diff --git a/.clang-tidy b/.clang-tidy
index 2cfcc2ac22b6ad25dfb70b6ae8aea71d183642da..3f407e0160e69deb2cf5a4bd18afb6b8951f5a99 100644
--- a/.clang-tidy
+++ b/.clang-tidy
@@ -1,4 +1,4 @@
-Checks: '-*,clang-diagnostic-*,llvm-*,misc-*,-misc-unused-parameters,readability-identifier-naming'
+Checks: '-*,clang-diagnostic-*,llvm-*,misc-*,-misc-unused-parameters,-misc-non-private-member-variables-in-classes,readability-identifier-naming'
 CheckOptions:
   - key:             readability-identifier-naming.ClassCase
     value:           CamelCase
diff --git a/.gitignore b/.gitignore
index fd308878407aa8e0c6745b1a837a94e3fff0b3e0..49273af86bd2bc7febd9ba765b4b6b935d065b34 100644
--- a/.gitignore
+++ b/.gitignore
@@ -43,6 +43,8 @@ autoconf/autom4te.cache
 /compile_commands.json
 # Visual Studio built-in CMake configuration
 /CMakeSettings.json
+# CLion project configuration
+/.idea
 
 #==============================================================================#
 # Directories to ignore (do not add trailing '/'s, they skip symlinks).
@@ -70,6 +72,8 @@ docs/_build
 # VS2017 and VSCode config files.
 .vscode
 .vs
+# clangd background index
+.clangd-index
 
 #==============================================================================#
 # Files created in tree by the Go bindings.
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 15f95d2e580c94ec4592b6c9c6bfa5cf16de136c..01252e7d0e17d5c3fde2162c4dda3cff39d7ed38 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -12,7 +12,7 @@ if(POLICY CMP0075)
 endif()
 
 if(NOT DEFINED LLVM_VERSION_MAJOR)
-  set(LLVM_VERSION_MAJOR 8)
+  set(LLVM_VERSION_MAJOR 9)
 endif()
 if(NOT DEFINED LLVM_VERSION_MINOR)
   set(LLVM_VERSION_MINOR 0)
@@ -104,27 +104,66 @@ endif()
 # LLVM_EXTERNAL_${project}_SOURCE_DIR using LLVM_ALL_PROJECTS
 # This allows an easy way of setting up a build directory for llvm and another
 # one for llvm+clang+... using the same sources.
-set(LLVM_ALL_PROJECTS "clang;libcxx;libcxxabi;lldb;compiler-rt;lld;polly;debuginfo-tests")
+set(LLVM_ALL_PROJECTS "clang;compiler-rt;debuginfo-tests;libclc;libcxx;libcxxabi;libunwind;lld;lldb;llgo;llvm;openmp;parallel-libs;polly;pstl")
 set(LLVM_ENABLE_PROJECTS "" CACHE STRING
 	"Semicolon-separated list of projects to build (${LLVM_ALL_PROJECTS}), or \"all\".")
 if( LLVM_ENABLE_PROJECTS STREQUAL "all" )
   set( LLVM_ENABLE_PROJECTS ${LLVM_ALL_PROJECTS})
 endif()
-foreach(proj ${LLVM_ENABLE_PROJECTS})
-  set(PROJ_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../${proj}")
-  if(NOT EXISTS "${PROJ_DIR}" OR NOT IS_DIRECTORY "${PROJ_DIR}")
-    message(FATAL_ERROR "LLVM_ENABLE_PROJECTS requests ${proj} but directory not found: ${PROJ_DIR}")
-  endif()
-  string(TOUPPER "${proj}" upper_proj)
-  STRING(REGEX REPLACE "-" "_" upper_proj ${upper_proj})
-  set(LLVM_EXTERNAL_${upper_proj}_SOURCE_DIR   "${CMAKE_CURRENT_SOURCE_DIR}/../${proj}")
-  # There is a widely spread opinion that clang-tools-extra should be merged
-  # into clang. The following simulates it by always enabling clang-tools-extra
-  # when enabling clang.
-  if (proj STREQUAL "clang")
-    set(LLVM_EXTERNAL_CLANG_TOOLS_EXTRA_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../clang-tools-extra")
-  endif()
-endforeach()
+
+# LLVM_ENABLE_PROJECTS_USED is `ON` if the user has ever used the
+# `LLVM_ENABLE_PROJECTS` CMake cache variable.  This exists for
+# several reasons:
+#
+# * As an indicator that the `LLVM_ENABLE_PROJECTS` list is now the single
+# source of truth for which projects to build. This means we will ignore user
+# supplied `LLVM_TOOL_<project>_BUILD` CMake cache variables and overwrite
+# them.
+#
+# * The case where the user previously had `LLVM_ENABLE_PROJECTS` set to a
+# non-empty list but now the user wishes to disable building all other projects
+# by setting `LLVM_ENABLE_PROJECTS` to an empty string. In that case we still
+# need to set the `LLVM_TOOL_${upper_proj}_BUILD` variables so that we disable
+# building all the projects that were previously enabled.
+set(LLVM_ENABLE_PROJECTS_USED OFF CACHE BOOL "")
+mark_as_advanced(LLVM_ENABLE_PROJECTS_USED)
+
+if (LLVM_ENABLE_PROJECTS_USED OR NOT LLVM_ENABLE_PROJECTS STREQUAL "")
+  set(LLVM_ENABLE_PROJECTS_USED ON CACHE BOOL "" FORCE)
+  foreach(proj ${LLVM_ALL_PROJECTS})
+    string(TOUPPER "${proj}" upper_proj)
+    string(REGEX REPLACE "-" "_" upper_proj ${upper_proj})
+    if ("${proj}" IN_LIST LLVM_ENABLE_PROJECTS)
+      message(STATUS "${proj} project is enabled")
+      set(SHOULD_ENABLE_PROJECT TRUE)
+      set(PROJ_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../${proj}")
+      if(NOT EXISTS "${PROJ_DIR}" OR NOT IS_DIRECTORY "${PROJ_DIR}")
+        message(FATAL_ERROR "LLVM_ENABLE_PROJECTS requests ${proj} but directory not found: ${PROJ_DIR}")
+      endif()
+      set(LLVM_EXTERNAL_${upper_proj}_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../${proj}")
+      # There is a widely spread opinion that clang-tools-extra should be merged
+      # into clang. The following simulates it by always enabling clang-tools-extra
+      # when enabling clang.
+      if (proj STREQUAL "clang")
+        set(LLVM_EXTERNAL_CLANG_TOOLS_EXTRA_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../clang-tools-extra")
+      endif()
+    else()
+      message(STATUS "${proj} project is disabled")
+      set(SHOULD_ENABLE_PROJECT FALSE)
+    endif()
+    # Force `LLVM_TOOL_${upper_proj}_BUILD` variables to have values that
+    # corresponds with `LLVM_ENABLE_PROJECTS`. This prevents the user setting
+    # `LLVM_TOOL_${upper_proj}_BUILD` variables externally. At some point
+    # we should deprecate allowing users to set these variables by turning them
+    # into normal CMake variables rather than cache variables.
+    set(LLVM_TOOL_${upper_proj}_BUILD
+      ${SHOULD_ENABLE_PROJECT}
+      CACHE
+      BOOL "Whether to build ${upper_proj} as part of LLVM" FORCE
+    )
+  endforeach()
+endif()
+unset(SHOULD_ENABLE_PROJECT)
 
 # Build llvm with ccache if the package is present
 set(LLVM_CCACHE_BUILD OFF CACHE BOOL "Set to ON for a ccache enabled build")
@@ -200,7 +239,7 @@ endif()
 include(VersionFromVCS)
 
 option(LLVM_APPEND_VC_REV
-  "Embed the version control system revision id in LLVM" ON)
+  "Embed the version control system revision in LLVM" ON)
 
 set(PACKAGE_NAME LLVM)
 set(PACKAGE_STRING "${PACKAGE_NAME} ${PACKAGE_VERSION}")
@@ -370,10 +409,6 @@ option(LLVM_ENABLE_LLD "Use lld as C and C++ linker." OFF)
 option(LLVM_ENABLE_PEDANTIC "Compile with pedantic enabled." ON)
 option(LLVM_ENABLE_WERROR "Fail and stop if a warning is triggered." OFF)
 
-if (MSVC)
-   option(LLVM_ENABLE_INCREMENTAL_LINK "Link incrementally. Enabling it might produce slower executables." OFF)
-endif()
-
 option(LLVM_ENABLE_DUMP "Enable dump functions even when assertions are disabled" OFF)
 
 if( NOT uppercase_CMAKE_BUILD_TYPE STREQUAL "DEBUG" )
@@ -387,9 +422,12 @@ option(LLVM_ENABLE_EXPENSIVE_CHECKS "Enable expensive checks" OFF)
 set(LLVM_ABI_BREAKING_CHECKS "WITH_ASSERTS" CACHE STRING
   "Enable abi-breaking checks.  Can be WITH_ASSERTS, FORCE_ON or FORCE_OFF.")
 
-option(LLVM_FORCE_USE_OLD_HOST_TOOLCHAIN
+option(LLVM_FORCE_USE_OLD_TOOLCHAIN
        "Set to ON to force using an old, unsupported host toolchain." OFF)
 
+option(LLVM_TEMPORARILY_ALLOW_OLD_TOOLCHAIN
+       "Set to ON to only warn when using a toolchain which is about to be deprecated, instead of emitting an error." OFF)
+
 option(LLVM_USE_INTEL_JITEVENTS
   "Use Intel JIT API to inform Intel(R) VTune(TM) Amplifier XE 2011 about JIT code"
   OFF)
@@ -756,13 +794,12 @@ set(LLVM_SRPM_USER_BINARY_SPECFILE ${CMAKE_CURRENT_SOURCE_DIR}/llvm.spec.in
 set(LLVM_SRPM_BINARY_SPECFILE ${CMAKE_CURRENT_BINARY_DIR}/llvm.spec)
 set(LLVM_SRPM_DIR "${CMAKE_CURRENT_BINARY_DIR}/srpm")
 
-# SVN_REVISION and GIT_COMMIT get set by the call to add_version_info_from_vcs.
-# DUMMY_VAR contains a version string which we don't care about.
-add_version_info_from_vcs(DUMMY_VAR)
-if ( SVN_REVISION )
-  set(LLVM_RPM_SPEC_REVISION "r${SVN_REVISION}")
-elseif ( GIT_COMMIT )
-  set (LLVM_RPM_SPEC_REVISION "g${GIT_COMMIT}")
+get_source_info(${CMAKE_CURRENT_SOURCE_DIR} revision repository)
+string(LENGTH "${revision}" revision_length)
+if(revision MATCHES "^[0-9]+$" AND revision_length LESS 40)
+  set(LLVM_RPM_SPEC_REVISION "r${revision}")
+else()
+  set(LLVM_RPM_SPEC_REVISION "${revision}")
 endif()
 
 configure_file(
diff --git a/CREDITS.TXT b/CREDITS.TXT
index 0a8154c8842f0830901f1a54b78fb4604bc0f56a..166f8f6d4f5138f99a435f8524d42c58b4bc566b 100644
--- a/CREDITS.TXT
+++ b/CREDITS.TXT
@@ -132,7 +132,7 @@ D: Basic-block autovectorization, PowerPC backend improvements
 
 N: Eric Fiselier
 E: eric@efcs.ca
-D: LIT patches and documentation.
+D: LIT patches and documentation
 
 N: Ryan Flynn
 E: pizza@parseerror.com
diff --git a/LICENSE.TXT b/LICENSE.TXT
index 461398bab7a7c9c353be548425b3065ae302eb23..fa6ac540007032cbd0ec772a1c72e6cb5527a4fe 100644
--- a/LICENSE.TXT
+++ b/LICENSE.TXT
@@ -1,10 +1,245 @@
 ==============================================================================
-LLVM Release License
+The LLVM Project is under the Apache License v2.0 with LLVM Exceptions:
+==============================================================================
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+    TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+    1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+    2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+    3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+    4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+    5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+    6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+    7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+    8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+    9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+    END OF TERMS AND CONDITIONS
+
+    APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+    Copyright [yyyy] [name of copyright owner]
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+
+
+---- LLVM Exceptions to the Apache 2.0 License ----
+
+As an exception, if, as a result of your compiling your source code, portions
+of this Software are embedded into an Object form of such source code, you
+may redistribute such embedded portions in such Object form without complying
+with the conditions of Sections 4(a), 4(b) and 4(d) of the License.
+
+In addition, if you combine or link compiled forms of this Software with
+software that is licensed under the GPLv2 ("Combined Software") and if a
+court of competent jurisdiction determines that the patent provision (Section
+3), the indemnity provision (Section 9) or other Section of the License
+conflicts with the conditions of the GPLv2, you may retroactively and
+prospectively choose to deem waived or otherwise exclude such Section(s) of
+the License, but only in their entirety and only with respect to the Combined
+Software.
+
+==============================================================================
+Software from third parties included in the LLVM Project:
+==============================================================================
+The LLVM Project contains third party software which is under different license
+terms. All such code will be identified clearly using at least one of two
+mechanisms:
+1) It will be in a separate directory tree with its own `LICENSE.txt` or
+   `LICENSE` file at the top containing the specific license and restrictions
+   which apply to that software, or
+2) It will contain specific license and restriction terms at the top of every
+   file.
+
+==============================================================================
+Legacy LLVM License (https://llvm.org/docs/DeveloperPolicy.html#legacy):
 ==============================================================================
 University of Illinois/NCSA
 Open Source License
 
-Copyright (c) 2003-2018 University of Illinois at Urbana-Champaign.
+Copyright (c) 2003-2019 University of Illinois at Urbana-Champaign.
 All rights reserved.
 
 Developed by:
@@ -42,27 +277,3 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH THE
 SOFTWARE.
 
-==============================================================================
-Copyrights and Licenses for Third Party Software Distributed with LLVM:
-==============================================================================
-The LLVM software contains code written by third parties.  Such software will
-have its own individual LICENSE.TXT file in the directory in which it appears.
-This file will describe the copyrights, license, and restrictions which apply
-to that code.
-
-The disclaimer of warranty in the University of Illinois Open Source License
-applies to all code in the LLVM Distribution, and nothing in any of the
-other licenses gives permission to use the names of the LLVM Team or the
-University of Illinois to endorse or promote products derived from this
-Software.
-
-The following pieces of software have additional or alternate copyrights,
-licenses, and/or restrictions:
-
-Program             Directory
--------             ---------
-Google Test         llvm/utils/unittest/googletest
-OpenBSD regex       llvm/lib/Support/{reg*, COPYRIGHT.regex}
-pyyaml tests        llvm/test/YAMLParser/{*.data, LICENSE.TXT}
-ARM contributions   llvm/lib/Target/ARM/LICENSE.TXT
-md5 contributions   llvm/lib/Support/MD5.cpp llvm/include/llvm/Support/MD5.h
diff --git a/LLVMBuild.txt b/LLVMBuild.txt
index 9cee3030a93559b9b0e85004fda03cb51db3e242..4cc65f21e854d8804b98f2fe06c2a78f1af1445d 100644
--- a/LLVMBuild.txt
+++ b/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./LLVMBuild.txt ------------------------------------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/bindings/LLVMBuild.txt b/bindings/LLVMBuild.txt
index 241ac0964f36f8c0b60a4218fe9cfcf239cbc380..9e54226ef53fcf2b466fe832318a77ac77519ef3 100644
--- a/bindings/LLVMBuild.txt
+++ b/bindings/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./bindings/LLVMBuild.txt ---------------------------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/bindings/go/llvm/IRBindings.cpp b/bindings/go/llvm/IRBindings.cpp
index b8e3063eb9ac0d49d118cf67ce58c507975a4c1b..102b79b8c9c669053227264afd16f5f542f16ac3 100644
--- a/bindings/go/llvm/IRBindings.cpp
+++ b/bindings/go/llvm/IRBindings.cpp
@@ -1,9 +1,8 @@
 //===- IRBindings.cpp - Additional bindings for ir ------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/bindings/go/llvm/IRBindings.h b/bindings/go/llvm/IRBindings.h
index 66b328c4325774d62b630ecfb55f7b843bca459c..04cecdafccd80ec2f9e9df6666dbcd3583fa348c 100644
--- a/bindings/go/llvm/IRBindings.h
+++ b/bindings/go/llvm/IRBindings.h
@@ -1,9 +1,8 @@
 //===- IRBindings.h - Additional bindings for IR ----------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/bindings/go/llvm/InstrumentationBindings.cpp b/bindings/go/llvm/InstrumentationBindings.cpp
index 8b7bafa77ad4cef661430cfa3292564c9cdf883c..316f77743443970911da12ccf5ac4927dbc9dbf2 100644
--- a/bindings/go/llvm/InstrumentationBindings.cpp
+++ b/bindings/go/llvm/InstrumentationBindings.cpp
@@ -1,9 +1,8 @@
 //===- InstrumentationBindings.cpp - instrumentation bindings -------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -16,6 +15,8 @@
 #include "llvm/IR/LegacyPassManager.h"
 #include "llvm/IR/Module.h"
 #include "llvm/Transforms/Instrumentation.h"
+#include "llvm/Transforms/Instrumentation/MemorySanitizer.h"
+#include "llvm/Transforms/Instrumentation/ThreadSanitizer.h"
 
 using namespace llvm;
 
@@ -28,11 +29,11 @@ void LLVMAddAddressSanitizerModulePass(LLVMPassManagerRef PM) {
 }
 
 void LLVMAddThreadSanitizerPass(LLVMPassManagerRef PM) {
-  unwrap(PM)->add(createThreadSanitizerPass());
+  unwrap(PM)->add(createThreadSanitizerLegacyPassPass());
 }
 
-void LLVMAddMemorySanitizerPass(LLVMPassManagerRef PM) {
-  unwrap(PM)->add(createMemorySanitizerPass());
+void LLVMAddMemorySanitizerLegacyPassPass(LLVMPassManagerRef PM) {
+  unwrap(PM)->add(createMemorySanitizerLegacyPassPass());
 }
 
 void LLVMAddDataFlowSanitizerPass(LLVMPassManagerRef PM,
diff --git a/bindings/go/llvm/InstrumentationBindings.h b/bindings/go/llvm/InstrumentationBindings.h
index 97af2d58c271433d70d09f1edd957f33c6fdab7e..143086c06274365450b935d7947b075d11a8b3ae 100644
--- a/bindings/go/llvm/InstrumentationBindings.h
+++ b/bindings/go/llvm/InstrumentationBindings.h
@@ -1,9 +1,8 @@
 //===- InstrumentationBindings.h - instrumentation bindings -----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -27,7 +26,7 @@ extern "C" {
 void LLVMAddAddressSanitizerFunctionPass(LLVMPassManagerRef PM);
 void LLVMAddAddressSanitizerModulePass(LLVMPassManagerRef PM);
 void LLVMAddThreadSanitizerPass(LLVMPassManagerRef PM);
-void LLVMAddMemorySanitizerPass(LLVMPassManagerRef PM);
+void LLVMAddMemorySanitizerLegacyPassPass(LLVMPassManagerRef PM);
 void LLVMAddDataFlowSanitizerPass(LLVMPassManagerRef PM, int ABIListFilesNum,
                                   const char **ABIListFiles);
 
diff --git a/bindings/go/llvm/SupportBindings.cpp b/bindings/go/llvm/SupportBindings.cpp
index 5e251b7da09232480b6fcf4fe61827ce6267f6f3..aac69a7c788d1ae425731c6fa1b214938f82e994 100644
--- a/bindings/go/llvm/SupportBindings.cpp
+++ b/bindings/go/llvm/SupportBindings.cpp
@@ -1,9 +1,8 @@
 //===- SupportBindings.cpp - Additional bindings for support --------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/bindings/go/llvm/SupportBindings.h b/bindings/go/llvm/SupportBindings.h
index efcd667514f48925b44bd3c7d545bdc883e1670f..db83e91ee5ab3c259cfb3320381b35edf05b6ba4 100644
--- a/bindings/go/llvm/SupportBindings.h
+++ b/bindings/go/llvm/SupportBindings.h
@@ -1,9 +1,8 @@
 //===- SupportBindings.h - Additional bindings for Support ------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/bindings/go/llvm/analysis.go b/bindings/go/llvm/analysis.go
index 3ae4b71def7c0f8a58e44362e17a4975fe836859..3a1c9d34b4dc373f12e15584a2de96ed85691213 100644
--- a/bindings/go/llvm/analysis.go
+++ b/bindings/go/llvm/analysis.go
@@ -1,9 +1,8 @@
 //===- analysis.go - Bindings for analysis --------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/bindings/go/llvm/bitreader.go b/bindings/go/llvm/bitreader.go
index c3bf07a19044bed6b12f51830726ed0726a73cb8..1954916e717da384a00bda8e33972768f86d806e 100644
--- a/bindings/go/llvm/bitreader.go
+++ b/bindings/go/llvm/bitreader.go
@@ -1,9 +1,8 @@
 //===- bitreader.go - Bindings for bitreader ------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/bindings/go/llvm/bitwriter.go b/bindings/go/llvm/bitwriter.go
index e03699c13b208c56781e68e9afddceea18fe6525..83780fc69713e47c887432e6a43cd8f2eb94827b 100644
--- a/bindings/go/llvm/bitwriter.go
+++ b/bindings/go/llvm/bitwriter.go
@@ -1,9 +1,8 @@
 //===- bitwriter.go - Bindings for bitwriter ------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/bindings/go/llvm/dibuilder.go b/bindings/go/llvm/dibuilder.go
index 1a98c07e7c90554d0cc713b17ba198f68d79c66e..69d5b911a8cc706b8f6438cac10a130f6b3ca9e6 100644
--- a/bindings/go/llvm/dibuilder.go
+++ b/bindings/go/llvm/dibuilder.go
@@ -1,9 +1,8 @@
 //===- dibuilder.go - Bindings for DIBuilder ------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/bindings/go/llvm/executionengine.go b/bindings/go/llvm/executionengine.go
index 91f8366ca75d7a9100cff1773d45e140d69530d0..5fa82047c1798c1697b62a56ff7781d55dbd1bc3 100644
--- a/bindings/go/llvm/executionengine.go
+++ b/bindings/go/llvm/executionengine.go
@@ -1,9 +1,8 @@
 //===- executionengine.go - Bindings for executionengine ------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/bindings/go/llvm/executionengine_test.go b/bindings/go/llvm/executionengine_test.go
index 3ca36d04569432d7d7334cff13f7a35db7b288de..4462f8fb20468bc61f255a71e175899f169bdb7e 100644
--- a/bindings/go/llvm/executionengine_test.go
+++ b/bindings/go/llvm/executionengine_test.go
@@ -1,9 +1,8 @@
 //===- executionengine_test.go - Tests for executionengine ----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/bindings/go/llvm/ir.go b/bindings/go/llvm/ir.go
index 1872a2ffe51092a31af4b685827d8ad0e6ffcb8d..c07a1b4ee5c112d128b64bc1af9b406a2d1c257c 100644
--- a/bindings/go/llvm/ir.go
+++ b/bindings/go/llvm/ir.go
@@ -1,9 +1,8 @@
 //===- ir.go - Bindings for ir --------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/bindings/go/llvm/ir_test.go b/bindings/go/llvm/ir_test.go
index 10f4968ba89f2c803eb9fbe04eb5ee33f2bcb530..5dd0598c012689286194f61b090feb807eccd721 100644
--- a/bindings/go/llvm/ir_test.go
+++ b/bindings/go/llvm/ir_test.go
@@ -1,9 +1,8 @@
 //===- ir_test.go - Tests for ir ------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -31,7 +30,7 @@ func testAttribute(t *testing.T, name string) {
 	fn.AddFunctionAttr(attr)
 	newattr := fn.GetEnumFunctionAttribute(kind)
 	if attr != newattr {
-		t.Errorf("got attribute mask %d, want %d", newattr, attr)
+		t.Errorf("got attribute %p, want %p", newattr.C, attr.C)
 	}
 
 	text := mod.String()
@@ -42,7 +41,7 @@ func testAttribute(t *testing.T, name string) {
 	fn.RemoveEnumFunctionAttribute(kind)
 	newattr = fn.GetEnumFunctionAttribute(kind)
 	if !newattr.IsNil() {
-		t.Errorf("got attribute mask %d, want 0", newattr)
+		t.Errorf("got attribute %p, want 0", newattr.C)
 	}
 }
 
diff --git a/bindings/go/llvm/linker.go b/bindings/go/llvm/linker.go
index ca16f7637b2b23d2db49506f1c12a61a5e8c210e..8e2e953fd46515960a7706f4f126cbc2bf95781d 100644
--- a/bindings/go/llvm/linker.go
+++ b/bindings/go/llvm/linker.go
@@ -1,9 +1,8 @@
 //===- linker.go - Bindings for linker ------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/bindings/go/llvm/llvm_dep.go b/bindings/go/llvm/llvm_dep.go
index 39b46759483edfaedbb24ab684985f75cb327ee6..9484e6ddf879e50b9ca61cef33be4ed24c27f018 100644
--- a/bindings/go/llvm/llvm_dep.go
+++ b/bindings/go/llvm/llvm_dep.go
@@ -1,9 +1,8 @@
 //===- llvm_dep.go - creates LLVM dependency ------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/bindings/go/llvm/string.go b/bindings/go/llvm/string.go
index bfe869dbc15b92c177f5ba1d3968194d48b30ded..bd48d4bc170a3329171af589d9361ec17ef01e53 100644
--- a/bindings/go/llvm/string.go
+++ b/bindings/go/llvm/string.go
@@ -1,9 +1,8 @@
 //===- string.go - Stringer implementation for Type -----------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/bindings/go/llvm/string_test.go b/bindings/go/llvm/string_test.go
index 3008f3ef937c99a608e32b82fccbc54e1ee21d44..56cbc9f3e7fd0f2711764e884b54216c174cee59 100644
--- a/bindings/go/llvm/string_test.go
+++ b/bindings/go/llvm/string_test.go
@@ -1,9 +1,8 @@
 //===- string_test.go - test Stringer implementation for Type -------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/bindings/go/llvm/support.go b/bindings/go/llvm/support.go
index 6f200861cca0babb368f7f44791d1d0c11496340..8a663fee502a5893efbcc1605afd84785a61ed86 100644
--- a/bindings/go/llvm/support.go
+++ b/bindings/go/llvm/support.go
@@ -1,9 +1,8 @@
 //===- support.go - Bindings for support ----------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/bindings/go/llvm/target.go b/bindings/go/llvm/target.go
index 5333397be32966b676e95f5e8d21e224550c1f32..cdd0fa30c3f5ffdfb472b0e8600734772070a2c6 100644
--- a/bindings/go/llvm/target.go
+++ b/bindings/go/llvm/target.go
@@ -1,9 +1,8 @@
 //===- target.go - Bindings for target ------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/bindings/go/llvm/transforms_coroutines.go b/bindings/go/llvm/transforms_coroutines.go
index 9486fcb249cba4231f4e607b247b0d47f38e9452..d0af76db473ba945ba8d57904f27161b2482eb0a 100644
--- a/bindings/go/llvm/transforms_coroutines.go
+++ b/bindings/go/llvm/transforms_coroutines.go
@@ -1,9 +1,8 @@
 //===- transforms_coroutines.go - Bindings for coroutines -----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/bindings/go/llvm/transforms_instrumentation.go b/bindings/go/llvm/transforms_instrumentation.go
index 73e2732cbd9553e88ec2c1453ad0681d438372b7..68e0ed6967979f4d77217341481a6ab7fef4b33b 100644
--- a/bindings/go/llvm/transforms_instrumentation.go
+++ b/bindings/go/llvm/transforms_instrumentation.go
@@ -1,9 +1,8 @@
 //===- transforms_instrumentation.go - Bindings for instrumentation -------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -32,8 +31,8 @@ func (pm PassManager) AddThreadSanitizerPass() {
 	C.LLVMAddThreadSanitizerPass(pm.C)
 }
 
-func (pm PassManager) AddMemorySanitizerPass() {
-	C.LLVMAddMemorySanitizerPass(pm.C)
+func (pm PassManager) AddMemorySanitizerLegacyPassPass() {
+	C.LLVMAddMemorySanitizerLegacyPassPass(pm.C)
 }
 
 func (pm PassManager) AddDataFlowSanitizerPass(abilist []string) {
diff --git a/bindings/go/llvm/transforms_ipo.go b/bindings/go/llvm/transforms_ipo.go
index 12d972b38f982ab45fc977ed641f5390aa684405..1dcb2af8bf243b00469cfef13fa2ad87fa5dbab8 100644
--- a/bindings/go/llvm/transforms_ipo.go
+++ b/bindings/go/llvm/transforms_ipo.go
@@ -1,9 +1,8 @@
 //===- transforms_ipo.go - Bindings for ipo -------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/bindings/go/llvm/transforms_pmbuilder.go b/bindings/go/llvm/transforms_pmbuilder.go
index b164e58812b1553cf4995f7085b32ec0d44977b4..f6b92ed099bc2b7234b4f0c0a6cd48c75e99a97f 100644
--- a/bindings/go/llvm/transforms_pmbuilder.go
+++ b/bindings/go/llvm/transforms_pmbuilder.go
@@ -1,9 +1,8 @@
 //===- transforms_pmbuilder.go - Bindings for PassManagerBuilder ----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/bindings/go/llvm/transforms_scalar.go b/bindings/go/llvm/transforms_scalar.go
index cb46f6881859b2bd6c53a0a5f5dbdce205600ef9..36fc13e00696bd6d1c0dcf558be1a84843ec4c78 100644
--- a/bindings/go/llvm/transforms_scalar.go
+++ b/bindings/go/llvm/transforms_scalar.go
@@ -1,9 +1,8 @@
 //===- transforms_scalar.go - Bindings for scalaropts ---------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/bindings/go/llvm/version.go b/bindings/go/llvm/version.go
index d0623be97a9baccd1480109abf0c1e57eff118e6..1183fe431b97ee50f76d5a73ad07eb749be756a1 100644
--- a/bindings/go/llvm/version.go
+++ b/bindings/go/llvm/version.go
@@ -1,9 +1,8 @@
 //===- version.go - LLVM version info -------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/bindings/ocaml/all_backends/all_backends_ocaml.c b/bindings/ocaml/all_backends/all_backends_ocaml.c
index 8fe7d9b5d593d7dce91e8f38e77f0cbdc0a7350d..ae4b496cb392053e756f57dfba0bfc6588197237 100644
--- a/bindings/ocaml/all_backends/all_backends_ocaml.c
+++ b/bindings/ocaml/all_backends/all_backends_ocaml.c
@@ -1,9 +1,9 @@
 /*===-- all_backends_ocaml.c - LLVM OCaml Glue ------------------*- C++ -*-===*\
 |*                                                                            *|
-|*                     The LLVM Compiler Infrastructure                       *|
-|*                                                                            *|
-|* This file is distributed under the University of Illinois Open Source      *|
-|* License. See LICENSE.TXT for details.                                      *|
+|* Part of the LLVM Project, under the Apache License v2.0 with LLVM          *|
+|* Exceptions.                                                                *|
+|* See https://llvm.org/LICENSE.txt for license information.                  *|
+|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception                    *|
 |*                                                                            *|
 |*===----------------------------------------------------------------------===*|
 |*                                                                            *|
diff --git a/bindings/ocaml/all_backends/llvm_all_backends.ml b/bindings/ocaml/all_backends/llvm_all_backends.ml
index f4f4725707bebadcaf6396fe9adb4cb557a363aa..b4df7cde23d9a0768f6c596cf286fb651b93f464 100644
--- a/bindings/ocaml/all_backends/llvm_all_backends.ml
+++ b/bindings/ocaml/all_backends/llvm_all_backends.ml
@@ -1,9 +1,8 @@
 (*===-- llvm_all_backends.ml - LLVM OCaml Interface -----------*- OCaml -*-===*
  *
- *                     The LLVM Compiler Infrastructure
- *
- * This file is distributed under the University of Illinois Open Source
- * License. See LICENSE.TXT for details.
+ * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+ * See https://llvm.org/LICENSE.txt for license information.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
  *
  *===----------------------------------------------------------------------===*)
 
diff --git a/bindings/ocaml/all_backends/llvm_all_backends.mli b/bindings/ocaml/all_backends/llvm_all_backends.mli
index 1808544706c636dd506f7af5d1de986df5913355..62a515b932a98cabcf8d3f414661affecab92f0f 100644
--- a/bindings/ocaml/all_backends/llvm_all_backends.mli
+++ b/bindings/ocaml/all_backends/llvm_all_backends.mli
@@ -1,9 +1,8 @@
 (*===-- llvm_all_backends.mli - LLVM OCaml Interface ----------*- OCaml -*-===*
  *
- *                     The LLVM Compiler Infrastructure
- *
- * This file is distributed under the University of Illinois Open Source
- * License. See LICENSE.TXT for details.
+ * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+ * See https://llvm.org/LICENSE.txt for license information.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
  *
  *===----------------------------------------------------------------------===*)
 
diff --git a/bindings/ocaml/analysis/analysis_ocaml.c b/bindings/ocaml/analysis/analysis_ocaml.c
index 8b8263d9f337a7838bcea7faf533da99114f97d6..af98e651e3b4a93139936e9cd7ab582d5b1523ee 100644
--- a/bindings/ocaml/analysis/analysis_ocaml.c
+++ b/bindings/ocaml/analysis/analysis_ocaml.c
@@ -1,9 +1,9 @@
 /*===-- analysis_ocaml.c - LLVM OCaml Glue ----------------------*- C++ -*-===*\
 |*                                                                            *|
-|*                     The LLVM Compiler Infrastructure                       *|
-|*                                                                            *|
-|* This file is distributed under the University of Illinois Open Source      *|
-|* License. See LICENSE.TXT for details.                                      *|
+|* Part of the LLVM Project, under the Apache License v2.0 with LLVM          *|
+|* Exceptions.                                                                *|
+|* See https://llvm.org/LICENSE.txt for license information.                  *|
+|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception                    *|
 |*                                                                            *|
 |*===----------------------------------------------------------------------===*|
 |*                                                                            *|
diff --git a/bindings/ocaml/analysis/llvm_analysis.ml b/bindings/ocaml/analysis/llvm_analysis.ml
index 8c11a63c091f80665377385de01a00a856a5e213..a8e5f4ef06ecf576c1696a5005991b0a27c9596c 100644
--- a/bindings/ocaml/analysis/llvm_analysis.ml
+++ b/bindings/ocaml/analysis/llvm_analysis.ml
@@ -1,9 +1,8 @@
 (*===-- llvm_analysis.ml - LLVM OCaml Interface ---------------*- OCaml -*-===*
  *
- *                     The LLVM Compiler Infrastructure
- *
- * This file is distributed under the University of Illinois Open Source
- * License. See LICENSE.TXT for details.
+ * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+ * See https://llvm.org/LICENSE.txt for license information.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
  *
  *===----------------------------------------------------------------------===*)
 
diff --git a/bindings/ocaml/analysis/llvm_analysis.mli b/bindings/ocaml/analysis/llvm_analysis.mli
index 03197cd41be63b0dcbd099e5b79261785f415fab..cf323b547d9a68a5d40bfe6327b02eee162816c4 100644
--- a/bindings/ocaml/analysis/llvm_analysis.mli
+++ b/bindings/ocaml/analysis/llvm_analysis.mli
@@ -1,9 +1,8 @@
 (*===-- llvm_analysis.mli - LLVM OCaml Interface --------------*- OCaml -*-===*
  *
- *                     The LLVM Compiler Infrastructure
- *
- * This file is distributed under the University of Illinois Open Source
- * License. See LICENSE.TXT for details.
+ * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+ * See https://llvm.org/LICENSE.txt for license information.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
  *
  *===----------------------------------------------------------------------===*)
 
diff --git a/bindings/ocaml/backends/backend_ocaml.c b/bindings/ocaml/backends/backend_ocaml.c
index 3e1a43897c77ca791fed37917ed400c79b9b81c2..16e68c541e3e0e2b6e5f671d750d2cb387179340 100644
--- a/bindings/ocaml/backends/backend_ocaml.c
+++ b/bindings/ocaml/backends/backend_ocaml.c
@@ -1,9 +1,9 @@
 /*===-- backend_ocaml.c - LLVM OCaml Glue -----------------------*- C++ -*-===*\
 |*                                                                            *|
-|*                     The LLVM Compiler Infrastructure                       *|
-|*                                                                            *|
-|* This file is distributed under the University of Illinois Open Source      *|
-|* License. See LICENSE.TXT for details.                                      *|
+|* Part of the LLVM Project, under the Apache License v2.0 with LLVM          *|
+|* Exceptions.                                                                *|
+|* See https://llvm.org/LICENSE.txt for license information.                  *|
+|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception                    *|
 |*                                                                            *|
 |*===----------------------------------------------------------------------===*|
 |*                                                                            *|
diff --git a/bindings/ocaml/backends/llvm_backend.ml.in b/bindings/ocaml/backends/llvm_backend.ml.in
index bd1e5860aa2d8967f0c27604de072dc37d1c7eb1..b80cc75c7d0eeb461c8cd08dd83a06b7a5121bee 100644
--- a/bindings/ocaml/backends/llvm_backend.ml.in
+++ b/bindings/ocaml/backends/llvm_backend.ml.in
@@ -1,9 +1,8 @@
 (*===-- llvm_backend.ml.in - LLVM OCaml Interface -------------*- OCaml -*-===*
  *
- *                     The LLVM Compiler Infrastructure
- *
- * This file is distributed under the University of Illinois Open Source
- * License. See LICENSE.TXT for details.
+ * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+ * See https://llvm.org/LICENSE.txt for license information.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
  *
  *===----------------------------------------------------------------------===*)
 
diff --git a/bindings/ocaml/backends/llvm_backend.mli.in b/bindings/ocaml/backends/llvm_backend.mli.in
index 9506789a7fddc83362d06dcbdba03fba5802b5d8..25b0f89823f26088f3f6c42a719e4d894163adce 100644
--- a/bindings/ocaml/backends/llvm_backend.mli.in
+++ b/bindings/ocaml/backends/llvm_backend.mli.in
@@ -1,9 +1,8 @@
 (*===-- llvm_backend.mli.in - LLVM OCaml Interface ------------*- OCaml -*-===*
  *
- *                     The LLVM Compiler Infrastructure
- *
- * This file is distributed under the University of Illinois Open Source
- * License. See LICENSE.TXT for details.
+ * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+ * See https://llvm.org/LICENSE.txt for license information.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
  *
  *===----------------------------------------------------------------------===*)
 
diff --git a/bindings/ocaml/bitreader/bitreader_ocaml.c b/bindings/ocaml/bitreader/bitreader_ocaml.c
index 6d957760cff730efcdb6de0a4117cc811152427c..1af554e24b5ac5dc5a3a9238819f1d63f9badffa 100644
--- a/bindings/ocaml/bitreader/bitreader_ocaml.c
+++ b/bindings/ocaml/bitreader/bitreader_ocaml.c
@@ -1,9 +1,9 @@
 /*===-- bitwriter_ocaml.c - LLVM OCaml Glue ---------------------*- C++ -*-===*\
 |*                                                                            *|
-|*                     The LLVM Compiler Infrastructure                       *|
-|*                                                                            *|
-|* This file is distributed under the University of Illinois Open Source      *|
-|* License. See LICENSE.TXT for details.                                      *|
+|* Part of the LLVM Project, under the Apache License v2.0 with LLVM          *|
+|* Exceptions.                                                                *|
+|* See https://llvm.org/LICENSE.txt for license information.                  *|
+|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception                    *|
 |*                                                                            *|
 |*===----------------------------------------------------------------------===*|
 |*                                                                            *|
diff --git a/bindings/ocaml/bitreader/llvm_bitreader.ml b/bindings/ocaml/bitreader/llvm_bitreader.ml
index b26efdd2c18f49331cfd47a501d34e2d0309b937..601089fa5da8555a3c77b2574f5fcaeaf8383402 100644
--- a/bindings/ocaml/bitreader/llvm_bitreader.ml
+++ b/bindings/ocaml/bitreader/llvm_bitreader.ml
@@ -1,9 +1,8 @@
 (*===-- llvm_bitreader.ml - LLVM OCaml Interface --------------*- OCaml -*-===*
  *
- *                     The LLVM Compiler Infrastructure
- *
- * This file is distributed under the University of Illinois Open Source
- * License. See LICENSE.TXT for details.
+ * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+ * See https://llvm.org/LICENSE.txt for license information.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
  *
  *===----------------------------------------------------------------------===*)
 
diff --git a/bindings/ocaml/bitreader/llvm_bitreader.mli b/bindings/ocaml/bitreader/llvm_bitreader.mli
index 435134337e076d7730dc0731391740f30585cda4..def8b84fe9e62e942d0f7bbb70c1f966c910a01c 100644
--- a/bindings/ocaml/bitreader/llvm_bitreader.mli
+++ b/bindings/ocaml/bitreader/llvm_bitreader.mli
@@ -1,9 +1,8 @@
 (*===-- llvm_bitreader.mli - LLVM OCaml Interface -------------*- OCaml -*-===*
  *
- *                     The LLVM Compiler Infrastructure
- *
- * This file is distributed under the University of Illinois Open Source
- * License. See LICENSE.TXT for details.
+ * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+ * See https://llvm.org/LICENSE.txt for license information.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
  *
  *===----------------------------------------------------------------------===*)
 
diff --git a/bindings/ocaml/bitwriter/bitwriter_ocaml.c b/bindings/ocaml/bitwriter/bitwriter_ocaml.c
index 04fd61917dc6205979d36b64c8c314bcb5effc94..6856d7547f0e0e3d8946bca9b7b426394d95c9df 100644
--- a/bindings/ocaml/bitwriter/bitwriter_ocaml.c
+++ b/bindings/ocaml/bitwriter/bitwriter_ocaml.c
@@ -1,9 +1,9 @@
 /*===-- bitwriter_ocaml.c - LLVM OCaml Glue ---------------------*- C++ -*-===*\
 |*                                                                            *|
-|*                     The LLVM Compiler Infrastructure                       *|
-|*                                                                            *|
-|* This file is distributed under the University of Illinois Open Source      *|
-|* License. See LICENSE.TXT for details.                                      *|
+|* Part of the LLVM Project, under the Apache License v2.0 with LLVM          *|
+|* Exceptions.                                                                *|
+|* See https://llvm.org/LICENSE.txt for license information.                  *|
+|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception                    *|
 |*                                                                            *|
 |*===----------------------------------------------------------------------===*|
 |*                                                                            *|
diff --git a/bindings/ocaml/bitwriter/llvm_bitwriter.ml b/bindings/ocaml/bitwriter/llvm_bitwriter.ml
index fca6efa4a75b2af6fdb6f412ec7f222c9209dee5..3750a02dcd0e9e1f6f964055fa1503dc381ddc5e 100644
--- a/bindings/ocaml/bitwriter/llvm_bitwriter.ml
+++ b/bindings/ocaml/bitwriter/llvm_bitwriter.ml
@@ -1,9 +1,8 @@
 (*===-- llvm_bitwriter.ml - LLVM OCaml Interface --------------*- OCaml -*-===*
  *
- *                     The LLVM Compiler Infrastructure
- *
- * This file is distributed under the University of Illinois Open Source
- * License. See LICENSE.TXT for details.
+ * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+ * See https://llvm.org/LICENSE.txt for license information.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
  *
  *===----------------------------------------------------------------------===
  *
diff --git a/bindings/ocaml/bitwriter/llvm_bitwriter.mli b/bindings/ocaml/bitwriter/llvm_bitwriter.mli
index 3d0f7808225727a0dccb20515f65c9d1357f2ab1..b8cc59c0f6713509ea2fff79d34a9888575f7766 100644
--- a/bindings/ocaml/bitwriter/llvm_bitwriter.mli
+++ b/bindings/ocaml/bitwriter/llvm_bitwriter.mli
@@ -1,9 +1,8 @@
 (*===-- llvm_bitwriter.mli - LLVM OCaml Interface -------------*- OCaml -*-===*
  *
- *                     The LLVM Compiler Infrastructure
- *
- * This file is distributed under the University of Illinois Open Source
- * License. See LICENSE.TXT for details.
+ * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+ * See https://llvm.org/LICENSE.txt for license information.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
  *
  *===----------------------------------------------------------------------===*)
 
diff --git a/bindings/ocaml/executionengine/executionengine_ocaml.c b/bindings/ocaml/executionengine/executionengine_ocaml.c
index a5e62aca8a4e102e9d0b185ef3a818da877c4e67..c83a8cadf4699e971c7688d093457ff3797c9afa 100644
--- a/bindings/ocaml/executionengine/executionengine_ocaml.c
+++ b/bindings/ocaml/executionengine/executionengine_ocaml.c
@@ -1,9 +1,9 @@
 /*===-- executionengine_ocaml.c - LLVM OCaml Glue ---------------*- C++ -*-===*\
 |*                                                                            *|
-|*                     The LLVM Compiler Infrastructure                       *|
-|*                                                                            *|
-|* This file is distributed under the University of Illinois Open Source      *|
-|* License. See LICENSE.TXT for details.                                      *|
+|* Part of the LLVM Project, under the Apache License v2.0 with LLVM          *|
+|* Exceptions.                                                                *|
+|* See https://llvm.org/LICENSE.txt for license information.                  *|
+|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception                    *|
 |*                                                                            *|
 |*===----------------------------------------------------------------------===*|
 |*                                                                            *|
diff --git a/bindings/ocaml/executionengine/llvm_executionengine.ml b/bindings/ocaml/executionengine/llvm_executionengine.ml
index 3f37e0c9d3d2c55c625b35662e57d1de559ebda0..5b202e2ead196a449c88a050795d0640520d9d77 100644
--- a/bindings/ocaml/executionengine/llvm_executionengine.ml
+++ b/bindings/ocaml/executionengine/llvm_executionengine.ml
@@ -1,9 +1,8 @@
 (*===-- llvm_executionengine.ml - LLVM OCaml Interface --------*- OCaml -*-===*
  *
- *                     The LLVM Compiler Infrastructure
- *
- * This file is distributed under the University of Illinois Open Source
- * License. See LICENSE.TXT for details.
+ * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+ * See https://llvm.org/LICENSE.txt for license information.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
  *
  *===----------------------------------------------------------------------===*)
 
diff --git a/bindings/ocaml/executionengine/llvm_executionengine.mli b/bindings/ocaml/executionengine/llvm_executionengine.mli
index bc076beaceab32c58f9ec96350684197cfbdaa27..3c5a1c06a4e034598a29da72ec101b951cca03d4 100644
--- a/bindings/ocaml/executionengine/llvm_executionengine.mli
+++ b/bindings/ocaml/executionengine/llvm_executionengine.mli
@@ -1,9 +1,8 @@
 (*===-- llvm_executionengine.mli - LLVM OCaml Interface -------*- OCaml -*-===*
  *
- *                     The LLVM Compiler Infrastructure
- *
- * This file is distributed under the University of Illinois Open Source
- * License. See LICENSE.TXT for details.
+ * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+ * See https://llvm.org/LICENSE.txt for license information.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
  *
  *===----------------------------------------------------------------------===*)
 
diff --git a/bindings/ocaml/irreader/irreader_ocaml.c b/bindings/ocaml/irreader/irreader_ocaml.c
index ce593db80a8ea89c43806e9d38698655ffe08183..63a339edb8de84b3191dcdc1d0186328b69b40da 100644
--- a/bindings/ocaml/irreader/irreader_ocaml.c
+++ b/bindings/ocaml/irreader/irreader_ocaml.c
@@ -1,9 +1,9 @@
 /*===-- irreader_ocaml.c - LLVM OCaml Glue ----------------------*- C++ -*-===*\
 |*                                                                            *|
-|*                     The LLVM Compiler Infrastructure                       *|
-|*                                                                            *|
-|* This file is distributed under the University of Illinois Open Source      *|
-|* License. See LICENSE.TXT for details.                                      *|
+|* Part of the LLVM Project, under the Apache License v2.0 with LLVM          *|
+|* Exceptions.                                                                *|
+|* See https://llvm.org/LICENSE.txt for license information.                  *|
+|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception                    *|
 |*                                                                            *|
 |*===----------------------------------------------------------------------===*|
 |*                                                                            *|
diff --git a/bindings/ocaml/irreader/llvm_irreader.ml b/bindings/ocaml/irreader/llvm_irreader.ml
index f757d629939c0f6f8a3607b37ed452fa878bd552..a8ece433181558d187daa5537d1ce6b55d325c43 100644
--- a/bindings/ocaml/irreader/llvm_irreader.ml
+++ b/bindings/ocaml/irreader/llvm_irreader.ml
@@ -1,9 +1,8 @@
 (*===-- llvm_irreader.ml - LLVM OCaml Interface ---------------*- OCaml -*-===*
  *
- *                     The LLVM Compiler Infrastructure
- *
- * This file is distributed under the University of Illinois Open Source
- * License. See LICENSE.TXT for details.
+ * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+ * See https://llvm.org/LICENSE.txt for license information.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
  *
  *===----------------------------------------------------------------------===*)
 
diff --git a/bindings/ocaml/irreader/llvm_irreader.mli b/bindings/ocaml/irreader/llvm_irreader.mli
index 2b1147373b5c6626a7146f12cc9f9d052814ba1c..bdb7d040845ab6381e4a55c9d35bcb820d7c2813 100644
--- a/bindings/ocaml/irreader/llvm_irreader.mli
+++ b/bindings/ocaml/irreader/llvm_irreader.mli
@@ -1,9 +1,8 @@
 (*===-- llvm_irreader.mli - LLVM OCaml Interface --------------*- OCaml -*-===*
  *
- *                     The LLVM Compiler Infrastructure
- *
- * This file is distributed under the University of Illinois Open Source
- * License. See LICENSE.TXT for details.
+ * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+ * See https://llvm.org/LICENSE.txt for license information.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
  *
  *===----------------------------------------------------------------------===*)
 
diff --git a/bindings/ocaml/linker/linker_ocaml.c b/bindings/ocaml/linker/linker_ocaml.c
index 08737bd5843c1c3fdeccc8265ed7408d673d9a64..75723d9ee2252eefbe344245b5ff9310eb76dad2 100644
--- a/bindings/ocaml/linker/linker_ocaml.c
+++ b/bindings/ocaml/linker/linker_ocaml.c
@@ -1,9 +1,9 @@
 /*===-- linker_ocaml.c - LLVM OCaml Glue ------------------------*- C++ -*-===*\
 |*                                                                            *|
-|*                     The LLVM Compiler Infrastructure                       *|
-|*                                                                            *|
-|* This file is distributed under the University of Illinois Open Source      *|
-|* License. See LICENSE.TXT for details.                                      *|
+|* Part of the LLVM Project, under the Apache License v2.0 with LLVM          *|
+|* Exceptions.                                                                *|
+|* See https://llvm.org/LICENSE.txt for license information.                  *|
+|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception                    *|
 |*                                                                            *|
 |*===----------------------------------------------------------------------===*|
 |*                                                                            *|
diff --git a/bindings/ocaml/linker/llvm_linker.ml b/bindings/ocaml/linker/llvm_linker.ml
index f2b64eeee91847f587117a6ccc5ba9a2222c85e3..e61e8fcb5fc536b319602d7a26153c8518abda41 100644
--- a/bindings/ocaml/linker/llvm_linker.ml
+++ b/bindings/ocaml/linker/llvm_linker.ml
@@ -1,9 +1,8 @@
 (*===-- llvm_linker.ml - LLVM OCaml Interface ------------------*- OCaml -*-===*
  *
- *                     The LLVM Compiler Infrastructure
- *
- * This file is distributed under the University of Illinois Open Source
- * License. See LICENSE.TXT for details.
+ * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+ * See https://llvm.org/LICENSE.txt for license information.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
  *
  *===----------------------------------------------------------------------===*)
 
diff --git a/bindings/ocaml/linker/llvm_linker.mli b/bindings/ocaml/linker/llvm_linker.mli
index 5f558ffb1162ac50db5249735afcd6fa8c7087b5..ed8f0b38c4eb61446f616d683008ab575729c95e 100644
--- a/bindings/ocaml/linker/llvm_linker.mli
+++ b/bindings/ocaml/linker/llvm_linker.mli
@@ -1,9 +1,8 @@
 (*===-- llvm_linker.mli - LLVM OCaml Interface -----------------*- OCaml -*-===*
  *
- *                     The LLVM Compiler Infrastructure
- *
- * This file is distributed under the University of Illinois Open Source
- * License. See LICENSE.TXT for details.
+ * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+ * See https://llvm.org/LICENSE.txt for license information.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
  *
  *===----------------------------------------------------------------------===*)
 
diff --git a/bindings/ocaml/llvm/llvm.ml b/bindings/ocaml/llvm/llvm.ml
index 5612fae69ffbe1119f1e2445c05db2f073410ca8..bbf19652560e7780d86e3b4a5aa796b42aa75e25 100644
--- a/bindings/ocaml/llvm/llvm.ml
+++ b/bindings/ocaml/llvm/llvm.ml
@@ -1,9 +1,8 @@
 (*===-- llvm/llvm.ml - LLVM OCaml Interface -------------------------------===*
  *
- *                     The LLVM Compiler Infrastructure
- *
- * This file is distributed under the University of Illinois Open Source
- * License. See LICENSE.TXT for details.
+ * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+ * See https://llvm.org/LICENSE.txt for license information.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
  *
  *===----------------------------------------------------------------------===*)
 
diff --git a/bindings/ocaml/llvm/llvm.mli b/bindings/ocaml/llvm/llvm.mli
index f12eb6efa61c797639d0d817ac17e4e04f79427b..c9eeee6b5136a3242bfa6d456698540a62e5f49b 100644
--- a/bindings/ocaml/llvm/llvm.mli
+++ b/bindings/ocaml/llvm/llvm.mli
@@ -1,9 +1,8 @@
 (*===-- llvm/llvm.mli - LLVM OCaml Interface ------------------------------===*
  *
- *                     The LLVM Compiler Infrastructure
- *
- * This file is distributed under the University of Illinois Open Source
- * License. See LICENSE.TXT for details.
+ * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+ * See https://llvm.org/LICENSE.txt for license information.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
  *
  *===----------------------------------------------------------------------===*)
 
diff --git a/bindings/ocaml/llvm/llvm_ocaml.c b/bindings/ocaml/llvm/llvm_ocaml.c
index cdf6c6a1206a620f7364761add4a7dbef441212b..1c524551ec6b0b37ccef80dfc0719d79d36d7a70 100644
--- a/bindings/ocaml/llvm/llvm_ocaml.c
+++ b/bindings/ocaml/llvm/llvm_ocaml.c
@@ -1,9 +1,9 @@
 /*===-- llvm_ocaml.c - LLVM OCaml Glue --------------------------*- C++ -*-===*\
 |*                                                                            *|
-|*                     The LLVM Compiler Infrastructure                       *|
-|*                                                                            *|
-|* This file is distributed under the University of Illinois Open Source      *|
-|* License. See LICENSE.TXT for details.                                      *|
+|* Part of the LLVM Project, under the Apache License v2.0 with LLVM          *|
+|* Exceptions.                                                                *|
+|* See https://llvm.org/LICENSE.txt for license information.                  *|
+|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception                    *|
 |*                                                                            *|
 |*===----------------------------------------------------------------------===*|
 |*                                                                            *|
diff --git a/bindings/ocaml/target/llvm_target.ml b/bindings/ocaml/target/llvm_target.ml
index 9e6b706b29ea8c2a8891302bc7f86c83a49d6892..0922ebe55345a0c5d6be9e206b4f0e970759e84f 100644
--- a/bindings/ocaml/target/llvm_target.ml
+++ b/bindings/ocaml/target/llvm_target.ml
@@ -1,9 +1,8 @@
 (*===-- llvm_target.ml - LLVM OCaml Interface ------------------*- OCaml -*-===*
  *
- *                     The LLVM Compiler Infrastructure
- *
- * This file is distributed under the University of Illinois Open Source
- * License. See LICENSE.TXT for details.
+ * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+ * See https://llvm.org/LICENSE.txt for license information.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
  *
  *===----------------------------------------------------------------------===*)
 
diff --git a/bindings/ocaml/target/llvm_target.mli b/bindings/ocaml/target/llvm_target.mli
index c59308c34e5434078c466562c18aa68bda239d5c..8d310793bea213c5c5bf9e69fa477ab633dfae6a 100644
--- a/bindings/ocaml/target/llvm_target.mli
+++ b/bindings/ocaml/target/llvm_target.mli
@@ -1,9 +1,8 @@
 (*===-- llvm_target.mli - LLVM OCaml Interface -----------------*- OCaml -*-===*
  *
- *                     The LLVM Compiler Infrastructure
- *
- * This file is distributed under the University of Illinois Open Source
- * License. See LICENSE.TXT for details.
+ * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+ * See https://llvm.org/LICENSE.txt for license information.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
  *
  *===----------------------------------------------------------------------===*)
 
diff --git a/bindings/ocaml/target/target_ocaml.c b/bindings/ocaml/target/target_ocaml.c
index 8872f42b5b68b6b67fe2a94ee1e07be407a6dd7c..cf48fbe45730543337b86f8925b4da71414ccf2a 100644
--- a/bindings/ocaml/target/target_ocaml.c
+++ b/bindings/ocaml/target/target_ocaml.c
@@ -1,9 +1,9 @@
 /*===-- target_ocaml.c - LLVM OCaml Glue ------------------------*- C++ -*-===*\
 |*                                                                            *|
-|*                     The LLVM Compiler Infrastructure                       *|
-|*                                                                            *|
-|* This file is distributed under the University of Illinois Open Source      *|
-|* License. See LICENSE.TXT for details.                                      *|
+|* Part of the LLVM Project, under the Apache License v2.0 with LLVM          *|
+|* Exceptions.                                                                *|
+|* See https://llvm.org/LICENSE.txt for license information.                  *|
+|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception                    *|
 |*                                                                            *|
 |*===----------------------------------------------------------------------===*|
 |*                                                                            *|
diff --git a/bindings/ocaml/transforms/ipo/ipo_ocaml.c b/bindings/ocaml/transforms/ipo/ipo_ocaml.c
index 9d8fb1eb711b6d90ad63f53400c9e6c93efa800b..e4226d8b370c3392f58c5c1e6343cf5f64ca9fb0 100644
--- a/bindings/ocaml/transforms/ipo/ipo_ocaml.c
+++ b/bindings/ocaml/transforms/ipo/ipo_ocaml.c
@@ -1,9 +1,9 @@
 /*===-- ipo_ocaml.c - LLVM OCaml Glue ---------------------------*- C++ -*-===*\
 |*                                                                            *|
-|*                     The LLVM Compiler Infrastructure                       *|
-|*                                                                            *|
-|* This file is distributed under the University of Illinois Open Source      *|
-|* License. See LICENSE.TXT for details.                                      *|
+|* Part of the LLVM Project, under the Apache License v2.0 with LLVM          *|
+|* Exceptions.                                                                *|
+|* See https://llvm.org/LICENSE.txt for license information.                  *|
+|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception                    *|
 |*                                                                            *|
 |*===----------------------------------------------------------------------===*|
 |*                                                                            *|
diff --git a/bindings/ocaml/transforms/ipo/llvm_ipo.ml b/bindings/ocaml/transforms/ipo/llvm_ipo.ml
index 1af7d67eac51f067e3ce91f0d61fbaf622af18b4..cc6b0507968f8dd639d6e99b775f8f4079717373 100644
--- a/bindings/ocaml/transforms/ipo/llvm_ipo.ml
+++ b/bindings/ocaml/transforms/ipo/llvm_ipo.ml
@@ -1,9 +1,8 @@
 (*===-- llvm_ipo.ml - LLVM OCaml Interface --------------------*- OCaml -*-===*
  *
- *                     The LLVM Compiler Infrastructure
- *
- * This file is distributed under the University of Illinois Open Source
- * License. See LICENSE.TXT for details.
+ * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+ * See https://llvm.org/LICENSE.txt for license information.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
  *
  *===----------------------------------------------------------------------===*)
 
diff --git a/bindings/ocaml/transforms/ipo/llvm_ipo.mli b/bindings/ocaml/transforms/ipo/llvm_ipo.mli
index 09a4860b91ee7f231de828d02c80c8797ab0be66..4ae988d8e579b1bba84658ad18edfd9985bb2d4c 100644
--- a/bindings/ocaml/transforms/ipo/llvm_ipo.mli
+++ b/bindings/ocaml/transforms/ipo/llvm_ipo.mli
@@ -1,9 +1,8 @@
 (*===-- llvm_ipo.mli - LLVM OCaml Interface -------------------*- OCaml -*-===*
  *
- *                     The LLVM Compiler Infrastructure
- *
- * This file is distributed under the University of Illinois Open Source
- * License. See LICENSE.TXT for details.
+ * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+ * See https://llvm.org/LICENSE.txt for license information.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
  *
  *===----------------------------------------------------------------------===*)
 
diff --git a/bindings/ocaml/transforms/passmgr_builder/llvm_passmgr_builder.ml b/bindings/ocaml/transforms/passmgr_builder/llvm_passmgr_builder.ml
index 60df44611dc7a2e33779d91884dc140d36dff9bb..f143eaf26eb02aec6ee507ce94d0db6f958a41b5 100644
--- a/bindings/ocaml/transforms/passmgr_builder/llvm_passmgr_builder.ml
+++ b/bindings/ocaml/transforms/passmgr_builder/llvm_passmgr_builder.ml
@@ -1,9 +1,8 @@
 (*===-- llvm_passmgr_builder.ml - LLVM OCaml Interface --------*- OCaml -*-===*
  *
- *                     The LLVM Compiler Infrastructure
- *
- * This file is distributed under the University of Illinois Open Source
- * License. See LICENSE.TXT for details.
+ * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+ * See https://llvm.org/LICENSE.txt for license information.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
  *
  *===----------------------------------------------------------------------===*)
 
diff --git a/bindings/ocaml/transforms/passmgr_builder/llvm_passmgr_builder.mli b/bindings/ocaml/transforms/passmgr_builder/llvm_passmgr_builder.mli
index ce162b1213fd2c0ea932603561018f05e1aeca28..8bb1005e6d461427b141771be69d3edf7a485e06 100644
--- a/bindings/ocaml/transforms/passmgr_builder/llvm_passmgr_builder.mli
+++ b/bindings/ocaml/transforms/passmgr_builder/llvm_passmgr_builder.mli
@@ -1,9 +1,8 @@
 (*===-- llvm_passmgr_builder.mli - LLVM OCaml Interface -------*- OCaml -*-===*
  *
- *                     The LLVM Compiler Infrastructure
- *
- * This file is distributed under the University of Illinois Open Source
- * License. See LICENSE.TXT for details.
+ * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+ * See https://llvm.org/LICENSE.txt for license information.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
  *
  *===----------------------------------------------------------------------===*)
 
diff --git a/bindings/ocaml/transforms/passmgr_builder/passmgr_builder_ocaml.c b/bindings/ocaml/transforms/passmgr_builder/passmgr_builder_ocaml.c
index a43863cea69112a7581600984bcaa55ac5496202..6d1f72efd87443e387936cfe62162ac9df82ef31 100644
--- a/bindings/ocaml/transforms/passmgr_builder/passmgr_builder_ocaml.c
+++ b/bindings/ocaml/transforms/passmgr_builder/passmgr_builder_ocaml.c
@@ -1,9 +1,9 @@
 /*===-- passmgr_builder_ocaml.c - LLVM OCaml Glue ---------------*- C++ -*-===*\
 |*                                                                            *|
-|*                     The LLVM Compiler Infrastructure                       *|
-|*                                                                            *|
-|* This file is distributed under the University of Illinois Open Source      *|
-|* License. See LICENSE.TXT for details.                                      *|
+|* Part of the LLVM Project, under the Apache License v2.0 with LLVM          *|
+|* Exceptions.                                                                *|
+|* See https://llvm.org/LICENSE.txt for license information.                  *|
+|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception                    *|
 |*                                                                            *|
 |*===----------------------------------------------------------------------===*|
 |*                                                                            *|
diff --git a/bindings/ocaml/transforms/scalar_opts/llvm_scalar_opts.ml b/bindings/ocaml/transforms/scalar_opts/llvm_scalar_opts.ml
index dcaf7e5e38f91bed82de3154d14bd82784b35b5a..0c99b0f34f76818fb68ec805240bc0748239f764 100644
--- a/bindings/ocaml/transforms/scalar_opts/llvm_scalar_opts.ml
+++ b/bindings/ocaml/transforms/scalar_opts/llvm_scalar_opts.ml
@@ -1,9 +1,8 @@
 (*===-- llvm_scalar_opts.ml - LLVM OCaml Interface ------------*- OCaml -*-===*
  *
- *                     The LLVM Compiler Infrastructure
- *
- * This file is distributed under the University of Illinois Open Source
- * License. See LICENSE.TXT for details.
+ * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+ * See https://llvm.org/LICENSE.txt for license information.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
  *
  *===----------------------------------------------------------------------===*)
 
diff --git a/bindings/ocaml/transforms/scalar_opts/llvm_scalar_opts.mli b/bindings/ocaml/transforms/scalar_opts/llvm_scalar_opts.mli
index 6767c001ac6b74b87c99cdf733567ea17b93401b..76d86ddf00147d95dd635d2fd21cd1999c6e48bc 100644
--- a/bindings/ocaml/transforms/scalar_opts/llvm_scalar_opts.mli
+++ b/bindings/ocaml/transforms/scalar_opts/llvm_scalar_opts.mli
@@ -1,9 +1,8 @@
 (*===-- llvm_scalar_opts.mli - LLVM OCaml Interface -----------*- OCaml -*-===*
  *
- *                     The LLVM Compiler Infrastructure
- *
- * This file is distributed under the University of Illinois Open Source
- * License. See LICENSE.TXT for details.
+ * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+ * See https://llvm.org/LICENSE.txt for license information.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
  *
  *===----------------------------------------------------------------------===*)
 
diff --git a/bindings/ocaml/transforms/scalar_opts/scalar_opts_ocaml.c b/bindings/ocaml/transforms/scalar_opts/scalar_opts_ocaml.c
index 0b52f36c0eda755e066c13103e00191d77103e24..cb3d50d31fb7c8d9e82c8ff16ab000aee21bfcda 100644
--- a/bindings/ocaml/transforms/scalar_opts/scalar_opts_ocaml.c
+++ b/bindings/ocaml/transforms/scalar_opts/scalar_opts_ocaml.c
@@ -1,9 +1,9 @@
 /*===-- scalar_opts_ocaml.c - LLVM OCaml Glue -------------------*- C++ -*-===*\
 |*                                                                            *|
-|*                     The LLVM Compiler Infrastructure                       *|
-|*                                                                            *|
-|* This file is distributed under the University of Illinois Open Source      *|
-|* License. See LICENSE.TXT for details.                                      *|
+|* Part of the LLVM Project, under the Apache License v2.0 with LLVM          *|
+|* Exceptions.                                                                *|
+|* See https://llvm.org/LICENSE.txt for license information.                  *|
+|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception                    *|
 |*                                                                            *|
 |*===----------------------------------------------------------------------===*|
 |*                                                                            *|
diff --git a/bindings/ocaml/transforms/utils/llvm_transform_utils.ml b/bindings/ocaml/transforms/utils/llvm_transform_utils.ml
index 135efe22b5a772ddbc06a5f550218c9ca9893050..20a50b107b3f945d0b0b38bd30cedd7f4047902a 100644
--- a/bindings/ocaml/transforms/utils/llvm_transform_utils.ml
+++ b/bindings/ocaml/transforms/utils/llvm_transform_utils.ml
@@ -1,9 +1,8 @@
 (*===-- llvm_transform_utils.ml - LLVM OCaml Interface --------*- OCaml -*-===*
  *
- *                     The LLVM Compiler Infrastructure
- *
- * This file is distributed under the University of Illinois Open Source
- * License. See LICENSE.TXT for details.
+ * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+ * See https://llvm.org/LICENSE.txt for license information.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
  *
  *===----------------------------------------------------------------------===*)
 
diff --git a/bindings/ocaml/transforms/utils/llvm_transform_utils.mli b/bindings/ocaml/transforms/utils/llvm_transform_utils.mli
index 1c2b07c34613bdec3293abbf2d2ae032da55f5ad..536f41dc4353538eedadcc14dd42d9eed37e41df 100644
--- a/bindings/ocaml/transforms/utils/llvm_transform_utils.mli
+++ b/bindings/ocaml/transforms/utils/llvm_transform_utils.mli
@@ -1,9 +1,8 @@
 (*===-- llvm_transform_utils.mli - LLVM OCaml Interface -------*- OCaml -*-===*
  *
- *                     The LLVM Compiler Infrastructure
- *
- * This file is distributed under the University of Illinois Open Source
- * License. See LICENSE.TXT for details.
+ * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+ * See https://llvm.org/LICENSE.txt for license information.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
  *
  *===----------------------------------------------------------------------===*)
 
diff --git a/bindings/ocaml/transforms/utils/transform_utils_ocaml.c b/bindings/ocaml/transforms/utils/transform_utils_ocaml.c
index f2c50d97523a21127fea3848528e6fddbb729c69..918eec1462223fac437c74e0a5fbbec3c7b0840a 100644
--- a/bindings/ocaml/transforms/utils/transform_utils_ocaml.c
+++ b/bindings/ocaml/transforms/utils/transform_utils_ocaml.c
@@ -1,9 +1,9 @@
 /*===-- transform_utils_ocaml.c - LLVM OCaml Glue ---------------*- C++ -*-===*\
 |*                                                                            *|
-|*                     The LLVM Compiler Infrastructure                       *|
-|*                                                                            *|
-|* This file is distributed under the University of Illinois Open Source      *|
-|* License. See LICENSE.TXT for details.                                      *|
+|* Part of the LLVM Project, under the Apache License v2.0 with LLVM          *|
+|* Exceptions.                                                                *|
+|* See https://llvm.org/LICENSE.txt for license information.                  *|
+|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception                    *|
 |*                                                                            *|
 |*===----------------------------------------------------------------------===*|
 |*                                                                            *|
diff --git a/bindings/ocaml/transforms/vectorize/llvm_vectorize.ml b/bindings/ocaml/transforms/vectorize/llvm_vectorize.ml
index f9f6be26d7f18372c427ff354bc52290dbf3445c..a2e280e1584892a8a010b78e7d40f6a7b875b97a 100644
--- a/bindings/ocaml/transforms/vectorize/llvm_vectorize.ml
+++ b/bindings/ocaml/transforms/vectorize/llvm_vectorize.ml
@@ -1,9 +1,8 @@
 (*===-- llvm_vectorize.ml - LLVM OCaml Interface --------------*- OCaml -*-===*
  *
- *                     The LLVM Compiler Infrastructure
- *
- * This file is distributed under the University of Illinois Open Source
- * License. See LICENSE.TXT for details.
+ * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+ * See https://llvm.org/LICENSE.txt for license information.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
  *
  *===----------------------------------------------------------------------===*)
 
diff --git a/bindings/ocaml/transforms/vectorize/llvm_vectorize.mli b/bindings/ocaml/transforms/vectorize/llvm_vectorize.mli
index f9b4ce5aae583e9732dee7f62b1abfe643626fa4..7376d9e6d81465bb7cef78a568124907c3efb957 100644
--- a/bindings/ocaml/transforms/vectorize/llvm_vectorize.mli
+++ b/bindings/ocaml/transforms/vectorize/llvm_vectorize.mli
@@ -1,9 +1,8 @@
 (*===-- llvm_vectorize.mli - LLVM OCaml Interface -------------*- OCaml -*-===*
  *
- *                     The LLVM Compiler Infrastructure
- *
- * This file is distributed under the University of Illinois Open Source
- * License. See LICENSE.TXT for details.
+ * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+ * See https://llvm.org/LICENSE.txt for license information.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
  *
  *===----------------------------------------------------------------------===*)
 
diff --git a/bindings/ocaml/transforms/vectorize/vectorize_ocaml.c b/bindings/ocaml/transforms/vectorize/vectorize_ocaml.c
index dcd9231b5bf275f7d66d334850ade09890e07dc2..ba9c132c85c29ef27c2ff7aa1df04330a9e6346d 100644
--- a/bindings/ocaml/transforms/vectorize/vectorize_ocaml.c
+++ b/bindings/ocaml/transforms/vectorize/vectorize_ocaml.c
@@ -1,9 +1,9 @@
 /*===-- vectorize_ocaml.c - LLVM OCaml Glue ---------------------*- C++ -*-===*\
 |*                                                                            *|
-|*                     The LLVM Compiler Infrastructure                       *|
-|*                                                                            *|
-|* This file is distributed under the University of Illinois Open Source      *|
-|* License. See LICENSE.TXT for details.                                      *|
+|* Part of the LLVM Project, under the Apache License v2.0 with LLVM          *|
+|* Exceptions.                                                                *|
+|* See https://llvm.org/LICENSE.txt for license information.                  *|
+|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception                    *|
 |*                                                                            *|
 |*===----------------------------------------------------------------------===*|
 |*                                                                            *|
diff --git a/bindings/python/llvm/common.py b/bindings/python/llvm/common.py
index b177f7fa47e918cb9c8279e84e5d72ae2ca35c0f..c29c5eecf74847e135b010f3e58ffb27bd2dc55c 100644
--- a/bindings/python/llvm/common.py
+++ b/bindings/python/llvm/common.py
@@ -1,9 +1,8 @@
 #===- common.py - Python LLVM Bindings -----------------------*- python -*--===#
 #
-#                     The LLVM Compiler Infrastructure
-#
-# This file is distributed under the University of Illinois Open Source
-# License. See LICENSE.TXT for details.
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 #
 #===------------------------------------------------------------------------===#
 
diff --git a/bindings/python/llvm/core.py b/bindings/python/llvm/core.py
index 6b3da6d867929ffeeca0af48e63594e2dc1b39ec..4d90f8652de13ea67df4ee6d24bdb25bd7de9db6 100644
--- a/bindings/python/llvm/core.py
+++ b/bindings/python/llvm/core.py
@@ -1,11 +1,11 @@
 #===- core.py - Python LLVM Bindings -------------------------*- python -*--===#
 #
-#                     The LLVM Compiler Infrastructure
-#
-# This file is distributed under the University of Illinois Open Source
-# License. See LICENSE.TXT for details.
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 #
 #===------------------------------------------------------------------------===#
+from __future__ import print_function
 
 from .common import LLVMObject
 from .common import c_object_p
@@ -18,6 +18,8 @@ from ctypes import byref
 from ctypes import c_char_p
 from ctypes import c_uint
 
+import sys
+
 __all__ = [
     "lib",
     "Enums",
@@ -235,7 +237,7 @@ class Module(LLVMObject):
         def __iter__(self):
             return self
         
-        def next(self):
+        def __next__(self):
             if not isinstance(self.function, Function):
                 raise StopIteration("")
             result = self.function
@@ -244,7 +246,10 @@ class Module(LLVMObject):
             else:
                 self.function = self.function.next
             return result
-    
+
+        if sys.version_info.major == 2:
+            next = __next__
+
     def __iter__(self):
         return Module.__function_iterator(self)
 
@@ -303,7 +308,7 @@ class Function(Value):
         def __iter__(self):
             return self
         
-        def next(self):
+        def __next__(self):
             if not isinstance(self.bb, BasicBlock):
                 raise StopIteration("")
             result = self.bb
@@ -312,6 +317,9 @@ class Function(Value):
             else:
                 self.bb = self.bb.next
             return result
+
+        if sys.version_info.major == 2:
+            next = __next__
     
     def __iter__(self):
         return Function.__bb_iterator(self)
@@ -380,7 +388,7 @@ class BasicBlock(LLVMObject):
         def __iter__(self):
             return self
         
-        def next(self):
+        def __next__(self):
             if not isinstance(self.inst, Instruction):
                 raise StopIteration("")
             result = self.inst
@@ -389,7 +397,10 @@ class BasicBlock(LLVMObject):
             else:
                 self.inst = self.inst.next
             return result
-    
+
+        if sys.version_info.major == 2:
+            next = __next__
+
     def __iter__(self):
         return BasicBlock.__inst_iterator(self)
 
@@ -605,7 +616,7 @@ def register_enumerations():
     ]
     for enum_class, enum_spec in enums:
         for name, value in enum_spec:
-            print name, value
+            print(name, value)
             enum_class.register(name, value)
     return enums
 
diff --git a/bindings/python/llvm/disassembler.py b/bindings/python/llvm/disassembler.py
index f2df275bf4a0969cb8bc741919817495deee261c..75625588911ca91d84e398c51e16419a180797f6 100644
--- a/bindings/python/llvm/disassembler.py
+++ b/bindings/python/llvm/disassembler.py
@@ -1,9 +1,8 @@
 #===- disassembler.py - Python LLVM Bindings -----------------*- python -*--===#
 #
-#                     The LLVM Compiler Infrastructure
-#
-# This file is distributed under the University of Illinois Open Source
-# License. See LICENSE.TXT for details.
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 #
 #===------------------------------------------------------------------------===#
 
diff --git a/bindings/python/llvm/enumerations.py b/bindings/python/llvm/enumerations.py
index f49d2faad351b09db190e59798fb2e30a45cb56c..ebb39a4ded831dc020c6aaa9586f6d32cf9d1cb5 100644
--- a/bindings/python/llvm/enumerations.py
+++ b/bindings/python/llvm/enumerations.py
@@ -1,9 +1,8 @@
 #===- enumerations.py - Python LLVM Enumerations -------------*- python -*--===#
 #
-#                     The LLVM Compiler Infrastructure
-#
-# This file is distributed under the University of Illinois Open Source
-# License. See LICENSE.TXT for details.
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 #
 #===------------------------------------------------------------------------===#
 
diff --git a/bindings/python/llvm/object.py b/bindings/python/llvm/object.py
index b427113e9cea945ac4b550ec585a93d55e779edb..e8841b6045f62ab3ec3ffe24be2676b3193a0ce6 100644
--- a/bindings/python/llvm/object.py
+++ b/bindings/python/llvm/object.py
@@ -1,9 +1,8 @@
 #===- object.py - Python Object Bindings --------------------*- python -*--===#
 #
-#                     The LLVM Compiler Infrastructure
-#
-# This file is distributed under the University of Illinois Open Source
-# License. See LICENSE.TXT for details.
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 #
 #===------------------------------------------------------------------------===#
 
diff --git a/bindings/python/llvm/tests/base.py b/bindings/python/llvm/tests/base.py
index 194f1a41192bfd5d09095e8745f5b8d622007a2d..aa435bc1f35f5603c7dc67642c9b286e85c29abf 100644
--- a/bindings/python/llvm/tests/base.py
+++ b/bindings/python/llvm/tests/base.py
@@ -1,6 +1,8 @@
 import os.path
+import sys
 import unittest
 
+
 POSSIBLE_TEST_BINARIES = [
     'libreadline.so.5',
     'libreadline.so.6',
@@ -15,6 +17,9 @@ POSSIBLE_TEST_BINARY_PATHS = [
 ]
 
 class TestBase(unittest.TestCase):
+    if sys.version_info.major == 2:
+        assertRaisesRegex = unittest.TestCase.assertRaisesRegexp
+
     def get_test_binary(self):
         """Helper to obtain a test binary for object file testing.
 
diff --git a/bindings/python/llvm/tests/test_bitreader.py b/bindings/python/llvm/tests/test_bitreader.py
index d5850091a91f29c4b7571f9837c116e481b2d65f..460005a2b87ab21947d7fa972d1767ae5ba9a996 100644
--- a/bindings/python/llvm/tests/test_bitreader.py
+++ b/bindings/python/llvm/tests/test_bitreader.py
@@ -1,3 +1,5 @@
+from __future__ import print_function
+
 from .base import TestBase
 from ..core import OpCode
 from ..core import MemoryBuffer
@@ -11,5 +13,5 @@ class TestBitReader(TestBase):
     def test_parse_bitcode(self):
         source = self.get_test_bc()
         m = parse_bitcode(MemoryBuffer(filename=source))
-        print m.target
-        print m.datalayout
+        print(m.target)
+        print(m.datalayout)
diff --git a/bindings/python/llvm/tests/test_core.py b/bindings/python/llvm/tests/test_core.py
index da7b635ec999b70020606db73e89c27b5742e748..68572b50b3d66afff8a0861799f67fa54ca7f924 100644
--- a/bindings/python/llvm/tests/test_core.py
+++ b/bindings/python/llvm/tests/test_core.py
@@ -1,3 +1,5 @@
+from __future__ import print_function
+
 from .base import TestBase
 from ..core import MemoryBuffer
 from ..core import PassRegistry
@@ -127,7 +129,7 @@ class TestCore(TestBase):
             self.assertEqual(inst.opcode, inst_list[i][1])
             for op in range(len(inst)):
                 o = inst.get_operand(op)
-                print o.name
+                print(o.name)
                 o.dump()
             inst.dump()
             i += 1
diff --git a/bindings/python/llvm/tests/test_disassembler.py b/bindings/python/llvm/tests/test_disassembler.py
index 37a04e4fc7e774f1e5a0106ac5f34d98a8f4033e..29f2f7060bac5c57309c6ad688ba722959dcc3ea 100644
--- a/bindings/python/llvm/tests/test_disassembler.py
+++ b/bindings/python/llvm/tests/test_disassembler.py
@@ -1,3 +1,5 @@
+from __future__ import print_function
+
 from .base import TestBase
 
 from ..disassembler import Disassembler, Option_UseMarkup
@@ -17,7 +19,7 @@ class TestDisassembler(TestBase):
         self.assertEqual(s, '\tjcxz\t-127')
 
     def test_nonexistent_triple(self):
-        with self.assertRaisesRegexp(Exception, "Could not obtain disassembler for triple"):
+        with self.assertRaisesRegex(Exception, "Could not obtain disassembler for triple"):
             Disassembler("nonexistent-triple-raises")
 
     def test_get_instructions(self):
@@ -38,6 +40,6 @@ class TestDisassembler(TestBase):
         disassembler = Disassembler(triple)
         disassembler.set_options(Option_UseMarkup)
         count, s = disassembler.get_instruction(sequence)
-        print s
+        print(s)
         self.assertEqual(count, 4)
         self.assertEqual(s, '\tpush\t{<reg:r4>, <reg:lr>}')
diff --git a/bindings/python/llvm/tests/test_object.py b/bindings/python/llvm/tests/test_object.py
index 3f92d8155b61337cf2b8fd1845e9521608a001da..a45b7beec3353922c04f87c3f8bc06b4c91940a6 100644
--- a/bindings/python/llvm/tests/test_object.py
+++ b/bindings/python/llvm/tests/test_object.py
@@ -1,3 +1,5 @@
+from numbers import Integral
+
 from .base import TestBase
 from ..object import ObjectFile
 from ..object import Relocation
@@ -20,9 +22,9 @@ class TestObjectFile(TestBase):
             count += 1
             assert isinstance(section, Section)
             assert isinstance(section.name, str)
-            assert isinstance(section.size, long)
+            assert isinstance(section.size, Integral)
             assert isinstance(section.contents, str)
-            assert isinstance(section.address, long)
+            assert isinstance(section.address, Integral)
             assert len(section.contents) == section.size
 
         self.assertGreater(count, 0)
@@ -38,8 +40,8 @@ class TestObjectFile(TestBase):
             count += 1
             assert isinstance(symbol, Symbol)
             assert isinstance(symbol.name, str)
-            assert isinstance(symbol.address, long)
-            assert isinstance(symbol.size, long)
+            assert isinstance(symbol.address, Integral)
+            assert isinstance(symbol.size, Integral)
 
         self.assertGreater(count, 0)
 
@@ -60,8 +62,8 @@ class TestObjectFile(TestBase):
         for section in o.get_sections():
             for relocation in section.get_relocations():
                 assert isinstance(relocation, Relocation)
-                assert isinstance(relocation.address, long)
-                assert isinstance(relocation.offset, long)
-                assert isinstance(relocation.type_number, long)
+                assert isinstance(relocation.address, Integral)
+                assert isinstance(relocation.offset, Integral)
+                assert isinstance(relocation.type_number, Integral)
                 assert isinstance(relocation.type_name, str)
                 assert isinstance(relocation.value_string, str)
diff --git a/cmake/config-ix.cmake b/cmake/config-ix.cmake
index 900c35ee4f0c9bb01c39097e8ba4b013b0c73505..18026d69e87cff54c27edb017045b7379f9b1f4a 100644
--- a/cmake/config-ix.cmake
+++ b/cmake/config-ix.cmake
@@ -28,7 +28,6 @@ check_include_file(dlfcn.h HAVE_DLFCN_H)
 check_include_file(errno.h HAVE_ERRNO_H)
 check_include_file(fcntl.h HAVE_FCNTL_H)
 check_include_file(link.h HAVE_LINK_H)
-check_include_file(malloc.h HAVE_MALLOC_H)
 check_include_file(malloc/malloc.h HAVE_MALLOC_MALLOC_H)
 if( NOT PURE_WINDOWS )
   check_include_file(pthread.h HAVE_PTHREAD_H)
@@ -325,6 +324,15 @@ else()
   unset(HAVE_FFI_CALL CACHE)
 endif( LLVM_ENABLE_FFI )
 
+# Whether we can use std::is_trivially_copyable to verify llvm::is_trivially_copyable.
+CHECK_CXX_SOURCE_COMPILES("
+#include <type_traits>
+struct T { int val; };
+static_assert(std::is_trivially_copyable<T>::value, \"ok\");
+int main() { return 0;}
+" HAVE_STD_IS_TRIVIALLY_COPYABLE)
+
+
 # Define LLVM_HAS_ATOMICS if gcc or MSVC atomic builtins are supported.
 include(CheckAtomic)
 
@@ -392,6 +400,8 @@ elseif (LLVM_NATIVE_ARCH MATCHES "arm64")
   set(LLVM_NATIVE_ARCH AArch64)
 elseif (LLVM_NATIVE_ARCH MATCHES "arm")
   set(LLVM_NATIVE_ARCH ARM)
+elseif (LLVM_NATIVE_ARCH MATCHES "avr")
+  set(LLVM_NATIVE_ARCH AVR)
 elseif (LLVM_NATIVE_ARCH MATCHES "mips")
   set(LLVM_NATIVE_ARCH Mips)
 elseif (LLVM_NATIVE_ARCH MATCHES "xcore")
diff --git a/cmake/modules/AddLLVM.cmake b/cmake/modules/AddLLVM.cmake
index c5aa961a292b4a2b8ba83c986aa51c78451ec5b3..95a88af3bbf3d5714ecb06342dcd473bf4505439 100644
--- a/cmake/modules/AddLLVM.cmake
+++ b/cmake/modules/AddLLVM.cmake
@@ -380,7 +380,7 @@ endfunction(set_windows_version_resource_properties)
 function(llvm_add_library name)
   cmake_parse_arguments(ARG
     "MODULE;SHARED;STATIC;OBJECT;DISABLE_LLVM_LINK_LLVM_DYLIB;SONAME;NO_INSTALL_RPATH"
-    "OUTPUT_NAME;PLUGIN_TOOL"
+    "OUTPUT_NAME;PLUGIN_TOOL;ENTITLEMENTS"
     "ADDITIONAL_HEADERS;DEPENDS;LINK_COMPONENTS;LINK_LIBS;OBJLIBS"
     ${ARGN})
   list(APPEND LLVM_COMMON_DEPENDS ${ARG_DEPENDS})
@@ -584,7 +584,7 @@ function(llvm_add_library name)
 
   if(ARG_SHARED OR ARG_MODULE)
     llvm_externalize_debuginfo(${name})
-    llvm_codesign(${name})
+    llvm_codesign(${name} ENTITLEMENTS ${ARG_ENTITLEMENTS})
   endif()
 endfunction()
 
@@ -616,11 +616,13 @@ endfunction()
 
 macro(add_llvm_library name)
   cmake_parse_arguments(ARG
-    "SHARED;BUILDTREE_ONLY"
+    "SHARED;BUILDTREE_ONLY;MODULE"
     ""
     ""
     ${ARGN})
-  if( BUILD_SHARED_LIBS OR ARG_SHARED )
+  if(ARG_MODULE)
+    llvm_add_library(${name} MODULE ${ARG_UNPARSED_ARGUMENTS})
+  elseif( BUILD_SHARED_LIBS OR ARG_SHARED )
     llvm_add_library(${name} SHARED ${ARG_UNPARSED_ARGUMENTS})
   else()
     llvm_add_library(${name} ${ARG_UNPARSED_ARGUMENTS})
@@ -629,11 +631,14 @@ macro(add_llvm_library name)
   # Libraries that are meant to only be exposed via the build tree only are
   # never installed and are only exported as a target in the special build tree
   # config file.
-  if (NOT ARG_BUILDTREE_ONLY)
+  if (NOT ARG_BUILDTREE_ONLY AND NOT ARG_MODULE)
     set_property( GLOBAL APPEND PROPERTY LLVM_LIBS ${name} )
   endif()
 
-  if( EXCLUDE_FROM_ALL )
+  if (ARG_MODULE AND NOT TARGET ${name})
+    # Add empty "phony" target
+    add_custom_target(${name})
+  elseif( EXCLUDE_FROM_ALL )
     set_target_properties( ${name} PROPERTIES EXCLUDE_FROM_ALL ON)
   elseif(ARG_BUILDTREE_ONLY)
     set_property(GLOBAL APPEND PROPERTY LLVM_EXPORTS_BUILDTREE_ONLY ${name})
@@ -642,7 +647,7 @@ macro(add_llvm_library name)
         ${name} STREQUAL "OptRemarks" OR
         (LLVM_LINK_LLVM_DYLIB AND ${name} STREQUAL "LLVM"))
       set(install_dir lib${LLVM_LIBDIR_SUFFIX})
-      if(ARG_SHARED OR BUILD_SHARED_LIBS)
+      if(ARG_MODULE OR ARG_SHARED OR BUILD_SHARED_LIBS)
         if(WIN32 OR CYGWIN OR MINGW)
           set(install_type RUNTIME)
           set(install_dir bin)
@@ -653,6 +658,10 @@ macro(add_llvm_library name)
         set(install_type ARCHIVE)
       endif()
 
+      if (ARG_MODULE)
+        set(install_type LIBRARY)
+      endif()
+
       if(${name} IN_LIST LLVM_DISTRIBUTION_COMPONENTS OR
           NOT LLVM_DISTRIBUTION_COMPONENTS)
         set(export_to_llvmexports EXPORT LLVMExports)
@@ -672,44 +681,12 @@ macro(add_llvm_library name)
     endif()
     set_property(GLOBAL APPEND PROPERTY LLVM_EXPORTS ${name})
   endif()
-  set_target_properties(${name} PROPERTIES FOLDER "Libraries")
-endmacro(add_llvm_library name)
-
-macro(add_llvm_loadable_module name)
-  llvm_add_library(${name} MODULE ${ARGN})
-  if(NOT TARGET ${name})
-    # Add empty "phony" target
-    add_custom_target(${name})
+  if (ARG_MODULE)
+    set_target_properties(${name} PROPERTIES FOLDER "Loadable modules")
   else()
-    if( EXCLUDE_FROM_ALL )
-      set_target_properties( ${name} PROPERTIES EXCLUDE_FROM_ALL ON)
-    else()
-      if (NOT LLVM_INSTALL_TOOLCHAIN_ONLY)
-        if(WIN32 OR CYGWIN)
-          # DLL platform
-          set(dlldir "bin")
-        else()
-          set(dlldir "lib${LLVM_LIBDIR_SUFFIX}")
-        endif()
-
-        if(${name} IN_LIST LLVM_DISTRIBUTION_COMPONENTS OR
-            NOT LLVM_DISTRIBUTION_COMPONENTS)
-          set(export_to_llvmexports EXPORT LLVMExports)
-          set_property(GLOBAL PROPERTY LLVM_HAS_EXPORTS True)
-        endif()
-
-        install(TARGETS ${name}
-                ${export_to_llvmexports}
-                LIBRARY DESTINATION ${dlldir}
-                ARCHIVE DESTINATION lib${LLVM_LIBDIR_SUFFIX})
-      endif()
-      set_property(GLOBAL APPEND PROPERTY LLVM_EXPORTS ${name})
-    endif()
+    set_target_properties(${name} PROPERTIES FOLDER "Libraries")
   endif()
-
-  set_target_properties(${name} PROPERTIES FOLDER "Loadable modules")
-endmacro(add_llvm_loadable_module name)
-
+endmacro(add_llvm_library name)
 
 macro(add_llvm_executable name)
   cmake_parse_arguments(ARG
@@ -934,14 +911,27 @@ macro(add_llvm_utility name)
 
   add_llvm_executable(${name} DISABLE_LLVM_LINK_LLVM_DYLIB ${ARGN})
   set_target_properties(${name} PROPERTIES FOLDER "Utils")
-  if( LLVM_INSTALL_UTILS AND LLVM_BUILD_UTILS )
-    install (TARGETS ${name}
-      RUNTIME DESTINATION ${LLVM_UTILS_INSTALL_DIR}
-      COMPONENT ${name})
-    if (NOT LLVM_ENABLE_IDE)
-      add_llvm_install_targets(install-${name}
-                               DEPENDS ${name}
-                               COMPONENT ${name})
+  if (NOT LLVM_INSTALL_TOOLCHAIN_ONLY)
+    if (LLVM_INSTALL_UTILS AND LLVM_BUILD_UTILS)
+      if (${name} IN_LIST LLVM_DISTRIBUTION_COMPONENTS OR
+          NOT LLVM_DISTRIBUTION_COMPONENTS)
+        set(export_to_llvmexports EXPORT LLVMExports)
+        set_property(GLOBAL PROPERTY LLVM_HAS_EXPORTS True)
+      endif()
+
+      install(TARGETS ${name}
+              ${export_to_llvmexports}
+              RUNTIME DESTINATION ${LLVM_UTILS_INSTALL_DIR}
+              COMPONENT ${name})
+
+      if (NOT LLVM_ENABLE_IDE)
+        add_llvm_install_targets(install-${name}
+                                 DEPENDS ${name}
+                                 COMPONENT ${name})
+      endif()
+      set_property(GLOBAL APPEND PROPERTY LLVM_EXPORTS ${name})
+    elseif(LLVM_BUILD_UTILS)
+      set_property(GLOBAL APPEND PROPERTY LLVM_EXPORTS_BUILDTREE_ONLY ${name})
     endif()
   endif()
 endmacro(add_llvm_utility name)
@@ -1300,7 +1290,6 @@ function(get_llvm_lit_path base_dir file_name)
   cmake_parse_arguments(ARG "ALLOW_EXTERNAL" "" "" ${ARGN})
 
   if (ARG_ALLOW_EXTERNAL)
-    set(LLVM_DEFAULT_EXTERNAL_LIT "${LLVM_EXTERNAL_LIT}")
     set (LLVM_EXTERNAL_LIT "" CACHE STRING "Command used to spawn lit")
     if ("${LLVM_EXTERNAL_LIT}" STREQUAL "")
       set(LLVM_EXTERNAL_LIT "${LLVM_DEFAULT_EXTERNAL_LIT}")
@@ -1643,7 +1632,16 @@ function(llvm_codesign name)
     return()
   endif()
 
-  if(APPLE)
+  if(CMAKE_GENERATOR STREQUAL "Xcode")
+    set_target_properties(${name} PROPERTIES
+      XCODE_ATTRIBUTE_CODE_SIGN_IDENTITY ${LLVM_CODESIGNING_IDENTITY}
+    )
+    if(DEFINED ARG_ENTITLEMENTS)
+      set_target_properties(${name} PROPERTIES
+        XCODE_ATTRIBUTE_CODE_SIGN_ENTITLEMENTS ${ARG_ENTITLEMENTS}
+      )
+    endif()
+  elseif(APPLE)
     if(NOT CMAKE_CODESIGN)
       set(CMAKE_CODESIGN xcrun codesign)
     endif()
@@ -1657,18 +1655,13 @@ function(llvm_codesign name)
     if(DEFINED ARG_ENTITLEMENTS)
       set(pass_entitlements --entitlements ${ARG_ENTITLEMENTS})
     endif()
-    if(CMAKE_GENERATOR STREQUAL "Xcode")
-      # Avoid double-signing error: Since output overwrites input, Xcode runs
-      # the post-build rule even if the actual build-step was skipped.
-      set(pass_force --force)
-    endif()
 
     add_custom_command(
       TARGET ${name} POST_BUILD
       COMMAND ${CMAKE_COMMAND} -E
               env CODESIGN_ALLOCATE=${CMAKE_CODESIGN_ALLOCATE}
               ${CMAKE_CODESIGN} -s ${LLVM_CODESIGNING_IDENTITY}
-              ${pass_entitlements} ${pass_force} $<TARGET_FILE:${name}>
+              ${pass_entitlements} $<TARGET_FILE:${name}>
     )
   endif()
 endfunction()
@@ -1724,35 +1717,35 @@ function(setup_dependency_debugging name)
   set_target_properties(${name} PROPERTIES RULE_LAUNCH_COMPILE ${sandbox_command})
 endfunction()
 
-# Figure out if we can track VC revisions.
-function(find_first_existing_file out_var)
-  foreach(file ${ARGN})
-    if(EXISTS "${file}")
-      set(${out_var} "${file}" PARENT_SCOPE)
-      return()
-    endif()
-  endforeach()
-endfunction()
-
-macro(find_first_existing_vc_file out_var path)
-    find_program(git_executable NAMES git git.exe git.cmd)
-    # Run from a subdirectory to force git to print an absolute path.
-    execute_process(COMMAND ${git_executable} rev-parse --git-dir
-      WORKING_DIRECTORY ${path}/cmake
-      RESULT_VARIABLE git_result
-      OUTPUT_VARIABLE git_dir
-      ERROR_QUIET)
-    if(git_result EQUAL 0)
-      string(STRIP "${git_dir}" git_dir)
-      set(${out_var} "${git_dir}/logs/HEAD")
-      # some branchless cases (e.g. 'repo') may not yet have .git/logs/HEAD
-      if (NOT EXISTS "${git_dir}/logs/HEAD")
-        file(WRITE "${git_dir}/logs/HEAD" "")
+function(find_first_existing_vc_file path out_var)
+  if(EXISTS "${path}/.svn")
+    set(svn_files
+      "${path}/.svn/wc.db"   # SVN 1.7
+      "${path}/.svn/entries" # SVN 1.6
+    )
+    foreach(file IN LISTS svn_files)
+      if(EXISTS "${file}")
+        set(${out_var} "${file}" PARENT_SCOPE)
+        return()
+      endif()
+    endforeach()
+  else()
+    find_package(Git)
+    if(GIT_FOUND)
+      execute_process(COMMAND ${GIT_EXECUTABLE} rev-parse --git-dir
+        WORKING_DIRECTORY ${path}
+        RESULT_VARIABLE git_result
+        OUTPUT_VARIABLE git_output
+        ERROR_QUIET)
+      if(git_result EQUAL 0)
+        string(STRIP "${git_output}" git_output)
+        get_filename_component(git_dir ${git_output} ABSOLUTE BASE_DIR ${path})
+        # Some branchless cases (e.g. 'repo') may not yet have .git/logs/HEAD
+        if (NOT EXISTS "${git_dir}/logs/HEAD")
+          file(WRITE "${git_dir}/logs/HEAD" "")
+        endif()
+        set(${out_var} "${git_dir}/logs/HEAD" PARENT_SCOPE)
       endif()
-    else()
-      find_first_existing_file(${out_var}
-        "${path}/.svn/wc.db"   # SVN 1.7
-        "${path}/.svn/entries" # SVN 1.6
-      )
     endif()
-endmacro()
+  endif()
+endfunction()
diff --git a/cmake/modules/CMakeLists.txt b/cmake/modules/CMakeLists.txt
index f5cc0006fa06ad5fc2a11707be1d3dac062013d4..9cf22b436fa7367d56e69a1d6c23b8155be956a6 100644
--- a/cmake/modules/CMakeLists.txt
+++ b/cmake/modules/CMakeLists.txt
@@ -41,6 +41,12 @@ set(LLVM_CONFIG_INCLUDE_DIRS
   "${LLVM_MAIN_INCLUDE_DIR}"
   "${LLVM_INCLUDE_DIR}"
   )
+set(LLVM_CONFIG_INCLUDE_DIR
+  "${LLVM_INCLUDE_DIR}"
+  )
+set(LLVM_CONFIG_MAIN_INCLUDE_DIR
+  "${LLVM_MAIN_INCLUDE_DIR}"
+  )
 set(LLVM_CONFIG_LIBRARY_DIRS
   "${LLVM_LIBRARY_DIR}"
   )
@@ -91,6 +97,8 @@ foreach(p ${_count})
 get_filename_component(LLVM_INSTALL_PREFIX \"\${LLVM_INSTALL_PREFIX}\" PATH)")
 endforeach(p)
 set(LLVM_CONFIG_INCLUDE_DIRS "\${LLVM_INSTALL_PREFIX}/include")
+set(LLVM_CONFIG_INCLUDE_DIR "${LLVM_CONFIG_INCLUDE_DIRS}")
+set(LLVM_CONFIG_MAIN_INCLUDE_DIR "${LLVM_CONFIG_INCLUDE_DIRS}")
 set(LLVM_CONFIG_LIBRARY_DIRS "\${LLVM_INSTALL_PREFIX}/lib\${LLVM_LIBDIR_SUFFIX}")
 set(LLVM_CONFIG_CMAKE_DIR "\${LLVM_INSTALL_PREFIX}/${LLVM_INSTALL_PACKAGE_DIR}")
 set(LLVM_CONFIG_BINARY_DIR "\${LLVM_INSTALL_PREFIX}")
diff --git a/cmake/modules/CheckCompilerVersion.cmake b/cmake/modules/CheckCompilerVersion.cmake
index adf500ad53a72236165c61176e908996337ba03f..b1cb5527422896631f8759add21d2ba2cf84be78 100644
--- a/cmake/modules/CheckCompilerVersion.cmake
+++ b/cmake/modules/CheckCompilerVersion.cmake
@@ -1,52 +1,94 @@
-# Check if the host compiler is new enough. LLVM requires at least GCC 4.8,
-# MSVC 2015 (Update 3), or Clang 3.1.
+# Check if the host compiler is new enough.
+# These versions are updated based on the following policy:
+#   llvm.org/docs/DeveloperPolicy.html#toolchain
 
 include(CheckCXXSourceCompiles)
 
-if(NOT DEFINED LLVM_COMPILER_CHECKED)
-  set(LLVM_COMPILER_CHECKED ON)
+set(GCC_MIN 4.8)
+set(GCC_SOFT_ERROR 5.1)
+set(CLANG_MIN 3.1)
+set(CLANG_SOFT_ERROR 3.5)
+set(APPLECLANG_MIN 3.1)
+set(APPLECLANG_SOFT_ERROR 6.0)
+set(MSVC_MIN 19.00.24213.1)
+set(MSVC_SOFT_ERROR 19.1)
 
-  if(NOT LLVM_FORCE_USE_OLD_TOOLCHAIN)
-    if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
-      if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS 4.8)
-        message(FATAL_ERROR "Host GCC version must be at least 4.8!")
-      endif()
-    elseif(CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
-      if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS 3.1)
-        message(FATAL_ERROR "Host Clang version must be at least 3.1!")
-      endif()
+# Map the above GCC versions to dates: https://gcc.gnu.org/develop.html#timeline
+set(GCC_MIN_DATE 20130322)
+set(GCC_SOFT_ERROR_DATE 20150422)
 
-      if (CMAKE_CXX_SIMULATE_ID MATCHES "MSVC")
-        if (CMAKE_CXX_SIMULATE_VERSION VERSION_LESS 19.0)
-          message(FATAL_ERROR "Host Clang must have at least -fms-compatibility-version=19.0")
-        endif()
-        set(CLANG_CL 1)
-      elseif(NOT LLVM_ENABLE_LIBCXX)
-        # Otherwise, test that we aren't using too old of a version of libstdc++
-        # with the Clang compiler. This is tricky as there is no real way to
-        # check the version of libstdc++ directly. Instead we test for a known
-        # bug in libstdc++4.6 that is fixed in libstdc++4.7.
-        set(OLD_CMAKE_REQUIRED_FLAGS ${CMAKE_REQUIRED_FLAGS})
-        set(OLD_CMAKE_REQUIRED_LIBRARIES ${CMAKE_REQUIRED_LIBRARIES})
-        set(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} -std=c++0x")
-        check_cxx_source_compiles("
-#include <atomic>
-std::atomic<float> x(0.0f);
-int main() { return (float)x; }"
-          LLVM_NO_OLD_LIBSTDCXX)
-        if(NOT LLVM_NO_OLD_LIBSTDCXX)
-          message(FATAL_ERROR "Host Clang must be able to find libstdc++4.8 or newer!")
-        endif()
-        set(CMAKE_REQUIRED_FLAGS ${OLD_CMAKE_REQUIRED_FLAGS})
-        set(CMAKE_REQUIRED_LIBRARIES ${OLD_CMAKE_REQUIRED_LIBRARIES})
-      endif()
-    elseif(CMAKE_CXX_COMPILER_ID MATCHES "MSVC")
-      if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS 19.0)
-        message(FATAL_ERROR "Host Visual Studio must be at least 2015")
-      elseif(CMAKE_CXX_COMPILER_VERSION VERSION_LESS 19.00.24213.1)
-        message(WARNING "Host Visual Studio should at least be 2015 Update 3 (MSVC 19.00.24213.1)"
-          "  due to miscompiles from earlier versions")
+
+if(DEFINED LLVM_COMPILER_CHECKED)
+  return()
+endif()
+set(LLVM_COMPILER_CHECKED ON)
+
+if(LLVM_FORCE_USE_OLD_TOOLCHAIN)
+  return()
+endif()
+
+function(check_compiler_version NAME NICE_NAME MINIMUM_VERSION SOFT_ERROR_VERSION)
+  if(NOT CMAKE_CXX_COMPILER_ID STREQUAL NAME)
+    return()
+  endif()
+  if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS MINIMUM_VERSION)
+    message(FATAL_ERROR "Host ${NICE_NAME} version must be at least ${MINIMUM_VERSION}, your version is ${CMAKE_CXX_COMPILER_VERSION}.")
+  elseif(CMAKE_CXX_COMPILER_VERSION VERSION_LESS SOFT_ERROR_VERSION)
+    if(LLVM_TEMPORARILY_ALLOW_OLD_TOOLCHAIN)
+      message(WARNING "Host ${NICE_NAME} version should be at least ${SOFT_ERROR_VERSION} because LLVM will soon use new C++ features which your toolchain version doesn't support. Your version is ${CMAKE_CXX_COMPILER_VERSION}. Ignoring because you've set LLVM_TEMPORARILY_ALLOW_OLD_TOOLCHAIN, but very soon your toolchain won't be supported.")
+    else()
+      message(FATAL_ERROR "Host ${NICE_NAME} version should be at least ${SOFT_ERROR_VERSION} because LLVM will soon use new C++ features which your toolchain version doesn't support. Your version is ${CMAKE_CXX_COMPILER_VERSION}. You can temporarily opt out using LLVM_TEMPORARILY_ALLOW_OLD_TOOLCHAIN, but very soon your toolchain won't be supported.")
+    endif()
+  endif()
+endfunction(check_compiler_version)
+
+check_compiler_version("GNU" "GCC" ${GCC_MIN} ${GCC_SOFT_ERROR})
+check_compiler_version("Clang" "Clang" ${CLANG_MIN} ${CLANG_SOFT_ERROR})
+check_compiler_version("AppleClang" "Apple Clang" ${APPLECLANG_MIN} ${APPLECLANG_SOFT_ERROR})
+check_compiler_version("MSVC" "Visual Studio" ${MSVC_MIN} ${MSVC_SOFT_ERROR})
+
+if(CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
+  if (CMAKE_CXX_SIMULATE_ID MATCHES "MSVC")
+    if (CMAKE_CXX_SIMULATE_VERSION VERSION_LESS MSVC_MIN)
+      message(FATAL_ERROR "Host Clang must have at least -fms-compatibility-version=${MSVC_MIN}, your version is ${CMAKE_CXX_COMPILER_VERSION}.")
+    endif()
+    set(CLANG_CL 1)
+  elseif(NOT LLVM_ENABLE_LIBCXX)
+    # Test that we aren't using too old of a version of libstdc++.
+    set(OLD_CMAKE_REQUIRED_FLAGS ${CMAKE_REQUIRED_FLAGS})
+    set(OLD_CMAKE_REQUIRED_LIBRARIES ${CMAKE_REQUIRED_LIBRARIES})
+    set(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} -std=c++0x")
+    check_cxx_source_compiles("
+#include <iosfwd>
+#if defined(__GLIBCXX__)
+#if __GLIBCXX__ < ${GCC_MIN_DATE}
+#error Unsupported libstdc++ version
+#endif
+#endif
+int main() { return 0; }
+"
+      LLVM_LIBSTDCXX_MIN)
+    if(NOT LLVM_LIBSTDCXX_MIN)
+      message(FATAL_ERROR "libstdc++ version must be at least ${GCC_MIN}.")
+    endif()
+    check_cxx_source_compiles("
+#include <iosfwd>
+#if defined(__GLIBCXX__)
+#if __GLIBCXX__ < ${GCC_SOFT_ERROR_DATE}
+#error Unsupported libstdc++ version
+#endif
+#endif
+int main() { return 0; }
+"
+      LLVM_LIBSTDCXX_SOFT_ERROR)
+    if(NOT LLVM_LIBSTDCXX_SOFT_ERROR)
+      if(LLVM_TEMPORARILY_ALLOW_OLD_TOOLCHAIN)
+        message(WARNING "libstdc++ version should be at least ${GCC_SOFT_ERROR} because LLVM will soon use new C++ features which your toolchain version doesn't support. Ignoring because you've set LLVM_TEMPORARILY_ALLOW_OLD_TOOLCHAIN, but very soon your toolchain won't be supported.")
+      else()
+        message(FATAL_ERROR "libstdc++ version should be at least ${GCC_SOFT_ERROR} because LLVM will soon use new C++ features which your toolchain version doesn't support. You can temporarily opt out using LLVM_TEMPORARILY_ALLOW_OLD_TOOLCHAIN, but very soon your toolchain won't be supported.")
       endif()
     endif()
+    set(CMAKE_REQUIRED_FLAGS ${OLD_CMAKE_REQUIRED_FLAGS})
+    set(CMAKE_REQUIRED_LIBRARIES ${OLD_CMAKE_REQUIRED_LIBRARIES})
   endif()
 endif()
diff --git a/cmake/modules/ChooseMSVCCRT.cmake b/cmake/modules/ChooseMSVCCRT.cmake
index cdd7deec4c84ce1790efb5c80432b73ff327e91b..0e6e1aa55254e51480ba489e54c545c7a6accc25 100644
--- a/cmake/modules/ChooseMSVCCRT.cmake
+++ b/cmake/modules/ChooseMSVCCRT.cmake
@@ -66,15 +66,6 @@ variables (LLVM_USE_CRT_DEBUG, etc) instead.")
       get_current_crt(LLVM_USE_CRT_${build}
         MSVC_CRT_REGEX
         CMAKE_CXX_FLAGS_${build})
-
-      # Make /MT the default in Release builds to make them faster
-      # and avoid the DLL function thunking.
-      if ((${build} STREQUAL "MINSIZEREL") OR
-          (${build} STREQUAL "RELEASE") OR
-          (${build} STREQUAL "RELWITHDEBINFO"))
-          set(LLVM_USE_CRT_${build} "MT")
-      endif()
-
       set(LLVM_USE_CRT_${build}
         "${LLVM_USE_CRT_${build}}"
         CACHE STRING "Specify VC++ CRT to use for ${build_type} configurations."
diff --git a/cmake/modules/CrossCompile.cmake b/cmake/modules/CrossCompile.cmake
index b239816c8253928446e2d4bcf9fe205433211dd5..bc3b210f01859d3a6aec3106aaa5fa476115fc4d 100644
--- a/cmake/modules/CrossCompile.cmake
+++ b/cmake/modules/CrossCompile.cmake
@@ -52,6 +52,7 @@ function(llvm_create_cross_target_internal target_name toolchain buildtype)
         -DLLVM_EXPERIMENTAL_TARGETS_TO_BUILD="${experimental_targets_to_build_arg}"
         -DLLVM_DEFAULT_TARGET_TRIPLE="${TARGET_TRIPLE}"
         -DLLVM_TARGET_ARCH="${LLVM_TARGET_ARCH}"
+        -DLLVM_TEMPORARILY_ALLOW_OLD_TOOLCHAIN="${LLVM_TEMPORARILY_ALLOW_OLD_TOOLCHAIN}"
         ${build_type_flags} ${linker_flag} ${external_clang_dir}
     WORKING_DIRECTORY ${LLVM_${target_name}_BUILD}
     DEPENDS CREATE_LLVM_${target_name}
diff --git a/cmake/modules/GenerateVersionFromCVS.cmake b/cmake/modules/GenerateVersionFromCVS.cmake
deleted file mode 100644
index 6b1c719834667d4c7f1338d947d1e8c03e76ae38..0000000000000000000000000000000000000000
--- a/cmake/modules/GenerateVersionFromCVS.cmake
+++ /dev/null
@@ -1,39 +0,0 @@
-# CMake project that writes Subversion revision information to a header.
-#
-# Input variables:
-#   SRC               - Source directory
-#   HEADER_FILE       - The header file to write
-#
-# The output header will contain macros FIRST_REPOSITORY and FIRST_REVISION,
-# and SECOND_REPOSITORY and SECOND_REVISION if requested, where "FIRST" and
-# "SECOND" are substituted with the names specified in the input variables.
-
-
-
-# Chop off cmake/modules/GetSVN.cmake
-get_filename_component(LLVM_DIR "${CMAKE_SCRIPT_MODE_FILE}" PATH)
-get_filename_component(LLVM_DIR "${LLVM_DIR}" PATH)
-get_filename_component(LLVM_DIR "${LLVM_DIR}" PATH)
-
-set(CMAKE_MODULE_PATH
-  ${CMAKE_MODULE_PATH}
-  "${LLVM_DIR}/cmake/modules")
-include(VersionFromVCS)
-
-# Handle strange terminals
-set(ENV{TERM} "dumb")
-
-function(append_info name path)
-  add_version_info_from_vcs(REVISION ${path})
-  string(STRIP "${REVISION}" REVISION)
-  file(APPEND "${HEADER_FILE}.txt"
-    "#define ${name} \"${REVISION}\"\n")
-endfunction()
-
-append_info(${NAME} "${SOURCE_DIR}")
-
-# Copy the file only if it has changed.
-execute_process(COMMAND ${CMAKE_COMMAND} -E copy_if_different
-  "${HEADER_FILE}.txt" "${HEADER_FILE}")
-file(REMOVE "${HEADER_FILE}.txt")
-
diff --git a/cmake/modules/GenerateVersionFromVCS.cmake b/cmake/modules/GenerateVersionFromVCS.cmake
new file mode 100644
index 0000000000000000000000000000000000000000..d8ec54df41ebd5eb1e02707b74050209e592142b
--- /dev/null
+++ b/cmake/modules/GenerateVersionFromVCS.cmake
@@ -0,0 +1,51 @@
+# CMake script that writes version control information to a header.
+#
+# Input variables:
+#   NAMES             - A list of names for each of the source directories.
+#   <NAME>_SOURCE_DIR - A path to source directory for each name in NAMES.
+#   HEADER_FILE       - The header file to write
+#
+# The output header will contain macros <NAME>_REPOSITORY and <NAME>_REVISION,
+# where "<NAME>" is substituted with the names specified in the input variables,
+# for each of the <NAME>_SOURCE_DIR given.
+
+get_filename_component(LLVM_CMAKE_DIR "${CMAKE_SCRIPT_MODE_FILE}" PATH)
+
+list(APPEND CMAKE_MODULE_PATH "${LLVM_CMAKE_DIR}")
+
+include(VersionFromVCS)
+
+# Handle strange terminals
+set(ENV{TERM} "dumb")
+
+function(append_info name path)
+  if(path)
+    get_source_info("${path}" revision repository)
+  endif()
+  if(revision)
+    file(APPEND "${HEADER_FILE}.tmp"
+      "#define ${name}_REVISION \"${revision}\"\n")
+  else()
+    file(APPEND "${HEADER_FILE}.tmp"
+      "#undef ${name}_REVISION\n")
+  endif()
+  if(repository)
+    file(APPEND "${HEADER_FILE}.tmp"
+      "#define ${name}_REPOSITORY \"${repository}\"\n")
+  else()
+    file(APPEND "${HEADER_FILE}.tmp"
+      "#undef ${name}_REPOSITORY\n")
+  endif()
+endfunction()
+
+foreach(name IN LISTS NAMES)
+  if(NOT DEFINED ${name}_SOURCE_DIR)
+    message(FATAL_ERROR "${name}_SOURCE_DIR is not defined")
+  endif()
+  append_info(${name} "${${name}_SOURCE_DIR}")
+endforeach()
+
+# Copy the file only if it has changed.
+execute_process(COMMAND ${CMAKE_COMMAND} -E copy_if_different
+  "${HEADER_FILE}.tmp" "${HEADER_FILE}")
+file(REMOVE "${HEADER_FILE}.tmp")
diff --git a/cmake/modules/GetSVN.cmake b/cmake/modules/GetSVN.cmake
deleted file mode 100644
index f729395f6e4bab54317fdb94e7c844f34f0bbe5a..0000000000000000000000000000000000000000
--- a/cmake/modules/GetSVN.cmake
+++ /dev/null
@@ -1,141 +0,0 @@
-# CMake project that writes Subversion revision information to a header.
-#
-# Input variables:
-#   SOURCE_DIRS - A list of source directories.
-#   NAMES       - A list of macro prefixes for each of the source directories.
-#   HEADER_FILE - The header file to write
-#
-# The output header will contain macros <NAME>_REPOSITORY and <NAME>_REVISION,
-# where "<NAME>" and is substituted with the names specified in the input
-# variables, for each of the SOURCE_DIRS given.
-
-# Chop off cmake/modules/GetSVN.cmake
-get_filename_component(LLVM_DIR "${CMAKE_SCRIPT_MODE_FILE}" PATH)
-get_filename_component(LLVM_DIR "${LLVM_DIR}" PATH)
-get_filename_component(LLVM_DIR "${LLVM_DIR}" PATH)
-
-# Handle strange terminals
-set(ENV{TERM} "dumb")
-
-macro(get_source_info_svn path revision repository)
-  # If svn is a bat file, find_program(Subversion) doesn't find it.
-  # Explicitly search for that here; Subversion_SVN_EXECUTABLE will override
-  # the find_program call in FindSubversion.cmake.
-  find_program(Subversion_SVN_EXECUTABLE NAMES svn svn.bat)
-
-  # FindSubversion does not work with symlinks. See PR 8437
-  if (NOT IS_SYMLINK "${path}")
-    find_package(Subversion)
-  endif()
-  if (Subversion_FOUND)
-    subversion_wc_info( ${path} Project )
-    if (Project_WC_REVISION)
-      set(${revision} ${Project_WC_REVISION} PARENT_SCOPE)
-    endif()
-    if (Project_WC_URL)
-      set(${repository} ${Project_WC_URL} PARENT_SCOPE)
-    endif()
-  endif()
-endmacro()
-
-macro(get_source_info_git_svn path revision repository)
-  find_program(git_executable NAMES git git.exe git.cmd)
-  if (git_executable)
-    execute_process(COMMAND ${git_executable} svn info
-      WORKING_DIRECTORY ${path}
-      TIMEOUT 5
-      RESULT_VARIABLE git_result
-      OUTPUT_VARIABLE git_output)
-    if (git_result EQUAL 0)
-      string(REGEX REPLACE "^(.*\n)?Revision: ([^\n]+).*"
-        "\\2" git_svn_rev "${git_output}")
-      set(${revision} ${git_svn_rev} PARENT_SCOPE)
-      string(REGEX REPLACE "^(.*\n)?URL: ([^\n]+).*"
-        "\\2" git_url "${git_output}")
-      set(${repository} ${git_url} PARENT_SCOPE)
-    endif()
-  endif()
-endmacro()
-
-macro(get_source_info_git path revision repository)
-  find_program(git_executable NAMES git git.exe git.cmd)
-  if (git_executable)
-    execute_process(COMMAND ${git_executable} log -1 --pretty=format:%H
-      WORKING_DIRECTORY ${path}
-      TIMEOUT 5
-      RESULT_VARIABLE git_result
-      OUTPUT_VARIABLE git_output)
-    if (git_result EQUAL 0)
-      set(${revision} ${git_output} PARENT_SCOPE)
-    endif()
-    execute_process(COMMAND ${git_executable} remote -v
-      WORKING_DIRECTORY ${path}
-      TIMEOUT 5
-      RESULT_VARIABLE git_result
-      OUTPUT_VARIABLE git_output)
-    if (git_result EQUAL 0)
-      string(REGEX REPLACE "^(.*\n)?[^ \t]+[ \t]+([^ \t\n]+)[ \t]+\\(fetch\\).*"
-        "\\2" git_url "${git_output}")
-      set(${repository} "${git_url}" PARENT_SCOPE)
-    endif()
-  endif()
-endmacro()
-
-function(get_source_info path revision repository)
-  if (EXISTS "${path}/.svn")
-    get_source_info_svn("${path}" revision repository)
-  elseif (EXISTS "${path}/.git/svn/refs")
-    get_source_info_git_svn("${path}" revision repository)
-  elseif (EXISTS "${path}/.git")
-    get_source_info_git("${path}" revision repository)
-  endif()
-endfunction()
-
-function(append_info name path)
-  get_source_info("${path}" revision repository)
-  string(STRIP "${revision}" revision)
-  string(STRIP "${repository}" repository)
-  file(APPEND "${HEADER_FILE}.txt"
-    "#define ${name}_REVISION \"${revision}\"\n")
-  file(APPEND "${HEADER_FILE}.txt"
-    "#define ${name}_REPOSITORY \"${repository}\"\n")
-endfunction()
-
-function(validate_inputs source_dirs names)
-  list(LENGTH source_dirs source_dirs_length)
-  list(LENGTH names names_length)
-  if (NOT source_dirs_length EQUAL names_length)
-    message(FATAL_ERROR
-            "GetSVN.cmake takes two arguments: a list of source directories, "
-            "and a list of names. Expected two lists must be of equal length, "
-            "but got ${source_dirs_length} source directories and "
-            "${names_length} names.")
-  endif()
-endfunction()
-
-if (DEFINED SOURCE_DIRS AND DEFINED NAMES)
-  validate_inputs("${SOURCE_DIRS}" "${NAMES}")
-
-  list(LENGTH SOURCE_DIRS source_dirs_length)
-  math(EXPR source_dirs_max_index ${source_dirs_length}-1)
-  foreach(index RANGE ${source_dirs_max_index})
-    list(GET SOURCE_DIRS ${index} source_dir)
-    list(GET NAMES ${index} name)
-    append_info(${name} ${source_dir})
-  endforeach()
-endif()
-
-# Allow -DFIRST_SOURCE_DIR arguments until Clang migrates to the new
-# -DSOURCE_DIRS argument.
-if(DEFINED FIRST_SOURCE_DIR)
-  append_info(${FIRST_NAME} "${FIRST_SOURCE_DIR}")
-  if(DEFINED SECOND_SOURCE_DIR)
-    append_info(${SECOND_NAME} "${SECOND_SOURCE_DIR}")
-  endif()
-endif()
-
-# Copy the file only if it has changed.
-execute_process(COMMAND ${CMAKE_COMMAND} -E copy_if_different
-  "${HEADER_FILE}.txt" "${HEADER_FILE}")
-file(REMOVE "${HEADER_FILE}.txt")
-
diff --git a/cmake/modules/HandleLLVMOptions.cmake b/cmake/modules/HandleLLVMOptions.cmake
index e8b0e27e000d2414a19ca2f481dfb23cae0e02b4..ca53f7f38ee06a16690023652c15811048554f6f 100644
--- a/cmake/modules/HandleLLVMOptions.cmake
+++ b/cmake/modules/HandleLLVMOptions.cmake
@@ -381,14 +381,6 @@ if( MSVC )
   # "Enforce type conversion rules".
   append("/Zc:rvalueCast" CMAKE_CXX_FLAGS)
 
-  if (CMAKE_CXX_COMPILER_ID MATCHES "MSVC" AND NOT LLVM_ENABLE_INCREMENTAL_LINK)
-    foreach(CONFIG RELEASE RELWITHDEBINFO MINSIZEREL)
-      foreach(FLAG EXE MODULE SHARED STATIC)
-        string(REGEX REPLACE "[-/](INCREMENTAL:YES|INCREMENTAL:NO|INCREMENTAL)" "/INCREMENTAL:NO" CMAKE_${FLAG}_LINKER_FLAGS_${CONFIG} "${CMAKE_${FLAG}_LINKER_FLAGS_${CONFIG}}")
-      endforeach()
-    endforeach()
-  endif()
-
   if (CMAKE_CXX_COMPILER_ID MATCHES "Clang" AND NOT LLVM_ENABLE_LTO)
     # clang-cl and cl by default produce non-deterministic binaries because
     # link.exe /incremental requires a timestamp in the .obj file.  clang-cl
@@ -792,6 +784,16 @@ if(LLVM_ENABLE_EH AND NOT LLVM_ENABLE_RTTI)
   message(FATAL_ERROR "Exception handling requires RTTI. You must set LLVM_ENABLE_RTTI to ON")
 endif()
 
+option(LLVM_USE_NEWPM "Build LLVM using the experimental new pass manager" Off)
+mark_as_advanced(LLVM_USE_NEWPM)
+if (LLVM_USE_NEWPM)
+  append("-fexperimental-new-pass-manager"
+    CMAKE_CXX_FLAGS
+    CMAKE_C_FLAGS
+    CMAKE_EXE_LINKER_FLAGS
+    CMAKE_SHARED_LINKER_FLAGS)
+endif()
+
 option(LLVM_ENABLE_IR_PGO "Build LLVM and tools with IR PGO instrumentation (deprecated)" Off)
 mark_as_advanced(LLVM_ENABLE_IR_PGO)
 
diff --git a/cmake/modules/LLVMConfig.cmake.in b/cmake/modules/LLVMConfig.cmake.in
index 7ca06381d90f56d14c06b8edcdb08e16ad3f6ea0..c3c3af37f6208d7fd59747f792a3738ef475749c 100644
--- a/cmake/modules/LLVMConfig.cmake.in
+++ b/cmake/modules/LLVMConfig.cmake.in
@@ -67,10 +67,12 @@ set(LLVM_LIBDIR_SUFFIX @LLVM_LIBDIR_SUFFIX@)
 set(LLVM_INCLUDE_DIRS "@LLVM_CONFIG_INCLUDE_DIRS@")
 set(LLVM_LIBRARY_DIRS "@LLVM_CONFIG_LIBRARY_DIRS@")
 
-# These variables are duplicated, but they must match the LLVM variables of the
-# same name. The variables ending in "S" could some day become lists, and are
-# preserved for convention and compatibility.
-set(LLVM_INCLUDE_DIR "@LLVM_CONFIG_INCLUDE_DIRS@")
+# These variables are duplicated for install tree but they have different
+# values for build tree.  LLVM_INCLUDE_DIRS contains both source
+# and generated include directories while the following variables have
+# them split.
+set(LLVM_INCLUDE_DIR "@LLVM_CONFIG_INCLUDE_DIR@")
+set(LLVM_MAIN_INCLUDE_DIR "@LLVM_CONFIG_MAIN_INCLUDE_DIR@")
 set(LLVM_LIBRARY_DIR "@LLVM_CONFIG_LIBRARY_DIRS@")
 
 set(LLVM_DEFINITIONS "@LLVM_DEFINITIONS@")
diff --git a/cmake/modules/TableGen.cmake b/cmake/modules/TableGen.cmake
index 2d942435222ee472d75b4e535c147ae4510e255b..3c84ae78a3497449946d0dd55825248b97b5e818 100644
--- a/cmake/modules/TableGen.cmake
+++ b/cmake/modules/TableGen.cmake
@@ -25,7 +25,7 @@ function(tablegen project ofn)
     file(RELATIVE_PATH ofn_rel
       ${CMAKE_BINARY_DIR} ${CMAKE_CURRENT_BINARY_DIR}/${ofn})
     set(additional_cmdline
-      -o ${ofn_rel}.tmp
+      -o ${ofn_rel}
       -d ${ofn_rel}.d
       WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
       DEPFILE ${CMAKE_CURRENT_BINARY_DIR}/${ofn}.d
@@ -36,7 +36,7 @@ function(tablegen project ofn)
     file(GLOB local_tds "*.td")
     file(GLOB_RECURSE global_tds "${LLVM_MAIN_INCLUDE_DIR}/llvm/*.td")
     set(additional_cmdline
-      -o ${CMAKE_CURRENT_BINARY_DIR}/${ofn}.tmp
+      -o ${CMAKE_CURRENT_BINARY_DIR}/${ofn}
       )
   endif()
 
@@ -69,8 +69,7 @@ function(tablegen project ofn)
   # dependency twice in the result file when
   # ("${${project}_TABLEGEN_TARGET}" STREQUAL "${${project}_TABLEGEN_EXE}")
   # but lets us having smaller and cleaner code here.
-  add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${ofn}.tmp
-    # Generate tablegen output in a temporary file.
+  add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${ofn}
     COMMAND ${${project}_TABLEGEN_EXE} ${ARGN} -I ${CMAKE_CURRENT_SOURCE_DIR}
     ${LLVM_TABLEGEN_FLAGS}
     ${LLVM_TARGET_DEFINITIONS_ABSOLUTE}
@@ -83,20 +82,9 @@ function(tablegen project ofn)
     ${LLVM_TARGET_DEFINITIONS_ABSOLUTE}
     COMMENT "Building ${ofn}..."
     )
-  add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${ofn}
-    # Only update the real output file if there are any differences.
-    # This prevents recompilation of all the files depending on it if there
-    # aren't any.
-    COMMAND ${CMAKE_COMMAND} -E copy_if_different
-        ${CMAKE_CURRENT_BINARY_DIR}/${ofn}.tmp
-        ${CMAKE_CURRENT_BINARY_DIR}/${ofn}
-    DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${ofn}.tmp
-    COMMENT "Updating ${ofn}..."
-    )
 
   # `make clean' must remove all those generated files:
-  set_property(DIRECTORY APPEND
-    PROPERTY ADDITIONAL_MAKE_CLEAN_FILES ${ofn}.tmp ${ofn})
+  set_property(DIRECTORY APPEND PROPERTY ADDITIONAL_MAKE_CLEAN_FILES ${ofn})
 
   set(TABLEGEN_OUTPUT ${TABLEGEN_OUTPUT} ${CMAKE_CURRENT_BINARY_DIR}/${ofn} PARENT_SCOPE)
   set_source_files_properties(${CMAKE_CURRENT_BINARY_DIR}/${ofn} PROPERTIES
diff --git a/cmake/modules/VersionFromVCS.cmake b/cmake/modules/VersionFromVCS.cmake
index 552fe77cdfb6890d43b34ff0fc478e29577951b7..56331a3a81fceb6dbb3c935ce0a5fe8bcb6a2648 100644
--- a/cmake/modules/VersionFromVCS.cmake
+++ b/cmake/modules/VersionFromVCS.cmake
@@ -3,90 +3,92 @@
 # existence of certain subdirectories under SOURCE_DIR (if provided as an
 # extra argument, otherwise uses CMAKE_CURRENT_SOURCE_DIR).
 
-function(add_version_info_from_vcs VERS)
-  SET(SOURCE_DIR ${ARGV1})
-  if("${SOURCE_DIR}" STREQUAL "")
-      SET(SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR})
-  endif()
-  string(REPLACE "svn" "" result "${${VERS}}")
-  if( EXISTS "${SOURCE_DIR}/.svn" )
-    set(result "${result}svn")
-    # FindSubversion does not work with symlinks. See PR 8437
-    if( NOT IS_SYMLINK "${SOURCE_DIR}" )
-      find_package(Subversion)
+function(get_source_info_svn path revision repository)
+  # If svn is a bat file, find_program(Subversion) doesn't find it.
+  # Explicitly search for that here; Subversion_SVN_EXECUTABLE will override
+  # the find_program call in FindSubversion.cmake.
+  find_program(Subversion_SVN_EXECUTABLE NAMES svn svn.bat)
+  find_package(Subversion)
+
+  # Subversion module does not work with symlinks, see PR8437.
+  get_filename_component(realpath ${path} REALPATH)
+  if(Subversion_FOUND)
+    subversion_wc_info(${realpath} Project)
+    if(Project_WC_REVISION)
+      set(${revision} ${Project_WC_REVISION} PARENT_SCOPE)
     endif()
-    if( Subversion_FOUND )
-      subversion_wc_info( ${SOURCE_DIR} Project )
-      if( Project_WC_REVISION )
-        set(SVN_REVISION ${Project_WC_REVISION} PARENT_SCOPE)
-        set(result "${result}-r${Project_WC_REVISION}")
-      endif()
-      if( Project_WC_URL )
-        set(LLVM_REPOSITORY ${Project_WC_URL} PARENT_SCOPE)
-      endif()
+    if(Project_WC_URL)
+      set(${repository} ${Project_WC_URL} PARENT_SCOPE)
     endif()
-  else()
-    find_program(git_executable NAMES git git.exe git.cmd)
-
-    if( git_executable )
-      # Run from a subdirectory to force git to print an absoute path.
-      execute_process(COMMAND ${git_executable} rev-parse --git-dir
-        WORKING_DIRECTORY ${SOURCE_DIR}/cmake
-        RESULT_VARIABLE git_result
-        OUTPUT_VARIABLE git_dir
-        ERROR_QUIET)
-      if(git_result EQUAL 0)
-        # Try to get a ref-id
-        string(STRIP "${git_dir}" git_dir)
-        set(result "${result}git")
-        if( EXISTS ${git_dir}/svn )
-          # Get the repository URL
-          execute_process(COMMAND
-            ${git_executable} svn info
-            WORKING_DIRECTORY ${SOURCE_DIR}
-            TIMEOUT 5
-            RESULT_VARIABLE git_result
-            OUTPUT_VARIABLE git_output
-            ERROR_QUIET)
-          if( git_result EQUAL 0 )
-            string(REGEX MATCH "URL: ([^ \n]*)" svn_url ${git_output})
-            if(svn_url)
-              set(LLVM_REPOSITORY ${CMAKE_MATCH_1} PARENT_SCOPE)
-            endif()
-          endif()
+  endif()
+endfunction()
 
-          # Get the svn revision number for this git commit if one exists.
-          execute_process(COMMAND ${git_executable} svn find-rev HEAD
-            WORKING_DIRECTORY ${SOURCE_DIR}
-            TIMEOUT 5
-            RESULT_VARIABLE git_result
-            OUTPUT_VARIABLE git_head_svn_rev_number
-            OUTPUT_STRIP_TRAILING_WHITESPACE)
-          if( git_result EQUAL 0 AND git_output)
-            set(SVN_REVISION ${git_head_svn_rev_number} PARENT_SCOPE)
-            set(git_svn_rev "-svn-${git_head_svn_rev_number}")
-          else()
-            set(git_svn_rev "")
-          endif()
+function(get_source_info_git path revision repository)
+  find_package(Git)
+  if(GIT_FOUND)
+    execute_process(COMMAND ${GIT_EXECUTABLE} rev-parse --git-dir
+      WORKING_DIRECTORY ${path}
+      RESULT_VARIABLE git_result
+      OUTPUT_VARIABLE git_output
+      ERROR_QUIET)
+    if(git_result EQUAL 0)
+      string(STRIP "${git_output}" git_output)
+      get_filename_component(git_dir ${git_output} ABSOLUTE BASE_DIR ${path})
+      if(EXISTS "${git_dir}/svn/refs")
+        execute_process(COMMAND ${GIT_EXECUTABLE} svn info
+          WORKING_DIRECTORY ${path}
+          RESULT_VARIABLE git_result
+          OUTPUT_VARIABLE git_output)
+        if(git_result EQUAL 0)
+          string(REGEX REPLACE "^(.*\n)?Revision: ([^\n]+).*"
+            "\\2" git_svn_rev "${git_output}")
+          set(${revision} ${git_svn_rev} PARENT_SCOPE)
+          string(REGEX REPLACE "^(.*\n)?URL: ([^\n]+).*"
+            "\\2" git_url "${git_output}")
+          set(${repository} ${git_url} PARENT_SCOPE)
         endif()
-
-        # Get the git ref id
-        execute_process(COMMAND
-          ${git_executable} rev-parse --short HEAD
-          WORKING_DIRECTORY ${SOURCE_DIR}
-          TIMEOUT 5
+      else()
+        execute_process(COMMAND ${GIT_EXECUTABLE} rev-parse HEAD
+          WORKING_DIRECTORY ${path}
           RESULT_VARIABLE git_result
-          OUTPUT_VARIABLE git_ref_id
-          OUTPUT_STRIP_TRAILING_WHITESPACE)
-
-        if( git_result EQUAL 0 )
-          set(GIT_COMMIT ${git_ref_id} PARENT_SCOPE)
-          set(result "${result}${git_svn_rev}-${git_ref_id}")
+          OUTPUT_VARIABLE git_output)
+        if(git_result EQUAL 0)
+          string(STRIP "${git_output}" git_output)
+          set(${revision} ${git_output} PARENT_SCOPE)
+        endif()
+        execute_process(COMMAND ${GIT_EXECUTABLE} rev-parse --abbrev-ref --symbolic-full-name @{upstream}
+          WORKING_DIRECTORY ${path}
+          RESULT_VARIABLE git_result
+          OUTPUT_VARIABLE git_output
+          ERROR_QUIET)
+        if(git_result EQUAL 0)
+          string(REPLACE "/" ";" branch ${git_output})
+          list(GET branch 0 remote)
         else()
-          set(result "${result}${git_svn_rev}")
+          set(remote "origin")
+        endif()
+        execute_process(COMMAND ${GIT_EXECUTABLE} remote get-url ${remote}
+          WORKING_DIRECTORY ${path}
+          RESULT_VARIABLE git_result
+          OUTPUT_VARIABLE git_output
+          ERROR_QUIET)
+        if(git_result EQUAL 0)
+          string(STRIP "${git_output}" git_output)
+          set(${repository} ${git_output} PARENT_SCOPE)
+        else()
+          set(${repository} ${path} PARENT_SCOPE)
         endif()
       endif()
     endif()
   endif()
-  set(${VERS} ${result} PARENT_SCOPE)
-endfunction(add_version_info_from_vcs)
+endfunction()
+
+function(get_source_info path revision repository)
+  if(EXISTS "${path}/.svn")
+    get_source_info_svn("${path}" revision_info repository_info)
+  else()
+    get_source_info_git("${path}" revision_info repository_info)
+  endif()
+  set(${repository} "${repository_info}" PARENT_SCOPE)
+  set(${revision} "${revision_info}" PARENT_SCOPE)
+endfunction()
diff --git a/cmake/platforms/WinMsvc.cmake b/cmake/platforms/WinMsvc.cmake
index f625d0e3c052412734b4f56af6722bada1ef4df3..d038df9795131fb248e63859e48735d6159524bc 100644
--- a/cmake/platforms/WinMsvc.cmake
+++ b/cmake/platforms/WinMsvc.cmake
@@ -92,7 +92,9 @@
 # won't see the value of any arguments the user passed via -D.  Since these are
 # necessary to properly configure MSVC in both the top-level configuration as well as
 # all feature-test invocations, we set environment variables with the values so that
-# these environments get inherited by child invocations.
+# these environments get inherited by child invocations. We can switch to
+# CMAKE_TRY_COMPILE_PLATFORM_VARIABLES once our minimum supported CMake version
+# is 3.6 or greater.
 function(init_user_prop prop)
   if(${prop})
     set(ENV{_${prop}} "${${prop}}")
diff --git a/docs/AMDGPU/AMDGPUAsmGFX7.rst b/docs/AMDGPU/AMDGPUAsmGFX7.rst
index 58c13b5556b436293f03db6b4c76a20e65ea5e4b..b2671498ee00837aa40ff72aa58af4e2f43a1377 100644
--- a/docs/AMDGPU/AMDGPUAsmGFX7.rst
+++ b/docs/AMDGPU/AMDGPUAsmGFX7.rst
@@ -32,146 +32,146 @@ DS
 .. parsed-literal::
 
     **INSTRUCTION**                    **DST**         **SRC0**      **SRC1**      **SRC2**           **MODIFIERS**
-    \ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|
-    ds_add_rtn_u32                 :ref:`vdst<amdgpu_synid7_vdst32_0>`,       :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata32_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_add_rtn_u64                 :ref:`vdst<amdgpu_synid7_vdst64_0>`,       :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata64_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_add_src2_u32                            :ref:`vaddr<amdgpu_synid7_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_add_src2_u64                            :ref:`vaddr<amdgpu_synid7_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_add_u32                                 :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata32_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_add_u64                                 :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata64_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_and_b32                                 :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata32_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_and_b64                                 :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata64_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_and_rtn_b32                 :ref:`vdst<amdgpu_synid7_vdst32_0>`,       :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata32_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_and_rtn_b64                 :ref:`vdst<amdgpu_synid7_vdst64_0>`,       :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata64_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_and_src2_b32                            :ref:`vaddr<amdgpu_synid7_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_and_src2_b64                            :ref:`vaddr<amdgpu_synid7_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_append                      :ref:`vdst<amdgpu_synid7_vdst32_0>`                                           :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_cmpst_b32                               :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata0<amdgpu_synid7_vdata32_0>`,   :ref:`vdata1<amdgpu_synid7_vdata32_0>`         :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_cmpst_b64                               :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata0<amdgpu_synid7_vdata64_0>`,   :ref:`vdata1<amdgpu_synid7_vdata64_0>`         :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_cmpst_f32                               :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata0<amdgpu_synid7_vdata32_0>`,   :ref:`vdata1<amdgpu_synid7_vdata32_0>`         :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_cmpst_f64                               :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata0<amdgpu_synid7_vdata64_0>`,   :ref:`vdata1<amdgpu_synid7_vdata64_0>`         :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_cmpst_rtn_b32               :ref:`vdst<amdgpu_synid7_vdst32_0>`,       :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata0<amdgpu_synid7_vdata32_0>`,   :ref:`vdata1<amdgpu_synid7_vdata32_0>`         :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_cmpst_rtn_b64               :ref:`vdst<amdgpu_synid7_vdst64_0>`,       :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata0<amdgpu_synid7_vdata64_0>`,   :ref:`vdata1<amdgpu_synid7_vdata64_0>`         :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_cmpst_rtn_f32               :ref:`vdst<amdgpu_synid7_vdst32_0>`,       :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata0<amdgpu_synid7_vdata32_0>`,   :ref:`vdata1<amdgpu_synid7_vdata32_0>`         :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_cmpst_rtn_f64               :ref:`vdst<amdgpu_synid7_vdst64_0>`,       :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata0<amdgpu_synid7_vdata64_0>`,   :ref:`vdata1<amdgpu_synid7_vdata64_0>`         :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_condxchg32_rtn_b64          :ref:`vdst<amdgpu_synid7_vdst64_0>`,       :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata64_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_consume                     :ref:`vdst<amdgpu_synid7_vdst32_0>`                                           :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_dec_rtn_u32                 :ref:`vdst<amdgpu_synid7_vdst32_0>`,       :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata32_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_dec_rtn_u64                 :ref:`vdst<amdgpu_synid7_vdst64_0>`,       :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata64_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_dec_src2_u32                            :ref:`vaddr<amdgpu_synid7_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_dec_src2_u64                            :ref:`vaddr<amdgpu_synid7_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_dec_u32                                 :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata32_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_dec_u64                                 :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata64_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_gws_barrier                             :ref:`vdata<amdgpu_synid7_vdata32_0>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_gws_init                                :ref:`vdata<amdgpu_synid7_vdata32_0>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_gws_sema_br                             :ref:`vdata<amdgpu_synid7_vdata32_0>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_gws_sema_p                                                                 :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_gws_sema_release_all                                                       :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_gws_sema_v                                                                 :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_inc_rtn_u32                 :ref:`vdst<amdgpu_synid7_vdst32_0>`,       :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata32_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_inc_rtn_u64                 :ref:`vdst<amdgpu_synid7_vdst64_0>`,       :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata64_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_inc_src2_u32                            :ref:`vaddr<amdgpu_synid7_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_inc_src2_u64                            :ref:`vaddr<amdgpu_synid7_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_inc_u32                                 :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata32_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_inc_u64                                 :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata64_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_max_f32                                 :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata32_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_max_f64                                 :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata64_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_max_i32                                 :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata32_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_max_i64                                 :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata64_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_max_rtn_f32                 :ref:`vdst<amdgpu_synid7_vdst32_0>`,       :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata32_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_max_rtn_f64                 :ref:`vdst<amdgpu_synid7_vdst64_0>`,       :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata64_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_max_rtn_i32                 :ref:`vdst<amdgpu_synid7_vdst32_0>`,       :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata32_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_max_rtn_i64                 :ref:`vdst<amdgpu_synid7_vdst64_0>`,       :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata64_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_max_rtn_u32                 :ref:`vdst<amdgpu_synid7_vdst32_0>`,       :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata32_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_max_rtn_u64                 :ref:`vdst<amdgpu_synid7_vdst64_0>`,       :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata64_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_max_src2_f32                            :ref:`vaddr<amdgpu_synid7_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_max_src2_f64                            :ref:`vaddr<amdgpu_synid7_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_max_src2_i32                            :ref:`vaddr<amdgpu_synid7_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_max_src2_i64                            :ref:`vaddr<amdgpu_synid7_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_max_src2_u32                            :ref:`vaddr<amdgpu_synid7_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_max_src2_u64                            :ref:`vaddr<amdgpu_synid7_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_max_u32                                 :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata32_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_max_u64                                 :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata64_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_min_f32                                 :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata32_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_min_f64                                 :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata64_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_min_i32                                 :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata32_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_min_i64                                 :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata64_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_min_rtn_f32                 :ref:`vdst<amdgpu_synid7_vdst32_0>`,       :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata32_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_min_rtn_f64                 :ref:`vdst<amdgpu_synid7_vdst64_0>`,       :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata64_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_min_rtn_i32                 :ref:`vdst<amdgpu_synid7_vdst32_0>`,       :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata32_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_min_rtn_i64                 :ref:`vdst<amdgpu_synid7_vdst64_0>`,       :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata64_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_min_rtn_u32                 :ref:`vdst<amdgpu_synid7_vdst32_0>`,       :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata32_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_min_rtn_u64                 :ref:`vdst<amdgpu_synid7_vdst64_0>`,       :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata64_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_min_src2_f32                            :ref:`vaddr<amdgpu_synid7_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_min_src2_f64                            :ref:`vaddr<amdgpu_synid7_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_min_src2_i32                            :ref:`vaddr<amdgpu_synid7_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_min_src2_i64                            :ref:`vaddr<amdgpu_synid7_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_min_src2_u32                            :ref:`vaddr<amdgpu_synid7_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_min_src2_u64                            :ref:`vaddr<amdgpu_synid7_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_min_u32                                 :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata32_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_min_u64                                 :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata64_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_mskor_b32                               :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata0<amdgpu_synid7_vdata32_0>`,   :ref:`vdata1<amdgpu_synid7_vdata32_0>`         :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_mskor_b64                               :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata0<amdgpu_synid7_vdata64_0>`,   :ref:`vdata1<amdgpu_synid7_vdata64_0>`         :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_mskor_rtn_b32               :ref:`vdst<amdgpu_synid7_vdst32_0>`,       :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata0<amdgpu_synid7_vdata32_0>`,   :ref:`vdata1<amdgpu_synid7_vdata32_0>`         :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_mskor_rtn_b64               :ref:`vdst<amdgpu_synid7_vdst64_0>`,       :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata0<amdgpu_synid7_vdata64_0>`,   :ref:`vdata1<amdgpu_synid7_vdata64_0>`         :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    \ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|
+    ds_add_rtn_u32                 :ref:`vdst<amdgpu_synid7_vdst32_0>`,       :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata32_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_add_rtn_u64                 :ref:`vdst<amdgpu_synid7_vdst64_0>`,       :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata64_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_add_src2_u32                            :ref:`vaddr<amdgpu_synid7_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_add_src2_u64                            :ref:`vaddr<amdgpu_synid7_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_add_u32                                 :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata32_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_add_u64                                 :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata64_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_and_b32                                 :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata32_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_and_b64                                 :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata64_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_and_rtn_b32                 :ref:`vdst<amdgpu_synid7_vdst32_0>`,       :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata32_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_and_rtn_b64                 :ref:`vdst<amdgpu_synid7_vdst64_0>`,       :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata64_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_and_src2_b32                            :ref:`vaddr<amdgpu_synid7_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_and_src2_b64                            :ref:`vaddr<amdgpu_synid7_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_append                      :ref:`vdst<amdgpu_synid7_vdst32_0>`                                           :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_cmpst_b32                               :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata0<amdgpu_synid7_vdata32_0>`,   :ref:`vdata1<amdgpu_synid7_vdata32_0>`         :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_cmpst_b64                               :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata0<amdgpu_synid7_vdata64_0>`,   :ref:`vdata1<amdgpu_synid7_vdata64_0>`         :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_cmpst_f32                               :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata0<amdgpu_synid7_vdata32_0>`,   :ref:`vdata1<amdgpu_synid7_vdata32_0>`         :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_cmpst_f64                               :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata0<amdgpu_synid7_vdata64_0>`,   :ref:`vdata1<amdgpu_synid7_vdata64_0>`         :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_cmpst_rtn_b32               :ref:`vdst<amdgpu_synid7_vdst32_0>`,       :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata0<amdgpu_synid7_vdata32_0>`,   :ref:`vdata1<amdgpu_synid7_vdata32_0>`         :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_cmpst_rtn_b64               :ref:`vdst<amdgpu_synid7_vdst64_0>`,       :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata0<amdgpu_synid7_vdata64_0>`,   :ref:`vdata1<amdgpu_synid7_vdata64_0>`         :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_cmpst_rtn_f32               :ref:`vdst<amdgpu_synid7_vdst32_0>`,       :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata0<amdgpu_synid7_vdata32_0>`,   :ref:`vdata1<amdgpu_synid7_vdata32_0>`         :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_cmpst_rtn_f64               :ref:`vdst<amdgpu_synid7_vdst64_0>`,       :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata0<amdgpu_synid7_vdata64_0>`,   :ref:`vdata1<amdgpu_synid7_vdata64_0>`         :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_condxchg32_rtn_b64          :ref:`vdst<amdgpu_synid7_vdst64_0>`,       :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata64_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_consume                     :ref:`vdst<amdgpu_synid7_vdst32_0>`                                           :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_dec_rtn_u32                 :ref:`vdst<amdgpu_synid7_vdst32_0>`,       :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata32_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_dec_rtn_u64                 :ref:`vdst<amdgpu_synid7_vdst64_0>`,       :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata64_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_dec_src2_u32                            :ref:`vaddr<amdgpu_synid7_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_dec_src2_u64                            :ref:`vaddr<amdgpu_synid7_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_dec_u32                                 :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata32_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_dec_u64                                 :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata64_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_gws_barrier                             :ref:`vdata<amdgpu_synid7_vdata32_0>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_gws_init                                :ref:`vdata<amdgpu_synid7_vdata32_0>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_gws_sema_br                             :ref:`vdata<amdgpu_synid7_vdata32_0>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_gws_sema_p                                                                 :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_gws_sema_release_all                                                       :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_gws_sema_v                                                                 :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_inc_rtn_u32                 :ref:`vdst<amdgpu_synid7_vdst32_0>`,       :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata32_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_inc_rtn_u64                 :ref:`vdst<amdgpu_synid7_vdst64_0>`,       :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata64_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_inc_src2_u32                            :ref:`vaddr<amdgpu_synid7_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_inc_src2_u64                            :ref:`vaddr<amdgpu_synid7_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_inc_u32                                 :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata32_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_inc_u64                                 :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata64_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_max_f32                                 :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata32_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_max_f64                                 :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata64_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_max_i32                                 :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata32_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_max_i64                                 :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata64_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_max_rtn_f32                 :ref:`vdst<amdgpu_synid7_vdst32_0>`,       :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata32_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_max_rtn_f64                 :ref:`vdst<amdgpu_synid7_vdst64_0>`,       :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata64_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_max_rtn_i32                 :ref:`vdst<amdgpu_synid7_vdst32_0>`,       :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata32_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_max_rtn_i64                 :ref:`vdst<amdgpu_synid7_vdst64_0>`,       :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata64_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_max_rtn_u32                 :ref:`vdst<amdgpu_synid7_vdst32_0>`,       :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata32_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_max_rtn_u64                 :ref:`vdst<amdgpu_synid7_vdst64_0>`,       :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata64_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_max_src2_f32                            :ref:`vaddr<amdgpu_synid7_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_max_src2_f64                            :ref:`vaddr<amdgpu_synid7_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_max_src2_i32                            :ref:`vaddr<amdgpu_synid7_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_max_src2_i64                            :ref:`vaddr<amdgpu_synid7_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_max_src2_u32                            :ref:`vaddr<amdgpu_synid7_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_max_src2_u64                            :ref:`vaddr<amdgpu_synid7_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_max_u32                                 :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata32_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_max_u64                                 :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata64_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_min_f32                                 :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata32_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_min_f64                                 :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata64_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_min_i32                                 :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata32_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_min_i64                                 :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata64_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_min_rtn_f32                 :ref:`vdst<amdgpu_synid7_vdst32_0>`,       :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata32_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_min_rtn_f64                 :ref:`vdst<amdgpu_synid7_vdst64_0>`,       :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata64_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_min_rtn_i32                 :ref:`vdst<amdgpu_synid7_vdst32_0>`,       :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata32_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_min_rtn_i64                 :ref:`vdst<amdgpu_synid7_vdst64_0>`,       :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata64_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_min_rtn_u32                 :ref:`vdst<amdgpu_synid7_vdst32_0>`,       :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata32_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_min_rtn_u64                 :ref:`vdst<amdgpu_synid7_vdst64_0>`,       :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata64_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_min_src2_f32                            :ref:`vaddr<amdgpu_synid7_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_min_src2_f64                            :ref:`vaddr<amdgpu_synid7_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_min_src2_i32                            :ref:`vaddr<amdgpu_synid7_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_min_src2_i64                            :ref:`vaddr<amdgpu_synid7_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_min_src2_u32                            :ref:`vaddr<amdgpu_synid7_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_min_src2_u64                            :ref:`vaddr<amdgpu_synid7_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_min_u32                                 :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata32_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_min_u64                                 :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata64_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_mskor_b32                               :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata0<amdgpu_synid7_vdata32_0>`,   :ref:`vdata1<amdgpu_synid7_vdata32_0>`         :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_mskor_b64                               :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata0<amdgpu_synid7_vdata64_0>`,   :ref:`vdata1<amdgpu_synid7_vdata64_0>`         :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_mskor_rtn_b32               :ref:`vdst<amdgpu_synid7_vdst32_0>`,       :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata0<amdgpu_synid7_vdata32_0>`,   :ref:`vdata1<amdgpu_synid7_vdata32_0>`         :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_mskor_rtn_b64               :ref:`vdst<amdgpu_synid7_vdst64_0>`,       :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata0<amdgpu_synid7_vdata64_0>`,   :ref:`vdata1<amdgpu_synid7_vdata64_0>`         :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
     ds_nop
-    ds_or_b32                                  :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata32_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_or_b64                                  :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata64_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_or_rtn_b32                  :ref:`vdst<amdgpu_synid7_vdst32_0>`,       :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata32_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_or_rtn_b64                  :ref:`vdst<amdgpu_synid7_vdst64_0>`,       :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata64_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_or_src2_b32                             :ref:`vaddr<amdgpu_synid7_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_or_src2_b64                             :ref:`vaddr<amdgpu_synid7_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_ordered_count               :ref:`vdst<amdgpu_synid7_vdst32_0>`,       :ref:`vaddr<amdgpu_synid7_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_read2_b32                   :ref:`vdst<amdgpu_synid7_vdst64_0>`::ref:`b32x2<amdgpu_synid7_type_dev>`, :ref:`vaddr<amdgpu_synid7_addr_ds>`                              :ref:`ds_offset8<amdgpu_synid_ds_offset8>` :ref:`ds_offset8<amdgpu_synid_ds_offset8>` :ref:`gds<amdgpu_synid_gds>`
-    ds_read2_b64                   :ref:`vdst<amdgpu_synid7_vdst128_0>`::ref:`b64x2<amdgpu_synid7_type_dev>`, :ref:`vaddr<amdgpu_synid7_addr_ds>`                              :ref:`ds_offset8<amdgpu_synid_ds_offset8>` :ref:`ds_offset8<amdgpu_synid_ds_offset8>` :ref:`gds<amdgpu_synid_gds>`
-    ds_read2st64_b32               :ref:`vdst<amdgpu_synid7_vdst64_0>`::ref:`b32x2<amdgpu_synid7_type_dev>`, :ref:`vaddr<amdgpu_synid7_addr_ds>`                              :ref:`ds_offset8<amdgpu_synid_ds_offset8>` :ref:`ds_offset8<amdgpu_synid_ds_offset8>` :ref:`gds<amdgpu_synid_gds>`
-    ds_read2st64_b64               :ref:`vdst<amdgpu_synid7_vdst128_0>`::ref:`b64x2<amdgpu_synid7_type_dev>`, :ref:`vaddr<amdgpu_synid7_addr_ds>`                              :ref:`ds_offset8<amdgpu_synid_ds_offset8>` :ref:`ds_offset8<amdgpu_synid_ds_offset8>` :ref:`gds<amdgpu_synid_gds>`
-    ds_read_b128                   :ref:`vdst<amdgpu_synid7_vdst128_0>`,       :ref:`vaddr<amdgpu_synid7_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_read_b32                    :ref:`vdst<amdgpu_synid7_vdst32_0>`,       :ref:`vaddr<amdgpu_synid7_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_read_b64                    :ref:`vdst<amdgpu_synid7_vdst64_0>`,       :ref:`vaddr<amdgpu_synid7_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_read_b96                    :ref:`vdst<amdgpu_synid7_vdst96_0>`,       :ref:`vaddr<amdgpu_synid7_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_read_i16                    :ref:`vdst<amdgpu_synid7_vdst32_0>`,       :ref:`vaddr<amdgpu_synid7_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_read_i8                     :ref:`vdst<amdgpu_synid7_vdst32_0>`,       :ref:`vaddr<amdgpu_synid7_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_read_u16                    :ref:`vdst<amdgpu_synid7_vdst32_0>`,       :ref:`vaddr<amdgpu_synid7_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_read_u8                     :ref:`vdst<amdgpu_synid7_vdst32_0>`,       :ref:`vaddr<amdgpu_synid7_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_rsub_rtn_u32                :ref:`vdst<amdgpu_synid7_vdst32_0>`,       :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata32_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_rsub_rtn_u64                :ref:`vdst<amdgpu_synid7_vdst64_0>`,       :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata64_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_rsub_src2_u32                           :ref:`vaddr<amdgpu_synid7_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_rsub_src2_u64                           :ref:`vaddr<amdgpu_synid7_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_rsub_u32                                :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata32_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_rsub_u64                                :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata64_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_sub_rtn_u32                 :ref:`vdst<amdgpu_synid7_vdst32_0>`,       :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata32_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_sub_rtn_u64                 :ref:`vdst<amdgpu_synid7_vdst64_0>`,       :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata64_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_sub_src2_u32                            :ref:`vaddr<amdgpu_synid7_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_sub_src2_u64                            :ref:`vaddr<amdgpu_synid7_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_sub_u32                                 :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata32_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_sub_u64                                 :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata64_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_swizzle_b32                 :ref:`vdst<amdgpu_synid7_vdst32_0>`,       :ref:`vaddr<amdgpu_synid7_addr_ds>`                              :ref:`sw_offset16<amdgpu_synid_sw_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_wrap_rtn_b32                :ref:`vdst<amdgpu_synid7_vdst32_0>`,       :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata0<amdgpu_synid7_vdata32_0>`,   :ref:`vdata1<amdgpu_synid7_vdata32_0>`         :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_write2_b32                              :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata0<amdgpu_synid7_vdata32_0>`,   :ref:`vdata1<amdgpu_synid7_vdata32_0>`         :ref:`ds_offset8<amdgpu_synid_ds_offset8>` :ref:`ds_offset8<amdgpu_synid_ds_offset8>` :ref:`gds<amdgpu_synid_gds>`
-    ds_write2_b64                              :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata0<amdgpu_synid7_vdata64_0>`,   :ref:`vdata1<amdgpu_synid7_vdata64_0>`         :ref:`ds_offset8<amdgpu_synid_ds_offset8>` :ref:`ds_offset8<amdgpu_synid_ds_offset8>` :ref:`gds<amdgpu_synid_gds>`
-    ds_write2st64_b32                          :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata0<amdgpu_synid7_vdata32_0>`,   :ref:`vdata1<amdgpu_synid7_vdata32_0>`         :ref:`ds_offset8<amdgpu_synid_ds_offset8>` :ref:`ds_offset8<amdgpu_synid_ds_offset8>` :ref:`gds<amdgpu_synid_gds>`
-    ds_write2st64_b64                          :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata0<amdgpu_synid7_vdata64_0>`,   :ref:`vdata1<amdgpu_synid7_vdata64_0>`         :ref:`ds_offset8<amdgpu_synid_ds_offset8>` :ref:`ds_offset8<amdgpu_synid_ds_offset8>` :ref:`gds<amdgpu_synid_gds>`
-    ds_write_b128                              :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata128_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_write_b16                               :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata32_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_write_b32                               :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata32_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_write_b64                               :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata64_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_write_b8                                :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata32_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_write_b96                               :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata96_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_write_src2_b32                          :ref:`vaddr<amdgpu_synid7_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_write_src2_b64                          :ref:`vaddr<amdgpu_synid7_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_wrxchg2_rtn_b32             :ref:`vdst<amdgpu_synid7_vdst64_0>`::ref:`b32x2<amdgpu_synid7_type_dev>`, :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata0<amdgpu_synid7_vdata32_0>`,   :ref:`vdata1<amdgpu_synid7_vdata32_0>`         :ref:`ds_offset8<amdgpu_synid_ds_offset8>` :ref:`ds_offset8<amdgpu_synid_ds_offset8>` :ref:`gds<amdgpu_synid_gds>`
-    ds_wrxchg2_rtn_b64             :ref:`vdst<amdgpu_synid7_vdst128_0>`::ref:`b64x2<amdgpu_synid7_type_dev>`, :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata0<amdgpu_synid7_vdata64_0>`,   :ref:`vdata1<amdgpu_synid7_vdata64_0>`         :ref:`ds_offset8<amdgpu_synid_ds_offset8>` :ref:`ds_offset8<amdgpu_synid_ds_offset8>` :ref:`gds<amdgpu_synid_gds>`
-    ds_wrxchg2st64_rtn_b32         :ref:`vdst<amdgpu_synid7_vdst64_0>`::ref:`b32x2<amdgpu_synid7_type_dev>`, :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata0<amdgpu_synid7_vdata32_0>`,   :ref:`vdata1<amdgpu_synid7_vdata32_0>`         :ref:`ds_offset8<amdgpu_synid_ds_offset8>` :ref:`ds_offset8<amdgpu_synid_ds_offset8>` :ref:`gds<amdgpu_synid_gds>`
-    ds_wrxchg2st64_rtn_b64         :ref:`vdst<amdgpu_synid7_vdst128_0>`::ref:`b64x2<amdgpu_synid7_type_dev>`, :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata0<amdgpu_synid7_vdata64_0>`,   :ref:`vdata1<amdgpu_synid7_vdata64_0>`         :ref:`ds_offset8<amdgpu_synid_ds_offset8>` :ref:`ds_offset8<amdgpu_synid_ds_offset8>` :ref:`gds<amdgpu_synid_gds>`
-    ds_wrxchg_rtn_b32              :ref:`vdst<amdgpu_synid7_vdst32_0>`,       :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata32_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_wrxchg_rtn_b64              :ref:`vdst<amdgpu_synid7_vdst64_0>`,       :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata64_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_xor_b32                                 :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata32_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_xor_b64                                 :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata64_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_xor_rtn_b32                 :ref:`vdst<amdgpu_synid7_vdst32_0>`,       :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata32_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_xor_rtn_b64                 :ref:`vdst<amdgpu_synid7_vdst64_0>`,       :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata64_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_xor_src2_b32                            :ref:`vaddr<amdgpu_synid7_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_xor_src2_b64                            :ref:`vaddr<amdgpu_synid7_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_or_b32                                  :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata32_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_or_b64                                  :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata64_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_or_rtn_b32                  :ref:`vdst<amdgpu_synid7_vdst32_0>`,       :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata32_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_or_rtn_b64                  :ref:`vdst<amdgpu_synid7_vdst64_0>`,       :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata64_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_or_src2_b32                             :ref:`vaddr<amdgpu_synid7_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_or_src2_b64                             :ref:`vaddr<amdgpu_synid7_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_ordered_count               :ref:`vdst<amdgpu_synid7_vdst32_0>`,       :ref:`vaddr<amdgpu_synid7_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_read2_b32                   :ref:`vdst<amdgpu_synid7_vdst64_0>`::ref:`b32x2<amdgpu_synid7_type_dev>`, :ref:`vaddr<amdgpu_synid7_addr_ds>`                              :ref:`offset8<amdgpu_synid_ds_offset8>` :ref:`offset8<amdgpu_synid_ds_offset8>` :ref:`gds<amdgpu_synid_gds>`
+    ds_read2_b64                   :ref:`vdst<amdgpu_synid7_vdst128_0>`::ref:`b64x2<amdgpu_synid7_type_dev>`, :ref:`vaddr<amdgpu_synid7_addr_ds>`                              :ref:`offset8<amdgpu_synid_ds_offset8>` :ref:`offset8<amdgpu_synid_ds_offset8>` :ref:`gds<amdgpu_synid_gds>`
+    ds_read2st64_b32               :ref:`vdst<amdgpu_synid7_vdst64_0>`::ref:`b32x2<amdgpu_synid7_type_dev>`, :ref:`vaddr<amdgpu_synid7_addr_ds>`                              :ref:`offset8<amdgpu_synid_ds_offset8>` :ref:`offset8<amdgpu_synid_ds_offset8>` :ref:`gds<amdgpu_synid_gds>`
+    ds_read2st64_b64               :ref:`vdst<amdgpu_synid7_vdst128_0>`::ref:`b64x2<amdgpu_synid7_type_dev>`, :ref:`vaddr<amdgpu_synid7_addr_ds>`                              :ref:`offset8<amdgpu_synid_ds_offset8>` :ref:`offset8<amdgpu_synid_ds_offset8>` :ref:`gds<amdgpu_synid_gds>`
+    ds_read_b128                   :ref:`vdst<amdgpu_synid7_vdst128_0>`,       :ref:`vaddr<amdgpu_synid7_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_read_b32                    :ref:`vdst<amdgpu_synid7_vdst32_0>`,       :ref:`vaddr<amdgpu_synid7_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_read_b64                    :ref:`vdst<amdgpu_synid7_vdst64_0>`,       :ref:`vaddr<amdgpu_synid7_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_read_b96                    :ref:`vdst<amdgpu_synid7_vdst96_0>`,       :ref:`vaddr<amdgpu_synid7_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_read_i16                    :ref:`vdst<amdgpu_synid7_vdst32_0>`,       :ref:`vaddr<amdgpu_synid7_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_read_i8                     :ref:`vdst<amdgpu_synid7_vdst32_0>`,       :ref:`vaddr<amdgpu_synid7_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_read_u16                    :ref:`vdst<amdgpu_synid7_vdst32_0>`,       :ref:`vaddr<amdgpu_synid7_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_read_u8                     :ref:`vdst<amdgpu_synid7_vdst32_0>`,       :ref:`vaddr<amdgpu_synid7_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_rsub_rtn_u32                :ref:`vdst<amdgpu_synid7_vdst32_0>`,       :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata32_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_rsub_rtn_u64                :ref:`vdst<amdgpu_synid7_vdst64_0>`,       :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata64_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_rsub_src2_u32                           :ref:`vaddr<amdgpu_synid7_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_rsub_src2_u64                           :ref:`vaddr<amdgpu_synid7_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_rsub_u32                                :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata32_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_rsub_u64                                :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata64_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_sub_rtn_u32                 :ref:`vdst<amdgpu_synid7_vdst32_0>`,       :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata32_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_sub_rtn_u64                 :ref:`vdst<amdgpu_synid7_vdst64_0>`,       :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata64_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_sub_src2_u32                            :ref:`vaddr<amdgpu_synid7_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_sub_src2_u64                            :ref:`vaddr<amdgpu_synid7_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_sub_u32                                 :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata32_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_sub_u64                                 :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata64_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_swizzle_b32                 :ref:`vdst<amdgpu_synid7_vdst32_0>`,       :ref:`vaddr<amdgpu_synid7_addr_ds>`                              :ref:`pattern<amdgpu_synid_sw_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_wrap_rtn_b32                :ref:`vdst<amdgpu_synid7_vdst32_0>`,       :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata0<amdgpu_synid7_vdata32_0>`,   :ref:`vdata1<amdgpu_synid7_vdata32_0>`         :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_write2_b32                              :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata0<amdgpu_synid7_vdata32_0>`,   :ref:`vdata1<amdgpu_synid7_vdata32_0>`         :ref:`offset8<amdgpu_synid_ds_offset8>` :ref:`offset8<amdgpu_synid_ds_offset8>` :ref:`gds<amdgpu_synid_gds>`
+    ds_write2_b64                              :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata0<amdgpu_synid7_vdata64_0>`,   :ref:`vdata1<amdgpu_synid7_vdata64_0>`         :ref:`offset8<amdgpu_synid_ds_offset8>` :ref:`offset8<amdgpu_synid_ds_offset8>` :ref:`gds<amdgpu_synid_gds>`
+    ds_write2st64_b32                          :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata0<amdgpu_synid7_vdata32_0>`,   :ref:`vdata1<amdgpu_synid7_vdata32_0>`         :ref:`offset8<amdgpu_synid_ds_offset8>` :ref:`offset8<amdgpu_synid_ds_offset8>` :ref:`gds<amdgpu_synid_gds>`
+    ds_write2st64_b64                          :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata0<amdgpu_synid7_vdata64_0>`,   :ref:`vdata1<amdgpu_synid7_vdata64_0>`         :ref:`offset8<amdgpu_synid_ds_offset8>` :ref:`offset8<amdgpu_synid_ds_offset8>` :ref:`gds<amdgpu_synid_gds>`
+    ds_write_b128                              :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata128_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_write_b16                               :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata32_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_write_b32                               :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata32_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_write_b64                               :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata64_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_write_b8                                :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata32_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_write_b96                               :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata96_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_write_src2_b32                          :ref:`vaddr<amdgpu_synid7_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_write_src2_b64                          :ref:`vaddr<amdgpu_synid7_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_wrxchg2_rtn_b32             :ref:`vdst<amdgpu_synid7_vdst64_0>`::ref:`b32x2<amdgpu_synid7_type_dev>`, :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata0<amdgpu_synid7_vdata32_0>`,   :ref:`vdata1<amdgpu_synid7_vdata32_0>`         :ref:`offset8<amdgpu_synid_ds_offset8>` :ref:`offset8<amdgpu_synid_ds_offset8>` :ref:`gds<amdgpu_synid_gds>`
+    ds_wrxchg2_rtn_b64             :ref:`vdst<amdgpu_synid7_vdst128_0>`::ref:`b64x2<amdgpu_synid7_type_dev>`, :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata0<amdgpu_synid7_vdata64_0>`,   :ref:`vdata1<amdgpu_synid7_vdata64_0>`         :ref:`offset8<amdgpu_synid_ds_offset8>` :ref:`offset8<amdgpu_synid_ds_offset8>` :ref:`gds<amdgpu_synid_gds>`
+    ds_wrxchg2st64_rtn_b32         :ref:`vdst<amdgpu_synid7_vdst64_0>`::ref:`b32x2<amdgpu_synid7_type_dev>`, :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata0<amdgpu_synid7_vdata32_0>`,   :ref:`vdata1<amdgpu_synid7_vdata32_0>`         :ref:`offset8<amdgpu_synid_ds_offset8>` :ref:`offset8<amdgpu_synid_ds_offset8>` :ref:`gds<amdgpu_synid_gds>`
+    ds_wrxchg2st64_rtn_b64         :ref:`vdst<amdgpu_synid7_vdst128_0>`::ref:`b64x2<amdgpu_synid7_type_dev>`, :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata0<amdgpu_synid7_vdata64_0>`,   :ref:`vdata1<amdgpu_synid7_vdata64_0>`         :ref:`offset8<amdgpu_synid_ds_offset8>` :ref:`offset8<amdgpu_synid_ds_offset8>` :ref:`gds<amdgpu_synid_gds>`
+    ds_wrxchg_rtn_b32              :ref:`vdst<amdgpu_synid7_vdst32_0>`,       :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata32_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_wrxchg_rtn_b64              :ref:`vdst<amdgpu_synid7_vdst64_0>`,       :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata64_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_xor_b32                                 :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata32_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_xor_b64                                 :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata64_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_xor_rtn_b32                 :ref:`vdst<amdgpu_synid7_vdst32_0>`,       :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata32_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_xor_rtn_b64                 :ref:`vdst<amdgpu_synid7_vdst64_0>`,       :ref:`vaddr<amdgpu_synid7_addr_ds>`,    :ref:`vdata<amdgpu_synid7_vdata64_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_xor_src2_b32                            :ref:`vaddr<amdgpu_synid7_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_xor_src2_b64                            :ref:`vaddr<amdgpu_synid7_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
 
 EXP
 -----------------------
@@ -188,7 +188,7 @@ FLAT
 .. parsed-literal::
 
     **INSTRUCTION**                    **DST**           **SRC0**      **SRC1**             **MODIFIERS**
-    \ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|
+    \ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|
     flat_atomic_add                :ref:`vdst<amdgpu_synid7_dst_flat_atomic32>`::ref:`opt<amdgpu_synid7_opt>`,     :ref:`vaddr<amdgpu_synid7_addr_flat>`,    :ref:`vdata<amdgpu_synid7_vdata32_0>`            :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
     flat_atomic_add_x2             :ref:`vdst<amdgpu_synid7_dst_flat_atomic64>`::ref:`opt<amdgpu_synid7_opt>`,     :ref:`vaddr<amdgpu_synid7_addr_flat>`,    :ref:`vdata<amdgpu_synid7_vdata64_0>`            :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
     flat_atomic_and                :ref:`vdst<amdgpu_synid7_dst_flat_atomic32>`::ref:`opt<amdgpu_synid7_opt>`,     :ref:`vaddr<amdgpu_synid7_addr_flat>`,    :ref:`vdata<amdgpu_synid7_vdata32_0>`            :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
@@ -338,56 +338,56 @@ MUBUF
 
 .. parsed-literal::
 
-    **INSTRUCTION**                    **DST**       **SRC0**             **SRC1**      **SRC2**      **SRC3**           **MODIFIERS**
-    \ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|
-    buffer_atomic_add                        :ref:`vdata<amdgpu_synid7_data_buf_atomic32>`::ref:`dst<amdgpu_synid7_ret>`,       :ref:`vaddr<amdgpu_synid7_addr_buf>`,    :ref:`srsrc<amdgpu_synid7_rsrc_buf>`,    :ref:`soffset<amdgpu_synid7_offset_buf>`        :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`addr64<amdgpu_synid_addr64>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_atomic_add_x2                     :ref:`vdata<amdgpu_synid7_data_buf_atomic64>`::ref:`dst<amdgpu_synid7_ret>`,       :ref:`vaddr<amdgpu_synid7_addr_buf>`,    :ref:`srsrc<amdgpu_synid7_rsrc_buf>`,    :ref:`soffset<amdgpu_synid7_offset_buf>`        :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`addr64<amdgpu_synid_addr64>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_atomic_and                        :ref:`vdata<amdgpu_synid7_data_buf_atomic32>`::ref:`dst<amdgpu_synid7_ret>`,       :ref:`vaddr<amdgpu_synid7_addr_buf>`,    :ref:`srsrc<amdgpu_synid7_rsrc_buf>`,    :ref:`soffset<amdgpu_synid7_offset_buf>`        :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`addr64<amdgpu_synid_addr64>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_atomic_and_x2                     :ref:`vdata<amdgpu_synid7_data_buf_atomic64>`::ref:`dst<amdgpu_synid7_ret>`,       :ref:`vaddr<amdgpu_synid7_addr_buf>`,    :ref:`srsrc<amdgpu_synid7_rsrc_buf>`,    :ref:`soffset<amdgpu_synid7_offset_buf>`        :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`addr64<amdgpu_synid_addr64>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_atomic_cmpswap                    :ref:`vdata<amdgpu_synid7_data_buf_atomic64>`::ref:`dst<amdgpu_synid7_ret>`::ref:`b32x2<amdgpu_synid7_type_dev>`, :ref:`vaddr<amdgpu_synid7_addr_buf>`,    :ref:`srsrc<amdgpu_synid7_rsrc_buf>`,    :ref:`soffset<amdgpu_synid7_offset_buf>`        :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`addr64<amdgpu_synid_addr64>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_atomic_cmpswap_x2                 :ref:`vdata<amdgpu_synid7_data_buf_atomic128>`::ref:`dst<amdgpu_synid7_ret>`::ref:`b64x2<amdgpu_synid7_type_dev>`, :ref:`vaddr<amdgpu_synid7_addr_buf>`,    :ref:`srsrc<amdgpu_synid7_rsrc_buf>`,    :ref:`soffset<amdgpu_synid7_offset_buf>`        :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`addr64<amdgpu_synid_addr64>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_atomic_dec                        :ref:`vdata<amdgpu_synid7_data_buf_atomic32>`::ref:`dst<amdgpu_synid7_ret>`::ref:`u32<amdgpu_synid7_type_dev>`,   :ref:`vaddr<amdgpu_synid7_addr_buf>`,    :ref:`srsrc<amdgpu_synid7_rsrc_buf>`,    :ref:`soffset<amdgpu_synid7_offset_buf>`        :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`addr64<amdgpu_synid_addr64>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_atomic_dec_x2                     :ref:`vdata<amdgpu_synid7_data_buf_atomic64>`::ref:`dst<amdgpu_synid7_ret>`::ref:`u64<amdgpu_synid7_type_dev>`,   :ref:`vaddr<amdgpu_synid7_addr_buf>`,    :ref:`srsrc<amdgpu_synid7_rsrc_buf>`,    :ref:`soffset<amdgpu_synid7_offset_buf>`        :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`addr64<amdgpu_synid_addr64>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_atomic_inc                        :ref:`vdata<amdgpu_synid7_data_buf_atomic32>`::ref:`dst<amdgpu_synid7_ret>`::ref:`u32<amdgpu_synid7_type_dev>`,   :ref:`vaddr<amdgpu_synid7_addr_buf>`,    :ref:`srsrc<amdgpu_synid7_rsrc_buf>`,    :ref:`soffset<amdgpu_synid7_offset_buf>`        :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`addr64<amdgpu_synid_addr64>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_atomic_inc_x2                     :ref:`vdata<amdgpu_synid7_data_buf_atomic64>`::ref:`dst<amdgpu_synid7_ret>`::ref:`u64<amdgpu_synid7_type_dev>`,   :ref:`vaddr<amdgpu_synid7_addr_buf>`,    :ref:`srsrc<amdgpu_synid7_rsrc_buf>`,    :ref:`soffset<amdgpu_synid7_offset_buf>`        :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`addr64<amdgpu_synid_addr64>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_atomic_or                         :ref:`vdata<amdgpu_synid7_data_buf_atomic32>`::ref:`dst<amdgpu_synid7_ret>`,       :ref:`vaddr<amdgpu_synid7_addr_buf>`,    :ref:`srsrc<amdgpu_synid7_rsrc_buf>`,    :ref:`soffset<amdgpu_synid7_offset_buf>`        :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`addr64<amdgpu_synid_addr64>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_atomic_or_x2                      :ref:`vdata<amdgpu_synid7_data_buf_atomic64>`::ref:`dst<amdgpu_synid7_ret>`,       :ref:`vaddr<amdgpu_synid7_addr_buf>`,    :ref:`srsrc<amdgpu_synid7_rsrc_buf>`,    :ref:`soffset<amdgpu_synid7_offset_buf>`        :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`addr64<amdgpu_synid_addr64>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_atomic_smax                       :ref:`vdata<amdgpu_synid7_data_buf_atomic32>`::ref:`dst<amdgpu_synid7_ret>`::ref:`s32<amdgpu_synid7_type_dev>`,   :ref:`vaddr<amdgpu_synid7_addr_buf>`,    :ref:`srsrc<amdgpu_synid7_rsrc_buf>`,    :ref:`soffset<amdgpu_synid7_offset_buf>`        :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`addr64<amdgpu_synid_addr64>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_atomic_smax_x2                    :ref:`vdata<amdgpu_synid7_data_buf_atomic64>`::ref:`dst<amdgpu_synid7_ret>`::ref:`s64<amdgpu_synid7_type_dev>`,   :ref:`vaddr<amdgpu_synid7_addr_buf>`,    :ref:`srsrc<amdgpu_synid7_rsrc_buf>`,    :ref:`soffset<amdgpu_synid7_offset_buf>`        :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`addr64<amdgpu_synid_addr64>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_atomic_smin                       :ref:`vdata<amdgpu_synid7_data_buf_atomic32>`::ref:`dst<amdgpu_synid7_ret>`::ref:`s32<amdgpu_synid7_type_dev>`,   :ref:`vaddr<amdgpu_synid7_addr_buf>`,    :ref:`srsrc<amdgpu_synid7_rsrc_buf>`,    :ref:`soffset<amdgpu_synid7_offset_buf>`        :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`addr64<amdgpu_synid_addr64>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_atomic_smin_x2                    :ref:`vdata<amdgpu_synid7_data_buf_atomic64>`::ref:`dst<amdgpu_synid7_ret>`::ref:`s64<amdgpu_synid7_type_dev>`,   :ref:`vaddr<amdgpu_synid7_addr_buf>`,    :ref:`srsrc<amdgpu_synid7_rsrc_buf>`,    :ref:`soffset<amdgpu_synid7_offset_buf>`        :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`addr64<amdgpu_synid_addr64>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_atomic_sub                        :ref:`vdata<amdgpu_synid7_data_buf_atomic32>`::ref:`dst<amdgpu_synid7_ret>`,       :ref:`vaddr<amdgpu_synid7_addr_buf>`,    :ref:`srsrc<amdgpu_synid7_rsrc_buf>`,    :ref:`soffset<amdgpu_synid7_offset_buf>`        :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`addr64<amdgpu_synid_addr64>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_atomic_sub_x2                     :ref:`vdata<amdgpu_synid7_data_buf_atomic64>`::ref:`dst<amdgpu_synid7_ret>`,       :ref:`vaddr<amdgpu_synid7_addr_buf>`,    :ref:`srsrc<amdgpu_synid7_rsrc_buf>`,    :ref:`soffset<amdgpu_synid7_offset_buf>`        :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`addr64<amdgpu_synid_addr64>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_atomic_swap                       :ref:`vdata<amdgpu_synid7_data_buf_atomic32>`::ref:`dst<amdgpu_synid7_ret>`,       :ref:`vaddr<amdgpu_synid7_addr_buf>`,    :ref:`srsrc<amdgpu_synid7_rsrc_buf>`,    :ref:`soffset<amdgpu_synid7_offset_buf>`        :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`addr64<amdgpu_synid_addr64>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_atomic_swap_x2                    :ref:`vdata<amdgpu_synid7_data_buf_atomic64>`::ref:`dst<amdgpu_synid7_ret>`,       :ref:`vaddr<amdgpu_synid7_addr_buf>`,    :ref:`srsrc<amdgpu_synid7_rsrc_buf>`,    :ref:`soffset<amdgpu_synid7_offset_buf>`        :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`addr64<amdgpu_synid_addr64>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_atomic_umax                       :ref:`vdata<amdgpu_synid7_data_buf_atomic32>`::ref:`dst<amdgpu_synid7_ret>`::ref:`u32<amdgpu_synid7_type_dev>`,   :ref:`vaddr<amdgpu_synid7_addr_buf>`,    :ref:`srsrc<amdgpu_synid7_rsrc_buf>`,    :ref:`soffset<amdgpu_synid7_offset_buf>`        :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`addr64<amdgpu_synid_addr64>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_atomic_umax_x2                    :ref:`vdata<amdgpu_synid7_data_buf_atomic64>`::ref:`dst<amdgpu_synid7_ret>`::ref:`u64<amdgpu_synid7_type_dev>`,   :ref:`vaddr<amdgpu_synid7_addr_buf>`,    :ref:`srsrc<amdgpu_synid7_rsrc_buf>`,    :ref:`soffset<amdgpu_synid7_offset_buf>`        :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`addr64<amdgpu_synid_addr64>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_atomic_umin                       :ref:`vdata<amdgpu_synid7_data_buf_atomic32>`::ref:`dst<amdgpu_synid7_ret>`::ref:`u32<amdgpu_synid7_type_dev>`,   :ref:`vaddr<amdgpu_synid7_addr_buf>`,    :ref:`srsrc<amdgpu_synid7_rsrc_buf>`,    :ref:`soffset<amdgpu_synid7_offset_buf>`        :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`addr64<amdgpu_synid_addr64>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_atomic_umin_x2                    :ref:`vdata<amdgpu_synid7_data_buf_atomic64>`::ref:`dst<amdgpu_synid7_ret>`::ref:`u64<amdgpu_synid7_type_dev>`,   :ref:`vaddr<amdgpu_synid7_addr_buf>`,    :ref:`srsrc<amdgpu_synid7_rsrc_buf>`,    :ref:`soffset<amdgpu_synid7_offset_buf>`        :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`addr64<amdgpu_synid_addr64>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_atomic_xor                        :ref:`vdata<amdgpu_synid7_data_buf_atomic32>`::ref:`dst<amdgpu_synid7_ret>`,       :ref:`vaddr<amdgpu_synid7_addr_buf>`,    :ref:`srsrc<amdgpu_synid7_rsrc_buf>`,    :ref:`soffset<amdgpu_synid7_offset_buf>`        :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`addr64<amdgpu_synid_addr64>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_atomic_xor_x2                     :ref:`vdata<amdgpu_synid7_data_buf_atomic64>`::ref:`dst<amdgpu_synid7_ret>`,       :ref:`vaddr<amdgpu_synid7_addr_buf>`,    :ref:`srsrc<amdgpu_synid7_rsrc_buf>`,    :ref:`soffset<amdgpu_synid7_offset_buf>`        :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`addr64<amdgpu_synid_addr64>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_load_dword              :ref:`vdst<amdgpu_synid7_dst_buf_lds>`,     :ref:`vaddr<amdgpu_synid7_addr_buf>`,           :ref:`srsrc<amdgpu_synid7_rsrc_buf>`,    :ref:`soffset<amdgpu_synid7_offset_buf>`                  :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`addr64<amdgpu_synid_addr64>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`lds<amdgpu_synid_lds>`
-    buffer_load_dwordx2            :ref:`vdst<amdgpu_synid7_dst_buf_64>`,     :ref:`vaddr<amdgpu_synid7_addr_buf>`,           :ref:`srsrc<amdgpu_synid7_rsrc_buf>`,    :ref:`soffset<amdgpu_synid7_offset_buf>`                  :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`addr64<amdgpu_synid_addr64>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_load_dwordx3            :ref:`vdst<amdgpu_synid7_dst_buf_96>`,     :ref:`vaddr<amdgpu_synid7_addr_buf>`,           :ref:`srsrc<amdgpu_synid7_rsrc_buf>`,    :ref:`soffset<amdgpu_synid7_offset_buf>`                  :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`addr64<amdgpu_synid_addr64>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_load_dwordx4            :ref:`vdst<amdgpu_synid7_dst_buf_128>`,     :ref:`vaddr<amdgpu_synid7_addr_buf>`,           :ref:`srsrc<amdgpu_synid7_rsrc_buf>`,    :ref:`soffset<amdgpu_synid7_offset_buf>`                  :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`addr64<amdgpu_synid_addr64>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_load_format_x           :ref:`vdst<amdgpu_synid7_dst_buf_lds>`,     :ref:`vaddr<amdgpu_synid7_addr_buf>`,           :ref:`srsrc<amdgpu_synid7_rsrc_buf>`,    :ref:`soffset<amdgpu_synid7_offset_buf>`                  :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`addr64<amdgpu_synid_addr64>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`lds<amdgpu_synid_lds>`
-    buffer_load_format_xy          :ref:`vdst<amdgpu_synid7_dst_buf_64>`,     :ref:`vaddr<amdgpu_synid7_addr_buf>`,           :ref:`srsrc<amdgpu_synid7_rsrc_buf>`,    :ref:`soffset<amdgpu_synid7_offset_buf>`                  :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`addr64<amdgpu_synid_addr64>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_load_format_xyz         :ref:`vdst<amdgpu_synid7_dst_buf_96>`,     :ref:`vaddr<amdgpu_synid7_addr_buf>`,           :ref:`srsrc<amdgpu_synid7_rsrc_buf>`,    :ref:`soffset<amdgpu_synid7_offset_buf>`                  :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`addr64<amdgpu_synid_addr64>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_load_format_xyzw        :ref:`vdst<amdgpu_synid7_dst_buf_128>`,     :ref:`vaddr<amdgpu_synid7_addr_buf>`,           :ref:`srsrc<amdgpu_synid7_rsrc_buf>`,    :ref:`soffset<amdgpu_synid7_offset_buf>`                  :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`addr64<amdgpu_synid_addr64>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_load_sbyte              :ref:`vdst<amdgpu_synid7_dst_buf_lds>`,     :ref:`vaddr<amdgpu_synid7_addr_buf>`,           :ref:`srsrc<amdgpu_synid7_rsrc_buf>`,    :ref:`soffset<amdgpu_synid7_offset_buf>`                  :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`addr64<amdgpu_synid_addr64>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`lds<amdgpu_synid_lds>`
-    buffer_load_sshort             :ref:`vdst<amdgpu_synid7_dst_buf_lds>`,     :ref:`vaddr<amdgpu_synid7_addr_buf>`,           :ref:`srsrc<amdgpu_synid7_rsrc_buf>`,    :ref:`soffset<amdgpu_synid7_offset_buf>`                  :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`addr64<amdgpu_synid_addr64>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`lds<amdgpu_synid_lds>`
-    buffer_load_ubyte              :ref:`vdst<amdgpu_synid7_dst_buf_lds>`,     :ref:`vaddr<amdgpu_synid7_addr_buf>`,           :ref:`srsrc<amdgpu_synid7_rsrc_buf>`,    :ref:`soffset<amdgpu_synid7_offset_buf>`                  :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`addr64<amdgpu_synid_addr64>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`lds<amdgpu_synid_lds>`
-    buffer_load_ushort             :ref:`vdst<amdgpu_synid7_dst_buf_lds>`,     :ref:`vaddr<amdgpu_synid7_addr_buf>`,           :ref:`srsrc<amdgpu_synid7_rsrc_buf>`,    :ref:`soffset<amdgpu_synid7_offset_buf>`                  :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`addr64<amdgpu_synid_addr64>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`lds<amdgpu_synid_lds>`
-    buffer_store_byte                        :ref:`vdata<amdgpu_synid7_vdata32_0>`,           :ref:`vaddr<amdgpu_synid7_addr_buf>`,    :ref:`srsrc<amdgpu_synid7_rsrc_buf>`,    :ref:`soffset<amdgpu_synid7_offset_buf>`        :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`addr64<amdgpu_synid_addr64>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_store_dword                       :ref:`vdata<amdgpu_synid7_vdata32_0>`,           :ref:`vaddr<amdgpu_synid7_addr_buf>`,    :ref:`srsrc<amdgpu_synid7_rsrc_buf>`,    :ref:`soffset<amdgpu_synid7_offset_buf>`        :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`addr64<amdgpu_synid_addr64>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_store_dwordx2                     :ref:`vdata<amdgpu_synid7_vdata64_0>`,           :ref:`vaddr<amdgpu_synid7_addr_buf>`,    :ref:`srsrc<amdgpu_synid7_rsrc_buf>`,    :ref:`soffset<amdgpu_synid7_offset_buf>`        :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`addr64<amdgpu_synid_addr64>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_store_dwordx3                     :ref:`vdata<amdgpu_synid7_vdata96_0>`,           :ref:`vaddr<amdgpu_synid7_addr_buf>`,    :ref:`srsrc<amdgpu_synid7_rsrc_buf>`,    :ref:`soffset<amdgpu_synid7_offset_buf>`        :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`addr64<amdgpu_synid_addr64>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_store_dwordx4                     :ref:`vdata<amdgpu_synid7_vdata128_0>`,           :ref:`vaddr<amdgpu_synid7_addr_buf>`,    :ref:`srsrc<amdgpu_synid7_rsrc_buf>`,    :ref:`soffset<amdgpu_synid7_offset_buf>`        :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`addr64<amdgpu_synid_addr64>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_store_format_x                    :ref:`vdata<amdgpu_synid7_vdata32_0>`,           :ref:`vaddr<amdgpu_synid7_addr_buf>`,    :ref:`srsrc<amdgpu_synid7_rsrc_buf>`,    :ref:`soffset<amdgpu_synid7_offset_buf>`        :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`addr64<amdgpu_synid_addr64>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_store_format_xy                   :ref:`vdata<amdgpu_synid7_vdata64_0>`,           :ref:`vaddr<amdgpu_synid7_addr_buf>`,    :ref:`srsrc<amdgpu_synid7_rsrc_buf>`,    :ref:`soffset<amdgpu_synid7_offset_buf>`        :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`addr64<amdgpu_synid_addr64>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_store_format_xyz                  :ref:`vdata<amdgpu_synid7_vdata96_0>`,           :ref:`vaddr<amdgpu_synid7_addr_buf>`,    :ref:`srsrc<amdgpu_synid7_rsrc_buf>`,    :ref:`soffset<amdgpu_synid7_offset_buf>`        :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`addr64<amdgpu_synid_addr64>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_store_format_xyzw                 :ref:`vdata<amdgpu_synid7_vdata128_0>`,           :ref:`vaddr<amdgpu_synid7_addr_buf>`,    :ref:`srsrc<amdgpu_synid7_rsrc_buf>`,    :ref:`soffset<amdgpu_synid7_offset_buf>`        :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`addr64<amdgpu_synid_addr64>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_store_short                       :ref:`vdata<amdgpu_synid7_vdata32_0>`,           :ref:`vaddr<amdgpu_synid7_addr_buf>`,    :ref:`srsrc<amdgpu_synid7_rsrc_buf>`,    :ref:`soffset<amdgpu_synid7_offset_buf>`        :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`addr64<amdgpu_synid_addr64>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    **INSTRUCTION**              **DST**   **SRC0**             **SRC1**   **SRC2**    **SRC3**    **MODIFIERS**
+    \ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|
+    buffer_atomic_add              :ref:`vdata<amdgpu_synid7_data_buf_atomic32>`::ref:`dst<amdgpu_synid7_ret>`,       :ref:`vaddr<amdgpu_synid7_addr_buf>`, :ref:`srsrc<amdgpu_synid7_rsrc_buf>`,  :ref:`soffset<amdgpu_synid7_offset_buf>` :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`addr64<amdgpu_synid_addr64>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_atomic_add_x2           :ref:`vdata<amdgpu_synid7_data_buf_atomic64>`::ref:`dst<amdgpu_synid7_ret>`,       :ref:`vaddr<amdgpu_synid7_addr_buf>`, :ref:`srsrc<amdgpu_synid7_rsrc_buf>`,  :ref:`soffset<amdgpu_synid7_offset_buf>` :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`addr64<amdgpu_synid_addr64>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_atomic_and              :ref:`vdata<amdgpu_synid7_data_buf_atomic32>`::ref:`dst<amdgpu_synid7_ret>`,       :ref:`vaddr<amdgpu_synid7_addr_buf>`, :ref:`srsrc<amdgpu_synid7_rsrc_buf>`,  :ref:`soffset<amdgpu_synid7_offset_buf>` :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`addr64<amdgpu_synid_addr64>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_atomic_and_x2           :ref:`vdata<amdgpu_synid7_data_buf_atomic64>`::ref:`dst<amdgpu_synid7_ret>`,       :ref:`vaddr<amdgpu_synid7_addr_buf>`, :ref:`srsrc<amdgpu_synid7_rsrc_buf>`,  :ref:`soffset<amdgpu_synid7_offset_buf>` :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`addr64<amdgpu_synid_addr64>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_atomic_cmpswap          :ref:`vdata<amdgpu_synid7_data_buf_atomic64>`::ref:`dst<amdgpu_synid7_ret>`::ref:`b32x2<amdgpu_synid7_type_dev>`, :ref:`vaddr<amdgpu_synid7_addr_buf>`, :ref:`srsrc<amdgpu_synid7_rsrc_buf>`,  :ref:`soffset<amdgpu_synid7_offset_buf>` :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`addr64<amdgpu_synid_addr64>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_atomic_cmpswap_x2       :ref:`vdata<amdgpu_synid7_data_buf_atomic128>`::ref:`dst<amdgpu_synid7_ret>`::ref:`b64x2<amdgpu_synid7_type_dev>`, :ref:`vaddr<amdgpu_synid7_addr_buf>`, :ref:`srsrc<amdgpu_synid7_rsrc_buf>`,  :ref:`soffset<amdgpu_synid7_offset_buf>` :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`addr64<amdgpu_synid_addr64>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_atomic_dec              :ref:`vdata<amdgpu_synid7_data_buf_atomic32>`::ref:`dst<amdgpu_synid7_ret>`::ref:`u32<amdgpu_synid7_type_dev>`,   :ref:`vaddr<amdgpu_synid7_addr_buf>`, :ref:`srsrc<amdgpu_synid7_rsrc_buf>`,  :ref:`soffset<amdgpu_synid7_offset_buf>` :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`addr64<amdgpu_synid_addr64>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_atomic_dec_x2           :ref:`vdata<amdgpu_synid7_data_buf_atomic64>`::ref:`dst<amdgpu_synid7_ret>`::ref:`u64<amdgpu_synid7_type_dev>`,   :ref:`vaddr<amdgpu_synid7_addr_buf>`, :ref:`srsrc<amdgpu_synid7_rsrc_buf>`,  :ref:`soffset<amdgpu_synid7_offset_buf>` :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`addr64<amdgpu_synid_addr64>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_atomic_inc              :ref:`vdata<amdgpu_synid7_data_buf_atomic32>`::ref:`dst<amdgpu_synid7_ret>`::ref:`u32<amdgpu_synid7_type_dev>`,   :ref:`vaddr<amdgpu_synid7_addr_buf>`, :ref:`srsrc<amdgpu_synid7_rsrc_buf>`,  :ref:`soffset<amdgpu_synid7_offset_buf>` :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`addr64<amdgpu_synid_addr64>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_atomic_inc_x2           :ref:`vdata<amdgpu_synid7_data_buf_atomic64>`::ref:`dst<amdgpu_synid7_ret>`::ref:`u64<amdgpu_synid7_type_dev>`,   :ref:`vaddr<amdgpu_synid7_addr_buf>`, :ref:`srsrc<amdgpu_synid7_rsrc_buf>`,  :ref:`soffset<amdgpu_synid7_offset_buf>` :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`addr64<amdgpu_synid_addr64>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_atomic_or               :ref:`vdata<amdgpu_synid7_data_buf_atomic32>`::ref:`dst<amdgpu_synid7_ret>`,       :ref:`vaddr<amdgpu_synid7_addr_buf>`, :ref:`srsrc<amdgpu_synid7_rsrc_buf>`,  :ref:`soffset<amdgpu_synid7_offset_buf>` :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`addr64<amdgpu_synid_addr64>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_atomic_or_x2            :ref:`vdata<amdgpu_synid7_data_buf_atomic64>`::ref:`dst<amdgpu_synid7_ret>`,       :ref:`vaddr<amdgpu_synid7_addr_buf>`, :ref:`srsrc<amdgpu_synid7_rsrc_buf>`,  :ref:`soffset<amdgpu_synid7_offset_buf>` :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`addr64<amdgpu_synid_addr64>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_atomic_smax             :ref:`vdata<amdgpu_synid7_data_buf_atomic32>`::ref:`dst<amdgpu_synid7_ret>`::ref:`s32<amdgpu_synid7_type_dev>`,   :ref:`vaddr<amdgpu_synid7_addr_buf>`, :ref:`srsrc<amdgpu_synid7_rsrc_buf>`,  :ref:`soffset<amdgpu_synid7_offset_buf>` :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`addr64<amdgpu_synid_addr64>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_atomic_smax_x2          :ref:`vdata<amdgpu_synid7_data_buf_atomic64>`::ref:`dst<amdgpu_synid7_ret>`::ref:`s64<amdgpu_synid7_type_dev>`,   :ref:`vaddr<amdgpu_synid7_addr_buf>`, :ref:`srsrc<amdgpu_synid7_rsrc_buf>`,  :ref:`soffset<amdgpu_synid7_offset_buf>` :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`addr64<amdgpu_synid_addr64>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_atomic_smin             :ref:`vdata<amdgpu_synid7_data_buf_atomic32>`::ref:`dst<amdgpu_synid7_ret>`::ref:`s32<amdgpu_synid7_type_dev>`,   :ref:`vaddr<amdgpu_synid7_addr_buf>`, :ref:`srsrc<amdgpu_synid7_rsrc_buf>`,  :ref:`soffset<amdgpu_synid7_offset_buf>` :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`addr64<amdgpu_synid_addr64>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_atomic_smin_x2          :ref:`vdata<amdgpu_synid7_data_buf_atomic64>`::ref:`dst<amdgpu_synid7_ret>`::ref:`s64<amdgpu_synid7_type_dev>`,   :ref:`vaddr<amdgpu_synid7_addr_buf>`, :ref:`srsrc<amdgpu_synid7_rsrc_buf>`,  :ref:`soffset<amdgpu_synid7_offset_buf>` :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`addr64<amdgpu_synid_addr64>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_atomic_sub              :ref:`vdata<amdgpu_synid7_data_buf_atomic32>`::ref:`dst<amdgpu_synid7_ret>`,       :ref:`vaddr<amdgpu_synid7_addr_buf>`, :ref:`srsrc<amdgpu_synid7_rsrc_buf>`,  :ref:`soffset<amdgpu_synid7_offset_buf>` :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`addr64<amdgpu_synid_addr64>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_atomic_sub_x2           :ref:`vdata<amdgpu_synid7_data_buf_atomic64>`::ref:`dst<amdgpu_synid7_ret>`,       :ref:`vaddr<amdgpu_synid7_addr_buf>`, :ref:`srsrc<amdgpu_synid7_rsrc_buf>`,  :ref:`soffset<amdgpu_synid7_offset_buf>` :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`addr64<amdgpu_synid_addr64>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_atomic_swap             :ref:`vdata<amdgpu_synid7_data_buf_atomic32>`::ref:`dst<amdgpu_synid7_ret>`,       :ref:`vaddr<amdgpu_synid7_addr_buf>`, :ref:`srsrc<amdgpu_synid7_rsrc_buf>`,  :ref:`soffset<amdgpu_synid7_offset_buf>` :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`addr64<amdgpu_synid_addr64>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_atomic_swap_x2          :ref:`vdata<amdgpu_synid7_data_buf_atomic64>`::ref:`dst<amdgpu_synid7_ret>`,       :ref:`vaddr<amdgpu_synid7_addr_buf>`, :ref:`srsrc<amdgpu_synid7_rsrc_buf>`,  :ref:`soffset<amdgpu_synid7_offset_buf>` :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`addr64<amdgpu_synid_addr64>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_atomic_umax             :ref:`vdata<amdgpu_synid7_data_buf_atomic32>`::ref:`dst<amdgpu_synid7_ret>`::ref:`u32<amdgpu_synid7_type_dev>`,   :ref:`vaddr<amdgpu_synid7_addr_buf>`, :ref:`srsrc<amdgpu_synid7_rsrc_buf>`,  :ref:`soffset<amdgpu_synid7_offset_buf>` :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`addr64<amdgpu_synid_addr64>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_atomic_umax_x2          :ref:`vdata<amdgpu_synid7_data_buf_atomic64>`::ref:`dst<amdgpu_synid7_ret>`::ref:`u64<amdgpu_synid7_type_dev>`,   :ref:`vaddr<amdgpu_synid7_addr_buf>`, :ref:`srsrc<amdgpu_synid7_rsrc_buf>`,  :ref:`soffset<amdgpu_synid7_offset_buf>` :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`addr64<amdgpu_synid_addr64>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_atomic_umin             :ref:`vdata<amdgpu_synid7_data_buf_atomic32>`::ref:`dst<amdgpu_synid7_ret>`::ref:`u32<amdgpu_synid7_type_dev>`,   :ref:`vaddr<amdgpu_synid7_addr_buf>`, :ref:`srsrc<amdgpu_synid7_rsrc_buf>`,  :ref:`soffset<amdgpu_synid7_offset_buf>` :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`addr64<amdgpu_synid_addr64>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_atomic_umin_x2          :ref:`vdata<amdgpu_synid7_data_buf_atomic64>`::ref:`dst<amdgpu_synid7_ret>`::ref:`u64<amdgpu_synid7_type_dev>`,   :ref:`vaddr<amdgpu_synid7_addr_buf>`, :ref:`srsrc<amdgpu_synid7_rsrc_buf>`,  :ref:`soffset<amdgpu_synid7_offset_buf>` :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`addr64<amdgpu_synid_addr64>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_atomic_xor              :ref:`vdata<amdgpu_synid7_data_buf_atomic32>`::ref:`dst<amdgpu_synid7_ret>`,       :ref:`vaddr<amdgpu_synid7_addr_buf>`, :ref:`srsrc<amdgpu_synid7_rsrc_buf>`,  :ref:`soffset<amdgpu_synid7_offset_buf>` :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`addr64<amdgpu_synid_addr64>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_atomic_xor_x2           :ref:`vdata<amdgpu_synid7_data_buf_atomic64>`::ref:`dst<amdgpu_synid7_ret>`,       :ref:`vaddr<amdgpu_synid7_addr_buf>`, :ref:`srsrc<amdgpu_synid7_rsrc_buf>`,  :ref:`soffset<amdgpu_synid7_offset_buf>` :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`addr64<amdgpu_synid_addr64>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_load_dword        :ref:`vdst<amdgpu_synid7_dst_buf_lds>`, :ref:`vaddr<amdgpu_synid7_addr_buf>`,           :ref:`srsrc<amdgpu_synid7_rsrc_buf>`, :ref:`soffset<amdgpu_synid7_offset_buf>`         :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`addr64<amdgpu_synid_addr64>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`lds<amdgpu_synid_lds>`
+    buffer_load_dwordx2      :ref:`vdst<amdgpu_synid7_dst_buf_64>`, :ref:`vaddr<amdgpu_synid7_addr_buf>`,           :ref:`srsrc<amdgpu_synid7_rsrc_buf>`, :ref:`soffset<amdgpu_synid7_offset_buf>`         :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`addr64<amdgpu_synid_addr64>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_load_dwordx3      :ref:`vdst<amdgpu_synid7_dst_buf_96>`, :ref:`vaddr<amdgpu_synid7_addr_buf>`,           :ref:`srsrc<amdgpu_synid7_rsrc_buf>`, :ref:`soffset<amdgpu_synid7_offset_buf>`         :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`addr64<amdgpu_synid_addr64>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_load_dwordx4      :ref:`vdst<amdgpu_synid7_dst_buf_128>`, :ref:`vaddr<amdgpu_synid7_addr_buf>`,           :ref:`srsrc<amdgpu_synid7_rsrc_buf>`, :ref:`soffset<amdgpu_synid7_offset_buf>`         :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`addr64<amdgpu_synid_addr64>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_load_format_x     :ref:`vdst<amdgpu_synid7_dst_buf_lds>`, :ref:`vaddr<amdgpu_synid7_addr_buf>`,           :ref:`srsrc<amdgpu_synid7_rsrc_buf>`, :ref:`soffset<amdgpu_synid7_offset_buf>`         :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`addr64<amdgpu_synid_addr64>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`lds<amdgpu_synid_lds>`
+    buffer_load_format_xy    :ref:`vdst<amdgpu_synid7_dst_buf_64>`, :ref:`vaddr<amdgpu_synid7_addr_buf>`,           :ref:`srsrc<amdgpu_synid7_rsrc_buf>`, :ref:`soffset<amdgpu_synid7_offset_buf>`         :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`addr64<amdgpu_synid_addr64>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_load_format_xyz   :ref:`vdst<amdgpu_synid7_dst_buf_96>`, :ref:`vaddr<amdgpu_synid7_addr_buf>`,           :ref:`srsrc<amdgpu_synid7_rsrc_buf>`, :ref:`soffset<amdgpu_synid7_offset_buf>`         :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`addr64<amdgpu_synid_addr64>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_load_format_xyzw  :ref:`vdst<amdgpu_synid7_dst_buf_128>`, :ref:`vaddr<amdgpu_synid7_addr_buf>`,           :ref:`srsrc<amdgpu_synid7_rsrc_buf>`, :ref:`soffset<amdgpu_synid7_offset_buf>`         :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`addr64<amdgpu_synid_addr64>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_load_sbyte        :ref:`vdst<amdgpu_synid7_dst_buf_lds>`, :ref:`vaddr<amdgpu_synid7_addr_buf>`,           :ref:`srsrc<amdgpu_synid7_rsrc_buf>`, :ref:`soffset<amdgpu_synid7_offset_buf>`         :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`addr64<amdgpu_synid_addr64>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`lds<amdgpu_synid_lds>`
+    buffer_load_sshort       :ref:`vdst<amdgpu_synid7_dst_buf_lds>`, :ref:`vaddr<amdgpu_synid7_addr_buf>`,           :ref:`srsrc<amdgpu_synid7_rsrc_buf>`, :ref:`soffset<amdgpu_synid7_offset_buf>`         :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`addr64<amdgpu_synid_addr64>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`lds<amdgpu_synid_lds>`
+    buffer_load_ubyte        :ref:`vdst<amdgpu_synid7_dst_buf_lds>`, :ref:`vaddr<amdgpu_synid7_addr_buf>`,           :ref:`srsrc<amdgpu_synid7_rsrc_buf>`, :ref:`soffset<amdgpu_synid7_offset_buf>`         :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`addr64<amdgpu_synid_addr64>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`lds<amdgpu_synid_lds>`
+    buffer_load_ushort       :ref:`vdst<amdgpu_synid7_dst_buf_lds>`, :ref:`vaddr<amdgpu_synid7_addr_buf>`,           :ref:`srsrc<amdgpu_synid7_rsrc_buf>`, :ref:`soffset<amdgpu_synid7_offset_buf>`         :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`addr64<amdgpu_synid_addr64>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`lds<amdgpu_synid_lds>`
+    buffer_store_byte              :ref:`vdata<amdgpu_synid7_vdata32_0>`,           :ref:`vaddr<amdgpu_synid7_addr_buf>`, :ref:`srsrc<amdgpu_synid7_rsrc_buf>`,  :ref:`soffset<amdgpu_synid7_offset_buf>` :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`addr64<amdgpu_synid_addr64>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_store_dword             :ref:`vdata<amdgpu_synid7_vdata32_0>`,           :ref:`vaddr<amdgpu_synid7_addr_buf>`, :ref:`srsrc<amdgpu_synid7_rsrc_buf>`,  :ref:`soffset<amdgpu_synid7_offset_buf>` :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`addr64<amdgpu_synid_addr64>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_store_dwordx2           :ref:`vdata<amdgpu_synid7_vdata64_0>`,           :ref:`vaddr<amdgpu_synid7_addr_buf>`, :ref:`srsrc<amdgpu_synid7_rsrc_buf>`,  :ref:`soffset<amdgpu_synid7_offset_buf>` :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`addr64<amdgpu_synid_addr64>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_store_dwordx3           :ref:`vdata<amdgpu_synid7_vdata96_0>`,           :ref:`vaddr<amdgpu_synid7_addr_buf>`, :ref:`srsrc<amdgpu_synid7_rsrc_buf>`,  :ref:`soffset<amdgpu_synid7_offset_buf>` :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`addr64<amdgpu_synid_addr64>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_store_dwordx4           :ref:`vdata<amdgpu_synid7_vdata128_0>`,           :ref:`vaddr<amdgpu_synid7_addr_buf>`, :ref:`srsrc<amdgpu_synid7_rsrc_buf>`,  :ref:`soffset<amdgpu_synid7_offset_buf>` :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`addr64<amdgpu_synid_addr64>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_store_format_x          :ref:`vdata<amdgpu_synid7_vdata32_0>`,           :ref:`vaddr<amdgpu_synid7_addr_buf>`, :ref:`srsrc<amdgpu_synid7_rsrc_buf>`,  :ref:`soffset<amdgpu_synid7_offset_buf>` :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`addr64<amdgpu_synid_addr64>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_store_format_xy         :ref:`vdata<amdgpu_synid7_vdata64_0>`,           :ref:`vaddr<amdgpu_synid7_addr_buf>`, :ref:`srsrc<amdgpu_synid7_rsrc_buf>`,  :ref:`soffset<amdgpu_synid7_offset_buf>` :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`addr64<amdgpu_synid_addr64>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_store_format_xyz        :ref:`vdata<amdgpu_synid7_vdata96_0>`,           :ref:`vaddr<amdgpu_synid7_addr_buf>`, :ref:`srsrc<amdgpu_synid7_rsrc_buf>`,  :ref:`soffset<amdgpu_synid7_offset_buf>` :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`addr64<amdgpu_synid_addr64>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_store_format_xyzw       :ref:`vdata<amdgpu_synid7_vdata128_0>`,           :ref:`vaddr<amdgpu_synid7_addr_buf>`, :ref:`srsrc<amdgpu_synid7_rsrc_buf>`,  :ref:`soffset<amdgpu_synid7_offset_buf>` :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`addr64<amdgpu_synid_addr64>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_store_short             :ref:`vdata<amdgpu_synid7_vdata32_0>`,           :ref:`vaddr<amdgpu_synid7_addr_buf>`, :ref:`srsrc<amdgpu_synid7_rsrc_buf>`,  :ref:`soffset<amdgpu_synid7_offset_buf>` :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`addr64<amdgpu_synid_addr64>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
     buffer_wbinvl1
     buffer_wbinvl1_vol
 
@@ -756,7 +756,7 @@ VOP3
 .. parsed-literal::
 
     **INSTRUCTION**                    **DST0**       **DST1**      **SRC0**        **SRC1**        **SRC2**            **MODIFIERS**
-    \ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|
+    \ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|
     v_add_f32_e64                  :ref:`vdst<amdgpu_synid7_vdst32_0>`,                :ref:`src0<amdgpu_synid7_src32_2>`::ref:`m<amdgpu_synid7_mod>`,     :ref:`src1<amdgpu_synid7_src32_2>`::ref:`m<amdgpu_synid7_mod>`                      :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
     v_add_f64                      :ref:`vdst<amdgpu_synid7_vdst64_0>`,                :ref:`src0<amdgpu_synid7_src64_1>`::ref:`m<amdgpu_synid7_mod>`,     :ref:`src1<amdgpu_synid7_src64_1>`::ref:`m<amdgpu_synid7_mod>`                      :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
     v_add_i32_e64                  :ref:`vdst<amdgpu_synid7_vdst32_0>`,      :ref:`sdst<amdgpu_synid7_sdst64_0>`,     :ref:`src0<amdgpu_synid7_src32_2>`,       :ref:`src1<amdgpu_synid7_src32_2>`
diff --git a/docs/AMDGPU/AMDGPUAsmGFX8.rst b/docs/AMDGPU/AMDGPUAsmGFX8.rst
index 9dc78e10089125b7263a21ac0fee4660fac3bdf4..a6dbc9be68f0b87321801813a60178340a781689 100644
--- a/docs/AMDGPU/AMDGPUAsmGFX8.rst
+++ b/docs/AMDGPU/AMDGPUAsmGFX8.rst
@@ -32,151 +32,151 @@ DS
 .. parsed-literal::
 
     **INSTRUCTION**                    **DST**         **SRC0**      **SRC1**      **SRC2**           **MODIFIERS**
-    \ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|
-    ds_add_f32                                 :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata32_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_add_rtn_f32                 :ref:`vdst<amdgpu_synid8_vdst32_0>`,       :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata32_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_add_rtn_u32                 :ref:`vdst<amdgpu_synid8_vdst32_0>`,       :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata32_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_add_rtn_u64                 :ref:`vdst<amdgpu_synid8_vdst64_0>`,       :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata64_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_add_src2_f32                            :ref:`vaddr<amdgpu_synid8_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_add_src2_u32                            :ref:`vaddr<amdgpu_synid8_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_add_src2_u64                            :ref:`vaddr<amdgpu_synid8_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_add_u32                                 :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata32_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_add_u64                                 :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata64_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_and_b32                                 :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata32_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_and_b64                                 :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata64_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_and_rtn_b32                 :ref:`vdst<amdgpu_synid8_vdst32_0>`,       :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata32_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_and_rtn_b64                 :ref:`vdst<amdgpu_synid8_vdst64_0>`,       :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata64_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_and_src2_b32                            :ref:`vaddr<amdgpu_synid8_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_and_src2_b64                            :ref:`vaddr<amdgpu_synid8_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_append                      :ref:`vdst<amdgpu_synid8_vdst32_0>`                                           :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_bpermute_b32                :ref:`vdst<amdgpu_synid8_vdst32_0>`,       :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata32_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>`
-    ds_cmpst_b32                               :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata0<amdgpu_synid8_vdata32_0>`,   :ref:`vdata1<amdgpu_synid8_vdata32_0>`         :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_cmpst_b64                               :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata0<amdgpu_synid8_vdata64_0>`,   :ref:`vdata1<amdgpu_synid8_vdata64_0>`         :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_cmpst_f32                               :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata0<amdgpu_synid8_vdata32_0>`,   :ref:`vdata1<amdgpu_synid8_vdata32_0>`         :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_cmpst_f64                               :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata0<amdgpu_synid8_vdata64_0>`,   :ref:`vdata1<amdgpu_synid8_vdata64_0>`         :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_cmpst_rtn_b32               :ref:`vdst<amdgpu_synid8_vdst32_0>`,       :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata0<amdgpu_synid8_vdata32_0>`,   :ref:`vdata1<amdgpu_synid8_vdata32_0>`         :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_cmpst_rtn_b64               :ref:`vdst<amdgpu_synid8_vdst64_0>`,       :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata0<amdgpu_synid8_vdata64_0>`,   :ref:`vdata1<amdgpu_synid8_vdata64_0>`         :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_cmpst_rtn_f32               :ref:`vdst<amdgpu_synid8_vdst32_0>`,       :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata0<amdgpu_synid8_vdata32_0>`,   :ref:`vdata1<amdgpu_synid8_vdata32_0>`         :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_cmpst_rtn_f64               :ref:`vdst<amdgpu_synid8_vdst64_0>`,       :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata0<amdgpu_synid8_vdata64_0>`,   :ref:`vdata1<amdgpu_synid8_vdata64_0>`         :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_condxchg32_rtn_b64          :ref:`vdst<amdgpu_synid8_vdst64_0>`,       :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata64_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_consume                     :ref:`vdst<amdgpu_synid8_vdst32_0>`                                           :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_dec_rtn_u32                 :ref:`vdst<amdgpu_synid8_vdst32_0>`,       :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata32_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_dec_rtn_u64                 :ref:`vdst<amdgpu_synid8_vdst64_0>`,       :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata64_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_dec_src2_u32                            :ref:`vaddr<amdgpu_synid8_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_dec_src2_u64                            :ref:`vaddr<amdgpu_synid8_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_dec_u32                                 :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata32_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_dec_u64                                 :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata64_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_gws_barrier                             :ref:`vdata<amdgpu_synid8_vdata32_0>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_gws_init                                :ref:`vdata<amdgpu_synid8_vdata32_0>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_gws_sema_br                             :ref:`vdata<amdgpu_synid8_vdata32_0>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_gws_sema_p                                                                 :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_gws_sema_release_all                                                       :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_gws_sema_v                                                                 :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_inc_rtn_u32                 :ref:`vdst<amdgpu_synid8_vdst32_0>`,       :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata32_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_inc_rtn_u64                 :ref:`vdst<amdgpu_synid8_vdst64_0>`,       :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata64_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_inc_src2_u32                            :ref:`vaddr<amdgpu_synid8_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_inc_src2_u64                            :ref:`vaddr<amdgpu_synid8_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_inc_u32                                 :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata32_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_inc_u64                                 :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata64_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_max_f32                                 :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata32_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_max_f64                                 :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata64_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_max_i32                                 :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata32_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_max_i64                                 :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata64_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_max_rtn_f32                 :ref:`vdst<amdgpu_synid8_vdst32_0>`,       :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata32_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_max_rtn_f64                 :ref:`vdst<amdgpu_synid8_vdst64_0>`,       :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata64_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_max_rtn_i32                 :ref:`vdst<amdgpu_synid8_vdst32_0>`,       :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata32_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_max_rtn_i64                 :ref:`vdst<amdgpu_synid8_vdst64_0>`,       :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata64_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_max_rtn_u32                 :ref:`vdst<amdgpu_synid8_vdst32_0>`,       :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata32_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_max_rtn_u64                 :ref:`vdst<amdgpu_synid8_vdst64_0>`,       :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata64_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_max_src2_f32                            :ref:`vaddr<amdgpu_synid8_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_max_src2_f64                            :ref:`vaddr<amdgpu_synid8_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_max_src2_i32                            :ref:`vaddr<amdgpu_synid8_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_max_src2_i64                            :ref:`vaddr<amdgpu_synid8_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_max_src2_u32                            :ref:`vaddr<amdgpu_synid8_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_max_src2_u64                            :ref:`vaddr<amdgpu_synid8_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_max_u32                                 :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata32_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_max_u64                                 :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata64_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_min_f32                                 :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata32_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_min_f64                                 :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata64_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_min_i32                                 :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata32_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_min_i64                                 :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata64_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_min_rtn_f32                 :ref:`vdst<amdgpu_synid8_vdst32_0>`,       :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata32_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_min_rtn_f64                 :ref:`vdst<amdgpu_synid8_vdst64_0>`,       :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata64_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_min_rtn_i32                 :ref:`vdst<amdgpu_synid8_vdst32_0>`,       :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata32_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_min_rtn_i64                 :ref:`vdst<amdgpu_synid8_vdst64_0>`,       :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata64_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_min_rtn_u32                 :ref:`vdst<amdgpu_synid8_vdst32_0>`,       :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata32_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_min_rtn_u64                 :ref:`vdst<amdgpu_synid8_vdst64_0>`,       :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata64_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_min_src2_f32                            :ref:`vaddr<amdgpu_synid8_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_min_src2_f64                            :ref:`vaddr<amdgpu_synid8_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_min_src2_i32                            :ref:`vaddr<amdgpu_synid8_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_min_src2_i64                            :ref:`vaddr<amdgpu_synid8_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_min_src2_u32                            :ref:`vaddr<amdgpu_synid8_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_min_src2_u64                            :ref:`vaddr<amdgpu_synid8_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_min_u32                                 :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata32_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_min_u64                                 :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata64_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_mskor_b32                               :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata0<amdgpu_synid8_vdata32_0>`,   :ref:`vdata1<amdgpu_synid8_vdata32_0>`         :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_mskor_b64                               :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata0<amdgpu_synid8_vdata64_0>`,   :ref:`vdata1<amdgpu_synid8_vdata64_0>`         :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_mskor_rtn_b32               :ref:`vdst<amdgpu_synid8_vdst32_0>`,       :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata0<amdgpu_synid8_vdata32_0>`,   :ref:`vdata1<amdgpu_synid8_vdata32_0>`         :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_mskor_rtn_b64               :ref:`vdst<amdgpu_synid8_vdst64_0>`,       :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata0<amdgpu_synid8_vdata64_0>`,   :ref:`vdata1<amdgpu_synid8_vdata64_0>`         :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    \ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|
+    ds_add_f32                                 :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata32_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_add_rtn_f32                 :ref:`vdst<amdgpu_synid8_vdst32_0>`,       :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata32_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_add_rtn_u32                 :ref:`vdst<amdgpu_synid8_vdst32_0>`,       :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata32_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_add_rtn_u64                 :ref:`vdst<amdgpu_synid8_vdst64_0>`,       :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata64_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_add_src2_f32                            :ref:`vaddr<amdgpu_synid8_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_add_src2_u32                            :ref:`vaddr<amdgpu_synid8_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_add_src2_u64                            :ref:`vaddr<amdgpu_synid8_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_add_u32                                 :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata32_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_add_u64                                 :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata64_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_and_b32                                 :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata32_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_and_b64                                 :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata64_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_and_rtn_b32                 :ref:`vdst<amdgpu_synid8_vdst32_0>`,       :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata32_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_and_rtn_b64                 :ref:`vdst<amdgpu_synid8_vdst64_0>`,       :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata64_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_and_src2_b32                            :ref:`vaddr<amdgpu_synid8_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_and_src2_b64                            :ref:`vaddr<amdgpu_synid8_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_append                      :ref:`vdst<amdgpu_synid8_vdst32_0>`                                           :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_bpermute_b32                :ref:`vdst<amdgpu_synid8_vdst32_0>`,       :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata32_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>`
+    ds_cmpst_b32                               :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata0<amdgpu_synid8_vdata32_0>`,   :ref:`vdata1<amdgpu_synid8_vdata32_0>`         :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_cmpst_b64                               :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata0<amdgpu_synid8_vdata64_0>`,   :ref:`vdata1<amdgpu_synid8_vdata64_0>`         :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_cmpst_f32                               :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata0<amdgpu_synid8_vdata32_0>`,   :ref:`vdata1<amdgpu_synid8_vdata32_0>`         :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_cmpst_f64                               :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata0<amdgpu_synid8_vdata64_0>`,   :ref:`vdata1<amdgpu_synid8_vdata64_0>`         :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_cmpst_rtn_b32               :ref:`vdst<amdgpu_synid8_vdst32_0>`,       :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata0<amdgpu_synid8_vdata32_0>`,   :ref:`vdata1<amdgpu_synid8_vdata32_0>`         :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_cmpst_rtn_b64               :ref:`vdst<amdgpu_synid8_vdst64_0>`,       :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata0<amdgpu_synid8_vdata64_0>`,   :ref:`vdata1<amdgpu_synid8_vdata64_0>`         :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_cmpst_rtn_f32               :ref:`vdst<amdgpu_synid8_vdst32_0>`,       :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata0<amdgpu_synid8_vdata32_0>`,   :ref:`vdata1<amdgpu_synid8_vdata32_0>`         :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_cmpst_rtn_f64               :ref:`vdst<amdgpu_synid8_vdst64_0>`,       :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata0<amdgpu_synid8_vdata64_0>`,   :ref:`vdata1<amdgpu_synid8_vdata64_0>`         :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_condxchg32_rtn_b64          :ref:`vdst<amdgpu_synid8_vdst64_0>`,       :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata64_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_consume                     :ref:`vdst<amdgpu_synid8_vdst32_0>`                                           :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_dec_rtn_u32                 :ref:`vdst<amdgpu_synid8_vdst32_0>`,       :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata32_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_dec_rtn_u64                 :ref:`vdst<amdgpu_synid8_vdst64_0>`,       :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata64_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_dec_src2_u32                            :ref:`vaddr<amdgpu_synid8_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_dec_src2_u64                            :ref:`vaddr<amdgpu_synid8_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_dec_u32                                 :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata32_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_dec_u64                                 :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata64_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_gws_barrier                             :ref:`vdata<amdgpu_synid8_vdata32_0>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_gws_init                                :ref:`vdata<amdgpu_synid8_vdata32_0>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_gws_sema_br                             :ref:`vdata<amdgpu_synid8_vdata32_0>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_gws_sema_p                                                                 :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_gws_sema_release_all                                                       :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_gws_sema_v                                                                 :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_inc_rtn_u32                 :ref:`vdst<amdgpu_synid8_vdst32_0>`,       :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata32_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_inc_rtn_u64                 :ref:`vdst<amdgpu_synid8_vdst64_0>`,       :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata64_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_inc_src2_u32                            :ref:`vaddr<amdgpu_synid8_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_inc_src2_u64                            :ref:`vaddr<amdgpu_synid8_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_inc_u32                                 :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata32_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_inc_u64                                 :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata64_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_max_f32                                 :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata32_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_max_f64                                 :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata64_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_max_i32                                 :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata32_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_max_i64                                 :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata64_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_max_rtn_f32                 :ref:`vdst<amdgpu_synid8_vdst32_0>`,       :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata32_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_max_rtn_f64                 :ref:`vdst<amdgpu_synid8_vdst64_0>`,       :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata64_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_max_rtn_i32                 :ref:`vdst<amdgpu_synid8_vdst32_0>`,       :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata32_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_max_rtn_i64                 :ref:`vdst<amdgpu_synid8_vdst64_0>`,       :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata64_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_max_rtn_u32                 :ref:`vdst<amdgpu_synid8_vdst32_0>`,       :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata32_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_max_rtn_u64                 :ref:`vdst<amdgpu_synid8_vdst64_0>`,       :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata64_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_max_src2_f32                            :ref:`vaddr<amdgpu_synid8_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_max_src2_f64                            :ref:`vaddr<amdgpu_synid8_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_max_src2_i32                            :ref:`vaddr<amdgpu_synid8_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_max_src2_i64                            :ref:`vaddr<amdgpu_synid8_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_max_src2_u32                            :ref:`vaddr<amdgpu_synid8_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_max_src2_u64                            :ref:`vaddr<amdgpu_synid8_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_max_u32                                 :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata32_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_max_u64                                 :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata64_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_min_f32                                 :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata32_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_min_f64                                 :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata64_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_min_i32                                 :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata32_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_min_i64                                 :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata64_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_min_rtn_f32                 :ref:`vdst<amdgpu_synid8_vdst32_0>`,       :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata32_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_min_rtn_f64                 :ref:`vdst<amdgpu_synid8_vdst64_0>`,       :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata64_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_min_rtn_i32                 :ref:`vdst<amdgpu_synid8_vdst32_0>`,       :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata32_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_min_rtn_i64                 :ref:`vdst<amdgpu_synid8_vdst64_0>`,       :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata64_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_min_rtn_u32                 :ref:`vdst<amdgpu_synid8_vdst32_0>`,       :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata32_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_min_rtn_u64                 :ref:`vdst<amdgpu_synid8_vdst64_0>`,       :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata64_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_min_src2_f32                            :ref:`vaddr<amdgpu_synid8_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_min_src2_f64                            :ref:`vaddr<amdgpu_synid8_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_min_src2_i32                            :ref:`vaddr<amdgpu_synid8_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_min_src2_i64                            :ref:`vaddr<amdgpu_synid8_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_min_src2_u32                            :ref:`vaddr<amdgpu_synid8_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_min_src2_u64                            :ref:`vaddr<amdgpu_synid8_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_min_u32                                 :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata32_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_min_u64                                 :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata64_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_mskor_b32                               :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata0<amdgpu_synid8_vdata32_0>`,   :ref:`vdata1<amdgpu_synid8_vdata32_0>`         :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_mskor_b64                               :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata0<amdgpu_synid8_vdata64_0>`,   :ref:`vdata1<amdgpu_synid8_vdata64_0>`         :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_mskor_rtn_b32               :ref:`vdst<amdgpu_synid8_vdst32_0>`,       :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata0<amdgpu_synid8_vdata32_0>`,   :ref:`vdata1<amdgpu_synid8_vdata32_0>`         :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_mskor_rtn_b64               :ref:`vdst<amdgpu_synid8_vdst64_0>`,       :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata0<amdgpu_synid8_vdata64_0>`,   :ref:`vdata1<amdgpu_synid8_vdata64_0>`         :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
     ds_nop
-    ds_or_b32                                  :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata32_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_or_b64                                  :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata64_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_or_rtn_b32                  :ref:`vdst<amdgpu_synid8_vdst32_0>`,       :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata32_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_or_rtn_b64                  :ref:`vdst<amdgpu_synid8_vdst64_0>`,       :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata64_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_or_src2_b32                             :ref:`vaddr<amdgpu_synid8_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_or_src2_b64                             :ref:`vaddr<amdgpu_synid8_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_ordered_count               :ref:`vdst<amdgpu_synid8_vdst32_0>`,       :ref:`vaddr<amdgpu_synid8_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_permute_b32                 :ref:`vdst<amdgpu_synid8_vdst32_0>`,       :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata32_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>`
-    ds_read2_b32                   :ref:`vdst<amdgpu_synid8_vdst64_0>`::ref:`b32x2<amdgpu_synid8_type_dev>`, :ref:`vaddr<amdgpu_synid8_addr_ds>`                              :ref:`ds_offset8<amdgpu_synid_ds_offset8>` :ref:`ds_offset8<amdgpu_synid_ds_offset8>` :ref:`gds<amdgpu_synid_gds>`
-    ds_read2_b64                   :ref:`vdst<amdgpu_synid8_vdst128_0>`::ref:`b64x2<amdgpu_synid8_type_dev>`, :ref:`vaddr<amdgpu_synid8_addr_ds>`                              :ref:`ds_offset8<amdgpu_synid_ds_offset8>` :ref:`ds_offset8<amdgpu_synid_ds_offset8>` :ref:`gds<amdgpu_synid_gds>`
-    ds_read2st64_b32               :ref:`vdst<amdgpu_synid8_vdst64_0>`::ref:`b32x2<amdgpu_synid8_type_dev>`, :ref:`vaddr<amdgpu_synid8_addr_ds>`                              :ref:`ds_offset8<amdgpu_synid_ds_offset8>` :ref:`ds_offset8<amdgpu_synid_ds_offset8>` :ref:`gds<amdgpu_synid_gds>`
-    ds_read2st64_b64               :ref:`vdst<amdgpu_synid8_vdst128_0>`::ref:`b64x2<amdgpu_synid8_type_dev>`, :ref:`vaddr<amdgpu_synid8_addr_ds>`                              :ref:`ds_offset8<amdgpu_synid_ds_offset8>` :ref:`ds_offset8<amdgpu_synid_ds_offset8>` :ref:`gds<amdgpu_synid_gds>`
-    ds_read_b128                   :ref:`vdst<amdgpu_synid8_vdst128_0>`,       :ref:`vaddr<amdgpu_synid8_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_read_b32                    :ref:`vdst<amdgpu_synid8_vdst32_0>`,       :ref:`vaddr<amdgpu_synid8_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_read_b64                    :ref:`vdst<amdgpu_synid8_vdst64_0>`,       :ref:`vaddr<amdgpu_synid8_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_read_b96                    :ref:`vdst<amdgpu_synid8_vdst96_0>`,       :ref:`vaddr<amdgpu_synid8_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_read_i16                    :ref:`vdst<amdgpu_synid8_vdst32_0>`,       :ref:`vaddr<amdgpu_synid8_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_read_i8                     :ref:`vdst<amdgpu_synid8_vdst32_0>`,       :ref:`vaddr<amdgpu_synid8_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_read_u16                    :ref:`vdst<amdgpu_synid8_vdst32_0>`,       :ref:`vaddr<amdgpu_synid8_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_read_u8                     :ref:`vdst<amdgpu_synid8_vdst32_0>`,       :ref:`vaddr<amdgpu_synid8_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_rsub_rtn_u32                :ref:`vdst<amdgpu_synid8_vdst32_0>`,       :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata32_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_rsub_rtn_u64                :ref:`vdst<amdgpu_synid8_vdst64_0>`,       :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata64_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_rsub_src2_u32                           :ref:`vaddr<amdgpu_synid8_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_rsub_src2_u64                           :ref:`vaddr<amdgpu_synid8_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_rsub_u32                                :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata32_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_rsub_u64                                :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata64_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_sub_rtn_u32                 :ref:`vdst<amdgpu_synid8_vdst32_0>`,       :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata32_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_sub_rtn_u64                 :ref:`vdst<amdgpu_synid8_vdst64_0>`,       :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata64_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_sub_src2_u32                            :ref:`vaddr<amdgpu_synid8_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_sub_src2_u64                            :ref:`vaddr<amdgpu_synid8_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_sub_u32                                 :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata32_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_sub_u64                                 :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata64_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_swizzle_b32                 :ref:`vdst<amdgpu_synid8_vdst32_0>`,       :ref:`vaddr<amdgpu_synid8_addr_ds>`                              :ref:`sw_offset16<amdgpu_synid_sw_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_wrap_rtn_b32                :ref:`vdst<amdgpu_synid8_vdst32_0>`,       :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata0<amdgpu_synid8_vdata32_0>`,   :ref:`vdata1<amdgpu_synid8_vdata32_0>`         :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_write2_b32                              :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata0<amdgpu_synid8_vdata32_0>`,   :ref:`vdata1<amdgpu_synid8_vdata32_0>`         :ref:`ds_offset8<amdgpu_synid_ds_offset8>` :ref:`ds_offset8<amdgpu_synid_ds_offset8>` :ref:`gds<amdgpu_synid_gds>`
-    ds_write2_b64                              :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata0<amdgpu_synid8_vdata64_0>`,   :ref:`vdata1<amdgpu_synid8_vdata64_0>`         :ref:`ds_offset8<amdgpu_synid_ds_offset8>` :ref:`ds_offset8<amdgpu_synid_ds_offset8>` :ref:`gds<amdgpu_synid_gds>`
-    ds_write2st64_b32                          :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata0<amdgpu_synid8_vdata32_0>`,   :ref:`vdata1<amdgpu_synid8_vdata32_0>`         :ref:`ds_offset8<amdgpu_synid_ds_offset8>` :ref:`ds_offset8<amdgpu_synid_ds_offset8>` :ref:`gds<amdgpu_synid_gds>`
-    ds_write2st64_b64                          :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata0<amdgpu_synid8_vdata64_0>`,   :ref:`vdata1<amdgpu_synid8_vdata64_0>`         :ref:`ds_offset8<amdgpu_synid_ds_offset8>` :ref:`ds_offset8<amdgpu_synid_ds_offset8>` :ref:`gds<amdgpu_synid_gds>`
-    ds_write_b128                              :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata128_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_write_b16                               :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata32_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_write_b32                               :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata32_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_write_b64                               :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata64_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_write_b8                                :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata32_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_write_b96                               :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata96_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_write_src2_b32                          :ref:`vaddr<amdgpu_synid8_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_write_src2_b64                          :ref:`vaddr<amdgpu_synid8_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_wrxchg2_rtn_b32             :ref:`vdst<amdgpu_synid8_vdst64_0>`::ref:`b32x2<amdgpu_synid8_type_dev>`, :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata0<amdgpu_synid8_vdata32_0>`,   :ref:`vdata1<amdgpu_synid8_vdata32_0>`         :ref:`ds_offset8<amdgpu_synid_ds_offset8>` :ref:`ds_offset8<amdgpu_synid_ds_offset8>` :ref:`gds<amdgpu_synid_gds>`
-    ds_wrxchg2_rtn_b64             :ref:`vdst<amdgpu_synid8_vdst128_0>`::ref:`b64x2<amdgpu_synid8_type_dev>`, :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata0<amdgpu_synid8_vdata64_0>`,   :ref:`vdata1<amdgpu_synid8_vdata64_0>`         :ref:`ds_offset8<amdgpu_synid_ds_offset8>` :ref:`ds_offset8<amdgpu_synid_ds_offset8>` :ref:`gds<amdgpu_synid_gds>`
-    ds_wrxchg2st64_rtn_b32         :ref:`vdst<amdgpu_synid8_vdst64_0>`::ref:`b32x2<amdgpu_synid8_type_dev>`, :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata0<amdgpu_synid8_vdata32_0>`,   :ref:`vdata1<amdgpu_synid8_vdata32_0>`         :ref:`ds_offset8<amdgpu_synid_ds_offset8>` :ref:`ds_offset8<amdgpu_synid_ds_offset8>` :ref:`gds<amdgpu_synid_gds>`
-    ds_wrxchg2st64_rtn_b64         :ref:`vdst<amdgpu_synid8_vdst128_0>`::ref:`b64x2<amdgpu_synid8_type_dev>`, :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata0<amdgpu_synid8_vdata64_0>`,   :ref:`vdata1<amdgpu_synid8_vdata64_0>`         :ref:`ds_offset8<amdgpu_synid_ds_offset8>` :ref:`ds_offset8<amdgpu_synid_ds_offset8>` :ref:`gds<amdgpu_synid_gds>`
-    ds_wrxchg_rtn_b32              :ref:`vdst<amdgpu_synid8_vdst32_0>`,       :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata32_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_wrxchg_rtn_b64              :ref:`vdst<amdgpu_synid8_vdst64_0>`,       :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata64_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_xor_b32                                 :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata32_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_xor_b64                                 :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata64_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_xor_rtn_b32                 :ref:`vdst<amdgpu_synid8_vdst32_0>`,       :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata32_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_xor_rtn_b64                 :ref:`vdst<amdgpu_synid8_vdst64_0>`,       :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata64_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_xor_src2_b32                            :ref:`vaddr<amdgpu_synid8_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_xor_src2_b64                            :ref:`vaddr<amdgpu_synid8_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_or_b32                                  :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata32_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_or_b64                                  :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata64_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_or_rtn_b32                  :ref:`vdst<amdgpu_synid8_vdst32_0>`,       :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata32_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_or_rtn_b64                  :ref:`vdst<amdgpu_synid8_vdst64_0>`,       :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata64_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_or_src2_b32                             :ref:`vaddr<amdgpu_synid8_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_or_src2_b64                             :ref:`vaddr<amdgpu_synid8_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_ordered_count               :ref:`vdst<amdgpu_synid8_vdst32_0>`,       :ref:`vaddr<amdgpu_synid8_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_permute_b32                 :ref:`vdst<amdgpu_synid8_vdst32_0>`,       :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata32_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>`
+    ds_read2_b32                   :ref:`vdst<amdgpu_synid8_vdst64_0>`::ref:`b32x2<amdgpu_synid8_type_dev>`, :ref:`vaddr<amdgpu_synid8_addr_ds>`                              :ref:`offset8<amdgpu_synid_ds_offset8>` :ref:`offset8<amdgpu_synid_ds_offset8>` :ref:`gds<amdgpu_synid_gds>`
+    ds_read2_b64                   :ref:`vdst<amdgpu_synid8_vdst128_0>`::ref:`b64x2<amdgpu_synid8_type_dev>`, :ref:`vaddr<amdgpu_synid8_addr_ds>`                              :ref:`offset8<amdgpu_synid_ds_offset8>` :ref:`offset8<amdgpu_synid_ds_offset8>` :ref:`gds<amdgpu_synid_gds>`
+    ds_read2st64_b32               :ref:`vdst<amdgpu_synid8_vdst64_0>`::ref:`b32x2<amdgpu_synid8_type_dev>`, :ref:`vaddr<amdgpu_synid8_addr_ds>`                              :ref:`offset8<amdgpu_synid_ds_offset8>` :ref:`offset8<amdgpu_synid_ds_offset8>` :ref:`gds<amdgpu_synid_gds>`
+    ds_read2st64_b64               :ref:`vdst<amdgpu_synid8_vdst128_0>`::ref:`b64x2<amdgpu_synid8_type_dev>`, :ref:`vaddr<amdgpu_synid8_addr_ds>`                              :ref:`offset8<amdgpu_synid_ds_offset8>` :ref:`offset8<amdgpu_synid_ds_offset8>` :ref:`gds<amdgpu_synid_gds>`
+    ds_read_b128                   :ref:`vdst<amdgpu_synid8_vdst128_0>`,       :ref:`vaddr<amdgpu_synid8_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_read_b32                    :ref:`vdst<amdgpu_synid8_vdst32_0>`,       :ref:`vaddr<amdgpu_synid8_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_read_b64                    :ref:`vdst<amdgpu_synid8_vdst64_0>`,       :ref:`vaddr<amdgpu_synid8_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_read_b96                    :ref:`vdst<amdgpu_synid8_vdst96_0>`,       :ref:`vaddr<amdgpu_synid8_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_read_i16                    :ref:`vdst<amdgpu_synid8_vdst32_0>`,       :ref:`vaddr<amdgpu_synid8_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_read_i8                     :ref:`vdst<amdgpu_synid8_vdst32_0>`,       :ref:`vaddr<amdgpu_synid8_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_read_u16                    :ref:`vdst<amdgpu_synid8_vdst32_0>`,       :ref:`vaddr<amdgpu_synid8_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_read_u8                     :ref:`vdst<amdgpu_synid8_vdst32_0>`,       :ref:`vaddr<amdgpu_synid8_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_rsub_rtn_u32                :ref:`vdst<amdgpu_synid8_vdst32_0>`,       :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata32_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_rsub_rtn_u64                :ref:`vdst<amdgpu_synid8_vdst64_0>`,       :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata64_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_rsub_src2_u32                           :ref:`vaddr<amdgpu_synid8_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_rsub_src2_u64                           :ref:`vaddr<amdgpu_synid8_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_rsub_u32                                :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata32_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_rsub_u64                                :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata64_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_sub_rtn_u32                 :ref:`vdst<amdgpu_synid8_vdst32_0>`,       :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata32_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_sub_rtn_u64                 :ref:`vdst<amdgpu_synid8_vdst64_0>`,       :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata64_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_sub_src2_u32                            :ref:`vaddr<amdgpu_synid8_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_sub_src2_u64                            :ref:`vaddr<amdgpu_synid8_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_sub_u32                                 :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata32_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_sub_u64                                 :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata64_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_swizzle_b32                 :ref:`vdst<amdgpu_synid8_vdst32_0>`,       :ref:`vaddr<amdgpu_synid8_addr_ds>`                              :ref:`pattern<amdgpu_synid_sw_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_wrap_rtn_b32                :ref:`vdst<amdgpu_synid8_vdst32_0>`,       :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata0<amdgpu_synid8_vdata32_0>`,   :ref:`vdata1<amdgpu_synid8_vdata32_0>`         :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_write2_b32                              :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata0<amdgpu_synid8_vdata32_0>`,   :ref:`vdata1<amdgpu_synid8_vdata32_0>`         :ref:`offset8<amdgpu_synid_ds_offset8>` :ref:`offset8<amdgpu_synid_ds_offset8>` :ref:`gds<amdgpu_synid_gds>`
+    ds_write2_b64                              :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata0<amdgpu_synid8_vdata64_0>`,   :ref:`vdata1<amdgpu_synid8_vdata64_0>`         :ref:`offset8<amdgpu_synid_ds_offset8>` :ref:`offset8<amdgpu_synid_ds_offset8>` :ref:`gds<amdgpu_synid_gds>`
+    ds_write2st64_b32                          :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata0<amdgpu_synid8_vdata32_0>`,   :ref:`vdata1<amdgpu_synid8_vdata32_0>`         :ref:`offset8<amdgpu_synid_ds_offset8>` :ref:`offset8<amdgpu_synid_ds_offset8>` :ref:`gds<amdgpu_synid_gds>`
+    ds_write2st64_b64                          :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata0<amdgpu_synid8_vdata64_0>`,   :ref:`vdata1<amdgpu_synid8_vdata64_0>`         :ref:`offset8<amdgpu_synid_ds_offset8>` :ref:`offset8<amdgpu_synid_ds_offset8>` :ref:`gds<amdgpu_synid_gds>`
+    ds_write_b128                              :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata128_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_write_b16                               :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata32_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_write_b32                               :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata32_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_write_b64                               :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata64_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_write_b8                                :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata32_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_write_b96                               :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata96_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_write_src2_b32                          :ref:`vaddr<amdgpu_synid8_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_write_src2_b64                          :ref:`vaddr<amdgpu_synid8_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_wrxchg2_rtn_b32             :ref:`vdst<amdgpu_synid8_vdst64_0>`::ref:`b32x2<amdgpu_synid8_type_dev>`, :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata0<amdgpu_synid8_vdata32_0>`,   :ref:`vdata1<amdgpu_synid8_vdata32_0>`         :ref:`offset8<amdgpu_synid_ds_offset8>` :ref:`offset8<amdgpu_synid_ds_offset8>` :ref:`gds<amdgpu_synid_gds>`
+    ds_wrxchg2_rtn_b64             :ref:`vdst<amdgpu_synid8_vdst128_0>`::ref:`b64x2<amdgpu_synid8_type_dev>`, :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata0<amdgpu_synid8_vdata64_0>`,   :ref:`vdata1<amdgpu_synid8_vdata64_0>`         :ref:`offset8<amdgpu_synid_ds_offset8>` :ref:`offset8<amdgpu_synid_ds_offset8>` :ref:`gds<amdgpu_synid_gds>`
+    ds_wrxchg2st64_rtn_b32         :ref:`vdst<amdgpu_synid8_vdst64_0>`::ref:`b32x2<amdgpu_synid8_type_dev>`, :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata0<amdgpu_synid8_vdata32_0>`,   :ref:`vdata1<amdgpu_synid8_vdata32_0>`         :ref:`offset8<amdgpu_synid_ds_offset8>` :ref:`offset8<amdgpu_synid_ds_offset8>` :ref:`gds<amdgpu_synid_gds>`
+    ds_wrxchg2st64_rtn_b64         :ref:`vdst<amdgpu_synid8_vdst128_0>`::ref:`b64x2<amdgpu_synid8_type_dev>`, :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata0<amdgpu_synid8_vdata64_0>`,   :ref:`vdata1<amdgpu_synid8_vdata64_0>`         :ref:`offset8<amdgpu_synid_ds_offset8>` :ref:`offset8<amdgpu_synid_ds_offset8>` :ref:`gds<amdgpu_synid_gds>`
+    ds_wrxchg_rtn_b32              :ref:`vdst<amdgpu_synid8_vdst32_0>`,       :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata32_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_wrxchg_rtn_b64              :ref:`vdst<amdgpu_synid8_vdst64_0>`,       :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata64_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_xor_b32                                 :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata32_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_xor_b64                                 :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata64_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_xor_rtn_b32                 :ref:`vdst<amdgpu_synid8_vdst32_0>`,       :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata32_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_xor_rtn_b64                 :ref:`vdst<amdgpu_synid8_vdst64_0>`,       :ref:`vaddr<amdgpu_synid8_addr_ds>`,    :ref:`vdata<amdgpu_synid8_vdata64_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_xor_src2_b32                            :ref:`vaddr<amdgpu_synid8_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_xor_src2_b64                            :ref:`vaddr<amdgpu_synid8_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
 
 EXP
 -----------------------
@@ -193,7 +193,7 @@ FLAT
 .. parsed-literal::
 
     **INSTRUCTION**                    **DST**           **SRC0**      **SRC1**             **MODIFIERS**
-    \ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|
+    \ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|
     flat_atomic_add                :ref:`vdst<amdgpu_synid8_dst_flat_atomic32>`::ref:`opt<amdgpu_synid8_opt>`,     :ref:`vaddr<amdgpu_synid8_addr_flat>`,    :ref:`vdata<amdgpu_synid8_vdata32_0>`            :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
     flat_atomic_add_x2             :ref:`vdst<amdgpu_synid8_dst_flat_atomic64>`::ref:`opt<amdgpu_synid8_opt>`,     :ref:`vaddr<amdgpu_synid8_addr_flat>`,    :ref:`vdata<amdgpu_synid8_vdata64_0>`            :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
     flat_atomic_and                :ref:`vdst<amdgpu_synid8_dst_flat_atomic32>`::ref:`opt<amdgpu_synid8_opt>`,     :ref:`vaddr<amdgpu_synid8_addr_flat>`,    :ref:`vdata<amdgpu_synid8_vdata32_0>`            :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
@@ -337,65 +337,65 @@ MUBUF
 
 .. parsed-literal::
 
-    **INSTRUCTION**                    **DST**       **SRC0**             **SRC1**      **SRC2**      **SRC3**           **MODIFIERS**
-    \ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|
-    buffer_atomic_add                        :ref:`vdata<amdgpu_synid8_data_buf_atomic32>`::ref:`dst<amdgpu_synid8_ret>`,       :ref:`vaddr<amdgpu_synid8_addr_buf>`,    :ref:`srsrc<amdgpu_synid8_rsrc_buf>`,    :ref:`soffset<amdgpu_synid8_offset_buf>`        :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_atomic_add_x2                     :ref:`vdata<amdgpu_synid8_data_buf_atomic64>`::ref:`dst<amdgpu_synid8_ret>`,       :ref:`vaddr<amdgpu_synid8_addr_buf>`,    :ref:`srsrc<amdgpu_synid8_rsrc_buf>`,    :ref:`soffset<amdgpu_synid8_offset_buf>`        :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_atomic_and                        :ref:`vdata<amdgpu_synid8_data_buf_atomic32>`::ref:`dst<amdgpu_synid8_ret>`,       :ref:`vaddr<amdgpu_synid8_addr_buf>`,    :ref:`srsrc<amdgpu_synid8_rsrc_buf>`,    :ref:`soffset<amdgpu_synid8_offset_buf>`        :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_atomic_and_x2                     :ref:`vdata<amdgpu_synid8_data_buf_atomic64>`::ref:`dst<amdgpu_synid8_ret>`,       :ref:`vaddr<amdgpu_synid8_addr_buf>`,    :ref:`srsrc<amdgpu_synid8_rsrc_buf>`,    :ref:`soffset<amdgpu_synid8_offset_buf>`        :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_atomic_cmpswap                    :ref:`vdata<amdgpu_synid8_data_buf_atomic64>`::ref:`dst<amdgpu_synid8_ret>`::ref:`b32x2<amdgpu_synid8_type_dev>`, :ref:`vaddr<amdgpu_synid8_addr_buf>`,    :ref:`srsrc<amdgpu_synid8_rsrc_buf>`,    :ref:`soffset<amdgpu_synid8_offset_buf>`        :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_atomic_cmpswap_x2                 :ref:`vdata<amdgpu_synid8_data_buf_atomic128>`::ref:`dst<amdgpu_synid8_ret>`::ref:`b64x2<amdgpu_synid8_type_dev>`, :ref:`vaddr<amdgpu_synid8_addr_buf>`,    :ref:`srsrc<amdgpu_synid8_rsrc_buf>`,    :ref:`soffset<amdgpu_synid8_offset_buf>`        :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_atomic_dec                        :ref:`vdata<amdgpu_synid8_data_buf_atomic32>`::ref:`dst<amdgpu_synid8_ret>`::ref:`u32<amdgpu_synid8_type_dev>`,   :ref:`vaddr<amdgpu_synid8_addr_buf>`,    :ref:`srsrc<amdgpu_synid8_rsrc_buf>`,    :ref:`soffset<amdgpu_synid8_offset_buf>`        :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_atomic_dec_x2                     :ref:`vdata<amdgpu_synid8_data_buf_atomic64>`::ref:`dst<amdgpu_synid8_ret>`::ref:`u64<amdgpu_synid8_type_dev>`,   :ref:`vaddr<amdgpu_synid8_addr_buf>`,    :ref:`srsrc<amdgpu_synid8_rsrc_buf>`,    :ref:`soffset<amdgpu_synid8_offset_buf>`        :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_atomic_inc                        :ref:`vdata<amdgpu_synid8_data_buf_atomic32>`::ref:`dst<amdgpu_synid8_ret>`::ref:`u32<amdgpu_synid8_type_dev>`,   :ref:`vaddr<amdgpu_synid8_addr_buf>`,    :ref:`srsrc<amdgpu_synid8_rsrc_buf>`,    :ref:`soffset<amdgpu_synid8_offset_buf>`        :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_atomic_inc_x2                     :ref:`vdata<amdgpu_synid8_data_buf_atomic64>`::ref:`dst<amdgpu_synid8_ret>`::ref:`u64<amdgpu_synid8_type_dev>`,   :ref:`vaddr<amdgpu_synid8_addr_buf>`,    :ref:`srsrc<amdgpu_synid8_rsrc_buf>`,    :ref:`soffset<amdgpu_synid8_offset_buf>`        :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_atomic_or                         :ref:`vdata<amdgpu_synid8_data_buf_atomic32>`::ref:`dst<amdgpu_synid8_ret>`,       :ref:`vaddr<amdgpu_synid8_addr_buf>`,    :ref:`srsrc<amdgpu_synid8_rsrc_buf>`,    :ref:`soffset<amdgpu_synid8_offset_buf>`        :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_atomic_or_x2                      :ref:`vdata<amdgpu_synid8_data_buf_atomic64>`::ref:`dst<amdgpu_synid8_ret>`,       :ref:`vaddr<amdgpu_synid8_addr_buf>`,    :ref:`srsrc<amdgpu_synid8_rsrc_buf>`,    :ref:`soffset<amdgpu_synid8_offset_buf>`        :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_atomic_smax                       :ref:`vdata<amdgpu_synid8_data_buf_atomic32>`::ref:`dst<amdgpu_synid8_ret>`::ref:`s32<amdgpu_synid8_type_dev>`,   :ref:`vaddr<amdgpu_synid8_addr_buf>`,    :ref:`srsrc<amdgpu_synid8_rsrc_buf>`,    :ref:`soffset<amdgpu_synid8_offset_buf>`        :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_atomic_smax_x2                    :ref:`vdata<amdgpu_synid8_data_buf_atomic64>`::ref:`dst<amdgpu_synid8_ret>`::ref:`s64<amdgpu_synid8_type_dev>`,   :ref:`vaddr<amdgpu_synid8_addr_buf>`,    :ref:`srsrc<amdgpu_synid8_rsrc_buf>`,    :ref:`soffset<amdgpu_synid8_offset_buf>`        :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_atomic_smin                       :ref:`vdata<amdgpu_synid8_data_buf_atomic32>`::ref:`dst<amdgpu_synid8_ret>`::ref:`s32<amdgpu_synid8_type_dev>`,   :ref:`vaddr<amdgpu_synid8_addr_buf>`,    :ref:`srsrc<amdgpu_synid8_rsrc_buf>`,    :ref:`soffset<amdgpu_synid8_offset_buf>`        :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_atomic_smin_x2                    :ref:`vdata<amdgpu_synid8_data_buf_atomic64>`::ref:`dst<amdgpu_synid8_ret>`::ref:`s64<amdgpu_synid8_type_dev>`,   :ref:`vaddr<amdgpu_synid8_addr_buf>`,    :ref:`srsrc<amdgpu_synid8_rsrc_buf>`,    :ref:`soffset<amdgpu_synid8_offset_buf>`        :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_atomic_sub                        :ref:`vdata<amdgpu_synid8_data_buf_atomic32>`::ref:`dst<amdgpu_synid8_ret>`,       :ref:`vaddr<amdgpu_synid8_addr_buf>`,    :ref:`srsrc<amdgpu_synid8_rsrc_buf>`,    :ref:`soffset<amdgpu_synid8_offset_buf>`        :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_atomic_sub_x2                     :ref:`vdata<amdgpu_synid8_data_buf_atomic64>`::ref:`dst<amdgpu_synid8_ret>`,       :ref:`vaddr<amdgpu_synid8_addr_buf>`,    :ref:`srsrc<amdgpu_synid8_rsrc_buf>`,    :ref:`soffset<amdgpu_synid8_offset_buf>`        :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_atomic_swap                       :ref:`vdata<amdgpu_synid8_data_buf_atomic32>`::ref:`dst<amdgpu_synid8_ret>`,       :ref:`vaddr<amdgpu_synid8_addr_buf>`,    :ref:`srsrc<amdgpu_synid8_rsrc_buf>`,    :ref:`soffset<amdgpu_synid8_offset_buf>`        :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_atomic_swap_x2                    :ref:`vdata<amdgpu_synid8_data_buf_atomic64>`::ref:`dst<amdgpu_synid8_ret>`,       :ref:`vaddr<amdgpu_synid8_addr_buf>`,    :ref:`srsrc<amdgpu_synid8_rsrc_buf>`,    :ref:`soffset<amdgpu_synid8_offset_buf>`        :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_atomic_umax                       :ref:`vdata<amdgpu_synid8_data_buf_atomic32>`::ref:`dst<amdgpu_synid8_ret>`::ref:`u32<amdgpu_synid8_type_dev>`,   :ref:`vaddr<amdgpu_synid8_addr_buf>`,    :ref:`srsrc<amdgpu_synid8_rsrc_buf>`,    :ref:`soffset<amdgpu_synid8_offset_buf>`        :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_atomic_umax_x2                    :ref:`vdata<amdgpu_synid8_data_buf_atomic64>`::ref:`dst<amdgpu_synid8_ret>`::ref:`u64<amdgpu_synid8_type_dev>`,   :ref:`vaddr<amdgpu_synid8_addr_buf>`,    :ref:`srsrc<amdgpu_synid8_rsrc_buf>`,    :ref:`soffset<amdgpu_synid8_offset_buf>`        :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_atomic_umin                       :ref:`vdata<amdgpu_synid8_data_buf_atomic32>`::ref:`dst<amdgpu_synid8_ret>`::ref:`u32<amdgpu_synid8_type_dev>`,   :ref:`vaddr<amdgpu_synid8_addr_buf>`,    :ref:`srsrc<amdgpu_synid8_rsrc_buf>`,    :ref:`soffset<amdgpu_synid8_offset_buf>`        :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_atomic_umin_x2                    :ref:`vdata<amdgpu_synid8_data_buf_atomic64>`::ref:`dst<amdgpu_synid8_ret>`::ref:`u64<amdgpu_synid8_type_dev>`,   :ref:`vaddr<amdgpu_synid8_addr_buf>`,    :ref:`srsrc<amdgpu_synid8_rsrc_buf>`,    :ref:`soffset<amdgpu_synid8_offset_buf>`        :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_atomic_xor                        :ref:`vdata<amdgpu_synid8_data_buf_atomic32>`::ref:`dst<amdgpu_synid8_ret>`,       :ref:`vaddr<amdgpu_synid8_addr_buf>`,    :ref:`srsrc<amdgpu_synid8_rsrc_buf>`,    :ref:`soffset<amdgpu_synid8_offset_buf>`        :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_atomic_xor_x2                     :ref:`vdata<amdgpu_synid8_data_buf_atomic64>`::ref:`dst<amdgpu_synid8_ret>`,       :ref:`vaddr<amdgpu_synid8_addr_buf>`,    :ref:`srsrc<amdgpu_synid8_rsrc_buf>`,    :ref:`soffset<amdgpu_synid8_offset_buf>`        :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_load_dword              :ref:`vdst<amdgpu_synid8_dst_buf_lds>`,     :ref:`vaddr<amdgpu_synid8_addr_buf>`,           :ref:`srsrc<amdgpu_synid8_rsrc_buf>`,    :ref:`soffset<amdgpu_synid8_offset_buf>`                  :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`lds<amdgpu_synid_lds>`
-    buffer_load_dwordx2            :ref:`vdst<amdgpu_synid8_dst_buf_64>`,     :ref:`vaddr<amdgpu_synid8_addr_buf>`,           :ref:`srsrc<amdgpu_synid8_rsrc_buf>`,    :ref:`soffset<amdgpu_synid8_offset_buf>`                  :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_load_dwordx3            :ref:`vdst<amdgpu_synid8_dst_buf_96>`,     :ref:`vaddr<amdgpu_synid8_addr_buf>`,           :ref:`srsrc<amdgpu_synid8_rsrc_buf>`,    :ref:`soffset<amdgpu_synid8_offset_buf>`                  :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_load_dwordx4            :ref:`vdst<amdgpu_synid8_dst_buf_128>`,     :ref:`vaddr<amdgpu_synid8_addr_buf>`,           :ref:`srsrc<amdgpu_synid8_rsrc_buf>`,    :ref:`soffset<amdgpu_synid8_offset_buf>`                  :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_load_format_d16_x       :ref:`vdst<amdgpu_synid8_dst_buf_d16_32>`,     :ref:`vaddr<amdgpu_synid8_addr_buf>`,           :ref:`srsrc<amdgpu_synid8_rsrc_buf>`,    :ref:`soffset<amdgpu_synid8_offset_buf>`                  :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_load_format_d16_xy      :ref:`vdst<amdgpu_synid8_dst_buf_d16_64>`,     :ref:`vaddr<amdgpu_synid8_addr_buf>`,           :ref:`srsrc<amdgpu_synid8_rsrc_buf>`,    :ref:`soffset<amdgpu_synid8_offset_buf>`                  :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_load_format_d16_xyz     :ref:`vdst<amdgpu_synid8_dst_buf_d16_96>`,     :ref:`vaddr<amdgpu_synid8_addr_buf>`,           :ref:`srsrc<amdgpu_synid8_rsrc_buf>`,    :ref:`soffset<amdgpu_synid8_offset_buf>`                  :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_load_format_d16_xyzw    :ref:`vdst<amdgpu_synid8_dst_buf_d16_128>`,     :ref:`vaddr<amdgpu_synid8_addr_buf>`,           :ref:`srsrc<amdgpu_synid8_rsrc_buf>`,    :ref:`soffset<amdgpu_synid8_offset_buf>`                  :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_load_format_x           :ref:`vdst<amdgpu_synid8_dst_buf_lds>`,     :ref:`vaddr<amdgpu_synid8_addr_buf>`,           :ref:`srsrc<amdgpu_synid8_rsrc_buf>`,    :ref:`soffset<amdgpu_synid8_offset_buf>`                  :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`lds<amdgpu_synid_lds>`
-    buffer_load_format_xy          :ref:`vdst<amdgpu_synid8_dst_buf_64>`,     :ref:`vaddr<amdgpu_synid8_addr_buf>`,           :ref:`srsrc<amdgpu_synid8_rsrc_buf>`,    :ref:`soffset<amdgpu_synid8_offset_buf>`                  :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_load_format_xyz         :ref:`vdst<amdgpu_synid8_dst_buf_96>`,     :ref:`vaddr<amdgpu_synid8_addr_buf>`,           :ref:`srsrc<amdgpu_synid8_rsrc_buf>`,    :ref:`soffset<amdgpu_synid8_offset_buf>`                  :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_load_format_xyzw        :ref:`vdst<amdgpu_synid8_dst_buf_128>`,     :ref:`vaddr<amdgpu_synid8_addr_buf>`,           :ref:`srsrc<amdgpu_synid8_rsrc_buf>`,    :ref:`soffset<amdgpu_synid8_offset_buf>`                  :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_load_sbyte              :ref:`vdst<amdgpu_synid8_dst_buf_lds>`,     :ref:`vaddr<amdgpu_synid8_addr_buf>`,           :ref:`srsrc<amdgpu_synid8_rsrc_buf>`,    :ref:`soffset<amdgpu_synid8_offset_buf>`                  :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`lds<amdgpu_synid_lds>`
-    buffer_load_sshort             :ref:`vdst<amdgpu_synid8_dst_buf_lds>`,     :ref:`vaddr<amdgpu_synid8_addr_buf>`,           :ref:`srsrc<amdgpu_synid8_rsrc_buf>`,    :ref:`soffset<amdgpu_synid8_offset_buf>`                  :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`lds<amdgpu_synid_lds>`
-    buffer_load_ubyte              :ref:`vdst<amdgpu_synid8_dst_buf_lds>`,     :ref:`vaddr<amdgpu_synid8_addr_buf>`,           :ref:`srsrc<amdgpu_synid8_rsrc_buf>`,    :ref:`soffset<amdgpu_synid8_offset_buf>`                  :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`lds<amdgpu_synid_lds>`
-    buffer_load_ushort             :ref:`vdst<amdgpu_synid8_dst_buf_lds>`,     :ref:`vaddr<amdgpu_synid8_addr_buf>`,           :ref:`srsrc<amdgpu_synid8_rsrc_buf>`,    :ref:`soffset<amdgpu_synid8_offset_buf>`                  :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`lds<amdgpu_synid_lds>`
-    buffer_store_byte                        :ref:`vdata<amdgpu_synid8_vdata32_0>`,           :ref:`vaddr<amdgpu_synid8_addr_buf>`,    :ref:`srsrc<amdgpu_synid8_rsrc_buf>`,    :ref:`soffset<amdgpu_synid8_offset_buf>`        :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_store_dword                       :ref:`vdata<amdgpu_synid8_vdata32_0>`,           :ref:`vaddr<amdgpu_synid8_addr_buf>`,    :ref:`srsrc<amdgpu_synid8_rsrc_buf>`,    :ref:`soffset<amdgpu_synid8_offset_buf>`        :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_store_dwordx2                     :ref:`vdata<amdgpu_synid8_vdata64_0>`,           :ref:`vaddr<amdgpu_synid8_addr_buf>`,    :ref:`srsrc<amdgpu_synid8_rsrc_buf>`,    :ref:`soffset<amdgpu_synid8_offset_buf>`        :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_store_dwordx3                     :ref:`vdata<amdgpu_synid8_vdata96_0>`,           :ref:`vaddr<amdgpu_synid8_addr_buf>`,    :ref:`srsrc<amdgpu_synid8_rsrc_buf>`,    :ref:`soffset<amdgpu_synid8_offset_buf>`        :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_store_dwordx4                     :ref:`vdata<amdgpu_synid8_vdata128_0>`,           :ref:`vaddr<amdgpu_synid8_addr_buf>`,    :ref:`srsrc<amdgpu_synid8_rsrc_buf>`,    :ref:`soffset<amdgpu_synid8_offset_buf>`        :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_store_format_d16_x                :ref:`vdata<amdgpu_synid8_data_buf_d16_32>`,           :ref:`vaddr<amdgpu_synid8_addr_buf>`,    :ref:`srsrc<amdgpu_synid8_rsrc_buf>`,    :ref:`soffset<amdgpu_synid8_offset_buf>`        :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_store_format_d16_xy               :ref:`vdata<amdgpu_synid8_data_buf_d16_64>`,           :ref:`vaddr<amdgpu_synid8_addr_buf>`,    :ref:`srsrc<amdgpu_synid8_rsrc_buf>`,    :ref:`soffset<amdgpu_synid8_offset_buf>`        :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_store_format_d16_xyz              :ref:`vdata<amdgpu_synid8_data_buf_d16_96>`,           :ref:`vaddr<amdgpu_synid8_addr_buf>`,    :ref:`srsrc<amdgpu_synid8_rsrc_buf>`,    :ref:`soffset<amdgpu_synid8_offset_buf>`        :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_store_format_d16_xyzw             :ref:`vdata<amdgpu_synid8_data_buf_d16_128>`,           :ref:`vaddr<amdgpu_synid8_addr_buf>`,    :ref:`srsrc<amdgpu_synid8_rsrc_buf>`,    :ref:`soffset<amdgpu_synid8_offset_buf>`        :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_store_format_x                    :ref:`vdata<amdgpu_synid8_vdata32_0>`,           :ref:`vaddr<amdgpu_synid8_addr_buf>`,    :ref:`srsrc<amdgpu_synid8_rsrc_buf>`,    :ref:`soffset<amdgpu_synid8_offset_buf>`        :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_store_format_xy                   :ref:`vdata<amdgpu_synid8_vdata64_0>`,           :ref:`vaddr<amdgpu_synid8_addr_buf>`,    :ref:`srsrc<amdgpu_synid8_rsrc_buf>`,    :ref:`soffset<amdgpu_synid8_offset_buf>`        :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_store_format_xyz                  :ref:`vdata<amdgpu_synid8_vdata96_0>`,           :ref:`vaddr<amdgpu_synid8_addr_buf>`,    :ref:`srsrc<amdgpu_synid8_rsrc_buf>`,    :ref:`soffset<amdgpu_synid8_offset_buf>`        :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_store_format_xyzw                 :ref:`vdata<amdgpu_synid8_vdata128_0>`,           :ref:`vaddr<amdgpu_synid8_addr_buf>`,    :ref:`srsrc<amdgpu_synid8_rsrc_buf>`,    :ref:`soffset<amdgpu_synid8_offset_buf>`        :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_store_lds_dword                   :ref:`srsrc<amdgpu_synid8_rsrc_buf>`,           :ref:`soffset<amdgpu_synid8_offset_buf>`                            :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`lds<amdgpu_synid_lds>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_store_short                       :ref:`vdata<amdgpu_synid8_vdata32_0>`,           :ref:`vaddr<amdgpu_synid8_addr_buf>`,    :ref:`srsrc<amdgpu_synid8_rsrc_buf>`,    :ref:`soffset<amdgpu_synid8_offset_buf>`        :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    **INSTRUCTION**                  **DST**   **SRC0**             **SRC1**    **SRC2**    **SRC3**    **MODIFIERS**
+    \ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|
+    buffer_atomic_add                  :ref:`vdata<amdgpu_synid8_data_buf_atomic32>`::ref:`dst<amdgpu_synid8_ret>`,       :ref:`vaddr<amdgpu_synid8_addr_buf>`,  :ref:`srsrc<amdgpu_synid8_rsrc_buf>`,  :ref:`soffset<amdgpu_synid8_offset_buf>` :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_atomic_add_x2               :ref:`vdata<amdgpu_synid8_data_buf_atomic64>`::ref:`dst<amdgpu_synid8_ret>`,       :ref:`vaddr<amdgpu_synid8_addr_buf>`,  :ref:`srsrc<amdgpu_synid8_rsrc_buf>`,  :ref:`soffset<amdgpu_synid8_offset_buf>` :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_atomic_and                  :ref:`vdata<amdgpu_synid8_data_buf_atomic32>`::ref:`dst<amdgpu_synid8_ret>`,       :ref:`vaddr<amdgpu_synid8_addr_buf>`,  :ref:`srsrc<amdgpu_synid8_rsrc_buf>`,  :ref:`soffset<amdgpu_synid8_offset_buf>` :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_atomic_and_x2               :ref:`vdata<amdgpu_synid8_data_buf_atomic64>`::ref:`dst<amdgpu_synid8_ret>`,       :ref:`vaddr<amdgpu_synid8_addr_buf>`,  :ref:`srsrc<amdgpu_synid8_rsrc_buf>`,  :ref:`soffset<amdgpu_synid8_offset_buf>` :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_atomic_cmpswap              :ref:`vdata<amdgpu_synid8_data_buf_atomic64>`::ref:`dst<amdgpu_synid8_ret>`::ref:`b32x2<amdgpu_synid8_type_dev>`, :ref:`vaddr<amdgpu_synid8_addr_buf>`,  :ref:`srsrc<amdgpu_synid8_rsrc_buf>`,  :ref:`soffset<amdgpu_synid8_offset_buf>` :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_atomic_cmpswap_x2           :ref:`vdata<amdgpu_synid8_data_buf_atomic128>`::ref:`dst<amdgpu_synid8_ret>`::ref:`b64x2<amdgpu_synid8_type_dev>`, :ref:`vaddr<amdgpu_synid8_addr_buf>`,  :ref:`srsrc<amdgpu_synid8_rsrc_buf>`,  :ref:`soffset<amdgpu_synid8_offset_buf>` :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_atomic_dec                  :ref:`vdata<amdgpu_synid8_data_buf_atomic32>`::ref:`dst<amdgpu_synid8_ret>`::ref:`u32<amdgpu_synid8_type_dev>`,   :ref:`vaddr<amdgpu_synid8_addr_buf>`,  :ref:`srsrc<amdgpu_synid8_rsrc_buf>`,  :ref:`soffset<amdgpu_synid8_offset_buf>` :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_atomic_dec_x2               :ref:`vdata<amdgpu_synid8_data_buf_atomic64>`::ref:`dst<amdgpu_synid8_ret>`::ref:`u64<amdgpu_synid8_type_dev>`,   :ref:`vaddr<amdgpu_synid8_addr_buf>`,  :ref:`srsrc<amdgpu_synid8_rsrc_buf>`,  :ref:`soffset<amdgpu_synid8_offset_buf>` :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_atomic_inc                  :ref:`vdata<amdgpu_synid8_data_buf_atomic32>`::ref:`dst<amdgpu_synid8_ret>`::ref:`u32<amdgpu_synid8_type_dev>`,   :ref:`vaddr<amdgpu_synid8_addr_buf>`,  :ref:`srsrc<amdgpu_synid8_rsrc_buf>`,  :ref:`soffset<amdgpu_synid8_offset_buf>` :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_atomic_inc_x2               :ref:`vdata<amdgpu_synid8_data_buf_atomic64>`::ref:`dst<amdgpu_synid8_ret>`::ref:`u64<amdgpu_synid8_type_dev>`,   :ref:`vaddr<amdgpu_synid8_addr_buf>`,  :ref:`srsrc<amdgpu_synid8_rsrc_buf>`,  :ref:`soffset<amdgpu_synid8_offset_buf>` :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_atomic_or                   :ref:`vdata<amdgpu_synid8_data_buf_atomic32>`::ref:`dst<amdgpu_synid8_ret>`,       :ref:`vaddr<amdgpu_synid8_addr_buf>`,  :ref:`srsrc<amdgpu_synid8_rsrc_buf>`,  :ref:`soffset<amdgpu_synid8_offset_buf>` :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_atomic_or_x2                :ref:`vdata<amdgpu_synid8_data_buf_atomic64>`::ref:`dst<amdgpu_synid8_ret>`,       :ref:`vaddr<amdgpu_synid8_addr_buf>`,  :ref:`srsrc<amdgpu_synid8_rsrc_buf>`,  :ref:`soffset<amdgpu_synid8_offset_buf>` :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_atomic_smax                 :ref:`vdata<amdgpu_synid8_data_buf_atomic32>`::ref:`dst<amdgpu_synid8_ret>`::ref:`s32<amdgpu_synid8_type_dev>`,   :ref:`vaddr<amdgpu_synid8_addr_buf>`,  :ref:`srsrc<amdgpu_synid8_rsrc_buf>`,  :ref:`soffset<amdgpu_synid8_offset_buf>` :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_atomic_smax_x2              :ref:`vdata<amdgpu_synid8_data_buf_atomic64>`::ref:`dst<amdgpu_synid8_ret>`::ref:`s64<amdgpu_synid8_type_dev>`,   :ref:`vaddr<amdgpu_synid8_addr_buf>`,  :ref:`srsrc<amdgpu_synid8_rsrc_buf>`,  :ref:`soffset<amdgpu_synid8_offset_buf>` :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_atomic_smin                 :ref:`vdata<amdgpu_synid8_data_buf_atomic32>`::ref:`dst<amdgpu_synid8_ret>`::ref:`s32<amdgpu_synid8_type_dev>`,   :ref:`vaddr<amdgpu_synid8_addr_buf>`,  :ref:`srsrc<amdgpu_synid8_rsrc_buf>`,  :ref:`soffset<amdgpu_synid8_offset_buf>` :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_atomic_smin_x2              :ref:`vdata<amdgpu_synid8_data_buf_atomic64>`::ref:`dst<amdgpu_synid8_ret>`::ref:`s64<amdgpu_synid8_type_dev>`,   :ref:`vaddr<amdgpu_synid8_addr_buf>`,  :ref:`srsrc<amdgpu_synid8_rsrc_buf>`,  :ref:`soffset<amdgpu_synid8_offset_buf>` :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_atomic_sub                  :ref:`vdata<amdgpu_synid8_data_buf_atomic32>`::ref:`dst<amdgpu_synid8_ret>`,       :ref:`vaddr<amdgpu_synid8_addr_buf>`,  :ref:`srsrc<amdgpu_synid8_rsrc_buf>`,  :ref:`soffset<amdgpu_synid8_offset_buf>` :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_atomic_sub_x2               :ref:`vdata<amdgpu_synid8_data_buf_atomic64>`::ref:`dst<amdgpu_synid8_ret>`,       :ref:`vaddr<amdgpu_synid8_addr_buf>`,  :ref:`srsrc<amdgpu_synid8_rsrc_buf>`,  :ref:`soffset<amdgpu_synid8_offset_buf>` :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_atomic_swap                 :ref:`vdata<amdgpu_synid8_data_buf_atomic32>`::ref:`dst<amdgpu_synid8_ret>`,       :ref:`vaddr<amdgpu_synid8_addr_buf>`,  :ref:`srsrc<amdgpu_synid8_rsrc_buf>`,  :ref:`soffset<amdgpu_synid8_offset_buf>` :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_atomic_swap_x2              :ref:`vdata<amdgpu_synid8_data_buf_atomic64>`::ref:`dst<amdgpu_synid8_ret>`,       :ref:`vaddr<amdgpu_synid8_addr_buf>`,  :ref:`srsrc<amdgpu_synid8_rsrc_buf>`,  :ref:`soffset<amdgpu_synid8_offset_buf>` :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_atomic_umax                 :ref:`vdata<amdgpu_synid8_data_buf_atomic32>`::ref:`dst<amdgpu_synid8_ret>`::ref:`u32<amdgpu_synid8_type_dev>`,   :ref:`vaddr<amdgpu_synid8_addr_buf>`,  :ref:`srsrc<amdgpu_synid8_rsrc_buf>`,  :ref:`soffset<amdgpu_synid8_offset_buf>` :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_atomic_umax_x2              :ref:`vdata<amdgpu_synid8_data_buf_atomic64>`::ref:`dst<amdgpu_synid8_ret>`::ref:`u64<amdgpu_synid8_type_dev>`,   :ref:`vaddr<amdgpu_synid8_addr_buf>`,  :ref:`srsrc<amdgpu_synid8_rsrc_buf>`,  :ref:`soffset<amdgpu_synid8_offset_buf>` :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_atomic_umin                 :ref:`vdata<amdgpu_synid8_data_buf_atomic32>`::ref:`dst<amdgpu_synid8_ret>`::ref:`u32<amdgpu_synid8_type_dev>`,   :ref:`vaddr<amdgpu_synid8_addr_buf>`,  :ref:`srsrc<amdgpu_synid8_rsrc_buf>`,  :ref:`soffset<amdgpu_synid8_offset_buf>` :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_atomic_umin_x2              :ref:`vdata<amdgpu_synid8_data_buf_atomic64>`::ref:`dst<amdgpu_synid8_ret>`::ref:`u64<amdgpu_synid8_type_dev>`,   :ref:`vaddr<amdgpu_synid8_addr_buf>`,  :ref:`srsrc<amdgpu_synid8_rsrc_buf>`,  :ref:`soffset<amdgpu_synid8_offset_buf>` :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_atomic_xor                  :ref:`vdata<amdgpu_synid8_data_buf_atomic32>`::ref:`dst<amdgpu_synid8_ret>`,       :ref:`vaddr<amdgpu_synid8_addr_buf>`,  :ref:`srsrc<amdgpu_synid8_rsrc_buf>`,  :ref:`soffset<amdgpu_synid8_offset_buf>` :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_atomic_xor_x2               :ref:`vdata<amdgpu_synid8_data_buf_atomic64>`::ref:`dst<amdgpu_synid8_ret>`,       :ref:`vaddr<amdgpu_synid8_addr_buf>`,  :ref:`srsrc<amdgpu_synid8_rsrc_buf>`,  :ref:`soffset<amdgpu_synid8_offset_buf>` :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_load_dword            :ref:`vdst<amdgpu_synid8_dst_buf_lds>`, :ref:`vaddr<amdgpu_synid8_addr_buf>`,           :ref:`srsrc<amdgpu_synid8_rsrc_buf>`,  :ref:`soffset<amdgpu_synid8_offset_buf>`         :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`lds<amdgpu_synid_lds>`
+    buffer_load_dwordx2          :ref:`vdst<amdgpu_synid8_dst_buf_64>`, :ref:`vaddr<amdgpu_synid8_addr_buf>`,           :ref:`srsrc<amdgpu_synid8_rsrc_buf>`,  :ref:`soffset<amdgpu_synid8_offset_buf>`         :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_load_dwordx3          :ref:`vdst<amdgpu_synid8_dst_buf_96>`, :ref:`vaddr<amdgpu_synid8_addr_buf>`,           :ref:`srsrc<amdgpu_synid8_rsrc_buf>`,  :ref:`soffset<amdgpu_synid8_offset_buf>`         :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_load_dwordx4          :ref:`vdst<amdgpu_synid8_dst_buf_128>`, :ref:`vaddr<amdgpu_synid8_addr_buf>`,           :ref:`srsrc<amdgpu_synid8_rsrc_buf>`,  :ref:`soffset<amdgpu_synid8_offset_buf>`         :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_load_format_d16_x     :ref:`vdst<amdgpu_synid8_dst_buf_d16_32>`, :ref:`vaddr<amdgpu_synid8_addr_buf>`,           :ref:`srsrc<amdgpu_synid8_rsrc_buf>`,  :ref:`soffset<amdgpu_synid8_offset_buf>`         :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_load_format_d16_xy    :ref:`vdst<amdgpu_synid8_dst_buf_d16_64>`, :ref:`vaddr<amdgpu_synid8_addr_buf>`,           :ref:`srsrc<amdgpu_synid8_rsrc_buf>`,  :ref:`soffset<amdgpu_synid8_offset_buf>`         :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_load_format_d16_xyz   :ref:`vdst<amdgpu_synid8_dst_buf_d16_96>`, :ref:`vaddr<amdgpu_synid8_addr_buf>`,           :ref:`srsrc<amdgpu_synid8_rsrc_buf>`,  :ref:`soffset<amdgpu_synid8_offset_buf>`         :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_load_format_d16_xyzw  :ref:`vdst<amdgpu_synid8_dst_buf_d16_128>`, :ref:`vaddr<amdgpu_synid8_addr_buf>`,           :ref:`srsrc<amdgpu_synid8_rsrc_buf>`,  :ref:`soffset<amdgpu_synid8_offset_buf>`         :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_load_format_x         :ref:`vdst<amdgpu_synid8_dst_buf_lds>`, :ref:`vaddr<amdgpu_synid8_addr_buf>`,           :ref:`srsrc<amdgpu_synid8_rsrc_buf>`,  :ref:`soffset<amdgpu_synid8_offset_buf>`         :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`lds<amdgpu_synid_lds>`
+    buffer_load_format_xy        :ref:`vdst<amdgpu_synid8_dst_buf_64>`, :ref:`vaddr<amdgpu_synid8_addr_buf>`,           :ref:`srsrc<amdgpu_synid8_rsrc_buf>`,  :ref:`soffset<amdgpu_synid8_offset_buf>`         :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_load_format_xyz       :ref:`vdst<amdgpu_synid8_dst_buf_96>`, :ref:`vaddr<amdgpu_synid8_addr_buf>`,           :ref:`srsrc<amdgpu_synid8_rsrc_buf>`,  :ref:`soffset<amdgpu_synid8_offset_buf>`         :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_load_format_xyzw      :ref:`vdst<amdgpu_synid8_dst_buf_128>`, :ref:`vaddr<amdgpu_synid8_addr_buf>`,           :ref:`srsrc<amdgpu_synid8_rsrc_buf>`,  :ref:`soffset<amdgpu_synid8_offset_buf>`         :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_load_sbyte            :ref:`vdst<amdgpu_synid8_dst_buf_lds>`, :ref:`vaddr<amdgpu_synid8_addr_buf>`,           :ref:`srsrc<amdgpu_synid8_rsrc_buf>`,  :ref:`soffset<amdgpu_synid8_offset_buf>`         :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`lds<amdgpu_synid_lds>`
+    buffer_load_sshort           :ref:`vdst<amdgpu_synid8_dst_buf_lds>`, :ref:`vaddr<amdgpu_synid8_addr_buf>`,           :ref:`srsrc<amdgpu_synid8_rsrc_buf>`,  :ref:`soffset<amdgpu_synid8_offset_buf>`         :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`lds<amdgpu_synid_lds>`
+    buffer_load_ubyte            :ref:`vdst<amdgpu_synid8_dst_buf_lds>`, :ref:`vaddr<amdgpu_synid8_addr_buf>`,           :ref:`srsrc<amdgpu_synid8_rsrc_buf>`,  :ref:`soffset<amdgpu_synid8_offset_buf>`         :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`lds<amdgpu_synid_lds>`
+    buffer_load_ushort           :ref:`vdst<amdgpu_synid8_dst_buf_lds>`, :ref:`vaddr<amdgpu_synid8_addr_buf>`,           :ref:`srsrc<amdgpu_synid8_rsrc_buf>`,  :ref:`soffset<amdgpu_synid8_offset_buf>`         :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`lds<amdgpu_synid_lds>`
+    buffer_store_byte                  :ref:`vdata<amdgpu_synid8_vdata32_0>`,           :ref:`vaddr<amdgpu_synid8_addr_buf>`,  :ref:`srsrc<amdgpu_synid8_rsrc_buf>`,  :ref:`soffset<amdgpu_synid8_offset_buf>` :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_store_dword                 :ref:`vdata<amdgpu_synid8_vdata32_0>`,           :ref:`vaddr<amdgpu_synid8_addr_buf>`,  :ref:`srsrc<amdgpu_synid8_rsrc_buf>`,  :ref:`soffset<amdgpu_synid8_offset_buf>` :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_store_dwordx2               :ref:`vdata<amdgpu_synid8_vdata64_0>`,           :ref:`vaddr<amdgpu_synid8_addr_buf>`,  :ref:`srsrc<amdgpu_synid8_rsrc_buf>`,  :ref:`soffset<amdgpu_synid8_offset_buf>` :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_store_dwordx3               :ref:`vdata<amdgpu_synid8_vdata96_0>`,           :ref:`vaddr<amdgpu_synid8_addr_buf>`,  :ref:`srsrc<amdgpu_synid8_rsrc_buf>`,  :ref:`soffset<amdgpu_synid8_offset_buf>` :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_store_dwordx4               :ref:`vdata<amdgpu_synid8_vdata128_0>`,           :ref:`vaddr<amdgpu_synid8_addr_buf>`,  :ref:`srsrc<amdgpu_synid8_rsrc_buf>`,  :ref:`soffset<amdgpu_synid8_offset_buf>` :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_store_format_d16_x          :ref:`vdata<amdgpu_synid8_data_buf_d16_32>`,           :ref:`vaddr<amdgpu_synid8_addr_buf>`,  :ref:`srsrc<amdgpu_synid8_rsrc_buf>`,  :ref:`soffset<amdgpu_synid8_offset_buf>` :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_store_format_d16_xy         :ref:`vdata<amdgpu_synid8_data_buf_d16_64>`,           :ref:`vaddr<amdgpu_synid8_addr_buf>`,  :ref:`srsrc<amdgpu_synid8_rsrc_buf>`,  :ref:`soffset<amdgpu_synid8_offset_buf>` :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_store_format_d16_xyz        :ref:`vdata<amdgpu_synid8_data_buf_d16_96>`,           :ref:`vaddr<amdgpu_synid8_addr_buf>`,  :ref:`srsrc<amdgpu_synid8_rsrc_buf>`,  :ref:`soffset<amdgpu_synid8_offset_buf>` :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_store_format_d16_xyzw       :ref:`vdata<amdgpu_synid8_data_buf_d16_128>`,           :ref:`vaddr<amdgpu_synid8_addr_buf>`,  :ref:`srsrc<amdgpu_synid8_rsrc_buf>`,  :ref:`soffset<amdgpu_synid8_offset_buf>` :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_store_format_x              :ref:`vdata<amdgpu_synid8_vdata32_0>`,           :ref:`vaddr<amdgpu_synid8_addr_buf>`,  :ref:`srsrc<amdgpu_synid8_rsrc_buf>`,  :ref:`soffset<amdgpu_synid8_offset_buf>` :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_store_format_xy             :ref:`vdata<amdgpu_synid8_vdata64_0>`,           :ref:`vaddr<amdgpu_synid8_addr_buf>`,  :ref:`srsrc<amdgpu_synid8_rsrc_buf>`,  :ref:`soffset<amdgpu_synid8_offset_buf>` :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_store_format_xyz            :ref:`vdata<amdgpu_synid8_vdata96_0>`,           :ref:`vaddr<amdgpu_synid8_addr_buf>`,  :ref:`srsrc<amdgpu_synid8_rsrc_buf>`,  :ref:`soffset<amdgpu_synid8_offset_buf>` :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_store_format_xyzw           :ref:`vdata<amdgpu_synid8_vdata128_0>`,           :ref:`vaddr<amdgpu_synid8_addr_buf>`,  :ref:`srsrc<amdgpu_synid8_rsrc_buf>`,  :ref:`soffset<amdgpu_synid8_offset_buf>` :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_store_lds_dword             :ref:`srsrc<amdgpu_synid8_rsrc_buf>`,           :ref:`soffset<amdgpu_synid8_offset_buf>`                 :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`lds<amdgpu_synid_lds>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_store_short                 :ref:`vdata<amdgpu_synid8_vdata32_0>`,           :ref:`vaddr<amdgpu_synid8_addr_buf>`,  :ref:`srsrc<amdgpu_synid8_rsrc_buf>`,  :ref:`soffset<amdgpu_synid8_offset_buf>` :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
     buffer_wbinvl1
     buffer_wbinvl1_vol
 
@@ -405,7 +405,7 @@ SMEM
 .. parsed-literal::
 
     **INSTRUCTION**                    **DST**       **SRC0**      **SRC1**      **SRC2**           **MODIFIERS**
-    \ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|
+    \ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|
     s_atc_probe                              :ref:`imm3<amdgpu_synid8_perm_smem>`,     :ref:`sbase<amdgpu_synid8_base_smem_addr>`,    :ref:`soffset<amdgpu_synid8_offset_smem_load>`
     s_atc_probe_buffer                       :ref:`imm3<amdgpu_synid8_perm_smem>`,     :ref:`sbase<amdgpu_synid8_base_smem_buf>`,    :ref:`soffset<amdgpu_synid8_offset_smem_load>`
     s_buffer_load_dword            :ref:`sdst<amdgpu_synid8_sdst32_0>`,     :ref:`sbase<amdgpu_synid8_base_smem_buf>`,    :ref:`soffset<amdgpu_synid8_offset_smem_load>`                  :ref:`glc<amdgpu_synid_glc>`
@@ -846,565 +846,565 @@ VOP2
 
 .. parsed-literal::
 
-    **INSTRUCTION**                    **DST0**      **DST1**      **SRC0**         **SRC1**        **SRC2**           **MODIFIERS**
-    \ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|
-    v_add_f16                      :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_0>`,        :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`
-    v_add_f16_dpp                  :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_dpp_sdwa_abs_neg>`,     :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_dpp_sdwa_abs_neg>`                    :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
-    v_add_f16_sdwa                 :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_dpp_sdwa_abs_neg>`,     :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_dpp_sdwa_abs_neg>`                    :ref:`clamp<amdgpu_synid_clamp>` :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
-    v_add_f32                      :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_0>`,        :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`
-    v_add_f32_dpp                  :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_dpp_sdwa_abs_neg>`,     :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_dpp_sdwa_abs_neg>`                    :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
-    v_add_f32_sdwa                 :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_dpp_sdwa_abs_neg>`,     :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_dpp_sdwa_abs_neg>`                    :ref:`clamp<amdgpu_synid_clamp>` :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
-    v_add_u16                      :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_0>`,        :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`
-    v_add_u16_dpp                  :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`,       :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`                      :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
-    v_add_u16_sdwa                 :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`,     :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`                    :ref:`clamp<amdgpu_synid_clamp>` :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
-    v_add_u32                      :ref:`vdst<amdgpu_synid8_vdst32_0>`,     :ref:`vcc<amdgpu_synid8_vcc_64>`,      :ref:`src0<amdgpu_synid8_src32_0>`,        :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`
-    v_add_u32_dpp                  :ref:`vdst<amdgpu_synid8_vdst32_0>`,     :ref:`vcc<amdgpu_synid8_vcc_64>`,      :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`,       :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`                      :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
-    v_add_u32_sdwa                 :ref:`vdst<amdgpu_synid8_vdst32_0>`,     :ref:`vcc<amdgpu_synid8_vcc_64>`,      :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`,     :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`                    :ref:`clamp<amdgpu_synid_clamp>` :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
-    v_addc_u32                     :ref:`vdst<amdgpu_synid8_vdst32_0>`,     :ref:`vcc<amdgpu_synid8_vcc_64>`,      :ref:`src0<amdgpu_synid8_src32_0>`,        :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`,      :ref:`vcc<amdgpu_synid8_vcc_64>`
-    v_addc_u32_dpp                 :ref:`vdst<amdgpu_synid8_vdst32_0>`,     :ref:`vcc<amdgpu_synid8_vcc_64>`,      :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`,       :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`,      :ref:`vcc<amdgpu_synid8_vcc_64>`            :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
-    v_addc_u32_sdwa                :ref:`vdst<amdgpu_synid8_vdst32_0>`,     :ref:`vcc<amdgpu_synid8_vcc_64>`,      :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`,     :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`,    :ref:`vcc<amdgpu_synid8_vcc_64>`            :ref:`clamp<amdgpu_synid_clamp>` :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
-    v_and_b32                      :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_0>`,        :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`
-    v_and_b32_dpp                  :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`,       :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`                      :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
-    v_and_b32_sdwa                 :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`,     :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`                    :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
-    v_ashrrev_i16                  :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_0>`::ref:`u16<amdgpu_synid8_type_dev>`,    :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`
-    v_ashrrev_i16_dpp              :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`u16<amdgpu_synid8_type_dev>`,   :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`                      :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
-    v_ashrrev_i16_sdwa             :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`::ref:`u16<amdgpu_synid8_type_dev>`, :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`                    :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
-    v_ashrrev_i32                  :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_0>`::ref:`u32<amdgpu_synid8_type_dev>`,    :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`
-    v_ashrrev_i32_dpp              :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`u32<amdgpu_synid8_type_dev>`,   :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`                      :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
-    v_ashrrev_i32_sdwa             :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`::ref:`u32<amdgpu_synid8_type_dev>`, :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`                    :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
-    v_cndmask_b32                  :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_0>`,        :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`,      :ref:`vcc<amdgpu_synid8_vcc_64>`
-    v_cndmask_b32_dpp              :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`,       :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`,      :ref:`vcc<amdgpu_synid8_vcc_64>`            :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
-    v_cndmask_b32_sdwa             :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`,     :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`,    :ref:`vcc<amdgpu_synid8_vcc_64>`            :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
-    v_ldexp_f16                    :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_0>`,        :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`i16<amdgpu_synid8_type_dev>`
-    v_ldexp_f16_dpp                :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_dpp_sdwa_abs_neg>`,     :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`i16<amdgpu_synid8_type_dev>`                  :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
-    v_ldexp_f16_sdwa               :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_dpp_sdwa_abs_neg>`,     :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`::ref:`i16<amdgpu_synid8_type_dev>`                :ref:`clamp<amdgpu_synid_clamp>` :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
-    v_lshlrev_b16                  :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_0>`::ref:`u16<amdgpu_synid8_type_dev>`,    :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`
-    v_lshlrev_b16_dpp              :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`u16<amdgpu_synid8_type_dev>`,   :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`                      :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
-    v_lshlrev_b16_sdwa             :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`::ref:`u16<amdgpu_synid8_type_dev>`, :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`                    :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
-    v_lshlrev_b32                  :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_0>`::ref:`u32<amdgpu_synid8_type_dev>`,    :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`
-    v_lshlrev_b32_dpp              :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`u32<amdgpu_synid8_type_dev>`,   :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`                      :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
-    v_lshlrev_b32_sdwa             :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`::ref:`u32<amdgpu_synid8_type_dev>`, :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`                    :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
-    v_lshrrev_b16                  :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_0>`::ref:`u16<amdgpu_synid8_type_dev>`,    :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`
-    v_lshrrev_b16_dpp              :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`u16<amdgpu_synid8_type_dev>`,   :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`                      :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
-    v_lshrrev_b16_sdwa             :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`::ref:`u16<amdgpu_synid8_type_dev>`, :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`                    :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
-    v_lshrrev_b32                  :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_0>`::ref:`u32<amdgpu_synid8_type_dev>`,    :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`
-    v_lshrrev_b32_dpp              :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`u32<amdgpu_synid8_type_dev>`,   :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`                      :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
-    v_lshrrev_b32_sdwa             :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`::ref:`u32<amdgpu_synid8_type_dev>`, :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`                    :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
-    v_mac_f16                      :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_0>`,        :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`
-    v_mac_f16_dpp                  :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_dpp_sdwa_abs_neg>`,     :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_dpp_sdwa_abs_neg>`                    :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
-    v_mac_f16_sdwa                 :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_dpp_sdwa_abs_neg>`,     :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_dpp_sdwa_abs_neg>`                    :ref:`clamp<amdgpu_synid_clamp>` :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
-    v_mac_f32                      :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_0>`,        :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`
-    v_mac_f32_dpp                  :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_dpp_sdwa_abs_neg>`,     :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_dpp_sdwa_abs_neg>`                    :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
-    v_mac_f32_sdwa                 :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_dpp_sdwa_abs_neg>`,     :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_dpp_sdwa_abs_neg>`                    :ref:`clamp<amdgpu_synid_clamp>` :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
-    v_madak_f16                    :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_0>`,        :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`,      :ref:`imm32<amdgpu_synid8_fimm16>`
-    v_madak_f32                    :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_0>`,        :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`,      :ref:`imm32<amdgpu_synid8_fimm32>`
-    v_madmk_f16                    :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_0>`,        :ref:`imm32<amdgpu_synid8_fimm16>`,      :ref:`vsrc2<amdgpu_synid8_vsrc32_0>`
-    v_madmk_f32                    :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_0>`,        :ref:`imm32<amdgpu_synid8_fimm32>`,      :ref:`vsrc2<amdgpu_synid8_vsrc32_0>`
-    v_max_f16                      :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_0>`,        :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`
-    v_max_f16_dpp                  :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_dpp_sdwa_abs_neg>`,     :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_dpp_sdwa_abs_neg>`                    :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
-    v_max_f16_sdwa                 :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_dpp_sdwa_abs_neg>`,     :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_dpp_sdwa_abs_neg>`                    :ref:`clamp<amdgpu_synid_clamp>` :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
-    v_max_f32                      :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_0>`,        :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`
-    v_max_f32_dpp                  :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_dpp_sdwa_abs_neg>`,     :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_dpp_sdwa_abs_neg>`                    :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
-    v_max_f32_sdwa                 :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_dpp_sdwa_abs_neg>`,     :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_dpp_sdwa_abs_neg>`                    :ref:`clamp<amdgpu_synid_clamp>` :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
-    v_max_i16                      :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_0>`,        :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`
-    v_max_i16_dpp                  :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`,       :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`                      :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
-    v_max_i16_sdwa                 :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`,     :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`                    :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
-    v_max_i32                      :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_0>`,        :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`
-    v_max_i32_dpp                  :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`,       :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`                      :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
-    v_max_i32_sdwa                 :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`,     :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`                    :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
-    v_max_u16                      :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_0>`,        :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`
-    v_max_u16_dpp                  :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`,       :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`                      :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
-    v_max_u16_sdwa                 :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`,     :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`                    :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
-    v_max_u32                      :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_0>`,        :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`
-    v_max_u32_dpp                  :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`,       :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`                      :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
-    v_max_u32_sdwa                 :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`,     :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`                    :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
-    v_min_f16                      :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_0>`,        :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`
-    v_min_f16_dpp                  :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_dpp_sdwa_abs_neg>`,     :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_dpp_sdwa_abs_neg>`                    :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
-    v_min_f16_sdwa                 :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_dpp_sdwa_abs_neg>`,     :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_dpp_sdwa_abs_neg>`                    :ref:`clamp<amdgpu_synid_clamp>` :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
-    v_min_f32                      :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_0>`,        :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`
-    v_min_f32_dpp                  :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_dpp_sdwa_abs_neg>`,     :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_dpp_sdwa_abs_neg>`                    :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
-    v_min_f32_sdwa                 :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_dpp_sdwa_abs_neg>`,     :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_dpp_sdwa_abs_neg>`                    :ref:`clamp<amdgpu_synid_clamp>` :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
-    v_min_i16                      :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_0>`,        :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`
-    v_min_i16_dpp                  :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`,       :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`                      :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
-    v_min_i16_sdwa                 :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`,     :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`                    :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
-    v_min_i32                      :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_0>`,        :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`
-    v_min_i32_dpp                  :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`,       :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`                      :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
-    v_min_i32_sdwa                 :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`,     :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`                    :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
-    v_min_u16                      :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_0>`,        :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`
-    v_min_u16_dpp                  :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`,       :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`                      :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
-    v_min_u16_sdwa                 :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`,     :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`                    :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
-    v_min_u32                      :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_0>`,        :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`
-    v_min_u32_dpp                  :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`,       :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`                      :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
-    v_min_u32_sdwa                 :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`,     :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`                    :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
-    v_mul_f16                      :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_0>`,        :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`
-    v_mul_f16_dpp                  :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_dpp_sdwa_abs_neg>`,     :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_dpp_sdwa_abs_neg>`                    :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
-    v_mul_f16_sdwa                 :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_dpp_sdwa_abs_neg>`,     :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_dpp_sdwa_abs_neg>`                    :ref:`clamp<amdgpu_synid_clamp>` :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
-    v_mul_f32                      :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_0>`,        :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`
-    v_mul_f32_dpp                  :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_dpp_sdwa_abs_neg>`,     :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_dpp_sdwa_abs_neg>`                    :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
-    v_mul_f32_sdwa                 :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_dpp_sdwa_abs_neg>`,     :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_dpp_sdwa_abs_neg>`                    :ref:`clamp<amdgpu_synid_clamp>` :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
-    v_mul_hi_i32_i24               :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_0>`,        :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`
-    v_mul_hi_i32_i24_dpp           :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`,       :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`                      :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
-    v_mul_hi_i32_i24_sdwa          :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`,     :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`                    :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
-    v_mul_hi_u32_u24               :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_0>`,        :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`
-    v_mul_hi_u32_u24_dpp           :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`,       :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`                      :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
-    v_mul_hi_u32_u24_sdwa          :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`,     :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`                    :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
-    v_mul_i32_i24                  :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_0>`,        :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`
-    v_mul_i32_i24_dpp              :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`,       :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`                      :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
-    v_mul_i32_i24_sdwa             :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`,     :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`                    :ref:`clamp<amdgpu_synid_clamp>` :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
-    v_mul_legacy_f32               :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_0>`,        :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`
-    v_mul_legacy_f32_dpp           :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_dpp_sdwa_abs_neg>`,     :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_dpp_sdwa_abs_neg>`                    :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
-    v_mul_legacy_f32_sdwa          :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_dpp_sdwa_abs_neg>`,     :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_dpp_sdwa_abs_neg>`                    :ref:`clamp<amdgpu_synid_clamp>` :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
-    v_mul_lo_u16                   :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_0>`,        :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`
-    v_mul_lo_u16_dpp               :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`,       :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`                      :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
-    v_mul_lo_u16_sdwa              :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`,     :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`                    :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
-    v_mul_u32_u24                  :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_0>`,        :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`
-    v_mul_u32_u24_dpp              :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`,       :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`                      :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
-    v_mul_u32_u24_sdwa             :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`,     :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`                    :ref:`clamp<amdgpu_synid_clamp>` :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
-    v_or_b32                       :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_0>`,        :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`
-    v_or_b32_dpp                   :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`,       :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`                      :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
-    v_or_b32_sdwa                  :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`,     :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`                    :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
-    v_sub_f16                      :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_0>`,        :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`
-    v_sub_f16_dpp                  :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_dpp_sdwa_abs_neg>`,     :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_dpp_sdwa_abs_neg>`                    :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
-    v_sub_f16_sdwa                 :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_dpp_sdwa_abs_neg>`,     :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_dpp_sdwa_abs_neg>`                    :ref:`clamp<amdgpu_synid_clamp>` :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
-    v_sub_f32                      :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_0>`,        :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`
-    v_sub_f32_dpp                  :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_dpp_sdwa_abs_neg>`,     :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_dpp_sdwa_abs_neg>`                    :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
-    v_sub_f32_sdwa                 :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_dpp_sdwa_abs_neg>`,     :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_dpp_sdwa_abs_neg>`                    :ref:`clamp<amdgpu_synid_clamp>` :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
-    v_sub_u16                      :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_0>`,        :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`
-    v_sub_u16_dpp                  :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`,       :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`                      :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
-    v_sub_u16_sdwa                 :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`,     :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`                    :ref:`clamp<amdgpu_synid_clamp>` :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
-    v_sub_u32                      :ref:`vdst<amdgpu_synid8_vdst32_0>`,     :ref:`vcc<amdgpu_synid8_vcc_64>`,      :ref:`src0<amdgpu_synid8_src32_0>`,        :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`
-    v_sub_u32_dpp                  :ref:`vdst<amdgpu_synid8_vdst32_0>`,     :ref:`vcc<amdgpu_synid8_vcc_64>`,      :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`,       :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`                      :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
-    v_sub_u32_sdwa                 :ref:`vdst<amdgpu_synid8_vdst32_0>`,     :ref:`vcc<amdgpu_synid8_vcc_64>`,      :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`,     :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`                    :ref:`clamp<amdgpu_synid_clamp>` :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
-    v_subb_u32                     :ref:`vdst<amdgpu_synid8_vdst32_0>`,     :ref:`vcc<amdgpu_synid8_vcc_64>`,      :ref:`src0<amdgpu_synid8_src32_0>`,        :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`,      :ref:`vcc<amdgpu_synid8_vcc_64>`
-    v_subb_u32_dpp                 :ref:`vdst<amdgpu_synid8_vdst32_0>`,     :ref:`vcc<amdgpu_synid8_vcc_64>`,      :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`,       :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`,      :ref:`vcc<amdgpu_synid8_vcc_64>`            :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
-    v_subb_u32_sdwa                :ref:`vdst<amdgpu_synid8_vdst32_0>`,     :ref:`vcc<amdgpu_synid8_vcc_64>`,      :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`,     :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`,    :ref:`vcc<amdgpu_synid8_vcc_64>`            :ref:`clamp<amdgpu_synid_clamp>` :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
-    v_subbrev_u32                  :ref:`vdst<amdgpu_synid8_vdst32_0>`,     :ref:`vcc<amdgpu_synid8_vcc_64>`,      :ref:`src0<amdgpu_synid8_src32_0>`,        :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`,      :ref:`vcc<amdgpu_synid8_vcc_64>`
-    v_subbrev_u32_dpp              :ref:`vdst<amdgpu_synid8_vdst32_0>`,     :ref:`vcc<amdgpu_synid8_vcc_64>`,      :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`,       :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`,      :ref:`vcc<amdgpu_synid8_vcc_64>`            :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
-    v_subbrev_u32_sdwa             :ref:`vdst<amdgpu_synid8_vdst32_0>`,     :ref:`vcc<amdgpu_synid8_vcc_64>`,      :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`,     :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`,    :ref:`vcc<amdgpu_synid8_vcc_64>`            :ref:`clamp<amdgpu_synid_clamp>` :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
-    v_subrev_f16                   :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_0>`,        :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`
-    v_subrev_f16_dpp               :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_dpp_sdwa_abs_neg>`,     :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_dpp_sdwa_abs_neg>`                    :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
-    v_subrev_f16_sdwa              :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_dpp_sdwa_abs_neg>`,     :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_dpp_sdwa_abs_neg>`                    :ref:`clamp<amdgpu_synid_clamp>` :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
-    v_subrev_f32                   :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_0>`,        :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`
-    v_subrev_f32_dpp               :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_dpp_sdwa_abs_neg>`,     :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_dpp_sdwa_abs_neg>`                    :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
-    v_subrev_f32_sdwa              :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_dpp_sdwa_abs_neg>`,     :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_dpp_sdwa_abs_neg>`                    :ref:`clamp<amdgpu_synid_clamp>` :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
-    v_subrev_u16                   :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_0>`,        :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`
-    v_subrev_u16_dpp               :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`,       :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`                      :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
-    v_subrev_u16_sdwa              :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`,     :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`                    :ref:`clamp<amdgpu_synid_clamp>` :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
-    v_subrev_u32                   :ref:`vdst<amdgpu_synid8_vdst32_0>`,     :ref:`vcc<amdgpu_synid8_vcc_64>`,      :ref:`src0<amdgpu_synid8_src32_0>`,        :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`
-    v_subrev_u32_dpp               :ref:`vdst<amdgpu_synid8_vdst32_0>`,     :ref:`vcc<amdgpu_synid8_vcc_64>`,      :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`,       :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`                      :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
-    v_subrev_u32_sdwa              :ref:`vdst<amdgpu_synid8_vdst32_0>`,     :ref:`vcc<amdgpu_synid8_vcc_64>`,      :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`,     :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`                    :ref:`clamp<amdgpu_synid_clamp>` :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
-    v_xor_b32                      :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_0>`,        :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`
-    v_xor_b32_dpp                  :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`,       :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`                      :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
-    v_xor_b32_sdwa                 :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`,     :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`                    :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
+    **INSTRUCTION**           **DST0**  **DST1** **SRC0**         **SRC1**        **SRC2**  **MODIFIERS**
+    \ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|
+    v_add_f16             :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`src0<amdgpu_synid8_src32_0>`,        :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`
+    v_add_f16_dpp         :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_dpp_sdwa_abs_neg>`,     :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_dpp_sdwa_abs_neg>`           :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
+    v_add_f16_sdwa        :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_dpp_sdwa_abs_neg>`,     :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_dpp_sdwa_abs_neg>`           :ref:`clamp<amdgpu_synid_clamp>` :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
+    v_add_f32             :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`src0<amdgpu_synid8_src32_0>`,        :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`
+    v_add_f32_dpp         :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_dpp_sdwa_abs_neg>`,     :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_dpp_sdwa_abs_neg>`           :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
+    v_add_f32_sdwa        :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_dpp_sdwa_abs_neg>`,     :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_dpp_sdwa_abs_neg>`           :ref:`clamp<amdgpu_synid_clamp>` :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
+    v_add_u16             :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`src0<amdgpu_synid8_src32_0>`,        :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`
+    v_add_u16_dpp         :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`,       :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`             :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
+    v_add_u16_sdwa        :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`,     :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`           :ref:`clamp<amdgpu_synid_clamp>` :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
+    v_add_u32             :ref:`vdst<amdgpu_synid8_vdst32_0>`, :ref:`vcc<amdgpu_synid8_vcc_64>`, :ref:`src0<amdgpu_synid8_src32_0>`,        :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`
+    v_add_u32_dpp         :ref:`vdst<amdgpu_synid8_vdst32_0>`, :ref:`vcc<amdgpu_synid8_vcc_64>`, :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`,       :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`             :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
+    v_add_u32_sdwa        :ref:`vdst<amdgpu_synid8_vdst32_0>`, :ref:`vcc<amdgpu_synid8_vcc_64>`, :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`,     :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`           :ref:`clamp<amdgpu_synid_clamp>` :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
+    v_addc_u32            :ref:`vdst<amdgpu_synid8_vdst32_0>`, :ref:`vcc<amdgpu_synid8_vcc_64>`, :ref:`src0<amdgpu_synid8_src32_0>`,        :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`,      :ref:`vcc<amdgpu_synid8_vcc_64>`
+    v_addc_u32_dpp        :ref:`vdst<amdgpu_synid8_vdst32_0>`, :ref:`vcc<amdgpu_synid8_vcc_64>`, :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`,       :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`,      :ref:`vcc<amdgpu_synid8_vcc_64>`   :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
+    v_addc_u32_sdwa       :ref:`vdst<amdgpu_synid8_vdst32_0>`, :ref:`vcc<amdgpu_synid8_vcc_64>`, :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`,     :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`,    :ref:`vcc<amdgpu_synid8_vcc_64>`   :ref:`clamp<amdgpu_synid_clamp>` :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
+    v_and_b32             :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`src0<amdgpu_synid8_src32_0>`,        :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`
+    v_and_b32_dpp         :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`,       :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`             :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
+    v_and_b32_sdwa        :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`,     :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`           :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
+    v_ashrrev_i16         :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`src0<amdgpu_synid8_src32_0>`::ref:`u16<amdgpu_synid8_type_dev>`,    :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`
+    v_ashrrev_i16_dpp     :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`u16<amdgpu_synid8_type_dev>`,   :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`             :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
+    v_ashrrev_i16_sdwa    :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`::ref:`u16<amdgpu_synid8_type_dev>`, :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`           :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
+    v_ashrrev_i32         :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`src0<amdgpu_synid8_src32_0>`::ref:`u32<amdgpu_synid8_type_dev>`,    :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`
+    v_ashrrev_i32_dpp     :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`u32<amdgpu_synid8_type_dev>`,   :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`             :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
+    v_ashrrev_i32_sdwa    :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`::ref:`u32<amdgpu_synid8_type_dev>`, :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`           :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
+    v_cndmask_b32         :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`src0<amdgpu_synid8_src32_0>`,        :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`,      :ref:`vcc<amdgpu_synid8_vcc_64>`
+    v_cndmask_b32_dpp     :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`,       :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`,      :ref:`vcc<amdgpu_synid8_vcc_64>`   :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
+    v_cndmask_b32_sdwa    :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`,     :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`,    :ref:`vcc<amdgpu_synid8_vcc_64>`   :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
+    v_ldexp_f16           :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`src0<amdgpu_synid8_src32_0>`,        :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`i16<amdgpu_synid8_type_dev>`
+    v_ldexp_f16_dpp       :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_dpp_sdwa_abs_neg>`,     :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`i16<amdgpu_synid8_type_dev>`         :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
+    v_ldexp_f16_sdwa      :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_dpp_sdwa_abs_neg>`,     :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`::ref:`i16<amdgpu_synid8_type_dev>`       :ref:`clamp<amdgpu_synid_clamp>` :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
+    v_lshlrev_b16         :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`src0<amdgpu_synid8_src32_0>`::ref:`u16<amdgpu_synid8_type_dev>`,    :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`
+    v_lshlrev_b16_dpp     :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`u16<amdgpu_synid8_type_dev>`,   :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`             :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
+    v_lshlrev_b16_sdwa    :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`::ref:`u16<amdgpu_synid8_type_dev>`, :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`           :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
+    v_lshlrev_b32         :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`src0<amdgpu_synid8_src32_0>`::ref:`u32<amdgpu_synid8_type_dev>`,    :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`
+    v_lshlrev_b32_dpp     :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`u32<amdgpu_synid8_type_dev>`,   :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`             :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
+    v_lshlrev_b32_sdwa    :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`::ref:`u32<amdgpu_synid8_type_dev>`, :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`           :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
+    v_lshrrev_b16         :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`src0<amdgpu_synid8_src32_0>`::ref:`u16<amdgpu_synid8_type_dev>`,    :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`
+    v_lshrrev_b16_dpp     :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`u16<amdgpu_synid8_type_dev>`,   :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`             :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
+    v_lshrrev_b16_sdwa    :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`::ref:`u16<amdgpu_synid8_type_dev>`, :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`           :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
+    v_lshrrev_b32         :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`src0<amdgpu_synid8_src32_0>`::ref:`u32<amdgpu_synid8_type_dev>`,    :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`
+    v_lshrrev_b32_dpp     :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`u32<amdgpu_synid8_type_dev>`,   :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`             :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
+    v_lshrrev_b32_sdwa    :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`::ref:`u32<amdgpu_synid8_type_dev>`, :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`           :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
+    v_mac_f16             :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`src0<amdgpu_synid8_src32_0>`,        :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`
+    v_mac_f16_dpp         :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_dpp_sdwa_abs_neg>`,     :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_dpp_sdwa_abs_neg>`           :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
+    v_mac_f16_sdwa        :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_dpp_sdwa_abs_neg>`,     :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_dpp_sdwa_abs_neg>`           :ref:`clamp<amdgpu_synid_clamp>` :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
+    v_mac_f32             :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`src0<amdgpu_synid8_src32_0>`,        :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`
+    v_mac_f32_dpp         :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_dpp_sdwa_abs_neg>`,     :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_dpp_sdwa_abs_neg>`           :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
+    v_mac_f32_sdwa        :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_dpp_sdwa_abs_neg>`,     :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_dpp_sdwa_abs_neg>`           :ref:`clamp<amdgpu_synid_clamp>` :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
+    v_madak_f16           :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`src0<amdgpu_synid8_src32_0>`,        :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`,      :ref:`imm32<amdgpu_synid8_fimm16>`
+    v_madak_f32           :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`src0<amdgpu_synid8_src32_0>`,        :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`,      :ref:`imm32<amdgpu_synid8_fimm32>`
+    v_madmk_f16           :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`src0<amdgpu_synid8_src32_0>`,        :ref:`imm32<amdgpu_synid8_fimm16>`,      :ref:`vsrc2<amdgpu_synid8_vsrc32_0>`
+    v_madmk_f32           :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`src0<amdgpu_synid8_src32_0>`,        :ref:`imm32<amdgpu_synid8_fimm32>`,      :ref:`vsrc2<amdgpu_synid8_vsrc32_0>`
+    v_max_f16             :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`src0<amdgpu_synid8_src32_0>`,        :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`
+    v_max_f16_dpp         :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_dpp_sdwa_abs_neg>`,     :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_dpp_sdwa_abs_neg>`           :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
+    v_max_f16_sdwa        :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_dpp_sdwa_abs_neg>`,     :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_dpp_sdwa_abs_neg>`           :ref:`clamp<amdgpu_synid_clamp>` :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
+    v_max_f32             :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`src0<amdgpu_synid8_src32_0>`,        :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`
+    v_max_f32_dpp         :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_dpp_sdwa_abs_neg>`,     :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_dpp_sdwa_abs_neg>`           :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
+    v_max_f32_sdwa        :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_dpp_sdwa_abs_neg>`,     :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_dpp_sdwa_abs_neg>`           :ref:`clamp<amdgpu_synid_clamp>` :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
+    v_max_i16             :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`src0<amdgpu_synid8_src32_0>`,        :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`
+    v_max_i16_dpp         :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`,       :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`             :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
+    v_max_i16_sdwa        :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`,     :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`           :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
+    v_max_i32             :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`src0<amdgpu_synid8_src32_0>`,        :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`
+    v_max_i32_dpp         :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`,       :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`             :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
+    v_max_i32_sdwa        :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`,     :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`           :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
+    v_max_u16             :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`src0<amdgpu_synid8_src32_0>`,        :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`
+    v_max_u16_dpp         :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`,       :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`             :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
+    v_max_u16_sdwa        :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`,     :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`           :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
+    v_max_u32             :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`src0<amdgpu_synid8_src32_0>`,        :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`
+    v_max_u32_dpp         :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`,       :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`             :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
+    v_max_u32_sdwa        :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`,     :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`           :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
+    v_min_f16             :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`src0<amdgpu_synid8_src32_0>`,        :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`
+    v_min_f16_dpp         :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_dpp_sdwa_abs_neg>`,     :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_dpp_sdwa_abs_neg>`           :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
+    v_min_f16_sdwa        :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_dpp_sdwa_abs_neg>`,     :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_dpp_sdwa_abs_neg>`           :ref:`clamp<amdgpu_synid_clamp>` :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
+    v_min_f32             :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`src0<amdgpu_synid8_src32_0>`,        :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`
+    v_min_f32_dpp         :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_dpp_sdwa_abs_neg>`,     :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_dpp_sdwa_abs_neg>`           :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
+    v_min_f32_sdwa        :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_dpp_sdwa_abs_neg>`,     :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_dpp_sdwa_abs_neg>`           :ref:`clamp<amdgpu_synid_clamp>` :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
+    v_min_i16             :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`src0<amdgpu_synid8_src32_0>`,        :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`
+    v_min_i16_dpp         :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`,       :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`             :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
+    v_min_i16_sdwa        :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`,     :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`           :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
+    v_min_i32             :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`src0<amdgpu_synid8_src32_0>`,        :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`
+    v_min_i32_dpp         :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`,       :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`             :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
+    v_min_i32_sdwa        :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`,     :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`           :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
+    v_min_u16             :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`src0<amdgpu_synid8_src32_0>`,        :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`
+    v_min_u16_dpp         :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`,       :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`             :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
+    v_min_u16_sdwa        :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`,     :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`           :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
+    v_min_u32             :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`src0<amdgpu_synid8_src32_0>`,        :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`
+    v_min_u32_dpp         :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`,       :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`             :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
+    v_min_u32_sdwa        :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`,     :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`           :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
+    v_mul_f16             :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`src0<amdgpu_synid8_src32_0>`,        :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`
+    v_mul_f16_dpp         :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_dpp_sdwa_abs_neg>`,     :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_dpp_sdwa_abs_neg>`           :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
+    v_mul_f16_sdwa        :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_dpp_sdwa_abs_neg>`,     :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_dpp_sdwa_abs_neg>`           :ref:`clamp<amdgpu_synid_clamp>` :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
+    v_mul_f32             :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`src0<amdgpu_synid8_src32_0>`,        :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`
+    v_mul_f32_dpp         :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_dpp_sdwa_abs_neg>`,     :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_dpp_sdwa_abs_neg>`           :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
+    v_mul_f32_sdwa        :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_dpp_sdwa_abs_neg>`,     :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_dpp_sdwa_abs_neg>`           :ref:`clamp<amdgpu_synid_clamp>` :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
+    v_mul_hi_i32_i24      :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`src0<amdgpu_synid8_src32_0>`,        :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`
+    v_mul_hi_i32_i24_dpp  :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`,       :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`             :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
+    v_mul_hi_i32_i24_sdwa :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`,     :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`           :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
+    v_mul_hi_u32_u24      :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`src0<amdgpu_synid8_src32_0>`,        :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`
+    v_mul_hi_u32_u24_dpp  :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`,       :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`             :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
+    v_mul_hi_u32_u24_sdwa :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`,     :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`           :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
+    v_mul_i32_i24         :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`src0<amdgpu_synid8_src32_0>`,        :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`
+    v_mul_i32_i24_dpp     :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`,       :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`             :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
+    v_mul_i32_i24_sdwa    :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`,     :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`           :ref:`clamp<amdgpu_synid_clamp>` :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
+    v_mul_legacy_f32      :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`src0<amdgpu_synid8_src32_0>`,        :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`
+    v_mul_legacy_f32_dpp  :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_dpp_sdwa_abs_neg>`,     :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_dpp_sdwa_abs_neg>`           :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
+    v_mul_legacy_f32_sdwa :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_dpp_sdwa_abs_neg>`,     :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_dpp_sdwa_abs_neg>`           :ref:`clamp<amdgpu_synid_clamp>` :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
+    v_mul_lo_u16          :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`src0<amdgpu_synid8_src32_0>`,        :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`
+    v_mul_lo_u16_dpp      :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`,       :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`             :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
+    v_mul_lo_u16_sdwa     :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`,     :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`           :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
+    v_mul_u32_u24         :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`src0<amdgpu_synid8_src32_0>`,        :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`
+    v_mul_u32_u24_dpp     :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`,       :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`             :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
+    v_mul_u32_u24_sdwa    :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`,     :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`           :ref:`clamp<amdgpu_synid_clamp>` :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
+    v_or_b32              :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`src0<amdgpu_synid8_src32_0>`,        :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`
+    v_or_b32_dpp          :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`,       :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`             :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
+    v_or_b32_sdwa         :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`,     :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`           :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
+    v_sub_f16             :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`src0<amdgpu_synid8_src32_0>`,        :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`
+    v_sub_f16_dpp         :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_dpp_sdwa_abs_neg>`,     :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_dpp_sdwa_abs_neg>`           :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
+    v_sub_f16_sdwa        :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_dpp_sdwa_abs_neg>`,     :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_dpp_sdwa_abs_neg>`           :ref:`clamp<amdgpu_synid_clamp>` :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
+    v_sub_f32             :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`src0<amdgpu_synid8_src32_0>`,        :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`
+    v_sub_f32_dpp         :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_dpp_sdwa_abs_neg>`,     :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_dpp_sdwa_abs_neg>`           :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
+    v_sub_f32_sdwa        :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_dpp_sdwa_abs_neg>`,     :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_dpp_sdwa_abs_neg>`           :ref:`clamp<amdgpu_synid_clamp>` :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
+    v_sub_u16             :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`src0<amdgpu_synid8_src32_0>`,        :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`
+    v_sub_u16_dpp         :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`,       :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`             :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
+    v_sub_u16_sdwa        :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`,     :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`           :ref:`clamp<amdgpu_synid_clamp>` :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
+    v_sub_u32             :ref:`vdst<amdgpu_synid8_vdst32_0>`, :ref:`vcc<amdgpu_synid8_vcc_64>`, :ref:`src0<amdgpu_synid8_src32_0>`,        :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`
+    v_sub_u32_dpp         :ref:`vdst<amdgpu_synid8_vdst32_0>`, :ref:`vcc<amdgpu_synid8_vcc_64>`, :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`,       :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`             :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
+    v_sub_u32_sdwa        :ref:`vdst<amdgpu_synid8_vdst32_0>`, :ref:`vcc<amdgpu_synid8_vcc_64>`, :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`,     :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`           :ref:`clamp<amdgpu_synid_clamp>` :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
+    v_subb_u32            :ref:`vdst<amdgpu_synid8_vdst32_0>`, :ref:`vcc<amdgpu_synid8_vcc_64>`, :ref:`src0<amdgpu_synid8_src32_0>`,        :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`,      :ref:`vcc<amdgpu_synid8_vcc_64>`
+    v_subb_u32_dpp        :ref:`vdst<amdgpu_synid8_vdst32_0>`, :ref:`vcc<amdgpu_synid8_vcc_64>`, :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`,       :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`,      :ref:`vcc<amdgpu_synid8_vcc_64>`   :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
+    v_subb_u32_sdwa       :ref:`vdst<amdgpu_synid8_vdst32_0>`, :ref:`vcc<amdgpu_synid8_vcc_64>`, :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`,     :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`,    :ref:`vcc<amdgpu_synid8_vcc_64>`   :ref:`clamp<amdgpu_synid_clamp>` :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
+    v_subbrev_u32         :ref:`vdst<amdgpu_synid8_vdst32_0>`, :ref:`vcc<amdgpu_synid8_vcc_64>`, :ref:`src0<amdgpu_synid8_src32_0>`,        :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`,      :ref:`vcc<amdgpu_synid8_vcc_64>`
+    v_subbrev_u32_dpp     :ref:`vdst<amdgpu_synid8_vdst32_0>`, :ref:`vcc<amdgpu_synid8_vcc_64>`, :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`,       :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`,      :ref:`vcc<amdgpu_synid8_vcc_64>`   :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
+    v_subbrev_u32_sdwa    :ref:`vdst<amdgpu_synid8_vdst32_0>`, :ref:`vcc<amdgpu_synid8_vcc_64>`, :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`,     :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`,    :ref:`vcc<amdgpu_synid8_vcc_64>`   :ref:`clamp<amdgpu_synid_clamp>` :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
+    v_subrev_f16          :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`src0<amdgpu_synid8_src32_0>`,        :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`
+    v_subrev_f16_dpp      :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_dpp_sdwa_abs_neg>`,     :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_dpp_sdwa_abs_neg>`           :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
+    v_subrev_f16_sdwa     :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_dpp_sdwa_abs_neg>`,     :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_dpp_sdwa_abs_neg>`           :ref:`clamp<amdgpu_synid_clamp>` :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
+    v_subrev_f32          :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`src0<amdgpu_synid8_src32_0>`,        :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`
+    v_subrev_f32_dpp      :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_dpp_sdwa_abs_neg>`,     :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_dpp_sdwa_abs_neg>`           :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
+    v_subrev_f32_sdwa     :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_dpp_sdwa_abs_neg>`,     :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_dpp_sdwa_abs_neg>`           :ref:`clamp<amdgpu_synid_clamp>` :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
+    v_subrev_u16          :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`src0<amdgpu_synid8_src32_0>`,        :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`
+    v_subrev_u16_dpp      :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`,       :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`             :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
+    v_subrev_u16_sdwa     :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`,     :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`           :ref:`clamp<amdgpu_synid_clamp>` :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
+    v_subrev_u32          :ref:`vdst<amdgpu_synid8_vdst32_0>`, :ref:`vcc<amdgpu_synid8_vcc_64>`, :ref:`src0<amdgpu_synid8_src32_0>`,        :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`
+    v_subrev_u32_dpp      :ref:`vdst<amdgpu_synid8_vdst32_0>`, :ref:`vcc<amdgpu_synid8_vcc_64>`, :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`,       :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`             :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
+    v_subrev_u32_sdwa     :ref:`vdst<amdgpu_synid8_vdst32_0>`, :ref:`vcc<amdgpu_synid8_vcc_64>`, :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`,     :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`           :ref:`clamp<amdgpu_synid_clamp>` :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
+    v_xor_b32             :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`src0<amdgpu_synid8_src32_0>`,        :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`
+    v_xor_b32_dpp         :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`,       :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`             :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
+    v_xor_b32_sdwa        :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`,     :ref:`vsrc1<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_sdwa_sext>`           :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
 
 VOP3
 -----------------------
 
 .. parsed-literal::
 
-    **INSTRUCTION**                    **DST0**       **DST1**      **SRC0**         **SRC1**        **SRC2**               **MODIFIERS**
-    \ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|
-    v_add_f16_e64                  :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_add_f32_e64                  :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_add_f64                      :ref:`vdst<amdgpu_synid8_vdst64_0>`,                :ref:`src0<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_add_u16_e64                  :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
-    v_add_u32_e64                  :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`sdst<amdgpu_synid8_sdst64_0>`,     :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
-    v_addc_u32_e64                 :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`sdst<amdgpu_synid8_sdst64_0>`,     :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`,       :ref:`ssrc2<amdgpu_synid8_ssrc64_1>`
-    v_alignbit_b32                 :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`,       :ref:`src2<amdgpu_synid8_src32_1>`
-    v_alignbyte_b32                :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`,       :ref:`src2<amdgpu_synid8_src32_1>`
-    v_and_b32_e64                  :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
-    v_ashrrev_i16_e64              :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`::ref:`u16<amdgpu_synid8_type_dev>`,    :ref:`src1<amdgpu_synid8_src32_1>`
-    v_ashrrev_i32_e64              :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`::ref:`u32<amdgpu_synid8_type_dev>`,    :ref:`src1<amdgpu_synid8_src32_1>`
-    v_ashrrev_i64                  :ref:`vdst<amdgpu_synid8_vdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`::ref:`u32<amdgpu_synid8_type_dev>`,    :ref:`src1<amdgpu_synid8_src64_1>`
-    v_bcnt_u32_b32                 :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
-    v_bfe_i32                      :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`::ref:`u32<amdgpu_synid8_type_dev>`,   :ref:`src2<amdgpu_synid8_src32_1>`::ref:`u32<amdgpu_synid8_type_dev>`
-    v_bfe_u32                      :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`,       :ref:`src2<amdgpu_synid8_src32_1>`
-    v_bfi_b32                      :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`,       :ref:`src2<amdgpu_synid8_src32_1>`
-    v_bfm_b32                      :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
-    v_bfrev_b32_e64                :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src<amdgpu_synid8_src32_1>`
-    v_ceil_f16_e64                 :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                                       :ref:`clamp<amdgpu_synid_clamp>`
-    v_ceil_f32_e64                 :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                                       :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_ceil_f64_e64                 :ref:`vdst<amdgpu_synid8_vdst64_0>`,                :ref:`src<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                                       :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    **INSTRUCTION**             **DST0**       **DST1**     **SRC0**         **SRC1**        **SRC2**            **MODIFIERS**
+    \ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|
+    v_add_f16_e64           :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_add_f32_e64           :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_add_f64               :ref:`vdst<amdgpu_synid8_vdst64_0>`,               :ref:`src0<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_add_u16_e64           :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
+    v_add_u32_e64           :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`sdst<amdgpu_synid8_sdst64_0>`,    :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
+    v_addc_u32_e64          :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`sdst<amdgpu_synid8_sdst64_0>`,    :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`,       :ref:`ssrc2<amdgpu_synid8_ssrc64_1>`
+    v_alignbit_b32          :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`,       :ref:`src2<amdgpu_synid8_src32_1>`
+    v_alignbyte_b32         :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`,       :ref:`src2<amdgpu_synid8_src32_1>`
+    v_and_b32_e64           :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
+    v_ashrrev_i16_e64       :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`::ref:`u16<amdgpu_synid8_type_dev>`,    :ref:`src1<amdgpu_synid8_src32_1>`
+    v_ashrrev_i32_e64       :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`::ref:`u32<amdgpu_synid8_type_dev>`,    :ref:`src1<amdgpu_synid8_src32_1>`
+    v_ashrrev_i64           :ref:`vdst<amdgpu_synid8_vdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`::ref:`u32<amdgpu_synid8_type_dev>`,    :ref:`src1<amdgpu_synid8_src64_1>`
+    v_bcnt_u32_b32          :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
+    v_bfe_i32               :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`::ref:`u32<amdgpu_synid8_type_dev>`,   :ref:`src2<amdgpu_synid8_src32_1>`::ref:`u32<amdgpu_synid8_type_dev>`
+    v_bfe_u32               :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`,       :ref:`src2<amdgpu_synid8_src32_1>`
+    v_bfi_b32               :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`,       :ref:`src2<amdgpu_synid8_src32_1>`
+    v_bfm_b32               :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
+    v_bfrev_b32_e64         :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src<amdgpu_synid8_src32_1>`
+    v_ceil_f16_e64          :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                                    :ref:`clamp<amdgpu_synid_clamp>`
+    v_ceil_f32_e64          :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                                    :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_ceil_f64_e64          :ref:`vdst<amdgpu_synid8_vdst64_0>`,               :ref:`src<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                                    :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
     v_clrexcp_e64
-    v_cmp_class_f16_e64            :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`b32<amdgpu_synid8_type_dev>`
-    v_cmp_class_f32_e64            :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`b32<amdgpu_synid8_type_dev>`
-    v_cmp_class_f64_e64            :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`b32<amdgpu_synid8_type_dev>`
-    v_cmp_eq_f16_e64               :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmp_eq_f32_e64               :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmp_eq_f64_e64               :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmp_eq_i16_e64               :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
-    v_cmp_eq_i32_e64               :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
-    v_cmp_eq_i64_e64               :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src64_1>`,        :ref:`src1<amdgpu_synid8_src64_1>`
-    v_cmp_eq_u16_e64               :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
-    v_cmp_eq_u32_e64               :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
-    v_cmp_eq_u64_e64               :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src64_1>`,        :ref:`src1<amdgpu_synid8_src64_1>`
-    v_cmp_f_f16_e64                :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmp_f_f32_e64                :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmp_f_f64_e64                :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmp_f_i16_e64                :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
-    v_cmp_f_i32_e64                :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
-    v_cmp_f_i64_e64                :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src64_1>`,        :ref:`src1<amdgpu_synid8_src64_1>`
-    v_cmp_f_u16_e64                :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
-    v_cmp_f_u32_e64                :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
-    v_cmp_f_u64_e64                :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src64_1>`,        :ref:`src1<amdgpu_synid8_src64_1>`
-    v_cmp_ge_f16_e64               :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmp_ge_f32_e64               :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmp_ge_f64_e64               :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmp_ge_i16_e64               :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
-    v_cmp_ge_i32_e64               :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
-    v_cmp_ge_i64_e64               :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src64_1>`,        :ref:`src1<amdgpu_synid8_src64_1>`
-    v_cmp_ge_u16_e64               :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
-    v_cmp_ge_u32_e64               :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
-    v_cmp_ge_u64_e64               :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src64_1>`,        :ref:`src1<amdgpu_synid8_src64_1>`
-    v_cmp_gt_f16_e64               :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmp_gt_f32_e64               :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmp_gt_f64_e64               :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmp_gt_i16_e64               :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
-    v_cmp_gt_i32_e64               :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
-    v_cmp_gt_i64_e64               :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src64_1>`,        :ref:`src1<amdgpu_synid8_src64_1>`
-    v_cmp_gt_u16_e64               :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
-    v_cmp_gt_u32_e64               :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
-    v_cmp_gt_u64_e64               :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src64_1>`,        :ref:`src1<amdgpu_synid8_src64_1>`
-    v_cmp_le_f16_e64               :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmp_le_f32_e64               :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmp_le_f64_e64               :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmp_le_i16_e64               :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
-    v_cmp_le_i32_e64               :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
-    v_cmp_le_i64_e64               :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src64_1>`,        :ref:`src1<amdgpu_synid8_src64_1>`
-    v_cmp_le_u16_e64               :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
-    v_cmp_le_u32_e64               :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
-    v_cmp_le_u64_e64               :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src64_1>`,        :ref:`src1<amdgpu_synid8_src64_1>`
-    v_cmp_lg_f16_e64               :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmp_lg_f32_e64               :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmp_lg_f64_e64               :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmp_lt_f16_e64               :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmp_lt_f32_e64               :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmp_lt_f64_e64               :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmp_lt_i16_e64               :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
-    v_cmp_lt_i32_e64               :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
-    v_cmp_lt_i64_e64               :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src64_1>`,        :ref:`src1<amdgpu_synid8_src64_1>`
-    v_cmp_lt_u16_e64               :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
-    v_cmp_lt_u32_e64               :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
-    v_cmp_lt_u64_e64               :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src64_1>`,        :ref:`src1<amdgpu_synid8_src64_1>`
-    v_cmp_ne_i16_e64               :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
-    v_cmp_ne_i32_e64               :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
-    v_cmp_ne_i64_e64               :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src64_1>`,        :ref:`src1<amdgpu_synid8_src64_1>`
-    v_cmp_ne_u16_e64               :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
-    v_cmp_ne_u32_e64               :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
-    v_cmp_ne_u64_e64               :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src64_1>`,        :ref:`src1<amdgpu_synid8_src64_1>`
-    v_cmp_neq_f16_e64              :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmp_neq_f32_e64              :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmp_neq_f64_e64              :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmp_nge_f16_e64              :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmp_nge_f32_e64              :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmp_nge_f64_e64              :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmp_ngt_f16_e64              :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmp_ngt_f32_e64              :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmp_ngt_f64_e64              :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmp_nle_f16_e64              :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmp_nle_f32_e64              :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmp_nle_f64_e64              :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmp_nlg_f16_e64              :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmp_nlg_f32_e64              :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmp_nlg_f64_e64              :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmp_nlt_f16_e64              :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmp_nlt_f32_e64              :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmp_nlt_f64_e64              :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmp_o_f16_e64                :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmp_o_f32_e64                :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmp_o_f64_e64                :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmp_t_i16_e64                :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
-    v_cmp_t_i32_e64                :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
-    v_cmp_t_i64_e64                :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src64_1>`,        :ref:`src1<amdgpu_synid8_src64_1>`
-    v_cmp_t_u16_e64                :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
-    v_cmp_t_u32_e64                :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
-    v_cmp_t_u64_e64                :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src64_1>`,        :ref:`src1<amdgpu_synid8_src64_1>`
-    v_cmp_tru_f16_e64              :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmp_tru_f32_e64              :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmp_tru_f64_e64              :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmp_u_f16_e64                :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmp_u_f32_e64                :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmp_u_f64_e64                :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmpx_class_f16_e64           :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`b32<amdgpu_synid8_type_dev>`
-    v_cmpx_class_f32_e64           :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`b32<amdgpu_synid8_type_dev>`
-    v_cmpx_class_f64_e64           :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`b32<amdgpu_synid8_type_dev>`
-    v_cmpx_eq_f16_e64              :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmpx_eq_f32_e64              :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmpx_eq_f64_e64              :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmpx_eq_i16_e64              :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
-    v_cmpx_eq_i32_e64              :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
-    v_cmpx_eq_i64_e64              :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src64_1>`,        :ref:`src1<amdgpu_synid8_src64_1>`
-    v_cmpx_eq_u16_e64              :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
-    v_cmpx_eq_u32_e64              :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
-    v_cmpx_eq_u64_e64              :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src64_1>`,        :ref:`src1<amdgpu_synid8_src64_1>`
-    v_cmpx_f_f16_e64               :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmpx_f_f32_e64               :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmpx_f_f64_e64               :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmpx_f_i16_e64               :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
-    v_cmpx_f_i32_e64               :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
-    v_cmpx_f_i64_e64               :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src64_1>`,        :ref:`src1<amdgpu_synid8_src64_1>`
-    v_cmpx_f_u16_e64               :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
-    v_cmpx_f_u32_e64               :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
-    v_cmpx_f_u64_e64               :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src64_1>`,        :ref:`src1<amdgpu_synid8_src64_1>`
-    v_cmpx_ge_f16_e64              :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmpx_ge_f32_e64              :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmpx_ge_f64_e64              :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmpx_ge_i16_e64              :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
-    v_cmpx_ge_i32_e64              :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
-    v_cmpx_ge_i64_e64              :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src64_1>`,        :ref:`src1<amdgpu_synid8_src64_1>`
-    v_cmpx_ge_u16_e64              :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
-    v_cmpx_ge_u32_e64              :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
-    v_cmpx_ge_u64_e64              :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src64_1>`,        :ref:`src1<amdgpu_synid8_src64_1>`
-    v_cmpx_gt_f16_e64              :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmpx_gt_f32_e64              :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmpx_gt_f64_e64              :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmpx_gt_i16_e64              :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
-    v_cmpx_gt_i32_e64              :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
-    v_cmpx_gt_i64_e64              :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src64_1>`,        :ref:`src1<amdgpu_synid8_src64_1>`
-    v_cmpx_gt_u16_e64              :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
-    v_cmpx_gt_u32_e64              :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
-    v_cmpx_gt_u64_e64              :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src64_1>`,        :ref:`src1<amdgpu_synid8_src64_1>`
-    v_cmpx_le_f16_e64              :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmpx_le_f32_e64              :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmpx_le_f64_e64              :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmpx_le_i16_e64              :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
-    v_cmpx_le_i32_e64              :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
-    v_cmpx_le_i64_e64              :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src64_1>`,        :ref:`src1<amdgpu_synid8_src64_1>`
-    v_cmpx_le_u16_e64              :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
-    v_cmpx_le_u32_e64              :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
-    v_cmpx_le_u64_e64              :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src64_1>`,        :ref:`src1<amdgpu_synid8_src64_1>`
-    v_cmpx_lg_f16_e64              :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmpx_lg_f32_e64              :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmpx_lg_f64_e64              :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmpx_lt_f16_e64              :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmpx_lt_f32_e64              :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmpx_lt_f64_e64              :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmpx_lt_i16_e64              :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
-    v_cmpx_lt_i32_e64              :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
-    v_cmpx_lt_i64_e64              :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src64_1>`,        :ref:`src1<amdgpu_synid8_src64_1>`
-    v_cmpx_lt_u16_e64              :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
-    v_cmpx_lt_u32_e64              :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
-    v_cmpx_lt_u64_e64              :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src64_1>`,        :ref:`src1<amdgpu_synid8_src64_1>`
-    v_cmpx_ne_i16_e64              :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
-    v_cmpx_ne_i32_e64              :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
-    v_cmpx_ne_i64_e64              :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src64_1>`,        :ref:`src1<amdgpu_synid8_src64_1>`
-    v_cmpx_ne_u16_e64              :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
-    v_cmpx_ne_u32_e64              :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
-    v_cmpx_ne_u64_e64              :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src64_1>`,        :ref:`src1<amdgpu_synid8_src64_1>`
-    v_cmpx_neq_f16_e64             :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmpx_neq_f32_e64             :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmpx_neq_f64_e64             :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmpx_nge_f16_e64             :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmpx_nge_f32_e64             :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmpx_nge_f64_e64             :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmpx_ngt_f16_e64             :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmpx_ngt_f32_e64             :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmpx_ngt_f64_e64             :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmpx_nle_f16_e64             :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmpx_nle_f32_e64             :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmpx_nle_f64_e64             :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmpx_nlg_f16_e64             :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmpx_nlg_f32_e64             :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmpx_nlg_f64_e64             :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmpx_nlt_f16_e64             :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmpx_nlt_f32_e64             :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmpx_nlt_f64_e64             :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmpx_o_f16_e64               :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmpx_o_f32_e64               :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmpx_o_f64_e64               :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmpx_t_i16_e64               :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
-    v_cmpx_t_i32_e64               :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
-    v_cmpx_t_i64_e64               :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src64_1>`,        :ref:`src1<amdgpu_synid8_src64_1>`
-    v_cmpx_t_u16_e64               :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
-    v_cmpx_t_u32_e64               :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
-    v_cmpx_t_u64_e64               :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src64_1>`,        :ref:`src1<amdgpu_synid8_src64_1>`
-    v_cmpx_tru_f16_e64             :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmpx_tru_f32_e64             :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmpx_tru_f64_e64             :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmpx_u_f16_e64               :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmpx_u_f32_e64               :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmpx_u_f64_e64               :ref:`sdst<amdgpu_synid8_sdst64_0>`,                :ref:`src0<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cndmask_b32_e64              :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`,       :ref:`ssrc2<amdgpu_synid8_ssrc64_1>`
-    v_cos_f16_e64                  :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                                       :ref:`clamp<amdgpu_synid_clamp>`
-    v_cos_f32_e64                  :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                                       :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_cubeid_f32                   :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,     :ref:`src2<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`             :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_cubema_f32                   :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,     :ref:`src2<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`             :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_cubesc_f32                   :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,     :ref:`src2<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`             :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_cubetc_f32                   :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,     :ref:`src2<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`             :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_cvt_f16_f32_e64              :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                                       :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_cvt_f16_i16_e64              :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src<amdgpu_synid8_src32_1>`                                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cvt_f16_u16_e64              :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src<amdgpu_synid8_src32_1>`                                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cvt_f32_f16_e64              :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                                       :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_cvt_f32_f64_e64              :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                                       :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_cvt_f32_i32_e64              :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src<amdgpu_synid8_src32_1>`                                         :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_cvt_f32_u32_e64              :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src<amdgpu_synid8_src32_1>`                                         :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_cvt_f32_ubyte0_e64           :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src<amdgpu_synid8_src32_1>`                                         :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_cvt_f32_ubyte1_e64           :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src<amdgpu_synid8_src32_1>`                                         :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_cvt_f32_ubyte2_e64           :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src<amdgpu_synid8_src32_1>`                                         :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_cvt_f32_ubyte3_e64           :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src<amdgpu_synid8_src32_1>`                                         :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_cvt_f64_f32_e64              :ref:`vdst<amdgpu_synid8_vdst64_0>`,                :ref:`src<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                                       :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_cvt_f64_i32_e64              :ref:`vdst<amdgpu_synid8_vdst64_0>`,                :ref:`src<amdgpu_synid8_src32_1>`                                         :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_cvt_f64_u32_e64              :ref:`vdst<amdgpu_synid8_vdst64_0>`,                :ref:`src<amdgpu_synid8_src32_1>`                                         :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_cvt_flr_i32_f32_e64          :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`
-    v_cvt_i16_f16_e64              :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`
-    v_cvt_i32_f32_e64              :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`
-    v_cvt_i32_f64_e64              :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`
-    v_cvt_off_f32_i4_e64           :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src<amdgpu_synid8_src32_1>`                                         :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_cvt_pk_i16_i32               :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
-    v_cvt_pk_u16_u32               :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
-    v_cvt_pk_u8_f32                :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`u32<amdgpu_synid8_type_dev>`,   :ref:`src2<amdgpu_synid8_src32_1>`::ref:`u32<amdgpu_synid8_type_dev>`
-    v_cvt_pkaccum_u8_f32           :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`u32<amdgpu_synid8_type_dev>`
-    v_cvt_pknorm_i16_f32           :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`
-    v_cvt_pknorm_u16_f32           :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`
-    v_cvt_pkrtz_f16_f32            :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`
-    v_cvt_rpi_i32_f32_e64          :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`
-    v_cvt_u16_f16_e64              :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`
-    v_cvt_u32_f32_e64              :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`
-    v_cvt_u32_f64_e64              :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`
-    v_div_fixup_f16                :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,     :ref:`src2<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`             :ref:`clamp<amdgpu_synid_clamp>`
-    v_div_fixup_f32                :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,     :ref:`src2<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`             :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_div_fixup_f64                :ref:`vdst<amdgpu_synid8_vdst64_0>`,                :ref:`src0<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,     :ref:`src2<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`             :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_div_fmas_f32                 :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,     :ref:`src2<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`             :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_div_fmas_f64                 :ref:`vdst<amdgpu_synid8_vdst64_0>`,                :ref:`src0<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,     :ref:`src2<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`             :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_div_scale_f32                :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`vcc<amdgpu_synid8_vcc_64>`,      :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`,       :ref:`src2<amdgpu_synid8_src32_1>`
-    v_div_scale_f64                :ref:`vdst<amdgpu_synid8_vdst64_0>`,      :ref:`vcc<amdgpu_synid8_vcc_64>`,      :ref:`src0<amdgpu_synid8_src64_1>`,        :ref:`src1<amdgpu_synid8_src64_1>`,       :ref:`src2<amdgpu_synid8_src64_1>`
-    v_exp_f16_e64                  :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                                       :ref:`clamp<amdgpu_synid_clamp>`
-    v_exp_f32_e64                  :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                                       :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_exp_legacy_f32_e64           :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                                       :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_ffbh_i32_e64                 :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src<amdgpu_synid8_src32_1>`
-    v_ffbh_u32_e64                 :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src<amdgpu_synid8_src32_1>`
-    v_ffbl_b32_e64                 :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src<amdgpu_synid8_src32_1>`
-    v_floor_f16_e64                :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                                       :ref:`clamp<amdgpu_synid_clamp>`
-    v_floor_f32_e64                :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                                       :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_floor_f64_e64                :ref:`vdst<amdgpu_synid8_vdst64_0>`,                :ref:`src<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                                       :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_fma_f16                      :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,     :ref:`src2<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`             :ref:`clamp<amdgpu_synid_clamp>`
-    v_fma_f32                      :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,     :ref:`src2<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`             :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_fma_f64                      :ref:`vdst<amdgpu_synid8_vdst64_0>`,                :ref:`src0<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,     :ref:`src2<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`             :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_fract_f16_e64                :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                                       :ref:`clamp<amdgpu_synid_clamp>`
-    v_fract_f32_e64                :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                                       :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_fract_f64_e64                :ref:`vdst<amdgpu_synid8_vdst64_0>`,                :ref:`src<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                                       :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_frexp_exp_i16_f16_e64        :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`
-    v_frexp_exp_i32_f32_e64        :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`
-    v_frexp_exp_i32_f64_e64        :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`
-    v_frexp_mant_f16_e64           :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                                       :ref:`clamp<amdgpu_synid_clamp>`
-    v_frexp_mant_f32_e64           :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                                       :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_frexp_mant_f64_e64           :ref:`vdst<amdgpu_synid8_vdst64_0>`,                :ref:`src<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                                       :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_interp_mov_f32_e64           :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`param<amdgpu_synid8_param>`::ref:`b32<amdgpu_synid8_type_dev>`,   :ref:`attr<amdgpu_synid8_attr>`::ref:`b32<amdgpu_synid8_type_dev>`                       :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_interp_p1_f32_e64            :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`vsrc<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`attr<amdgpu_synid8_attr>`::ref:`b32<amdgpu_synid8_type_dev>`                       :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_interp_p1ll_f16              :ref:`vdst<amdgpu_synid8_vdst32_0>`::ref:`f32<amdgpu_synid8_type_dev>`,            :ref:`vsrc<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`::ref:`f32<amdgpu_synid8_type_dev>`,  :ref:`attr<amdgpu_synid8_attr>`::ref:`b32<amdgpu_synid8_type_dev>`                       :ref:`high<amdgpu_synid_high>` :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_interp_p1lv_f16              :ref:`vdst<amdgpu_synid8_vdst32_0>`::ref:`f32<amdgpu_synid8_type_dev>`,            :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`::ref:`f32<amdgpu_synid8_type_dev>`, :ref:`attr<amdgpu_synid8_attr>`::ref:`b32<amdgpu_synid8_type_dev>`,   :ref:`vsrc2<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`::ref:`f16x2<amdgpu_synid8_type_dev>`      :ref:`high<amdgpu_synid_high>` :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_interp_p2_f16                :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`::ref:`f32<amdgpu_synid8_type_dev>`, :ref:`attr<amdgpu_synid8_attr>`::ref:`b32<amdgpu_synid8_type_dev>`,   :ref:`vsrc2<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`::ref:`f32<amdgpu_synid8_type_dev>`        :ref:`high<amdgpu_synid_high>` :ref:`clamp<amdgpu_synid_clamp>`
-    v_interp_p2_f32_e64            :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`vsrc<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`attr<amdgpu_synid8_attr>`::ref:`b32<amdgpu_synid8_type_dev>`                       :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_ldexp_f16_e64                :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`i16<amdgpu_synid8_type_dev>`                       :ref:`clamp<amdgpu_synid_clamp>`
-    v_ldexp_f32                    :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`i32<amdgpu_synid8_type_dev>`                       :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_ldexp_f64                    :ref:`vdst<amdgpu_synid8_vdst64_0>`,                :ref:`src0<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`i32<amdgpu_synid8_type_dev>`                       :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_lerp_u8                      :ref:`vdst<amdgpu_synid8_vdst32_0>`::ref:`u32<amdgpu_synid8_type_dev>`,            :ref:`src0<amdgpu_synid8_src32_1>`::ref:`b32<amdgpu_synid8_type_dev>`,    :ref:`src1<amdgpu_synid8_src32_1>`::ref:`b32<amdgpu_synid8_type_dev>`,   :ref:`src2<amdgpu_synid8_src32_1>`::ref:`b32<amdgpu_synid8_type_dev>`
-    v_log_f16_e64                  :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                                       :ref:`clamp<amdgpu_synid_clamp>`
-    v_log_f32_e64                  :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                                       :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_log_legacy_f32_e64           :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                                       :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_lshlrev_b16_e64              :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`::ref:`u16<amdgpu_synid8_type_dev>`,    :ref:`src1<amdgpu_synid8_src32_1>`
-    v_lshlrev_b32_e64              :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`::ref:`u32<amdgpu_synid8_type_dev>`,    :ref:`src1<amdgpu_synid8_src32_1>`
-    v_lshlrev_b64                  :ref:`vdst<amdgpu_synid8_vdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`::ref:`u32<amdgpu_synid8_type_dev>`,    :ref:`src1<amdgpu_synid8_src64_1>`
-    v_lshrrev_b16_e64              :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`::ref:`u16<amdgpu_synid8_type_dev>`,    :ref:`src1<amdgpu_synid8_src32_1>`
-    v_lshrrev_b32_e64              :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`::ref:`u32<amdgpu_synid8_type_dev>`,    :ref:`src1<amdgpu_synid8_src32_1>`
-    v_lshrrev_b64                  :ref:`vdst<amdgpu_synid8_vdst64_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`::ref:`u32<amdgpu_synid8_type_dev>`,    :ref:`src1<amdgpu_synid8_src64_1>`
-    v_mac_f16_e64                  :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_mac_f32_e64                  :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_mad_f16                      :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,     :ref:`src2<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`             :ref:`clamp<amdgpu_synid_clamp>`
-    v_mad_f32                      :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,     :ref:`src2<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`             :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_mad_i16                      :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`,       :ref:`src2<amdgpu_synid8_src32_1>`               :ref:`clamp<amdgpu_synid_clamp>`
-    v_mad_i32_i24                  :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`,       :ref:`src2<amdgpu_synid8_src32_1>`::ref:`i32<amdgpu_synid8_type_dev>`           :ref:`clamp<amdgpu_synid_clamp>`
-    v_mad_i64_i32                  :ref:`vdst<amdgpu_synid8_vdst64_0>`,      :ref:`sdst<amdgpu_synid8_sdst64_0>`,     :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`,       :ref:`src2<amdgpu_synid8_src64_1>`::ref:`i64<amdgpu_synid8_type_dev>`           :ref:`clamp<amdgpu_synid_clamp>`
-    v_mad_legacy_f32               :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,     :ref:`src2<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`             :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_mad_u16                      :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`,       :ref:`src2<amdgpu_synid8_src32_1>`               :ref:`clamp<amdgpu_synid_clamp>`
-    v_mad_u32_u24                  :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`,       :ref:`src2<amdgpu_synid8_src32_1>`::ref:`u32<amdgpu_synid8_type_dev>`           :ref:`clamp<amdgpu_synid_clamp>`
-    v_mad_u64_u32                  :ref:`vdst<amdgpu_synid8_vdst64_0>`,      :ref:`sdst<amdgpu_synid8_sdst64_0>`,     :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`,       :ref:`src2<amdgpu_synid8_src64_1>`::ref:`u64<amdgpu_synid8_type_dev>`           :ref:`clamp<amdgpu_synid_clamp>`
-    v_max3_f32                     :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,     :ref:`src2<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`             :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_max3_i32                     :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`,       :ref:`src2<amdgpu_synid8_src32_1>`
-    v_max3_u32                     :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`,       :ref:`src2<amdgpu_synid8_src32_1>`
-    v_max_f16_e64                  :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_max_f32_e64                  :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_max_f64                      :ref:`vdst<amdgpu_synid8_vdst64_0>`,                :ref:`src0<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_max_i16_e64                  :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
-    v_max_i32_e64                  :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
-    v_max_u16_e64                  :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
-    v_max_u32_e64                  :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
-    v_mbcnt_hi_u32_b32             :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
-    v_mbcnt_lo_u32_b32             :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
-    v_med3_f32                     :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,     :ref:`src2<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`             :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_med3_i32                     :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`,       :ref:`src2<amdgpu_synid8_src32_1>`
-    v_med3_u32                     :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`,       :ref:`src2<amdgpu_synid8_src32_1>`
-    v_min3_f32                     :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,     :ref:`src2<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`             :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_min3_i32                     :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`,       :ref:`src2<amdgpu_synid8_src32_1>`
-    v_min3_u32                     :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`,       :ref:`src2<amdgpu_synid8_src32_1>`
-    v_min_f16_e64                  :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_min_f32_e64                  :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_min_f64                      :ref:`vdst<amdgpu_synid8_vdst64_0>`,                :ref:`src0<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_min_i16_e64                  :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
-    v_min_i32_e64                  :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
-    v_min_u16_e64                  :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
-    v_min_u32_e64                  :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
-    v_mov_b32_e64                  :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src<amdgpu_synid8_src32_1>`
-    v_mov_fed_b32_e64              :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src<amdgpu_synid8_src32_1>`
-    v_movreld_b32_e64              :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src<amdgpu_synid8_src32_1>`
-    v_movrels_b32_e64              :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`vsrc<amdgpu_synid8_vsrc32_0>`
-    v_movrelsd_b32_e64             :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`vsrc<amdgpu_synid8_vsrc32_0>`
-    v_mqsad_pk_u16_u8              :ref:`vdst<amdgpu_synid8_vdst64_0>`::ref:`b64<amdgpu_synid8_type_dev>`,            :ref:`src0<amdgpu_synid8_src64_1>`::ref:`b64<amdgpu_synid8_type_dev>`,    :ref:`src1<amdgpu_synid8_src32_1>`::ref:`b32<amdgpu_synid8_type_dev>`,   :ref:`src2<amdgpu_synid8_src64_1>`::ref:`b64<amdgpu_synid8_type_dev>`           :ref:`clamp<amdgpu_synid_clamp>`
-    v_mqsad_u32_u8                 :ref:`vdst<amdgpu_synid8_vdst128_0>`::ref:`b128<amdgpu_synid8_type_dev>`,           :ref:`src0<amdgpu_synid8_src64_1>`::ref:`b64<amdgpu_synid8_type_dev>`,    :ref:`src1<amdgpu_synid8_src32_1>`::ref:`b32<amdgpu_synid8_type_dev>`,   :ref:`vsrc2<amdgpu_synid8_vsrc128_0>`::ref:`b128<amdgpu_synid8_type_dev>`         :ref:`clamp<amdgpu_synid_clamp>`
-    v_msad_u8                      :ref:`vdst<amdgpu_synid8_vdst32_0>`::ref:`u32<amdgpu_synid8_type_dev>`,            :ref:`src0<amdgpu_synid8_src32_1>`::ref:`b32<amdgpu_synid8_type_dev>`,    :ref:`src1<amdgpu_synid8_src32_1>`::ref:`b32<amdgpu_synid8_type_dev>`,   :ref:`src2<amdgpu_synid8_src32_1>`::ref:`b32<amdgpu_synid8_type_dev>`           :ref:`clamp<amdgpu_synid_clamp>`
-    v_mul_f16_e64                  :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_mul_f32_e64                  :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_mul_f64                      :ref:`vdst<amdgpu_synid8_vdst64_0>`,                :ref:`src0<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_mul_hi_i32                   :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
-    v_mul_hi_i32_i24_e64           :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
-    v_mul_hi_u32                   :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
-    v_mul_hi_u32_u24_e64           :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
-    v_mul_i32_i24_e64              :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
-    v_mul_legacy_f32_e64           :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_mul_lo_u16_e64               :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
-    v_mul_lo_u32                   :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
-    v_mul_u32_u24_e64              :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
+    v_cmp_class_f16_e64     :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`b32<amdgpu_synid8_type_dev>`
+    v_cmp_class_f32_e64     :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`b32<amdgpu_synid8_type_dev>`
+    v_cmp_class_f64_e64     :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`b32<amdgpu_synid8_type_dev>`
+    v_cmp_eq_f16_e64        :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmp_eq_f32_e64        :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmp_eq_f64_e64        :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmp_eq_i16_e64        :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
+    v_cmp_eq_i32_e64        :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
+    v_cmp_eq_i64_e64        :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src64_1>`,        :ref:`src1<amdgpu_synid8_src64_1>`
+    v_cmp_eq_u16_e64        :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
+    v_cmp_eq_u32_e64        :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
+    v_cmp_eq_u64_e64        :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src64_1>`,        :ref:`src1<amdgpu_synid8_src64_1>`
+    v_cmp_f_f16_e64         :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmp_f_f32_e64         :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmp_f_f64_e64         :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmp_f_i16_e64         :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
+    v_cmp_f_i32_e64         :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
+    v_cmp_f_i64_e64         :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src64_1>`,        :ref:`src1<amdgpu_synid8_src64_1>`
+    v_cmp_f_u16_e64         :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
+    v_cmp_f_u32_e64         :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
+    v_cmp_f_u64_e64         :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src64_1>`,        :ref:`src1<amdgpu_synid8_src64_1>`
+    v_cmp_ge_f16_e64        :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmp_ge_f32_e64        :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmp_ge_f64_e64        :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmp_ge_i16_e64        :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
+    v_cmp_ge_i32_e64        :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
+    v_cmp_ge_i64_e64        :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src64_1>`,        :ref:`src1<amdgpu_synid8_src64_1>`
+    v_cmp_ge_u16_e64        :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
+    v_cmp_ge_u32_e64        :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
+    v_cmp_ge_u64_e64        :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src64_1>`,        :ref:`src1<amdgpu_synid8_src64_1>`
+    v_cmp_gt_f16_e64        :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmp_gt_f32_e64        :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmp_gt_f64_e64        :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmp_gt_i16_e64        :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
+    v_cmp_gt_i32_e64        :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
+    v_cmp_gt_i64_e64        :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src64_1>`,        :ref:`src1<amdgpu_synid8_src64_1>`
+    v_cmp_gt_u16_e64        :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
+    v_cmp_gt_u32_e64        :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
+    v_cmp_gt_u64_e64        :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src64_1>`,        :ref:`src1<amdgpu_synid8_src64_1>`
+    v_cmp_le_f16_e64        :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmp_le_f32_e64        :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmp_le_f64_e64        :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmp_le_i16_e64        :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
+    v_cmp_le_i32_e64        :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
+    v_cmp_le_i64_e64        :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src64_1>`,        :ref:`src1<amdgpu_synid8_src64_1>`
+    v_cmp_le_u16_e64        :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
+    v_cmp_le_u32_e64        :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
+    v_cmp_le_u64_e64        :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src64_1>`,        :ref:`src1<amdgpu_synid8_src64_1>`
+    v_cmp_lg_f16_e64        :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmp_lg_f32_e64        :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmp_lg_f64_e64        :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmp_lt_f16_e64        :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmp_lt_f32_e64        :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmp_lt_f64_e64        :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmp_lt_i16_e64        :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
+    v_cmp_lt_i32_e64        :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
+    v_cmp_lt_i64_e64        :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src64_1>`,        :ref:`src1<amdgpu_synid8_src64_1>`
+    v_cmp_lt_u16_e64        :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
+    v_cmp_lt_u32_e64        :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
+    v_cmp_lt_u64_e64        :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src64_1>`,        :ref:`src1<amdgpu_synid8_src64_1>`
+    v_cmp_ne_i16_e64        :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
+    v_cmp_ne_i32_e64        :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
+    v_cmp_ne_i64_e64        :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src64_1>`,        :ref:`src1<amdgpu_synid8_src64_1>`
+    v_cmp_ne_u16_e64        :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
+    v_cmp_ne_u32_e64        :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
+    v_cmp_ne_u64_e64        :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src64_1>`,        :ref:`src1<amdgpu_synid8_src64_1>`
+    v_cmp_neq_f16_e64       :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmp_neq_f32_e64       :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmp_neq_f64_e64       :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmp_nge_f16_e64       :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmp_nge_f32_e64       :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmp_nge_f64_e64       :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmp_ngt_f16_e64       :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmp_ngt_f32_e64       :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmp_ngt_f64_e64       :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmp_nle_f16_e64       :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmp_nle_f32_e64       :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmp_nle_f64_e64       :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmp_nlg_f16_e64       :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmp_nlg_f32_e64       :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmp_nlg_f64_e64       :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmp_nlt_f16_e64       :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmp_nlt_f32_e64       :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmp_nlt_f64_e64       :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmp_o_f16_e64         :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmp_o_f32_e64         :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmp_o_f64_e64         :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmp_t_i16_e64         :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
+    v_cmp_t_i32_e64         :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
+    v_cmp_t_i64_e64         :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src64_1>`,        :ref:`src1<amdgpu_synid8_src64_1>`
+    v_cmp_t_u16_e64         :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
+    v_cmp_t_u32_e64         :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
+    v_cmp_t_u64_e64         :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src64_1>`,        :ref:`src1<amdgpu_synid8_src64_1>`
+    v_cmp_tru_f16_e64       :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmp_tru_f32_e64       :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmp_tru_f64_e64       :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmp_u_f16_e64         :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmp_u_f32_e64         :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmp_u_f64_e64         :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmpx_class_f16_e64    :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`b32<amdgpu_synid8_type_dev>`
+    v_cmpx_class_f32_e64    :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`b32<amdgpu_synid8_type_dev>`
+    v_cmpx_class_f64_e64    :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`b32<amdgpu_synid8_type_dev>`
+    v_cmpx_eq_f16_e64       :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmpx_eq_f32_e64       :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmpx_eq_f64_e64       :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmpx_eq_i16_e64       :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
+    v_cmpx_eq_i32_e64       :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
+    v_cmpx_eq_i64_e64       :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src64_1>`,        :ref:`src1<amdgpu_synid8_src64_1>`
+    v_cmpx_eq_u16_e64       :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
+    v_cmpx_eq_u32_e64       :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
+    v_cmpx_eq_u64_e64       :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src64_1>`,        :ref:`src1<amdgpu_synid8_src64_1>`
+    v_cmpx_f_f16_e64        :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmpx_f_f32_e64        :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmpx_f_f64_e64        :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmpx_f_i16_e64        :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
+    v_cmpx_f_i32_e64        :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
+    v_cmpx_f_i64_e64        :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src64_1>`,        :ref:`src1<amdgpu_synid8_src64_1>`
+    v_cmpx_f_u16_e64        :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
+    v_cmpx_f_u32_e64        :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
+    v_cmpx_f_u64_e64        :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src64_1>`,        :ref:`src1<amdgpu_synid8_src64_1>`
+    v_cmpx_ge_f16_e64       :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmpx_ge_f32_e64       :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmpx_ge_f64_e64       :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmpx_ge_i16_e64       :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
+    v_cmpx_ge_i32_e64       :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
+    v_cmpx_ge_i64_e64       :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src64_1>`,        :ref:`src1<amdgpu_synid8_src64_1>`
+    v_cmpx_ge_u16_e64       :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
+    v_cmpx_ge_u32_e64       :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
+    v_cmpx_ge_u64_e64       :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src64_1>`,        :ref:`src1<amdgpu_synid8_src64_1>`
+    v_cmpx_gt_f16_e64       :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmpx_gt_f32_e64       :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmpx_gt_f64_e64       :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmpx_gt_i16_e64       :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
+    v_cmpx_gt_i32_e64       :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
+    v_cmpx_gt_i64_e64       :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src64_1>`,        :ref:`src1<amdgpu_synid8_src64_1>`
+    v_cmpx_gt_u16_e64       :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
+    v_cmpx_gt_u32_e64       :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
+    v_cmpx_gt_u64_e64       :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src64_1>`,        :ref:`src1<amdgpu_synid8_src64_1>`
+    v_cmpx_le_f16_e64       :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmpx_le_f32_e64       :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmpx_le_f64_e64       :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmpx_le_i16_e64       :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
+    v_cmpx_le_i32_e64       :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
+    v_cmpx_le_i64_e64       :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src64_1>`,        :ref:`src1<amdgpu_synid8_src64_1>`
+    v_cmpx_le_u16_e64       :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
+    v_cmpx_le_u32_e64       :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
+    v_cmpx_le_u64_e64       :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src64_1>`,        :ref:`src1<amdgpu_synid8_src64_1>`
+    v_cmpx_lg_f16_e64       :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmpx_lg_f32_e64       :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmpx_lg_f64_e64       :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmpx_lt_f16_e64       :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmpx_lt_f32_e64       :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmpx_lt_f64_e64       :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmpx_lt_i16_e64       :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
+    v_cmpx_lt_i32_e64       :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
+    v_cmpx_lt_i64_e64       :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src64_1>`,        :ref:`src1<amdgpu_synid8_src64_1>`
+    v_cmpx_lt_u16_e64       :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
+    v_cmpx_lt_u32_e64       :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
+    v_cmpx_lt_u64_e64       :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src64_1>`,        :ref:`src1<amdgpu_synid8_src64_1>`
+    v_cmpx_ne_i16_e64       :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
+    v_cmpx_ne_i32_e64       :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
+    v_cmpx_ne_i64_e64       :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src64_1>`,        :ref:`src1<amdgpu_synid8_src64_1>`
+    v_cmpx_ne_u16_e64       :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
+    v_cmpx_ne_u32_e64       :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
+    v_cmpx_ne_u64_e64       :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src64_1>`,        :ref:`src1<amdgpu_synid8_src64_1>`
+    v_cmpx_neq_f16_e64      :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmpx_neq_f32_e64      :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmpx_neq_f64_e64      :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmpx_nge_f16_e64      :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmpx_nge_f32_e64      :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmpx_nge_f64_e64      :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmpx_ngt_f16_e64      :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmpx_ngt_f32_e64      :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmpx_ngt_f64_e64      :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmpx_nle_f16_e64      :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmpx_nle_f32_e64      :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmpx_nle_f64_e64      :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmpx_nlg_f16_e64      :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmpx_nlg_f32_e64      :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmpx_nlg_f64_e64      :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmpx_nlt_f16_e64      :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmpx_nlt_f32_e64      :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmpx_nlt_f64_e64      :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmpx_o_f16_e64        :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmpx_o_f32_e64        :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmpx_o_f64_e64        :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmpx_t_i16_e64        :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
+    v_cmpx_t_i32_e64        :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
+    v_cmpx_t_i64_e64        :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src64_1>`,        :ref:`src1<amdgpu_synid8_src64_1>`
+    v_cmpx_t_u16_e64        :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
+    v_cmpx_t_u32_e64        :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
+    v_cmpx_t_u64_e64        :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src64_1>`,        :ref:`src1<amdgpu_synid8_src64_1>`
+    v_cmpx_tru_f16_e64      :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmpx_tru_f32_e64      :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmpx_tru_f64_e64      :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmpx_u_f16_e64        :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmpx_u_f32_e64        :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmpx_u_f64_e64        :ref:`sdst<amdgpu_synid8_sdst64_0>`,               :ref:`src0<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cndmask_b32_e64       :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`,       :ref:`ssrc2<amdgpu_synid8_ssrc64_1>`
+    v_cos_f16_e64           :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                                    :ref:`clamp<amdgpu_synid_clamp>`
+    v_cos_f32_e64           :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                                    :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_cubeid_f32            :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,     :ref:`src2<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`          :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_cubema_f32            :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,     :ref:`src2<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`          :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_cubesc_f32            :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,     :ref:`src2<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`          :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_cubetc_f32            :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,     :ref:`src2<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`          :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_cvt_f16_f32_e64       :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                                    :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_cvt_f16_i16_e64       :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src<amdgpu_synid8_src32_1>`                                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cvt_f16_u16_e64       :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src<amdgpu_synid8_src32_1>`                                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cvt_f32_f16_e64       :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                                    :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_cvt_f32_f64_e64       :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                                    :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_cvt_f32_i32_e64       :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src<amdgpu_synid8_src32_1>`                                      :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_cvt_f32_u32_e64       :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src<amdgpu_synid8_src32_1>`                                      :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_cvt_f32_ubyte0_e64    :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src<amdgpu_synid8_src32_1>`                                      :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_cvt_f32_ubyte1_e64    :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src<amdgpu_synid8_src32_1>`                                      :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_cvt_f32_ubyte2_e64    :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src<amdgpu_synid8_src32_1>`                                      :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_cvt_f32_ubyte3_e64    :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src<amdgpu_synid8_src32_1>`                                      :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_cvt_f64_f32_e64       :ref:`vdst<amdgpu_synid8_vdst64_0>`,               :ref:`src<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                                    :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_cvt_f64_i32_e64       :ref:`vdst<amdgpu_synid8_vdst64_0>`,               :ref:`src<amdgpu_synid8_src32_1>`                                      :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_cvt_f64_u32_e64       :ref:`vdst<amdgpu_synid8_vdst64_0>`,               :ref:`src<amdgpu_synid8_src32_1>`                                      :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_cvt_flr_i32_f32_e64   :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`
+    v_cvt_i16_f16_e64       :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`
+    v_cvt_i32_f32_e64       :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`
+    v_cvt_i32_f64_e64       :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`
+    v_cvt_off_f32_i4_e64    :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src<amdgpu_synid8_src32_1>`                                      :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_cvt_pk_i16_i32        :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
+    v_cvt_pk_u16_u32        :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
+    v_cvt_pk_u8_f32         :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`u32<amdgpu_synid8_type_dev>`,   :ref:`src2<amdgpu_synid8_src32_1>`::ref:`u32<amdgpu_synid8_type_dev>`
+    v_cvt_pkaccum_u8_f32    :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`u32<amdgpu_synid8_type_dev>`
+    v_cvt_pknorm_i16_f32    :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`
+    v_cvt_pknorm_u16_f32    :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`
+    v_cvt_pkrtz_f16_f32     :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`
+    v_cvt_rpi_i32_f32_e64   :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`
+    v_cvt_u16_f16_e64       :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`
+    v_cvt_u32_f32_e64       :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`
+    v_cvt_u32_f64_e64       :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`
+    v_div_fixup_f16         :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,     :ref:`src2<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`          :ref:`clamp<amdgpu_synid_clamp>`
+    v_div_fixup_f32         :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,     :ref:`src2<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`          :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_div_fixup_f64         :ref:`vdst<amdgpu_synid8_vdst64_0>`,               :ref:`src0<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,     :ref:`src2<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`          :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_div_fmas_f32          :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,     :ref:`src2<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`          :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_div_fmas_f64          :ref:`vdst<amdgpu_synid8_vdst64_0>`,               :ref:`src0<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,     :ref:`src2<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`          :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_div_scale_f32         :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`vcc<amdgpu_synid8_vcc_64>`,     :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`,       :ref:`src2<amdgpu_synid8_src32_1>`
+    v_div_scale_f64         :ref:`vdst<amdgpu_synid8_vdst64_0>`,      :ref:`vcc<amdgpu_synid8_vcc_64>`,     :ref:`src0<amdgpu_synid8_src64_1>`,        :ref:`src1<amdgpu_synid8_src64_1>`,       :ref:`src2<amdgpu_synid8_src64_1>`
+    v_exp_f16_e64           :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                                    :ref:`clamp<amdgpu_synid_clamp>`
+    v_exp_f32_e64           :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                                    :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_exp_legacy_f32_e64    :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                                    :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_ffbh_i32_e64          :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src<amdgpu_synid8_src32_1>`
+    v_ffbh_u32_e64          :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src<amdgpu_synid8_src32_1>`
+    v_ffbl_b32_e64          :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src<amdgpu_synid8_src32_1>`
+    v_floor_f16_e64         :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                                    :ref:`clamp<amdgpu_synid_clamp>`
+    v_floor_f32_e64         :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                                    :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_floor_f64_e64         :ref:`vdst<amdgpu_synid8_vdst64_0>`,               :ref:`src<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                                    :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_fma_f16               :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,     :ref:`src2<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`          :ref:`clamp<amdgpu_synid_clamp>`
+    v_fma_f32               :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,     :ref:`src2<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`          :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_fma_f64               :ref:`vdst<amdgpu_synid8_vdst64_0>`,               :ref:`src0<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,     :ref:`src2<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`          :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_fract_f16_e64         :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                                    :ref:`clamp<amdgpu_synid_clamp>`
+    v_fract_f32_e64         :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                                    :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_fract_f64_e64         :ref:`vdst<amdgpu_synid8_vdst64_0>`,               :ref:`src<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                                    :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_frexp_exp_i16_f16_e64 :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`
+    v_frexp_exp_i32_f32_e64 :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`
+    v_frexp_exp_i32_f64_e64 :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`
+    v_frexp_mant_f16_e64    :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                                    :ref:`clamp<amdgpu_synid_clamp>`
+    v_frexp_mant_f32_e64    :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                                    :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_frexp_mant_f64_e64    :ref:`vdst<amdgpu_synid8_vdst64_0>`,               :ref:`src<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                                    :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_interp_mov_f32_e64    :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`param<amdgpu_synid8_param>`::ref:`b32<amdgpu_synid8_type_dev>`,   :ref:`attr<amdgpu_synid8_attr>`::ref:`b32<amdgpu_synid8_type_dev>`                    :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_interp_p1_f32_e64     :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`vsrc<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`attr<amdgpu_synid8_attr>`::ref:`b32<amdgpu_synid8_type_dev>`                    :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_interp_p1ll_f16       :ref:`vdst<amdgpu_synid8_vdst32_0>`::ref:`f32<amdgpu_synid8_type_dev>`,           :ref:`vsrc<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`::ref:`f32<amdgpu_synid8_type_dev>`,  :ref:`attr<amdgpu_synid8_attr>`::ref:`b32<amdgpu_synid8_type_dev>`                    :ref:`high<amdgpu_synid_high>` :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_interp_p1lv_f16       :ref:`vdst<amdgpu_synid8_vdst32_0>`::ref:`f32<amdgpu_synid8_type_dev>`,           :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`::ref:`f32<amdgpu_synid8_type_dev>`, :ref:`attr<amdgpu_synid8_attr>`::ref:`b32<amdgpu_synid8_type_dev>`,   :ref:`vsrc2<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`::ref:`f16x2<amdgpu_synid8_type_dev>`   :ref:`high<amdgpu_synid_high>` :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_interp_p2_f16         :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`::ref:`f32<amdgpu_synid8_type_dev>`, :ref:`attr<amdgpu_synid8_attr>`::ref:`b32<amdgpu_synid8_type_dev>`,   :ref:`vsrc2<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`::ref:`f32<amdgpu_synid8_type_dev>`     :ref:`high<amdgpu_synid_high>` :ref:`clamp<amdgpu_synid_clamp>`
+    v_interp_p2_f32_e64     :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`vsrc<amdgpu_synid8_vsrc32_0>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`attr<amdgpu_synid8_attr>`::ref:`b32<amdgpu_synid8_type_dev>`                    :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_ldexp_f16_e64         :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`i16<amdgpu_synid8_type_dev>`                    :ref:`clamp<amdgpu_synid_clamp>`
+    v_ldexp_f32             :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`i32<amdgpu_synid8_type_dev>`                    :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_ldexp_f64             :ref:`vdst<amdgpu_synid8_vdst64_0>`,               :ref:`src0<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`i32<amdgpu_synid8_type_dev>`                    :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_lerp_u8               :ref:`vdst<amdgpu_synid8_vdst32_0>`::ref:`u32<amdgpu_synid8_type_dev>`,           :ref:`src0<amdgpu_synid8_src32_1>`::ref:`b32<amdgpu_synid8_type_dev>`,    :ref:`src1<amdgpu_synid8_src32_1>`::ref:`b32<amdgpu_synid8_type_dev>`,   :ref:`src2<amdgpu_synid8_src32_1>`::ref:`b32<amdgpu_synid8_type_dev>`
+    v_log_f16_e64           :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                                    :ref:`clamp<amdgpu_synid_clamp>`
+    v_log_f32_e64           :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                                    :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_log_legacy_f32_e64    :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                                    :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_lshlrev_b16_e64       :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`::ref:`u16<amdgpu_synid8_type_dev>`,    :ref:`src1<amdgpu_synid8_src32_1>`
+    v_lshlrev_b32_e64       :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`::ref:`u32<amdgpu_synid8_type_dev>`,    :ref:`src1<amdgpu_synid8_src32_1>`
+    v_lshlrev_b64           :ref:`vdst<amdgpu_synid8_vdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`::ref:`u32<amdgpu_synid8_type_dev>`,    :ref:`src1<amdgpu_synid8_src64_1>`
+    v_lshrrev_b16_e64       :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`::ref:`u16<amdgpu_synid8_type_dev>`,    :ref:`src1<amdgpu_synid8_src32_1>`
+    v_lshrrev_b32_e64       :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`::ref:`u32<amdgpu_synid8_type_dev>`,    :ref:`src1<amdgpu_synid8_src32_1>`
+    v_lshrrev_b64           :ref:`vdst<amdgpu_synid8_vdst64_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`::ref:`u32<amdgpu_synid8_type_dev>`,    :ref:`src1<amdgpu_synid8_src64_1>`
+    v_mac_f16_e64           :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_mac_f32_e64           :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_mad_f16               :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,     :ref:`src2<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`          :ref:`clamp<amdgpu_synid_clamp>`
+    v_mad_f32               :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,     :ref:`src2<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`          :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_mad_i16               :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`,       :ref:`src2<amdgpu_synid8_src32_1>`            :ref:`clamp<amdgpu_synid_clamp>`
+    v_mad_i32_i24           :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`,       :ref:`src2<amdgpu_synid8_src32_1>`::ref:`i32<amdgpu_synid8_type_dev>`        :ref:`clamp<amdgpu_synid_clamp>`
+    v_mad_i64_i32           :ref:`vdst<amdgpu_synid8_vdst64_0>`,      :ref:`sdst<amdgpu_synid8_sdst64_0>`,    :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`,       :ref:`src2<amdgpu_synid8_src64_1>`::ref:`i64<amdgpu_synid8_type_dev>`        :ref:`clamp<amdgpu_synid_clamp>`
+    v_mad_legacy_f32        :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,     :ref:`src2<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`          :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_mad_u16               :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`,       :ref:`src2<amdgpu_synid8_src32_1>`            :ref:`clamp<amdgpu_synid_clamp>`
+    v_mad_u32_u24           :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`,       :ref:`src2<amdgpu_synid8_src32_1>`::ref:`u32<amdgpu_synid8_type_dev>`        :ref:`clamp<amdgpu_synid_clamp>`
+    v_mad_u64_u32           :ref:`vdst<amdgpu_synid8_vdst64_0>`,      :ref:`sdst<amdgpu_synid8_sdst64_0>`,    :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`,       :ref:`src2<amdgpu_synid8_src64_1>`::ref:`u64<amdgpu_synid8_type_dev>`        :ref:`clamp<amdgpu_synid_clamp>`
+    v_max3_f32              :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,     :ref:`src2<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`          :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_max3_i32              :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`,       :ref:`src2<amdgpu_synid8_src32_1>`
+    v_max3_u32              :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`,       :ref:`src2<amdgpu_synid8_src32_1>`
+    v_max_f16_e64           :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_max_f32_e64           :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_max_f64               :ref:`vdst<amdgpu_synid8_vdst64_0>`,               :ref:`src0<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_max_i16_e64           :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
+    v_max_i32_e64           :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
+    v_max_u16_e64           :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
+    v_max_u32_e64           :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
+    v_mbcnt_hi_u32_b32      :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
+    v_mbcnt_lo_u32_b32      :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
+    v_med3_f32              :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,     :ref:`src2<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`          :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_med3_i32              :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`,       :ref:`src2<amdgpu_synid8_src32_1>`
+    v_med3_u32              :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`,       :ref:`src2<amdgpu_synid8_src32_1>`
+    v_min3_f32              :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,     :ref:`src2<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`          :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_min3_i32              :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`,       :ref:`src2<amdgpu_synid8_src32_1>`
+    v_min3_u32              :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`,       :ref:`src2<amdgpu_synid8_src32_1>`
+    v_min_f16_e64           :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_min_f32_e64           :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_min_f64               :ref:`vdst<amdgpu_synid8_vdst64_0>`,               :ref:`src0<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_min_i16_e64           :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
+    v_min_i32_e64           :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
+    v_min_u16_e64           :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
+    v_min_u32_e64           :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
+    v_mov_b32_e64           :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src<amdgpu_synid8_src32_1>`
+    v_mov_fed_b32_e64       :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src<amdgpu_synid8_src32_1>`
+    v_movreld_b32_e64       :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src<amdgpu_synid8_src32_1>`
+    v_movrels_b32_e64       :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`vsrc<amdgpu_synid8_vsrc32_0>`
+    v_movrelsd_b32_e64      :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`vsrc<amdgpu_synid8_vsrc32_0>`
+    v_mqsad_pk_u16_u8       :ref:`vdst<amdgpu_synid8_vdst64_0>`::ref:`b64<amdgpu_synid8_type_dev>`,           :ref:`src0<amdgpu_synid8_src64_1>`::ref:`b64<amdgpu_synid8_type_dev>`,    :ref:`src1<amdgpu_synid8_src32_1>`::ref:`b32<amdgpu_synid8_type_dev>`,   :ref:`src2<amdgpu_synid8_src64_1>`::ref:`b64<amdgpu_synid8_type_dev>`        :ref:`clamp<amdgpu_synid_clamp>`
+    v_mqsad_u32_u8          :ref:`vdst<amdgpu_synid8_vdst128_0>`::ref:`b128<amdgpu_synid8_type_dev>`,          :ref:`src0<amdgpu_synid8_src64_1>`::ref:`b64<amdgpu_synid8_type_dev>`,    :ref:`src1<amdgpu_synid8_src32_1>`::ref:`b32<amdgpu_synid8_type_dev>`,   :ref:`vsrc2<amdgpu_synid8_vsrc128_0>`::ref:`b128<amdgpu_synid8_type_dev>`      :ref:`clamp<amdgpu_synid_clamp>`
+    v_msad_u8               :ref:`vdst<amdgpu_synid8_vdst32_0>`::ref:`u32<amdgpu_synid8_type_dev>`,           :ref:`src0<amdgpu_synid8_src32_1>`::ref:`b32<amdgpu_synid8_type_dev>`,    :ref:`src1<amdgpu_synid8_src32_1>`::ref:`b32<amdgpu_synid8_type_dev>`,   :ref:`src2<amdgpu_synid8_src32_1>`::ref:`b32<amdgpu_synid8_type_dev>`        :ref:`clamp<amdgpu_synid_clamp>`
+    v_mul_f16_e64           :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_mul_f32_e64           :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_mul_f64               :ref:`vdst<amdgpu_synid8_vdst64_0>`,               :ref:`src0<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_mul_hi_i32            :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
+    v_mul_hi_i32_i24_e64    :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
+    v_mul_hi_u32            :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
+    v_mul_hi_u32_u24_e64    :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
+    v_mul_i32_i24_e64       :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
+    v_mul_legacy_f32_e64    :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_mul_lo_u16_e64        :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
+    v_mul_lo_u32            :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
+    v_mul_u32_u24_e64       :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
     v_nop_e64
-    v_not_b32_e64                  :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src<amdgpu_synid8_src32_1>`
-    v_or_b32_e64                   :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
-    v_perm_b32                     :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`,       :ref:`src2<amdgpu_synid8_src32_1>`
-    v_qsad_pk_u16_u8               :ref:`vdst<amdgpu_synid8_vdst64_0>`::ref:`b64<amdgpu_synid8_type_dev>`,            :ref:`src0<amdgpu_synid8_src64_1>`::ref:`b64<amdgpu_synid8_type_dev>`,    :ref:`src1<amdgpu_synid8_src32_1>`::ref:`b32<amdgpu_synid8_type_dev>`,   :ref:`src2<amdgpu_synid8_src64_1>`::ref:`b64<amdgpu_synid8_type_dev>`           :ref:`clamp<amdgpu_synid_clamp>`
-    v_rcp_f16_e64                  :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                                       :ref:`clamp<amdgpu_synid_clamp>`
-    v_rcp_f32_e64                  :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                                       :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_rcp_f64_e64                  :ref:`vdst<amdgpu_synid8_vdst64_0>`,                :ref:`src<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                                       :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_rcp_iflag_f32_e64            :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                                       :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_readlane_b32                 :ref:`sdst<amdgpu_synid8_sdst32_2>`,                :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`,       :ref:`ssrc1<amdgpu_synid8_ssrc32_3>`
-    v_rndne_f16_e64                :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                                       :ref:`clamp<amdgpu_synid_clamp>`
-    v_rndne_f32_e64                :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                                       :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_rndne_f64_e64                :ref:`vdst<amdgpu_synid8_vdst64_0>`,                :ref:`src<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                                       :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_rsq_f16_e64                  :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                                       :ref:`clamp<amdgpu_synid_clamp>`
-    v_rsq_f32_e64                  :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                                       :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_rsq_f64_e64                  :ref:`vdst<amdgpu_synid8_vdst64_0>`,                :ref:`src<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                                       :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_sad_hi_u8                    :ref:`vdst<amdgpu_synid8_vdst32_0>`::ref:`u32<amdgpu_synid8_type_dev>`,            :ref:`src0<amdgpu_synid8_src32_1>`::ref:`u8x4<amdgpu_synid8_type_dev>`,   :ref:`src1<amdgpu_synid8_src32_1>`::ref:`u8x4<amdgpu_synid8_type_dev>`,  :ref:`src2<amdgpu_synid8_src32_1>`::ref:`u32<amdgpu_synid8_type_dev>`           :ref:`clamp<amdgpu_synid_clamp>`
-    v_sad_u16                      :ref:`vdst<amdgpu_synid8_vdst32_0>`::ref:`u32<amdgpu_synid8_type_dev>`,            :ref:`src0<amdgpu_synid8_src32_1>`::ref:`u16x2<amdgpu_synid8_type_dev>`,  :ref:`src1<amdgpu_synid8_src32_1>`::ref:`u16x2<amdgpu_synid8_type_dev>`, :ref:`src2<amdgpu_synid8_src32_1>`::ref:`u32<amdgpu_synid8_type_dev>`           :ref:`clamp<amdgpu_synid_clamp>`
-    v_sad_u32                      :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`,       :ref:`src2<amdgpu_synid8_src32_1>`               :ref:`clamp<amdgpu_synid_clamp>`
-    v_sad_u8                       :ref:`vdst<amdgpu_synid8_vdst32_0>`::ref:`u32<amdgpu_synid8_type_dev>`,            :ref:`src0<amdgpu_synid8_src32_1>`::ref:`u8x4<amdgpu_synid8_type_dev>`,   :ref:`src1<amdgpu_synid8_src32_1>`::ref:`u8x4<amdgpu_synid8_type_dev>`,  :ref:`src2<amdgpu_synid8_src32_1>`::ref:`u32<amdgpu_synid8_type_dev>`           :ref:`clamp<amdgpu_synid_clamp>`
-    v_sin_f16_e64                  :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                                       :ref:`clamp<amdgpu_synid_clamp>`
-    v_sin_f32_e64                  :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                                       :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_sqrt_f16_e64                 :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                                       :ref:`clamp<amdgpu_synid_clamp>`
-    v_sqrt_f32_e64                 :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                                       :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_sqrt_f64_e64                 :ref:`vdst<amdgpu_synid8_vdst64_0>`,                :ref:`src<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                                       :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_sub_f16_e64                  :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_sub_f32_e64                  :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_sub_u16_e64                  :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
-    v_sub_u32_e64                  :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`sdst<amdgpu_synid8_sdst64_0>`,     :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
-    v_subb_u32_e64                 :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`sdst<amdgpu_synid8_sdst64_0>`,     :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`,       :ref:`ssrc2<amdgpu_synid8_ssrc64_1>`
-    v_subbrev_u32_e64              :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`sdst<amdgpu_synid8_sdst64_0>`,     :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`,       :ref:`ssrc2<amdgpu_synid8_ssrc64_1>`
-    v_subrev_f16_e64               :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_subrev_f32_e64               :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_subrev_u16_e64               :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
-    v_subrev_u32_e64               :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`sdst<amdgpu_synid8_sdst64_0>`,     :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
-    v_trig_preop_f64               :ref:`vdst<amdgpu_synid8_vdst64_0>`,                :ref:`src0<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`u32<amdgpu_synid8_type_dev>`                       :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_trunc_f16_e64                :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                                       :ref:`clamp<amdgpu_synid_clamp>`
-    v_trunc_f32_e64                :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                                       :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_trunc_f64_e64                :ref:`vdst<amdgpu_synid8_vdst64_0>`,                :ref:`src<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                                       :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_writelane_b32                :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`ssrc0<amdgpu_synid8_ssrc32_4>`,       :ref:`ssrc1<amdgpu_synid8_ssrc32_3>`
-    v_xor_b32_e64                  :ref:`vdst<amdgpu_synid8_vdst32_0>`,                :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
+    v_not_b32_e64           :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src<amdgpu_synid8_src32_1>`
+    v_or_b32_e64            :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
+    v_perm_b32              :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`,       :ref:`src2<amdgpu_synid8_src32_1>`
+    v_qsad_pk_u16_u8        :ref:`vdst<amdgpu_synid8_vdst64_0>`::ref:`b64<amdgpu_synid8_type_dev>`,           :ref:`src0<amdgpu_synid8_src64_1>`::ref:`b64<amdgpu_synid8_type_dev>`,    :ref:`src1<amdgpu_synid8_src32_1>`::ref:`b32<amdgpu_synid8_type_dev>`,   :ref:`src2<amdgpu_synid8_src64_1>`::ref:`b64<amdgpu_synid8_type_dev>`        :ref:`clamp<amdgpu_synid_clamp>`
+    v_rcp_f16_e64           :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                                    :ref:`clamp<amdgpu_synid_clamp>`
+    v_rcp_f32_e64           :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                                    :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_rcp_f64_e64           :ref:`vdst<amdgpu_synid8_vdst64_0>`,               :ref:`src<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                                    :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_rcp_iflag_f32_e64     :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                                    :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_readlane_b32          :ref:`sdst<amdgpu_synid8_sdst32_2>`,               :ref:`vsrc0<amdgpu_synid8_vsrc32_0>`,       :ref:`ssrc1<amdgpu_synid8_ssrc32_3>`
+    v_rndne_f16_e64         :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                                    :ref:`clamp<amdgpu_synid_clamp>`
+    v_rndne_f32_e64         :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                                    :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_rndne_f64_e64         :ref:`vdst<amdgpu_synid8_vdst64_0>`,               :ref:`src<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                                    :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_rsq_f16_e64           :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                                    :ref:`clamp<amdgpu_synid_clamp>`
+    v_rsq_f32_e64           :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                                    :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_rsq_f64_e64           :ref:`vdst<amdgpu_synid8_vdst64_0>`,               :ref:`src<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                                    :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_sad_hi_u8             :ref:`vdst<amdgpu_synid8_vdst32_0>`::ref:`u32<amdgpu_synid8_type_dev>`,           :ref:`src0<amdgpu_synid8_src32_1>`::ref:`u8x4<amdgpu_synid8_type_dev>`,   :ref:`src1<amdgpu_synid8_src32_1>`::ref:`u8x4<amdgpu_synid8_type_dev>`,  :ref:`src2<amdgpu_synid8_src32_1>`::ref:`u32<amdgpu_synid8_type_dev>`        :ref:`clamp<amdgpu_synid_clamp>`
+    v_sad_u16               :ref:`vdst<amdgpu_synid8_vdst32_0>`::ref:`u32<amdgpu_synid8_type_dev>`,           :ref:`src0<amdgpu_synid8_src32_1>`::ref:`u16x2<amdgpu_synid8_type_dev>`,  :ref:`src1<amdgpu_synid8_src32_1>`::ref:`u16x2<amdgpu_synid8_type_dev>`, :ref:`src2<amdgpu_synid8_src32_1>`::ref:`u32<amdgpu_synid8_type_dev>`        :ref:`clamp<amdgpu_synid_clamp>`
+    v_sad_u32               :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`,       :ref:`src2<amdgpu_synid8_src32_1>`            :ref:`clamp<amdgpu_synid_clamp>`
+    v_sad_u8                :ref:`vdst<amdgpu_synid8_vdst32_0>`::ref:`u32<amdgpu_synid8_type_dev>`,           :ref:`src0<amdgpu_synid8_src32_1>`::ref:`u8x4<amdgpu_synid8_type_dev>`,   :ref:`src1<amdgpu_synid8_src32_1>`::ref:`u8x4<amdgpu_synid8_type_dev>`,  :ref:`src2<amdgpu_synid8_src32_1>`::ref:`u32<amdgpu_synid8_type_dev>`        :ref:`clamp<amdgpu_synid_clamp>`
+    v_sin_f16_e64           :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                                    :ref:`clamp<amdgpu_synid_clamp>`
+    v_sin_f32_e64           :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                                    :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_sqrt_f16_e64          :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                                    :ref:`clamp<amdgpu_synid_clamp>`
+    v_sqrt_f32_e64          :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                                    :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_sqrt_f64_e64          :ref:`vdst<amdgpu_synid8_vdst64_0>`,               :ref:`src<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                                    :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_sub_f16_e64           :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_sub_f32_e64           :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_sub_u16_e64           :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
+    v_sub_u32_e64           :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`sdst<amdgpu_synid8_sdst64_0>`,    :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
+    v_subb_u32_e64          :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`sdst<amdgpu_synid8_sdst64_0>`,    :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`,       :ref:`ssrc2<amdgpu_synid8_ssrc64_1>`
+    v_subbrev_u32_e64       :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`sdst<amdgpu_synid8_sdst64_0>`,    :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`,       :ref:`ssrc2<amdgpu_synid8_ssrc64_1>`
+    v_subrev_f16_e64        :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_subrev_f32_e64        :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_subrev_u16_e64        :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
+    v_subrev_u32_e64        :ref:`vdst<amdgpu_synid8_vdst32_0>`,      :ref:`sdst<amdgpu_synid8_sdst64_0>`,    :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
+    v_trig_preop_f64        :ref:`vdst<amdgpu_synid8_vdst64_0>`,               :ref:`src0<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid8_src32_1>`::ref:`u32<amdgpu_synid8_type_dev>`                    :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_trunc_f16_e64         :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                                    :ref:`clamp<amdgpu_synid_clamp>`
+    v_trunc_f32_e64         :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src<amdgpu_synid8_src32_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                                    :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_trunc_f64_e64         :ref:`vdst<amdgpu_synid8_vdst64_0>`,               :ref:`src<amdgpu_synid8_src64_1>`::ref:`m<amdgpu_synid8_mod_vop3_abs_neg>`                                    :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_writelane_b32         :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`ssrc0<amdgpu_synid8_ssrc32_4>`,       :ref:`ssrc1<amdgpu_synid8_ssrc32_3>`
+    v_xor_b32_e64           :ref:`vdst<amdgpu_synid8_vdst32_0>`,               :ref:`src0<amdgpu_synid8_src32_1>`,        :ref:`src1<amdgpu_synid8_src32_1>`
 
 VOPC
 -----------------------
diff --git a/docs/AMDGPU/AMDGPUAsmGFX9.rst b/docs/AMDGPU/AMDGPUAsmGFX9.rst
index 71052d0cf6e4ca1d96874f84ecc652165945e743..9dd3b9de260220f5485f94b17853006459c09c80 100644
--- a/docs/AMDGPU/AMDGPUAsmGFX9.rst
+++ b/docs/AMDGPU/AMDGPUAsmGFX9.rst
@@ -32,159 +32,159 @@ DS
 .. parsed-literal::
 
     **INSTRUCTION**                    **DST**         **SRC0**      **SRC1**      **SRC2**           **MODIFIERS**
-    \ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|
-    ds_add_f32                                 :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_add_rtn_f32                 :ref:`vdst<amdgpu_synid9_vdst32_0>`,       :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_add_rtn_u32                 :ref:`vdst<amdgpu_synid9_vdst32_0>`,       :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_add_rtn_u64                 :ref:`vdst<amdgpu_synid9_vdst64_0>`,       :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata64_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_add_src2_f32                            :ref:`vaddr<amdgpu_synid9_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_add_src2_u32                            :ref:`vaddr<amdgpu_synid9_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_add_src2_u64                            :ref:`vaddr<amdgpu_synid9_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_add_u32                                 :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_add_u64                                 :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata64_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_and_b32                                 :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_and_b64                                 :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata64_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_and_rtn_b32                 :ref:`vdst<amdgpu_synid9_vdst32_0>`,       :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_and_rtn_b64                 :ref:`vdst<amdgpu_synid9_vdst64_0>`,       :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata64_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_and_src2_b32                            :ref:`vaddr<amdgpu_synid9_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_and_src2_b64                            :ref:`vaddr<amdgpu_synid9_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_append                      :ref:`vdst<amdgpu_synid9_vdst32_0>`                                           :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_bpermute_b32                :ref:`vdst<amdgpu_synid9_vdst32_0>`,       :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>`
-    ds_cmpst_b32                               :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata0<amdgpu_synid9_vdata32_0>`,   :ref:`vdata1<amdgpu_synid9_vdata32_0>`         :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_cmpst_b64                               :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata0<amdgpu_synid9_vdata64_0>`,   :ref:`vdata1<amdgpu_synid9_vdata64_0>`         :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_cmpst_f32                               :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata0<amdgpu_synid9_vdata32_0>`,   :ref:`vdata1<amdgpu_synid9_vdata32_0>`         :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_cmpst_f64                               :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata0<amdgpu_synid9_vdata64_0>`,   :ref:`vdata1<amdgpu_synid9_vdata64_0>`         :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_cmpst_rtn_b32               :ref:`vdst<amdgpu_synid9_vdst32_0>`,       :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata0<amdgpu_synid9_vdata32_0>`,   :ref:`vdata1<amdgpu_synid9_vdata32_0>`         :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_cmpst_rtn_b64               :ref:`vdst<amdgpu_synid9_vdst64_0>`,       :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata0<amdgpu_synid9_vdata64_0>`,   :ref:`vdata1<amdgpu_synid9_vdata64_0>`         :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_cmpst_rtn_f32               :ref:`vdst<amdgpu_synid9_vdst32_0>`,       :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata0<amdgpu_synid9_vdata32_0>`,   :ref:`vdata1<amdgpu_synid9_vdata32_0>`         :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_cmpst_rtn_f64               :ref:`vdst<amdgpu_synid9_vdst64_0>`,       :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata0<amdgpu_synid9_vdata64_0>`,   :ref:`vdata1<amdgpu_synid9_vdata64_0>`         :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_condxchg32_rtn_b64          :ref:`vdst<amdgpu_synid9_vdst64_0>`,       :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata64_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_consume                     :ref:`vdst<amdgpu_synid9_vdst32_0>`                                           :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_dec_rtn_u32                 :ref:`vdst<amdgpu_synid9_vdst32_0>`,       :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_dec_rtn_u64                 :ref:`vdst<amdgpu_synid9_vdst64_0>`,       :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata64_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_dec_src2_u32                            :ref:`vaddr<amdgpu_synid9_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_dec_src2_u64                            :ref:`vaddr<amdgpu_synid9_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_dec_u32                                 :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_dec_u64                                 :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata64_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_gws_barrier                             :ref:`vdata<amdgpu_synid9_vdata32_0>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_gws_init                                :ref:`vdata<amdgpu_synid9_vdata32_0>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_gws_sema_br                             :ref:`vdata<amdgpu_synid9_vdata32_0>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_gws_sema_p                                                                 :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_gws_sema_release_all                                                       :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_gws_sema_v                                                                 :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_inc_rtn_u32                 :ref:`vdst<amdgpu_synid9_vdst32_0>`,       :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_inc_rtn_u64                 :ref:`vdst<amdgpu_synid9_vdst64_0>`,       :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata64_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_inc_src2_u32                            :ref:`vaddr<amdgpu_synid9_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_inc_src2_u64                            :ref:`vaddr<amdgpu_synid9_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_inc_u32                                 :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_inc_u64                                 :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata64_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_max_f32                                 :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_max_f64                                 :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata64_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_max_i32                                 :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_max_i64                                 :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata64_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_max_rtn_f32                 :ref:`vdst<amdgpu_synid9_vdst32_0>`,       :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_max_rtn_f64                 :ref:`vdst<amdgpu_synid9_vdst64_0>`,       :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata64_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_max_rtn_i32                 :ref:`vdst<amdgpu_synid9_vdst32_0>`,       :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_max_rtn_i64                 :ref:`vdst<amdgpu_synid9_vdst64_0>`,       :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata64_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_max_rtn_u32                 :ref:`vdst<amdgpu_synid9_vdst32_0>`,       :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_max_rtn_u64                 :ref:`vdst<amdgpu_synid9_vdst64_0>`,       :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata64_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_max_src2_f32                            :ref:`vaddr<amdgpu_synid9_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_max_src2_f64                            :ref:`vaddr<amdgpu_synid9_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_max_src2_i32                            :ref:`vaddr<amdgpu_synid9_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_max_src2_i64                            :ref:`vaddr<amdgpu_synid9_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_max_src2_u32                            :ref:`vaddr<amdgpu_synid9_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_max_src2_u64                            :ref:`vaddr<amdgpu_synid9_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_max_u32                                 :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_max_u64                                 :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata64_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_min_f32                                 :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_min_f64                                 :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata64_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_min_i32                                 :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_min_i64                                 :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata64_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_min_rtn_f32                 :ref:`vdst<amdgpu_synid9_vdst32_0>`,       :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_min_rtn_f64                 :ref:`vdst<amdgpu_synid9_vdst64_0>`,       :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata64_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_min_rtn_i32                 :ref:`vdst<amdgpu_synid9_vdst32_0>`,       :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_min_rtn_i64                 :ref:`vdst<amdgpu_synid9_vdst64_0>`,       :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata64_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_min_rtn_u32                 :ref:`vdst<amdgpu_synid9_vdst32_0>`,       :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_min_rtn_u64                 :ref:`vdst<amdgpu_synid9_vdst64_0>`,       :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata64_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_min_src2_f32                            :ref:`vaddr<amdgpu_synid9_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_min_src2_f64                            :ref:`vaddr<amdgpu_synid9_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_min_src2_i32                            :ref:`vaddr<amdgpu_synid9_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_min_src2_i64                            :ref:`vaddr<amdgpu_synid9_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_min_src2_u32                            :ref:`vaddr<amdgpu_synid9_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_min_src2_u64                            :ref:`vaddr<amdgpu_synid9_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_min_u32                                 :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_min_u64                                 :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata64_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_mskor_b32                               :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata0<amdgpu_synid9_vdata32_0>`,   :ref:`vdata1<amdgpu_synid9_vdata32_0>`         :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_mskor_b64                               :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata0<amdgpu_synid9_vdata64_0>`,   :ref:`vdata1<amdgpu_synid9_vdata64_0>`         :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_mskor_rtn_b32               :ref:`vdst<amdgpu_synid9_vdst32_0>`,       :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata0<amdgpu_synid9_vdata32_0>`,   :ref:`vdata1<amdgpu_synid9_vdata32_0>`         :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_mskor_rtn_b64               :ref:`vdst<amdgpu_synid9_vdst64_0>`,       :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata0<amdgpu_synid9_vdata64_0>`,   :ref:`vdata1<amdgpu_synid9_vdata64_0>`         :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    \ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|
+    ds_add_f32                                 :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_add_rtn_f32                 :ref:`vdst<amdgpu_synid9_vdst32_0>`,       :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_add_rtn_u32                 :ref:`vdst<amdgpu_synid9_vdst32_0>`,       :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_add_rtn_u64                 :ref:`vdst<amdgpu_synid9_vdst64_0>`,       :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata64_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_add_src2_f32                            :ref:`vaddr<amdgpu_synid9_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_add_src2_u32                            :ref:`vaddr<amdgpu_synid9_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_add_src2_u64                            :ref:`vaddr<amdgpu_synid9_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_add_u32                                 :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_add_u64                                 :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata64_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_and_b32                                 :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_and_b64                                 :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata64_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_and_rtn_b32                 :ref:`vdst<amdgpu_synid9_vdst32_0>`,       :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_and_rtn_b64                 :ref:`vdst<amdgpu_synid9_vdst64_0>`,       :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata64_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_and_src2_b32                            :ref:`vaddr<amdgpu_synid9_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_and_src2_b64                            :ref:`vaddr<amdgpu_synid9_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_append                      :ref:`vdst<amdgpu_synid9_vdst32_0>`                                           :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_bpermute_b32                :ref:`vdst<amdgpu_synid9_vdst32_0>`,       :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>`
+    ds_cmpst_b32                               :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata0<amdgpu_synid9_vdata32_0>`,   :ref:`vdata1<amdgpu_synid9_vdata32_0>`         :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_cmpst_b64                               :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata0<amdgpu_synid9_vdata64_0>`,   :ref:`vdata1<amdgpu_synid9_vdata64_0>`         :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_cmpst_f32                               :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata0<amdgpu_synid9_vdata32_0>`,   :ref:`vdata1<amdgpu_synid9_vdata32_0>`         :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_cmpst_f64                               :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata0<amdgpu_synid9_vdata64_0>`,   :ref:`vdata1<amdgpu_synid9_vdata64_0>`         :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_cmpst_rtn_b32               :ref:`vdst<amdgpu_synid9_vdst32_0>`,       :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata0<amdgpu_synid9_vdata32_0>`,   :ref:`vdata1<amdgpu_synid9_vdata32_0>`         :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_cmpst_rtn_b64               :ref:`vdst<amdgpu_synid9_vdst64_0>`,       :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata0<amdgpu_synid9_vdata64_0>`,   :ref:`vdata1<amdgpu_synid9_vdata64_0>`         :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_cmpst_rtn_f32               :ref:`vdst<amdgpu_synid9_vdst32_0>`,       :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata0<amdgpu_synid9_vdata32_0>`,   :ref:`vdata1<amdgpu_synid9_vdata32_0>`         :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_cmpst_rtn_f64               :ref:`vdst<amdgpu_synid9_vdst64_0>`,       :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata0<amdgpu_synid9_vdata64_0>`,   :ref:`vdata1<amdgpu_synid9_vdata64_0>`         :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_condxchg32_rtn_b64          :ref:`vdst<amdgpu_synid9_vdst64_0>`,       :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata64_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_consume                     :ref:`vdst<amdgpu_synid9_vdst32_0>`                                           :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_dec_rtn_u32                 :ref:`vdst<amdgpu_synid9_vdst32_0>`,       :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_dec_rtn_u64                 :ref:`vdst<amdgpu_synid9_vdst64_0>`,       :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata64_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_dec_src2_u32                            :ref:`vaddr<amdgpu_synid9_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_dec_src2_u64                            :ref:`vaddr<amdgpu_synid9_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_dec_u32                                 :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_dec_u64                                 :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata64_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_gws_barrier                             :ref:`vdata<amdgpu_synid9_vdata32_0>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_gws_init                                :ref:`vdata<amdgpu_synid9_vdata32_0>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_gws_sema_br                             :ref:`vdata<amdgpu_synid9_vdata32_0>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_gws_sema_p                                                                 :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_gws_sema_release_all                                                       :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_gws_sema_v                                                                 :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_inc_rtn_u32                 :ref:`vdst<amdgpu_synid9_vdst32_0>`,       :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_inc_rtn_u64                 :ref:`vdst<amdgpu_synid9_vdst64_0>`,       :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata64_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_inc_src2_u32                            :ref:`vaddr<amdgpu_synid9_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_inc_src2_u64                            :ref:`vaddr<amdgpu_synid9_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_inc_u32                                 :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_inc_u64                                 :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata64_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_max_f32                                 :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_max_f64                                 :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata64_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_max_i32                                 :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_max_i64                                 :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata64_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_max_rtn_f32                 :ref:`vdst<amdgpu_synid9_vdst32_0>`,       :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_max_rtn_f64                 :ref:`vdst<amdgpu_synid9_vdst64_0>`,       :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata64_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_max_rtn_i32                 :ref:`vdst<amdgpu_synid9_vdst32_0>`,       :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_max_rtn_i64                 :ref:`vdst<amdgpu_synid9_vdst64_0>`,       :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata64_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_max_rtn_u32                 :ref:`vdst<amdgpu_synid9_vdst32_0>`,       :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_max_rtn_u64                 :ref:`vdst<amdgpu_synid9_vdst64_0>`,       :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata64_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_max_src2_f32                            :ref:`vaddr<amdgpu_synid9_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_max_src2_f64                            :ref:`vaddr<amdgpu_synid9_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_max_src2_i32                            :ref:`vaddr<amdgpu_synid9_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_max_src2_i64                            :ref:`vaddr<amdgpu_synid9_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_max_src2_u32                            :ref:`vaddr<amdgpu_synid9_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_max_src2_u64                            :ref:`vaddr<amdgpu_synid9_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_max_u32                                 :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_max_u64                                 :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata64_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_min_f32                                 :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_min_f64                                 :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata64_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_min_i32                                 :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_min_i64                                 :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata64_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_min_rtn_f32                 :ref:`vdst<amdgpu_synid9_vdst32_0>`,       :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_min_rtn_f64                 :ref:`vdst<amdgpu_synid9_vdst64_0>`,       :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata64_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_min_rtn_i32                 :ref:`vdst<amdgpu_synid9_vdst32_0>`,       :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_min_rtn_i64                 :ref:`vdst<amdgpu_synid9_vdst64_0>`,       :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata64_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_min_rtn_u32                 :ref:`vdst<amdgpu_synid9_vdst32_0>`,       :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_min_rtn_u64                 :ref:`vdst<amdgpu_synid9_vdst64_0>`,       :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata64_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_min_src2_f32                            :ref:`vaddr<amdgpu_synid9_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_min_src2_f64                            :ref:`vaddr<amdgpu_synid9_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_min_src2_i32                            :ref:`vaddr<amdgpu_synid9_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_min_src2_i64                            :ref:`vaddr<amdgpu_synid9_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_min_src2_u32                            :ref:`vaddr<amdgpu_synid9_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_min_src2_u64                            :ref:`vaddr<amdgpu_synid9_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_min_u32                                 :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_min_u64                                 :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata64_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_mskor_b32                               :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata0<amdgpu_synid9_vdata32_0>`,   :ref:`vdata1<amdgpu_synid9_vdata32_0>`         :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_mskor_b64                               :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata0<amdgpu_synid9_vdata64_0>`,   :ref:`vdata1<amdgpu_synid9_vdata64_0>`         :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_mskor_rtn_b32               :ref:`vdst<amdgpu_synid9_vdst32_0>`,       :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata0<amdgpu_synid9_vdata32_0>`,   :ref:`vdata1<amdgpu_synid9_vdata32_0>`         :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_mskor_rtn_b64               :ref:`vdst<amdgpu_synid9_vdst64_0>`,       :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata0<amdgpu_synid9_vdata64_0>`,   :ref:`vdata1<amdgpu_synid9_vdata64_0>`         :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
     ds_nop
-    ds_or_b32                                  :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_or_b64                                  :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata64_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_or_rtn_b32                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,       :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_or_rtn_b64                  :ref:`vdst<amdgpu_synid9_vdst64_0>`,       :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata64_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_or_src2_b32                             :ref:`vaddr<amdgpu_synid9_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_or_src2_b64                             :ref:`vaddr<amdgpu_synid9_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_ordered_count               :ref:`vdst<amdgpu_synid9_vdst32_0>`,       :ref:`vaddr<amdgpu_synid9_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_permute_b32                 :ref:`vdst<amdgpu_synid9_vdst32_0>`,       :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>`
-    ds_read2_b32                   :ref:`vdst<amdgpu_synid9_vdst64_0>`::ref:`b32x2<amdgpu_synid9_type_dev>`, :ref:`vaddr<amdgpu_synid9_addr_ds>`                              :ref:`ds_offset8<amdgpu_synid_ds_offset8>` :ref:`ds_offset8<amdgpu_synid_ds_offset8>` :ref:`gds<amdgpu_synid_gds>`
-    ds_read2_b64                   :ref:`vdst<amdgpu_synid9_vdst128_0>`::ref:`b64x2<amdgpu_synid9_type_dev>`, :ref:`vaddr<amdgpu_synid9_addr_ds>`                              :ref:`ds_offset8<amdgpu_synid_ds_offset8>` :ref:`ds_offset8<amdgpu_synid_ds_offset8>` :ref:`gds<amdgpu_synid_gds>`
-    ds_read2st64_b32               :ref:`vdst<amdgpu_synid9_vdst64_0>`::ref:`b32x2<amdgpu_synid9_type_dev>`, :ref:`vaddr<amdgpu_synid9_addr_ds>`                              :ref:`ds_offset8<amdgpu_synid_ds_offset8>` :ref:`ds_offset8<amdgpu_synid_ds_offset8>` :ref:`gds<amdgpu_synid_gds>`
-    ds_read2st64_b64               :ref:`vdst<amdgpu_synid9_vdst128_0>`::ref:`b64x2<amdgpu_synid9_type_dev>`, :ref:`vaddr<amdgpu_synid9_addr_ds>`                              :ref:`ds_offset8<amdgpu_synid_ds_offset8>` :ref:`ds_offset8<amdgpu_synid_ds_offset8>` :ref:`gds<amdgpu_synid_gds>`
-    ds_read_b128                   :ref:`vdst<amdgpu_synid9_vdst128_0>`,       :ref:`vaddr<amdgpu_synid9_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_read_b32                    :ref:`vdst<amdgpu_synid9_vdst32_0>`,       :ref:`vaddr<amdgpu_synid9_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_read_b64                    :ref:`vdst<amdgpu_synid9_vdst64_0>`,       :ref:`vaddr<amdgpu_synid9_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_read_b96                    :ref:`vdst<amdgpu_synid9_vdst96_0>`,       :ref:`vaddr<amdgpu_synid9_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_read_i16                    :ref:`vdst<amdgpu_synid9_vdst32_0>`,       :ref:`vaddr<amdgpu_synid9_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_read_i8                     :ref:`vdst<amdgpu_synid9_vdst32_0>`,       :ref:`vaddr<amdgpu_synid9_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_read_i8_d16                 :ref:`vdst<amdgpu_synid9_vdst32_0>`,       :ref:`vaddr<amdgpu_synid9_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_read_i8_d16_hi              :ref:`vdst<amdgpu_synid9_vdst32_0>`,       :ref:`vaddr<amdgpu_synid9_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_read_u16                    :ref:`vdst<amdgpu_synid9_vdst32_0>`,       :ref:`vaddr<amdgpu_synid9_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_read_u16_d16                :ref:`vdst<amdgpu_synid9_vdst32_0>`,       :ref:`vaddr<amdgpu_synid9_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_read_u16_d16_hi             :ref:`vdst<amdgpu_synid9_vdst32_0>`,       :ref:`vaddr<amdgpu_synid9_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_read_u8                     :ref:`vdst<amdgpu_synid9_vdst32_0>`,       :ref:`vaddr<amdgpu_synid9_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_read_u8_d16                 :ref:`vdst<amdgpu_synid9_vdst32_0>`,       :ref:`vaddr<amdgpu_synid9_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_read_u8_d16_hi              :ref:`vdst<amdgpu_synid9_vdst32_0>`,       :ref:`vaddr<amdgpu_synid9_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_rsub_rtn_u32                :ref:`vdst<amdgpu_synid9_vdst32_0>`,       :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_rsub_rtn_u64                :ref:`vdst<amdgpu_synid9_vdst64_0>`,       :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata64_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_rsub_src2_u32                           :ref:`vaddr<amdgpu_synid9_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_rsub_src2_u64                           :ref:`vaddr<amdgpu_synid9_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_rsub_u32                                :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_rsub_u64                                :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata64_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_sub_rtn_u32                 :ref:`vdst<amdgpu_synid9_vdst32_0>`,       :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_sub_rtn_u64                 :ref:`vdst<amdgpu_synid9_vdst64_0>`,       :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata64_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_sub_src2_u32                            :ref:`vaddr<amdgpu_synid9_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_sub_src2_u64                            :ref:`vaddr<amdgpu_synid9_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_sub_u32                                 :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_sub_u64                                 :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata64_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_swizzle_b32                 :ref:`vdst<amdgpu_synid9_vdst32_0>`,       :ref:`vaddr<amdgpu_synid9_addr_ds>`                              :ref:`sw_offset16<amdgpu_synid_sw_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_wrap_rtn_b32                :ref:`vdst<amdgpu_synid9_vdst32_0>`,       :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata0<amdgpu_synid9_vdata32_0>`,   :ref:`vdata1<amdgpu_synid9_vdata32_0>`         :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_write2_b32                              :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata0<amdgpu_synid9_vdata32_0>`,   :ref:`vdata1<amdgpu_synid9_vdata32_0>`         :ref:`ds_offset8<amdgpu_synid_ds_offset8>` :ref:`ds_offset8<amdgpu_synid_ds_offset8>` :ref:`gds<amdgpu_synid_gds>`
-    ds_write2_b64                              :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata0<amdgpu_synid9_vdata64_0>`,   :ref:`vdata1<amdgpu_synid9_vdata64_0>`         :ref:`ds_offset8<amdgpu_synid_ds_offset8>` :ref:`ds_offset8<amdgpu_synid_ds_offset8>` :ref:`gds<amdgpu_synid_gds>`
-    ds_write2st64_b32                          :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata0<amdgpu_synid9_vdata32_0>`,   :ref:`vdata1<amdgpu_synid9_vdata32_0>`         :ref:`ds_offset8<amdgpu_synid_ds_offset8>` :ref:`ds_offset8<amdgpu_synid_ds_offset8>` :ref:`gds<amdgpu_synid_gds>`
-    ds_write2st64_b64                          :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata0<amdgpu_synid9_vdata64_0>`,   :ref:`vdata1<amdgpu_synid9_vdata64_0>`         :ref:`ds_offset8<amdgpu_synid_ds_offset8>` :ref:`ds_offset8<amdgpu_synid_ds_offset8>` :ref:`gds<amdgpu_synid_gds>`
-    ds_write_b128                              :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata128_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_write_b16                               :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_write_b16_d16_hi                        :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_write_b32                               :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_write_b64                               :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata64_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_write_b8                                :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_write_b8_d16_hi                         :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_write_b96                               :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata96_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_write_src2_b32                          :ref:`vaddr<amdgpu_synid9_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_write_src2_b64                          :ref:`vaddr<amdgpu_synid9_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_wrxchg2_rtn_b32             :ref:`vdst<amdgpu_synid9_vdst64_0>`::ref:`b32x2<amdgpu_synid9_type_dev>`, :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata0<amdgpu_synid9_vdata32_0>`,   :ref:`vdata1<amdgpu_synid9_vdata32_0>`         :ref:`ds_offset8<amdgpu_synid_ds_offset8>` :ref:`ds_offset8<amdgpu_synid_ds_offset8>` :ref:`gds<amdgpu_synid_gds>`
-    ds_wrxchg2_rtn_b64             :ref:`vdst<amdgpu_synid9_vdst128_0>`::ref:`b64x2<amdgpu_synid9_type_dev>`, :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata0<amdgpu_synid9_vdata64_0>`,   :ref:`vdata1<amdgpu_synid9_vdata64_0>`         :ref:`ds_offset8<amdgpu_synid_ds_offset8>` :ref:`ds_offset8<amdgpu_synid_ds_offset8>` :ref:`gds<amdgpu_synid_gds>`
-    ds_wrxchg2st64_rtn_b32         :ref:`vdst<amdgpu_synid9_vdst64_0>`::ref:`b32x2<amdgpu_synid9_type_dev>`, :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata0<amdgpu_synid9_vdata32_0>`,   :ref:`vdata1<amdgpu_synid9_vdata32_0>`         :ref:`ds_offset8<amdgpu_synid_ds_offset8>` :ref:`ds_offset8<amdgpu_synid_ds_offset8>` :ref:`gds<amdgpu_synid_gds>`
-    ds_wrxchg2st64_rtn_b64         :ref:`vdst<amdgpu_synid9_vdst128_0>`::ref:`b64x2<amdgpu_synid9_type_dev>`, :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata0<amdgpu_synid9_vdata64_0>`,   :ref:`vdata1<amdgpu_synid9_vdata64_0>`         :ref:`ds_offset8<amdgpu_synid_ds_offset8>` :ref:`ds_offset8<amdgpu_synid_ds_offset8>` :ref:`gds<amdgpu_synid_gds>`
-    ds_wrxchg_rtn_b32              :ref:`vdst<amdgpu_synid9_vdst32_0>`,       :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_wrxchg_rtn_b64              :ref:`vdst<amdgpu_synid9_vdst64_0>`,       :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata64_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_xor_b32                                 :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_xor_b64                                 :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata64_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_xor_rtn_b32                 :ref:`vdst<amdgpu_synid9_vdst32_0>`,       :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_xor_rtn_b64                 :ref:`vdst<amdgpu_synid9_vdst64_0>`,       :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata64_0>`                    :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_xor_src2_b32                            :ref:`vaddr<amdgpu_synid9_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
-    ds_xor_src2_b64                            :ref:`vaddr<amdgpu_synid9_addr_ds>`                              :ref:`ds_offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_or_b32                                  :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_or_b64                                  :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata64_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_or_rtn_b32                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,       :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_or_rtn_b64                  :ref:`vdst<amdgpu_synid9_vdst64_0>`,       :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata64_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_or_src2_b32                             :ref:`vaddr<amdgpu_synid9_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_or_src2_b64                             :ref:`vaddr<amdgpu_synid9_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_ordered_count               :ref:`vdst<amdgpu_synid9_vdst32_0>`,       :ref:`vaddr<amdgpu_synid9_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_permute_b32                 :ref:`vdst<amdgpu_synid9_vdst32_0>`,       :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>`
+    ds_read2_b32                   :ref:`vdst<amdgpu_synid9_vdst64_0>`::ref:`b32x2<amdgpu_synid9_type_dev>`, :ref:`vaddr<amdgpu_synid9_addr_ds>`                              :ref:`offset8<amdgpu_synid_ds_offset8>` :ref:`offset8<amdgpu_synid_ds_offset8>` :ref:`gds<amdgpu_synid_gds>`
+    ds_read2_b64                   :ref:`vdst<amdgpu_synid9_vdst128_0>`::ref:`b64x2<amdgpu_synid9_type_dev>`, :ref:`vaddr<amdgpu_synid9_addr_ds>`                              :ref:`offset8<amdgpu_synid_ds_offset8>` :ref:`offset8<amdgpu_synid_ds_offset8>` :ref:`gds<amdgpu_synid_gds>`
+    ds_read2st64_b32               :ref:`vdst<amdgpu_synid9_vdst64_0>`::ref:`b32x2<amdgpu_synid9_type_dev>`, :ref:`vaddr<amdgpu_synid9_addr_ds>`                              :ref:`offset8<amdgpu_synid_ds_offset8>` :ref:`offset8<amdgpu_synid_ds_offset8>` :ref:`gds<amdgpu_synid_gds>`
+    ds_read2st64_b64               :ref:`vdst<amdgpu_synid9_vdst128_0>`::ref:`b64x2<amdgpu_synid9_type_dev>`, :ref:`vaddr<amdgpu_synid9_addr_ds>`                              :ref:`offset8<amdgpu_synid_ds_offset8>` :ref:`offset8<amdgpu_synid_ds_offset8>` :ref:`gds<amdgpu_synid_gds>`
+    ds_read_b128                   :ref:`vdst<amdgpu_synid9_vdst128_0>`,       :ref:`vaddr<amdgpu_synid9_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_read_b32                    :ref:`vdst<amdgpu_synid9_vdst32_0>`,       :ref:`vaddr<amdgpu_synid9_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_read_b64                    :ref:`vdst<amdgpu_synid9_vdst64_0>`,       :ref:`vaddr<amdgpu_synid9_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_read_b96                    :ref:`vdst<amdgpu_synid9_vdst96_0>`,       :ref:`vaddr<amdgpu_synid9_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_read_i16                    :ref:`vdst<amdgpu_synid9_vdst32_0>`,       :ref:`vaddr<amdgpu_synid9_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_read_i8                     :ref:`vdst<amdgpu_synid9_vdst32_0>`,       :ref:`vaddr<amdgpu_synid9_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_read_i8_d16                 :ref:`vdst<amdgpu_synid9_vdst32_0>`,       :ref:`vaddr<amdgpu_synid9_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_read_i8_d16_hi              :ref:`vdst<amdgpu_synid9_vdst32_0>`,       :ref:`vaddr<amdgpu_synid9_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_read_u16                    :ref:`vdst<amdgpu_synid9_vdst32_0>`,       :ref:`vaddr<amdgpu_synid9_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_read_u16_d16                :ref:`vdst<amdgpu_synid9_vdst32_0>`,       :ref:`vaddr<amdgpu_synid9_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_read_u16_d16_hi             :ref:`vdst<amdgpu_synid9_vdst32_0>`,       :ref:`vaddr<amdgpu_synid9_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_read_u8                     :ref:`vdst<amdgpu_synid9_vdst32_0>`,       :ref:`vaddr<amdgpu_synid9_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_read_u8_d16                 :ref:`vdst<amdgpu_synid9_vdst32_0>`,       :ref:`vaddr<amdgpu_synid9_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_read_u8_d16_hi              :ref:`vdst<amdgpu_synid9_vdst32_0>`,       :ref:`vaddr<amdgpu_synid9_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_rsub_rtn_u32                :ref:`vdst<amdgpu_synid9_vdst32_0>`,       :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_rsub_rtn_u64                :ref:`vdst<amdgpu_synid9_vdst64_0>`,       :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata64_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_rsub_src2_u32                           :ref:`vaddr<amdgpu_synid9_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_rsub_src2_u64                           :ref:`vaddr<amdgpu_synid9_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_rsub_u32                                :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_rsub_u64                                :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata64_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_sub_rtn_u32                 :ref:`vdst<amdgpu_synid9_vdst32_0>`,       :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_sub_rtn_u64                 :ref:`vdst<amdgpu_synid9_vdst64_0>`,       :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata64_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_sub_src2_u32                            :ref:`vaddr<amdgpu_synid9_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_sub_src2_u64                            :ref:`vaddr<amdgpu_synid9_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_sub_u32                                 :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_sub_u64                                 :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata64_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_swizzle_b32                 :ref:`vdst<amdgpu_synid9_vdst32_0>`,       :ref:`vaddr<amdgpu_synid9_addr_ds>`                              :ref:`pattern<amdgpu_synid_sw_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_wrap_rtn_b32                :ref:`vdst<amdgpu_synid9_vdst32_0>`,       :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata0<amdgpu_synid9_vdata32_0>`,   :ref:`vdata1<amdgpu_synid9_vdata32_0>`         :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_write2_b32                              :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata0<amdgpu_synid9_vdata32_0>`,   :ref:`vdata1<amdgpu_synid9_vdata32_0>`         :ref:`offset8<amdgpu_synid_ds_offset8>` :ref:`offset8<amdgpu_synid_ds_offset8>` :ref:`gds<amdgpu_synid_gds>`
+    ds_write2_b64                              :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata0<amdgpu_synid9_vdata64_0>`,   :ref:`vdata1<amdgpu_synid9_vdata64_0>`         :ref:`offset8<amdgpu_synid_ds_offset8>` :ref:`offset8<amdgpu_synid_ds_offset8>` :ref:`gds<amdgpu_synid_gds>`
+    ds_write2st64_b32                          :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata0<amdgpu_synid9_vdata32_0>`,   :ref:`vdata1<amdgpu_synid9_vdata32_0>`         :ref:`offset8<amdgpu_synid_ds_offset8>` :ref:`offset8<amdgpu_synid_ds_offset8>` :ref:`gds<amdgpu_synid_gds>`
+    ds_write2st64_b64                          :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata0<amdgpu_synid9_vdata64_0>`,   :ref:`vdata1<amdgpu_synid9_vdata64_0>`         :ref:`offset8<amdgpu_synid_ds_offset8>` :ref:`offset8<amdgpu_synid_ds_offset8>` :ref:`gds<amdgpu_synid_gds>`
+    ds_write_b128                              :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata128_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_write_b16                               :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_write_b16_d16_hi                        :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_write_b32                               :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_write_b64                               :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata64_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_write_b8                                :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_write_b8_d16_hi                         :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_write_b96                               :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata96_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_write_src2_b32                          :ref:`vaddr<amdgpu_synid9_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_write_src2_b64                          :ref:`vaddr<amdgpu_synid9_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_wrxchg2_rtn_b32             :ref:`vdst<amdgpu_synid9_vdst64_0>`::ref:`b32x2<amdgpu_synid9_type_dev>`, :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata0<amdgpu_synid9_vdata32_0>`,   :ref:`vdata1<amdgpu_synid9_vdata32_0>`         :ref:`offset8<amdgpu_synid_ds_offset8>` :ref:`offset8<amdgpu_synid_ds_offset8>` :ref:`gds<amdgpu_synid_gds>`
+    ds_wrxchg2_rtn_b64             :ref:`vdst<amdgpu_synid9_vdst128_0>`::ref:`b64x2<amdgpu_synid9_type_dev>`, :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata0<amdgpu_synid9_vdata64_0>`,   :ref:`vdata1<amdgpu_synid9_vdata64_0>`         :ref:`offset8<amdgpu_synid_ds_offset8>` :ref:`offset8<amdgpu_synid_ds_offset8>` :ref:`gds<amdgpu_synid_gds>`
+    ds_wrxchg2st64_rtn_b32         :ref:`vdst<amdgpu_synid9_vdst64_0>`::ref:`b32x2<amdgpu_synid9_type_dev>`, :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata0<amdgpu_synid9_vdata32_0>`,   :ref:`vdata1<amdgpu_synid9_vdata32_0>`         :ref:`offset8<amdgpu_synid_ds_offset8>` :ref:`offset8<amdgpu_synid_ds_offset8>` :ref:`gds<amdgpu_synid_gds>`
+    ds_wrxchg2st64_rtn_b64         :ref:`vdst<amdgpu_synid9_vdst128_0>`::ref:`b64x2<amdgpu_synid9_type_dev>`, :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata0<amdgpu_synid9_vdata64_0>`,   :ref:`vdata1<amdgpu_synid9_vdata64_0>`         :ref:`offset8<amdgpu_synid_ds_offset8>` :ref:`offset8<amdgpu_synid_ds_offset8>` :ref:`gds<amdgpu_synid_gds>`
+    ds_wrxchg_rtn_b32              :ref:`vdst<amdgpu_synid9_vdst32_0>`,       :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_wrxchg_rtn_b64              :ref:`vdst<amdgpu_synid9_vdst64_0>`,       :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata64_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_xor_b32                                 :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_xor_b64                                 :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata64_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_xor_rtn_b32                 :ref:`vdst<amdgpu_synid9_vdst32_0>`,       :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_xor_rtn_b64                 :ref:`vdst<amdgpu_synid9_vdst64_0>`,       :ref:`vaddr<amdgpu_synid9_addr_ds>`,    :ref:`vdata<amdgpu_synid9_vdata64_0>`                    :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_xor_src2_b32                            :ref:`vaddr<amdgpu_synid9_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
+    ds_xor_src2_b64                            :ref:`vaddr<amdgpu_synid9_addr_ds>`                              :ref:`offset16<amdgpu_synid_ds_offset16>` :ref:`gds<amdgpu_synid_gds>`
 
 EXP
 -----------------------
@@ -201,297 +201,297 @@ FLAT
 .. parsed-literal::
 
     **INSTRUCTION**                    **DST**           **SRC0**      **SRC1**         **SRC2**           **MODIFIERS**
-    \ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|
-    flat_atomic_add                :ref:`vdst<amdgpu_synid9_dst_flat_atomic32>`::ref:`opt<amdgpu_synid9_opt>`,     :ref:`vaddr<amdgpu_synid9_addr_flat>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`                       :ref:`flat_offset12<amdgpu_synid_flat_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    flat_atomic_add_x2             :ref:`vdst<amdgpu_synid9_dst_flat_atomic64>`::ref:`opt<amdgpu_synid9_opt>`,     :ref:`vaddr<amdgpu_synid9_addr_flat>`,    :ref:`vdata<amdgpu_synid9_vdata64_0>`                       :ref:`flat_offset12<amdgpu_synid_flat_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    flat_atomic_and                :ref:`vdst<amdgpu_synid9_dst_flat_atomic32>`::ref:`opt<amdgpu_synid9_opt>`,     :ref:`vaddr<amdgpu_synid9_addr_flat>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`                       :ref:`flat_offset12<amdgpu_synid_flat_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    flat_atomic_and_x2             :ref:`vdst<amdgpu_synid9_dst_flat_atomic64>`::ref:`opt<amdgpu_synid9_opt>`,     :ref:`vaddr<amdgpu_synid9_addr_flat>`,    :ref:`vdata<amdgpu_synid9_vdata64_0>`                       :ref:`flat_offset12<amdgpu_synid_flat_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    flat_atomic_cmpswap            :ref:`vdst<amdgpu_synid9_dst_flat_atomic32>`::ref:`opt<amdgpu_synid9_opt>`,     :ref:`vaddr<amdgpu_synid9_addr_flat>`,    :ref:`vdata<amdgpu_synid9_vdata64_0>`::ref:`b32x2<amdgpu_synid9_type_dev>`                 :ref:`flat_offset12<amdgpu_synid_flat_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    flat_atomic_cmpswap_x2         :ref:`vdst<amdgpu_synid9_dst_flat_atomic64>`::ref:`opt<amdgpu_synid9_opt>`,     :ref:`vaddr<amdgpu_synid9_addr_flat>`,    :ref:`vdata<amdgpu_synid9_vdata128_0>`::ref:`b64x2<amdgpu_synid9_type_dev>`                 :ref:`flat_offset12<amdgpu_synid_flat_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    flat_atomic_dec                :ref:`vdst<amdgpu_synid9_dst_flat_atomic32>`::ref:`opt<amdgpu_synid9_opt>`::ref:`u32<amdgpu_synid9_type_dev>`, :ref:`vaddr<amdgpu_synid9_addr_flat>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`::ref:`u32<amdgpu_synid9_type_dev>`                   :ref:`flat_offset12<amdgpu_synid_flat_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    flat_atomic_dec_x2             :ref:`vdst<amdgpu_synid9_dst_flat_atomic64>`::ref:`opt<amdgpu_synid9_opt>`::ref:`u64<amdgpu_synid9_type_dev>`, :ref:`vaddr<amdgpu_synid9_addr_flat>`,    :ref:`vdata<amdgpu_synid9_vdata64_0>`::ref:`u64<amdgpu_synid9_type_dev>`                   :ref:`flat_offset12<amdgpu_synid_flat_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    flat_atomic_inc                :ref:`vdst<amdgpu_synid9_dst_flat_atomic32>`::ref:`opt<amdgpu_synid9_opt>`::ref:`u32<amdgpu_synid9_type_dev>`, :ref:`vaddr<amdgpu_synid9_addr_flat>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`::ref:`u32<amdgpu_synid9_type_dev>`                   :ref:`flat_offset12<amdgpu_synid_flat_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    flat_atomic_inc_x2             :ref:`vdst<amdgpu_synid9_dst_flat_atomic64>`::ref:`opt<amdgpu_synid9_opt>`::ref:`u64<amdgpu_synid9_type_dev>`, :ref:`vaddr<amdgpu_synid9_addr_flat>`,    :ref:`vdata<amdgpu_synid9_vdata64_0>`::ref:`u64<amdgpu_synid9_type_dev>`                   :ref:`flat_offset12<amdgpu_synid_flat_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    flat_atomic_or                 :ref:`vdst<amdgpu_synid9_dst_flat_atomic32>`::ref:`opt<amdgpu_synid9_opt>`,     :ref:`vaddr<amdgpu_synid9_addr_flat>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`                       :ref:`flat_offset12<amdgpu_synid_flat_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    flat_atomic_or_x2              :ref:`vdst<amdgpu_synid9_dst_flat_atomic64>`::ref:`opt<amdgpu_synid9_opt>`,     :ref:`vaddr<amdgpu_synid9_addr_flat>`,    :ref:`vdata<amdgpu_synid9_vdata64_0>`                       :ref:`flat_offset12<amdgpu_synid_flat_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    flat_atomic_smax               :ref:`vdst<amdgpu_synid9_dst_flat_atomic32>`::ref:`opt<amdgpu_synid9_opt>`::ref:`s32<amdgpu_synid9_type_dev>`, :ref:`vaddr<amdgpu_synid9_addr_flat>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`::ref:`s32<amdgpu_synid9_type_dev>`                   :ref:`flat_offset12<amdgpu_synid_flat_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    flat_atomic_smax_x2            :ref:`vdst<amdgpu_synid9_dst_flat_atomic64>`::ref:`opt<amdgpu_synid9_opt>`::ref:`s64<amdgpu_synid9_type_dev>`, :ref:`vaddr<amdgpu_synid9_addr_flat>`,    :ref:`vdata<amdgpu_synid9_vdata64_0>`::ref:`s64<amdgpu_synid9_type_dev>`                   :ref:`flat_offset12<amdgpu_synid_flat_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    flat_atomic_smin               :ref:`vdst<amdgpu_synid9_dst_flat_atomic32>`::ref:`opt<amdgpu_synid9_opt>`::ref:`s32<amdgpu_synid9_type_dev>`, :ref:`vaddr<amdgpu_synid9_addr_flat>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`::ref:`s32<amdgpu_synid9_type_dev>`                   :ref:`flat_offset12<amdgpu_synid_flat_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    flat_atomic_smin_x2            :ref:`vdst<amdgpu_synid9_dst_flat_atomic64>`::ref:`opt<amdgpu_synid9_opt>`::ref:`s64<amdgpu_synid9_type_dev>`, :ref:`vaddr<amdgpu_synid9_addr_flat>`,    :ref:`vdata<amdgpu_synid9_vdata64_0>`::ref:`s64<amdgpu_synid9_type_dev>`                   :ref:`flat_offset12<amdgpu_synid_flat_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    flat_atomic_sub                :ref:`vdst<amdgpu_synid9_dst_flat_atomic32>`::ref:`opt<amdgpu_synid9_opt>`,     :ref:`vaddr<amdgpu_synid9_addr_flat>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`                       :ref:`flat_offset12<amdgpu_synid_flat_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    flat_atomic_sub_x2             :ref:`vdst<amdgpu_synid9_dst_flat_atomic64>`::ref:`opt<amdgpu_synid9_opt>`,     :ref:`vaddr<amdgpu_synid9_addr_flat>`,    :ref:`vdata<amdgpu_synid9_vdata64_0>`                       :ref:`flat_offset12<amdgpu_synid_flat_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    flat_atomic_swap               :ref:`vdst<amdgpu_synid9_dst_flat_atomic32>`::ref:`opt<amdgpu_synid9_opt>`,     :ref:`vaddr<amdgpu_synid9_addr_flat>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`                       :ref:`flat_offset12<amdgpu_synid_flat_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    flat_atomic_swap_x2            :ref:`vdst<amdgpu_synid9_dst_flat_atomic64>`::ref:`opt<amdgpu_synid9_opt>`,     :ref:`vaddr<amdgpu_synid9_addr_flat>`,    :ref:`vdata<amdgpu_synid9_vdata64_0>`                       :ref:`flat_offset12<amdgpu_synid_flat_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    flat_atomic_umax               :ref:`vdst<amdgpu_synid9_dst_flat_atomic32>`::ref:`opt<amdgpu_synid9_opt>`::ref:`u32<amdgpu_synid9_type_dev>`, :ref:`vaddr<amdgpu_synid9_addr_flat>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`::ref:`u32<amdgpu_synid9_type_dev>`                   :ref:`flat_offset12<amdgpu_synid_flat_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    flat_atomic_umax_x2            :ref:`vdst<amdgpu_synid9_dst_flat_atomic64>`::ref:`opt<amdgpu_synid9_opt>`::ref:`u64<amdgpu_synid9_type_dev>`, :ref:`vaddr<amdgpu_synid9_addr_flat>`,    :ref:`vdata<amdgpu_synid9_vdata64_0>`::ref:`u64<amdgpu_synid9_type_dev>`                   :ref:`flat_offset12<amdgpu_synid_flat_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    flat_atomic_umin               :ref:`vdst<amdgpu_synid9_dst_flat_atomic32>`::ref:`opt<amdgpu_synid9_opt>`::ref:`u32<amdgpu_synid9_type_dev>`, :ref:`vaddr<amdgpu_synid9_addr_flat>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`::ref:`u32<amdgpu_synid9_type_dev>`                   :ref:`flat_offset12<amdgpu_synid_flat_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    flat_atomic_umin_x2            :ref:`vdst<amdgpu_synid9_dst_flat_atomic64>`::ref:`opt<amdgpu_synid9_opt>`::ref:`u64<amdgpu_synid9_type_dev>`, :ref:`vaddr<amdgpu_synid9_addr_flat>`,    :ref:`vdata<amdgpu_synid9_vdata64_0>`::ref:`u64<amdgpu_synid9_type_dev>`                   :ref:`flat_offset12<amdgpu_synid_flat_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    flat_atomic_xor                :ref:`vdst<amdgpu_synid9_dst_flat_atomic32>`::ref:`opt<amdgpu_synid9_opt>`,     :ref:`vaddr<amdgpu_synid9_addr_flat>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`                       :ref:`flat_offset12<amdgpu_synid_flat_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    flat_atomic_xor_x2             :ref:`vdst<amdgpu_synid9_dst_flat_atomic64>`::ref:`opt<amdgpu_synid9_opt>`,     :ref:`vaddr<amdgpu_synid9_addr_flat>`,    :ref:`vdata<amdgpu_synid9_vdata64_0>`                       :ref:`flat_offset12<amdgpu_synid_flat_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    flat_load_dword                :ref:`vdst<amdgpu_synid9_vdst32_0>`,         :ref:`vaddr<amdgpu_synid9_addr_flat>`                                 :ref:`flat_offset12<amdgpu_synid_flat_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    flat_load_dwordx2              :ref:`vdst<amdgpu_synid9_vdst64_0>`,         :ref:`vaddr<amdgpu_synid9_addr_flat>`                                 :ref:`flat_offset12<amdgpu_synid_flat_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    flat_load_dwordx3              :ref:`vdst<amdgpu_synid9_vdst96_0>`,         :ref:`vaddr<amdgpu_synid9_addr_flat>`                                 :ref:`flat_offset12<amdgpu_synid_flat_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    flat_load_dwordx4              :ref:`vdst<amdgpu_synid9_vdst128_0>`,         :ref:`vaddr<amdgpu_synid9_addr_flat>`                                 :ref:`flat_offset12<amdgpu_synid_flat_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    flat_load_sbyte                :ref:`vdst<amdgpu_synid9_vdst32_0>`,         :ref:`vaddr<amdgpu_synid9_addr_flat>`                                 :ref:`flat_offset12<amdgpu_synid_flat_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    flat_load_sbyte_d16            :ref:`vdst<amdgpu_synid9_vdst32_0>`,         :ref:`vaddr<amdgpu_synid9_addr_flat>`                                 :ref:`flat_offset12<amdgpu_synid_flat_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    flat_load_sbyte_d16_hi         :ref:`vdst<amdgpu_synid9_vdst32_0>`,         :ref:`vaddr<amdgpu_synid9_addr_flat>`                                 :ref:`flat_offset12<amdgpu_synid_flat_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    flat_load_short_d16            :ref:`vdst<amdgpu_synid9_vdst32_0>`,         :ref:`vaddr<amdgpu_synid9_addr_flat>`                                 :ref:`flat_offset12<amdgpu_synid_flat_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    flat_load_short_d16_hi         :ref:`vdst<amdgpu_synid9_vdst32_0>`,         :ref:`vaddr<amdgpu_synid9_addr_flat>`                                 :ref:`flat_offset12<amdgpu_synid_flat_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    flat_load_sshort               :ref:`vdst<amdgpu_synid9_vdst32_0>`,         :ref:`vaddr<amdgpu_synid9_addr_flat>`                                 :ref:`flat_offset12<amdgpu_synid_flat_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    flat_load_ubyte                :ref:`vdst<amdgpu_synid9_vdst32_0>`,         :ref:`vaddr<amdgpu_synid9_addr_flat>`                                 :ref:`flat_offset12<amdgpu_synid_flat_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    flat_load_ubyte_d16            :ref:`vdst<amdgpu_synid9_vdst32_0>`,         :ref:`vaddr<amdgpu_synid9_addr_flat>`                                 :ref:`flat_offset12<amdgpu_synid_flat_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    flat_load_ubyte_d16_hi         :ref:`vdst<amdgpu_synid9_vdst32_0>`,         :ref:`vaddr<amdgpu_synid9_addr_flat>`                                 :ref:`flat_offset12<amdgpu_synid_flat_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    flat_load_ushort               :ref:`vdst<amdgpu_synid9_vdst32_0>`,         :ref:`vaddr<amdgpu_synid9_addr_flat>`                                 :ref:`flat_offset12<amdgpu_synid_flat_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    flat_store_byte                              :ref:`vaddr<amdgpu_synid9_addr_flat>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`                       :ref:`flat_offset12<amdgpu_synid_flat_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    flat_store_byte_d16_hi                       :ref:`vaddr<amdgpu_synid9_addr_flat>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`                       :ref:`flat_offset12<amdgpu_synid_flat_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    flat_store_dword                             :ref:`vaddr<amdgpu_synid9_addr_flat>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`                       :ref:`flat_offset12<amdgpu_synid_flat_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    flat_store_dwordx2                           :ref:`vaddr<amdgpu_synid9_addr_flat>`,    :ref:`vdata<amdgpu_synid9_vdata64_0>`                       :ref:`flat_offset12<amdgpu_synid_flat_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    flat_store_dwordx3                           :ref:`vaddr<amdgpu_synid9_addr_flat>`,    :ref:`vdata<amdgpu_synid9_vdata96_0>`                       :ref:`flat_offset12<amdgpu_synid_flat_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    flat_store_dwordx4                           :ref:`vaddr<amdgpu_synid9_addr_flat>`,    :ref:`vdata<amdgpu_synid9_vdata128_0>`                       :ref:`flat_offset12<amdgpu_synid_flat_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    flat_store_short                             :ref:`vaddr<amdgpu_synid9_addr_flat>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`                       :ref:`flat_offset12<amdgpu_synid_flat_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    flat_store_short_d16_hi                      :ref:`vaddr<amdgpu_synid9_addr_flat>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`                       :ref:`flat_offset12<amdgpu_synid_flat_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    global_atomic_add              :ref:`vdst<amdgpu_synid9_dst_flat_atomic32>`::ref:`opt<amdgpu_synid9_opt>`,     :ref:`vaddr<amdgpu_synid9_vaddr_flat_global>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`,       :ref:`saddr<amdgpu_synid9_saddr_flat_global>`          :ref:`flat_offset13<amdgpu_synid_flat_offset13>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    global_atomic_add_x2           :ref:`vdst<amdgpu_synid9_dst_flat_atomic64>`::ref:`opt<amdgpu_synid9_opt>`,     :ref:`vaddr<amdgpu_synid9_vaddr_flat_global>`,    :ref:`vdata<amdgpu_synid9_vdata64_0>`,       :ref:`saddr<amdgpu_synid9_saddr_flat_global>`          :ref:`flat_offset13<amdgpu_synid_flat_offset13>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    global_atomic_and              :ref:`vdst<amdgpu_synid9_dst_flat_atomic32>`::ref:`opt<amdgpu_synid9_opt>`,     :ref:`vaddr<amdgpu_synid9_vaddr_flat_global>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`,       :ref:`saddr<amdgpu_synid9_saddr_flat_global>`          :ref:`flat_offset13<amdgpu_synid_flat_offset13>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    global_atomic_and_x2           :ref:`vdst<amdgpu_synid9_dst_flat_atomic64>`::ref:`opt<amdgpu_synid9_opt>`,     :ref:`vaddr<amdgpu_synid9_vaddr_flat_global>`,    :ref:`vdata<amdgpu_synid9_vdata64_0>`,       :ref:`saddr<amdgpu_synid9_saddr_flat_global>`          :ref:`flat_offset13<amdgpu_synid_flat_offset13>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    global_atomic_cmpswap          :ref:`vdst<amdgpu_synid9_dst_flat_atomic32>`::ref:`opt<amdgpu_synid9_opt>`,     :ref:`vaddr<amdgpu_synid9_vaddr_flat_global>`,    :ref:`vdata<amdgpu_synid9_vdata64_0>`::ref:`b32x2<amdgpu_synid9_type_dev>`, :ref:`saddr<amdgpu_synid9_saddr_flat_global>`          :ref:`flat_offset13<amdgpu_synid_flat_offset13>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    global_atomic_cmpswap_x2       :ref:`vdst<amdgpu_synid9_dst_flat_atomic64>`::ref:`opt<amdgpu_synid9_opt>`,     :ref:`vaddr<amdgpu_synid9_vaddr_flat_global>`,    :ref:`vdata<amdgpu_synid9_vdata128_0>`::ref:`b64x2<amdgpu_synid9_type_dev>`, :ref:`saddr<amdgpu_synid9_saddr_flat_global>`          :ref:`flat_offset13<amdgpu_synid_flat_offset13>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    global_atomic_dec              :ref:`vdst<amdgpu_synid9_dst_flat_atomic32>`::ref:`opt<amdgpu_synid9_opt>`::ref:`u32<amdgpu_synid9_type_dev>`, :ref:`vaddr<amdgpu_synid9_vaddr_flat_global>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`::ref:`u32<amdgpu_synid9_type_dev>`,   :ref:`saddr<amdgpu_synid9_saddr_flat_global>`          :ref:`flat_offset13<amdgpu_synid_flat_offset13>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    global_atomic_dec_x2           :ref:`vdst<amdgpu_synid9_dst_flat_atomic64>`::ref:`opt<amdgpu_synid9_opt>`::ref:`u64<amdgpu_synid9_type_dev>`, :ref:`vaddr<amdgpu_synid9_vaddr_flat_global>`,    :ref:`vdata<amdgpu_synid9_vdata64_0>`::ref:`u64<amdgpu_synid9_type_dev>`,   :ref:`saddr<amdgpu_synid9_saddr_flat_global>`          :ref:`flat_offset13<amdgpu_synid_flat_offset13>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    global_atomic_inc              :ref:`vdst<amdgpu_synid9_dst_flat_atomic32>`::ref:`opt<amdgpu_synid9_opt>`::ref:`u32<amdgpu_synid9_type_dev>`, :ref:`vaddr<amdgpu_synid9_vaddr_flat_global>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`::ref:`u32<amdgpu_synid9_type_dev>`,   :ref:`saddr<amdgpu_synid9_saddr_flat_global>`          :ref:`flat_offset13<amdgpu_synid_flat_offset13>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    global_atomic_inc_x2           :ref:`vdst<amdgpu_synid9_dst_flat_atomic64>`::ref:`opt<amdgpu_synid9_opt>`::ref:`u64<amdgpu_synid9_type_dev>`, :ref:`vaddr<amdgpu_synid9_vaddr_flat_global>`,    :ref:`vdata<amdgpu_synid9_vdata64_0>`::ref:`u64<amdgpu_synid9_type_dev>`,   :ref:`saddr<amdgpu_synid9_saddr_flat_global>`          :ref:`flat_offset13<amdgpu_synid_flat_offset13>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    global_atomic_or               :ref:`vdst<amdgpu_synid9_dst_flat_atomic32>`::ref:`opt<amdgpu_synid9_opt>`,     :ref:`vaddr<amdgpu_synid9_vaddr_flat_global>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`,       :ref:`saddr<amdgpu_synid9_saddr_flat_global>`          :ref:`flat_offset13<amdgpu_synid_flat_offset13>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    global_atomic_or_x2            :ref:`vdst<amdgpu_synid9_dst_flat_atomic64>`::ref:`opt<amdgpu_synid9_opt>`,     :ref:`vaddr<amdgpu_synid9_vaddr_flat_global>`,    :ref:`vdata<amdgpu_synid9_vdata64_0>`,       :ref:`saddr<amdgpu_synid9_saddr_flat_global>`          :ref:`flat_offset13<amdgpu_synid_flat_offset13>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    global_atomic_smax             :ref:`vdst<amdgpu_synid9_dst_flat_atomic32>`::ref:`opt<amdgpu_synid9_opt>`::ref:`s32<amdgpu_synid9_type_dev>`, :ref:`vaddr<amdgpu_synid9_vaddr_flat_global>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`::ref:`s32<amdgpu_synid9_type_dev>`,   :ref:`saddr<amdgpu_synid9_saddr_flat_global>`          :ref:`flat_offset13<amdgpu_synid_flat_offset13>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    global_atomic_smax_x2          :ref:`vdst<amdgpu_synid9_dst_flat_atomic64>`::ref:`opt<amdgpu_synid9_opt>`::ref:`s64<amdgpu_synid9_type_dev>`, :ref:`vaddr<amdgpu_synid9_vaddr_flat_global>`,    :ref:`vdata<amdgpu_synid9_vdata64_0>`::ref:`s64<amdgpu_synid9_type_dev>`,   :ref:`saddr<amdgpu_synid9_saddr_flat_global>`          :ref:`flat_offset13<amdgpu_synid_flat_offset13>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    global_atomic_smin             :ref:`vdst<amdgpu_synid9_dst_flat_atomic32>`::ref:`opt<amdgpu_synid9_opt>`::ref:`s32<amdgpu_synid9_type_dev>`, :ref:`vaddr<amdgpu_synid9_vaddr_flat_global>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`::ref:`s32<amdgpu_synid9_type_dev>`,   :ref:`saddr<amdgpu_synid9_saddr_flat_global>`          :ref:`flat_offset13<amdgpu_synid_flat_offset13>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    global_atomic_smin_x2          :ref:`vdst<amdgpu_synid9_dst_flat_atomic64>`::ref:`opt<amdgpu_synid9_opt>`::ref:`s64<amdgpu_synid9_type_dev>`, :ref:`vaddr<amdgpu_synid9_vaddr_flat_global>`,    :ref:`vdata<amdgpu_synid9_vdata64_0>`::ref:`s64<amdgpu_synid9_type_dev>`,   :ref:`saddr<amdgpu_synid9_saddr_flat_global>`          :ref:`flat_offset13<amdgpu_synid_flat_offset13>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    global_atomic_sub              :ref:`vdst<amdgpu_synid9_dst_flat_atomic32>`::ref:`opt<amdgpu_synid9_opt>`,     :ref:`vaddr<amdgpu_synid9_vaddr_flat_global>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`,       :ref:`saddr<amdgpu_synid9_saddr_flat_global>`          :ref:`flat_offset13<amdgpu_synid_flat_offset13>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    global_atomic_sub_x2           :ref:`vdst<amdgpu_synid9_dst_flat_atomic64>`::ref:`opt<amdgpu_synid9_opt>`,     :ref:`vaddr<amdgpu_synid9_vaddr_flat_global>`,    :ref:`vdata<amdgpu_synid9_vdata64_0>`,       :ref:`saddr<amdgpu_synid9_saddr_flat_global>`          :ref:`flat_offset13<amdgpu_synid_flat_offset13>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    global_atomic_swap             :ref:`vdst<amdgpu_synid9_dst_flat_atomic32>`::ref:`opt<amdgpu_synid9_opt>`,     :ref:`vaddr<amdgpu_synid9_vaddr_flat_global>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`,       :ref:`saddr<amdgpu_synid9_saddr_flat_global>`          :ref:`flat_offset13<amdgpu_synid_flat_offset13>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    global_atomic_swap_x2          :ref:`vdst<amdgpu_synid9_dst_flat_atomic64>`::ref:`opt<amdgpu_synid9_opt>`,     :ref:`vaddr<amdgpu_synid9_vaddr_flat_global>`,    :ref:`vdata<amdgpu_synid9_vdata64_0>`,       :ref:`saddr<amdgpu_synid9_saddr_flat_global>`          :ref:`flat_offset13<amdgpu_synid_flat_offset13>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    global_atomic_umax             :ref:`vdst<amdgpu_synid9_dst_flat_atomic32>`::ref:`opt<amdgpu_synid9_opt>`::ref:`u32<amdgpu_synid9_type_dev>`, :ref:`vaddr<amdgpu_synid9_vaddr_flat_global>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`::ref:`u32<amdgpu_synid9_type_dev>`,   :ref:`saddr<amdgpu_synid9_saddr_flat_global>`          :ref:`flat_offset13<amdgpu_synid_flat_offset13>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    global_atomic_umax_x2          :ref:`vdst<amdgpu_synid9_dst_flat_atomic64>`::ref:`opt<amdgpu_synid9_opt>`::ref:`u64<amdgpu_synid9_type_dev>`, :ref:`vaddr<amdgpu_synid9_vaddr_flat_global>`,    :ref:`vdata<amdgpu_synid9_vdata64_0>`::ref:`u64<amdgpu_synid9_type_dev>`,   :ref:`saddr<amdgpu_synid9_saddr_flat_global>`          :ref:`flat_offset13<amdgpu_synid_flat_offset13>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    global_atomic_umin             :ref:`vdst<amdgpu_synid9_dst_flat_atomic32>`::ref:`opt<amdgpu_synid9_opt>`::ref:`u32<amdgpu_synid9_type_dev>`, :ref:`vaddr<amdgpu_synid9_vaddr_flat_global>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`::ref:`u32<amdgpu_synid9_type_dev>`,   :ref:`saddr<amdgpu_synid9_saddr_flat_global>`          :ref:`flat_offset13<amdgpu_synid_flat_offset13>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    global_atomic_umin_x2          :ref:`vdst<amdgpu_synid9_dst_flat_atomic64>`::ref:`opt<amdgpu_synid9_opt>`::ref:`u64<amdgpu_synid9_type_dev>`, :ref:`vaddr<amdgpu_synid9_vaddr_flat_global>`,    :ref:`vdata<amdgpu_synid9_vdata64_0>`::ref:`u64<amdgpu_synid9_type_dev>`,   :ref:`saddr<amdgpu_synid9_saddr_flat_global>`          :ref:`flat_offset13<amdgpu_synid_flat_offset13>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    global_atomic_xor              :ref:`vdst<amdgpu_synid9_dst_flat_atomic32>`::ref:`opt<amdgpu_synid9_opt>`,     :ref:`vaddr<amdgpu_synid9_vaddr_flat_global>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`,       :ref:`saddr<amdgpu_synid9_saddr_flat_global>`          :ref:`flat_offset13<amdgpu_synid_flat_offset13>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    global_atomic_xor_x2           :ref:`vdst<amdgpu_synid9_dst_flat_atomic64>`::ref:`opt<amdgpu_synid9_opt>`,     :ref:`vaddr<amdgpu_synid9_vaddr_flat_global>`,    :ref:`vdata<amdgpu_synid9_vdata64_0>`,       :ref:`saddr<amdgpu_synid9_saddr_flat_global>`          :ref:`flat_offset13<amdgpu_synid_flat_offset13>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    global_load_dword              :ref:`vdst<amdgpu_synid9_vdst32_0>`,         :ref:`vaddr<amdgpu_synid9_vaddr_flat_global>`,    :ref:`saddr<amdgpu_synid9_saddr_flat_global>`                       :ref:`flat_offset13<amdgpu_synid_flat_offset13>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    global_load_dwordx2            :ref:`vdst<amdgpu_synid9_vdst64_0>`,         :ref:`vaddr<amdgpu_synid9_vaddr_flat_global>`,    :ref:`saddr<amdgpu_synid9_saddr_flat_global>`                       :ref:`flat_offset13<amdgpu_synid_flat_offset13>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    global_load_dwordx3            :ref:`vdst<amdgpu_synid9_vdst96_0>`,         :ref:`vaddr<amdgpu_synid9_vaddr_flat_global>`,    :ref:`saddr<amdgpu_synid9_saddr_flat_global>`                       :ref:`flat_offset13<amdgpu_synid_flat_offset13>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    global_load_dwordx4            :ref:`vdst<amdgpu_synid9_vdst128_0>`,         :ref:`vaddr<amdgpu_synid9_vaddr_flat_global>`,    :ref:`saddr<amdgpu_synid9_saddr_flat_global>`                       :ref:`flat_offset13<amdgpu_synid_flat_offset13>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    global_load_sbyte              :ref:`vdst<amdgpu_synid9_vdst32_0>`,         :ref:`vaddr<amdgpu_synid9_vaddr_flat_global>`,    :ref:`saddr<amdgpu_synid9_saddr_flat_global>`                       :ref:`flat_offset13<amdgpu_synid_flat_offset13>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    global_load_sbyte_d16          :ref:`vdst<amdgpu_synid9_vdst32_0>`,         :ref:`vaddr<amdgpu_synid9_vaddr_flat_global>`,    :ref:`saddr<amdgpu_synid9_saddr_flat_global>`                       :ref:`flat_offset13<amdgpu_synid_flat_offset13>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    global_load_sbyte_d16_hi       :ref:`vdst<amdgpu_synid9_vdst32_0>`,         :ref:`vaddr<amdgpu_synid9_vaddr_flat_global>`,    :ref:`saddr<amdgpu_synid9_saddr_flat_global>`                       :ref:`flat_offset13<amdgpu_synid_flat_offset13>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    global_load_short_d16          :ref:`vdst<amdgpu_synid9_vdst32_0>`,         :ref:`vaddr<amdgpu_synid9_vaddr_flat_global>`,    :ref:`saddr<amdgpu_synid9_saddr_flat_global>`                       :ref:`flat_offset13<amdgpu_synid_flat_offset13>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    global_load_short_d16_hi       :ref:`vdst<amdgpu_synid9_vdst32_0>`,         :ref:`vaddr<amdgpu_synid9_vaddr_flat_global>`,    :ref:`saddr<amdgpu_synid9_saddr_flat_global>`                       :ref:`flat_offset13<amdgpu_synid_flat_offset13>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    global_load_sshort             :ref:`vdst<amdgpu_synid9_vdst32_0>`,         :ref:`vaddr<amdgpu_synid9_vaddr_flat_global>`,    :ref:`saddr<amdgpu_synid9_saddr_flat_global>`                       :ref:`flat_offset13<amdgpu_synid_flat_offset13>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    global_load_ubyte              :ref:`vdst<amdgpu_synid9_vdst32_0>`,         :ref:`vaddr<amdgpu_synid9_vaddr_flat_global>`,    :ref:`saddr<amdgpu_synid9_saddr_flat_global>`                       :ref:`flat_offset13<amdgpu_synid_flat_offset13>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    global_load_ubyte_d16          :ref:`vdst<amdgpu_synid9_vdst32_0>`,         :ref:`vaddr<amdgpu_synid9_vaddr_flat_global>`,    :ref:`saddr<amdgpu_synid9_saddr_flat_global>`                       :ref:`flat_offset13<amdgpu_synid_flat_offset13>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    global_load_ubyte_d16_hi       :ref:`vdst<amdgpu_synid9_vdst32_0>`,         :ref:`vaddr<amdgpu_synid9_vaddr_flat_global>`,    :ref:`saddr<amdgpu_synid9_saddr_flat_global>`                       :ref:`flat_offset13<amdgpu_synid_flat_offset13>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    global_load_ushort             :ref:`vdst<amdgpu_synid9_vdst32_0>`,         :ref:`vaddr<amdgpu_synid9_vaddr_flat_global>`,    :ref:`saddr<amdgpu_synid9_saddr_flat_global>`                       :ref:`flat_offset13<amdgpu_synid_flat_offset13>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    global_store_byte                            :ref:`vaddr<amdgpu_synid9_vaddr_flat_global>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`,       :ref:`saddr<amdgpu_synid9_saddr_flat_global>`          :ref:`flat_offset13<amdgpu_synid_flat_offset13>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    global_store_byte_d16_hi                     :ref:`vaddr<amdgpu_synid9_vaddr_flat_global>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`,       :ref:`saddr<amdgpu_synid9_saddr_flat_global>`          :ref:`flat_offset13<amdgpu_synid_flat_offset13>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    global_store_dword                           :ref:`vaddr<amdgpu_synid9_vaddr_flat_global>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`,       :ref:`saddr<amdgpu_synid9_saddr_flat_global>`          :ref:`flat_offset13<amdgpu_synid_flat_offset13>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    global_store_dwordx2                         :ref:`vaddr<amdgpu_synid9_vaddr_flat_global>`,    :ref:`vdata<amdgpu_synid9_vdata64_0>`,       :ref:`saddr<amdgpu_synid9_saddr_flat_global>`          :ref:`flat_offset13<amdgpu_synid_flat_offset13>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    global_store_dwordx3                         :ref:`vaddr<amdgpu_synid9_vaddr_flat_global>`,    :ref:`vdata<amdgpu_synid9_vdata96_0>`,       :ref:`saddr<amdgpu_synid9_saddr_flat_global>`          :ref:`flat_offset13<amdgpu_synid_flat_offset13>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    global_store_dwordx4                         :ref:`vaddr<amdgpu_synid9_vaddr_flat_global>`,    :ref:`vdata<amdgpu_synid9_vdata128_0>`,       :ref:`saddr<amdgpu_synid9_saddr_flat_global>`          :ref:`flat_offset13<amdgpu_synid_flat_offset13>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    global_store_short                           :ref:`vaddr<amdgpu_synid9_vaddr_flat_global>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`,       :ref:`saddr<amdgpu_synid9_saddr_flat_global>`          :ref:`flat_offset13<amdgpu_synid_flat_offset13>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    global_store_short_d16_hi                    :ref:`vaddr<amdgpu_synid9_vaddr_flat_global>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`,       :ref:`saddr<amdgpu_synid9_saddr_flat_global>`          :ref:`flat_offset13<amdgpu_synid_flat_offset13>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    scratch_load_dword             :ref:`vdst<amdgpu_synid9_vdst32_0>`,         :ref:`vaddr<amdgpu_synid9_vaddr_flat_scratch>`,    :ref:`saddr<amdgpu_synid9_saddr_flat_scratch>`                       :ref:`flat_offset13<amdgpu_synid_flat_offset13>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    scratch_load_dwordx2           :ref:`vdst<amdgpu_synid9_vdst64_0>`,         :ref:`vaddr<amdgpu_synid9_vaddr_flat_scratch>`,    :ref:`saddr<amdgpu_synid9_saddr_flat_scratch>`                       :ref:`flat_offset13<amdgpu_synid_flat_offset13>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    scratch_load_dwordx3           :ref:`vdst<amdgpu_synid9_vdst96_0>`,         :ref:`vaddr<amdgpu_synid9_vaddr_flat_scratch>`,    :ref:`saddr<amdgpu_synid9_saddr_flat_scratch>`                       :ref:`flat_offset13<amdgpu_synid_flat_offset13>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    scratch_load_dwordx4           :ref:`vdst<amdgpu_synid9_vdst128_0>`,         :ref:`vaddr<amdgpu_synid9_vaddr_flat_scratch>`,    :ref:`saddr<amdgpu_synid9_saddr_flat_scratch>`                       :ref:`flat_offset13<amdgpu_synid_flat_offset13>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    scratch_load_sbyte             :ref:`vdst<amdgpu_synid9_vdst32_0>`,         :ref:`vaddr<amdgpu_synid9_vaddr_flat_scratch>`,    :ref:`saddr<amdgpu_synid9_saddr_flat_scratch>`                       :ref:`flat_offset13<amdgpu_synid_flat_offset13>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    scratch_load_sbyte_d16         :ref:`vdst<amdgpu_synid9_vdst32_0>`,         :ref:`vaddr<amdgpu_synid9_vaddr_flat_scratch>`,    :ref:`saddr<amdgpu_synid9_saddr_flat_scratch>`                       :ref:`flat_offset13<amdgpu_synid_flat_offset13>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    scratch_load_sbyte_d16_hi      :ref:`vdst<amdgpu_synid9_vdst32_0>`,         :ref:`vaddr<amdgpu_synid9_vaddr_flat_scratch>`,    :ref:`saddr<amdgpu_synid9_saddr_flat_scratch>`                       :ref:`flat_offset13<amdgpu_synid_flat_offset13>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    scratch_load_short_d16         :ref:`vdst<amdgpu_synid9_vdst32_0>`,         :ref:`vaddr<amdgpu_synid9_vaddr_flat_scratch>`,    :ref:`saddr<amdgpu_synid9_saddr_flat_scratch>`                       :ref:`flat_offset13<amdgpu_synid_flat_offset13>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    scratch_load_short_d16_hi      :ref:`vdst<amdgpu_synid9_vdst32_0>`,         :ref:`vaddr<amdgpu_synid9_vaddr_flat_scratch>`,    :ref:`saddr<amdgpu_synid9_saddr_flat_scratch>`                       :ref:`flat_offset13<amdgpu_synid_flat_offset13>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    scratch_load_sshort            :ref:`vdst<amdgpu_synid9_vdst32_0>`,         :ref:`vaddr<amdgpu_synid9_vaddr_flat_scratch>`,    :ref:`saddr<amdgpu_synid9_saddr_flat_scratch>`                       :ref:`flat_offset13<amdgpu_synid_flat_offset13>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    scratch_load_ubyte             :ref:`vdst<amdgpu_synid9_vdst32_0>`,         :ref:`vaddr<amdgpu_synid9_vaddr_flat_scratch>`,    :ref:`saddr<amdgpu_synid9_saddr_flat_scratch>`                       :ref:`flat_offset13<amdgpu_synid_flat_offset13>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    scratch_load_ubyte_d16         :ref:`vdst<amdgpu_synid9_vdst32_0>`,         :ref:`vaddr<amdgpu_synid9_vaddr_flat_scratch>`,    :ref:`saddr<amdgpu_synid9_saddr_flat_scratch>`                       :ref:`flat_offset13<amdgpu_synid_flat_offset13>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    scratch_load_ubyte_d16_hi      :ref:`vdst<amdgpu_synid9_vdst32_0>`,         :ref:`vaddr<amdgpu_synid9_vaddr_flat_scratch>`,    :ref:`saddr<amdgpu_synid9_saddr_flat_scratch>`                       :ref:`flat_offset13<amdgpu_synid_flat_offset13>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    scratch_load_ushort            :ref:`vdst<amdgpu_synid9_vdst32_0>`,         :ref:`vaddr<amdgpu_synid9_vaddr_flat_scratch>`,    :ref:`saddr<amdgpu_synid9_saddr_flat_scratch>`                       :ref:`flat_offset13<amdgpu_synid_flat_offset13>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    scratch_store_byte                           :ref:`vaddr<amdgpu_synid9_vaddr_flat_scratch>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`,       :ref:`saddr<amdgpu_synid9_saddr_flat_scratch>`          :ref:`flat_offset13<amdgpu_synid_flat_offset13>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    scratch_store_byte_d16_hi                    :ref:`vaddr<amdgpu_synid9_vaddr_flat_scratch>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`,       :ref:`saddr<amdgpu_synid9_saddr_flat_scratch>`          :ref:`flat_offset13<amdgpu_synid_flat_offset13>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    scratch_store_dword                          :ref:`vaddr<amdgpu_synid9_vaddr_flat_scratch>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`,       :ref:`saddr<amdgpu_synid9_saddr_flat_scratch>`          :ref:`flat_offset13<amdgpu_synid_flat_offset13>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    scratch_store_dwordx2                        :ref:`vaddr<amdgpu_synid9_vaddr_flat_scratch>`,    :ref:`vdata<amdgpu_synid9_vdata64_0>`,       :ref:`saddr<amdgpu_synid9_saddr_flat_scratch>`          :ref:`flat_offset13<amdgpu_synid_flat_offset13>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    scratch_store_dwordx3                        :ref:`vaddr<amdgpu_synid9_vaddr_flat_scratch>`,    :ref:`vdata<amdgpu_synid9_vdata96_0>`,       :ref:`saddr<amdgpu_synid9_saddr_flat_scratch>`          :ref:`flat_offset13<amdgpu_synid_flat_offset13>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    scratch_store_dwordx4                        :ref:`vaddr<amdgpu_synid9_vaddr_flat_scratch>`,    :ref:`vdata<amdgpu_synid9_vdata128_0>`,       :ref:`saddr<amdgpu_synid9_saddr_flat_scratch>`          :ref:`flat_offset13<amdgpu_synid_flat_offset13>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    scratch_store_short                          :ref:`vaddr<amdgpu_synid9_vaddr_flat_scratch>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`,       :ref:`saddr<amdgpu_synid9_saddr_flat_scratch>`          :ref:`flat_offset13<amdgpu_synid_flat_offset13>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    scratch_store_short_d16_hi                   :ref:`vaddr<amdgpu_synid9_vaddr_flat_scratch>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`,       :ref:`saddr<amdgpu_synid9_saddr_flat_scratch>`          :ref:`flat_offset13<amdgpu_synid_flat_offset13>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    \ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|
+    flat_atomic_add                :ref:`vdst<amdgpu_synid9_dst_flat_atomic32>`::ref:`opt<amdgpu_synid9_opt>`,     :ref:`vaddr<amdgpu_synid9_addr_flat>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`                       :ref:`offset12<amdgpu_synid_flat_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    flat_atomic_add_x2             :ref:`vdst<amdgpu_synid9_dst_flat_atomic64>`::ref:`opt<amdgpu_synid9_opt>`,     :ref:`vaddr<amdgpu_synid9_addr_flat>`,    :ref:`vdata<amdgpu_synid9_vdata64_0>`                       :ref:`offset12<amdgpu_synid_flat_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    flat_atomic_and                :ref:`vdst<amdgpu_synid9_dst_flat_atomic32>`::ref:`opt<amdgpu_synid9_opt>`,     :ref:`vaddr<amdgpu_synid9_addr_flat>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`                       :ref:`offset12<amdgpu_synid_flat_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    flat_atomic_and_x2             :ref:`vdst<amdgpu_synid9_dst_flat_atomic64>`::ref:`opt<amdgpu_synid9_opt>`,     :ref:`vaddr<amdgpu_synid9_addr_flat>`,    :ref:`vdata<amdgpu_synid9_vdata64_0>`                       :ref:`offset12<amdgpu_synid_flat_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    flat_atomic_cmpswap            :ref:`vdst<amdgpu_synid9_dst_flat_atomic32>`::ref:`opt<amdgpu_synid9_opt>`,     :ref:`vaddr<amdgpu_synid9_addr_flat>`,    :ref:`vdata<amdgpu_synid9_vdata64_0>`::ref:`b32x2<amdgpu_synid9_type_dev>`                 :ref:`offset12<amdgpu_synid_flat_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    flat_atomic_cmpswap_x2         :ref:`vdst<amdgpu_synid9_dst_flat_atomic64>`::ref:`opt<amdgpu_synid9_opt>`,     :ref:`vaddr<amdgpu_synid9_addr_flat>`,    :ref:`vdata<amdgpu_synid9_vdata128_0>`::ref:`b64x2<amdgpu_synid9_type_dev>`                 :ref:`offset12<amdgpu_synid_flat_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    flat_atomic_dec                :ref:`vdst<amdgpu_synid9_dst_flat_atomic32>`::ref:`opt<amdgpu_synid9_opt>`::ref:`u32<amdgpu_synid9_type_dev>`, :ref:`vaddr<amdgpu_synid9_addr_flat>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`::ref:`u32<amdgpu_synid9_type_dev>`                   :ref:`offset12<amdgpu_synid_flat_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    flat_atomic_dec_x2             :ref:`vdst<amdgpu_synid9_dst_flat_atomic64>`::ref:`opt<amdgpu_synid9_opt>`::ref:`u64<amdgpu_synid9_type_dev>`, :ref:`vaddr<amdgpu_synid9_addr_flat>`,    :ref:`vdata<amdgpu_synid9_vdata64_0>`::ref:`u64<amdgpu_synid9_type_dev>`                   :ref:`offset12<amdgpu_synid_flat_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    flat_atomic_inc                :ref:`vdst<amdgpu_synid9_dst_flat_atomic32>`::ref:`opt<amdgpu_synid9_opt>`::ref:`u32<amdgpu_synid9_type_dev>`, :ref:`vaddr<amdgpu_synid9_addr_flat>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`::ref:`u32<amdgpu_synid9_type_dev>`                   :ref:`offset12<amdgpu_synid_flat_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    flat_atomic_inc_x2             :ref:`vdst<amdgpu_synid9_dst_flat_atomic64>`::ref:`opt<amdgpu_synid9_opt>`::ref:`u64<amdgpu_synid9_type_dev>`, :ref:`vaddr<amdgpu_synid9_addr_flat>`,    :ref:`vdata<amdgpu_synid9_vdata64_0>`::ref:`u64<amdgpu_synid9_type_dev>`                   :ref:`offset12<amdgpu_synid_flat_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    flat_atomic_or                 :ref:`vdst<amdgpu_synid9_dst_flat_atomic32>`::ref:`opt<amdgpu_synid9_opt>`,     :ref:`vaddr<amdgpu_synid9_addr_flat>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`                       :ref:`offset12<amdgpu_synid_flat_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    flat_atomic_or_x2              :ref:`vdst<amdgpu_synid9_dst_flat_atomic64>`::ref:`opt<amdgpu_synid9_opt>`,     :ref:`vaddr<amdgpu_synid9_addr_flat>`,    :ref:`vdata<amdgpu_synid9_vdata64_0>`                       :ref:`offset12<amdgpu_synid_flat_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    flat_atomic_smax               :ref:`vdst<amdgpu_synid9_dst_flat_atomic32>`::ref:`opt<amdgpu_synid9_opt>`::ref:`s32<amdgpu_synid9_type_dev>`, :ref:`vaddr<amdgpu_synid9_addr_flat>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`::ref:`s32<amdgpu_synid9_type_dev>`                   :ref:`offset12<amdgpu_synid_flat_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    flat_atomic_smax_x2            :ref:`vdst<amdgpu_synid9_dst_flat_atomic64>`::ref:`opt<amdgpu_synid9_opt>`::ref:`s64<amdgpu_synid9_type_dev>`, :ref:`vaddr<amdgpu_synid9_addr_flat>`,    :ref:`vdata<amdgpu_synid9_vdata64_0>`::ref:`s64<amdgpu_synid9_type_dev>`                   :ref:`offset12<amdgpu_synid_flat_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    flat_atomic_smin               :ref:`vdst<amdgpu_synid9_dst_flat_atomic32>`::ref:`opt<amdgpu_synid9_opt>`::ref:`s32<amdgpu_synid9_type_dev>`, :ref:`vaddr<amdgpu_synid9_addr_flat>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`::ref:`s32<amdgpu_synid9_type_dev>`                   :ref:`offset12<amdgpu_synid_flat_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    flat_atomic_smin_x2            :ref:`vdst<amdgpu_synid9_dst_flat_atomic64>`::ref:`opt<amdgpu_synid9_opt>`::ref:`s64<amdgpu_synid9_type_dev>`, :ref:`vaddr<amdgpu_synid9_addr_flat>`,    :ref:`vdata<amdgpu_synid9_vdata64_0>`::ref:`s64<amdgpu_synid9_type_dev>`                   :ref:`offset12<amdgpu_synid_flat_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    flat_atomic_sub                :ref:`vdst<amdgpu_synid9_dst_flat_atomic32>`::ref:`opt<amdgpu_synid9_opt>`,     :ref:`vaddr<amdgpu_synid9_addr_flat>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`                       :ref:`offset12<amdgpu_synid_flat_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    flat_atomic_sub_x2             :ref:`vdst<amdgpu_synid9_dst_flat_atomic64>`::ref:`opt<amdgpu_synid9_opt>`,     :ref:`vaddr<amdgpu_synid9_addr_flat>`,    :ref:`vdata<amdgpu_synid9_vdata64_0>`                       :ref:`offset12<amdgpu_synid_flat_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    flat_atomic_swap               :ref:`vdst<amdgpu_synid9_dst_flat_atomic32>`::ref:`opt<amdgpu_synid9_opt>`,     :ref:`vaddr<amdgpu_synid9_addr_flat>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`                       :ref:`offset12<amdgpu_synid_flat_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    flat_atomic_swap_x2            :ref:`vdst<amdgpu_synid9_dst_flat_atomic64>`::ref:`opt<amdgpu_synid9_opt>`,     :ref:`vaddr<amdgpu_synid9_addr_flat>`,    :ref:`vdata<amdgpu_synid9_vdata64_0>`                       :ref:`offset12<amdgpu_synid_flat_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    flat_atomic_umax               :ref:`vdst<amdgpu_synid9_dst_flat_atomic32>`::ref:`opt<amdgpu_synid9_opt>`::ref:`u32<amdgpu_synid9_type_dev>`, :ref:`vaddr<amdgpu_synid9_addr_flat>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`::ref:`u32<amdgpu_synid9_type_dev>`                   :ref:`offset12<amdgpu_synid_flat_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    flat_atomic_umax_x2            :ref:`vdst<amdgpu_synid9_dst_flat_atomic64>`::ref:`opt<amdgpu_synid9_opt>`::ref:`u64<amdgpu_synid9_type_dev>`, :ref:`vaddr<amdgpu_synid9_addr_flat>`,    :ref:`vdata<amdgpu_synid9_vdata64_0>`::ref:`u64<amdgpu_synid9_type_dev>`                   :ref:`offset12<amdgpu_synid_flat_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    flat_atomic_umin               :ref:`vdst<amdgpu_synid9_dst_flat_atomic32>`::ref:`opt<amdgpu_synid9_opt>`::ref:`u32<amdgpu_synid9_type_dev>`, :ref:`vaddr<amdgpu_synid9_addr_flat>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`::ref:`u32<amdgpu_synid9_type_dev>`                   :ref:`offset12<amdgpu_synid_flat_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    flat_atomic_umin_x2            :ref:`vdst<amdgpu_synid9_dst_flat_atomic64>`::ref:`opt<amdgpu_synid9_opt>`::ref:`u64<amdgpu_synid9_type_dev>`, :ref:`vaddr<amdgpu_synid9_addr_flat>`,    :ref:`vdata<amdgpu_synid9_vdata64_0>`::ref:`u64<amdgpu_synid9_type_dev>`                   :ref:`offset12<amdgpu_synid_flat_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    flat_atomic_xor                :ref:`vdst<amdgpu_synid9_dst_flat_atomic32>`::ref:`opt<amdgpu_synid9_opt>`,     :ref:`vaddr<amdgpu_synid9_addr_flat>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`                       :ref:`offset12<amdgpu_synid_flat_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    flat_atomic_xor_x2             :ref:`vdst<amdgpu_synid9_dst_flat_atomic64>`::ref:`opt<amdgpu_synid9_opt>`,     :ref:`vaddr<amdgpu_synid9_addr_flat>`,    :ref:`vdata<amdgpu_synid9_vdata64_0>`                       :ref:`offset12<amdgpu_synid_flat_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    flat_load_dword                :ref:`vdst<amdgpu_synid9_vdst32_0>`,         :ref:`vaddr<amdgpu_synid9_addr_flat>`                                 :ref:`offset12<amdgpu_synid_flat_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    flat_load_dwordx2              :ref:`vdst<amdgpu_synid9_vdst64_0>`,         :ref:`vaddr<amdgpu_synid9_addr_flat>`                                 :ref:`offset12<amdgpu_synid_flat_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    flat_load_dwordx3              :ref:`vdst<amdgpu_synid9_vdst96_0>`,         :ref:`vaddr<amdgpu_synid9_addr_flat>`                                 :ref:`offset12<amdgpu_synid_flat_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    flat_load_dwordx4              :ref:`vdst<amdgpu_synid9_vdst128_0>`,         :ref:`vaddr<amdgpu_synid9_addr_flat>`                                 :ref:`offset12<amdgpu_synid_flat_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    flat_load_sbyte                :ref:`vdst<amdgpu_synid9_vdst32_0>`,         :ref:`vaddr<amdgpu_synid9_addr_flat>`                                 :ref:`offset12<amdgpu_synid_flat_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    flat_load_sbyte_d16            :ref:`vdst<amdgpu_synid9_vdst32_0>`,         :ref:`vaddr<amdgpu_synid9_addr_flat>`                                 :ref:`offset12<amdgpu_synid_flat_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    flat_load_sbyte_d16_hi         :ref:`vdst<amdgpu_synid9_vdst32_0>`,         :ref:`vaddr<amdgpu_synid9_addr_flat>`                                 :ref:`offset12<amdgpu_synid_flat_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    flat_load_short_d16            :ref:`vdst<amdgpu_synid9_vdst32_0>`,         :ref:`vaddr<amdgpu_synid9_addr_flat>`                                 :ref:`offset12<amdgpu_synid_flat_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    flat_load_short_d16_hi         :ref:`vdst<amdgpu_synid9_vdst32_0>`,         :ref:`vaddr<amdgpu_synid9_addr_flat>`                                 :ref:`offset12<amdgpu_synid_flat_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    flat_load_sshort               :ref:`vdst<amdgpu_synid9_vdst32_0>`,         :ref:`vaddr<amdgpu_synid9_addr_flat>`                                 :ref:`offset12<amdgpu_synid_flat_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    flat_load_ubyte                :ref:`vdst<amdgpu_synid9_vdst32_0>`,         :ref:`vaddr<amdgpu_synid9_addr_flat>`                                 :ref:`offset12<amdgpu_synid_flat_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    flat_load_ubyte_d16            :ref:`vdst<amdgpu_synid9_vdst32_0>`,         :ref:`vaddr<amdgpu_synid9_addr_flat>`                                 :ref:`offset12<amdgpu_synid_flat_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    flat_load_ubyte_d16_hi         :ref:`vdst<amdgpu_synid9_vdst32_0>`,         :ref:`vaddr<amdgpu_synid9_addr_flat>`                                 :ref:`offset12<amdgpu_synid_flat_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    flat_load_ushort               :ref:`vdst<amdgpu_synid9_vdst32_0>`,         :ref:`vaddr<amdgpu_synid9_addr_flat>`                                 :ref:`offset12<amdgpu_synid_flat_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    flat_store_byte                              :ref:`vaddr<amdgpu_synid9_addr_flat>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`                       :ref:`offset12<amdgpu_synid_flat_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    flat_store_byte_d16_hi                       :ref:`vaddr<amdgpu_synid9_addr_flat>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`                       :ref:`offset12<amdgpu_synid_flat_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    flat_store_dword                             :ref:`vaddr<amdgpu_synid9_addr_flat>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`                       :ref:`offset12<amdgpu_synid_flat_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    flat_store_dwordx2                           :ref:`vaddr<amdgpu_synid9_addr_flat>`,    :ref:`vdata<amdgpu_synid9_vdata64_0>`                       :ref:`offset12<amdgpu_synid_flat_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    flat_store_dwordx3                           :ref:`vaddr<amdgpu_synid9_addr_flat>`,    :ref:`vdata<amdgpu_synid9_vdata96_0>`                       :ref:`offset12<amdgpu_synid_flat_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    flat_store_dwordx4                           :ref:`vaddr<amdgpu_synid9_addr_flat>`,    :ref:`vdata<amdgpu_synid9_vdata128_0>`                       :ref:`offset12<amdgpu_synid_flat_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    flat_store_short                             :ref:`vaddr<amdgpu_synid9_addr_flat>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`                       :ref:`offset12<amdgpu_synid_flat_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    flat_store_short_d16_hi                      :ref:`vaddr<amdgpu_synid9_addr_flat>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`                       :ref:`offset12<amdgpu_synid_flat_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    global_atomic_add              :ref:`vdst<amdgpu_synid9_dst_flat_atomic32>`::ref:`opt<amdgpu_synid9_opt>`,     :ref:`vaddr<amdgpu_synid9_vaddr_flat_global>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`,       :ref:`saddr<amdgpu_synid9_saddr_flat_global>`          :ref:`offset13s<amdgpu_synid_flat_offset13s>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    global_atomic_add_x2           :ref:`vdst<amdgpu_synid9_dst_flat_atomic64>`::ref:`opt<amdgpu_synid9_opt>`,     :ref:`vaddr<amdgpu_synid9_vaddr_flat_global>`,    :ref:`vdata<amdgpu_synid9_vdata64_0>`,       :ref:`saddr<amdgpu_synid9_saddr_flat_global>`          :ref:`offset13s<amdgpu_synid_flat_offset13s>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    global_atomic_and              :ref:`vdst<amdgpu_synid9_dst_flat_atomic32>`::ref:`opt<amdgpu_synid9_opt>`,     :ref:`vaddr<amdgpu_synid9_vaddr_flat_global>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`,       :ref:`saddr<amdgpu_synid9_saddr_flat_global>`          :ref:`offset13s<amdgpu_synid_flat_offset13s>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    global_atomic_and_x2           :ref:`vdst<amdgpu_synid9_dst_flat_atomic64>`::ref:`opt<amdgpu_synid9_opt>`,     :ref:`vaddr<amdgpu_synid9_vaddr_flat_global>`,    :ref:`vdata<amdgpu_synid9_vdata64_0>`,       :ref:`saddr<amdgpu_synid9_saddr_flat_global>`          :ref:`offset13s<amdgpu_synid_flat_offset13s>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    global_atomic_cmpswap          :ref:`vdst<amdgpu_synid9_dst_flat_atomic32>`::ref:`opt<amdgpu_synid9_opt>`,     :ref:`vaddr<amdgpu_synid9_vaddr_flat_global>`,    :ref:`vdata<amdgpu_synid9_vdata64_0>`::ref:`b32x2<amdgpu_synid9_type_dev>`, :ref:`saddr<amdgpu_synid9_saddr_flat_global>`          :ref:`offset13s<amdgpu_synid_flat_offset13s>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    global_atomic_cmpswap_x2       :ref:`vdst<amdgpu_synid9_dst_flat_atomic64>`::ref:`opt<amdgpu_synid9_opt>`,     :ref:`vaddr<amdgpu_synid9_vaddr_flat_global>`,    :ref:`vdata<amdgpu_synid9_vdata128_0>`::ref:`b64x2<amdgpu_synid9_type_dev>`, :ref:`saddr<amdgpu_synid9_saddr_flat_global>`          :ref:`offset13s<amdgpu_synid_flat_offset13s>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    global_atomic_dec              :ref:`vdst<amdgpu_synid9_dst_flat_atomic32>`::ref:`opt<amdgpu_synid9_opt>`::ref:`u32<amdgpu_synid9_type_dev>`, :ref:`vaddr<amdgpu_synid9_vaddr_flat_global>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`::ref:`u32<amdgpu_synid9_type_dev>`,   :ref:`saddr<amdgpu_synid9_saddr_flat_global>`          :ref:`offset13s<amdgpu_synid_flat_offset13s>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    global_atomic_dec_x2           :ref:`vdst<amdgpu_synid9_dst_flat_atomic64>`::ref:`opt<amdgpu_synid9_opt>`::ref:`u64<amdgpu_synid9_type_dev>`, :ref:`vaddr<amdgpu_synid9_vaddr_flat_global>`,    :ref:`vdata<amdgpu_synid9_vdata64_0>`::ref:`u64<amdgpu_synid9_type_dev>`,   :ref:`saddr<amdgpu_synid9_saddr_flat_global>`          :ref:`offset13s<amdgpu_synid_flat_offset13s>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    global_atomic_inc              :ref:`vdst<amdgpu_synid9_dst_flat_atomic32>`::ref:`opt<amdgpu_synid9_opt>`::ref:`u32<amdgpu_synid9_type_dev>`, :ref:`vaddr<amdgpu_synid9_vaddr_flat_global>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`::ref:`u32<amdgpu_synid9_type_dev>`,   :ref:`saddr<amdgpu_synid9_saddr_flat_global>`          :ref:`offset13s<amdgpu_synid_flat_offset13s>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    global_atomic_inc_x2           :ref:`vdst<amdgpu_synid9_dst_flat_atomic64>`::ref:`opt<amdgpu_synid9_opt>`::ref:`u64<amdgpu_synid9_type_dev>`, :ref:`vaddr<amdgpu_synid9_vaddr_flat_global>`,    :ref:`vdata<amdgpu_synid9_vdata64_0>`::ref:`u64<amdgpu_synid9_type_dev>`,   :ref:`saddr<amdgpu_synid9_saddr_flat_global>`          :ref:`offset13s<amdgpu_synid_flat_offset13s>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    global_atomic_or               :ref:`vdst<amdgpu_synid9_dst_flat_atomic32>`::ref:`opt<amdgpu_synid9_opt>`,     :ref:`vaddr<amdgpu_synid9_vaddr_flat_global>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`,       :ref:`saddr<amdgpu_synid9_saddr_flat_global>`          :ref:`offset13s<amdgpu_synid_flat_offset13s>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    global_atomic_or_x2            :ref:`vdst<amdgpu_synid9_dst_flat_atomic64>`::ref:`opt<amdgpu_synid9_opt>`,     :ref:`vaddr<amdgpu_synid9_vaddr_flat_global>`,    :ref:`vdata<amdgpu_synid9_vdata64_0>`,       :ref:`saddr<amdgpu_synid9_saddr_flat_global>`          :ref:`offset13s<amdgpu_synid_flat_offset13s>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    global_atomic_smax             :ref:`vdst<amdgpu_synid9_dst_flat_atomic32>`::ref:`opt<amdgpu_synid9_opt>`::ref:`s32<amdgpu_synid9_type_dev>`, :ref:`vaddr<amdgpu_synid9_vaddr_flat_global>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`::ref:`s32<amdgpu_synid9_type_dev>`,   :ref:`saddr<amdgpu_synid9_saddr_flat_global>`          :ref:`offset13s<amdgpu_synid_flat_offset13s>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    global_atomic_smax_x2          :ref:`vdst<amdgpu_synid9_dst_flat_atomic64>`::ref:`opt<amdgpu_synid9_opt>`::ref:`s64<amdgpu_synid9_type_dev>`, :ref:`vaddr<amdgpu_synid9_vaddr_flat_global>`,    :ref:`vdata<amdgpu_synid9_vdata64_0>`::ref:`s64<amdgpu_synid9_type_dev>`,   :ref:`saddr<amdgpu_synid9_saddr_flat_global>`          :ref:`offset13s<amdgpu_synid_flat_offset13s>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    global_atomic_smin             :ref:`vdst<amdgpu_synid9_dst_flat_atomic32>`::ref:`opt<amdgpu_synid9_opt>`::ref:`s32<amdgpu_synid9_type_dev>`, :ref:`vaddr<amdgpu_synid9_vaddr_flat_global>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`::ref:`s32<amdgpu_synid9_type_dev>`,   :ref:`saddr<amdgpu_synid9_saddr_flat_global>`          :ref:`offset13s<amdgpu_synid_flat_offset13s>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    global_atomic_smin_x2          :ref:`vdst<amdgpu_synid9_dst_flat_atomic64>`::ref:`opt<amdgpu_synid9_opt>`::ref:`s64<amdgpu_synid9_type_dev>`, :ref:`vaddr<amdgpu_synid9_vaddr_flat_global>`,    :ref:`vdata<amdgpu_synid9_vdata64_0>`::ref:`s64<amdgpu_synid9_type_dev>`,   :ref:`saddr<amdgpu_synid9_saddr_flat_global>`          :ref:`offset13s<amdgpu_synid_flat_offset13s>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    global_atomic_sub              :ref:`vdst<amdgpu_synid9_dst_flat_atomic32>`::ref:`opt<amdgpu_synid9_opt>`,     :ref:`vaddr<amdgpu_synid9_vaddr_flat_global>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`,       :ref:`saddr<amdgpu_synid9_saddr_flat_global>`          :ref:`offset13s<amdgpu_synid_flat_offset13s>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    global_atomic_sub_x2           :ref:`vdst<amdgpu_synid9_dst_flat_atomic64>`::ref:`opt<amdgpu_synid9_opt>`,     :ref:`vaddr<amdgpu_synid9_vaddr_flat_global>`,    :ref:`vdata<amdgpu_synid9_vdata64_0>`,       :ref:`saddr<amdgpu_synid9_saddr_flat_global>`          :ref:`offset13s<amdgpu_synid_flat_offset13s>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    global_atomic_swap             :ref:`vdst<amdgpu_synid9_dst_flat_atomic32>`::ref:`opt<amdgpu_synid9_opt>`,     :ref:`vaddr<amdgpu_synid9_vaddr_flat_global>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`,       :ref:`saddr<amdgpu_synid9_saddr_flat_global>`          :ref:`offset13s<amdgpu_synid_flat_offset13s>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    global_atomic_swap_x2          :ref:`vdst<amdgpu_synid9_dst_flat_atomic64>`::ref:`opt<amdgpu_synid9_opt>`,     :ref:`vaddr<amdgpu_synid9_vaddr_flat_global>`,    :ref:`vdata<amdgpu_synid9_vdata64_0>`,       :ref:`saddr<amdgpu_synid9_saddr_flat_global>`          :ref:`offset13s<amdgpu_synid_flat_offset13s>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    global_atomic_umax             :ref:`vdst<amdgpu_synid9_dst_flat_atomic32>`::ref:`opt<amdgpu_synid9_opt>`::ref:`u32<amdgpu_synid9_type_dev>`, :ref:`vaddr<amdgpu_synid9_vaddr_flat_global>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`::ref:`u32<amdgpu_synid9_type_dev>`,   :ref:`saddr<amdgpu_synid9_saddr_flat_global>`          :ref:`offset13s<amdgpu_synid_flat_offset13s>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    global_atomic_umax_x2          :ref:`vdst<amdgpu_synid9_dst_flat_atomic64>`::ref:`opt<amdgpu_synid9_opt>`::ref:`u64<amdgpu_synid9_type_dev>`, :ref:`vaddr<amdgpu_synid9_vaddr_flat_global>`,    :ref:`vdata<amdgpu_synid9_vdata64_0>`::ref:`u64<amdgpu_synid9_type_dev>`,   :ref:`saddr<amdgpu_synid9_saddr_flat_global>`          :ref:`offset13s<amdgpu_synid_flat_offset13s>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    global_atomic_umin             :ref:`vdst<amdgpu_synid9_dst_flat_atomic32>`::ref:`opt<amdgpu_synid9_opt>`::ref:`u32<amdgpu_synid9_type_dev>`, :ref:`vaddr<amdgpu_synid9_vaddr_flat_global>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`::ref:`u32<amdgpu_synid9_type_dev>`,   :ref:`saddr<amdgpu_synid9_saddr_flat_global>`          :ref:`offset13s<amdgpu_synid_flat_offset13s>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    global_atomic_umin_x2          :ref:`vdst<amdgpu_synid9_dst_flat_atomic64>`::ref:`opt<amdgpu_synid9_opt>`::ref:`u64<amdgpu_synid9_type_dev>`, :ref:`vaddr<amdgpu_synid9_vaddr_flat_global>`,    :ref:`vdata<amdgpu_synid9_vdata64_0>`::ref:`u64<amdgpu_synid9_type_dev>`,   :ref:`saddr<amdgpu_synid9_saddr_flat_global>`          :ref:`offset13s<amdgpu_synid_flat_offset13s>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    global_atomic_xor              :ref:`vdst<amdgpu_synid9_dst_flat_atomic32>`::ref:`opt<amdgpu_synid9_opt>`,     :ref:`vaddr<amdgpu_synid9_vaddr_flat_global>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`,       :ref:`saddr<amdgpu_synid9_saddr_flat_global>`          :ref:`offset13s<amdgpu_synid_flat_offset13s>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    global_atomic_xor_x2           :ref:`vdst<amdgpu_synid9_dst_flat_atomic64>`::ref:`opt<amdgpu_synid9_opt>`,     :ref:`vaddr<amdgpu_synid9_vaddr_flat_global>`,    :ref:`vdata<amdgpu_synid9_vdata64_0>`,       :ref:`saddr<amdgpu_synid9_saddr_flat_global>`          :ref:`offset13s<amdgpu_synid_flat_offset13s>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    global_load_dword              :ref:`vdst<amdgpu_synid9_vdst32_0>`,         :ref:`vaddr<amdgpu_synid9_vaddr_flat_global>`,    :ref:`saddr<amdgpu_synid9_saddr_flat_global>`                       :ref:`offset13s<amdgpu_synid_flat_offset13s>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    global_load_dwordx2            :ref:`vdst<amdgpu_synid9_vdst64_0>`,         :ref:`vaddr<amdgpu_synid9_vaddr_flat_global>`,    :ref:`saddr<amdgpu_synid9_saddr_flat_global>`                       :ref:`offset13s<amdgpu_synid_flat_offset13s>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    global_load_dwordx3            :ref:`vdst<amdgpu_synid9_vdst96_0>`,         :ref:`vaddr<amdgpu_synid9_vaddr_flat_global>`,    :ref:`saddr<amdgpu_synid9_saddr_flat_global>`                       :ref:`offset13s<amdgpu_synid_flat_offset13s>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    global_load_dwordx4            :ref:`vdst<amdgpu_synid9_vdst128_0>`,         :ref:`vaddr<amdgpu_synid9_vaddr_flat_global>`,    :ref:`saddr<amdgpu_synid9_saddr_flat_global>`                       :ref:`offset13s<amdgpu_synid_flat_offset13s>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    global_load_sbyte              :ref:`vdst<amdgpu_synid9_vdst32_0>`,         :ref:`vaddr<amdgpu_synid9_vaddr_flat_global>`,    :ref:`saddr<amdgpu_synid9_saddr_flat_global>`                       :ref:`offset13s<amdgpu_synid_flat_offset13s>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    global_load_sbyte_d16          :ref:`vdst<amdgpu_synid9_vdst32_0>`,         :ref:`vaddr<amdgpu_synid9_vaddr_flat_global>`,    :ref:`saddr<amdgpu_synid9_saddr_flat_global>`                       :ref:`offset13s<amdgpu_synid_flat_offset13s>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    global_load_sbyte_d16_hi       :ref:`vdst<amdgpu_synid9_vdst32_0>`,         :ref:`vaddr<amdgpu_synid9_vaddr_flat_global>`,    :ref:`saddr<amdgpu_synid9_saddr_flat_global>`                       :ref:`offset13s<amdgpu_synid_flat_offset13s>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    global_load_short_d16          :ref:`vdst<amdgpu_synid9_vdst32_0>`,         :ref:`vaddr<amdgpu_synid9_vaddr_flat_global>`,    :ref:`saddr<amdgpu_synid9_saddr_flat_global>`                       :ref:`offset13s<amdgpu_synid_flat_offset13s>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    global_load_short_d16_hi       :ref:`vdst<amdgpu_synid9_vdst32_0>`,         :ref:`vaddr<amdgpu_synid9_vaddr_flat_global>`,    :ref:`saddr<amdgpu_synid9_saddr_flat_global>`                       :ref:`offset13s<amdgpu_synid_flat_offset13s>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    global_load_sshort             :ref:`vdst<amdgpu_synid9_vdst32_0>`,         :ref:`vaddr<amdgpu_synid9_vaddr_flat_global>`,    :ref:`saddr<amdgpu_synid9_saddr_flat_global>`                       :ref:`offset13s<amdgpu_synid_flat_offset13s>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    global_load_ubyte              :ref:`vdst<amdgpu_synid9_vdst32_0>`,         :ref:`vaddr<amdgpu_synid9_vaddr_flat_global>`,    :ref:`saddr<amdgpu_synid9_saddr_flat_global>`                       :ref:`offset13s<amdgpu_synid_flat_offset13s>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    global_load_ubyte_d16          :ref:`vdst<amdgpu_synid9_vdst32_0>`,         :ref:`vaddr<amdgpu_synid9_vaddr_flat_global>`,    :ref:`saddr<amdgpu_synid9_saddr_flat_global>`                       :ref:`offset13s<amdgpu_synid_flat_offset13s>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    global_load_ubyte_d16_hi       :ref:`vdst<amdgpu_synid9_vdst32_0>`,         :ref:`vaddr<amdgpu_synid9_vaddr_flat_global>`,    :ref:`saddr<amdgpu_synid9_saddr_flat_global>`                       :ref:`offset13s<amdgpu_synid_flat_offset13s>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    global_load_ushort             :ref:`vdst<amdgpu_synid9_vdst32_0>`,         :ref:`vaddr<amdgpu_synid9_vaddr_flat_global>`,    :ref:`saddr<amdgpu_synid9_saddr_flat_global>`                       :ref:`offset13s<amdgpu_synid_flat_offset13s>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    global_store_byte                            :ref:`vaddr<amdgpu_synid9_vaddr_flat_global>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`,       :ref:`saddr<amdgpu_synid9_saddr_flat_global>`          :ref:`offset13s<amdgpu_synid_flat_offset13s>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    global_store_byte_d16_hi                     :ref:`vaddr<amdgpu_synid9_vaddr_flat_global>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`,       :ref:`saddr<amdgpu_synid9_saddr_flat_global>`          :ref:`offset13s<amdgpu_synid_flat_offset13s>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    global_store_dword                           :ref:`vaddr<amdgpu_synid9_vaddr_flat_global>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`,       :ref:`saddr<amdgpu_synid9_saddr_flat_global>`          :ref:`offset13s<amdgpu_synid_flat_offset13s>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    global_store_dwordx2                         :ref:`vaddr<amdgpu_synid9_vaddr_flat_global>`,    :ref:`vdata<amdgpu_synid9_vdata64_0>`,       :ref:`saddr<amdgpu_synid9_saddr_flat_global>`          :ref:`offset13s<amdgpu_synid_flat_offset13s>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    global_store_dwordx3                         :ref:`vaddr<amdgpu_synid9_vaddr_flat_global>`,    :ref:`vdata<amdgpu_synid9_vdata96_0>`,       :ref:`saddr<amdgpu_synid9_saddr_flat_global>`          :ref:`offset13s<amdgpu_synid_flat_offset13s>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    global_store_dwordx4                         :ref:`vaddr<amdgpu_synid9_vaddr_flat_global>`,    :ref:`vdata<amdgpu_synid9_vdata128_0>`,       :ref:`saddr<amdgpu_synid9_saddr_flat_global>`          :ref:`offset13s<amdgpu_synid_flat_offset13s>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    global_store_short                           :ref:`vaddr<amdgpu_synid9_vaddr_flat_global>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`,       :ref:`saddr<amdgpu_synid9_saddr_flat_global>`          :ref:`offset13s<amdgpu_synid_flat_offset13s>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    global_store_short_d16_hi                    :ref:`vaddr<amdgpu_synid9_vaddr_flat_global>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`,       :ref:`saddr<amdgpu_synid9_saddr_flat_global>`          :ref:`offset13s<amdgpu_synid_flat_offset13s>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    scratch_load_dword             :ref:`vdst<amdgpu_synid9_vdst32_0>`,         :ref:`vaddr<amdgpu_synid9_vaddr_flat_scratch>`,    :ref:`saddr<amdgpu_synid9_saddr_flat_scratch>`                       :ref:`offset13s<amdgpu_synid_flat_offset13s>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    scratch_load_dwordx2           :ref:`vdst<amdgpu_synid9_vdst64_0>`,         :ref:`vaddr<amdgpu_synid9_vaddr_flat_scratch>`,    :ref:`saddr<amdgpu_synid9_saddr_flat_scratch>`                       :ref:`offset13s<amdgpu_synid_flat_offset13s>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    scratch_load_dwordx3           :ref:`vdst<amdgpu_synid9_vdst96_0>`,         :ref:`vaddr<amdgpu_synid9_vaddr_flat_scratch>`,    :ref:`saddr<amdgpu_synid9_saddr_flat_scratch>`                       :ref:`offset13s<amdgpu_synid_flat_offset13s>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    scratch_load_dwordx4           :ref:`vdst<amdgpu_synid9_vdst128_0>`,         :ref:`vaddr<amdgpu_synid9_vaddr_flat_scratch>`,    :ref:`saddr<amdgpu_synid9_saddr_flat_scratch>`                       :ref:`offset13s<amdgpu_synid_flat_offset13s>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    scratch_load_sbyte             :ref:`vdst<amdgpu_synid9_vdst32_0>`,         :ref:`vaddr<amdgpu_synid9_vaddr_flat_scratch>`,    :ref:`saddr<amdgpu_synid9_saddr_flat_scratch>`                       :ref:`offset13s<amdgpu_synid_flat_offset13s>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    scratch_load_sbyte_d16         :ref:`vdst<amdgpu_synid9_vdst32_0>`,         :ref:`vaddr<amdgpu_synid9_vaddr_flat_scratch>`,    :ref:`saddr<amdgpu_synid9_saddr_flat_scratch>`                       :ref:`offset13s<amdgpu_synid_flat_offset13s>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    scratch_load_sbyte_d16_hi      :ref:`vdst<amdgpu_synid9_vdst32_0>`,         :ref:`vaddr<amdgpu_synid9_vaddr_flat_scratch>`,    :ref:`saddr<amdgpu_synid9_saddr_flat_scratch>`                       :ref:`offset13s<amdgpu_synid_flat_offset13s>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    scratch_load_short_d16         :ref:`vdst<amdgpu_synid9_vdst32_0>`,         :ref:`vaddr<amdgpu_synid9_vaddr_flat_scratch>`,    :ref:`saddr<amdgpu_synid9_saddr_flat_scratch>`                       :ref:`offset13s<amdgpu_synid_flat_offset13s>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    scratch_load_short_d16_hi      :ref:`vdst<amdgpu_synid9_vdst32_0>`,         :ref:`vaddr<amdgpu_synid9_vaddr_flat_scratch>`,    :ref:`saddr<amdgpu_synid9_saddr_flat_scratch>`                       :ref:`offset13s<amdgpu_synid_flat_offset13s>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    scratch_load_sshort            :ref:`vdst<amdgpu_synid9_vdst32_0>`,         :ref:`vaddr<amdgpu_synid9_vaddr_flat_scratch>`,    :ref:`saddr<amdgpu_synid9_saddr_flat_scratch>`                       :ref:`offset13s<amdgpu_synid_flat_offset13s>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    scratch_load_ubyte             :ref:`vdst<amdgpu_synid9_vdst32_0>`,         :ref:`vaddr<amdgpu_synid9_vaddr_flat_scratch>`,    :ref:`saddr<amdgpu_synid9_saddr_flat_scratch>`                       :ref:`offset13s<amdgpu_synid_flat_offset13s>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    scratch_load_ubyte_d16         :ref:`vdst<amdgpu_synid9_vdst32_0>`,         :ref:`vaddr<amdgpu_synid9_vaddr_flat_scratch>`,    :ref:`saddr<amdgpu_synid9_saddr_flat_scratch>`                       :ref:`offset13s<amdgpu_synid_flat_offset13s>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    scratch_load_ubyte_d16_hi      :ref:`vdst<amdgpu_synid9_vdst32_0>`,         :ref:`vaddr<amdgpu_synid9_vaddr_flat_scratch>`,    :ref:`saddr<amdgpu_synid9_saddr_flat_scratch>`                       :ref:`offset13s<amdgpu_synid_flat_offset13s>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    scratch_load_ushort            :ref:`vdst<amdgpu_synid9_vdst32_0>`,         :ref:`vaddr<amdgpu_synid9_vaddr_flat_scratch>`,    :ref:`saddr<amdgpu_synid9_saddr_flat_scratch>`                       :ref:`offset13s<amdgpu_synid_flat_offset13s>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    scratch_store_byte                           :ref:`vaddr<amdgpu_synid9_vaddr_flat_scratch>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`,       :ref:`saddr<amdgpu_synid9_saddr_flat_scratch>`          :ref:`offset13s<amdgpu_synid_flat_offset13s>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    scratch_store_byte_d16_hi                    :ref:`vaddr<amdgpu_synid9_vaddr_flat_scratch>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`,       :ref:`saddr<amdgpu_synid9_saddr_flat_scratch>`          :ref:`offset13s<amdgpu_synid_flat_offset13s>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    scratch_store_dword                          :ref:`vaddr<amdgpu_synid9_vaddr_flat_scratch>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`,       :ref:`saddr<amdgpu_synid9_saddr_flat_scratch>`          :ref:`offset13s<amdgpu_synid_flat_offset13s>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    scratch_store_dwordx2                        :ref:`vaddr<amdgpu_synid9_vaddr_flat_scratch>`,    :ref:`vdata<amdgpu_synid9_vdata64_0>`,       :ref:`saddr<amdgpu_synid9_saddr_flat_scratch>`          :ref:`offset13s<amdgpu_synid_flat_offset13s>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    scratch_store_dwordx3                        :ref:`vaddr<amdgpu_synid9_vaddr_flat_scratch>`,    :ref:`vdata<amdgpu_synid9_vdata96_0>`,       :ref:`saddr<amdgpu_synid9_saddr_flat_scratch>`          :ref:`offset13s<amdgpu_synid_flat_offset13s>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    scratch_store_dwordx4                        :ref:`vaddr<amdgpu_synid9_vaddr_flat_scratch>`,    :ref:`vdata<amdgpu_synid9_vdata128_0>`,       :ref:`saddr<amdgpu_synid9_saddr_flat_scratch>`          :ref:`offset13s<amdgpu_synid_flat_offset13s>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    scratch_store_short                          :ref:`vaddr<amdgpu_synid9_vaddr_flat_scratch>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`,       :ref:`saddr<amdgpu_synid9_saddr_flat_scratch>`          :ref:`offset13s<amdgpu_synid_flat_offset13s>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    scratch_store_short_d16_hi                   :ref:`vaddr<amdgpu_synid9_vaddr_flat_scratch>`,    :ref:`vdata<amdgpu_synid9_vdata32_0>`,       :ref:`saddr<amdgpu_synid9_saddr_flat_scratch>`          :ref:`offset13s<amdgpu_synid_flat_offset13s>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
 
 MIMG
 -----------------------
 
 .. parsed-literal::
 
-    **INSTRUCTION**                    **DST**       **SRC0**       **SRC1**      **SRC2**           **MODIFIERS**
-    \ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|
-    image_atomic_add                         :ref:`vdata<amdgpu_synid9_data_mimg_atomic_reg>`::ref:`dst<amdgpu_synid9_ret>`, :ref:`vaddr<amdgpu_synid9_addr_mimg>`,    :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`          :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>`
-    image_atomic_and                         :ref:`vdata<amdgpu_synid9_data_mimg_atomic_reg>`::ref:`dst<amdgpu_synid9_ret>`, :ref:`vaddr<amdgpu_synid9_addr_mimg>`,    :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`          :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>`
-    image_atomic_cmpswap                     :ref:`vdata<amdgpu_synid9_data_mimg_atomic_cmp>`::ref:`dst<amdgpu_synid9_ret>`, :ref:`vaddr<amdgpu_synid9_addr_mimg>`,    :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`          :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>`
-    image_atomic_dec                         :ref:`vdata<amdgpu_synid9_data_mimg_atomic_reg>`::ref:`dst<amdgpu_synid9_ret>`, :ref:`vaddr<amdgpu_synid9_addr_mimg>`,    :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`          :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>`
-    image_atomic_inc                         :ref:`vdata<amdgpu_synid9_data_mimg_atomic_reg>`::ref:`dst<amdgpu_synid9_ret>`, :ref:`vaddr<amdgpu_synid9_addr_mimg>`,    :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`          :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>`
-    image_atomic_or                          :ref:`vdata<amdgpu_synid9_data_mimg_atomic_reg>`::ref:`dst<amdgpu_synid9_ret>`, :ref:`vaddr<amdgpu_synid9_addr_mimg>`,    :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`          :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>`
-    image_atomic_smax                        :ref:`vdata<amdgpu_synid9_data_mimg_atomic_reg>`::ref:`dst<amdgpu_synid9_ret>`, :ref:`vaddr<amdgpu_synid9_addr_mimg>`,    :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`          :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>`
-    image_atomic_smin                        :ref:`vdata<amdgpu_synid9_data_mimg_atomic_reg>`::ref:`dst<amdgpu_synid9_ret>`, :ref:`vaddr<amdgpu_synid9_addr_mimg>`,    :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`          :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>`
-    image_atomic_sub                         :ref:`vdata<amdgpu_synid9_data_mimg_atomic_reg>`::ref:`dst<amdgpu_synid9_ret>`, :ref:`vaddr<amdgpu_synid9_addr_mimg>`,    :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`          :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>`
-    image_atomic_swap                        :ref:`vdata<amdgpu_synid9_data_mimg_atomic_reg>`::ref:`dst<amdgpu_synid9_ret>`, :ref:`vaddr<amdgpu_synid9_addr_mimg>`,    :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`          :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>`
-    image_atomic_umax                        :ref:`vdata<amdgpu_synid9_data_mimg_atomic_reg>`::ref:`dst<amdgpu_synid9_ret>`, :ref:`vaddr<amdgpu_synid9_addr_mimg>`,    :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`          :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>`
-    image_atomic_umin                        :ref:`vdata<amdgpu_synid9_data_mimg_atomic_reg>`::ref:`dst<amdgpu_synid9_ret>`, :ref:`vaddr<amdgpu_synid9_addr_mimg>`,    :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`          :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>`
-    image_atomic_xor                         :ref:`vdata<amdgpu_synid9_data_mimg_atomic_reg>`::ref:`dst<amdgpu_synid9_ret>`, :ref:`vaddr<amdgpu_synid9_addr_mimg>`,    :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`          :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>`
-    image_gather4                  :ref:`vdst<amdgpu_synid9_dst_mimg_gather4>`,     :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,    :ref:`ssamp<amdgpu_synid9_samp_mimg>`          :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
-    image_gather4_b                :ref:`vdst<amdgpu_synid9_dst_mimg_gather4>`,     :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,    :ref:`ssamp<amdgpu_synid9_samp_mimg>`          :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
-    image_gather4_b_cl             :ref:`vdst<amdgpu_synid9_dst_mimg_gather4>`,     :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,    :ref:`ssamp<amdgpu_synid9_samp_mimg>`          :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
-    image_gather4_b_cl_o           :ref:`vdst<amdgpu_synid9_dst_mimg_gather4>`,     :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,    :ref:`ssamp<amdgpu_synid9_samp_mimg>`          :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
-    image_gather4_b_o              :ref:`vdst<amdgpu_synid9_dst_mimg_gather4>`,     :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,    :ref:`ssamp<amdgpu_synid9_samp_mimg>`          :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
-    image_gather4_c                :ref:`vdst<amdgpu_synid9_dst_mimg_gather4>`,     :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,    :ref:`ssamp<amdgpu_synid9_samp_mimg>`          :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
-    image_gather4_c_b              :ref:`vdst<amdgpu_synid9_dst_mimg_gather4>`,     :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,    :ref:`ssamp<amdgpu_synid9_samp_mimg>`          :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
-    image_gather4_c_b_cl           :ref:`vdst<amdgpu_synid9_dst_mimg_gather4>`,     :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,    :ref:`ssamp<amdgpu_synid9_samp_mimg>`          :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
-    image_gather4_c_b_cl_o         :ref:`vdst<amdgpu_synid9_dst_mimg_gather4>`,     :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,    :ref:`ssamp<amdgpu_synid9_samp_mimg>`          :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
-    image_gather4_c_b_o            :ref:`vdst<amdgpu_synid9_dst_mimg_gather4>`,     :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,    :ref:`ssamp<amdgpu_synid9_samp_mimg>`          :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
-    image_gather4_c_cl             :ref:`vdst<amdgpu_synid9_dst_mimg_gather4>`,     :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,    :ref:`ssamp<amdgpu_synid9_samp_mimg>`          :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
-    image_gather4_c_cl_o           :ref:`vdst<amdgpu_synid9_dst_mimg_gather4>`,     :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,    :ref:`ssamp<amdgpu_synid9_samp_mimg>`          :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
-    image_gather4_c_l              :ref:`vdst<amdgpu_synid9_dst_mimg_gather4>`,     :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,    :ref:`ssamp<amdgpu_synid9_samp_mimg>`          :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
-    image_gather4_c_l_o            :ref:`vdst<amdgpu_synid9_dst_mimg_gather4>`,     :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,    :ref:`ssamp<amdgpu_synid9_samp_mimg>`          :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
-    image_gather4_c_lz             :ref:`vdst<amdgpu_synid9_dst_mimg_gather4>`,     :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,    :ref:`ssamp<amdgpu_synid9_samp_mimg>`          :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
-    image_gather4_c_lz_o           :ref:`vdst<amdgpu_synid9_dst_mimg_gather4>`,     :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,    :ref:`ssamp<amdgpu_synid9_samp_mimg>`          :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
-    image_gather4_c_o              :ref:`vdst<amdgpu_synid9_dst_mimg_gather4>`,     :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,    :ref:`ssamp<amdgpu_synid9_samp_mimg>`          :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
-    image_gather4_cl               :ref:`vdst<amdgpu_synid9_dst_mimg_gather4>`,     :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,    :ref:`ssamp<amdgpu_synid9_samp_mimg>`          :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
-    image_gather4_cl_o             :ref:`vdst<amdgpu_synid9_dst_mimg_gather4>`,     :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,    :ref:`ssamp<amdgpu_synid9_samp_mimg>`          :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
-    image_gather4_l                :ref:`vdst<amdgpu_synid9_dst_mimg_gather4>`,     :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,    :ref:`ssamp<amdgpu_synid9_samp_mimg>`          :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
-    image_gather4_l_o              :ref:`vdst<amdgpu_synid9_dst_mimg_gather4>`,     :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,    :ref:`ssamp<amdgpu_synid9_samp_mimg>`          :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
-    image_gather4_lz               :ref:`vdst<amdgpu_synid9_dst_mimg_gather4>`,     :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,    :ref:`ssamp<amdgpu_synid9_samp_mimg>`          :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
-    image_gather4_lz_o             :ref:`vdst<amdgpu_synid9_dst_mimg_gather4>`,     :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,    :ref:`ssamp<amdgpu_synid9_samp_mimg>`          :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
-    image_gather4_o                :ref:`vdst<amdgpu_synid9_dst_mimg_gather4>`,     :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,    :ref:`ssamp<amdgpu_synid9_samp_mimg>`          :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
-    image_get_lod                  :ref:`vdst<amdgpu_synid9_dst_mimg_regular>`,     :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,    :ref:`ssamp<amdgpu_synid9_samp_mimg>`          :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>`
-    image_get_resinfo              :ref:`vdst<amdgpu_synid9_dst_mimg_regular>`,     :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`                    :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>`
-    image_load                     :ref:`vdst<amdgpu_synid9_dst_mimg_regular_d16>`,     :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`                    :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
-    image_load_mip                 :ref:`vdst<amdgpu_synid9_dst_mimg_regular_d16>`,     :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`                    :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
-    image_load_mip_pck             :ref:`vdst<amdgpu_synid9_dst_mimg_regular>`,     :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`                    :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>`
-    image_load_mip_pck_sgn         :ref:`vdst<amdgpu_synid9_dst_mimg_regular>`,     :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`                    :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>`
-    image_load_pck                 :ref:`vdst<amdgpu_synid9_dst_mimg_regular>`,     :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`                    :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>`
-    image_load_pck_sgn             :ref:`vdst<amdgpu_synid9_dst_mimg_regular>`,     :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`                    :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>`
-    image_sample                   :ref:`vdst<amdgpu_synid9_dst_mimg_regular_d16>`,     :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,    :ref:`ssamp<amdgpu_synid9_samp_mimg>`          :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
-    image_sample_b                 :ref:`vdst<amdgpu_synid9_dst_mimg_regular_d16>`,     :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,    :ref:`ssamp<amdgpu_synid9_samp_mimg>`          :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
-    image_sample_b_cl              :ref:`vdst<amdgpu_synid9_dst_mimg_regular_d16>`,     :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,    :ref:`ssamp<amdgpu_synid9_samp_mimg>`          :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
-    image_sample_b_cl_o            :ref:`vdst<amdgpu_synid9_dst_mimg_regular_d16>`,     :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,    :ref:`ssamp<amdgpu_synid9_samp_mimg>`          :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
-    image_sample_b_o               :ref:`vdst<amdgpu_synid9_dst_mimg_regular_d16>`,     :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,    :ref:`ssamp<amdgpu_synid9_samp_mimg>`          :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
-    image_sample_c                 :ref:`vdst<amdgpu_synid9_dst_mimg_regular_d16>`,     :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,    :ref:`ssamp<amdgpu_synid9_samp_mimg>`          :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
-    image_sample_c_b               :ref:`vdst<amdgpu_synid9_dst_mimg_regular_d16>`,     :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,    :ref:`ssamp<amdgpu_synid9_samp_mimg>`          :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
-    image_sample_c_b_cl            :ref:`vdst<amdgpu_synid9_dst_mimg_regular_d16>`,     :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,    :ref:`ssamp<amdgpu_synid9_samp_mimg>`          :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
-    image_sample_c_b_cl_o          :ref:`vdst<amdgpu_synid9_dst_mimg_regular_d16>`,     :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,    :ref:`ssamp<amdgpu_synid9_samp_mimg>`          :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
-    image_sample_c_b_o             :ref:`vdst<amdgpu_synid9_dst_mimg_regular_d16>`,     :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,    :ref:`ssamp<amdgpu_synid9_samp_mimg>`          :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
-    image_sample_c_cd              :ref:`vdst<amdgpu_synid9_dst_mimg_regular_d16>`,     :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,    :ref:`ssamp<amdgpu_synid9_samp_mimg>`          :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
-    image_sample_c_cd_cl           :ref:`vdst<amdgpu_synid9_dst_mimg_regular_d16>`,     :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,    :ref:`ssamp<amdgpu_synid9_samp_mimg>`          :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
-    image_sample_c_cd_cl_o         :ref:`vdst<amdgpu_synid9_dst_mimg_regular_d16>`,     :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,    :ref:`ssamp<amdgpu_synid9_samp_mimg>`          :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
-    image_sample_c_cd_o            :ref:`vdst<amdgpu_synid9_dst_mimg_regular_d16>`,     :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,    :ref:`ssamp<amdgpu_synid9_samp_mimg>`          :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
-    image_sample_c_cl              :ref:`vdst<amdgpu_synid9_dst_mimg_regular_d16>`,     :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,    :ref:`ssamp<amdgpu_synid9_samp_mimg>`          :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
-    image_sample_c_cl_o            :ref:`vdst<amdgpu_synid9_dst_mimg_regular_d16>`,     :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,    :ref:`ssamp<amdgpu_synid9_samp_mimg>`          :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
-    image_sample_c_d               :ref:`vdst<amdgpu_synid9_dst_mimg_regular_d16>`,     :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,    :ref:`ssamp<amdgpu_synid9_samp_mimg>`          :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
-    image_sample_c_d_cl            :ref:`vdst<amdgpu_synid9_dst_mimg_regular_d16>`,     :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,    :ref:`ssamp<amdgpu_synid9_samp_mimg>`          :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
-    image_sample_c_d_cl_o          :ref:`vdst<amdgpu_synid9_dst_mimg_regular_d16>`,     :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,    :ref:`ssamp<amdgpu_synid9_samp_mimg>`          :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
-    image_sample_c_d_o             :ref:`vdst<amdgpu_synid9_dst_mimg_regular_d16>`,     :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,    :ref:`ssamp<amdgpu_synid9_samp_mimg>`          :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
-    image_sample_c_l               :ref:`vdst<amdgpu_synid9_dst_mimg_regular_d16>`,     :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,    :ref:`ssamp<amdgpu_synid9_samp_mimg>`          :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
-    image_sample_c_l_o             :ref:`vdst<amdgpu_synid9_dst_mimg_regular_d16>`,     :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,    :ref:`ssamp<amdgpu_synid9_samp_mimg>`          :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
-    image_sample_c_lz              :ref:`vdst<amdgpu_synid9_dst_mimg_regular_d16>`,     :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,    :ref:`ssamp<amdgpu_synid9_samp_mimg>`          :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
-    image_sample_c_lz_o            :ref:`vdst<amdgpu_synid9_dst_mimg_regular_d16>`,     :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,    :ref:`ssamp<amdgpu_synid9_samp_mimg>`          :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
-    image_sample_c_o               :ref:`vdst<amdgpu_synid9_dst_mimg_regular_d16>`,     :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,    :ref:`ssamp<amdgpu_synid9_samp_mimg>`          :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
-    image_sample_cd                :ref:`vdst<amdgpu_synid9_dst_mimg_regular_d16>`,     :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,    :ref:`ssamp<amdgpu_synid9_samp_mimg>`          :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
-    image_sample_cd_cl             :ref:`vdst<amdgpu_synid9_dst_mimg_regular_d16>`,     :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,    :ref:`ssamp<amdgpu_synid9_samp_mimg>`          :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
-    image_sample_cd_cl_o           :ref:`vdst<amdgpu_synid9_dst_mimg_regular_d16>`,     :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,    :ref:`ssamp<amdgpu_synid9_samp_mimg>`          :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
-    image_sample_cd_o              :ref:`vdst<amdgpu_synid9_dst_mimg_regular_d16>`,     :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,    :ref:`ssamp<amdgpu_synid9_samp_mimg>`          :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
-    image_sample_cl                :ref:`vdst<amdgpu_synid9_dst_mimg_regular_d16>`,     :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,    :ref:`ssamp<amdgpu_synid9_samp_mimg>`          :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
-    image_sample_cl_o              :ref:`vdst<amdgpu_synid9_dst_mimg_regular_d16>`,     :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,    :ref:`ssamp<amdgpu_synid9_samp_mimg>`          :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
-    image_sample_d                 :ref:`vdst<amdgpu_synid9_dst_mimg_regular_d16>`,     :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,    :ref:`ssamp<amdgpu_synid9_samp_mimg>`          :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
-    image_sample_d_cl              :ref:`vdst<amdgpu_synid9_dst_mimg_regular_d16>`,     :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,    :ref:`ssamp<amdgpu_synid9_samp_mimg>`          :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
-    image_sample_d_cl_o            :ref:`vdst<amdgpu_synid9_dst_mimg_regular_d16>`,     :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,    :ref:`ssamp<amdgpu_synid9_samp_mimg>`          :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
-    image_sample_d_o               :ref:`vdst<amdgpu_synid9_dst_mimg_regular_d16>`,     :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,    :ref:`ssamp<amdgpu_synid9_samp_mimg>`          :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
-    image_sample_l                 :ref:`vdst<amdgpu_synid9_dst_mimg_regular_d16>`,     :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,    :ref:`ssamp<amdgpu_synid9_samp_mimg>`          :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
-    image_sample_l_o               :ref:`vdst<amdgpu_synid9_dst_mimg_regular_d16>`,     :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,    :ref:`ssamp<amdgpu_synid9_samp_mimg>`          :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
-    image_sample_lz                :ref:`vdst<amdgpu_synid9_dst_mimg_regular_d16>`,     :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,    :ref:`ssamp<amdgpu_synid9_samp_mimg>`          :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
-    image_sample_lz_o              :ref:`vdst<amdgpu_synid9_dst_mimg_regular_d16>`,     :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,    :ref:`ssamp<amdgpu_synid9_samp_mimg>`          :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
-    image_sample_o                 :ref:`vdst<amdgpu_synid9_dst_mimg_regular_d16>`,     :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,    :ref:`ssamp<amdgpu_synid9_samp_mimg>`          :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
-    image_store                              :ref:`vdata<amdgpu_synid9_data_mimg_store_d16>`,     :ref:`vaddr<amdgpu_synid9_addr_mimg>`,    :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`          :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
-    image_store_mip                          :ref:`vdata<amdgpu_synid9_data_mimg_store_d16>`,     :ref:`vaddr<amdgpu_synid9_addr_mimg>`,    :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`          :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
-    image_store_mip_pck                      :ref:`vdata<amdgpu_synid9_data_mimg_store>`,     :ref:`vaddr<amdgpu_synid9_addr_mimg>`,    :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`          :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>`
-    image_store_pck                          :ref:`vdata<amdgpu_synid9_data_mimg_store>`,     :ref:`vaddr<amdgpu_synid9_addr_mimg>`,    :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`          :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>`
+    **INSTRUCTION**            **DST**      **SRC0**       **SRC1**     **SRC2**       **MODIFIERS**
+    \ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|
+    image_atomic_add                :ref:`vdata<amdgpu_synid9_data_mimg_atomic_reg>`::ref:`dst<amdgpu_synid9_ret>`, :ref:`vaddr<amdgpu_synid9_addr_mimg>`,   :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`      :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>`
+    image_atomic_and                :ref:`vdata<amdgpu_synid9_data_mimg_atomic_reg>`::ref:`dst<amdgpu_synid9_ret>`, :ref:`vaddr<amdgpu_synid9_addr_mimg>`,   :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`      :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>`
+    image_atomic_cmpswap            :ref:`vdata<amdgpu_synid9_data_mimg_atomic_cmp>`::ref:`dst<amdgpu_synid9_ret>`, :ref:`vaddr<amdgpu_synid9_addr_mimg>`,   :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`      :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>`
+    image_atomic_dec                :ref:`vdata<amdgpu_synid9_data_mimg_atomic_reg>`::ref:`dst<amdgpu_synid9_ret>`, :ref:`vaddr<amdgpu_synid9_addr_mimg>`,   :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`      :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>`
+    image_atomic_inc                :ref:`vdata<amdgpu_synid9_data_mimg_atomic_reg>`::ref:`dst<amdgpu_synid9_ret>`, :ref:`vaddr<amdgpu_synid9_addr_mimg>`,   :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`      :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>`
+    image_atomic_or                 :ref:`vdata<amdgpu_synid9_data_mimg_atomic_reg>`::ref:`dst<amdgpu_synid9_ret>`, :ref:`vaddr<amdgpu_synid9_addr_mimg>`,   :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`      :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>`
+    image_atomic_smax               :ref:`vdata<amdgpu_synid9_data_mimg_atomic_reg>`::ref:`dst<amdgpu_synid9_ret>`, :ref:`vaddr<amdgpu_synid9_addr_mimg>`,   :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`      :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>`
+    image_atomic_smin               :ref:`vdata<amdgpu_synid9_data_mimg_atomic_reg>`::ref:`dst<amdgpu_synid9_ret>`, :ref:`vaddr<amdgpu_synid9_addr_mimg>`,   :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`      :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>`
+    image_atomic_sub                :ref:`vdata<amdgpu_synid9_data_mimg_atomic_reg>`::ref:`dst<amdgpu_synid9_ret>`, :ref:`vaddr<amdgpu_synid9_addr_mimg>`,   :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`      :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>`
+    image_atomic_swap               :ref:`vdata<amdgpu_synid9_data_mimg_atomic_reg>`::ref:`dst<amdgpu_synid9_ret>`, :ref:`vaddr<amdgpu_synid9_addr_mimg>`,   :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`      :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>`
+    image_atomic_umax               :ref:`vdata<amdgpu_synid9_data_mimg_atomic_reg>`::ref:`dst<amdgpu_synid9_ret>`, :ref:`vaddr<amdgpu_synid9_addr_mimg>`,   :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`      :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>`
+    image_atomic_umin               :ref:`vdata<amdgpu_synid9_data_mimg_atomic_reg>`::ref:`dst<amdgpu_synid9_ret>`, :ref:`vaddr<amdgpu_synid9_addr_mimg>`,   :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`      :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>`
+    image_atomic_xor                :ref:`vdata<amdgpu_synid9_data_mimg_atomic_reg>`::ref:`dst<amdgpu_synid9_ret>`, :ref:`vaddr<amdgpu_synid9_addr_mimg>`,   :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`      :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>`
+    image_gather4          :ref:`vdst<amdgpu_synid9_dst_mimg_gather4>`,    :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,   :ref:`ssamp<amdgpu_synid9_samp_mimg>`      :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
+    image_gather4_b        :ref:`vdst<amdgpu_synid9_dst_mimg_gather4>`,    :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,   :ref:`ssamp<amdgpu_synid9_samp_mimg>`      :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
+    image_gather4_b_cl     :ref:`vdst<amdgpu_synid9_dst_mimg_gather4>`,    :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,   :ref:`ssamp<amdgpu_synid9_samp_mimg>`      :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
+    image_gather4_b_cl_o   :ref:`vdst<amdgpu_synid9_dst_mimg_gather4>`,    :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,   :ref:`ssamp<amdgpu_synid9_samp_mimg>`      :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
+    image_gather4_b_o      :ref:`vdst<amdgpu_synid9_dst_mimg_gather4>`,    :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,   :ref:`ssamp<amdgpu_synid9_samp_mimg>`      :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
+    image_gather4_c        :ref:`vdst<amdgpu_synid9_dst_mimg_gather4>`,    :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,   :ref:`ssamp<amdgpu_synid9_samp_mimg>`      :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
+    image_gather4_c_b      :ref:`vdst<amdgpu_synid9_dst_mimg_gather4>`,    :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,   :ref:`ssamp<amdgpu_synid9_samp_mimg>`      :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
+    image_gather4_c_b_cl   :ref:`vdst<amdgpu_synid9_dst_mimg_gather4>`,    :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,   :ref:`ssamp<amdgpu_synid9_samp_mimg>`      :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
+    image_gather4_c_b_cl_o :ref:`vdst<amdgpu_synid9_dst_mimg_gather4>`,    :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,   :ref:`ssamp<amdgpu_synid9_samp_mimg>`      :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
+    image_gather4_c_b_o    :ref:`vdst<amdgpu_synid9_dst_mimg_gather4>`,    :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,   :ref:`ssamp<amdgpu_synid9_samp_mimg>`      :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
+    image_gather4_c_cl     :ref:`vdst<amdgpu_synid9_dst_mimg_gather4>`,    :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,   :ref:`ssamp<amdgpu_synid9_samp_mimg>`      :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
+    image_gather4_c_cl_o   :ref:`vdst<amdgpu_synid9_dst_mimg_gather4>`,    :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,   :ref:`ssamp<amdgpu_synid9_samp_mimg>`      :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
+    image_gather4_c_l      :ref:`vdst<amdgpu_synid9_dst_mimg_gather4>`,    :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,   :ref:`ssamp<amdgpu_synid9_samp_mimg>`      :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
+    image_gather4_c_l_o    :ref:`vdst<amdgpu_synid9_dst_mimg_gather4>`,    :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,   :ref:`ssamp<amdgpu_synid9_samp_mimg>`      :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
+    image_gather4_c_lz     :ref:`vdst<amdgpu_synid9_dst_mimg_gather4>`,    :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,   :ref:`ssamp<amdgpu_synid9_samp_mimg>`      :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
+    image_gather4_c_lz_o   :ref:`vdst<amdgpu_synid9_dst_mimg_gather4>`,    :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,   :ref:`ssamp<amdgpu_synid9_samp_mimg>`      :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
+    image_gather4_c_o      :ref:`vdst<amdgpu_synid9_dst_mimg_gather4>`,    :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,   :ref:`ssamp<amdgpu_synid9_samp_mimg>`      :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
+    image_gather4_cl       :ref:`vdst<amdgpu_synid9_dst_mimg_gather4>`,    :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,   :ref:`ssamp<amdgpu_synid9_samp_mimg>`      :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
+    image_gather4_cl_o     :ref:`vdst<amdgpu_synid9_dst_mimg_gather4>`,    :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,   :ref:`ssamp<amdgpu_synid9_samp_mimg>`      :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
+    image_gather4_l        :ref:`vdst<amdgpu_synid9_dst_mimg_gather4>`,    :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,   :ref:`ssamp<amdgpu_synid9_samp_mimg>`      :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
+    image_gather4_l_o      :ref:`vdst<amdgpu_synid9_dst_mimg_gather4>`,    :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,   :ref:`ssamp<amdgpu_synid9_samp_mimg>`      :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
+    image_gather4_lz       :ref:`vdst<amdgpu_synid9_dst_mimg_gather4>`,    :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,   :ref:`ssamp<amdgpu_synid9_samp_mimg>`      :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
+    image_gather4_lz_o     :ref:`vdst<amdgpu_synid9_dst_mimg_gather4>`,    :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,   :ref:`ssamp<amdgpu_synid9_samp_mimg>`      :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
+    image_gather4_o        :ref:`vdst<amdgpu_synid9_dst_mimg_gather4>`,    :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,   :ref:`ssamp<amdgpu_synid9_samp_mimg>`      :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
+    image_get_lod          :ref:`vdst<amdgpu_synid9_dst_mimg_regular>`,    :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,   :ref:`ssamp<amdgpu_synid9_samp_mimg>`      :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>`
+    image_get_resinfo      :ref:`vdst<amdgpu_synid9_dst_mimg_regular>`,    :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`               :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>`
+    image_load             :ref:`vdst<amdgpu_synid9_dst_mimg_regular_d16>`,    :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`               :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
+    image_load_mip         :ref:`vdst<amdgpu_synid9_dst_mimg_regular_d16>`,    :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`               :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
+    image_load_mip_pck     :ref:`vdst<amdgpu_synid9_dst_mimg_regular>`,    :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`               :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>`
+    image_load_mip_pck_sgn :ref:`vdst<amdgpu_synid9_dst_mimg_regular>`,    :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`               :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>`
+    image_load_pck         :ref:`vdst<amdgpu_synid9_dst_mimg_regular>`,    :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`               :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>`
+    image_load_pck_sgn     :ref:`vdst<amdgpu_synid9_dst_mimg_regular>`,    :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`               :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>`
+    image_sample           :ref:`vdst<amdgpu_synid9_dst_mimg_regular_d16>`,    :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,   :ref:`ssamp<amdgpu_synid9_samp_mimg>`      :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
+    image_sample_b         :ref:`vdst<amdgpu_synid9_dst_mimg_regular_d16>`,    :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,   :ref:`ssamp<amdgpu_synid9_samp_mimg>`      :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
+    image_sample_b_cl      :ref:`vdst<amdgpu_synid9_dst_mimg_regular_d16>`,    :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,   :ref:`ssamp<amdgpu_synid9_samp_mimg>`      :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
+    image_sample_b_cl_o    :ref:`vdst<amdgpu_synid9_dst_mimg_regular_d16>`,    :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,   :ref:`ssamp<amdgpu_synid9_samp_mimg>`      :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
+    image_sample_b_o       :ref:`vdst<amdgpu_synid9_dst_mimg_regular_d16>`,    :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,   :ref:`ssamp<amdgpu_synid9_samp_mimg>`      :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
+    image_sample_c         :ref:`vdst<amdgpu_synid9_dst_mimg_regular_d16>`,    :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,   :ref:`ssamp<amdgpu_synid9_samp_mimg>`      :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
+    image_sample_c_b       :ref:`vdst<amdgpu_synid9_dst_mimg_regular_d16>`,    :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,   :ref:`ssamp<amdgpu_synid9_samp_mimg>`      :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
+    image_sample_c_b_cl    :ref:`vdst<amdgpu_synid9_dst_mimg_regular_d16>`,    :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,   :ref:`ssamp<amdgpu_synid9_samp_mimg>`      :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
+    image_sample_c_b_cl_o  :ref:`vdst<amdgpu_synid9_dst_mimg_regular_d16>`,    :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,   :ref:`ssamp<amdgpu_synid9_samp_mimg>`      :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
+    image_sample_c_b_o     :ref:`vdst<amdgpu_synid9_dst_mimg_regular_d16>`,    :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,   :ref:`ssamp<amdgpu_synid9_samp_mimg>`      :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
+    image_sample_c_cd      :ref:`vdst<amdgpu_synid9_dst_mimg_regular_d16>`,    :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,   :ref:`ssamp<amdgpu_synid9_samp_mimg>`      :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
+    image_sample_c_cd_cl   :ref:`vdst<amdgpu_synid9_dst_mimg_regular_d16>`,    :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,   :ref:`ssamp<amdgpu_synid9_samp_mimg>`      :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
+    image_sample_c_cd_cl_o :ref:`vdst<amdgpu_synid9_dst_mimg_regular_d16>`,    :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,   :ref:`ssamp<amdgpu_synid9_samp_mimg>`      :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
+    image_sample_c_cd_o    :ref:`vdst<amdgpu_synid9_dst_mimg_regular_d16>`,    :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,   :ref:`ssamp<amdgpu_synid9_samp_mimg>`      :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
+    image_sample_c_cl      :ref:`vdst<amdgpu_synid9_dst_mimg_regular_d16>`,    :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,   :ref:`ssamp<amdgpu_synid9_samp_mimg>`      :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
+    image_sample_c_cl_o    :ref:`vdst<amdgpu_synid9_dst_mimg_regular_d16>`,    :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,   :ref:`ssamp<amdgpu_synid9_samp_mimg>`      :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
+    image_sample_c_d       :ref:`vdst<amdgpu_synid9_dst_mimg_regular_d16>`,    :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,   :ref:`ssamp<amdgpu_synid9_samp_mimg>`      :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
+    image_sample_c_d_cl    :ref:`vdst<amdgpu_synid9_dst_mimg_regular_d16>`,    :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,   :ref:`ssamp<amdgpu_synid9_samp_mimg>`      :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
+    image_sample_c_d_cl_o  :ref:`vdst<amdgpu_synid9_dst_mimg_regular_d16>`,    :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,   :ref:`ssamp<amdgpu_synid9_samp_mimg>`      :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
+    image_sample_c_d_o     :ref:`vdst<amdgpu_synid9_dst_mimg_regular_d16>`,    :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,   :ref:`ssamp<amdgpu_synid9_samp_mimg>`      :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
+    image_sample_c_l       :ref:`vdst<amdgpu_synid9_dst_mimg_regular_d16>`,    :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,   :ref:`ssamp<amdgpu_synid9_samp_mimg>`      :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
+    image_sample_c_l_o     :ref:`vdst<amdgpu_synid9_dst_mimg_regular_d16>`,    :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,   :ref:`ssamp<amdgpu_synid9_samp_mimg>`      :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
+    image_sample_c_lz      :ref:`vdst<amdgpu_synid9_dst_mimg_regular_d16>`,    :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,   :ref:`ssamp<amdgpu_synid9_samp_mimg>`      :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
+    image_sample_c_lz_o    :ref:`vdst<amdgpu_synid9_dst_mimg_regular_d16>`,    :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,   :ref:`ssamp<amdgpu_synid9_samp_mimg>`      :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
+    image_sample_c_o       :ref:`vdst<amdgpu_synid9_dst_mimg_regular_d16>`,    :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,   :ref:`ssamp<amdgpu_synid9_samp_mimg>`      :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
+    image_sample_cd        :ref:`vdst<amdgpu_synid9_dst_mimg_regular_d16>`,    :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,   :ref:`ssamp<amdgpu_synid9_samp_mimg>`      :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
+    image_sample_cd_cl     :ref:`vdst<amdgpu_synid9_dst_mimg_regular_d16>`,    :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,   :ref:`ssamp<amdgpu_synid9_samp_mimg>`      :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
+    image_sample_cd_cl_o   :ref:`vdst<amdgpu_synid9_dst_mimg_regular_d16>`,    :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,   :ref:`ssamp<amdgpu_synid9_samp_mimg>`      :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
+    image_sample_cd_o      :ref:`vdst<amdgpu_synid9_dst_mimg_regular_d16>`,    :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,   :ref:`ssamp<amdgpu_synid9_samp_mimg>`      :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
+    image_sample_cl        :ref:`vdst<amdgpu_synid9_dst_mimg_regular_d16>`,    :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,   :ref:`ssamp<amdgpu_synid9_samp_mimg>`      :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
+    image_sample_cl_o      :ref:`vdst<amdgpu_synid9_dst_mimg_regular_d16>`,    :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,   :ref:`ssamp<amdgpu_synid9_samp_mimg>`      :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
+    image_sample_d         :ref:`vdst<amdgpu_synid9_dst_mimg_regular_d16>`,    :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,   :ref:`ssamp<amdgpu_synid9_samp_mimg>`      :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
+    image_sample_d_cl      :ref:`vdst<amdgpu_synid9_dst_mimg_regular_d16>`,    :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,   :ref:`ssamp<amdgpu_synid9_samp_mimg>`      :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
+    image_sample_d_cl_o    :ref:`vdst<amdgpu_synid9_dst_mimg_regular_d16>`,    :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,   :ref:`ssamp<amdgpu_synid9_samp_mimg>`      :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
+    image_sample_d_o       :ref:`vdst<amdgpu_synid9_dst_mimg_regular_d16>`,    :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,   :ref:`ssamp<amdgpu_synid9_samp_mimg>`      :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
+    image_sample_l         :ref:`vdst<amdgpu_synid9_dst_mimg_regular_d16>`,    :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,   :ref:`ssamp<amdgpu_synid9_samp_mimg>`      :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
+    image_sample_l_o       :ref:`vdst<amdgpu_synid9_dst_mimg_regular_d16>`,    :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,   :ref:`ssamp<amdgpu_synid9_samp_mimg>`      :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
+    image_sample_lz        :ref:`vdst<amdgpu_synid9_dst_mimg_regular_d16>`,    :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,   :ref:`ssamp<amdgpu_synid9_samp_mimg>`      :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
+    image_sample_lz_o      :ref:`vdst<amdgpu_synid9_dst_mimg_regular_d16>`,    :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,   :ref:`ssamp<amdgpu_synid9_samp_mimg>`      :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
+    image_sample_o         :ref:`vdst<amdgpu_synid9_dst_mimg_regular_d16>`,    :ref:`vaddr<amdgpu_synid9_addr_mimg>`,     :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`,   :ref:`ssamp<amdgpu_synid9_samp_mimg>`      :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
+    image_store                     :ref:`vdata<amdgpu_synid9_data_mimg_store_d16>`,     :ref:`vaddr<amdgpu_synid9_addr_mimg>`,   :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`      :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
+    image_store_mip                 :ref:`vdata<amdgpu_synid9_data_mimg_store_d16>`,     :ref:`vaddr<amdgpu_synid9_addr_mimg>`,   :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`      :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>` :ref:`d16<amdgpu_synid_d16>`
+    image_store_mip_pck             :ref:`vdata<amdgpu_synid9_data_mimg_store>`,     :ref:`vaddr<amdgpu_synid9_addr_mimg>`,   :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`      :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>`
+    image_store_pck                 :ref:`vdata<amdgpu_synid9_data_mimg_store>`,     :ref:`vaddr<amdgpu_synid9_addr_mimg>`,   :ref:`srsrc<amdgpu_synid9_rsrc_mimg>`      :ref:`dmask<amdgpu_synid_dmask>` :ref:`unorm<amdgpu_synid_unorm>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`a16<amdgpu_synid_a16>` :ref:`lwe<amdgpu_synid_lwe>` :ref:`da<amdgpu_synid_da>`
 
 MUBUF
 -----------------------
 
 .. parsed-literal::
 
-    **INSTRUCTION**                    **DST**       **SRC0**             **SRC1**      **SRC2**      **SRC3**           **MODIFIERS**
-    \ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|
-    buffer_atomic_add                        :ref:`vdata<amdgpu_synid9_data_buf_atomic32>`::ref:`dst<amdgpu_synid9_ret>`,       :ref:`vaddr<amdgpu_synid9_addr_buf>`,    :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,    :ref:`soffset<amdgpu_synid9_offset_buf>`        :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_atomic_add_x2                     :ref:`vdata<amdgpu_synid9_data_buf_atomic64>`::ref:`dst<amdgpu_synid9_ret>`,       :ref:`vaddr<amdgpu_synid9_addr_buf>`,    :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,    :ref:`soffset<amdgpu_synid9_offset_buf>`        :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_atomic_and                        :ref:`vdata<amdgpu_synid9_data_buf_atomic32>`::ref:`dst<amdgpu_synid9_ret>`,       :ref:`vaddr<amdgpu_synid9_addr_buf>`,    :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,    :ref:`soffset<amdgpu_synid9_offset_buf>`        :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_atomic_and_x2                     :ref:`vdata<amdgpu_synid9_data_buf_atomic64>`::ref:`dst<amdgpu_synid9_ret>`,       :ref:`vaddr<amdgpu_synid9_addr_buf>`,    :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,    :ref:`soffset<amdgpu_synid9_offset_buf>`        :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_atomic_cmpswap                    :ref:`vdata<amdgpu_synid9_data_buf_atomic64>`::ref:`dst<amdgpu_synid9_ret>`::ref:`b32x2<amdgpu_synid9_type_dev>`, :ref:`vaddr<amdgpu_synid9_addr_buf>`,    :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,    :ref:`soffset<amdgpu_synid9_offset_buf>`        :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_atomic_cmpswap_x2                 :ref:`vdata<amdgpu_synid9_data_buf_atomic128>`::ref:`dst<amdgpu_synid9_ret>`::ref:`b64x2<amdgpu_synid9_type_dev>`, :ref:`vaddr<amdgpu_synid9_addr_buf>`,    :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,    :ref:`soffset<amdgpu_synid9_offset_buf>`        :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_atomic_dec                        :ref:`vdata<amdgpu_synid9_data_buf_atomic32>`::ref:`dst<amdgpu_synid9_ret>`::ref:`u32<amdgpu_synid9_type_dev>`,   :ref:`vaddr<amdgpu_synid9_addr_buf>`,    :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,    :ref:`soffset<amdgpu_synid9_offset_buf>`        :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_atomic_dec_x2                     :ref:`vdata<amdgpu_synid9_data_buf_atomic64>`::ref:`dst<amdgpu_synid9_ret>`::ref:`u64<amdgpu_synid9_type_dev>`,   :ref:`vaddr<amdgpu_synid9_addr_buf>`,    :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,    :ref:`soffset<amdgpu_synid9_offset_buf>`        :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_atomic_inc                        :ref:`vdata<amdgpu_synid9_data_buf_atomic32>`::ref:`dst<amdgpu_synid9_ret>`::ref:`u32<amdgpu_synid9_type_dev>`,   :ref:`vaddr<amdgpu_synid9_addr_buf>`,    :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,    :ref:`soffset<amdgpu_synid9_offset_buf>`        :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_atomic_inc_x2                     :ref:`vdata<amdgpu_synid9_data_buf_atomic64>`::ref:`dst<amdgpu_synid9_ret>`::ref:`u64<amdgpu_synid9_type_dev>`,   :ref:`vaddr<amdgpu_synid9_addr_buf>`,    :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,    :ref:`soffset<amdgpu_synid9_offset_buf>`        :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_atomic_or                         :ref:`vdata<amdgpu_synid9_data_buf_atomic32>`::ref:`dst<amdgpu_synid9_ret>`,       :ref:`vaddr<amdgpu_synid9_addr_buf>`,    :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,    :ref:`soffset<amdgpu_synid9_offset_buf>`        :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_atomic_or_x2                      :ref:`vdata<amdgpu_synid9_data_buf_atomic64>`::ref:`dst<amdgpu_synid9_ret>`,       :ref:`vaddr<amdgpu_synid9_addr_buf>`,    :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,    :ref:`soffset<amdgpu_synid9_offset_buf>`        :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_atomic_smax                       :ref:`vdata<amdgpu_synid9_data_buf_atomic32>`::ref:`dst<amdgpu_synid9_ret>`::ref:`s32<amdgpu_synid9_type_dev>`,   :ref:`vaddr<amdgpu_synid9_addr_buf>`,    :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,    :ref:`soffset<amdgpu_synid9_offset_buf>`        :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_atomic_smax_x2                    :ref:`vdata<amdgpu_synid9_data_buf_atomic64>`::ref:`dst<amdgpu_synid9_ret>`::ref:`s64<amdgpu_synid9_type_dev>`,   :ref:`vaddr<amdgpu_synid9_addr_buf>`,    :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,    :ref:`soffset<amdgpu_synid9_offset_buf>`        :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_atomic_smin                       :ref:`vdata<amdgpu_synid9_data_buf_atomic32>`::ref:`dst<amdgpu_synid9_ret>`::ref:`s32<amdgpu_synid9_type_dev>`,   :ref:`vaddr<amdgpu_synid9_addr_buf>`,    :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,    :ref:`soffset<amdgpu_synid9_offset_buf>`        :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_atomic_smin_x2                    :ref:`vdata<amdgpu_synid9_data_buf_atomic64>`::ref:`dst<amdgpu_synid9_ret>`::ref:`s64<amdgpu_synid9_type_dev>`,   :ref:`vaddr<amdgpu_synid9_addr_buf>`,    :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,    :ref:`soffset<amdgpu_synid9_offset_buf>`        :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_atomic_sub                        :ref:`vdata<amdgpu_synid9_data_buf_atomic32>`::ref:`dst<amdgpu_synid9_ret>`,       :ref:`vaddr<amdgpu_synid9_addr_buf>`,    :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,    :ref:`soffset<amdgpu_synid9_offset_buf>`        :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_atomic_sub_x2                     :ref:`vdata<amdgpu_synid9_data_buf_atomic64>`::ref:`dst<amdgpu_synid9_ret>`,       :ref:`vaddr<amdgpu_synid9_addr_buf>`,    :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,    :ref:`soffset<amdgpu_synid9_offset_buf>`        :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_atomic_swap                       :ref:`vdata<amdgpu_synid9_data_buf_atomic32>`::ref:`dst<amdgpu_synid9_ret>`,       :ref:`vaddr<amdgpu_synid9_addr_buf>`,    :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,    :ref:`soffset<amdgpu_synid9_offset_buf>`        :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_atomic_swap_x2                    :ref:`vdata<amdgpu_synid9_data_buf_atomic64>`::ref:`dst<amdgpu_synid9_ret>`,       :ref:`vaddr<amdgpu_synid9_addr_buf>`,    :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,    :ref:`soffset<amdgpu_synid9_offset_buf>`        :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_atomic_umax                       :ref:`vdata<amdgpu_synid9_data_buf_atomic32>`::ref:`dst<amdgpu_synid9_ret>`::ref:`u32<amdgpu_synid9_type_dev>`,   :ref:`vaddr<amdgpu_synid9_addr_buf>`,    :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,    :ref:`soffset<amdgpu_synid9_offset_buf>`        :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_atomic_umax_x2                    :ref:`vdata<amdgpu_synid9_data_buf_atomic64>`::ref:`dst<amdgpu_synid9_ret>`::ref:`u64<amdgpu_synid9_type_dev>`,   :ref:`vaddr<amdgpu_synid9_addr_buf>`,    :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,    :ref:`soffset<amdgpu_synid9_offset_buf>`        :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_atomic_umin                       :ref:`vdata<amdgpu_synid9_data_buf_atomic32>`::ref:`dst<amdgpu_synid9_ret>`::ref:`u32<amdgpu_synid9_type_dev>`,   :ref:`vaddr<amdgpu_synid9_addr_buf>`,    :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,    :ref:`soffset<amdgpu_synid9_offset_buf>`        :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_atomic_umin_x2                    :ref:`vdata<amdgpu_synid9_data_buf_atomic64>`::ref:`dst<amdgpu_synid9_ret>`::ref:`u64<amdgpu_synid9_type_dev>`,   :ref:`vaddr<amdgpu_synid9_addr_buf>`,    :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,    :ref:`soffset<amdgpu_synid9_offset_buf>`        :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_atomic_xor                        :ref:`vdata<amdgpu_synid9_data_buf_atomic32>`::ref:`dst<amdgpu_synid9_ret>`,       :ref:`vaddr<amdgpu_synid9_addr_buf>`,    :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,    :ref:`soffset<amdgpu_synid9_offset_buf>`        :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_atomic_xor_x2                     :ref:`vdata<amdgpu_synid9_data_buf_atomic64>`::ref:`dst<amdgpu_synid9_ret>`,       :ref:`vaddr<amdgpu_synid9_addr_buf>`,    :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,    :ref:`soffset<amdgpu_synid9_offset_buf>`        :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_load_dword              :ref:`vdst<amdgpu_synid9_dst_buf_lds>`,     :ref:`vaddr<amdgpu_synid9_addr_buf>`,           :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,    :ref:`soffset<amdgpu_synid9_offset_buf>`                  :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`lds<amdgpu_synid_lds>`
-    buffer_load_dwordx2            :ref:`vdst<amdgpu_synid9_dst_buf_64>`,     :ref:`vaddr<amdgpu_synid9_addr_buf>`,           :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,    :ref:`soffset<amdgpu_synid9_offset_buf>`                  :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_load_dwordx3            :ref:`vdst<amdgpu_synid9_dst_buf_96>`,     :ref:`vaddr<amdgpu_synid9_addr_buf>`,           :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,    :ref:`soffset<amdgpu_synid9_offset_buf>`                  :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_load_dwordx4            :ref:`vdst<amdgpu_synid9_dst_buf_128>`,     :ref:`vaddr<amdgpu_synid9_addr_buf>`,           :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,    :ref:`soffset<amdgpu_synid9_offset_buf>`                  :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_load_format_d16_hi_x    :ref:`vdst<amdgpu_synid9_dst_buf_32>`,     :ref:`vaddr<amdgpu_synid9_addr_buf>`,           :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,    :ref:`soffset<amdgpu_synid9_offset_buf>`                  :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_load_format_d16_x       :ref:`vdst<amdgpu_synid9_dst_buf_32>`,     :ref:`vaddr<amdgpu_synid9_addr_buf>`,           :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,    :ref:`soffset<amdgpu_synid9_offset_buf>`                  :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_load_format_d16_xy      :ref:`vdst<amdgpu_synid9_dst_buf_32>`,     :ref:`vaddr<amdgpu_synid9_addr_buf>`,           :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,    :ref:`soffset<amdgpu_synid9_offset_buf>`                  :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_load_format_d16_xyz     :ref:`vdst<amdgpu_synid9_dst_buf_64>`,     :ref:`vaddr<amdgpu_synid9_addr_buf>`,           :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,    :ref:`soffset<amdgpu_synid9_offset_buf>`                  :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_load_format_d16_xyzw    :ref:`vdst<amdgpu_synid9_dst_buf_64>`,     :ref:`vaddr<amdgpu_synid9_addr_buf>`,           :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,    :ref:`soffset<amdgpu_synid9_offset_buf>`                  :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_load_format_x           :ref:`vdst<amdgpu_synid9_dst_buf_lds>`,     :ref:`vaddr<amdgpu_synid9_addr_buf>`,           :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,    :ref:`soffset<amdgpu_synid9_offset_buf>`                  :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`lds<amdgpu_synid_lds>`
-    buffer_load_format_xy          :ref:`vdst<amdgpu_synid9_dst_buf_64>`,     :ref:`vaddr<amdgpu_synid9_addr_buf>`,           :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,    :ref:`soffset<amdgpu_synid9_offset_buf>`                  :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_load_format_xyz         :ref:`vdst<amdgpu_synid9_dst_buf_96>`,     :ref:`vaddr<amdgpu_synid9_addr_buf>`,           :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,    :ref:`soffset<amdgpu_synid9_offset_buf>`                  :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_load_format_xyzw        :ref:`vdst<amdgpu_synid9_dst_buf_128>`,     :ref:`vaddr<amdgpu_synid9_addr_buf>`,           :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,    :ref:`soffset<amdgpu_synid9_offset_buf>`                  :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_load_sbyte              :ref:`vdst<amdgpu_synid9_dst_buf_lds>`,     :ref:`vaddr<amdgpu_synid9_addr_buf>`,           :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,    :ref:`soffset<amdgpu_synid9_offset_buf>`                  :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`lds<amdgpu_synid_lds>`
-    buffer_load_sbyte_d16          :ref:`vdst<amdgpu_synid9_dst_buf_32>`,     :ref:`vaddr<amdgpu_synid9_addr_buf>`,           :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,    :ref:`soffset<amdgpu_synid9_offset_buf>`                  :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_load_sbyte_d16_hi       :ref:`vdst<amdgpu_synid9_dst_buf_32>`,     :ref:`vaddr<amdgpu_synid9_addr_buf>`,           :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,    :ref:`soffset<amdgpu_synid9_offset_buf>`                  :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_load_short_d16          :ref:`vdst<amdgpu_synid9_dst_buf_32>`,     :ref:`vaddr<amdgpu_synid9_addr_buf>`,           :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,    :ref:`soffset<amdgpu_synid9_offset_buf>`                  :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_load_short_d16_hi       :ref:`vdst<amdgpu_synid9_dst_buf_32>`,     :ref:`vaddr<amdgpu_synid9_addr_buf>`,           :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,    :ref:`soffset<amdgpu_synid9_offset_buf>`                  :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_load_sshort             :ref:`vdst<amdgpu_synid9_dst_buf_lds>`,     :ref:`vaddr<amdgpu_synid9_addr_buf>`,           :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,    :ref:`soffset<amdgpu_synid9_offset_buf>`                  :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`lds<amdgpu_synid_lds>`
-    buffer_load_ubyte              :ref:`vdst<amdgpu_synid9_dst_buf_lds>`,     :ref:`vaddr<amdgpu_synid9_addr_buf>`,           :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,    :ref:`soffset<amdgpu_synid9_offset_buf>`                  :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`lds<amdgpu_synid_lds>`
-    buffer_load_ubyte_d16          :ref:`vdst<amdgpu_synid9_dst_buf_32>`,     :ref:`vaddr<amdgpu_synid9_addr_buf>`,           :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,    :ref:`soffset<amdgpu_synid9_offset_buf>`                  :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_load_ubyte_d16_hi       :ref:`vdst<amdgpu_synid9_dst_buf_32>`,     :ref:`vaddr<amdgpu_synid9_addr_buf>`,           :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,    :ref:`soffset<amdgpu_synid9_offset_buf>`                  :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_load_ushort             :ref:`vdst<amdgpu_synid9_dst_buf_lds>`,     :ref:`vaddr<amdgpu_synid9_addr_buf>`,           :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,    :ref:`soffset<amdgpu_synid9_offset_buf>`                  :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`lds<amdgpu_synid_lds>`
-    buffer_store_byte                        :ref:`vdata<amdgpu_synid9_vdata32_0>`,           :ref:`vaddr<amdgpu_synid9_addr_buf>`,    :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,    :ref:`soffset<amdgpu_synid9_offset_buf>`        :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_store_byte_d16_hi                 :ref:`vdata<amdgpu_synid9_vdata32_0>`,           :ref:`vaddr<amdgpu_synid9_addr_buf>`,    :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,    :ref:`soffset<amdgpu_synid9_offset_buf>`        :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_store_dword                       :ref:`vdata<amdgpu_synid9_vdata32_0>`,           :ref:`vaddr<amdgpu_synid9_addr_buf>`,    :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,    :ref:`soffset<amdgpu_synid9_offset_buf>`        :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_store_dwordx2                     :ref:`vdata<amdgpu_synid9_vdata64_0>`,           :ref:`vaddr<amdgpu_synid9_addr_buf>`,    :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,    :ref:`soffset<amdgpu_synid9_offset_buf>`        :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_store_dwordx3                     :ref:`vdata<amdgpu_synid9_vdata96_0>`,           :ref:`vaddr<amdgpu_synid9_addr_buf>`,    :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,    :ref:`soffset<amdgpu_synid9_offset_buf>`        :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_store_dwordx4                     :ref:`vdata<amdgpu_synid9_vdata128_0>`,           :ref:`vaddr<amdgpu_synid9_addr_buf>`,    :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,    :ref:`soffset<amdgpu_synid9_offset_buf>`        :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_store_format_d16_hi_x             :ref:`vdata<amdgpu_synid9_vdata32_0>`,           :ref:`vaddr<amdgpu_synid9_addr_buf>`,    :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,    :ref:`soffset<amdgpu_synid9_offset_buf>`        :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_store_format_d16_x                :ref:`vdata<amdgpu_synid9_vdata32_0>`,           :ref:`vaddr<amdgpu_synid9_addr_buf>`,    :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,    :ref:`soffset<amdgpu_synid9_offset_buf>`        :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_store_format_d16_xy               :ref:`vdata<amdgpu_synid9_vdata32_0>`,           :ref:`vaddr<amdgpu_synid9_addr_buf>`,    :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,    :ref:`soffset<amdgpu_synid9_offset_buf>`        :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_store_format_d16_xyz              :ref:`vdata<amdgpu_synid9_vdata64_0>`,           :ref:`vaddr<amdgpu_synid9_addr_buf>`,    :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,    :ref:`soffset<amdgpu_synid9_offset_buf>`        :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_store_format_d16_xyzw             :ref:`vdata<amdgpu_synid9_vdata64_0>`,           :ref:`vaddr<amdgpu_synid9_addr_buf>`,    :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,    :ref:`soffset<amdgpu_synid9_offset_buf>`        :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_store_format_x                    :ref:`vdata<amdgpu_synid9_vdata32_0>`,           :ref:`vaddr<amdgpu_synid9_addr_buf>`,    :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,    :ref:`soffset<amdgpu_synid9_offset_buf>`        :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_store_format_xy                   :ref:`vdata<amdgpu_synid9_vdata64_0>`,           :ref:`vaddr<amdgpu_synid9_addr_buf>`,    :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,    :ref:`soffset<amdgpu_synid9_offset_buf>`        :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_store_format_xyz                  :ref:`vdata<amdgpu_synid9_vdata96_0>`,           :ref:`vaddr<amdgpu_synid9_addr_buf>`,    :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,    :ref:`soffset<amdgpu_synid9_offset_buf>`        :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_store_format_xyzw                 :ref:`vdata<amdgpu_synid9_vdata128_0>`,           :ref:`vaddr<amdgpu_synid9_addr_buf>`,    :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,    :ref:`soffset<amdgpu_synid9_offset_buf>`        :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_store_lds_dword                   :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,           :ref:`soffset<amdgpu_synid9_offset_buf>`                            :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`lds<amdgpu_synid_lds>`
-    buffer_store_short                       :ref:`vdata<amdgpu_synid9_vdata32_0>`,           :ref:`vaddr<amdgpu_synid9_addr_buf>`,    :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,    :ref:`soffset<amdgpu_synid9_offset_buf>`        :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
-    buffer_store_short_d16_hi                :ref:`vdata<amdgpu_synid9_vdata32_0>`,           :ref:`vaddr<amdgpu_synid9_addr_buf>`,    :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,    :ref:`soffset<amdgpu_synid9_offset_buf>`        :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`buf_offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    **INSTRUCTION**                  **DST**   **SRC0**             **SRC1**    **SRC2**    **SRC3**    **MODIFIERS**
+    \ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|
+    buffer_atomic_add                  :ref:`vdata<amdgpu_synid9_data_buf_atomic32>`::ref:`dst<amdgpu_synid9_ret>`,       :ref:`vaddr<amdgpu_synid9_addr_buf>`,  :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,  :ref:`soffset<amdgpu_synid9_offset_buf>` :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_atomic_add_x2               :ref:`vdata<amdgpu_synid9_data_buf_atomic64>`::ref:`dst<amdgpu_synid9_ret>`,       :ref:`vaddr<amdgpu_synid9_addr_buf>`,  :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,  :ref:`soffset<amdgpu_synid9_offset_buf>` :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_atomic_and                  :ref:`vdata<amdgpu_synid9_data_buf_atomic32>`::ref:`dst<amdgpu_synid9_ret>`,       :ref:`vaddr<amdgpu_synid9_addr_buf>`,  :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,  :ref:`soffset<amdgpu_synid9_offset_buf>` :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_atomic_and_x2               :ref:`vdata<amdgpu_synid9_data_buf_atomic64>`::ref:`dst<amdgpu_synid9_ret>`,       :ref:`vaddr<amdgpu_synid9_addr_buf>`,  :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,  :ref:`soffset<amdgpu_synid9_offset_buf>` :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_atomic_cmpswap              :ref:`vdata<amdgpu_synid9_data_buf_atomic64>`::ref:`dst<amdgpu_synid9_ret>`::ref:`b32x2<amdgpu_synid9_type_dev>`, :ref:`vaddr<amdgpu_synid9_addr_buf>`,  :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,  :ref:`soffset<amdgpu_synid9_offset_buf>` :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_atomic_cmpswap_x2           :ref:`vdata<amdgpu_synid9_data_buf_atomic128>`::ref:`dst<amdgpu_synid9_ret>`::ref:`b64x2<amdgpu_synid9_type_dev>`, :ref:`vaddr<amdgpu_synid9_addr_buf>`,  :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,  :ref:`soffset<amdgpu_synid9_offset_buf>` :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_atomic_dec                  :ref:`vdata<amdgpu_synid9_data_buf_atomic32>`::ref:`dst<amdgpu_synid9_ret>`::ref:`u32<amdgpu_synid9_type_dev>`,   :ref:`vaddr<amdgpu_synid9_addr_buf>`,  :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,  :ref:`soffset<amdgpu_synid9_offset_buf>` :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_atomic_dec_x2               :ref:`vdata<amdgpu_synid9_data_buf_atomic64>`::ref:`dst<amdgpu_synid9_ret>`::ref:`u64<amdgpu_synid9_type_dev>`,   :ref:`vaddr<amdgpu_synid9_addr_buf>`,  :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,  :ref:`soffset<amdgpu_synid9_offset_buf>` :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_atomic_inc                  :ref:`vdata<amdgpu_synid9_data_buf_atomic32>`::ref:`dst<amdgpu_synid9_ret>`::ref:`u32<amdgpu_synid9_type_dev>`,   :ref:`vaddr<amdgpu_synid9_addr_buf>`,  :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,  :ref:`soffset<amdgpu_synid9_offset_buf>` :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_atomic_inc_x2               :ref:`vdata<amdgpu_synid9_data_buf_atomic64>`::ref:`dst<amdgpu_synid9_ret>`::ref:`u64<amdgpu_synid9_type_dev>`,   :ref:`vaddr<amdgpu_synid9_addr_buf>`,  :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,  :ref:`soffset<amdgpu_synid9_offset_buf>` :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_atomic_or                   :ref:`vdata<amdgpu_synid9_data_buf_atomic32>`::ref:`dst<amdgpu_synid9_ret>`,       :ref:`vaddr<amdgpu_synid9_addr_buf>`,  :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,  :ref:`soffset<amdgpu_synid9_offset_buf>` :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_atomic_or_x2                :ref:`vdata<amdgpu_synid9_data_buf_atomic64>`::ref:`dst<amdgpu_synid9_ret>`,       :ref:`vaddr<amdgpu_synid9_addr_buf>`,  :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,  :ref:`soffset<amdgpu_synid9_offset_buf>` :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_atomic_smax                 :ref:`vdata<amdgpu_synid9_data_buf_atomic32>`::ref:`dst<amdgpu_synid9_ret>`::ref:`s32<amdgpu_synid9_type_dev>`,   :ref:`vaddr<amdgpu_synid9_addr_buf>`,  :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,  :ref:`soffset<amdgpu_synid9_offset_buf>` :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_atomic_smax_x2              :ref:`vdata<amdgpu_synid9_data_buf_atomic64>`::ref:`dst<amdgpu_synid9_ret>`::ref:`s64<amdgpu_synid9_type_dev>`,   :ref:`vaddr<amdgpu_synid9_addr_buf>`,  :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,  :ref:`soffset<amdgpu_synid9_offset_buf>` :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_atomic_smin                 :ref:`vdata<amdgpu_synid9_data_buf_atomic32>`::ref:`dst<amdgpu_synid9_ret>`::ref:`s32<amdgpu_synid9_type_dev>`,   :ref:`vaddr<amdgpu_synid9_addr_buf>`,  :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,  :ref:`soffset<amdgpu_synid9_offset_buf>` :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_atomic_smin_x2              :ref:`vdata<amdgpu_synid9_data_buf_atomic64>`::ref:`dst<amdgpu_synid9_ret>`::ref:`s64<amdgpu_synid9_type_dev>`,   :ref:`vaddr<amdgpu_synid9_addr_buf>`,  :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,  :ref:`soffset<amdgpu_synid9_offset_buf>` :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_atomic_sub                  :ref:`vdata<amdgpu_synid9_data_buf_atomic32>`::ref:`dst<amdgpu_synid9_ret>`,       :ref:`vaddr<amdgpu_synid9_addr_buf>`,  :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,  :ref:`soffset<amdgpu_synid9_offset_buf>` :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_atomic_sub_x2               :ref:`vdata<amdgpu_synid9_data_buf_atomic64>`::ref:`dst<amdgpu_synid9_ret>`,       :ref:`vaddr<amdgpu_synid9_addr_buf>`,  :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,  :ref:`soffset<amdgpu_synid9_offset_buf>` :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_atomic_swap                 :ref:`vdata<amdgpu_synid9_data_buf_atomic32>`::ref:`dst<amdgpu_synid9_ret>`,       :ref:`vaddr<amdgpu_synid9_addr_buf>`,  :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,  :ref:`soffset<amdgpu_synid9_offset_buf>` :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_atomic_swap_x2              :ref:`vdata<amdgpu_synid9_data_buf_atomic64>`::ref:`dst<amdgpu_synid9_ret>`,       :ref:`vaddr<amdgpu_synid9_addr_buf>`,  :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,  :ref:`soffset<amdgpu_synid9_offset_buf>` :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_atomic_umax                 :ref:`vdata<amdgpu_synid9_data_buf_atomic32>`::ref:`dst<amdgpu_synid9_ret>`::ref:`u32<amdgpu_synid9_type_dev>`,   :ref:`vaddr<amdgpu_synid9_addr_buf>`,  :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,  :ref:`soffset<amdgpu_synid9_offset_buf>` :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_atomic_umax_x2              :ref:`vdata<amdgpu_synid9_data_buf_atomic64>`::ref:`dst<amdgpu_synid9_ret>`::ref:`u64<amdgpu_synid9_type_dev>`,   :ref:`vaddr<amdgpu_synid9_addr_buf>`,  :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,  :ref:`soffset<amdgpu_synid9_offset_buf>` :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_atomic_umin                 :ref:`vdata<amdgpu_synid9_data_buf_atomic32>`::ref:`dst<amdgpu_synid9_ret>`::ref:`u32<amdgpu_synid9_type_dev>`,   :ref:`vaddr<amdgpu_synid9_addr_buf>`,  :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,  :ref:`soffset<amdgpu_synid9_offset_buf>` :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_atomic_umin_x2              :ref:`vdata<amdgpu_synid9_data_buf_atomic64>`::ref:`dst<amdgpu_synid9_ret>`::ref:`u64<amdgpu_synid9_type_dev>`,   :ref:`vaddr<amdgpu_synid9_addr_buf>`,  :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,  :ref:`soffset<amdgpu_synid9_offset_buf>` :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_atomic_xor                  :ref:`vdata<amdgpu_synid9_data_buf_atomic32>`::ref:`dst<amdgpu_synid9_ret>`,       :ref:`vaddr<amdgpu_synid9_addr_buf>`,  :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,  :ref:`soffset<amdgpu_synid9_offset_buf>` :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_atomic_xor_x2               :ref:`vdata<amdgpu_synid9_data_buf_atomic64>`::ref:`dst<amdgpu_synid9_ret>`,       :ref:`vaddr<amdgpu_synid9_addr_buf>`,  :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,  :ref:`soffset<amdgpu_synid9_offset_buf>` :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_load_dword            :ref:`vdst<amdgpu_synid9_dst_buf_lds>`, :ref:`vaddr<amdgpu_synid9_addr_buf>`,           :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,  :ref:`soffset<amdgpu_synid9_offset_buf>`         :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`lds<amdgpu_synid_lds>`
+    buffer_load_dwordx2          :ref:`vdst<amdgpu_synid9_dst_buf_64>`, :ref:`vaddr<amdgpu_synid9_addr_buf>`,           :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,  :ref:`soffset<amdgpu_synid9_offset_buf>`         :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_load_dwordx3          :ref:`vdst<amdgpu_synid9_dst_buf_96>`, :ref:`vaddr<amdgpu_synid9_addr_buf>`,           :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,  :ref:`soffset<amdgpu_synid9_offset_buf>`         :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_load_dwordx4          :ref:`vdst<amdgpu_synid9_dst_buf_128>`, :ref:`vaddr<amdgpu_synid9_addr_buf>`,           :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,  :ref:`soffset<amdgpu_synid9_offset_buf>`         :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_load_format_d16_hi_x  :ref:`vdst<amdgpu_synid9_dst_buf_32>`, :ref:`vaddr<amdgpu_synid9_addr_buf>`,           :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,  :ref:`soffset<amdgpu_synid9_offset_buf>`         :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_load_format_d16_x     :ref:`vdst<amdgpu_synid9_dst_buf_32>`, :ref:`vaddr<amdgpu_synid9_addr_buf>`,           :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,  :ref:`soffset<amdgpu_synid9_offset_buf>`         :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_load_format_d16_xy    :ref:`vdst<amdgpu_synid9_dst_buf_32>`, :ref:`vaddr<amdgpu_synid9_addr_buf>`,           :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,  :ref:`soffset<amdgpu_synid9_offset_buf>`         :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_load_format_d16_xyz   :ref:`vdst<amdgpu_synid9_dst_buf_64>`, :ref:`vaddr<amdgpu_synid9_addr_buf>`,           :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,  :ref:`soffset<amdgpu_synid9_offset_buf>`         :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_load_format_d16_xyzw  :ref:`vdst<amdgpu_synid9_dst_buf_64>`, :ref:`vaddr<amdgpu_synid9_addr_buf>`,           :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,  :ref:`soffset<amdgpu_synid9_offset_buf>`         :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_load_format_x         :ref:`vdst<amdgpu_synid9_dst_buf_lds>`, :ref:`vaddr<amdgpu_synid9_addr_buf>`,           :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,  :ref:`soffset<amdgpu_synid9_offset_buf>`         :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`lds<amdgpu_synid_lds>`
+    buffer_load_format_xy        :ref:`vdst<amdgpu_synid9_dst_buf_64>`, :ref:`vaddr<amdgpu_synid9_addr_buf>`,           :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,  :ref:`soffset<amdgpu_synid9_offset_buf>`         :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_load_format_xyz       :ref:`vdst<amdgpu_synid9_dst_buf_96>`, :ref:`vaddr<amdgpu_synid9_addr_buf>`,           :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,  :ref:`soffset<amdgpu_synid9_offset_buf>`         :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_load_format_xyzw      :ref:`vdst<amdgpu_synid9_dst_buf_128>`, :ref:`vaddr<amdgpu_synid9_addr_buf>`,           :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,  :ref:`soffset<amdgpu_synid9_offset_buf>`         :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_load_sbyte            :ref:`vdst<amdgpu_synid9_dst_buf_lds>`, :ref:`vaddr<amdgpu_synid9_addr_buf>`,           :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,  :ref:`soffset<amdgpu_synid9_offset_buf>`         :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`lds<amdgpu_synid_lds>`
+    buffer_load_sbyte_d16        :ref:`vdst<amdgpu_synid9_dst_buf_32>`, :ref:`vaddr<amdgpu_synid9_addr_buf>`,           :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,  :ref:`soffset<amdgpu_synid9_offset_buf>`         :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_load_sbyte_d16_hi     :ref:`vdst<amdgpu_synid9_dst_buf_32>`, :ref:`vaddr<amdgpu_synid9_addr_buf>`,           :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,  :ref:`soffset<amdgpu_synid9_offset_buf>`         :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_load_short_d16        :ref:`vdst<amdgpu_synid9_dst_buf_32>`, :ref:`vaddr<amdgpu_synid9_addr_buf>`,           :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,  :ref:`soffset<amdgpu_synid9_offset_buf>`         :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_load_short_d16_hi     :ref:`vdst<amdgpu_synid9_dst_buf_32>`, :ref:`vaddr<amdgpu_synid9_addr_buf>`,           :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,  :ref:`soffset<amdgpu_synid9_offset_buf>`         :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_load_sshort           :ref:`vdst<amdgpu_synid9_dst_buf_lds>`, :ref:`vaddr<amdgpu_synid9_addr_buf>`,           :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,  :ref:`soffset<amdgpu_synid9_offset_buf>`         :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`lds<amdgpu_synid_lds>`
+    buffer_load_ubyte            :ref:`vdst<amdgpu_synid9_dst_buf_lds>`, :ref:`vaddr<amdgpu_synid9_addr_buf>`,           :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,  :ref:`soffset<amdgpu_synid9_offset_buf>`         :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`lds<amdgpu_synid_lds>`
+    buffer_load_ubyte_d16        :ref:`vdst<amdgpu_synid9_dst_buf_32>`, :ref:`vaddr<amdgpu_synid9_addr_buf>`,           :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,  :ref:`soffset<amdgpu_synid9_offset_buf>`         :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_load_ubyte_d16_hi     :ref:`vdst<amdgpu_synid9_dst_buf_32>`, :ref:`vaddr<amdgpu_synid9_addr_buf>`,           :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,  :ref:`soffset<amdgpu_synid9_offset_buf>`         :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_load_ushort           :ref:`vdst<amdgpu_synid9_dst_buf_lds>`, :ref:`vaddr<amdgpu_synid9_addr_buf>`,           :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,  :ref:`soffset<amdgpu_synid9_offset_buf>`         :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>` :ref:`lds<amdgpu_synid_lds>`
+    buffer_store_byte                  :ref:`vdata<amdgpu_synid9_vdata32_0>`,           :ref:`vaddr<amdgpu_synid9_addr_buf>`,  :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,  :ref:`soffset<amdgpu_synid9_offset_buf>` :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_store_byte_d16_hi           :ref:`vdata<amdgpu_synid9_vdata32_0>`,           :ref:`vaddr<amdgpu_synid9_addr_buf>`,  :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,  :ref:`soffset<amdgpu_synid9_offset_buf>` :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_store_dword                 :ref:`vdata<amdgpu_synid9_vdata32_0>`,           :ref:`vaddr<amdgpu_synid9_addr_buf>`,  :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,  :ref:`soffset<amdgpu_synid9_offset_buf>` :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_store_dwordx2               :ref:`vdata<amdgpu_synid9_vdata64_0>`,           :ref:`vaddr<amdgpu_synid9_addr_buf>`,  :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,  :ref:`soffset<amdgpu_synid9_offset_buf>` :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_store_dwordx3               :ref:`vdata<amdgpu_synid9_vdata96_0>`,           :ref:`vaddr<amdgpu_synid9_addr_buf>`,  :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,  :ref:`soffset<amdgpu_synid9_offset_buf>` :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_store_dwordx4               :ref:`vdata<amdgpu_synid9_vdata128_0>`,           :ref:`vaddr<amdgpu_synid9_addr_buf>`,  :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,  :ref:`soffset<amdgpu_synid9_offset_buf>` :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_store_format_d16_hi_x       :ref:`vdata<amdgpu_synid9_vdata32_0>`,           :ref:`vaddr<amdgpu_synid9_addr_buf>`,  :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,  :ref:`soffset<amdgpu_synid9_offset_buf>` :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_store_format_d16_x          :ref:`vdata<amdgpu_synid9_vdata32_0>`,           :ref:`vaddr<amdgpu_synid9_addr_buf>`,  :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,  :ref:`soffset<amdgpu_synid9_offset_buf>` :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_store_format_d16_xy         :ref:`vdata<amdgpu_synid9_vdata32_0>`,           :ref:`vaddr<amdgpu_synid9_addr_buf>`,  :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,  :ref:`soffset<amdgpu_synid9_offset_buf>` :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_store_format_d16_xyz        :ref:`vdata<amdgpu_synid9_vdata64_0>`,           :ref:`vaddr<amdgpu_synid9_addr_buf>`,  :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,  :ref:`soffset<amdgpu_synid9_offset_buf>` :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_store_format_d16_xyzw       :ref:`vdata<amdgpu_synid9_vdata64_0>`,           :ref:`vaddr<amdgpu_synid9_addr_buf>`,  :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,  :ref:`soffset<amdgpu_synid9_offset_buf>` :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_store_format_x              :ref:`vdata<amdgpu_synid9_vdata32_0>`,           :ref:`vaddr<amdgpu_synid9_addr_buf>`,  :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,  :ref:`soffset<amdgpu_synid9_offset_buf>` :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_store_format_xy             :ref:`vdata<amdgpu_synid9_vdata64_0>`,           :ref:`vaddr<amdgpu_synid9_addr_buf>`,  :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,  :ref:`soffset<amdgpu_synid9_offset_buf>` :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_store_format_xyz            :ref:`vdata<amdgpu_synid9_vdata96_0>`,           :ref:`vaddr<amdgpu_synid9_addr_buf>`,  :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,  :ref:`soffset<amdgpu_synid9_offset_buf>` :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_store_format_xyzw           :ref:`vdata<amdgpu_synid9_vdata128_0>`,           :ref:`vaddr<amdgpu_synid9_addr_buf>`,  :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,  :ref:`soffset<amdgpu_synid9_offset_buf>` :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_store_lds_dword             :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,           :ref:`soffset<amdgpu_synid9_offset_buf>`                 :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`lds<amdgpu_synid_lds>`
+    buffer_store_short                 :ref:`vdata<amdgpu_synid9_vdata32_0>`,           :ref:`vaddr<amdgpu_synid9_addr_buf>`,  :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,  :ref:`soffset<amdgpu_synid9_offset_buf>` :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
+    buffer_store_short_d16_hi          :ref:`vdata<amdgpu_synid9_vdata32_0>`,           :ref:`vaddr<amdgpu_synid9_addr_buf>`,  :ref:`srsrc<amdgpu_synid9_rsrc_buf>`,  :ref:`soffset<amdgpu_synid9_offset_buf>` :ref:`idxen<amdgpu_synid_idxen>` :ref:`offen<amdgpu_synid_offen>` :ref:`offset12<amdgpu_synid_buf_offset12>` :ref:`glc<amdgpu_synid_glc>` :ref:`slc<amdgpu_synid_slc>`
     buffer_wbinvl1
     buffer_wbinvl1_vol
 
@@ -501,7 +501,7 @@ SMEM
 .. parsed-literal::
 
     **INSTRUCTION**                    **DST**       **SRC0**             **SRC1**      **SRC2**           **MODIFIERS**
-    \ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|
+    \ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|
     s_atc_probe                              :ref:`imm3<amdgpu_synid9_perm_smem>`,            :ref:`sbase<amdgpu_synid9_base_smem_addr>`,    :ref:`soffset<amdgpu_synid9_offset_smem_plain>`
     s_atc_probe_buffer                       :ref:`imm3<amdgpu_synid9_perm_smem>`,            :ref:`sbase<amdgpu_synid9_base_smem_buf>`,    :ref:`soffset<amdgpu_synid9_offset_smem_buf>`
     s_atomic_add                             :ref:`sdata<amdgpu_synid9_data_smem_atomic32>`::ref:`dst<amdgpu_synid9_ret>`,       :ref:`sbase<amdgpu_synid9_base_smem_addr>`,    :ref:`soffset<amdgpu_synid9_offset_smem_plain>`        :ref:`glc<amdgpu_synid_glc>`
@@ -1028,637 +1028,637 @@ VOP2
 
 .. parsed-literal::
 
-    **INSTRUCTION**                    **DST0**      **DST1**      **SRC0**        **SRC1**        **SRC2**           **MODIFIERS**
-    \ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|
-    v_add_co_u32                   :ref:`vdst<amdgpu_synid9_vdst32_0>`,     :ref:`vcc<amdgpu_synid9_vcc_64>`,      :ref:`src0<amdgpu_synid9_src32_0>`,       :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`
-    v_add_co_u32_dpp               :ref:`vdst<amdgpu_synid9_vdst32_0>`,     :ref:`vcc<amdgpu_synid9_vcc_64>`,      :ref:`vsrc0<amdgpu_synid9_vsrc32_0>`,      :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`                      :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
-    v_add_co_u32_sdwa              :ref:`vdst<amdgpu_synid9_vdst32_0>`,     :ref:`vcc<amdgpu_synid9_vcc_64>`,      :ref:`src0<amdgpu_synid9_src32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`,     :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`                    :ref:`clamp<amdgpu_synid_clamp>` :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
-    v_add_f16                      :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_0>`,       :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`
-    v_add_f16_dpp                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_dpp_sdwa_abs_neg>`,    :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_dpp_sdwa_abs_neg>`                    :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
-    v_add_f16_sdwa                 :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_0>`::ref:`m<amdgpu_synid9_mod_dpp_sdwa_abs_neg>`,     :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_dpp_sdwa_abs_neg>`                    :ref:`clamp<amdgpu_synid_clamp>` :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
-    v_add_f32                      :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_0>`,       :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`
-    v_add_f32_dpp                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_dpp_sdwa_abs_neg>`,    :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_dpp_sdwa_abs_neg>`                    :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
-    v_add_f32_sdwa                 :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_0>`::ref:`m<amdgpu_synid9_mod_dpp_sdwa_abs_neg>`,     :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_dpp_sdwa_abs_neg>`                    :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>` :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
-    v_add_u16                      :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_0>`,       :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`
-    v_add_u16_dpp                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid9_vsrc32_0>`,      :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`                      :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
-    v_add_u16_sdwa                 :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`,     :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`                    :ref:`clamp<amdgpu_synid_clamp>` :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
-    v_add_u32                      :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_0>`,       :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`
-    v_add_u32_dpp                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid9_vsrc32_0>`,      :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`                      :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
-    v_add_u32_sdwa                 :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`,     :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`                    :ref:`clamp<amdgpu_synid_clamp>` :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
-    v_addc_co_u32                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,     :ref:`vcc<amdgpu_synid9_vcc_64>`,      :ref:`src0<amdgpu_synid9_src32_0>`,       :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`,      :ref:`vcc<amdgpu_synid9_vcc_64>`
-    v_addc_co_u32_dpp              :ref:`vdst<amdgpu_synid9_vdst32_0>`,     :ref:`vcc<amdgpu_synid9_vcc_64>`,      :ref:`vsrc0<amdgpu_synid9_vsrc32_0>`,      :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`,      :ref:`vcc<amdgpu_synid9_vcc_64>`            :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
-    v_addc_co_u32_sdwa             :ref:`vdst<amdgpu_synid9_vdst32_0>`,     :ref:`vcc<amdgpu_synid9_vcc_64>`,      :ref:`src0<amdgpu_synid9_src32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`,     :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`,    :ref:`vcc<amdgpu_synid9_vcc_64>`            :ref:`clamp<amdgpu_synid_clamp>` :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
-    v_and_b32                      :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_0>`,       :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`
-    v_and_b32_dpp                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid9_vsrc32_0>`,      :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`                      :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
-    v_and_b32_sdwa                 :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`,     :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`                    :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
-    v_ashrrev_i16                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_0>`::ref:`u16<amdgpu_synid9_type_dev>`,   :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`
-    v_ashrrev_i16_dpp              :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid9_vsrc32_0>`::ref:`u16<amdgpu_synid9_type_dev>`,  :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`                      :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
-    v_ashrrev_i16_sdwa             :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`::ref:`u16<amdgpu_synid9_type_dev>`, :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`                    :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
-    v_ashrrev_i32                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_0>`::ref:`u32<amdgpu_synid9_type_dev>`,   :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`
-    v_ashrrev_i32_dpp              :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid9_vsrc32_0>`::ref:`u32<amdgpu_synid9_type_dev>`,  :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`                      :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
-    v_ashrrev_i32_sdwa             :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`::ref:`u32<amdgpu_synid9_type_dev>`, :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`                    :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
-    v_cndmask_b32                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_0>`,       :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`,      :ref:`vcc<amdgpu_synid9_vcc_64>`
-    v_cndmask_b32_dpp              :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid9_vsrc32_0>`,      :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`,      :ref:`vcc<amdgpu_synid9_vcc_64>`            :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
-    v_cndmask_b32_sdwa             :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`,     :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`,    :ref:`vcc<amdgpu_synid9_vcc_64>`            :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
-    v_ldexp_f16                    :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_0>`,       :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`i16<amdgpu_synid9_type_dev>`
-    v_ldexp_f16_dpp                :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_dpp_sdwa_abs_neg>`,    :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`i16<amdgpu_synid9_type_dev>`                  :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
-    v_ldexp_f16_sdwa               :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_0>`::ref:`m<amdgpu_synid9_mod_dpp_sdwa_abs_neg>`,     :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`::ref:`i16<amdgpu_synid9_type_dev>`                :ref:`clamp<amdgpu_synid_clamp>` :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
-    v_lshlrev_b16                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_0>`::ref:`u16<amdgpu_synid9_type_dev>`,   :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`
-    v_lshlrev_b16_dpp              :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid9_vsrc32_0>`::ref:`u16<amdgpu_synid9_type_dev>`,  :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`                      :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
-    v_lshlrev_b16_sdwa             :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`::ref:`u16<amdgpu_synid9_type_dev>`, :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`                    :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
-    v_lshlrev_b32                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_0>`::ref:`u32<amdgpu_synid9_type_dev>`,   :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`
-    v_lshlrev_b32_dpp              :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid9_vsrc32_0>`::ref:`u32<amdgpu_synid9_type_dev>`,  :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`                      :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
-    v_lshlrev_b32_sdwa             :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`::ref:`u32<amdgpu_synid9_type_dev>`, :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`                    :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
-    v_lshrrev_b16                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_0>`::ref:`u16<amdgpu_synid9_type_dev>`,   :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`
-    v_lshrrev_b16_dpp              :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid9_vsrc32_0>`::ref:`u16<amdgpu_synid9_type_dev>`,  :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`                      :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
-    v_lshrrev_b16_sdwa             :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`::ref:`u16<amdgpu_synid9_type_dev>`, :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`                    :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
-    v_lshrrev_b32                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_0>`::ref:`u32<amdgpu_synid9_type_dev>`,   :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`
-    v_lshrrev_b32_dpp              :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid9_vsrc32_0>`::ref:`u32<amdgpu_synid9_type_dev>`,  :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`                      :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
-    v_lshrrev_b32_sdwa             :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`::ref:`u32<amdgpu_synid9_type_dev>`, :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`                    :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
-    v_mac_f16                      :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_0>`,       :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`
-    v_mac_f16_dpp                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_dpp_sdwa_abs_neg>`,    :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_dpp_sdwa_abs_neg>`                    :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
-    v_mac_f32                      :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_0>`,       :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`
-    v_mac_f32_dpp                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_dpp_sdwa_abs_neg>`,    :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_dpp_sdwa_abs_neg>`                    :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
-    v_madak_f16                    :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_0>`,       :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`,      :ref:`imm32<amdgpu_synid9_fimm16>`
-    v_madak_f32                    :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_0>`,       :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`,      :ref:`imm32<amdgpu_synid9_fimm32>`
-    v_madmk_f16                    :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_0>`,       :ref:`imm32<amdgpu_synid9_fimm16>`,      :ref:`vsrc2<amdgpu_synid9_vsrc32_0>`
-    v_madmk_f32                    :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_0>`,       :ref:`imm32<amdgpu_synid9_fimm32>`,      :ref:`vsrc2<amdgpu_synid9_vsrc32_0>`
-    v_max_f16                      :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_0>`,       :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`
-    v_max_f16_dpp                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_dpp_sdwa_abs_neg>`,    :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_dpp_sdwa_abs_neg>`                    :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
-    v_max_f16_sdwa                 :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_0>`::ref:`m<amdgpu_synid9_mod_dpp_sdwa_abs_neg>`,     :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_dpp_sdwa_abs_neg>`                    :ref:`clamp<amdgpu_synid_clamp>` :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
-    v_max_f32                      :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_0>`,       :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`
-    v_max_f32_dpp                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_dpp_sdwa_abs_neg>`,    :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_dpp_sdwa_abs_neg>`                    :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
-    v_max_f32_sdwa                 :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_0>`::ref:`m<amdgpu_synid9_mod_dpp_sdwa_abs_neg>`,     :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_dpp_sdwa_abs_neg>`                    :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>` :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
-    v_max_i16                      :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_0>`,       :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`
-    v_max_i16_dpp                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid9_vsrc32_0>`,      :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`                      :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
-    v_max_i16_sdwa                 :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`,     :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`                    :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
-    v_max_i32                      :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_0>`,       :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`
-    v_max_i32_dpp                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid9_vsrc32_0>`,      :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`                      :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
-    v_max_i32_sdwa                 :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`,     :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`                    :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
-    v_max_u16                      :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_0>`,       :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`
-    v_max_u16_dpp                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid9_vsrc32_0>`,      :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`                      :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
-    v_max_u16_sdwa                 :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`,     :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`                    :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
-    v_max_u32                      :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_0>`,       :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`
-    v_max_u32_dpp                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid9_vsrc32_0>`,      :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`                      :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
-    v_max_u32_sdwa                 :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`,     :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`                    :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
-    v_min_f16                      :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_0>`,       :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`
-    v_min_f16_dpp                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_dpp_sdwa_abs_neg>`,    :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_dpp_sdwa_abs_neg>`                    :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
-    v_min_f16_sdwa                 :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_0>`::ref:`m<amdgpu_synid9_mod_dpp_sdwa_abs_neg>`,     :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_dpp_sdwa_abs_neg>`                    :ref:`clamp<amdgpu_synid_clamp>` :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
-    v_min_f32                      :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_0>`,       :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`
-    v_min_f32_dpp                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_dpp_sdwa_abs_neg>`,    :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_dpp_sdwa_abs_neg>`                    :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
-    v_min_f32_sdwa                 :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_0>`::ref:`m<amdgpu_synid9_mod_dpp_sdwa_abs_neg>`,     :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_dpp_sdwa_abs_neg>`                    :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>` :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
-    v_min_i16                      :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_0>`,       :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`
-    v_min_i16_dpp                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid9_vsrc32_0>`,      :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`                      :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
-    v_min_i16_sdwa                 :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`,     :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`                    :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
-    v_min_i32                      :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_0>`,       :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`
-    v_min_i32_dpp                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid9_vsrc32_0>`,      :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`                      :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
-    v_min_i32_sdwa                 :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`,     :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`                    :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
-    v_min_u16                      :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_0>`,       :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`
-    v_min_u16_dpp                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid9_vsrc32_0>`,      :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`                      :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
-    v_min_u16_sdwa                 :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`,     :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`                    :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
-    v_min_u32                      :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_0>`,       :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`
-    v_min_u32_dpp                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid9_vsrc32_0>`,      :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`                      :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
-    v_min_u32_sdwa                 :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`,     :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`                    :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
-    v_mul_f16                      :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_0>`,       :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`
-    v_mul_f16_dpp                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_dpp_sdwa_abs_neg>`,    :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_dpp_sdwa_abs_neg>`                    :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
-    v_mul_f16_sdwa                 :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_0>`::ref:`m<amdgpu_synid9_mod_dpp_sdwa_abs_neg>`,     :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_dpp_sdwa_abs_neg>`                    :ref:`clamp<amdgpu_synid_clamp>` :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
-    v_mul_f32                      :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_0>`,       :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`
-    v_mul_f32_dpp                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_dpp_sdwa_abs_neg>`,    :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_dpp_sdwa_abs_neg>`                    :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
-    v_mul_f32_sdwa                 :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_0>`::ref:`m<amdgpu_synid9_mod_dpp_sdwa_abs_neg>`,     :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_dpp_sdwa_abs_neg>`                    :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>` :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
-    v_mul_hi_i32_i24               :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_0>`,       :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`
-    v_mul_hi_i32_i24_dpp           :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid9_vsrc32_0>`,      :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`                      :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
-    v_mul_hi_i32_i24_sdwa          :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`,     :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`                    :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
-    v_mul_hi_u32_u24               :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_0>`,       :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`
-    v_mul_hi_u32_u24_dpp           :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid9_vsrc32_0>`,      :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`                      :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
-    v_mul_hi_u32_u24_sdwa          :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`,     :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`                    :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
-    v_mul_i32_i24                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_0>`,       :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`
-    v_mul_i32_i24_dpp              :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid9_vsrc32_0>`,      :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`                      :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
-    v_mul_i32_i24_sdwa             :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`,     :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`                    :ref:`clamp<amdgpu_synid_clamp>` :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
-    v_mul_legacy_f32               :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_0>`,       :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`
-    v_mul_legacy_f32_dpp           :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_dpp_sdwa_abs_neg>`,    :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_dpp_sdwa_abs_neg>`                    :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
-    v_mul_legacy_f32_sdwa          :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_0>`::ref:`m<amdgpu_synid9_mod_dpp_sdwa_abs_neg>`,     :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_dpp_sdwa_abs_neg>`                    :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>` :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
-    v_mul_lo_u16                   :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_0>`,       :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`
-    v_mul_lo_u16_dpp               :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid9_vsrc32_0>`,      :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`                      :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
-    v_mul_lo_u16_sdwa              :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`,     :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`                    :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
-    v_mul_u32_u24                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_0>`,       :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`
-    v_mul_u32_u24_dpp              :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid9_vsrc32_0>`,      :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`                      :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
-    v_mul_u32_u24_sdwa             :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`,     :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`                    :ref:`clamp<amdgpu_synid_clamp>` :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
-    v_or_b32                       :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_0>`,       :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`
-    v_or_b32_dpp                   :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid9_vsrc32_0>`,      :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`                      :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
-    v_or_b32_sdwa                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`,     :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`                    :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
-    v_sub_co_u32                   :ref:`vdst<amdgpu_synid9_vdst32_0>`,     :ref:`vcc<amdgpu_synid9_vcc_64>`,      :ref:`src0<amdgpu_synid9_src32_0>`,       :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`
-    v_sub_co_u32_dpp               :ref:`vdst<amdgpu_synid9_vdst32_0>`,     :ref:`vcc<amdgpu_synid9_vcc_64>`,      :ref:`vsrc0<amdgpu_synid9_vsrc32_0>`,      :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`                      :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
-    v_sub_co_u32_sdwa              :ref:`vdst<amdgpu_synid9_vdst32_0>`,     :ref:`vcc<amdgpu_synid9_vcc_64>`,      :ref:`src0<amdgpu_synid9_src32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`,     :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`                    :ref:`clamp<amdgpu_synid_clamp>` :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
-    v_sub_f16                      :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_0>`,       :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`
-    v_sub_f16_dpp                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_dpp_sdwa_abs_neg>`,    :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_dpp_sdwa_abs_neg>`                    :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
-    v_sub_f16_sdwa                 :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_0>`::ref:`m<amdgpu_synid9_mod_dpp_sdwa_abs_neg>`,     :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_dpp_sdwa_abs_neg>`                    :ref:`clamp<amdgpu_synid_clamp>` :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
-    v_sub_f32                      :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_0>`,       :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`
-    v_sub_f32_dpp                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_dpp_sdwa_abs_neg>`,    :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_dpp_sdwa_abs_neg>`                    :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
-    v_sub_f32_sdwa                 :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_0>`::ref:`m<amdgpu_synid9_mod_dpp_sdwa_abs_neg>`,     :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_dpp_sdwa_abs_neg>`                    :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>` :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
-    v_sub_u16                      :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_0>`,       :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`
-    v_sub_u16_dpp                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid9_vsrc32_0>`,      :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`                      :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
-    v_sub_u16_sdwa                 :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`,     :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`                    :ref:`clamp<amdgpu_synid_clamp>` :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
-    v_sub_u32                      :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_0>`,       :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`
-    v_sub_u32_dpp                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid9_vsrc32_0>`,      :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`                      :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
-    v_sub_u32_sdwa                 :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`,     :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`                    :ref:`clamp<amdgpu_synid_clamp>` :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
-    v_subb_co_u32                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,     :ref:`vcc<amdgpu_synid9_vcc_64>`,      :ref:`src0<amdgpu_synid9_src32_0>`,       :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`,      :ref:`vcc<amdgpu_synid9_vcc_64>`
-    v_subb_co_u32_dpp              :ref:`vdst<amdgpu_synid9_vdst32_0>`,     :ref:`vcc<amdgpu_synid9_vcc_64>`,      :ref:`vsrc0<amdgpu_synid9_vsrc32_0>`,      :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`,      :ref:`vcc<amdgpu_synid9_vcc_64>`            :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
-    v_subb_co_u32_sdwa             :ref:`vdst<amdgpu_synid9_vdst32_0>`,     :ref:`vcc<amdgpu_synid9_vcc_64>`,      :ref:`src0<amdgpu_synid9_src32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`,     :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`,    :ref:`vcc<amdgpu_synid9_vcc_64>`            :ref:`clamp<amdgpu_synid_clamp>` :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
-    v_subbrev_co_u32               :ref:`vdst<amdgpu_synid9_vdst32_0>`,     :ref:`vcc<amdgpu_synid9_vcc_64>`,      :ref:`src0<amdgpu_synid9_src32_0>`,       :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`,      :ref:`vcc<amdgpu_synid9_vcc_64>`
-    v_subbrev_co_u32_dpp           :ref:`vdst<amdgpu_synid9_vdst32_0>`,     :ref:`vcc<amdgpu_synid9_vcc_64>`,      :ref:`vsrc0<amdgpu_synid9_vsrc32_0>`,      :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`,      :ref:`vcc<amdgpu_synid9_vcc_64>`            :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
-    v_subbrev_co_u32_sdwa          :ref:`vdst<amdgpu_synid9_vdst32_0>`,     :ref:`vcc<amdgpu_synid9_vcc_64>`,      :ref:`src0<amdgpu_synid9_src32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`,     :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`,    :ref:`vcc<amdgpu_synid9_vcc_64>`            :ref:`clamp<amdgpu_synid_clamp>` :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
-    v_subrev_co_u32                :ref:`vdst<amdgpu_synid9_vdst32_0>`,     :ref:`vcc<amdgpu_synid9_vcc_64>`,      :ref:`src0<amdgpu_synid9_src32_0>`,       :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`
-    v_subrev_co_u32_dpp            :ref:`vdst<amdgpu_synid9_vdst32_0>`,     :ref:`vcc<amdgpu_synid9_vcc_64>`,      :ref:`vsrc0<amdgpu_synid9_vsrc32_0>`,      :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`                      :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
-    v_subrev_co_u32_sdwa           :ref:`vdst<amdgpu_synid9_vdst32_0>`,     :ref:`vcc<amdgpu_synid9_vcc_64>`,      :ref:`src0<amdgpu_synid9_src32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`,     :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`                    :ref:`clamp<amdgpu_synid_clamp>` :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
-    v_subrev_f16                   :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_0>`,       :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`
-    v_subrev_f16_dpp               :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_dpp_sdwa_abs_neg>`,    :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_dpp_sdwa_abs_neg>`                    :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
-    v_subrev_f16_sdwa              :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_0>`::ref:`m<amdgpu_synid9_mod_dpp_sdwa_abs_neg>`,     :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_dpp_sdwa_abs_neg>`                    :ref:`clamp<amdgpu_synid_clamp>` :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
-    v_subrev_f32                   :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_0>`,       :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`
-    v_subrev_f32_dpp               :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_dpp_sdwa_abs_neg>`,    :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_dpp_sdwa_abs_neg>`                    :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
-    v_subrev_f32_sdwa              :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_0>`::ref:`m<amdgpu_synid9_mod_dpp_sdwa_abs_neg>`,     :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_dpp_sdwa_abs_neg>`                    :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>` :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
-    v_subrev_u16                   :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_0>`,       :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`
-    v_subrev_u16_dpp               :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid9_vsrc32_0>`,      :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`                      :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
-    v_subrev_u16_sdwa              :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`,     :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`                    :ref:`clamp<amdgpu_synid_clamp>` :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
-    v_subrev_u32                   :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_0>`,       :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`
-    v_subrev_u32_dpp               :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid9_vsrc32_0>`,      :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`                      :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
-    v_subrev_u32_sdwa              :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`,     :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`                    :ref:`clamp<amdgpu_synid_clamp>` :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
-    v_xor_b32                      :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_0>`,       :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`
-    v_xor_b32_dpp                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid9_vsrc32_0>`,      :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`                      :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
-    v_xor_b32_sdwa                 :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`,     :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`                    :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
+    **INSTRUCTION**           **DST0**  **DST1** **SRC0**        **SRC1**        **SRC2**  **MODIFIERS**
+    \ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|
+    v_add_co_u32          :ref:`vdst<amdgpu_synid9_vdst32_0>`, :ref:`vcc<amdgpu_synid9_vcc_64>`, :ref:`src0<amdgpu_synid9_src32_0>`,       :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`
+    v_add_co_u32_dpp      :ref:`vdst<amdgpu_synid9_vdst32_0>`, :ref:`vcc<amdgpu_synid9_vcc_64>`, :ref:`vsrc0<amdgpu_synid9_vsrc32_0>`,      :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`             :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
+    v_add_co_u32_sdwa     :ref:`vdst<amdgpu_synid9_vdst32_0>`, :ref:`vcc<amdgpu_synid9_vcc_64>`, :ref:`src0<amdgpu_synid9_src32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`,     :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`           :ref:`clamp<amdgpu_synid_clamp>` :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
+    v_add_f16             :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`src0<amdgpu_synid9_src32_0>`,       :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`
+    v_add_f16_dpp         :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_dpp_sdwa_abs_neg>`,    :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_dpp_sdwa_abs_neg>`           :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
+    v_add_f16_sdwa        :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`src0<amdgpu_synid9_src32_0>`::ref:`m<amdgpu_synid9_mod_dpp_sdwa_abs_neg>`,     :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_dpp_sdwa_abs_neg>`           :ref:`clamp<amdgpu_synid_clamp>` :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
+    v_add_f32             :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`src0<amdgpu_synid9_src32_0>`,       :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`
+    v_add_f32_dpp         :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_dpp_sdwa_abs_neg>`,    :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_dpp_sdwa_abs_neg>`           :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
+    v_add_f32_sdwa        :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`src0<amdgpu_synid9_src32_0>`::ref:`m<amdgpu_synid9_mod_dpp_sdwa_abs_neg>`,     :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_dpp_sdwa_abs_neg>`           :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>` :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
+    v_add_u16             :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`src0<amdgpu_synid9_src32_0>`,       :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`
+    v_add_u16_dpp         :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid9_vsrc32_0>`,      :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`             :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
+    v_add_u16_sdwa        :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`src0<amdgpu_synid9_src32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`,     :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`           :ref:`clamp<amdgpu_synid_clamp>` :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
+    v_add_u32             :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`src0<amdgpu_synid9_src32_0>`,       :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`
+    v_add_u32_dpp         :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid9_vsrc32_0>`,      :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`             :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
+    v_add_u32_sdwa        :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`src0<amdgpu_synid9_src32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`,     :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`           :ref:`clamp<amdgpu_synid_clamp>` :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
+    v_addc_co_u32         :ref:`vdst<amdgpu_synid9_vdst32_0>`, :ref:`vcc<amdgpu_synid9_vcc_64>`, :ref:`src0<amdgpu_synid9_src32_0>`,       :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`,      :ref:`vcc<amdgpu_synid9_vcc_64>`
+    v_addc_co_u32_dpp     :ref:`vdst<amdgpu_synid9_vdst32_0>`, :ref:`vcc<amdgpu_synid9_vcc_64>`, :ref:`vsrc0<amdgpu_synid9_vsrc32_0>`,      :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`,      :ref:`vcc<amdgpu_synid9_vcc_64>`   :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
+    v_addc_co_u32_sdwa    :ref:`vdst<amdgpu_synid9_vdst32_0>`, :ref:`vcc<amdgpu_synid9_vcc_64>`, :ref:`src0<amdgpu_synid9_src32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`,     :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`,    :ref:`vcc<amdgpu_synid9_vcc_64>`   :ref:`clamp<amdgpu_synid_clamp>` :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
+    v_and_b32             :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`src0<amdgpu_synid9_src32_0>`,       :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`
+    v_and_b32_dpp         :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid9_vsrc32_0>`,      :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`             :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
+    v_and_b32_sdwa        :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`src0<amdgpu_synid9_src32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`,     :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`           :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
+    v_ashrrev_i16         :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`src0<amdgpu_synid9_src32_0>`::ref:`u16<amdgpu_synid9_type_dev>`,   :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`
+    v_ashrrev_i16_dpp     :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid9_vsrc32_0>`::ref:`u16<amdgpu_synid9_type_dev>`,  :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`             :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
+    v_ashrrev_i16_sdwa    :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`src0<amdgpu_synid9_src32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`::ref:`u16<amdgpu_synid9_type_dev>`, :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`           :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
+    v_ashrrev_i32         :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`src0<amdgpu_synid9_src32_0>`::ref:`u32<amdgpu_synid9_type_dev>`,   :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`
+    v_ashrrev_i32_dpp     :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid9_vsrc32_0>`::ref:`u32<amdgpu_synid9_type_dev>`,  :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`             :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
+    v_ashrrev_i32_sdwa    :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`src0<amdgpu_synid9_src32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`::ref:`u32<amdgpu_synid9_type_dev>`, :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`           :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
+    v_cndmask_b32         :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`src0<amdgpu_synid9_src32_0>`,       :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`,      :ref:`vcc<amdgpu_synid9_vcc_64>`
+    v_cndmask_b32_dpp     :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid9_vsrc32_0>`,      :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`,      :ref:`vcc<amdgpu_synid9_vcc_64>`   :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
+    v_cndmask_b32_sdwa    :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`src0<amdgpu_synid9_src32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`,     :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`,    :ref:`vcc<amdgpu_synid9_vcc_64>`   :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
+    v_ldexp_f16           :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`src0<amdgpu_synid9_src32_0>`,       :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`i16<amdgpu_synid9_type_dev>`
+    v_ldexp_f16_dpp       :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_dpp_sdwa_abs_neg>`,    :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`i16<amdgpu_synid9_type_dev>`         :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
+    v_ldexp_f16_sdwa      :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`src0<amdgpu_synid9_src32_0>`::ref:`m<amdgpu_synid9_mod_dpp_sdwa_abs_neg>`,     :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`::ref:`i16<amdgpu_synid9_type_dev>`       :ref:`clamp<amdgpu_synid_clamp>` :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
+    v_lshlrev_b16         :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`src0<amdgpu_synid9_src32_0>`::ref:`u16<amdgpu_synid9_type_dev>`,   :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`
+    v_lshlrev_b16_dpp     :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid9_vsrc32_0>`::ref:`u16<amdgpu_synid9_type_dev>`,  :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`             :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
+    v_lshlrev_b16_sdwa    :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`src0<amdgpu_synid9_src32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`::ref:`u16<amdgpu_synid9_type_dev>`, :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`           :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
+    v_lshlrev_b32         :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`src0<amdgpu_synid9_src32_0>`::ref:`u32<amdgpu_synid9_type_dev>`,   :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`
+    v_lshlrev_b32_dpp     :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid9_vsrc32_0>`::ref:`u32<amdgpu_synid9_type_dev>`,  :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`             :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
+    v_lshlrev_b32_sdwa    :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`src0<amdgpu_synid9_src32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`::ref:`u32<amdgpu_synid9_type_dev>`, :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`           :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
+    v_lshrrev_b16         :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`src0<amdgpu_synid9_src32_0>`::ref:`u16<amdgpu_synid9_type_dev>`,   :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`
+    v_lshrrev_b16_dpp     :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid9_vsrc32_0>`::ref:`u16<amdgpu_synid9_type_dev>`,  :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`             :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
+    v_lshrrev_b16_sdwa    :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`src0<amdgpu_synid9_src32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`::ref:`u16<amdgpu_synid9_type_dev>`, :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`           :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
+    v_lshrrev_b32         :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`src0<amdgpu_synid9_src32_0>`::ref:`u32<amdgpu_synid9_type_dev>`,   :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`
+    v_lshrrev_b32_dpp     :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid9_vsrc32_0>`::ref:`u32<amdgpu_synid9_type_dev>`,  :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`             :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
+    v_lshrrev_b32_sdwa    :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`src0<amdgpu_synid9_src32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`::ref:`u32<amdgpu_synid9_type_dev>`, :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`           :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
+    v_mac_f16             :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`src0<amdgpu_synid9_src32_0>`,       :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`
+    v_mac_f16_dpp         :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_dpp_sdwa_abs_neg>`,    :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_dpp_sdwa_abs_neg>`           :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
+    v_mac_f32             :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`src0<amdgpu_synid9_src32_0>`,       :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`
+    v_mac_f32_dpp         :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_dpp_sdwa_abs_neg>`,    :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_dpp_sdwa_abs_neg>`           :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
+    v_madak_f16           :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`src0<amdgpu_synid9_src32_0>`,       :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`,      :ref:`imm32<amdgpu_synid9_fimm16>`
+    v_madak_f32           :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`src0<amdgpu_synid9_src32_0>`,       :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`,      :ref:`imm32<amdgpu_synid9_fimm32>`
+    v_madmk_f16           :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`src0<amdgpu_synid9_src32_0>`,       :ref:`imm32<amdgpu_synid9_fimm16>`,      :ref:`vsrc2<amdgpu_synid9_vsrc32_0>`
+    v_madmk_f32           :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`src0<amdgpu_synid9_src32_0>`,       :ref:`imm32<amdgpu_synid9_fimm32>`,      :ref:`vsrc2<amdgpu_synid9_vsrc32_0>`
+    v_max_f16             :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`src0<amdgpu_synid9_src32_0>`,       :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`
+    v_max_f16_dpp         :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_dpp_sdwa_abs_neg>`,    :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_dpp_sdwa_abs_neg>`           :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
+    v_max_f16_sdwa        :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`src0<amdgpu_synid9_src32_0>`::ref:`m<amdgpu_synid9_mod_dpp_sdwa_abs_neg>`,     :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_dpp_sdwa_abs_neg>`           :ref:`clamp<amdgpu_synid_clamp>` :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
+    v_max_f32             :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`src0<amdgpu_synid9_src32_0>`,       :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`
+    v_max_f32_dpp         :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_dpp_sdwa_abs_neg>`,    :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_dpp_sdwa_abs_neg>`           :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
+    v_max_f32_sdwa        :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`src0<amdgpu_synid9_src32_0>`::ref:`m<amdgpu_synid9_mod_dpp_sdwa_abs_neg>`,     :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_dpp_sdwa_abs_neg>`           :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>` :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
+    v_max_i16             :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`src0<amdgpu_synid9_src32_0>`,       :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`
+    v_max_i16_dpp         :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid9_vsrc32_0>`,      :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`             :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
+    v_max_i16_sdwa        :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`src0<amdgpu_synid9_src32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`,     :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`           :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
+    v_max_i32             :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`src0<amdgpu_synid9_src32_0>`,       :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`
+    v_max_i32_dpp         :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid9_vsrc32_0>`,      :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`             :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
+    v_max_i32_sdwa        :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`src0<amdgpu_synid9_src32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`,     :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`           :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
+    v_max_u16             :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`src0<amdgpu_synid9_src32_0>`,       :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`
+    v_max_u16_dpp         :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid9_vsrc32_0>`,      :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`             :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
+    v_max_u16_sdwa        :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`src0<amdgpu_synid9_src32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`,     :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`           :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
+    v_max_u32             :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`src0<amdgpu_synid9_src32_0>`,       :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`
+    v_max_u32_dpp         :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid9_vsrc32_0>`,      :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`             :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
+    v_max_u32_sdwa        :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`src0<amdgpu_synid9_src32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`,     :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`           :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
+    v_min_f16             :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`src0<amdgpu_synid9_src32_0>`,       :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`
+    v_min_f16_dpp         :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_dpp_sdwa_abs_neg>`,    :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_dpp_sdwa_abs_neg>`           :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
+    v_min_f16_sdwa        :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`src0<amdgpu_synid9_src32_0>`::ref:`m<amdgpu_synid9_mod_dpp_sdwa_abs_neg>`,     :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_dpp_sdwa_abs_neg>`           :ref:`clamp<amdgpu_synid_clamp>` :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
+    v_min_f32             :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`src0<amdgpu_synid9_src32_0>`,       :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`
+    v_min_f32_dpp         :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_dpp_sdwa_abs_neg>`,    :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_dpp_sdwa_abs_neg>`           :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
+    v_min_f32_sdwa        :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`src0<amdgpu_synid9_src32_0>`::ref:`m<amdgpu_synid9_mod_dpp_sdwa_abs_neg>`,     :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_dpp_sdwa_abs_neg>`           :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>` :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
+    v_min_i16             :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`src0<amdgpu_synid9_src32_0>`,       :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`
+    v_min_i16_dpp         :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid9_vsrc32_0>`,      :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`             :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
+    v_min_i16_sdwa        :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`src0<amdgpu_synid9_src32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`,     :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`           :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
+    v_min_i32             :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`src0<amdgpu_synid9_src32_0>`,       :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`
+    v_min_i32_dpp         :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid9_vsrc32_0>`,      :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`             :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
+    v_min_i32_sdwa        :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`src0<amdgpu_synid9_src32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`,     :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`           :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
+    v_min_u16             :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`src0<amdgpu_synid9_src32_0>`,       :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`
+    v_min_u16_dpp         :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid9_vsrc32_0>`,      :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`             :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
+    v_min_u16_sdwa        :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`src0<amdgpu_synid9_src32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`,     :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`           :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
+    v_min_u32             :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`src0<amdgpu_synid9_src32_0>`,       :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`
+    v_min_u32_dpp         :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid9_vsrc32_0>`,      :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`             :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
+    v_min_u32_sdwa        :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`src0<amdgpu_synid9_src32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`,     :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`           :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
+    v_mul_f16             :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`src0<amdgpu_synid9_src32_0>`,       :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`
+    v_mul_f16_dpp         :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_dpp_sdwa_abs_neg>`,    :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_dpp_sdwa_abs_neg>`           :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
+    v_mul_f16_sdwa        :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`src0<amdgpu_synid9_src32_0>`::ref:`m<amdgpu_synid9_mod_dpp_sdwa_abs_neg>`,     :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_dpp_sdwa_abs_neg>`           :ref:`clamp<amdgpu_synid_clamp>` :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
+    v_mul_f32             :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`src0<amdgpu_synid9_src32_0>`,       :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`
+    v_mul_f32_dpp         :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_dpp_sdwa_abs_neg>`,    :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_dpp_sdwa_abs_neg>`           :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
+    v_mul_f32_sdwa        :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`src0<amdgpu_synid9_src32_0>`::ref:`m<amdgpu_synid9_mod_dpp_sdwa_abs_neg>`,     :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_dpp_sdwa_abs_neg>`           :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>` :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
+    v_mul_hi_i32_i24      :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`src0<amdgpu_synid9_src32_0>`,       :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`
+    v_mul_hi_i32_i24_dpp  :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid9_vsrc32_0>`,      :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`             :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
+    v_mul_hi_i32_i24_sdwa :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`src0<amdgpu_synid9_src32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`,     :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`           :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
+    v_mul_hi_u32_u24      :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`src0<amdgpu_synid9_src32_0>`,       :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`
+    v_mul_hi_u32_u24_dpp  :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid9_vsrc32_0>`,      :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`             :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
+    v_mul_hi_u32_u24_sdwa :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`src0<amdgpu_synid9_src32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`,     :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`           :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
+    v_mul_i32_i24         :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`src0<amdgpu_synid9_src32_0>`,       :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`
+    v_mul_i32_i24_dpp     :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid9_vsrc32_0>`,      :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`             :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
+    v_mul_i32_i24_sdwa    :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`src0<amdgpu_synid9_src32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`,     :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`           :ref:`clamp<amdgpu_synid_clamp>` :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
+    v_mul_legacy_f32      :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`src0<amdgpu_synid9_src32_0>`,       :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`
+    v_mul_legacy_f32_dpp  :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_dpp_sdwa_abs_neg>`,    :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_dpp_sdwa_abs_neg>`           :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
+    v_mul_legacy_f32_sdwa :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`src0<amdgpu_synid9_src32_0>`::ref:`m<amdgpu_synid9_mod_dpp_sdwa_abs_neg>`,     :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_dpp_sdwa_abs_neg>`           :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>` :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
+    v_mul_lo_u16          :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`src0<amdgpu_synid9_src32_0>`,       :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`
+    v_mul_lo_u16_dpp      :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid9_vsrc32_0>`,      :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`             :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
+    v_mul_lo_u16_sdwa     :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`src0<amdgpu_synid9_src32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`,     :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`           :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
+    v_mul_u32_u24         :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`src0<amdgpu_synid9_src32_0>`,       :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`
+    v_mul_u32_u24_dpp     :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid9_vsrc32_0>`,      :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`             :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
+    v_mul_u32_u24_sdwa    :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`src0<amdgpu_synid9_src32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`,     :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`           :ref:`clamp<amdgpu_synid_clamp>` :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
+    v_or_b32              :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`src0<amdgpu_synid9_src32_0>`,       :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`
+    v_or_b32_dpp          :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid9_vsrc32_0>`,      :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`             :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
+    v_or_b32_sdwa         :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`src0<amdgpu_synid9_src32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`,     :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`           :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
+    v_sub_co_u32          :ref:`vdst<amdgpu_synid9_vdst32_0>`, :ref:`vcc<amdgpu_synid9_vcc_64>`, :ref:`src0<amdgpu_synid9_src32_0>`,       :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`
+    v_sub_co_u32_dpp      :ref:`vdst<amdgpu_synid9_vdst32_0>`, :ref:`vcc<amdgpu_synid9_vcc_64>`, :ref:`vsrc0<amdgpu_synid9_vsrc32_0>`,      :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`             :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
+    v_sub_co_u32_sdwa     :ref:`vdst<amdgpu_synid9_vdst32_0>`, :ref:`vcc<amdgpu_synid9_vcc_64>`, :ref:`src0<amdgpu_synid9_src32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`,     :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`           :ref:`clamp<amdgpu_synid_clamp>` :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
+    v_sub_f16             :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`src0<amdgpu_synid9_src32_0>`,       :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`
+    v_sub_f16_dpp         :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_dpp_sdwa_abs_neg>`,    :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_dpp_sdwa_abs_neg>`           :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
+    v_sub_f16_sdwa        :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`src0<amdgpu_synid9_src32_0>`::ref:`m<amdgpu_synid9_mod_dpp_sdwa_abs_neg>`,     :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_dpp_sdwa_abs_neg>`           :ref:`clamp<amdgpu_synid_clamp>` :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
+    v_sub_f32             :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`src0<amdgpu_synid9_src32_0>`,       :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`
+    v_sub_f32_dpp         :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_dpp_sdwa_abs_neg>`,    :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_dpp_sdwa_abs_neg>`           :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
+    v_sub_f32_sdwa        :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`src0<amdgpu_synid9_src32_0>`::ref:`m<amdgpu_synid9_mod_dpp_sdwa_abs_neg>`,     :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_dpp_sdwa_abs_neg>`           :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>` :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
+    v_sub_u16             :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`src0<amdgpu_synid9_src32_0>`,       :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`
+    v_sub_u16_dpp         :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid9_vsrc32_0>`,      :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`             :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
+    v_sub_u16_sdwa        :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`src0<amdgpu_synid9_src32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`,     :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`           :ref:`clamp<amdgpu_synid_clamp>` :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
+    v_sub_u32             :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`src0<amdgpu_synid9_src32_0>`,       :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`
+    v_sub_u32_dpp         :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid9_vsrc32_0>`,      :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`             :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
+    v_sub_u32_sdwa        :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`src0<amdgpu_synid9_src32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`,     :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`           :ref:`clamp<amdgpu_synid_clamp>` :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
+    v_subb_co_u32         :ref:`vdst<amdgpu_synid9_vdst32_0>`, :ref:`vcc<amdgpu_synid9_vcc_64>`, :ref:`src0<amdgpu_synid9_src32_0>`,       :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`,      :ref:`vcc<amdgpu_synid9_vcc_64>`
+    v_subb_co_u32_dpp     :ref:`vdst<amdgpu_synid9_vdst32_0>`, :ref:`vcc<amdgpu_synid9_vcc_64>`, :ref:`vsrc0<amdgpu_synid9_vsrc32_0>`,      :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`,      :ref:`vcc<amdgpu_synid9_vcc_64>`   :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
+    v_subb_co_u32_sdwa    :ref:`vdst<amdgpu_synid9_vdst32_0>`, :ref:`vcc<amdgpu_synid9_vcc_64>`, :ref:`src0<amdgpu_synid9_src32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`,     :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`,    :ref:`vcc<amdgpu_synid9_vcc_64>`   :ref:`clamp<amdgpu_synid_clamp>` :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
+    v_subbrev_co_u32      :ref:`vdst<amdgpu_synid9_vdst32_0>`, :ref:`vcc<amdgpu_synid9_vcc_64>`, :ref:`src0<amdgpu_synid9_src32_0>`,       :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`,      :ref:`vcc<amdgpu_synid9_vcc_64>`
+    v_subbrev_co_u32_dpp  :ref:`vdst<amdgpu_synid9_vdst32_0>`, :ref:`vcc<amdgpu_synid9_vcc_64>`, :ref:`vsrc0<amdgpu_synid9_vsrc32_0>`,      :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`,      :ref:`vcc<amdgpu_synid9_vcc_64>`   :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
+    v_subbrev_co_u32_sdwa :ref:`vdst<amdgpu_synid9_vdst32_0>`, :ref:`vcc<amdgpu_synid9_vcc_64>`, :ref:`src0<amdgpu_synid9_src32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`,     :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`,    :ref:`vcc<amdgpu_synid9_vcc_64>`   :ref:`clamp<amdgpu_synid_clamp>` :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
+    v_subrev_co_u32       :ref:`vdst<amdgpu_synid9_vdst32_0>`, :ref:`vcc<amdgpu_synid9_vcc_64>`, :ref:`src0<amdgpu_synid9_src32_0>`,       :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`
+    v_subrev_co_u32_dpp   :ref:`vdst<amdgpu_synid9_vdst32_0>`, :ref:`vcc<amdgpu_synid9_vcc_64>`, :ref:`vsrc0<amdgpu_synid9_vsrc32_0>`,      :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`             :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
+    v_subrev_co_u32_sdwa  :ref:`vdst<amdgpu_synid9_vdst32_0>`, :ref:`vcc<amdgpu_synid9_vcc_64>`, :ref:`src0<amdgpu_synid9_src32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`,     :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`           :ref:`clamp<amdgpu_synid_clamp>` :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
+    v_subrev_f16          :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`src0<amdgpu_synid9_src32_0>`,       :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`
+    v_subrev_f16_dpp      :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_dpp_sdwa_abs_neg>`,    :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_dpp_sdwa_abs_neg>`           :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
+    v_subrev_f16_sdwa     :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`src0<amdgpu_synid9_src32_0>`::ref:`m<amdgpu_synid9_mod_dpp_sdwa_abs_neg>`,     :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_dpp_sdwa_abs_neg>`           :ref:`clamp<amdgpu_synid_clamp>` :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
+    v_subrev_f32          :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`src0<amdgpu_synid9_src32_0>`,       :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`
+    v_subrev_f32_dpp      :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_dpp_sdwa_abs_neg>`,    :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_dpp_sdwa_abs_neg>`           :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
+    v_subrev_f32_sdwa     :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`src0<amdgpu_synid9_src32_0>`::ref:`m<amdgpu_synid9_mod_dpp_sdwa_abs_neg>`,     :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_dpp_sdwa_abs_neg>`           :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>` :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
+    v_subrev_u16          :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`src0<amdgpu_synid9_src32_0>`,       :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`
+    v_subrev_u16_dpp      :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid9_vsrc32_0>`,      :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`             :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
+    v_subrev_u16_sdwa     :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`src0<amdgpu_synid9_src32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`,     :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`           :ref:`clamp<amdgpu_synid_clamp>` :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
+    v_subrev_u32          :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`src0<amdgpu_synid9_src32_0>`,       :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`
+    v_subrev_u32_dpp      :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid9_vsrc32_0>`,      :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`             :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
+    v_subrev_u32_sdwa     :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`src0<amdgpu_synid9_src32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`,     :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`           :ref:`clamp<amdgpu_synid_clamp>` :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
+    v_xor_b32             :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`src0<amdgpu_synid9_src32_0>`,       :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`
+    v_xor_b32_dpp         :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`vsrc0<amdgpu_synid9_vsrc32_0>`,      :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`             :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` :ref:`row_mask<amdgpu_synid_row_mask>` :ref:`bank_mask<amdgpu_synid_bank_mask>` :ref:`bound_ctrl<amdgpu_synid_bound_ctrl>`
+    v_xor_b32_sdwa        :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`src0<amdgpu_synid9_src32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`,     :ref:`vsrc1<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_sdwa_sext>`           :ref:`dst_sel<amdgpu_synid_dst_sel>` :ref:`dst_unused<amdgpu_synid_dst_unused>` :ref:`src0_sel<amdgpu_synid_src0_sel>` :ref:`src1_sel<amdgpu_synid_src1_sel>`
 
 VOP3
 -----------------------
 
 .. parsed-literal::
 
-    **INSTRUCTION**                    **DST0**       **DST1**      **SRC0**         **SRC1**        **SRC2**               **MODIFIERS**
-    \ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|
-    v_add3_u32                     :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`,       :ref:`src2<amdgpu_synid9_src32_1>`
-    v_add_co_u32_e64               :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`sdst<amdgpu_synid9_sdst64_0>`,     :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
-    v_add_f16_e64                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_add_f32_e64                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_add_f64                      :ref:`vdst<amdgpu_synid9_vdst64_0>`,                :ref:`src0<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_add_i16                      :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`                           :ref:`vop3_op_sel<amdgpu_synid_vop3_op_sel>` :ref:`clamp<amdgpu_synid_clamp>`
-    v_add_i32                      :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
-    v_add_lshl_u32                 :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`,       :ref:`src2<amdgpu_synid9_src32_1>`
-    v_add_u16_e64                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
-    v_add_u32_e64                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
-    v_addc_co_u32_e64              :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`sdst<amdgpu_synid9_sdst64_0>`,     :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`,       :ref:`ssrc2<amdgpu_synid9_ssrc64_1>`
-    v_alignbit_b32                 :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`,       :ref:`src2<amdgpu_synid9_src32_1>`
-    v_alignbyte_b32                :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`,       :ref:`src2<amdgpu_synid9_src32_1>`
-    v_and_b32_e64                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
-    v_and_or_b32                   :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`,       :ref:`src2<amdgpu_synid9_src32_1>`
-    v_ashrrev_i16_e64              :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`::ref:`u16<amdgpu_synid9_type_dev>`,    :ref:`src1<amdgpu_synid9_src32_1>`
-    v_ashrrev_i32_e64              :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`::ref:`u32<amdgpu_synid9_type_dev>`,    :ref:`src1<amdgpu_synid9_src32_1>`
-    v_ashrrev_i64                  :ref:`vdst<amdgpu_synid9_vdst64_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`::ref:`u32<amdgpu_synid9_type_dev>`,    :ref:`src1<amdgpu_synid9_src64_1>`
-    v_bcnt_u32_b32                 :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
-    v_bfe_i32                      :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`::ref:`u32<amdgpu_synid9_type_dev>`,   :ref:`src2<amdgpu_synid9_src32_1>`::ref:`u32<amdgpu_synid9_type_dev>`
-    v_bfe_u32                      :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`,       :ref:`src2<amdgpu_synid9_src32_1>`
-    v_bfi_b32                      :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`,       :ref:`src2<amdgpu_synid9_src32_1>`
-    v_bfm_b32                      :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
-    v_bfrev_b32_e64                :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src<amdgpu_synid9_src32_1>`
-    v_ceil_f16_e64                 :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                                       :ref:`clamp<amdgpu_synid_clamp>`
-    v_ceil_f32_e64                 :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                                       :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_ceil_f64_e64                 :ref:`vdst<amdgpu_synid9_vdst64_0>`,                :ref:`src<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                                       :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    **INSTRUCTION**                    **DST0**       **DST1**     **SRC0**         **SRC1**        **SRC2**            **MODIFIERS**
+    \ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|
+    v_add3_u32                     :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`,       :ref:`src2<amdgpu_synid9_src32_1>`
+    v_add_co_u32_e64               :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`sdst<amdgpu_synid9_sdst64_0>`,    :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
+    v_add_f16_e64                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_add_f32_e64                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_add_f64                      :ref:`vdst<amdgpu_synid9_vdst64_0>`,               :ref:`src0<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_add_i16                      :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`                        :ref:`op_sel<amdgpu_synid_vop3_op_sel>` :ref:`clamp<amdgpu_synid_clamp>`
+    v_add_i32                      :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
+    v_add_lshl_u32                 :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`,       :ref:`src2<amdgpu_synid9_src32_1>`
+    v_add_u16_e64                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
+    v_add_u32_e64                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
+    v_addc_co_u32_e64              :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`sdst<amdgpu_synid9_sdst64_0>`,    :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`,       :ref:`ssrc2<amdgpu_synid9_ssrc64_1>`
+    v_alignbit_b32                 :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`,       :ref:`src2<amdgpu_synid9_src32_1>`
+    v_alignbyte_b32                :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`,       :ref:`src2<amdgpu_synid9_src32_1>`
+    v_and_b32_e64                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
+    v_and_or_b32                   :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`,       :ref:`src2<amdgpu_synid9_src32_1>`
+    v_ashrrev_i16_e64              :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`::ref:`u16<amdgpu_synid9_type_dev>`,    :ref:`src1<amdgpu_synid9_src32_1>`
+    v_ashrrev_i32_e64              :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`::ref:`u32<amdgpu_synid9_type_dev>`,    :ref:`src1<amdgpu_synid9_src32_1>`
+    v_ashrrev_i64                  :ref:`vdst<amdgpu_synid9_vdst64_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`::ref:`u32<amdgpu_synid9_type_dev>`,    :ref:`src1<amdgpu_synid9_src64_1>`
+    v_bcnt_u32_b32                 :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
+    v_bfe_i32                      :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`::ref:`u32<amdgpu_synid9_type_dev>`,   :ref:`src2<amdgpu_synid9_src32_1>`::ref:`u32<amdgpu_synid9_type_dev>`
+    v_bfe_u32                      :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`,       :ref:`src2<amdgpu_synid9_src32_1>`
+    v_bfi_b32                      :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`,       :ref:`src2<amdgpu_synid9_src32_1>`
+    v_bfm_b32                      :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
+    v_bfrev_b32_e64                :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src<amdgpu_synid9_src32_1>`
+    v_ceil_f16_e64                 :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                                    :ref:`clamp<amdgpu_synid_clamp>`
+    v_ceil_f32_e64                 :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                                    :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_ceil_f64_e64                 :ref:`vdst<amdgpu_synid9_vdst64_0>`,               :ref:`src<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                                    :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
     v_clrexcp_e64
-    v_cmp_class_f16_e64            :ref:`sdst<amdgpu_synid9_sdst64_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`b32<amdgpu_synid9_type_dev>`
-    v_cmp_class_f32_e64            :ref:`sdst<amdgpu_synid9_sdst64_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`b32<amdgpu_synid9_type_dev>`
-    v_cmp_class_f64_e64            :ref:`sdst<amdgpu_synid9_sdst64_0>`,                :ref:`src0<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`b32<amdgpu_synid9_type_dev>`
-    v_cmp_eq_f16_e64               :ref:`sdst<amdgpu_synid9_sdst64_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmp_eq_f32_e64               :ref:`sdst<amdgpu_synid9_sdst64_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmp_eq_f64_e64               :ref:`sdst<amdgpu_synid9_sdst64_0>`,                :ref:`src0<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmp_eq_i16_e64               :ref:`sdst<amdgpu_synid9_sdst64_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
-    v_cmp_eq_i32_e64               :ref:`sdst<amdgpu_synid9_sdst64_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
-    v_cmp_eq_i64_e64               :ref:`sdst<amdgpu_synid9_sdst64_0>`,                :ref:`src0<amdgpu_synid9_src64_1>`,        :ref:`src1<amdgpu_synid9_src64_1>`
-    v_cmp_eq_u16_e64               :ref:`sdst<amdgpu_synid9_sdst64_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
-    v_cmp_eq_u32_e64               :ref:`sdst<amdgpu_synid9_sdst64_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
-    v_cmp_eq_u64_e64               :ref:`sdst<amdgpu_synid9_sdst64_0>`,                :ref:`src0<amdgpu_synid9_src64_1>`,        :ref:`src1<amdgpu_synid9_src64_1>`
-    v_cmp_f_f16_e64                :ref:`sdst<amdgpu_synid9_sdst64_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmp_f_f32_e64                :ref:`sdst<amdgpu_synid9_sdst64_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmp_f_f64_e64                :ref:`sdst<amdgpu_synid9_sdst64_0>`,                :ref:`src0<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmp_f_i16_e64                :ref:`sdst<amdgpu_synid9_sdst64_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
-    v_cmp_f_i32_e64                :ref:`sdst<amdgpu_synid9_sdst64_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
-    v_cmp_f_i64_e64                :ref:`sdst<amdgpu_synid9_sdst64_0>`,                :ref:`src0<amdgpu_synid9_src64_1>`,        :ref:`src1<amdgpu_synid9_src64_1>`
-    v_cmp_f_u16_e64                :ref:`sdst<amdgpu_synid9_sdst64_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
-    v_cmp_f_u32_e64                :ref:`sdst<amdgpu_synid9_sdst64_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
-    v_cmp_f_u64_e64                :ref:`sdst<amdgpu_synid9_sdst64_0>`,                :ref:`src0<amdgpu_synid9_src64_1>`,        :ref:`src1<amdgpu_synid9_src64_1>`
-    v_cmp_ge_f16_e64               :ref:`sdst<amdgpu_synid9_sdst64_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmp_ge_f32_e64               :ref:`sdst<amdgpu_synid9_sdst64_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmp_ge_f64_e64               :ref:`sdst<amdgpu_synid9_sdst64_0>`,                :ref:`src0<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmp_ge_i16_e64               :ref:`sdst<amdgpu_synid9_sdst64_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
-    v_cmp_ge_i32_e64               :ref:`sdst<amdgpu_synid9_sdst64_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
-    v_cmp_ge_i64_e64               :ref:`sdst<amdgpu_synid9_sdst64_0>`,                :ref:`src0<amdgpu_synid9_src64_1>`,        :ref:`src1<amdgpu_synid9_src64_1>`
-    v_cmp_ge_u16_e64               :ref:`sdst<amdgpu_synid9_sdst64_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
-    v_cmp_ge_u32_e64               :ref:`sdst<amdgpu_synid9_sdst64_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
-    v_cmp_ge_u64_e64               :ref:`sdst<amdgpu_synid9_sdst64_0>`,                :ref:`src0<amdgpu_synid9_src64_1>`,        :ref:`src1<amdgpu_synid9_src64_1>`
-    v_cmp_gt_f16_e64               :ref:`sdst<amdgpu_synid9_sdst64_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmp_gt_f32_e64               :ref:`sdst<amdgpu_synid9_sdst64_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmp_gt_f64_e64               :ref:`sdst<amdgpu_synid9_sdst64_0>`,                :ref:`src0<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmp_gt_i16_e64               :ref:`sdst<amdgpu_synid9_sdst64_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
-    v_cmp_gt_i32_e64               :ref:`sdst<amdgpu_synid9_sdst64_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
-    v_cmp_gt_i64_e64               :ref:`sdst<amdgpu_synid9_sdst64_0>`,                :ref:`src0<amdgpu_synid9_src64_1>`,        :ref:`src1<amdgpu_synid9_src64_1>`
-    v_cmp_gt_u16_e64               :ref:`sdst<amdgpu_synid9_sdst64_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
-    v_cmp_gt_u32_e64               :ref:`sdst<amdgpu_synid9_sdst64_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
-    v_cmp_gt_u64_e64               :ref:`sdst<amdgpu_synid9_sdst64_0>`,                :ref:`src0<amdgpu_synid9_src64_1>`,        :ref:`src1<amdgpu_synid9_src64_1>`
-    v_cmp_le_f16_e64               :ref:`sdst<amdgpu_synid9_sdst64_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmp_le_f32_e64               :ref:`sdst<amdgpu_synid9_sdst64_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmp_le_f64_e64               :ref:`sdst<amdgpu_synid9_sdst64_0>`,                :ref:`src0<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmp_le_i16_e64               :ref:`sdst<amdgpu_synid9_sdst64_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
-    v_cmp_le_i32_e64               :ref:`sdst<amdgpu_synid9_sdst64_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
-    v_cmp_le_i64_e64               :ref:`sdst<amdgpu_synid9_sdst64_0>`,                :ref:`src0<amdgpu_synid9_src64_1>`,        :ref:`src1<amdgpu_synid9_src64_1>`
-    v_cmp_le_u16_e64               :ref:`sdst<amdgpu_synid9_sdst64_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
-    v_cmp_le_u32_e64               :ref:`sdst<amdgpu_synid9_sdst64_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
-    v_cmp_le_u64_e64               :ref:`sdst<amdgpu_synid9_sdst64_0>`,                :ref:`src0<amdgpu_synid9_src64_1>`,        :ref:`src1<amdgpu_synid9_src64_1>`
-    v_cmp_lg_f16_e64               :ref:`sdst<amdgpu_synid9_sdst64_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmp_lg_f32_e64               :ref:`sdst<amdgpu_synid9_sdst64_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmp_lg_f64_e64               :ref:`sdst<amdgpu_synid9_sdst64_0>`,                :ref:`src0<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmp_lt_f16_e64               :ref:`sdst<amdgpu_synid9_sdst64_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmp_lt_f32_e64               :ref:`sdst<amdgpu_synid9_sdst64_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmp_lt_f64_e64               :ref:`sdst<amdgpu_synid9_sdst64_0>`,                :ref:`src0<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmp_lt_i16_e64               :ref:`sdst<amdgpu_synid9_sdst64_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
-    v_cmp_lt_i32_e64               :ref:`sdst<amdgpu_synid9_sdst64_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
-    v_cmp_lt_i64_e64               :ref:`sdst<amdgpu_synid9_sdst64_0>`,                :ref:`src0<amdgpu_synid9_src64_1>`,        :ref:`src1<amdgpu_synid9_src64_1>`
-    v_cmp_lt_u16_e64               :ref:`sdst<amdgpu_synid9_sdst64_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
-    v_cmp_lt_u32_e64               :ref:`sdst<amdgpu_synid9_sdst64_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
-    v_cmp_lt_u64_e64               :ref:`sdst<amdgpu_synid9_sdst64_0>`,                :ref:`src0<amdgpu_synid9_src64_1>`,        :ref:`src1<amdgpu_synid9_src64_1>`
-    v_cmp_ne_i16_e64               :ref:`sdst<amdgpu_synid9_sdst64_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
-    v_cmp_ne_i32_e64               :ref:`sdst<amdgpu_synid9_sdst64_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
-    v_cmp_ne_i64_e64               :ref:`sdst<amdgpu_synid9_sdst64_0>`,                :ref:`src0<amdgpu_synid9_src64_1>`,        :ref:`src1<amdgpu_synid9_src64_1>`
-    v_cmp_ne_u16_e64               :ref:`sdst<amdgpu_synid9_sdst64_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
-    v_cmp_ne_u32_e64               :ref:`sdst<amdgpu_synid9_sdst64_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
-    v_cmp_ne_u64_e64               :ref:`sdst<amdgpu_synid9_sdst64_0>`,                :ref:`src0<amdgpu_synid9_src64_1>`,        :ref:`src1<amdgpu_synid9_src64_1>`
-    v_cmp_neq_f16_e64              :ref:`sdst<amdgpu_synid9_sdst64_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmp_neq_f32_e64              :ref:`sdst<amdgpu_synid9_sdst64_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmp_neq_f64_e64              :ref:`sdst<amdgpu_synid9_sdst64_0>`,                :ref:`src0<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmp_nge_f16_e64              :ref:`sdst<amdgpu_synid9_sdst64_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmp_nge_f32_e64              :ref:`sdst<amdgpu_synid9_sdst64_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmp_nge_f64_e64              :ref:`sdst<amdgpu_synid9_sdst64_0>`,                :ref:`src0<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmp_ngt_f16_e64              :ref:`sdst<amdgpu_synid9_sdst64_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmp_ngt_f32_e64              :ref:`sdst<amdgpu_synid9_sdst64_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmp_ngt_f64_e64              :ref:`sdst<amdgpu_synid9_sdst64_0>`,                :ref:`src0<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmp_nle_f16_e64              :ref:`sdst<amdgpu_synid9_sdst64_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmp_nle_f32_e64              :ref:`sdst<amdgpu_synid9_sdst64_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmp_nle_f64_e64              :ref:`sdst<amdgpu_synid9_sdst64_0>`,                :ref:`src0<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmp_nlg_f16_e64              :ref:`sdst<amdgpu_synid9_sdst64_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmp_nlg_f32_e64              :ref:`sdst<amdgpu_synid9_sdst64_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmp_nlg_f64_e64              :ref:`sdst<amdgpu_synid9_sdst64_0>`,                :ref:`src0<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmp_nlt_f16_e64              :ref:`sdst<amdgpu_synid9_sdst64_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmp_nlt_f32_e64              :ref:`sdst<amdgpu_synid9_sdst64_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmp_nlt_f64_e64              :ref:`sdst<amdgpu_synid9_sdst64_0>`,                :ref:`src0<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmp_o_f16_e64                :ref:`sdst<amdgpu_synid9_sdst64_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmp_o_f32_e64                :ref:`sdst<amdgpu_synid9_sdst64_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmp_o_f64_e64                :ref:`sdst<amdgpu_synid9_sdst64_0>`,                :ref:`src0<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmp_t_i16_e64                :ref:`sdst<amdgpu_synid9_sdst64_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
-    v_cmp_t_i32_e64                :ref:`sdst<amdgpu_synid9_sdst64_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
-    v_cmp_t_i64_e64                :ref:`sdst<amdgpu_synid9_sdst64_0>`,                :ref:`src0<amdgpu_synid9_src64_1>`,        :ref:`src1<amdgpu_synid9_src64_1>`
-    v_cmp_t_u16_e64                :ref:`sdst<amdgpu_synid9_sdst64_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
-    v_cmp_t_u32_e64                :ref:`sdst<amdgpu_synid9_sdst64_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
-    v_cmp_t_u64_e64                :ref:`sdst<amdgpu_synid9_sdst64_0>`,                :ref:`src0<amdgpu_synid9_src64_1>`,        :ref:`src1<amdgpu_synid9_src64_1>`
-    v_cmp_tru_f16_e64              :ref:`sdst<amdgpu_synid9_sdst64_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmp_tru_f32_e64              :ref:`sdst<amdgpu_synid9_sdst64_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmp_tru_f64_e64              :ref:`sdst<amdgpu_synid9_sdst64_0>`,                :ref:`src0<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmp_u_f16_e64                :ref:`sdst<amdgpu_synid9_sdst64_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmp_u_f32_e64                :ref:`sdst<amdgpu_synid9_sdst64_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmp_u_f64_e64                :ref:`sdst<amdgpu_synid9_sdst64_0>`,                :ref:`src0<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmpx_class_f16_e64           :ref:`sdst<amdgpu_synid9_sdst64_1>`,                :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`b32<amdgpu_synid9_type_dev>`
-    v_cmpx_class_f32_e64           :ref:`sdst<amdgpu_synid9_sdst64_1>`,                :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`b32<amdgpu_synid9_type_dev>`
-    v_cmpx_class_f64_e64           :ref:`sdst<amdgpu_synid9_sdst64_1>`,                :ref:`src0<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`b32<amdgpu_synid9_type_dev>`
-    v_cmpx_eq_f16_e64              :ref:`sdst<amdgpu_synid9_sdst64_1>`,                :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmpx_eq_f32_e64              :ref:`sdst<amdgpu_synid9_sdst64_1>`,                :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmpx_eq_f64_e64              :ref:`sdst<amdgpu_synid9_sdst64_1>`,                :ref:`src0<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmpx_eq_i16_e64              :ref:`sdst<amdgpu_synid9_sdst64_1>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
-    v_cmpx_eq_i32_e64              :ref:`sdst<amdgpu_synid9_sdst64_1>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
-    v_cmpx_eq_i64_e64              :ref:`sdst<amdgpu_synid9_sdst64_1>`,                :ref:`src0<amdgpu_synid9_src64_1>`,        :ref:`src1<amdgpu_synid9_src64_1>`
-    v_cmpx_eq_u16_e64              :ref:`sdst<amdgpu_synid9_sdst64_1>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
-    v_cmpx_eq_u32_e64              :ref:`sdst<amdgpu_synid9_sdst64_1>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
-    v_cmpx_eq_u64_e64              :ref:`sdst<amdgpu_synid9_sdst64_1>`,                :ref:`src0<amdgpu_synid9_src64_1>`,        :ref:`src1<amdgpu_synid9_src64_1>`
-    v_cmpx_f_f16_e64               :ref:`sdst<amdgpu_synid9_sdst64_1>`,                :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmpx_f_f32_e64               :ref:`sdst<amdgpu_synid9_sdst64_1>`,                :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmpx_f_f64_e64               :ref:`sdst<amdgpu_synid9_sdst64_1>`,                :ref:`src0<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmpx_f_i16_e64               :ref:`sdst<amdgpu_synid9_sdst64_1>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
-    v_cmpx_f_i32_e64               :ref:`sdst<amdgpu_synid9_sdst64_1>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
-    v_cmpx_f_i64_e64               :ref:`sdst<amdgpu_synid9_sdst64_1>`,                :ref:`src0<amdgpu_synid9_src64_1>`,        :ref:`src1<amdgpu_synid9_src64_1>`
-    v_cmpx_f_u16_e64               :ref:`sdst<amdgpu_synid9_sdst64_1>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
-    v_cmpx_f_u32_e64               :ref:`sdst<amdgpu_synid9_sdst64_1>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
-    v_cmpx_f_u64_e64               :ref:`sdst<amdgpu_synid9_sdst64_1>`,                :ref:`src0<amdgpu_synid9_src64_1>`,        :ref:`src1<amdgpu_synid9_src64_1>`
-    v_cmpx_ge_f16_e64              :ref:`sdst<amdgpu_synid9_sdst64_1>`,                :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmpx_ge_f32_e64              :ref:`sdst<amdgpu_synid9_sdst64_1>`,                :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmpx_ge_f64_e64              :ref:`sdst<amdgpu_synid9_sdst64_1>`,                :ref:`src0<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmpx_ge_i16_e64              :ref:`sdst<amdgpu_synid9_sdst64_1>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
-    v_cmpx_ge_i32_e64              :ref:`sdst<amdgpu_synid9_sdst64_1>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
-    v_cmpx_ge_i64_e64              :ref:`sdst<amdgpu_synid9_sdst64_1>`,                :ref:`src0<amdgpu_synid9_src64_1>`,        :ref:`src1<amdgpu_synid9_src64_1>`
-    v_cmpx_ge_u16_e64              :ref:`sdst<amdgpu_synid9_sdst64_1>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
-    v_cmpx_ge_u32_e64              :ref:`sdst<amdgpu_synid9_sdst64_1>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
-    v_cmpx_ge_u64_e64              :ref:`sdst<amdgpu_synid9_sdst64_1>`,                :ref:`src0<amdgpu_synid9_src64_1>`,        :ref:`src1<amdgpu_synid9_src64_1>`
-    v_cmpx_gt_f16_e64              :ref:`sdst<amdgpu_synid9_sdst64_1>`,                :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmpx_gt_f32_e64              :ref:`sdst<amdgpu_synid9_sdst64_1>`,                :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmpx_gt_f64_e64              :ref:`sdst<amdgpu_synid9_sdst64_1>`,                :ref:`src0<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmpx_gt_i16_e64              :ref:`sdst<amdgpu_synid9_sdst64_1>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
-    v_cmpx_gt_i32_e64              :ref:`sdst<amdgpu_synid9_sdst64_1>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
-    v_cmpx_gt_i64_e64              :ref:`sdst<amdgpu_synid9_sdst64_1>`,                :ref:`src0<amdgpu_synid9_src64_1>`,        :ref:`src1<amdgpu_synid9_src64_1>`
-    v_cmpx_gt_u16_e64              :ref:`sdst<amdgpu_synid9_sdst64_1>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
-    v_cmpx_gt_u32_e64              :ref:`sdst<amdgpu_synid9_sdst64_1>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
-    v_cmpx_gt_u64_e64              :ref:`sdst<amdgpu_synid9_sdst64_1>`,                :ref:`src0<amdgpu_synid9_src64_1>`,        :ref:`src1<amdgpu_synid9_src64_1>`
-    v_cmpx_le_f16_e64              :ref:`sdst<amdgpu_synid9_sdst64_1>`,                :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmpx_le_f32_e64              :ref:`sdst<amdgpu_synid9_sdst64_1>`,                :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmpx_le_f64_e64              :ref:`sdst<amdgpu_synid9_sdst64_1>`,                :ref:`src0<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmpx_le_i16_e64              :ref:`sdst<amdgpu_synid9_sdst64_1>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
-    v_cmpx_le_i32_e64              :ref:`sdst<amdgpu_synid9_sdst64_1>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
-    v_cmpx_le_i64_e64              :ref:`sdst<amdgpu_synid9_sdst64_1>`,                :ref:`src0<amdgpu_synid9_src64_1>`,        :ref:`src1<amdgpu_synid9_src64_1>`
-    v_cmpx_le_u16_e64              :ref:`sdst<amdgpu_synid9_sdst64_1>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
-    v_cmpx_le_u32_e64              :ref:`sdst<amdgpu_synid9_sdst64_1>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
-    v_cmpx_le_u64_e64              :ref:`sdst<amdgpu_synid9_sdst64_1>`,                :ref:`src0<amdgpu_synid9_src64_1>`,        :ref:`src1<amdgpu_synid9_src64_1>`
-    v_cmpx_lg_f16_e64              :ref:`sdst<amdgpu_synid9_sdst64_1>`,                :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmpx_lg_f32_e64              :ref:`sdst<amdgpu_synid9_sdst64_1>`,                :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmpx_lg_f64_e64              :ref:`sdst<amdgpu_synid9_sdst64_1>`,                :ref:`src0<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmpx_lt_f16_e64              :ref:`sdst<amdgpu_synid9_sdst64_1>`,                :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmpx_lt_f32_e64              :ref:`sdst<amdgpu_synid9_sdst64_1>`,                :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmpx_lt_f64_e64              :ref:`sdst<amdgpu_synid9_sdst64_1>`,                :ref:`src0<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmpx_lt_i16_e64              :ref:`sdst<amdgpu_synid9_sdst64_1>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
-    v_cmpx_lt_i32_e64              :ref:`sdst<amdgpu_synid9_sdst64_1>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
-    v_cmpx_lt_i64_e64              :ref:`sdst<amdgpu_synid9_sdst64_1>`,                :ref:`src0<amdgpu_synid9_src64_1>`,        :ref:`src1<amdgpu_synid9_src64_1>`
-    v_cmpx_lt_u16_e64              :ref:`sdst<amdgpu_synid9_sdst64_1>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
-    v_cmpx_lt_u32_e64              :ref:`sdst<amdgpu_synid9_sdst64_1>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
-    v_cmpx_lt_u64_e64              :ref:`sdst<amdgpu_synid9_sdst64_1>`,                :ref:`src0<amdgpu_synid9_src64_1>`,        :ref:`src1<amdgpu_synid9_src64_1>`
-    v_cmpx_ne_i16_e64              :ref:`sdst<amdgpu_synid9_sdst64_1>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
-    v_cmpx_ne_i32_e64              :ref:`sdst<amdgpu_synid9_sdst64_1>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
-    v_cmpx_ne_i64_e64              :ref:`sdst<amdgpu_synid9_sdst64_1>`,                :ref:`src0<amdgpu_synid9_src64_1>`,        :ref:`src1<amdgpu_synid9_src64_1>`
-    v_cmpx_ne_u16_e64              :ref:`sdst<amdgpu_synid9_sdst64_1>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
-    v_cmpx_ne_u32_e64              :ref:`sdst<amdgpu_synid9_sdst64_1>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
-    v_cmpx_ne_u64_e64              :ref:`sdst<amdgpu_synid9_sdst64_1>`,                :ref:`src0<amdgpu_synid9_src64_1>`,        :ref:`src1<amdgpu_synid9_src64_1>`
-    v_cmpx_neq_f16_e64             :ref:`sdst<amdgpu_synid9_sdst64_1>`,                :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmpx_neq_f32_e64             :ref:`sdst<amdgpu_synid9_sdst64_1>`,                :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmpx_neq_f64_e64             :ref:`sdst<amdgpu_synid9_sdst64_1>`,                :ref:`src0<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmpx_nge_f16_e64             :ref:`sdst<amdgpu_synid9_sdst64_1>`,                :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmpx_nge_f32_e64             :ref:`sdst<amdgpu_synid9_sdst64_1>`,                :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmpx_nge_f64_e64             :ref:`sdst<amdgpu_synid9_sdst64_1>`,                :ref:`src0<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmpx_ngt_f16_e64             :ref:`sdst<amdgpu_synid9_sdst64_1>`,                :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmpx_ngt_f32_e64             :ref:`sdst<amdgpu_synid9_sdst64_1>`,                :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmpx_ngt_f64_e64             :ref:`sdst<amdgpu_synid9_sdst64_1>`,                :ref:`src0<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmpx_nle_f16_e64             :ref:`sdst<amdgpu_synid9_sdst64_1>`,                :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmpx_nle_f32_e64             :ref:`sdst<amdgpu_synid9_sdst64_1>`,                :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmpx_nle_f64_e64             :ref:`sdst<amdgpu_synid9_sdst64_1>`,                :ref:`src0<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmpx_nlg_f16_e64             :ref:`sdst<amdgpu_synid9_sdst64_1>`,                :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmpx_nlg_f32_e64             :ref:`sdst<amdgpu_synid9_sdst64_1>`,                :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmpx_nlg_f64_e64             :ref:`sdst<amdgpu_synid9_sdst64_1>`,                :ref:`src0<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmpx_nlt_f16_e64             :ref:`sdst<amdgpu_synid9_sdst64_1>`,                :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmpx_nlt_f32_e64             :ref:`sdst<amdgpu_synid9_sdst64_1>`,                :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmpx_nlt_f64_e64             :ref:`sdst<amdgpu_synid9_sdst64_1>`,                :ref:`src0<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmpx_o_f16_e64               :ref:`sdst<amdgpu_synid9_sdst64_1>`,                :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmpx_o_f32_e64               :ref:`sdst<amdgpu_synid9_sdst64_1>`,                :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmpx_o_f64_e64               :ref:`sdst<amdgpu_synid9_sdst64_1>`,                :ref:`src0<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmpx_t_i16_e64               :ref:`sdst<amdgpu_synid9_sdst64_1>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
-    v_cmpx_t_i32_e64               :ref:`sdst<amdgpu_synid9_sdst64_1>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
-    v_cmpx_t_i64_e64               :ref:`sdst<amdgpu_synid9_sdst64_1>`,                :ref:`src0<amdgpu_synid9_src64_1>`,        :ref:`src1<amdgpu_synid9_src64_1>`
-    v_cmpx_t_u16_e64               :ref:`sdst<amdgpu_synid9_sdst64_1>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
-    v_cmpx_t_u32_e64               :ref:`sdst<amdgpu_synid9_sdst64_1>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
-    v_cmpx_t_u64_e64               :ref:`sdst<amdgpu_synid9_sdst64_1>`,                :ref:`src0<amdgpu_synid9_src64_1>`,        :ref:`src1<amdgpu_synid9_src64_1>`
-    v_cmpx_tru_f16_e64             :ref:`sdst<amdgpu_synid9_sdst64_1>`,                :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmpx_tru_f32_e64             :ref:`sdst<amdgpu_synid9_sdst64_1>`,                :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmpx_tru_f64_e64             :ref:`sdst<amdgpu_synid9_sdst64_1>`,                :ref:`src0<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmpx_u_f16_e64               :ref:`sdst<amdgpu_synid9_sdst64_1>`,                :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmpx_u_f32_e64               :ref:`sdst<amdgpu_synid9_sdst64_1>`,                :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cmpx_u_f64_e64               :ref:`sdst<amdgpu_synid9_sdst64_1>`,                :ref:`src0<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cndmask_b32_e64              :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`,       :ref:`ssrc2<amdgpu_synid9_ssrc64_1>`
-    v_cos_f16_e64                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                                       :ref:`clamp<amdgpu_synid_clamp>`
-    v_cos_f32_e64                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                                       :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_cubeid_f32                   :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,     :ref:`src2<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`             :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_cubema_f32                   :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,     :ref:`src2<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`             :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_cubesc_f32                   :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,     :ref:`src2<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`             :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_cubetc_f32                   :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,     :ref:`src2<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`             :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_cvt_f16_f32_e64              :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                                       :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_cvt_f16_i16_e64              :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src<amdgpu_synid9_src32_1>`                                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cvt_f16_u16_e64              :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src<amdgpu_synid9_src32_1>`                                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_cvt_f32_f16_e64              :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                                       :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_cvt_f32_f64_e64              :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                                       :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_cvt_f32_i32_e64              :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src<amdgpu_synid9_src32_1>`                                         :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_cvt_f32_u32_e64              :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src<amdgpu_synid9_src32_1>`                                         :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_cvt_f32_ubyte0_e64           :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src<amdgpu_synid9_src32_1>`                                         :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_cvt_f32_ubyte1_e64           :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src<amdgpu_synid9_src32_1>`                                         :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_cvt_f32_ubyte2_e64           :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src<amdgpu_synid9_src32_1>`                                         :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_cvt_f32_ubyte3_e64           :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src<amdgpu_synid9_src32_1>`                                         :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_cvt_f64_f32_e64              :ref:`vdst<amdgpu_synid9_vdst64_0>`,                :ref:`src<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                                       :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_cvt_f64_i32_e64              :ref:`vdst<amdgpu_synid9_vdst64_0>`,                :ref:`src<amdgpu_synid9_src32_1>`                                         :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_cvt_f64_u32_e64              :ref:`vdst<amdgpu_synid9_vdst64_0>`,                :ref:`src<amdgpu_synid9_src32_1>`                                         :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_cvt_flr_i32_f32_e64          :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`
-    v_cvt_i16_f16_e64              :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                                       :ref:`clamp<amdgpu_synid_clamp>`
-    v_cvt_i32_f32_e64              :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                                       :ref:`clamp<amdgpu_synid_clamp>`
-    v_cvt_i32_f64_e64              :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                                       :ref:`clamp<amdgpu_synid_clamp>`
-    v_cvt_norm_i16_f16_e64         :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`
-    v_cvt_norm_u16_f16_e64         :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`
-    v_cvt_off_f32_i4_e64           :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src<amdgpu_synid9_src32_1>`                                         :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_cvt_pk_i16_i32               :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
-    v_cvt_pk_u16_u32               :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
-    v_cvt_pk_u8_f32                :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`u32<amdgpu_synid9_type_dev>`,   :ref:`src2<amdgpu_synid9_src32_1>`::ref:`u32<amdgpu_synid9_type_dev>`
-    v_cvt_pkaccum_u8_f32           :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`u32<amdgpu_synid9_type_dev>`
-    v_cvt_pknorm_i16_f16           :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                         :ref:`vop3_op_sel<amdgpu_synid_vop3_op_sel>`
-    v_cvt_pknorm_i16_f32           :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`
-    v_cvt_pknorm_u16_f16           :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                         :ref:`vop3_op_sel<amdgpu_synid_vop3_op_sel>`
-    v_cvt_pknorm_u16_f32           :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`
-    v_cvt_pkrtz_f16_f32            :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`
-    v_cvt_rpi_i32_f32_e64          :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`
-    v_cvt_u16_f16_e64              :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                                       :ref:`clamp<amdgpu_synid_clamp>`
-    v_cvt_u32_f32_e64              :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                                       :ref:`clamp<amdgpu_synid_clamp>`
-    v_cvt_u32_f64_e64              :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                                       :ref:`clamp<amdgpu_synid_clamp>`
-    v_div_fixup_f16                :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,     :ref:`src2<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`             :ref:`vop3_op_sel<amdgpu_synid_vop3_op_sel>` :ref:`clamp<amdgpu_synid_clamp>`
-    v_div_fixup_f32                :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,     :ref:`src2<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`             :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_div_fixup_f64                :ref:`vdst<amdgpu_synid9_vdst64_0>`,                :ref:`src0<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,     :ref:`src2<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`             :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_div_fixup_legacy_f16         :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,     :ref:`src2<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`             :ref:`clamp<amdgpu_synid_clamp>`
-    v_div_fmas_f32                 :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,     :ref:`src2<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`             :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_div_fmas_f64                 :ref:`vdst<amdgpu_synid9_vdst64_0>`,                :ref:`src0<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,     :ref:`src2<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`             :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_div_scale_f32                :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`vcc<amdgpu_synid9_vcc_64>`,      :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`,       :ref:`src2<amdgpu_synid9_src32_1>`
-    v_div_scale_f64                :ref:`vdst<amdgpu_synid9_vdst64_0>`,      :ref:`vcc<amdgpu_synid9_vcc_64>`,      :ref:`src0<amdgpu_synid9_src64_1>`,        :ref:`src1<amdgpu_synid9_src64_1>`,       :ref:`src2<amdgpu_synid9_src64_1>`
-    v_exp_f16_e64                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                                       :ref:`clamp<amdgpu_synid_clamp>`
-    v_exp_f32_e64                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                                       :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_exp_legacy_f32_e64           :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                                       :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_ffbh_i32_e64                 :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src<amdgpu_synid9_src32_1>`
-    v_ffbh_u32_e64                 :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src<amdgpu_synid9_src32_1>`
-    v_ffbl_b32_e64                 :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src<amdgpu_synid9_src32_1>`
-    v_floor_f16_e64                :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                                       :ref:`clamp<amdgpu_synid_clamp>`
-    v_floor_f32_e64                :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                                       :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_floor_f64_e64                :ref:`vdst<amdgpu_synid9_vdst64_0>`,                :ref:`src<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                                       :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_fma_f16                      :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,     :ref:`src2<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`             :ref:`vop3_op_sel<amdgpu_synid_vop3_op_sel>` :ref:`clamp<amdgpu_synid_clamp>`
-    v_fma_f32                      :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,     :ref:`src2<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`             :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_fma_f64                      :ref:`vdst<amdgpu_synid9_vdst64_0>`,                :ref:`src0<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,     :ref:`src2<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`             :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_fma_legacy_f16               :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,     :ref:`src2<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`             :ref:`clamp<amdgpu_synid_clamp>`
-    v_fract_f16_e64                :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                                       :ref:`clamp<amdgpu_synid_clamp>`
-    v_fract_f32_e64                :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                                       :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_fract_f64_e64                :ref:`vdst<amdgpu_synid9_vdst64_0>`,                :ref:`src<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                                       :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_frexp_exp_i16_f16_e64        :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`
-    v_frexp_exp_i32_f32_e64        :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`
-    v_frexp_exp_i32_f64_e64        :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`
-    v_frexp_mant_f16_e64           :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                                       :ref:`clamp<amdgpu_synid_clamp>`
-    v_frexp_mant_f32_e64           :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                                       :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_frexp_mant_f64_e64           :ref:`vdst<amdgpu_synid9_vdst64_0>`,                :ref:`src<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                                       :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_interp_mov_f32_e64           :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`param<amdgpu_synid9_param>`::ref:`b32<amdgpu_synid9_type_dev>`,   :ref:`attr<amdgpu_synid9_attr>`::ref:`b32<amdgpu_synid9_type_dev>`                       :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_interp_p1_f32_e64            :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`vsrc<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`attr<amdgpu_synid9_attr>`::ref:`b32<amdgpu_synid9_type_dev>`                       :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_interp_p1ll_f16              :ref:`vdst<amdgpu_synid9_vdst32_0>`::ref:`f32<amdgpu_synid9_type_dev>`,            :ref:`vsrc<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`::ref:`f32<amdgpu_synid9_type_dev>`,  :ref:`attr<amdgpu_synid9_attr>`::ref:`b32<amdgpu_synid9_type_dev>`                       :ref:`high<amdgpu_synid_high>` :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_interp_p1lv_f16              :ref:`vdst<amdgpu_synid9_vdst32_0>`::ref:`f32<amdgpu_synid9_type_dev>`,            :ref:`vsrc0<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`::ref:`f32<amdgpu_synid9_type_dev>`, :ref:`attr<amdgpu_synid9_attr>`::ref:`b32<amdgpu_synid9_type_dev>`,   :ref:`vsrc2<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`::ref:`f16x2<amdgpu_synid9_type_dev>`      :ref:`high<amdgpu_synid_high>` :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_interp_p2_f16                :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`vsrc0<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`::ref:`f32<amdgpu_synid9_type_dev>`, :ref:`attr<amdgpu_synid9_attr>`::ref:`b32<amdgpu_synid9_type_dev>`,   :ref:`vsrc2<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`::ref:`f32<amdgpu_synid9_type_dev>`        :ref:`high<amdgpu_synid_high>` :ref:`clamp<amdgpu_synid_clamp>`
-    v_interp_p2_f32_e64            :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`vsrc<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`attr<amdgpu_synid9_attr>`::ref:`b32<amdgpu_synid9_type_dev>`                       :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_interp_p2_legacy_f16         :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`vsrc0<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`::ref:`f32<amdgpu_synid9_type_dev>`, :ref:`attr<amdgpu_synid9_attr>`::ref:`b32<amdgpu_synid9_type_dev>`,   :ref:`vsrc2<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`::ref:`f32<amdgpu_synid9_type_dev>`        :ref:`high<amdgpu_synid_high>` :ref:`clamp<amdgpu_synid_clamp>`
-    v_ldexp_f16_e64                :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`i16<amdgpu_synid9_type_dev>`                       :ref:`clamp<amdgpu_synid_clamp>`
-    v_ldexp_f32                    :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`i32<amdgpu_synid9_type_dev>`                       :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_ldexp_f64                    :ref:`vdst<amdgpu_synid9_vdst64_0>`,                :ref:`src0<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`i32<amdgpu_synid9_type_dev>`                       :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_lerp_u8                      :ref:`vdst<amdgpu_synid9_vdst32_0>`::ref:`u32<amdgpu_synid9_type_dev>`,            :ref:`src0<amdgpu_synid9_src32_1>`::ref:`b32<amdgpu_synid9_type_dev>`,    :ref:`src1<amdgpu_synid9_src32_1>`::ref:`b32<amdgpu_synid9_type_dev>`,   :ref:`src2<amdgpu_synid9_src32_1>`::ref:`b32<amdgpu_synid9_type_dev>`
-    v_log_f16_e64                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                                       :ref:`clamp<amdgpu_synid_clamp>`
-    v_log_f32_e64                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                                       :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_log_legacy_f32_e64           :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                                       :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_lshl_add_u32                 :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`,       :ref:`src2<amdgpu_synid9_src32_1>`
-    v_lshl_or_b32                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`::ref:`u32<amdgpu_synid9_type_dev>`,   :ref:`src2<amdgpu_synid9_src32_1>`
-    v_lshlrev_b16_e64              :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`::ref:`u16<amdgpu_synid9_type_dev>`,    :ref:`src1<amdgpu_synid9_src32_1>`
-    v_lshlrev_b32_e64              :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`::ref:`u32<amdgpu_synid9_type_dev>`,    :ref:`src1<amdgpu_synid9_src32_1>`
-    v_lshlrev_b64                  :ref:`vdst<amdgpu_synid9_vdst64_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`::ref:`u32<amdgpu_synid9_type_dev>`,    :ref:`src1<amdgpu_synid9_src64_1>`
-    v_lshrrev_b16_e64              :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`::ref:`u16<amdgpu_synid9_type_dev>`,    :ref:`src1<amdgpu_synid9_src32_1>`
-    v_lshrrev_b32_e64              :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`::ref:`u32<amdgpu_synid9_type_dev>`,    :ref:`src1<amdgpu_synid9_src32_1>`
-    v_lshrrev_b64                  :ref:`vdst<amdgpu_synid9_vdst64_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`::ref:`u32<amdgpu_synid9_type_dev>`,    :ref:`src1<amdgpu_synid9_src64_1>`
-    v_mac_f16_e64                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_mac_f32_e64                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_mad_f16                      :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,     :ref:`src2<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`             :ref:`vop3_op_sel<amdgpu_synid_vop3_op_sel>` :ref:`clamp<amdgpu_synid_clamp>`
-    v_mad_f32                      :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,     :ref:`src2<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`             :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_mad_i16                      :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`,       :ref:`src2<amdgpu_synid9_src32_1>`               :ref:`vop3_op_sel<amdgpu_synid_vop3_op_sel>` :ref:`clamp<amdgpu_synid_clamp>`
-    v_mad_i32_i16                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`,       :ref:`src2<amdgpu_synid9_src32_1>`::ref:`i32<amdgpu_synid9_type_dev>`           :ref:`vop3_op_sel<amdgpu_synid_vop3_op_sel>` :ref:`clamp<amdgpu_synid_clamp>`
-    v_mad_i32_i24                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`,       :ref:`src2<amdgpu_synid9_src32_1>`::ref:`i32<amdgpu_synid9_type_dev>`           :ref:`clamp<amdgpu_synid_clamp>`
-    v_mad_i64_i32                  :ref:`vdst<amdgpu_synid9_vdst64_0>`,      :ref:`sdst<amdgpu_synid9_sdst64_0>`,     :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`,       :ref:`src2<amdgpu_synid9_src64_1>`::ref:`i64<amdgpu_synid9_type_dev>`           :ref:`clamp<amdgpu_synid_clamp>`
-    v_mad_legacy_f16               :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,     :ref:`src2<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`             :ref:`clamp<amdgpu_synid_clamp>`
-    v_mad_legacy_f32               :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,     :ref:`src2<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`             :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_mad_legacy_i16               :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`,       :ref:`src2<amdgpu_synid9_src32_1>`               :ref:`clamp<amdgpu_synid_clamp>`
-    v_mad_legacy_u16               :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`,       :ref:`src2<amdgpu_synid9_src32_1>`               :ref:`clamp<amdgpu_synid_clamp>`
-    v_mad_u16                      :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`,       :ref:`src2<amdgpu_synid9_src32_1>`               :ref:`vop3_op_sel<amdgpu_synid_vop3_op_sel>` :ref:`clamp<amdgpu_synid_clamp>`
-    v_mad_u32_u16                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`,       :ref:`src2<amdgpu_synid9_src32_1>`::ref:`u32<amdgpu_synid9_type_dev>`           :ref:`vop3_op_sel<amdgpu_synid_vop3_op_sel>` :ref:`clamp<amdgpu_synid_clamp>`
-    v_mad_u32_u24                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`,       :ref:`src2<amdgpu_synid9_src32_1>`::ref:`u32<amdgpu_synid9_type_dev>`           :ref:`clamp<amdgpu_synid_clamp>`
-    v_mad_u64_u32                  :ref:`vdst<amdgpu_synid9_vdst64_0>`,      :ref:`sdst<amdgpu_synid9_sdst64_0>`,     :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`,       :ref:`src2<amdgpu_synid9_src64_1>`::ref:`u64<amdgpu_synid9_type_dev>`           :ref:`clamp<amdgpu_synid_clamp>`
-    v_max3_f16                     :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,     :ref:`src2<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`             :ref:`vop3_op_sel<amdgpu_synid_vop3_op_sel>` :ref:`clamp<amdgpu_synid_clamp>`
-    v_max3_f32                     :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,     :ref:`src2<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`             :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_max3_i16                     :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`,       :ref:`src2<amdgpu_synid9_src32_1>`               :ref:`vop3_op_sel<amdgpu_synid_vop3_op_sel>`
-    v_max3_i32                     :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`,       :ref:`src2<amdgpu_synid9_src32_1>`
-    v_max3_u16                     :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`,       :ref:`src2<amdgpu_synid9_src32_1>`               :ref:`vop3_op_sel<amdgpu_synid_vop3_op_sel>`
-    v_max3_u32                     :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`,       :ref:`src2<amdgpu_synid9_src32_1>`
-    v_max_f16_e64                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_max_f32_e64                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_max_f64                      :ref:`vdst<amdgpu_synid9_vdst64_0>`,                :ref:`src0<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_max_i16_e64                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
-    v_max_i32_e64                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
-    v_max_u16_e64                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
-    v_max_u32_e64                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
-    v_mbcnt_hi_u32_b32             :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
-    v_mbcnt_lo_u32_b32             :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
-    v_med3_f16                     :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,     :ref:`src2<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`             :ref:`vop3_op_sel<amdgpu_synid_vop3_op_sel>` :ref:`clamp<amdgpu_synid_clamp>`
-    v_med3_f32                     :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,     :ref:`src2<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`             :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_med3_i16                     :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`,       :ref:`src2<amdgpu_synid9_src32_1>`               :ref:`vop3_op_sel<amdgpu_synid_vop3_op_sel>`
-    v_med3_i32                     :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`,       :ref:`src2<amdgpu_synid9_src32_1>`
-    v_med3_u16                     :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`,       :ref:`src2<amdgpu_synid9_src32_1>`               :ref:`vop3_op_sel<amdgpu_synid_vop3_op_sel>`
-    v_med3_u32                     :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`,       :ref:`src2<amdgpu_synid9_src32_1>`
-    v_min3_f16                     :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,     :ref:`src2<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`             :ref:`vop3_op_sel<amdgpu_synid_vop3_op_sel>` :ref:`clamp<amdgpu_synid_clamp>`
-    v_min3_f32                     :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,     :ref:`src2<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`             :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_min3_i16                     :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`,       :ref:`src2<amdgpu_synid9_src32_1>`               :ref:`vop3_op_sel<amdgpu_synid_vop3_op_sel>`
-    v_min3_i32                     :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`,       :ref:`src2<amdgpu_synid9_src32_1>`
-    v_min3_u16                     :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`,       :ref:`src2<amdgpu_synid9_src32_1>`               :ref:`vop3_op_sel<amdgpu_synid_vop3_op_sel>`
-    v_min3_u32                     :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`,       :ref:`src2<amdgpu_synid9_src32_1>`
-    v_min_f16_e64                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_min_f32_e64                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_min_f64                      :ref:`vdst<amdgpu_synid9_vdst64_0>`,                :ref:`src0<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_min_i16_e64                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
-    v_min_i32_e64                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
-    v_min_u16_e64                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
-    v_min_u32_e64                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
-    v_mov_b32_e64                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src<amdgpu_synid9_src32_1>`
-    v_mov_fed_b32_e64              :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src<amdgpu_synid9_src32_1>`
-    v_mqsad_pk_u16_u8              :ref:`vdst<amdgpu_synid9_vdst64_0>`::ref:`b64<amdgpu_synid9_type_dev>`,            :ref:`src0<amdgpu_synid9_src64_1>`::ref:`b64<amdgpu_synid9_type_dev>`,    :ref:`src1<amdgpu_synid9_src32_1>`::ref:`b32<amdgpu_synid9_type_dev>`,   :ref:`src2<amdgpu_synid9_src64_1>`::ref:`b64<amdgpu_synid9_type_dev>`           :ref:`clamp<amdgpu_synid_clamp>`
-    v_mqsad_u32_u8                 :ref:`vdst<amdgpu_synid9_vdst128_0>`::ref:`b128<amdgpu_synid9_type_dev>`,           :ref:`src0<amdgpu_synid9_src64_1>`::ref:`b64<amdgpu_synid9_type_dev>`,    :ref:`src1<amdgpu_synid9_src32_1>`::ref:`b32<amdgpu_synid9_type_dev>`,   :ref:`vsrc2<amdgpu_synid9_vsrc128_0>`::ref:`b128<amdgpu_synid9_type_dev>`         :ref:`clamp<amdgpu_synid_clamp>`
-    v_msad_u8                      :ref:`vdst<amdgpu_synid9_vdst32_0>`::ref:`u32<amdgpu_synid9_type_dev>`,            :ref:`src0<amdgpu_synid9_src32_1>`::ref:`b32<amdgpu_synid9_type_dev>`,    :ref:`src1<amdgpu_synid9_src32_1>`::ref:`b32<amdgpu_synid9_type_dev>`,   :ref:`src2<amdgpu_synid9_src32_1>`::ref:`b32<amdgpu_synid9_type_dev>`           :ref:`clamp<amdgpu_synid_clamp>`
-    v_mul_f16_e64                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_mul_f32_e64                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_mul_f64                      :ref:`vdst<amdgpu_synid9_vdst64_0>`,                :ref:`src0<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_mul_hi_i32                   :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
-    v_mul_hi_i32_i24_e64           :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
-    v_mul_hi_u32                   :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
-    v_mul_hi_u32_u24_e64           :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
-    v_mul_i32_i24_e64              :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
-    v_mul_legacy_f32_e64           :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_mul_lo_u16_e64               :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
-    v_mul_lo_u32                   :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
-    v_mul_u32_u24_e64              :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
+    v_cmp_class_f16_e64            :ref:`sdst<amdgpu_synid9_sdst64_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`b32<amdgpu_synid9_type_dev>`
+    v_cmp_class_f32_e64            :ref:`sdst<amdgpu_synid9_sdst64_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`b32<amdgpu_synid9_type_dev>`
+    v_cmp_class_f64_e64            :ref:`sdst<amdgpu_synid9_sdst64_0>`,               :ref:`src0<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`b32<amdgpu_synid9_type_dev>`
+    v_cmp_eq_f16_e64               :ref:`sdst<amdgpu_synid9_sdst64_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmp_eq_f32_e64               :ref:`sdst<amdgpu_synid9_sdst64_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmp_eq_f64_e64               :ref:`sdst<amdgpu_synid9_sdst64_0>`,               :ref:`src0<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmp_eq_i16_e64               :ref:`sdst<amdgpu_synid9_sdst64_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
+    v_cmp_eq_i32_e64               :ref:`sdst<amdgpu_synid9_sdst64_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
+    v_cmp_eq_i64_e64               :ref:`sdst<amdgpu_synid9_sdst64_0>`,               :ref:`src0<amdgpu_synid9_src64_1>`,        :ref:`src1<amdgpu_synid9_src64_1>`
+    v_cmp_eq_u16_e64               :ref:`sdst<amdgpu_synid9_sdst64_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
+    v_cmp_eq_u32_e64               :ref:`sdst<amdgpu_synid9_sdst64_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
+    v_cmp_eq_u64_e64               :ref:`sdst<amdgpu_synid9_sdst64_0>`,               :ref:`src0<amdgpu_synid9_src64_1>`,        :ref:`src1<amdgpu_synid9_src64_1>`
+    v_cmp_f_f16_e64                :ref:`sdst<amdgpu_synid9_sdst64_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmp_f_f32_e64                :ref:`sdst<amdgpu_synid9_sdst64_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmp_f_f64_e64                :ref:`sdst<amdgpu_synid9_sdst64_0>`,               :ref:`src0<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmp_f_i16_e64                :ref:`sdst<amdgpu_synid9_sdst64_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
+    v_cmp_f_i32_e64                :ref:`sdst<amdgpu_synid9_sdst64_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
+    v_cmp_f_i64_e64                :ref:`sdst<amdgpu_synid9_sdst64_0>`,               :ref:`src0<amdgpu_synid9_src64_1>`,        :ref:`src1<amdgpu_synid9_src64_1>`
+    v_cmp_f_u16_e64                :ref:`sdst<amdgpu_synid9_sdst64_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
+    v_cmp_f_u32_e64                :ref:`sdst<amdgpu_synid9_sdst64_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
+    v_cmp_f_u64_e64                :ref:`sdst<amdgpu_synid9_sdst64_0>`,               :ref:`src0<amdgpu_synid9_src64_1>`,        :ref:`src1<amdgpu_synid9_src64_1>`
+    v_cmp_ge_f16_e64               :ref:`sdst<amdgpu_synid9_sdst64_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmp_ge_f32_e64               :ref:`sdst<amdgpu_synid9_sdst64_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmp_ge_f64_e64               :ref:`sdst<amdgpu_synid9_sdst64_0>`,               :ref:`src0<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmp_ge_i16_e64               :ref:`sdst<amdgpu_synid9_sdst64_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
+    v_cmp_ge_i32_e64               :ref:`sdst<amdgpu_synid9_sdst64_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
+    v_cmp_ge_i64_e64               :ref:`sdst<amdgpu_synid9_sdst64_0>`,               :ref:`src0<amdgpu_synid9_src64_1>`,        :ref:`src1<amdgpu_synid9_src64_1>`
+    v_cmp_ge_u16_e64               :ref:`sdst<amdgpu_synid9_sdst64_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
+    v_cmp_ge_u32_e64               :ref:`sdst<amdgpu_synid9_sdst64_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
+    v_cmp_ge_u64_e64               :ref:`sdst<amdgpu_synid9_sdst64_0>`,               :ref:`src0<amdgpu_synid9_src64_1>`,        :ref:`src1<amdgpu_synid9_src64_1>`
+    v_cmp_gt_f16_e64               :ref:`sdst<amdgpu_synid9_sdst64_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmp_gt_f32_e64               :ref:`sdst<amdgpu_synid9_sdst64_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmp_gt_f64_e64               :ref:`sdst<amdgpu_synid9_sdst64_0>`,               :ref:`src0<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmp_gt_i16_e64               :ref:`sdst<amdgpu_synid9_sdst64_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
+    v_cmp_gt_i32_e64               :ref:`sdst<amdgpu_synid9_sdst64_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
+    v_cmp_gt_i64_e64               :ref:`sdst<amdgpu_synid9_sdst64_0>`,               :ref:`src0<amdgpu_synid9_src64_1>`,        :ref:`src1<amdgpu_synid9_src64_1>`
+    v_cmp_gt_u16_e64               :ref:`sdst<amdgpu_synid9_sdst64_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
+    v_cmp_gt_u32_e64               :ref:`sdst<amdgpu_synid9_sdst64_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
+    v_cmp_gt_u64_e64               :ref:`sdst<amdgpu_synid9_sdst64_0>`,               :ref:`src0<amdgpu_synid9_src64_1>`,        :ref:`src1<amdgpu_synid9_src64_1>`
+    v_cmp_le_f16_e64               :ref:`sdst<amdgpu_synid9_sdst64_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmp_le_f32_e64               :ref:`sdst<amdgpu_synid9_sdst64_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmp_le_f64_e64               :ref:`sdst<amdgpu_synid9_sdst64_0>`,               :ref:`src0<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmp_le_i16_e64               :ref:`sdst<amdgpu_synid9_sdst64_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
+    v_cmp_le_i32_e64               :ref:`sdst<amdgpu_synid9_sdst64_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
+    v_cmp_le_i64_e64               :ref:`sdst<amdgpu_synid9_sdst64_0>`,               :ref:`src0<amdgpu_synid9_src64_1>`,        :ref:`src1<amdgpu_synid9_src64_1>`
+    v_cmp_le_u16_e64               :ref:`sdst<amdgpu_synid9_sdst64_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
+    v_cmp_le_u32_e64               :ref:`sdst<amdgpu_synid9_sdst64_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
+    v_cmp_le_u64_e64               :ref:`sdst<amdgpu_synid9_sdst64_0>`,               :ref:`src0<amdgpu_synid9_src64_1>`,        :ref:`src1<amdgpu_synid9_src64_1>`
+    v_cmp_lg_f16_e64               :ref:`sdst<amdgpu_synid9_sdst64_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmp_lg_f32_e64               :ref:`sdst<amdgpu_synid9_sdst64_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmp_lg_f64_e64               :ref:`sdst<amdgpu_synid9_sdst64_0>`,               :ref:`src0<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmp_lt_f16_e64               :ref:`sdst<amdgpu_synid9_sdst64_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmp_lt_f32_e64               :ref:`sdst<amdgpu_synid9_sdst64_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmp_lt_f64_e64               :ref:`sdst<amdgpu_synid9_sdst64_0>`,               :ref:`src0<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmp_lt_i16_e64               :ref:`sdst<amdgpu_synid9_sdst64_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
+    v_cmp_lt_i32_e64               :ref:`sdst<amdgpu_synid9_sdst64_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
+    v_cmp_lt_i64_e64               :ref:`sdst<amdgpu_synid9_sdst64_0>`,               :ref:`src0<amdgpu_synid9_src64_1>`,        :ref:`src1<amdgpu_synid9_src64_1>`
+    v_cmp_lt_u16_e64               :ref:`sdst<amdgpu_synid9_sdst64_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
+    v_cmp_lt_u32_e64               :ref:`sdst<amdgpu_synid9_sdst64_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
+    v_cmp_lt_u64_e64               :ref:`sdst<amdgpu_synid9_sdst64_0>`,               :ref:`src0<amdgpu_synid9_src64_1>`,        :ref:`src1<amdgpu_synid9_src64_1>`
+    v_cmp_ne_i16_e64               :ref:`sdst<amdgpu_synid9_sdst64_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
+    v_cmp_ne_i32_e64               :ref:`sdst<amdgpu_synid9_sdst64_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
+    v_cmp_ne_i64_e64               :ref:`sdst<amdgpu_synid9_sdst64_0>`,               :ref:`src0<amdgpu_synid9_src64_1>`,        :ref:`src1<amdgpu_synid9_src64_1>`
+    v_cmp_ne_u16_e64               :ref:`sdst<amdgpu_synid9_sdst64_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
+    v_cmp_ne_u32_e64               :ref:`sdst<amdgpu_synid9_sdst64_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
+    v_cmp_ne_u64_e64               :ref:`sdst<amdgpu_synid9_sdst64_0>`,               :ref:`src0<amdgpu_synid9_src64_1>`,        :ref:`src1<amdgpu_synid9_src64_1>`
+    v_cmp_neq_f16_e64              :ref:`sdst<amdgpu_synid9_sdst64_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmp_neq_f32_e64              :ref:`sdst<amdgpu_synid9_sdst64_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmp_neq_f64_e64              :ref:`sdst<amdgpu_synid9_sdst64_0>`,               :ref:`src0<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmp_nge_f16_e64              :ref:`sdst<amdgpu_synid9_sdst64_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmp_nge_f32_e64              :ref:`sdst<amdgpu_synid9_sdst64_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmp_nge_f64_e64              :ref:`sdst<amdgpu_synid9_sdst64_0>`,               :ref:`src0<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmp_ngt_f16_e64              :ref:`sdst<amdgpu_synid9_sdst64_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmp_ngt_f32_e64              :ref:`sdst<amdgpu_synid9_sdst64_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmp_ngt_f64_e64              :ref:`sdst<amdgpu_synid9_sdst64_0>`,               :ref:`src0<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmp_nle_f16_e64              :ref:`sdst<amdgpu_synid9_sdst64_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmp_nle_f32_e64              :ref:`sdst<amdgpu_synid9_sdst64_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmp_nle_f64_e64              :ref:`sdst<amdgpu_synid9_sdst64_0>`,               :ref:`src0<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmp_nlg_f16_e64              :ref:`sdst<amdgpu_synid9_sdst64_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmp_nlg_f32_e64              :ref:`sdst<amdgpu_synid9_sdst64_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmp_nlg_f64_e64              :ref:`sdst<amdgpu_synid9_sdst64_0>`,               :ref:`src0<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmp_nlt_f16_e64              :ref:`sdst<amdgpu_synid9_sdst64_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmp_nlt_f32_e64              :ref:`sdst<amdgpu_synid9_sdst64_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmp_nlt_f64_e64              :ref:`sdst<amdgpu_synid9_sdst64_0>`,               :ref:`src0<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmp_o_f16_e64                :ref:`sdst<amdgpu_synid9_sdst64_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmp_o_f32_e64                :ref:`sdst<amdgpu_synid9_sdst64_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmp_o_f64_e64                :ref:`sdst<amdgpu_synid9_sdst64_0>`,               :ref:`src0<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmp_t_i16_e64                :ref:`sdst<amdgpu_synid9_sdst64_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
+    v_cmp_t_i32_e64                :ref:`sdst<amdgpu_synid9_sdst64_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
+    v_cmp_t_i64_e64                :ref:`sdst<amdgpu_synid9_sdst64_0>`,               :ref:`src0<amdgpu_synid9_src64_1>`,        :ref:`src1<amdgpu_synid9_src64_1>`
+    v_cmp_t_u16_e64                :ref:`sdst<amdgpu_synid9_sdst64_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
+    v_cmp_t_u32_e64                :ref:`sdst<amdgpu_synid9_sdst64_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
+    v_cmp_t_u64_e64                :ref:`sdst<amdgpu_synid9_sdst64_0>`,               :ref:`src0<amdgpu_synid9_src64_1>`,        :ref:`src1<amdgpu_synid9_src64_1>`
+    v_cmp_tru_f16_e64              :ref:`sdst<amdgpu_synid9_sdst64_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmp_tru_f32_e64              :ref:`sdst<amdgpu_synid9_sdst64_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmp_tru_f64_e64              :ref:`sdst<amdgpu_synid9_sdst64_0>`,               :ref:`src0<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmp_u_f16_e64                :ref:`sdst<amdgpu_synid9_sdst64_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmp_u_f32_e64                :ref:`sdst<amdgpu_synid9_sdst64_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmp_u_f64_e64                :ref:`sdst<amdgpu_synid9_sdst64_0>`,               :ref:`src0<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmpx_class_f16_e64           :ref:`sdst<amdgpu_synid9_sdst64_1>`,               :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`b32<amdgpu_synid9_type_dev>`
+    v_cmpx_class_f32_e64           :ref:`sdst<amdgpu_synid9_sdst64_1>`,               :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`b32<amdgpu_synid9_type_dev>`
+    v_cmpx_class_f64_e64           :ref:`sdst<amdgpu_synid9_sdst64_1>`,               :ref:`src0<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`b32<amdgpu_synid9_type_dev>`
+    v_cmpx_eq_f16_e64              :ref:`sdst<amdgpu_synid9_sdst64_1>`,               :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmpx_eq_f32_e64              :ref:`sdst<amdgpu_synid9_sdst64_1>`,               :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmpx_eq_f64_e64              :ref:`sdst<amdgpu_synid9_sdst64_1>`,               :ref:`src0<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmpx_eq_i16_e64              :ref:`sdst<amdgpu_synid9_sdst64_1>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
+    v_cmpx_eq_i32_e64              :ref:`sdst<amdgpu_synid9_sdst64_1>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
+    v_cmpx_eq_i64_e64              :ref:`sdst<amdgpu_synid9_sdst64_1>`,               :ref:`src0<amdgpu_synid9_src64_1>`,        :ref:`src1<amdgpu_synid9_src64_1>`
+    v_cmpx_eq_u16_e64              :ref:`sdst<amdgpu_synid9_sdst64_1>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
+    v_cmpx_eq_u32_e64              :ref:`sdst<amdgpu_synid9_sdst64_1>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
+    v_cmpx_eq_u64_e64              :ref:`sdst<amdgpu_synid9_sdst64_1>`,               :ref:`src0<amdgpu_synid9_src64_1>`,        :ref:`src1<amdgpu_synid9_src64_1>`
+    v_cmpx_f_f16_e64               :ref:`sdst<amdgpu_synid9_sdst64_1>`,               :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmpx_f_f32_e64               :ref:`sdst<amdgpu_synid9_sdst64_1>`,               :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmpx_f_f64_e64               :ref:`sdst<amdgpu_synid9_sdst64_1>`,               :ref:`src0<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmpx_f_i16_e64               :ref:`sdst<amdgpu_synid9_sdst64_1>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
+    v_cmpx_f_i32_e64               :ref:`sdst<amdgpu_synid9_sdst64_1>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
+    v_cmpx_f_i64_e64               :ref:`sdst<amdgpu_synid9_sdst64_1>`,               :ref:`src0<amdgpu_synid9_src64_1>`,        :ref:`src1<amdgpu_synid9_src64_1>`
+    v_cmpx_f_u16_e64               :ref:`sdst<amdgpu_synid9_sdst64_1>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
+    v_cmpx_f_u32_e64               :ref:`sdst<amdgpu_synid9_sdst64_1>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
+    v_cmpx_f_u64_e64               :ref:`sdst<amdgpu_synid9_sdst64_1>`,               :ref:`src0<amdgpu_synid9_src64_1>`,        :ref:`src1<amdgpu_synid9_src64_1>`
+    v_cmpx_ge_f16_e64              :ref:`sdst<amdgpu_synid9_sdst64_1>`,               :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmpx_ge_f32_e64              :ref:`sdst<amdgpu_synid9_sdst64_1>`,               :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmpx_ge_f64_e64              :ref:`sdst<amdgpu_synid9_sdst64_1>`,               :ref:`src0<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmpx_ge_i16_e64              :ref:`sdst<amdgpu_synid9_sdst64_1>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
+    v_cmpx_ge_i32_e64              :ref:`sdst<amdgpu_synid9_sdst64_1>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
+    v_cmpx_ge_i64_e64              :ref:`sdst<amdgpu_synid9_sdst64_1>`,               :ref:`src0<amdgpu_synid9_src64_1>`,        :ref:`src1<amdgpu_synid9_src64_1>`
+    v_cmpx_ge_u16_e64              :ref:`sdst<amdgpu_synid9_sdst64_1>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
+    v_cmpx_ge_u32_e64              :ref:`sdst<amdgpu_synid9_sdst64_1>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
+    v_cmpx_ge_u64_e64              :ref:`sdst<amdgpu_synid9_sdst64_1>`,               :ref:`src0<amdgpu_synid9_src64_1>`,        :ref:`src1<amdgpu_synid9_src64_1>`
+    v_cmpx_gt_f16_e64              :ref:`sdst<amdgpu_synid9_sdst64_1>`,               :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmpx_gt_f32_e64              :ref:`sdst<amdgpu_synid9_sdst64_1>`,               :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmpx_gt_f64_e64              :ref:`sdst<amdgpu_synid9_sdst64_1>`,               :ref:`src0<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmpx_gt_i16_e64              :ref:`sdst<amdgpu_synid9_sdst64_1>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
+    v_cmpx_gt_i32_e64              :ref:`sdst<amdgpu_synid9_sdst64_1>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
+    v_cmpx_gt_i64_e64              :ref:`sdst<amdgpu_synid9_sdst64_1>`,               :ref:`src0<amdgpu_synid9_src64_1>`,        :ref:`src1<amdgpu_synid9_src64_1>`
+    v_cmpx_gt_u16_e64              :ref:`sdst<amdgpu_synid9_sdst64_1>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
+    v_cmpx_gt_u32_e64              :ref:`sdst<amdgpu_synid9_sdst64_1>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
+    v_cmpx_gt_u64_e64              :ref:`sdst<amdgpu_synid9_sdst64_1>`,               :ref:`src0<amdgpu_synid9_src64_1>`,        :ref:`src1<amdgpu_synid9_src64_1>`
+    v_cmpx_le_f16_e64              :ref:`sdst<amdgpu_synid9_sdst64_1>`,               :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmpx_le_f32_e64              :ref:`sdst<amdgpu_synid9_sdst64_1>`,               :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmpx_le_f64_e64              :ref:`sdst<amdgpu_synid9_sdst64_1>`,               :ref:`src0<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmpx_le_i16_e64              :ref:`sdst<amdgpu_synid9_sdst64_1>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
+    v_cmpx_le_i32_e64              :ref:`sdst<amdgpu_synid9_sdst64_1>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
+    v_cmpx_le_i64_e64              :ref:`sdst<amdgpu_synid9_sdst64_1>`,               :ref:`src0<amdgpu_synid9_src64_1>`,        :ref:`src1<amdgpu_synid9_src64_1>`
+    v_cmpx_le_u16_e64              :ref:`sdst<amdgpu_synid9_sdst64_1>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
+    v_cmpx_le_u32_e64              :ref:`sdst<amdgpu_synid9_sdst64_1>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
+    v_cmpx_le_u64_e64              :ref:`sdst<amdgpu_synid9_sdst64_1>`,               :ref:`src0<amdgpu_synid9_src64_1>`,        :ref:`src1<amdgpu_synid9_src64_1>`
+    v_cmpx_lg_f16_e64              :ref:`sdst<amdgpu_synid9_sdst64_1>`,               :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmpx_lg_f32_e64              :ref:`sdst<amdgpu_synid9_sdst64_1>`,               :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmpx_lg_f64_e64              :ref:`sdst<amdgpu_synid9_sdst64_1>`,               :ref:`src0<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmpx_lt_f16_e64              :ref:`sdst<amdgpu_synid9_sdst64_1>`,               :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmpx_lt_f32_e64              :ref:`sdst<amdgpu_synid9_sdst64_1>`,               :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmpx_lt_f64_e64              :ref:`sdst<amdgpu_synid9_sdst64_1>`,               :ref:`src0<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmpx_lt_i16_e64              :ref:`sdst<amdgpu_synid9_sdst64_1>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
+    v_cmpx_lt_i32_e64              :ref:`sdst<amdgpu_synid9_sdst64_1>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
+    v_cmpx_lt_i64_e64              :ref:`sdst<amdgpu_synid9_sdst64_1>`,               :ref:`src0<amdgpu_synid9_src64_1>`,        :ref:`src1<amdgpu_synid9_src64_1>`
+    v_cmpx_lt_u16_e64              :ref:`sdst<amdgpu_synid9_sdst64_1>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
+    v_cmpx_lt_u32_e64              :ref:`sdst<amdgpu_synid9_sdst64_1>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
+    v_cmpx_lt_u64_e64              :ref:`sdst<amdgpu_synid9_sdst64_1>`,               :ref:`src0<amdgpu_synid9_src64_1>`,        :ref:`src1<amdgpu_synid9_src64_1>`
+    v_cmpx_ne_i16_e64              :ref:`sdst<amdgpu_synid9_sdst64_1>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
+    v_cmpx_ne_i32_e64              :ref:`sdst<amdgpu_synid9_sdst64_1>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
+    v_cmpx_ne_i64_e64              :ref:`sdst<amdgpu_synid9_sdst64_1>`,               :ref:`src0<amdgpu_synid9_src64_1>`,        :ref:`src1<amdgpu_synid9_src64_1>`
+    v_cmpx_ne_u16_e64              :ref:`sdst<amdgpu_synid9_sdst64_1>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
+    v_cmpx_ne_u32_e64              :ref:`sdst<amdgpu_synid9_sdst64_1>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
+    v_cmpx_ne_u64_e64              :ref:`sdst<amdgpu_synid9_sdst64_1>`,               :ref:`src0<amdgpu_synid9_src64_1>`,        :ref:`src1<amdgpu_synid9_src64_1>`
+    v_cmpx_neq_f16_e64             :ref:`sdst<amdgpu_synid9_sdst64_1>`,               :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmpx_neq_f32_e64             :ref:`sdst<amdgpu_synid9_sdst64_1>`,               :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmpx_neq_f64_e64             :ref:`sdst<amdgpu_synid9_sdst64_1>`,               :ref:`src0<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmpx_nge_f16_e64             :ref:`sdst<amdgpu_synid9_sdst64_1>`,               :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmpx_nge_f32_e64             :ref:`sdst<amdgpu_synid9_sdst64_1>`,               :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmpx_nge_f64_e64             :ref:`sdst<amdgpu_synid9_sdst64_1>`,               :ref:`src0<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmpx_ngt_f16_e64             :ref:`sdst<amdgpu_synid9_sdst64_1>`,               :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmpx_ngt_f32_e64             :ref:`sdst<amdgpu_synid9_sdst64_1>`,               :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmpx_ngt_f64_e64             :ref:`sdst<amdgpu_synid9_sdst64_1>`,               :ref:`src0<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmpx_nle_f16_e64             :ref:`sdst<amdgpu_synid9_sdst64_1>`,               :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmpx_nle_f32_e64             :ref:`sdst<amdgpu_synid9_sdst64_1>`,               :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmpx_nle_f64_e64             :ref:`sdst<amdgpu_synid9_sdst64_1>`,               :ref:`src0<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmpx_nlg_f16_e64             :ref:`sdst<amdgpu_synid9_sdst64_1>`,               :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmpx_nlg_f32_e64             :ref:`sdst<amdgpu_synid9_sdst64_1>`,               :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmpx_nlg_f64_e64             :ref:`sdst<amdgpu_synid9_sdst64_1>`,               :ref:`src0<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmpx_nlt_f16_e64             :ref:`sdst<amdgpu_synid9_sdst64_1>`,               :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmpx_nlt_f32_e64             :ref:`sdst<amdgpu_synid9_sdst64_1>`,               :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmpx_nlt_f64_e64             :ref:`sdst<amdgpu_synid9_sdst64_1>`,               :ref:`src0<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmpx_o_f16_e64               :ref:`sdst<amdgpu_synid9_sdst64_1>`,               :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmpx_o_f32_e64               :ref:`sdst<amdgpu_synid9_sdst64_1>`,               :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmpx_o_f64_e64               :ref:`sdst<amdgpu_synid9_sdst64_1>`,               :ref:`src0<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmpx_t_i16_e64               :ref:`sdst<amdgpu_synid9_sdst64_1>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
+    v_cmpx_t_i32_e64               :ref:`sdst<amdgpu_synid9_sdst64_1>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
+    v_cmpx_t_i64_e64               :ref:`sdst<amdgpu_synid9_sdst64_1>`,               :ref:`src0<amdgpu_synid9_src64_1>`,        :ref:`src1<amdgpu_synid9_src64_1>`
+    v_cmpx_t_u16_e64               :ref:`sdst<amdgpu_synid9_sdst64_1>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
+    v_cmpx_t_u32_e64               :ref:`sdst<amdgpu_synid9_sdst64_1>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
+    v_cmpx_t_u64_e64               :ref:`sdst<amdgpu_synid9_sdst64_1>`,               :ref:`src0<amdgpu_synid9_src64_1>`,        :ref:`src1<amdgpu_synid9_src64_1>`
+    v_cmpx_tru_f16_e64             :ref:`sdst<amdgpu_synid9_sdst64_1>`,               :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmpx_tru_f32_e64             :ref:`sdst<amdgpu_synid9_sdst64_1>`,               :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmpx_tru_f64_e64             :ref:`sdst<amdgpu_synid9_sdst64_1>`,               :ref:`src0<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmpx_u_f16_e64               :ref:`sdst<amdgpu_synid9_sdst64_1>`,               :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmpx_u_f32_e64               :ref:`sdst<amdgpu_synid9_sdst64_1>`,               :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cmpx_u_f64_e64               :ref:`sdst<amdgpu_synid9_sdst64_1>`,               :ref:`src0<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cndmask_b32_e64              :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`,       :ref:`ssrc2<amdgpu_synid9_ssrc64_1>`
+    v_cos_f16_e64                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                                    :ref:`clamp<amdgpu_synid_clamp>`
+    v_cos_f32_e64                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                                    :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_cubeid_f32                   :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,     :ref:`src2<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`          :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_cubema_f32                   :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,     :ref:`src2<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`          :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_cubesc_f32                   :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,     :ref:`src2<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`          :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_cubetc_f32                   :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,     :ref:`src2<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`          :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_cvt_f16_f32_e64              :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                                    :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_cvt_f16_i16_e64              :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src<amdgpu_synid9_src32_1>`                                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cvt_f16_u16_e64              :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src<amdgpu_synid9_src32_1>`                                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_cvt_f32_f16_e64              :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                                    :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_cvt_f32_f64_e64              :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                                    :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_cvt_f32_i32_e64              :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src<amdgpu_synid9_src32_1>`                                      :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_cvt_f32_u32_e64              :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src<amdgpu_synid9_src32_1>`                                      :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_cvt_f32_ubyte0_e64           :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src<amdgpu_synid9_src32_1>`                                      :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_cvt_f32_ubyte1_e64           :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src<amdgpu_synid9_src32_1>`                                      :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_cvt_f32_ubyte2_e64           :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src<amdgpu_synid9_src32_1>`                                      :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_cvt_f32_ubyte3_e64           :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src<amdgpu_synid9_src32_1>`                                      :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_cvt_f64_f32_e64              :ref:`vdst<amdgpu_synid9_vdst64_0>`,               :ref:`src<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                                    :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_cvt_f64_i32_e64              :ref:`vdst<amdgpu_synid9_vdst64_0>`,               :ref:`src<amdgpu_synid9_src32_1>`                                      :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_cvt_f64_u32_e64              :ref:`vdst<amdgpu_synid9_vdst64_0>`,               :ref:`src<amdgpu_synid9_src32_1>`                                      :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_cvt_flr_i32_f32_e64          :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`
+    v_cvt_i16_f16_e64              :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                                    :ref:`clamp<amdgpu_synid_clamp>`
+    v_cvt_i32_f32_e64              :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                                    :ref:`clamp<amdgpu_synid_clamp>`
+    v_cvt_i32_f64_e64              :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                                    :ref:`clamp<amdgpu_synid_clamp>`
+    v_cvt_norm_i16_f16_e64         :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`
+    v_cvt_norm_u16_f16_e64         :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`
+    v_cvt_off_f32_i4_e64           :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src<amdgpu_synid9_src32_1>`                                      :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_cvt_pk_i16_i32               :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
+    v_cvt_pk_u16_u32               :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
+    v_cvt_pk_u8_f32                :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`u32<amdgpu_synid9_type_dev>`,   :ref:`src2<amdgpu_synid9_src32_1>`::ref:`u32<amdgpu_synid9_type_dev>`
+    v_cvt_pkaccum_u8_f32           :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`u32<amdgpu_synid9_type_dev>`
+    v_cvt_pknorm_i16_f16           :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                      :ref:`op_sel<amdgpu_synid_vop3_op_sel>`
+    v_cvt_pknorm_i16_f32           :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`
+    v_cvt_pknorm_u16_f16           :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                      :ref:`op_sel<amdgpu_synid_vop3_op_sel>`
+    v_cvt_pknorm_u16_f32           :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`
+    v_cvt_pkrtz_f16_f32            :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`
+    v_cvt_rpi_i32_f32_e64          :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`
+    v_cvt_u16_f16_e64              :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                                    :ref:`clamp<amdgpu_synid_clamp>`
+    v_cvt_u32_f32_e64              :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                                    :ref:`clamp<amdgpu_synid_clamp>`
+    v_cvt_u32_f64_e64              :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                                    :ref:`clamp<amdgpu_synid_clamp>`
+    v_div_fixup_f16                :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,     :ref:`src2<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`          :ref:`op_sel<amdgpu_synid_vop3_op_sel>` :ref:`clamp<amdgpu_synid_clamp>`
+    v_div_fixup_f32                :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,     :ref:`src2<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`          :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_div_fixup_f64                :ref:`vdst<amdgpu_synid9_vdst64_0>`,               :ref:`src0<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,     :ref:`src2<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`          :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_div_fixup_legacy_f16         :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,     :ref:`src2<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`          :ref:`clamp<amdgpu_synid_clamp>`
+    v_div_fmas_f32                 :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,     :ref:`src2<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`          :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_div_fmas_f64                 :ref:`vdst<amdgpu_synid9_vdst64_0>`,               :ref:`src0<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,     :ref:`src2<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`          :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_div_scale_f32                :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`vcc<amdgpu_synid9_vcc_64>`,     :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`,       :ref:`src2<amdgpu_synid9_src32_1>`
+    v_div_scale_f64                :ref:`vdst<amdgpu_synid9_vdst64_0>`,      :ref:`vcc<amdgpu_synid9_vcc_64>`,     :ref:`src0<amdgpu_synid9_src64_1>`,        :ref:`src1<amdgpu_synid9_src64_1>`,       :ref:`src2<amdgpu_synid9_src64_1>`
+    v_exp_f16_e64                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                                    :ref:`clamp<amdgpu_synid_clamp>`
+    v_exp_f32_e64                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                                    :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_exp_legacy_f32_e64           :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                                    :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_ffbh_i32_e64                 :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src<amdgpu_synid9_src32_1>`
+    v_ffbh_u32_e64                 :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src<amdgpu_synid9_src32_1>`
+    v_ffbl_b32_e64                 :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src<amdgpu_synid9_src32_1>`
+    v_floor_f16_e64                :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                                    :ref:`clamp<amdgpu_synid_clamp>`
+    v_floor_f32_e64                :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                                    :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_floor_f64_e64                :ref:`vdst<amdgpu_synid9_vdst64_0>`,               :ref:`src<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                                    :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_fma_f16                      :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,     :ref:`src2<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`          :ref:`op_sel<amdgpu_synid_vop3_op_sel>` :ref:`clamp<amdgpu_synid_clamp>`
+    v_fma_f32                      :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,     :ref:`src2<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`          :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_fma_f64                      :ref:`vdst<amdgpu_synid9_vdst64_0>`,               :ref:`src0<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,     :ref:`src2<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`          :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_fma_legacy_f16               :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,     :ref:`src2<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`          :ref:`clamp<amdgpu_synid_clamp>`
+    v_fract_f16_e64                :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                                    :ref:`clamp<amdgpu_synid_clamp>`
+    v_fract_f32_e64                :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                                    :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_fract_f64_e64                :ref:`vdst<amdgpu_synid9_vdst64_0>`,               :ref:`src<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                                    :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_frexp_exp_i16_f16_e64        :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`
+    v_frexp_exp_i32_f32_e64        :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`
+    v_frexp_exp_i32_f64_e64        :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`
+    v_frexp_mant_f16_e64           :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                                    :ref:`clamp<amdgpu_synid_clamp>`
+    v_frexp_mant_f32_e64           :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                                    :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_frexp_mant_f64_e64           :ref:`vdst<amdgpu_synid9_vdst64_0>`,               :ref:`src<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                                    :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_interp_mov_f32_e64           :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`param<amdgpu_synid9_param>`::ref:`b32<amdgpu_synid9_type_dev>`,   :ref:`attr<amdgpu_synid9_attr>`::ref:`b32<amdgpu_synid9_type_dev>`                    :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_interp_p1_f32_e64            :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`vsrc<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`attr<amdgpu_synid9_attr>`::ref:`b32<amdgpu_synid9_type_dev>`                    :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_interp_p1ll_f16              :ref:`vdst<amdgpu_synid9_vdst32_0>`::ref:`f32<amdgpu_synid9_type_dev>`,           :ref:`vsrc<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`::ref:`f32<amdgpu_synid9_type_dev>`,  :ref:`attr<amdgpu_synid9_attr>`::ref:`b32<amdgpu_synid9_type_dev>`                    :ref:`high<amdgpu_synid_high>` :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_interp_p1lv_f16              :ref:`vdst<amdgpu_synid9_vdst32_0>`::ref:`f32<amdgpu_synid9_type_dev>`,           :ref:`vsrc0<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`::ref:`f32<amdgpu_synid9_type_dev>`, :ref:`attr<amdgpu_synid9_attr>`::ref:`b32<amdgpu_synid9_type_dev>`,   :ref:`vsrc2<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`::ref:`f16x2<amdgpu_synid9_type_dev>`   :ref:`high<amdgpu_synid_high>` :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_interp_p2_f16                :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`::ref:`f32<amdgpu_synid9_type_dev>`, :ref:`attr<amdgpu_synid9_attr>`::ref:`b32<amdgpu_synid9_type_dev>`,   :ref:`vsrc2<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`::ref:`f32<amdgpu_synid9_type_dev>`     :ref:`high<amdgpu_synid_high>` :ref:`clamp<amdgpu_synid_clamp>`
+    v_interp_p2_f32_e64            :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`vsrc<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`attr<amdgpu_synid9_attr>`::ref:`b32<amdgpu_synid9_type_dev>`                    :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_interp_p2_legacy_f16         :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`vsrc0<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`::ref:`f32<amdgpu_synid9_type_dev>`, :ref:`attr<amdgpu_synid9_attr>`::ref:`b32<amdgpu_synid9_type_dev>`,   :ref:`vsrc2<amdgpu_synid9_vsrc32_0>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`::ref:`f32<amdgpu_synid9_type_dev>`     :ref:`high<amdgpu_synid_high>` :ref:`clamp<amdgpu_synid_clamp>`
+    v_ldexp_f16_e64                :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`i16<amdgpu_synid9_type_dev>`                    :ref:`clamp<amdgpu_synid_clamp>`
+    v_ldexp_f32                    :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`i32<amdgpu_synid9_type_dev>`                    :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_ldexp_f64                    :ref:`vdst<amdgpu_synid9_vdst64_0>`,               :ref:`src0<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`i32<amdgpu_synid9_type_dev>`                    :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_lerp_u8                      :ref:`vdst<amdgpu_synid9_vdst32_0>`::ref:`u32<amdgpu_synid9_type_dev>`,           :ref:`src0<amdgpu_synid9_src32_1>`::ref:`b32<amdgpu_synid9_type_dev>`,    :ref:`src1<amdgpu_synid9_src32_1>`::ref:`b32<amdgpu_synid9_type_dev>`,   :ref:`src2<amdgpu_synid9_src32_1>`::ref:`b32<amdgpu_synid9_type_dev>`
+    v_log_f16_e64                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                                    :ref:`clamp<amdgpu_synid_clamp>`
+    v_log_f32_e64                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                                    :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_log_legacy_f32_e64           :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                                    :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_lshl_add_u32                 :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`,       :ref:`src2<amdgpu_synid9_src32_1>`
+    v_lshl_or_b32                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`::ref:`u32<amdgpu_synid9_type_dev>`,   :ref:`src2<amdgpu_synid9_src32_1>`
+    v_lshlrev_b16_e64              :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`::ref:`u16<amdgpu_synid9_type_dev>`,    :ref:`src1<amdgpu_synid9_src32_1>`
+    v_lshlrev_b32_e64              :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`::ref:`u32<amdgpu_synid9_type_dev>`,    :ref:`src1<amdgpu_synid9_src32_1>`
+    v_lshlrev_b64                  :ref:`vdst<amdgpu_synid9_vdst64_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`::ref:`u32<amdgpu_synid9_type_dev>`,    :ref:`src1<amdgpu_synid9_src64_1>`
+    v_lshrrev_b16_e64              :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`::ref:`u16<amdgpu_synid9_type_dev>`,    :ref:`src1<amdgpu_synid9_src32_1>`
+    v_lshrrev_b32_e64              :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`::ref:`u32<amdgpu_synid9_type_dev>`,    :ref:`src1<amdgpu_synid9_src32_1>`
+    v_lshrrev_b64                  :ref:`vdst<amdgpu_synid9_vdst64_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`::ref:`u32<amdgpu_synid9_type_dev>`,    :ref:`src1<amdgpu_synid9_src64_1>`
+    v_mac_f16_e64                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_mac_f32_e64                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_mad_f16                      :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,     :ref:`src2<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`          :ref:`op_sel<amdgpu_synid_vop3_op_sel>` :ref:`clamp<amdgpu_synid_clamp>`
+    v_mad_f32                      :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,     :ref:`src2<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`          :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_mad_i16                      :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`,       :ref:`src2<amdgpu_synid9_src32_1>`            :ref:`op_sel<amdgpu_synid_vop3_op_sel>` :ref:`clamp<amdgpu_synid_clamp>`
+    v_mad_i32_i16                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`,       :ref:`src2<amdgpu_synid9_src32_1>`::ref:`i32<amdgpu_synid9_type_dev>`        :ref:`op_sel<amdgpu_synid_vop3_op_sel>` :ref:`clamp<amdgpu_synid_clamp>`
+    v_mad_i32_i24                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`,       :ref:`src2<amdgpu_synid9_src32_1>`::ref:`i32<amdgpu_synid9_type_dev>`        :ref:`clamp<amdgpu_synid_clamp>`
+    v_mad_i64_i32                  :ref:`vdst<amdgpu_synid9_vdst64_0>`,      :ref:`sdst<amdgpu_synid9_sdst64_0>`,    :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`,       :ref:`src2<amdgpu_synid9_src64_1>`::ref:`i64<amdgpu_synid9_type_dev>`        :ref:`clamp<amdgpu_synid_clamp>`
+    v_mad_legacy_f16               :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,     :ref:`src2<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`          :ref:`clamp<amdgpu_synid_clamp>`
+    v_mad_legacy_f32               :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,     :ref:`src2<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`          :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_mad_legacy_i16               :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`,       :ref:`src2<amdgpu_synid9_src32_1>`            :ref:`clamp<amdgpu_synid_clamp>`
+    v_mad_legacy_u16               :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`,       :ref:`src2<amdgpu_synid9_src32_1>`            :ref:`clamp<amdgpu_synid_clamp>`
+    v_mad_u16                      :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`,       :ref:`src2<amdgpu_synid9_src32_1>`            :ref:`op_sel<amdgpu_synid_vop3_op_sel>` :ref:`clamp<amdgpu_synid_clamp>`
+    v_mad_u32_u16                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`,       :ref:`src2<amdgpu_synid9_src32_1>`::ref:`u32<amdgpu_synid9_type_dev>`        :ref:`op_sel<amdgpu_synid_vop3_op_sel>` :ref:`clamp<amdgpu_synid_clamp>`
+    v_mad_u32_u24                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`,       :ref:`src2<amdgpu_synid9_src32_1>`::ref:`u32<amdgpu_synid9_type_dev>`        :ref:`clamp<amdgpu_synid_clamp>`
+    v_mad_u64_u32                  :ref:`vdst<amdgpu_synid9_vdst64_0>`,      :ref:`sdst<amdgpu_synid9_sdst64_0>`,    :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`,       :ref:`src2<amdgpu_synid9_src64_1>`::ref:`u64<amdgpu_synid9_type_dev>`        :ref:`clamp<amdgpu_synid_clamp>`
+    v_max3_f16                     :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,     :ref:`src2<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`          :ref:`op_sel<amdgpu_synid_vop3_op_sel>` :ref:`clamp<amdgpu_synid_clamp>`
+    v_max3_f32                     :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,     :ref:`src2<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`          :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_max3_i16                     :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`,       :ref:`src2<amdgpu_synid9_src32_1>`            :ref:`op_sel<amdgpu_synid_vop3_op_sel>`
+    v_max3_i32                     :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`,       :ref:`src2<amdgpu_synid9_src32_1>`
+    v_max3_u16                     :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`,       :ref:`src2<amdgpu_synid9_src32_1>`            :ref:`op_sel<amdgpu_synid_vop3_op_sel>`
+    v_max3_u32                     :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`,       :ref:`src2<amdgpu_synid9_src32_1>`
+    v_max_f16_e64                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_max_f32_e64                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_max_f64                      :ref:`vdst<amdgpu_synid9_vdst64_0>`,               :ref:`src0<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_max_i16_e64                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
+    v_max_i32_e64                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
+    v_max_u16_e64                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
+    v_max_u32_e64                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
+    v_mbcnt_hi_u32_b32             :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
+    v_mbcnt_lo_u32_b32             :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
+    v_med3_f16                     :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,     :ref:`src2<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`          :ref:`op_sel<amdgpu_synid_vop3_op_sel>` :ref:`clamp<amdgpu_synid_clamp>`
+    v_med3_f32                     :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,     :ref:`src2<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`          :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_med3_i16                     :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`,       :ref:`src2<amdgpu_synid9_src32_1>`            :ref:`op_sel<amdgpu_synid_vop3_op_sel>`
+    v_med3_i32                     :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`,       :ref:`src2<amdgpu_synid9_src32_1>`
+    v_med3_u16                     :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`,       :ref:`src2<amdgpu_synid9_src32_1>`            :ref:`op_sel<amdgpu_synid_vop3_op_sel>`
+    v_med3_u32                     :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`,       :ref:`src2<amdgpu_synid9_src32_1>`
+    v_min3_f16                     :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,     :ref:`src2<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`          :ref:`op_sel<amdgpu_synid_vop3_op_sel>` :ref:`clamp<amdgpu_synid_clamp>`
+    v_min3_f32                     :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,     :ref:`src2<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`          :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_min3_i16                     :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`,       :ref:`src2<amdgpu_synid9_src32_1>`            :ref:`op_sel<amdgpu_synid_vop3_op_sel>`
+    v_min3_i32                     :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`,       :ref:`src2<amdgpu_synid9_src32_1>`
+    v_min3_u16                     :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`,       :ref:`src2<amdgpu_synid9_src32_1>`            :ref:`op_sel<amdgpu_synid_vop3_op_sel>`
+    v_min3_u32                     :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`,       :ref:`src2<amdgpu_synid9_src32_1>`
+    v_min_f16_e64                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_min_f32_e64                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_min_f64                      :ref:`vdst<amdgpu_synid9_vdst64_0>`,               :ref:`src0<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_min_i16_e64                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
+    v_min_i32_e64                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
+    v_min_u16_e64                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
+    v_min_u32_e64                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
+    v_mov_b32_e64                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src<amdgpu_synid9_src32_1>`
+    v_mov_fed_b32_e64              :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src<amdgpu_synid9_src32_1>`
+    v_mqsad_pk_u16_u8              :ref:`vdst<amdgpu_synid9_vdst64_0>`::ref:`b64<amdgpu_synid9_type_dev>`,           :ref:`src0<amdgpu_synid9_src64_1>`::ref:`b64<amdgpu_synid9_type_dev>`,    :ref:`src1<amdgpu_synid9_src32_1>`::ref:`b32<amdgpu_synid9_type_dev>`,   :ref:`src2<amdgpu_synid9_src64_1>`::ref:`b64<amdgpu_synid9_type_dev>`        :ref:`clamp<amdgpu_synid_clamp>`
+    v_mqsad_u32_u8                 :ref:`vdst<amdgpu_synid9_vdst128_0>`::ref:`b128<amdgpu_synid9_type_dev>`,          :ref:`src0<amdgpu_synid9_src64_1>`::ref:`b64<amdgpu_synid9_type_dev>`,    :ref:`src1<amdgpu_synid9_src32_1>`::ref:`b32<amdgpu_synid9_type_dev>`,   :ref:`vsrc2<amdgpu_synid9_vsrc128_0>`::ref:`b128<amdgpu_synid9_type_dev>`      :ref:`clamp<amdgpu_synid_clamp>`
+    v_msad_u8                      :ref:`vdst<amdgpu_synid9_vdst32_0>`::ref:`u32<amdgpu_synid9_type_dev>`,           :ref:`src0<amdgpu_synid9_src32_1>`::ref:`b32<amdgpu_synid9_type_dev>`,    :ref:`src1<amdgpu_synid9_src32_1>`::ref:`b32<amdgpu_synid9_type_dev>`,   :ref:`src2<amdgpu_synid9_src32_1>`::ref:`b32<amdgpu_synid9_type_dev>`        :ref:`clamp<amdgpu_synid_clamp>`
+    v_mul_f16_e64                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_mul_f32_e64                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_mul_f64                      :ref:`vdst<amdgpu_synid9_vdst64_0>`,               :ref:`src0<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_mul_hi_i32                   :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
+    v_mul_hi_i32_i24_e64           :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
+    v_mul_hi_u32                   :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
+    v_mul_hi_u32_u24_e64           :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
+    v_mul_i32_i24_e64              :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
+    v_mul_legacy_f32_e64           :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_mul_lo_u16_e64               :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
+    v_mul_lo_u32                   :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
+    v_mul_u32_u24_e64              :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
     v_nop_e64
-    v_not_b32_e64                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src<amdgpu_synid9_src32_1>`
-    v_or3_b32                      :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`,       :ref:`src2<amdgpu_synid9_src32_1>`
-    v_or_b32_e64                   :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
-    v_pack_b32_f16                 :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                         :ref:`vop3_op_sel<amdgpu_synid_vop3_op_sel>`
-    v_perm_b32                     :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`,       :ref:`src2<amdgpu_synid9_src32_1>`
-    v_qsad_pk_u16_u8               :ref:`vdst<amdgpu_synid9_vdst64_0>`::ref:`b64<amdgpu_synid9_type_dev>`,            :ref:`src0<amdgpu_synid9_src64_1>`::ref:`b64<amdgpu_synid9_type_dev>`,    :ref:`src1<amdgpu_synid9_src32_1>`::ref:`b32<amdgpu_synid9_type_dev>`,   :ref:`src2<amdgpu_synid9_src64_1>`::ref:`b64<amdgpu_synid9_type_dev>`           :ref:`clamp<amdgpu_synid_clamp>`
-    v_rcp_f16_e64                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                                       :ref:`clamp<amdgpu_synid_clamp>`
-    v_rcp_f32_e64                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                                       :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_rcp_f64_e64                  :ref:`vdst<amdgpu_synid9_vdst64_0>`,                :ref:`src<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                                       :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_rcp_iflag_f32_e64            :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                                       :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_readlane_b32                 :ref:`sdst<amdgpu_synid9_sdst32_2>`,                :ref:`vsrc0<amdgpu_synid9_vsrc32_0>`,       :ref:`ssrc1<amdgpu_synid9_ssrc32_3>`
-    v_rndne_f16_e64                :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                                       :ref:`clamp<amdgpu_synid_clamp>`
-    v_rndne_f32_e64                :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                                       :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_rndne_f64_e64                :ref:`vdst<amdgpu_synid9_vdst64_0>`,                :ref:`src<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                                       :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_rsq_f16_e64                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                                       :ref:`clamp<amdgpu_synid_clamp>`
-    v_rsq_f32_e64                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                                       :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_rsq_f64_e64                  :ref:`vdst<amdgpu_synid9_vdst64_0>`,                :ref:`src<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                                       :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_sad_hi_u8                    :ref:`vdst<amdgpu_synid9_vdst32_0>`::ref:`u32<amdgpu_synid9_type_dev>`,            :ref:`src0<amdgpu_synid9_src32_1>`::ref:`u8x4<amdgpu_synid9_type_dev>`,   :ref:`src1<amdgpu_synid9_src32_1>`::ref:`u8x4<amdgpu_synid9_type_dev>`,  :ref:`src2<amdgpu_synid9_src32_1>`::ref:`u32<amdgpu_synid9_type_dev>`           :ref:`clamp<amdgpu_synid_clamp>`
-    v_sad_u16                      :ref:`vdst<amdgpu_synid9_vdst32_0>`::ref:`u32<amdgpu_synid9_type_dev>`,            :ref:`src0<amdgpu_synid9_src32_1>`::ref:`u16x2<amdgpu_synid9_type_dev>`,  :ref:`src1<amdgpu_synid9_src32_1>`::ref:`u16x2<amdgpu_synid9_type_dev>`, :ref:`src2<amdgpu_synid9_src32_1>`::ref:`u32<amdgpu_synid9_type_dev>`           :ref:`clamp<amdgpu_synid_clamp>`
-    v_sad_u32                      :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`,       :ref:`src2<amdgpu_synid9_src32_1>`               :ref:`clamp<amdgpu_synid_clamp>`
-    v_sad_u8                       :ref:`vdst<amdgpu_synid9_vdst32_0>`::ref:`u32<amdgpu_synid9_type_dev>`,            :ref:`src0<amdgpu_synid9_src32_1>`::ref:`u8x4<amdgpu_synid9_type_dev>`,   :ref:`src1<amdgpu_synid9_src32_1>`::ref:`u8x4<amdgpu_synid9_type_dev>`,  :ref:`src2<amdgpu_synid9_src32_1>`::ref:`u32<amdgpu_synid9_type_dev>`           :ref:`clamp<amdgpu_synid_clamp>`
-    v_sat_pk_u8_i16_e64            :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src<amdgpu_synid9_src32_1>`
-    v_screen_partition_4se_b32_e64 :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src<amdgpu_synid9_src32_1>`
-    v_sin_f16_e64                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                                       :ref:`clamp<amdgpu_synid_clamp>`
-    v_sin_f32_e64                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                                       :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_sqrt_f16_e64                 :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                                       :ref:`clamp<amdgpu_synid_clamp>`
-    v_sqrt_f32_e64                 :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                                       :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_sqrt_f64_e64                 :ref:`vdst<amdgpu_synid9_vdst64_0>`,                :ref:`src<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                                       :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_sub_co_u32_e64               :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`sdst<amdgpu_synid9_sdst64_0>`,     :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
-    v_sub_f16_e64                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_sub_f32_e64                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_sub_i16                      :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`                           :ref:`vop3_op_sel<amdgpu_synid_vop3_op_sel>` :ref:`clamp<amdgpu_synid_clamp>`
-    v_sub_i32                      :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
-    v_sub_u16_e64                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
-    v_sub_u32_e64                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
-    v_subb_co_u32_e64              :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`sdst<amdgpu_synid9_sdst64_0>`,     :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`,       :ref:`ssrc2<amdgpu_synid9_ssrc64_1>`
-    v_subbrev_co_u32_e64           :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`sdst<amdgpu_synid9_sdst64_0>`,     :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`,       :ref:`ssrc2<amdgpu_synid9_ssrc64_1>`
-    v_subrev_co_u32_e64            :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`sdst<amdgpu_synid9_sdst64_0>`,     :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
-    v_subrev_f16_e64               :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>`
-    v_subrev_f32_e64               :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                         :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_subrev_u16_e64               :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
-    v_subrev_u32_e64               :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
-    v_trig_preop_f64               :ref:`vdst<amdgpu_synid9_vdst64_0>`,                :ref:`src0<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`u32<amdgpu_synid9_type_dev>`                       :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_trunc_f16_e64                :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                                       :ref:`clamp<amdgpu_synid_clamp>`
-    v_trunc_f32_e64                :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                                       :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_trunc_f64_e64                :ref:`vdst<amdgpu_synid9_vdst64_0>`,                :ref:`src<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                                       :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
-    v_writelane_b32                :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`ssrc0<amdgpu_synid9_ssrc32_4>`,       :ref:`ssrc1<amdgpu_synid9_ssrc32_3>`
-    v_xad_u32                      :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`,       :ref:`src2<amdgpu_synid9_src32_1>`
-    v_xor_b32_e64                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,                :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
+    v_not_b32_e64                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src<amdgpu_synid9_src32_1>`
+    v_or3_b32                      :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`,       :ref:`src2<amdgpu_synid9_src32_1>`
+    v_or_b32_e64                   :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
+    v_pack_b32_f16                 :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                      :ref:`op_sel<amdgpu_synid_vop3_op_sel>`
+    v_perm_b32                     :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`,       :ref:`src2<amdgpu_synid9_src32_1>`
+    v_qsad_pk_u16_u8               :ref:`vdst<amdgpu_synid9_vdst64_0>`::ref:`b64<amdgpu_synid9_type_dev>`,           :ref:`src0<amdgpu_synid9_src64_1>`::ref:`b64<amdgpu_synid9_type_dev>`,    :ref:`src1<amdgpu_synid9_src32_1>`::ref:`b32<amdgpu_synid9_type_dev>`,   :ref:`src2<amdgpu_synid9_src64_1>`::ref:`b64<amdgpu_synid9_type_dev>`        :ref:`clamp<amdgpu_synid_clamp>`
+    v_rcp_f16_e64                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                                    :ref:`clamp<amdgpu_synid_clamp>`
+    v_rcp_f32_e64                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                                    :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_rcp_f64_e64                  :ref:`vdst<amdgpu_synid9_vdst64_0>`,               :ref:`src<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                                    :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_rcp_iflag_f32_e64            :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                                    :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_readlane_b32                 :ref:`sdst<amdgpu_synid9_sdst32_2>`,               :ref:`vsrc0<amdgpu_synid9_vsrc32_0>`,       :ref:`ssrc1<amdgpu_synid9_ssrc32_3>`
+    v_rndne_f16_e64                :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                                    :ref:`clamp<amdgpu_synid_clamp>`
+    v_rndne_f32_e64                :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                                    :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_rndne_f64_e64                :ref:`vdst<amdgpu_synid9_vdst64_0>`,               :ref:`src<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                                    :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_rsq_f16_e64                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                                    :ref:`clamp<amdgpu_synid_clamp>`
+    v_rsq_f32_e64                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                                    :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_rsq_f64_e64                  :ref:`vdst<amdgpu_synid9_vdst64_0>`,               :ref:`src<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                                    :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_sad_hi_u8                    :ref:`vdst<amdgpu_synid9_vdst32_0>`::ref:`u32<amdgpu_synid9_type_dev>`,           :ref:`src0<amdgpu_synid9_src32_1>`::ref:`u8x4<amdgpu_synid9_type_dev>`,   :ref:`src1<amdgpu_synid9_src32_1>`::ref:`u8x4<amdgpu_synid9_type_dev>`,  :ref:`src2<amdgpu_synid9_src32_1>`::ref:`u32<amdgpu_synid9_type_dev>`        :ref:`clamp<amdgpu_synid_clamp>`
+    v_sad_u16                      :ref:`vdst<amdgpu_synid9_vdst32_0>`::ref:`u32<amdgpu_synid9_type_dev>`,           :ref:`src0<amdgpu_synid9_src32_1>`::ref:`u16x2<amdgpu_synid9_type_dev>`,  :ref:`src1<amdgpu_synid9_src32_1>`::ref:`u16x2<amdgpu_synid9_type_dev>`, :ref:`src2<amdgpu_synid9_src32_1>`::ref:`u32<amdgpu_synid9_type_dev>`        :ref:`clamp<amdgpu_synid_clamp>`
+    v_sad_u32                      :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`,       :ref:`src2<amdgpu_synid9_src32_1>`            :ref:`clamp<amdgpu_synid_clamp>`
+    v_sad_u8                       :ref:`vdst<amdgpu_synid9_vdst32_0>`::ref:`u32<amdgpu_synid9_type_dev>`,           :ref:`src0<amdgpu_synid9_src32_1>`::ref:`u8x4<amdgpu_synid9_type_dev>`,   :ref:`src1<amdgpu_synid9_src32_1>`::ref:`u8x4<amdgpu_synid9_type_dev>`,  :ref:`src2<amdgpu_synid9_src32_1>`::ref:`u32<amdgpu_synid9_type_dev>`        :ref:`clamp<amdgpu_synid_clamp>`
+    v_sat_pk_u8_i16_e64            :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src<amdgpu_synid9_src32_1>`
+    v_screen_partition_4se_b32_e64 :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src<amdgpu_synid9_src32_1>`
+    v_sin_f16_e64                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                                    :ref:`clamp<amdgpu_synid_clamp>`
+    v_sin_f32_e64                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                                    :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_sqrt_f16_e64                 :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                                    :ref:`clamp<amdgpu_synid_clamp>`
+    v_sqrt_f32_e64                 :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                                    :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_sqrt_f64_e64                 :ref:`vdst<amdgpu_synid9_vdst64_0>`,               :ref:`src<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                                    :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_sub_co_u32_e64               :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`sdst<amdgpu_synid9_sdst64_0>`,    :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
+    v_sub_f16_e64                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_sub_f32_e64                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_sub_i16                      :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`                        :ref:`op_sel<amdgpu_synid_vop3_op_sel>` :ref:`clamp<amdgpu_synid_clamp>`
+    v_sub_i32                      :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
+    v_sub_u16_e64                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
+    v_sub_u32_e64                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
+    v_subb_co_u32_e64              :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`sdst<amdgpu_synid9_sdst64_0>`,    :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`,       :ref:`ssrc2<amdgpu_synid9_ssrc64_1>`
+    v_subbrev_co_u32_e64           :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`sdst<amdgpu_synid9_sdst64_0>`,    :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`,       :ref:`ssrc2<amdgpu_synid9_ssrc64_1>`
+    v_subrev_co_u32_e64            :ref:`vdst<amdgpu_synid9_vdst32_0>`,      :ref:`sdst<amdgpu_synid9_sdst64_0>`,    :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
+    v_subrev_f16_e64               :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>`
+    v_subrev_f32_e64               :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                      :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_subrev_u16_e64               :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
+    v_subrev_u32_e64               :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
+    v_trig_preop_f64               :ref:`vdst<amdgpu_synid9_vdst64_0>`,               :ref:`src0<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`,      :ref:`src1<amdgpu_synid9_src32_1>`::ref:`u32<amdgpu_synid9_type_dev>`                    :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_trunc_f16_e64                :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                                    :ref:`clamp<amdgpu_synid_clamp>`
+    v_trunc_f32_e64                :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src<amdgpu_synid9_src32_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                                    :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_trunc_f64_e64                :ref:`vdst<amdgpu_synid9_vdst64_0>`,               :ref:`src<amdgpu_synid9_src64_1>`::ref:`m<amdgpu_synid9_mod_vop3_abs_neg>`                                    :ref:`clamp<amdgpu_synid_clamp>` :ref:`omod<amdgpu_synid_omod>`
+    v_writelane_b32                :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`ssrc0<amdgpu_synid9_ssrc32_4>`,       :ref:`ssrc1<amdgpu_synid9_ssrc32_3>`
+    v_xad_u32                      :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`,       :ref:`src2<amdgpu_synid9_src32_1>`
+    v_xor_b32_e64                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,               :ref:`src0<amdgpu_synid9_src32_1>`,        :ref:`src1<amdgpu_synid9_src32_1>`
 
 VOP3P
 -----------------------
 
 .. parsed-literal::
 
-    **INSTRUCTION**                    **DST**       **SRC0**        **SRC1**      **SRC2**           **MODIFIERS**
-    \ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|
-    v_mad_mix_f32                  :ref:`vdst<amdgpu_synid9_vdst32_0>`,     :ref:`src0<amdgpu_synid9_src32_1>`::ref:`fx<amdgpu_synid9_mad_type_dev>`,    :ref:`src1<amdgpu_synid9_src32_1>`::ref:`fx<amdgpu_synid9_mad_type_dev>`,  :ref:`src2<amdgpu_synid9_src32_1>`::ref:`fx<amdgpu_synid9_mad_type_dev>`        :ref:`mad_mix_op_sel<amdgpu_synid_mad_mix_op_sel>` :ref:`mad_mix_op_sel_hi<amdgpu_synid_mad_mix_op_sel_hi>` :ref:`clamp<amdgpu_synid_clamp>`
-    v_mad_mixhi_f16                :ref:`vdst<amdgpu_synid9_vdst32_0>`,     :ref:`src0<amdgpu_synid9_src32_1>`::ref:`fx<amdgpu_synid9_mad_type_dev>`,    :ref:`src1<amdgpu_synid9_src32_1>`::ref:`fx<amdgpu_synid9_mad_type_dev>`,  :ref:`src2<amdgpu_synid9_src32_1>`::ref:`fx<amdgpu_synid9_mad_type_dev>`        :ref:`mad_mix_op_sel<amdgpu_synid_mad_mix_op_sel>` :ref:`mad_mix_op_sel_hi<amdgpu_synid_mad_mix_op_sel_hi>` :ref:`clamp<amdgpu_synid_clamp>`
-    v_mad_mixlo_f16                :ref:`vdst<amdgpu_synid9_vdst32_0>`,     :ref:`src0<amdgpu_synid9_src32_1>`::ref:`fx<amdgpu_synid9_mad_type_dev>`,    :ref:`src1<amdgpu_synid9_src32_1>`::ref:`fx<amdgpu_synid9_mad_type_dev>`,  :ref:`src2<amdgpu_synid9_src32_1>`::ref:`fx<amdgpu_synid9_mad_type_dev>`        :ref:`mad_mix_op_sel<amdgpu_synid_mad_mix_op_sel>` :ref:`mad_mix_op_sel_hi<amdgpu_synid_mad_mix_op_sel_hi>` :ref:`clamp<amdgpu_synid_clamp>`
-    v_pk_add_f16                   :ref:`vdst<amdgpu_synid9_vdst32_0>`,     :ref:`src0<amdgpu_synid9_src32_1>`,       :ref:`src1<amdgpu_synid9_src32_1>`                     :ref:`op_sel<amdgpu_synid_op_sel>` :ref:`op_sel_hi<amdgpu_synid_op_sel_hi>` :ref:`neg_lo<amdgpu_synid_neg_lo>` :ref:`neg_hi<amdgpu_synid_neg_hi>` :ref:`clamp<amdgpu_synid_clamp>`
-    v_pk_add_i16                   :ref:`vdst<amdgpu_synid9_vdst32_0>`,     :ref:`src0<amdgpu_synid9_src32_1>`,       :ref:`src1<amdgpu_synid9_src32_1>`                     :ref:`op_sel<amdgpu_synid_op_sel>` :ref:`op_sel_hi<amdgpu_synid_op_sel_hi>` :ref:`clamp<amdgpu_synid_clamp>`
-    v_pk_add_u16                   :ref:`vdst<amdgpu_synid9_vdst32_0>`,     :ref:`src0<amdgpu_synid9_src32_1>`,       :ref:`src1<amdgpu_synid9_src32_1>`                     :ref:`op_sel<amdgpu_synid_op_sel>` :ref:`op_sel_hi<amdgpu_synid_op_sel_hi>` :ref:`clamp<amdgpu_synid_clamp>`
-    v_pk_ashrrev_i16               :ref:`vdst<amdgpu_synid9_vdst32_0>`,     :ref:`src0<amdgpu_synid9_src32_1>`::ref:`u16x2<amdgpu_synid9_type_dev>`, :ref:`src1<amdgpu_synid9_src32_1>`                     :ref:`op_sel<amdgpu_synid_op_sel>` :ref:`op_sel_hi<amdgpu_synid_op_sel_hi>`
-    v_pk_fma_f16                   :ref:`vdst<amdgpu_synid9_vdst32_0>`,     :ref:`src0<amdgpu_synid9_src32_1>`,       :ref:`src1<amdgpu_synid9_src32_1>`,     :ref:`src2<amdgpu_synid9_src32_1>`           :ref:`op_sel<amdgpu_synid_op_sel>` :ref:`op_sel_hi<amdgpu_synid_op_sel_hi>` :ref:`neg_lo<amdgpu_synid_neg_lo>` :ref:`neg_hi<amdgpu_synid_neg_hi>` :ref:`clamp<amdgpu_synid_clamp>`
-    v_pk_lshlrev_b16               :ref:`vdst<amdgpu_synid9_vdst32_0>`,     :ref:`src0<amdgpu_synid9_src32_1>`::ref:`u16x2<amdgpu_synid9_type_dev>`, :ref:`src1<amdgpu_synid9_src32_1>`                     :ref:`op_sel<amdgpu_synid_op_sel>` :ref:`op_sel_hi<amdgpu_synid_op_sel_hi>`
-    v_pk_lshrrev_b16               :ref:`vdst<amdgpu_synid9_vdst32_0>`,     :ref:`src0<amdgpu_synid9_src32_1>`::ref:`u16x2<amdgpu_synid9_type_dev>`, :ref:`src1<amdgpu_synid9_src32_1>`                     :ref:`op_sel<amdgpu_synid_op_sel>` :ref:`op_sel_hi<amdgpu_synid_op_sel_hi>`
-    v_pk_mad_i16                   :ref:`vdst<amdgpu_synid9_vdst32_0>`,     :ref:`src0<amdgpu_synid9_src32_1>`,       :ref:`src1<amdgpu_synid9_src32_1>`,     :ref:`src2<amdgpu_synid9_src32_1>`           :ref:`op_sel<amdgpu_synid_op_sel>` :ref:`op_sel_hi<amdgpu_synid_op_sel_hi>` :ref:`clamp<amdgpu_synid_clamp>`
-    v_pk_mad_u16                   :ref:`vdst<amdgpu_synid9_vdst32_0>`,     :ref:`src0<amdgpu_synid9_src32_1>`,       :ref:`src1<amdgpu_synid9_src32_1>`,     :ref:`src2<amdgpu_synid9_src32_1>`           :ref:`op_sel<amdgpu_synid_op_sel>` :ref:`op_sel_hi<amdgpu_synid_op_sel_hi>` :ref:`clamp<amdgpu_synid_clamp>`
-    v_pk_max_f16                   :ref:`vdst<amdgpu_synid9_vdst32_0>`,     :ref:`src0<amdgpu_synid9_src32_1>`,       :ref:`src1<amdgpu_synid9_src32_1>`                     :ref:`op_sel<amdgpu_synid_op_sel>` :ref:`op_sel_hi<amdgpu_synid_op_sel_hi>` :ref:`neg_lo<amdgpu_synid_neg_lo>` :ref:`neg_hi<amdgpu_synid_neg_hi>` :ref:`clamp<amdgpu_synid_clamp>`
-    v_pk_max_i16                   :ref:`vdst<amdgpu_synid9_vdst32_0>`,     :ref:`src0<amdgpu_synid9_src32_1>`,       :ref:`src1<amdgpu_synid9_src32_1>`                     :ref:`op_sel<amdgpu_synid_op_sel>` :ref:`op_sel_hi<amdgpu_synid_op_sel_hi>`
-    v_pk_max_u16                   :ref:`vdst<amdgpu_synid9_vdst32_0>`,     :ref:`src0<amdgpu_synid9_src32_1>`,       :ref:`src1<amdgpu_synid9_src32_1>`                     :ref:`op_sel<amdgpu_synid_op_sel>` :ref:`op_sel_hi<amdgpu_synid_op_sel_hi>`
-    v_pk_min_f16                   :ref:`vdst<amdgpu_synid9_vdst32_0>`,     :ref:`src0<amdgpu_synid9_src32_1>`,       :ref:`src1<amdgpu_synid9_src32_1>`                     :ref:`op_sel<amdgpu_synid_op_sel>` :ref:`op_sel_hi<amdgpu_synid_op_sel_hi>` :ref:`neg_lo<amdgpu_synid_neg_lo>` :ref:`neg_hi<amdgpu_synid_neg_hi>` :ref:`clamp<amdgpu_synid_clamp>`
-    v_pk_min_i16                   :ref:`vdst<amdgpu_synid9_vdst32_0>`,     :ref:`src0<amdgpu_synid9_src32_1>`,       :ref:`src1<amdgpu_synid9_src32_1>`                     :ref:`op_sel<amdgpu_synid_op_sel>` :ref:`op_sel_hi<amdgpu_synid_op_sel_hi>`
-    v_pk_min_u16                   :ref:`vdst<amdgpu_synid9_vdst32_0>`,     :ref:`src0<amdgpu_synid9_src32_1>`,       :ref:`src1<amdgpu_synid9_src32_1>`                     :ref:`op_sel<amdgpu_synid_op_sel>` :ref:`op_sel_hi<amdgpu_synid_op_sel_hi>`
-    v_pk_mul_f16                   :ref:`vdst<amdgpu_synid9_vdst32_0>`,     :ref:`src0<amdgpu_synid9_src32_1>`,       :ref:`src1<amdgpu_synid9_src32_1>`                     :ref:`op_sel<amdgpu_synid_op_sel>` :ref:`op_sel_hi<amdgpu_synid_op_sel_hi>` :ref:`neg_lo<amdgpu_synid_neg_lo>` :ref:`neg_hi<amdgpu_synid_neg_hi>` :ref:`clamp<amdgpu_synid_clamp>`
-    v_pk_mul_lo_u16                :ref:`vdst<amdgpu_synid9_vdst32_0>`,     :ref:`src0<amdgpu_synid9_src32_1>`,       :ref:`src1<amdgpu_synid9_src32_1>`                     :ref:`op_sel<amdgpu_synid_op_sel>` :ref:`op_sel_hi<amdgpu_synid_op_sel_hi>`
-    v_pk_sub_i16                   :ref:`vdst<amdgpu_synid9_vdst32_0>`,     :ref:`src0<amdgpu_synid9_src32_1>`,       :ref:`src1<amdgpu_synid9_src32_1>`                     :ref:`op_sel<amdgpu_synid_op_sel>` :ref:`op_sel_hi<amdgpu_synid_op_sel_hi>` :ref:`clamp<amdgpu_synid_clamp>`
-    v_pk_sub_u16                   :ref:`vdst<amdgpu_synid9_vdst32_0>`,     :ref:`src0<amdgpu_synid9_src32_1>`,       :ref:`src1<amdgpu_synid9_src32_1>`                     :ref:`op_sel<amdgpu_synid_op_sel>` :ref:`op_sel_hi<amdgpu_synid_op_sel_hi>` :ref:`clamp<amdgpu_synid_clamp>`
+    **INSTRUCTION**           **DST**      **SRC0**        **SRC1**     **SRC2**       **MODIFIERS**
+    \ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|\ |---|
+    v_mad_mix_f32         :ref:`vdst<amdgpu_synid9_vdst32_0>`,    :ref:`src0<amdgpu_synid9_src32_1>`::ref:`fx<amdgpu_synid9_mad_type_dev>`,    :ref:`src1<amdgpu_synid9_src32_1>`::ref:`fx<amdgpu_synid9_mad_type_dev>`, :ref:`src2<amdgpu_synid9_src32_1>`::ref:`fx<amdgpu_synid9_mad_type_dev>`    :ref:`m_op_sel<amdgpu_synid_mad_mix_op_sel>` :ref:`m_op_sel_hi<amdgpu_synid_mad_mix_op_sel_hi>` :ref:`clamp<amdgpu_synid_clamp>`
+    v_mad_mixhi_f16       :ref:`vdst<amdgpu_synid9_vdst32_0>`,    :ref:`src0<amdgpu_synid9_src32_1>`::ref:`fx<amdgpu_synid9_mad_type_dev>`,    :ref:`src1<amdgpu_synid9_src32_1>`::ref:`fx<amdgpu_synid9_mad_type_dev>`, :ref:`src2<amdgpu_synid9_src32_1>`::ref:`fx<amdgpu_synid9_mad_type_dev>`    :ref:`m_op_sel<amdgpu_synid_mad_mix_op_sel>` :ref:`m_op_sel_hi<amdgpu_synid_mad_mix_op_sel_hi>` :ref:`clamp<amdgpu_synid_clamp>`
+    v_mad_mixlo_f16       :ref:`vdst<amdgpu_synid9_vdst32_0>`,    :ref:`src0<amdgpu_synid9_src32_1>`::ref:`fx<amdgpu_synid9_mad_type_dev>`,    :ref:`src1<amdgpu_synid9_src32_1>`::ref:`fx<amdgpu_synid9_mad_type_dev>`, :ref:`src2<amdgpu_synid9_src32_1>`::ref:`fx<amdgpu_synid9_mad_type_dev>`    :ref:`m_op_sel<amdgpu_synid_mad_mix_op_sel>` :ref:`m_op_sel_hi<amdgpu_synid_mad_mix_op_sel_hi>` :ref:`clamp<amdgpu_synid_clamp>`
+    v_pk_add_f16          :ref:`vdst<amdgpu_synid9_vdst32_0>`,    :ref:`src0<amdgpu_synid9_src32_1>`,       :ref:`src1<amdgpu_synid9_src32_1>`                :ref:`op_sel<amdgpu_synid_op_sel>` :ref:`op_sel_hi<amdgpu_synid_op_sel_hi>` :ref:`neg_lo<amdgpu_synid_neg_lo>` :ref:`neg_hi<amdgpu_synid_neg_hi>` :ref:`clamp<amdgpu_synid_clamp>`
+    v_pk_add_i16          :ref:`vdst<amdgpu_synid9_vdst32_0>`,    :ref:`src0<amdgpu_synid9_src32_1>`,       :ref:`src1<amdgpu_synid9_src32_1>`                :ref:`op_sel<amdgpu_synid_op_sel>` :ref:`op_sel_hi<amdgpu_synid_op_sel_hi>` :ref:`clamp<amdgpu_synid_clamp>`
+    v_pk_add_u16          :ref:`vdst<amdgpu_synid9_vdst32_0>`,    :ref:`src0<amdgpu_synid9_src32_1>`,       :ref:`src1<amdgpu_synid9_src32_1>`                :ref:`op_sel<amdgpu_synid_op_sel>` :ref:`op_sel_hi<amdgpu_synid_op_sel_hi>` :ref:`clamp<amdgpu_synid_clamp>`
+    v_pk_ashrrev_i16      :ref:`vdst<amdgpu_synid9_vdst32_0>`,    :ref:`src0<amdgpu_synid9_src32_1>`::ref:`u16x2<amdgpu_synid9_type_dev>`, :ref:`src1<amdgpu_synid9_src32_1>`                :ref:`op_sel<amdgpu_synid_op_sel>` :ref:`op_sel_hi<amdgpu_synid_op_sel_hi>`
+    v_pk_fma_f16          :ref:`vdst<amdgpu_synid9_vdst32_0>`,    :ref:`src0<amdgpu_synid9_src32_1>`,       :ref:`src1<amdgpu_synid9_src32_1>`,    :ref:`src2<amdgpu_synid9_src32_1>`       :ref:`op_sel<amdgpu_synid_op_sel>` :ref:`op_sel_hi<amdgpu_synid_op_sel_hi>` :ref:`neg_lo<amdgpu_synid_neg_lo>` :ref:`neg_hi<amdgpu_synid_neg_hi>` :ref:`clamp<amdgpu_synid_clamp>`
+    v_pk_lshlrev_b16      :ref:`vdst<amdgpu_synid9_vdst32_0>`,    :ref:`src0<amdgpu_synid9_src32_1>`::ref:`u16x2<amdgpu_synid9_type_dev>`, :ref:`src1<amdgpu_synid9_src32_1>`                :ref:`op_sel<amdgpu_synid_op_sel>` :ref:`op_sel_hi<amdgpu_synid_op_sel_hi>`
+    v_pk_lshrrev_b16      :ref:`vdst<amdgpu_synid9_vdst32_0>`,    :ref:`src0<amdgpu_synid9_src32_1>`::ref:`u16x2<amdgpu_synid9_type_dev>`, :ref:`src1<amdgpu_synid9_src32_1>`                :ref:`op_sel<amdgpu_synid_op_sel>` :ref:`op_sel_hi<amdgpu_synid_op_sel_hi>`
+    v_pk_mad_i16          :ref:`vdst<amdgpu_synid9_vdst32_0>`,    :ref:`src0<amdgpu_synid9_src32_1>`,       :ref:`src1<amdgpu_synid9_src32_1>`,    :ref:`src2<amdgpu_synid9_src32_1>`       :ref:`op_sel<amdgpu_synid_op_sel>` :ref:`op_sel_hi<amdgpu_synid_op_sel_hi>` :ref:`clamp<amdgpu_synid_clamp>`
+    v_pk_mad_u16          :ref:`vdst<amdgpu_synid9_vdst32_0>`,    :ref:`src0<amdgpu_synid9_src32_1>`,       :ref:`src1<amdgpu_synid9_src32_1>`,    :ref:`src2<amdgpu_synid9_src32_1>`       :ref:`op_sel<amdgpu_synid_op_sel>` :ref:`op_sel_hi<amdgpu_synid_op_sel_hi>` :ref:`clamp<amdgpu_synid_clamp>`
+    v_pk_max_f16          :ref:`vdst<amdgpu_synid9_vdst32_0>`,    :ref:`src0<amdgpu_synid9_src32_1>`,       :ref:`src1<amdgpu_synid9_src32_1>`                :ref:`op_sel<amdgpu_synid_op_sel>` :ref:`op_sel_hi<amdgpu_synid_op_sel_hi>` :ref:`neg_lo<amdgpu_synid_neg_lo>` :ref:`neg_hi<amdgpu_synid_neg_hi>` :ref:`clamp<amdgpu_synid_clamp>`
+    v_pk_max_i16          :ref:`vdst<amdgpu_synid9_vdst32_0>`,    :ref:`src0<amdgpu_synid9_src32_1>`,       :ref:`src1<amdgpu_synid9_src32_1>`                :ref:`op_sel<amdgpu_synid_op_sel>` :ref:`op_sel_hi<amdgpu_synid_op_sel_hi>`
+    v_pk_max_u16          :ref:`vdst<amdgpu_synid9_vdst32_0>`,    :ref:`src0<amdgpu_synid9_src32_1>`,       :ref:`src1<amdgpu_synid9_src32_1>`                :ref:`op_sel<amdgpu_synid_op_sel>` :ref:`op_sel_hi<amdgpu_synid_op_sel_hi>`
+    v_pk_min_f16          :ref:`vdst<amdgpu_synid9_vdst32_0>`,    :ref:`src0<amdgpu_synid9_src32_1>`,       :ref:`src1<amdgpu_synid9_src32_1>`                :ref:`op_sel<amdgpu_synid_op_sel>` :ref:`op_sel_hi<amdgpu_synid_op_sel_hi>` :ref:`neg_lo<amdgpu_synid_neg_lo>` :ref:`neg_hi<amdgpu_synid_neg_hi>` :ref:`clamp<amdgpu_synid_clamp>`
+    v_pk_min_i16          :ref:`vdst<amdgpu_synid9_vdst32_0>`,    :ref:`src0<amdgpu_synid9_src32_1>`,       :ref:`src1<amdgpu_synid9_src32_1>`                :ref:`op_sel<amdgpu_synid_op_sel>` :ref:`op_sel_hi<amdgpu_synid_op_sel_hi>`
+    v_pk_min_u16          :ref:`vdst<amdgpu_synid9_vdst32_0>`,    :ref:`src0<amdgpu_synid9_src32_1>`,       :ref:`src1<amdgpu_synid9_src32_1>`                :ref:`op_sel<amdgpu_synid_op_sel>` :ref:`op_sel_hi<amdgpu_synid_op_sel_hi>`
+    v_pk_mul_f16          :ref:`vdst<amdgpu_synid9_vdst32_0>`,    :ref:`src0<amdgpu_synid9_src32_1>`,       :ref:`src1<amdgpu_synid9_src32_1>`                :ref:`op_sel<amdgpu_synid_op_sel>` :ref:`op_sel_hi<amdgpu_synid_op_sel_hi>` :ref:`neg_lo<amdgpu_synid_neg_lo>` :ref:`neg_hi<amdgpu_synid_neg_hi>` :ref:`clamp<amdgpu_synid_clamp>`
+    v_pk_mul_lo_u16       :ref:`vdst<amdgpu_synid9_vdst32_0>`,    :ref:`src0<amdgpu_synid9_src32_1>`,       :ref:`src1<amdgpu_synid9_src32_1>`                :ref:`op_sel<amdgpu_synid_op_sel>` :ref:`op_sel_hi<amdgpu_synid_op_sel_hi>`
+    v_pk_sub_i16          :ref:`vdst<amdgpu_synid9_vdst32_0>`,    :ref:`src0<amdgpu_synid9_src32_1>`,       :ref:`src1<amdgpu_synid9_src32_1>`                :ref:`op_sel<amdgpu_synid_op_sel>` :ref:`op_sel_hi<amdgpu_synid_op_sel_hi>` :ref:`clamp<amdgpu_synid_clamp>`
+    v_pk_sub_u16          :ref:`vdst<amdgpu_synid9_vdst32_0>`,    :ref:`src0<amdgpu_synid9_src32_1>`,       :ref:`src1<amdgpu_synid9_src32_1>`                :ref:`op_sel<amdgpu_synid_op_sel>` :ref:`op_sel_hi<amdgpu_synid_op_sel_hi>` :ref:`clamp<amdgpu_synid_clamp>`
 
 VOPC
 -----------------------
diff --git a/docs/AMDGPU/gfx9_mad_type_dev.rst b/docs/AMDGPU/gfx9_mad_type_dev.rst
index 53c78e441128883e447d0b2ef2573c59a2d5861e..0602f0821762762f3112f4a8471964b45a6d8795 100644
--- a/docs/AMDGPU/gfx9_mad_type_dev.rst
+++ b/docs/AMDGPU/gfx9_mad_type_dev.rst
@@ -12,6 +12,6 @@ fx
 
 This is an *f32* or *f16* operand depending on instruction modifiers:
 
-* Operand size is controlled by :ref:`mad_mix_op_sel_hi<amdgpu_synid_mad_mix_op_sel_hi>`.
-* Location of 16-bit operand is controlled by :ref:`mad_mix_op_sel<amdgpu_synid_mad_mix_op_sel>`.
+* Operand size is controlled by :ref:`m_op_sel_hi<amdgpu_synid_mad_mix_op_sel_hi>`.
+* Location of 16-bit operand is controlled by :ref:`m_op_sel<amdgpu_synid_mad_mix_op_sel>`.
 
diff --git a/docs/AMDGPU/gfx9_vaddr_flat_global.rst b/docs/AMDGPU/gfx9_vaddr_flat_global.rst
index 38beb6c44321bcdc81c30e05635439326e91e191..e08e8fb762d8d5325f5f2b0f8379905d584568c8 100644
--- a/docs/AMDGPU/gfx9_vaddr_flat_global.rst
+++ b/docs/AMDGPU/gfx9_vaddr_flat_global.rst
@@ -12,8 +12,8 @@ vaddr
 
 A 64-bit flat global address or a 32-bit offset depending on addressing mode:
 
-* Address = :ref:`vaddr<amdgpu_synid9_vaddr_flat_global>` + :ref:`flat_offset13<amdgpu_synid_flat_offset13>`. :ref:`vaddr<amdgpu_synid9_vaddr_flat_global>` is a 64-bit address. This mode is indicated by :ref:`saddr<amdgpu_synid9_saddr_flat_global>` set to :ref:`off<amdgpu_synid_off>`.
-* Address = :ref:`saddr<amdgpu_synid9_saddr_flat_global>` + :ref:`vaddr<amdgpu_synid9_vaddr_flat_global>` + :ref:`flat_offset13<amdgpu_synid_flat_offset13>`. :ref:`vaddr<amdgpu_synid9_vaddr_flat_global>` is a 32-bit offset. This mode is used when :ref:`saddr<amdgpu_synid9_saddr_flat_global>` is not :ref:`off<amdgpu_synid_off>`.
+* Address = :ref:`vaddr<amdgpu_synid9_vaddr_flat_global>` + :ref:`offset13s<amdgpu_synid_flat_offset13s>`. :ref:`vaddr<amdgpu_synid9_vaddr_flat_global>` is a 64-bit address. This mode is indicated by :ref:`saddr<amdgpu_synid9_saddr_flat_global>` set to :ref:`off<amdgpu_synid_off>`.
+* Address = :ref:`saddr<amdgpu_synid9_saddr_flat_global>` + :ref:`vaddr<amdgpu_synid9_vaddr_flat_global>` + :ref:`offset13s<amdgpu_synid_flat_offset13s>`. :ref:`vaddr<amdgpu_synid9_vaddr_flat_global>` is a 32-bit offset. This mode is used when :ref:`saddr<amdgpu_synid9_saddr_flat_global>` is not :ref:`off<amdgpu_synid_off>`.
 
 .. WARNING:: Assembler currently expects a 64-bit *vaddr* regardless of addressing mode. This have to be fixed.
 
diff --git a/docs/AMDGPUInstructionNotation.rst b/docs/AMDGPUInstructionNotation.rst
index 2b41d5b81949f4a478ac6e625a5eb5951e2d0cbf..7f23cf406cfece1dad2f671f16db318f177f0d4f 100644
--- a/docs/AMDGPUInstructionNotation.rst
+++ b/docs/AMDGPUInstructionNotation.rst
@@ -73,7 +73,7 @@ Where:
     :dst           An input operand which may also serve as a destination
                    if :ref:`glc<amdgpu_synid_glc>` modifier is specified.
     :fx            This is an *f32* or *f16* operand depending on
-                   :ref:`mad_mix_op_sel_hi<amdgpu_synid_mad_mix_op_sel_hi>` modifier.
+                   :ref:`m_op_sel_hi<amdgpu_synid_mad_mix_op_sel_hi>` modifier.
     :<type>        Operand *type* differs from *type*
                    :ref:`implied by the opcode name<amdgpu_syn_instruction_type>`.
                    This tag specifies actual operand *type*.
diff --git a/docs/AMDGPUModifierSyntax.rst b/docs/AMDGPUModifierSyntax.rst
index e2b8bb3f952a7c6d05c97db09c142db0cfe67ad7..1a555b678324d1e6e49f3760c973cfec1699059b 100644
--- a/docs/AMDGPUModifierSyntax.rst
+++ b/docs/AMDGPUModifierSyntax.rst
@@ -27,8 +27,8 @@ DS Modifiers
 
 .. _amdgpu_synid_ds_offset8:
 
-ds_offset8
-~~~~~~~~~~
+offset8
+~~~~~~~
 
 Specifies an immediate unsigned 8-bit offset, in bytes. The default value is 0.
 
@@ -50,8 +50,8 @@ Examples:
 
 .. _amdgpu_synid_ds_offset16:
 
-ds_offset16
-~~~~~~~~~~~
+offset16
+~~~~~~~~
 
 Specifies an immediate unsigned 16-bit offset, in bytes. The default value is 0.
 
@@ -73,8 +73,8 @@ Examples:
 
 .. _amdgpu_synid_sw_offset16:
 
-sw_offset16
-~~~~~~~~~~~
+pattern
+~~~~~~~
 
 This is a special modifier which may be used with *ds_swizzle_b32* instruction only.
 It specifies a swizzle pattern in numeric or symbolic form. The default value is 0.
@@ -205,8 +205,8 @@ FLAT Modifiers
 
 .. _amdgpu_synid_flat_offset12:
 
-flat_offset12
-~~~~~~~~~~~~~
+offset12
+~~~~~~~~
 
 Specifies an immediate unsigned 12-bit offset, in bytes. The default value is 0.
 
@@ -226,10 +226,10 @@ Examples:
   offset:4095
   offset:0xff
 
-.. _amdgpu_synid_flat_offset13:
+.. _amdgpu_synid_flat_offset13s:
 
-flat_offset13
-~~~~~~~~~~~~~
+offset13s
+~~~~~~~~~
 
 Specifies an immediate signed 13-bit offset, in bytes. The default value is 0.
 
@@ -238,7 +238,7 @@ Can be used with *global/scratch* opcodes only. GFX9 only.
     ============================ =======================================================
     Syntax                       Description
     ============================ =======================================================
-    offset:{-4096..+4095}        Specifies a 13-bit signed offset as an
+    offset:{-4096..4095}         Specifies a 13-bit signed offset as an
                                  :ref:`integer number <amdgpu_synid_integer_number>`.
     ============================ =======================================================
 
@@ -353,7 +353,7 @@ GFX7 and GFX8 only.
     r128                Specifies 128 bits texture resource size.
     =================== ================================================
 
-.. WARNING:: Using this modifier should descrease *rsrc* register size from 8 to 4 dwords, but assembler does not currently support this feature.
+.. WARNING:: Using this modifier should descrease *rsrc* operand size from 8 to 4 dwords, but assembler does not currently support this feature.
 
 tfe
 ~~~
@@ -545,8 +545,8 @@ GFX7 only. Cannot be used with :ref:`offen<amdgpu_synid_offen>` and
 
 .. _amdgpu_synid_buf_offset12:
 
-buf_offset12
-~~~~~~~~~~~~
+offset12
+~~~~~~~~
 
 Specifies an immediate unsigned 12-bit offset, in bytes. The default value is 0.
 
@@ -889,8 +889,8 @@ VOP3 Modifiers
 
 .. _amdgpu_synid_vop3_op_sel:
 
-vop3_op_sel
-~~~~~~~~~~~
+op_sel
+~~~~~~
 
 Selects the low [15:0] or high [31:16] operand bits for source and destination operands.
 By default, low bits are used for all operands.
@@ -1177,11 +1177,11 @@ GFX9 only.
 
 .. _amdgpu_synid_mad_mix_op_sel:
 
-mad_mix_op_sel
-~~~~~~~~~~~~~~
+m_op_sel
+~~~~~~~~
 
 This operand has meaning only for 16-bit source operands as indicated by
-:ref:`mad_mix_op_sel_hi<amdgpu_synid_mad_mix_op_sel_hi>`.
+:ref:`m_op_sel_hi<amdgpu_synid_mad_mix_op_sel_hi>`.
 It specifies to select either the low [15:0] or high [31:16] operand bits
 as input to the operation.
 
@@ -1206,8 +1206,8 @@ Examples:
 
 .. _amdgpu_synid_mad_mix_op_sel_hi:
 
-mad_mix_op_sel_hi
-~~~~~~~~~~~~~~~~~
+m_op_sel_hi
+~~~~~~~~~~~
 
 Selects the size of source operands: either 32 bits or 16 bits.
 By default, 32 bits are used for all source operands.
@@ -1218,7 +1218,7 @@ operands. First value controls src0, second value controls src1 and so on.
 The value 0 indicates 32 bits, the value 1 indicates 16 bits.
 
 The location of 16 bits in the operand may be specified by
-:ref:`mad_mix_op_sel<amdgpu_synid_mad_mix_op_sel>`.
+:ref:`m_op_sel<amdgpu_synid_mad_mix_op_sel>`.
 
     ======================================== ====================================
     Syntax                                   Description
diff --git a/docs/AMDGPUOperandSyntax.rst b/docs/AMDGPUOperandSyntax.rst
index 5f5d822526530f04cc34265caf50ae63d0589e2b..8713c72a253692615cca935808b5dce8cb8b7242 100644
--- a/docs/AMDGPUOperandSyntax.rst
+++ b/docs/AMDGPUOperandSyntax.rst
@@ -523,22 +523,22 @@ Floating-point *inline constants* are converted to
 :ref:`expected operand type<amdgpu_syn_instruction_type>`
 as described :ref:`here<amdgpu_synid_fp_const_conv>`.
 
-    ================================== ===================================================== ==================
-    Value                              Note                                                  Availability
-    ================================== ===================================================== ==================
-    0.0                                The same as integer constant 0.                       All GPUs
-    0.5                                Floating-point constant 0.5                           All GPUs
-    1.0                                Floating-point constant 1.0                           All GPUs
-    2.0                                Floating-point constant 2.0                           All GPUs
-    4.0                                Floating-point constant 4.0                           All GPUs
-    -0.5                               Floating-point constant -0.5                          All GPUs
-    -1.0                               Floating-point constant -1.0                          All GPUs
-    -2.0                               Floating-point constant -2.0                          All GPUs
-    -4.0                               Floating-point constant -4.0                          All GPUs
-    0.1592                             1.0/(2.0*pi). Use only for 16-bit operands.           GFX8, GFX9
-    0.15915494                         1.0/(2.0*pi). Use only for 16- and 32-bit operands.   GFX8, GFX9
-    0.159154943091895317852646485335   1.0/(2.0*pi).                                         GFX8, GFX9
-    ================================== ===================================================== ==================
+    ===================== ===================================================== ==================
+    Value                 Note                                                  Availability
+    ===================== ===================================================== ==================
+    0.0                   The same as integer constant 0.                       All GPUs
+    0.5                   Floating-point constant 0.5                           All GPUs
+    1.0                   Floating-point constant 1.0                           All GPUs
+    2.0                   Floating-point constant 2.0                           All GPUs
+    4.0                   Floating-point constant 4.0                           All GPUs
+    -0.5                  Floating-point constant -0.5                          All GPUs
+    -1.0                  Floating-point constant -1.0                          All GPUs
+    -2.0                  Floating-point constant -2.0                          All GPUs
+    -4.0                  Floating-point constant -4.0                          All GPUs
+    0.1592                1.0/(2.0*pi). Use only for 16-bit operands.           GFX8, GFX9
+    0.15915494            1.0/(2.0*pi). Use only for 16- and 32-bit operands.   GFX8, GFX9
+    0.15915494309189532   1.0/(2.0*pi).                                         GFX8, GFX9
+    ===================== ===================================================== ==================
 
 .. WARNING:: GFX7 does not support inline constants for *f16* operands.
 
@@ -950,13 +950,13 @@ When used as operands they are converted to
     ============== ============== =============== ====================================================================
     Expected type  Condition      Result          Note
     ============== ============== =============== ====================================================================
-    i16, u16, b16  cond(num, 16)  num.u16         Truncate to 16 bits.
-    i32, u32, b32  cond(num, 32)  num.u32         Truncate to 32 bits.
-    i64            cond(num, 32)  {-1, num.i32}   Truncate to 32 bits and then sign-extend the result to 64 bits.
-    u64, b64       cond(num, 32)  { 0, num.u32}   Truncate to 32 bits and then zero-extend the result to 64 bits.
-    f16            cond(num, 16)  num.u16         Use low 16 bits as an f16 value.
-    f32            cond(num, 32)  num.u32         Use low 32 bits as an f32 value.
-    f64            cond(num, 32)  {num.u32, 0}    Use low 32 bits of the number as high 32 bits
+    i16, u16, b16  cond(num,16)   num.u16         Truncate to 16 bits.
+    i32, u32, b32  cond(num,32)   num.u32         Truncate to 32 bits.
+    i64            cond(num,32)   {-1,num.i32}    Truncate to 32 bits and then sign-extend the result to 64 bits.
+    u64, b64       cond(num,32)   { 0,num.u32}    Truncate to 32 bits and then zero-extend the result to 64 bits.
+    f16            cond(num,16)   num.u16         Use low 16 bits as an f16 value.
+    f32            cond(num,32)   num.u32         Use low 32 bits as an f32 value.
+    f64            cond(num,32)   {num.u32,0}     Use low 32 bits of the number as high 32 bits
                                                   of the result; low 32 bits of the result are zeroed.
     ============== ============== =============== ====================================================================
 
@@ -972,14 +972,14 @@ Examples of valid literals:
 .. parsed-literal::
 
     // GFX9
-
-    v_add_u16 v0, 0xff00, v0                     // value after conversion: 0xff00
-    v_add_u16 v0, 0xffffffffffffff00, v0         // value after conversion: 0xff00
-    v_add_u16 v0, -256, v0                       // value after conversion: 0xff00
-
-    s_bfe_i64 s[0:1], 0xffefffff, s3             // value after conversion: 0xffffffffffefffff
-    s_bfe_u64 s[0:1], 0xffefffff, s3             // value after conversion: 0x00000000ffefffff
-    v_ceil_f64_e32 v[0:1], 0xffefffff            // value after conversion: 0xffefffff00000000 (-1.7976922776554302e308)
+                                             // Literal value after conversion:
+    v_add_u16 v0, 0xff00, v0                 //   0xff00
+    v_add_u16 v0, 0xffffffffffffff00, v0     //   0xff00
+    v_add_u16 v0, -256, v0                   //   0xff00
+                                             // Literal value after conversion:
+    s_bfe_i64 s[0:1], 0xffefffff, s3         //   0xffffffffffefffff
+    s_bfe_u64 s[0:1], 0xffefffff, s3         //   0x00000000ffefffff
+    v_ceil_f64_e32 v[0:1], 0xffefffff        //   0xffefffff00000000 (-1.7976922776554302e308)
 
 Examples of invalid literals:
 
@@ -987,8 +987,8 @@ Examples of invalid literals:
 
     // GFX9
 
-    v_add_u16 v0, 0x1ff00, v0               // conversion is not possible as truncated bits are not all 0 or 1
-    v_add_u16 v0, 0xffffffffffff00ff, v0    // conversion is not possible as truncated bits do not match MSB of the result
+    v_add_u16 v0, 0x1ff00, v0               // truncated bits are not all 0 or 1
+    v_add_u16 v0, 0xffffffffffff00ff, v0    // truncated bits do not match MSB of the result
 
 .. _amdgpu_synid_fp_lit_conv:
 
@@ -1004,12 +1004,12 @@ When used as operands they are converted to
     ============== ============== ================= =================================================================
     Expected type  Condition      Result            Note
     ============== ============== ================= =================================================================
-    i16, u16, b16  cond(num, 16)  f16(num)          Convert to f16 and use bits of the result as an integer value.
-    i32, u32, b32  cond(num, 32)  f32(num)          Convert to f32 and use bits of the result as an integer value.
+    i16, u16, b16  cond(num,16)   f16(num)          Convert to f16 and use bits of the result as an integer value.
+    i32, u32, b32  cond(num,32)   f32(num)          Convert to f32 and use bits of the result as an integer value.
     i64, u64, b64  false          \-                Conversion disabled because of an unclear semantics.
-    f16            cond(num, 16)  f16(num)          Convert to f16.
-    f32            cond(num, 32)  f32(num)          Convert to f32.
-    f64            true           {num.u32.hi, 0}   Use high 32 bits of the number as high 32 bits of the result;
+    f16            cond(num,16)   f16(num)          Convert to f16.
+    f32            cond(num,32)   f32(num)          Convert to f32.
+    f64            true           {num.u32.hi,0}    Use high 32 bits of the number as high 32 bits of the result;
                                                     zero-fill low 32 bits of the result.
 
                                                     Note that the result may differ from the original number.
@@ -1028,8 +1028,9 @@ Examples of valid literals:
     v_add_f16 v1, 65500.0, v2
     v_add_f32 v1, 65600.0, v2
 
-                                                 // value before conversion: 0x7fefffffffffffff (1.7976931348623157e308)
-    v_ceil_f64 v[0:1], 1.7976931348623157e308    // value after conversion:  0x7fefffff00000000 (1.7976922776554302e308)
+    // Literal value before conversion: 1.7976931348623157e308 (0x7fefffffffffffff)
+    // Literal value after conversion:  1.7976922776554302e308 (0x7fefffff00000000)
+    v_ceil_f64 v[0:1], 1.7976931348623157e308
 
 Examples of invalid literals:
 
@@ -1037,7 +1038,7 @@ Examples of invalid literals:
 
     // GFX9
 
-    v_add_f16 v1, 65600.0, v2                    // cannot be converted to f16 because of overflow
+    v_add_f16 v1, 65600.0, v2    // overflow
 
 .. _amdgpu_synid_exp_conv:
 
diff --git a/docs/AMDGPUUsage.rst b/docs/AMDGPUUsage.rst
index c60d60af5a82cf1204fb9854816f2762ff8c2ecc..7963543f50734e64a6d798138ded711cb82bea94 100644
--- a/docs/AMDGPUUsage.rst
+++ b/docs/AMDGPUUsage.rst
@@ -4875,11 +4875,27 @@ symbols do not affect code generation.
 .amdgcn.gfx_generation_number
 +++++++++++++++++++++++++++++
 
-Set to the GFX generation number of the target being assembled for. For
+Set to the GFX major generation number of the target being assembled for. For
 example, when assembling for a "GFX9" target this will be set to the integer
-value "9". The possible GFX generation numbers are presented in
+value "9". The possible GFX major generation numbers are presented in
 :ref:`amdgpu-processors`.
 
+.amdgcn.gfx_generation_minor
+++++++++++++++++++++++++++++
+
+Set to the GFX minor generation number of the target being assembled for. For
+example, when assembling for a "GFX810" target this will be set to the integer
+value "1". The possible GFX minor generation numbers are presented in
+:ref:`amdgpu-processors`.
+
+.amdgcn.gfx_generation_stepping
++++++++++++++++++++++++++++++++
+
+Set to the GFX stepping generation number of the target being assembled for.
+For example, when assembling for a "GFX704" target this will be set to the
+integer value "4". The possible GFX stepping generation numbers are presented
+in :ref:`amdgpu-processors`.
+
 .amdgcn.next_free_vgpr
 ++++++++++++++++++++++
 
diff --git a/docs/CMake.rst b/docs/CMake.rst
index f4f67db4d7e19618b5a5fd6b63d76b73833d83c9..30fa5bc37fa2faed38c91e97986be54b3347feaf 100644
--- a/docs/CMake.rst
+++ b/docs/CMake.rst
@@ -573,6 +573,18 @@ LLVM-specific variables
   options, which are passed to the CCACHE_MAXSIZE and CCACHE_DIR environment
   variables, respectively.
 
+**LLVM_FORCE_USE_OLD_TOOLCHAIN**:BOOL
+  If enabled, the compiler and standard library versions won't be checked. LLVM
+  may not compile at all, or might fail at runtime due to known bugs in these
+  toolchains.
+
+**LLVM_TEMPORARILY_ALLOW_OLD_TOOLCHAIN**:BOOL
+  If enabled, the compiler version check will only warn when using a toolchain
+  which is about to be deprecated, instead of emitting an error.
+
+**LLVM_USE_NEWPM**:BOOL
+  If enabled, use the experimental new pass manager.
+
 CMake Caches
 ============
 
@@ -774,7 +786,7 @@ Contents of ``<project dir>/<pass name>/CMakeLists.txt``:
 
 Note if you intend for this pass to be merged into the LLVM source tree at some
 point in the future it might make more sense to use LLVM's internal
-``add_llvm_loadable_module`` function instead by...
+``add_llvm_library`` function with the MODULE argument instead by...
 
 
 Adding the following to ``<project dir>/CMakeLists.txt`` (after
@@ -789,7 +801,7 @@ And then changing ``<project dir>/<pass name>/CMakeLists.txt`` to
 
 .. code-block:: cmake
 
-  add_llvm_loadable_module(LLVMPassname
+  add_llvm_library(LLVMPassname MODULE
     Pass.cpp
     )
 
diff --git a/docs/CodingStandards.rst b/docs/CodingStandards.rst
index 88d8ff804d981f333e57c64dff88dab3ca69c813..781b6bdae09654766eb2d0761627e05e9a9a4971 100644
--- a/docs/CodingStandards.rst
+++ b/docs/CodingStandards.rst
@@ -233,10 +233,9 @@ tree.  The standard header looks like this:
 
   //===-- llvm/Instruction.h - Instruction class definition -------*- C++ -*-===//
   //
-  //                     The LLVM Compiler Infrastructure
-  //
-  // This file is distributed under the University of Illinois Open Source
-  // License. See LICENSE.TXT for details.
+  // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+  // See https://llvm.org/LICENSE.txt for license information.
+  // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   //
   //===----------------------------------------------------------------------===//
   ///
diff --git a/docs/CommandGuide/FileCheck.rst b/docs/CommandGuide/FileCheck.rst
index 721d2c2e782e8c2949691b6074e4dc97d575c2c7..33d9fe8440450efb745c2f81403cff4c345d935e 100644
--- a/docs/CommandGuide/FileCheck.rst
+++ b/docs/CommandGuide/FileCheck.rst
@@ -111,13 +111,16 @@ and from the command line.
 
 .. option:: -v
 
-  Print directive pattern matches.
+  Print good directive pattern matches.  However, if ``-input-dump=fail`` or
+  ``-input-dump=always``, add those matches as input annotations instead.
 
 .. option:: -vv
 
   Print information helpful in diagnosing internal FileCheck issues, such as
   discarded overlapping ``CHECK-DAG:`` matches, implicit EOF pattern matches,
   and ``CHECK-NOT:`` patterns that do not have matches.  Implies ``-v``.
+  However, if ``-input-dump=fail`` or ``-input-dump=always``, just add that
+  information as input annotations instead.
 
 .. option:: --allow-deprecated-dag-overlap
 
diff --git a/docs/CommandGuide/llc.rst b/docs/CommandGuide/llc.rst
index 11dfc902d20c9d33723b838668fa816b0283b4e9..da096f1263ab8c37881aa01d759dfac89afe168f 100644
--- a/docs/CommandGuide/llc.rst
+++ b/docs/CommandGuide/llc.rst
@@ -87,9 +87,9 @@ End-user Options
 
    llvm-as < /dev/null | llc -march=xyz -mattr=help
 
-.. option:: --disable-fp-elim
+.. option:: --frame-pointer
 
- Disable frame pointer elimination optimization.
+ Specify effect of frame pointer elimination optimization (all,non-leaf,none).
 
 .. option:: --disable-excess-fp-precision
 
diff --git a/docs/CommandGuide/llvm-exegesis.rst b/docs/CommandGuide/llvm-exegesis.rst
index f27db9e57edc6c4a8d93764ba39178eb6e094911..878ced3c3bf3bf57face2a6ab8b7f99bfadef830 100644
--- a/docs/CommandGuide/llvm-exegesis.rst
+++ b/docs/CommandGuide/llvm-exegesis.rst
@@ -10,13 +10,13 @@ DESCRIPTION
 -----------
 
 :program:`llvm-exegesis` is a benchmarking tool that uses information available
-in LLVM to measure host machine instruction characteristics like latency or port
-decomposition.
+in LLVM to measure host machine instruction characteristics like latency,
+throughput, or port decomposition.
 
 Given an LLVM opcode name and a benchmarking mode, :program:`llvm-exegesis`
 generates a code snippet that makes execution as serial (resp. as parallel) as
-possible so that we can measure the latency (resp. uop decomposition) of the
-instruction.
+possible so that we can measure the latency (resp. inverse throughput/uop decomposition)
+of the instruction.
 The code snippet is jitted and executed on the host subtarget. The time taken
 (resp. resource usage) is measured using hardware performance counters. The
 result is printed out as YAML to the standard output.
@@ -37,11 +37,13 @@ instruction, run:
 
     $ llvm-exegesis -mode=latency -opcode-name=ADD64rr
 
-Measuring the uop decomposition of an instruction works similarly:
+Measuring the uop decomposition or inverse throughput of an instruction works similarly:
 
 .. code-block:: bash
 
     $ llvm-exegesis -mode=uops -opcode-name=ADD64rr
+    $ llvm-exegesis -mode=inverse_throughput -opcode-name=ADD64rr
+
 
 The output is a YAML document (the default is to write to stdout, but you can
 redirect the output to a file using `-benchmarks-file`):
@@ -186,9 +188,11 @@ OPTIONS
   Specify the custom code snippet to measure. See example 2 for details.
   Either `opcode-index`, `opcode-name` or `snippets-file` must be set.
 
-.. option:: -mode=[latency|uops|analysis]
+.. option:: -mode=[latency|uops|inverse_throughput|analysis]
 
- Specify the run mode.
+ Specify the run mode. Note that if you pick `analysis` mode, you also need
+ to specify at least one of the `-analysis-clusters-output-file=` and
+ `-analysis-inconsistencies-output-file=`.
 
 .. option:: -num-repetitions=<Number of repetition>
 
@@ -197,18 +201,18 @@ OPTIONS
 
 .. option:: -benchmarks-file=</path/to/file>
 
- File to read (`analysis` mode) or write (`latency`/`uops` modes) benchmark
- results. "-" uses stdin/stdout.
+ File to read (`analysis` mode) or write (`latency`/`uops`/`inverse_throughput`
+ modes) benchmark results. "-" uses stdin/stdout.
 
 .. option:: -analysis-clusters-output-file=</path/to/file>
 
  If provided, write the analysis clusters as CSV to this file. "-" prints to
- stdout.
+ stdout. By default, this analysis is not run.
 
 .. option:: -analysis-inconsistencies-output-file=</path/to/file>
 
  If non-empty, write inconsistencies found during analysis to this file. `-`
- prints to stdout.
+ prints to stdout. By default, this analysis is not run.
 
 .. option:: -analysis-numpoints=<dbscan numPoints parameter>
 
diff --git a/docs/CommandGuide/llvm-objdump.rst b/docs/CommandGuide/llvm-objdump.rst
index 0d0291520475d4b17b21459ece7df3d3cefa20bf..c3e7c166005b637cfe10bde4e91ece3558ef2f5c 100644
--- a/docs/CommandGuide/llvm-objdump.rst
+++ b/docs/CommandGuide/llvm-objdump.rst
@@ -16,12 +16,19 @@ stream.
 
 COMMANDS
 --------
-At least one of the following commands are required, and some commands can be combined with other commands:
+At least one of the following commands are required, and some commands can be
+combined with other commands:
 
-.. option:: -disassemble
+.. option:: -d, -disassemble
+
+  Display assembler mnemonics for the machine instructions. Disassembles all
+  text sections found in the input file(s).
+
+.. option:: -D, -disassemble-all
+
+  Display assembler mnemonics for the machine instructions. Disassembles all
+  sections found in the input file(s).
 
-  Display assembler mnemonics for the machine instructions
- 
 .. option:: -help
 
   Display usage information and exit. Does not stack with other commands.
@@ -45,14 +52,14 @@ At least one of the following commands are required, and some commands can be co
 .. option:: -version
 
   Display the version of this program. Does not stack with other commands.
-  
+
 OPTIONS
 -------
 :program:`llvm-objdump` supports the following options:
 
 .. option:: -arch=<architecture>
 
-  Specify the architecture to disassemble. see -version for available
+  Specify the architecture to disassemble. see ``-version`` for available
   architectures.
 
 .. option:: -cfg
@@ -68,14 +75,15 @@ OPTIONS
 
   Print line information from debug info if available.
 
-.. option:: -macho
+.. option:: -m, -macho
 
-  Use Mach-O specific object file parser.
+  Use Mach-O specific object file parser. Commands and other options may behave
+  differently when used with ``-macho``.
 
 .. option:: -mattr=<a1,+a2,-a3,...>
 
   Target specific attributes.
-  
+
 .. option:: -mc-x86-disable-arith-relaxation
 
   Disable relaxation of arithmetic instruction for X86.
@@ -83,26 +91,26 @@ OPTIONS
 .. option:: -stats
 
   Enable statistics output from program.
-  
+
 .. option:: -triple=<string>
 
-  Target triple to disassemble for, see -version for available targets.
-  
+  Target triple to disassemble for, see ``-version`` for available targets.
+
 .. option:: -x86-asm-syntax=<style>
 
   When used with the ``-disassemble`` option, choose style of code to emit from
   X86 backend. Supported values are:
 
    .. option:: att
-   
+
     AT&T-style assembly
-   
+
    .. option:: intel
-   
+
     Intel-style assembly
 
-   
-  The default disassembly style is **att**. 
+
+  The default disassembly style is **att**.
 
 BUGS
 ----
diff --git a/docs/CommandGuide/llvm-profdata.rst b/docs/CommandGuide/llvm-profdata.rst
index 96c91e309d79a53a27b0f6fb8fb91aeadaaa183b..f66fb499697c0037cb849231905002d8e1193ecb 100644
--- a/docs/CommandGuide/llvm-profdata.rst
+++ b/docs/CommandGuide/llvm-profdata.rst
@@ -203,7 +203,7 @@ OPTIONS
  annotations.
 
 .. option:: -topn=n
-	     
+
  Instruct the profile dumper to show the top ``n`` functions with the
  hottest basic blocks in the summary section. By default, the topn functions
  are not dumped.
@@ -216,6 +216,16 @@ OPTIONS
 
  Show the profiled sizes of the memory intrinsic calls for shown functions.
 
+.. option:: -value-cutoff=n
+
+ Show only those functions whose max count values are greater or equal to ``n``.
+ By default, the value-cutoff is set to 0.
+
+.. option:: -list-below-cutoff
+
+ Only output names of functions whose max count value are below the cutoff
+ value.
+
 EXIT STATUS
 -----------
 
diff --git a/docs/CommandGuide/llvm-readobj.rst b/docs/CommandGuide/llvm-readobj.rst
index 417fcd05c8a20732983c0a65630b74f74769e108..7d4679f2032f5cd3ac3daa02c24930fafccd31be 100644
--- a/docs/CommandGuide/llvm-readobj.rst
+++ b/docs/CommandGuide/llvm-readobj.rst
@@ -84,6 +84,10 @@ input. Otherwise, it will read from the specified ``filenames``.
 
  Display section groups (only for ELF object files).
 
+.. option:: -demangle, -C
+
+ Print demangled symbol names in the output.
+
 EXIT STATUS
 -----------
 
diff --git a/docs/CommandGuide/llvm-symbolizer.rst b/docs/CommandGuide/llvm-symbolizer.rst
index 7bcad1c12f119cf4a5ed313b25fc99b6025ed42d..b794f56246ff01098710f7d04015511a8ac382d6 100644
--- a/docs/CommandGuide/llvm-symbolizer.rst
+++ b/docs/CommandGuide/llvm-symbolizer.rst
@@ -68,11 +68,11 @@ EXAMPLE
 OPTIONS
 -------
 
-.. option:: -obj
+.. option:: -obj, -exe, -e
 
   Path to object file to be symbolized.
 
-.. option:: -functions=[none|short|linkage]
+.. option:: -functions[=<none|short|linkage>], -f
 
   Specify the way function names are printed (omit function name,
   print short function name, or print full linkage name, respectively).
@@ -83,11 +83,15 @@ OPTIONS
  Prefer function names stored in symbol table to function names
  in debug info sections. Defaults to true.
 
-.. option:: -demangle
+.. option:: -demangle, -C
 
  Print demangled function names. Defaults to true.
 
-.. option:: -inlining 
+.. option:: -no-demangle
+
+ Don't print demangled function names.
+
+.. option:: -inlining, -inlines, -i
 
  If a source code location is in an inlined function, prints all the
  inlnied frames. Defaults to true.
@@ -106,15 +110,24 @@ OPTIONS
  location, look for the debug info at the .dSYM path provided via the
  ``-dsym-hint`` flag. This flag can be used multiple times.
 
-.. option:: -print-address
+.. option:: -print-address, -addresses, -a
 
  Print address before the source code location. Defaults to false.
 
-.. option:: -pretty-print
+.. option:: -pretty-print, -p
 
  Print human readable output. If ``-inlining`` is specified, enclosing scope is
  prefixed by (inlined by). Refer to listed examples.
 
+.. option:: -basenames, -s
+
+ Strip directories when printing the file path.
+
+.. option:: -adjust-vma=<offset>
+
+ Add the specified offset to object file addresses when performing lookups. This
+ can be used to perform lookups as if the object were relocated by the offset.
+
 EXIT STATUS
 -----------
 
diff --git a/docs/CompileCudaWithLLVM.rst b/docs/CompileCudaWithLLVM.rst
index 95d6b0d9b82a19fef66520d88d92ed5dda32ade9..6e181c84e6881b9c956bcf06f9cc515ef4753c31 100644
--- a/docs/CompileCudaWithLLVM.rst
+++ b/docs/CompileCudaWithLLVM.rst
@@ -143,9 +143,9 @@ device side.
 ----------------------------
 
 In clang, ``math.h`` and ``cmath`` are available and `pass
-<https://github.com/llvm-mirror/test-suite/blob/master/External/CUDA/math_h.cu>`_
+<https://github.com/llvm/llvm-test-suite/blob/master/External/CUDA/math_h.cu>`_
 `tests
-<https://github.com/llvm-mirror/test-suite/blob/master/External/CUDA/cmath.cu>`_
+<https://github.com/llvm/llvm-test-suite/blob/master/External/CUDA/cmath.cu>`_
 adapted from libc++'s test suite.
 
 In nvcc ``math.h`` and ``cmath`` are mostly available.  Versions of ``::foof``
diff --git a/docs/CompilerWriterInfo.rst b/docs/CompilerWriterInfo.rst
index 838c36a4099c2537a4002edb0fbcb013d1be9f33..731d24a1c4e5e45465614156bc7ba28fb7f6b29b 100644
--- a/docs/CompilerWriterInfo.rst
+++ b/docs/CompilerWriterInfo.rst
@@ -16,16 +16,24 @@ Hardware
 AArch64 & ARM
 -------------
 
-* `ARMv8-A Architecture Reference Manual <http://infocenter.arm.com/help/index.jsp?topic=/com.arm.doc.ddi0487a.h/index.html>`_ (authentication required, free sign-up). This document covers both AArch64 and ARM instructions
+* `ARMv8-A Architecture Reference Manual <https://developer.arm.com/docs/ddi0487/latest>`_ This document covers both AArch64 and ARM instructions
 
-* `ARMv7-M Architecture Reference Manual <http://infocenter.arm.com/help/topic/com.arm.doc.ddi0403e.b/index.html>`_ (authentication required, free sign-up). This covers the Thumb2-only microcontrollers
+* `ARMv7-A Architecture Reference Manual <https://developer.arm.com/docs/ddi0406/latest>`_ This has some useful info on what is supported by older architecture versions.
 
-* `ARMv6-M Architecture Reference Manual <http://infocenter.arm.com/help/topic/com.arm.doc.ddi0419c/index.html>`_ (authentication required, free sign-up). This covers the Thumb1-only microcontrollers
+* `ARMv7-M Architecture Reference Manual <https://developer.arm.com/docs/ddi0403/latest>`_ This covers the Thumb2-only microcontrollers
+
+* `ARMv6-M Architecture Reference Manual <https://developer.arm.com/docs/ddi0419/latest>`_ This covers the Thumb1-only microcontrollers
 
 * `ARM C Language Extensions <http://infocenter.arm.com/help/topic/com.arm.doc.ihi0053c/IHI0053C_acle_2_0.pdf>`_
 
+* `ARM NEON Intrinsics Reference <http://infocenter.arm.com/help/topic/com.arm.doc.ihi0073b/IHI0073B_arm_neon_intrinsics_ref.pdf>`_
+
 * AArch32 `ABI Addenda and Errata <http://infocenter.arm.com/help/topic/com.arm.doc.ihi0045d/IHI0045D_ABI_addenda.pdf>`_
 
+* `Cortex-A57 Software Optimization Guide <http://infocenter.arm.com/help/topic/com.arm.doc.uan0015b/Cortex_A57_Software_Optimization_Guide_external.pdf>`_
+
+* `Run-time ABI for the ARM Architecture <http://infocenter.arm.com/help/topic/com.arm.doc.ihi0043d/IHI0043D_rtabi.pdf>`_ This documents the __aeabi_* helper functions.
+
 Itanium (ia64)
 --------------
 
@@ -119,7 +127,7 @@ ABI
 ===
 
 * `System V Application Binary Interface <http://www.sco.com/developers/gabi/latest/contents.html>`_
-* `Itanium C++ ABI <http://itanium-cxx-abi.github.io/cxx-abi/>`_
+* `Itanium C++ ABI <http://itanium-cxx-abi.github.io/cxx-abi/>`_ (This is used for all non-Windows targets.)
 
 Linux
 -----
@@ -127,6 +135,7 @@ Linux
 * `Linux extensions to gabi <https://github.com/hjl-tools/linux-abi/wiki/Linux-Extensions-to-gABI>`_
 * `PowerPC 64-bit ELF ABI Supplement <http://www.linuxbase.org/spec/ELF/ppc64/>`_
 * `Procedure Call Standard for the AArch64 Architecture <http://infocenter.arm.com/help/topic/com.arm.doc.ihi0055a/IHI0055A_aapcs64.pdf>`_
+* `Procedure Call Standard for the ARM Architecture <https://developer.arm.com/docs/ihi0042/latest>`_
 * `ELF for the ARM Architecture <http://infocenter.arm.com/help/topic/com.arm.doc.ihi0044e/IHI0044E_aaelf.pdf>`_
 * `ELF for the ARM 64-bit Architecture (AArch64) <http://infocenter.arm.com/help/topic/com.arm.doc.ihi0056a/IHI0056A_aaelf64.pdf>`_
 * `System z ELF ABI Supplement <http://legacy.redhat.com/pub/redhat/linux/7.1/es/os/s390x/doc/lzsabi0.pdf>`_
@@ -136,11 +145,16 @@ OS X
 
 * `Mach-O Runtime Architecture <http://developer.apple.com/documentation/Darwin/RuntimeArchitecture-date.html>`_
 * `Notes on Mach-O ABI <http://www.unsanity.org/archives/000044.php>`_
+* `ARM64 Function Calling Conventions <https://developer.apple.com/library/archive/documentation/Xcode/Conceptual/iPhoneOSABIReference/Articles/ARM64FunctionCallingConventions.html>`_
 
 Windows
 -------
 
 * `Microsoft PE/COFF Specification <http://www.microsoft.com/whdc/system/platform/firmware/pecoff.mspx>`_
+* `ARM64 exception handling <https://docs.microsoft.com/en-us/cpp/build/arm64-exception-handling>`_
+* `ARM exception handling <https://docs.microsoft.com/en-us/cpp/build/arm-exception-handling>`_
+* `Overview of ARM64 ABI conventions <https://docs.microsoft.com/en-us/cpp/build/arm64-windows-abi-conventions>`_
+* `Overview of ARM32 ABI Conventions <https://docs.microsoft.com/en-us/cpp/build/overview-of-arm-abi-conventions>`_
 
 NVPTX
 =====
diff --git a/docs/DeveloperPolicy.rst b/docs/DeveloperPolicy.rst
index 9125197a73a86a16a20036d5882ae915971a3690..003f75fdf5f9b68da30ab4920a30ab551a4240e0 100644
--- a/docs/DeveloperPolicy.rst
+++ b/docs/DeveloperPolicy.rst
@@ -22,7 +22,7 @@ This policy is also designed to accomplish the following objectives:
 
 #. Make life as simple and easy for contributors as possible.
 
-#. Keep the top of Subversion trees as stable as possible.
+#. Keep the top of tree as stable as possible.
 
 #. Establish awareness of the project's :ref:`copyright, license, and patent
    policies <copyright-license-patents>` with contributors to the project.
@@ -80,23 +80,19 @@ Making and Submitting a Patch
 When making a patch for review, the goal is to make it as easy for the reviewer
 to read it as possible.  As such, we recommend that you:
 
-#. Make your patch against the Subversion trunk, not a branch, and not an old
-   version of LLVM.  This makes it easy to apply the patch.  For information on
-   how to check out SVN trunk, please see the `Getting Started
-   Guide <GettingStarted.html#checkout>`_.
+#. Make your patch against git master, not a branch, and not an old version
+   of LLVM.  This makes it easy to apply the patch.  For information on how to
+   clone from git, please see the :ref:`Getting Started Guide
+   <checkout>`.
 
 #. Similarly, patches should be submitted soon after they are generated.  Old
    patches may not apply correctly if the underlying code changes between the
    time the patch was created and the time it is applied.
 
-#. Patches should be made with ``svn diff``, or similar. If you use a
+#. Patches should be made with ``git format-patch``, or similar. If you use a
    different tool, make sure it uses the ``diff -u`` format and that it
    doesn't contain clutter which makes it hard to read.
 
-#. If you are modifying generated files, such as the top-level ``configure``
-   script, please separate out those changes into a separate patch from the rest
-   of your changes.
-
 Once your patch is ready, submit it by emailing it to the appropriate project's
 commit mailing list (or commit it directly if applicable). Alternatively, some
 patches get sent to the project's development list or component of the LLVM bug
@@ -117,8 +113,8 @@ a file inline, making it difficult to work with for reviewers using that
 program.
 
 When submitting patches, please do not add confidentiality or non-disclosure
-notices to the patches themselves.  These notices conflict with the `LLVM
-License`_ and may result in your contribution being excluded.
+notices to the patches themselves.  These notices conflict with the LLVM
+licensing terms and may result in your contribution being excluded.
 
 .. _code review:
 
@@ -187,9 +183,9 @@ someone else will review it, allowing the patch to go unreviewed.  To solve this
 problem, we have a notion of an 'owner' for a piece of the code.  The sole
 responsibility of a code owner is to ensure that a commit to their area of the
 code is appropriately reviewed, either by themself or by someone else.  The list
-of current code owners can be found in the file
-`CODE_OWNERS.TXT <http://git.llvm.org/klaus/llvm/blob/master/CODE_OWNERS.TXT>`_
-in the root of the LLVM source tree.
+of current code owners can be found in the file `CODE_OWNERS.TXT
+<https://github.com/llvm/llvm-project/blob/master/llvm/CODE_OWNERS.TXT>`_ in the
+root of the LLVM source tree.
 
 Note that code ownership is completely different than reviewers: anyone can
 review a piece of code, and we welcome code review from anyone who is
@@ -642,6 +638,50 @@ In essences, these rules are necessary for targets to gain and retain their
 status, but also markers to define bit-rot, and will be used to clean up the
 tree from unmaintained targets.
 
+.. _toolchain:
+
+Updating Toolchain Requirements
+-------------------------------
+
+We intend to require newer toolchains as time goes by. This means LLVM's
+codebase can use newer versions of C++ as they get standardized. Requiring newer
+toolchains to build LLVM can be painful for those building LLVM; therefore, it
+will only be done through the following process:
+
+  * Generally, try to support LLVM and GCC versions from the last 3 years at a
+    minimum. This time-based guideline is not strict: we may support much older
+    compilers, or decide to support fewer versions.
+
+  * An RFC is sent to the `llvm-dev mailing list <http://lists.llvm.org/mailman/listinfo/llvm-dev>`_
+
+    - Detail upsides of the version increase (e.g. which newer C++ language or
+      library features LLVM should use; avoid miscompiles in particular compiler
+      versions, etc).
+    - Detail downsides on important platforms (e.g. Ubuntu LTS status).
+
+  * Once the RFC reaches consensus, update the CMake toolchain version checks as
+    well as the :doc:`getting started<GettingStarted>` guide. We want to
+    soft-error when developers compile LLVM. We say "soft-error" because the
+    error can be turned into a warning using a CMake flag. This is an important
+    step: LLVM still doesn't have code which requires the new toolchains, but it
+    soon will. If you compile LLVM but don't read the mailing list, we should
+    tell you!
+
+  * Ensure that at least one LLVM release has had this soft-error. Not all
+    developers compile LLVM top-of-tree. These release-bound developers should
+    also be told about upcoming changes.
+
+  * Turn the soft-error into a hard-error after said LLVM release has branched.
+
+  * Update the :doc:`coding standards<CodingStandards>` to allow the new
+    features we've explicitly approved in the RFC.
+
+  * Start using the new features in LLVM's codebase.
+
+Here's a `sample RFC
+<http://lists.llvm.org/pipermail/llvm-dev/2019-January/129452.html>`_ and the
+`corresponding change <https://reviews.llvm.org/D57264>`_.
+
 .. _copyright-license-patents:
 
 Copyright, License, and Patents
@@ -650,44 +690,227 @@ Copyright, License, and Patents
 .. note::
 
    This section deals with legal matters but does not provide legal advice.  We
-   are not lawyers --- please seek legal counsel from an attorney.
+   are not lawyers --- please seek legal counsel from a licensed attorney.
 
 This section addresses the issues of copyright, license and patents for the LLVM
-project.  The copyright for the code is held by the individual contributors of
-the code and the terms of its license to LLVM users and developers is the
-`University of Illinois/NCSA Open Source License
-<http://www.opensource.org/licenses/UoI-NCSA.php>`_ (with portions dual licensed
-under the `MIT License <http://www.opensource.org/licenses/mit-license.php>`_,
-see below).  As contributor to the LLVM project, you agree to allow any
-contributions to the project to licensed under these terms.
+project.  The copyright for the code is held by the contributors of
+the code.  The code is licensed under permissive `open source licensing terms`_,
+namely the Apache 2 license, which includes a copyright and `patent license`_.
+When you contribute code to the LLVM project, you license it under these terms.
+
+If you have questions or comments about these topics, please contact the
+`LLVM Developer's Mailing List <mailto:llvm-dev@lists.llvm.org>`_.  However,
+please realize that most compiler developers are not lawyers, and therefore you
+will not be getting official legal advice.
 
 Copyright
 ---------
 
-The LLVM project does not require copyright assignments, which means that the
-copyright for the code in the project is held by its respective contributors who
-have each agreed to release their contributed code under the terms of the `LLVM
-License`_.
+The LLVM project does not collect copyright assignments, which means that the
+copyright for the code in the project is held by the respective contributors.
+Because you (or your company)
+retain ownership of the code you contribute, you know it may only be used under
+the terms of the open source license you contributed it under: the license for
+your contributions cannot be changed in the future without your approval.
+
+Because the LLVM project does not require copyright assignments, changing the
+LLVM license requires tracking down the
+contributors to LLVM and getting them to agree that a license change is
+acceptable for their contributions.  We feel that a high burden for relicensing
+is good for the project, because contributors do not have to fear that their
+code will be used in a way with which they disagree.
+
+Relicensing
+-----------
 
-An implication of this is that the LLVM license is unlikely to ever change:
-changing it would require tracking down all the contributors to LLVM and getting
-them to agree that a license change is acceptable for their contribution.  Since
-there are no plans to change the license, this is not a cause for concern.
+The last paragraph notwithstanding, the LLVM Project is in the middle of a large
+effort to change licenses, which aims to solve several problems:
+
+* The old licenses made it difficult to move code from (e.g.) the compiler to
+  runtime libraries, because runtime libraries used a different license from the
+  rest of the compiler.
+* Some contributions were not submitted to LLVM due to concerns that
+  the patent grant required by the project was overly broad.
+* The patent grant was unique to the LLVM Project, not written by a lawyer, and
+  was difficult to determine what was protection was provided (if any).
+
+The scope of relicensing is all code that is considered part of the LLVM
+project, including the main LLVM repository, runtime libraries (compiler_rt,
+OpenMP, etc), Polly, and all other subprojects.  There are a few exceptions:
+
+* Code imported from other projects (e.g. Google Test, Autoconf, etc) will
+  remain as it is.  This code isn't developed as part of the LLVM project, it
+  is used by LLVM.
+* Some subprojects are impractical or uninteresting to relicense (e.g. llvm-gcc
+  and dragonegg). These will be split off from the LLVM project (e.g. to
+  separate Github projects), allowing interested people to continue their
+  development elsewhere.
+
+To relicense LLVM, we will be seeking approval from all of the copyright holders
+of code in the repository, or potentially remove/rewrite code if we cannot.
+This is a large
+and challenging project which will take a significant amount of time to
+complete.  In the interim, **all contributions to the project will be made under
+the terms of both the new license and the legacy license scheme** (each of which
+is described below).  The exception to this is the legacy patent grant, which
+will not be required for new contributions.
+
+When all of the code in the project has been converted to the new license or
+removed, we will drop the requirement to contribute under the legacy license.
+This will achieve the goal of having
+a single standardized license for the entire codebase.
+
+If you are a prior contributor to LLVM and have not done so already, please do
+*TODO* to allow us to use your code. *Add a link to a separate page here, which
+is probably a click through web form or something like that.  Details to be
+determined later*.
+
+
+.. _open source licensing terms:
+
+New LLVM Project License Framework
+----------------------------------
+
+Contributions to LLVM are licensed under the `Apache License, Version 2.0
+<https://www.apache.org/licenses/LICENSE-2.0>`_, with two limited
+exceptions intended to ensure that LLVM is very permissively licensed.
+Collectively, the name of this license is "Apache 2.0 License with LLVM
+exceptions".  The exceptions read:
+
+::
+
+   ---- LLVM Exceptions to the Apache 2.0 License ----
+
+   As an exception, if, as a result of your compiling your source code, portions
+   of this Software are embedded into an Object form of such source code, you
+   may redistribute such embedded portions in such Object form without complying
+   with the conditions of Sections 4(a), 4(b) and 4(d) of the License.
+
+   In addition, if you combine or link compiled forms of this Software with
+   software that is licensed under the GPLv2 ("Combined Software") and if a
+   court of competent jurisdiction determines that the patent provision (Section
+   3), the indemnity provision (Section 9) or other Section of the License
+   conflicts with the conditions of the GPLv2, you may retroactively and
+   prospectively choose to deem waived or otherwise exclude such Section(s) of
+   the License, but only in their entirety and only with respect to the Combined
+   Software.
+
+
+We intend to keep LLVM perpetually open source and available under a permissive
+license - this fosters the widest adoption of LLVM by
+**allowing commercial products to be derived from LLVM** with few restrictions
+and without a requirement for making any derived works also open source.  In
+particular, LLVM's license is not a "copyleft" license like the GPL.
+
+The "Apache 2.0 License with LLVM exceptions" allows you to:
+
+* freely download and use LLVM (in whole or in part) for personal, internal, or
+  commercial purposes.
+* include LLVM in packages or distributions you create.
+* combine LLVM with code licensed under every other major open source
+  license (including BSD, MIT, GPLv2, GPLv3...).
+* make changes to LLVM code without being required to contribute it back
+  to the project - contributions are appreciated though!
+
+However, it imposes these limitations on you:
+
+* You must retain the copyright notice if you redistribute LLVM: You cannot
+  strip the copyright headers off or replace them with your own.
+* Binaries that include LLVM must reproduce the copyright notice (e.g. in an
+  included README file or in an "About" box), unless the LLVM code was added as
+  a by-product of compilation.  For example, if an LLVM runtime library like
+  compiler_rt or libc++ was automatically included into your application by the
+  compiler, you do not need to attribute it.
+* You can't use our names to promote your products (LLVM derived or not) -
+  though you can make truthful statements about your use of the LLVM code,
+  without implying our sponsorship.
+* There's no warranty on LLVM at all.
+
+We want LLVM code to be widely used, and believe that this provides a model that
+is great for contributors and users of the project.  For more information about
+the Apache 2.0 License, please see the `Apache License FAQ
+<http://www.apache.org/foundation/license-faq.html>`_, maintained by the
+Apache Project.
 
-As a contributor to the project, this means that you (or your company) retain
-ownership of the code you contribute, that it cannot be used in a way that
-contradicts the license (which is a liberal BSD-style license), and that the
-license for your contributions won't change without your approval in the
-future.
 
-.. _LLVM License:
+.. note::
+
+   The LLVM Project includes some really old subprojects (dragonegg,
+   llvm-gcc-4.0, and llvm-gcc-4.2), which are licensed under **GPL
+   licenses**.  This code is not actively maintained - it does not even
+   build successfully.  This code is cleanly separated into distinct SVN
+   repositories from the rest of LLVM, and the LICENSE.txt files specifically
+   indicate that they contain GPL code.  When LLVM transitions from SVN to Git,
+   we plan to drop these code bases from the new repository structure.
 
-License
+
+.. _patent license:
+
+Patents
 -------
 
-We intend to keep LLVM perpetually open source and to use a liberal open source
-license. **As a contributor to the project, you agree that any contributions be
-licensed under the terms of the corresponding subproject.** All of the code in
+Section 3 of the Apache 2.0 license is a patent grant under which
+contributors of code to the project contribute the rights to use any of
+their patents that would otherwise be infringed by that code contribution
+(protecting uses of that code).  Further, the patent grant is revoked
+from anyone who files a patent lawsuit about code in LLVM - this protects the
+community by providing a "patent commons" for the code base and reducing the
+odds of patent lawsuits in general.
+
+The license specifically scopes which patents are included with code
+contributions.  To help explain this, the `Apache License FAQ
+<http://www.apache.org/foundation/license-faq.html>`_ explains this scope using
+some questions and answers, which we reproduce here for your convenience (for
+reference, the "ASF" is the Apache Software Foundation, the guidance still
+holds though)::
+
+   Q1: If I own a patent and contribute to a Work, and, at the time my
+   contribution is included in that Work, none of my patent's claims are subject
+   to Apache's Grant of Patent License, is there a way any of those claims would
+   later become subject to the Grant of Patent License solely due to subsequent
+   contributions by other parties who are not licensees of that patent.
+
+   A1: No.
+
+   Q2: If at any time after my contribution, I am able to license other patent
+   claims that would have been subject to Apache's Grant of Patent License if
+   they were licenseable by me at the time of my contribution, do those other
+   claims become subject to the Grant of Patent License?
+
+   A2: Yes.
+
+   Q3: If I own or control a licensable patent and contribute code to a specific
+   Apache product, which of my patent claims are subject to Apache's Grant of
+   Patent License?
+
+   A3:  The only patent claims that are licensed to the ASF are those you own or
+   have the right to license that read on your contribution or on the
+   combination of your contribution with the specific Apache product to which
+   you contributed as it existed at the time of your contribution. No additional
+   patent claims become licensed as a result of subsequent combinations of your
+   contribution with any other software. Note, however, that licensable patent
+   claims include those that you acquire in the future, as long as they read on
+   your original contribution as made at the original time. Once a patent claim
+   is subject to Apache's Grant of Patent License, it is licensed under the
+   terms of that Grant to the ASF and to recipients of any software distributed
+   by the ASF for any Apache software product whatsoever.
+
+.. _legacy:
+
+Legacy License Structure
+------------------------
+
+.. note::
+   The code base was previously licensed under the Terms described here.
+   We are in the middle of relicensing to a new approach (described above), but
+   until this effort is complete, the code is also still available under these
+   terms.  Once we finish the relicensing project, new versions of the code will
+   not be available under these terms.  However, nothing takes away your right
+   to use old versions under the licensing terms under which they were
+   originally released.
+
+We intend to keep LLVM perpetually open source and to use a permissive open
+source license.  The code in
 LLVM is available under the `University of Illinois/NCSA Open Source License
 <http://www.opensource.org/licenses/UoI-NCSA.php>`_, which boils down to
 this:
@@ -695,13 +918,13 @@ this:
 * You can freely distribute LLVM.
 * You must retain the copyright notice if you redistribute LLVM.
 * Binaries derived from LLVM must reproduce the copyright notice (e.g. in an
-  included readme file).
+  included README file).
 * You can't use our names to promote your LLVM derived products.
 * There's no warranty on LLVM at all.
 
 We believe this fosters the widest adoption of LLVM because it **allows
 commercial products to be derived from LLVM** with few restrictions and without
-a requirement for making any derived works also open source (i.e.  LLVM's
+a requirement for making any derived works also open source (i.e. LLVM's
 license is not a "copyleft" license like the GPL). We suggest that you read the
 `License <http://www.opensource.org/licenses/UoI-NCSA.php>`_ if further
 clarification is needed.
@@ -719,30 +942,3 @@ applications to the binary redistribution clause. This also means that it is ok
 to move code from (e.g.)  libc++ to the LLVM core without concern, but that code
 cannot be moved from the LLVM core to libc++ without the copyright owner's
 permission.
-
-Note that the LLVM Project does distribute dragonegg, **which is
-GPL.** This means that anything "linked" into dragonegg must itself be compatible
-with the GPL, and must be releasable under the terms of the GPL.  This implies
-that **any code linked into dragonegg and distributed to others may be subject to
-the viral aspects of the GPL** (for example, a proprietary code generator linked
-into dragonegg must be made available under the GPL).  This is not a problem for
-code already distributed under a more liberal license (like the UIUC license),
-and GPL-containing subprojects are kept in separate SVN repositories whose
-LICENSE.txt files specifically indicate that they contain GPL code.
-
-Patents
--------
-
-To the best of our knowledge, LLVM does not infringe on any patents (we have
-actually removed code from LLVM in the past that was found to infringe).  Having
-code in LLVM that infringes on patents would violate an important goal of the
-project by making it hard or impossible to reuse the code for arbitrary purposes
-(including commercial use).
-
-When contributing code, we expect contributors to notify us of any potential for
-patent-related trouble with their changes (including from third parties).  If
-you or your employer own the rights to a patent and would like to contribute
-code to LLVM that relies on it, we require that the copyright owner sign an
-agreement that allows any other user of LLVM to freely use your patent.  Please
-contact the `LLVM Foundation Board of Directors <mailto:board@llvm.org>`_ for more
-details.
diff --git a/docs/ExtendingLLVM.rst b/docs/ExtendingLLVM.rst
index 87f48c9934257d2aaa499dfe762ca3f550f6292b..389dfad168bb88955830faf287d366adddd32db7 100644
--- a/docs/ExtendingLLVM.rst
+++ b/docs/ExtendingLLVM.rst
@@ -253,10 +253,6 @@ Adding a fundamental type
    add enum ``LLVMTypeKind`` and modify
    ``LLVMTypeKind LLVMGetTypeKind(LLVMTypeRef Ty)`` for the new type
 
-#. ``llvm/include/llvm/IR/TypeBuilder.h``:
-
-   add new class to represent new type in the hierarchy
-
 #. ``llvm/lib/AsmParser/LLLexer.cpp``:
 
    add ability to parse in the type from text assembly
@@ -299,10 +295,6 @@ Adding a derived type
    add enum ``LLVMTypeKind`` and modify
    `LLVMTypeKind LLVMGetTypeKind(LLVMTypeRef Ty)` for the new type
 
-#. ``llvm/include/llvm/IR/TypeBuilder.h``:
-
-   add new class to represent new class in the hierarchy
-
 #. ``llvm/lib/AsmParser/LLLexer.cpp``:
 
    modify ``lltok::Kind LLLexer::LexIdentifier()`` to add ability to
diff --git a/docs/GettingStarted.rst b/docs/GettingStarted.rst
index ff6b6961ffd6cb9a9c50d959759a2c04114c05cd..f528bc6014fad5cc817f4cb19d25f457d6375d0d 100644
--- a/docs/GettingStarted.rst
+++ b/docs/GettingStarted.rst
@@ -8,23 +8,22 @@ Getting Started with the LLVM System
 Overview
 ========
 
-Welcome to LLVM! In order to get started, you first need to know some basic
-information.
+Welcome to the LLVM project! In order to get started, you first need to know
+some basic information.
 
-First, LLVM comes in three pieces. The first piece is the LLVM suite. This
-contains all of the tools, libraries, and header files needed to use LLVM.  It
-contains an assembler, disassembler, bitcode analyzer and bitcode optimizer.  It
-also contains basic regression tests that can be used to test the LLVM tools and
-the Clang front end.
+First, the LLVM project has multiple components. The core of the project is
+itself called "LLVM". This contains all of the tools, libraries, and header
+files needed to process an intermediate representation and convert it into
+object files.  It contains an assembler, disassembler, bitcode analyzer and
+bitcode optimizer.  It also contains basic regression tests.
 
-The second piece is the `Clang <http://clang.llvm.org/>`_ front end.  This
-component compiles C, C++, Objective C, and Objective C++ code into LLVM
-bitcode. Once compiled into LLVM bitcode, a program can be manipulated with the
-LLVM tools from the LLVM suite.
+Another piece is the `Clang <http://clang.llvm.org/>`_ front end.  This
+component compiles C, C++, Objective C, and Objective C++ code into LLVM bitcode
+-- and from there into object files, using LLVM.
 
-There is a third, optional piece called Test Suite.  It is a suite of programs
-with a testing harness that can be used to further test LLVM's functionality
-and performance.
+There are other components as well:
+the `libc++ C++ standard library <https://libcxx.llvm.org>`_,
+the `LLD linker <https://lld.llvm.org>`_, and more.
 
 Getting Started Quickly (A Summary)
 ===================================
@@ -39,90 +38,38 @@ Here's the short story for getting up and running quickly with LLVM:
 #. Read the documentation.
 #. Remember that you were warned twice about reading the documentation.
 
-   * In particular, the *relative paths specified are important*.
+#. Checkout LLVM (including related subprojects like Clang):
 
-#. Checkout LLVM:
+   * ``git clone https://github.com/llvm/llvm-project.git``
+   * Or, on windows, ``git clone --config core.autocrlf=false
+     https://github.com/llvm/llvm-project.git``
 
-   * ``cd where-you-want-llvm-to-live``
-   * ``svn co http://llvm.org/svn/llvm-project/llvm/trunk llvm``
+#. Configure and build LLVM and Clang:.
 
-#. Checkout Clang:
-
-   * ``cd where-you-want-llvm-to-live``
-   * ``cd llvm/tools``
-   * ``svn co http://llvm.org/svn/llvm-project/cfe/trunk clang``
-
-#. Checkout Extra Clang Tools **[Optional]**:
-
-   * ``cd where-you-want-llvm-to-live``
-   * ``cd llvm/tools/clang/tools``
-   * ``svn co http://llvm.org/svn/llvm-project/clang-tools-extra/trunk extra``
-
-#. Checkout LLD linker **[Optional]**:
-
-   * ``cd where-you-want-llvm-to-live``
-   * ``cd llvm/tools``
-   * ``svn co http://llvm.org/svn/llvm-project/lld/trunk lld``
-
-#. Checkout Polly Loop Optimizer **[Optional]**:
-
-   * ``cd where-you-want-llvm-to-live``
-   * ``cd llvm/tools``
-   * ``svn co http://llvm.org/svn/llvm-project/polly/trunk polly``
-
-#. Checkout Compiler-RT (required to build the sanitizers) **[Optional]**:
-
-   * ``cd where-you-want-llvm-to-live``
-   * ``cd llvm/projects``
-   * ``svn co http://llvm.org/svn/llvm-project/compiler-rt/trunk compiler-rt``
-
-#. Checkout Libomp (required for OpenMP support) **[Optional]**:
-
-   * ``cd where-you-want-llvm-to-live``
-   * ``cd llvm/projects``
-   * ``svn co http://llvm.org/svn/llvm-project/openmp/trunk openmp``
-
-#. Checkout libcxx and libcxxabi **[Optional]**:
-
-   * ``cd where-you-want-llvm-to-live``
-   * ``cd llvm/projects``
-   * ``svn co http://llvm.org/svn/llvm-project/libcxx/trunk libcxx``
-   * ``svn co http://llvm.org/svn/llvm-project/libcxxabi/trunk libcxxabi``
-
-#. Get the Test Suite Source Code **[Optional]**
-
-   * ``cd where-you-want-llvm-to-live``
-   * ``cd llvm/projects``
-   * ``svn co http://llvm.org/svn/llvm-project/test-suite/trunk test-suite``
-
-#. Configure and build LLVM and Clang:
-
-   *Warning:* Make sure you've checked out *all of* the source code
-   before trying to configure with cmake.  cmake does not pickup newly
-   added source directories in incremental builds.
-
-   The build uses `CMake <CMake.html>`_. LLVM requires CMake 3.4.3 to build. It
-   is generally recommended to use a recent CMake, especially if you're
-   generating Ninja build files. This is because the CMake project is constantly
-   improving the quality of the generators, and the Ninja generator gets a lot
-   of attention.
-
-   * ``cd where you want to build llvm``
+   * ``cd llvm-project``
    * ``mkdir build``
    * ``cd build``
-   * ``cmake -G <generator> [options] <path to llvm sources>``
+   * ``cmake -G <generator> [options] ../llvm``
 
      Some common generators are:
 
-     * ``Unix Makefiles`` --- for generating make-compatible parallel makefiles.
      * ``Ninja`` --- for generating `Ninja <https://ninja-build.org>`_
        build files. Most llvm developers use Ninja.
+     * ``Unix Makefiles`` --- for generating make-compatible parallel makefiles.
      * ``Visual Studio`` --- for generating Visual Studio projects and
        solutions.
      * ``Xcode`` --- for generating Xcode projects.
 
      Some Common options:
 
+     * ``-DLLVM_ENABLE_PROJECTS='...'`` --- semicolon-separated list of the LLVM
+       subprojects you'd like to additionally build. Can include any of: clang,
+       libcxx, libcxxabi, libunwind, lldb, compiler-rt, lld, polly, or
+       debuginfo-tests.
+
+       For example, to build LLVM, Clang, libcxx, and libcxxabi, use
+       ``-DLLVM_ENABLE_PROJECTS="clang;libcxx;libcxxabi"``.
+
      * ``-DCMAKE_INSTALL_PREFIX=directory`` --- Specify for *directory* the full
        pathname of where you want the LLVM tools and libraries to be installed
        (default ``/usr/local``).
@@ -135,16 +82,18 @@ Here's the short story for getting up and running quickly with LLVM:
 
    * Run your build tool of choice!
 
-     * The default target (i.e. ``make``) will build all of LLVM
+     * The default target (i.e. ``ninja`` or ``make``) will build all of LLVM.
 
-     * The ``check-all`` target (i.e. ``make check-all``) will run the
+     * The ``check-all`` target (i.e. ``ninja check-all``) will run the
        regression tests to ensure everything is in working order.
 
      * CMake will generate build targets for each tool and library, and most
        LLVM sub-projects generate their own ``check-<project>`` target.
 
-     * Running a serial build will be *slow*.  Make sure you run a
-       parallel build; for ``make``, use ``make -j``.
+     * Running a serial build will be *slow*.  Make sure you run a parallel
+       build. That's already done by default in Ninja; for ``make``, use
+       ``make -j NNN`` (with an appropriate value of NNN, e.g. number of CPUs
+       you have.)
 
    * For more information see `CMake <CMake.html>`_
 
@@ -172,7 +121,7 @@ OS                 Arch                  Compilers
 ================== ===================== =============
 Linux              x86\ :sup:`1`         GCC, Clang
 Linux              amd64                 GCC, Clang
-Linux              ARM\ :sup:`4`         GCC, Clang
+Linux              ARM                   GCC, Clang
 Linux              PowerPC               GCC, Clang
 Solaris            V9 (Ultrasparc)       GCC
 FreeBSD            x86\ :sup:`1`         GCC, Clang
@@ -192,7 +141,6 @@ Windows x64        x86-64                Visual Studio
   #. Code generation supported for 32-bit ABI only
   #. To use LLVM modules on Win32-based system, you may configure LLVM
      with ``-DBUILD_SHARED_LIBS=On``.
-  #. MCJIT not working well pre-v7, old JIT engine not supported any more.
 
 Note that Debug builds require a lot of time and disk space.  An LLVM-only build
 will need about 1-3 GB of space.  A full build of LLVM and Clang will need around
@@ -222,7 +170,7 @@ uses the package and provides other details.
 Package                                                     Version      Notes
 =========================================================== ============ ==========================================
 `GNU Make <http://savannah.gnu.org/projects/make>`_         3.79, 3.79.1 Makefile/build processor
-`GCC <http://gcc.gnu.org/>`_                                >=4.8.0      C/C++ compiler\ :sup:`1`
+`GCC <http://gcc.gnu.org/>`_                                >=5.1.0      C/C++ compiler\ :sup:`1`
 `python <http://www.python.org/>`_                          >=2.7        Automated test suite\ :sup:`2`
 `zlib <http://zlib.net>`_                                   >=1.2.3.4    Compression library\ :sup:`3`
 =========================================================== ============ ==========================================
@@ -272,15 +220,25 @@ Host C++ Toolchain, both Compiler and Standard Library
 ------------------------------------------------------
 
 LLVM is very demanding of the host C++ compiler, and as such tends to expose
-bugs in the compiler. We are also planning to follow improvements and
-developments in the C++ language and library reasonably closely. As such, we
-require a modern host C++ toolchain, both compiler and standard library, in
-order to build LLVM.
+bugs in the compiler. We also attempt to follow improvements and developments in
+the C++ language and library reasonably closely. As such, we require a modern
+host C++ toolchain, both compiler and standard library, in order to build LLVM.
 
-For the most popular host toolchains we check for specific minimum versions in
-our build systems:
+LLVM is written using the subset of C++ documented in :doc:`coding
+standards<CodingStandards>`. To enforce this language version, we check the most
+popular host toolchains for specific minimum versions in our build systems:
+
+* Clang 3.5
+* Apple Clang 6.0
+* GCC 5.1
+* Visual Studio 2017
+
+The below versions currently soft-error as we transition to the new compiler
+versions listed above. The LLVM codebase is currently known to compile correctly
+with the following compilers, though this will change in the near future:
 
 * Clang 3.1
+* Apple Clang 3.1
 * GCC 4.8
 * Visual Studio 2015 (Update 3)
 
@@ -334,33 +292,36 @@ The first step is to get a recent GCC toolchain installed. The most common
 distribution on which users have struggled with the version requirements is
 Ubuntu Precise, 12.04 LTS. For this distribution, one easy option is to install
 the `toolchain testing PPA`_ and use it to install a modern GCC. There is
-a really nice discussions of this on the `ask ubuntu stack exchange`_. However,
-not all users can use PPAs and there are many other distributions, so it may be
-necessary (or just useful, if you're here you *are* doing compiler development
-after all) to build and install GCC from source. It is also quite easy to do
-these days.
+a really nice discussions of this on the `ask ubuntu stack exchange`_ and a
+`github gist`_ with updated commands. However, not all users can use PPAs and
+there are many other distributions, so it may be necessary (or just useful, if
+you're here you *are* doing compiler development after all) to build and install
+GCC from source. It is also quite easy to do these days.
 
 .. _toolchain testing PPA:
   https://launchpad.net/~ubuntu-toolchain-r/+archive/test
 .. _ask ubuntu stack exchange:
-  http://askubuntu.com/questions/271388/how-to-install-gcc-4-8-in-ubuntu-12-04-from-the-terminal
+  https://askubuntu.com/questions/466651/how-do-i-use-the-latest-gcc-on-ubuntu/581497#58149
+.. _github gist:
+  https://gist.github.com/application2000/73fd6f4bf1be6600a2cf9f56315a2d91
 
-Easy steps for installing GCC 4.8.2:
+Easy steps for installing GCC 5.1.0:
 
 .. code-block:: console
 
-  % wget https://ftp.gnu.org/gnu/gcc/gcc-4.8.2/gcc-4.8.2.tar.bz2
-  % wget https://ftp.gnu.org/gnu/gcc/gcc-4.8.2/gcc-4.8.2.tar.bz2.sig
+  % gcc_version=5.1.0
+  % wget https://ftp.gnu.org/gnu/gcc/gcc-${gcc_version}/gcc-${gcc_version}.tar.bz2
+  % wget https://ftp.gnu.org/gnu/gcc/gcc-${gcc_version}/gcc-${gcc_version}.tar.bz2.sig
   % wget https://ftp.gnu.org/gnu/gnu-keyring.gpg
-  % signature_invalid=`gpg --verify --no-default-keyring --keyring ./gnu-keyring.gpg gcc-4.8.2.tar.bz2.sig`
+  % signature_invalid=`gpg --verify --no-default-keyring --keyring ./gnu-keyring.gpg gcc-${gcc_version}.tar.bz2.sig`
   % if [ $signature_invalid ]; then echo "Invalid signature" ; exit 1 ; fi
-  % tar -xvjf gcc-4.8.2.tar.bz2
-  % cd gcc-4.8.2
+  % tar -xvjf gcc-${gcc_version}.tar.bz2
+  % cd gcc-${gcc_version}
   % ./contrib/download_prerequisites
   % cd ..
-  % mkdir gcc-4.8.2-build
-  % cd gcc-4.8.2-build
-  % $PWD/../gcc-4.8.2/configure --prefix=$HOME/toolchains --enable-languages=c,c++
+  % mkdir gcc-${gcc_version}-build
+  % cd gcc-${gcc_version}-build
+  % $PWD/../gcc-${gcc_version}/configure --prefix=$HOME/toolchains --enable-languages=c,c++
   % make -j$(nproc)
   % make install
 
@@ -368,7 +329,7 @@ For more details, check out the excellent `GCC wiki entry`_, where I got most
 of this information from.
 
 .. _GCC wiki entry:
-  http://gcc.gnu.org/wiki/InstallingGCC
+  https://gcc.gnu.org/wiki/InstallingGCC
 
 Once you have a GCC toolchain, configure your build of LLVM to use the new
 toolchain for your host compiler and C++ standard library. Because the new
@@ -388,7 +349,7 @@ If you fail to set rpath, most LLVM binaries will fail on startup with a message
 from the loader similar to ``libstdc++.so.6: version `GLIBCXX_3.4.20' not
 found``. This means you need to tweak the -rpath linker flag.
 
-When you build Clang, you will need to give *it* access to modern C++11
+When you build Clang, you will need to give *it* access to modern C++
 standard library in order to use it as your new host in part of a bootstrap.
 There are two easy ways to do this, either build (and install) libc++ along
 with Clang and then use it with the ``-stdlib=libc++`` compile and link flag,
@@ -433,10 +394,9 @@ Unpacking the LLVM Archives
 ---------------------------
 
 If you have the LLVM distribution, you will need to unpack it before you can
-begin to compile it.  LLVM is distributed as a set of two files: the LLVM suite
-and the LLVM GCC front end compiled for your platform.  There is an additional
-test suite that is optional.  Each file is a TAR archive that is compressed with
-the gzip program.
+begin to compile it.  LLVM is distributed as a number of different
+subprojects. Each one has its own download which is a TAR archive that is
+compressed with the gzip program.
 
 The files are as follows, with *x.y* marking the version number:
 
@@ -444,336 +404,158 @@ The files are as follows, with *x.y* marking the version number:
 
   Source release for the LLVM libraries and tools.
 
-``llvm-test-x.y.tar.gz``
+``cfe-x.y.tar.gz``
 
-  Source release for the LLVM test-suite.
+  Source release for the Clang frontend.
 
 .. _checkout:
 
-Checkout LLVM from Subversion
------------------------------
+Checkout LLVM from Git
+----------------------
 
-If you have access to our Subversion repository, you can get a fresh copy of the
-entire source code.  All you need to do is check it out from Subversion as
-follows:
-
-* ``cd where-you-want-llvm-to-live``
-* Read-Only: ``svn co http://llvm.org/svn/llvm-project/llvm/trunk llvm``
-* Read-Write: ``svn co https://user@llvm.org/svn/llvm-project/llvm/trunk llvm``
-
-This will create an '``llvm``' directory in the current directory and fully
-populate it with the LLVM source code, Makefiles, test directories, and local
-copies of documentation files.
-
-If you want to get a specific release (as opposed to the most recent revision),
-you can check it out from the '``tags``' directory (instead of '``trunk``'). The
-following releases are located in the following subdirectories of the '``tags``'
-directory:
-
-* Release 3.5.0 and later: **RELEASE_350/final** and so on
-* Release 2.9 through 3.4: **RELEASE_29/final** and so on
-* Release 1.1 through 2.8: **RELEASE_11** and so on
-* Release 1.0: **RELEASE_1**
-
-If you would like to get the LLVM test suite (a separate package as of 1.4), you
-get it from the Subversion repository:
-
-.. code-block:: console
-
-  % cd llvm/projects
-  % svn co http://llvm.org/svn/llvm-project/test-suite/trunk test-suite
-
-By placing it in the ``llvm/projects``, it will be automatically configured by
-the LLVM cmake configuration.
-
-Git Mirror
-----------
-
-Git mirrors are available for a number of LLVM subprojects. These mirrors sync
-automatically with each Subversion commit and contain all necessary git-svn
-marks (so, you can recreate git-svn metadata locally). Note that right now
-mirrors reflect only ``trunk`` for each project.
+You can also checkout the source code for LLVM from Git. While the LLVM
+project's official source-code repository is Subversion, we are in the process
+of migrating to git. We currently recommend that all developers use Git for
+day-to-day development.
 
 .. note::
 
-  On Windows, first you will want to do ``git config --global core.autocrlf
-  false`` before you clone. This goes a long way toward ensuring that
-  line-endings will be handled correctly (the LLVM project mostly uses Linux
-  line-endings).
-
-You can do the read-only Git clone of LLVM via:
-
-.. code-block:: console
-
-  % git clone https://git.llvm.org/git/llvm.git/
+  Passing ``--config core.autocrlf=false`` should not be required in
+  the future after we adjust the .gitattribute settings correctly, but
+  is required for Windows users at the time of this writing.
 
-If you want to check out clang too, run:
+Simply run:
 
 .. code-block:: console
 
-  % cd llvm/tools
-  % git clone https://git.llvm.org/git/clang.git/
+  % git clone https://github.com/llvm/llvm-project.git
 
-If you want to check out compiler-rt (required to build the sanitizers), run:
+or on Windows,
 
 .. code-block:: console
 
-  % cd llvm/projects
-  % git clone https://git.llvm.org/git/compiler-rt.git/
-
-If you want to check out libomp (required for OpenMP support), run:
-
-.. code-block:: console
-
-  % cd llvm/projects
-  % git clone https://git.llvm.org/git/openmp.git/
-
-If you want to check out libcxx and libcxxabi (optional), run:
-
-.. code-block:: console
+  % git clone --config core.autocrlf=false https://github.com/llvm/llvm-project.git
 
-  % cd llvm/projects
-  % git clone https://git.llvm.org/git/libcxx.git/
-  % git clone https://git.llvm.org/git/libcxxabi.git/
+This will create an '``llvm-project``' directory in the current directory and
+fully populate it with all of the source code, test directories, and local
+copies of documentation files for LLVM and all the related subprojects. Note
+that unlike the tarballs, which contain each subproject in a separate file, the
+git repository contains all of the projects together.
 
-If you want to check out the Test Suite Source Code (optional), run:
-
-.. code-block:: console
-
-  % cd llvm/projects
-  % git clone https://git.llvm.org/git/test-suite.git/
-
-Since the upstream repository is in Subversion, you should use ``git
-pull --rebase`` instead of ``git pull`` to avoid generating a non-linear history
-in your clone.  To configure ``git pull`` to pass ``--rebase`` by default on the
-master branch, run the following command:
-
-.. code-block:: console
-
-  % git config branch.master.rebase true
+If you want to get a specific release (as opposed to the most recent revision),
+you can check out a tag after cloning the repository. E.g., `git checkout
+llvmorg-6.0.1` inside the ``llvm-project`` directory created by the above
+command.  Use `git tag -l` to list all of them.
 
-Sending patches with Git
-^^^^^^^^^^^^^^^^^^^^^^^^
+Sending patches
+^^^^^^^^^^^^^^^
 
 Please read `Developer Policy <DeveloperPolicy.html#one-off-patches>`_, too.
 
-Assume ``master`` points the upstream and ``mybranch`` points your working
-branch, and ``mybranch`` is rebased onto ``master``.  At first you may check
-sanity of whitespaces:
+We don't currently accept github pull requests, so you'll need to send patches
+either via emailing to llvm-commits, or, preferably, via :ref:`Phabricator
+<phabricator-reviews>`.
 
-.. code-block:: console
-
-  % git diff --check master..mybranch
-
-The easiest way to generate a patch is as below:
-
-.. code-block:: console
-
-  % git diff master..mybranch > /path/to/mybranch.diff
-
-It is a little different from svn-generated diff. git-diff-generated diff has
-prefixes like ``a/`` and ``b/``. Don't worry, most developers might know it
-could be accepted with ``patch -p1 -N``.
-
-But you may generate patchset with git-format-patch. It generates by-each-commit
-patchset. To generate patch files to attach to your article:
-
-.. code-block:: console
-
-  % git format-patch --no-attach master..mybranch -o /path/to/your/patchset
-
-If you would like to send patches directly, you may use git-send-email or
-git-imap-send. Here is an example to generate the patchset in Gmail's [Drafts].
-
-.. code-block:: console
-
-  % git format-patch --attach master..mybranch --stdout | git imap-send
-
-Then, your .git/config should have [imap] sections.
-
-.. code-block:: ini
+You'll generally want to make sure your branch has a single commit,
+corresponding to the review you wish to send, up-to-date with the upstream
+``origin/master`` branch, and doesn't contain merges. Once you have that, you
+can use ``git show`` or ``git format-patch`` to output the diff, and attach it
+to a Phabricator review (or to an email message).
 
-  [imap]
-        host = imaps://imap.gmail.com
-        user = your.gmail.account@gmail.com
-        pass = himitsu!
-        port = 993
-        sslverify = false
-  ; in English
-        folder = "[Gmail]/Drafts"
-  ; example for Japanese, "Modified UTF-7" encoded.
-        folder = "[Gmail]/&Tgtm+DBN-"
-  ; example for Traditional Chinese
-        folder = "[Gmail]/&g0l6Pw-"
-
-.. _developers-work-with-git-svn:
-
-For developers to work with git-svn
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-To set up clone from which you can submit code using ``git-svn``, run:
+However, using the "Arcanist" tool is often easier. After `installing
+arcanist`_, you can upload the latest commit using:
 
 .. code-block:: console
 
-  % git clone https://git.llvm.org/git/llvm.git/
-  % cd llvm
-  % git svn init https://llvm.org/svn/llvm-project/llvm/trunk --username=<username>
-  % git config svn-remote.svn.fetch :refs/remotes/origin/master
-  % git svn rebase -l  # -l avoids fetching ahead of the git mirror.
-
-  # If you have clang too:
-  % cd tools
-  % git clone https://git.llvm.org/git/clang.git/
-  % cd clang
-  % git svn init https://llvm.org/svn/llvm-project/cfe/trunk --username=<username>
-  % git config svn-remote.svn.fetch :refs/remotes/origin/master
-  % git svn rebase -l
+  % arc diff HEAD~1
 
-Likewise for compiler-rt, libomp and test-suite.
-
-To update this clone without generating git-svn tags that conflict with the
-upstream Git repo, run:
+Additionally, before sending a patch for review, please also try to ensure it's
+formatted properly. We use ``clang-format`` for this, which has git integration
+through the ``git-clang-format`` script. On some systems, it may already be
+installed (or be installable via your package manager). If so, you can simply
+run it -- the following command will format only the code changed in the most
+recent commit:
 
 .. code-block:: console
 
-  % git fetch && (cd tools/clang && git fetch)  # Get matching revisions of both trees.
-  % git checkout master
-  % git svn rebase -l
-  % (cd tools/clang &&
-     git checkout master &&
-     git svn rebase -l)
-
-Likewise for compiler-rt, libomp and test-suite.
-
-This leaves your working directories on their master branches, so you'll need to
-``checkout`` each working branch individually and ``rebase`` it on top of its
-parent branch.
-
-For those who wish to be able to update an llvm repo/revert patches easily using
-git-svn, please look in the directory for the scripts ``git-svnup`` and
-``git-svnrevert``.
-
-To perform the aforementioned update steps go into your source directory and
-just type ``git-svnup`` or ``git svnup`` and everything will just work.
-
-If one wishes to revert a commit with git-svn, but do not want the git hash to
-escape into the commit message, one can use the script ``git-svnrevert`` or
-``git svnrevert`` which will take in the git hash for the commit you want to
-revert, look up the appropriate svn revision, and output a message where all
-references to the git hash have been replaced with the svn revision.
-
-To commit back changes via git-svn, use ``git svn dcommit``:
-
-.. code-block:: console
-
-  % git svn dcommit
-
-Note that git-svn will create one SVN commit for each Git commit you have pending,
-so squash and edit each commit before executing ``dcommit`` to make sure they all
-conform to the coding standards and the developers' policy.
-
-On success, ``dcommit`` will rebase against the HEAD of SVN, so to avoid conflict,
-please make sure your current branch is up-to-date (via fetch/rebase) before
-proceeding.
+  % git clang-format HEAD~1
 
-The git-svn metadata can get out of sync after you mess around with branches and
-``dcommit``. When that happens, ``git svn dcommit`` stops working, complaining
-about files with uncommitted changes. The fix is to rebuild the metadata:
+Note that this modifies the files, but doesn't commit them -- you'll likely want
+to run
 
 .. code-block:: console
 
-  % rm -rf .git/svn
-  % git svn rebase -l
+  % git commit --amend -a
 
-Please, refer to the Git-SVN manual (``man git-svn``) for more information.
-
-For developers to work with a git monorepo
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+in order to update the last commit with all pending changes.
 
 .. note::
+  If you don't already have ``clang-format`` or ``git clang-format`` installed
+  on your system, the ``clang-format`` binary will be built alongside clang, and
+  the git integration can be run from
+  ``clang/tools/clang-format/git-clang-format``.
 
-   This set-up is using an unofficial mirror hosted on GitHub, use with caution.
-
-To set up a clone of all the llvm projects using a unified repository:
 
-.. code-block:: console
+.. _commit_from_git:
 
-  % export TOP_LEVEL_DIR=`pwd`
-  % git clone https://github.com/llvm-project/llvm-project-20170507/ llvm-project
-  % cd llvm-project
-  % git config branch.master.rebase true
-
-You can configure various build directory from this clone, starting with a build
-of LLVM alone:
-
-.. code-block:: console
-
-  % cd $TOP_LEVEL_DIR
-  % mkdir llvm-build && cd llvm-build
-  % cmake -GNinja ../llvm-project/llvm
-
-Or lldb:
-
-.. code-block:: console
-
-  % cd $TOP_LEVEL_DIR
-  % mkdir lldb-build && cd lldb-build
-  % cmake -GNinja ../llvm-project/llvm -DLLVM_ENABLE_PROJECTS=lldb
-
-Or a combination of multiple projects:
-
-.. code-block:: console
-
-  % cd $TOP_LEVEL_DIR
-  % mkdir clang-build && cd clang-build
-  % cmake -GNinja ../llvm-project/llvm -DLLVM_ENABLE_PROJECTS="clang;libcxx;libcxxabi"
+For developers to commit changes from Git
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 A helper script is provided in ``llvm/utils/git-svn/git-llvm``. After you add it
-to your path, you can push committed changes upstream with ``git llvm push``.
+to your path, you can push committed changes upstream with ``git llvm
+push``. While this creates a Subversion checkout and patches it under the hood,
+it does not require you to have interaction with it.
 
 .. code-block:: console
 
   % export PATH=$PATH:$TOP_LEVEL_DIR/llvm-project/llvm/utils/git-svn/
   % git llvm push
 
-While this is using SVN under the hood, it does not require any interaction from
-you with git-svn.
-After a few minutes, ``git pull`` should get back the changes as they were
-committed. Note that a current limitation is that ``git`` does not directly
-record file rename, and thus it is propagated to SVN as a combination of
-delete-add instead of a file rename.
+Within a couple minutes after pushing to subversion, the svn commit will have
+been converted back to a Git commit, and made its way into the official Git
+repository. At that point, ``git pull`` should get back the changes as they were
+committed.
 
-The SVN revision of each monorepo commit can be found in the commit notes.  git
-does not fetch notes by default. The following commands will fetch the notes and
-configure git to fetch future notes. Use ``git notes show $commit`` to look up
-the SVN revision of a git commit. The notes show up ``git log``, and searching
-the log is currently the recommended way to look up the git commit for a given
-SVN revision.
+You'll likely want to ``git pull --rebase`` to get the official git commit
+downloaded back to your repository. The SVN revision numbers of each commit can
+be found at the end of the commit message, e.g. ``llvm-svn: 350914``.
 
-.. code-block:: console
+You may also find the ``-n`` flag useful, like ``git llvm push -n``. This runs
+through all the steps of committing _without_ actually doing the commit, and
+tell you what it would have done. That can be useful if you're unsure whether
+the right thing will happen.
 
-  % git config --add remote.origin.fetch +refs/notes/commits:refs/notes/commits
-  % git fetch
+Checkout via SVN (deprecated)
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-If you are using `arc` to interact with Phabricator, you need to manually put it
-at the root of the checkout:
+Until we have fully migrated to Git, you may also get a fresh copy of
+the code from the official Subversion repository.
 
-.. code-block:: console
+* ``cd where-you-want-llvm-to-live``
+* Read-Only: ``svn co http://llvm.org/svn/llvm-project/llvm/trunk llvm``
+* Read-Write: ``svn co https://user@llvm.org/svn/llvm-project/llvm/trunk llvm``
 
-  % cd $TOP_LEVEL_DIR
-  % cp llvm/.arcconfig ./
-  % mkdir -p .git/info/
-  % echo .arcconfig >> .git/info/exclude
+This will create an '``llvm``' directory in the current directory and fully
+populate it with the LLVM source code, Makefiles, test directories, and local
+copies of documentation files.
 
+If you want to get a specific release (as opposed to the most recent revision),
+you can check it out from the '``tags``' directory (instead of '``trunk``'). The
+following releases are located in the following subdirectories of the '``tags``'
+directory:
+
+* Release 3.5.0 and later: **RELEASE_350/final** and so on
+* Release 2.9 through 3.4: **RELEASE_29/final** and so on
+* Release 1.1 through 2.8: **RELEASE_11** and so on
+* Release 1.0: **RELEASE_1**
 
 Local LLVM Configuration
 ------------------------
 
-Once checked out from the Subversion repository, the LLVM suite source code must
-be configured before being built. This process uses CMake.
-Unlinke the normal ``configure`` script, CMake
-generates the build files in whatever format you request as well as various
-``*.inc`` files, and ``llvm/include/Config/config.h``.
+Once checked out repository, the LLVM suite source code must be configured
+before being built. This process uses CMake.  Unlinke the normal ``configure``
+script, CMake generates the build files in whatever format you request as well
+as various ``*.inc`` files, and ``llvm/include/Config/config.h``.
 
 Variables are passed to ``cmake`` on the command line using the format
 ``-D<variable name>=<value>``. The following variables are some common options
@@ -797,19 +579,26 @@ used by people developing LLVM.
 |                         | running the install action of the build files.     |
 +-------------------------+----------------------------------------------------+
 | LLVM_TARGETS_TO_BUILD   | A semicolon delimited list controlling which       |
-|                         | targets will be built and linked into llc. This is |
-|                         | equivalent to the ``--enable-targets`` option in   |
-|                         | the configure script. The default list is defined  |
-|                         | as ``LLVM_ALL_TARGETS``, and can be set to include |
+|                         | targets will be built and linked into llvm.        |
+|                         | The default list is defined as                     |
+|                         | ``LLVM_ALL_TARGETS``, and can be set to include    |
 |                         | out-of-tree targets. The default value includes:   |
 |                         | ``AArch64, AMDGPU, ARM, BPF, Hexagon, Mips,        |
 |                         | MSP430, NVPTX, PowerPC, Sparc, SystemZ, X86,       |
 |                         | XCore``.                                           |
+|                         |                                                    |
 +-------------------------+----------------------------------------------------+
 | LLVM_ENABLE_DOXYGEN     | Build doxygen-based documentation from the source  |
 |                         | code This is disabled by default because it is     |
 |                         | slow and generates a lot of output.                |
 +-------------------------+----------------------------------------------------+
+| LLVM_ENABLE_PROJECTS    | A semicolon-delimited list selecting which of the  |
+|                         | other LLVM subprojects to additionally build. (Only|
+|                         | effective when using a side-by-side project layout |
+|                         | e.g. via git). The default list is empty. Can      |
+|                         | include: clang, libcxx, libcxxabi, libunwind, lldb,|
+|                         | compiler-rt, lld, polly, or debuginfo-tests.       |
++-------------------------+----------------------------------------------------+
 | LLVM_ENABLE_SPHINX      | Build sphinx-based documentation from the source   |
 |                         | code. This is disabled by default because it is    |
 |                         | slow and generates a lot of output. Sphinx version |
@@ -1030,10 +819,10 @@ Public header files exported from the LLVM library. The three main subdirectorie
 
 ``llvm/include/llvm/Config``
 
-  Header files configured by the ``configure`` script.
-  They wrap "standard" UNIX and C header files.  Source code can include these
-  header files which automatically take care of the conditional #includes that
-  the ``configure`` script generates.
+  Header files configured by ``cmake``.  They wrap "standard" UNIX and
+  C header files.  Source code can include these header files which
+  automatically take care of the conditional #includes that ``cmake``
+  generates.
 
 ``llvm/lib``
 ------------
@@ -1105,10 +894,11 @@ are intended to run quickly and cover a lot of territory without being exhaustiv
 ``test-suite``
 --------------
 
-A comprehensive correctness, performance, and benchmarking test suite for LLVM.
-Comes in a separate Subversion module because not every LLVM user is interested
-in such a comprehensive suite. For details see the :doc:`Testing Guide
-<TestingGuide>` document.
+A comprehensive correctness, performance, and benchmarking test suite
+for LLVM.  This comes in a ``separate git repository
+<https://github.com/llvm/llvm-test-suite>``, because it contains a
+large amount of third-party code under a variety of licenses. For
+details see the :doc:`Testing Guide <TestingGuide>` document.
 
 .. _tools:
 
@@ -1322,3 +1112,5 @@ write something up!).  For more information about LLVM, check out:
 * `LLVM Homepage <http://llvm.org/>`_
 * `LLVM Doxygen Tree <http://llvm.org/doxygen/>`_
 * `Starting a Project that Uses LLVM <http://llvm.org/docs/Projects.html>`_
+
+.. _installing arcanist: https://secure.phabricator.com/book/phabricator/article/arcanist_quick_start/
diff --git a/docs/HowToSubmitABug.rst b/docs/HowToSubmitABug.rst
index 7881a6e8dcc30fbb6571e6ef6ae425f1b090e835..d276ee8681f380121672c153fca703edc9d2437c 100644
--- a/docs/HowToSubmitABug.rst
+++ b/docs/HowToSubmitABug.rst
@@ -84,8 +84,8 @@ Compile-time optimization bugs
 ------------------------------
 
 If you find that a bug crashes in the optimizer, compile your test-case to a
-``.bc`` file by passing "``-emit-llvm -O0 -c -o foo.bc``".
-Then run:
+``.bc`` file by passing "``-emit-llvm -O1 -Xclang -disable-llvm-passes -c -o
+foo.bc``".  Then run:
 
 .. code-block:: bash
 
diff --git a/docs/LLVMBuild.txt b/docs/LLVMBuild.txt
index d5aea864ecdd849ce9128bff1396a6a8537a349d..00d82f664c16eb05cdef13298f4b8b9be0cefcd3 100644
--- a/docs/LLVMBuild.txt
+++ b/docs/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./docs/LLVMBuild.txt -------------------------------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/docs/LangRef.rst b/docs/LangRef.rst
index ea879a5db5b70f5ae78ef12b0ecf7b878833feea..044f3521ce6cfebda46a37812a38870bd69b82cf 100644
--- a/docs/LangRef.rst
+++ b/docs/LangRef.rst
@@ -2181,6 +2181,24 @@ volatile operations. The optimizers *may* change the order of volatile
 operations relative to non-volatile operations. This is not Java's
 "volatile" and has no cross-thread synchronization behavior.
 
+A volatile load or store may have additional target-specific semantics.
+Any volatile operation can have side effects, and any volatile operation
+can read and/or modify state which is not accessible via a regular load
+or store in this module. Volatile operations may use addresses which do
+not point to memory (like MMIO registers). This means the compiler may
+not use a volatile operation to prove a non-volatile access to that
+address has defined behavior.
+
+The allowed side-effects for volatile accesses are limited.  If a
+non-volatile store to a given address would be legal, a volatile
+operation may modify the memory at that address. A volatile operation
+may not modify any other memory accessible by the module being compiled.
+A volatile operation may not call any code in the current module.
+
+The compiler may assume execution will continue after a volatile operation,
+so operations which modify memory or may have undefined behavior can be
+hoisted past a volatile operation.
+
 IR-level volatile loads and stores cannot safely be optimized into
 llvm.memcpy or llvm.memmove intrinsics even when those intrinsics are
 flagged volatile. Likewise, the backend should never split or merge
@@ -4493,13 +4511,36 @@ DIGlobalVariable
 
 .. code-block:: text
 
-    !0 = !DIGlobalVariable(name: "foo", linkageName: "foo", scope: !1,
-                           file: !2, line: 7, type: !3, isLocal: true,
-                           isDefinition: false, variable: i32* @foo,
-                           declaration: !4)
+    @foo = global i32, !dbg !0
+    !0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression())
+    !1 = !DIGlobalVariable(name: "foo", linkageName: "foo", scope: !2,
+                           file: !3, line: 7, type: !4, isLocal: true,
+                           isDefinition: false, declaration: !5)
+
 
-All global variables should be referenced by the `globals:` field of a
-:ref:`compile unit <DICompileUnit>`.
+DIGlobalVariableExpression
+""""""""""""""""""""""""""
+
+``DIGlobalVariableExpression`` nodes tie a :ref:`DIGlobalVariable` together
+with a :ref:`DIExpression`.
+
+.. code-block:: text
+
+    @lower = global i32, !dbg !0
+    @upper = global i32, !dbg !1
+    !0 = !DIGlobalVariableExpression(
+             var: !2,
+             expr: !DIExpression(DW_OP_LLVM_fragment, 0, 32)
+             )
+    !1 = !DIGlobalVariableExpression(
+             var: !2,
+             expr: !DIExpression(DW_OP_LLVM_fragment, 32, 32)
+             )
+    !2 = !DIGlobalVariable(name: "split64", linkageName: "split64", scope: !3,
+                           file: !4, line: 8, type: !5, declaration: !6)
+
+All global variable expressions should be referenced by the `globals:` field of
+a :ref:`compile unit <DICompileUnit>`.
 
 .. _DISubprogram:
 
@@ -4599,6 +4640,8 @@ parameter, and it will be included in the ``variables:`` field of its
                           type: !3)
     !2 = !DILocalVariable(name: "y", scope: !5, file: !2, line: 7, type: !3)
 
+.. _DIExpression:
+
 DIExpression
 """"""""""""
 
@@ -5066,6 +5109,80 @@ For example, in the code below, the call instruction may only target the
     ...
     !0 = !{i64 (i64, i64)* @add, i64 (i64, i64)* @sub}
 
+'``callback``' Metadata
+^^^^^^^^^^^^^^^^^^^^^^^
+
+``callback`` metadata may be attached to a function declaration, or definition.
+(Call sites are excluded only due to the lack of a use case.) For ease of
+exposition, we'll refer to the function annotated w/ metadata as a broker
+function. The metadata describes how the arguments of a call to the broker are
+in turn passed to the callback function specified by the metadata. Thus, the
+``callback`` metadata provides a partial description of a call site inside the
+broker function with regards to the arguments of a call to the broker. The only
+semantic restriction on the broker function itself is that it is not allowed to
+inspect or modify arguments referenced in the ``callback`` metadata as
+pass-through to the callback function.
+
+The broker is not required to actually invoke the callback function at runtime.
+However, the assumptions about not inspecting or modifying arguments that would
+be passed to the specified callback function still hold, even if the callback
+function is not dynamically invoked. The broker is allowed to invoke the
+callback function more than once per invocation of the broker. The broker is
+also allowed to invoke (directly or indirectly) the function passed as a
+callback through another use. Finally, the broker is also allowed to relay the
+callback callee invocation to a different thread.
+
+The metadata is structured as follows: At the outer level, ``callback``
+metadata is a list of ``callback`` encodings. Each encoding starts with a
+constant ``i64`` which describes the argument position of the callback function
+in the call to the broker. The following elements, except the last, describe
+what arguments are passed to the callback function. Each element is again an
+``i64`` constant identifying the argument of the broker that is passed through,
+or ``i64 -1`` to indicate an unknown or inspected argument. The order in which
+they are listed has to be the same in which they are passed to the callback
+callee. The last element of the encoding is a boolean which specifies how
+variadic arguments of the broker are handled. If it is true, all variadic
+arguments of the broker are passed through to the callback function *after* the
+arguments encoded explicitly before.
+
+In the code below, the ``pthread_create`` function is marked as a broker
+through the ``!callback !1`` metadata. In the example, there is only one
+callback encoding, namely ``!2``, associated with the broker. This encoding
+identifies the callback function as the second argument of the broker (``i64
+2``) and the sole argument of the callback function as the third one of the
+broker function (``i64 3``).
+
+.. FIXME why does the llvm-sphinx-docs builder give a highlighting
+   error if the below is set to highlight as 'llvm', despite that we
+   have misc.highlighting_failure set?
+
+.. code-block:: text
+
+    declare !callback !1 dso_local i32 @pthread_create(i64*, %union.pthread_attr_t*, i8* (i8*)*, i8*)
+
+    ...
+    !2 = !{i64 2, i64 3, i1 false}
+    !1 = !{!2}
+
+Another example is shown below. The callback callee is the second argument of
+the ``__kmpc_fork_call`` function (``i64 2``). The callee is given two unknown
+values (each identified by a ``i64 -1``) and afterwards all
+variadic arguments that are passed to the ``__kmpc_fork_call`` call (due to the
+final ``i1 true``).
+
+.. FIXME why does the llvm-sphinx-docs builder give a highlighting
+   error if the below is set to highlight as 'llvm', despite that we
+   have misc.highlighting_failure set?
+
+.. code-block:: text
+
+    declare !callback !0 dso_local void @__kmpc_fork_call(%struct.ident_t*, i32, void (i32*, i32*, ...)*, ...)
+
+    ...
+    !1 = !{i64 2, i64 -1, i64 -1, i1 true}
+    !0 = !{!1}
+
+
 '``unpredictable``' Metadata
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
@@ -5140,7 +5257,7 @@ vectorization width and interleave count. These metadata should be used in
 conjunction with ``llvm.loop`` loop identification metadata. The
 ``llvm.loop.vectorize`` and ``llvm.loop.interleave`` metadata are only
 optimization hints and the optimizer will only interleave and vectorize loops if
-it believes it is safe to do so. The ``llvm.mem.parallel_loop_access`` metadata
+it believes it is safe to do so. The ``llvm.loop.parallel_accesses`` metadata
 which contains information about loop-carried memory dependencies can be helpful
 in determining the safety of these transformations.
 
@@ -5443,89 +5560,119 @@ Thes attributes in this metdata is added to all followup loops of the
 loop distribution pass. See
 :ref:`Transformation Metadata <transformation-metadata>` for details.
 
-'``llvm.mem``'
-^^^^^^^^^^^^^^^
+'``llvm.access.group``' Metadata
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-Metadata types used to annotate memory accesses with information helpful
-for optimizations are prefixed with ``llvm.mem``.
+``llvm.access.group`` metadata can be attached to any instruction that
+potentially accesses memory. It can point to a single distinct metadata
+node, which we call access group. This node represents all memory access
+instructions referring to it via ``llvm.access.group``. When an
+instruction belongs to multiple access groups, it can also point to a
+list of accesses groups, illustrated by the following example.
 
-'``llvm.mem.parallel_loop_access``' Metadata
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+.. code-block:: llvm
+
+   %val = load i32, i32* %arrayidx, !llvm.access.group !0
+   ...
+   !0 = !{!1, !2}
+   !1 = distinct !{}
+   !2 = distinct !{}
+
+It is illegal for the list node to be empty since it might be confused
+with an access group.
+
+The access group metadata node must be 'distinct' to avoid collapsing
+multiple access groups by content. A access group metadata node must
+always be empty which can be used to distinguish an access group
+metadata node from a list of access groups. Being empty avoids the
+situation that the content must be updated which, because metadata is
+immutable by design, would required finding and updating all references
+to the access group node.
+
+The access group can be used to refer to a memory access instruction
+without pointing to it directly (which is not possible in global
+metadata). Currently, the only metadata making use of it is
+``llvm.loop.parallel_accesses``.
+
+'``llvm.loop.parallel_accesses``' Metadata
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-The ``llvm.mem.parallel_loop_access`` metadata refers to a loop identifier,
-or metadata containing a list of loop identifiers for nested loops.
-The metadata is attached to memory accessing instructions and denotes that
-no loop carried memory dependence exist between it and other instructions denoted
-with the same loop identifier. The metadata on memory reads also implies that
-if conversion (i.e. speculative execution within a loop iteration) is safe.
-
-Precisely, given two instructions ``m1`` and ``m2`` that both have the
-``llvm.mem.parallel_loop_access`` metadata, with ``L1`` and ``L2`` being the
-set of loops associated with that metadata, respectively, then there is no loop
-carried dependence between ``m1`` and ``m2`` for loops in both ``L1`` and
-``L2``.
-
-As a special case, if all memory accessing instructions in a loop have
-``llvm.mem.parallel_loop_access`` metadata that refers to that loop, then the
-loop has no loop carried memory dependences and is considered to be a parallel
-loop.
-
-Note that if not all memory access instructions have such metadata referring to
-the loop, then the loop is considered not being trivially parallel. Additional
+The ``llvm.loop.parallel_accesses`` metadata refers to one or more
+access group metadata nodes (see ``llvm.access.group``). It denotes that
+no loop-carried memory dependence exist between it and other instructions
+in the loop with this metadata.
+
+Let ``m1`` and ``m2`` be two instructions that both have the
+``llvm.access.group`` metadata to the access group ``g1``, respectively
+``g2`` (which might be identical). If a loop contains both access groups
+in its ``llvm.loop.parallel_accesses`` metadata, then the compiler can
+assume that there is no dependency between ``m1`` and ``m2`` carried by
+this loop. Instructions that belong to multiple access groups are
+considered having this property if at least one of the access groups
+matches the ``llvm.loop.parallel_accesses`` list.
+
+If all memory-accessing instructions in a loop have
+``llvm.loop.parallel_accesses`` metadata that refers to that loop, then the
+loop has no loop carried memory dependences and is considered to be a
+parallel loop.
+
+Note that if not all memory access instructions belong to an access
+group referred to by ``llvm.loop.parallel_accesses``, then the loop must
+not be considered trivially parallel. Additional
 memory dependence analysis is required to make that determination. As a fail
 safe mechanism, this causes loops that were originally parallel to be considered
 sequential (if optimization passes that are unaware of the parallel semantics
 insert new memory instructions into the loop body).
 
 Example of a loop that is considered parallel due to its correct use of
-both ``llvm.loop`` and ``llvm.mem.parallel_loop_access``
-metadata types that refer to the same loop identifier metadata.
+both ``llvm.access.group`` and ``llvm.loop.parallel_accesses``
+metadata types.
 
 .. code-block:: llvm
 
    for.body:
      ...
-     %val0 = load i32, i32* %arrayidx, !llvm.mem.parallel_loop_access !0
+     %val0 = load i32, i32* %arrayidx, !llvm.access.group !1
      ...
-     store i32 %val0, i32* %arrayidx1, !llvm.mem.parallel_loop_access !0
+     store i32 %val0, i32* %arrayidx1, !llvm.access.group !1
      ...
      br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !0
 
    for.end:
    ...
-   !0 = !{!0}
+   !0 = distinct !{!0, !{!"llvm.loop.parallel_accesses", !1}}
+   !1 = distinct !{}
 
-It is also possible to have nested parallel loops. In that case the
-memory accesses refer to a list of loop identifier metadata nodes instead of
-the loop identifier metadata node directly:
+It is also possible to have nested parallel loops:
 
 .. code-block:: llvm
 
    outer.for.body:
      ...
-     %val1 = load i32, i32* %arrayidx3, !llvm.mem.parallel_loop_access !2
+     %val1 = load i32, i32* %arrayidx3, !llvm.access.group !4
      ...
      br label %inner.for.body
 
    inner.for.body:
      ...
-     %val0 = load i32, i32* %arrayidx1, !llvm.mem.parallel_loop_access !0
+     %val0 = load i32, i32* %arrayidx1, !llvm.access.group !3
      ...
-     store i32 %val0, i32* %arrayidx2, !llvm.mem.parallel_loop_access !0
+     store i32 %val0, i32* %arrayidx2, !llvm.access.group !3
      ...
      br i1 %exitcond, label %inner.for.end, label %inner.for.body, !llvm.loop !1
 
    inner.for.end:
      ...
-     store i32 %val1, i32* %arrayidx4, !llvm.mem.parallel_loop_access !2
+     store i32 %val1, i32* %arrayidx4, !llvm.access.group !4
      ...
      br i1 %exitcond, label %outer.for.end, label %outer.for.body, !llvm.loop !2
 
    outer.for.end:                                          ; preds = %for.body
    ...
-   !0 = !{!1, !2} ; a list of loop identifiers
-   !1 = !{!1} ; an identifier for the inner loop
-   !2 = !{!2} ; an identifier for the outer loop
+   !1 = distinct !{!1, !{!"llvm.loop.parallel_accesses", !3}}     ; metadata for the inner loop
+   !2 = distinct !{!2, !{!"llvm.loop.parallel_accesses", !3, !4}} ; metadata for the outer loop
+   !3 = distinct !{} ; access group for instructions in the inner loop (which are implicitly contained in outer loop as well)
+   !4 = distinct !{} ; access group for instructions in the outer, but not the inner loop
 
 '``irr_loop``' Metadata
 ^^^^^^^^^^^^^^^^^^^^^^^
@@ -6366,6 +6513,7 @@ control flow, not values (the one exception being the
 The terminator instructions are: ':ref:`ret <i_ret>`',
 ':ref:`br <i_br>`', ':ref:`switch <i_switch>`',
 ':ref:`indirectbr <i_indirectbr>`', ':ref:`invoke <i_invoke>`',
+':ref:`callbr <i_callbr>`'
 ':ref:`resume <i_resume>`', ':ref:`catchswitch <i_catchswitch>`',
 ':ref:`catchret <i_catchret>`',
 ':ref:`cleanupret <i_cleanupret>`',
@@ -6401,7 +6549,7 @@ The '``ret``' instruction optionally accepts a single argument, the
 return value. The type of the return value must be a ':ref:`first
 class <t_firstclass>`' type.
 
-A function is not :ref:`well formed <wellformed>` if it it has a non-void
+A function is not :ref:`well formed <wellformed>` if it has a non-void
 return type and contains a '``ret``' instruction with no return value or
 a return value with a type that does not match its type, or if it has a
 void return type and contains a '``ret``' instruction with a return
@@ -6636,7 +6784,7 @@ This instruction requires several arguments:
 #. The optional :ref:`Parameter Attributes <paramattrs>` list for return
    values. Only '``zeroext``', '``signext``', and '``inreg``' attributes
    are valid here.
-#. The optional addrspace attribute can be used to indicate the adress space
+#. The optional addrspace attribute can be used to indicate the address space
    of the called function. If it is not specified, the program address space
    from the :ref:`datalayout string<langref_datalayout>` will be used.
 #. '``ty``': the type of the call instruction itself which is also the
@@ -6690,6 +6838,85 @@ Example:
       %retval = invoke coldcc i32 %Testfnptr(i32 15) to label %Continue
                   unwind label %TestCleanup              ; i32:retval set
 
+.. _i_callbr:
+
+'``callbr``' Instruction
+^^^^^^^^^^^^^^^^^^^^^^^^
+
+Syntax:
+"""""""
+
+::
+
+      <result> = callbr [cconv] [ret attrs] [addrspace(<num>)] [<ty>|<fnty> <fnptrval>(<function args>) [fn attrs]
+                    [operand bundles] to label <normal label> or jump [other labels]
+
+Overview:
+"""""""""
+
+The '``callbr``' instruction causes control to transfer to a specified
+function, with the possibility of control flow transfer to either the
+'``normal``' label or one of the '``other``' labels.
+
+This instruction should only be used to implement the "goto" feature of gcc
+style inline assembly. Any other usage is an error in the IR verifier.
+
+Arguments:
+""""""""""
+
+This instruction requires several arguments:
+
+#. The optional "cconv" marker indicates which :ref:`calling
+   convention <callingconv>` the call should use. If none is
+   specified, the call defaults to using C calling conventions.
+#. The optional :ref:`Parameter Attributes <paramattrs>` list for return
+   values. Only '``zeroext``', '``signext``', and '``inreg``' attributes
+   are valid here.
+#. The optional addrspace attribute can be used to indicate the address space
+   of the called function. If it is not specified, the program address space
+   from the :ref:`datalayout string<langref_datalayout>` will be used.
+#. '``ty``': the type of the call instruction itself which is also the
+   type of the return value. Functions that return no value are marked
+   ``void``.
+#. '``fnty``': shall be the signature of the function being called. The
+   argument types must match the types implied by this signature. This
+   type can be omitted if the function is not varargs.
+#. '``fnptrval``': An LLVM value containing a pointer to a function to
+   be called. In most cases, this is a direct function call, but
+   indirect ``callbr``'s are just as possible, calling an arbitrary pointer
+   to function value.
+#. '``function args``': argument list whose types match the function
+   signature argument types and parameter attributes. All arguments must
+   be of :ref:`first class <t_firstclass>` type. If the function signature
+   indicates the function accepts a variable number of arguments, the
+   extra arguments can be specified.
+#. '``normal label``': the label reached when the called function
+   executes a '``ret``' instruction.
+#. '``other labels``': the labels reached when a callee transfers control
+   to a location other than the normal '``normal label``'
+#. The optional :ref:`function attributes <fnattrs>` list.
+#. The optional :ref:`operand bundles <opbundles>` list.
+
+Semantics:
+""""""""""
+
+This instruction is designed to operate as a standard '``call``'
+instruction in most regards. The primary difference is that it
+establishes an association with additional labels to define where control
+flow goes after the call.
+
+The only use of this today is to implement the "goto" feature of gcc inline
+assembly where additional labels can be provided as locations for the inline
+assembly to jump to.
+
+Example:
+""""""""
+
+.. code-block:: text
+
+      callbr void asm "", "r,x"(i32 %x, i8 *blockaddress(@foo, %fail))
+                  to label %normal or jump [label %fail]
+
 .. _i_resume:
 
 '``resume``' Instruction
@@ -6941,7 +7168,7 @@ Arguments:
 
 The argument to the '``fneg``' instruction must be a
 :ref:`floating-point <t_floating>` or :ref:`vector <t_vector>` of
-floating-point values. 
+floating-point values.
 
 Semantics:
 """"""""""
@@ -8553,14 +8780,18 @@ operation. The operation must be one of the following keywords:
 -  min
 -  umax
 -  umin
-
-The type of '<value>' must be an integer type whose bit width is a power
-of two greater than or equal to eight and less than or equal to a
-target-specific size limit. The type of the '``<pointer>``' operand must
-be a pointer to that type. If the ``atomicrmw`` is marked as
-``volatile``, then the optimizer is not allowed to modify the number or
-order of execution of this ``atomicrmw`` with other :ref:`volatile
-operations <volatile>`.
+-  fadd
+-  fsub
+
+For most of these operations, the type of '<value>' must be an integer
+type whose bit width is a power of two greater than or equal to eight
+and less than or equal to a target-specific size limit. For xchg, this
+may also be a floating point type with the same size constraints as
+integers.  For fadd/fsub, this must be a floating point type.  The
+type of the '``<pointer>``' operand must be a pointer to that type. If
+the ``atomicrmw`` is marked as ``volatile``, then the optimizer is not
+allowed to modify the number or order of execution of this
+``atomicrmw`` with other :ref:`volatile operations <volatile>`.
 
 A ``atomicrmw`` instruction can also take an optional
 ":ref:`syncscope <syncscope>`" argument.
@@ -8586,6 +8817,8 @@ operation argument:
    comparison)
 -  umin: ``*ptr = *ptr < val ? *ptr : val`` (using an unsigned
    comparison)
+- fadd: ``*ptr = *ptr + val`` (using floating point arithmetic)
+- fsub: ``*ptr = *ptr - val`` (using floating point arithmetic)
 
 Example:
 """"""""
@@ -9772,7 +10005,7 @@ This instruction requires several arguments:
 #. The optional :ref:`Parameter Attributes <paramattrs>` list for return
    values. Only '``zeroext``', '``signext``', and '``inreg``' attributes
    are valid here.
-#. The optional addrspace attribute can be used to indicate the adress space
+#. The optional addrspace attribute can be used to indicate the address space
    of the called function. If it is not specified, the program address space
    from the :ref:`datalayout string<langref_datalayout>` will be used.
 #. '``ty``': the type of the call instruction itself which is also the
@@ -12097,19 +12330,22 @@ integer type.
       declare i16 @llvm.bitreverse.i16(i16 <id>)
       declare i32 @llvm.bitreverse.i32(i32 <id>)
       declare i64 @llvm.bitreverse.i64(i64 <id>)
+      declare <4 x i32> @llvm.bitreverse.v4i32(<4 x i32> <id>)
 
 Overview:
 """""""""
 
 The '``llvm.bitreverse``' family of intrinsics is used to reverse the
-bitpattern of an integer value; for example ``0b10110110`` becomes
-``0b01101101``.
+bitpattern of an integer value or vector of integer values; for example
+``0b10110110`` becomes ``0b01101101``.
 
 Semantics:
 """"""""""
 
 The ``llvm.bitreverse.iN`` intrinsic returns an iN value that has bit
-``M`` in the input moved to bit ``N-M`` in the output.
+``M`` in the input moved to bit ``N-M`` in the output. The vector
+intrinsics, such as ``llvm.bitreverse.v4i32``, operate on a per-element
+basis and the element order is not affected.
 
 '``llvm.bswap.*``' Intrinsics
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -12125,14 +12361,14 @@ integer type that is an even number of bytes (i.e. BitWidth % 16 == 0).
       declare i16 @llvm.bswap.i16(i16 <id>)
       declare i32 @llvm.bswap.i32(i32 <id>)
       declare i64 @llvm.bswap.i64(i64 <id>)
+      declare <4 x i32> @llvm.bswap.v4i32(<4 x i32> <id>)
 
 Overview:
 """""""""
 
-The '``llvm.bswap``' family of intrinsics is used to byte swap integer
-values with an even number of bytes (positive multiple of 16 bits).
-These are useful for performing operations on data that is not in the
-target's native byte order.
+The '``llvm.bswap``' family of intrinsics is used to byte swap an integer
+value or vector of integer values with an even number of bytes (positive
+multiple of 16 bits).
 
 Semantics:
 """"""""""
@@ -12144,7 +12380,8 @@ swapped, so that if the input bytes are numbered 0, 1, 2, 3 then the
 returned i32 will have its bytes in 3, 2, 1, 0 order. The
 ``llvm.bswap.i48``, ``llvm.bswap.i64`` and other intrinsics extend this
 concept to additional even-byte lengths (6 bytes, 8 bytes and more,
-respectively).
+respectively). The vector intrinsics, such as ``llvm.bswap.v4i32``,
+operate on a per-element basis and the element order is not affected.
 
 '``llvm.ctpop.*``' Intrinsic
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -12932,7 +13169,8 @@ Arguments
 """"""""""
 
 The arguments (%a and %b) and the result may be of integer types of any bit
-width, but they must have the same bit width. ``%a`` and ``%b`` are the two
+width, but they must have the same bit width. The arguments may also work with
+int vectors of the same length and int size. ``%a`` and ``%b`` are the two
 values that will undergo signed fixed point multiplication. The argument
 ``%scale`` represents the scale of both operands, and must be a constant
 integer.
@@ -12948,7 +13186,7 @@ If the result value cannot be precisely represented in the given scale, the
 value is rounded up or down to the closest representable value. The rounding
 direction is unspecified.
 
-It is undefined behavior if the source value does not fit within the range of
+It is undefined behavior if the result value does not fit within the range of
 the fixed point type.
 
 
@@ -12965,6 +13203,65 @@ Examples
       %res = call i4 @llvm.smul.fix.i4(i4 3, i4 -3, i32 1)  ; %res = -5 (or -4) (1.5 x -1.5 = -2.25)
 
 
+'``llvm.umul.fix.*``' Intrinsics
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Syntax
+"""""""
+
+This is an overloaded intrinsic. You can use ``llvm.umul.fix``
+on any integer bit width or vectors of integers.
+
+::
+
+      declare i16 @llvm.umul.fix.i16(i16 %a, i16 %b, i32 %scale)
+      declare i32 @llvm.umul.fix.i32(i32 %a, i32 %b, i32 %scale)
+      declare i64 @llvm.umul.fix.i64(i64 %a, i64 %b, i32 %scale)
+      declare <4 x i32> @llvm.umul.fix.v4i32(<4 x i32> %a, <4 x i32> %b, i32 %scale)
+
+Overview
+"""""""""
+
+The '``llvm.umul.fix``' family of intrinsic functions perform unsigned
+fixed point multiplication on 2 arguments of the same scale.
+
+Arguments
+""""""""""
+
+The arguments (%a and %b) and the result may be of integer types of any bit
+width, but they must have the same bit width. The arguments may also work with
+int vectors of the same length and int size. ``%a`` and ``%b`` are the two
+values that will undergo unsigned fixed point multiplication. The argument
+``%scale`` represents the scale of both operands, and must be a constant
+integer.
+
+Semantics:
+""""""""""
+
+This operation performs unsigned fixed point multiplication on the 2 arguments of a
+specified scale. The result will also be returned in the same scale specified
+in the third argument.
+
+If the result value cannot be precisely represented in the given scale, the
+value is rounded up or down to the closest representable value. The rounding
+direction is unspecified.
+
+It is undefined behavior if the result value does not fit within the range of
+the fixed point type.
+
+
+Examples
+"""""""""
+
+.. code-block:: llvm
+
+      %res = call i4 @llvm.umul.fix.i4(i4 3, i4 2, i32 0)  ; %res = 6 (2 x 3 = 6)
+      %res = call i4 @llvm.umul.fix.i4(i4 3, i4 2, i32 1)  ; %res = 3 (1.5 x 1 = 1.5)
+
+      ; The result in the following could be rounded down to 3.5 or up to 4
+      %res = call i4 @llvm.umul.fix.i4(i4 15, i4 1, i32 1)  ; %res = 7 (or 8) (7.5 x 0.5 = 3.75)
+
+
 Specialised Arithmetic Intrinsics
 ---------------------------------
 
@@ -14957,13 +15254,13 @@ Syntax:
 Overview:
 """""""""
 
-The '``llvm.experimental.constrained.maxnum``' intrinsic returns the maximum 
+The '``llvm.experimental.constrained.maxnum``' intrinsic returns the maximum
 of the two arguments.
 
 Arguments:
 """"""""""
 
-The first two arguments and the return value are floating-point numbers 
+The first two arguments and the return value are floating-point numbers
 of the same type.
 
 The third and forth arguments specify the rounding mode and exception
@@ -15031,7 +15328,7 @@ Syntax:
 Overview:
 """""""""
 
-The '``llvm.experimental.constrained.ceil``' intrinsic returns the ceiling of the 
+The '``llvm.experimental.constrained.ceil``' intrinsic returns the ceiling of the
 first operand.
 
 Arguments:
@@ -15067,7 +15364,7 @@ Syntax:
 Overview:
 """""""""
 
-The '``llvm.experimental.constrained.floor``' intrinsic returns the floor of the 
+The '``llvm.experimental.constrained.floor``' intrinsic returns the floor of the
 first operand.
 
 Arguments:
@@ -15084,7 +15381,7 @@ Semantics:
 """"""""""
 
 This function returns the same values as the libm ``floor`` functions
-would and handles error conditions in the same way. 
+would and handles error conditions in the same way.
 
 
 '``llvm.experimental.constrained.round``' Intrinsic
@@ -15103,7 +15400,7 @@ Syntax:
 Overview:
 """""""""
 
-The '``llvm.experimental.constrained.round``' intrinsic returns the first 
+The '``llvm.experimental.constrained.round``' intrinsic returns the first
 operand rounded to the nearest integer.
 
 Arguments:
@@ -15139,8 +15436,8 @@ Syntax:
 Overview:
 """""""""
 
-The '``llvm.experimental.constrained.trunc``' intrinsic returns the first 
-operand rounded to the nearest integer not larger in magnitude than the 
+The '``llvm.experimental.constrained.trunc``' intrinsic returns the first
+operand rounded to the nearest integer not larger in magnitude than the
 operand.
 
 Arguments:
@@ -15428,40 +15725,40 @@ Syntax:
 
 ::
 
-      declare i32 @llvm.objectsize.i32(i8* <object>, i1 <min>, i1 <nullunknown>)
-      declare i64 @llvm.objectsize.i64(i8* <object>, i1 <min>, i1 <nullunknown>)
+      declare i32 @llvm.objectsize.i32(i8* <object>, i1 <min>, i1 <nullunknown>, i1 <dynamic>)
+      declare i64 @llvm.objectsize.i64(i8* <object>, i1 <min>, i1 <nullunknown>, i1 <dynamic>)
 
 Overview:
 """""""""
 
-The ``llvm.objectsize`` intrinsic is designed to provide information to
-the optimizers to determine at compile time whether a) an operation
-(like memcpy) will overflow a buffer that corresponds to an object, or
-b) that a runtime check for overflow isn't necessary. An object in this
-context means an allocation of a specific class, structure, array, or
-other object.
+The ``llvm.objectsize`` intrinsic is designed to provide information to the
+optimizer to determine whether a) an operation (like memcpy) will overflow a
+buffer that corresponds to an object, or b) that a runtime check for overflow
+isn't necessary. An object in this context means an allocation of a specific
+class, structure, array, or other object.
 
 Arguments:
 """"""""""
 
-The ``llvm.objectsize`` intrinsic takes three arguments. The first argument is
-a pointer to or into the ``object``. The second argument determines whether
-``llvm.objectsize`` returns 0 (if true) or -1 (if false) when the object size
-is unknown. The third argument controls how ``llvm.objectsize`` acts when
-``null`` in address space 0 is used as its pointer argument. If it's ``false``,
+The ``llvm.objectsize`` intrinsic takes four arguments. The first argument is a
+pointer to or into the ``object``. The second argument determines whether
+``llvm.objectsize`` returns 0 (if true) or -1 (if false) when the object size is
+unknown. The third argument controls how ``llvm.objectsize`` acts when ``null``
+in address space 0 is used as its pointer argument. If it's ``false``,
 ``llvm.objectsize`` reports 0 bytes available when given ``null``. Otherwise, if
 the ``null`` is in a non-zero address space or if ``true`` is given for the
-third argument of ``llvm.objectsize``, we assume its size is unknown.
+third argument of ``llvm.objectsize``, we assume its size is unknown. The fourth
+argument to ``llvm.objectsize`` determines if the value should be evaluated at
+runtime.
 
-The second and third arguments only accept constants.
+The second, third, and fourth arguments only accept constants.
 
 Semantics:
 """"""""""
 
-The ``llvm.objectsize`` intrinsic is lowered to a constant representing
-the size of the object concerned. If the size cannot be determined at
-compile time, ``llvm.objectsize`` returns ``i32/i64 -1 or 0`` (depending
-on the ``min`` argument).
+The ``llvm.objectsize`` intrinsic is lowered to a value representing the size of
+the object concerned. If the size cannot be determined, ``llvm.objectsize``
+returns ``i32/i64 -1 or 0`` (depending on the ``min`` argument).
 
 '``llvm.expect``' Intrinsic
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^
diff --git a/docs/LibFuzzer.rst b/docs/LibFuzzer.rst
index 0737fbbcd93050a554146675caa0ebf21f4a2efe..998dc7ca3ba0a502cef70f1b3ede57624f9457aa 100644
--- a/docs/LibFuzzer.rst
+++ b/docs/LibFuzzer.rst
@@ -414,8 +414,8 @@ A simple function that does something interesting if it receives the input
     return 0;
   }
   EOF
-  # Build test_fuzzer.cc with asan and link against libFuzzer.a
-  clang++ -fsanitize=address -fsanitize-coverage=trace-pc-guard test_fuzzer.cc libFuzzer.a
+  # Build test_fuzzer.cc with asan and link against libFuzzer.
+  clang++ -fsanitize=address,fuzzer test_fuzzer.cc
   # Run the fuzzer with no corpus.
   ./a.out
 
@@ -483,7 +483,7 @@ the fuzzing but is very likely to improve the results.
 Value Profile
 -------------
 
-With  ``-fsanitize-coverage=trace-cmp``
+With  ``-fsanitize-coverage=trace-cmp`` (default with ``-fsanitize=fuzzer``)
 and extra run-time flag ``-use_value_profile=1`` the fuzzer will
 collect value profiles for the parameters of compare instructions
 and treat some new values as new coverage.
@@ -544,7 +544,7 @@ Periodically restart both fuzzers so that they can use each other's findings.
 Currently, there is no simple way to run both fuzzing engines in parallel while sharing the same corpus dir.
 
 You may also use AFL on your target function ``LLVMFuzzerTestOneInput``:
-see an example `here <https://github.com/llvm-mirror/compiler-rt/tree/master/lib/fuzzer/afl>`__.
+see an example `here <https://github.com/llvm/llvm-project/tree/master/compiler-rt/lib/fuzzer/afl>`__.
 
 How good is my fuzzer?
 ----------------------
@@ -562,8 +562,9 @@ to visualize and study your code coverage
 User-supplied mutators
 ----------------------
 
-LibFuzzer allows to use custom (user-supplied) mutators,
-see FuzzerInterface.h_
+LibFuzzer allows to use custom (user-supplied) mutators, see
+`Structure-Aware Fuzzing <https://github.com/google/fuzzer-test-suite/blob/master/tutorial/structure-aware-fuzzing.md>`_
+for more details.
 
 Startup initialization
 ----------------------
@@ -645,10 +646,20 @@ coverage set of the process (since the fuzzer is in-process). In other words, by
 using more external dependencies we will slow down the fuzzer while the main
 reason for it to exist is extreme speed.
 
-Q. What about Windows then? The fuzzer contains code that does not build on Windows.
+Q. Does libFuzzer Support Windows?
 ------------------------------------------------------------------------------------
 
-Volunteers are welcome.
+Yes, libFuzzer now supports Windows. Initial support was added in r341082.
+Any build of Clang 9 supports it. You can download a build of Clang for Windows
+that has libFuzzer from
+`LLVM Snapshot Builds <https://llvm.org/builds/>`_.
+
+Using libFuzzer on Windows without ASAN is unsupported. Building fuzzers with the
+``/MD`` (dynamic runtime library) compile option is unsupported. Support for these
+may be added in the future. Linking fuzzers with the ``/INCREMENTAL`` link option
+(or the ``/DEBUG`` option which implies it) is also unsupported.
+
+Send any questions or comments to the mailing list: libfuzzer(#)googlegroups.com
 
 Q. When libFuzzer is not a good solution for a problem?
 ---------------------------------------------------------
@@ -741,7 +752,7 @@ Trophies
 .. _AddressSanitizer: http://clang.llvm.org/docs/AddressSanitizer.html
 .. _LeakSanitizer: http://clang.llvm.org/docs/LeakSanitizer.html
 .. _Heartbleed: http://en.wikipedia.org/wiki/Heartbleed
-.. _FuzzerInterface.h: https://github.com/llvm-mirror/compiler-rt/blob/master/lib/fuzzer/FuzzerInterface.h
+.. _FuzzerInterface.h: https://github.com/llvm/llvm-project/blob/master/compiler-rt/lib/fuzzer/FuzzerInterface.h
 .. _3.7.0: http://llvm.org/releases/3.7.0/docs/LibFuzzer.html
 .. _building Clang from trunk: http://clang.llvm.org/get_started.html
 .. _MemorySanitizer: http://clang.llvm.org/docs/MemorySanitizer.html
diff --git a/docs/Phabricator.rst b/docs/Phabricator.rst
index 640e1611da6c442d395dc9de9304ab6c85c1460a..ca23ab3f13078a7a14c51b80139ac05d5908f0f8 100644
--- a/docs/Phabricator.rst
+++ b/docs/Phabricator.rst
@@ -1,3 +1,5 @@
+.. _phabricator-reviews:
+
 =============================
 Code Reviews with Phabricator
 =============================
@@ -16,7 +18,7 @@ to respond to free-form comments in mail sent to the commits list.
 Sign up
 -------
 
-To get started with Phabricator, navigate to `http://reviews.llvm.org`_ and
+To get started with Phabricator, navigate to `https://reviews.llvm.org`_ and
 click the power icon in the top right. You can register with a GitHub account,
 a Google account, or you can create your own profile.
 
@@ -151,7 +153,7 @@ ends with the line:
   Differential Revision: <URL>
 
 where ``<URL>`` is the URL for the code review, starting with
-``http://reviews.llvm.org/``.
+``https://reviews.llvm.org/``.
 
 This allows people reading the version history to see the review for
 context. This also allows Phabricator to detect the commit, close the
@@ -162,11 +164,11 @@ be added automatically. If you don't want to use Arcanist, you can add the
 ``Differential Revision`` line (as the last line) to the commit message
 yourself.
 
-Using the Arcanist tool can simplify the process of committing reviewed code
-as it will retrieve reviewers, the ``Differential Revision``, etc from the review
-and place it in the commit message. Several methods of using Arcanist to commit
-code are given below. If you do not wish to use Arcanist then simply commit
-the reviewed patch as you would normally.
+Using the Arcanist tool can simplify the process of committing reviewed code as
+it will retrieve reviewers, the ``Differential Revision``, etc from the review
+and place it in the commit message. You may also commit an accepted change
+directly using ``git llvm push``, per the section in the :ref:`getting started
+guide <commit_from_git>`.
 
 Note that if you commit the change without using Arcanist and forget to add the
 ``Differential Revision`` line to your commit message then it is recommended
@@ -174,24 +176,9 @@ that you close the review manually. In the web UI, under "Leap Into Action" put
 the SVN revision number in the Comment, set the Action to "Close Revision" and
 click Submit.  Note the review must have been Accepted first.
 
-Subversion and Arcanist
-^^^^^^^^^^^^^^^^^^^^^^^
-
-On a clean Subversion working copy run the following (where ``<Revision>`` is
-the Phabricator review number):
-
-::
-
-  arc patch D<Revision>
-  arc commit --revision D<Revision>
-
-The first command will take the latest version of the reviewed patch and apply it to the working
-copy. The second command will commit this revision to trunk.
 
-git-svn and Arcanist
-^^^^^^^^^^^^^^^^^^^^
-
-This presumes that the git repository has been configured as described in :ref:`developers-work-with-git-svn`.
+Committing someone's change from Phabricator
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 On a clean Git repository on an up to date ``master`` branch run the
 following (where ``<Revision>`` is the Phabricator review number):
@@ -205,17 +192,31 @@ This will create a new branch called ``arcpatch-D<Revision>`` based on the
 current ``master`` and will create a commit corresponding to ``D<Revision>`` with a
 commit message derived from information in the Phabricator review.
 
-Check you are happy with the commit message and amend it if necessary. Now switch to
-the ``master`` branch and add the new commit to it and commit it to trunk. This
-can be done by running the following:
+Check you are happy with the commit message and amend it if necessary. Then,
+make sure the commit is up-to-date, and commit it. This can be done by running
+the following:
 
 ::
 
-  git checkout master
-  git merge --ff-only arcpatch-D<Revision>
-  git svn dcommit
+  git pull --rebase origin master
+  git show # Ensure the patch looks correct.
+  ninja check-$whatever # Rerun the appropriate tests if needed.
+  git llvm push
+
+Subversion and Arcanist (deprecated)
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
+To download a change from Phabricator and commit it with subversion, you should
+first make sure you have a clean working directory. Then run the following
+(where ``<Revision>`` is the Phabricator review number):
+
+::
+
+  arc patch D<Revision>
+  arc commit --revision D<Revision>
 
+The first command will take the latest version of the reviewed patch and apply
+it to the working copy. The second command will commit this revision to trunk.
 
 Abandoning a change
 -------------------
@@ -241,9 +242,9 @@ requests. We're looking into what the right long-term hosting for this is, but
 note that it is a derivative of an existing open source project, and so not
 trivially a good fit for an official LLVM project.
 
-.. _LLVM's Phabricator: http://reviews.llvm.org
-.. _`http://reviews.llvm.org`: http://reviews.llvm.org
-.. _Code Repository Browser: http://reviews.llvm.org/diffusion/
+.. _LLVM's Phabricator: https://reviews.llvm.org
+.. _`https://reviews.llvm.org`: https://reviews.llvm.org
+.. _Code Repository Browser: https://reviews.llvm.org/diffusion/
 .. _Arcanist Quick Start: https://secure.phabricator.com/book/phabricator/article/arcanist_quick_start/
 .. _Arcanist User Guide: https://secure.phabricator.com/book/phabricator/article/arcanist/
 .. _llvm-reviews GitHub project: https://github.com/r4nt/llvm-reviews/
diff --git a/docs/ProgrammersManual.rst b/docs/ProgrammersManual.rst
index 88c56700eb36967cb821b950c1cc7ba855ae03e1..2f89d9baa30d61e5bf91f1a9fc80fbd7bee9d671 100644
--- a/docs/ProgrammersManual.rst
+++ b/docs/ProgrammersManual.rst
@@ -656,19 +656,21 @@ taken is to report them to the user so that the user can attempt to fix the
 environment. In this case representing the error as a string makes perfect
 sense. LLVM provides the ``StringError`` class for this purpose. It takes two
 arguments: A string error message, and an equivalent ``std::error_code`` for
-interoperability:
+interoperability. It also provides a ``createStringError`` function to simplify
+common usage of this class:
 
 .. code-block:: c++
 
-  make_error<StringError>("Bad executable",
-                          make_error_code(errc::executable_format_error"));
+  // These two lines of code are equivalent:
+  make_error<StringError>("Bad executable", errc::executable_format_error);
+  createStringError(errc::executable_format_error, "Bad executable");
 
 If you're certain that the error you're building will never need to be converted
 to a ``std::error_code`` you can use the ``inconvertibleErrorCode()`` function:
 
 .. code-block:: c++
 
-  make_error<StringError>("Bad executable", inconvertibleErrorCode());
+  createStringError(inconvertibleErrorCode(), "Bad executable");
 
 This should be done only after careful consideration. If any attempt is made to
 convert this error to a ``std::error_code`` it will trigger immediate program
@@ -677,6 +679,14 @@ interoperability you should look for an existing ``std::error_code`` that you
 can convert to, and even (as painful as it is) consider introducing a new one as
 a stopgap measure.
 
+``createStringError`` can take ``printf`` style format specifiers to provide a
+formatted message:
+
+.. code-block:: c++
+
+  createStringError(errc::executable_format_error,
+                    "Bad executable: %s", FileName);
+
 Interoperability with std::error_code and ErrorOr
 """""""""""""""""""""""""""""""""""""""""""""""""
 
@@ -925,28 +935,86 @@ Building fallible iterators and iterator ranges
 
 The archive walking examples above retrieve archive members by index, however
 this requires considerable boiler-plate for iteration and error checking. We can
-clean this up by using ``Error`` with the "fallible iterator" pattern. The usual
-C++ iterator patterns do not allow for failure on increment, but we can
-incorporate support for it by having iterators hold an Error reference through
-which they can report failure. In this pattern, if an increment operation fails
-the failure is recorded via the Error reference and the iterator value is set to
-the end of the range in order to terminate the loop. This ensures that the
-dereference operation is safe anywhere that an ordinary iterator dereference
-would be safe (i.e. when the iterator is not equal to end). Where this pattern
-is followed (as in the ``llvm::object::Archive`` class) the result is much
-cleaner iteration idiom:
+clean this up by using the "fallible iterator" pattern, which supports the
+following natural iteration idiom for fallible containers like Archive:
 
 .. code-block:: c++
 
   Error Err;
   for (auto &Child : Ar->children(Err)) {
-    // Use Child - we only enter the loop when it's valid
+    // Use Child - only enter the loop when it's valid
+
+    // Allow early exit from the loop body, since we know that Err is success
+    // when we're inside the loop.
+    if (BailOutOn(Child))
+      return;
+
     ...
   }
   // Check Err after the loop to ensure it didn't break due to an error.
   if (Err)
     return Err;
 
+To enable this idiom, iterators over fallible containers are written in a
+natural style, with their ``++`` and ``--`` operators replaced with fallible
+``Error inc()`` and ``Error dec()`` functions. E.g.:
+
+.. code-block:: c++
+
+  class FallibleChildIterator {
+  public:
+    FallibleChildIterator(Archive &A, unsigned ChildIdx);
+    Archive::Child &operator*();
+    friend bool operator==(const ArchiveIterator &LHS,
+                           const ArchiveIterator &RHS);
+
+    // operator++/operator-- replaced with fallible increment / decrement:
+    Error inc() {
+      if (!A.childValid(ChildIdx + 1))
+        return make_error<BadArchiveMember>(...);
+      ++ChildIdx;
+      return Error::success();
+    }
+
+    Error dec() { ... }
+  };
+
+Instances of this kind of fallible iterator interface are then wrapped with the
+fallible_iterator utility which provides ``operator++`` and ``operator--``,
+returning any errors via a reference passed in to the wrapper at construction
+time. The fallible_iterator wrapper takes care of (a) jumping to the end of the
+range on error, and (b) marking the error as checked whenever an iterator is
+compared to ``end`` and found to be inequal (in particular: this marks the
+error as checked throughout the body of a range-based for loop), enabling early
+exit from the loop without redundant error checking.
+
+Instances of the fallible iterator interface (e.g. FallibleChildIterator above)
+are wrapped using the ``make_fallible_itr`` and ``make_fallible_end``
+functions. E.g.:
+
+.. code-block:: c++
+
+  class Archive {
+  public:
+    using child_iterator = fallible_iterator<FallibleChildIterator>;
+
+    child_iterator child_begin(Error &Err) {
+      return make_fallible_itr(FallibleChildIterator(*this, 0), Err);
+    }
+
+    child_iterator child_end() {
+      return make_fallible_end(FallibleChildIterator(*this, size()));
+    }
+
+    iterator_range<child_iterator> children(Error &Err) {
+      return make_range(child_begin(Err), child_end());
+    }
+  };
+
+Using the fallible_iterator utility allows for both natural construction of
+fallible iterators (using failing ``inc`` and ``dec`` operations) and
+relatively natural use of c++ iterator/loop idioms.
+
 .. _function_apis:
 
 More information on Error and its related utilities can be found in the
@@ -1456,7 +1524,7 @@ SmallVector has grown a few other minor advantages over std::vector, causing
 #. std::vector is exception-safe, and some implementations have pessimizations
    that copy elements when SmallVector would move them.
 
-#. SmallVector understands ``isPodLike<Type>`` and uses realloc aggressively.
+#. SmallVector understands ``llvm::is_trivially_copyable<Type>`` and uses realloc aggressively.
 
 #. Many LLVM APIs take a SmallVectorImpl as an out parameter (see the note
    below).
@@ -2905,37 +2973,6 @@ For example:
   GV->eraseFromParent();
 
 
-.. _create_types:
-
-How to Create Types
--------------------
-
-In generating IR, you may need some complex types.  If you know these types
-statically, you can use ``TypeBuilder<...>::get()``, defined in
-``llvm/Support/TypeBuilder.h``, to retrieve them.  ``TypeBuilder`` has two forms
-depending on whether you're building types for cross-compilation or native
-library use.  ``TypeBuilder<T, true>`` requires that ``T`` be independent of the
-host environment, meaning that it's built out of types from the ``llvm::types``
-(`doxygen <http://llvm.org/doxygen/namespacellvm_1_1types.html>`__) namespace
-and pointers, functions, arrays, etc. built of those.  ``TypeBuilder<T, false>``
-additionally allows native C types whose size may depend on the host compiler.
-For example,
-
-.. code-block:: c++
-
-  FunctionType *ft = TypeBuilder<types::i<8>(types::i<32>*), true>::get();
-
-is easier to read and write than the equivalent
-
-.. code-block:: c++
-
-  std::vector<const Type*> params;
-  params.push_back(PointerType::getUnqual(Type::Int32Ty));
-  FunctionType *ft = FunctionType::get(Type::Int8Ty, params, false);
-
-See the `class comment
-<http://llvm.org/doxygen/TypeBuilder_8h_source.html#l00001>`_ for more details.
-
 .. _threading:
 
 Threads and LLVM
@@ -3522,11 +3559,17 @@ Important Public Members of the ``Module`` class
   Look up the specified function in the ``Module`` SymbolTable_.  If it does not
   exist, return ``null``.
 
-* ``Function *getOrInsertFunction(const std::string &Name, const FunctionType
-  *T)``
-
-  Look up the specified function in the ``Module`` SymbolTable_.  If it does not
-  exist, add an external declaration for the function and return it.
+* ``FunctionCallee getOrInsertFunction(const std::string &Name,
+  const FunctionType *T)``
+
+  Look up the specified function in the ``Module`` SymbolTable_.  If
+  it does not exist, add an external declaration for the function and
+  return it. Note that the function signature already present may not
+  match the requested signature. Thus, in order to enable the common
+  usage of passing the result directly to EmitCall, the return type is
+  a struct of ``{FunctionType *T, Constant *FunctionPtr}``, rather
+  than simply the ``Function*`` with potentially an unexpected
+  signature.
 
 * ``std::string getTypeName(const Type *Ty)``
 
diff --git a/docs/Proposals/GitHubMove.rst b/docs/Proposals/GitHubMove.rst
index f12077504dba0e6253523a38541e36bdba0c581f..93b00d505aa0d40ba36d561c845b995d7771dfd3 100644
--- a/docs/Proposals/GitHubMove.rst
+++ b/docs/Proposals/GitHubMove.rst
@@ -105,8 +105,7 @@ A single revision number (e.g. r123456) thus identifies a consistent version of
 all LLVM sub-projects.
 
 Git does not use sequential integer revision number but instead uses a hash to
-identify each commit. (Linus mentioned that the lack of such revision number
-is "the only real design mistake" in Git [TorvaldRevNum]_.)
+identify each commit.
 
 The loss of a sequential integer revision number has been a sticking point in
 past discussions about Git:
@@ -853,6 +852,540 @@ Also, since the monorepo handles commits update across multiple projects, you're
 less like to encounter a build failure where a commit change an API in LLVM and
 another later one "fixes" the build in clang.
 
+Moving Local Branches to the Monorepo
+=====================================
+
+Suppose you have been developing against the existing LLVM git
+mirrors.  You have one or more git branches that you want to migrate
+to the "final monorepo".
+
+The simplest way to migrate such branches is with the
+``migrate-downstream-fork.py`` tool at
+https://github.com/jyknight/llvm-git-migration.
+
+Basic migration
+---------------
+
+Basic instructions for ``migrate-downstream-fork.py`` are in the
+Python script and are expanded on below to a more general recipe::
+
+  # Make a repository which will become your final local mirror of the
+  # monorepo.
+  mkdir my-monorepo
+  git -C my-monorepo init
+
+  # Add a remote to the monorepo.
+  git -C my-monorepo remote add upstream/monorepo https://github.com/llvm/llvm-project.git
+
+  # Add remotes for each git mirror you use, from upstream as well as
+  # your local mirror.  All projects are listed here but you need only
+  # import those for which you have local branches.
+  my_projects=( clang
+                clang-tools-extra
+                compiler-rt
+                debuginfo-tests
+                libcxx
+                libcxxabi
+                libunwind
+                lld
+                lldb
+                llvm
+                openmp
+                polly )
+  for p in ${my_projects[@]}; do
+    git -C my-monorepo remote add upstream/split/${p} https://github.com/llvm-mirror/${p}.git
+    git -C my-monorepo remote add local/split/${p} https://my.local.mirror.org/${p}.git
+  done
+
+  # Pull in all the commits.
+  git -C my-monorepo fetch --all
+
+  # Run migrate-downstream-fork to rewrite local branches on top of
+  # the upstream monorepo.
+  (
+     cd my-monorepo
+     migrate-downstream-fork.py \
+       refs/remotes/local \
+       refs/tags \
+       --new-repo-prefix=refs/remotes/upstream/monorepo \
+       --old-repo-prefix=refs/remotes/upstream/split \
+       --source-kind=split \
+       --revmap-out=monorepo-map.txt
+  )
+
+  # Octopus-merge the resulting local split histories to unify them.
+
+  # Assumes local work on local split mirrors is on master (and
+  # upstream is presumably represented by some other branch like
+  # upstream/master).
+  my_local_branch="master"
+
+  git -C my-monorepo branch --no-track local/octopus/master \
+    $(git -C my-monorepo merge-base refs/remotes/upstream/monorepo/master \
+                                    refs/remotes/local/split/llvm/${my_local_branch})
+  git -C my-monorepo checkout local/octopus/${my_local_branch}
+
+  subproject_branches=()
+  for p in ${my_projects[@]}; do
+    subproject_branch=${p}/local/monorepo/${my_local_branch}
+    git -C my-monorepo branch ${subproject_branch} \
+      refs/remotes/local/split/${p}/${my_local_branch}
+    if [[ "${p}" != "llvm" ]]; then
+      subproject_branches+=( ${subproject_branch} )
+    fi
+  done
+
+  git -C my-monorepo merge ${subproject_branches[@]}
+
+  for p in ${my_projects[@]}; do
+    subproject_branch=${p}/local/monorepo/${my_local_branch}
+    git -C my-monorepo branch -d ${subproject_branch}
+  done
+
+  # Create local branches for upstream monorepo branches.
+  for ref in $(git -C my-monorepo for-each-ref --format="%(refname)" \
+                   refs/remotes/upstream/monorepo); do
+    upstream_branch=${ref#refs/remotes/upstream/monorepo/}
+    git -C my-monorepo branch upstream/${upstream_branch} ${ref}
+  done
+
+The above gets you to a state like the following::
+
+  U1 - U2 - U3 <- upstream/master
+    \   \    \
+     \   \    - Llld1 - Llld2 -
+      \   \                    \
+       \   - Lclang1 - Lclang2-- Lmerge <- local/octopus/master
+        \                      /
+         - Lllvm1 - Lllvm2-----
+
+Each branched component has its branch rewritten on top of the
+monorepo and all components are unified by a giant octopus merge.
+
+If additional active local branches need to be preserved, the above
+operations following the assignment to ``my_local_branch`` should be
+done for each branch.  Ref paths will need to be updated to map the
+local branch to the corresponding upstream branch.  If local branches
+have no corresponding upstream branch, then the creation of
+``local/octopus/<local branch>`` need not use ``git-merge-base`` to
+pinpont its root commit; it may simply be branched from the
+appropriate component branch (say, ``llvm/local_release_X``).
+
+Zipping local history
+---------------------
+
+The octopus merge is suboptimal for many cases, because walking back
+through the history of one component leaves the other components fixed
+at a history that likely makes things unbuildable.
+
+Some downstream users track the order commits were made to subprojects
+with some kind of "umbrella" project that imports the project git
+mirrors as submodules, similar to the multirepo umbrella proposed
+above.  Such an umbrella repository looks something like this::
+
+   UM1 ---- UM2 -- UM3 -- UM4 ---- UM5 ---- UM6 ---- UM7 ---- UM8 <- master
+   |        |             |        |        |        |        |
+  Lllvm1   Llld1         Lclang1  Lclang2  Lllvm2   Llld2     Lmyproj1
+
+The vertical bars represent submodule updates to a particular local
+commit in the project mirror.  ``UM3`` in this case is a commit of
+some local umbrella repository state that is not a submodule update,
+perhaps a ``README`` or project build script update.  Commit ``UM8``
+updates a submodule of local project ``myproj``.
+
+The tool ``zip-downstream-fork.py`` at
+https://github.com/greened/llvm-git-migration/tree/zip can be used to
+convert the umbrella history into a monorepo-based history with
+commits in the order implied by submodule updates::
+
+  U1 - U2 - U3 <- upstream/master
+   \    \    \
+    \    -----\---------------                                    local/zip--.
+     \         \              \                                               |
+    - Lllvm1 - Llld1 - UM3 -  Lclang1 - Lclang2 - Lllvm2 - Llld2 - Lmyproj1 <-'
+
+
+The ``U*`` commits represent upstream commits to the monorepo master
+branch.  Each submodule update in the local ``UM*`` commits brought in
+a subproject tree at some local commit.  The trees in the ``L*1``
+commits represent merges from upstream.  These result in edges from
+the ``U*`` commits to their corresponding rewritten ``L*1`` commits.
+The ``L*2`` commits did not do any merges from upstream.
+
+Note that the merge from ``U2`` to ``Lclang1`` appears redundant, but
+if, say, ``U3`` changed some files in upstream clang, the ``Lclang1``
+commit appearing after the ``Llld1`` commit would actually represent a
+clang tree *earlier* in the upstream clang history.  We want the
+``local/zip`` branch to accurately represent the state of our umbrella
+history and so the edge ``U2 -> Lclang1`` is a visual reminder of what
+clang's tree actually looks like in ``Lclang1``.
+
+Even so, the edge ``U3 -> Llld1`` could be problematic for future
+merges from upstream.  git will think that we've already merged from
+``U3``, and we have, except for the state of the clang tree.  One
+possible migitation strategy is to manually diff clang between ``U2``
+and ``U3`` and apply those updates to ``local/zip``.  Another,
+possibly simpler strategy is to freeze local work on downstream
+branches and merge all submodules from the latest upstream before
+running ``zip-downstream-fork.py``.  If downstream merged each project
+from upstream in lockstep without any intervening local commits, then
+things should be fine without any special action.  We anticipate this
+to be the common case.
+
+The tree for ``Lclang1`` outside of clang will represent the state of
+things at ``U3`` since all of the upstream projects not participating
+in the umbrella history should be in a state respecting the commit
+``U3``.  The trees for llvm and lld should correctly represent commits
+``Lllvm1`` and ``Llld1``, respectively.
+
+Commit ``UM3`` changed files not related to submodules and we need
+somewhere to put them.  It is not safe in general to put them in the
+monorepo root directory because they may conflict with files in the
+monorepo.  Let's assume we want them in a directory ``local`` in the
+monorepo.
+
+**Example 1: Umbrella looks like the monorepo**
+
+For this example, we'll assume that each subproject appears in its own
+top-level directory in the umbrella, just as they do in the monorepo .
+Let's also assume that we want the files in directory ``myproj`` to
+appear in ``local/myproj``.
+
+Given the above run of ``migrate-downstream-fork.py``, a recipe to
+create the zipped history is below::
+
+  # Import any non-LLVM repositories the umbrella references.
+  git -C my-monorepo remote add localrepo \
+                                https://my.local.mirror.org/localrepo.git
+  git fetch localrepo
+
+  subprojects=( clang clang-tools-extra compiler-rt debuginfo-tests libclc
+                libcxx libcxxabi libunwind lld lldb llgo llvm openmp
+                parallel-libs polly pstl )
+
+  # Import histories for upstream split projects (this was probably
+  # already done for the ``migrate-downstream-fork.py`` run).
+  for project in ${subprojects[@]}; do
+    git remote add upstream/split/${project} \
+                   https://github.com/llvm-mirror/${subproject}.git
+    git fetch umbrella/split/${project}
+  done
+
+  # Import histories for downstream split projects (this was probably
+  # already done for the ``migrate-downstream-fork.py`` run).
+  for project in ${subprojects[@]}; do
+    git remote add local/split/${project} \
+                   https://my.local.mirror.org/${subproject}.git
+    git fetch local/split/${project}
+  done
+
+  # Import umbrella history.
+  git -C my-monorepo remote add umbrella \
+                                https://my.local.mirror.org/umbrella.git
+  git fetch umbrella
+
+  # Put myproj in local/myproj
+  echo "myproj local/myproj" > my-monorepo/submodule-map.txt
+
+  # Rewrite history
+  (
+    cd my-monorepo
+    zip-downstream-fork.py \
+      refs/remotes/umbrella \
+      --new-repo-prefix=refs/remotes/upstream/monorepo \
+      --old-repo-prefix=refs/remotes/upstream/split \
+      --revmap-in=monorepo-map.txt \
+      --revmap-out=zip-map.txt \
+      --subdir=local \
+      --submodule-map=submodule-map.txt \
+      --update-tags
+   )
+
+   # Create the zip branch (assuming umbrella master is wanted).
+   git -C my-monorepo branch --no-track local/zip/master refs/remotes/umbrella/master
+
+Note that if the umbrella has submodules to non-LLVM repositories,
+``zip-downstream-fork.py`` needs to know about them to be able to
+rewrite commits.  That is why the first step above is to fetch commits
+from such repositories.
+
+With ``--update-tags`` the tool will migrate annotated tags pointing
+to submodule commits that were inlined into the zipped history.  If
+the umbrella pulled in an upstream commit that happened to have a tag
+pointing to it, that tag will be migrated, which is almost certainly
+not what is wanted.  The tag can always be moved back to its original
+commit after rewriting, or the ``--update-tags`` option may be
+discarded and any local tags would then be migrated manually.
+
+**Example 2: Nested sources layout**
+
+The tool handles nested submodules (e.g. llvm is a submodule in
+umbrella and clang is a submodule in llvm).  The file
+``submodule-map.txt`` is a list of pairs, one per line.  The first
+pair item describes the path to a submodule in the umbrella
+repository.  The second pair item secribes the path where trees for
+that submodule should be written in the zipped history.  
+
+Let's say your umbrella repository is actually the llvm repository and
+it has submodules in the "nested sources" layout (clang in
+tools/clang, etc.).  Let's also say ``projects/myproj`` is a submodule
+pointing to some downstream repository.  The submodule map file should
+look like this (we still want myproj mapped the same way as
+previously)::
+
+  tools/clang clang
+  tools/clang/tools/extra clang-tools-extra
+  projects/compiler-rt compiler-rt
+  projects/debuginfo-tests debuginfo-tests
+  projects/libclc libclc
+  projects/libcxx libcxx
+  projects/libcxxabi libcxxabi
+  projects/libunwind libunwind
+  tools/lld lld
+  tools/lldb lldb
+  projects/openmp openmp
+  tools/polly polly
+  projects/myproj local/myproj
+
+If a submodule path does not appear in the map, the tools assumes it
+should be placed in the same place in the monorepo.  That means if you
+use the "nested sources" layout in your umrella, you *must* provide
+map entries for all of the projects in your umbrella (except llvm).
+Otherwise trees from submodule updates will appear underneath llvm in
+the zippped history.
+
+Because llvm is itself the umbrella, we use --subdir to write its
+content into ``llvm`` in the zippped history::
+
+  # Import any non-LLVM repositories the umbrella references.
+  git -C my-monorepo remote add localrepo \
+                                https://my.local.mirror.org/localrepo.git
+  git fetch localrepo
+
+  subprojects=( clang clang-tools-extra compiler-rt debuginfo-tests libclc
+                libcxx libcxxabi libunwind lld lldb llgo llvm openmp
+                parallel-libs polly pstl )
+
+  # Import histories for upstream split projects (this was probably
+  # already done for the ``migrate-downstream-fork.py`` run).
+  for project in ${subprojects[@]}; do
+    git remote add upstream/split/${project} \
+                   https://github.com/llvm-mirror/${subproject}.git
+    git fetch umbrella/split/${project}
+  done
+
+  # Import histories for downstream split projects (this was probably
+  # already done for the ``migrate-downstream-fork.py`` run).
+  for project in ${subprojects[@]}; do
+    git remote add local/split/${project} \
+                   https://my.local.mirror.org/${subproject}.git
+    git fetch local/split/${project}
+  done
+
+  # Import umbrella history.  We want this under a different refspec
+  # so zip-downstream-fork.py knows what it is.
+  git -C my-monorepo remote add umbrella \
+                                 https://my.local.mirror.org/llvm.git
+  git fetch umbrella
+
+  # Create the submodule map.
+  echo "tools/clang clang" > my-monorepo/submodule-map.txt
+  echo "tools/clang/tools/extra clang-tools-extra" >> my-monorepo/submodule-map.txt
+  echo "projects/compiler-rt compiler-rt" >> my-monorepo/submodule-map.txt
+  echo "projects/debuginfo-tests debuginfo-tests" >> my-monorepo/submodule-map.txt
+  echo "projects/libclc libclc" >> my-monorepo/submodule-map.txt
+  echo "projects/libcxx libcxx" >> my-monorepo/submodule-map.txt
+  echo "projects/libcxxabi libcxxabi" >> my-monorepo/submodule-map.txt
+  echo "projects/libunwind libunwind" >> my-monorepo/submodule-map.txt
+  echo "tools/lld lld" >> my-monorepo/submodule-map.txt
+  echo "tools/lldb lldb" >> my-monorepo/submodule-map.txt
+  echo "projects/openmp openmp" >> my-monorepo/submodule-map.txt
+  echo "tools/polly polly" >> my-monorepo/submodule-map.txt
+  echo "projects/myproj local/myproj" >> my-monorepo/submodule-map.txt
+
+  # Rewrite history
+  (
+    cd my-monorepo
+    zip-downstream-fork.py \
+      refs/remotes/umbrella \
+      --new-repo-prefix=refs/remotes/upstream/monorepo \
+      --old-repo-prefix=refs/remotes/upstream/split \
+      --revmap-in=monorepo-map.txt \
+      --revmap-out=zip-map.txt \
+      --subdir=llvm \
+      --submodule-map=submodule-map.txt \
+      --update-tags
+   )
+
+   # Create the zip branch (assuming umbrella master is wanted).
+   git -C my-monorepo branch --no-track local/zip/master refs/remotes/umbrella/master
+
+
+Comments at the top of ``zip-downstream-fork.py`` describe in more
+detail how the tool works and various implications of its operation.
+
+Importing local repositories
+----------------------------
+
+You may have additional repositories that integrate with the LLVM
+ecosystem, essentially extending it with new tools.  If such
+repositories are tightly coupled with LLVM, it may make sense to
+import them into your local mirror of the monorepo.
+
+If such repositores participated in the umbrella repository used
+during the zipping process above, they will automatically be added to
+the monorepo.  For downstream repositories that don't participate in
+an umbrella setup, the ``import-downstream-repo.py`` tool at
+https://github.com/greened/llvm-git-migration/tree/import can help with
+getting them into the monorepo.  A recipe follows::
+
+  # Import downstream repo history into the monorepo.
+  git -C my-monorepo remote add myrepo https://my.local.mirror.org/myrepo.git
+  git fetch myrepo
+
+  my_local_tags=( refs/tags/release
+                  refs/tags/hotfix )
+
+  (
+    cd my-monorepo
+    import-downstream-repo.py \
+      refs/remotes/myrepo \
+      ${my_local_tags[@]} \
+      --new-repo-prefix=refs/remotes/upstream/monorepo \
+      --subdir=myrepo \
+      --tag-prefix="myrepo-"
+   )
+
+   # Preserve release braches.
+   for ref in $(git -C my-monorepo for-each-ref --format="%(refname)" \
+                  refs/remotes/myrepo/release); do
+     branch=${ref#refs/remotes/myrepo/}
+     git -C my-monorepo branch --no-track myrepo/${branch} ${ref}
+   done
+
+   # Preserve master.
+   git -C my-monorepo branch --no-track myrepo/master refs/remotes/myrepo/master
+
+   # Merge master.
+   git -C my-monorepo checkout local/zip/master  # Or local/octopus/master
+   git -C my-monorepo merge myrepo/master
+
+You may want to merge other corresponding branches, for example
+``myrepo`` release branches if they were in lockstep with LLVM project
+releases.
+
+``--tag-prefix`` tells ``import-downstream-repo.py`` to rename
+annotated tags with the given prefix.  Due to limitations with
+``fast_filter_branch.py``, unannotated tags cannot be renamed
+(``fast_filter_branch.py`` considers them branches, not tags).  Since
+the upstream monorepo had its tags rewritten with an "llvmorg-"
+prefix, name conflicts should not be an issue.  ``--tag-prefix`` can
+be used to more clearly indicate which tags correspond to various
+imported repositories.
+
+Given this repository history::
+
+  R1 - R2 - R3 <- master
+       ^
+       |
+    release/1
+
+The above recipe results in a history like this::
+
+  U1 - U2 - U3 <- upstream/master
+   \    \    \
+    \    -----\---------------                                         local/zip--.
+     \         \              \                                                    |
+    - Lllvm1 - Llld1 - UM3 -  Lclang1 - Lclang2 - Lllvm2 - Llld2 - Lmyproj1 - M1 <-'
+                                                                             /
+                                                                 R1 - R2 - R3  <-.
+                                                                      ^           |
+                                                                      |           |
+                                                               myrepo-release/1   |
+                                                                                  |
+                                                                   myrepo/master--'
+
+Commits ``R1``, ``R2`` and ``R3`` have trees that *only* contain blobs
+from ``myrepo``.  If you require commits from ``myrepo`` to be
+interleaved with commits on local project branches (for example,
+interleaved with ``llvm1``, ``llvm2``, etc. above) and myrepo doesn't
+appear in an umbrella repository, a new tool will need to be
+developed.  Creating such a tool would involve:
+
+1. Modifying ``fast_filter_branch.py`` to optionally take a
+   revlist directly rather than generating it itself
+
+2. Creating a tool to generate an interleaved ordering of local
+   commits based on some criteria (``zip-downstream-fork.py`` uses the
+   umbrella history as its criterion)
+
+3. Generating such an ordering and feeding it to
+   ``fast_filter_branch.py`` as a revlist
+
+Some care will also likely need to be taken to handle merge commits,
+to ensure the parents of such commits migrate correctly.
+
+Scrubbing the Local Monorepo
+----------------------------
+
+Once all of the migrating, zipping and importing is done, it's time to
+clean up.  The python tools use ``git-fast-import`` which leaves a lot
+of cruft around and we want to shrink our new monorepo mirror as much
+as possible.  Here is one way to do it::
+
+  git -C my-monorepo checkout master
+
+  # Delete branches we no longer need.  Do this for any other branches
+  # you merged above.
+  git -C my-monorepo branch -D local/zip/master || true
+  git -C my-monorepo branch -D local/octopus/master || true
+
+  # Remove remotes.
+  git -C my-monorepo remote remove upstream/monorepo
+
+  for p in ${my_projects[@]}; do
+    git -C my-monorepo remote remove upstream/split/${p}
+    git -C my-monorepo remote remove local/split/${p}
+  done
+
+  git -C my-monorepo remote remove localrepo
+  git -C my-monorepo remote remove umbrella
+  git -C my-monorepo remote remove myrepo
+
+  # Add anything else here you don't need.  refs/tags/release is
+  # listed below assuming tags have been rewritten with a local prefix.
+  # If not, remove it from this list.
+  refs_to_clean=(
+    refs/original
+    refs/remotes
+    refs/tags/backups
+    refs/tags/release
+  )
+
+  git -C my-monorepo for-each-ref --format="%(refname)" ${refs_to_clean[@]} |
+    xargs -n1 --no-run-if-empty git -C my-monorepo update-ref -d
+
+  git -C my-monorepo reflog expire --all --expire=now
+
+  # fast_filter_branch.py might have gc running in the background.
+  while ! git -C my-monorepo \
+    -c gc.reflogExpire=0 \
+    -c gc.reflogExpireUnreachable=0 \
+    -c gc.rerereresolved=0 \
+    -c gc.rerereunresolved=0 \
+    -c gc.pruneExpire=now \
+    gc --prune=now; do
+    continue
+  done
+
+  # Takes a LOOOONG time!
+  git -C my-monorepo repack -A -d -f --depth=250 --window=250
+
+  git -C my-monorepo prune-packed
+  git -C my-monorepo prune
+
+You should now have a trim monorepo.  Upload it to your git server and
+happy hacking!
 
 References
 ==========
@@ -860,7 +1393,6 @@ References
 .. [LattnerRevNum] Chris Lattner, http://lists.llvm.org/pipermail/llvm-dev/2011-July/041739.html
 .. [TrickRevNum] Andrew Trick, http://lists.llvm.org/pipermail/llvm-dev/2011-July/041721.html
 .. [JSonnRevNum] Joerg Sonnenberg, http://lists.llvm.org/pipermail/llvm-dev/2011-July/041688.html
-.. [TorvaldRevNum] Linus Torvald, http://git.661346.n2.nabble.com/Git-commit-generation-numbers-td6584414.html
 .. [MatthewsRevNum] Chris Matthews, http://lists.llvm.org/pipermail/cfe-dev/2016-July/049886.html
 .. [submodules] Git submodules, https://git-scm.com/book/en/v2/Git-Tools-Submodules)
 .. [statuschecks] GitHub status-checks, https://help.github.com/articles/about-required-status-checks/
diff --git a/docs/ReleaseNotes.rst b/docs/ReleaseNotes.rst
index a3500ba3f743f502929ab726c5c7b36792af1aeb..52a8b11f3c5cae2e8e8df6f455d5f46aa22423a0 100644
--- a/docs/ReleaseNotes.rst
+++ b/docs/ReleaseNotes.rst
@@ -1,12 +1,12 @@
 ========================
-LLVM 8.0.0 Release Notes
+LLVM 9.0.0 Release Notes
 ========================
 
 .. contents::
     :local:
 
 .. warning::
-   These are in-progress notes for the upcoming LLVM 8 release.
+   These are in-progress notes for the upcoming LLVM 9 release.
    Release notes for previous releases can be found on
    `the Download Page <https://releases.llvm.org/download.html>`_.
 
@@ -15,7 +15,7 @@ Introduction
 ============
 
 This document contains the release notes for the LLVM Compiler Infrastructure,
-release 8.0.0.  Here we describe the status of LLVM, including major improvements
+release 9.0.0.  Here we describe the status of LLVM, including major improvements
 from the previous release, improvements in various subprojects of LLVM, and
 some of the current users of the code.  All LLVM releases may be downloaded
 from the `LLVM releases web site <https://llvm.org/releases/>`_.
@@ -40,9 +40,6 @@ Non-comprehensive list of changes in this release
    functionality, or simply have a lot to talk about), see the `NOTE` below
    for adding a new subsection.
 
-* The **llvm-cov** tool can now export lcov trace files using the
-  `-format=lcov` option of the `export` command.
-
 .. NOTE
    If you would like to document a larger change, then you can add a
    subsection about it right here. You can copy the following boilerplate
@@ -89,6 +86,12 @@ Changes to the AVR Target
 
  During this release ...
 
+Changes to the WebAssembly Target
+---------------------------------
+
+ During this release ...
+
+
 Changes to the OCaml bindings
 -----------------------------
 
@@ -101,7 +104,7 @@ Changes to the C API
 Changes to the DAG infrastructure
 ---------------------------------
 
-External Open Source Projects Using LLVM 8
+External Open Source Projects Using LLVM 9
 ==========================================
 
 * A project...
diff --git a/docs/ScudoHardenedAllocator.rst b/docs/ScudoHardenedAllocator.rst
index fcd5cefdac6d10ff0ea5ad373358c11aa71fb7c7..185c28f05f8ce13386a21e0ddedacf1aa22ff7a6 100644
--- a/docs/ScudoHardenedAllocator.rst
+++ b/docs/ScudoHardenedAllocator.rst
@@ -108,7 +108,7 @@ functions.
 
 You may also build Scudo like this: 
 
-.. code::
+.. code:: console
 
   cd $LLVM/projects/compiler-rt/lib
   clang++ -fPIC -std=c++11 -msse4.2 -O2 -I. scudo/*.cpp \
@@ -117,7 +117,7 @@ You may also build Scudo like this:
 
 and then use it with existing binaries as follows:
 
-.. code::
+.. code:: console
 
   LD_PRELOAD=`pwd`/libscudo.so ./a.out
 
@@ -151,7 +151,7 @@ can be assigned in the same string, separated by colons.
 
 For example, using the environment variable:
 
-.. code::
+.. code:: console
 
   SCUDO_OPTIONS="DeleteSizeMismatch=1:QuarantineSizeKb=64" ./a.out
 
@@ -201,3 +201,53 @@ Allocator related common Sanitizer options can also be passed through Scudo
 options, such as ``allocator_may_return_null`` or ``abort_on_error``. A detailed
 list including those can be found here:
 https://github.com/google/sanitizers/wiki/SanitizerCommonFlags.
+
+Error Types
+===========
+
+The allocator will output an error message, and potentially terminate the
+process, when an unexpected behavior is detected. The output usually starts with
+``"Scudo ERROR:"`` followed by a short summary of the problem that occurred as
+well as the pointer(s) involved. Once again, Scudo is meant to be a mitigation,
+and might not be the most useful of tools to help you root-cause the issue,
+please consider `ASan <https://github.com/google/sanitizers/wiki/AddressSanitizer>`_
+for this purpose.
+
+Here is a list of the current error messages and their potential cause:
+
+- ``"corrupted chunk header"``: the checksum verification of the chunk header
+  has failed. This is likely due to one of two things: the header was
+  overwritten (partially or totally), or the pointer passed to the function is
+  not a chunk at all;
+
+- ``"race on chunk header"``: two different threads are attempting to manipulate
+  the same header at the same time. This is usually symptomatic of a
+  race-condition or general lack of locking when performing operations on that
+  chunk;
+
+- ``"invalid chunk state"``: the chunk is not in the expected state for a given
+  operation, eg: it is not allocated when trying to free it, or it's not
+  quarantined when trying to recycle it, etc. A double-free is the typical
+  reason this error would occur;
+
+- ``"misaligned pointer"``: we strongly enforce basic alignment requirements, 8
+  bytes on 32-bit platforms, 16 bytes on 64-bit platforms. If a pointer passed
+  to our functions does not fit those, something is definitely wrong.
+
+- ``"allocation type mismatch"``: when the optional deallocation type mismatch
+  check is enabled, a deallocation function called on a chunk has to match the
+  type of function that was called to allocate it. Security implications of such
+  a mismatch are not necessarily obvious but situational at best;
+
+- ``"invalid sized delete"``: when the C++14 sized delete operator is used, and
+  the optional check enabled, this indicates that the size passed when
+  deallocating a chunk is not congruent with the one requested when allocating
+  it. This is likely to be a `compiler issue <https://software.intel.com/en-us/forums/intel-c-compiler/topic/783942>`_,
+  as was the case with Intel C++ Compiler, or some type confusion on the object
+  being deallocated;
+
+- ``"RSS limit exhausted"``: the maximum RSS optionally specified has been
+  exceeded;
+
+Several other error messages relate to parameter checking on the libc allocation
+APIs and are fairly straightforward to understand.
diff --git a/docs/SourceLevelDebugging.rst b/docs/SourceLevelDebugging.rst
index dab300ac7b76078843ecf5fbe22c6b0821a9cfe8..1a5ed2f2ad02156a456c5d2c16c863a82149c919 100644
--- a/docs/SourceLevelDebugging.rst
+++ b/docs/SourceLevelDebugging.rst
@@ -478,7 +478,7 @@ a C/C++ front-end would generate the following descriptors:
 
   ;; Define the compile unit.
   !1 = distinct !DICompileUnit(language: DW_LANG_C99, file: !2,
-                               producer: "clang version 4.0.0 (http://llvm.org/git/clang.git ae4deadbea242e8ea517eef662c30443f75bd086) (http://llvm.org/git/llvm.git 818b4c1539df3e51dc7e62c89ead4abfd348827d)",
+                               producer: "clang version 4.0.0",
                                isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug,
                                enums: !3, globals: !4)
 
@@ -506,7 +506,7 @@ a C/C++ front-end would generate the following descriptors:
   !7 = !{i32 2, !"Debug Info Version", i32 3}
 
   ;; Compiler identification
-  !8 = !{!"clang version 4.0.0 (http://llvm.org/git/clang.git ae4deadbea242e8ea517eef662c30443f75bd086) (http://llvm.org/git/llvm.git 818b4c1539df3e51dc7e62c89ead4abfd348827d)"}
+  !8 = !{!"clang version 4.0.0"}
 
 
 The align value in DIGlobalVariable description specifies variable alignment in
diff --git a/docs/SpeculativeLoadHardening.md b/docs/SpeculativeLoadHardening.md
index 0911b7c6ae23b7193e8cadb260618467546b39fd..de6dc015c570487cc939472be8e4da98c426740f 100644
--- a/docs/SpeculativeLoadHardening.md
+++ b/docs/SpeculativeLoadHardening.md
@@ -255,7 +255,7 @@ if (untrusted_size_from_caller < sizeof(local_buffer)) {
   memcpy(local_buffer, untrusted_data_from_caller,
          untrusted_size_from_caller);
   // The stack has now been smashed, writing an attacker-controlled
-  // address over the return adress.
+  // address over the return address.
   minor_processing(local_buffer);
   return;
   // Control will speculate to the attacker-written address.
diff --git a/docs/TableGen/LangIntro.rst b/docs/TableGen/LangIntro.rst
index ea46550ffc03a1f0bc7e77f8e7b8edc21b516cee..dcb03f58f793dae5c8957a979f635e1c89a04746 100644
--- a/docs/TableGen/LangIntro.rst
+++ b/docs/TableGen/LangIntro.rst
@@ -258,6 +258,16 @@ supported include:
 ``!if(a,b,c)``
   'b' if the result of 'int' or 'bit' operator 'a' is nonzero, 'c' otherwise.
 
+``!cond(condition_1 : val1, condition_2 : val2, ..., condition_n : valn)``
+    Instead of embedding !if inside !if which can get cumbersome,
+    one can use !cond. !cond returns 'val1' if the result of 'int' or 'bit'
+    operator 'condition1' is nonzero. Otherwise, it checks 'condition2'.
+    If 'condition2' is nonzero, returns 'val2', and so on.
+    If all conditions are zero, it reports an error.  
+
+    For example, to convert an integer 'x' into a string:
+      !cond(!lt(x,0) : "negative", !eq(x,0) : "zero", 1 : "positive")
+
 ``!eq(a,b)``
     'bit 1' if string a is equal to string b, 0 otherwise.  This only operates
     on string, int and bit objects.  Use !cast<string> to compare other types of
diff --git a/docs/TableGen/LangRef.rst b/docs/TableGen/LangRef.rst
index 2efee12ec9d5b991814cde6bafba0be76eaa96c5..a3dbf363151fe8f0af023859c321e1f0a8c5defe 100644
--- a/docs/TableGen/LangRef.rst
+++ b/docs/TableGen/LangRef.rst
@@ -102,6 +102,12 @@ wide variety of meanings:
                :!isa    !dag     !le      !lt        !ge
                :!gt     !ne
 
+TableGen also has !cond operator that needs a slightly different
+syntax compared to other "bang operators":
+
+.. productionlist::
+   CondOperator: !cond
+
 
 Syntax
 ======
@@ -140,7 +146,7 @@ considered to define the class if any of the following is true:
 #. The :token:`Body` in the :token:`ObjectBody` is present and is not empty.
 #. The :token:`BaseClassList` in the :token:`ObjectBody` is present.
 
-You can declare an empty class by giving and empty :token:`TemplateArgList`
+You can declare an empty class by giving an empty :token:`TemplateArgList`
 and an empty :token:`ObjectBody`. This can serve as a restricted form of
 forward declaration: note that records deriving from the forward-declared
 class will inherit no fields from it since the record expansion is done
@@ -315,6 +321,8 @@ The initial :token:`DagArg` is called the "operator" of the dag.
 
 .. productionlist::
    SimpleValue: `BangOperator` ["<" `Type` ">"] "(" `ValueListNE` ")"
+              :| `CondOperator` "(" `CondVal` ("," `CondVal`)* ")"
+   CondVal: `Value` ":" `Value`
 
 Bodies
 ------
diff --git a/docs/TestSuiteGuide.md b/docs/TestSuiteGuide.md
index fe7a2785c857016fcdde74571d188fcf9d07a793..d23b383a8d6c6f0ddaa344b4d3d0d10d43598157 100644
--- a/docs/TestSuiteGuide.md
+++ b/docs/TestSuiteGuide.md
@@ -17,7 +17,7 @@ Quickstart
 
    ```bash
    % mkdir venv
-   % virtualenv -p python2.7 venv
+   % virtualenv venv
    % . venv/bin/activate
    % pip install svn+http://llvm.org/svn/llvm-project/llvm/trunk/utils/lit
    % lit --version
@@ -27,7 +27,7 @@ Quickstart
 2. Check out the `test-suite` module with:
 
    ```bash
-   % svn co http://llvm.org/svn/llvm-project/test-suite/trunk test-suite
+   % git clone https://github.com/llvm/llvm-test-suite.git test-suite
    ```
 
 3. Create a build directory and use CMake to configure the suite. Use the
diff --git a/docs/TestSuiteMakefileGuide.rst b/docs/TestSuiteMakefileGuide.rst
index dfd12d72f678acf969f209a63bcd4403da25532f..246da5522a65c8a202ed26f623dac78cd0d89b50 100644
--- a/docs/TestSuiteMakefileGuide.rst
+++ b/docs/TestSuiteMakefileGuide.rst
@@ -14,14 +14,14 @@ the test suite creates temporary files during execution.
 
 To run the test suite, you need to use the following steps:
 
-#. ``cd`` into the ``llvm/projects`` directory in your source tree.
 #. Check out the ``test-suite`` module with:
 
    .. code-block:: bash
 
-       % svn co http://llvm.org/svn/llvm-project/test-suite/trunk test-suite
+       % git clone https://github.com/llvm/llvm-test-suite.git test-suite
 
-   This will get the test suite into ``llvm/projects/test-suite``.
+#. FIXME: these directions are outdated and won't work. Figure out
+   what the correct thing to do is, and write it down here.
 
 #. Configure and build ``llvm``.
 
diff --git a/docs/TestingGuide.rst b/docs/TestingGuide.rst
index e3db60dd9e788de898bf97c9d2f96e82e961db2d..949fecf4ccfcff5a12d276d5aa6ba8dfc43be701 100644
--- a/docs/TestingGuide.rst
+++ b/docs/TestingGuide.rst
@@ -29,10 +29,11 @@ later.
 LLVM Testing Infrastructure Organization
 ========================================
 
-The LLVM testing infrastructure contains two major categories of tests:
-regression tests and whole programs. The regression tests are contained
-inside the LLVM repository itself under ``llvm/test`` and are expected
-to always pass -- they should be run before every commit.
+The LLVM testing infrastructure contains three major categories of tests:
+unit tests, regression tests and whole programs. The unit tests and regression
+tests are contained inside the LLVM repository itself under ``llvm/unittests``
+and ``llvm/test`` respectively and are expected to always pass -- they should be
+run before every commit.
 
 The whole programs tests are referred to as the "LLVM test suite" (or
 "test-suite") and are in the ``test-suite`` module in subversion. For
@@ -40,6 +41,13 @@ historical reasons, these tests are also referred to as the "nightly
 tests" in places, which is less ambiguous than "test-suite" and remains
 in use although we run them much more often than nightly.
 
+Unit tests
+----------
+
+Unit tests are written using `Google Test <https://github.com/google/googletest/blob/master/googletest/docs/primer.md>`_
+and `Google Mock <https://github.com/google/googletest/blob/master/googlemock/docs/ForDummies.md>`_
+and are located in the ``llvm/unittests`` directory.
+
 Regression tests
 ----------------
 
@@ -90,16 +98,23 @@ test suite for more information . This test suite is located in the
 Quick start
 ===========
 
-The tests are located in two separate Subversion modules. The
-regressions tests are in the main "llvm" module under the directory
-``llvm/test`` (so you get these tests for free with the main LLVM tree).
-Use ``make check-all`` to run the regression tests after building LLVM.
+The tests are located in two separate Subversion modules. The unit and
+regression tests are in the main "llvm" module under the directories
+``llvm/unittests`` and ``llvm/test`` (so you get these tests for free with the
+main LLVM tree). Use ``make check-all`` to run the unit and regression tests
+after building LLVM.
 
 The ``test-suite`` module contains more comprehensive tests including whole C
 and C++ programs. See the :doc:`TestSuiteGuide` for details.
 
-Regression tests
-----------------
+Unit and Regression tests
+-------------------------
+
+To run all of the LLVM unit tests use the check-llvm-unit target:
+
+.. code-block:: bash
+
+    % make check-llvm-unit
 
 To run all of the LLVM regression tests use the check-llvm target:
 
@@ -107,6 +122,13 @@ To run all of the LLVM regression tests use the check-llvm target:
 
     % make check-llvm
 
+In order to get reasonable testing performance, build LLVM and subprojects
+in release mode, i.e.
+
+.. code-block:: bash
+
+    % cmake -DCMAKE_BUILD_TYPE="Release" -DLLVM_ENABLE_ASSERTIONS=On
+
 If you have `Clang <http://clang.llvm.org/>`_ checked out and built, you
 can run the LLVM and Clang tests simultaneously using:
 
@@ -143,15 +165,9 @@ or the :doc:`lit man page <CommandGuide/lit>`.
 Debugging Information tests
 ---------------------------
 
-To run debugging information tests simply checkout the tests inside
-clang/test directory.
-
-.. code-block:: bash
-
-    % cd clang/test
-    % svn co http://llvm.org/svn/llvm-project/debuginfo-tests/trunk debuginfo-tests
-
-These tests are already set up to run as part of clang regression tests.
+To run debugging information tests simply add the ``debuginfo-tests``
+project to your ``LLVM_ENABLE_PROJECTS`` define on the cmake
+command-line.
 
 Regression test structure
 =========================
diff --git a/docs/TypeMetadata.rst b/docs/TypeMetadata.rst
index 0382d0cbd44bf94bd889f64d4f7ec69b42b11376..84cf05ba70fc6521dc4bdece4d5c512862f26410 100644
--- a/docs/TypeMetadata.rst
+++ b/docs/TypeMetadata.rst
@@ -223,4 +223,4 @@ efficiently to minimize the sizes of the underlying bitsets.
       ret void
     }
 
-.. _GlobalLayoutBuilder: http://git.llvm.org/klaus/llvm/blob/master/include/llvm/Transforms/IPO/LowerTypeTests.h
+.. _GlobalLayoutBuilder: https://github.com/llvm/llvm-project/blob/master/llvm/include/llvm/Transforms/IPO/LowerTypeTests.h
diff --git a/docs/Vectorizers.rst b/docs/Vectorizers.rst
index 42e8d02f337eead4f993cfd34c27f532291631fd..fd1525c00cac285d11b7410714a36520377530e0 100644
--- a/docs/Vectorizers.rst
+++ b/docs/Vectorizers.rst
@@ -311,7 +311,7 @@ ignored (as other compilers do) are still being left un-vectorized.
 Vectorization of function calls
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-The Loop Vectorize can vectorize intrinsic math functions.
+The Loop Vectorizer can vectorize intrinsic math functions.
 See the table below for a list of these functions.
 
 +-----+-----+---------+
@@ -328,6 +328,11 @@ See the table below for a list of these functions.
 |     |     | fmuladd |
 +-----+-----+---------+
 
+Note that the optimizer may not be able to vectorize math library functions 
+that correspond to these intrinsics if the library calls access external state 
+such as "errno". To allow better optimization of C/C++ math library functions, 
+use "-fno-math-errno".
+
 The loop vectorizer knows about special instructions on the target and will
 vectorize a loop containing a function call that maps to the instructions. For
 example, the loop below will be vectorized on Intel x86 if the SSE4.1 roundps
@@ -369,7 +374,7 @@ Performance
 -----------
 
 This section shows the execution time of Clang on a simple benchmark:
-`gcc-loops <http://llvm.org/viewvc/llvm-project/test-suite/trunk/SingleSource/UnitTests/Vectorizer/>`_.
+`gcc-loops <https://github.com/llvm/llvm-test-suite/tree/master/SingleSource/UnitTests/Vectorizer>`_.
 This benchmarks is a collection of loops from the GCC autovectorization
 `page <http://gcc.gnu.org/projects/tree-ssa/vectorization.html>`_ by Dorit Nuzman.
 
diff --git a/docs/WritingAnLLVMPass.rst b/docs/WritingAnLLVMPass.rst
index 41f400740e84e78159b70eea4a11cb23b01d1ada..19dc6c11c9b6da55541875154f66035b66cc5d96 100644
--- a/docs/WritingAnLLVMPass.rst
+++ b/docs/WritingAnLLVMPass.rst
@@ -55,7 +55,7 @@ copy the following into ``CMakeLists.txt``:
 
 .. code-block:: cmake
 
-  add_llvm_loadable_module( LLVMHello
+  add_llvm_library( LLVMHello MODULE
     Hello.cpp
   
     PLUGIN_TOOL
diff --git a/docs/conf.py b/docs/conf.py
index 91c86569d8d7a6661f16de00c62d150f21797b57..000c0af1a63473972223c6fd3fda1352a6d60609 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -9,6 +9,7 @@
 #
 # All configuration values have a default; values that are commented out
 # serve to show the default.
+from __future__ import print_function
 
 import sys, os
 from datetime import date
@@ -50,9 +51,9 @@ copyright = u'2003-%d, LLVM Project' % date.today().year
 # built documents.
 #
 # The short version.
-version = '8'
+version = '9'
 # The full version, including alpha/beta/rc tags.
-release = '8'
+release = '9'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
@@ -219,7 +220,7 @@ man_pages = []
 # Automatically derive the list of man pages from the contents of the command
 # guide subdirectory.
 basedir = os.path.dirname(__file__)
-man_page_authors = "Maintained by The LLVM Team (http://llvm.org/)."
+man_page_authors = "Maintained by the LLVM Team (https://llvm.org/)."
 command_guide_subpath = 'CommandGuide'
 command_guide_path = os.path.join(basedir, command_guide_subpath)
 for name in os.listdir(command_guide_path):
@@ -234,14 +235,14 @@ for name in os.listdir(command_guide_path):
         header = f.readline().rstrip('\n')
 
         if len(header) != len(title):
-            print >>sys.stderr, (
+            print((
                 "error: invalid header in %r (does not match title)" % (
-                    file_subpath,))
+                    file_subpath,)), file=sys.stderr)
         if ' - ' not in title:
-            print >>sys.stderr, (
+            print((
                 ("error: invalid title in %r "
                  "(expected '<name> - <description>')") % (
-                    file_subpath,))
+                    file_subpath,)), file=sys.stderr)
 
         # Split the name out of the title.
         name,description = title.split(' - ', 1)
diff --git a/docs/index.rst b/docs/index.rst
index 698e6764bfb3ab9f67022dd8b249265bd9a65407..4527fe5105346b818e1bff548bcafc36b43efe55 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -232,7 +232,7 @@ For developers of applications which use LLVM as a library.
 
 `Documentation for Go bindings <http://godoc.org/llvm.org/llvm/bindings/go/llvm>`_
 
-`ViewVC Repository Browser <http://llvm.org/viewvc/>`_
+`Github Source Repository Browser <http://github.com/llvm/llvm-project//>`_
    ..
 
 :doc:`CompilerWriterInfo`
diff --git a/docs/llvm-objdump.1 b/docs/llvm-objdump.1
index 72d1b93f6bb37cb7038dbc356ffbc5e18765d44e..85c40d7da682279588f3756f6667ad71f49f94e5 100644
--- a/docs/llvm-objdump.1
+++ b/docs/llvm-objdump.1
@@ -1,5 +1,6 @@
-.\" This file is distributed under the University of Illinois Open Source
-.\" License. See LICENSE.TXT for details.
+.\" Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+.\" See https://llvm.org/LICENSE.txt for license information.
+.\" SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 .\"
 .Dd December 19, 2018
 .Dt LLVM-OBJDUMP 1
diff --git a/examples/BrainF/BrainF.cpp b/examples/BrainF/BrainF.cpp
index 8af34d04701e32ef713e0af0509321fc0112c39e..2937f8c6cc1d665025a5c18ff8962c08ab47ad8d 100644
--- a/examples/BrainF/BrainF.cpp
+++ b/examples/BrainF/BrainF.cpp
@@ -1,9 +1,8 @@
 //===-- BrainF.cpp - BrainF compiler example ------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -73,19 +72,18 @@ void BrainF::header(LLVMContext& C) {
                                                     Tys);
 
   //declare i32 @getchar()
-  getchar_func = cast<Function>(module->
-    getOrInsertFunction("getchar", IntegerType::getInt32Ty(C)));
+  getchar_func =
+      module->getOrInsertFunction("getchar", IntegerType::getInt32Ty(C));
 
   //declare i32 @putchar(i32)
-  putchar_func = cast<Function>(module->
-    getOrInsertFunction("putchar", IntegerType::getInt32Ty(C),
-                        IntegerType::getInt32Ty(C)));
+  putchar_func = module->getOrInsertFunction(
+      "putchar", IntegerType::getInt32Ty(C), IntegerType::getInt32Ty(C));
 
   //Function header
 
   //define void @brainf()
-  brainf_func = cast<Function>(module->
-    getOrInsertFunction("brainf", Type::getVoidTy(C)));
+  brainf_func = Function::Create(FunctionType::get(Type::getVoidTy(C), false),
+                                 Function::ExternalLinkage, "brainf", module);
 
   builder = new IRBuilder<>(BasicBlock::Create(C, label, brainf_func));
 
@@ -154,9 +152,9 @@ void BrainF::header(LLVMContext& C) {
       "aberrormsg");
 
     //declare i32 @puts(i8 *)
-    Function *puts_func = cast<Function>(module->
-      getOrInsertFunction("puts", IntegerType::getInt32Ty(C),
-                      PointerType::getUnqual(IntegerType::getInt8Ty(C))));
+    FunctionCallee puts_func = module->getOrInsertFunction(
+        "puts", IntegerType::getInt32Ty(C),
+        PointerType::getUnqual(IntegerType::getInt8Ty(C)));
 
     //brainf.aberror:
     aberrorbb = BasicBlock::Create(C, label, brainf_func);
diff --git a/examples/BrainF/BrainF.h b/examples/BrainF/BrainF.h
index 587be141b98f0728ae7b49a27aa9ce10228e47f6..3bd6b16f716a2a264dd08d08f7b93e0e33df2989 100644
--- a/examples/BrainF/BrainF.h
+++ b/examples/BrainF/BrainF.h
@@ -1,9 +1,8 @@
 //===-- BrainF.h - BrainF compiler class ------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -80,8 +79,8 @@ class BrainF {
     std::istream *in;
     Module *module;
     Function *brainf_func;
-    Function *getchar_func;
-    Function *putchar_func;
+    FunctionCallee getchar_func;
+    FunctionCallee putchar_func;
     Value *ptr_arr;
     Value *ptr_arrmax;
     BasicBlock *endbb;
diff --git a/examples/BrainF/BrainFDriver.cpp b/examples/BrainF/BrainFDriver.cpp
index 24aac4aeb1df489a7f4e9776269434c6d441a83e..1eef5fcf853e1e968f8cdfe0f47b6b27492d2c54 100644
--- a/examples/BrainF/BrainFDriver.cpp
+++ b/examples/BrainF/BrainFDriver.cpp
@@ -1,9 +1,8 @@
 //===-- BrainFDriver.cpp - BrainF compiler driver -------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -73,11 +72,14 @@ JIT("jit", cl::desc("Run program Just-In-Time"));
 //Add main function so can be fully compiled
 void addMainFunction(Module *mod) {
   //define i32 @main(i32 %argc, i8 **%argv)
-  Function *main_func = cast<Function>(mod->
-    getOrInsertFunction("main", IntegerType::getInt32Ty(mod->getContext()),
-                        IntegerType::getInt32Ty(mod->getContext()),
-                        PointerType::getUnqual(PointerType::getUnqual(
-                          IntegerType::getInt8Ty(mod->getContext())))));
+  FunctionType *main_func_fty = FunctionType::get(
+      Type::getInt32Ty(mod->getContext()),
+      {Type::getInt32Ty(mod->getContext()),
+       Type::getInt8Ty(mod->getContext())->getPointerTo()->getPointerTo()},
+      false);
+  Function *main_func =
+      Function::Create(main_func_fty, Function::ExternalLinkage, "main", mod);
+
   {
     Function::arg_iterator args = main_func->arg_begin();
     Value *arg_0 = &*args++;
diff --git a/examples/ExceptionDemo/ExceptionDemo.cpp b/examples/ExceptionDemo/ExceptionDemo.cpp
index a3182d2087fc05601dc1ed8933dd7e1fee8df5b0..91c8617f26d206bab1db693d7f31525207aa8cbd 100644
--- a/examples/ExceptionDemo/ExceptionDemo.cpp
+++ b/examples/ExceptionDemo/ExceptionDemo.cpp
@@ -1,9 +1,8 @@
 //===-- ExceptionDemo.cpp - An example using llvm Exceptions --------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/examples/Fibonacci/fibonacci.cpp b/examples/Fibonacci/fibonacci.cpp
index 662cb01dd37e9de96651d5dc71627d13047ed030..12393a414d0746813fb0d0104823d42639cccf84 100644
--- a/examples/Fibonacci/fibonacci.cpp
+++ b/examples/Fibonacci/fibonacci.cpp
@@ -1,9 +1,8 @@
 //===--- examples/Fibonacci/fibonacci.cpp - An example use of the JIT -----===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -52,9 +51,10 @@ using namespace llvm;
 static Function *CreateFibFunction(Module *M, LLVMContext &Context) {
   // Create the fib function and insert it into module M. This function is said
   // to return an int and take an int parameter.
+  FunctionType *FibFTy = FunctionType::get(Type::getInt32Ty(Context),
+                                           {Type::getInt32Ty(Context)}, false);
   Function *FibF =
-    cast<Function>(M->getOrInsertFunction("fib", Type::getInt32Ty(Context),
-                                          Type::getInt32Ty(Context)));
+      Function::Create(FibFTy, Function::ExternalLinkage, "fib", M);
 
   // Add a basic block to the function.
   BasicBlock *BB = BasicBlock::Create(Context, "EntryBlock", FibF);
diff --git a/examples/HowToUseJIT/HowToUseJIT.cpp b/examples/HowToUseJIT/HowToUseJIT.cpp
index f141fa5a7f541037a42643e309fa32e005f32ae3..30354d4cae64866e17f9722a781dc30ed456ae1a 100644
--- a/examples/HowToUseJIT/HowToUseJIT.cpp
+++ b/examples/HowToUseJIT/HowToUseJIT.cpp
@@ -1,9 +1,8 @@
 //===-- examples/HowToUseJIT/HowToUseJIT.cpp - An example use of the JIT --===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -70,8 +69,9 @@ int main() {
   // Create the add1 function entry and insert this entry into module M.  The
   // function will have a return type of "int" and take an argument of "int".
   Function *Add1F =
-    cast<Function>(M->getOrInsertFunction("add1", Type::getInt32Ty(Context),
-                                          Type::getInt32Ty(Context)));
+      Function::Create(FunctionType::get(Type::getInt32Ty(Context),
+                                         {Type::getInt32Ty(Context)}, false),
+                       Function::ExternalLinkage, "add1", M);
 
   // Add a basic block to the function. As before, it automatically inserts
   // because of the last argument.
@@ -100,7 +100,8 @@ int main() {
   // Now we're going to create function `foo', which returns an int and takes no
   // arguments.
   Function *FooF =
-    cast<Function>(M->getOrInsertFunction("foo", Type::getInt32Ty(Context)));
+      Function::Create(FunctionType::get(Type::getInt32Ty(Context), {}, false),
+                       Function::ExternalLinkage, "foo", M);
 
   // Add a basic block to the FooF function.
   BB = BasicBlock::Create(Context, "EntryBlock", FooF);
diff --git a/examples/Kaleidoscope/BuildingAJIT/Chapter1/KaleidoscopeJIT.h b/examples/Kaleidoscope/BuildingAJIT/Chapter1/KaleidoscopeJIT.h
index 1df5aff086935677b114ae4d2f68198fc325deed..25535bff432429c2a04a29fda737a6fbffb7d4ee 100644
--- a/examples/Kaleidoscope/BuildingAJIT/Chapter1/KaleidoscopeJIT.h
+++ b/examples/Kaleidoscope/BuildingAJIT/Chapter1/KaleidoscopeJIT.h
@@ -1,9 +1,8 @@
 //===- KaleidoscopeJIT.h - A simple JIT for Kaleidoscope --------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/examples/Kaleidoscope/BuildingAJIT/Chapter2/KaleidoscopeJIT.h b/examples/Kaleidoscope/BuildingAJIT/Chapter2/KaleidoscopeJIT.h
index 4dc08470dffabf70188014569572c051b59623e5..783ef7399f505b6823192a54fd81493c7398d6f1 100644
--- a/examples/Kaleidoscope/BuildingAJIT/Chapter2/KaleidoscopeJIT.h
+++ b/examples/Kaleidoscope/BuildingAJIT/Chapter2/KaleidoscopeJIT.h
@@ -1,9 +1,8 @@
 //===- KaleidoscopeJIT.h - A simple JIT for Kaleidoscope --------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/examples/Kaleidoscope/BuildingAJIT/Chapter3/KaleidoscopeJIT.h b/examples/Kaleidoscope/BuildingAJIT/Chapter3/KaleidoscopeJIT.h
index ce0111d2f6b845f6a0fb1ef382e9c892aa030df7..717fbb7d995118f1664ef5251e7ce600429f43bd 100644
--- a/examples/Kaleidoscope/BuildingAJIT/Chapter3/KaleidoscopeJIT.h
+++ b/examples/Kaleidoscope/BuildingAJIT/Chapter3/KaleidoscopeJIT.h
@@ -1,9 +1,8 @@
 //===- KaleidoscopeJIT.h - A simple JIT for Kaleidoscope --------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/examples/Kaleidoscope/BuildingAJIT/Chapter4/KaleidoscopeJIT.h b/examples/Kaleidoscope/BuildingAJIT/Chapter4/KaleidoscopeJIT.h
index ffca65fbcd4f52c30f3fa40508eeb9773b961ba9..3cf521ce2be005ce6f016c2d39ff74d462448bf2 100644
--- a/examples/Kaleidoscope/BuildingAJIT/Chapter4/KaleidoscopeJIT.h
+++ b/examples/Kaleidoscope/BuildingAJIT/Chapter4/KaleidoscopeJIT.h
@@ -1,9 +1,8 @@
 //===- KaleidoscopeJIT.h - A simple JIT for Kaleidoscope --------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/examples/Kaleidoscope/BuildingAJIT/Chapter5/KaleidoscopeJIT.h b/examples/Kaleidoscope/BuildingAJIT/Chapter5/KaleidoscopeJIT.h
index f1ae5b022895b8c9b2ddbd404072aa4634d5b09b..d6c9d12c538c26d8eb19f5854a3640f03e609910 100644
--- a/examples/Kaleidoscope/BuildingAJIT/Chapter5/KaleidoscopeJIT.h
+++ b/examples/Kaleidoscope/BuildingAJIT/Chapter5/KaleidoscopeJIT.h
@@ -1,9 +1,8 @@
 //===- KaleidoscopeJIT.h - A simple JIT for Kaleidoscope --------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/examples/Kaleidoscope/BuildingAJIT/Chapter5/RemoteJITUtils.h b/examples/Kaleidoscope/BuildingAJIT/Chapter5/RemoteJITUtils.h
index 48a19385596f2f3e1e60df17bd519cf1a5dde397..14a2815bc021f42e0a5c13152c6edaf8ff982b03 100644
--- a/examples/Kaleidoscope/BuildingAJIT/Chapter5/RemoteJITUtils.h
+++ b/examples/Kaleidoscope/BuildingAJIT/Chapter5/RemoteJITUtils.h
@@ -1,9 +1,8 @@
 //===-- RemoteJITUtils.h - Utilities for remote-JITing with LLI -*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/examples/Kaleidoscope/Chapter2/CMakeLists.txt b/examples/Kaleidoscope/Chapter2/CMakeLists.txt
index e4f275d01d4059cc104a938214c32b672fcd4c2a..563b8cc848f6ed4da9a444a6b9347162405040ec 100644
--- a/examples/Kaleidoscope/Chapter2/CMakeLists.txt
+++ b/examples/Kaleidoscope/Chapter2/CMakeLists.txt
@@ -11,3 +11,8 @@ if(LLVM_COMPILER_IS_GCC_COMPATIBLE)
     -Wno-unused-private-field
     )
 endif()
+
+if(MSVC)
+  # ignore "warning LNK4199: /DELAYLOAD:shell32.dll ignored; no imports found from shell32.dll"
+  target_link_libraries(Kaleidoscope-Ch2 PRIVATE "-ignore:4199")
+endif()
diff --git a/examples/Kaleidoscope/MCJIT/cached/genk-timing.py b/examples/Kaleidoscope/MCJIT/cached/genk-timing.py
index 96dd6db5da3dfa63aac781cfdef7c031ec8a64cc..87bbfbf32bda3ab7e1956c3d50927c362da3faed 100644
--- a/examples/Kaleidoscope/MCJIT/cached/genk-timing.py
+++ b/examples/Kaleidoscope/MCJIT/cached/genk-timing.py
@@ -1,5 +1,7 @@
 #!/usr/bin/env python
 
+from __future__ import print_function
+
 import sys
 import random
 
@@ -173,7 +175,7 @@ class KScriptGenerator:
 
 def generateKScript(filename, numFuncs, elementsPerFunc, funcsBetweenExec, callWeighting, timingScript):
     """ Generate a random Kaleidoscope script based on the given parameters """
-    print "Generating " + filename
+    print("Generating " + filename)
     print("  %d functions, %d elements per function, %d functions between execution" %
           (numFuncs, elementsPerFunc, funcsBetweenExec))
     print("  Call weighting = %f" % callWeighting)
@@ -200,7 +202,7 @@ def generateKScript(filename, numFuncs, elementsPerFunc, funcsBetweenExec, callW
     script.writeEmptyLine()
     script.writeFinalFunctionCounts()
     funcsCalled = len(script.calledFunctions)
-    print "  Called %d of %d functions, %d total" % (funcsCalled, numFuncs, script.totalCallsExecuted)
+    print("  Called %d of %d functions, %d total" % (funcsCalled, numFuncs, script.totalCallsExecuted))
     timingScript.writeTimingCall(filename, numFuncs, funcsCalled, script.totalCallsExecuted)
 
 # Execution begins here
@@ -216,4 +218,4 @@ dataSets = [(5000, 3,  50, 0.50), (5000, 10, 100, 0.10), (5000, 10, 5, 0.10), (5
 for (numFuncs, elementsPerFunc, funcsBetweenExec, callWeighting) in dataSets:
     filename = "test-%d-%d-%d-%d.k" % (numFuncs, elementsPerFunc, funcsBetweenExec, int(callWeighting * 100))
     generateKScript(filename, numFuncs, elementsPerFunc, funcsBetweenExec, callWeighting, timingScript)
-print "All done!"
+print("All done!")
diff --git a/examples/Kaleidoscope/MCJIT/cached/split-lib.py b/examples/Kaleidoscope/MCJIT/cached/split-lib.py
index 5cdcc6d46c0498f83ab8937b9c99c884fa514fbe..1aa80ee83ac4ca852a9dbaa1a0a5abe514636d00 100644
--- a/examples/Kaleidoscope/MCJIT/cached/split-lib.py
+++ b/examples/Kaleidoscope/MCJIT/cached/split-lib.py
@@ -1,5 +1,7 @@
 #!/usr/bin/env python
 
+from __future__ import print_function
+
 class TimingScriptGenerator:
     """Used to generate a bash script which will invoke the toy and time it"""
     def __init__(self, scriptname, outputname):
@@ -47,7 +49,7 @@ def splitScript(inputname, libGenScript, timingScript):
   infile = open(inputname, "r")
   libfile = open(libname, "w")
   callfile = open(callname, "w")
-  print "Splitting %s into %s and %s" % (inputname, callname, libname)
+  print("Splitting %s into %s and %s" % (inputname, callname, libname))
   for line in infile:
     if not line.startswith("#"):
       if line.startswith("print"):
@@ -67,4 +69,4 @@ script_list = ["test-5000-3-50-50.k", "test-5000-10-100-10.k", "test-5000-10-5-1
 
 for script in script_list:
   splitScript(script, libGenScript, timingScript)
-print "All done!"
+print("All done!")
diff --git a/examples/Kaleidoscope/MCJIT/complete/genk-timing.py b/examples/Kaleidoscope/MCJIT/complete/genk-timing.py
index 72591fe8bd4996e3939e4cf8cb9f1195b8d22d31..c3b4d23c002a15aee89144fbd69300fc4f0002cc 100644
--- a/examples/Kaleidoscope/MCJIT/complete/genk-timing.py
+++ b/examples/Kaleidoscope/MCJIT/complete/genk-timing.py
@@ -1,5 +1,7 @@
 #!/usr/bin/env python
 
+from __future__ import print_function
+
 import sys
 import random
 
@@ -178,7 +180,7 @@ class KScriptGenerator:
 
 def generateKScript(filename, numFuncs, elementsPerFunc, funcsBetweenExec, callWeighting, timingScript):
     """ Generate a random Kaleidoscope script based on the given parameters """
-    print "Generating " + filename
+    print("Generating " + filename)
     print("  %d functions, %d elements per function, %d functions between execution" %
           (numFuncs, elementsPerFunc, funcsBetweenExec))
     print("  Call weighting = %f" % callWeighting)
@@ -205,7 +207,7 @@ def generateKScript(filename, numFuncs, elementsPerFunc, funcsBetweenExec, callW
     script.writeEmptyLine()
     script.writeFinalFunctionCounts()
     funcsCalled = len(script.calledFunctions)
-    print "  Called %d of %d functions, %d total" % (funcsCalled, numFuncs, script.totalCallsExecuted)
+    print("  Called %d of %d functions, %d total" % (funcsCalled, numFuncs, script.totalCallsExecuted))
     timingScript.writeTimingCall(filename, numFuncs, funcsCalled, script.totalCallsExecuted)
 
 # Execution begins here
@@ -221,4 +223,4 @@ dataSets = [(5000, 3,  50, 0.50), (5000, 10, 100, 0.10), (5000, 10, 5, 0.10), (5
 for (numFuncs, elementsPerFunc, funcsBetweenExec, callWeighting) in dataSets:
     filename = "test-%d-%d-%d-%d.k" % (numFuncs, elementsPerFunc, funcsBetweenExec, int(callWeighting * 100))
     generateKScript(filename, numFuncs, elementsPerFunc, funcsBetweenExec, callWeighting, timingScript)
-print "All done!"
+print("All done!")
diff --git a/examples/Kaleidoscope/MCJIT/complete/split-lib.py b/examples/Kaleidoscope/MCJIT/complete/split-lib.py
index f6bec0276d858022279f762331132de4b392a48f..61c9a5b169fcd86221837e5ebd73d700206230dd 100644
--- a/examples/Kaleidoscope/MCJIT/complete/split-lib.py
+++ b/examples/Kaleidoscope/MCJIT/complete/split-lib.py
@@ -1,5 +1,7 @@
 #!/usr/bin/env python
 
+from __future__ import print_function
+
 class TimingScriptGenerator:
     """Used to generate a bash script which will invoke the toy and time it"""
     def __init__(self, scriptname, outputname):
@@ -47,7 +49,7 @@ def splitScript(inputname, libGenScript, timingScript):
   infile = open(inputname, "r")
   libfile = open(libname, "w")
   callfile = open(callname, "w")
-  print "Splitting %s into %s and %s" % (inputname, callname, libname)
+  print("Splitting %s into %s and %s" % (inputname, callname, libname))
   for line in infile:
     if not line.startswith("#"):
       if line.startswith("print"):
@@ -67,4 +69,4 @@ script_list = ["test-5000-3-50-50.k", "test-5000-10-100-10.k", "test-5000-10-5-1
 
 for script in script_list:
   splitScript(script, libGenScript, timingScript)
-print "All done!"
+print("All done!")
diff --git a/examples/Kaleidoscope/MCJIT/lazy/genk-timing.py b/examples/Kaleidoscope/MCJIT/lazy/genk-timing.py
index 96dd6db5da3dfa63aac781cfdef7c031ec8a64cc..87bbfbf32bda3ab7e1956c3d50927c362da3faed 100644
--- a/examples/Kaleidoscope/MCJIT/lazy/genk-timing.py
+++ b/examples/Kaleidoscope/MCJIT/lazy/genk-timing.py
@@ -1,5 +1,7 @@
 #!/usr/bin/env python
 
+from __future__ import print_function
+
 import sys
 import random
 
@@ -173,7 +175,7 @@ class KScriptGenerator:
 
 def generateKScript(filename, numFuncs, elementsPerFunc, funcsBetweenExec, callWeighting, timingScript):
     """ Generate a random Kaleidoscope script based on the given parameters """
-    print "Generating " + filename
+    print("Generating " + filename)
     print("  %d functions, %d elements per function, %d functions between execution" %
           (numFuncs, elementsPerFunc, funcsBetweenExec))
     print("  Call weighting = %f" % callWeighting)
@@ -200,7 +202,7 @@ def generateKScript(filename, numFuncs, elementsPerFunc, funcsBetweenExec, callW
     script.writeEmptyLine()
     script.writeFinalFunctionCounts()
     funcsCalled = len(script.calledFunctions)
-    print "  Called %d of %d functions, %d total" % (funcsCalled, numFuncs, script.totalCallsExecuted)
+    print("  Called %d of %d functions, %d total" % (funcsCalled, numFuncs, script.totalCallsExecuted))
     timingScript.writeTimingCall(filename, numFuncs, funcsCalled, script.totalCallsExecuted)
 
 # Execution begins here
@@ -216,4 +218,4 @@ dataSets = [(5000, 3,  50, 0.50), (5000, 10, 100, 0.10), (5000, 10, 5, 0.10), (5
 for (numFuncs, elementsPerFunc, funcsBetweenExec, callWeighting) in dataSets:
     filename = "test-%d-%d-%d-%d.k" % (numFuncs, elementsPerFunc, funcsBetweenExec, int(callWeighting * 100))
     generateKScript(filename, numFuncs, elementsPerFunc, funcsBetweenExec, callWeighting, timingScript)
-print "All done!"
+print("All done!")
diff --git a/examples/Kaleidoscope/include/KaleidoscopeJIT.h b/examples/Kaleidoscope/include/KaleidoscopeJIT.h
index 972773a64f7e4f2280599960a4d8f5346570dfe9..a2b0fee0ef9054765309a9d1ed4981bd3da380f0 100644
--- a/examples/Kaleidoscope/include/KaleidoscopeJIT.h
+++ b/examples/Kaleidoscope/include/KaleidoscopeJIT.h
@@ -1,9 +1,8 @@
 //===- KaleidoscopeJIT.h - A simple JIT for Kaleidoscope --------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/examples/LLVMBuild.txt b/examples/LLVMBuild.txt
index 4d06ffb995f2ae40ea682bac3630928059430635..582ccb0c1e3e7d1bdf53d28f00660ce8b76018ea 100644
--- a/examples/LLVMBuild.txt
+++ b/examples/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./examples/LLVMBuild.txt ---------------------------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/examples/ModuleMaker/ModuleMaker.cpp b/examples/ModuleMaker/ModuleMaker.cpp
index beff40a8d5060cfd140fb7bf0769eb12077d3d87..e409dd60e9b04710ab4b304f9e0f409441702c04 100644
--- a/examples/ModuleMaker/ModuleMaker.cpp
+++ b/examples/ModuleMaker/ModuleMaker.cpp
@@ -1,9 +1,8 @@
 //===- examples/ModuleMaker/ModuleMaker.cpp - Example project ---*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/examples/ParallelJIT/ParallelJIT.cpp b/examples/ParallelJIT/ParallelJIT.cpp
index ff44375e3921e27a21bc39bde34d20a73c4a1247..9c148a08017070900d4f790a501a34d4377681de 100644
--- a/examples/ParallelJIT/ParallelJIT.cpp
+++ b/examples/ParallelJIT/ParallelJIT.cpp
@@ -1,9 +1,8 @@
 //===-- examples/ParallelJIT/ParallelJIT.cpp - Exercise threaded-safe JIT -===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -48,20 +47,20 @@
 using namespace llvm;
 
 static Function* createAdd1(Module *M) {
+  LLVMContext &Context = M->getContext();
   // Create the add1 function entry and insert this entry into module M.  The
   // function will have a return type of "int" and take an argument of "int".
-  // The '0' terminates the list of argument types.
   Function *Add1F =
-    cast<Function>(M->getOrInsertFunction("add1",
-                                          Type::getInt32Ty(M->getContext()),
-                                          Type::getInt32Ty(M->getContext())));
+      Function::Create(FunctionType::get(Type::getInt32Ty(Context),
+                                         {Type::getInt32Ty(Context)}, false),
+                       Function::ExternalLinkage, "add1", M);
 
   // Add a basic block to the function. As before, it automatically inserts
   // because of the last argument.
-  BasicBlock *BB = BasicBlock::Create(M->getContext(), "EntryBlock", Add1F);
+  BasicBlock *BB = BasicBlock::Create(Context, "EntryBlock", Add1F);
 
   // Get pointers to the constant `1'.
-  Value *One = ConstantInt::get(Type::getInt32Ty(M->getContext()), 1);
+  Value *One = ConstantInt::get(Type::getInt32Ty(Context), 1);
 
   // Get pointers to the integer argument of the add1 function...
   assert(Add1F->arg_begin() != Add1F->arg_end()); // Make sure there's an arg
@@ -72,42 +71,43 @@ static Function* createAdd1(Module *M) {
   Instruction *Add = BinaryOperator::CreateAdd(One, ArgX, "addresult", BB);
 
   // Create the return instruction and add it to the basic block
-  ReturnInst::Create(M->getContext(), Add, BB);
+  ReturnInst::Create(Context, Add, BB);
 
   // Now, function add1 is ready.
   return Add1F;
 }
 
 static Function *CreateFibFunction(Module *M) {
+  LLVMContext &Context = M->getContext();
   // Create the fib function and insert it into module M.  This function is said
   // to return an int and take an int parameter.
-  Function *FibF = 
-    cast<Function>(M->getOrInsertFunction("fib",
-                                          Type::getInt32Ty(M->getContext()),
-                                          Type::getInt32Ty(M->getContext())));
+  FunctionType *FibFTy = FunctionType::get(Type::getInt32Ty(Context),
+                                           {Type::getInt32Ty(Context)}, false);
+  Function *FibF =
+      Function::Create(FibFTy, Function::ExternalLinkage, "fib", M);
 
   // Add a basic block to the function.
-  BasicBlock *BB = BasicBlock::Create(M->getContext(), "EntryBlock", FibF);
+  BasicBlock *BB = BasicBlock::Create(Context, "EntryBlock", FibF);
 
   // Get pointers to the constants.
-  Value *One = ConstantInt::get(Type::getInt32Ty(M->getContext()), 1);
-  Value *Two = ConstantInt::get(Type::getInt32Ty(M->getContext()), 2);
+  Value *One = ConstantInt::get(Type::getInt32Ty(Context), 1);
+  Value *Two = ConstantInt::get(Type::getInt32Ty(Context), 2);
 
   // Get pointer to the integer argument of the add1 function...
   Argument *ArgX = &*FibF->arg_begin(); // Get the arg.
   ArgX->setName("AnArg");            // Give it a nice symbolic name for fun.
 
   // Create the true_block.
-  BasicBlock *RetBB = BasicBlock::Create(M->getContext(), "return", FibF);
+  BasicBlock *RetBB = BasicBlock::Create(Context, "return", FibF);
   // Create an exit block.
-  BasicBlock* RecurseBB = BasicBlock::Create(M->getContext(), "recurse", FibF);
+  BasicBlock *RecurseBB = BasicBlock::Create(Context, "recurse", FibF);
 
   // Create the "if (arg < 2) goto exitbb"
   Value *CondInst = new ICmpInst(*BB, ICmpInst::ICMP_SLE, ArgX, Two, "cond");
   BranchInst::Create(RetBB, RecurseBB, CondInst, BB);
 
   // Create: ret int 1
-  ReturnInst::Create(M->getContext(), One, RetBB);
+  ReturnInst::Create(Context, One, RetBB);
 
   // create fib(x-1)
   Value *Sub = BinaryOperator::CreateSub(ArgX, One, "arg", RecurseBB);
@@ -122,7 +122,7 @@ static Function *CreateFibFunction(Module *M) {
     BinaryOperator::CreateAdd(CallFibX1, CallFibX2, "addresult", RecurseBB);
 
   // Create the return instruction and add it to the basic block
-  ReturnInst::Create(M->getContext(), Sum, RecurseBB);
+  ReturnInst::Create(Context, Sum, RecurseBB);
 
   return FibF;
 }
diff --git a/include/llvm-c/Analysis.h b/include/llvm-c/Analysis.h
index 36dcb89e0e08fd31c1264046538b86c65ace81a2..cb9e8ece3c5311e7c5c29271cbf7a783561bc0a1 100644
--- a/include/llvm-c/Analysis.h
+++ b/include/llvm-c/Analysis.h
@@ -1,9 +1,9 @@
 /*===-- llvm-c/Analysis.h - Analysis Library C Interface --------*- C++ -*-===*\
 |*                                                                            *|
-|*                     The LLVM Compiler Infrastructure                       *|
-|*                                                                            *|
-|* This file is distributed under the University of Illinois Open Source      *|
-|* License. See LICENSE.TXT for details.                                      *|
+|* Part of the LLVM Project, under the Apache License v2.0 with LLVM          *|
+|* Exceptions.                                                                *|
+|* See https://llvm.org/LICENSE.txt for license information.                  *|
+|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception                    *|
 |*                                                                            *|
 |*===----------------------------------------------------------------------===*|
 |*                                                                            *|
diff --git a/include/llvm-c/BitReader.h b/include/llvm-c/BitReader.h
index d1fc302767baca1fe9ea9d767cddd1bf2da98bdc..b307ee979f8a3986eb1553288cd6017fe7c906cb 100644
--- a/include/llvm-c/BitReader.h
+++ b/include/llvm-c/BitReader.h
@@ -1,9 +1,9 @@
 /*===-- llvm-c/BitReader.h - BitReader Library C Interface ------*- C++ -*-===*\
 |*                                                                            *|
-|*                     The LLVM Compiler Infrastructure                       *|
-|*                                                                            *|
-|* This file is distributed under the University of Illinois Open Source      *|
-|* License. See LICENSE.TXT for details.                                      *|
+|* Part of the LLVM Project, under the Apache License v2.0 with LLVM          *|
+|* Exceptions.                                                                *|
+|* See https://llvm.org/LICENSE.txt for license information.                  *|
+|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception                    *|
 |*                                                                            *|
 |*===----------------------------------------------------------------------===*|
 |*                                                                            *|
diff --git a/include/llvm-c/BitWriter.h b/include/llvm-c/BitWriter.h
index 797d03179ab3e991e065aa99dc26e4bedc59f9cd..187051555b9a29139fa070d9a7f3ad97950ee8e9 100644
--- a/include/llvm-c/BitWriter.h
+++ b/include/llvm-c/BitWriter.h
@@ -1,9 +1,9 @@
 /*===-- llvm-c/BitWriter.h - BitWriter Library C Interface ------*- C++ -*-===*\
 |*                                                                            *|
-|*                     The LLVM Compiler Infrastructure                       *|
-|*                                                                            *|
-|* This file is distributed under the University of Illinois Open Source      *|
-|* License. See LICENSE.TXT for details.                                      *|
+|* Part of the LLVM Project, under the Apache License v2.0 with LLVM          *|
+|* Exceptions.                                                                *|
+|* See https://llvm.org/LICENSE.txt for license information.                  *|
+|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception                    *|
 |*                                                                            *|
 |*===----------------------------------------------------------------------===*|
 |*                                                                            *|
diff --git a/include/llvm-c/Comdat.h b/include/llvm-c/Comdat.h
index 499996d68a53cb9fe5d5cbe6e1bf8e6174393bc2..81fee3fc9a6b4018a60fb1e95962a15b0b721c13 100644
--- a/include/llvm-c/Comdat.h
+++ b/include/llvm-c/Comdat.h
@@ -1,9 +1,9 @@
 /*===-- llvm-c/Comdat.h - Module Comdat C Interface -------------*- C++ -*-===*\
 |*                                                                            *|
-|*                     The LLVM Compiler Infrastructure                       *|
-|*                                                                            *|
-|* This file is distributed under the University of Illinois Open Source      *|
-|* License. See LICENSE.TXT for details.                                      *|
+|* Part of the LLVM Project, under the Apache License v2.0 with LLVM          *|
+|* Exceptions.                                                                *|
+|* See https://llvm.org/LICENSE.txt for license information.                  *|
+|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception                    *|
 |*                                                                            *|
 |*===----------------------------------------------------------------------===*|
 |*                                                                            *|
diff --git a/include/llvm-c/Core.h b/include/llvm-c/Core.h
index 0784275ace8cb393283c4b3f515662787aa5575d..9c521899c95bc6be4a048854ee66c2dd9fe67916 100644
--- a/include/llvm-c/Core.h
+++ b/include/llvm-c/Core.h
@@ -1,9 +1,9 @@
 /*===-- llvm-c/Core.h - Core Library C Interface ------------------*- C -*-===*\
 |*                                                                            *|
-|*                     The LLVM Compiler Infrastructure                       *|
-|*                                                                            *|
-|* This file is distributed under the University of Illinois Open Source      *|
-|* License. See LICENSE.TXT for details.                                      *|
+|* Part of the LLVM Project, under the Apache License v2.0 with LLVM          *|
+|* Exceptions.                                                                *|
+|* See https://llvm.org/LICENSE.txt for license information.                  *|
+|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception                    *|
 |*                                                                            *|
 |*===----------------------------------------------------------------------===*|
 |*                                                                            *|
@@ -65,6 +65,7 @@ typedef enum {
   LLVMInvoke         = 5,
   /* removed 6 due to API changes */
   LLVMUnreachable    = 7,
+  LLVMCallBr         = 67,
 
   /* Standard Unary Operators */
   LLVMFNeg           = 66,
@@ -520,6 +521,23 @@ void *LLVMContextGetDiagnosticContext(LLVMContextRef C);
 void LLVMContextSetYieldCallback(LLVMContextRef C, LLVMYieldCallback Callback,
                                  void *OpaqueHandle);
 
+/**
+ * Retrieve whether the given context is set to discard all value names.
+ *
+ * @see LLVMContext::shouldDiscardValueNames()
+ */
+LLVMBool LLVMContextShouldDiscardValueNames(LLVMContextRef C);
+
+/**
+ * Set whether the given context discards all value names.
+ *
+ * If true, only the names of GlobalValue objects will be available in the IR.
+ * This can be used to save memory and runtime, especially in release mode.
+ *
+ * @see LLVMContext::setDiscardValueNames()
+ */
+void LLVMContextSetDiscardValueNames(LLVMContextRef C, LLVMBool Discard);
+
 /**
  * Destroy a context instance.
  *
@@ -2068,9 +2086,14 @@ LLVMValueRef LLVMConstLShr(LLVMValueRef LHSConstant, LLVMValueRef RHSConstant);
 LLVMValueRef LLVMConstAShr(LLVMValueRef LHSConstant, LLVMValueRef RHSConstant);
 LLVMValueRef LLVMConstGEP(LLVMValueRef ConstantVal,
                           LLVMValueRef *ConstantIndices, unsigned NumIndices);
+LLVMValueRef LLVMConstGEP2(LLVMTypeRef Ty, LLVMValueRef ConstantVal,
+                           LLVMValueRef *ConstantIndices, unsigned NumIndices);
 LLVMValueRef LLVMConstInBoundsGEP(LLVMValueRef ConstantVal,
                                   LLVMValueRef *ConstantIndices,
                                   unsigned NumIndices);
+LLVMValueRef LLVMConstInBoundsGEP2(LLVMTypeRef Ty, LLVMValueRef ConstantVal,
+                                   LLVMValueRef *ConstantIndices,
+                                   unsigned NumIndices);
 LLVMValueRef LLVMConstTrunc(LLVMValueRef ConstantVal, LLVMTypeRef ToType);
 LLVMValueRef LLVMConstSExt(LLVMValueRef ConstantVal, LLVMTypeRef ToType);
 LLVMValueRef LLVMConstZExt(LLVMValueRef ConstantVal, LLVMTypeRef ToType);
@@ -2585,6 +2608,103 @@ LLVMValueRef LLVMGetPreviousParam(LLVMValueRef Arg);
  */
 void LLVMSetParamAlignment(LLVMValueRef Arg, unsigned Align);
 
+/**
+ * @}
+ */
+
+/**
+ * @defgroup LLVMCCoreValueGlobalIFunc IFuncs
+ *
+ * Functions in this group relate to indirect functions.
+ *
+ * Functions in this group expect LLVMValueRef instances that correspond
+ * to llvm::GlobalIFunc instances.
+ *
+ * @{
+ */
+
+/**
+ * Add a global indirect function to a module under a specified name.
+ *
+ * @see llvm::GlobalIFunc::create()
+ */
+LLVMValueRef LLVMAddGlobalIFunc(LLVMModuleRef M,
+                                const char *Name, size_t NameLen,
+                                LLVMTypeRef Ty, unsigned AddrSpace,
+                                LLVMValueRef Resolver);
+
+/**
+ * Obtain a GlobalIFunc value from a Module by its name.
+ *
+ * The returned value corresponds to a llvm::GlobalIFunc value.
+ *
+ * @see llvm::Module::getNamedIFunc()
+ */
+LLVMValueRef LLVMGetNamedGlobalIFunc(LLVMModuleRef M,
+                                     const char *Name, size_t NameLen);
+
+/**
+ * Obtain an iterator to the first GlobalIFunc in a Module.
+ *
+ * @see llvm::Module::ifunc_begin()
+ */
+LLVMValueRef LLVMGetFirstGlobalIFunc(LLVMModuleRef M);
+
+/**
+ * Obtain an iterator to the last GlobalIFunc in a Module.
+ *
+ * @see llvm::Module::ifunc_end()
+ */
+LLVMValueRef LLVMGetLastGlobalIFunc(LLVMModuleRef M);
+
+/**
+ * Advance a GlobalIFunc iterator to the next GlobalIFunc.
+ *
+ * Returns NULL if the iterator was already at the end and there are no more
+ * global aliases.
+ */
+LLVMValueRef LLVMGetNextGlobalIFunc(LLVMValueRef IFunc);
+
+/**
+ * Decrement a GlobalIFunc iterator to the previous GlobalIFunc.
+ *
+ * Returns NULL if the iterator was already at the beginning and there are
+ * no previous global aliases.
+ */
+LLVMValueRef LLVMGetPreviousGlobalIFunc(LLVMValueRef IFunc);
+  
+/**
+ * Retrieves the resolver function associated with this indirect function, or
+ * NULL if it doesn't not exist.
+ *
+ * @see llvm::GlobalIFunc::getResolver()
+ */
+LLVMValueRef LLVMGetGlobalIFuncResolver(LLVMValueRef IFunc);
+
+/**
+ * Sets the resolver function associated with this indirect function.
+ *
+ * @see llvm::GlobalIFunc::setResolver()
+ */
+void LLVMSetGlobalIFuncResolver(LLVMValueRef IFunc, LLVMValueRef Resolver);
+
+/**
+ * Remove a global indirect function from its parent module and delete it.
+ *
+ * @see llvm::GlobalIFunc::eraseFromParent()
+ */
+void LLVMEraseGlobalIFunc(LLVMValueRef IFunc);
+
+/**
+ * Remove a global indirect function from its parent module.
+ *
+ * This unlinks the global indirect function from its containing module but
+ * keeps it alive.
+ *
+ * @see llvm::GlobalIFunc::removeFromParent()
+ */
+void LLVMRemoveGlobalIFunc(LLVMValueRef IFunc);
+
 /**
  * @}
  */
@@ -2789,6 +2909,14 @@ LLVMBasicBlockRef LLVMGetPreviousBasicBlock(LLVMBasicBlockRef BB);
  */
 LLVMBasicBlockRef LLVMGetEntryBasicBlock(LLVMValueRef Fn);
 
+/**
+ * Create a new basic block without inserting it into a function.
+ *
+ * @see llvm::BasicBlock::Create()
+ */
+LLVMBasicBlockRef LLVMCreateBasicBlockInContext(LLVMContextRef C,
+                                                const char *Name);
+
 /**
  * Append a basic block to the end of a function.
  *
@@ -3074,6 +3202,13 @@ void LLVMRemoveCallSiteEnumAttribute(LLVMValueRef C, LLVMAttributeIndex Idx,
 void LLVMRemoveCallSiteStringAttribute(LLVMValueRef C, LLVMAttributeIndex Idx,
                                        const char *K, unsigned KLen);
 
+/**
+ * Obtain the function type called by this instruction.
+ *
+ * @see llvm::CallBase::getFunctionType()
+ */
+LLVMTypeRef LLVMGetCalledFunctionType(LLVMValueRef C);
+
 /**
  * Obtain the pointer to the function invoked by this instruction.
  *
@@ -3366,10 +3501,16 @@ LLVMValueRef LLVMBuildSwitch(LLVMBuilderRef, LLVMValueRef V,
                              LLVMBasicBlockRef Else, unsigned NumCases);
 LLVMValueRef LLVMBuildIndirectBr(LLVMBuilderRef B, LLVMValueRef Addr,
                                  unsigned NumDests);
+// LLVMBuildInvoke is deprecated in favor of LLVMBuildInvoke2, in preparation
+// for opaque pointer types.
 LLVMValueRef LLVMBuildInvoke(LLVMBuilderRef, LLVMValueRef Fn,
                              LLVMValueRef *Args, unsigned NumArgs,
                              LLVMBasicBlockRef Then, LLVMBasicBlockRef Catch,
                              const char *Name);
+LLVMValueRef LLVMBuildInvoke2(LLVMBuilderRef, LLVMTypeRef Ty, LLVMValueRef Fn,
+                              LLVMValueRef *Args, unsigned NumArgs,
+                              LLVMBasicBlockRef Then, LLVMBasicBlockRef Catch,
+                              const char *Name);
 LLVMValueRef LLVMBuildUnreachable(LLVMBuilderRef);
 
 /* Exception Handling */
@@ -3559,9 +3700,15 @@ LLVMValueRef LLVMBuildAlloca(LLVMBuilderRef, LLVMTypeRef Ty, const char *Name);
 LLVMValueRef LLVMBuildArrayAlloca(LLVMBuilderRef, LLVMTypeRef Ty,
                                   LLVMValueRef Val, const char *Name);
 LLVMValueRef LLVMBuildFree(LLVMBuilderRef, LLVMValueRef PointerVal);
+// LLVMBuildLoad is deprecated in favor of LLVMBuildLoad2, in preparation for
+// opaque pointer types.
 LLVMValueRef LLVMBuildLoad(LLVMBuilderRef, LLVMValueRef PointerVal,
                            const char *Name);
+LLVMValueRef LLVMBuildLoad2(LLVMBuilderRef, LLVMTypeRef Ty,
+                            LLVMValueRef PointerVal, const char *Name);
 LLVMValueRef LLVMBuildStore(LLVMBuilderRef, LLVMValueRef Val, LLVMValueRef Ptr);
+// LLVMBuildGEP, LLVMBuildInBoundsGEP, and LLVMBuildStructGEP are deprecated in
+// favor of LLVMBuild*GEP2, in preparation for opaque pointer types.
 LLVMValueRef LLVMBuildGEP(LLVMBuilderRef B, LLVMValueRef Pointer,
                           LLVMValueRef *Indices, unsigned NumIndices,
                           const char *Name);
@@ -3570,6 +3717,15 @@ LLVMValueRef LLVMBuildInBoundsGEP(LLVMBuilderRef B, LLVMValueRef Pointer,
                                   const char *Name);
 LLVMValueRef LLVMBuildStructGEP(LLVMBuilderRef B, LLVMValueRef Pointer,
                                 unsigned Idx, const char *Name);
+LLVMValueRef LLVMBuildGEP2(LLVMBuilderRef B, LLVMTypeRef Ty,
+                           LLVMValueRef Pointer, LLVMValueRef *Indices,
+                           unsigned NumIndices, const char *Name);
+LLVMValueRef LLVMBuildInBoundsGEP2(LLVMBuilderRef B, LLVMTypeRef Ty,
+                                   LLVMValueRef Pointer, LLVMValueRef *Indices,
+                                   unsigned NumIndices, const char *Name);
+LLVMValueRef LLVMBuildStructGEP2(LLVMBuilderRef B, LLVMTypeRef Ty,
+                                 LLVMValueRef Pointer, unsigned Idx,
+                                 const char *Name);
 LLVMValueRef LLVMBuildGlobalString(LLVMBuilderRef B, const char *Str,
                                    const char *Name);
 LLVMValueRef LLVMBuildGlobalStringPtr(LLVMBuilderRef B, const char *Str,
@@ -3616,11 +3772,16 @@ LLVMValueRef LLVMBuildCast(LLVMBuilderRef B, LLVMOpcode Op, LLVMValueRef Val,
                            LLVMTypeRef DestTy, const char *Name);
 LLVMValueRef LLVMBuildPointerCast(LLVMBuilderRef, LLVMValueRef Val,
                                   LLVMTypeRef DestTy, const char *Name);
-LLVMValueRef LLVMBuildIntCast(LLVMBuilderRef, LLVMValueRef Val, /*Signed cast!*/
-                              LLVMTypeRef DestTy, const char *Name);
+LLVMValueRef LLVMBuildIntCast2(LLVMBuilderRef, LLVMValueRef Val,
+                               LLVMTypeRef DestTy, LLVMBool IsSigned,
+                               const char *Name);
 LLVMValueRef LLVMBuildFPCast(LLVMBuilderRef, LLVMValueRef Val,
                              LLVMTypeRef DestTy, const char *Name);
 
+/** Deprecated: This cast is always signed. Use LLVMBuildIntCast2 instead. */
+LLVMValueRef LLVMBuildIntCast(LLVMBuilderRef, LLVMValueRef Val, /*Signed cast!*/
+                              LLVMTypeRef DestTy, const char *Name);
+
 /* Comparisons */
 LLVMValueRef LLVMBuildICmp(LLVMBuilderRef, LLVMIntPredicate Op,
                            LLVMValueRef LHS, LLVMValueRef RHS,
@@ -3631,9 +3792,14 @@ LLVMValueRef LLVMBuildFCmp(LLVMBuilderRef, LLVMRealPredicate Op,
 
 /* Miscellaneous instructions */
 LLVMValueRef LLVMBuildPhi(LLVMBuilderRef, LLVMTypeRef Ty, const char *Name);
+// LLVMBuildCall is deprecated in favor of LLVMBuildCall2, in preparation for
+// opaque pointer types.
 LLVMValueRef LLVMBuildCall(LLVMBuilderRef, LLVMValueRef Fn,
                            LLVMValueRef *Args, unsigned NumArgs,
                            const char *Name);
+LLVMValueRef LLVMBuildCall2(LLVMBuilderRef, LLVMTypeRef, LLVMValueRef Fn,
+                            LLVMValueRef *Args, unsigned NumArgs,
+                            const char *Name);
 LLVMValueRef LLVMBuildSelect(LLVMBuilderRef, LLVMValueRef If,
                              LLVMValueRef Then, LLVMValueRef Else,
                              const char *Name);
diff --git a/include/llvm-c/DataTypes.h b/include/llvm-c/DataTypes.h
index 7081c83ffc2b60b11e0358e10e2b86568246ad92..893b22b49ffca9dba9bf14c5fc512674ec3a1bfa 100644
--- a/include/llvm-c/DataTypes.h
+++ b/include/llvm-c/DataTypes.h
@@ -1,9 +1,9 @@
 /*===-- include/llvm-c/DataTypes.h - Define fixed size types ------*- C -*-===*\
 |*                                                                            *|
-|*                     The LLVM Compiler Infrastructure                       *|
-|*                                                                            *|
-|* This file is distributed under the University of Illinois Open Source      *|
-|* License. See LICENSE.TXT for details.                                      *|
+|* Part of the LLVM Project, under the Apache License v2.0 with LLVM          *|
+|* Exceptions.                                                                *|
+|* See https://llvm.org/LICENSE.txt for license information.                  *|
+|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception                    *|
 |*                                                                            *|
 |*===----------------------------------------------------------------------===*|
 |*                                                                            *|
diff --git a/include/llvm-c/DebugInfo.h b/include/llvm-c/DebugInfo.h
index bc341da3d521d939ca37fb6bc160c637f801b470..f840ce9055707946f013d5859ff9448e52454a7a 100644
--- a/include/llvm-c/DebugInfo.h
+++ b/include/llvm-c/DebugInfo.h
@@ -1,9 +1,8 @@
 //===------------ DebugInfo.h - LLVM C API Debug Info API -----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
@@ -54,7 +53,8 @@ typedef enum {
   LLVMDIFlagMainSubprogram = 1 << 21,
   LLVMDIFlagTypePassByValue = 1 << 22,
   LLVMDIFlagTypePassByReference = 1 << 23,
-  LLVMDIFlagFixedEnum = 1 << 24,
+  LLVMDIFlagEnumClass = 1 << 24,
+  LLVMDIFlagFixedEnum = LLVMDIFlagEnumClass, // Deprecated.
   LLVMDIFlagThunk = 1 << 25,
   LLVMDIFlagTrivial = 1 << 26,
   LLVMDIFlagBigEndian = 1 << 27,
diff --git a/include/llvm-c/Disassembler.h b/include/llvm-c/Disassembler.h
index 5e80b95848cf49f7d09bb6781973797876867920..3adcc3c47a3f568535904b31ff9f7f965b2d43f2 100644
--- a/include/llvm-c/Disassembler.h
+++ b/include/llvm-c/Disassembler.h
@@ -1,9 +1,9 @@
 /*===-- llvm-c/Disassembler.h - Disassembler Public C Interface ---*- C -*-===*\
 |*                                                                            *|
-|*                     The LLVM Compiler Infrastructure                       *|
-|*                                                                            *|
-|* This file is distributed under the University of Illinois Open Source      *|
-|* License. See LICENSE.TXT for details.                                      *|
+|* Part of the LLVM Project, under the Apache License v2.0 with LLVM          *|
+|* Exceptions.                                                                *|
+|* See https://llvm.org/LICENSE.txt for license information.                  *|
+|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception                    *|
 |*                                                                            *|
 |*===----------------------------------------------------------------------===*|
 |*                                                                            *|
diff --git a/include/llvm-c/DisassemblerTypes.h b/include/llvm-c/DisassemblerTypes.h
index e8754ac770555fa18e98cd62c519c6c5a988f58d..389e5ee454a8eecab57ff3329c202b415ee5b96b 100644
--- a/include/llvm-c/DisassemblerTypes.h
+++ b/include/llvm-c/DisassemblerTypes.h
@@ -1,9 +1,9 @@
 /*===-- llvm-c/DisassemblerTypedefs.h -----------------------------*- C -*-===*\
 |*                                                                            *|
-|*                     The LLVM Compiler Infrastructure                       *|
-|*                                                                            *|
-|* This file is distributed under the University of Illinois Open Source      *|
-|* License. See LICENSE.TXT for details.                                      *|
+|* Part of the LLVM Project, under the Apache License v2.0 with LLVM          *|
+|* Exceptions.                                                                *|
+|* See https://llvm.org/LICENSE.txt for license information.                  *|
+|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception                    *|
 |*                                                                            *|
 |*===----------------------------------------------------------------------===*/
 
diff --git a/include/llvm-c/Error.h b/include/llvm-c/Error.h
index 71e84661222bf39f2906703f8c9f32fa94a82591..40ea1fb1ff88f51d82d07affb9a2dda445e36c51 100644
--- a/include/llvm-c/Error.h
+++ b/include/llvm-c/Error.h
@@ -1,9 +1,9 @@
 /*===------- llvm-c/Error.h - llvm::Error class C Interface -------*- C -*-===*\
 |*                                                                            *|
-|*                     The LLVM Compiler Infrastructure                       *|
-|*                                                                            *|
-|* This file is distributed under the University of Illinois Open Source      *|
-|* License. See LICENSE.TXT for details.                                      *|
+|* Part of the LLVM Project, under the Apache License v2.0 with LLVM          *|
+|* Exceptions.                                                                *|
+|* See https://llvm.org/LICENSE.txt for license information.                  *|
+|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception                    *|
 |*                                                                            *|
 |*===----------------------------------------------------------------------===*|
 |*                                                                            *|
diff --git a/include/llvm-c/ErrorHandling.h b/include/llvm-c/ErrorHandling.h
index 2059b3aeb15875bd6e107aa29bf38bed563e7e9c..4927349d898352e682a436fe1df2242891d06d5f 100644
--- a/include/llvm-c/ErrorHandling.h
+++ b/include/llvm-c/ErrorHandling.h
@@ -1,9 +1,9 @@
 /*===-- llvm-c/ErrorHandling.h - Error Handling C Interface -------*- C -*-===*\
 |*                                                                            *|
-|*                     The LLVM Compiler Infrastructure                       *|
-|*                                                                            *|
-|* This file is distributed under the University of Illinois Open Source      *|
-|* License. See LICENSE.TXT for details.                                      *|
+|* Part of the LLVM Project, under the Apache License v2.0 with LLVM          *|
+|* Exceptions.                                                                *|
+|* See https://llvm.org/LICENSE.txt for license information.                  *|
+|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception                    *|
 |*                                                                            *|
 |*===----------------------------------------------------------------------===*|
 |*                                                                            *|
diff --git a/include/llvm-c/ExecutionEngine.h b/include/llvm-c/ExecutionEngine.h
index e8ebef9ab15d8a709046c167a18e80813288ede4..ef714cd0638480b6319d9d1b23b502a6ad94d6d5 100644
--- a/include/llvm-c/ExecutionEngine.h
+++ b/include/llvm-c/ExecutionEngine.h
@@ -1,9 +1,9 @@
 /*===-- llvm-c/ExecutionEngine.h - ExecutionEngine Lib C Iface --*- C++ -*-===*\
 |*                                                                            *|
-|*                     The LLVM Compiler Infrastructure                       *|
-|*                                                                            *|
-|* This file is distributed under the University of Illinois Open Source      *|
-|* License. See LICENSE.TXT for details.                                      *|
+|* Part of the LLVM Project, under the Apache License v2.0 with LLVM          *|
+|* Exceptions.                                                                *|
+|* See https://llvm.org/LICENSE.txt for license information.                  *|
+|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception                    *|
 |*                                                                            *|
 |*===----------------------------------------------------------------------===*|
 |*                                                                            *|
diff --git a/include/llvm-c/IRReader.h b/include/llvm-c/IRReader.h
index 5b58d9921fb0d0aa6567bc3d1061498a312ebdad..4d0b696e9583a318c2caa60780788993ace0ffdd 100644
--- a/include/llvm-c/IRReader.h
+++ b/include/llvm-c/IRReader.h
@@ -1,9 +1,9 @@
 /*===-- llvm-c/IRReader.h - IR Reader C Interface -----------------*- C -*-===*\
 |*                                                                            *|
-|*                     The LLVM Compiler Infrastructure                       *|
-|*                                                                            *|
-|* This file is distributed under the University of Illinois Open Source      *|
-|* License. See LICENSE.TXT for details.                                      *|
+|* Part of the LLVM Project, under the Apache License v2.0 with LLVM          *|
+|* Exceptions.                                                                *|
+|* See https://llvm.org/LICENSE.txt for license information.                  *|
+|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception                    *|
 |*                                                                            *|
 |*===----------------------------------------------------------------------===*|
 |*                                                                            *|
diff --git a/include/llvm-c/Initialization.h b/include/llvm-c/Initialization.h
index e45eafb139f2c00ac4e98d2d52fbcabba7622b0c..36c41dbd8d31d61cc1bd388625142eb515cf6939 100644
--- a/include/llvm-c/Initialization.h
+++ b/include/llvm-c/Initialization.h
@@ -1,9 +1,9 @@
 /*===-- llvm-c/Initialization.h - Initialization C Interface ------*- C -*-===*\
 |*                                                                            *|
-|*                     The LLVM Compiler Infrastructure                       *|
-|*                                                                            *|
-|* This file is distributed under the University of Illinois Open Source      *|
-|* License. See LICENSE.TXT for details.                                      *|
+|* Part of the LLVM Project, under the Apache License v2.0 with LLVM          *|
+|* Exceptions.                                                                *|
+|* See https://llvm.org/LICENSE.txt for license information.                  *|
+|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception                    *|
 |*                                                                            *|
 |*===----------------------------------------------------------------------===*|
 |*                                                                            *|
diff --git a/include/llvm-c/LinkTimeOptimizer.h b/include/llvm-c/LinkTimeOptimizer.h
index 8bcf59969ccb995b8fd5c3fb9bdcecb5cee7b2f5..19b4f5cf74918bfbd321096b0bc2b15c33d63c9d 100644
--- a/include/llvm-c/LinkTimeOptimizer.h
+++ b/include/llvm-c/LinkTimeOptimizer.h
@@ -1,9 +1,8 @@
 //===-- llvm/LinkTimeOptimizer.h - LTO Public C Interface -------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm-c/Linker.h b/include/llvm-c/Linker.h
index d02c37f94c8630fc43cd9d3819e2bc70cc974958..90851304166190bf60d17517e1460f922814c426 100644
--- a/include/llvm-c/Linker.h
+++ b/include/llvm-c/Linker.h
@@ -1,9 +1,9 @@
 /*===-- llvm-c/Linker.h - Module Linker C Interface -------------*- C++ -*-===*\
 |*                                                                            *|
-|*                     The LLVM Compiler Infrastructure                       *|
-|*                                                                            *|
-|* This file is distributed under the University of Illinois Open Source      *|
-|* License. See LICENSE.TXT for details.                                      *|
+|* Part of the LLVM Project, under the Apache License v2.0 with LLVM          *|
+|* Exceptions.                                                                *|
+|* See https://llvm.org/LICENSE.txt for license information.                  *|
+|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception                    *|
 |*                                                                            *|
 |*===----------------------------------------------------------------------===*|
 |*                                                                            *|
diff --git a/include/llvm-c/Object.h b/include/llvm-c/Object.h
index a2980e89fe3dff8202aab4a5383c3e19327938e9..bbea4555a09c1d25f5ee4e261d6aaac200aadd64 100644
--- a/include/llvm-c/Object.h
+++ b/include/llvm-c/Object.h
@@ -1,9 +1,9 @@
 /*===-- llvm-c/Object.h - Object Lib C Iface --------------------*- C++ -*-===*/
 /*                                                                            */
-/*                     The LLVM Compiler Infrastructure                       */
-/*                                                                            */
-/* This file is distributed under the University of Illinois Open Source      */
-/* License. See LICENSE.TXT for details.                                      */
+/* Part of the LLVM Project, under the Apache License v2.0 with LLVM          */
+/* Exceptions.                                                                */
+/* See https://llvm.org/LICENSE.txt for license information.                  */
+/* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception                    */
 /*                                                                            */
 /*===----------------------------------------------------------------------===*/
 /*                                                                            */
diff --git a/include/llvm-c/OptRemarks.h b/include/llvm-c/OptRemarks.h
index 6a90394e711c57962c462f8b17eb85f86146cf1b..b6cffc786c7f348a06a6103e2208e2c40dadb054 100644
--- a/include/llvm-c/OptRemarks.h
+++ b/include/llvm-c/OptRemarks.h
@@ -1,9 +1,9 @@
 /*===-- llvm-c/OptRemarks.h - OptRemarks Public C Interface -------*- C -*-===*\
 |*                                                                            *|
-|*                     The LLVM Compiler Infrastructure                       *|
-|*                                                                            *|
-|* This file is distributed under the University of Illinois Open Source      *|
-|* License. See LICENSE.TXT for details.                                      *|
+|* Part of the LLVM Project, under the Apache License v2.0 with LLVM          *|
+|* Exceptions.                                                                *|
+|* See https://llvm.org/LICENSE.txt for license information.                  *|
+|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception                    *|
 |*                                                                            *|
 |*===----------------------------------------------------------------------===*|
 |*                                                                            *|
diff --git a/include/llvm-c/OrcBindings.h b/include/llvm-c/OrcBindings.h
index 570db87fee947d3ed23ce2c2f72d32ba1b8537cc..9e92371b5a3a37df23847ac0db4f421f67ac2d92 100644
--- a/include/llvm-c/OrcBindings.h
+++ b/include/llvm-c/OrcBindings.h
@@ -1,9 +1,9 @@
 /*===----------- llvm-c/OrcBindings.h - Orc Lib C Iface ---------*- C++ -*-===*\
 |*                                                                            *|
-|*                     The LLVM Compiler Infrastructure                       *|
-|*                                                                            *|
-|* This file is distributed under the University of Illinois Open Source      *|
-|* License. See LICENSE.TXT for details.                                      *|
+|* Part of the LLVM Project, under the Apache License v2.0 with LLVM          *|
+|* Exceptions.                                                                *|
+|* See https://llvm.org/LICENSE.txt for license information.                  *|
+|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception                    *|
 |*                                                                            *|
 |*===----------------------------------------------------------------------===*|
 |*                                                                            *|
diff --git a/include/llvm-c/Support.h b/include/llvm-c/Support.h
index 37d5d72ff5dca3395f902d70c61377170bc9dc31..097f784246c5a2123512bbec5ec6e031324b3d47 100644
--- a/include/llvm-c/Support.h
+++ b/include/llvm-c/Support.h
@@ -1,9 +1,9 @@
 /*===-- llvm-c/Support.h - Support C Interface --------------------*- C -*-===*\
 |*                                                                            *|
-|*                     The LLVM Compiler Infrastructure                       *|
-|*                                                                            *|
-|* This file is distributed under the University of Illinois Open Source      *|
-|* License. See LICENSE.TXT for details.                                      *|
+|* Part of the LLVM Project, under the Apache License v2.0 with LLVM          *|
+|* Exceptions.                                                                *|
+|* See https://llvm.org/LICENSE.txt for license information.                  *|
+|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception                    *|
 |*                                                                            *|
 |*===----------------------------------------------------------------------===*|
 |*                                                                            *|
diff --git a/include/llvm-c/Target.h b/include/llvm-c/Target.h
index 03004ba5eec0c19feee23d41262d5868bde123dd..9a792a0d67be43521d1a529bed869c2feaeae154 100644
--- a/include/llvm-c/Target.h
+++ b/include/llvm-c/Target.h
@@ -1,9 +1,9 @@
 /*===-- llvm-c/Target.h - Target Lib C Iface --------------------*- C++ -*-===*/
 /*                                                                            */
-/*                     The LLVM Compiler Infrastructure                       */
-/*                                                                            */
-/* This file is distributed under the University of Illinois Open Source      */
-/* License. See LICENSE.TXT for details.                                      */
+/* Part of the LLVM Project, under the Apache License v2.0 with LLVM          */
+/* Exceptions.                                                                */
+/* See https://llvm.org/LICENSE.txt for license information.                  */
+/* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception                    */
 /*                                                                            */
 /*===----------------------------------------------------------------------===*/
 /*                                                                            */
diff --git a/include/llvm-c/TargetMachine.h b/include/llvm-c/TargetMachine.h
index bcfa0012c3cf43b519630da76bd968fc57bc38a7..28d7c096871e28e30c02b61be253a091a4bacb70 100644
--- a/include/llvm-c/TargetMachine.h
+++ b/include/llvm-c/TargetMachine.h
@@ -1,9 +1,9 @@
 /*===-- llvm-c/TargetMachine.h - Target Machine Library C Interface - C++ -*-=*\
 |*                                                                            *|
-|*                     The LLVM Compiler Infrastructure                       *|
-|*                                                                            *|
-|* This file is distributed under the University of Illinois Open Source      *|
-|* License. See LICENSE.TXT for details.                                      *|
+|* Part of the LLVM Project, under the Apache License v2.0 with LLVM          *|
+|* Exceptions.                                                                *|
+|* See https://llvm.org/LICENSE.txt for license information.                  *|
+|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception                    *|
 |*                                                                            *|
 |*===----------------------------------------------------------------------===*|
 |*                                                                            *|
@@ -39,7 +39,10 @@ typedef enum {
     LLVMRelocDefault,
     LLVMRelocStatic,
     LLVMRelocPIC,
-    LLVMRelocDynamicNoPic
+    LLVMRelocDynamicNoPic,
+    LLVMRelocROPI,
+    LLVMRelocRWPI,
+    LLVMRelocROPI_RWPI
 } LLVMRelocMode;
 
 typedef enum {
diff --git a/include/llvm-c/Transforms/AggressiveInstCombine.h b/include/llvm-c/Transforms/AggressiveInstCombine.h
index 8756a22e917a5b5b416f24cbe50a4c147110ee26..c0b0141c3da17bd67ffd2fc15ef0935b141d5933 100644
--- a/include/llvm-c/Transforms/AggressiveInstCombine.h
+++ b/include/llvm-c/Transforms/AggressiveInstCombine.h
@@ -1,9 +1,9 @@
 /*===-- AggressiveInstCombine.h ---------------------------------*- C++ -*-===*\
 |*                                                                            *|
-|*                     The LLVM Compiler Infrastructure                       *|
-|*                                                                            *|
-|* This file is distributed under the University of Illinois Open Source      *|
-|* License. See LICENSE.TXT for details.                                      *|
+|* Part of the LLVM Project, under the Apache License v2.0 with LLVM          *|
+|* Exceptions.                                                                *|
+|* See https://llvm.org/LICENSE.txt for license information.                  *|
+|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception                    *|
 |*                                                                            *|
 |*===----------------------------------------------------------------------===*|
 |*                                                                            *|
diff --git a/include/llvm-c/Transforms/Coroutines.h b/include/llvm-c/Transforms/Coroutines.h
index 827e30fb2d7cc639c7fa2547ddd6c8c8725286ab..227e7cf0a360ce5161917a5641cf3d8996fc7868 100644
--- a/include/llvm-c/Transforms/Coroutines.h
+++ b/include/llvm-c/Transforms/Coroutines.h
@@ -1,9 +1,9 @@
 /*===-- Coroutines.h - Coroutines Library C Interface -----------*- C++ -*-===*\
 |*                                                                            *|
-|*                     The LLVM Compiler Infrastructure                       *|
-|*                                                                            *|
-|* This file is distributed under the University of Illinois Open Source      *|
-|* License. See LICENSE.TXT for details.                                      *|
+|* Part of the LLVM Project, under the Apache License v2.0 with LLVM          *|
+|* Exceptions.                                                                *|
+|* See https://llvm.org/LICENSE.txt for license information.                  *|
+|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception                    *|
 |*                                                                            *|
 |*===----------------------------------------------------------------------===*|
 |*                                                                            *|
diff --git a/include/llvm-c/Transforms/IPO.h b/include/llvm-c/Transforms/IPO.h
index 7705b1864dc30070ca2071235d5404b8fa2352f9..7a82ed4641415b7099f117b0646b6c8690875c95 100644
--- a/include/llvm-c/Transforms/IPO.h
+++ b/include/llvm-c/Transforms/IPO.h
@@ -1,9 +1,9 @@
 /*===-- IPO.h - Interprocedural Transformations C Interface -----*- C++ -*-===*\
 |*                                                                            *|
-|*                     The LLVM Compiler Infrastructure                       *|
-|*                                                                            *|
-|* This file is distributed under the University of Illinois Open Source      *|
-|* License. See LICENSE.TXT for details.                                      *|
+|* Part of the LLVM Project, under the Apache License v2.0 with LLVM          *|
+|* Exceptions.                                                                *|
+|* See https://llvm.org/LICENSE.txt for license information.                  *|
+|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception                    *|
 |*                                                                            *|
 |*===----------------------------------------------------------------------===*|
 |*                                                                            *|
diff --git a/include/llvm-c/Transforms/InstCombine.h b/include/llvm-c/Transforms/InstCombine.h
index e1c1572d53dcfa62894bf4e97c631c0ed5808fe4..166f278d9a694fd92a26233cb05d9dc93cb394f7 100644
--- a/include/llvm-c/Transforms/InstCombine.h
+++ b/include/llvm-c/Transforms/InstCombine.h
@@ -1,9 +1,9 @@
 /*===-- Scalar.h - Scalar Transformation Library C Interface ----*- C++ -*-===*\
 |*                                                                            *|
-|*                     The LLVM Compiler Infrastructure                       *|
-|*                                                                            *|
-|* This file is distributed under the University of Illinois Open Source      *|
-|* License. See LICENSE.TXT for details.                                      *|
+|* Part of the LLVM Project, under the Apache License v2.0 with LLVM          *|
+|* Exceptions.                                                                *|
+|* See https://llvm.org/LICENSE.txt for license information.                  *|
+|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception                    *|
 |*                                                                            *|
 |*===----------------------------------------------------------------------===*|
 |*                                                                            *|
diff --git a/include/llvm-c/Transforms/PassManagerBuilder.h b/include/llvm-c/Transforms/PassManagerBuilder.h
index 69786b341ab4c2967e75c9a2fa581c8b7313f4b6..d164c00d49c50b7b333df825b6737d443284cec4 100644
--- a/include/llvm-c/Transforms/PassManagerBuilder.h
+++ b/include/llvm-c/Transforms/PassManagerBuilder.h
@@ -1,9 +1,9 @@
 /*===-- llvm-c/Transform/PassManagerBuilder.h - PMB C Interface ---*- C -*-===*\
 |*                                                                            *|
-|*                     The LLVM Compiler Infrastructure                       *|
-|*                                                                            *|
-|* This file is distributed under the University of Illinois Open Source      *|
-|* License. See LICENSE.TXT for details.                                      *|
+|* Part of the LLVM Project, under the Apache License v2.0 with LLVM          *|
+|* Exceptions.                                                                *|
+|* See https://llvm.org/LICENSE.txt for license information.                  *|
+|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception                    *|
 |*                                                                            *|
 |*===----------------------------------------------------------------------===*|
 |*                                                                            *|
diff --git a/include/llvm-c/Transforms/Scalar.h b/include/llvm-c/Transforms/Scalar.h
index 3c3bb4eb9b824ad6d84c6f96a79dc52f91641386..031cf98b2df26d7c8db950a21b5f6ebded1953e4 100644
--- a/include/llvm-c/Transforms/Scalar.h
+++ b/include/llvm-c/Transforms/Scalar.h
@@ -1,9 +1,9 @@
 /*===-- Scalar.h - Scalar Transformation Library C Interface ----*- C++ -*-===*\
 |*                                                                            *|
-|*                     The LLVM Compiler Infrastructure                       *|
-|*                                                                            *|
-|* This file is distributed under the University of Illinois Open Source      *|
-|* License. See LICENSE.TXT for details.                                      *|
+|* Part of the LLVM Project, under the Apache License v2.0 with LLVM          *|
+|* Exceptions.                                                                *|
+|* See https://llvm.org/LICENSE.txt for license information.                  *|
+|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception                    *|
 |*                                                                            *|
 |*===----------------------------------------------------------------------===*|
 |*                                                                            *|
diff --git a/include/llvm-c/Transforms/Utils.h b/include/llvm-c/Transforms/Utils.h
index f171f7fbbe3edf65917d4fbd59963d331226c25e..ff288336bd73e41084deef15a3f62e4f9402fcef 100644
--- a/include/llvm-c/Transforms/Utils.h
+++ b/include/llvm-c/Transforms/Utils.h
@@ -1,9 +1,9 @@
 /*===-- Utils.h - Transformation Utils Library C Interface ------*- C++ -*-===*\
 |*                                                                            *|
-|*                     The LLVM Compiler Infrastructure                       *|
-|*                                                                            *|
-|* This file is distributed under the University of Illinois Open Source      *|
-|* License. See LICENSE.TXT for details.                                      *|
+|* Part of the LLVM Project, under the Apache License v2.0 with LLVM          *|
+|* Exceptions.                                                                *|
+|* See https://llvm.org/LICENSE.txt for license information.                  *|
+|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception                    *|
 |*                                                                            *|
 |*===----------------------------------------------------------------------===*|
 |*                                                                            *|
diff --git a/include/llvm-c/Transforms/Vectorize.h b/include/llvm-c/Transforms/Vectorize.h
index e3f9961acfb12a3eb4793e0fafd27d443e4daf6a..e383481fe4f417756822ecc46becf41515b9e71b 100644
--- a/include/llvm-c/Transforms/Vectorize.h
+++ b/include/llvm-c/Transforms/Vectorize.h
@@ -1,10 +1,10 @@
 /*===---------------------------Vectorize.h --------------------- -*- C -*-===*\
 |*===----------- Vectorization Transformation Library C Interface ---------===*|
 |*                                                                            *|
-|*                     The LLVM Compiler Infrastructure                       *|
-|*                                                                            *|
-|* This file is distributed under the University of Illinois Open Source      *|
-|* License. See LICENSE.TXT for details.                                      *|
+|* Part of the LLVM Project, under the Apache License v2.0 with LLVM          *|
+|* Exceptions.                                                                *|
+|* See https://llvm.org/LICENSE.txt for license information.                  *|
+|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception                    *|
 |*                                                                            *|
 |*===----------------------------------------------------------------------===*|
 |*                                                                            *|
diff --git a/include/llvm-c/Types.h b/include/llvm-c/Types.h
index ce1acf3e04211a8fb69dd70513fe226d90f47d53..a2979bd3beb2fd88508584b20f6092f98dd52cc1 100644
--- a/include/llvm-c/Types.h
+++ b/include/llvm-c/Types.h
@@ -1,9 +1,9 @@
 /*===-- llvm-c/Support.h - C Interface Types declarations ---------*- C -*-===*\
 |*                                                                            *|
-|*                     The LLVM Compiler Infrastructure                       *|
-|*                                                                            *|
-|* This file is distributed under the University of Illinois Open Source      *|
-|* License. See LICENSE.TXT for details.                                      *|
+|* Part of the LLVM Project, under the Apache License v2.0 with LLVM          *|
+|* Exceptions.                                                                *|
+|* See https://llvm.org/LICENSE.txt for license information.                  *|
+|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception                    *|
 |*                                                                            *|
 |*===----------------------------------------------------------------------===*|
 |*                                                                            *|
diff --git a/include/llvm-c/lto.h b/include/llvm-c/lto.h
index 090cd34af4e9388fbf6103d29edac5c8c4a01803..a22f0bbade48c53ee1a596f6bd1e6e345c2af2af 100644
--- a/include/llvm-c/lto.h
+++ b/include/llvm-c/lto.h
@@ -1,9 +1,9 @@
 /*===-- llvm-c/lto.h - LTO Public C Interface ---------------------*- C -*-===*\
 |*                                                                            *|
-|*                     The LLVM Compiler Infrastructure                       *|
-|*                                                                            *|
-|* This file is distributed under the University of Illinois Open Source      *|
-|* License. See LICENSE.TXT for details.                                      *|
+|* Part of the LLVM Project, under the Apache License v2.0 with LLVM          *|
+|* Exceptions.                                                                *|
+|* See https://llvm.org/LICENSE.txt for license information.                  *|
+|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception                    *|
 |*                                                                            *|
 |*===----------------------------------------------------------------------===*|
 |*                                                                            *|
diff --git a/include/llvm/ADT/APFloat.h b/include/llvm/ADT/APFloat.h
index c6fa5ad674f6898e79ab7aaf604ef6ddf3d8f416..62558230691a7327d35189008c30170f534914c5 100644
--- a/include/llvm/ADT/APFloat.h
+++ b/include/llvm/ADT/APFloat.h
@@ -1,9 +1,8 @@
 //===- llvm/ADT/APFloat.h - Arbitrary Precision Floating Point ---*- C++ -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
diff --git a/include/llvm/ADT/APInt.h b/include/llvm/ADT/APInt.h
index 6e106ff8bf5dca7786acc1069e6b2739b18067e9..0f27dbf43ec3b4af0c4af24aa2f6b9c112e3efba 100644
--- a/include/llvm/ADT/APInt.h
+++ b/include/llvm/ADT/APInt.h
@@ -1,9 +1,8 @@
 //===-- llvm/ADT/APInt.h - For Arbitrary Precision Integer -----*- C++ -*--===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
diff --git a/include/llvm/ADT/APSInt.h b/include/llvm/ADT/APSInt.h
index 7ee2c4c62fced9963652357210d88ea445d2b96e..be366116141e24aa055423f10f9de4e138c95f2e 100644
--- a/include/llvm/ADT/APSInt.h
+++ b/include/llvm/ADT/APSInt.h
@@ -1,9 +1,8 @@
 //===-- llvm/ADT/APSInt.h - Arbitrary Precision Signed Int -----*- C++ -*--===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/ADT/AllocatorList.h b/include/llvm/ADT/AllocatorList.h
index 178c6742a87b90d41a75c78ada41e3e51d38cd8b..405a2e4264dfbd2d6c477505ab175e6c5fef41b8 100644
--- a/include/llvm/ADT/AllocatorList.h
+++ b/include/llvm/ADT/AllocatorList.h
@@ -1,9 +1,8 @@
 //===- llvm/ADT/AllocatorList.h - Custom allocator list ---------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/ADT/Any.h b/include/llvm/ADT/Any.h
index 7faa4c963d3d35ad43659102da2364b5e1f62e82..5dcd6e73c54f863a186d5451773c3b062479614f 100644
--- a/include/llvm/ADT/Any.h
+++ b/include/llvm/ADT/Any.h
@@ -1,9 +1,8 @@
 //===- Any.h - Generic type erased holder of any type -----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/ADT/ArrayRef.h b/include/llvm/ADT/ArrayRef.h
index 9cb25b09c6cb398ca5502df1b37400bcc3d37738..75bb7d690cc0e95fe8857171b4342faa0fc6f9e3 100644
--- a/include/llvm/ADT/ArrayRef.h
+++ b/include/llvm/ADT/ArrayRef.h
@@ -1,9 +1,8 @@
 //===- ArrayRef.h - Array Reference Wrapper ---------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -526,12 +525,6 @@ namespace llvm {
 
   /// @}
 
-  // ArrayRefs can be treated like a POD type.
-  template <typename T> struct isPodLike;
-  template <typename T> struct isPodLike<ArrayRef<T>> {
-    static const bool value = true;
-  };
-
   template <typename T> hash_code hash_value(ArrayRef<T> S) {
     return hash_combine_range(S.begin(), S.end());
   }
diff --git a/include/llvm/ADT/BitVector.h b/include/llvm/ADT/BitVector.h
index 9ab1da7c691301018e9260c896a1a184f030513b..fabf5d9cd348d10e8e22e3183f956f4bc71a86ae 100644
--- a/include/llvm/ADT/BitVector.h
+++ b/include/llvm/ADT/BitVector.h
@@ -1,9 +1,8 @@
 //===- llvm/ADT/BitVector.h - Bit vectors -----------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/ADT/BitmaskEnum.h b/include/llvm/ADT/BitmaskEnum.h
index 18c6ba5a3eb8b28d8ed20b0ffbedf4fd1b7c4d8c..1a18bc721b214edcda64a3fc9d6b287a8df617da 100644
--- a/include/llvm/ADT/BitmaskEnum.h
+++ b/include/llvm/ADT/BitmaskEnum.h
@@ -1,9 +1,8 @@
 //===-- llvm/ADT/BitmaskEnum.h ----------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/ADT/BreadthFirstIterator.h b/include/llvm/ADT/BreadthFirstIterator.h
index 6bc63c283b0975be2b883a68366147b988275998..cc6c8600e1063af394b5934045b77d79936cee9d 100644
--- a/include/llvm/ADT/BreadthFirstIterator.h
+++ b/include/llvm/ADT/BreadthFirstIterator.h
@@ -1,9 +1,8 @@
 //===- llvm/ADT/BreadthFirstIterator.h - Breadth First iterator -*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/ADT/CachedHashString.h b/include/llvm/ADT/CachedHashString.h
index d8f0e7afdd49c724a017c0e7dd8abe3db11c6ed2..80144fb87e0e4e9b721ea94a0017db134c1754ad 100644
--- a/include/llvm/ADT/CachedHashString.h
+++ b/include/llvm/ADT/CachedHashString.h
@@ -1,9 +1,8 @@
 //===- llvm/ADT/CachedHashString.h - Prehashed string/StringRef -*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/ADT/DAGDeltaAlgorithm.h b/include/llvm/ADT/DAGDeltaAlgorithm.h
index 41fdd43efb8a3aae2d4bb8b1ee3d4e0c0d597c86..d4cdc3c86048feaf475c3ac2cf5928a1e039be8a 100644
--- a/include/llvm/ADT/DAGDeltaAlgorithm.h
+++ b/include/llvm/ADT/DAGDeltaAlgorithm.h
@@ -1,9 +1,8 @@
 //===- DAGDeltaAlgorithm.h - A DAG Minimization Algorithm ------*- C++ -*--===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //===----------------------------------------------------------------------===//
 
 #ifndef LLVM_ADT_DAGDELTAALGORITHM_H
diff --git a/include/llvm/ADT/DeltaAlgorithm.h b/include/llvm/ADT/DeltaAlgorithm.h
index 6becb2a6010446dad8c0ba26e54a26d43dcd5d62..114b95499530290758f4a789ed72efba294e7520 100644
--- a/include/llvm/ADT/DeltaAlgorithm.h
+++ b/include/llvm/ADT/DeltaAlgorithm.h
@@ -1,9 +1,8 @@
 //===- DeltaAlgorithm.h - A Set Minimization Algorithm ---------*- C++ -*--===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //===----------------------------------------------------------------------===//
 
 #ifndef LLVM_ADT_DELTAALGORITHM_H
diff --git a/include/llvm/ADT/DenseMap.h b/include/llvm/ADT/DenseMap.h
index 1f50502fff92bad73eec7b41fc3fec39fd501429..e7cd370df6ff2ef21793119e55f898ac423302ca 100644
--- a/include/llvm/ADT/DenseMap.h
+++ b/include/llvm/ADT/DenseMap.h
@@ -1,9 +1,8 @@
 //===- llvm/ADT/DenseMap.h - Dense probed hash table ------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -146,7 +145,8 @@ public:
     }
 
     const KeyT EmptyKey = getEmptyKey(), TombstoneKey = getTombstoneKey();
-    if (isPodLike<KeyT>::value && isPodLike<ValueT>::value) {
+    if (is_trivially_copyable<KeyT>::value &&
+        is_trivially_copyable<ValueT>::value) {
       // Use a simpler loop when these are trivial types.
       for (BucketT *P = getBuckets(), *E = getBucketsEnd(); P != E; ++P)
         P->getFirst() = EmptyKey;
@@ -422,7 +422,8 @@ protected:
     setNumEntries(other.getNumEntries());
     setNumTombstones(other.getNumTombstones());
 
-    if (isPodLike<KeyT>::value && isPodLike<ValueT>::value)
+    if (is_trivially_copyable<KeyT>::value &&
+        is_trivially_copyable<ValueT>::value)
       memcpy(reinterpret_cast<void *>(getBuckets()), other.getBuckets(),
              getNumBuckets() * sizeof(BucketT));
     else
diff --git a/include/llvm/ADT/DenseMapInfo.h b/include/llvm/ADT/DenseMapInfo.h
index 5d12b424fb3756b15e4783707323483111aeaa86..18d6dffa9a52e594347968f72c14a5f73310f26b 100644
--- a/include/llvm/ADT/DenseMapInfo.h
+++ b/include/llvm/ADT/DenseMapInfo.h
@@ -1,9 +1,8 @@
 //===- llvm/ADT/DenseMapInfo.h - Type traits for DenseMap -------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/ADT/DenseSet.h b/include/llvm/ADT/DenseSet.h
index e85a38587e41dcd97366b64dc0eaeac4a0c97bca..5eaaf2773ac11c43607175399a079ca44c0fee71 100644
--- a/include/llvm/ADT/DenseSet.h
+++ b/include/llvm/ADT/DenseSet.h
@@ -1,9 +1,8 @@
 //===- llvm/ADT/DenseSet.h - Dense probed hash table ------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/ADT/DepthFirstIterator.h b/include/llvm/ADT/DepthFirstIterator.h
index 1f3766d3c9de03c88df4c1e6dfd91af5c025880d..11967f5eefcc13e8d70af6fe942dc8044436671a 100644
--- a/include/llvm/ADT/DepthFirstIterator.h
+++ b/include/llvm/ADT/DepthFirstIterator.h
@@ -1,9 +1,8 @@
 //===- llvm/ADT/DepthFirstIterator.h - Depth First iterator -----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/ADT/EpochTracker.h b/include/llvm/ADT/EpochTracker.h
index 49ef192364e8afb78f3b38f819d05d2e79a9d85d..a782b4756898e36f6725c635d3c990fdc54d3b21 100644
--- a/include/llvm/ADT/EpochTracker.h
+++ b/include/llvm/ADT/EpochTracker.h
@@ -1,9 +1,8 @@
 //===- llvm/ADT/EpochTracker.h - ADT epoch tracking --------------*- C++ -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/ADT/EquivalenceClasses.h b/include/llvm/ADT/EquivalenceClasses.h
index e3f48433c69fd910be19edd28a5a900a3658d2a2..2cb7108c07948876ed9ddc5c2eedc00fed256c36 100644
--- a/include/llvm/ADT/EquivalenceClasses.h
+++ b/include/llvm/ADT/EquivalenceClasses.h
@@ -1,9 +1,8 @@
 //===- llvm/ADT/EquivalenceClasses.h - Generic Equiv. Classes ---*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/ADT/FoldingSet.h b/include/llvm/ADT/FoldingSet.h
index e363e69d032aa13a56be35424e893e21f37a7dbd..d5837e51bcfcd576ce06d2f3fb260483de37bf0f 100644
--- a/include/llvm/ADT/FoldingSet.h
+++ b/include/llvm/ADT/FoldingSet.h
@@ -1,9 +1,8 @@
 //===- llvm/ADT/FoldingSet.h - Uniquing Hash Set ----------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/ADT/FunctionExtras.h b/include/llvm/ADT/FunctionExtras.h
index 2b75dc6ac219c255138610637bf4a5963134d67d..121aa527a5dac4518942e8277869116a9957ba3f 100644
--- a/include/llvm/ADT/FunctionExtras.h
+++ b/include/llvm/ADT/FunctionExtras.h
@@ -1,9 +1,8 @@
 //===- FunctionExtras.h - Function type erasure utilities -------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/include/llvm/ADT/GraphTraits.h b/include/llvm/ADT/GraphTraits.h
index d39b50fdc488a19a4f99dfecddf142300daba62e..3ce91225d80d54a9cda003381c8847eee5c6bf69 100644
--- a/include/llvm/ADT/GraphTraits.h
+++ b/include/llvm/ADT/GraphTraits.h
@@ -1,9 +1,8 @@
 //===- llvm/ADT/GraphTraits.h - Graph traits template -----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/ADT/Hashing.h b/include/llvm/ADT/Hashing.h
index 9175c545b7c911d7e0d1c20acefb4f8bfa5f69ba..f639aa2054c4f9ae42174920d23d80221ffcfd9e 100644
--- a/include/llvm/ADT/Hashing.h
+++ b/include/llvm/ADT/Hashing.h
@@ -1,9 +1,8 @@
 //===-- llvm/ADT/Hashing.h - Utilities for hashing --------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/ADT/ImmutableList.h b/include/llvm/ADT/ImmutableList.h
index 0541dc2566ed8e240f662c3ee46f3bfbaab798d9..c9ee494734e7f2d1f721b8b81552154205647887 100644
--- a/include/llvm/ADT/ImmutableList.h
+++ b/include/llvm/ADT/ImmutableList.h
@@ -1,9 +1,8 @@
 //==--- ImmutableList.h - Immutable (functional) list interface --*- C++ -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -242,10 +241,6 @@ template<typename T> struct DenseMapInfo<ImmutableList<T>> {
   }
 };
 
-template <typename T> struct isPodLike;
-template <typename T>
-struct isPodLike<ImmutableList<T>> { static const bool value = true; };
-
 } // end namespace llvm
 
 #endif // LLVM_ADT_IMMUTABLELIST_H
diff --git a/include/llvm/ADT/ImmutableMap.h b/include/llvm/ADT/ImmutableMap.h
index cbc27ff17ccf363e85c28ec39fb5a64067358f97..86fd7fefaec3de7be394ec0a42ac45b283505155 100644
--- a/include/llvm/ADT/ImmutableMap.h
+++ b/include/llvm/ADT/ImmutableMap.h
@@ -1,9 +1,8 @@
 //===--- ImmutableMap.h - Immutable (functional) map interface --*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/ADT/ImmutableSet.h b/include/llvm/ADT/ImmutableSet.h
index b1d5f4ac42e42887f711adef9fb7286b8087e9d5..5871054315334740ff0330e84d28f2639fab1658 100644
--- a/include/llvm/ADT/ImmutableSet.h
+++ b/include/llvm/ADT/ImmutableSet.h
@@ -1,9 +1,8 @@
 //===--- ImmutableSet.h - Immutable (functional) set interface --*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/ADT/IndexedMap.h b/include/llvm/ADT/IndexedMap.h
index 2ee80d2cde63a9c67ce7cc370d54be5d6b39fe04..b44f16b91d76be7dbffa234726d775e8b29861a9 100644
--- a/include/llvm/ADT/IndexedMap.h
+++ b/include/llvm/ADT/IndexedMap.h
@@ -1,9 +1,8 @@
 //===- llvm/ADT/IndexedMap.h - An index map implementation ------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/ADT/IntEqClasses.h b/include/llvm/ADT/IntEqClasses.h
index 0baee2f11a79daea1322bfaedc432db932eeea1d..08f46a3079ef97a0b7ae7f755d1f37daadf055d9 100644
--- a/include/llvm/ADT/IntEqClasses.h
+++ b/include/llvm/ADT/IntEqClasses.h
@@ -1,9 +1,8 @@
 //===-- llvm/ADT/IntEqClasses.h - Equiv. Classes of Integers ----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/ADT/IntervalMap.h b/include/llvm/ADT/IntervalMap.h
index b75c4920a8221aa3f960669b19a70f51cc3f243c..12828c4cfdab8818fccafbf5643046e20936746c 100644
--- a/include/llvm/ADT/IntervalMap.h
+++ b/include/llvm/ADT/IntervalMap.h
@@ -1,9 +1,8 @@
 //===- llvm/ADT/IntervalMap.h - A sorted interval map -----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -1134,6 +1133,19 @@ public:
     I.find(x);
     return I;
   }
+
+  /// overlaps(a, b) - Return true if the intervals in this map overlap with the
+  /// interval [a;b].
+  bool overlaps(KeyT a, KeyT b) {
+    assert(Traits::nonEmpty(a, b));
+    const_iterator I = find(a);
+    if (!I.valid())
+      return false;
+    // [a;b] and [x;y] overlap iff x<=b and a<=y. The find() call guarantees the
+    // second part (y = find(a).stop()), so it is sufficient to check the first
+    // one.
+    return !Traits::stopLess(b, I.start());
+  }
 };
 
 /// treeSafeLookup - Return the mapped value at x or NotFound, assuming a
diff --git a/include/llvm/ADT/IntrusiveRefCntPtr.h b/include/llvm/ADT/IntrusiveRefCntPtr.h
index 430ef86afbd95f1f3103535b8397cdbfa9cf2a51..6d97fe15db8b9ac502393eecd5ef7822a12f986e 100644
--- a/include/llvm/ADT/IntrusiveRefCntPtr.h
+++ b/include/llvm/ADT/IntrusiveRefCntPtr.h
@@ -1,9 +1,8 @@
 //==- llvm/ADT/IntrusiveRefCntPtr.h - Smart Refcounting Pointer --*- C++ -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/ADT/MapVector.h b/include/llvm/ADT/MapVector.h
index 47b4987f210aeb044cc53958b246a62498f9eaad..1de1124f4ea24cadf12aca00a4755dfd8f59e3fd 100644
--- a/include/llvm/ADT/MapVector.h
+++ b/include/llvm/ADT/MapVector.h
@@ -1,9 +1,8 @@
 //===- llvm/ADT/MapVector.h - Map w/ deterministic value order --*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/ADT/None.h b/include/llvm/ADT/None.h
index 4b6bc1e005b5a9d006c948f056be00350d2be651..004ca0ac50ac37975f9e87a016218f78ce6e6329 100644
--- a/include/llvm/ADT/None.h
+++ b/include/llvm/ADT/None.h
@@ -1,9 +1,8 @@
 //===-- None.h - Simple null value for implicit construction ------*- C++ -*-=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/ADT/Optional.h b/include/llvm/ADT/Optional.h
index 76937d632ae13a6f5325deb1c4473578a024d242..25a3185064f413225f0c63db53ea87114657dd75 100644
--- a/include/llvm/ADT/Optional.h
+++ b/include/llvm/ADT/Optional.h
@@ -1,9 +1,8 @@
 //===- Optional.h - Simple variant for passing optional values --*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -27,9 +26,11 @@
 
 namespace llvm {
 
+class raw_ostream;
+
 namespace optional_detail {
 /// Storage for any type.
-template <typename T, bool = isPodLike<T>::value> struct OptionalStorage {
+template <typename T, bool = is_trivially_copyable<T>::value> struct OptionalStorage {
   AlignedCharArrayUnion<T> storage;
   bool hasVal = false;
 
@@ -184,11 +185,6 @@ public:
 #endif
 };
 
-template <typename T> struct isPodLike<Optional<T>> {
-  // An Optional<T> is pod-like if T is.
-  static const bool value = isPodLike<T>::value;
-};
-
 template <typename T, typename U>
 bool operator==(const Optional<T> &X, const Optional<U> &Y) {
   if (X && Y)
@@ -323,6 +319,18 @@ template <typename T> bool operator>=(const T &X, const Optional<T> &Y) {
   return !(X < Y);
 }
 
+raw_ostream &operator<<(raw_ostream &OS, NoneType);
+
+template <typename T, typename = decltype(std::declval<raw_ostream &>()
+                                          << std::declval<const T &>())>
+raw_ostream &operator<<(raw_ostream &OS, const Optional<T> &O) {
+  if (O)
+    OS << *O;
+  else
+    OS << None;
+  return OS;
+}
+
 } // end namespace llvm
 
 #endif // LLVM_ADT_OPTIONAL_H
diff --git a/include/llvm/ADT/PackedVector.h b/include/llvm/ADT/PackedVector.h
index 3d53c49536d065d88479b31e551fdf80ec22c2c6..ae7f8cc8574361c649d914e3563b00839ff20c10 100644
--- a/include/llvm/ADT/PackedVector.h
+++ b/include/llvm/ADT/PackedVector.h
@@ -1,9 +1,8 @@
 //===- llvm/ADT/PackedVector.h - Packed values vector -----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/ADT/PointerEmbeddedInt.h b/include/llvm/ADT/PointerEmbeddedInt.h
index ab4e1048a5bc90d536d6d42276ed3fbc72b92f5f..3eb6edb034303918ec7a2f69e5b4c5f85f2e9b57 100644
--- a/include/llvm/ADT/PointerEmbeddedInt.h
+++ b/include/llvm/ADT/PointerEmbeddedInt.h
@@ -1,9 +1,8 @@
 //===- llvm/ADT/PointerEmbeddedInt.h ----------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/ADT/PointerIntPair.h b/include/llvm/ADT/PointerIntPair.h
index 884d05155bffa84ea68c584cdbe791435c0e6874..24a2bb67a36e2cbeee28dc2ddbf5815c72d55021 100644
--- a/include/llvm/ADT/PointerIntPair.h
+++ b/include/llvm/ADT/PointerIntPair.h
@@ -1,9 +1,8 @@
 //===- llvm/ADT/PointerIntPair.h - Pair for pointer and int -----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -15,6 +14,7 @@
 #define LLVM_ADT_POINTERINTPAIR_H
 
 #include "llvm/Support/PointerLikeTypeTraits.h"
+#include "llvm/Support/type_traits.h"
 #include <cassert>
 #include <cstdint>
 #include <limits>
@@ -42,6 +42,8 @@ template <typename PointerTy, unsigned IntBits, typename IntType = unsigned,
           typename PtrTraits = PointerLikeTypeTraits<PointerTy>,
           typename Info = PointerIntPairInfo<PointerTy, IntBits, PtrTraits>>
 class PointerIntPair {
+  // Used by MSVC visualizer and generally helpful for debugging/visualizing.
+  using InfoTy = Info;
   intptr_t Value = 0;
 
 public:
@@ -124,6 +126,19 @@ public:
   }
 };
 
+// Specialize is_trivially_copyable to avoid limitation of llvm::is_trivially_copyable
+// when compiled with gcc 4.9.
+template <typename PointerTy, unsigned IntBits, typename IntType,
+          typename PtrTraits,
+          typename Info>
+struct is_trivially_copyable<PointerIntPair<PointerTy, IntBits, IntType, PtrTraits, Info>> : std::true_type {
+#ifdef HAVE_STD_IS_TRIVIALLY_COPYABLE
+  static_assert(std::is_trivially_copyable<PointerIntPair<PointerTy, IntBits, IntType, PtrTraits, Info>>::value,
+                "inconsistent behavior between llvm:: and std:: implementation of is_trivially_copyable");
+#endif
+};
+
+
 template <typename PointerT, unsigned IntBits, typename PtrTraits>
 struct PointerIntPairInfo {
   static_assert(PtrTraits::NumLowBitsAvailable <
@@ -174,12 +189,6 @@ struct PointerIntPairInfo {
   }
 };
 
-template <typename T> struct isPodLike;
-template <typename PointerTy, unsigned IntBits, typename IntType>
-struct isPodLike<PointerIntPair<PointerTy, IntBits, IntType>> {
-  static const bool value = true;
-};
-
 // Provide specialization of DenseMapInfo for PointerIntPair.
 template <typename PointerTy, unsigned IntBits, typename IntType>
 struct DenseMapInfo<PointerIntPair<PointerTy, IntBits, IntType>> {
diff --git a/include/llvm/ADT/PointerSumType.h b/include/llvm/ADT/PointerSumType.h
index a19e45a462185b294c87f50c45cedb81e763790e..d467f83f58ac892757d8f9c949f8d1bfafd9c894 100644
--- a/include/llvm/ADT/PointerSumType.h
+++ b/include/llvm/ADT/PointerSumType.h
@@ -1,9 +1,8 @@
 //===- llvm/ADT/PointerSumType.h --------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/ADT/PointerUnion.h b/include/llvm/ADT/PointerUnion.h
index 315e58336cba33300ee6f88fa061bbb39be0f77f..0605429aa8aa02324857a7ec46380f88345d5159 100644
--- a/include/llvm/ADT/PointerUnion.h
+++ b/include/llvm/ADT/PointerUnion.h
@@ -1,9 +1,8 @@
 //===- llvm/ADT/PointerUnion.h - Discriminated Union of 2 Ptrs --*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/ADT/PostOrderIterator.h b/include/llvm/ADT/PostOrderIterator.h
index d77b12228cb15b219a95416dc342c55b319e26a4..2fe7447a8e7789d177a6b8f8cc6386318e20b519 100644
--- a/include/llvm/ADT/PostOrderIterator.h
+++ b/include/llvm/ADT/PostOrderIterator.h
@@ -1,9 +1,8 @@
 //===- llvm/ADT/PostOrderIterator.h - PostOrder iterator --------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/ADT/PriorityQueue.h b/include/llvm/ADT/PriorityQueue.h
index 8ba871e253043fc614cc1c3a21a46d55d447d657..cf79ee10ba7f712b90b2bf3b559b57f6d154dbb9 100644
--- a/include/llvm/ADT/PriorityQueue.h
+++ b/include/llvm/ADT/PriorityQueue.h
@@ -1,9 +1,8 @@
 //===- llvm/ADT/PriorityQueue.h - Priority queues ---------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/ADT/PriorityWorklist.h b/include/llvm/ADT/PriorityWorklist.h
index aa531f3337d9c673de8133a56c3c618f9a94ad2b..96d22c87557e6a0bce8fe3759a106565485d3d70 100644
--- a/include/llvm/ADT/PriorityWorklist.h
+++ b/include/llvm/ADT/PriorityWorklist.h
@@ -1,9 +1,8 @@
 //===- PriorityWorklist.h - Worklist with insertion priority ----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
diff --git a/include/llvm/ADT/SCCIterator.h b/include/llvm/ADT/SCCIterator.h
index ab1dc4613be03dae8d54b7746451f2d9d1479cdc..eb1a5d0938cf1bff87206333a1166a291eac38ad 100644
--- a/include/llvm/ADT/SCCIterator.h
+++ b/include/llvm/ADT/SCCIterator.h
@@ -1,9 +1,8 @@
 //===- ADT/SCCIterator.h - Strongly Connected Comp. Iter. -------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/include/llvm/ADT/STLExtras.h b/include/llvm/ADT/STLExtras.h
index 8685f0e4aada7f1372afb56d54aed2e0da56cc41..9a891d193e687598282176ec830af3f4a715f8b2 100644
--- a/include/llvm/ADT/STLExtras.h
+++ b/include/llvm/ADT/STLExtras.h
@@ -1,9 +1,8 @@
 //===- llvm/ADT/STLExtras.h - Useful STL related functions ------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -75,6 +74,12 @@ template <typename T> struct make_const_ptr {
   using type =
       typename std::add_pointer<typename std::add_const<T>::type>::type;
 };
+
+template <typename T> struct make_const_ref {
+  using type = typename std::add_lvalue_reference<
+      typename std::add_const<T>::type>::type;
+};
+
 //===----------------------------------------------------------------------===//
 //     Extra additions to <functional>
 //===----------------------------------------------------------------------===//
diff --git a/include/llvm/ADT/ScopeExit.h b/include/llvm/ADT/ScopeExit.h
index bd13755fa9999f0797014ad945a452e22b06d10f..712d91237739e99b54ccd790169770ab9215a9e8 100644
--- a/include/llvm/ADT/ScopeExit.h
+++ b/include/llvm/ADT/ScopeExit.h
@@ -1,9 +1,8 @@
 //===- llvm/ADT/ScopeExit.h - Execute code at scope exit --------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/ADT/ScopedHashTable.h b/include/llvm/ADT/ScopedHashTable.h
index 22b0c1bdaf4d0cb0fd4ae3dd6d66782588dea6dd..40c49ebc0be1ffb5c149049345ab7a1abf83f23c 100644
--- a/include/llvm/ADT/ScopedHashTable.h
+++ b/include/llvm/ADT/ScopedHashTable.h
@@ -1,9 +1,8 @@
 //===- ScopedHashTable.h - A simple scoped hash table -----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/ADT/Sequence.h b/include/llvm/ADT/Sequence.h
index 3d4a897bf9a9e7efc1f12d62e8937391a47e56d1..8c505f2010dd2f9d4d369c1071632b23dacfe694 100644
--- a/include/llvm/ADT/Sequence.h
+++ b/include/llvm/ADT/Sequence.h
@@ -1,9 +1,8 @@
 //===- Sequence.h - Utility for producing sequences of values ---*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/include/llvm/ADT/SetOperations.h b/include/llvm/ADT/SetOperations.h
index 7c9f2fbe066e2b9516518b2ddb29b9690406312a..037256a860b2c21f9622468d4e8eb6ba4d905e54 100644
--- a/include/llvm/ADT/SetOperations.h
+++ b/include/llvm/ADT/SetOperations.h
@@ -1,9 +1,8 @@
 //===-- llvm/ADT/SetOperations.h - Generic Set Operations -------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/ADT/SetVector.h b/include/llvm/ADT/SetVector.h
index 3d67810413209f406221a7a5900e8a2a62833e86..d0a0d28d1c813d5c006be44f7fe2ac91088ee441 100644
--- a/include/llvm/ADT/SetVector.h
+++ b/include/llvm/ADT/SetVector.h
@@ -1,9 +1,8 @@
 //===- llvm/ADT/SetVector.h - Set with insert order iteration ---*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/ADT/SmallBitVector.h b/include/llvm/ADT/SmallBitVector.h
index 0a73dbd6067182a66495d57233a62e700685e181..742450e6a951710c91a81bfdbb5aafc5428f3d08 100644
--- a/include/llvm/ADT/SmallBitVector.h
+++ b/include/llvm/ADT/SmallBitVector.h
@@ -1,9 +1,8 @@
 //===- llvm/ADT/SmallBitVector.h - 'Normally small' bit vectors -*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/ADT/SmallPtrSet.h b/include/llvm/ADT/SmallPtrSet.h
index db08e40257ba5d47e30ccce463b6b5088b131310..913518230d2d031251548a46c8d704dfea141436 100644
--- a/include/llvm/ADT/SmallPtrSet.h
+++ b/include/llvm/ADT/SmallPtrSet.h
@@ -1,9 +1,8 @@
 //===- llvm/ADT/SmallPtrSet.h - 'Normally small' pointer set ----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/ADT/SmallSet.h b/include/llvm/ADT/SmallSet.h
index 5d84627714bc34bf9d3cf250bdd65ce2424ea164..6b128c2e2992240d04af4ac46a5d4d080d9c97ea 100644
--- a/include/llvm/ADT/SmallSet.h
+++ b/include/llvm/ADT/SmallSet.h
@@ -1,9 +1,8 @@
 //===- llvm/ADT/SmallSet.h - 'Normally small' sets --------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/ADT/SmallString.h b/include/llvm/ADT/SmallString.h
index ff46e85ccb090281623869a2d5ab8d5e56ce3c45..898be80d0324e409a23f24a950dd1696ec61ed7d 100644
--- a/include/llvm/ADT/SmallString.h
+++ b/include/llvm/ADT/SmallString.h
@@ -1,9 +1,8 @@
 //===- llvm/ADT/SmallString.h - 'Normally small' strings --------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/ADT/SmallVector.h b/include/llvm/ADT/SmallVector.h
index 0636abbb1fbfdc8d2158b0438b747e7cb8585bd9..e3bfb90e1ba752f5424ed3ff097b45b993b1ff27 100644
--- a/include/llvm/ADT/SmallVector.h
+++ b/include/llvm/ADT/SmallVector.h
@@ -1,9 +1,8 @@
 //===- llvm/ADT/SmallVector.h - 'Normally small' vectors --------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -125,13 +124,9 @@ public:
   using const_pointer = const T *;
 
   // forward iterator creation methods.
-  LLVM_ATTRIBUTE_ALWAYS_INLINE
   iterator begin() { return (iterator)this->BeginX; }
-  LLVM_ATTRIBUTE_ALWAYS_INLINE
   const_iterator begin() const { return (const_iterator)this->BeginX; }
-  LLVM_ATTRIBUTE_ALWAYS_INLINE
   iterator end() { return begin() + size(); }
-  LLVM_ATTRIBUTE_ALWAYS_INLINE
   const_iterator end() const { return begin() + size(); }
 
   // reverse iterator creation methods.
@@ -150,12 +145,10 @@ public:
   /// Return a pointer to the vector's buffer, even if empty().
   const_pointer data() const { return const_pointer(begin()); }
 
-  LLVM_ATTRIBUTE_ALWAYS_INLINE
   reference operator[](size_type idx) {
     assert(idx < size());
     return begin()[idx];
   }
-  LLVM_ATTRIBUTE_ALWAYS_INLINE
   const_reference operator[](size_type idx) const {
     assert(idx < size());
     return begin()[idx];
@@ -180,9 +173,9 @@ public:
   }
 };
 
-/// SmallVectorTemplateBase<isPodLike = false> - This is where we put method
+/// SmallVectorTemplateBase<TriviallyCopyable = false> - This is where we put method
 /// implementations that are designed to work with non-POD-like T's.
-template <typename T, bool = isPodLike<T>::value>
+template <typename T, bool = is_trivially_copyable<T>::value>
 class SmallVectorTemplateBase : public SmallVectorTemplateCommon<T> {
 protected:
   SmallVectorTemplateBase(size_t Size) : SmallVectorTemplateCommon<T>(Size) {}
@@ -236,8 +229,8 @@ public:
 };
 
 // Define this out-of-line to dissuade the C++ compiler from inlining it.
-template <typename T, bool isPodLike>
-void SmallVectorTemplateBase<T, isPodLike>::grow(size_t MinSize) {
+template <typename T, bool TriviallyCopyable>
+void SmallVectorTemplateBase<T, TriviallyCopyable>::grow(size_t MinSize) {
   if (MinSize > UINT32_MAX)
     report_bad_alloc_error("SmallVector capacity overflow during allocation");
 
@@ -260,9 +253,8 @@ void SmallVectorTemplateBase<T, isPodLike>::grow(size_t MinSize) {
   this->Capacity = NewCapacity;
 }
 
-
-/// SmallVectorTemplateBase<isPodLike = true> - This is where we put method
-/// implementations that are designed to work with POD-like T's.
+/// SmallVectorTemplateBase<TriviallyCopyable = true> - This is where we put
+/// method implementations that are designed to work with POD-like T's.
 template <typename T>
 class SmallVectorTemplateBase<T, true> : public SmallVectorTemplateCommon<T> {
 protected:
@@ -331,7 +323,7 @@ public:
 protected:
   // Default ctor - Initialize to empty.
   explicit SmallVectorImpl(unsigned N)
-      : SmallVectorTemplateBase<T, isPodLike<T>::value>(N) {}
+      : SmallVectorTemplateBase<T>(N) {}
 
 public:
   SmallVectorImpl(const SmallVectorImpl &) = delete;
diff --git a/include/llvm/ADT/SparseBitVector.h b/include/llvm/ADT/SparseBitVector.h
index 84e73bcbace871d48d47c3b3124fe5bc49cecaac..12850e14f4ed9b272fcff1e17f87d5bacdcece67 100644
--- a/include/llvm/ADT/SparseBitVector.h
+++ b/include/llvm/ADT/SparseBitVector.h
@@ -1,9 +1,8 @@
 //===- llvm/ADT/SparseBitVector.h - Efficient Sparse BitVector --*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/ADT/SparseMultiSet.h b/include/llvm/ADT/SparseMultiSet.h
index 3c8637621510e68c966847f6401ee995cfe5e414..d9d3ff4592679e7c0aec10d67e2fdfc93a84c9cb 100644
--- a/include/llvm/ADT/SparseMultiSet.h
+++ b/include/llvm/ADT/SparseMultiSet.h
@@ -1,9 +1,8 @@
 //===- llvm/ADT/SparseMultiSet.h - Sparse multiset --------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/ADT/SparseSet.h b/include/llvm/ADT/SparseSet.h
index 74cc6dab8c743c46b451c31f528b05854b90bd47..a6eb9b942e804858b6f09a4d0534ea96a5a6a0dc 100644
--- a/include/llvm/ADT/SparseSet.h
+++ b/include/llvm/ADT/SparseSet.h
@@ -1,9 +1,8 @@
 //===- llvm/ADT/SparseSet.h - Sparse set ------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/ADT/Statistic.h b/include/llvm/ADT/Statistic.h
index 90c2eefceb6c0ac61756d594a07238783a1726a5..2ac59da596efe54b60e9c7748823846ab83e0b1e 100644
--- a/include/llvm/ADT/Statistic.h
+++ b/include/llvm/ADT/Statistic.h
@@ -1,9 +1,8 @@
 //===-- llvm/ADT/Statistic.h - Easy way to expose stats ---------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/ADT/StringExtras.h b/include/llvm/ADT/StringExtras.h
index 60a03633a8a65ab5bc44f652095ca4215f887dc7..16ac90bd6c896a536e2825b54389ea4eaa399fd6 100644
--- a/include/llvm/ADT/StringExtras.h
+++ b/include/llvm/ADT/StringExtras.h
@@ -1,9 +1,8 @@
 //===- llvm/ADT/StringExtras.h - Useful string functions --------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/ADT/StringMap.h b/include/llvm/ADT/StringMap.h
index a9f83d3f5091521385fbb7ab40387997b6382b96..d94e4836877da66938c86ad8cb222a32adda1d8c 100644
--- a/include/llvm/ADT/StringMap.h
+++ b/include/llvm/ADT/StringMap.h
@@ -1,9 +1,8 @@
 //===- StringMap.h - String Hash table map interface ------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/ADT/StringRef.h b/include/llvm/ADT/StringRef.h
index a5ba5b59b5a36a4a0dac3935f84cf624c2298378..4661b1e68b2fa257db6606044f9fd99214807277 100644
--- a/include/llvm/ADT/StringRef.h
+++ b/include/llvm/ADT/StringRef.h
@@ -1,9 +1,8 @@
 //===- StringRef.h - Constant String Reference Wrapper ----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -63,7 +62,6 @@ namespace llvm {
 
     // Workaround memcmp issue with null pointers (undefined behavior)
     // by providing a specialized version
-    LLVM_ATTRIBUTE_ALWAYS_INLINE
     static int compareMemory(const char *Lhs, const char *Rhs, size_t Length) {
       if (Length == 0) { return 0; }
       return ::memcmp(Lhs,Rhs,Length);
@@ -81,17 +79,14 @@ namespace llvm {
     StringRef(std::nullptr_t) = delete;
 
     /// Construct a string ref from a cstring.
-    LLVM_ATTRIBUTE_ALWAYS_INLINE
     /*implicit*/ StringRef(const char *Str)
         : Data(Str), Length(Str ? ::strlen(Str) : 0) {}
 
     /// Construct a string ref from a pointer and length.
-    LLVM_ATTRIBUTE_ALWAYS_INLINE
     /*implicit*/ constexpr StringRef(const char *data, size_t length)
         : Data(data), Length(length) {}
 
     /// Construct a string ref from an std::string.
-    LLVM_ATTRIBUTE_ALWAYS_INLINE
     /*implicit*/ StringRef(const std::string &Str)
       : Data(Str.data()), Length(Str.length()) {}
 
@@ -124,17 +119,14 @@ namespace llvm {
     /// data - Get a pointer to the start of the string (which may not be null
     /// terminated).
     LLVM_NODISCARD
-    LLVM_ATTRIBUTE_ALWAYS_INLINE
     const char *data() const { return Data; }
 
     /// empty - Check if the string is empty.
     LLVM_NODISCARD
-    LLVM_ATTRIBUTE_ALWAYS_INLINE
     bool empty() const { return Length == 0; }
 
     /// size - Get the string size.
     LLVM_NODISCARD
-    LLVM_ATTRIBUTE_ALWAYS_INLINE
     size_t size() const { return Length; }
 
     /// front - Get the first character in the string.
@@ -165,7 +157,6 @@ namespace llvm {
     /// equals - Check for string equality, this is more efficient than
     /// compare() when the relative ordering of inequal strings isn't needed.
     LLVM_NODISCARD
-    LLVM_ATTRIBUTE_ALWAYS_INLINE
     bool equals(StringRef RHS) const {
       return (Length == RHS.Length &&
               compareMemory(Data, RHS.Data, RHS.Length) == 0);
@@ -180,7 +171,6 @@ namespace llvm {
     /// compare - Compare two strings; the result is -1, 0, or 1 if this string
     /// is lexicographically less than, equal to, or greater than the \p RHS.
     LLVM_NODISCARD
-    LLVM_ATTRIBUTE_ALWAYS_INLINE
     int compare(StringRef RHS) const {
       // Check the prefix for a mismatch.
       if (int Res = compareMemory(Data, RHS.Data, std::min(Length, RHS.Length)))
@@ -263,7 +253,6 @@ namespace llvm {
 
     /// Check if this string starts with the given \p Prefix.
     LLVM_NODISCARD
-    LLVM_ATTRIBUTE_ALWAYS_INLINE
     bool startswith(StringRef Prefix) const {
       return Length >= Prefix.Length &&
              compareMemory(Data, Prefix.Data, Prefix.Length) == 0;
@@ -275,7 +264,6 @@ namespace llvm {
 
     /// Check if this string ends with the given \p Suffix.
     LLVM_NODISCARD
-    LLVM_ATTRIBUTE_ALWAYS_INLINE
     bool endswith(StringRef Suffix) const {
       return Length >= Suffix.Length &&
         compareMemory(end() - Suffix.Length, Suffix.Data, Suffix.Length) == 0;
@@ -294,7 +282,6 @@ namespace llvm {
     /// \returns The index of the first occurrence of \p C, or npos if not
     /// found.
     LLVM_NODISCARD
-    LLVM_ATTRIBUTE_ALWAYS_INLINE
     size_t find(char C, size_t From = 0) const {
       size_t FindBegin = std::min(From, Length);
       if (FindBegin < Length) { // Avoid calling memchr with nullptr.
@@ -317,7 +304,6 @@ namespace llvm {
     /// \returns The index of the first character satisfying \p F starting from
     /// \p From, or npos if not found.
     LLVM_NODISCARD
-    LLVM_ATTRIBUTE_ALWAYS_INLINE
     size_t find_if(function_ref<bool(char)> F, size_t From = 0) const {
       StringRef S = drop_front(From);
       while (!S.empty()) {
@@ -333,7 +319,6 @@ namespace llvm {
     /// \returns The index of the first character not satisfying \p F starting
     /// from \p From, or npos if not found.
     LLVM_NODISCARD
-    LLVM_ATTRIBUTE_ALWAYS_INLINE
     size_t find_if_not(function_ref<bool(char)> F, size_t From = 0) const {
       return find_if([F](char c) { return !F(c); }, From);
     }
@@ -444,19 +429,16 @@ namespace llvm {
     /// Return true if the given string is a substring of *this, and false
     /// otherwise.
     LLVM_NODISCARD
-    LLVM_ATTRIBUTE_ALWAYS_INLINE
     bool contains(StringRef Other) const { return find(Other) != npos; }
 
     /// Return true if the given character is contained in *this, and false
     /// otherwise.
     LLVM_NODISCARD
-    LLVM_ATTRIBUTE_ALWAYS_INLINE
     bool contains(char C) const { return find_first_of(C) != npos; }
 
     /// Return true if the given string is a substring of *this, and false
     /// otherwise.
     LLVM_NODISCARD
-    LLVM_ATTRIBUTE_ALWAYS_INLINE
     bool contains_lower(StringRef Other) const {
       return find_lower(Other) != npos;
     }
@@ -464,7 +446,6 @@ namespace llvm {
     /// Return true if the given character is contained in *this, and false
     /// otherwise.
     LLVM_NODISCARD
-    LLVM_ATTRIBUTE_ALWAYS_INLINE
     bool contains_lower(char C) const { return find_lower(C) != npos; }
 
     /// @}
@@ -594,7 +575,6 @@ namespace llvm {
     /// exceeds the number of characters remaining in the string, the string
     /// suffix (starting with \p Start) will be returned.
     LLVM_NODISCARD
-    LLVM_ATTRIBUTE_ALWAYS_INLINE
     StringRef substr(size_t Start, size_t N = npos) const {
       Start = std::min(Start, Length);
       return StringRef(Data + Start, std::min(N, Length - Start));
@@ -604,7 +584,6 @@ namespace llvm {
     /// elements remaining.  If \p N is greater than the length of the
     /// string, the entire string is returned.
     LLVM_NODISCARD
-    LLVM_ATTRIBUTE_ALWAYS_INLINE
     StringRef take_front(size_t N = 1) const {
       if (N >= size())
         return *this;
@@ -615,7 +594,6 @@ namespace llvm {
     /// elements remaining.  If \p N is greater than the length of the
     /// string, the entire string is returned.
     LLVM_NODISCARD
-    LLVM_ATTRIBUTE_ALWAYS_INLINE
     StringRef take_back(size_t N = 1) const {
       if (N >= size())
         return *this;
@@ -625,7 +603,6 @@ namespace llvm {
     /// Return the longest prefix of 'this' such that every character
     /// in the prefix satisfies the given predicate.
     LLVM_NODISCARD
-    LLVM_ATTRIBUTE_ALWAYS_INLINE
     StringRef take_while(function_ref<bool(char)> F) const {
       return substr(0, find_if_not(F));
     }
@@ -633,7 +610,6 @@ namespace llvm {
     /// Return the longest prefix of 'this' such that no character in
     /// the prefix satisfies the given predicate.
     LLVM_NODISCARD
-    LLVM_ATTRIBUTE_ALWAYS_INLINE
     StringRef take_until(function_ref<bool(char)> F) const {
       return substr(0, find_if(F));
     }
@@ -641,7 +617,6 @@ namespace llvm {
     /// Return a StringRef equal to 'this' but with the first \p N elements
     /// dropped.
     LLVM_NODISCARD
-    LLVM_ATTRIBUTE_ALWAYS_INLINE
     StringRef drop_front(size_t N = 1) const {
       assert(size() >= N && "Dropping more elements than exist");
       return substr(N);
@@ -650,7 +625,6 @@ namespace llvm {
     /// Return a StringRef equal to 'this' but with the last \p N elements
     /// dropped.
     LLVM_NODISCARD
-    LLVM_ATTRIBUTE_ALWAYS_INLINE
     StringRef drop_back(size_t N = 1) const {
       assert(size() >= N && "Dropping more elements than exist");
       return substr(0, size()-N);
@@ -659,7 +633,6 @@ namespace llvm {
     /// Return a StringRef equal to 'this', but with all characters satisfying
     /// the given predicate dropped from the beginning of the string.
     LLVM_NODISCARD
-    LLVM_ATTRIBUTE_ALWAYS_INLINE
     StringRef drop_while(function_ref<bool(char)> F) const {
       return substr(find_if_not(F));
     }
@@ -667,14 +640,12 @@ namespace llvm {
     /// Return a StringRef equal to 'this', but with all characters not
     /// satisfying the given predicate dropped from the beginning of the string.
     LLVM_NODISCARD
-    LLVM_ATTRIBUTE_ALWAYS_INLINE
     StringRef drop_until(function_ref<bool(char)> F) const {
       return substr(find_if(F));
     }
 
     /// Returns true if this StringRef has the given prefix and removes that
     /// prefix.
-    LLVM_ATTRIBUTE_ALWAYS_INLINE
     bool consume_front(StringRef Prefix) {
       if (!startswith(Prefix))
         return false;
@@ -685,7 +656,6 @@ namespace llvm {
 
     /// Returns true if this StringRef has the given suffix and removes that
     /// suffix.
-    LLVM_ATTRIBUTE_ALWAYS_INLINE
     bool consume_back(StringRef Suffix) {
       if (!endswith(Suffix))
         return false;
@@ -706,7 +676,6 @@ namespace llvm {
     /// will be returned. If this is less than \p Start, an empty string will
     /// be returned.
     LLVM_NODISCARD
-    LLVM_ATTRIBUTE_ALWAYS_INLINE
     StringRef slice(size_t Start, size_t End) const {
       Start = std::min(Start, Length);
       End = std::min(std::max(Start, End), Length);
@@ -894,12 +863,10 @@ namespace llvm {
   /// @name StringRef Comparison Operators
   /// @{
 
-  LLVM_ATTRIBUTE_ALWAYS_INLINE
   inline bool operator==(StringRef LHS, StringRef RHS) {
     return LHS.equals(RHS);
   }
 
-  LLVM_ATTRIBUTE_ALWAYS_INLINE
   inline bool operator!=(StringRef LHS, StringRef RHS) { return !(LHS == RHS); }
 
   inline bool operator<(StringRef LHS, StringRef RHS) {
@@ -928,10 +895,6 @@ namespace llvm {
   LLVM_NODISCARD
   hash_code hash_value(StringRef S);
 
-  // StringRefs can be treated like a POD type.
-  template <typename T> struct isPodLike;
-  template <> struct isPodLike<StringRef> { static const bool value = true; };
-
 } // end namespace llvm
 
 #endif // LLVM_ADT_STRINGREF_H
diff --git a/include/llvm/ADT/StringSet.h b/include/llvm/ADT/StringSet.h
index 9af44c07df795c7c5dd0032dae95540efd7c34ab..fcf9519b40e3ff02e0131ed8da2941e65f694ed6 100644
--- a/include/llvm/ADT/StringSet.h
+++ b/include/llvm/ADT/StringSet.h
@@ -1,9 +1,8 @@
 //===- StringSet.h - The LLVM Compiler Driver -------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open
-// Source License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/ADT/StringSwitch.h b/include/llvm/ADT/StringSwitch.h
index b7860b98ce5dd4d63de96849adb2118267027ced..fea911f6928bc51b86f2cdd5c9df575dd08dcd8b 100644
--- a/include/llvm/ADT/StringSwitch.h
+++ b/include/llvm/ADT/StringSwitch.h
@@ -1,9 +1,8 @@
 //===--- StringSwitch.h - Switch-on-literal-string Construct --------------===/
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //===----------------------------------------------------------------------===/
 //
 //  This file implements the StringSwitch template, which mimics a switch()
@@ -49,7 +48,6 @@ class StringSwitch {
   Optional<T> Result;
 
 public:
-  LLVM_ATTRIBUTE_ALWAYS_INLINE
   explicit StringSwitch(StringRef S)
   : Str(S), Result() { }
 
@@ -66,7 +64,6 @@ public:
   ~StringSwitch() = default;
 
   // Case-sensitive case matchers
-  LLVM_ATTRIBUTE_ALWAYS_INLINE
   StringSwitch &Case(StringLiteral S, T Value) {
     if (!Result && Str == S) {
       Result = std::move(Value);
@@ -74,7 +71,6 @@ public:
     return *this;
   }
 
-  LLVM_ATTRIBUTE_ALWAYS_INLINE
   StringSwitch& EndsWith(StringLiteral S, T Value) {
     if (!Result && Str.endswith(S)) {
       Result = std::move(Value);
@@ -82,7 +78,6 @@ public:
     return *this;
   }
 
-  LLVM_ATTRIBUTE_ALWAYS_INLINE
   StringSwitch& StartsWith(StringLiteral S, T Value) {
     if (!Result && Str.startswith(S)) {
       Result = std::move(Value);
@@ -90,51 +85,43 @@ public:
     return *this;
   }
 
-  LLVM_ATTRIBUTE_ALWAYS_INLINE
   StringSwitch &Cases(StringLiteral S0, StringLiteral S1, T Value) {
     return Case(S0, Value).Case(S1, Value);
   }
 
-  LLVM_ATTRIBUTE_ALWAYS_INLINE
   StringSwitch &Cases(StringLiteral S0, StringLiteral S1, StringLiteral S2,
                       T Value) {
     return Case(S0, Value).Cases(S1, S2, Value);
   }
 
-  LLVM_ATTRIBUTE_ALWAYS_INLINE
   StringSwitch &Cases(StringLiteral S0, StringLiteral S1, StringLiteral S2,
                       StringLiteral S3, T Value) {
     return Case(S0, Value).Cases(S1, S2, S3, Value);
   }
 
-  LLVM_ATTRIBUTE_ALWAYS_INLINE
   StringSwitch &Cases(StringLiteral S0, StringLiteral S1, StringLiteral S2,
                       StringLiteral S3, StringLiteral S4, T Value) {
     return Case(S0, Value).Cases(S1, S2, S3, S4, Value);
   }
 
-  LLVM_ATTRIBUTE_ALWAYS_INLINE
   StringSwitch &Cases(StringLiteral S0, StringLiteral S1, StringLiteral S2,
                       StringLiteral S3, StringLiteral S4, StringLiteral S5,
                       T Value) {
     return Case(S0, Value).Cases(S1, S2, S3, S4, S5, Value);
   }
 
-  LLVM_ATTRIBUTE_ALWAYS_INLINE
   StringSwitch &Cases(StringLiteral S0, StringLiteral S1, StringLiteral S2,
                       StringLiteral S3, StringLiteral S4, StringLiteral S5,
                       StringLiteral S6, T Value) {
     return Case(S0, Value).Cases(S1, S2, S3, S4, S5, S6, Value);
   }
 
-  LLVM_ATTRIBUTE_ALWAYS_INLINE
   StringSwitch &Cases(StringLiteral S0, StringLiteral S1, StringLiteral S2,
                       StringLiteral S3, StringLiteral S4, StringLiteral S5,
                       StringLiteral S6, StringLiteral S7, T Value) {
     return Case(S0, Value).Cases(S1, S2, S3, S4, S5, S6, S7, Value);
   }
 
-  LLVM_ATTRIBUTE_ALWAYS_INLINE
   StringSwitch &Cases(StringLiteral S0, StringLiteral S1, StringLiteral S2,
                       StringLiteral S3, StringLiteral S4, StringLiteral S5,
                       StringLiteral S6, StringLiteral S7, StringLiteral S8,
@@ -142,7 +129,6 @@ public:
     return Case(S0, Value).Cases(S1, S2, S3, S4, S5, S6, S7, S8, Value);
   }
 
-  LLVM_ATTRIBUTE_ALWAYS_INLINE
   StringSwitch &Cases(StringLiteral S0, StringLiteral S1, StringLiteral S2,
                       StringLiteral S3, StringLiteral S4, StringLiteral S5,
                       StringLiteral S6, StringLiteral S7, StringLiteral S8,
@@ -151,7 +137,6 @@ public:
   }
 
   // Case-insensitive case matchers.
-  LLVM_ATTRIBUTE_ALWAYS_INLINE
   StringSwitch &CaseLower(StringLiteral S, T Value) {
     if (!Result && Str.equals_lower(S))
       Result = std::move(Value);
@@ -159,7 +144,6 @@ public:
     return *this;
   }
 
-  LLVM_ATTRIBUTE_ALWAYS_INLINE
   StringSwitch &EndsWithLower(StringLiteral S, T Value) {
     if (!Result && Str.endswith_lower(S))
       Result = Value;
@@ -167,7 +151,6 @@ public:
     return *this;
   }
 
-  LLVM_ATTRIBUTE_ALWAYS_INLINE
   StringSwitch &StartsWithLower(StringLiteral S, T Value) {
     if (!Result && Str.startswith_lower(S))
       Result = std::move(Value);
@@ -175,31 +158,26 @@ public:
     return *this;
   }
 
-  LLVM_ATTRIBUTE_ALWAYS_INLINE
   StringSwitch &CasesLower(StringLiteral S0, StringLiteral S1, T Value) {
     return CaseLower(S0, Value).CaseLower(S1, Value);
   }
 
-  LLVM_ATTRIBUTE_ALWAYS_INLINE
   StringSwitch &CasesLower(StringLiteral S0, StringLiteral S1, StringLiteral S2,
                            T Value) {
     return CaseLower(S0, Value).CasesLower(S1, S2, Value);
   }
 
-  LLVM_ATTRIBUTE_ALWAYS_INLINE
   StringSwitch &CasesLower(StringLiteral S0, StringLiteral S1, StringLiteral S2,
                            StringLiteral S3, T Value) {
     return CaseLower(S0, Value).CasesLower(S1, S2, S3, Value);
   }
 
-  LLVM_ATTRIBUTE_ALWAYS_INLINE
   StringSwitch &CasesLower(StringLiteral S0, StringLiteral S1, StringLiteral S2,
                            StringLiteral S3, StringLiteral S4, T Value) {
     return CaseLower(S0, Value).CasesLower(S1, S2, S3, S4, Value);
   }
 
   LLVM_NODISCARD
-  LLVM_ATTRIBUTE_ALWAYS_INLINE
   R Default(T Value) {
     if (Result)
       return std::move(*Result);
@@ -207,7 +185,6 @@ public:
   }
 
   LLVM_NODISCARD
-  LLVM_ATTRIBUTE_ALWAYS_INLINE
   operator R() {
     assert(Result && "Fell off the end of a string-switch");
     return std::move(*Result);
diff --git a/include/llvm/ADT/TinyPtrVector.h b/include/llvm/ADT/TinyPtrVector.h
index 1b8e9aa658c34b2b0d0030597b678e41321dc253..ac82451a9b213377ed979e515687b6dfeca9d936 100644
--- a/include/llvm/ADT/TinyPtrVector.h
+++ b/include/llvm/ADT/TinyPtrVector.h
@@ -1,9 +1,8 @@
 //===- llvm/ADT/TinyPtrVector.h - 'Normally tiny' vectors -------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/ADT/Triple.h b/include/llvm/ADT/Triple.h
index 04b64e2fa60b00303ef7639a1a43dcd9eeeb0fc3..70d8879bf055d941eba5309d90cd1ad22476b234 100644
--- a/include/llvm/ADT/Triple.h
+++ b/include/llvm/ADT/Triple.h
@@ -1,9 +1,8 @@
 //===-- llvm/ADT/Triple.h - Target triple helper class ----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -60,7 +59,6 @@ public:
     mips64,         // MIPS64: mips64, mips64r6, mipsn32, mipsn32r6
     mips64el,       // MIPS64EL: mips64el, mips64r6el, mipsn32el, mipsn32r6el
     msp430,         // MSP430: msp430
-    nios2,          // NIOSII: nios2
     ppc,            // PPC: powerpc
     ppc64,          // PPC64: powerpc64, ppu
     ppc64le,        // PPC64LE: powerpc64le
@@ -187,7 +185,8 @@ public:
     AMDPAL,     // AMD PAL Runtime
     HermitCore, // HermitCore Unikernel/Multikernel
     Hurd,       // GNU/Hurd
-    LastOSType = Hurd
+    WASI,       // Experimental WebAssembly OS
+    LastOSType = WASI
   };
   enum EnvironmentType {
     UnknownEnvironment,
@@ -415,7 +414,7 @@ public:
     if (LHS[1] != Minor)
       return LHS[1] < Minor;
     if (LHS[2] != Micro)
-      return LHS[1] < Micro;
+      return LHS[2] < Micro;
 
     return false;
   }
@@ -524,32 +523,36 @@ public:
     return getOS() == Triple::Haiku;
   }
 
-  /// Checks if the environment could be MSVC.
-  bool isWindowsMSVCEnvironment() const {
-    return getOS() == Triple::Win32 &&
-           (getEnvironment() == Triple::UnknownEnvironment ||
-            getEnvironment() == Triple::MSVC);
+  /// Tests whether the OS is Windows.
+  bool isOSWindows() const {
+    return getOS() == Triple::Win32;
   }
 
   /// Checks if the environment is MSVC.
   bool isKnownWindowsMSVCEnvironment() const {
-    return getOS() == Triple::Win32 && getEnvironment() == Triple::MSVC;
+    return isOSWindows() && getEnvironment() == Triple::MSVC;
+  }
+
+  /// Checks if the environment could be MSVC.
+  bool isWindowsMSVCEnvironment() const {
+    return isKnownWindowsMSVCEnvironment() ||
+           (isOSWindows() && getEnvironment() == Triple::UnknownEnvironment);
   }
 
   bool isWindowsCoreCLREnvironment() const {
-    return getOS() == Triple::Win32 && getEnvironment() == Triple::CoreCLR;
+    return isOSWindows() && getEnvironment() == Triple::CoreCLR;
   }
 
   bool isWindowsItaniumEnvironment() const {
-    return getOS() == Triple::Win32 && getEnvironment() == Triple::Itanium;
+    return isOSWindows() && getEnvironment() == Triple::Itanium;
   }
 
   bool isWindowsCygwinEnvironment() const {
-    return getOS() == Triple::Win32 && getEnvironment() == Triple::Cygnus;
+    return isOSWindows() && getEnvironment() == Triple::Cygnus;
   }
 
   bool isWindowsGNUEnvironment() const {
-    return getOS() == Triple::Win32 && getEnvironment() == Triple::GNU;
+    return isOSWindows() && getEnvironment() == Triple::GNU;
   }
 
   /// Tests for either Cygwin or MinGW OS
@@ -563,11 +566,6 @@ public:
            isWindowsItaniumEnvironment();
   }
 
-  /// Tests whether the OS is Windows.
-  bool isOSWindows() const {
-    return getOS() == Triple::Win32;
-  }
-
   /// Tests whether the OS is NaCl (Native Client)
   bool isOSNaCl() const {
     return getOS() == Triple::NaCl;
@@ -588,6 +586,11 @@ public:
     return getOS() == Triple::Hurd;
   }
 
+  /// Tests whether the OS is WASI.
+  bool isOSWASI() const {
+    return getOS() == Triple::WASI;
+  }
+
   /// Tests whether the OS uses glibc.
   bool isOSGlibc() const {
     return (getOS() == Triple::Linux || getOS() == Triple::KFreeBSD ||
diff --git a/include/llvm/ADT/Twine.h b/include/llvm/ADT/Twine.h
index b60fd09813988e07517413ab127d742878052aa1..4140c22aad3d94a2bdd9b33085d53e5d232278fd 100644
--- a/include/llvm/ADT/Twine.h
+++ b/include/llvm/ADT/Twine.h
@@ -1,9 +1,8 @@
 //===- Twine.h - Fast Temporary String Concatenation ------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -274,6 +273,9 @@ namespace llvm {
 
       assert(isValid() && "Invalid twine!");
     }
+    /// Delete the implicit conversion from nullptr as Twine(const char *)
+    /// cannot take nullptr.
+    /*implicit*/ Twine(std::nullptr_t) = delete;
 
     /// Construct from an std::string.
     /*implicit*/ Twine(const std::string &Str) : LHSKind(StdStringKind) {
diff --git a/include/llvm/ADT/UniqueVector.h b/include/llvm/ADT/UniqueVector.h
index c86bedd0768707bfc29734d1c4005577366c99e5..bfea988f1702703b7ac59e449270887a6cc22829 100644
--- a/include/llvm/ADT/UniqueVector.h
+++ b/include/llvm/ADT/UniqueVector.h
@@ -1,9 +1,8 @@
 //===- llvm/ADT/UniqueVector.h ----------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/ADT/VariadicFunction.h b/include/llvm/ADT/VariadicFunction.h
index 9028abe4c72c91e060f0d2f393a439dc59d3144d..5aefb05ecdda636c9a95226bf20b9c938d8972b2 100644
--- a/include/llvm/ADT/VariadicFunction.h
+++ b/include/llvm/ADT/VariadicFunction.h
@@ -1,9 +1,8 @@
-//===--- VariadicFunctions.h - Variadic Functions ---------------*- C++ -*-===//
+//===- VariadicFunction.h - Variadic Functions ------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/ADT/bit.h b/include/llvm/ADT/bit.h
index a4aba7b6a9eeed76ab0870e64951a39cfdaaefff..a790d5ed2d21a4d41d8c87c334823d374898cd05 100644
--- a/include/llvm/ADT/bit.h
+++ b/include/llvm/ADT/bit.h
@@ -1,9 +1,8 @@
 //===-- llvm/ADT/bit.h - C++20 <bit> ----------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -41,11 +40,11 @@ template <typename To, typename From
           , typename = typename std::enable_if<__is_trivially_copyable(To)>::type
           , typename = typename std::enable_if<__is_trivially_copyable(From)>::type
 #else
-  // This case is GCC 4.x. clang with libc++ or libstdc++ never get here. Unlike
-  // llvm/Support/type_traits.h's isPodLike we don't want to provide a
-  // good-enough answer here: developers in that configuration will hit
-  // compilation failures on the bots instead of locally. That's acceptable
-  // because it's very few developers, and only until we move past C++11.
+// This case is GCC 4.x. clang with libc++ or libstdc++ never get here. Unlike
+// llvm/Support/type_traits.h's is_trivially_copyable we don't want to
+// provide a good-enough answer here: developers in that configuration will hit
+// compilation failures on the bots instead of locally. That's acceptable
+// because it's very few developers, and only until we move past C++11.
 #endif
 >
 inline To bit_cast(const From &from) noexcept {
diff --git a/include/llvm/ADT/edit_distance.h b/include/llvm/ADT/edit_distance.h
index b2e8ec5c3f6debc85f88b3c7938f6a89d7d749a3..4f51340086921a3385835d171acc7112f2952a85 100644
--- a/include/llvm/ADT/edit_distance.h
+++ b/include/llvm/ADT/edit_distance.h
@@ -1,9 +1,8 @@
 //===-- llvm/ADT/edit_distance.h - Array edit distance function --- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/ADT/fallible_iterator.h b/include/llvm/ADT/fallible_iterator.h
new file mode 100644
index 0000000000000000000000000000000000000000..6501ad2233cdb95136de1114b6239929c00b2bcb
--- /dev/null
+++ b/include/llvm/ADT/fallible_iterator.h
@@ -0,0 +1,243 @@
+//===--- fallible_iterator.h - Wrapper for fallible iterators ---*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_ADT_FALLIBLE_ITERATOR_H
+#define LLVM_ADT_FALLIBLE_ITERATOR_H
+
+#include "llvm/ADT/PointerIntPair.h"
+#include "llvm/ADT/iterator_range.h"
+#include "llvm/Support/Error.h"
+
+#include <type_traits>
+
+namespace llvm {
+
+/// A wrapper class for fallible iterators.
+///
+///   The fallible_iterator template wraps an underlying iterator-like class
+/// whose increment and decrement operations are replaced with fallible versions
+/// like:
+///
+///   @code{.cpp}
+///   Error inc();
+///   Error dec();
+///   @endcode
+///
+///   It produces an interface that is (mostly) compatible with a traditional
+/// c++ iterator, including ++ and -- operators that do not fail.
+///
+///   Instances of the wrapper are constructed with an instance of the
+/// underlying iterator and (for non-end iterators) a reference to an Error
+/// instance. If the underlying increment/decrement operations fail, the Error
+/// is returned via this reference, and the resulting iterator value set to an
+/// end-of-range sentinel value. This enables the following loop idiom:
+///
+///   @code{.cpp}
+///   class Archive { // E.g. Potentially malformed on-disk archive
+///   public:
+///     fallible_iterator<ArchiveChildItr> children_begin(Error &Err);
+///     fallible_iterator<ArchiveChildItr> children_end();
+///     iterator_range<fallible_iterator<ArchiveChildItr>>
+///     children(Error &Err) {
+///       return make_range(children_begin(Err), children_end());
+///     //...
+///   };
+///
+///   void walk(Archive &A) {
+///     Error Err = Error::success();
+///     for (auto &C : A.children(Err)) {
+///       // Loop body only entered when increment succeeds.
+///     }
+///     if (Err) {
+///       // handle error.
+///     }
+///   }
+///   @endcode
+///
+///   The wrapper marks the referenced Error as unchecked after each increment
+/// and/or decrement operation, and clears the unchecked flag when a non-end
+/// value is compared against end (since, by the increment invariant, not being
+/// an end value proves that there was no error, and is equivalent to checking
+/// that the Error is success). This allows early exits from the loop body
+/// without requiring redundant error checks.
+template <typename Underlying> class fallible_iterator {
+private:
+  template <typename T>
+  using enable_if_struct_deref_supported = std::enable_if<
+      !std::is_void<decltype(std::declval<T>().operator->())>::value,
+      decltype(std::declval<T>().operator->())>;
+
+public:
+  /// Construct a fallible iterator that *cannot* be used as an end-of-range
+  /// value.
+  ///
+  /// A value created by this method can be dereferenced, incremented,
+  /// decremented and compared, providing the underlying type supports it.
+  ///
+  /// The error that is passed in will be initially marked as checked, so if the
+  /// iterator is not used at all the Error need not be checked.
+  static fallible_iterator itr(Underlying I, Error &Err) {
+    (void)!!Err;
+    return fallible_iterator(std::move(I), &Err);
+  }
+
+  /// Construct a fallible iteratro that can be used as an end-of-range value.
+  ///
+  /// A value created by this method can be dereferenced (if the underlying
+  /// value points at a valid value) and compared, but not incremented or
+  /// decremented.
+  static fallible_iterator end(Underlying I) {
+    return fallible_iterator(std::move(I), nullptr);
+  }
+
+  /// Forward dereference to the underlying iterator.
+  auto operator*() -> decltype(*std::declval<Underlying>()) { return *I; }
+
+  /// Forward const dereference to the underlying iterator.
+  auto operator*() const -> decltype(*std::declval<const Underlying>()) {
+    return *I;
+  }
+
+  /// Forward structure dereference to the underlying iterator (if the
+  /// underlying iterator supports it).
+  template <typename T = Underlying>
+  typename enable_if_struct_deref_supported<T>::type operator->() {
+    return I.operator->();
+  }
+
+  /// Forward const structure dereference to the underlying iterator (if the
+  /// underlying iterator supports it).
+  template <typename T = Underlying>
+  typename enable_if_struct_deref_supported<const T>::type operator->() const {
+    return I.operator->();
+  }
+
+  /// Increment the fallible iterator.
+  ///
+  /// If the underlying 'inc' operation fails, this will set the Error value
+  /// and update this iterator value to point to end-of-range.
+  ///
+  /// The Error value is marked as needing checking, regardless of whether the
+  /// 'inc' operation succeeds or fails.
+  fallible_iterator &operator++() {
+    assert(getErrPtr() && "Cannot increment end iterator");
+    if (auto Err = I.inc())
+      handleError(std::move(Err));
+    else
+      resetCheckedFlag();
+    return *this;
+  }
+
+  /// Decrement the fallible iterator.
+  ///
+  /// If the underlying 'dec' operation fails, this will set the Error value
+  /// and update this iterator value to point to end-of-range.
+  ///
+  /// The Error value is marked as needing checking, regardless of whether the
+  /// 'dec' operation succeeds or fails.
+  fallible_iterator &operator--() {
+    assert(getErrPtr() && "Cannot decrement end iterator");
+    if (auto Err = I.dec())
+      handleError(std::move(Err));
+    else
+      resetCheckedFlag();
+    return *this;
+  }
+
+  /// Compare fallible iterators for equality.
+  ///
+  /// Returns true if both LHS and RHS are end-of-range values, or if both are
+  /// non-end-of-range values whose underlying iterator values compare equal.
+  ///
+  /// If this is a comparison between an end-of-range iterator and a
+  /// non-end-of-range iterator, then the Error (referenced by the
+  /// non-end-of-range value) is marked as checked: Since all
+  /// increment/decrement operations result in an end-of-range value, comparing
+  /// false against end-of-range is equivalent to checking that the Error value
+  /// is success. This flag management enables early returns from loop bodies
+  /// without redundant Error checks.
+  friend bool operator==(const fallible_iterator &LHS,
+                         const fallible_iterator &RHS) {
+    // If both iterators are in the end state they compare
+    // equal, regardless of whether either is valid.
+    if (LHS.isEnd() && RHS.isEnd())
+      return true;
+
+    assert(LHS.isValid() && RHS.isValid() &&
+           "Invalid iterators can only be compared against end");
+
+    bool Equal = LHS.I == RHS.I;
+
+    // If the iterators differ and this is a comparison against end then mark
+    // the Error as checked.
+    if (!Equal) {
+      if (LHS.isEnd())
+        (void)!!*RHS.getErrPtr();
+      else
+        (void)!!*LHS.getErrPtr();
+    }
+
+    return Equal;
+  }
+
+  /// Compare fallible iterators for inequality.
+  ///
+  /// See notes for operator==.
+  friend bool operator!=(const fallible_iterator &LHS,
+                         const fallible_iterator &RHS) {
+    return !(LHS == RHS);
+  }
+
+private:
+  fallible_iterator(Underlying I, Error *Err)
+      : I(std::move(I)), ErrState(Err, false) {}
+
+  Error *getErrPtr() const { return ErrState.getPointer(); }
+
+  bool isEnd() const { return getErrPtr() == nullptr; }
+
+  bool isValid() const { return !ErrState.getInt(); }
+
+  void handleError(Error Err) {
+    *getErrPtr() = std::move(Err);
+    ErrState.setPointer(nullptr);
+    ErrState.setInt(true);
+  }
+
+  void resetCheckedFlag() {
+    *getErrPtr() = Error::success();
+  }
+
+  Underlying I;
+  mutable PointerIntPair<Error *, 1> ErrState;
+};
+
+/// Convenience wrapper to make a fallible_iterator value from an instance
+/// of an underlying iterator and an Error reference.
+template <typename Underlying>
+fallible_iterator<Underlying> make_fallible_itr(Underlying I, Error &Err) {
+  return fallible_iterator<Underlying>::itr(std::move(I), Err);
+}
+
+/// Convenience wrapper to make a fallible_iterator end value from an instance
+/// of an underlying iterator.
+template <typename Underlying>
+fallible_iterator<Underlying> make_fallible_end(Underlying E) {
+  return fallible_iterator<Underlying>::end(std::move(E));
+}
+
+template <typename Underlying>
+iterator_range<fallible_iterator<Underlying>>
+make_fallible_range(Underlying I, Underlying E, Error &Err) {
+  return make_range(make_fallible_itr(std::move(I), Err),
+                    make_fallible_end(std::move(E)));
+}
+
+} // end namespace llvm
+
+#endif // LLVM_ADT_FALLIBLE_ITERATOR_H
diff --git a/include/llvm/ADT/ilist.h b/include/llvm/ADT/ilist.h
index 00bb6d528175acba972d463f17d59bc68ece8fd4..06c7abff965f619e71a031e804b0d6a8a1fb3dce 100644
--- a/include/llvm/ADT/ilist.h
+++ b/include/llvm/ADT/ilist.h
@@ -1,9 +1,8 @@
 //==-- llvm/ADT/ilist.h - Intrusive Linked List Template ---------*- C++ -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -66,9 +65,8 @@ template <typename NodeTy> struct ilist_callback_traits {
   void addNodeToList(NodeTy *) {}
   void removeNodeFromList(NodeTy *) {}
 
-  /// Callback before transferring nodes to this list.
-  ///
-  /// \pre \c this!=&OldList
+  /// Callback before transferring nodes to this list. The nodes may already be
+  /// in this same list.
   template <class Iterator>
   void transferNodesFromList(ilist_callback_traits &OldList, Iterator /*first*/,
                              Iterator /*last*/) {
@@ -287,8 +285,8 @@ private:
     if (position == last)
       return;
 
-    if (this != &L2) // Notify traits we moved the nodes...
-      this->transferNodesFromList(L2, first, last);
+    // Notify traits we moved the nodes...
+    this->transferNodesFromList(L2, first, last);
 
     base_list_type::splice(position, L2, first, last);
   }
diff --git a/include/llvm/ADT/ilist_base.h b/include/llvm/ADT/ilist_base.h
index 3d818a48d41d455832d49fe49ce90310c1dfb68f..b8c098b951aded74714c6c945000be05eadc5bf3 100644
--- a/include/llvm/ADT/ilist_base.h
+++ b/include/llvm/ADT/ilist_base.h
@@ -1,9 +1,8 @@
 //===- llvm/ADT/ilist_base.h - Intrusive List Base --------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/ADT/ilist_iterator.h b/include/llvm/ADT/ilist_iterator.h
index 671e644e0154283717756c52c598b83cfac3a78f..cbe5cefa96d16085a01c56b41840c77667044db8 100644
--- a/include/llvm/ADT/ilist_iterator.h
+++ b/include/llvm/ADT/ilist_iterator.h
@@ -1,9 +1,8 @@
 //===- llvm/ADT/ilist_iterator.h - Intrusive List Iterator ------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/ADT/ilist_node.h b/include/llvm/ADT/ilist_node.h
index dd0e6b4ec2b996e3d68b24e09399e34c4f29166f..e040d9630a1eba638dd8b3219dd69f92a7155bff 100644
--- a/include/llvm/ADT/ilist_node.h
+++ b/include/llvm/ADT/ilist_node.h
@@ -1,9 +1,8 @@
 //===- llvm/ADT/ilist_node.h - Intrusive Linked List Helper -----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/ADT/ilist_node_base.h b/include/llvm/ADT/ilist_node_base.h
index e5062ac4eaadba103b2c8242dbbbdb4f5fabd7bc..f6c518e6eed7a1749a45a38f3491b3c447172dea 100644
--- a/include/llvm/ADT/ilist_node_base.h
+++ b/include/llvm/ADT/ilist_node_base.h
@@ -1,9 +1,8 @@
 //===- llvm/ADT/ilist_node_base.h - Intrusive List Node Base -----*- C++ -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/ADT/ilist_node_options.h b/include/llvm/ADT/ilist_node_options.h
index 7ff4005f675757c758e86bd89e3f7b17d444679d..9b95cdbe08c49be050fca6277bd6403a9b3fb8fd 100644
--- a/include/llvm/ADT/ilist_node_options.h
+++ b/include/llvm/ADT/ilist_node_options.h
@@ -1,9 +1,8 @@
 //===- llvm/ADT/ilist_node_options.h - ilist_node Options -------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/ADT/iterator.h b/include/llvm/ADT/iterator.h
index 40e490cf7864ea69abb23662c35d3e9946531b93..467fd4c00ec5a45124876397a514061825618771 100644
--- a/include/llvm/ADT/iterator.h
+++ b/include/llvm/ADT/iterator.h
@@ -1,9 +1,8 @@
 //===- iterator.h - Utilities for using and defining iterators --*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/ADT/iterator_range.h b/include/llvm/ADT/iterator_range.h
index 2ba12866ecf3cfab809bcaadcd381fc4804af4fc..774c7c4e3366e503ebd9972a84053ddf6ff4e8ec 100644
--- a/include/llvm/ADT/iterator_range.h
+++ b/include/llvm/ADT/iterator_range.h
@@ -1,9 +1,8 @@
 //===- iterator_range.h - A range adaptor for iterators ---------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/include/llvm/ADT/simple_ilist.h b/include/llvm/ADT/simple_ilist.h
index 4c7598a1acb4ed744cf0abd1e8fc3ff146d57a65..9257b47b9cf89e943830062136bb70441e38d4c3 100644
--- a/include/llvm/ADT/simple_ilist.h
+++ b/include/llvm/ADT/simple_ilist.h
@@ -1,9 +1,8 @@
 //===- llvm/ADT/simple_ilist.h - Simple Intrusive List ----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/Analysis/AliasAnalysis.h b/include/llvm/Analysis/AliasAnalysis.h
index 2efcd9dafa195ad64ad7fa13bd9ce6527f85b9ed..4e55f0179af7a49bea770e93dedeceb66f715f69 100644
--- a/include/llvm/Analysis/AliasAnalysis.h
+++ b/include/llvm/Analysis/AliasAnalysis.h
@@ -1,9 +1,8 @@
 //===- llvm/Analysis/AliasAnalysis.h - Alias Analysis Interface -*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -43,7 +42,6 @@
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/Analysis/MemoryLocation.h"
 #include "llvm/Analysis/TargetLibraryInfo.h"
-#include "llvm/IR/CallSite.h"
 #include "llvm/IR/Function.h"
 #include "llvm/IR/Instruction.h"
 #include "llvm/IR/Instructions.h"
@@ -363,7 +361,8 @@ public:
 
   /// A convenience wrapper around the \c isMustAlias helper interface.
   bool isMustAlias(const Value *V1, const Value *V2) {
-    return alias(V1, 1, V2, 1) == MustAlias;
+    return alias(V1, LocationSize::precise(1), V2, LocationSize::precise(1)) ==
+           MustAlias;
   }
 
   /// Checks whether the given location points to constant memory, or if
@@ -381,15 +380,15 @@ public:
   /// \name Simple mod/ref information
   /// @{
 
-  /// Get the ModRef info associated with a pointer argument of a callsite. The
+  /// Get the ModRef info associated with a pointer argument of a call. The
   /// result's bits are set to indicate the allowed aliasing ModRef kinds. Note
   /// that these bits do not necessarily account for the overall behavior of
   /// the function, but rather only provide additional per-argument
   /// information. This never sets ModRefInfo::Must.
-  ModRefInfo getArgModRefInfo(ImmutableCallSite CS, unsigned ArgIdx);
+  ModRefInfo getArgModRefInfo(const CallBase *Call, unsigned ArgIdx);
 
   /// Return the behavior of the given call site.
-  FunctionModRefBehavior getModRefBehavior(ImmutableCallSite CS);
+  FunctionModRefBehavior getModRefBehavior(const CallBase *Call);
 
   /// Return the behavior when calling the given function.
   FunctionModRefBehavior getModRefBehavior(const Function *F);
@@ -405,8 +404,8 @@ public:
   /// property (e.g. calls to 'sin' and 'cos').
   ///
   /// This property corresponds to the GCC 'const' attribute.
-  bool doesNotAccessMemory(ImmutableCallSite CS) {
-    return getModRefBehavior(CS) == FMRB_DoesNotAccessMemory;
+  bool doesNotAccessMemory(const CallBase *Call) {
+    return getModRefBehavior(Call) == FMRB_DoesNotAccessMemory;
   }
 
   /// Checks if the specified function is known to never read or write memory.
@@ -433,8 +432,8 @@ public:
   /// absence of interfering store instructions, such as CSE of strlen calls.
   ///
   /// This property corresponds to the GCC 'pure' attribute.
-  bool onlyReadsMemory(ImmutableCallSite CS) {
-    return onlyReadsMemory(getModRefBehavior(CS));
+  bool onlyReadsMemory(const CallBase *Call) {
+    return onlyReadsMemory(getModRefBehavior(Call));
   }
 
   /// Checks if the specified function is known to only read from non-volatile
@@ -499,36 +498,12 @@ public:
 
   /// getModRefInfo (for call sites) - Return information about whether
   /// a particular call site modifies or reads the specified memory location.
-  ModRefInfo getModRefInfo(ImmutableCallSite CS, const MemoryLocation &Loc);
+  ModRefInfo getModRefInfo(const CallBase *Call, const MemoryLocation &Loc);
 
   /// getModRefInfo (for call sites) - A convenience wrapper.
-  ModRefInfo getModRefInfo(ImmutableCallSite CS, const Value *P,
-                           LocationSize Size) {
-    return getModRefInfo(CS, MemoryLocation(P, Size));
-  }
-
-  /// getModRefInfo (for calls) - Return information about whether
-  /// a particular call modifies or reads the specified memory location.
-  ModRefInfo getModRefInfo(const CallInst *C, const MemoryLocation &Loc) {
-    return getModRefInfo(ImmutableCallSite(C), Loc);
-  }
-
-  /// getModRefInfo (for calls) - A convenience wrapper.
-  ModRefInfo getModRefInfo(const CallInst *C, const Value *P,
+  ModRefInfo getModRefInfo(const CallBase *Call, const Value *P,
                            LocationSize Size) {
-    return getModRefInfo(C, MemoryLocation(P, Size));
-  }
-
-  /// getModRefInfo (for invokes) - Return information about whether
-  /// a particular invoke modifies or reads the specified memory location.
-  ModRefInfo getModRefInfo(const InvokeInst *I, const MemoryLocation &Loc) {
-    return getModRefInfo(ImmutableCallSite(I), Loc);
-  }
-
-  /// getModRefInfo (for invokes) - A convenience wrapper.
-  ModRefInfo getModRefInfo(const InvokeInst *I, const Value *P,
-                           LocationSize Size) {
-    return getModRefInfo(I, MemoryLocation(P, Size));
+    return getModRefInfo(Call, MemoryLocation(P, Size));
   }
 
   /// getModRefInfo (for loads) - Return information about whether
@@ -568,7 +543,7 @@ public:
 
   /// getModRefInfo (for cmpxchges) - A convenience wrapper.
   ModRefInfo getModRefInfo(const AtomicCmpXchgInst *CX, const Value *P,
-                           unsigned Size) {
+                           LocationSize Size) {
     return getModRefInfo(CX, MemoryLocation(P, Size));
   }
 
@@ -578,7 +553,7 @@ public:
 
   /// getModRefInfo (for atomicrmws) - A convenience wrapper.
   ModRefInfo getModRefInfo(const AtomicRMWInst *RMW, const Value *P,
-                           unsigned Size) {
+                           LocationSize Size) {
     return getModRefInfo(RMW, MemoryLocation(P, Size));
   }
 
@@ -625,8 +600,8 @@ public:
   ModRefInfo getModRefInfo(const Instruction *I,
                            const Optional<MemoryLocation> &OptLoc) {
     if (OptLoc == None) {
-      if (auto CS = ImmutableCallSite(I)) {
-        return createModRefInfo(getModRefBehavior(CS));
+      if (const auto *Call = dyn_cast<CallBase>(I)) {
+        return createModRefInfo(getModRefBehavior(Call));
       }
     }
 
@@ -660,12 +635,12 @@ public:
 
   /// Return information about whether a call and an instruction may refer to
   /// the same memory locations.
-  ModRefInfo getModRefInfo(Instruction *I, ImmutableCallSite Call);
+  ModRefInfo getModRefInfo(Instruction *I, const CallBase *Call);
 
   /// Return information about whether two call sites may refer to the same set
   /// of memory locations. See the AA documentation for details:
   ///   http://llvm.org/docs/AliasAnalysis.html#ModRefInfo
-  ModRefInfo getModRefInfo(ImmutableCallSite CS1, ImmutableCallSite CS2);
+  ModRefInfo getModRefInfo(const CallBase *Call1, const CallBase *Call2);
 
   /// Return information about whether a particular call site modifies
   /// or reads the specified memory location \p MemLoc before instruction \p I
@@ -776,25 +751,25 @@ public:
   /// that these bits do not necessarily account for the overall behavior of
   /// the function, but rather only provide additional per-argument
   /// information.
-  virtual ModRefInfo getArgModRefInfo(ImmutableCallSite CS,
+  virtual ModRefInfo getArgModRefInfo(const CallBase *Call,
                                       unsigned ArgIdx) = 0;
 
   /// Return the behavior of the given call site.
-  virtual FunctionModRefBehavior getModRefBehavior(ImmutableCallSite CS) = 0;
+  virtual FunctionModRefBehavior getModRefBehavior(const CallBase *Call) = 0;
 
   /// Return the behavior when calling the given function.
   virtual FunctionModRefBehavior getModRefBehavior(const Function *F) = 0;
 
   /// getModRefInfo (for call sites) - Return information about whether
   /// a particular call site modifies or reads the specified memory location.
-  virtual ModRefInfo getModRefInfo(ImmutableCallSite CS,
+  virtual ModRefInfo getModRefInfo(const CallBase *Call,
                                    const MemoryLocation &Loc) = 0;
 
   /// Return information about whether two call sites may refer to the same set
   /// of memory locations. See the AA documentation for details:
   ///   http://llvm.org/docs/AliasAnalysis.html#ModRefInfo
-  virtual ModRefInfo getModRefInfo(ImmutableCallSite CS1,
-                                   ImmutableCallSite CS2) = 0;
+  virtual ModRefInfo getModRefInfo(const CallBase *Call1,
+                                   const CallBase *Call2) = 0;
 
   /// @}
 };
@@ -826,26 +801,26 @@ public:
     return Result.pointsToConstantMemory(Loc, OrLocal);
   }
 
-  ModRefInfo getArgModRefInfo(ImmutableCallSite CS, unsigned ArgIdx) override {
-    return Result.getArgModRefInfo(CS, ArgIdx);
+  ModRefInfo getArgModRefInfo(const CallBase *Call, unsigned ArgIdx) override {
+    return Result.getArgModRefInfo(Call, ArgIdx);
   }
 
-  FunctionModRefBehavior getModRefBehavior(ImmutableCallSite CS) override {
-    return Result.getModRefBehavior(CS);
+  FunctionModRefBehavior getModRefBehavior(const CallBase *Call) override {
+    return Result.getModRefBehavior(Call);
   }
 
   FunctionModRefBehavior getModRefBehavior(const Function *F) override {
     return Result.getModRefBehavior(F);
   }
 
-  ModRefInfo getModRefInfo(ImmutableCallSite CS,
+  ModRefInfo getModRefInfo(const CallBase *Call,
                            const MemoryLocation &Loc) override {
-    return Result.getModRefInfo(CS, Loc);
+    return Result.getModRefInfo(Call, Loc);
   }
 
-  ModRefInfo getModRefInfo(ImmutableCallSite CS1,
-                           ImmutableCallSite CS2) override {
-    return Result.getModRefInfo(CS1, CS2);
+  ModRefInfo getModRefInfo(const CallBase *Call1,
+                           const CallBase *Call2) override {
+    return Result.getModRefInfo(Call1, Call2);
   }
 };
 
@@ -900,25 +875,28 @@ protected:
                  : CurrentResult.pointsToConstantMemory(Loc, OrLocal);
     }
 
-    ModRefInfo getArgModRefInfo(ImmutableCallSite CS, unsigned ArgIdx) {
-      return AAR ? AAR->getArgModRefInfo(CS, ArgIdx) : CurrentResult.getArgModRefInfo(CS, ArgIdx);
+    ModRefInfo getArgModRefInfo(const CallBase *Call, unsigned ArgIdx) {
+      return AAR ? AAR->getArgModRefInfo(Call, ArgIdx)
+                 : CurrentResult.getArgModRefInfo(Call, ArgIdx);
     }
 
-    FunctionModRefBehavior getModRefBehavior(ImmutableCallSite CS) {
-      return AAR ? AAR->getModRefBehavior(CS) : CurrentResult.getModRefBehavior(CS);
+    FunctionModRefBehavior getModRefBehavior(const CallBase *Call) {
+      return AAR ? AAR->getModRefBehavior(Call)
+                 : CurrentResult.getModRefBehavior(Call);
     }
 
     FunctionModRefBehavior getModRefBehavior(const Function *F) {
       return AAR ? AAR->getModRefBehavior(F) : CurrentResult.getModRefBehavior(F);
     }
 
-    ModRefInfo getModRefInfo(ImmutableCallSite CS, const MemoryLocation &Loc) {
-      return AAR ? AAR->getModRefInfo(CS, Loc)
-                 : CurrentResult.getModRefInfo(CS, Loc);
+    ModRefInfo getModRefInfo(const CallBase *Call, const MemoryLocation &Loc) {
+      return AAR ? AAR->getModRefInfo(Call, Loc)
+                 : CurrentResult.getModRefInfo(Call, Loc);
     }
 
-    ModRefInfo getModRefInfo(ImmutableCallSite CS1, ImmutableCallSite CS2) {
-      return AAR ? AAR->getModRefInfo(CS1, CS2) : CurrentResult.getModRefInfo(CS1, CS2);
+    ModRefInfo getModRefInfo(const CallBase *Call1, const CallBase *Call2) {
+      return AAR ? AAR->getModRefInfo(Call1, Call2)
+                 : CurrentResult.getModRefInfo(Call1, Call2);
     }
   };
 
@@ -950,11 +928,11 @@ public:
     return false;
   }
 
-  ModRefInfo getArgModRefInfo(ImmutableCallSite CS, unsigned ArgIdx) {
+  ModRefInfo getArgModRefInfo(const CallBase *Call, unsigned ArgIdx) {
     return ModRefInfo::ModRef;
   }
 
-  FunctionModRefBehavior getModRefBehavior(ImmutableCallSite CS) {
+  FunctionModRefBehavior getModRefBehavior(const CallBase *Call) {
     return FMRB_UnknownModRefBehavior;
   }
 
@@ -962,11 +940,11 @@ public:
     return FMRB_UnknownModRefBehavior;
   }
 
-  ModRefInfo getModRefInfo(ImmutableCallSite CS, const MemoryLocation &Loc) {
+  ModRefInfo getModRefInfo(const CallBase *Call, const MemoryLocation &Loc) {
     return ModRefInfo::ModRef;
   }
 
-  ModRefInfo getModRefInfo(ImmutableCallSite CS1, ImmutableCallSite CS2) {
+  ModRefInfo getModRefInfo(const CallBase *Call1, const CallBase *Call2) {
     return ModRefInfo::ModRef;
   }
 };
diff --git a/include/llvm/Analysis/AliasAnalysisEvaluator.h b/include/llvm/Analysis/AliasAnalysisEvaluator.h
index 0941814a56c3410dc92e01a1735e3ecc8425b123..972eceaa3ba92af7234881513107d42603664e35 100644
--- a/include/llvm/Analysis/AliasAnalysisEvaluator.h
+++ b/include/llvm/Analysis/AliasAnalysisEvaluator.h
@@ -1,9 +1,8 @@
 //===- AliasAnalysisEvaluator.h - Alias Analysis Accuracy Evaluator -------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/include/llvm/Analysis/AliasSetTracker.h b/include/llvm/Analysis/AliasSetTracker.h
index 7ed5cd5c4734601857b7b5fbc82f4a487ab4fb04..34a509b7f4bb2e919445958f2e1d1b1b15d2961c 100644
--- a/include/llvm/Analysis/AliasSetTracker.h
+++ b/include/llvm/Analysis/AliasSetTracker.h
@@ -1,9 +1,8 @@
 //===- llvm/Analysis/AliasSetTracker.h - Build Alias Sets -------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -37,6 +36,8 @@ namespace llvm {
 class AliasSetTracker;
 class BasicBlock;
 class LoadInst;
+class Loop;
+class MemorySSA;
 class AnyMemSetInst;
 class AnyMemTransferInst;
 class raw_ostream;
@@ -294,7 +295,8 @@ private:
   void removeFromTracker(AliasSetTracker &AST);
 
   void addPointer(AliasSetTracker &AST, PointerRec &Entry, LocationSize Size,
-                  const AAMDNodes &AAInfo, bool KnownMustAlias = false);
+                  const AAMDNodes &AAInfo, bool KnownMustAlias = false,
+                  bool SkipSizeUpdate = false);
   void addUnknownInst(Instruction *I, AliasAnalysis &AA);
 
   void removeUnknownInst(AliasSetTracker &AST, Instruction *I) {
@@ -310,10 +312,10 @@ private:
   }
 
 public:
-  /// Return true if the specified pointer "may" (or must) alias one of the
-  /// members in the set.
-  bool aliasesPointer(const Value *Ptr, LocationSize Size,
-                      const AAMDNodes &AAInfo, AliasAnalysis &AA) const;
+  /// If the specified pointer "may" (or must) alias one of the members in the
+  /// set return the appropriate AliasResult. Otherwise return NoAlias.
+  AliasResult aliasesPointer(const Value *Ptr, LocationSize Size,
+                             const AAMDNodes &AAInfo, AliasAnalysis &AA) const;
   bool aliasesUnknownInst(const Instruction *Inst, AliasAnalysis &AA) const;
 };
 
@@ -341,6 +343,8 @@ class AliasSetTracker {
   struct ASTCallbackVHDenseMapInfo : public DenseMapInfo<Value *> {};
 
   AliasAnalysis &AA;
+  MemorySSA *MSSA;
+  Loop *L;
   ilist<AliasSet> AliasSets;
 
   using PointerMapType = DenseMap<ASTCallbackVH, AliasSet::PointerRec *,
@@ -353,6 +357,8 @@ public:
   /// Create an empty collection of AliasSets, and use the specified alias
   /// analysis object to disambiguate load and store addresses.
   explicit AliasSetTracker(AliasAnalysis &aa) : AA(aa) {}
+  explicit AliasSetTracker(AliasAnalysis &aa, MemorySSA *mssa, Loop *l)
+      : AA(aa), MSSA(mssa), L(l) {}
   ~AliasSetTracker() { clear(); }
 
   /// These methods are used to add different types of instructions to the alias
@@ -377,6 +383,7 @@ public:
   void add(BasicBlock &BB);       // Add all instructions in basic block
   void add(const AliasSetTracker &AST); // Add alias relations from another AST
   void addUnknown(Instruction *I);
+  void addAllInstructionsInLoopUsingMSSA();
 
   void clear();
 
@@ -439,7 +446,8 @@ private:
 
   AliasSet &addPointer(MemoryLocation Loc, AliasSet::AccessLattice E);
   AliasSet *mergeAliasSetsForPointer(const Value *Ptr, LocationSize Size,
-                                     const AAMDNodes &AAInfo);
+                                     const AAMDNodes &AAInfo,
+                                     bool &MustAliasAll);
 
   /// Merge all alias sets into a single set that is considered to alias any
   /// pointer.
diff --git a/include/llvm/Analysis/AssumptionCache.h b/include/llvm/Analysis/AssumptionCache.h
index 46538b1fa86f124a561ff90d8c2547fb70e14980..b42846472f2eac169a84adc1638db5f46f8aba3b 100644
--- a/include/llvm/Analysis/AssumptionCache.h
+++ b/include/llvm/Analysis/AssumptionCache.h
@@ -1,9 +1,8 @@
 //===- llvm/Analysis/AssumptionCache.h - Track @llvm.assume -----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -104,6 +103,10 @@ public:
   /// not already be in the cache.
   void registerAssumption(CallInst *CI);
 
+  /// Remove an \@llvm.assume intrinsic from this function's cache if it has
+  /// been added to the cache earlier.
+  void unregisterAssumption(CallInst *CI);
+
   /// Update the cache of values being affected by this assumption (i.e.
   /// the values about which this assumption provides information).
   void updateAffectedValues(CallInst *CI);
@@ -209,6 +212,10 @@ public:
   /// existing cache will be returned.
   AssumptionCache &getAssumptionCache(Function &F);
 
+  /// Return the cached assumptions for a function if it has already been
+  /// scanned. Otherwise return nullptr.
+  AssumptionCache *lookupAssumptionCache(Function &F);
+
   AssumptionCacheTracker();
   ~AssumptionCacheTracker() override;
 
diff --git a/include/llvm/Analysis/BasicAliasAnalysis.h b/include/llvm/Analysis/BasicAliasAnalysis.h
index 6344e84b58ebe8eb5b7a39396a96d919d9ef0f9e..29f584cea8eb3131b06e59fe6155248f18595d70 100644
--- a/include/llvm/Analysis/BasicAliasAnalysis.h
+++ b/include/llvm/Analysis/BasicAliasAnalysis.h
@@ -1,9 +1,8 @@
 //===- BasicAliasAnalysis.h - Stateless, local Alias Analysis ---*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
@@ -21,7 +20,7 @@
 #include "llvm/Analysis/AliasAnalysis.h"
 #include "llvm/Analysis/AssumptionCache.h"
 #include "llvm/Analysis/MemoryLocation.h"
-#include "llvm/IR/CallSite.h"
+#include "llvm/IR/InstrTypes.h"
 #include "llvm/IR/PassManager.h"
 #include "llvm/Pass.h"
 #include <algorithm>
@@ -84,18 +83,18 @@ public:
 
   AliasResult alias(const MemoryLocation &LocA, const MemoryLocation &LocB);
 
-  ModRefInfo getModRefInfo(ImmutableCallSite CS, const MemoryLocation &Loc);
+  ModRefInfo getModRefInfo(const CallBase *Call, const MemoryLocation &Loc);
 
-  ModRefInfo getModRefInfo(ImmutableCallSite CS1, ImmutableCallSite CS2);
+  ModRefInfo getModRefInfo(const CallBase *Call1, const CallBase *Call2);
 
   /// Chases pointers until we find a (constant global) or not.
   bool pointsToConstantMemory(const MemoryLocation &Loc, bool OrLocal);
 
   /// Get the location associated with a pointer argument of a callsite.
-  ModRefInfo getArgModRefInfo(ImmutableCallSite CS, unsigned ArgIdx);
+  ModRefInfo getArgModRefInfo(const CallBase *Call, unsigned ArgIdx);
 
   /// Returns the behavior when calling the given call site.
-  FunctionModRefBehavior getModRefBehavior(ImmutableCallSite CS);
+  FunctionModRefBehavior getModRefBehavior(const CallBase *Call);
 
   /// Returns the behavior when calling the given function. For use when the
   /// call site is not known.
@@ -115,7 +114,7 @@ private:
     unsigned ZExtBits;
     unsigned SExtBits;
 
-    int64_t Scale;
+    APInt Scale;
 
     bool operator==(const VariableGEPIndex &Other) const {
       return V == Other.V && ZExtBits == Other.ZExtBits &&
@@ -133,10 +132,10 @@ private:
     // Base pointer of the GEP
     const Value *Base;
     // Total constant offset w.r.t the base from indexing into structs
-    int64_t StructOffset;
+    APInt StructOffset;
     // Total constant offset w.r.t the base from indexing through
     // pointers/arrays/vectors
-    int64_t OtherOffset;
+    APInt OtherOffset;
     // Scaled variable (non-constant) indices.
     SmallVector<VariableGEPIndex, 4> VarIndices;
   };
@@ -145,6 +144,8 @@ private:
   using LocPair = std::pair<MemoryLocation, MemoryLocation>;
   using AliasCacheTy = SmallDenseMap<LocPair, AliasResult, 8>;
   AliasCacheTy AliasCache;
+  using IsCapturedCacheTy = SmallDenseMap<const Value *, bool, 8>;
+  IsCapturedCacheTy IsCapturedCache;
 
   /// Tracks phi nodes we have visited.
   ///
@@ -189,7 +190,7 @@ private:
   bool
   constantOffsetHeuristic(const SmallVectorImpl<VariableGEPIndex> &VarIndices,
                           LocationSize V1Size, LocationSize V2Size,
-                          int64_t BaseOffset, AssumptionCache *AC,
+                          APInt BaseOffset, AssumptionCache *AC,
                           DominatorTree *DT);
 
   bool isValueEqualInPotentialCycles(const Value *V1, const Value *V2);
diff --git a/include/llvm/Analysis/BlockFrequencyInfo.h b/include/llvm/Analysis/BlockFrequencyInfo.h
index ca12db6208b8d0866553daa927d7daa62699b593..c73c7faba0e79a08e59b672c65b6572258d2a5b6 100644
--- a/include/llvm/Analysis/BlockFrequencyInfo.h
+++ b/include/llvm/Analysis/BlockFrequencyInfo.h
@@ -1,9 +1,8 @@
 //===- BlockFrequencyInfo.h - Block Frequency Analysis ----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -56,7 +55,7 @@ public:
 
   const Function *getFunction() const;
   const BranchProbabilityInfo *getBPI() const;
-  void view() const;
+  void view(StringRef = "BlockFrequencyDAGs") const;
 
   /// getblockFreq - Return block frequency. Return 0 if we don't have the
   /// information. Please note that initial frequency is equal to ENTRY_FREQ. It
diff --git a/include/llvm/Analysis/BlockFrequencyInfoImpl.h b/include/llvm/Analysis/BlockFrequencyInfoImpl.h
index 25b2efd33c98e65df4ca3b7c0683540b9caeede5..813bad49888043598feb9fa0afa3b1f06bda9f04 100644
--- a/include/llvm/Analysis/BlockFrequencyInfoImpl.h
+++ b/include/llvm/Analysis/BlockFrequencyInfoImpl.h
@@ -1,9 +1,8 @@
 //==- BlockFrequencyInfoImpl.h - Block Frequency Implementation --*- C++ -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -160,10 +159,6 @@ inline raw_ostream &operator<<(raw_ostream &OS, BlockMass X) {
 
 } // end namespace bfi_detail
 
-template <> struct isPodLike<bfi_detail::BlockMass> {
-  static const bool value = true;
-};
-
 /// Base class for BlockFrequencyInfoImpl
 ///
 /// BlockFrequencyInfoImplBase has supporting data structures and some
@@ -187,9 +182,9 @@ public:
   struct BlockNode {
     using IndexType = uint32_t;
 
-    IndexType Index = std::numeric_limits<uint32_t>::max();
+    IndexType Index;
 
-    BlockNode() = default;
+    BlockNode() : Index(std::numeric_limits<uint32_t>::max()) {}
     BlockNode(IndexType Index) : Index(Index) {}
 
     bool operator==(const BlockNode &X) const { return Index == X.Index; }
diff --git a/include/llvm/Analysis/BranchProbabilityInfo.h b/include/llvm/Analysis/BranchProbabilityInfo.h
index 45277db460907ddb9d165385aa3186fb4e7dae83..97cb730d16c7710607c74e66c3f72b5c264416b8 100644
--- a/include/llvm/Analysis/BranchProbabilityInfo.h
+++ b/include/llvm/Analysis/BranchProbabilityInfo.h
@@ -1,9 +1,8 @@
 //===- BranchProbabilityInfo.h - Branch Probability Analysis ----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Analysis/CFG.h b/include/llvm/Analysis/CFG.h
index caae0b6e2a8ffaba87a4ab28e4fd231230133dc8..bcff4fb87079550450fb425834f0384d3e6aacca 100644
--- a/include/llvm/Analysis/CFG.h
+++ b/include/llvm/Analysis/CFG.h
@@ -1,9 +1,8 @@
 //===-- Analysis/CFG.h - BasicBlock Analyses --------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Analysis/CFGPrinter.h b/include/llvm/Analysis/CFGPrinter.h
index 5996dd90bcfd5a5c0f6ade0f806263e68c3e6d2d..aaefc11653dda5cef454bbb6f0c28b0b392bef2d 100644
--- a/include/llvm/Analysis/CFGPrinter.h
+++ b/include/llvm/Analysis/CFGPrinter.h
@@ -1,9 +1,8 @@
 //===-- CFGPrinter.h - CFG printer external interface -----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Analysis/CFLAliasAnalysisUtils.h b/include/llvm/Analysis/CFLAliasAnalysisUtils.h
index 981a8ddc2289a84ea0ac9c92fd822a0d052b9eff..02f999a5b913d6ea3597358429b63a293baa96d9 100644
--- a/include/llvm/Analysis/CFLAliasAnalysisUtils.h
+++ b/include/llvm/Analysis/CFLAliasAnalysisUtils.h
@@ -1,9 +1,8 @@
 //=- CFLAliasAnalysisUtils.h - Utilities for CFL Alias Analysis ----*- C++-*-=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 // \file
diff --git a/include/llvm/Analysis/CFLAndersAliasAnalysis.h b/include/llvm/Analysis/CFLAndersAliasAnalysis.h
index 8ae72553ab94b59cbd87447111c681c3f01096ec..696aaeb68eeb8625fa380ff5b636c7c07b30f1bb 100644
--- a/include/llvm/Analysis/CFLAndersAliasAnalysis.h
+++ b/include/llvm/Analysis/CFLAndersAliasAnalysis.h
@@ -1,9 +1,8 @@
 //==- CFLAndersAliasAnalysis.h - Unification-based Alias Analysis -*- C++-*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/include/llvm/Analysis/CFLSteensAliasAnalysis.h b/include/llvm/Analysis/CFLSteensAliasAnalysis.h
index 09e366f11e18e434ecd7c10a8ceb3fa27caebfa9..2d3b43c6c6210e52ab857b96e9d68f53968463ce 100644
--- a/include/llvm/Analysis/CFLSteensAliasAnalysis.h
+++ b/include/llvm/Analysis/CFLSteensAliasAnalysis.h
@@ -1,9 +1,8 @@
 //==- CFLSteensAliasAnalysis.h - Unification-based Alias Analysis -*- C++-*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/include/llvm/Analysis/CGSCCPassManager.h b/include/llvm/Analysis/CGSCCPassManager.h
index 61b99f6c3e6b8cb5115f492dd93029d35e8734fe..6d26954633a007e5645e7862240eeb33cfd635f4 100644
--- a/include/llvm/Analysis/CGSCCPassManager.h
+++ b/include/llvm/Analysis/CGSCCPassManager.h
@@ -1,9 +1,8 @@
 //===- CGSCCPassManager.h - Call graph pass management ----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/include/llvm/Analysis/CallGraph.h b/include/llvm/Analysis/CallGraph.h
index f109cf2fac4dbeabe96de15301fed206ec68fc10..a743cbccb3d66706298ffefb6cdb8a4017a11896 100644
--- a/include/llvm/Analysis/CallGraph.h
+++ b/include/llvm/Analysis/CallGraph.h
@@ -1,9 +1,8 @@
 //===- CallGraph.h - Build a Module's call graph ----------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/include/llvm/Analysis/CallGraphSCCPass.h b/include/llvm/Analysis/CallGraphSCCPass.h
index ace54607634c8588c2e998df6e1eeec5093545c9..1b5b7e2f039e672fafbfe33c4e7f47de58f76297 100644
--- a/include/llvm/Analysis/CallGraphSCCPass.h
+++ b/include/llvm/Analysis/CallGraphSCCPass.h
@@ -1,9 +1,8 @@
 //===- CallGraphSCCPass.h - Pass that operates BU on call graph -*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Analysis/CallPrinter.h b/include/llvm/Analysis/CallPrinter.h
index 8b697d5aa1494cbaa7b45cf93bd0d3ce4baa724f..8d4159f3ddc02c28bd3ded281b96e024cb09f6f7 100644
--- a/include/llvm/Analysis/CallPrinter.h
+++ b/include/llvm/Analysis/CallPrinter.h
@@ -1,9 +1,8 @@
 //===-- CallPrinter.h - Call graph printer external interface ----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Analysis/CaptureTracking.h b/include/llvm/Analysis/CaptureTracking.h
index aaaaff9ae252c6ac9a93a3274c4cfc8ac6647f77..ca7abd34fea2437033d317db2f8653fcc08c1dba 100644
--- a/include/llvm/Analysis/CaptureTracking.h
+++ b/include/llvm/Analysis/CaptureTracking.h
@@ -1,9 +1,8 @@
 //===----- llvm/Analysis/CaptureTracking.h - Pointer capture ----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Analysis/CmpInstAnalysis.h b/include/llvm/Analysis/CmpInstAnalysis.h
index 0e9c6a96b0f41095234f5877c6f4a6b6bf4b83e8..3d34cd12aea479b40bf7ebb459878ed0a5ac77af 100644
--- a/include/llvm/Analysis/CmpInstAnalysis.h
+++ b/include/llvm/Analysis/CmpInstAnalysis.h
@@ -1,9 +1,8 @@
 //===-- CmpInstAnalysis.h - Utils to help fold compare insts ----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Analysis/CodeMetrics.h b/include/llvm/Analysis/CodeMetrics.h
index 75290223852265ba7d27c3e996fe5067b28be67c..1482b66a3080d9a76050e6c9c882e9535b0c4211 100644
--- a/include/llvm/Analysis/CodeMetrics.h
+++ b/include/llvm/Analysis/CodeMetrics.h
@@ -1,9 +1,8 @@
 //===- CodeMetrics.h - Code cost measurements -------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -17,7 +16,6 @@
 
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/SmallPtrSet.h"
-#include "llvm/IR/CallSite.h"
 
 namespace llvm {
 class AssumptionCache;
@@ -29,14 +27,6 @@ class DataLayout;
 class TargetTransformInfo;
 class Value;
 
-/// Check whether a call will lower to something small.
-///
-/// This tests checks whether this callsite will lower to something
-/// significantly cheaper than a traditional call, often a single
-/// instruction. Note that if isInstructionFree(CS.getInstruction()) would
-/// return true, so will this function.
-bool callIsSmall(ImmutableCallSite CS);
-
 /// Utility to calculate the size and a few similar metrics for a set
 /// of basic blocks.
 struct CodeMetrics {
diff --git a/include/llvm/Analysis/ConstantFolding.h b/include/llvm/Analysis/ConstantFolding.h
index 192c1abddcd2e6137f4f0a2447cd5b157af0d6f3..43a2df0d853eef65ab8915db4da927ab2b6bc93f 100644
--- a/include/llvm/Analysis/ConstantFolding.h
+++ b/include/llvm/Analysis/ConstantFolding.h
@@ -1,9 +1,8 @@
 //===-- ConstantFolding.h - Fold instructions into constants ----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -23,7 +22,7 @@
 namespace llvm {
 class APInt;
 template <typename T> class ArrayRef;
-class CallSite;
+class CallBase;
 class Constant;
 class ConstantExpr;
 class ConstantVector;
@@ -31,7 +30,6 @@ class DataLayout;
 class Function;
 class GlobalValue;
 class Instruction;
-class ImmutableCallSite;
 class TargetLibraryInfo;
 class Type;
 
@@ -139,11 +137,11 @@ Constant *ConstantFoldLoadThroughGEPIndices(Constant *C,
 
 /// canConstantFoldCallTo - Return true if its even possible to fold a call to
 /// the specified function.
-bool canConstantFoldCallTo(ImmutableCallSite CS, const Function *F);
+bool canConstantFoldCallTo(const CallBase *Call, const Function *F);
 
 /// ConstantFoldCall - Attempt to constant fold a call to the specified function
 /// with the specified arguments, returning null if unsuccessful.
-Constant *ConstantFoldCall(ImmutableCallSite CS, Function *F,
+Constant *ConstantFoldCall(const CallBase *Call, Function *F,
                            ArrayRef<Constant *> Operands,
                            const TargetLibraryInfo *TLI = nullptr);
 
@@ -155,7 +153,7 @@ Constant *ConstantFoldLoadThroughBitcast(Constant *C, Type *DestTy,
 
 /// Check whether the given call has no side-effects.
 /// Specifically checks for math routimes which sometimes set errno.
-bool isMathLibCallNoop(CallSite CS, const TargetLibraryInfo *TLI);
+bool isMathLibCallNoop(const CallBase *Call, const TargetLibraryInfo *TLI);
 }
 
 #endif
diff --git a/include/llvm/Analysis/DOTGraphTraitsPass.h b/include/llvm/Analysis/DOTGraphTraitsPass.h
index b7447a0547d53d8b77e0a6d04c58b1dbccef7550..0410a3314659f25cea6a32c278f5ad7c96e915ff 100644
--- a/include/llvm/Analysis/DOTGraphTraitsPass.h
+++ b/include/llvm/Analysis/DOTGraphTraitsPass.h
@@ -1,9 +1,8 @@
 //===-- DOTGraphTraitsPass.h - Print/View dotty graphs-----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Analysis/DemandedBits.h b/include/llvm/Analysis/DemandedBits.h
index f751d2edcfffb24e21237eb40d33304a2243d9f7..04db3eb57c18e8f1ff0c646ca687340e279d4c71 100644
--- a/include/llvm/Analysis/DemandedBits.h
+++ b/include/llvm/Analysis/DemandedBits.h
@@ -1,9 +1,8 @@
 //===- llvm/Analysis/DemandedBits.h - Determine demanded bits ---*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -57,14 +56,17 @@ public:
   /// Return true if, during analysis, I could not be reached.
   bool isInstructionDead(Instruction *I);
 
+  /// Return whether this use is dead by means of not having any demanded bits.
+  bool isUseDead(Use *U);
+
   void print(raw_ostream &OS);
 
 private:
   void performAnalysis();
   void determineLiveOperandBits(const Instruction *UserI,
-    const Instruction *I, unsigned OperandNo,
+    const Value *Val, unsigned OperandNo,
     const APInt &AOut, APInt &AB,
-    KnownBits &Known, KnownBits &Known2);
+    KnownBits &Known, KnownBits &Known2, bool &KnownBitsComputed);
 
   Function &F;
   AssumptionCache &AC;
@@ -75,6 +77,9 @@ private:
   // The set of visited instructions (non-integer-typed only).
   SmallPtrSet<Instruction*, 32> Visited;
   DenseMap<Instruction *, APInt> AliveBits;
+  // Uses with no demanded bits. If the user also has no demanded bits, the use
+  // might not be stored explicitly in this map, to save memory during analysis.
+  SmallPtrSet<Use *, 16> DeadUses;
 };
 
 class DemandedBitsWrapperPass : public FunctionPass {
diff --git a/include/llvm/Analysis/DependenceAnalysis.h b/include/llvm/Analysis/DependenceAnalysis.h
index c8ec737a2cb91ec7fad5f104568ee02e823e234e..997013a5fc8e39907b53913782ebc2a06aad8dc4 100644
--- a/include/llvm/Analysis/DependenceAnalysis.h
+++ b/include/llvm/Analysis/DependenceAnalysis.h
@@ -1,9 +1,8 @@
 //===-- llvm/Analysis/DependenceAnalysis.h -------------------- -*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -275,6 +274,10 @@ template <typename T> class ArrayRef;
                    LoopInfo *LI)
         : AA(AA), SE(SE), LI(LI), F(F) {}
 
+    /// Handle transitive invalidation when the cached analysis results go away.
+    bool invalidate(Function &F, const PreservedAnalyses &PA,
+                    FunctionAnalysisManager::Invalidator &Inv);
+
     /// depends - Tests for a dependence between the Src and Dst instructions.
     /// Returns NULL if no dependence; otherwise, returns a Dependence (or a
     /// FullDependence) with as much information as can be gleaned.
@@ -936,6 +939,17 @@ template <typename T> class ArrayRef;
     friend struct AnalysisInfoMixin<DependenceAnalysis>;
   }; // class DependenceAnalysis
 
+  /// Printer pass to dump DA results.
+  struct DependenceAnalysisPrinterPass
+      : public PassInfoMixin<DependenceAnalysisPrinterPass> {
+    DependenceAnalysisPrinterPass(raw_ostream &OS) : OS(OS) {}
+
+    PreservedAnalyses run(Function &F, FunctionAnalysisManager &FAM);
+
+  private:
+    raw_ostream &OS;
+  }; // class DependenceAnalysisPrinterPass
+
   /// Legacy pass manager pass to access dependence information
   class DependenceAnalysisWrapperPass : public FunctionPass {
   public:
diff --git a/include/llvm/Analysis/DivergenceAnalysis.h b/include/llvm/Analysis/DivergenceAnalysis.h
index d834862db09573261cfd49609ed94b6b7ddf5d59..3cfb9d13df94d094760ee816d81a5229057ced09 100644
--- a/include/llvm/Analysis/DivergenceAnalysis.h
+++ b/include/llvm/Analysis/DivergenceAnalysis.h
@@ -1,9 +1,8 @@
 //===- llvm/Analysis/DivergenceAnalysis.h - Divergence Analysis -*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Analysis/DomPrinter.h b/include/llvm/Analysis/DomPrinter.h
index 0ed28994995a2a0404ab7db50ad62b5452540005..a177f877b2952cd01543bc643f74aa9c77e5d694 100644
--- a/include/llvm/Analysis/DomPrinter.h
+++ b/include/llvm/Analysis/DomPrinter.h
@@ -1,9 +1,8 @@
 //===-- DomPrinter.h - Dom printer external interface ------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/IR/DomTreeUpdater.h b/include/llvm/Analysis/DomTreeUpdater.h
similarity index 97%
rename from include/llvm/IR/DomTreeUpdater.h
rename to include/llvm/Analysis/DomTreeUpdater.h
index e5bb092d21ca7ac6cc74d567d10ab990ad669276..fcfd3c12f52ab275ce915e0bfafa69e46ea82cb4 100644
--- a/include/llvm/IR/DomTreeUpdater.h
+++ b/include/llvm/Analysis/DomTreeUpdater.h
@@ -1,9 +1,8 @@
 //===- DomTreeUpdater.h - DomTree/Post DomTree Updater ----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -12,8 +11,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef LLVM_DOMTREEUPDATER_H
-#define LLVM_DOMTREEUPDATER_H
+#ifndef LLVM_ANALYSIS_DOMTREEUPDATER_H
+#define LLVM_ANALYSIS_DOMTREEUPDATER_H
 
 #include "llvm/Analysis/PostDominators.h"
 #include "llvm/IR/Dominators.h"
@@ -254,4 +253,4 @@ private:
 };
 } // namespace llvm
 
-#endif // LLVM_DOMTREEUPDATER_H
+#endif // LLVM_ANALYSIS_DOMTREEUPDATER_H
diff --git a/include/llvm/Analysis/DominanceFrontier.h b/include/llvm/Analysis/DominanceFrontier.h
index d94c420d71775d9ec821d7250ec220723ce80094..c0bf30e162dddce2c6b4ae62de4da9d4f528cdcf 100644
--- a/include/llvm/Analysis/DominanceFrontier.h
+++ b/include/llvm/Analysis/DominanceFrontier.h
@@ -1,9 +1,8 @@
 //===- llvm/Analysis/DominanceFrontier.h - Dominator Frontiers --*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Analysis/DominanceFrontierImpl.h b/include/llvm/Analysis/DominanceFrontierImpl.h
index 99224c0bf13134cf1204fe595e6f762c95aa7e9b..aa764be93b91343f2180deaa4250aaf5162f98f8 100644
--- a/include/llvm/Analysis/DominanceFrontierImpl.h
+++ b/include/llvm/Analysis/DominanceFrontierImpl.h
@@ -1,9 +1,8 @@
 //===- llvm/Analysis/DominanceFrontier.h - Dominator Frontiers --*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Analysis/EHPersonalities.h b/include/llvm/Analysis/EHPersonalities.h
index fe0e65b828cac2cbeae9ce310d69410a30fd23ac..d89aa11617b5ac21e0ce3d9f4ff38ba60e97a55a 100644
--- a/include/llvm/Analysis/EHPersonalities.h
+++ b/include/llvm/Analysis/EHPersonalities.h
@@ -1,9 +1,8 @@
 //===- EHPersonalities.h - Compute EH-related information -----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/Analysis/GlobalsModRef.h b/include/llvm/Analysis/GlobalsModRef.h
index 09cef68ce70f78296c7f44142b55fbc7e93a0895..14b209717921569aa53f213fc12ac5fda15f6d89 100644
--- a/include/llvm/Analysis/GlobalsModRef.h
+++ b/include/llvm/Analysis/GlobalsModRef.h
@@ -1,9 +1,8 @@
 //===- GlobalsModRef.h - Simple Mod/Ref AA for Globals ----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
@@ -88,7 +87,7 @@ public:
   AliasResult alias(const MemoryLocation &LocA, const MemoryLocation &LocB);
 
   using AAResultBase::getModRefInfo;
-  ModRefInfo getModRefInfo(ImmutableCallSite CS, const MemoryLocation &Loc);
+  ModRefInfo getModRefInfo(const CallBase *Call, const MemoryLocation &Loc);
 
   /// getModRefBehavior - Return the behavior of the specified function if
   /// called from the specified call site.  The call site may be null in which
@@ -98,7 +97,7 @@ public:
   /// getModRefBehavior - Return the behavior of the specified function if
   /// called from the specified call site.  The call site may be null in which
   /// case the most generic behavior of this function should be returned.
-  FunctionModRefBehavior getModRefBehavior(ImmutableCallSite CS);
+  FunctionModRefBehavior getModRefBehavior(const CallBase *Call);
 
 private:
   FunctionInfo *getFunctionInfo(const Function *F);
@@ -113,7 +112,7 @@ private:
   void CollectSCCMembership(CallGraph &CG);
 
   bool isNonEscapingGlobalNoAlias(const GlobalValue *GV, const Value *V);
-  ModRefInfo getModRefInfoForArgument(ImmutableCallSite CS,
+  ModRefInfo getModRefInfoForArgument(const CallBase *Call,
                                       const GlobalValue *GV);
 };
 
diff --git a/include/llvm/Analysis/GuardUtils.h b/include/llvm/Analysis/GuardUtils.h
index 3b151eeafc81db2494ba16d90af036bd48222edc..41e7b7c06c7501b17d4fd2ca0d2b3165dbae6da3 100644
--- a/include/llvm/Analysis/GuardUtils.h
+++ b/include/llvm/Analysis/GuardUtils.h
@@ -1,9 +1,8 @@
 //===-- GuardUtils.h - Utils for work with guards ---------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 // Utils that are used to perform analyzes related to guards and their
@@ -15,12 +14,31 @@
 
 namespace llvm {
 
+class BasicBlock;
 class User;
+class Value;
 
-/// Returns true iff \p U has semantics of a guard.
+/// Returns true iff \p U has semantics of a guard expressed in a form of call
+/// of llvm.experimental.guard intrinsic.
 bool isGuard(const User *U);
 
+/// Returns true iff \p U has semantics of a guard expressed in a form of a
+/// widenable conditional branch to deopt block.
+bool isGuardAsWidenableBranch(const User *U);
+
+/// If U is widenable branch looking like:
+///   %cond = ...
+///   %wc = call i1 @llvm.experimental.widenable.condition()
+///   %branch_cond = and i1 %cond, %wc
+///   br i1 %branch_cond, label %if_true_bb, label %if_false_bb ; <--- U
+/// The function returns true, and the values %cond and %wc and blocks
+/// %if_true_bb, if_false_bb are returned in
+/// the parameters (Condition, WidenableCondition, IfTrueBB and IfFalseFF)
+/// respectively. If \p U does not match this pattern, return false.
+bool parseWidenableBranch(const User *U, Value *&Condition,
+                          Value *&WidenableCondition, BasicBlock *&IfTrueBB,
+                          BasicBlock *&IfFalseBB);
+
 } // llvm
 
 #endif // LLVM_ANALYSIS_GUARDUTILS_H
-
diff --git a/include/llvm/Analysis/IVDescriptors.h b/include/llvm/Analysis/IVDescriptors.h
index 64b4ae23cc59de8e7d0790995f02105c0091cb45..254cabfc85ba743cac0a955d88777257672f3891 100644
--- a/include/llvm/Analysis/IVDescriptors.h
+++ b/include/llvm/Analysis/IVDescriptors.h
@@ -1,9 +1,8 @@
 //===- llvm/Analysis/IVDescriptors.h - IndVar Descriptors -------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Analysis/IVUsers.h b/include/llvm/Analysis/IVUsers.h
index 035b974c5c1dec9b203931544e78f0a331e4646e..f8ea3bcca229c4dae6c2391ab125e53eee5c7aae 100644
--- a/include/llvm/Analysis/IVUsers.h
+++ b/include/llvm/Analysis/IVUsers.h
@@ -1,9 +1,8 @@
 //===- llvm/Analysis/IVUsers.h - Induction Variable Users -------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Analysis/IndirectCallPromotionAnalysis.h b/include/llvm/Analysis/IndirectCallPromotionAnalysis.h
index be3a28424cf53dc465232b9569fc3658d6e5593c..8a05e913a91063b7a21b2a565ad7ca32eff3c63a 100644
--- a/include/llvm/Analysis/IndirectCallPromotionAnalysis.h
+++ b/include/llvm/Analysis/IndirectCallPromotionAnalysis.h
@@ -1,9 +1,8 @@
 //===- IndirectCallPromotionAnalysis.h - Indirect call analysis -*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/include/llvm/Analysis/IndirectCallSiteVisitor.h b/include/llvm/Analysis/IndirectCallSiteVisitor.h
deleted file mode 100644
index a30b59fee4f443cd59534a1dec89311a41070270..0000000000000000000000000000000000000000
--- a/include/llvm/Analysis/IndirectCallSiteVisitor.h
+++ /dev/null
@@ -1,40 +0,0 @@
-//===-- IndirectCallSiteVisitor.h - indirect call-sites visitor -----------===//
-//
-//                      The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This file implements defines a visitor class and a helper function that find
-// all indirect call-sites in a function.
-
-#ifndef LLVM_ANALYSIS_INDIRECTCALLSITEVISITOR_H
-#define LLVM_ANALYSIS_INDIRECTCALLSITEVISITOR_H
-
-#include "llvm/IR/InstVisitor.h"
-#include <vector>
-
-namespace llvm {
-// Visitor class that finds all indirect call sites.
-struct PGOIndirectCallSiteVisitor
-    : public InstVisitor<PGOIndirectCallSiteVisitor> {
-  std::vector<Instruction *> IndirectCallInsts;
-  PGOIndirectCallSiteVisitor() {}
-
-  void visitCallSite(CallSite CS) {
-    if (CS.isIndirectCall())
-      IndirectCallInsts.push_back(CS.getInstruction());
-  }
-};
-
-// Helper function that finds all indirect call sites.
-inline std::vector<Instruction *> findIndirectCallSites(Function &F) {
-  PGOIndirectCallSiteVisitor ICV;
-  ICV.visit(F);
-  return ICV.IndirectCallInsts;
-}
-}
-
-#endif
diff --git a/include/llvm/Analysis/IndirectCallVisitor.h b/include/llvm/Analysis/IndirectCallVisitor.h
new file mode 100644
index 0000000000000000000000000000000000000000..1d1f3f4cc5c0bc44fd12c5bc80ed0536158980a3
--- /dev/null
+++ b/include/llvm/Analysis/IndirectCallVisitor.h
@@ -0,0 +1,38 @@
+//===-- IndirectCallVisitor.h - indirect call visitor ---------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements defines a visitor class and a helper function that find
+// all indirect call-sites in a function.
+
+#ifndef LLVM_ANALYSIS_INDIRECTCALLVISITOR_H
+#define LLVM_ANALYSIS_INDIRECTCALLVISITOR_H
+
+#include "llvm/IR/InstVisitor.h"
+#include <vector>
+
+namespace llvm {
+// Visitor class that finds all indirect call.
+struct PGOIndirectCallVisitor : public InstVisitor<PGOIndirectCallVisitor> {
+  std::vector<Instruction *> IndirectCalls;
+  PGOIndirectCallVisitor() {}
+
+  void visitCallBase(CallBase &Call) {
+    if (Call.isIndirectCall())
+      IndirectCalls.push_back(&Call);
+  }
+};
+
+// Helper function that finds all indirect call sites.
+inline std::vector<Instruction *> findIndirectCalls(Function &F) {
+  PGOIndirectCallVisitor ICV;
+  ICV.visit(F);
+  return ICV.IndirectCalls;
+}
+} // namespace llvm
+
+#endif
diff --git a/include/llvm/Analysis/InlineCost.h b/include/llvm/Analysis/InlineCost.h
index 4c270354b0c4fd72046276efc12271166377f066..e386093351695226a1a460dcbb522e65fbdc9ef4 100644
--- a/include/llvm/Analysis/InlineCost.h
+++ b/include/llvm/Analysis/InlineCost.h
@@ -1,9 +1,8 @@
 //===- InlineCost.h - Cost analysis for inliner -----------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -232,7 +231,7 @@ getInlineCost(CallSite CS, Function *Callee, const InlineParams &Params,
               ProfileSummaryInfo *PSI, OptimizationRemarkEmitter *ORE);
 
 /// Minimal filter to detect invalid constructs for inlining.
-bool isInlineViable(Function &Callee);
+InlineResult isInlineViable(Function &Callee);
 }
 
 #endif
diff --git a/include/llvm/Analysis/InstructionPrecedenceTracking.h b/include/llvm/Analysis/InstructionPrecedenceTracking.h
index b754557ecc41d36780319f743ebbbdf183f8d809..eb72eac23ecce01e0643db88d83022cb4e2b015e 100644
--- a/include/llvm/Analysis/InstructionPrecedenceTracking.h
+++ b/include/llvm/Analysis/InstructionPrecedenceTracking.h
@@ -1,9 +1,8 @@
 //===-- InstructionPrecedenceTracking.h -------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 // Implements a class that is able to define some instructions as "special"
@@ -75,8 +74,14 @@ protected:
   virtual ~InstructionPrecedenceTracking() = default;
 
 public:
-  /// Clears cached information about this particular block.
-  void invalidateBlock(const BasicBlock *BB);
+  /// Notifies this tracking that we are going to insert a new instruction \p
+  /// Inst to the basic block \p BB. It makes all necessary updates to internal
+  /// caches to keep them consistent.
+  void insertInstructionTo(const Instruction *Inst, const BasicBlock *BB);
+
+  /// Notifies this tracking that we are going to remove the instruction \p Inst
+  /// It makes all necessary updates to internal caches to keep them consistent.
+  void removeInstruction(const Instruction *Inst);
 
   /// Invalidates all information from this tracking.
   void clear();
diff --git a/include/llvm/Analysis/InstructionSimplify.h b/include/llvm/Analysis/InstructionSimplify.h
index 6662e91037e1a2d43e584c848af4fd9a5d27927f..a9040439d94c1f61b3ff902bef2784e2ad6c4342 100644
--- a/include/llvm/Analysis/InstructionSimplify.h
+++ b/include/llvm/Analysis/InstructionSimplify.h
@@ -1,9 +1,8 @@
 //===-- InstructionSimplify.h - Fold instrs into simpler forms --*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -41,8 +40,8 @@ class Function;
 template <typename T, typename... TArgs> class AnalysisManager;
 template <class T> class ArrayRef;
 class AssumptionCache;
+class CallBase;
 class DominatorTree;
-class ImmutableCallSite;
 class DataLayout;
 class FastMathFlags;
 struct LoopStandardAnalysisResults;
@@ -239,15 +238,15 @@ Value *SimplifyFPBinOp(unsigned Opcode, Value *LHS, Value *RHS,
                        FastMathFlags FMF, const SimplifyQuery &Q);
 
 /// Given a callsite, fold the result or return null.
-Value *SimplifyCall(ImmutableCallSite CS, const SimplifyQuery &Q);
+Value *SimplifyCall(CallBase *Call, const SimplifyQuery &Q);
 
 /// Given a function and iterators over arguments, fold the result or return
 /// null.
-Value *SimplifyCall(ImmutableCallSite CS, Value *V, User::op_iterator ArgBegin,
+Value *SimplifyCall(CallBase *Call, Value *V, User::op_iterator ArgBegin,
                     User::op_iterator ArgEnd, const SimplifyQuery &Q);
 
 /// Given a function and set of arguments, fold the result or return null.
-Value *SimplifyCall(ImmutableCallSite CS, Value *V, ArrayRef<Value *> Args,
+Value *SimplifyCall(CallBase *Call, Value *V, ArrayRef<Value *> Args,
                     const SimplifyQuery &Q);
 
 /// See if we can compute a simplified version of this instruction. If not,
diff --git a/include/llvm/Analysis/Interval.h b/include/llvm/Analysis/Interval.h
index f3714dddedd5b08f231801ce8cfb0e51d1fc0c5f..5c9a4535bc7fd027ccf9179d0d8c92205fab62ff 100644
--- a/include/llvm/Analysis/Interval.h
+++ b/include/llvm/Analysis/Interval.h
@@ -1,9 +1,8 @@
 //===- llvm/Analysis/Interval.h - Interval Class Declaration ----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Analysis/IntervalIterator.h b/include/llvm/Analysis/IntervalIterator.h
index 6ffcae592e98bb53a38a34109ef16d0c438d37b1..efaaf9715b3de6d85ba6e69dc50fecf237b1b4e7 100644
--- a/include/llvm/Analysis/IntervalIterator.h
+++ b/include/llvm/Analysis/IntervalIterator.h
@@ -1,9 +1,8 @@
 //===- IntervalIterator.h - Interval Iterator Declaration -------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Analysis/IntervalPartition.h b/include/llvm/Analysis/IntervalPartition.h
index 50335165711f53a65733c49caa143612d7045bc8..5b127c25a2b82153fd2b8a0cc9eff8b97fe87806 100644
--- a/include/llvm/Analysis/IntervalPartition.h
+++ b/include/llvm/Analysis/IntervalPartition.h
@@ -1,9 +1,8 @@
 //===- IntervalPartition.h - Interval partition Calculation -----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Analysis/IteratedDominanceFrontier.h b/include/llvm/Analysis/IteratedDominanceFrontier.h
index 3083db75b81c1c1eb15a6f3b357d6e29dbb16c12..e7d19d1a815f7c3621945693056350597dcddc59 100644
--- a/include/llvm/Analysis/IteratedDominanceFrontier.h
+++ b/include/llvm/Analysis/IteratedDominanceFrontier.h
@@ -1,9 +1,8 @@
 //===- IteratedDominanceFrontier.h - Calculate IDF --------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/include/llvm/Analysis/LazyBlockFrequencyInfo.h b/include/llvm/Analysis/LazyBlockFrequencyInfo.h
index d1afb63d7e08670227c78d9f9e39e80086d2715f..0e7dc943bacf5807de0d8b7c01b8ea7c29afc659 100644
--- a/include/llvm/Analysis/LazyBlockFrequencyInfo.h
+++ b/include/llvm/Analysis/LazyBlockFrequencyInfo.h
@@ -1,9 +1,8 @@
 //===- LazyBlockFrequencyInfo.h - Lazy Block Frequency Analysis -*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Analysis/LazyBranchProbabilityInfo.h b/include/llvm/Analysis/LazyBranchProbabilityInfo.h
index 9e6bcfedcbb94486bb23bc2926d3ae34eb9171bc..cae0778cd16d808a3f38cb8fce9bbe1cddfbf08a 100644
--- a/include/llvm/Analysis/LazyBranchProbabilityInfo.h
+++ b/include/llvm/Analysis/LazyBranchProbabilityInfo.h
@@ -1,9 +1,8 @@
 //===- LazyBranchProbabilityInfo.h - Lazy Branch Probability ----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Analysis/LazyCallGraph.h b/include/llvm/Analysis/LazyCallGraph.h
index d1ec6a9dcc55d5f3a619f62394c036b8bb31eece..328654763b597e92a4501cfa2af79543f67f026d 100644
--- a/include/llvm/Analysis/LazyCallGraph.h
+++ b/include/llvm/Analysis/LazyCallGraph.h
@@ -1,9 +1,8 @@
 //===- LazyCallGraph.h - Analysis of a Module's call graph ------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/include/llvm/Analysis/LazyValueInfo.h b/include/llvm/Analysis/LazyValueInfo.h
index 1a4fdb59142778f3e2738bdb140dd5989f8562dc..570a5044f6f87360154c1b48d998856bf1056c02 100644
--- a/include/llvm/Analysis/LazyValueInfo.h
+++ b/include/llvm/Analysis/LazyValueInfo.h
@@ -1,9 +1,8 @@
 //===- LazyValueInfo.h - Value constraint analysis --------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Analysis/LegacyDivergenceAnalysis.h b/include/llvm/Analysis/LegacyDivergenceAnalysis.h
index fc426ad7fb645f0088eb415b625e07bd521776a0..0a338b816640e3e08ae015ef979687ecfa76b822 100644
--- a/include/llvm/Analysis/LegacyDivergenceAnalysis.h
+++ b/include/llvm/Analysis/LegacyDivergenceAnalysis.h
@@ -1,9 +1,8 @@
 //===- llvm/Analysis/LegacyDivergenceAnalysis.h - KernelDivergence Analysis -*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Analysis/Lint.h b/include/llvm/Analysis/Lint.h
index db5919fd91c72d175ca8d79381785d79dcb3c370..0fea81e215c9107762a398d03867308083ef3fd9 100644
--- a/include/llvm/Analysis/Lint.h
+++ b/include/llvm/Analysis/Lint.h
@@ -1,9 +1,8 @@
 //===-- llvm/Analysis/Lint.h - LLVM IR Lint ---------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Analysis/Loads.h b/include/llvm/Analysis/Loads.h
index f110c28bfc6d2feff77d3d1901ae25eb84a7c893..04401162fb3b39434871f99ed2206a39377e89cf 100644
--- a/include/llvm/Analysis/Loads.h
+++ b/include/llvm/Analysis/Loads.h
@@ -1,9 +1,8 @@
 //===- Loads.h - Local load analysis --------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Analysis/LoopAccessAnalysis.h b/include/llvm/Analysis/LoopAccessAnalysis.h
index b5a964de9eb0b61b6cf723c0941e57d74de169f6..fa34afaf9589ba73307147eb375b30602c65f260 100644
--- a/include/llvm/Analysis/LoopAccessAnalysis.h
+++ b/include/llvm/Analysis/LoopAccessAnalysis.h
@@ -1,9 +1,8 @@
 //===- llvm/Analysis/LoopAccessAnalysis.h -----------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -104,7 +103,9 @@ public:
     // Can vectorize safely without RT checks. All dependences are known to be
     // safe.
     Safe,
-    // Cannot vectorize due to unsafe or unknown dependencies.
+    // Can possibly vectorize with RT checks to overcome unknown dependencies.
+    PossiblySafeWithRtChecks,
+    // Cannot vectorize due to known unsafe dependencies.
     Unsafe,
   };
 
@@ -175,7 +176,7 @@ public:
 
   MemoryDepChecker(PredicatedScalarEvolution &PSE, const Loop *L)
       : PSE(PSE), InnermostLoop(L), AccessIdx(0), MaxSafeRegisterWidth(-1U),
-        ShouldRetryWithRuntimeCheck(false),
+        FoundNonConstantDistanceDependence(false),
         Status(VectorizationSafetyStatus::Safe), RecordDependences(true) {}
 
   /// Register the location (instructions are given increasing numbers)
@@ -218,7 +219,10 @@ public:
 
   /// In same cases when the dependency check fails we can still
   /// vectorize the loop with a dynamic array access check.
-  bool shouldRetryWithRuntimeCheck() { return ShouldRetryWithRuntimeCheck; }
+  bool shouldRetryWithRuntimeCheck() const {
+    return FoundNonConstantDistanceDependence &&
+           Status == VectorizationSafetyStatus::PossiblySafeWithRtChecks;
+  }
 
   /// Returns the memory dependences.  If null is returned we exceeded
   /// the MaxDependences threshold and this information is not
@@ -280,10 +284,11 @@ private:
 
   /// If we see a non-constant dependence distance we can still try to
   /// vectorize this loop with runtime checks.
-  bool ShouldRetryWithRuntimeCheck;
+  bool FoundNonConstantDistanceDependence;
 
   /// Result of the dependence checks, indicating whether the checked
-  /// dependences are safe for vectorization or not.
+  /// dependences are safe for vectorization, require RT checks or are known to
+  /// be unsafe.
   VectorizationSafetyStatus Status;
 
   //// True if Dependences reflects the dependences in the
@@ -319,7 +324,8 @@ private:
   bool couldPreventStoreLoadForward(uint64_t Distance, uint64_t TypeByteSize);
 
   /// Updates the current safety status with \p S. We can go from Safe to
-  /// to Unsafe.
+  /// either PossiblySafeWithRtChecks or Unsafe and from
+  /// PossiblySafeWithRtChecks to Unsafe.
   void mergeInStatus(VectorizationSafetyStatus S);
 };
 
diff --git a/include/llvm/Analysis/LoopAnalysisManager.h b/include/llvm/Analysis/LoopAnalysisManager.h
index 00e562c4f31f0bb71807179a2efdbec3f7437c8c..b87d981ec1eef8c5bda30716b74926df0e06bd12 100644
--- a/include/llvm/Analysis/LoopAnalysisManager.h
+++ b/include/llvm/Analysis/LoopAnalysisManager.h
@@ -1,9 +1,8 @@
 //===- LoopAnalysisManager.h - Loop analysis management ---------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/include/llvm/Analysis/LoopInfo.h b/include/llvm/Analysis/LoopInfo.h
index 30b29d66a1d11d97f3dc38a95860d4d5efd3273b..0899630fe8ead5c3435dc845f0dc11c2c4444cd4 100644
--- a/include/llvm/Analysis/LoopInfo.h
+++ b/include/llvm/Analysis/LoopInfo.h
@@ -1,9 +1,8 @@
 //===- llvm/Analysis/LoopInfo.h - Natural Loop Calculator -------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -408,6 +407,12 @@ public:
   /// Verify loop structure of this loop and all nested loops.
   void verifyLoopNest(DenseSet<const LoopT *> *Loops) const;
 
+  /// Returns true if the loop is annotated parallel.
+  ///
+  /// Derived classes can override this method using static template
+  /// polymorphism.
+  bool isAnnotatedParallel() const { return false; }
+
   /// Print loop with all the BBs inside it.
   void print(raw_ostream &OS, unsigned Depth = 0, bool Verbose = false) const;
 
@@ -989,6 +994,46 @@ public:
 /// Function to print a loop's contents as LLVM's text IR assembly.
 void printLoop(Loop &L, raw_ostream &OS, const std::string &Banner = "");
 
+/// Find and return the loop attribute node for the attribute @p Name in
+/// @p LoopID. Return nullptr if there is no such attribute.
+MDNode *findOptionMDForLoopID(MDNode *LoopID, StringRef Name);
+
+/// Find string metadata for a loop.
+///
+/// Returns the MDNode where the first operand is the metadata's name. The
+/// following operands are the metadata's values. If no metadata with @p Name is
+/// found, return nullptr.
+MDNode *findOptionMDForLoop(const Loop *TheLoop, StringRef Name);
+
+/// Return whether an MDNode might represent an access group.
+///
+/// Access group metadata nodes have to be distinct and empty. Being
+/// always-empty ensures that it never needs to be changed (which -- because
+/// MDNodes are designed immutable -- would require creating a new MDNode). Note
+/// that this is not a sufficient condition: not every distinct and empty NDNode
+/// is representing an access group.
+bool isValidAsAccessGroup(MDNode *AccGroup);
+
+/// Create a new LoopID after the loop has been transformed.
+///
+/// This can be used when no follow-up loop attributes are defined
+/// (llvm::makeFollowupLoopID returning None) to stop transformations to be
+/// applied again.
+///
+/// @param Context        The LLVMContext in which to create the new LoopID.
+/// @param OrigLoopID     The original LoopID; can be nullptr if the original
+///                       loop has no LoopID.
+/// @param RemovePrefixes Remove all loop attributes that have these prefixes.
+///                       Use to remove metadata of the transformation that has
+///                       been applied.
+/// @param AddAttrs       Add these loop attributes to the new LoopID.
+///
+/// @return A new LoopID that can be applied using Loop::setLoopID().
+llvm::MDNode *
+makePostTransformationMetadata(llvm::LLVMContext &Context, MDNode *OrigLoopID,
+                               llvm::ArrayRef<llvm::StringRef> RemovePrefixes,
+                               llvm::ArrayRef<llvm::MDNode *> AddAttrs);
+
 } // End llvm namespace
 
 #endif
diff --git a/include/llvm/Analysis/LoopInfoImpl.h b/include/llvm/Analysis/LoopInfoImpl.h
index d3054b7cf85a807af08c76309817e800513f77ac..ad4250831479c7b610302948ce7d65a76c3a137e 100644
--- a/include/llvm/Analysis/LoopInfoImpl.h
+++ b/include/llvm/Analysis/LoopInfoImpl.h
@@ -1,9 +1,8 @@
 //===- llvm/Analysis/LoopInfoImpl.h - Natural Loop Calculator ---*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -392,7 +391,10 @@ void LoopBase<BlockT, LoopT>::verifyLoopNest(
 template <class BlockT, class LoopT>
 void LoopBase<BlockT, LoopT>::print(raw_ostream &OS, unsigned Depth,
                                     bool Verbose) const {
-  OS.indent(Depth * 2) << "Loop at depth " << getLoopDepth() << " containing: ";
+  OS.indent(Depth * 2);
+  if (static_cast<const LoopT *>(this)->isAnnotatedParallel())
+    OS << "Parallel ";
+  OS << "Loop at depth " << getLoopDepth() << " containing: ";
 
   BlockT *H = getHeader();
   for (unsigned i = 0; i < getBlocks().size(); ++i) {
diff --git a/include/llvm/Analysis/LoopIterator.h b/include/llvm/Analysis/LoopIterator.h
index 91c54b23029b3af39a2c81696f3fd1bffe6603ed..fa4da4283f55a932547738d1f4b558c6972e2f5e 100644
--- a/include/llvm/Analysis/LoopIterator.h
+++ b/include/llvm/Analysis/LoopIterator.h
@@ -1,9 +1,8 @@
 //===--------- LoopIterator.h - Iterate over loop blocks --------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 // This file defines iterators to visit the basic blocks within a loop.
diff --git a/include/llvm/Analysis/LoopPass.h b/include/llvm/Analysis/LoopPass.h
index 86cfecd9df111c0f5adcbd0f32eb73948233f4c4..9215ab34ec6d15a1ea3525ff14e4f2453bccfef5 100644
--- a/include/llvm/Analysis/LoopPass.h
+++ b/include/llvm/Analysis/LoopPass.h
@@ -1,9 +1,8 @@
 //===- LoopPass.h - LoopPass class ----------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Analysis/LoopUnrollAnalyzer.h b/include/llvm/Analysis/LoopUnrollAnalyzer.h
index f45bf0b223b8ec18b2a6bef9fb71758ab21cca5b..5f332e3cac1623e9ef60653c4eac712d958d5113 100644
--- a/include/llvm/Analysis/LoopUnrollAnalyzer.h
+++ b/include/llvm/Analysis/LoopUnrollAnalyzer.h
@@ -1,9 +1,8 @@
 //===- llvm/Analysis/LoopUnrollAnalyzer.h - Loop Unroll Analyzer-*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Analysis/MemoryBuiltins.h b/include/llvm/Analysis/MemoryBuiltins.h
index 5418128f16ef840b5fd8f1c4786e35940f599f58..49b87a43a485295b2f5402af54cbfea9937b8021 100644
--- a/include/llvm/Analysis/MemoryBuiltins.h
+++ b/include/llvm/Analysis/MemoryBuiltins.h
@@ -1,9 +1,8 @@
 //==- llvm/Analysis/MemoryBuiltins.h - Calls to memory builtins --*- C++ -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -178,14 +177,13 @@ bool getObjectSize(const Value *Ptr, uint64_t &Size, const DataLayout &DL,
                    const TargetLibraryInfo *TLI, ObjectSizeOpts Opts = {});
 
 /// Try to turn a call to \@llvm.objectsize into an integer value of the given
-/// Type. Returns null on failure.
-/// If MustSucceed is true, this function will not return null, and may return
-/// conservative values governed by the second argument of the call to
-/// objectsize.
-ConstantInt *lowerObjectSizeCall(IntrinsicInst *ObjectSize,
-                                 const DataLayout &DL,
-                                 const TargetLibraryInfo *TLI,
-                                 bool MustSucceed);
+/// Type. Returns null on failure. If MustSucceed is true, this function will
+/// not return null, and may return conservative values governed by the second
+/// argument of the call to objectsize.
+Value *lowerObjectSizeCall(IntrinsicInst *ObjectSize, const DataLayout &DL,
+                           const TargetLibraryInfo *TLI, bool MustSucceed);
+
+
 
 using SizeOffsetType = std::pair<APInt, APInt>;
 
@@ -265,17 +263,17 @@ class ObjectSizeOffsetEvaluator
   Value *Zero;
   CacheMapTy CacheMap;
   PtrSetTy SeenVals;
-  bool RoundToAlign;
-
-  SizeOffsetEvalType unknown() {
-    return std::make_pair(nullptr, nullptr);
-  }
+  ObjectSizeOpts EvalOpts;
 
   SizeOffsetEvalType compute_(Value *V);
 
 public:
+  static SizeOffsetEvalType unknown() {
+    return std::make_pair(nullptr, nullptr);
+  }
+
   ObjectSizeOffsetEvaluator(const DataLayout &DL, const TargetLibraryInfo *TLI,
-                            LLVMContext &Context, bool RoundToAlign = false);
+                            LLVMContext &Context, ObjectSizeOpts EvalOpts = {});
 
   SizeOffsetEvalType compute(Value *V);
 
diff --git a/include/llvm/Analysis/MemoryDependenceAnalysis.h b/include/llvm/Analysis/MemoryDependenceAnalysis.h
index 52340b0cb51ca4e70c6bb626f50aee60b6a44730..de57499032acaad2be564844463786775e98d184 100644
--- a/include/llvm/Analysis/MemoryDependenceAnalysis.h
+++ b/include/llvm/Analysis/MemoryDependenceAnalysis.h
@@ -1,9 +1,8 @@
 //===- llvm/Analysis/MemoryDependenceAnalysis.h - Memory Deps ---*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -37,7 +36,6 @@
 namespace llvm {
 
 class AssumptionCache;
-class CallSite;
 class DominatorTree;
 class Function;
 class Instruction;
@@ -398,7 +396,7 @@ public:
   /// invalidated on the next non-local query or when an instruction is
   /// removed.  Clients must copy this data if they want it around longer than
   /// that.
-  const NonLocalDepInfo &getNonLocalCallDependency(CallSite QueryCS);
+  const NonLocalDepInfo &getNonLocalCallDependency(CallBase *QueryCall);
 
   /// Perform a full dependency query for an access to the QueryInst's
   /// specified memory location, returning the set of instructions that either
@@ -482,9 +480,9 @@ public:
   void releaseMemory();
 
 private:
-  MemDepResult getCallSiteDependencyFrom(CallSite C, bool isReadOnlyCall,
-                                         BasicBlock::iterator ScanIt,
-                                         BasicBlock *BB);
+  MemDepResult getCallDependencyFrom(CallBase *Call, bool isReadOnlyCall,
+                                     BasicBlock::iterator ScanIt,
+                                     BasicBlock *BB);
   bool getNonLocalPointerDepFromBB(Instruction *QueryInst,
                                    const PHITransAddr &Pointer,
                                    const MemoryLocation &Loc, bool isLoad,
diff --git a/include/llvm/Analysis/MemoryLocation.h b/include/llvm/Analysis/MemoryLocation.h
index cf839c5a1eb8764cd1ac8cafcf52d45e62f846e1..7c26353e618b6a90426c22fc4ea9df589bb5c67e 100644
--- a/include/llvm/Analysis/MemoryLocation.h
+++ b/include/llvm/Analysis/MemoryLocation.h
@@ -1,9 +1,8 @@
 //===- MemoryLocation.h - Memory location descriptions ----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
@@ -16,9 +15,9 @@
 #ifndef LLVM_ANALYSIS_MEMORYLOCATION_H
 #define LLVM_ANALYSIS_MEMORYLOCATION_H
 
-#include "llvm/ADT/Optional.h"
 #include "llvm/ADT/DenseMapInfo.h"
-#include "llvm/IR/CallSite.h"
+#include "llvm/ADT/Optional.h"
+#include "llvm/IR/Instructions.h"
 #include "llvm/IR/Metadata.h"
 
 namespace llvm {
@@ -135,6 +134,9 @@ public:
     return (Value & ImpreciseBit) == 0;
   }
 
+  // Convenience method to check if this LocationSize's value is 0.
+  bool isZero() const { return hasValue() && getValue() == 0; }
+
   bool operator==(const LocationSize &Other) const {
     return Value == Other.Value;
   }
@@ -231,11 +233,11 @@ public:
   static MemoryLocation getForDest(const AnyMemIntrinsic *MI);
 
   /// Return a location representing a particular argument of a call.
-  static MemoryLocation getForArgument(ImmutableCallSite CS, unsigned ArgIdx,
+  static MemoryLocation getForArgument(const CallBase *Call, unsigned ArgIdx,
                                        const TargetLibraryInfo *TLI);
-  static MemoryLocation getForArgument(ImmutableCallSite CS, unsigned ArgIdx,
+  static MemoryLocation getForArgument(const CallBase *Call, unsigned ArgIdx,
                                        const TargetLibraryInfo &TLI) {
-    return getForArgument(CS, ArgIdx, &TLI);
+    return getForArgument(Call, ArgIdx, &TLI);
   }
 
   explicit MemoryLocation(const Value *Ptr = nullptr,
diff --git a/include/llvm/Analysis/MemorySSA.h b/include/llvm/Analysis/MemorySSA.h
index 620083717eb462bb6105cc86f8574d3c4b447e6a..fa92fd341ecbcb863f1b7a5c418af6c5aab4ae2a 100644
--- a/include/llvm/Analysis/MemorySSA.h
+++ b/include/llvm/Analysis/MemorySSA.h
@@ -1,9 +1,8 @@
 //===- MemorySSA.h - Build Memory SSA ---------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -404,6 +403,7 @@ public:
 
   void resetOptimized() {
     OptimizedID = INVALID_MEMORYACCESS_ID;
+    setOperand(1, nullptr);
   }
 
   void print(raw_ostream &OS) const;
@@ -703,6 +703,7 @@ public:
   ~MemorySSA();
 
   MemorySSAWalker *getWalker();
+  MemorySSAWalker *getSkipSelfWalker();
 
   /// Given a memory Mod/Ref'ing instruction, get the MemorySSA
   /// access associated with it. If passed a basic block gets the memory phi
@@ -774,9 +775,6 @@ public:
   /// all uses, uses appear in the right places).  This is used by unit tests.
   void verifyMemorySSA() const;
 
-  /// Check clobber sanity for an access.
-  void checkClobberSanityAccess(const MemoryAccess *MA) const;
-
   /// Used in various insertion functions to specify whether we are talking
   /// about the beginning or end of a block.
   enum InsertionPlace { Beginning, End };
@@ -791,7 +789,6 @@ protected:
   void verifyDomination(Function &F) const;
   void verifyOrdering(Function &F) const;
   void verifyDominationNumbers(const Function &F) const;
-  void verifyClobberSanity(const Function &F) const;
 
   // This is used by the use optimizer and updater.
   AccessList *getWritableBlockAccesses(const BasicBlock *BB) const {
@@ -828,7 +825,9 @@ protected:
                                       const MemoryUseOrDef *Template = nullptr);
 
 private:
+  class ClobberWalkerBase;
   class CachingWalker;
+  class SkipSelfWalker;
   class OptimizeUses;
 
   CachingWalker *getWalkerImpl();
@@ -882,7 +881,9 @@ private:
   mutable DenseMap<const MemoryAccess *, unsigned long> BlockNumbering;
 
   // Memory SSA building info
+  std::unique_ptr<ClobberWalkerBase> WalkerBase;
   std::unique_ptr<CachingWalker> Walker;
+  std::unique_ptr<SkipSelfWalker> SkipWalker;
   unsigned NextID;
 };
 
diff --git a/include/llvm/Analysis/MemorySSAUpdater.h b/include/llvm/Analysis/MemorySSAUpdater.h
index 098876efdaf8ec3bb947447ad3e7ab284fffff47..58cf1cc6b5dccb5897cbebabafb80327ff4cdf7e 100644
--- a/include/llvm/Analysis/MemorySSAUpdater.h
+++ b/include/llvm/Analysis/MemorySSAUpdater.h
@@ -1,9 +1,8 @@
 //===- MemorySSAUpdater.h - Memory SSA Updater-------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -35,6 +34,7 @@
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/SmallSet.h"
 #include "llvm/ADT/SmallVector.h"
+#include "llvm/Analysis/LoopInfo.h"
 #include "llvm/Analysis/LoopIterator.h"
 #include "llvm/Analysis/MemorySSA.h"
 #include "llvm/IR/BasicBlock.h"
@@ -221,14 +221,14 @@ public:
   /// associated with it is erased from the program.  For example, if a store or
   /// load is simply erased (not replaced), removeMemoryAccess should be called
   /// on the MemoryAccess for that store/load.
-  void removeMemoryAccess(MemoryAccess *);
+  void removeMemoryAccess(MemoryAccess *, bool OptimizePhis = false);
 
   /// Remove MemoryAccess for a given instruction, if a MemoryAccess exists.
   /// This should be called when an instruction (load/store) is deleted from
   /// the program.
-  void removeMemoryAccess(const Instruction *I) {
+  void removeMemoryAccess(const Instruction *I, bool OptimizePhis = false) {
     if (MemoryAccess *MA = MSSA->getMemoryAccess(I))
-      removeMemoryAccess(MA);
+      removeMemoryAccess(MA, OptimizePhis);
   }
 
   /// Remove all MemoryAcceses in a set of BasicBlocks about to be deleted.
diff --git a/include/llvm/Analysis/ModuleSummaryAnalysis.h b/include/llvm/Analysis/ModuleSummaryAnalysis.h
index 9af7859cb4bf8435bba7240513654fab62a99759..1572a49e3384396f25f37aae09e5362897b34b95 100644
--- a/include/llvm/Analysis/ModuleSummaryAnalysis.h
+++ b/include/llvm/Analysis/ModuleSummaryAnalysis.h
@@ -1,9 +1,8 @@
 //===- ModuleSummaryAnalysis.h - Module summary index builder ---*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/include/llvm/Analysis/MustExecute.h b/include/llvm/Analysis/MustExecute.h
index b973447ca8221fb3d6d86f6b42701995062a8414..3ef539c89d97460a53c0cbb525aa751c656cc399 100644
--- a/include/llvm/Analysis/MustExecute.h
+++ b/include/llvm/Analysis/MustExecute.h
@@ -1,9 +1,8 @@
 //===- MustExecute.h - Is an instruction known to execute--------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
@@ -151,9 +150,9 @@ public:
       const;
 
   /// Inform the safety info that we are planning to insert a new instruction
-  /// into the basic block \p BB. It will make all cache updates to keep it
-  /// correct after this insertion.
-  void insertInstructionTo(const BasicBlock *BB);
+  /// \p Inst into the basic block \p BB. It will make all cache updates to keep
+  /// it correct after this insertion.
+  void insertInstructionTo(const Instruction *Inst, const BasicBlock *BB);
 
   /// Inform safety info that we are planning to remove the instruction \p Inst
   /// from its block. It will make all cache updates to keep it correct after
diff --git a/include/llvm/Analysis/ObjCARCAliasAnalysis.h b/include/llvm/Analysis/ObjCARCAliasAnalysis.h
index 559c77c308119bb06b363305d455618ee63d1c3f..ed15472a0fdcd431e27fadea58755f0c37194727 100644
--- a/include/llvm/Analysis/ObjCARCAliasAnalysis.h
+++ b/include/llvm/Analysis/ObjCARCAliasAnalysis.h
@@ -1,9 +1,8 @@
 //===- ObjCARCAliasAnalysis.h - ObjC ARC Alias Analysis ---------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
@@ -60,7 +59,7 @@ public:
   FunctionModRefBehavior getModRefBehavior(const Function *F);
 
   using AAResultBase::getModRefInfo;
-  ModRefInfo getModRefInfo(ImmutableCallSite CS, const MemoryLocation &Loc);
+  ModRefInfo getModRefInfo(const CallBase *Call, const MemoryLocation &Loc);
 };
 
 /// Analysis pass providing a never-invalidated alias analysis result.
diff --git a/include/llvm/Analysis/ObjCARCAnalysisUtils.h b/include/llvm/Analysis/ObjCARCAnalysisUtils.h
index 1f497fab35da8b8fe4bb9ded547d2dbcc93bc7f1..522abd756c9f641004ccf3c39b3d1b152130f69f 100644
--- a/include/llvm/Analysis/ObjCARCAnalysisUtils.h
+++ b/include/llvm/Analysis/ObjCARCAnalysisUtils.h
@@ -1,9 +1,8 @@
 //===- ObjCARCAnalysisUtils.h - ObjC ARC Analysis Utilities -----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/include/llvm/Analysis/ObjCARCInstKind.h b/include/llvm/Analysis/ObjCARCInstKind.h
index 018ea1f851bebaf35c5ef36f6d61e00ce1a225aa..9890d5f7dfdd52901dec2c64cbaa722ff9435243 100644
--- a/include/llvm/Analysis/ObjCARCInstKind.h
+++ b/include/llvm/Analysis/ObjCARCInstKind.h
@@ -1,9 +1,8 @@
 //===- ObjCARCInstKind.h - ARC instruction equivalence classes --*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/Analysis/OptimizationRemarkEmitter.h b/include/llvm/Analysis/OptimizationRemarkEmitter.h
index fa838696e2f876af21d4372071e60e2a51331813..a2b29555ce5ad4585e74c6a36872197b1126984c 100644
--- a/include/llvm/Analysis/OptimizationRemarkEmitter.h
+++ b/include/llvm/Analysis/OptimizationRemarkEmitter.h
@@ -1,9 +1,8 @@
 //===- OptimizationRemarkEmitter.h - Optimization Diagnostic ----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Analysis/OrderedBasicBlock.h b/include/llvm/Analysis/OrderedBasicBlock.h
index 0776aa626005f0739b4e6650ffb0e12254dde4c0..6823f68641989a9cdf6fdacf28f49e6ba4418739 100644
--- a/include/llvm/Analysis/OrderedBasicBlock.h
+++ b/include/llvm/Analysis/OrderedBasicBlock.h
@@ -1,9 +1,8 @@
 //===- llvm/Analysis/OrderedBasicBlock.h --------------------- -*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Analysis/OrderedInstructions.h b/include/llvm/Analysis/OrderedInstructions.h
index 7e3850b87c57541a71fffd2f270d3227d15a4d44..967b146b52debd6744531b3f0d615ca9220aa04c 100644
--- a/include/llvm/Analysis/OrderedInstructions.h
+++ b/include/llvm/Analysis/OrderedInstructions.h
@@ -1,9 +1,8 @@
 //===- llvm/Transforms/Utils/OrderedInstructions.h -------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Analysis/PHITransAddr.h b/include/llvm/Analysis/PHITransAddr.h
index 0a335b6be6c732cfb024e3a82e4390df142f705a..54a07f05347877950a2aa09ef185e551f485a5d6 100644
--- a/include/llvm/Analysis/PHITransAddr.h
+++ b/include/llvm/Analysis/PHITransAddr.h
@@ -1,9 +1,8 @@
 //===- PHITransAddr.h - PHI Translation for Addresses -----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Analysis/Passes.h b/include/llvm/Analysis/Passes.h
index 081dd5000835bd712420ef9d95f5725f35fbaa58..d9c97dff8c6efa79e352cac32e7b8380921bafec 100644
--- a/include/llvm/Analysis/Passes.h
+++ b/include/llvm/Analysis/Passes.h
@@ -1,9 +1,8 @@
 //===-- llvm/Analysis/Passes.h - Constructors for analyses ------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Analysis/PhiValues.h b/include/llvm/Analysis/PhiValues.h
index 76204ac1bc6c1ff45e8d2c3b3681d802eec38f3d..124fa219169486e746a29aa52313e63cea3a5461 100644
--- a/include/llvm/Analysis/PhiValues.h
+++ b/include/llvm/Analysis/PhiValues.h
@@ -1,9 +1,8 @@
 //===- PhiValues.h - Phi Value Analysis -------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Analysis/PostDominators.h b/include/llvm/Analysis/PostDominators.h
index f2dc8d135d71cb4a162c26f41682f874f4dca193..87d2e0318d0a0d4c48bfef12c75e667595beb334 100644
--- a/include/llvm/Analysis/PostDominators.h
+++ b/include/llvm/Analysis/PostDominators.h
@@ -1,9 +1,8 @@
 //=- llvm/Analysis/PostDominators.h - Post Dominator Calculation --*- C++ -*-=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Analysis/ProfileSummaryInfo.h b/include/llvm/Analysis/ProfileSummaryInfo.h
index 3aef4be72d71f49e0d7d0b07abcd874dcfed2132..636b5d6876f9c74d961fc6de95b318e5e93b523a 100644
--- a/include/llvm/Analysis/ProfileSummaryInfo.h
+++ b/include/llvm/Analysis/ProfileSummaryInfo.h
@@ -1,9 +1,8 @@
 //===- llvm/Analysis/ProfileSummaryInfo.h - profile summary ---*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Analysis/PtrUseVisitor.h b/include/llvm/Analysis/PtrUseVisitor.h
index b34b25c75040ad4a3f745a980c9348714ae829ed..dca8718b86d06aa73d3bd2ccc10ca67ce3a24be5 100644
--- a/include/llvm/Analysis/PtrUseVisitor.h
+++ b/include/llvm/Analysis/PtrUseVisitor.h
@@ -1,9 +1,8 @@
 //===- PtrUseVisitor.h - InstVisitors over a pointers uses ------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Analysis/RegionInfo.h b/include/llvm/Analysis/RegionInfo.h
index 27f6cc197927ce5fc5fd15025c481d94acd1fd93..8bcc3e8512003467beed17e9307520e91bdb929f 100644
--- a/include/llvm/Analysis/RegionInfo.h
+++ b/include/llvm/Analysis/RegionInfo.h
@@ -1,9 +1,8 @@
 //===- RegionInfo.h - SESE region analysis ----------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Analysis/RegionInfoImpl.h b/include/llvm/Analysis/RegionInfoImpl.h
index 5904214aa925432699962d5c6665d7938346520b..c59c09dd2095dd0e852ec31dc49204dc080d27d7 100644
--- a/include/llvm/Analysis/RegionInfoImpl.h
+++ b/include/llvm/Analysis/RegionInfoImpl.h
@@ -1,9 +1,8 @@
 //===- RegionInfoImpl.h - SESE region detection analysis --------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 // Detects single entry single exit regions in the control flow graph.
diff --git a/include/llvm/Analysis/RegionIterator.h b/include/llvm/Analysis/RegionIterator.h
index 4fd92fcde20b86b81f7d40af7749a87dc45d2284..72bc5bbcb506e7dc7af0be5433151fdd1d540bee 100644
--- a/include/llvm/Analysis/RegionIterator.h
+++ b/include/llvm/Analysis/RegionIterator.h
@@ -1,9 +1,8 @@
 //===- RegionIterator.h - Iterators to iteratate over Regions ---*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 // This file defines the iterators to iterate over the elements of a Region.
diff --git a/include/llvm/Analysis/RegionPass.h b/include/llvm/Analysis/RegionPass.h
index b3da91c89cbd63dd8dfbe5df1bc4b7d215136ea3..5b1864a37629783bdcba8afeb76f17b0c2243c98 100644
--- a/include/llvm/Analysis/RegionPass.h
+++ b/include/llvm/Analysis/RegionPass.h
@@ -1,9 +1,8 @@
 //===- RegionPass.h - RegionPass class --------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Analysis/RegionPrinter.h b/include/llvm/Analysis/RegionPrinter.h
index e132eaea567460a40ef6135c7fa4b1c4f60f959b..154ac35c486adb10c04fdbceea72a2d7566c7370 100644
--- a/include/llvm/Analysis/RegionPrinter.h
+++ b/include/llvm/Analysis/RegionPrinter.h
@@ -1,9 +1,8 @@
 //===-- RegionPrinter.h - Region printer external interface -----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Analysis/ScalarEvolution.h b/include/llvm/Analysis/ScalarEvolution.h
index 8f4200b07e5c7bd3a09fa7ccfa5c7c62ed5febef..f3a035117b3a907d1854545a7b4c7e4a6ea38df0 100644
--- a/include/llvm/Analysis/ScalarEvolution.h
+++ b/include/llvm/Analysis/ScalarEvolution.h
@@ -1,9 +1,8 @@
 //===- llvm/Analysis/ScalarEvolution.h - Scalar Evolution -------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -85,6 +84,9 @@ class SCEV : public FoldingSetNode {
   const unsigned short SCEVType;
 
 protected:
+  // Estimated complexity of this node's expression tree size.
+  const unsigned short ExpressionSize;
+
   /// This field is initialized to zero and may be used in subclasses to store
   /// miscellaneous information.
   unsigned short SubclassData = 0;
@@ -116,8 +118,9 @@ public:
     NoWrapMask = (1 << 3) - 1
   };
 
-  explicit SCEV(const FoldingSetNodeIDRef ID, unsigned SCEVTy)
-      : FastID(ID), SCEVType(SCEVTy) {}
+  explicit SCEV(const FoldingSetNodeIDRef ID, unsigned SCEVTy,
+                unsigned short ExpressionSize)
+      : FastID(ID), SCEVType(SCEVTy), ExpressionSize(ExpressionSize) {}
   SCEV(const SCEV &) = delete;
   SCEV &operator=(const SCEV &) = delete;
 
@@ -138,6 +141,19 @@ public:
   /// Return true if the specified scev is negated, but not a constant.
   bool isNonConstantNegative() const;
 
+  // Returns estimated size of the mathematical expression represented by this
+  // SCEV. The rules of its calculation are following:
+  // 1) Size of a SCEV without operands (like constants and SCEVUnknown) is 1;
+  // 2) Size SCEV with operands Op1, Op2, ..., OpN is calculated by formula:
+  //    (1 + Size(Op1) + ... + Size(OpN)).
+  // This value gives us an estimation of time we need to traverse through this
+  // SCEV and all its operands recursively. We may use it to avoid performing
+  // heavy transformations on SCEVs of excessive size for sake of saving the
+  // compilation time.
+  unsigned short getExpressionSize() const {
+    return ExpressionSize;
+  }
+
   /// Print out the internal representation of this scalar to the specified
   /// stream.  This should really only be used for debugging purposes.
   void print(raw_ostream &OS) const;
@@ -1273,7 +1289,7 @@ private:
     using EdgeExitInfo = std::pair<BasicBlock *, ExitLimit>;
 
     /// Initialize BackedgeTakenInfo from a list of exact exit counts.
-    BackedgeTakenInfo(SmallVectorImpl<EdgeExitInfo> &&ExitCounts, bool Complete,
+    BackedgeTakenInfo(ArrayRef<EdgeExitInfo> ExitCounts, bool Complete,
                       const SCEV *MaxCount, bool MaxOrZero);
 
     /// Test whether this BackedgeTakenInfo contains any computed information,
@@ -1826,15 +1842,15 @@ private:
                           bool NoWrap);
 
   /// Get add expr already created or create a new one.
-  const SCEV *getOrCreateAddExpr(SmallVectorImpl<const SCEV *> &Ops,
+  const SCEV *getOrCreateAddExpr(ArrayRef<const SCEV *> Ops,
                                  SCEV::NoWrapFlags Flags);
 
   /// Get mul expr already created or create a new one.
-  const SCEV *getOrCreateMulExpr(SmallVectorImpl<const SCEV *> &Ops,
+  const SCEV *getOrCreateMulExpr(ArrayRef<const SCEV *> Ops,
                                  SCEV::NoWrapFlags Flags);
 
   // Get addrec expr already created or create a new one.
-  const SCEV *getOrCreateAddRecExpr(SmallVectorImpl<const SCEV *> &Ops,
+  const SCEV *getOrCreateAddRecExpr(ArrayRef<const SCEV *> Ops,
                                     const Loop *L, SCEV::NoWrapFlags Flags);
 
   /// Return x if \p Val is f(x) where f is a 1-1 function.
diff --git a/include/llvm/Analysis/ScalarEvolutionAliasAnalysis.h b/include/llvm/Analysis/ScalarEvolutionAliasAnalysis.h
index 329be51e5eac6cf15a9b64bc4b294ee8c5817a38..d417850c0ee3fb5c623f4c0719e75d9eb86480e4 100644
--- a/include/llvm/Analysis/ScalarEvolutionAliasAnalysis.h
+++ b/include/llvm/Analysis/ScalarEvolutionAliasAnalysis.h
@@ -1,9 +1,8 @@
 //===- ScalarEvolutionAliasAnalysis.h - SCEV-based AA -----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/include/llvm/Analysis/ScalarEvolutionExpander.h b/include/llvm/Analysis/ScalarEvolutionExpander.h
index 58d42680d6bc96d3677963be11e7348ab7368e2c..a2c1258d172b5fa6963741fc0e649842eb646d9a 100644
--- a/include/llvm/Analysis/ScalarEvolutionExpander.h
+++ b/include/llvm/Analysis/ScalarEvolutionExpander.h
@@ -1,9 +1,8 @@
 //===---- llvm/Analysis/ScalarEvolutionExpander.h - SCEV Exprs --*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Analysis/ScalarEvolutionExpressions.h b/include/llvm/Analysis/ScalarEvolutionExpressions.h
index 42e76094eb2b4c4e1147fac3128c6d42cf88c30d..e187a9621a14232c156a4f575a95d91e8481d281 100644
--- a/include/llvm/Analysis/ScalarEvolutionExpressions.h
+++ b/include/llvm/Analysis/ScalarEvolutionExpressions.h
@@ -1,9 +1,8 @@
 //===- llvm/Analysis/ScalarEvolutionExpressions.h - SCEV Exprs --*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -51,7 +50,7 @@ class Type;
     ConstantInt *V;
 
     SCEVConstant(const FoldingSetNodeIDRef ID, ConstantInt *v) :
-      SCEV(ID, scConstant), V(v) {}
+      SCEV(ID, scConstant, 1), V(v) {}
 
   public:
     ConstantInt *getValue() const { return V; }
@@ -65,6 +64,13 @@ class Type;
     }
   };
 
+  static unsigned short computeExpressionSize(ArrayRef<const SCEV *> Args) {
+    APInt Size(16, 1);
+    for (auto *Arg : Args)
+      Size = Size.uadd_sat(APInt(16, Arg->getExpressionSize()));
+    return (unsigned short)Size.getZExtValue();
+  }
+
   /// This is the base class for unary cast operator classes.
   class SCEVCastExpr : public SCEV {
   protected:
@@ -142,9 +148,10 @@ class Type;
     const SCEV *const *Operands;
     size_t NumOperands;
 
-    SCEVNAryExpr(const FoldingSetNodeIDRef ID,
-                 enum SCEVTypes T, const SCEV *const *O, size_t N)
-      : SCEV(ID, T), Operands(O), NumOperands(N) {}
+    SCEVNAryExpr(const FoldingSetNodeIDRef ID, enum SCEVTypes T,
+                 const SCEV *const *O, size_t N)
+        : SCEV(ID, T, computeExpressionSize(makeArrayRef(O, N))), Operands(O),
+          NumOperands(N) {}
 
   public:
     size_t getNumOperands() const { return NumOperands; }
@@ -258,7 +265,8 @@ class Type;
     const SCEV *RHS;
 
     SCEVUDivExpr(const FoldingSetNodeIDRef ID, const SCEV *lhs, const SCEV *rhs)
-      : SCEV(ID, scUDivExpr), LHS(lhs), RHS(rhs) {}
+        : SCEV(ID, scUDivExpr, computeExpressionSize({lhs, rhs})), LHS(lhs),
+          RHS(rhs) {}
 
   public:
     const SCEV *getLHS() const { return LHS; }
@@ -411,7 +419,7 @@ class Type;
 
     SCEVUnknown(const FoldingSetNodeIDRef ID, Value *V,
                 ScalarEvolution *se, SCEVUnknown *next) :
-      SCEV(ID, scUnknown), CallbackVH(V), SE(se), Next(next) {}
+      SCEV(ID, scUnknown, 1), CallbackVH(V), SE(se), Next(next) {}
 
     // Implement CallbackVH.
     void deleted() override;
diff --git a/include/llvm/Analysis/ScalarEvolutionNormalization.h b/include/llvm/Analysis/ScalarEvolutionNormalization.h
index 51c92121c8f0ea7e98494266379102b8c7d31978..1a05594a46ec6a82adaf99fcf8efcc1a727684d9 100644
--- a/include/llvm/Analysis/ScalarEvolutionNormalization.h
+++ b/include/llvm/Analysis/ScalarEvolutionNormalization.h
@@ -1,9 +1,8 @@
 //===- llvm/Analysis/ScalarEvolutionNormalization.h - See below -*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Analysis/ScopedNoAliasAA.h b/include/llvm/Analysis/ScopedNoAliasAA.h
index 508968e16e5d4b958d4b89f898d57f097bdc9d80..94c6021806b7eaa3d2828e673448bf51ed517e0a 100644
--- a/include/llvm/Analysis/ScopedNoAliasAA.h
+++ b/include/llvm/Analysis/ScopedNoAliasAA.h
@@ -1,9 +1,8 @@
 //===- ScopedNoAliasAA.h - Scoped No-Alias Alias Analysis -------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -16,7 +15,7 @@
 #define LLVM_ANALYSIS_SCOPEDNOALIASAA_H
 
 #include "llvm/Analysis/AliasAnalysis.h"
-#include "llvm/IR/CallSite.h"
+#include "llvm/IR/InstrTypes.h"
 #include "llvm/IR/PassManager.h"
 #include "llvm/Pass.h"
 #include <memory>
@@ -41,8 +40,8 @@ public:
   }
 
   AliasResult alias(const MemoryLocation &LocA, const MemoryLocation &LocB);
-  ModRefInfo getModRefInfo(ImmutableCallSite CS, const MemoryLocation &Loc);
-  ModRefInfo getModRefInfo(ImmutableCallSite CS1, ImmutableCallSite CS2);
+  ModRefInfo getModRefInfo(const CallBase *Call, const MemoryLocation &Loc);
+  ModRefInfo getModRefInfo(const CallBase *Call1, const CallBase *Call2);
 
 private:
   bool mayAliasInScopes(const MDNode *Scopes, const MDNode *NoAlias) const;
diff --git a/include/llvm/Analysis/SparsePropagation.h b/include/llvm/Analysis/SparsePropagation.h
index 02a2e64268b7f91d1130e7cb278978414557d46c..fac92e4a25a4104244c24ec69ceceb585fdd240a 100644
--- a/include/llvm/Analysis/SparsePropagation.h
+++ b/include/llvm/Analysis/SparsePropagation.h
@@ -1,9 +1,8 @@
 //===- SparsePropagation.h - Sparse Conditional Property Propagation ------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -330,12 +329,8 @@ void SparseSolver<LatticeKey, LatticeVal, KeyInfo>::getFeasibleSuccessors(
     return;
   }
 
-  if (TI.isExceptionalTerminator()) {
-    Succs.assign(Succs.size(), true);
-    return;
-  }
-
-  if (isa<IndirectBrInst>(TI)) {
+  if (TI.isExceptionalTerminator() ||
+      TI.isIndirectTerminator()) {
     Succs.assign(Succs.size(), true);
     return;
   }
diff --git a/include/llvm/Analysis/StackSafetyAnalysis.h b/include/llvm/Analysis/StackSafetyAnalysis.h
index 8a151650a34c07e6922466aeab7154ac9054bed1..f9d8b08ac1425c38f11fd2a2cc647f591fae1fc5 100644
--- a/include/llvm/Analysis/StackSafetyAnalysis.h
+++ b/include/llvm/Analysis/StackSafetyAnalysis.h
@@ -1,9 +1,8 @@
 //===- StackSafetyAnalysis.h - Stack memory safety analysis -----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Analysis/SyncDependenceAnalysis.h b/include/llvm/Analysis/SyncDependenceAnalysis.h
index df693d9d8e8c2b2cb1eb78c50b604819df997a2c..099403b477577e7923129f01833eb3f89c2c27c0 100644
--- a/include/llvm/Analysis/SyncDependenceAnalysis.h
+++ b/include/llvm/Analysis/SyncDependenceAnalysis.h
@@ -1,9 +1,8 @@
 //===- SyncDependenceAnalysis.h - Divergent Branch Dependence -*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Analysis/SyntheticCountsUtils.h b/include/llvm/Analysis/SyntheticCountsUtils.h
index 87f4a0100b38901f51f9ee36cbbfe93a7d42dc30..b9b4c98bfc356932d618e3d9319ce2d1330892a0 100644
--- a/include/llvm/Analysis/SyntheticCountsUtils.h
+++ b/include/llvm/Analysis/SyntheticCountsUtils.h
@@ -1,9 +1,8 @@
 //===- SyntheticCountsUtils.h - utilities for count propagation--*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -36,16 +35,17 @@ public:
   using EdgeRef = typename CGT::EdgeRef;
   using SccTy = std::vector<NodeRef>;
 
-  using GetRelBBFreqTy = function_ref<Optional<Scaled64>(EdgeRef)>;
-  using GetCountTy = function_ref<uint64_t(NodeRef)>;
-  using AddCountTy = function_ref<void(NodeRef, uint64_t)>;
+  // Not all EdgeRef have information about the source of the edge. Hence
+  // NodeRef corresponding to the source of the EdgeRef is explicitly passed.
+  using GetProfCountTy = function_ref<Optional<Scaled64>(NodeRef, EdgeRef)>;
+  using AddCountTy = function_ref<void(NodeRef, Scaled64)>;
 
-  static void propagate(const CallGraphType &CG, GetRelBBFreqTy GetRelBBFreq,
-                        GetCountTy GetCount, AddCountTy AddCount);
+  static void propagate(const CallGraphType &CG, GetProfCountTy GetProfCount,
+                        AddCountTy AddCount);
 
 private:
-  static void propagateFromSCC(const SccTy &SCC, GetRelBBFreqTy GetRelBBFreq,
-                               GetCountTy GetCount, AddCountTy AddCount);
+  static void propagateFromSCC(const SccTy &SCC, GetProfCountTy GetProfCount,
+                               AddCountTy AddCount);
 };
 } // namespace llvm
 
diff --git a/include/llvm/Analysis/TargetFolder.h b/include/llvm/Analysis/TargetFolder.h
index ae75d3773362253c708ba4e57ca631372bfd8c00..b8047a46b93d98746f3319802410bd58b35d8532 100644
--- a/include/llvm/Analysis/TargetFolder.h
+++ b/include/llvm/Analysis/TargetFolder.h
@@ -1,9 +1,8 @@
 //====- TargetFolder.h - Constant folding helper ---------------*- C++ -*-====//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Analysis/TargetLibraryInfo.def b/include/llvm/Analysis/TargetLibraryInfo.def
index 518a85ee1a016949d06ed95eea057a14fd5c0f66..f73c69e54e7ca89b321579e9f91e5331ff718320 100644
--- a/include/llvm/Analysis/TargetLibraryInfo.def
+++ b/include/llvm/Analysis/TargetLibraryInfo.def
@@ -1,9 +1,8 @@
 //===-- TargetLibraryInfo.def - Library information -------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/Analysis/TargetLibraryInfo.h b/include/llvm/Analysis/TargetLibraryInfo.h
index a3fe834022f70877165666fe6b3cef6855994cfa..fc6811e88623b7f5b0543314ebf88f85c0812ec9 100644
--- a/include/llvm/Analysis/TargetLibraryInfo.h
+++ b/include/llvm/Analysis/TargetLibraryInfo.h
@@ -1,9 +1,8 @@
 //===-- TargetLibraryInfo.h - Library information ---------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/Analysis/TargetTransformInfo.h b/include/llvm/Analysis/TargetTransformInfo.h
index 6ddea8bbdce09ffe142ab7cecea912aa3d499b90..60dbf6775a644a3f215be20ec126f72fe2526dfd 100644
--- a/include/llvm/Analysis/TargetTransformInfo.h
+++ b/include/llvm/Analysis/TargetTransformInfo.h
@@ -1,9 +1,8 @@
 //===- TargetTransformInfo.h ------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
@@ -487,6 +486,10 @@ public:
   /// addressing mode expressions.
   bool shouldFavorPostInc() const;
 
+  /// Return true if LSR should make efforts to generate indexed addressing
+  /// modes that operate across loop iterations.
+  bool shouldFavorBackedgeIndex(const Loop *L) const;
+
   /// Return true if the target supports masked load/store
   /// AVX2 and AVX-512 targets allow masks for consecutive load and store
   bool isLegalMaskedStore(Type *DataType) const;
@@ -581,13 +584,17 @@ public:
   struct MemCmpExpansionOptions {
     // The list of available load sizes (in bytes), sorted in decreasing order.
     SmallVector<unsigned, 8> LoadSizes;
+    // Set to true to allow overlapping loads. For example, 7-byte compares can
+    // be done with two 4-byte compares instead of 4+2+1-byte compares. This
+    // requires all loads in LoadSizes to be doable in an unaligned way.
+    bool AllowOverlappingLoads = false;
   };
   const MemCmpExpansionOptions *enableMemCmpExpansion(bool IsZeroCmp) const;
 
   /// Enable matching of interleaved access groups.
   bool enableInterleavedAccessVectorization() const;
 
-  /// Enable matching of interleaved access groups that contain predicated 
+  /// Enable matching of interleaved access groups that contain predicated
   /// accesses or gaps and therefore vectorized using masked
   /// vector loads/stores.
   bool enableMaskedInterleavedAccessVectorization() const;
@@ -772,7 +779,7 @@ public:
   /// \return The cost of a shuffle instruction of kind Kind and of type Tp.
   /// The index and subtype parameters are used by the subvector insertion and
   /// extraction shuffle kinds to show the insert/extract point and the type of
-  /// the subvector being inserted/extracted. 
+  /// the subvector being inserted/extracted.
   /// NOTE: For subvector extractions Tp represents the source type.
   int getShuffleCost(ShuffleKind Kind, Type *Tp, int Index = 0,
                      Type *SubTp = nullptr) const;
@@ -930,6 +937,14 @@ public:
   bool areInlineCompatible(const Function *Caller,
                            const Function *Callee) const;
 
+  /// \returns True if the caller and callee agree on how \p Args will be passed
+  /// to the callee.
+  /// \param[out] Args The list of compatible arguments.  The implementation may
+  /// filter out any incompatible args from this list.
+  bool areFunctionArgsABICompatible(const Function *Caller,
+                                    const Function *Callee,
+                                    SmallPtrSetImpl<Argument *> &Args) const;
+
   /// The type of load/store indexing.
   enum MemIndexedMode {
     MIM_Unindexed,  ///< No indexing.
@@ -1054,6 +1069,7 @@ public:
                              TargetTransformInfo::LSRCost &C2) = 0;
   virtual bool canMacroFuseCmp() = 0;
   virtual bool shouldFavorPostInc() const = 0;
+  virtual bool shouldFavorBackedgeIndex(const Loop *L) const = 0;
   virtual bool isLegalMaskedStore(Type *DataType) = 0;
   virtual bool isLegalMaskedLoad(Type *DataType) = 0;
   virtual bool isLegalMaskedScatter(Type *DataType) = 0;
@@ -1175,6 +1191,9 @@ public:
       unsigned RemainingBytes, unsigned SrcAlign, unsigned DestAlign) const = 0;
   virtual bool areInlineCompatible(const Function *Caller,
                                    const Function *Callee) const = 0;
+  virtual bool
+  areFunctionArgsABICompatible(const Function *Caller, const Function *Callee,
+                               SmallPtrSetImpl<Argument *> &Args) const = 0;
   virtual bool isIndexedLoadLegal(MemIndexedMode Mode, Type *Ty) const = 0;
   virtual bool isIndexedStoreLegal(MemIndexedMode Mode,Type *Ty) const = 0;
   virtual unsigned getLoadStoreVecRegBitWidth(unsigned AddrSpace) const = 0;
@@ -1287,6 +1306,9 @@ public:
   bool shouldFavorPostInc() const override {
     return Impl.shouldFavorPostInc();
   }
+  bool shouldFavorBackedgeIndex(const Loop *L) const override {
+    return Impl.shouldFavorBackedgeIndex(L);
+  }
   bool isLegalMaskedStore(Type *DataType) override {
     return Impl.isLegalMaskedStore(DataType);
   }
@@ -1553,6 +1575,11 @@ public:
                            const Function *Callee) const override {
     return Impl.areInlineCompatible(Caller, Callee);
   }
+  bool areFunctionArgsABICompatible(
+      const Function *Caller, const Function *Callee,
+      SmallPtrSetImpl<Argument *> &Args) const override {
+    return Impl.areFunctionArgsABICompatible(Caller, Callee, Args);
+  }
   bool isIndexedLoadLegal(MemIndexedMode Mode, Type *Ty) const override {
     return Impl.isIndexedLoadLegal(Mode, Ty, getDataLayout());
   }
diff --git a/include/llvm/Analysis/TargetTransformInfoImpl.h b/include/llvm/Analysis/TargetTransformInfoImpl.h
index 5e79c5cdfe0353ad4e6c89f5e32db0fcb8a8b433..4705933750d16281490491eafc90719367e65229 100644
--- a/include/llvm/Analysis/TargetTransformInfoImpl.h
+++ b/include/llvm/Analysis/TargetTransformInfoImpl.h
@@ -1,9 +1,8 @@
 //===- TargetTransformInfoImpl.h --------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
@@ -254,6 +253,8 @@ public:
 
   bool shouldFavorPostInc() const { return false; }
 
+  bool shouldFavorBackedgeIndex(const Loop *L) const { return false; }
+
   bool isLegalMaskedStore(Type *DataType) { return false; }
 
   bool isLegalMaskedLoad(Type *DataType) { return false; }
@@ -526,6 +527,14 @@ public:
             Callee->getFnAttribute("target-features"));
   }
 
+  bool areFunctionArgsABICompatible(const Function *Caller, const Function *Callee,
+                                    SmallPtrSetImpl<Argument *> &Args) const {
+    return (Caller->getFnAttribute("target-cpu") ==
+            Callee->getFnAttribute("target-cpu")) &&
+           (Caller->getFnAttribute("target-features") ==
+            Callee->getFnAttribute("target-features"));
+  }
+
   bool isIndexedLoadLegal(TTI::MemIndexedMode Mode, Type *Ty,
                           const DataLayout &DL) const {
     return false;
diff --git a/include/llvm/Analysis/Trace.h b/include/llvm/Analysis/Trace.h
index b05d384ab1a39c8ca80f44b82c905fb77f1621ca..a1ffd03c4053030a23cb484fe1fbc8e505d27b4d 100644
--- a/include/llvm/Analysis/Trace.h
+++ b/include/llvm/Analysis/Trace.h
@@ -1,9 +1,8 @@
 //===- llvm/Analysis/Trace.h - Represent one trace of LLVM code -*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Analysis/TypeBasedAliasAnalysis.h b/include/llvm/Analysis/TypeBasedAliasAnalysis.h
index 7fcfdb3a817c8f705bc57496056a12e329076919..123500544a52176e925b3fe35ae6e97efbfd0c46 100644
--- a/include/llvm/Analysis/TypeBasedAliasAnalysis.h
+++ b/include/llvm/Analysis/TypeBasedAliasAnalysis.h
@@ -1,9 +1,8 @@
 //===- TypeBasedAliasAnalysis.h - Type-Based Alias Analysis -----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -17,7 +16,7 @@
 #define LLVM_ANALYSIS_TYPEBASEDALIASANALYSIS_H
 
 #include "llvm/Analysis/AliasAnalysis.h"
-#include "llvm/IR/CallSite.h"
+#include "llvm/IR/InstrTypes.h"
 #include "llvm/IR/PassManager.h"
 #include "llvm/Pass.h"
 #include <memory>
@@ -43,10 +42,10 @@ public:
 
   AliasResult alias(const MemoryLocation &LocA, const MemoryLocation &LocB);
   bool pointsToConstantMemory(const MemoryLocation &Loc, bool OrLocal);
-  FunctionModRefBehavior getModRefBehavior(ImmutableCallSite CS);
+  FunctionModRefBehavior getModRefBehavior(const CallBase *Call);
   FunctionModRefBehavior getModRefBehavior(const Function *F);
-  ModRefInfo getModRefInfo(ImmutableCallSite CS, const MemoryLocation &Loc);
-  ModRefInfo getModRefInfo(ImmutableCallSite CS1, ImmutableCallSite CS2);
+  ModRefInfo getModRefInfo(const CallBase *Call, const MemoryLocation &Loc);
+  ModRefInfo getModRefInfo(const CallBase *Call1, const CallBase *Call2);
 
 private:
   bool Aliases(const MDNode *A, const MDNode *B) const;
diff --git a/include/llvm/Analysis/TypeMetadataUtils.h b/include/llvm/Analysis/TypeMetadataUtils.h
index 3bf9c5d20741542603a8347a0c4ebc7ad8b49b8a..82cf8efeea54b663e77eb87d3c74f2f673be5429 100644
--- a/include/llvm/Analysis/TypeMetadataUtils.h
+++ b/include/llvm/Analysis/TypeMetadataUtils.h
@@ -1,9 +1,8 @@
 //===- TypeMetadataUtils.h - Utilities related to type metadata --*- C++ -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Analysis/Utils/Local.h b/include/llvm/Analysis/Utils/Local.h
index b4141bbff28d66b0179178d0dc19cece9fdd33e5..acbdf5dca32c79eed2ba4364247f0b8fdc8145a2 100644
--- a/include/llvm/Analysis/Utils/Local.h
+++ b/include/llvm/Analysis/Utils/Local.h
@@ -1,9 +1,8 @@
 //===- Local.h - Functions to perform local transformations -----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Analysis/ValueLattice.h b/include/llvm/Analysis/ValueLattice.h
index 0744ca617e48e433f253279a04f8f712ea7cfe6d..56519d7d0857cfea4f67d495fe5e6ba3e1f7d04d 100644
--- a/include/llvm/Analysis/ValueLattice.h
+++ b/include/llvm/Analysis/ValueLattice.h
@@ -1,9 +1,8 @@
 //===- ValueLattice.h - Value constraint analysis ---------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/Analysis/ValueLatticeUtils.h b/include/llvm/Analysis/ValueLatticeUtils.h
index 02072672e56ed174b3e81376ed211f111a9bfebc..a3bbb96129bffc5eea0778b54f29a709e3f1344e 100644
--- a/include/llvm/Analysis/ValueLatticeUtils.h
+++ b/include/llvm/Analysis/ValueLatticeUtils.h
@@ -1,9 +1,8 @@
 //===-- ValueLatticeUtils.h - Utils for solving lattices --------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Analysis/ValueTracking.h b/include/llvm/Analysis/ValueTracking.h
index 7c2e9a65ac27dcdf653e2818b6455186604c76b6..b3c07b1e246a8657a8ec6c87969a124c3994ef50 100644
--- a/include/llvm/Analysis/ValueTracking.h
+++ b/include/llvm/Analysis/ValueTracking.h
@@ -1,9 +1,8 @@
 //===- llvm/Analysis/ValueTracking.h - Walk computations --------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -297,10 +296,10 @@ class Value;
 
   /// This function returns call pointer argument that is considered the same by
   /// aliasing rules. You CAN'T use it to replace one value with another.
-  const Value *getArgumentAliasingToReturnedPointer(ImmutableCallSite CS);
-  inline Value *getArgumentAliasingToReturnedPointer(CallSite CS) {
-    return const_cast<Value *>(
-        getArgumentAliasingToReturnedPointer(ImmutableCallSite(CS)));
+  const Value *getArgumentAliasingToReturnedPointer(const CallBase *Call);
+  inline Value *getArgumentAliasingToReturnedPointer(CallBase *Call) {
+    return const_cast<Value *>(getArgumentAliasingToReturnedPointer(
+        const_cast<const CallBase *>(Call)));
   }
 
   // {launder,strip}.invariant.group returns pointer that aliases its argument,
@@ -309,7 +308,7 @@ class Value;
   // considered as capture. The arguments are not marked as returned neither,
   // because it would make it useless.
   bool isIntrinsicReturningPointerAliasingArgumentWithoutCapturing(
-      ImmutableCallSite CS);
+      const CallBase *Call);
 
   /// This method strips off any GEP address adjustments and pointer casts from
   /// the specified value, returning the original object being addressed. Note
diff --git a/include/llvm/Analysis/VectorUtils.h b/include/llvm/Analysis/VectorUtils.h
index 5d44a52d4c53c64e1ab57bb09b10b2b8200b9cdc..60ef6339de1b22075aee9e684a90774a1c50f760 100644
--- a/include/llvm/Analysis/VectorUtils.h
+++ b/include/llvm/Analysis/VectorUtils.h
@@ -1,9 +1,8 @@
 //===- llvm/Analysis/VectorUtils.h - Vector utilities -----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -117,8 +116,24 @@ computeMinimumValueSizes(ArrayRef<BasicBlock*> Blocks,
                          DemandedBits &DB,
                          const TargetTransformInfo *TTI=nullptr);
 
+/// Compute the union of two access-group lists.
+///
+/// If the list contains just one access group, it is returned directly. If the
+/// list is empty, returns nullptr.
+MDNode *uniteAccessGroups(MDNode *AccGroups1, MDNode *AccGroups2);
+
+/// Compute the access-group list of access groups that @p Inst1 and @p Inst2
+/// are both in. If either instruction does not access memory at all, it is
+/// considered to be in every list.
+///
+/// If the list contains just one access group, it is returned directly. If the
+/// list is empty, returns nullptr.
+MDNode *intersectAccessGroups(const Instruction *Inst1,
+                              const Instruction *Inst2);
+
 /// Specifically, let Kinds = [MD_tbaa, MD_alias_scope, MD_noalias, MD_fpmath,
-/// MD_nontemporal].  For K in Kinds, we get the MDNode for K from each of the
+/// MD_nontemporal, MD_access_group].
+/// For K in Kinds, we get the MDNode for K from each of the
 /// elements of VL, compute their "intersection" (i.e., the most generic
 /// metadata value that covers all of the individual values), and set I's
 /// metadata for M equal to the intersection value.
@@ -142,7 +157,7 @@ Constant *createBitMaskForGaps(IRBuilder<> &Builder, unsigned VF,
 
 /// Create a mask with replicated elements.
 ///
-/// This function creates a shuffle mask for replicating each of the \p VF 
+/// This function creates a shuffle mask for replicating each of the \p VF
 /// elements in a vector \p ReplicationFactor times. It can be used to
 /// transform a mask of \p VF elements into a mask of
 /// \p VF * \p ReplicationFactor elements used by a predicated
diff --git a/include/llvm/AsmParser/Parser.h b/include/llvm/AsmParser/Parser.h
index 285a7c022a24a95951c175936494d145c034b60a..b0c6034978055e12fb0f87b972e96c387a113fc2 100644
--- a/include/llvm/AsmParser/Parser.h
+++ b/include/llvm/AsmParser/Parser.h
@@ -1,9 +1,8 @@
 //===-- Parser.h - Parser for LLVM IR text assembly files -------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/AsmParser/SlotMapping.h b/include/llvm/AsmParser/SlotMapping.h
index bd7e8fcad8bcc8c9cd575c41740782dfa62565af..0e95eb816b4c6c11baf50c46dbc6f76b9c90bd68 100644
--- a/include/llvm/AsmParser/SlotMapping.h
+++ b/include/llvm/AsmParser/SlotMapping.h
@@ -1,9 +1,8 @@
 //===-- SlotMapping.h - Slot number mapping for unnamed values --*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/BinaryFormat/AMDGPUMetadataVerifier.h b/include/llvm/BinaryFormat/AMDGPUMetadataVerifier.h
index de44f41720edb3369350ac1d95fc1a4700971e5e..522721a73bf1f4d084ba299b44b9d247fb0c0db1 100644
--- a/include/llvm/BinaryFormat/AMDGPUMetadataVerifier.h
+++ b/include/llvm/BinaryFormat/AMDGPUMetadataVerifier.h
@@ -1,9 +1,8 @@
 //===- AMDGPUMetadataVerifier.h - MsgPack Types -----------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/BinaryFormat/COFF.h b/include/llvm/BinaryFormat/COFF.h
index 7b973c03cc80fc7d39d3d323065f21d913776c2d..e0543dea4c20084dbfeedf52c90ab21d9ab613b5 100644
--- a/include/llvm/BinaryFormat/COFF.h
+++ b/include/llvm/BinaryFormat/COFF.h
@@ -1,9 +1,8 @@
 //===-- llvm/BinaryFormat/COFF.h --------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -371,13 +370,15 @@ enum RelocationTypesARM : unsigned {
   IMAGE_REL_ARM_TOKEN = 0x0005,
   IMAGE_REL_ARM_BLX24 = 0x0008,
   IMAGE_REL_ARM_BLX11 = 0x0009,
+  IMAGE_REL_ARM_REL32 = 0x000A,
   IMAGE_REL_ARM_SECTION = 0x000E,
   IMAGE_REL_ARM_SECREL = 0x000F,
   IMAGE_REL_ARM_MOV32A = 0x0010,
   IMAGE_REL_ARM_MOV32T = 0x0011,
   IMAGE_REL_ARM_BRANCH20T = 0x0012,
   IMAGE_REL_ARM_BRANCH24T = 0x0014,
-  IMAGE_REL_ARM_BLX23T = 0x0015
+  IMAGE_REL_ARM_BLX23T = 0x0015,
+  IMAGE_REL_ARM_PAIR = 0x0016,
 };
 
 enum RelocationTypesARM64 : unsigned {
@@ -398,6 +399,7 @@ enum RelocationTypesARM64 : unsigned {
   IMAGE_REL_ARM64_ADDR64 = 0x000E,
   IMAGE_REL_ARM64_BRANCH19 = 0x000F,
   IMAGE_REL_ARM64_BRANCH14 = 0x0010,
+  IMAGE_REL_ARM64_REL32 = 0x0011,
 };
 
 enum COMDATType : unsigned {
diff --git a/include/llvm/BinaryFormat/Dwarf.def b/include/llvm/BinaryFormat/Dwarf.def
index cb9f7f5145d1fbc48ea71945e5a083dc1f18b01c..90ae0231945458c19382ecc4dcff2f94bfa3459b 100644
--- a/include/llvm/BinaryFormat/Dwarf.def
+++ b/include/llvm/BinaryFormat/Dwarf.def
@@ -1,9 +1,8 @@
 //===- llvm/Support/Dwarf.def - Dwarf definitions ---------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -43,7 +42,7 @@
 #endif
 
 #ifndef HANDLE_DW_LANG
-#define HANDLE_DW_LANG(ID, NAME, VERSION, VENDOR)
+#define HANDLE_DW_LANG(ID, NAME, LOWER_BOUND, VERSION, VENDOR)
 #endif
 
 #ifndef HANDLE_DW_ATE
@@ -632,50 +631,50 @@ HANDLE_DW_OP(0xfb, GNU_addr_index, 0, GNU)
 HANDLE_DW_OP(0xfc, GNU_const_index, 0, GNU)
 
 // DWARF languages.
-HANDLE_DW_LANG(0x0001, C89, 2, DWARF)
-HANDLE_DW_LANG(0x0002, C, 2, DWARF)
-HANDLE_DW_LANG(0x0003, Ada83, 2, DWARF)
-HANDLE_DW_LANG(0x0004, C_plus_plus, 2, DWARF)
-HANDLE_DW_LANG(0x0005, Cobol74, 2, DWARF)
-HANDLE_DW_LANG(0x0006, Cobol85, 2, DWARF)
-HANDLE_DW_LANG(0x0007, Fortran77, 2, DWARF)
-HANDLE_DW_LANG(0x0008, Fortran90, 2, DWARF)
-HANDLE_DW_LANG(0x0009, Pascal83, 2, DWARF)
-HANDLE_DW_LANG(0x000a, Modula2, 2, DWARF)
+HANDLE_DW_LANG(0x0001, C89, 0, 2, DWARF)
+HANDLE_DW_LANG(0x0002, C, 0, 2, DWARF)
+HANDLE_DW_LANG(0x0003, Ada83, 1, 2, DWARF)
+HANDLE_DW_LANG(0x0004, C_plus_plus, 0, 2, DWARF)
+HANDLE_DW_LANG(0x0005, Cobol74, 1, 2, DWARF)
+HANDLE_DW_LANG(0x0006, Cobol85, 1, 2, DWARF)
+HANDLE_DW_LANG(0x0007, Fortran77, 1, 2, DWARF)
+HANDLE_DW_LANG(0x0008, Fortran90, 1, 2, DWARF)
+HANDLE_DW_LANG(0x0009, Pascal83, 1, 2, DWARF)
+HANDLE_DW_LANG(0x000a, Modula2, 1, 2, DWARF)
 // New in DWARF v3:
-HANDLE_DW_LANG(0x000b, Java, 3, DWARF)
-HANDLE_DW_LANG(0x000c, C99, 3, DWARF)
-HANDLE_DW_LANG(0x000d, Ada95, 3, DWARF)
-HANDLE_DW_LANG(0x000e, Fortran95, 3, DWARF)
-HANDLE_DW_LANG(0x000f, PLI, 3, DWARF)
-HANDLE_DW_LANG(0x0010, ObjC, 3, DWARF)
-HANDLE_DW_LANG(0x0011, ObjC_plus_plus, 3, DWARF)
-HANDLE_DW_LANG(0x0012, UPC, 3, DWARF)
-HANDLE_DW_LANG(0x0013, D, 3, DWARF)
+HANDLE_DW_LANG(0x000b, Java, 0, 3, DWARF)
+HANDLE_DW_LANG(0x000c, C99, 0, 3, DWARF)
+HANDLE_DW_LANG(0x000d, Ada95, 1, 3, DWARF)
+HANDLE_DW_LANG(0x000e, Fortran95, 1, 3, DWARF)
+HANDLE_DW_LANG(0x000f, PLI, 1, 3, DWARF)
+HANDLE_DW_LANG(0x0010, ObjC, 0, 3, DWARF)
+HANDLE_DW_LANG(0x0011, ObjC_plus_plus, 0, 3, DWARF)
+HANDLE_DW_LANG(0x0012, UPC, 0, 3, DWARF)
+HANDLE_DW_LANG(0x0013, D, 0, 3, DWARF)
 // New in DWARF v4:
-HANDLE_DW_LANG(0x0014, Python, 4, DWARF)
+HANDLE_DW_LANG(0x0014, Python, 0, 4, DWARF)
 // New in DWARF v5:
-HANDLE_DW_LANG(0x0015, OpenCL, 5, DWARF)
-HANDLE_DW_LANG(0x0016, Go, 5, DWARF)
-HANDLE_DW_LANG(0x0017, Modula3, 5, DWARF)
-HANDLE_DW_LANG(0x0018, Haskell, 5, DWARF)
-HANDLE_DW_LANG(0x0019, C_plus_plus_03, 5, DWARF)
-HANDLE_DW_LANG(0x001a, C_plus_plus_11, 5, DWARF)
-HANDLE_DW_LANG(0x001b, OCaml, 5, DWARF)
-HANDLE_DW_LANG(0x001c, Rust, 5, DWARF)
-HANDLE_DW_LANG(0x001d, C11, 5, DWARF)
-HANDLE_DW_LANG(0x001e, Swift, 5, DWARF)
-HANDLE_DW_LANG(0x001f, Julia, 5, DWARF)
-HANDLE_DW_LANG(0x0020, Dylan, 5, DWARF)
-HANDLE_DW_LANG(0x0021, C_plus_plus_14, 5, DWARF)
-HANDLE_DW_LANG(0x0022, Fortran03, 5, DWARF)
-HANDLE_DW_LANG(0x0023, Fortran08, 5, DWARF)
-HANDLE_DW_LANG(0x0024, RenderScript, 5, DWARF)
-HANDLE_DW_LANG(0x0025, BLISS, 5, DWARF)
+HANDLE_DW_LANG(0x0015, OpenCL, 0, 5, DWARF)
+HANDLE_DW_LANG(0x0016, Go, 0, 5, DWARF)
+HANDLE_DW_LANG(0x0017, Modula3, 1, 5, DWARF)
+HANDLE_DW_LANG(0x0018, Haskell, 0, 5, DWARF)
+HANDLE_DW_LANG(0x0019, C_plus_plus_03, 0, 5, DWARF)
+HANDLE_DW_LANG(0x001a, C_plus_plus_11, 0, 5, DWARF)
+HANDLE_DW_LANG(0x001b, OCaml, 0, 5, DWARF)
+HANDLE_DW_LANG(0x001c, Rust, 0, 5, DWARF)
+HANDLE_DW_LANG(0x001d, C11, 0, 5, DWARF)
+HANDLE_DW_LANG(0x001e, Swift, 0, 5, DWARF)
+HANDLE_DW_LANG(0x001f, Julia, 1, 5, DWARF)
+HANDLE_DW_LANG(0x0020, Dylan, 0, 5, DWARF)
+HANDLE_DW_LANG(0x0021, C_plus_plus_14, 0, 5, DWARF)
+HANDLE_DW_LANG(0x0022, Fortran03, 1, 5, DWARF)
+HANDLE_DW_LANG(0x0023, Fortran08, 1, 5, DWARF)
+HANDLE_DW_LANG(0x0024, RenderScript, 0, 5, DWARF)
+HANDLE_DW_LANG(0x0025, BLISS, 0, 5, DWARF)
 // Vendor extensions:
-HANDLE_DW_LANG(0x8001, Mips_Assembler, 0, MIPS)
-HANDLE_DW_LANG(0x8e57, GOOGLE_RenderScript, 0, GOOGLE)
-HANDLE_DW_LANG(0xb000, BORLAND_Delphi, 0, BORLAND)
+HANDLE_DW_LANG(0x8001, Mips_Assembler, None, 0, MIPS)
+HANDLE_DW_LANG(0x8e57, GOOGLE_RenderScript, 0, 0, GOOGLE)
+HANDLE_DW_LANG(0xb000, BORLAND_Delphi, 0, 0, BORLAND)
 
 // DWARF attribute type encodings.
 HANDLE_DW_ATE(0x01, address, 2, DWARF)
diff --git a/include/llvm/BinaryFormat/Dwarf.h b/include/llvm/BinaryFormat/Dwarf.h
index c25b6cd0966b13ef37982833fb3591b81c18ae7a..3757c75f6ba1ce473aa07a30d3cd4cc520eba9ca 100644
--- a/include/llvm/BinaryFormat/Dwarf.h
+++ b/include/llvm/BinaryFormat/Dwarf.h
@@ -1,9 +1,8 @@
 //===-- llvm/BinaryFormat/Dwarf.h ---Dwarf Constants-------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -184,7 +183,8 @@ enum DefaultedMemberAttribute {
 };
 
 enum SourceLanguage {
-#define HANDLE_DW_LANG(ID, NAME, VERSION, VENDOR) DW_LANG_##NAME = ID,
+#define HANDLE_DW_LANG(ID, NAME, LOWER_BOUND, VERSION, VENDOR)                 \
+  DW_LANG_##NAME = ID,
 #include "llvm/BinaryFormat/Dwarf.def"
   DW_LANG_lo_user = 0x8000,
   DW_LANG_hi_user = 0xffff
@@ -490,6 +490,8 @@ unsigned AttributeEncodingVendor(TypeKind E);
 unsigned LanguageVendor(SourceLanguage L);
 /// @}
 
+Optional<unsigned> LanguageLowerBound(SourceLanguage L);
+
 /// A helper struct providing information about the byte size of DW_FORM
 /// values that vary in size depending on the DWARF version, address byte
 /// size, or DWARF32/DWARF64.
diff --git a/include/llvm/BinaryFormat/ELF.h b/include/llvm/BinaryFormat/ELF.h
index ce35d127d43390e764795c22c98bed1ac073e464..8e3860cd88c941152339db87388670d6519b3992 100644
--- a/include/llvm/BinaryFormat/ELF.h
+++ b/include/llvm/BinaryFormat/ELF.h
@@ -1,9 +1,8 @@
 //===- llvm/BinaryFormat/ELF.h - ELF constants and structures ---*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -1352,13 +1351,26 @@ enum {
 enum : unsigned {
   GNU_PROPERTY_STACK_SIZE = 1,
   GNU_PROPERTY_NO_COPY_ON_PROTECTED = 2,
-  GNU_PROPERTY_X86_FEATURE_1_AND = 0xc0000002
+  GNU_PROPERTY_X86_FEATURE_1_AND = 0xc0000002,
+  GNU_PROPERTY_X86_FEATURE_2_NEEDED = 0xc0008001,
+  GNU_PROPERTY_X86_FEATURE_2_USED = 0xc0010001,
 };
 
-// CET properties
-enum {
+// x86 processor feature bits.
+enum : unsigned {
   GNU_PROPERTY_X86_FEATURE_1_IBT = 1 << 0,
-  GNU_PROPERTY_X86_FEATURE_1_SHSTK = 1 << 1
+  GNU_PROPERTY_X86_FEATURE_1_SHSTK = 1 << 1,
+
+  GNU_PROPERTY_X86_FEATURE_2_X86 = 1 << 0,
+  GNU_PROPERTY_X86_FEATURE_2_X87 = 1 << 1,
+  GNU_PROPERTY_X86_FEATURE_2_MMX = 1 << 2,
+  GNU_PROPERTY_X86_FEATURE_2_XMM = 1 << 3,
+  GNU_PROPERTY_X86_FEATURE_2_YMM = 1 << 4,
+  GNU_PROPERTY_X86_FEATURE_2_ZMM = 1 << 5,
+  GNU_PROPERTY_X86_FEATURE_2_FXSR = 1 << 6,
+  GNU_PROPERTY_X86_FEATURE_2_XSAVE = 1 << 7,
+  GNU_PROPERTY_X86_FEATURE_2_XSAVEOPT = 1 << 8,
+  GNU_PROPERTY_X86_FEATURE_2_XSAVEC = 1 << 9,
 };
 
 // AMD specific notes. (Code Object V2)
diff --git a/include/llvm/BinaryFormat/MachO.def b/include/llvm/BinaryFormat/MachO.def
index 95de48d2b19eb0ef023ebfb83c9f2bd6742fa13f..76dcc58ba048c3b004008c9055be5fe73260fcb3 100644
--- a/include/llvm/BinaryFormat/MachO.def
+++ b/include/llvm/BinaryFormat/MachO.def
@@ -1,9 +1,8 @@
 //,,,-- llvm/Support/MachO.def - The MachO file definitions -----*- C++ -*-,,,//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //,,,----------------------------------------------------------------------,,,//
 //
diff --git a/include/llvm/BinaryFormat/MachO.h b/include/llvm/BinaryFormat/MachO.h
index c5294c76ebf729d4070f82d76acb09336001bcc7..08fe7803d408ba54894ac9416008f72e2c1ddcc5 100644
--- a/include/llvm/BinaryFormat/MachO.h
+++ b/include/llvm/BinaryFormat/MachO.h
@@ -1,9 +1,8 @@
 //===-- llvm/BinaryFormat/MachO.h - The MachO file format -------*- C++/-*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -335,6 +334,7 @@ enum {
   N_WEAK_DEF = 0x0080u,
   N_SYMBOL_RESOLVER = 0x0100u,
   N_ALT_ENTRY = 0x0200u,
+  N_COLD_FUNC = 0x0400u,
   // For undefined symbols coming from libraries, see GET_LIBRARY_ORDINAL()
   // as these are in the top 8 bits.
   SELF_LIBRARY_ORDINAL = 0x0,
@@ -486,7 +486,10 @@ enum PlatformType {
   PLATFORM_IOS = 2,
   PLATFORM_TVOS = 3,
   PLATFORM_WATCHOS = 4,
-  PLATFORM_BRIDGEOS = 5
+  PLATFORM_BRIDGEOS = 5,
+  PLATFORM_IOSSIMULATOR = 7,
+  PLATFORM_TVOSSIMULATOR = 8,
+  PLATFORM_WATCHOSSIMULATOR = 9
 };
 
 // Values for tools enum in build_tool_version.
diff --git a/include/llvm/BinaryFormat/Magic.h b/include/llvm/BinaryFormat/Magic.h
index 04801f810be37bee2dfb55e0bb36acfd8a1ef68f..05af2c367c8f2465e4b1c3f8c26ed6fa7bb43534 100644
--- a/include/llvm/BinaryFormat/Magic.h
+++ b/include/llvm/BinaryFormat/Magic.h
@@ -1,9 +1,8 @@
 //===- llvm/BinaryFormat/Magic.h - File magic identification ----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/BinaryFormat/MsgPack.def b/include/llvm/BinaryFormat/MsgPack.def
index 781b49f46aeb7bb3ef068bf9e6b16ddfdccd541c..7ad83ff21c428fa88cadcae74d0217d72199ce51 100644
--- a/include/llvm/BinaryFormat/MsgPack.def
+++ b/include/llvm/BinaryFormat/MsgPack.def
@@ -1,9 +1,8 @@
 //===- MsgPack.def - MessagePack definitions --------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
diff --git a/include/llvm/BinaryFormat/MsgPack.h b/include/llvm/BinaryFormat/MsgPack.h
index d431912a53e5612c844853280d1230c6ad22321a..9fda14b21c7116badcbe0cfa00e1d32bb629eb1a 100644
--- a/include/llvm/BinaryFormat/MsgPack.h
+++ b/include/llvm/BinaryFormat/MsgPack.h
@@ -1,9 +1,8 @@
 //===-- MsgPack.h - MessagePack Constants -----------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
diff --git a/include/llvm/BinaryFormat/MsgPackReader.h b/include/llvm/BinaryFormat/MsgPackReader.h
index 511c314074554ce70bd5f1cc2578d1a7d49c8d42..2d332f531b2324a1233e087eb7280a5d1a7a5a8c 100644
--- a/include/llvm/BinaryFormat/MsgPackReader.h
+++ b/include/llvm/BinaryFormat/MsgPackReader.h
@@ -1,9 +1,8 @@
 //===- MsgPackReader.h - Simple MsgPack reader ------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
diff --git a/include/llvm/BinaryFormat/MsgPackTypes.h b/include/llvm/BinaryFormat/MsgPackTypes.h
index f96cd4c338fd9e55edccbe0d784deefdcc0755ff..fcb11abd67664bed99e8a54e79e2b13cfdc75a4f 100644
--- a/include/llvm/BinaryFormat/MsgPackTypes.h
+++ b/include/llvm/BinaryFormat/MsgPackTypes.h
@@ -1,9 +1,8 @@
 //===- MsgPackTypes.h - MsgPack Types ---------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/BinaryFormat/MsgPackWriter.h b/include/llvm/BinaryFormat/MsgPackWriter.h
index 98af422c9f1939764d7196376b6c9fa07fbb8c84..3b610b774f77112e4d3323f432024a5a7dc81822 100644
--- a/include/llvm/BinaryFormat/MsgPackWriter.h
+++ b/include/llvm/BinaryFormat/MsgPackWriter.h
@@ -1,9 +1,8 @@
 //===- MsgPackWriter.h - Simple MsgPack writer ------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
diff --git a/include/llvm/BinaryFormat/Wasm.h b/include/llvm/BinaryFormat/Wasm.h
index ca752374e157087a22688c45d5e350bb99c52920..51f6ab2594b0cdd4183a2363f1fb6e3c454b64a4 100644
--- a/include/llvm/BinaryFormat/Wasm.h
+++ b/include/llvm/BinaryFormat/Wasm.h
@@ -1,9 +1,8 @@
 //===- Wasm.h - Wasm object file format -------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -26,7 +25,7 @@ const char WasmMagic[] = {'\0', 'a', 's', 'm'};
 // Wasm binary format version
 const uint32_t WasmVersion = 0x1;
 // Wasm linking metadata version
-const uint32_t WasmMetadataVersion = 0x1;
+const uint32_t WasmMetadataVersion = 0x2;
 // Wasm uses a 64k page size
 const uint32_t WasmPageSize = 65536;
 
@@ -43,6 +42,12 @@ struct WasmDylinkInfo {
   std::vector<StringRef> Needed; // Shared library depenedencies
 };
 
+struct WasmProducerInfo {
+  std::vector<std::pair<std::string, std::string>> Languages;
+  std::vector<std::pair<std::string, std::string>> Tools;
+  std::vector<std::pair<std::string, std::string>> SDKs;
+};
+
 struct WasmExport {
   StringRef Name;
   uint8_t Kind;
@@ -165,7 +170,8 @@ struct WasmSymbolInfo {
   StringRef Name;
   uint8_t Kind;
   uint32_t Flags;
-  StringRef Module; // For undefined symbols the module name of the import
+  StringRef ImportModule; // For undefined symbols the module of the import
+  StringRef ImportName;   // For undefined symbols the name of the import
   union {
     // For function or global symbols, the index in function or global index
     // space.
@@ -211,7 +217,7 @@ enum : unsigned {
   WASM_TYPE_F32 = 0x7D,
   WASM_TYPE_F64 = 0x7C,
   WASM_TYPE_V128 = 0x7B,
-  WASM_TYPE_ANYFUNC = 0x70,
+  WASM_TYPE_FUNCREF = 0x70,
   WASM_TYPE_EXCEPT_REF = 0x68,
   WASM_TYPE_FUNC = 0x60,
   WASM_TYPE_NORESULT = 0x40, // for blocks with no result values
@@ -229,7 +235,7 @@ enum : unsigned {
 // Opcodes used in initializer expressions.
 enum : unsigned {
   WASM_OPCODE_END = 0x0b,
-  WASM_OPCODE_GET_GLOBAL = 0x23,
+  WASM_OPCODE_GLOBAL_GET = 0x23,
   WASM_OPCODE_I32_CONST = 0x41,
   WASM_OPCODE_I64_CONST = 0x42,
   WASM_OPCODE_F32_CONST = 0x43,
@@ -284,6 +290,8 @@ const unsigned WASM_SYMBOL_BINDING_LOCAL = 0x2;
 const unsigned WASM_SYMBOL_VISIBILITY_DEFAULT = 0x0;
 const unsigned WASM_SYMBOL_VISIBILITY_HIDDEN = 0x4;
 const unsigned WASM_SYMBOL_UNDEFINED = 0x10;
+const unsigned WASM_SYMBOL_EXPORTED = 0x20;
+const unsigned WASM_SYMBOL_EXPLICIT_NAME = 0x40;
 
 #define WASM_RELOC(name, value) name = value,
 
@@ -304,13 +312,13 @@ enum class ValType {
 };
 
 struct WasmSignature {
-  SmallVector<wasm::ValType, 1> Returns;
-  SmallVector<wasm::ValType, 4> Params;
+  SmallVector<ValType, 1> Returns;
+  SmallVector<ValType, 4> Params;
   // Support empty and tombstone instances, needed by DenseMap.
   enum { Plain, Empty, Tombstone } State = Plain;
 
-  WasmSignature(SmallVector<wasm::ValType, 1> &&InReturns,
-                SmallVector<wasm::ValType, 4> &&InParams)
+  WasmSignature(SmallVector<ValType, 1> &&InReturns,
+                SmallVector<ValType, 4> &&InParams)
       : Returns(InReturns), Params(InParams) {}
   WasmSignature() = default;
 };
@@ -333,7 +341,7 @@ inline bool operator!=(const WasmGlobalType &LHS, const WasmGlobalType &RHS) {
   return !(LHS == RHS);
 }
 
-std::string toString(wasm::WasmSymbolType type);
+std::string toString(WasmSymbolType type);
 std::string relocTypetoString(uint32_t type);
 
 } // end namespace wasm
diff --git a/include/llvm/BinaryFormat/WasmRelocs.def b/include/llvm/BinaryFormat/WasmRelocs.def
index b3a08e70c1d5d55b8333f4663c1bd50cc5655733..59a786bb8fb9a515d681e10d6fdfb6dd2c066660 100644
--- a/include/llvm/BinaryFormat/WasmRelocs.def
+++ b/include/llvm/BinaryFormat/WasmRelocs.def
@@ -2,14 +2,14 @@
 #error "WASM_RELOC must be defined"
 #endif
 
-WASM_RELOC(R_WEBASSEMBLY_FUNCTION_INDEX_LEB,   0)
-WASM_RELOC(R_WEBASSEMBLY_TABLE_INDEX_SLEB,     1)
-WASM_RELOC(R_WEBASSEMBLY_TABLE_INDEX_I32,      2)
-WASM_RELOC(R_WEBASSEMBLY_MEMORY_ADDR_LEB,      3)
-WASM_RELOC(R_WEBASSEMBLY_MEMORY_ADDR_SLEB,     4)
-WASM_RELOC(R_WEBASSEMBLY_MEMORY_ADDR_I32,      5)
-WASM_RELOC(R_WEBASSEMBLY_TYPE_INDEX_LEB,       6)
-WASM_RELOC(R_WEBASSEMBLY_GLOBAL_INDEX_LEB,     7)
-WASM_RELOC(R_WEBASSEMBLY_FUNCTION_OFFSET_I32,  8)
-WASM_RELOC(R_WEBASSEMBLY_SECTION_OFFSET_I32,   9)
-WASM_RELOC(R_WEBASSEMBLY_EVENT_INDEX_LEB,     10)
+WASM_RELOC(R_WASM_FUNCTION_INDEX_LEB,   0)
+WASM_RELOC(R_WASM_TABLE_INDEX_SLEB,     1)
+WASM_RELOC(R_WASM_TABLE_INDEX_I32,      2)
+WASM_RELOC(R_WASM_MEMORY_ADDR_LEB,      3)
+WASM_RELOC(R_WASM_MEMORY_ADDR_SLEB,     4)
+WASM_RELOC(R_WASM_MEMORY_ADDR_I32,      5)
+WASM_RELOC(R_WASM_TYPE_INDEX_LEB,       6)
+WASM_RELOC(R_WASM_GLOBAL_INDEX_LEB,     7)
+WASM_RELOC(R_WASM_FUNCTION_OFFSET_I32,  8)
+WASM_RELOC(R_WASM_SECTION_OFFSET_I32,   9)
+WASM_RELOC(R_WASM_EVENT_INDEX_LEB,     10)
diff --git a/include/llvm/Bitcode/BitCodes.h b/include/llvm/Bitcode/BitCodes.h
index bf21e146e771d9e9adcc6b48d49d03d0124a884c..a0d8dfd68bf7177554b19b717d2feb0d26fe5d02 100644
--- a/include/llvm/Bitcode/BitCodes.h
+++ b/include/llvm/Bitcode/BitCodes.h
@@ -1,9 +1,8 @@
 //===- BitCodes.h - Enum values for the bitcode format ----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -160,8 +159,6 @@ public:
 
 };
 
-template <> struct isPodLike<BitCodeAbbrevOp> { static const bool value=true; };
-
 /// BitCodeAbbrev - This class represents an abbreviation record.  An
 /// abbreviation allows a complex record that has redundancy to be stored in a
 /// specialized format instead of the fully-general, fully-vbr, format.
diff --git a/include/llvm/Bitcode/BitcodeReader.h b/include/llvm/Bitcode/BitcodeReader.h
index ce8bdd9cf0b4a57c6dd5efdedcee0104c6e1406a..3e8e7edf79ce064a8bfbd1d8e47d4b1e14cd1f01 100644
--- a/include/llvm/Bitcode/BitcodeReader.h
+++ b/include/llvm/Bitcode/BitcodeReader.h
@@ -1,9 +1,8 @@
 //===- llvm/Bitcode/BitcodeReader.h - Bitcode reader ------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -51,6 +50,7 @@ class Module;
   struct BitcodeLTOInfo {
     bool IsThinLTO;
     bool HasSummary;
+    bool EnableSplitLTOUnit;
   };
 
   /// Represents a module in a bitcode file.
diff --git a/include/llvm/Bitcode/BitcodeWriter.h b/include/llvm/Bitcode/BitcodeWriter.h
index 0010cf6c0544265335bcb49ca4bd66bc5bd868a4..39061e09cda5d980ff3850ddeece65d6ec262a81 100644
--- a/include/llvm/Bitcode/BitcodeWriter.h
+++ b/include/llvm/Bitcode/BitcodeWriter.h
@@ -1,9 +1,8 @@
 //===- llvm/Bitcode/BitcodeWriter.h - Bitcode writers -----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Bitcode/BitcodeWriterPass.h b/include/llvm/Bitcode/BitcodeWriterPass.h
index 05044c9ae11c2e48698a118661a841a48ad2d90d..1773d1b9f11b23b6a193e39da037bb2f954b9e22 100644
--- a/include/llvm/Bitcode/BitcodeWriterPass.h
+++ b/include/llvm/Bitcode/BitcodeWriterPass.h
@@ -1,9 +1,8 @@
 //===-- BitcodeWriterPass.h - Bitcode writing pass --------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/include/llvm/Bitcode/BitstreamReader.h b/include/llvm/Bitcode/BitstreamReader.h
index 72e7619d9e1c398f727a6204ba8c55797541c311..90cbe7b4b1e3219f8738203dceb596019f82617c 100644
--- a/include/llvm/Bitcode/BitstreamReader.h
+++ b/include/llvm/Bitcode/BitstreamReader.h
@@ -1,9 +1,8 @@
 //===- BitstreamReader.h - Low-level bitstream reader interface -*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Bitcode/BitstreamWriter.h b/include/llvm/Bitcode/BitstreamWriter.h
index c854769e06229692c33e98ff7b99b7efceebf423..3de3998db524abe97b9b2c1c967926bbf3eb7f84 100644
--- a/include/llvm/Bitcode/BitstreamWriter.h
+++ b/include/llvm/Bitcode/BitstreamWriter.h
@@ -1,9 +1,8 @@
 //===- BitstreamWriter.h - Low-level bitstream writer interface -*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Bitcode/LLVMBitCodes.h b/include/llvm/Bitcode/LLVMBitCodes.h
index f0d11e9c16894e849e6d83f21374e5eb8e13cdab..71148ac55c3a3585d13ebd55365b0ffae59169fb 100644
--- a/include/llvm/Bitcode/LLVMBitCodes.h
+++ b/include/llvm/Bitcode/LLVMBitCodes.h
@@ -1,9 +1,8 @@
 //===- LLVMBitCodes.h - Enum values for the LLVM bitcode format -*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -407,7 +406,9 @@ enum RMWOperations {
   RMW_MAX = 7,
   RMW_MIN = 8,
   RMW_UMAX = 9,
-  RMW_UMIN = 10
+  RMW_UMIN = 10,
+  RMW_FADD = 11,
+  RMW_FSUB = 12
 };
 
 /// OverflowingBinaryOperatorOptionalFlags - Flags for serializing
@@ -534,6 +535,8 @@ enum FunctionCodes {
   // 54 is unused.
   FUNC_CODE_OPERAND_BUNDLE = 55, // OPERAND_BUNDLE: [tag#, value...]
   FUNC_CODE_INST_UNOP = 56,      // UNOP:       [opcode, ty, opval]
+  FUNC_CODE_INST_CALLBR = 57,    // CALLBR:     [attr, cc, norm, transfs,
+                                 //              fnty, fnid, args...]
 };
 
 enum UseListCodes {
diff --git a/include/llvm/CodeGen/AccelTable.h b/include/llvm/CodeGen/AccelTable.h
index 13928582f2dd7e8e2ed8e1cdb86bbccba82b982c..9731abd198034e77b472a11f383f00fe3e0a1343 100644
--- a/include/llvm/CodeGen/AccelTable.h
+++ b/include/llvm/CodeGen/AccelTable.h
@@ -1,9 +1,8 @@
 //==- include/llvm/CodeGen/AccelTable.h - Accelerator Tables -----*- C++ -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/CodeGen/Analysis.h b/include/llvm/CodeGen/Analysis.h
index d77aee66ed764068630a3d50427aa52759ae94c7..468768dea9e1bb718b40a686986e16e3833b1eff 100644
--- a/include/llvm/CodeGen/Analysis.h
+++ b/include/llvm/CodeGen/Analysis.h
@@ -1,9 +1,8 @@
 //===- CodeGen/Analysis.h - CodeGen LLVM IR Analysis Utilities --*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/CodeGen/AsmPrinter.h b/include/llvm/CodeGen/AsmPrinter.h
index 413901d218f961e1b70577f25925b3d77f52a1e2..fb12bb26b3ea632d7abc735fd18506d33657398c 100644
--- a/include/llvm/CodeGen/AsmPrinter.h
+++ b/include/llvm/CodeGen/AsmPrinter.h
@@ -1,9 +1,8 @@
 //===- llvm/CodeGen/AsmPrinter.h - AsmPrinter Framework ---------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -122,9 +121,6 @@ public:
   using GOTEquivUsePair = std::pair<const GlobalVariable *, unsigned>;
   MapVector<const MCSymbol *, GOTEquivUsePair> GlobalGOTEquivs;
 
-  /// Enable print [latency:throughput] in output.
-  bool EnablePrintSchedInfo = false;
-
 private:
   MCSymbol *CurrentFnBegin = nullptr;
   MCSymbol *CurrentFnEnd = nullptr;
@@ -227,6 +223,9 @@ public:
 
   void EmitToStreamer(MCStreamer &S, const MCInst &Inst);
 
+  /// Emits inital debug location directive.
+  void emitInitialRawDwarfLocDirective(const MachineFunction &MF);
+
   /// Return the current section we are emitting to.
   const MCSection *getCurrentSection() const;
 
diff --git a/include/llvm/CodeGen/AsmPrinterHandler.h b/include/llvm/CodeGen/AsmPrinterHandler.h
index a8b13200dd4e33eba1301e9ec5e1ffb296ee59ad..affb558f2fa66f632b66d90d47243a6d35aa2d33 100644
--- a/include/llvm/CodeGen/AsmPrinterHandler.h
+++ b/include/llvm/CodeGen/AsmPrinterHandler.h
@@ -1,9 +1,8 @@
 //===-- llvm/CodeGen/AsmPrinterHandler.h -----------------------*- C++ -*--===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/CodeGen/AtomicExpandUtils.h b/include/llvm/CodeGen/AtomicExpandUtils.h
index b1adf66e7ff42bae64c0f1eda00877deb43435f0..8a46c6e00d22b37b1804475fafb6fe2ccd0ff4eb 100644
--- a/include/llvm/CodeGen/AtomicExpandUtils.h
+++ b/include/llvm/CodeGen/AtomicExpandUtils.h
@@ -1,9 +1,8 @@
 //===- AtomicExpandUtils.h - Utilities for expanding atomic instructions --===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/CodeGen/BasicTTIImpl.h b/include/llvm/CodeGen/BasicTTIImpl.h
index f105d887c397f3386eb1dfe9acd6019c17faafc4..1e9aeab9711623990b6c8bdd6f1c16e35ea70648 100644
--- a/include/llvm/CodeGen/BasicTTIImpl.h
+++ b/include/llvm/CodeGen/BasicTTIImpl.h
@@ -1,9 +1,8 @@
 //===- BasicTTIImpl.h -------------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -997,7 +996,7 @@ public:
     // inside the loop.
     if (UseMaskForGaps)
       Cost += static_cast<T *>(this)->getArithmeticInstrCost(
-          BinaryOperator::And, MaskVT); 
+          BinaryOperator::And, MaskVT);
 
     return Cost;
   }
@@ -1116,6 +1115,9 @@ public:
   unsigned getIntrinsicInstrCost(
       Intrinsic::ID IID, Type *RetTy, ArrayRef<Type *> Tys, FastMathFlags FMF,
       unsigned ScalarizationCostPassed = std::numeric_limits<unsigned>::max()) {
+    unsigned RetVF = (RetTy->isVectorTy() ? RetTy->getVectorNumElements() : 1);
+    auto *ConcreteTTI = static_cast<T *>(this);
+
     SmallVector<unsigned, 2> ISDs;
     unsigned SingleCallCost = 10; // Library call cost. Make it expensive.
     switch (IID) {
@@ -1144,8 +1146,8 @@ public:
       if (ScalarCalls == 1)
         return 1; // Return cost of a scalar intrinsic. Assume it to be cheap.
 
-      unsigned ScalarCost = static_cast<T *>(this)->getIntrinsicInstrCost(
-          IID, ScalarRetTy, ScalarTys, FMF);
+      unsigned ScalarCost =
+          ConcreteTTI->getIntrinsicInstrCost(IID, ScalarRetTy, ScalarTys, FMF);
 
       return ScalarCalls * ScalarCost + ScalarizationCost;
     }
@@ -1227,44 +1229,123 @@ public:
     case Intrinsic::sideeffect:
       return 0;
     case Intrinsic::masked_store:
-      return static_cast<T *>(this)
-          ->getMaskedMemoryOpCost(Instruction::Store, Tys[0], 0, 0);
+      return ConcreteTTI->getMaskedMemoryOpCost(Instruction::Store, Tys[0], 0,
+                                                0);
     case Intrinsic::masked_load:
-      return static_cast<T *>(this)
-          ->getMaskedMemoryOpCost(Instruction::Load, RetTy, 0, 0);
+      return ConcreteTTI->getMaskedMemoryOpCost(Instruction::Load, RetTy, 0, 0);
     case Intrinsic::experimental_vector_reduce_add:
-      return static_cast<T *>(this)->getArithmeticReductionCost(
-          Instruction::Add, Tys[0], /*IsPairwiseForm=*/false);
+      return ConcreteTTI->getArithmeticReductionCost(Instruction::Add, Tys[0],
+                                                     /*IsPairwiseForm=*/false);
     case Intrinsic::experimental_vector_reduce_mul:
-      return static_cast<T *>(this)->getArithmeticReductionCost(
-          Instruction::Mul, Tys[0], /*IsPairwiseForm=*/false);
+      return ConcreteTTI->getArithmeticReductionCost(Instruction::Mul, Tys[0],
+                                                     /*IsPairwiseForm=*/false);
     case Intrinsic::experimental_vector_reduce_and:
-      return static_cast<T *>(this)->getArithmeticReductionCost(
-          Instruction::And, Tys[0], /*IsPairwiseForm=*/false);
+      return ConcreteTTI->getArithmeticReductionCost(Instruction::And, Tys[0],
+                                                     /*IsPairwiseForm=*/false);
     case Intrinsic::experimental_vector_reduce_or:
-      return static_cast<T *>(this)->getArithmeticReductionCost(
-          Instruction::Or, Tys[0], /*IsPairwiseForm=*/false);
+      return ConcreteTTI->getArithmeticReductionCost(Instruction::Or, Tys[0],
+                                                     /*IsPairwiseForm=*/false);
     case Intrinsic::experimental_vector_reduce_xor:
-      return static_cast<T *>(this)->getArithmeticReductionCost(
-          Instruction::Xor, Tys[0], /*IsPairwiseForm=*/false);
+      return ConcreteTTI->getArithmeticReductionCost(Instruction::Xor, Tys[0],
+                                                     /*IsPairwiseForm=*/false);
     case Intrinsic::experimental_vector_reduce_fadd:
-      return static_cast<T *>(this)->getArithmeticReductionCost(
-          Instruction::FAdd, Tys[0], /*IsPairwiseForm=*/false);
+      return ConcreteTTI->getArithmeticReductionCost(Instruction::FAdd, Tys[0],
+                                                     /*IsPairwiseForm=*/false);
     case Intrinsic::experimental_vector_reduce_fmul:
-      return static_cast<T *>(this)->getArithmeticReductionCost(
-          Instruction::FMul, Tys[0], /*IsPairwiseForm=*/false);
+      return ConcreteTTI->getArithmeticReductionCost(Instruction::FMul, Tys[0],
+                                                     /*IsPairwiseForm=*/false);
     case Intrinsic::experimental_vector_reduce_smax:
     case Intrinsic::experimental_vector_reduce_smin:
     case Intrinsic::experimental_vector_reduce_fmax:
     case Intrinsic::experimental_vector_reduce_fmin:
-      return static_cast<T *>(this)->getMinMaxReductionCost(
+      return ConcreteTTI->getMinMaxReductionCost(
           Tys[0], CmpInst::makeCmpResultType(Tys[0]), /*IsPairwiseForm=*/false,
           /*IsSigned=*/true);
     case Intrinsic::experimental_vector_reduce_umax:
     case Intrinsic::experimental_vector_reduce_umin:
-      return static_cast<T *>(this)->getMinMaxReductionCost(
+      return ConcreteTTI->getMinMaxReductionCost(
           Tys[0], CmpInst::makeCmpResultType(Tys[0]), /*IsPairwiseForm=*/false,
           /*IsSigned=*/false);
+    case Intrinsic::sadd_sat:
+    case Intrinsic::ssub_sat: {
+      Type *CondTy = Type::getInt1Ty(RetTy->getContext());
+      if (RetVF > 1)
+        CondTy = VectorType::get(CondTy, RetVF);
+
+      Type *OpTy = StructType::create({RetTy, CondTy});
+      Intrinsic::ID OverflowOp = IID == Intrinsic::sadd_sat
+                                     ? Intrinsic::sadd_with_overflow
+                                     : Intrinsic::ssub_with_overflow;
+
+      // SatMax -> Overflow && SumDiff < 0
+      // SatMin -> Overflow && SumDiff >= 0
+      unsigned Cost = 0;
+      Cost += ConcreteTTI->getIntrinsicInstrCost(
+          OverflowOp, OpTy, {RetTy, RetTy}, FMF, ScalarizationCostPassed);
+      Cost += ConcreteTTI->getCmpSelInstrCost(BinaryOperator::ICmp, RetTy,
+                                              CondTy, nullptr);
+      Cost += 2 * ConcreteTTI->getCmpSelInstrCost(BinaryOperator::Select, RetTy,
+                                                  CondTy, nullptr);
+      return Cost;
+    }
+    case Intrinsic::uadd_sat:
+    case Intrinsic::usub_sat: {
+      Type *CondTy = Type::getInt1Ty(RetTy->getContext());
+      if (RetVF > 1)
+        CondTy = VectorType::get(CondTy, RetVF);
+
+      Type *OpTy = StructType::create({RetTy, CondTy});
+      Intrinsic::ID OverflowOp = IID == Intrinsic::uadd_sat
+                                     ? Intrinsic::uadd_with_overflow
+                                     : Intrinsic::usub_with_overflow;
+
+      unsigned Cost = 0;
+      Cost += ConcreteTTI->getIntrinsicInstrCost(
+          OverflowOp, OpTy, {RetTy, RetTy}, FMF, ScalarizationCostPassed);
+      Cost += ConcreteTTI->getCmpSelInstrCost(BinaryOperator::Select, RetTy,
+                                              CondTy, nullptr);
+      return Cost;
+    }
+    case Intrinsic::sadd_with_overflow:
+    case Intrinsic::ssub_with_overflow: {
+      Type *SumTy = RetTy->getContainedType(0);
+      Type *OverflowTy = RetTy->getContainedType(1);
+      unsigned Opcode = IID == Intrinsic::sadd_with_overflow
+                            ? BinaryOperator::Add
+                            : BinaryOperator::Sub;
+
+      //   LHSSign -> LHS >= 0
+      //   RHSSign -> RHS >= 0
+      //   SumSign -> Sum >= 0
+      //
+      //   Add:
+      //   Overflow -> (LHSSign == RHSSign) && (LHSSign != SumSign)
+      //   Sub:
+      //   Overflow -> (LHSSign != RHSSign) && (LHSSign != SumSign)
+      unsigned Cost = 0;
+      Cost += ConcreteTTI->getArithmeticInstrCost(Opcode, SumTy);
+      Cost += 3 * ConcreteTTI->getCmpSelInstrCost(BinaryOperator::ICmp, SumTy,
+                                                  OverflowTy, nullptr);
+      Cost += 2 * ConcreteTTI->getCmpSelInstrCost(
+                      BinaryOperator::ICmp, OverflowTy, OverflowTy, nullptr);
+      Cost +=
+          ConcreteTTI->getArithmeticInstrCost(BinaryOperator::And, OverflowTy);
+      return Cost;
+    }
+    case Intrinsic::uadd_with_overflow:
+    case Intrinsic::usub_with_overflow: {
+      Type *SumTy = RetTy->getContainedType(0);
+      Type *OverflowTy = RetTy->getContainedType(1);
+      unsigned Opcode = IID == Intrinsic::uadd_with_overflow
+                            ? BinaryOperator::Add
+                            : BinaryOperator::Sub;
+
+      unsigned Cost = 0;
+      Cost += ConcreteTTI->getArithmeticInstrCost(Opcode, SumTy);
+      Cost += ConcreteTTI->getCmpSelInstrCost(BinaryOperator::ICmp, SumTy,
+                                              OverflowTy, nullptr);
+      return Cost;
+    }
     case Intrinsic::ctpop:
       ISDs.push_back(ISD::CTPOP);
       // In case of legalization use TCC_Expensive. This is cheaper than a
@@ -1305,17 +1386,16 @@ public:
     if (MinLegalCostI != LegalCost.end())
       return *MinLegalCostI;
 
-    auto MinCustomCostI = std::min_element(CustomCost.begin(), CustomCost.end());
+    auto MinCustomCostI =
+        std::min_element(CustomCost.begin(), CustomCost.end());
     if (MinCustomCostI != CustomCost.end())
       return *MinCustomCostI;
 
     // If we can't lower fmuladd into an FMA estimate the cost as a floating
     // point mul followed by an add.
     if (IID == Intrinsic::fmuladd)
-      return static_cast<T *>(this)
-                 ->getArithmeticInstrCost(BinaryOperator::FMul, RetTy) +
-             static_cast<T *>(this)
-                 ->getArithmeticInstrCost(BinaryOperator::FAdd, RetTy);
+      return ConcreteTTI->getArithmeticInstrCost(BinaryOperator::FMul, RetTy) +
+             ConcreteTTI->getArithmeticInstrCost(BinaryOperator::FAdd, RetTy);
 
     // Else, assume that we need to scalarize this intrinsic. For math builtins
     // this will emit a costly libcall, adding call overhead and spills. Make it
@@ -1333,7 +1413,7 @@ public:
           Ty = Ty->getScalarType();
         ScalarTys.push_back(Ty);
       }
-      unsigned ScalarCost = static_cast<T *>(this)->getIntrinsicInstrCost(
+      unsigned ScalarCost = ConcreteTTI->getIntrinsicInstrCost(
           IID, RetTy->getScalarType(), ScalarTys, FMF);
       for (unsigned i = 0, ie = Tys.size(); i != ie; ++i) {
         if (Tys[i]->isVectorTy()) {
diff --git a/include/llvm/CodeGen/BuiltinGCs.h b/include/llvm/CodeGen/BuiltinGCs.h
index 1767922fb5ac0c7d77c7c5f3a8c4445cb5b99bb6..d44183dab0f7fe52f3a137b97b8732863fdf385f 100644
--- a/include/llvm/CodeGen/BuiltinGCs.h
+++ b/include/llvm/CodeGen/BuiltinGCs.h
@@ -1,9 +1,8 @@
 //===-- BuiltinGCs.h - Garbage collector linkage hacks --------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/CodeGen/CalcSpillWeights.h b/include/llvm/CodeGen/CalcSpillWeights.h
index f85767f1fc119641085a66a54f777313590c8b61..9b8b7324f30a3a334c25c0640b087dfb3c0df00a 100644
--- a/include/llvm/CodeGen/CalcSpillWeights.h
+++ b/include/llvm/CodeGen/CalcSpillWeights.h
@@ -1,9 +1,8 @@
 //===- lib/CodeGen/CalcSpillWeights.h ---------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/CodeGen/CallingConvLower.h b/include/llvm/CodeGen/CallingConvLower.h
index efcf80ba0b4ea5761c469f0e0f23455bcca97e71..78aebbefc932e6cc5eaae51e7863baf217d8dffe 100644
--- a/include/llvm/CodeGen/CallingConvLower.h
+++ b/include/llvm/CodeGen/CallingConvLower.h
@@ -1,9 +1,8 @@
 //===- llvm/CallingConvLower.h - Calling Conventions ------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/CodeGen/CommandFlags.inc b/include/llvm/CodeGen/CommandFlags.inc
index 6535e068b6e819894ec17012cec0a6812432b057..5b9564c1062ffc38c5f7a75254bad4ea23b01c83 100644
--- a/include/llvm/CodeGen/CommandFlags.inc
+++ b/include/llvm/CodeGen/CommandFlags.inc
@@ -1,9 +1,8 @@
 //===-- CommandFlags.h - Command Line Flags Interface -----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -114,10 +113,16 @@ static cl::opt<TargetMachine::CodeGenFileType> FileType(
                clEnumValN(TargetMachine::CGFT_Null, "null",
                           "Emit nothing, for performance testing")));
 
-static cl::opt<bool>
-    DisableFPElim("disable-fp-elim",
-                  cl::desc("Disable frame pointer elimination optimization"),
-                  cl::init(false));
+static cl::opt<llvm::FramePointer::FP> FramePointerUsage(
+    "frame-pointer", cl::desc("Specify frame pointer elimination optimization"),
+    cl::init(llvm::FramePointer::None),
+    cl::values(
+        clEnumValN(llvm::FramePointer::All, "all",
+                   "Disable frame pointer elimination"),
+        clEnumValN(llvm::FramePointer::NonLeaf, "non-leaf",
+                   "Disable frame pointer elimination for non-leaf frame"),
+        clEnumValN(llvm::FramePointer::None, "none",
+                   "Enable frame pointer elimination")));
 
 static cl::opt<bool> EnableUnsafeFPMath(
     "enable-unsafe-fp-math",
@@ -368,9 +373,14 @@ setFunctionAttributes(StringRef CPU, StringRef Features, Module &M) {
       NewAttrs.addAttribute("target-cpu", CPU);
     if (!Features.empty())
       NewAttrs.addAttribute("target-features", Features);
-    if (DisableFPElim.getNumOccurrences() > 0)
-      NewAttrs.addAttribute("no-frame-pointer-elim",
-                            DisableFPElim ? "true" : "false");
+    if (FramePointerUsage.getNumOccurrences() > 0) {
+      if (FramePointerUsage == llvm::FramePointer::All)
+        NewAttrs.addAttribute("frame-pointer", "all");
+      else if (FramePointerUsage == llvm::FramePointer::NonLeaf)
+        NewAttrs.addAttribute("frame-pointer", "non-leaf");
+      else if (FramePointerUsage == llvm::FramePointer::None)
+        NewAttrs.addAttribute("frame-pointer", "none");
+    }
     if (DisableTailCalls.getNumOccurrences() > 0)
       NewAttrs.addAttribute("disable-tail-calls",
                             toStringRef(DisableTailCalls));
diff --git a/include/llvm/CodeGen/CostTable.h b/include/llvm/CodeGen/CostTable.h
index 48ad76971520db0ae6cd8f8641f6634c7e2c7017..52f3bfaea180b42e7e5536ee31a52939bee2a8a5 100644
--- a/include/llvm/CodeGen/CostTable.h
+++ b/include/llvm/CodeGen/CostTable.h
@@ -1,9 +1,8 @@
 //===-- CostTable.h - Instruction Cost Table handling -----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
diff --git a/include/llvm/CodeGen/DAGCombine.h b/include/llvm/CodeGen/DAGCombine.h
index 8b591900545121aab88fc76bdb65ebcbdd019e5a..9441873414555e07bed9ef1d0904ed0d03ea257c 100644
--- a/include/llvm/CodeGen/DAGCombine.h
+++ b/include/llvm/CodeGen/DAGCombine.h
@@ -1,9 +1,8 @@
 //===-- llvm/CodeGen/DAGCombine.h  ------- SelectionDAG Nodes ---*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/CodeGen/DFAPacketizer.h b/include/llvm/CodeGen/DFAPacketizer.h
index d3aabe22f21652e79aee0550bf4336a1de3af4ff..cf58ee0cabea97eb0c0b41f57e2c8b9697bf5807 100644
--- a/include/llvm/CodeGen/DFAPacketizer.h
+++ b/include/llvm/CodeGen/DFAPacketizer.h
@@ -1,9 +1,8 @@
 //===- llvm/CodeGen/DFAPacketizer.h - DFA Packetizer for VLIW ---*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 // This class implements a deterministic finite automaton (DFA) based
diff --git a/include/llvm/CodeGen/DIE.h b/include/llvm/CodeGen/DIE.h
index 7d486b1df56dc18cdbdbbdb320523fb2d67b1268..9e7167fd88c71ba0045b129ecd46d8c34078145b 100644
--- a/include/llvm/CodeGen/DIE.h
+++ b/include/llvm/CodeGen/DIE.h
@@ -1,9 +1,8 @@
 //===- lib/CodeGen/DIE.h - DWARF Info Entries -------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -800,7 +799,7 @@ class DIEUnit {
   const uint16_t Version; /// The Dwarf version number for this unit.
   const uint8_t AddrSize; /// The size in bytes of an address for this unit.
 protected:
-  ~DIEUnit() = default;
+  virtual ~DIEUnit() = default;
 
 public:
   DIEUnit(uint16_t Version, uint8_t AddrSize, dwarf::Tag UnitTag);
diff --git a/include/llvm/CodeGen/DIEValue.def b/include/llvm/CodeGen/DIEValue.def
index a3fce9b1d20cc7c57e781d0de78e06fbe1bcb56c..c6c4c9a477ce9b1414ad9ee186522b654fffb697 100644
--- a/include/llvm/CodeGen/DIEValue.def
+++ b/include/llvm/CodeGen/DIEValue.def
@@ -1,9 +1,8 @@
 //===- llvm/CodeGen/DIEValue.def - DIEValue types ---------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/CodeGen/DbgEntityHistoryCalculator.h b/include/llvm/CodeGen/DbgEntityHistoryCalculator.h
index befc28f084e7f5d5e8ab382095f2382d354324a0..b5374d89d7e3b9873da326138f66d3b617600145 100644
--- a/include/llvm/CodeGen/DbgEntityHistoryCalculator.h
+++ b/include/llvm/CodeGen/DbgEntityHistoryCalculator.h
@@ -1,9 +1,8 @@
 //===- llvm/CodeGen/DbgEntityHistoryCalculator.h ----------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/CodeGen/DebugHandlerBase.h b/include/llvm/CodeGen/DebugHandlerBase.h
index 4f0d14d317f2a11d87fab58e01b18f6119312a4b..3771d4892cfd0a25b70dcb667471ae33059626dc 100644
--- a/include/llvm/CodeGen/DebugHandlerBase.h
+++ b/include/llvm/CodeGen/DebugHandlerBase.h
@@ -1,9 +1,8 @@
 //===-- llvm/CodeGen/DebugHandlerBase.h -----------------------*- C++ -*--===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/CodeGen/DwarfStringPoolEntry.h b/include/llvm/CodeGen/DwarfStringPoolEntry.h
index 8b1a7af17bbf3e6d1becfd653ac762dc33fb1fc6..e189352a7b2d8627e12096082f1b9a4f337bd91e 100644
--- a/include/llvm/CodeGen/DwarfStringPoolEntry.h
+++ b/include/llvm/CodeGen/DwarfStringPoolEntry.h
@@ -1,9 +1,8 @@
 //===- llvm/CodeGen/DwarfStringPoolEntry.h - String pool entry --*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/CodeGen/EdgeBundles.h b/include/llvm/CodeGen/EdgeBundles.h
index c31fad246c9632e6fa19d934b42dcd5ec30c5a02..28cdf54e0575144fc17cdfb5030139ac2dc55073 100644
--- a/include/llvm/CodeGen/EdgeBundles.h
+++ b/include/llvm/CodeGen/EdgeBundles.h
@@ -1,9 +1,8 @@
 //===-------- EdgeBundles.h - Bundles of CFG edges --------------*- c++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/CodeGen/ExecutionDomainFix.h b/include/llvm/CodeGen/ExecutionDomainFix.h
index 338c214dd073402112fb42e612d01fbc52c56aa3..6836678e2101c881ae4b233a18342bda953f94be 100644
--- a/include/llvm/CodeGen/ExecutionDomainFix.h
+++ b/include/llvm/CodeGen/ExecutionDomainFix.h
@@ -1,9 +1,8 @@
 //==-- llvm/CodeGen/ExecutionDomainFix.h - Execution Domain Fix -*- C++ -*--==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/CodeGen/ExpandReductions.h b/include/llvm/CodeGen/ExpandReductions.h
index c6aaaad967b38fd3f35267dcc68d0f5eadab8aa0..5dbed07873c1140a1910d688f36b034febf59357 100644
--- a/include/llvm/CodeGen/ExpandReductions.h
+++ b/include/llvm/CodeGen/ExpandReductions.h
@@ -1,9 +1,8 @@
 //===----- ExpandReductions.h - Expand experimental reduction intrinsics --===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/CodeGen/FastISel.h b/include/llvm/CodeGen/FastISel.h
index 865d8a88b8cc87ed5f3ce535613f4461c123b1f1..394324b9d9a98549e5b48a6ff90165b376742094 100644
--- a/include/llvm/CodeGen/FastISel.h
+++ b/include/llvm/CodeGen/FastISel.h
@@ -1,9 +1,8 @@
 //===- FastISel.h - Definition of the FastISel class ------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
diff --git a/include/llvm/CodeGen/FaultMaps.h b/include/llvm/CodeGen/FaultMaps.h
index 55e25c9823b17db3ae6f46843c487a3ae118087e..a1e2349c413ecc1cce2f306e07363e2665f364a0 100644
--- a/include/llvm/CodeGen/FaultMaps.h
+++ b/include/llvm/CodeGen/FaultMaps.h
@@ -1,9 +1,8 @@
 //===- FaultMaps.h - The "FaultMaps" section --------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/CodeGen/FunctionLoweringInfo.h b/include/llvm/CodeGen/FunctionLoweringInfo.h
index 7c658515de0952ff84f72cd67f50a0ed610f7cf4..f5f37d1403a3a14ff90da44b48f33f3f3c19cb67 100644
--- a/include/llvm/CodeGen/FunctionLoweringInfo.h
+++ b/include/llvm/CodeGen/FunctionLoweringInfo.h
@@ -1,9 +1,8 @@
 //===- FunctionLoweringInfo.h - Lower functions from LLVM IR ---*- C++ -*--===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -16,6 +15,7 @@
 #define LLVM_CODEGEN_FUNCTIONLOWERINGINFO_H
 
 #include "llvm/ADT/APInt.h"
+#include "llvm/ADT/BitVector.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/IndexedMap.h"
 #include "llvm/ADT/Optional.h"
@@ -175,6 +175,10 @@ public:
   /// function arguments that are inserted after scheduling is completed.
   SmallVector<MachineInstr*, 8> ArgDbgValues;
 
+  /// Bitvector with a bit set if corresponding argument is described in
+  /// ArgDbgValues. Using arg numbers according to Argument numbering.
+  BitVector DescribedArgs;
+
   /// RegFixups - Registers which need to be replaced after isel is done.
   DenseMap<unsigned, unsigned> RegFixups;
 
diff --git a/include/llvm/CodeGen/GCMetadata.h b/include/llvm/CodeGen/GCMetadata.h
index 7fb27202c1226c1d88362e4f7b4136c21616bd4c..77cd356c49ddaafe12b5c1a1f3c15008f39557bb 100644
--- a/include/llvm/CodeGen/GCMetadata.h
+++ b/include/llvm/CodeGen/GCMetadata.h
@@ -1,9 +1,8 @@
 //===- GCMetadata.h - Garbage collector metadata ----------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/CodeGen/GCMetadataPrinter.h b/include/llvm/CodeGen/GCMetadataPrinter.h
index 5f1efb2ce02c2d81dc10531a6912678668927386..f9527c9f8752e93896507ffe889bdb00e1870325 100644
--- a/include/llvm/CodeGen/GCMetadataPrinter.h
+++ b/include/llvm/CodeGen/GCMetadataPrinter.h
@@ -1,9 +1,8 @@
 //===- llvm/CodeGen/GCMetadataPrinter.h - Prints asm GC tables --*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/CodeGen/GCStrategy.h b/include/llvm/CodeGen/GCStrategy.h
index 5a60cd7cb8236dd54e1d1bf1368370748f2ad3cf..c5731528da4ecd49f2397c08b54f4991ee628ca9 100644
--- a/include/llvm/CodeGen/GCStrategy.h
+++ b/include/llvm/CodeGen/GCStrategy.h
@@ -1,9 +1,8 @@
 //===- llvm/CodeGen/GCStrategy.h - Garbage collection -----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/CodeGen/GlobalISel/CSEInfo.h b/include/llvm/CodeGen/GlobalISel/CSEInfo.h
new file mode 100644
index 0000000000000000000000000000000000000000..97f3cd5c95b588b1742b19753792c9abd3827231
--- /dev/null
+++ b/include/llvm/CodeGen/GlobalISel/CSEInfo.h
@@ -0,0 +1,236 @@
+//===- llvm/CodeGen/GlobalISel/CSEInfo.h ------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+/// Provides analysis for continuously CSEing during GISel passes.
+//
+//===----------------------------------------------------------------------===//
+#ifndef LLVM_CODEGEN_GLOBALISEL_CSEINFO_H
+#define LLVM_CODEGEN_GLOBALISEL_CSEINFO_H
+
+#include "llvm/ADT/FoldingSet.h"
+#include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h"
+#include "llvm/CodeGen/GlobalISel/GISelWorkList.h"
+#include "llvm/CodeGen/GlobalISel/Utils.h"
+#include "llvm/CodeGen/MachineBasicBlock.h"
+#include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/IR/PassManager.h"
+#include "llvm/Pass.h"
+#include "llvm/Support/Allocator.h"
+
+namespace llvm {
+
+/// A class that wraps MachineInstrs and derives from FoldingSetNode in order to
+/// be uniqued in a CSEMap. The tradeoff here is extra memory allocations for
+/// UniqueMachineInstr vs making MachineInstr bigger.
+class UniqueMachineInstr : public FoldingSetNode {
+  friend class GISelCSEInfo;
+  const MachineInstr *MI;
+  explicit UniqueMachineInstr(const MachineInstr *MI) : MI(MI) {}
+
+public:
+  void Profile(FoldingSetNodeID &ID);
+};
+
+// Class representing some configuration that can be done during CSE analysis.
+// Currently it only supports shouldCSE method that each pass can set.
+class CSEConfig {
+public:
+  virtual ~CSEConfig() = default;
+  // Hook for defining which Generic instructions should be CSEd.
+  // GISelCSEInfo currently only calls this hook when dealing with generic
+  // opcodes.
+  virtual bool shouldCSEOpc(unsigned Opc);
+};
+
+// TODO: Find a better place for this.
+// Commonly used for O0 config.
+class CSEConfigConstantOnly : public CSEConfig {
+public:
+  virtual ~CSEConfigConstantOnly() = default;
+  virtual bool shouldCSEOpc(unsigned Opc) override;
+};
+
+/// The CSE Analysis object.
+/// This installs itself as a delegate to the MachineFunction to track
+/// new instructions as well as deletions. It however will not be able to
+/// track instruction mutations. In such cases, recordNewInstruction should be
+/// called (for eg inside MachineIRBuilder::recordInsertion).
+/// Also because of how just the instruction can be inserted without adding any
+/// operands to the instruction, instructions are uniqued and inserted lazily.
+/// CSEInfo should assert when trying to enter an incomplete instruction into
+/// the CSEMap. There is Opcode level granularity on which instructions can be
+/// CSE'd and for now, only Generic instructions are CSEable.
+class GISelCSEInfo : public GISelChangeObserver {
+  // Make it accessible only to CSEMIRBuilder.
+  friend class CSEMIRBuilder;
+
+  BumpPtrAllocator UniqueInstrAllocator;
+  FoldingSet<UniqueMachineInstr> CSEMap;
+  MachineRegisterInfo *MRI = nullptr;
+  MachineFunction *MF = nullptr;
+  std::unique_ptr<CSEConfig> CSEOpt;
+  /// Keep a cache of UniqueInstrs for each MachineInstr. In GISel,
+  /// often instructions are mutated (while their ID has completely changed).
+  /// Whenever mutation happens, invalidate the UniqueMachineInstr for the
+  /// MachineInstr
+  DenseMap<const MachineInstr *, UniqueMachineInstr *> InstrMapping;
+
+  /// Store instructions that are not fully formed in TemporaryInsts.
+  /// Also because CSE insertion happens lazily, we can remove insts from this
+  /// list and avoid inserting and then removing from the CSEMap.
+  GISelWorkList<8> TemporaryInsts;
+
+  // Only used in asserts.
+  DenseMap<unsigned, unsigned> OpcodeHitTable;
+
+  bool isUniqueMachineInstValid(const UniqueMachineInstr &UMI) const;
+
+  void invalidateUniqueMachineInstr(UniqueMachineInstr *UMI);
+
+  UniqueMachineInstr *getNodeIfExists(FoldingSetNodeID &ID,
+                                      MachineBasicBlock *MBB, void *&InsertPos);
+
+  /// Allocate and construct a new UniqueMachineInstr for MI and return.
+  UniqueMachineInstr *getUniqueInstrForMI(const MachineInstr *MI);
+
+  void insertNode(UniqueMachineInstr *UMI, void *InsertPos = nullptr);
+
+  /// Get the MachineInstr(Unique) if it exists already in the CSEMap and the
+  /// same MachineBasicBlock.
+  MachineInstr *getMachineInstrIfExists(FoldingSetNodeID &ID,
+                                        MachineBasicBlock *MBB,
+                                        void *&InsertPos);
+
+  /// Use this method to allocate a new UniqueMachineInstr for MI and insert it
+  /// into the CSEMap. MI should return true for shouldCSE(MI->getOpcode())
+  void insertInstr(MachineInstr *MI, void *InsertPos = nullptr);
+
+public:
+  GISelCSEInfo() = default;
+
+  virtual ~GISelCSEInfo();
+
+  void setMF(MachineFunction &MF);
+
+  /// Records a newly created inst in a list and lazily insert it to the CSEMap.
+  /// Sometimes, this method might be called with a partially constructed
+  /// MachineInstr,
+  //  (right after BuildMI without adding any operands) - and in such cases,
+  //  defer the hashing of the instruction to a later stage.
+  void recordNewInstruction(MachineInstr *MI);
+
+  /// Use this callback to inform CSE about a newly fully created instruction.
+  void handleRecordedInst(MachineInstr *MI);
+
+  /// Use this callback to insert all the recorded instructions. At this point,
+  /// all of these insts need to be fully constructed and should not be missing
+  /// any operands.
+  void handleRecordedInsts();
+
+  /// Remove this inst from the CSE map. If this inst has not been inserted yet,
+  /// it will be removed from the Tempinsts list if it exists.
+  void handleRemoveInst(MachineInstr *MI);
+
+  void releaseMemory();
+
+  void setCSEConfig(std::unique_ptr<CSEConfig> Opt) { CSEOpt = std::move(Opt); }
+
+  bool shouldCSE(unsigned Opc) const;
+
+  void analyze(MachineFunction &MF);
+
+  void countOpcodeHit(unsigned Opc);
+
+  void print();
+
+  // Observer API
+  void erasingInstr(MachineInstr &MI) override;
+  void createdInstr(MachineInstr &MI) override;
+  void changingInstr(MachineInstr &MI) override;
+  void changedInstr(MachineInstr &MI) override;
+};
+
+class TargetRegisterClass;
+class RegisterBank;
+
+// Simple builder class to easily profile properties about MIs.
+class GISelInstProfileBuilder {
+  FoldingSetNodeID &ID;
+  const MachineRegisterInfo &MRI;
+
+public:
+  GISelInstProfileBuilder(FoldingSetNodeID &ID, const MachineRegisterInfo &MRI)
+      : ID(ID), MRI(MRI) {}
+  // Profiling methods.
+  const GISelInstProfileBuilder &addNodeIDOpcode(unsigned Opc) const;
+  const GISelInstProfileBuilder &addNodeIDRegType(const LLT &Ty) const;
+  const GISelInstProfileBuilder &addNodeIDRegType(const unsigned) const;
+
+  const GISelInstProfileBuilder &
+  addNodeIDRegType(const TargetRegisterClass *RC) const;
+  const GISelInstProfileBuilder &addNodeIDRegType(const RegisterBank *RB) const;
+
+  const GISelInstProfileBuilder &addNodeIDRegNum(unsigned Reg) const;
+
+  const GISelInstProfileBuilder &addNodeIDImmediate(int64_t Imm) const;
+  const GISelInstProfileBuilder &
+  addNodeIDMBB(const MachineBasicBlock *MBB) const;
+
+  const GISelInstProfileBuilder &
+  addNodeIDMachineOperand(const MachineOperand &MO) const;
+
+  const GISelInstProfileBuilder &addNodeIDFlag(unsigned Flag) const;
+  const GISelInstProfileBuilder &addNodeID(const MachineInstr *MI) const;
+};
+
+/// Simple wrapper that does the following.
+/// 1) Lazily evaluate the MachineFunction to compute CSEable instructions.
+/// 2) Allows configuration of which instructions are CSEd through CSEConfig
+/// object. Provides a method called get which takes a CSEConfig object.
+class GISelCSEAnalysisWrapper {
+  GISelCSEInfo Info;
+  MachineFunction *MF = nullptr;
+  bool AlreadyComputed = false;
+
+public:
+  /// Takes a CSEConfig object that defines what opcodes get CSEd.
+  /// If CSEConfig is already set, and the CSE Analysis has been preserved,
+  /// it will not use the new CSEOpt(use Recompute to force using the new
+  /// CSEOpt).
+  GISelCSEInfo &get(std::unique_ptr<CSEConfig> CSEOpt, bool ReCompute = false);
+  void setMF(MachineFunction &MFunc) { MF = &MFunc; }
+  void setComputed(bool Computed) { AlreadyComputed = Computed; }
+  void releaseMemory() { Info.releaseMemory(); }
+};
+
+/// The actual analysis pass wrapper.
+class GISelCSEAnalysisWrapperPass : public MachineFunctionPass {
+  GISelCSEAnalysisWrapper Wrapper;
+
+public:
+  static char ID;
+  GISelCSEAnalysisWrapperPass() : MachineFunctionPass(ID) {
+    initializeGISelCSEAnalysisWrapperPassPass(*PassRegistry::getPassRegistry());
+  }
+
+  void getAnalysisUsage(AnalysisUsage &AU) const override;
+
+  const GISelCSEAnalysisWrapper &getCSEWrapper() const { return Wrapper; }
+  GISelCSEAnalysisWrapper &getCSEWrapper() { return Wrapper; }
+
+  bool runOnMachineFunction(MachineFunction &MF) override;
+
+  void releaseMemory() override {
+    Wrapper.releaseMemory();
+    Wrapper.setComputed(false);
+  }
+};
+
+} // namespace llvm
+
+#endif
diff --git a/include/llvm/CodeGen/GlobalISel/CSEMIRBuilder.h b/include/llvm/CodeGen/GlobalISel/CSEMIRBuilder.h
new file mode 100644
index 0000000000000000000000000000000000000000..4f95335db74b2ab71f64149854b4a55a32e87343
--- /dev/null
+++ b/include/llvm/CodeGen/GlobalISel/CSEMIRBuilder.h
@@ -0,0 +1,109 @@
+//===-- llvm/CodeGen/GlobalISel/CSEMIRBuilder.h  --*- C++ -*-==//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+/// \file
+/// This file implements a version of MachineIRBuilder which CSEs insts within
+/// a MachineBasicBlock.
+//===----------------------------------------------------------------------===//
+#ifndef LLVM_CODEGEN_GLOBALISEL_CSEMIRBUILDER_H
+#define LLVM_CODEGEN_GLOBALISEL_CSEMIRBUILDER_H
+
+#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
+#include "llvm/CodeGen/GlobalISel/Utils.h"
+
+namespace llvm {
+
+/// Defines a builder that does CSE of MachineInstructions using GISelCSEInfo.
+/// Eg usage.
+///
+///
+/// GISelCSEInfo *Info =
+/// &getAnalysis<GISelCSEAnalysisWrapperPass>().getCSEInfo(); CSEMIRBuilder
+/// CB(Builder.getState()); CB.setCSEInfo(Info); auto A = CB.buildConstant(s32,
+/// 42); auto B = CB.buildConstant(s32, 42); assert(A == B); unsigned CReg =
+/// MRI.createGenericVirtualRegister(s32); auto C = CB.buildConstant(CReg, 42);
+/// assert(C->getOpcode() == TargetOpcode::COPY);
+/// Explicitly passing in a register would materialize a copy if possible.
+/// CSEMIRBuilder also does trivial constant folding for binary ops.
+class CSEMIRBuilder : public MachineIRBuilder {
+
+  /// Returns true if A dominates B (within the same basic block).
+  /// Both iterators must be in the same basic block.
+  //
+  // TODO: Another approach for checking dominance is having two iterators and
+  // making them go towards each other until they meet or reach begin/end. Which
+  // approach is better? Should this even change dynamically? For G_CONSTANTS
+  // most of which will be at the top of the BB, the top down approach would be
+  // a better choice. Does IRTranslator placing constants at the beginning still
+  // make sense? Should this change based on Opcode?
+  bool dominates(MachineBasicBlock::const_iterator A,
+                 MachineBasicBlock::const_iterator B) const;
+
+  /// For given ID, find a machineinstr in the CSE Map. If found, check if it
+  /// dominates the current insertion point and if not, move it just before the
+  /// current insertion point and return it. If not found, return Null
+  /// MachineInstrBuilder.
+  MachineInstrBuilder getDominatingInstrForID(FoldingSetNodeID &ID,
+                                              void *&NodeInsertPos);
+  /// Simple check if we can CSE (we have the CSEInfo) or if this Opcode is
+  /// safe to CSE.
+  bool canPerformCSEForOpc(unsigned Opc) const;
+
+  void profileDstOp(const DstOp &Op, GISelInstProfileBuilder &B) const;
+
+  void profileDstOps(ArrayRef<DstOp> Ops, GISelInstProfileBuilder &B) const {
+    for (const DstOp &Op : Ops)
+      profileDstOp(Op, B);
+  }
+
+  void profileSrcOp(const SrcOp &Op, GISelInstProfileBuilder &B) const;
+
+  void profileSrcOps(ArrayRef<SrcOp> Ops, GISelInstProfileBuilder &B) const {
+    for (const SrcOp &Op : Ops)
+      profileSrcOp(Op, B);
+  }
+
+  void profileMBBOpcode(GISelInstProfileBuilder &B, unsigned Opc) const;
+
+  void profileEverything(unsigned Opc, ArrayRef<DstOp> DstOps,
+                         ArrayRef<SrcOp> SrcOps, Optional<unsigned> Flags,
+                         GISelInstProfileBuilder &B) const;
+
+  // Takes a MachineInstrBuilder and inserts it into the CSEMap using the
+  // NodeInsertPos.
+  MachineInstrBuilder memoizeMI(MachineInstrBuilder MIB, void *NodeInsertPos);
+
+  // If we have can CSE an instruction, but still need to materialize to a VReg,
+  // we emit a copy from the CSE'd inst to the VReg.
+  MachineInstrBuilder generateCopiesIfRequired(ArrayRef<DstOp> DstOps,
+                                               MachineInstrBuilder &MIB);
+
+  // If we have can CSE an instruction, but still need to materialize to a VReg,
+  // check if we can generate copies. It's not possible to return a single MIB,
+  // while emitting copies to multiple vregs.
+  bool checkCopyToDefsPossible(ArrayRef<DstOp> DstOps);
+
+public:
+  // Pull in base class constructors.
+  using MachineIRBuilder::MachineIRBuilder;
+  // Unhide buildInstr
+  MachineInstrBuilder buildInstr(unsigned Opc, ArrayRef<DstOp> DstOps,
+                                 ArrayRef<SrcOp> SrcOps,
+                                 Optional<unsigned> Flag = None) override;
+  // Bring in the other overload from the base class.
+  using MachineIRBuilder::buildConstant;
+
+  MachineInstrBuilder buildConstant(const DstOp &Res,
+                                    const ConstantInt &Val) override;
+
+  // Bring in the other overload from the base class.
+  using MachineIRBuilder::buildFConstant;
+  MachineInstrBuilder buildFConstant(const DstOp &Res,
+                                     const ConstantFP &Val) override;
+};
+} // namespace llvm
+#endif
diff --git a/include/llvm/CodeGen/GlobalISel/CallLowering.h b/include/llvm/CodeGen/GlobalISel/CallLowering.h
index 32980ce149c1c15098c12f35b98d7b499a0155e6..9b72b704a2e4a97e885a9005a09f3bb10a8eefa9 100644
--- a/include/llvm/CodeGen/GlobalISel/CallLowering.h
+++ b/include/llvm/CodeGen/GlobalISel/CallLowering.h
@@ -1,9 +1,8 @@
 //===- llvm/CodeGen/GlobalISel/CallLowering.h - Call lowering ---*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
@@ -40,6 +39,7 @@ class Value;
 class CallLowering {
   const TargetLowering *TLI;
 
+  virtual void anchor();
 public:
   struct ArgInfo {
     unsigned Reg;
@@ -108,6 +108,9 @@ public:
     MachineIRBuilder &MIRBuilder;
     MachineRegisterInfo &MRI;
     CCAssignFn *AssignFn;
+
+  private:
+    virtual void anchor();
   };
 
 protected:
diff --git a/include/llvm/CodeGen/GlobalISel/Combiner.h b/include/llvm/CodeGen/GlobalISel/Combiner.h
index 36a33deb4a642a6338fdee468191f1eca7f0737f..12a1f97077c94e3af1bb57a0d5b15de5192cbc69 100644
--- a/include/llvm/CodeGen/GlobalISel/Combiner.h
+++ b/include/llvm/CodeGen/GlobalISel/Combiner.h
@@ -1,9 +1,8 @@
 //== ----- llvm/CodeGen/GlobalISel/Combiner.h --------------------- == //
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -21,6 +20,7 @@
 namespace llvm {
 class MachineRegisterInfo;
 class CombinerInfo;
+class GISelCSEInfo;
 class TargetPassConfig;
 class MachineFunction;
 
@@ -28,14 +28,17 @@ class Combiner {
 public:
   Combiner(CombinerInfo &CombinerInfo, const TargetPassConfig *TPC);
 
-  bool combineMachineInstrs(MachineFunction &MF);
+  /// If CSEInfo is not null, then the Combiner will setup observer for
+  /// CSEInfo and instantiate a CSEMIRBuilder. Pass nullptr if CSE is not
+  /// needed.
+  bool combineMachineInstrs(MachineFunction &MF, GISelCSEInfo *CSEInfo);
 
 protected:
   CombinerInfo &CInfo;
 
   MachineRegisterInfo *MRI = nullptr;
   const TargetPassConfig *TPC;
-  MachineIRBuilder Builder;
+  std::unique_ptr<MachineIRBuilder> Builder;
 };
 
 } // End namespace llvm.
diff --git a/include/llvm/CodeGen/GlobalISel/CombinerHelper.h b/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
index 6e9ac01c1ee291a1dd6cff05c0b12f903d425f69..ee30ba988521382ec6191958509c1b92b59852ca 100644
--- a/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
+++ b/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
@@ -1,9 +1,8 @@
 //===-- llvm/CodeGen/GlobalISel/CombinerHelper.h --------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===--------------------------------------------------------------------===//
 //
diff --git a/include/llvm/CodeGen/GlobalISel/CombinerInfo.h b/include/llvm/CodeGen/GlobalISel/CombinerInfo.h
index d21aa3f725d92e77ad472fd91c9a1370749d1d2c..3b09a8e2b479fafd666eb39cbaf8740727f802ba 100644
--- a/include/llvm/CodeGen/GlobalISel/CombinerInfo.h
+++ b/include/llvm/CodeGen/GlobalISel/CombinerInfo.h
@@ -1,9 +1,8 @@
 //===- llvm/CodeGen/GlobalISel/CombinerInfo.h ------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/CodeGen/GlobalISel/ConstantFoldingMIRBuilder.h b/include/llvm/CodeGen/GlobalISel/ConstantFoldingMIRBuilder.h
index 118c65ed39632edb9170c73d67bf7988348d9086..e817d9b4550e58cd9107db61113f177cc856e063 100644
--- a/include/llvm/CodeGen/GlobalISel/ConstantFoldingMIRBuilder.h
+++ b/include/llvm/CodeGen/GlobalISel/ConstantFoldingMIRBuilder.h
@@ -1,9 +1,8 @@
 //===-- llvm/CodeGen/GlobalISel/ConstantFoldingMIRBuilder.h  --*- C++ -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
@@ -15,57 +14,6 @@
 
 namespace llvm {
 
-static Optional<APInt> ConstantFoldBinOp(unsigned Opcode, const unsigned Op1,
-                                         const unsigned Op2,
-                                         const MachineRegisterInfo &MRI) {
-  auto MaybeOp1Cst = getConstantVRegVal(Op1, MRI);
-  auto MaybeOp2Cst = getConstantVRegVal(Op2, MRI);
-  if (MaybeOp1Cst && MaybeOp2Cst) {
-    LLT Ty = MRI.getType(Op1);
-    APInt C1(Ty.getSizeInBits(), *MaybeOp1Cst, true);
-    APInt C2(Ty.getSizeInBits(), *MaybeOp2Cst, true);
-    switch (Opcode) {
-    default:
-      break;
-    case TargetOpcode::G_ADD:
-      return C1 + C2;
-    case TargetOpcode::G_AND:
-      return C1 & C2;
-    case TargetOpcode::G_ASHR:
-      return C1.ashr(C2);
-    case TargetOpcode::G_LSHR:
-      return C1.lshr(C2);
-    case TargetOpcode::G_MUL:
-      return C1 * C2;
-    case TargetOpcode::G_OR:
-      return C1 | C2;
-    case TargetOpcode::G_SHL:
-      return C1 << C2;
-    case TargetOpcode::G_SUB:
-      return C1 - C2;
-    case TargetOpcode::G_XOR:
-      return C1 ^ C2;
-    case TargetOpcode::G_UDIV:
-      if (!C2.getBoolValue())
-        break;
-      return C1.udiv(C2);
-    case TargetOpcode::G_SDIV:
-      if (!C2.getBoolValue())
-        break;
-      return C1.sdiv(C2);
-    case TargetOpcode::G_UREM:
-      if (!C2.getBoolValue())
-        break;
-      return C1.urem(C2);
-    case TargetOpcode::G_SREM:
-      if (!C2.getBoolValue())
-        break;
-      return C1.srem(C2);
-    }
-  }
-  return None;
-}
-
 /// An MIRBuilder which does trivial constant folding of binary ops.
 /// Calls to buildInstr will also try to constant fold binary ops.
 class ConstantFoldingMIRBuilder : public MachineIRBuilder {
diff --git a/include/llvm/CodeGen/GlobalISel/GISelChangeObserver.h b/include/llvm/CodeGen/GlobalISel/GISelChangeObserver.h
index d21c73309a5202f9ad3b4be9afa95e039bc4940c..e5691cb35174a347293741a76d1e77b4b9e8082f 100644
--- a/include/llvm/CodeGen/GlobalISel/GISelChangeObserver.h
+++ b/include/llvm/CodeGen/GlobalISel/GISelChangeObserver.h
@@ -1,9 +1,8 @@
-//===----- llvm/CodeGen/GlobalISel/GISelChangeObserver.h ------------------===//
+//===----- llvm/CodeGen/GlobalISel/GISelChangeObserver.h --------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -15,6 +14,7 @@
 #define LLVM_CODEGEN_GLOBALISEL_GISELCHANGEOBSERVER_H
 
 #include "llvm/ADT/SmallPtrSet.h"
+#include "llvm/CodeGen/MachineFunction.h"
 
 namespace llvm {
 class MachineInstr;
@@ -32,13 +32,20 @@ public:
   virtual ~GISelChangeObserver() {}
 
   /// An instruction is about to be erased.
-  virtual void erasingInstr(const MachineInstr &MI) = 0;
-  /// An instruction was created and inserted into the function.
-  virtual void createdInstr(const MachineInstr &MI) = 0;
+  virtual void erasingInstr(MachineInstr &MI) = 0;
+
+  /// An instruction has been created and inserted into the function.
+  /// Note that the instruction might not be a fully fledged instruction at this
+  /// point and won't be if the MachineFunction::Delegate is calling it. This is
+  /// because the delegate only sees the construction of the MachineInstr before
+  /// operands have been added.
+  virtual void createdInstr(MachineInstr &MI) = 0;
+
   /// This instruction is about to be mutated in some way.
-  virtual void changingInstr(const MachineInstr &MI) = 0;
+  virtual void changingInstr(MachineInstr &MI) = 0;
+
   /// This instruction was mutated in some way.
-  virtual void changedInstr(const MachineInstr &MI) = 0;
+  virtual void changedInstr(MachineInstr &MI) = 0;
 
   /// All the instructions using the given register are being changed.
   /// For convenience, finishedChangingAllUsesOfReg() will report the completion
@@ -51,5 +58,60 @@ public:
 
 };
 
+/// Simple wrapper observer that takes several observers, and calls
+/// each one for each event. If there are multiple observers (say CSE,
+/// Legalizer, Combiner), it's sufficient to register this to the machine
+/// function as the delegate.
+class GISelObserverWrapper : public MachineFunction::Delegate,
+                             public GISelChangeObserver {
+  SmallVector<GISelChangeObserver *, 4> Observers;
+
+public:
+  GISelObserverWrapper() = default;
+  GISelObserverWrapper(ArrayRef<GISelChangeObserver *> Obs)
+      : Observers(Obs.begin(), Obs.end()) {}
+  // Adds an observer.
+  void addObserver(GISelChangeObserver *O) { Observers.push_back(O); }
+  // Removes an observer from the list and does nothing if observer is not
+  // present.
+  void removeObserver(GISelChangeObserver *O) {
+    auto It = std::find(Observers.begin(), Observers.end(), O);
+    if (It != Observers.end())
+      Observers.erase(It);
+  }
+  // API for Observer.
+  void erasingInstr(MachineInstr &MI) override {
+    for (auto &O : Observers)
+      O->erasingInstr(MI);
+  }
+  void createdInstr(MachineInstr &MI) override {
+    for (auto &O : Observers)
+      O->createdInstr(MI);
+  }
+  void changingInstr(MachineInstr &MI) override {
+    for (auto &O : Observers)
+      O->changingInstr(MI);
+  }
+  void changedInstr(MachineInstr &MI) override {
+    for (auto &O : Observers)
+      O->changedInstr(MI);
+  }
+  // API for MachineFunction::Delegate
+  void MF_HandleInsertion(MachineInstr &MI) override { createdInstr(MI); }
+  void MF_HandleRemoval(MachineInstr &MI) override { erasingInstr(MI); }
+};
+
+/// A simple RAII based CSEInfo installer.
+/// Use this in a scope to install a delegate to the MachineFunction and reset
+/// it at the end of the scope.
+class RAIIDelegateInstaller {
+  MachineFunction &MF;
+  MachineFunction::Delegate *Delegate;
+
+public:
+  RAIIDelegateInstaller(MachineFunction &MF, MachineFunction::Delegate *Del);
+  ~RAIIDelegateInstaller();
+};
+
 } // namespace llvm
 #endif
diff --git a/include/llvm/CodeGen/GlobalISel/GISelWorkList.h b/include/llvm/CodeGen/GlobalISel/GISelWorkList.h
index b32c5af537ccd6578a7e3083ce90f5900ad674c6..626a66620fa1e9316af3eb5ea2e85b0e9b60d96c 100644
--- a/include/llvm/CodeGen/GlobalISel/GISelWorkList.h
+++ b/include/llvm/CodeGen/GlobalISel/GISelWorkList.h
@@ -1,9 +1,8 @@
 //===- GISelWorkList.h - Worklist for GISel passes ----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -18,6 +17,7 @@
 
 namespace llvm {
 
+class MachineInstr;
 class MachineFunction;
 
 // Worklist which mostly works similar to InstCombineWorkList, but on
@@ -25,23 +25,15 @@ class MachineFunction;
 // erasing an element doesn't move all elements over one place - instead just
 // nulls out the element of the vector.
 //
-// This worklist operates on instructions within a particular function. This is
-// important for acquiring the rights to modify/replace instructions a
-// GISelChangeObserver reports as the observer doesn't have the right to make
-// changes to the instructions it sees so we use our access to the
-// MachineFunction to establish that it's ok to add a given instruction to the
-// worklist.
-//
 // FIXME: Does it make sense to factor out common code with the
 // instcombinerWorkList?
 template<unsigned N>
 class GISelWorkList {
-  MachineFunction *MF;
   SmallVector<MachineInstr *, N> Worklist;
   DenseMap<MachineInstr *, unsigned> WorklistMap;
 
 public:
-  GISelWorkList(MachineFunction *MF) : MF(MF) {}
+  GISelWorkList() : WorklistMap(N) {}
 
   bool empty() const { return WorklistMap.empty(); }
 
@@ -49,27 +41,8 @@ public:
 
   /// Add the specified instruction to the worklist if it isn't already in it.
   void insert(MachineInstr *I) {
-    // It would be safe to add this instruction to the worklist regardless but
-    // for consistency with the const version, check that the instruction we're
-    // adding would have been accepted if we were given a const pointer instead.
-    insert(const_cast<const MachineInstr *>(I));
-  }
-
-  void insert(const MachineInstr *I) {
-    // Confirm we'd be able to find the non-const pointer we want to schedule if
-    // we wanted to. We have the right to schedule work that may modify any
-    // instruction in MF.
-    assert(I->getParent() && "Expected parent BB");
-    assert(I->getParent()->getParent() && "Expected parent function");
-    assert((!MF || I->getParent()->getParent() == MF) &&
-           "Expected parent function to be current function or not given");
-
-    // But don't actually do the search since we can derive it from the const
-    // pointer.
-    MachineInstr *NonConstI = const_cast<MachineInstr *>(I);
-    if (WorklistMap.try_emplace(NonConstI, Worklist.size()).second) {
-      Worklist.push_back(NonConstI);
-    }
+    if (WorklistMap.try_emplace(I, Worklist.size()).second)
+      Worklist.push_back(I);
   }
 
   /// Remove I from the worklist if it exists.
@@ -83,6 +56,11 @@ public:
     WorklistMap.erase(It);
   }
 
+  void clear() {
+    Worklist.clear();
+    WorklistMap.clear();
+  }
+
   MachineInstr *pop_back_val() {
     MachineInstr *I;
     do {
diff --git a/include/llvm/CodeGen/GlobalISel/IRTranslator.h b/include/llvm/CodeGen/GlobalISel/IRTranslator.h
index 04629f6b320464f99bcd6be503e56dbc50e6ef59..c75d823501acd7732b56f8c506c9ed3ba1614d2b 100644
--- a/include/llvm/CodeGen/GlobalISel/IRTranslator.h
+++ b/include/llvm/CodeGen/GlobalISel/IRTranslator.h
@@ -1,9 +1,8 @@
 //===- llvm/CodeGen/GlobalISel/IRTranslator.h - IRTranslator ----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
@@ -21,11 +20,11 @@
 
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/SmallVector.h"
-#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
+#include "llvm/CodeGen/GlobalISel/CSEMIRBuilder.h"
 #include "llvm/CodeGen/GlobalISel/Types.h"
 #include "llvm/CodeGen/MachineFunctionPass.h"
-#include "llvm/Support/Allocator.h"
 #include "llvm/IR/Intrinsics.h"
+#include "llvm/Support/Allocator.h"
 #include <memory>
 #include <utility>
 
@@ -217,6 +216,17 @@ private:
   bool translateOverflowIntrinsic(const CallInst &CI, unsigned Op,
                                   MachineIRBuilder &MIRBuilder);
 
+  /// Helper function for translateSimpleIntrinsic.
+  /// \return The generic opcode for \p IntrinsicID if \p IntrinsicID is a
+  /// simple intrinsic (ceil, fabs, etc.). Otherwise, returns
+  /// Intrinsic::not_intrinsic.
+  unsigned getSimpleIntrinsicOpcode(Intrinsic::ID ID);
+
+  /// Translates the intrinsics defined in getSimpleIntrinsicOpcode.
+  /// \return true if the translation succeeded.
+  bool translateSimpleIntrinsic(const CallInst &CI, Intrinsic::ID ID,
+                                MachineIRBuilder &MIRBuilder);
+
   bool translateKnownIntrinsic(const CallInst &CI, Intrinsic::ID ID,
                                MachineIRBuilder &MIRBuilder);
 
@@ -242,6 +252,8 @@ private:
 
   bool translateInvoke(const User &U, MachineIRBuilder &MIRBuilder);
 
+  bool translateCallBr(const User &U, MachineIRBuilder &MIRBuilder);
+
   bool translateLandingPad(const User &U, MachineIRBuilder &MIRBuilder);
 
   /// Translate one of LLVM's cast instructions into MachineInstrs, with the
@@ -444,11 +456,13 @@ private:
   // I.e., compared to regular MIBuilder, this one also inserts the instruction
   // in the current block, it can creates block, etc., basically a kind of
   // IRBuilder, but for Machine IR.
-  MachineIRBuilder CurBuilder;
+  // CSEMIRBuilder CurBuilder;
+  std::unique_ptr<MachineIRBuilder> CurBuilder;
 
   // Builder set to the entry block (just after ABI lowering instructions). Used
   // as a convenient location for Constants.
-  MachineIRBuilder EntryBuilder;
+  // CSEMIRBuilder EntryBuilder;
+  std::unique_ptr<MachineIRBuilder> EntryBuilder;
 
   // The MachineFunction currently being translated.
   MachineFunction *MF;
diff --git a/include/llvm/CodeGen/GlobalISel/InstructionSelect.h b/include/llvm/CodeGen/GlobalISel/InstructionSelect.h
index 01521c46ab6aa8ffcb1e04828378fd0624a5975c..1af46e0a9e76bfaf91a7ac6b977934726e302e3f 100644
--- a/include/llvm/CodeGen/GlobalISel/InstructionSelect.h
+++ b/include/llvm/CodeGen/GlobalISel/InstructionSelect.h
@@ -1,9 +1,8 @@
 //== llvm/CodeGen/GlobalISel/InstructionSelect.h -----------------*- C++ -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file This file describes the interface of the MachineFunctionPass
diff --git a/include/llvm/CodeGen/GlobalISel/InstructionSelector.h b/include/llvm/CodeGen/GlobalISel/InstructionSelector.h
index 471def7f45a3d1f73d6ba037dc5b51bafdd4a574..e4d05a57bd36557c008d37a5320078259ae060b2 100644
--- a/include/llvm/CodeGen/GlobalISel/InstructionSelector.h
+++ b/include/llvm/CodeGen/GlobalISel/InstructionSelector.h
@@ -1,9 +1,8 @@
 //===- llvm/CodeGen/GlobalISel/InstructionSelector.h ------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h b/include/llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h
index 2003a79f6b20618772c435ef344b6edd79a50291..87ef2f8452920a11648ae4a73f4e9cac1d96d756 100644
--- a/include/llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h
+++ b/include/llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h
@@ -1,9 +1,8 @@
 //===- llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h --------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/CodeGen/GlobalISel/LegalizationArtifactCombiner.h b/include/llvm/CodeGen/GlobalISel/LegalizationArtifactCombiner.h
index 20bec76501796c68b9848bb2bfaddea727ad0d2b..e7680e1bc126c10655df2944ed4c428b025ea118 100644
--- a/include/llvm/CodeGen/GlobalISel/LegalizationArtifactCombiner.h
+++ b/include/llvm/CodeGen/GlobalISel/LegalizationArtifactCombiner.h
@@ -1,9 +1,8 @@
 //===-- llvm/CodeGen/GlobalISel/LegalizationArtifactCombiner.h -----*- C++ -*-//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 // This file contains some helper functions which try to cleanup artifacts
@@ -81,11 +80,11 @@ public:
     if (mi_match(SrcReg, MRI, m_GTrunc(m_Reg(TruncSrc)))) {
       LLT DstTy = MRI.getType(DstReg);
       if (isInstUnsupported({TargetOpcode::G_AND, {DstTy}}) ||
-          isInstUnsupported({TargetOpcode::G_CONSTANT, {DstTy}}))
+          isConstantUnsupported(DstTy))
         return false;
       LLVM_DEBUG(dbgs() << ".. Combine MI: " << MI;);
       LLT SrcTy = MRI.getType(SrcReg);
-      APInt Mask = APInt::getAllOnesValue(SrcTy.getSizeInBits());
+      APInt Mask = APInt::getAllOnesValue(SrcTy.getScalarSizeInBits());
       auto MIBMask = Builder.buildConstant(DstTy, Mask.getZExtValue());
       Builder.buildAnd(DstReg, Builder.buildAnyExtOrTrunc(DstTy, TruncSrc),
                        MIBMask);
@@ -109,13 +108,15 @@ public:
     unsigned TruncSrc;
     if (mi_match(SrcReg, MRI, m_GTrunc(m_Reg(TruncSrc)))) {
       LLT DstTy = MRI.getType(DstReg);
-      if (isInstUnsupported({TargetOpcode::G_SHL, {DstTy}}) ||
-          isInstUnsupported({TargetOpcode::G_ASHR, {DstTy}}) ||
-          isInstUnsupported({TargetOpcode::G_CONSTANT, {DstTy}}))
+      // Guess on the RHS shift amount type, which should be re-legalized if
+      // applicable.
+      if (isInstUnsupported({TargetOpcode::G_SHL, {DstTy, DstTy}}) ||
+          isInstUnsupported({TargetOpcode::G_ASHR, {DstTy, DstTy}}) ||
+          isConstantUnsupported(DstTy))
         return false;
       LLVM_DEBUG(dbgs() << ".. Combine MI: " << MI;);
       LLT SrcTy = MRI.getType(SrcReg);
-      unsigned ShAmt = DstTy.getSizeInBits() - SrcTy.getSizeInBits();
+      unsigned ShAmt = DstTy.getScalarSizeInBits() - SrcTy.getScalarSizeInBits();
       auto MIBShAmt = Builder.buildConstant(DstTy, ShAmt);
       auto MIBShl = Builder.buildInstr(
           TargetOpcode::G_SHL, {DstTy},
@@ -150,7 +151,7 @@ public:
       } else {
         // G_[SZ]EXT (G_IMPLICIT_DEF) -> G_CONSTANT 0 because the top
         // bits will be 0 for G_ZEXT and 0/1 for the G_SEXT.
-        if (isInstUnsupported({TargetOpcode::G_CONSTANT, {DstTy}}))
+        if (isConstantUnsupported(DstTy))
           return false;
         LLVM_DEBUG(dbgs() << ".. Combine G_[SZ]EXT(G_IMPLICIT_DEF): " << MI;);
         Builder.buildConstant(DstReg, 0);
@@ -162,6 +163,16 @@ public:
     return false;
   }
 
+  static unsigned getMergeOpcode(LLT OpTy, LLT DestTy) {
+    if (OpTy.isVector() && DestTy.isVector())
+      return TargetOpcode::G_CONCAT_VECTORS;
+
+    if (OpTy.isVector() && !DestTy.isVector())
+      return TargetOpcode::G_BUILD_VECTOR;
+
+    return TargetOpcode::G_MERGE_VALUES;
+  }
+
   bool tryCombineMerges(MachineInstr &MI,
                         SmallVectorImpl<MachineInstr *> &DeadInsts) {
 
@@ -170,16 +181,10 @@ public:
 
     unsigned NumDefs = MI.getNumOperands() - 1;
 
-    unsigned MergingOpcode;
     LLT OpTy = MRI.getType(MI.getOperand(NumDefs).getReg());
     LLT DestTy = MRI.getType(MI.getOperand(0).getReg());
-    if (OpTy.isVector() && DestTy.isVector())
-      MergingOpcode = TargetOpcode::G_CONCAT_VECTORS;
-    else if (OpTy.isVector() && !DestTy.isVector())
-      MergingOpcode = TargetOpcode::G_BUILD_VECTOR;
-    else
-      MergingOpcode = TargetOpcode::G_MERGE_VALUES;
 
+    unsigned MergingOpcode = getMergeOpcode(OpTy, DestTy);
     MachineInstr *MergeI =
         getOpcodeDef(MergingOpcode, MI.getOperand(NumDefs).getReg(), MRI);
 
@@ -248,6 +253,65 @@ public:
     return true;
   }
 
+  static bool isMergeLikeOpcode(unsigned Opc) {
+    switch (Opc) {
+    case TargetOpcode::G_MERGE_VALUES:
+    case TargetOpcode::G_BUILD_VECTOR:
+    case TargetOpcode::G_CONCAT_VECTORS:
+      return true;
+    default:
+      return false;
+    }
+  }
+
+  bool tryCombineExtract(MachineInstr &MI,
+                         SmallVectorImpl<MachineInstr *> &DeadInsts) {
+    assert(MI.getOpcode() == TargetOpcode::G_EXTRACT);
+
+    // Try to use the source registers from a G_MERGE_VALUES
+    //
+    // %2 = G_MERGE_VALUES %0, %1
+    // %3 = G_EXTRACT %2, N
+    // =>
+    //
+    // for N < %2.getSizeInBits() / 2
+    //     %3 = G_EXTRACT %0, N
+    //
+    // for N >= %2.getSizeInBits() / 2
+    //    %3 = G_EXTRACT %1, (N - %0.getSizeInBits()
+
+    unsigned Src = lookThroughCopyInstrs(MI.getOperand(1).getReg());
+    MachineInstr *MergeI = MRI.getVRegDef(Src);
+    if (!MergeI || !isMergeLikeOpcode(MergeI->getOpcode()))
+      return false;
+
+    LLT DstTy = MRI.getType(MI.getOperand(0).getReg());
+    LLT SrcTy = MRI.getType(Src);
+
+    // TODO: Do we need to check if the resulting extract is supported?
+    unsigned ExtractDstSize = DstTy.getSizeInBits();
+    unsigned Offset = MI.getOperand(2).getImm();
+    unsigned NumMergeSrcs = MergeI->getNumOperands() - 1;
+    unsigned MergeSrcSize = SrcTy.getSizeInBits() / NumMergeSrcs;
+    unsigned MergeSrcIdx = Offset / MergeSrcSize;
+
+    // Compute the offset of the last bit the extract needs.
+    unsigned EndMergeSrcIdx = (Offset + ExtractDstSize - 1) / MergeSrcSize;
+
+    // Can't handle the case where the extract spans multiple inputs.
+    if (MergeSrcIdx != EndMergeSrcIdx)
+      return false;
+
+    // TODO: We could modify MI in place in most cases.
+    Builder.setInstr(MI);
+    Builder.buildExtract(
+      MI.getOperand(0).getReg(),
+      MergeI->getOperand(MergeSrcIdx + 1).getReg(),
+      Offset - MergeSrcIdx * MergeSrcSize);
+    markInstAndDefDead(MI, *MergeI, DeadInsts);
+    return true;
+  }
+
   /// Try to combine away MI.
   /// Returns true if it combined away the MI.
   /// Adds instructions that are dead as a result of the combine
@@ -265,6 +329,8 @@ public:
       return tryCombineSExt(MI, DeadInsts);
     case TargetOpcode::G_UNMERGE_VALUES:
       return tryCombineMerges(MI, DeadInsts);
+    case TargetOpcode::G_EXTRACT:
+      return tryCombineExtract(MI, DeadInsts);
     case TargetOpcode::G_TRUNC: {
       bool Changed = false;
       for (auto &Use : MRI.use_instructions(MI.getOperand(0).getReg()))
@@ -275,6 +341,23 @@ public:
   }
 
 private:
+
+  static unsigned getArtifactSrcReg(const MachineInstr &MI) {
+    switch (MI.getOpcode()) {
+    case TargetOpcode::COPY:
+    case TargetOpcode::G_TRUNC:
+    case TargetOpcode::G_ZEXT:
+    case TargetOpcode::G_ANYEXT:
+    case TargetOpcode::G_SEXT:
+    case TargetOpcode::G_UNMERGE_VALUES:
+      return MI.getOperand(MI.getNumOperands() - 1).getReg();
+    case TargetOpcode::G_EXTRACT:
+      return MI.getOperand(1).getReg();
+    default:
+      llvm_unreachable("Not a legalization artifact happen");
+    }
+  }
+
   /// Mark MI as dead. If a def of one of MI's operands, DefMI, would also be
   /// dead due to MI being killed, then mark DefMI as dead too.
   /// Some of the combines (extends(trunc)), try to walk through redundant
@@ -295,8 +378,8 @@ private:
     // and as a result, %3, %2, %1 are dead.
     MachineInstr *PrevMI = &MI;
     while (PrevMI != &DefMI) {
-      unsigned PrevRegSrc =
-          PrevMI->getOperand(PrevMI->getNumOperands() - 1).getReg();
+      unsigned PrevRegSrc = getArtifactSrcReg(*PrevMI);
+
       MachineInstr *TmpDef = MRI.getVRegDef(PrevRegSrc);
       if (MRI.hasOneUse(PrevRegSrc)) {
         if (TmpDef != &DefMI) {
@@ -320,6 +403,15 @@ private:
     return Step.Action == Unsupported || Step.Action == NotFound;
   }
 
+  bool isConstantUnsupported(LLT Ty) const {
+    if (!Ty.isVector())
+      return isInstUnsupported({TargetOpcode::G_CONSTANT, {Ty}});
+
+    LLT EltTy = Ty.getElementType();
+    return isInstUnsupported({TargetOpcode::G_CONSTANT, {EltTy}}) ||
+           isInstUnsupported({TargetOpcode::G_BUILD_VECTOR, {Ty, EltTy}});
+  }
+
   /// Looks through copy instructions and returns the actual
   /// source register.
   unsigned lookThroughCopyInstrs(unsigned Reg) {
diff --git a/include/llvm/CodeGen/GlobalISel/Legalizer.h b/include/llvm/CodeGen/GlobalISel/Legalizer.h
index 8284ab6dac65e083241287fae13e74aeb8d9ad20..01816509d4d65d3a9d02998e7fc4eddc375af2c0 100644
--- a/include/llvm/CodeGen/GlobalISel/Legalizer.h
+++ b/include/llvm/CodeGen/GlobalISel/Legalizer.h
@@ -1,9 +1,8 @@
-//== llvm/CodeGen/GlobalISel/LegalizePass.h ------------- -*- C++ -*-==//
+//== llvm/CodeGen/GlobalISel/Legalizer.h ---------------- -*- C++ -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h b/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h
index 4d3e4a3763865c86f1d64b485a8d3e7d72474726..136356cc109792a7c15034d35fabea31d72e6f58 100644
--- a/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h
+++ b/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h
@@ -1,9 +1,8 @@
 //== llvm/CodeGen/GlobalISel/LegalizerHelper.h ---------------- -*- C++ -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -49,9 +48,10 @@ public:
     UnableToLegalize,
   };
 
-  LegalizerHelper(MachineFunction &MF, GISelChangeObserver &Observer);
+  LegalizerHelper(MachineFunction &MF, GISelChangeObserver &Observer,
+                  MachineIRBuilder &B);
   LegalizerHelper(MachineFunction &MF, const LegalizerInfo &LI,
-                  GISelChangeObserver &Observer);
+                  GISelChangeObserver &Observer, MachineIRBuilder &B);
 
   /// Replace \p MI by a sequence of legal instructions that can implement the
   /// same operation. Note that this means \p MI may be deleted, so any iterator
@@ -86,11 +86,11 @@ public:
   /// Legalize a vector instruction by increasing the number of vector elements
   /// involved and ignoring the added elements later.
   LegalizeResult moreElementsVector(MachineInstr &MI, unsigned TypeIdx,
-                                    LLT WideTy);
+                                    LLT MoreTy);
 
   /// Expose MIRBuilder so clients can set their own RecordInsertInstruction
   /// functions
-  MachineIRBuilder MIRBuilder;
+  MachineIRBuilder &MIRBuilder;
 
   /// Expose LegalizerInfo so the clients can re-use.
   const LegalizerInfo &getLegalizerInfo() const { return LI; }
@@ -103,18 +103,97 @@ private:
   void widenScalarSrc(MachineInstr &MI, LLT WideTy, unsigned OpIdx,
                       unsigned ExtOpcode);
 
+  /// Legalize a single operand \p OpIdx of the machine instruction \p MI as a
+  /// Use by truncating the operand's type to \p NarrowTy using G_TRUNC, and
+  /// replacing the vreg of the operand in place.
+  void narrowScalarSrc(MachineInstr &MI, LLT NarrowTy, unsigned OpIdx);
+
   /// Legalize a single operand \p OpIdx of the machine instruction \p MI as a
   /// Def by extending the operand's type to \p WideTy and truncating it back
   /// with the \p TruncOpcode, and replacing the vreg of the operand in place.
   void widenScalarDst(MachineInstr &MI, LLT WideTy, unsigned OpIdx = 0,
                       unsigned TruncOpcode = TargetOpcode::G_TRUNC);
 
+  // Legalize a single operand \p OpIdx of the machine instruction \p MI as a
+  // Def by truncating the operand's type to \p NarrowTy, replacing in place and
+  // extending back with \p ExtOpcode.
+  void narrowScalarDst(MachineInstr &MI, LLT NarrowTy, unsigned OpIdx,
+                       unsigned ExtOpcode);
+  /// Legalize a single operand \p OpIdx of the machine instruction \p MI as a
+  /// Def by performing it with additional vector elements and extracting the
+  /// result elements, and replacing the vreg of the operand in place.
+  void moreElementsVectorDst(MachineInstr &MI, LLT MoreTy, unsigned OpIdx);
+
+  LegalizeResult
+  widenScalarMergeValues(MachineInstr &MI, unsigned TypeIdx, LLT WideTy);
+  LegalizeResult
+  widenScalarUnmergeValues(MachineInstr &MI, unsigned TypeIdx, LLT WideTy);
+  LegalizeResult
+  widenScalarExtract(MachineInstr &MI, unsigned TypeIdx, LLT WideTy);
+  LegalizeResult
+  widenScalarInsert(MachineInstr &MI, unsigned TypeIdx, LLT WideTy);
+
   /// Helper function to split a wide generic register into bitwise blocks with
   /// the given Type (which implies the number of blocks needed). The generic
   /// registers created are appended to Ops, starting at bit 0 of Reg.
   void extractParts(unsigned Reg, LLT Ty, int NumParts,
                     SmallVectorImpl<unsigned> &VRegs);
 
+  /// Version which handles irregular splits.
+  bool extractParts(unsigned Reg, LLT RegTy, LLT MainTy,
+                    LLT &LeftoverTy,
+                    SmallVectorImpl<unsigned> &VRegs,
+                    SmallVectorImpl<unsigned> &LeftoverVRegs);
+
+  /// Helper function to build a wide generic register \p DstReg of type \p
+  /// RegTy from smaller parts. This will produce a G_MERGE_VALUES,
+  /// G_BUILD_VECTOR, G_CONCAT_VECTORS, or sequence of G_INSERT as appropriate
+  /// for the types.
+  ///
+  /// \p PartRegs must be registers of type \p PartTy.
+  ///
+  /// If \p ResultTy does not evenly break into \p PartTy sized pieces, the
+  /// remainder must be specified with \p LeftoverRegs of type \p LeftoverTy.
+  void insertParts(unsigned DstReg, LLT ResultTy,
+                   LLT PartTy, ArrayRef<unsigned> PartRegs,
+                   LLT LeftoverTy = LLT(), ArrayRef<unsigned> LeftoverRegs = {});
+
+  LegalizeResult fewerElementsVectorImplicitDef(MachineInstr &MI,
+                                                unsigned TypeIdx, LLT NarrowTy);
+
+  /// Legalize a simple vector instruction where all operands are the same type
+  /// by splitting into multiple components.
+  LegalizeResult fewerElementsVectorBasic(MachineInstr &MI, unsigned TypeIdx,
+                                          LLT NarrowTy);
+
+  /// Legalize a instruction with a vector type where each operand may have a
+  /// different element type. All type indexes must have the same number of
+  /// elements.
+  LegalizeResult fewerElementsVectorMultiEltType(MachineInstr &MI,
+                                                 unsigned TypeIdx, LLT NarrowTy);
+
+  LegalizeResult fewerElementsVectorCasts(MachineInstr &MI, unsigned TypeIdx,
+                                          LLT NarrowTy);
+
+  LegalizeResult
+  fewerElementsVectorCmp(MachineInstr &MI, unsigned TypeIdx, LLT NarrowTy);
+
+  LegalizeResult
+  fewerElementsVectorSelect(MachineInstr &MI, unsigned TypeIdx, LLT NarrowTy);
+
+  LegalizeResult
+  reduceLoadStoreWidth(MachineInstr &MI, unsigned TypeIdx, LLT NarrowTy);
+
+  LegalizeResult narrowScalarShiftByConstant(MachineInstr &MI, const APInt &Amt,
+                                             LLT HalfTy, LLT ShiftAmtTy);
+
+  LegalizeResult narrowScalarShift(MachineInstr &MI, unsigned TypeIdx, LLT Ty);
+  LegalizeResult narrowScalarMul(MachineInstr &MI, unsigned TypeIdx, LLT Ty);
+  LegalizeResult narrowScalarExtract(MachineInstr &MI, unsigned TypeIdx, LLT Ty);
+  LegalizeResult narrowScalarInsert(MachineInstr &MI, unsigned TypeIdx, LLT Ty);
+
+  LegalizeResult narrowScalarSelect(MachineInstr &MI, unsigned TypeIdx, LLT Ty);
+
   LegalizeResult lowerBitCount(MachineInstr &MI, unsigned TypeIdx, LLT Ty);
 
   MachineRegisterInfo &MRI;
diff --git a/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h b/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h
index 13776dd3e87d18dd76a2ab8eb5ee07b37d6d9834..59791738fdf6c3699673f4379c7219fb3c90a5e9 100644
--- a/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h
+++ b/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h
@@ -1,9 +1,8 @@
 //===- llvm/CodeGen/GlobalISel/LegalizerInfo.h ------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -205,15 +204,40 @@ LegalityPredicate typePairAndMemSizeInSet(
     std::initializer_list<TypePairAndMemSize> TypesAndMemSizeInit);
 /// True iff the specified type index is a scalar.
 LegalityPredicate isScalar(unsigned TypeIdx);
+/// True iff the specified type index is a vector.
+LegalityPredicate isVector(unsigned TypeIdx);
+/// True iff the specified type index is a pointer (with any address space).
+LegalityPredicate isPointer(unsigned TypeIdx);
+/// True iff the specified type index is a pointer with the specified address
+/// space.
+LegalityPredicate isPointer(unsigned TypeIdx, unsigned AddrSpace);
+
 /// True iff the specified type index is a scalar that's narrower than the given
 /// size.
 LegalityPredicate narrowerThan(unsigned TypeIdx, unsigned Size);
+
 /// True iff the specified type index is a scalar that's wider than the given
 /// size.
 LegalityPredicate widerThan(unsigned TypeIdx, unsigned Size);
+
+/// True iff the specified type index is a scalar or vector with an element type
+/// that's narrower than the given size.
+LegalityPredicate scalarOrEltNarrowerThan(unsigned TypeIdx, unsigned Size);
+
+/// True iff the specified type index is a scalar or a vector with an element
+/// type that's wider than the given size.
+LegalityPredicate scalarOrEltWiderThan(unsigned TypeIdx, unsigned Size);
+
 /// True iff the specified type index is a scalar whose size is not a power of
 /// 2.
 LegalityPredicate sizeNotPow2(unsigned TypeIdx);
+
+/// True iff the specified type index is a scalar or vector whose element size
+/// is not a power of 2.
+LegalityPredicate scalarOrEltSizeNotPow2(unsigned TypeIdx);
+
+/// True iff the specified type indices are both the same bit size.
+LegalityPredicate sameSize(unsigned TypeIdx0, unsigned TypeIdx1);
 /// True iff the specified MMO index has a size that is not a power of 2
 LegalityPredicate memSizeInBytesNotPow2(unsigned MMOIdx);
 /// True iff the specified type index is a vector whose element count is not a
@@ -228,13 +252,25 @@ LegalityPredicate atomicOrderingAtLeastOrStrongerThan(unsigned MMOIdx,
 namespace LegalizeMutations {
 /// Select this specific type for the given type index.
 LegalizeMutation changeTo(unsigned TypeIdx, LLT Ty);
+
 /// Keep the same type as the given type index.
 LegalizeMutation changeTo(unsigned TypeIdx, unsigned FromTypeIdx);
-/// Widen the type for the given type index to the next power of 2.
-LegalizeMutation widenScalarToNextPow2(unsigned TypeIdx, unsigned Min = 0);
+
+/// Keep the same scalar or element type as the given type index.
+LegalizeMutation changeElementTo(unsigned TypeIdx, unsigned FromTypeIdx);
+
+/// Keep the same scalar or element type as the given type.
+LegalizeMutation changeElementTo(unsigned TypeIdx, LLT Ty);
+
+/// Widen the scalar type or vector element type for the given type index to the
+/// next power of 2.
+LegalizeMutation widenScalarOrEltToNextPow2(unsigned TypeIdx, unsigned Min = 0);
+
 /// Add more elements to the type for the given type index to the next power of
 /// 2.
 LegalizeMutation moreElementsToNextPow2(unsigned TypeIdx, unsigned Min = 0);
+/// Break up the vector type for the given type index into the element type.
+LegalizeMutation scalarize(unsigned TypeIdx);
 } // end namespace LegalizeMutations
 
 /// A single rule in a legalizer info ruleset.
@@ -597,13 +633,29 @@ public:
     return actionForCartesianProduct(LegalizeAction::Custom, Types0, Types1);
   }
 
+  /// Unconditionally custom lower.
+  LegalizeRuleSet &custom() {
+    return customIf(always);
+  }
+
   /// Widen the scalar to the next power of two that is at least MinSize.
   /// No effect if the type is not a scalar or is a power of two.
   LegalizeRuleSet &widenScalarToNextPow2(unsigned TypeIdx,
                                          unsigned MinSize = 0) {
     using namespace LegalityPredicates;
-    return actionIf(LegalizeAction::WidenScalar, sizeNotPow2(typeIdx(TypeIdx)),
-                    LegalizeMutations::widenScalarToNextPow2(TypeIdx, MinSize));
+    return actionIf(
+        LegalizeAction::WidenScalar, sizeNotPow2(typeIdx(TypeIdx)),
+        LegalizeMutations::widenScalarOrEltToNextPow2(TypeIdx, MinSize));
+  }
+
+  /// Widen the scalar or vector element type to the next power of two that is
+  /// at least MinSize.  No effect if the scalar size is a power of two.
+  LegalizeRuleSet &widenScalarOrEltToNextPow2(unsigned TypeIdx,
+                                              unsigned MinSize = 0) {
+    using namespace LegalityPredicates;
+    return actionIf(
+        LegalizeAction::WidenScalar, scalarOrEltSizeNotPow2(typeIdx(TypeIdx)),
+        LegalizeMutations::widenScalarOrEltToNextPow2(TypeIdx, MinSize));
   }
 
   LegalizeRuleSet &narrowScalar(unsigned TypeIdx, LegalizeMutation Mutation) {
@@ -612,6 +664,21 @@ public:
                     Mutation);
   }
 
+  LegalizeRuleSet &scalarize(unsigned TypeIdx) {
+    using namespace LegalityPredicates;
+    return actionIf(LegalizeAction::FewerElements, isVector(typeIdx(TypeIdx)),
+                    LegalizeMutations::scalarize(TypeIdx));
+  }
+
+  /// Ensure the scalar is at least as wide as Ty.
+  LegalizeRuleSet &minScalarOrElt(unsigned TypeIdx, const LLT &Ty) {
+    using namespace LegalityPredicates;
+    using namespace LegalizeMutations;
+    return actionIf(LegalizeAction::WidenScalar,
+                    scalarOrEltNarrowerThan(TypeIdx, Ty.getScalarSizeInBits()),
+                    changeElementTo(typeIdx(TypeIdx), Ty));
+  }
+
   /// Ensure the scalar is at least as wide as Ty.
   LegalizeRuleSet &minScalar(unsigned TypeIdx, const LLT &Ty) {
     using namespace LegalityPredicates;
@@ -621,6 +688,15 @@ public:
                     changeTo(typeIdx(TypeIdx), Ty));
   }
 
+  /// Ensure the scalar is at most as wide as Ty.
+  LegalizeRuleSet &maxScalarOrElt(unsigned TypeIdx, const LLT &Ty) {
+    using namespace LegalityPredicates;
+    using namespace LegalizeMutations;
+    return actionIf(LegalizeAction::NarrowScalar,
+                    scalarOrEltWiderThan(TypeIdx, Ty.getScalarSizeInBits()),
+                    changeElementTo(typeIdx(TypeIdx), Ty));
+  }
+
   /// Ensure the scalar is at most as wide as Ty.
   LegalizeRuleSet &maxScalar(unsigned TypeIdx, const LLT &Ty) {
     using namespace LegalityPredicates;
@@ -637,12 +713,12 @@ public:
                                const LLT &Ty) {
     using namespace LegalityPredicates;
     using namespace LegalizeMutations;
-    return actionIf(LegalizeAction::NarrowScalar,
-                    [=](const LegalityQuery &Query) {
-                      return widerThan(TypeIdx, Ty.getSizeInBits()) &&
-                             Predicate(Query);
-                    },
-                    changeTo(typeIdx(TypeIdx), Ty));
+    return actionIf(
+        LegalizeAction::NarrowScalar,
+        [=](const LegalityQuery &Query) {
+          return widerThan(TypeIdx, Ty.getSizeInBits()) && Predicate(Query);
+        },
+        changeElementTo(typeIdx(TypeIdx), Ty));
   }
 
   /// Limit the range of scalar sizes to MinTy and MaxTy.
@@ -652,6 +728,12 @@ public:
     return minScalar(TypeIdx, MinTy).maxScalar(TypeIdx, MaxTy);
   }
 
+  /// Limit the range of scalar sizes to MinTy and MaxTy.
+  LegalizeRuleSet &clampScalarOrElt(unsigned TypeIdx, const LLT &MinTy,
+                                    const LLT &MaxTy) {
+    return minScalarOrElt(TypeIdx, MinTy).maxScalarOrElt(TypeIdx, MaxTy);
+  }
+
   /// Widen the scalar to match the size of another.
   LegalizeRuleSet &minScalarSameAs(unsigned TypeIdx, unsigned LargeTypeIdx) {
     typeIdx(TypeIdx);
@@ -661,8 +743,9 @@ public:
                  Query.Types[TypeIdx].getSizeInBits();
         },
         [=](const LegalityQuery &Query) {
+          LLT T = Query.Types[LargeTypeIdx];
           return std::make_pair(TypeIdx,
-                                Query.Types[LargeTypeIdx].getElementType());
+                                T.isVector() ? T.getElementType() : T);
         });
   }
 
@@ -691,7 +774,7 @@ public:
         [=](const LegalityQuery &Query) {
           LLT VecTy = Query.Types[TypeIdx];
           return std::make_pair(
-              TypeIdx, LLT::vector(MinElements, VecTy.getScalarSizeInBits()));
+              TypeIdx, LLT::vector(MinElements, VecTy.getElementType()));
         });
   }
   /// Limit the number of elements in EltTy vectors to at most MaxElements.
@@ -708,10 +791,8 @@ public:
         },
         [=](const LegalityQuery &Query) {
           LLT VecTy = Query.Types[TypeIdx];
-          if (MaxElements == 1)
-            return std::make_pair(TypeIdx, VecTy.getElementType());
-          return std::make_pair(
-              TypeIdx, LLT::vector(MaxElements, VecTy.getScalarSizeInBits()));
+          LLT NewTy = LLT::scalarOrVector(MaxElements, VecTy.getElementType());
+          return std::make_pair(TypeIdx, NewTy);
         });
   }
   /// Limit the number of elements for the given vectors to at least MinTy's
diff --git a/include/llvm/CodeGen/GlobalISel/Localizer.h b/include/llvm/CodeGen/GlobalISel/Localizer.h
index 1e2d4763e5e1d01210cd6a1ae17e1beadf6d3e6d..cfc7c3567c5d51d2d8941ebd4b96e2c9d60d6192 100644
--- a/include/llvm/CodeGen/GlobalISel/Localizer.h
+++ b/include/llvm/CodeGen/GlobalISel/Localizer.h
@@ -1,9 +1,8 @@
 //== llvm/CodeGen/GlobalISel/Localizer.h - Localizer -------------*- C++ -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/CodeGen/GlobalISel/MIPatternMatch.h b/include/llvm/CodeGen/GlobalISel/MIPatternMatch.h
index f77f9a8df7ee4bd6641955d605607ae161a3d1af..1f4ee8aac0c27c4b1ad6a4859885bea32303046b 100644
--- a/include/llvm/CodeGen/GlobalISel/MIPatternMatch.h
+++ b/include/llvm/CodeGen/GlobalISel/MIPatternMatch.h
@@ -1,9 +1,8 @@
 //== ----- llvm/CodeGen/GlobalISel/MIPatternMatch.h --------------------- == //
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h b/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h
index 1745b9702d8f9da587e082f177a76ff140e6ecd9..6e1ff7c3eb5629e02f1437dbc4c71598c81f2d1e 100644
--- a/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h
+++ b/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h
@@ -1,9 +1,8 @@
 //===-- llvm/CodeGen/GlobalISel/MachineIRBuilder.h - MIBuilder --*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
@@ -14,6 +13,7 @@
 #ifndef LLVM_CODEGEN_GLOBALISEL_MACHINEIRBUILDER_H
 #define LLVM_CODEGEN_GLOBALISEL_MACHINEIRBUILDER_H
 
+#include "llvm/CodeGen/GlobalISel/CSEInfo.h"
 #include "llvm/CodeGen/GlobalISel/Types.h"
 
 #include "llvm/CodeGen/LowLevelType.h"
@@ -52,6 +52,8 @@ struct MachineIRBuilderState {
   /// @}
 
   GISelChangeObserver *Observer;
+
+  GISelCSEInfo *CSEInfo;
 };
 
 class DstOp {
@@ -64,6 +66,7 @@ class DstOp {
 public:
   enum class DstType { Ty_LLT, Ty_Reg, Ty_RC };
   DstOp(unsigned R) : Reg(R), Ty(DstType::Ty_Reg) {}
+  DstOp(const MachineOperand &Op) : Reg(Op.getReg()), Ty(DstType::Ty_Reg) {}
   DstOp(const LLT &T) : LLTTy(T), Ty(DstType::Ty_LLT) {}
   DstOp(const TargetRegisterClass *TRC) : RC(TRC), Ty(DstType::Ty_RC) {}
 
@@ -81,8 +84,6 @@ public:
     }
   }
 
-  DstType getType() const { return Ty; }
-
   LLT getLLTTy(const MachineRegisterInfo &MRI) const {
     switch (Ty) {
     case DstType::Ty_RC:
@@ -95,6 +96,20 @@ public:
     llvm_unreachable("Unrecognised DstOp::DstType enum");
   }
 
+  unsigned getReg() const {
+    assert(Ty == DstType::Ty_Reg && "Not a register");
+    return Reg;
+  }
+
+  const TargetRegisterClass *getRegClass() const {
+    switch (Ty) {
+    case DstType::Ty_RC:
+      return RC;
+    default:
+      llvm_unreachable("Not a RC Operand");
+    }
+  }
+
   DstType getDstOpKind() const { return Ty; }
 
 private:
@@ -111,6 +126,7 @@ class SrcOp {
 public:
   enum class SrcType { Ty_Reg, Ty_MIB, Ty_Predicate };
   SrcOp(unsigned R) : Reg(R), Ty(SrcType::Ty_Reg) {}
+  SrcOp(const MachineOperand &Op) : Reg(Op.getReg()), Ty(SrcType::Ty_Reg) {}
   SrcOp(const MachineInstrBuilder &MIB) : SrcMIB(MIB), Ty(SrcType::Ty_MIB) {}
   SrcOp(const CmpInst::Predicate P) : Pred(P), Ty(SrcType::Ty_Predicate) {}
 
@@ -187,6 +203,7 @@ protected:
   void validateTruncExt(const LLT &Dst, const LLT &Src, bool IsExtend);
 
   void validateBinaryOp(const LLT &Res, const LLT &Op0, const LLT &Op1);
+  void validateShiftOp(const LLT &Res, const LLT &Op0, const LLT &Op1);
 
   void validateSelectOp(const LLT &ResTy, const LLT &TstTy, const LLT &Op0Ty,
                         const LLT &Op1Ty);
@@ -215,21 +232,35 @@ public:
     return *State.MF;
   }
 
+  const MachineFunction &getMF() const {
+    assert(State.MF && "MachineFunction is not set");
+    return *State.MF;
+  }
+
   /// Getter for DebugLoc
   const DebugLoc &getDL() { return State.DL; }
 
   /// Getter for MRI
   MachineRegisterInfo *getMRI() { return State.MRI; }
+  const MachineRegisterInfo *getMRI() const { return State.MRI; }
 
   /// Getter for the State
   MachineIRBuilderState &getState() { return State; }
 
   /// Getter for the basic block we currently build.
-  MachineBasicBlock &getMBB() {
+  const MachineBasicBlock &getMBB() const {
     assert(State.MBB && "MachineBasicBlock is not set");
     return *State.MBB;
   }
 
+  MachineBasicBlock &getMBB() {
+    return const_cast<MachineBasicBlock &>(
+        const_cast<const MachineIRBuilder *>(this)->getMBB());
+  }
+
+  GISelCSEInfo *getCSEInfo() { return State.CSEInfo; }
+  const GISelCSEInfo *getCSEInfo() const { return State.CSEInfo; }
+
   /// Current insertion point for new instructions.
   MachineBasicBlock::iterator getInsertPt() { return State.II; }
 
@@ -239,10 +270,12 @@ public:
   void setInsertPt(MachineBasicBlock &MBB, MachineBasicBlock::iterator II);
   /// @}
 
+  void setCSEInfo(GISelCSEInfo *Info);
+
   /// \name Setters for the insertion point.
   /// @{
   /// Set the MachineFunction where to build instructions.
-  void setMF(MachineFunction &);
+  void setMF(MachineFunction &MF);
 
   /// Set the insertion point to the  end of \p MBB.
   /// \pre \p MBB must be contained by getMF().
@@ -432,6 +465,15 @@ public:
   /// \return The newly created instruction.
   MachineInstrBuilder buildSExt(const DstOp &Res, const SrcOp &Op);
 
+  /// \return The opcode of the extension the target wants to use for boolean
+  /// values.
+  unsigned getBoolExtOp(bool IsVec, bool IsFP) const;
+
+  // Build and insert \p Res = G_ANYEXT \p Op, \p Res = G_SEXT \p Op, or \p Res
+  // = G_ZEXT \p Op depending on how the target wants to extend boolean values.
+  MachineInstrBuilder buildBoolExt(const DstOp &Res, const SrcOp &Op,
+                                   bool IsFP);
+
   /// Build and insert \p Res = G_ZEXT \p Op
   ///
   /// G_ZEXT produces a register of the specified width, with bits 0 to
@@ -534,7 +576,8 @@ public:
   ///      type.
   ///
   /// \return The newly created instruction.
-  MachineInstrBuilder buildConstant(const DstOp &Res, const ConstantInt &Val);
+  virtual MachineInstrBuilder buildConstant(const DstOp &Res,
+                                            const ConstantInt &Val);
 
   /// Build and insert \p Res = G_CONSTANT \p Val
   ///
@@ -545,6 +588,7 @@ public:
   ///
   /// \return The newly created instruction.
   MachineInstrBuilder buildConstant(const DstOp &Res, int64_t Val);
+  MachineInstrBuilder buildConstant(const DstOp &Res, const APInt &Val);
 
   /// Build and insert \p Res = G_FCONSTANT \p Val
   ///
@@ -555,7 +599,8 @@ public:
   /// \pre \p Res must be a generic virtual register with scalar type.
   ///
   /// \return The newly created instruction.
-  MachineInstrBuilder buildFConstant(const DstOp &Res, const ConstantFP &Val);
+  virtual MachineInstrBuilder buildFConstant(const DstOp &Res,
+                                             const ConstantFP &Val);
 
   MachineInstrBuilder buildFConstant(const DstOp &Res, double Val);
 
@@ -669,6 +714,11 @@ public:
   MachineInstrBuilder buildBuildVector(const DstOp &Res,
                                        ArrayRef<unsigned> Ops);
 
+  /// Build and insert \p Res = G_BUILD_VECTOR with \p Src replicated to fill
+  /// the number of elements
+  MachineInstrBuilder buildSplatVector(const DstOp &Res,
+                                       const SrcOp &Src);
+
   /// Build and insert \p Res = G_BUILD_VECTOR_TRUNC \p Op0, ...
   ///
   /// G_BUILD_VECTOR_TRUNC creates a vector value from multiple scalar registers
@@ -1096,6 +1146,36 @@ public:
     return buildInstr(TargetOpcode::G_MUL, {Dst}, {Src0, Src1}, Flags);
   }
 
+  MachineInstrBuilder buildUMulH(const DstOp &Dst, const SrcOp &Src0,
+                                 const SrcOp &Src1,
+                                 Optional<unsigned> Flags = None) {
+    return buildInstr(TargetOpcode::G_UMULH, {Dst}, {Src0, Src1}, Flags);
+  }
+
+  MachineInstrBuilder buildSMulH(const DstOp &Dst, const SrcOp &Src0,
+                                 const SrcOp &Src1,
+                                 Optional<unsigned> Flags = None) {
+    return buildInstr(TargetOpcode::G_SMULH, {Dst}, {Src0, Src1}, Flags);
+  }
+
+  MachineInstrBuilder buildShl(const DstOp &Dst, const SrcOp &Src0,
+                               const SrcOp &Src1,
+                               Optional<unsigned> Flags = None) {
+    return buildInstr(TargetOpcode::G_SHL, {Dst}, {Src0, Src1}, Flags);
+  }
+
+  MachineInstrBuilder buildLShr(const DstOp &Dst, const SrcOp &Src0,
+                                const SrcOp &Src1,
+                                Optional<unsigned> Flags = None) {
+    return buildInstr(TargetOpcode::G_LSHR, {Dst}, {Src0, Src1}, Flags);
+  }
+
+  MachineInstrBuilder buildAShr(const DstOp &Dst, const SrcOp &Src0,
+                                const SrcOp &Src1,
+                                Optional<unsigned> Flags = None) {
+    return buildInstr(TargetOpcode::G_ASHR, {Dst}, {Src0, Src1}, Flags);
+  }
+
   /// Build and insert \p Res = G_AND \p Op0, \p Op1
   ///
   /// G_AND sets \p Res to the bitwise and of integer parameters \p Op0 and \p
diff --git a/include/llvm/CodeGen/GlobalISel/RegBankSelect.h b/include/llvm/CodeGen/GlobalISel/RegBankSelect.h
index c53ae416e60b3f587f5c7ab818af2967d5a217cb..c31ca5c537e1c5e783d25a8c8a902accd02b6220 100644
--- a/include/llvm/CodeGen/GlobalISel/RegBankSelect.h
+++ b/include/llvm/CodeGen/GlobalISel/RegBankSelect.h
@@ -1,9 +1,8 @@
 //=- llvm/CodeGen/GlobalISel/RegBankSelect.h - Reg Bank Selector --*- C++ -*-=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/CodeGen/GlobalISel/RegisterBank.h b/include/llvm/CodeGen/GlobalISel/RegisterBank.h
index d5612e17393cc1576fc59803b4b02c77d5fb8434..f528d1a460128c499bded03e2bf7a1a479ade50b 100644
--- a/include/llvm/CodeGen/GlobalISel/RegisterBank.h
+++ b/include/llvm/CodeGen/GlobalISel/RegisterBank.h
@@ -1,9 +1,8 @@
 //==-- llvm/CodeGen/GlobalISel/RegisterBank.h - Register Bank ----*- C++ -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/CodeGen/GlobalISel/RegisterBankInfo.h b/include/llvm/CodeGen/GlobalISel/RegisterBankInfo.h
index 82fd7eddb68ae1cee0d2b7a1f2425b2a41d5b507..37a63d1bae09787bc5aeb81a6e9231946658511e 100644
--- a/include/llvm/CodeGen/GlobalISel/RegisterBankInfo.h
+++ b/include/llvm/CodeGen/GlobalISel/RegisterBankInfo.h
@@ -1,9 +1,8 @@
 //===- llvm/CodeGen/GlobalISel/RegisterBankInfo.h ---------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -103,8 +102,8 @@ public:
   /// Currently the TableGen-like file would look like:
   /// \code
   /// PartialMapping[] = {
-  /// /*32-bit add*/ {0, 32, GPR},
-  /// /*2x32-bit add*/ {0, 32, GPR}, {0, 32, GPR}, // <-- Same entry 3x
+  /// /*32-bit add*/    {0, 32, GPR}, // Scalar entry repeated for first vec elt.
+  /// /*2x32-bit add*/  {0, 32, GPR}, {32, 32, GPR},
   /// /*<2x32-bit> vadd {0, 64, VPR}
   /// }; // PartialMapping duplicated.
   ///
@@ -118,14 +117,15 @@ public:
   /// With the array of pointer, we would have:
   /// \code
   /// PartialMapping[] = {
-  /// /*32-bit add*/ {0, 32, GPR},
+  /// /*32-bit add lower */ {0, 32, GPR},
+  /// /*32-bit add upper */ {32, 32, GPR},
   /// /*<2x32-bit> vadd {0, 64, VPR}
   /// }; // No more duplication.
   ///
   /// BreakDowns[] = {
   /// /*AddBreakDown*/ &PartialMapping[0],
-  /// /*2xAddBreakDown*/ &PartialMapping[0], &PartialMapping[0],
-  /// /*VAddBreakDown*/ &PartialMapping[1]
+  /// /*2xAddBreakDown*/ &PartialMapping[0], &PartialMapping[1],
+  /// /*VAddBreakDown*/ &PartialMapping[2]
   /// }; // Addresses of PartialMapping duplicated (smaller).
   ///
   /// ValueMapping[] {
@@ -160,6 +160,10 @@ public:
     const PartialMapping *begin() const { return BreakDown; }
     const PartialMapping *end() const { return BreakDown + NumBreakDowns; }
 
+    /// \return true if all partial mappings are the same size and register
+    /// bank.
+    bool partsAllUniform() const;
+
     /// Check if this ValueMapping is valid.
     bool isValid() const { return BreakDown && NumBreakDowns; }
 
@@ -617,6 +621,15 @@ public:
     return &A != &B;
   }
 
+  /// Get the cost of using \p ValMapping to decompose a register. This is
+  /// similar to ::copyCost, except for cases where multiple copy-like
+  /// operations need to be inserted. If the register is used as a source
+  /// operand and already has a bank assigned, \p CurBank is non-null.
+  virtual unsigned getBreakDownCost(const ValueMapping &ValMapping,
+                                    const RegisterBank *CurBank = nullptr) const {
+    return std::numeric_limits<unsigned>::max();
+  }
+
   /// Constrain the (possibly generic) virtual register \p Reg to \p RC.
   ///
   /// \pre \p Reg is a virtual register that either has a bank or a class.
diff --git a/include/llvm/CodeGen/GlobalISel/Types.h b/include/llvm/CodeGen/GlobalISel/Types.h
index 7b22e343a7f86447b3ab5348b73046fd86300f02..4fd7043ba02dea2eb80dac5f7ae2d557ad056bcb 100644
--- a/include/llvm/CodeGen/GlobalISel/Types.h
+++ b/include/llvm/CodeGen/GlobalISel/Types.h
@@ -1,9 +1,8 @@
 //===- llvm/CodeGen/GlobalISel/Types.h - Types used by GISel ----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
diff --git a/include/llvm/CodeGen/GlobalISel/Utils.h b/include/llvm/CodeGen/GlobalISel/Utils.h
index 51e3a27329721eae69e4eee5d3d6adc52a025cab..dd14a0cf87ee32fcd4647e145911ad41175965f4 100644
--- a/include/llvm/CodeGen/GlobalISel/Utils.h
+++ b/include/llvm/CodeGen/GlobalISel/Utils.h
@@ -1,9 +1,8 @@
 //==-- llvm/CodeGen/GlobalISel/Utils.h ---------------------------*- C++ -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -108,5 +107,8 @@ APFloat getAPFloatFromSize(double Val, unsigned Size);
 /// fallback.
 void getSelectionDAGFallbackAnalysisUsage(AnalysisUsage &AU);
 
+Optional<APInt> ConstantFoldBinOp(unsigned Opcode, const unsigned Op1,
+                                  const unsigned Op2,
+                                  const MachineRegisterInfo &MRI);
 } // End namespace llvm.
 #endif
diff --git a/include/llvm/CodeGen/ISDOpcodes.h b/include/llvm/CodeGen/ISDOpcodes.h
index 9c918ae1104f9dbcec2e9acd0efe198a0755abe1..ed4bfe7ad2de1284f3a7ea78bff63837df323bd8 100644
--- a/include/llvm/CodeGen/ISDOpcodes.h
+++ b/include/llvm/CodeGen/ISDOpcodes.h
@@ -1,9 +1,8 @@
 //===-- llvm/CodeGen/ISDOpcodes.h - CodeGen opcodes -------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -272,12 +271,12 @@ namespace ISD {
     /// resulting value is this minimum value.
     SSUBSAT, USUBSAT,
 
-    /// RESULT = SMULFIX(LHS, RHS, SCALE) - Perform fixed point multiplication on
+    /// RESULT = [US]MULFIX(LHS, RHS, SCALE) - Perform fixed point multiplication on
     /// 2 integers with the same width and scale. SCALE represents the scale of
     /// both operands as fixed point numbers. This SCALE parameter must be a
     /// constant integer. A scale of zero is effectively performing
     /// multiplication on 2 integers.
-    SMULFIX,
+    SMULFIX, UMULFIX,
 
     /// Simple binary floating point operators.
     FADD, FSUB, FMUL, FDIV, FREM,
@@ -580,7 +579,9 @@ namespace ISD {
     /// is often a storage-only type but has native conversions.
     FP16_TO_FP, FP_TO_FP16,
 
-    /// Perform various unary floating-point operations inspired by libm.
+    /// Perform various unary floating-point operations inspired by libm. For
+    /// FPOWI, the result is undefined if if the integer operand doesn't fit
+    /// into 32 bits.
     FNEG, FABS, FSQRT, FCBRT, FSIN, FCOS, FPOWI, FPOW,
     FLOG, FLOG2, FLOG10, FEXP, FEXP2,
     FCEIL, FTRUNC, FRINT, FNEARBYINT, FROUND, FFLOOR,
@@ -666,6 +667,9 @@ namespace ISD {
     /// SDOperands.
     INLINEASM,
 
+    /// INLINEASM_BR - Terminator version of inline asm. Used by asm-goto.
+    INLINEASM_BR,
+
     /// EH_LABEL - Represents a label in mid basic block used to track
     /// locations needed for debug and exception handling tables.  These nodes
     /// take a chain as input and return a chain.
@@ -819,6 +823,8 @@ namespace ISD {
     ATOMIC_LOAD_MAX,
     ATOMIC_LOAD_UMIN,
     ATOMIC_LOAD_UMAX,
+    ATOMIC_LOAD_FADD,
+    ATOMIC_LOAD_FSUB,
 
     // Masked load and store - consecutive vector load and store operations
     // with additional mask operand that prevents memory accesses to the
diff --git a/include/llvm/CodeGen/IntrinsicLowering.h b/include/llvm/CodeGen/IntrinsicLowering.h
index 597d684909c168824d62fc46e8fd842c26e54d7f..daf2d9a47801daed5f1a3c00516ffcbd2d0e03af 100644
--- a/include/llvm/CodeGen/IntrinsicLowering.h
+++ b/include/llvm/CodeGen/IntrinsicLowering.h
@@ -1,9 +1,8 @@
 //===-- IntrinsicLowering.h - Intrinsic Function Lowering -------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -31,10 +30,6 @@ class IntrinsicLowering {
 public:
   explicit IntrinsicLowering(const DataLayout &DL) : DL(DL), Warned(false) {}
 
-  /// Add all of the prototypes that might be needed by an intrinsic lowering
-  /// implementation to be inserted into the module specified.
-  void AddPrototypes(Module &M);
-
   /// Replace a call to the specified intrinsic function.
   /// If an intrinsic function must be implemented by the code generator
   /// (such as va_start), this function should print a message and abort.
diff --git a/include/llvm/CodeGen/LatencyPriorityQueue.h b/include/llvm/CodeGen/LatencyPriorityQueue.h
index 9b8d83ce77ca58398cf7021c45152a778e3129e3..95f4c6473542ef22cee247e9216a49c1a7229089 100644
--- a/include/llvm/CodeGen/LatencyPriorityQueue.h
+++ b/include/llvm/CodeGen/LatencyPriorityQueue.h
@@ -1,9 +1,8 @@
 //===---- LatencyPriorityQueue.h - A latency-oriented priority queue ------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/CodeGen/LazyMachineBlockFrequencyInfo.h b/include/llvm/CodeGen/LazyMachineBlockFrequencyInfo.h
index 221f16a03f1672889b17a6354bfe6a9cd83e8f59..ca99c6c89b191f7c2a824dcdbf54bed306c16e4a 100644
--- a/include/llvm/CodeGen/LazyMachineBlockFrequencyInfo.h
+++ b/include/llvm/CodeGen/LazyMachineBlockFrequencyInfo.h
@@ -1,9 +1,8 @@
 ///===- LazyMachineBlockFrequencyInfo.h - Lazy Block Frequency -*- C++ -*--===//
 ///
-///                     The LLVM Compiler Infrastructure
-///
-/// This file is distributed under the University of Illinois Open Source
-/// License. See LICENSE.TXT for details.
+/// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+/// See https://llvm.org/LICENSE.txt for license information.
+/// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ///
 ///===---------------------------------------------------------------------===//
 /// \file
diff --git a/include/llvm/CodeGen/LexicalScopes.h b/include/llvm/CodeGen/LexicalScopes.h
index 3ba503487823dfaf824e30eae3f961a18b1e8bdf..253d4734995b5c617d5d2144316cc7421d413a20 100644
--- a/include/llvm/CodeGen/LexicalScopes.h
+++ b/include/llvm/CodeGen/LexicalScopes.h
@@ -1,9 +1,8 @@
 //===- LexicalScopes.cpp - Collecting lexical scope info --------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/CodeGen/LinkAllAsmWriterComponents.h b/include/llvm/CodeGen/LinkAllAsmWriterComponents.h
index 38fcb37b1e69e97c13cb56704ac373b50eb7cb5b..75a5c359630e218cf075de088c0f7e4c118a55c9 100644
--- a/include/llvm/CodeGen/LinkAllAsmWriterComponents.h
+++ b/include/llvm/CodeGen/LinkAllAsmWriterComponents.h
@@ -1,9 +1,8 @@
 //===- llvm/Codegen/LinkAllAsmWriterComponents.h ----------------*- C++ -*-===//
 //
-//                      The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/CodeGen/LinkAllCodegenComponents.h b/include/llvm/CodeGen/LinkAllCodegenComponents.h
index 18c13ca8f59877f6e8f1b2747e43d601d214c852..56c93b24147e5f113468a6f018627f9575d7e191 100644
--- a/include/llvm/CodeGen/LinkAllCodegenComponents.h
+++ b/include/llvm/CodeGen/LinkAllCodegenComponents.h
@@ -1,9 +1,8 @@
 //===- llvm/Codegen/LinkAllCodegenComponents.h ------------------*- C++ -*-===//
 //
-//                      The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/CodeGen/LiveInterval.h b/include/llvm/CodeGen/LiveInterval.h
index cdf9ad2588cf37f3567f97ab762fa535ba2d9c3d..622c1248b36a7857a851b7af60fbccc59e7935cc 100644
--- a/include/llvm/CodeGen/LiveInterval.h
+++ b/include/llvm/CodeGen/LiveInterval.h
@@ -1,9 +1,8 @@
 //===- llvm/CodeGen/LiveInterval.h - Interval representation ----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/CodeGen/LiveIntervalUnion.h b/include/llvm/CodeGen/LiveIntervalUnion.h
index 9e2799bd44142c7c3b911f7d7fcaf4d81e284fb3..05506d2c3bc6328753bb67a01e12819ab1dd1162 100644
--- a/include/llvm/CodeGen/LiveIntervalUnion.h
+++ b/include/llvm/CodeGen/LiveIntervalUnion.h
@@ -1,9 +1,8 @@
 //===- LiveIntervalUnion.h - Live interval union data struct ---*- C++ -*--===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/CodeGen/LiveIntervals.h b/include/llvm/CodeGen/LiveIntervals.h
index 16ab1dc475c41bf2ff5901a92c81de829233dd99..d0791bec5c4189c228aa8fcd7fdd9cde6c58edf6 100644
--- a/include/llvm/CodeGen/LiveIntervals.h
+++ b/include/llvm/CodeGen/LiveIntervals.h
@@ -1,9 +1,8 @@
 //===- LiveIntervals.h - Live Interval Analysis -----------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/CodeGen/LivePhysRegs.h b/include/llvm/CodeGen/LivePhysRegs.h
index 7312902e21b71f0c3873c9a7d0a3f596d14c41ba..50da0b3d5c4832ecfe273a4e1c283aa579453f3a 100644
--- a/include/llvm/CodeGen/LivePhysRegs.h
+++ b/include/llvm/CodeGen/LivePhysRegs.h
@@ -1,9 +1,8 @@
 //===- llvm/CodeGen/LivePhysRegs.h - Live Physical Register Set -*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/CodeGen/LiveRangeEdit.h b/include/llvm/CodeGen/LiveRangeEdit.h
index 53830297c525c0074fd175e81f4f75065b3ea65e..6519937ec071882c03e184ef6f1b5c4caccd1af5 100644
--- a/include/llvm/CodeGen/LiveRangeEdit.h
+++ b/include/llvm/CodeGen/LiveRangeEdit.h
@@ -1,9 +1,8 @@
 //===- LiveRangeEdit.h - Basic tools for split and spill --------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/CodeGen/LiveRegMatrix.h b/include/llvm/CodeGen/LiveRegMatrix.h
index f62a55c7308558fc9d45f61314941617cb2589a8..ab4d44f9a611761a0da08ad4b8cbf107b5baa275 100644
--- a/include/llvm/CodeGen/LiveRegMatrix.h
+++ b/include/llvm/CodeGen/LiveRegMatrix.h
@@ -1,9 +1,8 @@
 //===- LiveRegMatrix.h - Track register interference ----------*- C++ -*---===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/CodeGen/LiveRegUnits.h b/include/llvm/CodeGen/LiveRegUnits.h
index 5e9dd8b3cdf6983387469fd9975eb171bbfdca85..7dbb2feab8bfe6e4e107e9dd7ae6c87ec2e28cab 100644
--- a/include/llvm/CodeGen/LiveRegUnits.h
+++ b/include/llvm/CodeGen/LiveRegUnits.h
@@ -1,9 +1,8 @@
 //===- llvm/CodeGen/LiveRegUnits.h - Register Unit Set ----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/CodeGen/LiveStacks.h b/include/llvm/CodeGen/LiveStacks.h
index 44ed785f7b53de62759b6186eae3de28f47e7be6..7c4c64d515dfb0b5c9b560f02d9ffdc968d4d98a 100644
--- a/include/llvm/CodeGen/LiveStacks.h
+++ b/include/llvm/CodeGen/LiveStacks.h
@@ -1,9 +1,8 @@
 //===- LiveStacks.h - Live Stack Slot Analysis ------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/CodeGen/LiveVariables.h b/include/llvm/CodeGen/LiveVariables.h
index ed8da8662106dee3262d64359b5ff490f6237172..71de306e29420325880e459bd98f3906038d5510 100644
--- a/include/llvm/CodeGen/LiveVariables.h
+++ b/include/llvm/CodeGen/LiveVariables.h
@@ -1,9 +1,8 @@
 //===-- llvm/CodeGen/LiveVariables.h - Live Variable Analysis ---*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/CodeGen/LoopTraversal.h b/include/llvm/CodeGen/LoopTraversal.h
index 750da0143c0df3892447522a7b76ccbf2b4154e8..e5810ef1ef2634b66f8f22040e76770c6ac15519 100644
--- a/include/llvm/CodeGen/LoopTraversal.h
+++ b/include/llvm/CodeGen/LoopTraversal.h
@@ -1,9 +1,8 @@
 //==------ llvm/CodeGen/LoopTraversal.h - Loop Traversal -*- C++ -*---------==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/CodeGen/LowLevelType.h b/include/llvm/CodeGen/LowLevelType.h
index a3c5c9329f53a4df7bb98d025fb977c2089b32aa..687233e4e16823186c3eb1e75526e3dea55cdc62 100644
--- a/include/llvm/CodeGen/LowLevelType.h
+++ b/include/llvm/CodeGen/LowLevelType.h
@@ -1,9 +1,8 @@
 //== llvm/CodeGen/LowLevelType.h ------------------------------- -*- C++ -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/CodeGen/MIRParser/MIRParser.h b/include/llvm/CodeGen/MIRParser/MIRParser.h
index e199a1f69ad7078a5dfeaade4c7e3086095d66ce..6a04e48e533ce548173eb991e8e5025b8c6844d2 100644
--- a/include/llvm/CodeGen/MIRParser/MIRParser.h
+++ b/include/llvm/CodeGen/MIRParser/MIRParser.h
@@ -1,9 +1,8 @@
 //===- MIRParser.h - MIR serialization format parser ------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/CodeGen/MIRPrinter.h b/include/llvm/CodeGen/MIRPrinter.h
index 078c4b2f6072e3fa6af584cb6505ecda6bddd599..b95cf0fe804f76c69d922a33d96b9ca44a5c9742 100644
--- a/include/llvm/CodeGen/MIRPrinter.h
+++ b/include/llvm/CodeGen/MIRPrinter.h
@@ -1,9 +1,8 @@
 //===- MIRPrinter.h - MIR serialization format printer --------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/CodeGen/MIRYamlMapping.h b/include/llvm/CodeGen/MIRYamlMapping.h
index 98ac81915dc05c25d153d2b211b9eb798acae5c5..b600ec92e18d6a4ee866cfb8a70bbaff7f501911 100644
--- a/include/llvm/CodeGen/MIRYamlMapping.h
+++ b/include/llvm/CodeGen/MIRYamlMapping.h
@@ -1,9 +1,8 @@
-//===- MIRYAMLMapping.h - Describes the mapping between MIR and YAML ------===//
+//===- MIRYamlMapping.h - Describes the mapping between MIR and YAML ------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/CodeGen/MachORelocation.h b/include/llvm/CodeGen/MachORelocation.h
index cbb49695af75344c61f835cd6565a73a907c7d34..0185c7cbe018a7e0adb08d72dec23e765c751200 100644
--- a/include/llvm/CodeGen/MachORelocation.h
+++ b/include/llvm/CodeGen/MachORelocation.h
@@ -1,9 +1,8 @@
 //=== MachORelocation.h - Mach-O Relocation Info ----------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/CodeGen/MachineBasicBlock.h b/include/llvm/CodeGen/MachineBasicBlock.h
index ec2f270fcb3f2f83a935697346638318327cd04f..34e18ba9de67e5ddd5fefd964db6fa327caa0064 100644
--- a/include/llvm/CodeGen/MachineBasicBlock.h
+++ b/include/llvm/CodeGen/MachineBasicBlock.h
@@ -1,9 +1,8 @@
 //===- llvm/CodeGen/MachineBasicBlock.h -------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/CodeGen/MachineBlockFrequencyInfo.h b/include/llvm/CodeGen/MachineBlockFrequencyInfo.h
index 5b4b99ca0a5d84dcdad13d81bf4c6d896ef6e40b..a438ecfcc25ed28017db6d7c1f02b43afe22aca8 100644
--- a/include/llvm/CodeGen/MachineBlockFrequencyInfo.h
+++ b/include/llvm/CodeGen/MachineBlockFrequencyInfo.h
@@ -1,9 +1,8 @@
 //===- MachineBlockFrequencyInfo.h - MBB Frequency Analysis -----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/CodeGen/MachineBranchProbabilityInfo.h b/include/llvm/CodeGen/MachineBranchProbabilityInfo.h
index 81b0524cf0a4d44e0bfa9a1cb8ad52475b0aed4d..2b9b2030eb97a111051b40183dc40960dfd8a626 100644
--- a/include/llvm/CodeGen/MachineBranchProbabilityInfo.h
+++ b/include/llvm/CodeGen/MachineBranchProbabilityInfo.h
@@ -1,9 +1,8 @@
 //=- MachineBranchProbabilityInfo.h - Branch Probability Analysis -*- C++ -*-=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/CodeGen/MachineCombinerPattern.h b/include/llvm/CodeGen/MachineCombinerPattern.h
index 586535f771c287b88121e3dc2aaa443d398b25e5..4f4034baf8013097ec32b8be08bfdd50ec8b8d8f 100644
--- a/include/llvm/CodeGen/MachineCombinerPattern.h
+++ b/include/llvm/CodeGen/MachineCombinerPattern.h
@@ -1,10 +1,9 @@
 //===-- llvm/CodeGen/MachineCombinerPattern.h - Instruction pattern supported by
 // combiner  ------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/CodeGen/MachineConstantPool.h b/include/llvm/CodeGen/MachineConstantPool.h
index b0b5420a884b6213358a060690c532d8067c1c88..4d07b620a4b4c372347e1a3f79e3d83a0e6ae2f4 100644
--- a/include/llvm/CodeGen/MachineConstantPool.h
+++ b/include/llvm/CodeGen/MachineConstantPool.h
@@ -1,9 +1,8 @@
 //===- CodeGen/MachineConstantPool.h - Abstract Constant Pool ---*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/CodeGen/MachineDominanceFrontier.h b/include/llvm/CodeGen/MachineDominanceFrontier.h
index 75d75bc3669ae9201fb9ed830c6a9f96e8aee47d..f7bbd07a63ab5a82e33a070f1836c5c5fac3581b 100644
--- a/include/llvm/CodeGen/MachineDominanceFrontier.h
+++ b/include/llvm/CodeGen/MachineDominanceFrontier.h
@@ -1,9 +1,8 @@
 //===- llvm/CodeGen/MachineDominanceFrontier.h ------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/CodeGen/MachineDominators.h b/include/llvm/CodeGen/MachineDominators.h
index e3d3d169db9769c4521c9c3176d7bf81a371b0f8..d2200080b8978994a799e318524e0e09192c5c5e 100644
--- a/include/llvm/CodeGen/MachineDominators.h
+++ b/include/llvm/CodeGen/MachineDominators.h
@@ -1,9 +1,8 @@
 //==- llvm/CodeGen/MachineDominators.h - Machine Dom Calculation -*- C++ -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/CodeGen/MachineFrameInfo.h b/include/llvm/CodeGen/MachineFrameInfo.h
index c2706a21a1778f1f1ec8bbf1ecaafd126ab99a2c..402a2dc606ad20cfdf311bffa5c0f69d61df90ec 100644
--- a/include/llvm/CodeGen/MachineFrameInfo.h
+++ b/include/llvm/CodeGen/MachineFrameInfo.h
@@ -1,9 +1,8 @@
 //===-- CodeGen/MachineFrameInfo.h - Abstract Stack Frame Rep. --*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/CodeGen/MachineFunction.h b/include/llvm/CodeGen/MachineFunction.h
index 35305bd53b28908fe0fcf45396822838c1043c5c..34ceb15aebde134847ca213f69c0aafdd854c062 100644
--- a/include/llvm/CodeGen/MachineFunction.h
+++ b/include/llvm/CodeGen/MachineFunction.h
@@ -1,9 +1,8 @@
 //===- llvm/CodeGen/MachineFunction.h ---------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -86,7 +85,7 @@ template <> struct ilist_callback_traits<MachineBasicBlock> {
 
   template <class Iterator>
   void transferNodesFromList(ilist_callback_traits &OldList, Iterator, Iterator) {
-    llvm_unreachable("Never transfer between lists");
+    assert(this == &OldList && "never transfer MBBs between functions");
   }
 };
 
@@ -372,16 +371,18 @@ public:
 
   public:
     virtual ~Delegate() = default;
-    virtual void MF_HandleInsertion(const MachineInstr &MI) = 0;
-    virtual void MF_HandleRemoval(const MachineInstr &MI) = 0;
+    /// Callback after an insertion. This should not modify the MI directly.
+    virtual void MF_HandleInsertion(MachineInstr &MI) = 0;
+    /// Callback before a removal. This should not modify the MI directly.
+    virtual void MF_HandleRemoval(MachineInstr &MI) = 0;
   };
 
 private:
   Delegate *TheDelegate = nullptr;
 
   // Callbacks for insertion and removal.
-  void handleInsertion(const MachineInstr &MI);
-  void handleRemoval(const MachineInstr &MI);
+  void handleInsertion(MachineInstr &MI);
+  void handleRemoval(MachineInstr &MI);
   friend struct ilist_traits<MachineInstr>;
 
 public:
diff --git a/include/llvm/CodeGen/MachineFunctionPass.h b/include/llvm/CodeGen/MachineFunctionPass.h
index 6d978daa20181bc68073ebc2c90102779f0f56d8..caaf22c2139e31af29e115556528ea167cf327a9 100644
--- a/include/llvm/CodeGen/MachineFunctionPass.h
+++ b/include/llvm/CodeGen/MachineFunctionPass.h
@@ -1,9 +1,8 @@
 //===-- MachineFunctionPass.h - Pass for MachineFunctions --------*-C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/CodeGen/MachineInstr.h b/include/llvm/CodeGen/MachineInstr.h
index ea1a2a536fc7377e0e33eadbee60acbbdb3be6a6..4cb39c5f0a6ad67b8f1a138ca4019c2402136b6d 100644
--- a/include/llvm/CodeGen/MachineInstr.h
+++ b/include/llvm/CodeGen/MachineInstr.h
@@ -1,9 +1,8 @@
 //===- llvm/CodeGen/MachineInstr.h - MachineInstr class ---------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -1012,10 +1011,13 @@ public:
   }
   bool isKill() const { return getOpcode() == TargetOpcode::KILL; }
   bool isImplicitDef() const { return getOpcode()==TargetOpcode::IMPLICIT_DEF; }
-  bool isInlineAsm() const { return getOpcode() == TargetOpcode::INLINEASM; }
+  bool isInlineAsm() const {
+    return getOpcode() == TargetOpcode::INLINEASM ||
+           getOpcode() == TargetOpcode::INLINEASM_BR;
+  }
 
   bool isMSInlineAsm() const {
-    return getOpcode() == TargetOpcode::INLINEASM && getInlineAsmDialect();
+    return isInlineAsm() && getInlineAsmDialect() == InlineAsm::AD_Intel;
   }
 
   bool isStackAligningInlineAsm() const;
@@ -1400,6 +1402,19 @@ public:
   /// Return true if all the defs of this instruction are dead.
   bool allDefsAreDead() const;
 
+  /// Return a valid size if the instruction is a spill instruction.
+  Optional<unsigned> getSpillSize(const TargetInstrInfo *TII) const;
+
+  /// Return a valid size if the instruction is a folded spill instruction.
+  Optional<unsigned> getFoldedSpillSize(const TargetInstrInfo *TII) const;
+
+  /// Return a valid size if the instruction is a restore instruction.
+  Optional<unsigned> getRestoreSize(const TargetInstrInfo *TII) const;
+
+  /// Return a valid size if the instruction is a folded restore instruction.
+  Optional<unsigned>
+  getFoldedRestoreSize(const TargetInstrInfo *TII) const;
+
   /// Copy implicit register operands from specified
   /// instruction to this instruction.
   void copyImplicitOps(MachineFunction &MF, const MachineInstr &MI);
@@ -1526,6 +1541,8 @@ public:
   /// not modify the MIFlags of this MachineInstr.
   uint16_t mergeFlagsWith(const MachineInstr& Other) const;
 
+  static uint16_t copyFlagsFromInstruction(const Instruction &I);
+
   /// Copy all flags to MachineInst MIFlags
   void copyIRFlags(const Instruction &I);
 
diff --git a/include/llvm/CodeGen/MachineInstrBuilder.h b/include/llvm/CodeGen/MachineInstrBuilder.h
index b5e523f655e7b18defdbac4df650e36a97aa88bb..4a8cd6861a987bdc878311e2ec447a9599d7a69d 100644
--- a/include/llvm/CodeGen/MachineInstrBuilder.h
+++ b/include/llvm/CodeGen/MachineInstrBuilder.h
@@ -1,9 +1,8 @@
 //===- CodeGen/MachineInstrBuilder.h - Simplify creation of MIs --*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -80,6 +79,11 @@ public:
   /// explicitly.
   MachineInstr *getInstr() const { return MI; }
 
+  /// Get the register for the operand index.
+  /// The operand at the index should be a register (asserted by
+  /// MachineOperand).
+  unsigned getReg(unsigned Idx) { return MI->getOperand(Idx).getReg(); }
+
   /// Add a new virtual register operand.
   const MachineInstrBuilder &addReg(unsigned RegNo, unsigned flags = 0,
                                     unsigned SubReg = 0) const {
diff --git a/include/llvm/CodeGen/MachineInstrBundle.h b/include/llvm/CodeGen/MachineInstrBundle.h
index b5341fd1ae496d3307626b712c5f9533b839bbe2..9bb5b633566551186538dbc73d74134abf302e84 100644
--- a/include/llvm/CodeGen/MachineInstrBundle.h
+++ b/include/llvm/CodeGen/MachineInstrBundle.h
@@ -1,9 +1,8 @@
-//===-- CodeGen/MachineInstBundle.h - MI bundle utilities -------*- C++ -*-===//
+//===- llvm/CodeGen/MachineInstrBundle.h - MI bundle utilities --*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/CodeGen/MachineInstrBundleIterator.h b/include/llvm/CodeGen/MachineInstrBundleIterator.h
index 5fe4964ff116571762823806d3f6c1ba1437aa47..0f59563e7e1bec7239304ed2e0c93d715c8264a4 100644
--- a/include/llvm/CodeGen/MachineInstrBundleIterator.h
+++ b/include/llvm/CodeGen/MachineInstrBundleIterator.h
@@ -1,9 +1,8 @@
 //===- llvm/CodeGen/MachineInstrBundleIterator.h ----------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/CodeGen/MachineJumpTableInfo.h b/include/llvm/CodeGen/MachineJumpTableInfo.h
index 25a3e6b556a3aed580b3ef471c87e5c6ecff4aa3..11781145b378fd3b4493abf32a18a13daf90b4e9 100644
--- a/include/llvm/CodeGen/MachineJumpTableInfo.h
+++ b/include/llvm/CodeGen/MachineJumpTableInfo.h
@@ -1,9 +1,8 @@
 //===-- CodeGen/MachineJumpTableInfo.h - Abstract Jump Tables  --*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/CodeGen/MachineLoopInfo.h b/include/llvm/CodeGen/MachineLoopInfo.h
index 917fb90380f50b3a4c8589ff83bffeaa20d8e3d7..da6df59c739c2c554fe2a86d8b313159beff0bb3 100644
--- a/include/llvm/CodeGen/MachineLoopInfo.h
+++ b/include/llvm/CodeGen/MachineLoopInfo.h
@@ -1,9 +1,8 @@
 //===- llvm/CodeGen/MachineLoopInfo.h - Natural Loop Calculator -*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/CodeGen/MachineMemOperand.h b/include/llvm/CodeGen/MachineMemOperand.h
index 078ef7ca510c369ccf602727a875f7285d1c3f43..12fae2f5663db2a3e40d20b656acaf07b5f970a2 100644
--- a/include/llvm/CodeGen/MachineMemOperand.h
+++ b/include/llvm/CodeGen/MachineMemOperand.h
@@ -1,9 +1,8 @@
 //==- llvm/CodeGen/MachineMemOperand.h - MachineMemOperand class -*- C++ -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -267,13 +266,13 @@ public:
   bool isAtomic() const { return getOrdering() != AtomicOrdering::NotAtomic; }
 
   /// Returns true if this memory operation doesn't have any ordering
-  /// constraints other than normal aliasing. Volatile and atomic memory
-  /// operations can't be reordered.
-  ///
-  /// Currently, we don't model the difference between volatile and atomic
-  /// operations. They should retain their ordering relative to all memory
-  /// operations.
-  bool isUnordered() const { return !isVolatile(); }
+  /// constraints other than normal aliasing. Volatile and (ordered) atomic
+  /// memory operations can't be reordered. 
+  bool isUnordered() const {
+    return (getOrdering() == AtomicOrdering::NotAtomic ||
+            getOrdering() == AtomicOrdering::Unordered) &&
+           !isVolatile();
+  }
 
   /// Update this MachineMemOperand to reflect the alignment of MMO, if it has a
   /// greater alignment. This must only be used when the new alignment applies
diff --git a/include/llvm/CodeGen/MachineModuleInfo.h b/include/llvm/CodeGen/MachineModuleInfo.h
index 4371420bc7a2bb83f935a0ddb50d47b99c9d1bdd..9b81dc6a45e5263478fade9a97a6b86a07dfda23 100644
--- a/include/llvm/CodeGen/MachineModuleInfo.h
+++ b/include/llvm/CodeGen/MachineModuleInfo.h
@@ -1,9 +1,8 @@
 //===-- llvm/CodeGen/MachineModuleInfo.h ------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -114,10 +113,9 @@ class MachineModuleInfo : public ImmutablePass {
   /// True if debugging information is available in this module.
   bool DbgInfoAvailable;
 
-  /// True if this module calls VarArg function with floating-point arguments.
-  /// This is used to emit an undefined reference to _fltused on Windows
-  /// targets.
-  bool UsesVAFloatArgument;
+  /// True if this module is being built for windows/msvc, and uses floating
+  /// point.  This is used to emit an undefined reference to _fltused.
+  bool UsesMSVCFloatingPoint;
 
   /// True if the module calls the __morestack function indirectly, as is
   /// required under the large code model on x86. This is used to emit
@@ -187,13 +185,9 @@ public:
   bool hasDebugInfo() const { return DbgInfoAvailable; }
   void setDebugInfoAvailability(bool avail) { DbgInfoAvailable = avail; }
 
-  bool usesVAFloatArgument() const {
-    return UsesVAFloatArgument;
-  }
+  bool usesMSVCFloatingPoint() const { return UsesMSVCFloatingPoint; }
 
-  void setUsesVAFloatArgument(bool b) {
-    UsesVAFloatArgument = b;
-  }
+  void setUsesMSVCFloatingPoint(bool b) { UsesMSVCFloatingPoint = b; }
 
   bool usesMorestackAddr() const {
     return UsesMorestackAddr;
@@ -258,14 +252,6 @@ public:
   /// \}
 }; // End class MachineModuleInfo
 
-//===- MMI building helpers -----------------------------------------------===//
-
-/// Determine if any floating-point values are being passed to this variadic
-/// function, and set the MachineModuleInfo's usesVAFloatArgument flag if so.
-/// This flag is used to emit an undefined reference to _fltused on Windows,
-/// which will link in MSVCRT's floating-point support.
-void computeUsesVAFloatArgument(const CallInst &I, MachineModuleInfo &MMI);
-
 } // end namespace llvm
 
 #endif // LLVM_CODEGEN_MACHINEMODULEINFO_H
diff --git a/include/llvm/CodeGen/MachineModuleInfoImpls.h b/include/llvm/CodeGen/MachineModuleInfoImpls.h
index 17df1fa792b745a21b8bf4fab7c587069e624d6c..746e9223961300c76d0410391b597bdd548aead0 100644
--- a/include/llvm/CodeGen/MachineModuleInfoImpls.h
+++ b/include/llvm/CodeGen/MachineModuleInfoImpls.h
@@ -1,9 +1,8 @@
 //===- llvm/CodeGen/MachineModuleInfoImpls.h --------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/CodeGen/MachineOperand.h b/include/llvm/CodeGen/MachineOperand.h
index 53e8889d118a03c11da291abe10a5165afda7d71..ddcdc07618c973b0327c60bb308b1f3c39951594 100644
--- a/include/llvm/CodeGen/MachineOperand.h
+++ b/include/llvm/CodeGen/MachineOperand.h
@@ -1,9 +1,8 @@
 //===-- llvm/CodeGen/MachineOperand.h - MachineOperand class ----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/CodeGen/MachineOptimizationRemarkEmitter.h b/include/llvm/CodeGen/MachineOptimizationRemarkEmitter.h
index a7ce870400c2d01fb24e9f9316e1ac342be00c7e..9a0fd56253673b511586af17bd52bea3fa7ba14c 100644
--- a/include/llvm/CodeGen/MachineOptimizationRemarkEmitter.h
+++ b/include/llvm/CodeGen/MachineOptimizationRemarkEmitter.h
@@ -1,9 +1,8 @@
 ///===- MachineOptimizationRemarkEmitter.h - Opt Diagnostics -*- C++ -*----===//
 ///
-///                     The LLVM Compiler Infrastructure
-///
-/// This file is distributed under the University of Illinois Open Source
-/// License. See LICENSE.TXT for details.
+/// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+/// See https://llvm.org/LICENSE.txt for license information.
+/// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ///
 ///===---------------------------------------------------------------------===//
 /// \file
diff --git a/include/llvm/CodeGen/MachineOutliner.h b/include/llvm/CodeGen/MachineOutliner.h
index bfd1e994053a75a0c4a8ccce324abbe752431a3b..377df4e18a2a0063bb45d0195031d205111b8614 100644
--- a/include/llvm/CodeGen/MachineOutliner.h
+++ b/include/llvm/CodeGen/MachineOutliner.h
@@ -1,9 +1,8 @@
 //===---- MachineOutliner.h - Outliner data structures ------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
diff --git a/include/llvm/CodeGen/MachinePassRegistry.h b/include/llvm/CodeGen/MachinePassRegistry.h
index a031c92d914f408193d430aed0480ab042e26c36..f5b3723db0aabebcd372bf86027cbd008a21f4c3 100644
--- a/include/llvm/CodeGen/MachinePassRegistry.h
+++ b/include/llvm/CodeGen/MachinePassRegistry.h
@@ -1,9 +1,8 @@
 //===- llvm/CodeGen/MachinePassRegistry.h -----------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/CodeGen/MachinePipeliner.h b/include/llvm/CodeGen/MachinePipeliner.h
new file mode 100644
index 0000000000000000000000000000000000000000..a30e4b91edcfc1504cfe542adc282a460805ee1a
--- /dev/null
+++ b/include/llvm/CodeGen/MachinePipeliner.h
@@ -0,0 +1,614 @@
+//===- MachinePipeliner.h - Machine Software Pipeliner Pass -------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// An implementation of the Swing Modulo Scheduling (SMS) software pipeliner.
+//
+// Software pipelining (SWP) is an instruction scheduling technique for loops
+// that overlap loop iterations and exploits ILP via a compiler transformation.
+//
+// Swing Modulo Scheduling is an implementation of software pipelining
+// that generates schedules that are near optimal in terms of initiation
+// interval, register requirements, and stage count. See the papers:
+//
+// "Swing Modulo Scheduling: A Lifetime-Sensitive Approach", by J. Llosa,
+// A. Gonzalez, E. Ayguade, and M. Valero. In PACT '96 Proceedings of the 1996
+// Conference on Parallel Architectures and Compilation Techiniques.
+//
+// "Lifetime-Sensitive Modulo Scheduling in a Production Environment", by J.
+// Llosa, E. Ayguade, A. Gonzalez, M. Valero, and J. Eckhardt. In IEEE
+// Transactions on Computers, Vol. 50, No. 3, 2001.
+//
+// "An Implementation of Swing Modulo Scheduling With Extensions for
+// Superblocks", by T. Lattner, Master's Thesis, University of Illinois at
+// Urbana-Champaign, 2005.
+//
+//
+// The SMS algorithm consists of three main steps after computing the minimal
+// initiation interval (MII).
+// 1) Analyze the dependence graph and compute information about each
+//    instruction in the graph.
+// 2) Order the nodes (instructions) by priority based upon the heuristics
+//    described in the algorithm.
+// 3) Attempt to schedule the nodes in the specified order using the MII.
+//
+//===----------------------------------------------------------------------===//
+#ifndef LLVM_LIB_CODEGEN_MACHINEPIPELINER_H
+#define LLVM_LIB_CODEGEN_MACHINEPIPELINER_H
+
+#include "llvm/CodeGen/MachineDominators.h"
+#include "llvm/CodeGen/RegisterClassInfo.h"
+#include "llvm/CodeGen/ScheduleDAGInstrs.h"
+#include "llvm/CodeGen/TargetInstrInfo.h"
+
+namespace llvm {
+
+class NodeSet;
+class SMSchedule;
+
+extern cl::opt<bool> SwpEnableCopyToPhi;
+
+/// The main class in the implementation of the target independent
+/// software pipeliner pass.
+class MachinePipeliner : public MachineFunctionPass {
+public:
+  MachineFunction *MF = nullptr;
+  const MachineLoopInfo *MLI = nullptr;
+  const MachineDominatorTree *MDT = nullptr;
+  const InstrItineraryData *InstrItins;
+  const TargetInstrInfo *TII = nullptr;
+  RegisterClassInfo RegClassInfo;
+  bool disabledByPragma = false;
+  unsigned II_setByPragma = 0;
+
+#ifndef NDEBUG
+  static int NumTries;
+#endif
+
+  /// Cache the target analysis information about the loop.
+  struct LoopInfo {
+    MachineBasicBlock *TBB = nullptr;
+    MachineBasicBlock *FBB = nullptr;
+    SmallVector<MachineOperand, 4> BrCond;
+    MachineInstr *LoopInductionVar = nullptr;
+    MachineInstr *LoopCompare = nullptr;
+  };
+  LoopInfo LI;
+
+  static char ID;
+
+  MachinePipeliner() : MachineFunctionPass(ID) {
+    initializeMachinePipelinerPass(*PassRegistry::getPassRegistry());
+  }
+
+  bool runOnMachineFunction(MachineFunction &MF) override;
+
+  void getAnalysisUsage(AnalysisUsage &AU) const override {
+    AU.addRequired<AAResultsWrapperPass>();
+    AU.addPreserved<AAResultsWrapperPass>();
+    AU.addRequired<MachineLoopInfo>();
+    AU.addRequired<MachineDominatorTree>();
+    AU.addRequired<LiveIntervals>();
+    MachineFunctionPass::getAnalysisUsage(AU);
+  }
+
+private:
+  void preprocessPhiNodes(MachineBasicBlock &B);
+  bool canPipelineLoop(MachineLoop &L);
+  bool scheduleLoop(MachineLoop &L);
+  bool swingModuloScheduler(MachineLoop &L);
+  void setPragmaPipelineOptions(MachineLoop &L);
+};
+
+/// This class builds the dependence graph for the instructions in a loop,
+/// and attempts to schedule the instructions using the SMS algorithm.
+class SwingSchedulerDAG : public ScheduleDAGInstrs {
+  MachinePipeliner &Pass;
+  /// The minimum initiation interval between iterations for this schedule.
+  unsigned MII = 0;
+  /// The maximum initiation interval between iterations for this schedule.
+  unsigned MAX_II = 0;
+  /// Set to true if a valid pipelined schedule is found for the loop.
+  bool Scheduled = false;
+  MachineLoop &Loop;
+  LiveIntervals &LIS;
+  const RegisterClassInfo &RegClassInfo;
+  unsigned II_setByPragma = 0;
+
+  /// A toplogical ordering of the SUnits, which is needed for changing
+  /// dependences and iterating over the SUnits.
+  ScheduleDAGTopologicalSort Topo;
+
+  struct NodeInfo {
+    int ASAP = 0;
+    int ALAP = 0;
+    int ZeroLatencyDepth = 0;
+    int ZeroLatencyHeight = 0;
+
+    NodeInfo() = default;
+  };
+  /// Computed properties for each node in the graph.
+  std::vector<NodeInfo> ScheduleInfo;
+
+  enum OrderKind { BottomUp = 0, TopDown = 1 };
+  /// Computed node ordering for scheduling.
+  SetVector<SUnit *> NodeOrder;
+
+  using NodeSetType = SmallVector<NodeSet, 8>;
+  using ValueMapTy = DenseMap<unsigned, unsigned>;
+  using MBBVectorTy = SmallVectorImpl<MachineBasicBlock *>;
+  using InstrMapTy = DenseMap<MachineInstr *, MachineInstr *>;
+
+  /// Instructions to change when emitting the final schedule.
+  DenseMap<SUnit *, std::pair<unsigned, int64_t>> InstrChanges;
+
+  /// We may create a new instruction, so remember it because it
+  /// must be deleted when the pass is finished.
+  SmallPtrSet<MachineInstr *, 4> NewMIs;
+
+  /// Ordered list of DAG postprocessing steps.
+  std::vector<std::unique_ptr<ScheduleDAGMutation>> Mutations;
+
+  /// Helper class to implement Johnson's circuit finding algorithm.
+  class Circuits {
+    std::vector<SUnit> &SUnits;
+    SetVector<SUnit *> Stack;
+    BitVector Blocked;
+    SmallVector<SmallPtrSet<SUnit *, 4>, 10> B;
+    SmallVector<SmallVector<int, 4>, 16> AdjK;
+    // Node to Index from ScheduleDAGTopologicalSort
+    std::vector<int> *Node2Idx;
+    unsigned NumPaths;
+    static unsigned MaxPaths;
+
+  public:
+    Circuits(std::vector<SUnit> &SUs, ScheduleDAGTopologicalSort &Topo)
+        : SUnits(SUs), Blocked(SUs.size()), B(SUs.size()), AdjK(SUs.size()) {
+      Node2Idx = new std::vector<int>(SUs.size());
+      unsigned Idx = 0;
+      for (const auto &NodeNum : Topo)
+        Node2Idx->at(NodeNum) = Idx++;
+    }
+
+    ~Circuits() { delete Node2Idx; }
+
+    /// Reset the data structures used in the circuit algorithm.
+    void reset() {
+      Stack.clear();
+      Blocked.reset();
+      B.assign(SUnits.size(), SmallPtrSet<SUnit *, 4>());
+      NumPaths = 0;
+    }
+
+    void createAdjacencyStructure(SwingSchedulerDAG *DAG);
+    bool circuit(int V, int S, NodeSetType &NodeSets, bool HasBackedge = false);
+    void unblock(int U);
+  };
+
+  struct CopyToPhiMutation : public ScheduleDAGMutation {
+    void apply(ScheduleDAGInstrs *DAG) override;
+  };
+
+public:
+  SwingSchedulerDAG(MachinePipeliner &P, MachineLoop &L, LiveIntervals &lis,
+                    const RegisterClassInfo &rci, unsigned II)
+      : ScheduleDAGInstrs(*P.MF, P.MLI, false), Pass(P), Loop(L), LIS(lis),
+        RegClassInfo(rci), II_setByPragma(II), Topo(SUnits, &ExitSU) {
+    P.MF->getSubtarget().getSMSMutations(Mutations);
+    if (SwpEnableCopyToPhi)
+      Mutations.push_back(llvm::make_unique<CopyToPhiMutation>());
+  }
+
+  void schedule() override;
+  void finishBlock() override;
+
+  /// Return true if the loop kernel has been scheduled.
+  bool hasNewSchedule() { return Scheduled; }
+
+  /// Return the earliest time an instruction may be scheduled.
+  int getASAP(SUnit *Node) { return ScheduleInfo[Node->NodeNum].ASAP; }
+
+  /// Return the latest time an instruction my be scheduled.
+  int getALAP(SUnit *Node) { return ScheduleInfo[Node->NodeNum].ALAP; }
+
+  /// The mobility function, which the number of slots in which
+  /// an instruction may be scheduled.
+  int getMOV(SUnit *Node) { return getALAP(Node) - getASAP(Node); }
+
+  /// The depth, in the dependence graph, for a node.
+  unsigned getDepth(SUnit *Node) { return Node->getDepth(); }
+
+  /// The maximum unweighted length of a path from an arbitrary node to the
+  /// given node in which each edge has latency 0
+  int getZeroLatencyDepth(SUnit *Node) {
+    return ScheduleInfo[Node->NodeNum].ZeroLatencyDepth;
+  }
+
+  /// The height, in the dependence graph, for a node.
+  unsigned getHeight(SUnit *Node) { return Node->getHeight(); }
+
+  /// The maximum unweighted length of a path from the given node to an
+  /// arbitrary node in which each edge has latency 0
+  int getZeroLatencyHeight(SUnit *Node) {
+    return ScheduleInfo[Node->NodeNum].ZeroLatencyHeight;
+  }
+
+  /// Return true if the dependence is a back-edge in the data dependence graph.
+  /// Since the DAG doesn't contain cycles, we represent a cycle in the graph
+  /// using an anti dependence from a Phi to an instruction.
+  bool isBackedge(SUnit *Source, const SDep &Dep) {
+    if (Dep.getKind() != SDep::Anti)
+      return false;
+    return Source->getInstr()->isPHI() || Dep.getSUnit()->getInstr()->isPHI();
+  }
+
+  bool isLoopCarriedDep(SUnit *Source, const SDep &Dep, bool isSucc = true);
+
+  /// The distance function, which indicates that operation V of iteration I
+  /// depends on operations U of iteration I-distance.
+  unsigned getDistance(SUnit *U, SUnit *V, const SDep &Dep) {
+    // Instructions that feed a Phi have a distance of 1. Computing larger
+    // values for arrays requires data dependence information.
+    if (V->getInstr()->isPHI() && Dep.getKind() == SDep::Anti)
+      return 1;
+    return 0;
+  }
+
+  void applyInstrChange(MachineInstr *MI, SMSchedule &Schedule);
+
+  void fixupRegisterOverlaps(std::deque<SUnit *> &Instrs);
+
+  /// Return the new base register that was stored away for the changed
+  /// instruction.
+  unsigned getInstrBaseReg(SUnit *SU) {
+    DenseMap<SUnit *, std::pair<unsigned, int64_t>>::iterator It =
+        InstrChanges.find(SU);
+    if (It != InstrChanges.end())
+      return It->second.first;
+    return 0;
+  }
+
+  void addMutation(std::unique_ptr<ScheduleDAGMutation> Mutation) {
+    Mutations.push_back(std::move(Mutation));
+  }
+
+  static bool classof(const ScheduleDAGInstrs *DAG) { return true; }
+
+private:
+  void addLoopCarriedDependences(AliasAnalysis *AA);
+  void updatePhiDependences();
+  void changeDependences();
+  unsigned calculateResMII();
+  unsigned calculateRecMII(NodeSetType &RecNodeSets);
+  void findCircuits(NodeSetType &NodeSets);
+  void fuseRecs(NodeSetType &NodeSets);
+  void removeDuplicateNodes(NodeSetType &NodeSets);
+  void computeNodeFunctions(NodeSetType &NodeSets);
+  void registerPressureFilter(NodeSetType &NodeSets);
+  void colocateNodeSets(NodeSetType &NodeSets);
+  void checkNodeSets(NodeSetType &NodeSets);
+  void groupRemainingNodes(NodeSetType &NodeSets);
+  void addConnectedNodes(SUnit *SU, NodeSet &NewSet,
+                         SetVector<SUnit *> &NodesAdded);
+  void computeNodeOrder(NodeSetType &NodeSets);
+  void checkValidNodeOrder(const NodeSetType &Circuits) const;
+  bool schedulePipeline(SMSchedule &Schedule);
+  void generatePipelinedLoop(SMSchedule &Schedule);
+  void generateProlog(SMSchedule &Schedule, unsigned LastStage,
+                      MachineBasicBlock *KernelBB, ValueMapTy *VRMap,
+                      MBBVectorTy &PrologBBs);
+  void generateEpilog(SMSchedule &Schedule, unsigned LastStage,
+                      MachineBasicBlock *KernelBB, ValueMapTy *VRMap,
+                      MBBVectorTy &EpilogBBs, MBBVectorTy &PrologBBs);
+  void generateExistingPhis(MachineBasicBlock *NewBB, MachineBasicBlock *BB1,
+                            MachineBasicBlock *BB2, MachineBasicBlock *KernelBB,
+                            SMSchedule &Schedule, ValueMapTy *VRMap,
+                            InstrMapTy &InstrMap, unsigned LastStageNum,
+                            unsigned CurStageNum, bool IsLast);
+  void generatePhis(MachineBasicBlock *NewBB, MachineBasicBlock *BB1,
+                    MachineBasicBlock *BB2, MachineBasicBlock *KernelBB,
+                    SMSchedule &Schedule, ValueMapTy *VRMap,
+                    InstrMapTy &InstrMap, unsigned LastStageNum,
+                    unsigned CurStageNum, bool IsLast);
+  void removeDeadInstructions(MachineBasicBlock *KernelBB,
+                              MBBVectorTy &EpilogBBs);
+  void splitLifetimes(MachineBasicBlock *KernelBB, MBBVectorTy &EpilogBBs,
+                      SMSchedule &Schedule);
+  void addBranches(MBBVectorTy &PrologBBs, MachineBasicBlock *KernelBB,
+                   MBBVectorTy &EpilogBBs, SMSchedule &Schedule,
+                   ValueMapTy *VRMap);
+  bool computeDelta(MachineInstr &MI, unsigned &Delta);
+  void updateMemOperands(MachineInstr &NewMI, MachineInstr &OldMI,
+                         unsigned Num);
+  MachineInstr *cloneInstr(MachineInstr *OldMI, unsigned CurStageNum,
+                           unsigned InstStageNum);
+  MachineInstr *cloneAndChangeInstr(MachineInstr *OldMI, unsigned CurStageNum,
+                                    unsigned InstStageNum,
+                                    SMSchedule &Schedule);
+  void updateInstruction(MachineInstr *NewMI, bool LastDef,
+                         unsigned CurStageNum, unsigned InstrStageNum,
+                         SMSchedule &Schedule, ValueMapTy *VRMap);
+  MachineInstr *findDefInLoop(unsigned Reg);
+  unsigned getPrevMapVal(unsigned StageNum, unsigned PhiStage, unsigned LoopVal,
+                         unsigned LoopStage, ValueMapTy *VRMap,
+                         MachineBasicBlock *BB);
+  void rewritePhiValues(MachineBasicBlock *NewBB, unsigned StageNum,
+                        SMSchedule &Schedule, ValueMapTy *VRMap,
+                        InstrMapTy &InstrMap);
+  void rewriteScheduledInstr(MachineBasicBlock *BB, SMSchedule &Schedule,
+                             InstrMapTy &InstrMap, unsigned CurStageNum,
+                             unsigned PhiNum, MachineInstr *Phi,
+                             unsigned OldReg, unsigned NewReg,
+                             unsigned PrevReg = 0);
+  bool canUseLastOffsetValue(MachineInstr *MI, unsigned &BasePos,
+                             unsigned &OffsetPos, unsigned &NewBase,
+                             int64_t &NewOffset);
+  void postprocessDAG();
+  /// Set the Minimum Initiation Interval for this schedule attempt.
+  void setMII(unsigned ResMII, unsigned RecMII);
+  /// Set the Maximum Initiation Interval for this schedule attempt.
+  void setMAX_II();
+};
+
+/// A NodeSet contains a set of SUnit DAG nodes with additional information
+/// that assigns a priority to the set.
+class NodeSet {
+  SetVector<SUnit *> Nodes;
+  bool HasRecurrence = false;
+  unsigned RecMII = 0;
+  int MaxMOV = 0;
+  unsigned MaxDepth = 0;
+  unsigned Colocate = 0;
+  SUnit *ExceedPressure = nullptr;
+  unsigned Latency = 0;
+
+public:
+  using iterator = SetVector<SUnit *>::const_iterator;
+
+  NodeSet() = default;
+  NodeSet(iterator S, iterator E) : Nodes(S, E), HasRecurrence(true) {
+    Latency = 0;
+    for (unsigned i = 0, e = Nodes.size(); i < e; ++i)
+      for (const SDep &Succ : Nodes[i]->Succs)
+        if (Nodes.count(Succ.getSUnit()))
+          Latency += Succ.getLatency();
+  }
+
+  bool insert(SUnit *SU) { return Nodes.insert(SU); }
+
+  void insert(iterator S, iterator E) { Nodes.insert(S, E); }
+
+  template <typename UnaryPredicate> bool remove_if(UnaryPredicate P) {
+    return Nodes.remove_if(P);
+  }
+
+  unsigned count(SUnit *SU) const { return Nodes.count(SU); }
+
+  bool hasRecurrence() { return HasRecurrence; };
+
+  unsigned size() const { return Nodes.size(); }
+
+  bool empty() const { return Nodes.empty(); }
+
+  SUnit *getNode(unsigned i) const { return Nodes[i]; };
+
+  void setRecMII(unsigned mii) { RecMII = mii; };
+
+  void setColocate(unsigned c) { Colocate = c; };
+
+  void setExceedPressure(SUnit *SU) { ExceedPressure = SU; }
+
+  bool isExceedSU(SUnit *SU) { return ExceedPressure == SU; }
+
+  int compareRecMII(NodeSet &RHS) { return RecMII - RHS.RecMII; }
+
+  int getRecMII() { return RecMII; }
+
+  /// Summarize node functions for the entire node set.
+  void computeNodeSetInfo(SwingSchedulerDAG *SSD) {
+    for (SUnit *SU : *this) {
+      MaxMOV = std::max(MaxMOV, SSD->getMOV(SU));
+      MaxDepth = std::max(MaxDepth, SSD->getDepth(SU));
+    }
+  }
+
+  unsigned getLatency() { return Latency; }
+
+  unsigned getMaxDepth() { return MaxDepth; }
+
+  void clear() {
+    Nodes.clear();
+    RecMII = 0;
+    HasRecurrence = false;
+    MaxMOV = 0;
+    MaxDepth = 0;
+    Colocate = 0;
+    ExceedPressure = nullptr;
+  }
+
+  operator SetVector<SUnit *> &() { return Nodes; }
+
+  /// Sort the node sets by importance. First, rank them by recurrence MII,
+  /// then by mobility (least mobile done first), and finally by depth.
+  /// Each node set may contain a colocate value which is used as the first
+  /// tie breaker, if it's set.
+  bool operator>(const NodeSet &RHS) const {
+    if (RecMII == RHS.RecMII) {
+      if (Colocate != 0 && RHS.Colocate != 0 && Colocate != RHS.Colocate)
+        return Colocate < RHS.Colocate;
+      if (MaxMOV == RHS.MaxMOV)
+        return MaxDepth > RHS.MaxDepth;
+      return MaxMOV < RHS.MaxMOV;
+    }
+    return RecMII > RHS.RecMII;
+  }
+
+  bool operator==(const NodeSet &RHS) const {
+    return RecMII == RHS.RecMII && MaxMOV == RHS.MaxMOV &&
+           MaxDepth == RHS.MaxDepth;
+  }
+
+  bool operator!=(const NodeSet &RHS) const { return !operator==(RHS); }
+
+  iterator begin() { return Nodes.begin(); }
+  iterator end() { return Nodes.end(); }
+  void print(raw_ostream &os) const;
+
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+  LLVM_DUMP_METHOD void dump() const;
+#endif
+};
+
+/// This class represents the scheduled code.  The main data structure is a
+/// map from scheduled cycle to instructions.  During scheduling, the
+/// data structure explicitly represents all stages/iterations.   When
+/// the algorithm finshes, the schedule is collapsed into a single stage,
+/// which represents instructions from different loop iterations.
+///
+/// The SMS algorithm allows negative values for cycles, so the first cycle
+/// in the schedule is the smallest cycle value.
+class SMSchedule {
+private:
+  /// Map from execution cycle to instructions.
+  DenseMap<int, std::deque<SUnit *>> ScheduledInstrs;
+
+  /// Map from instruction to execution cycle.
+  std::map<SUnit *, int> InstrToCycle;
+
+  /// Map for each register and the max difference between its uses and def.
+  /// The first element in the pair is the max difference in stages. The
+  /// second is true if the register defines a Phi value and loop value is
+  /// scheduled before the Phi.
+  std::map<unsigned, std::pair<unsigned, bool>> RegToStageDiff;
+
+  /// Keep track of the first cycle value in the schedule.  It starts
+  /// as zero, but the algorithm allows negative values.
+  int FirstCycle = 0;
+
+  /// Keep track of the last cycle value in the schedule.
+  int LastCycle = 0;
+
+  /// The initiation interval (II) for the schedule.
+  int InitiationInterval = 0;
+
+  /// Target machine information.
+  const TargetSubtargetInfo &ST;
+
+  /// Virtual register information.
+  MachineRegisterInfo &MRI;
+
+  std::unique_ptr<DFAPacketizer> Resources;
+
+public:
+  SMSchedule(MachineFunction *mf)
+      : ST(mf->getSubtarget()), MRI(mf->getRegInfo()),
+        Resources(ST.getInstrInfo()->CreateTargetScheduleState(ST)) {}
+
+  void reset() {
+    ScheduledInstrs.clear();
+    InstrToCycle.clear();
+    RegToStageDiff.clear();
+    FirstCycle = 0;
+    LastCycle = 0;
+    InitiationInterval = 0;
+  }
+
+  /// Set the initiation interval for this schedule.
+  void setInitiationInterval(int ii) { InitiationInterval = ii; }
+
+  /// Return the first cycle in the completed schedule.  This
+  /// can be a negative value.
+  int getFirstCycle() const { return FirstCycle; }
+
+  /// Return the last cycle in the finalized schedule.
+  int getFinalCycle() const { return FirstCycle + InitiationInterval - 1; }
+
+  /// Return the cycle of the earliest scheduled instruction in the dependence
+  /// chain.
+  int earliestCycleInChain(const SDep &Dep);
+
+  /// Return the cycle of the latest scheduled instruction in the dependence
+  /// chain.
+  int latestCycleInChain(const SDep &Dep);
+
+  void computeStart(SUnit *SU, int *MaxEarlyStart, int *MinLateStart,
+                    int *MinEnd, int *MaxStart, int II, SwingSchedulerDAG *DAG);
+  bool insert(SUnit *SU, int StartCycle, int EndCycle, int II);
+
+  /// Iterators for the cycle to instruction map.
+  using sched_iterator = DenseMap<int, std::deque<SUnit *>>::iterator;
+  using const_sched_iterator =
+      DenseMap<int, std::deque<SUnit *>>::const_iterator;
+
+  /// Return true if the instruction is scheduled at the specified stage.
+  bool isScheduledAtStage(SUnit *SU, unsigned StageNum) {
+    return (stageScheduled(SU) == (int)StageNum);
+  }
+
+  /// Return the stage for a scheduled instruction.  Return -1 if
+  /// the instruction has not been scheduled.
+  int stageScheduled(SUnit *SU) const {
+    std::map<SUnit *, int>::const_iterator it = InstrToCycle.find(SU);
+    if (it == InstrToCycle.end())
+      return -1;
+    return (it->second - FirstCycle) / InitiationInterval;
+  }
+
+  /// Return the cycle for a scheduled instruction. This function normalizes
+  /// the first cycle to be 0.
+  unsigned cycleScheduled(SUnit *SU) const {
+    std::map<SUnit *, int>::const_iterator it = InstrToCycle.find(SU);
+    assert(it != InstrToCycle.end() && "Instruction hasn't been scheduled.");
+    return (it->second - FirstCycle) % InitiationInterval;
+  }
+
+  /// Return the maximum stage count needed for this schedule.
+  unsigned getMaxStageCount() {
+    return (LastCycle - FirstCycle) / InitiationInterval;
+  }
+
+  /// Return the max. number of stages/iterations that can occur between a
+  /// register definition and its uses.
+  unsigned getStagesForReg(int Reg, unsigned CurStage) {
+    std::pair<unsigned, bool> Stages = RegToStageDiff[Reg];
+    if (CurStage > getMaxStageCount() && Stages.first == 0 && Stages.second)
+      return 1;
+    return Stages.first;
+  }
+
+  /// The number of stages for a Phi is a little different than other
+  /// instructions. The minimum value computed in RegToStageDiff is 1
+  /// because we assume the Phi is needed for at least 1 iteration.
+  /// This is not the case if the loop value is scheduled prior to the
+  /// Phi in the same stage.  This function returns the number of stages
+  /// or iterations needed between the Phi definition and any uses.
+  unsigned getStagesForPhi(int Reg) {
+    std::pair<unsigned, bool> Stages = RegToStageDiff[Reg];
+    if (Stages.second)
+      return Stages.first;
+    return Stages.first - 1;
+  }
+
+  /// Return the instructions that are scheduled at the specified cycle.
+  std::deque<SUnit *> &getInstructions(int cycle) {
+    return ScheduledInstrs[cycle];
+  }
+
+  bool isValidSchedule(SwingSchedulerDAG *SSD);
+  void finalizeSchedule(SwingSchedulerDAG *SSD);
+  void orderDependence(SwingSchedulerDAG *SSD, SUnit *SU,
+                       std::deque<SUnit *> &Insts);
+  bool isLoopCarried(SwingSchedulerDAG *SSD, MachineInstr &Phi);
+  bool isLoopCarriedDefOfUse(SwingSchedulerDAG *SSD, MachineInstr *Def,
+                             MachineOperand &MO);
+  void print(raw_ostream &os) const;
+  void dump() const;
+};
+
+} // end namespace llvm
+
+#endif // LLVM_LIB_CODEGEN_MACHINEPIPELINER_H
diff --git a/include/llvm/CodeGen/MachinePostDominators.h b/include/llvm/CodeGen/MachinePostDominators.h
index c6a41598ce32c2697085b57d8d7c02d1fa5e103f..b67e6b52ac8fee120c6bd02d444734155419c623 100644
--- a/include/llvm/CodeGen/MachinePostDominators.h
+++ b/include/llvm/CodeGen/MachinePostDominators.h
@@ -1,9 +1,8 @@
-//=- llvm/CodeGen/MachineDominators.h ----------------------------*- C++ -*-==//
+//===- llvm/CodeGen/MachinePostDominators.h ----------------------*- C++ -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/CodeGen/MachineRegionInfo.h b/include/llvm/CodeGen/MachineRegionInfo.h
index 8394b58d0a16c81600572f885031af9d99ccb707..6d9fb9b9100a8dd418685c416472ce804159608b 100644
--- a/include/llvm/CodeGen/MachineRegionInfo.h
+++ b/include/llvm/CodeGen/MachineRegionInfo.h
@@ -1,9 +1,8 @@
 //===- llvm/CodeGen/MachineRegionInfo.h -------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/CodeGen/MachineRegisterInfo.h b/include/llvm/CodeGen/MachineRegisterInfo.h
index fef010a23ef9d3b3e9477f4d73fa1d79c64c2392..d25cd98bc55dec5136c857f6f56d85816a36b322 100644
--- a/include/llvm/CodeGen/MachineRegisterInfo.h
+++ b/include/llvm/CodeGen/MachineRegisterInfo.h
@@ -1,9 +1,8 @@
 //===- llvm/CodeGen/MachineRegisterInfo.h -----------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/CodeGen/MachineSSAUpdater.h b/include/llvm/CodeGen/MachineSSAUpdater.h
index 5e91246b402ca5482ec9c4fb4ac9b641633a0981..0319ec77467171ccefb2ed093c2a5b114bcf6a8a 100644
--- a/include/llvm/CodeGen/MachineSSAUpdater.h
+++ b/include/llvm/CodeGen/MachineSSAUpdater.h
@@ -1,9 +1,8 @@
 //===- MachineSSAUpdater.h - Unstructured SSA Update Tool -------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/CodeGen/MachineScheduler.h b/include/llvm/CodeGen/MachineScheduler.h
index 4bc31ae7c61aca6eecb0a7d2349f2c21852d0dbc..705799877028141543ea8368eda809b3e2cd57d1 100644
--- a/include/llvm/CodeGen/MachineScheduler.h
+++ b/include/llvm/CodeGen/MachineScheduler.h
@@ -1,9 +1,8 @@
 //===- MachineScheduler.h - MachineInstr Scheduling Pass --------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/CodeGen/MachineTraceMetrics.h b/include/llvm/CodeGen/MachineTraceMetrics.h
index 9d8db393ca92a6e20c84fe15a17ffa1ba9b4156c..025989504177fa5fa15881704eecd6015bec8255 100644
--- a/include/llvm/CodeGen/MachineTraceMetrics.h
+++ b/include/llvm/CodeGen/MachineTraceMetrics.h
@@ -1,9 +1,8 @@
 //===- lib/CodeGen/MachineTraceMetrics.h - Super-scalar metrics -*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/CodeGen/MacroFusion.h b/include/llvm/CodeGen/MacroFusion.h
index a77226ddaf33ad2a0e09fdbb0ff960c214332798..3a140fe63fdee773cf896daa9b7956cfc2ebf097 100644
--- a/include/llvm/CodeGen/MacroFusion.h
+++ b/include/llvm/CodeGen/MacroFusion.h
@@ -1,9 +1,8 @@
 //===- MacroFusion.h - Macro Fusion -----------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/CodeGen/PBQP/CostAllocator.h b/include/llvm/CodeGen/PBQP/CostAllocator.h
index bde451ae1fccd24748e7b53e3043f4b742d0e9cf..0d6d8a31317bfbe4e23cbd3f9fbbbf55b18b4bcf 100644
--- a/include/llvm/CodeGen/PBQP/CostAllocator.h
+++ b/include/llvm/CodeGen/PBQP/CostAllocator.h
@@ -1,9 +1,8 @@
 //===- CostAllocator.h - PBQP Cost Allocator --------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/CodeGen/PBQP/Graph.h b/include/llvm/CodeGen/PBQP/Graph.h
index a6d88b057dcb51da30a19d2dfaa8b216633493ef..c2cd6dadae5f8b46b2a317d73464ff7d070930fd 100644
--- a/include/llvm/CodeGen/PBQP/Graph.h
+++ b/include/llvm/CodeGen/PBQP/Graph.h
@@ -1,9 +1,8 @@
 //===- Graph.h - PBQP Graph -------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/CodeGen/PBQP/Math.h b/include/llvm/CodeGen/PBQP/Math.h
index d1432a3053c4369c764f4771c997460dcf04f636..8b014ccbb07ba72655c1024b7184d8644a474140 100644
--- a/include/llvm/CodeGen/PBQP/Math.h
+++ b/include/llvm/CodeGen/PBQP/Math.h
@@ -1,9 +1,8 @@
 //===- Math.h - PBQP Vector and Matrix classes ------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/CodeGen/PBQP/ReductionRules.h b/include/llvm/CodeGen/PBQP/ReductionRules.h
index 21b99027970da085123e13d66aa4b2e1cf9f600e..51822d082badd82a18c962923bd7edc8906dc4a6 100644
--- a/include/llvm/CodeGen/PBQP/ReductionRules.h
+++ b/include/llvm/CodeGen/PBQP/ReductionRules.h
@@ -1,9 +1,8 @@
 //===- ReductionRules.h - Reduction Rules -----------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/CodeGen/PBQP/Solution.h b/include/llvm/CodeGen/PBQP/Solution.h
index 4d4379fbc2c2178d83eead0d091d71df28527ce1..d5b1474f0f4c7ea564b63ddc49a88ad76d8cd059 100644
--- a/include/llvm/CodeGen/PBQP/Solution.h
+++ b/include/llvm/CodeGen/PBQP/Solution.h
@@ -1,9 +1,8 @@
 //===- Solution.h - PBQP Solution -------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/CodeGen/PBQPRAConstraint.h b/include/llvm/CodeGen/PBQPRAConstraint.h
index 995467dc56d8672711c059eb9fc0c4315970b06a..876ab97a669fd6e15a85559c08a570c7a2a1be1b 100644
--- a/include/llvm/CodeGen/PBQPRAConstraint.h
+++ b/include/llvm/CodeGen/PBQPRAConstraint.h
@@ -1,9 +1,8 @@
-//===- RegAllocPBQP.h -------------------------------------------*- C++ -*-===//
+//===- llvm/CodeGen/PBQPRAConstraint.h --------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/CodeGen/ParallelCG.h b/include/llvm/CodeGen/ParallelCG.h
index dbf09ea31e20ca6bf372fe70f6498a831d6e5b8f..a44715d4fc4fa316db20093e397cf044187f19ce 100644
--- a/include/llvm/CodeGen/ParallelCG.h
+++ b/include/llvm/CodeGen/ParallelCG.h
@@ -1,9 +1,8 @@
 //===-- llvm/CodeGen/ParallelCG.h - Parallel code generation ----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/CodeGen/Passes.h b/include/llvm/CodeGen/Passes.h
index acf1ebb5bc8396a203e0dd501a5473412ecdde4c..c59473a3f3b44e516443acb0f1d4fe7730758b3c 100644
--- a/include/llvm/CodeGen/Passes.h
+++ b/include/llvm/CodeGen/Passes.h
@@ -1,9 +1,8 @@
 //===-- Passes.h - Target independent code generation passes ----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/CodeGen/PreISelIntrinsicLowering.h b/include/llvm/CodeGen/PreISelIntrinsicLowering.h
index b7f83e515b7e95acc0c510609bd1d1fbe6ca0b40..73d7d779e55b6737c369c4592e3fc7fc099b078a 100644
--- a/include/llvm/CodeGen/PreISelIntrinsicLowering.h
+++ b/include/llvm/CodeGen/PreISelIntrinsicLowering.h
@@ -1,9 +1,8 @@
 //===- PreISelIntrinsicLowering.h - Pre-ISel intrinsic lowering pass ------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/CodeGen/PseudoSourceValue.h b/include/llvm/CodeGen/PseudoSourceValue.h
index f66191bc9fb40621569049220bb724c2b91cca9c..4920f23fe837ff58f5db3bbcf22a3fb7aa115d9e 100644
--- a/include/llvm/CodeGen/PseudoSourceValue.h
+++ b/include/llvm/CodeGen/PseudoSourceValue.h
@@ -1,9 +1,8 @@
 //===-- llvm/CodeGen/PseudoSourceValue.h ------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/CodeGen/ReachingDefAnalysis.h b/include/llvm/CodeGen/ReachingDefAnalysis.h
index b21b745c8fd1c4f8fc871411558e92c3ffdd3372..a599fb62f5e2ae084ef0b631e23f31699c662f05 100644
--- a/include/llvm/CodeGen/ReachingDefAnalysis.h
+++ b/include/llvm/CodeGen/ReachingDefAnalysis.h
@@ -1,15 +1,14 @@
 //==--- llvm/CodeGen/ReachingDefAnalysis.h - Reaching Def Analysis -*- C++ -*---==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 /// \file Reaching Defs Analysis pass.
 ///
-/// This pass tracks for each instruction what is the “closest” reaching def of
+/// This pass tracks for each instruction what is the "closest" reaching def of
 /// a given register. It is used by BreakFalseDeps (for clearance calculation)
 /// and ExecutionDomainFix (for arbitrating conflicting domains).
 ///
diff --git a/include/llvm/CodeGen/RegAllocPBQP.h b/include/llvm/CodeGen/RegAllocPBQP.h
index ba9763077d09b044f6e453cdb4dce652e1cae8c8..f7f92248f4ce99d7895146b1f7a8d535c363ee8a 100644
--- a/include/llvm/CodeGen/RegAllocPBQP.h
+++ b/include/llvm/CodeGen/RegAllocPBQP.h
@@ -1,9 +1,8 @@
 //===- RegAllocPBQP.h -------------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/CodeGen/RegAllocRegistry.h b/include/llvm/CodeGen/RegAllocRegistry.h
index b518fbb9c9da66f677b581d8291cac198f4a05b0..7077aa30450ba7cc51281fa0072385b9062a8317 100644
--- a/include/llvm/CodeGen/RegAllocRegistry.h
+++ b/include/llvm/CodeGen/RegAllocRegistry.h
@@ -1,9 +1,8 @@
 //===- llvm/CodeGen/RegAllocRegistry.h --------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/CodeGen/RegisterClassInfo.h b/include/llvm/CodeGen/RegisterClassInfo.h
index 97113c575815b6a09667be5df22f18cd5716e234..14af5c4d090d4cc314e7d6fb8ab7464e97be16bb 100644
--- a/include/llvm/CodeGen/RegisterClassInfo.h
+++ b/include/llvm/CodeGen/RegisterClassInfo.h
@@ -1,9 +1,8 @@
 //===- RegisterClassInfo.h - Dynamic Register Class Info --------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/CodeGen/RegisterPressure.h b/include/llvm/CodeGen/RegisterPressure.h
index 79054b9e33b783cbf1fe2896c6c25ea36f5752e6..5bbaa03fd75158695c8015ce3bfb858fdbfa0fc5 100644
--- a/include/llvm/CodeGen/RegisterPressure.h
+++ b/include/llvm/CodeGen/RegisterPressure.h
@@ -1,9 +1,8 @@
 //===- RegisterPressure.h - Dynamic Register Pressure -----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -132,10 +131,6 @@ public:
   }
 };
 
-template <> struct isPodLike<PressureChange> {
-   static const bool value = true;
-};
-
 /// List of PressureChanges in order of increasing, unique PSetID.
 ///
 /// Use a small fixed number, because we can fit more PressureChanges in an
diff --git a/include/llvm/CodeGen/RegisterScavenging.h b/include/llvm/CodeGen/RegisterScavenging.h
index b6bd028a8cacbcc15f2d615f338d10af5079d9ea..478ea38e0febd11ef8769085da000e206e68966e 100644
--- a/include/llvm/CodeGen/RegisterScavenging.h
+++ b/include/llvm/CodeGen/RegisterScavenging.h
@@ -1,9 +1,8 @@
 //===- RegisterScavenging.h - Machine register scavenging -------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/CodeGen/RegisterUsageInfo.h b/include/llvm/CodeGen/RegisterUsageInfo.h
index efecc61d9c3039dbb7ef4c12278c78b565f64dce..33554550b9dcaba83d61dcdc1f1266c3c979b8d3 100644
--- a/include/llvm/CodeGen/RegisterUsageInfo.h
+++ b/include/llvm/CodeGen/RegisterUsageInfo.h
@@ -1,9 +1,8 @@
 //==- RegisterUsageInfo.h - Register Usage Informartion Storage --*- C++ -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/include/llvm/CodeGen/ResourcePriorityQueue.h b/include/llvm/CodeGen/ResourcePriorityQueue.h
index 8d582ee298b6ff1af3e720921cba260d8c6077f8..81587a3170ce1e9202f725ff6190f13257870c6f 100644
--- a/include/llvm/CodeGen/ResourcePriorityQueue.h
+++ b/include/llvm/CodeGen/ResourcePriorityQueue.h
@@ -1,9 +1,8 @@
 //===----- ResourcePriorityQueue.h - A DFA-oriented priority queue -------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/CodeGen/RuntimeLibcalls.h b/include/llvm/CodeGen/RuntimeLibcalls.h
index 28567a1ce437535f37ba51b019a550271e6509d5..f71f39e5bf033e262deaf841a40c7d4284f60a03 100644
--- a/include/llvm/CodeGen/RuntimeLibcalls.h
+++ b/include/llvm/CodeGen/RuntimeLibcalls.h
@@ -1,9 +1,8 @@
 //===-- CodeGen/RuntimeLibcalls.h - Runtime Library Calls -------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/CodeGen/SDNodeProperties.td b/include/llvm/CodeGen/SDNodeProperties.td
index 83bbab2fdc8de84129d24c49e2c0cd3429d4ceac..d25e0bda26a95dc6da01923244a640cb20e3809d 100644
--- a/include/llvm/CodeGen/SDNodeProperties.td
+++ b/include/llvm/CodeGen/SDNodeProperties.td
@@ -1,9 +1,8 @@
 //===- SDNodeProperties.td - Common code for DAG isels ---*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/CodeGen/ScheduleDAG.h b/include/llvm/CodeGen/ScheduleDAG.h
index 0870d67db390c4dfee649df311d8e633ce9e49bc..68614dded8179935f7cea6a67c297d9f2b33f8af 100644
--- a/include/llvm/CodeGen/ScheduleDAG.h
+++ b/include/llvm/CodeGen/ScheduleDAG.h
@@ -1,9 +1,8 @@
 //===- llvm/CodeGen/ScheduleDAG.h - Common Base Class -----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -239,9 +238,6 @@ class TargetRegisterInfo;
     void dump(const TargetRegisterInfo *TRI = nullptr) const;
   };
 
-  template <>
-  struct isPodLike<SDep> { static const bool value = true; };
-
   /// Scheduling unit. This is a node in the scheduling DAG.
   class SUnit {
   private:
diff --git a/include/llvm/CodeGen/ScheduleDAGInstrs.h b/include/llvm/CodeGen/ScheduleDAGInstrs.h
index daad18125db92f823b83ea2f01622e9f03e7f8ac..fa7f886a0d4d7bd6f6f0fb3a447fd16e464680b4 100644
--- a/include/llvm/CodeGen/ScheduleDAGInstrs.h
+++ b/include/llvm/CodeGen/ScheduleDAGInstrs.h
@@ -1,9 +1,8 @@
 //===- ScheduleDAGInstrs.h - MachineInstr Scheduling ------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/CodeGen/ScheduleDAGMutation.h b/include/llvm/CodeGen/ScheduleDAGMutation.h
index 5c236427e0b8cb76c9194e80acc37a93f0ca299a..d1dd72859a380ebc7a81d1d4e043cb6fab852764 100644
--- a/include/llvm/CodeGen/ScheduleDAGMutation.h
+++ b/include/llvm/CodeGen/ScheduleDAGMutation.h
@@ -1,9 +1,8 @@
 //===- ScheduleDAGMutation.h - MachineInstr Scheduling ----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/CodeGen/ScheduleDFS.h b/include/llvm/CodeGen/ScheduleDFS.h
index 3ecc033ac35a8e6079fe2382e4acf1bf74c80915..d60deab95f5dc3a7abb2208e1fa9b9309df862f8 100644
--- a/include/llvm/CodeGen/ScheduleDFS.h
+++ b/include/llvm/CodeGen/ScheduleDFS.h
@@ -1,9 +1,8 @@
-//===- ScheduleDAGILP.h - ILP metric for ScheduleDAGInstrs ------*- C++ -*-===//
+//===- ScheduleDFS.h - ILP metric for ScheduleDAGInstrs ---------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/CodeGen/ScheduleHazardRecognizer.h b/include/llvm/CodeGen/ScheduleHazardRecognizer.h
index ace4a2d836ca81d2fd72dc668033d16b3955d226..37590f496ca296fae6c1f1f223c063caa71bbe2b 100644
--- a/include/llvm/CodeGen/ScheduleHazardRecognizer.h
+++ b/include/llvm/CodeGen/ScheduleHazardRecognizer.h
@@ -1,9 +1,8 @@
 //=- llvm/CodeGen/ScheduleHazardRecognizer.h - Scheduling Support -*- C++ -*-=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/CodeGen/SchedulerRegistry.h b/include/llvm/CodeGen/SchedulerRegistry.h
index fbe559f25556c86f09999b82601622c309228382..0ccfaafd9e50255e7f0ec693b80b91b496590924 100644
--- a/include/llvm/CodeGen/SchedulerRegistry.h
+++ b/include/llvm/CodeGen/SchedulerRegistry.h
@@ -1,9 +1,8 @@
 //===- llvm/CodeGen/SchedulerRegistry.h -------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/CodeGen/ScoreboardHazardRecognizer.h b/include/llvm/CodeGen/ScoreboardHazardRecognizer.h
index 3f75d108f282501891896708d2d5a2a729f7e34c..ac67f3008fa7f6ede87a7dd5a5cf4e17c0218f08 100644
--- a/include/llvm/CodeGen/ScoreboardHazardRecognizer.h
+++ b/include/llvm/CodeGen/ScoreboardHazardRecognizer.h
@@ -1,9 +1,8 @@
 //=- llvm/CodeGen/ScoreboardHazardRecognizer.h - Schedule Support -*- C++ -*-=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/CodeGen/SelectionDAG.h b/include/llvm/CodeGen/SelectionDAG.h
index 8093fdab5491eb10787fb091b0962022c09abdcc..dfd1d49a5d1483224e1bf7855c2c6b0cb34889ba 100644
--- a/include/llvm/CodeGen/SelectionDAG.h
+++ b/include/llvm/CodeGen/SelectionDAG.h
@@ -1,9 +1,8 @@
 //===- llvm/CodeGen/SelectionDAG.h - InstSelection DAG ----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -308,6 +307,9 @@ public:
         : DAGUpdateListener(DAG), Callback(std::move(Callback)) {}
 
     void NodeDeleted(SDNode *N, SDNode *E) override { Callback(N, E); }
+
+   private:
+    virtual void anchor();
   };
 
   /// When true, additional steps are taken to
@@ -989,10 +991,6 @@ public:
 
   /// Gets a node for an atomic op, produces result (if relevant)
   /// and chain and takes 2 operands.
-  SDValue getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDValue Chain,
-                    SDValue Ptr, SDValue Val, const Value *PtrVal,
-                    unsigned Alignment, AtomicOrdering Ordering,
-                    SyncScope::ID SSID);
   SDValue getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDValue Chain,
                     SDValue Ptr, SDValue Val, MachineMemOperand *MMO);
 
@@ -1128,6 +1126,13 @@ public:
   /// Expand the specified \c ISD::VACOPY node as the Legalize pass would.
   SDValue expandVACopy(SDNode *Node);
 
+  /// Returs an GlobalAddress of the function from the current module with
+  /// name matching the given ExternalSymbol. Additionally can provide the
+  /// matched function.
+  /// Panics the function doesn't exists.
+  SDValue getSymbolFunctionGlobalAddress(SDValue Op,
+                                         Function **TargetFunction = nullptr);
+
   /// *Mutate* the specified node in-place to have the
   /// specified operands.  If the resultant node already exists in the DAG,
   /// this does not modify the specified node, instead it returns the node that
@@ -1144,6 +1149,11 @@ public:
                                SDValue Op3, SDValue Op4, SDValue Op5);
   SDNode *UpdateNodeOperands(SDNode *N, ArrayRef<SDValue> Ops);
 
+  /// Creates a new TokenFactor containing \p Vals. If \p Vals contains 64k
+  /// values or more, move values into new TokenFactors in 64k-1 blocks, until
+  /// the final TokenFactor has less than 64k operands.
+  SDValue getTokenFactor(const SDLoc &DL, SmallVectorImpl<SDValue> &Vals);
+
   /// *Mutate* the specified machine node's memory references to the provided
   /// list.
   void setNodeMemRefs(MachineSDNode *N,
@@ -1348,21 +1358,20 @@ public:
   /// with this SelectionDAG.
   bool hasDebugValues() const { return !DbgInfo->empty(); }
 
-  SDDbgInfo::DbgIterator DbgBegin() { return DbgInfo->DbgBegin(); }
-  SDDbgInfo::DbgIterator DbgEnd()   { return DbgInfo->DbgEnd(); }
+  SDDbgInfo::DbgIterator DbgBegin() const { return DbgInfo->DbgBegin(); }
+  SDDbgInfo::DbgIterator DbgEnd() const  { return DbgInfo->DbgEnd(); }
 
-  SDDbgInfo::DbgIterator ByvalParmDbgBegin() {
+  SDDbgInfo::DbgIterator ByvalParmDbgBegin() const {
     return DbgInfo->ByvalParmDbgBegin();
   }
-
-  SDDbgInfo::DbgIterator ByvalParmDbgEnd()   {
+  SDDbgInfo::DbgIterator ByvalParmDbgEnd() const {
     return DbgInfo->ByvalParmDbgEnd();
   }
 
-  SDDbgInfo::DbgLabelIterator DbgLabelBegin() {
+  SDDbgInfo::DbgLabelIterator DbgLabelBegin() const {
     return DbgInfo->DbgLabelBegin();
   }
-  SDDbgInfo::DbgLabelIterator DbgLabelEnd() {
+  SDDbgInfo::DbgLabelIterator DbgLabelEnd() const {
     return DbgInfo->DbgLabelEnd();
   }
 
@@ -1385,11 +1394,11 @@ public:
                            const SDNode *N2);
 
   SDValue FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL, EVT VT,
-                                 SDNode *Cst1, SDNode *Cst2);
+                                 SDNode *N1, SDNode *N2);
 
   SDValue FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL, EVT VT,
-                                 const ConstantSDNode *Cst1,
-                                 const ConstantSDNode *Cst2);
+                                 const ConstantSDNode *C1,
+                                 const ConstantSDNode *C2);
 
   SDValue FoldConstantVectorArithmetic(unsigned Opcode, const SDLoc &DL, EVT VT,
                                        ArrayRef<SDValue> Ops,
@@ -1432,18 +1441,6 @@ public:
   KnownBits computeKnownBits(SDValue Op, const APInt &DemandedElts,
                              unsigned Depth = 0) const;
 
-  /// \copydoc SelectionDAG::computeKnownBits(SDValue,unsigned)
-  void computeKnownBits(SDValue Op, KnownBits &Known,
-                        unsigned Depth = 0) const {
-    Known = computeKnownBits(Op, Depth);
-  }
-
-  /// \copydoc SelectionDAG::computeKnownBits(SDValue,const APInt&,unsigned)
-  void computeKnownBits(SDValue Op, KnownBits &Known, const APInt &DemandedElts,
-                        unsigned Depth = 0) const {
-    Known = computeKnownBits(Op, DemandedElts, Depth);
-  }
-
   /// Used to represent the possible overflow behavior of an operation.
   /// Never: the operation cannot overflow.
   /// Always: the operation will always overflow.
diff --git a/include/llvm/CodeGen/SelectionDAGAddressAnalysis.h b/include/llvm/CodeGen/SelectionDAGAddressAnalysis.h
index 2b2c48d57bc0fac3a04f9dbf09491194ab843c68..f168b846fdaf4232690805a74186e359f2fe77b7 100644
--- a/include/llvm/CodeGen/SelectionDAGAddressAnalysis.h
+++ b/include/llvm/CodeGen/SelectionDAGAddressAnalysis.h
@@ -1,9 +1,8 @@
 //===- SelectionDAGAddressAnalysis.h - DAG Address Analysis -----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -60,6 +59,9 @@ public:
 
   /// Parses tree in Ptr for base, index, offset addresses.
   static BaseIndexOffset match(const LSBaseSDNode *N, const SelectionDAG &DAG);
+
+  void print(raw_ostream& OS) const;
+  void dump() const;
 };
 
 } // end namespace llvm
diff --git a/include/llvm/CodeGen/SelectionDAGISel.h b/include/llvm/CodeGen/SelectionDAGISel.h
index 6758c55c696afa4b8030b177d3b0d82acb1ce849..2acb9226d7d7b118e2781b1f10a0795678d78b8a 100644
--- a/include/llvm/CodeGen/SelectionDAGISel.h
+++ b/include/llvm/CodeGen/SelectionDAGISel.h
@@ -1,9 +1,8 @@
 //===-- llvm/CodeGen/SelectionDAGISel.h - Common Base Class------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -303,7 +302,7 @@ public:
 private:
 
   // Calls to these functions are generated by tblgen.
-  void Select_INLINEASM(SDNode *N);
+  void Select_INLINEASM(SDNode *N, bool Branch);
   void Select_READ_REGISTER(SDNode *Op);
   void Select_WRITE_REGISTER(SDNode *Op);
   void Select_UNDEF(SDNode *N);
diff --git a/include/llvm/CodeGen/SelectionDAGNodes.h b/include/llvm/CodeGen/SelectionDAGNodes.h
index 2931717b72deb1dafe45c442a5e904a57ea84fc8..c0dd9d1e12f0ff60884d883417a14121f87e02b5 100644
--- a/include/llvm/CodeGen/SelectionDAGNodes.h
+++ b/include/llvm/CodeGen/SelectionDAGNodes.h
@@ -1,9 +1,8 @@
 //===- llvm/CodeGen/SelectionDAGNodes.h - SelectionDAG Nodes ----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -184,6 +183,7 @@ public:
   inline unsigned getNumOperands() const;
   inline const SDValue &getOperand(unsigned i) const;
   inline uint64_t getConstantOperandVal(unsigned i) const;
+  inline const APInt &getConstantOperandAPInt(unsigned i) const;
   inline bool isTargetMemoryOpcode() const;
   inline bool isTargetOpcode() const;
   inline bool isMachineOpcode() const;
@@ -232,7 +232,6 @@ template<> struct DenseMapInfo<SDValue> {
     return LHS == RHS;
   }
 };
-template <> struct isPodLike<SDValue> { static const bool value = true; };
 
 /// Allow casting operators to work directly on
 /// SDValues as if they were SDNode*'s.
@@ -898,9 +897,17 @@ public:
   /// Return the number of values used by this operation.
   unsigned getNumOperands() const { return NumOperands; }
 
+  /// Return the maximum number of operands that a SDNode can hold.
+  static constexpr size_t getMaxNumOperands() {
+    return std::numeric_limits<decltype(SDNode::NumOperands)>::max();
+  }
+
   /// Helper method returns the integer value of a ConstantSDNode operand.
   inline uint64_t getConstantOperandVal(unsigned Num) const;
 
+  /// Helper method returns the APInt of a ConstantSDNode operand.
+  inline const APInt &getConstantOperandAPInt(unsigned Num) const;
+
   const SDValue &getOperand(unsigned Num) const {
     assert(Num < NumOperands && "Invalid child # of SDNode!");
     return OperandList[Num];
@@ -1128,6 +1135,10 @@ inline uint64_t SDValue::getConstantOperandVal(unsigned i) const {
   return Node->getConstantOperandVal(i);
 }
 
+inline const APInt &SDValue::getConstantOperandAPInt(unsigned i) const {
+  return Node->getConstantOperandAPInt(i);
+}
+
 inline bool SDValue::isTargetOpcode() const {
   return Node->isTargetOpcode();
 }
@@ -1356,6 +1367,8 @@ public:
            N->getOpcode() == ISD::ATOMIC_LOAD_MAX     ||
            N->getOpcode() == ISD::ATOMIC_LOAD_UMIN    ||
            N->getOpcode() == ISD::ATOMIC_LOAD_UMAX    ||
+           N->getOpcode() == ISD::ATOMIC_LOAD_FADD    ||
+           N->getOpcode() == ISD::ATOMIC_LOAD_FSUB    ||
            N->getOpcode() == ISD::ATOMIC_LOAD         ||
            N->getOpcode() == ISD::ATOMIC_STORE        ||
            N->getOpcode() == ISD::MLOAD               ||
@@ -1408,6 +1421,8 @@ public:
            N->getOpcode() == ISD::ATOMIC_LOAD_MAX     ||
            N->getOpcode() == ISD::ATOMIC_LOAD_UMIN    ||
            N->getOpcode() == ISD::ATOMIC_LOAD_UMAX    ||
+           N->getOpcode() == ISD::ATOMIC_LOAD_FADD    ||
+           N->getOpcode() == ISD::ATOMIC_LOAD_FSUB    ||
            N->getOpcode() == ISD::ATOMIC_LOAD         ||
            N->getOpcode() == ISD::ATOMIC_STORE;
   }
@@ -1536,6 +1551,10 @@ uint64_t SDNode::getConstantOperandVal(unsigned Num) const {
   return cast<ConstantSDNode>(getOperand(Num))->getZExtValue();
 }
 
+const APInt &SDNode::getConstantOperandAPInt(unsigned Num) const {
+  return cast<ConstantSDNode>(getOperand(Num))->getAPIntValue();
+}
+
 class ConstantFPSDNode : public SDNode {
   friend class SelectionDAG;
 
@@ -1614,9 +1633,9 @@ ConstantSDNode *isConstOrConstSplat(SDValue N, bool AllowUndefs = false);
 ConstantFPSDNode *isConstOrConstSplatFP(SDValue N, bool AllowUndefs = false);
 
 /// Return true if the value is a constant 0 integer or a splatted vector of
-/// a constant 0 integer (with no undefs).
+/// a constant 0 integer (with no undefs by default).
 /// Build vector implicit truncation is not an issue for null values.
-bool isNullOrNullSplat(SDValue V);
+bool isNullOrNullSplat(SDValue V, bool AllowUndefs = false);
 
 /// Return true if the value is a constant 1 integer or a splatted vector of a
 /// constant 1 integer (with no undefs).
@@ -2487,15 +2506,18 @@ namespace ISD {
 
   /// Attempt to match a unary predicate against a scalar/splat constant or
   /// every element of a constant BUILD_VECTOR.
+  /// If AllowUndef is true, then UNDEF elements will pass nullptr to Match.
   bool matchUnaryPredicate(SDValue Op,
-                           std::function<bool(ConstantSDNode *)> Match);
+                           std::function<bool(ConstantSDNode *)> Match,
+                           bool AllowUndefs = false);
 
   /// Attempt to match a binary predicate against a pair of scalar/splat
   /// constants or every element of a pair of constant BUILD_VECTORs.
+  /// If AllowUndef is true, then UNDEF elements will pass nullptr to Match.
   bool matchBinaryPredicate(
       SDValue LHS, SDValue RHS,
-      std::function<bool(ConstantSDNode *, ConstantSDNode *)> Match);
-
+      std::function<bool(ConstantSDNode *, ConstantSDNode *)> Match,
+      bool AllowUndefs = false);
 } // end namespace ISD
 
 } // end namespace llvm
diff --git a/include/llvm/CodeGen/SelectionDAGTargetInfo.h b/include/llvm/CodeGen/SelectionDAGTargetInfo.h
index 45c1df48a5e6f68af4af26a86771ec5943d81277..7c9f57beb660b6dd8210ddca1a6875e69f16aa59 100644
--- a/include/llvm/CodeGen/SelectionDAGTargetInfo.h
+++ b/include/llvm/CodeGen/SelectionDAGTargetInfo.h
@@ -1,9 +1,8 @@
 //==- llvm/CodeGen/SelectionDAGTargetInfo.h - SelectionDAG Info --*- C++ -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/CodeGen/SlotIndexes.h b/include/llvm/CodeGen/SlotIndexes.h
index 8c8a7be459fd5a605c23b3b16b30fda12e22ff06..63461d61bf3485532a53d095270f1f6206da9e1e 100644
--- a/include/llvm/CodeGen/SlotIndexes.h
+++ b/include/llvm/CodeGen/SlotIndexes.h
@@ -1,9 +1,8 @@
 //===- llvm/CodeGen/SlotIndexes.h - Slot indexes representation -*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -302,8 +301,6 @@ class raw_ostream;
     }
   };
 
-  template <> struct isPodLike<SlotIndex> { static const bool value = true; };
-
   inline raw_ostream& operator<<(raw_ostream &os, SlotIndex li) {
     li.print(os);
     return os;
diff --git a/include/llvm/CodeGen/StackMaps.h b/include/llvm/CodeGen/StackMaps.h
index 8be9ae378557eafe01c4321e0b0d6c4dd0e98146..d7d88de6f68258befb714a54a840690a4fc9856e 100644
--- a/include/llvm/CodeGen/StackMaps.h
+++ b/include/llvm/CodeGen/StackMaps.h
@@ -1,9 +1,8 @@
 //===- StackMaps.h - StackMaps ----------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/CodeGen/StackProtector.h b/include/llvm/CodeGen/StackProtector.h
index a506ac636a17a21f276b08737819582d66cc522a..ed52db3e6269ba68619159b5ea232f7e3da06d8d 100644
--- a/include/llvm/CodeGen/StackProtector.h
+++ b/include/llvm/CodeGen/StackProtector.h
@@ -1,9 +1,8 @@
 //===- StackProtector.h - Stack Protector Insertion -------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/CodeGen/TailDuplicator.h b/include/llvm/CodeGen/TailDuplicator.h
index be6562c85f2ef69584dfb588f06e67f7da0fbdd7..358798d5ed601413064282a2f93031e021e08e28 100644
--- a/include/llvm/CodeGen/TailDuplicator.h
+++ b/include/llvm/CodeGen/TailDuplicator.h
@@ -1,9 +1,8 @@
 //===- llvm/CodeGen/TailDuplicator.h ----------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/CodeGen/TargetCallingConv.h b/include/llvm/CodeGen/TargetCallingConv.h
index 7d138f5851713d3eff32bc9417502509ee75f965..f82c05dc82de0e87567f124ed555208cf38946d0 100644
--- a/include/llvm/CodeGen/TargetCallingConv.h
+++ b/include/llvm/CodeGen/TargetCallingConv.h
@@ -1,9 +1,8 @@
 //===-- llvm/CodeGen/TargetCallingConv.h - Calling Convention ---*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/CodeGen/TargetFrameLowering.h b/include/llvm/CodeGen/TargetFrameLowering.h
index f8effee998e3be12c0a0d094a316eee6df493bb7..754ee5cba01f541df55cca62974ae78671f1d2bf 100644
--- a/include/llvm/CodeGen/TargetFrameLowering.h
+++ b/include/llvm/CodeGen/TargetFrameLowering.h
@@ -1,9 +1,8 @@
 //===-- llvm/CodeGen/TargetFrameLowering.h ----------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -207,8 +206,11 @@ public:
     return false;
   }
 
-  /// Return true if the target needs to disable frame pointer elimination.
-  virtual bool noFramePointerElim(const MachineFunction &MF) const;
+  /// Return true if the target wants to keep the frame pointer regardless of
+  /// the function attribute "frame-pointer".
+  virtual bool keepFramePointer(const MachineFunction &MF) const {
+    return false;
+  }
 
   /// hasFP - Return true if the specified function should have a dedicated
   /// frame pointer register. For most targets this is true only if the function
@@ -259,6 +261,17 @@ public:
     return getFrameIndexReference(MF, FI, FrameReg);
   }
 
+  /// getNonLocalFrameIndexReference - This method returns the offset used to
+  /// reference a frame index location. The offset can be from either FP/BP/SP
+  /// based on which base register is returned by llvm.localaddress.
+  virtual int getNonLocalFrameIndexReference(const MachineFunction &MF,
+                                       int FI) const {
+    // By default, dispatch to getFrameIndexReference. Interested targets can
+    // override this.
+    unsigned FrameReg;
+    return getFrameIndexReference(MF, FI, FrameReg);
+  }
+
   /// This method determines which of the registers reported by
   /// TargetRegisterInfo::getCalleeSavedRegs() should actually get saved.
   /// The default implementation checks populates the \p SavedRegs bitset with
diff --git a/include/llvm/CodeGen/TargetInstrInfo.h b/include/llvm/CodeGen/TargetInstrInfo.h
index 961b90e9bc12d2e10f1537c53a38c786256f439c..b732be625de5967e0bd6c4a77803ec3946ce12e3 100644
--- a/include/llvm/CodeGen/TargetInstrInfo.h
+++ b/include/llvm/CodeGen/TargetInstrInfo.h
@@ -1,9 +1,8 @@
 //===- llvm/CodeGen/TargetInstrInfo.h - Instruction Info --------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -429,6 +428,13 @@ public:
 
     RegSubRegPair(unsigned Reg = 0, unsigned SubReg = 0)
         : Reg(Reg), SubReg(SubReg) {}
+
+    bool operator==(const RegSubRegPair& P) const {
+      return Reg == P.Reg && SubReg == P.SubReg;
+    }
+    bool operator!=(const RegSubRegPair& P) const {
+      return !(*this == P);
+    }
   };
 
   /// A pair composed of a pair of a register and a sub-register index,
diff --git a/include/llvm/CodeGen/TargetLowering.h b/include/llvm/CodeGen/TargetLowering.h
index d4e6da80a4c8f8cc23ec2a7e888d3e7a0daadbc6..fa71024a25f38a9b36e40806518c15364da43cab 100644
--- a/include/llvm/CodeGen/TargetLowering.h
+++ b/include/llvm/CodeGen/TargetLowering.h
@@ -1,9 +1,8 @@
 //===- llvm/CodeGen/TargetLowering.h - Target Lowering Info -----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
@@ -195,7 +194,11 @@ public:
           IsNest(false), IsByVal(false), IsInAlloca(false), IsReturned(false),
           IsSwiftSelf(false), IsSwiftError(false) {}
 
-    void setAttributes(ImmutableCallSite *CS, unsigned ArgIdx);
+    void setAttributes(const CallBase *Call, unsigned ArgIdx);
+
+    void setAttributes(ImmutableCallSite *CS, unsigned ArgIdx) {
+      return setAttributes(cast<CallBase>(CS->getInstruction()), ArgIdx);
+    }
   };
   using ArgListTy = std::vector<ArgListEntry>;
 
@@ -643,6 +646,13 @@ public:
     return RepRegClassCostForVT[VT.SimpleTy];
   }
 
+  /// Return true if SHIFT instructions should be expanded to SHIFT_PARTS
+  /// instructions, and false if a library call is preferred (e.g for code-size
+  /// reasons).
+  virtual bool shouldExpandShift(SelectionDAG &DAG, SDNode *N) const {
+    return true;
+  }
+
   /// Return true if the target has native support for the specified value type.
   /// This means that it has a register that directly holds it without
   /// promotions or expansions.
@@ -830,6 +840,7 @@ public:
     default:
       llvm_unreachable("Unexpected fixed point operation.");
     case ISD::SMULFIX:
+    case ISD::UMULFIX:
       Supported = isSupportedFixedPointOperation(Op, VT, Scale);
       break;
     }
@@ -1515,7 +1526,7 @@ public:
   /// performs validation and error handling, returns the function. Otherwise,
   /// returns nullptr. Must be previously inserted by insertSSPDeclarations.
   /// Should be used only when getIRStackGuard returns nullptr.
-  virtual Value *getSSPStackGuardCheck(const Module &M) const;
+  virtual Function *getSSPStackGuardCheck(const Module &M) const;
 
 protected:
   Value *getDefaultSafeStackPointerLocation(IRBuilder<> &IRB,
@@ -1716,8 +1727,9 @@ public:
 
   /// Returns how the IR-level AtomicExpand pass should expand the given
   /// AtomicRMW, if at all. Default is to never expand.
-  virtual AtomicExpansionKind shouldExpandAtomicRMWInIR(AtomicRMWInst *) const {
-    return AtomicExpansionKind::None;
+  virtual AtomicExpansionKind shouldExpandAtomicRMWInIR(AtomicRMWInst *RMW) const {
+    return RMW->isFloatingPointOperation() ?
+      AtomicExpansionKind::CmpXChg : AtomicExpansionKind::None;
   }
 
   /// On some platforms, an AtomicRMW that never actually modifies the value
@@ -1762,6 +1774,8 @@ public:
       Action != TypeSplitVector;
   }
 
+  virtual bool isProfitableToCombineMinNumMaxNum(EVT VT) const { return true; }
+
   /// Return true if a select of constants (select Cond, C1, C2) should be
   /// transformed into simple math ops with the condition value. For example:
   /// select Cond, C1, C1-1 --> add (zext Cond), C1-1
@@ -2155,6 +2169,8 @@ public:
     case ISD::UADDO:
     case ISD::ADDC:
     case ISD::ADDE:
+    case ISD::SADDSAT:
+    case ISD::UADDSAT:
     case ISD::FMINNUM:
     case ISD::FMAXNUM:
     case ISD::FMINIMUM:
@@ -2268,6 +2284,16 @@ public:
     return false;
   }
 
+  /// Return true if sinking I's operands to the same basic block as I is
+  /// profitable, e.g. because the operands can be folded into a target
+  /// instruction during instruction selection. After calling the function
+  /// \p Ops contains the Uses to sink ordered by dominance (dominating users
+  /// come first).
+  virtual bool shouldSinkOperands(Instruction *I,
+                                  SmallVectorImpl<Use *> &Ops) const {
+    return false;
+  }
+
   /// Return true if the target supplies and combines to a paired load
   /// two loaded values of type LoadedType next to each other in memory.
   /// RequiredAlignment gives the minimal alignment constraints that must be met
@@ -2407,6 +2433,12 @@ public:
     return false;
   }
 
+  /// Try to convert an extract element of a vector binary operation into an
+  /// extract element followed by a scalar operation.
+  virtual bool shouldScalarizeBinop(SDValue VecOp) const {
+    return false;
+  }
+
   // Return true if it is profitable to use a scalar input to a BUILD_VECTOR
   // even if the vector itself has multiple uses.
   virtual bool aggressivelyPreferBuildVectorSources(EVT VecVT) const {
@@ -2889,16 +2921,6 @@ public:
   bool ShrinkDemandedOp(SDValue Op, unsigned BitWidth, const APInt &Demanded,
                         TargetLoweringOpt &TLO) const;
 
-  /// Helper for SimplifyDemandedBits that can simplify an operation with
-  /// multiple uses.  This function simplifies operand \p OpIdx of \p User and
-  /// then updates \p User with the simplified version. No other uses of
-  /// \p OpIdx are updated. If \p User is the only user of \p OpIdx, this
-  /// function behaves exactly like function SimplifyDemandedBits declared
-  /// below except that it also updates the DAG by calling
-  /// DCI.CommitTargetLoweringOpt.
-  bool SimplifyDemandedBits(SDNode *User, unsigned OpIdx, const APInt &Demanded,
-                            DAGCombinerInfo &DCI, TargetLoweringOpt &TLO) const;
-
   /// Look at Op.  At this point, we know that only the DemandedBits bits of the
   /// result of Op are ever used downstream.  If we can use this information to
   /// simplify Op, create a new simplified DAG node and return true, returning
@@ -3090,6 +3112,15 @@ public:
     return true;
   }
 
+  /// Return true if it is profitable to fold a pair of shifts into a mask.
+  /// This is usually true on most targets. But some targets, like Thumb1,
+  /// have immediate shift instructions, but no immediate "and" instruction;
+  /// this makes the fold unprofitable.
+  virtual bool shouldFoldShiftPairToMask(const SDNode *N,
+                                         CombineLevel Level) const {
+    return true;
+  }
+
   // Return true if it is profitable to combine a BUILD_VECTOR with a stride-pattern
   // to a shuffle and a truncate.
   // Example of such a combine:
@@ -3631,6 +3662,12 @@ public:
                                             std::vector<SDValue> &Ops,
                                             SelectionDAG &DAG) const;
 
+  // Lower custom output constraints. If invalid, return SDValue().
+  virtual SDValue LowerAsmOutputForConstraint(SDValue &Chain, SDValue *Flag,
+                                              SDLoc DL,
+                                              const AsmOperandInfo &OpInfo,
+                                              SelectionDAG &DAG) const;
+
   //===--------------------------------------------------------------------===//
   // Div utility functions
   //
@@ -3782,6 +3819,14 @@ public:
   /// \returns True, if the expansion was successful, false otherwise
   bool expandCTTZ(SDNode *N, SDValue &Result, SelectionDAG &DAG) const;
 
+  /// Expand ABS nodes. Expands vector/scalar ABS nodes,
+  /// vector nodes can only succeed if all operations are legal/custom.
+  /// (ABS x) -> (XOR (ADD x, (SRA x, type_size)), (SRA x, type_size))
+  /// \param N Node to expand
+  /// \param Result output after conversion
+  /// \returns True, if the expansion was successful, false otherwise
+  bool expandABS(SDNode *N, SDValue &Result, SelectionDAG &DAG) const;
+
   /// Turn load of vector type into a load of the individual elements.
   /// \param LD load to expand
   /// \returns MERGE_VALUEs of the scalar loads with their chains.
@@ -3821,13 +3866,11 @@ public:
 
   /// Method for building the DAG expansion of ISD::[US][ADD|SUB]SAT. This
   /// method accepts integers as its arguments.
-  SDValue getExpandedSaturationAdditionSubtraction(SDNode *Node,
-                                                   SelectionDAG &DAG) const;
+  SDValue expandAddSubSat(SDNode *Node, SelectionDAG &DAG) const;
 
   /// Method for building the DAG expansion of ISD::SMULFIX. This method accepts
   /// integers as its arguments.
-  SDValue getExpandedFixedPointMultiplication(SDNode *Node,
-                                              SelectionDAG &DAG) const;
+  SDValue expandFixedPointMul(SDNode *Node, SelectionDAG &DAG) const;
 
   //===--------------------------------------------------------------------===//
   // Instruction Emitting Hooks
@@ -3880,9 +3923,10 @@ public:
   SDValue lowerCmpEqZeroToCtlzSrl(SDValue Op, SelectionDAG &DAG) const;
 
 private:
-  SDValue simplifySetCCWithAnd(EVT VT, SDValue N0, SDValue N1,
-                               ISD::CondCode Cond, DAGCombinerInfo &DCI,
-                               const SDLoc &DL) const;
+  SDValue foldSetCCWithAnd(EVT VT, SDValue N0, SDValue N1, ISD::CondCode Cond,
+                           const SDLoc &DL, DAGCombinerInfo &DCI) const;
+  SDValue foldSetCCWithBinOp(EVT VT, SDValue N0, SDValue N1, ISD::CondCode Cond,
+                             const SDLoc &DL, DAGCombinerInfo &DCI) const;
 
   SDValue optimizeSetCCOfSignedTruncationCheck(EVT SCCVT, SDValue N0,
                                                SDValue N1, ISD::CondCode Cond,
diff --git a/include/llvm/CodeGen/TargetLoweringObjectFileImpl.h b/include/llvm/CodeGen/TargetLoweringObjectFileImpl.h
index 052d1f8bc6863c4f65c82b90776cd6fc22d651f8..a1fb81cb009d2f05c8e7a1c1dda98e048717298a 100644
--- a/include/llvm/CodeGen/TargetLoweringObjectFileImpl.h
+++ b/include/llvm/CodeGen/TargetLoweringObjectFileImpl.h
@@ -1,9 +1,8 @@
 //==- llvm/CodeGen/TargetLoweringObjectFileImpl.h - Object Info --*- C++ -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/CodeGen/TargetOpcodes.h b/include/llvm/CodeGen/TargetOpcodes.h
index d0d959c4ae116608921bc1dd071f9e95e61f659c..080a244f6f6964e92245a031d76f0558d2864a71 100644
--- a/include/llvm/CodeGen/TargetOpcodes.h
+++ b/include/llvm/CodeGen/TargetOpcodes.h
@@ -1,9 +1,8 @@
 //===-- llvm/CodeGen/TargetOpcodes.h - Target Indep Opcodes -----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/CodeGen/TargetPassConfig.h b/include/llvm/CodeGen/TargetPassConfig.h
index 3288711a335d2a8ef31fe4ea307a8ba52565e62d..1def50dbb9d1e3db3abc3aa2935986f791c73ec9 100644
--- a/include/llvm/CodeGen/TargetPassConfig.h
+++ b/include/llvm/CodeGen/TargetPassConfig.h
@@ -1,9 +1,8 @@
 //===- TargetPassConfig.h - Code Generation pass options --------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -75,9 +74,6 @@ public:
   }
 };
 
-template <> struct isPodLike<IdentifyingPassPtr> {
-  static const bool value = true;
-};
 
 /// Target-Independent Code Generator Pass Configuration Options.
 ///
@@ -319,6 +315,10 @@ public:
   /// when GlobalISel failed and isGlobalISelAbortEnabled is false.
   virtual bool reportDiagnosticWhenGlobalISelFallback() const;
 
+  /// Check whether continuous CSE should be enabled in GISel passes.
+  /// By default, it's enabled for non O0 levels.
+  virtual bool isGISelCSEEnabled() const;
+
 protected:
   // Helper to verify the analysis is really immutable.
   void setOpt(bool &Opt, bool Val);
diff --git a/include/llvm/CodeGen/TargetRegisterInfo.h b/include/llvm/CodeGen/TargetRegisterInfo.h
index 0fbff3137653d059ffbd24eccd80fb69ee69f01e..5ed1e448575fc1ea783dc505e1a9a626db2aa5aa 100644
--- a/include/llvm/CodeGen/TargetRegisterInfo.h
+++ b/include/llvm/CodeGen/TargetRegisterInfo.h
@@ -1,9 +1,8 @@
 //==- CodeGen/TargetRegisterInfo.h - Target Register Information -*- C++ -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/CodeGen/TargetSchedule.h b/include/llvm/CodeGen/TargetSchedule.h
index 6173925e23a11265415df657ac07aa62614dab5e..cce85c8d7b0dc5718fd1eb6feacb4f04f37ead6d 100644
--- a/include/llvm/CodeGen/TargetSchedule.h
+++ b/include/llvm/CodeGen/TargetSchedule.h
@@ -1,9 +1,8 @@
 //===- llvm/CodeGen/TargetSchedule.h - Sched Machine Model ------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/CodeGen/TargetSubtargetInfo.h b/include/llvm/CodeGen/TargetSubtargetInfo.h
index 968e4c4b810273ae3873459158d7f611358f9079..bf0e9b208334c4a670f23ec2073a178dd95d43fe 100644
--- a/include/llvm/CodeGen/TargetSubtargetInfo.h
+++ b/include/llvm/CodeGen/TargetSubtargetInfo.h
@@ -1,9 +1,8 @@
 //===- llvm/CodeGen/TargetSubtargetInfo.h - Target Information --*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -190,9 +189,6 @@ public:
   /// TargetLowering preference). It does not yet disable the postRA scheduler.
   virtual bool enableMachineScheduler() const;
 
-  /// Support printing of [latency:throughput] comment in output .S file.
-  virtual bool supportPrintSchedInfo() const { return false; }
-
   /// True if the machine scheduler should disable the TLI preference
   /// for preRA scheduling with the source level scheduler.
   virtual bool enableMachineSchedDefaultSched() const { return true; }
@@ -286,10 +282,6 @@ public:
   /// possible.
   virtual bool enableSubRegLiveness() const { return false; }
 
-  /// Returns string representation of scheduler comment
-  std::string getSchedInfoStr(const MachineInstr &MI) const;
-  std::string getSchedInfoStr(MCInst const &MCI) const override;
-
   /// This is called after a .mir file was loaded.
   virtual void mirFileLoaded(MachineFunction &MF) const;
 };
diff --git a/include/llvm/CodeGen/UnreachableBlockElim.h b/include/llvm/CodeGen/UnreachableBlockElim.h
index 3e7afd4cd4337181be1595a2fb85d6322ceb18d2..d52d7c3c5b49e033556583c79728d2cd837e267d 100644
--- a/include/llvm/CodeGen/UnreachableBlockElim.h
+++ b/include/llvm/CodeGen/UnreachableBlockElim.h
@@ -1,9 +1,8 @@
 //===-- UnreachableBlockElim.h - Remove unreachable blocks for codegen --===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/CodeGen/ValueTypes.h b/include/llvm/CodeGen/ValueTypes.h
index d2ef4a94f8e2be99ddfc019203a855e9f418d651..c540c94f79d99fa4c5961be642ae778c956a5f11 100644
--- a/include/llvm/CodeGen/ValueTypes.h
+++ b/include/llvm/CodeGen/ValueTypes.h
@@ -1,9 +1,8 @@
 //===- CodeGen/ValueTypes.h - Low-Level Target independ. types --*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/CodeGen/ValueTypes.td b/include/llvm/CodeGen/ValueTypes.td
index 0abb4ece1d144c8b015f911a5a3680b006d5f2b1..f82faf259c70c16d0d7aed9d76c2a8cb622ecbc5 100644
--- a/include/llvm/CodeGen/ValueTypes.td
+++ b/include/llvm/CodeGen/ValueTypes.td
@@ -1,9 +1,8 @@
 //===- ValueTypes.td - ValueType definitions ---------------*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/CodeGen/VirtRegMap.h b/include/llvm/CodeGen/VirtRegMap.h
index 6a8e50a7e5f5c4801222b4a62a4e83c23d07ac51..7bdecbed5872295ec6df767d19e56a4577a48499 100644
--- a/include/llvm/CodeGen/VirtRegMap.h
+++ b/include/llvm/CodeGen/VirtRegMap.h
@@ -1,9 +1,8 @@
 //===- llvm/CodeGen/VirtRegMap.h - Virtual Register Map ---------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/CodeGen/WasmEHFuncInfo.h b/include/llvm/CodeGen/WasmEHFuncInfo.h
index 219fff988f6e0f974bfc7ad4e497d713ffc28ded..aaca8474349ab0dc251607418e3d82e5853fcaa9 100644
--- a/include/llvm/CodeGen/WasmEHFuncInfo.h
+++ b/include/llvm/CodeGen/WasmEHFuncInfo.h
@@ -1,9 +1,8 @@
 //===--- llvm/CodeGen/WasmEHFuncInfo.h --------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/CodeGen/WinEHFuncInfo.h b/include/llvm/CodeGen/WinEHFuncInfo.h
index 8043024626a0c5ff168901e7eee30d9a7eb17217..f098316de793af1662cdb1f710dc2fa6a7bc4696 100644
--- a/include/llvm/CodeGen/WinEHFuncInfo.h
+++ b/include/llvm/CodeGen/WinEHFuncInfo.h
@@ -1,9 +1,8 @@
 //===- llvm/CodeGen/WinEHFuncInfo.h -----------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Config/AsmParsers.def.in b/include/llvm/Config/AsmParsers.def.in
index d63675351c8024c228c31dd581ae73fc2a63e471..2e5fa33ace2ea5502a76cc2171a9439ff453026b 100644
--- a/include/llvm/Config/AsmParsers.def.in
+++ b/include/llvm/Config/AsmParsers.def.in
@@ -1,9 +1,9 @@
 /*===- llvm/Config/AsmParsers.def - LLVM Assembly Parsers -------*- C++ -*-===*\
 |*                                                                            *|
-|*                     The LLVM Compiler Infrastructure                       *|
-|*                                                                            *|
-|* This file is distributed under the University of Illinois Open Source      *|
-|* License. See LICENSE.TXT for details.                                      *|
+|* Part of the LLVM Project, under the Apache License v2.0 with LLVM          *|
+|* Exceptions.                                                                *|
+|* See https://llvm.org/LICENSE.txt for license information.                  *|
+|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception                    *|
 |*                                                                            *|
 |*===----------------------------------------------------------------------===*|
 |*                                                                            *|
diff --git a/include/llvm/Config/AsmPrinters.def.in b/include/llvm/Config/AsmPrinters.def.in
index f0152a4aa979f6ad08c12a495d2ed99f75c5c027..56419d9c147e4daf505fe0b5d5c3fa86f2f2214d 100644
--- a/include/llvm/Config/AsmPrinters.def.in
+++ b/include/llvm/Config/AsmPrinters.def.in
@@ -1,9 +1,9 @@
 /*===- llvm/Config/AsmPrinters.def - LLVM Assembly Printers -----*- C++ -*-===*\
 |*                                                                            *|
-|*                     The LLVM Compiler Infrastructure                       *|
-|*                                                                            *|
-|* This file is distributed under the University of Illinois Open Source      *|
-|* License. See LICENSE.TXT for details.                                      *|
+|* Part of the LLVM Project, under the Apache License v2.0 with LLVM          *|
+|* Exceptions.                                                                *|
+|* See https://llvm.org/LICENSE.txt for license information.                  *|
+|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception                    *|
 |*                                                                            *|
 |*===----------------------------------------------------------------------===*|
 |*                                                                            *|
diff --git a/include/llvm/Config/Disassemblers.def.in b/include/llvm/Config/Disassemblers.def.in
index d3a9bbdeaeac18f73293dbecb853e00f8ffacdb9..b110b1588d35b8a9525c89731f952fe0d2f80170 100644
--- a/include/llvm/Config/Disassemblers.def.in
+++ b/include/llvm/Config/Disassemblers.def.in
@@ -1,9 +1,9 @@
 /*===- llvm/Config/Disassemblers.def - LLVM Assembly Parsers ----*- C++ -*-===*\
 |*                                                                            *|
-|*                     The LLVM Compiler Infrastructure                       *|
-|*                                                                            *|
-|* This file is distributed under the University of Illinois Open Source      *|
-|* License. See LICENSE.TXT for details.                                      *|
+|* Part of the LLVM Project, under the Apache License v2.0 with LLVM          *|
+|* Exceptions.                                                                *|
+|* See https://llvm.org/LICENSE.txt for license information.                  *|
+|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception                    *|
 |*                                                                            *|
 |*===----------------------------------------------------------------------===*|
 |*                                                                            *|
diff --git a/include/llvm/Config/Targets.def.in b/include/llvm/Config/Targets.def.in
index d589ecee12cc495404e4446967e51e8c5d1a56a0..0da9b948c9f71f6287fa9e48c42404ee93e55070 100644
--- a/include/llvm/Config/Targets.def.in
+++ b/include/llvm/Config/Targets.def.in
@@ -1,9 +1,9 @@
 /*===- llvm/Config/Targets.def - LLVM Target Architectures ------*- C++ -*-===*\
 |*                                                                            *|
-|*                     The LLVM Compiler Infrastructure                       *|
-|*                                                                            *|
-|* This file is distributed under the University of Illinois Open Source      *|
-|* License. See LICENSE.TXT for details.                                      *|
+|* Part of the LLVM Project, under the Apache License v2.0 with LLVM          *|
+|* Exceptions.                                                                *|
+|* See https://llvm.org/LICENSE.txt for license information.                  *|
+|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception                    *|
 |*                                                                            *|
 |*===----------------------------------------------------------------------===*|
 |*                                                                            *|
diff --git a/include/llvm/Config/abi-breaking.h.cmake b/include/llvm/Config/abi-breaking.h.cmake
index 7ae401e5b8a8c7df2826869e3569d6a4cb517642..89c66bd7d11b7386decd3673e5fa2cd5e2c303b9 100644
--- a/include/llvm/Config/abi-breaking.h.cmake
+++ b/include/llvm/Config/abi-breaking.h.cmake
@@ -1,9 +1,9 @@
 /*===------- llvm/Config/abi-breaking.h - llvm configuration -------*- C -*-===*/
 /*                                                                            */
-/*                     The LLVM Compiler Infrastructure                       */
-/*                                                                            */
-/* This file is distributed under the University of Illinois Open Source      */
-/* License. See LICENSE.TXT for details.                                      */
+/* Part of the LLVM Project, under the Apache License v2.0 with LLVM          */
+/* Exceptions.                                                                */
+/* See https://llvm.org/LICENSE.txt for license information.                  */
+/* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception                    */
 /*                                                                            */
 /*===----------------------------------------------------------------------===*/
 
diff --git a/include/llvm/Config/config.h.cmake b/include/llvm/Config/config.h.cmake
index 03bbd74d6d325e1b42b095801eb9e2e3e7521c68..53e6c202d13a5fe55a5b71e41241a207c437d145 100644
--- a/include/llvm/Config/config.h.cmake
+++ b/include/llvm/Config/config.h.cmake
@@ -127,9 +127,6 @@
 /* Define to 1 if you have the `mallinfo' function. */
 #cmakedefine HAVE_MALLINFO ${HAVE_MALLINFO}
 
-/* Define to 1 if you have the <malloc.h> header file. */
-#cmakedefine HAVE_MALLOC_H ${HAVE_MALLOC_H}
-
 /* Define to 1 if you have the <malloc/malloc.h> header file. */
 #cmakedefine HAVE_MALLOC_MALLOC_H ${HAVE_MALLOC_MALLOC_H}
 
@@ -338,6 +335,9 @@
 /* Define as the return type of signal handlers (`int' or `void'). */
 #cmakedefine RETSIGTYPE ${RETSIGTYPE}
 
+/* Define if std::is_trivially_copyable is supported */
+#cmakedefine HAVE_STD_IS_TRIVIALLY_COPYABLE ${HAVE_STD_IS_TRIVIALLY_COPYABLE}
+
 /* Define to a function implementing stricmp */
 #cmakedefine stricmp ${stricmp}
 
diff --git a/include/llvm/Config/llvm-config.h.cmake b/include/llvm/Config/llvm-config.h.cmake
index 872a31553d11a772756a08a2c7dd5bb5818bdc58..e6b6f78f71cde3460ecb79d861b9606e88ac8e28 100644
--- a/include/llvm/Config/llvm-config.h.cmake
+++ b/include/llvm/Config/llvm-config.h.cmake
@@ -1,9 +1,9 @@
 /*===------- llvm/Config/llvm-config.h - llvm configuration -------*- C -*-===*/
 /*                                                                            */
-/*                     The LLVM Compiler Infrastructure                       */
-/*                                                                            */
-/* This file is distributed under the University of Illinois Open Source      */
-/* License. See LICENSE.TXT for details.                                      */
+/* Part of the LLVM Project, under the Apache License v2.0 with LLVM          */
+/* Exceptions.                                                                */
+/* See https://llvm.org/LICENSE.txt for license information.                  */
+/* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception                    */
 /*                                                                            */
 /*===----------------------------------------------------------------------===*/
 
diff --git a/include/llvm/DebugInfo/CodeView/AppendingTypeTableBuilder.h b/include/llvm/DebugInfo/CodeView/AppendingTypeTableBuilder.h
index bd1743511ed4bd441a509c1bc0bf1f163db0ab14..0ac8b651939d80d6fe35c5510691ae5a6019ab61 100644
--- a/include/llvm/DebugInfo/CodeView/AppendingTypeTableBuilder.h
+++ b/include/llvm/DebugInfo/CodeView/AppendingTypeTableBuilder.h
@@ -1,9 +1,8 @@
 //===- AppendingTypeTableBuilder.h -------------------------------*- C++-*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/CodeView/CVRecord.h b/include/llvm/DebugInfo/CodeView/CVRecord.h
index 11ca9ff108de731553c6f581435b1b5a6cdde63a..1df3d16e586067e38c932659eb035074221ef2c4 100644
--- a/include/llvm/DebugInfo/CodeView/CVRecord.h
+++ b/include/llvm/DebugInfo/CodeView/CVRecord.h
@@ -1,9 +1,8 @@
-//===- RecordIterator.h -----------------------------------------*- C++ -*-===//
+//===- CVRecord.h -----------------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/CodeView/CVSymbolVisitor.h b/include/llvm/DebugInfo/CodeView/CVSymbolVisitor.h
index 7c8cd121751a2a9d1788fb77ff2b6dbb3968c414..1615ff41df125fe0d3d80e1fe9260541d342de66 100644
--- a/include/llvm/DebugInfo/CodeView/CVSymbolVisitor.h
+++ b/include/llvm/DebugInfo/CodeView/CVSymbolVisitor.h
@@ -1,9 +1,8 @@
 //===- CVSymbolVisitor.h ----------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/CodeView/CVTypeVisitor.h b/include/llvm/DebugInfo/CodeView/CVTypeVisitor.h
index b765ba1abb4d75463c355f507297a30250b8bc2c..7538cb2c254800eb3a61cb8b324e76033944d812 100644
--- a/include/llvm/DebugInfo/CodeView/CVTypeVisitor.h
+++ b/include/llvm/DebugInfo/CodeView/CVTypeVisitor.h
@@ -1,9 +1,8 @@
 //===- CVTypeVisitor.h ------------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/CodeView/CodeView.h b/include/llvm/DebugInfo/CodeView/CodeView.h
index 8e0d9f608e93758161885e541fc5c9cc2f8123c4..4fbf507a6b7ddcba9672d315b109767dd43c258b 100644
--- a/include/llvm/DebugInfo/CodeView/CodeView.h
+++ b/include/llvm/DebugInfo/CodeView/CodeView.h
@@ -1,9 +1,8 @@
 //===- CodeView.h -----------------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/DebugInfo/CodeView/CodeViewError.h b/include/llvm/DebugInfo/CodeView/CodeViewError.h
index d4615d02220dfe5c1d0541181582571fb7dcb97b..9990c8d05d1cfa83ea83d30e04606baa9b05d3e3 100644
--- a/include/llvm/DebugInfo/CodeView/CodeViewError.h
+++ b/include/llvm/DebugInfo/CodeView/CodeViewError.h
@@ -1,9 +1,8 @@
 //===- CodeViewError.h - Error extensions for CodeView ----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/CodeView/CodeViewRecordIO.h b/include/llvm/DebugInfo/CodeView/CodeViewRecordIO.h
index 94f104ff772c2ef4cb357ad17a941bb8cb044573..b2476f16d1002b22e34a3a90488c7426efce8447 100644
--- a/include/llvm/DebugInfo/CodeView/CodeViewRecordIO.h
+++ b/include/llvm/DebugInfo/CodeView/CodeViewRecordIO.h
@@ -1,9 +1,8 @@
 //===- CodeViewRecordIO.h ---------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/CodeView/CodeViewRegisters.def b/include/llvm/DebugInfo/CodeView/CodeViewRegisters.def
index fdfcf4d53a236a2aedea762b5e1befea12f8dbba..0593bc0571c6021f16ef0205c3a04d7e460388ad 100644
--- a/include/llvm/DebugInfo/CodeView/CodeViewRegisters.def
+++ b/include/llvm/DebugInfo/CodeView/CodeViewRegisters.def
@@ -1,9 +1,8 @@
 //===-- CodeViewRegisters.def - CodeView registers --------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/DebugInfo/CodeView/CodeViewSymbols.def b/include/llvm/DebugInfo/CodeView/CodeViewSymbols.def
index b5f1cc0198dce6e7139e21371cb26fde8e320dbb..e36d80466137d181eedd5b1c0ac97495a94124fe 100644
--- a/include/llvm/DebugInfo/CodeView/CodeViewSymbols.def
+++ b/include/llvm/DebugInfo/CodeView/CodeViewSymbols.def
@@ -1,9 +1,8 @@
 //===-- CodeViewSymbols.def - All CodeView leaf types -----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/DebugInfo/CodeView/CodeViewTypes.def b/include/llvm/DebugInfo/CodeView/CodeViewTypes.def
index e9a479dba496ce466717430ecd29d9eb77c22653..a31111eb80a4ed3a5120ec1404ee9065669f6a85 100644
--- a/include/llvm/DebugInfo/CodeView/CodeViewTypes.def
+++ b/include/llvm/DebugInfo/CodeView/CodeViewTypes.def
@@ -1,9 +1,8 @@
 //===-- CodeViewTypes.def - All CodeView leaf types -------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/DebugInfo/CodeView/ContinuationRecordBuilder.h b/include/llvm/DebugInfo/CodeView/ContinuationRecordBuilder.h
index 7f851a2595dc0f773b5021ac0dfec970cd87b61c..cd67f783211e329d431963608c3a41d595cf9fff 100644
--- a/include/llvm/DebugInfo/CodeView/ContinuationRecordBuilder.h
+++ b/include/llvm/DebugInfo/CodeView/ContinuationRecordBuilder.h
@@ -1,9 +1,8 @@
 //===- ContinuationRecordBuilder.h ------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/CodeView/DebugChecksumsSubsection.h b/include/llvm/DebugInfo/CodeView/DebugChecksumsSubsection.h
index 78b284563afd90023a1ddde8f930ee04b6700c54..01f83676afdf3122b65abd5b5ae25afd0555efbe 100644
--- a/include/llvm/DebugInfo/CodeView/DebugChecksumsSubsection.h
+++ b/include/llvm/DebugInfo/CodeView/DebugChecksumsSubsection.h
@@ -1,9 +1,8 @@
 //===- DebugChecksumsSubsection.h -------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/CodeView/DebugCrossExSubsection.h b/include/llvm/DebugInfo/CodeView/DebugCrossExSubsection.h
index 2f9e9814d998d5000f43effa90958459282d03ad..64a78a7cef214e11ad31b5c6bf568a04e644b406 100644
--- a/include/llvm/DebugInfo/CodeView/DebugCrossExSubsection.h
+++ b/include/llvm/DebugInfo/CodeView/DebugCrossExSubsection.h
@@ -1,9 +1,8 @@
 //===- DebugCrossExSubsection.h ---------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/CodeView/DebugCrossImpSubsection.h b/include/llvm/DebugInfo/CodeView/DebugCrossImpSubsection.h
index 8be7ef265c82e9e65a7ce107ea9889d9180dd879..e7683cb2a9c4a020769d7d58f0c05bdb10df321a 100644
--- a/include/llvm/DebugInfo/CodeView/DebugCrossImpSubsection.h
+++ b/include/llvm/DebugInfo/CodeView/DebugCrossImpSubsection.h
@@ -1,9 +1,8 @@
-//===- DebugCrossExSubsection.h ---------------------------------*- C++ -*-===//
+//===- DebugCrossImpSubsection.h --------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/CodeView/DebugFrameDataSubsection.h b/include/llvm/DebugInfo/CodeView/DebugFrameDataSubsection.h
index 847d93f0e98585a8a4a8eb543b023400fc78bfd1..d5cd640231f99ff0945b12ff2d56ffd70540a42d 100644
--- a/include/llvm/DebugInfo/CodeView/DebugFrameDataSubsection.h
+++ b/include/llvm/DebugInfo/CodeView/DebugFrameDataSubsection.h
@@ -1,9 +1,8 @@
 //===- DebugFrameDataSubsection.h ------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/CodeView/DebugInlineeLinesSubsection.h b/include/llvm/DebugInfo/CodeView/DebugInlineeLinesSubsection.h
index b88c0eae1de282bbfba1aeb48679c3a8037cab99..1ca2bd0296959945a0f466658e9fd049795b4885 100644
--- a/include/llvm/DebugInfo/CodeView/DebugInlineeLinesSubsection.h
+++ b/include/llvm/DebugInfo/CodeView/DebugInlineeLinesSubsection.h
@@ -1,9 +1,8 @@
 //===- DebugInlineeLinesSubsection.h ----------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/CodeView/DebugLinesSubsection.h b/include/llvm/DebugInfo/CodeView/DebugLinesSubsection.h
index 53044b6c3dc8ba22def3eaf8209177c7c6b3cec9..1f8e56c5311fa241f07df9630d21409bf4ddfc7c 100644
--- a/include/llvm/DebugInfo/CodeView/DebugLinesSubsection.h
+++ b/include/llvm/DebugInfo/CodeView/DebugLinesSubsection.h
@@ -1,9 +1,8 @@
 //===- DebugLinesSubsection.h -----------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/CodeView/DebugStringTableSubsection.h b/include/llvm/DebugInfo/CodeView/DebugStringTableSubsection.h
index bebc960223ccc5ae85451a66829ed29fd51f2458..6e5b8adddd4aef430c8e459458d7ae35cbc4b0ea 100644
--- a/include/llvm/DebugInfo/CodeView/DebugStringTableSubsection.h
+++ b/include/llvm/DebugInfo/CodeView/DebugStringTableSubsection.h
@@ -1,9 +1,8 @@
 //===- DebugStringTableSubsection.h - CodeView String Table -----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/CodeView/DebugSubsection.h b/include/llvm/DebugInfo/CodeView/DebugSubsection.h
index e427e0006a55be7e83d161b9cafd084f8830af20..66272870efdacbc559c93f592ab06b0375fcb292 100644
--- a/include/llvm/DebugInfo/CodeView/DebugSubsection.h
+++ b/include/llvm/DebugInfo/CodeView/DebugSubsection.h
@@ -1,9 +1,8 @@
 //===- DebugSubsection.h ------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/CodeView/DebugSubsectionRecord.h b/include/llvm/DebugInfo/CodeView/DebugSubsectionRecord.h
index fc0cf0d1d90eca0d940e944168b1c584975a8d42..bcb379f00d68a2b71212ca6ae9770c567add17e8 100644
--- a/include/llvm/DebugInfo/CodeView/DebugSubsectionRecord.h
+++ b/include/llvm/DebugInfo/CodeView/DebugSubsectionRecord.h
@@ -1,9 +1,8 @@
 //===- DebugSubsectionRecord.h ----------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/CodeView/DebugSubsectionVisitor.h b/include/llvm/DebugInfo/CodeView/DebugSubsectionVisitor.h
index 75f749dfa933407b5c858f7ba7b8ce5403aa7e51..720b1b49581fa72c546823b17b383d7ed8d66987 100644
--- a/include/llvm/DebugInfo/CodeView/DebugSubsectionVisitor.h
+++ b/include/llvm/DebugInfo/CodeView/DebugSubsectionVisitor.h
@@ -1,9 +1,8 @@
 //===- DebugSubsectionVisitor.h -----------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/CodeView/DebugSymbolRVASubsection.h b/include/llvm/DebugInfo/CodeView/DebugSymbolRVASubsection.h
index a4c04b55eb4c2e1b6f2c3e58fc17bb4d08a7e5db..91b740ce6b9aee4e9349eb8f472f7527bc2cde35 100644
--- a/include/llvm/DebugInfo/CodeView/DebugSymbolRVASubsection.h
+++ b/include/llvm/DebugInfo/CodeView/DebugSymbolRVASubsection.h
@@ -1,9 +1,8 @@
 //===- DebugSymbolRVASubsection.h -------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/CodeView/DebugSymbolsSubsection.h b/include/llvm/DebugInfo/CodeView/DebugSymbolsSubsection.h
index dfda7deb6cb41934d1d9afe8fdb6e62e842d47b9..784fc59484b96e0117ce0654bbb1a0ac2d08b2f6 100644
--- a/include/llvm/DebugInfo/CodeView/DebugSymbolsSubsection.h
+++ b/include/llvm/DebugInfo/CodeView/DebugSymbolsSubsection.h
@@ -1,9 +1,8 @@
 //===- DebugSymbolsSubsection.h --------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/CodeView/DebugUnknownSubsection.h b/include/llvm/DebugInfo/CodeView/DebugUnknownSubsection.h
index ea9a96ca8d68e27e61e3d8dfb68b235598462dd4..fa7df325499fe93aaaf1c3c1e28708ab351a8293 100644
--- a/include/llvm/DebugInfo/CodeView/DebugUnknownSubsection.h
+++ b/include/llvm/DebugInfo/CodeView/DebugUnknownSubsection.h
@@ -1,9 +1,8 @@
 //===- DebugUnknownSubsection.h -----------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/CodeView/EnumTables.h b/include/llvm/DebugInfo/CodeView/EnumTables.h
index ee0f0f7c6023a20639387f4e0aac05c1d0091f1e..74f8c7176811deb6a3edebb572f0c1026631dd85 100644
--- a/include/llvm/DebugInfo/CodeView/EnumTables.h
+++ b/include/llvm/DebugInfo/CodeView/EnumTables.h
@@ -1,9 +1,8 @@
 //===- EnumTables.h - Enum to string conversion tables ----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/CodeView/Formatters.h b/include/llvm/DebugInfo/CodeView/Formatters.h
index 278ad02a39cd74766bfd835507e044f1c9b465eb..7d04a6a89bef61f5fab05202bb46aaa096dde1df 100644
--- a/include/llvm/DebugInfo/CodeView/Formatters.h
+++ b/include/llvm/DebugInfo/CodeView/Formatters.h
@@ -1,9 +1,8 @@
 //===- Formatters.h ---------------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/CodeView/FunctionId.h b/include/llvm/DebugInfo/CodeView/FunctionId.h
index 1af3da810b5aef5dc75d2053473316cb6dcb1792..bc102278819cdc7486a8f75b10dff275591a742d 100644
--- a/include/llvm/DebugInfo/CodeView/FunctionId.h
+++ b/include/llvm/DebugInfo/CodeView/FunctionId.h
@@ -1,9 +1,8 @@
 //===- FunctionId.h ---------------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/CodeView/GUID.h b/include/llvm/DebugInfo/CodeView/GUID.h
index a055ce9e2e4516c0776db60a751e829db6e6662f..5f807e6f7eebd2658d19e02fc2516360e0a6093c 100644
--- a/include/llvm/DebugInfo/CodeView/GUID.h
+++ b/include/llvm/DebugInfo/CodeView/GUID.h
@@ -1,9 +1,8 @@
 //===- GUID.h ---------------------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/CodeView/GlobalTypeTableBuilder.h b/include/llvm/DebugInfo/CodeView/GlobalTypeTableBuilder.h
index c4704168ed3439273a7578d565794ea914c503c9..a43ce20edde67423e62bce856ebe37a85bd5b510 100644
--- a/include/llvm/DebugInfo/CodeView/GlobalTypeTableBuilder.h
+++ b/include/llvm/DebugInfo/CodeView/GlobalTypeTableBuilder.h
@@ -1,9 +1,8 @@
 //===- GlobalTypeTableBuilder.h ----------------------------------*- C++-*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -74,14 +73,30 @@ public:
                            CreateFunc Create) {
     auto Result = HashedRecords.try_emplace(Hash, nextTypeIndex());
 
-    if (LLVM_UNLIKELY(Result.second)) {
+    if (LLVM_UNLIKELY(Result.second /*inserted*/ ||
+                      Result.first->second.isSimple())) {
       uint8_t *Stable = RecordStorage.Allocate<uint8_t>(RecordSize);
       MutableArrayRef<uint8_t> Data(Stable, RecordSize);
-      SeenRecords.push_back(Create(Data));
+      ArrayRef<uint8_t> StableRecord = Create(Data);
+      if (StableRecord.empty()) {
+        // Records with forward references into the Type stream will be deferred
+        // for insertion at a later time, on the second pass.
+        Result.first->getSecond() = TypeIndex(SimpleTypeKind::NotTranslated);
+        return TypeIndex(SimpleTypeKind::NotTranslated);
+      }
+      if (Result.first->second.isSimple()) {
+        assert(Result.first->second.getIndex() ==
+               (uint32_t)SimpleTypeKind::NotTranslated);
+        // On the second pass, update with index to remapped record. The
+        // (initially misbehaved) record will now come *after* other records
+        // resolved in the first pass, with proper *back* references in the
+        // stream.
+        Result.first->second = nextTypeIndex();
+      }
+      SeenRecords.push_back(StableRecord);
       SeenHashes.push_back(Hash);
     }
 
-    // Update the caller's copy of Record to point a stable copy.
     return Result.first->second;
   }
 
diff --git a/include/llvm/DebugInfo/CodeView/LazyRandomTypeCollection.h b/include/llvm/DebugInfo/CodeView/LazyRandomTypeCollection.h
index 383f7dd9fb6a7d6e4fadcc5ffe7dc2cd6718dd84..4e03627e9580db51af67097bc6487cf382920220 100644
--- a/include/llvm/DebugInfo/CodeView/LazyRandomTypeCollection.h
+++ b/include/llvm/DebugInfo/CodeView/LazyRandomTypeCollection.h
@@ -1,9 +1,8 @@
 //===- LazyRandomTypeCollection.h -------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/CodeView/Line.h b/include/llvm/DebugInfo/CodeView/Line.h
index ac229c3375137dda10dd88a80df18664d6604a3a..eb2aa154df1bd9b96207a498475a3fe4623c42b3 100644
--- a/include/llvm/DebugInfo/CodeView/Line.h
+++ b/include/llvm/DebugInfo/CodeView/Line.h
@@ -1,9 +1,8 @@
 //===- Line.h ---------------------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/CodeView/MergingTypeTableBuilder.h b/include/llvm/DebugInfo/CodeView/MergingTypeTableBuilder.h
index 9030918ebbb325a61e9360998760d1861abd1cba..1b2f6d29a9b625d2b4e5dd3097786342ef286a16 100644
--- a/include/llvm/DebugInfo/CodeView/MergingTypeTableBuilder.h
+++ b/include/llvm/DebugInfo/CodeView/MergingTypeTableBuilder.h
@@ -1,9 +1,8 @@
 //===- MergingTypeTableBuilder.h ---------------------------------*- C++-*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/CodeView/RecordName.h b/include/llvm/DebugInfo/CodeView/RecordName.h
index b022108df3d653dd25426802bd1c7679c65fa88d..cc09db8933bdbb6aa97832e4a5ff75a3475a10ee 100644
--- a/include/llvm/DebugInfo/CodeView/RecordName.h
+++ b/include/llvm/DebugInfo/CodeView/RecordName.h
@@ -1,9 +1,8 @@
 //===- RecordName.h ------------------------------------------- *- C++ --*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/CodeView/RecordSerialization.h b/include/llvm/DebugInfo/CodeView/RecordSerialization.h
index 36237e1a4d9e426a8bb1f995340b2d6df0db0b98..618146cebfdb84ea3266a547c978f8ca0754133c 100644
--- a/include/llvm/DebugInfo/CodeView/RecordSerialization.h
+++ b/include/llvm/DebugInfo/CodeView/RecordSerialization.h
@@ -1,9 +1,8 @@
 //===- RecordSerialization.h ------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/CodeView/SimpleTypeSerializer.h b/include/llvm/DebugInfo/CodeView/SimpleTypeSerializer.h
index a85d9270186bf3befa33a509f9f5b57b2187224f..3ca09b445a301b35b950aa66166e8e6048dca4e1 100644
--- a/include/llvm/DebugInfo/CodeView/SimpleTypeSerializer.h
+++ b/include/llvm/DebugInfo/CodeView/SimpleTypeSerializer.h
@@ -1,9 +1,8 @@
 //===- SimpleTypeSerializer.h -----------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/CodeView/StringsAndChecksums.h b/include/llvm/DebugInfo/CodeView/StringsAndChecksums.h
index 22a333e631a03327239fed3fee358f2123c17977..22a283e785e1fb2fb671445939fa9fc4a87fb0ae 100644
--- a/include/llvm/DebugInfo/CodeView/StringsAndChecksums.h
+++ b/include/llvm/DebugInfo/CodeView/StringsAndChecksums.h
@@ -1,9 +1,8 @@
 //===- StringsAndChecksums.h ------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/CodeView/SymbolDeserializer.h b/include/llvm/DebugInfo/CodeView/SymbolDeserializer.h
index 6b5dd2d20d170b60bd8a5bf9ed3b1eb6dadd187f..62761cb87c8174868f201b2d7d255d9ffba875c9 100644
--- a/include/llvm/DebugInfo/CodeView/SymbolDeserializer.h
+++ b/include/llvm/DebugInfo/CodeView/SymbolDeserializer.h
@@ -1,9 +1,8 @@
 //===- SymbolDeserializer.h -------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/CodeView/SymbolDumpDelegate.h b/include/llvm/DebugInfo/CodeView/SymbolDumpDelegate.h
index 823636c398def110da630ab3e7bdcf1cd8c1ea15..12f45dcb21ffc54086fdb7e808c014e098daa82b 100644
--- a/include/llvm/DebugInfo/CodeView/SymbolDumpDelegate.h
+++ b/include/llvm/DebugInfo/CodeView/SymbolDumpDelegate.h
@@ -1,9 +1,8 @@
 //===-- SymbolDumpDelegate.h ------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/CodeView/SymbolDumper.h b/include/llvm/DebugInfo/CodeView/SymbolDumper.h
index 215da2e2b522dd17bf6a0939ac4aa61c7e2d82b1..d832a48b12653cd94b5a8d5007c49c0b193e5f4b 100644
--- a/include/llvm/DebugInfo/CodeView/SymbolDumper.h
+++ b/include/llvm/DebugInfo/CodeView/SymbolDumper.h
@@ -1,9 +1,8 @@
 //===-- SymbolDumper.h - CodeView symbol info dumper ------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/CodeView/SymbolRecord.h b/include/llvm/DebugInfo/CodeView/SymbolRecord.h
index b58825c4a7887a90b54a13edd15ce7d3e87e8998..ac7b106e7b96ee30337f3f47b5a05d88a320b77c 100644
--- a/include/llvm/DebugInfo/CodeView/SymbolRecord.h
+++ b/include/llvm/DebugInfo/CodeView/SymbolRecord.h
@@ -1,9 +1,8 @@
 //===- SymbolRecord.h -------------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/CodeView/SymbolRecordHelpers.h b/include/llvm/DebugInfo/CodeView/SymbolRecordHelpers.h
index 3713fe118eaa9843e100ae60666d58dba4afad41..57dbc56c0769da6611853b315bdfbece6ce32ce3 100644
--- a/include/llvm/DebugInfo/CodeView/SymbolRecordHelpers.h
+++ b/include/llvm/DebugInfo/CodeView/SymbolRecordHelpers.h
@@ -1,9 +1,8 @@
 //===- SymbolRecordHelpers.h ------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/CodeView/SymbolRecordMapping.h b/include/llvm/DebugInfo/CodeView/SymbolRecordMapping.h
index 391e8f127665a790f64a280188980f990f8c3633..34368b6185d65ae6b2654ee5b15222b6a5839e8c 100644
--- a/include/llvm/DebugInfo/CodeView/SymbolRecordMapping.h
+++ b/include/llvm/DebugInfo/CodeView/SymbolRecordMapping.h
@@ -1,9 +1,8 @@
 //===- SymbolRecordMapping.h ------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/CodeView/SymbolSerializer.h b/include/llvm/DebugInfo/CodeView/SymbolSerializer.h
index f4d8ab0c3c2ed25cccd49ad26cd6fd5685226a28..08271c1600727369cf112358a7ae417f37b6111f 100644
--- a/include/llvm/DebugInfo/CodeView/SymbolSerializer.h
+++ b/include/llvm/DebugInfo/CodeView/SymbolSerializer.h
@@ -1,9 +1,8 @@
 //===- SymbolSerializer.h ---------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/CodeView/SymbolVisitorCallbackPipeline.h b/include/llvm/DebugInfo/CodeView/SymbolVisitorCallbackPipeline.h
index e29511a67b7f3b4345728ea2e5eccca1ddfa77b4..145d63a6fe61e1cb0bba76fe447da1c3d1db3424 100644
--- a/include/llvm/DebugInfo/CodeView/SymbolVisitorCallbackPipeline.h
+++ b/include/llvm/DebugInfo/CodeView/SymbolVisitorCallbackPipeline.h
@@ -1,9 +1,8 @@
 //===- SymbolVisitorCallbackPipeline.h --------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/CodeView/SymbolVisitorCallbacks.h b/include/llvm/DebugInfo/CodeView/SymbolVisitorCallbacks.h
index 0816f7c62656f22ffc4456c33a0630532447038d..1a4d5b9d31df67daa6e49a71f9f11aabdadc5f7c 100644
--- a/include/llvm/DebugInfo/CodeView/SymbolVisitorCallbacks.h
+++ b/include/llvm/DebugInfo/CodeView/SymbolVisitorCallbacks.h
@@ -1,9 +1,8 @@
 //===- SymbolVisitorCallbacks.h ---------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/CodeView/SymbolVisitorDelegate.h b/include/llvm/DebugInfo/CodeView/SymbolVisitorDelegate.h
index a2a3c6f18fba1594ca65d0018d07e4e453c9c812..368d8b288315c2ece9b0eb9279bfffd7a46fcc3a 100644
--- a/include/llvm/DebugInfo/CodeView/SymbolVisitorDelegate.h
+++ b/include/llvm/DebugInfo/CodeView/SymbolVisitorDelegate.h
@@ -1,9 +1,8 @@
 //===-- SymbolVisitorDelegate.h ---------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/CodeView/TypeCollection.h b/include/llvm/DebugInfo/CodeView/TypeCollection.h
index e9fc9b0de8efe1a946835514971e2e6b76be07cd..58b1dd058c1a2ba5777c26b362206abd18245729 100644
--- a/include/llvm/DebugInfo/CodeView/TypeCollection.h
+++ b/include/llvm/DebugInfo/CodeView/TypeCollection.h
@@ -1,9 +1,8 @@
 //===- TypeCollection.h - A collection of CodeView type records -*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/CodeView/TypeDeserializer.h b/include/llvm/DebugInfo/CodeView/TypeDeserializer.h
index 9887d901773a05bec70453973c8a2d5ca85468e6..d2ee3478b90b3dff4bf4e4347eb60cf4ee5d383e 100644
--- a/include/llvm/DebugInfo/CodeView/TypeDeserializer.h
+++ b/include/llvm/DebugInfo/CodeView/TypeDeserializer.h
@@ -1,9 +1,8 @@
 //===- TypeDeserializer.h ---------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/CodeView/TypeDumpVisitor.h b/include/llvm/DebugInfo/CodeView/TypeDumpVisitor.h
index afb8b3636361ba94a4277dc5447cd5b2de64feba..41a219ae5a7b365f6b23601e59147ae35806276e 100644
--- a/include/llvm/DebugInfo/CodeView/TypeDumpVisitor.h
+++ b/include/llvm/DebugInfo/CodeView/TypeDumpVisitor.h
@@ -1,9 +1,8 @@
 //===-- TypeDumpVisitor.h - CodeView type info dumper -----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/CodeView/TypeHashing.h b/include/llvm/DebugInfo/CodeView/TypeHashing.h
index 1f732d29a538f95e68bc2710813c91e780b446cc..c2fbff64ceb8569c08df9f4c4f3e2b3820de1c76 100644
--- a/include/llvm/DebugInfo/CodeView/TypeHashing.h
+++ b/include/llvm/DebugInfo/CodeView/TypeHashing.h
@@ -1,9 +1,8 @@
 //===- TypeHashing.h ---------------------------------------------*- C++-*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -85,6 +84,8 @@ struct GloballyHashedType {
   }
   std::array<uint8_t, 8> Hash;
 
+  bool empty() const { return *(const uint64_t*)Hash.data() == 0; }
+
   /// Given a sequence of bytes representing a record, compute a global hash for
   /// this record.  Due to the nature of global hashes incorporating the hashes
   /// of referenced records, this function requires a list of types and ids
@@ -108,8 +109,33 @@ struct GloballyHashedType {
   template <typename Range>
   static std::vector<GloballyHashedType> hashTypes(Range &&Records) {
     std::vector<GloballyHashedType> Hashes;
-    for (const auto &R : Records)
-      Hashes.push_back(hashType(R, Hashes, Hashes));
+    bool UnresolvedRecords = false;
+    for (const auto &R : Records) {
+      GloballyHashedType H = hashType(R, Hashes, Hashes);
+      if (H.empty())
+        UnresolvedRecords = true;
+      Hashes.push_back(H);
+    }
+
+    // In some rare cases, there might be records with forward references in the
+    // stream. Several passes might be needed to fully hash each record in the
+    // Type stream. However this occurs on very small OBJs generated by MASM,
+    // with a dozen records at most. Therefore this codepath isn't
+    // time-critical, as it isn't taken in 99% of cases.
+    while (UnresolvedRecords) {
+      UnresolvedRecords = false;
+      auto HashIt = Hashes.begin();
+      for (const auto &R : Records) {
+        if (HashIt->empty()) {
+          GloballyHashedType H = hashType(R, Hashes, Hashes);
+          if (H.empty())
+            UnresolvedRecords = true;
+          else
+            *HashIt = H;
+        }
+        ++HashIt;
+      }
+    }
 
     return Hashes;
   }
diff --git a/include/llvm/DebugInfo/CodeView/TypeIndex.h b/include/llvm/DebugInfo/CodeView/TypeIndex.h
index 58463a6b13dff3ed560644515b73f931f233d621..b9e2562bfc2b1b01f8399b69c74dca31e87ea82a 100644
--- a/include/llvm/DebugInfo/CodeView/TypeIndex.h
+++ b/include/llvm/DebugInfo/CodeView/TypeIndex.h
@@ -1,9 +1,8 @@
 //===- TypeIndex.h ----------------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/CodeView/TypeIndexDiscovery.h b/include/llvm/DebugInfo/CodeView/TypeIndexDiscovery.h
index c424a09ece890fce4f8388558cb167051635cdf9..469768787274d5a06a1fc4ab9b22d3c51f34f601 100644
--- a/include/llvm/DebugInfo/CodeView/TypeIndexDiscovery.h
+++ b/include/llvm/DebugInfo/CodeView/TypeIndexDiscovery.h
@@ -1,9 +1,8 @@
 //===- TypeIndexDiscovery.h -------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/CodeView/TypeRecord.h b/include/llvm/DebugInfo/CodeView/TypeRecord.h
index e7b5293dc51a419dfe9fee3f59fb9d8faa8dc153..b147dd6c3d05c07616ac6d71e0b0d8b451003d51 100644
--- a/include/llvm/DebugInfo/CodeView/TypeRecord.h
+++ b/include/llvm/DebugInfo/CodeView/TypeRecord.h
@@ -1,9 +1,8 @@
 //===- TypeRecord.h ---------------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -95,6 +94,11 @@ struct MemberAttributes {
     return MP == MethodKind::IntroducingVirtual ||
            MP == MethodKind::PureIntroducingVirtual;
   }
+
+  /// Is this method static.
+  bool isStatic() const {
+    return getMethodKind() == MethodKind::Static;
+  }
 };
 
 // Does not correspond to any tag, this is the tail of an LF_POINTER record
diff --git a/include/llvm/DebugInfo/CodeView/TypeRecordHelpers.h b/include/llvm/DebugInfo/CodeView/TypeRecordHelpers.h
index 389472ed1aeae1c1fefca14614693f7e0deb2ae5..e84704d99ddc836af123f92692da19fd504ba1d4 100644
--- a/include/llvm/DebugInfo/CodeView/TypeRecordHelpers.h
+++ b/include/llvm/DebugInfo/CodeView/TypeRecordHelpers.h
@@ -1,9 +1,8 @@
 //===- TypeRecordHelpers.h --------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/CodeView/TypeRecordMapping.h b/include/llvm/DebugInfo/CodeView/TypeRecordMapping.h
index cbe8d6066bb9f159231543ea0daf83e8c4606e71..b885d54afce6143ef61548df03f7e1df041f159e 100644
--- a/include/llvm/DebugInfo/CodeView/TypeRecordMapping.h
+++ b/include/llvm/DebugInfo/CodeView/TypeRecordMapping.h
@@ -1,9 +1,8 @@
 //===- TypeRecordMapping.h --------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/CodeView/TypeStreamMerger.h b/include/llvm/DebugInfo/CodeView/TypeStreamMerger.h
index a84f074237de0a8784903627c7b554515711cb98..d0506cce81762f09f77dc0055e78e28df1472b33 100644
--- a/include/llvm/DebugInfo/CodeView/TypeStreamMerger.h
+++ b/include/llvm/DebugInfo/CodeView/TypeStreamMerger.h
@@ -1,9 +1,8 @@
 //===- TypeStreamMerger.h ---------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -84,20 +83,20 @@ Error mergeTypeAndIdRecords(MergingTypeTableBuilder &DestIds,
                             MergingTypeTableBuilder &DestTypes,
                             SmallVectorImpl<TypeIndex> &SourceToDest,
                             const CVTypeArray &IdsAndTypes,
-                            Optional<EndPrecompRecord> &EndPrecomp);
+                            Optional<uint32_t> &PCHSignature);
 
 Error mergeTypeAndIdRecords(GlobalTypeTableBuilder &DestIds,
                             GlobalTypeTableBuilder &DestTypes,
                             SmallVectorImpl<TypeIndex> &SourceToDest,
                             const CVTypeArray &IdsAndTypes,
                             ArrayRef<GloballyHashedType> Hashes,
-                            Optional<EndPrecompRecord> &EndPrecomp);
+                            Optional<uint32_t> &PCHSignature);
 
 Error mergeTypeRecords(GlobalTypeTableBuilder &Dest,
                        SmallVectorImpl<TypeIndex> &SourceToDest,
                        const CVTypeArray &Types,
                        ArrayRef<GloballyHashedType> Hashes,
-                       Optional<EndPrecompRecord> &EndPrecomp);
+                       Optional<uint32_t> &PCHSignature);
 
 Error mergeIdRecords(GlobalTypeTableBuilder &Dest, ArrayRef<TypeIndex> Types,
                      SmallVectorImpl<TypeIndex> &SourceToDest,
diff --git a/include/llvm/DebugInfo/CodeView/TypeSymbolEmitter.h b/include/llvm/DebugInfo/CodeView/TypeSymbolEmitter.h
index dfba83d62fcee45d7d57779dd5682e9feb4511c5..4f2e5deb10b4dbc0cdfbe009d79c8271fcc5a2c7 100644
--- a/include/llvm/DebugInfo/CodeView/TypeSymbolEmitter.h
+++ b/include/llvm/DebugInfo/CodeView/TypeSymbolEmitter.h
@@ -1,9 +1,8 @@
 //===- TypeSymbolEmitter.h --------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/CodeView/TypeTableCollection.h b/include/llvm/DebugInfo/CodeView/TypeTableCollection.h
index 80326a0ffd39d491debbd712ccd66b80d2d9ddf3..5cbe3400e02920cf7b6c8b78a8f391ae51a68740 100644
--- a/include/llvm/DebugInfo/CodeView/TypeTableCollection.h
+++ b/include/llvm/DebugInfo/CodeView/TypeTableCollection.h
@@ -1,9 +1,8 @@
 //===- TypeTableCollection.h ---------------------------------- *- C++ --*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/CodeView/TypeVisitorCallbackPipeline.h b/include/llvm/DebugInfo/CodeView/TypeVisitorCallbackPipeline.h
index 126fb8abb0da814b45fdb6d4b47a200174071684..fb0b579d6a06b70b5b752153b998c74b3472cc6b 100644
--- a/include/llvm/DebugInfo/CodeView/TypeVisitorCallbackPipeline.h
+++ b/include/llvm/DebugInfo/CodeView/TypeVisitorCallbackPipeline.h
@@ -1,9 +1,8 @@
 //===- TypeVisitorCallbackPipeline.h ----------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/CodeView/TypeVisitorCallbacks.h b/include/llvm/DebugInfo/CodeView/TypeVisitorCallbacks.h
index d7a473306bc20fd817e3f26c9f725d5747450f19..33f8b1f24b1b4ce00868ef43ab32770ddaf33713 100644
--- a/include/llvm/DebugInfo/CodeView/TypeVisitorCallbacks.h
+++ b/include/llvm/DebugInfo/CodeView/TypeVisitorCallbacks.h
@@ -1,9 +1,8 @@
 //===- TypeVisitorCallbacks.h -----------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/DIContext.h b/include/llvm/DebugInfo/DIContext.h
index bbdd5e0d9c3fb7bef06108acfc611a06217c8799..a41ab21412d0ac14d1c44bfc743d76e636541a2b 100644
--- a/include/llvm/DebugInfo/DIContext.h
+++ b/include/llvm/DebugInfo/DIContext.h
@@ -1,9 +1,8 @@
 //===- DIContext.h ----------------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -81,7 +80,7 @@ class DIInliningInfo {
 public:
   DIInliningInfo() = default;
 
-  DILineInfo getFrame(unsigned Index) const {
+  const DILineInfo & getFrame(unsigned Index) const {
     assert(Index < Frames.size());
     return Frames[Index];
   }
@@ -98,6 +97,11 @@ public:
   void addFrame(const DILineInfo &Frame) {
     Frames.push_back(Frame);
   }
+  
+  void resize(unsigned i) {
+    Frames.resize(i);
+  }
+  
 };
 
 /// Container for description of a global variable.
diff --git a/include/llvm/DebugInfo/DWARF/DWARFAbbreviationDeclaration.h b/include/llvm/DebugInfo/DWARF/DWARFAbbreviationDeclaration.h
index 84b23398b8ccd8840af83de2b55b0e3f6ee18e29..ccf2891c2e21cb539125c2afc381cdb3b9c63c0d 100644
--- a/include/llvm/DebugInfo/DWARF/DWARFAbbreviationDeclaration.h
+++ b/include/llvm/DebugInfo/DWARF/DWARFAbbreviationDeclaration.h
@@ -1,9 +1,8 @@
 //===- DWARFAbbreviationDeclaration.h ---------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/DWARF/DWARFAcceleratorTable.h b/include/llvm/DebugInfo/DWARF/DWARFAcceleratorTable.h
index 1d448728338f5087f823ba56314f25a4013b39ca..0f09a5959a7349323ec0fe996e533734ea6f2a28 100644
--- a/include/llvm/DebugInfo/DWARF/DWARFAcceleratorTable.h
+++ b/include/llvm/DebugInfo/DWARF/DWARFAcceleratorTable.h
@@ -1,9 +1,8 @@
 //===- DWARFAcceleratorTable.h ----------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/DWARF/DWARFAddressRange.h b/include/llvm/DebugInfo/DWARF/DWARFAddressRange.h
index 5a7df5c353e8838d0d18699758cb6894a2ce495d..56d46c696fd6e2d8157c21b61f0c805de5df0912 100644
--- a/include/llvm/DebugInfo/DWARF/DWARFAddressRange.h
+++ b/include/llvm/DebugInfo/DWARF/DWARFAddressRange.h
@@ -1,9 +1,8 @@
 //===- DWARFAddressRange.h --------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/DWARF/DWARFAttribute.h b/include/llvm/DebugInfo/DWARF/DWARFAttribute.h
index f0672bb0ca75853d316917d54030871c9218e735..fc6f0a4436051d37ee02f638f40453aa98ef6d8f 100644
--- a/include/llvm/DebugInfo/DWARF/DWARFAttribute.h
+++ b/include/llvm/DebugInfo/DWARF/DWARFAttribute.h
@@ -1,9 +1,8 @@
 //===- DWARFAttribute.h -----------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/DWARF/DWARFCompileUnit.h b/include/llvm/DebugInfo/DWARF/DWARFCompileUnit.h
index 33797419a7b8fb687ab0da09902a81bbeb51dbcc..16b9bfb5de5611934825b72f41f8695d09a42cdf 100644
--- a/include/llvm/DebugInfo/DWARF/DWARFCompileUnit.h
+++ b/include/llvm/DebugInfo/DWARF/DWARFCompileUnit.h
@@ -1,9 +1,8 @@
 //===- DWARFCompileUnit.h ---------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/DWARF/DWARFContext.h b/include/llvm/DebugInfo/DWARF/DWARFContext.h
index dbb6be04544be7a1ab204f52f55dcd484536e7c2..e41592783b17c1c9d102a99b5eccc62b35a0f736 100644
--- a/include/llvm/DebugInfo/DWARF/DWARFContext.h
+++ b/include/llvm/DebugInfo/DWARF/DWARFContext.h
@@ -1,9 +1,8 @@
 //===- DWARFContext.h -------------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===/
 
diff --git a/include/llvm/DebugInfo/DWARF/DWARFDataExtractor.h b/include/llvm/DebugInfo/DWARF/DWARFDataExtractor.h
index 1ed087520b30747cceeeb62ad8338cb5ec6dcb59..7c2a159b71fa65f7dc29bad5d80dd6637a61fbcb 100644
--- a/include/llvm/DebugInfo/DWARF/DWARFDataExtractor.h
+++ b/include/llvm/DebugInfo/DWARF/DWARFDataExtractor.h
@@ -1,9 +1,8 @@
 //===- DWARFDataExtractor.h -------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/DWARF/DWARFDebugAbbrev.h b/include/llvm/DebugInfo/DWARF/DWARFDebugAbbrev.h
index d277ec382ba56be432497b84179e9414311f061a..28fd8484b4a99ef6952c34629d6d40a55cd979db 100644
--- a/include/llvm/DebugInfo/DWARF/DWARFDebugAbbrev.h
+++ b/include/llvm/DebugInfo/DWARF/DWARFDebugAbbrev.h
@@ -1,9 +1,8 @@
 //===- DWARFDebugAbbrev.h ---------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/DWARF/DWARFDebugAddr.h b/include/llvm/DebugInfo/DWARF/DWARFDebugAddr.h
index ffbd1b06d1e2c4cbced5d97aff4b4a22529d8f6e..a98bf282fe7c756ca107f3a26f97d77f4c85fa89 100644
--- a/include/llvm/DebugInfo/DWARF/DWARFDebugAddr.h
+++ b/include/llvm/DebugInfo/DWARF/DWARFDebugAddr.h
@@ -1,9 +1,8 @@
 //===- DWARFDebugAddr.h -------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/DWARF/DWARFDebugArangeSet.h b/include/llvm/DebugInfo/DWARF/DWARFDebugArangeSet.h
index ab46fac39f7c63c0c2b67825771293878a12c323..5b6c578bc3bf879987b3f47580c92a76dded5c90 100644
--- a/include/llvm/DebugInfo/DWARF/DWARFDebugArangeSet.h
+++ b/include/llvm/DebugInfo/DWARF/DWARFDebugArangeSet.h
@@ -1,9 +1,8 @@
 //===- DWARFDebugArangeSet.h ------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/DWARF/DWARFDebugAranges.h b/include/llvm/DebugInfo/DWARF/DWARFDebugAranges.h
index ea71a50f3270d0ebb5788b4f4e239ea3b5b2154c..6d3af24f90a848124a64ac94ef43af6f211caa57 100644
--- a/include/llvm/DebugInfo/DWARF/DWARFDebugAranges.h
+++ b/include/llvm/DebugInfo/DWARF/DWARFDebugAranges.h
@@ -1,9 +1,8 @@
 //===- DWARFDebugAranges.h --------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/DWARF/DWARFDebugFrame.h b/include/llvm/DebugInfo/DWARF/DWARFDebugFrame.h
index 7dc07d774aba9cf08eb829af248e608b575e4690..d960f4bc9b1c72aad42e2c466e26ec4a08b7c894 100644
--- a/include/llvm/DebugInfo/DWARF/DWARFDebugFrame.h
+++ b/include/llvm/DebugInfo/DWARF/DWARFDebugFrame.h
@@ -1,9 +1,8 @@
 //===- DWARFDebugFrame.h - Parsing of .debug_frame --------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/DWARF/DWARFDebugInfoEntry.h b/include/llvm/DebugInfo/DWARF/DWARFDebugInfoEntry.h
index 88c8f57bc33cab1708707378631878d4137f4274..f50063b24370086ab3c69ed66af4c60a9b544659 100644
--- a/include/llvm/DebugInfo/DWARF/DWARFDebugInfoEntry.h
+++ b/include/llvm/DebugInfo/DWARF/DWARFDebugInfoEntry.h
@@ -1,9 +1,8 @@
 //===- DWARFDebugInfoEntry.h ------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/DWARF/DWARFDebugLine.h b/include/llvm/DebugInfo/DWARF/DWARFDebugLine.h
index d50af5a057f1c7acbd7f1be13a404c65d50be3cd..9e40c8444661d5a5bf40b7b4c88e4828c6e356d8 100644
--- a/include/llvm/DebugInfo/DWARF/DWARFDebugLine.h
+++ b/include/llvm/DebugInfo/DWARF/DWARFDebugLine.h
@@ -1,9 +1,8 @@
 //===- DWARFDebugLine.h -----------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/DWARF/DWARFDebugLoc.h b/include/llvm/DebugInfo/DWARF/DWARFDebugLoc.h
index da2098e15402bb883913216cd4894740a55986f3..dbed95944326e919b5880650d51b26e8e7176129 100644
--- a/include/llvm/DebugInfo/DWARF/DWARFDebugLoc.h
+++ b/include/llvm/DebugInfo/DWARF/DWARFDebugLoc.h
@@ -1,9 +1,8 @@
 //===- DWARFDebugLoc.h ------------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/DWARF/DWARFDebugMacro.h b/include/llvm/DebugInfo/DWARF/DWARFDebugMacro.h
index bfe2fc3ac02d7d0590508a2218d29469b7aef496..a6c125990ca7aa06ea89ad70d60222403165ba88 100644
--- a/include/llvm/DebugInfo/DWARF/DWARFDebugMacro.h
+++ b/include/llvm/DebugInfo/DWARF/DWARFDebugMacro.h
@@ -1,9 +1,8 @@
 //===- DWARFDebugMacro.h ----------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/DWARF/DWARFDebugPubTable.h b/include/llvm/DebugInfo/DWARF/DWARFDebugPubTable.h
index 9e1656eb1615c4336671c5dde0512aa409a0b4cf..99e91ca9031970d065e7c6714821d0d67e8245f1 100644
--- a/include/llvm/DebugInfo/DWARF/DWARFDebugPubTable.h
+++ b/include/llvm/DebugInfo/DWARF/DWARFDebugPubTable.h
@@ -1,9 +1,8 @@
 //===- DWARFDebugPubTable.h -------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/DWARF/DWARFDebugRangeList.h b/include/llvm/DebugInfo/DWARF/DWARFDebugRangeList.h
index bc26edf006477350734b9130f53c58aa4394f359..c1c0f420f61570fb1d10c79f65c0c14ff7ccb471 100644
--- a/include/llvm/DebugInfo/DWARF/DWARFDebugRangeList.h
+++ b/include/llvm/DebugInfo/DWARF/DWARFDebugRangeList.h
@@ -1,9 +1,8 @@
 //===- DWARFDebugRangeList.h ------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/DWARF/DWARFDebugRnglists.h b/include/llvm/DebugInfo/DWARF/DWARFDebugRnglists.h
index 5cc8d789e5980e1311d4cad3907fce076519586b..c6661a8bfa0bc8a15973d666c2a99b87734bf8db 100644
--- a/include/llvm/DebugInfo/DWARF/DWARFDebugRnglists.h
+++ b/include/llvm/DebugInfo/DWARF/DWARFDebugRnglists.h
@@ -1,9 +1,8 @@
 //===- DWARFDebugRnglists.h -------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/DWARF/DWARFDie.h b/include/llvm/DebugInfo/DWARF/DWARFDie.h
index 56d46cd739a27108b5bc9419d1fb850694f6e651..21e68f983bb3521d2d56210d9e9c6f584e919188 100644
--- a/include/llvm/DebugInfo/DWARF/DWARFDie.h
+++ b/include/llvm/DebugInfo/DWARF/DWARFDie.h
@@ -1,9 +1,8 @@
 //===- DWARFDie.h -----------------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/DWARF/DWARFExpression.h b/include/llvm/DebugInfo/DWARF/DWARFExpression.h
index 3fad68a9b48b930675505448bf243f2cb817a6b0..8049094cc16fef251130cf45a94ee98386d53ece 100644
--- a/include/llvm/DebugInfo/DWARF/DWARFExpression.h
+++ b/include/llvm/DebugInfo/DWARF/DWARFExpression.h
@@ -1,9 +1,8 @@
 //===--- DWARFExpression.h - DWARF Expression handling ----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/DWARF/DWARFFormValue.h b/include/llvm/DebugInfo/DWARF/DWARFFormValue.h
index edf9442acd054674da49357d24bde7466b9ffa0d..e092d96b201880bdc88cf85db82cd4f852c15d52 100644
--- a/include/llvm/DebugInfo/DWARF/DWARFFormValue.h
+++ b/include/llvm/DebugInfo/DWARF/DWARFFormValue.h
@@ -1,9 +1,8 @@
 //===- DWARFFormValue.h -----------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -61,7 +60,6 @@ public:
 
   dwarf::Form getForm() const { return Form; }
   uint64_t getRawUValue() const { return Value.uval; }
-  uint64_t getSectionIndex() const { return Value.SectionIndex; }
   void setForm(dwarf::Form F) { Form = F; }
   void setUValue(uint64_t V) { Value.uval = V; }
   void setSValue(int64_t V) { Value.sval = V; }
@@ -75,6 +73,10 @@ public:
   bool isFormClass(FormClass FC) const;
   const DWARFUnit *getUnit() const { return U; }
   void dump(raw_ostream &OS, DIDumpOptions DumpOpts = DIDumpOptions()) const;
+  void dumpSectionedAddress(raw_ostream &OS, DIDumpOptions DumpOpts,
+                            SectionedAddress SA) const;
+  static void dumpAddressSection(const DWARFObject &Obj, raw_ostream &OS,
+                                 DIDumpOptions DumpOpts, uint64_t SectionIndex);
 
   /// Extracts a value in \p Data at offset \p *OffsetPtr. The information
   /// in \p FormParams is needed to interpret some forms. The optional
diff --git a/include/llvm/DebugInfo/DWARF/DWARFGdbIndex.h b/include/llvm/DebugInfo/DWARF/DWARFGdbIndex.h
index 073e02903c39e1ebf72416d974a71439c26a227e..38cd42ddb88389804240bdf1219d16e84e65abfc 100644
--- a/include/llvm/DebugInfo/DWARF/DWARFGdbIndex.h
+++ b/include/llvm/DebugInfo/DWARF/DWARFGdbIndex.h
@@ -1,9 +1,8 @@
 //===- DWARFGdbIndex.h ------------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/DWARF/DWARFListTable.h b/include/llvm/DebugInfo/DWARF/DWARFListTable.h
index 9b987314f209495505382c2afac559a495295dcb..6c13db3d87c0d247090e8d1df672cd2f56631567 100644
--- a/include/llvm/DebugInfo/DWARF/DWARFListTable.h
+++ b/include/llvm/DebugInfo/DWARF/DWARFListTable.h
@@ -1,9 +1,8 @@
 //===- DWARFListTable.h -----------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/DWARF/DWARFObject.h b/include/llvm/DebugInfo/DWARF/DWARFObject.h
index d611b5d075c8ea13ab3194ae073b0ed7c63e4857..1bba74a25d0e849e15c262651c5ee1ea2ea6c738 100644
--- a/include/llvm/DebugInfo/DWARF/DWARFObject.h
+++ b/include/llvm/DebugInfo/DWARF/DWARFObject.h
@@ -1,9 +1,8 @@
 //===- DWARFObject.h --------------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===-----------------------------------------------------------------------===/
 
diff --git a/include/llvm/DebugInfo/DWARF/DWARFRelocMap.h b/include/llvm/DebugInfo/DWARF/DWARFRelocMap.h
index f51838424614a601ba1b3a2e0ef5fbab2af0366a..a8f5b2840c648cbe2ad5091298ba50582c109309 100644
--- a/include/llvm/DebugInfo/DWARF/DWARFRelocMap.h
+++ b/include/llvm/DebugInfo/DWARF/DWARFRelocMap.h
@@ -1,9 +1,8 @@
 //===- DWARFRelocMap.h ------------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/DWARF/DWARFSection.h b/include/llvm/DebugInfo/DWARF/DWARFSection.h
index 7f82359652971ac2bf23afcfb16796f09c164386..bb0004615d75970fef5313885a212ff733664dbe 100644
--- a/include/llvm/DebugInfo/DWARF/DWARFSection.h
+++ b/include/llvm/DebugInfo/DWARF/DWARFSection.h
@@ -1,9 +1,8 @@
 //===- DWARFSection.h -------------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/DWARF/DWARFTypeUnit.h b/include/llvm/DebugInfo/DWARF/DWARFTypeUnit.h
index 8ca5ba13fc2334fb73eff7a293b29a9c67ae81c8..90d89375fd353bb2d5d0cffba42b0112a0b211de 100644
--- a/include/llvm/DebugInfo/DWARF/DWARFTypeUnit.h
+++ b/include/llvm/DebugInfo/DWARF/DWARFTypeUnit.h
@@ -1,9 +1,8 @@
 //===- DWARFTypeUnit.h ------------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/DWARF/DWARFUnit.h b/include/llvm/DebugInfo/DWARF/DWARFUnit.h
index 9909def1072355eb0959ba5e802f006c12b967d2..e82ce8ed6c2823723c2cc087021bbbef38aa98e0 100644
--- a/include/llvm/DebugInfo/DWARF/DWARFUnit.h
+++ b/include/llvm/DebugInfo/DWARF/DWARFUnit.h
@@ -1,9 +1,8 @@
 //===- DWARFUnit.h ----------------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -410,7 +409,7 @@ public:
     return None;
   }
 
-  void collectAddressRanges(DWARFAddressRangesVector &CURanges);
+  Expected<DWARFAddressRangesVector> collectAddressRanges();
 
   /// Returns subprogram DIE with address range encompassing the provided
   /// address. The pointer is alive as long as parsed compile unit DIEs are not
diff --git a/include/llvm/DebugInfo/DWARF/DWARFUnitIndex.h b/include/llvm/DebugInfo/DWARF/DWARFUnitIndex.h
index 16be5f9401c03d5cf044d992026d072cfe13244a..fc8c707c512e50212d6721e9253e9dc8c91e3720 100644
--- a/include/llvm/DebugInfo/DWARF/DWARFUnitIndex.h
+++ b/include/llvm/DebugInfo/DWARF/DWARFUnitIndex.h
@@ -1,9 +1,8 @@
 //===- DWARFUnitIndex.h -----------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/DWARF/DWARFVerifier.h b/include/llvm/DebugInfo/DWARF/DWARFVerifier.h
index e47fbea5646ed523cd55555f9f09bc117ffff623..f1268f22027214107b30a275b4b72831eb0d88c6 100644
--- a/include/llvm/DebugInfo/DWARF/DWARFVerifier.h
+++ b/include/llvm/DebugInfo/DWARF/DWARFVerifier.h
@@ -1,9 +1,8 @@
 //===- DWARFVerifier.h ----------------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/MSF/IMSFFile.h b/include/llvm/DebugInfo/MSF/IMSFFile.h
index f98e715e6b1576c7a94f51081561f28c61956964..7e80f96b89ae1bf34114b2d995a09b97fc9aa794 100644
--- a/include/llvm/DebugInfo/MSF/IMSFFile.h
+++ b/include/llvm/DebugInfo/MSF/IMSFFile.h
@@ -1,9 +1,8 @@
 //===- IMSFFile.h - Abstract base class for an MSF file ---------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/MSF/MSFBuilder.h b/include/llvm/DebugInfo/MSF/MSFBuilder.h
index 3de98c4ecba8dab10a65c53a23c15beab193709b..282870f5b3f1b4ed8fa2bf83ceb508889f4341e7 100644
--- a/include/llvm/DebugInfo/MSF/MSFBuilder.h
+++ b/include/llvm/DebugInfo/MSF/MSFBuilder.h
@@ -1,9 +1,8 @@
 //===- MSFBuilder.h - MSF Directory & Metadata Builder ----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/MSF/MSFCommon.h b/include/llvm/DebugInfo/MSF/MSFCommon.h
index 2db2b71df4a7433481591f99a9132065c2fec4a8..83331b14b8af10dacacb2f19c1ed9594cc12cf73 100644
--- a/include/llvm/DebugInfo/MSF/MSFCommon.h
+++ b/include/llvm/DebugInfo/MSF/MSFCommon.h
@@ -1,9 +1,8 @@
 //===- MSFCommon.h - Common types and functions for MSF files ---*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/MSF/MSFError.h b/include/llvm/DebugInfo/MSF/MSFError.h
index 5c043a7837b368624ebe45d71e93f83ac411db64..fbc4e6928536f80105210df47e2fa89e311e1a31 100644
--- a/include/llvm/DebugInfo/MSF/MSFError.h
+++ b/include/llvm/DebugInfo/MSF/MSFError.h
@@ -1,9 +1,8 @@
 //===- MSFError.h - Error extensions for MSF Files --------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/MSF/MappedBlockStream.h b/include/llvm/DebugInfo/MSF/MappedBlockStream.h
index f65e52922da775b9e330f904bd53aa593ca30d2e..593d781b990e1ba8b14f50bde15ce30febb80c8d 100644
--- a/include/llvm/DebugInfo/MSF/MappedBlockStream.h
+++ b/include/llvm/DebugInfo/MSF/MappedBlockStream.h
@@ -1,9 +1,8 @@
 //==- MappedBlockStream.h - Discontiguous stream data in an MSF --*- C++ -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/PDB/ConcreteSymbolEnumerator.h b/include/llvm/DebugInfo/PDB/ConcreteSymbolEnumerator.h
index ac7f19637ab165b5d99d86625d5da2e575f81c14..49ba20af7263601772b6600d4e17d07d6eab5e62 100644
--- a/include/llvm/DebugInfo/PDB/ConcreteSymbolEnumerator.h
+++ b/include/llvm/DebugInfo/PDB/ConcreteSymbolEnumerator.h
@@ -1,9 +1,8 @@
 //===- ConcreteSymbolEnumerator.h -------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/PDB/DIA/DIADataStream.h b/include/llvm/DebugInfo/PDB/DIA/DIADataStream.h
index 881d7329ab662155bf4678036a42d5b2ed2035be..f05b58c555074e762872d908ab2076dc6f6b6065 100644
--- a/include/llvm/DebugInfo/PDB/DIA/DIADataStream.h
+++ b/include/llvm/DebugInfo/PDB/DIA/DIADataStream.h
@@ -1,9 +1,8 @@
 //===- DIADataStream.h - DIA implementation of IPDBDataStream ---*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/PDB/DIA/DIAEnumDebugStreams.h b/include/llvm/DebugInfo/PDB/DIA/DIAEnumDebugStreams.h
index 1f129052d034dc80935f753e46358f9dc28e1b52..8a00ad45291a356ab7b27f17a037f58b350d5264 100644
--- a/include/llvm/DebugInfo/PDB/DIA/DIAEnumDebugStreams.h
+++ b/include/llvm/DebugInfo/PDB/DIA/DIAEnumDebugStreams.h
@@ -1,9 +1,8 @@
 //==- DIAEnumDebugStreams.h - DIA Debug Stream Enumerator impl ---*- C++ -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/PDB/DIA/DIAEnumFrameData.h b/include/llvm/DebugInfo/PDB/DIA/DIAEnumFrameData.h
index f3b02f07e6485272c45e378133db5bb71764078f..bd417c0746b178f1edf581985117fbc3a88e4889 100644
--- a/include/llvm/DebugInfo/PDB/DIA/DIAEnumFrameData.h
+++ b/include/llvm/DebugInfo/PDB/DIA/DIAEnumFrameData.h
@@ -1,9 +1,8 @@
 //==- DIAEnumFrameData.h --------------------------------------- -*- C++ -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/PDB/DIA/DIAEnumInjectedSources.h b/include/llvm/DebugInfo/PDB/DIA/DIAEnumInjectedSources.h
index 4669a8d31196e0b4d1425626fa154b051ceb45cc..1f75ca27c4f84ea966d720e35d510c2075cb5817 100644
--- a/include/llvm/DebugInfo/PDB/DIA/DIAEnumInjectedSources.h
+++ b/include/llvm/DebugInfo/PDB/DIA/DIAEnumInjectedSources.h
@@ -1,9 +1,8 @@
 //==- DIAEnumInjectedSources.h - DIA Injected Sources Enumerator -*- C++ -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/PDB/DIA/DIAEnumLineNumbers.h b/include/llvm/DebugInfo/PDB/DIA/DIAEnumLineNumbers.h
index f1cb6268a26dc37c18102a65e4a9b2550184a28d..8800baac105d452589c063b2430a4e8116df1439 100644
--- a/include/llvm/DebugInfo/PDB/DIA/DIAEnumLineNumbers.h
+++ b/include/llvm/DebugInfo/PDB/DIA/DIAEnumLineNumbers.h
@@ -1,9 +1,8 @@
 //==- DIAEnumLineNumbers.h - DIA Line Number Enumerator impl -----*- C++ -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/PDB/DIA/DIAEnumSectionContribs.h b/include/llvm/DebugInfo/PDB/DIA/DIAEnumSectionContribs.h
index ac2ae317d263c861d0b37b32213d061bb045ab5e..be8613bfba9dee20f823c1ade56f78bff53ecf29 100644
--- a/include/llvm/DebugInfo/PDB/DIA/DIAEnumSectionContribs.h
+++ b/include/llvm/DebugInfo/PDB/DIA/DIAEnumSectionContribs.h
@@ -1,9 +1,8 @@
 //==- DIAEnumSectionContribs.h --------------------------------- -*- C++ -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/PDB/DIA/DIAEnumSourceFiles.h b/include/llvm/DebugInfo/PDB/DIA/DIAEnumSourceFiles.h
index dac3df06a178a9a02a01a452088a95bf800f7ca8..61278994ed364b6df8357fae09e043ba5d8e5565 100644
--- a/include/llvm/DebugInfo/PDB/DIA/DIAEnumSourceFiles.h
+++ b/include/llvm/DebugInfo/PDB/DIA/DIAEnumSourceFiles.h
@@ -1,9 +1,8 @@
 //==- DIAEnumSourceFiles.h - DIA Source File Enumerator impl -----*- C++ -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/PDB/DIA/DIAEnumSymbols.h b/include/llvm/DebugInfo/PDB/DIA/DIAEnumSymbols.h
index 9689859ae0f8c6174ad6d350265796d782034bf4..f55342cea2e55ae64bc84e517b0a23d6d943b0e0 100644
--- a/include/llvm/DebugInfo/PDB/DIA/DIAEnumSymbols.h
+++ b/include/llvm/DebugInfo/PDB/DIA/DIAEnumSymbols.h
@@ -1,9 +1,8 @@
 //==- DIAEnumSymbols.h - DIA Symbol Enumerator impl --------------*- C++ -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/PDB/DIA/DIAEnumTables.h b/include/llvm/DebugInfo/PDB/DIA/DIAEnumTables.h
index f4f856ebb6fd64b50dbefd47ebf80303e55b85c8..057cb06fc8cacefd5e28bdb69076d04444b0af71 100644
--- a/include/llvm/DebugInfo/PDB/DIA/DIAEnumTables.h
+++ b/include/llvm/DebugInfo/PDB/DIA/DIAEnumTables.h
@@ -1,9 +1,8 @@
 //===- DIAEnumTables.h - DIA Tables Enumerator Impl -------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/PDB/DIA/DIAError.h b/include/llvm/DebugInfo/PDB/DIA/DIAError.h
index 2b33a65a0a14475fe4cbb57077908fada6310ad0..96d960599f7ef7b5c243014fca02b29a2fec143d 100644
--- a/include/llvm/DebugInfo/PDB/DIA/DIAError.h
+++ b/include/llvm/DebugInfo/PDB/DIA/DIAError.h
@@ -1,9 +1,8 @@
 //===- DIAError.h - Error extensions for PDB DIA implementation -*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/PDB/DIA/DIAFrameData.h b/include/llvm/DebugInfo/PDB/DIA/DIAFrameData.h
index 0ce6cfc93030d043da7dc9e48698db7f3251c722..c04f7cd00836f5c1331fb4ca7ab07b68550190df 100644
--- a/include/llvm/DebugInfo/PDB/DIA/DIAFrameData.h
+++ b/include/llvm/DebugInfo/PDB/DIA/DIAFrameData.h
@@ -1,9 +1,8 @@
 //===- DIAFrameData.h - DIA Impl. of IPDBFrameData ---------------- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/PDB/DIA/DIAInjectedSource.h b/include/llvm/DebugInfo/PDB/DIA/DIAInjectedSource.h
index 635508da84eaac6a238ad6927ed6fcc0225af01c..8be06f80fc6e4b335fbb0411eb3d6f7304c18a2a 100644
--- a/include/llvm/DebugInfo/PDB/DIA/DIAInjectedSource.h
+++ b/include/llvm/DebugInfo/PDB/DIA/DIAInjectedSource.h
@@ -1,9 +1,8 @@
 //===- DIAInjectedSource.h - DIA impl for IPDBInjectedSource ----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/PDB/DIA/DIALineNumber.h b/include/llvm/DebugInfo/PDB/DIA/DIALineNumber.h
index a59e3a19c8c22ee2251e5b02f4093b5ada121c1a..d8bb272207639d142ab521e5fd4bfc2bd9ad8515 100644
--- a/include/llvm/DebugInfo/PDB/DIA/DIALineNumber.h
+++ b/include/llvm/DebugInfo/PDB/DIA/DIALineNumber.h
@@ -1,9 +1,8 @@
 //===- DIALineNumber.h - DIA implementation of IPDBLineNumber ---*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/PDB/DIA/DIARawSymbol.h b/include/llvm/DebugInfo/PDB/DIA/DIARawSymbol.h
index 5d4f855c63ca674f953ea45db48a902ca1e8d686..7f201d3a4e361d0517a9b7811df94fbd6f246a3f 100644
--- a/include/llvm/DebugInfo/PDB/DIA/DIARawSymbol.h
+++ b/include/llvm/DebugInfo/PDB/DIA/DIARawSymbol.h
@@ -1,9 +1,8 @@
 //===- DIARawSymbol.h - DIA implementation of IPDBRawSymbol ----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/PDB/DIA/DIASectionContrib.h b/include/llvm/DebugInfo/PDB/DIA/DIASectionContrib.h
index 4688f1f91a89b308c3eae68608b75192a5a39fc9..0972831e8b1671daacc0e48e78c2c92b34492609 100644
--- a/include/llvm/DebugInfo/PDB/DIA/DIASectionContrib.h
+++ b/include/llvm/DebugInfo/PDB/DIA/DIASectionContrib.h
@@ -1,9 +1,8 @@
 //===- DIASectionContrib.h - DIA Impl. of IPDBSectionContrib ------ C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/PDB/DIA/DIASession.h b/include/llvm/DebugInfo/PDB/DIA/DIASession.h
index 592e061a8d83f13755b13a6eb097e090e71fcbaa..6f62e6061f56e2304381c7c8bd83cd9bbc2d0d43 100644
--- a/include/llvm/DebugInfo/PDB/DIA/DIASession.h
+++ b/include/llvm/DebugInfo/PDB/DIA/DIASession.h
@@ -1,9 +1,8 @@
 //===- DIASession.h - DIA implementation of IPDBSession ---------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/PDB/DIA/DIASourceFile.h b/include/llvm/DebugInfo/PDB/DIA/DIASourceFile.h
index 1088ea54981cfae65990c093fc5a55bc40c01d98..96edfc9f9e2910b76eaa84e193ff7bddb6460dd4 100644
--- a/include/llvm/DebugInfo/PDB/DIA/DIASourceFile.h
+++ b/include/llvm/DebugInfo/PDB/DIA/DIASourceFile.h
@@ -1,9 +1,8 @@
 //===- DIASourceFile.h - DIA implementation of IPDBSourceFile ---*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/PDB/DIA/DIASupport.h b/include/llvm/DebugInfo/PDB/DIA/DIASupport.h
index 92ebc04ae5a4ae465507fbb250520d48e78aa7da..1a7c2f3aeeaba522c2cc2be9e2545bdd307f434c 100644
--- a/include/llvm/DebugInfo/PDB/DIA/DIASupport.h
+++ b/include/llvm/DebugInfo/PDB/DIA/DIASupport.h
@@ -1,9 +1,8 @@
 //===- DIASupport.h - Common header includes for DIA ------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 // Common defines and header includes for all LLVMDebugInfoPDBDIA.  The
diff --git a/include/llvm/DebugInfo/PDB/DIA/DIATable.h b/include/llvm/DebugInfo/PDB/DIA/DIATable.h
index ce93fa0b86c34527052f9ab989a3535d67ba92ca..65396a042f06a3ccfffb70dc42bb14ac9a21d85a 100644
--- a/include/llvm/DebugInfo/PDB/DIA/DIATable.h
+++ b/include/llvm/DebugInfo/PDB/DIA/DIATable.h
@@ -1,9 +1,8 @@
 //===- DIATable.h - DIA implementation of IPDBTable -------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/PDB/DIA/DIAUtils.h b/include/llvm/DebugInfo/PDB/DIA/DIAUtils.h
index aa843e05de706ba653b19b6aad153f229dfc73dc..5e01d8f10a6e70c4616ab9cd9e64611f613bedd2 100644
--- a/include/llvm/DebugInfo/PDB/DIA/DIAUtils.h
+++ b/include/llvm/DebugInfo/PDB/DIA/DIAUtils.h
@@ -1,9 +1,8 @@
 //===- DIAUtils.h - Utility functions for working with DIA ------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/PDB/GenericError.h b/include/llvm/DebugInfo/PDB/GenericError.h
index 997f13f5f30ece7d53428948adff39c69460020e..ec85d92d2a9273349b26bc4551db106f52c86367 100644
--- a/include/llvm/DebugInfo/PDB/GenericError.h
+++ b/include/llvm/DebugInfo/PDB/GenericError.h
@@ -1,9 +1,8 @@
-//===- Error.h - system_error extensions for PDB ----------------*- C++ -*-===//
+//===- GenericError.h - system_error extensions for PDB ---------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/PDB/IPDBDataStream.h b/include/llvm/DebugInfo/PDB/IPDBDataStream.h
index 0d7a286a11a6e32d8ffaf844b2134fe33df8a028..4d0589a8791599cde72ca803c62214a57961737d 100644
--- a/include/llvm/DebugInfo/PDB/IPDBDataStream.h
+++ b/include/llvm/DebugInfo/PDB/IPDBDataStream.h
@@ -1,9 +1,8 @@
 //===- IPDBDataStream.h - base interface for child enumerator ---*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/PDB/IPDBEnumChildren.h b/include/llvm/DebugInfo/PDB/IPDBEnumChildren.h
index 7017f2600e9b9192a9902ad2ab517ae085d72e5f..bfa67d39bc768d41e85ee1eec339faaf1cf663f6 100644
--- a/include/llvm/DebugInfo/PDB/IPDBEnumChildren.h
+++ b/include/llvm/DebugInfo/PDB/IPDBEnumChildren.h
@@ -1,9 +1,8 @@
 //===- IPDBEnumChildren.h - base interface for child enumerator -*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/PDB/IPDBFrameData.h b/include/llvm/DebugInfo/PDB/IPDBFrameData.h
index 74679215b88017286aadd7dafc98060ddbae8cd9..24138b380db4aee2dcce6c8fe23a48acaaf48083 100644
--- a/include/llvm/DebugInfo/PDB/IPDBFrameData.h
+++ b/include/llvm/DebugInfo/PDB/IPDBFrameData.h
@@ -1,9 +1,8 @@
 //===- IPDBFrameData.h - base interface for frame data ----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/PDB/IPDBInjectedSource.h b/include/llvm/DebugInfo/PDB/IPDBInjectedSource.h
index e75d64af92bb5c36db7035edcd660242b8a162f5..56e85d1faecdf85a6bcb05bb36382b78a2fdd9b3 100644
--- a/include/llvm/DebugInfo/PDB/IPDBInjectedSource.h
+++ b/include/llvm/DebugInfo/PDB/IPDBInjectedSource.h
@@ -1,9 +1,8 @@
 //===- IPDBInjectedSource.h - base class for PDB injected file --*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/PDB/IPDBLineNumber.h b/include/llvm/DebugInfo/PDB/IPDBLineNumber.h
index e20080f2fbfcd3e8d5b3c126ad783f601f7903d3..77e88999497e5dca6a541032436313e1198207a8 100644
--- a/include/llvm/DebugInfo/PDB/IPDBLineNumber.h
+++ b/include/llvm/DebugInfo/PDB/IPDBLineNumber.h
@@ -1,9 +1,8 @@
 //===- IPDBLineNumber.h - base interface for PDB line no. info ---*- C++-*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/PDB/IPDBRawSymbol.h b/include/llvm/DebugInfo/PDB/IPDBRawSymbol.h
index 7c818d7cadebb0b343ca6f1d6185d8291617623f..b24e712e3b781fa99ef253ccf8dd40dd11de9e8b 100644
--- a/include/llvm/DebugInfo/PDB/IPDBRawSymbol.h
+++ b/include/llvm/DebugInfo/PDB/IPDBRawSymbol.h
@@ -1,9 +1,8 @@
 //===- IPDBRawSymbol.h - base interface for PDB symbol types ----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/PDB/IPDBSectionContrib.h b/include/llvm/DebugInfo/PDB/IPDBSectionContrib.h
index 4fda62404672574c5ab0fbd9677462535d1dd07f..c5cf4bbe556007e3046095fb1f02155ac0708b0f 100644
--- a/include/llvm/DebugInfo/PDB/IPDBSectionContrib.h
+++ b/include/llvm/DebugInfo/PDB/IPDBSectionContrib.h
@@ -1,9 +1,8 @@
 //==- IPDBSectionContrib.h - Interfaces for PDB SectionContribs --*- C++ -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/PDB/IPDBSession.h b/include/llvm/DebugInfo/PDB/IPDBSession.h
index 88fd02c0a345287a8c82a213416a2dd627c1c908..aa8d9c76d63eaebc508cc1b01dd8539aff753d20 100644
--- a/include/llvm/DebugInfo/PDB/IPDBSession.h
+++ b/include/llvm/DebugInfo/PDB/IPDBSession.h
@@ -1,9 +1,8 @@
 //===- IPDBSession.h - base interface for a PDB symbol context --*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/PDB/IPDBSourceFile.h b/include/llvm/DebugInfo/PDB/IPDBSourceFile.h
index 3676c4030b13f89385d9d4315647d0381e723f86..d7e49fb70580185fda575d879f2df11f747f9599 100644
--- a/include/llvm/DebugInfo/PDB/IPDBSourceFile.h
+++ b/include/llvm/DebugInfo/PDB/IPDBSourceFile.h
@@ -1,9 +1,8 @@
 //===- IPDBSourceFile.h - base interface for a PDB source file --*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/PDB/IPDBTable.h b/include/llvm/DebugInfo/PDB/IPDBTable.h
index 4561c4e847b2b48d4b0de72ced85ca9df7d669cf..55ca230d58c41f15d2ef5c7db78a72a47b10a51d 100644
--- a/include/llvm/DebugInfo/PDB/IPDBTable.h
+++ b/include/llvm/DebugInfo/PDB/IPDBTable.h
@@ -1,9 +1,8 @@
 //===- IPDBTable.h - Base Interface for a PDB Symbol Context ----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/PDB/Native/DbiModuleDescriptor.h b/include/llvm/DebugInfo/PDB/Native/DbiModuleDescriptor.h
index 9eef4041d0a1ecb401c0ad5e413c6ad39eefb5dc..568f0c98c559700e20349990deba930dd38cbeb7 100644
--- a/include/llvm/DebugInfo/PDB/Native/DbiModuleDescriptor.h
+++ b/include/llvm/DebugInfo/PDB/Native/DbiModuleDescriptor.h
@@ -1,9 +1,8 @@
 //===- DbiModuleDescriptor.h - PDB module information -----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/PDB/Native/DbiModuleDescriptorBuilder.h b/include/llvm/DebugInfo/PDB/Native/DbiModuleDescriptorBuilder.h
index ac7f741afefa7a5162bc067fc59c597592808850..4f5d28bbd05aa1a84bc789575bb1ec53441438fa 100644
--- a/include/llvm/DebugInfo/PDB/Native/DbiModuleDescriptorBuilder.h
+++ b/include/llvm/DebugInfo/PDB/Native/DbiModuleDescriptorBuilder.h
@@ -1,9 +1,8 @@
 //===- DbiModuleDescriptorBuilder.h - PDB module information ----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/PDB/Native/DbiModuleList.h b/include/llvm/DebugInfo/PDB/Native/DbiModuleList.h
index 5f6e7ab92a967c67d621cef3c3ae3ed4aeb248be..14223273c898c0464a4cc9e395c79c6e47c70299 100644
--- a/include/llvm/DebugInfo/PDB/Native/DbiModuleList.h
+++ b/include/llvm/DebugInfo/PDB/Native/DbiModuleList.h
@@ -1,9 +1,8 @@
 //===- DbiModuleList.h - PDB module information list ------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/PDB/Native/DbiStream.h b/include/llvm/DebugInfo/PDB/Native/DbiStream.h
index a3ca607efbef51eb765df45b27c10fc62866b0ce..7d75c159b7aebe1d241282b93da629002526c0d9 100644
--- a/include/llvm/DebugInfo/PDB/Native/DbiStream.h
+++ b/include/llvm/DebugInfo/PDB/Native/DbiStream.h
@@ -1,9 +1,8 @@
 //===- DbiStream.h - PDB Dbi Stream (Stream 3) Access -----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -11,6 +10,7 @@
 #define LLVM_DEBUGINFO_PDB_RAW_PDBDBISTREAM_H
 
 #include "llvm/DebugInfo/CodeView/DebugSubsection.h"
+#include "llvm/DebugInfo/CodeView/DebugFrameDataSubsection.h"
 #include "llvm/DebugInfo/MSF/MappedBlockStream.h"
 #include "llvm/DebugInfo/PDB/Native/DbiModuleDescriptor.h"
 #include "llvm/DebugInfo/PDB/Native/DbiModuleList.h"
@@ -80,7 +80,10 @@ public:
 
   FixedStreamArray<object::coff_section> getSectionHeaders() const;
 
-  FixedStreamArray<object::FpoData> getFpoRecords();
+  bool hasOldFpoRecords() const;
+  FixedStreamArray<object::FpoData> getOldFpoRecords() const;
+  bool hasNewFpoRecords() const;
+  const codeview::DebugFrameDataSubsectionRef &getNewFpoRecords() const;
 
   FixedStreamArray<SecMapEntry> getSectionMap() const;
   void visitSectionContributions(ISectionContribVisitor &Visitor) const;
@@ -91,7 +94,11 @@ private:
   Error initializeSectionContributionData();
   Error initializeSectionHeadersData(PDBFile *Pdb);
   Error initializeSectionMapData();
-  Error initializeFpoRecords(PDBFile *Pdb);
+  Error initializeOldFpoRecords(PDBFile *Pdb);
+  Error initializeNewFpoRecords(PDBFile *Pdb);
+
+  Expected<std::unique_ptr<msf::MappedBlockStream>>
+  createIndexedStreamForHeaderType(PDBFile *Pdb, DbgHeaderType Type) const;
 
   std::unique_ptr<BinaryStream> Stream;
 
@@ -117,8 +124,11 @@ private:
   std::unique_ptr<msf::MappedBlockStream> SectionHeaderStream;
   FixedStreamArray<object::coff_section> SectionHeaders;
 
-  std::unique_ptr<msf::MappedBlockStream> FpoStream;
-  FixedStreamArray<object::FpoData> FpoRecords;
+  std::unique_ptr<msf::MappedBlockStream> OldFpoStream;
+  FixedStreamArray<object::FpoData> OldFpoRecords;
+  
+  std::unique_ptr<msf::MappedBlockStream> NewFpoStream;
+  codeview::DebugFrameDataSubsectionRef NewFpoRecords;
 
   const DbiStreamHeader *Header;
 };
diff --git a/include/llvm/DebugInfo/PDB/Native/DbiStreamBuilder.h b/include/llvm/DebugInfo/PDB/Native/DbiStreamBuilder.h
index b538de5766771921f39d8c542a22f47eafb3343d..d9be238af07bb58b1e38055f9e1e1b6d4d1c39c7 100644
--- a/include/llvm/DebugInfo/PDB/Native/DbiStreamBuilder.h
+++ b/include/llvm/DebugInfo/PDB/Native/DbiStreamBuilder.h
@@ -1,9 +1,8 @@
 //===- DbiStreamBuilder.h - PDB Dbi Stream Creation -------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/PDB/Native/EnumTables.h b/include/llvm/DebugInfo/PDB/Native/EnumTables.h
index c018445630fed9f3fb79e16ac95458a98f185182..70161fadf7d25e5ccff50be9a85af10a6685096b 100644
--- a/include/llvm/DebugInfo/PDB/Native/EnumTables.h
+++ b/include/llvm/DebugInfo/PDB/Native/EnumTables.h
@@ -1,9 +1,8 @@
 //===- EnumTables.h - Enum to string conversion tables ----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/PDB/Native/Formatters.h b/include/llvm/DebugInfo/PDB/Native/Formatters.h
index 7d5eab2e2a0901438e37501832cfedfede9301ef..29c957eeb5e04bfedba5cc50e5f85b637d23c3b6 100644
--- a/include/llvm/DebugInfo/PDB/Native/Formatters.h
+++ b/include/llvm/DebugInfo/PDB/Native/Formatters.h
@@ -1,9 +1,8 @@
 //===- Formatters.h ---------------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/PDB/Native/GSIStreamBuilder.h b/include/llvm/DebugInfo/PDB/Native/GSIStreamBuilder.h
index 4c39ca762b5b74f874068aa30457c39ec447f746..a497956000283de4f161fa3fa03c08efb2ea9b4b 100644
--- a/include/llvm/DebugInfo/PDB/Native/GSIStreamBuilder.h
+++ b/include/llvm/DebugInfo/PDB/Native/GSIStreamBuilder.h
@@ -1,9 +1,8 @@
 //===- GSIStreamBuilder.h - PDB Publics/Globals Stream Creation -*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/PDB/Native/GlobalsStream.h b/include/llvm/DebugInfo/PDB/Native/GlobalsStream.h
index 7f84564ee988e19e9c820e26e44510a01e1b9df4..404baaa250771004118177a0e9b4428f6342faaa 100644
--- a/include/llvm/DebugInfo/PDB/Native/GlobalsStream.h
+++ b/include/llvm/DebugInfo/PDB/Native/GlobalsStream.h
@@ -1,9 +1,8 @@
 //===- GlobalsStream.h - PDB Index of Symbols by Name -----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/PDB/Native/Hash.h b/include/llvm/DebugInfo/PDB/Native/Hash.h
index 1f11d43ecdd4ecc22e388b39bc409bce781e9c83..b048d878a12cd059a6e1011e8c91285520629038 100644
--- a/include/llvm/DebugInfo/PDB/Native/Hash.h
+++ b/include/llvm/DebugInfo/PDB/Native/Hash.h
@@ -1,9 +1,8 @@
 //===- Hash.h - PDB hash functions ------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/PDB/Native/HashTable.h b/include/llvm/DebugInfo/PDB/Native/HashTable.h
index 34cc6179688b5f14d905cbfaf33a101a1e85011a..e8f08c68cd3bb3d3de8be77a21a0c7e785ea0140 100644
--- a/include/llvm/DebugInfo/PDB/Native/HashTable.h
+++ b/include/llvm/DebugInfo/PDB/Native/HashTable.h
@@ -1,9 +1,8 @@
 //===- HashTable.h - PDB Hash Table -----------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/PDB/Native/ISectionContribVisitor.h b/include/llvm/DebugInfo/PDB/Native/ISectionContribVisitor.h
index fb00d6ad4bc785df035f65963a0d9636f01c8f44..717dce2f27379bbb7d75a1ff4778c632ef3c22d0 100644
--- a/include/llvm/DebugInfo/PDB/Native/ISectionContribVisitor.h
+++ b/include/llvm/DebugInfo/PDB/Native/ISectionContribVisitor.h
@@ -1,9 +1,8 @@
 //===- ISectionContribVisitor.h ---------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/PDB/Native/InfoStream.h b/include/llvm/DebugInfo/PDB/Native/InfoStream.h
index 8c52b042f289ae1376ebcaa0912ab0f4724f7888..315b09356ae30c2c1148a8acd313a4368623601a 100644
--- a/include/llvm/DebugInfo/PDB/Native/InfoStream.h
+++ b/include/llvm/DebugInfo/PDB/Native/InfoStream.h
@@ -1,9 +1,8 @@
 //===- InfoStream.h - PDB Info Stream (Stream 1) Access ---------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/PDB/Native/InfoStreamBuilder.h b/include/llvm/DebugInfo/PDB/Native/InfoStreamBuilder.h
index 101127a355f53c896a7f3def502a50d4f3bc8922..208a37c45d4988d23f35de6b45368430bb0c4c0b 100644
--- a/include/llvm/DebugInfo/PDB/Native/InfoStreamBuilder.h
+++ b/include/llvm/DebugInfo/PDB/Native/InfoStreamBuilder.h
@@ -1,9 +1,8 @@
 //===- InfoStreamBuilder.h - PDB Info Stream Creation -----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/PDB/Native/ModuleDebugStream.h b/include/llvm/DebugInfo/PDB/Native/ModuleDebugStream.h
index 8d590df288f378d5010210beb2ccbd9820f0b39c..161aa32ff63958222fe4eef408e86301d193a471 100644
--- a/include/llvm/DebugInfo/PDB/Native/ModuleDebugStream.h
+++ b/include/llvm/DebugInfo/PDB/Native/ModuleDebugStream.h
@@ -1,9 +1,8 @@
 //===- ModuleDebugStream.h - PDB Module Info Stream Access ------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/PDB/Native/NamedStreamMap.h b/include/llvm/DebugInfo/PDB/Native/NamedStreamMap.h
index 01b8f1b5da5632b0b3248f87e3f1a0147de5edde..c49d796356c7bf2c146f12555d4fbd6d2903a24f 100644
--- a/include/llvm/DebugInfo/PDB/Native/NamedStreamMap.h
+++ b/include/llvm/DebugInfo/PDB/Native/NamedStreamMap.h
@@ -1,9 +1,8 @@
 //===- NamedStreamMap.h - PDB Named Stream Map ------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/PDB/Native/NativeCompilandSymbol.h b/include/llvm/DebugInfo/PDB/Native/NativeCompilandSymbol.h
index 3cd465503044265d959251266b7d0b2909a48c15..50d437642d0f736ef4b2566e1f5dd4918cd65143 100644
--- a/include/llvm/DebugInfo/PDB/Native/NativeCompilandSymbol.h
+++ b/include/llvm/DebugInfo/PDB/Native/NativeCompilandSymbol.h
@@ -1,9 +1,8 @@
 //===- NativeCompilandSymbol.h - native impl for compiland syms -*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/PDB/Native/NativeEnumGlobals.h b/include/llvm/DebugInfo/PDB/Native/NativeEnumGlobals.h
index 4442a1ec41fb7eb91d0c9029f8b1576113f4f3a7..073878afd12937dbe6c2c53393d15db0796f374e 100644
--- a/include/llvm/DebugInfo/PDB/Native/NativeEnumGlobals.h
+++ b/include/llvm/DebugInfo/PDB/Native/NativeEnumGlobals.h
@@ -1,9 +1,8 @@
 //==- NativeEnumGlobals.h - Native Global Enumerator impl --------*- C++ -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/PDB/Native/NativeEnumModules.h b/include/llvm/DebugInfo/PDB/Native/NativeEnumModules.h
index c268641a1008bc9e77bfffa29558c745f60449b4..94f1ee18ed9f43cbd4376d546b3860f6f9d58840 100644
--- a/include/llvm/DebugInfo/PDB/Native/NativeEnumModules.h
+++ b/include/llvm/DebugInfo/PDB/Native/NativeEnumModules.h
@@ -1,9 +1,8 @@
 //==- NativeEnumModules.h - Native Module Enumerator impl --------*- C++ -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/PDB/Native/NativeEnumTypes.h b/include/llvm/DebugInfo/PDB/Native/NativeEnumTypes.h
index f8ac1655dc6111006962b1bf76a0b6b3a42fa830..25c56567384fbf28a3102906ab728a14904927a6 100644
--- a/include/llvm/DebugInfo/PDB/Native/NativeEnumTypes.h
+++ b/include/llvm/DebugInfo/PDB/Native/NativeEnumTypes.h
@@ -1,9 +1,8 @@
 //==- NativeEnumTypes.h - Native Type Enumerator impl ------------*- C++ -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/PDB/Native/NativeExeSymbol.h b/include/llvm/DebugInfo/PDB/Native/NativeExeSymbol.h
index f4030da1d0267f9d5a19b1fc825e4873c61d7ad6..280358d02305c71f94271062e9645388daa5ed25 100644
--- a/include/llvm/DebugInfo/PDB/Native/NativeExeSymbol.h
+++ b/include/llvm/DebugInfo/PDB/Native/NativeExeSymbol.h
@@ -1,9 +1,8 @@
 //===- NativeExeSymbol.h - native impl for PDBSymbolExe ---------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/PDB/Native/NativeRawSymbol.h b/include/llvm/DebugInfo/PDB/Native/NativeRawSymbol.h
index 6505a7d39573e0324bb2364d5e1fc0c572d47567..4133be2207136aeb3003351f7b83d003edb60681 100644
--- a/include/llvm/DebugInfo/PDB/Native/NativeRawSymbol.h
+++ b/include/llvm/DebugInfo/PDB/Native/NativeRawSymbol.h
@@ -1,9 +1,8 @@
 //==- NativeRawSymbol.h - Native implementation of IPDBRawSymbol -*- C++ -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/PDB/Native/NativeSession.h b/include/llvm/DebugInfo/PDB/Native/NativeSession.h
index 4878e47d31217f7d4871365d3ae172331e2df960..ee7d8cdec93bf2bc93ace654911a8dd68dac5028 100644
--- a/include/llvm/DebugInfo/PDB/Native/NativeSession.h
+++ b/include/llvm/DebugInfo/PDB/Native/NativeSession.h
@@ -1,9 +1,8 @@
 //===- NativeSession.h - Native implementation of IPDBSession ---*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/PDB/Native/NativeSymbolEnumerator.h b/include/llvm/DebugInfo/PDB/Native/NativeSymbolEnumerator.h
index acc5eb8ff2c2e35a0630c31362c9794a9d519b70..0635850978991fa96a07fcf990e20be467b0ff14 100644
--- a/include/llvm/DebugInfo/PDB/Native/NativeSymbolEnumerator.h
+++ b/include/llvm/DebugInfo/PDB/Native/NativeSymbolEnumerator.h
@@ -1,9 +1,8 @@
 //===- NativeSymbolEnumerator.h - info about enumerator values --*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/PDB/Native/NativeTypeArray.h b/include/llvm/DebugInfo/PDB/Native/NativeTypeArray.h
index 10e68e6df4500d542d01496cb58c4c2f57d7153c..262864fd709fa3a09c5c53482a63a782f1eef74f 100644
--- a/include/llvm/DebugInfo/PDB/Native/NativeTypeArray.h
+++ b/include/llvm/DebugInfo/PDB/Native/NativeTypeArray.h
@@ -1,9 +1,8 @@
 //===- NativeTypeArray.h ------------------------------------------ C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/PDB/Native/NativeTypeBuiltin.h b/include/llvm/DebugInfo/PDB/Native/NativeTypeBuiltin.h
index 725dfb89222f12db6bfb635f97cd05b8eaaeac0e..8bb09f05d0bc64b47888ac7ac3286abf9ea14e3c 100644
--- a/include/llvm/DebugInfo/PDB/Native/NativeTypeBuiltin.h
+++ b/include/llvm/DebugInfo/PDB/Native/NativeTypeBuiltin.h
@@ -1,9 +1,8 @@
 //===- NativeTypeBuiltin.h ---------------------------------------- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/PDB/Native/NativeTypeEnum.h b/include/llvm/DebugInfo/PDB/Native/NativeTypeEnum.h
index a5cbefc181114576d1de7c6e2f72615b890d9ce3..2068c88fc74a04a6a0b6ac577ed59d53b825b253 100644
--- a/include/llvm/DebugInfo/PDB/Native/NativeTypeEnum.h
+++ b/include/llvm/DebugInfo/PDB/Native/NativeTypeEnum.h
@@ -1,9 +1,8 @@
 //===- NativeTypeEnum.h - info about enum type ------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/PDB/Native/NativeTypeFunctionSig.h b/include/llvm/DebugInfo/PDB/Native/NativeTypeFunctionSig.h
index 1b1b87f6581f86e2186bf95d3f38bd9727ee8b39..a7ea287dffc8e6b741dcf828f361549d1565f12a 100644
--- a/include/llvm/DebugInfo/PDB/Native/NativeTypeFunctionSig.h
+++ b/include/llvm/DebugInfo/PDB/Native/NativeTypeFunctionSig.h
@@ -1,9 +1,8 @@
 //===- NativeTypeFunctionSig.h - info about function signature ---*- C++-*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/PDB/Native/NativeTypePointer.h b/include/llvm/DebugInfo/PDB/Native/NativeTypePointer.h
index bcb7431fecf1bb916eb54ad71b29d7ca4da283be..446f77db0f6c6d6266ab9832ebe00617268d8698 100644
--- a/include/llvm/DebugInfo/PDB/Native/NativeTypePointer.h
+++ b/include/llvm/DebugInfo/PDB/Native/NativeTypePointer.h
@@ -1,9 +1,8 @@
 //===- NativeTypePointer.h - info about pointer type -------------*- C++-*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/PDB/Native/NativeTypeTypedef.h b/include/llvm/DebugInfo/PDB/Native/NativeTypeTypedef.h
index 06eb6fcf376499a62d3ba27fc72ef7941b382dc4..fe8a6f7f2bda2d727c5f99bfb65b9843d1a8bd9a 100644
--- a/include/llvm/DebugInfo/PDB/Native/NativeTypeTypedef.h
+++ b/include/llvm/DebugInfo/PDB/Native/NativeTypeTypedef.h
@@ -1,9 +1,8 @@
 //===- NativeTypeTypedef.h - info about typedef ------------------*- C++-*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/PDB/Native/NativeTypeUDT.h b/include/llvm/DebugInfo/PDB/Native/NativeTypeUDT.h
index 84821d8731bef634b6632fbfe453579ce00dbcde..8f4dee3e658cb781dc7cab4f0f39ee031eeb1c8b 100644
--- a/include/llvm/DebugInfo/PDB/Native/NativeTypeUDT.h
+++ b/include/llvm/DebugInfo/PDB/Native/NativeTypeUDT.h
@@ -1,9 +1,8 @@
 //===- NativeTypeUDT.h - info about class/struct type ------------*- C++-*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/PDB/Native/NativeTypeVTShape.h b/include/llvm/DebugInfo/PDB/Native/NativeTypeVTShape.h
index a996f34ef8598691c028908be85eb6c418d462ad..4ec0f9bf6b3dcf9caf16516e303be7c75f6daad9 100644
--- a/include/llvm/DebugInfo/PDB/Native/NativeTypeVTShape.h
+++ b/include/llvm/DebugInfo/PDB/Native/NativeTypeVTShape.h
@@ -1,9 +1,8 @@
 //===- NativeTypeVTShape.h - info about virtual table shape ------*- C++-*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/PDB/Native/PDBFile.h b/include/llvm/DebugInfo/PDB/Native/PDBFile.h
index 5e39ac3e37b722b12a8b912e879869270521310c..cb9bd07125e87c7ec5e1f7900c03457c16e87ceb 100644
--- a/include/llvm/DebugInfo/PDB/Native/PDBFile.h
+++ b/include/llvm/DebugInfo/PDB/Native/PDBFile.h
@@ -1,9 +1,8 @@
 //===- PDBFile.h - Low level interface to a PDB file ------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/PDB/Native/PDBFileBuilder.h b/include/llvm/DebugInfo/PDB/Native/PDBFileBuilder.h
index 37458749a8d85d578debc0f1ddbf9e14264fac66..72000bdc011ac9e77016ee2df11ee6ef486dbdb5 100644
--- a/include/llvm/DebugInfo/PDB/Native/PDBFileBuilder.h
+++ b/include/llvm/DebugInfo/PDB/Native/PDBFileBuilder.h
@@ -1,9 +1,8 @@
 //===- PDBFileBuilder.h - PDB File Creation ---------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/PDB/Native/PDBStringTable.h b/include/llvm/DebugInfo/PDB/Native/PDBStringTable.h
index 29167c966d427f9973053b34d81ee818f5f8b106..57f0b64a32a681ce9b2677495ef91aa1e042b14d 100644
--- a/include/llvm/DebugInfo/PDB/Native/PDBStringTable.h
+++ b/include/llvm/DebugInfo/PDB/Native/PDBStringTable.h
@@ -1,9 +1,8 @@
 //===- PDBStringTable.h - PDB String Table -----------------------*- C++-*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/PDB/Native/PDBStringTableBuilder.h b/include/llvm/DebugInfo/PDB/Native/PDBStringTableBuilder.h
index 0f81c18eafe6df320c0189b4e9da957d4bdee103..57267ef5c6c599c1fa7d0d3c9afbdf0a5f5de37e 100644
--- a/include/llvm/DebugInfo/PDB/Native/PDBStringTableBuilder.h
+++ b/include/llvm/DebugInfo/PDB/Native/PDBStringTableBuilder.h
@@ -1,9 +1,8 @@
 //===- PDBStringTableBuilder.h - PDB String Table Builder -------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/DebugInfo/PDB/Native/PublicsStream.h b/include/llvm/DebugInfo/PDB/Native/PublicsStream.h
index 2d0222a9071a0d871d5a07e4d1651ecd261b8198..ee28d108df8b67d7284900f5a150d41cd53bc893 100644
--- a/include/llvm/DebugInfo/PDB/Native/PublicsStream.h
+++ b/include/llvm/DebugInfo/PDB/Native/PublicsStream.h
@@ -1,9 +1,8 @@
 //===- PublicsStream.h - PDB Public Symbol Stream -------- ------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/PDB/Native/RawConstants.h b/include/llvm/DebugInfo/PDB/Native/RawConstants.h
index fbbd3318d958ab4bbddcae6c11da6c60a872d0d9..0dde5ef66932d302830ef4618721e130871cf5a9 100644
--- a/include/llvm/DebugInfo/PDB/Native/RawConstants.h
+++ b/include/llvm/DebugInfo/PDB/Native/RawConstants.h
@@ -1,9 +1,8 @@
 //===- RawConstants.h -------------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/PDB/Native/RawError.h b/include/llvm/DebugInfo/PDB/Native/RawError.h
index 97d11b4f20d12143d003d407a79b6d1aab1469f4..aadb64c2e3f17a3dcf9e7ad225f3f4506ab73de4 100644
--- a/include/llvm/DebugInfo/PDB/Native/RawError.h
+++ b/include/llvm/DebugInfo/PDB/Native/RawError.h
@@ -1,9 +1,8 @@
 //===- RawError.h - Error extensions for raw PDB implementation -*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/PDB/Native/RawTypes.h b/include/llvm/DebugInfo/PDB/Native/RawTypes.h
index 8f6d6611c032879ce37cb9eb471246825129c31f..84b0cb39bf132e0570147837ccfdf108ae3d6c03 100644
--- a/include/llvm/DebugInfo/PDB/Native/RawTypes.h
+++ b/include/llvm/DebugInfo/PDB/Native/RawTypes.h
@@ -1,9 +1,8 @@
 //===- RawTypes.h -----------------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/PDB/Native/SymbolCache.h b/include/llvm/DebugInfo/PDB/Native/SymbolCache.h
index 08e1d41e6ee93903092d5ca05b31598a81724720..0b15ab474f7101d12df92cca88cc072888808611 100644
--- a/include/llvm/DebugInfo/PDB/Native/SymbolCache.h
+++ b/include/llvm/DebugInfo/PDB/Native/SymbolCache.h
@@ -1,9 +1,8 @@
 //==- SymbolCache.h - Cache of native symbols and ids ------------*- C++ -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/PDB/Native/SymbolStream.h b/include/llvm/DebugInfo/PDB/Native/SymbolStream.h
index ae9f7d657b701e2c3ad7cbaf4b75497b9008e1d1..4fe1bd9734e46d6bb48784eaff340a8edc8416b4 100644
--- a/include/llvm/DebugInfo/PDB/Native/SymbolStream.h
+++ b/include/llvm/DebugInfo/PDB/Native/SymbolStream.h
@@ -1,9 +1,8 @@
 //===- SymbolStream.cpp - PDB Symbol Stream Access --------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/PDB/Native/TpiHashing.h b/include/llvm/DebugInfo/PDB/Native/TpiHashing.h
index c2996ccf1825d027563add93fd0727ba2bda4026..4ac60a80e7017164f1ec678f74cb949b9036d84f 100644
--- a/include/llvm/DebugInfo/PDB/Native/TpiHashing.h
+++ b/include/llvm/DebugInfo/PDB/Native/TpiHashing.h
@@ -1,9 +1,8 @@
 //===- TpiHashing.h ---------------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/PDB/Native/TpiStream.h b/include/llvm/DebugInfo/PDB/Native/TpiStream.h
index b76576a7a263bb4675a9dfab39cd348549a46450..1b7fd2d54cb22412c0ae0199995ae922ed230fce 100644
--- a/include/llvm/DebugInfo/PDB/Native/TpiStream.h
+++ b/include/llvm/DebugInfo/PDB/Native/TpiStream.h
@@ -1,9 +1,8 @@
 //===- TpiStream.cpp - PDB Type Info (TPI) Stream 2 Access ------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/PDB/Native/TpiStreamBuilder.h b/include/llvm/DebugInfo/PDB/Native/TpiStreamBuilder.h
index 411720d6f56b5694308a627066d05bd7b3921890..72d98e9c2c4d199b7cefb4b0ffef2c1ba895d10a 100644
--- a/include/llvm/DebugInfo/PDB/Native/TpiStreamBuilder.h
+++ b/include/llvm/DebugInfo/PDB/Native/TpiStreamBuilder.h
@@ -1,9 +1,8 @@
 //===- TpiStreamBuilder.h - PDB Tpi Stream Creation -------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/PDB/PDB.h b/include/llvm/DebugInfo/PDB/PDB.h
index 9f9da39ca6ccd476e517427887ffc3d5a006b039..6d734dc2f243ac40913195d111a4f25aa37f7fbe 100644
--- a/include/llvm/DebugInfo/PDB/PDB.h
+++ b/include/llvm/DebugInfo/PDB/PDB.h
@@ -1,9 +1,8 @@
 //===- PDB.h - base header file for creating a PDB reader -------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/PDB/PDBContext.h b/include/llvm/DebugInfo/PDB/PDBContext.h
index 0ce49f5ef92237fac249f4a9d13ecaf93e6d712b..e404e07f7583e2f28c09890d7fc3c6b5f7c666a7 100644
--- a/include/llvm/DebugInfo/PDB/PDBContext.h
+++ b/include/llvm/DebugInfo/PDB/PDBContext.h
@@ -1,9 +1,8 @@
 //===-- PDBContext.h --------------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===/
 
diff --git a/include/llvm/DebugInfo/PDB/PDBExtras.h b/include/llvm/DebugInfo/PDB/PDBExtras.h
index aaec71aa8c90da17eb8ebc191cf13fd009cd3cfe..b9a8d8f6ac020dc1e8f4506d77c3279ca0b01955 100644
--- a/include/llvm/DebugInfo/PDB/PDBExtras.h
+++ b/include/llvm/DebugInfo/PDB/PDBExtras.h
@@ -1,9 +1,8 @@
 //===- PDBExtras.h - helper functions and classes for PDBs ------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/PDB/PDBSymDumper.h b/include/llvm/DebugInfo/PDB/PDBSymDumper.h
index c976935c48e027c890439e77b3ac49284d94cd25..f81b15f2353d702f813f8a137a7272bb1367dd2e 100644
--- a/include/llvm/DebugInfo/PDB/PDBSymDumper.h
+++ b/include/llvm/DebugInfo/PDB/PDBSymDumper.h
@@ -1,9 +1,8 @@
 //===- PDBSymDumper.h - base interface for PDB symbol dumper *- C++ -----*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/PDB/PDBSymbol.h b/include/llvm/DebugInfo/PDB/PDBSymbol.h
index 3a74f7c3aace35887432cdd10add558e806b8701..d9004a8894d9778f0e3c38578135f897e2fd2cdb 100644
--- a/include/llvm/DebugInfo/PDB/PDBSymbol.h
+++ b/include/llvm/DebugInfo/PDB/PDBSymbol.h
@@ -1,9 +1,8 @@
 //===- PDBSymbol.h - base class for user-facing symbol types -----*- C++-*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/PDB/PDBSymbolAnnotation.h b/include/llvm/DebugInfo/PDB/PDBSymbolAnnotation.h
index ef00df15cb0a16600479b0658399cfaf22a2d3d1..c76466a97b66d6e8e78dc8bca0e20e162ceabdf3 100644
--- a/include/llvm/DebugInfo/PDB/PDBSymbolAnnotation.h
+++ b/include/llvm/DebugInfo/PDB/PDBSymbolAnnotation.h
@@ -1,9 +1,8 @@
 //===- PDBSymbolAnnotation.h - Accessors for querying PDB annotations ---*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 #ifndef LLVM_DEBUGINFO_PDB_PDBSYMBOLANNOTATION_H
diff --git a/include/llvm/DebugInfo/PDB/PDBSymbolBlock.h b/include/llvm/DebugInfo/PDB/PDBSymbolBlock.h
index 2cf9c72a8886a7bdc9f1de6ea6b5078dd76b24f1..cf471450d9891eec21643084b03ae2b8271abf18 100644
--- a/include/llvm/DebugInfo/PDB/PDBSymbolBlock.h
+++ b/include/llvm/DebugInfo/PDB/PDBSymbolBlock.h
@@ -1,9 +1,8 @@
 //===- PDBSymbolBlock.h - Accessors for querying PDB blocks -------------*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 #ifndef LLVM_DEBUGINFO_PDB_PDBSYMBOLBLOCK_H
diff --git a/include/llvm/DebugInfo/PDB/PDBSymbolCompiland.h b/include/llvm/DebugInfo/PDB/PDBSymbolCompiland.h
index 04dbd962ebd43102cb5e7c12a2c3ecdb044a2d9b..ca8b39d03f86f2012ce613c866048456965f6eaf 100644
--- a/include/llvm/DebugInfo/PDB/PDBSymbolCompiland.h
+++ b/include/llvm/DebugInfo/PDB/PDBSymbolCompiland.h
@@ -1,9 +1,8 @@
 //===- PDBSymbolCompiland.h - Accessors for querying PDB compilands -----*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 #ifndef LLVM_DEBUGINFO_PDB_PDBSYMBOLCOMPILAND_H
diff --git a/include/llvm/DebugInfo/PDB/PDBSymbolCompilandDetails.h b/include/llvm/DebugInfo/PDB/PDBSymbolCompilandDetails.h
index 3d651a464d94c9c3cd50e955522341b1529cb69f..b82bb6c0a352b9bb32b81b6a6783d09795553fb2 100644
--- a/include/llvm/DebugInfo/PDB/PDBSymbolCompilandDetails.h
+++ b/include/llvm/DebugInfo/PDB/PDBSymbolCompilandDetails.h
@@ -1,9 +1,8 @@
 //===- PDBSymbolCompilandDetails.h - PDB compiland details ------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/PDB/PDBSymbolCompilandEnv.h b/include/llvm/DebugInfo/PDB/PDBSymbolCompilandEnv.h
index ffc408314d9afbc50c5511391ab7af749b065c69..61607a03593d8dc727c08289e77cdd771e2ecffa 100644
--- a/include/llvm/DebugInfo/PDB/PDBSymbolCompilandEnv.h
+++ b/include/llvm/DebugInfo/PDB/PDBSymbolCompilandEnv.h
@@ -1,9 +1,8 @@
 //===- PDBSymbolCompilandEnv.h - compiland environment variables *- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/PDB/PDBSymbolCustom.h b/include/llvm/DebugInfo/PDB/PDBSymbolCustom.h
index c29e4c31d3f34af4336f506e48050ea1ac39e3a5..75a86411643a62c6a8686c2be383f29860add0e8 100644
--- a/include/llvm/DebugInfo/PDB/PDBSymbolCustom.h
+++ b/include/llvm/DebugInfo/PDB/PDBSymbolCustom.h
@@ -1,9 +1,8 @@
 //===- PDBSymbolCustom.h - compiler-specific types --------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/PDB/PDBSymbolData.h b/include/llvm/DebugInfo/PDB/PDBSymbolData.h
index 217e1e976e6be837e1afec89613dbfb8c8b70230..7e9b69d7cf4bb84a60be89bf1ba832de887562a4 100644
--- a/include/llvm/DebugInfo/PDB/PDBSymbolData.h
+++ b/include/llvm/DebugInfo/PDB/PDBSymbolData.h
@@ -1,9 +1,8 @@
 //===- PDBSymbolData.h - PDB data (e.g. variable) accessors -----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/PDB/PDBSymbolExe.h b/include/llvm/DebugInfo/PDB/PDBSymbolExe.h
index 366d0cf4777f9384eda315d40aee8fcf99bc0d0c..1a9fb240a248849a96d5ac7f121f162be8f8c830 100644
--- a/include/llvm/DebugInfo/PDB/PDBSymbolExe.h
+++ b/include/llvm/DebugInfo/PDB/PDBSymbolExe.h
@@ -1,9 +1,8 @@
 //===- PDBSymbolExe.h - Accessors for querying executables in a PDB ----*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/PDB/PDBSymbolFunc.h b/include/llvm/DebugInfo/PDB/PDBSymbolFunc.h
index 129e557c7f25df974b278f43b1ab42e9910e5407..6be27c8d3bc7e1294872a145a2017f2889421456 100644
--- a/include/llvm/DebugInfo/PDB/PDBSymbolFunc.h
+++ b/include/llvm/DebugInfo/PDB/PDBSymbolFunc.h
@@ -1,9 +1,8 @@
 //===- PDBSymbolFunc.h - class representing a function instance -*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/PDB/PDBSymbolFuncDebugEnd.h b/include/llvm/DebugInfo/PDB/PDBSymbolFuncDebugEnd.h
index 18db8a50fd1b2cfb9f3533fc35ec95bec1ad9627..7152249cbd038e3336d3855e3b25d9489c7ee859 100644
--- a/include/llvm/DebugInfo/PDB/PDBSymbolFuncDebugEnd.h
+++ b/include/llvm/DebugInfo/PDB/PDBSymbolFuncDebugEnd.h
@@ -1,9 +1,8 @@
 //===- PDBSymbolFuncDebugEnd.h - function end bounds info -------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/PDB/PDBSymbolFuncDebugStart.h b/include/llvm/DebugInfo/PDB/PDBSymbolFuncDebugStart.h
index 83d82f0cbcc5d45593b9fd3ace1763272e002320..3125c271d2e8a75d6976ab67d71eaafa60a86f09 100644
--- a/include/llvm/DebugInfo/PDB/PDBSymbolFuncDebugStart.h
+++ b/include/llvm/DebugInfo/PDB/PDBSymbolFuncDebugStart.h
@@ -1,9 +1,8 @@
 //===- PDBSymbolFuncDebugStart.h - function start bounds info ---*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/PDB/PDBSymbolLabel.h b/include/llvm/DebugInfo/PDB/PDBSymbolLabel.h
index 8b2617fcd757c414e4cbbf6e8be32095574a01e3..3625e23f014f15e44d3f52028d027f8c32b43502 100644
--- a/include/llvm/DebugInfo/PDB/PDBSymbolLabel.h
+++ b/include/llvm/DebugInfo/PDB/PDBSymbolLabel.h
@@ -1,9 +1,8 @@
 //===- PDBSymbolLabel.h - label info ----------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/PDB/PDBSymbolPublicSymbol.h b/include/llvm/DebugInfo/PDB/PDBSymbolPublicSymbol.h
index 9def3edb469aaf9327cb6ac5ccb3db779662059e..e2b2545d78ec1a0a50e0fb5c601981326a8f385e 100644
--- a/include/llvm/DebugInfo/PDB/PDBSymbolPublicSymbol.h
+++ b/include/llvm/DebugInfo/PDB/PDBSymbolPublicSymbol.h
@@ -1,9 +1,8 @@
 //===- PDBSymbolPublicSymbol.h - public symbol info -------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/PDB/PDBSymbolThunk.h b/include/llvm/DebugInfo/PDB/PDBSymbolThunk.h
index 7bb0555362dbd2c1c0709013d2eee34aef6480fe..274de8b0b16f8e6b63730dba206bbd0b0350161c 100644
--- a/include/llvm/DebugInfo/PDB/PDBSymbolThunk.h
+++ b/include/llvm/DebugInfo/PDB/PDBSymbolThunk.h
@@ -1,9 +1,8 @@
 //===- PDBSymbolThunk.h - Support for querying PDB thunks ---------------*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/PDB/PDBSymbolTypeArray.h b/include/llvm/DebugInfo/PDB/PDBSymbolTypeArray.h
index 488f668bdc10d4d5dfcf66c6c704358b41cef88d..c0215c9ee4b1808521b1d20cba4880ecac5ac255 100644
--- a/include/llvm/DebugInfo/PDB/PDBSymbolTypeArray.h
+++ b/include/llvm/DebugInfo/PDB/PDBSymbolTypeArray.h
@@ -1,9 +1,8 @@
 //===- PDBSymbolTypeArray.h - array type information ------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/PDB/PDBSymbolTypeBaseClass.h b/include/llvm/DebugInfo/PDB/PDBSymbolTypeBaseClass.h
index 550deedd7504883af06f924d43828776c9b0880e..bab292ee0d46eb72ebaf4fdfdc34d8733aca0b50 100644
--- a/include/llvm/DebugInfo/PDB/PDBSymbolTypeBaseClass.h
+++ b/include/llvm/DebugInfo/PDB/PDBSymbolTypeBaseClass.h
@@ -1,9 +1,8 @@
 //===- PDBSymbolTypeBaseClass.h - base class type information ---*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/PDB/PDBSymbolTypeBuiltin.h b/include/llvm/DebugInfo/PDB/PDBSymbolTypeBuiltin.h
index e07e88802b8f1efda533217b52020f9aac50856f..7d94c3c97a2b1dde7900e37a30deaf908043be2a 100644
--- a/include/llvm/DebugInfo/PDB/PDBSymbolTypeBuiltin.h
+++ b/include/llvm/DebugInfo/PDB/PDBSymbolTypeBuiltin.h
@@ -1,9 +1,8 @@
 //===- PDBSymbolTypeBuiltin.h - builtin type information --------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/PDB/PDBSymbolTypeCustom.h b/include/llvm/DebugInfo/PDB/PDBSymbolTypeCustom.h
index 0d8979c9c5c5669013deb6d2fc9e7a9deeb322d8..dc647aff48d3ce436be4bb4405ceea5349d28b8e 100644
--- a/include/llvm/DebugInfo/PDB/PDBSymbolTypeCustom.h
+++ b/include/llvm/DebugInfo/PDB/PDBSymbolTypeCustom.h
@@ -1,9 +1,8 @@
 //===- PDBSymbolTypeCustom.h - custom compiler type information -*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/PDB/PDBSymbolTypeDimension.h b/include/llvm/DebugInfo/PDB/PDBSymbolTypeDimension.h
index 58292a63501f7474042b46def21ed85fc6d0e0b5..7a9e43785d67f37f7e4b2df0bbf9597377c45033 100644
--- a/include/llvm/DebugInfo/PDB/PDBSymbolTypeDimension.h
+++ b/include/llvm/DebugInfo/PDB/PDBSymbolTypeDimension.h
@@ -1,9 +1,8 @@
 //===- PDBSymbolTypeDimension.h - array dimension type info -----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/PDB/PDBSymbolTypeEnum.h b/include/llvm/DebugInfo/PDB/PDBSymbolTypeEnum.h
index f463047bb5b55484dcd92bd17e30f6e922e7f492..3ac72801b202d8df6a9493194030040f7fb6a84d 100644
--- a/include/llvm/DebugInfo/PDB/PDBSymbolTypeEnum.h
+++ b/include/llvm/DebugInfo/PDB/PDBSymbolTypeEnum.h
@@ -1,9 +1,8 @@
 //===- PDBSymbolTypeEnum.h - enum type info ---------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/PDB/PDBSymbolTypeFriend.h b/include/llvm/DebugInfo/PDB/PDBSymbolTypeFriend.h
index 5b940b0737afad7e73c6e7d564e4b9b6002c98af..c4d9dd6308a36b4c3719b0a67bbd104d08838680 100644
--- a/include/llvm/DebugInfo/PDB/PDBSymbolTypeFriend.h
+++ b/include/llvm/DebugInfo/PDB/PDBSymbolTypeFriend.h
@@ -1,9 +1,8 @@
 //===- PDBSymbolTypeFriend.h - friend type info -----------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/PDB/PDBSymbolTypeFunctionArg.h b/include/llvm/DebugInfo/PDB/PDBSymbolTypeFunctionArg.h
index 074cb418fc8292b726c6565ca291acc537ad7e74..22d3623496f293b28ca69ee3c235f13ad3f51f39 100644
--- a/include/llvm/DebugInfo/PDB/PDBSymbolTypeFunctionArg.h
+++ b/include/llvm/DebugInfo/PDB/PDBSymbolTypeFunctionArg.h
@@ -1,9 +1,8 @@
 //===- PDBSymbolTypeFunctionArg.h - function arg type info ------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/PDB/PDBSymbolTypeFunctionSig.h b/include/llvm/DebugInfo/PDB/PDBSymbolTypeFunctionSig.h
index dfdf436197c3733d24d740bb4ed275c5fe91711a..a1491ca2e415ec043507a35b8e8526a70405c9f5 100644
--- a/include/llvm/DebugInfo/PDB/PDBSymbolTypeFunctionSig.h
+++ b/include/llvm/DebugInfo/PDB/PDBSymbolTypeFunctionSig.h
@@ -1,9 +1,8 @@
 //===- PDBSymbolTypeFunctionSig.h - function signature type info *- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/PDB/PDBSymbolTypeManaged.h b/include/llvm/DebugInfo/PDB/PDBSymbolTypeManaged.h
index d716abd640c638253a9ea0b1ae60c80838846f55..6bc70bca82e74a35e99f8f017f24dab093d70d95 100644
--- a/include/llvm/DebugInfo/PDB/PDBSymbolTypeManaged.h
+++ b/include/llvm/DebugInfo/PDB/PDBSymbolTypeManaged.h
@@ -1,9 +1,8 @@
 //===- PDBSymbolTypeManaged.h - managed type info ---------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/PDB/PDBSymbolTypePointer.h b/include/llvm/DebugInfo/PDB/PDBSymbolTypePointer.h
index 300d6722fc4d6d95a754cd98e73ec22db329212e..b36f459e880cd03ce87d15bf473f8d9a294c87f8 100644
--- a/include/llvm/DebugInfo/PDB/PDBSymbolTypePointer.h
+++ b/include/llvm/DebugInfo/PDB/PDBSymbolTypePointer.h
@@ -1,9 +1,8 @@
 //===- PDBSymbolTypePointer.h - pointer type info ---------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/PDB/PDBSymbolTypeTypedef.h b/include/llvm/DebugInfo/PDB/PDBSymbolTypeTypedef.h
index d6e2a36486d5ff0d4ec001bce2017c52d10b2098..2712d0617e0e1d717233a7580ebe6dd549b349c6 100644
--- a/include/llvm/DebugInfo/PDB/PDBSymbolTypeTypedef.h
+++ b/include/llvm/DebugInfo/PDB/PDBSymbolTypeTypedef.h
@@ -1,9 +1,8 @@
 //===- PDBSymbolTypeTypedef.h - typedef type info ---------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/PDB/PDBSymbolTypeUDT.h b/include/llvm/DebugInfo/PDB/PDBSymbolTypeUDT.h
index 937dd6c87221c6b701638271ba3ffbc75eb572a3..3e73ad7ac85a3290e3e0fe702c047c377bb10213 100644
--- a/include/llvm/DebugInfo/PDB/PDBSymbolTypeUDT.h
+++ b/include/llvm/DebugInfo/PDB/PDBSymbolTypeUDT.h
@@ -1,9 +1,8 @@
 //===- PDBSymbolTypeUDT.h - UDT type info -----------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/PDB/PDBSymbolTypeVTable.h b/include/llvm/DebugInfo/PDB/PDBSymbolTypeVTable.h
index 6efce4bbd686d23695e6c7f5f962c1e1958a0063..e8161d311ea7f674652872f3f47ca7ce13389263 100644
--- a/include/llvm/DebugInfo/PDB/PDBSymbolTypeVTable.h
+++ b/include/llvm/DebugInfo/PDB/PDBSymbolTypeVTable.h
@@ -1,9 +1,8 @@
 //===- PDBSymbolTypeVTable.h - VTable type info -----------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/PDB/PDBSymbolTypeVTableShape.h b/include/llvm/DebugInfo/PDB/PDBSymbolTypeVTableShape.h
index 8949052b0c0f62901b5e845f06640d7b357cc886..614060867042a54a0997a5c8d300067aea8f7fbb 100644
--- a/include/llvm/DebugInfo/PDB/PDBSymbolTypeVTableShape.h
+++ b/include/llvm/DebugInfo/PDB/PDBSymbolTypeVTableShape.h
@@ -1,9 +1,8 @@
 //===- PDBSymbolTypeVTableShape.h - VTable shape info -----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/PDB/PDBSymbolUnknown.h b/include/llvm/DebugInfo/PDB/PDBSymbolUnknown.h
index e935ac6ce0dce1391eb7f9f8ef4253639b81b519..cc29d38c2578c519ccf62205a4cce71d622eef63 100644
--- a/include/llvm/DebugInfo/PDB/PDBSymbolUnknown.h
+++ b/include/llvm/DebugInfo/PDB/PDBSymbolUnknown.h
@@ -1,9 +1,8 @@
 //===- PDBSymbolUnknown.h - unknown symbol type -----------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/PDB/PDBSymbolUsingNamespace.h b/include/llvm/DebugInfo/PDB/PDBSymbolUsingNamespace.h
index 4e8c99fc8d89060cd2a1a43a06a23784131fc0e3..fd812cb2f7936391ee5109fadf9cb4e1fc7568bc 100644
--- a/include/llvm/DebugInfo/PDB/PDBSymbolUsingNamespace.h
+++ b/include/llvm/DebugInfo/PDB/PDBSymbolUsingNamespace.h
@@ -1,9 +1,8 @@
 //===- PDBSymbolUsingNamespace.h - using namespace info ---------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/PDB/PDBTypes.h b/include/llvm/DebugInfo/PDB/PDBTypes.h
index 917f3ed73910af8f87682ebc702afbe92135c5bc..6d26b64ddf7264a53db6f170fa834e9827332039 100644
--- a/include/llvm/DebugInfo/PDB/PDBTypes.h
+++ b/include/llvm/DebugInfo/PDB/PDBTypes.h
@@ -1,9 +1,8 @@
 //===- PDBTypes.h - Defines enums for various fields contained in PDB ----====//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/PDB/UDTLayout.h b/include/llvm/DebugInfo/PDB/UDTLayout.h
index c4234c191e21c033b6b2d5397c339bfec7289810..c67b093b63c0b80e437ee5e71d7b541a6c7b16ef 100644
--- a/include/llvm/DebugInfo/PDB/UDTLayout.h
+++ b/include/llvm/DebugInfo/PDB/UDTLayout.h
@@ -1,9 +1,8 @@
 //===- UDTLayout.h - UDT layout info ----------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/DebugInfo/Symbolize/DIPrinter.h b/include/llvm/DebugInfo/Symbolize/DIPrinter.h
index ab82be3706d8391c0b24ec5da3d547f67548ef00..71663f30a59ff6314e277f9537ed9dfd4b3fd03a 100644
--- a/include/llvm/DebugInfo/Symbolize/DIPrinter.h
+++ b/include/llvm/DebugInfo/Symbolize/DIPrinter.h
@@ -1,9 +1,8 @@
 //===- llvm/DebugInfo/Symbolize/DIPrinter.h ---------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -30,6 +29,7 @@ class DIPrinter {
   bool PrintPretty;
   int PrintSourceContext;
   bool Verbose;
+  bool Basenames;
 
   void print(const DILineInfo &Info, bool Inlined);
   void printContext(const std::string &FileName, int64_t Line);
@@ -37,10 +37,10 @@ class DIPrinter {
 public:
   DIPrinter(raw_ostream &OS, bool PrintFunctionNames = true,
             bool PrintPretty = false, int PrintSourceContext = 0,
-            bool Verbose = false)
+            bool Verbose = false, bool Basenames = false)
       : OS(OS), PrintFunctionNames(PrintFunctionNames),
         PrintPretty(PrintPretty), PrintSourceContext(PrintSourceContext),
-        Verbose(Verbose) {}
+        Verbose(Verbose), Basenames(Basenames) {}
 
   DIPrinter &operator<<(const DILineInfo &Info);
   DIPrinter &operator<<(const DIInliningInfo &Info);
diff --git a/include/llvm/DebugInfo/Symbolize/SymbolizableModule.h b/include/llvm/DebugInfo/Symbolize/SymbolizableModule.h
index e576a91e887cc6c3a47f1f692b0f49e4909ffebc..f0862d0b85d2b74e83695c5b2ce46250ded1ee20 100644
--- a/include/llvm/DebugInfo/Symbolize/SymbolizableModule.h
+++ b/include/llvm/DebugInfo/Symbolize/SymbolizableModule.h
@@ -1,9 +1,8 @@
 //===- SymbolizableModule.h -------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/DebugInfo/Symbolize/Symbolize.h b/include/llvm/DebugInfo/Symbolize/Symbolize.h
index 289148f569db63d0a27a10ff4cfc58e6c874e7cf..4e57fe43847811f2012f07558ad06470f3a20176 100644
--- a/include/llvm/DebugInfo/Symbolize/Symbolize.h
+++ b/include/llvm/DebugInfo/Symbolize/Symbolize.h
@@ -1,9 +1,8 @@
 //===- Symbolize.h ----------------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -42,13 +41,16 @@ public:
     bool RelativeAddresses : 1;
     std::string DefaultArch;
     std::vector<std::string> DsymHints;
+    std::string FallbackDebugPath;
 
     Options(FunctionNameKind PrintFunctions = FunctionNameKind::LinkageName,
             bool UseSymbolTable = true, bool Demangle = true,
-            bool RelativeAddresses = false, std::string DefaultArch = "")
+            bool RelativeAddresses = false, std::string DefaultArch = "",
+            std::string FallbackDebugPath = "")
         : PrintFunctions(PrintFunctions), UseSymbolTable(UseSymbolTable),
           Demangle(Demangle), RelativeAddresses(RelativeAddresses),
-          DefaultArch(std::move(DefaultArch)) {}
+          DefaultArch(std::move(DefaultArch)),
+          FallbackDebugPath(std::move(FallbackDebugPath)) {}
   };
 
   LLVMSymbolizer(const Options &Opts = Options()) : Opts(Opts) {}
diff --git a/include/llvm/Demangle/Compiler.h b/include/llvm/Demangle/Compiler.h
deleted file mode 100644
index 248d6e3a7faaee02c095b8bb1079ff1ff4fd16f7..0000000000000000000000000000000000000000
--- a/include/llvm/Demangle/Compiler.h
+++ /dev/null
@@ -1,93 +0,0 @@
-//===--- Compiler.h ---------------------------------------------*- C++ -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//
-// This file contains a variety of feature test macros copied from
-// include/llvm/Support/Compiler.h so that LLVMDemangle does not need to take
-// a dependency on LLVMSupport.
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_DEMANGLE_COMPILER_H
-#define LLVM_DEMANGLE_COMPILER_H
-
-#ifdef _MSC_VER
-// snprintf is implemented in VS 2015
-#if _MSC_VER < 1900
-#define snprintf _snprintf_s
-#endif
-#endif
-
-#ifndef __has_feature
-#define __has_feature(x) 0
-#endif
-
-#ifndef __has_cpp_attribute
-#define __has_cpp_attribute(x) 0
-#endif
-
-#ifndef __has_attribute
-#define __has_attribute(x) 0
-#endif
-
-#ifndef __has_builtin
-#define __has_builtin(x) 0
-#endif
-
-#ifndef LLVM_GNUC_PREREQ
-#if defined(__GNUC__) && defined(__GNUC_MINOR__) && defined(__GNUC_PATCHLEVEL__)
-#define LLVM_GNUC_PREREQ(maj, min, patch)                                      \
-  ((__GNUC__ << 20) + (__GNUC_MINOR__ << 10) + __GNUC_PATCHLEVEL__ >=          \
-   ((maj) << 20) + ((min) << 10) + (patch))
-#elif defined(__GNUC__) && defined(__GNUC_MINOR__)
-#define LLVM_GNUC_PREREQ(maj, min, patch)                                      \
-  ((__GNUC__ << 20) + (__GNUC_MINOR__ << 10) >= ((maj) << 20) + ((min) << 10))
-#else
-#define LLVM_GNUC_PREREQ(maj, min, patch) 0
-#endif
-#endif
-
-#if __has_attribute(used) || LLVM_GNUC_PREREQ(3, 1, 0)
-#define LLVM_ATTRIBUTE_USED __attribute__((__used__))
-#else
-#define LLVM_ATTRIBUTE_USED
-#endif
-
-#if __has_builtin(__builtin_unreachable) || LLVM_GNUC_PREREQ(4, 5, 0)
-#define LLVM_BUILTIN_UNREACHABLE __builtin_unreachable()
-#elif defined(_MSC_VER)
-#define LLVM_BUILTIN_UNREACHABLE __assume(false)
-#endif
-
-#if __has_attribute(noinline) || LLVM_GNUC_PREREQ(3, 4, 0)
-#define LLVM_ATTRIBUTE_NOINLINE __attribute__((noinline))
-#elif defined(_MSC_VER)
-#define LLVM_ATTRIBUTE_NOINLINE __declspec(noinline)
-#else
-#define LLVM_ATTRIBUTE_NOINLINE
-#endif
-
-#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
-#define LLVM_DUMP_METHOD LLVM_ATTRIBUTE_NOINLINE LLVM_ATTRIBUTE_USED
-#else
-#define LLVM_DUMP_METHOD LLVM_ATTRIBUTE_NOINLINE
-#endif
-
-#if __cplusplus > 201402L && __has_cpp_attribute(fallthrough)
-#define LLVM_FALLTHROUGH [[fallthrough]]
-#elif __has_cpp_attribute(gnu::fallthrough)
-#define LLVM_FALLTHROUGH [[gnu::fallthrough]]
-#elif !__cplusplus
-// Workaround for llvm.org/PR23435, since clang 3.6 and below emit a spurious
-// error when __has_cpp_attribute is given a scoped attribute in C mode.
-#define LLVM_FALLTHROUGH
-#elif __has_cpp_attribute(clang::fallthrough)
-#define LLVM_FALLTHROUGH [[clang::fallthrough]]
-#else
-#define LLVM_FALLTHROUGH
-#endif
-
-#endif
diff --git a/include/llvm/Demangle/Demangle.h b/include/llvm/Demangle/Demangle.h
index 4c9dc9569e18835621ff6f924867bc734138dd12..6fea7ef13f11d8d298d9054a042581bdf50bb8ec 100644
--- a/include/llvm/Demangle/Demangle.h
+++ b/include/llvm/Demangle/Demangle.h
@@ -1,9 +1,8 @@
 //===--- Demangle.h ---------------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -11,6 +10,7 @@
 #define LLVM_DEMANGLE_DEMANGLE_H
 
 #include <cstddef>
+#include <string>
 
 namespace llvm {
 /// This is a llvm local version of __cxa_demangle. Other than the name and
@@ -36,6 +36,13 @@ enum MSDemangleFlags { MSDF_None = 0, MSDF_DumpBackrefs = 1 << 0 };
 char *microsoftDemangle(const char *mangled_name, char *buf, size_t *n,
                         int *status, MSDemangleFlags Flags = MSDF_None);
 
+/// Attempt to demangle a string using different demangling schemes.
+/// The function uses heuristics to determine which demangling scheme to use.
+/// \param MangledName - reference to string to demangle.
+/// \returns - the demangled string, or a copy of the input string if no
+/// demangling occurred.
+std::string demangle(const std::string &MangledName);
+
 /// "Partial" demangler. This supports demangling a string into an AST
 /// (typically an intermediate stage in itaniumDemangle) and querying certain
 /// properties or partially printing the demangled name.
diff --git a/include/llvm/Demangle/DemangleConfig.h b/include/llvm/Demangle/DemangleConfig.h
new file mode 100644
index 0000000000000000000000000000000000000000..73f89d357c856082d0f8b56ab840af7c295a5f92
--- /dev/null
+++ b/include/llvm/Demangle/DemangleConfig.h
@@ -0,0 +1,99 @@
+//===--- DemangleConfig.h ---------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains a variety of feature test macros copied from
+// include/llvm/Support/Compiler.h so that LLVMDemangle does not need to take
+// a dependency on LLVMSupport.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_DEMANGLE_COMPILER_H
+#define LLVM_DEMANGLE_COMPILER_H
+
+#ifdef _MSC_VER
+// snprintf is implemented in VS 2015
+#if _MSC_VER < 1900
+#define snprintf _snprintf_s
+#endif
+#endif
+
+#ifndef __has_feature
+#define __has_feature(x) 0
+#endif
+
+#ifndef __has_cpp_attribute
+#define __has_cpp_attribute(x) 0
+#endif
+
+#ifndef __has_attribute
+#define __has_attribute(x) 0
+#endif
+
+#ifndef __has_builtin
+#define __has_builtin(x) 0
+#endif
+
+#ifndef DEMANGLE_GNUC_PREREQ
+#if defined(__GNUC__) && defined(__GNUC_MINOR__) && defined(__GNUC_PATCHLEVEL__)
+#define DEMANGLE_GNUC_PREREQ(maj, min, patch)                           \
+  ((__GNUC__ << 20) + (__GNUC_MINOR__ << 10) + __GNUC_PATCHLEVEL__ >=          \
+   ((maj) << 20) + ((min) << 10) + (patch))
+#elif defined(__GNUC__) && defined(__GNUC_MINOR__)
+#define DEMANGLE_GNUC_PREREQ(maj, min, patch)                           \
+  ((__GNUC__ << 20) + (__GNUC_MINOR__ << 10) >= ((maj) << 20) + ((min) << 10))
+#else
+#define DEMANGLE_GNUC_PREREQ(maj, min, patch) 0
+#endif
+#endif
+
+#if __has_attribute(used) || DEMANGLE_GNUC_PREREQ(3, 1, 0)
+#define DEMANGLE_ATTRIBUTE_USED __attribute__((__used__))
+#else
+#define DEMANGLE_ATTRIBUTE_USED
+#endif
+
+#if __has_builtin(__builtin_unreachable) || DEMANGLE_GNUC_PREREQ(4, 5, 0)
+#define DEMANGLE_UNREACHABLE __builtin_unreachable()
+#elif defined(_MSC_VER)
+#define DEMANGLE_UNREACHABLE __assume(false)
+#else
+#define DEMANGLE_UNREACHABLE
+#endif
+
+#if __has_attribute(noinline) || DEMANGLE_GNUC_PREREQ(3, 4, 0)
+#define DEMANGLE_ATTRIBUTE_NOINLINE __attribute__((noinline))
+#elif defined(_MSC_VER)
+#define DEMANGLE_ATTRIBUTE_NOINLINE __declspec(noinline)
+#else
+#define DEMANGLE_ATTRIBUTE_NOINLINE
+#endif
+
+#if !defined(NDEBUG)
+#define DEMANGLE_DUMP_METHOD DEMANGLE_ATTRIBUTE_NOINLINE DEMANGLE_ATTRIBUTE_USED
+#else
+#define DEMANGLE_DUMP_METHOD DEMANGLE_ATTRIBUTE_NOINLINE
+#endif
+
+#if __cplusplus > 201402L && __has_cpp_attribute(fallthrough)
+#define DEMANGLE_FALLTHROUGH [[fallthrough]]
+#elif __has_cpp_attribute(gnu::fallthrough)
+#define DEMANGLE_FALLTHROUGH [[gnu::fallthrough]]
+#elif !__cplusplus
+// Workaround for llvm.org/PR23435, since clang 3.6 and below emit a spurious
+// error when __has_cpp_attribute is given a scoped attribute in C mode.
+#define DEMANGLE_FALLTHROUGH
+#elif __has_cpp_attribute(clang::fallthrough)
+#define DEMANGLE_FALLTHROUGH [[clang::fallthrough]]
+#else
+#define DEMANGLE_FALLTHROUGH
+#endif
+
+#define DEMANGLE_NAMESPACE_BEGIN namespace llvm { namespace itanium_demangle {
+#define DEMANGLE_NAMESPACE_END } }
+
+#endif
diff --git a/include/llvm/Demangle/ItaniumDemangle.h b/include/llvm/Demangle/ItaniumDemangle.h
index 0b9187f30a5a627d3c628fa59984d34820e25e1f..df06aa62d335feced300f0affb9b88752c5c9f22 100644
--- a/include/llvm/Demangle/ItaniumDemangle.h
+++ b/include/llvm/Demangle/ItaniumDemangle.h
@@ -1,23 +1,26 @@
 //===------------------------- ItaniumDemangle.h ----------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
-// This file is dual licensed under the MIT and the University of Illinois Open
-// Source Licenses. See LICENSE.TXT for details.
+//===----------------------------------------------------------------------===//
+//
+// Generic itanium demangler library. This file has two byte-per-byte identical
+// copies in the source tree, one in libcxxabi, and the other in llvm.
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef LLVM_DEMANGLE_ITANIUMDEMANGLE_H
-#define LLVM_DEMANGLE_ITANIUMDEMANGLE_H
+#ifndef DEMANGLE_ITANIUMDEMANGLE_H
+#define DEMANGLE_ITANIUMDEMANGLE_H
 
 // FIXME: (possibly) incomplete list of features that clang mangles that this
 // file does not yet support:
 //   - C++ modules TS
 
-#include "llvm/Demangle/Compiler.h"
-#include "llvm/Demangle/StringView.h"
-#include "llvm/Demangle/Utility.h"
-
+#include "DemangleConfig.h"
+#include "StringView.h"
+#include "Utility.h"
 #include <cassert>
 #include <cctype>
 #include <cstdio>
@@ -95,8 +98,8 @@
     X(BracedExpr) \
     X(BracedRangeExpr)
 
-namespace llvm {
-namespace itanium_demangle {
+DEMANGLE_NAMESPACE_BEGIN
+
 // Base class of all AST nodes. The AST is built by the parser, then is
 // traversed by the printLeft/Right functions to produce a demangled string.
 class Node {
@@ -194,7 +197,7 @@ public:
   virtual ~Node() = default;
 
 #ifndef NDEBUG
-  LLVM_DUMP_METHOD void dump() const;
+  DEMANGLE_DUMP_METHOD void dump() const;
 #endif
 };
 
@@ -1278,7 +1281,7 @@ public:
     case SpecialSubKind::iostream:
       return StringView("basic_iostream");
     }
-    LLVM_BUILTIN_UNREACHABLE;
+    DEMANGLE_UNREACHABLE;
   }
 
   void printLeft(OutputStream &S) const override {
@@ -1330,7 +1333,7 @@ public:
     case SpecialSubKind::iostream:
       return StringView("iostream");
     }
-    LLVM_BUILTIN_UNREACHABLE;
+    DEMANGLE_UNREACHABLE;
   }
 
   void printLeft(OutputStream &S) const override {
@@ -2476,6 +2479,12 @@ AbstractManglingParser<Derived, Alloc>::parseUnnamedTypeName(NameState *) {
       return nullptr;
     return make<ClosureTypeName>(Params, Count);
   }
+  if (consumeIf("Ub")) {
+    (void)parseNumber();
+    if (!consumeIf('_'))
+      return nullptr;
+    return make<NameType>("'block-literal'");
+  }
   return nullptr;
 }
 
@@ -3467,7 +3476,7 @@ Node *AbstractManglingParser<Derived, Alloc>::parseType() {
       Result = getDerived().parseFunctionType();
       break;
     }
-    LLVM_FALLTHROUGH;
+    DEMANGLE_FALLTHROUGH;
   }
   case 'U': {
     Result = getDerived().parseQualifiedType();
@@ -3754,7 +3763,7 @@ Node *AbstractManglingParser<Derived, Alloc>::parseType() {
       // substitution table.
       return Sub;
     }
-    LLVM_FALLTHROUGH;
+    DEMANGLE_FALLTHROUGH;
   }
   //        ::= <class-enum-type>
   default: {
@@ -5139,7 +5148,7 @@ AbstractManglingParser<Derived, Alloc>::parseTemplateArgs(bool TagTemplates) {
 // extension      ::= ___Z <encoding> _block_invoke_<decimal-digit>+
 template <typename Derived, typename Alloc>
 Node *AbstractManglingParser<Derived, Alloc>::parse() {
-  if (consumeIf("_Z")) {
+  if (consumeIf("_Z") || consumeIf("__Z")) {
     Node *Encoding = getDerived().parseEncoding();
     if (Encoding == nullptr)
       return nullptr;
@@ -5152,7 +5161,7 @@ Node *AbstractManglingParser<Derived, Alloc>::parse() {
     return Encoding;
   }
 
-  if (consumeIf("___Z")) {
+  if (consumeIf("___Z") || consumeIf("____Z")) {
     Node *Encoding = getDerived().parseEncoding();
     if (Encoding == nullptr || !consumeIf("_block_invoke"))
       return nullptr;
@@ -5178,7 +5187,6 @@ struct ManglingParser : AbstractManglingParser<ManglingParser<Alloc>, Alloc> {
                                Alloc>::AbstractManglingParser;
 };
 
-}  // namespace itanium_demangle
-}  // namespace llvm
+DEMANGLE_NAMESPACE_END
 
-#endif // LLVM_DEMANGLE_ITANIUMDEMANGLE_H
+#endif // DEMANGLE_ITANIUMDEMANGLE_H
diff --git a/include/llvm/Demangle/MicrosoftDemangle.h b/include/llvm/Demangle/MicrosoftDemangle.h
index b186758ebe24ba5aa2573fc682fce34206187fbf..a23e3273509fa4e28806998093cd82c1b537c5f7 100644
--- a/include/llvm/Demangle/MicrosoftDemangle.h
+++ b/include/llvm/Demangle/MicrosoftDemangle.h
@@ -1,16 +1,15 @@
 //===------------------------- MicrosoftDemangle.h --------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is dual licensed under the MIT and the University of Illinois Open
-// Source Licenses. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #ifndef LLVM_DEMANGLE_MICROSOFT_DEMANGLE_H
 #define LLVM_DEMANGLE_MICROSOFT_DEMANGLE_H
 
-#include "llvm/Demangle/Compiler.h"
+#include "llvm/Demangle/DemangleConfig.h"
 #include "llvm/Demangle/MicrosoftDemangleNodes.h"
 #include "llvm/Demangle/StringView.h"
 #include "llvm/Demangle/Utility.h"
@@ -245,7 +244,7 @@ private:
   FuncClass demangleFunctionClass(StringView &MangledName);
   CallingConv demangleCallingConvention(StringView &MangledName);
   StorageClass demangleVariableStorageClass(StringView &MangledName);
-  void demangleThrowSpecification(StringView &MangledName);
+  bool demangleThrowSpecification(StringView &MangledName);
   wchar_t demangleWcharLiteral(StringView &MangledName);
   uint8_t demangleCharLiteral(StringView &MangledName);
 
diff --git a/include/llvm/Demangle/MicrosoftDemangleNodes.h b/include/llvm/Demangle/MicrosoftDemangleNodes.h
index 1eca6762475a0f502fcee27f39ffadd16648408a..b478418086b6cd124976a9304e24a66f269fae35 100644
--- a/include/llvm/Demangle/MicrosoftDemangleNodes.h
+++ b/include/llvm/Demangle/MicrosoftDemangleNodes.h
@@ -1,11 +1,30 @@
+//===- MicrosoftDemangleNodes.h ---------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines the AST nodes used in the MSVC demangler.
+//
+//===----------------------------------------------------------------------===//
+
 #ifndef LLVM_SUPPORT_MICROSOFTDEMANGLENODES_H
 #define LLVM_SUPPORT_MICROSOFTDEMANGLENODES_H
 
-#include "llvm/Demangle/Compiler.h"
+#include "llvm/Demangle/DemangleConfig.h"
 #include "llvm/Demangle/StringView.h"
 #include <array>
 
+namespace llvm {
+namespace itanium_demangle {
 class OutputStream;
+}
+}
+
+using llvm::itanium_demangle::OutputStream;
+using llvm::itanium_demangle::StringView;
 
 namespace llvm {
 namespace ms_demangle {
@@ -53,6 +72,7 @@ enum class ReferenceKind : uint8_t { None, LValueRef, RValueRef };
 enum OutputFlags {
   OF_Default = 0,
   OF_NoCallingConvention = 1,
+  OF_NoTagSpecifier = 2,
 };
 
 // Types
@@ -235,7 +255,7 @@ struct Node {
 
   virtual void output(OutputStream &OS, OutputFlags Flags) const = 0;
 
-  std::string toString() const;
+  std::string toString(OutputFlags Flags = OF_Default) const;
 
 private:
   NodeKind Kind;
@@ -322,6 +342,9 @@ struct FunctionSignatureNode : public TypeNode {
 
   // Function parameters
   NodeArrayNode *Params = nullptr;
+
+  // True if the function type is noexcept
+  bool IsNoexcept = false;
 };
 
 struct IdentifierNode : public Node {
@@ -598,4 +621,4 @@ struct FunctionSymbolNode : public SymbolNode {
 } // namespace ms_demangle
 } // namespace llvm
 
-#endif
\ No newline at end of file
+#endif
diff --git a/include/llvm/Demangle/README.txt b/include/llvm/Demangle/README.txt
new file mode 100644
index 0000000000000000000000000000000000000000..514ff6dd16f2cc3e394808db52874f8cf60cdcdb
--- /dev/null
+++ b/include/llvm/Demangle/README.txt
@@ -0,0 +1,52 @@
+Itanium Name Demangler Library
+==============================
+
+Introduction
+------------
+
+This directory contains the generic itanium name demangler library. The main
+purpose of the library is to demangle C++ symbols, i.e. convert the string
+"_Z1fv" into "f()". You can also use the CRTP base ManglingParser to perform
+some simple analysis on the mangled name, or (in LLVM) use the opaque
+ItaniumPartialDemangler to query the demangled AST.
+
+Why are there multiple copies of the this library in the source tree?
+---------------------------------------------------------------------
+
+This directory is mirrored between libcxxabi/demangle and
+llvm/include/llvm/Demangle. The simple reason for this is that both projects
+need to demangle symbols, but neither can depend on each other. libcxxabi needs
+the demangler to implement __cxa_demangle, which is part of the itanium ABI
+spec. LLVM needs a copy for a bunch of places, but doesn't want to use the
+system's __cxa_demangle because it a) might not be available (i.e., on Windows),
+and b) probably isn't that up-to-date on the latest language features.
+
+The copy of the demangler in LLVM has some extra stuff that aren't needed in
+libcxxabi (ie, the MSVC demangler, ItaniumPartialDemangler), which depend on the
+shared generic components. Despite these differences, we want to keep the "core"
+generic demangling library identical between both copies to simplify development
+and testing.
+
+If you're working on the generic library, then do the work first in libcxxabi,
+then run the cp-to-llvm.sh script in src/demangle. This script takes as an
+argument the path to llvm, and re-copies the changes you made to libcxxabi over.
+Note that this script just blindly overwrites all changes to the generic library
+in llvm, so be careful.
+
+Because the core demangler needs to work in libcxxabi, everything needs to be
+declared in an anonymous namespace (see DEMANGLE_NAMESPACE_BEGIN), and you can't
+introduce any code that depends on the libcxx dylib.
+
+Hopefully, when LLVM becomes a monorepo, we can de-duplicate this code, and have
+both LLVM and libcxxabi depend on a shared demangler library.
+
+Testing
+-------
+
+The tests are split up between libcxxabi/test/{unit,}test_demangle.cpp, and
+llvm/unittest/Demangle. The llvm directory should only get tests for stuff not
+included in the core library. In the future though, we should probably move all
+the tests to LLVM.
+
+It is also a really good idea to run libFuzzer after non-trivial changes, see
+libcxxabi/fuzz/cxa_demangle_fuzzer.cpp and https://llvm.org/docs/LibFuzzer.html.
diff --git a/include/llvm/Demangle/StringView.h b/include/llvm/Demangle/StringView.h
index a89deda694c2095836813fab713384fd92c172ba..ceb6c79580669df1d2e36dedbed4fc96b195f857 100644
--- a/include/llvm/Demangle/StringView.h
+++ b/include/llvm/Demangle/StringView.h
@@ -1,22 +1,25 @@
 //===--- StringView.h -------------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+//===----------------------------------------------------------------------===//
 //
+// FIXME: Use std::string_view instead when we support C++17.
 //
-// This file contains a limited version of LLVM's StringView class.  It is
-// copied here so that LLVMDemangle need not take a dependency on LLVMSupport.
 //===----------------------------------------------------------------------===//
 
-#ifndef LLVM_DEMANGLE_STRINGVIEW_H
-#define LLVM_DEMANGLE_STRINGVIEW_H
+#ifndef DEMANGLE_STRINGVIEW_H
+#define DEMANGLE_STRINGVIEW_H
 
+#include "DemangleConfig.h"
 #include <algorithm>
 #include <cassert>
 #include <cstring>
 
+DEMANGLE_NAMESPACE_BEGIN
+
 class StringView {
   const char *First;
   const char *Last;
@@ -43,7 +46,7 @@ public:
     if (FindBegin < size()) {
       // Just forward to memchr, which is faster than a hand-rolled loop.
       if (const void *P = ::memchr(First + FindBegin, C, size() - FindBegin))
-        return static_cast<const char *>(P) - First;
+        return size_t(static_cast<const char *>(P) - First);
     }
     return npos;
   }
@@ -118,4 +121,6 @@ inline bool operator==(const StringView &LHS, const StringView &RHS) {
          std::equal(LHS.begin(), LHS.end(), RHS.begin());
 }
 
+DEMANGLE_NAMESPACE_END
+
 #endif
diff --git a/include/llvm/Demangle/Utility.h b/include/llvm/Demangle/Utility.h
index 1d1601c81635ab81969182fabab34644f0e8ae8a..ec23859af46afce15f3f367a31eb149ddda23e31 100644
--- a/include/llvm/Demangle/Utility.h
+++ b/include/llvm/Demangle/Utility.h
@@ -1,25 +1,27 @@
 //===--- Utility.h ----------------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+//===----------------------------------------------------------------------===//
 //
+// Provide some utility classes for use in the demangler(s).
 //
-// This file contains several utility classes used by the demangle library.
 //===----------------------------------------------------------------------===//
 
-#ifndef LLVM_DEMANGLE_UTILITY_H
-#define LLVM_DEMANGLE_UTILITY_H
+#ifndef DEMANGLE_UTILITY_H
+#define DEMANGLE_UTILITY_H
 
 #include "StringView.h"
-
 #include <cstdint>
 #include <cstdlib>
 #include <cstring>
 #include <iterator>
 #include <limits>
 
+DEMANGLE_NAMESPACE_BEGIN
+
 // Stream that AST nodes write their string representation into after the AST
 // has been parsed.
 class OutputStream {
@@ -184,4 +186,6 @@ inline bool initializeOutputStream(char *Buf, size_t *N, OutputStream &S,
   return true;
 }
 
+DEMANGLE_NAMESPACE_END
+
 #endif
diff --git a/include/llvm/ExecutionEngine/ExecutionEngine.h b/include/llvm/ExecutionEngine/ExecutionEngine.h
index b61cb24fa5fb5e6afcc90f484b76c82648172da3..68cc920483b63e9dff8a014729d11ce9649ff6e1 100644
--- a/include/llvm/ExecutionEngine/ExecutionEngine.h
+++ b/include/llvm/ExecutionEngine/ExecutionEngine.h
@@ -1,9 +1,8 @@
 //===- ExecutionEngine.h - Abstract Execution Engine Interface --*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/ExecutionEngine/GenericValue.h b/include/llvm/ExecutionEngine/GenericValue.h
index 504e30a018b69fdfac94e938f9c3e38bd94a2e8a..1ca989da1b7ee7a5cb4bc7a84259d92daeb18c6a 100644
--- a/include/llvm/ExecutionEngine/GenericValue.h
+++ b/include/llvm/ExecutionEngine/GenericValue.h
@@ -1,9 +1,8 @@
 //===- GenericValue.h - Represent any type of LLVM value --------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/ExecutionEngine/Interpreter.h b/include/llvm/ExecutionEngine/Interpreter.h
index a14707840ad83af943782376a148a0a7f919633c..0749409766e390db81e220fe1b495e4d5a2a6921 100644
--- a/include/llvm/ExecutionEngine/Interpreter.h
+++ b/include/llvm/ExecutionEngine/Interpreter.h
@@ -1,9 +1,8 @@
 //===-- Interpreter.h - Abstract Execution Engine Interface -----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/ExecutionEngine/JITEventListener.h b/include/llvm/ExecutionEngine/JITEventListener.h
index 1b08379b8c3b119a3aadc6ecc3bf7d70f709c220..606b6f7cc1284116407d2ab531083481aa633168 100644
--- a/include/llvm/ExecutionEngine/JITEventListener.h
+++ b/include/llvm/ExecutionEngine/JITEventListener.h
@@ -1,9 +1,8 @@
 //===- JITEventListener.h - Exposes events from JIT compilation -*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/ExecutionEngine/JITSymbol.h b/include/llvm/ExecutionEngine/JITSymbol.h
index 05c9590726dfc0130c5f1476f59210b98c6290d7..da1352f617402bc209d8420a6850b6c42afa68d4 100644
--- a/include/llvm/ExecutionEngine/JITSymbol.h
+++ b/include/llvm/ExecutionEngine/JITSymbol.h
@@ -1,9 +1,8 @@
 //===- JITSymbol.h - JIT symbol abstraction ---------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/ExecutionEngine/MCJIT.h b/include/llvm/ExecutionEngine/MCJIT.h
index 66ddb7cdb87598fed9c4dac52b990c79b308b450..8253bf98963b8d55ea27cdcc7f120bf68abcaf38 100644
--- a/include/llvm/ExecutionEngine/MCJIT.h
+++ b/include/llvm/ExecutionEngine/MCJIT.h
@@ -1,9 +1,8 @@
 //===-- MCJIT.h - MC-Based Just-In-Time Execution Engine --------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/ExecutionEngine/OProfileWrapper.h b/include/llvm/ExecutionEngine/OProfileWrapper.h
index 05da594a94a8da7c1ddb488ce7c5f563b13104fb..b13d7f6e245b6ffbb7e6119e374d924a7e499764 100644
--- a/include/llvm/ExecutionEngine/OProfileWrapper.h
+++ b/include/llvm/ExecutionEngine/OProfileWrapper.h
@@ -1,9 +1,8 @@
 //===-- OProfileWrapper.h - OProfile JIT API Wrapper ------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 // This file defines a OProfileWrapper object that detects if the oprofile
diff --git a/include/llvm/ExecutionEngine/ObjectCache.h b/include/llvm/ExecutionEngine/ObjectCache.h
index 077044408e095922aee2044fc0410cb802cc5f87..47e94f18a1c7aee1d8e0a3f50c4f0bb961b890a2 100644
--- a/include/llvm/ExecutionEngine/ObjectCache.h
+++ b/include/llvm/ExecutionEngine/ObjectCache.h
@@ -1,9 +1,8 @@
 //===-- ObjectCache.h - Class definition for the ObjectCache ----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h b/include/llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h
index 884878925cde8e81070d517f241bd79639bf380c..ca1ce40cdaef1fd1598b58bde15092ce09cc9543 100644
--- a/include/llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h
+++ b/include/llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h
@@ -1,9 +1,8 @@
 //===- CompileOnDemandLayer.h - Compile each function on demand -*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/ExecutionEngine/Orc/CompileUtils.h b/include/llvm/ExecutionEngine/Orc/CompileUtils.h
index f34f88311ba526b2bb45f94e5a5c6abd4c96fa8b..15a656678135d0cdd05e6064a7dd8c2a13ee8f5f 100644
--- a/include/llvm/ExecutionEngine/Orc/CompileUtils.h
+++ b/include/llvm/ExecutionEngine/Orc/CompileUtils.h
@@ -1,9 +1,8 @@
 //===- CompileUtils.h - Utilities for compiling IR in the JIT ---*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/ExecutionEngine/Orc/Core.h b/include/llvm/ExecutionEngine/Orc/Core.h
index 39d306e0bd4c4cf51f33c520fee4c03b98b12115..299247a98f5210382d13defea6be6ba326dbe061 100644
--- a/include/llvm/ExecutionEngine/Orc/Core.h
+++ b/include/llvm/ExecutionEngine/Orc/Core.h
@@ -1,9 +1,8 @@
 //===------ Core.h -- Core ORC APIs (Layer, JITDylib, etc.) -----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/ExecutionEngine/Orc/ExecutionUtils.h b/include/llvm/ExecutionEngine/Orc/ExecutionUtils.h
index 88559f822e5d06c505d05030055138535b427a55..40dd415d584875a58e42afe98a5e185a08d38ac7 100644
--- a/include/llvm/ExecutionEngine/Orc/ExecutionUtils.h
+++ b/include/llvm/ExecutionEngine/Orc/ExecutionUtils.h
@@ -1,9 +1,8 @@
 //===- ExecutionUtils.h - Utilities for executing code in Orc ---*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/ExecutionEngine/Orc/GlobalMappingLayer.h b/include/llvm/ExecutionEngine/Orc/GlobalMappingLayer.h
index a8a88d7cb2d2c67eb00b94ed777f5aa619c85009..a4e43d4e1c9c2d928896ca83bc0dd98857c387fb 100644
--- a/include/llvm/ExecutionEngine/Orc/GlobalMappingLayer.h
+++ b/include/llvm/ExecutionEngine/Orc/GlobalMappingLayer.h
@@ -1,9 +1,8 @@
 //===- GlobalMappingLayer.h - Run all IR through a functor ------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/ExecutionEngine/Orc/IRCompileLayer.h b/include/llvm/ExecutionEngine/Orc/IRCompileLayer.h
index 30d71e69cd70cb7236e132da76dc5c22de19478a..ecff09b98f87389ef8454dcb6dfeeccc45ed432e 100644
--- a/include/llvm/ExecutionEngine/Orc/IRCompileLayer.h
+++ b/include/llvm/ExecutionEngine/Orc/IRCompileLayer.h
@@ -1,9 +1,8 @@
 //===- IRCompileLayer.h -- Eagerly compile IR for JIT -----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/ExecutionEngine/Orc/IRTransformLayer.h b/include/llvm/ExecutionEngine/Orc/IRTransformLayer.h
index 49e65b9f2a805fe8ae559171149c11ebddcf9a76..8890a572f65a8037920b8950f3bc143d03c5ce13 100644
--- a/include/llvm/ExecutionEngine/Orc/IRTransformLayer.h
+++ b/include/llvm/ExecutionEngine/Orc/IRTransformLayer.h
@@ -1,9 +1,8 @@
 //===- IRTransformLayer.h - Run all IR through a functor --------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/ExecutionEngine/Orc/IndirectionUtils.h b/include/llvm/ExecutionEngine/Orc/IndirectionUtils.h
index c2527802f6a75bf2bdea38a4096b71a5b22bad6b..2ea9ac1c2b2f9941c037dc6b61fb0303f1d42217 100644
--- a/include/llvm/ExecutionEngine/Orc/IndirectionUtils.h
+++ b/include/llvm/ExecutionEngine/Orc/IndirectionUtils.h
@@ -1,9 +1,8 @@
 //===- IndirectionUtils.h - Utilities for adding indirections ---*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/ExecutionEngine/Orc/JITTargetMachineBuilder.h b/include/llvm/ExecutionEngine/Orc/JITTargetMachineBuilder.h
index eb9b6bf2dea6a02caac915558a7c496ce96867d2..bcbd72e68f154b47afe7e7393d9f7a6b429be89a 100644
--- a/include/llvm/ExecutionEngine/Orc/JITTargetMachineBuilder.h
+++ b/include/llvm/ExecutionEngine/Orc/JITTargetMachineBuilder.h
@@ -1,9 +1,8 @@
 //===- JITTargetMachineBuilder.h - Build TargetMachines for JIT -*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/ExecutionEngine/Orc/LLJIT.h b/include/llvm/ExecutionEngine/Orc/LLJIT.h
index ce3e5d519c73cf64b0e56d97650938ad8d0f00e5..b0ef20dec4d576dd5fe62d66ae0b172893715a73 100644
--- a/include/llvm/ExecutionEngine/Orc/LLJIT.h
+++ b/include/llvm/ExecutionEngine/Orc/LLJIT.h
@@ -1,9 +1,8 @@
 //===----- LLJIT.h -- An ORC-based JIT for compiling LLVM IR ----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for 3Bdetails.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/ExecutionEngine/Orc/LambdaResolver.h b/include/llvm/ExecutionEngine/Orc/LambdaResolver.h
index 7b6f3d2f92ab654f69a1ac84c17d3f61aad24c69..92efdeded36254c4b91855fab2957a6b6f9dab23 100644
--- a/include/llvm/ExecutionEngine/Orc/LambdaResolver.h
+++ b/include/llvm/ExecutionEngine/Orc/LambdaResolver.h
@@ -1,9 +1,8 @@
 //===- LambdaResolverMM - Redirect symbol lookup via a functor --*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/ExecutionEngine/Orc/Layer.h b/include/llvm/ExecutionEngine/Orc/Layer.h
index cd797445a2e6b3476de2a15a193d9ef94ac65f91..8f9bd704395e622ba005a4963a00d69d2a5f3d01 100644
--- a/include/llvm/ExecutionEngine/Orc/Layer.h
+++ b/include/llvm/ExecutionEngine/Orc/Layer.h
@@ -1,9 +1,8 @@
 //===---------------- Layer.h -- Layer interfaces --------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/ExecutionEngine/Orc/LazyEmittingLayer.h b/include/llvm/ExecutionEngine/Orc/LazyEmittingLayer.h
index 46761b0ca7e189cee5028c510a93379aad37b859..fdf64d06db1377d3e91cc8e59649faaf7db19845 100644
--- a/include/llvm/ExecutionEngine/Orc/LazyEmittingLayer.h
+++ b/include/llvm/ExecutionEngine/Orc/LazyEmittingLayer.h
@@ -1,9 +1,8 @@
 //===- LazyEmittingLayer.h - Lazily emit IR to lower JIT layers -*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/ExecutionEngine/Orc/LazyReexports.h b/include/llvm/ExecutionEngine/Orc/LazyReexports.h
index b5041325bce2152939951ffe5641b24f70667ed6..9fdd1d15f782c0cb9bb051fd34a824814384a648 100644
--- a/include/llvm/ExecutionEngine/Orc/LazyReexports.h
+++ b/include/llvm/ExecutionEngine/Orc/LazyReexports.h
@@ -1,9 +1,8 @@
 //===------ LazyReexports.h -- Utilities for lazy reexports -----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/ExecutionEngine/Orc/Legacy.h b/include/llvm/ExecutionEngine/Orc/Legacy.h
index 4c6162ac4b8b4bea1d00d0b683c28c380904ebbd..e0e552652769eee47458fe791496a44e62a9db16 100644
--- a/include/llvm/ExecutionEngine/Orc/Legacy.h
+++ b/include/llvm/ExecutionEngine/Orc/Legacy.h
@@ -1,9 +1,8 @@
 //===--- Legacy.h -- Adapters for ExecutionEngine API interop ---*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/ExecutionEngine/Orc/NullResolver.h b/include/llvm/ExecutionEngine/Orc/NullResolver.h
index 03fefb69a9289b90cfe94b9b8ab4f7cf96de8d35..ffa37a13d06497ba21c9e685cdcc71a5b248c964 100644
--- a/include/llvm/ExecutionEngine/Orc/NullResolver.h
+++ b/include/llvm/ExecutionEngine/Orc/NullResolver.h
@@ -1,9 +1,8 @@
 //===------ NullResolver.h - Reject symbol lookup requests ------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/ExecutionEngine/Orc/ObjectTransformLayer.h b/include/llvm/ExecutionEngine/Orc/ObjectTransformLayer.h
index 44d6b490e19da5d790fde83645d57bf0b133b40c..de4603f98f83eced39429009ca2b6b16665481d7 100644
--- a/include/llvm/ExecutionEngine/Orc/ObjectTransformLayer.h
+++ b/include/llvm/ExecutionEngine/Orc/ObjectTransformLayer.h
@@ -1,9 +1,8 @@
 //===- ObjectTransformLayer.h - Run all objects through functor -*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/ExecutionEngine/Orc/OrcABISupport.h b/include/llvm/ExecutionEngine/Orc/OrcABISupport.h
index 49e7b534f21c490878d239c24792b2477b72e810..38246bc480b64fc0563304758b08eff981b30d48 100644
--- a/include/llvm/ExecutionEngine/Orc/OrcABISupport.h
+++ b/include/llvm/ExecutionEngine/Orc/OrcABISupport.h
@@ -1,9 +1,8 @@
 //===- OrcABISupport.h - ABI support code -----------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -287,7 +286,7 @@ class OrcMips64 {
 public:
   static const unsigned PointerSize = 8;
   static const unsigned TrampolineSize = 40;
-  static const unsigned ResolverCodeSize = 0x11C;
+  static const unsigned ResolverCodeSize = 0x120;
 
   using IndirectStubsInfo = GenericIndirectStubsInfo<32>;
   using JITReentryFn = JITTargetAddress (*)(void *CallbackMgr,
diff --git a/include/llvm/ExecutionEngine/Orc/OrcError.h b/include/llvm/ExecutionEngine/Orc/OrcError.h
index dc60e8d74e9755f89dbf9a392c3bd5b6b614c627..e5d6a3eca85f2e4cebd7ea447e48f50cc81c13c7 100644
--- a/include/llvm/ExecutionEngine/Orc/OrcError.h
+++ b/include/llvm/ExecutionEngine/Orc/OrcError.h
@@ -1,9 +1,8 @@
 //===------ OrcError.h - Reject symbol lookup requests ------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/ExecutionEngine/Orc/OrcRemoteTargetClient.h b/include/llvm/ExecutionEngine/Orc/OrcRemoteTargetClient.h
index 3e07f5cf37423fddf34d846746e6845b1184a012..8b875b7906e110e4d73318b8b32745c9ed628e5f 100644
--- a/include/llvm/ExecutionEngine/Orc/OrcRemoteTargetClient.h
+++ b/include/llvm/ExecutionEngine/Orc/OrcRemoteTargetClient.h
@@ -1,9 +1,8 @@
 //===- OrcRemoteTargetClient.h - Orc Remote-target Client -------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/ExecutionEngine/Orc/OrcRemoteTargetRPCAPI.h b/include/llvm/ExecutionEngine/Orc/OrcRemoteTargetRPCAPI.h
index 8db9e317a18aca4f8936ded7429a68cfa38a71c3..e7b598d8f812136d14aacef109f749ecd28a9017 100644
--- a/include/llvm/ExecutionEngine/Orc/OrcRemoteTargetRPCAPI.h
+++ b/include/llvm/ExecutionEngine/Orc/OrcRemoteTargetRPCAPI.h
@@ -1,9 +1,8 @@
 //===- OrcRemoteTargetRPCAPI.h - Orc Remote-target RPC API ------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/ExecutionEngine/Orc/OrcRemoteTargetServer.h b/include/llvm/ExecutionEngine/Orc/OrcRemoteTargetServer.h
index acbc1682fa5d06772b1c8e86e8d25efd563c451e..968448112be991c6587b732a0cab5d18f46a3210 100644
--- a/include/llvm/ExecutionEngine/Orc/OrcRemoteTargetServer.h
+++ b/include/llvm/ExecutionEngine/Orc/OrcRemoteTargetServer.h
@@ -1,9 +1,8 @@
 //===- OrcRemoteTargetServer.h - Orc Remote-target Server -------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/ExecutionEngine/Orc/RPCSerialization.h b/include/llvm/ExecutionEngine/Orc/RPCSerialization.h
index 1e5f6ced597abe03d5c7fcd9012edc6fe17117eb..07c7471afc6a85c9b68397e564040e0c12b3f64d 100644
--- a/include/llvm/ExecutionEngine/Orc/RPCSerialization.h
+++ b/include/llvm/ExecutionEngine/Orc/RPCSerialization.h
@@ -1,9 +1,8 @@
 //===- llvm/ExecutionEngine/Orc/RPCSerialization.h --------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -128,123 +127,85 @@ template <typename T>
 class RPCTypeName<Expected<T>> {
 public:
   static const char* getName() {
-    std::lock_guard<std::mutex> Lock(NameMutex);
-    if (Name.empty())
+    static std::string Name = [] {
+      std::string Name;
       raw_string_ostream(Name) << "Expected<"
                                << RPCTypeNameSequence<T>()
                                << ">";
+      return Name;
+    }();
     return Name.data();
   }
-
-private:
-  static std::mutex NameMutex;
-  static std::string Name;
 };
 
-template <typename T>
-std::mutex RPCTypeName<Expected<T>>::NameMutex;
-
-template <typename T>
-std::string RPCTypeName<Expected<T>>::Name;
-
 template <typename T1, typename T2>
 class RPCTypeName<std::pair<T1, T2>> {
 public:
   static const char* getName() {
-    std::lock_guard<std::mutex> Lock(NameMutex);
-    if (Name.empty())
+    static std::string Name = [] {
+      std::string Name;
       raw_string_ostream(Name) << "std::pair<" << RPCTypeNameSequence<T1, T2>()
                                << ">";
+      return Name;
+    }();
     return Name.data();
   }
-private:
-  static std::mutex NameMutex;
-  static std::string Name;
 };
 
-template <typename T1, typename T2>
-std::mutex RPCTypeName<std::pair<T1, T2>>::NameMutex;
-template <typename T1, typename T2>
-std::string RPCTypeName<std::pair<T1, T2>>::Name;
-
 template <typename... ArgTs>
 class RPCTypeName<std::tuple<ArgTs...>> {
 public:
   static const char* getName() {
-    std::lock_guard<std::mutex> Lock(NameMutex);
-    if (Name.empty())
+    static std::string Name = [] {
+      std::string Name;
       raw_string_ostream(Name) << "std::tuple<"
                                << RPCTypeNameSequence<ArgTs...>() << ">";
+      return Name;
+    }();
     return Name.data();
   }
-private:
-  static std::mutex NameMutex;
-  static std::string Name;
 };
 
-template <typename... ArgTs>
-std::mutex RPCTypeName<std::tuple<ArgTs...>>::NameMutex;
-template <typename... ArgTs>
-std::string RPCTypeName<std::tuple<ArgTs...>>::Name;
-
 template <typename T>
 class RPCTypeName<std::vector<T>> {
 public:
   static const char*getName() {
-    std::lock_guard<std::mutex> Lock(NameMutex);
-    if (Name.empty())
+    static std::string Name = [] {
+      std::string Name;
       raw_string_ostream(Name) << "std::vector<" << RPCTypeName<T>::getName()
                                << ">";
+      return Name;
+    }();
     return Name.data();
   }
-
-private:
-  static std::mutex NameMutex;
-  static std::string Name;
 };
 
-template <typename T>
-std::mutex RPCTypeName<std::vector<T>>::NameMutex;
-template <typename T>
-std::string RPCTypeName<std::vector<T>>::Name;
-
 template <typename T> class RPCTypeName<std::set<T>> {
 public:
   static const char *getName() {
-    std::lock_guard<std::mutex> Lock(NameMutex);
-    if (Name.empty())
+    static std::string Name = [] {
+      std::string Name;
       raw_string_ostream(Name)
           << "std::set<" << RPCTypeName<T>::getName() << ">";
+      return Name;
+    }();
     return Name.data();
   }
-
-private:
-  static std::mutex NameMutex;
-  static std::string Name;
 };
 
-template <typename T> std::mutex RPCTypeName<std::set<T>>::NameMutex;
-template <typename T> std::string RPCTypeName<std::set<T>>::Name;
-
 template <typename K, typename V> class RPCTypeName<std::map<K, V>> {
 public:
   static const char *getName() {
-    std::lock_guard<std::mutex> Lock(NameMutex);
-    if (Name.empty())
+    static std::string Name = [] {
+      std::string Name;
       raw_string_ostream(Name)
           << "std::map<" << RPCTypeNameSequence<K, V>() << ">";
+      return Name;
+    }();
     return Name.data();
   }
-
-private:
-  static std::mutex NameMutex;
-  static std::string Name;
 };
 
-template <typename K, typename V>
-std::mutex RPCTypeName<std::map<K, V>>::NameMutex;
-template <typename K, typename V> std::string RPCTypeName<std::map<K, V>>::Name;
-
 /// The SerializationTraits<ChannelT, T> class describes how to serialize and
 /// deserialize an instance of type T to/from an abstract channel of type
 /// ChannelT. It also provides a representation of the type's name via the
diff --git a/include/llvm/ExecutionEngine/Orc/RPCUtils.h b/include/llvm/ExecutionEngine/Orc/RPCUtils.h
index 953b73e10e43da94e43bb8aa6593554db3edae85..3b11e1b283de7973887b683485c0ba77318820e6 100644
--- a/include/llvm/ExecutionEngine/Orc/RPCUtils.h
+++ b/include/llvm/ExecutionEngine/Orc/RPCUtils.h
@@ -1,9 +1,8 @@
-//===------- RPCUTils.h - Utilities for building RPC APIs -------*- C++ -*-===//
+//===- RPCUtils.h - Utilities for building RPC APIs -------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -152,25 +151,17 @@ public:
 
   /// Returns the full function prototype as a string.
   static const char *getPrototype() {
-    std::lock_guard<std::mutex> Lock(NameMutex);
-    if (Name.empty())
+    static std::string Name = [] {
+      std::string Name;
       raw_string_ostream(Name)
           << RPCTypeName<RetT>::getName() << " " << DerivedFunc::getName()
           << "(" << llvm::orc::rpc::RPCTypeNameSequence<ArgTs...>() << ")";
+      return Name;
+    }();
     return Name.data();
   }
-
-private:
-  static std::mutex NameMutex;
-  static std::string Name;
 };
 
-template <typename DerivedFunc, typename RetT, typename... ArgTs>
-std::mutex Function<DerivedFunc, RetT(ArgTs...)>::NameMutex;
-
-template <typename DerivedFunc, typename RetT, typename... ArgTs>
-std::string Function<DerivedFunc, RetT(ArgTs...)>::Name;
-
 /// Allocates RPC function ids during autonegotiation.
 /// Specializations of this class must provide four members:
 ///
diff --git a/include/llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h b/include/llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h
index 401f6e3fa811612cc9970e26ff20484adc9f2147..cd9ec36da0dea08ae683be8a112026b19979b5ce 100644
--- a/include/llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h
+++ b/include/llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h
@@ -1,9 +1,8 @@
 //===- RTDyldObjectLinkingLayer.h - RTDyld-based jit linking  ---*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -443,11 +442,14 @@ public:
 private:
   ExecutionSession &ES;
 
-  std::map<VModuleKey, std::unique_ptr<LinkedObject>> LinkedObjects;
   ResourcesGetter GetResources;
   NotifyLoadedFtor NotifyLoaded;
   NotifyFinalizedFtor NotifyFinalized;
   NotifyFreedFtor NotifyFreed;
+
+  // NB!  `LinkedObjects` needs to be destroyed before `NotifyFreed` because
+  // `~ConcreteLinkedObject` calls `NotifyFreed`
+  std::map<VModuleKey, std::unique_ptr<LinkedObject>> LinkedObjects;
   bool ProcessAllSections = false;
 };
 
diff --git a/include/llvm/ExecutionEngine/Orc/RawByteChannel.h b/include/llvm/ExecutionEngine/Orc/RawByteChannel.h
index db810f4ef2e53a4d12983a4647b5cab6fc702fb7..46b7c59450e69bb973b6293f9ce799640298bc4f 100644
--- a/include/llvm/ExecutionEngine/Orc/RawByteChannel.h
+++ b/include/llvm/ExecutionEngine/Orc/RawByteChannel.h
@@ -1,9 +1,8 @@
 //===- llvm/ExecutionEngine/Orc/RawByteChannel.h ----------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/ExecutionEngine/Orc/RemoteObjectLayer.h b/include/llvm/ExecutionEngine/Orc/RemoteObjectLayer.h
index 955e77607a18b76bdff50d08262b6e246ceded72..9550edceb1008af85cf400158c7112a1909549a8 100644
--- a/include/llvm/ExecutionEngine/Orc/RemoteObjectLayer.h
+++ b/include/llvm/ExecutionEngine/Orc/RemoteObjectLayer.h
@@ -1,9 +1,8 @@
 //===------ RemoteObjectLayer.h - Forwards objs to a remote -----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/ExecutionEngine/Orc/SymbolStringPool.h b/include/llvm/ExecutionEngine/Orc/SymbolStringPool.h
index 717076e25609018c23362262993dc9fcc80dd2d3..39fa74bf313262f3006b28edca448059048ce7a8 100644
--- a/include/llvm/ExecutionEngine/Orc/SymbolStringPool.h
+++ b/include/llvm/ExecutionEngine/Orc/SymbolStringPool.h
@@ -1,9 +1,8 @@
 //===- SymbolStringPool.h - Multi-threaded pool for JIT symbols -*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/ExecutionEngine/Orc/ThreadSafeModule.h b/include/llvm/ExecutionEngine/Orc/ThreadSafeModule.h
index bf946de532d3e2e5e50baaa84f31e3e247c289e7..5787500387c4bb5dc4b34369fc11f29488423559 100644
--- a/include/llvm/ExecutionEngine/Orc/ThreadSafeModule.h
+++ b/include/llvm/ExecutionEngine/Orc/ThreadSafeModule.h
@@ -1,9 +1,8 @@
 //===----------- ThreadSafeModule.h -- Layer interfaces ---------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/ExecutionEngine/OrcMCJITReplacement.h b/include/llvm/ExecutionEngine/OrcMCJITReplacement.h
index 4cd5648b2fc23167be496c89716d724099df8048..6cca1933f39f614df56bdc3f57413f5dce223f96 100644
--- a/include/llvm/ExecutionEngine/OrcMCJITReplacement.h
+++ b/include/llvm/ExecutionEngine/OrcMCJITReplacement.h
@@ -1,9 +1,8 @@
 //===---- OrcMCJITReplacement.h - Orc-based MCJIT replacement ---*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/ExecutionEngine/RTDyldMemoryManager.h b/include/llvm/ExecutionEngine/RTDyldMemoryManager.h
index 23d651f6d1b63b1239ee350f24c3778bf6b10bd5..c7c87ecdfa09a980aee76a9215a0e5eb954c3498 100644
--- a/include/llvm/ExecutionEngine/RTDyldMemoryManager.h
+++ b/include/llvm/ExecutionEngine/RTDyldMemoryManager.h
@@ -1,9 +1,8 @@
 //===-- RTDyldMemoryManager.cpp - Memory manager for MC-JIT -----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/ExecutionEngine/RuntimeDyld.h b/include/llvm/ExecutionEngine/RuntimeDyld.h
index e419ee05e566364868a0600dffb9e4bb67e189ae..2af9203c26a4a3874fc9d8dd8617682da0879ec2 100644
--- a/include/llvm/ExecutionEngine/RuntimeDyld.h
+++ b/include/llvm/ExecutionEngine/RuntimeDyld.h
@@ -1,9 +1,8 @@
 //===- RuntimeDyld.h - Run-time dynamic linker for MC-JIT -------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/ExecutionEngine/RuntimeDyldChecker.h b/include/llvm/ExecutionEngine/RuntimeDyldChecker.h
index 13fc5fd5a3e74328f4afc72abd9c3fd24d042303..ee925c219065566ec65980ee9b3be26caddae2ce 100644
--- a/include/llvm/ExecutionEngine/RuntimeDyldChecker.h
+++ b/include/llvm/ExecutionEngine/RuntimeDyldChecker.h
@@ -1,9 +1,8 @@
 //===---- RuntimeDyldChecker.h - RuntimeDyld tester framework -----*- C++ -*-=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/ExecutionEngine/SectionMemoryManager.h b/include/llvm/ExecutionEngine/SectionMemoryManager.h
index 3cf131c27778e0bc23e914a85c135010222a405a..d7316425da2f6556fd02b653baaa00664c65f54d 100644
--- a/include/llvm/ExecutionEngine/SectionMemoryManager.h
+++ b/include/llvm/ExecutionEngine/SectionMemoryManager.h
@@ -1,9 +1,8 @@
 //===- SectionMemoryManager.h - Memory manager for MCJIT/RtDyld -*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/FuzzMutate/FuzzerCLI.h b/include/llvm/FuzzMutate/FuzzerCLI.h
index 3333e96db166e6a317f4126b3962accada438a5b..2a16e43a6ab3d077839fbe9c19750b2c28f52613 100644
--- a/include/llvm/FuzzMutate/FuzzerCLI.h
+++ b/include/llvm/FuzzMutate/FuzzerCLI.h
@@ -1,9 +1,8 @@
 //===-- FuzzerCLI.h - Common logic for CLIs of fuzzers ----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/FuzzMutate/IRMutator.h b/include/llvm/FuzzMutate/IRMutator.h
index 9aa9d6d6a4bc2cddf183794d35823e422da54bc6..40a1ce8aeec9cfc80b384de256d4b1aea9e8d13f 100644
--- a/include/llvm/FuzzMutate/IRMutator.h
+++ b/include/llvm/FuzzMutate/IRMutator.h
@@ -1,9 +1,8 @@
 //===-- IRMutator.h - Mutation engine for fuzzing IR ------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/FuzzMutate/OpDescriptor.h b/include/llvm/FuzzMutate/OpDescriptor.h
index dd30fda99bea23ed5d2eeb554ac6ad11cb67b9c4..d6c98cd949a29da7aecf9cf2a5da4a74d70e8077 100644
--- a/include/llvm/FuzzMutate/OpDescriptor.h
+++ b/include/llvm/FuzzMutate/OpDescriptor.h
@@ -1,9 +1,8 @@
 //===-- OpDescriptor.h ------------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/FuzzMutate/Operations.h b/include/llvm/FuzzMutate/Operations.h
index 668bd952ebb202e8977fa803b0a98c496e9b5c9e..2eb4c38c2aeb48015674435ef8d96ee017098dd8 100644
--- a/include/llvm/FuzzMutate/Operations.h
+++ b/include/llvm/FuzzMutate/Operations.h
@@ -1,9 +1,8 @@
 //===-- Operations.h - ----------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/FuzzMutate/Random.h b/include/llvm/FuzzMutate/Random.h
index 3a5f46a07554d07d71359ca74c7756af737f0718..615b15f04ceb7b087106952b5a544cdcb135f68c 100644
--- a/include/llvm/FuzzMutate/Random.h
+++ b/include/llvm/FuzzMutate/Random.h
@@ -1,9 +1,8 @@
 //===--- Random.h - Utilities for random sampling -------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/FuzzMutate/RandomIRBuilder.h b/include/llvm/FuzzMutate/RandomIRBuilder.h
index 5cf3f0b227096faa63d66e5d559de0a50386c15f..f3b609702e9db5fed92c7d6095e7fd281855086b 100644
--- a/include/llvm/FuzzMutate/RandomIRBuilder.h
+++ b/include/llvm/FuzzMutate/RandomIRBuilder.h
@@ -1,9 +1,8 @@
-//===-- Mutator.h - Utils for randomly mutation IR --------------*- C++ -*-===//
+//===- RandomIRBuilder.h - Utils for randomly mutation IR -------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/IR/Argument.h b/include/llvm/IR/Argument.h
index 497dca44547cdac01cd2f3a3a92dd21fa4be0a38..2de5c71120ee9f12d177117e501558457127df91 100644
--- a/include/llvm/IR/Argument.h
+++ b/include/llvm/IR/Argument.h
@@ -1,9 +1,8 @@
 //===-- llvm/Argument.h - Definition of the Argument class ------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/IR/AssemblyAnnotationWriter.h b/include/llvm/IR/AssemblyAnnotationWriter.h
index 6e1f5c43e12ee9e2e966e1cdfa61383322c835c2..3fd3c57a6796395e69015305b2a2948979f50350 100644
--- a/include/llvm/IR/AssemblyAnnotationWriter.h
+++ b/include/llvm/IR/AssemblyAnnotationWriter.h
@@ -1,9 +1,8 @@
 //===-- AssemblyAnnotationWriter.h - Annotation .ll files -------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/IR/Attributes.h b/include/llvm/IR/Attributes.h
index 9fc4614af0105bff3b45d1e9eb9f812d629cbcd7..8d7f4018e846c70b0e49d45746e824b57254e106 100644
--- a/include/llvm/IR/Attributes.h
+++ b/include/llvm/IR/Attributes.h
@@ -1,9 +1,8 @@
 //===- llvm/Attributes.h - Container for Attributes -------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/IR/AutoUpgrade.h b/include/llvm/IR/AutoUpgrade.h
index 8cf574c6a1386575974ba049d41d50d86085ce92..9900319336a74911bacb7d74030db23444e2e768 100644
--- a/include/llvm/IR/AutoUpgrade.h
+++ b/include/llvm/IR/AutoUpgrade.h
@@ -1,9 +1,8 @@
 //===- AutoUpgrade.h - AutoUpgrade Helpers ----------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/IR/BasicBlock.h b/include/llvm/IR/BasicBlock.h
index 99eac33f742ec2a7c8d0c5a7d87bba65bbd09098..fd9efb8417ea9a7fe27bea4473e9e6251fd6c825 100644
--- a/include/llvm/IR/BasicBlock.h
+++ b/include/llvm/IR/BasicBlock.h
@@ -1,9 +1,8 @@
 //===- llvm/BasicBlock.h - Represent a basic block in the VM ----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -363,7 +362,7 @@ public:
   /// This is actually not used to update the Predecessor list, but is actually
   /// used to update the PHI nodes that reside in the block.  Note that this
   /// should be called while the predecessor still refers to this block.
-  void removePredecessor(BasicBlock *Pred, bool DontDeleteUselessPHIs = false);
+  void removePredecessor(BasicBlock *Pred, bool KeepOneInputPHIs = false);
 
   bool canSplitPredecessors() const;
 
diff --git a/include/llvm/IR/CFG.h b/include/llvm/IR/CFG.h
index 8385c4647e12c2bd65b40af07446d624c470ff5e..55aff7137e86f61a3735a842691c6adf432b21c9 100644
--- a/include/llvm/IR/CFG.h
+++ b/include/llvm/IR/CFG.h
@@ -1,9 +1,8 @@
 //===- CFG.h ----------------------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
@@ -238,10 +237,6 @@ public:
   }
 };
 
-template <typename T, typename U> struct isPodLike<SuccIterator<T, U>> {
-  static const bool value = isPodLike<T>::value;
-};
-
 using succ_iterator = SuccIterator<Instruction, BasicBlock>;
 using succ_const_iterator = SuccIterator<const Instruction, const BasicBlock>;
 using succ_range = iterator_range<succ_iterator>;
diff --git a/include/llvm/IR/CFGDiff.h b/include/llvm/IR/CFGDiff.h
index da4373f7bce23c44cbd7903c7e1b250e7225b8fe..57b62dd66a474038821cd4abdacbe4d270cdae69 100644
--- a/include/llvm/IR/CFGDiff.h
+++ b/include/llvm/IR/CFGDiff.h
@@ -1,9 +1,8 @@
 //===- CFGDiff.h - Define a CFG snapshot. -----------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/IR/CallSite.h b/include/llvm/IR/CallSite.h
index a3e78049f4bef2c4971dc72d58ba9f36a6a874ec..1454874e4efea855663403f48b879860257ba7df 100644
--- a/include/llvm/IR/CallSite.h
+++ b/include/llvm/IR/CallSite.h
@@ -1,15 +1,14 @@
 //===- CallSite.h - Abstract Call & Invoke instrs ---------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 // This file defines the CallSite class, which is a handy wrapper for code that
-// wants to treat Call and Invoke instructions in a generic way. When in non-
-// mutation context (e.g. an analysis) ImmutableCallSite should be used.
+// wants to treat Call, Invoke and CallBr instructions in a generic way. When
+// in non-mutation context (e.g. an analysis) ImmutableCallSite should be used.
 // Finally, when some degree of customization is necessary between these two
 // extremes, CallSiteBase<> can be supplied with fine-tuned parameters.
 //
@@ -18,7 +17,7 @@
 // They are efficiently copyable, assignable and constructable, with cost
 // equivalent to copying a pointer (notice that they have only a single data
 // member). The internal representation carries a flag which indicates which of
-// the two variants is enclosed. This allows for cheaper checks when various
+// the three variants is enclosed. This allows for cheaper checks when various
 // accessors of CallSite are employed.
 //
 //===----------------------------------------------------------------------===//
@@ -49,45 +48,50 @@ namespace Intrinsic {
 enum ID : unsigned;
 }
 
-template <typename FunTy = const Function,
-          typename BBTy = const BasicBlock,
-          typename ValTy = const Value,
-          typename UserTy = const User,
-          typename UseTy = const Use,
-          typename InstrTy = const Instruction,
+template <typename FunTy = const Function, typename BBTy = const BasicBlock,
+          typename ValTy = const Value, typename UserTy = const User,
+          typename UseTy = const Use, typename InstrTy = const Instruction,
           typename CallTy = const CallInst,
           typename InvokeTy = const InvokeInst,
+          typename CallBrTy = const CallBrInst,
           typename IterTy = User::const_op_iterator>
 class CallSiteBase {
 protected:
-  PointerIntPair<InstrTy*, 1, bool> I;
+  PointerIntPair<InstrTy *, 2, int> I;
 
   CallSiteBase() = default;
-  CallSiteBase(CallTy *CI) : I(CI, true) { assert(CI); }
-  CallSiteBase(InvokeTy *II) : I(II, false) { assert(II); }
+  CallSiteBase(CallTy *CI) : I(CI, 1) { assert(CI); }
+  CallSiteBase(InvokeTy *II) : I(II, 0) { assert(II); }
+  CallSiteBase(CallBrTy *CBI) : I(CBI, 2) { assert(CBI); }
   explicit CallSiteBase(ValTy *II) { *this = get(II); }
 
 private:
   /// This static method is like a constructor. It will create an appropriate
-  /// call site for a Call or Invoke instruction, but it can also create a null
-  /// initialized CallSiteBase object for something which is NOT a call site.
+  /// call site for a Call, Invoke or CallBr instruction, but it can also create
+  /// a null initialized CallSiteBase object for something which is NOT a call
+  /// site.
   static CallSiteBase get(ValTy *V) {
     if (InstrTy *II = dyn_cast<InstrTy>(V)) {
       if (II->getOpcode() == Instruction::Call)
         return CallSiteBase(static_cast<CallTy*>(II));
-      else if (II->getOpcode() == Instruction::Invoke)
+      if (II->getOpcode() == Instruction::Invoke)
         return CallSiteBase(static_cast<InvokeTy*>(II));
+      if (II->getOpcode() == Instruction::CallBr)
+        return CallSiteBase(static_cast<CallBrTy *>(II));
     }
     return CallSiteBase();
   }
 
 public:
-  /// Return true if a CallInst is enclosed. Note that !isCall() does not mean
-  /// an InvokeInst is enclosed. It may also signify a NULL instruction pointer.
-  bool isCall() const { return I.getInt(); }
+  /// Return true if a CallInst is enclosed.
+  bool isCall() const { return I.getInt() == 1; }
+
+  /// Return true if a InvokeInst is enclosed. !I.getInt() may also signify a
+  /// NULL instruction pointer, so check that.
+  bool isInvoke() const { return getInstruction() && I.getInt() == 0; }
 
-  /// Return true if a InvokeInst is enclosed.
-  bool isInvoke() const { return getInstruction() && !I.getInt(); }
+  /// Return true if a CallBrInst is enclosed.
+  bool isCallBr() const { return I.getInt() == 2; }
 
   InstrTy *getInstruction() const { return I.getPointer(); }
   InstrTy *operator->() const { return I.getPointer(); }
@@ -98,7 +102,7 @@ public:
 
   /// Return the pointer to function that is being called.
   ValTy *getCalledValue() const {
-    assert(getInstruction() && "Not a call or invoke instruction!");
+    assert(getInstruction() && "Not a call, invoke or callbr instruction!");
     return *getCallee();
   }
 
@@ -115,16 +119,19 @@ public:
       return false;
     if (isa<FunTy>(V) || isa<Constant>(V))
       return false;
-    if (const CallInst *CI = dyn_cast<CallInst>(getInstruction())) {
-      if (CI->isInlineAsm())
+    if (const CallBase *CB = dyn_cast<CallBase>(getInstruction()))
+      if (CB->isInlineAsm())
         return false;
-    }
     return true;
   }
 
-  /// Set the callee to the specified value.
+  /// Set the callee to the specified value.  Unlike the function of the same
+  /// name on CallBase, does not modify the type!
   void setCalledFunction(Value *V) {
-    assert(getInstruction() && "Not a call or invoke instruction!");
+    assert(getInstruction() && "Not a call, callbr, or invoke instruction!");
+    assert(cast<PointerType>(V->getType())->getElementType() ==
+               cast<CallBase>(getInstruction())->getFunctionType() &&
+           "New callee type does not match FunctionType on call");
     *getCallee() = V;
   }
 
@@ -189,7 +196,7 @@ public:
   }
 
   void setArgument(unsigned ArgNo, Value* newVal) {
-    assert(getInstruction() && "Not a call or invoke instruction!");
+    assert(getInstruction() && "Not a call, invoke or callbr instruction!");
     assert(arg_begin() + ArgNo < arg_end() && "Argument # out of range!");
     getInstruction()->setOperand(ArgNo, newVal);
   }
@@ -203,7 +210,7 @@ public:
   /// Given a use for an argument, get the argument number that corresponds to
   /// it.
   unsigned getArgumentNo(const Use *U) const {
-    assert(getInstruction() && "Not a call or invoke instruction!");
+    assert(getInstruction() && "Not a call, invoke or callbr instruction!");
     assert(isArgOperand(U) && "Argument # out of range!");
     return U - arg_begin();
   }
@@ -227,7 +234,7 @@ public:
   /// Given a use for a data operand, get the data operand number that
   /// corresponds to it.
   unsigned getDataOperandNo(const Use *U) const {
-    assert(getInstruction() && "Not a call or invoke instruction!");
+    assert(getInstruction() && "Not a call, invoke or callbr instruction!");
     assert(isDataOperand(U) && "Data operand # out of range!");
     return U - data_operands_begin();
   }
@@ -237,18 +244,19 @@ public:
   using data_operand_iterator = IterTy;
 
   /// data_operands_begin/data_operands_end - Return iterators iterating over
-  /// the call / invoke argument list and bundle operands.  For invokes, this is
-  /// the set of instruction operands except the invoke target and the two
-  /// successor blocks; and for calls this is the set of instruction operands
-  /// except the call target.
+  /// the call / invoke / callbr argument list and bundle operands. For invokes,
+  /// this is the set of instruction operands except the invoke target and the
+  /// two successor blocks; for calls this is the set of instruction operands
+  /// except the call target; for callbrs the number of labels to skip must be
+  /// determined first.
 
   IterTy data_operands_begin() const {
     assert(getInstruction() && "Not a call or invoke instruction!");
-    return (*this)->op_begin();
+    return cast<CallBase>(getInstruction())->data_operands_begin();
   }
   IterTy data_operands_end() const {
     assert(getInstruction() && "Not a call or invoke instruction!");
-    return (*this)->op_end() - (isCall() ? 1 : 3);
+    return cast<CallBase>(getInstruction())->data_operands_end();
   }
   iterator_range<IterTy> data_ops() const {
     return make_range(data_operands_begin(), data_operands_end());
@@ -277,17 +285,19 @@ public:
     return isCall() && cast<CallInst>(getInstruction())->isTailCall();
   }
 
-#define CALLSITE_DELEGATE_GETTER(METHOD) \
-  InstrTy *II = getInstruction();    \
-  return isCall()                        \
-    ? cast<CallInst>(II)->METHOD         \
-    : cast<InvokeInst>(II)->METHOD
+#define CALLSITE_DELEGATE_GETTER(METHOD)                                       \
+  InstrTy *II = getInstruction();                                              \
+  return isCall() ? cast<CallInst>(II)->METHOD                                 \
+                  : isCallBr() ? cast<CallBrInst>(II)->METHOD                  \
+                                : cast<InvokeInst>(II)->METHOD
 
-#define CALLSITE_DELEGATE_SETTER(METHOD) \
-  InstrTy *II = getInstruction();    \
-  if (isCall())                          \
-    cast<CallInst>(II)->METHOD;          \
-  else                                   \
+#define CALLSITE_DELEGATE_SETTER(METHOD)                                       \
+  InstrTy *II = getInstruction();                                              \
+  if (isCall())                                                                \
+    cast<CallInst>(II)->METHOD;                                                \
+  else if (isCallBr())                                                         \
+    cast<CallBrInst>(II)->METHOD;                                              \
+  else                                                                         \
     cast<InvokeInst>(II)->METHOD
 
   unsigned getNumArgOperands() const {
@@ -303,9 +313,7 @@ public:
   }
 
   bool isInlineAsm() const {
-    if (isCall())
-      return cast<CallInst>(getInstruction())->isInlineAsm();
-    return false;
+    return cast<CallBase>(getInstruction())->isInlineAsm();
   }
 
   /// Get the calling convention of the call.
@@ -389,10 +397,10 @@ public:
   /// Return true if the data operand at index \p i directly or indirectly has
   /// the attribute \p A.
   ///
-  /// Normal call or invoke arguments have per operand attributes, as specified
-  /// in the attribute set attached to this instruction, while operand bundle
-  /// operands may have some attributes implied by the type of its containing
-  /// operand bundle.
+  /// Normal call, invoke or callbr arguments have per operand attributes, as
+  /// specified in the attribute set attached to this instruction, while operand
+  /// bundle operands may have some attributes implied by the type of its
+  /// containing operand bundle.
   bool dataOperandHasImpliedAttr(unsigned i, Attribute::AttrKind Kind) const {
     CALLSITE_DELEGATE_GETTER(dataOperandHasImpliedAttr(i, Kind));
   }
@@ -580,13 +588,9 @@ public:
 #undef CALLSITE_DELEGATE_SETTER
 
   void getOperandBundlesAsDefs(SmallVectorImpl<OperandBundleDef> &Defs) const {
-    const Instruction *II = getInstruction();
     // Since this is actually a getter that "looks like" a setter, don't use the
     // above macros to avoid confusion.
-    if (isCall())
-      cast<CallInst>(II)->getOperandBundlesAsDefs(Defs);
-    else
-      cast<InvokeInst>(II)->getOperandBundlesAsDefs(Defs);
+    cast<CallBase>(getInstruction())->getOperandBundlesAsDefs(Defs);
   }
 
   /// Determine whether this data operand is not captured.
@@ -662,12 +666,13 @@ private:
 
 class CallSite : public CallSiteBase<Function, BasicBlock, Value, User, Use,
                                      Instruction, CallInst, InvokeInst,
-                                     User::op_iterator> {
+                                     CallBrInst, User::op_iterator> {
 public:
   CallSite() = default;
   CallSite(CallSiteBase B) : CallSiteBase(B) {}
   CallSite(CallInst *CI) : CallSiteBase(CI) {}
   CallSite(InvokeInst *II) : CallSiteBase(II) {}
+  CallSite(CallBrInst *CBI) : CallSiteBase(CBI) {}
   explicit CallSite(Instruction *II) : CallSiteBase(II) {}
   explicit CallSite(Value *V) : CallSiteBase(V) {}
 
@@ -683,6 +688,182 @@ private:
   User::op_iterator getCallee() const;
 };
 
+/// AbstractCallSite
+///
+/// An abstract call site is a wrapper that allows to treat direct,
+/// indirect, and callback calls the same. If an abstract call site
+/// represents a direct or indirect call site it behaves like a stripped
+/// down version of a normal call site object. The abstract call site can
+/// also represent a callback call, thus the fact that the initially
+/// called function (=broker) may invoke a third one (=callback callee).
+/// In this case, the abstract call site hides the middle man, hence the
+/// broker function. The result is a representation of the callback call,
+/// inside the broker, but in the context of the original call to the broker.
+///
+/// There are up to three functions involved when we talk about callback call
+/// sites. The caller (1), which invokes the broker function. The broker
+/// function (2), that will invoke the callee zero or more times. And finally
+/// the callee (3), which is the target of the callback call.
+///
+/// The abstract call site will handle the mapping from parameters to arguments
+/// depending on the semantic of the broker function. However, it is important
+/// to note that the mapping is often partial. Thus, some arguments of the
+/// call/invoke instruction are mapped to parameters of the callee while others
+/// are not.
+class AbstractCallSite {
+public:
+
+  /// The encoding of a callback with regards to the underlying instruction.
+  struct CallbackInfo {
+
+    /// For direct/indirect calls the parameter encoding is empty. If it is not,
+    /// the abstract call site represents a callback. In that case, the first
+    /// element of the encoding vector represents which argument of the call
+    /// site CS is the callback callee. The remaining elements map parameters
+    /// (identified by their position) to the arguments that will be passed
+    /// through (also identified by position but in the call site instruction).
+    ///
+    /// NOTE that we use LLVM argument numbers (starting at 0) and not
+    /// clang/soruce argument numbers (starting at 1). The -1 entries represent
+    /// unknown values that are passed to the callee.
+    using ParameterEncodingTy = SmallVector<int, 0>;
+    ParameterEncodingTy ParameterEncoding;
+
+  };
+
+private:
+
+  /// The underlying call site:
+  ///   caller -> callee,             if this is a direct or indirect call site
+  ///   caller -> broker function,    if this is a callback call site
+  CallSite CS;
+
+  /// The encoding of a callback with regards to the underlying instruction.
+  CallbackInfo CI;
+
+public:
+  /// Sole constructor for abstract call sites (ACS).
+  ///
+  /// An abstract call site can only be constructed through a llvm::Use because
+  /// each operand (=use) of an instruction could potentially be a different
+  /// abstract call site. Furthermore, even if the value of the llvm::Use is the
+  /// same, and the user is as well, the abstract call sites might not be.
+  ///
+  /// If a use is not associated with an abstract call site the constructed ACS
+  /// will evaluate to false if converted to a boolean.
+  ///
+  /// If the use is the callee use of a call or invoke instruction, the
+  /// constructed abstract call site will behave as a llvm::CallSite would.
+  ///
+  /// If the use is not a callee use of a call or invoke instruction, the
+  /// callback metadata is used to determine the argument <-> parameter mapping
+  /// as well as the callee of the abstract call site.
+  AbstractCallSite(const Use *U);
+
+  /// Conversion operator to conveniently check for a valid/initialized ACS.
+  explicit operator bool() const { return (bool)CS; }
+
+  /// Return the underlying instruction.
+  Instruction *getInstruction() const { return CS.getInstruction(); }
+
+  /// Return the call site abstraction for the underlying instruction.
+  CallSite getCallSite() const { return CS; }
+
+  /// Return true if this ACS represents a direct call.
+  bool isDirectCall() const {
+    return !isCallbackCall() && !CS.isIndirectCall();
+  }
+
+  /// Return true if this ACS represents an indirect call.
+  bool isIndirectCall() const {
+    return !isCallbackCall() && CS.isIndirectCall();
+  }
+
+  /// Return true if this ACS represents a callback call.
+  bool isCallbackCall() const {
+    // For a callback call site the callee is ALWAYS stored first in the
+    // transitive values vector. Thus, a non-empty vector indicates a callback.
+    return !CI.ParameterEncoding.empty();
+  }
+
+  /// Return true if @p UI is the use that defines the callee of this ACS.
+  bool isCallee(Value::const_user_iterator UI) const {
+    return isCallee(&UI.getUse());
+  }
+
+  /// Return true if @p U is the use that defines the callee of this ACS.
+  bool isCallee(const Use *U) const {
+    if (isDirectCall())
+      return CS.isCallee(U);
+
+    assert(!CI.ParameterEncoding.empty() &&
+           "Callback without parameter encoding!");
+
+    return (int)CS.getArgumentNo(U) == CI.ParameterEncoding[0];
+  }
+
+  /// Return the number of parameters of the callee.
+  unsigned getNumArgOperands() const {
+    if (isDirectCall())
+      return CS.getNumArgOperands();
+    // Subtract 1 for the callee encoding.
+    return CI.ParameterEncoding.size() - 1;
+  }
+
+  /// Return the operand index of the underlying instruction associated with @p
+  /// Arg.
+  int getCallArgOperandNo(Argument &Arg) const {
+    return getCallArgOperandNo(Arg.getArgNo());
+  }
+
+  /// Return the operand index of the underlying instruction associated with
+  /// the function parameter number @p ArgNo or -1 if there is none.
+  int getCallArgOperandNo(unsigned ArgNo) const {
+    if (isDirectCall())
+      return ArgNo;
+    // Add 1 for the callee encoding.
+    return CI.ParameterEncoding[ArgNo + 1];
+  }
+
+  /// Return the operand of the underlying instruction associated with @p Arg.
+  Value *getCallArgOperand(Argument &Arg) const {
+    return getCallArgOperand(Arg.getArgNo());
+  }
+
+  /// Return the operand of the underlying instruction associated with the
+  /// function parameter number @p ArgNo or nullptr if there is none.
+  Value *getCallArgOperand(unsigned ArgNo) const {
+    if (isDirectCall())
+      return CS.getArgOperand(ArgNo);
+    // Add 1 for the callee encoding.
+    return CI.ParameterEncoding[ArgNo + 1] >= 0
+               ? CS.getArgOperand(CI.ParameterEncoding[ArgNo + 1])
+               : nullptr;
+  }
+
+  /// Return the operand index of the underlying instruction associated with the
+  /// callee of this ACS. Only valid for callback calls!
+  int getCallArgOperandNoForCallee() const {
+    assert(isCallbackCall());
+    assert(CI.ParameterEncoding.size() && CI.ParameterEncoding[0] > 0);
+    return CI.ParameterEncoding[0];
+  }
+
+  /// Return the pointer to function that is being called.
+  Value *getCalledValue() const {
+    if (isDirectCall())
+      return CS.getCalledValue();
+    return CS.getArgOperand(getCallArgOperandNoForCallee());
+  }
+
+  /// Return the function being called if this is a direct call, otherwise
+  /// return null (if it's an indirect call).
+  Function *getCalledFunction() const {
+    Value *V = getCalledValue();
+    return V ? dyn_cast<Function>(V->stripPointerCasts()) : nullptr;
+  }
+};
+
 template <> struct DenseMapInfo<CallSite> {
   using BaseInfo = DenseMapInfo<decltype(CallSite::I)>;
 
@@ -713,6 +894,7 @@ public:
   ImmutableCallSite() = default;
   ImmutableCallSite(const CallInst *CI) : CallSiteBase(CI) {}
   ImmutableCallSite(const InvokeInst *II) : CallSiteBase(II) {}
+  ImmutableCallSite(const CallBrInst *CBI) : CallSiteBase(CBI) {}
   explicit ImmutableCallSite(const Instruction *II) : CallSiteBase(II) {}
   explicit ImmutableCallSite(const Value *V) : CallSiteBase(V) {}
   ImmutableCallSite(CallSite CS) : CallSiteBase(CS.getInstruction()) {}
diff --git a/include/llvm/IR/CallingConv.h b/include/llvm/IR/CallingConv.h
index 49c3be960373326106e9442405c979398e7b30f3..399c6ad521fafa37af9007dd62a69b7bbe15273d 100644
--- a/include/llvm/IR/CallingConv.h
+++ b/include/llvm/IR/CallingConv.h
@@ -1,9 +1,8 @@
 //===- llvm/CallingConv.h - LLVM Calling Conventions ------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/IR/Comdat.h b/include/llvm/IR/Comdat.h
index 555121e928f79e1c56610d3a7ca88de8e0763d39..f712a16dd318ec029e1ebf8f8bb8bfa85dce1537 100644
--- a/include/llvm/IR/Comdat.h
+++ b/include/llvm/IR/Comdat.h
@@ -1,9 +1,8 @@
 //===- llvm/IR/Comdat.h - Comdat definitions --------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/IR/Constant.h b/include/llvm/IR/Constant.h
index 98437f8eff1fa435109857fa5853942b3fe72135..e71ac273c08005d2c9680d8cfc210680ba8e4629 100644
--- a/include/llvm/IR/Constant.h
+++ b/include/llvm/IR/Constant.h
@@ -1,9 +1,8 @@
 //===-- llvm/Constant.h - Constant class definition -------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/IR/ConstantFolder.h b/include/llvm/IR/ConstantFolder.h
index da5bba7ba14122e99c4a67e77b7429238fdd47a1..1971cb854e340925f046651235b6c5bd6978f80d 100644
--- a/include/llvm/IR/ConstantFolder.h
+++ b/include/llvm/IR/ConstantFolder.h
@@ -1,9 +1,8 @@
 //===- ConstantFolder.h - Constant folding helper ---------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/IR/ConstantRange.h b/include/llvm/IR/ConstantRange.h
index 1adda3269abca76b711dfbad212ae8d2183b3b67..86c8c30f5e4039a5307ca9ad6ead09371ea8a576 100644
--- a/include/llvm/IR/ConstantRange.h
+++ b/include/llvm/IR/ConstantRange.h
@@ -1,9 +1,8 @@
 //===- ConstantRange.h - Represent a range ----------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/IR/Constants.h b/include/llvm/IR/Constants.h
index afc93cd61d474da770cb5948179bf45a6944f114..ca56e8b9328ca5d6ee3b0a083a9054c67bcb961e 100644
--- a/include/llvm/IR/Constants.h
+++ b/include/llvm/IR/Constants.h
@@ -1,9 +1,8 @@
 //===-- llvm/Constants.h - Constant class subclass definitions --*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/IR/DIBuilder.h b/include/llvm/IR/DIBuilder.h
index 1fe7b36db00b5ef19d37232f20540ec0e8c44a41..b575632af7da3df1975159258e3e6468ef8ed90f 100644
--- a/include/llvm/IR/DIBuilder.h
+++ b/include/llvm/IR/DIBuilder.h
@@ -1,9 +1,8 @@
 //===- DIBuilder.h - Debug Information Builder ------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -502,11 +501,11 @@ namespace llvm {
     /// \param Elements       Enumeration elements.
     /// \param UnderlyingType Underlying type of a C++11/ObjC fixed enum.
     /// \param UniqueIdentifier A unique identifier for the enum.
-    /// \param IsFixed Boolean flag indicate if this is C++11/ObjC fixed enum.
+    /// \param IsScoped Boolean flag indicate if this is C++11/ObjC 'enum class'.
     DICompositeType *createEnumerationType(
         DIScope *Scope, StringRef Name, DIFile *File, unsigned LineNumber,
         uint64_t SizeInBits, uint32_t AlignInBits, DINodeArray Elements,
-        DIType *UnderlyingType, StringRef UniqueIdentifier = "", bool IsFixed = false);
+        DIType *UnderlyingType, StringRef UniqueIdentifier = "", bool IsScoped = false);
 
     /// Create subroutine type.
     /// \param ParameterTypes  An array of subroutine parameter types. This
diff --git a/include/llvm/IR/DataLayout.h b/include/llvm/IR/DataLayout.h
index d796a65e61293631fa0d468f76ef823a0d4096bc..274ec2351d8be700965d3ea398608e5ba1777b9b 100644
--- a/include/llvm/IR/DataLayout.h
+++ b/include/llvm/IR/DataLayout.h
@@ -1,9 +1,8 @@
 //===- llvm/DataLayout.h - Data size & alignment info -----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -334,6 +333,9 @@ public:
   /// the backends/clients are updated.
   unsigned getPointerSize(unsigned AS = 0) const;
 
+  /// Returns the maximum pointer size over all address spaces.
+  unsigned getMaxPointerSize() const;
+
   // Index size used for address calculation.
   unsigned getIndexSize(unsigned AS) const;
 
@@ -361,6 +363,11 @@ public:
     return getPointerSize(AS) * 8;
   }
 
+  /// Returns the maximum pointer size over all address spaces.
+  unsigned getMaxPointerSizeInBits() const {
+    return getMaxPointerSize() * 8;
+  }
+
   /// Size in bits of index used for address calculation in getelementptr.
   unsigned getIndexSizeInBits(unsigned AS) const {
     return getIndexSize(AS) * 8;
diff --git a/include/llvm/IR/DebugInfo.h b/include/llvm/IR/DebugInfo.h
index 01178af3c9ff58f0861772ef8f15c2abfa7e292b..171e1621889fbbe001a7e145e2dff718b76b4f51 100644
--- a/include/llvm/IR/DebugInfo.h
+++ b/include/llvm/IR/DebugInfo.h
@@ -1,9 +1,8 @@
 //===- DebugInfo.h - Debug Information Helpers ------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/IR/DebugInfoFlags.def b/include/llvm/IR/DebugInfoFlags.def
index a0a9c6bcacb8639a46fd0867f0bec11272236d07..f19c04dd9759c84d96a40f27228b928df87dea21 100644
--- a/include/llvm/IR/DebugInfoFlags.def
+++ b/include/llvm/IR/DebugInfoFlags.def
@@ -1,9 +1,8 @@
 //===- llvm/IR/DebugInfoFlags.def - Debug info flag definitions -*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -54,7 +53,7 @@ HANDLE_DI_FLAG((1 << 20), NoReturn)
 HANDLE_DI_FLAG((1 << 21), MainSubprogram)
 HANDLE_DI_FLAG((1 << 22), TypePassByValue)
 HANDLE_DI_FLAG((1 << 23), TypePassByReference)
-HANDLE_DI_FLAG((1 << 24), FixedEnum)
+HANDLE_DI_FLAG((1 << 24), EnumClass)
 HANDLE_DI_FLAG((1 << 25), Thunk)
 HANDLE_DI_FLAG((1 << 26), Trivial)
 HANDLE_DI_FLAG((1 << 27), BigEndian)
diff --git a/include/llvm/IR/DebugInfoMetadata.h b/include/llvm/IR/DebugInfoMetadata.h
index 0830b5ccbeca6327f41da041d717c0ffcda9c53f..385f6e7a65f3666afb4a6dda89ec91b28a82f50a 100644
--- a/include/llvm/IR/DebugInfoMetadata.h
+++ b/include/llvm/IR/DebugInfoMetadata.h
@@ -1,9 +1,8 @@
 //===- llvm/IR/DebugInfoMetadata.h - Debug info metadata --------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -1430,6 +1429,9 @@ class DILocation : public MDNode {
 
   /// Reverse transformation as getPrefixEncodingFromUnsigned.
   static unsigned getUnsignedFromPrefixEncoding(unsigned U) {
+    if (U & 1)
+      return 0;
+    U >>= 1;
     return (U & 0x20) ? (((U >> 1) & 0xfe0) | (U & 0x1f)) : (U & 0x1f);
   }
 
@@ -1448,6 +1450,14 @@ class DILocation : public MDNode {
                         getRawInlinedAt(), isImplicitCode());
   }
 
+  static unsigned encodeComponent(unsigned C) {
+    return (C == 0) ? 1U : (getPrefixEncodingFromUnsigned(C) << 1);
+  }
+
+  static unsigned encodingBits(unsigned C) {
+    return (C == 0) ? 1 : (C > 0x1f ? 14 : 7);
+  }
+
 public:
   // Disallow replacing operands.
   void replaceOperandWith(unsigned I, Metadata *New) = delete;
@@ -1518,20 +1528,32 @@ public:
   /// order. If the lowest bit is 1, the current component is empty, and the
   /// next component will start in the next bit. Otherwise, the current
   /// component is non-empty, and its content starts in the next bit. The
-  /// length of each components is either 5 bit or 12 bit: if the 7th bit
+  /// value of each components is either 5 bit or 12 bit: if the 7th bit
   /// is 0, the bit 2~6 (5 bits) are used to represent the component; if the
   /// 7th bit is 1, the bit 2~6 (5 bits) and 8~14 (7 bits) are combined to
-  /// represent the component.
+  /// represent the component. Thus, the number of bits used for a component
+  /// is either 0 (if it and all the next components are empty); 1 - if it is
+  /// empty; 7 - if its value is up to and including 0x1f (lsb and msb are both
+  /// 0); or 14, if its value is up to and including 0x1ff. Note that the last
+  /// component is also capped at 0x1ff, even in the case when both first
+  /// components are 0, and we'd technically have 29 bits available.
+  ///
+  /// For precise control over the data being encoded in the discriminator,
+  /// use encodeDiscriminator/decodeDiscriminator.
 
   inline unsigned getDiscriminator() const;
 
   /// Returns a new DILocation with updated \p Discriminator.
   inline const DILocation *cloneWithDiscriminator(unsigned Discriminator) const;
 
-  /// Returns a new DILocation with updated base discriminator \p BD.
-  inline const DILocation *setBaseDiscriminator(unsigned BD) const;
+  /// Returns a new DILocation with updated base discriminator \p BD. Only the
+  /// base discriminator is set in the new DILocation, the other encoded values
+  /// are elided.
+  /// If the discriminator cannot be encoded, the function returns None.
+  inline Optional<const DILocation *> cloneWithBaseDiscriminator(unsigned BD) const;
 
-  /// Returns the duplication factor stored in the discriminator.
+  /// Returns the duplication factor stored in the discriminator, or 1 if no
+  /// duplication factor (or 0) is encoded.
   inline unsigned getDuplicationFactor() const;
 
   /// Returns the copy identifier stored in the discriminator.
@@ -1540,9 +1562,11 @@ public:
   /// Returns the base discriminator stored in the discriminator.
   inline unsigned getBaseDiscriminator() const;
 
-  /// Returns a new DILocation with duplication factor \p DF encoded in the
-  /// discriminator.
-  inline const DILocation *cloneWithDuplicationFactor(unsigned DF) const;
+  /// Returns a new DILocation with duplication factor \p DF * current
+  /// duplication factor encoded in the discriminator. The current duplication
+  /// factor is as defined by getDuplicationFactor().
+  /// Returns None if encoding failed.
+  inline Optional<const DILocation *> cloneByMultiplyingDuplicationFactor(unsigned DF) const;
 
   /// When two instructions are combined into a single instruction we also
   /// need to combine the original locations into a single location.
@@ -1563,19 +1587,32 @@ public:
 
   /// Returns the base discriminator for a given encoded discriminator \p D.
   static unsigned getBaseDiscriminatorFromDiscriminator(unsigned D) {
-    if ((D & 1) == 0)
-      return getUnsignedFromPrefixEncoding(D >> 1);
-    else
-      return 0;
-  }
-
-  /// Returns the duplication factor for a given encoded discriminator \p D.
+    return getUnsignedFromPrefixEncoding(D);
+  }
+
+  /// Raw encoding of the discriminator. APIs such as cloneWithDuplicationFactor
+  /// have certain special case behavior (e.g. treating empty duplication factor
+  /// as the value '1').
+  /// This API, in conjunction with cloneWithDiscriminator, may be used to encode
+  /// the raw values provided. \p BD: base discriminator \p DF: duplication factor
+  /// \p CI: copy index
+  /// The return is None if the values cannot be encoded in 32 bits - for
+  /// example, values for BD or DF larger than 12 bits. Otherwise, the return
+  /// is the encoded value.
+  static Optional<unsigned> encodeDiscriminator(unsigned BD, unsigned DF, unsigned CI);
+
+  /// Raw decoder for values in an encoded discriminator D.
+  static void decodeDiscriminator(unsigned D, unsigned &BD, unsigned &DF,
+                                  unsigned &CI);
+
+  /// Returns the duplication factor for a given encoded discriminator \p D, or
+  /// 1 if no value or 0 is encoded.
   static unsigned getDuplicationFactorFromDiscriminator(unsigned D) {
     D = getNextComponentInDiscriminator(D);
-    if (D == 0 || (D & 1))
+    unsigned Ret = getUnsignedFromPrefixEncoding(D);
+    if (Ret == 0)
       return 1;
-    else
-      return getUnsignedFromPrefixEncoding(D >> 1);
+    return Ret;
   }
 
   /// Returns the copy identifier for a given encoded discriminator \p D.
@@ -1999,28 +2036,26 @@ unsigned DILocation::getCopyIdentifier() const {
   return getCopyIdentifierFromDiscriminator(getDiscriminator());
 }
 
-const DILocation *DILocation::setBaseDiscriminator(unsigned D) const {
-  if (D == 0)
+Optional<const DILocation *> DILocation::cloneWithBaseDiscriminator(unsigned D) const {
+  unsigned BD, DF, CI;
+  decodeDiscriminator(getDiscriminator(), BD, DF, CI);
+  if (D == BD)
     return this;
-  else
-    return cloneWithDiscriminator(getPrefixEncodingFromUnsigned(D) << 1);
+  if (Optional<unsigned> Encoded = encodeDiscriminator(D, DF, CI))
+    return cloneWithDiscriminator(*Encoded);
+  return None;
 }
 
-const DILocation *DILocation::cloneWithDuplicationFactor(unsigned DF) const {
+Optional<const DILocation *> DILocation::cloneByMultiplyingDuplicationFactor(unsigned DF) const {
   DF *= getDuplicationFactor();
   if (DF <= 1)
     return this;
 
   unsigned BD = getBaseDiscriminator();
-  unsigned CI = getCopyIdentifier() << (DF > 0x1f ? 14 : 7);
-  unsigned D = CI | (getPrefixEncodingFromUnsigned(DF) << 1);
-
-  if (BD == 0)
-    D = (D << 1) | 1;
-  else
-    D = (D << (BD > 0x1f ? 14 : 7)) | (getPrefixEncodingFromUnsigned(BD) << 1);
-
-  return cloneWithDiscriminator(D);
+  unsigned CI = getCopyIdentifier();
+  if (Optional<unsigned> D = encodeDiscriminator(BD, DF, CI))
+    return cloneWithDiscriminator(*D);
+  return None;
 }
 
 class DINamespace : public DIScope {
@@ -2475,6 +2510,12 @@ public:
   /// return true with an offset of zero.
   bool extractIfOffset(int64_t &Offset) const;
 
+  /// Checks if the last 4 elements of the expression are DW_OP_constu <DWARF
+  /// Address Space> DW_OP_swap DW_OP_xderef and extracts the <DWARF Address
+  /// Space>.
+  static const DIExpression *extractAddressClass(const DIExpression *Expr,
+                                                 unsigned &AddrClass);
+
   /// Constants for DIExpression::prepend.
   enum { NoDeref = false, WithDeref = true, WithStackValue = true };
 
diff --git a/include/llvm/IR/DebugLoc.h b/include/llvm/IR/DebugLoc.h
index 4f0d7f51b5f9acd9b709eb4554c9bc5b21beb595..780d17a33661f8670a085afd8dd0e917093481d6 100644
--- a/include/llvm/IR/DebugLoc.h
+++ b/include/llvm/IR/DebugLoc.h
@@ -1,9 +1,8 @@
 //===- DebugLoc.h - Debug Location Information ------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/IR/DerivedTypes.h b/include/llvm/IR/DerivedTypes.h
index 9526d6287d2f83763a7fddf70af768ce710b13dd..5bf37294bb2ec837bbb350d185a75b4de95858e7 100644
--- a/include/llvm/IR/DerivedTypes.h
+++ b/include/llvm/IR/DerivedTypes.h
@@ -1,9 +1,8 @@
 //===- llvm/DerivedTypes.h - Classes for handling data types ----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -158,6 +157,38 @@ unsigned Type::getFunctionNumParams() const {
   return cast<FunctionType>(this)->getNumParams();
 }
 
+/// A handy container for a FunctionType+Callee-pointer pair, which can be
+/// passed around as a single entity. This assists in replacing the use of
+/// PointerType::getElementType() to access the function's type, since that's
+/// slated for removal as part of the [opaque pointer types] project.
+class FunctionCallee {
+public:
+  // Allow implicit conversion from types which have a getFunctionType member
+  // (e.g. Function and InlineAsm).
+  template <typename T, typename U = decltype(&T::getFunctionType)>
+  FunctionCallee(T *Fn)
+      : FnTy(Fn ? Fn->getFunctionType() : nullptr), Callee(Fn) {}
+
+  FunctionCallee(FunctionType *FnTy, Value *Callee)
+      : FnTy(FnTy), Callee(Callee) {
+    assert((FnTy == nullptr) == (Callee == nullptr));
+  }
+
+  FunctionCallee(std::nullptr_t) {}
+
+  FunctionCallee() = default;
+
+  FunctionType *getFunctionType() { return FnTy; }
+
+  Value *getCallee() { return Callee; }
+
+  explicit operator bool() { return Callee; }
+
+private:
+  FunctionType *FnTy = nullptr;
+  Value *Callee = nullptr;
+};
+
 /// Common super class of ArrayType, StructType and VectorType.
 class CompositeType : public Type {
 protected:
diff --git a/include/llvm/IR/DerivedUser.h b/include/llvm/IR/DerivedUser.h
index 67c483d3c497723c5759f069bb2a3494d2be0747..a25d316c2d60bc5ec64fdab9d6646520fe016560 100644
--- a/include/llvm/IR/DerivedUser.h
+++ b/include/llvm/IR/DerivedUser.h
@@ -1,9 +1,8 @@
 //===- DerivedUser.h - Base for non-IR Users --------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/IR/DiagnosticHandler.h b/include/llvm/IR/DiagnosticHandler.h
index 51873bea3d413598207109aa541aeff339de2773..af1e2977d48aa7725b323b0d6093b06164f1c860 100644
--- a/include/llvm/IR/DiagnosticHandler.h
+++ b/include/llvm/IR/DiagnosticHandler.h
@@ -1,9 +1,8 @@
 //===- DiagnosticHandler.h - DiagnosticHandler class for LLVM -*- C++ ---*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 // Base DiagnosticHandler class declaration. Derive from this class to provide
diff --git a/include/llvm/IR/DiagnosticInfo.h b/include/llvm/IR/DiagnosticInfo.h
index b5ed2c7d5097812d4fc8e2640fcc96442f788624..ab4c692e18b8b1a10da8250068b64a001404dcfc 100644
--- a/include/llvm/IR/DiagnosticInfo.h
+++ b/include/llvm/IR/DiagnosticInfo.h
@@ -1,9 +1,8 @@
 //===- llvm/IR/DiagnosticInfo.h - Diagnostic Declaration --------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -101,6 +100,7 @@ private:
   /// Severity gives the severity of the diagnostic.
   const DiagnosticSeverity Severity;
 
+  virtual void anchor();
 public:
   DiagnosticInfo(/* DiagnosticKind */ int Kind, DiagnosticSeverity Severity)
       : Kind(Kind), Severity(Severity) {}
@@ -210,6 +210,7 @@ public:
 };
 
 class DiagnosticInfoStackSize : public DiagnosticInfoResourceLimit {
+  virtual void anchor() override;
 public:
   DiagnosticInfoStackSize(const Function &Fn, uint64_t StackSize,
                           DiagnosticSeverity Severity = DS_Warning,
@@ -360,6 +361,7 @@ public:
 
 /// Common features for diagnostics with an associated location.
 class DiagnosticInfoWithLocationBase : public DiagnosticInfo {
+  virtual void anchor() override;
 public:
   /// \p Fn is the function where the diagnostic is being emitted. \p Loc is
   /// the location information to use in the diagnostic.
@@ -598,6 +600,7 @@ operator<<(RemarkT &R,
 /// Common features for diagnostics dealing with optimization remarks
 /// that are used by IR passes.
 class DiagnosticInfoIROptimization : public DiagnosticInfoOptimizationBase {
+  virtual void anchor() override;
 public:
   /// \p PassName is the name of the pass emitting this diagnostic. \p
   /// RemarkName is a textual identifier for the remark (single-word,
@@ -818,6 +821,7 @@ private:
 /// Diagnostic information for optimization analysis remarks related to
 /// floating-point non-commutativity.
 class OptimizationRemarkAnalysisFPCommute : public OptimizationRemarkAnalysis {
+  virtual void anchor();
 public:
   /// \p PassName is the name of the pass emitting this diagnostic. If this name
   /// matches the regular expression given in -Rpass-analysis=, then the
@@ -859,6 +863,7 @@ private:
 /// Diagnostic information for optimization analysis remarks related to
 /// pointer aliasing.
 class OptimizationRemarkAnalysisAliasing : public OptimizationRemarkAnalysis {
+  virtual void anchor();
 public:
   /// \p PassName is the name of the pass emitting this diagnostic. If this name
   /// matches the regular expression given in -Rpass-analysis=, then the
diff --git a/include/llvm/IR/DiagnosticPrinter.h b/include/llvm/IR/DiagnosticPrinter.h
index 25c47cdd1a1208bf5de7683f70d7f8718dea10c7..102932ceefa518bc29aaebde8c81578680638500 100644
--- a/include/llvm/IR/DiagnosticPrinter.h
+++ b/include/llvm/IR/DiagnosticPrinter.h
@@ -1,9 +1,8 @@
 //===- llvm/Support/DiagnosticPrinter.h - Diagnostic Printer ----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/IR/Dominators.h b/include/llvm/IR/Dominators.h
index f7da47d076632b11da74a24347c9602b461c5db6..fef1c6abf8c2baa46f2c7c537c4f8b405d892385 100644
--- a/include/llvm/IR/Dominators.h
+++ b/include/llvm/IR/Dominators.h
@@ -1,9 +1,8 @@
 //===- Dominators.h - Dominator Info Calculation ----------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/IR/Function.h b/include/llvm/IR/Function.h
index 630f47e8bb5763712f8043b05053e6582b10bb23..7184cb49af97c83ea6510a394a24b33580332e3f 100644
--- a/include/llvm/IR/Function.h
+++ b/include/llvm/IR/Function.h
@@ -1,9 +1,8 @@
 //===- llvm/Function.h - Class to represent a single function ---*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/IR/GVMaterializer.h b/include/llvm/IR/GVMaterializer.h
index 675abeb6ec3a2e475c1c6c7521122538e05f44d4..d62da41ebc296626365bdc05794877798963bde2 100644
--- a/include/llvm/IR/GVMaterializer.h
+++ b/include/llvm/IR/GVMaterializer.h
@@ -1,9 +1,8 @@
 //===- GVMaterializer.h - Interface for GV materializers --------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/IR/GetElementPtrTypeIterator.h b/include/llvm/IR/GetElementPtrTypeIterator.h
index 3c143ea5f703eb7df62f6d88687b0c104f8bab21..9b257abc7c1f1ea1ff631ee3a14a9e754e7a5118 100644
--- a/include/llvm/IR/GetElementPtrTypeIterator.h
+++ b/include/llvm/IR/GetElementPtrTypeIterator.h
@@ -1,9 +1,8 @@
 //===- GetElementPtrTypeIterator.h ------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/IR/GlobalAlias.h b/include/llvm/IR/GlobalAlias.h
index 450583baaa3c1b2dc1ef42bb3bfa46cc1e8b1b99..3cd40570130039b34d5ebc6d73b53c0fe06b605c 100644
--- a/include/llvm/IR/GlobalAlias.h
+++ b/include/llvm/IR/GlobalAlias.h
@@ -1,9 +1,8 @@
 //===-------- llvm/GlobalAlias.h - GlobalAlias class ------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/IR/GlobalIFunc.h b/include/llvm/IR/GlobalIFunc.h
index ef51315a6f5d9eb957f5202ba91844af10454b3d..bc0d3c053cce26e5c22f8321054b908470e1041d 100644
--- a/include/llvm/IR/GlobalIFunc.h
+++ b/include/llvm/IR/GlobalIFunc.h
@@ -1,9 +1,8 @@
 //===-------- llvm/GlobalIFunc.h - GlobalIFunc class ------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
diff --git a/include/llvm/IR/GlobalIndirectSymbol.h b/include/llvm/IR/GlobalIndirectSymbol.h
index 22c00686c549f4b3c66e167b0395b1b23a778a06..8bc3f90b94aaf23e389161bdcf41c405b2837c43 100644
--- a/include/llvm/IR/GlobalIndirectSymbol.h
+++ b/include/llvm/IR/GlobalIndirectSymbol.h
@@ -1,9 +1,8 @@
 //===- llvm/GlobalIndirectSymbol.h - GlobalIndirectSymbol class -*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/IR/GlobalObject.h b/include/llvm/IR/GlobalObject.h
index 1fd3568100c2673ee0340e1003a55b2914d36c8c..b8ab6140ebe7ba1c0c86c3da060d6fa4df5995cd 100644
--- a/include/llvm/IR/GlobalObject.h
+++ b/include/llvm/IR/GlobalObject.h
@@ -1,9 +1,8 @@
 //===-- llvm/GlobalObject.h - Class to represent global objects -*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/IR/GlobalValue.h b/include/llvm/IR/GlobalValue.h
index c07d4051c803d307aa791262d3d8df9a7bdf7da6..d78ce622bc5dd099a4b6e91d29b1759b7ac0c1f0 100644
--- a/include/llvm/IR/GlobalValue.h
+++ b/include/llvm/IR/GlobalValue.h
@@ -1,9 +1,8 @@
 //===-- llvm/GlobalValue.h - Class to represent a global value --*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/IR/GlobalVariable.h b/include/llvm/IR/GlobalVariable.h
index 03b9ec46ebb439481849156c0c8754f414031cef..2e2c8c47791351c17cfd3ffd3be744ff7eadb383 100644
--- a/include/llvm/IR/GlobalVariable.h
+++ b/include/llvm/IR/GlobalVariable.h
@@ -1,9 +1,8 @@
 //===-- llvm/GlobalVariable.h - GlobalVariable class ------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/IR/IRBuilder.h b/include/llvm/IR/IRBuilder.h
index e89c44380d0b0435248f49c13228cab292e4b9e9..2e0619d54f8601cce9d9f38db4bb626d8d147f67 100644
--- a/include/llvm/IR/IRBuilder.h
+++ b/include/llvm/IR/IRBuilder.h
@@ -1,9 +1,8 @@
 //===- llvm/IRBuilder.h - Builder for LLVM Instructions ---------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -889,19 +888,95 @@ public:
   }
 
   /// Create an invoke instruction.
-  InvokeInst *CreateInvoke(Value *Callee, BasicBlock *NormalDest,
-                           BasicBlock *UnwindDest,
+  InvokeInst *CreateInvoke(FunctionType *Ty, Value *Callee,
+                           BasicBlock *NormalDest, BasicBlock *UnwindDest,
+                           ArrayRef<Value *> Args,
+                           ArrayRef<OperandBundleDef> OpBundles,
+                           const Twine &Name = "") {
+    return Insert(
+        InvokeInst::Create(Ty, Callee, NormalDest, UnwindDest, Args, OpBundles),
+        Name);
+  }
+  InvokeInst *CreateInvoke(FunctionType *Ty, Value *Callee,
+                           BasicBlock *NormalDest, BasicBlock *UnwindDest,
                            ArrayRef<Value *> Args = None,
                            const Twine &Name = "") {
-    return Insert(InvokeInst::Create(Callee, NormalDest, UnwindDest, Args),
+    return Insert(InvokeInst::Create(Ty, Callee, NormalDest, UnwindDest, Args),
                   Name);
   }
+
+  InvokeInst *CreateInvoke(FunctionCallee Callee, BasicBlock *NormalDest,
+                           BasicBlock *UnwindDest, ArrayRef<Value *> Args,
+                           ArrayRef<OperandBundleDef> OpBundles,
+                           const Twine &Name = "") {
+    return CreateInvoke(Callee.getFunctionType(), Callee.getCallee(),
+                        NormalDest, UnwindDest, Args, OpBundles, Name);
+  }
+
+  InvokeInst *CreateInvoke(FunctionCallee Callee, BasicBlock *NormalDest,
+                           BasicBlock *UnwindDest,
+                           ArrayRef<Value *> Args = None,
+                           const Twine &Name = "") {
+    return CreateInvoke(Callee.getFunctionType(), Callee.getCallee(),
+                        NormalDest, UnwindDest, Args, Name);
+  }
+
+  // Deprecated [opaque pointer types]
   InvokeInst *CreateInvoke(Value *Callee, BasicBlock *NormalDest,
                            BasicBlock *UnwindDest, ArrayRef<Value *> Args,
                            ArrayRef<OperandBundleDef> OpBundles,
                            const Twine &Name = "") {
-    return Insert(InvokeInst::Create(Callee, NormalDest, UnwindDest, Args,
-                                     OpBundles), Name);
+    return CreateInvoke(
+        cast<FunctionType>(
+            cast<PointerType>(Callee->getType())->getElementType()),
+        Callee, NormalDest, UnwindDest, Args, OpBundles, Name);
+  }
+
+  // Deprecated [opaque pointer types]
+  InvokeInst *CreateInvoke(Value *Callee, BasicBlock *NormalDest,
+                           BasicBlock *UnwindDest,
+                           ArrayRef<Value *> Args = None,
+                           const Twine &Name = "") {
+    return CreateInvoke(
+        cast<FunctionType>(
+            cast<PointerType>(Callee->getType())->getElementType()),
+        Callee, NormalDest, UnwindDest, Args, Name);
+  }
+
+  /// \brief Create a callbr instruction.
+  CallBrInst *CreateCallBr(FunctionType *Ty, Value *Callee,
+                           BasicBlock *DefaultDest,
+                           ArrayRef<BasicBlock *> IndirectDests,
+                           ArrayRef<Value *> Args = None,
+                           const Twine &Name = "") {
+    return Insert(CallBrInst::Create(Ty, Callee, DefaultDest, IndirectDests,
+                                     Args), Name);
+  }
+  CallBrInst *CreateCallBr(FunctionType *Ty, Value *Callee,
+                           BasicBlock *DefaultDest,
+                           ArrayRef<BasicBlock *> IndirectDests,
+                           ArrayRef<Value *> Args,
+                           ArrayRef<OperandBundleDef> OpBundles,
+                           const Twine &Name = "") {
+    return Insert(
+        CallBrInst::Create(Ty, Callee, DefaultDest, IndirectDests, Args,
+                           OpBundles), Name);
+  }
+
+  CallBrInst *CreateCallBr(FunctionCallee Callee, BasicBlock *DefaultDest,
+                           ArrayRef<BasicBlock *> IndirectDests,
+                           ArrayRef<Value *> Args = None,
+                           const Twine &Name = "") {
+    return CreateCallBr(Callee.getFunctionType(), Callee.getCallee(),
+                        DefaultDest, IndirectDests, Args, Name);
+  }
+  CallBrInst *CreateCallBr(FunctionCallee Callee, BasicBlock *DefaultDest,
+                           ArrayRef<BasicBlock *> IndirectDests,
+                           ArrayRef<Value *> Args,
+                           ArrayRef<OperandBundleDef> OpBundles,
+                           const Twine &Name = "") {
+    return CreateCallBr(Callee.getFunctionType(), Callee.getCallee(),
+                        DefaultDest, IndirectDests, Args, Name);
   }
 
   ResumeInst *CreateResume(Value *Exn) {
@@ -964,7 +1039,7 @@ private:
   }
 
   Value *foldConstant(Instruction::BinaryOps Opc, Value *L,
-                      Value *R, const Twine &Name = nullptr) const {
+                      Value *R, const Twine &Name) const {
     auto *LC = dyn_cast<Constant>(L);
     auto *RC = dyn_cast<Constant>(R);
     return (LC && RC) ? Insert(Folder.CreateBinOp(Opc, LC, RC), Name) : nullptr;
@@ -1312,22 +1387,35 @@ public:
     return Insert(new AllocaInst(Ty, DL.getAllocaAddrSpace(), ArraySize), Name);
   }
 
-  /// Provided to resolve 'CreateLoad(Ptr, "...")' correctly, instead of
+  /// Provided to resolve 'CreateLoad(Ty, Ptr, "...")' correctly, instead of
   /// converting the string to 'bool' for the isVolatile parameter.
-  LoadInst *CreateLoad(Value *Ptr, const char *Name) {
-    return Insert(new LoadInst(Ptr), Name);
-  }
-
-  LoadInst *CreateLoad(Value *Ptr, const Twine &Name = "") {
-    return Insert(new LoadInst(Ptr), Name);
+  LoadInst *CreateLoad(Type *Ty, Value *Ptr, const char *Name) {
+    return Insert(new LoadInst(Ty, Ptr), Name);
   }
 
   LoadInst *CreateLoad(Type *Ty, Value *Ptr, const Twine &Name = "") {
     return Insert(new LoadInst(Ty, Ptr), Name);
   }
 
+  LoadInst *CreateLoad(Type *Ty, Value *Ptr, bool isVolatile,
+                       const Twine &Name = "") {
+    return Insert(new LoadInst(Ty, Ptr, Twine(), isVolatile), Name);
+  }
+
+  // Deprecated [opaque pointer types]
+  LoadInst *CreateLoad(Value *Ptr, const char *Name) {
+    return CreateLoad(Ptr->getType()->getPointerElementType(), Ptr, Name);
+  }
+
+  // Deprecated [opaque pointer types]
+  LoadInst *CreateLoad(Value *Ptr, const Twine &Name = "") {
+    return CreateLoad(Ptr->getType()->getPointerElementType(), Ptr, Name);
+  }
+
+  // Deprecated [opaque pointer types]
   LoadInst *CreateLoad(Value *Ptr, bool isVolatile, const Twine &Name = "") {
-    return Insert(new LoadInst(Ptr, nullptr, isVolatile), Name);
+    return CreateLoad(Ptr->getType()->getPointerElementType(), Ptr, isVolatile,
+                      Name);
   }
 
   StoreInst *CreateStore(Value *Val, Value *Ptr, bool isVolatile = false) {
@@ -1337,24 +1425,43 @@ public:
   /// Provided to resolve 'CreateAlignedLoad(Ptr, Align, "...")'
   /// correctly, instead of converting the string to 'bool' for the isVolatile
   /// parameter.
-  LoadInst *CreateAlignedLoad(Value *Ptr, unsigned Align, const char *Name) {
-    LoadInst *LI = CreateLoad(Ptr, Name);
+  LoadInst *CreateAlignedLoad(Type *Ty, Value *Ptr, unsigned Align,
+                              const char *Name) {
+    LoadInst *LI = CreateLoad(Ty, Ptr, Name);
     LI->setAlignment(Align);
     return LI;
   }
-  LoadInst *CreateAlignedLoad(Value *Ptr, unsigned Align,
+  LoadInst *CreateAlignedLoad(Type *Ty, Value *Ptr, unsigned Align,
                               const Twine &Name = "") {
-    LoadInst *LI = CreateLoad(Ptr, Name);
+    LoadInst *LI = CreateLoad(Ty, Ptr, Name);
     LI->setAlignment(Align);
     return LI;
   }
-  LoadInst *CreateAlignedLoad(Value *Ptr, unsigned Align, bool isVolatile,
-                              const Twine &Name = "") {
-    LoadInst *LI = CreateLoad(Ptr, isVolatile, Name);
+  LoadInst *CreateAlignedLoad(Type *Ty, Value *Ptr, unsigned Align,
+                              bool isVolatile, const Twine &Name = "") {
+    LoadInst *LI = CreateLoad(Ty, Ptr, isVolatile, Name);
     LI->setAlignment(Align);
     return LI;
   }
 
+  // Deprecated [opaque pointer types]
+  LoadInst *CreateAlignedLoad(Value *Ptr, unsigned Align, const char *Name) {
+    return CreateAlignedLoad(Ptr->getType()->getPointerElementType(), Ptr,
+                             Align, Name);
+  }
+  // Deprecated [opaque pointer types]
+  LoadInst *CreateAlignedLoad(Value *Ptr, unsigned Align,
+                              const Twine &Name = "") {
+    return CreateAlignedLoad(Ptr->getType()->getPointerElementType(), Ptr,
+                             Align, Name);
+  }
+  // Deprecated [opaque pointer types]
+  LoadInst *CreateAlignedLoad(Value *Ptr, unsigned Align, bool isVolatile,
+                              const Twine &Name = "") {
+    return CreateAlignedLoad(Ptr->getType()->getPointerElementType(), Ptr,
+                             Align, isVolatile, Name);
+  }
+
   StoreInst *CreateAlignedStore(Value *Val, Value *Ptr, unsigned Align,
                                 bool isVolatile = false) {
     StoreInst *SI = CreateStore(Val, Ptr, isVolatile);
@@ -1491,50 +1598,69 @@ public:
     return Insert(GetElementPtrInst::CreateInBounds(Ty, Ptr, Idxs), Name);
   }
 
-  Value *CreateConstGEP1_64(Value *Ptr, uint64_t Idx0, const Twine &Name = "") {
+  Value *CreateConstGEP1_64(Type *Ty, Value *Ptr, uint64_t Idx0,
+                            const Twine &Name = "") {
     Value *Idx = ConstantInt::get(Type::getInt64Ty(Context), Idx0);
 
     if (auto *PC = dyn_cast<Constant>(Ptr))
-      return Insert(Folder.CreateGetElementPtr(nullptr, PC, Idx), Name);
+      return Insert(Folder.CreateGetElementPtr(Ty, PC, Idx), Name);
 
-    return Insert(GetElementPtrInst::Create(nullptr, Ptr, Idx), Name);
+    return Insert(GetElementPtrInst::Create(Ty, Ptr, Idx), Name);
   }
 
-  Value *CreateConstInBoundsGEP1_64(Value *Ptr, uint64_t Idx0,
+  Value *CreateConstGEP1_64(Value *Ptr, uint64_t Idx0, const Twine &Name = "") {
+    return CreateConstGEP1_64(nullptr, Ptr, Idx0, Name);
+  }
+
+  Value *CreateConstInBoundsGEP1_64(Type *Ty, Value *Ptr, uint64_t Idx0,
                                     const Twine &Name = "") {
     Value *Idx = ConstantInt::get(Type::getInt64Ty(Context), Idx0);
 
     if (auto *PC = dyn_cast<Constant>(Ptr))
-      return Insert(Folder.CreateInBoundsGetElementPtr(nullptr, PC, Idx), Name);
+      return Insert(Folder.CreateInBoundsGetElementPtr(Ty, PC, Idx), Name);
+
+    return Insert(GetElementPtrInst::CreateInBounds(Ty, Ptr, Idx), Name);
+  }
 
-    return Insert(GetElementPtrInst::CreateInBounds(nullptr, Ptr, Idx), Name);
+  Value *CreateConstInBoundsGEP1_64(Value *Ptr, uint64_t Idx0,
+                                    const Twine &Name = "") {
+    return CreateConstInBoundsGEP1_64(nullptr, Ptr, Idx0, Name);
   }
 
-  Value *CreateConstGEP2_64(Value *Ptr, uint64_t Idx0, uint64_t Idx1,
-                    const Twine &Name = "") {
+  Value *CreateConstGEP2_64(Type *Ty, Value *Ptr, uint64_t Idx0, uint64_t Idx1,
+                            const Twine &Name = "") {
     Value *Idxs[] = {
       ConstantInt::get(Type::getInt64Ty(Context), Idx0),
       ConstantInt::get(Type::getInt64Ty(Context), Idx1)
     };
 
     if (auto *PC = dyn_cast<Constant>(Ptr))
-      return Insert(Folder.CreateGetElementPtr(nullptr, PC, Idxs), Name);
+      return Insert(Folder.CreateGetElementPtr(Ty, PC, Idxs), Name);
 
-    return Insert(GetElementPtrInst::Create(nullptr, Ptr, Idxs), Name);
+    return Insert(GetElementPtrInst::Create(Ty, Ptr, Idxs), Name);
   }
 
-  Value *CreateConstInBoundsGEP2_64(Value *Ptr, uint64_t Idx0, uint64_t Idx1,
-                                    const Twine &Name = "") {
+  Value *CreateConstGEP2_64(Value *Ptr, uint64_t Idx0, uint64_t Idx1,
+                            const Twine &Name = "") {
+    return CreateConstGEP2_64(nullptr, Ptr, Idx0, Idx1, Name);
+  }
+
+  Value *CreateConstInBoundsGEP2_64(Type *Ty, Value *Ptr, uint64_t Idx0,
+                                    uint64_t Idx1, const Twine &Name = "") {
     Value *Idxs[] = {
       ConstantInt::get(Type::getInt64Ty(Context), Idx0),
       ConstantInt::get(Type::getInt64Ty(Context), Idx1)
     };
 
     if (auto *PC = dyn_cast<Constant>(Ptr))
-      return Insert(Folder.CreateInBoundsGetElementPtr(nullptr, PC, Idxs),
-                    Name);
+      return Insert(Folder.CreateInBoundsGetElementPtr(Ty, PC, Idxs), Name);
+
+    return Insert(GetElementPtrInst::CreateInBounds(Ty, Ptr, Idxs), Name);
+  }
 
-    return Insert(GetElementPtrInst::CreateInBounds(nullptr, Ptr, Idxs), Name);
+  Value *CreateConstInBoundsGEP2_64(Value *Ptr, uint64_t Idx0, uint64_t Idx1,
+                                    const Twine &Name = "") {
+    return CreateConstInBoundsGEP2_64(nullptr, Ptr, Idx0, Idx1, Name);
   }
 
   Value *CreateStructGEP(Type *Ty, Value *Ptr, unsigned Idx,
@@ -1880,15 +2006,8 @@ public:
     return Insert(PHINode::Create(Ty, NumReservedValues), Name);
   }
 
-  CallInst *CreateCall(Value *Callee, ArrayRef<Value *> Args = None,
-                       const Twine &Name = "", MDNode *FPMathTag = nullptr) {
-    auto *PTy = cast<PointerType>(Callee->getType());
-    auto *FTy = cast<FunctionType>(PTy->getElementType());
-    return CreateCall(FTy, Callee, Args, Name, FPMathTag);
-  }
-
   CallInst *CreateCall(FunctionType *FTy, Value *Callee,
-                       ArrayRef<Value *> Args, const Twine &Name = "",
+                       ArrayRef<Value *> Args = None, const Twine &Name = "",
                        MDNode *FPMathTag = nullptr) {
     CallInst *CI = CallInst::Create(FTy, Callee, Args, DefaultOperandBundles);
     if (isa<FPMathOperator>(CI))
@@ -1896,18 +2015,43 @@ public:
     return Insert(CI, Name);
   }
 
-  CallInst *CreateCall(Value *Callee, ArrayRef<Value *> Args,
+  CallInst *CreateCall(FunctionType *FTy, Value *Callee, ArrayRef<Value *> Args,
                        ArrayRef<OperandBundleDef> OpBundles,
                        const Twine &Name = "", MDNode *FPMathTag = nullptr) {
-    CallInst *CI = CallInst::Create(Callee, Args, OpBundles);
+    CallInst *CI = CallInst::Create(FTy, Callee, Args, OpBundles);
     if (isa<FPMathOperator>(CI))
       CI = cast<CallInst>(setFPAttrs(CI, FPMathTag, FMF));
     return Insert(CI, Name);
   }
 
-  CallInst *CreateCall(Function *Callee, ArrayRef<Value *> Args,
+  CallInst *CreateCall(FunctionCallee Callee, ArrayRef<Value *> Args = None,
                        const Twine &Name = "", MDNode *FPMathTag = nullptr) {
-    return CreateCall(Callee->getFunctionType(), Callee, Args, Name, FPMathTag);
+    return CreateCall(Callee.getFunctionType(), Callee.getCallee(), Args, Name,
+                      FPMathTag);
+  }
+
+  CallInst *CreateCall(FunctionCallee Callee, ArrayRef<Value *> Args,
+                       ArrayRef<OperandBundleDef> OpBundles,
+                       const Twine &Name = "", MDNode *FPMathTag = nullptr) {
+    return CreateCall(Callee.getFunctionType(), Callee.getCallee(), Args,
+                      OpBundles, Name, FPMathTag);
+  }
+
+  // Deprecated [opaque pointer types]
+  CallInst *CreateCall(Value *Callee, ArrayRef<Value *> Args = None,
+                       const Twine &Name = "", MDNode *FPMathTag = nullptr) {
+    return CreateCall(
+        cast<FunctionType>(Callee->getType()->getPointerElementType()), Callee,
+        Args, Name, FPMathTag);
+  }
+
+  // Deprecated [opaque pointer types]
+  CallInst *CreateCall(Value *Callee, ArrayRef<Value *> Args,
+                       ArrayRef<OperandBundleDef> OpBundles,
+                       const Twine &Name = "", MDNode *FPMathTag = nullptr) {
+    return CreateCall(
+        cast<FunctionType>(Callee->getType()->getPointerElementType()), Callee,
+        Args, OpBundles, Name, FPMathTag);
   }
 
   Value *CreateSelect(Value *C, Value *True, Value *False,
@@ -2126,11 +2270,12 @@ public:
 private:
   /// Helper function that creates an assume intrinsic call that
   /// represents an alignment assumption on the provided Ptr, Mask, Type
-  /// and Offset.
+  /// and Offset. It may be sometimes useful to do some other logic
+  /// based on this alignment check, thus it can be stored into 'TheCheck'.
   CallInst *CreateAlignmentAssumptionHelper(const DataLayout &DL,
                                             Value *PtrValue, Value *Mask,
-                                            Type *IntPtrTy,
-                                            Value *OffsetValue) {
+                                            Type *IntPtrTy, Value *OffsetValue,
+                                            Value **TheCheck) {
     Value *PtrIntValue = CreatePtrToInt(PtrValue, IntPtrTy, "ptrint");
 
     if (OffsetValue) {
@@ -2149,6 +2294,9 @@ private:
     Value *Zero = ConstantInt::get(IntPtrTy, 0);
     Value *MaskedPtr = CreateAnd(PtrIntValue, Mask, "maskedptr");
     Value *InvCond = CreateICmpEQ(MaskedPtr, Zero, "maskcond");
+    if (TheCheck)
+      *TheCheck = InvCond;
+
     return CreateAssumption(InvCond);
   }
 
@@ -2159,17 +2307,22 @@ public:
   /// An optional offset can be provided, and if it is provided, the offset
   /// must be subtracted from the provided pointer to get the pointer with the
   /// specified alignment.
+  ///
+  /// It may be sometimes useful to do some other logic
+  /// based on this alignment check, thus it can be stored into 'TheCheck'.
   CallInst *CreateAlignmentAssumption(const DataLayout &DL, Value *PtrValue,
                                       unsigned Alignment,
-                                      Value *OffsetValue = nullptr) {
+                                      Value *OffsetValue = nullptr,
+                                      Value **TheCheck = nullptr) {
     assert(isa<PointerType>(PtrValue->getType()) &&
            "trying to create an alignment assumption on a non-pointer?");
+    assert(Alignment != 0 && "Invalid Alignment");
     auto *PtrTy = cast<PointerType>(PtrValue->getType());
     Type *IntPtrTy = getIntPtrTy(DL, PtrTy->getAddressSpace());
 
-    Value *Mask = ConstantInt::get(IntPtrTy, Alignment > 0 ? Alignment - 1 : 0);
+    Value *Mask = ConstantInt::get(IntPtrTy, Alignment - 1);
     return CreateAlignmentAssumptionHelper(DL, PtrValue, Mask, IntPtrTy,
-                                           OffsetValue);
+                                           OffsetValue, TheCheck);
   }
 
   /// Create an assume intrinsic call that represents an alignment
@@ -2179,29 +2332,28 @@ public:
   /// must be subtracted from the provided pointer to get the pointer with the
   /// specified alignment.
   ///
+  /// It may be sometimes useful to do some other logic
+  /// based on this alignment check, thus it can be stored into 'TheCheck'.
+  ///
   /// This overload handles the condition where the Alignment is dependent
   /// on an existing value rather than a static value.
   CallInst *CreateAlignmentAssumption(const DataLayout &DL, Value *PtrValue,
                                       Value *Alignment,
-                                      Value *OffsetValue = nullptr) {
+                                      Value *OffsetValue = nullptr,
+                                      Value **TheCheck = nullptr) {
     assert(isa<PointerType>(PtrValue->getType()) &&
            "trying to create an alignment assumption on a non-pointer?");
     auto *PtrTy = cast<PointerType>(PtrValue->getType());
     Type *IntPtrTy = getIntPtrTy(DL, PtrTy->getAddressSpace());
 
     if (Alignment->getType() != IntPtrTy)
-      Alignment = CreateIntCast(Alignment, IntPtrTy, /*isSigned*/ true,
+      Alignment = CreateIntCast(Alignment, IntPtrTy, /*isSigned*/ false,
                                 "alignmentcast");
-    Value *IsPositive =
-        CreateICmp(CmpInst::ICMP_SGT, Alignment,
-                   ConstantInt::get(Alignment->getType(), 0), "ispositive");
-    Value *PositiveMask =
-        CreateSub(Alignment, ConstantInt::get(IntPtrTy, 1), "positivemask");
-    Value *Mask = CreateSelect(IsPositive, PositiveMask,
-                               ConstantInt::get(IntPtrTy, 0), "mask");
+
+    Value *Mask = CreateSub(Alignment, ConstantInt::get(IntPtrTy, 1), "mask");
 
     return CreateAlignmentAssumptionHelper(DL, PtrValue, Mask, IntPtrTy,
-                                           OffsetValue);
+                                           OffsetValue, TheCheck);
   }
 };
 
diff --git a/include/llvm/IR/IRPrintingPasses.h b/include/llvm/IR/IRPrintingPasses.h
index 75f80567dbd5392c0018076c6fd2d7eb38711fe0..3be9449c1a9365175a7bf056d9811b02cc86115e 100644
--- a/include/llvm/IR/IRPrintingPasses.h
+++ b/include/llvm/IR/IRPrintingPasses.h
@@ -1,9 +1,8 @@
 //===- IRPrintingPasses.h - Passes to print out IR constructs ---*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/include/llvm/IR/InlineAsm.h b/include/llvm/IR/InlineAsm.h
index 1519a45d59e91968f2915c3fe9b13a68f25c509c..2aac807623a976d53fbc53f9f5fa1c5ee0fa6d42 100644
--- a/include/llvm/IR/InlineAsm.h
+++ b/include/llvm/IR/InlineAsm.h
@@ -1,9 +1,8 @@
 //===- llvm/InlineAsm.h - Class to represent inline asm strings -*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/IR/InstIterator.h b/include/llvm/IR/InstIterator.h
index 2988fc935dd5d169aa523f6bf1ede8de9e81c6c8..054fe4e9cbe9920fb0d331c88221a49bc92b8c82 100644
--- a/include/llvm/IR/InstIterator.h
+++ b/include/llvm/IR/InstIterator.h
@@ -1,9 +1,8 @@
 //===- InstIterator.h - Classes for inst iteration --------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/IR/InstVisitor.h b/include/llvm/IR/InstVisitor.h
index 5e9b18d90a0cade0c32f7649472cfa21c73c91d6..fbeb2caf14e67bfad4485c5198826cb15278d826 100644
--- a/include/llvm/IR/InstVisitor.h
+++ b/include/llvm/IR/InstVisitor.h
@@ -1,9 +1,8 @@
 //===- InstVisitor.h - Instruction visitor templates ------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -218,14 +217,17 @@ public:
   RetTy visitVACopyInst(VACopyInst &I)            { DELEGATE(IntrinsicInst); }
   RetTy visitIntrinsicInst(IntrinsicInst &I)      { DELEGATE(CallInst); }
 
-  // Call and Invoke are slightly different as they delegate first through
-  // a generic CallSite visitor.
+  // Call, Invoke and CallBr are slightly different as they delegate first
+  // through a generic CallSite visitor.
   RetTy visitCallInst(CallInst &I) {
     return static_cast<SubClass*>(this)->visitCallSite(&I);
   }
   RetTy visitInvokeInst(InvokeInst &I) {
     return static_cast<SubClass*>(this)->visitCallSite(&I);
   }
+  RetTy visitCallBrInst(CallBrInst &I) {
+    return static_cast<SubClass *>(this)->visitCallSite(&I);
+  }
 
   // While terminators don't have a distinct type modeling them, we support
   // intercepting them with dedicated a visitor callback.
@@ -268,17 +270,23 @@ public:
   RetTy visitCmpInst(CmpInst &I)                  { DELEGATE(Instruction);}
   RetTy visitUnaryInstruction(UnaryInstruction &I){ DELEGATE(Instruction);}
 
-  // Provide a special visitor for a 'callsite' that visits both calls and
-  // invokes. When unimplemented, properly delegates to either the terminator or
-  // regular instruction visitor.
+  // The next level delegation for `CallBase` is slightly more complex in order
+  // to support visiting cases where the call is also a terminator.
+  RetTy visitCallBase(CallBase &I) {
+    if (isa<InvokeInst>(I) || isa<CallBrInst>(I))
+      return static_cast<SubClass *>(this)->visitTerminator(I);
+
+    DELEGATE(Instruction);
+  }
+
+  // Provide a legacy visitor for a 'callsite' that visits calls, invokes,
+  // and calbrs.
+  //
+  // Prefer overriding the type system based `CallBase` instead.
   RetTy visitCallSite(CallSite CS) {
     assert(CS);
     Instruction &I = *CS.getInstruction();
-    if (CS.isCall())
-      DELEGATE(Instruction);
-
-    assert(CS.isInvoke());
-    return static_cast<SubClass *>(this)->visitTerminator(I);
+    DELEGATE(CallBase);
   }
 
   // If the user wants a 'default' case, they can choose to override this
diff --git a/include/llvm/IR/InstrTypes.h b/include/llvm/IR/InstrTypes.h
index 4611a61e0b034b5fc151c396a46f2a89880346c3..6f506687ee5523e4b611a15bae05d993f3bdf9a1 100644
--- a/include/llvm/IR/InstrTypes.h
+++ b/include/llvm/IR/InstrTypes.h
@@ -1,9 +1,8 @@
 //===- llvm/InstrTypes.h - Important Instruction subclasses -----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -28,6 +27,7 @@
 #include "llvm/IR/CallingConv.h"
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/DerivedTypes.h"
+#include "llvm/IR/Function.h"
 #include "llvm/IR/Instruction.h"
 #include "llvm/IR/LLVMContext.h"
 #include "llvm/IR/OperandTraits.h"
@@ -46,6 +46,10 @@
 
 namespace llvm {
 
+namespace Intrinsic {
+enum ID : unsigned;
+}
+
 //===----------------------------------------------------------------------===//
 //                          UnaryInstruction Class
 //===----------------------------------------------------------------------===//
@@ -1029,16 +1033,26 @@ protected:
       return 0;
     case Instruction::Invoke:
       return 2;
+    case Instruction::CallBr:
+      return getNumSubclassExtraOperandsDynamic();
     }
     llvm_unreachable("Invalid opcode!");
   }
 
+  /// Get the number of extra operands for instructions that don't have a fixed
+  /// number of extra operands.
+  unsigned getNumSubclassExtraOperandsDynamic() const;
+
 public:
   using Instruction::getContext;
 
   static bool classof(const Instruction *I) {
     return I->getOpcode() == Instruction::Call ||
-           I->getOpcode() == Instruction::Invoke;
+           I->getOpcode() == Instruction::Invoke ||
+           I->getOpcode() == Instruction::CallBr;
+  }
+  static bool classof(const Value *V) {
+    return isa<Instruction>(V) && classof(cast<Instruction>(V));
   }
 
   FunctionType *getFunctionType() const { return FTy; }
@@ -1048,6 +1062,60 @@ public:
     this->FTy = FTy;
   }
 
+  DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value);
+
+  /// data_operands_begin/data_operands_end - Return iterators iterating over
+  /// the call / invoke argument list and bundle operands.  For invokes, this is
+  /// the set of instruction operands except the invoke target and the two
+  /// successor blocks; and for calls this is the set of instruction operands
+  /// except the call target.
+  User::op_iterator data_operands_begin() { return op_begin(); }
+  User::const_op_iterator data_operands_begin() const {
+    return const_cast<CallBase *>(this)->data_operands_begin();
+  }
+  User::op_iterator data_operands_end() {
+    // Walk from the end of the operands over the called operand and any
+    // subclass operands.
+    return op_end() - getNumSubclassExtraOperands() - 1;
+  }
+  User::const_op_iterator data_operands_end() const {
+    return const_cast<CallBase *>(this)->data_operands_end();
+  }
+  iterator_range<User::op_iterator> data_ops() {
+    return make_range(data_operands_begin(), data_operands_end());
+  }
+  iterator_range<User::const_op_iterator> data_ops() const {
+    return make_range(data_operands_begin(), data_operands_end());
+  }
+  bool data_operands_empty() const {
+    return data_operands_end() == data_operands_begin();
+  }
+  unsigned data_operands_size() const {
+    return std::distance(data_operands_begin(), data_operands_end());
+  }
+
+  bool isDataOperand(const Use *U) const {
+    assert(this == U->getUser() &&
+           "Only valid to query with a use of this instruction!");
+    return data_operands_begin() <= U && U < data_operands_end();
+  }
+  bool isDataOperand(Value::const_user_iterator UI) const {
+    return isDataOperand(&UI.getUse());
+  }
+
+  /// Given a value use iterator, return the data operand corresponding to it.
+  /// Iterator must actually correspond to a data operand.
+  unsigned getDataOperandNo(Value::const_user_iterator UI) const {
+    return getDataOperandNo(&UI.getUse());
+  }
+
+  /// Given a use for a data operand, get the data operand number that
+  /// corresponds to it.
+  unsigned getDataOperandNo(const Use *U) const {
+    assert(isDataOperand(U) && "Data operand # out of range!");
+    return U - data_operands_begin();
+  }
+
   /// Return the iterator pointing to the beginning of the argument list.
   User::op_iterator arg_begin() { return op_begin(); }
   User::const_op_iterator arg_begin() const {
@@ -1056,25 +1124,33 @@ public:
 
   /// Return the iterator pointing to the end of the argument list.
   User::op_iterator arg_end() {
-    // Walk from the end of the operands over the called operand, the subclass
-    // operands, and any operands for bundles to find the end of the argument
+    // From the end of the data operands, walk backwards past the bundle
     // operands.
-    return op_end() - getNumTotalBundleOperands() -
-           getNumSubclassExtraOperands() - 1;
+    return data_operands_end() - getNumTotalBundleOperands();
   }
   User::const_op_iterator arg_end() const {
     return const_cast<CallBase *>(this)->arg_end();
   }
 
   /// Iteration adapter for range-for loops.
+  iterator_range<User::op_iterator> args() {
+    return make_range(arg_begin(), arg_end());
+  }
+  iterator_range<User::const_op_iterator> args() const {
+    return make_range(arg_begin(), arg_end());
+  }
+  bool arg_empty() const { return arg_end() == arg_begin(); }
+  unsigned arg_size() const { return arg_end() - arg_begin(); }
+
+  // Legacy API names that duplicate the above and will be removed once users
+  // are migrated.
   iterator_range<User::op_iterator> arg_operands() {
     return make_range(arg_begin(), arg_end());
   }
   iterator_range<User::const_op_iterator> arg_operands() const {
     return make_range(arg_begin(), arg_end());
   }
-
-  unsigned getNumArgOperands() const { return arg_end() - arg_begin(); }
+  unsigned getNumArgOperands() const { return arg_size(); }
 
   Value *getArgOperand(unsigned i) const {
     assert(i < getNumArgOperands() && "Out of bounds!");
@@ -1096,7 +1172,20 @@ public:
     return User::getOperandUse(i);
   }
 
-  DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value);
+  bool isArgOperand(const Use *U) const {
+    assert(this == U->getUser() &&
+           "Only valid to query with a use of this instruction!");
+    return arg_begin() <= U && U < arg_end();
+  }
+  bool isArgOperand(Value::const_user_iterator UI) const {
+    return isArgOperand(&UI.getUse());
+  }
+
+  /// Returns true if this CallSite passes the given Value* as an argument to
+  /// the called function.
+  bool hasArgument(const Value *V) const {
+    return llvm::any_of(args(), [V](const Value *Arg) { return Arg == V; });
+  }
 
   Value *getCalledOperand() const { return Op<CalledOperandOpEndIdx>(); }
 
@@ -1113,13 +1202,45 @@ public:
     return dyn_cast_or_null<Function>(getCalledOperand());
   }
 
+  /// Return true if the callsite is an indirect call.
+  bool isIndirectCall() const;
+
+  /// Determine whether the passed iterator points to the callee operand's Use.
+  bool isCallee(Value::const_user_iterator UI) const {
+    return isCallee(&UI.getUse());
+  }
+
+  /// Determine whether this Use is the callee operand's Use.
+  bool isCallee(const Use *U) const { return &getCalledOperandUse() == U; }
+
+  /// Helper to get the caller (the parent function).
+  Function *getCaller();
+  const Function *getCaller() const {
+    return const_cast<CallBase *>(this)->getCaller();
+  }
+
+  /// Tests if this call site must be tail call optimized. Only a CallInst can
+  /// be tail call optimized.
+  bool isMustTailCall() const;
+
+  /// Tests if this call site is marked as a tail call.
+  bool isTailCall() const;
+
+  /// Returns the intrinsic ID of the intrinsic called or
+  /// Intrinsic::not_intrinsic if the called function is not an intrinsic, or if
+  /// this is an indirect call.
+  Intrinsic::ID getIntrinsicID() const;
+
   void setCalledOperand(Value *V) { Op<CalledOperandOpEndIdx>() = V; }
 
   /// Sets the function called, including updating the function type.
-  void setCalledFunction(Value *Fn) {
-    setCalledFunction(
-        cast<FunctionType>(cast<PointerType>(Fn->getType())->getElementType()),
-        Fn);
+  void setCalledFunction(Function *Fn) {
+    setCalledFunction(Fn->getFunctionType(), Fn);
+  }
+
+  /// Sets the function called, including updating the function type.
+  void setCalledFunction(FunctionCallee Fn) {
+    setCalledFunction(Fn.getFunctionType(), Fn.getCallee());
   }
 
   /// Sets the function called, including updating to the specified function
@@ -1128,6 +1249,9 @@ public:
     this->FTy = FTy;
     assert(FTy == cast<FunctionType>(
                       cast<PointerType>(Fn->getType())->getElementType()));
+    // This function doesn't mutate the return type, only the function
+    // type. Seems broken, but I'm just gonna stick an assert in for now.
+    assert(getType() == FTy->getReturnType());
     setCalledOperand(Fn);
   }
 
@@ -1142,6 +1266,9 @@ public:
                                (ID << 2));
   }
 
+  /// Check if this call is an inline asm statement.
+  bool isInlineAsm() const { return isa<InlineAsm>(getCalledOperand()); }
+
   /// \name Attribute API
   ///
   /// These methods access and modify attributes on this call (including
@@ -1304,6 +1431,55 @@ public:
     return bundleOperandHasAttr(i - 1, Kind);
   }
 
+  /// Determine whether this data operand is not captured.
+  // FIXME: Once this API is no longer duplicated in `CallSite`, rename this to
+  // better indicate that this may return a conservative answer.
+  bool doesNotCapture(unsigned OpNo) const {
+    return dataOperandHasImpliedAttr(OpNo + 1, Attribute::NoCapture);
+  }
+
+  /// Determine whether this argument is passed by value.
+  bool isByValArgument(unsigned ArgNo) const {
+    return paramHasAttr(ArgNo, Attribute::ByVal);
+  }
+
+  /// Determine whether this argument is passed in an alloca.
+  bool isInAllocaArgument(unsigned ArgNo) const {
+    return paramHasAttr(ArgNo, Attribute::InAlloca);
+  }
+
+  /// Determine whether this argument is passed by value or in an alloca.
+  bool isByValOrInAllocaArgument(unsigned ArgNo) const {
+    return paramHasAttr(ArgNo, Attribute::ByVal) ||
+           paramHasAttr(ArgNo, Attribute::InAlloca);
+  }
+
+  /// Determine if there are is an inalloca argument. Only the last argument can
+  /// have the inalloca attribute.
+  bool hasInAllocaArgument() const {
+    return !arg_empty() && paramHasAttr(arg_size() - 1, Attribute::InAlloca);
+  }
+
+  // FIXME: Once this API is no longer duplicated in `CallSite`, rename this to
+  // better indicate that this may return a conservative answer.
+  bool doesNotAccessMemory(unsigned OpNo) const {
+    return dataOperandHasImpliedAttr(OpNo + 1, Attribute::ReadNone);
+  }
+
+  // FIXME: Once this API is no longer duplicated in `CallSite`, rename this to
+  // better indicate that this may return a conservative answer.
+  bool onlyReadsMemory(unsigned OpNo) const {
+    return dataOperandHasImpliedAttr(OpNo + 1, Attribute::ReadOnly) ||
+           dataOperandHasImpliedAttr(OpNo + 1, Attribute::ReadNone);
+  }
+
+  // FIXME: Once this API is no longer duplicated in `CallSite`, rename this to
+  // better indicate that this may return a conservative answer.
+  bool doesNotReadMemory(unsigned OpNo) const {
+    return dataOperandHasImpliedAttr(OpNo + 1, Attribute::WriteOnly) ||
+           dataOperandHasImpliedAttr(OpNo + 1, Attribute::ReadNone);
+  }
+
   /// Extract the alignment of the return value.
   unsigned getRetAlignment() const { return Attrs.getRetAlignment(); }
 
@@ -1324,6 +1500,11 @@ public:
     return Attrs.getDereferenceableOrNullBytes(i);
   }
 
+  /// Return true if the return value is known to be not null.
+  /// This may be because it has the nonnull attribute, or because at least
+  /// one byte is dereferenceable and the pointer is in addrspace(0).
+  bool isReturnNonNull() const;
+
   /// Determine if the return value is marked with NoAlias attribute.
   bool returnDoesNotAlias() const {
     return Attrs.hasAttribute(AttributeList::ReturnIndex, Attribute::NoAlias);
@@ -1477,6 +1658,16 @@ public:
            Idx < getBundleOperandsEndIndex();
   }
 
+  /// Returns true if the use is a bundle operand.
+  bool isBundleOperand(const Use *U) const {
+    assert(this == U->getUser() &&
+           "Only valid to query with a use of this instruction!");
+    return hasOperandBundles() && isBundleOperand(U - op_begin());
+  }
+  bool isBundleOperand(Value::const_user_iterator UI) const {
+    return isBundleOperand(&UI.getUse());
+  }
+
   /// Return the total number operands (not operand bundles) used by
   /// every operand bundle in this OperandBundleUser.
   unsigned getNumTotalBundleOperands() const {
diff --git a/include/llvm/IR/Instruction.def b/include/llvm/IR/Instruction.def
index 58e4e2e1d6cc50b1564438e2d58c2642fcf073f9..41cdf613ad64d0e74b5db9b330d13e72dc551c25 100644
--- a/include/llvm/IR/Instruction.def
+++ b/include/llvm/IR/Instruction.def
@@ -1,9 +1,8 @@
 //===-- llvm/Instruction.def - File that describes Instructions -*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -135,89 +134,90 @@ HANDLE_TERM_INST  ( 7, Unreachable   , UnreachableInst)
 HANDLE_TERM_INST  ( 8, CleanupRet    , CleanupReturnInst)
 HANDLE_TERM_INST  ( 9, CatchRet      , CatchReturnInst)
 HANDLE_TERM_INST  (10, CatchSwitch   , CatchSwitchInst)
-  LAST_TERM_INST  (10)
+HANDLE_TERM_INST  (11, CallBr        , CallBrInst) // A call-site terminator
+  LAST_TERM_INST  (11)
 
 // Standard unary operators...
- FIRST_UNARY_INST(11)
-HANDLE_UNARY_INST(11, FNeg  , UnaryOperator)
-  LAST_UNARY_INST(11)
+ FIRST_UNARY_INST(12)
+HANDLE_UNARY_INST(12, FNeg  , UnaryOperator)
+  LAST_UNARY_INST(12)
 
 // Standard binary operators...
- FIRST_BINARY_INST(12)
-HANDLE_BINARY_INST(12, Add  , BinaryOperator)
-HANDLE_BINARY_INST(13, FAdd , BinaryOperator)
-HANDLE_BINARY_INST(14, Sub  , BinaryOperator)
-HANDLE_BINARY_INST(15, FSub , BinaryOperator)
-HANDLE_BINARY_INST(16, Mul  , BinaryOperator)
-HANDLE_BINARY_INST(17, FMul , BinaryOperator)
-HANDLE_BINARY_INST(18, UDiv , BinaryOperator)
-HANDLE_BINARY_INST(19, SDiv , BinaryOperator)
-HANDLE_BINARY_INST(20, FDiv , BinaryOperator)
-HANDLE_BINARY_INST(21, URem , BinaryOperator)
-HANDLE_BINARY_INST(22, SRem , BinaryOperator)
-HANDLE_BINARY_INST(23, FRem , BinaryOperator)
+ FIRST_BINARY_INST(13)
+HANDLE_BINARY_INST(13, Add  , BinaryOperator)
+HANDLE_BINARY_INST(14, FAdd , BinaryOperator)
+HANDLE_BINARY_INST(15, Sub  , BinaryOperator)
+HANDLE_BINARY_INST(16, FSub , BinaryOperator)
+HANDLE_BINARY_INST(17, Mul  , BinaryOperator)
+HANDLE_BINARY_INST(18, FMul , BinaryOperator)
+HANDLE_BINARY_INST(19, UDiv , BinaryOperator)
+HANDLE_BINARY_INST(20, SDiv , BinaryOperator)
+HANDLE_BINARY_INST(21, FDiv , BinaryOperator)
+HANDLE_BINARY_INST(22, URem , BinaryOperator)
+HANDLE_BINARY_INST(23, SRem , BinaryOperator)
+HANDLE_BINARY_INST(24, FRem , BinaryOperator)
 
 // Logical operators (integer operands)
-HANDLE_BINARY_INST(24, Shl  , BinaryOperator) // Shift left  (logical)
-HANDLE_BINARY_INST(25, LShr , BinaryOperator) // Shift right (logical)
-HANDLE_BINARY_INST(26, AShr , BinaryOperator) // Shift right (arithmetic)
-HANDLE_BINARY_INST(27, And  , BinaryOperator)
-HANDLE_BINARY_INST(28, Or   , BinaryOperator)
-HANDLE_BINARY_INST(29, Xor  , BinaryOperator)
-  LAST_BINARY_INST(29)
+HANDLE_BINARY_INST(25, Shl  , BinaryOperator) // Shift left  (logical)
+HANDLE_BINARY_INST(26, LShr , BinaryOperator) // Shift right (logical)
+HANDLE_BINARY_INST(27, AShr , BinaryOperator) // Shift right (arithmetic)
+HANDLE_BINARY_INST(28, And  , BinaryOperator)
+HANDLE_BINARY_INST(29, Or   , BinaryOperator)
+HANDLE_BINARY_INST(30, Xor  , BinaryOperator)
+  LAST_BINARY_INST(30)
 
 // Memory operators...
- FIRST_MEMORY_INST(30)
-HANDLE_MEMORY_INST(30, Alloca, AllocaInst)  // Stack management
-HANDLE_MEMORY_INST(31, Load  , LoadInst  )  // Memory manipulation instrs
-HANDLE_MEMORY_INST(32, Store , StoreInst )
-HANDLE_MEMORY_INST(33, GetElementPtr, GetElementPtrInst)
-HANDLE_MEMORY_INST(34, Fence , FenceInst )
-HANDLE_MEMORY_INST(35, AtomicCmpXchg , AtomicCmpXchgInst )
-HANDLE_MEMORY_INST(36, AtomicRMW , AtomicRMWInst )
-  LAST_MEMORY_INST(36)
+ FIRST_MEMORY_INST(31)
+HANDLE_MEMORY_INST(31, Alloca, AllocaInst)  // Stack management
+HANDLE_MEMORY_INST(32, Load  , LoadInst  )  // Memory manipulation instrs
+HANDLE_MEMORY_INST(33, Store , StoreInst )
+HANDLE_MEMORY_INST(34, GetElementPtr, GetElementPtrInst)
+HANDLE_MEMORY_INST(35, Fence , FenceInst )
+HANDLE_MEMORY_INST(36, AtomicCmpXchg , AtomicCmpXchgInst )
+HANDLE_MEMORY_INST(37, AtomicRMW , AtomicRMWInst )
+  LAST_MEMORY_INST(37)
 
 // Cast operators ...
 // NOTE: The order matters here because CastInst::isEliminableCastPair
 // NOTE: (see Instructions.cpp) encodes a table based on this ordering.
- FIRST_CAST_INST(37)
-HANDLE_CAST_INST(37, Trunc   , TruncInst   )  // Truncate integers
-HANDLE_CAST_INST(38, ZExt    , ZExtInst    )  // Zero extend integers
-HANDLE_CAST_INST(39, SExt    , SExtInst    )  // Sign extend integers
-HANDLE_CAST_INST(40, FPToUI  , FPToUIInst  )  // floating point -> UInt
-HANDLE_CAST_INST(41, FPToSI  , FPToSIInst  )  // floating point -> SInt
-HANDLE_CAST_INST(42, UIToFP  , UIToFPInst  )  // UInt -> floating point
-HANDLE_CAST_INST(43, SIToFP  , SIToFPInst  )  // SInt -> floating point
-HANDLE_CAST_INST(44, FPTrunc , FPTruncInst )  // Truncate floating point
-HANDLE_CAST_INST(45, FPExt   , FPExtInst   )  // Extend floating point
-HANDLE_CAST_INST(46, PtrToInt, PtrToIntInst)  // Pointer -> Integer
-HANDLE_CAST_INST(47, IntToPtr, IntToPtrInst)  // Integer -> Pointer
-HANDLE_CAST_INST(48, BitCast , BitCastInst )  // Type cast
-HANDLE_CAST_INST(49, AddrSpaceCast, AddrSpaceCastInst)  // addrspace cast
-  LAST_CAST_INST(49)
-
- FIRST_FUNCLETPAD_INST(50)
-HANDLE_FUNCLETPAD_INST(50, CleanupPad, CleanupPadInst)
-HANDLE_FUNCLETPAD_INST(51, CatchPad  , CatchPadInst)
-  LAST_FUNCLETPAD_INST(51)
+ FIRST_CAST_INST(38)
+HANDLE_CAST_INST(38, Trunc   , TruncInst   )  // Truncate integers
+HANDLE_CAST_INST(39, ZExt    , ZExtInst    )  // Zero extend integers
+HANDLE_CAST_INST(40, SExt    , SExtInst    )  // Sign extend integers
+HANDLE_CAST_INST(41, FPToUI  , FPToUIInst  )  // floating point -> UInt
+HANDLE_CAST_INST(42, FPToSI  , FPToSIInst  )  // floating point -> SInt
+HANDLE_CAST_INST(43, UIToFP  , UIToFPInst  )  // UInt -> floating point
+HANDLE_CAST_INST(44, SIToFP  , SIToFPInst  )  // SInt -> floating point
+HANDLE_CAST_INST(45, FPTrunc , FPTruncInst )  // Truncate floating point
+HANDLE_CAST_INST(46, FPExt   , FPExtInst   )  // Extend floating point
+HANDLE_CAST_INST(47, PtrToInt, PtrToIntInst)  // Pointer -> Integer
+HANDLE_CAST_INST(48, IntToPtr, IntToPtrInst)  // Integer -> Pointer
+HANDLE_CAST_INST(49, BitCast , BitCastInst )  // Type cast
+HANDLE_CAST_INST(50, AddrSpaceCast, AddrSpaceCastInst)  // addrspace cast
+  LAST_CAST_INST(50)
+
+ FIRST_FUNCLETPAD_INST(51)
+HANDLE_FUNCLETPAD_INST(51, CleanupPad, CleanupPadInst)
+HANDLE_FUNCLETPAD_INST(52, CatchPad  , CatchPadInst)
+  LAST_FUNCLETPAD_INST(52)
 
 // Other operators...
- FIRST_OTHER_INST(52)
-HANDLE_OTHER_INST(52, ICmp   , ICmpInst   )  // Integer comparison instruction
-HANDLE_OTHER_INST(53, FCmp   , FCmpInst   )  // Floating point comparison instr.
-HANDLE_OTHER_INST(54, PHI    , PHINode    )  // PHI node instruction
-HANDLE_OTHER_INST(55, Call   , CallInst   )  // Call a function
-HANDLE_OTHER_INST(56, Select , SelectInst )  // select instruction
-HANDLE_USER_INST (57, UserOp1, Instruction)  // May be used internally in a pass
-HANDLE_USER_INST (58, UserOp2, Instruction)  // Internal to passes only
-HANDLE_OTHER_INST(59, VAArg  , VAArgInst  )  // vaarg instruction
-HANDLE_OTHER_INST(60, ExtractElement, ExtractElementInst)// extract from vector
-HANDLE_OTHER_INST(61, InsertElement, InsertElementInst)  // insert into vector
-HANDLE_OTHER_INST(62, ShuffleVector, ShuffleVectorInst)  // shuffle two vectors.
-HANDLE_OTHER_INST(63, ExtractValue, ExtractValueInst)// extract from aggregate
-HANDLE_OTHER_INST(64, InsertValue, InsertValueInst)  // insert into aggregate
-HANDLE_OTHER_INST(65, LandingPad, LandingPadInst)  // Landing pad instruction.
-  LAST_OTHER_INST(65)
+ FIRST_OTHER_INST(53)
+HANDLE_OTHER_INST(53, ICmp   , ICmpInst   )  // Integer comparison instruction
+HANDLE_OTHER_INST(54, FCmp   , FCmpInst   )  // Floating point comparison instr.
+HANDLE_OTHER_INST(55, PHI    , PHINode    )  // PHI node instruction
+HANDLE_OTHER_INST(56, Call   , CallInst   )  // Call a function
+HANDLE_OTHER_INST(57, Select , SelectInst )  // select instruction
+HANDLE_USER_INST (58, UserOp1, Instruction)  // May be used internally in a pass
+HANDLE_USER_INST (59, UserOp2, Instruction)  // Internal to passes only
+HANDLE_OTHER_INST(60, VAArg  , VAArgInst  )  // vaarg instruction
+HANDLE_OTHER_INST(61, ExtractElement, ExtractElementInst)// extract from vector
+HANDLE_OTHER_INST(62, InsertElement, InsertElementInst)  // insert into vector
+HANDLE_OTHER_INST(63, ShuffleVector, ShuffleVectorInst)  // shuffle two vectors.
+HANDLE_OTHER_INST(64, ExtractValue, ExtractValueInst)// extract from aggregate
+HANDLE_OTHER_INST(65, InsertValue, InsertValueInst)  // insert into aggregate
+HANDLE_OTHER_INST(66, LandingPad, LandingPadInst)  // Landing pad instruction.
+  LAST_OTHER_INST(66)
 
 #undef  FIRST_TERM_INST
 #undef HANDLE_TERM_INST
diff --git a/include/llvm/IR/Instruction.h b/include/llvm/IR/Instruction.h
index 7a81f700ee4ecccd6ca45652776c421a3295b287..b940d125a277af0bcf5e1d00af86674acd7d262f 100644
--- a/include/llvm/IR/Instruction.h
+++ b/include/llvm/IR/Instruction.h
@@ -1,9 +1,8 @@
 //===-- llvm/Instruction.h - Instruction class definition -------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -136,6 +135,9 @@ public:
   bool isExceptionalTerminator() const {
     return isExceptionalTerminator(getOpcode());
   }
+  bool isIndirectTerminator() const {
+    return isIndirectTerminator(getOpcode());
+  }
 
   static const char* getOpcodeName(unsigned OpCode);
 
@@ -203,6 +205,17 @@ public:
     }
   }
 
+  /// Returns true if the OpCode is a terminator with indirect targets.
+  static inline bool isIndirectTerminator(unsigned OpCode) {
+    switch (OpCode) {
+    case Instruction::IndirectBr:
+    case Instruction::CallBr:
+      return true;
+    default:
+      return false;
+    }
+  }
+
   //===--------------------------------------------------------------------===//
   // Metadata manipulation.
   //===--------------------------------------------------------------------===//
@@ -582,6 +595,10 @@ public:
     }
   }
 
+  /// Return true if the instruction is a llvm.lifetime.start or
+  /// llvm.lifetime.end marker.
+  bool isLifetimeStartOrEnd() const;
+
   /// Return a pointer to the next non-debug instruction in the same basic
   /// block as 'this', or nullptr if no such instruction exists.
   const Instruction *getNextNonDebugInstruction() const;
diff --git a/include/llvm/IR/Instructions.h b/include/llvm/IR/Instructions.h
index 7700e7caa9cd5572b8164582cf0f9ddcac30bc93..a82cedab25d4bddda0a4c112f07728979d702328 100644
--- a/include/llvm/IR/Instructions.h
+++ b/include/llvm/IR/Instructions.h
@@ -1,9 +1,8 @@
 //===- llvm/Instructions.h - Instruction subclass definitions ---*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -175,47 +174,58 @@ protected:
   LoadInst *cloneImpl() const;
 
 public:
-  LoadInst(Value *Ptr, const Twine &NameStr, Instruction *InsertBefore);
-  LoadInst(Value *Ptr, const Twine &NameStr, BasicBlock *InsertAtEnd);
-  LoadInst(Type *Ty, Value *Ptr, const Twine &NameStr, bool isVolatile = false,
+  LoadInst(Type *Ty, Value *Ptr, const Twine &NameStr = "",
            Instruction *InsertBefore = nullptr);
-  LoadInst(Value *Ptr, const Twine &NameStr, bool isVolatile = false,
-           Instruction *InsertBefore = nullptr)
-      : LoadInst(cast<PointerType>(Ptr->getType())->getElementType(), Ptr,
-                 NameStr, isVolatile, InsertBefore) {}
-  LoadInst(Value *Ptr, const Twine &NameStr, bool isVolatile,
+  LoadInst(Type *Ty, Value *Ptr, const Twine &NameStr, BasicBlock *InsertAtEnd);
+  LoadInst(Type *Ty, Value *Ptr, const Twine &NameStr, bool isVolatile,
+           Instruction *InsertBefore = nullptr);
+  LoadInst(Type *Ty, Value *Ptr, const Twine &NameStr, bool isVolatile,
            BasicBlock *InsertAtEnd);
-  LoadInst(Value *Ptr, const Twine &NameStr, bool isVolatile, unsigned Align,
-           Instruction *InsertBefore = nullptr)
-      : LoadInst(cast<PointerType>(Ptr->getType())->getElementType(), Ptr,
-                 NameStr, isVolatile, Align, InsertBefore) {}
   LoadInst(Type *Ty, Value *Ptr, const Twine &NameStr, bool isVolatile,
            unsigned Align, Instruction *InsertBefore = nullptr);
-  LoadInst(Value *Ptr, const Twine &NameStr, bool isVolatile,
+  LoadInst(Type *Ty, Value *Ptr, const Twine &NameStr, bool isVolatile,
            unsigned Align, BasicBlock *InsertAtEnd);
-  LoadInst(Value *Ptr, const Twine &NameStr, bool isVolatile, unsigned Align,
-           AtomicOrdering Order, SyncScope::ID SSID = SyncScope::System,
-           Instruction *InsertBefore = nullptr)
-      : LoadInst(cast<PointerType>(Ptr->getType())->getElementType(), Ptr,
-                 NameStr, isVolatile, Align, Order, SSID, InsertBefore) {}
   LoadInst(Type *Ty, Value *Ptr, const Twine &NameStr, bool isVolatile,
            unsigned Align, AtomicOrdering Order,
            SyncScope::ID SSID = SyncScope::System,
            Instruction *InsertBefore = nullptr);
-  LoadInst(Value *Ptr, const Twine &NameStr, bool isVolatile,
+  LoadInst(Type *Ty, Value *Ptr, const Twine &NameStr, bool isVolatile,
            unsigned Align, AtomicOrdering Order, SyncScope::ID SSID,
            BasicBlock *InsertAtEnd);
-  LoadInst(Value *Ptr, const char *NameStr, Instruction *InsertBefore);
-  LoadInst(Value *Ptr, const char *NameStr, BasicBlock *InsertAtEnd);
-  LoadInst(Type *Ty, Value *Ptr, const char *NameStr = nullptr,
-           bool isVolatile = false, Instruction *InsertBefore = nullptr);
-  explicit LoadInst(Value *Ptr, const char *NameStr = nullptr,
-                    bool isVolatile = false,
+
+  // Deprecated [opaque pointer types]
+  explicit LoadInst(Value *Ptr, const Twine &NameStr = "",
                     Instruction *InsertBefore = nullptr)
-      : LoadInst(cast<PointerType>(Ptr->getType())->getElementType(), Ptr,
-                 NameStr, isVolatile, InsertBefore) {}
-  LoadInst(Value *Ptr, const char *NameStr, bool isVolatile,
-           BasicBlock *InsertAtEnd);
+      : LoadInst(Ptr->getType()->getPointerElementType(), Ptr, NameStr,
+                 InsertBefore) {}
+  LoadInst(Value *Ptr, const Twine &NameStr, BasicBlock *InsertAtEnd)
+      : LoadInst(Ptr->getType()->getPointerElementType(), Ptr, NameStr,
+                 InsertAtEnd) {}
+  LoadInst(Value *Ptr, const Twine &NameStr, bool isVolatile,
+           Instruction *InsertBefore = nullptr)
+      : LoadInst(Ptr->getType()->getPointerElementType(), Ptr, NameStr,
+                 isVolatile, InsertBefore) {}
+  LoadInst(Value *Ptr, const Twine &NameStr, bool isVolatile,
+           BasicBlock *InsertAtEnd)
+      : LoadInst(Ptr->getType()->getPointerElementType(), Ptr, NameStr,
+                 isVolatile, InsertAtEnd) {}
+  LoadInst(Value *Ptr, const Twine &NameStr, bool isVolatile, unsigned Align,
+           Instruction *InsertBefore = nullptr)
+      : LoadInst(Ptr->getType()->getPointerElementType(), Ptr, NameStr,
+                 isVolatile, Align, InsertBefore) {}
+  LoadInst(Value *Ptr, const Twine &NameStr, bool isVolatile, unsigned Align,
+           BasicBlock *InsertAtEnd)
+      : LoadInst(Ptr->getType()->getPointerElementType(), Ptr, NameStr,
+                 isVolatile, Align, InsertAtEnd) {}
+  LoadInst(Value *Ptr, const Twine &NameStr, bool isVolatile, unsigned Align,
+           AtomicOrdering Order, SyncScope::ID SSID = SyncScope::System,
+           Instruction *InsertBefore = nullptr)
+      : LoadInst(Ptr->getType()->getPointerElementType(), Ptr, NameStr,
+                 isVolatile, Align, Order, SSID, InsertBefore) {}
+  LoadInst(Value *Ptr, const Twine &NameStr, bool isVolatile, unsigned Align,
+           AtomicOrdering Order, SyncScope::ID SSID, BasicBlock *InsertAtEnd)
+      : LoadInst(Ptr->getType()->getPointerElementType(), Ptr, NameStr,
+                 isVolatile, Align, Order, SSID, InsertAtEnd) {}
 
   /// Return true if this is a load from a volatile memory location.
   bool isVolatile() const { return getSubclassDataFromInstruction() & 1; }
@@ -714,8 +724,14 @@ public:
     /// *p = old <unsigned v ? old : v
     UMin,
 
+    /// *p = old + v
+    FAdd,
+
+    /// *p = old - v
+    FSub,
+
     FIRST_BINOP = Xchg,
-    LAST_BINOP = UMin,
+    LAST_BINOP = FSub,
     BAD_BINOP
   };
 
@@ -737,6 +753,16 @@ public:
 
   static StringRef getOperationName(BinOp Op);
 
+  static bool isFPOperation(BinOp Op) {
+    switch (Op) {
+    case AtomicRMWInst::FAdd:
+    case AtomicRMWInst::FSub:
+      return true;
+    default:
+      return false;
+    }
+  }
+
   void setOperation(BinOp Operation) {
     unsigned short SubclassData = getSubclassDataFromInstruction();
     setInstructionSubclassData((SubclassData & 31) |
@@ -794,6 +820,10 @@ public:
     return getPointerOperand()->getType()->getPointerAddressSpace();
   }
 
+  bool isFloatingPointOperation() const {
+    return isFPOperation(getOperation());
+  }
+
   // Methods for support type inquiry through isa, cast, and dyn_cast:
   static bool classof(const Instruction *I) {
     return I->getOpcode() == Instruction::AtomicRMW;
@@ -1433,36 +1463,25 @@ class CallInst : public CallBase {
                   ArrayRef<OperandBundleDef> Bundles, const Twine &NameStr,
                   Instruction *InsertBefore);
 
-  inline CallInst(Value *Func, ArrayRef<Value *> Args,
-                  ArrayRef<OperandBundleDef> Bundles, const Twine &NameStr,
-                  Instruction *InsertBefore)
-      : CallInst(cast<FunctionType>(
-                     cast<PointerType>(Func->getType())->getElementType()),
-                 Func, Args, Bundles, NameStr, InsertBefore) {}
-
-  inline CallInst(Value *Func, ArrayRef<Value *> Args, const Twine &NameStr,
-                  Instruction *InsertBefore)
-      : CallInst(Func, Args, None, NameStr, InsertBefore) {}
+  inline CallInst(FunctionType *Ty, Value *Func, ArrayRef<Value *> Args,
+                  const Twine &NameStr, Instruction *InsertBefore)
+      : CallInst(Ty, Func, Args, None, NameStr, InsertBefore) {}
 
   /// Construct a CallInst given a range of arguments.
   /// Construct a CallInst from a range of arguments
-  inline CallInst(Value *Func, ArrayRef<Value *> Args,
+  inline CallInst(FunctionType *Ty, Value *Func, ArrayRef<Value *> Args,
                   ArrayRef<OperandBundleDef> Bundles, const Twine &NameStr,
                   BasicBlock *InsertAtEnd);
 
-  explicit CallInst(Value *F, const Twine &NameStr, Instruction *InsertBefore);
+  explicit CallInst(FunctionType *Ty, Value *F, const Twine &NameStr,
+                    Instruction *InsertBefore);
 
-  CallInst(Value *F, const Twine &NameStr, BasicBlock *InsertAtEnd);
+  CallInst(FunctionType *ty, Value *F, const Twine &NameStr,
+           BasicBlock *InsertAtEnd);
 
-  void init(Value *Func, ArrayRef<Value *> Args,
-            ArrayRef<OperandBundleDef> Bundles, const Twine &NameStr) {
-    init(cast<FunctionType>(
-             cast<PointerType>(Func->getType())->getElementType()),
-         Func, Args, Bundles, NameStr);
-  }
   void init(FunctionType *FTy, Value *Func, ArrayRef<Value *> Args,
             ArrayRef<OperandBundleDef> Bundles, const Twine &NameStr);
-  void init(Value *Func, const Twine &NameStr);
+  void init(FunctionType *FTy, Value *Func, const Twine &NameStr);
 
   /// Compute the number of operands to allocate.
   static int ComputeNumOperands(int NumArgs, int NumBundleInputs = 0) {
@@ -1478,21 +1497,9 @@ protected:
   CallInst *cloneImpl() const;
 
 public:
-  static CallInst *Create(Value *Func, ArrayRef<Value *> Args,
-                          ArrayRef<OperandBundleDef> Bundles = None,
-                          const Twine &NameStr = "",
+  static CallInst *Create(FunctionType *Ty, Value *F, const Twine &NameStr = "",
                           Instruction *InsertBefore = nullptr) {
-    return Create(cast<FunctionType>(
-                      cast<PointerType>(Func->getType())->getElementType()),
-                  Func, Args, Bundles, NameStr, InsertBefore);
-  }
-
-  static CallInst *Create(Value *Func, ArrayRef<Value *> Args,
-                          const Twine &NameStr,
-                          Instruction *InsertBefore = nullptr) {
-    return Create(cast<FunctionType>(
-                      cast<PointerType>(Func->getType())->getElementType()),
-                  Func, Args, None, NameStr, InsertBefore);
+    return new (ComputeNumOperands(0)) CallInst(Ty, F, NameStr, InsertBefore);
   }
 
   static CallInst *Create(FunctionType *Ty, Value *Func, ArrayRef<Value *> Args,
@@ -1514,7 +1521,18 @@ public:
         CallInst(Ty, Func, Args, Bundles, NameStr, InsertBefore);
   }
 
-  static CallInst *Create(Value *Func, ArrayRef<Value *> Args,
+  static CallInst *Create(FunctionType *Ty, Value *F, const Twine &NameStr,
+                          BasicBlock *InsertAtEnd) {
+    return new (ComputeNumOperands(0)) CallInst(Ty, F, NameStr, InsertAtEnd);
+  }
+
+  static CallInst *Create(FunctionType *Ty, Value *Func, ArrayRef<Value *> Args,
+                          const Twine &NameStr, BasicBlock *InsertAtEnd) {
+    return new (ComputeNumOperands(Args.size()))
+        CallInst(Ty, Func, Args, None, NameStr, InsertAtEnd);
+  }
+
+  static CallInst *Create(FunctionType *Ty, Value *Func, ArrayRef<Value *> Args,
                           ArrayRef<OperandBundleDef> Bundles,
                           const Twine &NameStr, BasicBlock *InsertAtEnd) {
     const int NumOperands =
@@ -1522,23 +1540,99 @@ public:
     const unsigned DescriptorBytes = Bundles.size() * sizeof(BundleOpInfo);
 
     return new (NumOperands, DescriptorBytes)
-        CallInst(Func, Args, Bundles, NameStr, InsertAtEnd);
+        CallInst(Ty, Func, Args, Bundles, NameStr, InsertAtEnd);
   }
 
-  static CallInst *Create(Value *Func, ArrayRef<Value *> Args,
+  static CallInst *Create(FunctionCallee Func, const Twine &NameStr = "",
+                          Instruction *InsertBefore = nullptr) {
+    return Create(Func.getFunctionType(), Func.getCallee(), NameStr,
+                  InsertBefore);
+  }
+
+  static CallInst *Create(FunctionCallee Func, ArrayRef<Value *> Args,
+                          ArrayRef<OperandBundleDef> Bundles = None,
+                          const Twine &NameStr = "",
+                          Instruction *InsertBefore = nullptr) {
+    return Create(Func.getFunctionType(), Func.getCallee(), Args, Bundles,
+                  NameStr, InsertBefore);
+  }
+
+  static CallInst *Create(FunctionCallee Func, ArrayRef<Value *> Args,
+                          const Twine &NameStr,
+                          Instruction *InsertBefore = nullptr) {
+    return Create(Func.getFunctionType(), Func.getCallee(), Args, NameStr,
+                  InsertBefore);
+  }
+
+  static CallInst *Create(FunctionCallee Func, const Twine &NameStr,
+                          BasicBlock *InsertAtEnd) {
+    return Create(Func.getFunctionType(), Func.getCallee(), NameStr,
+                  InsertAtEnd);
+  }
+
+  static CallInst *Create(FunctionCallee Func, ArrayRef<Value *> Args,
                           const Twine &NameStr, BasicBlock *InsertAtEnd) {
-    return new (ComputeNumOperands(Args.size()))
-        CallInst(Func, Args, None, NameStr, InsertAtEnd);
+    return Create(Func.getFunctionType(), Func.getCallee(), Args, NameStr,
+                  InsertAtEnd);
   }
 
-  static CallInst *Create(Value *F, const Twine &NameStr = "",
+  static CallInst *Create(FunctionCallee Func, ArrayRef<Value *> Args,
+                          ArrayRef<OperandBundleDef> Bundles,
+                          const Twine &NameStr, BasicBlock *InsertAtEnd) {
+    return Create(Func.getFunctionType(), Func.getCallee(), Args, Bundles,
+                  NameStr, InsertAtEnd);
+  }
+
+  // Deprecated [opaque pointer types]
+  static CallInst *Create(Value *Func, const Twine &NameStr = "",
+                          Instruction *InsertBefore = nullptr) {
+    return Create(cast<FunctionType>(
+                      cast<PointerType>(Func->getType())->getElementType()),
+                  Func, NameStr, InsertBefore);
+  }
+
+  // Deprecated [opaque pointer types]
+  static CallInst *Create(Value *Func, ArrayRef<Value *> Args,
+                          const Twine &NameStr,
+                          Instruction *InsertBefore = nullptr) {
+    return Create(cast<FunctionType>(
+                      cast<PointerType>(Func->getType())->getElementType()),
+                  Func, Args, NameStr, InsertBefore);
+  }
+
+  // Deprecated [opaque pointer types]
+  static CallInst *Create(Value *Func, ArrayRef<Value *> Args,
+                          ArrayRef<OperandBundleDef> Bundles = None,
+                          const Twine &NameStr = "",
                           Instruction *InsertBefore = nullptr) {
-    return new (ComputeNumOperands(0)) CallInst(F, NameStr, InsertBefore);
+    return Create(cast<FunctionType>(
+                      cast<PointerType>(Func->getType())->getElementType()),
+                  Func, Args, Bundles, NameStr, InsertBefore);
   }
 
-  static CallInst *Create(Value *F, const Twine &NameStr,
+  // Deprecated [opaque pointer types]
+  static CallInst *Create(Value *Func, const Twine &NameStr,
                           BasicBlock *InsertAtEnd) {
-    return new (ComputeNumOperands(0)) CallInst(F, NameStr, InsertAtEnd);
+    return Create(cast<FunctionType>(
+                      cast<PointerType>(Func->getType())->getElementType()),
+                  Func, NameStr, InsertAtEnd);
+  }
+
+  // Deprecated [opaque pointer types]
+  static CallInst *Create(Value *Func, ArrayRef<Value *> Args,
+                          const Twine &NameStr, BasicBlock *InsertAtEnd) {
+    return Create(cast<FunctionType>(
+                      cast<PointerType>(Func->getType())->getElementType()),
+                  Func, Args, NameStr, InsertAtEnd);
+  }
+
+  // Deprecated [opaque pointer types]
+  static CallInst *Create(Value *Func, ArrayRef<Value *> Args,
+                          ArrayRef<OperandBundleDef> Bundles,
+                          const Twine &NameStr, BasicBlock *InsertAtEnd) {
+    return Create(cast<FunctionType>(
+                      cast<PointerType>(Func->getType())->getElementType()),
+                  Func, Args, Bundles, NameStr, InsertAtEnd);
   }
 
   /// Create a clone of \p CI with a different set of operand bundles and
@@ -1628,9 +1722,6 @@ public:
     addAttribute(AttributeList::FunctionIndex, Attribute::ReturnsTwice);
   }
 
-  /// Check if this call is an inline asm statement.
-  bool isInlineAsm() const { return isa<InlineAsm>(getCalledOperand()); }
-
   // Methods for support type inquiry through isa, cast, and dyn_cast:
   static bool classof(const Instruction *I) {
     return I->getOpcode() == Instruction::Call;
@@ -1647,18 +1738,15 @@ private:
   }
 };
 
-CallInst::CallInst(Value *Func, ArrayRef<Value *> Args,
+CallInst::CallInst(FunctionType *Ty, Value *Func, ArrayRef<Value *> Args,
                    ArrayRef<OperandBundleDef> Bundles, const Twine &NameStr,
                    BasicBlock *InsertAtEnd)
-    : CallBase(cast<FunctionType>(
-                   cast<PointerType>(Func->getType())->getElementType())
-                   ->getReturnType(),
-               Instruction::Call,
+    : CallBase(Ty->getReturnType(), Instruction::Call,
                OperandTraits<CallBase>::op_end(this) -
                    (Args.size() + CountBundleInputs(Bundles) + 1),
                unsigned(Args.size() + CountBundleInputs(Bundles) + 1),
                InsertAtEnd) {
-  init(Func, Args, Bundles, NameStr);
+  init(Ty, Func, Args, Bundles, NameStr);
 }
 
 CallInst::CallInst(FunctionType *Ty, Value *Func, ArrayRef<Value *> Args,
@@ -3561,36 +3649,17 @@ class InvokeInst : public CallBase {
   /// Construct an InvokeInst given a range of arguments.
   ///
   /// Construct an InvokeInst from a range of arguments
-  inline InvokeInst(Value *Func, BasicBlock *IfNormal, BasicBlock *IfException,
-                    ArrayRef<Value *> Args, ArrayRef<OperandBundleDef> Bundles,
-                    int NumOperands, const Twine &NameStr,
-                    Instruction *InsertBefore)
-      : InvokeInst(cast<FunctionType>(
-                       cast<PointerType>(Func->getType())->getElementType()),
-                   Func, IfNormal, IfException, Args, Bundles, NumOperands,
-                   NameStr, InsertBefore) {}
-
   inline InvokeInst(FunctionType *Ty, Value *Func, BasicBlock *IfNormal,
                     BasicBlock *IfException, ArrayRef<Value *> Args,
                     ArrayRef<OperandBundleDef> Bundles, int NumOperands,
                     const Twine &NameStr, Instruction *InsertBefore);
-  /// Construct an InvokeInst given a range of arguments.
-  ///
-  /// Construct an InvokeInst from a range of arguments
-  inline InvokeInst(Value *Func, BasicBlock *IfNormal, BasicBlock *IfException,
-                    ArrayRef<Value *> Args, ArrayRef<OperandBundleDef> Bundles,
-                    int NumOperands, const Twine &NameStr,
-                    BasicBlock *InsertAtEnd);
 
-  void init(Value *Func, BasicBlock *IfNormal, BasicBlock *IfException,
-            ArrayRef<Value *> Args, ArrayRef<OperandBundleDef> Bundles,
-            const Twine &NameStr) {
-    init(cast<FunctionType>(
-             cast<PointerType>(Func->getType())->getElementType()),
-         Func, IfNormal, IfException, Args, Bundles, NameStr);
-  }
+  inline InvokeInst(FunctionType *Ty, Value *Func, BasicBlock *IfNormal,
+                    BasicBlock *IfException, ArrayRef<Value *> Args,
+                    ArrayRef<OperandBundleDef> Bundles, int NumOperands,
+                    const Twine &NameStr, BasicBlock *InsertAtEnd);
 
-  void init(FunctionType *FTy, Value *Func, BasicBlock *IfNormal,
+  void init(FunctionType *Ty, Value *Func, BasicBlock *IfNormal,
             BasicBlock *IfException, ArrayRef<Value *> Args,
             ArrayRef<OperandBundleDef> Bundles, const Twine &NameStr);
 
@@ -3608,27 +3677,6 @@ protected:
   InvokeInst *cloneImpl() const;
 
 public:
-  static InvokeInst *Create(Value *Func, BasicBlock *IfNormal,
-                            BasicBlock *IfException, ArrayRef<Value *> Args,
-                            const Twine &NameStr,
-                            Instruction *InsertBefore = nullptr) {
-    return Create(cast<FunctionType>(
-                      cast<PointerType>(Func->getType())->getElementType()),
-                  Func, IfNormal, IfException, Args, None, NameStr,
-                  InsertBefore);
-  }
-
-  static InvokeInst *Create(Value *Func, BasicBlock *IfNormal,
-                            BasicBlock *IfException, ArrayRef<Value *> Args,
-                            ArrayRef<OperandBundleDef> Bundles = None,
-                            const Twine &NameStr = "",
-                            Instruction *InsertBefore = nullptr) {
-    return Create(cast<FunctionType>(
-                      cast<PointerType>(Func->getType())->getElementType()),
-                  Func, IfNormal, IfException, Args, Bundles, NameStr,
-                  InsertBefore);
-  }
-
   static InvokeInst *Create(FunctionType *Ty, Value *Func, BasicBlock *IfNormal,
                             BasicBlock *IfException, ArrayRef<Value *> Args,
                             const Twine &NameStr,
@@ -3653,16 +3701,16 @@ public:
                    NameStr, InsertBefore);
   }
 
-  static InvokeInst *Create(Value *Func,
-                            BasicBlock *IfNormal, BasicBlock *IfException,
-                            ArrayRef<Value *> Args, const Twine &NameStr,
-                            BasicBlock *InsertAtEnd) {
+  static InvokeInst *Create(FunctionType *Ty, Value *Func, BasicBlock *IfNormal,
+                            BasicBlock *IfException, ArrayRef<Value *> Args,
+                            const Twine &NameStr, BasicBlock *InsertAtEnd) {
     int NumOperands = ComputeNumOperands(Args.size());
-    return new (NumOperands) InvokeInst(Func, IfNormal, IfException, Args, None,
-                                        NumOperands, NameStr, InsertAtEnd);
+    return new (NumOperands)
+        InvokeInst(Ty, Func, IfNormal, IfException, Args, None, NumOperands,
+                   NameStr, InsertAtEnd);
   }
 
-  static InvokeInst *Create(Value *Func, BasicBlock *IfNormal,
+  static InvokeInst *Create(FunctionType *Ty, Value *Func, BasicBlock *IfNormal,
                             BasicBlock *IfException, ArrayRef<Value *> Args,
                             ArrayRef<OperandBundleDef> Bundles,
                             const Twine &NameStr, BasicBlock *InsertAtEnd) {
@@ -3671,10 +3719,85 @@ public:
     unsigned DescriptorBytes = Bundles.size() * sizeof(BundleOpInfo);
 
     return new (NumOperands, DescriptorBytes)
-        InvokeInst(Func, IfNormal, IfException, Args, Bundles, NumOperands,
+        InvokeInst(Ty, Func, IfNormal, IfException, Args, Bundles, NumOperands,
                    NameStr, InsertAtEnd);
   }
 
+  static InvokeInst *Create(FunctionCallee Func, BasicBlock *IfNormal,
+                            BasicBlock *IfException, ArrayRef<Value *> Args,
+                            const Twine &NameStr,
+                            Instruction *InsertBefore = nullptr) {
+    return Create(Func.getFunctionType(), Func.getCallee(), IfNormal,
+                  IfException, Args, None, NameStr, InsertBefore);
+  }
+
+  static InvokeInst *Create(FunctionCallee Func, BasicBlock *IfNormal,
+                            BasicBlock *IfException, ArrayRef<Value *> Args,
+                            ArrayRef<OperandBundleDef> Bundles = None,
+                            const Twine &NameStr = "",
+                            Instruction *InsertBefore = nullptr) {
+    return Create(Func.getFunctionType(), Func.getCallee(), IfNormal,
+                  IfException, Args, Bundles, NameStr, InsertBefore);
+  }
+
+  static InvokeInst *Create(FunctionCallee Func, BasicBlock *IfNormal,
+                            BasicBlock *IfException, ArrayRef<Value *> Args,
+                            const Twine &NameStr, BasicBlock *InsertAtEnd) {
+    return Create(Func.getFunctionType(), Func.getCallee(), IfNormal,
+                  IfException, Args, NameStr, InsertAtEnd);
+  }
+
+  static InvokeInst *Create(FunctionCallee Func, BasicBlock *IfNormal,
+                            BasicBlock *IfException, ArrayRef<Value *> Args,
+                            ArrayRef<OperandBundleDef> Bundles,
+                            const Twine &NameStr, BasicBlock *InsertAtEnd) {
+    return Create(Func.getFunctionType(), Func.getCallee(), IfNormal,
+                  IfException, Args, Bundles, NameStr, InsertAtEnd);
+  }
+
+  // Deprecated [opaque pointer types]
+  static InvokeInst *Create(Value *Func, BasicBlock *IfNormal,
+                            BasicBlock *IfException, ArrayRef<Value *> Args,
+                            const Twine &NameStr,
+                            Instruction *InsertBefore = nullptr) {
+    return Create(cast<FunctionType>(
+                      cast<PointerType>(Func->getType())->getElementType()),
+                  Func, IfNormal, IfException, Args, None, NameStr,
+                  InsertBefore);
+  }
+
+  // Deprecated [opaque pointer types]
+  static InvokeInst *Create(Value *Func, BasicBlock *IfNormal,
+                            BasicBlock *IfException, ArrayRef<Value *> Args,
+                            ArrayRef<OperandBundleDef> Bundles = None,
+                            const Twine &NameStr = "",
+                            Instruction *InsertBefore = nullptr) {
+    return Create(cast<FunctionType>(
+                      cast<PointerType>(Func->getType())->getElementType()),
+                  Func, IfNormal, IfException, Args, Bundles, NameStr,
+                  InsertBefore);
+  }
+
+  // Deprecated [opaque pointer types]
+  static InvokeInst *Create(Value *Func, BasicBlock *IfNormal,
+                            BasicBlock *IfException, ArrayRef<Value *> Args,
+                            const Twine &NameStr, BasicBlock *InsertAtEnd) {
+    return Create(cast<FunctionType>(
+                      cast<PointerType>(Func->getType())->getElementType()),
+                  Func, IfNormal, IfException, Args, NameStr, InsertAtEnd);
+  }
+
+  // Deprecated [opaque pointer types]
+  static InvokeInst *Create(Value *Func, BasicBlock *IfNormal,
+                            BasicBlock *IfException, ArrayRef<Value *> Args,
+                            ArrayRef<OperandBundleDef> Bundles,
+                            const Twine &NameStr, BasicBlock *InsertAtEnd) {
+    return Create(cast<FunctionType>(
+                      cast<PointerType>(Func->getType())->getElementType()),
+                  Func, IfNormal, IfException, Args, Bundles, NameStr,
+                  InsertAtEnd);
+  }
+
   /// Create a clone of \p II with a different set of operand bundles and
   /// insert it before \p InsertPt.
   ///
@@ -3753,17 +3876,257 @@ InvokeInst::InvokeInst(FunctionType *Ty, Value *Func, BasicBlock *IfNormal,
   init(Ty, Func, IfNormal, IfException, Args, Bundles, NameStr);
 }
 
-InvokeInst::InvokeInst(Value *Func, BasicBlock *IfNormal,
+InvokeInst::InvokeInst(FunctionType *Ty, Value *Func, BasicBlock *IfNormal,
                        BasicBlock *IfException, ArrayRef<Value *> Args,
                        ArrayRef<OperandBundleDef> Bundles, int NumOperands,
                        const Twine &NameStr, BasicBlock *InsertAtEnd)
-    : CallBase(cast<FunctionType>(
-                   cast<PointerType>(Func->getType())->getElementType())
-                   ->getReturnType(),
-               Instruction::Invoke,
+    : CallBase(Ty->getReturnType(), Instruction::Invoke,
                OperandTraits<CallBase>::op_end(this) - NumOperands, NumOperands,
                InsertAtEnd) {
-  init(Func, IfNormal, IfException, Args, Bundles, NameStr);
+  init(Ty, Func, IfNormal, IfException, Args, Bundles, NameStr);
+}
+
+//===----------------------------------------------------------------------===//
+//                              CallBrInst Class
+//===----------------------------------------------------------------------===//
+
+/// CallBr instruction, tracking function calls that may not return control but
+/// instead transfer it to a third location. The SubclassData field is used to
+/// hold the calling convention of the call.
+///
+class CallBrInst : public CallBase {
+
+  unsigned NumIndirectDests;
+
+  CallBrInst(const CallBrInst &BI);
+
+  /// Construct a CallBrInst given a range of arguments.
+  ///
+  /// Construct a CallBrInst from a range of arguments
+  inline CallBrInst(FunctionType *Ty, Value *Func, BasicBlock *DefaultDest,
+                    ArrayRef<BasicBlock *> IndirectDests,
+                    ArrayRef<Value *> Args,
+                    ArrayRef<OperandBundleDef> Bundles, int NumOperands,
+                    const Twine &NameStr, Instruction *InsertBefore);
+
+  inline CallBrInst(FunctionType *Ty, Value *Func, BasicBlock *DefaultDest,
+                    ArrayRef<BasicBlock *> IndirectDests,
+                    ArrayRef<Value *> Args,
+                    ArrayRef<OperandBundleDef> Bundles, int NumOperands,
+                    const Twine &NameStr, BasicBlock *InsertAtEnd);
+
+  void init(FunctionType *FTy, Value *Func, BasicBlock *DefaultDest,
+            ArrayRef<BasicBlock *> IndirectDests, ArrayRef<Value *> Args,
+            ArrayRef<OperandBundleDef> Bundles, const Twine &NameStr);
+
+  /// Compute the number of operands to allocate.
+  static int ComputeNumOperands(int NumArgs, int NumIndirectDests,
+                                int NumBundleInputs = 0) {
+    // We need one operand for the called function, plus our extra operands and
+    // the input operand counts provided.
+    return 2 + NumIndirectDests + NumArgs + NumBundleInputs;
+  }
+
+protected:
+  // Note: Instruction needs to be a friend here to call cloneImpl.
+  friend class Instruction;
+
+  CallBrInst *cloneImpl() const;
+
+public:
+  static CallBrInst *Create(FunctionType *Ty, Value *Func,
+                            BasicBlock *DefaultDest,
+                            ArrayRef<BasicBlock *> IndirectDests,
+                            ArrayRef<Value *> Args, const Twine &NameStr,
+                            Instruction *InsertBefore = nullptr) {
+    int NumOperands = ComputeNumOperands(Args.size(), IndirectDests.size());
+    return new (NumOperands)
+        CallBrInst(Ty, Func, DefaultDest, IndirectDests, Args, None,
+                   NumOperands, NameStr, InsertBefore);
+  }
+
+  static CallBrInst *Create(FunctionType *Ty, Value *Func,
+                            BasicBlock *DefaultDest,
+                            ArrayRef<BasicBlock *> IndirectDests,
+                            ArrayRef<Value *> Args,
+                            ArrayRef<OperandBundleDef> Bundles = None,
+                            const Twine &NameStr = "",
+                            Instruction *InsertBefore = nullptr) {
+    int NumOperands = ComputeNumOperands(Args.size(), IndirectDests.size(),
+                                         CountBundleInputs(Bundles));
+    unsigned DescriptorBytes = Bundles.size() * sizeof(BundleOpInfo);
+
+    return new (NumOperands, DescriptorBytes)
+        CallBrInst(Ty, Func, DefaultDest, IndirectDests, Args, Bundles,
+                   NumOperands, NameStr, InsertBefore);
+  }
+
+  static CallBrInst *Create(FunctionType *Ty, Value *Func,
+                            BasicBlock *DefaultDest,
+                            ArrayRef<BasicBlock *> IndirectDests,
+                            ArrayRef<Value *> Args, const Twine &NameStr,
+                            BasicBlock *InsertAtEnd) {
+    int NumOperands = ComputeNumOperands(Args.size(), IndirectDests.size());
+    return new (NumOperands)
+        CallBrInst(Ty, Func, DefaultDest, IndirectDests, Args, None,
+                   NumOperands, NameStr, InsertAtEnd);
+  }
+
+  static CallBrInst *Create(FunctionType *Ty, Value *Func,
+                            BasicBlock *DefaultDest,
+                            ArrayRef<BasicBlock *> IndirectDests,
+                            ArrayRef<Value *> Args,
+                            ArrayRef<OperandBundleDef> Bundles,
+                            const Twine &NameStr, BasicBlock *InsertAtEnd) {
+    int NumOperands = ComputeNumOperands(Args.size(), IndirectDests.size(),
+                                         CountBundleInputs(Bundles));
+    unsigned DescriptorBytes = Bundles.size() * sizeof(BundleOpInfo);
+
+    return new (NumOperands, DescriptorBytes)
+        CallBrInst(Ty, Func, DefaultDest, IndirectDests, Args, Bundles,
+                   NumOperands, NameStr, InsertAtEnd);
+  }
+
+  static CallBrInst *Create(FunctionCallee Func, BasicBlock *DefaultDest,
+                            ArrayRef<BasicBlock *> IndirectDests,
+                            ArrayRef<Value *> Args, const Twine &NameStr,
+                            Instruction *InsertBefore = nullptr) {
+    return Create(Func.getFunctionType(), Func.getCallee(), DefaultDest,
+                  IndirectDests, Args, NameStr, InsertBefore);
+  }
+
+  static CallBrInst *Create(FunctionCallee Func, BasicBlock *DefaultDest,
+                            ArrayRef<BasicBlock *> IndirectDests,
+                            ArrayRef<Value *> Args,
+                            ArrayRef<OperandBundleDef> Bundles = None,
+                            const Twine &NameStr = "",
+                            Instruction *InsertBefore = nullptr) {
+    return Create(Func.getFunctionType(), Func.getCallee(), DefaultDest,
+                  IndirectDests, Args, Bundles, NameStr, InsertBefore);
+  }
+
+  static CallBrInst *Create(FunctionCallee Func, BasicBlock *DefaultDest,
+                            ArrayRef<BasicBlock *> IndirectDests,
+                            ArrayRef<Value *> Args, const Twine &NameStr,
+                            BasicBlock *InsertAtEnd) {
+    return Create(Func.getFunctionType(), Func.getCallee(), DefaultDest,
+                  IndirectDests, Args, NameStr, InsertAtEnd);
+  }
+
+  static CallBrInst *Create(FunctionCallee Func,
+                            BasicBlock *DefaultDest,
+                            ArrayRef<BasicBlock *> IndirectDests,
+                            ArrayRef<Value *> Args,
+                            ArrayRef<OperandBundleDef> Bundles,
+                            const Twine &NameStr, BasicBlock *InsertAtEnd) {
+    return Create(Func.getFunctionType(), Func.getCallee(), DefaultDest,
+                  IndirectDests, Args, Bundles, NameStr, InsertAtEnd);
+  }
+
+  /// Create a clone of \p CBI with a different set of operand bundles and
+  /// insert it before \p InsertPt.
+  ///
+  /// The returned callbr instruction is identical to \p CBI in every way
+  /// except that the operand bundles for the new instruction are set to the
+  /// operand bundles in \p Bundles.
+  static CallBrInst *Create(CallBrInst *CBI,
+                            ArrayRef<OperandBundleDef> Bundles,
+                            Instruction *InsertPt = nullptr);
+
+  /// Return the number of callbr indirect dest labels.
+  ///
+  unsigned getNumIndirectDests() const { return NumIndirectDests; }
+
+  /// getIndirectDestLabel - Return the i-th indirect dest label.
+  ///
+  Value *getIndirectDestLabel(unsigned i) const {
+    assert(i < getNumIndirectDests() && "Out of bounds!");
+    return getOperand(i + getNumArgOperands() + getNumTotalBundleOperands() +
+                      1);
+  }
+
+  Value *getIndirectDestLabelUse(unsigned i) const {
+    assert(i < getNumIndirectDests() && "Out of bounds!");
+    return getOperandUse(i + getNumArgOperands() + getNumTotalBundleOperands() +
+                         1);
+  }
+
+  // Return the destination basic blocks...
+  BasicBlock *getDefaultDest() const {
+    return cast<BasicBlock>(*(&Op<-1>() - getNumIndirectDests() - 1));
+  }
+  BasicBlock *getIndirectDest(unsigned i) const {
+    return cast<BasicBlock>(*(&Op<-1>() - getNumIndirectDests() + i));
+  }
+  SmallVector<BasicBlock *, 16> getIndirectDests() const {
+    SmallVector<BasicBlock *, 16> IndirectDests;
+    for (unsigned i = 0, e = getNumIndirectDests(); i < e; ++i)
+      IndirectDests.push_back(getIndirectDest(i));
+    return IndirectDests;
+  }
+  void setDefaultDest(BasicBlock *B) {
+    *(&Op<-1>() - getNumIndirectDests() - 1) = reinterpret_cast<Value *>(B);
+  }
+  void setIndirectDest(unsigned i, BasicBlock *B) {
+    *(&Op<-1>() - getNumIndirectDests() + i) = reinterpret_cast<Value *>(B);
+  }
+
+  BasicBlock *getSuccessor(unsigned i) const {
+    assert(i < getNumSuccessors() + 1 &&
+           "Successor # out of range for callbr!");
+    return i == 0 ? getDefaultDest() : getIndirectDest(i - 1);
+  }
+
+  void setSuccessor(unsigned idx, BasicBlock *NewSucc) {
+    assert(idx < getNumIndirectDests() + 1 &&
+           "Successor # out of range for callbr!");
+    *(&Op<-1>() - getNumIndirectDests() -1 + idx) =
+        reinterpret_cast<Value *>(NewSucc);
+  }
+
+  unsigned getNumSuccessors() const { return getNumIndirectDests() + 1; }
+
+  // Methods for support type inquiry through isa, cast, and dyn_cast:
+  static bool classof(const Instruction *I) {
+    return (I->getOpcode() == Instruction::CallBr);
+  }
+  static bool classof(const Value *V) {
+    return isa<Instruction>(V) && classof(cast<Instruction>(V));
+  }
+
+private:
+
+  // Shadow Instruction::setInstructionSubclassData with a private forwarding
+  // method so that subclasses cannot accidentally use it.
+  void setInstructionSubclassData(unsigned short D) {
+    Instruction::setInstructionSubclassData(D);
+  }
+};
+
+CallBrInst::CallBrInst(FunctionType *Ty, Value *Func, BasicBlock *DefaultDest,
+                       ArrayRef<BasicBlock *> IndirectDests,
+                       ArrayRef<Value *> Args,
+                       ArrayRef<OperandBundleDef> Bundles, int NumOperands,
+                       const Twine &NameStr, Instruction *InsertBefore)
+    : CallBase(Ty->getReturnType(), Instruction::CallBr,
+               OperandTraits<CallBase>::op_end(this) - NumOperands, NumOperands,
+               InsertBefore) {
+  init(Ty, Func, DefaultDest, IndirectDests, Args, Bundles, NameStr);
+}
+
+CallBrInst::CallBrInst(FunctionType *Ty, Value *Func, BasicBlock *DefaultDest,
+                       ArrayRef<BasicBlock *> IndirectDests,
+                       ArrayRef<Value *> Args,
+                       ArrayRef<OperandBundleDef> Bundles, int NumOperands,
+                       const Twine &NameStr, BasicBlock *InsertAtEnd)
+    : CallBase(
+          cast<FunctionType>(
+              cast<PointerType>(Func->getType())->getElementType())
+              ->getReturnType(),
+          Instruction::CallBr,
+          OperandTraits<CallBase>::op_end(this) - NumOperands, NumOperands,
+          InsertAtEnd) {
+  init(Ty, Func, DefaultDest, IndirectDests, Args, Bundles, NameStr);
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/include/llvm/IR/IntrinsicInst.h b/include/llvm/IR/IntrinsicInst.h
index 80a7a705257483d4f5cd6db0f178353acd5c22a8..62bb4fa61d7e99256d8df8428f35b1d83b61274e 100644
--- a/include/llvm/IR/IntrinsicInst.h
+++ b/include/llvm/IR/IntrinsicInst.h
@@ -1,9 +1,8 @@
 //===-- llvm/IntrinsicInst.h - Intrinsic Instruction Wrappers ---*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/IR/Intrinsics.h b/include/llvm/IR/Intrinsics.h
index e1e17f983ff88c9f23bfdf504c8233791ecf7dd4..ad5e7d1ac193b81ca392255f5a0e68c68c7865c6 100644
--- a/include/llvm/IR/Intrinsics.h
+++ b/include/llvm/IR/Intrinsics.h
@@ -1,9 +1,8 @@
-//===-- llvm/Instrinsics.h - LLVM Intrinsic Function Handling ---*- C++ -*-===//
+//===- Intrinsics.h - LLVM Intrinsic Function Handling ----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/IR/Intrinsics.td b/include/llvm/IR/Intrinsics.td
index f503d3ebbdfebd496b1d5ca63253075e3b383849..090ee60ede76a3dab6054f68f655d39534351a6b 100644
--- a/include/llvm/IR/Intrinsics.td
+++ b/include/llvm/IR/Intrinsics.td
@@ -1,9 +1,8 @@
 //===- Intrinsics.td - Defines all LLVM intrinsics ---------*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -157,10 +156,15 @@ class LLVMMatchType<int num>
 // the intrinsic is overloaded, so the matched type should be declared as iAny.
 class LLVMExtendedType<int num> : LLVMMatchType<num>;
 class LLVMTruncatedType<int num> : LLVMMatchType<num>;
-class LLVMVectorSameWidth<int num, LLVMType elty>
-  : LLVMMatchType<num> {
+
+// Match the scalar/vector of another intrinsic parameter but with a different
+// element type. Either both are scalars or both are vectors with the same
+// number of elements.
+class LLVMScalarOrSameVectorWidth<int idx, LLVMType elty>
+  : LLVMMatchType<idx> {
   ValueType ElTy = elty.VT;
 }
+
 class LLVMPointerTo<int num> : LLVMMatchType<num>;
 class LLVMPointerToElt<int num> : LLVMMatchType<num>;
 class LLVMVectorOfAnyPointersToElt<int num> : LLVMMatchType<num>;
@@ -414,6 +418,13 @@ def int_localescape : Intrinsic<[], [llvm_vararg_ty]>;
 def int_localrecover : Intrinsic<[llvm_ptr_ty],
                                  [llvm_ptr_ty, llvm_ptr_ty, llvm_i32_ty],
                                  [IntrNoMem]>;
+
+// Given the frame pointer passed into an SEH filter function, returns a
+// pointer to the local variable area suitable for use with llvm.localrecover.
+def int_eh_recoverfp : Intrinsic<[llvm_ptr_ty],
+                                 [llvm_ptr_ty, llvm_ptr_ty],
+                                 [IntrNoMem]>;
+
 // Note: we treat stacksave/stackrestore as writemem because we don't otherwise
 // model their dependencies on allocas.
 def int_stacksave     : Intrinsic<[llvm_ptr_ty]>,
@@ -547,7 +558,8 @@ def int_siglongjmp : Intrinsic<[], [llvm_ptr_ty, llvm_i32_ty], [IntrNoReturn]>;
 
 // Internal interface for object size checking
 def int_objectsize : Intrinsic<[llvm_anyint_ty],
-                               [llvm_anyptr_ty, llvm_i1_ty, llvm_i1_ty],
+                               [llvm_anyptr_ty, llvm_i1_ty,
+                                llvm_i1_ty, llvm_i1_ty],
                                [IntrNoMem, IntrSpeculatable]>,
                                GCCBuiltin<"__builtin_object_size">;
 
@@ -790,24 +802,30 @@ def int_adjust_trampoline : Intrinsic<[llvm_ptr_ty], [llvm_ptr_ty],
 //
 
 // Expose the carry flag from add operations on two integrals.
-def int_sadd_with_overflow : Intrinsic<[llvm_anyint_ty, llvm_i1_ty],
+def int_sadd_with_overflow : Intrinsic<[llvm_anyint_ty,
+                                        LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>],
                                        [LLVMMatchType<0>, LLVMMatchType<0>],
                                        [IntrNoMem, IntrSpeculatable]>;
-def int_uadd_with_overflow : Intrinsic<[llvm_anyint_ty, llvm_i1_ty],
+def int_uadd_with_overflow : Intrinsic<[llvm_anyint_ty,
+                                        LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>],
                                        [LLVMMatchType<0>, LLVMMatchType<0>],
                                        [IntrNoMem, IntrSpeculatable]>;
 
-def int_ssub_with_overflow : Intrinsic<[llvm_anyint_ty, llvm_i1_ty],
+def int_ssub_with_overflow : Intrinsic<[llvm_anyint_ty,
+                                        LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>],
                                        [LLVMMatchType<0>, LLVMMatchType<0>],
                                        [IntrNoMem, IntrSpeculatable]>;
-def int_usub_with_overflow : Intrinsic<[llvm_anyint_ty, llvm_i1_ty],
+def int_usub_with_overflow : Intrinsic<[llvm_anyint_ty,
+                                        LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>],
                                        [LLVMMatchType<0>, LLVMMatchType<0>],
                                        [IntrNoMem, IntrSpeculatable]>;
 
-def int_smul_with_overflow : Intrinsic<[llvm_anyint_ty, llvm_i1_ty],
+def int_smul_with_overflow : Intrinsic<[llvm_anyint_ty,
+                                        LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>],
                                        [LLVMMatchType<0>, LLVMMatchType<0>],
                                        [IntrNoMem, IntrSpeculatable]>;
-def int_umul_with_overflow : Intrinsic<[llvm_anyint_ty, llvm_i1_ty],
+def int_umul_with_overflow : Intrinsic<[llvm_anyint_ty,
+                                        LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>],
                                        [LLVMMatchType<0>, LLVMMatchType<0>],
                                        [IntrNoMem, IntrSpeculatable]>;
 
@@ -832,6 +850,10 @@ def int_smul_fix : Intrinsic<[llvm_anyint_ty],
                              [LLVMMatchType<0>, LLVMMatchType<0>, llvm_i32_ty],
                              [IntrNoMem, IntrSpeculatable, Commutative]>;
 
+def int_umul_fix : Intrinsic<[llvm_anyint_ty],
+                             [LLVMMatchType<0>, LLVMMatchType<0>, llvm_i32_ty],
+                             [IntrNoMem, IntrSpeculatable, Commutative]>;
+
 //===------------------------- Memory Use Markers -------------------------===//
 //
 def int_lifetime_start  : Intrinsic<[],
@@ -995,35 +1017,35 @@ def int_is_constant : Intrinsic<[llvm_i1_ty], [llvm_any_ty], [IntrNoMem], "llvm.
 def int_masked_store : Intrinsic<[], [llvm_anyvector_ty,
                                       LLVMAnyPointerType<LLVMMatchType<0>>,
                                       llvm_i32_ty,
-                                      LLVMVectorSameWidth<0, llvm_i1_ty>],
+                                      LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>],
                                  [IntrArgMemOnly]>;
 
 def int_masked_load  : Intrinsic<[llvm_anyvector_ty],
                                  [LLVMAnyPointerType<LLVMMatchType<0>>, llvm_i32_ty,
-                                  LLVMVectorSameWidth<0, llvm_i1_ty>, LLVMMatchType<0>],
+                                  LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, LLVMMatchType<0>],
                                  [IntrReadMem, IntrArgMemOnly]>;
 
 def int_masked_gather: Intrinsic<[llvm_anyvector_ty],
                                  [LLVMVectorOfAnyPointersToElt<0>, llvm_i32_ty,
-                                  LLVMVectorSameWidth<0, llvm_i1_ty>,
+                                  LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
                                   LLVMMatchType<0>],
                                  [IntrReadMem]>;
 
 def int_masked_scatter: Intrinsic<[],
                                   [llvm_anyvector_ty,
                                    LLVMVectorOfAnyPointersToElt<0>, llvm_i32_ty,
-                                   LLVMVectorSameWidth<0, llvm_i1_ty>]>;
+                                   LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>]>;
 
 def int_masked_expandload: Intrinsic<[llvm_anyvector_ty],
                                      [LLVMPointerToElt<0>,
-                                      LLVMVectorSameWidth<0, llvm_i1_ty>,
+                                      LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
                                       LLVMMatchType<0>],
                                      [IntrReadMem]>;
 
 def int_masked_compressstore: Intrinsic<[],
                                      [llvm_anyvector_ty,
                                       LLVMPointerToElt<0>,
-                                      LLVMVectorSameWidth<0, llvm_i1_ty>],
+                                      LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>],
                                      [IntrArgMemOnly]>;
 
 // Test whether a pointer is associated with a type metadata identifier.
@@ -1042,6 +1064,9 @@ def int_icall_branch_funnel : Intrinsic<[], [llvm_vararg_ty], []>;
 def int_load_relative: Intrinsic<[llvm_ptr_ty], [llvm_ptr_ty, llvm_anyint_ty],
                                  [IntrReadMem, IntrArgMemOnly]>;
 
+def int_hwasan_check_memaccess :
+  Intrinsic<[], [llvm_ptr_ty, llvm_ptr_ty, llvm_i32_ty], [IntrInaccessibleMemOnly]>;
+
 // Xray intrinsics
 //===----------------------------------------------------------------------===//
 // Custom event logging for x-ray.
diff --git a/include/llvm/IR/IntrinsicsAArch64.td b/include/llvm/IR/IntrinsicsAArch64.td
index ff25750fe399bc8f7fae882096e333ee123543d3..3abeff4bb71cb66efc867dbf958bca874067b56b 100644
--- a/include/llvm/IR/IntrinsicsAArch64.td
+++ b/include/llvm/IR/IntrinsicsAArch64.td
@@ -1,9 +1,8 @@
 //===- IntrinsicsAARCH64.td - Defines AARCH64 intrinsics ---*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/IR/IntrinsicsAMDGPU.td b/include/llvm/IR/IntrinsicsAMDGPU.td
index 3ea364c78457e70e507e8b0cc8860a12f779585c..151329cce18fd5fc3e464538e43e3a58220ef5e3 100644
--- a/include/llvm/IR/IntrinsicsAMDGPU.td
+++ b/include/llvm/IR/IntrinsicsAMDGPU.td
@@ -1,9 +1,8 @@
 //===- IntrinsicsAMDGPU.td - Defines AMDGPU intrinsics -----*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -392,6 +391,35 @@ class AMDGPULDSF32Intrin<string clang_builtin> :
     [IntrArgMemOnly, NoCapture<0>]
 >;
 
+class AMDGPUDSOrderedIntrinsic : Intrinsic<
+  [llvm_i32_ty],
+  // M0 = {hi16:address, lo16:waveID}. Allow passing M0 as a pointer, so that
+  // the bit packing can be optimized at the IR level.
+  [LLVMQualPointerType<llvm_i32_ty, 2>, // IntToPtr(M0)
+   llvm_i32_ty, // value to add or swap
+   llvm_i32_ty, // ordering
+   llvm_i32_ty, // scope
+   llvm_i1_ty,  // isVolatile
+   llvm_i32_ty, // ordered count index (OA index), also added to the address
+   llvm_i1_ty,  // wave release, usually set to 1
+   llvm_i1_ty], // wave done, set to 1 for the last ordered instruction
+  [NoCapture<0>]
+>;
+
+class AMDGPUDSAppendConsumedIntrinsic : Intrinsic<
+  [llvm_i32_ty],
+  [llvm_anyptr_ty, // LDS or GDS ptr
+   llvm_i1_ty], // isVolatile
+   [IntrConvergent, IntrArgMemOnly, NoCapture<0>]
+>;
+
+def int_amdgcn_ds_ordered_add : AMDGPUDSOrderedIntrinsic;
+def int_amdgcn_ds_ordered_swap : AMDGPUDSOrderedIntrinsic;
+
+// The pointer argument is assumed to be dynamically uniform if a VGPR.
+def int_amdgcn_ds_append : AMDGPUDSAppendConsumedIntrinsic;
+def int_amdgcn_ds_consume : AMDGPUDSAppendConsumedIntrinsic;
+
 def int_amdgcn_ds_fadd : AMDGPULDSF32Intrin<"__builtin_amdgcn_ds_faddf">;
 def int_amdgcn_ds_fmin : AMDGPULDSF32Intrin<"__builtin_amdgcn_ds_fminf">;
 def int_amdgcn_ds_fmax : AMDGPULDSF32Intrin<"__builtin_amdgcn_ds_fmaxf">;
@@ -590,7 +618,7 @@ class AMDGPUDimSampleProfile<string opmod,
                              AMDGPUDimProps dim,
                              AMDGPUSampleVariant sample> : AMDGPUDimProfile<opmod, dim> {
   let IsSample = 1;
-  let RetTypes = [llvm_anyfloat_ty];
+  let RetTypes = [llvm_any_ty];
   let ExtraAddrArgs = sample.ExtraAddrArgs;
   let Gradients = sample.Gradients;
   let LodClampMip = sample.LodOrClamp;
@@ -683,11 +711,11 @@ defset list<AMDGPUImageDimIntrinsic> AMDGPUImageDimIntrinsics = {
   }
 
   defm int_amdgcn_image_load
-    : AMDGPUImageDimIntrinsicsAll<"LOAD", [llvm_anyfloat_ty], [], [IntrReadMem],
+    : AMDGPUImageDimIntrinsicsAll<"LOAD", [llvm_any_ty], [], [IntrReadMem],
                                   [SDNPMemOperand]>,
       AMDGPUImageDMaskIntrinsic;
   defm int_amdgcn_image_load_mip
-    : AMDGPUImageDimIntrinsicsNoMsaa<"LOAD_MIP", [llvm_anyfloat_ty], [],
+    : AMDGPUImageDimIntrinsicsNoMsaa<"LOAD_MIP", [llvm_any_ty], [],
                                      [IntrReadMem], [SDNPMemOperand], 1>,
       AMDGPUImageDMaskIntrinsic;
 
@@ -1148,6 +1176,20 @@ def int_amdgcn_interp_p2 :
             [IntrNoMem, IntrSpeculatable]>;
           // See int_amdgcn_v_interp_p1 for why this is IntrNoMem.
 
+// __builtin_amdgcn_interp_p1_f16 <i>, <attr_chan>, <attr>, <high>, <m0>
+def int_amdgcn_interp_p1_f16 :
+  GCCBuiltin<"__builtin_amdgcn_interp_p1_f16">,
+  Intrinsic<[llvm_float_ty],
+            [llvm_float_ty, llvm_i32_ty, llvm_i32_ty, llvm_i1_ty, llvm_i32_ty],
+            [IntrNoMem, IntrSpeculatable]>;
+
+// __builtin_amdgcn_interp_p2_f16 <p1>, <j>, <attr_chan>, <attr>, <high>, <m0>
+def int_amdgcn_interp_p2_f16 :
+  GCCBuiltin<"__builtin_amdgcn_interp_p2_f16">,
+  Intrinsic<[llvm_half_ty],
+            [llvm_float_ty, llvm_float_ty, llvm_i32_ty, llvm_i32_ty, llvm_i1_ty, llvm_i32_ty],
+            [IntrNoMem, IntrSpeculatable]>;
+
 // Pixel shaders only: whether the current pixel is live (i.e. not a helper
 // invocation for derivative computation).
 def int_amdgcn_ps_live : Intrinsic <
diff --git a/include/llvm/IR/IntrinsicsARM.td b/include/llvm/IR/IntrinsicsARM.td
index 4e11f9c29dd056234666f6da633061a3e074f202..229c20abaf77941044e628650877697592a5e6b5 100644
--- a/include/llvm/IR/IntrinsicsARM.td
+++ b/include/llvm/IR/IntrinsicsARM.td
@@ -1,9 +1,8 @@
 //===- IntrinsicsARM.td - Defines ARM intrinsics -----------*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/IR/IntrinsicsBPF.td b/include/llvm/IR/IntrinsicsBPF.td
index 94eca8e40332b9e934864deb75f09994ef3c9f63..d7595a2a77007755ee9ad28f271c736400f7cc30 100644
--- a/include/llvm/IR/IntrinsicsBPF.td
+++ b/include/llvm/IR/IntrinsicsBPF.td
@@ -1,9 +1,8 @@
 //===- IntrinsicsBPF.td - Defines BPF intrinsics -----------*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/IR/IntrinsicsHexagon.td b/include/llvm/IR/IntrinsicsHexagon.td
index ecc69a679553ffe99cb35047021c3a41e587547a..3e3166da8d50ba08bf7e52dec6993c1cdc9a34d5 100644
--- a/include/llvm/IR/IntrinsicsHexagon.td
+++ b/include/llvm/IR/IntrinsicsHexagon.td
@@ -1,8 +1,7 @@
 //===- IntrinsicsHexagon.td - Defines Hexagon intrinsics ---*- tablegen -*-===//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/IR/IntrinsicsMips.td b/include/llvm/IR/IntrinsicsMips.td
index 421a79be4ebc22881230d1e835bc8f7cc8676d2d..26189dc37575cec2e4b4b3d242c22df8884023c7 100644
--- a/include/llvm/IR/IntrinsicsMips.td
+++ b/include/llvm/IR/IntrinsicsMips.td
@@ -1,9 +1,8 @@
 //===- IntrinsicsMips.td - Defines Mips intrinsics ---------*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/IR/IntrinsicsNVVM.td b/include/llvm/IR/IntrinsicsNVVM.td
index 7f694f68969e651618c8f0f0eca21cfc6ebb98f5..cf072c70ebafdf5be0f7f59473d4f5cba23d34f4 100644
--- a/include/llvm/IR/IntrinsicsNVVM.td
+++ b/include/llvm/IR/IntrinsicsNVVM.td
@@ -1,9 +1,8 @@
 //===- IntrinsicsNVVM.td - Defines NVVM intrinsics ---------*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -3674,11 +3673,19 @@ multiclass PTXReadSRegIntrinsic_v4i32<string regname> {
 class PTXReadSRegIntrinsic_r32<string name>
   : Intrinsic<[llvm_i32_ty], [], [IntrNoMem]>,
     GCCBuiltin<"__nvvm_read_ptx_sreg_" # name>;
-
 class PTXReadSRegIntrinsic_r64<string name>
   : Intrinsic<[llvm_i64_ty], [], [IntrNoMem]>,
     GCCBuiltin<"__nvvm_read_ptx_sreg_" # name>;
 
+// Intrinsics to read registers with non-constant values. E.g. the values that
+// do change over the kernel lifetime. Such reads should not be CSE'd.
+class PTXReadNCSRegIntrinsic_r32<string name>
+  : Intrinsic<[llvm_i32_ty], [], [IntrInaccessibleMemOnly]>,
+    GCCBuiltin<"__nvvm_read_ptx_sreg_" # name>;
+class PTXReadNCSRegIntrinsic_r64<string name>
+  : Intrinsic<[llvm_i64_ty], [], [IntrInaccessibleMemOnly]>,
+    GCCBuiltin<"__nvvm_read_ptx_sreg_" # name>;
+
 defm int_nvvm_read_ptx_sreg_tid : PTXReadSRegIntrinsic_v4i32<"tid">;
 defm int_nvvm_read_ptx_sreg_ntid : PTXReadSRegIntrinsic_v4i32<"ntid">;
 
@@ -3704,13 +3711,13 @@ def int_nvvm_read_ptx_sreg_lanemask_ge :
 def int_nvvm_read_ptx_sreg_lanemask_gt :
     PTXReadSRegIntrinsic_r32<"lanemask_gt">;
 
-def int_nvvm_read_ptx_sreg_clock : PTXReadSRegIntrinsic_r32<"clock">;
-def int_nvvm_read_ptx_sreg_clock64 : PTXReadSRegIntrinsic_r64<"clock64">;
+def int_nvvm_read_ptx_sreg_clock : PTXReadNCSRegIntrinsic_r32<"clock">;
+def int_nvvm_read_ptx_sreg_clock64 : PTXReadNCSRegIntrinsic_r64<"clock64">;
 
-def int_nvvm_read_ptx_sreg_pm0 : PTXReadSRegIntrinsic_r32<"pm0">;
-def int_nvvm_read_ptx_sreg_pm1 : PTXReadSRegIntrinsic_r32<"pm1">;
-def int_nvvm_read_ptx_sreg_pm2 : PTXReadSRegIntrinsic_r32<"pm2">;
-def int_nvvm_read_ptx_sreg_pm3 : PTXReadSRegIntrinsic_r32<"pm3">;
+def int_nvvm_read_ptx_sreg_pm0 : PTXReadNCSRegIntrinsic_r32<"pm0">;
+def int_nvvm_read_ptx_sreg_pm1 : PTXReadNCSRegIntrinsic_r32<"pm1">;
+def int_nvvm_read_ptx_sreg_pm2 : PTXReadNCSRegIntrinsic_r32<"pm2">;
+def int_nvvm_read_ptx_sreg_pm3 : PTXReadNCSRegIntrinsic_r32<"pm3">;
 
 def int_nvvm_read_ptx_sreg_warpsize : PTXReadSRegIntrinsic_r32<"warpsize">;
 
diff --git a/include/llvm/IR/IntrinsicsPowerPC.td b/include/llvm/IR/IntrinsicsPowerPC.td
index 62b2e8f77e7d8b3b870559ef60c0bb7609d0f4b1..e6b08842015e1171eba9f3c4da0b59eca05bc014 100644
--- a/include/llvm/IR/IntrinsicsPowerPC.td
+++ b/include/llvm/IR/IntrinsicsPowerPC.td
@@ -1,9 +1,8 @@
 //===- IntrinsicsPowerPC.td - Defines PowerPC intrinsics ---*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/IR/IntrinsicsRISCV.td b/include/llvm/IR/IntrinsicsRISCV.td
index 0ac7348b56dbf87d1aecaa33510732b544afcea8..5b13fc41f513a54e98edb8e81e8b1d0e0f5f16a2 100644
--- a/include/llvm/IR/IntrinsicsRISCV.td
+++ b/include/llvm/IR/IntrinsicsRISCV.td
@@ -1,9 +1,8 @@
 //===- IntrinsicsRISCV.td - Defines RISCV intrinsics -------*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -41,4 +40,29 @@ def int_riscv_masked_cmpxchg_i32
                                 llvm_i32_ty, llvm_i32_ty],
                 [IntrArgMemOnly, NoCapture<0>]>;
 
+class MaskedAtomicRMW64Intrinsic
+    : Intrinsic<[llvm_i64_ty],
+                [llvm_anyptr_ty, llvm_i64_ty, llvm_i64_ty, llvm_i64_ty],
+                [IntrArgMemOnly, NoCapture<0>]>;
+
+class MaskedAtomicRMW64WithSextIntrinsic
+    : Intrinsic<[llvm_i64_ty],
+                [llvm_anyptr_ty, llvm_i64_ty, llvm_i64_ty, llvm_i64_ty,
+                 llvm_i64_ty],
+                [IntrArgMemOnly, NoCapture<0>]>;
+
+def int_riscv_masked_atomicrmw_xchg_i64 : MaskedAtomicRMW64Intrinsic;
+def int_riscv_masked_atomicrmw_add_i64  : MaskedAtomicRMW64Intrinsic;
+def int_riscv_masked_atomicrmw_sub_i64  : MaskedAtomicRMW64Intrinsic;
+def int_riscv_masked_atomicrmw_nand_i64 : MaskedAtomicRMW64Intrinsic;
+def int_riscv_masked_atomicrmw_max_i64  : MaskedAtomicRMW64WithSextIntrinsic;
+def int_riscv_masked_atomicrmw_min_i64  : MaskedAtomicRMW64WithSextIntrinsic;
+def int_riscv_masked_atomicrmw_umax_i64 : MaskedAtomicRMW64Intrinsic;
+def int_riscv_masked_atomicrmw_umin_i64 : MaskedAtomicRMW64Intrinsic;
+
+def int_riscv_masked_cmpxchg_i64
+    : Intrinsic<[llvm_i64_ty], [llvm_anyptr_ty, llvm_i64_ty, llvm_i64_ty,
+                                llvm_i64_ty, llvm_i64_ty],
+                [IntrArgMemOnly, NoCapture<0>]>;
+
 } // TargetPrefix = "riscv"
diff --git a/include/llvm/IR/IntrinsicsSystemZ.td b/include/llvm/IR/IntrinsicsSystemZ.td
index caa2ec209a3107fd780bc6a057e134f9d5c5bd65..91e66dd9c5230b267993dda80fb69269afb1b2eb 100644
--- a/include/llvm/IR/IntrinsicsSystemZ.td
+++ b/include/llvm/IR/IntrinsicsSystemZ.td
@@ -1,9 +1,8 @@
 //===- IntrinsicsSystemZ.td - Defines SystemZ intrinsics ---*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/IR/IntrinsicsWebAssembly.td b/include/llvm/IR/IntrinsicsWebAssembly.td
index ff5964c3aaba884ccb44fed7a5c11b8d709e391f..bfeb7063ced58ba667d399ef5444be3bfe3d77c0 100644
--- a/include/llvm/IR/IntrinsicsWebAssembly.td
+++ b/include/llvm/IR/IntrinsicsWebAssembly.td
@@ -1,9 +1,8 @@
 //===- IntrinsicsWebAssembly.td - Defines wasm intrinsics --*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
@@ -24,18 +23,6 @@ def int_wasm_memory_grow : Intrinsic<[llvm_anyint_ty],
                                      [llvm_i32_ty, LLVMMatchType<0>],
                                      []>;
 
-// These are the old names.
-def int_wasm_mem_size : Intrinsic<[llvm_anyint_ty],
-                                  [llvm_i32_ty],
-                                  [IntrReadMem]>;
-def int_wasm_mem_grow : Intrinsic<[llvm_anyint_ty],
-                                  [llvm_i32_ty, LLVMMatchType<0>],
-                                  []>;
-
-// These are the old old names. They also lack the immediate field.
-def int_wasm_current_memory : Intrinsic<[llvm_anyint_ty], [], [IntrReadMem]>;
-def int_wasm_grow_memory : Intrinsic<[llvm_anyint_ty], [LLVMMatchType<0>], []>;
-
 //===----------------------------------------------------------------------===//
 // Saturating float-to-int conversions
 //===----------------------------------------------------------------------===//
@@ -62,11 +49,10 @@ def int_wasm_get_exception : Intrinsic<[llvm_ptr_ty], [llvm_token_ty],
                                        [IntrHasSideEffects]>;
 def int_wasm_get_ehselector : Intrinsic<[llvm_i32_ty], [llvm_token_ty],
                                         [IntrHasSideEffects]>;
-
-// wasm.catch returns the pointer to the exception object caught by wasm 'catch'
-// instruction.
-def int_wasm_catch : Intrinsic<[llvm_ptr_ty], [llvm_i32_ty],
-                               [IntrHasSideEffects]>;
+// This is the same as llvm.wasm.get.exception except that it does not take a
+// token operand. This is only for instruction selection purpose.
+def int_wasm_extract_exception : Intrinsic<[llvm_ptr_ty], [],
+                                           [IntrHasSideEffects]>;
 
 // WebAssembly EH must maintain the landingpads in the order assigned to them
 // by WasmEHPrepare pass to generate landingpad table in EHStreamer. This is
@@ -124,4 +110,18 @@ def int_wasm_alltrue :
             [llvm_anyvector_ty],
             [IntrNoMem, IntrSpeculatable]>;
 
+//===----------------------------------------------------------------------===//
+// Bulk memory intrinsics
+//===----------------------------------------------------------------------===//
+
+def int_wasm_memory_init :
+  Intrinsic<[],
+            [llvm_i32_ty, llvm_i32_ty, llvm_ptr_ty, llvm_i32_ty, llvm_i32_ty],
+            [IntrWriteMem, IntrInaccessibleMemOrArgMemOnly, WriteOnly<2>,
+             IntrHasSideEffects]>;
+def int_wasm_data_drop :
+  Intrinsic<[],
+            [llvm_i32_ty],
+            [IntrNoDuplicate, IntrHasSideEffects]>;
+
 } // TargetPrefix = "wasm"
diff --git a/include/llvm/IR/IntrinsicsX86.td b/include/llvm/IR/IntrinsicsX86.td
index a59dbe7ea4a42150b375e7d6e88ee8538e4cfc13..a8c8cba56f4a2c267b44693d7ef7ec448fc1a0d6 100644
--- a/include/llvm/IR/IntrinsicsX86.td
+++ b/include/llvm/IR/IntrinsicsX86.td
@@ -1,9 +1,8 @@
 //===- IntrinsicsX86.td - Defines X86 intrinsics -----------*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -27,12 +26,6 @@ let TargetPrefix = "x86" in {
 
   // Marks the EH guard slot node created in LLVM IR prior to code generation.
   def int_x86_seh_ehguard : Intrinsic<[], [llvm_ptr_ty], []>;
-
-  // Given a pointer to the end of an EH registration object, returns the true
-  // parent frame address that can be used with llvm.localrecover.
-  def int_x86_seh_recoverfp : Intrinsic<[llvm_ptr_ty],
-                                        [llvm_ptr_ty, llvm_ptr_ty],
-                                        [IntrNoMem]>;
 }
 
 //===----------------------------------------------------------------------===//
@@ -364,18 +357,6 @@ let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
 
 // Integer arithmetic ops.
 let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
-  def int_x86_sse2_padds_b : GCCBuiltin<"__builtin_ia32_paddsb128">,
-              Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty,
-                         llvm_v16i8_ty], [IntrNoMem, Commutative]>;
-  def int_x86_sse2_padds_w : GCCBuiltin<"__builtin_ia32_paddsw128">,
-              Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty,
-                         llvm_v8i16_ty], [IntrNoMem, Commutative]>;
-  def int_x86_sse2_psubs_b : GCCBuiltin<"__builtin_ia32_psubsb128">,
-              Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty,
-                         llvm_v16i8_ty], [IntrNoMem]>;
-  def int_x86_sse2_psubs_w : GCCBuiltin<"__builtin_ia32_psubsw128">,
-              Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty,
-                         llvm_v8i16_ty], [IntrNoMem]>;
   def int_x86_sse2_pmulhu_w : GCCBuiltin<"__builtin_ia32_pmulhuw128">,
               Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty,
                          llvm_v8i16_ty], [IntrNoMem, Commutative]>;
@@ -1324,21 +1305,12 @@ let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
 
 // BITALG bits shuffle
 let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
-  def int_x86_avx512_mask_vpshufbitqmb_128 :
-    GCCBuiltin<"__builtin_ia32_vpshufbitqmb128_mask">,
-    Intrinsic<[llvm_i16_ty],
-              [llvm_v16i8_ty, llvm_v16i8_ty, llvm_i16_ty],
-              [IntrNoMem]>;
-  def int_x86_avx512_mask_vpshufbitqmb_256 :
-    GCCBuiltin<"__builtin_ia32_vpshufbitqmb256_mask">,
-    Intrinsic<[llvm_i32_ty],
-              [llvm_v32i8_ty, llvm_v32i8_ty, llvm_i32_ty],
-              [IntrNoMem]>;
-  def int_x86_avx512_mask_vpshufbitqmb_512 :
-    GCCBuiltin<"__builtin_ia32_vpshufbitqmb512_mask">,
-    Intrinsic<[llvm_i64_ty],
-              [llvm_v64i8_ty, llvm_v64i8_ty, llvm_i64_ty],
-              [IntrNoMem]>;
+  def int_x86_avx512_vpshufbitqmb_128 :
+    Intrinsic<[llvm_v16i1_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_vpshufbitqmb_256 :
+    Intrinsic<[llvm_v32i1_ty], [llvm_v32i8_ty, llvm_v32i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_vpshufbitqmb_512 :
+    Intrinsic<[llvm_v64i1_ty], [llvm_v64i8_ty, llvm_v64i8_ty], [IntrNoMem]>;
 }
 
 //===----------------------------------------------------------------------===//
@@ -1346,18 +1318,6 @@ let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
 
 // Integer arithmetic ops.
 let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
-  def int_x86_avx2_padds_b : GCCBuiltin<"__builtin_ia32_paddsb256">,
-              Intrinsic<[llvm_v32i8_ty], [llvm_v32i8_ty,
-                         llvm_v32i8_ty], [IntrNoMem, Commutative]>;
-  def int_x86_avx2_padds_w : GCCBuiltin<"__builtin_ia32_paddsw256">,
-              Intrinsic<[llvm_v16i16_ty], [llvm_v16i16_ty,
-                         llvm_v16i16_ty], [IntrNoMem, Commutative]>;
-  def int_x86_avx2_psubs_b : GCCBuiltin<"__builtin_ia32_psubsb256">,
-              Intrinsic<[llvm_v32i8_ty], [llvm_v32i8_ty,
-                         llvm_v32i8_ty], [IntrNoMem]>;
-  def int_x86_avx2_psubs_w : GCCBuiltin<"__builtin_ia32_psubsw256">,
-              Intrinsic<[llvm_v16i16_ty], [llvm_v16i16_ty,
-                         llvm_v16i16_ty], [IntrNoMem]>;
   def int_x86_avx2_pmulhu_w : GCCBuiltin<"__builtin_ia32_pmulhuw256">,
               Intrinsic<[llvm_v16i16_ty], [llvm_v16i16_ty,
                          llvm_v16i16_ty], [IntrNoMem, Commutative]>;
@@ -1494,18 +1454,15 @@ let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
               Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty,
                          llvm_i32_ty], [IntrNoMem]>;
 
-  def int_x86_avx512_mask_pmultishift_qb_128:
-        GCCBuiltin<"__builtin_ia32_vpmultishiftqb128_mask">,
-        Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty,
-                   llvm_v16i8_ty, llvm_v16i8_ty, llvm_i16_ty], [IntrNoMem]>;
-  def int_x86_avx512_mask_pmultishift_qb_256:
-        GCCBuiltin<"__builtin_ia32_vpmultishiftqb256_mask">,
-        Intrinsic<[llvm_v32i8_ty], [llvm_v32i8_ty,
-                   llvm_v32i8_ty, llvm_v32i8_ty, llvm_i32_ty], [IntrNoMem]>;
-  def int_x86_avx512_mask_pmultishift_qb_512:
-        GCCBuiltin<"__builtin_ia32_vpmultishiftqb512_mask">,
-        Intrinsic<[llvm_v64i8_ty], [llvm_v64i8_ty,
-                   llvm_v64i8_ty, llvm_v64i8_ty, llvm_i64_ty], [IntrNoMem]>;
+  def int_x86_avx512_pmultishift_qb_128:
+        GCCBuiltin<"__builtin_ia32_vpmultishiftqb128">,
+        Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_pmultishift_qb_256:
+        GCCBuiltin<"__builtin_ia32_vpmultishiftqb256">,
+        Intrinsic<[llvm_v32i8_ty], [llvm_v32i8_ty, llvm_v32i8_ty], [IntrNoMem]>;
+  def int_x86_avx512_pmultishift_qb_512:
+        GCCBuiltin<"__builtin_ia32_vpmultishiftqb512">,
+        Intrinsic<[llvm_v64i8_ty], [llvm_v64i8_ty, llvm_v64i8_ty], [IntrNoMem]>;
 }
 
 // Pack ops.
@@ -1715,83 +1672,6 @@ let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
   def int_x86_avx512_psrav_w_512 : GCCBuiltin<"__builtin_ia32_psrav32hi">,
               Intrinsic<[llvm_v32i16_ty], [llvm_v32i16_ty, llvm_v32i16_ty],
                         [IntrNoMem]>;
-
-  def int_x86_avx512_prorv_d_128 : GCCBuiltin<"__builtin_ia32_prorvd128">,
-              Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty,
-                         llvm_v4i32_ty], [IntrNoMem]>;
-  def int_x86_avx512_prorv_d_256 : GCCBuiltin<"__builtin_ia32_prorvd256">,
-              Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty,
-                         llvm_v8i32_ty], [IntrNoMem]>;
-  def int_x86_avx512_prorv_d_512 : GCCBuiltin<"__builtin_ia32_prorvd512">,
-              Intrinsic<[llvm_v16i32_ty], [llvm_v16i32_ty,
-                         llvm_v16i32_ty], [IntrNoMem]>;
-  def int_x86_avx512_prorv_q_128 : GCCBuiltin<"__builtin_ia32_prorvq128">,
-              Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty,
-                         llvm_v2i64_ty], [IntrNoMem]>;
-  def int_x86_avx512_prorv_q_256 : GCCBuiltin<"__builtin_ia32_prorvq256">,
-              Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty,
-                         llvm_v4i64_ty], [IntrNoMem]>;
-  def int_x86_avx512_prorv_q_512 : GCCBuiltin<"__builtin_ia32_prorvq512">,
-              Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty,
-                         llvm_v8i64_ty], [IntrNoMem]>;
-
-   def int_x86_avx512_prol_d_128 : GCCBuiltin<"__builtin_ia32_prold128">,
-              Intrinsic<[llvm_v4i32_ty] , [llvm_v4i32_ty,
-                         llvm_i32_ty], [IntrNoMem]>;
-  def int_x86_avx512_prol_d_256 : GCCBuiltin<"__builtin_ia32_prold256">,
-              Intrinsic<[llvm_v8i32_ty] , [llvm_v8i32_ty,
-                         llvm_i32_ty], [IntrNoMem]>;
-  def int_x86_avx512_prol_d_512 : GCCBuiltin<"__builtin_ia32_prold512">,
-              Intrinsic<[llvm_v16i32_ty] , [llvm_v16i32_ty,
-                         llvm_i32_ty], [IntrNoMem]>;
-  def int_x86_avx512_prol_q_128 : GCCBuiltin<"__builtin_ia32_prolq128">,
-              Intrinsic<[llvm_v2i64_ty] , [llvm_v2i64_ty,
-                         llvm_i32_ty], [IntrNoMem]>;
-  def int_x86_avx512_prol_q_256 : GCCBuiltin<"__builtin_ia32_prolq256">,
-              Intrinsic<[llvm_v4i64_ty] , [llvm_v4i64_ty,
-                         llvm_i32_ty], [IntrNoMem]>;
-  def int_x86_avx512_prol_q_512 : GCCBuiltin<"__builtin_ia32_prolq512">,
-              Intrinsic<[llvm_v8i64_ty] , [llvm_v8i64_ty,
-                         llvm_i32_ty], [IntrNoMem]>;
-
-
-  def int_x86_avx512_prolv_d_128 : GCCBuiltin<"__builtin_ia32_prolvd128">,
-              Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty,
-                         llvm_v4i32_ty], [IntrNoMem]>;
-  def int_x86_avx512_prolv_d_256 : GCCBuiltin<"__builtin_ia32_prolvd256">,
-              Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty,
-                         llvm_v8i32_ty], [IntrNoMem]>;
-  def int_x86_avx512_prolv_d_512 : GCCBuiltin<"__builtin_ia32_prolvd512">,
-              Intrinsic<[llvm_v16i32_ty], [llvm_v16i32_ty,
-                         llvm_v16i32_ty], [IntrNoMem]>;
-  def int_x86_avx512_prolv_q_128 : GCCBuiltin<"__builtin_ia32_prolvq128">,
-              Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty,
-                         llvm_v2i64_ty], [IntrNoMem]>;
-  def int_x86_avx512_prolv_q_256 : GCCBuiltin<"__builtin_ia32_prolvq256">,
-              Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty,
-                         llvm_v4i64_ty], [IntrNoMem]>;
-  def int_x86_avx512_prolv_q_512 : GCCBuiltin<"__builtin_ia32_prolvq512">,
-              Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty,
-                         llvm_v8i64_ty], [IntrNoMem]>;
-  def int_x86_avx512_pror_d_128 : GCCBuiltin<"__builtin_ia32_prord128">,
-              Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty,
-                         llvm_i32_ty], [IntrNoMem]>;
-  def int_x86_avx512_pror_d_256 : GCCBuiltin<"__builtin_ia32_prord256">,
-              Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty,
-                         llvm_i32_ty], [IntrNoMem]>;
-  def int_x86_avx512_pror_d_512 : GCCBuiltin<"__builtin_ia32_prord512">,
-              Intrinsic<[llvm_v16i32_ty], [llvm_v16i32_ty,
-                         llvm_i32_ty], [IntrNoMem]>;
-  def int_x86_avx512_pror_q_128 : GCCBuiltin<"__builtin_ia32_prorq128">,
-              Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty,
-                         llvm_i32_ty], [IntrNoMem]>;
-  def int_x86_avx512_pror_q_256 : GCCBuiltin<"__builtin_ia32_prorq256">,
-              Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty,
-                         llvm_i32_ty], [IntrNoMem]>;
-  def int_x86_avx512_pror_q_512 : GCCBuiltin<"__builtin_ia32_prorq512">,
-              Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty,
-                         llvm_i32_ty], [IntrNoMem]>;
-
 }
 
 // Gather ops
@@ -2028,31 +1908,6 @@ let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
   def int_x86_xop_vfrcz_ps_256 : GCCBuiltin<"__builtin_ia32_vfrczps256">,
               Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty], [IntrNoMem]>;
 
-  def int_x86_xop_vpcomb : GCCBuiltin<"__builtin_ia32_vpcomb">,
-              Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty,
-                         llvm_i8_ty], [IntrNoMem]>;
-  def int_x86_xop_vpcomw : GCCBuiltin<"__builtin_ia32_vpcomw">,
-              Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty,
-                         llvm_i8_ty], [IntrNoMem]>;
-  def int_x86_xop_vpcomd : GCCBuiltin<"__builtin_ia32_vpcomd">,
-              Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty,
-                         llvm_i8_ty], [IntrNoMem]>;
-  def int_x86_xop_vpcomq : GCCBuiltin<"__builtin_ia32_vpcomq">,
-              Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty,
-                         llvm_i8_ty], [IntrNoMem]>;
-  def int_x86_xop_vpcomub : GCCBuiltin<"__builtin_ia32_vpcomub">,
-              Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty,
-                         llvm_i8_ty], [IntrNoMem]>;
-  def int_x86_xop_vpcomuw : GCCBuiltin<"__builtin_ia32_vpcomuw">,
-              Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty,
-                         llvm_i8_ty], [IntrNoMem]>;
-  def int_x86_xop_vpcomud : GCCBuiltin<"__builtin_ia32_vpcomud">,
-              Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty,
-                         llvm_i8_ty], [IntrNoMem]>;
-  def int_x86_xop_vpcomuq : GCCBuiltin<"__builtin_ia32_vpcomuq">,
-              Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty,
-                         llvm_i8_ty], [IntrNoMem]>;
-
   def int_x86_xop_vphaddbd :
               GCCBuiltin<"__builtin_ia32_vphaddbd">,
               Intrinsic<[llvm_v4i32_ty], [llvm_v16i8_ty], [IntrNoMem]>;
@@ -2163,32 +2018,6 @@ let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
               Intrinsic<[llvm_v16i8_ty],
                         [llvm_v16i8_ty, llvm_v16i8_ty, llvm_v16i8_ty],
                         [IntrNoMem]>;
-
-  def int_x86_xop_vprotb : GCCBuiltin<"__builtin_ia32_vprotb">,
-              Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty],
-                        [IntrNoMem]>;
-  def int_x86_xop_vprotd : GCCBuiltin<"__builtin_ia32_vprotd">,
-              Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty],
-                        [IntrNoMem]>;
-  def int_x86_xop_vprotq : GCCBuiltin<"__builtin_ia32_vprotq">,
-              Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty],
-                        [IntrNoMem]>;
-  def int_x86_xop_vprotw : GCCBuiltin<"__builtin_ia32_vprotw">,
-              Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty],
-                        [IntrNoMem]>;
-  def int_x86_xop_vprotbi : GCCBuiltin<"__builtin_ia32_vprotbi">,
-              Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_i8_ty],
-                        [IntrNoMem]>;
-  def int_x86_xop_vprotdi : GCCBuiltin<"__builtin_ia32_vprotdi">,
-              Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_i8_ty],
-                        [IntrNoMem]>;
-  def int_x86_xop_vprotqi : GCCBuiltin<"__builtin_ia32_vprotqi">,
-              Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_i8_ty],
-                        [IntrNoMem]>;
-  def int_x86_xop_vprotwi : GCCBuiltin<"__builtin_ia32_vprotwi">,
-              Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_i8_ty],
-                        [IntrNoMem]>;
-
   def int_x86_xop_vpshab :
               GCCBuiltin<"__builtin_ia32_vpshab">,
               Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty],
@@ -2859,10 +2688,12 @@ let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
 
 // Vector convert
 let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
-  def int_x86_avx512_mask_cvtdq2ps_512 :
-        GCCBuiltin<"__builtin_ia32_cvtdq2ps512_mask">,
-          Intrinsic<[llvm_v16f32_ty],
-          [llvm_v16i32_ty, llvm_v16f32_ty,  llvm_i16_ty,  llvm_i32_ty],
+  def int_x86_avx512_sitofp_round :
+          Intrinsic<[llvm_anyfloat_ty], [llvm_anyint_ty, llvm_i32_ty],
+          [IntrNoMem]>;
+
+  def int_x86_avx512_uitofp_round :
+          Intrinsic<[llvm_anyfloat_ty], [llvm_anyint_ty, llvm_i32_ty],
           [IntrNoMem]>;
 
   def int_x86_avx512_mask_cvtpd2dq_128 :
@@ -3033,30 +2864,12 @@ let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
           [llvm_v8f32_ty, llvm_v8i64_ty,  llvm_i8_ty,  llvm_i32_ty],
           [IntrNoMem]>;
 
-  def int_x86_avx512_mask_cvtqq2pd_512 :
-        GCCBuiltin<"__builtin_ia32_cvtqq2pd512_mask">,
-          Intrinsic<[llvm_v8f64_ty],
-          [llvm_v8i64_ty, llvm_v8f64_ty,  llvm_i8_ty,  llvm_i32_ty],
-          [IntrNoMem]>;
-
   def int_x86_avx512_mask_cvtqq2ps_128 :
         GCCBuiltin<"__builtin_ia32_cvtqq2ps128_mask">,
           Intrinsic<[llvm_v4f32_ty],
           [llvm_v2i64_ty, llvm_v4f32_ty,  llvm_i8_ty],
           [IntrNoMem]>;
 
-  def int_x86_avx512_mask_cvtqq2ps_256 :
-        GCCBuiltin<"__builtin_ia32_cvtqq2ps256_mask">,
-          Intrinsic<[llvm_v4f32_ty],
-          [llvm_v4i64_ty, llvm_v4f32_ty,  llvm_i8_ty],
-          [IntrNoMem]>;
-
-  def int_x86_avx512_mask_cvtqq2ps_512 :
-        GCCBuiltin<"__builtin_ia32_cvtqq2ps512_mask">,
-          Intrinsic<[llvm_v8f32_ty],
-          [llvm_v8i64_ty, llvm_v8f32_ty,  llvm_i8_ty,  llvm_i32_ty],
-          [IntrNoMem]>;
-
   def int_x86_avx512_mask_cvttpd2dq_128 :
         GCCBuiltin<"__builtin_ia32_cvttpd2dq128_mask">,
           Intrinsic<[llvm_v4i32_ty],
@@ -3183,36 +2996,12 @@ let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
           [llvm_v8f32_ty, llvm_v8i64_ty,  llvm_i8_ty,  llvm_i32_ty],
           [IntrNoMem]>;
 
-  def int_x86_avx512_mask_cvtudq2ps_512 :
-        GCCBuiltin<"__builtin_ia32_cvtudq2ps512_mask">,
-          Intrinsic<[llvm_v16f32_ty],
-          [llvm_v16i32_ty, llvm_v16f32_ty,  llvm_i16_ty,  llvm_i32_ty],
-          [IntrNoMem]>;
-
-  def int_x86_avx512_mask_cvtuqq2pd_512 :
-        GCCBuiltin<"__builtin_ia32_cvtuqq2pd512_mask">,
-          Intrinsic<[llvm_v8f64_ty],
-          [llvm_v8i64_ty, llvm_v8f64_ty,  llvm_i8_ty,  llvm_i32_ty],
-          [IntrNoMem]>;
-
   def int_x86_avx512_mask_cvtuqq2ps_128 :
         GCCBuiltin<"__builtin_ia32_cvtuqq2ps128_mask">,
           Intrinsic<[llvm_v4f32_ty],
           [llvm_v2i64_ty, llvm_v4f32_ty,  llvm_i8_ty],
           [IntrNoMem]>;
 
-  def int_x86_avx512_mask_cvtuqq2ps_256 :
-        GCCBuiltin<"__builtin_ia32_cvtuqq2ps256_mask">,
-          Intrinsic<[llvm_v4f32_ty],
-          [llvm_v4i64_ty, llvm_v4f32_ty,  llvm_i8_ty],
-          [IntrNoMem]>;
-
-  def int_x86_avx512_mask_cvtuqq2ps_512 :
-        GCCBuiltin<"__builtin_ia32_cvtuqq2ps512_mask">,
-          Intrinsic<[llvm_v8f32_ty],
-          [llvm_v8i64_ty, llvm_v8f32_ty,  llvm_i8_ty,  llvm_i32_ty],
-          [IntrNoMem]>;
-
   def int_x86_avx512_mask_rndscale_pd_128 : GCCBuiltin<"__builtin_ia32_rndscalepd_128_mask">,
         Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_i32_ty,
                                      llvm_v2f64_ty,  llvm_i8_ty], [IntrNoMem]>;
@@ -3677,18 +3466,6 @@ let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
 }
 // Integer arithmetic ops
 let TargetPrefix = "x86" in {
-  def int_x86_avx512_padds_b_512 : GCCBuiltin<"__builtin_ia32_paddsb512">,
-          Intrinsic<[llvm_v64i8_ty], [llvm_v64i8_ty, llvm_v64i8_ty],
-                    [IntrNoMem]>;
-  def int_x86_avx512_padds_w_512 : GCCBuiltin<"__builtin_ia32_paddsw512">,
-          Intrinsic<[llvm_v32i16_ty], [llvm_v32i16_ty, llvm_v32i16_ty],
-                     [IntrNoMem]>;
-  def int_x86_avx512_psubs_b_512 : GCCBuiltin<"__builtin_ia32_psubsb512">,
-          Intrinsic<[llvm_v64i8_ty], [llvm_v64i8_ty, llvm_v64i8_ty],
-                     [IntrNoMem]>;
-  def int_x86_avx512_psubs_w_512 : GCCBuiltin<"__builtin_ia32_psubsw512">,
-          Intrinsic<[llvm_v32i16_ty], [llvm_v32i16_ty, llvm_v32i16_ty],
-                     [IntrNoMem]>;
   def int_x86_avx512_pmulhu_w_512 : GCCBuiltin<"__builtin_ia32_pmulhuw512">,
               Intrinsic<[llvm_v32i16_ty], [llvm_v32i16_ty,
                          llvm_v32i16_ty], [IntrNoMem, Commutative]>;
@@ -3720,265 +3497,235 @@ let TargetPrefix = "x86" in {
 
 // Gather and Scatter ops
 let TargetPrefix = "x86" in {
-  def int_x86_avx512_gather_dpd_512  : GCCBuiltin<"__builtin_ia32_gathersiv8df">,
+  // NOTE: These are deprecated in favor of the versions that take a vXi1 mask.
+  def int_x86_avx512_gather_dpd_512  :
           Intrinsic<[llvm_v8f64_ty], [llvm_v8f64_ty, llvm_ptr_ty,
                      llvm_v8i32_ty, llvm_i8_ty, llvm_i32_ty],
                     [IntrReadMem, IntrArgMemOnly]>;
-  def int_x86_avx512_gather_dps_512  : GCCBuiltin<"__builtin_ia32_gathersiv16sf">,
+  def int_x86_avx512_gather_dps_512  :
           Intrinsic<[llvm_v16f32_ty], [llvm_v16f32_ty, llvm_ptr_ty,
                      llvm_v16i32_ty, llvm_i16_ty, llvm_i32_ty],
                     [IntrReadMem, IntrArgMemOnly]>;
-  def int_x86_avx512_gather_qpd_512  : GCCBuiltin<"__builtin_ia32_gatherdiv8df">,
+  def int_x86_avx512_gather_qpd_512  :
           Intrinsic<[llvm_v8f64_ty], [llvm_v8f64_ty, llvm_ptr_ty,
                      llvm_v8i64_ty, llvm_i8_ty, llvm_i32_ty],
                     [IntrReadMem, IntrArgMemOnly]>;
-  def int_x86_avx512_gather_qps_512  : GCCBuiltin<"__builtin_ia32_gatherdiv16sf">,
+  def int_x86_avx512_gather_qps_512  :
           Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty, llvm_ptr_ty,
                      llvm_v8i64_ty, llvm_i8_ty, llvm_i32_ty],
                     [IntrReadMem, IntrArgMemOnly]>;
 
 
-  def int_x86_avx512_gather_dpq_512  : GCCBuiltin<"__builtin_ia32_gathersiv8di">,
+  def int_x86_avx512_gather_dpq_512  :
           Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, llvm_ptr_ty,
                      llvm_v8i32_ty, llvm_i8_ty, llvm_i32_ty],
                     [IntrReadMem, IntrArgMemOnly]>;
-  def int_x86_avx512_gather_dpi_512  : GCCBuiltin<"__builtin_ia32_gathersiv16si">,
+  def int_x86_avx512_gather_dpi_512  :
           Intrinsic<[llvm_v16i32_ty], [llvm_v16i32_ty, llvm_ptr_ty,
                      llvm_v16i32_ty, llvm_i16_ty, llvm_i32_ty],
                     [IntrReadMem, IntrArgMemOnly]>;
-  def int_x86_avx512_gather_qpq_512  : GCCBuiltin<"__builtin_ia32_gatherdiv8di">,
+  def int_x86_avx512_gather_qpq_512  :
           Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, llvm_ptr_ty,
                      llvm_v8i64_ty, llvm_i8_ty, llvm_i32_ty],
                     [IntrReadMem, IntrArgMemOnly]>;
-  def int_x86_avx512_gather_qpi_512  : GCCBuiltin<"__builtin_ia32_gatherdiv16si">,
+  def int_x86_avx512_gather_qpi_512  :
           Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty, llvm_ptr_ty,
                      llvm_v8i64_ty, llvm_i8_ty, llvm_i32_ty],
                     [IntrReadMem, IntrArgMemOnly]>;
 
   def int_x86_avx512_gather3div2_df :
-        GCCBuiltin<"__builtin_ia32_gather3div2df">,
           Intrinsic<[llvm_v2f64_ty],
           [llvm_v2f64_ty, llvm_ptr_ty, llvm_v2i64_ty, llvm_i8_ty, llvm_i32_ty],
           [IntrReadMem, IntrArgMemOnly]>;
 
   def int_x86_avx512_gather3div2_di :
-        GCCBuiltin<"__builtin_ia32_gather3div2di">,
           Intrinsic<[llvm_v2i64_ty],
           [llvm_v2i64_ty, llvm_ptr_ty, llvm_v2i64_ty, llvm_i8_ty, llvm_i32_ty],
           [IntrReadMem, IntrArgMemOnly]>;
 
   def int_x86_avx512_gather3div4_df :
-        GCCBuiltin<"__builtin_ia32_gather3div4df">,
           Intrinsic<[llvm_v4f64_ty],
           [llvm_v4f64_ty, llvm_ptr_ty, llvm_v4i64_ty, llvm_i8_ty, llvm_i32_ty],
           [IntrReadMem, IntrArgMemOnly]>;
 
   def int_x86_avx512_gather3div4_di :
-        GCCBuiltin<"__builtin_ia32_gather3div4di">,
           Intrinsic<[llvm_v4i64_ty],
           [llvm_v4i64_ty, llvm_ptr_ty, llvm_v4i64_ty, llvm_i8_ty, llvm_i32_ty],
           [IntrReadMem, IntrArgMemOnly]>;
 
   def int_x86_avx512_gather3div4_sf :
-        GCCBuiltin<"__builtin_ia32_gather3div4sf">,
           Intrinsic<[llvm_v4f32_ty],
           [llvm_v4f32_ty, llvm_ptr_ty, llvm_v2i64_ty, llvm_i8_ty, llvm_i32_ty],
           [IntrReadMem, IntrArgMemOnly]>;
 
   def int_x86_avx512_gather3div4_si :
-        GCCBuiltin<"__builtin_ia32_gather3div4si">,
           Intrinsic<[llvm_v4i32_ty],
           [llvm_v4i32_ty, llvm_ptr_ty, llvm_v2i64_ty, llvm_i8_ty, llvm_i32_ty],
           [IntrReadMem, IntrArgMemOnly]>;
 
   def int_x86_avx512_gather3div8_sf :
-        GCCBuiltin<"__builtin_ia32_gather3div8sf">,
           Intrinsic<[llvm_v4f32_ty],
           [llvm_v4f32_ty, llvm_ptr_ty, llvm_v4i64_ty, llvm_i8_ty, llvm_i32_ty],
           [IntrReadMem, IntrArgMemOnly]>;
 
   def int_x86_avx512_gather3div8_si :
-        GCCBuiltin<"__builtin_ia32_gather3div8si">,
           Intrinsic<[llvm_v4i32_ty],
           [llvm_v4i32_ty, llvm_ptr_ty, llvm_v4i64_ty, llvm_i8_ty, llvm_i32_ty],
           [IntrReadMem, IntrArgMemOnly]>;
 
   def int_x86_avx512_gather3siv2_df :
-        GCCBuiltin<"__builtin_ia32_gather3siv2df">,
           Intrinsic<[llvm_v2f64_ty],
           [llvm_v2f64_ty, llvm_ptr_ty, llvm_v4i32_ty, llvm_i8_ty, llvm_i32_ty],
           [IntrReadMem, IntrArgMemOnly]>;
 
   def int_x86_avx512_gather3siv2_di :
-        GCCBuiltin<"__builtin_ia32_gather3siv2di">,
           Intrinsic<[llvm_v2i64_ty],
           [llvm_v2i64_ty, llvm_ptr_ty, llvm_v4i32_ty, llvm_i8_ty, llvm_i32_ty],
           [IntrReadMem, IntrArgMemOnly]>;
 
   def int_x86_avx512_gather3siv4_df :
-        GCCBuiltin<"__builtin_ia32_gather3siv4df">,
           Intrinsic<[llvm_v4f64_ty],
           [llvm_v4f64_ty, llvm_ptr_ty, llvm_v4i32_ty, llvm_i8_ty, llvm_i32_ty],
           [IntrReadMem, IntrArgMemOnly]>;
 
   def int_x86_avx512_gather3siv4_di :
-        GCCBuiltin<"__builtin_ia32_gather3siv4di">,
           Intrinsic<[llvm_v4i64_ty],
           [llvm_v4i64_ty, llvm_ptr_ty, llvm_v4i32_ty, llvm_i8_ty, llvm_i32_ty],
           [IntrReadMem, IntrArgMemOnly]>;
 
   def int_x86_avx512_gather3siv4_sf :
-        GCCBuiltin<"__builtin_ia32_gather3siv4sf">,
           Intrinsic<[llvm_v4f32_ty],
           [llvm_v4f32_ty, llvm_ptr_ty, llvm_v4i32_ty, llvm_i8_ty, llvm_i32_ty],
           [IntrReadMem, IntrArgMemOnly]>;
 
   def int_x86_avx512_gather3siv4_si :
-        GCCBuiltin<"__builtin_ia32_gather3siv4si">,
           Intrinsic<[llvm_v4i32_ty],
           [llvm_v4i32_ty, llvm_ptr_ty, llvm_v4i32_ty, llvm_i8_ty, llvm_i32_ty],
           [IntrReadMem, IntrArgMemOnly]>;
 
   def int_x86_avx512_gather3siv8_sf :
-        GCCBuiltin<"__builtin_ia32_gather3siv8sf">,
           Intrinsic<[llvm_v8f32_ty],
           [llvm_v8f32_ty, llvm_ptr_ty, llvm_v8i32_ty, llvm_i8_ty, llvm_i32_ty],
           [IntrReadMem, IntrArgMemOnly]>;
 
   def int_x86_avx512_gather3siv8_si :
-        GCCBuiltin<"__builtin_ia32_gather3siv8si">,
           Intrinsic<[llvm_v8i32_ty],
           [llvm_v8i32_ty, llvm_ptr_ty, llvm_v8i32_ty, llvm_i8_ty, llvm_i32_ty],
           [IntrReadMem, IntrArgMemOnly]>;
 
 // scatter
-  def int_x86_avx512_scatter_dpd_512  : GCCBuiltin<"__builtin_ia32_scattersiv8df">,
+  // NOTE: These are deprecated in favor of the versions that take a vXi1 mask.
+  def int_x86_avx512_scatter_dpd_512  :
           Intrinsic<[], [llvm_ptr_ty, llvm_i8_ty,
                         llvm_v8i32_ty, llvm_v8f64_ty, llvm_i32_ty],
                     [IntrArgMemOnly]>;
-  def int_x86_avx512_scatter_dps_512  : GCCBuiltin<"__builtin_ia32_scattersiv16sf">,
+  def int_x86_avx512_scatter_dps_512  :
           Intrinsic<[], [llvm_ptr_ty, llvm_i16_ty,
                        llvm_v16i32_ty, llvm_v16f32_ty, llvm_i32_ty],
                     [IntrArgMemOnly]>;
-  def int_x86_avx512_scatter_qpd_512  : GCCBuiltin<"__builtin_ia32_scatterdiv8df">,
+  def int_x86_avx512_scatter_qpd_512  :
           Intrinsic<[], [llvm_ptr_ty, llvm_i8_ty,
                      llvm_v8i64_ty, llvm_v8f64_ty, llvm_i32_ty],
                     [IntrArgMemOnly]>;
-  def int_x86_avx512_scatter_qps_512  : GCCBuiltin<"__builtin_ia32_scatterdiv16sf">,
+  def int_x86_avx512_scatter_qps_512  :
           Intrinsic<[], [llvm_ptr_ty, llvm_i8_ty,
                      llvm_v8i64_ty, llvm_v8f32_ty, llvm_i32_ty],
                     [IntrArgMemOnly]>;
 
 
-  def int_x86_avx512_scatter_dpq_512  : GCCBuiltin<"__builtin_ia32_scattersiv8di">,
+  def int_x86_avx512_scatter_dpq_512  :
           Intrinsic<[], [llvm_ptr_ty, llvm_i8_ty,
                          llvm_v8i32_ty, llvm_v8i64_ty, llvm_i32_ty],
                     [IntrArgMemOnly]>;
-  def int_x86_avx512_scatter_dpi_512  : GCCBuiltin<"__builtin_ia32_scattersiv16si">,
+  def int_x86_avx512_scatter_dpi_512  :
           Intrinsic<[], [llvm_ptr_ty, llvm_i16_ty,
                      llvm_v16i32_ty, llvm_v16i32_ty, llvm_i32_ty],
                     [IntrArgMemOnly]>;
-  def int_x86_avx512_scatter_qpq_512  : GCCBuiltin<"__builtin_ia32_scatterdiv8di">,
+  def int_x86_avx512_scatter_qpq_512  :
           Intrinsic<[], [llvm_ptr_ty, llvm_i8_ty,llvm_v8i64_ty, llvm_v8i64_ty,
                          llvm_i32_ty],
                     [IntrArgMemOnly]>;
-  def int_x86_avx512_scatter_qpi_512  : GCCBuiltin<"__builtin_ia32_scatterdiv16si">,
+  def int_x86_avx512_scatter_qpi_512  :
           Intrinsic<[], [llvm_ptr_ty, llvm_i8_ty, llvm_v8i64_ty, llvm_v8i32_ty,
                          llvm_i32_ty],
                     [IntrArgMemOnly]>;
 
   def int_x86_avx512_scatterdiv2_df :
-       GCCBuiltin<"__builtin_ia32_scatterdiv2df">,
         Intrinsic<[],
         [llvm_ptr_ty, llvm_i8_ty, llvm_v2i64_ty, llvm_v2f64_ty, llvm_i32_ty],
         [IntrArgMemOnly]>;
 
   def int_x86_avx512_scatterdiv2_di :
-        GCCBuiltin<"__builtin_ia32_scatterdiv2di">,
           Intrinsic<[],
           [llvm_ptr_ty, llvm_i8_ty, llvm_v2i64_ty, llvm_v2i64_ty, llvm_i32_ty],
           [IntrArgMemOnly]>;
 
   def int_x86_avx512_scatterdiv4_df :
-        GCCBuiltin<"__builtin_ia32_scatterdiv4df">,
           Intrinsic<[],
           [llvm_ptr_ty, llvm_i8_ty, llvm_v4i64_ty, llvm_v4f64_ty, llvm_i32_ty],
           [IntrArgMemOnly]>;
 
   def int_x86_avx512_scatterdiv4_di :
-        GCCBuiltin<"__builtin_ia32_scatterdiv4di">,
           Intrinsic<[],
           [llvm_ptr_ty, llvm_i8_ty, llvm_v4i64_ty, llvm_v4i64_ty, llvm_i32_ty],
           [IntrArgMemOnly]>;
 
   def int_x86_avx512_scatterdiv4_sf :
-        GCCBuiltin<"__builtin_ia32_scatterdiv4sf">,
           Intrinsic<[],
           [llvm_ptr_ty, llvm_i8_ty, llvm_v2i64_ty, llvm_v4f32_ty, llvm_i32_ty],
           [IntrArgMemOnly]>;
 
   def int_x86_avx512_scatterdiv4_si :
-        GCCBuiltin<"__builtin_ia32_scatterdiv4si">,
           Intrinsic<[],
           [llvm_ptr_ty, llvm_i8_ty, llvm_v2i64_ty, llvm_v4i32_ty, llvm_i32_ty],
           [IntrArgMemOnly]>;
 
   def int_x86_avx512_scatterdiv8_sf :
-        GCCBuiltin<"__builtin_ia32_scatterdiv8sf">,
           Intrinsic<[],
           [llvm_ptr_ty, llvm_i8_ty, llvm_v4i64_ty, llvm_v4f32_ty, llvm_i32_ty],
           [IntrArgMemOnly]>;
 
   def int_x86_avx512_scatterdiv8_si :
-        GCCBuiltin<"__builtin_ia32_scatterdiv8si">,
           Intrinsic<[],
           [llvm_ptr_ty, llvm_i8_ty, llvm_v4i64_ty, llvm_v4i32_ty, llvm_i32_ty],
           [IntrArgMemOnly]>;
 
   def int_x86_avx512_scattersiv2_df :
-        GCCBuiltin<"__builtin_ia32_scattersiv2df">,
           Intrinsic<[],
           [llvm_ptr_ty, llvm_i8_ty, llvm_v4i32_ty, llvm_v2f64_ty, llvm_i32_ty],
           [IntrArgMemOnly]>;
 
   def int_x86_avx512_scattersiv2_di :
-        GCCBuiltin<"__builtin_ia32_scattersiv2di">,
           Intrinsic<[],
           [llvm_ptr_ty, llvm_i8_ty, llvm_v4i32_ty, llvm_v2i64_ty, llvm_i32_ty],
           [IntrArgMemOnly]>;
 
   def int_x86_avx512_scattersiv4_df :
-        GCCBuiltin<"__builtin_ia32_scattersiv4df">,
           Intrinsic<[],
           [llvm_ptr_ty, llvm_i8_ty, llvm_v4i32_ty, llvm_v4f64_ty, llvm_i32_ty],
           [IntrArgMemOnly]>;
 
   def int_x86_avx512_scattersiv4_di :
-        GCCBuiltin<"__builtin_ia32_scattersiv4di">,
           Intrinsic<[],
           [llvm_ptr_ty, llvm_i8_ty, llvm_v4i32_ty, llvm_v4i64_ty, llvm_i32_ty],
           [IntrArgMemOnly]>;
 
   def int_x86_avx512_scattersiv4_sf :
-        GCCBuiltin<"__builtin_ia32_scattersiv4sf">,
           Intrinsic<[],
           [llvm_ptr_ty, llvm_i8_ty, llvm_v4i32_ty, llvm_v4f32_ty, llvm_i32_ty],
           [IntrArgMemOnly]>;
 
   def int_x86_avx512_scattersiv4_si :
-        GCCBuiltin<"__builtin_ia32_scattersiv4si">,
           Intrinsic<[],
           [llvm_ptr_ty, llvm_i8_ty, llvm_v4i32_ty, llvm_v4i32_ty, llvm_i32_ty],
           [IntrArgMemOnly]>;
 
   def int_x86_avx512_scattersiv8_sf :
-        GCCBuiltin<"__builtin_ia32_scattersiv8sf">,
           Intrinsic<[],
           [llvm_ptr_ty, llvm_i8_ty, llvm_v8i32_ty, llvm_v8f32_ty, llvm_i32_ty],
           [IntrArgMemOnly]>;
 
   def int_x86_avx512_scattersiv8_si :
-        GCCBuiltin<"__builtin_ia32_scattersiv8si">,
           Intrinsic<[],
           [llvm_ptr_ty, llvm_i8_ty, llvm_v8i32_ty, llvm_v8i32_ty, llvm_i32_ty],
           [IntrArgMemOnly]>;
@@ -4012,40 +3759,261 @@ let TargetPrefix = "x86" in {
                      llvm_i32_ty, llvm_i32_ty], [IntrArgMemOnly]>;
 }
 
-// AVX-512 conflict detection instruction
-// Instructions that count the number of leading zero bits
+// AVX512 gather/scatter intrinsics that use vXi1 masks.
 let TargetPrefix = "x86" in {
-  def int_x86_avx512_mask_conflict_d_128 :
-          GCCBuiltin<"__builtin_ia32_vpconflictsi_128_mask">,
+  def int_x86_avx512_mask_gather_dpd_512  :
+          Intrinsic<[llvm_v8f64_ty], [llvm_v8f64_ty, llvm_ptr_ty,
+                     llvm_v8i32_ty, llvm_v8i1_ty, llvm_i32_ty],
+                    [IntrReadMem, IntrArgMemOnly]>;
+  def int_x86_avx512_mask_gather_dps_512  :
+          Intrinsic<[llvm_v16f32_ty], [llvm_v16f32_ty, llvm_ptr_ty,
+                     llvm_v16i32_ty, llvm_v16i1_ty, llvm_i32_ty],
+                    [IntrReadMem, IntrArgMemOnly]>;
+  def int_x86_avx512_mask_gather_qpd_512  :
+          Intrinsic<[llvm_v8f64_ty], [llvm_v8f64_ty, llvm_ptr_ty,
+                     llvm_v8i64_ty, llvm_v8i1_ty, llvm_i32_ty],
+                    [IntrReadMem, IntrArgMemOnly]>;
+  def int_x86_avx512_mask_gather_qps_512  :
+          Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty, llvm_ptr_ty,
+                     llvm_v8i64_ty, llvm_v8i1_ty, llvm_i32_ty],
+                    [IntrReadMem, IntrArgMemOnly]>;
+
+
+  def int_x86_avx512_mask_gather_dpq_512  :
+          Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, llvm_ptr_ty,
+                     llvm_v8i32_ty, llvm_v8i1_ty, llvm_i32_ty],
+                    [IntrReadMem, IntrArgMemOnly]>;
+  def int_x86_avx512_mask_gather_dpi_512  :
+          Intrinsic<[llvm_v16i32_ty], [llvm_v16i32_ty, llvm_ptr_ty,
+                     llvm_v16i32_ty, llvm_v16i1_ty, llvm_i32_ty],
+                    [IntrReadMem, IntrArgMemOnly]>;
+  def int_x86_avx512_mask_gather_qpq_512  :
+          Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, llvm_ptr_ty,
+                     llvm_v8i64_ty, llvm_v8i1_ty, llvm_i32_ty],
+                    [IntrReadMem, IntrArgMemOnly]>;
+  def int_x86_avx512_mask_gather_qpi_512  :
+          Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty, llvm_ptr_ty,
+                     llvm_v8i64_ty, llvm_v8i1_ty, llvm_i32_ty],
+                    [IntrReadMem, IntrArgMemOnly]>;
+
+  def int_x86_avx512_mask_gather3div2_df :
+          Intrinsic<[llvm_v2f64_ty],
+          [llvm_v2f64_ty, llvm_ptr_ty, llvm_v2i64_ty, llvm_v2i1_ty, llvm_i32_ty],
+          [IntrReadMem, IntrArgMemOnly]>;
+
+  def int_x86_avx512_mask_gather3div2_di :
+          Intrinsic<[llvm_v2i64_ty],
+          [llvm_v2i64_ty, llvm_ptr_ty, llvm_v2i64_ty, llvm_v2i1_ty, llvm_i32_ty],
+          [IntrReadMem, IntrArgMemOnly]>;
+
+  def int_x86_avx512_mask_gather3div4_df :
+          Intrinsic<[llvm_v4f64_ty],
+          [llvm_v4f64_ty, llvm_ptr_ty, llvm_v4i64_ty, llvm_v4i1_ty, llvm_i32_ty],
+          [IntrReadMem, IntrArgMemOnly]>;
+
+  def int_x86_avx512_mask_gather3div4_di :
+          Intrinsic<[llvm_v4i64_ty],
+          [llvm_v4i64_ty, llvm_ptr_ty, llvm_v4i64_ty, llvm_v4i1_ty, llvm_i32_ty],
+          [IntrReadMem, IntrArgMemOnly]>;
+
+  def int_x86_avx512_mask_gather3div4_sf :
+          Intrinsic<[llvm_v4f32_ty],
+          [llvm_v4f32_ty, llvm_ptr_ty, llvm_v2i64_ty, llvm_v2i1_ty, llvm_i32_ty],
+          [IntrReadMem, IntrArgMemOnly]>;
+
+  def int_x86_avx512_mask_gather3div4_si :
           Intrinsic<[llvm_v4i32_ty],
-                    [llvm_v4i32_ty, llvm_v4i32_ty, llvm_i8_ty],
-                    [IntrNoMem]>;
-  def int_x86_avx512_mask_conflict_d_256 :
-          GCCBuiltin<"__builtin_ia32_vpconflictsi_256_mask">,
-          Intrinsic<[llvm_v8i32_ty],
-                    [llvm_v8i32_ty, llvm_v8i32_ty, llvm_i8_ty],
-                    [IntrNoMem]>;
-  def int_x86_avx512_mask_conflict_d_512 :
-          GCCBuiltin<"__builtin_ia32_vpconflictsi_512_mask">,
-          Intrinsic<[llvm_v16i32_ty],
-                    [llvm_v16i32_ty, llvm_v16i32_ty, llvm_i16_ty],
-                    [IntrNoMem]>;
+          [llvm_v4i32_ty, llvm_ptr_ty, llvm_v2i64_ty, llvm_v2i1_ty, llvm_i32_ty],
+          [IntrReadMem, IntrArgMemOnly]>;
+
+  def int_x86_avx512_mask_gather3div8_sf :
+          Intrinsic<[llvm_v4f32_ty],
+          [llvm_v4f32_ty, llvm_ptr_ty, llvm_v4i64_ty, llvm_v4i1_ty, llvm_i32_ty],
+          [IntrReadMem, IntrArgMemOnly]>;
+
+  def int_x86_avx512_mask_gather3div8_si :
+          Intrinsic<[llvm_v4i32_ty],
+          [llvm_v4i32_ty, llvm_ptr_ty, llvm_v4i64_ty, llvm_v4i1_ty, llvm_i32_ty],
+          [IntrReadMem, IntrArgMemOnly]>;
+
+  def int_x86_avx512_mask_gather3siv2_df :
+          Intrinsic<[llvm_v2f64_ty],
+          [llvm_v2f64_ty, llvm_ptr_ty, llvm_v4i32_ty, llvm_v2i1_ty, llvm_i32_ty],
+          [IntrReadMem, IntrArgMemOnly]>;
 
-  def int_x86_avx512_mask_conflict_q_128 :
-          GCCBuiltin<"__builtin_ia32_vpconflictdi_128_mask">,
+  def int_x86_avx512_mask_gather3siv2_di :
           Intrinsic<[llvm_v2i64_ty],
-                    [llvm_v2i64_ty, llvm_v2i64_ty, llvm_i8_ty],
-                    [IntrNoMem]>;
-  def int_x86_avx512_mask_conflict_q_256 :
-          GCCBuiltin<"__builtin_ia32_vpconflictdi_256_mask">,
+          [llvm_v2i64_ty, llvm_ptr_ty, llvm_v4i32_ty, llvm_v2i1_ty, llvm_i32_ty],
+          [IntrReadMem, IntrArgMemOnly]>;
+
+  def int_x86_avx512_mask_gather3siv4_df :
+          Intrinsic<[llvm_v4f64_ty],
+          [llvm_v4f64_ty, llvm_ptr_ty, llvm_v4i32_ty, llvm_v4i1_ty, llvm_i32_ty],
+          [IntrReadMem, IntrArgMemOnly]>;
+
+  def int_x86_avx512_mask_gather3siv4_di :
           Intrinsic<[llvm_v4i64_ty],
-                    [llvm_v4i64_ty, llvm_v4i64_ty, llvm_i8_ty],
-                    [IntrNoMem]>;
-  def int_x86_avx512_mask_conflict_q_512 :
-          GCCBuiltin<"__builtin_ia32_vpconflictdi_512_mask">,
-          Intrinsic<[llvm_v8i64_ty],
-                    [llvm_v8i64_ty, llvm_v8i64_ty, llvm_i8_ty],
-                    [IntrNoMem]>;
+          [llvm_v4i64_ty, llvm_ptr_ty, llvm_v4i32_ty, llvm_v4i1_ty, llvm_i32_ty],
+          [IntrReadMem, IntrArgMemOnly]>;
+
+  def int_x86_avx512_mask_gather3siv4_sf :
+          Intrinsic<[llvm_v4f32_ty],
+          [llvm_v4f32_ty, llvm_ptr_ty, llvm_v4i32_ty, llvm_v4i1_ty, llvm_i32_ty],
+          [IntrReadMem, IntrArgMemOnly]>;
+
+  def int_x86_avx512_mask_gather3siv4_si :
+          Intrinsic<[llvm_v4i32_ty],
+          [llvm_v4i32_ty, llvm_ptr_ty, llvm_v4i32_ty, llvm_v4i1_ty, llvm_i32_ty],
+          [IntrReadMem, IntrArgMemOnly]>;
+
+  def int_x86_avx512_mask_gather3siv8_sf :
+          Intrinsic<[llvm_v8f32_ty],
+          [llvm_v8f32_ty, llvm_ptr_ty, llvm_v8i32_ty, llvm_v8i1_ty, llvm_i32_ty],
+          [IntrReadMem, IntrArgMemOnly]>;
+
+  def int_x86_avx512_mask_gather3siv8_si :
+          Intrinsic<[llvm_v8i32_ty],
+          [llvm_v8i32_ty, llvm_ptr_ty, llvm_v8i32_ty, llvm_v8i1_ty, llvm_i32_ty],
+          [IntrReadMem, IntrArgMemOnly]>;
+
+  def int_x86_avx512_mask_scatter_dpd_512  :
+          Intrinsic<[], [llvm_ptr_ty, llvm_v8i1_ty,
+                        llvm_v8i32_ty, llvm_v8f64_ty, llvm_i32_ty],
+                    [IntrArgMemOnly]>;
+  def int_x86_avx512_mask_scatter_dps_512  :
+          Intrinsic<[], [llvm_ptr_ty, llvm_v16i1_ty,
+                       llvm_v16i32_ty, llvm_v16f32_ty, llvm_i32_ty],
+                    [IntrArgMemOnly]>;
+  def int_x86_avx512_mask_scatter_qpd_512  :
+          Intrinsic<[], [llvm_ptr_ty, llvm_v8i1_ty,
+                     llvm_v8i64_ty, llvm_v8f64_ty, llvm_i32_ty],
+                    [IntrArgMemOnly]>;
+  def int_x86_avx512_mask_scatter_qps_512  :
+          Intrinsic<[], [llvm_ptr_ty, llvm_v8i1_ty,
+                     llvm_v8i64_ty, llvm_v8f32_ty, llvm_i32_ty],
+                    [IntrArgMemOnly]>;
+
+
+  def int_x86_avx512_mask_scatter_dpq_512  :
+          Intrinsic<[], [llvm_ptr_ty, llvm_v8i1_ty,
+                         llvm_v8i32_ty, llvm_v8i64_ty, llvm_i32_ty],
+                    [IntrArgMemOnly]>;
+  def int_x86_avx512_mask_scatter_dpi_512  :
+          Intrinsic<[], [llvm_ptr_ty, llvm_v16i1_ty,
+                     llvm_v16i32_ty, llvm_v16i32_ty, llvm_i32_ty],
+                    [IntrArgMemOnly]>;
+  def int_x86_avx512_mask_scatter_qpq_512  :
+          Intrinsic<[], [llvm_ptr_ty, llvm_v8i1_ty,llvm_v8i64_ty, llvm_v8i64_ty,
+                         llvm_i32_ty],
+                    [IntrArgMemOnly]>;
+  def int_x86_avx512_mask_scatter_qpi_512  :
+          Intrinsic<[], [llvm_ptr_ty, llvm_v8i1_ty, llvm_v8i64_ty, llvm_v8i32_ty,
+                         llvm_i32_ty],
+                    [IntrArgMemOnly]>;
+
+  def int_x86_avx512_mask_scatterdiv2_df :
+        Intrinsic<[],
+        [llvm_ptr_ty, llvm_v2i1_ty, llvm_v2i64_ty, llvm_v2f64_ty, llvm_i32_ty],
+        [IntrArgMemOnly]>;
+
+  def int_x86_avx512_mask_scatterdiv2_di :
+          Intrinsic<[],
+          [llvm_ptr_ty, llvm_v2i1_ty, llvm_v2i64_ty, llvm_v2i64_ty, llvm_i32_ty],
+          [IntrArgMemOnly]>;
+
+  def int_x86_avx512_mask_scatterdiv4_df :
+          Intrinsic<[],
+          [llvm_ptr_ty, llvm_v4i1_ty, llvm_v4i64_ty, llvm_v4f64_ty, llvm_i32_ty],
+          [IntrArgMemOnly]>;
+
+  def int_x86_avx512_mask_scatterdiv4_di :
+          Intrinsic<[],
+          [llvm_ptr_ty, llvm_v4i1_ty, llvm_v4i64_ty, llvm_v4i64_ty, llvm_i32_ty],
+          [IntrArgMemOnly]>;
+
+  def int_x86_avx512_mask_scatterdiv4_sf :
+          Intrinsic<[],
+          [llvm_ptr_ty, llvm_v2i1_ty, llvm_v2i64_ty, llvm_v4f32_ty, llvm_i32_ty],
+          [IntrArgMemOnly]>;
+
+  def int_x86_avx512_mask_scatterdiv4_si :
+          Intrinsic<[],
+          [llvm_ptr_ty, llvm_v2i1_ty, llvm_v2i64_ty, llvm_v4i32_ty, llvm_i32_ty],
+          [IntrArgMemOnly]>;
+
+  def int_x86_avx512_mask_scatterdiv8_sf :
+          Intrinsic<[],
+          [llvm_ptr_ty, llvm_v4i1_ty, llvm_v4i64_ty, llvm_v4f32_ty, llvm_i32_ty],
+          [IntrArgMemOnly]>;
+
+  def int_x86_avx512_mask_scatterdiv8_si :
+          Intrinsic<[],
+          [llvm_ptr_ty, llvm_v4i1_ty, llvm_v4i64_ty, llvm_v4i32_ty, llvm_i32_ty],
+          [IntrArgMemOnly]>;
+
+  def int_x86_avx512_mask_scattersiv2_df :
+          Intrinsic<[],
+          [llvm_ptr_ty, llvm_v2i1_ty, llvm_v4i32_ty, llvm_v2f64_ty, llvm_i32_ty],
+          [IntrArgMemOnly]>;
+
+  def int_x86_avx512_mask_scattersiv2_di :
+          Intrinsic<[],
+          [llvm_ptr_ty, llvm_v2i1_ty, llvm_v4i32_ty, llvm_v2i64_ty, llvm_i32_ty],
+          [IntrArgMemOnly]>;
+
+  def int_x86_avx512_mask_scattersiv4_df :
+          Intrinsic<[],
+          [llvm_ptr_ty, llvm_v4i1_ty, llvm_v4i32_ty, llvm_v4f64_ty, llvm_i32_ty],
+          [IntrArgMemOnly]>;
+
+  def int_x86_avx512_mask_scattersiv4_di :
+          Intrinsic<[],
+          [llvm_ptr_ty, llvm_v4i1_ty, llvm_v4i32_ty, llvm_v4i64_ty, llvm_i32_ty],
+          [IntrArgMemOnly]>;
+
+  def int_x86_avx512_mask_scattersiv4_sf :
+          Intrinsic<[],
+          [llvm_ptr_ty, llvm_v4i1_ty, llvm_v4i32_ty, llvm_v4f32_ty, llvm_i32_ty],
+          [IntrArgMemOnly]>;
+
+  def int_x86_avx512_mask_scattersiv4_si :
+          Intrinsic<[],
+          [llvm_ptr_ty, llvm_v4i1_ty, llvm_v4i32_ty, llvm_v4i32_ty, llvm_i32_ty],
+          [IntrArgMemOnly]>;
+
+  def int_x86_avx512_mask_scattersiv8_sf :
+          Intrinsic<[],
+          [llvm_ptr_ty, llvm_v8i1_ty, llvm_v8i32_ty, llvm_v8f32_ty, llvm_i32_ty],
+          [IntrArgMemOnly]>;
+
+  def int_x86_avx512_mask_scattersiv8_si :
+          Intrinsic<[],
+          [llvm_ptr_ty, llvm_v8i1_ty, llvm_v8i32_ty, llvm_v8i32_ty, llvm_i32_ty],
+          [IntrArgMemOnly]>;
+}
+
+// AVX-512 conflict detection instruction
+// Instructions that count the number of leading zero bits
+let TargetPrefix = "x86" in {
+  def int_x86_avx512_conflict_d_128 :
+          GCCBuiltin<"__builtin_ia32_vpconflictsi_128">,
+          Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty], [IntrNoMem]>;
+  def int_x86_avx512_conflict_d_256 :
+          GCCBuiltin<"__builtin_ia32_vpconflictsi_256">,
+          Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty], [IntrNoMem]>;
+  def int_x86_avx512_conflict_d_512 :
+          GCCBuiltin<"__builtin_ia32_vpconflictsi_512">,
+          Intrinsic<[llvm_v16i32_ty], [llvm_v16i32_ty], [IntrNoMem]>;
+
+  def int_x86_avx512_conflict_q_128 :
+          GCCBuiltin<"__builtin_ia32_vpconflictdi_128">,
+          Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty], [IntrNoMem]>;
+  def int_x86_avx512_conflict_q_256 :
+          GCCBuiltin<"__builtin_ia32_vpconflictdi_256">,
+          Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty], [IntrNoMem]>;
+  def int_x86_avx512_conflict_q_512 :
+          GCCBuiltin<"__builtin_ia32_vpconflictdi_512">,
+          Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty], [IntrNoMem]>;
 }
 
 // Compares
@@ -4061,387 +4029,16 @@ let TargetPrefix = "x86" in {
 
 // Compress, Expand
 let TargetPrefix = "x86" in {
-  def int_x86_avx512_mask_compress_ps_512 :
-                             GCCBuiltin<"__builtin_ia32_compresssf512_mask">,
-        Intrinsic<[llvm_v16f32_ty], [llvm_v16f32_ty, llvm_v16f32_ty,
-                   llvm_i16_ty], [IntrNoMem]>;
-  def int_x86_avx512_mask_compress_pd_512 :
-                             GCCBuiltin<"__builtin_ia32_compressdf512_mask">,
-        Intrinsic<[llvm_v8f64_ty], [llvm_v8f64_ty, llvm_v8f64_ty,
-                   llvm_i8_ty], [IntrNoMem]>;
-  def int_x86_avx512_mask_compress_ps_256 :
-                             GCCBuiltin<"__builtin_ia32_compresssf256_mask">,
-        Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty, llvm_v8f32_ty,
-                   llvm_i8_ty], [IntrNoMem]>;
-  def int_x86_avx512_mask_compress_pd_256 :
-                             GCCBuiltin<"__builtin_ia32_compressdf256_mask">,
-        Intrinsic<[llvm_v4f64_ty], [llvm_v4f64_ty, llvm_v4f64_ty,
-                   llvm_i8_ty], [IntrNoMem]>;
-  def int_x86_avx512_mask_compress_ps_128 :
-                             GCCBuiltin<"__builtin_ia32_compresssf128_mask">,
-        Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty,
-                   llvm_i8_ty], [IntrNoMem]>;
-  def int_x86_avx512_mask_compress_pd_128 :
-                             GCCBuiltin<"__builtin_ia32_compressdf128_mask">,
-        Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty,
-                   llvm_i8_ty], [IntrNoMem]>;
-
-  def int_x86_avx512_mask_compress_d_512 :
-                             GCCBuiltin<"__builtin_ia32_compresssi512_mask">,
-        Intrinsic<[llvm_v16i32_ty], [llvm_v16i32_ty, llvm_v16i32_ty,
-                   llvm_i16_ty], [IntrNoMem]>;
-  def int_x86_avx512_mask_compress_q_512 :
-                             GCCBuiltin<"__builtin_ia32_compressdi512_mask">,
-        Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, llvm_v8i64_ty,
-                   llvm_i8_ty], [IntrNoMem]>;
-  def int_x86_avx512_mask_compress_d_256 :
-                             GCCBuiltin<"__builtin_ia32_compresssi256_mask">,
-        Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty, llvm_v8i32_ty,
-                   llvm_i8_ty], [IntrNoMem]>;
-  def int_x86_avx512_mask_compress_q_256 :
-                             GCCBuiltin<"__builtin_ia32_compressdi256_mask">,
-        Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty, llvm_v4i64_ty,
-                   llvm_i8_ty], [IntrNoMem]>;
-  def int_x86_avx512_mask_compress_d_128 :
-                             GCCBuiltin<"__builtin_ia32_compresssi128_mask">,
-        Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty,
-                   llvm_i8_ty], [IntrNoMem]>;
-  def int_x86_avx512_mask_compress_q_128 :
-                             GCCBuiltin<"__builtin_ia32_compressdi128_mask">,
-        Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty,
-                   llvm_i8_ty], [IntrNoMem]>;
-
-  def int_x86_avx512_mask_compress_b_512 :
-                             GCCBuiltin<"__builtin_ia32_compressqi512_mask">,
-        Intrinsic<[llvm_v64i8_ty], [llvm_v64i8_ty, llvm_v64i8_ty,
-                   llvm_i64_ty], [IntrNoMem]>;
-  def int_x86_avx512_mask_compress_w_512 :
-                             GCCBuiltin<"__builtin_ia32_compresshi512_mask">,
-        Intrinsic<[llvm_v32i16_ty], [llvm_v32i16_ty, llvm_v32i16_ty,
-                   llvm_i32_ty], [IntrNoMem]>;
-  def int_x86_avx512_mask_compress_b_256 :
-                             GCCBuiltin<"__builtin_ia32_compressqi256_mask">,
-        Intrinsic<[llvm_v32i8_ty], [llvm_v32i8_ty, llvm_v32i8_ty,
-                   llvm_i32_ty], [IntrNoMem]>;
-  def int_x86_avx512_mask_compress_w_256 :
-                             GCCBuiltin<"__builtin_ia32_compresshi256_mask">,
-        Intrinsic<[llvm_v16i16_ty], [llvm_v16i16_ty, llvm_v16i16_ty,
-                   llvm_i16_ty], [IntrNoMem]>;
-  def int_x86_avx512_mask_compress_b_128 :
-                             GCCBuiltin<"__builtin_ia32_compressqi128_mask">,
-        Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty,
-                   llvm_i16_ty], [IntrNoMem]>;
-  def int_x86_avx512_mask_compress_w_128 :
-                             GCCBuiltin<"__builtin_ia32_compresshi128_mask">,
-        Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty,
-                   llvm_i8_ty], [IntrNoMem]>;
-
-// expand
-  def int_x86_avx512_mask_expand_ps_512 :
-                             GCCBuiltin<"__builtin_ia32_expandsf512_mask">,
-        Intrinsic<[llvm_v16f32_ty], [llvm_v16f32_ty, llvm_v16f32_ty,
-                   llvm_i16_ty], [IntrNoMem]>;
-  def int_x86_avx512_mask_expand_pd_512 :
-                             GCCBuiltin<"__builtin_ia32_expanddf512_mask">,
-        Intrinsic<[llvm_v8f64_ty], [llvm_v8f64_ty, llvm_v8f64_ty,
-                   llvm_i8_ty], [IntrNoMem]>;
-  def int_x86_avx512_mask_expand_ps_256 :
-                             GCCBuiltin<"__builtin_ia32_expandsf256_mask">,
-        Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty, llvm_v8f32_ty,
-                   llvm_i8_ty], [IntrNoMem]>;
-  def int_x86_avx512_mask_expand_pd_256 :
-                             GCCBuiltin<"__builtin_ia32_expanddf256_mask">,
-        Intrinsic<[llvm_v4f64_ty], [llvm_v4f64_ty, llvm_v4f64_ty,
-                   llvm_i8_ty], [IntrNoMem]>;
-  def int_x86_avx512_mask_expand_ps_128 :
-                             GCCBuiltin<"__builtin_ia32_expandsf128_mask">,
-        Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty,
-                   llvm_i8_ty], [IntrNoMem]>;
-  def int_x86_avx512_mask_expand_pd_128 :
-                             GCCBuiltin<"__builtin_ia32_expanddf128_mask">,
-        Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty,
-                   llvm_i8_ty], [IntrNoMem]>;
-
-  def int_x86_avx512_mask_expand_d_512 :
-                             GCCBuiltin<"__builtin_ia32_expandsi512_mask">,
-        Intrinsic<[llvm_v16i32_ty], [llvm_v16i32_ty, llvm_v16i32_ty,
-                   llvm_i16_ty], [IntrNoMem]>;
-  def int_x86_avx512_mask_expand_q_512 :
-                             GCCBuiltin<"__builtin_ia32_expanddi512_mask">,
-        Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, llvm_v8i64_ty,
-                   llvm_i8_ty], [IntrNoMem]>;
-  def int_x86_avx512_mask_expand_d_256 :
-                             GCCBuiltin<"__builtin_ia32_expandsi256_mask">,
-        Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty, llvm_v8i32_ty,
-                   llvm_i8_ty], [IntrNoMem]>;
-  def int_x86_avx512_mask_expand_q_256 :
-                             GCCBuiltin<"__builtin_ia32_expanddi256_mask">,
-        Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty, llvm_v4i64_ty,
-                   llvm_i8_ty], [IntrNoMem]>;
-  def int_x86_avx512_mask_expand_d_128 :
-                             GCCBuiltin<"__builtin_ia32_expandsi128_mask">,
-        Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty,
-                   llvm_i8_ty], [IntrNoMem]>;
-  def int_x86_avx512_mask_expand_q_128 :
-                             GCCBuiltin<"__builtin_ia32_expanddi128_mask">,
-        Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty,
-                   llvm_i8_ty], [IntrNoMem]>;
-
-  def int_x86_avx512_mask_expand_b_512 :
-                            GCCBuiltin<"__builtin_ia32_expandqi512_mask">,
-        Intrinsic<[llvm_v64i8_ty], [llvm_v64i8_ty, llvm_v64i8_ty,
-                   llvm_i64_ty], [IntrNoMem]>;
-  def int_x86_avx512_mask_expand_w_512 :
-                            GCCBuiltin<"__builtin_ia32_expandhi512_mask">,
-        Intrinsic<[llvm_v32i16_ty], [llvm_v32i16_ty, llvm_v32i16_ty,
-                   llvm_i32_ty], [IntrNoMem]>;
-  def int_x86_avx512_mask_expand_b_256 :
-                            GCCBuiltin<"__builtin_ia32_expandqi256_mask">,
-        Intrinsic<[llvm_v32i8_ty], [llvm_v32i8_ty, llvm_v32i8_ty,
-                   llvm_i32_ty], [IntrNoMem]>;
-  def int_x86_avx512_mask_expand_w_256 :
-                            GCCBuiltin<"__builtin_ia32_expandhi256_mask">,
-        Intrinsic<[llvm_v16i16_ty], [llvm_v16i16_ty, llvm_v16i16_ty,
-                   llvm_i16_ty], [IntrNoMem]>;
-  def int_x86_avx512_mask_expand_b_128 :
-                            GCCBuiltin<"__builtin_ia32_expandqi128_mask">,
-        Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty,
-                   llvm_i16_ty], [IntrNoMem]>;
-  def int_x86_avx512_mask_expand_w_128 :
-                            GCCBuiltin<"__builtin_ia32_expandhi128_mask">,
-        Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty,
-                   llvm_i8_ty], [IntrNoMem]>;
-}
-
-// VBMI2 Concat & Shift
-let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
-  def int_x86_avx512_vpshld_q_512 :
-        GCCBuiltin<"__builtin_ia32_vpshldq512">,
-        Intrinsic<[llvm_v8i64_ty],
-                  [llvm_v8i64_ty, llvm_v8i64_ty, llvm_i32_ty], [IntrNoMem]>;
-  def int_x86_avx512_vpshld_q_256 :
-        GCCBuiltin<"__builtin_ia32_vpshldq256">,
-        Intrinsic<[llvm_v4i64_ty],
-                  [llvm_v4i64_ty, llvm_v4i64_ty, llvm_i32_ty], [IntrNoMem]>;
-  def int_x86_avx512_vpshld_q_128 :
-        GCCBuiltin<"__builtin_ia32_vpshldq128">,
-        Intrinsic<[llvm_v2i64_ty],
-                  [llvm_v2i64_ty, llvm_v2i64_ty, llvm_i32_ty], [IntrNoMem]>;
-
-  def int_x86_avx512_vpshld_d_512 :
-        GCCBuiltin<"__builtin_ia32_vpshldd512">,
-        Intrinsic<[llvm_v16i32_ty],
-                  [llvm_v16i32_ty, llvm_v16i32_ty, llvm_i32_ty], [IntrNoMem]>;
-  def int_x86_avx512_vpshld_d_256 :
-        GCCBuiltin<"__builtin_ia32_vpshldd256">,
-        Intrinsic<[llvm_v8i32_ty],
-                  [llvm_v8i32_ty, llvm_v8i32_ty, llvm_i32_ty], [IntrNoMem]>;
-  def int_x86_avx512_vpshld_d_128 :
-        GCCBuiltin<"__builtin_ia32_vpshldd128">,
-        Intrinsic<[llvm_v4i32_ty],
-                  [llvm_v4i32_ty, llvm_v4i32_ty, llvm_i32_ty], [IntrNoMem]>;
-
-  def int_x86_avx512_vpshld_w_512 :
-        GCCBuiltin<"__builtin_ia32_vpshldw512">,
-        Intrinsic<[llvm_v32i16_ty],
-                  [llvm_v32i16_ty, llvm_v32i16_ty, llvm_i32_ty], [IntrNoMem]>;
-  def int_x86_avx512_vpshld_w_256 :
-        GCCBuiltin<"__builtin_ia32_vpshldw256">,
-        Intrinsic<[llvm_v16i16_ty],
-                  [llvm_v16i16_ty, llvm_v16i16_ty, llvm_i32_ty], [IntrNoMem]>;
-  def int_x86_avx512_vpshld_w_128 :
-        GCCBuiltin<"__builtin_ia32_vpshldw128">,
-        Intrinsic<[llvm_v8i16_ty],
-                  [llvm_v8i16_ty, llvm_v8i16_ty, llvm_i32_ty], [IntrNoMem]>;
-
-  def int_x86_avx512_vpshrd_q_512 :
-        GCCBuiltin<"__builtin_ia32_vpshrdq512">,
-        Intrinsic<[llvm_v8i64_ty],
-                  [llvm_v8i64_ty, llvm_v8i64_ty, llvm_i32_ty], [IntrNoMem]>;
-  def int_x86_avx512_vpshrd_q_256 :
-        GCCBuiltin<"__builtin_ia32_vpshrdq256">,
-        Intrinsic<[llvm_v4i64_ty],
-                  [llvm_v4i64_ty, llvm_v4i64_ty, llvm_i32_ty], [IntrNoMem]>;
-  def int_x86_avx512_vpshrd_q_128 :
-        GCCBuiltin<"__builtin_ia32_vpshrdq128">,
-        Intrinsic<[llvm_v2i64_ty],
-                  [llvm_v2i64_ty, llvm_v2i64_ty, llvm_i32_ty], [IntrNoMem]>;
-
-  def int_x86_avx512_vpshrd_d_512 :
-        GCCBuiltin<"__builtin_ia32_vpshrdd512">,
-        Intrinsic<[llvm_v16i32_ty],
-                  [llvm_v16i32_ty, llvm_v16i32_ty, llvm_i32_ty], [IntrNoMem]>;
-  def int_x86_avx512_vpshrd_d_256 :
-        GCCBuiltin<"__builtin_ia32_vpshrdd256">,
-        Intrinsic<[llvm_v8i32_ty],
-                  [llvm_v8i32_ty, llvm_v8i32_ty, llvm_i32_ty], [IntrNoMem]>;
-  def int_x86_avx512_vpshrd_d_128 :
-        GCCBuiltin<"__builtin_ia32_vpshrdd128">,
-        Intrinsic<[llvm_v4i32_ty],
-                  [llvm_v4i32_ty, llvm_v4i32_ty, llvm_i32_ty], [IntrNoMem]>;
-
-  def int_x86_avx512_vpshrd_w_512 :
-        GCCBuiltin<"__builtin_ia32_vpshrdw512">,
-        Intrinsic<[llvm_v32i16_ty],
-                  [llvm_v32i16_ty, llvm_v32i16_ty, llvm_i32_ty], [IntrNoMem]>;
-  def int_x86_avx512_vpshrd_w_256 :
-        GCCBuiltin<"__builtin_ia32_vpshrdw256">,
-        Intrinsic<[llvm_v16i16_ty],
-                  [llvm_v16i16_ty, llvm_v16i16_ty, llvm_i32_ty], [IntrNoMem]>;
-  def int_x86_avx512_vpshrd_w_128 :
-        GCCBuiltin<"__builtin_ia32_vpshrdw128">,
-        Intrinsic<[llvm_v8i16_ty],
-                  [llvm_v8i16_ty, llvm_v8i16_ty, llvm_i32_ty], [IntrNoMem]>;
-
-  def int_x86_avx512_mask_vpshldv_w_128 :
-        GCCBuiltin<"__builtin_ia32_vpshldvw128_mask">,
-        Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty,
-                   llvm_v8i16_ty, llvm_i8_ty], [IntrNoMem]>;
-  def int_x86_avx512_maskz_vpshldv_w_128 :
-        GCCBuiltin<"__builtin_ia32_vpshldvw128_maskz">,
-        Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty,
-                   llvm_v8i16_ty, llvm_i8_ty], [IntrNoMem]>;
-  def int_x86_avx512_mask_vpshldv_w_256 :
-        GCCBuiltin<"__builtin_ia32_vpshldvw256_mask">,
-        Intrinsic<[llvm_v16i16_ty], [llvm_v16i16_ty, llvm_v16i16_ty,
-                   llvm_v16i16_ty, llvm_i16_ty], [IntrNoMem]>;
-  def int_x86_avx512_maskz_vpshldv_w_256 :
-        GCCBuiltin<"__builtin_ia32_vpshldvw256_maskz">,
-        Intrinsic<[llvm_v16i16_ty], [llvm_v16i16_ty, llvm_v16i16_ty,
-                   llvm_v16i16_ty, llvm_i16_ty], [IntrNoMem]>;
-  def int_x86_avx512_mask_vpshldv_w_512 :
-        GCCBuiltin<"__builtin_ia32_vpshldvw512_mask">,
-        Intrinsic<[llvm_v32i16_ty], [llvm_v32i16_ty, llvm_v32i16_ty,
-                   llvm_v32i16_ty, llvm_i32_ty], [IntrNoMem]>;
-  def int_x86_avx512_maskz_vpshldv_w_512 :
-        GCCBuiltin<"__builtin_ia32_vpshldvw512_maskz">,
-        Intrinsic<[llvm_v32i16_ty], [llvm_v32i16_ty, llvm_v32i16_ty,
-                   llvm_v32i16_ty, llvm_i32_ty], [IntrNoMem]>;
-
-  def int_x86_avx512_mask_vpshldv_q_128 :
-        GCCBuiltin<"__builtin_ia32_vpshldvq128_mask">,
-        Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty,
-                   llvm_v2i64_ty, llvm_i8_ty], [IntrNoMem]>;
-  def int_x86_avx512_maskz_vpshldv_q_128 :
-        GCCBuiltin<"__builtin_ia32_vpshldvq128_maskz">,
-        Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty,
-                   llvm_v2i64_ty, llvm_i8_ty], [IntrNoMem]>;
-  def int_x86_avx512_mask_vpshldv_q_256 :
-        GCCBuiltin<"__builtin_ia32_vpshldvq256_mask">,
-        Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty, llvm_v4i64_ty,
-                   llvm_v4i64_ty, llvm_i8_ty], [IntrNoMem]>;
-  def int_x86_avx512_maskz_vpshldv_q_256 :
-        GCCBuiltin<"__builtin_ia32_vpshldvq256_maskz">,
-        Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty, llvm_v4i64_ty,
-                   llvm_v4i64_ty, llvm_i8_ty], [IntrNoMem]>;
-  def int_x86_avx512_mask_vpshldv_q_512 :
-        GCCBuiltin<"__builtin_ia32_vpshldvq512_mask">,
-        Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, llvm_v8i64_ty,
-                   llvm_v8i64_ty, llvm_i8_ty], [IntrNoMem]>;
-  def int_x86_avx512_maskz_vpshldv_q_512 :
-        GCCBuiltin<"__builtin_ia32_vpshldvq512_maskz">,
-        Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, llvm_v8i64_ty,
-                   llvm_v8i64_ty, llvm_i8_ty], [IntrNoMem]>;
-
-  def int_x86_avx512_mask_vpshldv_d_128 :
-        GCCBuiltin<"__builtin_ia32_vpshldvd128_mask">,
-        Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty,
-                   llvm_v4i32_ty, llvm_i8_ty], [IntrNoMem]>;
-  def int_x86_avx512_maskz_vpshldv_d_128 :
-        GCCBuiltin<"__builtin_ia32_vpshldvd128_maskz">,
-        Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty,
-                   llvm_v4i32_ty, llvm_i8_ty], [IntrNoMem]>;
-  def int_x86_avx512_mask_vpshldv_d_256 :
-        GCCBuiltin<"__builtin_ia32_vpshldvd256_mask">,
-        Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty, llvm_v8i32_ty,
-                   llvm_v8i32_ty, llvm_i8_ty], [IntrNoMem]>;
-  def int_x86_avx512_maskz_vpshldv_d_256 :
-        GCCBuiltin<"__builtin_ia32_vpshldvd256_maskz">,
-        Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty, llvm_v8i32_ty,
-                   llvm_v8i32_ty, llvm_i8_ty], [IntrNoMem]>;
-  def int_x86_avx512_mask_vpshldv_d_512 :
-        GCCBuiltin<"__builtin_ia32_vpshldvd512_mask">,
-        Intrinsic<[llvm_v16i32_ty], [llvm_v16i32_ty, llvm_v16i32_ty,
-                   llvm_v16i32_ty, llvm_i16_ty], [IntrNoMem]>;
-  def int_x86_avx512_maskz_vpshldv_d_512 :
-        GCCBuiltin<"__builtin_ia32_vpshldvd512_maskz">,
-        Intrinsic<[llvm_v16i32_ty], [llvm_v16i32_ty, llvm_v16i32_ty,
-                   llvm_v16i32_ty, llvm_i16_ty], [IntrNoMem]>;
-
-  def int_x86_avx512_mask_vpshrdv_w_128 :
-        GCCBuiltin<"__builtin_ia32_vpshrdvw128_mask">,
-        Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty,
-                   llvm_v8i16_ty, llvm_i8_ty], [IntrNoMem]>;
-  def int_x86_avx512_maskz_vpshrdv_w_128 :
-        GCCBuiltin<"__builtin_ia32_vpshrdvw128_maskz">,
-        Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty,
-                   llvm_v8i16_ty, llvm_i8_ty], [IntrNoMem]>;
-  def int_x86_avx512_mask_vpshrdv_w_256 :
-        GCCBuiltin<"__builtin_ia32_vpshrdvw256_mask">,
-        Intrinsic<[llvm_v16i16_ty], [llvm_v16i16_ty, llvm_v16i16_ty,
-                   llvm_v16i16_ty, llvm_i16_ty], [IntrNoMem]>;
-  def int_x86_avx512_maskz_vpshrdv_w_256 :
-        GCCBuiltin<"__builtin_ia32_vpshrdvw256_maskz">,
-        Intrinsic<[llvm_v16i16_ty], [llvm_v16i16_ty, llvm_v16i16_ty,
-                   llvm_v16i16_ty, llvm_i16_ty], [IntrNoMem]>;
-  def int_x86_avx512_mask_vpshrdv_w_512 :
-        GCCBuiltin<"__builtin_ia32_vpshrdvw512_mask">,
-        Intrinsic<[llvm_v32i16_ty], [llvm_v32i16_ty, llvm_v32i16_ty,
-                   llvm_v32i16_ty, llvm_i32_ty], [IntrNoMem]>;
-  def int_x86_avx512_maskz_vpshrdv_w_512 :
-        GCCBuiltin<"__builtin_ia32_vpshrdvw512_maskz">,
-        Intrinsic<[llvm_v32i16_ty], [llvm_v32i16_ty, llvm_v32i16_ty,
-                   llvm_v32i16_ty, llvm_i32_ty], [IntrNoMem]>;
-
-  def int_x86_avx512_mask_vpshrdv_q_128 :
-        GCCBuiltin<"__builtin_ia32_vpshrdvq128_mask">,
-        Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty,
-                   llvm_v2i64_ty, llvm_i8_ty], [IntrNoMem]>;
-  def int_x86_avx512_maskz_vpshrdv_q_128 :
-        GCCBuiltin<"__builtin_ia32_vpshrdvq128_maskz">,
-        Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty,
-                   llvm_v2i64_ty, llvm_i8_ty], [IntrNoMem]>;
-  def int_x86_avx512_mask_vpshrdv_q_256 :
-        GCCBuiltin<"__builtin_ia32_vpshrdvq256_mask">,
-        Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty, llvm_v4i64_ty,
-                   llvm_v4i64_ty, llvm_i8_ty], [IntrNoMem]>;
-  def int_x86_avx512_maskz_vpshrdv_q_256 :
-        GCCBuiltin<"__builtin_ia32_vpshrdvq256_maskz">,
-        Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty, llvm_v4i64_ty,
-                   llvm_v4i64_ty, llvm_i8_ty], [IntrNoMem]>;
-  def int_x86_avx512_mask_vpshrdv_q_512 :
-        GCCBuiltin<"__builtin_ia32_vpshrdvq512_mask">,
-        Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, llvm_v8i64_ty,
-                   llvm_v8i64_ty, llvm_i8_ty], [IntrNoMem]>;
-  def int_x86_avx512_maskz_vpshrdv_q_512 :
-        GCCBuiltin<"__builtin_ia32_vpshrdvq512_maskz">,
-        Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, llvm_v8i64_ty,
-                   llvm_v8i64_ty, llvm_i8_ty], [IntrNoMem]>;
-
-  def int_x86_avx512_mask_vpshrdv_d_128 :
-        GCCBuiltin<"__builtin_ia32_vpshrdvd128_mask">,
-        Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty,
-                   llvm_v4i32_ty, llvm_i8_ty], [IntrNoMem]>;
-  def int_x86_avx512_maskz_vpshrdv_d_128 :
-        GCCBuiltin<"__builtin_ia32_vpshrdvd128_maskz">,
-        Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty,
-                   llvm_v4i32_ty, llvm_i8_ty], [IntrNoMem]>;
-  def int_x86_avx512_mask_vpshrdv_d_256 :
-        GCCBuiltin<"__builtin_ia32_vpshrdvd256_mask">,
-        Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty, llvm_v8i32_ty,
-                   llvm_v8i32_ty, llvm_i8_ty], [IntrNoMem]>;
-  def int_x86_avx512_maskz_vpshrdv_d_256 :
-        GCCBuiltin<"__builtin_ia32_vpshrdvd256_maskz">,
-        Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty, llvm_v8i32_ty,
-                   llvm_v8i32_ty, llvm_i8_ty], [IntrNoMem]>;
-  def int_x86_avx512_mask_vpshrdv_d_512 :
-        GCCBuiltin<"__builtin_ia32_vpshrdvd512_mask">,
-        Intrinsic<[llvm_v16i32_ty], [llvm_v16i32_ty, llvm_v16i32_ty,
-                   llvm_v16i32_ty, llvm_i16_ty], [IntrNoMem]>;
-  def int_x86_avx512_maskz_vpshrdv_d_512 :
-        GCCBuiltin<"__builtin_ia32_vpshrdvd512_maskz">,
-        Intrinsic<[llvm_v16i32_ty], [llvm_v16i32_ty, llvm_v16i32_ty,
-                   llvm_v16i32_ty, llvm_i16_ty], [IntrNoMem]>;
+  def int_x86_avx512_mask_compress :
+        Intrinsic<[llvm_anyvector_ty],
+                  [LLVMMatchType<0>, LLVMMatchType<0>,
+                   LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>],
+                  [IntrNoMem]>;
+  def int_x86_avx512_mask_expand :
+        Intrinsic<[llvm_anyvector_ty],
+                  [LLVMMatchType<0>, LLVMMatchType<0>,
+                   LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>],
+                  [IntrNoMem]>;
 }
 
 // truncate
@@ -4655,10 +4252,6 @@ let TargetPrefix = "x86" in {
           Intrinsic<[],
                     [llvm_ptr_ty, llvm_v2i64_ty, llvm_i8_ty],
                     [IntrArgMemOnly]>;
-  def int_x86_avx512_mask_pmov_qd_256 : // FIXME: Replace with trunc+select.
-          Intrinsic<[llvm_v4i32_ty],
-                    [llvm_v4i64_ty, llvm_v4i32_ty, llvm_i8_ty],
-                    [IntrNoMem]>;
   def int_x86_avx512_mask_pmov_qd_mem_256 :
           GCCBuiltin<"__builtin_ia32_pmovqd256mem_mask">,
           Intrinsic<[],
@@ -4684,10 +4277,6 @@ let TargetPrefix = "x86" in {
           Intrinsic<[],
                     [llvm_ptr_ty, llvm_v4i64_ty, llvm_i8_ty],
                     [IntrArgMemOnly]>;
-  def int_x86_avx512_mask_pmov_qd_512 : // FIXME: Replace with trunc+select.
-          Intrinsic<[llvm_v8i32_ty],
-                    [llvm_v8i64_ty, llvm_v8i32_ty, llvm_i8_ty],
-                    [IntrNoMem]>;
   def int_x86_avx512_mask_pmov_qd_mem_512 :
           GCCBuiltin<"__builtin_ia32_pmovqd512mem_mask">,
           Intrinsic<[],
@@ -4921,10 +4510,6 @@ let TargetPrefix = "x86" in {
           Intrinsic<[],
                     [llvm_ptr_ty, llvm_v8i16_ty, llvm_i8_ty],
                     [IntrArgMemOnly]>;
-  def int_x86_avx512_mask_pmov_wb_256 : // FIXME: Replace with trunc+select.
-          Intrinsic<[llvm_v16i8_ty],
-                    [llvm_v16i16_ty, llvm_v16i8_ty, llvm_i16_ty],
-                    [IntrNoMem]>;
   def int_x86_avx512_mask_pmov_wb_mem_256 :
           GCCBuiltin<"__builtin_ia32_pmovwb256mem_mask">,
           Intrinsic<[],
@@ -4950,10 +4535,6 @@ let TargetPrefix = "x86" in {
           Intrinsic<[],
                     [llvm_ptr_ty, llvm_v16i16_ty, llvm_i16_ty],
                     [IntrArgMemOnly]>;
-  def int_x86_avx512_mask_pmov_wb_512 : // FIXME: Replace with trunc+select.
-          Intrinsic<[llvm_v32i8_ty],
-                    [llvm_v32i16_ty, llvm_v32i8_ty, llvm_i32_ty],
-                    [IntrNoMem]>;
   def int_x86_avx512_mask_pmov_wb_mem_512 :
           GCCBuiltin<"__builtin_ia32_pmovwb512mem_mask">,
           Intrinsic<[],
diff --git a/include/llvm/IR/IntrinsicsXCore.td b/include/llvm/IR/IntrinsicsXCore.td
index b614e1ed6ec0b8facd21ff784013f9548df19fbd..7fe8bdfd3bd011074075ad935df700eb8b7264a2 100644
--- a/include/llvm/IR/IntrinsicsXCore.td
+++ b/include/llvm/IR/IntrinsicsXCore.td
@@ -1,9 +1,8 @@
 //==- IntrinsicsXCore.td - XCore intrinsics                 -*- tablegen -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/IR/LLVMContext.h b/include/llvm/IR/LLVMContext.h
index ebd445553167bd6496a76acfe31db49ed262fdc5..f28480ee4c6f60a278e00beb5eaa544045e8826a 100644
--- a/include/llvm/IR/LLVMContext.h
+++ b/include/llvm/IR/LLVMContext.h
@@ -1,9 +1,8 @@
 //===- llvm/LLVMContext.h - Class for managing "global" state ---*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -102,6 +101,8 @@ public:
     MD_associated = 22,               // "associated"
     MD_callees = 23,                  // "callees"
     MD_irr_loop = 24,                 // "irr_loop"
+    MD_access_group = 25,             // "llvm.access.group"
+    MD_callback = 26,                 // "callback"
   };
 
   /// Known operand bundle tag IDs, which always have the same value.  All
diff --git a/include/llvm/IR/LegacyPassManager.h b/include/llvm/IR/LegacyPassManager.h
index 5257a0eed488cd9b1574a146e8e5d91a9f3bb8f6..d6bb79ab60199c84735bf736bcbdc2ac28b8f70e 100644
--- a/include/llvm/IR/LegacyPassManager.h
+++ b/include/llvm/IR/LegacyPassManager.h
@@ -1,9 +1,8 @@
 //===- LegacyPassManager.h - Legacy Container for Passes --------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/IR/LegacyPassManagers.h b/include/llvm/IR/LegacyPassManagers.h
index 51a2eb2a146dbfd0a7ee3f3df03bba4e254585d4..72bc80fb538157305d01038413e7579469d45030 100644
--- a/include/llvm/IR/LegacyPassManagers.h
+++ b/include/llvm/IR/LegacyPassManagers.h
@@ -1,9 +1,8 @@
 //===- LegacyPassManagers.h - Legacy Pass Infrastructure --------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/IR/LegacyPassNameParser.h b/include/llvm/IR/LegacyPassNameParser.h
index 4cec081964081a38c131b7abc15d0f7d12a5ee4a..30820e750350dfa3e5c1ab77a33bb98af79ba543 100644
--- a/include/llvm/IR/LegacyPassNameParser.h
+++ b/include/llvm/IR/LegacyPassNameParser.h
@@ -1,9 +1,8 @@
 //===- LegacyPassNameParser.h -----------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/IR/MDBuilder.h b/include/llvm/IR/MDBuilder.h
index 174616c7ab1dcd521ada89374e62bde3971633eb..3a2b1bddf45dba3ad55ecb0d8b11f06e7ff10b6f 100644
--- a/include/llvm/IR/MDBuilder.h
+++ b/include/llvm/IR/MDBuilder.h
@@ -1,9 +1,8 @@
 //===---- llvm/MDBuilder.h - Builder for LLVM metadata ----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -94,6 +93,17 @@ public:
   /// calls.
   MDNode *createCallees(ArrayRef<Function *> Callees);
 
+  //===------------------------------------------------------------------===//
+  // Callback metadata.
+  //===------------------------------------------------------------------===//
+
+  /// Return metadata describing a callback (see llvm::AbstractCallSite).
+  MDNode *createCallbackEncoding(unsigned CalleeArgNo, ArrayRef<int> Arguments,
+                                 bool VarArgsArePassed);
+
+  /// Merge the new callback encoding \p NewCB into \p ExistingCallbacks.
+  MDNode *mergeCallbackEncodings(MDNode *ExistingCallbacks, MDNode *NewCB);
+
   //===------------------------------------------------------------------===//
   // AA metadata.
   //===------------------------------------------------------------------===//
diff --git a/include/llvm/IR/Mangler.h b/include/llvm/IR/Mangler.h
index 0261c00f524cf2d55cae66754e2972822ed456ea..e4a05ab46a650a44eca0905e3487f147e8a91846 100644
--- a/include/llvm/IR/Mangler.h
+++ b/include/llvm/IR/Mangler.h
@@ -1,9 +1,8 @@
 //===-- llvm/IR/Mangler.h - Self-contained name mangler ---------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/IR/Metadata.def b/include/llvm/IR/Metadata.def
index 70a03f28b488f819b9611d43c87934af74e8ace8..d54401382e009bd0377995e68df4f5d22a655f71 100644
--- a/include/llvm/IR/Metadata.def
+++ b/include/llvm/IR/Metadata.def
@@ -1,9 +1,8 @@
 //===- llvm/IR/Metadata.def - Metadata definitions --------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/IR/Metadata.h b/include/llvm/IR/Metadata.h
index be82c4efc1159899d121a612d6d7cab368fc29c8..7ca2540181ba06ef598c40fc29eb8c31ebead7e6 100644
--- a/include/llvm/IR/Metadata.h
+++ b/include/llvm/IR/Metadata.h
@@ -1,9 +1,8 @@
 //===- llvm/IR/Metadata.h - Metadata definitions ----------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/IR/Module.h b/include/llvm/IR/Module.h
index ef4a4a9840b2f5799930edd61322829f430d5ca1..7373b84854119d1f6bada03e3e70726f98d47ec5 100644
--- a/include/llvm/IR/Module.h
+++ b/include/llvm/IR/Module.h
@@ -1,9 +1,8 @@
 //===- llvm/Module.h - C++ class to represent a VM module -------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -333,16 +332,18 @@ public:
   /// Look up the specified function in the module symbol table. Four
   /// possibilities:
   ///   1. If it does not exist, add a prototype for the function and return it.
-  ///   2. If it exists, and has a local linkage, the existing function is
-  ///      renamed and a new one is inserted.
-  ///   3. Otherwise, if the existing function has the correct prototype, return
+  ///   2. Otherwise, if the existing function has the correct prototype, return
   ///      the existing function.
-  ///   4. Finally, the function exists but has the wrong prototype: return the
+  ///   3. Finally, the function exists but has the wrong prototype: return the
   ///      function with a constantexpr cast to the right prototype.
-  Constant *getOrInsertFunction(StringRef Name, FunctionType *T,
-                                AttributeList AttributeList);
+  ///
+  /// In all cases, the returned value is a FunctionCallee wrapper around the
+  /// 'FunctionType *T' passed in, as well as a 'Value*' either of the Function or
+  /// the bitcast to the function.
+  FunctionCallee getOrInsertFunction(StringRef Name, FunctionType *T,
+                                     AttributeList AttributeList);
 
-  Constant *getOrInsertFunction(StringRef Name, FunctionType *T);
+  FunctionCallee getOrInsertFunction(StringRef Name, FunctionType *T);
 
   /// Look up the specified function in the module symbol table. If it does not
   /// exist, add a prototype for the function and return it. This function
@@ -350,11 +351,10 @@ public:
   /// or a ConstantExpr BitCast of that type if the named function has a
   /// different type. This version of the method takes a list of
   /// function arguments, which makes it easier for clients to use.
-  template<typename... ArgsTy>
-  Constant *getOrInsertFunction(StringRef Name,
-                                AttributeList AttributeList,
-                                Type *RetTy, ArgsTy... Args)
-  {
+  template <typename... ArgsTy>
+  FunctionCallee getOrInsertFunction(StringRef Name,
+                                     AttributeList AttributeList, Type *RetTy,
+                                     ArgsTy... Args) {
     SmallVector<Type*, sizeof...(ArgsTy)> ArgTys{Args...};
     return getOrInsertFunction(Name,
                                FunctionType::get(RetTy, ArgTys, false),
@@ -362,11 +362,18 @@ public:
   }
 
   /// Same as above, but without the attributes.
-  template<typename... ArgsTy>
-  Constant *getOrInsertFunction(StringRef Name, Type *RetTy, ArgsTy... Args) {
+  template <typename... ArgsTy>
+  FunctionCallee getOrInsertFunction(StringRef Name, Type *RetTy,
+                                     ArgsTy... Args) {
     return getOrInsertFunction(Name, AttributeList{}, RetTy, Args...);
   }
 
+  // Avoid an incorrect ordering that'd otherwise compile incorrectly.
+  template <typename... ArgsTy>
+  FunctionCallee
+  getOrInsertFunction(StringRef Name, AttributeList AttributeList,
+                      FunctionType *Invalid, ArgsTy... Args) = delete;
+
   /// Look up the specified function in the module symbol table. If it does not
   /// exist, return null.
   Function *getFunction(StringRef Name) const;
@@ -403,11 +410,15 @@ public:
   }
 
   /// Look up the specified global in the module symbol table.
-  ///   1. If it does not exist, add a declaration of the global and return it.
-  ///   2. Else, the global exists but has the wrong type: return the function
-  ///      with a constantexpr cast to the right type.
-  ///   3. Finally, if the existing global is the correct declaration, return
-  ///      the existing global.
+  /// If it does not exist, invoke a callback to create a declaration of the
+  /// global and return it. The global is constantexpr casted to the expected
+  /// type if necessary.
+  Constant *
+  getOrInsertGlobal(StringRef Name, Type *Ty,
+                    function_ref<GlobalVariable *()> CreateGlobalCallback);
+
+  /// Look up the specified global in the module symbol table. If required, this
+  /// overload constructs the global variable using its constructor's defaults.
   Constant *getOrInsertGlobal(StringRef Name, Type *Ty);
 
 /// @}
diff --git a/include/llvm/IR/ModuleSlotTracker.h b/include/llvm/IR/ModuleSlotTracker.h
index eb26fba906ea8cea513692bc2e87ebfa8820b1eb..85f8ff938366c875335ccac2d677d3dd06075b55 100644
--- a/include/llvm/IR/ModuleSlotTracker.h
+++ b/include/llvm/IR/ModuleSlotTracker.h
@@ -1,9 +1,8 @@
 //===-- llvm/IR/ModuleSlotTracker.h -----------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/IR/ModuleSummaryIndex.h b/include/llvm/IR/ModuleSummaryIndex.h
index 6653795d5034b84377a328525cde071012d1b38c..bcb3aa063e1337cadb49184953c7578989253c6c 100644
--- a/include/llvm/IR/ModuleSummaryIndex.h
+++ b/include/llvm/IR/ModuleSummaryIndex.h
@@ -1,9 +1,8 @@
 //===- llvm/ModuleSummaryIndex.h - Module Summary Index ---------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -831,6 +830,13 @@ private:
   /// union.
   bool HaveGVs;
 
+  // True if the index was created for a module compiled with -fsplit-lto-unit.
+  bool EnableSplitLTOUnit;
+
+  // True if some of the modules were compiled with -fsplit-lto-unit and
+  // some were not. Set when the combined index is created during the thin link.
+  bool PartiallySplitLTOUnits = false;
+
   std::set<std::string> CfiFunctionDefs;
   std::set<std::string> CfiFunctionDecls;
 
@@ -850,7 +856,9 @@ private:
 
 public:
   // See HaveGVs variable comment.
-  ModuleSummaryIndex(bool HaveGVs) : HaveGVs(HaveGVs), Saver(Alloc) {}
+  ModuleSummaryIndex(bool HaveGVs, bool EnableSplitLTOUnit = false)
+      : HaveGVs(HaveGVs), EnableSplitLTOUnit(EnableSplitLTOUnit), Saver(Alloc) {
+  }
 
   bool haveGVs() const { return HaveGVs; }
 
@@ -940,6 +948,12 @@ public:
     SkipModuleByDistributedBackend = true;
   }
 
+  bool enableSplitLTOUnit() const { return EnableSplitLTOUnit; }
+  void setEnableSplitLTOUnit() { EnableSplitLTOUnit = true; }
+
+  bool partiallySplitLTOUnits() const { return PartiallySplitLTOUnits; }
+  void setPartiallySplitLTOUnits() { PartiallySplitLTOUnits = true; }
+
   bool isGlobalValueLive(const GlobalValueSummary *GVS) const {
     return !WithGlobalValueDeadStripping || GVS->isLive();
   }
diff --git a/include/llvm/IR/ModuleSummaryIndexYAML.h b/include/llvm/IR/ModuleSummaryIndexYAML.h
index a88ee26b51c3534abc1b2b280cd9fcd89a911c09..c3feeeeeae16b71da9dc569f70628c9d966ce7ca 100644
--- a/include/llvm/IR/ModuleSummaryIndexYAML.h
+++ b/include/llvm/IR/ModuleSummaryIndexYAML.h
@@ -1,9 +1,8 @@
 //===-- llvm/ModuleSummaryIndexYAML.h - YAML I/O for summary ----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/IR/NoFolder.h b/include/llvm/IR/NoFolder.h
index def07ffe2ff671d00c4bd111e4a39fa917481c94..7fd303933c2283eeb06a20d4612c593cc817c7b4 100644
--- a/include/llvm/IR/NoFolder.h
+++ b/include/llvm/IR/NoFolder.h
@@ -1,9 +1,8 @@
 //===- NoFolder.h - Constant folding helper ---------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/IR/OperandTraits.h b/include/llvm/IR/OperandTraits.h
index c618aff3df9aea354c7128bd52de62f62860cfcf..979ad35019f8c79570804b67f63d5740d3fe13b4 100644
--- a/include/llvm/IR/OperandTraits.h
+++ b/include/llvm/IR/OperandTraits.h
@@ -1,9 +1,8 @@
 //===-- llvm/OperandTraits.h - OperandTraits class definition ---*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/IR/Operator.h b/include/llvm/IR/Operator.h
index 6b387bbcccb10746905818cc30e3578b42ea74f4..613d4342ef5fcecd8e389636319fb216f53807fd 100644
--- a/include/llvm/IR/Operator.h
+++ b/include/llvm/IR/Operator.h
@@ -1,9 +1,8 @@
 //===-- llvm/Operator.h - Operator utility subclass -------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/IR/OptBisect.h b/include/llvm/IR/OptBisect.h
index aa24c94c01302bcde3aeaaa717b1cc691cfd69ce..8d51b6856dda2178064d05f1d01955e2420cb315 100644
--- a/include/llvm/IR/OptBisect.h
+++ b/include/llvm/IR/OptBisect.h
@@ -1,9 +1,8 @@
 //===- llvm/IR/OptBisect.h - LLVM Bisect support ----------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
diff --git a/include/llvm/IR/PassInstrumentation.h b/include/llvm/IR/PassInstrumentation.h
index 08dac1c4a2746599f86f46368e5eaa08585aa87f..f8a1196871cfcc4ff193d6711a5c8d18a6315a9e 100644
--- a/include/llvm/IR/PassInstrumentation.h
+++ b/include/llvm/IR/PassInstrumentation.h
@@ -1,9 +1,8 @@
 //===- llvm/IR/PassInstrumentation.h ----------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/include/llvm/IR/PassManager.h b/include/llvm/IR/PassManager.h
index 738a2242eea0879f58ef836fc4a5fc3ef4f0a1de..47bad5b2f1995bc91c17db4d5111d7c646d2bcb8 100644
--- a/include/llvm/IR/PassManager.h
+++ b/include/llvm/IR/PassManager.h
@@ -1,9 +1,8 @@
 //===- PassManager.h - Pass management infrastructure -----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/include/llvm/IR/PassManagerInternal.h b/include/llvm/IR/PassManagerInternal.h
index 5ad68be6274230f2501ccf9c20a7670d255ff1ee..58198bf67b112d027d8ba02e9b3974d8f7bebc49 100644
--- a/include/llvm/IR/PassManagerInternal.h
+++ b/include/llvm/IR/PassManagerInternal.h
@@ -1,9 +1,8 @@
 //===- PassManager internal APIs and implementation details -----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/include/llvm/IR/PassTimingInfo.h b/include/llvm/IR/PassTimingInfo.h
index e9945f997f4395b65b4a82b9935e8432b8144b0b..ddddbdfa3a30d025e85b3cf3fbd332344bc16d15 100644
--- a/include/llvm/IR/PassTimingInfo.h
+++ b/include/llvm/IR/PassTimingInfo.h
@@ -1,9 +1,8 @@
 //===- PassTimingInfo.h - pass execution timing -----------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/include/llvm/IR/PatternMatch.h b/include/llvm/IR/PatternMatch.h
index e1e7c7268b5d690a7e08b77caf5b97310a15e21b..6c51d487737018d99109fec698d811c351165586 100644
--- a/include/llvm/IR/PatternMatch.h
+++ b/include/llvm/IR/PatternMatch.h
@@ -1,9 +1,8 @@
 //===- PatternMatch.h - Match on the LLVM IR --------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -31,7 +30,6 @@
 
 #include "llvm/ADT/APFloat.h"
 #include "llvm/ADT/APInt.h"
-#include "llvm/IR/CallSite.h"
 #include "llvm/IR/Constant.h"
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/InstrTypes.h"
@@ -1465,6 +1463,20 @@ struct UAddWithOverflow_match {
       if (AddExpr.match(ICmpRHS) && (ICmpLHS == AddLHS || ICmpLHS == AddRHS))
         return L.match(AddLHS) && R.match(AddRHS) && S.match(ICmpRHS);
 
+    // Match special-case for increment-by-1.
+    if (Pred == ICmpInst::ICMP_EQ) {
+      // (a + 1) == 0
+      // (1 + a) == 0
+      if (AddExpr.match(ICmpLHS) && m_ZeroInt().match(ICmpRHS) &&
+          (m_One().match(AddLHS) || m_One().match(AddRHS)))
+        return L.match(AddLHS) && R.match(AddRHS) && S.match(ICmpLHS);
+      // 0 == (a + 1)
+      // 0 == (1 + a)
+      if (m_ZeroInt().match(ICmpLHS) && AddExpr.match(ICmpRHS) &&
+          (m_One().match(AddLHS) || m_One().match(AddRHS)))
+        return L.match(AddLHS) && R.match(AddRHS) && S.match(ICmpRHS);
+    }
+
     return false;
   }
 };
@@ -1486,8 +1498,10 @@ template <typename Opnd_t> struct Argument_match {
   Argument_match(unsigned OpIdx, const Opnd_t &V) : OpI(OpIdx), Val(V) {}
 
   template <typename OpTy> bool match(OpTy *V) {
-    CallSite CS(V);
-    return CS.isCall() && Val.match(CS.getArgument(OpI));
+    // FIXME: Should likely be switched to use `CallBase`.
+    if (const auto *CI = dyn_cast<CallInst>(V))
+      return Val.match(CI->getArgOperand(OpI));
+    return false;
   }
 };
 
diff --git a/include/llvm/IR/PredIteratorCache.h b/include/llvm/IR/PredIteratorCache.h
index 81f535311431acec72e18406634d3edafc7cd848..cc835277910bf8dbb9f46b994c1aa9476151625d 100644
--- a/include/llvm/IR/PredIteratorCache.h
+++ b/include/llvm/IR/PredIteratorCache.h
@@ -1,9 +1,8 @@
 //===- PredIteratorCache.h - pred_iterator Cache ----------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/IR/ProfileSummary.h b/include/llvm/IR/ProfileSummary.h
index e38663770a13935cecc18f436a9eb2bf74bfabc9..f1d49cc16630e0b926e55b2690b33dab980a7503 100644
--- a/include/llvm/IR/ProfileSummary.h
+++ b/include/llvm/IR/ProfileSummary.h
@@ -1,9 +1,8 @@
 //===- ProfileSummary.h - Profile summary data structure. -------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/IR/RuntimeLibcalls.def b/include/llvm/IR/RuntimeLibcalls.def
index 89005120cdc1cde2d433c75b0bd385d9c6b09883..9d7268d5567978cf4e6bf0959586c51741f9e1b5 100644
--- a/include/llvm/IR/RuntimeLibcalls.def
+++ b/include/llvm/IR/RuntimeLibcalls.def
@@ -1,9 +1,8 @@
 //===-- llvm/RuntimeLibcalls.def - File that describes libcalls -*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/IR/SafepointIRVerifier.h b/include/llvm/IR/SafepointIRVerifier.h
index 092050d1d2079f3072020f3a9e4cdbc81f254062..cbdb8bd425643481c1281bedf484a7c2fc6e137e 100644
--- a/include/llvm/IR/SafepointIRVerifier.h
+++ b/include/llvm/IR/SafepointIRVerifier.h
@@ -1,9 +1,8 @@
 //===- SafepointIRVerifier.h - Checks for GC relocation problems *- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/IR/Statepoint.h b/include/llvm/IR/Statepoint.h
index 8908e1b0d09085fd349d376bebc968968750c207..aec0e9b6d87bea44b4f865c67b10bb85e74ed99d 100644
--- a/include/llvm/IR/Statepoint.h
+++ b/include/llvm/IR/Statepoint.h
@@ -1,14 +1,13 @@
 //===- llvm/IR/Statepoint.h - gc.statepoint utilities -----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 // This file contains utility functions and a wrapper class analogous to
-// CallSite for accessing the fields of gc.statepoint, gc.relocate,
+// CallBase for accessing the fields of gc.statepoint, gc.relocate,
 // gc.result intrinsics; and some general utilities helpful when dealing with
 // gc.statepoint.
 //
@@ -21,7 +20,6 @@
 #include "llvm/ADT/iterator_range.h"
 #include "llvm/IR/Attributes.h"
 #include "llvm/IR/BasicBlock.h"
-#include "llvm/IR/CallSite.h"
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/Function.h"
 #include "llvm/IR/Instruction.h"
@@ -57,42 +55,39 @@ enum class StatepointFlags {
 class GCRelocateInst;
 class GCResultInst;
 
-bool isStatepoint(ImmutableCallSite CS);
+bool isStatepoint(const CallBase *Call);
 bool isStatepoint(const Value *V);
 bool isStatepoint(const Value &V);
 
-bool isGCRelocate(ImmutableCallSite CS);
+bool isGCRelocate(const CallBase *Call);
 bool isGCRelocate(const Value *V);
 
-bool isGCResult(ImmutableCallSite CS);
+bool isGCResult(const CallBase *Call);
 bool isGCResult(const Value *V);
 
-/// Analogous to CallSiteBase, this provides most of the actual
+/// A wrapper around a GC intrinsic call, this provides most of the actual
 /// functionality for Statepoint and ImmutableStatepoint.  It is
 /// templatized to allow easily specializing of const and non-const
-/// concrete subtypes.  This is structured analogous to CallSite
-/// rather than the IntrinsicInst.h helpers since we need to support
-/// invokable statepoints.
+/// concrete subtypes.
 template <typename FunTy, typename InstructionTy, typename ValueTy,
-          typename CallSiteTy>
+          typename CallBaseTy>
 class StatepointBase {
-  CallSiteTy StatepointCS;
+  CallBaseTy *StatepointCall;
 
 protected:
   explicit StatepointBase(InstructionTy *I) {
     if (isStatepoint(I)) {
-      StatepointCS = CallSiteTy(I);
-      assert(StatepointCS && "isStatepoint implies CallSite");
+      StatepointCall = cast<CallBaseTy>(I);
     }
   }
 
-  explicit StatepointBase(CallSiteTy CS) {
-    if (isStatepoint(CS))
-      StatepointCS = CS;
+  explicit StatepointBase(CallBaseTy *Call) {
+    if (isStatepoint(Call))
+      StatepointCall = Call;
   }
 
 public:
-  using arg_iterator = typename CallSiteTy::arg_iterator;
+  using arg_iterator = typename CallBaseTy::const_op_iterator;
 
   enum {
     IDPos = 0,
@@ -107,30 +102,30 @@ public:
   void *operator new(size_t s) = delete;
 
   explicit operator bool() const {
-    // We do not assign non-statepoint CallSites to StatepointCS.
-    return (bool)StatepointCS;
+    // We do not assign non-statepoint call instructions to StatepointCall.
+    return (bool)StatepointCall;
   }
 
-  /// Return the underlying CallSite.
-  CallSiteTy getCallSite() const {
+  /// Return the underlying call instruction.
+  CallBaseTy *getCall() const {
     assert(*this && "check validity first!");
-    return StatepointCS;
+    return StatepointCall;
   }
 
   uint64_t getFlags() const {
-    return cast<ConstantInt>(getCallSite().getArgument(FlagsPos))
+    return cast<ConstantInt>(getCall()->getArgOperand(FlagsPos))
         ->getZExtValue();
   }
 
   /// Return the ID associated with this statepoint.
   uint64_t getID() const {
-    const Value *IDVal = getCallSite().getArgument(IDPos);
+    const Value *IDVal = getCall()->getArgOperand(IDPos);
     return cast<ConstantInt>(IDVal)->getZExtValue();
   }
 
   /// Return the number of patchable bytes associated with this statepoint.
   uint32_t getNumPatchBytes() const {
-    const Value *NumPatchBytesVal = getCallSite().getArgument(NumPatchBytesPos);
+    const Value *NumPatchBytesVal = getCall()->getArgOperand(NumPatchBytesPos);
     uint64_t NumPatchBytes =
       cast<ConstantInt>(NumPatchBytesVal)->getZExtValue();
     assert(isInt<32>(NumPatchBytes) && "should fit in 32 bits!");
@@ -139,12 +134,11 @@ public:
 
   /// Return the value actually being called or invoked.
   ValueTy *getCalledValue() const {
-    return getCallSite().getArgument(CalledFunctionPos);
+    return getCall()->getArgOperand(CalledFunctionPos);
   }
 
-  InstructionTy *getInstruction() const {
-    return getCallSite().getInstruction();
-  }
+  // FIXME: Migrate users of this to `getCall` and remove it.
+  InstructionTy *getInstruction() const { return getCall(); }
 
   /// Return the function being called if this is a direct call, otherwise
   /// return null (if it's an indirect call).
@@ -153,12 +147,12 @@ public:
   }
 
   /// Return the caller function for this statepoint.
-  FunTy *getCaller() const { return getCallSite().getCaller(); }
+  FunTy *getCaller() const { return getCall()->getCaller(); }
 
   /// Determine if the statepoint cannot unwind.
   bool doesNotThrow() const {
     Function *F = getCalledFunction();
-    return getCallSite().doesNotThrow() || (F ? F->doesNotThrow() : false);
+    return getCall()->doesNotThrow() || (F ? F->doesNotThrow() : false);
   }
 
   /// Return the type of the value returned by the call underlying the
@@ -171,18 +165,18 @@ public:
 
   /// Number of arguments to be passed to the actual callee.
   int getNumCallArgs() const {
-    const Value *NumCallArgsVal = getCallSite().getArgument(NumCallArgsPos);
+    const Value *NumCallArgsVal = getCall()->getArgOperand(NumCallArgsPos);
     return cast<ConstantInt>(NumCallArgsVal)->getZExtValue();
   }
 
   size_t arg_size() const { return getNumCallArgs(); }
-  typename CallSiteTy::arg_iterator arg_begin() const {
-    assert(CallArgsBeginPos <= (int)getCallSite().arg_size());
-    return getCallSite().arg_begin() + CallArgsBeginPos;
+  arg_iterator arg_begin() const {
+    assert(CallArgsBeginPos <= (int)getCall()->arg_size());
+    return getCall()->arg_begin() + CallArgsBeginPos;
   }
-  typename CallSiteTy::arg_iterator arg_end() const {
+  arg_iterator arg_end() const {
     auto I = arg_begin() + arg_size();
-    assert((getCallSite().arg_end() - I) >= 0);
+    assert((getCall()->arg_end() - I) >= 0);
     return I;
   }
 
@@ -199,8 +193,8 @@ public:
   /// Return true if the call or the callee has the given attribute.
   bool paramHasAttr(unsigned i, Attribute::AttrKind A) const {
     Function *F = getCalledFunction();
-    return getCallSite().paramHasAttr(i + CallArgsBeginPos, A) ||
-          (F ? F->getAttributes().hasAttribute(i, A) : false);
+    return getCall()->paramHasAttr(i + CallArgsBeginPos, A) ||
+           (F ? F->getAttributes().hasAttribute(i, A) : false);
   }
 
   /// Number of GC transition args.
@@ -208,14 +202,14 @@ public:
     const Value *NumGCTransitionArgs = *arg_end();
     return cast<ConstantInt>(NumGCTransitionArgs)->getZExtValue();
   }
-  typename CallSiteTy::arg_iterator gc_transition_args_begin() const {
+  arg_iterator gc_transition_args_begin() const {
     auto I = arg_end() + 1;
-    assert((getCallSite().arg_end() - I) >= 0);
+    assert((getCall()->arg_end() - I) >= 0);
     return I;
   }
-  typename CallSiteTy::arg_iterator gc_transition_args_end() const {
+  arg_iterator gc_transition_args_end() const {
     auto I = gc_transition_args_begin() + getNumTotalGCTransitionArgs();
-    assert((getCallSite().arg_end() - I) >= 0);
+    assert((getCall()->arg_end() - I) >= 0);
     return I;
   }
 
@@ -231,14 +225,14 @@ public:
     return cast<ConstantInt>(NumVMSArgs)->getZExtValue();
   }
 
-  typename CallSiteTy::arg_iterator deopt_begin() const {
+  arg_iterator deopt_begin() const {
     auto I = gc_transition_args_end() + 1;
-    assert((getCallSite().arg_end() - I) >= 0);
+    assert((getCall()->arg_end() - I) >= 0);
     return I;
   }
-  typename CallSiteTy::arg_iterator deopt_end() const {
+  arg_iterator deopt_end() const {
     auto I = deopt_begin() + getNumTotalVMSArgs();
-    assert((getCallSite().arg_end() - I) >= 0);
+    assert((getCall()->arg_end() - I) >= 0);
     return I;
   }
 
@@ -247,15 +241,11 @@ public:
     return make_range(deopt_begin(), deopt_end());
   }
 
-  typename CallSiteTy::arg_iterator gc_args_begin() const {
-    return deopt_end();
-  }
-  typename CallSiteTy::arg_iterator gc_args_end() const {
-    return getCallSite().arg_end();
-  }
+  arg_iterator gc_args_begin() const { return deopt_end(); }
+  arg_iterator gc_args_end() const { return getCall()->arg_end(); }
 
   unsigned gcArgsStartIdx() const {
-    return gc_args_begin() - getInstruction()->op_begin();
+    return gc_args_begin() - getCall()->op_begin();
   }
 
   /// range adapter for gc arguments
@@ -304,25 +294,24 @@ public:
 /// to a gc.statepoint.
 class ImmutableStatepoint
     : public StatepointBase<const Function, const Instruction, const Value,
-                            ImmutableCallSite> {
-  using Base =
-      StatepointBase<const Function, const Instruction, const Value,
-                     ImmutableCallSite>;
+                            const CallBase> {
+  using Base = StatepointBase<const Function, const Instruction, const Value,
+                              const CallBase>;
 
 public:
   explicit ImmutableStatepoint(const Instruction *I) : Base(I) {}
-  explicit ImmutableStatepoint(ImmutableCallSite CS) : Base(CS) {}
+  explicit ImmutableStatepoint(const CallBase *Call) : Base(Call) {}
 };
 
 /// A specialization of it's base class for read-write access
 /// to a gc.statepoint.
 class Statepoint
-    : public StatepointBase<Function, Instruction, Value, CallSite> {
-  using Base = StatepointBase<Function, Instruction, Value, CallSite>;
+    : public StatepointBase<Function, Instruction, Value, CallBase> {
+  using Base = StatepointBase<Function, Instruction, Value, CallBase>;
 
 public:
   explicit Statepoint(Instruction *I) : Base(I) {}
-  explicit Statepoint(CallSite CS) : Base(CS) {}
+  explicit Statepoint(CallBase *Call) : Base(Call) {}
 };
 
 /// Common base class for representing values projected from a statepoint.
@@ -347,14 +336,14 @@ public:
   }
 
   /// The statepoint with which this gc.relocate is associated.
-  const Instruction *getStatepoint() const {
+  const CallBase *getStatepoint() const {
     const Value *Token = getArgOperand(0);
 
     // This takes care both of relocates for call statepoints and relocates
     // on normal path of invoke statepoint.
     if (!isa<LandingPadInst>(Token)) {
       assert(isStatepoint(Token));
-      return cast<Instruction>(Token);
+      return cast<CallBase>(Token);
     }
 
     // This relocate is on exceptional path of an invoke statepoint
@@ -366,7 +355,7 @@ public:
            "safepoint block should be well formed");
     assert(isStatepoint(InvokeBB->getTerminator()));
 
-    return InvokeBB->getTerminator();
+    return cast<CallBase>(InvokeBB->getTerminator());
   }
 };
 
@@ -395,13 +384,11 @@ public:
   }
 
   Value *getBasePtr() const {
-    ImmutableCallSite CS(getStatepoint());
-    return *(CS.arg_begin() + getBasePtrIndex());
+    return *(getStatepoint()->arg_begin() + getBasePtrIndex());
   }
 
   Value *getDerivedPtr() const {
-    ImmutableCallSite CS(getStatepoint());
-    return *(CS.arg_begin() + getDerivedPtrIndex());
+    return *(getStatepoint()->arg_begin() + getDerivedPtrIndex());
   }
 };
 
@@ -418,28 +405,25 @@ public:
 };
 
 template <typename FunTy, typename InstructionTy, typename ValueTy,
-          typename CallSiteTy>
+          typename CallBaseTy>
 std::vector<const GCRelocateInst *>
-StatepointBase<FunTy, InstructionTy, ValueTy, CallSiteTy>::getRelocates()
+StatepointBase<FunTy, InstructionTy, ValueTy, CallBaseTy>::getRelocates()
     const {
-
   std::vector<const GCRelocateInst *> Result;
 
-  CallSiteTy StatepointCS = getCallSite();
-
   // Search for relocated pointers.  Note that working backwards from the
   // gc_relocates ensures that we only get pairs which are actually relocated
   // and used after the statepoint.
-  for (const User *U : getInstruction()->users())
+  for (const User *U : StatepointCall->users())
     if (auto *Relocate = dyn_cast<GCRelocateInst>(U))
       Result.push_back(Relocate);
 
-  if (!StatepointCS.isInvoke())
+  auto *StatepointInvoke = dyn_cast<InvokeInst>(StatepointCall);
+  if (!StatepointInvoke)
     return Result;
 
   // We need to scan thorough exceptional relocations if it is invoke statepoint
-  LandingPadInst *LandingPad =
-      cast<InvokeInst>(getInstruction())->getLandingPadInst();
+  LandingPadInst *LandingPad = StatepointInvoke->getLandingPadInst();
 
   // Search for gc relocates that are attached to this landingpad.
   for (const User *LandingPadUser : LandingPad->users()) {
diff --git a/include/llvm/IR/SymbolTableListTraits.h b/include/llvm/IR/SymbolTableListTraits.h
index 87ce902c2811b19c67b28fe4a047868c7a6d120d..5b793e5dbf2836517b2d24b72681f85db1b19b65 100644
--- a/include/llvm/IR/SymbolTableListTraits.h
+++ b/include/llvm/IR/SymbolTableListTraits.h
@@ -1,9 +1,8 @@
 //===- llvm/SymbolTableListTraits.h - Traits for iplist ---------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/IR/TrackingMDRef.h b/include/llvm/IR/TrackingMDRef.h
index 084efada221fdfe9355281c2502f8c34537d1efa..d7377398b91b3cdd6c3eb4a462cb4e5e48390348 100644
--- a/include/llvm/IR/TrackingMDRef.h
+++ b/include/llvm/IR/TrackingMDRef.h
@@ -1,9 +1,8 @@
 //===- llvm/IR/TrackingMDRef.h - Tracking Metadata references ---*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/IR/Type.h b/include/llvm/IR/Type.h
index 9c1f99d1b3a2d40a968b7dadfef1300909d1f374..5f9f44e815237e1096d7e9c85e31a3bcdb93bfbb 100644
--- a/include/llvm/IR/Type.h
+++ b/include/llvm/IR/Type.h
@@ -1,9 +1,8 @@
 //===- llvm/Type.h - Classes for handling data types ------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -467,28 +466,6 @@ template <> struct isa_impl<PointerType, Type> {
   }
 };
 
-//===----------------------------------------------------------------------===//
-// Provide specializations of GraphTraits to be able to treat a type as a
-// graph of sub types.
-
-template <> struct GraphTraits<Type *> {
-  using NodeRef = Type *;
-  using ChildIteratorType = Type::subtype_iterator;
-
-  static NodeRef getEntryNode(Type *T) { return T; }
-  static ChildIteratorType child_begin(NodeRef N) { return N->subtype_begin(); }
-  static ChildIteratorType child_end(NodeRef N) { return N->subtype_end(); }
-};
-
-template <> struct GraphTraits<const Type*> {
-  using NodeRef = const Type *;
-  using ChildIteratorType = Type::subtype_iterator;
-
-  static NodeRef getEntryNode(NodeRef T) { return T; }
-  static ChildIteratorType child_begin(NodeRef N) { return N->subtype_begin(); }
-  static ChildIteratorType child_end(NodeRef N) { return N->subtype_end(); }
-};
-
 // Create wrappers for C Binding types (see CBindingWrapping.h).
 DEFINE_ISA_CONVERSION_FUNCTIONS(Type, LLVMTypeRef)
 
diff --git a/include/llvm/IR/TypeBuilder.h b/include/llvm/IR/TypeBuilder.h
deleted file mode 100644
index d2c6f00079da4cb2fef9a841b9e60818dd11d30c..0000000000000000000000000000000000000000
--- a/include/llvm/IR/TypeBuilder.h
+++ /dev/null
@@ -1,407 +0,0 @@
-//===---- llvm/TypeBuilder.h - Builder for LLVM types -----------*- C++ -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This file defines the TypeBuilder class, which is used as a convenient way to
-// create LLVM types with a consistent and simplified interface.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_IR_TYPEBUILDER_H
-#define LLVM_IR_TYPEBUILDER_H
-
-#include "llvm/IR/DerivedTypes.h"
-#include "llvm/IR/LLVMContext.h"
-#include <climits>
-
-namespace llvm {
-
-/// TypeBuilder - This provides a uniform API for looking up types
-/// known at compile time.  To support cross-compilation, we define a
-/// series of tag types in the llvm::types namespace, like i<N>,
-/// ieee_float, ppc_fp128, etc.  TypeBuilder<T, false> allows T to be
-/// any of these, a native C type (whose size may depend on the host
-/// compiler), or a pointer, function, or struct type built out of
-/// these.  TypeBuilder<T, true> removes native C types from this set
-/// to guarantee that its result is suitable for cross-compilation.
-/// We define the primitive types, pointer types, and functions up to
-/// 5 arguments here, but to use this class with your own types,
-/// you'll need to specialize it.  For example, say you want to call a
-/// function defined externally as:
-///
-/// \code{.cpp}
-///
-///   struct MyType {
-///     int32 a;
-///     int32 *b;
-///     void *array[1];  // Intended as a flexible array.
-///   };
-///   int8 AFunction(struct MyType *value);
-///
-/// \endcode
-///
-/// You'll want to use
-///   Function::Create(TypeBuilder<types::i<8>(MyType*), true>::get(), ...)
-/// to declare the function, but when you first try this, your compiler will
-/// complain that TypeBuilder<MyType, true>::get() doesn't exist. To fix this,
-/// write:
-///
-/// \code{.cpp}
-///
-///   namespace llvm {
-///   template<bool xcompile> class TypeBuilder<MyType, xcompile> {
-///   public:
-///     static StructType *get(LLVMContext &Context) {
-///       // If you cache this result, be sure to cache it separately
-///       // for each LLVMContext.
-///       return StructType::get(
-///         TypeBuilder<types::i<32>, xcompile>::get(Context),
-///         TypeBuilder<types::i<32>*, xcompile>::get(Context),
-///         TypeBuilder<types::i<8>*[], xcompile>::get(Context),
-///         nullptr);
-///     }
-///
-///     // You may find this a convenient place to put some constants
-///     // to help with getelementptr.  They don't have any effect on
-///     // the operation of TypeBuilder.
-///     enum Fields {
-///       FIELD_A,
-///       FIELD_B,
-///       FIELD_ARRAY
-///     };
-///   }
-///   }  // namespace llvm
-///
-/// \endcode
-///
-/// TypeBuilder cannot handle recursive types or types you only know at runtime.
-/// If you try to give it a recursive type, it will deadlock, infinitely
-/// recurse, or do something similarly undesirable.
-template<typename T, bool cross_compilable> class TypeBuilder {};
-
-// Types for use with cross-compilable TypeBuilders.  These correspond
-// exactly with an LLVM-native type.
-namespace types {
-/// i<N> corresponds to the LLVM IntegerType with N bits.
-template<uint32_t num_bits> class i {};
-
-// The following classes represent the LLVM floating types.
-class ieee_float {};
-class ieee_double {};
-class x86_fp80 {};
-class fp128 {};
-class ppc_fp128 {};
-// X86 MMX.
-class x86_mmx {};
-}  // namespace types
-
-// LLVM doesn't have const or volatile types.
-template<typename T, bool cross> class TypeBuilder<const T, cross>
-  : public TypeBuilder<T, cross> {};
-template<typename T, bool cross> class TypeBuilder<volatile T, cross>
-  : public TypeBuilder<T, cross> {};
-template<typename T, bool cross> class TypeBuilder<const volatile T, cross>
-  : public TypeBuilder<T, cross> {};
-
-// Pointers
-template<typename T, bool cross> class TypeBuilder<T*, cross> {
-public:
-  static PointerType *get(LLVMContext &Context) {
-    return PointerType::getUnqual(TypeBuilder<T,cross>::get(Context));
-  }
-};
-
-/// There is no support for references
-template<typename T, bool cross> class TypeBuilder<T&, cross> {};
-
-// Arrays
-template<typename T, size_t N, bool cross> class TypeBuilder<T[N], cross> {
-public:
-  static ArrayType *get(LLVMContext &Context) {
-    return ArrayType::get(TypeBuilder<T, cross>::get(Context), N);
-  }
-};
-/// LLVM uses an array of length 0 to represent an unknown-length array.
-template<typename T, bool cross> class TypeBuilder<T[], cross> {
-public:
-  static ArrayType *get(LLVMContext &Context) {
-    return ArrayType::get(TypeBuilder<T, cross>::get(Context), 0);
-  }
-};
-
-// Define the C integral types only for TypeBuilder<T, false>.
-//
-// C integral types do not have a defined size. It would be nice to use the
-// stdint.h-defined typedefs that do have defined sizes, but we'd run into the
-// following problem:
-//
-// On an ILP32 machine, stdint.h might define:
-//
-//   typedef int int32_t;
-//   typedef long long int64_t;
-//   typedef long size_t;
-//
-// If we defined TypeBuilder<int32_t> and TypeBuilder<int64_t>, then any use of
-// TypeBuilder<size_t> would fail.  We couldn't define TypeBuilder<size_t> in
-// addition to the defined-size types because we'd get duplicate definitions on
-// platforms where stdint.h instead defines:
-//
-//   typedef int int32_t;
-//   typedef long long int64_t;
-//   typedef int size_t;
-//
-// So we define all the primitive C types and nothing else.
-#define DEFINE_INTEGRAL_TYPEBUILDER(T) \
-template<> class TypeBuilder<T, false> { \
-public: \
-  static IntegerType *get(LLVMContext &Context) { \
-    return IntegerType::get(Context, sizeof(T) * CHAR_BIT); \
-  } \
-}; \
-template<> class TypeBuilder<T, true> { \
-  /* We provide a definition here so users don't accidentally */ \
-  /* define these types to work. */ \
-}
-DEFINE_INTEGRAL_TYPEBUILDER(char);
-DEFINE_INTEGRAL_TYPEBUILDER(signed char);
-DEFINE_INTEGRAL_TYPEBUILDER(unsigned char);
-DEFINE_INTEGRAL_TYPEBUILDER(short);
-DEFINE_INTEGRAL_TYPEBUILDER(unsigned short);
-DEFINE_INTEGRAL_TYPEBUILDER(int);
-DEFINE_INTEGRAL_TYPEBUILDER(unsigned int);
-DEFINE_INTEGRAL_TYPEBUILDER(long);
-DEFINE_INTEGRAL_TYPEBUILDER(unsigned long);
-#ifdef _MSC_VER
-DEFINE_INTEGRAL_TYPEBUILDER(__int64);
-DEFINE_INTEGRAL_TYPEBUILDER(unsigned __int64);
-#else /* _MSC_VER */
-DEFINE_INTEGRAL_TYPEBUILDER(long long);
-DEFINE_INTEGRAL_TYPEBUILDER(unsigned long long);
-#endif /* _MSC_VER */
-#undef DEFINE_INTEGRAL_TYPEBUILDER
-
-template<uint32_t num_bits, bool cross>
-class TypeBuilder<types::i<num_bits>, cross> {
-public:
-  static IntegerType *get(LLVMContext &C) {
-    return IntegerType::get(C, num_bits);
-  }
-};
-
-template<> class TypeBuilder<float, false> {
-public:
-  static Type *get(LLVMContext& C) {
-    return Type::getFloatTy(C);
-  }
-};
-template<> class TypeBuilder<float, true> {};
-
-template<> class TypeBuilder<double, false> {
-public:
-  static Type *get(LLVMContext& C) {
-    return Type::getDoubleTy(C);
-  }
-};
-template<> class TypeBuilder<double, true> {};
-
-template<bool cross> class TypeBuilder<types::ieee_float, cross> {
-public:
-  static Type *get(LLVMContext& C) { return Type::getFloatTy(C); }
-};
-template<bool cross> class TypeBuilder<types::ieee_double, cross> {
-public:
-  static Type *get(LLVMContext& C) { return Type::getDoubleTy(C); }
-};
-template<bool cross> class TypeBuilder<types::x86_fp80, cross> {
-public:
-  static Type *get(LLVMContext& C) { return Type::getX86_FP80Ty(C); }
-};
-template<bool cross> class TypeBuilder<types::fp128, cross> {
-public:
-  static Type *get(LLVMContext& C) { return Type::getFP128Ty(C); }
-};
-template<bool cross> class TypeBuilder<types::ppc_fp128, cross> {
-public:
-  static Type *get(LLVMContext& C) { return Type::getPPC_FP128Ty(C); }
-};
-template<bool cross> class TypeBuilder<types::x86_mmx, cross> {
-public:
-  static Type *get(LLVMContext& C) { return Type::getX86_MMXTy(C); }
-};
-
-template<bool cross> class TypeBuilder<void, cross> {
-public:
-  static Type *get(LLVMContext &C) {
-    return Type::getVoidTy(C);
-  }
-};
-
-/// void* is disallowed in LLVM types, but it occurs often enough in C code that
-/// we special case it.
-template<> class TypeBuilder<void*, false>
-  : public TypeBuilder<types::i<8>*, false> {};
-template<> class TypeBuilder<const void*, false>
-  : public TypeBuilder<types::i<8>*, false> {};
-template<> class TypeBuilder<volatile void*, false>
-  : public TypeBuilder<types::i<8>*, false> {};
-template<> class TypeBuilder<const volatile void*, false>
-  : public TypeBuilder<types::i<8>*, false> {};
-
-template<typename R, bool cross> class TypeBuilder<R(), cross> {
-public:
-  static FunctionType *get(LLVMContext &Context) {
-    return FunctionType::get(TypeBuilder<R, cross>::get(Context), false);
-  }
-};
-template<typename R, typename A1, bool cross> class TypeBuilder<R(A1), cross> {
-public:
-  static FunctionType *get(LLVMContext &Context) {
-    Type *params[] = {
-      TypeBuilder<A1, cross>::get(Context),
-    };
-    return FunctionType::get(TypeBuilder<R, cross>::get(Context),
-                             params, false);
-  }
-};
-template<typename R, typename A1, typename A2, bool cross>
-class TypeBuilder<R(A1, A2), cross> {
-public:
-  static FunctionType *get(LLVMContext &Context) {
-    Type *params[] = {
-      TypeBuilder<A1, cross>::get(Context),
-      TypeBuilder<A2, cross>::get(Context),
-    };
-    return FunctionType::get(TypeBuilder<R, cross>::get(Context),
-                             params, false);
-  }
-};
-template<typename R, typename A1, typename A2, typename A3, bool cross>
-class TypeBuilder<R(A1, A2, A3), cross> {
-public:
-  static FunctionType *get(LLVMContext &Context) {
-    Type *params[] = {
-      TypeBuilder<A1, cross>::get(Context),
-      TypeBuilder<A2, cross>::get(Context),
-      TypeBuilder<A3, cross>::get(Context),
-    };
-    return FunctionType::get(TypeBuilder<R, cross>::get(Context),
-                             params, false);
-  }
-};
-
-template<typename R, typename A1, typename A2, typename A3, typename A4,
-         bool cross>
-class TypeBuilder<R(A1, A2, A3, A4), cross> {
-public:
-  static FunctionType *get(LLVMContext &Context) {
-    Type *params[] = {
-      TypeBuilder<A1, cross>::get(Context),
-      TypeBuilder<A2, cross>::get(Context),
-      TypeBuilder<A3, cross>::get(Context),
-      TypeBuilder<A4, cross>::get(Context),
-    };
-    return FunctionType::get(TypeBuilder<R, cross>::get(Context),
-                             params, false);
-  }
-};
-
-template<typename R, typename A1, typename A2, typename A3, typename A4,
-         typename A5, bool cross>
-class TypeBuilder<R(A1, A2, A3, A4, A5), cross> {
-public:
-  static FunctionType *get(LLVMContext &Context) {
-    Type *params[] = {
-      TypeBuilder<A1, cross>::get(Context),
-      TypeBuilder<A2, cross>::get(Context),
-      TypeBuilder<A3, cross>::get(Context),
-      TypeBuilder<A4, cross>::get(Context),
-      TypeBuilder<A5, cross>::get(Context),
-    };
-    return FunctionType::get(TypeBuilder<R, cross>::get(Context),
-                             params, false);
-  }
-};
-
-template<typename R, bool cross> class TypeBuilder<R(...), cross> {
-public:
-  static FunctionType *get(LLVMContext &Context) {
-    return FunctionType::get(TypeBuilder<R, cross>::get(Context), true);
-  }
-};
-template<typename R, typename A1, bool cross>
-class TypeBuilder<R(A1, ...), cross> {
-public:
-  static FunctionType *get(LLVMContext &Context) {
-    Type *params[] = {
-      TypeBuilder<A1, cross>::get(Context),
-    };
-    return FunctionType::get(TypeBuilder<R, cross>::get(Context), params, true);
-  }
-};
-template<typename R, typename A1, typename A2, bool cross>
-class TypeBuilder<R(A1, A2, ...), cross> {
-public:
-  static FunctionType *get(LLVMContext &Context) {
-    Type *params[] = {
-      TypeBuilder<A1, cross>::get(Context),
-      TypeBuilder<A2, cross>::get(Context),
-    };
-    return FunctionType::get(TypeBuilder<R, cross>::get(Context),
-                                   params, true);
-  }
-};
-template<typename R, typename A1, typename A2, typename A3, bool cross>
-class TypeBuilder<R(A1, A2, A3, ...), cross> {
-public:
-  static FunctionType *get(LLVMContext &Context) {
-    Type *params[] = {
-      TypeBuilder<A1, cross>::get(Context),
-      TypeBuilder<A2, cross>::get(Context),
-      TypeBuilder<A3, cross>::get(Context),
-    };
-    return FunctionType::get(TypeBuilder<R, cross>::get(Context),
-                                   params, true);
-  }
-};
-
-template<typename R, typename A1, typename A2, typename A3, typename A4,
-         bool cross>
-class TypeBuilder<R(A1, A2, A3, A4, ...), cross> {
-public:
-  static FunctionType *get(LLVMContext &Context) {
-    Type *params[] = {
-      TypeBuilder<A1, cross>::get(Context),
-      TypeBuilder<A2, cross>::get(Context),
-      TypeBuilder<A3, cross>::get(Context),
-      TypeBuilder<A4, cross>::get(Context),
-    };
-    return FunctionType::get(TypeBuilder<R, cross>::get(Context),
-                             params, true);
-  }
-};
-
-template<typename R, typename A1, typename A2, typename A3, typename A4,
-         typename A5, bool cross>
-class TypeBuilder<R(A1, A2, A3, A4, A5, ...), cross> {
-public:
-  static FunctionType *get(LLVMContext &Context) {
-    Type *params[] = {
-      TypeBuilder<A1, cross>::get(Context),
-      TypeBuilder<A2, cross>::get(Context),
-      TypeBuilder<A3, cross>::get(Context),
-      TypeBuilder<A4, cross>::get(Context),
-      TypeBuilder<A5, cross>::get(Context),
-    };
-    return FunctionType::get(TypeBuilder<R, cross>::get(Context),
-                                   params, true);
-  }
-};
-
-}  // namespace llvm
-
-#endif
diff --git a/include/llvm/IR/TypeFinder.h b/include/llvm/IR/TypeFinder.h
index c050c388d398f8c857c177543d43f801ba736a41..a83f85ea84c303e1a47bb194845a6fe1b09cde00 100644
--- a/include/llvm/IR/TypeFinder.h
+++ b/include/llvm/IR/TypeFinder.h
@@ -1,9 +1,8 @@
 //===- llvm/IR/TypeFinder.h - Class to find used struct types ---*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/IR/Use.h b/include/llvm/IR/Use.h
index 25c44e0871a9a44b5d679e42107ceca41cba807e..6d29d47acd630e5732371c963fb6fd7f003c2c16 100644
--- a/include/llvm/IR/Use.h
+++ b/include/llvm/IR/Use.h
@@ -1,9 +1,8 @@
 //===- llvm/Use.h - Definition of the Use class -----------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/include/llvm/IR/UseListOrder.h b/include/llvm/IR/UseListOrder.h
index b6bb0f19a0aaf6264437edfbfde7589c50e0de61..a1f313e269b29ca712e633be67eacb40a723d4b1 100644
--- a/include/llvm/IR/UseListOrder.h
+++ b/include/llvm/IR/UseListOrder.h
@@ -1,9 +1,8 @@
 //===- llvm/IR/UseListOrder.h - LLVM Use List Order -------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/IR/User.h b/include/llvm/IR/User.h
index aea31467f2fa5d6a5f93a41de87b9dfdb367ffbb..19d87c5c621d99a81ab81b244d7380764687d15e 100644
--- a/include/llvm/IR/User.h
+++ b/include/llvm/IR/User.h
@@ -1,9 +1,8 @@
 //===- llvm/User.h - User class definition ----------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/IR/Value.def b/include/llvm/IR/Value.def
index e2ddba0aa1596eac50b317d4f231940a86719c3b..aaf1651979a9627cdbef142a8f1dd1474aff7edd 100644
--- a/include/llvm/IR/Value.def
+++ b/include/llvm/IR/Value.def
@@ -1,9 +1,8 @@
 //===-------- llvm/IR/Value.def - File that describes Values ---v-*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/IR/Value.h b/include/llvm/IR/Value.h
index 4f3a45c684fcc01488f65d11f049dfcb5b9fb0cd..d0de008db265f7c1ce1fa34ab9a1213430c1e945 100644
--- a/include/llvm/IR/Value.h
+++ b/include/llvm/IR/Value.h
@@ -1,9 +1,8 @@
 //===- llvm/Value.h - Definition of the Value class -------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/IR/ValueHandle.h b/include/llvm/IR/ValueHandle.h
index d94472ce1be18d2724a8b434e0f9c98d4c4e0ad6..1135d796f7ed7def28191f78d224c03a9a3e2c13 100644
--- a/include/llvm/IR/ValueHandle.h
+++ b/include/llvm/IR/ValueHandle.h
@@ -1,9 +1,8 @@
 //===- ValueHandle.h - Value Smart Pointer classes --------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -309,15 +308,6 @@ struct DenseMapInfo<AssertingVH<T>> {
   }
 };
 
-template <typename T>
-struct isPodLike<AssertingVH<T>> {
-#ifdef NDEBUG
-  static const bool value = true;
-#else
-  static const bool value = false;
-#endif
-};
-
 /// Value handle that tracks a Value across RAUW.
 ///
 /// TrackingVH is designed for situations where a client needs to hold a handle
@@ -549,14 +539,6 @@ template <typename T> struct DenseMapInfo<PoisoningVH<T>> {
   }
 };
 
-template <typename T> struct isPodLike<PoisoningVH<T>> {
-#ifdef NDEBUG
-  static const bool value = true;
-#else
-  static const bool value = false;
-#endif
-};
-
 } // end namespace llvm
 
 #endif // LLVM_IR_VALUEHANDLE_H
diff --git a/include/llvm/IR/ValueMap.h b/include/llvm/IR/ValueMap.h
index e7e33918a61320a5970100e323fcf07b2995ce41..6a79b1d387f3808124b2412066c4db611aa88eaf 100644
--- a/include/llvm/IR/ValueMap.h
+++ b/include/llvm/IR/ValueMap.h
@@ -1,9 +1,8 @@
 //===- ValueMap.h - Safe map from Values to data ----------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/IR/ValueSymbolTable.h b/include/llvm/IR/ValueSymbolTable.h
index 012e717c7470c3e987ce73d71daff80ac494bc6b..105ea73857afecfcc64c1a4dd53108519613c424 100644
--- a/include/llvm/IR/ValueSymbolTable.h
+++ b/include/llvm/IR/ValueSymbolTable.h
@@ -1,9 +1,8 @@
 //===- llvm/ValueSymbolTable.h - Implement a Value Symtab -------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/IR/Verifier.h b/include/llvm/IR/Verifier.h
index 7255132e1e6534b85ca1f0c43c3b33c33259e0da..62c33c8325ebbbcdc531a3e673d80b71dc0889cc 100644
--- a/include/llvm/IR/Verifier.h
+++ b/include/llvm/IR/Verifier.h
@@ -1,9 +1,8 @@
 //===- Verifier.h - LLVM IR Verifier ----------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/IRReader/IRReader.h b/include/llvm/IRReader/IRReader.h
index bedde8954fbb8af4c5c849f587b7ed491acc7875..05171300b602d12b72d7442d62de69621328a968 100644
--- a/include/llvm/IRReader/IRReader.h
+++ b/include/llvm/IRReader/IRReader.h
@@ -1,9 +1,8 @@
 //===---- llvm/IRReader/IRReader.h - Reader for LLVM IR files ---*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -21,11 +20,22 @@
 namespace llvm {
 
 class StringRef;
+class MemoryBuffer;
 class MemoryBufferRef;
 class Module;
 class SMDiagnostic;
 class LLVMContext;
 
+/// If the given MemoryBuffer holds a bitcode image, return a Module
+/// for it which does lazy deserialization of function bodies.  Otherwise,
+/// attempt to parse it as LLVM Assembly and return a fully populated
+/// Module. The ShouldLazyLoadMetadata flag is passed down to the bitcode
+/// reader to optionally enable lazy metadata loading. This takes ownership
+/// of \p Buffer.
+std::unique_ptr<Module> getLazyIRModule(std::unique_ptr<MemoryBuffer> Buffer,
+                                        SMDiagnostic &Err, LLVMContext &Context,
+                                        bool ShouldLazyLoadMetadata = false);
+
 /// If the given file holds a bitcode image, return a Module
 /// for it which does lazy deserialization of function bodies.  Otherwise,
 /// attempt to parse it as LLVM Assembly and return a fully populated
diff --git a/include/llvm/InitializePasses.h b/include/llvm/InitializePasses.h
index 206089cf3a8f4a95896573f8c4b51dcec9d2c23b..fe6c17182a8e0b9f69f7dcc9744024a052a37a5d 100644
--- a/include/llvm/InitializePasses.h
+++ b/include/llvm/InitializePasses.h
@@ -1,9 +1,8 @@
 //===- llvm/InitializePasses.h - Initialize All Passes ----------*- C++ -*-===//
 //
-//                      The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -85,6 +84,7 @@ void initializeBranchProbabilityInfoWrapperPassPass(PassRegistry&);
 void initializeBranchRelaxationPass(PassRegistry&);
 void initializeBreakCriticalEdgesPass(PassRegistry&);
 void initializeBreakFalseDepsPass(PassRegistry&);
+void initializeCanonicalizeAliasesLegacyPassPass(PassRegistry &);
 void initializeCFGOnlyPrinterLegacyPassPass(PassRegistry&);
 void initializeCFGOnlyViewerLegacyPassPass(PassRegistry&);
 void initializeCFGPrinterLegacyPassPass(PassRegistry&);
@@ -198,6 +198,7 @@ void initializeLegacyDivergenceAnalysisPass(PassRegistry&);
 void initializeLegacyLICMPassPass(PassRegistry&);
 void initializeLegacyLoopSinkPassPass(PassRegistry&);
 void initializeLegalizerPass(PassRegistry&);
+void initializeGISelCSEAnalysisWrapperPassPass(PassRegistry &);
 void initializeLibCallsShrinkWrapLegacyPassPass(PassRegistry&);
 void initializeLintPass(PassRegistry&);
 void initializeLiveDebugValuesPass(PassRegistry&);
@@ -239,6 +240,7 @@ void initializeLowerAtomicLegacyPassPass(PassRegistry&);
 void initializeLowerEmuTLSPass(PassRegistry&);
 void initializeLowerExpectIntrinsicPass(PassRegistry&);
 void initializeLowerGuardIntrinsicLegacyPassPass(PassRegistry&);
+void initializeLowerWidenableConditionLegacyPassPass(PassRegistry&);
 void initializeLowerIntrinsicsPass(PassRegistry&);
 void initializeLowerInvokeLegacyPassPass(PassRegistry&);
 void initializeLowerSwitchPass(PassRegistry&);
@@ -273,7 +275,7 @@ void initializeMemDerefPrinterPass(PassRegistry&);
 void initializeMemoryDependenceWrapperPassPass(PassRegistry&);
 void initializeMemorySSAPrinterLegacyPassPass(PassRegistry&);
 void initializeMemorySSAWrapperPassPass(PassRegistry&);
-void initializeMemorySanitizerPass(PassRegistry&);
+void initializeMemorySanitizerLegacyPassPass(PassRegistry&);
 void initializeMergeFunctionsPass(PassRegistry&);
 void initializeMergeICmpsPass(PassRegistry&);
 void initializeMergedLoadStoreMotionLegacyPassPass(PassRegistry&);
@@ -390,7 +392,7 @@ void initializeTailDuplicatePass(PassRegistry&);
 void initializeTargetLibraryInfoWrapperPassPass(PassRegistry&);
 void initializeTargetPassConfigPass(PassRegistry&);
 void initializeTargetTransformInfoWrapperPassPass(PassRegistry&);
-void initializeThreadSanitizerPass(PassRegistry&);
+void initializeThreadSanitizerLegacyPassPass(PassRegistry&);
 void initializeTwoAddressInstructionPassPass(PassRegistry&);
 void initializeTypeBasedAAWrapperPassPass(PassRegistry&);
 void initializeUnifyFunctionExitNodesPass(PassRegistry&);
diff --git a/include/llvm/LTO/Caching.h b/include/llvm/LTO/Caching.h
index 7201ab31f5b08a10aa45eeed5afd012f6eb1d36b..c20d819208336ea7006ea2daeee5e3bceea5a14b 100644
--- a/include/llvm/LTO/Caching.h
+++ b/include/llvm/LTO/Caching.h
@@ -1,9 +1,8 @@
 //===- Caching.h - LLVM Link Time Optimizer Configuration -----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/LTO/Config.h b/include/llvm/LTO/Config.h
index 7058602c3ee2ef01f1456c61ee7d410583c17bb7..f0e1b1d16490768e634dd31cb034aa30c0ea2bad 100644
--- a/include/llvm/LTO/Config.h
+++ b/include/llvm/LTO/Config.h
@@ -1,9 +1,8 @@
 //===-Config.h - LLVM Link Time Optimizer Configuration -------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/LTO/LTO.h b/include/llvm/LTO/LTO.h
index 15390873056af5b56f921a3acff143ce7db6c886..d3949aa33b26a77a416c24fb1b554c78f2955fd7 100644
--- a/include/llvm/LTO/LTO.h
+++ b/include/llvm/LTO/LTO.h
@@ -1,9 +1,8 @@
 //===-LTO.h - LLVM Link Time Optimizer ------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -400,6 +399,9 @@ private:
   Error runThinLTO(AddStreamFn AddStream, NativeObjectCache Cache);
 
   mutable bool CalledGetMaxTasks = false;
+
+  // Use Optional to distinguish false from not yet initialized.
+  Optional<bool> EnableSplitLTOUnit;
 };
 
 /// The resolution for a symbol. The linker must provide a SymbolResolution for
diff --git a/include/llvm/LTO/LTOBackend.h b/include/llvm/LTO/LTOBackend.h
index d4743f6940ff931c471f2b7c01c7f159247a7536..4ff8a1993d49ff133fe0ed8deb1282dae35a5dc6 100644
--- a/include/llvm/LTO/LTOBackend.h
+++ b/include/llvm/LTO/LTOBackend.h
@@ -1,9 +1,8 @@
 //===-LTOBackend.h - LLVM Link Time Optimizer Backend ---------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/LTO/SummaryBasedOptimizations.h b/include/llvm/LTO/SummaryBasedOptimizations.h
index ad3a8e7dc77b4a1b003a6feeb82fbad6afeae932..6697c821a5ead6f306a910ca4104bba65a926fa1 100644
--- a/include/llvm/LTO/SummaryBasedOptimizations.h
+++ b/include/llvm/LTO/SummaryBasedOptimizations.h
@@ -1,9 +1,8 @@
 //=- llvm/LTO/SummaryBasedOptimizations.h -Link time optimizations-*- C++ -*-=//
 //
-//                      The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/LTO/legacy/LTOCodeGenerator.h b/include/llvm/LTO/legacy/LTOCodeGenerator.h
index 8f23b7cb4574f0ab8b6b6acad549b0ec167bbd8d..37fef191a28c3fb3bd44f167a6350a8384368374 100644
--- a/include/llvm/LTO/legacy/LTOCodeGenerator.h
+++ b/include/llvm/LTO/legacy/LTOCodeGenerator.h
@@ -1,9 +1,8 @@
 //===-LTOCodeGenerator.h - LLVM Link Time Optimizer -----------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/LTO/legacy/LTOModule.h b/include/llvm/LTO/legacy/LTOModule.h
index 017e223ed8a616520719ff4a12b019562e04a4f1..006521bf397834552227013c3106bcfff0e06f66 100644
--- a/include/llvm/LTO/legacy/LTOModule.h
+++ b/include/llvm/LTO/legacy/LTOModule.h
@@ -1,9 +1,8 @@
 //===-LTOModule.h - LLVM Link Time Optimizer ------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/LTO/legacy/ThinLTOCodeGenerator.h b/include/llvm/LTO/legacy/ThinLTOCodeGenerator.h
index d4c69a1ce260b7337d1ab49dad9bacfa864db806..1730e83628d5227b4da325b91c4cf9abb82042bb 100644
--- a/include/llvm/LTO/legacy/ThinLTOCodeGenerator.h
+++ b/include/llvm/LTO/legacy/ThinLTOCodeGenerator.h
@@ -1,9 +1,8 @@
 //===-ThinLTOCodeGenerator.h - LLVM Link Time Optimizer -------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/LTO/legacy/UpdateCompilerUsed.h b/include/llvm/LTO/legacy/UpdateCompilerUsed.h
index 4be0027e97d7cabc31db534355e8b3812f7bec43..af9d62b4af3080113ab874c7b0592ee596736b4b 100644
--- a/include/llvm/LTO/legacy/UpdateCompilerUsed.h
+++ b/include/llvm/LTO/legacy/UpdateCompilerUsed.h
@@ -1,9 +1,8 @@
 //==------ UpdateCompilerUsed.h - LLVM Link Time Optimizer Utility --------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/LineEditor/LineEditor.h b/include/llvm/LineEditor/LineEditor.h
index 68995d0633add4b67477212b8f467be967ec56fe..0beaf1bb23a90b1bd1f559fcefd5d61a0a10c170 100644
--- a/include/llvm/LineEditor/LineEditor.h
+++ b/include/llvm/LineEditor/LineEditor.h
@@ -1,9 +1,8 @@
 //===-- llvm/LineEditor/LineEditor.h - line editor --------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/LinkAllIR.h b/include/llvm/LinkAllIR.h
index 4f4af7187be451905c6db29b75598385a6763390..4b0aabeee7017a280cc0f832a98cc5aac8437076 100644
--- a/include/llvm/LinkAllIR.h
+++ b/include/llvm/LinkAllIR.h
@@ -1,9 +1,8 @@
 //===----- LinkAllIR.h - Reference All VMCore Code --------------*- C++ -*-===//
 //
-//                      The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/LinkAllPasses.h b/include/llvm/LinkAllPasses.h
index a31caeea2eedcf6dbbffb87521fe38ccf1dae0d6..ec81b1618cc5674fe4b19ef6be35e3453b2dd365 100644
--- a/include/llvm/LinkAllPasses.h
+++ b/include/llvm/LinkAllPasses.h
@@ -1,9 +1,8 @@
 //===- llvm/LinkAllPasses.h ------------ Reference All Passes ---*- C++ -*-===//
 //
-//                      The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -230,7 +229,8 @@ namespace {
       llvm::TargetLibraryInfo TLI(TLII);
       llvm::AliasAnalysis AA(TLI);
       llvm::AliasSetTracker X(AA);
-      X.add(nullptr, 0, llvm::AAMDNodes()); // for -print-alias-sets
+      X.add(nullptr, llvm::LocationSize::unknown(),
+            llvm::AAMDNodes()); // for -print-alias-sets
       (void) llvm::AreStatisticsEnabled();
       (void) llvm::sys::RunningOnValgrind();
     }
diff --git a/include/llvm/Linker/IRMover.h b/include/llvm/Linker/IRMover.h
index 235ada47cef47900d5f2203f5cf4223fcd923c43..e5df83f01fe38115d6373d89700aae802319e4d1 100644
--- a/include/llvm/Linker/IRMover.h
+++ b/include/llvm/Linker/IRMover.h
@@ -1,9 +1,8 @@
 //===- IRMover.h ------------------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/Linker/Linker.h b/include/llvm/Linker/Linker.h
index 7776c720ec5379a6d56f36f2ba74ef97512586b0..c9b1d42b3903d1e746d2ec07d946006ca47f2d49 100644
--- a/include/llvm/Linker/Linker.h
+++ b/include/llvm/Linker/Linker.h
@@ -1,9 +1,8 @@
 //===- Linker.h - Module Linker Interface -----------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/MC/ConstantPools.h b/include/llvm/MC/ConstantPools.h
index ef33250204ecff8d52bf2e4326b83bd7e9a4dff5..2fe5ce252c941176058fcde3aaaf7e186e0beebe 100644
--- a/include/llvm/MC/ConstantPools.h
+++ b/include/llvm/MC/ConstantPools.h
@@ -1,9 +1,8 @@
-//===- ConstantPool.h - Keep track of assembler-generated  ------*- C++ -*-===//
+//===- ConstantPools.h - Keep track of assembler-generated ------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/MC/LaneBitmask.h b/include/llvm/MC/LaneBitmask.h
index 8c0b4ecb8fd40c3d7e347d8ca78855785f98f95d..d5f69287a265b0a2be3b3c24b0f0299644c6a25c 100644
--- a/include/llvm/MC/LaneBitmask.h
+++ b/include/llvm/MC/LaneBitmask.h
@@ -1,9 +1,8 @@
 //===- llvm/MC/LaneBitmask.h ------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
diff --git a/include/llvm/MC/MCAsmBackend.h b/include/llvm/MC/MCAsmBackend.h
index 07835c21fcedb1696559d4b1436a1538b0dffe22..1f3ad6c1e547885ed2c2f05855a43502c30284c5 100644
--- a/include/llvm/MC/MCAsmBackend.h
+++ b/include/llvm/MC/MCAsmBackend.h
@@ -1,9 +1,8 @@
 //===- llvm/MC/MCAsmBackend.h - MC Asm Backend ------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -88,6 +87,22 @@ public:
     return false;
   }
 
+  /// Hook to check if extra nop bytes must be inserted for alignment directive.
+  /// For some targets this may be necessary in order to support linker
+  /// relaxation. The number of bytes to insert are returned in Size.
+  virtual bool shouldInsertExtraNopBytesForCodeAlign(const MCAlignFragment &AF,
+                                                     unsigned &Size) {
+    return false;
+  }
+
+  /// Hook which indicates if the target requires a fixup to be generated when
+  /// handling an align directive in an executable section
+  virtual bool shouldInsertFixupForCodeAlign(MCAssembler &Asm,
+                                             const MCAsmLayout &Layout,
+                                             MCAlignFragment &AF) {
+    return false;
+  }
+
   /// Apply the \p Value for given \p Fixup into the provided data fragment, at
   /// the offset specified by the fixup and following the fixup kind as
   /// appropriate. Errors (such as an out of range fixup value) should be
diff --git a/include/llvm/MC/MCAsmInfo.h b/include/llvm/MC/MCAsmInfo.h
index 120fb8fa74921f3ef65851f49be14ac66d8a5474..e861df5bb3dd939a3a567575971da9393dc2916b 100644
--- a/include/llvm/MC/MCAsmInfo.h
+++ b/include/llvm/MC/MCAsmInfo.h
@@ -1,9 +1,8 @@
 //===-- llvm/MC/MCAsmInfo.h - Asm info --------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -492,7 +491,7 @@ public:
   StringRef getPrivateLabelPrefix() const { return PrivateLabelPrefix; }
 
   bool hasLinkerPrivateGlobalPrefix() const {
-    return LinkerPrivateGlobalPrefix[0] != '\0';
+    return !LinkerPrivateGlobalPrefix.empty();
   }
 
   StringRef getLinkerPrivateGlobalPrefix() const {
diff --git a/include/llvm/MC/MCAsmInfoCOFF.h b/include/llvm/MC/MCAsmInfoCOFF.h
index 01c8ae49a6fcd4163eb986f4a29e0d51e213db34..1dfb4750af66a91d45fb7355bf2445e949eee31f 100644
--- a/include/llvm/MC/MCAsmInfoCOFF.h
+++ b/include/llvm/MC/MCAsmInfoCOFF.h
@@ -1,9 +1,8 @@
 //===- MCAsmInfoCOFF.h - COFF asm properties --------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/MC/MCAsmInfoDarwin.h b/include/llvm/MC/MCAsmInfoDarwin.h
index a533d604a89e1ab833f2dc2c9906259cb0d6b329..c889ce99cebeb87de2467c0c5ec2edde0d904ae8 100644
--- a/include/llvm/MC/MCAsmInfoDarwin.h
+++ b/include/llvm/MC/MCAsmInfoDarwin.h
@@ -1,9 +1,8 @@
 //===- MCAsmInfoDarwin.h - Darwin asm properties ----------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/MC/MCAsmInfoELF.h b/include/llvm/MC/MCAsmInfoELF.h
index f113afc9885e7dc8fc61ab826fa7e294ccd20034..aa2e5873e2c6816a76126694c378b5069d191500 100644
--- a/include/llvm/MC/MCAsmInfoELF.h
+++ b/include/llvm/MC/MCAsmInfoELF.h
@@ -1,9 +1,8 @@
 //===- llvm/MC/MCAsmInfoELF.h - ELF Asm info --------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/MC/MCAsmInfoWasm.h b/include/llvm/MC/MCAsmInfoWasm.h
index 71c6ee28df7069dc3bab0a8e17adead205963bfb..3afc610b8b07b65a96dc8a20c6887cf3501d6cfa 100644
--- a/include/llvm/MC/MCAsmInfoWasm.h
+++ b/include/llvm/MC/MCAsmInfoWasm.h
@@ -1,9 +1,8 @@
 //===-- llvm/MC/MCAsmInfoWasm.h - Wasm Asm info -----------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/MC/MCAsmLayout.h b/include/llvm/MC/MCAsmLayout.h
index b711db319302945d3a59974aec5b6724a0f21db8..45ac96f0b81ec26995e9a0c7fc102c22fe0bc40d 100644
--- a/include/llvm/MC/MCAsmLayout.h
+++ b/include/llvm/MC/MCAsmLayout.h
@@ -1,9 +1,8 @@
 //===- MCAsmLayout.h - Assembly Layout Object -------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/MC/MCAsmMacro.h b/include/llvm/MC/MCAsmMacro.h
index 135fa4f2e33d64a800b95ecf0d018eceaaa67f21..364d3b5f36662b009e70d4c1df7c59207fe53f0f 100644
--- a/include/llvm/MC/MCAsmMacro.h
+++ b/include/llvm/MC/MCAsmMacro.h
@@ -1,9 +1,8 @@
 //===- MCAsmMacro.h - Assembly Macros ---------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/MC/MCAssembler.h b/include/llvm/MC/MCAssembler.h
index 986c6e17548fc3501c329a314c66d5a86975fb5c..4543018901a40b4d6b2d7df65e1bb5a25d889bad 100644
--- a/include/llvm/MC/MCAssembler.h
+++ b/include/llvm/MC/MCAssembler.h
@@ -1,9 +1,8 @@
 //===- MCAssembler.h - Object File Generation -------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/MC/MCCodeEmitter.h b/include/llvm/MC/MCCodeEmitter.h
index f1b0b784a2df6ee5a6d5ef2839f2b86a523b0e5b..04b4367ada7b7a3045b47820499de965e7a21ed8 100644
--- a/include/llvm/MC/MCCodeEmitter.h
+++ b/include/llvm/MC/MCCodeEmitter.h
@@ -1,9 +1,8 @@
 //===- llvm/MC/MCCodeEmitter.h - Instruction Encoding -----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/MC/MCCodePadder.h b/include/llvm/MC/MCCodePadder.h
index 4dde6bf59272440441debf06905c2411aa452371..f7b1a2113a9acb10d3bf3afa9d994bdf8de5eb0c 100644
--- a/include/llvm/MC/MCCodePadder.h
+++ b/include/llvm/MC/MCCodePadder.h
@@ -1,9 +1,8 @@
-//===- llvm/MC/CodePadder.h - MC Code Padder --------------------*- C++ -*-===//
+//===- llvm/MC/MCCodePadder.h - MC Code Padder ------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/MC/MCCodeView.h b/include/llvm/MC/MCCodeView.h
index cef03a409f9506c903be3739ae138511a9c06228..2126354cded6c73701dc332435fcf2ed8e61c26b 100644
--- a/include/llvm/MC/MCCodeView.h
+++ b/include/llvm/MC/MCCodeView.h
@@ -1,9 +1,8 @@
 //===- MCCodeView.h - Machine Code CodeView support -------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/MC/MCContext.h b/include/llvm/MC/MCContext.h
index 3b8ac8b79e210e250b6ea4ea975d7e80c4c1db3c..b964965cf47fc43d5df9a64279d3f70709960865 100644
--- a/include/llvm/MC/MCContext.h
+++ b/include/llvm/MC/MCContext.h
@@ -1,9 +1,8 @@
 //===- MCContext.h - Machine Code Context -----------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/MC/MCDirectives.h b/include/llvm/MC/MCDirectives.h
index 8c74b169135b10deca31807bf8f72a07d2402a3a..4029264c202635910e5637d205fa5258dc2522cf 100644
--- a/include/llvm/MC/MCDirectives.h
+++ b/include/llvm/MC/MCDirectives.h
@@ -1,9 +1,8 @@
 //===- MCDirectives.h - Enums for directives on various targets -*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -20,6 +19,7 @@ enum MCSymbolAttr {
   MCSA_Invalid = 0,    ///< Not a valid directive.
 
   // Various directives in alphabetical order.
+  MCSA_Cold,                ///< .cold (MachO)
   MCSA_ELF_TypeFunction,    ///< .type _foo, STT_FUNC  # aka @function
   MCSA_ELF_TypeIndFunction, ///< .type _foo, STT_GNU_IFUNC
   MCSA_ELF_TypeObject,      ///< .type _foo, STT_OBJECT  # aka @object
diff --git a/include/llvm/MC/MCDisassembler/MCDisassembler.h b/include/llvm/MC/MCDisassembler/MCDisassembler.h
index 7f09c05ccf2abfdbf6ec8cb7707dc209716a4161..268f3ccad8897d6413ec1806e50493b03b5a4d1e 100644
--- a/include/llvm/MC/MCDisassembler/MCDisassembler.h
+++ b/include/llvm/MC/MCDisassembler/MCDisassembler.h
@@ -1,9 +1,8 @@
 //===- llvm/MC/MCDisassembler.h - Disassembler interface --------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -17,6 +16,7 @@
 namespace llvm {
 
 template <typename T> class ArrayRef;
+class StringRef;
 class MCContext;
 class MCInst;
 class MCSubtargetInfo;
@@ -80,6 +80,23 @@ public:
                                       raw_ostream &VStream,
                                       raw_ostream &CStream) const = 0;
 
+  /// May parse any prelude that precedes instructions after the start of a
+  /// symbol. Needed for some targets, e.g. WebAssembly.
+  ///
+  /// \param Name     - The name of the symbol.
+  /// \param Size     - The number of bytes consumed.
+  /// \param Address  - The address, in the memory space of region, of the first
+  ///                   byte of the symbol.
+  /// \param Bytes    - A reference to the actual bytes at the symbol location.
+  /// \param VStream  - The stream to print warnings and diagnostic messages on.
+  /// \param CStream  - The stream to print comments and annotations on.
+  /// \return         - MCDisassembler::Success if the bytes are valid,
+  ///                   MCDisassembler::Fail if the bytes were invalid.
+  virtual DecodeStatus onSymbolStart(StringRef Name, uint64_t &Size,
+                                     ArrayRef<uint8_t> Bytes, uint64_t Address,
+                                     raw_ostream &VStream,
+                                     raw_ostream &CStream) const;
+
 private:
   MCContext &Ctx;
 
diff --git a/include/llvm/MC/MCDisassembler/MCExternalSymbolizer.h b/include/llvm/MC/MCDisassembler/MCExternalSymbolizer.h
index df909a0dccd3fb4367dac7c71f2412d235e33f0c..ffac5ee5cb1f61b4772195fba17c9fd826189d18 100644
--- a/include/llvm/MC/MCDisassembler/MCExternalSymbolizer.h
+++ b/include/llvm/MC/MCDisassembler/MCExternalSymbolizer.h
@@ -1,9 +1,8 @@
 //===-- llvm/MC/MCExternalSymbolizer.h - ------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/MC/MCDisassembler/MCRelocationInfo.h b/include/llvm/MC/MCDisassembler/MCRelocationInfo.h
index 6030ae660d38b843c17462ac9b16df531e23be53..efc59da1933539fa4c6dcc40d959230bb40a24d5 100644
--- a/include/llvm/MC/MCDisassembler/MCRelocationInfo.h
+++ b/include/llvm/MC/MCDisassembler/MCRelocationInfo.h
@@ -1,9 +1,8 @@
 //===- llvm/MC/MCRelocationInfo.h -------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/MC/MCDisassembler/MCSymbolizer.h b/include/llvm/MC/MCDisassembler/MCSymbolizer.h
index 0bfa569474ec59e40a7e586eef4a862862cbed88..b7ca83a5f16c053ff68e451cfc9772fd0b9ba9fe 100644
--- a/include/llvm/MC/MCDisassembler/MCSymbolizer.h
+++ b/include/llvm/MC/MCDisassembler/MCSymbolizer.h
@@ -1,9 +1,8 @@
 //===- llvm/MC/MCSymbolizer.h - MCSymbolizer class --------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/MC/MCDwarf.h b/include/llvm/MC/MCDwarf.h
index aeaa4e55879b08983bef054277ac874379657fa0..8eaa1523fe843ecbba5a4cbb1b13701d3ce9b42a 100644
--- a/include/llvm/MC/MCDwarf.h
+++ b/include/llvm/MC/MCDwarf.h
@@ -1,9 +1,8 @@
 //===- MCDwarf.h - Machine Code Dwarf support -------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -599,6 +598,7 @@ struct MCDwarfFrameInfo {
   bool IsSignalFrame = false;
   bool IsSimple = false;
   unsigned RAReg = static_cast<unsigned>(INT_MAX);
+  bool IsBKeyFrame = false;
 };
 
 class MCDwarfFrameEmitter {
diff --git a/include/llvm/MC/MCELFObjectWriter.h b/include/llvm/MC/MCELFObjectWriter.h
index f226d6a45a5ab637500c2f1a8d4e8292ea718148..9ff7011df1d9e7f14c2dfa77e620793b3b7a4ef2 100644
--- a/include/llvm/MC/MCELFObjectWriter.h
+++ b/include/llvm/MC/MCELFObjectWriter.h
@@ -1,9 +1,8 @@
 //===- llvm/MC/MCELFObjectWriter.h - ELF Object Writer ----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/MC/MCELFStreamer.h b/include/llvm/MC/MCELFStreamer.h
index 3797079661e42df2bddc418632fae90c32cdbe4d..8838d53d75b5680014b5097194bfdecadb84e400 100644
--- a/include/llvm/MC/MCELFStreamer.h
+++ b/include/llvm/MC/MCELFStreamer.h
@@ -1,9 +1,8 @@
 //===- MCELFStreamer.h - MCStreamer ELF Object File Interface ---*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/MC/MCExpr.h b/include/llvm/MC/MCExpr.h
index 8cb6b86fd672eedd9d8e57a117fb6fb8146f80bc..853163032911a5c08542c946e3a284d586bc402b 100644
--- a/include/llvm/MC/MCExpr.h
+++ b/include/llvm/MC/MCExpr.h
@@ -1,9 +1,8 @@
 //===- MCExpr.h - Assembly Level Expressions --------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/MC/MCFixedLenDisassembler.h b/include/llvm/MC/MCFixedLenDisassembler.h
index ad34d9494bb9f9e1fa72bd9d162fefbade5283f9..218ae0d13189767b84b600f62ff34e791b5e4d45 100644
--- a/include/llvm/MC/MCFixedLenDisassembler.h
+++ b/include/llvm/MC/MCFixedLenDisassembler.h
@@ -1,9 +1,8 @@
 //===-- llvm/MC/MCFixedLenDisassembler.h - Decoder driver -------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 // Fixed length disassembler decoder state machine driver.
diff --git a/include/llvm/MC/MCFixup.h b/include/llvm/MC/MCFixup.h
index 5f301eafc556795a3e2470d5ddc820959f206e62..559a19e90569bc0650818e39574165c5ea69562a 100644
--- a/include/llvm/MC/MCFixup.h
+++ b/include/llvm/MC/MCFixup.h
@@ -1,9 +1,8 @@
 //===-- llvm/MC/MCFixup.h - Instruction Relocation and Patching -*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/MC/MCFixupKindInfo.h b/include/llvm/MC/MCFixupKindInfo.h
index 483abb39403fff8fee18ae42a2ca60d2c6f64f5e..0ea34866db6ac94ac6ec19f0380966954c405fcc 100644
--- a/include/llvm/MC/MCFixupKindInfo.h
+++ b/include/llvm/MC/MCFixupKindInfo.h
@@ -1,9 +1,8 @@
 //===-- llvm/MC/MCFixupKindInfo.h - Fixup Descriptors -----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/MC/MCFragment.h b/include/llvm/MC/MCFragment.h
index c999c9fc4f17cfdc2b018188d080963884b99864..aadf2ce725eac45058a5c7b35d0bd357beacf573 100644
--- a/include/llvm/MC/MCFragment.h
+++ b/include/llvm/MC/MCFragment.h
@@ -1,9 +1,8 @@
 //===- MCFragment.h - Fragment type hierarchy -------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/MC/MCInst.h b/include/llvm/MC/MCInst.h
index d501b686bb2e79d83fc397db4531d46b43850483..9675b786745f10301a0c2a920c71c436a60fa43e 100644
--- a/include/llvm/MC/MCInst.h
+++ b/include/llvm/MC/MCInst.h
@@ -1,9 +1,8 @@
 //===- llvm/MC/MCInst.h - MCInst class --------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -154,8 +153,6 @@ public:
   bool evaluateAsConstantImm(int64_t &Imm) const;
 };
 
-template <> struct isPodLike<MCOperand> { static const bool value = true; };
-
 /// Instances of this class represent a single low-level machine
 /// instruction.
 class MCInst {
diff --git a/include/llvm/MC/MCInstBuilder.h b/include/llvm/MC/MCInstBuilder.h
index c5c4f481e7df8b7f1da0d9c96b43e7b833ac091c..0c8e01fdc412c4da668bd3609a0c8cf85d059c0c 100644
--- a/include/llvm/MC/MCInstBuilder.h
+++ b/include/llvm/MC/MCInstBuilder.h
@@ -1,9 +1,8 @@
 //===-- llvm/MC/MCInstBuilder.h - Simplify creation of MCInsts --*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/MC/MCInstPrinter.h b/include/llvm/MC/MCInstPrinter.h
index df221e1db0e71c2f26aa31582a0666319d34271f..916aa8928c73ed3a730ce1362357be91cb326fbd 100644
--- a/include/llvm/MC/MCInstPrinter.h
+++ b/include/llvm/MC/MCInstPrinter.h
@@ -1,9 +1,8 @@
 //===- MCInstPrinter.h - MCInst to target assembly syntax -------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/MC/MCInstrAnalysis.h b/include/llvm/MC/MCInstrAnalysis.h
index 200f10f7d64beaa698d3e9560d1a81c2e3c86f3f..dfefd7e72777efb7b598a728875268f6cb66f47b 100644
--- a/include/llvm/MC/MCInstrAnalysis.h
+++ b/include/llvm/MC/MCInstrAnalysis.h
@@ -1,9 +1,8 @@
 //===- llvm/MC/MCInstrAnalysis.h - InstrDesc target hooks -------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/MC/MCInstrDesc.h b/include/llvm/MC/MCInstrDesc.h
index 61e7d09afbcbc4f6507c89c5b936612396be049c..ad5a3e9c675f36b1284b0b713cd12dd22a88c772 100644
--- a/include/llvm/MC/MCInstrDesc.h
+++ b/include/llvm/MC/MCInstrDesc.h
@@ -1,9 +1,8 @@
 //===-- llvm/MC/MCInstrDesc.h - Instruction Descriptors -*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/MC/MCInstrInfo.h b/include/llvm/MC/MCInstrInfo.h
index 18da87cf89296cf2c04b01061c37eea91ddc144b..874b1e46795b1e4c911b93a924c7ef186982ada0 100644
--- a/include/llvm/MC/MCInstrInfo.h
+++ b/include/llvm/MC/MCInstrInfo.h
@@ -1,9 +1,8 @@
 //===-- llvm/MC/MCInstrInfo.h - Target Instruction Info ---------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/MC/MCInstrItineraries.h b/include/llvm/MC/MCInstrItineraries.h
index fe81376e0db72008532129f610f32485980b0179..485aa663272efe1edb9d9b72dc91dd6df6fa9e4b 100644
--- a/include/llvm/MC/MCInstrItineraries.h
+++ b/include/llvm/MC/MCInstrItineraries.h
@@ -1,9 +1,8 @@
 //===- llvm/MC/MCInstrItineraries.h - Scheduling ----------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/MC/MCLabel.h b/include/llvm/MC/MCLabel.h
index aaf70691fc01aae0649f6d36851caf9b3a6a371e..0b8afac8f7548d8272b2a696b5ab54c92a7f3e7f 100644
--- a/include/llvm/MC/MCLabel.h
+++ b/include/llvm/MC/MCLabel.h
@@ -1,9 +1,8 @@
 //===- MCLabel.h - Machine Code Directional Local Labels --------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/MC/MCLinkerOptimizationHint.h b/include/llvm/MC/MCLinkerOptimizationHint.h
index f0fd07f43cf33ac567eef97cb14ae6c7d2e1c62b..f2a1364ad88416726e9442625b514da4499fb327 100644
--- a/include/llvm/MC/MCLinkerOptimizationHint.h
+++ b/include/llvm/MC/MCLinkerOptimizationHint.h
@@ -1,10 +1,9 @@
 //===- MCLinkerOptimizationHint.h - LOH interface ---------------*- C++ -*-===//
 //
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/MC/MCMachObjectWriter.h b/include/llvm/MC/MCMachObjectWriter.h
index 22fbeb72a4ec6c4b3e3b5d03dfe35be3f02e87df..278aebee99acdc5350a88c3a3d301b3edc47de35 100644
--- a/include/llvm/MC/MCMachObjectWriter.h
+++ b/include/llvm/MC/MCMachObjectWriter.h
@@ -1,9 +1,8 @@
 //===- llvm/MC/MCMachObjectWriter.h - Mach Object Writer --------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/MC/MCObjectFileInfo.h b/include/llvm/MC/MCObjectFileInfo.h
index f8142ccd8ac5b68ba95fc92731e4c87a2e0ede6c..0c1ade48fecc929f1be129eecdf9e852b79bdc82 100644
--- a/include/llvm/MC/MCObjectFileInfo.h
+++ b/include/llvm/MC/MCObjectFileInfo.h
@@ -1,9 +1,8 @@
 //===-- llvm/MC/MCObjectFileInfo.h - Object File Info -----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/MC/MCObjectStreamer.h b/include/llvm/MC/MCObjectStreamer.h
index 892909656c150a070023cf22e0552ccec02c670f..8affca49490fa47bccb2e15f64b0be1f897a78c8 100644
--- a/include/llvm/MC/MCObjectStreamer.h
+++ b/include/llvm/MC/MCObjectStreamer.h
@@ -1,9 +1,8 @@
 //===- MCObjectStreamer.h - MCStreamer Object File Interface ----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -116,8 +115,7 @@ public:
   void EmitSLEB128Value(const MCExpr *Value) override;
   void EmitWeakReference(MCSymbol *Alias, const MCSymbol *Symbol) override;
   void ChangeSection(MCSection *Section, const MCExpr *Subsection) override;
-  void EmitInstruction(const MCInst &Inst, const MCSubtargetInfo &STI,
-                       bool = false) override;
+  void EmitInstruction(const MCInst &Inst, const MCSubtargetInfo &STI) override;
 
   /// Emit an instruction to a special fragment, because this instruction
   /// can change its size during relaxation.
diff --git a/include/llvm/MC/MCObjectWriter.h b/include/llvm/MC/MCObjectWriter.h
index 8bae2bf200832f8132f011ba58495d559381bd6e..2547b2b7c9c18937562126d8011cf27676a2ec5a 100644
--- a/include/llvm/MC/MCObjectWriter.h
+++ b/include/llvm/MC/MCObjectWriter.h
@@ -1,9 +1,8 @@
 //===- llvm/MC/MCObjectWriter.h - Object File Writer Interface --*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/MC/MCParser/AsmCond.h b/include/llvm/MC/MCParser/AsmCond.h
index a6e0fbd7f3375e9523455bcb50e9c9120660f813..ea21550100814d1dd48b9ff314ddc4cf1effc9a3 100644
--- a/include/llvm/MC/MCParser/AsmCond.h
+++ b/include/llvm/MC/MCParser/AsmCond.h
@@ -1,9 +1,8 @@
 //===- AsmCond.h - Assembly file conditional assembly  ----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/MC/MCParser/AsmLexer.h b/include/llvm/MC/MCParser/AsmLexer.h
index 2e9b8dfa3b260f7b59252d4a1d9a4762cb04750e..b7294493b2f801992633109c9f3bb720e678a07d 100644
--- a/include/llvm/MC/MCParser/AsmLexer.h
+++ b/include/llvm/MC/MCParser/AsmLexer.h
@@ -1,9 +1,8 @@
 //===- AsmLexer.h - Lexer for Assembly Files --------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/MC/MCParser/MCAsmLexer.h b/include/llvm/MC/MCParser/MCAsmLexer.h
index ea13d1cdc09f04907402243461ec53f09900ee52..e89abeaac94c2f7e53747c8625efeaa24091f160 100644
--- a/include/llvm/MC/MCParser/MCAsmLexer.h
+++ b/include/llvm/MC/MCParser/MCAsmLexer.h
@@ -1,9 +1,8 @@
 //===- llvm/MC/MCAsmLexer.h - Abstract Asm Lexer Interface ------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/MC/MCParser/MCAsmParser.h b/include/llvm/MC/MCParser/MCAsmParser.h
index b80289878e6e61f34dcfe8b5bfe296a5e9bb6f9c..da5653ee71d3ba75467d8e686d071fb1b81fd479 100644
--- a/include/llvm/MC/MCParser/MCAsmParser.h
+++ b/include/llvm/MC/MCParser/MCAsmParser.h
@@ -1,9 +1,8 @@
 //===- llvm/MC/MCAsmParser.h - Abstract Asm Parser Interface ----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -130,9 +129,6 @@ protected: // Can only create subclasses.
   /// Flag tracking whether any errors have been encountered.
   bool HadError = false;
 
-  /// Enable print [latency:throughput] in output file.
-  bool EnablePrintSchedInfo = false;
-
   bool ShowParsedOperands = false;
 
 public:
@@ -166,9 +162,6 @@ public:
   bool getShowParsedOperands() const { return ShowParsedOperands; }
   void setShowParsedOperands(bool Value) { ShowParsedOperands = Value; }
 
-  void setEnablePrintSchedInfo(bool Value) { EnablePrintSchedInfo = Value; }
-  bool shouldPrintSchedInfo() const { return EnablePrintSchedInfo; }
-
   /// Run the parser on the input source buffer.
   virtual bool Run(bool NoInitialTextSection, bool NoFinalize = false) = 0;
 
diff --git a/include/llvm/MC/MCParser/MCAsmParserExtension.h b/include/llvm/MC/MCParser/MCAsmParserExtension.h
index 1a132bceddc57c4def1217e3ea2cafbb30ddecf1..5d2afe81a54ba7422137e4dcd7853fc3c10129fe 100644
--- a/include/llvm/MC/MCParser/MCAsmParserExtension.h
+++ b/include/llvm/MC/MCParser/MCAsmParserExtension.h
@@ -1,9 +1,8 @@
 //===- llvm/MC/MCAsmParserExtension.h - Asm Parser Hooks --------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/MC/MCParser/MCAsmParserUtils.h b/include/llvm/MC/MCParser/MCAsmParserUtils.h
index 84173bb9cb8ebe97a1af130b374c67ad3d8dc1ed..d692da7402fefef57cb127809859b1fff379a77b 100644
--- a/include/llvm/MC/MCParser/MCAsmParserUtils.h
+++ b/include/llvm/MC/MCParser/MCAsmParserUtils.h
@@ -1,9 +1,8 @@
 //===- llvm/MC/MCAsmParserUtils.h - Asm Parser Utilities --------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/MC/MCParser/MCParsedAsmOperand.h b/include/llvm/MC/MCParser/MCParsedAsmOperand.h
index 4af76ac2a858a71cd6727f4b64617f8c8f46a0e9..2b6e2aa48b8f1761d73741b8a09133ab5ac55e56 100644
--- a/include/llvm/MC/MCParser/MCParsedAsmOperand.h
+++ b/include/llvm/MC/MCParser/MCParsedAsmOperand.h
@@ -1,9 +1,8 @@
 //===- llvm/MC/MCParsedAsmOperand.h - Asm Parser Operand --------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/MC/MCParser/MCTargetAsmParser.h b/include/llvm/MC/MCParser/MCTargetAsmParser.h
index bb979422f0a5b0841a8fa5e7120343609fdfc342..c5683b74c4cefa14c98b855569d1b0ff80e3c49d 100644
--- a/include/llvm/MC/MCParser/MCTargetAsmParser.h
+++ b/include/llvm/MC/MCParser/MCTargetAsmParser.h
@@ -1,9 +1,8 @@
 //===- llvm/MC/MCTargetAsmParser.h - Target Assembly Parser -----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -490,6 +489,9 @@ public:
                                               MCContext &Ctx) {
     return nullptr;
   }
+
+  // For any checks or cleanups at the end of parsing.
+  virtual void onEndOfFile() {}
 };
 
 } // end namespace llvm
diff --git a/include/llvm/MC/MCRegisterInfo.h b/include/llvm/MC/MCRegisterInfo.h
index 8d8c677c77eafcea0e36f901e75504b593535ba5..92d39c3fcfb7e3fd70a0bc5e42c7527e64daf843 100644
--- a/include/llvm/MC/MCRegisterInfo.h
+++ b/include/llvm/MC/MCRegisterInfo.h
@@ -1,9 +1,8 @@
 //===- MC/MCRegisterInfo.h - Target Register Description --------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/MC/MCSchedule.h b/include/llvm/MC/MCSchedule.h
index 689ac73cbdd1c37f6aeb882b7ba9b1dfeb6eb001..df3248ee6e863f1528c2efaac9e57d4ee32cea06 100644
--- a/include/llvm/MC/MCSchedule.h
+++ b/include/llvm/MC/MCSchedule.h
@@ -1,9 +1,8 @@
 //===-- llvm/MC/MCSchedule.h - Scheduling -----------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -15,6 +14,7 @@
 #ifndef LLVM_MC_MCSCHEDULE_H
 #define LLVM_MC_MCSCHEDULE_H
 
+#include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/Optional.h"
 #include "llvm/Config/llvm-config.h"
 #include "llvm/Support/DataTypes.h"
@@ -370,6 +370,11 @@ struct MCSchedModel {
   getReciprocalThroughput(const MCSubtargetInfo &STI, const MCInstrInfo &MCII,
                           const MCInst &Inst) const;
 
+  /// Returns the maximum forwarding delay for register reads dependent on
+  /// writes of scheduling class WriteResourceIdx.
+  static unsigned getForwardingDelayCycles(ArrayRef<MCReadAdvanceEntry> Entries,
+                                           unsigned WriteResourceIdx = 0);
+
   /// Returns the default initialized model.
   static const MCSchedModel &GetDefaultSchedModel() { return Default; }
   static const MCSchedModel Default;
diff --git a/include/llvm/MC/MCSection.h b/include/llvm/MC/MCSection.h
index eb210b4e9dfa2b484ea80ec5e877c816d3f20dcc..27490dc89c6196fc5073be1c07d3a0e1881f2d6b 100644
--- a/include/llvm/MC/MCSection.h
+++ b/include/llvm/MC/MCSection.h
@@ -1,9 +1,8 @@
 //===- MCSection.h - Machine Code Sections ----------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/MC/MCSectionCOFF.h b/include/llvm/MC/MCSectionCOFF.h
index 24b9f8898ebb48c02420a628ff515e0fcfb1a4f9..8be95e0f1de5dd8e16ad5260d65d54444ab9b22d 100644
--- a/include/llvm/MC/MCSectionCOFF.h
+++ b/include/llvm/MC/MCSectionCOFF.h
@@ -1,9 +1,8 @@
 //===- MCSectionCOFF.h - COFF Machine Code Sections -------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -63,8 +62,6 @@ private:
   }
 
 public:
-  ~MCSectionCOFF();
-
   /// Decides whether a '.section' directive should be printed before the
   /// section name
   bool ShouldOmitSectionDirective(StringRef Name, const MCAsmInfo &MAI) const;
diff --git a/include/llvm/MC/MCSectionELF.h b/include/llvm/MC/MCSectionELF.h
index 00c289c6bd6ebfbf83d01f2670361e7d102b5b1d..fe6b2d7afc79411a6065a78fc81c8de915e8cef0 100644
--- a/include/llvm/MC/MCSectionELF.h
+++ b/include/llvm/MC/MCSectionELF.h
@@ -1,9 +1,8 @@
 //===- MCSectionELF.h - ELF Machine Code Sections ---------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -64,8 +63,6 @@ private:
   void setSectionName(StringRef Name) { SectionName = Name; }
 
 public:
-  ~MCSectionELF();
-
   /// Decides whether a '.section' directive should be printed before the
   /// section name
   bool ShouldOmitSectionDirective(StringRef Name, const MCAsmInfo &MAI) const;
diff --git a/include/llvm/MC/MCSectionMachO.h b/include/llvm/MC/MCSectionMachO.h
index 89db09cbdbdc175aec55a89d47308228f96dcdfe..2c73661fb1fd65cb0614797e8d72c883920bba96 100644
--- a/include/llvm/MC/MCSectionMachO.h
+++ b/include/llvm/MC/MCSectionMachO.h
@@ -1,9 +1,8 @@
 //===- MCSectionMachO.h - MachO Machine Code Sections -----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/MC/MCSectionWasm.h b/include/llvm/MC/MCSectionWasm.h
index ab4cd7b007ec9fe99907a05399716af59486c4fd..0e576b7ba404fdf957671f3d84bdf159ce057443 100644
--- a/include/llvm/MC/MCSectionWasm.h
+++ b/include/llvm/MC/MCSectionWasm.h
@@ -1,9 +1,8 @@
 //===- MCSectionWasm.h - Wasm Machine Code Sections -------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -49,14 +48,10 @@ class MCSectionWasm final : public MCSection {
       : MCSection(SV_Wasm, K, Begin), SectionName(Section), UniqueID(UniqueID),
         Group(group) {}
 
-  void setSectionName(StringRef Name) { SectionName = Name; }
-
 public:
-  ~MCSectionWasm();
-
   /// Decides whether a '.section' directive should be printed before the
   /// section name
-  bool ShouldOmitSectionDirective(StringRef Name, const MCAsmInfo &MAI) const;
+  bool shouldOmitSectionDirective(StringRef Name, const MCAsmInfo &MAI) const;
 
   StringRef getSectionName() const { return SectionName; }
   const MCSymbolWasm *getGroup() const { return Group; }
diff --git a/include/llvm/MC/MCStreamer.h b/include/llvm/MC/MCStreamer.h
index 849ea9a624e899455a1fbc11f95ed44d7b479b62..dc2386fca85f23a49d24ed213a44625118e80e0f 100644
--- a/include/llvm/MC/MCStreamer.h
+++ b/include/llvm/MC/MCStreamer.h
@@ -1,9 +1,8 @@
 //===- MCStreamer.h - High-level Streaming Machine Code Output --*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -806,6 +805,8 @@ public:
                                        Optional<StringRef> Source,
                                        unsigned CUID = 0);
 
+  virtual void EmitCFIBKeyFrame();
+
   /// This implements the DWARF2 '.loc fileno lineno ...' assembler
   /// directive.
   virtual void EmitDwarfLocDirective(unsigned FileNo, unsigned Line,
@@ -951,9 +952,7 @@ public:
   virtual void EmitAddrsigSym(const MCSymbol *Sym) {}
 
   /// Emit the given \p Instruction into the current section.
-  /// PrintSchedInfo == true then schedul comment should be added to output
-  virtual void EmitInstruction(const MCInst &Inst, const MCSubtargetInfo &STI,
-                               bool PrintSchedInfo = false);
+  virtual void EmitInstruction(const MCInst &Inst, const MCSubtargetInfo &STI);
 
   /// Set the bundle alignment mode from now on in the section.
   /// The argument is the power of 2 to which the alignment is set. The
diff --git a/include/llvm/MC/MCSubtargetInfo.h b/include/llvm/MC/MCSubtargetInfo.h
index b3ce523d9c0ca4da9c5a3a8156019a600ef1c4f0..c7472a28c7747329f758af1a96a24d29bf27959c 100644
--- a/include/llvm/MC/MCSubtargetInfo.h
+++ b/include/llvm/MC/MCSubtargetInfo.h
@@ -1,9 +1,8 @@
 //===- llvm/MC/MCSubtargetInfo.h - Subtarget Information --------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -153,6 +152,16 @@ public:
     return 0;
   }
 
+  /// Return the set of ReadAdvance entries declared by the scheduling class
+  /// descriptor in input.
+  ArrayRef<MCReadAdvanceEntry>
+  getReadAdvanceEntries(const MCSchedClassDesc &SC) const {
+    if (!SC.NumReadAdvanceEntries)
+      return ArrayRef<MCReadAdvanceEntry>();
+    return ArrayRef<MCReadAdvanceEntry>(&ReadAdvanceTable[SC.ReadAdvanceIdx],
+                                        SC.NumReadAdvanceEntries);
+  }
+
   /// Get scheduling itinerary of a CPU.
   InstrItineraryData getInstrItineraryForCPU(StringRef CPU) const;
 
@@ -171,11 +180,6 @@ public:
     auto Found = std::lower_bound(ProcDesc.begin(), ProcDesc.end(), CPU);
     return Found != ProcDesc.end() && StringRef(Found->Key) == CPU;
   }
-
-  /// Returns string representation of scheduler comment
-  virtual std::string getSchedInfoStr(MCInst const &MCI) const {
-    return {};
-  }
 };
 
 } // end namespace llvm
diff --git a/include/llvm/MC/MCSymbol.h b/include/llvm/MC/MCSymbol.h
index 4681a1be60c4467b883ba6dd560f9dd9cbcd8a2c..273e5a339f3f369a59483b2978facad60774e552 100644
--- a/include/llvm/MC/MCSymbol.h
+++ b/include/llvm/MC/MCSymbol.h
@@ -1,9 +1,8 @@
 //===- MCSymbol.h - Machine Code Symbols ------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/MC/MCSymbolCOFF.h b/include/llvm/MC/MCSymbolCOFF.h
index 7918c353dc1546f310174e2aa2959b2816050a4f..94087ce871aedad51fd6b8f72a2c415a966c4b38 100644
--- a/include/llvm/MC/MCSymbolCOFF.h
+++ b/include/llvm/MC/MCSymbolCOFF.h
@@ -1,9 +1,8 @@
 //===- MCSymbolCOFF.h -  ----------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/MC/MCSymbolELF.h b/include/llvm/MC/MCSymbolELF.h
index bbcd22e8e7dbe15fac4238ee86a28b3198d133fe..34e5c4344affcd1febda6884e5ddec20772df39d 100644
--- a/include/llvm/MC/MCSymbolELF.h
+++ b/include/llvm/MC/MCSymbolELF.h
@@ -1,9 +1,8 @@
 //===- MCSymbolELF.h -  -----------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 #ifndef LLVM_MC_MCSYMBOLELF_H
diff --git a/include/llvm/MC/MCSymbolMachO.h b/include/llvm/MC/MCSymbolMachO.h
index 6125c2050976849e38ec9b9268ffb348ea9a4cbf..8f9ff56470a77432845d9c2e6b4c22f91a806cba 100644
--- a/include/llvm/MC/MCSymbolMachO.h
+++ b/include/llvm/MC/MCSymbolMachO.h
@@ -1,9 +1,8 @@
 //===- MCSymbolMachO.h -  ---------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 #ifndef LLVM_MC_MCSYMBOLMACHO_H
@@ -35,6 +34,7 @@ class MCSymbolMachO : public MCSymbol {
     SF_WeakDefinition                       = 0x0080,
     SF_SymbolResolver                       = 0x0100,
     SF_AltEntry                             = 0x0200,
+    SF_Cold                                 = 0x0400,
 
     // Common alignment
     SF_CommonAlignmentMask                  = 0xF0FF,
@@ -98,6 +98,10 @@ public:
     return getFlags() & SF_AltEntry;
   }
 
+  void setCold() const { modifyFlags(SF_Cold, SF_Cold); }
+
+  bool isCold() const { return getFlags() & SF_Cold; }
+
   void setDesc(unsigned Value) const {
     assert(Value == (Value & SF_DescFlagsMask) &&
            "Invalid .desc value!");
diff --git a/include/llvm/MC/MCSymbolWasm.h b/include/llvm/MC/MCSymbolWasm.h
index 8e66dc881d0fba45ca0cac9278fd8202b0fdb853..88759fe4d9e6b72c570cf9d06a2d27718578f296 100644
--- a/include/llvm/MC/MCSymbolWasm.h
+++ b/include/llvm/MC/MCSymbolWasm.h
@@ -1,9 +1,8 @@
 //===- MCSymbolWasm.h -  ----------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 #ifndef LLVM_MC_MCSYMBOLWASM_H
@@ -19,7 +18,8 @@ class MCSymbolWasm : public MCSymbol {
   bool IsWeak = false;
   bool IsHidden = false;
   bool IsComdat = false;
-  std::string ModuleName;
+  Optional<std::string> ImportModule;
+  Optional<std::string> ImportName;
   wasm::WasmSignature *Signature = nullptr;
   Optional<wasm::WasmGlobalType> GlobalType;
   Optional<wasm::WasmEventType> EventType;
@@ -32,7 +32,7 @@ public:
   // Use a module name of "env" for now, for compatibility with existing tools.
   // This is temporary, and may change, as the ABI is not yet stable.
   MCSymbolWasm(const StringMapEntry<bool> *Name, bool isTemporary)
-      : MCSymbol(SymbolKindWasm, Name, isTemporary), ModuleName("env") {}
+      : MCSymbol(SymbolKindWasm, Name, isTemporary) {}
   static bool classof(const MCSymbol *S) { return S->isWasm(); }
 
   const MCExpr *getSize() const { return SymbolSize; }
@@ -46,6 +46,13 @@ public:
   wasm::WasmSymbolType getType() const { return Type; }
   void setType(wasm::WasmSymbolType type) { Type = type; }
 
+  bool isExported() const {
+    return getFlags() & wasm::WASM_SYMBOL_EXPORTED;
+  }
+  void setExported() const {
+    modifyFlags(wasm::WASM_SYMBOL_EXPORTED, wasm::WASM_SYMBOL_EXPORTED);
+  }
+
   bool isWeak() const { return IsWeak; }
   void setWeak(bool isWeak) { IsWeak = isWeak; }
 
@@ -55,8 +62,21 @@ public:
   bool isComdat() const { return IsComdat; }
   void setComdat(bool isComdat) { IsComdat = isComdat; }
 
-  const StringRef getModuleName() const { return ModuleName; }
-  void setModuleName(StringRef Name) { ModuleName = Name; }
+  const StringRef getImportModule() const {
+      if (ImportModule.hasValue()) {
+          return ImportModule.getValue();
+      }
+      return "env";
+  }
+  void setImportModule(StringRef Name) { ImportModule = Name; }
+
+  const StringRef getImportName() const {
+      if (ImportName.hasValue()) {
+          return ImportName.getValue();
+      }
+      return getName();
+  }
+  void setImportName(StringRef Name) { ImportName = Name; }
 
   const wasm::WasmSignature *getSignature() const { return Signature; }
   void setSignature(wasm::WasmSignature *Sig) { Signature = Sig; }
diff --git a/include/llvm/MC/MCTargetOptions.h b/include/llvm/MC/MCTargetOptions.h
index f5d330fbeb22f9fb6715778e6356acf37893869e..087d155393c216251027b712d35165c081586fa9 100644
--- a/include/llvm/MC/MCTargetOptions.h
+++ b/include/llvm/MC/MCTargetOptions.h
@@ -1,9 +1,8 @@
 //===- MCTargetOptions.h - MC Target Options --------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/MC/MCTargetOptionsCommandFlags.inc b/include/llvm/MC/MCTargetOptionsCommandFlags.inc
index 5172fa44511fa9b8548291e6295342959ea2949d..f8129463b54ca144aab6a2f237878e2f4265992b 100644
--- a/include/llvm/MC/MCTargetOptionsCommandFlags.inc
+++ b/include/llvm/MC/MCTargetOptionsCommandFlags.inc
@@ -1,9 +1,8 @@
 //===-- MCTargetOptionsCommandFlags.h --------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/MC/MCValue.h b/include/llvm/MC/MCValue.h
index 11f5082ed3f4849feba5702ad612cbfe4e14e936..0be7ce7055c5ac41dadd81e0cbb3b435ba3f7066 100644
--- a/include/llvm/MC/MCValue.h
+++ b/include/llvm/MC/MCValue.h
@@ -1,9 +1,8 @@
 //===-- llvm/MC/MCValue.h - MCValue class -----------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/MC/MCWasmObjectWriter.h b/include/llvm/MC/MCWasmObjectWriter.h
index 6b788cfe96b9a16d741f5eefc40447d5eb45d5e9..4adbca28f116ea3a1e25178f4af6eb1a0cb5d729 100644
--- a/include/llvm/MC/MCWasmObjectWriter.h
+++ b/include/llvm/MC/MCWasmObjectWriter.h
@@ -1,9 +1,8 @@
 //===-- llvm/MC/MCWasmObjectWriter.h - Wasm Object Writer -------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/MC/MCWasmStreamer.h b/include/llvm/MC/MCWasmStreamer.h
index 01e6a437928785dda831b1046ad86ad851b57f5e..2d7f2b9975c9d5293f55a8773177163daaaa656e 100644
--- a/include/llvm/MC/MCWasmStreamer.h
+++ b/include/llvm/MC/MCWasmStreamer.h
@@ -1,9 +1,8 @@
 //===- MCWasmStreamer.h - MCStreamer Wasm Object File Interface -*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/MC/MCWin64EH.h b/include/llvm/MC/MCWin64EH.h
index 1a9f6f403d7c6a62647415a2d9e105925879a20d..60ec06e61b7c8ce28c0d6767a9d4177837a79ffc 100644
--- a/include/llvm/MC/MCWin64EH.h
+++ b/include/llvm/MC/MCWin64EH.h
@@ -1,9 +1,8 @@
 //===- MCWin64EH.h - Machine Code Win64 EH support --------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/MC/MCWinCOFFObjectWriter.h b/include/llvm/MC/MCWinCOFFObjectWriter.h
index c1d35ea1f6baaad1413290ceb70be19484d0004f..3fe124fd7f1c4657b21898a6c20ea1ad17e1ca7a 100644
--- a/include/llvm/MC/MCWinCOFFObjectWriter.h
+++ b/include/llvm/MC/MCWinCOFFObjectWriter.h
@@ -1,9 +1,8 @@
 //===- llvm/MC/MCWinCOFFObjectWriter.h - Win COFF Object Writer -*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/MC/MCWinCOFFStreamer.h b/include/llvm/MC/MCWinCOFFStreamer.h
index 0049d04b4b3fb8148ada2c3db84efcdecd2fd91a..c1c1ec56cb48457b5b787bcf24a864c03798708e 100644
--- a/include/llvm/MC/MCWinCOFFStreamer.h
+++ b/include/llvm/MC/MCWinCOFFStreamer.h
@@ -1,9 +1,8 @@
 //===- MCWinCOFFStreamer.h - COFF Object File Interface ---------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/MC/MCWinEH.h b/include/llvm/MC/MCWinEH.h
index 98ef0367a11d115dc4f536dd079c8e86393a0260..b1c28c0ecae79471354574235362efb082e8af2b 100644
--- a/include/llvm/MC/MCWinEH.h
+++ b/include/llvm/MC/MCWinEH.h
@@ -1,9 +1,8 @@
 //===- MCWinEH.h - Windows Unwinding Support --------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/MC/MachineLocation.h b/include/llvm/MC/MachineLocation.h
index 91ed661ebeab5d828b63c864003e73685bd04681..5872540e6104dea8b1e8baba962ee4381464650c 100644
--- a/include/llvm/MC/MachineLocation.h
+++ b/include/llvm/MC/MachineLocation.h
@@ -1,9 +1,8 @@
 //===- llvm/MC/MachineLocation.h --------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 // The MachineLocation class is used to represent a simple location in a machine
diff --git a/include/llvm/MC/SectionKind.h b/include/llvm/MC/SectionKind.h
index 66eb9ec56d14fa93973139085fb58a74aa436942..0342c4cfbbdedd1ae50e904ac3b832fc8822b2e3 100644
--- a/include/llvm/MC/SectionKind.h
+++ b/include/llvm/MC/SectionKind.h
@@ -1,9 +1,8 @@
 //===-- llvm/MC/SectionKind.h - Classification of sections ------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/MC/StringTableBuilder.h b/include/llvm/MC/StringTableBuilder.h
index 265260fcee4d2f5e03e07b663412e9b689ee7cd7..c83eca4e512d743c1595a03d285a4ef40e1b0380 100644
--- a/include/llvm/MC/StringTableBuilder.h
+++ b/include/llvm/MC/StringTableBuilder.h
@@ -1,9 +1,8 @@
 //===- StringTableBuilder.h - String table building utility -----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/MC/SubtargetFeature.h b/include/llvm/MC/SubtargetFeature.h
index 76c7dd560800b4aa36a2305f30c1da8339f9e81d..5f49d3894ae6e6ca9093dab4517de83f32440b87 100644
--- a/include/llvm/MC/SubtargetFeature.h
+++ b/include/llvm/MC/SubtargetFeature.h
@@ -1,9 +1,8 @@
 //===- llvm/MC/SubtargetFeature.h - CPU characteristics ---------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/MCA/Context.h b/include/llvm/MCA/Context.h
index 6b2bee0fdc4210779189c4fe8e7b43dfe9426ed5..a9f3e05ce4483377ba42bcd99a55da49b2b3acbf 100644
--- a/include/llvm/MCA/Context.h
+++ b/include/llvm/MCA/Context.h
@@ -1,9 +1,8 @@
 //===---------------------------- Context.h ---------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/include/llvm/MCA/HWEventListener.h b/include/llvm/MCA/HWEventListener.h
index 3b32b2cd657721d7cb2694e5f7d0c2c684dedfcb..1857ad289c64b45502b4d171528072ccd88fb796 100644
--- a/include/llvm/MCA/HWEventListener.h
+++ b/include/llvm/MCA/HWEventListener.h
@@ -1,9 +1,8 @@
 //===----------------------- HWEventListener.h ------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/include/llvm/MCA/HardwareUnits/HardwareUnit.h b/include/llvm/MCA/HardwareUnits/HardwareUnit.h
index 104a2009f219d234986557de065cafd3ebe70aca..f6e178bcff1019add575558465aedda430fc99c5 100644
--- a/include/llvm/MCA/HardwareUnits/HardwareUnit.h
+++ b/include/llvm/MCA/HardwareUnits/HardwareUnit.h
@@ -1,9 +1,8 @@
 //===-------------------------- HardwareUnit.h ------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/include/llvm/MCA/HardwareUnits/LSUnit.h b/include/llvm/MCA/HardwareUnits/LSUnit.h
index e217fc50f780c5d78cc4a4051dd322c204660e52..b8a9f2755b0553fc77c998653345b4c87aa3a97e 100644
--- a/include/llvm/MCA/HardwareUnits/LSUnit.h
+++ b/include/llvm/MCA/HardwareUnits/LSUnit.h
@@ -1,9 +1,8 @@
 //===------------------------- LSUnit.h --------------------------*- C++-*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/include/llvm/MCA/HardwareUnits/RegisterFile.h b/include/llvm/MCA/HardwareUnits/RegisterFile.h
index c23ab038923481363f0d5718af4ca56f56dd983f..36506327bd292f4fec6bdcb86bf5b6dbc10d91a1 100644
--- a/include/llvm/MCA/HardwareUnits/RegisterFile.h
+++ b/include/llvm/MCA/HardwareUnits/RegisterFile.h
@@ -1,9 +1,8 @@
 //===--------------------- RegisterFile.h -----------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
@@ -21,6 +20,7 @@
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/MC/MCRegisterInfo.h"
 #include "llvm/MC/MCSchedule.h"
+#include "llvm/MC/MCSubtargetInfo.h"
 #include "llvm/MCA/HardwareUnits/HardwareUnit.h"
 #include "llvm/Support/Error.h"
 
@@ -196,7 +196,7 @@ public:
 
   // Collect writes that are in a data dependency with RS, and update RS
   // internal state.
-  void addRegisterRead(ReadState &RS, SmallVectorImpl<WriteRef> &Writes) const;
+  void addRegisterRead(ReadState &RS, const MCSubtargetInfo &STI) const;
 
   // Removes write \param WS from the register mappings.
   // Physical registers may be released to reflect this update.
diff --git a/include/llvm/MCA/HardwareUnits/ResourceManager.h b/include/llvm/MCA/HardwareUnits/ResourceManager.h
index 5429f2b8cf0a5c928f40a32ab1d26c3ae137e921..3addaaf4775989d835297a789be0eef4547b7014 100644
--- a/include/llvm/MCA/HardwareUnits/ResourceManager.h
+++ b/include/llvm/MCA/HardwareUnits/ResourceManager.h
@@ -1,9 +1,8 @@
 //===--------------------- ResourceManager.h --------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
@@ -72,7 +71,7 @@ class DefaultResourceStrategy final : public ResourceStrategy {
   ///
   /// There is one bit set for every available resource unit.
   /// It defaults to the value of field ResourceSizeMask in ResourceState.
-  const unsigned ResourceUnitMask;
+  const uint64_t ResourceUnitMask;
 
   /// A simple round-robin selector for processor resource units.
   /// Each bit of this mask identifies a sub resource within a group.
@@ -139,17 +138,42 @@ class ResourceState {
   /// An index to the MCProcResourceDesc entry in the processor model.
   const unsigned ProcResourceDescIndex;
   /// A resource mask. This is generated by the tool with the help of
-  /// function `mca::createProcResourceMasks' (see Support.h).
+  /// function `mca::computeProcResourceMasks' (see Support.h).
+  ///
+  /// Field ResourceMask only has one bit set if this resource state describes a
+  /// processor resource unit (i.e. this is not a group). That means, we can
+  /// quickly check if a resource is a group by simply counting the number of
+  /// bits that are set in the mask.
+  ///
+  /// The most significant bit of a mask (MSB) uniquely identifies a resource.
+  /// Remaining bits are used to describe the composition of a group (Group).
+  ///
+  /// Example (little endian):
+  ///            Resource |  Mask      |  MSB       |  Group
+  ///            ---------+------------+------------+------------
+  ///            A        |  0b000001  |  0b000001  |  0b000000
+  ///                     |            |            |
+  ///            B        |  0b000010  |  0b000010  |  0b000000
+  ///                     |            |            |
+  ///            C        |  0b010000  |  0b010000  |  0b000000
+  ///                     |            |            |
+  ///            D        |  0b110010  |  0b100000  |  0b010010
+  ///
+  /// In this example, resources A, B and C are processor resource units.
+  /// Only resource D is a group resource, and it contains resources B and C.
+  /// That is because MSB(B) and MSB(C) are both contained within Group(D).
   const uint64_t ResourceMask;
 
   /// A ProcResource can have multiple units.
   ///
-  /// For processor resource groups,
-  /// this field default to the value of field `ResourceMask`; the number of
-  /// bits set is equal to the cardinality of the group.  For normal (i.e.
-  /// non-group) resources, the number of bits set in this mask is equivalent
-  /// to the number of units declared by the processor model (see field
-  /// 'NumUnits' in 'ProcResourceUnits').
+  /// For processor resource groups this field is a mask of contained resource
+  /// units. It is obtained from ResourceMask by clearing the highest set bit.
+  /// The number of resource units in a group can be simply computed as the
+  /// population count of this field.
+  ///
+  /// For normal (i.e. non-group) resources, the number of bits set in this mask
+  /// is equivalent to the number of units declared by the processor model (see
+  /// field 'NumUnits' in 'ProcResourceUnits').
   uint64_t ResourceSizeMask;
 
   /// A mask of ready units.
@@ -277,16 +301,54 @@ typedef std::pair<unsigned, unsigned> BufferUsageEntry;
 /// In future, it can be extended to support itineraries too through the same
 /// public interface.
 class ResourceManager {
-  // The resource manager owns all the ResourceState.
+  // Set of resources available on the subtarget.
+  //
+  // There is an instance of ResourceState for every resource declared by the
+  // target scheduling model.
+  //
+  // Elements of this vector are ordered by resource kind. In particular,
+  // resource units take precedence over resource groups.
+  //
+  // The index of a processor resource in this vector depends on the value of
+  // its mask (see the description of field ResourceState::ResourceMask).  In
+  // particular, it is computed as the position of the most significant bit set
+  // (MSB) in the mask plus one (since we want to ignore the invalid resource
+  // descriptor at index zero).
+  //
+  // Example (little endian):
+  //
+  //             Resource | Mask    |  MSB    | Index
+  //             ---------+---------+---------+-------
+  //                 A    | 0b00001 | 0b00001 |   1
+  //                      |         |         |
+  //                 B    | 0b00100 | 0b00100 |   3
+  //                      |         |         |
+  //                 C    | 0b10010 | 0b10000 |   5
+  //
+  //
+  // The same index is also used to address elements within vector `Strategies`
+  // and vector `Resource2Groups`.
   std::vector<std::unique_ptr<ResourceState>> Resources;
   std::vector<std::unique_ptr<ResourceStrategy>> Strategies;
 
+  // Used to quickly identify groups that own a particular resource unit.
+  std::vector<uint64_t> Resource2Groups;
+
+  // A table to map processor resource IDs to processor resource masks.
+  SmallVector<uint64_t, 8> ProcResID2Mask;
+
   // Keeps track of which resources are busy, and how many cycles are left
   // before those become usable again.
   SmallDenseMap<ResourceRef, unsigned> BusyResources;
 
-  // A table to map processor resource IDs to processor resource masks.
-  SmallVector<uint64_t, 8> ProcResID2Mask;
+  // Set of processor resource units available on the target.
+  uint64_t ProcResUnitMask;
+
+  // Set of processor resource units that are available during this cycle.
+  uint64_t AvailableProcResUnits;
+
+  // Set of processor resource groups that are currently reserved.
+  uint64_t ReservedResourceGroups;
 
   // Returns the actual resource unit that will be used.
   ResourceRef selectPipe(uint64_t ResourceID);
@@ -335,11 +397,14 @@ public:
   // Release a previously reserved processor resource.
   void releaseResource(uint64_t ResourceID);
 
-  // Returns true if all resources are in-order, and there is at least one
-  // resource which is a dispatch hazard (BufferSize = 0).
-  bool mustIssueImmediately(const InstrDesc &Desc) const;
+  // Returns a zero mask if resources requested by Desc are all available during
+  // this cycle. It returns a non-zero mask value only if there are unavailable
+  // processor resources; each bit set in the mask represents a busy processor
+  // resource unit or a reserved processor resource group.
+  uint64_t checkAvailability(const InstrDesc &Desc) const;
 
-  bool canBeIssued(const InstrDesc &Desc) const;
+  uint64_t getProcResUnitMask() const { return ProcResUnitMask; }
+  uint64_t getAvailableProcResUnits() const { return AvailableProcResUnits; }
 
   void issueInstruction(
       const InstrDesc &Desc,
diff --git a/include/llvm/MCA/HardwareUnits/RetireControlUnit.h b/include/llvm/MCA/HardwareUnits/RetireControlUnit.h
index 71360e984ade5d48914b375df9e628abbf58274c..06290141739e5663a8e53f39e5002e0bd66fc948 100644
--- a/include/llvm/MCA/HardwareUnits/RetireControlUnit.h
+++ b/include/llvm/MCA/HardwareUnits/RetireControlUnit.h
@@ -1,9 +1,8 @@
 //===---------------------- RetireControlUnit.h -----------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/include/llvm/MCA/HardwareUnits/Scheduler.h b/include/llvm/MCA/HardwareUnits/Scheduler.h
index 351ea4827df9a112c3cb3e14607c97eda19146e2..f1cfcbe6b2b0570eecec1693e439419b955dadae 100644
--- a/include/llvm/MCA/HardwareUnits/Scheduler.h
+++ b/include/llvm/MCA/HardwareUnits/Scheduler.h
@@ -1,9 +1,8 @@
 //===--------------------- Scheduler.h ------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
@@ -68,22 +67,6 @@ public:
 /// resources. This class is also responsible for tracking the progress of
 /// instructions from the dispatch stage, until the write-back stage.
 ///
-/// An instruction dispatched to the Scheduler is initially placed into either
-/// the 'WaitSet' or the 'ReadySet' depending on the availability of the input
-/// operands.
-///
-/// An instruction is moved from the WaitSet to the ReadySet when register
-/// operands become available, and all memory dependencies are met.
-/// Instructions that are moved from the WaitSet to the ReadySet transition
-/// in state from 'IS_AVAILABLE' to 'IS_READY'.
-///
-/// On every cycle, the Scheduler checks if it can promote instructions from the
-/// WaitSet to the ReadySet.
-///
-/// An Instruction is moved from the ReadySet the `IssuedSet` when it is issued
-/// to a (one or more) pipeline(s). This event also causes an instruction state
-/// transition (i.e. from state IS_READY, to state IS_EXECUTING). An Instruction
-/// leaves the IssuedSet when it reaches the write-back stage.
 class Scheduler : public HardwareUnit {
   LSUnit &LSU;
 
@@ -93,10 +76,48 @@ class Scheduler : public HardwareUnit {
   // Hardware resources that are managed by this scheduler.
   std::unique_ptr<ResourceManager> Resources;
 
+  // Instructions dispatched to the Scheduler are internally classified based on
+  // the instruction stage (see Instruction::InstrStage).
+  //
+  // An Instruction dispatched to the Scheduler is added to the WaitSet if not
+  // all its register operands are available, and at least one latency is unknown.
+  // By construction, the WaitSet only contains instructions that are in the
+  // IS_DISPATCHED stage.
+  //
+  // An Instruction transitions from the WaitSet to the PendingSet if the
+  // instruction is not ready yet, but the latency of every register read is known.
+  // Instructions in the PendingSet are expected to be in the IS_PENDING stage.
+  //
+  // Instructions in the PendingSet are immediately dominated only by
+  // instructions that have already been issued to the underlying pipelines.
+  // In the presence of bottlenecks caused by data dependencies, the PendingSet
+  // can be inspected to identify problematic data dependencies between
+  // instructions.
+  //
+  // An instruction is moved to the ReadySet when all register operands become
+  // available, and all memory dependencies are met.  Instructions that are
+  // moved from the PendingSet to the ReadySet transition in state from
+  // 'IS_PENDING' to 'IS_READY'.
+  //
+  // On every cycle, the Scheduler checks if it can promote instructions from the
+  // PendingSet to the ReadySet.
+  //
+  // An Instruction is moved from the ReadySet to the `IssuedSet` when it starts
+  // exection. This event also causes an instruction state transition (i.e. from
+  // state IS_READY, to state IS_EXECUTING). An Instruction leaves the IssuedSet
+  // only when it reaches the write-back stage.
   std::vector<InstRef> WaitSet;
+  std::vector<InstRef> PendingSet;
   std::vector<InstRef> ReadySet;
   std::vector<InstRef> IssuedSet;
 
+  // A mask of busy resource units. It defaults to the empty set (i.e. a zero
+  // mask), and it is cleared at the beginning of every cycle.
+  // It is updated every time the scheduler fails to issue an instruction from
+  // the ready set due to unavailable pipeline resources.
+  // Each bit of the mask represents an unavailable resource.
+  uint64_t BusyResourceUnits;
+
   /// Verify the given selection strategy and set the Strategy member
   /// accordingly.  If no strategy is provided, the DefaultSchedulerStrategy is
   /// used.
@@ -112,9 +133,14 @@ class Scheduler : public HardwareUnit {
   // vector 'Executed'.
   void updateIssuedSet(SmallVectorImpl<InstRef> &Executed);
 
-  // Try to promote instructions from WaitSet to ReadySet.
+  // Try to promote instructions from the PendingSet to the ReadySet.
   // Add promoted instructions to the 'Ready' vector in input.
-  void promoteToReadySet(SmallVectorImpl<InstRef> &Ready);
+  // Returns true if at least one instruction was promoted.
+  bool promoteToReadySet(SmallVectorImpl<InstRef> &Ready);
+
+  // Try to promote instructions from the WaitSet to the PendingSet.
+  // Returns true if at least one instruction was promoted.
+  bool promoteToPendingSet();
 
 public:
   Scheduler(const MCSchedModel &Model, LSUnit &Lsu)
@@ -127,7 +153,7 @@ public:
 
   Scheduler(std::unique_ptr<ResourceManager> RM, LSUnit &Lsu,
             std::unique_ptr<SchedulerStrategy> SelectStrategy)
-      : LSU(Lsu), Resources(std::move(RM)) {
+      : LSU(Lsu), Resources(std::move(RM)), BusyResourceUnits(0) {
     initializeStrategy(std::move(SelectStrategy));
   }
 
@@ -195,6 +221,16 @@ public:
   /// resources are not available.
   InstRef select();
 
+  /// Returns a mask of busy resources. Each bit of the mask identifies a unique
+  /// processor resource unit. In the absence of bottlenecks caused by resource
+  /// pressure, the mask value returned by this method is always zero.
+  uint64_t getBusyResourceUnits() const { return BusyResourceUnits; }
+  bool arePipelinesFullyUsed() const {
+    return !Resources->getAvailableProcResUnits();
+  }
+  bool isReadySetEmpty() const { return ReadySet.empty(); }
+  bool isWaitSetEmpty() const { return WaitSet.empty(); }
+
 #ifndef NDEBUG
   // Update the ready queues.
   void dump() const;
diff --git a/include/llvm/MCA/InstrBuilder.h b/include/llvm/MCA/InstrBuilder.h
index 5f998db5e4ce0c232e7b110113608cc1125193e1..690016354f7a7a361e68314810b451a8c81fb489 100644
--- a/include/llvm/MCA/InstrBuilder.h
+++ b/include/llvm/MCA/InstrBuilder.h
@@ -1,9 +1,8 @@
 //===--------------------- InstrBuilder.h -----------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/include/llvm/MCA/Instruction.h b/include/llvm/MCA/Instruction.h
index 148651b22dac04657c598516970db241180182df..658b7fe4f891c6e6ad9a291b3daea1a90fa7b0c2 100644
--- a/include/llvm/MCA/Instruction.h
+++ b/include/llvm/MCA/Instruction.h
@@ -1,9 +1,8 @@
 //===--------------------- Instruction.h ------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
@@ -28,6 +27,7 @@
 #include <memory>
 
 namespace llvm {
+
 namespace mca {
 
 constexpr int UNKNOWN_CYCLES = -512;
@@ -137,8 +137,8 @@ class WriteState {
 public:
   WriteState(const WriteDescriptor &Desc, unsigned RegID,
              bool clearsSuperRegs = false, bool writesZero = false)
-      : WD(&Desc), CyclesLeft(UNKNOWN_CYCLES), RegisterID(RegID),
-        PRFID(0), ClearsSuperRegs(clearsSuperRegs), WritesZero(writesZero),
+      : WD(&Desc), CyclesLeft(UNKNOWN_CYCLES), RegisterID(RegID), PRFID(0),
+        ClearsSuperRegs(clearsSuperRegs), WritesZero(writesZero),
         IsEliminated(false), DependentWrite(nullptr), PartialWrite(nullptr),
         DependentWriteCyclesLeft(0) {}
 
@@ -154,7 +154,9 @@ public:
   void addUser(ReadState *Use, int ReadAdvance);
   void addUser(WriteState *Use);
 
-  unsigned getDependentWriteCyclesLeft() const { return DependentWriteCyclesLeft; }
+  unsigned getDependentWriteCyclesLeft() const {
+    return DependentWriteCyclesLeft;
+  }
 
   unsigned getNumUsers() const {
     unsigned NumUsers = Users.size();
@@ -166,6 +168,14 @@ public:
   bool clearsSuperRegisters() const { return ClearsSuperRegs; }
   bool isWriteZero() const { return WritesZero; }
   bool isEliminated() const { return IsEliminated; }
+
+  bool isReady() const {
+    if (getDependentWrite())
+      return false;
+    unsigned CyclesLeft = getDependentWriteCyclesLeft();
+    return !CyclesLeft || CyclesLeft < getLatency();
+  }
+
   bool isExecuted() const {
     return CyclesLeft != UNKNOWN_CYCLES && CyclesLeft <= 0;
   }
@@ -237,6 +247,7 @@ public:
   unsigned getRegisterID() const { return RegisterID; }
   unsigned getRegisterFileID() const { return PRFID; }
 
+  bool isPending() const { return !IndependentFromDef && CyclesLeft > 0; }
   bool isReady() const { return IsReady; }
   bool isImplicitRead() const { return RD->isImplicitRead(); }
 
@@ -327,9 +338,16 @@ struct InstrDesc {
   // A list of buffered resources consumed by this instruction.
   SmallVector<uint64_t, 4> Buffers;
 
+  unsigned UsedProcResUnits;
+  unsigned UsedProcResGroups;
+
   unsigned MaxLatency;
   // Number of MicroOps for this instruction.
   unsigned NumMicroOps;
+  // SchedClassID used to construct this InstrDesc.
+  // This information is currently used by views to do fast queries on the
+  // subtarget when computing the reciprocal throughput.
+  unsigned SchedClassID;
 
   bool MayLoad;
   bool MayStore;
@@ -337,6 +355,10 @@ struct InstrDesc {
   bool BeginGroup;
   bool EndGroup;
 
+  // True if all buffered resources are in-order, and there is at least one
+  // buffer which is a dispatch hazard (BufferSize = 0).
+  bool MustIssueImmediately;
+
   // A zero latency instruction doesn't consume any scheduler resources.
   bool isZeroLatency() const { return !MaxLatency && Resources.empty(); }
 
@@ -399,12 +421,13 @@ public:
 /// that are sent to the various components of the simulated hardware pipeline.
 class Instruction : public InstructionBase {
   enum InstrStage {
-    IS_INVALID,   // Instruction in an invalid state.
-    IS_AVAILABLE, // Instruction dispatched but operands are not ready.
-    IS_READY,     // Instruction dispatched and operands ready.
-    IS_EXECUTING, // Instruction issued.
-    IS_EXECUTED,  // Instruction executed. Values are written back.
-    IS_RETIRED    // Instruction retired.
+    IS_INVALID,    // Instruction in an invalid state.
+    IS_DISPATCHED, // Instruction dispatched but operands are not ready.
+    IS_PENDING,    // Instruction is not ready, but operand latency is known.
+    IS_READY,      // Instruction dispatched and operands ready.
+    IS_EXECUTING,  // Instruction issued.
+    IS_EXECUTED,   // Instruction executed. Values are written back.
+    IS_RETIRED     // Instruction retired.
   };
 
   // The current instruction stage.
@@ -434,15 +457,18 @@ public:
   // all the definitions.
   void execute();
 
-  // Force a transition from the IS_AVAILABLE state to the IS_READY state if
-  // input operands are all ready. State transitions normally occur at the
-  // beginning of a new cycle (see method cycleEvent()). However, the scheduler
-  // may decide to promote instructions from the wait queue to the ready queue
-  // as the result of another issue event.  This method is called every time the
-  // instruction might have changed in state.
+  // Force a transition from the IS_DISPATCHED state to the IS_READY or
+  // IS_PENDING state. State transitions normally occur either at the beginning
+  // of a new cycle (see method cycleEvent()), or as a result of another issue
+  // event. This method is called every time the instruction might have changed
+  // in state. It internally delegates to method updateDispatched() and
+  // updateWaiting().
   void update();
+  bool updateDispatched();
+  bool updatePending();
 
-  bool isDispatched() const { return Stage == IS_AVAILABLE; }
+  bool isDispatched() const { return Stage == IS_DISPATCHED; }
+  bool isPending() const { return Stage == IS_PENDING; }
   bool isReady() const { return Stage == IS_READY; }
   bool isExecuting() const { return Stage == IS_EXECUTING; }
   bool isExecuted() const { return Stage == IS_EXECUTED; }
@@ -454,7 +480,7 @@ public:
                   [](const WriteState &W) { return W.isEliminated(); });
   }
 
-  // Forces a transition from state IS_AVAILABLE to state IS_EXECUTED.
+  // Forces a transition from state IS_DISPATCHED to state IS_EXECUTED.
   void forceExecuted();
 
   void retire() {
@@ -541,4 +567,4 @@ public:
 } // namespace mca
 } // namespace llvm
 
-#endif  // LLVM_MCA_INSTRUCTION_H
+#endif // LLVM_MCA_INSTRUCTION_H
diff --git a/include/llvm/MCA/Pipeline.h b/include/llvm/MCA/Pipeline.h
index acd256060bdd77a6f73c142eb3e38e72595de490..935033f67f8bfebfa914231c79305114c5a45553 100644
--- a/include/llvm/MCA/Pipeline.h
+++ b/include/llvm/MCA/Pipeline.h
@@ -1,9 +1,8 @@
 //===--------------------- Pipeline.h ---------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/include/llvm/MCA/SourceMgr.h b/include/llvm/MCA/SourceMgr.h
index 5e0ca6419f5d4057185d8fb123edd53eed6bdb06..dbe31db1b1ddbd97c41c6178729daa245ba5a359 100644
--- a/include/llvm/MCA/SourceMgr.h
+++ b/include/llvm/MCA/SourceMgr.h
@@ -1,9 +1,8 @@
 //===--------------------- SourceMgr.h --------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/include/llvm/MCA/Stages/DispatchStage.h b/include/llvm/MCA/Stages/DispatchStage.h
index f015cd7522eb54edf0e22e0cf412c4f352861ab2..e39f03e3263b0f322f2c72184065ef7d6b9af88c 100644
--- a/include/llvm/MCA/Stages/DispatchStage.h
+++ b/include/llvm/MCA/Stages/DispatchStage.h
@@ -1,9 +1,8 @@
 //===----------------------- DispatchStage.h --------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
@@ -62,8 +61,6 @@ class DispatchStage final : public Stage {
   bool canDispatch(const InstRef &IR) const;
   Error dispatch(InstRef IR);
 
-  void updateRAWDependencies(ReadState &RS, const MCSubtargetInfo &STI);
-
   void notifyInstructionDispatched(const InstRef &IR,
                                    ArrayRef<unsigned> UsedPhysRegs,
                                    unsigned uOps) const;
diff --git a/include/llvm/MCA/Stages/EntryStage.h b/include/llvm/MCA/Stages/EntryStage.h
index cd9a65b8cc2bcfaa7a0161bee4dc9d71b449f5ed..59a2daff886e4a31342a21ee6e7153480687f804 100644
--- a/include/llvm/MCA/Stages/EntryStage.h
+++ b/include/llvm/MCA/Stages/EntryStage.h
@@ -1,9 +1,8 @@
 //===---------------------- EntryStage.h ------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/include/llvm/MCA/Stages/ExecuteStage.h b/include/llvm/MCA/Stages/ExecuteStage.h
index 4f40c5c65d6bc951acf7b47edebeacb2d301dfa4..ec9eae04a568e9c980e6e45fcf5e2bca0097b22d 100644
--- a/include/llvm/MCA/Stages/ExecuteStage.h
+++ b/include/llvm/MCA/Stages/ExecuteStage.h
@@ -1,9 +1,8 @@
 //===---------------------- ExecuteStage.h ----------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
@@ -65,7 +64,7 @@ public:
 
   void notifyInstructionIssued(
       const InstRef &IR,
-      ArrayRef<std::pair<ResourceRef, ResourceCycles>> Used) const;
+      MutableArrayRef<std::pair<ResourceRef, ResourceCycles>> Used) const;
   void notifyInstructionExecuted(const InstRef &IR) const;
   void notifyInstructionReady(const InstRef &IR) const;
   void notifyResourceAvailable(const ResourceRef &RR) const;
diff --git a/include/llvm/MCA/Stages/InstructionTables.h b/include/llvm/MCA/Stages/InstructionTables.h
index 50f84fc6fff94c9c455e52a26c6dbe7716e3ab70..4b463c9b51c15230f9f4b3193b2ce7c3910ec1c9 100644
--- a/include/llvm/MCA/Stages/InstructionTables.h
+++ b/include/llvm/MCA/Stages/InstructionTables.h
@@ -1,9 +1,8 @@
 //===--------------------- InstructionTables.h ------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
@@ -32,7 +31,8 @@ class InstructionTables final : public Stage {
   SmallVector<uint64_t, 8> Masks;
 
 public:
-  InstructionTables(const MCSchedModel &Model) : Stage(), SM(Model) {
+  InstructionTables(const MCSchedModel &Model)
+      : Stage(), SM(Model), Masks(Model.getNumProcResourceKinds()) {
     computeProcResourceMasks(Model, Masks);
   }
 
diff --git a/include/llvm/MCA/Stages/RetireStage.h b/include/llvm/MCA/Stages/RetireStage.h
index 2051ce5c86adf990ee87f4f3b1acffbf807c9a46..08c216ac7bf43408ded5f66837bbcd2de4a3cda7 100644
--- a/include/llvm/MCA/Stages/RetireStage.h
+++ b/include/llvm/MCA/Stages/RetireStage.h
@@ -1,9 +1,8 @@
 //===---------------------- RetireStage.h -----------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/include/llvm/MCA/Stages/Stage.h b/include/llvm/MCA/Stages/Stage.h
index fc7ab569bb0f73a30566c75309e8263e126df4c8..46b242caa6cff25c88c896991dd54e33e33325b8 100644
--- a/include/llvm/MCA/Stages/Stage.h
+++ b/include/llvm/MCA/Stages/Stage.h
@@ -1,9 +1,8 @@
 //===---------------------- Stage.h -----------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/include/llvm/MCA/Support.h b/include/llvm/MCA/Support.h
index 5c5a35454330b70ee32fbf194d3956f9f127e4a1..fc36ed492d128c9c5c3cfa87addceb605513ba04 100644
--- a/include/llvm/MCA/Support.h
+++ b/include/llvm/MCA/Support.h
@@ -1,9 +1,8 @@
 //===--------------------- Support.h ----------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
@@ -61,24 +60,13 @@ public:
     return (Denominator == 1) ? Numerator : (double)Numerator / Denominator;
   }
 
+  unsigned getNumerator() const { return Numerator; }
+  unsigned getDenominator() const { return Denominator; }
+
   // Add the components of RHS to this instance.  Instead of calculating
   // the final value here, we keep track of the numerator and denominator
   // separately, to reduce floating point error.
-  ResourceCycles &operator+=(const ResourceCycles &RHS) {
-    if (Denominator == RHS.Denominator)
-      Numerator += RHS.Numerator;
-    else {
-      // Create a common denominator for LHS and RHS by calculating the least
-      // common multiple from the GCD.
-      unsigned GCD = GreatestCommonDivisor64(Denominator, RHS.Denominator);
-      unsigned LCM = (Denominator * RHS.Denominator) / GCD;
-      unsigned LHSNumerator = Numerator * (LCM / Denominator);
-      unsigned RHSNumerator = RHS.Numerator * (LCM / RHS.Denominator);
-      Numerator = LHSNumerator + RHSNumerator;
-      Denominator = LCM;
-    }
-    return *this;
-  }
+  ResourceCycles &operator+=(const ResourceCycles &RHS);
 };
 
 /// Populates vector Masks with processor resource masks.
@@ -104,7 +92,7 @@ public:
 /// Resource masks are used by the ResourceManager to solve set membership
 /// problems with simple bit manipulation operations.
 void computeProcResourceMasks(const MCSchedModel &SM,
-                              SmallVectorImpl<uint64_t> &Masks);
+                              MutableArrayRef<uint64_t> Masks);
 
 /// Compute the reciprocal block throughput from a set of processor resource
 /// cycles. The reciprocal block throughput is computed as the MAX between:
diff --git a/include/llvm/Object/Archive.h b/include/llvm/Object/Archive.h
index 9ef1e4875191fc14bf7d73f4d68ce5b12eb5b1cb..c40278a4f923b72b9f46695add95e5268991f587 100644
--- a/include/llvm/Object/Archive.h
+++ b/include/llvm/Object/Archive.h
@@ -1,9 +1,8 @@
 //===- Archive.h - ar archive file format -----------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -16,6 +15,7 @@
 
 #include "llvm/ADT/Optional.h"
 #include "llvm/ADT/StringRef.h"
+#include "llvm/ADT/fallible_iterator.h"
 #include "llvm/ADT/iterator_range.h"
 #include "llvm/Object/Binary.h"
 #include "llvm/Support/Chrono.h"
@@ -143,44 +143,38 @@ public:
     getAsBinary(LLVMContext *Context = nullptr) const;
   };
 
-  class child_iterator {
+  class ChildFallibleIterator {
     Child C;
-    Error *E = nullptr;
 
   public:
-    child_iterator() : C(Child(nullptr, nullptr, nullptr)) {}
-    child_iterator(const Child &C, Error *E) : C(C), E(E) {}
+    ChildFallibleIterator() : C(Child(nullptr, nullptr, nullptr)) {}
+    ChildFallibleIterator(const Child &C) : C(C) {}
 
     const Child *operator->() const { return &C; }
     const Child &operator*() const { return C; }
 
-    bool operator==(const child_iterator &other) const {
+    bool operator==(const ChildFallibleIterator &other) const {
       // Ignore errors here: If an error occurred during increment then getNext
       // will have been set to child_end(), and the following comparison should
       // do the right thing.
       return C == other.C;
     }
 
-    bool operator!=(const child_iterator &other) const {
+    bool operator!=(const ChildFallibleIterator &other) const {
       return !(*this == other);
     }
 
-    // Code in loops with child_iterators must check for errors on each loop
-    // iteration.  And if there is an error break out of the loop.
-    child_iterator &operator++() { // Preincrement
-      assert(E && "Can't increment iterator with no Error attached");
-      ErrorAsOutParameter ErrAsOutParam(E);
-      if (auto ChildOrErr = C.getNext())
-        C = *ChildOrErr;
-      else {
-        C = C.getParent()->child_end().C;
-        *E = ChildOrErr.takeError();
-        E = nullptr;
-      }
-      return *this;
+    Error inc() {
+      auto NextChild = C.getNext();
+      if (!NextChild)
+        return NextChild.takeError();
+      C = std::move(*NextChild);
+      return Error::success();
     }
   };
 
+  using child_iterator = fallible_iterator<ChildFallibleIterator>;
+
   class Symbol {
     const Archive *Parent;
     uint32_t SymbolIndex;
diff --git a/include/llvm/Object/ArchiveWriter.h b/include/llvm/Object/ArchiveWriter.h
index 495b943d04c0058e0050d54ac53fb43ca045e5a5..4e796eb1adfc595ff760eab83d3f1160c2a343ae 100644
--- a/include/llvm/Object/ArchiveWriter.h
+++ b/include/llvm/Object/ArchiveWriter.h
@@ -1,9 +1,8 @@
 //===- ArchiveWriter.h - ar archive file format writer ----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Object/Binary.h b/include/llvm/Object/Binary.h
index 99745e24b8c862ded32c9850ab700878f048d3c6..8ce2d5aa402bcd3f5f1ec52fbc61578154d5d2bc 100644
--- a/include/llvm/Object/Binary.h
+++ b/include/llvm/Object/Binary.h
@@ -1,9 +1,8 @@
 //===- Binary.h - A generic binary file -------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Object/COFF.h b/include/llvm/Object/COFF.h
index 5fe475ee8f46efbd84c65ff527dc3f111bb5793d..c22a482aa15fc5d7cb722c1c7524519b5adba75b 100644
--- a/include/llvm/Object/COFF.h
+++ b/include/llvm/Object/COFF.h
@@ -1,9 +1,8 @@
 //===- COFF.h - COFF object file implementation -----------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -1022,6 +1021,8 @@ public:
 
   ArrayRef<uint8_t> getSymbolAuxData(COFFSymbolRef Symbol) const;
 
+  uint32_t getSymbolIndex(COFFSymbolRef Symbol) const;
+
   size_t getSymbolTableEntrySize() const {
     if (COFFHeader)
       return sizeof(coff_symbol16);
diff --git a/include/llvm/Object/COFFImportFile.h b/include/llvm/Object/COFFImportFile.h
index 0a4556ad88845fb2af86ae7171a03a10aa1c8f94..d051d7f09d0ed3a6d3d2e132deb066d150efe820 100644
--- a/include/llvm/Object/COFFImportFile.h
+++ b/include/llvm/Object/COFFImportFile.h
@@ -1,9 +1,8 @@
 //===- COFFImportFile.h - COFF short import file implementation -*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Object/COFFModuleDefinition.h b/include/llvm/Object/COFFModuleDefinition.h
index be139a2833b0b45c93b55c1a534ba87dd24037f0..ab52259fea1a365c0e7f7803ff878d024f86fa57 100644
--- a/include/llvm/Object/COFFModuleDefinition.h
+++ b/include/llvm/Object/COFFModuleDefinition.h
@@ -1,9 +1,8 @@
 //===--- COFFModuleDefinition.h ---------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Object/CVDebugRecord.h b/include/llvm/Object/CVDebugRecord.h
index faad72c0df29fbd4c0132c85164de7daf0d1c682..d41c7391f70199f1160bf5c6d06e61d4d32146ce 100644
--- a/include/llvm/Object/CVDebugRecord.h
+++ b/include/llvm/Object/CVDebugRecord.h
@@ -1,9 +1,8 @@
 //===- CVDebugRecord.h ------------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/Object/Decompressor.h b/include/llvm/Object/Decompressor.h
index 2a77d2ffbf687f676b0b0b72e53f8325a4657ffe..cc918481b308fe1dc12dc87ba32fdd91876ab03a 100644
--- a/include/llvm/Object/Decompressor.h
+++ b/include/llvm/Object/Decompressor.h
@@ -1,9 +1,8 @@
 //===-- Decompressor.h ------------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===/
 
diff --git a/include/llvm/Object/ELF.h b/include/llvm/Object/ELF.h
index bcdc190cc7dccceab916ec1e5f81745638b92ca3..1ba32b4875c5be6b74e459021d53bcc49f603063 100644
--- a/include/llvm/Object/ELF.h
+++ b/include/llvm/Object/ELF.h
@@ -1,9 +1,8 @@
 //===- ELF.h - ELF object file implementation -------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Object/ELFObjectFile.h b/include/llvm/Object/ELFObjectFile.h
index 0f620681cd99a421d34b6f496aa0fd84d07c7a98..aab7340a9e89c60c9981f493323d992c57886522 100644
--- a/include/llvm/Object/ELFObjectFile.h
+++ b/include/llvm/Object/ELFObjectFile.h
@@ -1,9 +1,8 @@
 //===- ELFObjectFile.h - ELF object file implementation ---------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -441,7 +440,16 @@ Expected<StringRef> ELFObjectFile<ELFT>::getSymbolName(DataRefImpl Sym) const {
   auto SymStrTabOrErr = EF.getStringTable(StringTableSec);
   if (!SymStrTabOrErr)
     return SymStrTabOrErr.takeError();
-  return ESym->getName(*SymStrTabOrErr);
+  Expected<StringRef> Name = ESym->getName(*SymStrTabOrErr);
+
+  // If the symbol name is empty use the section name.
+  if ((!Name || Name->empty()) && ESym->getType() == ELF::STT_SECTION) {
+    StringRef SecName;
+    Expected<section_iterator> Sec = getSymbolSection(Sym);
+    if (Sec && !(*Sec)->getName(SecName))
+      return SecName;
+  }
+  return Name;
 }
 
 template <class ELFT>
diff --git a/include/llvm/Object/ELFTypes.h b/include/llvm/Object/ELFTypes.h
index ec3c8e7bae465ba4b788ad2b350ac60679ca5aa1..45bbd6c4dea531c80d333668a54c5b23362557dc 100644
--- a/include/llvm/Object/ELFTypes.h
+++ b/include/llvm/Object/ELFTypes.h
@@ -1,9 +1,8 @@
 //===- ELFTypes.h - Endian specific types for ELF ---------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/Object/Error.h b/include/llvm/Object/Error.h
index eb938338715d64793dd3eed8e725975c2097869c..b7bbf06fc86de316256b8085055f2bda5c71fa4c 100644
--- a/include/llvm/Object/Error.h
+++ b/include/llvm/Object/Error.h
@@ -1,9 +1,8 @@
 //===- Error.h - system_error extensions for Object -------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -50,6 +49,7 @@ inline std::error_code make_error_code(object_error e) {
 /// Currently inherits from ECError for easy interoperability with
 /// std::error_code, but this will be removed in the future.
 class BinaryError : public ErrorInfo<BinaryError, ECError> {
+  virtual void anchor();
 public:
   static char ID;
   BinaryError() {
diff --git a/include/llvm/Object/IRObjectFile.h b/include/llvm/Object/IRObjectFile.h
index 993359b766a173a5981c12c9354bba493f822e08..39454b05f0d49ebe96a12af1bdd92ea6ba5a0f17 100644
--- a/include/llvm/Object/IRObjectFile.h
+++ b/include/llvm/Object/IRObjectFile.h
@@ -1,9 +1,8 @@
 //===- IRObjectFile.h - LLVM IR object file implementation ------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Object/IRSymtab.h b/include/llvm/Object/IRSymtab.h
index 5f6a024cd13298bd9750c9475a4fcc64aa40add7..0601d995fd1832decdf79b9e0d2fe457c0e03898 100644
--- a/include/llvm/Object/IRSymtab.h
+++ b/include/llvm/Object/IRSymtab.h
@@ -1,9 +1,8 @@
 //===- IRSymtab.h - data definitions for IR symbol tables -------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Object/MachO.h b/include/llvm/Object/MachO.h
index 159c1765ab861f514ba3122cb67ef83c3390f12a..79ec8bb0bcd76bdf6ebc979c9a410c30214689ee 100644
--- a/include/llvm/Object/MachO.h
+++ b/include/llvm/Object/MachO.h
@@ -1,9 +1,8 @@
 //===- MachO.h - MachO object file implementation ---------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -356,7 +355,7 @@ public:
   basic_symbol_iterator symbol_end() const override;
 
   // MachO specific.
-  basic_symbol_iterator getSymbolByIndex(unsigned Index) const;
+  symbol_iterator getSymbolByIndex(unsigned Index) const;
   uint64_t getSymbolIndex(DataRefImpl Symb) const;
 
   section_iterator section_begin() const override;
@@ -616,6 +615,9 @@ public:
     case MachO::PLATFORM_TVOS: return "tvos";
     case MachO::PLATFORM_WATCHOS: return "watchos";
     case MachO::PLATFORM_BRIDGEOS: return "bridgeos";
+    case MachO::PLATFORM_IOSSIMULATOR: return "iossimulator";
+    case MachO::PLATFORM_TVOSSIMULATOR: return "tvossimulator";
+    case MachO::PLATFORM_WATCHOSSIMULATOR: return "watchossimulator";
     default:
       std::string ret;
       raw_string_ostream ss(ret);
diff --git a/include/llvm/Object/MachOUniversal.h b/include/llvm/Object/MachOUniversal.h
index 9e70b0bc30c040d7dbe4786a19c86b27f5ed2b11..5bf724f2c8b2ef1bd2b00155c3663d52e7744d18 100644
--- a/include/llvm/Object/MachOUniversal.h
+++ b/include/llvm/Object/MachOUniversal.h
@@ -1,9 +1,8 @@
 //===- MachOUniversal.h - Mach-O universal binaries -------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Object/ModuleSymbolTable.h b/include/llvm/Object/ModuleSymbolTable.h
index c3cbc27998e5ac2b057ad23fdc7927fd96c66382..4c582fbcda81107a97c1cec3b5555cb4156263bc 100644
--- a/include/llvm/Object/ModuleSymbolTable.h
+++ b/include/llvm/Object/ModuleSymbolTable.h
@@ -1,9 +1,8 @@
 //===- ModuleSymbolTable.h - symbol table for in-memory IR ------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Object/ObjectFile.h b/include/llvm/Object/ObjectFile.h
index 036c99cb6baff0d5f967302b40f965dff391c38c..f13775c914df808ecdfd4454158e1f3011f8bba4 100644
--- a/include/llvm/Object/ObjectFile.h
+++ b/include/llvm/Object/ObjectFile.h
@@ -1,9 +1,8 @@
 //===- ObjectFile.h - File format independent object file -------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Object/RelocVisitor.h b/include/llvm/Object/RelocVisitor.h
index 9a978de2e59968e354ce01e3b8f6828beb0afcc3..76f3fab2febb8471d586c32337ef9f1498867685 100644
--- a/include/llvm/Object/RelocVisitor.h
+++ b/include/llvm/Object/RelocVisitor.h
@@ -1,9 +1,8 @@
 //===- RelocVisitor.h - Visitor for object file relocations -----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -101,6 +100,8 @@ private:
     case Triple::arm:
     case Triple::armeb:
       return visitARM(Rel, R, Value);
+    case Triple::avr:
+      return visitAVR(Rel, R, Value);
     case Triple::lanai:
       return visitLanai(Rel, R, Value);
     case Triple::mipsel:
@@ -259,6 +260,16 @@ private:
     return 0;
   }
 
+  uint64_t visitAVR(uint32_t Rel, RelocationRef R, uint64_t Value) {
+    if (Rel == ELF::R_AVR_16) {
+      return (Value + getELFAddend(R)) & 0xFFFF;
+    } else if (Rel == ELF::R_AVR_32) {
+      return (Value + getELFAddend(R)) & 0xFFFFFFFF;
+    }
+    HasError = true;
+    return 0;
+  }
+
   uint64_t visitLanai(uint32_t Rel, RelocationRef R, uint64_t Value) {
     if (Rel == ELF::R_LANAI_32)
       return (Value + getELFAddend(R)) & 0xFFFFFFFF;
@@ -325,17 +336,17 @@ private:
   uint64_t visitWasm(uint32_t Rel, RelocationRef R, uint64_t Value) {
     if (ObjToVisit.getArch() == Triple::wasm32) {
       switch (Rel) {
-      case wasm::R_WEBASSEMBLY_FUNCTION_INDEX_LEB:
-      case wasm::R_WEBASSEMBLY_TABLE_INDEX_SLEB:
-      case wasm::R_WEBASSEMBLY_TABLE_INDEX_I32:
-      case wasm::R_WEBASSEMBLY_MEMORY_ADDR_LEB:
-      case wasm::R_WEBASSEMBLY_MEMORY_ADDR_SLEB:
-      case wasm::R_WEBASSEMBLY_MEMORY_ADDR_I32:
-      case wasm::R_WEBASSEMBLY_TYPE_INDEX_LEB:
-      case wasm::R_WEBASSEMBLY_GLOBAL_INDEX_LEB:
-      case wasm::R_WEBASSEMBLY_FUNCTION_OFFSET_I32:
-      case wasm::R_WEBASSEMBLY_SECTION_OFFSET_I32:
-      case wasm::R_WEBASSEMBLY_EVENT_INDEX_LEB:
+      case wasm::R_WASM_FUNCTION_INDEX_LEB:
+      case wasm::R_WASM_TABLE_INDEX_SLEB:
+      case wasm::R_WASM_TABLE_INDEX_I32:
+      case wasm::R_WASM_MEMORY_ADDR_LEB:
+      case wasm::R_WASM_MEMORY_ADDR_SLEB:
+      case wasm::R_WASM_MEMORY_ADDR_I32:
+      case wasm::R_WASM_TYPE_INDEX_LEB:
+      case wasm::R_WASM_GLOBAL_INDEX_LEB:
+      case wasm::R_WASM_FUNCTION_OFFSET_I32:
+      case wasm::R_WASM_SECTION_OFFSET_I32:
+      case wasm::R_WASM_EVENT_INDEX_LEB:
         // For wasm section, its offset at 0 -- ignoring Value
         return 0;
       }
diff --git a/include/llvm/Object/StackMapParser.h b/include/llvm/Object/StackMapParser.h
index 557db5afa8256473cff43beee0bbcb35f380e4f4..8817516eb2128f1c14ee9ec21270548db60712a3 100644
--- a/include/llvm/Object/StackMapParser.h
+++ b/include/llvm/Object/StackMapParser.h
@@ -1,9 +1,8 @@
 //===- StackMapParser.h - StackMap Parsing Support --------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/Object/SymbolSize.h b/include/llvm/Object/SymbolSize.h
index 1a1dc8752943077bb437d6f88ea490190881b343..085623e35907d5f6e3b9de8ec3c0ac1e4b1b0b41 100644
--- a/include/llvm/Object/SymbolSize.h
+++ b/include/llvm/Object/SymbolSize.h
@@ -1,9 +1,8 @@
 //===- SymbolSize.h ---------------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //
 //===----------------------------------------------------------------------===//
diff --git a/include/llvm/Object/SymbolicFile.h b/include/llvm/Object/SymbolicFile.h
index 5b9549bc34492fdf852686899c36940d116b9096..6d95d7a895f290eda8ab357c394e94e94c1d8ecd 100644
--- a/include/llvm/Object/SymbolicFile.h
+++ b/include/llvm/Object/SymbolicFile.h
@@ -1,9 +1,8 @@
 //===- SymbolicFile.h - Interface that only provides symbols ----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Object/Wasm.h b/include/llvm/Object/Wasm.h
index ed857652a0486149658e157987586f7693e4e2b0..a0a2c4bf6696271940305ff98c2fa3f2cedf7696 100644
--- a/include/llvm/Object/Wasm.h
+++ b/include/llvm/Object/Wasm.h
@@ -1,9 +1,8 @@
-//===- WasmObjectFile.h - Wasm object file implementation -------*- C++ -*-===//
+//===- Wasm.h - Wasm object file implementation -----------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -130,6 +129,7 @@ public:
   static bool classof(const Binary *v) { return v->isWasm(); }
 
   const wasm::WasmDylinkInfo &dylinkInfo() const { return DylinkInfo; }
+  const wasm::WasmProducerInfo &getProducerInfo() const { return ProducerInfo; }
   ArrayRef<wasm::WasmSignature> types() const { return Signatures; }
   ArrayRef<uint32_t> functionTypes() const { return FunctionTypes; }
   ArrayRef<wasm::WasmImport> imports() const { return Imports; }
@@ -149,7 +149,6 @@ public:
   uint32_t getNumImportedGlobals() const { return NumImportedGlobals; }
   uint32_t getNumImportedFunctions() const { return NumImportedFunctions; }
   uint32_t getNumImportedEvents() const { return NumImportedEvents; }
-
   void moveSymbolNext(DataRefImpl &Symb) const override;
 
   uint32_t getSymbolFlags(DataRefImpl Symb) const override;
@@ -222,13 +221,13 @@ private:
   bool isValidDataSymbol(uint32_t Index) const;
   bool isValidSectionSymbol(uint32_t Index) const;
   wasm::WasmFunction &getDefinedFunction(uint32_t Index);
+  const wasm::WasmFunction &getDefinedFunction(uint32_t Index) const;
   wasm::WasmGlobal &getDefinedGlobal(uint32_t Index);
   wasm::WasmEvent &getDefinedEvent(uint32_t Index);
 
   const WasmSection &getWasmSection(DataRefImpl Ref) const;
   const wasm::WasmRelocation &getWasmRelocation(DataRefImpl Ref) const;
 
-  const uint8_t *getPtr(size_t Offset) const;
   Error parseSection(WasmSection &Sec);
   Error parseCustomSection(WasmSection &Sec, ReadContext &Ctx);
 
@@ -252,11 +251,13 @@ private:
   Error parseLinkingSection(ReadContext &Ctx);
   Error parseLinkingSectionSymtab(ReadContext &Ctx);
   Error parseLinkingSectionComdat(ReadContext &Ctx);
+  Error parseProducersSection(ReadContext &Ctx);
   Error parseRelocSection(StringRef Name, ReadContext &Ctx);
 
   wasm::WasmObjectHeader Header;
   std::vector<WasmSection> Sections;
   wasm::WasmDylinkInfo DylinkInfo;
+  wasm::WasmProducerInfo ProducerInfo;
   std::vector<wasm::WasmSignature> Signatures;
   std::vector<uint32_t> FunctionTypes;
   std::vector<wasm::WasmTable> Tables;
diff --git a/include/llvm/Object/WasmTraits.h b/include/llvm/Object/WasmTraits.h
index 049d72f79e4173d55aea93f7a729bd5cebe2a801..3eee8e71b1876ad9a0aab7b96363d1e8e52eda8e 100644
--- a/include/llvm/Object/WasmTraits.h
+++ b/include/llvm/Object/WasmTraits.h
@@ -1,9 +1,8 @@
 //===- WasmTraits.h - DenseMap traits for the Wasm structures ---*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Object/WindowsResource.h b/include/llvm/Object/WindowsResource.h
index a077c82871bfc6a0183b1c2a8e8140ca9e646d4d..fd04e608cf5fdcc7d902ca573c0d68db7932a6c5 100644
--- a/include/llvm/Object/WindowsResource.h
+++ b/include/llvm/Object/WindowsResource.h
@@ -1,9 +1,8 @@
 //===-- WindowsResource.h ---------------------------------------*- C++-*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===---------------------------------------------------------------------===//
 //
diff --git a/include/llvm/ObjectYAML/COFFYAML.h b/include/llvm/ObjectYAML/COFFYAML.h
index 78f021fc0386987e15140ff241bf8b3ad7992e04..eec5af928f6d1753bcf33a1eb87fa43facfd3a72 100644
--- a/include/llvm/ObjectYAML/COFFYAML.h
+++ b/include/llvm/ObjectYAML/COFFYAML.h
@@ -1,9 +1,8 @@
 //===- COFFYAML.h - COFF YAMLIO implementation ------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -58,7 +57,13 @@ LLVM_YAML_STRONG_TYPEDEF(uint8_t, AuxSymbolType)
 struct Relocation {
   uint32_t VirtualAddress;
   uint16_t Type;
+
+  // Normally a Relocation can refer to the symbol via its name.
+  // It can also use a direct symbol table index instead (with no name
+  // specified), allowing disambiguating between multiple symbols with the
+  // same name or crafting intentionally broken files for testing.
   StringRef SymbolName;
+  Optional<uint32_t> SymbolTableIndex;
 };
 
 struct Section {
diff --git a/include/llvm/ObjectYAML/CodeViewYAMLDebugSections.h b/include/llvm/ObjectYAML/CodeViewYAMLDebugSections.h
index d620008e22d218885d1e3cbb4a5c0ed4570a4df8..9cbacb88b51826b2596a00f31dc97583c6f297c3 100644
--- a/include/llvm/ObjectYAML/CodeViewYAMLDebugSections.h
+++ b/include/llvm/ObjectYAML/CodeViewYAMLDebugSections.h
@@ -1,9 +1,8 @@
 //=- CodeViewYAMLDebugSections.h - CodeView YAMLIO debug sections -*- C++ -*-=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/ObjectYAML/CodeViewYAMLSymbols.h b/include/llvm/ObjectYAML/CodeViewYAMLSymbols.h
index 791193c78f193be53a080f5f887778fcc178c573..7c05c9eea05edfdeed1b2f43537b934067274510 100644
--- a/include/llvm/ObjectYAML/CodeViewYAMLSymbols.h
+++ b/include/llvm/ObjectYAML/CodeViewYAMLSymbols.h
@@ -1,9 +1,8 @@
 //===- CodeViewYAMLSymbols.h - CodeView YAMLIO Symbol implementation ------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/ObjectYAML/CodeViewYAMLTypeHashing.h b/include/llvm/ObjectYAML/CodeViewYAMLTypeHashing.h
index 344966fe68919e166364ca6dfc1005f6bb29e5ca..d6cec8d310eb65ab4478560d03acb271f8e1b557 100644
--- a/include/llvm/ObjectYAML/CodeViewYAMLTypeHashing.h
+++ b/include/llvm/ObjectYAML/CodeViewYAMLTypeHashing.h
@@ -1,9 +1,8 @@
 //==- CodeViewYAMLTypeHashing.h - CodeView YAMLIO Type hashing ----*- C++-*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/ObjectYAML/CodeViewYAMLTypes.h b/include/llvm/ObjectYAML/CodeViewYAMLTypes.h
index 1b1306df4f53cea47dc6dcbb90b30c6ae75ad879..04b5e0ba3aa1a0992f3b2bbb7b6d3a30d4642f2d 100644
--- a/include/llvm/ObjectYAML/CodeViewYAMLTypes.h
+++ b/include/llvm/ObjectYAML/CodeViewYAMLTypes.h
@@ -1,9 +1,8 @@
 //==- CodeViewYAMLTypes.h - CodeView YAMLIO Type implementation --*- C++ -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/ObjectYAML/DWARFEmitter.h b/include/llvm/ObjectYAML/DWARFEmitter.h
index ce32274219303e25891c0d235b3c154cdc4b6d40..2ccc876d502383a463dd5c337d16d8860ce22ef6 100644
--- a/include/llvm/ObjectYAML/DWARFEmitter.h
+++ b/include/llvm/ObjectYAML/DWARFEmitter.h
@@ -1,9 +1,8 @@
 //===--- DWARFEmitter.h - ---------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/include/llvm/ObjectYAML/DWARFYAML.h b/include/llvm/ObjectYAML/DWARFYAML.h
index 705c887789455e7d29166a573d6ccb9edb68ff21..78d736c3ef05c1e6dfb86eb8135080a16d1c4e73 100644
--- a/include/llvm/ObjectYAML/DWARFYAML.h
+++ b/include/llvm/ObjectYAML/DWARFYAML.h
@@ -1,9 +1,8 @@
 //===- DWARFYAML.h - DWARF YAMLIO implementation ----------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
diff --git a/include/llvm/ObjectYAML/ELFYAML.h b/include/llvm/ObjectYAML/ELFYAML.h
index 92081f065dce4513b571190a896f5464ff338366..e194e59a4851d724369d192e7e4e2762dec80780 100644
--- a/include/llvm/ObjectYAML/ELFYAML.h
+++ b/include/llvm/ObjectYAML/ELFYAML.h
@@ -1,9 +1,8 @@
 //===- ELFYAML.h - ELF YAMLIO implementation --------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
@@ -44,6 +43,8 @@ LLVM_YAML_STRONG_TYPEDEF(uint8_t, ELF_ELFDATA)
 LLVM_YAML_STRONG_TYPEDEF(uint8_t, ELF_ELFOSABI)
 // Just use 64, since it can hold 32-bit values too.
 LLVM_YAML_STRONG_TYPEDEF(uint64_t, ELF_EF)
+// Just use 64, since it can hold 32-bit values too.
+LLVM_YAML_STRONG_TYPEDEF(uint64_t, ELF_DYNTAG)
 LLVM_YAML_STRONG_TYPEDEF(uint32_t, ELF_PF)
 LLVM_YAML_STRONG_TYPEDEF(uint32_t, ELF_SHT)
 LLVM_YAML_STRONG_TYPEDEF(uint32_t, ELF_REL)
@@ -68,6 +69,7 @@ struct FileHeader {
   ELF_ELFCLASS Class;
   ELF_ELFDATA Data;
   ELF_ELFOSABI OSABI;
+  llvm::yaml::Hex8 ABIVersion;
   ELF_ET Type;
   ELF_EM Machine;
   ELF_EF Flags;
@@ -107,8 +109,14 @@ struct SectionOrType {
   StringRef sectionNameOrType;
 };
 
+struct DynamicEntry {
+  ELF_DYNTAG Tag;
+  llvm::yaml::Hex64 Val;
+};
+
 struct Section {
   enum class SectionKind {
+    Dynamic,
     Group,
     RawContent,
     Relocation,
@@ -121,13 +129,23 @@ struct Section {
   ELF_SHF Flags;
   llvm::yaml::Hex64 Address;
   StringRef Link;
-  StringRef Info;
   llvm::yaml::Hex64 AddressAlign;
   Optional<llvm::yaml::Hex64> EntSize;
 
   Section(SectionKind Kind) : Kind(Kind) {}
   virtual ~Section();
 };
+
+struct DynamicSection : Section {
+  std::vector<DynamicEntry> Entries;
+
+  DynamicSection() : Section(SectionKind::Dynamic) {}
+
+  static bool classof(const Section *S) {
+    return S->Kind == SectionKind::Dynamic;
+  }
+};
+
 struct RawContentSection : Section {
   yaml::BinaryRef Content;
   llvm::yaml::Hex64 Size;
@@ -153,6 +171,7 @@ struct Group : Section {
   // Members of a group contain a flag and a list of section indices
   // that are part of the group.
   std::vector<SectionOrType> Members;
+  StringRef Signature; /* Info */
 
   Group() : Section(SectionKind::Group) {}
 
@@ -170,6 +189,7 @@ struct Relocation {
 
 struct RelocationSection : Section {
   std::vector<Relocation> Relocations;
+  StringRef RelocatableSec; /* Info */
 
   RelocationSection() : Section(SectionKind::Relocation) {}
 
@@ -214,6 +234,7 @@ struct Object {
 } // end namespace ELFYAML
 } // end namespace llvm
 
+LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::ELFYAML::DynamicEntry)
 LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::ELFYAML::ProgramHeader)
 LLVM_YAML_IS_SEQUENCE_VECTOR(std::unique_ptr<llvm::ELFYAML::Section>)
 LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::ELFYAML::Symbol)
@@ -296,6 +317,11 @@ struct ScalarEnumerationTraits<ELFYAML::ELF_REL> {
   static void enumeration(IO &IO, ELFYAML::ELF_REL &Value);
 };
 
+template <>
+struct ScalarEnumerationTraits<ELFYAML::ELF_DYNTAG> {
+  static void enumeration(IO &IO, ELFYAML::ELF_DYNTAG &Value);
+};
+
 template <>
 struct ScalarEnumerationTraits<ELFYAML::ELF_RSS> {
   static void enumeration(IO &IO, ELFYAML::ELF_RSS &Value);
@@ -351,6 +377,10 @@ struct MappingTraits<ELFYAML::LocalGlobalWeakSymbols> {
   static void mapping(IO &IO, ELFYAML::LocalGlobalWeakSymbols &Symbols);
 };
 
+template <> struct MappingTraits<ELFYAML::DynamicEntry> {
+  static void mapping(IO &IO, ELFYAML::DynamicEntry &Rel);
+};
+
 template <> struct MappingTraits<ELFYAML::Relocation> {
   static void mapping(IO &IO, ELFYAML::Relocation &Rel);
 };
diff --git a/include/llvm/ObjectYAML/MachOYAML.h b/include/llvm/ObjectYAML/MachOYAML.h
index cec4f86185f08bcdddeb49a5b3aba9b7484a4523..d7e1c033f43b8d81a1e918593602ffde079e959e 100644
--- a/include/llvm/ObjectYAML/MachOYAML.h
+++ b/include/llvm/ObjectYAML/MachOYAML.h
@@ -1,9 +1,8 @@
 //===- MachOYAML.h - Mach-O YAMLIO implementation ---------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
diff --git a/include/llvm/ObjectYAML/ObjectYAML.h b/include/llvm/ObjectYAML/ObjectYAML.h
index 00ce86430fca42dfc19bfc8ccc56b03dd5872532..56f5f3947840cf95a0aeb2ca6e94dfe8df16b905 100644
--- a/include/llvm/ObjectYAML/ObjectYAML.h
+++ b/include/llvm/ObjectYAML/ObjectYAML.h
@@ -1,9 +1,8 @@
 //===- ObjectYAML.h ---------------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/ObjectYAML/WasmYAML.h b/include/llvm/ObjectYAML/WasmYAML.h
index 406dd7cb515f745fff27a26ef9d16d777c55ec86..f5260bbb1ed4d75f3e2b648f6af84e3d47ce33c5 100644
--- a/include/llvm/ObjectYAML/WasmYAML.h
+++ b/include/llvm/ObjectYAML/WasmYAML.h
@@ -1,9 +1,8 @@
 //===- WasmYAML.h - Wasm YAMLIO implementation ------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
@@ -123,6 +122,11 @@ struct NameEntry {
   StringRef Name;
 };
 
+struct ProducerEntry {
+  std::string Name;
+  std::string Version;
+};
+
 struct SegmentInfo {
   uint32_t Index;
   StringRef Name;
@@ -224,6 +228,19 @@ struct LinkingSection : CustomSection {
   std::vector<Comdat> Comdats;
 };
 
+struct ProducersSection : CustomSection {
+  ProducersSection() : CustomSection("producers") {}
+
+  static bool classof(const Section *S) {
+    auto C = dyn_cast<CustomSection>(S);
+    return C && C->Name == "producers";
+  }
+
+  std::vector<ProducerEntry> Languages;
+  std::vector<ProducerEntry> Tools;
+  std::vector<ProducerEntry> SDKs;
+};
+
 struct TypeSection : Section {
   TypeSection() : Section(wasm::WASM_SEC_TYPE) {}
 
@@ -366,6 +383,7 @@ LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::WasmYAML::Function)
 LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::WasmYAML::LocalDecl)
 LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::WasmYAML::Relocation)
 LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::WasmYAML::NameEntry)
+LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::WasmYAML::ProducerEntry)
 LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::WasmYAML::SegmentInfo)
 LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::WasmYAML::SymbolInfo)
 LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::WasmYAML::InitFunction)
@@ -444,6 +462,10 @@ template <> struct MappingTraits<WasmYAML::NameEntry> {
   static void mapping(IO &IO, WasmYAML::NameEntry &NameEntry);
 };
 
+template <> struct MappingTraits<WasmYAML::ProducerEntry> {
+  static void mapping(IO &IO, WasmYAML::ProducerEntry &ProducerEntry);
+};
+
 template <> struct MappingTraits<WasmYAML::SegmentInfo> {
   static void mapping(IO &IO, WasmYAML::SegmentInfo &SegmentInfo);
 };
diff --git a/include/llvm/ObjectYAML/YAML.h b/include/llvm/ObjectYAML/YAML.h
index 163cd8dfcf08c5c28cfc5f448b972a0c5c4bccc4..a14bfbc5f38a36a6000ea6f44d71b6b186b5b029 100644
--- a/include/llvm/ObjectYAML/YAML.h
+++ b/include/llvm/ObjectYAML/YAML.h
@@ -1,9 +1,8 @@
 //===- YAML.h ---------------------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/Option/Arg.h b/include/llvm/Option/Arg.h
index d0086bb6d611832effa65ce675f80a24cb04aa80..d3623a0c1e4b4c0e51a0b023e0da02225436bf19 100644
--- a/include/llvm/Option/Arg.h
+++ b/include/llvm/Option/Arg.h
@@ -1,9 +1,8 @@
 //===- Arg.h - Parsed Argument Classes --------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
diff --git a/include/llvm/Option/ArgList.h b/include/llvm/Option/ArgList.h
index 687c8cbb02f9e66146e7e46f74c7fea2decd1f71..a0cd2b8a400c50451e4ecfc66c66681a2b62112a 100644
--- a/include/llvm/Option/ArgList.h
+++ b/include/llvm/Option/ArgList.h
@@ -1,9 +1,8 @@
 //===- ArgList.h - Argument List Management ---------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/Option/OptParser.td b/include/llvm/Option/OptParser.td
index 9c373741770b0268d5e3a15e9c6ac4a656e8cf51..a68f17a8b10b307cfe33209c15e1fbe02127f873 100644
--- a/include/llvm/Option/OptParser.td
+++ b/include/llvm/Option/OptParser.td
@@ -1,9 +1,8 @@
 //===--- OptParser.td - Common Option Parsing Interfaces ------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Option/OptSpecifier.h b/include/llvm/Option/OptSpecifier.h
index 84c3cf8ad534d202eb9c64fbe45f335194c8dd34..7a5fcfb18b388625dbbb98ea303edcf1e1a56a1c 100644
--- a/include/llvm/Option/OptSpecifier.h
+++ b/include/llvm/Option/OptSpecifier.h
@@ -1,9 +1,8 @@
 //===- OptSpecifier.h - Option Specifiers -----------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/Option/OptTable.h b/include/llvm/Option/OptTable.h
index fdb05d8a15afb33c7d5d12cc4f046a3f69155924..5db30436069dbe7c5f3f494f948c34194dce8f4f 100644
--- a/include/llvm/Option/OptTable.h
+++ b/include/llvm/Option/OptTable.h
@@ -1,9 +1,8 @@
 //===- OptTable.h - Option Table --------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/Option/Option.h b/include/llvm/Option/Option.h
index b09f6043b7a942ddf683808d3ec3adac17e6c99e..c11487e3f9d7c63dfb34b5f0e4769ea0c88a045f 100644
--- a/include/llvm/Option/Option.h
+++ b/include/llvm/Option/Option.h
@@ -1,9 +1,8 @@
 //===- Option.h - Abstract Driver Options -----------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/Pass.h b/include/llvm/Pass.h
index 5935a0853d32f30f755fc1687ed1a941a41e199d..329f7eaba73dddd30ca707d99d1e89a75a481ed3 100644
--- a/include/llvm/Pass.h
+++ b/include/llvm/Pass.h
@@ -1,9 +1,8 @@
 //===- llvm/Pass.h - Base class for Passes ----------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/PassAnalysisSupport.h b/include/llvm/PassAnalysisSupport.h
index a075eb5574720498054f95347089355ebe1915b1..1228534deb95dc4e49b3b8eb33ca348ad6437188 100644
--- a/include/llvm/PassAnalysisSupport.h
+++ b/include/llvm/PassAnalysisSupport.h
@@ -1,9 +1,8 @@
 //===- llvm/PassAnalysisSupport.h - Analysis Pass Support code --*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/PassInfo.h b/include/llvm/PassInfo.h
index 2f1ab4d43377ef3b46c2b2bee9219834eb0c4c51..686fc044ebcb33abc846834bac957c858a524ed0 100644
--- a/include/llvm/PassInfo.h
+++ b/include/llvm/PassInfo.h
@@ -1,9 +1,8 @@
 //===- llvm/PassInfo.h - Pass Info class ------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/PassRegistry.h b/include/llvm/PassRegistry.h
index 57462138c5aeb86bca7f9abff04e608e31c8ae83..b9a015430c102af03a6773413266d46b1aa15523 100644
--- a/include/llvm/PassRegistry.h
+++ b/include/llvm/PassRegistry.h
@@ -1,9 +1,8 @@
 //===- llvm/PassRegistry.h - Pass Information Registry ----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/PassSupport.h b/include/llvm/PassSupport.h
index 1bf23dcba50b9bb3d1f915ce38667926ad369d80..ab90217ce4a8aeb41fe4834edb1138f2e605d6a6 100644
--- a/include/llvm/PassSupport.h
+++ b/include/llvm/PassSupport.h
@@ -1,9 +1,8 @@
 //===- llvm/PassSupport.h - Pass Support code -------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Passes/PassBuilder.h b/include/llvm/Passes/PassBuilder.h
index fa59345a02cf72126f7251f17baa51ddf0834389..9c915c61b96400e727d9177939f09cf5384e326a 100644
--- a/include/llvm/Passes/PassBuilder.h
+++ b/include/llvm/Passes/PassBuilder.h
@@ -1,9 +1,8 @@
 //===- Parsing, selection, and construction of pass pipelines --*- C++ -*--===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
@@ -56,7 +55,7 @@ struct PGOOptions {
 
 /// This class provides access to building LLVM's passes.
 ///
-/// It's members provide the baseline state available to passes during their
+/// Its members provide the baseline state available to passes during their
 /// construction. The \c PassRegistry.def file specifies how to construct all
 /// of the built-in passes, and those may reference these members during
 /// construction.
diff --git a/include/llvm/Passes/PassPlugin.h b/include/llvm/Passes/PassPlugin.h
index af8f11a7a352e8dacadc1aeb62d96c76dbd9a30b..013b7a827c47ddb51bb39383c3f5f139b213ae78 100644
--- a/include/llvm/Passes/PassPlugin.h
+++ b/include/llvm/Passes/PassPlugin.h
@@ -1,9 +1,8 @@
 //===- llvm/Passes/PassPlugin.h - Public Plugin API -----------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Passes/StandardInstrumentations.h b/include/llvm/Passes/StandardInstrumentations.h
index b6a73b0f4874e4ec25e25d1233d55fa02671dcc5..56725436a7742041db0a136e89334fa0929a1a27 100644
--- a/include/llvm/Passes/StandardInstrumentations.h
+++ b/include/llvm/Passes/StandardInstrumentations.h
@@ -1,9 +1,8 @@
 //===- StandardInstrumentations.h ------------------------------*- C++ -*--===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
@@ -16,14 +15,48 @@
 #ifndef LLVM_PASSES_STANDARDINSTRUMENTATIONS_H
 #define LLVM_PASSES_STANDARDINSTRUMENTATIONS_H
 
+#include "llvm/ADT/SmallVector.h"
 #include "llvm/IR/PassInstrumentation.h"
 #include "llvm/IR/PassTimingInfo.h"
 
+#include <string>
+#include <utility>
+
 namespace llvm {
 
+class Module;
+
+/// Instrumentation to print IR before/after passes.
+///
+/// Needs state to be able to print module after pass that invalidates IR unit
+/// (typically Loop or SCC).
+class PrintIRInstrumentation {
+public:
+  PrintIRInstrumentation() = default;
+  ~PrintIRInstrumentation();
+
+  void registerCallbacks(PassInstrumentationCallbacks &PIC);
+
+private:
+  bool printBeforePass(StringRef PassID, Any IR);
+  void printAfterPass(StringRef PassID, Any IR);
+  void printAfterPassInvalidated(StringRef PassID);
+
+  using PrintModuleDesc = std::tuple<const Module *, std::string, StringRef>;
+
+  void pushModuleDesc(StringRef PassID, Any IR);
+  PrintModuleDesc popModuleDesc(StringRef PassID);
+
+  /// Stack of Module description, enough to print the module after a given
+  /// pass.
+  SmallVector<PrintModuleDesc, 2> ModuleDescStack;
+  bool StoreModuleDesc = false;
+};
+
 /// This class provides an interface to register all the standard pass
 /// instrumentations and manages their state (if any).
 class StandardInstrumentations {
+  PrintIRInstrumentation PrintIR;
   TimePassesHandler TimePasses;
 
 public:
diff --git a/include/llvm/ProfileData/Coverage/CoverageMapping.h b/include/llvm/ProfileData/Coverage/CoverageMapping.h
index beaa36553287741299c269e3d8b8eb920817e3ed..11758ac4cf2fa247665b60e51e235b02a3f3b19c 100644
--- a/include/llvm/ProfileData/Coverage/CoverageMapping.h
+++ b/include/llvm/ProfileData/Coverage/CoverageMapping.h
@@ -1,9 +1,8 @@
 //===- CoverageMapping.h - Code coverage mapping support --------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/ProfileData/Coverage/CoverageMappingReader.h b/include/llvm/ProfileData/Coverage/CoverageMappingReader.h
index c88c71a6d6f40bc5293486d6061891c27ea2886a..dbb1976df792a76449fb33c8c64f7a83ab24c3f1 100644
--- a/include/llvm/ProfileData/Coverage/CoverageMappingReader.h
+++ b/include/llvm/ProfileData/Coverage/CoverageMappingReader.h
@@ -1,9 +1,8 @@
 //===- CoverageMappingReader.h - Code coverage mapping reader ---*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/ProfileData/Coverage/CoverageMappingWriter.h b/include/llvm/ProfileData/Coverage/CoverageMappingWriter.h
index 86fb1bdf177374a8caa7ca564ef814cc8ddd62f8..5f88cacdfcbb1f31dab6fb8a925764d975412c52 100644
--- a/include/llvm/ProfileData/Coverage/CoverageMappingWriter.h
+++ b/include/llvm/ProfileData/Coverage/CoverageMappingWriter.h
@@ -1,9 +1,8 @@
 //===- CoverageMappingWriter.h - Code coverage mapping writer ---*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/ProfileData/GCOV.h b/include/llvm/ProfileData/GCOV.h
index a088f63a6915278245e105bf2c0f81366102a410..27b76b577c10d459e48bce6415486ac1198995c0 100644
--- a/include/llvm/ProfileData/GCOV.h
+++ b/include/llvm/ProfileData/GCOV.h
@@ -1,9 +1,8 @@
 //===- GCOV.h - LLVM coverage tool ------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/ProfileData/InstrProf.h b/include/llvm/ProfileData/InstrProf.h
index dc45021fc47de76b21d687e642c65489464033d1..9ea1b9bd2fe3f47f836ed970de8bfe0bb57e5536 100644
--- a/include/llvm/ProfileData/InstrProf.h
+++ b/include/llvm/ProfileData/InstrProf.h
@@ -1,9 +1,8 @@
 //===- InstrProf.h - Instrumented profiling format support ------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -1041,6 +1040,9 @@ struct Header {
 void getMemOPSizeRangeFromOption(StringRef Str, int64_t &RangeStart,
                                  int64_t &RangeLast);
 
+// Create the variable for the profile file name.
+void createProfileFileNameVar(Module &M, StringRef InstrProfileOutput);
+
 } // end namespace llvm
 
 #endif // LLVM_PROFILEDATA_INSTRPROF_H
diff --git a/include/llvm/ProfileData/InstrProfData.inc b/include/llvm/ProfileData/InstrProfData.inc
index 454620ed997af731f178491b3fdd3b109ee6b03d..e1e2df55699a7060bca9e341992375656bc35de3 100644
--- a/include/llvm/ProfileData/InstrProfData.inc
+++ b/include/llvm/ProfileData/InstrProfData.inc
@@ -1,9 +1,8 @@
 /*===-- InstrProfData.inc - instr profiling runtime structures -*- C++ -*-=== *\
 |*
-|*                     The LLVM Compiler Infrastructure
-|*
-|* This file is distributed under the University of Illinois Open Source
-|* License. See LICENSE.TXT for details.
+|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+|* See https://llvm.org/LICENSE.txt for license information.
+|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 |*
 \*===----------------------------------------------------------------------===*/
 /*
@@ -250,22 +249,22 @@ COVMAP_HEADER(uint32_t, Int32Ty, Version, \
 #define INSTR_PROF_DATA_DEFINED
 INSTR_PROF_SECT_ENTRY(IPSK_data, \
                       INSTR_PROF_QUOTE(INSTR_PROF_DATA_COMMON), \
-                      INSTR_PROF_QUOTE(INSTR_PROF_DATA_COFF), "__DATA,")
+                      INSTR_PROF_DATA_COFF, "__DATA,")
 INSTR_PROF_SECT_ENTRY(IPSK_cnts, \
                       INSTR_PROF_QUOTE(INSTR_PROF_CNTS_COMMON), \
-                      INSTR_PROF_QUOTE(INSTR_PROF_CNTS_COFF), "__DATA,")
+                      INSTR_PROF_CNTS_COFF, "__DATA,")
 INSTR_PROF_SECT_ENTRY(IPSK_name, \
                       INSTR_PROF_QUOTE(INSTR_PROF_NAME_COMMON), \
-                      INSTR_PROF_QUOTE(INSTR_PROF_NAME_COFF), "__DATA,")
+                      INSTR_PROF_NAME_COFF, "__DATA,")
 INSTR_PROF_SECT_ENTRY(IPSK_vals, \
                       INSTR_PROF_QUOTE(INSTR_PROF_VALS_COMMON), \
-                      INSTR_PROF_QUOTE(INSTR_PROF_VALS_COFF), "__DATA,")
+                      INSTR_PROF_VALS_COFF, "__DATA,")
 INSTR_PROF_SECT_ENTRY(IPSK_vnodes, \
                       INSTR_PROF_QUOTE(INSTR_PROF_VNODES_COMMON), \
-                      INSTR_PROF_QUOTE(INSTR_PROF_VNODES_COFF), "__DATA,")
+                      INSTR_PROF_VNODES_COFF, "__DATA,")
 INSTR_PROF_SECT_ENTRY(IPSK_covmap, \
                       INSTR_PROF_QUOTE(INSTR_PROF_COVMAP_COMMON), \
-                      INSTR_PROF_QUOTE(INSTR_PROF_COVMAP_COFF), "__LLVM_COV,")
+                      INSTR_PROF_COVMAP_COFF, "__LLVM_COV,")
 
 #undef INSTR_PROF_SECT_ENTRY
 #endif
@@ -655,13 +654,15 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure,
 #define INSTR_PROF_VALS_COMMON __llvm_prf_vals
 #define INSTR_PROF_VNODES_COMMON __llvm_prf_vnds
 #define INSTR_PROF_COVMAP_COMMON __llvm_covmap
-/* Win32 */
-#define INSTR_PROF_DATA_COFF .lprfd
-#define INSTR_PROF_NAME_COFF .lprfn
-#define INSTR_PROF_CNTS_COFF .lprfc
-#define INSTR_PROF_VALS_COFF .lprfv
-#define INSTR_PROF_VNODES_COFF .lprfnd
-#define INSTR_PROF_COVMAP_COFF .lcovmap
+/* Windows section names. Because these section names contain dollar characters,
+ * they must be quoted.
+ */
+#define INSTR_PROF_DATA_COFF ".lprfd$M"
+#define INSTR_PROF_NAME_COFF ".lprfn$M"
+#define INSTR_PROF_CNTS_COFF ".lprfc$M"
+#define INSTR_PROF_VALS_COFF ".lprfv$M"
+#define INSTR_PROF_VNODES_COFF ".lprfnd$M"
+#define INSTR_PROF_COVMAP_COFF ".lcovmap$M"
 
 #ifdef _WIN32
 /* Runtime section names and name strings.  */
@@ -677,31 +678,18 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure,
 #define INSTR_PROF_COVMAP_SECT_NAME INSTR_PROF_COVMAP_COFF
 #else
 /* Runtime section names and name strings.  */
-#define INSTR_PROF_DATA_SECT_NAME INSTR_PROF_DATA_COMMON
-#define INSTR_PROF_NAME_SECT_NAME INSTR_PROF_NAME_COMMON
-#define INSTR_PROF_CNTS_SECT_NAME INSTR_PROF_CNTS_COMMON
+#define INSTR_PROF_DATA_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_DATA_COMMON)
+#define INSTR_PROF_NAME_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_NAME_COMMON)
+#define INSTR_PROF_CNTS_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_CNTS_COMMON)
 /* Array of pointers. Each pointer points to a list
  * of value nodes associated with one value site.
  */
-#define INSTR_PROF_VALS_SECT_NAME INSTR_PROF_VALS_COMMON
+#define INSTR_PROF_VALS_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_VALS_COMMON)
 /* Value profile nodes section. */
-#define INSTR_PROF_VNODES_SECT_NAME INSTR_PROF_VNODES_COMMON
-#define INSTR_PROF_COVMAP_SECT_NAME INSTR_PROF_COVMAP_COMMON
+#define INSTR_PROF_VNODES_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_VNODES_COMMON)
+#define INSTR_PROF_COVMAP_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVMAP_COMMON)
 #endif
 
-#define INSTR_PROF_DATA_SECT_NAME_STR                                          \
-  INSTR_PROF_QUOTE(INSTR_PROF_DATA_SECT_NAME)
-#define INSTR_PROF_NAME_SECT_NAME_STR                                          \
-  INSTR_PROF_QUOTE(INSTR_PROF_NAME_SECT_NAME)
-#define INSTR_PROF_CNTS_SECT_NAME_STR                                          \
-  INSTR_PROF_QUOTE(INSTR_PROF_CNTS_SECT_NAME)
-#define INSTR_PROF_COVMAP_SECT_NAME_STR                                        \
-  INSTR_PROF_QUOTE(INSTR_PROF_COVMAP_SECT_NAME)
-#define INSTR_PROF_VALS_SECT_NAME_STR                                          \
-  INSTR_PROF_QUOTE(INSTR_PROF_VALS_SECT_NAME)
-#define INSTR_PROF_VNODES_SECT_NAME_STR                                        \
-  INSTR_PROF_QUOTE(INSTR_PROF_VNODES_SECT_NAME)
-
 /* Macros to define start/stop section symbol for a given
  * section on Linux. For instance
  * INSTR_PROF_SECT_START(INSTR_PROF_DATA_SECT_NAME) will
diff --git a/include/llvm/ProfileData/InstrProfReader.h b/include/llvm/ProfileData/InstrProfReader.h
index 08d78227611708ef782f2869f170f0dc67c10c90..d465420f3d6b3eb7c585f5e96c7cb327041bb9af 100644
--- a/include/llvm/ProfileData/InstrProfReader.h
+++ b/include/llvm/ProfileData/InstrProfReader.h
@@ -1,9 +1,8 @@
 //===- InstrProfReader.h - Instrumented profiling readers -------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/ProfileData/InstrProfWriter.h b/include/llvm/ProfileData/InstrProfWriter.h
index 8107ab386fe2363096dddfaa984c63f13631b9f4..b0ab31ddfc3e701f5a48f578a780102db6972c4e 100644
--- a/include/llvm/ProfileData/InstrProfWriter.h
+++ b/include/llvm/ProfileData/InstrProfWriter.h
@@ -1,9 +1,8 @@
 //===- InstrProfWriter.h - Instrumented profiling writer --------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/ProfileData/ProfileCommon.h b/include/llvm/ProfileData/ProfileCommon.h
index 087588f06340129466173e1c8720b2a8b7270e78..b52f88643206208095bef7269b97840b20bc51e3 100644
--- a/include/llvm/ProfileData/ProfileCommon.h
+++ b/include/llvm/ProfileData/ProfileCommon.h
@@ -1,9 +1,8 @@
 //===- ProfileCommon.h - Common profiling APIs. -----------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/ProfileData/SampleProf.h b/include/llvm/ProfileData/SampleProf.h
index 927dfd246878ebdf2346de3f582125c6e940e9cf..ca3e2dec561424373e07c064cb64bebbda8ad876 100644
--- a/include/llvm/ProfileData/SampleProf.h
+++ b/include/llvm/ProfileData/SampleProf.h
@@ -1,9 +1,8 @@
 //===- SampleProf.h - Sampling profiling format support ---------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/ProfileData/SampleProfReader.h b/include/llvm/ProfileData/SampleProfReader.h
index 5cc729e42cc88f218dd37496ea5f9f4cda1e7401..32b8f91d2ed9e4484459eec2846a06b4bc45864e 100644
--- a/include/llvm/ProfileData/SampleProfReader.h
+++ b/include/llvm/ProfileData/SampleProfReader.h
@@ -1,9 +1,8 @@
 //===- SampleProfReader.h - Read LLVM sample profile data -------*- C++ -*-===//
 //
-//                      The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/ProfileData/SampleProfWriter.h b/include/llvm/ProfileData/SampleProfWriter.h
index d5ac6e53e4f723c40b45429c9d67049df813f64c..81e6e3ab0b4a3c4733b7fa2aea34b557518cf339 100644
--- a/include/llvm/ProfileData/SampleProfWriter.h
+++ b/include/llvm/ProfileData/SampleProfWriter.h
@@ -1,9 +1,8 @@
 //===- SampleProfWriter.h - Write LLVM sample profile data ------*- C++ -*-===//
 //
-//                      The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Support/AArch64TargetParser.def b/include/llvm/Support/AArch64TargetParser.def
index 6a7d16adefc6ad81a77228b831ccd7db9023abf4..d8c69c2ed370a5f5c702fa176848d5bb5057cbba 100644
--- a/include/llvm/Support/AArch64TargetParser.def
+++ b/include/llvm/Support/AArch64TargetParser.def
@@ -1,9 +1,8 @@
 //===- AARCH64TargetParser.def - AARCH64 target parsing defines ---------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -73,6 +72,8 @@ AARCH64_ARCH_EXT_NAME("rcpc",     AArch64::AEK_RCPC,     "+rcpc", "-rcpc")
 AARCH64_ARCH_EXT_NAME("rng",      AArch64::AEK_RAND,     "+rand",  "-rand")
 AARCH64_ARCH_EXT_NAME("memtag",   AArch64::AEK_MTE,      "+mte",   "-mte")
 AARCH64_ARCH_EXT_NAME("ssbs",     AArch64::AEK_SSBS,     "+ssbs",  "-ssbs")
+AARCH64_ARCH_EXT_NAME("sb",       AArch64::AEK_SB,       "+sb",    "-sb")
+AARCH64_ARCH_EXT_NAME("predres",  AArch64::AEK_PREDRES,  "+predres", "-predres")
 #undef AARCH64_ARCH_EXT_NAME
 
 #ifndef AARCH64_CPU_NAME
@@ -100,8 +101,8 @@ AARCH64_CPU_NAME("exynos-m2", ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false,
                 (AArch64::AEK_CRC))
 AARCH64_CPU_NAME("exynos-m3", ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false,
                 (AArch64::AEK_CRC))
-AARCH64_CPU_NAME("exynos-m4", ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false,
-                (AArch64::AEK_CRC))
+AARCH64_CPU_NAME("exynos-m4", ARMV8_2A, FK_CRYPTO_NEON_FP_ARMV8, false,
+                (AArch64::AEK_FP16 | AArch64::AEK_DOTPROD))
 AARCH64_CPU_NAME("falkor", ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false,
                 (AArch64::AEK_CRC | AArch64::AEK_RDM))
 AARCH64_CPU_NAME("saphira", ARMV8_3A, FK_CRYPTO_NEON_FP_ARMV8, false,
diff --git a/include/llvm/Support/AArch64TargetParser.h b/include/llvm/Support/AArch64TargetParser.h
index aea406217c46a09aa15f5bd58af5f25b62ef393e..faf1c5c237d313a3d5a22cd6794ef71ba548e6ab 100644
--- a/include/llvm/Support/AArch64TargetParser.h
+++ b/include/llvm/Support/AArch64TargetParser.h
@@ -1,9 +1,8 @@
 //===-- AArch64TargetParser - Parser for AArch64 features -------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -48,6 +47,8 @@ enum ArchExtKind : unsigned {
   AEK_RAND =        1 << 18,
   AEK_MTE =         1 << 19,
   AEK_SSBS =        1 << 20,
+  AEK_SB =          1 << 21,
+  AEK_PREDRES =     1 << 22,
 };
 
 enum class ArchKind {
diff --git a/include/llvm/Support/AMDGPUMetadata.h b/include/llvm/Support/AMDGPUMetadata.h
index 84851c07499d43f4078f0fa6e403980371f7ce0f..f43be51d093358218be79b7c995afcb1cc7ed812 100644
--- a/include/llvm/Support/AMDGPUMetadata.h
+++ b/include/llvm/Support/AMDGPUMetadata.h
@@ -1,9 +1,8 @@
 //===--- AMDGPUMetadata.h ---------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Support/AMDHSAKernelDescriptor.h b/include/llvm/Support/AMDHSAKernelDescriptor.h
index 751699e3a19f74ebcd83b394d99ec339368bc72e..dca1a3137ac39cd8fbf56e740cd1099f62600b52 100644
--- a/include/llvm/Support/AMDHSAKernelDescriptor.h
+++ b/include/llvm/Support/AMDHSAKernelDescriptor.h
@@ -1,9 +1,8 @@
 //===--- AMDHSAKernelDescriptor.h -----------------------------*- C++ -*---===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Support/ARMAttributeParser.h b/include/llvm/Support/ARMAttributeParser.h
index 919f39721f866fe44ad4a190005c873e1aaaeb4d..c7aeaf99cbbad9491ffba2c59450ff80d145df5e 100644
--- a/include/llvm/Support/ARMAttributeParser.h
+++ b/include/llvm/Support/ARMAttributeParser.h
@@ -1,9 +1,8 @@
 //===--- ARMAttributeParser.h - ARM Attribute Information Printer ---------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/Support/ARMBuildAttributes.h b/include/llvm/Support/ARMBuildAttributes.h
index b8a03765a7c0abcc2c4d267940aec274e87d9222..1299c0b7d8693bc6cfcd6b7006a90ab56f2d33d0 100644
--- a/include/llvm/Support/ARMBuildAttributes.h
+++ b/include/llvm/Support/ARMBuildAttributes.h
@@ -1,9 +1,8 @@
 //===-- ARMBuildAttributes.h - ARM Build Attributes -------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Support/ARMEHABI.h b/include/llvm/Support/ARMEHABI.h
index 9b052df0a908a6113813b43c11ca2ba1f1b02c33..3fbb56d65eb8b7731e97acba9e4fa246fbc8faad 100644
--- a/include/llvm/Support/ARMEHABI.h
+++ b/include/llvm/Support/ARMEHABI.h
@@ -1,9 +1,8 @@
 //===--- ARMEHABI.h - ARM Exception Handling ABI ----------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Support/ARMTargetParser.def b/include/llvm/Support/ARMTargetParser.def
index adf64390ff2df79ce4349fccf5c451aca15932e7..35f8333e823c9a221285c05ab2dd2508c6c3f656 100644
--- a/include/llvm/Support/ARMTargetParser.def
+++ b/include/llvm/Support/ARMTargetParser.def
@@ -1,9 +1,8 @@
 //===- ARMTargetParser.def - ARM target parsing defines ---------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -158,6 +157,7 @@ ARM_ARCH_EXT_NAME("iwmmxt2",  ARM::AEK_IWMMXT2,  nullptr,  nullptr)
 ARM_ARCH_EXT_NAME("maverick", ARM::AEK_MAVERICK, nullptr,  nullptr)
 ARM_ARCH_EXT_NAME("xscale",   ARM::AEK_XSCALE,   nullptr,  nullptr)
 ARM_ARCH_EXT_NAME("fp16fml",  ARM::AEK_FP16FML,  "+fp16fml", "-fp16fml")
+ARM_ARCH_EXT_NAME("sb",       ARM::AEK_SB,       "+sb",      "-sb")
 #undef ARM_ARCH_EXT_NAME
 
 #ifndef ARM_HW_DIV_NAME
@@ -265,7 +265,8 @@ ARM_CPU_NAME("cyclone", ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false, ARM::AEK_CRC)
 ARM_CPU_NAME("exynos-m1", ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false, ARM::AEK_CRC)
 ARM_CPU_NAME("exynos-m2", ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false, ARM::AEK_CRC)
 ARM_CPU_NAME("exynos-m3", ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false, ARM::AEK_CRC)
-ARM_CPU_NAME("exynos-m4", ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false, ARM::AEK_CRC)
+ARM_CPU_NAME("exynos-m4", ARMV8_2A, FK_CRYPTO_NEON_FP_ARMV8, false,
+             (ARM::AEK_FP16 | ARM::AEK_DOTPROD))
 ARM_CPU_NAME("kryo", ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false, ARM::AEK_CRC)
 // Non-standard Arch names.
 ARM_CPU_NAME("iwmmxt", IWMMXT, FK_NONE, true, ARM::AEK_NONE)
diff --git a/include/llvm/Support/ARMTargetParser.h b/include/llvm/Support/ARMTargetParser.h
index e41675a1f6087194d3e071bf0a44ab0d9d1b11f5..edd47ef82ad91890bde1604b0785de207a165ef7 100644
--- a/include/llvm/Support/ARMTargetParser.h
+++ b/include/llvm/Support/ARMTargetParser.h
@@ -1,9 +1,8 @@
 //===-- ARMTargetParser - Parser for ARM target features --------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -45,6 +44,7 @@ enum ArchExtKind : unsigned {
   AEK_SHA2    =     1 << 15,
   AEK_AES     =     1 << 16,
   AEK_FP16FML =     1 << 17,
+  AEK_SB      =     1 << 18,
   // Unsupported extensions.
   AEK_OS = 0x8000000,
   AEK_IWMMXT = 0x10000000,
diff --git a/include/llvm/Support/ARMWinEH.h b/include/llvm/Support/ARMWinEH.h
index 60174503ad4924e6b76a03846c38f3080573585c..b1b821b78a4e262226cf83cce3a66f88fb0a2451 100644
--- a/include/llvm/Support/ARMWinEH.h
+++ b/include/llvm/Support/ARMWinEH.h
@@ -1,9 +1,8 @@
-//===-- llvm/Support/WinARMEH.h - Windows on ARM EH Constants ---*- C++ -*-===//
+//===-- llvm/Support/ARMWinEH.h - Windows on ARM EH Constants ---*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/Support/AlignOf.h b/include/llvm/Support/AlignOf.h
index 9e7a62b85e342bc4ad460a6ff765eca3e00df598..d12401f0eb49a32983cdd9feab1103e1ca280b9b 100644
--- a/include/llvm/Support/AlignOf.h
+++ b/include/llvm/Support/AlignOf.h
@@ -1,9 +1,8 @@
 //===--- AlignOf.h - Portable calculation of type alignment -----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Support/Allocator.h b/include/llvm/Support/Allocator.h
index 42d08378a6774a1da39ca28782eb5dc17fb622c5..09e967b98abcbb60b60a07dbf7c9c7edf5a46d8c 100644
--- a/include/llvm/Support/Allocator.h
+++ b/include/llvm/Support/Allocator.h
@@ -1,9 +1,8 @@
 //===- Allocator.h - Simple memory allocation abstraction -------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/include/llvm/Support/ArrayRecycler.h b/include/llvm/Support/ArrayRecycler.h
index 68696be6bf3d13e695552bd5ded42eff8d303c2e..5256ce80c0282cea9c906912504cb071a5ade4e0 100644
--- a/include/llvm/Support/ArrayRecycler.h
+++ b/include/llvm/Support/ArrayRecycler.h
@@ -1,9 +1,8 @@
 //==- llvm/Support/ArrayRecycler.h - Recycling of Arrays ---------*- C++ -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Support/Atomic.h b/include/llvm/Support/Atomic.h
index 552313f0c2412bfebc895832ac06d8159f607497..a8445fddc1a85537ecaae14f29998a7777bc99ad 100644
--- a/include/llvm/Support/Atomic.h
+++ b/include/llvm/Support/Atomic.h
@@ -1,9 +1,8 @@
 //===- llvm/Support/Atomic.h - Atomic Operations -----------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Support/AtomicOrdering.h b/include/llvm/Support/AtomicOrdering.h
index a679ab30243ee14e9c3937f181c4a59dac25c538..763bc3ea7b282a7b8d713f0b718e5c78841ace82 100644
--- a/include/llvm/Support/AtomicOrdering.h
+++ b/include/llvm/Support/AtomicOrdering.h
@@ -1,9 +1,8 @@
 //===-- llvm/Support/AtomicOrdering.h ---Atomic Ordering---------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
diff --git a/include/llvm/Support/BinaryByteStream.h b/include/llvm/Support/BinaryByteStream.h
index 9808d3b721576b3fafa8a02f1cf2bc7e9d4b6496..7acce9a038885762fb3fecac194184ce17247bcc 100644
--- a/include/llvm/Support/BinaryByteStream.h
+++ b/include/llvm/Support/BinaryByteStream.h
@@ -1,9 +1,8 @@
 //===- BinaryByteStream.h ---------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //===----------------------------------------------------------------------===//
 // A BinaryStream which stores data in a single continguous memory buffer.
 //===----------------------------------------------------------------------===//
diff --git a/include/llvm/Support/BinaryItemStream.h b/include/llvm/Support/BinaryItemStream.h
index 278723ddf8dafb0592ed5d696d7b33e4d9525747..4cd66adcc01ab8cee7274c8bf2c5951abf78dc27 100644
--- a/include/llvm/Support/BinaryItemStream.h
+++ b/include/llvm/Support/BinaryItemStream.h
@@ -1,9 +1,8 @@
 //===- BinaryItemStream.h ---------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/Support/BinaryStream.h b/include/llvm/Support/BinaryStream.h
index 7677214e48ee59e03ad895e52926e3151060e23b..fcf4398550eeebd2e75fd20587955562c17e67ff 100644
--- a/include/llvm/Support/BinaryStream.h
+++ b/include/llvm/Support/BinaryStream.h
@@ -1,9 +1,8 @@
 //===- BinaryStream.h - Base interface for a stream of data -----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/Support/BinaryStreamArray.h b/include/llvm/Support/BinaryStreamArray.h
index 7c110fcb6a4b1be8bdeadfeb05725f21ded8aacb..96d09db69ae53685b3d31b9c24ff660f033e0a82 100644
--- a/include/llvm/Support/BinaryStreamArray.h
+++ b/include/llvm/Support/BinaryStreamArray.h
@@ -1,9 +1,8 @@
 //===- BinaryStreamArray.h - Array backed by an arbitrary stream *- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/Support/BinaryStreamError.h b/include/llvm/Support/BinaryStreamError.h
index 7d9699d5363910a05e2bd200411a6ac4ead916bb..cf6e034ffd2cebd0d906dc1157ad6d342cac9be6 100644
--- a/include/llvm/Support/BinaryStreamError.h
+++ b/include/llvm/Support/BinaryStreamError.h
@@ -1,9 +1,8 @@
 //===- BinaryStreamError.h - Error extensions for Binary Streams *- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/Support/BinaryStreamReader.h b/include/llvm/Support/BinaryStreamReader.h
index 392958de30d5656aff5c0aa037a3f8a455930755..32e5728448291a4417b8df2a1e31c2b013f67913 100644
--- a/include/llvm/Support/BinaryStreamReader.h
+++ b/include/llvm/Support/BinaryStreamReader.h
@@ -1,9 +1,8 @@
 //===- BinaryStreamReader.h - Reads objects from a binary stream *- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/Support/BinaryStreamRef.h b/include/llvm/Support/BinaryStreamRef.h
index d8dc1392c01c229d6acd55bf21de22ac4e0d3747..7427b8da5b43554caafc3c66fefe70be79cdb82a 100644
--- a/include/llvm/Support/BinaryStreamRef.h
+++ b/include/llvm/Support/BinaryStreamRef.h
@@ -1,9 +1,8 @@
 //===- BinaryStreamRef.h - A copyable reference to a stream -----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/Support/BinaryStreamWriter.h b/include/llvm/Support/BinaryStreamWriter.h
index 6e8a68a30474260b6a72faeea260d694119ea31a..ae019c81e157e5236f0b7b916f08cb3c46f38e30 100644
--- a/include/llvm/Support/BinaryStreamWriter.h
+++ b/include/llvm/Support/BinaryStreamWriter.h
@@ -1,9 +1,8 @@
 //===- BinaryStreamWriter.h - Writes objects to a BinaryStream ---*- C++-*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/Support/BlockFrequency.h b/include/llvm/Support/BlockFrequency.h
index 4b468f7acb322f3a8a40f3c83277075db485444c..18fb60e1904b3558b09836286b37644b90b1617c 100644
--- a/include/llvm/Support/BlockFrequency.h
+++ b/include/llvm/Support/BlockFrequency.h
@@ -1,9 +1,8 @@
 //===-------- BlockFrequency.h - Block Frequency Wrapper --------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Support/BranchProbability.h b/include/llvm/Support/BranchProbability.h
index 3a88e71c24804234c247c51bc98a4889883aecfa..dd0aba025abfb6188241823e1718c4e6e677459c 100644
--- a/include/llvm/Support/BranchProbability.h
+++ b/include/llvm/Support/BranchProbability.h
@@ -1,9 +1,8 @@
 //===- BranchProbability.h - Branch Probability Wrapper ---------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Support/BuryPointer.h b/include/llvm/Support/BuryPointer.h
index 53f1f395b922947713e4ab5f8a98a8d291f5a343..276a5b7089c3e7e89327f5efa4ce70097ba8f238 100644
--- a/include/llvm/Support/BuryPointer.h
+++ b/include/llvm/Support/BuryPointer.h
@@ -1,9 +1,8 @@
 //===- llvm/Support/BuryPointer.h - Memory Manipulation/Leak ----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/Support/CBindingWrapping.h b/include/llvm/Support/CBindingWrapping.h
index f60f99d376ad433163ed8904d62bb1d421e8bc36..46d6b4e3fa7d5917d95a61245aaaa7740a2b68b9 100644
--- a/include/llvm/Support/CBindingWrapping.h
+++ b/include/llvm/Support/CBindingWrapping.h
@@ -1,9 +1,8 @@
-//===- llvm/Support/CBindingWrapph.h - C Interface Wrapping -----*- C++ -*-===//
+//===- llvm/Support/CBindingWrapping.h - C Interface Wrapping ---*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Support/CFGUpdate.h b/include/llvm/Support/CFGUpdate.h
index 63c24a3d2a20e45cbc2a6860fefdc6d8b9546a15..eeaf5d0a21acd39df01f56c7632bd52a9acf5675 100644
--- a/include/llvm/Support/CFGUpdate.h
+++ b/include/llvm/Support/CFGUpdate.h
@@ -1,9 +1,8 @@
 //===- CFGUpdate.h - Encode a CFG Edge Update. ------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Support/CMakeLists.txt b/include/llvm/Support/CMakeLists.txt
index bba962a5de10643cdff3d3b4ea11449f71a41b87..680be8fdf3911913c4c623845adc61748bbac4fd 100644
--- a/include/llvm/Support/CMakeLists.txt
+++ b/include/llvm/Support/CMakeLists.txt
@@ -1,38 +1,21 @@
-find_first_existing_vc_file(llvm_vc "${LLVM_MAIN_SRC_DIR}")
+find_first_existing_vc_file("${LLVM_MAIN_SRC_DIR}" llvm_vc)
 
 # The VC revision include that we want to generate.
 set(version_inc "${CMAKE_CURRENT_BINARY_DIR}/VCSRevision.h")
 
-set(get_svn_script "${LLVM_CMAKE_PATH}/GenerateVersionFromCVS.cmake")
+set(generate_vcs_version_script "${LLVM_CMAKE_PATH}/GenerateVersionFromVCS.cmake")
 
-file(WRITE "${version_inc}.undef" "#undef LLVM_REVISION\n")
-if((DEFINED llvm_vc) AND LLVM_APPEND_VC_REV)
-
-  execute_process(COMMAND ${CMAKE_COMMAND} -E compare_files
-      "${version_inc}.undef" "${version_inc}"
-      RESULT_VARIABLE files_not_equal
-      OUTPUT_QUIET
-      ERROR_QUIET)
-  # Remove ${version_inc} if it doesn't define a revision. This will force it
-  # to be regenerated when toggling LLVM_APPEND_VC_REV from OFF to ON.
-  if(NOT files_not_equal)
-    file(REMOVE "${version_inc}")
-  endif()
-
-  # Create custom target to generate the VC revision include.
-  add_custom_command(OUTPUT "${version_inc}"
-    DEPENDS "${llvm_vc}" "${get_svn_script}"
-    COMMAND
-    ${CMAKE_COMMAND} "-DSOURCE_DIR=${LLVM_MAIN_SRC_DIR}"
-                     "-DNAME=LLVM_REVISION"
-                     "-DHEADER_FILE=${version_inc}"
-                     -P "${get_svn_script}")
-else()
-  # Make sure ${version_inc} doesn't define a revision
-  execute_process(COMMAND ${CMAKE_COMMAND} -E copy_if_different
-    "${version_inc}.undef" "${version_inc}")
+if(llvm_vc AND LLVM_APPEND_VC_REV)
+  set(llvm_source_dir ${LLVM_MAIN_SRC_DIR})
 endif()
-file(REMOVE "${version_inc}.undef")
+
+# Create custom target to generate the VC revision include.
+add_custom_command(OUTPUT "${version_inc}"
+  DEPENDS "${llvm_vc}" "${generate_vcs_version_script}"
+  COMMAND ${CMAKE_COMMAND} "-DNAMES=LLVM"
+                           "-DLLVM_SOURCE_DIR=${llvm_source_dir}"
+                           "-DHEADER_FILE=${version_inc}"
+                           -P "${generate_vcs_version_script}")
 
 # Mark the generated header as being generated.
 set_source_files_properties("${version_inc}"
diff --git a/include/llvm/Support/COM.h b/include/llvm/Support/COM.h
index a2d5a7a68ba9f866b24d23fbf2a258a300555259..d59966f849b48546fd9e6cd0ee0547324d9cfc61 100644
--- a/include/llvm/Support/COM.h
+++ b/include/llvm/Support/COM.h
@@ -1,9 +1,8 @@
 //===- llvm/Support/COM.h ---------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/include/llvm/Support/CachePruning.h b/include/llvm/Support/CachePruning.h
index cf3f8ec67a52ce9fd7866b1b2f8467cbcf7ac8e5..a72a86439f6a5f7985bfb6cb4ef7ddd25fdf96c7 100644
--- a/include/llvm/Support/CachePruning.h
+++ b/include/llvm/Support/CachePruning.h
@@ -1,9 +1,8 @@
 //=- CachePruning.h - Helper to manage the pruning of a cache dir -*- C++ -*-=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Support/Capacity.h b/include/llvm/Support/Capacity.h
index 7460f9825bd32a33ceb97e15bf6f19530849937d..6b99e0aaa48853b882a2055008df0bb415cc00e1 100644
--- a/include/llvm/Support/Capacity.h
+++ b/include/llvm/Support/Capacity.h
@@ -1,9 +1,8 @@
 //===--- Capacity.h - Generic computation of ADT memory use -----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Support/Casting.h b/include/llvm/Support/Casting.h
index 3f21e0f9ebc33a316869e6fa8d5a57fe8bc3b99f..cddee3642fa9e54ba1c8e7434038a641954ef339 100644
--- a/include/llvm/Support/Casting.h
+++ b/include/llvm/Support/Casting.h
@@ -1,9 +1,8 @@
 //===- llvm/Support/Casting.h - Allow flexible, checked, casts --*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Support/CheckedArithmetic.h b/include/llvm/Support/CheckedArithmetic.h
index 039c374136ffc68122448c5c9f42ffc71f727935..8f7fbde8c3aeeccaa94e27e47fbb328e795f2d22 100644
--- a/include/llvm/Support/CheckedArithmetic.h
+++ b/include/llvm/Support/CheckedArithmetic.h
@@ -1,9 +1,8 @@
 //==-- llvm/Support/CheckedArithmetic.h - Safe arithmetical operations *- C++ //
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Support/Chrono.h b/include/llvm/Support/Chrono.h
index 57677e8d5cf149326f6fe32f8a16cd4cb1a9917c..334ab60835a44a95d44bd6b9e9e27fe35fe66d6c 100644
--- a/include/llvm/Support/Chrono.h
+++ b/include/llvm/Support/Chrono.h
@@ -1,9 +1,8 @@
 //===- llvm/Support/Chrono.h - Utilities for Timing Manipulation-*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -34,21 +33,21 @@ template <typename D = std::chrono::nanoseconds>
 using TimePoint = std::chrono::time_point<std::chrono::system_clock, D>;
 
 /// Convert a TimePoint to std::time_t
-LLVM_ATTRIBUTE_ALWAYS_INLINE inline std::time_t toTimeT(TimePoint<> TP) {
+inline std::time_t toTimeT(TimePoint<> TP) {
   using namespace std::chrono;
   return system_clock::to_time_t(
       time_point_cast<system_clock::time_point::duration>(TP));
 }
 
 /// Convert a std::time_t to a TimePoint
-LLVM_ATTRIBUTE_ALWAYS_INLINE inline TimePoint<std::chrono::seconds>
+inline TimePoint<std::chrono::seconds>
 toTimePoint(std::time_t T) {
   using namespace std::chrono;
   return time_point_cast<seconds>(system_clock::from_time_t(T));
 }
 
 /// Convert a std::time_t + nanoseconds to a TimePoint
-LLVM_ATTRIBUTE_ALWAYS_INLINE inline TimePoint<>
+inline TimePoint<>
 toTimePoint(std::time_t T, uint32_t nsec) {
   using namespace std::chrono;
   return time_point_cast<nanoseconds>(system_clock::from_time_t(T))
diff --git a/include/llvm/Support/CodeGen.h b/include/llvm/Support/CodeGen.h
index ce3a62382f9e41a6a1754b947122c255e9f5af63..3175b29a4dd9a6e61c783730a01f9fea15c8085f 100644
--- a/include/llvm/Support/CodeGen.h
+++ b/include/llvm/Support/CodeGen.h
@@ -1,9 +1,8 @@
 //===-- llvm/Support/CodeGen.h - CodeGen Concepts ---------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -50,13 +49,18 @@ namespace llvm {
   // Code generation optimization level.
   namespace CodeGenOpt {
     enum Level {
-      None,        // -O0
-      Less,        // -O1
-      Default,     // -O2, -Os
-      Aggressive   // -O3
+      None = 0,      // -O0
+      Less = 1,      // -O1
+      Default = 2,   // -O2, -Os
+      Aggressive = 3 // -O3
     };
   }
 
+  // Specify effect of frame pointer elimination optimization.
+  namespace FramePointer {
+    enum FP {All, NonLeaf, None};
+  }
+
 }  // end llvm namespace
 
 #endif
diff --git a/include/llvm/Support/CodeGenCoverage.h b/include/llvm/Support/CodeGenCoverage.h
index c863be35b82231a6b5888aac16b039143204366e..0b1af779ffb0695c88835302245257fb60c68edf 100644
--- a/include/llvm/Support/CodeGenCoverage.h
+++ b/include/llvm/Support/CodeGenCoverage.h
@@ -1,9 +1,8 @@
 //== llvm/Support/CodeGenCoverage.h ------------------------------*- C++ -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file This file provides rule coverage tracking for tablegen-erated CodeGen.
diff --git a/include/llvm/Support/CommandLine.h b/include/llvm/Support/CommandLine.h
index cd3543c130e085970e4ccf780428f5b5048685c7..a78020f010fa082168c68f62557fe2a36ed8b830 100644
--- a/include/llvm/Support/CommandLine.h
+++ b/include/llvm/Support/CommandLine.h
@@ -1,9 +1,8 @@
 //===- llvm/Support/CommandLine.h - Command line handler --------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -156,6 +155,9 @@ enum OptionHidden {   // Control whether -help shows this option
 // enabled, and used, the value for the flag comes from the suffix of the
 // argument.
 //
+// AlwaysPrefix - Only allow the behavior enabled by the Prefix flag and reject
+// the Option=Value form.
+//
 // Grouping - With this option enabled, multiple letter options are allowed to
 // bunch together with only a single hyphen for the whole group.  This allows
 // emulation of the behavior that ls uses for example: ls -la === ls -l -a
@@ -165,7 +167,8 @@ enum FormattingFlags {
   NormalFormatting = 0x00, // Nothing special
   Positional = 0x01,       // Is a positional argument, no '-' required
   Prefix = 0x02,           // Can this option directly prefix its value?
-  Grouping = 0x03          // Can this option group with other options?
+  AlwaysPrefix = 0x03,     // Can this option only directly prefix its value?
+  Grouping = 0x04          // Can this option group with other options?
 };
 
 enum MiscFlags {             // Miscellaneous flags to adjust argument
@@ -265,7 +268,7 @@ class Option {
   // detail representing the non-value
   unsigned Value : 2;
   unsigned HiddenFlag : 2; // enum OptionHidden
-  unsigned Formatting : 2; // enum FormattingFlags
+  unsigned Formatting : 3; // enum FormattingFlags
   unsigned Misc : 3;
   unsigned Position = 0;       // Position of last occurrence of the option
   unsigned AdditionalVals = 0; // Greater than 0 for multi-valued option.
diff --git a/include/llvm/Support/Compiler.h b/include/llvm/Support/Compiler.h
index 14e4d6e9714047ac211d45c1634f98d7d09b9185..a81488a2e4c955e18f83b110b1647064707f0d2b 100644
--- a/include/llvm/Support/Compiler.h
+++ b/include/llvm/Support/Compiler.h
@@ -1,9 +1,8 @@
 //===-- llvm/Support/Compiler.h - Compiler abstraction support --*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Support/Compression.h b/include/llvm/Support/Compression.h
index f7258f4bf8f825b78f614c1a09950bccac46c619..5bc0e56913fe2689975301e321227f5591cf50e7 100644
--- a/include/llvm/Support/Compression.h
+++ b/include/llvm/Support/Compression.h
@@ -1,9 +1,8 @@
 //===-- llvm/Support/Compression.h ---Compression----------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Support/ConvertUTF.h b/include/llvm/Support/ConvertUTF.h
index 6ae56c2470bb2b83082ae0f83bcdae4cae090bf0..1add185330fa02d7fca2f8d9f89695af4342cba5 100644
--- a/include/llvm/Support/ConvertUTF.h
+++ b/include/llvm/Support/ConvertUTF.h
@@ -1,9 +1,8 @@
 /*===--- ConvertUTF.h - Universal Character Names conversions ---------------===
  *
- *                     The LLVM Compiler Infrastructure
- *
- * This file is distributed under the University of Illinois Open Source
- * License. See LICENSE.TXT for details.
+ * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+ * See https://llvm.org/LICENSE.txt for license information.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
  *
  *==------------------------------------------------------------------------==*/
 /*
diff --git a/include/llvm/Support/CrashRecoveryContext.h b/include/llvm/Support/CrashRecoveryContext.h
index 7b3fd4f882e4bd3bd9eb2f564bb2bb51ac185539..feb449e2899c3c78db5276c2a5bacf6c173c26a9 100644
--- a/include/llvm/Support/CrashRecoveryContext.h
+++ b/include/llvm/Support/CrashRecoveryContext.h
@@ -1,9 +1,8 @@
 //===--- CrashRecoveryContext.h - Crash Recovery ----------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/Support/DJB.h b/include/llvm/Support/DJB.h
index e03111473362e170cc67b2737166c9af800c63f5..8a04a324a5dc6cccaaa6af7b0e9b340f0755587e 100644
--- a/include/llvm/Support/DJB.h
+++ b/include/llvm/Support/DJB.h
@@ -1,9 +1,8 @@
 //===-- llvm/Support/DJB.h ---DJB Hash --------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Support/DOTGraphTraits.h b/include/llvm/Support/DOTGraphTraits.h
index 4381b5bf163345c32d6dce539db60a9274a9e0e4..ec01b7d9576ae53e4954c183330bda8ff9660f35 100644
--- a/include/llvm/Support/DOTGraphTraits.h
+++ b/include/llvm/Support/DOTGraphTraits.h
@@ -1,9 +1,8 @@
-//===-- llvm/Support/DotGraphTraits.h - Customize .dot output ---*- C++ -*-===//
+//===-- llvm/Support/DOTGraphTraits.h - Customize .dot output ---*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Support/DataExtractor.h b/include/llvm/Support/DataExtractor.h
index 2b1639856e7942aa1776dd9625fb7fd8e34668ca..6b08a2a2a4452e0ff813e8829f5d6e832c6ed822 100644
--- a/include/llvm/Support/DataExtractor.h
+++ b/include/llvm/Support/DataExtractor.h
@@ -1,9 +1,8 @@
 //===-- DataExtractor.h -----------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/Support/DataTypes.h b/include/llvm/Support/DataTypes.h
index ad60a5b3f300c8af5d254dea83847fb59fe5679a..a3fcc82531b7191d907d03a09c70eaf3f47fcdb8 100644
--- a/include/llvm/Support/DataTypes.h
+++ b/include/llvm/Support/DataTypes.h
@@ -1,9 +1,8 @@
 //===-- llvm/Support/DataTypes.h - Define fixed size types ------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Support/Debug.h b/include/llvm/Support/Debug.h
index df86dbb82414a9fcabe3258f94ae96a92a308a2c..64b730951bdaba0f6337e70654190ab99b3e959b 100644
--- a/include/llvm/Support/Debug.h
+++ b/include/llvm/Support/Debug.h
@@ -1,9 +1,8 @@
 //===- llvm/Support/Debug.h - Easy way to add debug output ------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Support/DebugCounter.h b/include/llvm/Support/DebugCounter.h
index 6eadd5c6aefff6bbb77939dc2fe69585ad1446e3..e7d1fa68f21ae665ed142fa269446750afefb941 100644
--- a/include/llvm/Support/DebugCounter.h
+++ b/include/llvm/Support/DebugCounter.h
@@ -1,9 +1,8 @@
 //===- llvm/Support/DebugCounter.h - Debug counter support ------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/include/llvm/Support/DynamicLibrary.h b/include/llvm/Support/DynamicLibrary.h
index 9563b483f6d57d828e7b6a89999ae62452eadd09..95d5ba281e225061a68b812b19ab899684d15265 100644
--- a/include/llvm/Support/DynamicLibrary.h
+++ b/include/llvm/Support/DynamicLibrary.h
@@ -1,9 +1,8 @@
 //===-- llvm/Support/DynamicLibrary.h - Portable Dynamic Library -*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Support/Endian.h b/include/llvm/Support/Endian.h
index a4d3f4ff793df74e2472ec0d2f1def31f3d868fd..a0632bbc4b5815f2c9c7f19ae07ac7063dc2d414 100644
--- a/include/llvm/Support/Endian.h
+++ b/include/llvm/Support/Endian.h
@@ -1,9 +1,8 @@
 //===- Endian.h - Utilities for IO with endian specific data ----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Support/EndianStream.h b/include/llvm/Support/EndianStream.h
index 9742e253ad3e1e8f3b90a5cd1a4ad1aa98d412f1..87898038d2162c4c3f54e4c556b13dbec03aa871 100644
--- a/include/llvm/Support/EndianStream.h
+++ b/include/llvm/Support/EndianStream.h
@@ -1,9 +1,8 @@
 //===- EndianStream.h - Stream ops with endian specific data ----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Support/Errc.h b/include/llvm/Support/Errc.h
index dce42782a0d36c3ba7f05b98ffd0a00a2f975069..9be8e5705a54cbda0fe1dcdba394d3e1ba698ed0 100644
--- a/include/llvm/Support/Errc.h
+++ b/include/llvm/Support/Errc.h
@@ -1,9 +1,8 @@
 //===- llvm/Support/Errc.h - Defines the llvm::errc enum --------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Support/Errno.h b/include/llvm/Support/Errno.h
index 8069c3639df32b3f6c92ef2947f61c3974cdebff..aedb5fb292b84d81d0e4b490070a33ed93325dca 100644
--- a/include/llvm/Support/Errno.h
+++ b/include/llvm/Support/Errno.h
@@ -1,9 +1,8 @@
 //===- llvm/Support/Errno.h - Portable+convenient errno handling -*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Support/Error.h b/include/llvm/Support/Error.h
index 4962812e750e6007ce0024c1217c82506c7fff64..23a48ed03e44627dc2274ad823290252ddd6a12b 100644
--- a/include/llvm/Support/Error.h
+++ b/include/llvm/Support/Error.h
@@ -1,9 +1,8 @@
 //===- llvm/Support/Error.h - Recoverable error handling --------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -1066,6 +1065,8 @@ private:
 class ECError : public ErrorInfo<ECError> {
   friend Error errorCodeToError(std::error_code);
 
+  virtual void anchor() override;
+
 public:
   void setErrorCode(std::error_code EC) { this->EC = EC; }
   std::error_code convertToErrorCode() const override { return EC; }
@@ -1175,7 +1176,7 @@ Error createStringError(std::error_code EC, char const *Msg);
 /// show more detailed information to the user.
 class FileError final : public ErrorInfo<FileError> {
 
-  friend Error createFileError(std::string, Error);
+  friend Error createFileError(const Twine &, Error);
 
 public:
   void log(raw_ostream &OS) const override {
@@ -1192,15 +1193,15 @@ public:
   static char ID;
 
 private:
-  FileError(std::string F, std::unique_ptr<ErrorInfoBase> E) {
+  FileError(const Twine &F, std::unique_ptr<ErrorInfoBase> E) {
     assert(E && "Cannot create FileError from Error success value.");
-    assert(!F.empty() &&
+    assert(!F.isTriviallyEmpty() &&
            "The file name provided to FileError must not be empty.");
-    FileName = F;
+    FileName = F.str();
     Err = std::move(E);
   }
 
-  static Error build(std::string F, Error E) {
+  static Error build(const Twine &F, Error E) {
     return Error(std::unique_ptr<FileError>(new FileError(F, E.takePayload())));
   }
 
@@ -1210,11 +1211,17 @@ private:
 
 /// Concatenate a source file path and/or name with an Error. The resulting
 /// Error is unchecked.
-inline Error createFileError(std::string F, Error E) {
+inline Error createFileError(const Twine &F, Error E) {
   return FileError::build(F, std::move(E));
 }
 
-Error createFileError(std::string F, ErrorSuccess) = delete;
+/// Concatenate a source file path and/or name with a std::error_code 
+/// to form an Error object.
+inline Error createFileError(const Twine &F, std::error_code EC) {
+  return createFileError(F, errorCodeToError(EC));
+}
+
+Error createFileError(const Twine &F, ErrorSuccess) = delete;
 
 /// Helper for check-and-exit error handling.
 ///
diff --git a/include/llvm/Support/ErrorHandling.h b/include/llvm/Support/ErrorHandling.h
index fec39e59a717071b00b2feb6aebcd7d0bbf5b4dc..f75c2984a9ff7031a2d0c1c216a50060621b3b16 100644
--- a/include/llvm/Support/ErrorHandling.h
+++ b/include/llvm/Support/ErrorHandling.h
@@ -1,9 +1,8 @@
 //===- llvm/Support/ErrorHandling.h - Fatal error handling ------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Support/ErrorOr.h b/include/llvm/Support/ErrorOr.h
index e6ce764ad822a702f79f13ff070d017a601b202f..8211f4d8a098b458802dfba1d58ed2656da5b760 100644
--- a/include/llvm/Support/ErrorOr.h
+++ b/include/llvm/Support/ErrorOr.h
@@ -1,9 +1,8 @@
 //===- llvm/Support/ErrorOr.h - Error Smart Pointer -------------*- C++ -*-===//
 //
-//                             The LLVM Linker
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
diff --git a/include/llvm/Support/FileCheck.h b/include/llvm/Support/FileCheck.h
index 4061a26e22c50f04abdc801f600fb5d55ccc52bb..01e9362a36540ec18756599e49658189111e8001 100644
--- a/include/llvm/Support/FileCheck.h
+++ b/include/llvm/Support/FileCheck.h
@@ -1,9 +1,8 @@
 //==-- llvm/Support/FileCheck.h ---------------------------*- C++ -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Support/FileOutputBuffer.h b/include/llvm/Support/FileOutputBuffer.h
index 68226ca55502f7475fa0a5e320c8f2c660286dac..999f551ebf2d575a9508b325b42925edd2fc674e 100644
--- a/include/llvm/Support/FileOutputBuffer.h
+++ b/include/llvm/Support/FileOutputBuffer.h
@@ -1,9 +1,8 @@
 //=== FileOutputBuffer.h - File Output Buffer -------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -33,11 +32,6 @@ public:
   enum {
     /// set the 'x' bit on the resulting file
     F_executable = 1,
-
-    /// the contents of the new file are initialized from the file that exists
-    /// at the location (if present).  This allows in-place modification of an
-    /// existing file.
-    F_modify = 2
   };
 
   /// Factory method to create an OutputBuffer object which manages a read/write
diff --git a/include/llvm/Support/FileSystem.h b/include/llvm/Support/FileSystem.h
index 827e2e91eea218b2e684170e65b2978b71e974ee..156250faabe0f10f33f3e602abc9ad91866d6ea2 100644
--- a/include/llvm/Support/FileSystem.h
+++ b/include/llvm/Support/FileSystem.h
@@ -1,9 +1,8 @@
 //===- llvm/Support/FileSystem.h - File System OS Concept -------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -302,10 +301,7 @@ public:
 /// relative/../path => <current-directory>/relative/../path
 ///
 /// @param path A path that is modified to be an absolute path.
-/// @returns errc::success if \a path has been made absolute, otherwise a
-///          platform-specific error_code.
-std::error_code make_absolute(const Twine &current_directory,
-                              SmallVectorImpl<char> &path);
+void make_absolute(const Twine &current_directory, SmallVectorImpl<char> &path);
 
 /// Make \a path an absolute path.
 ///
diff --git a/include/llvm/Support/FileUtilities.h b/include/llvm/Support/FileUtilities.h
index 2ee2c60b9964b6159d2f26dfc6542d6411099909..16b2206924c3d8b9f4c84ad3012bfaf6350ceb52 100644
--- a/include/llvm/Support/FileUtilities.h
+++ b/include/llvm/Support/FileUtilities.h
@@ -1,9 +1,8 @@
 //===- llvm/Support/FileUtilities.h - File System Utilities -----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Support/Format.h b/include/llvm/Support/Format.h
index bcbd2bec572280f50de2a8b3be8f014708dbc2b9..77dcbaebf1a3051c946f1977273ed901bb404125 100644
--- a/include/llvm/Support/Format.h
+++ b/include/llvm/Support/Format.h
@@ -1,9 +1,8 @@
 //===- Format.h - Efficient printf-style formatting for streams -*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Support/FormatAdapters.h b/include/llvm/Support/FormatAdapters.h
index 8320eaad39a9d012c240a402ddba5c9b7721e921..a0e8cc43919129b9db05e1651b21d53338440205 100644
--- a/include/llvm/Support/FormatAdapters.h
+++ b/include/llvm/Support/FormatAdapters.h
@@ -1,9 +1,8 @@
 //===- FormatAdapters.h - Formatters for common LLVM types -----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/Support/FormatCommon.h b/include/llvm/Support/FormatCommon.h
index 36fbad296c3f2bd1b83ff6a9fdd2724027d077b9..3c119d12529aeedad61cd619ca521f200f4e6420 100644
--- a/include/llvm/Support/FormatCommon.h
+++ b/include/llvm/Support/FormatCommon.h
@@ -1,9 +1,8 @@
-//===- FormatAdapters.h - Formatters for common LLVM types -----*- C++ -*-===//
+//===- FormatCommon.h - Formatters for common LLVM types --------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/Support/FormatProviders.h b/include/llvm/Support/FormatProviders.h
index 4e57034ff98e9336478548cfba0fb8ed2a46fe44..629a4845716a42b9405e5255b01e88c255f46c4d 100644
--- a/include/llvm/Support/FormatProviders.h
+++ b/include/llvm/Support/FormatProviders.h
@@ -1,9 +1,8 @@
 //===- FormatProviders.h - Formatters for common LLVM types -----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Support/FormatVariadic.h b/include/llvm/Support/FormatVariadic.h
index b0f582513e07a4c59e81cb695a12a0d43c453ac5..5bbda9dd626eb9b2dc47489a5bf0173182b699eb 100644
--- a/include/llvm/Support/FormatVariadic.h
+++ b/include/llvm/Support/FormatVariadic.h
@@ -1,9 +1,8 @@
 //===- FormatVariadic.h - Efficient type-safe string formatting --*- C++-*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Support/FormatVariadicDetails.h b/include/llvm/Support/FormatVariadicDetails.h
index 56dda430efda31fde92ab1ef8d9c07ed76cbd2d9..e3c185134daa4d4c1bf2612174f753053a4bf2e4 100644
--- a/include/llvm/Support/FormatVariadicDetails.h
+++ b/include/llvm/Support/FormatVariadicDetails.h
@@ -1,9 +1,8 @@
 //===- FormatVariadicDetails.h - Helpers for FormatVariadic.h ----*- C++-*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -21,6 +20,8 @@ class Error;
 
 namespace detail {
 class format_adapter {
+  virtual void anchor();
+
 protected:
   virtual ~format_adapter() {}
 
diff --git a/include/llvm/Support/FormattedStream.h b/include/llvm/Support/FormattedStream.h
index 4a135cd23174f007c21cc0fc64c6e144f9f85ab9..b49c8d86531dbcaf49b1f78aec9a8988ff66b9b0 100644
--- a/include/llvm/Support/FormattedStream.h
+++ b/include/llvm/Support/FormattedStream.h
@@ -1,9 +1,8 @@
 //===-- llvm/Support/FormattedStream.h - Formatted streams ------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Support/GenericDomTree.h b/include/llvm/Support/GenericDomTree.h
index b3018bac310ac95e3ce024e60f01998c579c38cd..53a613ea420e484c8f64be41a8aac74c7c345b44 100644
--- a/include/llvm/Support/GenericDomTree.h
+++ b/include/llvm/Support/GenericDomTree.h
@@ -1,9 +1,8 @@
 //===- GenericDomTree.h - Generic dominator trees for graphs ----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/include/llvm/Support/GenericDomTreeConstruction.h b/include/llvm/Support/GenericDomTreeConstruction.h
index 971e8305a112c6c85e82af8ec92d48ea22b6fd80..69cc721203f20607cd1e6b0a9bf2a29eb3d9749d 100644
--- a/include/llvm/Support/GenericDomTreeConstruction.h
+++ b/include/llvm/Support/GenericDomTreeConstruction.h
@@ -1,9 +1,8 @@
 //===- GenericDomTreeConstruction.h - Dominator Calculation ------*- C++ -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/include/llvm/Support/GlobPattern.h b/include/llvm/Support/GlobPattern.h
index c9436a13c1a3d91ec3a31b7f1185ebc5e3dbe7bd..66a4cd94c12ada304d7e9d19ce65737ca08abbce 100644
--- a/include/llvm/Support/GlobPattern.h
+++ b/include/llvm/Support/GlobPattern.h
@@ -1,9 +1,8 @@
 //===-- GlobPattern.h - glob pattern matcher implementation -*- C++ -*-----===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Support/GraphWriter.h b/include/llvm/Support/GraphWriter.h
index 02d98bec16e2b3bb6ab83978782d85b0110a1894..466a0449e2579e9604e7e08d82d3d4fa52c34900 100644
--- a/include/llvm/Support/GraphWriter.h
+++ b/include/llvm/Support/GraphWriter.h
@@ -1,9 +1,8 @@
 //===- llvm/Support/GraphWriter.h - Write graph to a .dot file --*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Support/Host.h b/include/llvm/Support/Host.h
index 57c79c0b9fdf6da0fab339b9174d91cb4870025f..b37cc514c92e2b6a0eaef7b9c44ed0dae5dfc923 100644
--- a/include/llvm/Support/Host.h
+++ b/include/llvm/Support/Host.h
@@ -1,9 +1,8 @@
 //===- llvm/Support/Host.h - Host machine characteristics --------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Support/InitLLVM.h b/include/llvm/Support/InitLLVM.h
index 0f629c9ac92d87673c757235aae3020c5c6fc276..57a0ebce9d459e61dc79f7a48a643b519977367e 100644
--- a/include/llvm/Support/InitLLVM.h
+++ b/include/llvm/Support/InitLLVM.h
@@ -1,9 +1,8 @@
 //===- InitLLVM.h -----------------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/Support/ItaniumManglingCanonicalizer.h b/include/llvm/Support/ItaniumManglingCanonicalizer.h
index 34eb9f7deaaf78ec1489c87aae73b4477e4979d2..6920000340d4f5643ae28b4fd7e1a7b8bfd006b4 100644
--- a/include/llvm/Support/ItaniumManglingCanonicalizer.h
+++ b/include/llvm/Support/ItaniumManglingCanonicalizer.h
@@ -1,9 +1,8 @@
 //===--- ItaniumManglingCanonicalizer.h -------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Support/JSON.h b/include/llvm/Support/JSON.h
index 7a04fd52bc50e4f052f791334823e4113fe0364f..e3cb950663232d0df6aa93bb184da86a6cde3370 100644
--- a/include/llvm/Support/JSON.h
+++ b/include/llvm/Support/JSON.h
@@ -1,9 +1,8 @@
 //===--- JSON.h - JSON values, parsing and serialization -------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===---------------------------------------------------------------------===//
 ///
@@ -481,6 +480,7 @@ private:
   mutable llvm::AlignedCharArrayUnion<bool, double, int64_t, llvm::StringRef,
                                       std::string, json::Array, json::Object>
       Union;
+  friend bool operator==(const Value &, const Value &);
 };
 
 bool operator==(const Value &, const Value &);
diff --git a/include/llvm/Support/JamCRC.h b/include/llvm/Support/JamCRC.h
index 846d6cea98281ab42fc33c66d85e7c02cfaeafb3..b6fc4e7b9b033995a7f66aca66666be3686d3efb 100644
--- a/include/llvm/Support/JamCRC.h
+++ b/include/llvm/Support/JamCRC.h
@@ -1,9 +1,8 @@
 //===-- llvm/Support/JamCRC.h - Cyclic Redundancy Check ---------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Support/KnownBits.h b/include/llvm/Support/KnownBits.h
index 259df9546c5795cb8c9a8cf82a87f724746c5978..8983a06cf48a9bebe21401f1697a85111dc99ee3 100644
--- a/include/llvm/Support/KnownBits.h
+++ b/include/llvm/Support/KnownBits.h
@@ -1,9 +1,8 @@
 //===- llvm/Support/KnownBits.h - Stores known zeros/ones -------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Support/LEB128.h b/include/llvm/Support/LEB128.h
index 9feb07229225845e20154a4e583378fecba311c0..eb43c85e1fb3efb1ee602f7784737c459c4f2aaf 100644
--- a/include/llvm/Support/LEB128.h
+++ b/include/llvm/Support/LEB128.h
@@ -1,9 +1,8 @@
 //===- llvm/Support/LEB128.h - [SU]LEB128 utility functions -----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Support/LineIterator.h b/include/llvm/Support/LineIterator.h
index 892d289976cbc3385dcff6ef9acec3b64c2f2120..c9f10ca975ae7b85c84f93ebe2464d1976943519 100644
--- a/include/llvm/Support/LineIterator.h
+++ b/include/llvm/Support/LineIterator.h
@@ -1,9 +1,8 @@
 //===- LineIterator.h - Iterator to read a text buffer's lines --*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/Support/LockFileManager.h b/include/llvm/Support/LockFileManager.h
index 86db0b2b1020ea964a8755a754bdf0fd3917a662..57e4fbd84cd954630b846d4424a0e91813341119 100644
--- a/include/llvm/Support/LockFileManager.h
+++ b/include/llvm/Support/LockFileManager.h
@@ -1,9 +1,8 @@
 //===--- LockFileManager.h - File-level locking utility ---------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 #ifndef LLVM_SUPPORT_LOCKFILEMANAGER_H
diff --git a/include/llvm/Support/LowLevelTypeImpl.h b/include/llvm/Support/LowLevelTypeImpl.h
index a0a5a52d206e46000e8c0e9c62763cfbe9ed6a7c..efe5c51d1a55f670f3532b62e35913dcfef0ba41 100644
--- a/include/llvm/Support/LowLevelTypeImpl.h
+++ b/include/llvm/Support/LowLevelTypeImpl.h
@@ -1,9 +1,8 @@
 //== llvm/Support/LowLevelTypeImpl.h --------------------------- -*- C++ -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -46,8 +45,8 @@ public:
                SizeInBits, /*AddressSpace=*/0};
   }
 
-  /// Get a low-level pointer in the given address space (defaulting to 0).
-  static LLT pointer(uint16_t AddressSpace, unsigned SizeInBits) {
+  /// Get a low-level pointer in the given address space.
+  static LLT pointer(unsigned AddressSpace, unsigned SizeInBits) {
     assert(SizeInBits > 0 && "invalid pointer size");
     return LLT{/*isPointer=*/true, /*isVector=*/false, /*NumElements=*/0,
                SizeInBits, AddressSpace};
@@ -71,6 +70,14 @@ public:
                ScalarTy.isPointer() ? ScalarTy.getAddressSpace() : 0};
   }
 
+  static LLT scalarOrVector(uint16_t NumElements, LLT ScalarTy) {
+    return NumElements == 1 ? ScalarTy : LLT::vector(NumElements, ScalarTy);
+  }
+
+  static LLT scalarOrVector(uint16_t NumElements, unsigned ScalarSize) {
+    return scalarOrVector(NumElements, LLT::scalar(ScalarSize));
+  }
+
   explicit LLT(bool isPointer, bool isVector, uint16_t NumElements,
                unsigned SizeInBits, unsigned AddressSpace) {
     init(isPointer, isVector, NumElements, SizeInBits, AddressSpace);
@@ -104,6 +111,26 @@ public:
     return getScalarSizeInBits() * getNumElements();
   }
 
+  LLT getScalarType() const {
+    return isVector() ? getElementType() : *this;
+  }
+
+  /// If this type is a vector, return a vector with the same number of elements
+  /// but the new element type. Otherwise, return the new element type.
+  LLT changeElementType(LLT NewEltTy) const {
+    return isVector() ? LLT::vector(getNumElements(), NewEltTy) : NewEltTy;
+  }
+
+  /// If this type is a vector, return a vector with the same number of elements
+  /// but the new element size. Otherwise, return the new element type. Invalid
+  /// for pointer types. For pointer types, use changeElementType.
+  LLT changeElementSize(unsigned NewEltSize) const {
+    assert(!getScalarType().isPointer() &&
+           "invalid to directly change element size for pointers");
+    return isVector() ? LLT::vector(getNumElements(), NewEltSize)
+                      : LLT::scalar(NewEltSize);
+  }
+
   unsigned getScalarSizeInBits() const {
     assert(RawData != 0 && "Invalid Type");
     if (!IsVector) {
@@ -147,6 +174,7 @@ public:
   bool operator!=(const LLT &RHS) const { return !(*this == RHS); }
 
   friend struct DenseMapInfo<LLT>;
+  friend class GISelInstProfileBuilder;
 
 private:
   /// LLT is packed into 64 bits as follows:
@@ -169,10 +197,10 @@ private:
   static const constexpr BitFieldInfo ScalarSizeFieldInfo{32, 0};
   /// * Pointer (isPointer == 1 && isVector == 0):
   ///   SizeInBits: 16;
-  ///   AddressSpace: 23;
+  ///   AddressSpace: 24;
   static const constexpr BitFieldInfo PointerSizeFieldInfo{16, 0};
   static const constexpr BitFieldInfo PointerAddressSpaceFieldInfo{
-      23, PointerSizeFieldInfo[0] + PointerSizeFieldInfo[1]};
+      24, PointerSizeFieldInfo[0] + PointerSizeFieldInfo[1]};
   /// * Vector-of-non-pointer (isPointer == 0 && isVector == 1):
   ///   NumElements: 16;
   ///   SizeOfElement: 32;
@@ -182,13 +210,13 @@ private:
   /// * Vector-of-pointer (isPointer == 1 && isVector == 1):
   ///   NumElements: 16;
   ///   SizeOfElement: 16;
-  ///   AddressSpace: 23;
+  ///   AddressSpace: 24;
   static const constexpr BitFieldInfo PointerVectorElementsFieldInfo{16, 0};
   static const constexpr BitFieldInfo PointerVectorSizeFieldInfo{
       16,
       PointerVectorElementsFieldInfo[1] + PointerVectorElementsFieldInfo[0]};
   static const constexpr BitFieldInfo PointerVectorAddressSpaceFieldInfo{
-      23, PointerVectorSizeFieldInfo[1] + PointerVectorSizeFieldInfo[0]};
+      24, PointerVectorSizeFieldInfo[1] + PointerVectorSizeFieldInfo[0]};
 
   uint64_t IsPointer : 1;
   uint64_t IsVector : 1;
@@ -231,6 +259,11 @@ private:
             maskAndShift(AddressSpace, PointerVectorAddressSpaceFieldInfo);
     }
   }
+
+  uint64_t getUniqueRAWLLTData() const {
+    return ((uint64_t)RawData) << 2 | ((uint64_t)IsPointer) << 1 |
+           ((uint64_t)IsVector);
+  }
 };
 
 inline raw_ostream& operator<<(raw_ostream &OS, const LLT &Ty) {
@@ -250,8 +283,7 @@ template<> struct DenseMapInfo<LLT> {
     return Invalid;
   }
   static inline unsigned getHashValue(const LLT &Ty) {
-    uint64_t Val = ((uint64_t)Ty.RawData) << 2 | ((uint64_t)Ty.IsPointer) << 1 |
-                   ((uint64_t)Ty.IsVector);
+    uint64_t Val = Ty.getUniqueRAWLLTData();
     return DenseMapInfo<uint64_t>::getHashValue(Val);
   }
   static bool isEqual(const LLT &LHS, const LLT &RHS) {
diff --git a/include/llvm/Support/MSVCErrorWorkarounds.h b/include/llvm/Support/MSVCErrorWorkarounds.h
index 053ecf64d1e9d200cdfc9e58c5b38c545a6504e5..30e8febae20b9a91db0a02678feea245b9c166d2 100644
--- a/include/llvm/Support/MSVCErrorWorkarounds.h
+++ b/include/llvm/Support/MSVCErrorWorkarounds.h
@@ -1,9 +1,8 @@
 //===--- MSVCErrorWorkarounds.h - Enable future<Error> in MSVC --*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Support/MachineValueType.h b/include/llvm/Support/MachineValueType.h
index 552dea05029c9a7d5bf90e6e0d00a32d85dbf434..7651b28a63380cdf2c1847ea0efcf0c0f4342b0d 100644
--- a/include/llvm/Support/MachineValueType.h
+++ b/include/llvm/Support/MachineValueType.h
@@ -1,9 +1,8 @@
 //===- Support/MachineValueType.h - Machine-Level types ---------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Support/ManagedStatic.h b/include/llvm/Support/ManagedStatic.h
index b4bf3210cc7384423749ec8e5379871cfabc3491..441f24e05afc589e597aa288737e060edcdd0699 100644
--- a/include/llvm/Support/ManagedStatic.h
+++ b/include/llvm/Support/ManagedStatic.h
@@ -1,9 +1,8 @@
 //===-- llvm/Support/ManagedStatic.h - Static Global wrapper ----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Support/MathExtras.h b/include/llvm/Support/MathExtras.h
index b59f21b4998e3d4663de53249cbcee6ca75b7397..e902a725659adb690d8bc8e0073b83d309914173 100644
--- a/include/llvm/Support/MathExtras.h
+++ b/include/llvm/Support/MathExtras.h
@@ -1,9 +1,8 @@
 //===-- llvm/Support/MathExtras.h - Useful math functions -------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Support/MemAlloc.h b/include/llvm/Support/MemAlloc.h
index d06c659cfba6b4c0cede4557517f5f2bb605c8ea..ed076ffa52d6012c42f7fbf487aae59571ba7693 100644
--- a/include/llvm/Support/MemAlloc.h
+++ b/include/llvm/Support/MemAlloc.h
@@ -1,9 +1,8 @@
 //===- MemAlloc.h - Memory allocation functions -----------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/include/llvm/Support/Memory.h b/include/llvm/Support/Memory.h
index fa026d49a61b6c689e465f76c57f680a6ef23b3d..2f4cafe40c805bdc77f4581ed877b5cbea365647 100644
--- a/include/llvm/Support/Memory.h
+++ b/include/llvm/Support/Memory.h
@@ -1,9 +1,8 @@
 //===- llvm/Support/Memory.h - Memory Support -------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Support/MemoryBuffer.h b/include/llvm/Support/MemoryBuffer.h
index 8933295d4ea4eeebc2c1bd74d1b1c328884f20a9..44052697d06c662b08b87c6a306fb509268ad907 100644
--- a/include/llvm/Support/MemoryBuffer.h
+++ b/include/llvm/Support/MemoryBuffer.h
@@ -1,9 +1,8 @@
 //===--- MemoryBuffer.h - Memory Buffer Interface ---------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Support/MipsABIFlags.h b/include/llvm/Support/MipsABIFlags.h
index 12c350015b2109962cf7304fddfe2e3f415f0a5f..d3233f645fb9aca50e1157fa46722e513754f4bb 100644
--- a/include/llvm/Support/MipsABIFlags.h
+++ b/include/llvm/Support/MipsABIFlags.h
@@ -1,9 +1,8 @@
 //===--- MipsABIFlags.h - MIPS ABI flags ----------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Support/Mutex.h b/include/llvm/Support/Mutex.h
index 680d94b24ef517d2a0e4bd886ebbced9893b68c5..c3abfc7a78063891ece4f0f05679bd1da8edbf33 100644
--- a/include/llvm/Support/Mutex.h
+++ b/include/llvm/Support/Mutex.h
@@ -1,9 +1,8 @@
 //===- llvm/Support/Mutex.h - Mutex Operating System Concept -----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Support/MutexGuard.h b/include/llvm/Support/MutexGuard.h
index 641d64d949880df08606936e1ee985221159fd11..d86ced145816cd02b06ea98c090002afebc4cd27 100644
--- a/include/llvm/Support/MutexGuard.h
+++ b/include/llvm/Support/MutexGuard.h
@@ -1,9 +1,8 @@
 //===-- Support/MutexGuard.h - Acquire/Release Mutex In Scope ---*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Support/NativeFormatting.h b/include/llvm/Support/NativeFormatting.h
index 6d1dd7b422fe6199dad8d365416720f5684a12e5..825a44c77c0016047d82d06aefe4e2d150b1cea7 100644
--- a/include/llvm/Support/NativeFormatting.h
+++ b/include/llvm/Support/NativeFormatting.h
@@ -1,9 +1,8 @@
 //===- NativeFormatting.h - Low level formatting helpers ---------*- C++-*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/Support/OnDiskHashTable.h b/include/llvm/Support/OnDiskHashTable.h
index 912e2700d1a01e4994eedb05db038fc3d594295d..d84da92aab9b098e73f2b46b25ce40f02eac7007 100644
--- a/include/llvm/Support/OnDiskHashTable.h
+++ b/include/llvm/Support/OnDiskHashTable.h
@@ -1,9 +1,8 @@
 //===--- OnDiskHashTable.h - On-Disk Hash Table Implementation --*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
diff --git a/include/llvm/Support/Options.h b/include/llvm/Support/Options.h
index dd321c6a19840795c0ce99b487b40453b9ed13c7..d02ef85a75bf181d2b5f07eb1bdac8e2dc897e3b 100644
--- a/include/llvm/Support/Options.h
+++ b/include/llvm/Support/Options.h
@@ -1,9 +1,8 @@
 //===- llvm/Support/Options.h - Debug options support -----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/include/llvm/Support/Parallel.h b/include/llvm/Support/Parallel.h
index 1462265343be786b9ec3f80f68358bd0eb4be4d2..9843b9518cd53cf8864014bb226264fad27df8b9 100644
--- a/include/llvm/Support/Parallel.h
+++ b/include/llvm/Support/Parallel.h
@@ -1,9 +1,8 @@
 //===- llvm/Support/Parallel.h - Parallel algorithms ----------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/Support/Path.h b/include/llvm/Support/Path.h
index 76de887b7cb4edd616ea6e241238d65f7ca655e5..5c0bee58f188ab22182c787b5d74905b1f9c6d52 100644
--- a/include/llvm/Support/Path.h
+++ b/include/llvm/Support/Path.h
@@ -1,9 +1,8 @@
 //===- llvm/Support/Path.h - Path Operating System Concept ------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Support/PluginLoader.h b/include/llvm/Support/PluginLoader.h
index bdbb134b28ebba5066cd7b4cf5bed255dc6d36d4..c0c516bdae03e56e468d5fc6f2f572af57de4c9b 100644
--- a/include/llvm/Support/PluginLoader.h
+++ b/include/llvm/Support/PluginLoader.h
@@ -1,9 +1,8 @@
 //===-- llvm/Support/PluginLoader.h - Plugin Loader for Tools ---*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Support/PointerLikeTypeTraits.h b/include/llvm/Support/PointerLikeTypeTraits.h
index 1710b57131d1252134dbcdbcf5bff051a6bb8c34..1e7e5b53ca6570d0254df71a62eb9b29e98049a6 100644
--- a/include/llvm/Support/PointerLikeTypeTraits.h
+++ b/include/llvm/Support/PointerLikeTypeTraits.h
@@ -1,9 +1,8 @@
 //===- llvm/Support/PointerLikeTypeTraits.h - Pointer Traits ----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Support/PrettyStackTrace.h b/include/llvm/Support/PrettyStackTrace.h
index 4d64fe4ef727cb89b106ffaad5a2fdbdc87d1fe6..f6ed7c94ba31004214ac1a861ee5e0ddb760e326 100644
--- a/include/llvm/Support/PrettyStackTrace.h
+++ b/include/llvm/Support/PrettyStackTrace.h
@@ -1,9 +1,8 @@
 //===- llvm/Support/PrettyStackTrace.h - Pretty Crash Handling --*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Support/Printable.h b/include/llvm/Support/Printable.h
index cb55d41316e3f3c9e0f5a54798bead929e7e2c96..0f8670d0419c7689537a0ccddcc0b4ecc8d3eecf 100644
--- a/include/llvm/Support/Printable.h
+++ b/include/llvm/Support/Printable.h
@@ -1,9 +1,8 @@
 //===--- Printable.h - Print function helpers -------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Support/Process.h b/include/llvm/Support/Process.h
index f9f1cac862789ba383a1f3bd4b8bcc7ed0ede052..9a70304f5c7018fc19cc16e81131429bb914847f 100644
--- a/include/llvm/Support/Process.h
+++ b/include/llvm/Support/Process.h
@@ -1,9 +1,8 @@
 //===- llvm/Support/Process.h -----------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/include/llvm/Support/Program.h b/include/llvm/Support/Program.h
index 1f4dbdce332389ff19912c315324b6127d325976..6b2315c5da8d0ff04a2ca9dff221013dcd6a677a 100644
--- a/include/llvm/Support/Program.h
+++ b/include/llvm/Support/Program.h
@@ -1,9 +1,8 @@
 //===- llvm/Support/Program.h ------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Support/RWMutex.h b/include/llvm/Support/RWMutex.h
index 5ac3e558999b9ecb9e44853461afeb4d36ce7384..9cd57cbd65a1da5649212908c2095397e7252fd7 100644
--- a/include/llvm/Support/RWMutex.h
+++ b/include/llvm/Support/RWMutex.h
@@ -1,9 +1,8 @@
 //===- RWMutex.h - Reader/Writer Mutual Exclusion Lock ----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Support/RandomNumberGenerator.h b/include/llvm/Support/RandomNumberGenerator.h
index 1399dab815f83a8f21e63e81fe39bb0f0b23fb1d..55d6876cc5e41f45d4eabe44c07656b7265fe74b 100644
--- a/include/llvm/Support/RandomNumberGenerator.h
+++ b/include/llvm/Support/RandomNumberGenerator.h
@@ -1,9 +1,8 @@
 //==- llvm/Support/RandomNumberGenerator.h - RNG for diversity ---*- C++ -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Support/Recycler.h b/include/llvm/Support/Recycler.h
index 53db2e86d12d3da1489433fd9074b0b667240007..bbd9ae321ae30c6eda22349585882c586f75644c 100644
--- a/include/llvm/Support/Recycler.h
+++ b/include/llvm/Support/Recycler.h
@@ -1,9 +1,8 @@
 //==- llvm/Support/Recycler.h - Recycling Allocator --------------*- C++ -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Support/RecyclingAllocator.h b/include/llvm/Support/RecyclingAllocator.h
index 32b033b17946385f5a9585c5b37686fa52e7874c..2c29dacfe21272df5f5a2606a2f22973202c3245 100644
--- a/include/llvm/Support/RecyclingAllocator.h
+++ b/include/llvm/Support/RecyclingAllocator.h
@@ -1,9 +1,8 @@
 //==- llvm/Support/RecyclingAllocator.h - Recycling Allocator ----*- C++ -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Support/Regex.h b/include/llvm/Support/Regex.h
index d901eb1e3ffba8bce1b6f3010390cf7269f9d26c..2d19b10fd890762283038e8e0f868e4d4d1f6839 100644
--- a/include/llvm/Support/Regex.h
+++ b/include/llvm/Support/Regex.h
@@ -1,9 +1,8 @@
 //===-- Regex.h - Regular Expression matcher implementation -*- C++ -*-----===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Support/Registry.h b/include/llvm/Support/Registry.h
index 02fd5b9354a1af873cd824dd77f8918e61f0a709..4d8aa5f1470df076e2c8e0743ef1b872fc125ae5 100644
--- a/include/llvm/Support/Registry.h
+++ b/include/llvm/Support/Registry.h
@@ -1,9 +1,8 @@
 //=== Registry.h - Linker-supported plugin registries -----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -81,17 +80,17 @@ namespace llvm {
 
     /// Iterators for registry entries.
     ///
-    class iterator {
+    class iterator
+        : public llvm::iterator_facade_base<iterator, std::forward_iterator_tag,
+                                            const entry> {
       const node *Cur;
 
     public:
       explicit iterator(const node *N) : Cur(N) {}
 
       bool operator==(const iterator &That) const { return Cur == That.Cur; }
-      bool operator!=(const iterator &That) const { return Cur != That.Cur; }
       iterator &operator++() { Cur = Cur->Next; return *this; }
       const entry &operator*() const { return Cur->Val; }
-      const entry *operator->() const { return &Cur->Val; }
     };
 
     // begin is not defined here in order to avoid usage of an undefined static
diff --git a/include/llvm/Support/SHA1.h b/include/llvm/Support/SHA1.h
index 1fc60a878f945cf4da0497cecdcfb260f37eda6d..87fe94bbd5cd36d0e183a0d4a93cdb8f2d1ed178 100644
--- a/include/llvm/Support/SHA1.h
+++ b/include/llvm/Support/SHA1.h
@@ -1,9 +1,8 @@
 //==- SHA1.h - SHA1 implementation for LLVM                     --*- C++ -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 // This code is taken from public domain
diff --git a/include/llvm/Support/SMLoc.h b/include/llvm/Support/SMLoc.h
index c74feff378d63d7fa8bf9dc81749f7629e796c15..d8607034ee86d90a81f47d5e87a48d8b81f71066 100644
--- a/include/llvm/Support/SMLoc.h
+++ b/include/llvm/Support/SMLoc.h
@@ -1,9 +1,8 @@
 //===- SMLoc.h - Source location for use with diagnostics -------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Support/SaveAndRestore.h b/include/llvm/Support/SaveAndRestore.h
index 8e11789907ad0a240ad780910c45cb4663990956..3c0333b7119a6f3b6c36bda3f46dfedb0ce5d171 100644
--- a/include/llvm/Support/SaveAndRestore.h
+++ b/include/llvm/Support/SaveAndRestore.h
@@ -1,9 +1,8 @@
 //===-- SaveAndRestore.h - Utility  -------------------------------*- C++ -*-=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
diff --git a/include/llvm/Support/ScaledNumber.h b/include/llvm/Support/ScaledNumber.h
index 3bd3ccedc42cfe2461698ccece8143f863ed4cbf..552da34f357b78c54f90a2bba2d1387040c931d7 100644
--- a/include/llvm/Support/ScaledNumber.h
+++ b/include/llvm/Support/ScaledNumber.h
@@ -1,9 +1,8 @@
 //===- llvm/Support/ScaledNumber.h - Support for scaled numbers -*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -887,10 +886,6 @@ template <class DigitsT> void ScaledNumber<DigitsT>::shiftRight(int32_t Shift) {
   Digits >>= Shift;
 }
 
-template <typename T> struct isPodLike;
-template <typename T> struct isPodLike<ScaledNumber<T>> {
-  static const bool value = true;
-};
 
 } // end namespace llvm
 
diff --git a/include/llvm/Support/ScopedPrinter.h b/include/llvm/Support/ScopedPrinter.h
index 34c1a287ee10607544d4eb4ef7f299ce88825dbe..169965531b01cf60d9b43c54f54d801d9407b7f7 100644
--- a/include/llvm/Support/ScopedPrinter.h
+++ b/include/llvm/Support/ScopedPrinter.h
@@ -1,9 +1,8 @@
 //===-- ScopedPrinter.h ---------------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/Support/Signals.h b/include/llvm/Support/Signals.h
index f25a049699040d06be85ce3f6fd6748117293e2b..9f2cb850c4680ebd81d0e207a1951d43dbccfeb1 100644
--- a/include/llvm/Support/Signals.h
+++ b/include/llvm/Support/Signals.h
@@ -1,9 +1,8 @@
 //===- llvm/Support/Signals.h - Signal Handling support ----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Support/SmallVectorMemoryBuffer.h b/include/llvm/Support/SmallVectorMemoryBuffer.h
index c4a600e7f37ddea9e95efc788adcaee5eb3aebd5..b63b58e3a8ba9bf010012bd4858b25d8f1efc013 100644
--- a/include/llvm/Support/SmallVectorMemoryBuffer.h
+++ b/include/llvm/Support/SmallVectorMemoryBuffer.h
@@ -1,9 +1,8 @@
 //===- SmallVectorMemoryBuffer.h --------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Support/Solaris/sys/regset.h b/include/llvm/Support/Solaris/sys/regset.h
index 6a69ebe718a19173620578a5fc310a7252f4fb65..6bd98fa826a6444932fe16a7b61145df4aa2233d 100644
--- a/include/llvm/Support/Solaris/sys/regset.h
+++ b/include/llvm/Support/Solaris/sys/regset.h
@@ -1,9 +1,8 @@
 /*===- llvm/Support/Solaris/sys/regset.h ------------------------*- C++ -*-===*
  *
- *                     The LLVM Compiler Infrastructure
- *
- * This file is distributed under the University of Illinois Open Source
- * License. See LICENSE.TXT for details.
+ * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+ * See https://llvm.org/LICENSE.txt for license information.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
  *
  *===----------------------------------------------------------------------===*
  *
diff --git a/include/llvm/Support/SourceMgr.h b/include/llvm/Support/SourceMgr.h
index 63ac893239d170f9dcdc1a221fd941560d01fabd..7b081d32f99e531c0882c1a2c526788b58ac3596 100644
--- a/include/llvm/Support/SourceMgr.h
+++ b/include/llvm/Support/SourceMgr.h
@@ -1,9 +1,8 @@
 //===- SourceMgr.h - Manager for Source Buffers & Diagnostics ---*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Support/SpecialCaseList.h b/include/llvm/Support/SpecialCaseList.h
index fd62fc48047bad1e2880f5be4d8ae2568ccaca93..b7400266f4dfa61f42c5e088d612549cee0dd222 100644
--- a/include/llvm/Support/SpecialCaseList.h
+++ b/include/llvm/Support/SpecialCaseList.h
@@ -1,9 +1,8 @@
 //===-- SpecialCaseList.h - special case list for sanitizers ----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //===----------------------------------------------------------------------===//
 //
 // This is a utility class used to parse user-provided text files with
diff --git a/include/llvm/Support/StringPool.h b/include/llvm/Support/StringPool.h
index bb5fd07f0d00954e0c68b5756c0b81d60e4655fa..a4f45916f53d600e561d40cd979ff0410c50a4ca 100644
--- a/include/llvm/Support/StringPool.h
+++ b/include/llvm/Support/StringPool.h
@@ -1,9 +1,8 @@
 //===- StringPool.h - Interned string pool ----------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Support/StringSaver.h b/include/llvm/Support/StringSaver.h
index 6b77d487333b4a5e68edc64712bffc9f7c1a1fb1..c54044e3986ca64579d7663beb81e315bd72d864 100644
--- a/include/llvm/Support/StringSaver.h
+++ b/include/llvm/Support/StringSaver.h
@@ -1,9 +1,8 @@
 //===- llvm/Support/StringSaver.h -------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/Support/SwapByteOrder.h b/include/llvm/Support/SwapByteOrder.h
index 71d3724950abb7461bde135912d286bcc0a5aa75..f58b4d911d4db959351dca3ba972cbb75cfd7086 100644
--- a/include/llvm/Support/SwapByteOrder.h
+++ b/include/llvm/Support/SwapByteOrder.h
@@ -1,9 +1,8 @@
 //===- SwapByteOrder.h - Generic and optimized byte swaps -------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Support/SymbolRemappingReader.h b/include/llvm/Support/SymbolRemappingReader.h
index b457b9e817e42f03717bb2dc434ec7095c7716bf..2b9ab570eb8b3710021651ba78254250e787b53e 100644
--- a/include/llvm/Support/SymbolRemappingReader.h
+++ b/include/llvm/Support/SymbolRemappingReader.h
@@ -1,9 +1,8 @@
 //===- SymbolRemappingReader.h - Read symbol remapping file -----*- C++ -*-===//
 //
-//                      The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Support/SystemUtils.h b/include/llvm/Support/SystemUtils.h
index bd60793d15545c6f1ad28cc481421ca9a278611f..77deddb9ee1c23da97d160a49cb0b4a05a1dcb33 100644
--- a/include/llvm/Support/SystemUtils.h
+++ b/include/llvm/Support/SystemUtils.h
@@ -1,9 +1,8 @@
 //===- SystemUtils.h - Utilities to do low-level system stuff ---*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Support/TarWriter.h b/include/llvm/Support/TarWriter.h
index 639f61b538922312398ddd1b35897655c7b02444..71164e2ef961d1c56141e13f49db7198825d1384 100644
--- a/include/llvm/Support/TarWriter.h
+++ b/include/llvm/Support/TarWriter.h
@@ -1,9 +1,8 @@
 //===-- llvm/Support/TarWriter.h - Tar archive file creator -----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/Support/TargetOpcodes.def b/include/llvm/Support/TargetOpcodes.def
index 3245276bbf362bb1b345aa2a4dd7cf93aa04b830..661227492784fbfed04124df730a05c26ef36cbd 100644
--- a/include/llvm/Support/TargetOpcodes.def
+++ b/include/llvm/Support/TargetOpcodes.def
@@ -1,9 +1,8 @@
 //===-- llvm/Support/TargetOpcodes.def - Target Indep Opcodes ---*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -29,6 +28,7 @@
 ///
 HANDLE_TARGET_OPCODE(PHI)
 HANDLE_TARGET_OPCODE(INLINEASM)
+HANDLE_TARGET_OPCODE(INLINEASM_BR)
 HANDLE_TARGET_OPCODE(CFI_INSTRUCTION)
 HANDLE_TARGET_OPCODE(EH_LABEL)
 HANDLE_TARGET_OPCODE(GC_LABEL)
@@ -481,6 +481,9 @@ HANDLE_TARGET_OPCODE(G_UITOFP)
 /// Generic FP absolute value.
 HANDLE_TARGET_OPCODE(G_FABS)
 
+/// Generic FP canonicalize value.
+HANDLE_TARGET_OPCODE(G_FCANONICALIZE)
+
 /// Generic pointer offset
 HANDLE_TARGET_OPCODE(G_GEP)
 
@@ -518,6 +521,21 @@ HANDLE_TARGET_OPCODE(G_CTPOP)
 /// Generic byte swap.
 HANDLE_TARGET_OPCODE(G_BSWAP)
 
+/// Floating point ceil.
+HANDLE_TARGET_OPCODE(G_FCEIL)
+
+/// Floating point cosine.
+HANDLE_TARGET_OPCODE(G_FCOS)
+
+/// Floating point sine.
+HANDLE_TARGET_OPCODE(G_FSIN)
+
+/// Floating point square root.
+HANDLE_TARGET_OPCODE(G_FSQRT)
+
+/// Floating point floor.
+HANDLE_TARGET_OPCODE(G_FFLOOR)
+
 /// Generic AddressSpaceCast.
 HANDLE_TARGET_OPCODE(G_ADDRSPACE_CAST)
 
diff --git a/include/llvm/Support/TargetParser.h b/include/llvm/Support/TargetParser.h
index ace11ed410a38fafe24b2ceab67e31028163eed7..7efd022219e45fb4795619c3cb9d2929a9153034 100644
--- a/include/llvm/Support/TargetParser.h
+++ b/include/llvm/Support/TargetParser.h
@@ -1,9 +1,8 @@
 //===-- TargetParser - Parser for target features ---------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Support/TargetRegistry.h b/include/llvm/Support/TargetRegistry.h
index 1bafc4e687da1227748bb2c063c6bfab52675edd..5854d57fba933c768f21844fe7474eecbc22afff 100644
--- a/include/llvm/Support/TargetRegistry.h
+++ b/include/llvm/Support/TargetRegistry.h
@@ -1,9 +1,8 @@
 //===- Support/TargetRegistry.h - Target Registration -----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Support/TargetSelect.h b/include/llvm/Support/TargetSelect.h
index 582785cb69a554a61becd51e597b86f68b35395b..9ffb84c4a5701b4f2540c53aab29c79320d9d0fc 100644
--- a/include/llvm/Support/TargetSelect.h
+++ b/include/llvm/Support/TargetSelect.h
@@ -1,9 +1,8 @@
 //===- TargetSelect.h - Target Selection & Registration ---------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Support/TaskQueue.h b/include/llvm/Support/TaskQueue.h
index 49981adb763d546162f197cf504d2ae87f5c2077..df2ffdee2cc24ad6f417e8181b2883de5cf8a0f5 100644
--- a/include/llvm/Support/TaskQueue.h
+++ b/include/llvm/Support/TaskQueue.h
@@ -1,9 +1,8 @@
 //===-- llvm/Support/TaskQueue.h - A TaskQueue implementation ---*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Support/ThreadLocal.h b/include/llvm/Support/ThreadLocal.h
index 885bd18e83563d1a94b5bccf3d492fba3732f5cf..d6838c15fc3454b347823262a79acc3b1878be2d 100644
--- a/include/llvm/Support/ThreadLocal.h
+++ b/include/llvm/Support/ThreadLocal.h
@@ -1,9 +1,8 @@
 //===- llvm/Support/ThreadLocal.h - Thread Local Data ------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Support/ThreadPool.h b/include/llvm/Support/ThreadPool.h
index 4fdbd528b2127ae50803b39a1cd85f5cb26e7d80..4bcbaa3142fd4f1c2d9b17d827a3bf9b7163ade8 100644
--- a/include/llvm/Support/ThreadPool.h
+++ b/include/llvm/Support/ThreadPool.h
@@ -1,9 +1,8 @@
 //===-- llvm/Support/ThreadPool.h - A ThreadPool implementation -*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Support/Threading.h b/include/llvm/Support/Threading.h
index ba7ece5e72ba78428eddb003a0e296e908fe2657..23b115f0b47816a988198660bc0b96827a28ef16 100644
--- a/include/llvm/Support/Threading.h
+++ b/include/llvm/Support/Threading.h
@@ -1,9 +1,8 @@
 //===-- llvm/Support/Threading.h - Control multithreading mode --*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Support/Timer.h b/include/llvm/Support/Timer.h
index a11c3ce3ff2247e2543a477b437261a5cf5c9037..09f742249000fd92ff3fe283c99ec25abe9f5915 100644
--- a/include/llvm/Support/Timer.h
+++ b/include/llvm/Support/Timer.h
@@ -1,9 +1,8 @@
 //===-- llvm/Support/Timer.h - Interval Timing Support ----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/Support/ToolOutputFile.h b/include/llvm/Support/ToolOutputFile.h
index cf3bc2fb0171a4b94809c1b6271fab4e0e465cbe..a99e327f8db7040263c15b2ee63a621b32c06ff7 100644
--- a/include/llvm/Support/ToolOutputFile.h
+++ b/include/llvm/Support/ToolOutputFile.h
@@ -1,9 +1,8 @@
 //===- ToolOutputFile.h - Output files for compiler-like tools -----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Support/TrailingObjects.h b/include/llvm/Support/TrailingObjects.h
index 490bd94f4cd5685fda33bf48d4ccb98da2338180..8cf4f7aed7f890886595267a2f8ee642c3f72d8b 100644
--- a/include/llvm/Support/TrailingObjects.h
+++ b/include/llvm/Support/TrailingObjects.h
@@ -1,9 +1,8 @@
 //===--- TrailingObjects.h - Variable-length classes ------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
diff --git a/include/llvm/Support/TrigramIndex.h b/include/llvm/Support/TrigramIndex.h
index da0b6daf47ed0ef037578e83daad5754cf61028e..9351c2db169a7b732ac75a063fea5ce044d2428b 100644
--- a/include/llvm/Support/TrigramIndex.h
+++ b/include/llvm/Support/TrigramIndex.h
@@ -1,9 +1,8 @@
 //===-- TrigramIndex.h - a heuristic for SpecialCaseList --------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //===----------------------------------------------------------------------===//
 //
 // TrigramIndex implements a heuristic for SpecialCaseList that allows to
diff --git a/include/llvm/Support/TypeName.h b/include/llvm/Support/TypeName.h
index 0eb7ead98b21f0e25271435e6a4ae0a5133cf264..236490a250119136f1e4c2479ee7cfa93dbfcf75 100644
--- a/include/llvm/Support/TypeName.h
+++ b/include/llvm/Support/TypeName.h
@@ -1,9 +1,8 @@
 //===- TypeName.h -----------------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/Support/Unicode.h b/include/llvm/Support/Unicode.h
index 983acaf03635606e389564afda4b36a96b1bc285..ca17bba2fbb42d9165dbfd2724ae28ffcc1fd01e 100644
--- a/include/llvm/Support/Unicode.h
+++ b/include/llvm/Support/Unicode.h
@@ -1,9 +1,8 @@
 //===- llvm/Support/Unicode.h - Unicode character properties  -*- C++ -*-=====//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Support/UnicodeCharRanges.h b/include/llvm/Support/UnicodeCharRanges.h
index 3cf4a6d96602829afbbe6b410e62c2ef7c7d5b0a..4b59f8a92b769b6207c1076b270620a6360f82a1 100644
--- a/include/llvm/Support/UnicodeCharRanges.h
+++ b/include/llvm/Support/UnicodeCharRanges.h
@@ -1,9 +1,8 @@
 //===--- UnicodeCharRanges.h - Types and functions for character ranges ---===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 #ifndef LLVM_SUPPORT_UNICODECHARRANGES_H
diff --git a/include/llvm/Support/UniqueLock.h b/include/llvm/Support/UniqueLock.h
index 91dc911036d5f74db0162e2ef2932ed228201696..0a887ad5965da403a2e7482cb6c492b34c4a5aa2 100644
--- a/include/llvm/Support/UniqueLock.h
+++ b/include/llvm/Support/UniqueLock.h
@@ -1,9 +1,8 @@
 //===- Support/UniqueLock.h - Acquire/Release Mutex In Scope ----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Support/Valgrind.h b/include/llvm/Support/Valgrind.h
index 084b901b326c10b20b582420281ce652eed87772..1e14dfec9a6186cff4b71aca3d0766c8ab3ed204 100644
--- a/include/llvm/Support/Valgrind.h
+++ b/include/llvm/Support/Valgrind.h
@@ -1,9 +1,8 @@
 //===- llvm/Support/Valgrind.h - Communication with Valgrind ----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Support/VersionTuple.h b/include/llvm/Support/VersionTuple.h
index e85a188e54b45c9075bbb1b47a504c3bdc3b4f74..14736d6b28f0537e927475e6f42f5c6d11690928 100644
--- a/include/llvm/Support/VersionTuple.h
+++ b/include/llvm/Support/VersionTuple.h
@@ -1,9 +1,8 @@
 //===- VersionTuple.h - Version Number Handling -----------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
diff --git a/include/llvm/Support/VirtualFileSystem.h b/include/llvm/Support/VirtualFileSystem.h
index 3d4094eeb11106c13b27dadbdd7611028b124fe6..7b7c9f865ace9706c61094c5b6070a8eecdecabf 100644
--- a/include/llvm/Support/VirtualFileSystem.h
+++ b/include/llvm/Support/VirtualFileSystem.h
@@ -1,9 +1,8 @@
 //===- VirtualFileSystem.h - Virtual File System Layer ----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -24,6 +23,7 @@
 #include "llvm/Support/Chrono.h"
 #include "llvm/Support/ErrorOr.h"
 #include "llvm/Support/FileSystem.h"
+#include "llvm/Support/Path.h"
 #include "llvm/Support/SourceMgr.h"
 #include <cassert>
 #include <cstdint>
@@ -383,6 +383,8 @@ protected:
 
 private:
   IntrusiveRefCntPtr<FileSystem> FS;
+
+  virtual void anchor();
 };
 
 namespace detail {
@@ -493,6 +495,230 @@ struct YAMLVFSEntry {
   std::string RPath;
 };
 
+class VFSFromYamlDirIterImpl;
+class RedirectingFileSystemParser;
+
+/// A virtual file system parsed from a YAML file.
+///
+/// Currently, this class allows creating virtual directories and mapping
+/// virtual file paths to existing external files, available in \c ExternalFS.
+///
+/// The basic structure of the parsed file is:
+/// \verbatim
+/// {
+///   'version': <version number>,
+///   <optional configuration>
+///   'roots': [
+///              <directory entries>
+///            ]
+/// }
+/// \endverbatim
+///
+/// All configuration options are optional.
+///   'case-sensitive': <boolean, default=true>
+///   'use-external-names': <boolean, default=true>
+///   'overlay-relative': <boolean, default=false>
+///   'fallthrough': <boolean, default=true>
+///
+/// Virtual directories are represented as
+/// \verbatim
+/// {
+///   'type': 'directory',
+///   'name': <string>,
+///   'contents': [ <file or directory entries> ]
+/// }
+/// \endverbatim
+///
+/// The default attributes for virtual directories are:
+/// \verbatim
+/// MTime = now() when created
+/// Perms = 0777
+/// User = Group = 0
+/// Size = 0
+/// UniqueID = unspecified unique value
+/// \endverbatim
+///
+/// Re-mapped files are represented as
+/// \verbatim
+/// {
+///   'type': 'file',
+///   'name': <string>,
+///   'use-external-name': <boolean> # Optional
+///   'external-contents': <path to external file>
+/// }
+/// \endverbatim
+///
+/// and inherit their attributes from the external contents.
+///
+/// In both cases, the 'name' field may contain multiple path components (e.g.
+/// /path/to/file). However, any directory that contains more than one child
+/// must be uniquely represented by a directory entry.
+class RedirectingFileSystem : public vfs::FileSystem {
+public:
+  enum EntryKind { EK_Directory, EK_File };
+
+  /// A single file or directory in the VFS.
+  class Entry {
+    EntryKind Kind;
+    std::string Name;
+
+  public:
+    Entry(EntryKind K, StringRef Name) : Kind(K), Name(Name) {}
+    virtual ~Entry() = default;
+
+    StringRef getName() const { return Name; }
+    EntryKind getKind() const { return Kind; }
+  };
+
+  class RedirectingDirectoryEntry : public Entry {
+    std::vector<std::unique_ptr<Entry>> Contents;
+    Status S;
+
+  public:
+    RedirectingDirectoryEntry(StringRef Name,
+                              std::vector<std::unique_ptr<Entry>> Contents,
+                              Status S)
+        : Entry(EK_Directory, Name), Contents(std::move(Contents)),
+          S(std::move(S)) {}
+    RedirectingDirectoryEntry(StringRef Name, Status S)
+        : Entry(EK_Directory, Name), S(std::move(S)) {}
+
+    Status getStatus() { return S; }
+
+    void addContent(std::unique_ptr<Entry> Content) {
+      Contents.push_back(std::move(Content));
+    }
+
+    Entry *getLastContent() const { return Contents.back().get(); }
+
+    using iterator = decltype(Contents)::iterator;
+
+    iterator contents_begin() { return Contents.begin(); }
+    iterator contents_end() { return Contents.end(); }
+
+    static bool classof(const Entry *E) { return E->getKind() == EK_Directory; }
+  };
+
+  class RedirectingFileEntry : public Entry {
+  public:
+    enum NameKind { NK_NotSet, NK_External, NK_Virtual };
+
+  private:
+    std::string ExternalContentsPath;
+    NameKind UseName;
+
+  public:
+    RedirectingFileEntry(StringRef Name, StringRef ExternalContentsPath,
+                         NameKind UseName)
+        : Entry(EK_File, Name), ExternalContentsPath(ExternalContentsPath),
+          UseName(UseName) {}
+
+    StringRef getExternalContentsPath() const { return ExternalContentsPath; }
+
+    /// whether to use the external path as the name for this file.
+    bool useExternalName(bool GlobalUseExternalName) const {
+      return UseName == NK_NotSet ? GlobalUseExternalName
+                                  : (UseName == NK_External);
+    }
+
+    NameKind getUseName() const { return UseName; }
+
+    static bool classof(const Entry *E) { return E->getKind() == EK_File; }
+  };
+
+private:
+  friend class VFSFromYamlDirIterImpl;
+  friend class RedirectingFileSystemParser;
+
+  /// The root(s) of the virtual file system.
+  std::vector<std::unique_ptr<Entry>> Roots;
+
+  /// The file system to use for external references.
+  IntrusiveRefCntPtr<FileSystem> ExternalFS;
+
+  /// If IsRelativeOverlay is set, this represents the directory
+  /// path that should be prefixed to each 'external-contents' entry
+  /// when reading from YAML files.
+  std::string ExternalContentsPrefixDir;
+
+  /// @name Configuration
+  /// @{
+
+  /// Whether to perform case-sensitive comparisons.
+  ///
+  /// Currently, case-insensitive matching only works correctly with ASCII.
+  bool CaseSensitive = true;
+
+  /// IsRelativeOverlay marks whether a ExternalContentsPrefixDir path must
+  /// be prefixed in every 'external-contents' when reading from YAML files.
+  bool IsRelativeOverlay = false;
+
+  /// Whether to use to use the value of 'external-contents' for the
+  /// names of files.  This global value is overridable on a per-file basis.
+  bool UseExternalNames = true;
+
+  /// Whether to attempt a file lookup in external file system after it wasn't
+  /// found in VFS.
+  bool IsFallthrough = true;
+  /// @}
+
+  /// Virtual file paths and external files could be canonicalized without "..",
+  /// "." and "./" in their paths. FIXME: some unittests currently fail on
+  /// win32 when using remove_dots and remove_leading_dotslash on paths.
+  bool UseCanonicalizedPaths =
+#ifdef _WIN32
+      false;
+#else
+      true;
+#endif
+
+  RedirectingFileSystem(IntrusiveRefCntPtr<FileSystem> ExternalFS)
+      : ExternalFS(std::move(ExternalFS)) {}
+
+  /// Looks up the path <tt>[Start, End)</tt> in \p From, possibly
+  /// recursing into the contents of \p From if it is a directory.
+  ErrorOr<Entry *> lookupPath(llvm::sys::path::const_iterator Start,
+                              llvm::sys::path::const_iterator End,
+                              Entry *From) const;
+
+  /// Get the status of a given an \c Entry.
+  ErrorOr<Status> status(const Twine &Path, Entry *E);
+
+public:
+  /// Looks up \p Path in \c Roots.
+  ErrorOr<Entry *> lookupPath(const Twine &Path) const;
+
+  /// Parses \p Buffer, which is expected to be in YAML format and
+  /// returns a virtual file system representing its contents.
+  static RedirectingFileSystem *
+  create(std::unique_ptr<MemoryBuffer> Buffer,
+         SourceMgr::DiagHandlerTy DiagHandler, StringRef YAMLFilePath,
+         void *DiagContext, IntrusiveRefCntPtr<FileSystem> ExternalFS);
+
+  ErrorOr<Status> status(const Twine &Path) override;
+  ErrorOr<std::unique_ptr<File>> openFileForRead(const Twine &Path) override;
+
+  std::error_code getRealPath(const Twine &Path,
+                              SmallVectorImpl<char> &Output) const override;
+
+  llvm::ErrorOr<std::string> getCurrentWorkingDirectory() const override;
+
+  std::error_code setCurrentWorkingDirectory(const Twine &Path) override;
+
+  std::error_code isLocal(const Twine &Path, bool &Result) override;
+
+  directory_iterator dir_begin(const Twine &Dir, std::error_code &EC) override;
+
+  void setExternalContentsPrefixDir(StringRef PrefixDir);
+
+  StringRef getExternalContentsPrefixDir() const;
+
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+  LLVM_DUMP_METHOD void dump() const;
+  LLVM_DUMP_METHOD void dumpEntry(Entry *E, int NumSpaces = 0) const;
+#endif
+};
+
 /// Collect all pairs of <virtual path, real path> entries from the
 /// \p YAMLFilePath. This is used by the module dependency collector to forward
 /// the entries into the reproducer output VFS YAML file.
@@ -526,6 +752,8 @@ public:
     OverlayDir.assign(OverlayDirectory.str());
   }
 
+  const std::vector<YAMLVFSEntry> &getMappings() const { return Mappings; }
+
   void write(llvm::raw_ostream &OS);
 };
 
diff --git a/include/llvm/Support/Watchdog.h b/include/llvm/Support/Watchdog.h
index 01e1d926eb9569684cf93d54c73942a7c623b2b3..281595e8f2722524cfa25fc7216eca3f2f36a5fa 100644
--- a/include/llvm/Support/Watchdog.h
+++ b/include/llvm/Support/Watchdog.h
@@ -1,9 +1,8 @@
 //===--- Watchdog.h - Watchdog timer ----------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Support/Win64EH.h b/include/llvm/Support/Win64EH.h
index e27bf1b3a1a5082744e21073d38becf356d013d1..bdd23b41594ee0154061a238c4392c888f21b678 100644
--- a/include/llvm/Support/Win64EH.h
+++ b/include/llvm/Support/Win64EH.h
@@ -1,9 +1,8 @@
 //===-- llvm/Support/Win64EH.h ---Win64 EH Constants-------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Support/WindowsError.h b/include/llvm/Support/WindowsError.h
index 63bfe597654654f274e603bc5e7aca065cd182ce..19540522412438b40f9404efa8c174f7422aa481 100644
--- a/include/llvm/Support/WindowsError.h
+++ b/include/llvm/Support/WindowsError.h
@@ -1,9 +1,8 @@
 //===-- WindowsError.h - Support for mapping windows errors to posix-------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/Support/WithColor.h b/include/llvm/Support/WithColor.h
index 76842d1c3dc8ef6b07f041465527e2bbac2015d8..f4e10758117943d4c0a87d6443de3777a755c110 100644
--- a/include/llvm/Support/WithColor.h
+++ b/include/llvm/Support/WithColor.h
@@ -1,9 +1,8 @@
 //===- WithColor.h ----------------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/Support/X86DisassemblerDecoderCommon.h b/include/llvm/Support/X86DisassemblerDecoderCommon.h
index 466dd309909ac39ad3709e8caabf07483789d339..83c867169f5229ba1c23e45dff7b433c2e60520c 100644
--- a/include/llvm/Support/X86DisassemblerDecoderCommon.h
+++ b/include/llvm/Support/X86DisassemblerDecoderCommon.h
@@ -1,9 +1,8 @@
 //===-- X86DisassemblerDecoderCommon.h - Disassembler decoder ---*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Support/X86TargetParser.def b/include/llvm/Support/X86TargetParser.def
index e9bede545d3f71ced08b874d405a37f6132faf11..fb82228c4dff4c337ceec1f8c0cf85e17bf8fb02 100644
--- a/include/llvm/Support/X86TargetParser.def
+++ b/include/llvm/Support/X86TargetParser.def
@@ -1,9 +1,8 @@
 //===- X86TargetParser.def - X86 target parsing defines ---------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Support/YAMLParser.h b/include/llvm/Support/YAMLParser.h
index 5b031a9a4270c6f290faa280936257bc38874c67..3570119a3bfdb65805d28ddafc73ce952936a658 100644
--- a/include/llvm/Support/YAMLParser.h
+++ b/include/llvm/Support/YAMLParser.h
@@ -1,9 +1,8 @@
 //===- YAMLParser.h - Simple YAML parser ------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Support/YAMLTraits.h b/include/llvm/Support/YAMLTraits.h
index 3d790e96fff76d18f986b1e6aeb53258305c6ee7..e991f3b3be15e32353c50977b3786257e11bb5b9 100644
--- a/include/llvm/Support/YAMLTraits.h
+++ b/include/llvm/Support/YAMLTraits.h
@@ -1,9 +1,8 @@
 //===- llvm/Support/YAMLTraits.h --------------------------------*- C++ -*-===//
 //
-//                             The LLVM Linker
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/Support/circular_raw_ostream.h b/include/llvm/Support/circular_raw_ostream.h
index b46fd7f730c914b72c5c83d0817cfbab3fbd7b96..4ecdb17376f15913b2f4162b6aff86d17a837d9d 100644
--- a/include/llvm/Support/circular_raw_ostream.h
+++ b/include/llvm/Support/circular_raw_ostream.h
@@ -1,9 +1,8 @@
 //===-- llvm/Support/circular_raw_ostream.h - Buffered streams --*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Support/raw_os_ostream.h b/include/llvm/Support/raw_os_ostream.h
index a983aeb90879c5fe71eb85d5d094dcc585ed23a8..c51a94da3a28d2c7c159311e09a52d097fa66225 100644
--- a/include/llvm/Support/raw_os_ostream.h
+++ b/include/llvm/Support/raw_os_ostream.h
@@ -1,9 +1,8 @@
 //===- raw_os_ostream.h - std::ostream adaptor for raw_ostream --*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Support/raw_ostream.h b/include/llvm/Support/raw_ostream.h
index 9a86f3a371e087b9fe9563cf8baf9908f55a73a8..1eb13c78d93351fb2c17d1efb65e6b9e04ac5f96 100644
--- a/include/llvm/Support/raw_ostream.h
+++ b/include/llvm/Support/raw_ostream.h
@@ -1,9 +1,8 @@
 //===--- raw_ostream.h - Raw output stream ----------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -554,6 +553,8 @@ class buffer_ostream : public raw_svector_ostream {
   raw_ostream &OS;
   SmallVector<char, 0> Buffer;
 
+  virtual void anchor() override;
+
 public:
   buffer_ostream(raw_ostream &OS) : raw_svector_ostream(Buffer), OS(OS) {}
   ~buffer_ostream() override { OS << str(); }
diff --git a/include/llvm/Support/raw_sha1_ostream.h b/include/llvm/Support/raw_sha1_ostream.h
index bd55d98b7c1d6b70eab36f5796f64737a2c1f4e1..3991691796b598065935707813d7fc57f2c0e7db 100644
--- a/include/llvm/Support/raw_sha1_ostream.h
+++ b/include/llvm/Support/raw_sha1_ostream.h
@@ -1,9 +1,8 @@
 //==- raw_sha1_ostream.h - raw_ostream that compute SHA1        --*- C++ -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Support/thread.h b/include/llvm/Support/thread.h
index 787a513d6017616283ae249f00ae28e5fc973379..084ed16166feccae798036d9992d365ba6ea0984 100644
--- a/include/llvm/Support/thread.h
+++ b/include/llvm/Support/thread.h
@@ -1,9 +1,8 @@
 //===-- llvm/Support/thread.h - Wrapper for <thread> ------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Support/type_traits.h b/include/llvm/Support/type_traits.h
index e7b8f2517b8aa975a26923eb07b534f48e77a737..c8c6a76a90f1d4c221469e57681dba284988138a 100644
--- a/include/llvm/Support/type_traits.h
+++ b/include/llvm/Support/type_traits.h
@@ -1,9 +1,8 @@
 //===- llvm/Support/type_traits.h - Simplfied type traits -------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -25,35 +24,6 @@
 
 namespace llvm {
 
-/// isPodLike - This is a type trait that is used to determine whether a given
-/// type can be copied around with memcpy instead of running ctors etc.
-template <typename T>
-struct isPodLike {
-  // std::is_trivially_copyable is available in libc++ with clang, libstdc++
-  // that comes with GCC 5.  MSVC 2015 and newer also have
-  // std::is_trivially_copyable.
-#if (__has_feature(is_trivially_copyable) && defined(_LIBCPP_VERSION)) ||      \
-    (defined(__GNUC__) && __GNUC__ >= 5) || defined(_MSC_VER)
-  // If the compiler supports the is_trivially_copyable trait use it, as it
-  // matches the definition of isPodLike closely.
-  static const bool value = std::is_trivially_copyable<T>::value;
-#elif __has_feature(is_trivially_copyable)
-  // Use the internal name if the compiler supports is_trivially_copyable but we
-  // don't know if the standard library does. This is the case for clang in
-  // conjunction with libstdc++ from GCC 4.x.
-  static const bool value = __is_trivially_copyable(T);
-#else
-  // If we don't know anything else, we can (at least) assume that all non-class
-  // types are PODs.
-  static const bool value = !std::is_class<T>::value;
-#endif
-};
-
-// std::pair's are pod-like if their elements are.
-template<typename T, typename U>
-struct isPodLike<std::pair<T, U>> {
-  static const bool value = isPodLike<T>::value && isPodLike<U>::value;
-};
 
 /// Metafunction that determines whether the given type is either an
 /// integral type or an enumeration type, including enum classes.
@@ -120,6 +90,12 @@ template<typename T> union move_construction_triviality_helper {
     move_construction_triviality_helper(move_construction_triviality_helper&&) = default;
     ~move_construction_triviality_helper() = default;
 };
+
+template<class T>
+union trivial_helper {
+    T t;
+};
+
 } // end namespace detail
 
 /// An implementation of `std::is_trivially_copy_constructible` since we have
@@ -144,6 +120,78 @@ struct is_trivially_move_constructible<T &> : std::true_type {};
 template <typename T>
 struct is_trivially_move_constructible<T &&> : std::true_type {};
 
+
+template <typename T>
+struct is_copy_assignable {
+  template<class F>
+    static auto get(F*) -> decltype(std::declval<F &>() = std::declval<const F &>(), std::true_type{});
+    static std::false_type get(...);
+    static constexpr bool value = decltype(get((T*)nullptr))::value;
+};
+
+template <typename T>
+struct is_move_assignable {
+  template<class F>
+    static auto get(F*) -> decltype(std::declval<F &>() = std::declval<F &&>(), std::true_type{});
+    static std::false_type get(...);
+    static constexpr bool value = decltype(get((T*)nullptr))::value;
+};
+
+
+// An implementation of `std::is_trivially_copyable` since STL version
+// is not equally supported by all compilers, especially GCC 4.9.
+// Uniform implementation of this trait is important for ABI compatibility
+// as it has an impact on SmallVector's ABI (among others).
+template <typename T>
+class is_trivially_copyable {
+
+  // copy constructors
+  static constexpr bool has_trivial_copy_constructor =
+      std::is_copy_constructible<detail::trivial_helper<T>>::value;
+  static constexpr bool has_deleted_copy_constructor =
+      !std::is_copy_constructible<T>::value;
+
+  // move constructors
+  static constexpr bool has_trivial_move_constructor =
+      std::is_move_constructible<detail::trivial_helper<T>>::value;
+  static constexpr bool has_deleted_move_constructor =
+      !std::is_move_constructible<T>::value;
+
+  // copy assign
+  static constexpr bool has_trivial_copy_assign =
+      is_copy_assignable<detail::trivial_helper<T>>::value;
+  static constexpr bool has_deleted_copy_assign =
+      !is_copy_assignable<T>::value;
+
+  // move assign
+  static constexpr bool has_trivial_move_assign =
+      is_move_assignable<detail::trivial_helper<T>>::value;
+  static constexpr bool has_deleted_move_assign =
+      !is_move_assignable<T>::value;
+
+  // destructor
+  static constexpr bool has_trivial_destructor =
+      std::is_destructible<detail::trivial_helper<T>>::value;
+
+  public:
+
+  static constexpr bool value =
+      has_trivial_destructor &&
+      (has_deleted_move_assign || has_trivial_move_assign) &&
+      (has_deleted_move_constructor || has_trivial_move_constructor) &&
+      (has_deleted_copy_assign || has_trivial_copy_assign) &&
+      (has_deleted_copy_constructor || has_trivial_copy_constructor);
+
+#ifdef HAVE_STD_IS_TRIVIALLY_COPYABLE
+  static_assert(value == std::is_trivially_copyable<T>::value,
+                "inconsistent behavior between llvm:: and std:: implementation of is_trivially_copyable");
+#endif
+};
+template <typename T>
+class is_trivially_copyable<T*> : public std::true_type {
+};
+
+
 } // end namespace llvm
 
 // If the compiler supports detecting whether a class is final, define
diff --git a/include/llvm/TableGen/Error.h b/include/llvm/TableGen/Error.h
index de4d3bf5478230c8c19d7bd8aac21a2128965a5b..7c83b62986205e660d2a473d15adfdc09bcded87 100644
--- a/include/llvm/TableGen/Error.h
+++ b/include/llvm/TableGen/Error.h
@@ -1,9 +1,8 @@
 //===- llvm/TableGen/Error.h - tblgen error handling helpers ----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/TableGen/Main.h b/include/llvm/TableGen/Main.h
index 670572dc81034ff9ec6fea85c477f0482013db8a..e464cd4d4fb574f23a429e66a5724ac0cc62658b 100644
--- a/include/llvm/TableGen/Main.h
+++ b/include/llvm/TableGen/Main.h
@@ -1,9 +1,8 @@
 //===- llvm/TableGen/Main.h - tblgen entry point ----------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/TableGen/Record.h b/include/llvm/TableGen/Record.h
index e022bc82b4e46bb318e881dd0d78bd904bc9ae05..e4a07bbb71be20a8dd52c1a45d7fbbaa6d1bf792 100644
--- a/include/llvm/TableGen/Record.h
+++ b/include/llvm/TableGen/Record.h
@@ -1,9 +1,8 @@
 //===- llvm/TableGen/Record.h - Classes for Table Records -------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -316,6 +315,7 @@ protected:
     IK_TernOpInit,
     IK_UnOpInit,
     IK_LastOpInit,
+    IK_CondOpInit,
     IK_FoldOpInit,
     IK_IsAOpInit,
     IK_StringInit,
@@ -912,6 +912,83 @@ public:
   std::string getAsString() const override;
 };
 
+/// !cond(condition_1: value1, ... , condition_n: value)
+/// Selects the first value for which condition is true.
+/// Otherwise reports an error.
+class CondOpInit final : public TypedInit, public FoldingSetNode,
+                      public TrailingObjects<CondOpInit, Init *> {
+  unsigned NumConds;
+  RecTy *ValType;
+
+  CondOpInit(unsigned NC, RecTy *Type)
+    : TypedInit(IK_CondOpInit, Type),
+      NumConds(NC), ValType(Type) {}
+
+  size_t numTrailingObjects(OverloadToken<Init *>) const {
+    return 2*NumConds;
+  }
+
+public:
+  CondOpInit(const CondOpInit &) = delete;
+  CondOpInit &operator=(const CondOpInit &) = delete;
+
+  static bool classof(const Init *I) {
+    return I->getKind() == IK_CondOpInit;
+  }
+
+  static CondOpInit *get(ArrayRef<Init*> C, ArrayRef<Init*> V,
+                        RecTy *Type);
+
+  void Profile(FoldingSetNodeID &ID) const;
+
+  RecTy *getValType() const { return ValType; }
+
+  unsigned getNumConds() const { return NumConds; }
+
+  Init *getCond(unsigned Num) const {
+    assert(Num < NumConds && "Condition number out of range!");
+    return getTrailingObjects<Init *>()[Num];
+  }
+
+  Init *getVal(unsigned Num) const {
+    assert(Num < NumConds && "Val number out of range!");
+    return getTrailingObjects<Init *>()[Num+NumConds];
+  }
+
+  ArrayRef<Init *> getConds() const {
+    return makeArrayRef(getTrailingObjects<Init *>(), NumConds);
+  }
+
+  ArrayRef<Init *> getVals() const {
+    return makeArrayRef(getTrailingObjects<Init *>()+NumConds, NumConds);
+  }
+
+  Init *Fold(Record *CurRec) const;
+
+  Init *resolveReferences(Resolver &R) const override;
+
+  bool isConcrete() const override;
+  bool isComplete() const override;
+  std::string getAsString() const override;
+
+  using const_case_iterator = SmallVectorImpl<Init*>::const_iterator;
+  using const_val_iterator = SmallVectorImpl<Init*>::const_iterator;
+
+  inline const_case_iterator  arg_begin() const { return getConds().begin(); }
+  inline const_case_iterator  arg_end  () const { return getConds().end(); }
+
+  inline size_t              case_size () const { return NumConds; }
+  inline bool                case_empty() const { return NumConds == 0; }
+
+  inline const_val_iterator name_begin() const { return getVals().begin();}
+  inline const_val_iterator name_end  () const { return getVals().end(); }
+
+  inline size_t              val_size () const { return NumConds; }
+  inline bool                val_empty() const { return NumConds == 0; }
+
+  Init *getBit(unsigned Bit) const override;
+};
+
 /// !foldl (a, b, expr, start, lst) - Fold over a list.
 class FoldOpInit : public TypedInit, public FoldingSetNode {
 private:
diff --git a/include/llvm/TableGen/SearchableTable.td b/include/llvm/TableGen/SearchableTable.td
index 1089d363eb6f7ad8cdcddcf0b3dcd0f60c227e19..2680c71218ea59717e9e3a5443072604e258f0fc 100644
--- a/include/llvm/TableGen/SearchableTable.td
+++ b/include/llvm/TableGen/SearchableTable.td
@@ -1,9 +1,8 @@
 //===- SearchableTable.td ----------------------------------*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/TableGen/SetTheory.h b/include/llvm/TableGen/SetTheory.h
index 4b32f9e3da8fb8e83545d7c06414d6263364c243..35156424b0d30eb0e3e6f0974a9d40e7fb915674 100644
--- a/include/llvm/TableGen/SetTheory.h
+++ b/include/llvm/TableGen/SetTheory.h
@@ -1,9 +1,8 @@
 //===- SetTheory.h - Generate ordered sets from DAG expressions -*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/TableGen/StringMatcher.h b/include/llvm/TableGen/StringMatcher.h
index 3aa3540d616da60a19d51291af98201d457410f9..795b7a6d41dcc6aac711517f81d5c5cea38110a1 100644
--- a/include/llvm/TableGen/StringMatcher.h
+++ b/include/llvm/TableGen/StringMatcher.h
@@ -1,9 +1,8 @@
 //===- StringMatcher.h - Generate a matcher for input strings ---*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/TableGen/StringToOffsetTable.h b/include/llvm/TableGen/StringToOffsetTable.h
index 4b11e889ea6c7e980ad91898f76665885e25e29c..76ce51893907c8c5230fe01acc8bb7009992ba26 100644
--- a/include/llvm/TableGen/StringToOffsetTable.h
+++ b/include/llvm/TableGen/StringToOffsetTable.h
@@ -1,9 +1,8 @@
 //===- StringToOffsetTable.h - Emit a big concatenated string ---*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/TableGen/TableGenBackend.h b/include/llvm/TableGen/TableGenBackend.h
index d226f1f1af7b406bb3d3b72c760867c54e67410a..a426e4217578438e60177b17fbf473834bf107d8 100644
--- a/include/llvm/TableGen/TableGenBackend.h
+++ b/include/llvm/TableGen/TableGenBackend.h
@@ -1,9 +1,8 @@
 //===- llvm/TableGen/TableGenBackend.h - Backend utilities ------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -23,6 +22,8 @@ class raw_ostream;
 /// raw_ostream.
 void emitSourceFileHeader(StringRef Desc, raw_ostream &OS);
 
+extern bool TimeRegions;
+
 } // End llvm namespace
 
 #endif
diff --git a/include/llvm/Target/CodeGenCWrappers.h b/include/llvm/Target/CodeGenCWrappers.h
index 3ad77c5d5e000e919f0fbc5be083ea987f831111..a995463570535d04ccb0c378639c076760b88c73 100644
--- a/include/llvm/Target/CodeGenCWrappers.h
+++ b/include/llvm/Target/CodeGenCWrappers.h
@@ -1,9 +1,8 @@
 //===- llvm/Target/CodeGenCWrappers.h - CodeGen C Wrappers ------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Target/GenericOpcodes.td b/include/llvm/Target/GenericOpcodes.td
index 775221a9f9772c7236dcab85bf49c4b96171ed86..e4b452e91b9da9298f064a73d5791a3e7fec147e 100644
--- a/include/llvm/Target/GenericOpcodes.td
+++ b/include/llvm/Target/GenericOpcodes.td
@@ -1,9 +1,8 @@
 //===-- GenericOpcodes.td - Opcodes used with GlobalISel ---*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -93,12 +92,14 @@ def G_BITCAST : GenericInstruction {
   let hasSideEffects = 0;
 }
 
+// Only supports scalar result types
 def G_CONSTANT : GenericInstruction {
   let OutOperandList = (outs type0:$dst);
   let InOperandList = (ins unknown:$imm);
   let hasSideEffects = 0;
 }
 
+// Only supports scalar result types
 def G_FCONSTANT : GenericInstruction {
   let OutOperandList = (outs type0:$dst);
   let InOperandList = (ins unknown:$imm);
@@ -122,31 +123,31 @@ def G_VAARG : GenericInstruction {
 
 def G_CTLZ : GenericInstruction {
   let OutOperandList = (outs type0:$dst);
-  let InOperandList = (ins type0:$src);
+  let InOperandList = (ins type1:$src);
   let hasSideEffects = 0;
 }
 
 def G_CTLZ_ZERO_UNDEF : GenericInstruction {
   let OutOperandList = (outs type0:$dst);
-  let InOperandList = (ins type0:$src);
+  let InOperandList = (ins type1:$src);
   let hasSideEffects = 0;
 }
 
 def G_CTTZ : GenericInstruction {
   let OutOperandList = (outs type0:$dst);
-  let InOperandList = (ins type0:$src);
+  let InOperandList = (ins type1:$src);
   let hasSideEffects = 0;
 }
 
 def G_CTTZ_ZERO_UNDEF : GenericInstruction {
   let OutOperandList = (outs type0:$dst);
-  let InOperandList = (ins type0:$src);
+  let InOperandList = (ins type1:$src);
   let hasSideEffects = 0;
 }
 
 def G_CTPOP : GenericInstruction {
   let OutOperandList = (outs type0:$dst);
-  let InOperandList = (ins type0:$src);
+  let InOperandList = (ins type1:$src);
   let hasSideEffects = 0;
 }
 
@@ -255,21 +256,21 @@ def G_XOR : GenericInstruction {
 // Generic left-shift.
 def G_SHL : GenericInstruction {
   let OutOperandList = (outs type0:$dst);
-  let InOperandList = (ins type0:$src1, type0:$src2);
+  let InOperandList = (ins type0:$src1, type1:$src2);
   let hasSideEffects = 0;
 }
 
 // Generic logical right-shift.
 def G_LSHR : GenericInstruction {
   let OutOperandList = (outs type0:$dst);
-  let InOperandList = (ins type0:$src1, type0:$src2);
+  let InOperandList = (ins type0:$src1, type1:$src2);
   let hasSideEffects = 0;
 }
 
 // Generic arithmetic right-shift.
 def G_ASHR : GenericInstruction {
   let OutOperandList = (outs type0:$dst);
-  let InOperandList = (ins type0:$src1, type0:$src2);
+  let InOperandList = (ins type0:$src1, type1:$src2);
   let hasSideEffects = 0;
 }
 
@@ -454,6 +455,12 @@ def G_FABS : GenericInstruction {
   let hasSideEffects = 0;
 }
 
+def G_FCANONICALIZE : GenericInstruction {
+  let OutOperandList = (outs type0:$dst);
+  let InOperandList = (ins type0:$src);
+  let hasSideEffects = 0;
+}
+
 //------------------------------------------------------------------------------
 // Floating Point Binary ops.
 //------------------------------------------------------------------------------
@@ -547,6 +554,44 @@ def G_FLOG10 : GenericInstruction {
   let hasSideEffects = 0;
 }
 
+// Floating point ceiling of a value.
+def G_FCEIL : GenericInstruction {
+  let OutOperandList = (outs type0:$dst);
+  let InOperandList = (ins type0:$src1);
+  let hasSideEffects = 0;
+}
+
+// Floating point cosine of a value.
+def G_FCOS : GenericInstruction {
+  let OutOperandList = (outs type0:$dst);
+  let InOperandList = (ins type0:$src1);
+  let hasSideEffects = 0;
+}
+
+// Floating point sine of a value.
+def G_FSIN : GenericInstruction {
+  let OutOperandList = (outs type0:$dst);
+  let InOperandList = (ins type0:$src1);
+  let hasSideEffects = 0;
+}
+
+// Floating point square root of a value.
+// This returns NaN for negative nonzero values.
+// NOTE: Unlike libm sqrt(), this never sets errno. In all other respects it's
+// libm-conformant.
+def G_FSQRT : GenericInstruction {
+  let OutOperandList = (outs type0:$dst);
+  let InOperandList = (ins type0:$src1);
+  let hasSideEffects = 0;
+}
+
+// Floating point floor of a value.
+def G_FFLOOR : GenericInstruction {
+  let OutOperandList = (outs type0:$dst);
+  let InOperandList = (ins type0:$src1);
+  let hasSideEffects = 0;
+}
+
 //------------------------------------------------------------------------------
 // Opcodes for LLVM Intrinsics
 //------------------------------------------------------------------------------
@@ -682,7 +727,9 @@ def G_MERGE_VALUES : GenericInstruction {
   let hasSideEffects = 0;
 }
 
-/// Create a vector from multiple scalar registers.
+/// Create a vector from multiple scalar registers. No implicit
+/// conversion is performed (i.e. the result element type must be the
+/// same as all source operands)
 def G_BUILD_VECTOR : GenericInstruction {
   let OutOperandList = (outs type0:$dst);
   let InOperandList = (ins type1:$src0, variable_ops);
diff --git a/include/llvm/Target/GlobalISel/RegisterBank.td b/include/llvm/Target/GlobalISel/RegisterBank.td
index 4dfd139e9fb6ff16731bf66afcd687ded18c4a79..51578b66b160bb6e6a8fefd82da750dbe163f162 100644
--- a/include/llvm/Target/GlobalISel/RegisterBank.td
+++ b/include/llvm/Target/GlobalISel/RegisterBank.td
@@ -1,9 +1,8 @@
 //===- RegisterBank.td - Register bank definitions ---------*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Target/GlobalISel/SelectionDAGCompat.td b/include/llvm/Target/GlobalISel/SelectionDAGCompat.td
index af26375802a250c21dee6cfbf0689ee3fc757963..6b0434a0758cd44bacbc96d6dcdc2f310e47a0cc 100644
--- a/include/llvm/Target/GlobalISel/SelectionDAGCompat.td
+++ b/include/llvm/Target/GlobalISel/SelectionDAGCompat.td
@@ -1,9 +1,8 @@
 //===- TargetGlobalISel.td - Common code for GlobalISel ----*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -89,6 +88,12 @@ def : GINodeEquiv<G_CTLZ_ZERO_UNDEF, ctlz_zero_undef>;
 def : GINodeEquiv<G_CTTZ_ZERO_UNDEF, cttz_zero_undef>;
 def : GINodeEquiv<G_CTPOP, ctpop>;
 def : GINodeEquiv<G_EXTRACT_VECTOR_ELT, vector_extract>;
+def : GINodeEquiv<G_FCEIL, fceil>;
+def : GINodeEquiv<G_FCOS, fcos>;
+def : GINodeEquiv<G_FSIN, fsin>;
+def : GINodeEquiv<G_FABS, fabs>;
+def : GINodeEquiv<G_FSQRT, fsqrt>;
+def : GINodeEquiv<G_FFLOOR, ffloor>;
 
 // Broadly speaking G_LOAD is equivalent to ISD::LOAD but there are some
 // complications that tablegen must take care of. For example, Predicates such
diff --git a/include/llvm/Target/GlobalISel/Target.td b/include/llvm/Target/GlobalISel/Target.td
index 6740f404a9d379c3e93c42c782042dad8b9906a2..538ca65e1162ea3f2632df515fd61ca11b53eb5d 100644
--- a/include/llvm/Target/GlobalISel/Target.td
+++ b/include/llvm/Target/GlobalISel/Target.td
@@ -1,9 +1,8 @@
 //===- Target.td - Define GlobalISel rules -----------------*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Target/Target.td b/include/llvm/Target/Target.td
index e4b827babb92c71a3e8f1a52317d414f7588ec94..380e8485816747fa343533a5582ff2632fc3d8cf 100644
--- a/include/llvm/Target/Target.td
+++ b/include/llvm/Target/Target.td
@@ -1,9 +1,8 @@
 //===- Target.td - Target Independent TableGen interface ---*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -934,6 +933,15 @@ def INLINEASM : StandardPseudoInstruction {
   let AsmString = "";
   let hasSideEffects = 0;  // Note side effect is encoded in an operand.
 }
+def INLINEASM_BR : StandardPseudoInstruction {
+  let OutOperandList = (outs);
+  let InOperandList = (ins variable_ops);
+  let AsmString = "";
+  let hasSideEffects = 0;  // Note side effect is encoded in an operand.
+  let isTerminator = 1;
+  let isBranch = 1;
+  let isIndirectBranch = 1;
+}
 def CFI_INSTRUCTION : StandardPseudoInstruction {
   let OutOperandList = (outs);
   let InOperandList = (ins i32imm:$id);
diff --git a/include/llvm/Target/TargetCallingConv.td b/include/llvm/Target/TargetCallingConv.td
index 95d2b42262940dbfdfe4027585c6115dde1d9c62..11ed4f5b808059a6871bb8119311d02c77ec366f 100644
--- a/include/llvm/Target/TargetCallingConv.td
+++ b/include/llvm/Target/TargetCallingConv.td
@@ -1,9 +1,8 @@
 //===- TargetCallingConv.td - Target Calling Conventions ---*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -160,6 +159,11 @@ class CCDelegateTo<CallingConv cc> : CCAction {
 /// that the target supports.
 class CallingConv<list<CCAction> actions> {
   list<CCAction> Actions = actions;
+
+  /// If true, this calling convention will be emitted as externally visible in
+  /// the llvm namespaces instead of as a static function.
+  bit Entry = 0;
+
   bit Custom = 0;
 }
 
diff --git a/include/llvm/Target/TargetInstrPredicate.td b/include/llvm/Target/TargetInstrPredicate.td
index 4b2c57b34c2e2035599cd09e41b3a726e72600e2..5623461c648dacc91589377a7b385995edd6966f 100644
--- a/include/llvm/Target/TargetInstrPredicate.td
+++ b/include/llvm/Target/TargetInstrPredicate.td
@@ -1,9 +1,8 @@
 //===- TargetInstrPredicate.td - ---------------------------*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Target/TargetIntrinsicInfo.h b/include/llvm/Target/TargetIntrinsicInfo.h
index 6a92bdee747e7c475d4fb6d43e91c81574ca8385..ef571b15153e7db3ae1b474eef4297b6ba705964 100644
--- a/include/llvm/Target/TargetIntrinsicInfo.h
+++ b/include/llvm/Target/TargetIntrinsicInfo.h
@@ -1,9 +1,8 @@
 //===-- llvm/Target/TargetIntrinsicInfo.h - Instruction Info ----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Target/TargetItinerary.td b/include/llvm/Target/TargetItinerary.td
index 182054d8444ed40dc50c8c1615c466cfa82edb3f..b68ed045520ca8693ba40f8079e4e57680d82dc6 100644
--- a/include/llvm/Target/TargetItinerary.td
+++ b/include/llvm/Target/TargetItinerary.td
@@ -1,9 +1,8 @@
 //===- TargetItinerary.td - Target Itinierary Description --*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Target/TargetLoweringObjectFile.h b/include/llvm/Target/TargetLoweringObjectFile.h
index e80f2bf82f26d6eca31fedd7361efcae509c1960..f96e3532dfcf73f4e4b23436ba0b61bb74738fdb 100644
--- a/include/llvm/Target/TargetLoweringObjectFile.h
+++ b/include/llvm/Target/TargetLoweringObjectFile.h
@@ -1,9 +1,8 @@
 //===-- llvm/Target/TargetLoweringObjectFile.h - Object Info ----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Target/TargetMachine.h b/include/llvm/Target/TargetMachine.h
index 3eafcc25583aceaacb8b242360646868c1e78dde..7dd9b99b682f9b210613870f4aaf525b5c331971 100644
--- a/include/llvm/Target/TargetMachine.h
+++ b/include/llvm/Target/TargetMachine.h
@@ -1,9 +1,8 @@
 //===-- llvm/Target/TargetMachine.h - Target Information --------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Target/TargetOptions.h b/include/llvm/Target/TargetOptions.h
index b18101d92833d590020cfa0ea232886137497a54..6b76a11651ae8e3468e20910b25bff921dab551b 100644
--- a/include/llvm/Target/TargetOptions.h
+++ b/include/llvm/Target/TargetOptions.h
@@ -1,9 +1,8 @@
 //===-- llvm/Target/TargetOptions.h - Target Options ------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Target/TargetPfmCounters.td b/include/llvm/Target/TargetPfmCounters.td
index dac150f03445a109461f6da55558ff9f59fb4087..e1d5013c12916edcde2a95b58a14ae6eb68d5ebe 100644
--- a/include/llvm/Target/TargetPfmCounters.td
+++ b/include/llvm/Target/TargetPfmCounters.td
@@ -1,9 +1,8 @@
 //===- TargetPfmCounters.td - Target Pfm Counters -*- tablegen ----------*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Target/TargetSchedule.td b/include/llvm/Target/TargetSchedule.td
index 808e183f5a5f9b57aa79cf04dd045d903286e23c..cbfad5fcdeae14f610ca73a88007fba02a5ec176 100644
--- a/include/llvm/Target/TargetSchedule.td
+++ b/include/llvm/Target/TargetSchedule.td
@@ -1,9 +1,8 @@
 //===- TargetSchedule.td - Target Independent Scheduling ---*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Target/TargetSelectionDAG.td b/include/llvm/Target/TargetSelectionDAG.td
index eb5a14bd21b8a6c461d7c83e0f7bbdf7349e686b..174d97a77ee53bcc8770710ba0d2f96dd76190b6 100644
--- a/include/llvm/Target/TargetSelectionDAG.td
+++ b/include/llvm/Target/TargetSelectionDAG.td
@@ -1,9 +1,8 @@
 //===- TargetSelectionDAG.td - Common code for DAG isels ---*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -125,7 +124,7 @@ def SDTIntSatNoShOp : SDTypeProfile<1, 2, [   // ssat with no shift
 def SDTIntBinHiLoOp : SDTypeProfile<2, 2, [ // mulhi, mullo, sdivrem, udivrem
   SDTCisSameAs<0, 1>, SDTCisSameAs<0, 2>, SDTCisSameAs<0, 3>,SDTCisInt<0>
 ]>;
-def SDTIntScaledBinOp : SDTypeProfile<1, 3, [  // smulfix
+def SDTIntScaledBinOp : SDTypeProfile<1, 3, [  // smulfix, umulfix
   SDTCisSameAs<0, 1>, SDTCisSameAs<0, 2>, SDTCisInt<0>, SDTCisInt<3>
 ]>;
 
@@ -265,6 +264,11 @@ def SDTAtomic3 : SDTypeProfile<1, 3, [
 def SDTAtomic2 : SDTypeProfile<1, 2, [
   SDTCisSameAs<0,2>, SDTCisInt<0>, SDTCisPtrTy<1>
 ]>;
+
+def SDTFPAtomic2 : SDTypeProfile<1, 2, [
+  SDTCisSameAs<0,2>, SDTCisFP<0>, SDTCisPtrTy<1>
+]>;
+
 def SDTAtomicStore : SDTypeProfile<0, 2, [
   SDTCisPtrTy<0>, SDTCisInt<1>
 ]>;
@@ -385,7 +389,9 @@ def saddsat    : SDNode<"ISD::SADDSAT"   , SDTIntBinOp, [SDNPCommutative]>;
 def uaddsat    : SDNode<"ISD::UADDSAT"   , SDTIntBinOp, [SDNPCommutative]>;
 def ssubsat    : SDNode<"ISD::SSUBSAT"   , SDTIntBinOp>;
 def usubsat    : SDNode<"ISD::USUBSAT"   , SDTIntBinOp>;
+
 def smulfix    : SDNode<"ISD::SMULFIX"   , SDTIntScaledBinOp, [SDNPCommutative]>;
+def umulfix    : SDNode<"ISD::UMULFIX"   , SDTIntScaledBinOp, [SDNPCommutative]>;
 
 def sext_inreg : SDNode<"ISD::SIGN_EXTEND_INREG", SDTExtInreg>;
 def sext_invec : SDNode<"ISD::SIGN_EXTEND_VECTOR_INREG", SDTExtInvec>;
@@ -511,6 +517,11 @@ def atomic_load_umin : SDNode<"ISD::ATOMIC_LOAD_UMIN", SDTAtomic2,
                     [SDNPHasChain, SDNPMayStore, SDNPMayLoad, SDNPMemOperand]>;
 def atomic_load_umax : SDNode<"ISD::ATOMIC_LOAD_UMAX", SDTAtomic2,
                     [SDNPHasChain, SDNPMayStore, SDNPMayLoad, SDNPMemOperand]>;
+def atomic_load_fadd : SDNode<"ISD::ATOMIC_LOAD_FADD" , SDTFPAtomic2,
+                    [SDNPHasChain, SDNPMayStore, SDNPMayLoad, SDNPMemOperand]>;
+def atomic_load_fsub : SDNode<"ISD::ATOMIC_LOAD_FSUB" , SDTFPAtomic2,
+                    [SDNPHasChain, SDNPMayStore, SDNPMayLoad, SDNPMemOperand]>;
+
 def atomic_load      : SDNode<"ISD::ATOMIC_LOAD", SDTAtomicLoad,
                     [SDNPHasChain, SDNPMayLoad, SDNPMemOperand]>;
 def atomic_store     : SDNode<"ISD::ATOMIC_STORE", SDTAtomicStore,
diff --git a/include/llvm/Testing/Support/Error.h b/include/llvm/Testing/Support/Error.h
index 0e5b5403ce8759a9484c04ccc3f6c37b2cbf3f78..85328f26440ba65e1ec20c91e00e5bccbc49297a 100644
--- a/include/llvm/Testing/Support/Error.h
+++ b/include/llvm/Testing/Support/Error.h
@@ -1,9 +1,8 @@
 //===- llvm/Testing/Support/Error.h ---------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/Testing/Support/SupportHelpers.h b/include/llvm/Testing/Support/SupportHelpers.h
index b2975ec395d5500bc77d09d2198a7e78d86aac4d..a0f034d50fa10eba67227b3f7c88ad3586c1fb60 100644
--- a/include/llvm/Testing/Support/SupportHelpers.h
+++ b/include/llvm/Testing/Support/SupportHelpers.h
@@ -1,9 +1,8 @@
 //===- Testing/Support/SupportHelpers.h -----------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/TextAPI/ELF/ELFStub.h b/include/llvm/TextAPI/ELF/ELFStub.h
index fa54e6f8b711d7018d245df9e1d75e9b74df5782..76b2af121662890bb82e276e35f81acc57fc8e35 100644
--- a/include/llvm/TextAPI/ELF/ELFStub.h
+++ b/include/llvm/TextAPI/ELF/ELFStub.h
@@ -1,9 +1,8 @@
 //===- ELFStub.h ------------------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===-----------------------------------------------------------------------===/
 ///
diff --git a/include/llvm/TextAPI/ELF/TBEHandler.h b/include/llvm/TextAPI/ELF/TBEHandler.h
index 91521c656fa230c90c2af273d516bd02176ef079..1748fd13f3dce7dbc248cc52aab916ccb1b7362a 100644
--- a/include/llvm/TextAPI/ELF/TBEHandler.h
+++ b/include/llvm/TextAPI/ELF/TBEHandler.h
@@ -1,9 +1,8 @@
 //===- TBEHandler.h ---------------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===-----------------------------------------------------------------------===/
 ///
diff --git a/include/llvm/ToolDrivers/llvm-dlltool/DlltoolDriver.h b/include/llvm/ToolDrivers/llvm-dlltool/DlltoolDriver.h
index 964b0f7620a2b56f13413523b4997a769b040a34..d144f62f1cc1ad9c5e6141afceb57086b3b74cda 100644
--- a/include/llvm/ToolDrivers/llvm-dlltool/DlltoolDriver.h
+++ b/include/llvm/ToolDrivers/llvm-dlltool/DlltoolDriver.h
@@ -1,9 +1,8 @@
 //===- DlltoolDriver.h - dlltool.exe-compatible driver ----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/ToolDrivers/llvm-lib/LibDriver.h b/include/llvm/ToolDrivers/llvm-lib/LibDriver.h
index a4806ac4ad69849be41492a8de70d92093d288f6..7d0735f1e3ab328f0e3c517a7c6062762d42ac60 100644
--- a/include/llvm/ToolDrivers/llvm-lib/LibDriver.h
+++ b/include/llvm/ToolDrivers/llvm-lib/LibDriver.h
@@ -1,9 +1,8 @@
 //===- llvm-lib/LibDriver.h - lib.exe-compatible driver ---------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Transforms/AggressiveInstCombine/AggressiveInstCombine.h b/include/llvm/Transforms/AggressiveInstCombine/AggressiveInstCombine.h
index f970acdc741f1fbaad8b33c22b01f863850a7577..887c8807904e49216236fbee41a780813147e336 100644
--- a/include/llvm/Transforms/AggressiveInstCombine/AggressiveInstCombine.h
+++ b/include/llvm/Transforms/AggressiveInstCombine/AggressiveInstCombine.h
@@ -1,9 +1,8 @@
 //===- AggressiveInstCombine.h - AggressiveInstCombine pass -----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/include/llvm/Transforms/Coroutines.h b/include/llvm/Transforms/Coroutines.h
index 51beb44fdc566c7bbb833d36d617dc5b0a8b30ac..9df3ec0f3ef4f4162a7c81009024eea61b8f4866 100644
--- a/include/llvm/Transforms/Coroutines.h
+++ b/include/llvm/Transforms/Coroutines.h
@@ -1,9 +1,8 @@
 //===-- Coroutines.h - Coroutine Transformations ----------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 // Declare accessor functions for coroutine lowering passes.
diff --git a/include/llvm/Transforms/IPO.h b/include/llvm/Transforms/IPO.h
index 11d363b1200bca4e26a57e6dc30396832be07fa4..4f7a1bae12b99559c91d732c46dd416a263d4c7f 100644
--- a/include/llvm/Transforms/IPO.h
+++ b/include/llvm/Transforms/IPO.h
@@ -1,9 +1,8 @@
 //===- llvm/Transforms/IPO.h - Interprocedural Transformations --*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Transforms/IPO/AlwaysInliner.h b/include/llvm/Transforms/IPO/AlwaysInliner.h
index b52c0fdbd2c937fe7f883e3cc78bce5ffd9fed79..64e25230f6da43e5f04d2bb6c23365d04736a521 100644
--- a/include/llvm/Transforms/IPO/AlwaysInliner.h
+++ b/include/llvm/Transforms/IPO/AlwaysInliner.h
@@ -1,9 +1,8 @@
 //===-- AlwaysInliner.h - Pass to inline "always_inline" functions --------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
diff --git a/include/llvm/Transforms/IPO/ArgumentPromotion.h b/include/llvm/Transforms/IPO/ArgumentPromotion.h
index 49ca6cc73393aba3357a273db0cbea497f403e04..c8afb7bdcd65fe9bae4fd77594223fdd051c3708 100644
--- a/include/llvm/Transforms/IPO/ArgumentPromotion.h
+++ b/include/llvm/Transforms/IPO/ArgumentPromotion.h
@@ -1,9 +1,8 @@
 //===- ArgumentPromotion.h - Promote by-reference arguments -----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/Transforms/IPO/CalledValuePropagation.h b/include/llvm/Transforms/IPO/CalledValuePropagation.h
index 352bdc7ac17f13be93049e04778bf1c5c414a2d9..c2626d0867b4d9c00279e29144ea3ebf6b4e1a85 100644
--- a/include/llvm/Transforms/IPO/CalledValuePropagation.h
+++ b/include/llvm/Transforms/IPO/CalledValuePropagation.h
@@ -1,9 +1,8 @@
 //===- CalledValuePropagation.h - Propagate called values -------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Transforms/IPO/ConstantMerge.h b/include/llvm/Transforms/IPO/ConstantMerge.h
index e04d3ae1a40edbd60b9640acf817f61d2bc30be3..12d38b5f58fa0d20145383801aa883c039e9c263 100644
--- a/include/llvm/Transforms/IPO/ConstantMerge.h
+++ b/include/llvm/Transforms/IPO/ConstantMerge.h
@@ -1,9 +1,8 @@
 //===- ConstantMerge.h - Merge duplicate global constants -------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Transforms/IPO/CrossDSOCFI.h b/include/llvm/Transforms/IPO/CrossDSOCFI.h
index 0979f5b79e868245bd9b2941088562e93129cd30..8440df6397299b83aeb0e2ea7502ebb75c929dc0 100644
--- a/include/llvm/Transforms/IPO/CrossDSOCFI.h
+++ b/include/llvm/Transforms/IPO/CrossDSOCFI.h
@@ -1,9 +1,8 @@
 //===-- CrossDSOCFI.cpp - Externalize this module's CFI checks --*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Transforms/IPO/DeadArgumentElimination.h b/include/llvm/Transforms/IPO/DeadArgumentElimination.h
index ba5666f20a9bf072e73cfc874dad20c24049d1bf..73797bc10017c3dc3c31616732d8d5880e61e1ef 100644
--- a/include/llvm/Transforms/IPO/DeadArgumentElimination.h
+++ b/include/llvm/Transforms/IPO/DeadArgumentElimination.h
@@ -1,9 +1,8 @@
 //===- DeadArgumentElimination.h - Eliminate Dead Args ----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Transforms/IPO/ElimAvailExtern.h b/include/llvm/Transforms/IPO/ElimAvailExtern.h
index 94cb954fd2d5a804c82c03e589914c855c46b144..92c319b3cce30a20d5b2d69cfd6672de772325b9 100644
--- a/include/llvm/Transforms/IPO/ElimAvailExtern.h
+++ b/include/llvm/Transforms/IPO/ElimAvailExtern.h
@@ -1,9 +1,8 @@
 //===- ElimAvailExtern.h - Optimize Global Variables ------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Transforms/IPO/ForceFunctionAttrs.h b/include/llvm/Transforms/IPO/ForceFunctionAttrs.h
index ff8a6546f0591a125b2d96d157575a745e5fd94b..7379009b2592c8c512e4a8b6bae27d90c00689cd 100644
--- a/include/llvm/Transforms/IPO/ForceFunctionAttrs.h
+++ b/include/llvm/Transforms/IPO/ForceFunctionAttrs.h
@@ -1,9 +1,8 @@
 //===-- ForceFunctionAttrs.h - Force function attrs for debugging ---------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/include/llvm/Transforms/IPO/FunctionAttrs.h b/include/llvm/Transforms/IPO/FunctionAttrs.h
index 901fed7a0fa4f1fe7e51bda2baaf7d00de3b6e73..ce61eea05c7998d32024fc9f1e2eaac7643091ee 100644
--- a/include/llvm/Transforms/IPO/FunctionAttrs.h
+++ b/include/llvm/Transforms/IPO/FunctionAttrs.h
@@ -1,9 +1,8 @@
 //===- FunctionAttrs.h - Compute function attributes ------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Transforms/IPO/FunctionImport.h b/include/llvm/Transforms/IPO/FunctionImport.h
index c2103b637266d877264a4361b9d7e9b266601b84..bbf270c400af1facc1160d1e9cb3b510f5b2967b 100644
--- a/include/llvm/Transforms/IPO/FunctionImport.h
+++ b/include/llvm/Transforms/IPO/FunctionImport.h
@@ -1,9 +1,8 @@
 //===- llvm/Transforms/IPO/FunctionImport.h - ThinLTO importing -*- C++ -*-===//
 //
-//                      The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/Transforms/IPO/GlobalDCE.h b/include/llvm/Transforms/IPO/GlobalDCE.h
index 7ca241f4645a9153c3094d734dda1056ae3066d3..c434484d1ae350a0550286adfa7eda2ff005d4f7 100644
--- a/include/llvm/Transforms/IPO/GlobalDCE.h
+++ b/include/llvm/Transforms/IPO/GlobalDCE.h
@@ -1,9 +1,8 @@
 //===-- GlobalDCE.h - DCE unreachable internal functions ------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Transforms/IPO/GlobalOpt.h b/include/llvm/Transforms/IPO/GlobalOpt.h
index 5b4878604eab13b89f2e80bc7a3b8f2842a33172..48a861ff2cf8f488d94525ff4638f0988e2f359f 100644
--- a/include/llvm/Transforms/IPO/GlobalOpt.h
+++ b/include/llvm/Transforms/IPO/GlobalOpt.h
@@ -1,9 +1,8 @@
 //===- GlobalOpt.h - Optimize Global Variables ------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Transforms/IPO/GlobalSplit.h b/include/llvm/Transforms/IPO/GlobalSplit.h
index 56cefb7886fec9df85f9afdd5700bb3ef89377ec..690b23a2d78535a227dc602396ae54c3c0658988 100644
--- a/include/llvm/Transforms/IPO/GlobalSplit.h
+++ b/include/llvm/Transforms/IPO/GlobalSplit.h
@@ -1,9 +1,8 @@
 //===- GlobalSplit.h - global variable splitter -----------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Transforms/IPO/HotColdSplitting.h b/include/llvm/Transforms/IPO/HotColdSplitting.h
index 57e9a9e691873b4b9a71904b8a692df81473fe51..73668844590d74f3b9e80d4b743087a83258c4fe 100644
--- a/include/llvm/Transforms/IPO/HotColdSplitting.h
+++ b/include/llvm/Transforms/IPO/HotColdSplitting.h
@@ -1,9 +1,8 @@
 //===- HotColdSplitting.h ---- Outline Cold Regions -------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //===----------------------------------------------------------------------===//
 //
 // This pass outlines cold regions to a separate function.
diff --git a/include/llvm/Transforms/IPO/InferFunctionAttrs.h b/include/llvm/Transforms/IPO/InferFunctionAttrs.h
index 54e1c243ae272a816bbe9354f930437fd446c193..bb7907fb8ac8b091d4c4870645d866a1d1817f37 100644
--- a/include/llvm/Transforms/IPO/InferFunctionAttrs.h
+++ b/include/llvm/Transforms/IPO/InferFunctionAttrs.h
@@ -1,9 +1,8 @@
 //===-- InferFunctionAttrs.h - Infer implicit function attributes ---------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
diff --git a/include/llvm/Transforms/IPO/Inliner.h b/include/llvm/Transforms/IPO/Inliner.h
index 610e4500e4b1b3fd94a1f968017332110613b2eb..8202b94d5a937b26a50e771f859fe48cf22f4cd2 100644
--- a/include/llvm/Transforms/IPO/Inliner.h
+++ b/include/llvm/Transforms/IPO/Inliner.h
@@ -1,9 +1,8 @@
 //===- Inliner.h - Inliner pass and infrastructure --------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/Transforms/IPO/Internalize.h b/include/llvm/Transforms/IPO/Internalize.h
index 45d676d9f77b5640ef2821d8697aaed54bd470a6..a7efd3e172a89be44fac3493a38779b99d0c758d 100644
--- a/include/llvm/Transforms/IPO/Internalize.h
+++ b/include/llvm/Transforms/IPO/Internalize.h
@@ -1,9 +1,8 @@
 //====- Internalize.h - Internalization API ---------------------*- C++ -*-===//
 //
-//                      The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Transforms/IPO/LowerTypeTests.h b/include/llvm/Transforms/IPO/LowerTypeTests.h
index bc448386b63da9af6531beaf5c4acca176f977bc..39b23f5957db701cd56052bb5105f4d5978b06b8 100644
--- a/include/llvm/Transforms/IPO/LowerTypeTests.h
+++ b/include/llvm/Transforms/IPO/LowerTypeTests.h
@@ -1,9 +1,8 @@
 //===- LowerTypeTests.h - type metadata lowering pass -----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Transforms/IPO/PartialInlining.h b/include/llvm/Transforms/IPO/PartialInlining.h
index ec6dd36dae06eb6d23535697be277b0f743eed27..3b8297d6598756ad2e5df3616ff24b3fcb4cef8f 100644
--- a/include/llvm/Transforms/IPO/PartialInlining.h
+++ b/include/llvm/Transforms/IPO/PartialInlining.h
@@ -1,9 +1,8 @@
 //===- PartialInlining.h - Inline parts of functions ------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Transforms/IPO/PassManagerBuilder.h b/include/llvm/Transforms/IPO/PassManagerBuilder.h
index 276306f686ffacfd8718907fad2217ad3c59d1f1..f82cfdc063d54c4fe301d1fea5462931283129dc 100644
--- a/include/llvm/Transforms/IPO/PassManagerBuilder.h
+++ b/include/llvm/Transforms/IPO/PassManagerBuilder.h
@@ -1,9 +1,8 @@
 // llvm/Transforms/IPO/PassManagerBuilder.h - Build Standard Pass -*- C++ -*-=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Transforms/IPO/SCCP.h b/include/llvm/Transforms/IPO/SCCP.h
index fdb7865fbac31113c1b51815467d1d4da55f99e0..3c40d44ca9de5ec2742b3aa77cca13b5e14dd1e2 100644
--- a/include/llvm/Transforms/IPO/SCCP.h
+++ b/include/llvm/Transforms/IPO/SCCP.h
@@ -1,9 +1,8 @@
 //===- SCCP.h - Sparse Conditional Constant Propagation ---------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Transforms/IPO/SampleProfile.h b/include/llvm/Transforms/IPO/SampleProfile.h
index af4a933ec1f66e0010be1b471ba1a836875dbb06..a5ad44551bf64aa751622c132f0352312b3a3e88 100644
--- a/include/llvm/Transforms/IPO/SampleProfile.h
+++ b/include/llvm/Transforms/IPO/SampleProfile.h
@@ -1,9 +1,8 @@
 //===- SampleProfile.h - SamplePGO pass ---------- --------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Transforms/IPO/StripDeadPrototypes.h b/include/llvm/Transforms/IPO/StripDeadPrototypes.h
index 5a05cd75c9d556b7f25b31453fe06a2e44775ec8..f4a15c36afc919e10a2e2411094d03aa363a15f8 100644
--- a/include/llvm/Transforms/IPO/StripDeadPrototypes.h
+++ b/include/llvm/Transforms/IPO/StripDeadPrototypes.h
@@ -1,9 +1,8 @@
 //===-- StripDeadPrototypes.h - Remove unused function declarations -------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Transforms/IPO/ThinLTOBitcodeWriter.h b/include/llvm/Transforms/IPO/ThinLTOBitcodeWriter.h
index bf04bbfe92d81c5412fc087b5a63a02929778241..7acb922b37e12304b4206c8acf40840b543ebae9 100644
--- a/include/llvm/Transforms/IPO/ThinLTOBitcodeWriter.h
+++ b/include/llvm/Transforms/IPO/ThinLTOBitcodeWriter.h
@@ -1,9 +1,8 @@
 //===- ThinLTOBitcodeWriter.h - Bitcode writing pass for ThinLTO ----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Transforms/IPO/WholeProgramDevirt.h b/include/llvm/Transforms/IPO/WholeProgramDevirt.h
index bf2c79b0751e5c03d5e4153c62c9072cc3fded19..509fcc867060d08e3a6cde8c35c2f4e026269bec 100644
--- a/include/llvm/Transforms/IPO/WholeProgramDevirt.h
+++ b/include/llvm/Transforms/IPO/WholeProgramDevirt.h
@@ -1,9 +1,8 @@
 //===- WholeProgramDevirt.h - Whole-program devirt pass ---------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Transforms/InstCombine/InstCombine.h b/include/llvm/Transforms/InstCombine/InstCombine.h
index ab25fe08553a410aa2936922b5a09c36d5b4b086..8894d96e591fff605753f762ececa75c66f8d899 100644
--- a/include/llvm/Transforms/InstCombine/InstCombine.h
+++ b/include/llvm/Transforms/InstCombine/InstCombine.h
@@ -1,9 +1,8 @@
 //===- InstCombine.h - InstCombine pass -------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/include/llvm/Transforms/InstCombine/InstCombineWorklist.h b/include/llvm/Transforms/InstCombine/InstCombineWorklist.h
index f860b4b865553bd29358b633247827fcffe1c876..6c33bdbafbd2e6f6305b90bd164ab798d1039352 100644
--- a/include/llvm/Transforms/InstCombine/InstCombineWorklist.h
+++ b/include/llvm/Transforms/InstCombine/InstCombineWorklist.h
@@ -1,9 +1,8 @@
 //===- InstCombineWorklist.h - Worklist for InstCombine pass ----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/Transforms/Instrumentation.h b/include/llvm/Transforms/Instrumentation.h
index 3624cd266af5e780239a8b06de84845470a52a42..29cb281813682845996c34c038237dd34751d653 100644
--- a/include/llvm/Transforms/Instrumentation.h
+++ b/include/llvm/Transforms/Instrumentation.h
@@ -1,9 +1,8 @@
 //===- Transforms/Instrumentation.h - Instrumentation passes ----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -152,17 +151,9 @@ ModulePass *createAddressSanitizerModulePass(bool CompileKernel = false,
                                              bool UseGlobalsGC = true,
                                              bool UseOdrIndicator = true);
 
-// Insert MemorySanitizer instrumentation (detection of uninitialized reads)
-FunctionPass *createMemorySanitizerPass(int TrackOrigins = 0,
-                                        bool Recover = false,
-                                        bool EnableKmsan = false);
-
 FunctionPass *createHWAddressSanitizerPass(bool CompileKernel = false,
                                            bool Recover = false);
 
-// Insert ThreadSanitizer (race detection) instrumentation
-FunctionPass *createThreadSanitizerPass();
-
 // Insert DataFlowSanitizer (dynamic data flow analysis) instrumentation
 ModulePass *createDataFlowSanitizerPass(
     const std::vector<std::string> &ABIListFiles = std::vector<std::string>(),
@@ -230,7 +221,6 @@ static inline uint32_t scaleBranchCount(uint64_t Count, uint64_t Scale) {
   assert(Scaled <= std::numeric_limits<uint32_t>::max() && "overflow 32-bits");
   return Scaled;
 }
-
 } // end namespace llvm
 
 #endif // LLVM_TRANSFORMS_INSTRUMENTATION_H
diff --git a/include/llvm/Transforms/Instrumentation/BoundsChecking.h b/include/llvm/Transforms/Instrumentation/BoundsChecking.h
index 3d4f62c121c25c4576e5587637a083bfff43ca6e..120c6a8fb09f7a3c74e4d8d4fc15067ad08a09c4 100644
--- a/include/llvm/Transforms/Instrumentation/BoundsChecking.h
+++ b/include/llvm/Transforms/Instrumentation/BoundsChecking.h
@@ -1,9 +1,8 @@
 //===- BoundsChecking.h - Bounds checking instrumentation -------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/Transforms/Instrumentation/CGProfile.h b/include/llvm/Transforms/Instrumentation/CGProfile.h
index c06c1a28715e97375ed0b065b05f2a8ed51ae87f..28fd3804dec91b4e837f361f79ed7792385e2f6b 100644
--- a/include/llvm/Transforms/Instrumentation/CGProfile.h
+++ b/include/llvm/Transforms/Instrumentation/CGProfile.h
@@ -1,9 +1,8 @@
 //===- Transforms/Instrumentation/CGProfile.h -------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/include/llvm/Transforms/Instrumentation/ControlHeightReduction.h b/include/llvm/Transforms/Instrumentation/ControlHeightReduction.h
index 460342d1631b4c4f51746cb26ce4d2f2b08d551d..18b428582046a736ca3777fa7e81cde499e386ea 100644
--- a/include/llvm/Transforms/Instrumentation/ControlHeightReduction.h
+++ b/include/llvm/Transforms/Instrumentation/ControlHeightReduction.h
@@ -1,9 +1,8 @@
 //===- ControlHeightReduction.h - Control Height Reduction ------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Transforms/Instrumentation/GCOVProfiler.h b/include/llvm/Transforms/Instrumentation/GCOVProfiler.h
index dd55fbe29eedf4d62d814d64be5f496650ce5197..b3971e49754ea774b29b16e5f8b894e7a2b9447c 100644
--- a/include/llvm/Transforms/Instrumentation/GCOVProfiler.h
+++ b/include/llvm/Transforms/Instrumentation/GCOVProfiler.h
@@ -1,9 +1,8 @@
 //===- Transforms/Instrumentation/GCOVProfiler.h ----------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/include/llvm/Transforms/Instrumentation/InstrProfiling.h b/include/llvm/Transforms/Instrumentation/InstrProfiling.h
index 13fb3db4ae6fd571e19740117fdc3382da2e1a2b..4fad6506810bd91a1b18f4c3278ca5896b239fec 100644
--- a/include/llvm/Transforms/Instrumentation/InstrProfiling.h
+++ b/include/llvm/Transforms/Instrumentation/InstrProfiling.h
@@ -1,9 +1,8 @@
 //===- Transforms/Instrumentation/InstrProfiling.h --------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/include/llvm/Transforms/Instrumentation/MemorySanitizer.h b/include/llvm/Transforms/Instrumentation/MemorySanitizer.h
new file mode 100644
index 0000000000000000000000000000000000000000..0739d9e58a61fca091e0691cb3038c9628e5b22d
--- /dev/null
+++ b/include/llvm/Transforms/Instrumentation/MemorySanitizer.h
@@ -0,0 +1,50 @@
+//===- Transforms/Instrumentation/MemorySanitizer.h - MSan Pass -----------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines the memoy sanitizer pass.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_TRANSFORMS_INSTRUMENTATION_MEMORYSANITIZER_H
+#define LLVM_TRANSFORMS_INSTRUMENTATION_MEMORYSANITIZER_H
+
+#include "llvm/IR/PassManager.h"
+#include "llvm/Pass.h"
+
+namespace llvm {
+
+struct MemorySanitizerOptions {
+  MemorySanitizerOptions() = default;
+  MemorySanitizerOptions(int TrackOrigins, bool Recover, bool Kernel)
+      : TrackOrigins(TrackOrigins), Recover(Recover), Kernel(Kernel) {}
+  int TrackOrigins = 0;
+  bool Recover = false;
+  bool Kernel = false;
+};
+
+// Insert MemorySanitizer instrumentation (detection of uninitialized reads)
+FunctionPass *
+createMemorySanitizerLegacyPassPass(MemorySanitizerOptions Options = {});
+
+/// A function pass for msan instrumentation.
+///
+/// Instruments functions to detect unitialized reads. This function pass
+/// inserts calls to runtime library functions. If the functions aren't declared
+/// yet, the pass inserts the declarations. Otherwise the existing globals are
+/// used.
+struct MemorySanitizerPass : public PassInfoMixin<MemorySanitizerPass> {
+  MemorySanitizerPass(MemorySanitizerOptions Options) : Options(Options) {}
+
+  PreservedAnalyses run(Function &F, FunctionAnalysisManager &FAM);
+
+private:
+  MemorySanitizerOptions Options;
+};
+}
+
+#endif /* LLVM_TRANSFORMS_INSTRUMENTATION_MEMORYSANITIZER_H */
diff --git a/include/llvm/Transforms/Instrumentation/PGOInstrumentation.h b/include/llvm/Transforms/Instrumentation/PGOInstrumentation.h
index fdc5df68a669d551eba4d010b3e548fa2419cc9b..4f22e28232e72f1b357c13ba999eabf38c2420a0 100644
--- a/include/llvm/Transforms/Instrumentation/PGOInstrumentation.h
+++ b/include/llvm/Transforms/Instrumentation/PGOInstrumentation.h
@@ -1,9 +1,8 @@
 //===- Transforms/Instrumentation/PGOInstrumentation.h ----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Transforms/Instrumentation/ThreadSanitizer.h b/include/llvm/Transforms/Instrumentation/ThreadSanitizer.h
new file mode 100644
index 0000000000000000000000000000000000000000..b4e7d9924ff611ca55265de692d909d30d391844
--- /dev/null
+++ b/include/llvm/Transforms/Instrumentation/ThreadSanitizer.h
@@ -0,0 +1,32 @@
+//===- Transforms/Instrumentation/ThreadSanitizer.h - TSan Pass -----------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines the thread sanitizer pass.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_TRANSFORMS_INSTRUMENTATION_THREADSANITIZER_H
+#define LLVM_TRANSFORMS_INSTRUMENTATION_THREADSANITIZER_H
+
+#include "llvm/IR/PassManager.h"
+#include "llvm/Pass.h"
+
+namespace llvm {
+// Insert ThreadSanitizer (race detection) instrumentation
+FunctionPass *createThreadSanitizerLegacyPassPass();
+
+/// A function pass for tsan instrumentation.
+///
+/// Instruments functions to detect race conditions reads. This function pass
+/// inserts calls to runtime library functions. If the functions aren't declared
+/// yet, the pass inserts the declarations. Otherwise the existing globals are
+struct ThreadSanitizerPass : public PassInfoMixin<ThreadSanitizerPass> {
+  PreservedAnalyses run(Function &F, FunctionAnalysisManager &FAM);
+};
+} // namespace llvm
+#endif /* LLVM_TRANSFORMS_INSTRUMENTATION_THREADSANITIZER_H */
diff --git a/include/llvm/Transforms/ObjCARC.h b/include/llvm/Transforms/ObjCARC.h
index 1897adc2ffbf5960efb3604837135b6a3f03980e..2f114c75e2e2ac0e96f1b621adbbc482ade9c668 100644
--- a/include/llvm/Transforms/ObjCARC.h
+++ b/include/llvm/Transforms/ObjCARC.h
@@ -1,9 +1,8 @@
 //===-- ObjCARC.h - ObjCARC Scalar Transformations --------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Transforms/Scalar.h b/include/llvm/Transforms/Scalar.h
index 9019c3be4944d6a12c23f0ae5c24371dad8ef3b5..97a4c06cdc904a4775545ebe0bd864e382f0d51a 100644
--- a/include/llvm/Transforms/Scalar.h
+++ b/include/llvm/Transforms/Scalar.h
@@ -1,9 +1,8 @@
 //===-- Scalar.h - Scalar Transformations -----------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -358,6 +357,12 @@ Pass *createLowerAtomicPass();
 //
 Pass *createLowerGuardIntrinsicPass();
 
+//===----------------------------------------------------------------------===//
+//
+// LowerWidenableCondition - Lower widenable condition to i1 true.
+//
+Pass *createLowerWidenableConditionPass();
+
 //===----------------------------------------------------------------------===//
 //
 // MergeICmps - Merge integer comparison chains into a memcmp
@@ -471,6 +476,7 @@ FunctionPass *createLoopDataPrefetchPass();
 
 ///===---------------------------------------------------------------------===//
 ModulePass *createNameAnonGlobalPass();
+ModulePass *createCanonicalizeAliasesPass();
 
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Transforms/Scalar/ADCE.h b/include/llvm/Transforms/Scalar/ADCE.h
index f98af62c1a76f06e89ded4f861859fcc9d9cf453..7d8b7ae68c004614a9b40d5eed1b9abc4b3a0cb9 100644
--- a/include/llvm/Transforms/Scalar/ADCE.h
+++ b/include/llvm/Transforms/Scalar/ADCE.h
@@ -1,9 +1,8 @@
 //===- ADCE.h - Aggressive dead code elimination ----------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Transforms/Scalar/AlignmentFromAssumptions.h b/include/llvm/Transforms/Scalar/AlignmentFromAssumptions.h
index 61975036e9ff4bc49b641057794f62b16760b028..fb1687e1ac5dea3071dd81b4ee396a954252d8e2 100644
--- a/include/llvm/Transforms/Scalar/AlignmentFromAssumptions.h
+++ b/include/llvm/Transforms/Scalar/AlignmentFromAssumptions.h
@@ -1,9 +1,8 @@
 //===---- AlignmentFromAssumptions.h ----------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Transforms/Scalar/BDCE.h b/include/llvm/Transforms/Scalar/BDCE.h
index d7d2730a80336e51c7db0ae7874d8dae9f82b806..996622bccdba502656b99820a11e0709f4561a8f 100644
--- a/include/llvm/Transforms/Scalar/BDCE.h
+++ b/include/llvm/Transforms/Scalar/BDCE.h
@@ -1,9 +1,8 @@
 //===---- BDCE.cpp - Bit-tracking dead code elimination ---------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Transforms/Scalar/CallSiteSplitting.h b/include/llvm/Transforms/Scalar/CallSiteSplitting.h
index b2ca2a1c09aee550b49b4631ce10280fa4c8a7bb..b6055639e8a8fc9d001073d852594fe2ea801706 100644
--- a/include/llvm/Transforms/Scalar/CallSiteSplitting.h
+++ b/include/llvm/Transforms/Scalar/CallSiteSplitting.h
@@ -1,9 +1,8 @@
 //===- CallSiteSplitting..h - Callsite Splitting ------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/Transforms/Scalar/ConstantHoisting.h b/include/llvm/Transforms/Scalar/ConstantHoisting.h
index ba32e122fa1017022dcc69a910e584429ba56b32..29311ca2dae14544b02a320f25748b7736d1d33b 100644
--- a/include/llvm/Transforms/Scalar/ConstantHoisting.h
+++ b/include/llvm/Transforms/Scalar/ConstantHoisting.h
@@ -1,9 +1,8 @@
 //==- ConstantHoisting.h - Prepare code for expensive constants --*- C++ -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Transforms/Scalar/CorrelatedValuePropagation.h b/include/llvm/Transforms/Scalar/CorrelatedValuePropagation.h
index 20930699b5578c6c327935d3b65e45d7a4e8ab36..25795de5d951fa2b3c1cdc0543498f01efad5109 100644
--- a/include/llvm/Transforms/Scalar/CorrelatedValuePropagation.h
+++ b/include/llvm/Transforms/Scalar/CorrelatedValuePropagation.h
@@ -1,9 +1,8 @@
 //===- CorrelatedValuePropagation.h -----------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/Transforms/Scalar/DCE.h b/include/llvm/Transforms/Scalar/DCE.h
index 273346cf81d95c12be2017efd00a310824b0ab7a..974e4b20d152d38737f16f9cd65d08e3fbf88264 100644
--- a/include/llvm/Transforms/Scalar/DCE.h
+++ b/include/llvm/Transforms/Scalar/DCE.h
@@ -1,9 +1,8 @@
 //===- DCE.h - Dead code elimination ----------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Transforms/Scalar/DeadStoreElimination.h b/include/llvm/Transforms/Scalar/DeadStoreElimination.h
index cfeb2181423217c60d9e052c10d20ce93dfbb05c..b66b0de90c790bef33834d941387829b9ab0c39c 100644
--- a/include/llvm/Transforms/Scalar/DeadStoreElimination.h
+++ b/include/llvm/Transforms/Scalar/DeadStoreElimination.h
@@ -1,9 +1,8 @@
 //===- DeadStoreElimination.h - Fast Dead Store Elimination -----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Transforms/Scalar/DivRemPairs.h b/include/llvm/Transforms/Scalar/DivRemPairs.h
index 0a4346f33b12b8e080399a4d38465bbb9568ea38..7401e02cb4ab455e6c4411dc251263171317cb1b 100644
--- a/include/llvm/Transforms/Scalar/DivRemPairs.h
+++ b/include/llvm/Transforms/Scalar/DivRemPairs.h
@@ -1,9 +1,8 @@
 //===- DivRemPairs.h - Hoist/decompose integer division and remainder -----===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Transforms/Scalar/EarlyCSE.h b/include/llvm/Transforms/Scalar/EarlyCSE.h
index faf03a4ec489a51800c04e7f2b0c1574460c8afb..1e7fd71dcbf4db73be4ea669f1237139e26b31cd 100644
--- a/include/llvm/Transforms/Scalar/EarlyCSE.h
+++ b/include/llvm/Transforms/Scalar/EarlyCSE.h
@@ -1,9 +1,8 @@
 //===- EarlyCSE.h - Simple and fast CSE pass --------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Transforms/Scalar/Float2Int.h b/include/llvm/Transforms/Scalar/Float2Int.h
index 206ee980109b4fdbb20f73ee851dc409740b7a96..06aeb83225273a475c2d566840a2ce5d2de95c52 100644
--- a/include/llvm/Transforms/Scalar/Float2Int.h
+++ b/include/llvm/Transforms/Scalar/Float2Int.h
@@ -1,9 +1,8 @@
 //===-- Float2Int.h - Demote floating point ops to work on integers -------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Transforms/Scalar/GVN.h b/include/llvm/Transforms/Scalar/GVN.h
index 784de7f9fe26bfbe63daaad0353d8a633844d132..9fe00a9e7f2d79e434cf6698ce7ad004ecd5dcac 100644
--- a/include/llvm/Transforms/Scalar/GVN.h
+++ b/include/llvm/Transforms/Scalar/GVN.h
@@ -1,9 +1,8 @@
 //===- GVN.h - Eliminate redundant values and loads -------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
@@ -27,6 +26,7 @@
 #include "llvm/IR/Dominators.h"
 #include "llvm/IR/InstrTypes.h"
 #include "llvm/IR/PassManager.h"
+#include "llvm/IR/ValueHandle.h"
 #include "llvm/Support/Allocator.h"
 #include "llvm/Support/Compiler.h"
 #include <cstdint>
@@ -180,7 +180,12 @@ private:
 
   // Map the block to reversed postorder traversal number. It is used to
   // find back edge easily.
-  DenseMap<const BasicBlock *, uint32_t> BlockRPONumber;
+  DenseMap<AssertingVH<BasicBlock>, uint32_t> BlockRPONumber;
+
+  // This is set 'true' initially and also when new blocks have been added to
+  // the function being analyzed. This boolean is used to control the updating
+  // of BlockRPONumber prior to accessing the contents of BlockRPONumber.
+  bool InvalidBlockRPONumbers = true;
 
   using LoadDepVect = SmallVector<NonLocalDepResult, 64>;
   using AvailValInBlkVect = SmallVector<gvn::AvailableValueInBlock, 64>;
diff --git a/include/llvm/Transforms/Scalar/GVNExpression.h b/include/llvm/Transforms/Scalar/GVNExpression.h
index 8b346969b1e9045805499bc535c278e141b8edf3..3dc4515f85a15cd5c21e54a01090856a39d01a19 100644
--- a/include/llvm/Transforms/Scalar/GVNExpression.h
+++ b/include/llvm/Transforms/Scalar/GVNExpression.h
@@ -1,9 +1,8 @@
 //===- GVNExpression.h - GVN Expression classes -----------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Transforms/Scalar/GuardWidening.h b/include/llvm/Transforms/Scalar/GuardWidening.h
index 2bc0940ac715ea80174d1809a9eff33170446112..3adea0529c61886e15b774fa884570024f8a0ad4 100644
--- a/include/llvm/Transforms/Scalar/GuardWidening.h
+++ b/include/llvm/Transforms/Scalar/GuardWidening.h
@@ -1,9 +1,8 @@
 //===- GuardWidening.h - ----------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Transforms/Scalar/IVUsersPrinter.h b/include/llvm/Transforms/Scalar/IVUsersPrinter.h
index fad00d86a95f1a652e06b0ddf22beb998ed57881..a1f20d9ca98351b9fb7de5af4245fd68f1002a0a 100644
--- a/include/llvm/Transforms/Scalar/IVUsersPrinter.h
+++ b/include/llvm/Transforms/Scalar/IVUsersPrinter.h
@@ -1,9 +1,8 @@
 //===- IVUsersPrinter.h - Induction Variable Users Printing -----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/Transforms/Scalar/IndVarSimplify.h b/include/llvm/Transforms/Scalar/IndVarSimplify.h
index e321c8fc6e9cf0549e1cb574f6d0ef228a4ffb44..3c20537ab76aaeb2addd403e6fe23f11400d8305 100644
--- a/include/llvm/Transforms/Scalar/IndVarSimplify.h
+++ b/include/llvm/Transforms/Scalar/IndVarSimplify.h
@@ -1,9 +1,8 @@
 //===- IndVarSimplify.h - Induction Variable Simplification -----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Transforms/Scalar/InductiveRangeCheckElimination.h b/include/llvm/Transforms/Scalar/InductiveRangeCheckElimination.h
index 311c549b8326ca0aec625bacb4f4e335a462b74a..b1e700714e5195369b4869afb653f12df5a24a25 100644
--- a/include/llvm/Transforms/Scalar/InductiveRangeCheckElimination.h
+++ b/include/llvm/Transforms/Scalar/InductiveRangeCheckElimination.h
@@ -1,9 +1,8 @@
 //===- InductiveRangeCheckElimination.h - IRCE ------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Transforms/Scalar/InstSimplifyPass.h b/include/llvm/Transforms/Scalar/InstSimplifyPass.h
index da79a13eb7cf9d4f4e0d1696409f4cd101ad23da..0c30b62605366f6cc3acad344b6613f3c8d37489 100644
--- a/include/llvm/Transforms/Scalar/InstSimplifyPass.h
+++ b/include/llvm/Transforms/Scalar/InstSimplifyPass.h
@@ -1,9 +1,8 @@
 //===- InstSimplifyPass.h ---------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/include/llvm/Transforms/Scalar/JumpThreading.h b/include/llvm/Transforms/Scalar/JumpThreading.h
index 78518941ceda94b9789e044cc7de4036b42cbced..0464d40c45e68913104ec4ef4144785748b4c9e8 100644
--- a/include/llvm/Transforms/Scalar/JumpThreading.h
+++ b/include/llvm/Transforms/Scalar/JumpThreading.h
@@ -1,9 +1,8 @@
 //===- JumpThreading.h - thread control through conditional BBs -*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -23,7 +22,7 @@
 #include "llvm/Analysis/AliasAnalysis.h"
 #include "llvm/Analysis/BlockFrequencyInfo.h"
 #include "llvm/Analysis/BranchProbabilityInfo.h"
-#include "llvm/IR/DomTreeUpdater.h"
+#include "llvm/Analysis/DomTreeUpdater.h"
 #include "llvm/IR/ValueHandle.h"
 #include <memory>
 #include <utility>
@@ -139,7 +138,11 @@ public:
   bool ProcessImpliedCondition(BasicBlock *BB);
 
   bool SimplifyPartiallyRedundantLoad(LoadInst *LI);
+  void UnfoldSelectInstr(BasicBlock *Pred, BasicBlock *BB, SelectInst *SI,
+                         PHINode *SIUse, unsigned Idx);
+
   bool TryToUnfoldSelect(CmpInst *CondCmp, BasicBlock *BB);
+  bool TryToUnfoldSelect(SwitchInst *SI, BasicBlock *BB);
   bool TryToUnfoldSelectInCurrBB(BasicBlock *BB);
 
   bool ProcessGuards(BasicBlock *BB);
diff --git a/include/llvm/Transforms/Scalar/LICM.h b/include/llvm/Transforms/Scalar/LICM.h
index 68ad190c7647db04a81eff628e84df331fdefe89..ea947ea0fa4c221fa15d9557aac7e992c9580727 100644
--- a/include/llvm/Transforms/Scalar/LICM.h
+++ b/include/llvm/Transforms/Scalar/LICM.h
@@ -1,9 +1,8 @@
 //===- LICM.h - Loop Invariant Code Motion Pass -------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Transforms/Scalar/LoopAccessAnalysisPrinter.h b/include/llvm/Transforms/Scalar/LoopAccessAnalysisPrinter.h
index e1b33799578b1744ea20005fbfbc43d067ee1cbf..3f250fc1ce8c3fc701f45dda62b94597423ac316 100644
--- a/include/llvm/Transforms/Scalar/LoopAccessAnalysisPrinter.h
+++ b/include/llvm/Transforms/Scalar/LoopAccessAnalysisPrinter.h
@@ -1,9 +1,8 @@
 //===- llvm/Analysis/LoopAccessAnalysisPrinter.h ----------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/Transforms/Scalar/LoopDataPrefetch.h b/include/llvm/Transforms/Scalar/LoopDataPrefetch.h
index e1ad67ac6fff449d71486dd4ae6e943458c7c228..9ebd5984cea97ba1b23810b5168a25175057dd62 100644
--- a/include/llvm/Transforms/Scalar/LoopDataPrefetch.h
+++ b/include/llvm/Transforms/Scalar/LoopDataPrefetch.h
@@ -1,10 +1,9 @@
 //===-------- LoopDataPrefetch.h - Loop Data Prefetching Pass ---*- C++ -*-===//
 //
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/include/llvm/Transforms/Scalar/LoopDeletion.h b/include/llvm/Transforms/Scalar/LoopDeletion.h
index 7b8cb1e115c97a84d698cf2fccdf01dfe341dd3a..557616e2e6bac954810637e81844ee6f3dc09ab3 100644
--- a/include/llvm/Transforms/Scalar/LoopDeletion.h
+++ b/include/llvm/Transforms/Scalar/LoopDeletion.h
@@ -1,9 +1,8 @@
 //===- LoopDeletion.h - Loop Deletion ---------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Transforms/Scalar/LoopDistribute.h b/include/llvm/Transforms/Scalar/LoopDistribute.h
index 2bf1c9d696d5b051fe4dec0e24907d7e04d794a7..1a82176490c50dbb35f55ea8c40085dd1e74fe4e 100644
--- a/include/llvm/Transforms/Scalar/LoopDistribute.h
+++ b/include/llvm/Transforms/Scalar/LoopDistribute.h
@@ -1,9 +1,8 @@
 //===- LoopDistribute.cpp - Loop Distribution Pass --------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Transforms/Scalar/LoopIdiomRecognize.h b/include/llvm/Transforms/Scalar/LoopIdiomRecognize.h
index 7added8d2c61712ee615ed356624790760d4965a..d2fff8bb5743bc4818dd4253061ed66911baec84 100644
--- a/include/llvm/Transforms/Scalar/LoopIdiomRecognize.h
+++ b/include/llvm/Transforms/Scalar/LoopIdiomRecognize.h
@@ -1,9 +1,8 @@
 //===- LoopIdiomRecognize.h - Loop Idiom Recognize Pass ---------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Transforms/Scalar/LoopInstSimplify.h b/include/llvm/Transforms/Scalar/LoopInstSimplify.h
index 04dc79c3fa5736fdbb1cd44ae75b33bb6b50b446..f6e86d11ed9561c94c1df808be2f8bc31baf2889 100644
--- a/include/llvm/Transforms/Scalar/LoopInstSimplify.h
+++ b/include/llvm/Transforms/Scalar/LoopInstSimplify.h
@@ -1,9 +1,8 @@
 //===- LoopInstSimplify.h - Loop Inst Simplify Pass -------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Transforms/Scalar/LoopLoadElimination.h b/include/llvm/Transforms/Scalar/LoopLoadElimination.h
index b0514a4a7c9895b7df2b008aa6004ea7cce6027e..65b9aabb8f51893d3d7c59cf3c2d1e3e9675c694 100644
--- a/include/llvm/Transforms/Scalar/LoopLoadElimination.h
+++ b/include/llvm/Transforms/Scalar/LoopLoadElimination.h
@@ -1,9 +1,8 @@
 //===- LoopLoadElimination.h ------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Transforms/Scalar/LoopPassManager.h b/include/llvm/Transforms/Scalar/LoopPassManager.h
index 46ebb74c413c5991bd9d0c0a6cd6f3a7b59e1956..61ec58585fd0819ffd75bef77f4479dbfeff9ee8 100644
--- a/include/llvm/Transforms/Scalar/LoopPassManager.h
+++ b/include/llvm/Transforms/Scalar/LoopPassManager.h
@@ -1,9 +1,8 @@
 //===- LoopPassManager.h - Loop pass management -----------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/include/llvm/Transforms/Scalar/LoopPredication.h b/include/llvm/Transforms/Scalar/LoopPredication.h
index 57398bdb6bd1c9b968d8fd838279164e7674bed7..252daafab7a30940d401d4fcc2185603309b38b2 100644
--- a/include/llvm/Transforms/Scalar/LoopPredication.h
+++ b/include/llvm/Transforms/Scalar/LoopPredication.h
@@ -1,9 +1,8 @@
 //===- LoopPredication.h - Guard based loop predication pass ----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Transforms/Scalar/LoopRotation.h b/include/llvm/Transforms/Scalar/LoopRotation.h
index ea8d5618e6f7d5c94f27ef5a5128f03118b3fa4b..254e6072906a2d9696ed03b8e3376d6a0a2cc032 100644
--- a/include/llvm/Transforms/Scalar/LoopRotation.h
+++ b/include/llvm/Transforms/Scalar/LoopRotation.h
@@ -1,9 +1,8 @@
 //===- LoopRotation.h - Loop Rotation -------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Transforms/Scalar/LoopSimplifyCFG.h b/include/llvm/Transforms/Scalar/LoopSimplifyCFG.h
index 7628c7413eac5e92cfd84a53f5c6f018fa1c6cc4..2d718592aef54d35136b2c19fd78e2950746039d 100644
--- a/include/llvm/Transforms/Scalar/LoopSimplifyCFG.h
+++ b/include/llvm/Transforms/Scalar/LoopSimplifyCFG.h
@@ -1,9 +1,8 @@
 //===- LoopSimplifyCFG.cpp - Loop CFG Simplification Pass -------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Transforms/Scalar/LoopSink.h b/include/llvm/Transforms/Scalar/LoopSink.h
index 371a7c8d2c446c526b5d1945cd4d2a00a5550996..234c48cbebc590e1304d63a40deebeedb700c57d 100644
--- a/include/llvm/Transforms/Scalar/LoopSink.h
+++ b/include/llvm/Transforms/Scalar/LoopSink.h
@@ -1,9 +1,8 @@
 //===- LoopSink.h - Loop Sink Pass ------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Transforms/Scalar/LoopStrengthReduce.h b/include/llvm/Transforms/Scalar/LoopStrengthReduce.h
index 62c038a3857d69d6096858d42e8a08be7dfd51c1..5cf805bc49393a8a9a2ac8f298a2bef127a02e82 100644
--- a/include/llvm/Transforms/Scalar/LoopStrengthReduce.h
+++ b/include/llvm/Transforms/Scalar/LoopStrengthReduce.h
@@ -1,9 +1,8 @@
 //===- LoopStrengthReduce.h - Loop Strength Reduce Pass ---------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Transforms/Scalar/LoopUnrollAndJamPass.h b/include/llvm/Transforms/Scalar/LoopUnrollAndJamPass.h
index fc69aa361059e004e2a77791a7f0a8129f304cee..7920269b0fb27f0f2dac15fb17cb8ddaed2942c0 100644
--- a/include/llvm/Transforms/Scalar/LoopUnrollAndJamPass.h
+++ b/include/llvm/Transforms/Scalar/LoopUnrollAndJamPass.h
@@ -1,9 +1,8 @@
 //===- LoopUnrollAndJamPass.h -----------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/Transforms/Scalar/LoopUnrollPass.h b/include/llvm/Transforms/Scalar/LoopUnrollPass.h
index e38e983cc9eb69b3d3efc3f338f506ef13dc7a6a..1445373eb4f2d7a669546521eb67c4364d60818d 100644
--- a/include/llvm/Transforms/Scalar/LoopUnrollPass.h
+++ b/include/llvm/Transforms/Scalar/LoopUnrollPass.h
@@ -1,9 +1,8 @@
 //===- LoopUnrollPass.h -----------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/Transforms/Scalar/LowerAtomic.h b/include/llvm/Transforms/Scalar/LowerAtomic.h
index a4a2e7aafe449e90d4db67df76c342fe7a3c0a20..40f8ca571f19acbba680485d8d8df4a8f2ae939f 100644
--- a/include/llvm/Transforms/Scalar/LowerAtomic.h
+++ b/include/llvm/Transforms/Scalar/LowerAtomic.h
@@ -1,9 +1,8 @@
 //===- LowerAtomic.cpp - Lower atomic intrinsics ----------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/include/llvm/Transforms/Scalar/LowerExpectIntrinsic.h b/include/llvm/Transforms/Scalar/LowerExpectIntrinsic.h
index b6ee6523697c18540e69c7fedae4f351d9639fd7..4e47ff70d55747cfcec9b8442bfe1177d52f2a79 100644
--- a/include/llvm/Transforms/Scalar/LowerExpectIntrinsic.h
+++ b/include/llvm/Transforms/Scalar/LowerExpectIntrinsic.h
@@ -1,9 +1,8 @@
 //===- LowerExpectIntrinsic.h - LowerExpectIntrinsic pass -------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/include/llvm/Transforms/Scalar/LowerGuardIntrinsic.h b/include/llvm/Transforms/Scalar/LowerGuardIntrinsic.h
index a9f19f6b84b4500c4c483968a510c67db8729a1f..ce97b9e4c386907e97ddd9a682d3df79046f2990 100644
--- a/include/llvm/Transforms/Scalar/LowerGuardIntrinsic.h
+++ b/include/llvm/Transforms/Scalar/LowerGuardIntrinsic.h
@@ -1,9 +1,8 @@
 //===--- LowerGuardIntrinsic.h - Lower the guard intrinsic ---------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Transforms/Scalar/LowerWidenableCondition.h b/include/llvm/Transforms/Scalar/LowerWidenableCondition.h
new file mode 100644
index 0000000000000000000000000000000000000000..7c1e64b8f3a934b5071fcc0decb9f2ec7a1a2f57
--- /dev/null
+++ b/include/llvm/Transforms/Scalar/LowerWidenableCondition.h
@@ -0,0 +1,26 @@
+//===--- LowerWidenableCondition.h - Lower the guard intrinsic ---------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This pass lowers the llvm.widenable.condition intrinsic to default value
+// which is i1 true.
+//
+//===----------------------------------------------------------------------===//
+#ifndef LLVM_TRANSFORMS_SCALAR_LOWERWIDENABLECONDITION_H
+#define LLVM_TRANSFORMS_SCALAR_LOWERWIDENABLECONDITION_H
+
+#include "llvm/IR/PassManager.h"
+
+namespace llvm {
+
+struct LowerWidenableConditionPass : PassInfoMixin<LowerWidenableConditionPass> {
+  PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
+};
+
+}
+
+#endif //LLVM_TRANSFORMS_SCALAR_LOWERWIDENABLECONDITION_H
diff --git a/include/llvm/Transforms/Scalar/MakeGuardsExplicit.h b/include/llvm/Transforms/Scalar/MakeGuardsExplicit.h
index 41b4aada2baac97310b301167139e46611ec1b9a..525174734303270c1bcbec1b7a627f1426b55f0a 100644
--- a/include/llvm/Transforms/Scalar/MakeGuardsExplicit.h
+++ b/include/llvm/Transforms/Scalar/MakeGuardsExplicit.h
@@ -1,9 +1,8 @@
 //===-- MakeGuardsExplicit.h - Turn guard intrinsics into guard branches --===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Transforms/Scalar/MemCpyOptimizer.h b/include/llvm/Transforms/Scalar/MemCpyOptimizer.h
index 046c808bd051122dbdee34fd1914de92abb2b7f1..5386f58b2b8201fb61d1bc94bd66e55f0d307e67 100644
--- a/include/llvm/Transforms/Scalar/MemCpyOptimizer.h
+++ b/include/llvm/Transforms/Scalar/MemCpyOptimizer.h
@@ -1,9 +1,8 @@
 //===- MemCpyOptimizer.h - memcpy optimization ------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Transforms/Scalar/MergedLoadStoreMotion.h b/include/llvm/Transforms/Scalar/MergedLoadStoreMotion.h
index 48df09cdec9ec61fcccf541b1e699a3575b7e752..9071a56532f823eeba5761f63471aabeaffc27dd 100644
--- a/include/llvm/Transforms/Scalar/MergedLoadStoreMotion.h
+++ b/include/llvm/Transforms/Scalar/MergedLoadStoreMotion.h
@@ -1,9 +1,8 @@
 //===- MergedLoadStoreMotion.h - merge and hoist/sink load/stores ---------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Transforms/Scalar/NaryReassociate.h b/include/llvm/Transforms/Scalar/NaryReassociate.h
index e835bd5f0761c46b3dc4020fe3a38f8009a859e4..26f5fe185dd526f93ad05d10c5c47fd3e8c7353e 100644
--- a/include/llvm/Transforms/Scalar/NaryReassociate.h
+++ b/include/llvm/Transforms/Scalar/NaryReassociate.h
@@ -1,9 +1,8 @@
 //===- NaryReassociate.h - Reassociate n-ary expressions --------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Transforms/Scalar/NewGVN.h b/include/llvm/Transforms/Scalar/NewGVN.h
index 3f7541863a19f3d17d895bdb78fefb5a3756f0cb..1f3680fec79cfd6541a67c6d5ed05fd2a8f4397c 100644
--- a/include/llvm/Transforms/Scalar/NewGVN.h
+++ b/include/llvm/Transforms/Scalar/NewGVN.h
@@ -1,9 +1,8 @@
 //===- NewGVN.h - Global Value Numbering Pass -------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Transforms/Scalar/PartiallyInlineLibCalls.h b/include/llvm/Transforms/Scalar/PartiallyInlineLibCalls.h
index 7f73831e0eb3c7f416cb2a5c76068a288c0ff301..fd5a06c5051d8acbab4d7b69dcb78c221b52228c 100644
--- a/include/llvm/Transforms/Scalar/PartiallyInlineLibCalls.h
+++ b/include/llvm/Transforms/Scalar/PartiallyInlineLibCalls.h
@@ -1,9 +1,8 @@
 //===--- PartiallyInlineLibCalls.h - Partially inline libcalls --*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Transforms/Scalar/Reassociate.h b/include/llvm/Transforms/Scalar/Reassociate.h
index ba7586dffd9d7df8383affbba9816245b09e415f..487471c65bb17acd189366ba1a9c6867deacf108 100644
--- a/include/llvm/Transforms/Scalar/Reassociate.h
+++ b/include/llvm/Transforms/Scalar/Reassociate.h
@@ -1,9 +1,8 @@
 //===- Reassociate.h - Reassociate binary expressions -----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Transforms/Scalar/RewriteStatepointsForGC.h b/include/llvm/Transforms/Scalar/RewriteStatepointsForGC.h
index 128f176f44203b281c8e618f5e72729e14fa510a..12773c16dcc2972c7060c5294e705fe22d694eec 100644
--- a/include/llvm/Transforms/Scalar/RewriteStatepointsForGC.h
+++ b/include/llvm/Transforms/Scalar/RewriteStatepointsForGC.h
@@ -1,9 +1,8 @@
 //===- RewriteStatepointsForGC.h - ------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Transforms/Scalar/SCCP.h b/include/llvm/Transforms/Scalar/SCCP.h
index 0abbb32fde6a03cd78a5bc21e8826f21acb85bc6..0ffd983eb3e0b70d12a4671c4ce82dfc9cc69968 100644
--- a/include/llvm/Transforms/Scalar/SCCP.h
+++ b/include/llvm/Transforms/Scalar/SCCP.h
@@ -1,9 +1,8 @@
 //===- SCCP.cpp - Sparse Conditional Constant Propagation -------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Transforms/Scalar/SROA.h b/include/llvm/Transforms/Scalar/SROA.h
index b36c6f492be12a2c919f7c6733dee5bf3c7a9cfa..bd3a072013b107603f89f9e283fd7bb28d65f67c 100644
--- a/include/llvm/Transforms/Scalar/SROA.h
+++ b/include/llvm/Transforms/Scalar/SROA.h
@@ -1,9 +1,8 @@
 //===- SROA.h - Scalar Replacement Of Aggregates ----------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/include/llvm/Transforms/Scalar/Scalarizer.h b/include/llvm/Transforms/Scalar/Scalarizer.h
index 1a0b9a2b638c8a2b77ab3f3a300a468ddc3db1a9..81363130e2e3aa7de619332a20f125b52499da67 100644
--- a/include/llvm/Transforms/Scalar/Scalarizer.h
+++ b/include/llvm/Transforms/Scalar/Scalarizer.h
@@ -1,9 +1,8 @@
 //===- Scalarizer.h --- Scalarize vector operations -----------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Transforms/Scalar/SimpleLoopUnswitch.h b/include/llvm/Transforms/Scalar/SimpleLoopUnswitch.h
index eed50ec96161b67e32f4e1359a00698792abe1bd..33c1faaeee0bbd2298ad097c277e00cd720daded 100644
--- a/include/llvm/Transforms/Scalar/SimpleLoopUnswitch.h
+++ b/include/llvm/Transforms/Scalar/SimpleLoopUnswitch.h
@@ -1,9 +1,8 @@
 //===- SimpleLoopUnswitch.h - Hoist loop-invariant control flow -*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/Transforms/Scalar/SimplifyCFG.h b/include/llvm/Transforms/Scalar/SimplifyCFG.h
index ce0a35fc06bd41aa4b02391ad415969d16345138..f9792d38bbe6bf5d9f47535b9845e2f0a9e3e5b9 100644
--- a/include/llvm/Transforms/Scalar/SimplifyCFG.h
+++ b/include/llvm/Transforms/Scalar/SimplifyCFG.h
@@ -1,9 +1,8 @@
 //===- SimplifyCFG.h - Simplify and canonicalize the CFG --------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/include/llvm/Transforms/Scalar/Sink.h b/include/llvm/Transforms/Scalar/Sink.h
index f9b3cb0fae39772848f33439a8c512af43d28d72..6cbe964d158006f67193caaa01a29c0207a48f8b 100644
--- a/include/llvm/Transforms/Scalar/Sink.h
+++ b/include/llvm/Transforms/Scalar/Sink.h
@@ -1,9 +1,8 @@
 //===-- Sink.h - Code Sinking -----------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Transforms/Scalar/SpeculateAroundPHIs.h b/include/llvm/Transforms/Scalar/SpeculateAroundPHIs.h
index 4a0bfd7547239dc90a0a5fe472a1b86c9e4e6449..3c7dafe71e8e1fc4acd3beba0c6f8a89b325c1b1 100644
--- a/include/llvm/Transforms/Scalar/SpeculateAroundPHIs.h
+++ b/include/llvm/Transforms/Scalar/SpeculateAroundPHIs.h
@@ -1,9 +1,8 @@
 //===- SpeculateAroundPHIs.h - Speculate around PHIs ------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/Transforms/Scalar/SpeculativeExecution.h b/include/llvm/Transforms/Scalar/SpeculativeExecution.h
index d00e950222a07abbdf5fe1292292b9c36912ee98..14da864832137675606f20353ca3ba5d328c4ca9 100644
--- a/include/llvm/Transforms/Scalar/SpeculativeExecution.h
+++ b/include/llvm/Transforms/Scalar/SpeculativeExecution.h
@@ -1,9 +1,8 @@
 //===- SpeculativeExecution.h -----------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Transforms/Scalar/TailRecursionElimination.h b/include/llvm/Transforms/Scalar/TailRecursionElimination.h
index 793f9bc152ed2b52298e966a2aa09d43c0c5ae9b..9068676445044b5f3c0f208a56062adf5bcd0e7a 100644
--- a/include/llvm/Transforms/Scalar/TailRecursionElimination.h
+++ b/include/llvm/Transforms/Scalar/TailRecursionElimination.h
@@ -1,9 +1,8 @@
 //===---- TailRecursionElimination.h ----------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Transforms/Scalar/WarnMissedTransforms.h b/include/llvm/Transforms/Scalar/WarnMissedTransforms.h
index 018b22a932e6cf1bd95b190d76f7c14f04967080..2d5942a3f569012a800d91e93c87e3c19645cad2 100644
--- a/include/llvm/Transforms/Scalar/WarnMissedTransforms.h
+++ b/include/llvm/Transforms/Scalar/WarnMissedTransforms.h
@@ -1,9 +1,8 @@
 //===- WarnMissedTransforms.h -----------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Transforms/Utils.h b/include/llvm/Transforms/Utils.h
index 378552775c77fc671a4277a8b4a67d2d8c8901b4..6e03453babf1a348d62e7a5160879f8a82a54091 100644
--- a/include/llvm/Transforms/Utils.h
+++ b/include/llvm/Transforms/Utils.h
@@ -1,9 +1,8 @@
 //===- llvm/Transforms/Utils.h - Utility Transformations --------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Transforms/Utils/ASanStackFrameLayout.h b/include/llvm/Transforms/Utils/ASanStackFrameLayout.h
index eaad06a108190d58e4c44378cba365c80fe9072d..0b570c0d134230d1a41eae44c46bbe5313c7e5ce 100644
--- a/include/llvm/Transforms/Utils/ASanStackFrameLayout.h
+++ b/include/llvm/Transforms/Utils/ASanStackFrameLayout.h
@@ -1,9 +1,8 @@
 //===- ASanStackFrameLayout.h - ComputeASanStackFrameLayout -----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Transforms/Utils/AddDiscriminators.h b/include/llvm/Transforms/Utils/AddDiscriminators.h
index 4dad06e6c1254224eacae8fae2218ba225830dcc..f512c6c06331961a3a4337adfcaeded6a30386a8 100644
--- a/include/llvm/Transforms/Utils/AddDiscriminators.h
+++ b/include/llvm/Transforms/Utils/AddDiscriminators.h
@@ -1,9 +1,8 @@
 //===- AddDiscriminators.h --------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Transforms/Utils/BasicBlockUtils.h b/include/llvm/Transforms/Utils/BasicBlockUtils.h
index a0fc18825a5894e29ab8ddf12ca74199f435be75..4e763289cefa23c3426fc10957a31ac00da34021 100644
--- a/include/llvm/Transforms/Utils/BasicBlockUtils.h
+++ b/include/llvm/Transforms/Utils/BasicBlockUtils.h
@@ -1,9 +1,8 @@
 //===- Transform/Utils/BasicBlockUtils.h - BasicBlock Utils -----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -18,9 +17,9 @@
 // FIXME: Move to this file: BasicBlock::removePredecessor, BB::splitBasicBlock
 
 #include "llvm/ADT/ArrayRef.h"
+#include "llvm/Analysis/DomTreeUpdater.h"
 #include "llvm/IR/BasicBlock.h"
 #include "llvm/IR/CFG.h"
-#include "llvm/IR/DomTreeUpdater.h"
 #include "llvm/IR/InstrTypes.h"
 #include <cassert>
 
@@ -40,8 +39,27 @@ class ReturnInst;
 class TargetLibraryInfo;
 class Value;
 
+/// Replace contents of every block in \p BBs with single unreachable
+/// instruction. If \p Updates is specified, collect all necessary DT updates
+/// into this vector. If \p KeepOneInputPHIs is true, one-input Phis in
+/// successors of blocks being deleted will be preserved.
+void DetatchDeadBlocks(ArrayRef <BasicBlock *> BBs,
+                       SmallVectorImpl<DominatorTree::UpdateType> *Updates,
+                       bool KeepOneInputPHIs = false);
+
 /// Delete the specified block, which must have no predecessors.
-void DeleteDeadBlock(BasicBlock *BB, DomTreeUpdater *DTU = nullptr);
+void DeleteDeadBlock(BasicBlock *BB, DomTreeUpdater *DTU = nullptr,
+                     bool KeepOneInputPHIs = false);
+
+/// Delete the specified blocks from \p BB. The set of deleted blocks must have
+/// no predecessors that are not being deleted themselves. \p BBs must have no
+/// duplicating blocks. If there are loops among this set of blocks, all
+/// relevant loop info updates should be done before this function is called.
+/// If \p KeepOneInputPHIs is true, one-input Phis in successors of blocks
+/// being deleted will be preserved.
+void DeleteDeadBlocks(ArrayRef <BasicBlock *> BBs,
+                      DomTreeUpdater *DTU = nullptr,
+                      bool KeepOneInputPHIs = false);
 
 /// We know that BB has one predecessor. If there are any single-entry PHI nodes
 /// in it, fold them away. This handles the case when all entries to the PHI
@@ -88,7 +106,7 @@ struct CriticalEdgeSplittingOptions {
   LoopInfo *LI;
   MemorySSAUpdater *MSSAU;
   bool MergeIdenticalEdges = false;
-  bool DontDeleteUselessPHIs = false;
+  bool KeepOneInputPHIs = false;
   bool PreserveLCSSA = false;
 
   CriticalEdgeSplittingOptions(DominatorTree *DT = nullptr,
@@ -101,8 +119,8 @@ struct CriticalEdgeSplittingOptions {
     return *this;
   }
 
-  CriticalEdgeSplittingOptions &setDontDeleteUselessPHIs() {
-    DontDeleteUselessPHIs = true;
+  CriticalEdgeSplittingOptions &setKeepOneInputPHIs() {
+    KeepOneInputPHIs = true;
     return *this;
   }
 
diff --git a/include/llvm/Transforms/Utils/BreakCriticalEdges.h b/include/llvm/Transforms/Utils/BreakCriticalEdges.h
index 9cc81a176cb6a2c5b934594efba8b5f058173df2..3644f1ed7a13ebb5a642950eb633dbcc832e3ee8 100644
--- a/include/llvm/Transforms/Utils/BreakCriticalEdges.h
+++ b/include/llvm/Transforms/Utils/BreakCriticalEdges.h
@@ -1,9 +1,8 @@
 //===- BreakCriticalEdges.h - Critical Edge Elimination Pass --------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Transforms/Utils/BuildLibCalls.h b/include/llvm/Transforms/Utils/BuildLibCalls.h
index 28efce6ac3fb9ba46d49b61aa77d0b8f9150574d..6c61cbeb0ba20f8ed78c470d8d6b340615cb4cbc 100644
--- a/include/llvm/Transforms/Utils/BuildLibCalls.h
+++ b/include/llvm/Transforms/Utils/BuildLibCalls.h
@@ -1,9 +1,8 @@
 //===- BuildLibCalls.h - Utility builder for libcalls -----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Transforms/Utils/BypassSlowDivision.h b/include/llvm/Transforms/Utils/BypassSlowDivision.h
index 6eca5ed2154e2320364898de4b2468122ea517b7..471055921fa80b980eeea40c3c4366b48f6277b6 100644
--- a/include/llvm/Transforms/Utils/BypassSlowDivision.h
+++ b/include/llvm/Transforms/Utils/BypassSlowDivision.h
@@ -1,9 +1,8 @@
 //===- llvm/Transforms/Utils/BypassSlowDivision.h ---------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Transforms/Utils/CallPromotionUtils.h b/include/llvm/Transforms/Utils/CallPromotionUtils.h
index 6e8ece723638057f2c121c43f96e3dfade6ff291..d9d171c6d8bddf99a454d280d0c8ee87ed8bda25 100644
--- a/include/llvm/Transforms/Utils/CallPromotionUtils.h
+++ b/include/llvm/Transforms/Utils/CallPromotionUtils.h
@@ -1,9 +1,8 @@
 //===- CallPromotionUtils.h - Utilities for call promotion ------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Transforms/Utils/CanonicalizeAliases.h b/include/llvm/Transforms/Utils/CanonicalizeAliases.h
new file mode 100644
index 0000000000000000000000000000000000000000..8f23a041a24e62c8f43ecc6448c5fe3fbf4dee0c
--- /dev/null
+++ b/include/llvm/Transforms/Utils/CanonicalizeAliases.h
@@ -0,0 +1,31 @@
+//===-- CanonicalizeAliases.h - Alias Canonicalization Pass -----*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file canonicalizes aliases.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_TRANSFORMS_UTILS_CANONICALIZE_ALIASES_H
+#define LLVM_TRANSFORMS_UTILS_CANONICALIZE_ALIASES_H
+
+#include "llvm/IR/Module.h"
+#include "llvm/IR/PassManager.h"
+
+namespace llvm {
+
+/// Simple pass that canonicalizes aliases.
+class CanonicalizeAliasesPass : public PassInfoMixin<CanonicalizeAliasesPass> {
+public:
+  CanonicalizeAliasesPass() = default;
+
+  PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
+};
+
+} // end namespace llvm
+
+#endif // LLVM_TRANSFORMS_UTILS_CANONICALIZE_ALIASESH
diff --git a/include/llvm/Transforms/Utils/Cloning.h b/include/llvm/Transforms/Utils/Cloning.h
index f5e997324fc8917d05321adb2dc0bab6165d9ddb..86775b1af758649668f9a5feb068571373408c78 100644
--- a/include/llvm/Transforms/Utils/Cloning.h
+++ b/include/llvm/Transforms/Utils/Cloning.h
@@ -1,9 +1,8 @@
 //===- Cloning.h - Clone various parts of LLVM programs ---------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -269,6 +268,13 @@ BasicBlock *DuplicateInstructionsInSplitBetween(BasicBlock *BB,
                                                 ValueToValueMapTy &ValueMapping,
                                                 DomTreeUpdater &DTU);
 
+/// Updates profile information by adjusting the entry count by adding
+/// entryDelta then scaling callsite information by the new count divided by the
+/// old count. VMap is used during inlinng to also update the new clone
+void updateProfileCallee(
+    Function *Callee, int64_t entryDelta,
+    const ValueMap<const Value *, WeakTrackingVH> *VMap = nullptr);
+
 } // end namespace llvm
 
 #endif // LLVM_TRANSFORMS_UTILS_CLONING_H
diff --git a/include/llvm/Transforms/Utils/CodeExtractor.h b/include/llvm/Transforms/Utils/CodeExtractor.h
index 498ff5a7883f3157b2760f7e8eb82cec8d74694b..becbf0ea62fe91689e6a235317304483f1b0a9d1 100644
--- a/include/llvm/Transforms/Utils/CodeExtractor.h
+++ b/include/llvm/Transforms/Utils/CodeExtractor.h
@@ -1,9 +1,8 @@
 //===- Transform/Utils/CodeExtractor.h - Code extraction util ---*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -27,6 +26,8 @@ class BasicBlock;
 class BlockFrequency;
 class BlockFrequencyInfo;
 class BranchProbabilityInfo;
+class AssumptionCache;
+class CallInst;
 class DominatorTree;
 class Function;
 class Instruction;
@@ -56,6 +57,7 @@ class Value;
     const bool AggregateArgs;
     BlockFrequencyInfo *BFI;
     BranchProbabilityInfo *BPI;
+    AssumptionCache *AC;
 
     // If true, varargs functions can be extracted.
     bool AllowVarArgs;
@@ -84,6 +86,7 @@ class Value;
     CodeExtractor(ArrayRef<BasicBlock *> BBs, DominatorTree *DT = nullptr,
                   bool AggregateArgs = false, BlockFrequencyInfo *BFI = nullptr,
                   BranchProbabilityInfo *BPI = nullptr,
+                  AssumptionCache *AC = nullptr,
                   bool AllowVarArgs = false, bool AllowAlloca = false,
                   std::string Suffix = "");
 
@@ -94,6 +97,7 @@ class Value;
     CodeExtractor(DominatorTree &DT, Loop &L, bool AggregateArgs = false,
                   BlockFrequencyInfo *BFI = nullptr,
                   BranchProbabilityInfo *BPI = nullptr,
+                  AssumptionCache *AC = nullptr,
                   std::string Suffix = "");
 
     /// Perform the extraction, returning the new function.
@@ -164,10 +168,9 @@ class Value;
         DenseMap<BasicBlock *, BlockFrequency> &ExitWeights,
         BranchProbabilityInfo *BPI);
 
-    void emitCallAndSwitchStatement(Function *newFunction,
-                                    BasicBlock *newHeader,
-                                    ValueSet &inputs,
-                                    ValueSet &outputs);
+    CallInst *emitCallAndSwitchStatement(Function *newFunction,
+                                         BasicBlock *newHeader,
+                                         ValueSet &inputs, ValueSet &outputs);
   };
 
 } // end namespace llvm
diff --git a/include/llvm/Transforms/Utils/CtorUtils.h b/include/llvm/Transforms/Utils/CtorUtils.h
index 63e564dcb87a9cd7b0120b885ef80f68b8f5fb7e..3625ee662b1ce03ea8d2de65bb446bae26a4a521 100644
--- a/include/llvm/Transforms/Utils/CtorUtils.h
+++ b/include/llvm/Transforms/Utils/CtorUtils.h
@@ -1,9 +1,8 @@
 //===- CtorUtils.h - Helpers for working with global_ctors ------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Transforms/Utils/EntryExitInstrumenter.h b/include/llvm/Transforms/Utils/EntryExitInstrumenter.h
index f50c5c922081603364207d8747fccf29210b1e4d..3913693af3594afb96836211a1b5ec5a9fca888f 100644
--- a/include/llvm/Transforms/Utils/EntryExitInstrumenter.h
+++ b/include/llvm/Transforms/Utils/EntryExitInstrumenter.h
@@ -1,9 +1,8 @@
 //===- EntryExitInstrumenter.h - Function Entry/Exit Instrumentation ------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Transforms/Utils/EscapeEnumerator.h b/include/llvm/Transforms/Utils/EscapeEnumerator.h
index 1256dfdaca172d01226c6e59fec6a102dd41fe7b..e667796c841bc1ab4012940afa5528a36d005e42 100644
--- a/include/llvm/Transforms/Utils/EscapeEnumerator.h
+++ b/include/llvm/Transforms/Utils/EscapeEnumerator.h
@@ -1,9 +1,8 @@
 //===-- EscapeEnumerator.h --------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Transforms/Utils/Evaluator.h b/include/llvm/Transforms/Utils/Evaluator.h
index 9908ae6fd393753165e8fb5199b60fed56e24884..bffd65f71b2ef09b334d9a93f457989f6630f36e 100644
--- a/include/llvm/Transforms/Utils/Evaluator.h
+++ b/include/llvm/Transforms/Utils/Evaluator.h
@@ -1,9 +1,8 @@
 //===- Evaluator.h - LLVM IR evaluator --------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Transforms/Utils/FunctionComparator.h b/include/llvm/Transforms/Utils/FunctionComparator.h
index 35ba0950343c06ee8685ab14e3f2001de3b64556..4e2571b1d0b6f6b799728ab03f82a0f1b8565973 100644
--- a/include/llvm/Transforms/Utils/FunctionComparator.h
+++ b/include/llvm/Transforms/Utils/FunctionComparator.h
@@ -1,9 +1,8 @@
 //===- FunctionComparator.h - Function Comparator ---------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Transforms/Utils/FunctionImportUtils.h b/include/llvm/Transforms/Utils/FunctionImportUtils.h
index e24398b90012b355d0f409df813d3f758b1fb0c7..9c2a9ea531eafab7c2eb1456be97653737e96433 100644
--- a/include/llvm/Transforms/Utils/FunctionImportUtils.h
+++ b/include/llvm/Transforms/Utils/FunctionImportUtils.h
@@ -1,9 +1,8 @@
 //===- FunctionImportUtils.h - Importing support utilities -----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -44,6 +43,11 @@ class FunctionImportGlobalProcessing {
   /// to promote any non-renamable values.
   SmallPtrSet<GlobalValue *, 8> Used;
 
+  /// Keep track of any COMDATs that require renaming (because COMDAT
+  /// leader was promoted and renamed). Maps from original COMDAT to one
+  /// with new name.
+  DenseMap<const Comdat *, Comdat *> RenamedComdats;
+
   /// Check if we should promote the given local value to global scope.
   bool shouldPromoteLocalToGlobal(const GlobalValue *SGV);
 
diff --git a/include/llvm/Transforms/Utils/GlobalStatus.h b/include/llvm/Transforms/Utils/GlobalStatus.h
index 8cc265bdf81d27f57086141333780321a13ad543..519593c96766c779b8bcaa3e1cbd00cfbe29fa7c 100644
--- a/include/llvm/Transforms/Utils/GlobalStatus.h
+++ b/include/llvm/Transforms/Utils/GlobalStatus.h
@@ -1,9 +1,8 @@
 //===- GlobalStatus.h - Compute status info for globals ---------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/Transforms/Utils/GuardUtils.h b/include/llvm/Transforms/Utils/GuardUtils.h
index 537045edafe48739edceccf6905a860c9c31eb25..3b365c56a5c01be4b9a6f64ed46a0a9d218950b5 100644
--- a/include/llvm/Transforms/Utils/GuardUtils.h
+++ b/include/llvm/Transforms/Utils/GuardUtils.h
@@ -1,9 +1,8 @@
 //===-- GuardUtils.h - Utils for work with guards ---------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 // Utils that are used to perform transformations related to guards and their
diff --git a/include/llvm/Transforms/Utils/ImportedFunctionsInliningStatistics.h b/include/llvm/Transforms/Utils/ImportedFunctionsInliningStatistics.h
index b55a9893bcf76506cb289ca7c7e1f13e2740372f..033ea05b77fa4b1df86c234142f16842f49ba496 100644
--- a/include/llvm/Transforms/Utils/ImportedFunctionsInliningStatistics.h
+++ b/include/llvm/Transforms/Utils/ImportedFunctionsInliningStatistics.h
@@ -1,9 +1,8 @@
-//===-- ImportedFunctionsInliningStats.h ------------------------*- C++ -*-===//
+//===-- ImportedFunctionsInliningStatistics.h -------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 // Generating inliner statistics for imported functions, mostly useful for
diff --git a/include/llvm/Transforms/Utils/IntegerDivision.h b/include/llvm/Transforms/Utils/IntegerDivision.h
index 5d9927eb51b20561c9d8930d0d43d87f9a123af2..35cae9aa2269edc26308ba66e8171cf020c2dfae 100644
--- a/include/llvm/Transforms/Utils/IntegerDivision.h
+++ b/include/llvm/Transforms/Utils/IntegerDivision.h
@@ -1,9 +1,8 @@
 //===- llvm/Transforms/Utils/IntegerDivision.h ------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Transforms/Utils/LCSSA.h b/include/llvm/Transforms/Utils/LCSSA.h
index fe717e5f6635ba1dc3dce8ac28173956a0025dd4..b01c8022a65bc8ca519b8ac51ba02f4f591fd594 100644
--- a/include/llvm/Transforms/Utils/LCSSA.h
+++ b/include/llvm/Transforms/Utils/LCSSA.h
@@ -1,9 +1,8 @@
 //===- LCSSA.h - Loop-closed SSA transform Pass -----------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Transforms/Utils/LibCallsShrinkWrap.h b/include/llvm/Transforms/Utils/LibCallsShrinkWrap.h
index c9df532e5794a83f357322f162be4296c57c1290..ff1537ace329ea4678c2d08afdce60e266c69c5d 100644
--- a/include/llvm/Transforms/Utils/LibCallsShrinkWrap.h
+++ b/include/llvm/Transforms/Utils/LibCallsShrinkWrap.h
@@ -1,9 +1,8 @@
 //===- LibCallsShrinkWrap.h - Shrink Wrap Library Calls -------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Transforms/Utils/Local.h b/include/llvm/Transforms/Utils/Local.h
index ec8b0eda36416a38c57207503c1c49d0fad3aefd..641b645825fcab8ecea00bf663959d32e7cf9929 100644
--- a/include/llvm/Transforms/Utils/Local.h
+++ b/include/llvm/Transforms/Utils/Local.h
@@ -1,9 +1,8 @@
 //===- Local.h - Functions to perform local transformations -----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -21,12 +20,11 @@
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/TinyPtrVector.h"
 #include "llvm/Analysis/AliasAnalysis.h"
+#include "llvm/Analysis/DomTreeUpdater.h"
 #include "llvm/Analysis/Utils/Local.h"
-#include "llvm/IR/CallSite.h"
 #include "llvm/IR/Constant.h"
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/DataLayout.h"
-#include "llvm/IR/DomTreeUpdater.h"
 #include "llvm/IR/Dominators.h"
 #include "llvm/IR/GetElementPtrTypeIterator.h"
 #include "llvm/IR/Operator.h"
@@ -342,6 +340,18 @@ void replaceDbgValueForAlloca(AllocaInst *AI, Value *NewAllocaAddress,
 /// Returns true if any debug users were updated.
 bool salvageDebugInfo(Instruction &I);
 
+/// Implementation of salvageDebugInfo, applying only to instructions in
+/// \p Insns, rather than all debug users of \p I.
+bool salvageDebugInfoForDbgValues(Instruction &I,
+                                  ArrayRef<DbgVariableIntrinsic *> Insns);
+
+/// Given an instruction \p I and DIExpression \p DIExpr operating on it, write
+/// the effects of \p I into the returned DIExpression, or return nullptr if
+/// it cannot be salvaged. \p StackVal: whether DW_OP_stack_value should be
+/// appended to the expression.
+DIExpression *salvageDebugInfoImpl(Instruction &I, DIExpression *DIExpr,
+                                   bool StackVal);
+
 /// Point debug users of \p From to \p To or salvage them. Use this function
 /// only when replacing all uses of \p From with \p To, with a guarantee that
 /// \p From is going to be deleted.
@@ -426,7 +436,7 @@ unsigned replaceDominatedUsesWith(Value *From, Value *To, DominatorTree &DT,
 unsigned replaceDominatedUsesWith(Value *From, Value *To, DominatorTree &DT,
                                   const BasicBlock *BB);
 
-/// Return true if the CallSite CS calls a gc leaf function.
+/// Return true if this call calls a gc leaf function.
 ///
 /// A leaf function is a function that does not safepoint the thread during its
 /// execution.  During a call or invoke to such a function, the callers stack
@@ -434,7 +444,7 @@ unsigned replaceDominatedUsesWith(Value *From, Value *To, DominatorTree &DT,
 ///
 /// Most passes can and should ignore this information, and it is only used
 /// during lowering by the GC infrastructure.
-bool callsGCLeafFunction(ImmutableCallSite CS, const TargetLibraryInfo &TLI);
+bool callsGCLeafFunction(const CallBase *Call, const TargetLibraryInfo &TLI);
 
 /// Copy a nonnull metadata node to a new load instruction.
 ///
@@ -456,8 +466,7 @@ void dropDebugUsers(Instruction &I);
 /// \p DomBlock, by moving its instructions to the insertion point \p InsertPt.
 ///
 /// The moved instructions receive the insertion point debug location values
-/// (DILocations) and their debug intrinsic instructions (dbg.values) are
-/// removed.
+/// (DILocations) and their debug intrinsic instructions are removed.
 void hoistAllInstructionsInto(BasicBlock *DomBlock, Instruction *InsertPt,
                               BasicBlock *BB);
 
diff --git a/include/llvm/Transforms/Utils/LoopRotationUtils.h b/include/llvm/Transforms/Utils/LoopRotationUtils.h
index cd5bc43010181538dc3e090c0c26d1e6dfae513a..1e80722ed8b8d12f8ae58a636c3f1c38b5bcc866 100644
--- a/include/llvm/Transforms/Utils/LoopRotationUtils.h
+++ b/include/llvm/Transforms/Utils/LoopRotationUtils.h
@@ -1,9 +1,8 @@
 //===- LoopRotationUtils.h - Utilities to perform loop rotation -*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Transforms/Utils/LoopSimplify.h b/include/llvm/Transforms/Utils/LoopSimplify.h
index 166da2738ffd6ca031189d4ca58a501a2e56c0f9..938e46dac43af8e19ae0786a0cf23bf6285b787b 100644
--- a/include/llvm/Transforms/Utils/LoopSimplify.h
+++ b/include/llvm/Transforms/Utils/LoopSimplify.h
@@ -1,9 +1,8 @@
 //===- LoopSimplify.h - Loop Canonicalization Pass --------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Transforms/Utils/LoopUtils.h b/include/llvm/Transforms/Utils/LoopUtils.h
index faf9131e6aca44d3a005481676632791c854a496..a5883ac79bb6d875fdd6fe04ddf532909b0e3401 100644
--- a/include/llvm/Transforms/Utils/LoopUtils.h
+++ b/include/llvm/Transforms/Utils/LoopUtils.h
@@ -1,9 +1,8 @@
 //===- llvm/Transforms/Utils/LoopUtils.h - Loop utilities -------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -41,6 +40,8 @@ class BasicBlock;
 class DataLayout;
 class Loop;
 class LoopInfo;
+class MemoryAccess;
+class MemorySSAUpdater;
 class OptimizationRemarkEmitter;
 class PredicatedScalarEvolution;
 class PredIteratorCache;
@@ -78,7 +79,8 @@ bool formLCSSAForInstructions(SmallVectorImpl<Instruction *> &Worklist,
 ///
 /// Looks at all instructions in the loop which have uses outside of the
 /// current loop. For each, an LCSSA PHI node is inserted and the uses outside
-/// the loop are rewritten to use this node.
+/// the loop are rewritten to use this node. Sub-loops must be in LCSSA form
+/// already.
 ///
 /// LoopInfo and DominatorTree are required and preserved.
 ///
@@ -109,8 +111,8 @@ bool formLCSSARecursively(Loop &L, DominatorTree &DT, LoopInfo *LI,
 /// arguments. Diagnostics is emitted via \p ORE. It returns changed status.
 bool sinkRegion(DomTreeNode *, AliasAnalysis *, LoopInfo *, DominatorTree *,
                 TargetLibraryInfo *, TargetTransformInfo *, Loop *,
-                AliasSetTracker *, ICFLoopSafetyInfo *,
-                OptimizationRemarkEmitter *ORE);
+                AliasSetTracker *, MemorySSAUpdater *, ICFLoopSafetyInfo *,
+                bool, int &, OptimizationRemarkEmitter *);
 
 /// Walk the specified region of the CFG (defined by all blocks
 /// dominated by the specified block, and that are in the current loop) in depth
@@ -122,7 +124,8 @@ bool sinkRegion(DomTreeNode *, AliasAnalysis *, LoopInfo *, DominatorTree *,
 /// ORE. It returns changed status.
 bool hoistRegion(DomTreeNode *, AliasAnalysis *, LoopInfo *, DominatorTree *,
                  TargetLibraryInfo *, Loop *, AliasSetTracker *,
-                 ICFLoopSafetyInfo *, OptimizationRemarkEmitter *ORE);
+                 MemorySSAUpdater *, ICFLoopSafetyInfo *, bool, int &,
+                 OptimizationRemarkEmitter *);
 
 /// This function deletes dead loops. The caller of this function needs to
 /// guarantee that the loop is infact dead.
@@ -146,14 +149,12 @@ void deleteDeadLoop(Loop *L, DominatorTree *DT, ScalarEvolution *SE,
 /// LoopInfo, DominatorTree, Loop, AliasSet information for all instructions
 /// of the loop and loop safety information as arguments.
 /// Diagnostics is emitted via \p ORE. It returns changed status.
-bool promoteLoopAccessesToScalars(const SmallSetVector<Value *, 8> &,
-                                  SmallVectorImpl<BasicBlock *> &,
-                                  SmallVectorImpl<Instruction *> &,
-                                  PredIteratorCache &, LoopInfo *,
-                                  DominatorTree *, const TargetLibraryInfo *,
-                                  Loop *, AliasSetTracker *,
-                                  ICFLoopSafetyInfo *,
-                                  OptimizationRemarkEmitter *);
+bool promoteLoopAccessesToScalars(
+    const SmallSetVector<Value *, 8> &, SmallVectorImpl<BasicBlock *> &,
+    SmallVectorImpl<Instruction *> &, SmallVectorImpl<MemoryAccess *> &,
+    PredIteratorCache &, LoopInfo *, DominatorTree *, const TargetLibraryInfo *,
+    Loop *, AliasSetTracker *, MemorySSAUpdater *, ICFLoopSafetyInfo *,
+    OptimizationRemarkEmitter *);
 
 /// Does a BFS from a given node to all of its children inside a given loop.
 /// The returned vector of nodes includes the starting point.
@@ -168,7 +169,7 @@ SmallVector<Instruction *, 8> findDefsUsedOutsideOfLoop(Loop *L);
 /// If it has a value (e.g. {"llvm.distribute", 1} return the value as an
 /// operand or null otherwise.  If the string metadata is not found return
 /// Optional's not-a-value.
-Optional<const MDOperand *> findStringMetadataForLoop(Loop *TheLoop,
+Optional<const MDOperand *> findStringMetadataForLoop(const Loop *TheLoop,
                                                       StringRef Name);
 
 /// Find named metadata for a loop with an integer value.
@@ -274,7 +275,9 @@ void getLoopAnalysisUsage(AnalysisUsage &AU);
 /// If \p ORE is set use it to emit optimization remarks.
 bool canSinkOrHoistInst(Instruction &I, AAResults *AA, DominatorTree *DT,
                         Loop *CurLoop, AliasSetTracker *CurAST,
-                        bool TargetExecutesOncePerLoop,
+                        MemorySSAUpdater *MSSAU, bool TargetExecutesOncePerLoop,
+                        bool NoOfMemAccessesTooLarge,
+                        int *LicmMssaOptCap = nullptr,
                         OptimizationRemarkEmitter *ORE = nullptr);
 
 /// Returns a Min/Max operation corresponding to MinMaxRecurrenceKind.
@@ -321,6 +324,23 @@ Value *createTargetReduction(IRBuilder<> &B, const TargetTransformInfo *TTI,
 /// Flag set: NSW, NUW, exact, and all of fast-math.
 void propagateIRFlags(Value *I, ArrayRef<Value *> VL, Value *OpValue = nullptr);
 
+/// Returns true if we can prove that \p S is defined and always negative in
+/// loop \p L.
+bool isKnownNegativeInLoop(const SCEV *S, const Loop *L, ScalarEvolution &SE);
+
+/// Returns true if we can prove that \p S is defined and always non-negative in
+/// loop \p L.
+bool isKnownNonNegativeInLoop(const SCEV *S, const Loop *L,
+                              ScalarEvolution &SE);
+
+/// Returns true if \p S is defined and never is equal to signed/unsigned max.
+bool cannotBeMaxInLoop(const SCEV *S, const Loop *L, ScalarEvolution &SE,
+                       bool Signed);
+
+/// Returns true if \p S is defined and never is equal to signed/unsigned min.
+bool cannotBeMinInLoop(const SCEV *S, const Loop *L, ScalarEvolution &SE,
+                       bool Signed);
+
 } // end namespace llvm
 
 #endif // LLVM_TRANSFORMS_UTILS_LOOPUTILS_H
diff --git a/include/llvm/Transforms/Utils/LoopVersioning.h b/include/llvm/Transforms/Utils/LoopVersioning.h
index fcd734b37a1f935897890481b7ea3ff1f5a2db9c..355c4d7dc6d87162142096626175ea32e54299a5 100644
--- a/include/llvm/Transforms/Utils/LoopVersioning.h
+++ b/include/llvm/Transforms/Utils/LoopVersioning.h
@@ -1,9 +1,8 @@
 //===- LoopVersioning.h - Utility to version a loop -------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Transforms/Utils/LowerInvoke.h b/include/llvm/Transforms/Utils/LowerInvoke.h
index 12774c7fd1f7b44af2b7250b96ea74234b175765..c1198b08d3ded8842635593e79045a0982362bbc 100644
--- a/include/llvm/Transforms/Utils/LowerInvoke.h
+++ b/include/llvm/Transforms/Utils/LowerInvoke.h
@@ -1,9 +1,8 @@
 //===- LowerInvoke.h - Eliminate Invoke instructions ----------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Transforms/Utils/LowerMemIntrinsics.h b/include/llvm/Transforms/Utils/LowerMemIntrinsics.h
index 2b7d0f67a3245aeb2ba5e180f4dd8a6dcf0efa4a..8e9d7b522c78b282ffa5c73036bf7e3221083a49 100644
--- a/include/llvm/Transforms/Utils/LowerMemIntrinsics.h
+++ b/include/llvm/Transforms/Utils/LowerMemIntrinsics.h
@@ -1,9 +1,8 @@
-//===- llvm/Transforms/Utils/LowerMemintrinsics.h ---------------*- C++ -*-===//
+//===- llvm/Transforms/Utils/LowerMemIntrinsics.h ---------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Transforms/Utils/Mem2Reg.h b/include/llvm/Transforms/Utils/Mem2Reg.h
index 407684338a3b7460ef0d80637b86d48f6a675918..76c1c2c5bffeca1b2a7232fcfb6991c9bd00360d 100644
--- a/include/llvm/Transforms/Utils/Mem2Reg.h
+++ b/include/llvm/Transforms/Utils/Mem2Reg.h
@@ -1,9 +1,8 @@
 //===- Mem2Reg.h - The -mem2reg pass, a wrapper around the Utils lib ------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Transforms/Utils/ModuleUtils.h b/include/llvm/Transforms/Utils/ModuleUtils.h
index 14615c25d093bb9baff580d4f2cd36c948b627a2..c69af558874164e354fe9991278845703c9a4950 100644
--- a/include/llvm/Transforms/Utils/ModuleUtils.h
+++ b/include/llvm/Transforms/Utils/ModuleUtils.h
@@ -1,9 +1,8 @@
 //===-- ModuleUtils.h - Functions to manipulate Modules ---------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -22,6 +21,7 @@ namespace llvm {
 template <typename T> class ArrayRef;
 class Module;
 class Function;
+class FunctionCallee;
 class GlobalValue;
 class GlobalVariable;
 class Constant;
@@ -40,24 +40,36 @@ void appendToGlobalCtors(Module &M, Function *F, int Priority,
 void appendToGlobalDtors(Module &M, Function *F, int Priority,
                          Constant *Data = nullptr);
 
-// Validate the result of Module::getOrInsertFunction called for an interface
-// function of given sanitizer. If the instrumented module defines a function
-// with the same name, their prototypes must match, otherwise
-// getOrInsertFunction returns a bitcast.
-Function *checkSanitizerInterfaceFunction(Constant *FuncOrBitcast);
-
-Function *declareSanitizerInitFunction(Module &M, StringRef InitName,
-                                       ArrayRef<Type *> InitArgTypes);
+FunctionCallee declareSanitizerInitFunction(Module &M, StringRef InitName,
+                                            ArrayRef<Type *> InitArgTypes);
 
 /// Creates sanitizer constructor function, and calls sanitizer's init
 /// function from it.
 /// \return Returns pair of pointers to constructor, and init functions
 /// respectively.
-std::pair<Function *, Function *> createSanitizerCtorAndInitFunctions(
+std::pair<Function *, FunctionCallee> createSanitizerCtorAndInitFunctions(
+    Module &M, StringRef CtorName, StringRef InitName,
+    ArrayRef<Type *> InitArgTypes, ArrayRef<Value *> InitArgs,
+    StringRef VersionCheckName = StringRef());
+
+/// Creates sanitizer constructor function lazily. If a constructor and init
+/// function already exist, this function returns it. Otherwise it calls \c
+/// createSanitizerCtorAndInitFunctions. The FunctionsCreatedCallback is invoked
+/// in that case, passing the new Ctor and Init function.
+///
+/// \return Returns pair of pointers to constructor, and init functions
+/// respectively.
+std::pair<Function *, FunctionCallee> getOrCreateSanitizerCtorAndInitFunctions(
     Module &M, StringRef CtorName, StringRef InitName,
     ArrayRef<Type *> InitArgTypes, ArrayRef<Value *> InitArgs,
+    function_ref<void(Function *, FunctionCallee)> FunctionsCreatedCallback,
     StringRef VersionCheckName = StringRef());
 
+// Creates and returns a sanitizer init function without argument if it doesn't
+// exist, and adds it to the global constructors list. Otherwise it returns the
+// existing function.
+Function *getOrCreateInitFunction(Module &M, StringRef Name);
+
 /// Rename all the anon globals in the module using a hash computed from
 /// the list of public globals in the module.
 bool nameUnamedGlobals(Module &M);
diff --git a/include/llvm/Transforms/Utils/NameAnonGlobals.h b/include/llvm/Transforms/Utils/NameAnonGlobals.h
index 17fc902eebf882d97e5b197691ceb7126b8fe747..659ebe33ffa6992b49c274cb640c950109a5905f 100644
--- a/include/llvm/Transforms/Utils/NameAnonGlobals.h
+++ b/include/llvm/Transforms/Utils/NameAnonGlobals.h
@@ -1,9 +1,8 @@
 //===-- NameAnonGlobals.h - Anonymous Global Naming Pass --------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Transforms/Utils/PredicateInfo.h b/include/llvm/Transforms/Utils/PredicateInfo.h
index 2fc38089f3f1f42d49dc96ebf632df529f170334..da4a5dcc28c006a47c44a8d748b42398456e4e81 100644
--- a/include/llvm/Transforms/Utils/PredicateInfo.h
+++ b/include/llvm/Transforms/Utils/PredicateInfo.h
@@ -1,9 +1,8 @@
 //===- PredicateInfo.h - Build PredicateInfo ----------------------*-C++-*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
diff --git a/include/llvm/Transforms/Utils/PromoteMemToReg.h b/include/llvm/Transforms/Utils/PromoteMemToReg.h
index 5ddfbe2bf05881a5c5d68458966ae8019b2742b2..b2b4507bbc74c2e93bf028eb2a09a8dd214251c6 100644
--- a/include/llvm/Transforms/Utils/PromoteMemToReg.h
+++ b/include/llvm/Transforms/Utils/PromoteMemToReg.h
@@ -1,9 +1,8 @@
 //===- PromoteMemToReg.h - Promote Allocas to Scalars -----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Transforms/Utils/SSAUpdater.h b/include/llvm/Transforms/Utils/SSAUpdater.h
index d02607acbbb57909c3cabd5d81772eff20d179c1..22b2295cc9d70e49804e2cb32793dc70af11f438 100644
--- a/include/llvm/Transforms/Utils/SSAUpdater.h
+++ b/include/llvm/Transforms/Utils/SSAUpdater.h
@@ -1,9 +1,8 @@
 //===- SSAUpdater.h - Unstructured SSA Update Tool --------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -148,7 +147,7 @@ public:
   /// Insts is a list of loads and stores to promote, and Name is the basename
   /// for the PHIs to insert. After this is complete, the loads and stores are
   /// removed from the code.
-  void run(const SmallVectorImpl<Instruction *> &Insts) const;
+  void run(const SmallVectorImpl<Instruction *> &Insts);
 
   /// Return true if the specified instruction is in the Inst list.
   ///
@@ -159,7 +158,7 @@ public:
 
   /// This hook is invoked after all the stores are found and inserted as
   /// available values.
-  virtual void doExtraRewritesBeforeFinalDeletion() const {}
+  virtual void doExtraRewritesBeforeFinalDeletion() {}
 
   /// Clients can choose to implement this to get notified right before
   /// a load is RAUW'd another value.
diff --git a/include/llvm/Transforms/Utils/SSAUpdaterBulk.h b/include/llvm/Transforms/Utils/SSAUpdaterBulk.h
index 53a608f018049db6f0972a20afd10e3c405d0bf2..5d17d6f3d285605e79238de6abd2e64e7b404ee6 100644
--- a/include/llvm/Transforms/Utils/SSAUpdaterBulk.h
+++ b/include/llvm/Transforms/Utils/SSAUpdaterBulk.h
@@ -1,9 +1,8 @@
 //===- SSAUpdaterBulk.h - Unstructured SSA Update Tool ----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Transforms/Utils/SSAUpdaterImpl.h b/include/llvm/Transforms/Utils/SSAUpdaterImpl.h
index cab0f3e7157578ea05a7331faa6ec66cb2da3957..ee06893ca66021dfcefc5d8f351ea195e48c321c 100644
--- a/include/llvm/Transforms/Utils/SSAUpdaterImpl.h
+++ b/include/llvm/Transforms/Utils/SSAUpdaterImpl.h
@@ -1,9 +1,8 @@
 //===- SSAUpdaterImpl.h - SSA Updater Implementation ------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Transforms/Utils/SanitizerStats.h b/include/llvm/Transforms/Utils/SanitizerStats.h
index d36e34258a3fc7f70b0e49d421b482b1549a442b..14e8ae045cddd168247350d79b25935f780e4b48 100644
--- a/include/llvm/Transforms/Utils/SanitizerStats.h
+++ b/include/llvm/Transforms/Utils/SanitizerStats.h
@@ -1,9 +1,8 @@
 //===- SanitizerStats.h - Sanitizer statistics gathering  -------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Transforms/Utils/SimplifyIndVar.h b/include/llvm/Transforms/Utils/SimplifyIndVar.h
index a1dfed29a22d32aaadb713f8f42736a538c27c88..dec73ef057e8fcc27c035690b9c7db2f6279466f 100644
--- a/include/llvm/Transforms/Utils/SimplifyIndVar.h
+++ b/include/llvm/Transforms/Utils/SimplifyIndVar.h
@@ -1,9 +1,8 @@
 //===-- llvm/Transforms/Utils/SimplifyIndVar.h - Indvar Utils ---*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Transforms/Utils/SimplifyLibCalls.h b/include/llvm/Transforms/Utils/SimplifyLibCalls.h
index 025bcd44e3107d1e3af3851b08ef97b5df396684..44eef091a2605b2f2576231f7b3cda1cd8f47d52 100644
--- a/include/llvm/Transforms/Utils/SimplifyLibCalls.h
+++ b/include/llvm/Transforms/Utils/SimplifyLibCalls.h
@@ -1,9 +1,8 @@
 //===- SimplifyLibCalls.h - Library call simplifier -------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Transforms/Utils/SplitModule.h b/include/llvm/Transforms/Utils/SplitModule.h
index d2c31f2701accc2ee0dcb055f62eb61b3f3f8ab7..7839c5d9a589488677fc1863bf248740d8f4f956 100644
--- a/include/llvm/Transforms/Utils/SplitModule.h
+++ b/include/llvm/Transforms/Utils/SplitModule.h
@@ -1,9 +1,8 @@
 //===- SplitModule.h - Split a module into partitions -----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Transforms/Utils/SymbolRewriter.h b/include/llvm/Transforms/Utils/SymbolRewriter.h
index 5f6488e08b5af3a9a3632770d42f04f8412777cb..ce9dcaf2b74f92b91a523c9e9d69c2c654597c4f 100644
--- a/include/llvm/Transforms/Utils/SymbolRewriter.h
+++ b/include/llvm/Transforms/Utils/SymbolRewriter.h
@@ -1,9 +1,8 @@
 //===- SymbolRewriter.h - Symbol Rewriting Pass -----------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Transforms/Utils/UnifyFunctionExitNodes.h b/include/llvm/Transforms/Utils/UnifyFunctionExitNodes.h
index 222c601ad60842381cfa246ac1fce63cbbac9fd4..f68534ecd2ebbf616dc4e75ded8c61b93affae0c 100644
--- a/include/llvm/Transforms/Utils/UnifyFunctionExitNodes.h
+++ b/include/llvm/Transforms/Utils/UnifyFunctionExitNodes.h
@@ -1,9 +1,8 @@
 //===-- UnifyFunctionExitNodes.h - Ensure fn's have one return --*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Transforms/Utils/UnrollLoop.h b/include/llvm/Transforms/Utils/UnrollLoop.h
index 70e936d75008a30ebcac45141013ba0f2eae8260..8f3824e150a841eaeef477df6851ccf490bad930 100644
--- a/include/llvm/Transforms/Utils/UnrollLoop.h
+++ b/include/llvm/Transforms/Utils/UnrollLoop.h
@@ -1,9 +1,8 @@
 //===- llvm/Transforms/Utils/UnrollLoop.h - Unrolling utilities -*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Transforms/Utils/VNCoercion.h b/include/llvm/Transforms/Utils/VNCoercion.h
index 1baa9b66e491906e07421bff18e8a85bbe851a38..f67b9ed0afddd8c6607e7cce13c513ea19b68967 100644
--- a/include/llvm/Transforms/Utils/VNCoercion.h
+++ b/include/llvm/Transforms/Utils/VNCoercion.h
@@ -1,9 +1,8 @@
 //===- VNCoercion.h - Value Numbering Coercion Utilities --------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file / This file provides routines used by LLVM's value numbering passes to
diff --git a/include/llvm/Transforms/Utils/ValueMapper.h b/include/llvm/Transforms/Utils/ValueMapper.h
index 4ecb23ea19518752ab9909c529556281bed63f38..1952a210291eba2d1489447d1ca24e6478c00d6f 100644
--- a/include/llvm/Transforms/Utils/ValueMapper.h
+++ b/include/llvm/Transforms/Utils/ValueMapper.h
@@ -1,9 +1,8 @@
 //===- ValueMapper.h - Remapping for constants and metadata -----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Transforms/Vectorize.h b/include/llvm/Transforms/Vectorize.h
index 70f9a2e0741b3a1aebcbe5c65a30c7fc872f9b52..a411b5664ddaf5124cfda7ecde495542dbed981b 100644
--- a/include/llvm/Transforms/Vectorize.h
+++ b/include/llvm/Transforms/Vectorize.h
@@ -1,9 +1,8 @@
 //===-- Vectorize.h - Vectorization Transformations -------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Transforms/Vectorize/LoadStoreVectorizer.h b/include/llvm/Transforms/Vectorize/LoadStoreVectorizer.h
index 6b37d7093c442c7c4c99c9e40ca631a660532638..f72c76c6f0f2f6be005352d44d5be7181747bec1 100644
--- a/include/llvm/Transforms/Vectorize/LoadStoreVectorizer.h
+++ b/include/llvm/Transforms/Vectorize/LoadStoreVectorizer.h
@@ -1,9 +1,8 @@
 //===- LoadStoreVectorizer.cpp - GPU Load & Store Vectorizer --------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h b/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
index 5c7bba048607e5ef7e6e40c348956c8b0f527b24..ea0a1c2654b407bc8b924a1e40b1350fe7e4f404 100644
--- a/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
+++ b/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h
@@ -1,9 +1,8 @@
 //===- llvm/Transforms/Vectorize/LoopVectorizationLegality.h ----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -99,11 +98,7 @@ public:
                      OptimizationRemarkEmitter &ORE);
 
   /// Mark the loop L as already vectorized by setting the width to 1.
-  void setAlreadyVectorized() {
-    IsVectorized.Value = 1;
-    Hint Hints[] = {IsVectorized};
-    writeHintsToMetadata(Hints);
-  }
+  void setAlreadyVectorized();
 
   bool allowVectorization(Function *F, Loop *L,
                           bool VectorizeOnlyWhenForced) const;
@@ -152,15 +147,6 @@ private:
   /// Checks string hint with one operand and set value if valid.
   void setHint(StringRef Name, Metadata *Arg);
 
-  /// Create a new hint from name / value pair.
-  MDNode *createHintMetadata(StringRef Name, unsigned V) const;
-
-  /// Matches metadata with hint name.
-  bool matchesHintMetadataName(MDNode *Node, ArrayRef<Hint> HintTypes);
-
-  /// Sets current hints into loop metadata, keeping other values intact.
-  void writeHintsToMetadata(ArrayRef<Hint> HintTypes);
-
   /// The loop these hints belong to.
   const Loop *TheLoop;
 
diff --git a/include/llvm/Transforms/Vectorize/LoopVectorize.h b/include/llvm/Transforms/Vectorize/LoopVectorize.h
index d9c4f7b023c14fa689463aebf7b70cbecf9a3289..cdbe5f6351a52e00a088233a4d24a193b4f0a81c 100644
--- a/include/llvm/Transforms/Vectorize/LoopVectorize.h
+++ b/include/llvm/Transforms/Vectorize/LoopVectorize.h
@@ -1,9 +1,8 @@
 //===- LoopVectorize.h ------------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/Transforms/Vectorize/SLPVectorizer.h b/include/llvm/Transforms/Vectorize/SLPVectorizer.h
index 3152e8192fc5dbd69b26ea881ed28b0152c47ad8..9be70557b5b9ece3135f73c37970ba3130f34179 100644
--- a/include/llvm/Transforms/Vectorize/SLPVectorizer.h
+++ b/include/llvm/Transforms/Vectorize/SLPVectorizer.h
@@ -1,9 +1,8 @@
 //===- SLPVectorizer.h ------------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 // This pass implements the Bottom Up SLP vectorizer. It detects consecutive
diff --git a/include/llvm/WindowsManifest/WindowsManifestMerger.h b/include/llvm/WindowsManifest/WindowsManifestMerger.h
index 302d3705887b73719ed9b5f55f0fdf64acc15a82..935c930ad91da7953f7301eb6ebd8a7c79c411ae 100644
--- a/include/llvm/WindowsManifest/WindowsManifestMerger.h
+++ b/include/llvm/WindowsManifest/WindowsManifestMerger.h
@@ -1,9 +1,8 @@
 //===-- WindowsManifestMerger.h ---------------------------------*- C++-*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===---------------------------------------------------------------------===//
 //
diff --git a/include/llvm/WindowsResource/ResourceProcessor.h b/include/llvm/WindowsResource/ResourceProcessor.h
index 4ca0a4b05bd0c155f231377b335a80a35b222310..4e99c05f4cd98259bff94b80714a866d61cd1f1f 100644
--- a/include/llvm/WindowsResource/ResourceProcessor.h
+++ b/include/llvm/WindowsResource/ResourceProcessor.h
@@ -1,9 +1,8 @@
 //===-- ResourceProcessor.h -------------------------------------*- C++-*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===---------------------------------------------------------------------===//
 
diff --git a/include/llvm/WindowsResource/ResourceScriptToken.h b/include/llvm/WindowsResource/ResourceScriptToken.h
index 494ae3222a4bbf3d92a6df56525615eecf3e6455..254121cd318a3845911d5e04bf22fbbd60e6cb46 100644
--- a/include/llvm/WindowsResource/ResourceScriptToken.h
+++ b/include/llvm/WindowsResource/ResourceScriptToken.h
@@ -1,9 +1,8 @@
 //===-- ResourceScriptToken.h -----------------------------------*- C++-*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===---------------------------------------------------------------------===//
 //
diff --git a/include/llvm/WindowsResource/ResourceScriptTokenList.h b/include/llvm/WindowsResource/ResourceScriptTokenList.h
index 0beed117c3e73dd4393b3d156a48b5d09e4185bf..6b44dccaa35f90bb58c6c7dd63e344f215593273 100644
--- a/include/llvm/WindowsResource/ResourceScriptTokenList.h
+++ b/include/llvm/WindowsResource/ResourceScriptTokenList.h
@@ -1,9 +1,8 @@
 //===-- ResourceScriptTokenList.h -------------------------------*- C++-*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===---------------------------------------------------------------------===//
 //
diff --git a/include/llvm/XRay/BlockIndexer.h b/include/llvm/XRay/BlockIndexer.h
index b42fa17f3fb7bd34e13071b8e43e7c5933787334..dafd2b5a523096a3c52e366785374ad9fc3a7fec 100644
--- a/include/llvm/XRay/BlockIndexer.h
+++ b/include/llvm/XRay/BlockIndexer.h
@@ -1,9 +1,8 @@
 //===- BlockIndexer.h - FDR Block Indexing Visitor ------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/XRay/BlockPrinter.h b/include/llvm/XRay/BlockPrinter.h
index bfb21e2395172f693f3e29cf6480379603ac608b..949258085332d58ff51ad285c0c76e1be1510af3 100644
--- a/include/llvm/XRay/BlockPrinter.h
+++ b/include/llvm/XRay/BlockPrinter.h
@@ -1,9 +1,8 @@
 //===- BlockPrinter.h - FDR Block Pretty Printer -------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/XRay/BlockVerifier.h b/include/llvm/XRay/BlockVerifier.h
index 46371c13891a7a79143f010527ab3bd80a68dbf7..c848fdf084bcc30bc5f32a9d17afebca4986d679 100644
--- a/include/llvm/XRay/BlockVerifier.h
+++ b/include/llvm/XRay/BlockVerifier.h
@@ -1,9 +1,8 @@
 //===- BlockVerifier.h - FDR Block Verifier -------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/XRay/FDRLogBuilder.h b/include/llvm/XRay/FDRLogBuilder.h
index b5e9ed5c406b19a73d28f1aa571d99149d00fecb..3f49dc6dcb9d606800ddc96f84601b44f81c3b16 100644
--- a/include/llvm/XRay/FDRLogBuilder.h
+++ b/include/llvm/XRay/FDRLogBuilder.h
@@ -1,9 +1,8 @@
 //===- FDRLogBuilder.h - XRay FDR Log Building Utility --------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 #ifndef LLVM_INCLUDE_LLVM_XRAY_FDRLOGBUILDER_H_
diff --git a/include/llvm/XRay/FDRRecordConsumer.h b/include/llvm/XRay/FDRRecordConsumer.h
index e856e1540558e903da993e1ff97ccd1c12fcd08f..4fbb1d41d0da2bb7c629b734fcf0a18df6f4d2c1 100644
--- a/include/llvm/XRay/FDRRecordConsumer.h
+++ b/include/llvm/XRay/FDRRecordConsumer.h
@@ -1,9 +1,8 @@
 //===- FDRRecordConsumer.h - XRay Flight Data Recorder Mode Records -------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 #ifndef LLVM_INCLUDE_LLVM_XRAY_FDRRECORDCONSUMER_H_
diff --git a/include/llvm/XRay/FDRRecordProducer.h b/include/llvm/XRay/FDRRecordProducer.h
index efdba2a67b7bec66cede54e29c97ca06ba912346..b530a85bc7e13334f5fcb0f714983529f5954363 100644
--- a/include/llvm/XRay/FDRRecordProducer.h
+++ b/include/llvm/XRay/FDRRecordProducer.h
@@ -1,9 +1,8 @@
 //===- FDRRecordProducer.h - XRay FDR Mode Record Producer ----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 #ifndef LLVM_INCLUDE_LLVM_XRAY_FDRRECORDPRODUCER_H_
diff --git a/include/llvm/XRay/FDRRecords.h b/include/llvm/XRay/FDRRecords.h
index 8a84f4d0c1fbc6e7427108005dab0a50de704fe6..a8ce74bd88fbd3a960a3e6bf7cb968d600bc26d2 100644
--- a/include/llvm/XRay/FDRRecords.h
+++ b/include/llvm/XRay/FDRRecords.h
@@ -1,9 +1,8 @@
 //===- FDRRecords.h - XRay Flight Data Recorder Mode Records --------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/XRay/FDRTraceExpander.h b/include/llvm/XRay/FDRTraceExpander.h
index 02a21bed5ce916fabab76ee8cbe22dc408bacdb3..f3c36cf4cf66ba5557827e106811096d1b5e8276 100644
--- a/include/llvm/XRay/FDRTraceExpander.h
+++ b/include/llvm/XRay/FDRTraceExpander.h
@@ -1,9 +1,8 @@
 //===- FDRTraceExpander.h - XRay FDR Mode Log Expander --------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/XRay/FDRTraceWriter.h b/include/llvm/XRay/FDRTraceWriter.h
index 7b3b5fa25eff4b31dae47b45ddfb7b2058444839..2498877e27c18adcb6f899a56acf7956d6ac2c65 100644
--- a/include/llvm/XRay/FDRTraceWriter.h
+++ b/include/llvm/XRay/FDRTraceWriter.h
@@ -1,9 +1,8 @@
 //===- FDRTraceWriter.h - XRay FDR Trace Writer -----------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/XRay/FileHeaderReader.h b/include/llvm/XRay/FileHeaderReader.h
index 3b8809bdbb3426446a4cf7a01c0e44234ce73475..1c9681cfd9aff82f950129488c2728ee37b80010 100644
--- a/include/llvm/XRay/FileHeaderReader.h
+++ b/include/llvm/XRay/FileHeaderReader.h
@@ -1,9 +1,8 @@
 //===- FileHeaderReader.h - XRay Trace File Header Reading Function -------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/XRay/Graph.h b/include/llvm/XRay/Graph.h
index a4d34a8a4be34d228f4fe37ab3ea68b5cf10bed7..004681512800f9b80c08ac8791f1b36acc12333f 100644
--- a/include/llvm/XRay/Graph.h
+++ b/include/llvm/XRay/Graph.h
@@ -1,9 +1,8 @@
 //===-- Graph.h - XRay Graph Class ------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/XRay/InstrumentationMap.h b/include/llvm/XRay/InstrumentationMap.h
index 42bfca36a20bc0fde9eb2b2d9b32de33cdb4bf09..5cbe5c44893b21d7884dac5b01aff23f14f4261e 100644
--- a/include/llvm/XRay/InstrumentationMap.h
+++ b/include/llvm/XRay/InstrumentationMap.h
@@ -1,9 +1,8 @@
 //===- InstrumentationMap.h - XRay Instrumentation Map ----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/XRay/Profile.h b/include/llvm/XRay/Profile.h
index 9365630358e8b64547fe50d1bf5e325a8c35d733..79d9b53387f3933d41cc420b03d259c394cba7ba 100644
--- a/include/llvm/XRay/Profile.h
+++ b/include/llvm/XRay/Profile.h
@@ -1,9 +1,8 @@
 //===- Profile.h - XRay Profile Abstraction -------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/XRay/RecordPrinter.h b/include/llvm/XRay/RecordPrinter.h
index 649c64ab6f5cbfbee1af0418f795ef3a349ca69b..7c7b7a32c56dc6c1a4e371c9aee0be7060ab41e2 100644
--- a/include/llvm/XRay/RecordPrinter.h
+++ b/include/llvm/XRay/RecordPrinter.h
@@ -1,9 +1,8 @@
 //===- RecordPrinter.h - FDR Record Printer -------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/XRay/Trace.h b/include/llvm/XRay/Trace.h
index 924addd1560d8e93b257f6dd265325d46963ca17..eb1f03b2a0d4a3cf65219727fbb3f47ed95cf537 100644
--- a/include/llvm/XRay/Trace.h
+++ b/include/llvm/XRay/Trace.h
@@ -1,9 +1,8 @@
 //===- Trace.h - XRay Trace Abstraction -----------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/XRay/XRayRecord.h b/include/llvm/XRay/XRayRecord.h
index 7685ec95838a4ed67e1f2fd59cccd53cadac9c5a..546b02303b6a8adeb9622f6b263f58ccc8bc24a1 100644
--- a/include/llvm/XRay/XRayRecord.h
+++ b/include/llvm/XRay/XRayRecord.h
@@ -1,9 +1,8 @@
 //===- XRayRecord.h - XRay Trace Record -----------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/XRay/YAMLXRayRecord.h b/include/llvm/XRay/YAMLXRayRecord.h
index 6150196ed98df939f1ac6354e8fc8965d872e1bb..bc8b03548d6e2b3f6599ff85b6df19836c1cde84 100644
--- a/include/llvm/XRay/YAMLXRayRecord.h
+++ b/include/llvm/XRay/YAMLXRayRecord.h
@@ -1,9 +1,8 @@
 //===- YAMLXRayRecord.h - XRay Record YAML Support Definitions ------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/include/llvm/module.modulemap b/include/llvm/module.modulemap
index 3b6e2a8a0ba42d51e4eaa3273bf6d9750abf5c39..65c9495e930644f3d82ac906f2d5cd513c0743ca 100644
--- a/include/llvm/module.modulemap
+++ b/include/llvm/module.modulemap
@@ -7,7 +7,11 @@ module LLVM_Analysis {
   textual header "Analysis/TargetLibraryInfo.def"
 }
 
-module LLVM_AsmParser { requires cplusplus umbrella "AsmParser" module * { export * } }
+module LLVM_AsmParser {
+  requires cplusplus
+  umbrella "AsmParser"
+  module * { export * }
+}
 
 // A module covering CodeGen/ and Target/. These are intertwined
 // and codependent, and thus notionally form a single module.
@@ -27,14 +31,20 @@ module LLVM_Backend {
     textual header "CodeGen/CommandFlags.inc"
     textual header "CodeGen/DIEValue.def"
   }
+}
 
-  module Target {
-    umbrella "Target"
-    module * { export * }
-  }
+// FIXME: Make this as a submodule of LLVM_Backend again.
+//        Doing so causes a linker error in clang-format.
+module LLVM_Backend_Target {
+  umbrella "Target"
+  module * { export * }
 }
 
-module LLVM_Bitcode { requires cplusplus umbrella "Bitcode" module * { export * } }
+module LLVM_Bitcode {
+ requires cplusplus
+ umbrella "Bitcode"
+ module * { export * }
+}
 
 
 module LLVM_BinaryFormat {
@@ -210,7 +220,7 @@ module LLVM_intrinsic_gen {
   module IR_ConstantRange { header "IR/ConstantRange.h" export * }
   module IR_Dominators { header "IR/Dominators.h" export * }
   module Analysis_PostDominators { header "Analysis/PostDominators.h" export * }
-  module IR_DomTreeUpdater { header "IR/DomTreeUpdater.h" export * }
+  module Analysis_DomTreeUpdater { header "Analysis/DomTreeUpdater.h" export * }
   module IR_IRBuilder { header "IR/IRBuilder.h" export * }
   module IR_PassManager { header "IR/PassManager.h" export * }
   module IR_PredIteratorCache { header "IR/PredIteratorCache.h" export * }
@@ -244,9 +254,23 @@ module LLVM_IR {
   textual header "IR/RuntimeLibcalls.def"
 }
 
-module LLVM_IRReader { requires cplusplus umbrella "IRReader" module * { export * } }
-module LLVM_LineEditor { requires cplusplus umbrella "LineEditor" module * { export * } }
-module LLVM_LTO { requires cplusplus umbrella "LTO" module * { export * } }
+module LLVM_IRReader {
+  requires cplusplus
+  umbrella "IRReader"
+  module * { export * }
+}
+
+module LLVM_LineEditor {
+  requires cplusplus
+  umbrella "LineEditor"
+  module * { export * }
+}
+
+module LLVM_LTO {
+  requires cplusplus
+  umbrella "LTO"
+  module * { export * }
+}
 
 module LLVM_MC {
   requires cplusplus
@@ -273,7 +297,11 @@ module LLVM_Object {
   module * { export * }
 }
 
-module LLVM_Option { requires cplusplus umbrella "Option" module * { export * } }
+module LLVM_Option {
+  requires cplusplus
+  umbrella "Option"
+  module * { export * }
+}
 
 module LLVM_ProfileData {
   requires cplusplus
@@ -291,7 +319,11 @@ module LLVM_Support_TargetRegistry {
   export *
 }
 
-module LLVM_TableGen { requires cplusplus umbrella "TableGen" module * { export * } }
+module LLVM_TableGen {
+  requires cplusplus
+  umbrella "TableGen"
+  module * { export * }
+}
 
 module LLVM_Transforms {
   requires cplusplus
diff --git a/lib/Analysis/AliasAnalysis.cpp b/lib/Analysis/AliasAnalysis.cpp
index 8ed48390818b988f001de71fd5f47f5760996a4b..c024e3a2c5b18120be1abf20bebf6a2364ba1cd4 100644
--- a/lib/Analysis/AliasAnalysis.cpp
+++ b/lib/Analysis/AliasAnalysis.cpp
@@ -1,9 +1,8 @@
 //==- AliasAnalysis.cpp - Generic Alias Analysis Interface Implementation --==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -40,7 +39,6 @@
 #include "llvm/IR/Argument.h"
 #include "llvm/IR/Attributes.h"
 #include "llvm/IR/BasicBlock.h"
-#include "llvm/IR/CallSite.h"
 #include "llvm/IR/Instruction.h"
 #include "llvm/IR/Instructions.h"
 #include "llvm/IR/Module.h"
@@ -118,11 +116,11 @@ bool AAResults::pointsToConstantMemory(const MemoryLocation &Loc,
   return false;
 }
 
-ModRefInfo AAResults::getArgModRefInfo(ImmutableCallSite CS, unsigned ArgIdx) {
+ModRefInfo AAResults::getArgModRefInfo(const CallBase *Call, unsigned ArgIdx) {
   ModRefInfo Result = ModRefInfo::ModRef;
 
   for (const auto &AA : AAs) {
-    Result = intersectModRef(Result, AA->getArgModRefInfo(CS, ArgIdx));
+    Result = intersectModRef(Result, AA->getArgModRefInfo(Call, ArgIdx));
 
     // Early-exit the moment we reach the bottom of the lattice.
     if (isNoModRef(Result))
@@ -132,11 +130,11 @@ ModRefInfo AAResults::getArgModRefInfo(ImmutableCallSite CS, unsigned ArgIdx) {
   return Result;
 }
 
-ModRefInfo AAResults::getModRefInfo(Instruction *I, ImmutableCallSite Call) {
+ModRefInfo AAResults::getModRefInfo(Instruction *I, const CallBase *Call2) {
   // We may have two calls.
-  if (auto CS = ImmutableCallSite(I)) {
+  if (const auto *Call1 = dyn_cast<CallBase>(I)) {
     // Check if the two calls modify the same memory.
-    return getModRefInfo(CS, Call);
+    return getModRefInfo(Call1, Call2);
   } else if (I->isFenceLike()) {
     // If this is a fence, just return ModRef.
     return ModRefInfo::ModRef;
@@ -146,19 +144,19 @@ ModRefInfo AAResults::getModRefInfo(Instruction *I, ImmutableCallSite Call) {
     // is that if the call references what this instruction
     // defines, it must be clobbered by this location.
     const MemoryLocation DefLoc = MemoryLocation::get(I);
-    ModRefInfo MR = getModRefInfo(Call, DefLoc);
+    ModRefInfo MR = getModRefInfo(Call2, DefLoc);
     if (isModOrRefSet(MR))
       return setModAndRef(MR);
   }
   return ModRefInfo::NoModRef;
 }
 
-ModRefInfo AAResults::getModRefInfo(ImmutableCallSite CS,
+ModRefInfo AAResults::getModRefInfo(const CallBase *Call,
                                     const MemoryLocation &Loc) {
   ModRefInfo Result = ModRefInfo::ModRef;
 
   for (const auto &AA : AAs) {
-    Result = intersectModRef(Result, AA->getModRefInfo(CS, Loc));
+    Result = intersectModRef(Result, AA->getModRefInfo(Call, Loc));
 
     // Early-exit the moment we reach the bottom of the lattice.
     if (isNoModRef(Result))
@@ -167,7 +165,7 @@ ModRefInfo AAResults::getModRefInfo(ImmutableCallSite CS,
 
   // Try to refine the mod-ref info further using other API entry points to the
   // aggregate set of AA results.
-  auto MRB = getModRefBehavior(CS);
+  auto MRB = getModRefBehavior(Call);
   if (MRB == FMRB_DoesNotAccessMemory ||
       MRB == FMRB_OnlyAccessesInaccessibleMem)
     return ModRefInfo::NoModRef;
@@ -181,15 +179,16 @@ ModRefInfo AAResults::getModRefInfo(ImmutableCallSite CS,
     bool IsMustAlias = true;
     ModRefInfo AllArgsMask = ModRefInfo::NoModRef;
     if (doesAccessArgPointees(MRB)) {
-      for (auto AI = CS.arg_begin(), AE = CS.arg_end(); AI != AE; ++AI) {
+      for (auto AI = Call->arg_begin(), AE = Call->arg_end(); AI != AE; ++AI) {
         const Value *Arg = *AI;
         if (!Arg->getType()->isPointerTy())
           continue;
-        unsigned ArgIdx = std::distance(CS.arg_begin(), AI);
-        MemoryLocation ArgLoc = MemoryLocation::getForArgument(CS, ArgIdx, TLI);
+        unsigned ArgIdx = std::distance(Call->arg_begin(), AI);
+        MemoryLocation ArgLoc =
+            MemoryLocation::getForArgument(Call, ArgIdx, TLI);
         AliasResult ArgAlias = alias(ArgLoc, Loc);
         if (ArgAlias != NoAlias) {
-          ModRefInfo ArgMask = getArgModRefInfo(CS, ArgIdx);
+          ModRefInfo ArgMask = getArgModRefInfo(Call, ArgIdx);
           AllArgsMask = unionModRef(AllArgsMask, ArgMask);
         }
         // Conservatively clear IsMustAlias unless only MustAlias is found.
@@ -213,12 +212,12 @@ ModRefInfo AAResults::getModRefInfo(ImmutableCallSite CS,
   return Result;
 }
 
-ModRefInfo AAResults::getModRefInfo(ImmutableCallSite CS1,
-                                    ImmutableCallSite CS2) {
+ModRefInfo AAResults::getModRefInfo(const CallBase *Call1,
+                                    const CallBase *Call2) {
   ModRefInfo Result = ModRefInfo::ModRef;
 
   for (const auto &AA : AAs) {
-    Result = intersectModRef(Result, AA->getModRefInfo(CS1, CS2));
+    Result = intersectModRef(Result, AA->getModRefInfo(Call1, Call2));
 
     // Early-exit the moment we reach the bottom of the lattice.
     if (isNoModRef(Result))
@@ -228,59 +227,61 @@ ModRefInfo AAResults::getModRefInfo(ImmutableCallSite CS1,
   // Try to refine the mod-ref info further using other API entry points to the
   // aggregate set of AA results.
 
-  // If CS1 or CS2 are readnone, they don't interact.
-  auto CS1B = getModRefBehavior(CS1);
-  if (CS1B == FMRB_DoesNotAccessMemory)
+  // If Call1 or Call2 are readnone, they don't interact.
+  auto Call1B = getModRefBehavior(Call1);
+  if (Call1B == FMRB_DoesNotAccessMemory)
     return ModRefInfo::NoModRef;
 
-  auto CS2B = getModRefBehavior(CS2);
-  if (CS2B == FMRB_DoesNotAccessMemory)
+  auto Call2B = getModRefBehavior(Call2);
+  if (Call2B == FMRB_DoesNotAccessMemory)
     return ModRefInfo::NoModRef;
 
   // If they both only read from memory, there is no dependence.
-  if (onlyReadsMemory(CS1B) && onlyReadsMemory(CS2B))
+  if (onlyReadsMemory(Call1B) && onlyReadsMemory(Call2B))
     return ModRefInfo::NoModRef;
 
-  // If CS1 only reads memory, the only dependence on CS2 can be
-  // from CS1 reading memory written by CS2.
-  if (onlyReadsMemory(CS1B))
+  // If Call1 only reads memory, the only dependence on Call2 can be
+  // from Call1 reading memory written by Call2.
+  if (onlyReadsMemory(Call1B))
     Result = clearMod(Result);
-  else if (doesNotReadMemory(CS1B))
+  else if (doesNotReadMemory(Call1B))
     Result = clearRef(Result);
 
-  // If CS2 only access memory through arguments, accumulate the mod/ref
-  // information from CS1's references to the memory referenced by
-  // CS2's arguments.
-  if (onlyAccessesArgPointees(CS2B)) {
-    if (!doesAccessArgPointees(CS2B))
+  // If Call2 only access memory through arguments, accumulate the mod/ref
+  // information from Call1's references to the memory referenced by
+  // Call2's arguments.
+  if (onlyAccessesArgPointees(Call2B)) {
+    if (!doesAccessArgPointees(Call2B))
       return ModRefInfo::NoModRef;
     ModRefInfo R = ModRefInfo::NoModRef;
     bool IsMustAlias = true;
-    for (auto I = CS2.arg_begin(), E = CS2.arg_end(); I != E; ++I) {
+    for (auto I = Call2->arg_begin(), E = Call2->arg_end(); I != E; ++I) {
       const Value *Arg = *I;
       if (!Arg->getType()->isPointerTy())
         continue;
-      unsigned CS2ArgIdx = std::distance(CS2.arg_begin(), I);
-      auto CS2ArgLoc = MemoryLocation::getForArgument(CS2, CS2ArgIdx, TLI);
-
-      // ArgModRefCS2 indicates what CS2 might do to CS2ArgLoc, and the
-      // dependence of CS1 on that location is the inverse:
-      // - If CS2 modifies location, dependence exists if CS1 reads or writes.
-      // - If CS2 only reads location, dependence exists if CS1 writes.
-      ModRefInfo ArgModRefCS2 = getArgModRefInfo(CS2, CS2ArgIdx);
+      unsigned Call2ArgIdx = std::distance(Call2->arg_begin(), I);
+      auto Call2ArgLoc =
+          MemoryLocation::getForArgument(Call2, Call2ArgIdx, TLI);
+
+      // ArgModRefC2 indicates what Call2 might do to Call2ArgLoc, and the
+      // dependence of Call1 on that location is the inverse:
+      // - If Call2 modifies location, dependence exists if Call1 reads or
+      //   writes.
+      // - If Call2 only reads location, dependence exists if Call1 writes.
+      ModRefInfo ArgModRefC2 = getArgModRefInfo(Call2, Call2ArgIdx);
       ModRefInfo ArgMask = ModRefInfo::NoModRef;
-      if (isModSet(ArgModRefCS2))
+      if (isModSet(ArgModRefC2))
         ArgMask = ModRefInfo::ModRef;
-      else if (isRefSet(ArgModRefCS2))
+      else if (isRefSet(ArgModRefC2))
         ArgMask = ModRefInfo::Mod;
 
-      // ModRefCS1 indicates what CS1 might do to CS2ArgLoc, and we use
+      // ModRefC1 indicates what Call1 might do to Call2ArgLoc, and we use
       // above ArgMask to update dependence info.
-      ModRefInfo ModRefCS1 = getModRefInfo(CS1, CS2ArgLoc);
-      ArgMask = intersectModRef(ArgMask, ModRefCS1);
+      ModRefInfo ModRefC1 = getModRefInfo(Call1, Call2ArgLoc);
+      ArgMask = intersectModRef(ArgMask, ModRefC1);
 
       // Conservatively clear IsMustAlias unless only MustAlias is found.
-      IsMustAlias &= isMustSet(ModRefCS1);
+      IsMustAlias &= isMustSet(ModRefC1);
 
       R = intersectModRef(unionModRef(R, ArgMask), Result);
       if (R == Result) {
@@ -298,31 +299,32 @@ ModRefInfo AAResults::getModRefInfo(ImmutableCallSite CS1,
     return IsMustAlias ? setMust(R) : clearMust(R);
   }
 
-  // If CS1 only accesses memory through arguments, check if CS2 references
-  // any of the memory referenced by CS1's arguments. If not, return NoModRef.
-  if (onlyAccessesArgPointees(CS1B)) {
-    if (!doesAccessArgPointees(CS1B))
+  // If Call1 only accesses memory through arguments, check if Call2 references
+  // any of the memory referenced by Call1's arguments. If not, return NoModRef.
+  if (onlyAccessesArgPointees(Call1B)) {
+    if (!doesAccessArgPointees(Call1B))
       return ModRefInfo::NoModRef;
     ModRefInfo R = ModRefInfo::NoModRef;
     bool IsMustAlias = true;
-    for (auto I = CS1.arg_begin(), E = CS1.arg_end(); I != E; ++I) {
+    for (auto I = Call1->arg_begin(), E = Call1->arg_end(); I != E; ++I) {
       const Value *Arg = *I;
       if (!Arg->getType()->isPointerTy())
         continue;
-      unsigned CS1ArgIdx = std::distance(CS1.arg_begin(), I);
-      auto CS1ArgLoc = MemoryLocation::getForArgument(CS1, CS1ArgIdx, TLI);
-
-      // ArgModRefCS1 indicates what CS1 might do to CS1ArgLoc; if CS1 might
-      // Mod CS1ArgLoc, then we care about either a Mod or a Ref by CS2. If
-      // CS1 might Ref, then we care only about a Mod by CS2.
-      ModRefInfo ArgModRefCS1 = getArgModRefInfo(CS1, CS1ArgIdx);
-      ModRefInfo ModRefCS2 = getModRefInfo(CS2, CS1ArgLoc);
-      if ((isModSet(ArgModRefCS1) && isModOrRefSet(ModRefCS2)) ||
-          (isRefSet(ArgModRefCS1) && isModSet(ModRefCS2)))
-        R = intersectModRef(unionModRef(R, ArgModRefCS1), Result);
+      unsigned Call1ArgIdx = std::distance(Call1->arg_begin(), I);
+      auto Call1ArgLoc =
+          MemoryLocation::getForArgument(Call1, Call1ArgIdx, TLI);
+
+      // ArgModRefC1 indicates what Call1 might do to Call1ArgLoc; if Call1
+      // might Mod Call1ArgLoc, then we care about either a Mod or a Ref by
+      // Call2. If Call1 might Ref, then we care only about a Mod by Call2.
+      ModRefInfo ArgModRefC1 = getArgModRefInfo(Call1, Call1ArgIdx);
+      ModRefInfo ModRefC2 = getModRefInfo(Call2, Call1ArgLoc);
+      if ((isModSet(ArgModRefC1) && isModOrRefSet(ModRefC2)) ||
+          (isRefSet(ArgModRefC1) && isModSet(ModRefC2)))
+        R = intersectModRef(unionModRef(R, ArgModRefC1), Result);
 
       // Conservatively clear IsMustAlias unless only MustAlias is found.
-      IsMustAlias &= isMustSet(ModRefCS2);
+      IsMustAlias &= isMustSet(ModRefC2);
 
       if (R == Result) {
         // On early exit, not all args were checked, cannot set Must.
@@ -342,11 +344,11 @@ ModRefInfo AAResults::getModRefInfo(ImmutableCallSite CS1,
   return Result;
 }
 
-FunctionModRefBehavior AAResults::getModRefBehavior(ImmutableCallSite CS) {
+FunctionModRefBehavior AAResults::getModRefBehavior(const CallBase *Call) {
   FunctionModRefBehavior Result = FMRB_UnknownModRefBehavior;
 
   for (const auto &AA : AAs) {
-    Result = FunctionModRefBehavior(Result & AA->getModRefBehavior(CS));
+    Result = FunctionModRefBehavior(Result & AA->getModRefBehavior(Call));
 
     // Early-exit the moment we reach the bottom of the lattice.
     if (Result == FMRB_DoesNotAccessMemory)
@@ -558,8 +560,8 @@ ModRefInfo AAResults::callCapturesBefore(const Instruction *I,
       isa<Constant>(Object))
     return ModRefInfo::ModRef;
 
-  ImmutableCallSite CS(I);
-  if (!CS.getInstruction() || CS.getInstruction() == Object)
+  const auto *Call = dyn_cast<CallBase>(I);
+  if (!Call || Call == Object)
     return ModRefInfo::ModRef;
 
   if (PointerMayBeCapturedBefore(Object, /* ReturnCaptures */ true,
@@ -572,14 +574,14 @@ ModRefInfo AAResults::callCapturesBefore(const Instruction *I,
   ModRefInfo R = ModRefInfo::NoModRef;
   bool IsMustAlias = true;
   // Set flag only if no May found and all operands processed.
-  for (auto CI = CS.data_operands_begin(), CE = CS.data_operands_end();
+  for (auto CI = Call->data_operands_begin(), CE = Call->data_operands_end();
        CI != CE; ++CI, ++ArgNo) {
     // Only look at the no-capture or byval pointer arguments.  If this
     // pointer were passed to arguments that were neither of these, then it
     // couldn't be no-capture.
     if (!(*CI)->getType()->isPointerTy() ||
-        (!CS.doesNotCapture(ArgNo) &&
-         ArgNo < CS.getNumArgOperands() && !CS.isByValArgument(ArgNo)))
+        (!Call->doesNotCapture(ArgNo) && ArgNo < Call->getNumArgOperands() &&
+         !Call->isByValArgument(ArgNo)))
       continue;
 
     AliasResult AR = alias(MemoryLocation(*CI), MemoryLocation(Object));
@@ -591,9 +593,9 @@ ModRefInfo AAResults::callCapturesBefore(const Instruction *I,
       IsMustAlias = false;
     if (AR == NoAlias)
       continue;
-    if (CS.doesNotAccessMemory(ArgNo))
+    if (Call->doesNotAccessMemory(ArgNo))
       continue;
-    if (CS.onlyReadsMemory(ArgNo)) {
+    if (Call->onlyReadsMemory(ArgNo)) {
       R = ModRefInfo::Ref;
       continue;
     }
@@ -775,8 +777,8 @@ AAResults llvm::createLegacyPMAAResults(Pass &P, Function &F,
 }
 
 bool llvm::isNoAliasCall(const Value *V) {
-  if (auto CS = ImmutableCallSite(V))
-    return CS.hasRetAttr(Attribute::NoAlias);
+  if (const auto *Call = dyn_cast<CallBase>(V))
+    return Call->hasRetAttr(Attribute::NoAlias);
   return false;
 }
 
diff --git a/lib/Analysis/AliasAnalysisEvaluator.cpp b/lib/Analysis/AliasAnalysisEvaluator.cpp
index 764ae9160350586781141ccb8f28005dbe64fe8c..e83703867e09a1180cf8dcb432bcb2d401bd8ee5 100644
--- a/lib/Analysis/AliasAnalysisEvaluator.cpp
+++ b/lib/Analysis/AliasAnalysisEvaluator.cpp
@@ -1,9 +1,8 @@
 //===- AliasAnalysisEvaluator.cpp - Alias Analysis Accuracy Evaluator -----===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -66,11 +65,10 @@ static inline void PrintModRefResults(const char *Msg, bool P, Instruction *I,
   }
 }
 
-static inline void PrintModRefResults(const char *Msg, bool P, CallSite CSA,
-                                      CallSite CSB, Module *M) {
+static inline void PrintModRefResults(const char *Msg, bool P, CallBase *CallA,
+                                      CallBase *CallB, Module *M) {
   if (PrintAll || P) {
-    errs() << "  " << Msg << ": " << *CSA.getInstruction() << " <-> "
-           << *CSB.getInstruction() << '\n';
+    errs() << "  " << Msg << ": " << *CallA << " <-> " << *CallB << '\n';
   }
 }
 
@@ -98,7 +96,7 @@ void AAEvaluator::runInternal(Function &F, AAResults &AA) {
   ++FunctionCount;
 
   SetVector<Value *> Pointers;
-  SmallSetVector<CallSite, 16> CallSites;
+  SmallSetVector<CallBase *, 16> Calls;
   SetVector<Value *> Loads;
   SetVector<Value *> Stores;
 
@@ -114,16 +112,16 @@ void AAEvaluator::runInternal(Function &F, AAResults &AA) {
     if (EvalAAMD && isa<StoreInst>(&*I))
       Stores.insert(&*I);
     Instruction &Inst = *I;
-    if (auto CS = CallSite(&Inst)) {
-      Value *Callee = CS.getCalledValue();
+    if (auto *Call = dyn_cast<CallBase>(&Inst)) {
+      Value *Callee = Call->getCalledValue();
       // Skip actual functions for direct function calls.
       if (!isa<Function>(Callee) && isInterestingPointer(Callee))
         Pointers.insert(Callee);
       // Consider formals.
-      for (Use &DataOp : CS.data_ops())
+      for (Use &DataOp : Call->data_ops())
         if (isInterestingPointer(DataOp))
           Pointers.insert(DataOp);
-      CallSites.insert(CS);
+      Calls.insert(Call);
     } else {
       // Consider all operands.
       for (Instruction::op_iterator OI = Inst.op_begin(), OE = Inst.op_end();
@@ -136,19 +134,21 @@ void AAEvaluator::runInternal(Function &F, AAResults &AA) {
   if (PrintAll || PrintNoAlias || PrintMayAlias || PrintPartialAlias ||
       PrintMustAlias || PrintNoModRef || PrintMod || PrintRef || PrintModRef)
     errs() << "Function: " << F.getName() << ": " << Pointers.size()
-           << " pointers, " << CallSites.size() << " call sites\n";
+           << " pointers, " << Calls.size() << " call sites\n";
 
   // iterate over the worklist, and run the full (n^2)/2 disambiguations
   for (SetVector<Value *>::iterator I1 = Pointers.begin(), E = Pointers.end();
        I1 != E; ++I1) {
-    uint64_t I1Size = MemoryLocation::UnknownSize;
+    auto I1Size = LocationSize::unknown();
     Type *I1ElTy = cast<PointerType>((*I1)->getType())->getElementType();
-    if (I1ElTy->isSized()) I1Size = DL.getTypeStoreSize(I1ElTy);
+    if (I1ElTy->isSized())
+      I1Size = LocationSize::precise(DL.getTypeStoreSize(I1ElTy));
 
     for (SetVector<Value *>::iterator I2 = Pointers.begin(); I2 != I1; ++I2) {
-      uint64_t I2Size = MemoryLocation::UnknownSize;
-      Type *I2ElTy =cast<PointerType>((*I2)->getType())->getElementType();
-      if (I2ElTy->isSized()) I2Size = DL.getTypeStoreSize(I2ElTy);
+      auto I2Size = LocationSize::unknown();
+      Type *I2ElTy = cast<PointerType>((*I2)->getType())->getElementType();
+      if (I2ElTy->isSized())
+        I2Size = LocationSize::precise(DL.getTypeStoreSize(I2ElTy));
 
       AliasResult AR = AA.alias(*I1, I1Size, *I2, I2Size);
       switch (AR) {
@@ -228,49 +228,48 @@ void AAEvaluator::runInternal(Function &F, AAResults &AA) {
   }
 
   // Mod/ref alias analysis: compare all pairs of calls and values
-  for (CallSite C : CallSites) {
-    Instruction *I = C.getInstruction();
-
+  for (CallBase *Call : Calls) {
     for (auto Pointer : Pointers) {
-      uint64_t Size = MemoryLocation::UnknownSize;
+      auto Size = LocationSize::unknown();
       Type *ElTy = cast<PointerType>(Pointer->getType())->getElementType();
-      if (ElTy->isSized()) Size = DL.getTypeStoreSize(ElTy);
+      if (ElTy->isSized())
+        Size = LocationSize::precise(DL.getTypeStoreSize(ElTy));
 
-      switch (AA.getModRefInfo(C, Pointer, Size)) {
+      switch (AA.getModRefInfo(Call, Pointer, Size)) {
       case ModRefInfo::NoModRef:
-        PrintModRefResults("NoModRef", PrintNoModRef, I, Pointer,
+        PrintModRefResults("NoModRef", PrintNoModRef, Call, Pointer,
                            F.getParent());
         ++NoModRefCount;
         break;
       case ModRefInfo::Mod:
-        PrintModRefResults("Just Mod", PrintMod, I, Pointer, F.getParent());
+        PrintModRefResults("Just Mod", PrintMod, Call, Pointer, F.getParent());
         ++ModCount;
         break;
       case ModRefInfo::Ref:
-        PrintModRefResults("Just Ref", PrintRef, I, Pointer, F.getParent());
+        PrintModRefResults("Just Ref", PrintRef, Call, Pointer, F.getParent());
         ++RefCount;
         break;
       case ModRefInfo::ModRef:
-        PrintModRefResults("Both ModRef", PrintModRef, I, Pointer,
+        PrintModRefResults("Both ModRef", PrintModRef, Call, Pointer,
                            F.getParent());
         ++ModRefCount;
         break;
       case ModRefInfo::Must:
-        PrintModRefResults("Must", PrintMust, I, Pointer, F.getParent());
+        PrintModRefResults("Must", PrintMust, Call, Pointer, F.getParent());
         ++MustCount;
         break;
       case ModRefInfo::MustMod:
-        PrintModRefResults("Just Mod (MustAlias)", PrintMustMod, I, Pointer,
+        PrintModRefResults("Just Mod (MustAlias)", PrintMustMod, Call, Pointer,
                            F.getParent());
         ++MustModCount;
         break;
       case ModRefInfo::MustRef:
-        PrintModRefResults("Just Ref (MustAlias)", PrintMustRef, I, Pointer,
+        PrintModRefResults("Just Ref (MustAlias)", PrintMustRef, Call, Pointer,
                            F.getParent());
         ++MustRefCount;
         break;
       case ModRefInfo::MustModRef:
-        PrintModRefResults("Both ModRef (MustAlias)", PrintMustModRef, I,
+        PrintModRefResults("Both ModRef (MustAlias)", PrintMustModRef, Call,
                            Pointer, F.getParent());
         ++MustModRefCount;
         break;
@@ -279,44 +278,46 @@ void AAEvaluator::runInternal(Function &F, AAResults &AA) {
   }
 
   // Mod/ref alias analysis: compare all pairs of calls
-  for (auto C = CallSites.begin(), Ce = CallSites.end(); C != Ce; ++C) {
-    for (auto D = CallSites.begin(); D != Ce; ++D) {
-      if (D == C)
+  for (CallBase *CallA : Calls) {
+    for (CallBase *CallB : Calls) {
+      if (CallA == CallB)
         continue;
-      switch (AA.getModRefInfo(*C, *D)) {
+      switch (AA.getModRefInfo(CallA, CallB)) {
       case ModRefInfo::NoModRef:
-        PrintModRefResults("NoModRef", PrintNoModRef, *C, *D, F.getParent());
+        PrintModRefResults("NoModRef", PrintNoModRef, CallA, CallB,
+                           F.getParent());
         ++NoModRefCount;
         break;
       case ModRefInfo::Mod:
-        PrintModRefResults("Just Mod", PrintMod, *C, *D, F.getParent());
+        PrintModRefResults("Just Mod", PrintMod, CallA, CallB, F.getParent());
         ++ModCount;
         break;
       case ModRefInfo::Ref:
-        PrintModRefResults("Just Ref", PrintRef, *C, *D, F.getParent());
+        PrintModRefResults("Just Ref", PrintRef, CallA, CallB, F.getParent());
         ++RefCount;
         break;
       case ModRefInfo::ModRef:
-        PrintModRefResults("Both ModRef", PrintModRef, *C, *D, F.getParent());
+        PrintModRefResults("Both ModRef", PrintModRef, CallA, CallB,
+                           F.getParent());
         ++ModRefCount;
         break;
       case ModRefInfo::Must:
-        PrintModRefResults("Must", PrintMust, *C, *D, F.getParent());
+        PrintModRefResults("Must", PrintMust, CallA, CallB, F.getParent());
         ++MustCount;
         break;
       case ModRefInfo::MustMod:
-        PrintModRefResults("Just Mod (MustAlias)", PrintMustMod, *C, *D,
+        PrintModRefResults("Just Mod (MustAlias)", PrintMustMod, CallA, CallB,
                            F.getParent());
         ++MustModCount;
         break;
       case ModRefInfo::MustRef:
-        PrintModRefResults("Just Ref (MustAlias)", PrintMustRef, *C, *D,
+        PrintModRefResults("Just Ref (MustAlias)", PrintMustRef, CallA, CallB,
                            F.getParent());
         ++MustRefCount;
         break;
       case ModRefInfo::MustModRef:
-        PrintModRefResults("Both ModRef (MustAlias)", PrintMustModRef, *C, *D,
-                           F.getParent());
+        PrintModRefResults("Both ModRef (MustAlias)", PrintMustModRef, CallA,
+                           CallB, F.getParent());
         ++MustModRefCount;
         break;
       }
diff --git a/lib/Analysis/AliasAnalysisSummary.cpp b/lib/Analysis/AliasAnalysisSummary.cpp
index 2b4879453beba38f393d0418f75253796bc7cc57..2f3396a44117a256619ca5518f9a9e7e3f7339b8 100644
--- a/lib/Analysis/AliasAnalysisSummary.cpp
+++ b/lib/Analysis/AliasAnalysisSummary.cpp
@@ -73,28 +73,28 @@ AliasAttrs getExternallyVisibleAttrs(AliasAttrs Attr) {
 }
 
 Optional<InstantiatedValue> instantiateInterfaceValue(InterfaceValue IValue,
-                                                      CallSite CS) {
+                                                      CallBase &Call) {
   auto Index = IValue.Index;
-  auto Value = (Index == 0) ? CS.getInstruction() : CS.getArgument(Index - 1);
-  if (Value->getType()->isPointerTy())
-    return InstantiatedValue{Value, IValue.DerefLevel};
+  auto *V = (Index == 0) ? &Call : Call.getArgOperand(Index - 1);
+  if (V->getType()->isPointerTy())
+    return InstantiatedValue{V, IValue.DerefLevel};
   return None;
 }
 
 Optional<InstantiatedRelation>
-instantiateExternalRelation(ExternalRelation ERelation, CallSite CS) {
-  auto From = instantiateInterfaceValue(ERelation.From, CS);
+instantiateExternalRelation(ExternalRelation ERelation, CallBase &Call) {
+  auto From = instantiateInterfaceValue(ERelation.From, Call);
   if (!From)
     return None;
-  auto To = instantiateInterfaceValue(ERelation.To, CS);
+  auto To = instantiateInterfaceValue(ERelation.To, Call);
   if (!To)
     return None;
   return InstantiatedRelation{*From, *To, ERelation.Offset};
 }
 
 Optional<InstantiatedAttr> instantiateExternalAttribute(ExternalAttribute EAttr,
-                                                        CallSite CS) {
-  auto Value = instantiateInterfaceValue(EAttr.IValue, CS);
+                                                        CallBase &Call) {
+  auto Value = instantiateInterfaceValue(EAttr.IValue, Call);
   if (!Value)
     return None;
   return InstantiatedAttr{*Value, EAttr.Attr};
diff --git a/lib/Analysis/AliasAnalysisSummary.h b/lib/Analysis/AliasAnalysisSummary.h
index fb93a12420f852cbbec460db8a69524c96b4dccf..fe75b03cedef06b7e7dffef99c0b647dca75525a 100644
--- a/lib/Analysis/AliasAnalysisSummary.h
+++ b/lib/Analysis/AliasAnalysisSummary.h
@@ -1,9 +1,8 @@
 //=====- CFLSummary.h - Abstract stratified sets implementation. --------=====//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
@@ -38,7 +37,7 @@
 #include "llvm/ADT/DenseMapInfo.h"
 #include "llvm/ADT/Optional.h"
 #include "llvm/ADT/SmallVector.h"
-#include "llvm/IR/CallSite.h"
+#include "llvm/IR/InstrTypes.h"
 #include <bitset>
 
 namespace llvm {
@@ -196,12 +195,13 @@ struct AliasSummary {
   SmallVector<ExternalAttribute, 8> RetParamAttributes;
 };
 
-/// This is the result of instantiating InterfaceValue at a particular callsite
+/// This is the result of instantiating InterfaceValue at a particular call
 struct InstantiatedValue {
   Value *Val;
   unsigned DerefLevel;
 };
-Optional<InstantiatedValue> instantiateInterfaceValue(InterfaceValue, CallSite);
+Optional<InstantiatedValue> instantiateInterfaceValue(InterfaceValue IValue,
+                                                      CallBase &Call);
 
 inline bool operator==(InstantiatedValue LHS, InstantiatedValue RHS) {
   return LHS.Val == RHS.Val && LHS.DerefLevel == RHS.DerefLevel;
@@ -229,8 +229,8 @@ struct InstantiatedRelation {
   InstantiatedValue From, To;
   int64_t Offset;
 };
-Optional<InstantiatedRelation> instantiateExternalRelation(ExternalRelation,
-                                                           CallSite);
+Optional<InstantiatedRelation>
+instantiateExternalRelation(ExternalRelation ERelation, CallBase &Call);
 
 /// This is the result of instantiating ExternalAttribute at a particular
 /// callsite
@@ -238,8 +238,8 @@ struct InstantiatedAttr {
   InstantiatedValue IValue;
   AliasAttrs Attr;
 };
-Optional<InstantiatedAttr> instantiateExternalAttribute(ExternalAttribute,
-                                                        CallSite);
+Optional<InstantiatedAttr> instantiateExternalAttribute(ExternalAttribute EAttr,
+                                                        CallBase &Call);
 }
 
 template <> struct DenseMapInfo<cflaa::InstantiatedValue> {
diff --git a/lib/Analysis/AliasSetTracker.cpp b/lib/Analysis/AliasSetTracker.cpp
index c152b0ddecae5e84c25990dad83f2ef565e66f03..a6e5b9fab558d09b6fd3d3da0530443e72ca3c5f 100644
--- a/lib/Analysis/AliasSetTracker.cpp
+++ b/lib/Analysis/AliasSetTracker.cpp
@@ -1,9 +1,8 @@
 //===- AliasSetTracker.cpp - Alias Sets Tracker implementation-------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -14,9 +13,10 @@
 #include "llvm/Analysis/AliasSetTracker.h"
 #include "llvm/Analysis/AliasAnalysis.h"
 #include "llvm/Analysis/GuardUtils.h"
+#include "llvm/Analysis/LoopInfo.h"
 #include "llvm/Analysis/MemoryLocation.h"
+#include "llvm/Analysis/MemorySSA.h"
 #include "llvm/Config/llvm-config.h"
-#include "llvm/IR/CallSite.h"
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/DataLayout.h"
 #include "llvm/IR/Function.h"
@@ -128,24 +128,24 @@ void AliasSet::removeFromTracker(AliasSetTracker &AST) {
 
 void AliasSet::addPointer(AliasSetTracker &AST, PointerRec &Entry,
                           LocationSize Size, const AAMDNodes &AAInfo,
-                          bool KnownMustAlias) {
+                          bool KnownMustAlias, bool SkipSizeUpdate) {
   assert(!Entry.hasAliasSet() && "Entry already in set!");
 
   // Check to see if we have to downgrade to _may_ alias.
-  if (isMustAlias() && !KnownMustAlias)
+  if (isMustAlias())
     if (PointerRec *P = getSomePointer()) {
-      AliasAnalysis &AA = AST.getAliasAnalysis();
-      AliasResult Result =
-          AA.alias(MemoryLocation(P->getValue(), P->getSize(), P->getAAInfo()),
-                   MemoryLocation(Entry.getValue(), Size, AAInfo));
-      if (Result != MustAlias) {
-        Alias = SetMayAlias;
-        AST.TotalMayAliasSetSize += size();
-      } else {
-        // First entry of must alias must have maximum size!
+      if (!KnownMustAlias) {
+        AliasAnalysis &AA = AST.getAliasAnalysis();
+        AliasResult Result = AA.alias(
+            MemoryLocation(P->getValue(), P->getSize(), P->getAAInfo()),
+            MemoryLocation(Entry.getValue(), Size, AAInfo));
+        if (Result != MustAlias) {
+          Alias = SetMayAlias;
+          AST.TotalMayAliasSetSize += size();
+        }
+        assert(Result != NoAlias && "Cannot be part of must set!");
+      } else if (!SkipSizeUpdate)
         P->updateSizeAndAAInfo(Size, AAInfo);
-      }
-      assert(Result != NoAlias && "Cannot be part of must set!");
     }
 
   Entry.setAliasSet(this);
@@ -185,14 +185,15 @@ void AliasSet::addUnknownInst(Instruction *I, AliasAnalysis &AA) {
   Access = ModRefAccess;
 }
 
-/// aliasesPointer - Return true if the specified pointer "may" (or must)
-/// alias one of the members in the set.
+/// aliasesPointer - If the specified pointer "may" (or must) alias one of the
+/// members in the set return the appropriate AliasResult. Otherwise return
+/// NoAlias.
 ///
-bool AliasSet::aliasesPointer(const Value *Ptr, LocationSize Size,
-                              const AAMDNodes &AAInfo,
-                              AliasAnalysis &AA) const {
+AliasResult AliasSet::aliasesPointer(const Value *Ptr, LocationSize Size,
+                                     const AAMDNodes &AAInfo,
+                                     AliasAnalysis &AA) const {
   if (AliasAny)
-    return true;
+    return MayAlias;
 
   if (Alias == SetMustAlias) {
     assert(UnknownInsts.empty() && "Illegal must alias set!");
@@ -209,9 +210,10 @@ bool AliasSet::aliasesPointer(const Value *Ptr, LocationSize Size,
   // If this is a may-alias set, we have to check all of the pointers in the set
   // to be sure it doesn't alias the set...
   for (iterator I = begin(), E = end(); I != E; ++I)
-    if (AA.alias(MemoryLocation(Ptr, Size, AAInfo),
-                 MemoryLocation(I.getPointer(), I.getSize(), I.getAAInfo())))
-      return true;
+    if (AliasResult AR = AA.alias(
+            MemoryLocation(Ptr, Size, AAInfo),
+            MemoryLocation(I.getPointer(), I.getSize(), I.getAAInfo())))
+      return AR;
 
   // Check the unknown instructions...
   if (!UnknownInsts.empty()) {
@@ -219,10 +221,10 @@ bool AliasSet::aliasesPointer(const Value *Ptr, LocationSize Size,
       if (auto *Inst = getUnknownInst(i))
         if (isModOrRefSet(
                 AA.getModRefInfo(Inst, MemoryLocation(Ptr, Size, AAInfo))))
-          return true;
+          return MayAlias;
   }
 
-  return false;
+  return NoAlias;
 }
 
 bool AliasSet::aliasesUnknownInst(const Instruction *Inst,
@@ -236,7 +238,8 @@ bool AliasSet::aliasesUnknownInst(const Instruction *Inst,
 
   for (unsigned i = 0, e = UnknownInsts.size(); i != e; ++i) {
     if (auto *UnknownInst = getUnknownInst(i)) {
-      ImmutableCallSite C1(UnknownInst), C2(Inst);
+      const auto *C1 = dyn_cast<CallBase>(UnknownInst);
+      const auto *C2 = dyn_cast<CallBase>(Inst);
       if (!C1 || !C2 || isModOrRefSet(AA.getModRefInfo(C1, C2)) ||
           isModOrRefSet(AA.getModRefInfo(C2, C1)))
         return true;
@@ -288,25 +291,38 @@ void AliasSetTracker::clear() {
   AliasSets.clear();
 }
 
-
 /// mergeAliasSetsForPointer - Given a pointer, merge all alias sets that may
 /// alias the pointer. Return the unified set, or nullptr if no set that aliases
-/// the pointer was found.
+/// the pointer was found. MustAliasAll is updated to true/false if the pointer
+/// is found to MustAlias all the sets it merged.
 AliasSet *AliasSetTracker::mergeAliasSetsForPointer(const Value *Ptr,
                                                     LocationSize Size,
-                                                    const AAMDNodes &AAInfo) {
+                                                    const AAMDNodes &AAInfo,
+                                                    bool &MustAliasAll) {
   AliasSet *FoundSet = nullptr;
+  AliasResult AllAR = MustAlias;
   for (iterator I = begin(), E = end(); I != E;) {
     iterator Cur = I++;
-    if (Cur->Forward || !Cur->aliasesPointer(Ptr, Size, AAInfo, AA)) continue;
+    if (Cur->Forward)
+      continue;
+
+    AliasResult AR = Cur->aliasesPointer(Ptr, Size, AAInfo, AA);
+    if (AR == NoAlias)
+      continue;
 
-    if (!FoundSet) {      // If this is the first alias set ptr can go into.
-      FoundSet = &*Cur;   // Remember it.
-    } else {              // Otherwise, we must merge the sets.
-      FoundSet->mergeSetIn(*Cur, *this);     // Merge in contents.
+    AllAR =
+        AliasResult(AllAR & AR); // Possible downgrade to May/Partial, even No
+
+    if (!FoundSet) {
+      // If this is the first alias set ptr can go into, remember it.
+      FoundSet = &*Cur;
+    } else {
+      // Otherwise, we must merge the sets.
+      FoundSet->mergeSetIn(*Cur, *this);
     }
   }
 
+  MustAliasAll = (AllAR == MustAlias);
   return FoundSet;
 }
 
@@ -316,10 +332,13 @@ AliasSet *AliasSetTracker::findAliasSetForUnknownInst(Instruction *Inst) {
     iterator Cur = I++;
     if (Cur->Forward || !Cur->aliasesUnknownInst(Inst, AA))
       continue;
-    if (!FoundSet)            // If this is the first alias set ptr can go into.
-      FoundSet = &*Cur;       // Remember it.
-    else   // Otherwise, we must merge the sets.
-      FoundSet->mergeSetIn(*Cur, *this);     // Merge in contents.
+    if (!FoundSet) {
+      // If this is the first alias set ptr can go into, remember it.
+      FoundSet = &*Cur;
+    } else {
+      // Otherwise, we must merge the sets.
+      FoundSet->mergeSetIn(*Cur, *this);
+    }
   }
   return FoundSet;
 }
@@ -329,7 +348,7 @@ AliasSet &AliasSetTracker::getAliasSetFor(const MemoryLocation &MemLoc) {
   Value * const Pointer = const_cast<Value*>(MemLoc.Ptr);
   const LocationSize Size = MemLoc.Size;
   const AAMDNodes &AAInfo = MemLoc.AATags;
-  
+
   AliasSet::PointerRec &Entry = getEntryFor(Pointer);
 
   if (AliasAnyAS) {
@@ -348,6 +367,7 @@ AliasSet &AliasSetTracker::getAliasSetFor(const MemoryLocation &MemLoc) {
     return *AliasAnyAS;
   }
 
+  bool MustAliasAll = false;
   // Check to see if the pointer is already known.
   if (Entry.hasAliasSet()) {
     // If the size changed, we may need to merge several alias sets.
@@ -356,20 +376,21 @@ AliasSet &AliasSetTracker::getAliasSetFor(const MemoryLocation &MemLoc) {
     // is NoAlias, mergeAliasSetsForPointer(undef, ...) will not find the
     // the right set for undef, even if it exists.
     if (Entry.updateSizeAndAAInfo(Size, AAInfo))
-      mergeAliasSetsForPointer(Pointer, Size, AAInfo);
+      mergeAliasSetsForPointer(Pointer, Size, AAInfo, MustAliasAll);
     // Return the set!
     return *Entry.getAliasSet(*this)->getForwardedTarget(*this);
   }
 
-  if (AliasSet *AS = mergeAliasSetsForPointer(Pointer, Size, AAInfo)) {
+  if (AliasSet *AS =
+          mergeAliasSetsForPointer(Pointer, Size, AAInfo, MustAliasAll)) {
     // Add it to the alias set it aliases.
-    AS->addPointer(*this, Entry, Size, AAInfo);
+    AS->addPointer(*this, Entry, Size, AAInfo, MustAliasAll);
     return *AS;
   }
 
   // Otherwise create a new alias set to hold the loaded pointer.
   AliasSets.push_back(new AliasSet());
-  AliasSets.back().addPointer(*this, Entry, Size, AAInfo);
+  AliasSets.back().addPointer(*this, Entry, Size, AAInfo, true);
   return AliasSets.back();
 }
 
@@ -422,14 +443,12 @@ void AliasSetTracker::addUnknown(Instruction *Inst) {
   if (!Inst->mayReadOrWriteMemory())
     return; // doesn't alias anything
 
-  AliasSet *AS = findAliasSetForUnknownInst(Inst);
-  if (AS) {
+  if (AliasSet *AS = findAliasSetForUnknownInst(Inst)) {
     AS->addUnknownInst(Inst, AA);
     return;
   }
   AliasSets.push_back(new AliasSet());
-  AS = &AliasSets.back();
-  AS->addUnknownInst(Inst, AA);
+  AliasSets.back().addUnknownInst(Inst, AA);
 }
 
 void AliasSetTracker::add(Instruction *I) {
@@ -446,44 +465,44 @@ void AliasSetTracker::add(Instruction *I) {
     return add(MTI);
 
   // Handle all calls with known mod/ref sets genericall
-  CallSite CS(I);
-  if (CS && CS.onlyAccessesArgMemory()) {
-    auto getAccessFromModRef = [](ModRefInfo MRI) {
-      if (isRefSet(MRI) && isModSet(MRI))
-        return AliasSet::ModRefAccess;
-      else if (isModSet(MRI))
-        return AliasSet::ModAccess;
-      else if (isRefSet(MRI))
-        return AliasSet::RefAccess;
-      else
-        return AliasSet::NoAccess;
-     
-    };
-    
-    ModRefInfo CallMask = createModRefInfo(AA.getModRefBehavior(CS));
-
-    // Some intrinsics are marked as modifying memory for control flow
-    // modelling purposes, but don't actually modify any specific memory
-    // location. 
-    using namespace PatternMatch;
-    if (I->use_empty() && match(I, m_Intrinsic<Intrinsic::invariant_start>()))
-      CallMask = clearMod(CallMask);
-
-    for (auto AI = CS.arg_begin(), AE = CS.arg_end(); AI != AE; ++AI) {
-      const Value *Arg = *AI;
-      if (!Arg->getType()->isPointerTy())
-        continue;
-      unsigned ArgIdx = std::distance(CS.arg_begin(), AI);
-      MemoryLocation ArgLoc = MemoryLocation::getForArgument(CS, ArgIdx,
-                                                             nullptr);
-      ModRefInfo ArgMask = AA.getArgModRefInfo(CS, ArgIdx);
-      ArgMask = intersectModRef(CallMask, ArgMask);
-      if (!isNoModRef(ArgMask))
-        addPointer(ArgLoc, getAccessFromModRef(ArgMask));
+  if (auto *Call = dyn_cast<CallBase>(I))
+    if (Call->onlyAccessesArgMemory()) {
+      auto getAccessFromModRef = [](ModRefInfo MRI) {
+        if (isRefSet(MRI) && isModSet(MRI))
+          return AliasSet::ModRefAccess;
+        else if (isModSet(MRI))
+          return AliasSet::ModAccess;
+        else if (isRefSet(MRI))
+          return AliasSet::RefAccess;
+        else
+          return AliasSet::NoAccess;
+      };
+
+      ModRefInfo CallMask = createModRefInfo(AA.getModRefBehavior(Call));
+
+      // Some intrinsics are marked as modifying memory for control flow
+      // modelling purposes, but don't actually modify any specific memory
+      // location.
+      using namespace PatternMatch;
+      if (Call->use_empty() &&
+          match(Call, m_Intrinsic<Intrinsic::invariant_start>()))
+        CallMask = clearMod(CallMask);
+
+      for (auto IdxArgPair : enumerate(Call->args())) {
+        int ArgIdx = IdxArgPair.index();
+        const Value *Arg = IdxArgPair.value();
+        if (!Arg->getType()->isPointerTy())
+          continue;
+        MemoryLocation ArgLoc =
+            MemoryLocation::getForArgument(Call, ArgIdx, nullptr);
+        ModRefInfo ArgMask = AA.getArgModRefInfo(Call, ArgIdx);
+        ArgMask = intersectModRef(CallMask, ArgMask);
+        if (!isNoModRef(ArgMask))
+          addPointer(ArgLoc, getAccessFromModRef(ArgMask));
+      }
+      return;
     }
-    return;
-  }
-  
+
   return addUnknown(I);
 }
 
@@ -516,6 +535,15 @@ void AliasSetTracker::add(const AliasSetTracker &AST) {
   }
 }
 
+void AliasSetTracker::addAllInstructionsInLoopUsingMSSA() {
+  assert(MSSA && L && "MSSA and L must be available");
+  for (const BasicBlock *BB : L->blocks())
+    if (auto *Accesses = MSSA->getBlockAccesses(BB))
+      for (auto &Access : *Accesses)
+        if (auto *MUD = dyn_cast<MemoryUseOrDef>(&Access))
+          add(MUD->getMemoryInst());
+}
+
 // deleteValue method - This method is used to remove a pointer value from the
 // AliasSetTracker entirely.  It should be used when an instruction is deleted
 // from the program to update the AST.  If you don't use this, you would have
@@ -563,9 +591,8 @@ void AliasSetTracker::copyValue(Value *From, Value *To) {
   I = PointerMap.find_as(From);
   // Add it to the alias set it aliases...
   AliasSet *AS = I->second->getAliasSet(*this);
-  AS->addPointer(*this, Entry, I->second->getSize(),
-                 I->second->getAAInfo(),
-                 true);
+  AS->addPointer(*this, Entry, I->second->getSize(), I->second->getAAInfo(),
+                 true, true);
 }
 
 AliasSet &AliasSetTracker::mergeAllAliasSets() {
diff --git a/lib/Analysis/Analysis.cpp b/lib/Analysis/Analysis.cpp
index bb8742123a0f08beafedbf39f46899304d963c0d..d46a8d8e306c7e6b3d68ca8a857e8957024f16df 100644
--- a/lib/Analysis/Analysis.cpp
+++ b/lib/Analysis/Analysis.cpp
@@ -1,9 +1,8 @@
 //===-- Analysis.cpp ------------------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Analysis/AssumptionCache.cpp b/lib/Analysis/AssumptionCache.cpp
index 8bfd24ccf77b99addca30fd135ca1b914b9e95bb..cf2f845dee0a2d932a74e28e37aea05d35e38c25 100644
--- a/lib/Analysis/AssumptionCache.cpp
+++ b/lib/Analysis/AssumptionCache.cpp
@@ -1,9 +1,8 @@
 //===- AssumptionCache.cpp - Cache finding @llvm.assume calls -------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -54,11 +53,11 @@ AssumptionCache::getOrInsertAffectedValues(Value *V) {
   return AVIP.first->second;
 }
 
-void AssumptionCache::updateAffectedValues(CallInst *CI) {
+static void findAffectedValues(CallInst *CI,
+                               SmallVectorImpl<Value *> &Affected) {
   // Note: This code must be kept in-sync with the code in
   // computeKnownBitsFromAssume in ValueTracking.
 
-  SmallVector<Value *, 16> Affected;
   auto AddAffected = [&Affected](Value *V) {
     if (isa<Argument>(V)) {
       Affected.push_back(V);
@@ -109,6 +108,11 @@ void AssumptionCache::updateAffectedValues(CallInst *CI) {
       AddAffectedFromEq(B);
     }
   }
+}
+
+void AssumptionCache::updateAffectedValues(CallInst *CI) {
+  SmallVector<Value *, 16> Affected;
+  findAffectedValues(CI, Affected);
 
   for (auto &AV : Affected) {
     auto &AVV = getOrInsertAffectedValues(AV);
@@ -117,6 +121,18 @@ void AssumptionCache::updateAffectedValues(CallInst *CI) {
   }
 }
 
+void AssumptionCache::unregisterAssumption(CallInst *CI) {
+  SmallVector<Value *, 16> Affected;
+  findAffectedValues(CI, Affected);
+
+  for (auto &AV : Affected) {
+    auto AVI = AffectedValues.find_as(AV);
+    if (AVI != AffectedValues.end())
+      AffectedValues.erase(AVI);
+  }
+  remove_if(AssumeHandles, [CI](WeakTrackingVH &VH) { return CI == VH; });
+}
+
 void AssumptionCache::AffectedValueCallbackVH::deleted() {
   auto AVI = AC->AffectedValues.find(getValPtr());
   if (AVI != AC->AffectedValues.end())
@@ -241,6 +257,13 @@ AssumptionCache &AssumptionCacheTracker::getAssumptionCache(Function &F) {
   return *IP.first->second;
 }
 
+AssumptionCache *AssumptionCacheTracker::lookupAssumptionCache(Function &F) {
+  auto I = AssumptionCaches.find_as(&F);
+  if (I != AssumptionCaches.end())
+    return I->second.get();
+  return nullptr;
+}
+
 void AssumptionCacheTracker::verifyAnalysis() const {
   // FIXME: In the long term the verifier should not be controllable with a
   // flag. We should either fix all passes to correctly update the assumption
diff --git a/lib/Analysis/BasicAliasAnalysis.cpp b/lib/Analysis/BasicAliasAnalysis.cpp
index b7aa395ab849cc0ab3de2b136ced39cd175c5dc8..382a70b80666a5bde5d4999f391c16566e105fed 100644
--- a/lib/Analysis/BasicAliasAnalysis.cpp
+++ b/lib/Analysis/BasicAliasAnalysis.cpp
@@ -1,9 +1,8 @@
 //===- BasicAliasAnalysis.cpp - Stateless Alias Analysis Impl -------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -31,7 +30,6 @@
 #include "llvm/Analysis/PhiValues.h"
 #include "llvm/IR/Argument.h"
 #include "llvm/IR/Attributes.h"
-#include "llvm/IR/CallSite.h"
 #include "llvm/IR/Constant.h"
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/DataLayout.h"
@@ -68,6 +66,16 @@ using namespace llvm;
 /// Enable analysis of recursive PHI nodes.
 static cl::opt<bool> EnableRecPhiAnalysis("basicaa-recphi", cl::Hidden,
                                           cl::init(false));
+
+/// By default, even on 32-bit architectures we use 64-bit integers for
+/// calculations. This will allow us to more-aggressively decompose indexing
+/// expressions calculated using i64 values (e.g., long long in C) which is
+/// common enough to worry about.
+static cl::opt<bool> ForceAtLeast64Bits("basicaa-force-at-least-64b",
+                                        cl::Hidden, cl::init(true));
+static cl::opt<bool> DoubleCalcBits("basicaa-double-calc-bits",
+                                    cl::Hidden, cl::init(false));
+
 /// SearchLimitReached / SearchTimes shows how often the limit of
 /// to decompose GEPs is reached. It will affect the precision
 /// of basic alias analysis.
@@ -108,25 +116,44 @@ bool BasicAAResult::invalidate(Function &Fn, const PreservedAnalyses &PA,
 
 /// Returns true if the pointer is to a function-local object that never
 /// escapes from the function.
-static bool isNonEscapingLocalObject(const Value *V) {
+static bool isNonEscapingLocalObject(
+    const Value *V,
+    SmallDenseMap<const Value *, bool, 8> *IsCapturedCache = nullptr) {
+  SmallDenseMap<const Value *, bool, 8>::iterator CacheIt;
+  if (IsCapturedCache) {
+    bool Inserted;
+    std::tie(CacheIt, Inserted) = IsCapturedCache->insert({V, false});
+    if (!Inserted)
+      // Found cached result, return it!
+      return CacheIt->second;
+  }
+
   // If this is a local allocation, check to see if it escapes.
-  if (isa<AllocaInst>(V) || isNoAliasCall(V))
+  if (isa<AllocaInst>(V) || isNoAliasCall(V)) {
     // Set StoreCaptures to True so that we can assume in our callers that the
     // pointer is not the result of a load instruction. Currently
     // PointerMayBeCaptured doesn't have any special analysis for the
     // StoreCaptures=false case; if it did, our callers could be refined to be
     // more precise.
-    return !PointerMayBeCaptured(V, false, /*StoreCaptures=*/true);
+    auto Ret = !PointerMayBeCaptured(V, false, /*StoreCaptures=*/true);
+    if (IsCapturedCache)
+      CacheIt->second = Ret;
+    return Ret;
+  }
 
   // If this is an argument that corresponds to a byval or noalias argument,
   // then it has not escaped before entering the function.  Check if it escapes
   // inside the function.
   if (const Argument *A = dyn_cast<Argument>(V))
-    if (A->hasByValAttr() || A->hasNoAliasAttr())
+    if (A->hasByValAttr() || A->hasNoAliasAttr()) {
       // Note even if the argument is marked nocapture, we still need to check
       // for copies made inside the function. The nocapture attribute only
       // specifies that there are no copies made that outlive the function.
-      return !PointerMayBeCaptured(V, false, /*StoreCaptures=*/true);
+      auto Ret = !PointerMayBeCaptured(V, false, /*StoreCaptures=*/true);
+      if (IsCapturedCache)
+        CacheIt->second = Ret;
+      return Ret;
+    }
 
   return false;
 }
@@ -134,7 +161,7 @@ static bool isNonEscapingLocalObject(const Value *V) {
 /// Returns true if the pointer is one which would have been considered an
 /// escape by isNonEscapingLocalObject.
 static bool isEscapeSource(const Value *V) {
-  if (ImmutableCallSite(V))
+  if (isa<CallBase>(V))
     return true;
 
   if (isa<Argument>(V))
@@ -381,13 +408,22 @@ static bool isObjectSize(const Value *V, uint64_t Size, const DataLayout &DL,
 }
 
 /// To ensure a pointer offset fits in an integer of size PointerSize
-/// (in bits) when that size is smaller than 64. This is an issue in
-/// particular for 32b programs with negative indices that rely on two's
-/// complement wrap-arounds for precise alias information.
-static int64_t adjustToPointerSize(int64_t Offset, unsigned PointerSize) {
-  assert(PointerSize <= 64 && "Invalid PointerSize!");
-  unsigned ShiftBits = 64 - PointerSize;
-  return (int64_t)((uint64_t)Offset << ShiftBits) >> ShiftBits;
+/// (in bits) when that size is smaller than the maximum pointer size. This is
+/// an issue, for example, in particular for 32b pointers with negative indices
+/// that rely on two's complement wrap-arounds for precise alias information
+/// where the maximum pointer size is 64b.
+static APInt adjustToPointerSize(APInt Offset, unsigned PointerSize) {
+  assert(PointerSize <= Offset.getBitWidth() && "Invalid PointerSize!");
+  unsigned ShiftBits = Offset.getBitWidth() - PointerSize;
+  return (Offset << ShiftBits).ashr(ShiftBits);
+}
+
+static unsigned getMaxPointerSize(const DataLayout &DL) {
+  unsigned MaxPointerSize = DL.getMaxPointerSizeInBits();
+  if (MaxPointerSize < 64 && ForceAtLeast64Bits) MaxPointerSize = 64;
+  if (DoubleCalcBits) MaxPointerSize *= 2;
+
+  return MaxPointerSize;
 }
 
 /// If V is a symbolic pointer expression, decompose it into a base pointer
@@ -410,8 +446,7 @@ bool BasicAAResult::DecomposeGEPExpression(const Value *V,
   unsigned MaxLookup = MaxLookupSearchDepth;
   SearchTimes++;
 
-  Decomposed.StructOffset = 0;
-  Decomposed.OtherOffset = 0;
+  unsigned MaxPointerSize = getMaxPointerSize(DL);
   Decomposed.VarIndices.clear();
   do {
     // See if this is a bitcast or GEP.
@@ -436,7 +471,7 @@ bool BasicAAResult::DecomposeGEPExpression(const Value *V,
 
     const GEPOperator *GEPOp = dyn_cast<GEPOperator>(Op);
     if (!GEPOp) {
-      if (auto CS = ImmutableCallSite(V)) {
+      if (const auto *Call = dyn_cast<CallBase>(V)) {
         // CaptureTracking can know about special capturing properties of some
         // intrinsics like launder.invariant.group, that can't be expressed with
         // the attributes, but have properties like returning aliasing pointer.
@@ -446,7 +481,7 @@ bool BasicAAResult::DecomposeGEPExpression(const Value *V,
         // because it should be in sync with CaptureTracking. Not using it may
         // cause weird miscompilations where 2 aliasing pointers are assumed to
         // noalias.
-        if (auto *RP = getArgumentAliasingToReturnedPointer(CS)) {
+        if (auto *RP = getArgumentAliasingToReturnedPointer(Call)) {
           V = RP;
           continue;
         }
@@ -501,13 +536,15 @@ bool BasicAAResult::DecomposeGEPExpression(const Value *V,
         if (CIdx->isZero())
           continue;
         Decomposed.OtherOffset +=
-          DL.getTypeAllocSize(GTI.getIndexedType()) * CIdx->getSExtValue();
+          (DL.getTypeAllocSize(GTI.getIndexedType()) *
+            CIdx->getValue().sextOrSelf(MaxPointerSize))
+              .sextOrTrunc(MaxPointerSize);
         continue;
       }
 
       GepHasConstantOffset = false;
 
-      uint64_t Scale = DL.getTypeAllocSize(GTI.getIndexedType());
+      APInt Scale(MaxPointerSize, DL.getTypeAllocSize(GTI.getIndexedType()));
       unsigned ZExtBits = 0, SExtBits = 0;
 
       // If the integer type is smaller than the pointer size, it is implicitly
@@ -519,20 +556,34 @@ bool BasicAAResult::DecomposeGEPExpression(const Value *V,
       // Use GetLinearExpression to decompose the index into a C1*V+C2 form.
       APInt IndexScale(Width, 0), IndexOffset(Width, 0);
       bool NSW = true, NUW = true;
+      const Value *OrigIndex = Index;
       Index = GetLinearExpression(Index, IndexScale, IndexOffset, ZExtBits,
                                   SExtBits, DL, 0, AC, DT, NSW, NUW);
 
-      // All GEP math happens in the width of the pointer type,
-      // so we can truncate the value to 64-bits as we don't handle
-      // currently pointers larger than 64 bits and we would crash
-      // later. TODO: Make `Scale` an APInt to avoid this problem.
-      if (IndexScale.getBitWidth() > 64)
-        IndexScale = IndexScale.sextOrTrunc(64);
-
       // The GEP index scale ("Scale") scales C1*V+C2, yielding (C1*V+C2)*Scale.
       // This gives us an aggregate computation of (C1*Scale)*V + C2*Scale.
-      Decomposed.OtherOffset += IndexOffset.getSExtValue() * Scale;
-      Scale *= IndexScale.getSExtValue();
+
+      // It can be the case that, even through C1*V+C2 does not overflow for
+      // relevant values of V, (C2*Scale) can overflow. In that case, we cannot
+      // decompose the expression in this way.
+      //
+      // FIXME: C1*Scale and the other operations in the decomposed
+      // (C1*Scale)*V+C2*Scale can also overflow. We should check for this
+      // possibility.
+      APInt WideScaledOffset = IndexOffset.sextOrTrunc(MaxPointerSize*2) *
+                                 Scale.sext(MaxPointerSize*2);
+      if (WideScaledOffset.getMinSignedBits() > MaxPointerSize) {
+        Index = OrigIndex;
+        IndexScale = 1;
+        IndexOffset = 0;
+
+        ZExtBits = SExtBits = 0;
+        if (PointerSize > Width)
+          SExtBits += PointerSize - Width;
+      } else {
+        Decomposed.OtherOffset += IndexOffset.sextOrTrunc(MaxPointerSize) * Scale;
+        Scale *= IndexScale.sextOrTrunc(MaxPointerSize);
+      }
 
       // If we already had an occurrence of this index variable, merge this
       // scale into it.  For example, we want to handle:
@@ -552,9 +603,8 @@ bool BasicAAResult::DecomposeGEPExpression(const Value *V,
       // pointer size.
       Scale = adjustToPointerSize(Scale, PointerSize);
 
-      if (Scale) {
-        VariableGEPIndex Entry = {Index, ZExtBits, SExtBits,
-                                  static_cast<int64_t>(Scale)};
+      if (!!Scale) {
+        VariableGEPIndex Entry = {Index, ZExtBits, SExtBits, Scale};
         Decomposed.VarIndices.push_back(Entry);
       }
     }
@@ -640,8 +690,8 @@ bool BasicAAResult::pointsToConstantMemory(const MemoryLocation &Loc,
 }
 
 /// Returns the behavior when calling the given call site.
-FunctionModRefBehavior BasicAAResult::getModRefBehavior(ImmutableCallSite CS) {
-  if (CS.doesNotAccessMemory())
+FunctionModRefBehavior BasicAAResult::getModRefBehavior(const CallBase *Call) {
+  if (Call->doesNotAccessMemory())
     // Can't do better than this.
     return FMRB_DoesNotAccessMemory;
 
@@ -649,23 +699,23 @@ FunctionModRefBehavior BasicAAResult::getModRefBehavior(ImmutableCallSite CS) {
 
   // If the callsite knows it only reads memory, don't return worse
   // than that.
-  if (CS.onlyReadsMemory())
+  if (Call->onlyReadsMemory())
     Min = FMRB_OnlyReadsMemory;
-  else if (CS.doesNotReadMemory())
+  else if (Call->doesNotReadMemory())
     Min = FMRB_DoesNotReadMemory;
 
-  if (CS.onlyAccessesArgMemory())
+  if (Call->onlyAccessesArgMemory())
     Min = FunctionModRefBehavior(Min & FMRB_OnlyAccessesArgumentPointees);
-  else if (CS.onlyAccessesInaccessibleMemory())
+  else if (Call->onlyAccessesInaccessibleMemory())
     Min = FunctionModRefBehavior(Min & FMRB_OnlyAccessesInaccessibleMem);
-  else if (CS.onlyAccessesInaccessibleMemOrArgMem())
+  else if (Call->onlyAccessesInaccessibleMemOrArgMem())
     Min = FunctionModRefBehavior(Min & FMRB_OnlyAccessesInaccessibleOrArgMem);
 
-  // If CS has operand bundles then aliasing attributes from the function it
-  // calls do not directly apply to the CallSite.  This can be made more
-  // precise in the future.
-  if (!CS.hasOperandBundles())
-    if (const Function *F = CS.getCalledFunction())
+  // If the call has operand bundles then aliasing attributes from the function
+  // it calls do not directly apply to the call.  This can be made more precise
+  // in the future.
+  if (!Call->hasOperandBundles())
+    if (const Function *F = Call->getCalledFunction())
       Min =
           FunctionModRefBehavior(Min & getBestAAResults().getModRefBehavior(F));
 
@@ -698,9 +748,9 @@ FunctionModRefBehavior BasicAAResult::getModRefBehavior(const Function *F) {
 }
 
 /// Returns true if this is a writeonly (i.e Mod only) parameter.
-static bool isWriteOnlyParam(ImmutableCallSite CS, unsigned ArgIdx,
+static bool isWriteOnlyParam(const CallBase *Call, unsigned ArgIdx,
                              const TargetLibraryInfo &TLI) {
-  if (CS.paramHasAttr(ArgIdx, Attribute::WriteOnly))
+  if (Call->paramHasAttr(ArgIdx, Attribute::WriteOnly))
     return true;
 
   // We can bound the aliasing properties of memset_pattern16 just as we can
@@ -710,7 +760,8 @@ static bool isWriteOnlyParam(ImmutableCallSite CS, unsigned ArgIdx,
   // FIXME Consider handling this in InferFunctionAttr.cpp together with other
   // attributes.
   LibFunc F;
-  if (CS.getCalledFunction() && TLI.getLibFunc(*CS.getCalledFunction(), F) &&
+  if (Call->getCalledFunction() &&
+      TLI.getLibFunc(*Call->getCalledFunction(), F) &&
       F == LibFunc_memset_pattern16 && TLI.has(F))
     if (ArgIdx == 0)
       return true;
@@ -722,23 +773,23 @@ static bool isWriteOnlyParam(ImmutableCallSite CS, unsigned ArgIdx,
   return false;
 }
 
-ModRefInfo BasicAAResult::getArgModRefInfo(ImmutableCallSite CS,
+ModRefInfo BasicAAResult::getArgModRefInfo(const CallBase *Call,
                                            unsigned ArgIdx) {
   // Checking for known builtin intrinsics and target library functions.
-  if (isWriteOnlyParam(CS, ArgIdx, TLI))
+  if (isWriteOnlyParam(Call, ArgIdx, TLI))
     return ModRefInfo::Mod;
 
-  if (CS.paramHasAttr(ArgIdx, Attribute::ReadOnly))
+  if (Call->paramHasAttr(ArgIdx, Attribute::ReadOnly))
     return ModRefInfo::Ref;
 
-  if (CS.paramHasAttr(ArgIdx, Attribute::ReadNone))
+  if (Call->paramHasAttr(ArgIdx, Attribute::ReadNone))
     return ModRefInfo::NoModRef;
 
-  return AAResultBase::getArgModRefInfo(CS, ArgIdx);
+  return AAResultBase::getArgModRefInfo(Call, ArgIdx);
 }
 
-static bool isIntrinsicCall(ImmutableCallSite CS, Intrinsic::ID IID) {
-  const IntrinsicInst *II = dyn_cast<IntrinsicInst>(CS.getInstruction());
+static bool isIntrinsicCall(const CallBase *Call, Intrinsic::ID IID) {
+  const IntrinsicInst *II = dyn_cast<IntrinsicInst>(Call);
   return II && II->getIntrinsicID() == IID;
 }
 
@@ -784,6 +835,7 @@ AliasResult BasicAAResult::alias(const MemoryLocation &LocA,
   // SmallDenseMap if it ever grows larger.
   // FIXME: This should really be shrink_to_inline_capacity_and_clear().
   AliasCache.shrink_and_clear();
+  IsCapturedCache.shrink_and_clear();
   VisitedPhiBBs.clear();
   return Alias;
 }
@@ -794,9 +846,9 @@ AliasResult BasicAAResult::alias(const MemoryLocation &LocA,
 /// Since we only look at local properties of this function, we really can't
 /// say much about this query.  We do, however, use simple "address taken"
 /// analysis on local objects.
-ModRefInfo BasicAAResult::getModRefInfo(ImmutableCallSite CS,
+ModRefInfo BasicAAResult::getModRefInfo(const CallBase *Call,
                                         const MemoryLocation &Loc) {
-  assert(notDifferentParent(CS.getInstruction(), Loc.Ptr) &&
+  assert(notDifferentParent(Call, Loc.Ptr) &&
          "AliasAnalysis query involving multiple functions!");
 
   const Value *Object = GetUnderlyingObject(Loc.Ptr, DL);
@@ -807,15 +859,21 @@ ModRefInfo BasicAAResult::getModRefInfo(ImmutableCallSite CS,
   // contents of the alloca into argument registers or stack slots, so there is
   // no lifetime issue.
   if (isa<AllocaInst>(Object))
-    if (const CallInst *CI = dyn_cast<CallInst>(CS.getInstruction()))
+    if (const CallInst *CI = dyn_cast<CallInst>(Call))
       if (CI->isTailCall() &&
           !CI->getAttributes().hasAttrSomewhere(Attribute::ByVal))
         return ModRefInfo::NoModRef;
 
+  // Stack restore is able to modify unescaped dynamic allocas. Assume it may
+  // modify them even though the alloca is not escaped.
+  if (auto *AI = dyn_cast<AllocaInst>(Object))
+    if (!AI->isStaticAlloca() && isIntrinsicCall(Call, Intrinsic::stackrestore))
+      return ModRefInfo::Mod;
+
   // If the pointer is to a locally allocated object that does not escape,
   // then the call can not mod/ref the pointer unless the call takes the pointer
   // as an argument, and itself doesn't capture it.
-  if (!isa<Constant>(Object) && CS.getInstruction() != Object &&
+  if (!isa<Constant>(Object) && Call != Object &&
       isNonEscapingLocalObject(Object)) {
 
     // Optimistically assume that call doesn't touch Object and check this
@@ -824,19 +882,20 @@ ModRefInfo BasicAAResult::getModRefInfo(ImmutableCallSite CS,
     bool IsMustAlias = true;
 
     unsigned OperandNo = 0;
-    for (auto CI = CS.data_operands_begin(), CE = CS.data_operands_end();
+    for (auto CI = Call->data_operands_begin(), CE = Call->data_operands_end();
          CI != CE; ++CI, ++OperandNo) {
       // Only look at the no-capture or byval pointer arguments.  If this
       // pointer were passed to arguments that were neither of these, then it
       // couldn't be no-capture.
       if (!(*CI)->getType()->isPointerTy() ||
-          (!CS.doesNotCapture(OperandNo) &&
-           OperandNo < CS.getNumArgOperands() && !CS.isByValArgument(OperandNo)))
+          (!Call->doesNotCapture(OperandNo) &&
+           OperandNo < Call->getNumArgOperands() &&
+           !Call->isByValArgument(OperandNo)))
         continue;
 
       // Call doesn't access memory through this operand, so we don't care
       // if it aliases with Object.
-      if (CS.doesNotAccessMemory(OperandNo))
+      if (Call->doesNotAccessMemory(OperandNo))
         continue;
 
       // If this is a no-capture pointer argument, see if we can tell that it
@@ -845,17 +904,17 @@ ModRefInfo BasicAAResult::getModRefInfo(ImmutableCallSite CS,
           getBestAAResults().alias(MemoryLocation(*CI), MemoryLocation(Object));
       if (AR != MustAlias)
         IsMustAlias = false;
-      // Operand doesnt alias 'Object', continue looking for other aliases
+      // Operand doesn't alias 'Object', continue looking for other aliases
       if (AR == NoAlias)
         continue;
       // Operand aliases 'Object', but call doesn't modify it. Strengthen
       // initial assumption and keep looking in case if there are more aliases.
-      if (CS.onlyReadsMemory(OperandNo)) {
+      if (Call->onlyReadsMemory(OperandNo)) {
         Result = setRef(Result);
         continue;
       }
       // Operand aliases 'Object' but call only writes into it.
-      if (CS.doesNotReadMemory(OperandNo)) {
+      if (Call->doesNotReadMemory(OperandNo)) {
         Result = setMod(Result);
         continue;
       }
@@ -879,17 +938,16 @@ ModRefInfo BasicAAResult::getModRefInfo(ImmutableCallSite CS,
     }
   }
 
-  // If the CallSite is to malloc or calloc, we can assume that it doesn't
+  // If the call is to malloc or calloc, we can assume that it doesn't
   // modify any IR visible value.  This is only valid because we assume these
   // routines do not read values visible in the IR.  TODO: Consider special
   // casing realloc and strdup routines which access only their arguments as
   // well.  Or alternatively, replace all of this with inaccessiblememonly once
   // that's implemented fully.
-  auto *Inst = CS.getInstruction();
-  if (isMallocOrCallocLikeFn(Inst, &TLI)) {
+  if (isMallocOrCallocLikeFn(Call, &TLI)) {
     // Be conservative if the accessed pointer may alias the allocation -
     // fallback to the generic handling below.
-    if (getBestAAResults().alias(MemoryLocation(Inst), Loc) == NoAlias)
+    if (getBestAAResults().alias(MemoryLocation(Call), Loc) == NoAlias)
       return ModRefInfo::NoModRef;
   }
 
@@ -897,7 +955,7 @@ ModRefInfo BasicAAResult::getModRefInfo(ImmutableCallSite CS,
   // operands, i.e., source and destination of any given memcpy must no-alias.
   // If Loc must-aliases either one of these two locations, then it necessarily
   // no-aliases the other.
-  if (auto *Inst = dyn_cast<AnyMemCpyInst>(CS.getInstruction())) {
+  if (auto *Inst = dyn_cast<AnyMemCpyInst>(Call)) {
     AliasResult SrcAA, DestAA;
 
     if ((SrcAA = getBestAAResults().alias(MemoryLocation::getForSource(Inst),
@@ -921,7 +979,7 @@ ModRefInfo BasicAAResult::getModRefInfo(ImmutableCallSite CS,
   // While the assume intrinsic is marked as arbitrarily writing so that
   // proper control dependencies will be maintained, it never aliases any
   // particular memory location.
-  if (isIntrinsicCall(CS, Intrinsic::assume))
+  if (isIntrinsicCall(Call, Intrinsic::assume))
     return ModRefInfo::NoModRef;
 
   // Like assumes, guard intrinsics are also marked as arbitrarily writing so
@@ -931,7 +989,7 @@ ModRefInfo BasicAAResult::getModRefInfo(ImmutableCallSite CS,
   // *Unlike* assumes, guard intrinsics are modeled as reading memory since the
   // heap state at the point the guard is issued needs to be consistent in case
   // the guard invokes the "deopt" continuation.
-  if (isIntrinsicCall(CS, Intrinsic::experimental_guard))
+  if (isIntrinsicCall(Call, Intrinsic::experimental_guard))
     return ModRefInfo::Ref;
 
   // Like assumes, invariant.start intrinsics were also marked as arbitrarily
@@ -957,20 +1015,20 @@ ModRefInfo BasicAAResult::getModRefInfo(ImmutableCallSite CS,
   // The transformation will cause the second store to be ignored (based on
   // rules of invariant.start)  and print 40, while the first program always
   // prints 50.
-  if (isIntrinsicCall(CS, Intrinsic::invariant_start))
+  if (isIntrinsicCall(Call, Intrinsic::invariant_start))
     return ModRefInfo::Ref;
 
   // The AAResultBase base class has some smarts, lets use them.
-  return AAResultBase::getModRefInfo(CS, Loc);
+  return AAResultBase::getModRefInfo(Call, Loc);
 }
 
-ModRefInfo BasicAAResult::getModRefInfo(ImmutableCallSite CS1,
-                                        ImmutableCallSite CS2) {
+ModRefInfo BasicAAResult::getModRefInfo(const CallBase *Call1,
+                                        const CallBase *Call2) {
   // While the assume intrinsic is marked as arbitrarily writing so that
   // proper control dependencies will be maintained, it never aliases any
   // particular memory location.
-  if (isIntrinsicCall(CS1, Intrinsic::assume) ||
-      isIntrinsicCall(CS2, Intrinsic::assume))
+  if (isIntrinsicCall(Call1, Intrinsic::assume) ||
+      isIntrinsicCall(Call2, Intrinsic::assume))
     return ModRefInfo::NoModRef;
 
   // Like assumes, guard intrinsics are also marked as arbitrarily writing so
@@ -981,21 +1039,21 @@ ModRefInfo BasicAAResult::getModRefInfo(ImmutableCallSite CS1,
   // heap state at the point the guard is issued needs to be consistent in case
   // the guard invokes the "deopt" continuation.
 
-  // NB! This function is *not* commutative, so we specical case two
+  // NB! This function is *not* commutative, so we special case two
   // possibilities for guard intrinsics.
 
-  if (isIntrinsicCall(CS1, Intrinsic::experimental_guard))
-    return isModSet(createModRefInfo(getModRefBehavior(CS2)))
+  if (isIntrinsicCall(Call1, Intrinsic::experimental_guard))
+    return isModSet(createModRefInfo(getModRefBehavior(Call2)))
                ? ModRefInfo::Ref
                : ModRefInfo::NoModRef;
 
-  if (isIntrinsicCall(CS2, Intrinsic::experimental_guard))
-    return isModSet(createModRefInfo(getModRefBehavior(CS1)))
+  if (isIntrinsicCall(Call2, Intrinsic::experimental_guard))
+    return isModSet(createModRefInfo(getModRefBehavior(Call1)))
                ? ModRefInfo::Mod
                : ModRefInfo::NoModRef;
 
   // The AAResultBase base class has some smarts, lets use them.
-  return AAResultBase::getModRefInfo(CS1, CS2);
+  return AAResultBase::getModRefInfo(Call1, Call2);
 }
 
 /// Provide ad-hoc rules to disambiguate accesses through two GEP operators,
@@ -1033,8 +1091,12 @@ static AliasResult aliasSameBasePointerGEPs(const GEPOperator *GEP1,
 
   // If the last (struct) indices are constants and are equal, the other indices
   // might be also be dynamically equal, so the GEPs can alias.
-  if (C1 && C2 && C1->getSExtValue() == C2->getSExtValue())
-    return MayAlias;
+  if (C1 && C2) {
+    unsigned BitWidth = std::max(C1->getBitWidth(), C2->getBitWidth());
+    if (C1->getValue().sextOrSelf(BitWidth) ==
+        C2->getValue().sextOrSelf(BitWidth))
+      return MayAlias;
+  }
 
   // Find the last-indexed type of the GEP, i.e., the type you'd get if
   // you stripped the last index.
@@ -1117,6 +1179,10 @@ static AliasResult aliasSameBasePointerGEPs(const GEPOperator *GEP1,
     return MayAlias;
   }
 
+  if (C1->getValue().getActiveBits() > 64 ||
+      C2->getValue().getActiveBits() > 64)
+    return MayAlias;
+
   // We know that:
   // - both GEPs begin indexing from the exact same pointer;
   // - the last indices in both GEPs are constants, indexing into a struct;
@@ -1197,8 +1263,8 @@ bool BasicAAResult::isGEPBaseAtNegativeOffset(const GEPOperator *GEPOp,
       !DecompObject.VarIndices.empty())
     return false;
 
-  int64_t ObjectBaseOffset = DecompObject.StructOffset +
-                             DecompObject.OtherOffset;
+  APInt ObjectBaseOffset = DecompObject.StructOffset +
+                           DecompObject.OtherOffset;
 
   // If the GEP has no variable indices, we know the precise offset
   // from the base, then use it. If the GEP has variable indices,
@@ -1206,10 +1272,11 @@ bool BasicAAResult::isGEPBaseAtNegativeOffset(const GEPOperator *GEPOp,
   // false in that case.
   if (!DecompGEP.VarIndices.empty())
     return false;
-  int64_t GEPBaseOffset = DecompGEP.StructOffset;
+
+  APInt GEPBaseOffset = DecompGEP.StructOffset;
   GEPBaseOffset += DecompGEP.OtherOffset;
 
-  return (GEPBaseOffset >= ObjectBaseOffset + (int64_t)ObjectAccessSize);
+  return GEPBaseOffset.sge(ObjectBaseOffset + (int64_t)ObjectAccessSize);
 }
 
 /// Provides a bunch of ad-hoc rules to disambiguate a GEP instruction against
@@ -1224,13 +1291,17 @@ BasicAAResult::aliasGEP(const GEPOperator *GEP1, LocationSize V1Size,
                         LocationSize V2Size, const AAMDNodes &V2AAInfo,
                         const Value *UnderlyingV1, const Value *UnderlyingV2) {
   DecomposedGEP DecompGEP1, DecompGEP2;
+  unsigned MaxPointerSize = getMaxPointerSize(DL);
+  DecompGEP1.StructOffset = DecompGEP1.OtherOffset = APInt(MaxPointerSize, 0);
+  DecompGEP2.StructOffset = DecompGEP2.OtherOffset = APInt(MaxPointerSize, 0);
+
   bool GEP1MaxLookupReached =
     DecomposeGEPExpression(GEP1, DecompGEP1, DL, &AC, DT);
   bool GEP2MaxLookupReached =
     DecomposeGEPExpression(V2, DecompGEP2, DL, &AC, DT);
 
-  int64_t GEP1BaseOffset = DecompGEP1.StructOffset + DecompGEP1.OtherOffset;
-  int64_t GEP2BaseOffset = DecompGEP2.StructOffset + DecompGEP2.OtherOffset;
+  APInt GEP1BaseOffset = DecompGEP1.StructOffset + DecompGEP1.OtherOffset;
+  APInt GEP2BaseOffset = DecompGEP2.StructOffset + DecompGEP2.OtherOffset;
 
   assert(DecompGEP1.Base == UnderlyingV1 && DecompGEP2.Base == UnderlyingV2 &&
          "DecomposeGEPExpression returned a result different from "
@@ -1348,9 +1419,9 @@ BasicAAResult::aliasGEP(const GEPOperator *GEP1, LocationSize V1Size,
   // that the objects are partially overlapping.  If the difference is
   // greater, we know they do not overlap.
   if (GEP1BaseOffset != 0 && DecompGEP1.VarIndices.empty()) {
-    if (GEP1BaseOffset >= 0) {
+    if (GEP1BaseOffset.sge(0)) {
       if (V2Size != LocationSize::unknown()) {
-        if ((uint64_t)GEP1BaseOffset < V2Size.getValue())
+        if (GEP1BaseOffset.ult(V2Size.getValue()))
           return PartialAlias;
         return NoAlias;
       }
@@ -1365,7 +1436,7 @@ BasicAAResult::aliasGEP(const GEPOperator *GEP1, LocationSize V1Size,
       // stripped a gep with negative index ('gep <ptr>, -1, ...).
       if (V1Size != LocationSize::unknown() &&
           V2Size != LocationSize::unknown()) {
-        if (-(uint64_t)GEP1BaseOffset < V1Size.getValue())
+        if ((-GEP1BaseOffset).ult(V1Size.getValue()))
           return PartialAlias;
         return NoAlias;
       }
@@ -1373,7 +1444,7 @@ BasicAAResult::aliasGEP(const GEPOperator *GEP1, LocationSize V1Size,
   }
 
   if (!DecompGEP1.VarIndices.empty()) {
-    uint64_t Modulo = 0;
+    APInt Modulo(MaxPointerSize, 0);
     bool AllPositive = true;
     for (unsigned i = 0, e = DecompGEP1.VarIndices.size(); i != e; ++i) {
 
@@ -1381,7 +1452,7 @@ BasicAAResult::aliasGEP(const GEPOperator *GEP1, LocationSize V1Size,
       // Grab the least significant bit set in any of the scales. We
       // don't need std::abs here (even if the scale's negative) as we'll
       // be ^'ing Modulo with itself later.
-      Modulo |= (uint64_t)DecompGEP1.VarIndices[i].Scale;
+      Modulo |= DecompGEP1.VarIndices[i].Scale;
 
       if (AllPositive) {
         // If the Value could change between cycles, then any reasoning about
@@ -1402,9 +1473,9 @@ BasicAAResult::aliasGEP(const GEPOperator *GEP1, LocationSize V1Size,
         // If the variable begins with a zero then we know it's
         // positive, regardless of whether the value is signed or
         // unsigned.
-        int64_t Scale = DecompGEP1.VarIndices[i].Scale;
+        APInt Scale = DecompGEP1.VarIndices[i].Scale;
         AllPositive =
-            (SignKnownZero && Scale >= 0) || (SignKnownOne && Scale < 0);
+            (SignKnownZero && Scale.sge(0)) || (SignKnownOne && Scale.slt(0));
       }
     }
 
@@ -1413,18 +1484,18 @@ BasicAAResult::aliasGEP(const GEPOperator *GEP1, LocationSize V1Size,
     // We can compute the difference between the two addresses
     // mod Modulo. Check whether that difference guarantees that the
     // two locations do not alias.
-    uint64_t ModOffset = (uint64_t)GEP1BaseOffset & (Modulo - 1);
+    APInt ModOffset = GEP1BaseOffset & (Modulo - 1);
     if (V1Size != LocationSize::unknown() &&
-        V2Size != LocationSize::unknown() && ModOffset >= V2Size.getValue() &&
-        V1Size.getValue() <= Modulo - ModOffset)
+        V2Size != LocationSize::unknown() && ModOffset.uge(V2Size.getValue()) &&
+        (Modulo - ModOffset).uge(V1Size.getValue()))
       return NoAlias;
 
     // If we know all the variables are positive, then GEP1 >= GEP1BasePtr.
     // If GEP1BasePtr > V2 (GEP1BaseOffset > 0) then we know the pointers
     // don't alias if V2Size can fit in the gap between V2 and GEP1BasePtr.
-    if (AllPositive && GEP1BaseOffset > 0 &&
+    if (AllPositive && GEP1BaseOffset.sgt(0) &&
         V2Size != LocationSize::unknown() &&
-        V2Size.getValue() <= (uint64_t)GEP1BaseOffset)
+        GEP1BaseOffset.uge(V2Size.getValue()))
       return NoAlias;
 
     if (constantOffsetHeuristic(DecompGEP1.VarIndices, V1Size, V2Size,
@@ -1638,7 +1709,7 @@ AliasResult BasicAAResult::aliasCheck(const Value *V1, LocationSize V1Size,
                                       const Value *O1, const Value *O2) {
   // If either of the memory references is empty, it doesn't matter what the
   // pointer values are.
-  if (V1Size == 0 || V2Size == 0)
+  if (V1Size.isZero() || V2Size.isZero())
     return NoAlias;
 
   // Strip off any casts if they exist.
@@ -1703,9 +1774,9 @@ AliasResult BasicAAResult::aliasCheck(const Value *V1, LocationSize V1Size,
     // temporary store the nocapture argument's value in a temporary memory
     // location if that memory location doesn't escape. Or it may pass a
     // nocapture value to other functions as long as they don't capture it.
-    if (isEscapeSource(O1) && isNonEscapingLocalObject(O2))
+    if (isEscapeSource(O1) && isNonEscapingLocalObject(O2, &IsCapturedCache))
       return NoAlias;
-    if (isEscapeSource(O2) && isNonEscapingLocalObject(O1))
+    if (isEscapeSource(O2) && isNonEscapingLocalObject(O1, &IsCapturedCache))
       return NoAlias;
   }
 
@@ -1830,7 +1901,7 @@ void BasicAAResult::GetIndexDifference(
   for (unsigned i = 0, e = Src.size(); i != e; ++i) {
     const Value *V = Src[i].V;
     unsigned ZExtBits = Src[i].ZExtBits, SExtBits = Src[i].SExtBits;
-    int64_t Scale = Src[i].Scale;
+    APInt Scale = Src[i].Scale;
 
     // Find V in Dest.  This is N^2, but pointer indices almost never have more
     // than a few variable indexes.
@@ -1850,7 +1921,7 @@ void BasicAAResult::GetIndexDifference(
     }
 
     // If we didn't consume this entry, add it to the end of the Dest list.
-    if (Scale) {
+    if (!!Scale) {
       VariableGEPIndex Entry = {V, ZExtBits, SExtBits, -Scale};
       Dest.push_back(Entry);
     }
@@ -1859,7 +1930,7 @@ void BasicAAResult::GetIndexDifference(
 
 bool BasicAAResult::constantOffsetHeuristic(
     const SmallVectorImpl<VariableGEPIndex> &VarIndices,
-    LocationSize MaybeV1Size, LocationSize MaybeV2Size, int64_t BaseOffset,
+    LocationSize MaybeV1Size, LocationSize MaybeV2Size, APInt BaseOffset,
     AssumptionCache *AC, DominatorTree *DT) {
   if (VarIndices.size() != 2 || MaybeV1Size == LocationSize::unknown() ||
       MaybeV2Size == LocationSize::unknown())
@@ -1904,14 +1975,15 @@ bool BasicAAResult::constantOffsetHeuristic(
   // the minimum distance between %i and %i + 5 is 3.
   APInt MinDiff = V0Offset - V1Offset, Wrapped = -MinDiff;
   MinDiff = APIntOps::umin(MinDiff, Wrapped);
-  uint64_t MinDiffBytes = MinDiff.getZExtValue() * std::abs(Var0.Scale);
+  APInt MinDiffBytes =
+    MinDiff.zextOrTrunc(Var0.Scale.getBitWidth()) * Var0.Scale.abs();
 
   // We can't definitely say whether GEP1 is before or after V2 due to wrapping
   // arithmetic (i.e. for some values of GEP1 and V2 GEP1 < V2, and for other
   // values GEP1 > V2). We'll therefore only declare NoAlias if both V1Size and
   // V2Size can fit in the MinDiffBytes gap.
-  return V1Size + std::abs(BaseOffset) <= MinDiffBytes &&
-         V2Size + std::abs(BaseOffset) <= MinDiffBytes;
+  return MinDiffBytes.uge(V1Size + BaseOffset.abs()) &&
+         MinDiffBytes.uge(V2Size + BaseOffset.abs());
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/lib/Analysis/BlockFrequencyInfo.cpp b/lib/Analysis/BlockFrequencyInfo.cpp
index 41c2958952139eb47a705b8ec968289fd0af0bba..398303dd31b416a170acac849d3e7d8d9529c51b 100644
--- a/lib/Analysis/BlockFrequencyInfo.cpp
+++ b/lib/Analysis/BlockFrequencyInfo.cpp
@@ -1,9 +1,8 @@
 //===- BlockFrequencyInfo.cpp - Block Frequency Analysis ------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -252,8 +251,8 @@ void BlockFrequencyInfo::setBlockFreqAndScale(
 
 /// Pop up a ghostview window with the current block frequency propagation
 /// rendered using dot.
-void BlockFrequencyInfo::view() const {
-  ViewGraph(const_cast<BlockFrequencyInfo *>(this), "BlockFrequencyDAGs");
+void BlockFrequencyInfo::view(StringRef title) const {
+  ViewGraph(const_cast<BlockFrequencyInfo *>(this), title);
 }
 
 const Function *BlockFrequencyInfo::getFunction() const {
diff --git a/lib/Analysis/BlockFrequencyInfoImpl.cpp b/lib/Analysis/BlockFrequencyInfoImpl.cpp
index 08ebcc47a807b18faad1d7101b13a62028dfa9b0..f0b9ea6f553ec8d41c6d838b21db9a9f6c08baca 100644
--- a/lib/Analysis/BlockFrequencyInfoImpl.cpp
+++ b/lib/Analysis/BlockFrequencyInfoImpl.cpp
@@ -1,9 +1,8 @@
 //===- BlockFrequencyImplInfo.cpp - Block Frequency Info Implementation ---===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Analysis/BranchProbabilityInfo.cpp b/lib/Analysis/BranchProbabilityInfo.cpp
index 7f544b27fe9d6ab79f900b5baa3256e6af548e66..e4de67b8b8443b1ab42f529a57996d66bee3ab3c 100644
--- a/lib/Analysis/BranchProbabilityInfo.cpp
+++ b/lib/Analysis/BranchProbabilityInfo.cpp
@@ -1,9 +1,8 @@
 //===- BranchProbabilityInfo.cpp - Branch Probability Analysis ------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Analysis/CFG.cpp b/lib/Analysis/CFG.cpp
index aa880a62b75464a8de4b77603aec0a7499e85c76..d71dd116ce31d3ec0bea26c67115e78ffc2640b6 100644
--- a/lib/Analysis/CFG.cpp
+++ b/lib/Analysis/CFG.cpp
@@ -1,9 +1,8 @@
 //===-- CFG.cpp - BasicBlock analysis --------------------------------------==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Analysis/CFGPrinter.cpp b/lib/Analysis/CFGPrinter.cpp
index 6d01e9d5d447b3a0f81bbe53d8bffa3f4ac2720a..619b675b58d88bbe88306081652e10aac227e344 100644
--- a/lib/Analysis/CFGPrinter.cpp
+++ b/lib/Analysis/CFGPrinter.cpp
@@ -1,9 +1,8 @@
 //===- CFGPrinter.cpp - DOT printer for the control flow graph ------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Analysis/CFLAndersAliasAnalysis.cpp b/lib/Analysis/CFLAndersAliasAnalysis.cpp
index 1c61dd369a05fbe7b9a3081c57fd7f5c27b7301d..2af64e81f2a5a1ea9abe26b8c142bb57d071b75c 100644
--- a/lib/Analysis/CFLAndersAliasAnalysis.cpp
+++ b/lib/Analysis/CFLAndersAliasAnalysis.cpp
@@ -1,9 +1,8 @@
 //===- CFLAndersAliasAnalysis.cpp - Unification-based Alias Analysis ------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Analysis/CFLGraph.h b/lib/Analysis/CFLGraph.h
index 12121d7174339bcb58e15d1c371762040877d965..5783c5bc9bdd5ca9e4a7e8c6c90788706baa42a1 100644
--- a/lib/Analysis/CFLGraph.h
+++ b/lib/Analysis/CFLGraph.h
@@ -1,9 +1,8 @@
 //===- CFLGraph.h - Abstract stratified sets implementation. -----*- C++-*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -25,7 +24,6 @@
 #include "llvm/Analysis/TargetLibraryInfo.h"
 #include "llvm/IR/Argument.h"
 #include "llvm/IR/BasicBlock.h"
-#include "llvm/IR/CallSite.h"
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/DataLayout.h"
 #include "llvm/IR/Function.h"
@@ -191,9 +189,9 @@ template <typename CFLAA> class CFLGraphBuilder {
 
     // Returns possible functions called by CS into the given SmallVectorImpl.
     // Returns true if targets found, false otherwise.
-    static bool getPossibleTargets(CallSite CS,
+    static bool getPossibleTargets(CallBase &Call,
                                    SmallVectorImpl<Function *> &Output) {
-      if (auto *Fn = CS.getCalledFunction()) {
+      if (auto *Fn = Call.getCalledFunction()) {
         Output.push_back(Fn);
         return true;
       }
@@ -370,11 +368,11 @@ template <typename CFLAA> class CFLGraphBuilder {
       return !Fn->hasExactDefinition();
     }
 
-    bool tryInterproceduralAnalysis(CallSite CS,
+    bool tryInterproceduralAnalysis(CallBase &Call,
                                     const SmallVectorImpl<Function *> &Fns) {
       assert(Fns.size() > 0);
 
-      if (CS.arg_size() > MaxSupportedArgsInSummary)
+      if (Call.arg_size() > MaxSupportedArgsInSummary)
         return false;
 
       // Exit early if we'll fail anyway
@@ -382,7 +380,7 @@ template <typename CFLAA> class CFLGraphBuilder {
         if (isFunctionExternal(Fn) || Fn->isVarArg())
           return false;
         // Fail if the caller does not provide enough arguments
-        assert(Fn->arg_size() <= CS.arg_size());
+        assert(Fn->arg_size() <= Call.arg_size());
         if (!AA.getAliasSummary(*Fn))
           return false;
       }
@@ -393,7 +391,7 @@ template <typename CFLAA> class CFLGraphBuilder {
 
         auto &RetParamRelations = Summary->RetParamRelations;
         for (auto &Relation : RetParamRelations) {
-          auto IRelation = instantiateExternalRelation(Relation, CS);
+          auto IRelation = instantiateExternalRelation(Relation, Call);
           if (IRelation.hasValue()) {
             Graph.addNode(IRelation->From);
             Graph.addNode(IRelation->To);
@@ -403,7 +401,7 @@ template <typename CFLAA> class CFLGraphBuilder {
 
         auto &RetParamAttributes = Summary->RetParamAttributes;
         for (auto &Attribute : RetParamAttributes) {
-          auto IAttr = instantiateExternalAttribute(Attribute, CS);
+          auto IAttr = instantiateExternalAttribute(Attribute, Call);
           if (IAttr.hasValue())
             Graph.addNode(IAttr->IValue, IAttr->Attr);
         }
@@ -412,37 +410,35 @@ template <typename CFLAA> class CFLGraphBuilder {
       return true;
     }
 
-    void visitCallSite(CallSite CS) {
-      auto Inst = CS.getInstruction();
-
+    void visitCallBase(CallBase &Call) {
       // Make sure all arguments and return value are added to the graph first
-      for (Value *V : CS.args())
+      for (Value *V : Call.args())
         if (V->getType()->isPointerTy())
           addNode(V);
-      if (Inst->getType()->isPointerTy())
-        addNode(Inst);
+      if (Call.getType()->isPointerTy())
+        addNode(&Call);
 
       // Check if Inst is a call to a library function that
       // allocates/deallocates on the heap. Those kinds of functions do not
       // introduce any aliases.
       // TODO: address other common library functions such as realloc(),
       // strdup(), etc.
-      if (isMallocOrCallocLikeFn(Inst, &TLI) || isFreeCall(Inst, &TLI))
+      if (isMallocOrCallocLikeFn(&Call, &TLI) || isFreeCall(&Call, &TLI))
         return;
 
       // TODO: Add support for noalias args/all the other fun function
       // attributes that we can tack on.
       SmallVector<Function *, 4> Targets;
-      if (getPossibleTargets(CS, Targets))
-        if (tryInterproceduralAnalysis(CS, Targets))
+      if (getPossibleTargets(Call, Targets))
+        if (tryInterproceduralAnalysis(Call, Targets))
           return;
 
       // Because the function is opaque, we need to note that anything
       // could have happened to the arguments (unless the function is marked
       // readonly or readnone), and that the result could alias just about
       // anything, too (unless the result is marked noalias).
-      if (!CS.onlyReadsMemory())
-        for (Value *V : CS.args()) {
+      if (!Call.onlyReadsMemory())
+        for (Value *V : Call.args()) {
           if (V->getType()->isPointerTy()) {
             // The argument itself escapes.
             Graph.addAttr(InstantiatedValue{V, 0}, getAttrEscaped());
@@ -453,12 +449,12 @@ template <typename CFLAA> class CFLGraphBuilder {
           }
         }
 
-      if (Inst->getType()->isPointerTy()) {
-        auto *Fn = CS.getCalledFunction();
+      if (Call.getType()->isPointerTy()) {
+        auto *Fn = Call.getCalledFunction();
         if (Fn == nullptr || !Fn->returnDoesNotAlias())
           // No need to call addNode() since we've added Inst at the
           // beginning of this function and we know it is not a global.
-          Graph.addAttr(InstantiatedValue{Inst, 0}, getAttrUnknown());
+          Graph.addAttr(InstantiatedValue{&Call, 0}, getAttrUnknown());
       }
     }
 
diff --git a/lib/Analysis/CFLSteensAliasAnalysis.cpp b/lib/Analysis/CFLSteensAliasAnalysis.cpp
index 30ce13578e5433f994fda6615c9fde4743a703ee..44b1834f70bf694fec63424507edf30060c8f915 100644
--- a/lib/Analysis/CFLSteensAliasAnalysis.cpp
+++ b/lib/Analysis/CFLSteensAliasAnalysis.cpp
@@ -1,9 +1,8 @@
 //===- CFLSteensAliasAnalysis.cpp - Unification-based Alias Analysis ------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Analysis/CGSCCPassManager.cpp b/lib/Analysis/CGSCCPassManager.cpp
index fd2292ced017d4415ef7746fc094e75974d87763..e811dc3a802739a5c82a7e3552f9745d52fe2c2c 100644
--- a/lib/Analysis/CGSCCPassManager.cpp
+++ b/lib/Analysis/CGSCCPassManager.cpp
@@ -1,9 +1,8 @@
 //===- CGSCCPassManager.cpp - Managing & running CGSCC passes -------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Analysis/CMakeLists.txt b/lib/Analysis/CMakeLists.txt
index c57d8ef69d69b7d45a931af2176d6c4a08de0c04..3cc9fe3c17155b2c0ed31b77aa50982abc2785a0 100644
--- a/lib/Analysis/CMakeLists.txt
+++ b/lib/Analysis/CMakeLists.txt
@@ -27,6 +27,7 @@ add_llvm_library(LLVMAnalysis
   DependenceAnalysis.cpp
   DivergenceAnalysis.cpp
   DomPrinter.cpp
+  DomTreeUpdater.cpp
   DominanceFrontier.cpp
   EHPersonalities.cpp
   GlobalsModRef.cpp
diff --git a/lib/Analysis/CallGraph.cpp b/lib/Analysis/CallGraph.cpp
index 0da678e1611b851683df63a762cfdaa43f8faaa1..e4e9e71adbda2949a858cdbe2d293b1f5c8d052b 100644
--- a/lib/Analysis/CallGraph.cpp
+++ b/lib/Analysis/CallGraph.cpp
@@ -1,9 +1,8 @@
 //===- CallGraph.cpp - Build a Module's call graph ------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Analysis/CallGraphSCCPass.cpp b/lib/Analysis/CallGraphSCCPass.cpp
index 0aed57a39387cc89b2ea218c1fd5515868b14737..ee01f2536fb8540dc0f9d1ac1ced2369c748fb9d 100644
--- a/lib/Analysis/CallGraphSCCPass.cpp
+++ b/lib/Analysis/CallGraphSCCPass.cpp
@@ -1,9 +1,8 @@
 //===- CallGraphSCCPass.cpp - Pass that operates BU on call graph ---------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Analysis/CallPrinter.cpp b/lib/Analysis/CallPrinter.cpp
index e7017e77652af7c176eb0143b0056576badac36d..d24cbd104bf62eab8223c9beb2df2e7a0eb33510 100644
--- a/lib/Analysis/CallPrinter.cpp
+++ b/lib/Analysis/CallPrinter.cpp
@@ -1,9 +1,8 @@
 //===- CallPrinter.cpp - DOT printer for call graph -----------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Analysis/CaptureTracking.cpp b/lib/Analysis/CaptureTracking.cpp
index 5cac5e5ca616a4f9a4706f1ea398d08bda8afbce..d3bd556f8bc63ce810837498634b188e3e224431 100644
--- a/lib/Analysis/CaptureTracking.cpp
+++ b/lib/Analysis/CaptureTracking.cpp
@@ -1,9 +1,8 @@
 //===--- CaptureTracking.cpp - Determine whether a pointer is captured ----===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -23,7 +22,6 @@
 #include "llvm/Analysis/CFG.h"
 #include "llvm/Analysis/OrderedBasicBlock.h"
 #include "llvm/Analysis/ValueTracking.h"
-#include "llvm/IR/CallSite.h"
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/Dominators.h"
 #include "llvm/IR/Instructions.h"
@@ -240,11 +238,12 @@ void llvm::PointerMayBeCaptured(const Value *V, CaptureTracker *Tracker,
     switch (I->getOpcode()) {
     case Instruction::Call:
     case Instruction::Invoke: {
-      CallSite CS(I);
+      auto *Call = cast<CallBase>(I);
       // Not captured if the callee is readonly, doesn't return a copy through
       // its return value and doesn't unwind (a readonly function can leak bits
       // by throwing an exception or not depending on the input value).
-      if (CS.onlyReadsMemory() && CS.doesNotThrow() && I->getType()->isVoidTy())
+      if (Call->onlyReadsMemory() && Call->doesNotThrow() &&
+          Call->getType()->isVoidTy())
         break;
 
       // The pointer is not captured if returned pointer is not captured.
@@ -252,14 +251,14 @@ void llvm::PointerMayBeCaptured(const Value *V, CaptureTracker *Tracker,
       // marked with nocapture do not capture. This means that places like
       // GetUnderlyingObject in ValueTracking or DecomposeGEPExpression
       // in BasicAA also need to know about this property.
-      if (isIntrinsicReturningPointerAliasingArgumentWithoutCapturing(CS)) {
-        AddUses(I);
+      if (isIntrinsicReturningPointerAliasingArgumentWithoutCapturing(Call)) {
+        AddUses(Call);
         break;
       }
 
       // Volatile operations effectively capture the memory location that they
       // load and store to.
-      if (auto *MI = dyn_cast<MemIntrinsic>(I))
+      if (auto *MI = dyn_cast<MemIntrinsic>(Call))
         if (MI->isVolatile())
           if (Tracker->captured(U))
             return;
@@ -271,13 +270,14 @@ void llvm::PointerMayBeCaptured(const Value *V, CaptureTracker *Tracker,
       // that loading a value from a pointer does not cause the pointer to be
       // captured, even though the loaded value might be the pointer itself
       // (think of self-referential objects).
-      CallSite::data_operand_iterator B =
-        CS.data_operands_begin(), E = CS.data_operands_end();
-      for (CallSite::data_operand_iterator A = B; A != E; ++A)
-        if (A->get() == V && !CS.doesNotCapture(A - B))
+      for (auto IdxOpPair : enumerate(Call->data_ops())) {
+        int Idx = IdxOpPair.index();
+        Value *A = IdxOpPair.value();
+        if (A == V && !Call->doesNotCapture(Idx))
           // The parameter is not marked 'nocapture' - captured.
           if (Tracker->captured(U))
             return;
+      }
       break;
     }
     case Instruction::Load:
diff --git a/lib/Analysis/CmpInstAnalysis.cpp b/lib/Analysis/CmpInstAnalysis.cpp
index 27071babec5c85f0924a177cce0243868c958486..a5757be2c4f409507940f224e59fddf7912bee5a 100644
--- a/lib/Analysis/CmpInstAnalysis.cpp
+++ b/lib/Analysis/CmpInstAnalysis.cpp
@@ -1,9 +1,8 @@
 //===- CmpInstAnalysis.cpp - Utils to help fold compares ---------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Analysis/CodeMetrics.cpp b/lib/Analysis/CodeMetrics.cpp
index 46cc87d2b1781a658fa9d5059d2a5dc2b6d349ae..627d955c865f2dbff2a5d6e17f1faa390b86f236 100644
--- a/lib/Analysis/CodeMetrics.cpp
+++ b/lib/Analysis/CodeMetrics.cpp
@@ -1,9 +1,8 @@
 //===- CodeMetrics.cpp - Code cost measurements ---------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -16,7 +15,6 @@
 #include "llvm/Analysis/LoopInfo.h"
 #include "llvm/Analysis/TargetTransformInfo.h"
 #include "llvm/Analysis/ValueTracking.h"
-#include "llvm/IR/CallSite.h"
 #include "llvm/IR/DataLayout.h"
 #include "llvm/IR/Function.h"
 #include "llvm/Support/Debug.h"
@@ -126,14 +124,12 @@ void CodeMetrics::analyzeBasicBlock(const BasicBlock *BB,
       continue;
 
     // Special handling for calls.
-    if (isa<CallInst>(I) || isa<InvokeInst>(I)) {
-      ImmutableCallSite CS(&I);
-
-      if (const Function *F = CS.getCalledFunction()) {
+    if (const auto *Call = dyn_cast<CallBase>(&I)) {
+      if (const Function *F = Call->getCalledFunction()) {
         // If a function is both internal and has a single use, then it is
         // extremely likely to get inlined in the future (it was probably
         // exposed by an interleaved devirtualization pass).
-        if (!CS.isNoInline() && F->hasInternalLinkage() && F->hasOneUse())
+        if (!Call->isNoInline() && F->hasInternalLinkage() && F->hasOneUse())
           ++NumInlineCandidates;
 
         // If this call is to function itself, then the function is recursive.
@@ -148,7 +144,7 @@ void CodeMetrics::analyzeBasicBlock(const BasicBlock *BB,
       } else {
         // We don't want inline asm to count as a call - that would prevent loop
         // unrolling. The argument setup cost is still real, though.
-        if (!isa<InlineAsm>(CS.getCalledValue()))
+        if (!Call->isInlineAsm())
           ++NumCalls;
       }
     }
diff --git a/lib/Analysis/ConstantFolding.cpp b/lib/Analysis/ConstantFolding.cpp
index 6180886267470e441ff51e3c00344f18e48bc6b1..3b4e60315662a89cd46a4eff71acddb56737be7f 100644
--- a/lib/Analysis/ConstantFolding.cpp
+++ b/lib/Analysis/ConstantFolding.cpp
@@ -1,9 +1,8 @@
 //===-- ConstantFolding.cpp - Fold instructions into constants ------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -1025,9 +1024,9 @@ Constant *ConstantFoldInstOperandsImpl(const Value *InstOrCE, unsigned Opcode,
   case Instruction::FCmp: llvm_unreachable("Invalid for compares");
   case Instruction::Call:
     if (auto *F = dyn_cast<Function>(Ops.back())) {
-      ImmutableCallSite CS(cast<CallInst>(InstOrCE));
-      if (canConstantFoldCallTo(CS, F))
-        return ConstantFoldCall(CS, F, Ops.slice(0, Ops.size() - 1), TLI);
+      const auto *Call = cast<CallBase>(InstOrCE);
+      if (canConstantFoldCallTo(Call, F))
+        return ConstantFoldCall(Call, F, Ops.slice(0, Ops.size() - 1), TLI);
     }
     return nullptr;
   case Instruction::Select:
@@ -1367,8 +1366,8 @@ llvm::ConstantFoldLoadThroughGEPIndices(Constant *C,
 //  Constant Folding for Calls
 //
 
-bool llvm::canConstantFoldCallTo(ImmutableCallSite CS, const Function *F) {
-  if (CS.isNoBuiltin() || CS.isStrictFP())
+bool llvm::canConstantFoldCallTo(const CallBase *Call, const Function *F) {
+  if (Call->isNoBuiltin() || Call->isStrictFP())
     return false;
   switch (F->getIntrinsicID()) {
   case Intrinsic::fabs:
@@ -1629,10 +1628,22 @@ static bool isManifestConstant(const Constant *c) {
   return false;
 }
 
+static bool getConstIntOrUndef(Value *Op, const APInt *&C) {
+  if (auto *CI = dyn_cast<ConstantInt>(Op)) {
+    C = &CI->getValue();
+    return true;
+  }
+  if (isa<UndefValue>(Op)) {
+    C = nullptr;
+    return true;
+  }
+  return false;
+}
+
 Constant *ConstantFoldScalarCall(StringRef Name, unsigned IntrinsicID, Type *Ty,
                                  ArrayRef<Constant *> Operands,
                                  const TargetLibraryInfo *TLI,
-                                 ImmutableCallSite CS) {
+                                 const CallBase *Call) {
   if (Operands.size() == 1) {
     if (IntrinsicID == Intrinsic::is_constant) {
       // We know we have a "Constant" argument. But we want to only
@@ -1643,8 +1654,10 @@ Constant *ConstantFoldScalarCall(StringRef Name, unsigned IntrinsicID, Type *Ty,
       return nullptr;
     }
     if (isa<UndefValue>(Operands[0])) {
-      // cosine(arg) is between -1 and 1. cosine(invalid arg) is NaN
-      if (IntrinsicID == Intrinsic::cos)
+      // cosine(arg) is between -1 and 1. cosine(invalid arg) is NaN.
+      // ctpop() is between 0 and bitwidth, pick 0 for undef.
+      if (IntrinsicID == Intrinsic::cos ||
+          IntrinsicID == Intrinsic::ctpop)
         return Constant::getNullValue(Ty);
       if (IntrinsicID == Intrinsic::bswap ||
           IntrinsicID == Intrinsic::bitreverse ||
@@ -1658,9 +1671,10 @@ Constant *ConstantFoldScalarCall(StringRef Name, unsigned IntrinsicID, Type *Ty,
       if (IntrinsicID == Intrinsic::launder_invariant_group ||
           IntrinsicID == Intrinsic::strip_invariant_group) {
         // If instruction is not yet put in a basic block (e.g. when cloning
-        // a function during inlining), CS caller may not be available.
-        // So check CS's BB first before querying CS.getCaller.
-        const Function *Caller = CS.getParent() ? CS.getCaller() : nullptr;
+        // a function during inlining), Call's caller may not be available.
+        // So check Call's BB first before querying Call->getCaller.
+        const Function *Caller =
+            Call->getParent() ? Call->getCaller() : nullptr;
         if (Caller &&
             !NullPointerIsDefined(
                 Caller, Operands[0]->getType()->getPointerAddressSpace())) {
@@ -1995,62 +2009,92 @@ Constant *ConstantFoldScalarCall(StringRef Name, unsigned IntrinsicID, Type *Ty,
       return nullptr;
     }
 
-    if (auto *Op1 = dyn_cast<ConstantInt>(Operands[0])) {
-      if (auto *Op2 = dyn_cast<ConstantInt>(Operands[1])) {
+    if (Operands[0]->getType()->isIntegerTy() &&
+        Operands[1]->getType()->isIntegerTy()) {
+      const APInt *C0, *C1;
+      if (!getConstIntOrUndef(Operands[0], C0) ||
+          !getConstIntOrUndef(Operands[1], C1))
+        return nullptr;
+
+      switch (IntrinsicID) {
+      default: break;
+      case Intrinsic::smul_with_overflow:
+      case Intrinsic::umul_with_overflow:
+        // Even if both operands are undef, we cannot fold muls to undef
+        // in the general case. For example, on i2 there are no inputs
+        // that would produce { i2 -1, i1 true } as the result.
+        if (!C0 || !C1)
+          return Constant::getNullValue(Ty);
+        LLVM_FALLTHROUGH;
+      case Intrinsic::sadd_with_overflow:
+      case Intrinsic::uadd_with_overflow:
+      case Intrinsic::ssub_with_overflow:
+      case Intrinsic::usub_with_overflow: {
+        if (!C0 || !C1)
+          return UndefValue::get(Ty);
+
+        APInt Res;
+        bool Overflow;
         switch (IntrinsicID) {
-        default: break;
+        default: llvm_unreachable("Invalid case");
         case Intrinsic::sadd_with_overflow:
+          Res = C0->sadd_ov(*C1, Overflow);
+          break;
         case Intrinsic::uadd_with_overflow:
+          Res = C0->uadd_ov(*C1, Overflow);
+          break;
         case Intrinsic::ssub_with_overflow:
+          Res = C0->ssub_ov(*C1, Overflow);
+          break;
         case Intrinsic::usub_with_overflow:
+          Res = C0->usub_ov(*C1, Overflow);
+          break;
         case Intrinsic::smul_with_overflow:
-        case Intrinsic::umul_with_overflow: {
-          APInt Res;
-          bool Overflow;
-          switch (IntrinsicID) {
-          default: llvm_unreachable("Invalid case");
-          case Intrinsic::sadd_with_overflow:
-            Res = Op1->getValue().sadd_ov(Op2->getValue(), Overflow);
-            break;
-          case Intrinsic::uadd_with_overflow:
-            Res = Op1->getValue().uadd_ov(Op2->getValue(), Overflow);
-            break;
-          case Intrinsic::ssub_with_overflow:
-            Res = Op1->getValue().ssub_ov(Op2->getValue(), Overflow);
-            break;
-          case Intrinsic::usub_with_overflow:
-            Res = Op1->getValue().usub_ov(Op2->getValue(), Overflow);
-            break;
-          case Intrinsic::smul_with_overflow:
-            Res = Op1->getValue().smul_ov(Op2->getValue(), Overflow);
-            break;
-          case Intrinsic::umul_with_overflow:
-            Res = Op1->getValue().umul_ov(Op2->getValue(), Overflow);
-            break;
-          }
-          Constant *Ops[] = {
-            ConstantInt::get(Ty->getContext(), Res),
-            ConstantInt::get(Type::getInt1Ty(Ty->getContext()), Overflow)
-          };
-          return ConstantStruct::get(cast<StructType>(Ty), Ops);
-        }
-        case Intrinsic::uadd_sat:
-          return ConstantInt::get(Ty, Op1->getValue().uadd_sat(Op2->getValue()));
-        case Intrinsic::sadd_sat:
-          return ConstantInt::get(Ty, Op1->getValue().sadd_sat(Op2->getValue()));
-        case Intrinsic::usub_sat:
-          return ConstantInt::get(Ty, Op1->getValue().usub_sat(Op2->getValue()));
-        case Intrinsic::ssub_sat:
-          return ConstantInt::get(Ty, Op1->getValue().ssub_sat(Op2->getValue()));
-        case Intrinsic::cttz:
-          if (Op2->isOne() && Op1->isZero()) // cttz(0, 1) is undef.
-            return UndefValue::get(Ty);
-          return ConstantInt::get(Ty, Op1->getValue().countTrailingZeros());
-        case Intrinsic::ctlz:
-          if (Op2->isOne() && Op1->isZero()) // ctlz(0, 1) is undef.
-            return UndefValue::get(Ty);
-          return ConstantInt::get(Ty, Op1->getValue().countLeadingZeros());
+          Res = C0->smul_ov(*C1, Overflow);
+          break;
+        case Intrinsic::umul_with_overflow:
+          Res = C0->umul_ov(*C1, Overflow);
+          break;
         }
+        Constant *Ops[] = {
+          ConstantInt::get(Ty->getContext(), Res),
+          ConstantInt::get(Type::getInt1Ty(Ty->getContext()), Overflow)
+        };
+        return ConstantStruct::get(cast<StructType>(Ty), Ops);
+      }
+      case Intrinsic::uadd_sat:
+      case Intrinsic::sadd_sat:
+        if (!C0 && !C1)
+          return UndefValue::get(Ty);
+        if (!C0 || !C1)
+          return Constant::getAllOnesValue(Ty);
+        if (IntrinsicID == Intrinsic::uadd_sat)
+          return ConstantInt::get(Ty, C0->uadd_sat(*C1));
+        else
+          return ConstantInt::get(Ty, C0->sadd_sat(*C1));
+      case Intrinsic::usub_sat:
+      case Intrinsic::ssub_sat:
+        if (!C0 && !C1)
+          return UndefValue::get(Ty);
+        if (!C0 || !C1)
+          return Constant::getNullValue(Ty);
+        if (IntrinsicID == Intrinsic::usub_sat)
+          return ConstantInt::get(Ty, C0->usub_sat(*C1));
+        else
+          return ConstantInt::get(Ty, C0->ssub_sat(*C1));
+      case Intrinsic::cttz:
+      case Intrinsic::ctlz:
+        assert(C1 && "Must be constant int");
+
+        // cttz(0, 1) and ctlz(0, 1) are undef.
+        if (C1->isOneValue() && (!C0 || C0->isNullValue()))
+          return UndefValue::get(Ty);
+        if (!C0)
+          return Constant::getNullValue(Ty);
+        if (IntrinsicID == Intrinsic::cttz)
+          return ConstantInt::get(Ty, C0->countTrailingZeros());
+        else
+          return ConstantInt::get(Ty, C0->countLeadingZeros());
       }
 
       return nullptr;
@@ -2136,26 +2180,33 @@ Constant *ConstantFoldScalarCall(StringRef Name, unsigned IntrinsicID, Type *Ty,
   }
 
   if (IntrinsicID == Intrinsic::fshl || IntrinsicID == Intrinsic::fshr) {
-    auto *C0 = dyn_cast<ConstantInt>(Operands[0]);
-    auto *C1 = dyn_cast<ConstantInt>(Operands[1]);
-    auto *C2 = dyn_cast<ConstantInt>(Operands[2]);
-    if (!(C0 && C1 && C2))
+    const APInt *C0, *C1, *C2;
+    if (!getConstIntOrUndef(Operands[0], C0) ||
+        !getConstIntOrUndef(Operands[1], C1) ||
+        !getConstIntOrUndef(Operands[2], C2))
       return nullptr;
 
+    bool IsRight = IntrinsicID == Intrinsic::fshr;
+    if (!C2)
+      return Operands[IsRight ? 1 : 0];
+    if (!C0 && !C1)
+      return UndefValue::get(Ty);
+
     // The shift amount is interpreted as modulo the bitwidth. If the shift
     // amount is effectively 0, avoid UB due to oversized inverse shift below.
-    unsigned BitWidth = C0->getBitWidth();
-    unsigned ShAmt = C2->getValue().urem(BitWidth);
-    bool IsRight = IntrinsicID == Intrinsic::fshr;
+    unsigned BitWidth = C2->getBitWidth();
+    unsigned ShAmt = C2->urem(BitWidth);
     if (!ShAmt)
-      return IsRight ? C1 : C0;
+      return Operands[IsRight ? 1 : 0];
 
-    // (X << ShlAmt) | (Y >> LshrAmt)
-    const APInt &X = C0->getValue();
-    const APInt &Y = C1->getValue();
+    // (C0 << ShlAmt) | (C1 >> LshrAmt)
     unsigned LshrAmt = IsRight ? ShAmt : BitWidth - ShAmt;
     unsigned ShlAmt = !IsRight ? ShAmt : BitWidth - ShAmt;
-    return ConstantInt::get(Ty->getContext(), X.shl(ShlAmt) | Y.lshr(LshrAmt));
+    if (!C0)
+      return ConstantInt::get(Ty, C1->lshr(LshrAmt));
+    if (!C1)
+      return ConstantInt::get(Ty, C0->shl(ShlAmt));
+    return ConstantInt::get(Ty, C0->shl(ShlAmt) | C1->lshr(LshrAmt));
   }
 
   return nullptr;
@@ -2165,7 +2216,7 @@ Constant *ConstantFoldVectorCall(StringRef Name, unsigned IntrinsicID,
                                  VectorType *VTy, ArrayRef<Constant *> Operands,
                                  const DataLayout &DL,
                                  const TargetLibraryInfo *TLI,
-                                 ImmutableCallSite CS) {
+                                 const CallBase *Call) {
   SmallVector<Constant *, 4> Result(VTy->getNumElements());
   SmallVector<Constant *, 4> Lane(Operands.size());
   Type *Ty = VTy->getElementType();
@@ -2228,7 +2279,8 @@ Constant *ConstantFoldVectorCall(StringRef Name, unsigned IntrinsicID,
     }
 
     // Use the regular scalar folding to simplify this column.
-    Constant *Folded = ConstantFoldScalarCall(Name, IntrinsicID, Ty, Lane, TLI, CS);
+    Constant *Folded =
+        ConstantFoldScalarCall(Name, IntrinsicID, Ty, Lane, TLI, Call);
     if (!Folded)
       return nullptr;
     Result[I] = Folded;
@@ -2239,11 +2291,10 @@ Constant *ConstantFoldVectorCall(StringRef Name, unsigned IntrinsicID,
 
 } // end anonymous namespace
 
-Constant *
-llvm::ConstantFoldCall(ImmutableCallSite CS, Function *F,
-                       ArrayRef<Constant *> Operands,
-                       const TargetLibraryInfo *TLI) {
-  if (CS.isNoBuiltin() || CS.isStrictFP())
+Constant *llvm::ConstantFoldCall(const CallBase *Call, Function *F,
+                                 ArrayRef<Constant *> Operands,
+                                 const TargetLibraryInfo *TLI) {
+  if (Call->isNoBuiltin() || Call->isStrictFP())
     return nullptr;
   if (!F->hasName())
     return nullptr;
@@ -2253,17 +2304,19 @@ llvm::ConstantFoldCall(ImmutableCallSite CS, Function *F,
 
   if (auto *VTy = dyn_cast<VectorType>(Ty))
     return ConstantFoldVectorCall(Name, F->getIntrinsicID(), VTy, Operands,
-                                  F->getParent()->getDataLayout(), TLI, CS);
+                                  F->getParent()->getDataLayout(), TLI, Call);
 
-  return ConstantFoldScalarCall(Name, F->getIntrinsicID(), Ty, Operands, TLI, CS);
+  return ConstantFoldScalarCall(Name, F->getIntrinsicID(), Ty, Operands, TLI,
+                                Call);
 }
 
-bool llvm::isMathLibCallNoop(CallSite CS, const TargetLibraryInfo *TLI) {
+bool llvm::isMathLibCallNoop(const CallBase *Call,
+                             const TargetLibraryInfo *TLI) {
   // FIXME: Refactor this code; this duplicates logic in LibCallsShrinkWrap
   // (and to some extent ConstantFoldScalarCall).
-  if (CS.isNoBuiltin() || CS.isStrictFP())
+  if (Call->isNoBuiltin() || Call->isStrictFP())
     return false;
-  Function *F = CS.getCalledFunction();
+  Function *F = Call->getCalledFunction();
   if (!F)
     return false;
 
@@ -2271,8 +2324,8 @@ bool llvm::isMathLibCallNoop(CallSite CS, const TargetLibraryInfo *TLI) {
   if (!TLI || !TLI->getLibFunc(*F, Func))
     return false;
 
-  if (CS.getNumArgOperands() == 1) {
-    if (ConstantFP *OpC = dyn_cast<ConstantFP>(CS.getArgOperand(0))) {
+  if (Call->getNumArgOperands() == 1) {
+    if (ConstantFP *OpC = dyn_cast<ConstantFP>(Call->getArgOperand(0))) {
       const APFloat &Op = OpC->getValueAPF();
       switch (Func) {
       case LibFunc_logl:
@@ -2370,9 +2423,9 @@ bool llvm::isMathLibCallNoop(CallSite CS, const TargetLibraryInfo *TLI) {
     }
   }
 
-  if (CS.getNumArgOperands() == 2) {
-    ConstantFP *Op0C = dyn_cast<ConstantFP>(CS.getArgOperand(0));
-    ConstantFP *Op1C = dyn_cast<ConstantFP>(CS.getArgOperand(1));
+  if (Call->getNumArgOperands() == 2) {
+    ConstantFP *Op0C = dyn_cast<ConstantFP>(Call->getArgOperand(0));
+    ConstantFP *Op1C = dyn_cast<ConstantFP>(Call->getArgOperand(1));
     if (Op0C && Op1C) {
       const APFloat &Op0 = Op0C->getValueAPF();
       const APFloat &Op1 = Op1C->getValueAPF();
diff --git a/lib/Analysis/CostModel.cpp b/lib/Analysis/CostModel.cpp
index 3d55bf20bb4021a40f593dce4daa4f0a722fa89f..bf0cdbfd0c8b5a545fbfbb34c0eab303512bd0ee 100644
--- a/lib/Analysis/CostModel.cpp
+++ b/lib/Analysis/CostModel.cpp
@@ -1,9 +1,8 @@
 //===- CostModel.cpp ------ Cost Model Analysis ---------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Analysis/Delinearization.cpp b/lib/Analysis/Delinearization.cpp
index 4cafb7da16d3bfe49463cc7ee37c878981fc6932..c1043e446beb6d1fa6660819f6229b1da607ba93 100644
--- a/lib/Analysis/Delinearization.cpp
+++ b/lib/Analysis/Delinearization.cpp
@@ -1,9 +1,8 @@
 //===---- Delinearization.cpp - MultiDimensional Index Delinearization ----===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Analysis/DemandedBits.cpp b/lib/Analysis/DemandedBits.cpp
index 0382787fbefe788e0209bdae08a5427ca01c9bee..0df47a9eda6ef2a7f611358c7dfde42630e357cc 100644
--- a/lib/Analysis/DemandedBits.cpp
+++ b/lib/Analysis/DemandedBits.cpp
@@ -1,9 +1,8 @@
 //===- DemandedBits.cpp - Determine demanded bits -------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -21,8 +20,7 @@
 
 #include "llvm/Analysis/DemandedBits.h"
 #include "llvm/ADT/APInt.h"
-#include "llvm/ADT/SmallPtrSet.h"
-#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/Analysis/AssumptionCache.h"
 #include "llvm/Analysis/ValueTracking.h"
@@ -85,8 +83,9 @@ static bool isAlwaysLive(Instruction *I) {
 }
 
 void DemandedBits::determineLiveOperandBits(
-    const Instruction *UserI, const Instruction *I, unsigned OperandNo,
-    const APInt &AOut, APInt &AB, KnownBits &Known, KnownBits &Known2) {
+    const Instruction *UserI, const Value *Val, unsigned OperandNo,
+    const APInt &AOut, APInt &AB, KnownBits &Known, KnownBits &Known2,
+    bool &KnownBitsComputed) {
   unsigned BitWidth = AB.getBitWidth();
 
   // We're called once per operand, but for some instructions, we need to
@@ -97,7 +96,11 @@ void DemandedBits::determineLiveOperandBits(
   // provided here.
   auto ComputeKnownBits =
       [&](unsigned BitWidth, const Value *V1, const Value *V2) {
-        const DataLayout &DL = I->getModule()->getDataLayout();
+        if (KnownBitsComputed)
+          return;
+        KnownBitsComputed = true;
+
+        const DataLayout &DL = UserI->getModule()->getDataLayout();
         Known = KnownBits(BitWidth);
         computeKnownBits(V1, Known, DL, 0, &AC, UserI, &DT);
 
@@ -129,7 +132,7 @@ void DemandedBits::determineLiveOperandBits(
           // We need some output bits, so we need all bits of the
           // input to the left of, and including, the leftmost bit
           // known to be one.
-          ComputeKnownBits(BitWidth, I, nullptr);
+          ComputeKnownBits(BitWidth, Val, nullptr);
           AB = APInt::getHighBitsSet(BitWidth,
                  std::min(BitWidth, Known.countMaxLeadingZeros()+1));
         }
@@ -139,7 +142,7 @@ void DemandedBits::determineLiveOperandBits(
           // We need some output bits, so we need all bits of the
           // input to the right of, and including, the rightmost bit
           // known to be one.
-          ComputeKnownBits(BitWidth, I, nullptr);
+          ComputeKnownBits(BitWidth, Val, nullptr);
           AB = APInt::getLowBitsSet(BitWidth,
                  std::min(BitWidth, Known.countMaxTrailingZeros()+1));
         }
@@ -234,14 +237,11 @@ void DemandedBits::determineLiveOperandBits(
     // other operand are dead (unless they're both zero, in which
     // case they can't both be dead, so just mark the LHS bits as
     // dead).
-    if (OperandNo == 0) {
-      ComputeKnownBits(BitWidth, I, UserI->getOperand(1));
+    ComputeKnownBits(BitWidth, UserI->getOperand(0), UserI->getOperand(1));
+    if (OperandNo == 0)
       AB &= ~Known2.Zero;
-    } else {
-      if (!isa<Instruction>(UserI->getOperand(0)))
-        ComputeKnownBits(BitWidth, UserI->getOperand(0), I);
+    else
       AB &= ~(Known.Zero & ~Known2.Zero);
-    }
     break;
   case Instruction::Or:
     AB = AOut;
@@ -250,14 +250,11 @@ void DemandedBits::determineLiveOperandBits(
     // other operand are dead (unless they're both one, in which
     // case they can't both be dead, so just mark the LHS bits as
     // dead).
-    if (OperandNo == 0) {
-      ComputeKnownBits(BitWidth, I, UserI->getOperand(1));
+    ComputeKnownBits(BitWidth, UserI->getOperand(0), UserI->getOperand(1));
+    if (OperandNo == 0)
       AB &= ~Known2.One;
-    } else {
-      if (!isa<Instruction>(UserI->getOperand(0)))
-        ComputeKnownBits(BitWidth, UserI->getOperand(0), I);
+    else
       AB &= ~(Known.One & ~Known2.One);
-    }
     break;
   case Instruction::Xor:
   case Instruction::PHI:
@@ -314,8 +311,9 @@ void DemandedBits::performAnalysis() {
 
   Visited.clear();
   AliveBits.clear();
+  DeadUses.clear();
 
-  SmallVector<Instruction*, 128> Worklist;
+  SmallSetVector<Instruction*, 16> Worklist;
 
   // Collect the set of "root" instructions that are known live.
   for (Instruction &I : instructions(F)) {
@@ -330,7 +328,7 @@ void DemandedBits::performAnalysis() {
     Type *T = I.getType();
     if (T->isIntOrIntVectorTy()) {
       if (AliveBits.try_emplace(&I, T->getScalarSizeInBits(), 0).second)
-        Worklist.push_back(&I);
+        Worklist.insert(&I);
 
       continue;
     }
@@ -341,7 +339,7 @@ void DemandedBits::performAnalysis() {
         Type *T = J->getType();
         if (T->isIntOrIntVectorTy())
           AliveBits[J] = APInt::getAllOnesValue(T->getScalarSizeInBits());
-        Worklist.push_back(J);
+        Worklist.insert(J);
       }
     }
     // To save memory, we don't add I to the Visited set here. Instead, we
@@ -358,7 +356,8 @@ void DemandedBits::performAnalysis() {
     APInt AOut;
     if (UserI->getType()->isIntOrIntVectorTy()) {
       AOut = AliveBits[UserI];
-      LLVM_DEBUG(dbgs() << " Alive Out: " << AOut);
+      LLVM_DEBUG(dbgs() << " Alive Out: 0x"
+                        << Twine::utohexstr(AOut.getLimitedValue()));
     }
     LLVM_DEBUG(dbgs() << "\n");
 
@@ -366,26 +365,40 @@ void DemandedBits::performAnalysis() {
       Visited.insert(UserI);
 
     KnownBits Known, Known2;
+    bool KnownBitsComputed = false;
     // Compute the set of alive bits for each operand. These are anded into the
     // existing set, if any, and if that changes the set of alive bits, the
     // operand is added to the work-list.
     for (Use &OI : UserI->operands()) {
-      if (Instruction *I = dyn_cast<Instruction>(OI)) {
-        Type *T = I->getType();
-        if (T->isIntOrIntVectorTy()) {
-          unsigned BitWidth = T->getScalarSizeInBits();
-          APInt AB = APInt::getAllOnesValue(BitWidth);
-          if (UserI->getType()->isIntOrIntVectorTy() && !AOut &&
-              !isAlwaysLive(UserI)) {
-            AB = APInt(BitWidth, 0);
-          } else {
-            // If all bits of the output are dead, then all bits of the input
-            // Bits of each operand that are used to compute alive bits of the
-            // output are alive, all others are dead.
-            determineLiveOperandBits(UserI, I, OI.getOperandNo(), AOut, AB,
-                                     Known, Known2);
-          }
+      // We also want to detect dead uses of arguments, but will only store
+      // demanded bits for instructions.
+      Instruction *I = dyn_cast<Instruction>(OI);
+      if (!I && !isa<Argument>(OI))
+        continue;
+
+      Type *T = OI->getType();
+      if (T->isIntOrIntVectorTy()) {
+        unsigned BitWidth = T->getScalarSizeInBits();
+        APInt AB = APInt::getAllOnesValue(BitWidth);
+        if (UserI->getType()->isIntOrIntVectorTy() && !AOut &&
+            !isAlwaysLive(UserI)) {
+          // If all bits of the output are dead, then all bits of the input
+          // are also dead.
+          AB = APInt(BitWidth, 0);
+        } else {
+          // Bits of each operand that are used to compute alive bits of the
+          // output are alive, all others are dead.
+          determineLiveOperandBits(UserI, OI, OI.getOperandNo(), AOut, AB,
+                                   Known, Known2, KnownBitsComputed);
+
+          // Keep track of uses which have no demanded bits.
+          if (AB.isNullValue())
+            DeadUses.insert(&OI);
+          else
+            DeadUses.erase(&OI);
+        }
 
+        if (I) {
           // If we've added to the set of alive bits (or the operand has not
           // been previously visited), then re-queue the operand to be visited
           // again.
@@ -397,11 +410,11 @@ void DemandedBits::performAnalysis() {
           APInt ABNew = AB | ABPrev;
           if (ABNew != ABPrev || ABI == AliveBits.end()) {
             AliveBits[I] = std::move(ABNew);
-            Worklist.push_back(I);
+            Worklist.insert(I);
           }
-        } else if (!Visited.count(I)) {
-          Worklist.push_back(I);
         }
+      } else if (I && !Visited.count(I)) {
+        Worklist.insert(I);
       }
     }
   }
@@ -426,6 +439,31 @@ bool DemandedBits::isInstructionDead(Instruction *I) {
     !isAlwaysLive(I);
 }
 
+bool DemandedBits::isUseDead(Use *U) {
+  // We only track integer uses, everything else is assumed live.
+  if (!(*U)->getType()->isIntOrIntVectorTy())
+    return false;
+
+  // Uses by always-live instructions are never dead.
+  Instruction *UserI = cast<Instruction>(U->getUser());
+  if (isAlwaysLive(UserI))
+    return false;
+
+  performAnalysis();
+  if (DeadUses.count(U))
+    return true;
+
+  // If no output bits are demanded, no input bits are demanded and the use
+  // is dead. These uses might not be explicitly present in the DeadUses map.
+  if (UserI->getType()->isIntOrIntVectorTy()) {
+    auto Found = AliveBits.find(UserI);
+    if (Found != AliveBits.end() && Found->second.isNullValue())
+      return true;
+  }
+
+  return false;
+}
+
 void DemandedBits::print(raw_ostream &OS) {
   performAnalysis();
   for (auto &KV : AliveBits) {
diff --git a/lib/Analysis/DependenceAnalysis.cpp b/lib/Analysis/DependenceAnalysis.cpp
index b544ae5f535d3abf27249771db6e8236a67bfc77..8337b9b96ea23b53a600b953bac54edc9d68b81f 100644
--- a/lib/Analysis/DependenceAnalysis.cpp
+++ b/lib/Analysis/DependenceAnalysis.cpp
@@ -1,9 +1,8 @@
 //===-- DependenceAnalysis.cpp - DA Implementation --------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -194,6 +193,13 @@ void DependenceAnalysisWrapperPass::print(raw_ostream &OS,
   dumpExampleDependence(OS, info.get());
 }
 
+PreservedAnalyses
+DependenceAnalysisPrinterPass::run(Function &F, FunctionAnalysisManager &FAM) {
+  OS << "'Dependence Analysis' for function '" << F.getName() << "':\n";
+  dumpExampleDependence(OS, &FAM.getResult<DependenceAnalysis>(F));
+  return PreservedAnalyses::all();
+}
+
 //===----------------------------------------------------------------------===//
 // Dependence methods
 
@@ -3362,6 +3368,19 @@ static void dumpSmallBitVector(SmallBitVector &BV) {
 }
 #endif
 
+bool DependenceInfo::invalidate(Function &F, const PreservedAnalyses &PA,
+                                FunctionAnalysisManager::Invalidator &Inv) {
+  // Check if the analysis itself has been invalidated.
+  auto PAC = PA.getChecker<DependenceAnalysis>();
+  if (!PAC.preserved() && !PAC.preservedSet<AllAnalysesOn<Function>>())
+    return true;
+
+  // Check transitive dependencies.
+  return Inv.invalidate<AAManager>(F, PA) ||
+         Inv.invalidate<ScalarEvolutionAnalysis>(F, PA) ||
+         Inv.invalidate<LoopAnalysis>(F, PA);
+}
+
 // depends -
 // Returns NULL if there is no dependence.
 // Otherwise, return a Dependence with as many details as possible.
@@ -3503,7 +3522,7 @@ DependenceInfo::depends(Instruction *Src, Instruction *Dst,
   // to either Separable or Coupled).
   //
   // Next, we consider 1 and 2. The intersection of the GroupLoops is empty.
-  // Next, 1 and 3. The intersectionof their GroupLoops = {2}, not empty,
+  // Next, 1 and 3. The intersection of their GroupLoops = {2}, not empty,
   // so Pair[3].Group = {0, 1, 3} and Done = false.
   //
   // Next, we compare 2 against 3. The intersection of the GroupLoops is empty.
diff --git a/lib/Analysis/DivergenceAnalysis.cpp b/lib/Analysis/DivergenceAnalysis.cpp
index 7ba23854a3cc53efb3fdd68d7ca4ea78bf87a0fc..0ccd59ef2bfd46134d79a3c855125862847383d3 100644
--- a/lib/Analysis/DivergenceAnalysis.cpp
+++ b/lib/Analysis/DivergenceAnalysis.cpp
@@ -1,9 +1,8 @@
 //===- DivergenceAnalysis.cpp --------- Divergence Analysis Implementation -==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Analysis/DomPrinter.cpp b/lib/Analysis/DomPrinter.cpp
index 8abc0e7d0df957fbc74375f109023645a60ce2d0..d9f43dd746efb7c2c7a4fcaf8207fb087489ad87 100644
--- a/lib/Analysis/DomPrinter.cpp
+++ b/lib/Analysis/DomPrinter.cpp
@@ -1,9 +1,8 @@
 //===- DomPrinter.cpp - DOT printer for the dominance trees    ------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/IR/DomTreeUpdater.cpp b/lib/Analysis/DomTreeUpdater.cpp
similarity index 98%
rename from lib/IR/DomTreeUpdater.cpp
rename to lib/Analysis/DomTreeUpdater.cpp
index b72c1b77c2ce06507164a7df71c98124f4431c55..e4d505b8f1ad9515c2272ec0f27dd91cc1630425 100644
--- a/lib/IR/DomTreeUpdater.cpp
+++ b/lib/Analysis/DomTreeUpdater.cpp
@@ -1,9 +1,8 @@
 //===- DomTreeUpdater.cpp - DomTree/Post DomTree Updater --------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -12,7 +11,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "llvm/IR/DomTreeUpdater.h"
+#include "llvm/Analysis/DomTreeUpdater.h"
 #include "llvm/Analysis/PostDominators.h"
 #include "llvm/IR/Dominators.h"
 #include "llvm/Support/GenericDomTree.h"
diff --git a/lib/Analysis/DominanceFrontier.cpp b/lib/Analysis/DominanceFrontier.cpp
index de7f62cf4ecd7ef9bb1f862baf4f0e5b1e69d48f..f9a554acb7ea36818b48f60ecb7b3b764b11423f 100644
--- a/lib/Analysis/DominanceFrontier.cpp
+++ b/lib/Analysis/DominanceFrontier.cpp
@@ -1,9 +1,8 @@
 //===- DominanceFrontier.cpp - Dominance Frontier Calculation -------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Analysis/EHPersonalities.cpp b/lib/Analysis/EHPersonalities.cpp
index 0df73aeebbdc2f0eea5fef145a443ace1456bda2..2242541696a44b3fb04f2ee9ba935700f2fbe5f1 100644
--- a/lib/Analysis/EHPersonalities.cpp
+++ b/lib/Analysis/EHPersonalities.cpp
@@ -1,9 +1,8 @@
 //===- EHPersonalities.cpp - Compute EH-related information ---------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Analysis/GlobalsModRef.cpp b/lib/Analysis/GlobalsModRef.cpp
index 2c503609d96bc961dcdc156c10b06deb3b0c62da..4649b30cd1f5a7dd72315206cf67d9e96669b8a2 100644
--- a/lib/Analysis/GlobalsModRef.cpp
+++ b/lib/Analysis/GlobalsModRef.cpp
@@ -1,9 +1,8 @@
 //===- GlobalsModRef.cpp - Simple Mod/Ref Analysis for Globals ------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -255,11 +254,11 @@ FunctionModRefBehavior GlobalsAAResult::getModRefBehavior(const Function *F) {
 }
 
 FunctionModRefBehavior
-GlobalsAAResult::getModRefBehavior(ImmutableCallSite CS) {
+GlobalsAAResult::getModRefBehavior(const CallBase *Call) {
   FunctionModRefBehavior Min = FMRB_UnknownModRefBehavior;
 
-  if (!CS.hasOperandBundles())
-    if (const Function *F = CS.getCalledFunction())
+  if (!Call->hasOperandBundles())
+    if (const Function *F = Call->getCalledFunction())
       if (FunctionInfo *FI = getFunctionInfo(F)) {
         if (!isModOrRefSet(FI->getModRefInfo()))
           Min = FMRB_DoesNotAccessMemory;
@@ -267,7 +266,7 @@ GlobalsAAResult::getModRefBehavior(ImmutableCallSite CS) {
           Min = FMRB_OnlyReadsMemory;
       }
 
-  return FunctionModRefBehavior(AAResultBase::getModRefBehavior(CS) & Min);
+  return FunctionModRefBehavior(AAResultBase::getModRefBehavior(Call) & Min);
 }
 
 /// Returns the function info for the function, or null if we don't have
@@ -366,14 +365,14 @@ bool GlobalsAAResult::AnalyzeUsesOfPointer(Value *V,
     } else if (Operator::getOpcode(I) == Instruction::BitCast) {
       if (AnalyzeUsesOfPointer(I, Readers, Writers, OkayStoreDest))
         return true;
-    } else if (auto CS = CallSite(I)) {
+    } else if (auto *Call = dyn_cast<CallBase>(I)) {
       // Make sure that this is just the function being called, not that it is
       // passing into the function.
-      if (CS.isDataOperand(&U)) {
+      if (Call->isDataOperand(&U)) {
         // Detect calls to free.
-        if (CS.isArgOperand(&U) && isFreeCall(I, &TLI)) {
+        if (Call->isArgOperand(&U) && isFreeCall(I, &TLI)) {
           if (Writers)
-            Writers->insert(CS->getParent()->getParent());
+            Writers->insert(Call->getParent()->getParent());
         } else {
           return true; // Argument of an unknown call.
         }
@@ -576,15 +575,15 @@ void GlobalsAAResult::AnalyzeCallGraph(CallGraph &CG, Module &M) {
 
         // We handle calls specially because the graph-relevant aspects are
         // handled above.
-        if (auto CS = CallSite(&I)) {
-          if (isAllocationFn(&I, &TLI) || isFreeCall(&I, &TLI)) {
+        if (auto *Call = dyn_cast<CallBase>(&I)) {
+          if (isAllocationFn(Call, &TLI) || isFreeCall(Call, &TLI)) {
             // FIXME: It is completely unclear why this is necessary and not
             // handled by the above graph code.
             FI.addModRefInfo(ModRefInfo::ModRef);
-          } else if (Function *Callee = CS.getCalledFunction()) {
+          } else if (Function *Callee = Call->getCalledFunction()) {
             // The callgraph doesn't include intrinsic calls.
             if (Callee->isIntrinsic()) {
-              if (isa<DbgInfoIntrinsic>(I))
+              if (isa<DbgInfoIntrinsic>(Call))
                 // Don't let dbg intrinsics affect alias info.
                 continue;
 
@@ -597,7 +596,7 @@ void GlobalsAAResult::AnalyzeCallGraph(CallGraph &CG, Module &M) {
         }
 
         // All non-call instructions we use the primary predicates for whether
-        // thay read or write memory.
+        // they read or write memory.
         if (I.mayReadFromMemory())
           FI.addModRefInfo(ModRefInfo::Ref);
         if (I.mayWriteToMemory())
@@ -791,10 +790,10 @@ bool GlobalsAAResult::isNonEscapingGlobalNoAlias(const GlobalValue *GV,
     }
 
     // FIXME: It would be good to handle other obvious no-alias cases here, but
-    // it isn't clear how to do so reasonbly without building a small version
+    // it isn't clear how to do so reasonably without building a small version
     // of BasicAA into this code. We could recurse into AAResultBase::alias
     // here but that seems likely to go poorly as we're inside the
-    // implementation of such a query. Until then, just conservatievly retun
+    // implementation of such a query. Until then, just conservatively return
     // false.
     return false;
   } while (!Inputs.empty());
@@ -885,16 +884,16 @@ AliasResult GlobalsAAResult::alias(const MemoryLocation &LocA,
   return AAResultBase::alias(LocA, LocB);
 }
 
-ModRefInfo GlobalsAAResult::getModRefInfoForArgument(ImmutableCallSite CS,
+ModRefInfo GlobalsAAResult::getModRefInfoForArgument(const CallBase *Call,
                                                      const GlobalValue *GV) {
-  if (CS.doesNotAccessMemory())
+  if (Call->doesNotAccessMemory())
     return ModRefInfo::NoModRef;
   ModRefInfo ConservativeResult =
-      CS.onlyReadsMemory() ? ModRefInfo::Ref : ModRefInfo::ModRef;
+      Call->onlyReadsMemory() ? ModRefInfo::Ref : ModRefInfo::ModRef;
 
   // Iterate through all the arguments to the called function. If any argument
   // is based on GV, return the conservative result.
-  for (auto &A : CS.args()) {
+  for (auto &A : Call->args()) {
     SmallVector<Value*, 4> Objects;
     GetUnderlyingObjects(A, Objects, DL);
 
@@ -914,7 +913,7 @@ ModRefInfo GlobalsAAResult::getModRefInfoForArgument(ImmutableCallSite CS,
   return ModRefInfo::NoModRef;
 }
 
-ModRefInfo GlobalsAAResult::getModRefInfo(ImmutableCallSite CS,
+ModRefInfo GlobalsAAResult::getModRefInfo(const CallBase *Call,
                                           const MemoryLocation &Loc) {
   ModRefInfo Known = ModRefInfo::ModRef;
 
@@ -923,15 +922,15 @@ ModRefInfo GlobalsAAResult::getModRefInfo(ImmutableCallSite CS,
   if (const GlobalValue *GV =
           dyn_cast<GlobalValue>(GetUnderlyingObject(Loc.Ptr, DL)))
     if (GV->hasLocalLinkage())
-      if (const Function *F = CS.getCalledFunction())
+      if (const Function *F = Call->getCalledFunction())
         if (NonAddressTakenGlobals.count(GV))
           if (const FunctionInfo *FI = getFunctionInfo(F))
             Known = unionModRef(FI->getModRefInfoForGlobal(*GV),
-                                getModRefInfoForArgument(CS, GV));
+                                getModRefInfoForArgument(Call, GV));
 
   if (!isModOrRefSet(Known))
     return ModRefInfo::NoModRef; // No need to query other mod/ref analyses
-  return intersectModRef(Known, AAResultBase::getModRefInfo(CS, Loc));
+  return intersectModRef(Known, AAResultBase::getModRefInfo(Call, Loc));
 }
 
 GlobalsAAResult::GlobalsAAResult(const DataLayout &DL,
diff --git a/lib/Analysis/GuardUtils.cpp b/lib/Analysis/GuardUtils.cpp
index 08fa6abeafb527dea417740ee795142b1d5eba52..c099ee8e41c66efcfd9a4a380969d519a968adb4 100644
--- a/lib/Analysis/GuardUtils.cpp
+++ b/lib/Analysis/GuardUtils.cpp
@@ -1,9 +1,8 @@
 //===-- GuardUtils.cpp - Utils for work with guards -------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 // Utils that are used to perform analyzes related to guards and their
@@ -19,3 +18,27 @@ bool llvm::isGuard(const User *U) {
   using namespace llvm::PatternMatch;
   return match(U, m_Intrinsic<Intrinsic::experimental_guard>());
 }
+
+bool llvm::isGuardAsWidenableBranch(const User *U) {
+  Value *Condition, *WidenableCondition;
+  BasicBlock *GuardedBB, *DeoptBB;
+  if (!parseWidenableBranch(U, Condition, WidenableCondition, GuardedBB,
+                            DeoptBB))
+    return false;
+  using namespace llvm::PatternMatch;
+  for (auto &Insn : *DeoptBB) {
+    if (match(&Insn, m_Intrinsic<Intrinsic::experimental_deoptimize>()))
+      return true;
+    if (Insn.mayHaveSideEffects())
+      return false;
+  }
+  return false;
+}
+
+bool llvm::parseWidenableBranch(const User *U, Value *&Condition,
+                                Value *&WidenableCondition,
+                                BasicBlock *&IfTrueBB, BasicBlock *&IfFalseBB) {
+  using namespace llvm::PatternMatch;
+  return match(U, m_Br(m_And(m_Value(Condition), m_Value(WidenableCondition)),
+                       IfTrueBB, IfFalseBB));
+}
diff --git a/lib/Analysis/IVDescriptors.cpp b/lib/Analysis/IVDescriptors.cpp
index aaebc4a481ec5db351a12432f313a44356774859..a452a52b94de75e7aa0b82efb7aa41b56aee5d57 100644
--- a/lib/Analysis/IVDescriptors.cpp
+++ b/lib/Analysis/IVDescriptors.cpp
@@ -1,9 +1,8 @@
 //===- llvm/Analysis/IVDescriptors.cpp - IndVar Descriptors -----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -15,6 +14,7 @@
 #include "llvm/ADT/ScopeExit.h"
 #include "llvm/Analysis/AliasAnalysis.h"
 #include "llvm/Analysis/BasicAliasAnalysis.h"
+#include "llvm/Analysis/DomTreeUpdater.h"
 #include "llvm/Analysis/GlobalsModRef.h"
 #include "llvm/Analysis/InstructionSimplify.h"
 #include "llvm/Analysis/LoopInfo.h"
@@ -26,7 +26,6 @@
 #include "llvm/Analysis/ScalarEvolutionExpressions.h"
 #include "llvm/Analysis/TargetTransformInfo.h"
 #include "llvm/Analysis/ValueTracking.h"
-#include "llvm/IR/DomTreeUpdater.h"
 #include "llvm/IR/Dominators.h"
 #include "llvm/IR/Instructions.h"
 #include "llvm/IR/Module.h"
@@ -1010,7 +1009,7 @@ bool InductionDescriptor::isInductionPHI(PHINode *Phi, const Loop *TheLoop,
   // If we started from an UnknownSCEV, and managed to build an addRecurrence
   // only after enabling Assume with PSCEV, this means we may have encountered
   // cast instructions that required adding a runtime check in order to
-  // guarantee the correctness of the AddRecurence respresentation of the
+  // guarantee the correctness of the AddRecurrence respresentation of the
   // induction.
   if (PhiScev != AR && SymbolicPhi) {
     SmallVector<Instruction *, 2> Casts;
diff --git a/lib/Analysis/IVUsers.cpp b/lib/Analysis/IVUsers.cpp
index 609e5e3a1448bdd482d6663d6f9af3ce7c022dde..681a0cf7e981492a0625ee9bc17a812d214475b5 100644
--- a/lib/Analysis/IVUsers.cpp
+++ b/lib/Analysis/IVUsers.cpp
@@ -1,9 +1,8 @@
 //===- IVUsers.cpp - Induction Variable Users -------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Analysis/IndirectCallPromotionAnalysis.cpp b/lib/Analysis/IndirectCallPromotionAnalysis.cpp
index 4659c0a00629cb4512aa681e009c7636f13d42db..6ff840efcb642df4e37299be10a296a79ed8f1b4 100644
--- a/lib/Analysis/IndirectCallPromotionAnalysis.cpp
+++ b/lib/Analysis/IndirectCallPromotionAnalysis.cpp
@@ -1,9 +1,8 @@
 //===-- IndirectCallPromotionAnalysis.cpp - Find promotion candidates ===//
 //
-//                      The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -15,7 +14,7 @@
 
 #include "llvm/Analysis/IndirectCallPromotionAnalysis.h"
 #include "llvm/ADT/STLExtras.h"
-#include "llvm/Analysis/IndirectCallSiteVisitor.h"
+#include "llvm/Analysis/IndirectCallVisitor.h"
 #include "llvm/IR/CallSite.h"
 #include "llvm/IR/InstIterator.h"
 #include "llvm/IR/InstVisitor.h"
diff --git a/lib/Analysis/InlineCost.cpp b/lib/Analysis/InlineCost.cpp
index a3347dbcb93350dbe9d6519bf4a96f900712e09c..574a1b6c8410a8d4ec29fd6edc877e374cea354a 100644
--- a/lib/Analysis/InlineCost.cpp
+++ b/lib/Analysis/InlineCost.cpp
@@ -1,9 +1,8 @@
 //===- InlineCost.cpp - Cost analysis for inliner -------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -51,19 +50,19 @@ static cl::opt<int> InlineThreshold(
     cl::desc("Control the amount of inlining to perform (default = 225)"));
 
 static cl::opt<int> HintThreshold(
-    "inlinehint-threshold", cl::Hidden, cl::init(325),
+    "inlinehint-threshold", cl::Hidden, cl::init(325), cl::ZeroOrMore, 
     cl::desc("Threshold for inlining functions with inline hint"));
 
 static cl::opt<int>
     ColdCallSiteThreshold("inline-cold-callsite-threshold", cl::Hidden,
-                          cl::init(45),
+                          cl::init(45), cl::ZeroOrMore,
                           cl::desc("Threshold for inlining cold callsites"));
 
 // We introduce this threshold to help performance of instrumentation based
 // PGO before we actually hook up inliner with analysis passes such as BPI and
 // BFI.
 static cl::opt<int> ColdThreshold(
-    "inlinecold-threshold", cl::Hidden, cl::init(45),
+    "inlinecold-threshold", cl::Hidden, cl::init(45), cl::ZeroOrMore, 
     cl::desc("Threshold for inlining functions with cold attribute"));
 
 static cl::opt<int>
@@ -77,7 +76,7 @@ static cl::opt<int> LocallyHotCallSiteThreshold(
 
 static cl::opt<int> ColdCallSiteRelFreq(
     "cold-callsite-rel-freq", cl::Hidden, cl::init(2), cl::ZeroOrMore,
-    cl::desc("Maxmimum block frequency, expressed as a percentage of caller's "
+    cl::desc("Maximum block frequency, expressed as a percentage of caller's "
              "entry frequency, for a callsite to be cold in the absence of "
              "profile information."));
 
@@ -88,7 +87,7 @@ static cl::opt<int> HotCallSiteRelFreq(
              "profile information."));
 
 static cl::opt<bool> OptComputeFullInlineCost(
-    "inline-cost-full", cl::Hidden, cl::init(false),
+    "inline-cost-full", cl::Hidden, cl::init(false), cl::ZeroOrMore,
     cl::desc("Compute the full inline cost of a call site even when the cost "
              "exceeds the threshold."));
 
@@ -1178,7 +1177,7 @@ bool CallAnalyzer::simplifyCallSite(Function *F, CallSite CS) {
   // because we have to continually rebuild the argument list even when no
   // simplifications can be performed. Until that is fixed with remapping
   // inside of instsimplify, directly constant fold calls here.
-  if (!canConstantFoldCallTo(CS, F))
+  if (!canConstantFoldCallTo(cast<CallBase>(CS.getInstruction()), F))
     return false;
 
   // Try to re-map the arguments to constants.
@@ -1194,7 +1193,8 @@ bool CallAnalyzer::simplifyCallSite(Function *F, CallSite CS) {
 
     ConstantArgs.push_back(C);
   }
-  if (Constant *C = ConstantFoldCall(CS, F, ConstantArgs)) {
+  if (Constant *C = ConstantFoldCall(cast<CallBase>(CS.getInstruction()), F,
+                                     ConstantArgs)) {
     SimplifiedValues[CS.getInstruction()] = C;
     return true;
   }
@@ -1676,7 +1676,7 @@ ConstantInt *CallAnalyzer::stripAndComputeInBoundsConstantOffsets(Value *&V) {
 /// blocks to see if all their incoming edges are dead or not.
 void CallAnalyzer::findDeadBlocks(BasicBlock *CurrBB, BasicBlock *NextBB) {
   auto IsEdgeDead = [&](BasicBlock *Pred, BasicBlock *Succ) {
-    // A CFG edge is dead if the predecessor is dead or the predessor has a
+    // A CFG edge is dead if the predecessor is dead or the predecessor has a
     // known successor which is not the one under exam.
     return (DeadBlocks.count(Pred) ||
             (KnownSuccessors[Pred] && KnownSuccessors[Pred] != Succ));
@@ -1731,6 +1731,13 @@ InlineResult CallAnalyzer::analyzeCall(CallSite CS) {
   // Update the threshold based on callsite properties
   updateThreshold(CS, F);
 
+  // While Threshold depends on commandline options that can take negative
+  // values, we want to enforce the invariant that the computed threshold and
+  // bonuses are non-negative.
+  assert(Threshold >= 0);
+  assert(SingleBBBonus >= 0);
+  assert(VectorBonus >= 0);
+
   // Speculatively apply all possible bonuses to Threshold. If cost exceeds
   // this Threshold any time, and cost cannot decrease, we can stop processing
   // the rest of the function body.
@@ -2016,9 +2023,10 @@ InlineCost llvm::getInlineCost(
   // Calls to functions with always-inline attributes should be inlined
   // whenever possible.
   if (CS.hasFnAttr(Attribute::AlwaysInline)) {
-    if (isInlineViable(*Callee))
+    auto IsViable = isInlineViable(*Callee);
+    if (IsViable)
       return llvm::InlineCost::getAlways("always inline attribute");
-    return llvm::InlineCost::getNever("inapplicable always inline attribute");
+    return llvm::InlineCost::getNever(IsViable.message);
   }
 
   // Never inline functions with conflicting attributes (unless callee has
@@ -2066,13 +2074,16 @@ InlineCost llvm::getInlineCost(
   return llvm::InlineCost::get(CA.getCost(), CA.getThreshold());
 }
 
-bool llvm::isInlineViable(Function &F) {
+InlineResult llvm::isInlineViable(Function &F) {
   bool ReturnsTwice = F.hasFnAttribute(Attribute::ReturnsTwice);
   for (Function::iterator BI = F.begin(), BE = F.end(); BI != BE; ++BI) {
     // Disallow inlining of functions which contain indirect branches or
     // blockaddresses.
-    if (isa<IndirectBrInst>(BI->getTerminator()) || BI->hasAddressTaken())
-      return false;
+    if (isa<IndirectBrInst>(BI->getTerminator()))
+      return "contains indirect branches";
+
+    if (BI->hasAddressTaken())
+      return "uses block address";
 
     for (auto &II : *BI) {
       CallSite CS(&II);
@@ -2081,13 +2092,13 @@ bool llvm::isInlineViable(Function &F) {
 
       // Disallow recursive calls.
       if (&F == CS.getCalledFunction())
-        return false;
+        return "recursive call";
 
       // Disallow calls which expose returns-twice to a function not previously
       // attributed as such.
       if (!ReturnsTwice && CS.isCall() &&
           cast<CallInst>(CS.getInstruction())->canReturnTwice())
-        return false;
+        return "exposes returns-twice attribute";
 
       if (CS.getCalledFunction())
         switch (CS.getCalledFunction()->getIntrinsicID()) {
@@ -2096,12 +2107,14 @@ bool llvm::isInlineViable(Function &F) {
         // Disallow inlining of @llvm.icall.branch.funnel because current
         // backend can't separate call targets from call arguments.
         case llvm::Intrinsic::icall_branch_funnel:
+          return "disallowed inlining of @llvm.icall.branch.funnel";
         // Disallow inlining functions that call @llvm.localescape. Doing this
         // correctly would require major changes to the inliner.
         case llvm::Intrinsic::localescape:
+          return "disallowed inlining of @llvm.localescape";
         // Disallow inlining of functions that initialize VarArgs with va_start.
         case llvm::Intrinsic::vastart:
-          return false;
+          return "contains VarArgs initialized with va_start";
         }
     }
   }
diff --git a/lib/Analysis/InstCount.cpp b/lib/Analysis/InstCount.cpp
index 95ab6ee3db5bd4714bc48afba87ebca9b01ca88a..943a99a5f46dec3916dab791367609a5ecbacda2 100644
--- a/lib/Analysis/InstCount.cpp
+++ b/lib/Analysis/InstCount.cpp
@@ -1,9 +1,8 @@
 //===-- InstCount.cpp - Collects the count of all instructions ------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Analysis/InstructionPrecedenceTracking.cpp b/lib/Analysis/InstructionPrecedenceTracking.cpp
index b98975b006a445bd1f0f255f1e560294b3e26fa1..0afce7ebbd311862e2d026df9d9031558e2ca458 100644
--- a/lib/Analysis/InstructionPrecedenceTracking.cpp
+++ b/lib/Analysis/InstructionPrecedenceTracking.cpp
@@ -1,9 +1,8 @@
 //===-- InstructionPrecedenceTracking.cpp -----------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 // Implements a class that is able to define some instructions as "special"
@@ -99,9 +98,17 @@ void InstructionPrecedenceTracking::validateAll() const {
 }
 #endif
 
-void InstructionPrecedenceTracking::invalidateBlock(const BasicBlock *BB) {
+void InstructionPrecedenceTracking::insertInstructionTo(const Instruction *Inst,
+                                                        const BasicBlock *BB) {
+  if (isSpecialInstruction(Inst))
+    FirstSpecialInsts.erase(BB);
   OI.invalidateBlock(BB);
-  FirstSpecialInsts.erase(BB);
+}
+
+void InstructionPrecedenceTracking::removeInstruction(const Instruction *Inst) {
+  if (isSpecialInstruction(Inst))
+    FirstSpecialInsts.erase(Inst->getParent());
+  OI.invalidateBlock(Inst->getParent());
 }
 
 void InstructionPrecedenceTracking::clear() {
diff --git a/lib/Analysis/InstructionSimplify.cpp b/lib/Analysis/InstructionSimplify.cpp
index ccf907c144f0d1a7535e667a1f788a819b0122aa..843b3e97c0767501c653bb142f0aabc776b8c5df 100644
--- a/lib/Analysis/InstructionSimplify.cpp
+++ b/lib/Analysis/InstructionSimplify.cpp
@@ -1,9 +1,8 @@
 //===- InstructionSimplify.cpp - Fold instruction operands ----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -34,6 +33,8 @@
 #include "llvm/IR/Dominators.h"
 #include "llvm/IR/GetElementPtrTypeIterator.h"
 #include "llvm/IR/GlobalAlias.h"
+#include "llvm/IR/InstrTypes.h"
+#include "llvm/IR/Instructions.h"
 #include "llvm/IR/Operator.h"
 #include "llvm/IR/PatternMatch.h"
 #include "llvm/IR/ValueHandle.h"
@@ -672,8 +673,8 @@ static Constant *stripAndComputeConstantOffsets(const DataLayout &DL, Value *&V,
         break;
       V = GA->getAliasee();
     } else {
-      if (auto CS = CallSite(V))
-        if (Value *RV = CS.getReturnedArgOperand()) {
+      if (auto *Call = dyn_cast<CallBase>(V))
+        if (Value *RV = Call->getReturnedArgOperand()) {
           V = RV;
           continue;
         }
@@ -4941,7 +4942,15 @@ static Value *simplifyUnaryIntrinsic(Function *F, Value *Op0,
   case Intrinsic::log2:
     // log2(exp2(x)) -> x
     if (Q.CxtI->hasAllowReassoc() &&
-        match(Op0, m_Intrinsic<Intrinsic::exp2>(m_Value(X)))) return X;
+        (match(Op0, m_Intrinsic<Intrinsic::exp2>(m_Value(X))) ||
+         match(Op0, m_Intrinsic<Intrinsic::pow>(m_SpecificFP(2.0),
+                                                m_Value(X))))) return X;
+    break;
+  case Intrinsic::log10:
+    // log10(pow(10.0, x)) -> x
+    if (Q.CxtI->hasAllowReassoc() &&
+        match(Op0, m_Intrinsic<Intrinsic::pow>(m_SpecificFP(10.0),
+                                               m_Value(X)))) return X;
     break;
   default:
     break;
@@ -5138,7 +5147,7 @@ static Value *simplifyIntrinsic(Function *F, IterTy ArgBegin, IterTy ArgEnd,
 }
 
 template <typename IterTy>
-static Value *SimplifyCall(ImmutableCallSite CS, Value *V, IterTy ArgBegin,
+static Value *SimplifyCall(CallBase *Call, Value *V, IterTy ArgBegin,
                            IterTy ArgEnd, const SimplifyQuery &Q,
                            unsigned MaxRecurse) {
   Type *Ty = V->getType();
@@ -5159,7 +5168,7 @@ static Value *SimplifyCall(ImmutableCallSite CS, Value *V, IterTy ArgBegin,
     if (Value *Ret = simplifyIntrinsic(F, ArgBegin, ArgEnd, Q))
       return Ret;
 
-  if (!canConstantFoldCallTo(CS, F))
+  if (!canConstantFoldCallTo(Call, F))
     return nullptr;
 
   SmallVector<Constant *, 4> ConstantArgs;
@@ -5171,24 +5180,22 @@ static Value *SimplifyCall(ImmutableCallSite CS, Value *V, IterTy ArgBegin,
     ConstantArgs.push_back(C);
   }
 
-  return ConstantFoldCall(CS, F, ConstantArgs, Q.TLI);
+  return ConstantFoldCall(Call, F, ConstantArgs, Q.TLI);
 }
 
-Value *llvm::SimplifyCall(ImmutableCallSite CS, Value *V,
-                          User::op_iterator ArgBegin, User::op_iterator ArgEnd,
-                          const SimplifyQuery &Q) {
-  return ::SimplifyCall(CS, V, ArgBegin, ArgEnd, Q, RecursionLimit);
+Value *llvm::SimplifyCall(CallBase *Call, Value *V, User::op_iterator ArgBegin,
+                          User::op_iterator ArgEnd, const SimplifyQuery &Q) {
+  return ::SimplifyCall(Call, V, ArgBegin, ArgEnd, Q, RecursionLimit);
 }
 
-Value *llvm::SimplifyCall(ImmutableCallSite CS, Value *V,
-                          ArrayRef<Value *> Args, const SimplifyQuery &Q) {
-  return ::SimplifyCall(CS, V, Args.begin(), Args.end(), Q, RecursionLimit);
+Value *llvm::SimplifyCall(CallBase *Call, Value *V, ArrayRef<Value *> Args,
+                          const SimplifyQuery &Q) {
+  return ::SimplifyCall(Call, V, Args.begin(), Args.end(), Q, RecursionLimit);
 }
 
-Value *llvm::SimplifyCall(ImmutableCallSite ICS, const SimplifyQuery &Q) {
-  CallSite CS(const_cast<Instruction*>(ICS.getInstruction()));
-  return ::SimplifyCall(CS, CS.getCalledValue(), CS.arg_begin(), CS.arg_end(),
-                        Q, RecursionLimit);
+Value *llvm::SimplifyCall(CallBase *Call, const SimplifyQuery &Q) {
+  return ::SimplifyCall(Call, Call->getCalledValue(), Call->arg_begin(),
+                        Call->arg_end(), Q, RecursionLimit);
 }
 
 /// See if we can compute a simplified version of this instruction.
@@ -5327,8 +5334,7 @@ Value *llvm::SimplifyInstruction(Instruction *I, const SimplifyQuery &SQ,
     Result = SimplifyPHINode(cast<PHINode>(I), Q);
     break;
   case Instruction::Call: {
-    CallSite CS(cast<CallInst>(I));
-    Result = SimplifyCall(CS, Q);
+    Result = SimplifyCall(cast<CallInst>(I), Q);
     break;
   }
 #define HANDLE_CAST_INST(num, opc, clas) case Instruction::opc:
diff --git a/lib/Analysis/Interval.cpp b/lib/Analysis/Interval.cpp
index 6d5de22cb93f52ac746a82644bbe0455186961d1..07d6e27c13befc2a34cc679eb2b6047eef9f2995 100644
--- a/lib/Analysis/Interval.cpp
+++ b/lib/Analysis/Interval.cpp
@@ -1,9 +1,8 @@
 //===- Interval.cpp - Interval class code ---------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Analysis/IntervalPartition.cpp b/lib/Analysis/IntervalPartition.cpp
index c777d91b67c6262e0f656700bc6c990881129604..d12db010db6aa097c8758743f70e2b6b7ea1a9ad 100644
--- a/lib/Analysis/IntervalPartition.cpp
+++ b/lib/Analysis/IntervalPartition.cpp
@@ -1,9 +1,8 @@
 //===- IntervalPartition.cpp - Interval Partition module code -------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Analysis/IteratedDominanceFrontier.cpp b/lib/Analysis/IteratedDominanceFrontier.cpp
index 000fe5ddad540f665c8c17b219fb700292f6954d..307bb29db15b84ca32ecdccdc9f42134b8f09d07 100644
--- a/lib/Analysis/IteratedDominanceFrontier.cpp
+++ b/lib/Analysis/IteratedDominanceFrontier.cpp
@@ -1,9 +1,8 @@
 //===- IteratedDominanceFrontier.cpp - Compute IDF ------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Analysis/LLVMBuild.txt b/lib/Analysis/LLVMBuild.txt
index 8a87b980b0a8964024e1a537116a7118b88808b2..d73b55f037fa3b31e2d498fa5022bfb3a5577710 100644
--- a/lib/Analysis/LLVMBuild.txt
+++ b/lib/Analysis/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/Analysis/LLVMBuild.txt -----------------------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Analysis/LazyBlockFrequencyInfo.cpp b/lib/Analysis/LazyBlockFrequencyInfo.cpp
index 93c23bca96af7ceb4ce1e2c33717dcc454d51417..439758560284171e008fbe7e66f0f4b2666364dd 100644
--- a/lib/Analysis/LazyBlockFrequencyInfo.cpp
+++ b/lib/Analysis/LazyBlockFrequencyInfo.cpp
@@ -1,9 +1,8 @@
 //===- LazyBlockFrequencyInfo.cpp - Lazy Block Frequency Analysis ---------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Analysis/LazyBranchProbabilityInfo.cpp b/lib/Analysis/LazyBranchProbabilityInfo.cpp
index 429b78c3a47e598bfa37791b6e14adc528305ec0..f2592c26b3736ccdc35aa2cbbbf10ac2195e2d5f 100644
--- a/lib/Analysis/LazyBranchProbabilityInfo.cpp
+++ b/lib/Analysis/LazyBranchProbabilityInfo.cpp
@@ -1,9 +1,8 @@
 //===- LazyBranchProbabilityInfo.cpp - Lazy Branch Probability Analysis ---===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Analysis/LazyCallGraph.cpp b/lib/Analysis/LazyCallGraph.cpp
index 3f22ada803c9f404acce8709e32c92cc480de944..726f52a95855d9b8a5bcfca2519ed669ff8e0011 100644
--- a/lib/Analysis/LazyCallGraph.cpp
+++ b/lib/Analysis/LazyCallGraph.cpp
@@ -1,9 +1,8 @@
 //===- LazyCallGraph.cpp - Analysis of a Module's call graph --------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Analysis/LazyValueInfo.cpp b/lib/Analysis/LazyValueInfo.cpp
index 110c085d3f353224dac76da6543b969797a6b2f5..4e915be303a404273772969a09a68d637f9bc20e 100644
--- a/lib/Analysis/LazyValueInfo.cpp
+++ b/lib/Analysis/LazyValueInfo.cpp
@@ -1,9 +1,8 @@
 //===- LazyValueInfo.cpp - Value constraint analysis ------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -625,7 +624,7 @@ bool LazyValueInfoImpl::solveBlockValueImpl(ValueLatticeElement &Res,
   // and the like to prove non-nullness, but it's not clear that's worth it
   // compile time wise.  The context-insensitive value walk done inside
   // isKnownNonZero gets most of the profitable cases at much less expense.
-  // This does mean that we have a sensativity to where the defining
+  // This does mean that we have a sensitivity to where the defining
   // instruction is placed, even if it could legally be hoisted much higher.
   // That is unfortunate.
   PointerType *PT = dyn_cast<PointerType>(BBI->getType());
@@ -824,7 +823,9 @@ void LazyValueInfoImpl::intersectAssumeOrGuardBlockValueConstantRange(
   if (!GuardDecl || GuardDecl->use_empty())
     return;
 
-  for (Instruction &I : make_range(BBI->getIterator().getReverse(),
+  if (BBI->getIterator() == BBI->getParent()->begin())
+    return;
+  for (Instruction &I : make_range(std::next(BBI->getIterator().getReverse()),
                                    BBI->getParent()->rend())) {
     Value *Cond = nullptr;
     if (match(&I, m_Intrinsic<Intrinsic::experimental_guard>(m_Value(Cond))))
diff --git a/lib/Analysis/LegacyDivergenceAnalysis.cpp b/lib/Analysis/LegacyDivergenceAnalysis.cpp
index 5540859ebddae4e6395aabde4458b58874d920fa..52212e1c42aa6ab2393abb8dc99a68b76f00df9e 100644
--- a/lib/Analysis/LegacyDivergenceAnalysis.cpp
+++ b/lib/Analysis/LegacyDivergenceAnalysis.cpp
@@ -1,10 +1,9 @@
 //===- LegacyDivergenceAnalysis.cpp --------- Legacy Divergence Analysis
 //Implementation -==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Analysis/Lint.cpp b/lib/Analysis/Lint.cpp
index db919bd233bf89e547c9c05721027ec143c9dc64..37ce56d268b47f50562ce2c9c905b1a1c7b37e09 100644
--- a/lib/Analysis/Lint.cpp
+++ b/lib/Analysis/Lint.cpp
@@ -1,9 +1,8 @@
 //===-- Lint.cpp - Check for common errors in LLVM IR ---------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -330,12 +329,12 @@ void Lint::visitCallSite(CallSite CS) {
       // Check that the memcpy arguments don't overlap. The AliasAnalysis API
       // isn't expressive enough for what we really want to do. Known partial
       // overlap is not distinguished from the case where nothing is known.
-      uint64_t Size = 0;
+      auto Size = LocationSize::unknown();
       if (const ConstantInt *Len =
               dyn_cast<ConstantInt>(findValue(MCI->getLength(),
                                               /*OffsetOk=*/false)))
         if (Len->getValue().isIntN(32))
-          Size = Len->getValue().getZExtValue();
+          Size = LocationSize::precise(Len->getValue().getZExtValue());
       Assert(AA->alias(MCI->getSource(), Size, MCI->getDest(), Size) !=
                  MustAlias,
              "Undefined behavior: memcpy source and destination overlap", &I);
diff --git a/lib/Analysis/Loads.cpp b/lib/Analysis/Loads.cpp
index d319d4c249d33f7162192c9575352e8387a52902..7da9bd718a51cb5d059a66a6a5236f25bbcc8ca5 100644
--- a/lib/Analysis/Loads.cpp
+++ b/lib/Analysis/Loads.cpp
@@ -1,9 +1,8 @@
 //===- Loads.cpp - Local load analysis ------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -107,8 +106,8 @@ static bool isDereferenceableAndAlignedPointer(
     return isDereferenceableAndAlignedPointer(ASC->getOperand(0), Align, Size,
                                               DL, CtxI, DT, Visited);
 
-  if (auto CS = ImmutableCallSite(V))
-    if (auto *RP = getArgumentAliasingToReturnedPointer(CS))
+  if (const auto *Call = dyn_cast<CallBase>(V))
+    if (auto *RP = getArgumentAliasingToReturnedPointer(Call))
       return isDereferenceableAndAlignedPointer(RP, Align, Size, DL, CtxI, DT,
                                                 Visited);
 
@@ -281,9 +280,17 @@ bool llvm::isSafeToLoadUnconditionally(Value *V, unsigned Align,
     Value *AccessedPtr;
     unsigned AccessedAlign;
     if (LoadInst *LI = dyn_cast<LoadInst>(BBI)) {
+      // Ignore volatile loads. The execution of a volatile load cannot
+      // be used to prove an address is backed by regular memory; it can,
+      // for example, point to an MMIO register.
+      if (LI->isVolatile())
+        continue;
       AccessedPtr = LI->getPointerOperand();
       AccessedAlign = LI->getAlignment();
     } else if (StoreInst *SI = dyn_cast<StoreInst>(BBI)) {
+      // Ignore volatile stores (see comment for loads).
+      if (SI->isVolatile())
+        continue;
       AccessedPtr = SI->getPointerOperand();
       AccessedAlign = SI->getAlignment();
     } else
@@ -345,7 +352,7 @@ Value *llvm::FindAvailablePtrLoadStore(Value *Ptr, Type *AccessTy,
   const DataLayout &DL = ScanBB->getModule()->getDataLayout();
 
   // Try to get the store size for the type.
-  uint64_t AccessSize = DL.getTypeStoreSize(AccessTy);
+  auto AccessSize = LocationSize::precise(DL.getTypeStoreSize(AccessTy));
 
   Value *StrippedPtr = Ptr->stripPointerCasts();
 
diff --git a/lib/Analysis/LoopAccessAnalysis.cpp b/lib/Analysis/LoopAccessAnalysis.cpp
index 245f318e308a3658ae9ec6077a06a29b4be2c64e..5130807cadd238ee1c96fb9644927839dbbf155e 100644
--- a/lib/Analysis/LoopAccessAnalysis.cpp
+++ b/lib/Analysis/LoopAccessAnalysis.cpp
@@ -1,9 +1,8 @@
 //===- LoopAccessAnalysis.cpp - Loop Access Analysis Implementation --------==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -342,7 +341,7 @@ void RuntimePointerChecking::groupChecks(
   //
   // The above case requires that we have an UnknownDependence between
   // accesses to the same underlying object. This cannot happen unless
-  // ShouldRetryWithRuntimeCheck is set, and therefore UseDependencies
+  // FoundNonConstantDistanceDependence is set, and therefore UseDependencies
   // is also false. In this case we will use the fallback path and create
   // separate checking groups for all pointers.
 
@@ -556,7 +555,7 @@ public:
   /// perform dependency checking.
   ///
   /// Note that this can later be cleared if we retry memcheck analysis without
-  /// dependency checking (i.e. ShouldRetryWithRuntimeCheck).
+  /// dependency checking (i.e. FoundNonConstantDistanceDependence).
   bool isDependencyCheckNeeded() { return !CheckDeps.empty(); }
 
   /// We decided that no dependence analysis would be used.  Reset the state.
@@ -604,8 +603,8 @@ private:
   ///
   /// Note that, this is different from isDependencyCheckNeeded.  When we retry
   /// memcheck analysis without dependency checking
-  /// (i.e. ShouldRetryWithRuntimeCheck), isDependencyCheckNeeded is cleared
-  /// while this remains set if we have potentially dependent accesses.
+  /// (i.e. FoundNonConstantDistanceDependence), isDependencyCheckNeeded is
+  /// cleared while this remains set if we have potentially dependent accesses.
   bool IsRTCheckAnalysisNeeded;
 
   /// The SCEV predicate containing all the SCEV-related assumptions.
@@ -1014,7 +1013,7 @@ int64_t llvm::getPtrStride(PredicatedScalarEvolution &PSE, Value *Ptr,
     return 0;
   }
 
-  // The accesss function must stride over the innermost loop.
+  // The access function must stride over the innermost loop.
   if (Lp != AR->getLoop()) {
     LLVM_DEBUG(dbgs() << "LAA: Bad stride - Not striding over innermost loop "
                       << *Ptr << " SCEV: " << *AR << "\n");
@@ -1086,7 +1085,7 @@ int64_t llvm::getPtrStride(PredicatedScalarEvolution &PSE, Value *Ptr,
     if (Assume) {
       // We can avoid this case by adding a run-time check.
       LLVM_DEBUG(dbgs() << "LAA: Non unit strided pointer which is not either "
-                        << "inbouds or in address space 0 may wrap:\n"
+                        << "inbounds or in address space 0 may wrap:\n"
                         << "LAA:   Pointer: " << *Ptr << "\n"
                         << "LAA:   SCEV: " << *AR << "\n"
                         << "LAA:   Added an overflow assumption\n");
@@ -1230,6 +1229,7 @@ MemoryDepChecker::Dependence::isSafeForVectorization(DepType Type) {
     return VectorizationSafetyStatus::Safe;
 
   case Unknown:
+    return VectorizationSafetyStatus::PossiblySafeWithRtChecks;
   case ForwardButPreventsForwarding:
   case Backward:
   case BackwardVectorizableButPreventsForwarding:
@@ -1345,7 +1345,7 @@ static bool isSafeDependenceDistance(const DataLayout &DL, ScalarEvolution &SE,
   // where Step is the absolute stride of the memory accesses in bytes,
   // then there is no dependence.
   //
-  // Ratioanle:
+  // Rationale:
   // We basically want to check if the absolute distance (|Dist/Step|)
   // is >= the loop iteration count (or > BackedgeTakenCount).
   // This is equivalent to the Strong SIV Test (Practical Dependence Testing,
@@ -1368,7 +1368,7 @@ static bool isSafeDependenceDistance(const DataLayout &DL, ScalarEvolution &SE,
 
   // The dependence distance can be positive/negative, so we sign extend Dist;
   // The multiplication of the absolute stride in bytes and the
-  // backdgeTakenCount is non-negative, so we zero extend Product.
+  // backedgeTakenCount is non-negative, so we zero extend Product.
   if (DistTypeSize > ProductTypeSize)
     CastedProduct = SE.getZeroExtendExpr(Product, Dist.getType());
   else
@@ -1491,7 +1491,7 @@ MemoryDepChecker::isDependent(const MemAccessInfo &A, unsigned AIdx,
       return Dependence::NoDep;
 
     LLVM_DEBUG(dbgs() << "LAA: Dependence because of non-constant distance\n");
-    ShouldRetryWithRuntimeCheck = true;
+    FoundNonConstantDistanceDependence = true;
     return Dependence::Unknown;
   }
 
@@ -2251,7 +2251,7 @@ void LoopAccessInfo::collectStridedAccess(Value *MemAccess) {
 
   // Match the types so we can compare the stride and the BETakenCount.
   // The Stride can be positive/negative, so we sign extend Stride;
-  // The backdgeTakenCount is non-negative, so we zero extend BETakenCount.
+  // The backedgeTakenCount is non-negative, so we zero extend BETakenCount.
   const DataLayout &DL = TheLoop->getHeader()->getModule()->getDataLayout();
   uint64_t StrideTypeSize = DL.getTypeAllocSize(StrideExpr->getType());
   uint64_t BETypeSize = DL.getTypeAllocSize(BETakenCount->getType());
diff --git a/lib/Analysis/LoopAnalysisManager.cpp b/lib/Analysis/LoopAnalysisManager.cpp
index 2a3b29d7fbcaabeed5259ccc537c5aa461457610..8ae92e17399cd0fb1f7bb315dd8adfb6362a6a78 100644
--- a/lib/Analysis/LoopAnalysisManager.cpp
+++ b/lib/Analysis/LoopAnalysisManager.cpp
@@ -1,9 +1,8 @@
 //===- LoopAnalysisManager.cpp - Loop analysis management -----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Analysis/LoopInfo.cpp b/lib/Analysis/LoopInfo.cpp
index 6c779bf2cca03bb5b694ae8798a87915f10ef059..9518c6e551c007642604253f84c74e6e47f2d88f 100644
--- a/lib/Analysis/LoopInfo.cpp
+++ b/lib/Analysis/LoopInfo.cpp
@@ -1,9 +1,8 @@
 //===- LoopInfo.cpp - Natural Loop Calculator -----------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -255,35 +254,13 @@ void Loop::setLoopID(MDNode *LoopID) const {
 }
 
 void Loop::setLoopAlreadyUnrolled() {
-  MDNode *LoopID = getLoopID();
-  // First remove any existing loop unrolling metadata.
-  SmallVector<Metadata *, 4> MDs;
-  // Reserve first location for self reference to the LoopID metadata node.
-  MDs.push_back(nullptr);
-
-  if (LoopID) {
-    for (unsigned i = 1, ie = LoopID->getNumOperands(); i < ie; ++i) {
-      bool IsUnrollMetadata = false;
-      MDNode *MD = dyn_cast<MDNode>(LoopID->getOperand(i));
-      if (MD) {
-        const MDString *S = dyn_cast<MDString>(MD->getOperand(0));
-        IsUnrollMetadata = S && S->getString().startswith("llvm.loop.unroll.");
-      }
-      if (!IsUnrollMetadata)
-        MDs.push_back(LoopID->getOperand(i));
-    }
-  }
-
-  // Add unroll(disable) metadata to disable future unrolling.
   LLVMContext &Context = getHeader()->getContext();
-  SmallVector<Metadata *, 1> DisableOperands;
-  DisableOperands.push_back(MDString::get(Context, "llvm.loop.unroll.disable"));
-  MDNode *DisableNode = MDNode::get(Context, DisableOperands);
-  MDs.push_back(DisableNode);
 
-  MDNode *NewLoopID = MDNode::get(Context, MDs);
-  // Set operand 0 to refer to the loop id itself.
-  NewLoopID->replaceOperandWith(0, NewLoopID);
+  MDNode *DisableUnrollMD =
+      MDNode::get(Context, MDString::get(Context, "llvm.loop.unroll.disable"));
+  MDNode *LoopID = getLoopID();
+  MDNode *NewLoopID = makePostTransformationMetadata(
+      Context, LoopID, {"llvm.loop.unroll."}, {DisableUnrollMD});
   setLoopID(NewLoopID);
 }
 
@@ -293,16 +270,50 @@ bool Loop::isAnnotatedParallel() const {
   if (!DesiredLoopIdMetadata)
     return false;
 
+  MDNode *ParallelAccesses =
+      findOptionMDForLoop(this, "llvm.loop.parallel_accesses");
+  SmallPtrSet<MDNode *, 4>
+      ParallelAccessGroups; // For scalable 'contains' check.
+  if (ParallelAccesses) {
+    for (const MDOperand &MD : drop_begin(ParallelAccesses->operands(), 1)) {
+      MDNode *AccGroup = cast<MDNode>(MD.get());
+      assert(isValidAsAccessGroup(AccGroup) &&
+             "List item must be an access group");
+      ParallelAccessGroups.insert(AccGroup);
+    }
+  }
+
   // The loop branch contains the parallel loop metadata. In order to ensure
   // that any parallel-loop-unaware optimization pass hasn't added loop-carried
   // dependencies (thus converted the loop back to a sequential loop), check
-  // that all the memory instructions in the loop contain parallelism metadata
-  // that point to the same unique "loop id metadata" the loop branch does.
+  // that all the memory instructions in the loop belong to an access group that
+  // is parallel to this loop.
   for (BasicBlock *BB : this->blocks()) {
     for (Instruction &I : *BB) {
       if (!I.mayReadOrWriteMemory())
         continue;
 
+      if (MDNode *AccessGroup = I.getMetadata(LLVMContext::MD_access_group)) {
+        auto ContainsAccessGroup = [&ParallelAccessGroups](MDNode *AG) -> bool {
+          if (AG->getNumOperands() == 0) {
+            assert(isValidAsAccessGroup(AG) && "Item must be an access group");
+            return ParallelAccessGroups.count(AG);
+          }
+
+          for (const MDOperand &AccessListItem : AG->operands()) {
+            MDNode *AccGroup = cast<MDNode>(AccessListItem.get());
+            assert(isValidAsAccessGroup(AccGroup) &&
+                   "List item must be an access group");
+            if (ParallelAccessGroups.count(AccGroup))
+              return true;
+          }
+          return false;
+        };
+
+        if (ContainsAccessGroup(AccessGroup))
+          continue;
+      }
+
       // The memory instruction can refer to the loop identifier metadata
       // directly or indirectly through another list metadata (in case of
       // nested parallel loops). The loop identifier metadata refers to
@@ -693,6 +704,80 @@ void llvm::printLoop(Loop &L, raw_ostream &OS, const std::string &Banner) {
   }
 }
 
+MDNode *llvm::findOptionMDForLoopID(MDNode *LoopID, StringRef Name) {
+  // No loop metadata node, no loop properties.
+  if (!LoopID)
+    return nullptr;
+
+  // First operand should refer to the metadata node itself, for legacy reasons.
+  assert(LoopID->getNumOperands() > 0 && "requires at least one operand");
+  assert(LoopID->getOperand(0) == LoopID && "invalid loop id");
+
+  // Iterate over the metdata node operands and look for MDString metadata.
+  for (unsigned i = 1, e = LoopID->getNumOperands(); i < e; ++i) {
+    MDNode *MD = dyn_cast<MDNode>(LoopID->getOperand(i));
+    if (!MD || MD->getNumOperands() < 1)
+      continue;
+    MDString *S = dyn_cast<MDString>(MD->getOperand(0));
+    if (!S)
+      continue;
+    // Return the operand node if MDString holds expected metadata.
+    if (Name.equals(S->getString()))
+      return MD;
+  }
+
+  // Loop property not found.
+  return nullptr;
+}
+
+MDNode *llvm::findOptionMDForLoop(const Loop *TheLoop, StringRef Name) {
+  return findOptionMDForLoopID(TheLoop->getLoopID(), Name);
+}
+
+bool llvm::isValidAsAccessGroup(MDNode *Node) {
+  return Node->getNumOperands() == 0 && Node->isDistinct();
+}
+
+MDNode *llvm::makePostTransformationMetadata(LLVMContext &Context,
+                                             MDNode *OrigLoopID,
+                                             ArrayRef<StringRef> RemovePrefixes,
+                                             ArrayRef<MDNode *> AddAttrs) {
+  // First remove any existing loop metadata related to this transformation.
+  SmallVector<Metadata *, 4> MDs;
+
+  // Reserve first location for self reference to the LoopID metadata node.
+  TempMDTuple TempNode = MDNode::getTemporary(Context, None);
+  MDs.push_back(TempNode.get());
+
+  // Remove metadata for the transformation that has been applied or that became
+  // outdated.
+  if (OrigLoopID) {
+    for (unsigned i = 1, ie = OrigLoopID->getNumOperands(); i < ie; ++i) {
+      bool IsVectorMetadata = false;
+      Metadata *Op = OrigLoopID->getOperand(i);
+      if (MDNode *MD = dyn_cast<MDNode>(Op)) {
+        const MDString *S = dyn_cast<MDString>(MD->getOperand(0));
+        if (S)
+          IsVectorMetadata =
+              llvm::any_of(RemovePrefixes, [S](StringRef Prefix) -> bool {
+                return S->getString().startswith(Prefix);
+              });
+      }
+      if (!IsVectorMetadata)
+        MDs.push_back(Op);
+    }
+  }
+
+  // Add metadata to avoid reapplying a transformation, such as
+  // llvm.loop.unroll.disable and llvm.loop.isvectorized.
+  MDs.append(AddAttrs.begin(), AddAttrs.end());
+
+  MDNode *NewLoopID = MDNode::getDistinct(Context, MDs);
+  // Replace the temporary node with a self-reference.
+  NewLoopID->replaceOperandWith(0, NewLoopID);
+  return NewLoopID;
+}
+
 //===----------------------------------------------------------------------===//
 // LoopInfo implementation
 //
diff --git a/lib/Analysis/LoopPass.cpp b/lib/Analysis/LoopPass.cpp
index a68f114b83a0dbebcc13f048c9c6248c2e1fbdb2..e36370be3ce1d46e67b46855b0005c57a1ace3d9 100644
--- a/lib/Analysis/LoopPass.cpp
+++ b/lib/Analysis/LoopPass.cpp
@@ -1,9 +1,8 @@
 //===- LoopPass.cpp - Loop Pass and Loop Pass Manager ---------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Analysis/LoopUnrollAnalyzer.cpp b/lib/Analysis/LoopUnrollAnalyzer.cpp
index c8b91a7a1a516e0ae2f675bd164b812a501ce2e5..1728b5e9f6d2fbcc4ec3a702f4a1f8d6a55d417b 100644
--- a/lib/Analysis/LoopUnrollAnalyzer.cpp
+++ b/lib/Analysis/LoopUnrollAnalyzer.cpp
@@ -1,9 +1,8 @@
 //===- LoopUnrollAnalyzer.cpp - Unrolling Effect Estimation -----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Analysis/MemDepPrinter.cpp b/lib/Analysis/MemDepPrinter.cpp
index 5a6bbd7b2ac6a5231489bc19365f65e826bd52bf..6e1bb50e88936576227b13cc1500f84082037c69 100644
--- a/lib/Analysis/MemDepPrinter.cpp
+++ b/lib/Analysis/MemDepPrinter.cpp
@@ -1,9 +1,8 @@
 //===- MemDepPrinter.cpp - Printer for MemoryDependenceAnalysis -----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -13,7 +12,6 @@
 #include "llvm/ADT/SetVector.h"
 #include "llvm/Analysis/MemoryDependenceAnalysis.h"
 #include "llvm/Analysis/Passes.h"
-#include "llvm/IR/CallSite.h"
 #include "llvm/IR/InstIterator.h"
 #include "llvm/IR/LLVMContext.h"
 #include "llvm/Support/ErrorHandling.h"
@@ -106,9 +104,9 @@ bool MemDepPrinter::runOnFunction(Function &F) {
     if (!Res.isNonLocal()) {
       Deps[Inst].insert(std::make_pair(getInstTypePair(Res),
                                        static_cast<BasicBlock *>(nullptr)));
-    } else if (auto CS = CallSite(Inst)) {
+    } else if (auto *Call = dyn_cast<CallBase>(Inst)) {
       const MemoryDependenceResults::NonLocalDepInfo &NLDI =
-        MDA.getNonLocalCallDependency(CS);
+          MDA.getNonLocalCallDependency(Call);
 
       DepSet &InstDeps = Deps[Inst];
       for (const NonLocalDepEntry &I : NLDI) {
diff --git a/lib/Analysis/MemDerefPrinter.cpp b/lib/Analysis/MemDerefPrinter.cpp
index 4a136c5a0c6dc44d14caef37eb4717cdcfe59683..5f95a18b8a13558926d3fa4284243e2238ff8093 100644
--- a/lib/Analysis/MemDerefPrinter.cpp
+++ b/lib/Analysis/MemDerefPrinter.cpp
@@ -1,9 +1,8 @@
 //===- MemDerefPrinter.cpp - Printer for isDereferenceablePointer ---------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Analysis/MemoryBuiltins.cpp b/lib/Analysis/MemoryBuiltins.cpp
index 686ad294378c9ca3defe4580b6c3918f56bf668b..56332bb5fe8ddc37266021c7184d9a5905cf94da 100644
--- a/lib/Analysis/MemoryBuiltins.cpp
+++ b/lib/Analysis/MemoryBuiltins.cpp
@@ -1,9 +1,8 @@
 //===- MemoryBuiltins.cpp - Identify calls to memory builtins -------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -442,10 +441,10 @@ bool llvm::getObjectSize(const Value *Ptr, uint64_t &Size, const DataLayout &DL,
   return true;
 }
 
-ConstantInt *llvm::lowerObjectSizeCall(IntrinsicInst *ObjectSize,
-                                       const DataLayout &DL,
-                                       const TargetLibraryInfo *TLI,
-                                       bool MustSucceed) {
+Value *llvm::lowerObjectSizeCall(IntrinsicInst *ObjectSize,
+                                 const DataLayout &DL,
+                                 const TargetLibraryInfo *TLI,
+                                 bool MustSucceed) {
   assert(ObjectSize->getIntrinsicID() == Intrinsic::objectsize &&
          "ObjectSize must be a call to llvm.objectsize!");
 
@@ -462,13 +461,35 @@ ConstantInt *llvm::lowerObjectSizeCall(IntrinsicInst *ObjectSize,
   EvalOptions.NullIsUnknownSize =
       cast<ConstantInt>(ObjectSize->getArgOperand(2))->isOne();
 
-  // FIXME: Does it make sense to just return a failure value if the size won't
-  // fit in the output and `!MustSucceed`?
-  uint64_t Size;
   auto *ResultType = cast<IntegerType>(ObjectSize->getType());
-  if (getObjectSize(ObjectSize->getArgOperand(0), Size, DL, TLI, EvalOptions) &&
-      isUIntN(ResultType->getBitWidth(), Size))
-    return ConstantInt::get(ResultType, Size);
+  bool StaticOnly = cast<ConstantInt>(ObjectSize->getArgOperand(3))->isZero();
+  if (StaticOnly) {
+    // FIXME: Does it make sense to just return a failure value if the size won't
+    // fit in the output and `!MustSucceed`?
+    uint64_t Size;
+    if (getObjectSize(ObjectSize->getArgOperand(0), Size, DL, TLI, EvalOptions) &&
+        isUIntN(ResultType->getBitWidth(), Size))
+      return ConstantInt::get(ResultType, Size);
+  } else {
+    LLVMContext &Ctx = ObjectSize->getFunction()->getContext();
+    ObjectSizeOffsetEvaluator Eval(DL, TLI, Ctx, EvalOptions);
+    SizeOffsetEvalType SizeOffsetPair =
+        Eval.compute(ObjectSize->getArgOperand(0));
+
+    if (SizeOffsetPair != ObjectSizeOffsetEvaluator::unknown()) {
+      IRBuilder<TargetFolder> Builder(Ctx, TargetFolder(DL));
+      Builder.SetInsertPoint(ObjectSize);
+
+      // If we've outside the end of the object, then we can always access
+      // exactly 0 bytes.
+      Value *ResultSize =
+          Builder.CreateSub(SizeOffsetPair.first, SizeOffsetPair.second);
+      Value *UseZero =
+          Builder.CreateICmpULT(SizeOffsetPair.first, SizeOffsetPair.second);
+      return Builder.CreateSelect(UseZero, ConstantInt::get(ResultType, 0),
+                                  ResultSize);
+    }
+  }
 
   if (!MustSucceed)
     return nullptr;
@@ -743,9 +764,9 @@ SizeOffsetType ObjectSizeOffsetVisitor::visitInstruction(Instruction &I) {
 
 ObjectSizeOffsetEvaluator::ObjectSizeOffsetEvaluator(
     const DataLayout &DL, const TargetLibraryInfo *TLI, LLVMContext &Context,
-    bool RoundToAlign)
+    ObjectSizeOpts EvalOpts)
     : DL(DL), TLI(TLI), Context(Context), Builder(Context, TargetFolder(DL)),
-      RoundToAlign(RoundToAlign) {
+      EvalOpts(EvalOpts) {
   // IntTy and Zero must be set for each compute() since the address space may
   // be different for later objects.
 }
@@ -774,10 +795,7 @@ SizeOffsetEvalType ObjectSizeOffsetEvaluator::compute(Value *V) {
 }
 
 SizeOffsetEvalType ObjectSizeOffsetEvaluator::compute_(Value *V) {
-  ObjectSizeOpts ObjSizeOptions;
-  ObjSizeOptions.RoundToAlign = RoundToAlign;
-
-  ObjectSizeOffsetVisitor Visitor(DL, TLI, Context, ObjSizeOptions);
+  ObjectSizeOffsetVisitor Visitor(DL, TLI, Context, EvalOpts);
   SizeOffsetType Const = Visitor.compute(V);
   if (Visitor.bothKnown(Const))
     return std::make_pair(ConstantInt::get(Context, Const.first),
diff --git a/lib/Analysis/MemoryDependenceAnalysis.cpp b/lib/Analysis/MemoryDependenceAnalysis.cpp
index 2fe012d3865bcf6194441e234ae7d74a6c3254d2..98c3ae40d471fe5371a4a62626e12f34231bb9cb 100644
--- a/lib/Analysis/MemoryDependenceAnalysis.cpp
+++ b/lib/Analysis/MemoryDependenceAnalysis.cpp
@@ -1,9 +1,8 @@
 //===- MemoryDependenceAnalysis.cpp - Mem Deps Implementation -------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -31,7 +30,6 @@
 #include "llvm/Analysis/ValueTracking.h"
 #include "llvm/IR/Attributes.h"
 #include "llvm/IR/BasicBlock.h"
-#include "llvm/IR/CallSite.h"
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/DataLayout.h"
 #include "llvm/IR/DerivedTypes.h"
@@ -182,8 +180,8 @@ static ModRefInfo GetLocation(const Instruction *Inst, MemoryLocation &Loc,
 }
 
 /// Private helper for finding the local dependencies of a call site.
-MemDepResult MemoryDependenceResults::getCallSiteDependencyFrom(
-    CallSite CS, bool isReadOnlyCall, BasicBlock::iterator ScanIt,
+MemDepResult MemoryDependenceResults::getCallDependencyFrom(
+    CallBase *Call, bool isReadOnlyCall, BasicBlock::iterator ScanIt,
     BasicBlock *BB) {
   unsigned Limit = BlockScanLimit;
 
@@ -205,21 +203,21 @@ MemDepResult MemoryDependenceResults::getCallSiteDependencyFrom(
     ModRefInfo MR = GetLocation(Inst, Loc, TLI);
     if (Loc.Ptr) {
       // A simple instruction.
-      if (isModOrRefSet(AA.getModRefInfo(CS, Loc)))
+      if (isModOrRefSet(AA.getModRefInfo(Call, Loc)))
         return MemDepResult::getClobber(Inst);
       continue;
     }
 
-    if (auto InstCS = CallSite(Inst)) {
+    if (auto *CallB = dyn_cast<CallBase>(Inst)) {
       // If these two calls do not interfere, look past it.
-      if (isNoModRef(AA.getModRefInfo(CS, InstCS))) {
-        // If the two calls are the same, return InstCS as a Def, so that
-        // CS can be found redundant and eliminated.
+      if (isNoModRef(AA.getModRefInfo(Call, CallB))) {
+        // If the two calls are the same, return Inst as a Def, so that
+        // Call can be found redundant and eliminated.
         if (isReadOnlyCall && !isModSet(MR) &&
-            CS.getInstruction()->isIdenticalToWhenDefined(Inst))
+            Call->isIdenticalToWhenDefined(CallB))
           return MemDepResult::getDef(Inst);
 
-        // Otherwise if the two calls don't interact (e.g. InstCS is readnone)
+        // Otherwise if the two calls don't interact (e.g. CallB is readnone)
         // keep scanning.
         continue;
       } else
@@ -750,11 +748,10 @@ MemDepResult MemoryDependenceResults::getDependency(Instruction *QueryInst) {
 
       LocalCache = getPointerDependencyFrom(
           MemLoc, isLoad, ScanPos->getIterator(), QueryParent, QueryInst);
-    } else if (isa<CallInst>(QueryInst) || isa<InvokeInst>(QueryInst)) {
-      CallSite QueryCS(QueryInst);
-      bool isReadOnly = AA.onlyReadsMemory(QueryCS);
-      LocalCache = getCallSiteDependencyFrom(
-          QueryCS, isReadOnly, ScanPos->getIterator(), QueryParent);
+    } else if (auto *QueryCall = dyn_cast<CallBase>(QueryInst)) {
+      bool isReadOnly = AA.onlyReadsMemory(QueryCall);
+      LocalCache = getCallDependencyFrom(QueryCall, isReadOnly,
+                                         ScanPos->getIterator(), QueryParent);
     } else
       // Non-memory instruction.
       LocalCache = MemDepResult::getUnknown();
@@ -780,11 +777,11 @@ static void AssertSorted(MemoryDependenceResults::NonLocalDepInfo &Cache,
 #endif
 
 const MemoryDependenceResults::NonLocalDepInfo &
-MemoryDependenceResults::getNonLocalCallDependency(CallSite QueryCS) {
-  assert(getDependency(QueryCS.getInstruction()).isNonLocal() &&
+MemoryDependenceResults::getNonLocalCallDependency(CallBase *QueryCall) {
+  assert(getDependency(QueryCall).isNonLocal() &&
          "getNonLocalCallDependency should only be used on calls with "
          "non-local deps!");
-  PerInstNLInfo &CacheP = NonLocalDeps[QueryCS.getInstruction()];
+  PerInstNLInfo &CacheP = NonLocalDeps[QueryCall];
   NonLocalDepInfo &Cache = CacheP.first;
 
   // This is the set of blocks that need to be recomputed.  In the cached case,
@@ -814,14 +811,14 @@ MemoryDependenceResults::getNonLocalCallDependency(CallSite QueryCS) {
     //     << Cache.size() << " cached: " << *QueryInst;
   } else {
     // Seed DirtyBlocks with each of the preds of QueryInst's block.
-    BasicBlock *QueryBB = QueryCS.getInstruction()->getParent();
+    BasicBlock *QueryBB = QueryCall->getParent();
     for (BasicBlock *Pred : PredCache.get(QueryBB))
       DirtyBlocks.push_back(Pred);
     ++NumUncacheNonLocal;
   }
 
   // isReadonlyCall - If this is a read-only call, we can be more aggressive.
-  bool isReadonlyCall = AA.onlyReadsMemory(QueryCS);
+  bool isReadonlyCall = AA.onlyReadsMemory(QueryCall);
 
   SmallPtrSet<BasicBlock *, 32> Visited;
 
@@ -865,8 +862,8 @@ MemoryDependenceResults::getNonLocalCallDependency(CallSite QueryCS) {
       if (Instruction *Inst = ExistingResult->getResult().getInst()) {
         ScanPos = Inst->getIterator();
         // We're removing QueryInst's use of Inst.
-        RemoveFromReverseMap(ReverseNonLocalDeps, Inst,
-                             QueryCS.getInstruction());
+        RemoveFromReverseMap<Instruction *>(ReverseNonLocalDeps, Inst,
+                                            QueryCall);
       }
     }
 
@@ -874,8 +871,7 @@ MemoryDependenceResults::getNonLocalCallDependency(CallSite QueryCS) {
     MemDepResult Dep;
 
     if (ScanPos != DirtyBB->begin()) {
-      Dep =
-          getCallSiteDependencyFrom(QueryCS, isReadonlyCall, ScanPos, DirtyBB);
+      Dep = getCallDependencyFrom(QueryCall, isReadonlyCall, ScanPos, DirtyBB);
     } else if (DirtyBB != &DirtyBB->getParent()->getEntryBlock()) {
       // No dependence found.  If this is the entry block of the function, it is
       // a clobber, otherwise it is unknown.
@@ -897,7 +893,7 @@ MemoryDependenceResults::getNonLocalCallDependency(CallSite QueryCS) {
       // Keep the ReverseNonLocalDeps map up to date so we can efficiently
       // update this when we remove instructions.
       if (Instruction *Inst = Dep.getInst())
-        ReverseNonLocalDeps[Inst].insert(QueryCS.getInstruction());
+        ReverseNonLocalDeps[Inst].insert(QueryCall);
     } else {
 
       // If the block *is* completely transparent to the load, we need to check
diff --git a/lib/Analysis/MemoryLocation.cpp b/lib/Analysis/MemoryLocation.cpp
index c0605f6ad3797adce2f9552411b52932cf48193d..163830eee797e49cbcd37e13246df83a8f4ef824 100644
--- a/lib/Analysis/MemoryLocation.cpp
+++ b/lib/Analysis/MemoryLocation.cpp
@@ -1,9 +1,8 @@
 //===- MemoryLocation.cpp - Memory location descriptions -------------------==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -37,8 +36,9 @@ MemoryLocation MemoryLocation::get(const LoadInst *LI) {
   LI->getAAMetadata(AATags);
   const auto &DL = LI->getModule()->getDataLayout();
 
-  return MemoryLocation(LI->getPointerOperand(),
-                        DL.getTypeStoreSize(LI->getType()), AATags);
+  return MemoryLocation(
+      LI->getPointerOperand(),
+      LocationSize::precise(DL.getTypeStoreSize(LI->getType())), AATags);
 }
 
 MemoryLocation MemoryLocation::get(const StoreInst *SI) {
@@ -47,7 +47,8 @@ MemoryLocation MemoryLocation::get(const StoreInst *SI) {
   const auto &DL = SI->getModule()->getDataLayout();
 
   return MemoryLocation(SI->getPointerOperand(),
-                        DL.getTypeStoreSize(SI->getValueOperand()->getType()),
+                        LocationSize::precise(DL.getTypeStoreSize(
+                            SI->getValueOperand()->getType())),
                         AATags);
 }
 
@@ -64,9 +65,10 @@ MemoryLocation MemoryLocation::get(const AtomicCmpXchgInst *CXI) {
   CXI->getAAMetadata(AATags);
   const auto &DL = CXI->getModule()->getDataLayout();
 
-  return MemoryLocation(
-      CXI->getPointerOperand(),
-      DL.getTypeStoreSize(CXI->getCompareOperand()->getType()), AATags);
+  return MemoryLocation(CXI->getPointerOperand(),
+                        LocationSize::precise(DL.getTypeStoreSize(
+                            CXI->getCompareOperand()->getType())),
+                        AATags);
 }
 
 MemoryLocation MemoryLocation::get(const AtomicRMWInst *RMWI) {
@@ -75,7 +77,8 @@ MemoryLocation MemoryLocation::get(const AtomicRMWInst *RMWI) {
   const auto &DL = RMWI->getModule()->getDataLayout();
 
   return MemoryLocation(RMWI->getPointerOperand(),
-                        DL.getTypeStoreSize(RMWI->getValOperand()->getType()),
+                        LocationSize::precise(DL.getTypeStoreSize(
+                            RMWI->getValOperand()->getType())),
                         AATags);
 }
 
@@ -88,9 +91,9 @@ MemoryLocation MemoryLocation::getForSource(const AtomicMemTransferInst *MTI) {
 }
 
 MemoryLocation MemoryLocation::getForSource(const AnyMemTransferInst *MTI) {
-  uint64_t Size = MemoryLocation::UnknownSize;
+  auto Size = LocationSize::unknown();
   if (ConstantInt *C = dyn_cast<ConstantInt>(MTI->getLength()))
-    Size = C->getValue().getZExtValue();
+    Size = LocationSize::precise(C->getValue().getZExtValue());
 
   // memcpy/memmove can have AA tags. For memcpy, they apply
   // to both the source and the destination.
@@ -109,9 +112,9 @@ MemoryLocation MemoryLocation::getForDest(const AtomicMemIntrinsic *MI) {
 }
 
 MemoryLocation MemoryLocation::getForDest(const AnyMemIntrinsic *MI) {
-  uint64_t Size = MemoryLocation::UnknownSize;
+  auto Size = LocationSize::unknown();
   if (ConstantInt *C = dyn_cast<ConstantInt>(MI->getLength()))
-    Size = C->getValue().getZExtValue();
+    Size = LocationSize::precise(C->getValue().getZExtValue());
 
   // memcpy/memmove can have AA tags. For memcpy, they apply
   // to both the source and the destination.
@@ -121,15 +124,15 @@ MemoryLocation MemoryLocation::getForDest(const AnyMemIntrinsic *MI) {
   return MemoryLocation(MI->getRawDest(), Size, AATags);
 }
 
-MemoryLocation MemoryLocation::getForArgument(ImmutableCallSite CS,
+MemoryLocation MemoryLocation::getForArgument(const CallBase *Call,
                                               unsigned ArgIdx,
                                               const TargetLibraryInfo *TLI) {
   AAMDNodes AATags;
-  CS->getAAMetadata(AATags);
-  const Value *Arg = CS.getArgument(ArgIdx);
+  Call->getAAMetadata(AATags);
+  const Value *Arg = Call->getArgOperand(ArgIdx);
 
   // We may be able to produce an exact size for known intrinsics.
-  if (const IntrinsicInst *II = dyn_cast<IntrinsicInst>(CS.getInstruction())) {
+  if (const IntrinsicInst *II = dyn_cast<IntrinsicInst>(Call)) {
     const DataLayout &DL = II->getModule()->getDataLayout();
 
     switch (II->getIntrinsicID()) {
@@ -141,7 +144,8 @@ MemoryLocation MemoryLocation::getForArgument(ImmutableCallSite CS,
       assert((ArgIdx == 0 || ArgIdx == 1) &&
              "Invalid argument index for memory intrinsic");
       if (ConstantInt *LenCI = dyn_cast<ConstantInt>(II->getArgOperand(2)))
-        return MemoryLocation(Arg, LenCI->getZExtValue(), AATags);
+        return MemoryLocation(Arg, LocationSize::precise(LenCI->getZExtValue()),
+                              AATags);
       break;
 
     case Intrinsic::lifetime_start:
@@ -149,27 +153,37 @@ MemoryLocation MemoryLocation::getForArgument(ImmutableCallSite CS,
     case Intrinsic::invariant_start:
       assert(ArgIdx == 1 && "Invalid argument index");
       return MemoryLocation(
-          Arg, cast<ConstantInt>(II->getArgOperand(0))->getZExtValue(), AATags);
+          Arg,
+          LocationSize::precise(
+              cast<ConstantInt>(II->getArgOperand(0))->getZExtValue()),
+          AATags);
 
     case Intrinsic::invariant_end:
       // The first argument to an invariant.end is a "descriptor" type (e.g. a
       // pointer to a empty struct) which is never actually dereferenced.
       if (ArgIdx == 0)
-        return MemoryLocation(Arg, 0, AATags);
+        return MemoryLocation(Arg, LocationSize::precise(0), AATags);
       assert(ArgIdx == 2 && "Invalid argument index");
       return MemoryLocation(
-          Arg, cast<ConstantInt>(II->getArgOperand(1))->getZExtValue(), AATags);
+          Arg,
+          LocationSize::precise(
+              cast<ConstantInt>(II->getArgOperand(1))->getZExtValue()),
+          AATags);
 
     case Intrinsic::arm_neon_vld1:
       assert(ArgIdx == 0 && "Invalid argument index");
       // LLVM's vld1 and vst1 intrinsics currently only support a single
       // vector register.
-      return MemoryLocation(Arg, DL.getTypeStoreSize(II->getType()), AATags);
+      return MemoryLocation(
+          Arg, LocationSize::precise(DL.getTypeStoreSize(II->getType())),
+          AATags);
 
     case Intrinsic::arm_neon_vst1:
       assert(ArgIdx == 0 && "Invalid argument index");
-      return MemoryLocation(
-          Arg, DL.getTypeStoreSize(II->getArgOperand(1)->getType()), AATags);
+      return MemoryLocation(Arg,
+                            LocationSize::precise(DL.getTypeStoreSize(
+                                II->getArgOperand(1)->getType())),
+                            AATags);
     }
   }
 
@@ -178,18 +192,20 @@ MemoryLocation MemoryLocation::getForArgument(ImmutableCallSite CS,
   // LoopIdiomRecognizer likes to turn loops into calls to memset_pattern16
   // whenever possible.
   LibFunc F;
-  if (TLI && CS.getCalledFunction() &&
-      TLI->getLibFunc(*CS.getCalledFunction(), F) &&
+  if (TLI && Call->getCalledFunction() &&
+      TLI->getLibFunc(*Call->getCalledFunction(), F) &&
       F == LibFunc_memset_pattern16 && TLI->has(F)) {
     assert((ArgIdx == 0 || ArgIdx == 1) &&
            "Invalid argument index for memset_pattern16");
     if (ArgIdx == 1)
-      return MemoryLocation(Arg, 16, AATags);
-    if (const ConstantInt *LenCI = dyn_cast<ConstantInt>(CS.getArgument(2)))
-      return MemoryLocation(Arg, LenCI->getZExtValue(), AATags);
+      return MemoryLocation(Arg, LocationSize::precise(16), AATags);
+    if (const ConstantInt *LenCI =
+            dyn_cast<ConstantInt>(Call->getArgOperand(2)))
+      return MemoryLocation(Arg, LocationSize::precise(LenCI->getZExtValue()),
+                            AATags);
   }
   // FIXME: Handle memset_pattern4 and memset_pattern8 also.
 
-  return MemoryLocation(CS.getArgument(ArgIdx), LocationSize::unknown(),
+  return MemoryLocation(Call->getArgOperand(ArgIdx), LocationSize::unknown(),
                         AATags);
 }
diff --git a/lib/Analysis/MemorySSA.cpp b/lib/Analysis/MemorySSA.cpp
index 3d98fca823ad492a184bab5950adf21e3025e7ba..a2309d8179c8646b7f15cf647d890d850e67c7af 100644
--- a/lib/Analysis/MemorySSA.cpp
+++ b/lib/Analysis/MemorySSA.cpp
@@ -1,9 +1,8 @@
 //===- MemorySSA.cpp - Memory SSA Builder ---------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -30,7 +29,6 @@
 #include "llvm/Config/llvm-config.h"
 #include "llvm/IR/AssemblyAnnotationWriter.h"
 #include "llvm/IR/BasicBlock.h"
-#include "llvm/IR/CallSite.h"
 #include "llvm/IR/Dominators.h"
 #include "llvm/IR/Function.h"
 #include "llvm/IR/Instruction.h"
@@ -131,9 +129,9 @@ public:
       : MemoryLocOrCall(MUD->getMemoryInst()) {}
 
   MemoryLocOrCall(Instruction *Inst) {
-    if (ImmutableCallSite(Inst)) {
+    if (auto *C = dyn_cast<CallBase>(Inst)) {
       IsCall = true;
-      CS = ImmutableCallSite(Inst);
+      Call = C;
     } else {
       IsCall = false;
       // There is no such thing as a memorylocation for a fence inst, and it is
@@ -145,9 +143,9 @@ public:
 
   explicit MemoryLocOrCall(const MemoryLocation &Loc) : Loc(Loc) {}
 
-  ImmutableCallSite getCS() const {
+  const CallBase *getCall() const {
     assert(IsCall);
-    return CS;
+    return Call;
   }
 
   MemoryLocation getLoc() const {
@@ -162,16 +160,17 @@ public:
     if (!IsCall)
       return Loc == Other.Loc;
 
-    if (CS.getCalledValue() != Other.CS.getCalledValue())
+    if (Call->getCalledValue() != Other.Call->getCalledValue())
       return false;
 
-    return CS.arg_size() == Other.CS.arg_size() &&
-           std::equal(CS.arg_begin(), CS.arg_end(), Other.CS.arg_begin());
+    return Call->arg_size() == Other.Call->arg_size() &&
+           std::equal(Call->arg_begin(), Call->arg_end(),
+                      Other.Call->arg_begin());
   }
 
 private:
   union {
-    ImmutableCallSite CS;
+    const CallBase *Call;
     MemoryLocation Loc;
   };
 };
@@ -197,9 +196,9 @@ template <> struct DenseMapInfo<MemoryLocOrCall> {
 
     hash_code hash =
         hash_combine(MLOC.IsCall, DenseMapInfo<const Value *>::getHashValue(
-                                      MLOC.getCS().getCalledValue()));
+                                      MLOC.getCall()->getCalledValue()));
 
-    for (const Value *Arg : MLOC.getCS().args())
+    for (const Value *Arg : MLOC.getCall()->args())
       hash = hash_combine(hash, DenseMapInfo<const Value *>::getHashValue(Arg));
     return hash;
   }
@@ -258,7 +257,7 @@ static ClobberAlias instructionClobbersQuery(const MemoryDef *MD,
                                              AliasAnalysis &AA) {
   Instruction *DefInst = MD->getMemoryInst();
   assert(DefInst && "Defining instruction not actually an instruction");
-  ImmutableCallSite UseCS(UseInst);
+  const auto *UseCall = dyn_cast<CallBase>(UseInst);
   Optional<AliasResult> AR;
 
   if (const IntrinsicInst *II = dyn_cast<IntrinsicInst>(DefInst)) {
@@ -271,7 +270,7 @@ static ClobberAlias instructionClobbersQuery(const MemoryDef *MD,
     // context.
     switch (II->getIntrinsicID()) {
     case Intrinsic::lifetime_start:
-      if (UseCS)
+      if (UseCall)
         return {false, NoAlias};
       AR = AA.alias(MemoryLocation(II->getArgOperand(1)), UseLoc);
       return {AR != NoAlias, AR};
@@ -285,8 +284,8 @@ static ClobberAlias instructionClobbersQuery(const MemoryDef *MD,
     }
   }
 
-  if (UseCS) {
-    ModRefInfo I = AA.getModRefInfo(DefInst, UseCS);
+  if (UseCall) {
+    ModRefInfo I = AA.getModRefInfo(DefInst, UseCall);
     AR = isMustSet(I) ? MustAlias : MayAlias;
     return {isModOrRefSet(I), AR};
   }
@@ -332,11 +331,12 @@ struct UpwardsMemoryQuery {
   // The MemoryAccess we actually got called with, used to test local domination
   const MemoryAccess *OriginalAccess = nullptr;
   Optional<AliasResult> AR = MayAlias;
+  bool SkipSelfAccess = false;
 
   UpwardsMemoryQuery() = default;
 
   UpwardsMemoryQuery(const Instruction *Inst, const MemoryAccess *Access)
-      : IsCall(ImmutableCallSite(Inst)), Inst(Inst), OriginalAccess(Access) {
+      : IsCall(isa<CallBase>(Inst)), Inst(Inst), OriginalAccess(Access) {
     if (!IsCall)
       StartingLoc = MemoryLocation::get(Inst);
   }
@@ -380,7 +380,7 @@ static bool isUseTriviallyOptimizableToLiveOnEntry(AliasAnalysis &AA,
 /// \param Query     The UpwardsMemoryQuery we used for our search.
 /// \param AA        The AliasAnalysis we used for our search.
 /// \param AllowImpreciseClobber Always false, unless we do relaxed verify.
-static void
+LLVM_ATTRIBUTE_UNUSED static void
 checkClobberSanity(const MemoryAccess *Start, MemoryAccess *ClobberAt,
                    const MemoryLocation &StartLoc, const MemorySSA &MSSA,
                    const UpwardsMemoryQuery &Query, AliasAnalysis &AA,
@@ -535,13 +535,13 @@ class ClobberWalker {
   ///
   /// This does not test for whether StopAt is a clobber
   UpwardsWalkResult
-  walkToPhiOrClobber(DefPath &Desc,
-                     const MemoryAccess *StopAt = nullptr) const {
+  walkToPhiOrClobber(DefPath &Desc, const MemoryAccess *StopAt = nullptr,
+                     const MemoryAccess *SkipStopAt = nullptr) const {
     assert(!isa<MemoryUse>(Desc.Last) && "Uses don't exist in my world");
 
     for (MemoryAccess *Current : def_chain(Desc.Last)) {
       Desc.Last = Current;
-      if (Current == StopAt)
+      if (Current == StopAt || Current == SkipStopAt)
         return {Current, false, MayAlias};
 
       if (auto *MD = dyn_cast<MemoryDef>(Current)) {
@@ -619,9 +619,16 @@ class ClobberWalker {
       if (!VisitedPhis.insert({Node.Last, Node.Loc}).second)
         continue;
 
-      UpwardsWalkResult Res = walkToPhiOrClobber(Node, /*StopAt=*/StopWhere);
+      const MemoryAccess *SkipStopWhere = nullptr;
+      if (Query->SkipSelfAccess && Node.Loc == Query->StartingLoc) {
+        assert(isa<MemoryDef>(Query->OriginalAccess));
+        SkipStopWhere = Query->OriginalAccess;
+      }
+
+      UpwardsWalkResult Res = walkToPhiOrClobber(Node, /*StopAt=*/StopWhere,
+                                                 /*SkipStopAt=*/SkipStopWhere);
       if (Res.IsKnownClobber) {
-        assert(Res.Result != StopWhere);
+        assert(Res.Result != StopWhere && Res.Result != SkipStopWhere);
         // If this wasn't a cache hit, we hit a clobber when walking. That's a
         // failure.
         TerminatedPath Term{Res.Result, PathIndex};
@@ -633,10 +640,13 @@ class ClobberWalker {
         continue;
       }
 
-      if (Res.Result == StopWhere) {
+      if (Res.Result == StopWhere || Res.Result == SkipStopWhere) {
         // We've hit our target. Save this path off for if we want to continue
-        // walking.
-        NewPaused.push_back(PathIndex);
+        // walking. If we are in the mode of skipping the OriginalAccess, and
+        // we've reached back to the OriginalAccess, do not save path, we've
+        // just looped back to self.
+        if (Res.Result != SkipStopWhere)
+          NewPaused.push_back(PathIndex);
         continue;
       }
 
@@ -907,7 +917,8 @@ public:
     }
 
 #ifdef EXPENSIVE_CHECKS
-    checkClobberSanity(Current, Result, Q.StartingLoc, MSSA, Q, AA);
+    if (!Q.SkipSelfAccess)
+      checkClobberSanity(Current, Result, Q.StartingLoc, MSSA, Q, AA);
 #endif
     return Result;
   }
@@ -935,28 +946,76 @@ struct RenamePassData {
 
 namespace llvm {
 
+class MemorySSA::ClobberWalkerBase {
+  ClobberWalker Walker;
+  MemorySSA *MSSA;
+
+public:
+  ClobberWalkerBase(MemorySSA *M, AliasAnalysis *A, DominatorTree *D)
+      : Walker(*M, *A, *D), MSSA(M) {}
+
+  MemoryAccess *getClobberingMemoryAccessBase(MemoryAccess *,
+                                              const MemoryLocation &);
+  // Second argument (bool), defines whether the clobber search should skip the
+  // original queried access. If true, there will be a follow-up query searching
+  // for a clobber access past "self". Note that the Optimized access is not
+  // updated if a new clobber is found by this SkipSelf search. If this
+  // additional query becomes heavily used we may decide to cache the result.
+  // Walker instantiations will decide how to set the SkipSelf bool.
+  MemoryAccess *getClobberingMemoryAccessBase(MemoryAccess *, bool);
+  void verify(const MemorySSA *MSSA) { Walker.verify(MSSA); }
+};
+
 /// A MemorySSAWalker that does AA walks to disambiguate accesses. It no
 /// longer does caching on its own, but the name has been retained for the
 /// moment.
 class MemorySSA::CachingWalker final : public MemorySSAWalker {
-  ClobberWalker Walker;
-
-  MemoryAccess *getClobberingMemoryAccess(MemoryAccess *, UpwardsMemoryQuery &);
+  ClobberWalkerBase *Walker;
 
 public:
-  CachingWalker(MemorySSA *, AliasAnalysis *, DominatorTree *);
+  CachingWalker(MemorySSA *M, ClobberWalkerBase *W)
+      : MemorySSAWalker(M), Walker(W) {}
   ~CachingWalker() override = default;
 
   using MemorySSAWalker::getClobberingMemoryAccess;
 
-  MemoryAccess *getClobberingMemoryAccess(MemoryAccess *) override;
-  MemoryAccess *getClobberingMemoryAccess(MemoryAccess *,
-                                          const MemoryLocation &) override;
-  void invalidateInfo(MemoryAccess *) override;
+  MemoryAccess *getClobberingMemoryAccess(MemoryAccess *MA) override;
+  MemoryAccess *getClobberingMemoryAccess(MemoryAccess *MA,
+                                          const MemoryLocation &Loc) override;
+
+  void invalidateInfo(MemoryAccess *MA) override {
+    if (auto *MUD = dyn_cast<MemoryUseOrDef>(MA))
+      MUD->resetOptimized();
+  }
+
+  void verify(const MemorySSA *MSSA) override {
+    MemorySSAWalker::verify(MSSA);
+    Walker->verify(MSSA);
+  }
+};
+
+class MemorySSA::SkipSelfWalker final : public MemorySSAWalker {
+  ClobberWalkerBase *Walker;
+
+public:
+  SkipSelfWalker(MemorySSA *M, ClobberWalkerBase *W)
+      : MemorySSAWalker(M), Walker(W) {}
+  ~SkipSelfWalker() override = default;
+
+  using MemorySSAWalker::getClobberingMemoryAccess;
+
+  MemoryAccess *getClobberingMemoryAccess(MemoryAccess *MA) override;
+  MemoryAccess *getClobberingMemoryAccess(MemoryAccess *MA,
+                                          const MemoryLocation &Loc) override;
+
+  void invalidateInfo(MemoryAccess *MA) override {
+    if (auto *MUD = dyn_cast<MemoryUseOrDef>(MA))
+      MUD->resetOptimized();
+  }
 
   void verify(const MemorySSA *MSSA) override {
     MemorySSAWalker::verify(MSSA);
-    Walker.verify(MSSA);
+    Walker->verify(MSSA);
   }
 };
 
@@ -1095,7 +1154,7 @@ void MemorySSA::markUnreachableAsLiveOnEntry(BasicBlock *BB) {
 
 MemorySSA::MemorySSA(Function &Func, AliasAnalysis *AA, DominatorTree *DT)
     : AA(AA), DT(DT), F(Func), LiveOnEntryDef(nullptr), Walker(nullptr),
-      NextID(0) {
+      SkipWalker(nullptr), NextID(0) {
   buildMemorySSA();
 }
 
@@ -1426,10 +1485,25 @@ MemorySSA::CachingWalker *MemorySSA::getWalkerImpl() {
   if (Walker)
     return Walker.get();
 
-  Walker = llvm::make_unique<CachingWalker>(this, AA, DT);
+  if (!WalkerBase)
+    WalkerBase = llvm::make_unique<ClobberWalkerBase>(this, AA, DT);
+
+  Walker = llvm::make_unique<CachingWalker>(this, WalkerBase.get());
   return Walker.get();
 }
 
+MemorySSAWalker *MemorySSA::getSkipSelfWalker() {
+  if (SkipWalker)
+    return SkipWalker.get();
+
+  if (!WalkerBase)
+    WalkerBase = llvm::make_unique<ClobberWalkerBase>(this, AA, DT);
+
+  SkipWalker = llvm::make_unique<SkipSelfWalker>(this, WalkerBase.get());
+  return SkipWalker.get();
+ }
+
+
 // This is a helper function used by the creation routines. It places NewAccess
 // into the access and defs lists for a given basic block, at the given
 // insertion point.
@@ -1704,34 +1778,16 @@ void MemorySSA::verifyMemorySSA() const {
   verifyOrdering(F);
   verifyDominationNumbers(F);
   Walker->verify(this);
-  verifyClobberSanity(F);
-}
-
-/// Check sanity of the clobbering instruction for access MA.
-void MemorySSA::checkClobberSanityAccess(const MemoryAccess *MA) const {
-  if (const auto *MUD = dyn_cast<MemoryUseOrDef>(MA)) {
-    if (!MUD->isOptimized())
-      return;
-    auto *I = MUD->getMemoryInst();
-    auto Loc = MemoryLocation::getOrNone(I);
-    if (Loc == None)
-      return;
-    auto *Clobber = MUD->getOptimized();
-    UpwardsMemoryQuery Q(I, MUD);
-    checkClobberSanity(MUD, Clobber, *Loc, *this, Q, *AA, true);
-  }
-}
-
-void MemorySSA::verifyClobberSanity(const Function &F) const {
-#if !defined(NDEBUG) && defined(EXPENSIVE_CHECKS)
-  for (const BasicBlock &BB : F) {
-    const AccessList *Accesses = getBlockAccesses(&BB);
-    if (!Accesses)
-      continue;
-    for (const MemoryAccess &MA : *Accesses)
-      checkClobberSanityAccess(&MA);
-  }
-#endif
+  // Previously, the verification used to also verify that the clobberingAccess
+  // cached by MemorySSA is the same as the clobberingAccess found at a later
+  // query to AA. This does not hold true in general due to the current fragility
+  // of BasicAA which has arbitrary caps on the things it analyzes before giving
+  // up. As a result, transformations that are correct, will lead to BasicAA
+  // returning different Alias answers before and after that transformation.
+  // Invalidating MemorySSA is not an option, as the results in BasicAA can be so
+  // random, in the worst case we'd need to rebuild MemorySSA from scratch after
+  // every transformation, which defeats the purpose of using it. For such an
+  // example, see test4 added in D51960.
 }
 
 /// Verify that all of the blocks we believe to have valid domination numbers
@@ -2131,25 +2187,11 @@ void MemorySSAWrapperPass::print(raw_ostream &OS, const Module *M) const {
 
 MemorySSAWalker::MemorySSAWalker(MemorySSA *M) : MSSA(M) {}
 
-MemorySSA::CachingWalker::CachingWalker(MemorySSA *M, AliasAnalysis *A,
-                                        DominatorTree *D)
-    : MemorySSAWalker(M), Walker(*M, *A, *D) {}
-
-void MemorySSA::CachingWalker::invalidateInfo(MemoryAccess *MA) {
-  if (auto *MUD = dyn_cast<MemoryUseOrDef>(MA))
-    MUD->resetOptimized();
-}
-
-/// Walk the use-def chains starting at \p MA and find
+/// Walk the use-def chains starting at \p StartingAccess and find
 /// the MemoryAccess that actually clobbers Loc.
 ///
 /// \returns our clobbering memory access
-MemoryAccess *MemorySSA::CachingWalker::getClobberingMemoryAccess(
-    MemoryAccess *StartingAccess, UpwardsMemoryQuery &Q) {
-  return Walker.findClobber(StartingAccess, Q);
-}
-
-MemoryAccess *MemorySSA::CachingWalker::getClobberingMemoryAccess(
+MemoryAccess *MemorySSA::ClobberWalkerBase::getClobberingMemoryAccessBase(
     MemoryAccess *StartingAccess, const MemoryLocation &Loc) {
   if (isa<MemoryPhi>(StartingAccess))
     return StartingAccess;
@@ -2162,7 +2204,7 @@ MemoryAccess *MemorySSA::CachingWalker::getClobberingMemoryAccess(
 
   // Conservatively, fences are always clobbers, so don't perform the walk if we
   // hit a fence.
-  if (!ImmutableCallSite(I) && I->isFenceLike())
+  if (!isa<CallBase>(I) && I->isFenceLike())
     return StartingUseOrDef;
 
   UpwardsMemoryQuery Q;
@@ -2173,11 +2215,12 @@ MemoryAccess *MemorySSA::CachingWalker::getClobberingMemoryAccess(
 
   // Unlike the other function, do not walk to the def of a def, because we are
   // handed something we already believe is the clobbering access.
+  // We never set SkipSelf to true in Q in this method.
   MemoryAccess *DefiningAccess = isa<MemoryUse>(StartingUseOrDef)
                                      ? StartingUseOrDef->getDefiningAccess()
                                      : StartingUseOrDef;
 
-  MemoryAccess *Clobber = getClobberingMemoryAccess(DefiningAccess, Q);
+  MemoryAccess *Clobber = Walker.findClobber(DefiningAccess, Q);
   LLVM_DEBUG(dbgs() << "Starting Memory SSA clobber for " << *I << " is ");
   LLVM_DEBUG(dbgs() << *StartingUseOrDef << "\n");
   LLVM_DEBUG(dbgs() << "Final Memory SSA clobber for " << *I << " is ");
@@ -2186,23 +2229,29 @@ MemoryAccess *MemorySSA::CachingWalker::getClobberingMemoryAccess(
 }
 
 MemoryAccess *
-MemorySSA::CachingWalker::getClobberingMemoryAccess(MemoryAccess *MA) {
+MemorySSA::ClobberWalkerBase::getClobberingMemoryAccessBase(MemoryAccess *MA,
+                                                            bool SkipSelf) {
   auto *StartingAccess = dyn_cast<MemoryUseOrDef>(MA);
   // If this is a MemoryPhi, we can't do anything.
   if (!StartingAccess)
     return MA;
 
+  bool IsOptimized = false;
+
   // If this is an already optimized use or def, return the optimized result.
   // Note: Currently, we store the optimized def result in a separate field,
   // since we can't use the defining access.
-  if (StartingAccess->isOptimized())
-    return StartingAccess->getOptimized();
+  if (StartingAccess->isOptimized()) {
+    if (!SkipSelf || !isa<MemoryDef>(StartingAccess))
+      return StartingAccess->getOptimized();
+    IsOptimized = true;
+  }
 
   const Instruction *I = StartingAccess->getMemoryInst();
   // We can't sanely do anything with a fence, since they conservatively clobber
   // all memory, and have no locations to get pointers from to try to
   // disambiguate.
-  if (!ImmutableCallSite(I) && I->isFenceLike())
+  if (!isa<CallBase>(I) && I->isFenceLike())
     return StartingAccess;
 
   UpwardsMemoryQuery Q(I, StartingAccess);
@@ -2214,32 +2263,70 @@ MemorySSA::CachingWalker::getClobberingMemoryAccess(MemoryAccess *MA) {
     return LiveOnEntry;
   }
 
-  // Start with the thing we already think clobbers this location
-  MemoryAccess *DefiningAccess = StartingAccess->getDefiningAccess();
+  MemoryAccess *OptimizedAccess;
+  if (!IsOptimized) {
+    // Start with the thing we already think clobbers this location
+    MemoryAccess *DefiningAccess = StartingAccess->getDefiningAccess();
+
+    // At this point, DefiningAccess may be the live on entry def.
+    // If it is, we will not get a better result.
+    if (MSSA->isLiveOnEntryDef(DefiningAccess)) {
+      StartingAccess->setOptimized(DefiningAccess);
+      StartingAccess->setOptimizedAccessType(None);
+      return DefiningAccess;
+    }
 
-  // At this point, DefiningAccess may be the live on entry def.
-  // If it is, we will not get a better result.
-  if (MSSA->isLiveOnEntryDef(DefiningAccess)) {
-    StartingAccess->setOptimized(DefiningAccess);
-    StartingAccess->setOptimizedAccessType(None);
-    return DefiningAccess;
-  }
+    OptimizedAccess = Walker.findClobber(DefiningAccess, Q);
+    StartingAccess->setOptimized(OptimizedAccess);
+    if (MSSA->isLiveOnEntryDef(OptimizedAccess))
+      StartingAccess->setOptimizedAccessType(None);
+    else if (Q.AR == MustAlias)
+      StartingAccess->setOptimizedAccessType(MustAlias);
+  } else
+    OptimizedAccess = StartingAccess->getOptimized();
 
-  MemoryAccess *Result = getClobberingMemoryAccess(DefiningAccess, Q);
   LLVM_DEBUG(dbgs() << "Starting Memory SSA clobber for " << *I << " is ");
-  LLVM_DEBUG(dbgs() << *DefiningAccess << "\n");
-  LLVM_DEBUG(dbgs() << "Final Memory SSA clobber for " << *I << " is ");
-  LLVM_DEBUG(dbgs() << *Result << "\n");
-
-  StartingAccess->setOptimized(Result);
-  if (MSSA->isLiveOnEntryDef(Result))
-    StartingAccess->setOptimizedAccessType(None);
-  else if (Q.AR == MustAlias)
-    StartingAccess->setOptimizedAccessType(MustAlias);
+  LLVM_DEBUG(dbgs() << *StartingAccess << "\n");
+  LLVM_DEBUG(dbgs() << "Optimized Memory SSA clobber for " << *I << " is ");
+  LLVM_DEBUG(dbgs() << *OptimizedAccess << "\n");
+
+  MemoryAccess *Result;
+  if (SkipSelf && isa<MemoryPhi>(OptimizedAccess) &&
+      isa<MemoryDef>(StartingAccess)) {
+    assert(isa<MemoryDef>(Q.OriginalAccess));
+    Q.SkipSelfAccess = true;
+    Result = Walker.findClobber(OptimizedAccess, Q);
+  } else
+    Result = OptimizedAccess;
+
+  LLVM_DEBUG(dbgs() << "Result Memory SSA clobber [SkipSelf = " << SkipSelf);
+  LLVM_DEBUG(dbgs() << "] for " << *I << " is " << *Result << "\n");
 
   return Result;
 }
 
+MemoryAccess *
+MemorySSA::CachingWalker::getClobberingMemoryAccess(MemoryAccess *MA) {
+  return Walker->getClobberingMemoryAccessBase(MA, false);
+}
+
+MemoryAccess *
+MemorySSA::CachingWalker::getClobberingMemoryAccess(MemoryAccess *MA,
+                                                    const MemoryLocation &Loc) {
+  return Walker->getClobberingMemoryAccessBase(MA, Loc);
+}
+
+MemoryAccess *
+MemorySSA::SkipSelfWalker::getClobberingMemoryAccess(MemoryAccess *MA) {
+  return Walker->getClobberingMemoryAccessBase(MA, true);
+}
+
+MemoryAccess *
+MemorySSA::SkipSelfWalker::getClobberingMemoryAccess(MemoryAccess *MA,
+                                                    const MemoryLocation &Loc) {
+  return Walker->getClobberingMemoryAccessBase(MA, Loc);
+}
+
 MemoryAccess *
 DoNothingMemorySSAWalker::getClobberingMemoryAccess(MemoryAccess *MA) {
   if (auto *Use = dyn_cast<MemoryUseOrDef>(MA))
diff --git a/lib/Analysis/MemorySSAUpdater.cpp b/lib/Analysis/MemorySSAUpdater.cpp
index 6c817d20368456b1196c47d9485845154d62c61a..53950be58db2d7828bb9a97e19185d4a3da71239 100644
--- a/lib/Analysis/MemorySSAUpdater.cpp
+++ b/lib/Analysis/MemorySSAUpdater.cpp
@@ -1,9 +1,8 @@
 //===-- MemorySSAUpdater.cpp - Memory SSA Updater--------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------===//
 //
@@ -599,7 +598,7 @@ void MemorySSAUpdater::applyUpdates(ArrayRef<CFGUpdate> Updates,
 
   if (!RevDeleteUpdates.empty()) {
     // Update for inserted edges: use newDT and snapshot CFG as if deletes had
-    // not occured.
+    // not occurred.
     // FIXME: This creates a new DT, so it's more expensive to do mix
     // delete/inserts vs just inserts. We can do an incremental update on the DT
     // to revert deletes, than re-delete the edges. Teaching DT to do this, is
@@ -697,7 +696,7 @@ void MemorySSAUpdater::applyInsertUpdates(ArrayRef<CFGUpdate> Updates,
 
   // Map a BB to its predecessors: added + previously existing. To get a
   // deterministic order, store predecessors as SetVectors. The order in each
-  // will be defined by teh order in Updates (fixed) and the order given by
+  // will be defined by the order in Updates (fixed) and the order given by
   // children<> (also fixed). Since we further iterate over these ordered sets,
   // we lose the information of multiple edges possibly existing between two
   // blocks, so we'll keep and EdgeCount map for that.
@@ -1052,7 +1051,7 @@ void MemorySSAUpdater::wireOldPredecessorsToNewImmediatePredecessor(
   }
 }
 
-void MemorySSAUpdater::removeMemoryAccess(MemoryAccess *MA) {
+void MemorySSAUpdater::removeMemoryAccess(MemoryAccess *MA, bool OptimizePhis) {
   assert(!MSSA->isLiveOnEntryDef(MA) &&
          "Trying to remove the live on entry def");
   // We can only delete phi nodes if they have no uses, or we can replace all
@@ -1071,6 +1070,8 @@ void MemorySSAUpdater::removeMemoryAccess(MemoryAccess *MA) {
     NewDefTarget = cast<MemoryUseOrDef>(MA)->getDefiningAccess();
   }
 
+  SmallSetVector<MemoryPhi *, 4> PhisToCheck;
+
   // Re-point the uses at our defining access
   if (!isa<MemoryUse>(MA) && !MA->use_empty()) {
     // Reset optimized on users of this store, and reset the uses.
@@ -1090,6 +1091,9 @@ void MemorySSAUpdater::removeMemoryAccess(MemoryAccess *MA) {
       Use &U = *MA->use_begin();
       if (auto *MUD = dyn_cast<MemoryUseOrDef>(U.getUser()))
         MUD->resetOptimized();
+      if (OptimizePhis)
+        if (MemoryPhi *MP = dyn_cast<MemoryPhi>(U.getUser()))
+          PhisToCheck.insert(MP);
       U.set(NewDefTarget);
     }
   }
@@ -1098,6 +1102,21 @@ void MemorySSAUpdater::removeMemoryAccess(MemoryAccess *MA) {
   // are doing things here
   MSSA->removeFromLookups(MA);
   MSSA->removeFromLists(MA);
+
+  // Optionally optimize Phi uses. This will recursively remove trivial phis.
+  if (!PhisToCheck.empty()) {
+    SmallVector<WeakVH, 16> PhisToOptimize{PhisToCheck.begin(),
+                                           PhisToCheck.end()};
+    PhisToCheck.clear();
+
+    unsigned PhisSize = PhisToOptimize.size();
+    while (PhisSize-- > 0)
+      if (MemoryPhi *MP =
+              cast_or_null<MemoryPhi>(PhisToOptimize.pop_back_val())) {
+        auto OperRange = MP->operands();
+        tryRemoveTrivialPhi(MP, OperRange);
+      }
+  }
 }
 
 void MemorySSAUpdater::removeBlocks(
diff --git a/lib/Analysis/ModuleDebugInfoPrinter.cpp b/lib/Analysis/ModuleDebugInfoPrinter.cpp
index 1e321f17d59fa32d139db380a4fdb33e2162377e..519242759824a0ace7fff9733ff9151acce9ea7b 100644
--- a/lib/Analysis/ModuleDebugInfoPrinter.cpp
+++ b/lib/Analysis/ModuleDebugInfoPrinter.cpp
@@ -1,9 +1,8 @@
 //===-- ModuleDebugInfoPrinter.cpp - Prints module debug info metadata ----===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Analysis/ModuleSummaryAnalysis.cpp b/lib/Analysis/ModuleSummaryAnalysis.cpp
index 6bda1d1b1a36016ed675d04bdbb745ab53abdbd4..58bf469493b6416517b437904c91391daf068c45 100644
--- a/lib/Analysis/ModuleSummaryAnalysis.cpp
+++ b/lib/Analysis/ModuleSummaryAnalysis.cpp
@@ -1,9 +1,8 @@
 //===- ModuleSummaryAnalysis.cpp - Module summary index builder -----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -71,6 +70,11 @@ cl::opt<FunctionSummary::ForceSummaryHotnessType, true> FSEC(
                           "all-non-critical", "All non-critical edges."),
                clEnumValN(FunctionSummary::FSHT_All, "all", "All edges.")));
 
+cl::opt<std::string> ModuleSummaryDotFile(
+    "module-summary-dot-file", cl::init(""), cl::Hidden,
+    cl::value_desc("filename"),
+    cl::desc("File to emit dot graph of new summary into."));
+
 // Walk through the operands of a given User via worklist iteration and populate
 // the set of GlobalValue references encountered. Invoked either on an
 // Instruction or a GlobalVariable (which walks its initializer).
@@ -457,7 +461,11 @@ ModuleSummaryIndex llvm::buildModuleSummaryIndex(
     std::function<BlockFrequencyInfo *(const Function &F)> GetBFICallback,
     ProfileSummaryInfo *PSI) {
   assert(PSI);
-  ModuleSummaryIndex Index(/*HaveGVs=*/true);
+  bool EnableSplitLTOUnit = false;
+  if (auto *MD = mdconst::extract_or_null<ConstantInt>(
+          M.getModuleFlag("EnableSplitLTOUnit")))
+    EnableSplitLTOUnit = MD->getZExtValue();
+  ModuleSummaryIndex Index(/*HaveGVs=*/true, EnableSplitLTOUnit);
 
   // Identify the local values in the llvm.used and llvm.compiler.used sets,
   // which should not be exported as they would then require renaming and
@@ -622,6 +630,15 @@ ModuleSummaryIndex llvm::buildModuleSummaryIndex(
     }
   }
 
+  if (!ModuleSummaryDotFile.empty()) {
+    std::error_code EC;
+    raw_fd_ostream OSDot(ModuleSummaryDotFile, EC, sys::fs::OpenFlags::F_None);
+    if (EC)
+      report_fatal_error(Twine("Failed to open dot file ") +
+                         ModuleSummaryDotFile + ": " + EC.message() + "\n");
+    Index.exportToDot(OSDot);
+  }
+
   return Index;
 }
 
diff --git a/lib/Analysis/MustExecute.cpp b/lib/Analysis/MustExecute.cpp
index 281b8f5ab987e803f8bd615c86ab3436c0838793..e22831cd6a9271eee52a43278e1f24a242b593c9 100644
--- a/lib/Analysis/MustExecute.cpp
+++ b/lib/Analysis/MustExecute.cpp
@@ -1,9 +1,8 @@
 //===- MustExecute.cpp - Printer for isGuaranteedToExecute ----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -83,16 +82,15 @@ void ICFLoopSafetyInfo::computeLoopSafetyInfo(const Loop *CurLoop) {
   computeBlockColors(CurLoop);
 }
 
-void ICFLoopSafetyInfo::insertInstructionTo(const BasicBlock *BB) {
-  ICF.invalidateBlock(BB);
-  MW.invalidateBlock(BB);
+void ICFLoopSafetyInfo::insertInstructionTo(const Instruction *Inst,
+                                            const BasicBlock *BB) {
+  ICF.insertInstructionTo(Inst, BB);
+  MW.insertInstructionTo(Inst, BB);
 }
 
 void ICFLoopSafetyInfo::removeInstruction(const Instruction *Inst) {
-  // TODO: So far we just conservatively drop cache, but maybe we can not do it
-  // when Inst is not an ICF instruction. Follow-up on that.
-  ICF.invalidateBlock(Inst->getParent());
-  MW.invalidateBlock(Inst->getParent());
+  ICF.removeInstruction(Inst);
+  MW.removeInstruction(Inst);
 }
 
 void LoopSafetyInfo::computeBlockColors(const Loop *CurLoop) {
diff --git a/lib/Analysis/ObjCARCAliasAnalysis.cpp b/lib/Analysis/ObjCARCAliasAnalysis.cpp
index 096ea661ecb6a9d86530f5e1151e56e875a88d61..f768319f0a4dd42b52239f1ffcf80ac6dd641d71 100644
--- a/lib/Analysis/ObjCARCAliasAnalysis.cpp
+++ b/lib/Analysis/ObjCARCAliasAnalysis.cpp
@@ -1,9 +1,8 @@
 //===- ObjCARCAliasAnalysis.cpp - ObjC ARC Optimization -------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
@@ -106,12 +105,12 @@ FunctionModRefBehavior ObjCARCAAResult::getModRefBehavior(const Function *F) {
   return AAResultBase::getModRefBehavior(F);
 }
 
-ModRefInfo ObjCARCAAResult::getModRefInfo(ImmutableCallSite CS,
+ModRefInfo ObjCARCAAResult::getModRefInfo(const CallBase *Call,
                                           const MemoryLocation &Loc) {
   if (!EnableARCOpts)
-    return AAResultBase::getModRefInfo(CS, Loc);
+    return AAResultBase::getModRefInfo(Call, Loc);
 
-  switch (GetBasicARCInstKind(CS.getInstruction())) {
+  switch (GetBasicARCInstKind(Call)) {
   case ARCInstKind::Retain:
   case ARCInstKind::RetainRV:
   case ARCInstKind::Autorelease:
@@ -128,7 +127,7 @@ ModRefInfo ObjCARCAAResult::getModRefInfo(ImmutableCallSite CS,
     break;
   }
 
-  return AAResultBase::getModRefInfo(CS, Loc);
+  return AAResultBase::getModRefInfo(Call, Loc);
 }
 
 ObjCARCAAResult ObjCARCAA::run(Function &F, FunctionAnalysisManager &AM) {
diff --git a/lib/Analysis/ObjCARCAnalysisUtils.cpp b/lib/Analysis/ObjCARCAnalysisUtils.cpp
index d6db6386c38bd8969da1e9a0126cfd7f07054b83..56d1cb42122581a10e04cbd1c35b679db5ca246b 100644
--- a/lib/Analysis/ObjCARCAnalysisUtils.cpp
+++ b/lib/Analysis/ObjCARCAnalysisUtils.cpp
@@ -1,9 +1,8 @@
 //===- ObjCARCAnalysisUtils.cpp -------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Analysis/ObjCARCInstKind.cpp b/lib/Analysis/ObjCARCInstKind.cpp
index 31c432711834a945d6481cc964085609519b4afa..59994bae8bee966f5519e96cc17b5cbaad12c736 100644
--- a/lib/Analysis/ObjCARCInstKind.cpp
+++ b/lib/Analysis/ObjCARCInstKind.cpp
@@ -1,9 +1,8 @@
 //===- ARCInstKind.cpp - ObjC ARC Optimization ----------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/lib/Analysis/OptimizationRemarkEmitter.cpp b/lib/Analysis/OptimizationRemarkEmitter.cpp
index 8ece0a2a3ed3d078936648bd1d60a645c074581e..72c40a0be2325d85e8ff9f5bcfde4c9a6faa577a 100644
--- a/lib/Analysis/OptimizationRemarkEmitter.cpp
+++ b/lib/Analysis/OptimizationRemarkEmitter.cpp
@@ -1,9 +1,8 @@
 //===- OptimizationRemarkEmitter.cpp - Optimization Diagnostic --*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Analysis/OrderedBasicBlock.cpp b/lib/Analysis/OrderedBasicBlock.cpp
index 5f4fe0f7dda2ca7981efe236dcc3ae32c4596e82..3cc6319028d7824e7280aa25fffa6c5a16891baf 100644
--- a/lib/Analysis/OrderedBasicBlock.cpp
+++ b/lib/Analysis/OrderedBasicBlock.cpp
@@ -1,9 +1,8 @@
 //===- OrderedBasicBlock.cpp --------------------------------- -*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Analysis/OrderedInstructions.cpp b/lib/Analysis/OrderedInstructions.cpp
index 7b155208c02ec9a1a4c11e17a322f9e27a4ecad4..458c0a7de6c226a2dea883035c7af3b0800b77c6 100644
--- a/lib/Analysis/OrderedInstructions.cpp
+++ b/lib/Analysis/OrderedInstructions.cpp
@@ -1,9 +1,8 @@
 //===-- OrderedInstructions.cpp - Instruction dominance function ---------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Analysis/PHITransAddr.cpp b/lib/Analysis/PHITransAddr.cpp
index 858f08f6537a88db1b1572988787c42d359e469a..7f77ab146c4c542d5243143ae4f95bca1f0d1a38 100644
--- a/lib/Analysis/PHITransAddr.cpp
+++ b/lib/Analysis/PHITransAddr.cpp
@@ -1,9 +1,8 @@
 //===- PHITransAddr.cpp - PHI Translation for Addresses -------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Analysis/PhiValues.cpp b/lib/Analysis/PhiValues.cpp
index 729227c8669706ecbe1be2f76ea0f499dbc7284b..49749bc44746dc9032d6693c59ff10ec54859166 100644
--- a/lib/Analysis/PhiValues.cpp
+++ b/lib/Analysis/PhiValues.cpp
@@ -1,9 +1,8 @@
 //===- PhiValues.cpp - Phi Value Analysis ---------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Analysis/PostDominators.cpp b/lib/Analysis/PostDominators.cpp
index e6b660fe26d7a3bd6b7f646adc187669d68ab01d..4afe22bd5342afe1da2253f8d84b313bff487f02 100644
--- a/lib/Analysis/PostDominators.cpp
+++ b/lib/Analysis/PostDominators.cpp
@@ -1,9 +1,8 @@
 //===- PostDominators.cpp - Post-Dominator Calculation --------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Analysis/ProfileSummaryInfo.cpp b/lib/Analysis/ProfileSummaryInfo.cpp
index 1d70c75f2e1c9d532edf6531589e155721874fe1..bd7817ce67fb55568a39e2865c1dbc547b8acb11 100644
--- a/lib/Analysis/ProfileSummaryInfo.cpp
+++ b/lib/Analysis/ProfileSummaryInfo.cpp
@@ -1,9 +1,8 @@
 //===- ProfileSummaryInfo.cpp - Global profile summary information --------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Analysis/PtrUseVisitor.cpp b/lib/Analysis/PtrUseVisitor.cpp
index 1fdaf4d55b590be6dbabb01939e99f55d5457089..f78446c6fb49241c9bb49f5b30ec61643fd85b42 100644
--- a/lib/Analysis/PtrUseVisitor.cpp
+++ b/lib/Analysis/PtrUseVisitor.cpp
@@ -1,9 +1,8 @@
 //===- PtrUseVisitor.cpp - InstVisitors over a pointers uses --------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Analysis/RegionInfo.cpp b/lib/Analysis/RegionInfo.cpp
index 2bd611350f4670167c3a5933d028460886f927ab..8ba38adfb0d233c561e072206f9cfefe6a398016 100644
--- a/lib/Analysis/RegionInfo.cpp
+++ b/lib/Analysis/RegionInfo.cpp
@@ -1,9 +1,8 @@
 //===- RegionInfo.cpp - SESE region detection analysis --------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 // Detects single entry single exit regions in the control flow graph.
diff --git a/lib/Analysis/RegionPass.cpp b/lib/Analysis/RegionPass.cpp
index a101ff109199b6d9ee753b0966835dd14043a548..0e43af65e6cbe016bbc6f77fbe52cc09a0184b2b 100644
--- a/lib/Analysis/RegionPass.cpp
+++ b/lib/Analysis/RegionPass.cpp
@@ -1,9 +1,8 @@
 //===- RegionPass.cpp - Region Pass and Region Pass Manager ---------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Analysis/RegionPrinter.cpp b/lib/Analysis/RegionPrinter.cpp
index 5986b8c4e0c30aa3181cf11e678d92edb5fbc8ca..5bdcb31fbe99316af61d15e5daa9c86f275d6372 100644
--- a/lib/Analysis/RegionPrinter.cpp
+++ b/lib/Analysis/RegionPrinter.cpp
@@ -1,9 +1,8 @@
 //===- RegionPrinter.cpp - Print regions tree pass ------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 // Print out the region tree of a function using dotty/graphviz.
diff --git a/lib/Analysis/ScalarEvolution.cpp b/lib/Analysis/ScalarEvolution.cpp
index e5134f2eeda90e48dee2aa890a7c6ad7a6f0ee87..cca425bf550e18fdfaa7daa819b5b7c917eebe18 100644
--- a/lib/Analysis/ScalarEvolution.cpp
+++ b/lib/Analysis/ScalarEvolution.cpp
@@ -1,9 +1,8 @@
 //===- ScalarEvolution.cpp - Scalar Evolution Analysis --------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -212,6 +211,11 @@ static cl::opt<unsigned>
                   cl::desc("Max coefficients in AddRec during evolving"),
                   cl::init(8));
 
+static cl::opt<unsigned>
+    HugeExprThreshold("scalar-evolution-huge-expr-threshold", cl::Hidden,
+                  cl::desc("Size of the expression which is considered huge"),
+                  cl::init(4096));
+
 //===----------------------------------------------------------------------===//
 //                           SCEV class definitions
 //===----------------------------------------------------------------------===//
@@ -393,7 +397,7 @@ bool SCEV::isNonConstantNegative() const {
 }
 
 SCEVCouldNotCompute::SCEVCouldNotCompute() :
-  SCEV(FoldingSetNodeIDRef(), scCouldNotCompute) {}
+  SCEV(FoldingSetNodeIDRef(), scCouldNotCompute, 0) {}
 
 bool SCEVCouldNotCompute::classof(const SCEV *S) {
   return S->getSCEVType() == scCouldNotCompute;
@@ -422,7 +426,7 @@ ScalarEvolution::getConstant(Type *Ty, uint64_t V, bool isSigned) {
 
 SCEVCastExpr::SCEVCastExpr(const FoldingSetNodeIDRef ID,
                            unsigned SCEVTy, const SCEV *op, Type *ty)
-  : SCEV(ID, SCEVTy), Op(op), Ty(ty) {}
+  : SCEV(ID, SCEVTy, computeExpressionSize(op)), Op(op), Ty(ty) {}
 
 SCEVTruncateExpr::SCEVTruncateExpr(const FoldingSetNodeIDRef ID,
                                    const SCEV *op, Type *ty)
@@ -846,6 +850,17 @@ static inline int sizeOfSCEV(const SCEV *S) {
   return F.Size;
 }
 
+/// Returns true if the subtree of \p S contains at least HugeExprThreshold
+/// nodes.
+static bool isHugeExpression(const SCEV *S) {
+  return S->getExpressionSize() >= HugeExprThreshold;
+}
+
+/// Returns true of \p Ops contains a huge SCEV (see definition above).
+static bool hasHugeExpression(ArrayRef<const SCEV *> Ops) {
+  return any_of(Ops, isHugeExpression);
+}
+
 namespace {
 
 struct SCEVDivision : public SCEVVisitor<SCEVDivision, void> {
@@ -2295,7 +2310,7 @@ CollectAddOperandsWithScales(DenseMap<const SCEV *, APInt> &M,
 // can't-overflow flags for the operation if possible.
 static SCEV::NoWrapFlags
 StrengthenNoWrapFlags(ScalarEvolution *SE, SCEVTypes Type,
-                      const SmallVectorImpl<const SCEV *> &Ops,
+                      const ArrayRef<const SCEV *> Ops,
                       SCEV::NoWrapFlags Flags) {
   using namespace std::placeholders;
 
@@ -2405,7 +2420,7 @@ const SCEV *ScalarEvolution::getAddExpr(SmallVectorImpl<const SCEV *> &Ops,
   }
 
   // Limit recursion calls depth.
-  if (Depth > MaxArithDepth)
+  if (Depth > MaxArithDepth || hasHugeExpression(Ops))
     return getOrCreateAddExpr(Ops, Flags);
 
   // Okay, check to see if the same value occurs in the operand list more than
@@ -2743,7 +2758,7 @@ const SCEV *ScalarEvolution::getAddExpr(SmallVectorImpl<const SCEV *> &Ops,
 }
 
 const SCEV *
-ScalarEvolution::getOrCreateAddExpr(SmallVectorImpl<const SCEV *> &Ops,
+ScalarEvolution::getOrCreateAddExpr(ArrayRef<const SCEV *> Ops,
                                     SCEV::NoWrapFlags Flags) {
   FoldingSetNodeID ID;
   ID.AddInteger(scAddExpr);
@@ -2765,7 +2780,7 @@ ScalarEvolution::getOrCreateAddExpr(SmallVectorImpl<const SCEV *> &Ops,
 }
 
 const SCEV *
-ScalarEvolution::getOrCreateAddRecExpr(SmallVectorImpl<const SCEV *> &Ops,
+ScalarEvolution::getOrCreateAddRecExpr(ArrayRef<const SCEV *> Ops,
                                        const Loop *L, SCEV::NoWrapFlags Flags) {
   FoldingSetNodeID ID;
   ID.AddInteger(scAddRecExpr);
@@ -2788,7 +2803,7 @@ ScalarEvolution::getOrCreateAddRecExpr(SmallVectorImpl<const SCEV *> &Ops,
 }
 
 const SCEV *
-ScalarEvolution::getOrCreateMulExpr(SmallVectorImpl<const SCEV *> &Ops,
+ScalarEvolution::getOrCreateMulExpr(ArrayRef<const SCEV *> Ops,
                                     SCEV::NoWrapFlags Flags) {
   FoldingSetNodeID ID;
   ID.AddInteger(scMulExpr);
@@ -2884,7 +2899,7 @@ const SCEV *ScalarEvolution::getMulExpr(SmallVectorImpl<const SCEV *> &Ops,
   Flags = StrengthenNoWrapFlags(this, scMulExpr, Ops, Flags);
 
   // Limit recursion calls depth.
-  if (Depth > MaxArithDepth)
+  if (Depth > MaxArithDepth || hasHugeExpression(Ops))
     return getOrCreateMulExpr(Ops, Flags);
 
   // If there are any constants, fold them together.
@@ -3057,7 +3072,8 @@ const SCEV *ScalarEvolution::getMulExpr(SmallVectorImpl<const SCEV *> &Ops,
       // Limit max number of arguments to avoid creation of unreasonably big
       // SCEVAddRecs with very complex operands.
       if (AddRec->getNumOperands() + OtherAddRec->getNumOperands() - 1 >
-          MaxAddRecSize)
+          MaxAddRecSize || isHugeExpression(AddRec) ||
+          isHugeExpression(OtherAddRec))
         continue;
 
       bool Overflow = false;
@@ -3090,7 +3106,7 @@ const SCEV *ScalarEvolution::getMulExpr(SmallVectorImpl<const SCEV *> &Ops,
         AddRecOps.push_back(getAddExpr(SumOps, SCEV::FlagAnyWrap, Depth + 1));
       }
       if (!Overflow) {
-        const SCEV *NewAddRec = getAddRecExpr(AddRecOps, AddRec->getLoop(),
+        const SCEV *NewAddRec = getAddRecExpr(AddRecOps, AddRecLoop,
                                               SCEV::FlagAnyWrap);
         if (Ops.size() == 2) return NewAddRec;
         Ops[Idx] = NewAddRec;
@@ -3892,7 +3908,7 @@ void ScalarEvolution::eraseValueFromMap(Value *V) {
 }
 
 /// Check whether value has nuw/nsw/exact set but SCEV does not.
-/// TODO: In reality it is better to check the poison recursevely
+/// TODO: In reality it is better to check the poison recursively
 /// but this is better than nothing.
 static bool SCEVLostPoisonFlags(const SCEV *S, const Value *V) {
   if (auto *I = dyn_cast<Instruction>(V)) {
@@ -6128,7 +6144,7 @@ const SCEV *ScalarEvolution::createSCEV(Value *V) {
     // to obey basic rules for definitions dominating uses which this
     // analysis depends on.
     if (!DT.isReachableFromEntry(I->getParent()))
-      return getUnknown(V);
+      return getUnknown(UndefValue::get(V->getType()));
   } else if (ConstantInt *CI = dyn_cast<ConstantInt>(V))
     return getConstant(CI);
   else if (isa<ConstantPointerNull>(V))
@@ -6972,8 +6988,8 @@ ScalarEvolution::ExitLimit::ExitLimit(const SCEV *E, const SCEV *M,
 /// Allocate memory for BackedgeTakenInfo and copy the not-taken count of each
 /// computable exit into a persistent ExitNotTakenInfo array.
 ScalarEvolution::BackedgeTakenInfo::BackedgeTakenInfo(
-    SmallVectorImpl<ScalarEvolution::BackedgeTakenInfo::EdgeExitInfo>
-        &&ExitCounts,
+    ArrayRef<ScalarEvolution::BackedgeTakenInfo::EdgeExitInfo>
+        ExitCounts,
     bool Complete, const SCEV *MaxCount, bool MaxOrZero)
     : MaxAndComplete(MaxCount, Complete), MaxOrZero(MaxOrZero) {
   using EdgeExitInfo = ScalarEvolution::BackedgeTakenInfo::EdgeExitInfo;
@@ -8087,44 +8103,46 @@ const SCEV *ScalarEvolution::computeSCEVAtScope(const SCEV *V, const Loop *L) {
   // exit value from the loop without using SCEVs.
   if (const SCEVUnknown *SU = dyn_cast<SCEVUnknown>(V)) {
     if (Instruction *I = dyn_cast<Instruction>(SU->getValue())) {
-      const Loop *LI = this->LI[I->getParent()];
-      if (LI && LI->getParentLoop() == L)  // Looking for loop exit value.
-        if (PHINode *PN = dyn_cast<PHINode>(I))
-          if (PN->getParent() == LI->getHeader()) {
-            // Okay, there is no closed form solution for the PHI node.  Check
-            // to see if the loop that contains it has a known backedge-taken
-            // count.  If so, we may be able to force computation of the exit
-            // value.
-            const SCEV *BackedgeTakenCount = getBackedgeTakenCount(LI);
-            if (const SCEVConstant *BTCC =
-                  dyn_cast<SCEVConstant>(BackedgeTakenCount)) {
-
-              // This trivial case can show up in some degenerate cases where
-              // the incoming IR has not yet been fully simplified.
-              if (BTCC->getValue()->isZero()) {
-                Value *InitValue = nullptr;
-                bool MultipleInitValues = false;
-                for (unsigned i = 0; i < PN->getNumIncomingValues(); i++) {
-                  if (!LI->contains(PN->getIncomingBlock(i))) {
-                    if (!InitValue)
-                      InitValue = PN->getIncomingValue(i);
-                    else if (InitValue != PN->getIncomingValue(i)) {
-                      MultipleInitValues = true;
-                      break;
-                    }
+      if (PHINode *PN = dyn_cast<PHINode>(I)) {
+        const Loop *LI = this->LI[I->getParent()];
+        // Looking for loop exit value.
+        if (LI && LI->getParentLoop() == L &&
+            PN->getParent() == LI->getHeader()) {
+          // Okay, there is no closed form solution for the PHI node.  Check
+          // to see if the loop that contains it has a known backedge-taken
+          // count.  If so, we may be able to force computation of the exit
+          // value.
+          const SCEV *BackedgeTakenCount = getBackedgeTakenCount(LI);
+          if (const SCEVConstant *BTCC =
+                dyn_cast<SCEVConstant>(BackedgeTakenCount)) {
+
+            // This trivial case can show up in some degenerate cases where
+            // the incoming IR has not yet been fully simplified.
+            if (BTCC->getValue()->isZero()) {
+              Value *InitValue = nullptr;
+              bool MultipleInitValues = false;
+              for (unsigned i = 0; i < PN->getNumIncomingValues(); i++) {
+                if (!LI->contains(PN->getIncomingBlock(i))) {
+                  if (!InitValue)
+                    InitValue = PN->getIncomingValue(i);
+                  else if (InitValue != PN->getIncomingValue(i)) {
+                    MultipleInitValues = true;
+                    break;
                   }
-                  if (!MultipleInitValues && InitValue)
-                    return getSCEV(InitValue);
                 }
+                if (!MultipleInitValues && InitValue)
+                  return getSCEV(InitValue);
               }
-              // Okay, we know how many times the containing loop executes.  If
-              // this is a constant evolving PHI node, get the final value at
-              // the specified iteration number.
-              Constant *RV =
-                  getConstantEvolutionLoopExitValue(PN, BTCC->getAPInt(), LI);
-              if (RV) return getSCEV(RV);
             }
+            // Okay, we know how many times the containing loop executes.  If
+            // this is a constant evolving PHI node, get the final value at
+            // the specified iteration number.
+            Constant *RV =
+                getConstantEvolutionLoopExitValue(PN, BTCC->getAPInt(), LI);
+            if (RV) return getSCEV(RV);
           }
+        }
+      }
 
       // Okay, this is an expression that we cannot symbolically evaluate
       // into a SCEV.  Check to see if it's possible to symbolically evaluate
diff --git a/lib/Analysis/ScalarEvolutionAliasAnalysis.cpp b/lib/Analysis/ScalarEvolutionAliasAnalysis.cpp
index 61f625477ca6bf32b3070714208130a4d4c9446f..4f66b4d1b8f45e4311dcd32caf680656d10a89f8 100644
--- a/lib/Analysis/ScalarEvolutionAliasAnalysis.cpp
+++ b/lib/Analysis/ScalarEvolutionAliasAnalysis.cpp
@@ -1,9 +1,8 @@
 //===- ScalarEvolutionAliasAnalysis.cpp - SCEV-based Alias Analysis -------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -27,7 +26,7 @@ AliasResult SCEVAAResult::alias(const MemoryLocation &LocA,
   // If either of the memory references is empty, it doesn't matter what the
   // pointer values are. This allows the code below to ignore this special
   // case.
-  if (LocA.Size == 0 || LocB.Size == 0)
+  if (LocA.Size.isZero() || LocB.Size.isZero())
     return NoAlias;
 
   // This is SCEVAAResult. Get the SCEVs!
@@ -82,10 +81,10 @@ AliasResult SCEVAAResult::alias(const MemoryLocation &LocA,
   Value *BO = GetBaseValue(BS);
   if ((AO && AO != LocA.Ptr) || (BO && BO != LocB.Ptr))
     if (alias(MemoryLocation(AO ? AO : LocA.Ptr,
-                             AO ? +MemoryLocation::UnknownSize : LocA.Size,
+                             AO ? LocationSize::unknown() : LocA.Size,
                              AO ? AAMDNodes() : LocA.AATags),
               MemoryLocation(BO ? BO : LocB.Ptr,
-                             BO ? +MemoryLocation::UnknownSize : LocB.Size,
+                             BO ? LocationSize::unknown() : LocB.Size,
                              BO ? AAMDNodes() : LocB.AATags)) == NoAlias)
       return NoAlias;
 
diff --git a/lib/Analysis/ScalarEvolutionExpander.cpp b/lib/Analysis/ScalarEvolutionExpander.cpp
index ca5cf1663b83af74db62bce5f57f76ba749996ba..939907ce1b5186a23e3526e4186a878b076bc2b2 100644
--- a/lib/Analysis/ScalarEvolutionExpander.cpp
+++ b/lib/Analysis/ScalarEvolutionExpander.cpp
@@ -1,9 +1,8 @@
 //===- ScalarEvolutionExpander.cpp - Scalar Evolution Analysis ------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Analysis/ScalarEvolutionNormalization.cpp b/lib/Analysis/ScalarEvolutionNormalization.cpp
index 3740039b8f8674d253a65b6ba371863837cd8fb7..209ae66ca53e3617e3c272a0744311dae93c73a1 100644
--- a/lib/Analysis/ScalarEvolutionNormalization.cpp
+++ b/lib/Analysis/ScalarEvolutionNormalization.cpp
@@ -1,9 +1,8 @@
 //===- ScalarEvolutionNormalization.cpp - See below -----------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Analysis/ScopedNoAliasAA.cpp b/lib/Analysis/ScopedNoAliasAA.cpp
index f12275aff387007c683efbffce9dc97e61db4f20..dcf30c2f0a7fc9afa5c08b14c8283f042ec62c04 100644
--- a/lib/Analysis/ScopedNoAliasAA.cpp
+++ b/lib/Analysis/ScopedNoAliasAA.cpp
@@ -1,9 +1,8 @@
 //===- ScopedNoAliasAA.cpp - Scoped No-Alias Alias Analysis ---------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -95,39 +94,36 @@ AliasResult ScopedNoAliasAAResult::alias(const MemoryLocation &LocA,
   return AAResultBase::alias(LocA, LocB);
 }
 
-ModRefInfo ScopedNoAliasAAResult::getModRefInfo(ImmutableCallSite CS,
+ModRefInfo ScopedNoAliasAAResult::getModRefInfo(const CallBase *Call,
                                                 const MemoryLocation &Loc) {
   if (!EnableScopedNoAlias)
-    return AAResultBase::getModRefInfo(CS, Loc);
+    return AAResultBase::getModRefInfo(Call, Loc);
 
-  if (!mayAliasInScopes(Loc.AATags.Scope, CS.getInstruction()->getMetadata(
-                                              LLVMContext::MD_noalias)))
+  if (!mayAliasInScopes(Loc.AATags.Scope,
+                        Call->getMetadata(LLVMContext::MD_noalias)))
     return ModRefInfo::NoModRef;
 
-  if (!mayAliasInScopes(
-          CS.getInstruction()->getMetadata(LLVMContext::MD_alias_scope),
-          Loc.AATags.NoAlias))
+  if (!mayAliasInScopes(Call->getMetadata(LLVMContext::MD_alias_scope),
+                        Loc.AATags.NoAlias))
     return ModRefInfo::NoModRef;
 
-  return AAResultBase::getModRefInfo(CS, Loc);
+  return AAResultBase::getModRefInfo(Call, Loc);
 }
 
-ModRefInfo ScopedNoAliasAAResult::getModRefInfo(ImmutableCallSite CS1,
-                                                ImmutableCallSite CS2) {
+ModRefInfo ScopedNoAliasAAResult::getModRefInfo(const CallBase *Call1,
+                                                const CallBase *Call2) {
   if (!EnableScopedNoAlias)
-    return AAResultBase::getModRefInfo(CS1, CS2);
+    return AAResultBase::getModRefInfo(Call1, Call2);
 
-  if (!mayAliasInScopes(
-          CS1.getInstruction()->getMetadata(LLVMContext::MD_alias_scope),
-          CS2.getInstruction()->getMetadata(LLVMContext::MD_noalias)))
+  if (!mayAliasInScopes(Call1->getMetadata(LLVMContext::MD_alias_scope),
+                        Call2->getMetadata(LLVMContext::MD_noalias)))
     return ModRefInfo::NoModRef;
 
-  if (!mayAliasInScopes(
-          CS2.getInstruction()->getMetadata(LLVMContext::MD_alias_scope),
-          CS1.getInstruction()->getMetadata(LLVMContext::MD_noalias)))
+  if (!mayAliasInScopes(Call2->getMetadata(LLVMContext::MD_alias_scope),
+                        Call1->getMetadata(LLVMContext::MD_noalias)))
     return ModRefInfo::NoModRef;
 
-  return AAResultBase::getModRefInfo(CS1, CS2);
+  return AAResultBase::getModRefInfo(Call1, Call2);
 }
 
 static void collectMDInDomain(const MDNode *List, const MDNode *Domain,
diff --git a/lib/Analysis/StackSafetyAnalysis.cpp b/lib/Analysis/StackSafetyAnalysis.cpp
index 49d9b3f57c620c314f76d4d5c0d206ca5ad8473f..2bca7cf60fdd2f3aa722e5400135d0fdff09b492 100644
--- a/lib/Analysis/StackSafetyAnalysis.cpp
+++ b/lib/Analysis/StackSafetyAnalysis.cpp
@@ -1,9 +1,8 @@
 //===- StackSafetyAnalysis.cpp - Stack memory safety analysis -------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -323,11 +322,8 @@ bool StackSafetyLocalAnalysis::analyzeAllUses(const Value *Ptr, UseInfo &US) {
       case Instruction::Invoke: {
         ImmutableCallSite CS(I);
 
-        if (const IntrinsicInst *II = dyn_cast<IntrinsicInst>(I)) {
-          if (II->getIntrinsicID() == Intrinsic::lifetime_start ||
-              II->getIntrinsicID() == Intrinsic::lifetime_end)
-            break;
-        }
+        if (I->isLifetimeStartOrEnd())
+          break;
 
         if (const MemIntrinsic *MI = dyn_cast<MemIntrinsic>(I)) {
           US.updateRange(getMemIntrinsicAccessRange(MI, UI, Ptr));
diff --git a/lib/Analysis/StratifiedSets.h b/lib/Analysis/StratifiedSets.h
index 2f20cd12506c117458d19abbbf4d8e6099e90a36..60ea2451b0ef9a988d56fce1a8ff96e304f15428 100644
--- a/lib/Analysis/StratifiedSets.h
+++ b/lib/Analysis/StratifiedSets.h
@@ -1,9 +1,8 @@
 //===- StratifiedSets.h - Abstract stratified sets implementation. --------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Analysis/SyncDependenceAnalysis.cpp b/lib/Analysis/SyncDependenceAnalysis.cpp
index e1a7e4476d123de989357475ce37b4f9f8a4c4d1..54b4bb3113baf5bc0e86870e20340ffc4f712db4 100644
--- a/lib/Analysis/SyncDependenceAnalysis.cpp
+++ b/lib/Analysis/SyncDependenceAnalysis.cpp
@@ -1,10 +1,9 @@
 //===- SyncDependenceAnalysis.cpp - Divergent Branch Dependence Calculation
 //--===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Analysis/SyntheticCountsUtils.cpp b/lib/Analysis/SyntheticCountsUtils.cpp
index 386396bcff364f3cf0222a3f7994fd4c46023686..22766e5f07f544efc82ece5cdb308a1b4067133e 100644
--- a/lib/Analysis/SyntheticCountsUtils.cpp
+++ b/lib/Analysis/SyntheticCountsUtils.cpp
@@ -1,9 +1,8 @@
 //===--- SyntheticCountsUtils.cpp - synthetic counts propagation utils ---===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -26,8 +25,7 @@ using namespace llvm;
 // Given an SCC, propagate entry counts along the edge of the SCC nodes.
 template <typename CallGraphType>
 void SyntheticCountsUtils<CallGraphType>::propagateFromSCC(
-    const SccTy &SCC, GetRelBBFreqTy GetRelBBFreq, GetCountTy GetCount,
-    AddCountTy AddCount) {
+    const SccTy &SCC, GetProfCountTy GetProfCount, AddCountTy AddCount) {
 
   DenseSet<NodeRef> SCCNodes;
   SmallVector<std::pair<NodeRef, EdgeRef>, 8> SCCEdges, NonSCCEdges;
@@ -54,17 +52,13 @@ void SyntheticCountsUtils<CallGraphType>::propagateFromSCC(
   // This ensures that the order of
   // traversal of nodes within the SCC doesn't affect the final result.
 
-  DenseMap<NodeRef, uint64_t> AdditionalCounts;
+  DenseMap<NodeRef, Scaled64> AdditionalCounts;
   for (auto &E : SCCEdges) {
-    auto OptRelFreq = GetRelBBFreq(E.second);
-    if (!OptRelFreq)
+    auto OptProfCount = GetProfCount(E.first, E.second);
+    if (!OptProfCount)
       continue;
-    Scaled64 RelFreq = OptRelFreq.getValue();
-    auto Caller = E.first;
     auto Callee = CGT::edge_dest(E.second);
-    RelFreq *= Scaled64(GetCount(Caller), 0);
-    uint64_t AdditionalCount = RelFreq.toInt<uint64_t>();
-    AdditionalCounts[Callee] += AdditionalCount;
+    AdditionalCounts[Callee] += OptProfCount.getValue();
   }
 
   // Update the counts for the nodes in the SCC.
@@ -73,14 +67,11 @@ void SyntheticCountsUtils<CallGraphType>::propagateFromSCC(
 
   // Now update the counts for nodes outside the SCC.
   for (auto &E : NonSCCEdges) {
-    auto OptRelFreq = GetRelBBFreq(E.second);
-    if (!OptRelFreq)
+    auto OptProfCount = GetProfCount(E.first, E.second);
+    if (!OptProfCount)
       continue;
-    Scaled64 RelFreq = OptRelFreq.getValue();
-    auto Caller = E.first;
     auto Callee = CGT::edge_dest(E.second);
-    RelFreq *= Scaled64(GetCount(Caller), 0);
-    AddCount(Callee, RelFreq.toInt<uint64_t>());
+    AddCount(Callee, OptProfCount.getValue());
   }
 }
 
@@ -94,8 +85,7 @@ void SyntheticCountsUtils<CallGraphType>::propagateFromSCC(
 
 template <typename CallGraphType>
 void SyntheticCountsUtils<CallGraphType>::propagate(const CallGraphType &CG,
-                                                    GetRelBBFreqTy GetRelBBFreq,
-                                                    GetCountTy GetCount,
+                                                    GetProfCountTy GetProfCount,
                                                     AddCountTy AddCount) {
   std::vector<SccTy> SCCs;
 
@@ -107,7 +97,7 @@ void SyntheticCountsUtils<CallGraphType>::propagate(const CallGraphType &CG,
   // The scc iterator returns the scc in bottom-up order, so reverse the SCCs
   // and call propagateFromSCC.
   for (auto &SCC : reverse(SCCs))
-    propagateFromSCC(SCC, GetRelBBFreq, GetCount, AddCount);
+    propagateFromSCC(SCC, GetProfCount, AddCount);
 }
 
 template class llvm::SyntheticCountsUtils<const CallGraph *>;
diff --git a/lib/Analysis/TargetLibraryInfo.cpp b/lib/Analysis/TargetLibraryInfo.cpp
index 4643f75da42d14a10b39954a9e9d4061a39984e1..29e460ec78540a266326c8de0af1e01ef5b42d3a 100644
--- a/lib/Analysis/TargetLibraryInfo.cpp
+++ b/lib/Analysis/TargetLibraryInfo.cpp
@@ -1,9 +1,8 @@
 //===-- TargetLibraryInfo.cpp - Runtime library information ----------------==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -161,25 +160,66 @@ static void initialize(TargetLibraryInfoImpl &TLI, const Triple &T,
   }
 
   if (T.isOSWindows() && !T.isOSCygMing()) {
-    // Win32 does not support long double
+    // XXX: The earliest documentation available at the moment is for VS2015/VC19:
+    // https://docs.microsoft.com/en-us/cpp/c-runtime-library/floating-point-support?view=vs-2015
+    // XXX: In order to use an MSVCRT older than VC19,
+    // the specific library version must be explicit in the target triple,
+    // e.g., x86_64-pc-windows-msvc18.
+    bool hasPartialC99 = true;
+    if (T.isKnownWindowsMSVCEnvironment()) {
+      unsigned Major, Minor, Micro;
+      T.getEnvironmentVersion(Major, Minor, Micro);
+      hasPartialC99 = (Major == 0 || Major >= 19);
+    }
+
+    // Latest targets support C89 math functions, in part.
+    bool isARM = (T.getArch() == Triple::aarch64 ||
+                  T.getArch() == Triple::arm);
+    bool hasPartialFloat = (isARM ||
+                            T.getArch() == Triple::x86_64);
+
+    // Win32 does not support float C89 math functions, in general.
+    if (!hasPartialFloat) {
+      TLI.setUnavailable(LibFunc_acosf);
+      TLI.setUnavailable(LibFunc_asinf);
+      TLI.setUnavailable(LibFunc_atan2f);
+      TLI.setUnavailable(LibFunc_atanf);
+      TLI.setUnavailable(LibFunc_ceilf);
+      TLI.setUnavailable(LibFunc_cosf);
+      TLI.setUnavailable(LibFunc_coshf);
+      TLI.setUnavailable(LibFunc_expf);
+      TLI.setUnavailable(LibFunc_floorf);
+      TLI.setUnavailable(LibFunc_fmodf);
+      TLI.setUnavailable(LibFunc_log10f);
+      TLI.setUnavailable(LibFunc_logf);
+      TLI.setUnavailable(LibFunc_modff);
+      TLI.setUnavailable(LibFunc_powf);
+      TLI.setUnavailable(LibFunc_sinf);
+      TLI.setUnavailable(LibFunc_sinhf);
+      TLI.setUnavailable(LibFunc_sqrtf);
+      TLI.setUnavailable(LibFunc_tanf);
+      TLI.setUnavailable(LibFunc_tanhf);
+    }
+    if (!isARM)
+      TLI.setUnavailable(LibFunc_fabsf);
+    TLI.setUnavailable(LibFunc_frexpf);
+    TLI.setUnavailable(LibFunc_ldexpf);
+
+    // Win32 does not support long double C89 math functions.
     TLI.setUnavailable(LibFunc_acosl);
     TLI.setUnavailable(LibFunc_asinl);
-    TLI.setUnavailable(LibFunc_atanl);
     TLI.setUnavailable(LibFunc_atan2l);
+    TLI.setUnavailable(LibFunc_atanl);
     TLI.setUnavailable(LibFunc_ceill);
-    TLI.setUnavailable(LibFunc_copysignl);
     TLI.setUnavailable(LibFunc_cosl);
     TLI.setUnavailable(LibFunc_coshl);
     TLI.setUnavailable(LibFunc_expl);
-    TLI.setUnavailable(LibFunc_fabsf); // Win32 and Win64 both lack fabsf
     TLI.setUnavailable(LibFunc_fabsl);
     TLI.setUnavailable(LibFunc_floorl);
-    TLI.setUnavailable(LibFunc_fmaxl);
-    TLI.setUnavailable(LibFunc_fminl);
     TLI.setUnavailable(LibFunc_fmodl);
     TLI.setUnavailable(LibFunc_frexpl);
-    TLI.setUnavailable(LibFunc_ldexpf);
     TLI.setUnavailable(LibFunc_ldexpl);
+    TLI.setUnavailable(LibFunc_log10l);
     TLI.setUnavailable(LibFunc_logl);
     TLI.setUnavailable(LibFunc_modfl);
     TLI.setUnavailable(LibFunc_powl);
@@ -189,81 +229,66 @@ static void initialize(TargetLibraryInfoImpl &TLI, const Triple &T,
     TLI.setUnavailable(LibFunc_tanl);
     TLI.setUnavailable(LibFunc_tanhl);
 
-    // Win32 only has C89 math
-    TLI.setUnavailable(LibFunc_acosh);
-    TLI.setUnavailable(LibFunc_acoshf);
+    // Win32 does not fully support C99 math functions.
+    if (!hasPartialC99) {
+      TLI.setUnavailable(LibFunc_acosh);
+      TLI.setUnavailable(LibFunc_acoshf);
+      TLI.setUnavailable(LibFunc_asinh);
+      TLI.setUnavailable(LibFunc_asinhf);
+      TLI.setUnavailable(LibFunc_atanh);
+      TLI.setUnavailable(LibFunc_atanhf);
+      TLI.setAvailableWithName(LibFunc_cabs, "_cabs");
+      TLI.setUnavailable(LibFunc_cabsf);
+      TLI.setUnavailable(LibFunc_cbrt);
+      TLI.setUnavailable(LibFunc_cbrtf);
+      TLI.setAvailableWithName(LibFunc_copysign, "_copysign");
+      TLI.setAvailableWithName(LibFunc_copysignf, "_copysignf");
+      TLI.setUnavailable(LibFunc_exp2);
+      TLI.setUnavailable(LibFunc_exp2f);
+      TLI.setUnavailable(LibFunc_expm1);
+      TLI.setUnavailable(LibFunc_expm1f);
+      TLI.setUnavailable(LibFunc_fmax);
+      TLI.setUnavailable(LibFunc_fmaxf);
+      TLI.setUnavailable(LibFunc_fmin);
+      TLI.setUnavailable(LibFunc_fminf);
+      TLI.setUnavailable(LibFunc_log1p);
+      TLI.setUnavailable(LibFunc_log1pf);
+      TLI.setUnavailable(LibFunc_log2);
+      TLI.setUnavailable(LibFunc_log2f);
+      TLI.setAvailableWithName(LibFunc_logb, "_logb");
+      if (hasPartialFloat)
+        TLI.setAvailableWithName(LibFunc_logbf, "_logbf");
+      else
+        TLI.setUnavailable(LibFunc_logbf);
+      TLI.setUnavailable(LibFunc_rint);
+      TLI.setUnavailable(LibFunc_rintf);
+      TLI.setUnavailable(LibFunc_round);
+      TLI.setUnavailable(LibFunc_roundf);
+      TLI.setUnavailable(LibFunc_trunc);
+      TLI.setUnavailable(LibFunc_truncf);
+    }
+
+    // Win32 does not support long double C99 math functions.
     TLI.setUnavailable(LibFunc_acoshl);
-    TLI.setUnavailable(LibFunc_asinh);
-    TLI.setUnavailable(LibFunc_asinhf);
     TLI.setUnavailable(LibFunc_asinhl);
-    TLI.setUnavailable(LibFunc_atanh);
-    TLI.setUnavailable(LibFunc_atanhf);
     TLI.setUnavailable(LibFunc_atanhl);
-    TLI.setUnavailable(LibFunc_cabs);
-    TLI.setUnavailable(LibFunc_cabsf);
     TLI.setUnavailable(LibFunc_cabsl);
-    TLI.setUnavailable(LibFunc_cbrt);
-    TLI.setUnavailable(LibFunc_cbrtf);
     TLI.setUnavailable(LibFunc_cbrtl);
-    TLI.setUnavailable(LibFunc_exp2);
-    TLI.setUnavailable(LibFunc_exp2f);
+    TLI.setUnavailable(LibFunc_copysignl);
     TLI.setUnavailable(LibFunc_exp2l);
-    TLI.setUnavailable(LibFunc_expm1);
-    TLI.setUnavailable(LibFunc_expm1f);
     TLI.setUnavailable(LibFunc_expm1l);
-    TLI.setUnavailable(LibFunc_log2);
-    TLI.setUnavailable(LibFunc_log2f);
-    TLI.setUnavailable(LibFunc_log2l);
-    TLI.setUnavailable(LibFunc_log1p);
-    TLI.setUnavailable(LibFunc_log1pf);
+    TLI.setUnavailable(LibFunc_fmaxl);
+    TLI.setUnavailable(LibFunc_fminl);
     TLI.setUnavailable(LibFunc_log1pl);
-    TLI.setUnavailable(LibFunc_logb);
-    TLI.setUnavailable(LibFunc_logbf);
+    TLI.setUnavailable(LibFunc_log2l);
     TLI.setUnavailable(LibFunc_logbl);
-    TLI.setUnavailable(LibFunc_nearbyint);
-    TLI.setUnavailable(LibFunc_nearbyintf);
     TLI.setUnavailable(LibFunc_nearbyintl);
-    TLI.setUnavailable(LibFunc_rint);
-    TLI.setUnavailable(LibFunc_rintf);
     TLI.setUnavailable(LibFunc_rintl);
-    TLI.setUnavailable(LibFunc_round);
-    TLI.setUnavailable(LibFunc_roundf);
     TLI.setUnavailable(LibFunc_roundl);
-    TLI.setUnavailable(LibFunc_trunc);
-    TLI.setUnavailable(LibFunc_truncf);
     TLI.setUnavailable(LibFunc_truncl);
 
-    // Win32 provides some C99 math with mangled names
-    TLI.setAvailableWithName(LibFunc_copysign, "_copysign");
-
-    if (T.getArch() == Triple::x86) {
-      // Win32 on x86 implements single-precision math functions as macros
-      TLI.setUnavailable(LibFunc_acosf);
-      TLI.setUnavailable(LibFunc_asinf);
-      TLI.setUnavailable(LibFunc_atanf);
-      TLI.setUnavailable(LibFunc_atan2f);
-      TLI.setUnavailable(LibFunc_ceilf);
-      TLI.setUnavailable(LibFunc_copysignf);
-      TLI.setUnavailable(LibFunc_cosf);
-      TLI.setUnavailable(LibFunc_coshf);
-      TLI.setUnavailable(LibFunc_expf);
-      TLI.setUnavailable(LibFunc_floorf);
-      TLI.setUnavailable(LibFunc_fminf);
-      TLI.setUnavailable(LibFunc_fmaxf);
-      TLI.setUnavailable(LibFunc_fmodf);
-      TLI.setUnavailable(LibFunc_logf);
-      TLI.setUnavailable(LibFunc_log10f);
-      TLI.setUnavailable(LibFunc_modff);
-      TLI.setUnavailable(LibFunc_powf);
-      TLI.setUnavailable(LibFunc_sinf);
-      TLI.setUnavailable(LibFunc_sinhf);
-      TLI.setUnavailable(LibFunc_sqrtf);
-      TLI.setUnavailable(LibFunc_tanf);
-      TLI.setUnavailable(LibFunc_tanhf);
-    }
-
-    // Win32 does *not* provide these functions, but they are
-    // generally available on POSIX-compliant systems:
+    // Win32 does not support these functions, but
+    // they are generally available on POSIX-compliant systems.
     TLI.setUnavailable(LibFunc_access);
     TLI.setUnavailable(LibFunc_bcmp);
     TLI.setUnavailable(LibFunc_bcopy);
@@ -318,12 +343,6 @@ static void initialize(TargetLibraryInfoImpl &TLI, const Triple &T,
     TLI.setUnavailable(LibFunc_utime);
     TLI.setUnavailable(LibFunc_utimes);
     TLI.setUnavailable(LibFunc_write);
-
-    // Win32 does *not* provide provide these functions, but they are
-    // specified by C99:
-    TLI.setUnavailable(LibFunc_atoll);
-    TLI.setUnavailable(LibFunc_frexpf);
-    TLI.setUnavailable(LibFunc_llabs);
   }
 
   switch (T.getOS()) {
diff --git a/lib/Analysis/TargetTransformInfo.cpp b/lib/Analysis/TargetTransformInfo.cpp
index 79fe6dc7d87cb26dc4fa2b5989fbf3a6493f434c..7e453bfa1dff48ef7103cbc1a8e0c5d1a20b0d8e 100644
--- a/lib/Analysis/TargetTransformInfo.cpp
+++ b/lib/Analysis/TargetTransformInfo.cpp
@@ -1,9 +1,8 @@
 //===- llvm/Analysis/TargetTransformInfo.cpp ------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -163,6 +162,10 @@ bool TargetTransformInfo::shouldFavorPostInc() const {
   return TTIImpl->shouldFavorPostInc();
 }
 
+bool TargetTransformInfo::shouldFavorBackedgeIndex(const Loop *L) const {
+  return TTIImpl->shouldFavorBackedgeIndex(L);
+}
+
 bool TargetTransformInfo::isLegalMaskedStore(Type *DataType) const {
   return TTIImpl->isLegalMaskedStore(DataType);
 }
@@ -625,6 +628,12 @@ bool TargetTransformInfo::areInlineCompatible(const Function *Caller,
   return TTIImpl->areInlineCompatible(Caller, Callee);
 }
 
+bool TargetTransformInfo::areFunctionArgsABICompatible(
+    const Function *Caller, const Function *Callee,
+    SmallPtrSetImpl<Argument *> &Args) const {
+  return TTIImpl->areFunctionArgsABICompatible(Caller, Callee, Args);
+}
+
 bool TargetTransformInfo::isIndexedLoadLegal(MemIndexedMode Mode,
                                              Type *Ty) const {
   return TTIImpl->isIndexedLoadLegal(Mode, Ty);
diff --git a/lib/Analysis/Trace.cpp b/lib/Analysis/Trace.cpp
index 4dec53151ed629c173c33c1d78b72e43d6c59476..879c7172d038935185a91bbf6ef1f5ad51e3629f 100644
--- a/lib/Analysis/Trace.cpp
+++ b/lib/Analysis/Trace.cpp
@@ -1,9 +1,8 @@
 //===- Trace.cpp - Implementation of Trace class --------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Analysis/TypeBasedAliasAnalysis.cpp b/lib/Analysis/TypeBasedAliasAnalysis.cpp
index 25a154edf4ac8eead0149dd8ff6068ed02d055e1..708061873874aa6307f56199bbd4a1cbf201a67d 100644
--- a/lib/Analysis/TypeBasedAliasAnalysis.cpp
+++ b/lib/Analysis/TypeBasedAliasAnalysis.cpp
@@ -1,9 +1,8 @@
 //===- TypeBasedAliasAnalysis.cpp - Type-Based Alias Analysis -------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -399,20 +398,20 @@ bool TypeBasedAAResult::pointsToConstantMemory(const MemoryLocation &Loc,
 }
 
 FunctionModRefBehavior
-TypeBasedAAResult::getModRefBehavior(ImmutableCallSite CS) {
+TypeBasedAAResult::getModRefBehavior(const CallBase *Call) {
   if (!EnableTBAA)
-    return AAResultBase::getModRefBehavior(CS);
+    return AAResultBase::getModRefBehavior(Call);
 
   FunctionModRefBehavior Min = FMRB_UnknownModRefBehavior;
 
   // If this is an "immutable" type, we can assume the call doesn't write
   // to memory.
-  if (const MDNode *M = CS.getInstruction()->getMetadata(LLVMContext::MD_tbaa))
+  if (const MDNode *M = Call->getMetadata(LLVMContext::MD_tbaa))
     if ((!isStructPathTBAA(M) && TBAANode(M).isTypeImmutable()) ||
         (isStructPathTBAA(M) && TBAAStructTagNode(M).isTypeImmutable()))
       Min = FMRB_OnlyReadsMemory;
 
-  return FunctionModRefBehavior(AAResultBase::getModRefBehavior(CS) & Min);
+  return FunctionModRefBehavior(AAResultBase::getModRefBehavior(Call) & Min);
 }
 
 FunctionModRefBehavior TypeBasedAAResult::getModRefBehavior(const Function *F) {
@@ -420,33 +419,30 @@ FunctionModRefBehavior TypeBasedAAResult::getModRefBehavior(const Function *F) {
   return AAResultBase::getModRefBehavior(F);
 }
 
-ModRefInfo TypeBasedAAResult::getModRefInfo(ImmutableCallSite CS,
+ModRefInfo TypeBasedAAResult::getModRefInfo(const CallBase *Call,
                                             const MemoryLocation &Loc) {
   if (!EnableTBAA)
-    return AAResultBase::getModRefInfo(CS, Loc);
+    return AAResultBase::getModRefInfo(Call, Loc);
 
   if (const MDNode *L = Loc.AATags.TBAA)
-    if (const MDNode *M =
-            CS.getInstruction()->getMetadata(LLVMContext::MD_tbaa))
+    if (const MDNode *M = Call->getMetadata(LLVMContext::MD_tbaa))
       if (!Aliases(L, M))
         return ModRefInfo::NoModRef;
 
-  return AAResultBase::getModRefInfo(CS, Loc);
+  return AAResultBase::getModRefInfo(Call, Loc);
 }
 
-ModRefInfo TypeBasedAAResult::getModRefInfo(ImmutableCallSite CS1,
-                                            ImmutableCallSite CS2) {
+ModRefInfo TypeBasedAAResult::getModRefInfo(const CallBase *Call1,
+                                            const CallBase *Call2) {
   if (!EnableTBAA)
-    return AAResultBase::getModRefInfo(CS1, CS2);
+    return AAResultBase::getModRefInfo(Call1, Call2);
 
-  if (const MDNode *M1 =
-          CS1.getInstruction()->getMetadata(LLVMContext::MD_tbaa))
-    if (const MDNode *M2 =
-            CS2.getInstruction()->getMetadata(LLVMContext::MD_tbaa))
+  if (const MDNode *M1 = Call1->getMetadata(LLVMContext::MD_tbaa))
+    if (const MDNode *M2 = Call2->getMetadata(LLVMContext::MD_tbaa))
       if (!Aliases(M1, M2))
         return ModRefInfo::NoModRef;
 
-  return AAResultBase::getModRefInfo(CS1, CS2);
+  return AAResultBase::getModRefInfo(Call1, Call2);
 }
 
 bool MDNode::isTBAAVtableAccess() const {
diff --git a/lib/Analysis/TypeMetadataUtils.cpp b/lib/Analysis/TypeMetadataUtils.cpp
index bd13a43b8d46217be59153941d656b1346eff611..9311dfbc6eba0e21fd60085119d7db9bfd5a1124 100644
--- a/lib/Analysis/TypeMetadataUtils.cpp
+++ b/lib/Analysis/TypeMetadataUtils.cpp
@@ -1,9 +1,8 @@
 //===- TypeMetadataUtils.cpp - Utilities related to type metadata ---------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Analysis/ValueLattice.cpp b/lib/Analysis/ValueLattice.cpp
index 7de437ca480e42589905169d6750edec1e894080..a0115a0eec36c174342c7eeaed75e0258087c746 100644
--- a/lib/Analysis/ValueLattice.cpp
+++ b/lib/Analysis/ValueLattice.cpp
@@ -1,9 +1,8 @@
 //===- ValueLattice.cpp - Value constraint analysis -------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Analysis/ValueLatticeUtils.cpp b/lib/Analysis/ValueLatticeUtils.cpp
index 22c9de4fe94d954e69cfb747e99e9af6f9db705c..3f9287e26ce7e93dd1253c9bfeed5eefb2314672 100644
--- a/lib/Analysis/ValueLatticeUtils.cpp
+++ b/lib/Analysis/ValueLatticeUtils.cpp
@@ -1,9 +1,8 @@
 //===-- ValueLatticeUtils.cpp - Utils for solving lattices ------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Analysis/ValueTracking.cpp b/lib/Analysis/ValueTracking.cpp
index 6f0196069f2d64ff9d819049b2f8376dcb8c24a6..6f88a6b34ca61eea6d0daf9d4dbdd332d21c3c76 100644
--- a/lib/Analysis/ValueTracking.cpp
+++ b/lib/Analysis/ValueTracking.cpp
@@ -1,9 +1,8 @@
 //===- ValueTracking.cpp - Walk computations to compute properties --------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -2023,10 +2022,10 @@ bool isKnownNonZero(const Value *V, unsigned Depth, const Query &Q) {
       if (Q.IIQ.getMetadata(LI, LLVMContext::MD_nonnull))
         return true;
 
-    if (auto CS = ImmutableCallSite(V)) {
-      if (CS.isReturnNonNull())
+    if (const auto *Call = dyn_cast<CallBase>(V)) {
+      if (Call->isReturnNonNull())
         return true;
-      if (const auto *RP = getArgumentAliasingToReturnedPointer(CS))
+      if (const auto *RP = getArgumentAliasingToReturnedPointer(Call))
         return isKnownNonZero(RP, Depth, Q);
     }
   }
@@ -2037,11 +2036,33 @@ bool isKnownNonZero(const Value *V, unsigned Depth, const Query &Q) {
     if (isKnownNonNullFromDominatingCondition(V, Q.CxtI, Q.DT))
       return true;
 
+    // Look through bitcast operations, GEPs, and int2ptr instructions as they
+    // do not alter the value, or at least not the nullness property of the
+    // value, e.g., int2ptr is allowed to zero/sign extend the value.
+    //
+    // Note that we have to take special care to avoid looking through
+    // truncating casts, e.g., int2ptr/ptr2int with appropriate sizes, as well
+    // as casts that can alter the value, e.g., AddrSpaceCasts.
     if (const GEPOperator *GEP = dyn_cast<GEPOperator>(V))
       if (isGEPKnownNonNull(GEP, Depth, Q))
         return true;
+
+    if (auto *BCO = dyn_cast<BitCastOperator>(V))
+      return isKnownNonZero(BCO->getOperand(0), Depth, Q);
+
+    if (auto *I2P = dyn_cast<IntToPtrInst>(V))
+      if (Q.DL.getTypeSizeInBits(I2P->getSrcTy()) <=
+          Q.DL.getTypeSizeInBits(I2P->getDestTy()))
+        return isKnownNonZero(I2P->getOperand(0), Depth, Q);
   }
 
+  // Similar to int2ptr above, we can look through ptr2int here if the cast
+  // is a no-op or an extend and not a truncate.
+  if (auto *P2I = dyn_cast<PtrToIntInst>(V))
+    if (Q.DL.getTypeSizeInBits(P2I->getSrcTy()) <=
+        Q.DL.getTypeSizeInBits(P2I->getDestTy()))
+      return isKnownNonZero(P2I->getOperand(0), Depth, Q);
+
   unsigned BitWidth = getBitWidth(V->getType()->getScalarType(), Q.DL);
 
   // X | Y != 0 if X != 0 or Y != 0.
@@ -3389,7 +3410,14 @@ Value *llvm::GetPointerBaseWithConstantOffset(Value *Ptr, int64_t &Offset,
       if (!GEP->accumulateConstantOffset(DL, GEPOffset))
         break;
 
-      ByteOffset += GEPOffset.getSExtValue();
+      APInt OrigByteOffset(ByteOffset);
+      ByteOffset += GEPOffset.sextOrTrunc(ByteOffset.getBitWidth());
+      if (ByteOffset.getMinSignedBits() > 64) {
+        // Stop traversal if the pointer offset wouldn't fit into int64_t
+        // (this should be removed if Offset is updated to an APInt)
+        ByteOffset = OrigByteOffset;
+        break;
+      }
 
       Ptr = GEP->getPointerOperand();
     } else if (Operator::getOpcode(Ptr) == Instruction::BitCast ||
@@ -3617,21 +3645,21 @@ uint64_t llvm::GetStringLength(const Value *V, unsigned CharSize) {
   return Len == ~0ULL ? 1 : Len;
 }
 
-const Value *llvm::getArgumentAliasingToReturnedPointer(ImmutableCallSite CS) {
-  assert(CS &&
-         "getArgumentAliasingToReturnedPointer only works on nonnull CallSite");
-  if (const Value *RV = CS.getReturnedArgOperand())
+const Value *llvm::getArgumentAliasingToReturnedPointer(const CallBase *Call) {
+  assert(Call &&
+         "getArgumentAliasingToReturnedPointer only works on nonnull calls");
+  if (const Value *RV = Call->getReturnedArgOperand())
     return RV;
   // This can be used only as a aliasing property.
-  if (isIntrinsicReturningPointerAliasingArgumentWithoutCapturing(CS))
-    return CS.getArgOperand(0);
+  if (isIntrinsicReturningPointerAliasingArgumentWithoutCapturing(Call))
+    return Call->getArgOperand(0);
   return nullptr;
 }
 
 bool llvm::isIntrinsicReturningPointerAliasingArgumentWithoutCapturing(
-    ImmutableCallSite CS) {
-  return CS.getIntrinsicID() == Intrinsic::launder_invariant_group ||
-         CS.getIntrinsicID() == Intrinsic::strip_invariant_group;
+    const CallBase *Call) {
+  return Call->getIntrinsicID() == Intrinsic::launder_invariant_group ||
+         Call->getIntrinsicID() == Intrinsic::strip_invariant_group;
 }
 
 /// \p PN defines a loop-variant pointer to an object.  Check if the
@@ -3679,7 +3707,7 @@ Value *llvm::GetUnderlyingObject(Value *V, const DataLayout &DL,
       // An alloca can't be further simplified.
       return V;
     } else {
-      if (auto CS = CallSite(V)) {
+      if (auto *Call = dyn_cast<CallBase>(V)) {
         // CaptureTracking can know about special capturing properties of some
         // intrinsics like launder.invariant.group, that can't be expressed with
         // the attributes, but have properties like returning aliasing pointer.
@@ -3689,7 +3717,7 @@ Value *llvm::GetUnderlyingObject(Value *V, const DataLayout &DL,
         // because it should be in sync with CaptureTracking. Not using it may
         // cause weird miscompilations where 2 aliasing pointers are assumed to
         // noalias.
-        if (auto *RP = getArgumentAliasingToReturnedPointer(CS)) {
+        if (auto *RP = getArgumentAliasingToReturnedPointer(Call)) {
           V = RP;
           continue;
         }
@@ -3822,8 +3850,7 @@ bool llvm::onlyUsedByLifetimeMarkers(const Value *V) {
     const IntrinsicInst *II = dyn_cast<IntrinsicInst>(U);
     if (!II) return false;
 
-    if (II->getIntrinsicID() != Intrinsic::lifetime_start &&
-        II->getIntrinsicID() != Intrinsic::lifetime_end)
+    if (!II->isLifetimeStartOrEnd())
       return false;
   }
   return true;
@@ -3895,6 +3922,7 @@ bool llvm::isSafeToSpeculativelyExecute(const Value *V,
   case Instruction::VAArg:
   case Instruction::Alloca:
   case Instruction::Invoke:
+  case Instruction::CallBr:
   case Instruction::PHI:
   case Instruction::Store:
   case Instruction::Ret:
@@ -4327,7 +4355,7 @@ bool llvm::isGuaranteedToTransferExecutionToSuccessor(const Instruction *I) {
 }
 
 bool llvm::isGuaranteedToTransferExecutionToSuccessor(const BasicBlock *BB) {
-  // TODO: This is slightly consdervative for invoke instruction since exiting
+  // TODO: This is slightly conservative for invoke instruction since exiting
   // via an exception *is* normal control for them.
   for (auto I = BB->begin(), E = BB->end(); I != E; ++I)
     if (!isGuaranteedToTransferExecutionToSuccessor(&*I))
@@ -5236,21 +5264,16 @@ static bool isMatchingOps(const Value *ALHS, const Value *ARHS,
   return IsMatchingOps || IsSwappedOps;
 }
 
-/// Return true if "icmp1 APred ALHS ARHS" implies "icmp2 BPred BLHS BRHS" is
-/// true.  Return false if "icmp1 APred ALHS ARHS" implies "icmp2 BPred BLHS
-/// BRHS" is false.  Otherwise, return None if we can't infer anything.
+/// Return true if "icmp1 APred X, Y" implies "icmp2 BPred X, Y" is true.
+/// Return false if "icmp1 APred X, Y" implies "icmp2 BPred X, Y" is false.
+/// Otherwise, return None if we can't infer anything.
 static Optional<bool> isImpliedCondMatchingOperands(CmpInst::Predicate APred,
-                                                    const Value *ALHS,
-                                                    const Value *ARHS,
                                                     CmpInst::Predicate BPred,
-                                                    const Value *BLHS,
-                                                    const Value *BRHS,
-                                                    bool IsSwappedOps) {
-  // Canonicalize the operands so they're matching.
-  if (IsSwappedOps) {
-    std::swap(BLHS, BRHS);
+                                                    bool AreSwappedOps) {
+  // Canonicalize the predicate as if the operands were not commuted.
+  if (AreSwappedOps)
     BPred = ICmpInst::getSwappedPredicate(BPred);
-  }
+
   if (CmpInst::isImpliedTrueByMatchingCmp(APred, BPred))
     return true;
   if (CmpInst::isImpliedFalseByMatchingCmp(APred, BPred))
@@ -5259,15 +5282,14 @@ static Optional<bool> isImpliedCondMatchingOperands(CmpInst::Predicate APred,
   return None;
 }
 
-/// Return true if "icmp1 APred ALHS C1" implies "icmp2 BPred BLHS C2" is
-/// true.  Return false if "icmp1 APred ALHS C1" implies "icmp2 BPred BLHS
-/// C2" is false.  Otherwise, return None if we can't infer anything.
+/// Return true if "icmp APred X, C1" implies "icmp BPred X, C2" is true.
+/// Return false if "icmp APred X, C1" implies "icmp BPred X, C2" is false.
+/// Otherwise, return None if we can't infer anything.
 static Optional<bool>
-isImpliedCondMatchingImmOperands(CmpInst::Predicate APred, const Value *ALHS,
+isImpliedCondMatchingImmOperands(CmpInst::Predicate APred,
                                  const ConstantInt *C1,
                                  CmpInst::Predicate BPred,
-                                 const Value *BLHS, const ConstantInt *C2) {
-  assert(ALHS == BLHS && "LHS operands must match.");
+                                 const ConstantInt *C2) {
   ConstantRange DomCR =
       ConstantRange::makeExactICmpRegion(APred, C1->getValue());
   ConstantRange CR =
@@ -5299,10 +5321,10 @@ static Optional<bool> isImpliedCondICmps(const ICmpInst *LHS,
   ICmpInst::Predicate BPred = RHS->getPredicate();
 
   // Can we infer anything when the two compares have matching operands?
-  bool IsSwappedOps;
-  if (isMatchingOps(ALHS, ARHS, BLHS, BRHS, IsSwappedOps)) {
+  bool AreSwappedOps;
+  if (isMatchingOps(ALHS, ARHS, BLHS, BRHS, AreSwappedOps)) {
     if (Optional<bool> Implication = isImpliedCondMatchingOperands(
-            APred, ALHS, ARHS, BPred, BLHS, BRHS, IsSwappedOps))
+            APred, BPred, AreSwappedOps))
       return Implication;
     // No amount of additional analysis will infer the second condition, so
     // early exit.
@@ -5313,8 +5335,7 @@ static Optional<bool> isImpliedCondICmps(const ICmpInst *LHS,
   // constants (not necessarily matching)?
   if (ALHS == BLHS && isa<ConstantInt>(ARHS) && isa<ConstantInt>(BRHS)) {
     if (Optional<bool> Implication = isImpliedCondMatchingImmOperands(
-            APred, ALHS, cast<ConstantInt>(ARHS), BPred, BLHS,
-            cast<ConstantInt>(BRHS)))
+            APred, cast<ConstantInt>(ARHS), BPred, cast<ConstantInt>(BRHS)))
       return Implication;
     // No amount of additional analysis will infer the second condition, so
     // early exit.
diff --git a/lib/Analysis/VectorUtils.cpp b/lib/Analysis/VectorUtils.cpp
index 9ebb01684c8a5d4ab1568031872abe122569b92e..f64ee6a0fb9a7d5a909af9726b5ef328523bdb78 100644
--- a/lib/Analysis/VectorUtils.cpp
+++ b/lib/Analysis/VectorUtils.cpp
@@ -1,9 +1,8 @@
 //===----------- VectorUtils.cpp - Vectorizer utility functions -----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -49,6 +48,10 @@ bool llvm::isTriviallyVectorizable(Intrinsic::ID ID) {
   case Intrinsic::cttz:
   case Intrinsic::fshl:
   case Intrinsic::fshr:
+  case Intrinsic::sadd_sat:
+  case Intrinsic::ssub_sat:
+  case Intrinsic::uadd_sat:
+  case Intrinsic::usub_sat:
   case Intrinsic::sqrt: // Begin floating-point.
   case Intrinsic::sin:
   case Intrinsic::cos:
@@ -464,16 +467,100 @@ llvm::computeMinimumValueSizes(ArrayRef<BasicBlock *> Blocks, DemandedBits &DB,
   return MinBWs;
 }
 
+/// Add all access groups in @p AccGroups to @p List.
+template <typename ListT>
+static void addToAccessGroupList(ListT &List, MDNode *AccGroups) {
+  // Interpret an access group as a list containing itself.
+  if (AccGroups->getNumOperands() == 0) {
+    assert(isValidAsAccessGroup(AccGroups) && "Node must be an access group");
+    List.insert(AccGroups);
+    return;
+  }
+
+  for (auto &AccGroupListOp : AccGroups->operands()) {
+    auto *Item = cast<MDNode>(AccGroupListOp.get());
+    assert(isValidAsAccessGroup(Item) && "List item must be an access group");
+    List.insert(Item);
+  }
+}
+
+MDNode *llvm::uniteAccessGroups(MDNode *AccGroups1, MDNode *AccGroups2) {
+  if (!AccGroups1)
+    return AccGroups2;
+  if (!AccGroups2)
+    return AccGroups1;
+  if (AccGroups1 == AccGroups2)
+    return AccGroups1;
+
+  SmallSetVector<Metadata *, 4> Union;
+  addToAccessGroupList(Union, AccGroups1);
+  addToAccessGroupList(Union, AccGroups2);
+
+  if (Union.size() == 0)
+    return nullptr;
+  if (Union.size() == 1)
+    return cast<MDNode>(Union.front());
+
+  LLVMContext &Ctx = AccGroups1->getContext();
+  return MDNode::get(Ctx, Union.getArrayRef());
+}
+
+MDNode *llvm::intersectAccessGroups(const Instruction *Inst1,
+                                    const Instruction *Inst2) {
+  bool MayAccessMem1 = Inst1->mayReadOrWriteMemory();
+  bool MayAccessMem2 = Inst2->mayReadOrWriteMemory();
+
+  if (!MayAccessMem1 && !MayAccessMem2)
+    return nullptr;
+  if (!MayAccessMem1)
+    return Inst2->getMetadata(LLVMContext::MD_access_group);
+  if (!MayAccessMem2)
+    return Inst1->getMetadata(LLVMContext::MD_access_group);
+
+  MDNode *MD1 = Inst1->getMetadata(LLVMContext::MD_access_group);
+  MDNode *MD2 = Inst2->getMetadata(LLVMContext::MD_access_group);
+  if (!MD1 || !MD2)
+    return nullptr;
+  if (MD1 == MD2)
+    return MD1;
+
+  // Use set for scalable 'contains' check.
+  SmallPtrSet<Metadata *, 4> AccGroupSet2;
+  addToAccessGroupList(AccGroupSet2, MD2);
+
+  SmallVector<Metadata *, 4> Intersection;
+  if (MD1->getNumOperands() == 0) {
+    assert(isValidAsAccessGroup(MD1) && "Node must be an access group");
+    if (AccGroupSet2.count(MD1))
+      Intersection.push_back(MD1);
+  } else {
+    for (const MDOperand &Node : MD1->operands()) {
+      auto *Item = cast<MDNode>(Node.get());
+      assert(isValidAsAccessGroup(Item) && "List item must be an access group");
+      if (AccGroupSet2.count(Item))
+        Intersection.push_back(Item);
+    }
+  }
+
+  if (Intersection.size() == 0)
+    return nullptr;
+  if (Intersection.size() == 1)
+    return cast<MDNode>(Intersection.front());
+
+  LLVMContext &Ctx = Inst1->getContext();
+  return MDNode::get(Ctx, Intersection);
+}
+
 /// \returns \p I after propagating metadata from \p VL.
 Instruction *llvm::propagateMetadata(Instruction *Inst, ArrayRef<Value *> VL) {
   Instruction *I0 = cast<Instruction>(VL[0]);
   SmallVector<std::pair<unsigned, MDNode *>, 4> Metadata;
   I0->getAllMetadataOtherThanDebugLoc(Metadata);
 
-  for (auto Kind :
-       {LLVMContext::MD_tbaa, LLVMContext::MD_alias_scope,
-        LLVMContext::MD_noalias, LLVMContext::MD_fpmath,
-        LLVMContext::MD_nontemporal, LLVMContext::MD_invariant_load}) {
+  for (auto Kind : {LLVMContext::MD_tbaa, LLVMContext::MD_alias_scope,
+                    LLVMContext::MD_noalias, LLVMContext::MD_fpmath,
+                    LLVMContext::MD_nontemporal, LLVMContext::MD_invariant_load,
+                    LLVMContext::MD_access_group}) {
     MDNode *MD = I0->getMetadata(Kind);
 
     for (int J = 1, E = VL.size(); MD && J != E; ++J) {
@@ -494,6 +581,9 @@ Instruction *llvm::propagateMetadata(Instruction *Inst, ArrayRef<Value *> VL) {
       case LLVMContext::MD_invariant_load:
         MD = MDNode::intersect(MD, IMD);
         break;
+      case LLVMContext::MD_access_group:
+        MD = intersectAccessGroups(Inst, IJ);
+        break;
       default:
         llvm_unreachable("unhandled metadata");
       }
@@ -901,7 +991,7 @@ void InterleavedAccessInfo::analyzeInterleaving(
     // that all the pointers in the group don't wrap.
     // So we check only group member 0 (which is always guaranteed to exist),
     // and group member Factor - 1; If the latter doesn't exist we rely on
-    // peeling (if it is a non-reveresed accsess -- see Case 3).
+    // peeling (if it is a non-reversed accsess -- see Case 3).
     Value *FirstMemberPtr = getLoadStorePointerOperand(Group->getMember(0));
     if (!getPtrStride(PSE, FirstMemberPtr, TheLoop, Strides, /*Assume=*/false,
                       /*ShouldCheckWrap=*/true)) {
diff --git a/lib/AsmParser/LLLexer.cpp b/lib/AsmParser/LLLexer.cpp
index eab7ec81953609cc050f7b7af47307ebac74d2ae..461117c92b86f5255d8a947ef246dc3334a6642e 100644
--- a/lib/AsmParser/LLLexer.cpp
+++ b/lib/AsmParser/LLLexer.cpp
@@ -1,9 +1,8 @@
 //===- LLLexer.cpp - Lexer for .ll Files ----------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -859,6 +858,7 @@ lltok::Kind LLLexer::LexIdentifier() {
   INSTKEYWORD(invoke,      Invoke);
   INSTKEYWORD(resume,      Resume);
   INSTKEYWORD(unreachable, Unreachable);
+  INSTKEYWORD(callbr,      CallBr);
 
   INSTKEYWORD(alloca,      Alloca);
   INSTKEYWORD(load,        Load);
diff --git a/lib/AsmParser/LLLexer.h b/lib/AsmParser/LLLexer.h
index 21deb6e08910bda8e99fbead656615ae06e0edcc..4d3a2920e937475ece2c2878a7476ad30647d7c1 100644
--- a/lib/AsmParser/LLLexer.h
+++ b/lib/AsmParser/LLLexer.h
@@ -1,9 +1,8 @@
 //===- LLLexer.h - Lexer for LLVM Assembly Files ----------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/AsmParser/LLParser.cpp b/lib/AsmParser/LLParser.cpp
index f887372060b5344a2ed11e442c788414573b2bee..2a36bb189840964e1fcd3975412dfc6997aeb4be 100644
--- a/lib/AsmParser/LLParser.cpp
+++ b/lib/AsmParser/LLParser.cpp
@@ -1,9 +1,8 @@
 //===-- LLParser.cpp - Parser Class ---------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -164,6 +163,14 @@ bool LLParser::ValidateEndOfModule() {
       AS = AS.addAttributes(Context, AttributeList::FunctionIndex,
                             AttributeSet::get(Context, FnAttrs));
       II->setAttributes(AS);
+    } else if (CallBrInst *CBI = dyn_cast<CallBrInst>(V)) {
+      AttributeList AS = CBI->getAttributes();
+      AttrBuilder FnAttrs(AS.getFnAttributes());
+      AS = AS.removeAttributes(Context, AttributeList::FunctionIndex);
+      FnAttrs.merge(B);
+      AS = AS.addAttributes(Context, AttributeList::FunctionIndex,
+                            AttributeSet::get(Context, FnAttrs));
+      CBI->setAttributes(AS);
     } else if (auto *GV = dyn_cast<GlobalVariable>(V)) {
       AttrBuilder Attrs(GV->getAttributes());
       Attrs.merge(B);
@@ -5567,6 +5574,7 @@ int LLParser::ParseInstruction(Instruction *&Inst, BasicBlock *BB,
   case lltok::kw_catchswitch: return ParseCatchSwitch(Inst, PFS);
   case lltok::kw_catchpad:    return ParseCatchPad(Inst, PFS);
   case lltok::kw_cleanuppad:  return ParseCleanupPad(Inst, PFS);
+  case lltok::kw_callbr:      return ParseCallBr(Inst, PFS);
   // Unary Operators.
   case lltok::kw_fneg: {
     FastMathFlags FMF = EatFastMathFlagsIfPresent();
@@ -6185,6 +6193,124 @@ bool LLParser::ParseUnaryOp(Instruction *&Inst, PerFunctionState &PFS,
   return false;
 }
 
+/// ParseCallBr
+///   ::= 'callbr' OptionalCallingConv OptionalAttrs Type Value ParamList
+///       OptionalAttrs OptionalOperandBundles 'to' TypeAndValue
+///       '[' LabelList ']'
+bool LLParser::ParseCallBr(Instruction *&Inst, PerFunctionState &PFS) {
+  LocTy CallLoc = Lex.getLoc();
+  AttrBuilder RetAttrs, FnAttrs;
+  std::vector<unsigned> FwdRefAttrGrps;
+  LocTy NoBuiltinLoc;
+  unsigned CC;
+  Type *RetType = nullptr;
+  LocTy RetTypeLoc;
+  ValID CalleeID;
+  SmallVector<ParamInfo, 16> ArgList;
+  SmallVector<OperandBundleDef, 2> BundleList;
+
+  BasicBlock *DefaultDest;
+  if (ParseOptionalCallingConv(CC) || ParseOptionalReturnAttrs(RetAttrs) ||
+      ParseType(RetType, RetTypeLoc, true /*void allowed*/) ||
+      ParseValID(CalleeID) || ParseParameterList(ArgList, PFS) ||
+      ParseFnAttributeValuePairs(FnAttrs, FwdRefAttrGrps, false,
+                                 NoBuiltinLoc) ||
+      ParseOptionalOperandBundles(BundleList, PFS) ||
+      ParseToken(lltok::kw_to, "expected 'to' in callbr") ||
+      ParseTypeAndBasicBlock(DefaultDest, PFS) ||
+      ParseToken(lltok::lsquare, "expected '[' in callbr"))
+    return true;
+
+  // Parse the destination list.
+  SmallVector<BasicBlock *, 16> IndirectDests;
+
+  if (Lex.getKind() != lltok::rsquare) {
+    BasicBlock *DestBB;
+    if (ParseTypeAndBasicBlock(DestBB, PFS))
+      return true;
+    IndirectDests.push_back(DestBB);
+
+    while (EatIfPresent(lltok::comma)) {
+      if (ParseTypeAndBasicBlock(DestBB, PFS))
+        return true;
+      IndirectDests.push_back(DestBB);
+    }
+  }
+
+  if (ParseToken(lltok::rsquare, "expected ']' at end of block list"))
+    return true;
+
+  // If RetType is a non-function pointer type, then this is the short syntax
+  // for the call, which means that RetType is just the return type.  Infer the
+  // rest of the function argument types from the arguments that are present.
+  FunctionType *Ty = dyn_cast<FunctionType>(RetType);
+  if (!Ty) {
+    // Pull out the types of all of the arguments...
+    std::vector<Type *> ParamTypes;
+    for (unsigned i = 0, e = ArgList.size(); i != e; ++i)
+      ParamTypes.push_back(ArgList[i].V->getType());
+
+    if (!FunctionType::isValidReturnType(RetType))
+      return Error(RetTypeLoc, "Invalid result type for LLVM function");
+
+    Ty = FunctionType::get(RetType, ParamTypes, false);
+  }
+
+  CalleeID.FTy = Ty;
+
+  // Look up the callee.
+  Value *Callee;
+  if (ConvertValIDToValue(PointerType::getUnqual(Ty), CalleeID, Callee, &PFS,
+                          /*IsCall=*/true))
+    return true;
+
+  if (isa<InlineAsm>(Callee) && !Ty->getReturnType()->isVoidTy())
+    return Error(RetTypeLoc, "asm-goto outputs not supported");
+
+  // Set up the Attribute for the function.
+  SmallVector<Value *, 8> Args;
+  SmallVector<AttributeSet, 8> ArgAttrs;
+
+  // Loop through FunctionType's arguments and ensure they are specified
+  // correctly.  Also, gather any parameter attributes.
+  FunctionType::param_iterator I = Ty->param_begin();
+  FunctionType::param_iterator E = Ty->param_end();
+  for (unsigned i = 0, e = ArgList.size(); i != e; ++i) {
+    Type *ExpectedTy = nullptr;
+    if (I != E) {
+      ExpectedTy = *I++;
+    } else if (!Ty->isVarArg()) {
+      return Error(ArgList[i].Loc, "too many arguments specified");
+    }
+
+    if (ExpectedTy && ExpectedTy != ArgList[i].V->getType())
+      return Error(ArgList[i].Loc, "argument is not of expected type '" +
+                                       getTypeString(ExpectedTy) + "'");
+    Args.push_back(ArgList[i].V);
+    ArgAttrs.push_back(ArgList[i].Attrs);
+  }
+
+  if (I != E)
+    return Error(CallLoc, "not enough parameters specified for call");
+
+  if (FnAttrs.hasAlignmentAttr())
+    return Error(CallLoc, "callbr instructions may not have an alignment");
+
+  // Finish off the Attribute and check them
+  AttributeList PAL =
+      AttributeList::get(Context, AttributeSet::get(Context, FnAttrs),
+                         AttributeSet::get(Context, RetAttrs), ArgAttrs);
+
+  CallBrInst *CBI =
+      CallBrInst::Create(Ty, Callee, DefaultDest, IndirectDests, Args,
+                         BundleList);
+  CBI->setCallingConv(CC);
+  CBI->setAttributes(PAL);
+  ForwardRefAttrGroups[CBI] = FwdRefAttrGrps;
+  Inst = CBI;
+  return false;
+}
+
 //===----------------------------------------------------------------------===//
 // Binary Operators.
 //===----------------------------------------------------------------------===//
@@ -6816,6 +6942,7 @@ int LLParser::ParseAtomicRMW(Instruction *&Inst, PerFunctionState &PFS) {
   AtomicOrdering Ordering = AtomicOrdering::NotAtomic;
   SyncScope::ID SSID = SyncScope::System;
   bool isVolatile = false;
+  bool IsFP = false;
   AtomicRMWInst::BinOp Operation;
 
   if (EatIfPresent(lltok::kw_volatile))
@@ -6834,6 +6961,14 @@ int LLParser::ParseAtomicRMW(Instruction *&Inst, PerFunctionState &PFS) {
   case lltok::kw_min: Operation = AtomicRMWInst::Min; break;
   case lltok::kw_umax: Operation = AtomicRMWInst::UMax; break;
   case lltok::kw_umin: Operation = AtomicRMWInst::UMin; break;
+  case lltok::kw_fadd:
+    Operation = AtomicRMWInst::FAdd;
+    IsFP = true;
+    break;
+  case lltok::kw_fsub:
+    Operation = AtomicRMWInst::FSub;
+    IsFP = true;
+    break;
   }
   Lex.Lex();  // Eat the operation.
 
@@ -6850,10 +6985,25 @@ int LLParser::ParseAtomicRMW(Instruction *&Inst, PerFunctionState &PFS) {
   if (cast<PointerType>(Ptr->getType())->getElementType() != Val->getType())
     return Error(ValLoc, "atomicrmw value and pointer type do not match");
 
-  if (!Val->getType()->isIntegerTy()) {
-    return Error(ValLoc, "atomicrmw " +
-                 AtomicRMWInst::getOperationName(Operation) +
-                 " operand must be an integer");
+  if (Operation == AtomicRMWInst::Xchg) {
+    if (!Val->getType()->isIntegerTy() &&
+        !Val->getType()->isFloatingPointTy()) {
+      return Error(ValLoc, "atomicrmw " +
+                   AtomicRMWInst::getOperationName(Operation) +
+                   " operand must be an integer or floating point type");
+    }
+  } else if (IsFP) {
+    if (!Val->getType()->isFloatingPointTy()) {
+      return Error(ValLoc, "atomicrmw " +
+                   AtomicRMWInst::getOperationName(Operation) +
+                   " operand must be a floating point type");
+    }
+  } else {
+    if (!Val->getType()->isIntegerTy()) {
+      return Error(ValLoc, "atomicrmw " +
+                   AtomicRMWInst::getOperationName(Operation) +
+                   " operand must be an integer");
+    }
   }
 
   unsigned Size = Val->getType()->getPrimitiveSizeInBits();
@@ -7519,7 +7669,7 @@ bool LLParser::ParseArgs(std::vector<uint64_t> &Args) {
   return false;
 }
 
-auto FwdVIRef = (GlobalValueSummaryMapTy::value_type *)-8;
+static const auto FwdVIRef = (GlobalValueSummaryMapTy::value_type *)-8;
 
 static void resolveFwdRef(ValueInfo *Fwd, ValueInfo &Resolved) {
   bool ReadOnly = Fwd->isReadOnly();
diff --git a/lib/AsmParser/LLParser.h b/lib/AsmParser/LLParser.h
index 5a0fc297265d4d1dc649bc31084de1f1dcbdd3f9..95aea0c775a0e825c62b72b3a51650a2c62ed3a6 100644
--- a/lib/AsmParser/LLParser.h
+++ b/lib/AsmParser/LLParser.h
@@ -1,9 +1,8 @@
 //===-- LLParser.h - Parser Class -------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -571,6 +570,7 @@ namespace llvm {
     bool ParseCatchSwitch(Instruction *&Inst, PerFunctionState &PFS);
     bool ParseCatchPad(Instruction *&Inst, PerFunctionState &PFS);
     bool ParseCleanupPad(Instruction *&Inst, PerFunctionState &PFS);
+    bool ParseCallBr(Instruction *&Inst, PerFunctionState &PFS);
 
     bool ParseUnaryOp(Instruction *&Inst, PerFunctionState &PFS, unsigned Opc,
                       unsigned OperandType);
diff --git a/lib/AsmParser/LLToken.h b/lib/AsmParser/LLToken.h
index c2e2795a9467be6c4026aa5ea29091dd3c242163..88eeae11a4b663385f7d27335b44e177a4d7eaae 100644
--- a/lib/AsmParser/LLToken.h
+++ b/lib/AsmParser/LLToken.h
@@ -1,9 +1,8 @@
 //===- LLToken.h - Token Codes for LLVM Assembly Files ----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -328,6 +327,7 @@ enum Kind {
   kw_catchret,
   kw_catchpad,
   kw_cleanuppad,
+  kw_callbr,
 
   kw_alloca,
   kw_load,
diff --git a/lib/AsmParser/LLVMBuild.txt b/lib/AsmParser/LLVMBuild.txt
index 82dba8c15bb8d1c2da98fe9e76e3221a49445342..cacb753f6a9dfd9790ec8b6daaff4fa2ffdf2ea3 100644
--- a/lib/AsmParser/LLVMBuild.txt
+++ b/lib/AsmParser/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/AsmParser/LLVMBuild.txt ----------------------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/AsmParser/Parser.cpp b/lib/AsmParser/Parser.cpp
index 1205dff24e8a5b2768388f97f6a6173cf32ed878..b13c6237f411881539504df9d32a2f94e84dece5 100644
--- a/lib/AsmParser/Parser.cpp
+++ b/lib/AsmParser/Parser.cpp
@@ -1,9 +1,8 @@
 //===- Parser.cpp - Main dispatch module for the Parser library -----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/BinaryFormat/AMDGPUMetadataVerifier.cpp b/lib/BinaryFormat/AMDGPUMetadataVerifier.cpp
index b789f646b5f6f920880ff76c912d1f31c8730987..48f996ef4ef5fbc4817fbc1cf38a055c7899ac1e 100644
--- a/lib/BinaryFormat/AMDGPUMetadataVerifier.cpp
+++ b/lib/BinaryFormat/AMDGPUMetadataVerifier.cpp
@@ -1,9 +1,8 @@
 //===- AMDGPUMetadataVerifier.cpp - MsgPack Types ---------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/BinaryFormat/Dwarf.cpp b/lib/BinaryFormat/Dwarf.cpp
index fe8ce2bd7aa36558d586415516f86abc7066517a..73f04a3ed2897643052d6c5352a359876c62912c 100644
--- a/lib/BinaryFormat/Dwarf.cpp
+++ b/lib/BinaryFormat/Dwarf.cpp
@@ -1,9 +1,8 @@
 //===-- llvm/BinaryFormat/Dwarf.cpp - Dwarf Framework ------------*- C++-*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -301,7 +300,7 @@ StringRef llvm::dwarf::LanguageString(unsigned Language) {
   switch (Language) {
   default:
     return StringRef();
-#define HANDLE_DW_LANG(ID, NAME, VERSION, VENDOR)                              \
+#define HANDLE_DW_LANG(ID, NAME, LOWER_BOUND, VERSION, VENDOR)                 \
   case DW_LANG_##NAME:                                                         \
     return "DW_LANG_" #NAME;
 #include "llvm/BinaryFormat/Dwarf.def"
@@ -310,7 +309,7 @@ StringRef llvm::dwarf::LanguageString(unsigned Language) {
 
 unsigned llvm::dwarf::getLanguage(StringRef LanguageString) {
   return StringSwitch<unsigned>(LanguageString)
-#define HANDLE_DW_LANG(ID, NAME, VERSION, VENDOR)                              \
+#define HANDLE_DW_LANG(ID, NAME, LOWER_BOUND, VERSION, VENDOR)                 \
   .Case("DW_LANG_" #NAME, DW_LANG_##NAME)
 #include "llvm/BinaryFormat/Dwarf.def"
       .Default(0);
@@ -320,7 +319,7 @@ unsigned llvm::dwarf::LanguageVersion(dwarf::SourceLanguage Lang) {
   switch (Lang) {
   default:
     return 0;
-#define HANDLE_DW_LANG(ID, NAME, VERSION, VENDOR)                              \
+#define HANDLE_DW_LANG(ID, NAME, LOWER_BOUND, VERSION, VENDOR)                 \
   case DW_LANG_##NAME:                                                         \
     return VERSION;
 #include "llvm/BinaryFormat/Dwarf.def"
@@ -331,13 +330,24 @@ unsigned llvm::dwarf::LanguageVendor(dwarf::SourceLanguage Lang) {
   switch (Lang) {
   default:
     return 0;
-#define HANDLE_DW_LANG(ID, NAME, VERSION, VENDOR)                              \
+#define HANDLE_DW_LANG(ID, NAME, LOWER_BOUND, VERSION, VENDOR)                 \
   case DW_LANG_##NAME:                                                         \
     return DWARF_VENDOR_##VENDOR;
 #include "llvm/BinaryFormat/Dwarf.def"
   }
 }
 
+Optional<unsigned> llvm::dwarf::LanguageLowerBound(dwarf::SourceLanguage Lang) {
+  switch (Lang) {
+  default:
+    return None;
+#define HANDLE_DW_LANG(ID, NAME, LOWER_BOUND, VERSION, VENDOR)                 \
+  case DW_LANG_##NAME:                                                         \
+    return LOWER_BOUND;
+#include "llvm/BinaryFormat/Dwarf.def"
+  }
+}
+
 StringRef llvm::dwarf::CaseString(unsigned Case) {
   switch (Case) {
   case DW_ID_case_sensitive:
diff --git a/lib/BinaryFormat/LLVMBuild.txt b/lib/BinaryFormat/LLVMBuild.txt
index d7d4dcb5f23d38d7cb2e2bd2ce4ab6637bb8a649..7e5dccde41f88c9736118110b0017acaeb73255c 100644
--- a/lib/BinaryFormat/LLVMBuild.txt
+++ b/lib/BinaryFormat/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/BinaryFormat/LLVMBuild.txt -------------------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/BinaryFormat/Magic.cpp b/lib/BinaryFormat/Magic.cpp
index 78efa6ec87be8c614d22a565c47eae296466e31b..5e9d6e73257a3c889fc8584c7aac86d4ea6f9fe6 100644
--- a/lib/BinaryFormat/Magic.cpp
+++ b/lib/BinaryFormat/Magic.cpp
@@ -1,9 +1,8 @@
 //===- llvm/BinaryFormat/Magic.cpp - File magic identification --*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/BinaryFormat/MsgPackReader.cpp b/lib/BinaryFormat/MsgPackReader.cpp
index b510fdba960808c9742e7e0824dbe01de9498303..872a6e0e29f8317450684be860b4b3e3bddc145b 100644
--- a/lib/BinaryFormat/MsgPackReader.cpp
+++ b/lib/BinaryFormat/MsgPackReader.cpp
@@ -1,9 +1,8 @@
 //===- MsgPackReader.cpp - Simple MsgPack reader ----------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
diff --git a/lib/BinaryFormat/MsgPackTypes.cpp b/lib/BinaryFormat/MsgPackTypes.cpp
index 4a8f70b10fb83e4c32cc01996fe1f2fc080dbf31..a887e168bb6d91c7897c894d3abc796388f4f128 100644
--- a/lib/BinaryFormat/MsgPackTypes.cpp
+++ b/lib/BinaryFormat/MsgPackTypes.cpp
@@ -1,9 +1,8 @@
 //===- MsgPackTypes.cpp - MsgPack Types -------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/BinaryFormat/MsgPackWriter.cpp b/lib/BinaryFormat/MsgPackWriter.cpp
index d024bb0fcdb2302cb03c8c8a176ec747f60e8cdb..b4d70e8f78c12e0f3f4d63ec6dd5402349591954 100644
--- a/lib/BinaryFormat/MsgPackWriter.cpp
+++ b/lib/BinaryFormat/MsgPackWriter.cpp
@@ -1,9 +1,8 @@
 //===- MsgPackWriter.cpp - Simple MsgPack writer ----------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
diff --git a/lib/BinaryFormat/Wasm.cpp b/lib/BinaryFormat/Wasm.cpp
index 94d40bf02a390c0438d7ec2e1f72375c91964a6e..1d36b2cb6fbdb4e1a639da07f87b9feb150dfc09 100644
--- a/lib/BinaryFormat/Wasm.cpp
+++ b/lib/BinaryFormat/Wasm.cpp
@@ -1,16 +1,15 @@
 //===-- llvm/BinaryFormat/Wasm.cpp -------------------------------*- C++-*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #include "llvm/BinaryFormat/Wasm.h"
 
-std::string llvm::wasm::toString(wasm::WasmSymbolType type) {
-  switch (type) {
+std::string llvm::wasm::toString(wasm::WasmSymbolType Type) {
+  switch (Type) {
   case wasm::WASM_SYMBOL_TYPE_FUNCTION:
     return "WASM_SYMBOL_TYPE_FUNCTION";
   case wasm::WASM_SYMBOL_TYPE_GLOBAL:
@@ -25,8 +24,8 @@ std::string llvm::wasm::toString(wasm::WasmSymbolType type) {
   llvm_unreachable("unknown symbol type");
 }
 
-std::string llvm::wasm::relocTypetoString(uint32_t type) {
-  switch (type) {
+std::string llvm::wasm::relocTypetoString(uint32_t Type) {
+  switch (Type) {
 #define WASM_RELOC(NAME, VALUE)                                                \
   case VALUE:                                                                  \
     return #NAME;
diff --git a/lib/Bitcode/LLVMBuild.txt b/lib/Bitcode/LLVMBuild.txt
index af9936bbe82967efad3dc7635d4a0f1cebdf5c8a..3698fe6172c30ca8bcc995414cebc8e95454da1b 100644
--- a/lib/Bitcode/LLVMBuild.txt
+++ b/lib/Bitcode/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/Bitcode/LLVMBuild.txt ------------------------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Bitcode/Reader/BitReader.cpp b/lib/Bitcode/Reader/BitReader.cpp
index 3ec45956b3e50f8dad03ecf95030401c565a3ced..5ac893aef14e4686c2e79887fe74a80ac84527cd 100644
--- a/lib/Bitcode/Reader/BitReader.cpp
+++ b/lib/Bitcode/Reader/BitReader.cpp
@@ -1,9 +1,8 @@
 //===-- BitReader.cpp -----------------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Bitcode/Reader/BitcodeReader.cpp b/lib/Bitcode/Reader/BitcodeReader.cpp
index 846ce3a4f7aa72042cf5d80022f9b626c7ab074a..b94bb66a0d66c1043aba7b90f80acfb512b640d1 100644
--- a/lib/Bitcode/Reader/BitcodeReader.cpp
+++ b/lib/Bitcode/Reader/BitcodeReader.cpp
@@ -1,9 +1,8 @@
 //===- BitcodeReader.cpp - Internal BitcodeReader implementation ----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -1035,6 +1034,8 @@ static AtomicRMWInst::BinOp getDecodedRMWOperation(unsigned Val) {
   case bitc::RMW_MIN: return AtomicRMWInst::Min;
   case bitc::RMW_UMAX: return AtomicRMWInst::UMax;
   case bitc::RMW_UMIN: return AtomicRMWInst::UMin;
+  case bitc::RMW_FADD: return AtomicRMWInst::FAdd;
+  case bitc::RMW_FSUB: return AtomicRMWInst::FSub;
   }
 }
 
@@ -2423,16 +2424,18 @@ Error BitcodeReader::parseConstants() {
         Elts.push_back(ValueList.getConstantFwdRef(Record[OpNum++], ElTy));
       }
 
-      if (PointeeType &&
-          PointeeType !=
-              cast<PointerType>(Elts[0]->getType()->getScalarType())
-                  ->getElementType())
-        return error("Explicit gep operator type does not match pointee type "
-                     "of pointer operand");
-
       if (Elts.size() < 1)
         return error("Invalid gep with no operands");
 
+      Type *ImplicitPointeeType =
+          cast<PointerType>(Elts[0]->getType()->getScalarType())
+              ->getElementType();
+      if (!PointeeType)
+        PointeeType = ImplicitPointeeType;
+      else if (PointeeType != ImplicitPointeeType)
+        return error("Explicit gep operator type does not match pointee type "
+                     "of pointer operand");
+
       ArrayRef<Constant *> Indices(Elts.begin() + 1, Elts.end());
       V = ConstantExpr::getGetElementPtr(PointeeType, Elts[0], Indices,
                                          InBounds, InRangeIndex);
@@ -3720,16 +3723,16 @@ Error BitcodeReader::parseFunctionBody(Function *F) {
           return error("EXTRACTVAL: Invalid type");
         if ((unsigned)Index != Index)
           return error("Invalid value");
-        if (IsStruct && Index >= CurTy->subtypes().size())
+        if (IsStruct && Index >= CurTy->getStructNumElements())
           return error("EXTRACTVAL: Invalid struct index");
         if (IsArray && Index >= CurTy->getArrayNumElements())
           return error("EXTRACTVAL: Invalid array index");
         EXTRACTVALIdx.push_back((unsigned)Index);
 
         if (IsStruct)
-          CurTy = CurTy->subtypes()[Index];
+          CurTy = CurTy->getStructElementType(Index);
         else
-          CurTy = CurTy->subtypes()[0];
+          CurTy = CurTy->getArrayElementType();
       }
 
       I = ExtractValueInst::Create(Agg, EXTRACTVALIdx);
@@ -3762,16 +3765,16 @@ Error BitcodeReader::parseFunctionBody(Function *F) {
           return error("INSERTVAL: Invalid type");
         if ((unsigned)Index != Index)
           return error("Invalid value");
-        if (IsStruct && Index >= CurTy->subtypes().size())
+        if (IsStruct && Index >= CurTy->getStructNumElements())
           return error("INSERTVAL: Invalid struct index");
         if (IsArray && Index >= CurTy->getArrayNumElements())
           return error("INSERTVAL: Invalid array index");
 
         INSERTVALIdx.push_back((unsigned)Index);
         if (IsStruct)
-          CurTy = CurTy->subtypes()[Index];
+          CurTy = CurTy->getStructElementType(Index);
         else
-          CurTy = CurTy->subtypes()[0];
+          CurTy = CurTy->getArrayElementType();
       }
 
       if (CurTy != Val->getType())
@@ -4210,7 +4213,8 @@ Error BitcodeReader::parseFunctionBody(Function *F) {
         }
       }
 
-      I = InvokeInst::Create(Callee, NormalBB, UnwindBB, Ops, OperandBundles);
+      I = InvokeInst::Create(FTy, Callee, NormalBB, UnwindBB, Ops,
+                             OperandBundles);
       OperandBundles.clear();
       InstructionList.push_back(I);
       cast<InvokeInst>(I)->setCallingConv(
@@ -4227,6 +4231,74 @@ Error BitcodeReader::parseFunctionBody(Function *F) {
       InstructionList.push_back(I);
       break;
     }
+    case bitc::FUNC_CODE_INST_CALLBR: {
+      // CALLBR: [attr, cc, norm, transfs, fty, fnid, args]
+      unsigned OpNum = 0;
+      AttributeList PAL = getAttributes(Record[OpNum++]);
+      unsigned CCInfo = Record[OpNum++];
+
+      BasicBlock *DefaultDest = getBasicBlock(Record[OpNum++]);
+      unsigned NumIndirectDests = Record[OpNum++];
+      SmallVector<BasicBlock *, 16> IndirectDests;
+      for (unsigned i = 0, e = NumIndirectDests; i != e; ++i)
+        IndirectDests.push_back(getBasicBlock(Record[OpNum++]));
+
+      FunctionType *FTy = nullptr;
+      if (CCInfo >> bitc::CALL_EXPLICIT_TYPE & 1 &&
+          !(FTy = dyn_cast<FunctionType>(getTypeByID(Record[OpNum++]))))
+        return error("Explicit call type is not a function type");
+
+      Value *Callee;
+      if (getValueTypePair(Record, OpNum, NextValueNo, Callee))
+        return error("Invalid record");
+
+      PointerType *OpTy = dyn_cast<PointerType>(Callee->getType());
+      if (!OpTy)
+        return error("Callee is not a pointer type");
+      if (!FTy) {
+        FTy = dyn_cast<FunctionType>(OpTy->getElementType());
+        if (!FTy)
+          return error("Callee is not of pointer to function type");
+      } else if (OpTy->getElementType() != FTy)
+        return error("Explicit call type does not match pointee type of "
+                     "callee operand");
+      if (Record.size() < FTy->getNumParams() + OpNum)
+        return error("Insufficient operands to call");
+
+      SmallVector<Value*, 16> Args;
+      // Read the fixed params.
+      for (unsigned i = 0, e = FTy->getNumParams(); i != e; ++i, ++OpNum) {
+        if (FTy->getParamType(i)->isLabelTy())
+          Args.push_back(getBasicBlock(Record[OpNum]));
+        else
+          Args.push_back(getValue(Record, OpNum, NextValueNo,
+                                  FTy->getParamType(i)));
+        if (!Args.back())
+          return error("Invalid record");
+      }
+
+      // Read type/value pairs for varargs params.
+      if (!FTy->isVarArg()) {
+        if (OpNum != Record.size())
+          return error("Invalid record");
+      } else {
+        while (OpNum != Record.size()) {
+          Value *Op;
+          if (getValueTypePair(Record, OpNum, NextValueNo, Op))
+            return error("Invalid record");
+          Args.push_back(Op);
+        }
+      }
+
+      I = CallBrInst::Create(FTy, Callee, DefaultDest, IndirectDests, Args,
+                             OperandBundles);
+      OperandBundles.clear();
+      InstructionList.push_back(I);
+      cast<CallBrInst>(I)->setCallingConv(
+          static_cast<CallingConv::ID>((0x7ff & CCInfo) >> bitc::CALL_CCONV));
+      cast<CallBrInst>(I)->setAttributes(PAL);
+      break;
+    }
     case bitc::FUNC_CODE_INST_UNREACHABLE: // UNREACHABLE
       I = new UnreachableInst(Context);
       InstructionList.push_back(I);
@@ -4402,7 +4474,7 @@ Error BitcodeReader::parseFunctionBody(Function *F) {
       unsigned Align;
       if (Error Err = parseAlignmentValue(Record[OpNum], Align))
         return Err;
-      I = new LoadInst(Op, "", Record[OpNum+1], Align, Ordering, SSID);
+      I = new LoadInst(Ty, Op, "", Record[OpNum + 1], Align, Ordering, SSID);
 
       InstructionList.push_back(I);
       break;
@@ -5294,18 +5366,30 @@ Error ModuleSummaryIndexBitcodeReader::parseEntireSummary(unsigned ID) {
       break;
     case bitc::FS_FLAGS: {  // [flags]
       uint64_t Flags = Record[0];
-      // Scan flags (set only on the combined index).
-      assert(Flags <= 0x3 && "Unexpected bits in flag");
+      // Scan flags.
+      assert(Flags <= 0x1f && "Unexpected bits in flag");
 
       // 1 bit: WithGlobalValueDeadStripping flag.
+      // Set on combined index only.
       if (Flags & 0x1)
         TheIndex.setWithGlobalValueDeadStripping();
       // 1 bit: SkipModuleByDistributedBackend flag.
+      // Set on combined index only.
       if (Flags & 0x2)
         TheIndex.setSkipModuleByDistributedBackend();
       // 1 bit: HasSyntheticEntryCounts flag.
+      // Set on combined index only.
       if (Flags & 0x4)
         TheIndex.setHasSyntheticEntryCounts();
+      // 1 bit: DisableSplitLTOUnit flag.
+      // Set on per module indexes. It is up to the client to validate
+      // the consistency of this flag across modules being linked.
+      if (Flags & 0x8)
+        TheIndex.setEnableSplitLTOUnit();
+      // 1 bit: PartiallySplitLTOUnits flag.
+      // Set on combined index only.
+      if (Flags & 0x10)
+        TheIndex.setPartiallySplitLTOUnits();
       break;
     }
     case bitc::FS_VALUE_GUID: { // [valueid, refguid]
@@ -5917,6 +6001,46 @@ Expected<std::unique_ptr<ModuleSummaryIndex>> BitcodeModule::getSummary() {
   return std::move(Index);
 }
 
+static Expected<bool> getEnableSplitLTOUnitFlag(BitstreamCursor &Stream,
+                                                unsigned ID) {
+  if (Stream.EnterSubBlock(ID))
+    return error("Invalid record");
+  SmallVector<uint64_t, 64> Record;
+
+  while (true) {
+    BitstreamEntry Entry = Stream.advanceSkippingSubblocks();
+
+    switch (Entry.Kind) {
+    case BitstreamEntry::SubBlock: // Handled for us already.
+    case BitstreamEntry::Error:
+      return error("Malformed block");
+    case BitstreamEntry::EndBlock:
+      // If no flags record found, conservatively return true to mimic
+      // behavior before this flag was added.
+      return true;
+    case BitstreamEntry::Record:
+      // The interesting case.
+      break;
+    }
+
+    // Look for the FS_FLAGS record.
+    Record.clear();
+    auto BitCode = Stream.readRecord(Entry.ID, Record);
+    switch (BitCode) {
+    default: // Default behavior: ignore.
+      break;
+    case bitc::FS_FLAGS: { // [flags]
+      uint64_t Flags = Record[0];
+      // Scan flags.
+      assert(Flags <= 0x1f && "Unexpected bits in flag");
+
+      return Flags & 0x8;
+    }
+    }
+  }
+  llvm_unreachable("Exit infinite loop");
+}
+
 // Check if the given bitcode buffer contains a global value summary block.
 Expected<BitcodeLTOInfo> BitcodeModule::getLTOInfo() {
   BitstreamCursor Stream(Buffer);
@@ -5932,14 +6056,27 @@ Expected<BitcodeLTOInfo> BitcodeModule::getLTOInfo() {
     case BitstreamEntry::Error:
       return error("Malformed block");
     case BitstreamEntry::EndBlock:
-      return BitcodeLTOInfo{/*IsThinLTO=*/false, /*HasSummary=*/false};
+      return BitcodeLTOInfo{/*IsThinLTO=*/false, /*HasSummary=*/false,
+                            /*EnableSplitLTOUnit=*/false};
 
     case BitstreamEntry::SubBlock:
-      if (Entry.ID == bitc::GLOBALVAL_SUMMARY_BLOCK_ID)
-        return BitcodeLTOInfo{/*IsThinLTO=*/true, /*HasSummary=*/true};
+      if (Entry.ID == bitc::GLOBALVAL_SUMMARY_BLOCK_ID) {
+        Expected<bool> EnableSplitLTOUnit =
+            getEnableSplitLTOUnitFlag(Stream, Entry.ID);
+        if (!EnableSplitLTOUnit)
+          return EnableSplitLTOUnit.takeError();
+        return BitcodeLTOInfo{/*IsThinLTO=*/true, /*HasSummary=*/true,
+                              *EnableSplitLTOUnit};
+      }
 
-      if (Entry.ID == bitc::FULL_LTO_GLOBALVAL_SUMMARY_BLOCK_ID)
-        return BitcodeLTOInfo{/*IsThinLTO=*/false, /*HasSummary=*/true};
+      if (Entry.ID == bitc::FULL_LTO_GLOBALVAL_SUMMARY_BLOCK_ID) {
+        Expected<bool> EnableSplitLTOUnit =
+            getEnableSplitLTOUnitFlag(Stream, Entry.ID);
+        if (!EnableSplitLTOUnit)
+          return EnableSplitLTOUnit.takeError();
+        return BitcodeLTOInfo{/*IsThinLTO=*/false, /*HasSummary=*/true,
+                              *EnableSplitLTOUnit};
+      }
 
       // Ignore other sub-blocks.
       if (Stream.SkipBlock())
diff --git a/lib/Bitcode/Reader/BitstreamReader.cpp b/lib/Bitcode/Reader/BitstreamReader.cpp
index 771cf3d927bc35882ca1a83d46c65fd6ce4339e4..985944738cbd2b72d099c42098ef39f07edd32d4 100644
--- a/lib/Bitcode/Reader/BitstreamReader.cpp
+++ b/lib/Bitcode/Reader/BitstreamReader.cpp
@@ -1,9 +1,8 @@
 //===- BitstreamReader.cpp - BitstreamReader implementation ---------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Bitcode/Reader/LLVMBuild.txt b/lib/Bitcode/Reader/LLVMBuild.txt
index c85a87bfebc87ec5f827e490b8b4264136723e95..c43f819750c5a2b7da1541bb73156bd23163bb76 100644
--- a/lib/Bitcode/Reader/LLVMBuild.txt
+++ b/lib/Bitcode/Reader/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/Bitcode/Reader/LLVMBuild.txt -----------------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Bitcode/Reader/MetadataLoader.cpp b/lib/Bitcode/Reader/MetadataLoader.cpp
index 3289aa0acddd9bc7ce9136df03496a7c74dec803..f6b26104149956da2de2047da18f15caf7af2c73 100644
--- a/lib/Bitcode/Reader/MetadataLoader.cpp
+++ b/lib/Bitcode/Reader/MetadataLoader.cpp
@@ -1,9 +1,8 @@
 //===- MetadataLoader.cpp - Internal BitcodeReader implementation ---------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Bitcode/Reader/MetadataLoader.h b/lib/Bitcode/Reader/MetadataLoader.h
index 07a77a086f3240b1c85e1dfb5ca4200157fa7bb3..fe2b202732499c63ed2053a67882eaa1f6815386 100644
--- a/lib/Bitcode/Reader/MetadataLoader.h
+++ b/lib/Bitcode/Reader/MetadataLoader.h
@@ -1,9 +1,8 @@
 //===-- Bitcode/Reader/MetadataLoader.h - Load Metadatas -------*- C++ -*-====//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Bitcode/Reader/ValueList.cpp b/lib/Bitcode/Reader/ValueList.cpp
index b3945a37408f2684dae51a5b8c68fd35fbb78040..4b482de640c03628de895da1d6b66f88bb96101b 100644
--- a/lib/Bitcode/Reader/ValueList.cpp
+++ b/lib/Bitcode/Reader/ValueList.cpp
@@ -1,9 +1,8 @@
 //===- ValueList.cpp - Internal BitcodeReader implementation --------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Bitcode/Reader/ValueList.h b/lib/Bitcode/Reader/ValueList.h
index 5ad7899347ad97f95a99de8a2f7e7f01e21b5219..c636a74d2f6743ee56aebcb3c85fde9c3467af98 100644
--- a/lib/Bitcode/Reader/ValueList.h
+++ b/lib/Bitcode/Reader/ValueList.h
@@ -1,9 +1,8 @@
 //===-- Bitcode/Reader/ValueList.h - Number values --------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Bitcode/Writer/BitWriter.cpp b/lib/Bitcode/Writer/BitWriter.cpp
index 763cd12aa2d71976c8724031f922b2cc2a5ed5fa..76ca89147e52f4a49a044079bd7d26fa012d8876 100644
--- a/lib/Bitcode/Writer/BitWriter.cpp
+++ b/lib/Bitcode/Writer/BitWriter.cpp
@@ -1,9 +1,8 @@
 //===-- BitWriter.cpp -----------------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Bitcode/Writer/BitcodeWriter.cpp b/lib/Bitcode/Writer/BitcodeWriter.cpp
index 68d79edceafb5509659c20c9afa02ef6b9a4c56b..a15ad55b0a3e49a5be8a891a752bf0efc61f0919 100644
--- a/lib/Bitcode/Writer/BitcodeWriter.cpp
+++ b/lib/Bitcode/Writer/BitcodeWriter.cpp
@@ -1,9 +1,8 @@
 //===- Bitcode/Writer/BitcodeWriter.cpp - Bitcode Writer ------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -560,6 +559,8 @@ static unsigned getEncodedRMWOperation(AtomicRMWInst::BinOp Op) {
   case AtomicRMWInst::Min: return bitc::RMW_MIN;
   case AtomicRMWInst::UMax: return bitc::RMW_UMAX;
   case AtomicRMWInst::UMin: return bitc::RMW_UMIN;
+  case AtomicRMWInst::FAdd: return bitc::RMW_FADD;
+  case AtomicRMWInst::FSub: return bitc::RMW_FSUB;
   }
 }
 
@@ -2776,6 +2777,41 @@ void ModuleBitcodeWriter::writeInstruction(const Instruction &I,
       Vals.push_back(VE.getValueID(CatchSwitch.getUnwindDest()));
     break;
   }
+  case Instruction::CallBr: {
+    const CallBrInst *CBI = cast<CallBrInst>(&I);
+    const Value *Callee = CBI->getCalledValue();
+    FunctionType *FTy = CBI->getFunctionType();
+
+    if (CBI->hasOperandBundles())
+      writeOperandBundles(CBI, InstID);
+
+    Code = bitc::FUNC_CODE_INST_CALLBR;
+
+    Vals.push_back(VE.getAttributeListID(CBI->getAttributes()));
+
+    Vals.push_back(CBI->getCallingConv() << bitc::CALL_CCONV |
+                   1 << bitc::CALL_EXPLICIT_TYPE);
+
+    Vals.push_back(VE.getValueID(CBI->getDefaultDest()));
+    Vals.push_back(CBI->getNumIndirectDests());
+    for (unsigned i = 0, e = CBI->getNumIndirectDests(); i != e; ++i)
+      Vals.push_back(VE.getValueID(CBI->getIndirectDest(i)));
+
+    Vals.push_back(VE.getTypeID(FTy));
+    pushValueAndType(Callee, InstID, Vals);
+
+    // Emit value #'s for the fixed parameters.
+    for (unsigned i = 0, e = FTy->getNumParams(); i != e; ++i)
+      pushValue(I.getOperand(i), InstID, Vals); // fixed param.
+
+    // Emit type/value pairs for varargs params.
+    if (FTy->isVarArg()) {
+      for (unsigned i = FTy->getNumParams(), e = CBI->getNumArgOperands();
+           i != e; ++i)
+        pushValueAndType(I.getOperand(i), InstID, Vals); // vararg
+    }
+    break;
+  }
   case Instruction::Unreachable:
     Code = bitc::FUNC_CODE_INST_UNREACHABLE;
     AbbrevToUse = FUNCTION_INST_UNREACHABLE_ABBREV;
@@ -3618,6 +3654,13 @@ void ModuleBitcodeWriterBase::writePerModuleGlobalValueSummary() {
 
   Stream.EmitRecord(bitc::FS_VERSION, ArrayRef<uint64_t>{INDEX_VERSION});
 
+  // Write the index flags.
+  uint64_t Flags = 0;
+  // Bits 1-3 are set only in the combined index, skip them.
+  if (Index->enableSplitLTOUnit())
+    Flags |= 0x8;
+  Stream.EmitRecord(bitc::FS_FLAGS, ArrayRef<uint64_t>{Flags});
+
   if (Index->begin() == Index->end()) {
     Stream.ExitBlock();
     return;
@@ -3734,6 +3777,10 @@ void IndexBitcodeWriter::writeCombinedGlobalValueSummary() {
     Flags |= 0x2;
   if (Index.hasSyntheticEntryCounts())
     Flags |= 0x4;
+  if (Index.enableSplitLTOUnit())
+    Flags |= 0x8;
+  if (Index.partiallySplitLTOUnits())
+    Flags |= 0x10;
   Stream.EmitRecord(bitc::FS_FLAGS, ArrayRef<uint64_t>{Flags});
 
   for (const auto &GVI : valueIds()) {
diff --git a/lib/Bitcode/Writer/BitcodeWriterPass.cpp b/lib/Bitcode/Writer/BitcodeWriterPass.cpp
index 41212e575f8eaaf5c709301904766a53148a89e1..6796cf8cee540ed2a0226693f6cefc788bd978be 100644
--- a/lib/Bitcode/Writer/BitcodeWriterPass.cpp
+++ b/lib/Bitcode/Writer/BitcodeWriterPass.cpp
@@ -1,9 +1,8 @@
 //===- BitcodeWriterPass.cpp - Bitcode writing pass -----------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Bitcode/Writer/LLVMBuild.txt b/lib/Bitcode/Writer/LLVMBuild.txt
index ef6dc9f901e2c145dd7cdd8247fe1c1ec3d1d2b3..34c99cbb93094dfc31f753b283a95767e55cb88c 100644
--- a/lib/Bitcode/Writer/LLVMBuild.txt
+++ b/lib/Bitcode/Writer/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/Bitcode/Writer/LLVMBuild.txt -----------------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Bitcode/Writer/ValueEnumerator.cpp b/lib/Bitcode/Writer/ValueEnumerator.cpp
index deb04f1bb36c8f4e5b3aaaf31e973ba754d673b0..c735efab9c1b21248e9466e08b16c08e33ade334 100644
--- a/lib/Bitcode/Writer/ValueEnumerator.cpp
+++ b/lib/Bitcode/Writer/ValueEnumerator.cpp
@@ -1,9 +1,8 @@
 //===- ValueEnumerator.cpp - Number values and types for bitcode writer ---===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -415,10 +414,8 @@ ValueEnumerator::ValueEnumerator(const Module &M,
           EnumerateMetadata(&F, MD->getMetadata());
         }
         EnumerateType(I.getType());
-        if (const CallInst *CI = dyn_cast<CallInst>(&I))
-          EnumerateAttributes(CI->getAttributes());
-        else if (const InvokeInst *II = dyn_cast<InvokeInst>(&I))
-          EnumerateAttributes(II->getAttributes());
+        if (const auto *Call = dyn_cast<CallBase>(&I))
+          EnumerateAttributes(Call->getAttributes());
 
         // Enumerate metadata attached with this instruction.
         MDs.clear();
diff --git a/lib/Bitcode/Writer/ValueEnumerator.h b/lib/Bitcode/Writer/ValueEnumerator.h
index 011356c32601e3d8e1e7a8453a1ccbc0bcffcf31..112f0b4a1dc465c04ad43f14a62902c46a854324 100644
--- a/lib/Bitcode/Writer/ValueEnumerator.h
+++ b/lib/Bitcode/Writer/ValueEnumerator.h
@@ -1,9 +1,8 @@
 //===- Bitcode/Writer/ValueEnumerator.h - Number values ---------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/AggressiveAntiDepBreaker.cpp b/lib/CodeGen/AggressiveAntiDepBreaker.cpp
index 632ea8e9cdc417a1d659c1e3339a3f3c0085699c..444f618d8b8ce117159fc7aaf7175dcfe8229454 100644
--- a/lib/CodeGen/AggressiveAntiDepBreaker.cpp
+++ b/lib/CodeGen/AggressiveAntiDepBreaker.cpp
@@ -1,9 +1,8 @@
 //===- AggressiveAntiDepBreaker.cpp - Anti-dep breaker --------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/AggressiveAntiDepBreaker.h b/lib/CodeGen/AggressiveAntiDepBreaker.h
index 5dce3c2499e51f4e719550f1559a0f010b8d46da..0cf2e6d78f7ffaad097ee56181397b99b0bb8814 100644
--- a/lib/CodeGen/AggressiveAntiDepBreaker.h
+++ b/lib/CodeGen/AggressiveAntiDepBreaker.h
@@ -1,9 +1,8 @@
 //==- llvm/CodeGen/AggressiveAntiDepBreaker.h - Anti-Dep Support -*- C++ -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/AllocationOrder.cpp b/lib/CodeGen/AllocationOrder.cpp
index 37dcb0be824e87b8dceaaa6bff2d1ffdc2982e5f..c99800659bfd81aa881693fad4e86fa14b1138a6 100644
--- a/lib/CodeGen/AllocationOrder.cpp
+++ b/lib/CodeGen/AllocationOrder.cpp
@@ -1,9 +1,8 @@
 //===-- llvm/CodeGen/AllocationOrder.cpp - Allocation Order ---------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/AllocationOrder.h b/lib/CodeGen/AllocationOrder.h
index 467bcc2edc6f69379a5cd39543688bbe390386da..9247dd8449366128a2be2fd2301e1fda4d1407cf 100644
--- a/lib/CodeGen/AllocationOrder.h
+++ b/lib/CodeGen/AllocationOrder.h
@@ -1,9 +1,8 @@
 //===-- llvm/CodeGen/AllocationOrder.h - Allocation Order -*- C++ -*-------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/Analysis.cpp b/lib/CodeGen/Analysis.cpp
index 27dce7fd7b79981e882e1d56adfedd5fd7994256..9e3ab2454de758a1550e0bfc7a2f41408822d3b8 100644
--- a/lib/CodeGen/Analysis.cpp
+++ b/lib/CodeGen/Analysis.cpp
@@ -1,9 +1,8 @@
 //===-- Analysis.cpp - CodeGen LLVM IR Analysis Utilities -----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -546,6 +545,21 @@ bool llvm::attributesPermitTailCall(const Function *F, const Instruction *I,
     CalleeAttrs.removeAttribute(Attribute::SExt);
   }
 
+  // Drop sext and zext return attributes if the result is not used.
+  // This enables tail calls for code like:
+  //
+  // define void @caller() {
+  // entry:
+  //   %unused_result = tail call zeroext i1 @callee()
+  //   br label %retlabel
+  // retlabel:
+  //   ret void
+  // }
+  if (I->use_empty()) {
+    CalleeAttrs.removeAttribute(Attribute::SExt);
+    CalleeAttrs.removeAttribute(Attribute::ZExt);
+  }
+
   // If they're still different, there's some facet we don't understand
   // (currently only "inreg", but in future who knows). It may be OK but the
   // only safe option is to reject the tail call.
diff --git a/lib/CodeGen/AntiDepBreaker.h b/lib/CodeGen/AntiDepBreaker.h
index d937162879818d8f30ca266bc1f539c32cb1d96e..b1114859513639de3d3281a4768f9899a23191c2 100644
--- a/lib/CodeGen/AntiDepBreaker.h
+++ b/lib/CodeGen/AntiDepBreaker.h
@@ -1,9 +1,8 @@
 //===- llvm/CodeGen/AntiDepBreaker.h - Anti-Dependence Breaking -*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/AsmPrinter/ARMException.cpp b/lib/CodeGen/AsmPrinter/ARMException.cpp
index 9011f025f5958c8c54e991e7283d1a72504510ba..f6ef85a5b78f1ad9f19549ee318a1d26362f0920 100644
--- a/lib/CodeGen/AsmPrinter/ARMException.cpp
+++ b/lib/CodeGen/AsmPrinter/ARMException.cpp
@@ -1,9 +1,8 @@
 //===-- CodeGen/AsmPrinter/ARMException.cpp - ARM EHABI Exception Impl ----===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/AsmPrinter/AccelTable.cpp b/lib/CodeGen/AsmPrinter/AccelTable.cpp
index 95875ccb8a0b9eb6611f10758237c257ca56e743..2de473b57a3026b5996c063abdeabcde4b7b7a69 100644
--- a/lib/CodeGen/AsmPrinter/AccelTable.cpp
+++ b/lib/CodeGen/AsmPrinter/AccelTable.cpp
@@ -1,9 +1,8 @@
 //===- llvm/CodeGen/AsmPrinter/AccelTable.cpp - Accelerator Tables --------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/AsmPrinter/AddressPool.cpp b/lib/CodeGen/AsmPrinter/AddressPool.cpp
index f8143b903d5157126906da63a62c50437944f9fc..f11c7de5ed8afadce9fd03f33bf2b4c6698c5c79 100644
--- a/lib/CodeGen/AsmPrinter/AddressPool.cpp
+++ b/lib/CodeGen/AsmPrinter/AddressPool.cpp
@@ -1,9 +1,8 @@
 //===- llvm/CodeGen/AddressPool.cpp - Dwarf Debug Framework ---------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -24,17 +23,24 @@ unsigned AddressPool::getIndex(const MCSymbol *Sym, bool TLS) {
   return IterBool.first->second.Number;
 }
 
-
-void AddressPool::emitHeader(AsmPrinter &Asm, MCSection *Section) {
+MCSymbol *AddressPool::emitHeader(AsmPrinter &Asm, MCSection *Section) {
   static const uint8_t AddrSize = Asm.getDataLayout().getPointerSize();
-  uint64_t Length = sizeof(uint16_t) // version
-                  + sizeof(uint8_t)  // address_size
-                  + sizeof(uint8_t)  // segment_selector_size
-                  + AddrSize * Pool.size(); // entries
-  Asm.emitInt32(Length); // TODO: Support DWARF64 format.
+  StringRef Prefix = "debug_addr_";
+  MCSymbol *BeginLabel = Asm.createTempSymbol(Prefix + "start");
+  MCSymbol *EndLabel = Asm.createTempSymbol(Prefix + "end");
+
+  Asm.OutStreamer->AddComment("Length of contribution");
+  Asm.EmitLabelDifference(EndLabel, BeginLabel,
+                          4); // TODO: Support DWARF64 format.
+  Asm.OutStreamer->EmitLabel(BeginLabel);
+  Asm.OutStreamer->AddComment("DWARF version number");
   Asm.emitInt16(Asm.getDwarfVersion());
+  Asm.OutStreamer->AddComment("Address size");
   Asm.emitInt8(AddrSize);
+  Asm.OutStreamer->AddComment("Segment selector size");
   Asm.emitInt8(0); // TODO: Support non-zero segment_selector_size.
+
+  return EndLabel;
 }
 
 // Emit addresses into the section given.
@@ -45,8 +51,10 @@ void AddressPool::emit(AsmPrinter &Asm, MCSection *AddrSection) {
   // Start the dwarf addr section.
   Asm.OutStreamer->SwitchSection(AddrSection);
 
+  MCSymbol *EndLabel = nullptr;
+
   if (Asm.getDwarfVersion() >= 5)
-    emitHeader(Asm, AddrSection);
+    EndLabel = emitHeader(Asm, AddrSection);
 
   // Define the symbol that marks the start of the contribution.
   // It is referenced via DW_AT_addr_base.
@@ -63,4 +71,7 @@ void AddressPool::emit(AsmPrinter &Asm, MCSection *AddrSection) {
 
   for (const MCExpr *Entry : Entries)
     Asm.OutStreamer->EmitValue(Entry, Asm.getDataLayout().getPointerSize());
+
+  if (EndLabel)
+    Asm.OutStreamer->EmitLabel(EndLabel);
 }
diff --git a/lib/CodeGen/AsmPrinter/AddressPool.h b/lib/CodeGen/AsmPrinter/AddressPool.h
index 2209c7eb50edc3de78b8527e4a32beb5ec2e8478..f92cf72093ca03c828ea1647736194105e6e593c 100644
--- a/lib/CodeGen/AsmPrinter/AddressPool.h
+++ b/lib/CodeGen/AsmPrinter/AddressPool.h
@@ -1,9 +1,8 @@
 //===- llvm/CodeGen/AddressPool.h - Dwarf Debug Framework -------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -55,7 +54,7 @@ public:
   void setLabel(MCSymbol *Sym) { AddressTableBaseSym = Sym; }
 
 private:
-  void emitHeader(AsmPrinter &Asm, MCSection *Section);
+  MCSymbol *emitHeader(AsmPrinter &Asm, MCSection *Section);
 
   /// Symbol designates the start of the contribution to the address table.
   MCSymbol *AddressTableBaseSym = nullptr;
diff --git a/lib/CodeGen/AsmPrinter/AsmPrinter.cpp b/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
index 7070451e333073a7be91b788450ce8c9569c8976..5319519ddc0d70d6d90a6edf682ccdcbd801191c 100644
--- a/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
+++ b/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
@@ -1,9 +1,8 @@
 //===- AsmPrinter.cpp - Common AsmPrinter code ----------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -60,7 +59,6 @@
 #include "llvm/CodeGen/TargetLowering.h"
 #include "llvm/CodeGen/TargetOpcodes.h"
 #include "llvm/CodeGen/TargetRegisterInfo.h"
-#include "llvm/CodeGen/TargetSubtargetInfo.h"
 #include "llvm/IR/BasicBlock.h"
 #include "llvm/IR/Comdat.h"
 #include "llvm/IR/Constant.h"
@@ -143,10 +141,6 @@ static const char *const CodeViewLineTablesGroupDescription =
 
 STATISTIC(EmittedInsts, "Number of machine instrs printed");
 
-static cl::opt<bool>
-    PrintSchedule("print-schedule", cl::Hidden, cl::init(false),
-                  cl::desc("Print 'sched: [latency:throughput]' in .s output"));
-
 char AsmPrinter::ID = 0;
 
 using gcp_map_type = DenseMap<GCStrategy *, std::unique_ptr<GCMetadataPrinter>>;
@@ -232,6 +226,12 @@ void AsmPrinter::EmitToStreamer(MCStreamer &S, const MCInst &Inst) {
   S.EmitInstruction(Inst, getSubtargetInfo());
 }
 
+void AsmPrinter::emitInitialRawDwarfLocDirective(const MachineFunction &MF) {
+  assert(DD && "Dwarf debug file is not defined.");
+  assert(OutStreamer->hasRawTextSupport() && "Expected assembly output mode.");
+  (void)DD->emitInitialLocDirective(MF, /*CUID=*/0);
+}
+
 /// getCurrentSection() - Return the current section we are emitting to.
 const MCSection *AsmPrinter::getCurrentSection() const {
   return OutStreamer->getCurrentSectionOnly();
@@ -658,6 +658,9 @@ void AsmPrinter::EmitFunctionHeader() {
   if (MAI->hasDotTypeDotSizeDirective())
     OutStreamer->EmitSymbolAttribute(CurrentFnSym, MCSA_ELF_TypeFunction);
 
+  if (F.hasFnAttribute(Attribute::Cold))
+    OutStreamer->EmitSymbolAttribute(CurrentFnSym, MCSA_Cold);
+
   if (isVerbose()) {
     F.printAsOperand(OutStreamer->GetCommentOS(),
                    /*PrintType=*/false, F.getParent());
@@ -738,74 +741,30 @@ void AsmPrinter::EmitFunctionEntryLabel() {
 }
 
 /// emitComments - Pretty-print comments for instructions.
-/// It returns true iff the sched comment was emitted.
-///   Otherwise it returns false.
-static bool emitComments(const MachineInstr &MI, raw_ostream &CommentOS,
-                         AsmPrinter *AP) {
+static void emitComments(const MachineInstr &MI, raw_ostream &CommentOS) {
   const MachineFunction *MF = MI.getMF();
   const TargetInstrInfo *TII = MF->getSubtarget().getInstrInfo();
 
   // Check for spills and reloads
-  int FI;
-
-  const MachineFrameInfo &MFI = MF->getFrameInfo();
-  bool Commented = false;
-
-  auto getSize =
-      [&MFI](const SmallVectorImpl<const MachineMemOperand *> &Accesses) {
-        unsigned Size = 0;
-        for (auto A : Accesses)
-          if (MFI.isSpillSlotObjectIndex(
-                  cast<FixedStackPseudoSourceValue>(A->getPseudoValue())
-                      ->getFrameIndex()))
-            Size += A->getSize();
-        return Size;
-      };
 
   // We assume a single instruction only has a spill or reload, not
   // both.
-  const MachineMemOperand *MMO;
-  SmallVector<const MachineMemOperand *, 2> Accesses;
-  if (TII->isLoadFromStackSlotPostFE(MI, FI)) {
-    if (MFI.isSpillSlotObjectIndex(FI)) {
-      MMO = *MI.memoperands_begin();
-      CommentOS << MMO->getSize() << "-byte Reload";
-      Commented = true;
-    }
-  } else if (TII->hasLoadFromStackSlot(MI, Accesses)) {
-    if (auto Size = getSize(Accesses)) {
-      CommentOS << Size << "-byte Folded Reload";
-      Commented = true;
-    }
-  } else if (TII->isStoreToStackSlotPostFE(MI, FI)) {
-    if (MFI.isSpillSlotObjectIndex(FI)) {
-      MMO = *MI.memoperands_begin();
-      CommentOS << MMO->getSize() << "-byte Spill";
-      Commented = true;
-    }
-  } else if (TII->hasStoreToStackSlot(MI, Accesses)) {
-    if (auto Size = getSize(Accesses)) {
-      CommentOS << Size << "-byte Folded Spill";
-      Commented = true;
-    }
+  Optional<unsigned> Size;
+  if ((Size = MI.getRestoreSize(TII))) {
+    CommentOS << *Size << "-byte Reload\n";
+  } else if ((Size = MI.getFoldedRestoreSize(TII))) {
+    if (*Size)
+      CommentOS << *Size << "-byte Folded Reload\n";
+  } else if ((Size = MI.getSpillSize(TII))) {
+    CommentOS << *Size << "-byte Spill\n";
+  } else if ((Size = MI.getFoldedSpillSize(TII))) {
+    if (*Size)
+      CommentOS << *Size << "-byte Folded Spill\n";
   }
 
   // Check for spill-induced copies
-  if (MI.getAsmPrinterFlag(MachineInstr::ReloadReuse)) {
-    Commented = true;
-    CommentOS << " Reload Reuse";
-  }
-
-  if (Commented) {
-    if (AP->EnablePrintSchedInfo) {
-      // If any comment was added above and we need sched info comment then add
-      // this new comment just after the above comment w/o "\n" between them.
-      CommentOS << " " << MF->getSubtarget().getSchedInfoStr(MI) << "\n";
-      return true;
-    }
-    CommentOS << "\n";
-  }
-  return false;
+  if (MI.getAsmPrinterFlag(MachineInstr::ReloadReuse))
+    CommentOS << " Reload Reuse\n";
 }
 
 /// emitImplicitDef - This method emits the specified machine instruction
@@ -1093,10 +1052,8 @@ void AsmPrinter::EmitFunctionBody() {
         }
       }
 
-      if (isVerbose() && emitComments(MI, OutStreamer->GetCommentOS(), this)) {
-        MachineInstr *MIP = const_cast<MachineInstr *>(&MI);
-        MIP->setAsmPrinterFlag(MachineInstr::NoSchedComment);
-      }
+      if (isVerbose())
+        emitComments(MI, OutStreamer->GetCommentOS());
 
       switch (MI.getOpcode()) {
       case TargetOpcode::CFI_INSTRUCTION:
@@ -1110,6 +1067,7 @@ void AsmPrinter::EmitFunctionBody() {
         OutStreamer->EmitLabel(MI.getOperand(0).getMCSymbol());
         break;
       case TargetOpcode::INLINEASM:
+      case TargetOpcode::INLINEASM_BR:
         EmitInlineAsm(&MI);
         break;
       case TargetOpcode::DBG_VALUE:
@@ -1329,9 +1287,19 @@ void AsmPrinter::emitGlobalIndirectSymbol(Module &M,
   else
     assert(GIS.hasLocalLinkage() && "Invalid alias or ifunc linkage");
 
+  bool IsFunction = GIS.getType()->getPointerElementType()->isFunctionTy();
+
+  // Treat bitcasts of functions as functions also. This is important at least
+  // on WebAssembly where object and function addresses can't alias each other.
+  if (!IsFunction)
+    if (auto *CE = dyn_cast<ConstantExpr>(GIS.getIndirectSymbol()))
+      if (CE->getOpcode() == Instruction::BitCast)
+        IsFunction =
+          CE->getOperand(0)->getType()->getPointerElementType()->isFunctionTy();
+
   // Set the symbol type to function if the alias has a function type.
   // This affects codegen when the aliasee is not a function.
-  if (GIS.getType()->getPointerElementType()->isFunctionTy()) {
+  if (IsFunction) {
     OutStreamer->EmitSymbolAttribute(Name, MCSA_ELF_TypeFunction);
     if (isa<GlobalIFunc>(GIS))
       OutStreamer->EmitSymbolAttribute(Name, MCSA_ELF_TypeIndFunction);
@@ -1628,11 +1596,6 @@ void AsmPrinter::SetupMachineFunction(MachineFunction &MF) {
   }
 
   ORE = &getAnalysis<MachineOptimizationRemarkEmitterPass>().getORE();
-
-  const TargetSubtargetInfo &STI = MF.getSubtarget();
-  EnablePrintSchedInfo = PrintSchedule.getNumOccurrences()
-                             ? PrintSchedule
-                             : STI.supportPrintSchedInfo();
 }
 
 namespace {
@@ -1905,8 +1868,7 @@ bool AsmPrinter::EmitSpecialLLVMGlobal(const GlobalVariable *GV) {
 }
 
 /// EmitLLVMUsedList - For targets that define a MAI::UsedDirective, mark each
-/// global in the specified llvm.used list for which emitUsedDirectiveFor
-/// is true, as being used with this directive.
+/// global in the specified llvm.used list.
 void AsmPrinter::EmitLLVMUsedList(const ConstantArray *InitList) {
   // Should be an array of 'i8*'.
   for (unsigned i = 0, e = InitList->getNumOperands(); i != e; ++i) {
diff --git a/lib/CodeGen/AsmPrinter/AsmPrinterDwarf.cpp b/lib/CodeGen/AsmPrinter/AsmPrinterDwarf.cpp
index afce3ad3133b3376ae4445a639842cb4e5a15452..7a62b6bfc6a4502424f9f72b8534eb8b16f63a97 100644
--- a/lib/CodeGen/AsmPrinter/AsmPrinterDwarf.cpp
+++ b/lib/CodeGen/AsmPrinter/AsmPrinterDwarf.cpp
@@ -1,9 +1,8 @@
 //===-- AsmPrinterDwarf.cpp - AsmPrinter Dwarf Support --------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/AsmPrinter/AsmPrinterInlineAsm.cpp b/lib/CodeGen/AsmPrinter/AsmPrinterInlineAsm.cpp
index 62103e3107c008a749ca89db3bc6525cf84db9c4..52acabec06d1b1133a7e634a76dc906d5ff46229 100644
--- a/lib/CodeGen/AsmPrinter/AsmPrinterInlineAsm.cpp
+++ b/lib/CodeGen/AsmPrinter/AsmPrinterInlineAsm.cpp
@@ -1,9 +1,8 @@
 //===-- AsmPrinterInlineAsm.cpp - AsmPrinter Inline Asm Handling ----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -19,7 +18,6 @@
 #include "llvm/CodeGen/MachineModuleInfo.h"
 #include "llvm/CodeGen/TargetInstrInfo.h"
 #include "llvm/CodeGen/TargetRegisterInfo.h"
-#include "llvm/CodeGen/TargetSubtargetInfo.h"
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/DataLayout.h"
 #include "llvm/IR/InlineAsm.h"
@@ -155,7 +153,6 @@ void AsmPrinter::EmitInlineAsm(StringRef Str, const MCSubtargetInfo &STI,
                        " we don't have an asm parser for this target\n");
   Parser->setAssemblerDialect(Dialect);
   Parser->setTargetParser(*TAP.get());
-  Parser->setEnablePrintSchedInfo(EnablePrintSchedInfo);
   // Enable lexing Masm binary and hex integer literals in intel inline
   // assembly.
   if (Dialect == InlineAsm::AD_Intel)
@@ -436,9 +433,16 @@ static void EmitGCCInlineAsmStr(const char *AsmStr, const MachineInstr *MI,
           ++OpNo;  // Skip over the ID number.
 
           if (Modifier[0] == 'l') { // Labels are target independent.
-            // FIXME: What if the operand isn't an MBB, report error?
-            const MCSymbol *Sym = MI->getOperand(OpNo).getMBB()->getSymbol();
-            Sym->print(OS, AP->MAI);
+            if (MI->getOperand(OpNo).isBlockAddress()) {
+              const BlockAddress *BA = MI->getOperand(OpNo).getBlockAddress();
+              MCSymbol *Sym = AP->GetBlockAddressSymbol(BA);
+              Sym->print(OS, AP->MAI);
+            } else if (MI->getOperand(OpNo).isMBB()) {
+              const MCSymbol *Sym = MI->getOperand(OpNo).getMBB()->getSymbol();
+              Sym->print(OS, AP->MAI);
+            } else {
+              Error = true;
+            }
           } else {
             if (InlineAsm::isMemKind(OpFlags)) {
               Error = AP->PrintAsmMemoryOperand(MI, OpNo, InlineAsmVariant,
diff --git a/lib/CodeGen/AsmPrinter/ByteStreamer.h b/lib/CodeGen/AsmPrinter/ByteStreamer.h
index 2163cc7e3e117a066d8d026bcd3484844a025d6b..7a2902d68070db60c1337663e2d28d07ade5b2cc 100644
--- a/lib/CodeGen/AsmPrinter/ByteStreamer.h
+++ b/lib/CodeGen/AsmPrinter/ByteStreamer.h
@@ -1,9 +1,8 @@
 //===-- llvm/CodeGen/ByteStreamer.h - ByteStreamer class --------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/AsmPrinter/CodeViewDebug.cpp b/lib/CodeGen/AsmPrinter/CodeViewDebug.cpp
index 4132b1a30c42ab537812bf581792ae8b003e761b..60731f1e676c7facf69a2a8084701ad21a38a79f 100644
--- a/lib/CodeGen/AsmPrinter/CodeViewDebug.cpp
+++ b/lib/CodeGen/AsmPrinter/CodeViewDebug.cpp
@@ -1,9 +1,8 @@
 //===- llvm/lib/CodeGen/AsmPrinter/CodeViewDebug.cpp ----------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -124,6 +123,8 @@ CodeViewDebug::CodeViewDebug(AsmPrinter *AP)
   TheCPU =
       mapArchToCVCPUType(Triple(MMI->getModule()->getTargetTriple()).getArch());
 
+  collectGlobalVariableInfo();
+
   // Check if we should emit type record hashes.
   ConstantInt *GH = mdconst::extract_or_null<ConstantInt>(
       MMI->getModule()->getModuleFlag("CodeViewGHash"));
@@ -1038,6 +1039,7 @@ void CodeViewDebug::emitDebugInfoForFunction(const Function *GV,
     endSymbolRecord(FrameProcEnd);
 
     emitLocalVariableList(FI, FI.Locals);
+    emitGlobalVariableList(FI.Globals);
     emitLexicalBlockList(FI.ChildBlocks, FI);
 
     // Emit inlined call site information. Only emit functions inlined directly
@@ -1833,11 +1835,24 @@ TypeIndex CodeViewDebug::lowerTypeMemberFunction(const DISubroutineType *Ty,
 
   unsigned Index = 0;
   SmallVector<TypeIndex, 8> ArgTypeIndices;
-  TypeIndex ReturnTypeIndex = getTypeIndex(ReturnAndArgs[Index++]);
+  TypeIndex ReturnTypeIndex = TypeIndex::Void();
+  if (ReturnAndArgs.size() > Index) {
+    ReturnTypeIndex = getTypeIndex(ReturnAndArgs[Index++]);
+  }
 
+  // If the first argument is a pointer type and this isn't a static method,
+  // treat it as the special 'this' parameter, which is encoded separately from
+  // the arguments.
   TypeIndex ThisTypeIndex;
-  if (!IsStaticMethod && ReturnAndArgs.size() > 1)
-    ThisTypeIndex = getTypeIndexForThisPtr(ReturnAndArgs[Index++], Ty);
+  if (!IsStaticMethod && ReturnAndArgs.size() > Index) {
+    if (const DIDerivedType *PtrTy =
+            dyn_cast_or_null<DIDerivedType>(ReturnAndArgs[Index].resolve())) {
+      if (PtrTy->getTag() == dwarf::DW_TAG_pointer_type) {
+        ThisTypeIndex = getTypeIndexForThisPtr(PtrTy, Ty);
+        Index++;
+      }
+    }
+  }
 
   while (Index < ReturnAndArgs.size())
     ArgTypeIndices.push_back(getTypeIndex(ReturnAndArgs[Index++]));
@@ -2046,6 +2061,7 @@ void CodeViewDebug::clear() {
   GlobalUDTs.clear();
   TypeIndices.clear();
   CompleteTypeIndices.clear();
+  ScopeGlobals.clear();
 }
 
 void CodeViewDebug::collectMemberInfo(ClassInfo &Info,
@@ -2392,9 +2408,10 @@ TypeIndex CodeViewDebug::getTypeIndex(DITypeRef TypeRef, DITypeRef ClassTyRef) {
 }
 
 codeview::TypeIndex
-CodeViewDebug::getTypeIndexForThisPtr(DITypeRef TypeRef,
+CodeViewDebug::getTypeIndexForThisPtr(const DIDerivedType *PtrTy,
                                       const DISubroutineType *SubroutineTy) {
-  const DIType *Ty = TypeRef.resolve();
+  assert(PtrTy->getTag() == dwarf::DW_TAG_pointer_type &&
+         "this type must be a pointer type");
 
   PointerOptions Options = PointerOptions::None;
   if (SubroutineTy->getFlags() & DINode::DIFlags::FlagLValueReference)
@@ -2407,13 +2424,13 @@ CodeViewDebug::getTypeIndexForThisPtr(DITypeRef TypeRef,
   // so that the TypeIndex for the this pointer can be shared with the type
   // index for other pointers to this class type.  If there is a ref qualifier
   // then we lookup the pointer using the subroutine as the parent type.
-  auto I = TypeIndices.find({Ty, SubroutineTy});
+  auto I = TypeIndices.find({PtrTy, SubroutineTy});
   if (I != TypeIndices.end())
     return I->second;
 
   TypeLoweringScope S(*this);
-  TypeIndex TI = lowerTypePointer(cast<DIDerivedType>(Ty), Options);
-  return recordTypeIndexForDINode(Ty, TI, SubroutineTy);
+  TypeIndex TI = lowerTypePointer(PtrTy, Options);
+  return recordTypeIndexForDINode(PtrTy, TI, SubroutineTy);
 }
 
 TypeIndex CodeViewDebug::getTypeIndexForReferenceTo(DITypeRef TypeRef) {
@@ -2648,6 +2665,7 @@ void CodeViewDebug::emitLexicalBlock(const LexicalBlock &Block,
 
   // Emit variables local to this lexical block.
   emitLocalVariableList(FI, Block.Locals);
+  emitGlobalVariableList(Block.Globals);
 
   // Emit lexical blocks contained within this block.
   emitLexicalBlockList(Block.Children, FI);
@@ -2661,9 +2679,10 @@ void CodeViewDebug::emitLexicalBlock(const LexicalBlock &Block,
 void CodeViewDebug::collectLexicalBlockInfo(
         SmallVectorImpl<LexicalScope *> &Scopes,
         SmallVectorImpl<LexicalBlock *> &Blocks,
-        SmallVectorImpl<LocalVariable> &Locals) {
+        SmallVectorImpl<LocalVariable> &Locals,
+        SmallVectorImpl<CVGlobalVariable> &Globals) {
   for (LexicalScope *Scope : Scopes)
-    collectLexicalBlockInfo(*Scope, Blocks, Locals);
+    collectLexicalBlockInfo(*Scope, Blocks, Locals, Globals);
 }
 
 /// Populate the lexical blocks and local variable lists of the parent with
@@ -2671,45 +2690,58 @@ void CodeViewDebug::collectLexicalBlockInfo(
 void CodeViewDebug::collectLexicalBlockInfo(
     LexicalScope &Scope,
     SmallVectorImpl<LexicalBlock *> &ParentBlocks,
-    SmallVectorImpl<LocalVariable> &ParentLocals) {
+    SmallVectorImpl<LocalVariable> &ParentLocals,
+    SmallVectorImpl<CVGlobalVariable> &ParentGlobals) {
   if (Scope.isAbstractScope())
     return;
 
-  auto LocalsIter = ScopeVariables.find(&Scope);
-  if (LocalsIter == ScopeVariables.end()) {
-    // This scope does not contain variables and can be eliminated.
-    collectLexicalBlockInfo(Scope.getChildren(), ParentBlocks, ParentLocals);
-    return;
-  }
-  SmallVectorImpl<LocalVariable> &Locals = LocalsIter->second;
-
+  // Gather information about the lexical scope including local variables,
+  // global variables, and address ranges.
+  bool IgnoreScope = false;
+  auto LI = ScopeVariables.find(&Scope);
+  SmallVectorImpl<LocalVariable> *Locals =
+      LI != ScopeVariables.end() ? &LI->second : nullptr;
+  auto GI = ScopeGlobals.find(Scope.getScopeNode());
+  SmallVectorImpl<CVGlobalVariable> *Globals =
+      GI != ScopeGlobals.end() ? GI->second.get() : nullptr;
   const DILexicalBlock *DILB = dyn_cast<DILexicalBlock>(Scope.getScopeNode());
-  if (!DILB) {
-    // This scope is not a lexical block and can be eliminated, but keep any
-    // local variables it contains.
-    ParentLocals.append(Locals.begin(), Locals.end());
-    collectLexicalBlockInfo(Scope.getChildren(), ParentBlocks, ParentLocals);
-    return;
-  }
-
   const SmallVectorImpl<InsnRange> &Ranges = Scope.getRanges();
-  if (Ranges.size() != 1 || !getLabelAfterInsn(Ranges.front().second)) {
-    // This lexical block scope has too many address ranges to represent in the
-    // current CodeView format or does not have a valid address range.
-    // Eliminate this lexical scope and promote any locals it contains to the
-    // parent scope.
-    //
-    // For lexical scopes with multiple address ranges you may be tempted to
-    // construct a single range covering every instruction where the block is
-    // live and everything in between.  Unfortunately, Visual Studio only
-    // displays variables from the first matching lexical block scope.  If the
-    // first lexical block contains exception handling code or cold code which
-    // is moved to the bottom of the routine creating a single range covering
-    // nearly the entire routine, then it will hide all other lexical blocks
-    // and the variables they contain.
-    //
-    ParentLocals.append(Locals.begin(), Locals.end());
-    collectLexicalBlockInfo(Scope.getChildren(), ParentBlocks, ParentLocals);
+
+  // Ignore lexical scopes which do not contain variables.
+  if (!Locals && !Globals)
+    IgnoreScope = true;
+
+  // Ignore lexical scopes which are not lexical blocks.
+  if (!DILB)
+    IgnoreScope = true;
+
+  // Ignore scopes which have too many address ranges to represent in the
+  // current CodeView format or do not have a valid address range.
+  //
+  // For lexical scopes with multiple address ranges you may be tempted to
+  // construct a single range covering every instruction where the block is
+  // live and everything in between.  Unfortunately, Visual Studio only
+  // displays variables from the first matching lexical block scope.  If the
+  // first lexical block contains exception handling code or cold code which
+  // is moved to the bottom of the routine creating a single range covering
+  // nearly the entire routine, then it will hide all other lexical blocks
+  // and the variables they contain.
+  if (Ranges.size() != 1 || !getLabelAfterInsn(Ranges.front().second))
+    IgnoreScope = true;
+
+  if (IgnoreScope) {
+    // This scope can be safely ignored and eliminating it will reduce the
+    // size of the debug information. Be sure to collect any variable and scope
+    // information from the this scope or any of its children and collapse them
+    // into the parent scope.
+    if (Locals)
+      ParentLocals.append(Locals->begin(), Locals->end());
+    if (Globals)
+      ParentGlobals.append(Globals->begin(), Globals->end());
+    collectLexicalBlockInfo(Scope.getChildren(),
+                            ParentBlocks,
+                            ParentLocals,
+                            ParentGlobals);
     return;
   }
 
@@ -2720,8 +2752,8 @@ void CodeViewDebug::collectLexicalBlockInfo(
   if (!BlockInsertion.second)
     return;
 
-  // Create a lexical block containing the local variables and collect the
-  // the lexical block information for the children.
+  // Create a lexical block containing the variables and collect the the
+  // lexical block information for the children.
   const InsnRange &Range = Ranges.front();
   assert(Range.first && Range.second);
   LexicalBlock &Block = BlockInsertion.first->second;
@@ -2730,9 +2762,15 @@ void CodeViewDebug::collectLexicalBlockInfo(
   assert(Block.Begin && "missing label for scope begin");
   assert(Block.End && "missing label for scope end");
   Block.Name = DILB->getName();
-  Block.Locals = std::move(Locals);
+  if (Locals)
+    Block.Locals = std::move(*Locals);
+  if (Globals)
+    Block.Globals = std::move(*Globals);
   ParentBlocks.push_back(&Block);
-  collectLexicalBlockInfo(Scope.getChildren(), Block.Children, Block.Locals);
+  collectLexicalBlockInfo(Scope.getChildren(),
+                          Block.Children,
+                          Block.Locals,
+                          Block.Globals);
 }
 
 void CodeViewDebug::endFunctionImpl(const MachineFunction *MF) {
@@ -2744,7 +2782,10 @@ void CodeViewDebug::endFunctionImpl(const MachineFunction *MF) {
 
   // Build the lexical block structure to emit for this routine.
   if (LexicalScope *CFS = LScopes.getCurrentFunctionScope())
-    collectLexicalBlockInfo(*CFS, CurFn->ChildBlocks, CurFn->Locals);
+    collectLexicalBlockInfo(*CFS,
+                            CurFn->ChildBlocks,
+                            CurFn->Locals,
+                            CurFn->Globals);
 
   // Clear the scope and variable information from the map which will not be
   // valid after we have finished processing this routine.  This also prepares
@@ -2861,7 +2902,7 @@ void CodeViewDebug::emitDebugInfoForUDTs(
   }
 }
 
-void CodeViewDebug::emitDebugInfoForGlobals() {
+void CodeViewDebug::collectGlobalVariableInfo() {
   DenseMap<const DIGlobalVariableExpression *, const GlobalVariable *>
       GlobalMap;
   for (const GlobalVariable &GV : MMI->getModule()->globals()) {
@@ -2874,42 +2915,56 @@ void CodeViewDebug::emitDebugInfoForGlobals() {
   NamedMDNode *CUs = MMI->getModule()->getNamedMetadata("llvm.dbg.cu");
   for (const MDNode *Node : CUs->operands()) {
     const auto *CU = cast<DICompileUnit>(Node);
-
-    // First, emit all globals that are not in a comdat in a single symbol
-    // substream. MSVC doesn't like it if the substream is empty, so only open
-    // it if we have at least one global to emit.
-    switchToDebugSectionForSymbol(nullptr);
-    MCSymbol *EndLabel = nullptr;
     for (const auto *GVE : CU->getGlobalVariables()) {
-      if (const auto *GV = GlobalMap.lookup(GVE))
-        if (!GV->hasComdat() && !GV->isDeclarationForLinker()) {
-          if (!EndLabel) {
-            OS.AddComment("Symbol subsection for globals");
-            EndLabel = beginCVSubsection(DebugSubsectionKind::Symbols);
-          }
-          // FIXME: emitDebugInfoForGlobal() doesn't handle DIExpressions.
-          emitDebugInfoForGlobal(GVE->getVariable(), GV, Asm->getSymbol(GV));
-        }
+      const auto *GV = GlobalMap.lookup(GVE);
+      if (!GV || GV->isDeclarationForLinker())
+        continue;
+      const DIGlobalVariable *DIGV = GVE->getVariable();
+      DIScope *Scope = DIGV->getScope();
+      SmallVector<CVGlobalVariable, 1> *VariableList;
+      if (Scope && isa<DILocalScope>(Scope)) {
+        // Locate a global variable list for this scope, creating one if
+        // necessary.
+        auto Insertion = ScopeGlobals.insert(
+            {Scope, std::unique_ptr<GlobalVariableList>()});
+        if (Insertion.second)
+          Insertion.first->second = llvm::make_unique<GlobalVariableList>();
+        VariableList = Insertion.first->second.get();
+      } else if (GV->hasComdat())
+        // Emit this global variable into a COMDAT section.
+        VariableList = &ComdatVariables;
+      else
+        // Emit this globla variable in a single global symbol section.
+        VariableList = &GlobalVariables;
+      CVGlobalVariable CVGV = {DIGV, GV};
+      VariableList->emplace_back(std::move(CVGV));
     }
-    if (EndLabel)
-      endCVSubsection(EndLabel);
+  }
+}
 
-    // Second, emit each global that is in a comdat into its own .debug$S
-    // section along with its own symbol substream.
-    for (const auto *GVE : CU->getGlobalVariables()) {
-      if (const auto *GV = GlobalMap.lookup(GVE)) {
-        if (GV->hasComdat()) {
-          MCSymbol *GVSym = Asm->getSymbol(GV);
-          OS.AddComment("Symbol subsection for " +
-                        Twine(GlobalValue::dropLLVMManglingEscape(GV->getName())));
-          switchToDebugSectionForSymbol(GVSym);
-          EndLabel = beginCVSubsection(DebugSubsectionKind::Symbols);
-          // FIXME: emitDebugInfoForGlobal() doesn't handle DIExpressions.
-          emitDebugInfoForGlobal(GVE->getVariable(), GV, GVSym);
-          endCVSubsection(EndLabel);
-        }
-      }
-    }
+void CodeViewDebug::emitDebugInfoForGlobals() {
+  // First, emit all globals that are not in a comdat in a single symbol
+  // substream. MSVC doesn't like it if the substream is empty, so only open
+  // it if we have at least one global to emit.
+  switchToDebugSectionForSymbol(nullptr);
+  if (!GlobalVariables.empty()) {
+    OS.AddComment("Symbol subsection for globals");
+    MCSymbol *EndLabel = beginCVSubsection(DebugSubsectionKind::Symbols);
+    emitGlobalVariableList(GlobalVariables);
+    endCVSubsection(EndLabel);
+  }
+
+  // Second, emit each global that is in a comdat into its own .debug$S
+  // section along with its own symbol substream.
+  for (const CVGlobalVariable &CVGV : ComdatVariables) {
+    MCSymbol *GVSym = Asm->getSymbol(CVGV.GV);
+    OS.AddComment("Symbol subsection for " +
+            Twine(GlobalValue::dropLLVMManglingEscape(CVGV.GV->getName())));
+    switchToDebugSectionForSymbol(GVSym);
+    MCSymbol *EndLabel = beginCVSubsection(DebugSubsectionKind::Symbols);
+    // FIXME: emitDebugInfoForGlobal() doesn't handle DIExpressions.
+    emitDebugInfoForGlobal(CVGV.DIGV, CVGV.GV, GVSym);
+    endCVSubsection(EndLabel);
   }
 }
 
@@ -2925,6 +2980,15 @@ void CodeViewDebug::emitDebugInfoForRetainedTypes() {
   }
 }
 
+// Emit each global variable in the specified array.
+void CodeViewDebug::emitGlobalVariableList(ArrayRef<CVGlobalVariable> Globals) {
+  for (const CVGlobalVariable &CVGV : Globals) {
+    MCSymbol *GVSym = Asm->getSymbol(CVGV.GV);
+    // FIXME: emitDebugInfoForGlobal() doesn't handle DIExpressions.
+    emitDebugInfoForGlobal(CVGV.DIGV, CVGV.GV, GVSym);
+  }
+}
+
 void CodeViewDebug::emitDebugInfoForGlobal(const DIGlobalVariable *DIGV,
                                            const GlobalVariable *GV,
                                            MCSymbol *GVSym) {
diff --git a/lib/CodeGen/AsmPrinter/CodeViewDebug.h b/lib/CodeGen/AsmPrinter/CodeViewDebug.h
index fcc7c3290eb2c4aa1388dacf638cded494bb67b9..7cf7b19ec70b48ca0e52b4dec69c37bc7c85393d 100644
--- a/lib/CodeGen/AsmPrinter/CodeViewDebug.h
+++ b/lib/CodeGen/AsmPrinter/CodeViewDebug.h
@@ -1,9 +1,8 @@
 //===- llvm/lib/CodeGen/AsmPrinter/CodeViewDebug.h --------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -99,6 +98,11 @@ class LLVM_LIBRARY_VISIBILITY CodeViewDebug : public DebugHandlerBase {
     bool UseReferenceType = false;
   };
 
+  struct CVGlobalVariable {
+    const DIGlobalVariable *DIGV;
+    const GlobalVariable *GV;
+  };
+
   struct InlineSite {
     SmallVector<LocalVariable, 1> InlinedLocals;
     SmallVector<const DILocation *, 1> ChildSites;
@@ -112,6 +116,7 @@ class LLVM_LIBRARY_VISIBILITY CodeViewDebug : public DebugHandlerBase {
   // Combines information from DILexicalBlock and LexicalScope.
   struct LexicalBlock {
     SmallVector<LocalVariable, 1> Locals;
+    SmallVector<CVGlobalVariable, 1> Globals;
     SmallVector<LexicalBlock *, 1> Children;
     const MCSymbol *Begin;
     const MCSymbol *End;
@@ -134,6 +139,7 @@ class LLVM_LIBRARY_VISIBILITY CodeViewDebug : public DebugHandlerBase {
     SmallVector<const DILocation *, 1> ChildSites;
 
     SmallVector<LocalVariable, 1> Locals;
+    SmallVector<CVGlobalVariable, 1> Globals;
 
     std::unordered_map<const DILexicalBlockBase*, LexicalBlock> LexicalBlocks;
 
@@ -183,6 +189,17 @@ class LLVM_LIBRARY_VISIBILITY CodeViewDebug : public DebugHandlerBase {
   // and LexicalBlocks.
   DenseMap<const LexicalScope *, SmallVector<LocalVariable, 1>> ScopeVariables;
 
+  // Map to separate global variables according to the lexical scope they
+  // belong in. A null local scope represents the global scope.
+  typedef SmallVector<CVGlobalVariable, 1> GlobalVariableList;
+  DenseMap<const DIScope*, std::unique_ptr<GlobalVariableList> > ScopeGlobals;
+
+  // Array of global variables which  need to be emitted into a COMDAT section.
+  SmallVector<CVGlobalVariable, 1> ComdatVariables;
+
+  // Array of non-COMDAT global variables.
+  SmallVector<CVGlobalVariable, 1> GlobalVariables;
+
   /// The set of comdat .debug$S sections that we've seen so far. Each section
   /// must start with a magic version number that must only be emitted once.
   /// This set tracks which sections we've already opened.
@@ -288,13 +305,13 @@ class LLVM_LIBRARY_VISIBILITY CodeViewDebug : public DebugHandlerBase {
 
   void emitDebugInfoForFunction(const Function *GV, FunctionInfo &FI);
 
-  void emitDebugInfoForGlobals();
-
   void emitDebugInfoForRetainedTypes();
 
   void
   emitDebugInfoForUDTs(ArrayRef<std::pair<std::string, const DIType *>> UDTs);
 
+  void emitDebugInfoForGlobals();
+  void emitGlobalVariableList(ArrayRef<CVGlobalVariable> Globals);
   void emitDebugInfoForGlobal(const DIGlobalVariable *DIGV,
                               const GlobalVariable *GV, MCSymbol *GVSym);
 
@@ -319,6 +336,7 @@ class LLVM_LIBRARY_VISIBILITY CodeViewDebug : public DebugHandlerBase {
 
   using InlinedEntity = DbgValueHistoryMap::InlinedEntity;
 
+  void collectGlobalVariableInfo();
   void collectVariableInfo(const DISubprogram *SP);
 
   void collectVariableInfoFromMFTable(DenseSet<InlinedEntity> &Processed);
@@ -327,10 +345,12 @@ class LLVM_LIBRARY_VISIBILITY CodeViewDebug : public DebugHandlerBase {
   // scopes, and populate it with local variables.
   void collectLexicalBlockInfo(SmallVectorImpl<LexicalScope *> &Scopes,
                                SmallVectorImpl<LexicalBlock *> &Blocks,
-                               SmallVectorImpl<LocalVariable> &Locals);
+                               SmallVectorImpl<LocalVariable> &Locals,
+                               SmallVectorImpl<CVGlobalVariable> &Globals);
   void collectLexicalBlockInfo(LexicalScope &Scope,
                                SmallVectorImpl<LexicalBlock *> &ParentBlocks,
-                               SmallVectorImpl<LocalVariable> &ParentLocals);
+                               SmallVectorImpl<LocalVariable> &ParentLocals,
+                               SmallVectorImpl<CVGlobalVariable> &ParentGlobals);
 
   /// Records information about a local variable in the appropriate scope. In
   /// particular, locals from inlined code live inside the inlining site.
@@ -356,7 +376,7 @@ class LLVM_LIBRARY_VISIBILITY CodeViewDebug : public DebugHandlerBase {
                                    DITypeRef ClassTyRef = DITypeRef());
 
   codeview::TypeIndex
-  getTypeIndexForThisPtr(DITypeRef TypeRef,
+  getTypeIndexForThisPtr(const DIDerivedType *PtrTy,
                          const DISubroutineType *SubroutineTy);
 
   codeview::TypeIndex getTypeIndexForReferenceTo(DITypeRef TypeRef);
diff --git a/lib/CodeGen/AsmPrinter/DIE.cpp b/lib/CodeGen/AsmPrinter/DIE.cpp
index e27659494f08125d0125ed860fed1435037c0d2f..a07b54773039b5a9fd35ff5310c3246d94fec7b1 100644
--- a/lib/CodeGen/AsmPrinter/DIE.cpp
+++ b/lib/CodeGen/AsmPrinter/DIE.cpp
@@ -1,9 +1,8 @@
 //===--- lib/CodeGen/DIE.cpp - DWARF Info Entries -------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/AsmPrinter/DIEHash.cpp b/lib/CodeGen/AsmPrinter/DIEHash.cpp
index b8f1202494d7bf5df8dd74c2d837f0a2f9d17dca..75d9430c35193e49e90cbd87e39210bffd261d30 100644
--- a/lib/CodeGen/AsmPrinter/DIEHash.cpp
+++ b/lib/CodeGen/AsmPrinter/DIEHash.cpp
@@ -1,9 +1,8 @@
 //===-- llvm/CodeGen/DIEHash.cpp - Dwarf Hashing Framework ----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/AsmPrinter/DIEHash.h b/lib/CodeGen/AsmPrinter/DIEHash.h
index dae517ab2c29857796b2343e8d06028c30604428..2e49514c98be383d6d9765567a2c98fa3a41d509 100644
--- a/lib/CodeGen/AsmPrinter/DIEHash.h
+++ b/lib/CodeGen/AsmPrinter/DIEHash.h
@@ -1,9 +1,8 @@
 //===-- llvm/CodeGen/DIEHash.h - Dwarf Hashing Framework -------*- C++ -*--===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/AsmPrinter/DbgEntityHistoryCalculator.cpp b/lib/CodeGen/AsmPrinter/DbgEntityHistoryCalculator.cpp
index 09867822c30ac9f1c281cdc0108911edc96a3377..4f828da55e271c3419ccd09d63f0f621c179f6c2 100644
--- a/lib/CodeGen/AsmPrinter/DbgEntityHistoryCalculator.cpp
+++ b/lib/CodeGen/AsmPrinter/DbgEntityHistoryCalculator.cpp
@@ -1,9 +1,8 @@
 //===- llvm/CodeGen/AsmPrinter/DbgEntityHistoryCalculator.cpp -------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/CodeGen/AsmPrinter/DebugHandlerBase.cpp b/lib/CodeGen/AsmPrinter/DebugHandlerBase.cpp
index 551cd36d19843d6cf1c0a68a37a5365ef5341008..0680d2040fb4d0faaf6f05760fe821de188bcbca 100644
--- a/lib/CodeGen/AsmPrinter/DebugHandlerBase.cpp
+++ b/lib/CodeGen/AsmPrinter/DebugHandlerBase.cpp
@@ -1,9 +1,8 @@
 //===-- llvm/lib/CodeGen/AsmPrinter/DebugHandlerBase.cpp -------*- C++ -*--===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -216,24 +215,36 @@ void DebugHandlerBase::beginFunction(const MachineFunction *MF) {
     if (Ranges.empty())
       continue;
 
-    // The first mention of a function argument gets the CurrentFnBegin
-    // label, so arguments are visible when breaking at function entry.
+    auto IsDescribedByReg = [](const MachineInstr *MI) {
+      return MI->getOperand(0).isReg() && MI->getOperand(0).getReg();
+    };
+
+    // The first mention of a function argument gets the CurrentFnBegin label,
+    // so arguments are visible when breaking at function entry.
+    //
+    // We do not change the label for values that are described by registers,
+    // as that could place them above their defining instructions. We should
+    // ideally not change the labels for constant debug values either, since
+    // doing that violates the ranges that are calculated in the history map.
+    // However, we currently do not emit debug values for constant arguments
+    // directly at the start of the function, so this code is still useful.
     const DILocalVariable *DIVar = Ranges.front().first->getDebugVariable();
     if (DIVar->isParameter() &&
         getDISubprogram(DIVar->getScope())->describes(&MF->getFunction())) {
-      LabelsBeforeInsn[Ranges.front().first] = Asm->getFunctionBegin();
+      if (!IsDescribedByReg(Ranges.front().first))
+        LabelsBeforeInsn[Ranges.front().first] = Asm->getFunctionBegin();
       if (Ranges.front().first->getDebugExpression()->isFragment()) {
         // Mark all non-overlapping initial fragments.
         for (auto I = Ranges.begin(); I != Ranges.end(); ++I) {
           const DIExpression *Fragment = I->first->getDebugExpression();
-          if (std::all_of(Ranges.begin(), I,
+          if (std::any_of(Ranges.begin(), I,
                           [&](DbgValueHistoryMap::InstrRange Pred) {
-                            return !Fragment->fragmentsOverlap(
+                            return Fragment->fragmentsOverlap(
                                 Pred.first->getDebugExpression());
                           }))
-            LabelsBeforeInsn[I->first] = Asm->getFunctionBegin();
-          else
             break;
+          if (!IsDescribedByReg(I->first))
+            LabelsBeforeInsn[I->first] = Asm->getFunctionBegin();
         }
       }
     }
diff --git a/lib/CodeGen/AsmPrinter/DebugLocEntry.h b/lib/CodeGen/AsmPrinter/DebugLocEntry.h
index befa4b941c8dbe7e439b5e68caca68773cb0e4f6..40360ba4d34abb65e580c1055c9be07cc018bf23 100644
--- a/lib/CodeGen/AsmPrinter/DebugLocEntry.h
+++ b/lib/CodeGen/AsmPrinter/DebugLocEntry.h
@@ -1,9 +1,8 @@
 //===-- llvm/CodeGen/DebugLocEntry.h - Entry in debug_loc list -*- C++ -*--===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/CodeGen/AsmPrinter/DebugLocStream.cpp b/lib/CodeGen/AsmPrinter/DebugLocStream.cpp
index 7e8ed7104af3419a4ab0c1b9f5f854202ccc8d4f..f483d532ff073031c70bcf33459b3d7f316c930e 100644
--- a/lib/CodeGen/AsmPrinter/DebugLocStream.cpp
+++ b/lib/CodeGen/AsmPrinter/DebugLocStream.cpp
@@ -1,9 +1,8 @@
 //===- DebugLocStream.cpp - DWARF debug_loc stream --------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/CodeGen/AsmPrinter/DebugLocStream.h b/lib/CodeGen/AsmPrinter/DebugLocStream.h
index 8dcf5cbc1889dd7ca87240f7ab73dd28ef42d740..789291771b5a19dfb85679255380b76ed54567b7 100644
--- a/lib/CodeGen/AsmPrinter/DebugLocStream.h
+++ b/lib/CodeGen/AsmPrinter/DebugLocStream.h
@@ -1,9 +1,8 @@
 //===--- lib/CodeGen/DebugLocStream.h - DWARF debug_loc stream --*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/CodeGen/AsmPrinter/DwarfCFIException.cpp b/lib/CodeGen/AsmPrinter/DwarfCFIException.cpp
index 1990456cc555c50497585d47e994a5b33c3d0479..207a7284dafadf647eb746e513fec0e589aa0818 100644
--- a/lib/CodeGen/AsmPrinter/DwarfCFIException.cpp
+++ b/lib/CodeGen/AsmPrinter/DwarfCFIException.cpp
@@ -1,9 +1,8 @@
 //===-- CodeGen/AsmPrinter/DwarfException.cpp - Dwarf Exception Impl ------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp b/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp
index 1dca3f0fce5baf647f26da923f5d14e9746812a0..0a30ede2d2d4ae6797de7adfe4aa7c77745d7278 100644
--- a/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp
+++ b/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp
@@ -1,9 +1,8 @@
 //===- llvm/CodeGen/DwarfCompileUnit.cpp - Dwarf Compile Units ------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -168,6 +167,7 @@ DIE *DwarfCompileUnit::getOrCreateGlobalVariableDIE(
   // Add location.
   bool addToAccelTable = false;
   DIELoc *Loc = nullptr;
+  Optional<unsigned> NVPTXAddressSpace;
   std::unique_ptr<DIEDwarfExpression> DwarfExpr;
   for (const auto &GE : GlobalExprs) {
     const GlobalVariable *Global = GE.Var;
@@ -201,8 +201,24 @@ DIE *DwarfCompileUnit::getOrCreateGlobalVariableDIE(
       DwarfExpr = llvm::make_unique<DIEDwarfExpression>(*Asm, *this, *Loc);
     }
 
-    if (Expr)
+    if (Expr) {
+      // According to
+      // https://docs.nvidia.com/cuda/archive/10.0/ptx-writers-guide-to-interoperability/index.html#cuda-specific-dwarf
+      // cuda-gdb requires DW_AT_address_class for all variables to be able to
+      // correctly interpret address space of the variable address.
+      // Decode DW_OP_constu <DWARF Address Space> DW_OP_swap DW_OP_xderef
+      // sequence for the NVPTX + gdb target.
+      unsigned LocalNVPTXAddressSpace;
+      if (Asm->TM.getTargetTriple().isNVPTX() && DD->tuneForGDB()) {
+        const DIExpression *NewExpr =
+            DIExpression::extractAddressClass(Expr, LocalNVPTXAddressSpace);
+        if (NewExpr != Expr) {
+          Expr = NewExpr;
+          NVPTXAddressSpace = LocalNVPTXAddressSpace;
+        }
+      }
       DwarfExpr->addFragmentOffset(Expr);
+    }
 
     if (Global) {
       const MCSymbol *Sym = Asm->getSymbol(Global);
@@ -247,6 +263,15 @@ DIE *DwarfCompileUnit::getOrCreateGlobalVariableDIE(
       DwarfExpr->setMemoryLocationKind();
     DwarfExpr->addExpression(Expr);
   }
+  if (Asm->TM.getTargetTriple().isNVPTX() && DD->tuneForGDB()) {
+    // According to
+    // https://docs.nvidia.com/cuda/archive/10.0/ptx-writers-guide-to-interoperability/index.html#cuda-specific-dwarf
+    // cuda-gdb requires DW_AT_address_class for all variables to be able to
+    // correctly interpret address space of the variable address.
+    const unsigned NVPTX_ADDR_global_space = 5;
+    addUInt(*VariableDIE, dwarf::DW_AT_address_class, dwarf::DW_FORM_data1,
+            NVPTXAddressSpace ? *NVPTXAddressSpace : NVPTX_ADDR_global_space);
+  }
   if (Loc)
     addBlock(*VariableDIE, dwarf::DW_AT_location, DwarfExpr->finalize());
 
@@ -592,6 +617,7 @@ DIE *DwarfCompileUnit::constructVariableDIEImpl(const DbgVariable &DV,
   if (!DV.hasFrameIndexExprs())
     return VariableDie;
 
+  Optional<unsigned> NVPTXAddressSpace;
   DIELoc *Loc = new (DIEValueAllocator) DIELoc;
   DIEDwarfExpression DwarfExpr(*Asm, *this, *Loc);
   for (auto &Fragment : DV.getFrameIndexExprs()) {
@@ -603,7 +629,23 @@ DIE *DwarfCompileUnit::constructVariableDIEImpl(const DbgVariable &DV,
     SmallVector<uint64_t, 8> Ops;
     Ops.push_back(dwarf::DW_OP_plus_uconst);
     Ops.push_back(Offset);
-    Ops.append(Expr->elements_begin(), Expr->elements_end());
+    // According to
+    // https://docs.nvidia.com/cuda/archive/10.0/ptx-writers-guide-to-interoperability/index.html#cuda-specific-dwarf
+    // cuda-gdb requires DW_AT_address_class for all variables to be able to
+    // correctly interpret address space of the variable address.
+    // Decode DW_OP_constu <DWARF Address Space> DW_OP_swap DW_OP_xderef
+    // sequence for the NVPTX + gdb target.
+    unsigned LocalNVPTXAddressSpace;
+    if (Asm->TM.getTargetTriple().isNVPTX() && DD->tuneForGDB()) {
+      const DIExpression *NewExpr =
+          DIExpression::extractAddressClass(Expr, LocalNVPTXAddressSpace);
+      if (NewExpr != Expr) {
+        Expr = NewExpr;
+        NVPTXAddressSpace = LocalNVPTXAddressSpace;
+      }
+    }
+    if (Expr)
+      Ops.append(Expr->elements_begin(), Expr->elements_end());
     DIExpressionCursor Cursor(Ops);
     DwarfExpr.setMemoryLocationKind();
     if (const MCSymbol *FrameSymbol = Asm->getFunctionFrameSymbol())
@@ -613,6 +655,15 @@ DIE *DwarfCompileUnit::constructVariableDIEImpl(const DbgVariable &DV,
           *Asm->MF->getSubtarget().getRegisterInfo(), Cursor, FrameReg);
     DwarfExpr.addExpression(std::move(Cursor));
   }
+  if (Asm->TM.getTargetTriple().isNVPTX() && DD->tuneForGDB()) {
+    // According to
+    // https://docs.nvidia.com/cuda/archive/10.0/ptx-writers-guide-to-interoperability/index.html#cuda-specific-dwarf
+    // cuda-gdb requires DW_AT_address_class for all variables to be able to
+    // correctly interpret address space of the variable address.
+    const unsigned NVPTX_ADDR_local_space = 6;
+    addUInt(*VariableDie, dwarf::DW_AT_address_class, dwarf::DW_FORM_data1,
+            NVPTXAddressSpace ? *NVPTXAddressSpace : NVPTX_ADDR_local_space);
+  }
   addBlock(*VariableDie, dwarf::DW_AT_location, DwarfExpr.finalize());
 
   return VariableDie;
diff --git a/lib/CodeGen/AsmPrinter/DwarfCompileUnit.h b/lib/CodeGen/AsmPrinter/DwarfCompileUnit.h
index 9ec22f68c12f1a0dc4b69b4f5cca2de589c75143..c5ec49503c6bca83d743c4df51dcb81112136adb 100644
--- a/lib/CodeGen/AsmPrinter/DwarfCompileUnit.h
+++ b/lib/CodeGen/AsmPrinter/DwarfCompileUnit.h
@@ -1,9 +1,8 @@
 //===- llvm/CodeGen/DwarfCompileUnit.h - Dwarf Compile Unit -----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/AsmPrinter/DwarfDebug.cpp b/lib/CodeGen/AsmPrinter/DwarfDebug.cpp
index 2d8c9e497388155f705894fe502285fbc0c4dc4b..1106b859859ea95c8115e14305f628a614d95579 100644
--- a/lib/CodeGen/AsmPrinter/DwarfDebug.cpp
+++ b/lib/CodeGen/AsmPrinter/DwarfDebug.cpp
@@ -1,9 +1,8 @@
 //===- llvm/CodeGen/DwarfDebug.cpp - Dwarf Debug Framework ----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -315,7 +314,7 @@ DwarfDebug::DwarfDebug(AsmPrinter *A, Module *M)
       IsDarwin(A->TM.getTargetTriple().isOSDarwin()) {
   const Triple &TT = Asm->TM.getTargetTriple();
 
-  // Make sure we know our "debugger tuning."  The target option takes
+  // Make sure we know our "debugger tuning".  The target option takes
   // precedence; fall back to triple-based defaults.
   if (Asm->TM.Options.DebuggerTuning != DebuggerKind::Default)
     DebuggerTuning = Asm->TM.Options.DebuggerTuning;
@@ -573,15 +572,6 @@ void DwarfDebug::finishUnitAttributes(const DICompileUnit *DIUnit,
   DIE &Die = NewCU.getUnitDie();
   StringRef FN = DIUnit->getFilename();
 
-  // LTO with assembly output shares a single line table amongst multiple CUs.
-  // To avoid the compilation directory being ambiguous, let the line table
-  // explicitly describe the directory of all files, never relying on the
-  // compilation directory.
-  if (!Asm->OutStreamer->hasRawTextSupport() || SingleCU)
-    Asm->OutStreamer->emitDwarfFile0Directive(
-        CompilationDir, FN, NewCU.getMD5AsBytes(DIUnit->getFile()),
-        DIUnit->getSource(), NewCU.getUniqueID());
-
   StringRef Producer = DIUnit->getProducer();
   StringRef Flags = DIUnit->getFlags();
   if (!Flags.empty() && !useAppleExtensionAttributes()) {
@@ -649,6 +639,16 @@ DwarfDebug::getOrCreateDwarfCompileUnit(const DICompileUnit *DIUnit) {
   for (auto *IE : DIUnit->getImportedEntities())
     NewCU.addImportedEntity(IE);
 
+  // LTO with assembly output shares a single line table amongst multiple CUs.
+  // To avoid the compilation directory being ambiguous, let the line table
+  // explicitly describe the directory of all files, never relying on the
+  // compilation directory.
+  if (!Asm->OutStreamer->hasRawTextSupport() || SingleCU)
+    Asm->OutStreamer->emitDwarfFile0Directive(
+        CompilationDir, DIUnit->getFilename(),
+        NewCU.getMD5AsBytes(DIUnit->getFile()), DIUnit->getSource(),
+        NewCU.getUniqueID());
+
   if (useSplitDwarf()) {
     NewCU.setSkeleton(constructSkeletonCU(NewCU));
     NewCU.setSection(Asm->getObjFileLowering().getDwarfInfoDWOSection());
@@ -1179,6 +1179,18 @@ DwarfDebug::buildLocationList(SmallVectorImpl<DebugLocEntry> &DebugLoc,
     LLVM_DEBUG(dbgs() << "DotDebugLoc: " << *Begin << "\n");
 
     auto Value = getDebugLocValue(Begin);
+
+    // Omit entries with empty ranges as they do not have any effect in DWARF.
+    if (StartLabel == EndLabel) {
+      // If this is a fragment, we must still add the value to the list of
+      // open ranges, since it may describe non-overlapping parts of the
+      // variable.
+      if (DIExpr->isFragment())
+        OpenRanges.push_back(Value);
+      LLVM_DEBUG(dbgs() << "Omitting location list entry with empty range.\n");
+      continue;
+    }
+
     DebugLocEntry Loc(StartLabel, EndLabel, Value);
     bool couldMerge = false;
 
@@ -1476,7 +1488,7 @@ void DwarfDebug::beginInstruction(const MachineInstr *MI) {
   // We have an explicit location, different from the previous location.
   // Don't repeat a line-0 record, but otherwise emit the new location.
   // (The new location might be an explicit line 0, which we do emit.)
-  if (PrevInstLoc && DL.getLine() == 0 && LastAsmLine == 0)
+  if (DL.getLine() == 0 && LastAsmLine == 0)
     return;
   unsigned Flags = 0;
   if (DL == PrologEndLoc) {
@@ -1508,6 +1520,46 @@ static DebugLoc findPrologueEndLoc(const MachineFunction *MF) {
   return DebugLoc();
 }
 
+/// Register a source line with debug info. Returns the  unique label that was
+/// emitted and which provides correspondence to the source line list.
+static void recordSourceLine(AsmPrinter &Asm, unsigned Line, unsigned Col,
+                             const MDNode *S, unsigned Flags, unsigned CUID,
+                             uint16_t DwarfVersion,
+                             ArrayRef<std::unique_ptr<DwarfCompileUnit>> DCUs) {
+  StringRef Fn;
+  unsigned FileNo = 1;
+  unsigned Discriminator = 0;
+  if (auto *Scope = cast_or_null<DIScope>(S)) {
+    Fn = Scope->getFilename();
+    if (Line != 0 && DwarfVersion >= 4)
+      if (auto *LBF = dyn_cast<DILexicalBlockFile>(Scope))
+        Discriminator = LBF->getDiscriminator();
+
+    FileNo = static_cast<DwarfCompileUnit &>(*DCUs[CUID])
+                 .getOrCreateSourceID(Scope->getFile());
+  }
+  Asm.OutStreamer->EmitDwarfLocDirective(FileNo, Line, Col, Flags, 0,
+                                         Discriminator, Fn);
+}
+
+DebugLoc DwarfDebug::emitInitialLocDirective(const MachineFunction &MF,
+                                             unsigned CUID) {
+  // Get beginning of function.
+  if (DebugLoc PrologEndLoc = findPrologueEndLoc(&MF)) {
+    // Ensure the compile unit is created if the function is called before
+    // beginFunction().
+    (void)getOrCreateDwarfCompileUnit(
+        MF.getFunction().getSubprogram()->getUnit());
+    // We'd like to list the prologue as "not statements" but GDB behaves
+    // poorly if we do that. Revisit this with caution/GDB (7.5+) testing.
+    const DISubprogram *SP = PrologEndLoc->getInlinedAtScope()->getSubprogram();
+    ::recordSourceLine(*Asm, SP->getScopeLine(), 0, SP, DWARF2_FLAG_IS_STMT,
+                       CUID, getDwarfVersion(), getUnits());
+    return PrologEndLoc;
+  }
+  return DebugLoc();
+}
+
 // Gather pre-function debug information.  Assumes being called immediately
 // after the function entry point has been emitted.
 void DwarfDebug::beginFunctionImpl(const MachineFunction *MF) {
@@ -1530,13 +1582,8 @@ void DwarfDebug::beginFunctionImpl(const MachineFunction *MF) {
     Asm->OutStreamer->getContext().setDwarfCompileUnitID(CU.getUniqueID());
 
   // Record beginning of function.
-  PrologEndLoc = findPrologueEndLoc(MF);
-  if (PrologEndLoc) {
-    // We'd like to list the prologue as "not statements" but GDB behaves
-    // poorly if we do that. Revisit this with caution/GDB (7.5+) testing.
-    auto *SP = PrologEndLoc->getInlinedAtScope()->getSubprogram();
-    recordSourceLine(SP->getScopeLine(), 0, SP, DWARF2_FLAG_IS_STMT);
-  }
+  PrologEndLoc = emitInitialLocDirective(
+      *MF, Asm->OutStreamer->getContext().getDwarfCompileUnitID());
 }
 
 void DwarfDebug::skippedNonDebugFunction() {
@@ -1634,21 +1681,9 @@ void DwarfDebug::endFunctionImpl(const MachineFunction *MF) {
 // emitted and which provides correspondence to the source line list.
 void DwarfDebug::recordSourceLine(unsigned Line, unsigned Col, const MDNode *S,
                                   unsigned Flags) {
-  StringRef Fn;
-  unsigned FileNo = 1;
-  unsigned Discriminator = 0;
-  if (auto *Scope = cast_or_null<DIScope>(S)) {
-    Fn = Scope->getFilename();
-    if (Line != 0 && getDwarfVersion() >= 4)
-      if (auto *LBF = dyn_cast<DILexicalBlockFile>(Scope))
-        Discriminator = LBF->getDiscriminator();
-
-    unsigned CUID = Asm->OutStreamer->getContext().getDwarfCompileUnitID();
-    FileNo = static_cast<DwarfCompileUnit &>(*InfoHolder.getUnits()[CUID])
-              .getOrCreateSourceID(Scope->getFile());
-  }
-  Asm->OutStreamer->EmitDwarfLocDirective(FileNo, Line, Col, Flags, 0,
-                                          Discriminator, Fn);
+  ::recordSourceLine(*Asm, Line, Col, S, Flags,
+                     Asm->OutStreamer->getContext().getDwarfCompileUnitID(),
+                     getDwarfVersion(), getUnits());
 }
 
 //===----------------------------------------------------------------------===//
@@ -1917,6 +1952,7 @@ static void emitDebugLocValue(const AsmPrinter &AP, const DIBasicType *BT,
 void DebugLocEntry::finalize(const AsmPrinter &AP,
                              DebugLocStream::ListBuilder &List,
                              const DIBasicType *BT) {
+  assert(Begin != End && "unexpected location list entry with empty range");
   DebugLocStream::EntryBuilder Entry(List, Begin, End);
   BufferByteStreamer Streamer = Entry.getStreamer();
   DebugLocDwarfExpression DwarfExpr(AP.getDwarfVersion(), Streamer);
@@ -1942,8 +1978,10 @@ void DebugLocEntry::finalize(const AsmPrinter &AP,
 void DwarfDebug::emitDebugLocEntryLocation(const DebugLocStream::Entry &Entry) {
   // Emit the size.
   Asm->OutStreamer->AddComment("Loc expr size");
-  Asm->emitInt16(DebugLocs.getBytes(Entry).size());
-
+  if (getDwarfVersion() >= 5)
+    Asm->EmitULEB128(DebugLocs.getBytes(Entry).size());
+  else
+    Asm->emitInt16(DebugLocs.getBytes(Entry).size());
   // Emit the entry.
   APByteStreamer Streamer(*Asm);
   emitDebugLocEntry(Streamer, Entry);
@@ -2086,9 +2124,9 @@ void DwarfDebug::emitDebugLoc() {
 }
 
 void DwarfDebug::emitDebugLocDWO() {
-  Asm->OutStreamer->SwitchSection(
-      Asm->getObjFileLowering().getDwarfLocDWOSection());
   for (const auto &List : DebugLocs.getLists()) {
+    Asm->OutStreamer->SwitchSection(
+        Asm->getObjFileLowering().getDwarfLocDWOSection());
     Asm->OutStreamer->EmitLabel(List.Label);
     for (const auto &Entry : DebugLocs.getEntries(List)) {
       // GDB only supports startx_length in pre-standard split-DWARF.
@@ -2362,8 +2400,8 @@ static void emitRangeList(DwarfDebug &DD, AsmPrinter *Asm,
   }
 }
 
-void emitDebugRangesImpl(DwarfDebug &DD, AsmPrinter *Asm,
-                         const DwarfFile &Holder, MCSymbol *TableEnd) {
+static void emitDebugRangesImpl(DwarfDebug &DD, AsmPrinter *Asm,
+                                const DwarfFile &Holder, MCSymbol *TableEnd) {
   for (const RangeSpanList &List : Holder.getRangeLists())
     emitRangeList(DD, Asm, List);
 
diff --git a/lib/CodeGen/AsmPrinter/DwarfDebug.h b/lib/CodeGen/AsmPrinter/DwarfDebug.h
index 8a31e989b289a4c3c8c87310be2cd9041f3557f3..f0d50a4764090a48dda59c192f6f42913ac29f69 100644
--- a/lib/CodeGen/AsmPrinter/DwarfDebug.h
+++ b/lib/CodeGen/AsmPrinter/DwarfDebug.h
@@ -1,9 +1,8 @@
 //===- llvm/CodeGen/DwarfDebug.h - Dwarf Debug Framework --------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -593,6 +592,9 @@ public:
   /// Emit all Dwarf sections that should come after the content.
   void endModule() override;
 
+  /// Emits inital debug location directive.
+  DebugLoc emitInitialLocDirective(const MachineFunction &MF, unsigned CUID);
+
   /// Process beginning of an instruction.
   void beginInstruction(const MachineInstr *MI) override;
 
diff --git a/lib/CodeGen/AsmPrinter/DwarfException.h b/lib/CodeGen/AsmPrinter/DwarfException.h
index b57ea8fc6322615f6d3bcb9eb5b6359346b1a42e..24bbf58b91ec93552636c265da9aeeaab66b6f2a 100644
--- a/lib/CodeGen/AsmPrinter/DwarfException.h
+++ b/lib/CodeGen/AsmPrinter/DwarfException.h
@@ -1,9 +1,8 @@
 //===-- DwarfException.h - Dwarf Exception Framework -----------*- C++ -*--===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/AsmPrinter/DwarfExpression.cpp b/lib/CodeGen/AsmPrinter/DwarfExpression.cpp
index 19c350afbf1776a5c42cd026143bf1dbe50ec95c..558d33ac0e6bdf4d3d11a802784daf06023767a7 100644
--- a/lib/CodeGen/AsmPrinter/DwarfExpression.cpp
+++ b/lib/CodeGen/AsmPrinter/DwarfExpression.cpp
@@ -1,9 +1,8 @@
 //===- llvm/CodeGen/DwarfExpression.cpp - Dwarf Debug Framework -----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/AsmPrinter/DwarfExpression.h b/lib/CodeGen/AsmPrinter/DwarfExpression.h
index d47c4d1d296967300a90f47615ceaba6d7cbe7af..1f7b89fe5fcabaeca4938c51e0dab5903ca46a3f 100644
--- a/lib/CodeGen/AsmPrinter/DwarfExpression.h
+++ b/lib/CodeGen/AsmPrinter/DwarfExpression.h
@@ -1,9 +1,8 @@
 //===- llvm/CodeGen/DwarfExpression.h - Dwarf Compile Unit ------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -190,7 +189,7 @@ protected:
   /// DW_OP_stack_value.  Unfortunately, DW_OP_stack_value was not available
   /// until DWARF 4, so we will continue to generate DW_OP_constu <const> for
   /// DWARF 2 and DWARF 3. Technically, this is incorrect since DW_OP_const
-  /// <const> actually describes a value at a constant addess, not a constant
+  /// <const> actually describes a value at a constant address, not a constant
   /// value.  However, in the past there was no better way to describe a
   /// constant value, so the producers and consumers started to rely on
   /// heuristics to disambiguate the value vs. location status of the
diff --git a/lib/CodeGen/AsmPrinter/DwarfFile.cpp b/lib/CodeGen/AsmPrinter/DwarfFile.cpp
index 78ccad4814111b77bfff2801c6d786c19e70fe63..e3c9095d13438819a690c5f39415783566c90559 100644
--- a/lib/CodeGen/AsmPrinter/DwarfFile.cpp
+++ b/lib/CodeGen/AsmPrinter/DwarfFile.cpp
@@ -1,9 +1,8 @@
 //===- llvm/CodeGen/DwarfFile.cpp - Dwarf Debug Framework -----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -44,6 +43,11 @@ void DwarfFile::emitUnit(DwarfUnit *TheU, bool UseOffsets) {
   if (!S)
     return;
 
+  // Skip CUs that ended up not being needed (split CUs that were abandoned
+  // because they added no information beyond the non-split CU)
+  if (llvm::empty(TheU->getUnitDie().values()))
+    return;
+
   Asm->OutStreamer->SwitchSection(S);
   TheU->emitHeader(UseOffsets);
   Asm->emitDwarfDIE(TheU->getUnitDie());
@@ -63,6 +67,11 @@ void DwarfFile::computeSizeAndOffsets() {
     if (TheU->getCUNode()->isDebugDirectivesOnly())
       continue;
 
+    // Skip CUs that ended up not being needed (split CUs that were abandoned
+    // because they added no information beyond the non-split CU)
+    if (llvm::empty(TheU->getUnitDie().values()))
+      return;
+
     TheU->setDebugSectionOffset(SecOffset);
     SecOffset += computeSizeAndOffsetsForUnit(TheU.get());
   }
diff --git a/lib/CodeGen/AsmPrinter/DwarfFile.h b/lib/CodeGen/AsmPrinter/DwarfFile.h
index 51acca8c1e53424e5a710813c525be0d7359e618..b3026a35b8567209f886d871533488198db8a870 100644
--- a/lib/CodeGen/AsmPrinter/DwarfFile.h
+++ b/lib/CodeGen/AsmPrinter/DwarfFile.h
@@ -1,9 +1,8 @@
 //===- llvm/CodeGen/DwarfFile.h - Dwarf Debug Framework ---------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/CodeGen/AsmPrinter/DwarfStringPool.cpp b/lib/CodeGen/AsmPrinter/DwarfStringPool.cpp
index 02016534a774d21f6f12e9e0d51e2d5b13ef6045..2a76dcb1b0827658e33a8ec162dccdade22719bd 100644
--- a/lib/CodeGen/AsmPrinter/DwarfStringPool.cpp
+++ b/lib/CodeGen/AsmPrinter/DwarfStringPool.cpp
@@ -1,9 +1,8 @@
 //===- llvm/CodeGen/DwarfStringPool.cpp - Dwarf Debug Framework -----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/CodeGen/AsmPrinter/DwarfStringPool.h b/lib/CodeGen/AsmPrinter/DwarfStringPool.h
index f484540d8d37aacf038f716299a09b27b89f7530..c5f5637fdae3fa4a5facd86582c62bf2ceedb00c 100644
--- a/lib/CodeGen/AsmPrinter/DwarfStringPool.h
+++ b/lib/CodeGen/AsmPrinter/DwarfStringPool.h
@@ -1,9 +1,8 @@
 //===- llvm/CodeGen/DwarfStringPool.h - Dwarf Debug Framework ---*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/CodeGen/AsmPrinter/DwarfUnit.cpp b/lib/CodeGen/AsmPrinter/DwarfUnit.cpp
index 991dac7cf35b0e96f6d5476abfd2334d26e031e9..8534bd12ba22171ec7640c31b01e25fb6447fa51 100644
--- a/lib/CodeGen/AsmPrinter/DwarfUnit.cpp
+++ b/lib/CodeGen/AsmPrinter/DwarfUnit.cpp
@@ -1,9 +1,8 @@
 //===-- llvm/CodeGen/DwarfUnit.cpp - Dwarf Type and Compile Units ---------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -1361,7 +1360,7 @@ void DwarfUnit::constructEnumTypeDIE(DIE &Buffer, const DICompositeType *CTy) {
   if (DTy) {
     if (DD->getDwarfVersion() >= 3)
       addType(Buffer, DTy);
-    if (DD->getDwarfVersion() >= 4 && (CTy->getFlags() & DINode::FlagFixedEnum))
+    if (DD->getDwarfVersion() >= 4 && (CTy->getFlags() & DINode::FlagEnumClass))
       addFlag(Buffer, dwarf::DW_AT_enum_class);
   }
 
diff --git a/lib/CodeGen/AsmPrinter/DwarfUnit.h b/lib/CodeGen/AsmPrinter/DwarfUnit.h
index a59ebb7c1465c93c7cd27f0a0841408de4fe7722..2278bb0ebe168fe9681cb7441e9441e192bf96ba 100644
--- a/lib/CodeGen/AsmPrinter/DwarfUnit.h
+++ b/lib/CodeGen/AsmPrinter/DwarfUnit.h
@@ -1,9 +1,8 @@
 //===-- llvm/CodeGen/DwarfUnit.h - Dwarf Compile Unit ---*- C++ -*--===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/AsmPrinter/EHStreamer.cpp b/lib/CodeGen/AsmPrinter/EHStreamer.cpp
index 7599121de2b06c9c84c1c598b008b384e5aece63..2786f8d9f6a4031948540e609d5cb68fc4b8e42c 100644
--- a/lib/CodeGen/AsmPrinter/EHStreamer.cpp
+++ b/lib/CodeGen/AsmPrinter/EHStreamer.cpp
@@ -1,9 +1,8 @@
 //===- CodeGen/AsmPrinter/EHStreamer.cpp - Exception Directive Streamer ---===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/AsmPrinter/EHStreamer.h b/lib/CodeGen/AsmPrinter/EHStreamer.h
index ce912d032c6dac48d50709b223f3b89dc5a1125f..e62cf17a05d4b95060c9cc8454844c2bfa9d8b81 100644
--- a/lib/CodeGen/AsmPrinter/EHStreamer.h
+++ b/lib/CodeGen/AsmPrinter/EHStreamer.h
@@ -1,9 +1,8 @@
 //===- EHStreamer.h - Exception Handling Directive Streamer -----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/AsmPrinter/ErlangGCPrinter.cpp b/lib/CodeGen/AsmPrinter/ErlangGCPrinter.cpp
index 34677ecc9e6945212e313b03249ad1cc1e1d35e4..39392b79e96075e874c7d689411a071529365cc6 100644
--- a/lib/CodeGen/AsmPrinter/ErlangGCPrinter.cpp
+++ b/lib/CodeGen/AsmPrinter/ErlangGCPrinter.cpp
@@ -1,9 +1,8 @@
 //===- ErlangGCPrinter.cpp - Erlang/OTP frametable emitter ----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/AsmPrinter/LLVMBuild.txt b/lib/CodeGen/AsmPrinter/LLVMBuild.txt
index bde8148d259bc1e6a8e76fd85d438e6a9c81601a..88d7c13ce035b0469264b281285050d9b895eef9 100644
--- a/lib/CodeGen/AsmPrinter/LLVMBuild.txt
+++ b/lib/CodeGen/AsmPrinter/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/CodeGen/AsmPrinter/LLVMBuild.txt -------------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/CodeGen/AsmPrinter/OcamlGCPrinter.cpp b/lib/CodeGen/AsmPrinter/OcamlGCPrinter.cpp
index 3479a00def23b412b35f439e9e416c79d6b744bb..3145cc90dc73b4a98942ff1e6f9d5285b2b24493 100644
--- a/lib/CodeGen/AsmPrinter/OcamlGCPrinter.cpp
+++ b/lib/CodeGen/AsmPrinter/OcamlGCPrinter.cpp
@@ -1,9 +1,8 @@
 //===- OcamlGCPrinter.cpp - Ocaml frametable emitter ----------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/AsmPrinter/WasmException.cpp b/lib/CodeGen/AsmPrinter/WasmException.cpp
index 527e5ae50146a5d98bbbe6d4cd34fa8d9d043c54..444b0ed17b6d3aaadb408cd6f3e1f11ad0796145 100644
--- a/lib/CodeGen/AsmPrinter/WasmException.cpp
+++ b/lib/CodeGen/AsmPrinter/WasmException.cpp
@@ -1,9 +1,8 @@
 //===-- CodeGen/AsmPrinter/WasmException.cpp - Wasm Exception Impl --------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -19,10 +18,10 @@
 using namespace llvm;
 
 void WasmException::endModule() {
-  // This is the symbol used in 'throw' and 'if_except' instruction to denote
+  // This is the symbol used in 'throw' and 'br_on_exn' instruction to denote
   // this is a C++ exception. This symbol has to be emitted somewhere once in
   // the module.  Check if the symbol has already been created, i.e., we have at
-  // least one 'throw' or 'if_except' instruction in the module, and emit the
+  // least one 'throw' or 'br_on_exn' instruction in the module, and emit the
   // symbol only if so.
   SmallString<60> NameStr;
   Mangler::getNameWithPrefix(NameStr, "__cpp_exception", Asm->getDataLayout());
diff --git a/lib/CodeGen/AsmPrinter/WasmException.h b/lib/CodeGen/AsmPrinter/WasmException.h
index cbdb42457cf80eaf75eb19b303365a310223e4b6..1893b6b2df43d1a5ef56bd6e2ec3748419cda164 100644
--- a/lib/CodeGen/AsmPrinter/WasmException.h
+++ b/lib/CodeGen/AsmPrinter/WasmException.h
@@ -1,9 +1,8 @@
 //===-- WasmException.h - Wasm Exception Framework -------------*- C++ -*--===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/AsmPrinter/WinCFGuard.cpp b/lib/CodeGen/AsmPrinter/WinCFGuard.cpp
index 18d37caf57ee494a42d7bb262134fc91e543758f..290be81c6baabef6155607701111e0406aed5b72 100644
--- a/lib/CodeGen/AsmPrinter/WinCFGuard.cpp
+++ b/lib/CodeGen/AsmPrinter/WinCFGuard.cpp
@@ -1,9 +1,8 @@
 //===-- CodeGen/AsmPrinter/WinCFGuard.cpp - Control Flow Guard Impl ------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/AsmPrinter/WinCFGuard.h b/lib/CodeGen/AsmPrinter/WinCFGuard.h
index 28f119e359661f4d2ca89eaeddb9c716d2164f0f..def0a59ab0075d0e0fcbc2a34a3065e4a1e7e0fb 100644
--- a/lib/CodeGen/AsmPrinter/WinCFGuard.h
+++ b/lib/CodeGen/AsmPrinter/WinCFGuard.h
@@ -1,9 +1,8 @@
 //===-- WinCFGuard.h - Windows Control Flow Guard Handling ----*- C++ -*--===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/AsmPrinter/WinException.cpp b/lib/CodeGen/AsmPrinter/WinException.cpp
index 2a97a2fde43d1520752c2fe132e25b004c538375..c870419056ba669971a376c9753d39b44390bb8b 100644
--- a/lib/CodeGen/AsmPrinter/WinException.cpp
+++ b/lib/CodeGen/AsmPrinter/WinException.cpp
@@ -1,9 +1,8 @@
 //===-- CodeGen/AsmPrinter/WinException.cpp - Dwarf Exception Impl ------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -545,15 +544,17 @@ void WinException::emitCSpecificHandlerTable(const MachineFunction *MF) {
       OS.AddComment(Comment);
   };
 
-  // Emit a label assignment with the SEH frame offset so we can use it for
-  // llvm.x86.seh.recoverfp.
-  StringRef FLinkageName =
-      GlobalValue::dropLLVMManglingEscape(MF->getFunction().getName());
-  MCSymbol *ParentFrameOffset =
-      Ctx.getOrCreateParentFrameOffsetSymbol(FLinkageName);
-  const MCExpr *MCOffset =
-      MCConstantExpr::create(FuncInfo.SEHSetFrameOffset, Ctx);
-  Asm->OutStreamer->EmitAssignment(ParentFrameOffset, MCOffset);
+  if (!isAArch64) {
+    // Emit a label assignment with the SEH frame offset so we can use it for
+    // llvm.eh.recoverfp.
+    StringRef FLinkageName =
+        GlobalValue::dropLLVMManglingEscape(MF->getFunction().getName());
+    MCSymbol *ParentFrameOffset =
+        Ctx.getOrCreateParentFrameOffsetSymbol(FLinkageName);
+    const MCExpr *MCOffset =
+        MCConstantExpr::create(FuncInfo.SEHSetFrameOffset, Ctx);
+    Asm->OutStreamer->EmitAssignment(ParentFrameOffset, MCOffset);
+  }
 
   // Use the assembler to compute the number of table entries through label
   // difference and division.
@@ -936,8 +937,7 @@ void WinException::emitEHRegistrationOffsetLabel(const WinEHFuncInfo &FuncInfo,
   int FI = FuncInfo.EHRegNodeFrameIndex;
   if (FI != INT_MAX) {
     const TargetFrameLowering *TFI = Asm->MF->getSubtarget().getFrameLowering();
-    unsigned UnusedReg;
-    Offset = TFI->getFrameIndexReference(*Asm->MF, FI, UnusedReg);
+    Offset = TFI->getNonLocalFrameIndexReference(*Asm->MF, FI);
   }
 
   MCContext &Ctx = Asm->OutContext;
diff --git a/lib/CodeGen/AsmPrinter/WinException.h b/lib/CodeGen/AsmPrinter/WinException.h
index 728cde3b250279b11ca871b2e4db4cdfc7799a7e..9491d7d69c2a4addaae77d6912988a4c7cc3c069 100644
--- a/lib/CodeGen/AsmPrinter/WinException.h
+++ b/lib/CodeGen/AsmPrinter/WinException.h
@@ -1,9 +1,8 @@
 //===-- WinException.h - Windows Exception Handling ----------*- C++ -*--===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -68,7 +67,7 @@ class LLVM_LIBRARY_VISIBILITY WinException : public EHStreamer {
       const MachineFunction *MF, const WinEHFuncInfo &FuncInfo,
       SmallVectorImpl<std::pair<const MCExpr *, int>> &IPToStateTable);
 
-  /// Emits the label used with llvm.x86.seh.recoverfp, which is used by
+  /// Emits the label used with llvm.eh.recoverfp, which is used by
   /// outlined funclets.
   void emitEHRegistrationOffsetLabel(const WinEHFuncInfo &FuncInfo,
                                      StringRef FLinkageName);
diff --git a/lib/CodeGen/AtomicExpandPass.cpp b/lib/CodeGen/AtomicExpandPass.cpp
index 95581c09dd1c46d5487ae2bd60b21fe1f40292cf..c247a8afcf700989cf3815c4d5f383dc528cd432 100644
--- a/lib/CodeGen/AtomicExpandPass.cpp
+++ b/lib/CodeGen/AtomicExpandPass.cpp
@@ -1,9 +1,8 @@
 //===- AtomicExpandPass.cpp - Expand atomic instructions ------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -382,7 +381,7 @@ LoadInst *AtomicExpand::convertAtomicLoadToIntegerType(LoadInst *LI) {
                               Addr->getType()->getPointerAddressSpace());
   Value *NewAddr = Builder.CreateBitCast(Addr, PT);
 
-  auto *NewLI = Builder.CreateLoad(NewAddr);
+  auto *NewLI = Builder.CreateLoad(NewTy, NewAddr);
   NewLI->setAlignment(LI->getAlignment());
   NewLI->setVolatile(LI->isVolatile());
   NewLI->setAtomic(LI->getOrdering(), LI->getSyncScopeID());
@@ -496,11 +495,26 @@ static void createCmpXchgInstFun(IRBuilder<> &Builder, Value *Addr,
                                  Value *Loaded, Value *NewVal,
                                  AtomicOrdering MemOpOrder,
                                  Value *&Success, Value *&NewLoaded) {
+  Type *OrigTy = NewVal->getType();
+
+  // This code can go away when cmpxchg supports FP types.
+  bool NeedBitcast = OrigTy->isFloatingPointTy();
+  if (NeedBitcast) {
+    IntegerType *IntTy = Builder.getIntNTy(OrigTy->getPrimitiveSizeInBits());
+    unsigned AS = Addr->getType()->getPointerAddressSpace();
+    Addr = Builder.CreateBitCast(Addr, IntTy->getPointerTo(AS));
+    NewVal = Builder.CreateBitCast(NewVal, IntTy);
+    Loaded = Builder.CreateBitCast(Loaded, IntTy);
+  }
+
   Value* Pair = Builder.CreateAtomicCmpXchg(
       Addr, Loaded, NewVal, MemOpOrder,
       AtomicCmpXchgInst::getStrongestFailureOrdering(MemOpOrder));
   Success = Builder.CreateExtractValue(Pair, 1, "success");
   NewLoaded = Builder.CreateExtractValue(Pair, 0, "newloaded");
+
+  if (NeedBitcast)
+    NewLoaded = Builder.CreateBitCast(NewLoaded, OrigTy);
 }
 
 /// Emit IR to implement the given atomicrmw operation on values in registers,
@@ -535,6 +549,10 @@ static Value *performAtomicOp(AtomicRMWInst::BinOp Op, IRBuilder<> &Builder,
   case AtomicRMWInst::UMin:
     NewVal = Builder.CreateICmpULE(Loaded, Inc);
     return Builder.CreateSelect(NewVal, Loaded, Inc, "new");
+  case AtomicRMWInst::FAdd:
+    return Builder.CreateFAdd(Loaded, Inc, "new");
+  case AtomicRMWInst::FSub:
+    return Builder.CreateFSub(Loaded, Inc, "new");
   default:
     llvm_unreachable("Unknown atomic op");
   }
@@ -1533,6 +1551,8 @@ static ArrayRef<RTLIB::Libcall> GetRMWLibcall(AtomicRMWInst::BinOp Op) {
   case AtomicRMWInst::Min:
   case AtomicRMWInst::UMax:
   case AtomicRMWInst::UMin:
+  case AtomicRMWInst::FAdd:
+  case AtomicRMWInst::FSub:
     // No atomic libcalls are available for max/min/umax/umin.
     return {};
   }
@@ -1734,7 +1754,7 @@ bool AtomicExpand::expandAtomicOpToLibcall(
   for (Value *Arg : Args)
     ArgTys.push_back(Arg->getType());
   FunctionType *FnType = FunctionType::get(ResultTy, ArgTys, false);
-  Constant *LibcallFn =
+  FunctionCallee LibcallFn =
       M->getOrInsertFunction(TLI->getLibcallName(RTLibType), FnType, Attr);
   CallInst *Call = Builder.CreateCall(LibcallFn, Args);
   Call->setAttributes(Attr);
@@ -1749,8 +1769,8 @@ bool AtomicExpand::expandAtomicOpToLibcall(
     // from call}
     Type *FinalResultTy = I->getType();
     Value *V = UndefValue::get(FinalResultTy);
-    Value *ExpectedOut =
-        Builder.CreateAlignedLoad(AllocaCASExpected, AllocaAlignment);
+    Value *ExpectedOut = Builder.CreateAlignedLoad(
+        CASExpected->getType(), AllocaCASExpected, AllocaAlignment);
     Builder.CreateLifetimeEnd(AllocaCASExpected_i8, SizeVal64);
     V = Builder.CreateInsertValue(V, ExpectedOut, 0);
     V = Builder.CreateInsertValue(V, Result, 1);
@@ -1760,7 +1780,8 @@ bool AtomicExpand::expandAtomicOpToLibcall(
     if (UseSizedLibcall)
       V = Builder.CreateBitOrPointerCast(Result, I->getType());
     else {
-      V = Builder.CreateAlignedLoad(AllocaResult, AllocaAlignment);
+      V = Builder.CreateAlignedLoad(I->getType(), AllocaResult,
+                                    AllocaAlignment);
       Builder.CreateLifetimeEnd(AllocaResult_i8, SizeVal64);
     }
     I->replaceAllUsesWith(V);
diff --git a/lib/CodeGen/BasicTargetTransformInfo.cpp b/lib/CodeGen/BasicTargetTransformInfo.cpp
index d11f375b176e91a314917d3c478aca72ab717bff..57cefae2066a9cd51dab881a541d2fa9f64dcbfc 100644
--- a/lib/CodeGen/BasicTargetTransformInfo.cpp
+++ b/lib/CodeGen/BasicTargetTransformInfo.cpp
@@ -1,9 +1,8 @@
 //===- BasicTargetTransformInfo.cpp - Basic target-independent TTI impl ---===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/lib/CodeGen/BranchFolding.cpp b/lib/CodeGen/BranchFolding.cpp
index efbfd5f4ab2c582e9b06674810eb0111de99ef0f..95b0ddec63b81066f9b6ead5037c885e8579c37b 100644
--- a/lib/CodeGen/BranchFolding.cpp
+++ b/lib/CodeGen/BranchFolding.cpp
@@ -1,9 +1,8 @@
 //===- BranchFolding.cpp - Fold machine code branch instructions ----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -1183,29 +1182,6 @@ bool BranchFolder::TailMergeBlocks(MachineFunction &MF) {
           }
         }
 
-        // Failing case: the only way IBB can be reached from PBB is via
-        // exception handling.  Happens for landing pads.  Would be nice to have
-        // a bit in the edge so we didn't have to do all this.
-        if (IBB->isEHPad()) {
-          MachineFunction::iterator IP = ++PBB->getIterator();
-          MachineBasicBlock *PredNextBB = nullptr;
-          if (IP != MF.end())
-            PredNextBB = &*IP;
-          if (!TBB) {
-            if (IBB != PredNextBB)      // fallthrough
-              continue;
-          } else if (FBB) {
-            if (TBB != IBB && FBB != IBB)   // cbr then ubr
-              continue;
-          } else if (Cond.empty()) {
-            if (TBB != IBB)               // ubr
-              continue;
-          } else {
-            if (TBB != IBB && IBB != PredNextBB)  // cbr
-              continue;
-          }
-        }
-
         // Remove the unconditional branch at the end, if any.
         if (TBB && (Cond.empty() || FBB)) {
           DebugLoc dl = PBB->findBranchDebugLoc();
diff --git a/lib/CodeGen/BranchFolding.h b/lib/CodeGen/BranchFolding.h
index accd0ab7317bb8adb5eadd420271e5527bea735d..761ff9c7d54e2e291033f48eee2571d81819a07c 100644
--- a/lib/CodeGen/BranchFolding.h
+++ b/lib/CodeGen/BranchFolding.h
@@ -1,9 +1,8 @@
 //===- BranchFolding.h - Fold machine code branch instructions --*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/CodeGen/BranchRelaxation.cpp b/lib/CodeGen/BranchRelaxation.cpp
index c092da2b66026fde6833867b94dbbe9253838df5..3ad6266d4f357cfe7238cadc3885c2d4b13bda4a 100644
--- a/lib/CodeGen/BranchRelaxation.cpp
+++ b/lib/CodeGen/BranchRelaxation.cpp
@@ -1,9 +1,8 @@
 //===- BranchRelaxation.cpp -----------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/CodeGen/BreakFalseDeps.cpp b/lib/CodeGen/BreakFalseDeps.cpp
index 210699cbf2397ac5964dcb1e89c101fe267739c7..cc4b2caa9bed94fcffc08307c99cc7dadd666a7e 100644
--- a/lib/CodeGen/BreakFalseDeps.cpp
+++ b/lib/CodeGen/BreakFalseDeps.cpp
@@ -1,9 +1,8 @@
 //==- llvm/CodeGen/BreakFalseDeps.cpp - Break False Dependency Fix -*- C++ -*==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/BuiltinGCs.cpp b/lib/CodeGen/BuiltinGCs.cpp
index 93939e573b7baa41052cf40c6be5c82cb8b6b753..bfc10cb3fef2aa66974f47fbf37388c1a37d5632 100644
--- a/lib/CodeGen/BuiltinGCs.cpp
+++ b/lib/CodeGen/BuiltinGCs.cpp
@@ -1,9 +1,8 @@
 //===- BuiltinGCs.cpp - Boilerplate for our built in GC types -------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/CFIInstrInserter.cpp b/lib/CodeGen/CFIInstrInserter.cpp
index c4799855a2b30eed983974b647511ea9a692da4d..1a4d54231cfd491f5424f842ccf17db72bc11021 100644
--- a/lib/CodeGen/CFIInstrInserter.cpp
+++ b/lib/CodeGen/CFIInstrInserter.cpp
@@ -1,9 +1,8 @@
 //===------ CFIInstrInserter.cpp - Insert additional CFI instructions -----===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/CalcSpillWeights.cpp b/lib/CodeGen/CalcSpillWeights.cpp
index 02347b9f0b5cd2740d2940c52c5f0926c341a7c7..7164fdfb788671f34b16c62fc016075e1540103e 100644
--- a/lib/CodeGen/CalcSpillWeights.cpp
+++ b/lib/CodeGen/CalcSpillWeights.cpp
@@ -1,9 +1,8 @@
 //===- CalcSpillWeights.cpp -----------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/CodeGen/CallingConvLower.cpp b/lib/CodeGen/CallingConvLower.cpp
index 3593089b206dbab41440f460d84a7a0f3b882029..497fcb1478495e1572c174614e1ef42b10d43559 100644
--- a/lib/CodeGen/CallingConvLower.cpp
+++ b/lib/CodeGen/CallingConvLower.cpp
@@ -1,9 +1,8 @@
 //===-- CallingConvLower.cpp - Calling Conventions ------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/CodeGen.cpp b/lib/CodeGen/CodeGen.cpp
index 66166482c78bcd7ca31566a3f7b339c8cf929a61..748db034d272bfaedcc3770c65c4387adf65ce7b 100644
--- a/lib/CodeGen/CodeGen.cpp
+++ b/lib/CodeGen/CodeGen.cpp
@@ -1,9 +1,8 @@
 //===-- CodeGen.cpp -------------------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/CodeGenPrepare.cpp b/lib/CodeGen/CodeGenPrepare.cpp
index 392e8bc0db48021a640b87c392969a4343d67846..14f56279e851236206e8fbc6d59ec54749f8b9fe 100644
--- a/lib/CodeGen/CodeGenPrepare.cpp
+++ b/lib/CodeGen/CodeGenPrepare.cpp
@@ -1,9 +1,8 @@
 //===- CodeGenPrepare.cpp - Prepare a function for code generation --------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -376,6 +375,8 @@ class TypePromotionTransaction;
         SmallVectorImpl<Instruction *> &SpeculativelyMovedExts);
     bool splitBranchCondition(Function &F);
     bool simplifyOffsetableRelocate(Instruction &I);
+
+    bool tryToSinkFreeOperands(Instruction *I);
   };
 
 } // end anonymous namespace
@@ -654,6 +655,16 @@ bool CodeGenPrepare::isMergingEmptyBlockProfitable(BasicBlock *BB,
         BB->getSinglePredecessor()->getSingleSuccessor()))
     return false;
 
+  // Skip merging if the block's successor is also a successor to any callbr
+  // that leads to this block.
+  // FIXME: Is this really needed? Is this a correctness issue?
+  for (pred_iterator PI = pred_begin(BB), E = pred_end(BB); PI != E; ++PI) {
+    if (auto *CBI = dyn_cast<CallBrInst>((*PI)->getTerminator()))
+      for (unsigned i = 0, e = CBI->getNumSuccessors(); i != e; ++i)
+        if (DestBB == CBI->getSuccessor(i))
+          return false;
+  }
+
   // Try to skip merging if the unique predecessor of BB is terminated by a
   // switch or indirect branch instruction, and BB is used as an incoming block
   // of PHIs in DestBB. In such case, merging BB and DestBB would cause ISel to
@@ -1148,54 +1159,54 @@ static bool OptimizeNoopCopyExpression(CastInst *CI, const TargetLowering &TLI,
   return SinkCast(CI);
 }
 
-/// Try to combine CI into a call to the llvm.uadd.with.overflow intrinsic if
-/// possible.
-///
-/// Return true if any changes were made.
-static bool CombineUAddWithOverflow(CmpInst *CI) {
+static void replaceMathCmpWithIntrinsic(BinaryOperator *BO, CmpInst *Cmp,
+                                        Instruction *InsertPt,
+                                        Intrinsic::ID IID) {
+  IRBuilder<> Builder(InsertPt);
+  Value *MathOV = Builder.CreateBinaryIntrinsic(IID, BO->getOperand(0),
+                                                BO->getOperand(1));
+  Value *Math = Builder.CreateExtractValue(MathOV, 0, "math");
+  Value *OV = Builder.CreateExtractValue(MathOV, 1, "ov");
+  BO->replaceAllUsesWith(Math);
+  Cmp->replaceAllUsesWith(OV);
+  BO->eraseFromParent();
+  Cmp->eraseFromParent();
+}
+
+/// Try to combine the compare into a call to the llvm.uadd.with.overflow
+/// intrinsic. Return true if any changes were made.
+static bool combineToUAddWithOverflow(CmpInst *Cmp, const TargetLowering &TLI,
+                                      const DataLayout &DL) {
   Value *A, *B;
-  Instruction *AddI;
-  if (!match(CI,
-             m_UAddWithOverflow(m_Value(A), m_Value(B), m_Instruction(AddI))))
+  BinaryOperator *Add;
+  if (!match(Cmp, m_UAddWithOverflow(m_Value(A), m_Value(B), m_BinOp(Add))))
     return false;
 
-  Type *Ty = AddI->getType();
+  // Allow the transform as long as we have an integer type that is not
+  // obviously illegal and unsupported.
+  Type *Ty = Add->getType();
   if (!isa<IntegerType>(Ty))
     return false;
+  EVT CodegenVT = TLI.getValueType(DL, Ty);
+  if (!CodegenVT.isSimple() && TLI.isOperationExpand(ISD::UADDO, CodegenVT))
+    return false;
 
-  // We don't want to move around uses of condition values this late, so we we
+  // We don't want to move around uses of condition values this late, so we
   // check if it is legal to create the call to the intrinsic in the basic
-  // block containing the icmp:
-
-  if (AddI->getParent() != CI->getParent() && !AddI->hasOneUse())
+  // block containing the icmp.
+  if (Add->getParent() != Cmp->getParent() && !Add->hasOneUse())
     return false;
 
 #ifndef NDEBUG
   // Someday m_UAddWithOverflow may get smarter, but this is a safe assumption
   // for now:
-  if (AddI->hasOneUse())
-    assert(*AddI->user_begin() == CI && "expected!");
+  if (Add->hasOneUse())
+    assert(*Add->user_begin() == Cmp && "expected!");
 #endif
 
-  Module *M = CI->getModule();
-  Value *F = Intrinsic::getDeclaration(M, Intrinsic::uadd_with_overflow, Ty);
-
-  auto *InsertPt = AddI->hasOneUse() ? CI : AddI;
-
-  DebugLoc Loc = CI->getDebugLoc();
-  auto *UAddWithOverflow =
-      CallInst::Create(F, {A, B}, "uadd.overflow", InsertPt);
-  UAddWithOverflow->setDebugLoc(Loc);
-  auto *UAdd = ExtractValueInst::Create(UAddWithOverflow, 0, "uadd", InsertPt);
-  UAdd->setDebugLoc(Loc);
-  auto *Overflow =
-      ExtractValueInst::Create(UAddWithOverflow, 1, "overflow", InsertPt);
-  Overflow->setDebugLoc(Loc);
-
-  CI->replaceAllUsesWith(Overflow);
-  AddI->replaceAllUsesWith(UAdd);
-  CI->eraseFromParent();
-  AddI->eraseFromParent();
+  Instruction *InPt = Add->hasOneUse() ? cast<Instruction>(Cmp)
+                                       : cast<Instruction>(Add);
+  replaceMathCmpWithIntrinsic(Add, Cmp, InPt, Intrinsic::uadd_with_overflow);
   return true;
 }
 
@@ -1205,18 +1216,19 @@ static bool CombineUAddWithOverflow(CmpInst *CI) {
 /// lose; some adjustment may be wanted there.
 ///
 /// Return true if any changes are made.
-static bool SinkCmpExpression(CmpInst *CI, const TargetLowering *TLI) {
-  BasicBlock *DefBB = CI->getParent();
+static bool sinkCmpExpression(CmpInst *Cmp, const TargetLowering &TLI) {
+  if (TLI.hasMultipleConditionRegisters())
+    return false;
 
   // Avoid sinking soft-FP comparisons, since this can move them into a loop.
-  if (TLI && TLI->useSoftFloat() && isa<FCmpInst>(CI))
+  if (TLI.useSoftFloat() && isa<FCmpInst>(Cmp))
     return false;
 
   // Only insert a cmp in each block once.
   DenseMap<BasicBlock*, CmpInst*> InsertedCmps;
 
   bool MadeChange = false;
-  for (Value::user_iterator UI = CI->user_begin(), E = CI->user_end();
+  for (Value::user_iterator UI = Cmp->user_begin(), E = Cmp->user_end();
        UI != E; ) {
     Use &TheUse = UI.getUse();
     Instruction *User = cast<Instruction>(*UI);
@@ -1230,6 +1242,7 @@ static bool SinkCmpExpression(CmpInst *CI, const TargetLowering *TLI) {
 
     // Figure out which BB this cmp is used in.
     BasicBlock *UserBB = User->getParent();
+    BasicBlock *DefBB = Cmp->getParent();
 
     // If this user is in the same block as the cmp, don't change the cmp.
     if (UserBB == DefBB) continue;
@@ -1241,10 +1254,11 @@ static bool SinkCmpExpression(CmpInst *CI, const TargetLowering *TLI) {
       BasicBlock::iterator InsertPt = UserBB->getFirstInsertionPt();
       assert(InsertPt != UserBB->end());
       InsertedCmp =
-          CmpInst::Create(CI->getOpcode(), CI->getPredicate(),
-                          CI->getOperand(0), CI->getOperand(1), "", &*InsertPt);
+          CmpInst::Create(Cmp->getOpcode(), Cmp->getPredicate(),
+                          Cmp->getOperand(0), Cmp->getOperand(1), "",
+                          &*InsertPt);
       // Propagate the debug info.
-      InsertedCmp->setDebugLoc(CI->getDebugLoc());
+      InsertedCmp->setDebugLoc(Cmp->getDebugLoc());
     }
 
     // Replace a use of the cmp with a use of the new cmp.
@@ -1254,19 +1268,20 @@ static bool SinkCmpExpression(CmpInst *CI, const TargetLowering *TLI) {
   }
 
   // If we removed all uses, nuke the cmp.
-  if (CI->use_empty()) {
-    CI->eraseFromParent();
+  if (Cmp->use_empty()) {
+    Cmp->eraseFromParent();
     MadeChange = true;
   }
 
   return MadeChange;
 }
 
-static bool OptimizeCmpExpression(CmpInst *CI, const TargetLowering *TLI) {
-  if (SinkCmpExpression(CI, TLI))
+static bool optimizeCmpExpression(CmpInst *Cmp, const TargetLowering &TLI,
+                                  const DataLayout &DL) {
+  if (sinkCmpExpression(Cmp, TLI))
     return true;
 
-  if (CombineUAddWithOverflow(CI))
+  if (combineToUAddWithOverflow(Cmp, TLI, DL))
     return true;
 
   return false;
@@ -1704,9 +1719,23 @@ bool CodeGenPrepare::optimizeCallInst(CallInst *CI, bool &ModifiedDT) {
   if (II) {
     switch (II->getIntrinsicID()) {
     default: break;
+    case Intrinsic::experimental_widenable_condition: {
+      // Give up on future widening oppurtunties so that we can fold away dead
+      // paths and merge blocks before going into block-local instruction
+      // selection.   
+      if (II->use_empty()) {
+        II->eraseFromParent();
+        return true;
+      }
+      Constant *RetVal = ConstantInt::getTrue(II->getContext());
+      resetIteratorIfInvalidatedWhileCalling(BB, [&]() {
+        replaceAndRecursivelySimplify(CI, RetVal, TLInfo, nullptr);
+      });
+      return true;
+    }
     case Intrinsic::objectsize: {
       // Lower all uses of llvm.objectsize.*
-      ConstantInt *RetVal =
+      Value *RetVal =
           lowerObjectSizeCall(II, *DL, TLInfo, /*MustSucceed=*/true);
 
       resetIteratorIfInvalidatedWhileCalling(BB, [&]() {
@@ -1735,6 +1764,7 @@ bool CodeGenPrepare::optimizeCallInst(CallInst *CI, bool &ModifiedDT) {
       InsertedInsts.insert(ExtVal);
       return true;
     }
+
     case Intrinsic::launder_invariant_group:
     case Intrinsic::strip_invariant_group: {
       Value *ArgVal = II->getArgOperand(0);
@@ -1846,10 +1876,8 @@ bool CodeGenPrepare::dupRetToEnableTailCallOpts(BasicBlock *BB) {
   // return is the first instruction in the block.
   if (PN) {
     BasicBlock::iterator BI = BB->begin();
-    do { ++BI; } while (isa<DbgInfoIntrinsic>(BI));
-    if (&*BI == BCI)
-      // Also skip over the bitcast.
-      ++BI;
+    // Skip over debug and the bitcast.
+    do { ++BI; } while (isa<DbgInfoIntrinsic>(BI) || &*BI == BCI);
     if (&*BI != RetI)
       return false;
   } else {
@@ -5160,11 +5188,11 @@ bool CodeGenPrepare::splitLargeGEPOffsets() {
       }
 
       // Generate a new GEP to replace the current one.
-      IRBuilder<> Builder(GEP);
+      LLVMContext &Ctx = GEP->getContext();
       Type *IntPtrTy = DL->getIntPtrType(GEP->getType());
       Type *I8PtrTy =
-          Builder.getInt8PtrTy(GEP->getType()->getPointerAddressSpace());
-      Type *I8Ty = Builder.getInt8Ty();
+          Type::getInt8PtrTy(Ctx, GEP->getType()->getPointerAddressSpace());
+      Type *I8Ty = Type::getInt8Ty(Ctx);
 
       if (!NewBaseGEP) {
         // Create a new base if we don't have one yet.  Find the insertion
@@ -5200,6 +5228,7 @@ bool CodeGenPrepare::splitLargeGEPOffsets() {
         NewGEPBases.insert(NewBaseGEP);
       }
 
+      IRBuilder<> Builder(GEP);
       Value *NewGEP = NewBaseGEP;
       if (Offset == BaseOffset) {
         if (GEP->getType() != I8PtrTy)
@@ -5957,6 +5986,48 @@ bool CodeGenPrepare::optimizeShuffleVectorInst(ShuffleVectorInst *SVI) {
   return MadeChange;
 }
 
+bool CodeGenPrepare::tryToSinkFreeOperands(Instruction *I) {
+  // If the operands of I can be folded into a target instruction together with
+  // I, duplicate and sink them.
+  SmallVector<Use *, 4> OpsToSink;
+  if (!TLI || !TLI->shouldSinkOperands(I, OpsToSink))
+    return false;
+
+  // OpsToSink can contain multiple uses in a use chain (e.g.
+  // (%u1 with %u1 = shufflevector), (%u2 with %u2 = zext %u1)). The dominating
+  // uses must come first, which means they are sunk first, temporarily creating
+  // invalid IR. This will be fixed once their dominated users are sunk and
+  // updated.
+  BasicBlock *TargetBB = I->getParent();
+  bool Changed = false;
+  SmallVector<Use *, 4> ToReplace;
+  for (Use *U : OpsToSink) {
+    auto *UI = cast<Instruction>(U->get());
+    if (UI->getParent() == TargetBB || isa<PHINode>(UI))
+      continue;
+    ToReplace.push_back(U);
+  }
+
+  SmallPtrSet<Instruction *, 4> MaybeDead;
+  for (Use *U : ToReplace) {
+    auto *UI = cast<Instruction>(U->get());
+    Instruction *NI = UI->clone();
+    MaybeDead.insert(UI);
+    LLVM_DEBUG(dbgs() << "Sinking " << *UI << " to user " << *I << "\n");
+    NI->insertBefore(I);
+    InsertedInsts.insert(NI);
+    U->set(NI);
+    Changed = true;
+  }
+
+  // Remove instructions that are dead after sinking.
+  for (auto *I : MaybeDead)
+    if (!I->hasNUsesOrMore(1))
+      I->eraseFromParent();
+
+  return Changed;
+}
+
 bool CodeGenPrepare::optimizeSwitchInst(SwitchInst *SI) {
   if (!TLI || !DL)
     return false;
@@ -6700,8 +6771,8 @@ bool CodeGenPrepare::optimizeInst(Instruction *I, bool &ModifiedDT) {
   }
 
   if (CmpInst *CI = dyn_cast<CmpInst>(I))
-    if (!TLI || !TLI->hasMultipleConditionRegisters())
-      return OptimizeCmpExpression(CI, TLI);
+    if (TLI && optimizeCmpExpression(CI, *TLI, *DL))
+      return true;
 
   if (LoadInst *LI = dyn_cast<LoadInst>(I)) {
     LI->setMetadata(LLVMContext::MD_invariant_group, nullptr);
@@ -6771,6 +6842,9 @@ bool CodeGenPrepare::optimizeInst(Instruction *I, bool &ModifiedDT) {
     return false;
   }
 
+  if (tryToSinkFreeOperands(I))
+    return true;
+
   if (CallInst *CI = dyn_cast<CallInst>(I))
     return optimizeCallInst(CI, ModifiedDT);
 
diff --git a/lib/CodeGen/CriticalAntiDepBreaker.cpp b/lib/CodeGen/CriticalAntiDepBreaker.cpp
index 5a5960b16130b317a300d6c8149fb4c155aa4042..4144c243a3414d721965b1440e47d8ccd62f8f46 100644
--- a/lib/CodeGen/CriticalAntiDepBreaker.cpp
+++ b/lib/CodeGen/CriticalAntiDepBreaker.cpp
@@ -1,9 +1,8 @@
 //===- CriticalAntiDepBreaker.cpp - Anti-dep breaker ----------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/CriticalAntiDepBreaker.h b/lib/CodeGen/CriticalAntiDepBreaker.h
index 09c4423a2f057319d5097d18bdc4c3f62eac48f4..4e127ce525c8e04b56109c4842be258f17c39960 100644
--- a/lib/CodeGen/CriticalAntiDepBreaker.h
+++ b/lib/CodeGen/CriticalAntiDepBreaker.h
@@ -1,9 +1,8 @@
 //===- llvm/CodeGen/CriticalAntiDepBreaker.h - Anti-Dep Support -*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/DFAPacketizer.cpp b/lib/CodeGen/DFAPacketizer.cpp
index 68034afe98d535e999b4142fd7d943460627a8b5..b99be5d7a87c40b4f53dc33de03bdb5e9433b8f6 100644
--- a/lib/CodeGen/DFAPacketizer.cpp
+++ b/lib/CodeGen/DFAPacketizer.cpp
@@ -1,9 +1,8 @@
 //=- llvm/CodeGen/DFAPacketizer.cpp - DFA Packetizer for VLIW -*- C++ -*-=====//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 // This class implements a deterministic finite automaton (DFA) based
diff --git a/lib/CodeGen/DeadMachineInstructionElim.cpp b/lib/CodeGen/DeadMachineInstructionElim.cpp
index ff44c5660baddb766d4ed73b20d9e835443b551d..36c7de67f5d8299f390e3c3bc32824c72d58d4ae 100644
--- a/lib/CodeGen/DeadMachineInstructionElim.cpp
+++ b/lib/CodeGen/DeadMachineInstructionElim.cpp
@@ -1,9 +1,8 @@
 //===- DeadMachineInstructionElim.cpp - Remove dead machine instructions --===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/DetectDeadLanes.cpp b/lib/CodeGen/DetectDeadLanes.cpp
index c83db476a4ded1472b438d0fa205185c696fb381..fe78acf4d80a1e343d584232576375f97bdd4d80 100644
--- a/lib/CodeGen/DetectDeadLanes.cpp
+++ b/lib/CodeGen/DetectDeadLanes.cpp
@@ -1,9 +1,8 @@
 //===- DetectDeadLanes.cpp - SubRegister Lane Usage Analysis --*- C++ -*---===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/DwarfEHPrepare.cpp b/lib/CodeGen/DwarfEHPrepare.cpp
index 4586649d17f086d652eaccec76231f3114cfc3df..6b36a93c62c3d03aa0d58dc9b073c923301d5ece 100644
--- a/lib/CodeGen/DwarfEHPrepare.cpp
+++ b/lib/CodeGen/DwarfEHPrepare.cpp
@@ -1,9 +1,8 @@
 //===- DwarfEHPrepare - Prepare exception handling for code generation ----===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -46,7 +45,7 @@ namespace {
 
   class DwarfEHPrepare : public FunctionPass {
     // RewindFunction - _Unwind_Resume or the target equivalent.
-    Constant *RewindFunction = nullptr;
+    FunctionCallee RewindFunction = nullptr;
 
     DominatorTree *DT = nullptr;
     const TargetLowering *TLI = nullptr;
diff --git a/lib/CodeGen/EarlyIfConversion.cpp b/lib/CodeGen/EarlyIfConversion.cpp
index 098afd885f2fe00e0bb48f275f3f93a15e6fe5c2..0a83760befaa3942e0c25e3e55c434882dade4b5 100644
--- a/lib/CodeGen/EarlyIfConversion.cpp
+++ b/lib/CodeGen/EarlyIfConversion.cpp
@@ -1,9 +1,8 @@
 //===-- EarlyIfConversion.cpp - If-conversion on SSA form machine code ----===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -398,6 +397,13 @@ bool SSAIfConv::canConvertIf(MachineBasicBlock *MBB) {
     return false;
   }
 
+  // Make sure the analyzed branch is conditional; one of the successors
+  // could be a landing pad. (Empty landing pads can be generated on Windows.)
+  if (Cond.empty()) {
+    LLVM_DEBUG(dbgs() << "AnalyzeBranch found an unconditional branch.\n");
+    return false;
+  }
+
   // AnalyzeBranch doesn't set FBB on a fall-through branch.
   // Make sure it is always set.
   FBB = TBB == Succ0 ? Succ1 : Succ0;
diff --git a/lib/CodeGen/EdgeBundles.cpp b/lib/CodeGen/EdgeBundles.cpp
index 54c53eb1631222a0dc0439c6f47c29c7e8bd86c2..e073c84f309043cb37cac8545a0b912f163943c4 100644
--- a/lib/CodeGen/EdgeBundles.cpp
+++ b/lib/CodeGen/EdgeBundles.cpp
@@ -1,9 +1,8 @@
 //===-------- EdgeBundles.cpp - Bundles of CFG edges ----------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/ExecutionDomainFix.cpp b/lib/CodeGen/ExecutionDomainFix.cpp
index 458dcf2b0e26db67ac94e3c100d21c382ad99a0d..8fb725a82965b3097a6234056601541ad456e6fa 100644
--- a/lib/CodeGen/ExecutionDomainFix.cpp
+++ b/lib/CodeGen/ExecutionDomainFix.cpp
@@ -1,9 +1,8 @@
 //===- ExecutionDomainFix.cpp - Fix execution domain issues ----*- C++ -*--===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/CodeGen/ExpandISelPseudos.cpp b/lib/CodeGen/ExpandISelPseudos.cpp
index ec586a2caea3a490029d769eaf7f6698c39f02de..a27c2e3c39f6a1dfe00c1e7d5d12980d8d32309d 100644
--- a/lib/CodeGen/ExpandISelPseudos.cpp
+++ b/lib/CodeGen/ExpandISelPseudos.cpp
@@ -1,9 +1,8 @@
 //===-- llvm/CodeGen/ExpandISelPseudos.cpp ----------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/ExpandMemCmp.cpp b/lib/CodeGen/ExpandMemCmp.cpp
index d7562cbf1e90c59e23643db15185d1640f0d3776..0cd33805d9d264a9062718e0fbc8ba34f340c7dd 100644
--- a/lib/CodeGen/ExpandMemCmp.cpp
+++ b/lib/CodeGen/ExpandMemCmp.cpp
@@ -1,9 +1,8 @@
 //===--- ExpandMemCmp.cpp - Expand memcmp() to load/stores ----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -66,23 +65,18 @@ class MemCmpExpansion {
   // Represents the decomposition in blocks of the expansion. For example,
   // comparing 33 bytes on X86+sse can be done with 2x16-byte loads and
   // 1x1-byte load, which would be represented as [{16, 0}, {16, 16}, {32, 1}.
-  // TODO(courbet): Involve the target more in this computation. On X86, 7
-  // bytes can be done more efficiently with two overlaping 4-byte loads than
-  // covering the interval with [{4, 0},{2, 4},{1, 6}}.
   struct LoadEntry {
     LoadEntry(unsigned LoadSize, uint64_t Offset)
         : LoadSize(LoadSize), Offset(Offset) {
-      assert(Offset % LoadSize == 0 && "invalid load entry");
     }
 
-    uint64_t getGEPIndex() const { return Offset / LoadSize; }
-
     // The size of the load for this block, in bytes.
-    const unsigned LoadSize;
-    // The offset of this load WRT the base pointer, in bytes.
-    const uint64_t Offset;
+    unsigned LoadSize;
+    // The offset of this load from the base pointer, in bytes.
+    uint64_t Offset;
   };
-  SmallVector<LoadEntry, 8> LoadSequence;
+  using LoadEntryVector = SmallVector<LoadEntry, 8>;
+  LoadEntryVector LoadSequence;
 
   void createLoadCmpBlocks();
   void createResultBlock();
@@ -92,13 +86,23 @@ class MemCmpExpansion {
   void emitLoadCompareBlock(unsigned BlockIndex);
   void emitLoadCompareBlockMultipleLoads(unsigned BlockIndex,
                                          unsigned &LoadIndex);
-  void emitLoadCompareByteBlock(unsigned BlockIndex, unsigned GEPIndex);
+  void emitLoadCompareByteBlock(unsigned BlockIndex, unsigned OffsetBytes);
   void emitMemCmpResultBlock();
   Value *getMemCmpExpansionZeroCase();
   Value *getMemCmpEqZeroOneBlock();
   Value *getMemCmpOneBlock();
+  Value *getPtrToElementAtOffset(Value *Source, Type *LoadSizeType,
+                                 uint64_t OffsetBytes);
+
+  static LoadEntryVector
+  computeGreedyLoadSequence(uint64_t Size, llvm::ArrayRef<unsigned> LoadSizes,
+                            unsigned MaxNumLoads, unsigned &NumLoadsNonOneByte);
+  static LoadEntryVector
+  computeOverlappingLoadSequence(uint64_t Size, unsigned MaxLoadSize,
+                                 unsigned MaxNumLoads,
+                                 unsigned &NumLoadsNonOneByte);
 
- public:
+public:
   MemCmpExpansion(CallInst *CI, uint64_t Size,
                   const TargetTransformInfo::MemCmpExpansionOptions &Options,
                   unsigned MaxNumLoads, const bool IsUsedForZeroCmp,
@@ -110,6 +114,76 @@ class MemCmpExpansion {
   Value *getMemCmpExpansion();
 };
 
+MemCmpExpansion::LoadEntryVector MemCmpExpansion::computeGreedyLoadSequence(
+    uint64_t Size, llvm::ArrayRef<unsigned> LoadSizes,
+    const unsigned MaxNumLoads, unsigned &NumLoadsNonOneByte) {
+  NumLoadsNonOneByte = 0;
+  LoadEntryVector LoadSequence;
+  uint64_t Offset = 0;
+  while (Size && !LoadSizes.empty()) {
+    const unsigned LoadSize = LoadSizes.front();
+    const uint64_t NumLoadsForThisSize = Size / LoadSize;
+    if (LoadSequence.size() + NumLoadsForThisSize > MaxNumLoads) {
+      // Do not expand if the total number of loads is larger than what the
+      // target allows. Note that it's important that we exit before completing
+      // the expansion to avoid using a ton of memory to store the expansion for
+      // large sizes.
+      return {};
+    }
+    if (NumLoadsForThisSize > 0) {
+      for (uint64_t I = 0; I < NumLoadsForThisSize; ++I) {
+        LoadSequence.push_back({LoadSize, Offset});
+        Offset += LoadSize;
+      }
+      if (LoadSize > 1)
+        ++NumLoadsNonOneByte;
+      Size = Size % LoadSize;
+    }
+    LoadSizes = LoadSizes.drop_front();
+  }
+  return LoadSequence;
+}
+
+MemCmpExpansion::LoadEntryVector
+MemCmpExpansion::computeOverlappingLoadSequence(uint64_t Size,
+                                                const unsigned MaxLoadSize,
+                                                const unsigned MaxNumLoads,
+                                                unsigned &NumLoadsNonOneByte) {
+  // These are already handled by the greedy approach.
+  if (Size < 2 || MaxLoadSize < 2)
+    return {};
+
+  // We try to do as many non-overlapping loads as possible starting from the
+  // beginning.
+  const uint64_t NumNonOverlappingLoads = Size / MaxLoadSize;
+  assert(NumNonOverlappingLoads && "there must be at least one load");
+  // There remain 0 to (MaxLoadSize - 1) bytes to load, this will be done with
+  // an overlapping load.
+  Size = Size - NumNonOverlappingLoads * MaxLoadSize;
+  // Bail if we do not need an overloapping store, this is already handled by
+  // the greedy approach.
+  if (Size == 0)
+    return {};
+  // Bail if the number of loads (non-overlapping + potential overlapping one)
+  // is larger than the max allowed.
+  if ((NumNonOverlappingLoads + 1) > MaxNumLoads)
+    return {};
+
+  // Add non-overlapping loads.
+  LoadEntryVector LoadSequence;
+  uint64_t Offset = 0;
+  for (uint64_t I = 0; I < NumNonOverlappingLoads; ++I) {
+    LoadSequence.push_back({MaxLoadSize, Offset});
+    Offset += MaxLoadSize;
+  }
+
+  // Add the last overlapping load.
+  assert(Size > 0 && Size < MaxLoadSize && "broken invariant");
+  LoadSequence.push_back({MaxLoadSize, Offset - (MaxLoadSize - Size)});
+  NumLoadsNonOneByte = 1;
+  return LoadSequence;
+}
+
 // Initialize the basic block structure required for expansion of memcmp call
 // with given maximum load size and memcmp size parameter.
 // This structure includes:
@@ -133,38 +207,31 @@ MemCmpExpansion::MemCmpExpansion(
       Builder(CI) {
   assert(Size > 0 && "zero blocks");
   // Scale the max size down if the target can load more bytes than we need.
-  size_t LoadSizeIndex = 0;
-  while (LoadSizeIndex < Options.LoadSizes.size() &&
-         Options.LoadSizes[LoadSizeIndex] > Size) {
-    ++LoadSizeIndex;
+  llvm::ArrayRef<unsigned> LoadSizes(Options.LoadSizes);
+  while (!LoadSizes.empty() && LoadSizes.front() > Size) {
+    LoadSizes = LoadSizes.drop_front();
   }
-  this->MaxLoadSize = Options.LoadSizes[LoadSizeIndex];
+  assert(!LoadSizes.empty() && "cannot load Size bytes");
+  MaxLoadSize = LoadSizes.front();
   // Compute the decomposition.
-  uint64_t CurSize = Size;
-  uint64_t Offset = 0;
-  while (CurSize && LoadSizeIndex < Options.LoadSizes.size()) {
-    const unsigned LoadSize = Options.LoadSizes[LoadSizeIndex];
-    assert(LoadSize > 0 && "zero load size");
-    const uint64_t NumLoadsForThisSize = CurSize / LoadSize;
-    if (LoadSequence.size() + NumLoadsForThisSize > MaxNumLoads) {
-      // Do not expand if the total number of loads is larger than what the
-      // target allows. Note that it's important that we exit before completing
-      // the expansion to avoid using a ton of memory to store the expansion for
-      // large sizes.
-      LoadSequence.clear();
-      return;
-    }
-    if (NumLoadsForThisSize > 0) {
-      for (uint64_t I = 0; I < NumLoadsForThisSize; ++I) {
-        LoadSequence.push_back({LoadSize, Offset});
-        Offset += LoadSize;
-      }
-      if (LoadSize > 1) {
-        ++NumLoadsNonOneByte;
-      }
-      CurSize = CurSize % LoadSize;
+  unsigned GreedyNumLoadsNonOneByte = 0;
+  LoadSequence = computeGreedyLoadSequence(Size, LoadSizes, MaxNumLoads,
+                                           GreedyNumLoadsNonOneByte);
+  NumLoadsNonOneByte = GreedyNumLoadsNonOneByte;
+  assert(LoadSequence.size() <= MaxNumLoads && "broken invariant");
+  // If we allow overlapping loads and the load sequence is not already optimal,
+  // use overlapping loads.
+  if (Options.AllowOverlappingLoads &&
+      (LoadSequence.empty() || LoadSequence.size() > 2)) {
+    unsigned OverlappingNumLoadsNonOneByte = 0;
+    auto OverlappingLoads = computeOverlappingLoadSequence(
+        Size, MaxLoadSize, MaxNumLoads, OverlappingNumLoadsNonOneByte);
+    if (!OverlappingLoads.empty() &&
+        (LoadSequence.empty() ||
+         OverlappingLoads.size() < LoadSequence.size())) {
+      LoadSequence = OverlappingLoads;
+      NumLoadsNonOneByte = OverlappingNumLoadsNonOneByte;
     }
-    ++LoadSizeIndex;
   }
   assert(LoadSequence.size() <= MaxNumLoads && "broken invariant");
 }
@@ -189,30 +256,32 @@ void MemCmpExpansion::createResultBlock() {
                                    EndBlock->getParent(), EndBlock);
 }
 
+/// Return a pointer to an element of type `LoadSizeType` at offset
+/// `OffsetBytes`.
+Value *MemCmpExpansion::getPtrToElementAtOffset(Value *Source,
+                                                Type *LoadSizeType,
+                                                uint64_t OffsetBytes) {
+  if (OffsetBytes > 0) {
+    auto *ByteType = Type::getInt8Ty(CI->getContext());
+    Source = Builder.CreateGEP(
+        ByteType, Builder.CreateBitCast(Source, ByteType->getPointerTo()),
+        ConstantInt::get(ByteType, OffsetBytes));
+  }
+  return Builder.CreateBitCast(Source, LoadSizeType->getPointerTo());
+}
+
 // This function creates the IR instructions for loading and comparing 1 byte.
 // It loads 1 byte from each source of the memcmp parameters with the given
 // GEPIndex. It then subtracts the two loaded values and adds this result to the
 // final phi node for selecting the memcmp result.
 void MemCmpExpansion::emitLoadCompareByteBlock(unsigned BlockIndex,
-                                               unsigned GEPIndex) {
-  Value *Source1 = CI->getArgOperand(0);
-  Value *Source2 = CI->getArgOperand(1);
-
+                                               unsigned OffsetBytes) {
   Builder.SetInsertPoint(LoadCmpBlocks[BlockIndex]);
   Type *LoadSizeType = Type::getInt8Ty(CI->getContext());
-  // Cast source to LoadSizeType*.
-  if (Source1->getType() != LoadSizeType)
-    Source1 = Builder.CreateBitCast(Source1, LoadSizeType->getPointerTo());
-  if (Source2->getType() != LoadSizeType)
-    Source2 = Builder.CreateBitCast(Source2, LoadSizeType->getPointerTo());
-
-  // Get the base address using the GEPIndex.
-  if (GEPIndex != 0) {
-    Source1 = Builder.CreateGEP(LoadSizeType, Source1,
-                                ConstantInt::get(LoadSizeType, GEPIndex));
-    Source2 = Builder.CreateGEP(LoadSizeType, Source2,
-                                ConstantInt::get(LoadSizeType, GEPIndex));
-  }
+  Value *Source1 =
+      getPtrToElementAtOffset(CI->getArgOperand(0), LoadSizeType, OffsetBytes);
+  Value *Source2 =
+      getPtrToElementAtOffset(CI->getArgOperand(1), LoadSizeType, OffsetBytes);
 
   Value *LoadSrc1 = Builder.CreateLoad(LoadSizeType, Source1);
   Value *LoadSrc2 = Builder.CreateLoad(LoadSizeType, Source2);
@@ -270,24 +339,10 @@ Value *MemCmpExpansion::getCompareLoadPairs(unsigned BlockIndex,
     IntegerType *LoadSizeType =
         IntegerType::get(CI->getContext(), CurLoadEntry.LoadSize * 8);
 
-    Value *Source1 = CI->getArgOperand(0);
-    Value *Source2 = CI->getArgOperand(1);
-
-    // Cast source to LoadSizeType*.
-    if (Source1->getType() != LoadSizeType)
-      Source1 = Builder.CreateBitCast(Source1, LoadSizeType->getPointerTo());
-    if (Source2->getType() != LoadSizeType)
-      Source2 = Builder.CreateBitCast(Source2, LoadSizeType->getPointerTo());
-
-    // Get the base address using a GEP.
-    if (CurLoadEntry.Offset != 0) {
-      Source1 = Builder.CreateGEP(
-          LoadSizeType, Source1,
-          ConstantInt::get(LoadSizeType, CurLoadEntry.getGEPIndex()));
-      Source2 = Builder.CreateGEP(
-          LoadSizeType, Source2,
-          ConstantInt::get(LoadSizeType, CurLoadEntry.getGEPIndex()));
-    }
+    Value *Source1 = getPtrToElementAtOffset(CI->getArgOperand(0), LoadSizeType,
+                                             CurLoadEntry.Offset);
+    Value *Source2 = getPtrToElementAtOffset(CI->getArgOperand(1), LoadSizeType,
+                                             CurLoadEntry.Offset);
 
     // Get a constant or load a value for each source address.
     Value *LoadSrc1 = nullptr;
@@ -378,8 +433,7 @@ void MemCmpExpansion::emitLoadCompareBlock(unsigned BlockIndex) {
   const LoadEntry &CurLoadEntry = LoadSequence[BlockIndex];
 
   if (CurLoadEntry.LoadSize == 1) {
-    MemCmpExpansion::emitLoadCompareByteBlock(BlockIndex,
-                                              CurLoadEntry.getGEPIndex());
+    MemCmpExpansion::emitLoadCompareByteBlock(BlockIndex, CurLoadEntry.Offset);
     return;
   }
 
@@ -388,25 +442,12 @@ void MemCmpExpansion::emitLoadCompareBlock(unsigned BlockIndex) {
   Type *MaxLoadType = IntegerType::get(CI->getContext(), MaxLoadSize * 8);
   assert(CurLoadEntry.LoadSize <= MaxLoadSize && "Unexpected load type");
 
-  Value *Source1 = CI->getArgOperand(0);
-  Value *Source2 = CI->getArgOperand(1);
-
   Builder.SetInsertPoint(LoadCmpBlocks[BlockIndex]);
-  // Cast source to LoadSizeType*.
-  if (Source1->getType() != LoadSizeType)
-    Source1 = Builder.CreateBitCast(Source1, LoadSizeType->getPointerTo());
-  if (Source2->getType() != LoadSizeType)
-    Source2 = Builder.CreateBitCast(Source2, LoadSizeType->getPointerTo());
 
-  // Get the base address using a GEP.
-  if (CurLoadEntry.Offset != 0) {
-    Source1 = Builder.CreateGEP(
-        LoadSizeType, Source1,
-        ConstantInt::get(LoadSizeType, CurLoadEntry.getGEPIndex()));
-    Source2 = Builder.CreateGEP(
-        LoadSizeType, Source2,
-        ConstantInt::get(LoadSizeType, CurLoadEntry.getGEPIndex()));
-  }
+  Value *Source1 = getPtrToElementAtOffset(CI->getArgOperand(0), LoadSizeType,
+                                           CurLoadEntry.Offset);
+  Value *Source2 = getPtrToElementAtOffset(CI->getArgOperand(1), LoadSizeType,
+                                           CurLoadEntry.Offset);
 
   // Load LoadSizeType from the base address.
   Value *LoadSrc1 = Builder.CreateLoad(LoadSizeType, Source1);
@@ -694,7 +735,6 @@ static bool expandMemCmp(CallInst *CI, const TargetTransformInfo *TTI,
   if (SizeVal == 0) {
     return false;
   }
-
   // TTI call to check if target would like to expand memcmp. Also, get the
   // available load sizes.
   const bool IsUsedForZeroCmp = isOnlyUsedInZeroEqualityComparison(CI);
diff --git a/lib/CodeGen/ExpandPostRAPseudos.cpp b/lib/CodeGen/ExpandPostRAPseudos.cpp
index f2a2bcbb94b196b13552cc7b5705723312d213a2..0ab70aff7dc48f9d4300361b2e2ebd75e0a70379 100644
--- a/lib/CodeGen/ExpandPostRAPseudos.cpp
+++ b/lib/CodeGen/ExpandPostRAPseudos.cpp
@@ -1,9 +1,8 @@
 //===-- ExpandPostRAPseudos.cpp - Pseudo instruction expansion pass -------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/ExpandReductions.cpp b/lib/CodeGen/ExpandReductions.cpp
index 7552ba8cd85db71a72f165ad99367526b61f745d..f44d89156b6cce4da286def7ef398133848c2040 100644
--- a/lib/CodeGen/ExpandReductions.cpp
+++ b/lib/CodeGen/ExpandReductions.cpp
@@ -1,9 +1,8 @@
 //===--- ExpandReductions.cpp - Expand experimental reduction intrinsics --===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/FEntryInserter.cpp b/lib/CodeGen/FEntryInserter.cpp
index 4ddf9f92836c0cb9946a7f47923cb34e6c7f7765..a122f490884eee6489f5293dd830067138e44d7a 100644
--- a/lib/CodeGen/FEntryInserter.cpp
+++ b/lib/CodeGen/FEntryInserter.cpp
@@ -1,9 +1,8 @@
 //===-- FEntryInsertion.cpp - Patchable prologues for LLVM -------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/FaultMaps.cpp b/lib/CodeGen/FaultMaps.cpp
index 361558a0e562d36a90bc4b6398013d430874a386..600f72d320eb590818e1a4a9f5fa210eb0bdaf86 100644
--- a/lib/CodeGen/FaultMaps.cpp
+++ b/lib/CodeGen/FaultMaps.cpp
@@ -1,9 +1,8 @@
 //===- FaultMaps.cpp ------------------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/CodeGen/FuncletLayout.cpp b/lib/CodeGen/FuncletLayout.cpp
index 581cd423f2d4e99c0b3804a291cdb9bfea42933e..75f6d0b8f0bfe1b8fda2666caedf3bb59c9bdc35 100644
--- a/lib/CodeGen/FuncletLayout.cpp
+++ b/lib/CodeGen/FuncletLayout.cpp
@@ -1,9 +1,8 @@
 //===-- FuncletLayout.cpp - Contiguously lay out funclets -----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/GCMetadata.cpp b/lib/CodeGen/GCMetadata.cpp
index 1c80556dfef57943142f6cfe8c59b3af1649a4c8..9c53550eaa9d4827503c439a5b3d81553474fefc 100644
--- a/lib/CodeGen/GCMetadata.cpp
+++ b/lib/CodeGen/GCMetadata.cpp
@@ -1,9 +1,8 @@
 //===-- GCMetadata.cpp - Garbage collector metadata -----------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/GCMetadataPrinter.cpp b/lib/CodeGen/GCMetadataPrinter.cpp
index bc7beb6f6c2d3ee9e4a273739726adc5342dffd3..500dba9aea37e3dcf0472a04cf1012c4cca7e47e 100644
--- a/lib/CodeGen/GCMetadataPrinter.cpp
+++ b/lib/CodeGen/GCMetadataPrinter.cpp
@@ -1,9 +1,8 @@
 //===- GCMetadataPrinter.cpp - Garbage collection infrastructure ----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/GCRootLowering.cpp b/lib/CodeGen/GCRootLowering.cpp
index e8ccd84b0b93e4ebe0c8798e49500c78d4916003..90571d090bfbc170bf590cdb726dd8a6ae553356 100644
--- a/lib/CodeGen/GCRootLowering.cpp
+++ b/lib/CodeGen/GCRootLowering.cpp
@@ -1,9 +1,8 @@
 //===-- GCRootLowering.cpp - Garbage collection infrastructure ------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -214,7 +213,7 @@ bool LowerIntrinsics::DoLowering(Function &F, GCStrategy &S) {
       }
       case Intrinsic::gcread: {
         // Replace a read barrier with a simple load.
-        Value *Ld = new LoadInst(CI->getArgOperand(1), "", CI);
+        Value *Ld = new LoadInst(CI->getType(), CI->getArgOperand(1), "", CI);
         Ld->takeName(CI);
         CI->replaceAllUsesWith(Ld);
         CI->eraseFromParent();
diff --git a/lib/CodeGen/GCStrategy.cpp b/lib/CodeGen/GCStrategy.cpp
index 6be4c16c6301e750f957e02b4fd1cc31138aa9a4..43d06b0f82e9491d8f6f760eefac352271c72c85 100644
--- a/lib/CodeGen/GCStrategy.cpp
+++ b/lib/CodeGen/GCStrategy.cpp
@@ -1,9 +1,8 @@
 //===- GCStrategy.cpp - Garbage Collector Description ---------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/GlobalISel/CMakeLists.txt b/lib/CodeGen/GlobalISel/CMakeLists.txt
index 5f13692bbee1f454a7a442eecf377bb29826191f..da2fd3b239a2f2da68c79fc4ba5de1b808f4fe68 100644
--- a/lib/CodeGen/GlobalISel/CMakeLists.txt
+++ b/lib/CodeGen/GlobalISel/CMakeLists.txt
@@ -1,4 +1,6 @@
 add_llvm_library(LLVMGlobalISel
+        CSEInfo.cpp
+        CSEMIRBuilder.cpp
         CallLowering.cpp
         GlobalISel.cpp
         Combiner.cpp
diff --git a/lib/CodeGen/GlobalISel/CSEInfo.cpp b/lib/CodeGen/GlobalISel/CSEInfo.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..aaf09c8c938d3dbf729851502dc2dd7625027d61
--- /dev/null
+++ b/lib/CodeGen/GlobalISel/CSEInfo.cpp
@@ -0,0 +1,369 @@
+//===- CSEInfo.cpp ------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+//
+//===----------------------------------------------------------------------===//
+#include "llvm/CodeGen/GlobalISel/CSEInfo.h"
+#include "llvm/CodeGen/MachineRegisterInfo.h"
+
+#define DEBUG_TYPE "cseinfo"
+
+using namespace llvm;
+char llvm::GISelCSEAnalysisWrapperPass::ID = 0;
+INITIALIZE_PASS_BEGIN(GISelCSEAnalysisWrapperPass, DEBUG_TYPE,
+                      "Analysis containing CSE Info", false, true)
+INITIALIZE_PASS_END(GISelCSEAnalysisWrapperPass, DEBUG_TYPE,
+                    "Analysis containing CSE Info", false, true)
+
+/// -------- UniqueMachineInstr -------------//
+
+void UniqueMachineInstr::Profile(FoldingSetNodeID &ID) {
+  GISelInstProfileBuilder(ID, MI->getMF()->getRegInfo()).addNodeID(MI);
+}
+/// -----------------------------------------
+
+/// --------- CSEConfig ---------- ///
+bool CSEConfig::shouldCSEOpc(unsigned Opc) {
+  switch (Opc) {
+  default:
+    break;
+  case TargetOpcode::G_ADD:
+  case TargetOpcode::G_AND:
+  case TargetOpcode::G_ASHR:
+  case TargetOpcode::G_LSHR:
+  case TargetOpcode::G_MUL:
+  case TargetOpcode::G_OR:
+  case TargetOpcode::G_SHL:
+  case TargetOpcode::G_SUB:
+  case TargetOpcode::G_XOR:
+  case TargetOpcode::G_UDIV:
+  case TargetOpcode::G_SDIV:
+  case TargetOpcode::G_UREM:
+  case TargetOpcode::G_SREM:
+  case TargetOpcode::G_CONSTANT:
+  case TargetOpcode::G_FCONSTANT:
+  case TargetOpcode::G_ZEXT:
+  case TargetOpcode::G_SEXT:
+  case TargetOpcode::G_ANYEXT:
+  case TargetOpcode::G_UNMERGE_VALUES:
+  case TargetOpcode::G_TRUNC:
+    return true;
+  }
+  return false;
+}
+
+bool CSEConfigConstantOnly::shouldCSEOpc(unsigned Opc) {
+  return Opc == TargetOpcode::G_CONSTANT;
+}
+/// -----------------------------------------
+
+/// -------- GISelCSEInfo -------------//
+void GISelCSEInfo::setMF(MachineFunction &MF) {
+  this->MF = &MF;
+  this->MRI = &MF.getRegInfo();
+}
+
+GISelCSEInfo::~GISelCSEInfo() {}
+
+bool GISelCSEInfo::isUniqueMachineInstValid(
+    const UniqueMachineInstr &UMI) const {
+  // Should we check here and assert that the instruction has been fully
+  // constructed?
+  // FIXME: Any other checks required to be done here? Remove this method if
+  // none.
+  return true;
+}
+
+void GISelCSEInfo::invalidateUniqueMachineInstr(UniqueMachineInstr *UMI) {
+  bool Removed = CSEMap.RemoveNode(UMI);
+  (void)Removed;
+  assert(Removed && "Invalidation called on invalid UMI");
+  // FIXME: Should UMI be deallocated/destroyed?
+}
+
+UniqueMachineInstr *GISelCSEInfo::getNodeIfExists(FoldingSetNodeID &ID,
+                                                  MachineBasicBlock *MBB,
+                                                  void *&InsertPos) {
+  auto *Node = CSEMap.FindNodeOrInsertPos(ID, InsertPos);
+  if (Node) {
+    if (!isUniqueMachineInstValid(*Node)) {
+      invalidateUniqueMachineInstr(Node);
+      return nullptr;
+    }
+
+    if (Node->MI->getParent() != MBB)
+      return nullptr;
+  }
+  return Node;
+}
+
+void GISelCSEInfo::insertNode(UniqueMachineInstr *UMI, void *InsertPos) {
+  handleRecordedInsts();
+  assert(UMI);
+  UniqueMachineInstr *MaybeNewNode = UMI;
+  if (InsertPos)
+    CSEMap.InsertNode(UMI, InsertPos);
+  else
+    MaybeNewNode = CSEMap.GetOrInsertNode(UMI);
+  if (MaybeNewNode != UMI) {
+    // A similar node exists in the folding set. Let's ignore this one.
+    return;
+  }
+  assert(InstrMapping.count(UMI->MI) == 0 &&
+         "This instruction should not be in the map");
+  InstrMapping[UMI->MI] = MaybeNewNode;
+}
+
+UniqueMachineInstr *GISelCSEInfo::getUniqueInstrForMI(const MachineInstr *MI) {
+  assert(shouldCSE(MI->getOpcode()) && "Trying to CSE an unsupported Node");
+  auto *Node = new (UniqueInstrAllocator) UniqueMachineInstr(MI);
+  return Node;
+}
+
+void GISelCSEInfo::insertInstr(MachineInstr *MI, void *InsertPos) {
+  assert(MI);
+  // If it exists in temporary insts, remove it.
+  TemporaryInsts.remove(MI);
+  auto *Node = getUniqueInstrForMI(MI);
+  insertNode(Node, InsertPos);
+}
+
+MachineInstr *GISelCSEInfo::getMachineInstrIfExists(FoldingSetNodeID &ID,
+                                                    MachineBasicBlock *MBB,
+                                                    void *&InsertPos) {
+  handleRecordedInsts();
+  if (auto *Inst = getNodeIfExists(ID, MBB, InsertPos)) {
+    LLVM_DEBUG(dbgs() << "CSEInfo::Found Instr " << *Inst->MI;);
+    return const_cast<MachineInstr *>(Inst->MI);
+  }
+  return nullptr;
+}
+
+void GISelCSEInfo::countOpcodeHit(unsigned Opc) {
+#ifndef NDEBUG
+  if (OpcodeHitTable.count(Opc))
+    OpcodeHitTable[Opc] += 1;
+  else
+    OpcodeHitTable[Opc] = 1;
+#endif
+  // Else do nothing.
+}
+
+void GISelCSEInfo::recordNewInstruction(MachineInstr *MI) {
+  if (shouldCSE(MI->getOpcode())) {
+    TemporaryInsts.insert(MI);
+    LLVM_DEBUG(dbgs() << "CSEInfo::Recording new MI " << *MI);
+  }
+}
+
+void GISelCSEInfo::handleRecordedInst(MachineInstr *MI) {
+  assert(shouldCSE(MI->getOpcode()) && "Invalid instruction for CSE");
+  auto *UMI = InstrMapping.lookup(MI);
+  LLVM_DEBUG(dbgs() << "CSEInfo::Handling recorded MI " << *MI);
+  if (UMI) {
+    // Invalidate this MI.
+    invalidateUniqueMachineInstr(UMI);
+    InstrMapping.erase(MI);
+  }
+  /// Now insert the new instruction.
+  if (UMI) {
+    /// We'll reuse the same UniqueMachineInstr to avoid the new
+    /// allocation.
+    *UMI = UniqueMachineInstr(MI);
+    insertNode(UMI, nullptr);
+  } else {
+    /// This is a new instruction. Allocate a new UniqueMachineInstr and
+    /// Insert.
+    insertInstr(MI);
+  }
+}
+
+void GISelCSEInfo::handleRemoveInst(MachineInstr *MI) {
+  if (auto *UMI = InstrMapping.lookup(MI)) {
+    invalidateUniqueMachineInstr(UMI);
+    InstrMapping.erase(MI);
+  }
+  TemporaryInsts.remove(MI);
+}
+
+void GISelCSEInfo::handleRecordedInsts() {
+  while (!TemporaryInsts.empty()) {
+    auto *MI = TemporaryInsts.pop_back_val();
+    handleRecordedInst(MI);
+  }
+}
+
+bool GISelCSEInfo::shouldCSE(unsigned Opc) const {
+  // Only GISel opcodes are CSEable
+  if (!isPreISelGenericOpcode(Opc))
+    return false;
+  assert(CSEOpt.get() && "CSEConfig not set");
+  return CSEOpt->shouldCSEOpc(Opc);
+}
+
+void GISelCSEInfo::erasingInstr(MachineInstr &MI) { handleRemoveInst(&MI); }
+void GISelCSEInfo::createdInstr(MachineInstr &MI) { recordNewInstruction(&MI); }
+void GISelCSEInfo::changingInstr(MachineInstr &MI) {
+  // For now, perform erase, followed by insert.
+  erasingInstr(MI);
+  createdInstr(MI);
+}
+void GISelCSEInfo::changedInstr(MachineInstr &MI) { changingInstr(MI); }
+
+void GISelCSEInfo::analyze(MachineFunction &MF) {
+  setMF(MF);
+  for (auto &MBB : MF) {
+    if (MBB.empty())
+      continue;
+    for (MachineInstr &MI : MBB) {
+      if (!shouldCSE(MI.getOpcode()))
+        continue;
+      LLVM_DEBUG(dbgs() << "CSEInfo::Add MI: " << MI);
+      insertInstr(&MI);
+    }
+  }
+}
+
+void GISelCSEInfo::releaseMemory() {
+  print();
+  CSEMap.clear();
+  InstrMapping.clear();
+  UniqueInstrAllocator.Reset();
+  TemporaryInsts.clear();
+  CSEOpt.reset();
+  MRI = nullptr;
+  MF = nullptr;
+#ifndef NDEBUG
+  OpcodeHitTable.clear();
+#endif
+}
+
+void GISelCSEInfo::print() {
+  LLVM_DEBUG(for (auto &It
+                  : OpcodeHitTable) {
+    dbgs() << "CSEInfo::CSE Hit for Opc " << It.first << " : " << It.second
+           << "\n";
+  };);
+}
+/// -----------------------------------------
+// ---- Profiling methods for FoldingSetNode --- //
+const GISelInstProfileBuilder &
+GISelInstProfileBuilder::addNodeID(const MachineInstr *MI) const {
+  addNodeIDMBB(MI->getParent());
+  addNodeIDOpcode(MI->getOpcode());
+  for (auto &Op : MI->operands())
+    addNodeIDMachineOperand(Op);
+  addNodeIDFlag(MI->getFlags());
+  return *this;
+}
+
+const GISelInstProfileBuilder &
+GISelInstProfileBuilder::addNodeIDOpcode(unsigned Opc) const {
+  ID.AddInteger(Opc);
+  return *this;
+}
+
+const GISelInstProfileBuilder &
+GISelInstProfileBuilder::addNodeIDRegType(const LLT &Ty) const {
+  uint64_t Val = Ty.getUniqueRAWLLTData();
+  ID.AddInteger(Val);
+  return *this;
+}
+
+const GISelInstProfileBuilder &
+GISelInstProfileBuilder::addNodeIDRegType(const TargetRegisterClass *RC) const {
+  ID.AddPointer(RC);
+  return *this;
+}
+
+const GISelInstProfileBuilder &
+GISelInstProfileBuilder::addNodeIDRegType(const RegisterBank *RB) const {
+  ID.AddPointer(RB);
+  return *this;
+}
+
+const GISelInstProfileBuilder &
+GISelInstProfileBuilder::addNodeIDImmediate(int64_t Imm) const {
+  ID.AddInteger(Imm);
+  return *this;
+}
+
+const GISelInstProfileBuilder &
+GISelInstProfileBuilder::addNodeIDRegNum(unsigned Reg) const {
+  ID.AddInteger(Reg);
+  return *this;
+}
+
+const GISelInstProfileBuilder &
+GISelInstProfileBuilder::addNodeIDRegType(const unsigned Reg) const {
+  addNodeIDMachineOperand(MachineOperand::CreateReg(Reg, false));
+  return *this;
+}
+
+const GISelInstProfileBuilder &
+GISelInstProfileBuilder::addNodeIDMBB(const MachineBasicBlock *MBB) const {
+  ID.AddPointer(MBB);
+  return *this;
+}
+
+const GISelInstProfileBuilder &
+GISelInstProfileBuilder::addNodeIDFlag(unsigned Flag) const {
+  if (Flag)
+    ID.AddInteger(Flag);
+  return *this;
+}
+
+const GISelInstProfileBuilder &GISelInstProfileBuilder::addNodeIDMachineOperand(
+    const MachineOperand &MO) const {
+  if (MO.isReg()) {
+    unsigned Reg = MO.getReg();
+    if (!MO.isDef())
+      addNodeIDRegNum(Reg);
+    LLT Ty = MRI.getType(Reg);
+    if (Ty.isValid())
+      addNodeIDRegType(Ty);
+    auto *RB = MRI.getRegBankOrNull(Reg);
+    if (RB)
+      addNodeIDRegType(RB);
+    auto *RC = MRI.getRegClassOrNull(Reg);
+    if (RC)
+      addNodeIDRegType(RC);
+    assert(!MO.isImplicit() && "Unhandled case");
+  } else if (MO.isImm())
+    ID.AddInteger(MO.getImm());
+  else if (MO.isCImm())
+    ID.AddPointer(MO.getCImm());
+  else if (MO.isFPImm())
+    ID.AddPointer(MO.getFPImm());
+  else if (MO.isPredicate())
+    ID.AddInteger(MO.getPredicate());
+  else
+    llvm_unreachable("Unhandled operand type");
+  // Handle other types
+  return *this;
+}
+
+GISelCSEInfo &GISelCSEAnalysisWrapper::get(std::unique_ptr<CSEConfig> CSEOpt,
+                                           bool Recompute) {
+  if (!AlreadyComputed || Recompute) {
+    Info.setCSEConfig(std::move(CSEOpt));
+    Info.analyze(*MF);
+    AlreadyComputed = true;
+  }
+  return Info;
+}
+void GISelCSEAnalysisWrapperPass::getAnalysisUsage(AnalysisUsage &AU) const {
+  AU.setPreservesAll();
+  MachineFunctionPass::getAnalysisUsage(AU);
+}
+
+bool GISelCSEAnalysisWrapperPass::runOnMachineFunction(MachineFunction &MF) {
+  releaseMemory();
+  Wrapper.setMF(MF);
+  return false;
+}
diff --git a/lib/CodeGen/GlobalISel/CSEMIRBuilder.cpp b/lib/CodeGen/GlobalISel/CSEMIRBuilder.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..461bc6038c2c44c1064f9c03c0fde626cb95e904
--- /dev/null
+++ b/lib/CodeGen/GlobalISel/CSEMIRBuilder.cpp
@@ -0,0 +1,244 @@
+//===-- llvm/CodeGen/GlobalISel/CSEMIRBuilder.cpp - MIBuilder--*- C++ -*-==//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+/// \file
+/// This file implements the CSEMIRBuilder class which CSEs as it builds
+/// instructions.
+//===----------------------------------------------------------------------===//
+//
+
+#include "llvm/CodeGen/GlobalISel/CSEMIRBuilder.h"
+#include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h"
+
+using namespace llvm;
+
+bool CSEMIRBuilder::dominates(MachineBasicBlock::const_iterator A,
+                              MachineBasicBlock::const_iterator B) const {
+  auto MBBEnd = getMBB().end();
+  if (B == MBBEnd)
+    return true;
+  assert(A->getParent() == B->getParent() &&
+         "Iterators should be in same block");
+  const MachineBasicBlock *BBA = A->getParent();
+  MachineBasicBlock::const_iterator I = BBA->begin();
+  for (; &*I != A && &*I != B; ++I)
+    ;
+  return &*I == A;
+}
+
+MachineInstrBuilder
+CSEMIRBuilder::getDominatingInstrForID(FoldingSetNodeID &ID,
+                                       void *&NodeInsertPos) {
+  GISelCSEInfo *CSEInfo = getCSEInfo();
+  assert(CSEInfo && "Can't get here without setting CSEInfo");
+  MachineBasicBlock *CurMBB = &getMBB();
+  MachineInstr *MI =
+      CSEInfo->getMachineInstrIfExists(ID, CurMBB, NodeInsertPos);
+  if (MI) {
+    CSEInfo->countOpcodeHit(MI->getOpcode());
+    auto CurrPos = getInsertPt();
+    if (!dominates(MI, CurrPos))
+      CurMBB->splice(CurrPos, CurMBB, MI);
+    return MachineInstrBuilder(getMF(), MI);
+  }
+  return MachineInstrBuilder();
+}
+
+bool CSEMIRBuilder::canPerformCSEForOpc(unsigned Opc) const {
+  const GISelCSEInfo *CSEInfo = getCSEInfo();
+  if (!CSEInfo || !CSEInfo->shouldCSE(Opc))
+    return false;
+  return true;
+}
+
+void CSEMIRBuilder::profileDstOp(const DstOp &Op,
+                                 GISelInstProfileBuilder &B) const {
+  switch (Op.getDstOpKind()) {
+  case DstOp::DstType::Ty_RC:
+    B.addNodeIDRegType(Op.getRegClass());
+    break;
+  default:
+    B.addNodeIDRegType(Op.getLLTTy(*getMRI()));
+    break;
+  }
+}
+
+void CSEMIRBuilder::profileSrcOp(const SrcOp &Op,
+                                 GISelInstProfileBuilder &B) const {
+  switch (Op.getSrcOpKind()) {
+  case SrcOp::SrcType::Ty_Predicate:
+    B.addNodeIDImmediate(static_cast<int64_t>(Op.getPredicate()));
+    break;
+  default:
+    B.addNodeIDRegType(Op.getReg());
+    break;
+  }
+}
+
+void CSEMIRBuilder::profileMBBOpcode(GISelInstProfileBuilder &B,
+                                     unsigned Opc) const {
+  // First add the MBB (Local CSE).
+  B.addNodeIDMBB(&getMBB());
+  // Then add the opcode.
+  B.addNodeIDOpcode(Opc);
+}
+
+void CSEMIRBuilder::profileEverything(unsigned Opc, ArrayRef<DstOp> DstOps,
+                                      ArrayRef<SrcOp> SrcOps,
+                                      Optional<unsigned> Flags,
+                                      GISelInstProfileBuilder &B) const {
+
+  profileMBBOpcode(B, Opc);
+  // Then add the DstOps.
+  profileDstOps(DstOps, B);
+  // Then add the SrcOps.
+  profileSrcOps(SrcOps, B);
+  // Add Flags if passed in.
+  if (Flags)
+    B.addNodeIDFlag(*Flags);
+}
+
+MachineInstrBuilder CSEMIRBuilder::memoizeMI(MachineInstrBuilder MIB,
+                                             void *NodeInsertPos) {
+  assert(canPerformCSEForOpc(MIB->getOpcode()) &&
+         "Attempting to CSE illegal op");
+  MachineInstr *MIBInstr = MIB;
+  getCSEInfo()->insertInstr(MIBInstr, NodeInsertPos);
+  return MIB;
+}
+
+bool CSEMIRBuilder::checkCopyToDefsPossible(ArrayRef<DstOp> DstOps) {
+  if (DstOps.size() == 1)
+    return true; // always possible to emit copy to just 1 vreg.
+
+  return std::all_of(DstOps.begin(), DstOps.end(), [](const DstOp &Op) {
+    DstOp::DstType DT = Op.getDstOpKind();
+    return DT == DstOp::DstType::Ty_LLT || DT == DstOp::DstType::Ty_RC;
+  });
+}
+
+MachineInstrBuilder
+CSEMIRBuilder::generateCopiesIfRequired(ArrayRef<DstOp> DstOps,
+                                        MachineInstrBuilder &MIB) {
+  assert(checkCopyToDefsPossible(DstOps) &&
+         "Impossible return a single MIB with copies to multiple defs");
+  if (DstOps.size() == 1) {
+    const DstOp &Op = DstOps[0];
+    if (Op.getDstOpKind() == DstOp::DstType::Ty_Reg)
+      return buildCopy(Op.getReg(), MIB->getOperand(0).getReg());
+  }
+  return MIB;
+}
+
+MachineInstrBuilder CSEMIRBuilder::buildInstr(unsigned Opc,
+                                              ArrayRef<DstOp> DstOps,
+                                              ArrayRef<SrcOp> SrcOps,
+                                              Optional<unsigned> Flag) {
+  switch (Opc) {
+  default:
+    break;
+  case TargetOpcode::G_ADD:
+  case TargetOpcode::G_AND:
+  case TargetOpcode::G_ASHR:
+  case TargetOpcode::G_LSHR:
+  case TargetOpcode::G_MUL:
+  case TargetOpcode::G_OR:
+  case TargetOpcode::G_SHL:
+  case TargetOpcode::G_SUB:
+  case TargetOpcode::G_XOR:
+  case TargetOpcode::G_UDIV:
+  case TargetOpcode::G_SDIV:
+  case TargetOpcode::G_UREM:
+  case TargetOpcode::G_SREM: {
+    // Try to constant fold these.
+    assert(SrcOps.size() == 2 && "Invalid sources");
+    assert(DstOps.size() == 1 && "Invalid dsts");
+    if (Optional<APInt> Cst = ConstantFoldBinOp(Opc, SrcOps[0].getReg(),
+                                                SrcOps[1].getReg(), *getMRI()))
+      return buildConstant(DstOps[0], Cst->getSExtValue());
+    break;
+  }
+  }
+  bool CanCopy = checkCopyToDefsPossible(DstOps);
+  if (!canPerformCSEForOpc(Opc))
+    return MachineIRBuilder::buildInstr(Opc, DstOps, SrcOps, Flag);
+  // If we can CSE this instruction, but involves generating copies to multiple
+  // regs, give up. This frequently happens to UNMERGEs.
+  if (!CanCopy) {
+    auto MIB = MachineIRBuilder::buildInstr(Opc, DstOps, SrcOps, Flag);
+    // CSEInfo would have tracked this instruction. Remove it from the temporary
+    // insts.
+    getCSEInfo()->handleRemoveInst(&*MIB);
+    return MIB;
+  }
+  FoldingSetNodeID ID;
+  GISelInstProfileBuilder ProfBuilder(ID, *getMRI());
+  void *InsertPos = nullptr;
+  profileEverything(Opc, DstOps, SrcOps, Flag, ProfBuilder);
+  MachineInstrBuilder MIB = getDominatingInstrForID(ID, InsertPos);
+  if (MIB) {
+    // Handle generating copies here.
+    return generateCopiesIfRequired(DstOps, MIB);
+  }
+  // This instruction does not exist in the CSEInfo. Build it and CSE it.
+  MachineInstrBuilder NewMIB =
+      MachineIRBuilder::buildInstr(Opc, DstOps, SrcOps, Flag);
+  return memoizeMI(NewMIB, InsertPos);
+}
+
+MachineInstrBuilder CSEMIRBuilder::buildConstant(const DstOp &Res,
+                                                 const ConstantInt &Val) {
+  constexpr unsigned Opc = TargetOpcode::G_CONSTANT;
+  if (!canPerformCSEForOpc(Opc))
+    return MachineIRBuilder::buildConstant(Res, Val);
+
+  // For vectors, CSE the element only for now.
+  LLT Ty = Res.getLLTTy(*getMRI());
+  if (Ty.isVector())
+    return buildSplatVector(Res, buildConstant(Ty.getElementType(), Val));
+
+  FoldingSetNodeID ID;
+  GISelInstProfileBuilder ProfBuilder(ID, *getMRI());
+  void *InsertPos = nullptr;
+  profileMBBOpcode(ProfBuilder, Opc);
+  profileDstOp(Res, ProfBuilder);
+  ProfBuilder.addNodeIDMachineOperand(MachineOperand::CreateCImm(&Val));
+  MachineInstrBuilder MIB = getDominatingInstrForID(ID, InsertPos);
+  if (MIB) {
+    // Handle generating copies here.
+    return generateCopiesIfRequired({Res}, MIB);
+  }
+
+  MachineInstrBuilder NewMIB = MachineIRBuilder::buildConstant(Res, Val);
+  return memoizeMI(NewMIB, InsertPos);
+}
+
+MachineInstrBuilder CSEMIRBuilder::buildFConstant(const DstOp &Res,
+                                                  const ConstantFP &Val) {
+  constexpr unsigned Opc = TargetOpcode::G_FCONSTANT;
+  if (!canPerformCSEForOpc(Opc))
+    return MachineIRBuilder::buildFConstant(Res, Val);
+
+  // For vectors, CSE the element only for now.
+  LLT Ty = Res.getLLTTy(*getMRI());
+  if (Ty.isVector())
+    return buildSplatVector(Res, buildFConstant(Ty.getElementType(), Val));
+
+  FoldingSetNodeID ID;
+  GISelInstProfileBuilder ProfBuilder(ID, *getMRI());
+  void *InsertPos = nullptr;
+  profileMBBOpcode(ProfBuilder, Opc);
+  profileDstOp(Res, ProfBuilder);
+  ProfBuilder.addNodeIDMachineOperand(MachineOperand::CreateFPImm(&Val));
+  MachineInstrBuilder MIB = getDominatingInstrForID(ID, InsertPos);
+  if (MIB) {
+    // Handle generating copies here.
+    return generateCopiesIfRequired({Res}, MIB);
+  }
+  MachineInstrBuilder NewMIB = MachineIRBuilder::buildFConstant(Res, Val);
+  return memoizeMI(NewMIB, InsertPos);
+}
diff --git a/lib/CodeGen/GlobalISel/CallLowering.cpp b/lib/CodeGen/GlobalISel/CallLowering.cpp
index da972eae756c9816169f05158d0c65cf229172c2..a7c2a2ee4ee1881e5b306a443089f9a58e3bb1ea 100644
--- a/lib/CodeGen/GlobalISel/CallLowering.cpp
+++ b/lib/CodeGen/GlobalISel/CallLowering.cpp
@@ -1,9 +1,8 @@
 //===-- lib/CodeGen/GlobalISel/CallLowering.cpp - Call lowering -----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
@@ -23,6 +22,8 @@
 
 using namespace llvm;
 
+void CallLowering::anchor() {}
+
 bool CallLowering::lowerCall(
     MachineIRBuilder &MIRBuilder, ImmutableCallSite CS, unsigned ResReg,
     ArrayRef<unsigned> ArgRegs, std::function<unsigned()> GetCalleeReg) const {
@@ -180,3 +181,5 @@ unsigned CallLowering::ValueHandler::extendRegister(unsigned ValReg,
   }
   llvm_unreachable("unable to extend register");
 }
+
+void CallLowering::ValueHandler::anchor() {}
diff --git a/lib/CodeGen/GlobalISel/Combiner.cpp b/lib/CodeGen/GlobalISel/Combiner.cpp
index 90fd54ec244ba34b5a45ac892f4a2182dda10e85..093702bd45d998a47df2626f1191a12b9c802ba8 100644
--- a/lib/CodeGen/GlobalISel/Combiner.cpp
+++ b/lib/CodeGen/GlobalISel/Combiner.cpp
@@ -1,9 +1,8 @@
 //===-- lib/CodeGen/GlobalISel/Combiner.cpp -------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -13,7 +12,9 @@
 
 #include "llvm/CodeGen/GlobalISel/Combiner.h"
 #include "llvm/ADT/PostOrderIterator.h"
+#include "llvm/CodeGen/GlobalISel/CSEInfo.h"
 #include "llvm/CodeGen/GlobalISel/CombinerInfo.h"
+#include "llvm/CodeGen/GlobalISel/CSEMIRBuilder.h"
 #include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h"
 #include "llvm/CodeGen/GlobalISel/GISelWorkList.h"
 #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
@@ -35,38 +36,33 @@ namespace {
 /// instruction creation will schedule that instruction for a future visit.
 /// Other Combiner implementations may require more complex behaviour from
 /// their GISelChangeObserver subclass.
-class WorkListMaintainer : public GISelChangeObserver,
-                           public MachineFunction::Delegate {
+class WorkListMaintainer : public GISelChangeObserver {
   using WorkListTy = GISelWorkList<512>;
-  MachineFunction &MF;
   WorkListTy &WorkList;
   /// The instructions that have been created but we want to report once they
   /// have their operands. This is only maintained if debug output is requested.
   SmallPtrSet<const MachineInstr *, 4> CreatedInstrs;
 
 public:
-  WorkListMaintainer(MachineFunction &MF, WorkListTy &WorkList)
-      : GISelChangeObserver(), MF(MF), WorkList(WorkList) {
-    MF.setDelegate(this);
-  }
+  WorkListMaintainer(WorkListTy &WorkList)
+      : GISelChangeObserver(), WorkList(WorkList) {}
   virtual ~WorkListMaintainer() {
-    MF.resetDelegate(this);
   }
 
-  void erasingInstr(const MachineInstr &MI) override {
-    LLVM_DEBUG(dbgs() << "Erased: " << MI << "\n");
+  void erasingInstr(MachineInstr &MI) override {
+    LLVM_DEBUG(dbgs() << "Erasing: " << MI << "\n");
     WorkList.remove(&MI);
   }
-  void createdInstr(const MachineInstr &MI) override {
+  void createdInstr(MachineInstr &MI) override {
     LLVM_DEBUG(dbgs() << "Creating: " << MI << "\n");
     WorkList.insert(&MI);
     LLVM_DEBUG(CreatedInstrs.insert(&MI));
   }
-  void changingInstr(const MachineInstr &MI) override {
+  void changingInstr(MachineInstr &MI) override {
     LLVM_DEBUG(dbgs() << "Changing: " << MI << "\n");
     WorkList.insert(&MI);
   }
-  void changedInstr(const MachineInstr &MI) override {
+  void changedInstr(MachineInstr &MI) override {
     LLVM_DEBUG(dbgs() << "Changed: " << MI << "\n");
     WorkList.insert(&MI);
   }
@@ -79,13 +75,6 @@ public:
     });
     LLVM_DEBUG(CreatedInstrs.clear());
   }
-
-  void MF_HandleInsertion(const MachineInstr &MI) override {
-    createdInstr(MI);
-  }
-  void MF_HandleRemoval(const MachineInstr &MI) override {
-    erasingInstr(MI);
-  }
 };
 }
 
@@ -94,15 +83,20 @@ Combiner::Combiner(CombinerInfo &Info, const TargetPassConfig *TPC)
   (void)this->TPC; // FIXME: Remove when used.
 }
 
-bool Combiner::combineMachineInstrs(MachineFunction &MF) {
+bool Combiner::combineMachineInstrs(MachineFunction &MF,
+                                    GISelCSEInfo *CSEInfo) {
   // If the ISel pipeline failed, do not bother running this pass.
   // FIXME: Should this be here or in individual combiner passes.
   if (MF.getProperties().hasProperty(
           MachineFunctionProperties::Property::FailedISel))
     return false;
 
+  Builder =
+      CSEInfo ? make_unique<CSEMIRBuilder>() : make_unique<MachineIRBuilder>();
   MRI = &MF.getRegInfo();
-  Builder.setMF(MF);
+  Builder->setMF(MF);
+  if (CSEInfo)
+    Builder->setCSEInfo(CSEInfo);
 
   LLVM_DEBUG(dbgs() << "Generic MI Combiner for: " << MF.getName() << '\n');
 
@@ -110,14 +104,19 @@ bool Combiner::combineMachineInstrs(MachineFunction &MF) {
 
   bool MFChanged = false;
   bool Changed;
+  MachineIRBuilder &B = *Builder.get();
 
   do {
     // Collect all instructions. Do a post order traversal for basic blocks and
     // insert with list bottom up, so while we pop_back_val, we'll traverse top
     // down RPOT.
     Changed = false;
-    GISelWorkList<512> WorkList(&MF);
-    WorkListMaintainer Observer(MF, WorkList);
+    GISelWorkList<512> WorkList;
+    WorkListMaintainer Observer(WorkList);
+    GISelObserverWrapper WrapperObserver(&Observer);
+    if (CSEInfo)
+      WrapperObserver.addObserver(CSEInfo);
+    RAIIDelegateInstaller DelInstall(MF, &WrapperObserver);
     for (MachineBasicBlock *MBB : post_order(&MF)) {
       if (MBB->empty())
         continue;
@@ -137,7 +136,7 @@ bool Combiner::combineMachineInstrs(MachineFunction &MF) {
     while (!WorkList.empty()) {
       MachineInstr *CurrInst = WorkList.pop_back_val();
       LLVM_DEBUG(dbgs() << "\nTry combining " << *CurrInst;);
-      Changed |= CInfo.combine(Observer, *CurrInst, Builder);
+      Changed |= CInfo.combine(WrapperObserver, *CurrInst, B);
       Observer.reportFullyCreatedInstrs();
     }
     MFChanged |= Changed;
diff --git a/lib/CodeGen/GlobalISel/CombinerHelper.cpp b/lib/CodeGen/GlobalISel/CombinerHelper.cpp
index b1c5670a6dec3f76398394b1785b180ec4fa2449..a2b1e8e1eb4afed0182f3b8122497e850c21d4e9 100644
--- a/lib/CodeGen/GlobalISel/CombinerHelper.cpp
+++ b/lib/CodeGen/GlobalISel/CombinerHelper.cpp
@@ -1,9 +1,8 @@
 //===-- lib/CodeGen/GlobalISel/GICombinerHelper.cpp -----------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 #include "llvm/CodeGen/GlobalISel/CombinerHelper.h"
@@ -182,6 +181,14 @@ bool CombinerHelper::tryCombineExtendingLoads(MachineInstr &MI) {
   if (!LoadValueTy.isScalar())
     return false;
 
+  // Most architectures are going to legalize <s8 loads into at least a 1 byte
+  // load, and the MMOs can only describe memory accesses in multiples of bytes.
+  // If we try to perform extload combining on those, we can end up with
+  // %a(s8) = extload %ptr (load 1 byte from %ptr)
+  // ... which is an illegal extload instruction.
+  if (LoadValueTy.getSizeInBits() < 8)
+    return false;
+
   // Find the preferred type aside from the any-extends (unless it's the only
   // one) and non-extending ops. We'll emit an extending load to that type and
   // and emit a variant of (extend (trunc X)) for the others according to the
diff --git a/lib/CodeGen/GlobalISel/GISelChangeObserver.cpp b/lib/CodeGen/GlobalISel/GISelChangeObserver.cpp
index 993a919826fa6a3eeb54c34ff515d874f56837ef..5a532c995ce2719ce68cdc99663cdb15d0382144 100644
--- a/lib/CodeGen/GlobalISel/GISelChangeObserver.cpp
+++ b/lib/CodeGen/GlobalISel/GISelChangeObserver.cpp
@@ -1,9 +1,8 @@
 //===-- lib/CodeGen/GlobalISel/GISelChangeObserver.cpp --------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -29,3 +28,12 @@ void GISelChangeObserver::finishedChangingAllUsesOfReg() {
     changedInstr(*ChangedMI);
 }
 
+RAIIDelegateInstaller::RAIIDelegateInstaller(MachineFunction &MF,
+                                             MachineFunction::Delegate *Del)
+    : MF(MF), Delegate(Del) {
+  // Register this as the delegate for handling insertions and deletions of
+  // instructions.
+  MF.setDelegate(Del);
+}
+
+RAIIDelegateInstaller::~RAIIDelegateInstaller() { MF.resetDelegate(Delegate); }
diff --git a/lib/CodeGen/GlobalISel/GlobalISel.cpp b/lib/CodeGen/GlobalISel/GlobalISel.cpp
index 00c6a9d631586ea727599c6c4affcf1184d57b1a..e0391e6f6467b08d7cb20227c0142217be9f53e0 100644
--- a/lib/CodeGen/GlobalISel/GlobalISel.cpp
+++ b/lib/CodeGen/GlobalISel/GlobalISel.cpp
@@ -1,9 +1,8 @@
 //===-- llvm/CodeGen/GlobalISel/GlobalIsel.cpp --- GlobalISel ----*- C++ -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/lib/CodeGen/GlobalISel/IRTranslator.cpp b/lib/CodeGen/GlobalISel/IRTranslator.cpp
index 33313756a3899d6f3fbcef46a57b9b325a484437..7db54079af1fba0f3723bde2ed9ddca3b722251d 100644
--- a/lib/CodeGen/GlobalISel/IRTranslator.cpp
+++ b/lib/CodeGen/GlobalISel/IRTranslator.cpp
@@ -1,9 +1,8 @@
 //===- llvm/CodeGen/GlobalISel/IRTranslator.cpp - IRTranslator ---*- C++ -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
@@ -17,8 +16,10 @@
 #include "llvm/ADT/SmallSet.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/Analysis/OptimizationRemarkEmitter.h"
+#include "llvm/Analysis/ValueTracking.h"
 #include "llvm/CodeGen/Analysis.h"
 #include "llvm/CodeGen/GlobalISel/CallLowering.h"
+#include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h"
 #include "llvm/CodeGen/LowLevelType.h"
 #include "llvm/CodeGen/MachineBasicBlock.h"
 #include "llvm/CodeGen/MachineFrameInfo.h"
@@ -75,11 +76,16 @@
 
 using namespace llvm;
 
+static cl::opt<bool>
+    EnableCSEInIRTranslator("enable-cse-in-irtranslator",
+                            cl::desc("Should enable CSE in irtranslator"),
+                            cl::Optional, cl::init(false));
 char IRTranslator::ID = 0;
 
 INITIALIZE_PASS_BEGIN(IRTranslator, DEBUG_TYPE, "IRTranslator LLVM IR -> MI",
                 false, false)
 INITIALIZE_PASS_DEPENDENCY(TargetPassConfig)
+INITIALIZE_PASS_DEPENDENCY(GISelCSEAnalysisWrapperPass)
 INITIALIZE_PASS_END(IRTranslator, DEBUG_TYPE, "IRTranslator LLVM IR -> MI",
                 false, false)
 
@@ -105,20 +111,24 @@ IRTranslator::IRTranslator() : MachineFunctionPass(ID) {
 }
 
 #ifndef NDEBUG
+namespace {
 /// Verify that every instruction created has the same DILocation as the
 /// instruction being translated.
-class DILocationVerifier : MachineFunction::Delegate {
-  MachineFunction &MF;
+class DILocationVerifier : public GISelChangeObserver {
   const Instruction *CurrInst = nullptr;
 
 public:
-  DILocationVerifier(MachineFunction &MF) : MF(MF) { MF.setDelegate(this); }
-  ~DILocationVerifier() { MF.resetDelegate(this); }
+  DILocationVerifier() = default;
+  ~DILocationVerifier() = default;
 
   const Instruction *getCurrentInst() const { return CurrInst; }
   void setCurrentInst(const Instruction *Inst) { CurrInst = Inst; }
 
-  void MF_HandleInsertion(const MachineInstr &MI) override {
+  void erasingInstr(MachineInstr &MI) override {}
+  void changingInstr(MachineInstr &MI) override {}
+  void changedInstr(MachineInstr &MI) override {}
+
+  void createdInstr(MachineInstr &MI) override {
     assert(getCurrentInst() && "Inserted instruction without a current MI");
 
     // Only print the check message if we're actually checking it.
@@ -129,14 +139,15 @@ public:
     assert(CurrInst->getDebugLoc() == MI.getDebugLoc() &&
            "Line info was not transferred to all instructions");
   }
-  void MF_HandleRemoval(const MachineInstr &MI) override {}
 };
+} // namespace
 #endif // ifndef NDEBUG
 
 
 void IRTranslator::getAnalysisUsage(AnalysisUsage &AU) const {
   AU.addRequired<StackProtector>();
   AU.addRequired<TargetPassConfig>();
+  AU.addRequired<GISelCSEAnalysisWrapperPass>();
   getSelectionDAGFallbackAnalysisUsage(AU);
   MachineFunctionPass::getAnalysisUsage(AU);
 }
@@ -309,12 +320,13 @@ bool IRTranslator::translateBinaryOp(unsigned Opcode, const User &U,
   unsigned Op0 = getOrCreateVReg(*U.getOperand(0));
   unsigned Op1 = getOrCreateVReg(*U.getOperand(1));
   unsigned Res = getOrCreateVReg(U);
-  auto FBinOp = MIRBuilder.buildInstr(Opcode).addDef(Res).addUse(Op0).addUse(Op1);
+  uint16_t Flags = 0;
   if (isa<Instruction>(U)) {
-    MachineInstr *FBinOpMI = FBinOp.getInstr();
     const Instruction &I = cast<Instruction>(U);
-    FBinOpMI->copyIRFlags(I);
+    Flags = MachineInstr::copyFlagsFromInstruction(I);
   }
+
+  MIRBuilder.buildInstr(Opcode, {Res}, {Op0, Op1}, Flags);
   return true;
 }
 
@@ -333,7 +345,7 @@ bool IRTranslator::translateFSub(const User &U, MachineIRBuilder &MIRBuilder) {
 bool IRTranslator::translateFNeg(const User &U, MachineIRBuilder &MIRBuilder) {
   MIRBuilder.buildInstr(TargetOpcode::G_FNEG)
       .addDef(getOrCreateVReg(U))
-      .addUse(getOrCreateVReg(*U.getOperand(1)));
+      .addUse(getOrCreateVReg(*U.getOperand(0)));
   return true;
 }
 
@@ -355,8 +367,8 @@ bool IRTranslator::translateCompare(const User &U,
     MIRBuilder.buildCopy(
         Res, getOrCreateVReg(*Constant::getAllOnesValue(CI->getType())));
   else {
-    auto FCmp = MIRBuilder.buildFCmp(Pred, Res, Op0, Op1);
-    FCmp->copyIRFlags(*CI);
+    MIRBuilder.buildInstr(TargetOpcode::G_FCMP, {Res}, {Pred, Op0, Op1},
+                          MachineInstr::copyFlagsFromInstruction(*CI));
   }
 
   return true;
@@ -591,13 +603,13 @@ bool IRTranslator::translateSelect(const User &U,
   ArrayRef<unsigned> Op1Regs = getOrCreateVRegs(*U.getOperand(2));
 
   const SelectInst &SI = cast<SelectInst>(U);
-  const CmpInst *Cmp = dyn_cast<CmpInst>(SI.getCondition());
+  uint16_t Flags = 0;
+  if (const CmpInst *Cmp = dyn_cast<CmpInst>(SI.getCondition()))
+    Flags = MachineInstr::copyFlagsFromInstruction(*Cmp);
+
   for (unsigned i = 0; i < ResRegs.size(); ++i) {
-    auto Select =
-        MIRBuilder.buildSelect(ResRegs[i], Tst, Op0Regs[i], Op1Regs[i]);
-    if (Cmp && isa<FPMathOperator>(Cmp)) {
-      Select->copyIRFlags(*Cmp);
-    }
+    MIRBuilder.buildInstr(TargetOpcode::G_SELECT, {ResRegs[i]},
+                          {Tst, Op0Regs[i], Op1Regs[i]}, Flags);
   }
 
   return true;
@@ -777,19 +789,109 @@ bool IRTranslator::translateOverflowIntrinsic(const CallInst &CI, unsigned Op,
   return true;
 }
 
+unsigned IRTranslator::getSimpleIntrinsicOpcode(Intrinsic::ID ID) {
+  switch (ID) {
+    default:
+      break;
+    case Intrinsic::bswap:
+      return TargetOpcode::G_BSWAP;
+    case Intrinsic::ceil:
+      return TargetOpcode::G_FCEIL;
+    case Intrinsic::cos:
+      return TargetOpcode::G_FCOS;
+    case Intrinsic::ctpop:
+      return TargetOpcode::G_CTPOP;
+    case Intrinsic::exp:
+      return TargetOpcode::G_FEXP;
+    case Intrinsic::exp2:
+      return TargetOpcode::G_FEXP2;
+    case Intrinsic::fabs:
+      return TargetOpcode::G_FABS;
+    case Intrinsic::canonicalize:
+      return TargetOpcode::G_FCANONICALIZE;
+    case Intrinsic::floor:
+      return TargetOpcode::G_FFLOOR;
+    case Intrinsic::fma:
+      return TargetOpcode::G_FMA;
+    case Intrinsic::log:
+      return TargetOpcode::G_FLOG;
+    case Intrinsic::log2:
+      return TargetOpcode::G_FLOG2;
+    case Intrinsic::log10:
+      return TargetOpcode::G_FLOG10;
+    case Intrinsic::pow:
+      return TargetOpcode::G_FPOW;
+    case Intrinsic::round:
+      return TargetOpcode::G_INTRINSIC_ROUND;
+    case Intrinsic::sin:
+      return TargetOpcode::G_FSIN;
+    case Intrinsic::sqrt:
+      return TargetOpcode::G_FSQRT;
+    case Intrinsic::trunc:
+      return TargetOpcode::G_INTRINSIC_TRUNC;
+  }
+  return Intrinsic::not_intrinsic;
+}
+
+bool IRTranslator::translateSimpleIntrinsic(const CallInst &CI,
+                                            Intrinsic::ID ID,
+                                            MachineIRBuilder &MIRBuilder) {
+
+  unsigned Op = getSimpleIntrinsicOpcode(ID);
+
+  // Is this a simple intrinsic?
+  if (Op == Intrinsic::not_intrinsic)
+    return false;
+
+  // Yes. Let's translate it.
+  SmallVector<llvm::SrcOp, 4> VRegs;
+  for (auto &Arg : CI.arg_operands())
+    VRegs.push_back(getOrCreateVReg(*Arg));
+
+  MIRBuilder.buildInstr(Op, {getOrCreateVReg(CI)}, VRegs,
+                        MachineInstr::copyFlagsFromInstruction(CI));
+  return true;
+}
+
 bool IRTranslator::translateKnownIntrinsic(const CallInst &CI, Intrinsic::ID ID,
                                            MachineIRBuilder &MIRBuilder) {
+
+  // If this is a simple intrinsic (that is, we just need to add a def of
+  // a vreg, and uses for each arg operand, then translate it.
+  if (translateSimpleIntrinsic(CI, ID, MIRBuilder))
+    return true;
+
   switch (ID) {
   default:
     break;
   case Intrinsic::lifetime_start:
-  case Intrinsic::lifetime_end:
-    // Stack coloring is not enabled in O0 (which we care about now) so we can
-    // drop these. Make sure someone notices when we start compiling at higher
-    // opts though.
-    if (MF->getTarget().getOptLevel() != CodeGenOpt::None)
-      return false;
+  case Intrinsic::lifetime_end: {
+    // No stack colouring in O0, discard region information.
+    if (MF->getTarget().getOptLevel() == CodeGenOpt::None)
+      return true;
+
+    unsigned Op = ID == Intrinsic::lifetime_start ? TargetOpcode::LIFETIME_START
+                                                  : TargetOpcode::LIFETIME_END;
+
+    // Get the underlying objects for the location passed on the lifetime
+    // marker.
+    SmallVector<Value *, 4> Allocas;
+    GetUnderlyingObjects(CI.getArgOperand(1), Allocas, *DL);
+
+    // Iterate over each underlying object, creating lifetime markers for each
+    // static alloca. Quit if we find a non-static alloca.
+    for (Value *V : Allocas) {
+      AllocaInst *AI = dyn_cast<AllocaInst>(V);
+      if (!AI)
+        continue;
+
+      if (!AI->isStaticAlloca())
+        return true;
+
+      MIRBuilder.buildInstr(Op).addFrameIndex(getOrCreateFrameIndex(*AI));
+    }
     return true;
+  }
   case Intrinsic::dbg_declare: {
     const DbgDeclareInst &DI = cast<DbgDeclareInst>(CI);
     assert(DI.getVariable() && "Missing variable");
@@ -837,10 +939,11 @@ bool IRTranslator::translateKnownIntrinsic(const CallInst &CI, Intrinsic::ID ID,
     Value *Ptr = CI.getArgOperand(0);
     unsigned ListSize = TLI.getVaListSizeInBits(*DL) / 8;
 
+    // FIXME: Get alignment
     MIRBuilder.buildInstr(TargetOpcode::G_VASTART)
         .addUse(getOrCreateVReg(*Ptr))
         .addMemOperand(MF->getMachineMemOperand(
-            MachinePointerInfo(Ptr), MachineMemOperand::MOStore, ListSize, 0));
+            MachinePointerInfo(Ptr), MachineMemOperand::MOStore, ListSize, 1));
     return true;
   }
   case Intrinsic::dbg_value: {
@@ -878,75 +981,6 @@ bool IRTranslator::translateKnownIntrinsic(const CallInst &CI, Intrinsic::ID ID,
     return translateOverflowIntrinsic(CI, TargetOpcode::G_UMULO, MIRBuilder);
   case Intrinsic::smul_with_overflow:
     return translateOverflowIntrinsic(CI, TargetOpcode::G_SMULO, MIRBuilder);
-  case Intrinsic::pow: {
-    auto Pow = MIRBuilder.buildInstr(TargetOpcode::G_FPOW)
-        .addDef(getOrCreateVReg(CI))
-        .addUse(getOrCreateVReg(*CI.getArgOperand(0)))
-        .addUse(getOrCreateVReg(*CI.getArgOperand(1)));
-    Pow->copyIRFlags(CI);
-    return true;
-  }
-  case Intrinsic::exp: {
-    auto Exp = MIRBuilder.buildInstr(TargetOpcode::G_FEXP)
-        .addDef(getOrCreateVReg(CI))
-        .addUse(getOrCreateVReg(*CI.getArgOperand(0)));
-    Exp->copyIRFlags(CI);
-    return true;
-  }
-  case Intrinsic::exp2: {
-    auto Exp2 = MIRBuilder.buildInstr(TargetOpcode::G_FEXP2)
-        .addDef(getOrCreateVReg(CI))
-        .addUse(getOrCreateVReg(*CI.getArgOperand(0)));
-    Exp2->copyIRFlags(CI);
-    return true;
-  }
-  case Intrinsic::log: {
-    auto Log = MIRBuilder.buildInstr(TargetOpcode::G_FLOG)
-        .addDef(getOrCreateVReg(CI))
-        .addUse(getOrCreateVReg(*CI.getArgOperand(0)));
-    Log->copyIRFlags(CI);
-    return true;
-  }
-  case Intrinsic::log2: {
-    auto Log2 = MIRBuilder.buildInstr(TargetOpcode::G_FLOG2)
-        .addDef(getOrCreateVReg(CI))
-        .addUse(getOrCreateVReg(*CI.getArgOperand(0)));
-    Log2->copyIRFlags(CI);
-    return true;
-  }
-  case Intrinsic::log10: {
-    auto Log10 = MIRBuilder.buildInstr(TargetOpcode::G_FLOG10)
-        .addDef(getOrCreateVReg(CI))
-        .addUse(getOrCreateVReg(*CI.getArgOperand(0)));
-    Log10->copyIRFlags(CI);
-    return true;
-  }
-  case Intrinsic::fabs: {
-    auto Fabs = MIRBuilder.buildInstr(TargetOpcode::G_FABS)
-        .addDef(getOrCreateVReg(CI))
-        .addUse(getOrCreateVReg(*CI.getArgOperand(0)));
-    Fabs->copyIRFlags(CI);
-    return true;
-  }
-  case Intrinsic::trunc:
-    MIRBuilder.buildInstr(TargetOpcode::G_INTRINSIC_TRUNC)
-        .addDef(getOrCreateVReg(CI))
-        .addUse(getOrCreateVReg(*CI.getArgOperand(0)));
-    return true;
-  case Intrinsic::round:
-    MIRBuilder.buildInstr(TargetOpcode::G_INTRINSIC_ROUND)
-        .addDef(getOrCreateVReg(CI))
-        .addUse(getOrCreateVReg(*CI.getArgOperand(0)));
-    return true;
-  case Intrinsic::fma: {
-    auto FMA = MIRBuilder.buildInstr(TargetOpcode::G_FMA)
-        .addDef(getOrCreateVReg(CI))
-        .addUse(getOrCreateVReg(*CI.getArgOperand(0)))
-        .addUse(getOrCreateVReg(*CI.getArgOperand(1)))
-        .addUse(getOrCreateVReg(*CI.getArgOperand(2)));
-    FMA->copyIRFlags(CI);
-    return true;
-  }
   case Intrinsic::fmuladd: {
     const TargetMachine &TM = MF->getTarget();
     const TargetLowering &TLI = *MF->getSubtarget().getTargetLowering();
@@ -958,14 +992,14 @@ bool IRTranslator::translateKnownIntrinsic(const CallInst &CI, Intrinsic::ID ID,
         TLI.isFMAFasterThanFMulAndFAdd(TLI.getValueType(*DL, CI.getType()))) {
       // TODO: Revisit this to see if we should move this part of the
       // lowering to the combiner.
-      auto FMA =  MIRBuilder.buildInstr(TargetOpcode::G_FMA, {Dst}, {Op0, Op1, Op2});
-      FMA->copyIRFlags(CI);
+      MIRBuilder.buildInstr(TargetOpcode::G_FMA, {Dst}, {Op0, Op1, Op2},
+                            MachineInstr::copyFlagsFromInstruction(CI));
     } else {
       LLT Ty = getLLTForType(*CI.getType(), *DL);
-      auto FMul = MIRBuilder.buildInstr(TargetOpcode::G_FMUL, {Ty}, {Op0, Op1});
-      FMul->copyIRFlags(CI);
-      auto FAdd =  MIRBuilder.buildInstr(TargetOpcode::G_FADD, {Dst}, {FMul, Op2});
-      FAdd->copyIRFlags(CI);
+      auto FMul = MIRBuilder.buildInstr(TargetOpcode::G_FMUL, {Ty}, {Op0, Op1},
+                                        MachineInstr::copyFlagsFromInstruction(CI));
+      MIRBuilder.buildInstr(TargetOpcode::G_FADD, {Dst}, {FMul, Op2},
+                            MachineInstr::copyFlagsFromInstruction(CI));
     }
     return true;
   }
@@ -1026,12 +1060,6 @@ bool IRTranslator::translateKnownIntrinsic(const CallInst &CI, Intrinsic::ID ID,
         .addUse(getOrCreateVReg(*CI.getArgOperand(0)));
     return true;
   }
-  case Intrinsic::ctpop: {
-    MIRBuilder.buildInstr(TargetOpcode::G_CTPOP)
-        .addDef(getOrCreateVReg(CI))
-        .addUse(getOrCreateVReg(*CI.getArgOperand(0)));
-    return true;
-  }
   case Intrinsic::invariant_start: {
     LLT PtrTy = getLLTForType(*CI.getArgOperand(0)->getType(), *DL);
     unsigned Undef = MRI->createGenericVirtualRegister(PtrTy);
@@ -1161,9 +1189,13 @@ bool IRTranslator::translateCall(const User &U, MachineIRBuilder &MIRBuilder) {
   TargetLowering::IntrinsicInfo Info;
   // TODO: Add a GlobalISel version of getTgtMemIntrinsic.
   if (TLI.getTgtMemIntrinsic(Info, CI, *MF, ID)) {
+    unsigned Align = Info.align;
+    if (Align == 0)
+      Align = DL->getABITypeAlignment(Info.memVT.getTypeForEVT(F->getContext()));
+
     uint64_t Size = Info.memVT.getStoreSize();
     MIB.addMemOperand(MF->getMachineMemOperand(MachinePointerInfo(Info.ptrVal),
-                                               Info.flags, Size, Info.align));
+                                               Info.flags, Size, Align));
   }
 
   return true;
@@ -1225,6 +1257,12 @@ bool IRTranslator::translateInvoke(const User &U,
   return true;
 }
 
+bool IRTranslator::translateCallBr(const User &U,
+                                   MachineIRBuilder &MIRBuilder) {
+  // FIXME: Implement this.
+  return false;
+}
+
 bool IRTranslator::translateLandingPad(const User &U,
                                        MachineIRBuilder &MIRBuilder) {
   const LandingPadInst &LP = cast<LandingPadInst>(U);
@@ -1546,12 +1584,14 @@ bool IRTranslator::translateAtomicRMW(const User &U,
 
 void IRTranslator::finishPendingPhis() {
 #ifndef NDEBUG
-  DILocationVerifier Verifier(*MF);
+  DILocationVerifier Verifier;
+  GISelObserverWrapper WrapperObserver(&Verifier);
+  RAIIDelegateInstaller DelInstall(*MF, &WrapperObserver);
 #endif // ifndef NDEBUG
   for (auto &Phi : PendingPHIs) {
     const PHINode *PI = Phi.first;
     ArrayRef<MachineInstr *> ComponentPHIs = Phi.second;
-    EntryBuilder.setDebugLoc(PI->getDebugLoc());
+    EntryBuilder->setDebugLoc(PI->getDebugLoc());
 #ifndef NDEBUG
     Verifier.setCurrentInst(PI);
 #endif // ifndef NDEBUG
@@ -1592,11 +1632,12 @@ bool IRTranslator::valueIsSplit(const Value &V,
 }
 
 bool IRTranslator::translate(const Instruction &Inst) {
-  CurBuilder.setDebugLoc(Inst.getDebugLoc());
-  EntryBuilder.setDebugLoc(Inst.getDebugLoc());
+  CurBuilder->setDebugLoc(Inst.getDebugLoc());
+  EntryBuilder->setDebugLoc(Inst.getDebugLoc());
   switch(Inst.getOpcode()) {
-#define HANDLE_INST(NUM, OPCODE, CLASS) \
-    case Instruction::OPCODE: return translate##OPCODE(Inst, CurBuilder);
+#define HANDLE_INST(NUM, OPCODE, CLASS)                                        \
+  case Instruction::OPCODE:                                                    \
+    return translate##OPCODE(Inst, *CurBuilder.get());
 #include "llvm/IR/Instruction.def"
   default:
     return false;
@@ -1605,11 +1646,11 @@ bool IRTranslator::translate(const Instruction &Inst) {
 
 bool IRTranslator::translate(const Constant &C, unsigned Reg) {
   if (auto CI = dyn_cast<ConstantInt>(&C))
-    EntryBuilder.buildConstant(Reg, *CI);
+    EntryBuilder->buildConstant(Reg, *CI);
   else if (auto CF = dyn_cast<ConstantFP>(&C))
-    EntryBuilder.buildFConstant(Reg, *CF);
+    EntryBuilder->buildFConstant(Reg, *CF);
   else if (isa<UndefValue>(C))
-    EntryBuilder.buildUndef(Reg);
+    EntryBuilder->buildUndef(Reg);
   else if (isa<ConstantPointerNull>(C)) {
     // As we are trying to build a constant val of 0 into a pointer,
     // insert a cast to make them correct with respect to types.
@@ -1617,9 +1658,9 @@ bool IRTranslator::translate(const Constant &C, unsigned Reg) {
     auto *ZeroTy = Type::getIntNTy(C.getContext(), NullSize);
     auto *ZeroVal = ConstantInt::get(ZeroTy, 0);
     unsigned ZeroReg = getOrCreateVReg(*ZeroVal);
-    EntryBuilder.buildCast(Reg, ZeroReg);
+    EntryBuilder->buildCast(Reg, ZeroReg);
   } else if (auto GV = dyn_cast<GlobalValue>(&C))
-    EntryBuilder.buildGlobalValue(Reg, GV);
+    EntryBuilder->buildGlobalValue(Reg, GV);
   else if (auto CAZ = dyn_cast<ConstantAggregateZero>(&C)) {
     if (!CAZ->getType()->isVectorTy())
       return false;
@@ -1631,7 +1672,7 @@ bool IRTranslator::translate(const Constant &C, unsigned Reg) {
       Constant &Elt = *CAZ->getElementValue(i);
       Ops.push_back(getOrCreateVReg(Elt));
     }
-    EntryBuilder.buildBuildVector(Reg, Ops);
+    EntryBuilder->buildBuildVector(Reg, Ops);
   } else if (auto CV = dyn_cast<ConstantDataVector>(&C)) {
     // Return the scalar if it is a <1 x Ty> vector.
     if (CV->getNumElements() == 1)
@@ -1641,11 +1682,12 @@ bool IRTranslator::translate(const Constant &C, unsigned Reg) {
       Constant &Elt = *CV->getElementAsConstant(i);
       Ops.push_back(getOrCreateVReg(Elt));
     }
-    EntryBuilder.buildBuildVector(Reg, Ops);
+    EntryBuilder->buildBuildVector(Reg, Ops);
   } else if (auto CE = dyn_cast<ConstantExpr>(&C)) {
     switch(CE->getOpcode()) {
-#define HANDLE_INST(NUM, OPCODE, CLASS)                         \
-      case Instruction::OPCODE: return translate##OPCODE(*CE, EntryBuilder);
+#define HANDLE_INST(NUM, OPCODE, CLASS)                                        \
+  case Instruction::OPCODE:                                                    \
+    return translate##OPCODE(*CE, *EntryBuilder.get());
 #include "llvm/IR/Instruction.def"
     default:
       return false;
@@ -1657,9 +1699,9 @@ bool IRTranslator::translate(const Constant &C, unsigned Reg) {
     for (unsigned i = 0; i < CV->getNumOperands(); ++i) {
       Ops.push_back(getOrCreateVReg(*CV->getOperand(i)));
     }
-    EntryBuilder.buildBuildVector(Reg, Ops);
+    EntryBuilder->buildBuildVector(Reg, Ops);
   } else if (auto *BA = dyn_cast<BlockAddress>(&C)) {
-    EntryBuilder.buildBlockAddress(Reg, BA);
+    EntryBuilder->buildBlockAddress(Reg, BA);
   } else
     return false;
 
@@ -1676,8 +1718,8 @@ void IRTranslator::finalizeFunction() {
   // MachineIRBuilder::DebugLoc can outlive the DILocation it holds. Clear it
   // to avoid accessing free’d memory (in runOnMachineFunction) and to avoid
   // destroying it twice (in ~IRTranslator() and ~LLVMContext())
-  EntryBuilder = MachineIRBuilder();
-  CurBuilder = MachineIRBuilder();
+  EntryBuilder.reset();
+  CurBuilder.reset();
 }
 
 bool IRTranslator::runOnMachineFunction(MachineFunction &CurMF) {
@@ -1685,12 +1727,31 @@ bool IRTranslator::runOnMachineFunction(MachineFunction &CurMF) {
   const Function &F = MF->getFunction();
   if (F.empty())
     return false;
+  GISelCSEAnalysisWrapper &Wrapper =
+      getAnalysis<GISelCSEAnalysisWrapperPass>().getCSEWrapper();
+  // Set the CSEConfig and run the analysis.
+  GISelCSEInfo *CSEInfo = nullptr;
+  TPC = &getAnalysis<TargetPassConfig>();
+  bool EnableCSE = EnableCSEInIRTranslator.getNumOccurrences()
+                       ? EnableCSEInIRTranslator
+                       : TPC->isGISelCSEEnabled();
+
+  if (EnableCSE) {
+    EntryBuilder = make_unique<CSEMIRBuilder>(CurMF);
+    std::unique_ptr<CSEConfig> Config = make_unique<CSEConfig>();
+    CSEInfo = &Wrapper.get(std::move(Config));
+    EntryBuilder->setCSEInfo(CSEInfo);
+    CurBuilder = make_unique<CSEMIRBuilder>(CurMF);
+    CurBuilder->setCSEInfo(CSEInfo);
+  } else {
+    EntryBuilder = make_unique<MachineIRBuilder>();
+    CurBuilder = make_unique<MachineIRBuilder>();
+  }
   CLI = MF->getSubtarget().getCallLowering();
-  CurBuilder.setMF(*MF);
-  EntryBuilder.setMF(*MF);
+  CurBuilder->setMF(*MF);
+  EntryBuilder->setMF(*MF);
   MRI = &MF->getRegInfo();
   DL = &F.getParent()->getDataLayout();
-  TPC = &getAnalysis<TargetPassConfig>();
   ORE = llvm::make_unique<OptimizationRemarkEmitter>(&F);
 
   assert(PendingPHIs.empty() && "stale PHIs");
@@ -1709,7 +1770,7 @@ bool IRTranslator::runOnMachineFunction(MachineFunction &CurMF) {
   // Setup a separate basic-block for the arguments and constants
   MachineBasicBlock *EntryBB = MF->CreateMachineBasicBlock();
   MF->push_back(EntryBB);
-  EntryBuilder.setMBB(*EntryBB);
+  EntryBuilder->setMBB(*EntryBB);
 
   // Create all blocks, in IR order, to preserve the layout.
   for (const BasicBlock &BB: F) {
@@ -1746,7 +1807,7 @@ bool IRTranslator::runOnMachineFunction(MachineFunction &CurMF) {
     }
   }
 
-  if (!CLI->lowerFormalArguments(EntryBuilder, F, VRegArgs)) {
+  if (!CLI->lowerFormalArguments(*EntryBuilder.get(), F, VRegArgs)) {
     OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure",
                                F.getSubprogram(), &F.getEntryBlock());
     R << "unable to lower arguments: " << ore::NV("Prototype", F.getType());
@@ -1763,22 +1824,27 @@ bool IRTranslator::runOnMachineFunction(MachineFunction &CurMF) {
       assert(VRegs.empty() && "VRegs already populated?");
       VRegs.push_back(VArg);
     } else {
-      unpackRegs(*ArgIt, VArg, EntryBuilder);
+      unpackRegs(*ArgIt, VArg, *EntryBuilder.get());
     }
     ArgIt++;
   }
 
   // Need to visit defs before uses when translating instructions.
+  GISelObserverWrapper WrapperObserver;
+  if (EnableCSE && CSEInfo)
+    WrapperObserver.addObserver(CSEInfo);
   {
     ReversePostOrderTraversal<const Function *> RPOT(&F);
 #ifndef NDEBUG
-    DILocationVerifier Verifier(*MF);
+    DILocationVerifier Verifier;
+    WrapperObserver.addObserver(&Verifier);
 #endif // ifndef NDEBUG
+    RAIIDelegateInstaller DelInstall(*MF, &WrapperObserver);
     for (const BasicBlock *BB : RPOT) {
       MachineBasicBlock &MBB = getMBB(*BB);
       // Set the insertion point of all the following translations to
       // the end of this basic block.
-      CurBuilder.setMBB(MBB);
+      CurBuilder->setMBB(MBB);
 
       for (const Instruction &Inst : *BB) {
 #ifndef NDEBUG
@@ -1803,6 +1869,9 @@ bool IRTranslator::runOnMachineFunction(MachineFunction &CurMF) {
         return false;
       }
     }
+#ifndef NDEBUG
+    WrapperObserver.removeObserver(&Verifier);
+#endif
   }
 
   finishPendingPhis();
diff --git a/lib/CodeGen/GlobalISel/InstructionSelect.cpp b/lib/CodeGen/GlobalISel/InstructionSelect.cpp
index c83c791327e486806827c3084463e5e5eb67583c..4434c7e208b7c894af4ebe34d7c04839a42c4176 100644
--- a/lib/CodeGen/GlobalISel/InstructionSelect.cpp
+++ b/lib/CodeGen/GlobalISel/InstructionSelect.cpp
@@ -1,9 +1,8 @@
 //===- llvm/CodeGen/GlobalISel/InstructionSelect.cpp - InstructionSelect ---==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/lib/CodeGen/GlobalISel/InstructionSelector.cpp b/lib/CodeGen/GlobalISel/InstructionSelector.cpp
index 38913e4afcba30228438017f879983489490abf5..d8cd71f64b9c0d803d31996689a70ee04f184b3a 100644
--- a/lib/CodeGen/GlobalISel/InstructionSelector.cpp
+++ b/lib/CodeGen/GlobalISel/InstructionSelector.cpp
@@ -1,9 +1,8 @@
 //===- llvm/CodeGen/GlobalISel/InstructionSelector.cpp --------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/GlobalISel/LLVMBuild.txt b/lib/CodeGen/GlobalISel/LLVMBuild.txt
index a2daa3b222a6d77ba0750d0a8173ebd68f4b6a87..deeed37d7a959ec823b0c33ba601c5d6dc7299fc 100644
--- a/lib/CodeGen/GlobalISel/LLVMBuild.txt
+++ b/lib/CodeGen/GlobalISel/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/CodeGen/GlobalISel/LLVMBuild.txt -----------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/CodeGen/GlobalISel/LegalityPredicates.cpp b/lib/CodeGen/GlobalISel/LegalityPredicates.cpp
index 94eab9ae00c87c8e074bdfc29146ea00433f5993..07e0cb662b5824b67b1489866e583374302a0363 100644
--- a/lib/CodeGen/GlobalISel/LegalityPredicates.cpp
+++ b/lib/CodeGen/GlobalISel/LegalityPredicates.cpp
@@ -1,9 +1,8 @@
 //===- lib/CodeGen/GlobalISel/LegalizerPredicates.cpp - Predicates --------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -57,10 +56,30 @@ LegalityPredicate LegalityPredicates::isScalar(unsigned TypeIdx) {
   };
 }
 
+LegalityPredicate LegalityPredicates::isVector(unsigned TypeIdx) {
+  return [=](const LegalityQuery &Query) {
+    return Query.Types[TypeIdx].isVector();
+  };
+}
+
+LegalityPredicate LegalityPredicates::isPointer(unsigned TypeIdx) {
+  return [=](const LegalityQuery &Query) {
+    return Query.Types[TypeIdx].isPointer();
+  };
+}
+
+LegalityPredicate LegalityPredicates::isPointer(unsigned TypeIdx,
+                                                unsigned AddrSpace) {
+  return [=](const LegalityQuery &Query) {
+    LLT Ty = Query.Types[TypeIdx];
+    return Ty.isPointer() && Ty.getAddressSpace() == AddrSpace;
+  };
+}
+
 LegalityPredicate LegalityPredicates::narrowerThan(unsigned TypeIdx,
                                                    unsigned Size) {
   return [=](const LegalityQuery &Query) {
-    const LLT &QueryTy = Query.Types[TypeIdx];
+    const LLT QueryTy = Query.Types[TypeIdx];
     return QueryTy.isScalar() && QueryTy.getSizeInBits() < Size;
   };
 }
@@ -68,18 +87,49 @@ LegalityPredicate LegalityPredicates::narrowerThan(unsigned TypeIdx,
 LegalityPredicate LegalityPredicates::widerThan(unsigned TypeIdx,
                                                 unsigned Size) {
   return [=](const LegalityQuery &Query) {
-    const LLT &QueryTy = Query.Types[TypeIdx];
+    const LLT QueryTy = Query.Types[TypeIdx];
     return QueryTy.isScalar() && QueryTy.getSizeInBits() > Size;
   };
 }
 
+LegalityPredicate LegalityPredicates::scalarOrEltNarrowerThan(unsigned TypeIdx,
+                                                              unsigned Size) {
+  return [=](const LegalityQuery &Query) {
+    const LLT QueryTy = Query.Types[TypeIdx];
+    return QueryTy.getScalarSizeInBits() < Size;
+  };
+}
+
+LegalityPredicate LegalityPredicates::scalarOrEltWiderThan(unsigned TypeIdx,
+                                                           unsigned Size) {
+  return [=](const LegalityQuery &Query) {
+    const LLT QueryTy = Query.Types[TypeIdx];
+    return QueryTy.getScalarSizeInBits() > Size;
+  };
+}
+
+LegalityPredicate LegalityPredicates::scalarOrEltSizeNotPow2(unsigned TypeIdx) {
+  return [=](const LegalityQuery &Query) {
+    const LLT QueryTy = Query.Types[TypeIdx];
+    return !isPowerOf2_32(QueryTy.getScalarSizeInBits());
+  };
+}
+
 LegalityPredicate LegalityPredicates::sizeNotPow2(unsigned TypeIdx) {
   return [=](const LegalityQuery &Query) {
-    const LLT &QueryTy = Query.Types[TypeIdx];
+    const LLT QueryTy = Query.Types[TypeIdx];
     return QueryTy.isScalar() && !isPowerOf2_32(QueryTy.getSizeInBits());
   };
 }
 
+LegalityPredicate LegalityPredicates::sameSize(unsigned TypeIdx0,
+                                               unsigned TypeIdx1) {
+  return [=](const LegalityQuery &Query) {
+    return Query.Types[TypeIdx0].getSizeInBits() ==
+           Query.Types[TypeIdx1].getSizeInBits();
+  };
+}
+
 LegalityPredicate LegalityPredicates::memSizeInBytesNotPow2(unsigned MMOIdx) {
   return [=](const LegalityQuery &Query) {
     return !isPowerOf2_32(Query.MMODescrs[MMOIdx].SizeInBits / 8);
@@ -88,8 +138,8 @@ LegalityPredicate LegalityPredicates::memSizeInBytesNotPow2(unsigned MMOIdx) {
 
 LegalityPredicate LegalityPredicates::numElementsNotPow2(unsigned TypeIdx) {
   return [=](const LegalityQuery &Query) {
-    const LLT &QueryTy = Query.Types[TypeIdx];
-    return QueryTy.isVector() && isPowerOf2_32(QueryTy.getNumElements());
+    const LLT QueryTy = Query.Types[TypeIdx];
+    return QueryTy.isVector() && !isPowerOf2_32(QueryTy.getNumElements());
   };
 }
 
diff --git a/lib/CodeGen/GlobalISel/LegalizeMutations.cpp b/lib/CodeGen/GlobalISel/LegalizeMutations.cpp
index a29b32ecdc03a2b50d852c8d7a14174c3e925b79..fcbecf90a8455eea37344b6385d31c19f1d4ad3d 100644
--- a/lib/CodeGen/GlobalISel/LegalizeMutations.cpp
+++ b/lib/CodeGen/GlobalISel/LegalizeMutations.cpp
@@ -1,9 +1,8 @@
 //===- lib/CodeGen/GlobalISel/LegalizerMutations.cpp - Mutations ----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -27,25 +26,46 @@ LegalizeMutation LegalizeMutations::changeTo(unsigned TypeIdx,
   };
 }
 
-LegalizeMutation LegalizeMutations::widenScalarToNextPow2(unsigned TypeIdx,
-                                                          unsigned Min) {
+LegalizeMutation LegalizeMutations::changeElementTo(unsigned TypeIdx,
+                                                    unsigned FromTypeIdx) {
   return [=](const LegalityQuery &Query) {
-    unsigned NewSizeInBits =
-        1 << Log2_32_Ceil(Query.Types[TypeIdx].getSizeInBits());
-    if (NewSizeInBits < Min)
-      NewSizeInBits = Min;
-    return std::make_pair(TypeIdx, LLT::scalar(NewSizeInBits));
+    const LLT OldTy = Query.Types[TypeIdx];
+    const LLT NewTy = Query.Types[FromTypeIdx];
+    return std::make_pair(TypeIdx, OldTy.changeElementType(NewTy));
+  };
+}
+
+LegalizeMutation LegalizeMutations::changeElementTo(unsigned TypeIdx,
+                                                    LLT NewEltTy) {
+  return [=](const LegalityQuery &Query) {
+    const LLT OldTy = Query.Types[TypeIdx];
+    return std::make_pair(TypeIdx, OldTy.changeElementType(NewEltTy));
+  };
+}
+
+LegalizeMutation LegalizeMutations::widenScalarOrEltToNextPow2(unsigned TypeIdx,
+                                                               unsigned Min) {
+  return [=](const LegalityQuery &Query) {
+    const LLT Ty = Query.Types[TypeIdx];
+    unsigned NewEltSizeInBits =
+        std::max(1u << Log2_32_Ceil(Ty.getScalarSizeInBits()), Min);
+    return std::make_pair(TypeIdx, Ty.changeElementSize(NewEltSizeInBits));
   };
 }
 
 LegalizeMutation LegalizeMutations::moreElementsToNextPow2(unsigned TypeIdx,
                                                            unsigned Min) {
   return [=](const LegalityQuery &Query) {
-    const LLT &VecTy = Query.Types[TypeIdx];
-    unsigned NewNumElements = 1 << Log2_32_Ceil(VecTy.getNumElements());
-    if (NewNumElements < Min)
-      NewNumElements = Min;
-    return std::make_pair(
-        TypeIdx, LLT::vector(NewNumElements, VecTy.getScalarSizeInBits()));
+    const LLT VecTy = Query.Types[TypeIdx];
+    unsigned NewNumElements =
+        std::max(1u << Log2_32_Ceil(VecTy.getNumElements()), Min);
+    return std::make_pair(TypeIdx,
+                          LLT::vector(NewNumElements, VecTy.getElementType()));
+  };
+}
+
+LegalizeMutation LegalizeMutations::scalarize(unsigned TypeIdx) {
+  return [=](const LegalityQuery &Query) {
+    return std::make_pair(TypeIdx, Query.Types[TypeIdx].getElementType());
   };
 }
diff --git a/lib/CodeGen/GlobalISel/Legalizer.cpp b/lib/CodeGen/GlobalISel/Legalizer.cpp
index df2dcace4212c9033519f385420085780f49f3a0..02fa4d12038130c131bf2759b7904ba2c7b47db6 100644
--- a/lib/CodeGen/GlobalISel/Legalizer.cpp
+++ b/lib/CodeGen/GlobalISel/Legalizer.cpp
@@ -1,9 +1,8 @@
 //===-- llvm/CodeGen/GlobalISel/Legalizer.cpp -----------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -16,6 +15,8 @@
 #include "llvm/CodeGen/GlobalISel/Legalizer.h"
 #include "llvm/ADT/PostOrderIterator.h"
 #include "llvm/ADT/SetVector.h"
+#include "llvm/CodeGen/GlobalISel/CSEInfo.h"
+#include "llvm/CodeGen/GlobalISel/CSEMIRBuilder.h"
 #include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h"
 #include "llvm/CodeGen/GlobalISel/GISelWorkList.h"
 #include "llvm/CodeGen/GlobalISel/LegalizationArtifactCombiner.h"
@@ -33,11 +34,17 @@
 
 using namespace llvm;
 
+static cl::opt<bool>
+    EnableCSEInLegalizer("enable-cse-in-legalizer",
+                         cl::desc("Should enable CSE in Legalizer"),
+                         cl::Optional, cl::init(false));
+
 char Legalizer::ID = 0;
 INITIALIZE_PASS_BEGIN(Legalizer, DEBUG_TYPE,
                       "Legalize the Machine IR a function's Machine IR", false,
                       false)
 INITIALIZE_PASS_DEPENDENCY(TargetPassConfig)
+INITIALIZE_PASS_DEPENDENCY(GISelCSEAnalysisWrapperPass)
 INITIALIZE_PASS_END(Legalizer, DEBUG_TYPE,
                     "Legalize the Machine IR a function's Machine IR", false,
                     false)
@@ -48,6 +55,8 @@ Legalizer::Legalizer() : MachineFunctionPass(ID) {
 
 void Legalizer::getAnalysisUsage(AnalysisUsage &AU) const {
   AU.addRequired<TargetPassConfig>();
+  AU.addRequired<GISelCSEAnalysisWrapperPass>();
+  AU.addPreserved<GISelCSEAnalysisWrapperPass>();
   getSelectionDAGFallbackAnalysisUsage(AU);
   MachineFunctionPass::getAnalysisUsage(AU);
 }
@@ -67,12 +76,14 @@ static bool isArtifact(const MachineInstr &MI) {
   case TargetOpcode::G_UNMERGE_VALUES:
   case TargetOpcode::G_CONCAT_VECTORS:
   case TargetOpcode::G_BUILD_VECTOR:
+  case TargetOpcode::G_EXTRACT:
     return true;
   }
 }
 using InstListTy = GISelWorkList<256>;
 using ArtifactListTy = GISelWorkList<128>;
 
+namespace {
 class LegalizerWorkListManager : public GISelChangeObserver {
   InstListTy &InstList;
   ArtifactListTy &ArtifactList;
@@ -81,7 +92,7 @@ public:
   LegalizerWorkListManager(InstListTy &Insts, ArtifactListTy &Arts)
       : InstList(Insts), ArtifactList(Arts) {}
 
-  void createdInstr(const MachineInstr &MI) override {
+  void createdInstr(MachineInstr &MI) override {
     // Only legalize pre-isel generic instructions.
     // Legalization process could generate Target specific pseudo
     // instructions with generic types. Don't record them
@@ -94,23 +105,24 @@ public:
     LLVM_DEBUG(dbgs() << ".. .. New MI: " << MI);
   }
 
-  void erasingInstr(const MachineInstr &MI) override {
+  void erasingInstr(MachineInstr &MI) override {
     LLVM_DEBUG(dbgs() << ".. .. Erasing: " << MI);
     InstList.remove(&MI);
     ArtifactList.remove(&MI);
   }
 
-  void changingInstr(const MachineInstr &MI) override {
+  void changingInstr(MachineInstr &MI) override {
     LLVM_DEBUG(dbgs() << ".. .. Changing MI: " << MI);
   }
 
-  void changedInstr(const MachineInstr &MI) override {
+  void changedInstr(MachineInstr &MI) override {
     // When insts change, we want to revisit them to legalize them again.
     // We'll consider them the same as created.
     LLVM_DEBUG(dbgs() << ".. .. Changed MI: " << MI);
     createdInstr(MI);
   }
 };
+} // namespace
 
 bool Legalizer::runOnMachineFunction(MachineFunction &MF) {
   // If the ISel pipeline failed, do not bother running that pass.
@@ -120,14 +132,16 @@ bool Legalizer::runOnMachineFunction(MachineFunction &MF) {
   LLVM_DEBUG(dbgs() << "Legalize Machine IR for: " << MF.getName() << '\n');
   init(MF);
   const TargetPassConfig &TPC = getAnalysis<TargetPassConfig>();
+  GISelCSEAnalysisWrapper &Wrapper =
+      getAnalysis<GISelCSEAnalysisWrapperPass>().getCSEWrapper();
   MachineOptimizationRemarkEmitter MORE(MF, /*MBFI=*/nullptr);
 
   const size_t NumBlocks = MF.size();
   MachineRegisterInfo &MRI = MF.getRegInfo();
 
   // Populate Insts
-  InstListTy InstList(&MF);
-  ArtifactListTy ArtifactList(&MF);
+  InstListTy InstList;
+  ArtifactListTy ArtifactList;
   ReversePostOrderTraversal<MachineFunction *> RPOT(&MF);
   // Perform legalization bottom up so we can DCE as we legalize.
   // Traverse BB in RPOT and within each basic block, add insts top down,
@@ -146,12 +160,35 @@ bool Legalizer::runOnMachineFunction(MachineFunction &MF) {
         InstList.insert(&MI);
     }
   }
+  std::unique_ptr<MachineIRBuilder> MIRBuilder;
+  GISelCSEInfo *CSEInfo = nullptr;
+  bool EnableCSE = EnableCSEInLegalizer.getNumOccurrences()
+                       ? EnableCSEInLegalizer
+                       : TPC.isGISelCSEEnabled();
+
+  if (EnableCSE) {
+    MIRBuilder = make_unique<CSEMIRBuilder>();
+    std::unique_ptr<CSEConfig> Config = make_unique<CSEConfig>();
+    CSEInfo = &Wrapper.get(std::move(Config));
+    MIRBuilder->setCSEInfo(CSEInfo);
+  } else
+    MIRBuilder = make_unique<MachineIRBuilder>();
+  // This observer keeps the worklist updated.
   LegalizerWorkListManager WorkListObserver(InstList, ArtifactList);
-  LegalizerHelper Helper(MF, WorkListObserver);
+  // We want both WorkListObserver as well as CSEInfo to observe all changes.
+  // Use the wrapper observer.
+  GISelObserverWrapper WrapperObserver(&WorkListObserver);
+  if (EnableCSE && CSEInfo)
+    WrapperObserver.addObserver(CSEInfo);
+  // Now install the observer as the delegate to MF.
+  // This will keep all the observers notified about new insertions/deletions.
+  RAIIDelegateInstaller DelInstall(MF, &WrapperObserver);
+  LegalizerHelper Helper(MF, WrapperObserver, *MIRBuilder.get());
   const LegalizerInfo &LInfo(Helper.getLegalizerInfo());
-  LegalizationArtifactCombiner ArtCombiner(Helper.MIRBuilder, MF.getRegInfo(), LInfo);
-  auto RemoveDeadInstFromLists = [&WorkListObserver](MachineInstr *DeadMI) {
-    WorkListObserver.erasingInstr(*DeadMI);
+  LegalizationArtifactCombiner ArtCombiner(*MIRBuilder.get(), MF.getRegInfo(),
+                                           LInfo);
+  auto RemoveDeadInstFromLists = [&WrapperObserver](MachineInstr *DeadMI) {
+    WrapperObserver.erasingInstr(*DeadMI);
   };
   bool Changed = false;
   do {
diff --git a/lib/CodeGen/GlobalISel/LegalizerHelper.cpp b/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
index 266409dd1fabceb3c2d329bc1337e1a4619af098..b3170b039e7518b085e2d6207e87a29a50ca47da 100644
--- a/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
+++ b/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
@@ -1,9 +1,8 @@
 //===-- llvm/CodeGen/GlobalISel/LegalizerHelper.cpp -----------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -30,17 +29,49 @@
 using namespace llvm;
 using namespace LegalizeActions;
 
+/// Try to break down \p OrigTy into \p NarrowTy sized pieces.
+///
+/// Returns the number of \p NarrowTy elements needed to reconstruct \p OrigTy,
+/// with any leftover piece as type \p LeftoverTy
+///
+/// Returns -1 if the breakdown is not satisfiable.
+static int getNarrowTypeBreakDown(LLT OrigTy, LLT NarrowTy, LLT &LeftoverTy) {
+  assert(!LeftoverTy.isValid() && "this is an out argument");
+
+  unsigned Size = OrigTy.getSizeInBits();
+  unsigned NarrowSize = NarrowTy.getSizeInBits();
+  unsigned NumParts = Size / NarrowSize;
+  unsigned LeftoverSize = Size - NumParts * NarrowSize;
+  assert(Size > NarrowSize);
+
+  if (LeftoverSize == 0)
+    return NumParts;
+
+  if (NarrowTy.isVector()) {
+    unsigned EltSize = OrigTy.getScalarSizeInBits();
+    if (LeftoverSize % EltSize != 0)
+      return -1;
+    LeftoverTy = LLT::scalarOrVector(LeftoverSize / EltSize, EltSize);
+  } else {
+    LeftoverTy = LLT::scalar(LeftoverSize);
+  }
+
+  return NumParts;
+}
+
 LegalizerHelper::LegalizerHelper(MachineFunction &MF,
-                                 GISelChangeObserver &Observer)
-    : MRI(MF.getRegInfo()), LI(*MF.getSubtarget().getLegalizerInfo()),
-      Observer(Observer) {
+                                 GISelChangeObserver &Observer,
+                                 MachineIRBuilder &Builder)
+    : MIRBuilder(Builder), MRI(MF.getRegInfo()),
+      LI(*MF.getSubtarget().getLegalizerInfo()), Observer(Observer) {
   MIRBuilder.setMF(MF);
   MIRBuilder.setChangeObserver(Observer);
 }
 
 LegalizerHelper::LegalizerHelper(MachineFunction &MF, const LegalizerInfo &LI,
-                                 GISelChangeObserver &Observer)
-    : MRI(MF.getRegInfo()), LI(LI), Observer(Observer) {
+                                 GISelChangeObserver &Observer,
+                                 MachineIRBuilder &B)
+    : MIRBuilder(B), MRI(MF.getRegInfo()), LI(LI), Observer(Observer) {
   MIRBuilder.setMF(MF);
   MIRBuilder.setChangeObserver(Observer);
 }
@@ -68,6 +99,9 @@ LegalizerHelper::legalizeInstrStep(MachineInstr &MI) {
   case FewerElements:
     LLVM_DEBUG(dbgs() << ".. Reduce number of elements\n");
     return fewerElementsVector(MI, Step.TypeIdx, Step.NewType);
+  case MoreElements:
+    LLVM_DEBUG(dbgs() << ".. Increase number of elements\n");
+    return moreElementsVector(MI, Step.TypeIdx, Step.NewType);
   case Custom:
     LLVM_DEBUG(dbgs() << ".. Custom legalization\n");
     return LI.legalizeCustom(MI, MRI, MIRBuilder, Observer) ? Legalized
@@ -85,6 +119,96 @@ void LegalizerHelper::extractParts(unsigned Reg, LLT Ty, int NumParts,
   MIRBuilder.buildUnmerge(VRegs, Reg);
 }
 
+bool LegalizerHelper::extractParts(unsigned Reg, LLT RegTy,
+                                   LLT MainTy, LLT &LeftoverTy,
+                                   SmallVectorImpl<unsigned> &VRegs,
+                                   SmallVectorImpl<unsigned> &LeftoverRegs) {
+  assert(!LeftoverTy.isValid() && "this is an out argument");
+
+  unsigned RegSize = RegTy.getSizeInBits();
+  unsigned MainSize = MainTy.getSizeInBits();
+  unsigned NumParts = RegSize / MainSize;
+  unsigned LeftoverSize = RegSize - NumParts * MainSize;
+
+  // Use an unmerge when possible.
+  if (LeftoverSize == 0) {
+    for (unsigned I = 0; I < NumParts; ++I)
+      VRegs.push_back(MRI.createGenericVirtualRegister(MainTy));
+    MIRBuilder.buildUnmerge(VRegs, Reg);
+    return true;
+  }
+
+  if (MainTy.isVector()) {
+    unsigned EltSize = MainTy.getScalarSizeInBits();
+    if (LeftoverSize % EltSize != 0)
+      return false;
+    LeftoverTy = LLT::scalarOrVector(LeftoverSize / EltSize, EltSize);
+  } else {
+    LeftoverTy = LLT::scalar(LeftoverSize);
+  }
+
+  // For irregular sizes, extract the individual parts.
+  for (unsigned I = 0; I != NumParts; ++I) {
+    unsigned NewReg = MRI.createGenericVirtualRegister(MainTy);
+    VRegs.push_back(NewReg);
+    MIRBuilder.buildExtract(NewReg, Reg, MainSize * I);
+  }
+
+  for (unsigned Offset = MainSize * NumParts; Offset < RegSize;
+       Offset += LeftoverSize) {
+    unsigned NewReg = MRI.createGenericVirtualRegister(LeftoverTy);
+    LeftoverRegs.push_back(NewReg);
+    MIRBuilder.buildExtract(NewReg, Reg, Offset);
+  }
+
+  return true;
+}
+
+void LegalizerHelper::insertParts(unsigned DstReg,
+                                  LLT ResultTy, LLT PartTy,
+                                  ArrayRef<unsigned> PartRegs,
+                                  LLT LeftoverTy,
+                                  ArrayRef<unsigned> LeftoverRegs) {
+  if (!LeftoverTy.isValid()) {
+    assert(LeftoverRegs.empty());
+
+    if (!ResultTy.isVector()) {
+      MIRBuilder.buildMerge(DstReg, PartRegs);
+      return;
+    }
+
+    if (PartTy.isVector())
+      MIRBuilder.buildConcatVectors(DstReg, PartRegs);
+    else
+      MIRBuilder.buildBuildVector(DstReg, PartRegs);
+    return;
+  }
+
+  unsigned PartSize = PartTy.getSizeInBits();
+  unsigned LeftoverPartSize = LeftoverTy.getSizeInBits();
+
+  unsigned CurResultReg = MRI.createGenericVirtualRegister(ResultTy);
+  MIRBuilder.buildUndef(CurResultReg);
+
+  unsigned Offset = 0;
+  for (unsigned PartReg : PartRegs) {
+    unsigned NewResultReg = MRI.createGenericVirtualRegister(ResultTy);
+    MIRBuilder.buildInsert(NewResultReg, CurResultReg, PartReg, Offset);
+    CurResultReg = NewResultReg;
+    Offset += PartSize;
+  }
+
+  for (unsigned I = 0, E = LeftoverRegs.size(); I != E; ++I) {
+    // Use the original output register for the final insert to avoid a copy.
+    unsigned NewResultReg = (I + 1 == E) ?
+      DstReg : MRI.createGenericVirtualRegister(ResultTy);
+
+    MIRBuilder.buildInsert(NewResultReg, CurResultReg, LeftoverRegs[I], Offset);
+    CurResultReg = NewResultReg;
+    Offset += LeftoverPartSize;
+  }
+}
+
 static RTLIB::Libcall getRTLibDesc(unsigned Opcode, unsigned Size) {
   switch (Opcode) {
   case TargetOpcode::G_SDIV:
@@ -114,6 +238,9 @@ static RTLIB::Libcall getRTLibDesc(unsigned Opcode, unsigned Size) {
   case TargetOpcode::G_FDIV:
     assert((Size == 32 || Size == 64) && "Unsupported size");
     return Size == 64 ? RTLIB::DIV_F64 : RTLIB::DIV_F32;
+  case TargetOpcode::G_FEXP:
+    assert((Size == 32 || Size == 64) && "Unsupported size");
+    return Size == 64 ? RTLIB::EXP_F64 : RTLIB::EXP_F32;
   case TargetOpcode::G_FREM:
     return Size == 64 ? RTLIB::REM_F64 : RTLIB::REM_F32;
   case TargetOpcode::G_FPOW:
@@ -121,6 +248,26 @@ static RTLIB::Libcall getRTLibDesc(unsigned Opcode, unsigned Size) {
   case TargetOpcode::G_FMA:
     assert((Size == 32 || Size == 64) && "Unsupported size");
     return Size == 64 ? RTLIB::FMA_F64 : RTLIB::FMA_F32;
+  case TargetOpcode::G_FSIN:
+    assert((Size == 32 || Size == 64 || Size == 128) && "Unsupported size");
+    return Size == 128 ? RTLIB::SIN_F128
+                       : Size == 64 ? RTLIB::SIN_F64 : RTLIB::SIN_F32;
+  case TargetOpcode::G_FCOS:
+    assert((Size == 32 || Size == 64 || Size == 128) && "Unsupported size");
+    return Size == 128 ? RTLIB::COS_F128
+                       : Size == 64 ? RTLIB::COS_F64 : RTLIB::COS_F32;
+  case TargetOpcode::G_FLOG10:
+    assert((Size == 32 || Size == 64 || Size == 128) && "Unsupported size");
+    return Size == 128 ? RTLIB::LOG10_F128
+                       : Size == 64 ? RTLIB::LOG10_F64 : RTLIB::LOG10_F32;
+  case TargetOpcode::G_FLOG:
+    assert((Size == 32 || Size == 64 || Size == 128) && "Unsupported size");
+    return Size == 128 ? RTLIB::LOG_F128
+                       : Size == 64 ? RTLIB::LOG_F64 : RTLIB::LOG_F32;
+  case TargetOpcode::G_FLOG2:
+    assert((Size == 32 || Size == 64 || Size == 128) && "Unsupported size");
+    return Size == 128 ? RTLIB::LOG2_F128
+                       : Size == 64 ? RTLIB::LOG2_F64 : RTLIB::LOG2_F32;
   }
   llvm_unreachable("Unknown libcall function");
 }
@@ -212,7 +359,17 @@ LegalizerHelper::libcall(MachineInstr &MI) {
   case TargetOpcode::G_FDIV:
   case TargetOpcode::G_FMA:
   case TargetOpcode::G_FPOW:
-  case TargetOpcode::G_FREM: {
+  case TargetOpcode::G_FREM:
+  case TargetOpcode::G_FCOS:
+  case TargetOpcode::G_FSIN:
+  case TargetOpcode::G_FLOG10:
+  case TargetOpcode::G_FLOG:
+  case TargetOpcode::G_FLOG2:
+  case TargetOpcode::G_FEXP: {
+    if (Size > 64) {
+      LLVM_DEBUG(dbgs() << "Size " << Size << " too large to legalize.\n");
+      return UnableToLegalize;
+    }
     Type *HLTy = Size == 64 ? Type::getDoubleTy(Ctx) : Type::getFloatTy(Ctx);
     auto Status = simpleLibcall(MI, MIRBuilder, Size, HLTy);
     if (Status != Legalized)
@@ -281,10 +438,6 @@ LegalizerHelper::libcall(MachineInstr &MI) {
 LegalizerHelper::LegalizeResult LegalizerHelper::narrowScalar(MachineInstr &MI,
                                                               unsigned TypeIdx,
                                                               LLT NarrowTy) {
-  // FIXME: Don't know how to handle secondary types yet.
-  if (TypeIdx != 0 && MI.getOpcode() != TargetOpcode::G_EXTRACT)
-    return UnableToLegalize;
-
   MIRBuilder.setInstr(MI);
 
   uint64_t SizeOp0 = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
@@ -346,212 +499,113 @@ LegalizerHelper::LegalizeResult LegalizerHelper::narrowScalar(MachineInstr &MI,
     MI.eraseFromParent();
     return Legalized;
   }
-  case TargetOpcode::G_EXTRACT: {
-    if (TypeIdx != 1)
-      return UnableToLegalize;
-
-    int64_t SizeOp1 = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits();
-    // FIXME: add support for when SizeOp1 isn't an exact multiple of
-    // NarrowSize.
-    if (SizeOp1 % NarrowSize != 0)
-      return UnableToLegalize;
-    int NumParts = SizeOp1 / NarrowSize;
-
-    SmallVector<unsigned, 2> SrcRegs, DstRegs;
-    SmallVector<uint64_t, 2> Indexes;
-    extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, SrcRegs);
-
-    unsigned OpReg = MI.getOperand(0).getReg();
-    uint64_t OpStart = MI.getOperand(2).getImm();
-    uint64_t OpSize = MRI.getType(OpReg).getSizeInBits();
-    for (int i = 0; i < NumParts; ++i) {
-      unsigned SrcStart = i * NarrowSize;
-
-      if (SrcStart + NarrowSize <= OpStart || SrcStart >= OpStart + OpSize) {
-        // No part of the extract uses this subregister, ignore it.
-        continue;
-      } else if (SrcStart == OpStart && NarrowTy == MRI.getType(OpReg)) {
-        // The entire subregister is extracted, forward the value.
-        DstRegs.push_back(SrcRegs[i]);
-        continue;
-      }
-
-      // OpSegStart is where this destination segment would start in OpReg if it
-      // extended infinitely in both directions.
-      int64_t ExtractOffset;
-      uint64_t SegSize;
-      if (OpStart < SrcStart) {
-        ExtractOffset = 0;
-        SegSize = std::min(NarrowSize, OpStart + OpSize - SrcStart);
-      } else {
-        ExtractOffset = OpStart - SrcStart;
-        SegSize = std::min(SrcStart + NarrowSize - OpStart, OpSize);
-      }
-
-      unsigned SegReg = SrcRegs[i];
-      if (ExtractOffset != 0 || SegSize != NarrowSize) {
-        // A genuine extract is needed.
-        SegReg = MRI.createGenericVirtualRegister(LLT::scalar(SegSize));
-        MIRBuilder.buildExtract(SegReg, SrcRegs[i], ExtractOffset);
-      }
-
-      DstRegs.push_back(SegReg);
-    }
-
-    unsigned DstReg = MI.getOperand(0).getReg();
-    if(MRI.getType(DstReg).isVector())
-      MIRBuilder.buildBuildVector(DstReg, DstRegs);
-    else
-      MIRBuilder.buildMerge(DstReg, DstRegs);
-    MI.eraseFromParent();
-    return Legalized;
-  }
-  case TargetOpcode::G_INSERT: {
+  case TargetOpcode::G_SUB: {
     // FIXME: add support for when SizeOp0 isn't an exact multiple of
     // NarrowSize.
     if (SizeOp0 % NarrowSize != 0)
       return UnableToLegalize;
 
-    int NumParts = SizeOp0 / NarrowSize;
-
-    SmallVector<unsigned, 2> SrcRegs, DstRegs;
-    SmallVector<uint64_t, 2> Indexes;
-    extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, SrcRegs);
+    int NumParts = SizeOp0 / NarrowTy.getSizeInBits();
 
-    unsigned OpReg = MI.getOperand(2).getReg();
-    uint64_t OpStart = MI.getOperand(3).getImm();
-    uint64_t OpSize = MRI.getType(OpReg).getSizeInBits();
-    for (int i = 0; i < NumParts; ++i) {
-      unsigned DstStart = i * NarrowSize;
-
-      if (DstStart + NarrowSize <= OpStart || DstStart >= OpStart + OpSize) {
-        // No part of the insert affects this subregister, forward the original.
-        DstRegs.push_back(SrcRegs[i]);
-        continue;
-      } else if (DstStart == OpStart && NarrowTy == MRI.getType(OpReg)) {
-        // The entire subregister is defined by this insert, forward the new
-        // value.
-        DstRegs.push_back(OpReg);
-        continue;
-      }
+    SmallVector<unsigned, 2> Src1Regs, Src2Regs, DstRegs;
+    extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, Src1Regs);
+    extractParts(MI.getOperand(2).getReg(), NarrowTy, NumParts, Src2Regs);
 
-      // OpSegStart is where this destination segment would start in OpReg if it
-      // extended infinitely in both directions.
-      int64_t ExtractOffset, InsertOffset;
-      uint64_t SegSize;
-      if (OpStart < DstStart) {
-        InsertOffset = 0;
-        ExtractOffset = DstStart - OpStart;
-        SegSize = std::min(NarrowSize, OpStart + OpSize - DstStart);
-      } else {
-        InsertOffset = OpStart - DstStart;
-        ExtractOffset = 0;
-        SegSize =
-            std::min(NarrowSize - InsertOffset, OpStart + OpSize - DstStart);
-      }
+    unsigned DstReg = MRI.createGenericVirtualRegister(NarrowTy);
+    unsigned BorrowOut = MRI.createGenericVirtualRegister(LLT::scalar(1));
+    MIRBuilder.buildInstr(TargetOpcode::G_USUBO, {DstReg, BorrowOut},
+                          {Src1Regs[0], Src2Regs[0]});
+    DstRegs.push_back(DstReg);
+    unsigned BorrowIn = BorrowOut;
+    for (int i = 1; i < NumParts; ++i) {
+      DstReg = MRI.createGenericVirtualRegister(NarrowTy);
+      BorrowOut = MRI.createGenericVirtualRegister(LLT::scalar(1));
 
-      unsigned SegReg = OpReg;
-      if (ExtractOffset != 0 || SegSize != OpSize) {
-        // A genuine extract is needed.
-        SegReg = MRI.createGenericVirtualRegister(LLT::scalar(SegSize));
-        MIRBuilder.buildExtract(SegReg, OpReg, ExtractOffset);
-      }
+      MIRBuilder.buildInstr(TargetOpcode::G_USUBE, {DstReg, BorrowOut},
+                            {Src1Regs[i], Src2Regs[i], BorrowIn});
 
-      unsigned DstReg = MRI.createGenericVirtualRegister(NarrowTy);
-      MIRBuilder.buildInsert(DstReg, SrcRegs[i], SegReg, InsertOffset);
       DstRegs.push_back(DstReg);
+      BorrowIn = BorrowOut;
     }
-
-    assert(DstRegs.size() == (unsigned)NumParts && "not all parts covered");
-    unsigned DstReg = MI.getOperand(0).getReg();
-    if(MRI.getType(DstReg).isVector())
-      MIRBuilder.buildBuildVector(DstReg, DstRegs);
-    else
-      MIRBuilder.buildMerge(DstReg, DstRegs);
+    MIRBuilder.buildMerge(MI.getOperand(0).getReg(), DstRegs);
     MI.eraseFromParent();
     return Legalized;
   }
+  case TargetOpcode::G_MUL:
+    return narrowScalarMul(MI, TypeIdx, NarrowTy);
+  case TargetOpcode::G_EXTRACT:
+    return narrowScalarExtract(MI, TypeIdx, NarrowTy);
+  case TargetOpcode::G_INSERT:
+    return narrowScalarInsert(MI, TypeIdx, NarrowTy);
   case TargetOpcode::G_LOAD: {
-    // FIXME: add support for when SizeOp0 isn't an exact multiple of
-    // NarrowSize.
-    if (SizeOp0 % NarrowSize != 0)
-      return UnableToLegalize;
-
     const auto &MMO = **MI.memoperands_begin();
-    // This implementation doesn't work for atomics. Give up instead of doing
-    // something invalid.
-    if (MMO.getOrdering() != AtomicOrdering::NotAtomic ||
-        MMO.getFailureOrdering() != AtomicOrdering::NotAtomic)
+    unsigned DstReg = MI.getOperand(0).getReg();
+    LLT DstTy = MRI.getType(DstReg);
+    if (DstTy.isVector())
       return UnableToLegalize;
 
-    int NumParts = SizeOp0 / NarrowSize;
-    LLT OffsetTy = LLT::scalar(
-        MRI.getType(MI.getOperand(1).getReg()).getScalarSizeInBits());
-
-    SmallVector<unsigned, 2> DstRegs;
-    for (int i = 0; i < NumParts; ++i) {
-      unsigned DstReg = MRI.createGenericVirtualRegister(NarrowTy);
-      unsigned SrcReg = 0;
-      unsigned Adjustment = i * NarrowSize / 8;
-      unsigned Alignment = MinAlign(MMO.getAlignment(), Adjustment);
-
-      MachineMemOperand *SplitMMO = MIRBuilder.getMF().getMachineMemOperand(
-          MMO.getPointerInfo().getWithOffset(Adjustment), MMO.getFlags(),
-          NarrowSize / 8, Alignment, MMO.getAAInfo(), MMO.getRanges(),
-          MMO.getSyncScopeID(), MMO.getOrdering(), MMO.getFailureOrdering());
-
-      MIRBuilder.materializeGEP(SrcReg, MI.getOperand(1).getReg(), OffsetTy,
-                                Adjustment);
+    if (8 * MMO.getSize() != DstTy.getSizeInBits()) {
+      unsigned TmpReg = MRI.createGenericVirtualRegister(NarrowTy);
+      auto &MMO = **MI.memoperands_begin();
+      MIRBuilder.buildLoad(TmpReg, MI.getOperand(1).getReg(), MMO);
+      MIRBuilder.buildAnyExt(DstReg, TmpReg);
+      MI.eraseFromParent();
+      return Legalized;
+    }
 
-      MIRBuilder.buildLoad(DstReg, SrcReg, *SplitMMO);
+    return reduceLoadStoreWidth(MI, TypeIdx, NarrowTy);
+  }
+  case TargetOpcode::G_ZEXTLOAD:
+  case TargetOpcode::G_SEXTLOAD: {
+    bool ZExt = MI.getOpcode() == TargetOpcode::G_ZEXTLOAD;
+    unsigned DstReg = MI.getOperand(0).getReg();
+    unsigned PtrReg = MI.getOperand(1).getReg();
 
-      DstRegs.push_back(DstReg);
+    unsigned TmpReg = MRI.createGenericVirtualRegister(NarrowTy);
+    auto &MMO = **MI.memoperands_begin();
+    if (MMO.getSize() * 8 == NarrowSize) {
+      MIRBuilder.buildLoad(TmpReg, PtrReg, MMO);
+    } else {
+      unsigned ExtLoad = ZExt ? TargetOpcode::G_ZEXTLOAD
+        : TargetOpcode::G_SEXTLOAD;
+      MIRBuilder.buildInstr(ExtLoad)
+        .addDef(TmpReg)
+        .addUse(PtrReg)
+        .addMemOperand(&MMO);
     }
-    unsigned DstReg = MI.getOperand(0).getReg();
-    if(MRI.getType(DstReg).isVector())
-      MIRBuilder.buildBuildVector(DstReg, DstRegs);
+
+    if (ZExt)
+      MIRBuilder.buildZExt(DstReg, TmpReg);
     else
-      MIRBuilder.buildMerge(DstReg, DstRegs);
+      MIRBuilder.buildSExt(DstReg, TmpReg);
+
     MI.eraseFromParent();
     return Legalized;
   }
   case TargetOpcode::G_STORE: {
-    // FIXME: add support for when SizeOp0 isn't an exact multiple of
-    // NarrowSize.
-    if (SizeOp0 % NarrowSize != 0)
-      return UnableToLegalize;
-
     const auto &MMO = **MI.memoperands_begin();
-    // This implementation doesn't work for atomics. Give up instead of doing
-    // something invalid.
-    if (MMO.getOrdering() != AtomicOrdering::NotAtomic ||
-        MMO.getFailureOrdering() != AtomicOrdering::NotAtomic)
+
+    unsigned SrcReg = MI.getOperand(0).getReg();
+    LLT SrcTy = MRI.getType(SrcReg);
+    if (SrcTy.isVector())
       return UnableToLegalize;
 
     int NumParts = SizeOp0 / NarrowSize;
-    LLT OffsetTy = LLT::scalar(
-        MRI.getType(MI.getOperand(1).getReg()).getScalarSizeInBits());
-
-    SmallVector<unsigned, 2> SrcRegs;
-    extractParts(MI.getOperand(0).getReg(), NarrowTy, NumParts, SrcRegs);
-
-    for (int i = 0; i < NumParts; ++i) {
-      unsigned DstReg = 0;
-      unsigned Adjustment = i * NarrowSize / 8;
-      unsigned Alignment = MinAlign(MMO.getAlignment(), Adjustment);
-
-      MachineMemOperand *SplitMMO = MIRBuilder.getMF().getMachineMemOperand(
-          MMO.getPointerInfo().getWithOffset(Adjustment), MMO.getFlags(),
-          NarrowSize / 8, Alignment, MMO.getAAInfo(), MMO.getRanges(),
-          MMO.getSyncScopeID(), MMO.getOrdering(), MMO.getFailureOrdering());
-
-      MIRBuilder.materializeGEP(DstReg, MI.getOperand(1).getReg(), OffsetTy,
-                                Adjustment);
+    unsigned HandledSize = NumParts * NarrowTy.getSizeInBits();
+    unsigned LeftoverBits = SrcTy.getSizeInBits() - HandledSize;
+    if (SrcTy.isVector() && LeftoverBits != 0)
+      return UnableToLegalize;
 
-      MIRBuilder.buildStore(SrcRegs[i], DstReg, *SplitMMO);
+    if (8 * MMO.getSize() != SrcTy.getSizeInBits()) {
+      unsigned TmpReg = MRI.createGenericVirtualRegister(NarrowTy);
+      auto &MMO = **MI.memoperands_begin();
+      MIRBuilder.buildTrunc(TmpReg, SrcReg);
+      MIRBuilder.buildStore(TmpReg, MI.getOperand(1).getReg(), MMO);
+      MI.eraseFromParent();
+      return Legalized;
     }
-    MI.eraseFromParent();
-    return Legalized;
+
+    return reduceLoadStoreWidth(MI, 0, NarrowTy);
   }
   case TargetOpcode::G_CONSTANT: {
     // FIXME: add support for when SizeOp0 isn't an exact multiple of
@@ -578,6 +632,8 @@ LegalizerHelper::LegalizeResult LegalizerHelper::narrowScalar(MachineInstr &MI,
     MI.eraseFromParent();
     return Legalized;
   }
+  case TargetOpcode::G_SELECT:
+    return narrowScalarSelect(MI, TypeIdx, NarrowTy);
   case TargetOpcode::G_AND:
   case TargetOpcode::G_OR:
   case TargetOpcode::G_XOR: {
@@ -631,6 +687,38 @@ LegalizerHelper::LegalizeResult LegalizerHelper::narrowScalar(MachineInstr &MI,
     MI.eraseFromParent();
     return Legalized;
   }
+  case TargetOpcode::G_SHL:
+  case TargetOpcode::G_LSHR:
+  case TargetOpcode::G_ASHR:
+    return narrowScalarShift(MI, TypeIdx, NarrowTy);
+  case TargetOpcode::G_CTLZ:
+  case TargetOpcode::G_CTLZ_ZERO_UNDEF:
+  case TargetOpcode::G_CTTZ:
+  case TargetOpcode::G_CTTZ_ZERO_UNDEF:
+  case TargetOpcode::G_CTPOP:
+    if (TypeIdx != 0)
+      return UnableToLegalize; // TODO
+
+    Observer.changingInstr(MI);
+    narrowScalarDst(MI, NarrowTy, 0, TargetOpcode::G_ZEXT);
+    Observer.changedInstr(MI);
+    return Legalized;
+  case TargetOpcode::G_INTTOPTR:
+    if (TypeIdx != 1)
+      return UnableToLegalize;
+
+    Observer.changingInstr(MI);
+    narrowScalarSrc(MI, NarrowTy, 1);
+    Observer.changedInstr(MI);
+    return Legalized;
+  case TargetOpcode::G_PTRTOINT:
+    if (TypeIdx != 0)
+      return UnableToLegalize;
+
+    Observer.changingInstr(MI);
+    narrowScalarDst(MI, NarrowTy, 0, TargetOpcode::G_ZEXT);
+    Observer.changedInstr(MI);
+    return Legalized;
   }
 }
 
@@ -641,6 +729,14 @@ void LegalizerHelper::widenScalarSrc(MachineInstr &MI, LLT WideTy,
   MO.setReg(ExtB->getOperand(0).getReg());
 }
 
+void LegalizerHelper::narrowScalarSrc(MachineInstr &MI, LLT NarrowTy,
+                                      unsigned OpIdx) {
+  MachineOperand &MO = MI.getOperand(OpIdx);
+  auto ExtB = MIRBuilder.buildInstr(TargetOpcode::G_TRUNC, {NarrowTy},
+                                    {MO.getReg()});
+  MO.setReg(ExtB->getOperand(0).getReg());
+}
+
 void LegalizerHelper::widenScalarDst(MachineInstr &MI, LLT WideTy,
                                      unsigned OpIdx, unsigned TruncOpcode) {
   MachineOperand &MO = MI.getOperand(OpIdx);
@@ -650,6 +746,145 @@ void LegalizerHelper::widenScalarDst(MachineInstr &MI, LLT WideTy,
   MO.setReg(DstExt);
 }
 
+void LegalizerHelper::narrowScalarDst(MachineInstr &MI, LLT NarrowTy,
+                                      unsigned OpIdx, unsigned ExtOpcode) {
+  MachineOperand &MO = MI.getOperand(OpIdx);
+  unsigned DstTrunc = MRI.createGenericVirtualRegister(NarrowTy);
+  MIRBuilder.setInsertPt(MIRBuilder.getMBB(), ++MIRBuilder.getInsertPt());
+  MIRBuilder.buildInstr(ExtOpcode, {MO.getReg()}, {DstTrunc});
+  MO.setReg(DstTrunc);
+}
+
+void LegalizerHelper::moreElementsVectorDst(MachineInstr &MI, LLT WideTy,
+                                            unsigned OpIdx) {
+  MachineOperand &MO = MI.getOperand(OpIdx);
+  unsigned DstExt = MRI.createGenericVirtualRegister(WideTy);
+  MIRBuilder.setInsertPt(MIRBuilder.getMBB(), ++MIRBuilder.getInsertPt());
+  MIRBuilder.buildExtract(MO.getReg(), DstExt, 0);
+  MO.setReg(DstExt);
+}
+
+LegalizerHelper::LegalizeResult
+LegalizerHelper::widenScalarMergeValues(MachineInstr &MI, unsigned TypeIdx,
+                                        LLT WideTy) {
+  if (TypeIdx != 1)
+    return UnableToLegalize;
+
+  unsigned DstReg = MI.getOperand(0).getReg();
+  LLT DstTy = MRI.getType(DstReg);
+  if (!DstTy.isScalar())
+    return UnableToLegalize;
+
+  unsigned NumOps = MI.getNumOperands();
+  unsigned NumSrc = MI.getNumOperands() - 1;
+  unsigned PartSize = DstTy.getSizeInBits() / NumSrc;
+
+  unsigned Src1 = MI.getOperand(1).getReg();
+  unsigned ResultReg = MIRBuilder.buildZExt(DstTy, Src1)->getOperand(0).getReg();
+
+  for (unsigned I = 2; I != NumOps; ++I) {
+    const unsigned Offset = (I - 1) * PartSize;
+
+    unsigned SrcReg = MI.getOperand(I).getReg();
+    assert(MRI.getType(SrcReg) == LLT::scalar(PartSize));
+
+    auto ZextInput = MIRBuilder.buildZExt(DstTy, SrcReg);
+
+    unsigned NextResult = I + 1 == NumOps ? DstReg :
+      MRI.createGenericVirtualRegister(DstTy);
+
+    auto ShiftAmt = MIRBuilder.buildConstant(DstTy, Offset);
+    auto Shl = MIRBuilder.buildShl(DstTy, ZextInput, ShiftAmt);
+    MIRBuilder.buildOr(NextResult, ResultReg, Shl);
+    ResultReg = NextResult;
+  }
+
+  MI.eraseFromParent();
+  return Legalized;
+}
+
+LegalizerHelper::LegalizeResult
+LegalizerHelper::widenScalarUnmergeValues(MachineInstr &MI, unsigned TypeIdx,
+                                          LLT WideTy) {
+  if (TypeIdx != 0)
+    return UnableToLegalize;
+
+  unsigned NumDst = MI.getNumOperands() - 1;
+  unsigned SrcReg = MI.getOperand(NumDst).getReg();
+  LLT SrcTy = MRI.getType(SrcReg);
+  if (!SrcTy.isScalar())
+    return UnableToLegalize;
+
+  unsigned Dst0Reg = MI.getOperand(0).getReg();
+  LLT DstTy = MRI.getType(Dst0Reg);
+  if (!DstTy.isScalar())
+    return UnableToLegalize;
+
+  unsigned NewSrcSize = NumDst * WideTy.getSizeInBits();
+  LLT NewSrcTy = LLT::scalar(NewSrcSize);
+  unsigned SizeDiff = WideTy.getSizeInBits() - DstTy.getSizeInBits();
+
+  auto WideSrc = MIRBuilder.buildZExt(NewSrcTy, SrcReg);
+
+  for (unsigned I = 1; I != NumDst; ++I) {
+    auto ShiftAmt = MIRBuilder.buildConstant(NewSrcTy, SizeDiff * I);
+    auto Shl = MIRBuilder.buildShl(NewSrcTy, WideSrc, ShiftAmt);
+    WideSrc = MIRBuilder.buildOr(NewSrcTy, WideSrc, Shl);
+  }
+
+  Observer.changingInstr(MI);
+
+  MI.getOperand(NumDst).setReg(WideSrc->getOperand(0).getReg());
+  for (unsigned I = 0; I != NumDst; ++I)
+    widenScalarDst(MI, WideTy, I);
+
+  Observer.changedInstr(MI);
+
+  return Legalized;
+}
+
+LegalizerHelper::LegalizeResult
+LegalizerHelper::widenScalarExtract(MachineInstr &MI, unsigned TypeIdx,
+                                    LLT WideTy) {
+  if (TypeIdx != 1)
+    return UnableToLegalize;
+
+  unsigned SrcReg = MI.getOperand(1).getReg();
+  LLT SrcTy = MRI.getType(SrcReg);
+  if (!SrcTy.isVector())
+    return UnableToLegalize;
+
+  unsigned DstReg = MI.getOperand(0).getReg();
+  LLT DstTy = MRI.getType(DstReg);
+  if (DstTy != SrcTy.getElementType())
+    return UnableToLegalize;
+
+  unsigned Offset = MI.getOperand(2).getImm();
+  if (Offset % SrcTy.getScalarSizeInBits() != 0)
+    return UnableToLegalize;
+
+  Observer.changingInstr(MI);
+  widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT);
+
+  MI.getOperand(2).setImm((WideTy.getSizeInBits() / SrcTy.getSizeInBits()) *
+                          Offset);
+  widenScalarDst(MI, WideTy.getScalarType(), 0);
+  Observer.changedInstr(MI);
+  return Legalized;
+}
+
+LegalizerHelper::LegalizeResult
+LegalizerHelper::widenScalarInsert(MachineInstr &MI, unsigned TypeIdx,
+                                   LLT WideTy) {
+  if (TypeIdx != 0)
+    return UnableToLegalize;
+  Observer.changingInstr(MI);
+  widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT);
+  widenScalarDst(MI, WideTy);
+  Observer.changedInstr(MI);
+  return Legalized;
+}
+
 LegalizerHelper::LegalizeResult
 LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) {
   MIRBuilder.setInstr(MI);
@@ -657,6 +892,14 @@ LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) {
   switch (MI.getOpcode()) {
   default:
     return UnableToLegalize;
+  case TargetOpcode::G_EXTRACT:
+    return widenScalarExtract(MI, TypeIdx, WideTy);
+  case TargetOpcode::G_INSERT:
+    return widenScalarInsert(MI, TypeIdx, WideTy);
+  case TargetOpcode::G_MERGE_VALUES:
+    return widenScalarMergeValues(MI, TypeIdx, WideTy);
+  case TargetOpcode::G_UNMERGE_VALUES:
+    return widenScalarUnmergeValues(MI, TypeIdx, WideTy);
   case TargetOpcode::G_UADDO:
   case TargetOpcode::G_USUBO: {
     if (TypeIdx == 1)
@@ -688,19 +931,28 @@ LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) {
   case TargetOpcode::G_CTLZ:
   case TargetOpcode::G_CTLZ_ZERO_UNDEF:
   case TargetOpcode::G_CTPOP: {
+    if (TypeIdx == 0) {
+      Observer.changingInstr(MI);
+      widenScalarDst(MI, WideTy, 0);
+      Observer.changedInstr(MI);
+      return Legalized;
+    }
+
+    unsigned SrcReg = MI.getOperand(1).getReg();
+
     // First ZEXT the input.
-    auto MIBSrc = MIRBuilder.buildZExt(WideTy, MI.getOperand(1).getReg());
-    LLT CurTy = MRI.getType(MI.getOperand(0).getReg());
+    auto MIBSrc = MIRBuilder.buildZExt(WideTy, SrcReg);
+    LLT CurTy = MRI.getType(SrcReg);
     if (MI.getOpcode() == TargetOpcode::G_CTTZ) {
       // The count is the same in the larger type except if the original
       // value was zero.  This can be handled by setting the bit just off
       // the top of the original type.
       auto TopBit =
           APInt::getOneBitSet(WideTy.getSizeInBits(), CurTy.getSizeInBits());
-      MIBSrc = MIRBuilder.buildInstr(
-          TargetOpcode::G_OR, {WideTy},
-          {MIBSrc, MIRBuilder.buildConstant(WideTy, TopBit.getSExtValue())});
+      MIBSrc = MIRBuilder.buildOr(
+        WideTy, MIBSrc, MIRBuilder.buildConstant(WideTy, TopBit));
     }
+
     // Perform the operation at the larger size.
     auto MIBNewOp = MIRBuilder.buildInstr(MI.getOpcode(), {WideTy}, {MIBSrc});
     // This is already the correct result for CTPOP and CTTZs
@@ -712,22 +964,43 @@ LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) {
           TargetOpcode::G_SUB, {WideTy},
           {MIBNewOp, MIRBuilder.buildConstant(WideTy, SizeDiff)});
     }
-    auto &TII = *MI.getMF()->getSubtarget().getInstrInfo();
-    // Make the original instruction a trunc now, and update its source.
+
+    MIRBuilder.buildZExtOrTrunc(MI.getOperand(0), MIBNewOp);
+    MI.eraseFromParent();
+    return Legalized;
+  }
+  case TargetOpcode::G_BSWAP: {
     Observer.changingInstr(MI);
-    MI.setDesc(TII.get(TargetOpcode::G_TRUNC));
-    MI.getOperand(1).setReg(MIBNewOp->getOperand(0).getReg());
+    unsigned DstReg = MI.getOperand(0).getReg();
+
+    unsigned ShrReg = MRI.createGenericVirtualRegister(WideTy);
+    unsigned DstExt = MRI.createGenericVirtualRegister(WideTy);
+    unsigned ShiftAmtReg = MRI.createGenericVirtualRegister(WideTy);
+    widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT);
+
+    MI.getOperand(0).setReg(DstExt);
+
+    MIRBuilder.setInsertPt(MIRBuilder.getMBB(), ++MIRBuilder.getInsertPt());
+
+    LLT Ty = MRI.getType(DstReg);
+    unsigned DiffBits = WideTy.getScalarSizeInBits() - Ty.getScalarSizeInBits();
+    MIRBuilder.buildConstant(ShiftAmtReg, DiffBits);
+    MIRBuilder.buildInstr(TargetOpcode::G_LSHR)
+      .addDef(ShrReg)
+      .addUse(DstExt)
+      .addUse(ShiftAmtReg);
+
+    MIRBuilder.buildTrunc(DstReg, ShrReg);
     Observer.changedInstr(MI);
     return Legalized;
   }
-
   case TargetOpcode::G_ADD:
   case TargetOpcode::G_AND:
   case TargetOpcode::G_MUL:
   case TargetOpcode::G_OR:
   case TargetOpcode::G_XOR:
   case TargetOpcode::G_SUB:
-    // Perform operation at larger width (any extension is fine here, high bits
+    // Perform operation at larger width (any extension is fines here, high bits
     // don't affect the result) and then truncate the result back to the
     // original type.
     Observer.changingInstr(MI);
@@ -738,12 +1011,18 @@ LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) {
     return Legalized;
 
   case TargetOpcode::G_SHL:
-    Observer.changingInstr(MI);
-    widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT);
-    // The "number of bits to shift" operand must preserve its value as an
-    // unsigned integer:
-    widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ZEXT);
-    widenScalarDst(MI, WideTy);
+      Observer.changingInstr(MI);
+
+    if (TypeIdx == 0) {
+      widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT);
+      widenScalarDst(MI, WideTy);
+    } else {
+      assert(TypeIdx == 1);
+      // The "number of bits to shift" operand must preserve its value as an
+      // unsigned integer:
+      widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ZEXT);
+    }
+
     Observer.changedInstr(MI);
     return Legalized;
 
@@ -757,18 +1036,26 @@ LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) {
     return Legalized;
 
   case TargetOpcode::G_ASHR:
+  case TargetOpcode::G_LSHR:
     Observer.changingInstr(MI);
-    widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_SEXT);
-    // The "number of bits to shift" operand must preserve its value as an
-    // unsigned integer:
-    widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ZEXT);
-    widenScalarDst(MI, WideTy);
-    Observer.changedInstr(MI);
-    return Legalized;
 
+    if (TypeIdx == 0) {
+      unsigned CvtOp = MI.getOpcode() == TargetOpcode::G_ASHR ?
+        TargetOpcode::G_SEXT : TargetOpcode::G_ZEXT;
+
+      widenScalarSrc(MI, WideTy, 1, CvtOp);
+      widenScalarDst(MI, WideTy);
+    } else {
+      assert(TypeIdx == 1);
+      // The "number of bits to shift" operand must preserve its value as an
+      // unsigned integer:
+      widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ZEXT);
+    }
+
+    Observer.changedInstr(MI);
+    return Legalized;
   case TargetOpcode::G_UDIV:
   case TargetOpcode::G_UREM:
-  case TargetOpcode::G_LSHR:
     Observer.changingInstr(MI);
     widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ZEXT);
     widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ZEXT);
@@ -777,15 +1064,19 @@ LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) {
     return Legalized;
 
   case TargetOpcode::G_SELECT:
-    if (TypeIdx != 0)
-      return UnableToLegalize;
-    // Perform operation at larger width (any extension is fine here, high bits
-    // don't affect the result) and then truncate the result back to the
-    // original type.
     Observer.changingInstr(MI);
-    widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ANYEXT);
-    widenScalarSrc(MI, WideTy, 3, TargetOpcode::G_ANYEXT);
-    widenScalarDst(MI, WideTy);
+    if (TypeIdx == 0) {
+      // Perform operation at larger width (any extension is fine here, high
+      // bits don't affect the result) and then truncate the result back to the
+      // original type.
+      widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ANYEXT);
+      widenScalarSrc(MI, WideTy, 3, TargetOpcode::G_ANYEXT);
+      widenScalarDst(MI, WideTy);
+    } else {
+      bool IsVec = MRI.getType(MI.getOperand(1).getReg()).isVector();
+      // Explicit extension is required here since high bits affect the result.
+      widenScalarSrc(MI, WideTy, 1, MIRBuilder.getBoolExtOp(IsVec, false));
+    }
     Observer.changedInstr(MI);
     return Legalized;
 
@@ -814,23 +1105,7 @@ LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) {
     Observer.changedInstr(MI);
     return Legalized;
 
-  case TargetOpcode::G_INSERT:
-    if (TypeIdx != 0)
-      return UnableToLegalize;
-    Observer.changingInstr(MI);
-    widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT);
-    widenScalarDst(MI, WideTy);
-    Observer.changedInstr(MI);
-    return Legalized;
-
   case TargetOpcode::G_LOAD:
-    // For some types like i24, we might try to widen to i32. To properly handle
-    // this we should be using a dedicated extending load, until then avoid
-    // trying to legalize.
-    if (alignTo(MRI.getType(MI.getOperand(0).getReg()).getSizeInBits(), 8) !=
-        WideTy.getSizeInBits())
-      return UnableToLegalize;
-    LLVM_FALLTHROUGH;
   case TargetOpcode::G_SEXTLOAD:
   case TargetOpcode::G_ZEXTLOAD:
     Observer.changingInstr(MI);
@@ -839,12 +1114,19 @@ LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) {
     return Legalized;
 
   case TargetOpcode::G_STORE: {
-    if (MRI.getType(MI.getOperand(0).getReg()) != LLT::scalar(1) ||
-        WideTy != LLT::scalar(8))
+    if (TypeIdx != 0)
+      return UnableToLegalize;
+
+    LLT Ty = MRI.getType(MI.getOperand(0).getReg());
+    if (!isPowerOf2_32(Ty.getSizeInBits()))
       return UnableToLegalize;
 
     Observer.changingInstr(MI);
-    widenScalarSrc(MI, WideTy, 0, TargetOpcode::G_ZEXT);
+
+    unsigned ExtType = Ty.getScalarSizeInBits() == 1 ?
+      TargetOpcode::G_ZEXT : TargetOpcode::G_ANYEXT;
+    widenScalarSrc(MI, WideTy, 0, ExtType);
+
     Observer.changedInstr(MI);
     return Legalized;
   }
@@ -866,14 +1148,19 @@ LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) {
     bool LosesInfo;
     switch (WideTy.getSizeInBits()) {
     case 32:
-      Val.convert(APFloat::IEEEsingle(), APFloat::rmTowardZero, &LosesInfo);
+      Val.convert(APFloat::IEEEsingle(), APFloat::rmNearestTiesToEven,
+                  &LosesInfo);
       break;
     case 64:
-      Val.convert(APFloat::IEEEdouble(), APFloat::rmTowardZero, &LosesInfo);
+      Val.convert(APFloat::IEEEdouble(), APFloat::rmNearestTiesToEven,
+                  &LosesInfo);
       break;
     default:
-      llvm_unreachable("Unhandled fp widen type");
+      return UnableToLegalize;
     }
+
+    assert(!LosesInfo && "extend should always be lossless");
+
     Observer.changingInstr(MI);
     SrcMO.setFPImm(ConstantFP::get(Ctx, Val));
 
@@ -881,6 +1168,12 @@ LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) {
     Observer.changedInstr(MI);
     return Legalized;
   }
+  case TargetOpcode::G_IMPLICIT_DEF: {
+    Observer.changingInstr(MI);
+    widenScalarDst(MI, WideTy);
+    Observer.changedInstr(MI);
+    return Legalized;
+  }
   case TargetOpcode::G_BRCOND:
     Observer.changingInstr(MI);
     widenScalarSrc(MI, WideTy, 0, TargetOpcode::G_ANYEXT);
@@ -936,7 +1229,21 @@ LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) {
     Observer.changedInstr(MI);
     return Legalized;
   }
-  case TargetOpcode::G_EXTRACT_VECTOR_ELT:
+  case TargetOpcode::G_EXTRACT_VECTOR_ELT: {
+    if (TypeIdx == 0) {
+      unsigned VecReg = MI.getOperand(1).getReg();
+      LLT VecTy = MRI.getType(VecReg);
+      Observer.changingInstr(MI);
+
+      widenScalarSrc(MI, LLT::vector(VecTy.getNumElements(),
+                                     WideTy.getSizeInBits()),
+                     1, TargetOpcode::G_SEXT);
+
+      widenScalarDst(MI, WideTy, 0);
+      Observer.changedInstr(MI);
+      return Legalized;
+    }
+
     if (TypeIdx != 2)
       return UnableToLegalize;
     Observer.changingInstr(MI);
@@ -944,6 +1251,50 @@ LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) {
     Observer.changedInstr(MI);
     return Legalized;
   }
+  case TargetOpcode::G_FADD:
+  case TargetOpcode::G_FMUL:
+  case TargetOpcode::G_FSUB:
+  case TargetOpcode::G_FMA:
+  case TargetOpcode::G_FNEG:
+  case TargetOpcode::G_FABS:
+  case TargetOpcode::G_FCANONICALIZE:
+  case TargetOpcode::G_FDIV:
+  case TargetOpcode::G_FREM:
+  case TargetOpcode::G_FCEIL:
+  case TargetOpcode::G_FFLOOR:
+  case TargetOpcode::G_FCOS:
+  case TargetOpcode::G_FSIN:
+  case TargetOpcode::G_FLOG10:
+  case TargetOpcode::G_FLOG:
+  case TargetOpcode::G_FLOG2:
+  case TargetOpcode::G_FSQRT:
+  case TargetOpcode::G_FEXP:
+    assert(TypeIdx == 0);
+    Observer.changingInstr(MI);
+
+    for (unsigned I = 1, E = MI.getNumOperands(); I != E; ++I)
+      widenScalarSrc(MI, WideTy, I, TargetOpcode::G_FPEXT);
+
+    widenScalarDst(MI, WideTy, 0, TargetOpcode::G_FPTRUNC);
+    Observer.changedInstr(MI);
+    return Legalized;
+  case TargetOpcode::G_INTTOPTR:
+    if (TypeIdx != 1)
+      return UnableToLegalize;
+
+    Observer.changingInstr(MI);
+    widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ZEXT);
+    Observer.changedInstr(MI);
+    return Legalized;
+  case TargetOpcode::G_PTRTOINT:
+    if (TypeIdx != 0)
+      return UnableToLegalize;
+
+    Observer.changingInstr(MI);
+    widenScalarDst(MI, WideTy, 0);
+    Observer.changedInstr(MI);
+    return Legalized;
+  }
 }
 
 LegalizerHelper::LegalizeResult
@@ -1140,87 +1491,1066 @@ LegalizerHelper::lower(MachineInstr &MI, unsigned TypeIdx, LLT Ty) {
     MI.eraseFromParent();
     return Legalized;
   }
+  case G_USUBO: {
+    unsigned Res = MI.getOperand(0).getReg();
+    unsigned BorrowOut = MI.getOperand(1).getReg();
+    unsigned LHS = MI.getOperand(2).getReg();
+    unsigned RHS = MI.getOperand(3).getReg();
+
+    MIRBuilder.buildSub(Res, LHS, RHS);
+    MIRBuilder.buildICmp(CmpInst::ICMP_ULT, BorrowOut, LHS, RHS);
+
+    MI.eraseFromParent();
+    return Legalized;
   }
+  case G_USUBE: {
+    unsigned Res = MI.getOperand(0).getReg();
+    unsigned BorrowOut = MI.getOperand(1).getReg();
+    unsigned LHS = MI.getOperand(2).getReg();
+    unsigned RHS = MI.getOperand(3).getReg();
+    unsigned BorrowIn = MI.getOperand(4).getReg();
+
+    unsigned TmpRes = MRI.createGenericVirtualRegister(Ty);
+    unsigned ZExtBorrowIn = MRI.createGenericVirtualRegister(Ty);
+    unsigned LHS_EQ_RHS = MRI.createGenericVirtualRegister(LLT::scalar(1));
+    unsigned LHS_ULT_RHS = MRI.createGenericVirtualRegister(LLT::scalar(1));
+
+    MIRBuilder.buildSub(TmpRes, LHS, RHS);
+    MIRBuilder.buildZExt(ZExtBorrowIn, BorrowIn);
+    MIRBuilder.buildSub(Res, TmpRes, ZExtBorrowIn);
+    MIRBuilder.buildICmp(CmpInst::ICMP_EQ, LHS_EQ_RHS, LHS, RHS);
+    MIRBuilder.buildICmp(CmpInst::ICMP_ULT, LHS_ULT_RHS, LHS, RHS);
+    MIRBuilder.buildSelect(BorrowOut, LHS_EQ_RHS, BorrowIn, LHS_ULT_RHS);
+
+    MI.eraseFromParent();
+    return Legalized;
+  }
+  }
+}
+
+LegalizerHelper::LegalizeResult LegalizerHelper::fewerElementsVectorImplicitDef(
+    MachineInstr &MI, unsigned TypeIdx, LLT NarrowTy) {
+  SmallVector<unsigned, 2> DstRegs;
+
+  unsigned NarrowSize = NarrowTy.getSizeInBits();
+  unsigned DstReg = MI.getOperand(0).getReg();
+  unsigned Size = MRI.getType(DstReg).getSizeInBits();
+  int NumParts = Size / NarrowSize;
+  // FIXME: Don't know how to handle the situation where the small vectors
+  // aren't all the same size yet.
+  if (Size % NarrowSize != 0)
+    return UnableToLegalize;
+
+  for (int i = 0; i < NumParts; ++i) {
+    unsigned TmpReg = MRI.createGenericVirtualRegister(NarrowTy);
+    MIRBuilder.buildUndef(TmpReg);
+    DstRegs.push_back(TmpReg);
+  }
+
+  if (NarrowTy.isVector())
+    MIRBuilder.buildConcatVectors(DstReg, DstRegs);
+  else
+    MIRBuilder.buildBuildVector(DstReg, DstRegs);
+
+  MI.eraseFromParent();
+  return Legalized;
 }
 
 LegalizerHelper::LegalizeResult
-LegalizerHelper::fewerElementsVector(MachineInstr &MI, unsigned TypeIdx,
-                                     LLT NarrowTy) {
-  // FIXME: Don't know how to handle secondary types yet.
+LegalizerHelper::fewerElementsVectorBasic(MachineInstr &MI, unsigned TypeIdx,
+                                          LLT NarrowTy) {
+  const unsigned Opc = MI.getOpcode();
+  const unsigned NumOps = MI.getNumOperands() - 1;
+  const unsigned NarrowSize = NarrowTy.getSizeInBits();
+  const unsigned DstReg = MI.getOperand(0).getReg();
+  const unsigned Flags = MI.getFlags();
+  const LLT DstTy = MRI.getType(DstReg);
+  const unsigned Size = DstTy.getSizeInBits();
+  const int NumParts = Size / NarrowSize;
+  const LLT EltTy = DstTy.getElementType();
+  const unsigned EltSize = EltTy.getSizeInBits();
+  const unsigned BitsForNumParts = NarrowSize * NumParts;
+
+  // Check if we have any leftovers. If we do, then only handle the case where
+  // the leftover is one element.
+  if (BitsForNumParts != Size && BitsForNumParts + EltSize != Size)
+    return UnableToLegalize;
+
+  if (BitsForNumParts != Size) {
+    unsigned AccumDstReg = MRI.createGenericVirtualRegister(DstTy);
+    MIRBuilder.buildUndef(AccumDstReg);
+
+    // Handle the pieces which evenly divide into the requested type with
+    // extract/op/insert sequence.
+    for (unsigned Offset = 0; Offset < BitsForNumParts; Offset += NarrowSize) {
+      SmallVector<SrcOp, 4> SrcOps;
+      for (unsigned I = 1, E = MI.getNumOperands(); I != E; ++I) {
+        unsigned PartOpReg = MRI.createGenericVirtualRegister(NarrowTy);
+        MIRBuilder.buildExtract(PartOpReg, MI.getOperand(I).getReg(), Offset);
+        SrcOps.push_back(PartOpReg);
+      }
+
+      unsigned PartDstReg = MRI.createGenericVirtualRegister(NarrowTy);
+      MIRBuilder.buildInstr(Opc, {PartDstReg}, SrcOps, Flags);
+
+      unsigned PartInsertReg = MRI.createGenericVirtualRegister(DstTy);
+      MIRBuilder.buildInsert(PartInsertReg, AccumDstReg, PartDstReg, Offset);
+      AccumDstReg = PartInsertReg;
+      Offset += NarrowSize;
+    }
+
+    // Handle the remaining element sized leftover piece.
+    SmallVector<SrcOp, 4> SrcOps;
+    for (unsigned I = 1, E = MI.getNumOperands(); I != E; ++I) {
+      unsigned PartOpReg = MRI.createGenericVirtualRegister(EltTy);
+      MIRBuilder.buildExtract(PartOpReg, MI.getOperand(I).getReg(),
+                              BitsForNumParts);
+      SrcOps.push_back(PartOpReg);
+    }
+
+    unsigned PartDstReg = MRI.createGenericVirtualRegister(EltTy);
+    MIRBuilder.buildInstr(Opc, {PartDstReg}, SrcOps, Flags);
+    MIRBuilder.buildInsert(DstReg, AccumDstReg, PartDstReg, BitsForNumParts);
+    MI.eraseFromParent();
+
+    return Legalized;
+  }
+
+  SmallVector<unsigned, 2> DstRegs, Src0Regs, Src1Regs, Src2Regs;
+
+  extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, Src0Regs);
+
+  if (NumOps >= 2)
+    extractParts(MI.getOperand(2).getReg(), NarrowTy, NumParts, Src1Regs);
+
+  if (NumOps >= 3)
+    extractParts(MI.getOperand(3).getReg(), NarrowTy, NumParts, Src2Regs);
+
+  for (int i = 0; i < NumParts; ++i) {
+    unsigned DstReg = MRI.createGenericVirtualRegister(NarrowTy);
+
+    if (NumOps == 1)
+      MIRBuilder.buildInstr(Opc, {DstReg}, {Src0Regs[i]}, Flags);
+    else if (NumOps == 2) {
+      MIRBuilder.buildInstr(Opc, {DstReg}, {Src0Regs[i], Src1Regs[i]}, Flags);
+    } else if (NumOps == 3) {
+      MIRBuilder.buildInstr(Opc, {DstReg},
+                            {Src0Regs[i], Src1Regs[i], Src2Regs[i]}, Flags);
+    }
+
+    DstRegs.push_back(DstReg);
+  }
+
+  if (NarrowTy.isVector())
+    MIRBuilder.buildConcatVectors(DstReg, DstRegs);
+  else
+    MIRBuilder.buildBuildVector(DstReg, DstRegs);
+
+  MI.eraseFromParent();
+  return Legalized;
+}
+
+// Handle splitting vector operations which need to have the same number of
+// elements in each type index, but each type index may have a different element
+// type.
+//
+// e.g.  <4 x s64> = G_SHL <4 x s64>, <4 x s32> ->
+//       <2 x s64> = G_SHL <2 x s64>, <2 x s32>
+//       <2 x s64> = G_SHL <2 x s64>, <2 x s32>
+//
+// Also handles some irregular breakdown cases, e.g.
+// e.g.  <3 x s64> = G_SHL <3 x s64>, <3 x s32> ->
+//       <2 x s64> = G_SHL <2 x s64>, <2 x s32>
+//             s64 = G_SHL s64, s32
+LegalizerHelper::LegalizeResult
+LegalizerHelper::fewerElementsVectorMultiEltType(
+  MachineInstr &MI, unsigned TypeIdx, LLT NarrowTyArg) {
   if (TypeIdx != 0)
     return UnableToLegalize;
 
-  MIRBuilder.setInstr(MI);
-  switch (MI.getOpcode()) {
-  default:
+  const LLT NarrowTy0 = NarrowTyArg;
+  const unsigned NewNumElts =
+      NarrowTy0.isVector() ? NarrowTy0.getNumElements() : 1;
+
+  const unsigned DstReg = MI.getOperand(0).getReg();
+  LLT DstTy = MRI.getType(DstReg);
+  LLT LeftoverTy0;
+
+  // All of the operands need to have the same number of elements, so if we can
+  // determine a type breakdown for the result type, we can for all of the
+  // source types.
+  int NumParts = getNarrowTypeBreakDown(DstTy, NarrowTy0, LeftoverTy0);
+  if (NumParts < 0)
     return UnableToLegalize;
-  case TargetOpcode::G_ADD: {
-    unsigned NarrowSize = NarrowTy.getSizeInBits();
-    unsigned DstReg = MI.getOperand(0).getReg();
-    unsigned Size = MRI.getType(DstReg).getSizeInBits();
-    int NumParts = Size / NarrowSize;
+
+  SmallVector<MachineInstrBuilder, 4> NewInsts;
+
+  SmallVector<unsigned, 4> DstRegs, LeftoverDstRegs;
+  SmallVector<unsigned, 4> PartRegs, LeftoverRegs;
+
+  for (unsigned I = 1, E = MI.getNumOperands(); I != E; ++I) {
+    LLT LeftoverTy;
+    unsigned SrcReg = MI.getOperand(I).getReg();
+    LLT SrcTyI = MRI.getType(SrcReg);
+    LLT NarrowTyI = LLT::scalarOrVector(NewNumElts, SrcTyI.getScalarType());
+    LLT LeftoverTyI;
+
+    // Split this operand into the requested typed registers, and any leftover
+    // required to reproduce the original type.
+    if (!extractParts(SrcReg, SrcTyI, NarrowTyI, LeftoverTyI, PartRegs,
+                      LeftoverRegs))
+      return UnableToLegalize;
+
+    if (I == 1) {
+      // For the first operand, create an instruction for each part and setup
+      // the result.
+      for (unsigned PartReg : PartRegs) {
+        unsigned PartDstReg = MRI.createGenericVirtualRegister(NarrowTy0);
+        NewInsts.push_back(MIRBuilder.buildInstrNoInsert(MI.getOpcode())
+                               .addDef(PartDstReg)
+                               .addUse(PartReg));
+        DstRegs.push_back(PartDstReg);
+      }
+
+      for (unsigned LeftoverReg : LeftoverRegs) {
+        unsigned PartDstReg = MRI.createGenericVirtualRegister(LeftoverTy0);
+        NewInsts.push_back(MIRBuilder.buildInstrNoInsert(MI.getOpcode())
+                               .addDef(PartDstReg)
+                               .addUse(LeftoverReg));
+        LeftoverDstRegs.push_back(PartDstReg);
+      }
+    } else {
+      assert(NewInsts.size() == PartRegs.size() + LeftoverRegs.size());
+
+      // Add the newly created operand splits to the existing instructions. The
+      // odd-sized pieces are ordered after the requested NarrowTyArg sized
+      // pieces.
+      unsigned InstCount = 0;
+      for (unsigned J = 0, JE = PartRegs.size(); J != JE; ++J)
+        NewInsts[InstCount++].addUse(PartRegs[J]);
+      for (unsigned J = 0, JE = LeftoverRegs.size(); J != JE; ++J)
+        NewInsts[InstCount++].addUse(LeftoverRegs[J]);
+    }
+
+    PartRegs.clear();
+    LeftoverRegs.clear();
+  }
+
+  // Insert the newly built operations and rebuild the result register.
+  for (auto &MIB : NewInsts)
+    MIRBuilder.insertInstr(MIB);
+
+  insertParts(DstReg, DstTy, NarrowTy0, DstRegs, LeftoverTy0, LeftoverDstRegs);
+
+  MI.eraseFromParent();
+  return Legalized;
+}
+
+LegalizerHelper::LegalizeResult
+LegalizerHelper::fewerElementsVectorCasts(MachineInstr &MI, unsigned TypeIdx,
+                                          LLT NarrowTy) {
+  if (TypeIdx != 0)
+    return UnableToLegalize;
+
+  unsigned DstReg = MI.getOperand(0).getReg();
+  unsigned SrcReg = MI.getOperand(1).getReg();
+  LLT DstTy = MRI.getType(DstReg);
+  LLT SrcTy = MRI.getType(SrcReg);
+
+  LLT NarrowTy0 = NarrowTy;
+  LLT NarrowTy1;
+  unsigned NumParts;
+
+  if (NarrowTy.isVector()) {
+    // Uneven breakdown not handled.
+    NumParts = DstTy.getNumElements() / NarrowTy.getNumElements();
+    if (NumParts * NarrowTy.getNumElements() != DstTy.getNumElements())
+      return UnableToLegalize;
+
+    NarrowTy1 = LLT::vector(NumParts, SrcTy.getElementType().getSizeInBits());
+  } else {
+    NumParts = DstTy.getNumElements();
+    NarrowTy1 = SrcTy.getElementType();
+  }
+
+  SmallVector<unsigned, 4> SrcRegs, DstRegs;
+  extractParts(SrcReg, NarrowTy1, NumParts, SrcRegs);
+
+  for (unsigned I = 0; I < NumParts; ++I) {
+    unsigned DstReg = MRI.createGenericVirtualRegister(NarrowTy0);
+    MachineInstr *NewInst = MIRBuilder.buildInstr(MI.getOpcode())
+      .addDef(DstReg)
+      .addUse(SrcRegs[I]);
+
+    NewInst->setFlags(MI.getFlags());
+    DstRegs.push_back(DstReg);
+  }
+
+  if (NarrowTy.isVector())
+    MIRBuilder.buildConcatVectors(DstReg, DstRegs);
+  else
+    MIRBuilder.buildBuildVector(DstReg, DstRegs);
+
+  MI.eraseFromParent();
+  return Legalized;
+}
+
+LegalizerHelper::LegalizeResult
+LegalizerHelper::fewerElementsVectorCmp(MachineInstr &MI, unsigned TypeIdx,
+                                        LLT NarrowTy) {
+  unsigned DstReg = MI.getOperand(0).getReg();
+  unsigned Src0Reg = MI.getOperand(2).getReg();
+  LLT DstTy = MRI.getType(DstReg);
+  LLT SrcTy = MRI.getType(Src0Reg);
+
+  unsigned NumParts;
+  LLT NarrowTy0, NarrowTy1;
+
+  if (TypeIdx == 0) {
+    unsigned NewElts = NarrowTy.isVector() ? NarrowTy.getNumElements() : 1;
+    unsigned OldElts = DstTy.getNumElements();
+
+    NarrowTy0 = NarrowTy;
+    NumParts = NarrowTy.isVector() ? (OldElts / NewElts) : DstTy.getNumElements();
+    NarrowTy1 = NarrowTy.isVector() ?
+      LLT::vector(NarrowTy.getNumElements(), SrcTy.getScalarSizeInBits()) :
+      SrcTy.getElementType();
+
+  } else {
+    unsigned NewElts = NarrowTy.isVector() ? NarrowTy.getNumElements() : 1;
+    unsigned OldElts = SrcTy.getNumElements();
+
+    NumParts = NarrowTy.isVector() ? (OldElts / NewElts) :
+      NarrowTy.getNumElements();
+    NarrowTy0 = LLT::vector(NarrowTy.getNumElements(),
+                            DstTy.getScalarSizeInBits());
+    NarrowTy1 = NarrowTy;
+  }
+
+  // FIXME: Don't know how to handle the situation where the small vectors
+  // aren't all the same size yet.
+  if (NarrowTy1.isVector() &&
+      NarrowTy1.getNumElements() * NumParts != DstTy.getNumElements())
+    return UnableToLegalize;
+
+  CmpInst::Predicate Pred
+    = static_cast<CmpInst::Predicate>(MI.getOperand(1).getPredicate());
+
+  SmallVector<unsigned, 2> Src1Regs, Src2Regs, DstRegs;
+  extractParts(MI.getOperand(2).getReg(), NarrowTy1, NumParts, Src1Regs);
+  extractParts(MI.getOperand(3).getReg(), NarrowTy1, NumParts, Src2Regs);
+
+  for (unsigned I = 0; I < NumParts; ++I) {
+    unsigned DstReg = MRI.createGenericVirtualRegister(NarrowTy0);
+    DstRegs.push_back(DstReg);
+
+    if (MI.getOpcode() == TargetOpcode::G_ICMP)
+      MIRBuilder.buildICmp(Pred, DstReg, Src1Regs[I], Src2Regs[I]);
+    else {
+      MachineInstr *NewCmp
+        = MIRBuilder.buildFCmp(Pred, DstReg, Src1Regs[I], Src2Regs[I]);
+      NewCmp->setFlags(MI.getFlags());
+    }
+  }
+
+  if (NarrowTy1.isVector())
+    MIRBuilder.buildConcatVectors(DstReg, DstRegs);
+  else
+    MIRBuilder.buildBuildVector(DstReg, DstRegs);
+
+  MI.eraseFromParent();
+  return Legalized;
+}
+
+LegalizerHelper::LegalizeResult
+LegalizerHelper::fewerElementsVectorSelect(MachineInstr &MI, unsigned TypeIdx,
+                                           LLT NarrowTy) {
+  unsigned DstReg = MI.getOperand(0).getReg();
+  unsigned CondReg = MI.getOperand(1).getReg();
+
+  unsigned NumParts = 0;
+  LLT NarrowTy0, NarrowTy1;
+
+  LLT DstTy = MRI.getType(DstReg);
+  LLT CondTy = MRI.getType(CondReg);
+  unsigned Size = DstTy.getSizeInBits();
+
+  assert(TypeIdx == 0 || CondTy.isVector());
+
+  if (TypeIdx == 0) {
+    NarrowTy0 = NarrowTy;
+    NarrowTy1 = CondTy;
+
+    unsigned NarrowSize = NarrowTy0.getSizeInBits();
     // FIXME: Don't know how to handle the situation where the small vectors
     // aren't all the same size yet.
     if (Size % NarrowSize != 0)
       return UnableToLegalize;
 
-    SmallVector<unsigned, 2> Src1Regs, Src2Regs, DstRegs;
-    extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, Src1Regs);
-    extractParts(MI.getOperand(2).getReg(), NarrowTy, NumParts, Src2Regs);
+    NumParts = Size / NarrowSize;
 
-    for (int i = 0; i < NumParts; ++i) {
-      unsigned DstReg = MRI.createGenericVirtualRegister(NarrowTy);
-      MIRBuilder.buildAdd(DstReg, Src1Regs[i], Src2Regs[i]);
-      DstRegs.push_back(DstReg);
+    // Need to break down the condition type
+    if (CondTy.isVector()) {
+      if (CondTy.getNumElements() == NumParts)
+        NarrowTy1 = CondTy.getElementType();
+      else
+        NarrowTy1 = LLT::vector(CondTy.getNumElements() / NumParts,
+                                CondTy.getScalarSizeInBits());
+    }
+  } else {
+    NumParts = CondTy.getNumElements();
+    if (NarrowTy.isVector()) {
+      // TODO: Handle uneven breakdown.
+      if (NumParts * NarrowTy.getNumElements() != CondTy.getNumElements())
+        return UnableToLegalize;
+
+      return UnableToLegalize;
+    } else {
+      NarrowTy0 = DstTy.getElementType();
+      NarrowTy1 = NarrowTy;
     }
+  }
+
+  SmallVector<unsigned, 2> DstRegs, Src0Regs, Src1Regs, Src2Regs;
+  if (CondTy.isVector())
+    extractParts(MI.getOperand(1).getReg(), NarrowTy1, NumParts, Src0Regs);
+
+  extractParts(MI.getOperand(2).getReg(), NarrowTy0, NumParts, Src1Regs);
+  extractParts(MI.getOperand(3).getReg(), NarrowTy0, NumParts, Src2Regs);
+
+  for (unsigned i = 0; i < NumParts; ++i) {
+    unsigned DstReg = MRI.createGenericVirtualRegister(NarrowTy0);
+    MIRBuilder.buildSelect(DstReg, CondTy.isVector() ? Src0Regs[i] : CondReg,
+                           Src1Regs[i], Src2Regs[i]);
+    DstRegs.push_back(DstReg);
+  }
 
+  if (NarrowTy0.isVector())
     MIRBuilder.buildConcatVectors(DstReg, DstRegs);
-    MI.eraseFromParent();
-    return Legalized;
+  else
+    MIRBuilder.buildBuildVector(DstReg, DstRegs);
+
+  MI.eraseFromParent();
+  return Legalized;
+}
+
+LegalizerHelper::LegalizeResult
+LegalizerHelper::reduceLoadStoreWidth(MachineInstr &MI, unsigned TypeIdx,
+                                      LLT NarrowTy) {
+  // FIXME: Don't know how to handle secondary types yet.
+  if (TypeIdx != 0)
+    return UnableToLegalize;
+
+  MachineMemOperand *MMO = *MI.memoperands_begin();
+
+  // This implementation doesn't work for atomics. Give up instead of doing
+  // something invalid.
+  if (MMO->getOrdering() != AtomicOrdering::NotAtomic ||
+      MMO->getFailureOrdering() != AtomicOrdering::NotAtomic)
+    return UnableToLegalize;
+
+  bool IsLoad = MI.getOpcode() == TargetOpcode::G_LOAD;
+  unsigned ValReg = MI.getOperand(0).getReg();
+  unsigned AddrReg = MI.getOperand(1).getReg();
+  LLT ValTy = MRI.getType(ValReg);
+
+  int NumParts = -1;
+  LLT LeftoverTy;
+  SmallVector<unsigned, 8> NarrowRegs, NarrowLeftoverRegs;
+  if (IsLoad) {
+    NumParts = getNarrowTypeBreakDown(ValTy, NarrowTy, LeftoverTy);
+  } else {
+    if (extractParts(ValReg, ValTy, NarrowTy, LeftoverTy, NarrowRegs,
+                     NarrowLeftoverRegs))
+      NumParts = NarrowRegs.size();
   }
-  case TargetOpcode::G_LOAD:
-  case TargetOpcode::G_STORE: {
-    bool IsLoad = MI.getOpcode() == TargetOpcode::G_LOAD;
-    unsigned ValReg = MI.getOperand(0).getReg();
-    unsigned AddrReg = MI.getOperand(1).getReg();
-    unsigned NarrowSize = NarrowTy.getSizeInBits();
-    unsigned Size = MRI.getType(ValReg).getSizeInBits();
-    unsigned NumParts = Size / NarrowSize;
-
-    SmallVector<unsigned, 8> NarrowRegs;
-    if (!IsLoad)
-      extractParts(ValReg, NarrowTy, NumParts, NarrowRegs);
-
-    const LLT OffsetTy =
-        LLT::scalar(MRI.getType(AddrReg).getScalarSizeInBits());
-    MachineFunction &MF = *MI.getMF();
-    MachineMemOperand *MMO = *MI.memoperands_begin();
-    for (unsigned Idx = 0; Idx < NumParts; ++Idx) {
-      unsigned Adjustment = Idx * NarrowTy.getSizeInBits() / 8;
-      unsigned Alignment = MinAlign(MMO->getAlignment(), Adjustment);
+
+  if (NumParts == -1)
+    return UnableToLegalize;
+
+  const LLT OffsetTy = LLT::scalar(MRI.getType(AddrReg).getScalarSizeInBits());
+
+  unsigned TotalSize = ValTy.getSizeInBits();
+
+  // Split the load/store into PartTy sized pieces starting at Offset. If this
+  // is a load, return the new registers in ValRegs. For a store, each elements
+  // of ValRegs should be PartTy. Returns the next offset that needs to be
+  // handled.
+  auto splitTypePieces = [=](LLT PartTy, SmallVectorImpl<unsigned> &ValRegs,
+                             unsigned Offset) -> unsigned {
+    MachineFunction &MF = MIRBuilder.getMF();
+    unsigned PartSize = PartTy.getSizeInBits();
+    for (unsigned Idx = 0, E = NumParts; Idx != E && Offset < TotalSize;
+         Offset += PartSize, ++Idx) {
+      unsigned ByteSize = PartSize / 8;
+      unsigned ByteOffset = Offset / 8;
       unsigned NewAddrReg = 0;
-      MIRBuilder.materializeGEP(NewAddrReg, AddrReg, OffsetTy, Adjustment);
-      MachineMemOperand &NewMMO = *MF.getMachineMemOperand(
-          MMO->getPointerInfo().getWithOffset(Adjustment), MMO->getFlags(),
-          NarrowTy.getSizeInBits() / 8, Alignment);
+
+      MIRBuilder.materializeGEP(NewAddrReg, AddrReg, OffsetTy, ByteOffset);
+
+      MachineMemOperand *NewMMO =
+        MF.getMachineMemOperand(MMO, ByteOffset, ByteSize);
+
       if (IsLoad) {
-        unsigned Dst = MRI.createGenericVirtualRegister(NarrowTy);
-        NarrowRegs.push_back(Dst);
-        MIRBuilder.buildLoad(Dst, NewAddrReg, NewMMO);
+        unsigned Dst = MRI.createGenericVirtualRegister(PartTy);
+        ValRegs.push_back(Dst);
+        MIRBuilder.buildLoad(Dst, NewAddrReg, *NewMMO);
       } else {
-        MIRBuilder.buildStore(NarrowRegs[Idx], NewAddrReg, NewMMO);
+        MIRBuilder.buildStore(ValRegs[Idx], NewAddrReg, *NewMMO);
       }
     }
-    if (IsLoad) {
-      if (NarrowTy.isVector())
-        MIRBuilder.buildConcatVectors(ValReg, NarrowRegs);
-      else
-        MIRBuilder.buildBuildVector(ValReg, NarrowRegs);
-    }
+
+    return Offset;
+  };
+
+  unsigned HandledOffset = splitTypePieces(NarrowTy, NarrowRegs, 0);
+
+  // Handle the rest of the register if this isn't an even type breakdown.
+  if (LeftoverTy.isValid())
+    splitTypePieces(LeftoverTy, NarrowLeftoverRegs, HandledOffset);
+
+  if (IsLoad) {
+    insertParts(ValReg, ValTy, NarrowTy, NarrowRegs,
+                LeftoverTy, NarrowLeftoverRegs);
+  }
+
+  MI.eraseFromParent();
+  return Legalized;
+}
+
+LegalizerHelper::LegalizeResult
+LegalizerHelper::fewerElementsVector(MachineInstr &MI, unsigned TypeIdx,
+                                     LLT NarrowTy) {
+  using namespace TargetOpcode;
+
+  MIRBuilder.setInstr(MI);
+  switch (MI.getOpcode()) {
+  case G_IMPLICIT_DEF:
+    return fewerElementsVectorImplicitDef(MI, TypeIdx, NarrowTy);
+  case G_AND:
+  case G_OR:
+  case G_XOR:
+  case G_ADD:
+  case G_SUB:
+  case G_MUL:
+  case G_SMULH:
+  case G_UMULH:
+  case G_FADD:
+  case G_FMUL:
+  case G_FSUB:
+  case G_FNEG:
+  case G_FABS:
+  case G_FCANONICALIZE:
+  case G_FDIV:
+  case G_FREM:
+  case G_FMA:
+  case G_FPOW:
+  case G_FEXP:
+  case G_FEXP2:
+  case G_FLOG:
+  case G_FLOG2:
+  case G_FLOG10:
+  case G_FCEIL:
+  case G_FFLOOR:
+  case G_INTRINSIC_ROUND:
+  case G_INTRINSIC_TRUNC:
+  case G_FCOS:
+  case G_FSIN:
+  case G_FSQRT:
+  case G_BSWAP:
+    return fewerElementsVectorBasic(MI, TypeIdx, NarrowTy);
+  case G_SHL:
+  case G_LSHR:
+  case G_ASHR:
+    return fewerElementsVectorMultiEltType(MI, TypeIdx, NarrowTy);
+  case G_ZEXT:
+  case G_SEXT:
+  case G_ANYEXT:
+  case G_FPEXT:
+  case G_FPTRUNC:
+  case G_SITOFP:
+  case G_UITOFP:
+  case G_FPTOSI:
+  case G_FPTOUI:
+  case G_INTTOPTR:
+  case G_PTRTOINT:
+  case G_ADDRSPACE_CAST:
+    return fewerElementsVectorCasts(MI, TypeIdx, NarrowTy);
+  case G_ICMP:
+  case G_FCMP:
+    return fewerElementsVectorCmp(MI, TypeIdx, NarrowTy);
+  case G_SELECT:
+    return fewerElementsVectorSelect(MI, TypeIdx, NarrowTy);
+  case G_LOAD:
+  case G_STORE:
+    return reduceLoadStoreWidth(MI, TypeIdx, NarrowTy);
+  default:
+    return UnableToLegalize;
+  }
+}
+
+LegalizerHelper::LegalizeResult
+LegalizerHelper::narrowScalarShiftByConstant(MachineInstr &MI, const APInt &Amt,
+                                             const LLT HalfTy, const LLT AmtTy) {
+
+  unsigned InL = MRI.createGenericVirtualRegister(HalfTy);
+  unsigned InH = MRI.createGenericVirtualRegister(HalfTy);
+  MIRBuilder.buildUnmerge({InL, InH}, MI.getOperand(1).getReg());
+
+  if (Amt.isNullValue()) {
+    MIRBuilder.buildMerge(MI.getOperand(0).getReg(), {InL, InH});
     MI.eraseFromParent();
     return Legalized;
   }
+
+  LLT NVT = HalfTy;
+  unsigned NVTBits = HalfTy.getSizeInBits();
+  unsigned VTBits = 2 * NVTBits;
+
+  SrcOp Lo(0), Hi(0);
+  if (MI.getOpcode() == TargetOpcode::G_SHL) {
+    if (Amt.ugt(VTBits)) {
+      Lo = Hi = MIRBuilder.buildConstant(NVT, 0);
+    } else if (Amt.ugt(NVTBits)) {
+      Lo = MIRBuilder.buildConstant(NVT, 0);
+      Hi = MIRBuilder.buildShl(NVT, InL,
+                               MIRBuilder.buildConstant(AmtTy, Amt - NVTBits));
+    } else if (Amt == NVTBits) {
+      Lo = MIRBuilder.buildConstant(NVT, 0);
+      Hi = InL;
+    } else {
+      Lo = MIRBuilder.buildShl(NVT, InL, MIRBuilder.buildConstant(AmtTy, Amt));
+      auto OrLHS =
+          MIRBuilder.buildShl(NVT, InH, MIRBuilder.buildConstant(AmtTy, Amt));
+      auto OrRHS = MIRBuilder.buildLShr(
+          NVT, InL, MIRBuilder.buildConstant(AmtTy, -Amt + NVTBits));
+      Hi = MIRBuilder.buildOr(NVT, OrLHS, OrRHS);
+    }
+  } else if (MI.getOpcode() == TargetOpcode::G_LSHR) {
+    if (Amt.ugt(VTBits)) {
+      Lo = Hi = MIRBuilder.buildConstant(NVT, 0);
+    } else if (Amt.ugt(NVTBits)) {
+      Lo = MIRBuilder.buildLShr(NVT, InH,
+                                MIRBuilder.buildConstant(AmtTy, Amt - NVTBits));
+      Hi = MIRBuilder.buildConstant(NVT, 0);
+    } else if (Amt == NVTBits) {
+      Lo = InH;
+      Hi = MIRBuilder.buildConstant(NVT, 0);
+    } else {
+      auto ShiftAmtConst = MIRBuilder.buildConstant(AmtTy, Amt);
+
+      auto OrLHS = MIRBuilder.buildLShr(NVT, InL, ShiftAmtConst);
+      auto OrRHS = MIRBuilder.buildShl(
+          NVT, InH, MIRBuilder.buildConstant(AmtTy, -Amt + NVTBits));
+
+      Lo = MIRBuilder.buildOr(NVT, OrLHS, OrRHS);
+      Hi = MIRBuilder.buildLShr(NVT, InH, ShiftAmtConst);
+    }
+  } else {
+    if (Amt.ugt(VTBits)) {
+      Hi = Lo = MIRBuilder.buildAShr(
+          NVT, InH, MIRBuilder.buildConstant(AmtTy, NVTBits - 1));
+    } else if (Amt.ugt(NVTBits)) {
+      Lo = MIRBuilder.buildAShr(NVT, InH,
+                                MIRBuilder.buildConstant(AmtTy, Amt - NVTBits));
+      Hi = MIRBuilder.buildAShr(NVT, InH,
+                                MIRBuilder.buildConstant(AmtTy, NVTBits - 1));
+    } else if (Amt == NVTBits) {
+      Lo = InH;
+      Hi = MIRBuilder.buildAShr(NVT, InH,
+                                MIRBuilder.buildConstant(AmtTy, NVTBits - 1));
+    } else {
+      auto ShiftAmtConst = MIRBuilder.buildConstant(AmtTy, Amt);
+
+      auto OrLHS = MIRBuilder.buildLShr(NVT, InL, ShiftAmtConst);
+      auto OrRHS = MIRBuilder.buildShl(
+          NVT, InH, MIRBuilder.buildConstant(AmtTy, -Amt + NVTBits));
+
+      Lo = MIRBuilder.buildOr(NVT, OrLHS, OrRHS);
+      Hi = MIRBuilder.buildAShr(NVT, InH, ShiftAmtConst);
+    }
+  }
+
+  MIRBuilder.buildMerge(MI.getOperand(0).getReg(), {Lo.getReg(), Hi.getReg()});
+  MI.eraseFromParent();
+
+  return Legalized;
+}
+
+// TODO: Optimize if constant shift amount.
+LegalizerHelper::LegalizeResult
+LegalizerHelper::narrowScalarShift(MachineInstr &MI, unsigned TypeIdx,
+                                   LLT RequestedTy) {
+  if (TypeIdx == 1) {
+    Observer.changingInstr(MI);
+    narrowScalarSrc(MI, RequestedTy, 2);
+    Observer.changedInstr(MI);
+    return Legalized;
   }
+
+  unsigned DstReg = MI.getOperand(0).getReg();
+  LLT DstTy = MRI.getType(DstReg);
+  if (DstTy.isVector())
+    return UnableToLegalize;
+
+  unsigned Amt = MI.getOperand(2).getReg();
+  LLT ShiftAmtTy = MRI.getType(Amt);
+  const unsigned DstEltSize = DstTy.getScalarSizeInBits();
+  if (DstEltSize % 2 != 0)
+    return UnableToLegalize;
+
+  // Ignore the input type. We can only go to exactly half the size of the
+  // input. If that isn't small enough, the resulting pieces will be further
+  // legalized.
+  const unsigned NewBitSize = DstEltSize / 2;
+  const LLT HalfTy = LLT::scalar(NewBitSize);
+  const LLT CondTy = LLT::scalar(1);
+
+  if (const MachineInstr *KShiftAmt =
+          getOpcodeDef(TargetOpcode::G_CONSTANT, Amt, MRI)) {
+    return narrowScalarShiftByConstant(
+        MI, KShiftAmt->getOperand(1).getCImm()->getValue(), HalfTy, ShiftAmtTy);
+  }
+
+  // TODO: Expand with known bits.
+
+  // Handle the fully general expansion by an unknown amount.
+  auto NewBits = MIRBuilder.buildConstant(ShiftAmtTy, NewBitSize);
+
+  unsigned InL = MRI.createGenericVirtualRegister(HalfTy);
+  unsigned InH = MRI.createGenericVirtualRegister(HalfTy);
+  MIRBuilder.buildUnmerge({InL, InH}, MI.getOperand(1).getReg());
+
+  auto AmtExcess = MIRBuilder.buildSub(ShiftAmtTy, Amt, NewBits);
+  auto AmtLack = MIRBuilder.buildSub(ShiftAmtTy, NewBits, Amt);
+
+  auto Zero = MIRBuilder.buildConstant(ShiftAmtTy, 0);
+  auto IsShort = MIRBuilder.buildICmp(ICmpInst::ICMP_ULT, CondTy, Amt, NewBits);
+  auto IsZero = MIRBuilder.buildICmp(ICmpInst::ICMP_EQ, CondTy, Amt, Zero);
+
+  unsigned ResultRegs[2];
+  switch (MI.getOpcode()) {
+  case TargetOpcode::G_SHL: {
+    // Short: ShAmt < NewBitSize
+    auto LoS = MIRBuilder.buildShl(HalfTy, InH, Amt);
+
+    auto OrLHS = MIRBuilder.buildShl(HalfTy, InH, Amt);
+    auto OrRHS = MIRBuilder.buildLShr(HalfTy, InL, AmtLack);
+    auto HiS = MIRBuilder.buildOr(HalfTy, OrLHS, OrRHS);
+
+    // Long: ShAmt >= NewBitSize
+    auto LoL = MIRBuilder.buildConstant(HalfTy, 0);         // Lo part is zero.
+    auto HiL = MIRBuilder.buildShl(HalfTy, InL, AmtExcess); // Hi from Lo part.
+
+    auto Lo = MIRBuilder.buildSelect(HalfTy, IsShort, LoS, LoL);
+    auto Hi = MIRBuilder.buildSelect(
+        HalfTy, IsZero, InH, MIRBuilder.buildSelect(HalfTy, IsShort, HiS, HiL));
+
+    ResultRegs[0] = Lo.getReg(0);
+    ResultRegs[1] = Hi.getReg(0);
+    break;
+  }
+  case TargetOpcode::G_LSHR: {
+    // Short: ShAmt < NewBitSize
+    auto HiS = MIRBuilder.buildLShr(HalfTy, InH, Amt);
+
+    auto OrLHS = MIRBuilder.buildLShr(HalfTy, InL, Amt);
+    auto OrRHS = MIRBuilder.buildShl(HalfTy, InH, AmtLack);
+    auto LoS = MIRBuilder.buildOr(HalfTy, OrLHS, OrRHS);
+
+    // Long: ShAmt >= NewBitSize
+    auto HiL = MIRBuilder.buildConstant(HalfTy, 0);          // Hi part is zero.
+    auto LoL = MIRBuilder.buildLShr(HalfTy, InH, AmtExcess); // Lo from Hi part.
+
+    auto Lo = MIRBuilder.buildSelect(
+        HalfTy, IsZero, InL, MIRBuilder.buildSelect(HalfTy, IsShort, LoS, LoL));
+    auto Hi = MIRBuilder.buildSelect(HalfTy, IsShort, HiS, HiL);
+
+    ResultRegs[0] = Lo.getReg(0);
+    ResultRegs[1] = Hi.getReg(0);
+    break;
+  }
+  case TargetOpcode::G_ASHR: {
+    // Short: ShAmt < NewBitSize
+    auto HiS = MIRBuilder.buildAShr(HalfTy, InH, Amt);
+
+    auto OrLHS = MIRBuilder.buildLShr(HalfTy, InL, Amt);
+    auto OrRHS = MIRBuilder.buildLShr(HalfTy, InH, AmtLack);
+    auto LoS = MIRBuilder.buildOr(HalfTy, OrLHS, OrRHS);
+
+    // Long: ShAmt >= NewBitSize
+
+    // Sign of Hi part.
+    auto HiL = MIRBuilder.buildAShr(
+        HalfTy, InH, MIRBuilder.buildConstant(ShiftAmtTy, NewBitSize - 1));
+
+    auto LoL = MIRBuilder.buildAShr(HalfTy, InH, AmtExcess); // Lo from Hi part.
+
+    auto Lo = MIRBuilder.buildSelect(
+        HalfTy, IsZero, InL, MIRBuilder.buildSelect(HalfTy, IsShort, LoS, LoL));
+
+    auto Hi = MIRBuilder.buildSelect(HalfTy, IsShort, HiS, HiL);
+
+    ResultRegs[0] = Lo.getReg(0);
+    ResultRegs[1] = Hi.getReg(0);
+    break;
+  }
+  default:
+    llvm_unreachable("not a shift");
+  }
+
+  MIRBuilder.buildMerge(DstReg, ResultRegs);
+  MI.eraseFromParent();
+  return Legalized;
+}
+
+LegalizerHelper::LegalizeResult
+LegalizerHelper::moreElementsVector(MachineInstr &MI, unsigned TypeIdx,
+                                    LLT MoreTy) {
+  MIRBuilder.setInstr(MI);
+  unsigned Opc = MI.getOpcode();
+  switch (Opc) {
+  case TargetOpcode::G_IMPLICIT_DEF: {
+    Observer.changingInstr(MI);
+    moreElementsVectorDst(MI, MoreTy, 0);
+    Observer.changedInstr(MI);
+    return Legalized;
+  }
+  default:
+    return UnableToLegalize;
+  }
+}
+
+LegalizerHelper::LegalizeResult
+LegalizerHelper::narrowScalarMul(MachineInstr &MI, unsigned TypeIdx, LLT NewTy) {
+  unsigned DstReg = MI.getOperand(0).getReg();
+  unsigned Src0 = MI.getOperand(1).getReg();
+  unsigned Src1 = MI.getOperand(2).getReg();
+  LLT Ty = MRI.getType(DstReg);
+  if (Ty.isVector())
+    return UnableToLegalize;
+
+  unsigned Size = Ty.getSizeInBits();
+  unsigned NewSize = Size / 2;
+  if (Size != 2 * NewSize)
+    return UnableToLegalize;
+
+  LLT HalfTy = LLT::scalar(NewSize);
+  // TODO: if HalfTy != NewTy, handle the breakdown all at once?
+
+  unsigned ShiftAmt = MRI.createGenericVirtualRegister(Ty);
+  unsigned Lo = MRI.createGenericVirtualRegister(HalfTy);
+  unsigned Hi = MRI.createGenericVirtualRegister(HalfTy);
+  unsigned ExtLo = MRI.createGenericVirtualRegister(Ty);
+  unsigned ExtHi = MRI.createGenericVirtualRegister(Ty);
+  unsigned ShiftedHi = MRI.createGenericVirtualRegister(Ty);
+
+  SmallVector<unsigned, 2> Src0Parts;
+  SmallVector<unsigned, 2> Src1Parts;
+
+  extractParts(Src0, HalfTy, 2, Src0Parts);
+  extractParts(Src1, HalfTy, 2, Src1Parts);
+
+  MIRBuilder.buildMul(Lo, Src0Parts[0], Src1Parts[0]);
+
+  // TODO: Use smulh or umulh depending on what the target has.
+  MIRBuilder.buildUMulH(Hi, Src0Parts[1], Src1Parts[1]);
+
+  MIRBuilder.buildConstant(ShiftAmt, NewSize);
+  MIRBuilder.buildAnyExt(ExtHi, Hi);
+  MIRBuilder.buildShl(ShiftedHi, ExtHi, ShiftAmt);
+
+  MIRBuilder.buildZExt(ExtLo, Lo);
+  MIRBuilder.buildOr(DstReg, ExtLo, ShiftedHi);
+  MI.eraseFromParent();
+  return Legalized;
+}
+
+
+LegalizerHelper::LegalizeResult
+LegalizerHelper::narrowScalarExtract(MachineInstr &MI, unsigned TypeIdx,
+                                     LLT NarrowTy) {
+  if (TypeIdx != 1)
+    return UnableToLegalize;
+
+  uint64_t NarrowSize = NarrowTy.getSizeInBits();
+
+  int64_t SizeOp1 = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits();
+  // FIXME: add support for when SizeOp1 isn't an exact multiple of
+  // NarrowSize.
+  if (SizeOp1 % NarrowSize != 0)
+    return UnableToLegalize;
+  int NumParts = SizeOp1 / NarrowSize;
+
+  SmallVector<unsigned, 2> SrcRegs, DstRegs;
+  SmallVector<uint64_t, 2> Indexes;
+  extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, SrcRegs);
+
+  unsigned OpReg = MI.getOperand(0).getReg();
+  uint64_t OpStart = MI.getOperand(2).getImm();
+  uint64_t OpSize = MRI.getType(OpReg).getSizeInBits();
+  for (int i = 0; i < NumParts; ++i) {
+    unsigned SrcStart = i * NarrowSize;
+
+    if (SrcStart + NarrowSize <= OpStart || SrcStart >= OpStart + OpSize) {
+      // No part of the extract uses this subregister, ignore it.
+      continue;
+    } else if (SrcStart == OpStart && NarrowTy == MRI.getType(OpReg)) {
+      // The entire subregister is extracted, forward the value.
+      DstRegs.push_back(SrcRegs[i]);
+      continue;
+    }
+
+    // OpSegStart is where this destination segment would start in OpReg if it
+    // extended infinitely in both directions.
+    int64_t ExtractOffset;
+    uint64_t SegSize;
+    if (OpStart < SrcStart) {
+      ExtractOffset = 0;
+      SegSize = std::min(NarrowSize, OpStart + OpSize - SrcStart);
+    } else {
+      ExtractOffset = OpStart - SrcStart;
+      SegSize = std::min(SrcStart + NarrowSize - OpStart, OpSize);
+    }
+
+    unsigned SegReg = SrcRegs[i];
+    if (ExtractOffset != 0 || SegSize != NarrowSize) {
+      // A genuine extract is needed.
+      SegReg = MRI.createGenericVirtualRegister(LLT::scalar(SegSize));
+      MIRBuilder.buildExtract(SegReg, SrcRegs[i], ExtractOffset);
+    }
+
+    DstRegs.push_back(SegReg);
+  }
+
+  unsigned DstReg = MI.getOperand(0).getReg();
+  if(MRI.getType(DstReg).isVector())
+    MIRBuilder.buildBuildVector(DstReg, DstRegs);
+  else
+    MIRBuilder.buildMerge(DstReg, DstRegs);
+  MI.eraseFromParent();
+  return Legalized;
+}
+
+LegalizerHelper::LegalizeResult
+LegalizerHelper::narrowScalarInsert(MachineInstr &MI, unsigned TypeIdx,
+                                    LLT NarrowTy) {
+  // FIXME: Don't know how to handle secondary types yet.
+  if (TypeIdx != 0)
+    return UnableToLegalize;
+
+  uint64_t SizeOp0 = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
+  uint64_t NarrowSize = NarrowTy.getSizeInBits();
+
+  // FIXME: add support for when SizeOp0 isn't an exact multiple of
+  // NarrowSize.
+  if (SizeOp0 % NarrowSize != 0)
+    return UnableToLegalize;
+
+  int NumParts = SizeOp0 / NarrowSize;
+
+  SmallVector<unsigned, 2> SrcRegs, DstRegs;
+  SmallVector<uint64_t, 2> Indexes;
+  extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, SrcRegs);
+
+  unsigned OpReg = MI.getOperand(2).getReg();
+  uint64_t OpStart = MI.getOperand(3).getImm();
+  uint64_t OpSize = MRI.getType(OpReg).getSizeInBits();
+  for (int i = 0; i < NumParts; ++i) {
+    unsigned DstStart = i * NarrowSize;
+
+    if (DstStart + NarrowSize <= OpStart || DstStart >= OpStart + OpSize) {
+      // No part of the insert affects this subregister, forward the original.
+      DstRegs.push_back(SrcRegs[i]);
+      continue;
+    } else if (DstStart == OpStart && NarrowTy == MRI.getType(OpReg)) {
+      // The entire subregister is defined by this insert, forward the new
+      // value.
+      DstRegs.push_back(OpReg);
+      continue;
+    }
+
+    // OpSegStart is where this destination segment would start in OpReg if it
+    // extended infinitely in both directions.
+    int64_t ExtractOffset, InsertOffset;
+    uint64_t SegSize;
+    if (OpStart < DstStart) {
+      InsertOffset = 0;
+      ExtractOffset = DstStart - OpStart;
+      SegSize = std::min(NarrowSize, OpStart + OpSize - DstStart);
+    } else {
+      InsertOffset = OpStart - DstStart;
+      ExtractOffset = 0;
+      SegSize =
+        std::min(NarrowSize - InsertOffset, OpStart + OpSize - DstStart);
+    }
+
+    unsigned SegReg = OpReg;
+    if (ExtractOffset != 0 || SegSize != OpSize) {
+      // A genuine extract is needed.
+      SegReg = MRI.createGenericVirtualRegister(LLT::scalar(SegSize));
+      MIRBuilder.buildExtract(SegReg, OpReg, ExtractOffset);
+    }
+
+    unsigned DstReg = MRI.createGenericVirtualRegister(NarrowTy);
+    MIRBuilder.buildInsert(DstReg, SrcRegs[i], SegReg, InsertOffset);
+    DstRegs.push_back(DstReg);
+  }
+
+  assert(DstRegs.size() == (unsigned)NumParts && "not all parts covered");
+  unsigned DstReg = MI.getOperand(0).getReg();
+  if(MRI.getType(DstReg).isVector())
+    MIRBuilder.buildBuildVector(DstReg, DstRegs);
+  else
+    MIRBuilder.buildMerge(DstReg, DstRegs);
+  MI.eraseFromParent();
+  return Legalized;
+}
+
+LegalizerHelper::LegalizeResult
+LegalizerHelper::narrowScalarSelect(MachineInstr &MI, unsigned TypeIdx,
+                                    LLT NarrowTy) {
+  if (TypeIdx != 0)
+    return UnableToLegalize;
+
+  unsigned CondReg = MI.getOperand(1).getReg();
+  LLT CondTy = MRI.getType(CondReg);
+  if (CondTy.isVector()) // TODO: Handle vselect
+    return UnableToLegalize;
+
+  unsigned DstReg = MI.getOperand(0).getReg();
+  LLT DstTy = MRI.getType(DstReg);
+
+  SmallVector<unsigned, 4> DstRegs, DstLeftoverRegs;
+  SmallVector<unsigned, 4> Src1Regs, Src1LeftoverRegs;
+  SmallVector<unsigned, 4> Src2Regs, Src2LeftoverRegs;
+  LLT LeftoverTy;
+  if (!extractParts(MI.getOperand(2).getReg(), DstTy, NarrowTy, LeftoverTy,
+                    Src1Regs, Src1LeftoverRegs))
+    return UnableToLegalize;
+
+  LLT Unused;
+  if (!extractParts(MI.getOperand(3).getReg(), DstTy, NarrowTy, Unused,
+                    Src2Regs, Src2LeftoverRegs))
+    llvm_unreachable("inconsistent extractParts result");
+
+  for (unsigned I = 0, E = Src1Regs.size(); I != E; ++I) {
+    auto Select = MIRBuilder.buildSelect(NarrowTy,
+                                         CondReg, Src1Regs[I], Src2Regs[I]);
+    DstRegs.push_back(Select->getOperand(0).getReg());
+  }
+
+  for (unsigned I = 0, E = Src1LeftoverRegs.size(); I != E; ++I) {
+    auto Select = MIRBuilder.buildSelect(
+      LeftoverTy, CondReg, Src1LeftoverRegs[I], Src2LeftoverRegs[I]);
+    DstLeftoverRegs.push_back(Select->getOperand(0).getReg());
+  }
+
+  insertParts(DstReg, DstTy, NarrowTy, DstRegs,
+              LeftoverTy, DstLeftoverRegs);
+
+  MI.eraseFromParent();
+  return Legalized;
 }
 
 LegalizerHelper::LegalizeResult
@@ -1244,7 +2574,7 @@ LegalizerHelper::lowerBitCount(MachineInstr &MI, unsigned TypeIdx, LLT Ty) {
   case TargetOpcode::G_CTLZ: {
     unsigned SrcReg = MI.getOperand(1).getReg();
     unsigned Len = Ty.getSizeInBits();
-    if (isSupported({TargetOpcode::G_CTLZ_ZERO_UNDEF, {Ty}})) {
+    if (isSupported({TargetOpcode::G_CTLZ_ZERO_UNDEF, {Ty, Ty}})) {
       // If CTLZ_ZERO_UNDEF is supported, emit that and a select for zero.
       auto MIBCtlzZU = MIRBuilder.buildInstr(TargetOpcode::G_CTLZ_ZERO_UNDEF,
                                              {Ty}, {SrcReg});
@@ -1294,7 +2624,7 @@ LegalizerHelper::lowerBitCount(MachineInstr &MI, unsigned TypeIdx, LLT Ty) {
   case TargetOpcode::G_CTTZ: {
     unsigned SrcReg = MI.getOperand(1).getReg();
     unsigned Len = Ty.getSizeInBits();
-    if (isSupported({TargetOpcode::G_CTTZ_ZERO_UNDEF, {Ty}})) {
+    if (isSupported({TargetOpcode::G_CTTZ_ZERO_UNDEF, {Ty, Ty}})) {
       // If CTTZ_ZERO_UNDEF is legal or custom, emit that and a select with
       // zero.
       auto MIBCttzZU = MIRBuilder.buildInstr(TargetOpcode::G_CTTZ_ZERO_UNDEF,
@@ -1319,8 +2649,8 @@ LegalizerHelper::lowerBitCount(MachineInstr &MI, unsigned TypeIdx, LLT Ty) {
         TargetOpcode::G_AND, {Ty},
         {MIBNot, MIRBuilder.buildInstr(TargetOpcode::G_ADD, {Ty},
                                        {SrcReg, MIBCstNeg1})});
-    if (!isSupported({TargetOpcode::G_CTPOP, {Ty}}) &&
-        isSupported({TargetOpcode::G_CTLZ, {Ty}})) {
+    if (!isSupported({TargetOpcode::G_CTPOP, {Ty, Ty}}) &&
+        isSupported({TargetOpcode::G_CTLZ, {Ty, Ty}})) {
       auto MIBCstLen = MIRBuilder.buildConstant(Ty, Len);
       MIRBuilder.buildInstr(
           TargetOpcode::G_SUB, {MI.getOperand(0).getReg()},
diff --git a/lib/CodeGen/GlobalISel/LegalizerInfo.cpp b/lib/CodeGen/GlobalISel/LegalizerInfo.cpp
index 926af3fd76c6760c13f3483df2577a77c44456ed..f2fc2e78a9effd9bed252562279a65ef3d1341fb 100644
--- a/lib/CodeGen/GlobalISel/LegalizerInfo.cpp
+++ b/lib/CodeGen/GlobalISel/LegalizerInfo.cpp
@@ -1,9 +1,8 @@
 //===- lib/CodeGen/GlobalISel/LegalizerInfo.cpp - Legalizer ---------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -59,6 +58,66 @@ raw_ostream &LegalityQuery::print(raw_ostream &OS) const {
   return OS;
 }
 
+#ifndef NDEBUG
+// Make sure the returned mutation makes sense for the match type.
+static bool mutationIsSane(const LegalizeRule &Rule,
+                           const LegalityQuery &Q,
+                           std::pair<unsigned, LLT> Mutation) {
+  const unsigned TypeIdx = Mutation.first;
+  const LLT OldTy = Q.Types[TypeIdx];
+  const LLT NewTy = Mutation.second;
+
+  switch (Rule.getAction()) {
+  case FewerElements:
+  case MoreElements: {
+    if (!OldTy.isVector())
+      return false;
+
+    if (NewTy.isVector()) {
+      if (Rule.getAction() == FewerElements) {
+        // Make sure the element count really decreased.
+        if (NewTy.getNumElements() >= OldTy.getNumElements())
+          return false;
+      } else {
+        // Make sure the element count really increased.
+        if (NewTy.getNumElements() <= OldTy.getNumElements())
+          return false;
+      }
+    }
+
+    // Make sure the element type didn't change.
+    return NewTy.getScalarType() == OldTy.getElementType();
+  }
+  case NarrowScalar:
+  case WidenScalar: {
+    if (OldTy.isVector()) {
+      // Number of elements should not change.
+      if (!NewTy.isVector() || OldTy.getNumElements() != NewTy.getNumElements())
+        return false;
+    } else {
+      // Both types must be vectors
+      if (NewTy.isVector())
+        return false;
+    }
+
+    if (Rule.getAction() == NarrowScalar)  {
+      // Make sure the size really decreased.
+      if (NewTy.getScalarSizeInBits() >= OldTy.getScalarSizeInBits())
+        return false;
+    } else {
+      // Make sure the size really increased.
+      if (NewTy.getScalarSizeInBits() <= OldTy.getScalarSizeInBits())
+        return false;
+    }
+
+    return true;
+  }
+  default:
+    return true;
+  }
+}
+#endif
+
 LegalizeActionStep LegalizeRuleSet::apply(const LegalityQuery &Query) const {
   LLVM_DEBUG(dbgs() << "Applying legalizer ruleset to: "; Query.print(dbgs());
              dbgs() << "\n");
@@ -66,12 +125,15 @@ LegalizeActionStep LegalizeRuleSet::apply(const LegalityQuery &Query) const {
     LLVM_DEBUG(dbgs() << ".. fallback to legacy rules (no rules defined)\n");
     return {LegalizeAction::UseLegacyRules, 0, LLT{}};
   }
-  for (const auto &Rule : Rules) {
+  for (const LegalizeRule &Rule : Rules) {
     if (Rule.match(Query)) {
       LLVM_DEBUG(dbgs() << ".. match\n");
       std::pair<unsigned, LLT> Mutation = Rule.determineMutation(Query);
       LLVM_DEBUG(dbgs() << ".. .. " << (unsigned)Rule.getAction() << ", "
                         << Mutation.first << ", " << Mutation.second << "\n");
+      assert(mutationIsSane(Rule, Query, Mutation) &&
+             "legality mutation invalid for match");
+
       assert((Query.Types[Mutation.first] != Mutation.second ||
               Rule.getAction() == Lower ||
               Rule.getAction() == MoreElements ||
@@ -180,16 +242,14 @@ void LegalizerInfo::computeTables() {
         if (TypeIdx < ScalarSizeChangeStrategies[OpcodeIdx].size() &&
             ScalarSizeChangeStrategies[OpcodeIdx][TypeIdx] != nullptr)
           S = ScalarSizeChangeStrategies[OpcodeIdx][TypeIdx];
-        llvm::sort(ScalarSpecifiedActions.begin(),
-                   ScalarSpecifiedActions.end());
+        llvm::sort(ScalarSpecifiedActions);
         checkPartialSizeAndActionsVector(ScalarSpecifiedActions);
         setScalarAction(Opcode, TypeIdx, S(ScalarSpecifiedActions));
       }
 
       // 2. Handle pointer types
       for (auto PointerSpecifiedActions : AddressSpace2SpecifiedActions) {
-        llvm::sort(PointerSpecifiedActions.second.begin(),
-                   PointerSpecifiedActions.second.end());
+        llvm::sort(PointerSpecifiedActions.second);
         checkPartialSizeAndActionsVector(PointerSpecifiedActions.second);
         // For pointer types, we assume that there isn't a meaningfull way
         // to change the number of bits used in the pointer.
@@ -201,8 +261,7 @@ void LegalizerInfo::computeTables() {
       // 3. Handle vector types
       SizeAndActionsVec ElementSizesSeen;
       for (auto VectorSpecifiedActions : ElemSize2SpecifiedActions) {
-        llvm::sort(VectorSpecifiedActions.second.begin(),
-                   VectorSpecifiedActions.second.end());
+        llvm::sort(VectorSpecifiedActions.second);
         const uint16_t ElementSize = VectorSpecifiedActions.first;
         ElementSizesSeen.push_back({ElementSize, Legal});
         checkPartialSizeAndActionsVector(VectorSpecifiedActions.second);
@@ -585,7 +644,7 @@ const MachineInstr *llvm::machineFunctionIsIllegal(const MachineFunction &MF) {
     for (const MachineBasicBlock &MBB : MF)
       for (const MachineInstr &MI : MBB)
         if (isPreISelGenericOpcode(MI.getOpcode()) && !MLI->isLegal(MI, MRI))
-	  return &MI;
+          return &MI;
   }
   return nullptr;
 }
diff --git a/lib/CodeGen/GlobalISel/Localizer.cpp b/lib/CodeGen/GlobalISel/Localizer.cpp
index 52b340753a50eb50cb9101af4b70e7271d95d7e7..e61cddf114a01ae37795b862eb513bba6d127d48 100644
--- a/lib/CodeGen/GlobalISel/Localizer.cpp
+++ b/lib/CodeGen/GlobalISel/Localizer.cpp
@@ -1,9 +1,8 @@
 //===- Localizer.cpp ---------------------- Localize some instrs -*- C++ -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp b/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp
index c1109e611777dfe7abd9f261406cce476052f6c7..c01eee1fe8886b560a23c831801a89e16d97465b 100644
--- a/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp
+++ b/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp
@@ -1,9 +1,8 @@
 //===-- llvm/CodeGen/GlobalISel/MachineIRBuilder.cpp - MIBuilder--*- C++ -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
@@ -17,6 +16,7 @@
 #include "llvm/CodeGen/MachineInstrBuilder.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
 #include "llvm/CodeGen/TargetInstrInfo.h"
+#include "llvm/CodeGen/TargetLowering.h"
 #include "llvm/CodeGen/TargetOpcodes.h"
 #include "llvm/CodeGen/TargetSubtargetInfo.h"
 #include "llvm/IR/DebugInfo.h"
@@ -46,6 +46,8 @@ void MachineIRBuilder::setInstr(MachineInstr &MI) {
   State.II = MI.getIterator();
 }
 
+void MachineIRBuilder::setCSEInfo(GISelCSEInfo *Info) { State.CSEInfo = Info; }
+
 void MachineIRBuilder::setInsertPt(MachineBasicBlock &MBB,
                                    MachineBasicBlock::iterator II) {
   assert(MBB.getParent() == &getMF() &&
@@ -183,6 +185,12 @@ void MachineIRBuilder::validateBinaryOp(const LLT &Res, const LLT &Op0,
   assert((Res == Op0 && Res == Op1) && "type mismatch");
 }
 
+void MachineIRBuilder::validateShiftOp(const LLT &Res, const LLT &Op0,
+                                       const LLT &Op1) {
+  assert((Res.isScalar() || Res.isVector()) && "invalid operand type");
+  assert((Res == Op0) && "type mismatch");
+}
+
 MachineInstrBuilder MachineIRBuilder::buildGEP(unsigned Res, unsigned Op0,
                                                unsigned Op1) {
   assert(getMRI()->getType(Res).isPointer() &&
@@ -241,36 +249,60 @@ MachineInstrBuilder MachineIRBuilder::buildCopy(const DstOp &Res,
 MachineInstrBuilder MachineIRBuilder::buildConstant(const DstOp &Res,
                                                     const ConstantInt &Val) {
   LLT Ty = Res.getLLTTy(*getMRI());
+  LLT EltTy = Ty.getScalarType();
+  assert(EltTy.getScalarSizeInBits() == Val.getBitWidth() &&
+         "creating constant with the wrong size");
+
+  if (Ty.isVector()) {
+    auto Const = buildInstr(TargetOpcode::G_CONSTANT)
+    .addDef(getMRI()->createGenericVirtualRegister(EltTy))
+    .addCImm(&Val);
+    return buildSplatVector(Res, Const);
+  }
 
-  assert((Ty.isScalar() || Ty.isPointer()) && "invalid operand type");
-
-  const ConstantInt *NewVal = &Val;
-  if (Ty.getSizeInBits() != Val.getBitWidth())
-    NewVal = ConstantInt::get(getMF().getFunction().getContext(),
-                              Val.getValue().sextOrTrunc(Ty.getSizeInBits()));
-
-  auto MIB = buildInstr(TargetOpcode::G_CONSTANT);
-  Res.addDefToMIB(*getMRI(), MIB);
-  MIB.addCImm(NewVal);
-  return MIB;
+  auto Const = buildInstr(TargetOpcode::G_CONSTANT);
+  Res.addDefToMIB(*getMRI(), Const);
+  Const.addCImm(&Val);
+  return Const;
 }
 
 MachineInstrBuilder MachineIRBuilder::buildConstant(const DstOp &Res,
                                                     int64_t Val) {
   auto IntN = IntegerType::get(getMF().getFunction().getContext(),
-                               Res.getLLTTy(*getMRI()).getSizeInBits());
+                               Res.getLLTTy(*getMRI()).getScalarSizeInBits());
   ConstantInt *CI = ConstantInt::get(IntN, Val, true);
   return buildConstant(Res, *CI);
 }
 
 MachineInstrBuilder MachineIRBuilder::buildFConstant(const DstOp &Res,
                                                      const ConstantFP &Val) {
-  assert(Res.getLLTTy(*getMRI()).isScalar() && "invalid operand type");
+  LLT Ty = Res.getLLTTy(*getMRI());
+  LLT EltTy = Ty.getScalarType();
 
-  auto MIB = buildInstr(TargetOpcode::G_FCONSTANT);
-  Res.addDefToMIB(*getMRI(), MIB);
-  MIB.addFPImm(&Val);
-  return MIB;
+  assert(APFloat::getSizeInBits(Val.getValueAPF().getSemantics())
+         == EltTy.getSizeInBits() &&
+         "creating fconstant with the wrong size");
+
+  assert(!Ty.isPointer() && "invalid operand type");
+
+  if (Ty.isVector()) {
+    auto Const = buildInstr(TargetOpcode::G_FCONSTANT)
+    .addDef(getMRI()->createGenericVirtualRegister(EltTy))
+    .addFPImm(&Val);
+
+    return buildSplatVector(Res, Const);
+  }
+
+  auto Const = buildInstr(TargetOpcode::G_FCONSTANT);
+  Res.addDefToMIB(*getMRI(), Const);
+  Const.addFPImm(&Val);
+  return Const;
+}
+
+MachineInstrBuilder MachineIRBuilder::buildConstant(const DstOp &Res,
+                                                    const APInt &Val) {
+  ConstantInt *CI = ConstantInt::get(getMF().getFunction().getContext(), Val);
+  return buildConstant(Res, *CI);
 }
 
 MachineInstrBuilder MachineIRBuilder::buildFConstant(const DstOp &Res,
@@ -278,7 +310,7 @@ MachineInstrBuilder MachineIRBuilder::buildFConstant(const DstOp &Res,
   LLT DstTy = Res.getLLTTy(*getMRI());
   auto &Ctx = getMF().getFunction().getContext();
   auto *CFP =
-      ConstantFP::get(Ctx, getAPFloatFromSize(Val, DstTy.getSizeInBits()));
+      ConstantFP::get(Ctx, getAPFloatFromSize(Val, DstTy.getScalarSizeInBits()));
   return buildFConstant(Res, *CFP);
 }
 
@@ -342,6 +374,25 @@ MachineInstrBuilder MachineIRBuilder::buildZExt(const DstOp &Res,
   return buildInstr(TargetOpcode::G_ZEXT, Res, Op);
 }
 
+unsigned MachineIRBuilder::getBoolExtOp(bool IsVec, bool IsFP) const {
+  const auto *TLI = getMF().getSubtarget().getTargetLowering();
+  switch (TLI->getBooleanContents(IsVec, IsFP)) {
+  case TargetLoweringBase::ZeroOrNegativeOneBooleanContent:
+    return TargetOpcode::G_SEXT;
+  case TargetLoweringBase::ZeroOrOneBooleanContent:
+    return TargetOpcode::G_ZEXT;
+  default:
+    return TargetOpcode::G_ANYEXT;
+  }
+}
+
+MachineInstrBuilder MachineIRBuilder::buildBoolExt(const DstOp &Res,
+                                                   const SrcOp &Op,
+                                                   bool IsFP) {
+  unsigned ExtOp = getBoolExtOp(getMRI()->getType(Op.getReg()).isVector(), IsFP);
+  return buildInstr(ExtOp, Res, Op);
+}
+
 MachineInstrBuilder MachineIRBuilder::buildExtOrTrunc(unsigned ExtOpc,
                                                       const DstOp &Res,
                                                       const SrcOp &Op) {
@@ -504,6 +555,12 @@ MachineInstrBuilder MachineIRBuilder::buildBuildVector(const DstOp &Res,
   return buildInstr(TargetOpcode::G_BUILD_VECTOR, Res, TmpVec);
 }
 
+MachineInstrBuilder MachineIRBuilder::buildSplatVector(const DstOp &Res,
+                                                       const SrcOp &Src) {
+  SmallVector<SrcOp, 8> TmpVec(Res.getLLTTy(*getMRI()).getNumElements(), Src);
+  return buildInstr(TargetOpcode::G_BUILD_VECTOR, Res, TmpVec);
+}
+
 MachineInstrBuilder
 MachineIRBuilder::buildBuildVectorTrunc(const DstOp &Res,
                                         ArrayRef<unsigned> Ops) {
@@ -801,11 +858,8 @@ MachineInstrBuilder MachineIRBuilder::buildInstr(unsigned Opc,
   }
   case TargetOpcode::G_ADD:
   case TargetOpcode::G_AND:
-  case TargetOpcode::G_ASHR:
-  case TargetOpcode::G_LSHR:
   case TargetOpcode::G_MUL:
   case TargetOpcode::G_OR:
-  case TargetOpcode::G_SHL:
   case TargetOpcode::G_SUB:
   case TargetOpcode::G_XOR:
   case TargetOpcode::G_UDIV:
@@ -819,6 +873,17 @@ MachineInstrBuilder MachineIRBuilder::buildInstr(unsigned Opc,
                      SrcOps[0].getLLTTy(*getMRI()),
                      SrcOps[1].getLLTTy(*getMRI()));
     break;
+  }
+  case TargetOpcode::G_SHL:
+  case TargetOpcode::G_ASHR:
+  case TargetOpcode::G_LSHR: {
+    assert(DstOps.size() == 1 && "Invalid Dst");
+    assert(SrcOps.size() == 2 && "Invalid Srcs");
+    validateShiftOp(DstOps[0].getLLTTy(*getMRI()),
+                    SrcOps[0].getLLTTy(*getMRI()),
+                    SrcOps[1].getLLTTy(*getMRI()));
+    break;
+  }
   case TargetOpcode::G_SEXT:
   case TargetOpcode::G_ZEXT:
   case TargetOpcode::G_ANYEXT:
@@ -828,7 +893,7 @@ MachineInstrBuilder MachineIRBuilder::buildInstr(unsigned Opc,
                      SrcOps[0].getLLTTy(*getMRI()), true);
     break;
   case TargetOpcode::G_TRUNC:
-  case TargetOpcode::G_FPTRUNC:
+  case TargetOpcode::G_FPTRUNC: {
     assert(DstOps.size() == 1 && "Invalid Dst");
     assert(SrcOps.size() == 1 && "Invalid Srcs");
     validateTruncExt(DstOps[0].getLLTTy(*getMRI()),
@@ -941,7 +1006,7 @@ MachineInstrBuilder MachineIRBuilder::buildInstr(unsigned Opc,
            "type mismatch in input list");
     assert(SrcOps.size() * SrcOps[0].getLLTTy(*getMRI()).getSizeInBits() ==
                DstOps[0].getLLTTy(*getMRI()).getSizeInBits() &&
-           "input scalars do not exactly cover the outpur vector register");
+           "input scalars do not exactly cover the output vector register");
     break;
   }
   case TargetOpcode::G_BUILD_VECTOR_TRUNC: {
@@ -974,7 +1039,7 @@ MachineInstrBuilder MachineIRBuilder::buildInstr(unsigned Opc,
            "type mismatch in input list");
     assert(SrcOps.size() * SrcOps[0].getLLTTy(*getMRI()).getSizeInBits() ==
                DstOps[0].getLLTTy(*getMRI()).getSizeInBits() &&
-           "input vectors do not exactly cover the outpur vector register");
+           "input vectors do not exactly cover the output vector register");
     break;
   }
   case TargetOpcode::G_UADDE: {
diff --git a/lib/CodeGen/GlobalISel/RegBankSelect.cpp b/lib/CodeGen/GlobalISel/RegBankSelect.cpp
index 98df7d34857e3bbb978a2484f62a36925b5c0226..2c231709240126a30933665c6c17c3207061cb8c 100644
--- a/lib/CodeGen/GlobalISel/RegBankSelect.cpp
+++ b/lib/CodeGen/GlobalISel/RegBankSelect.cpp
@@ -1,9 +1,8 @@
 //==- llvm/CodeGen/GlobalISel/RegBankSelect.cpp - RegBankSelect --*- C++ -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
@@ -115,7 +114,7 @@ bool RegBankSelect::assignmentMatch(
   // By default we assume we will have to repair something.
   OnlyAssign = false;
   // Each part of a break down needs to end up in a different register.
-  // In other word, Reg assignement does not match.
+  // In other word, Reg assignment does not match.
   if (ValMapping.NumBreakDowns != 1)
     return false;
 
@@ -136,33 +135,78 @@ bool RegBankSelect::repairReg(
     MachineOperand &MO, const RegisterBankInfo::ValueMapping &ValMapping,
     RegBankSelect::RepairingPlacement &RepairPt,
     const iterator_range<SmallVectorImpl<unsigned>::const_iterator> &NewVRegs) {
-  if (ValMapping.NumBreakDowns != 1 && !TPC->isGlobalISelAbortEnabled())
-    return false;
-  assert(ValMapping.NumBreakDowns == 1 && "Not yet implemented");
+
+  assert(ValMapping.NumBreakDowns == size(NewVRegs) && "need new vreg for each breakdown");
+
   // An empty range of new register means no repairing.
   assert(!empty(NewVRegs) && "We should not have to repair");
 
-  // Assume we are repairing a use and thus, the original reg will be
-  // the source of the repairing.
-  unsigned Src = MO.getReg();
-  unsigned Dst = *NewVRegs.begin();
-
-  // If we repair a definition, swap the source and destination for
-  // the repairing.
-  if (MO.isDef())
-    std::swap(Src, Dst);
-
-  assert((RepairPt.getNumInsertPoints() == 1 ||
-          TargetRegisterInfo::isPhysicalRegister(Dst)) &&
-         "We are about to create several defs for Dst");
-
-  // Build the instruction used to repair, then clone it at the right
-  // places. Avoiding buildCopy bypasses the check that Src and Dst have the
-  // same types because the type is a placeholder when this function is called.
-  MachineInstr *MI =
-      MIRBuilder.buildInstrNoInsert(TargetOpcode::COPY).addDef(Dst).addUse(Src);
-  LLVM_DEBUG(dbgs() << "Copy: " << printReg(Src) << " to: " << printReg(Dst)
-                    << '\n');
+  MachineInstr *MI;
+  if (ValMapping.NumBreakDowns == 1) {
+    // Assume we are repairing a use and thus, the original reg will be
+    // the source of the repairing.
+    unsigned Src = MO.getReg();
+    unsigned Dst = *NewVRegs.begin();
+
+    // If we repair a definition, swap the source and destination for
+    // the repairing.
+    if (MO.isDef())
+      std::swap(Src, Dst);
+
+    assert((RepairPt.getNumInsertPoints() == 1 ||
+            TargetRegisterInfo::isPhysicalRegister(Dst)) &&
+           "We are about to create several defs for Dst");
+
+    // Build the instruction used to repair, then clone it at the right
+    // places. Avoiding buildCopy bypasses the check that Src and Dst have the
+    // same types because the type is a placeholder when this function is called.
+    MI = MIRBuilder.buildInstrNoInsert(TargetOpcode::COPY)
+      .addDef(Dst)
+      .addUse(Src);
+    LLVM_DEBUG(dbgs() << "Copy: " << printReg(Src) << " to: " << printReg(Dst)
+               << '\n');
+  } else {
+    // TODO: Support with G_IMPLICIT_DEF + G_INSERT sequence or G_EXTRACT
+    // sequence.
+    assert(ValMapping.partsAllUniform() && "irregular breakdowns not supported");
+
+    LLT RegTy = MRI->getType(MO.getReg());
+    assert(!RegTy.isPointer() && "not implemented");
+
+    // FIXME: We could handle split vectors with concat_vectors easily, but this
+    // would require an agreement on the type of registers with the
+    // target. Currently createVRegs just uses scalar types, and expects the
+    // target code to replace this type (which we won't know about here)
+    assert((RegTy.isScalar() ||
+            RegTy.getNumElements() == ValMapping.NumBreakDowns) &&
+           "only basic vector breakdowns currently supported");
+
+    if (MO.isDef()) {
+      unsigned MergeOp = RegTy.isScalar() ?
+        TargetOpcode::G_MERGE_VALUES : TargetOpcode::G_BUILD_VECTOR;
+
+      auto MergeBuilder =
+        MIRBuilder.buildInstrNoInsert(MergeOp)
+        .addDef(MO.getReg());
+
+      for (unsigned SrcReg : NewVRegs)
+        MergeBuilder.addUse(SrcReg);
+
+      MI = MergeBuilder;
+    } else {
+      MachineInstrBuilder UnMergeBuilder =
+        MIRBuilder.buildInstrNoInsert(TargetOpcode::G_UNMERGE_VALUES);
+      for (unsigned DefReg : NewVRegs)
+        UnMergeBuilder.addDef(DefReg);
+
+      UnMergeBuilder.addUse(MO.getReg());
+      MI = UnMergeBuilder;
+    }
+  }
+
+  if (RepairPt.getNumInsertPoints() != 1)
+    report_fatal_error("need testcase to support multiple insertion points");
+
   // TODO:
   // Check if MI is legal. if not, we need to legalize all the
   // instructions we are going to insert.
@@ -195,7 +239,8 @@ uint64_t RegBankSelect::getRepairCost(
   const RegisterBank *CurRegBank = RBI->getRegBank(MO.getReg(), *MRI, *TRI);
   // If MO does not have a register bank, we should have just been
   // able to set one unless we have to break the value down.
-  assert((!IsSameNumOfValues || CurRegBank) && "We should not have to repair");
+  assert(CurRegBank || MO.isDef());
+
   // Def: Val <- NewDefs
   //     Same number of values: copy
   //     Different number: Val = build_sequence Defs1, Defs2, ...
@@ -206,6 +251,9 @@ uint64_t RegBankSelect::getRepairCost(
   // We should remember that this value is available somewhere else to
   // coalesce the value.
 
+  if (ValMapping.NumBreakDowns != 1)
+    return RBI->getBreakDownCost(ValMapping, CurRegBank);
+
   if (IsSameNumOfValues) {
     const RegisterBank *DesiredRegBrank = ValMapping.BreakDown[0].RegBank;
     // If we repair a definition, swap the source and destination for
@@ -528,7 +576,7 @@ RegBankSelect::MappingCost RegBankSelect::computeMapping(
 bool RegBankSelect::applyMapping(
     MachineInstr &MI, const RegisterBankInfo::InstructionMapping &InstrMapping,
     SmallVectorImpl<RegBankSelect::RepairingPlacement> &RepairPts) {
-  // OpdMapper will hold all the information needed for the rewritting.
+  // OpdMapper will hold all the information needed for the rewriting.
   RegisterBankInfo::OperandsMapper OpdMapper(MI, InstrMapping, *MRI);
 
   // First, place the repairing code.
@@ -714,18 +762,23 @@ RegBankSelect::RepairingPlacement::RepairingPlacement(
     // - Terminators must be the last instructions:
     //   * Before, move the insert point before the first terminator.
     //   * After, we have to split the outcoming edges.
-    unsigned Reg = MO.getReg();
     if (Before) {
       // Check whether Reg is defined by any terminator.
-      MachineBasicBlock::iterator It = MI;
-      for (auto Begin = MI.getParent()->begin();
-           --It != Begin && It->isTerminator();)
-        if (It->modifiesRegister(Reg, &TRI)) {
-          // Insert the repairing code right after the definition.
-          addInsertPoint(*It, /*Before*/ false);
-          return;
-        }
-      addInsertPoint(*It, /*Before*/ true);
+      MachineBasicBlock::reverse_iterator It = MI;
+      auto REnd = MI.getParent()->rend();
+
+      for (; It != REnd && It->isTerminator(); ++It) {
+        assert(!It->modifiesRegister(MO.getReg(), &TRI) &&
+               "copy insertion in middle of terminators not handled");
+      }
+
+      if (It == REnd) {
+        addInsertPoint(*MI.getParent()->begin(), true);
+        return;
+      }
+
+      // We are sure to be right before the first terminator.
+      addInsertPoint(*It, /*Before*/ false);
       return;
     }
     // Make sure Reg is not redefined by other terminators, otherwise
@@ -733,7 +786,8 @@ RegBankSelect::RepairingPlacement::RepairingPlacement(
     for (MachineBasicBlock::iterator It = MI, End = MI.getParent()->end();
          ++It != End;)
       // The machine verifier should reject this kind of code.
-      assert(It->modifiesRegister(Reg, &TRI) && "Do not know where to split");
+      assert(It->modifiesRegister(MO.getReg(), &TRI) &&
+             "Do not know where to split");
     // Split each outcoming edges.
     MachineBasicBlock &Src = *MI.getParent();
     for (auto &Succ : Src.successors())
diff --git a/lib/CodeGen/GlobalISel/RegisterBank.cpp b/lib/CodeGen/GlobalISel/RegisterBank.cpp
index 16f67a217ce15939dccdb70ca2b321c283d206d9..4e41f338934db6a59a77c0dc0c331a13ccafabef 100644
--- a/lib/CodeGen/GlobalISel/RegisterBank.cpp
+++ b/lib/CodeGen/GlobalISel/RegisterBank.cpp
@@ -1,9 +1,8 @@
 //===- llvm/CodeGen/GlobalISel/RegisterBank.cpp - Register Bank --*- C++ -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/lib/CodeGen/GlobalISel/RegisterBankInfo.cpp b/lib/CodeGen/GlobalISel/RegisterBankInfo.cpp
index 28404e52d6ea9954bbb08ad7457df69c26968e5f..1b6399452938ef551901b4facde245ec17633d17 100644
--- a/lib/CodeGen/GlobalISel/RegisterBankInfo.cpp
+++ b/lib/CodeGen/GlobalISel/RegisterBankInfo.cpp
@@ -1,9 +1,8 @@
 //===- llvm/CodeGen/GlobalISel/RegisterBankInfo.cpp --------------*- C++ -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
@@ -498,6 +497,19 @@ void RegisterBankInfo::PartialMapping::print(raw_ostream &OS) const {
     OS << "nullptr";
 }
 
+bool RegisterBankInfo::ValueMapping::partsAllUniform() const {
+  if (NumBreakDowns < 2)
+    return true;
+
+  const PartialMapping *First = begin();
+  for (const PartialMapping *Part = First + 1; Part != end(); ++Part) {
+    if (Part->Length != First->Length || Part->RegBank != First->RegBank)
+      return false;
+  }
+
+  return true;
+}
+
 bool RegisterBankInfo::ValueMapping::verify(unsigned MeaningfulBitWidth) const {
   assert(NumBreakDowns && "Value mapped nowhere?!");
   unsigned OrigValueBitWidth = 0;
diff --git a/lib/CodeGen/GlobalISel/Utils.cpp b/lib/CodeGen/GlobalISel/Utils.cpp
index 4d3a375355919833b45968145fdbcd05dd2ccadc..237d892c7f70cb4e5d24026adf3d01b7a7d2615e 100644
--- a/lib/CodeGen/GlobalISel/Utils.cpp
+++ b/lib/CodeGen/GlobalISel/Utils.cpp
@@ -1,9 +1,8 @@
 //===- llvm/CodeGen/GlobalISel/Utils.cpp -------------------------*- C++ -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file This file implements the utility functions used by the GlobalISel
@@ -235,6 +234,57 @@ APFloat llvm::getAPFloatFromSize(double Val, unsigned Size) {
   return APF;
 }
 
+Optional<APInt> llvm::ConstantFoldBinOp(unsigned Opcode, const unsigned Op1,
+                                        const unsigned Op2,
+                                        const MachineRegisterInfo &MRI) {
+  auto MaybeOp1Cst = getConstantVRegVal(Op1, MRI);
+  auto MaybeOp2Cst = getConstantVRegVal(Op2, MRI);
+  if (MaybeOp1Cst && MaybeOp2Cst) {
+    LLT Ty = MRI.getType(Op1);
+    APInt C1(Ty.getSizeInBits(), *MaybeOp1Cst, true);
+    APInt C2(Ty.getSizeInBits(), *MaybeOp2Cst, true);
+    switch (Opcode) {
+    default:
+      break;
+    case TargetOpcode::G_ADD:
+      return C1 + C2;
+    case TargetOpcode::G_AND:
+      return C1 & C2;
+    case TargetOpcode::G_ASHR:
+      return C1.ashr(C2);
+    case TargetOpcode::G_LSHR:
+      return C1.lshr(C2);
+    case TargetOpcode::G_MUL:
+      return C1 * C2;
+    case TargetOpcode::G_OR:
+      return C1 | C2;
+    case TargetOpcode::G_SHL:
+      return C1 << C2;
+    case TargetOpcode::G_SUB:
+      return C1 - C2;
+    case TargetOpcode::G_XOR:
+      return C1 ^ C2;
+    case TargetOpcode::G_UDIV:
+      if (!C2.getBoolValue())
+        break;
+      return C1.udiv(C2);
+    case TargetOpcode::G_SDIV:
+      if (!C2.getBoolValue())
+        break;
+      return C1.sdiv(C2);
+    case TargetOpcode::G_UREM:
+      if (!C2.getBoolValue())
+        break;
+      return C1.urem(C2);
+    case TargetOpcode::G_SREM:
+      if (!C2.getBoolValue())
+        break;
+      return C1.srem(C2);
+    }
+  }
+  return None;
+}
+
 void llvm::getSelectionDAGFallbackAnalysisUsage(AnalysisUsage &AU) {
   AU.addPreserved<StackProtector>();
 }
diff --git a/lib/CodeGen/GlobalMerge.cpp b/lib/CodeGen/GlobalMerge.cpp
index d3364952f2444a726c5d6abb2d2de0d10edcde51..8b25d486dbc330cecc6e391dbadb6de581205335 100644
--- a/lib/CodeGen/GlobalMerge.cpp
+++ b/lib/CodeGen/GlobalMerge.cpp
@@ -1,9 +1,8 @@
 //===- GlobalMerge.cpp - Internal globals merging -------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/IfConversion.cpp b/lib/CodeGen/IfConversion.cpp
index 5666626ab311f8837973cb892af2bbe2c96c6ebb..f1ab6815eb95d1509a5732ae29ffeb1bfbecea41 100644
--- a/lib/CodeGen/IfConversion.cpp
+++ b/lib/CodeGen/IfConversion.cpp
@@ -1,9 +1,8 @@
 //===- IfConversion.cpp - Machine code if conversion pass -----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -1444,7 +1443,7 @@ bool IfConverter::IfConvertSimple(BBInfo &BBI, IfcvtKind Kind) {
   Redefs.init(*TRI);
 
   if (MRI->tracksLiveness()) {
-    // Initialize liveins to the first BB. These are potentiall redefined by
+    // Initialize liveins to the first BB. These are potentially redefined by
     // predicated instructions.
     Redefs.addLiveIns(CvtMBB);
     Redefs.addLiveIns(NextMBB);
@@ -2148,7 +2147,7 @@ void IfConverter::MergeBlocks(BBInfo &ToBBI, BBInfo &FromBBI, bool AddEdges) {
       // Calculate the edge probability for the edge from ToBBI.BB to Succ,
       // which is a portion of the edge probability from FromMBB to Succ. The
       // portion ratio is the edge probability from ToBBI.BB to FromMBB (if
-      // FromBBI is a successor of ToBBI.BB. See comment below for excepion).
+      // FromBBI is a successor of ToBBI.BB. See comment below for exception).
       NewProb = MBPI->getEdgeProbability(&FromMBB, Succ);
 
       // To2FromProb is 0 when FromMBB is not a successor of ToBBI.BB. This
diff --git a/lib/CodeGen/ImplicitNullChecks.cpp b/lib/CodeGen/ImplicitNullChecks.cpp
index f411ee6745d0ed6f23326ea902a0d595de5fe6f4..75f4eef5dc1c3091e4b41224807f949a598ef345 100644
--- a/lib/CodeGen/ImplicitNullChecks.cpp
+++ b/lib/CodeGen/ImplicitNullChecks.cpp
@@ -1,9 +1,8 @@
 //===- ImplicitNullChecks.cpp - Fold null checks into memory accesses -----===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -236,8 +235,11 @@ bool ImplicitNullChecks::canHandle(const MachineInstr *MI) {
   assert(!llvm::any_of(MI->operands(), IsRegMask) &&
          "Calls were filtered out above!");
 
-  auto IsUnordered = [](MachineMemOperand *MMO) { return MMO->isUnordered(); };
-  return llvm::all_of(MI->memoperands(), IsUnordered);
+  // TODO: This should be isUnordered (see D57601) once test cases are written
+  // demonstrating that.
+  auto IsSimple = [](MachineMemOperand *MMO) {
+    return !MMO->isVolatile() && !MMO->isAtomic(); };
+  return llvm::all_of(MI->memoperands(), IsSimple);
 }
 
 ImplicitNullChecks::DependenceResult
diff --git a/lib/CodeGen/IndirectBrExpandPass.cpp b/lib/CodeGen/IndirectBrExpandPass.cpp
index 7b05ebf820fd1d68b325179bff960c1fecf73ad0..7ac093ba4a71479df704ceeb1a048af57641a149 100644
--- a/lib/CodeGen/IndirectBrExpandPass.cpp
+++ b/lib/CodeGen/IndirectBrExpandPass.cpp
@@ -1,9 +1,8 @@
 //===- IndirectBrExpandPass.cpp - Expand indirectbr to switch -------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
@@ -149,11 +148,9 @@ bool IndirectBrExpandPass::runOnFunction(Function &F) {
     ConstantInt *BBIndexC = ConstantInt::get(ITy, BBIndex);
 
     // Now rewrite the blockaddress to an integer constant based on the index.
-    // FIXME: We could potentially preserve the uses as arguments to inline asm.
-    // This would allow some uses such as diagnostic information in crashes to
-    // have higher quality even when this transform is enabled, but would break
-    // users that round-trip blockaddresses through inline assembly and then
-    // back into an indirectbr.
+    // FIXME: This part doesn't properly recognize other uses of blockaddress
+    // expressions, for instance, where they are used to pass labels to
+    // asm-goto. This part of the pass needs a rework.
     BA->replaceAllUsesWith(ConstantExpr::getIntToPtr(BBIndexC, BA->getType()));
   }
 
diff --git a/lib/CodeGen/InlineSpiller.cpp b/lib/CodeGen/InlineSpiller.cpp
index 007e9283d833d22709957f80a054250672f45370..647168b1c33eb2d459be012a03e5d54ea201c5cf 100644
--- a/lib/CodeGen/InlineSpiller.cpp
+++ b/lib/CodeGen/InlineSpiller.cpp
@@ -1,9 +1,8 @@
 //===- InlineSpiller.cpp - Insert spills and restores inline --------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -76,6 +75,10 @@ STATISTIC(NumRemats,          "Number of rematerialized defs for spilling");
 
 static cl::opt<bool> DisableHoisting("disable-spill-hoist", cl::Hidden,
                                      cl::desc("Disable inline spill hoisting"));
+static cl::opt<bool>
+RestrictStatepointRemat("restrict-statepoint-remat",
+                       cl::init(false), cl::Hidden,
+                       cl::desc("Restrict remat for statepoint operands"));
 
 namespace {
 
@@ -215,6 +218,7 @@ private:
   void eliminateRedundantSpills(LiveInterval &LI, VNInfo *VNI);
 
   void markValueUsed(LiveInterval*, VNInfo*);
+  bool canGuaranteeAssignmentAfterRemat(unsigned VReg, MachineInstr &MI);
   bool reMaterializeFor(LiveInterval &, MachineInstr &MI);
   void reMaterializeAll();
 
@@ -514,6 +518,28 @@ void InlineSpiller::markValueUsed(LiveInterval *LI, VNInfo *VNI) {
   } while (!WorkList.empty());
 }
 
+bool InlineSpiller::canGuaranteeAssignmentAfterRemat(unsigned VReg,
+                                                     MachineInstr &MI) {
+  if (!RestrictStatepointRemat)
+    return true;
+  // Here's a quick explanation of the problem we're trying to handle here:
+  // * There are some pseudo instructions with more vreg uses than there are
+  //   physical registers on the machine.
+  // * This is normally handled by spilling the vreg, and folding the reload
+  //   into the user instruction.  (Thus decreasing the number of used vregs
+  //   until the remainder can be assigned to physregs.)
+  // * However, since we may try to spill vregs in any order, we can end up
+  //   trying to spill each operand to the instruction, and then rematting it
+  //   instead.  When that happens, the new live intervals (for the remats) are
+  //   expected to be trivially assignable (i.e. RS_Done).  However, since we
+  //   may have more remats than physregs, we're guaranteed to fail to assign
+  //   one.
+  // At the moment, we only handle this for STATEPOINTs since they're the only
+  // psuedo op where we've seen this.  If we start seeing other instructions
+  // with the same problem, we need to revisit this.
+  return (MI.getOpcode() != TargetOpcode::STATEPOINT);
+}
+
 /// reMaterializeFor - Attempt to rematerialize before MI instead of reloading.
 bool InlineSpiller::reMaterializeFor(LiveInterval &VirtReg, MachineInstr &MI) {
   // Analyze instruction
@@ -569,6 +595,14 @@ bool InlineSpiller::reMaterializeFor(LiveInterval &VirtReg, MachineInstr &MI) {
     return true;
   }
 
+  // If we can't guarantee that we'll be able to actually assign the new vreg,
+  // we can't remat.
+  if (!canGuaranteeAssignmentAfterRemat(VirtReg.reg, MI)) {
+    markValueUsed(&VirtReg, ParentVNI);
+    LLVM_DEBUG(dbgs() << "\tcannot remat for " << UseIdx << '\t' << MI);
+    return false;
+  }
+
   // Allocate a new register for the remat.
   unsigned NewVReg = Edit->createFrom(Original);
 
diff --git a/lib/CodeGen/InterferenceCache.cpp b/lib/CodeGen/InterferenceCache.cpp
index 82f6e8d8e23435f3a0457bc445bfada114aac75b..7b50dac4cd1a78bd33899774b223bb3be24c63a7 100644
--- a/lib/CodeGen/InterferenceCache.cpp
+++ b/lib/CodeGen/InterferenceCache.cpp
@@ -1,9 +1,8 @@
 //===- InterferenceCache.cpp - Caching per-block interference -------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/InterferenceCache.h b/lib/CodeGen/InterferenceCache.h
index 160e2b16e2943a995a05a2f38c2074eb0cba3539..50c6ac62d194156d982e2bc16611280704980db9 100644
--- a/lib/CodeGen/InterferenceCache.h
+++ b/lib/CodeGen/InterferenceCache.h
@@ -1,9 +1,8 @@
 //===- InterferenceCache.h - Caching per-block interference ----*- C++ -*--===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/InterleavedAccessPass.cpp b/lib/CodeGen/InterleavedAccessPass.cpp
index fd2ff162630a8912a1f43264b80abc993163f57f..2f8012aeeefb06af5e99e6111bf8bbd1654cb1ff 100644
--- a/lib/CodeGen/InterleavedAccessPass.cpp
+++ b/lib/CodeGen/InterleavedAccessPass.cpp
@@ -1,9 +1,8 @@
 //===- InterleavedAccessPass.cpp ------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/InterleavedLoadCombinePass.cpp b/lib/CodeGen/InterleavedLoadCombinePass.cpp
index 989fa164ad2dd2ef3887e1353f8647019b909aac..5524924f869dd597c8cd8f3ea8eff35bd55766ed 100644
--- a/lib/CodeGen/InterleavedLoadCombinePass.cpp
+++ b/lib/CodeGen/InterleavedLoadCombinePass.cpp
@@ -1,9 +1,8 @@
 //===- InterleavedLoadCombine.cpp - Combine Interleaved Loads ---*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -1219,7 +1218,7 @@ bool InterleavedLoadCombineImpl::combine(std::list<VectorInfo> &InterleavedLoad,
                                       "interleaved.wide.ptrcast");
 
   // Create the wide load and update the MemorySSA.
-  auto LI = Builder.CreateAlignedLoad(CI, InsertionPoint->getAlignment(),
+  auto LI = Builder.CreateAlignedLoad(ILTy, CI, InsertionPoint->getAlignment(),
                                       "interleaved.wide.load");
   auto MSSAU = MemorySSAUpdater(&MSSA);
   MemoryUse *MSSALoad = cast<MemoryUse>(MSSAU.createMemoryAccessBefore(
diff --git a/lib/CodeGen/IntrinsicLowering.cpp b/lib/CodeGen/IntrinsicLowering.cpp
index 707113bd973b742bfb692857cead20e2406432e4..8cbd8bcaeabbb6e3cdd27b50fd613dbce3cb4867 100644
--- a/lib/CodeGen/IntrinsicLowering.cpp
+++ b/lib/CodeGen/IntrinsicLowering.cpp
@@ -1,9 +1,8 @@
 //===-- IntrinsicLowering.cpp - Intrinsic Lowering default implementation -===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -24,39 +23,6 @@
 #include "llvm/Support/raw_ostream.h"
 using namespace llvm;
 
-template <class ArgIt>
-static void EnsureFunctionExists(Module &M, const char *Name,
-                                 ArgIt ArgBegin, ArgIt ArgEnd,
-                                 Type *RetTy) {
-  // Insert a correctly-typed definition now.
-  std::vector<Type *> ParamTys;
-  for (ArgIt I = ArgBegin; I != ArgEnd; ++I)
-    ParamTys.push_back(I->getType());
-  M.getOrInsertFunction(Name, FunctionType::get(RetTy, ParamTys, false));
-}
-
-static void EnsureFPIntrinsicsExist(Module &M, Function &Fn,
-                                    const char *FName,
-                                    const char *DName, const char *LDName) {
-  // Insert definitions for all the floating point types.
-  switch((int)Fn.arg_begin()->getType()->getTypeID()) {
-  case Type::FloatTyID:
-    EnsureFunctionExists(M, FName, Fn.arg_begin(), Fn.arg_end(),
-                         Type::getFloatTy(M.getContext()));
-    break;
-  case Type::DoubleTyID:
-    EnsureFunctionExists(M, DName, Fn.arg_begin(), Fn.arg_end(),
-                         Type::getDoubleTy(M.getContext()));
-    break;
-  case Type::X86_FP80TyID:
-  case Type::FP128TyID:
-  case Type::PPC_FP128TyID:
-    EnsureFunctionExists(M, LDName, Fn.arg_begin(), Fn.arg_end(),
-                         Fn.arg_begin()->getType());
-    break;
-  }
-}
-
 /// This function is used when we want to lower an intrinsic call to a call of
 /// an external function. This handles hard cases such as when there was already
 /// a prototype for the external function, but that prototype doesn't match the
@@ -72,8 +38,8 @@ static CallInst *ReplaceCallWith(const char *NewFn, CallInst *CI,
   std::vector<Type *> ParamTys;
   for (ArgIt I = ArgBegin; I != ArgEnd; ++I)
     ParamTys.push_back((*I)->getType());
-  Constant* FCache = M->getOrInsertFunction(NewFn,
-                                  FunctionType::get(RetTy, ParamTys, false));
+  FunctionCallee FCache =
+      M->getOrInsertFunction(NewFn, FunctionType::get(RetTy, ParamTys, false));
 
   IRBuilder<> Builder(CI->getParent(), CI->getIterator());
   SmallVector<Value *, 8> Args(ArgBegin, ArgEnd);
@@ -92,75 +58,6 @@ static CallInst *ReplaceCallWith(const char *NewFn, CallInst *CI,
 #  define setjmp_undefined_for_msvc
 #endif
 
-void IntrinsicLowering::AddPrototypes(Module &M) {
-  LLVMContext &Context = M.getContext();
-  for (auto &F : M)
-    if (F.isDeclaration() && !F.use_empty())
-      switch (F.getIntrinsicID()) {
-      default: break;
-      case Intrinsic::setjmp:
-        EnsureFunctionExists(M, "setjmp", F.arg_begin(), F.arg_end(),
-                             Type::getInt32Ty(M.getContext()));
-        break;
-      case Intrinsic::longjmp:
-        EnsureFunctionExists(M, "longjmp", F.arg_begin(), F.arg_end(),
-                             Type::getVoidTy(M.getContext()));
-        break;
-      case Intrinsic::siglongjmp:
-        EnsureFunctionExists(M, "abort", F.arg_end(), F.arg_end(),
-                             Type::getVoidTy(M.getContext()));
-        break;
-      case Intrinsic::memcpy:
-        M.getOrInsertFunction("memcpy",
-          Type::getInt8PtrTy(Context),
-                              Type::getInt8PtrTy(Context),
-                              Type::getInt8PtrTy(Context),
-                              DL.getIntPtrType(Context));
-        break;
-      case Intrinsic::memmove:
-        M.getOrInsertFunction("memmove",
-          Type::getInt8PtrTy(Context),
-                              Type::getInt8PtrTy(Context),
-                              Type::getInt8PtrTy(Context),
-                              DL.getIntPtrType(Context));
-        break;
-      case Intrinsic::memset:
-        M.getOrInsertFunction("memset",
-          Type::getInt8PtrTy(Context),
-                              Type::getInt8PtrTy(Context),
-                              Type::getInt32Ty(M.getContext()),
-                              DL.getIntPtrType(Context));
-        break;
-      case Intrinsic::sqrt:
-        EnsureFPIntrinsicsExist(M, F, "sqrtf", "sqrt", "sqrtl");
-        break;
-      case Intrinsic::sin:
-        EnsureFPIntrinsicsExist(M, F, "sinf", "sin", "sinl");
-        break;
-      case Intrinsic::cos:
-        EnsureFPIntrinsicsExist(M, F, "cosf", "cos", "cosl");
-        break;
-      case Intrinsic::pow:
-        EnsureFPIntrinsicsExist(M, F, "powf", "pow", "powl");
-        break;
-      case Intrinsic::log:
-        EnsureFPIntrinsicsExist(M, F, "logf", "log", "logl");
-        break;
-      case Intrinsic::log2:
-        EnsureFPIntrinsicsExist(M, F, "log2f", "log2", "log2l");
-        break;
-      case Intrinsic::log10:
-        EnsureFPIntrinsicsExist(M, F, "log10f", "log10", "log10l");
-        break;
-      case Intrinsic::exp:
-        EnsureFPIntrinsicsExist(M, F, "expf", "exp", "expl");
-        break;
-      case Intrinsic::exp2:
-        EnsureFPIntrinsicsExist(M, F, "exp2f", "exp2", "exp2l");
-        break;
-      }
-}
-
 /// Emit the code to lower bswap of V before the specified instruction IP.
 static Value *LowerBSWAP(LLVMContext &Context, Value *V, Instruction *IP) {
   assert(V->getType()->isIntOrIntVectorTy() && "Can't bswap a non-integer type!");
@@ -601,7 +498,7 @@ bool IntrinsicLowering::LowerToByteSwap(CallInst *CI) {
 
   // Okay, we can do this xform, do so now.
   Module *M = CI->getModule();
-  Constant *Int = Intrinsic::getDeclaration(M, Intrinsic::bswap, Ty);
+  Function *Int = Intrinsic::getDeclaration(M, Intrinsic::bswap, Ty);
 
   Value *Op = CI->getArgOperand(0);
   Op = CallInst::Create(Int, Op, CI->getName(), CI);
diff --git a/lib/CodeGen/LLVMBuild.txt b/lib/CodeGen/LLVMBuild.txt
index 07ea9dcaea7aef2422a6b29014492c7755d841c5..0a766fd7aca60fa5b759018985e9192c8af5b2e8 100644
--- a/lib/CodeGen/LLVMBuild.txt
+++ b/lib/CodeGen/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/CodeGen/LLVMBuild.txt ------------------------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/CodeGen/LLVMTargetMachine.cpp b/lib/CodeGen/LLVMTargetMachine.cpp
index 52e832cc38c1cd3e819d68915a341da2391c4bfe..1d10a6c9afd583bebc82dc1d7292e8b6277836f8 100644
--- a/lib/CodeGen/LLVMTargetMachine.cpp
+++ b/lib/CodeGen/LLVMTargetMachine.cpp
@@ -1,9 +1,8 @@
 //===-- LLVMTargetMachine.cpp - Implement the LLVMTargetMachine class -----===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/LatencyPriorityQueue.cpp b/lib/CodeGen/LatencyPriorityQueue.cpp
index f9f33a98a9d1c392f3b145eb6a69362932dbfd58..8a7a41d0f763a5f9c4375c3e7d7142010f78cfa3 100644
--- a/lib/CodeGen/LatencyPriorityQueue.cpp
+++ b/lib/CodeGen/LatencyPriorityQueue.cpp
@@ -1,9 +1,8 @@
 //===---- LatencyPriorityQueue.cpp - A latency-oriented priority queue ----===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/LazyMachineBlockFrequencyInfo.cpp b/lib/CodeGen/LazyMachineBlockFrequencyInfo.cpp
index 5b52cc66a2971a4e5b1153bc47e4e0c828ce78ae..200ac0ba15bfbfa0df73c165e3db1cdd639b215c 100644
--- a/lib/CodeGen/LazyMachineBlockFrequencyInfo.cpp
+++ b/lib/CodeGen/LazyMachineBlockFrequencyInfo.cpp
@@ -1,9 +1,8 @@
 ///===- LazyMachineBlockFrequencyInfo.cpp - Lazy Machine Block Frequency --===//
 ///
-///                     The LLVM Compiler Infrastructure
-///
-/// This file is distributed under the University of Illinois Open Source
-/// License. See LICENSE.TXT for details.
+/// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+/// See https://llvm.org/LICENSE.txt for license information.
+/// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ///
 ///===---------------------------------------------------------------------===//
 /// \file
diff --git a/lib/CodeGen/LexicalScopes.cpp b/lib/CodeGen/LexicalScopes.cpp
index d06821bdfcce7a43abd9ca3c4c3ea92dbdcccea0..503821537ed9263a6f9981982be2ba4c8a36140e 100644
--- a/lib/CodeGen/LexicalScopes.cpp
+++ b/lib/CodeGen/LexicalScopes.cpp
@@ -1,9 +1,8 @@
 //===- LexicalScopes.cpp - Collecting lexical scope info ------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/LiveDebugValues.cpp b/lib/CodeGen/LiveDebugValues.cpp
index fc0ebea2d36c1fe0d3dfdbb8d52f8bb6f4403d93..449f10c7ef721d30a1ff562d9872370a7484138d 100644
--- a/lib/CodeGen/LiveDebugValues.cpp
+++ b/lib/CodeGen/LiveDebugValues.cpp
@@ -1,9 +1,8 @@
 //===- LiveDebugValues.cpp - Tracking Debug Value MIs ---------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
@@ -35,13 +34,14 @@
 #include "llvm/CodeGen/MachineMemOperand.h"
 #include "llvm/CodeGen/MachineOperand.h"
 #include "llvm/CodeGen/PseudoSourceValue.h"
+#include "llvm/CodeGen/RegisterScavenging.h"
 #include "llvm/CodeGen/TargetFrameLowering.h"
 #include "llvm/CodeGen/TargetInstrInfo.h"
 #include "llvm/CodeGen/TargetLowering.h"
 #include "llvm/CodeGen/TargetRegisterInfo.h"
 #include "llvm/CodeGen/TargetSubtargetInfo.h"
-#include "llvm/CodeGen/RegisterScavenging.h"
 #include "llvm/Config/llvm-config.h"
+#include "llvm/IR/DIBuilder.h"
 #include "llvm/IR/DebugInfoMetadata.h"
 #include "llvm/IR/DebugLoc.h"
 #include "llvm/IR/Function.h"
@@ -86,6 +86,8 @@ private:
   BitVector CalleeSavedRegs;
   LexicalScopes LS;
 
+  enum struct TransferKind { TransferCopy, TransferSpill, TransferRestore };
+
   /// Keeps track of lexical scopes associated with a user value's source
   /// location.
   class UserValueScopes {
@@ -125,15 +127,30 @@ private:
 
   /// A pair of debug variable and value location.
   struct VarLoc {
+    // The location at which a spilled variable resides. It consists of a
+    // register and an offset.
+    struct SpillLoc {
+      unsigned SpillBase;
+      int SpillOffset;
+      bool operator==(const SpillLoc &Other) const {
+        return SpillBase == Other.SpillBase && SpillOffset == Other.SpillOffset;
+      }
+    };
+
     const DebugVariable Var;
     const MachineInstr &MI; ///< Only used for cloning a new DBG_VALUE.
     mutable UserValueScopes UVS;
-    enum { InvalidKind = 0, RegisterKind } Kind = InvalidKind;
+    enum VarLocKind {
+      InvalidKind = 0,
+      RegisterKind,
+      SpillLocKind
+    } Kind = InvalidKind;
 
     /// The value location. Stored separately to avoid repeatedly
     /// extracting it from MI.
     union {
       uint64_t RegNo;
+      SpillLoc SpillLocation;
       uint64_t Hash;
     } Loc;
 
@@ -150,6 +167,17 @@ private:
       }
     }
 
+    /// The constructor for spill locations.
+    VarLoc(const MachineInstr &MI, unsigned SpillBase, int SpillOffset,
+           LexicalScopes &LS)
+        : Var(MI.getDebugVariable(), MI.getDebugLoc()->getInlinedAt()), MI(MI),
+          UVS(MI.getDebugLoc(), LS) {
+      assert(MI.isDebugValue() && "not a DBG_VALUE");
+      assert(MI.getNumOperands() == 4 && "malformed DBG_VALUE");
+      Kind = SpillLocKind;
+      Loc.SpillLocation = {SpillBase, SpillOffset};
+    }
+
     /// If this variable is described by a register, return it,
     /// otherwise return 0.
     unsigned isDescribedByReg() const {
@@ -237,15 +265,23 @@ private:
 
   bool isSpillInstruction(const MachineInstr &MI, MachineFunction *MF,
                           unsigned &Reg);
-  int extractSpillBaseRegAndOffset(const MachineInstr &MI, unsigned &Reg);
+  /// If a given instruction is identified as a spill, return the spill location
+  /// and set \p Reg to the spilled register.
+  Optional<VarLoc::SpillLoc> isRestoreInstruction(const MachineInstr &MI,
+                                                  MachineFunction *MF,
+                                                  unsigned &Reg);
+  /// Given a spill instruction, extract the register and offset used to
+  /// address the spill location in a target independent way.
+  VarLoc::SpillLoc extractSpillBaseRegAndOffset(const MachineInstr &MI);
   void insertTransferDebugPair(MachineInstr &MI, OpenRangesSet &OpenRanges,
                                TransferMap &Transfers, VarLocMap &VarLocIDs,
-                               unsigned OldVarID, unsigned NewReg = 0);
+                               unsigned OldVarID, TransferKind Kind,
+                               unsigned NewReg = 0);
 
   void transferDebugValue(const MachineInstr &MI, OpenRangesSet &OpenRanges,
                           VarLocMap &VarLocIDs);
-  void transferSpillInst(MachineInstr &MI, OpenRangesSet &OpenRanges,
-                         VarLocMap &VarLocIDs, TransferMap &Transfers);
+  void transferSpillOrRestoreInst(MachineInstr &MI, OpenRangesSet &OpenRanges,
+                                  VarLocMap &VarLocIDs, TransferMap &Transfers);
   void transferRegisterCopy(MachineInstr &MI, OpenRangesSet &OpenRanges,
                             VarLocMap &VarLocIDs, TransferMap &Transfers);
   void transferRegisterDef(MachineInstr &MI, OpenRangesSet &OpenRanges,
@@ -339,10 +375,8 @@ void LiveDebugValues::printVarLocInMBB(const MachineFunction &MF,
 }
 #endif
 
-/// Given a spill instruction, extract the register and offset used to
-/// address the spill location in a target independent way.
-int LiveDebugValues::extractSpillBaseRegAndOffset(const MachineInstr &MI,
-                                                  unsigned &Reg) {
+LiveDebugValues::VarLoc::SpillLoc
+LiveDebugValues::extractSpillBaseRegAndOffset(const MachineInstr &MI) {
   assert(MI.hasOneMemOperand() &&
          "Spill instruction does not have exactly one memory operand?");
   auto MMOI = MI.memoperands_begin();
@@ -351,7 +385,9 @@ int LiveDebugValues::extractSpillBaseRegAndOffset(const MachineInstr &MI,
          "Inconsistent memory operand in spill instruction");
   int FI = cast<FixedStackPseudoSourceValue>(PVal)->getFrameIndex();
   const MachineBasicBlock *MBB = MI.getParent();
-  return TFI->getFrameIndexReference(*MBB->getParent(), FI, Reg);
+  unsigned Reg;
+  int Offset = TFI->getFrameIndexReference(*MBB->getParent(), FI, Reg);
+  return {Reg, Offset};
 }
 
 /// End all previous ranges related to @MI and start a new range from @MI
@@ -387,11 +423,28 @@ void LiveDebugValues::transferDebugValue(const MachineInstr &MI,
 /// otherwise it is variable's location on the stack.
 void LiveDebugValues::insertTransferDebugPair(
     MachineInstr &MI, OpenRangesSet &OpenRanges, TransferMap &Transfers,
-    VarLocMap &VarLocIDs, unsigned OldVarID, unsigned NewReg) {
+    VarLocMap &VarLocIDs, unsigned OldVarID, TransferKind Kind,
+    unsigned NewReg) {
   const MachineInstr *DMI = &VarLocIDs[OldVarID].MI;
   MachineFunction *MF = MI.getParent()->getParent();
   MachineInstr *NewDMI;
-  if (NewReg) {
+
+  auto ProcessVarLoc = [&MI, &OpenRanges, &Transfers,
+                        &VarLocIDs](VarLoc &VL, MachineInstr *NewDMI) {
+    unsigned LocId = VarLocIDs.insert(VL);
+    OpenRanges.insert(LocId, VL.Var);
+    // The newly created DBG_VALUE instruction NewDMI must be inserted after
+    // MI. Keep track of the pairing.
+    TransferDebugPair MIP = {&MI, NewDMI};
+    Transfers.push_back(MIP);
+  };
+
+  // End all previous ranges of Var.
+  OpenRanges.erase(VarLocIDs[OldVarID].Var);
+  switch (Kind) {
+  case TransferKind::TransferCopy: {
+    assert(NewReg &&
+           "No register supplied when handling a copy of a debug value");
     // Create a DBG_VALUE instruction to describe the Var in its new
     // register location.
     NewDMI = BuildMI(*MF, DMI->getDebugLoc(), DMI->getDesc(),
@@ -399,33 +452,43 @@ void LiveDebugValues::insertTransferDebugPair(
                      DMI->getDebugVariable(), DMI->getDebugExpression());
     if (DMI->isIndirectDebugValue())
       NewDMI->getOperand(1).setImm(DMI->getOperand(1).getImm());
+    VarLoc VL(*NewDMI, LS);
+    ProcessVarLoc(VL, NewDMI);
     LLVM_DEBUG(dbgs() << "Creating DBG_VALUE inst for register copy: ";
                NewDMI->print(dbgs(), false, false, false, TII));
-  } else {
+    return;
+  }
+  case TransferKind::TransferSpill: {
     // Create a DBG_VALUE instruction to describe the Var in its spilled
     // location.
-    unsigned SpillBase;
-    int SpillOffset = extractSpillBaseRegAndOffset(MI, SpillBase);
-    auto *SpillExpr = DIExpression::prepend(DMI->getDebugExpression(),
-                                            DIExpression::NoDeref, SpillOffset);
-    NewDMI = BuildMI(*MF, DMI->getDebugLoc(), DMI->getDesc(), true, SpillBase,
-                     DMI->getDebugVariable(), SpillExpr);
+    VarLoc::SpillLoc SpillLocation = extractSpillBaseRegAndOffset(MI);
+    auto *SpillExpr =
+        DIExpression::prepend(DMI->getDebugExpression(), DIExpression::NoDeref,
+                              SpillLocation.SpillOffset);
+    NewDMI =
+        BuildMI(*MF, DMI->getDebugLoc(), DMI->getDesc(), true,
+                SpillLocation.SpillBase, DMI->getDebugVariable(), SpillExpr);
+    VarLoc VL(*NewDMI, SpillLocation.SpillBase, SpillLocation.SpillOffset, LS);
+    ProcessVarLoc(VL, NewDMI);
     LLVM_DEBUG(dbgs() << "Creating DBG_VALUE inst for spill: ";
                NewDMI->print(dbgs(), false, false, false, TII));
+    return;
   }
-
-  // The newly created DBG_VALUE instruction NewDMI must be inserted after
-  // MI. Keep track of the pairing.
-  TransferDebugPair MIP = {&MI, NewDMI};
-  Transfers.push_back(MIP);
-
-  // End all previous ranges of Var.
-  OpenRanges.erase(VarLocIDs[OldVarID].Var);
-
-  // Add the VarLoc to OpenRanges.
-  VarLoc VL(*NewDMI, LS);
-  unsigned LocID = VarLocIDs.insert(VL);
-  OpenRanges.insert(LocID, VL.Var);
+  case TransferKind::TransferRestore: {
+    assert(NewReg &&
+           "No register supplied when handling a restore of a debug value");
+    MachineFunction *MF = MI.getMF();
+    DIBuilder DIB(*const_cast<Function &>(MF->getFunction()).getParent());
+    NewDMI = BuildMI(*MF, DMI->getDebugLoc(), DMI->getDesc(), false, NewReg,
+                     DMI->getDebugVariable(), DIB.createExpression());
+    VarLoc VL(*NewDMI, LS);
+    ProcessVarLoc(VL, NewDMI);
+    LLVM_DEBUG(dbgs() << "Creating DBG_VALUE inst for register restore: ";
+               NewDMI->print(dbgs(), false, false, false, TII));
+    return;
+  }
+  }
+  llvm_unreachable("Invalid transfer kind");
 }
 
 /// A definition of a register may mark the end of a range.
@@ -471,24 +534,15 @@ void LiveDebugValues::transferRegisterDef(MachineInstr &MI,
 /// other spills). We do not handle this yet (more than one memory operand).
 bool LiveDebugValues::isSpillInstruction(const MachineInstr &MI,
                                          MachineFunction *MF, unsigned &Reg) {
-  const MachineFrameInfo &FrameInfo = MF->getFrameInfo();
-  int FI;
   SmallVector<const MachineMemOperand*, 1> Accesses;
 
   // TODO: Handle multiple stores folded into one.
   if (!MI.hasOneMemOperand())
     return false;
 
-  // To identify a spill instruction, use the same criteria as in AsmPrinter.
-  if (!((TII->isStoreToStackSlotPostFE(MI, FI) &&
-         FrameInfo.isSpillSlotObjectIndex(FI)) ||
-        (TII->hasStoreToStackSlot(MI, Accesses) &&
-         llvm::any_of(Accesses, [&FrameInfo](const MachineMemOperand *MMO) {
-           return FrameInfo.isSpillSlotObjectIndex(
-               cast<FixedStackPseudoSourceValue>(MMO->getPseudoValue())
-                   ->getFrameIndex());
-         }))))
-    return false;
+  if (!MI.getSpillSize(TII) && !MI.getFoldedSpillSize(TII))
+    return false; // This is not a spill instruction, since no valid size was
+                  // returned from either function.
 
   auto isKilledReg = [&](const MachineOperand MO, unsigned &Reg) {
     if (!MO.isReg() || !MO.isUse()) {
@@ -525,29 +579,67 @@ bool LiveDebugValues::isSpillInstruction(const MachineInstr &MI,
   return false;
 }
 
+Optional<LiveDebugValues::VarLoc::SpillLoc>
+LiveDebugValues::isRestoreInstruction(const MachineInstr &MI,
+                                      MachineFunction *MF, unsigned &Reg) {
+  if (!MI.hasOneMemOperand())
+    return None;
+
+  // FIXME: Handle folded restore instructions with more than one memory
+  // operand.
+  if (MI.getRestoreSize(TII)) {
+    Reg = MI.getOperand(0).getReg();
+    return extractSpillBaseRegAndOffset(MI);
+  }
+  return None;
+}
+
 /// A spilled register may indicate that we have to end the current range of
 /// a variable and create a new one for the spill location.
+/// A restored register may indicate the reverse situation.
 /// We don't want to insert any instructions in process(), so we just create
 /// the DBG_VALUE without inserting it and keep track of it in \p Transfers.
 /// It will be inserted into the BB when we're done iterating over the
 /// instructions.
-void LiveDebugValues::transferSpillInst(MachineInstr &MI,
-                                        OpenRangesSet &OpenRanges,
-                                        VarLocMap &VarLocIDs,
-                                        TransferMap &Transfers) {
-  unsigned Reg;
+void LiveDebugValues::transferSpillOrRestoreInst(MachineInstr &MI,
+                                                 OpenRangesSet &OpenRanges,
+                                                 VarLocMap &VarLocIDs,
+                                                 TransferMap &Transfers) {
   MachineFunction *MF = MI.getMF();
-  if (!isSpillInstruction(MI, MF, Reg))
-    return;
+  TransferKind TKind;
+  unsigned Reg;
+  Optional<VarLoc::SpillLoc> Loc;
 
-  // Check if the register is the location of a debug value.
+  LLVM_DEBUG(dbgs() << "Examining instruction: "; MI.dump(););
+
+  if (isSpillInstruction(MI, MF, Reg)) {
+    TKind = TransferKind::TransferSpill;
+    LLVM_DEBUG(dbgs() << "Recognized as spill: "; MI.dump(););
+    LLVM_DEBUG(dbgs() << "Register: " << Reg << " " << printReg(Reg, TRI)
+                      << "\n");
+  } else {
+    if (!(Loc = isRestoreInstruction(MI, MF, Reg)))
+      return;
+    TKind = TransferKind::TransferRestore;
+    LLVM_DEBUG(dbgs() << "Recognized as restore: "; MI.dump(););
+    LLVM_DEBUG(dbgs() << "Register: " << Reg << " " << printReg(Reg, TRI)
+                      << "\n");
+  }
+  // Check if the register or spill location is the location of a debug value.
   for (unsigned ID : OpenRanges.getVarLocs()) {
-    if (VarLocIDs[ID].isDescribedByReg() == Reg) {
+    if (TKind == TransferKind::TransferSpill &&
+        VarLocIDs[ID].isDescribedByReg() == Reg) {
       LLVM_DEBUG(dbgs() << "Spilling Register " << printReg(Reg, TRI) << '('
                         << VarLocIDs[ID].Var.getVar()->getName() << ")\n");
-      insertTransferDebugPair(MI, OpenRanges, Transfers, VarLocIDs, ID);
-      return;
-    }
+    } else if (TKind == TransferKind::TransferRestore &&
+               VarLocIDs[ID].Loc.SpillLocation == *Loc) {
+      LLVM_DEBUG(dbgs() << "Restoring Register " << printReg(Reg, TRI) << '('
+                        << VarLocIDs[ID].Var.getVar()->getName() << ")\n");
+    } else
+      continue;
+    insertTransferDebugPair(MI, OpenRanges, Transfers, VarLocIDs, ID, TKind,
+                            Reg);
+    return;
   }
 }
 
@@ -585,7 +677,7 @@ void LiveDebugValues::transferRegisterCopy(MachineInstr &MI,
   for (unsigned ID : OpenRanges.getVarLocs()) {
     if (VarLocIDs[ID].isDescribedByReg() == SrcReg) {
       insertTransferDebugPair(MI, OpenRanges, Transfers, VarLocIDs, ID,
-                              DestReg);
+                              TransferKind::TransferCopy, DestReg);
       return;
     }
   }
@@ -625,7 +717,7 @@ bool LiveDebugValues::process(MachineInstr &MI, OpenRangesSet &OpenRanges,
   transferRegisterDef(MI, OpenRanges, VarLocIDs);
   if (transferChanges) {
     transferRegisterCopy(MI, OpenRanges, VarLocIDs, Transfers);
-    transferSpillInst(MI, OpenRanges, VarLocIDs, Transfers);
+    transferSpillOrRestoreInst(MI, OpenRanges, VarLocIDs, Transfers);
   }
   Changed = transferTerminatorInst(MI, OpenRanges, OutLocs, VarLocIDs);
   return Changed;
diff --git a/lib/CodeGen/LiveDebugVariables.cpp b/lib/CodeGen/LiveDebugVariables.cpp
index d0d889782a358e9c6e4c37eddd313ffabb238559..f5afa15bdb29e9dcaacd7342f8fd336ae7bdede5 100644
--- a/lib/CodeGen/LiveDebugVariables.cpp
+++ b/lib/CodeGen/LiveDebugVariables.cpp
@@ -1,9 +1,8 @@
 //===- LiveDebugVariables.cpp - Tracking debug info variables -------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -71,6 +70,7 @@ EnableLDV("live-debug-variables", cl::init(true),
           cl::desc("Enable the live debug variables pass"), cl::Hidden);
 
 STATISTIC(NumInsertedDebugValues, "Number of DBG_VALUEs inserted");
+STATISTIC(NumInsertedDebugLabels, "Number of DBG_LABELs inserted");
 
 char LiveDebugVariables::ID = 0;
 
@@ -339,6 +339,37 @@ public:
   void print(raw_ostream &, const TargetRegisterInfo *);
 };
 
+/// A user label is a part of a debug info user label.
+class UserLabel {
+  const DILabel *Label; ///< The debug info label we are part of.
+  DebugLoc dl;          ///< The debug location for the label. This is
+                        ///< used by dwarf writer to find lexical scope.
+  SlotIndex loc;        ///< Slot used by the debug label.
+
+  /// Insert a DBG_LABEL into MBB at Idx.
+  void insertDebugLabel(MachineBasicBlock *MBB, SlotIndex Idx,
+                        LiveIntervals &LIS, const TargetInstrInfo &TII);
+
+public:
+  /// Create a new UserLabel.
+  UserLabel(const DILabel *label, DebugLoc L, SlotIndex Idx)
+      : Label(label), dl(std::move(L)), loc(Idx) {}
+
+  /// Does this UserLabel match the parameters?
+  bool match(const DILabel *L, const DILocation *IA,
+             const SlotIndex Index) const {
+    return Label == L && dl->getInlinedAt() == IA && loc == Index;
+  }
+
+  /// Recreate DBG_LABEL instruction from data structures.
+  void emitDebugLabel(LiveIntervals &LIS, const TargetInstrInfo &TII);
+
+  /// Return DebugLoc of this UserLabel.
+  DebugLoc getDebugLoc() { return dl; }
+
+  void print(raw_ostream &, const TargetRegisterInfo *);
+};
+
 /// Implementation of the LiveDebugVariables pass.
 class LDVImpl {
   LiveDebugVariables &pass;
@@ -356,6 +387,9 @@ class LDVImpl {
   /// All allocated UserValue instances.
   SmallVector<std::unique_ptr<UserValue>, 8> userValues;
 
+  /// All allocated UserLabel instances.
+  SmallVector<std::unique_ptr<UserLabel>, 2> userLabels;
+
   /// Map virtual register to eq class leader.
   using VRMap = DenseMap<unsigned, UserValue *>;
   VRMap virtRegToEqClass;
@@ -379,6 +413,14 @@ class LDVImpl {
   /// \returns True if the DBG_VALUE instruction should be deleted.
   bool handleDebugValue(MachineInstr &MI, SlotIndex Idx);
 
+  /// Add DBG_LABEL instruction to UserLabel.
+  ///
+  /// \param MI DBG_LABEL instruction
+  /// \param Idx Last valid SlotIndex before instruction.
+  ///
+  /// \returns True if the DBG_LABEL instruction should be deleted.
+  bool handleDebugLabel(MachineInstr &MI, SlotIndex Idx);
+
   /// Collect and erase all DBG_VALUE instructions, adding a UserValue def
   /// for each instruction.
   ///
@@ -400,6 +442,7 @@ public:
   void clear() {
     MF = nullptr;
     userValues.clear();
+    userLabels.clear();
     virtRegToEqClass.clear();
     userVarMap.clear();
     // Make sure we call emitDebugValues if the machine function was modified.
@@ -445,12 +488,21 @@ static void printDebugLoc(const DebugLoc &DL, raw_ostream &CommentOS,
   CommentOS << " ]";
 }
 
-static void printExtendedName(raw_ostream &OS, const DILocalVariable *V,
+static void printExtendedName(raw_ostream &OS, const DINode *Node,
                               const DILocation *DL) {
-  const LLVMContext &Ctx = V->getContext();
-  StringRef Res = V->getName();
+  const LLVMContext &Ctx = Node->getContext();
+  StringRef Res;
+  unsigned Line;
+  if (const auto *V = dyn_cast<const DILocalVariable>(Node)) {
+    Res = V->getName();
+    Line = V->getLine();
+  } else if (const auto *L = dyn_cast<const DILabel>(Node)) {
+    Res = L->getName();
+    Line = L->getLine();
+  }
+
   if (!Res.empty())
-    OS << Res << "," << V->getLine();
+    OS << Res << "," << Line;
   if (auto *InlinedAt = DL->getInlinedAt()) {
     if (DebugLoc InlinedAtDL = InlinedAt) {
       OS << " @[";
@@ -461,9 +513,8 @@ static void printExtendedName(raw_ostream &OS, const DILocalVariable *V,
 }
 
 void UserValue::print(raw_ostream &OS, const TargetRegisterInfo *TRI) {
-  auto *DV = cast<DILocalVariable>(Variable);
   OS << "!\"";
-  printExtendedName(OS, DV, dl);
+  printExtendedName(OS, Variable, dl);
 
   OS << "\"\t";
   for (LocMap::const_iterator I = locInts.begin(); I.valid(); ++I) {
@@ -483,10 +534,22 @@ void UserValue::print(raw_ostream &OS, const TargetRegisterInfo *TRI) {
   OS << '\n';
 }
 
+void UserLabel::print(raw_ostream &OS, const TargetRegisterInfo *TRI) {
+  OS << "!\"";
+  printExtendedName(OS, Label, dl);
+
+  OS << "\"\t";
+  OS << loc;
+  OS << '\n';
+}
+
 void LDVImpl::print(raw_ostream &OS) {
   OS << "********** DEBUG VARIABLES **********\n";
-  for (unsigned i = 0, e = userValues.size(); i != e; ++i)
-    userValues[i]->print(OS, TRI);
+  for (auto &userValue : userValues)
+    userValue->print(OS, TRI);
+  OS << "********** DEBUG LABELS **********\n";
+  for (auto &userLabel : userLabels)
+    userLabel->print(OS, TRI);
 }
 #endif
 
@@ -587,6 +650,29 @@ bool LDVImpl::handleDebugValue(MachineInstr &MI, SlotIndex Idx) {
   return true;
 }
 
+bool LDVImpl::handleDebugLabel(MachineInstr &MI, SlotIndex Idx) {
+  // DBG_LABEL label
+  if (MI.getNumOperands() != 1 || !MI.getOperand(0).isMetadata()) {
+    LLVM_DEBUG(dbgs() << "Can't handle " << MI);
+    return false;
+  }
+
+  // Get or create the UserLabel for label here.
+  const DILabel *Label = MI.getDebugLabel();
+  const DebugLoc &DL = MI.getDebugLoc();
+  bool Found = false;
+  for (auto const &L : userLabels) {
+    if (L->match(Label, DL->getInlinedAt(), Idx)) {
+      Found = true;
+      break;
+    }
+  }
+  if (!Found)
+    userLabels.push_back(llvm::make_unique<UserLabel>(Label, DL, Idx));
+
+  return true;
+}
+
 bool LDVImpl::collectDebugValues(MachineFunction &mf) {
   bool Changed = false;
   for (MachineFunction::iterator MFI = mf.begin(), MFE = mf.end(); MFI != MFE;
@@ -610,7 +696,8 @@ bool LDVImpl::collectDebugValues(MachineFunction &mf) {
       do {
         // Only handle DBG_VALUE in handleDebugValue(). Skip all other
         // kinds of debug instructions.
-        if (MBBI->isDebugValue() && handleDebugValue(*MBBI, Idx)) {
+        if ((MBBI->isDebugValue() && handleDebugValue(*MBBI, Idx)) ||
+            (MBBI->isDebugLabel() && handleDebugLabel(*MBBI, Idx))) {
           MBBI = MBB->erase(MBBI);
           Changed = true;
         } else
@@ -1247,6 +1334,15 @@ void UserValue::insertDebugValue(MachineBasicBlock *MBB, SlotIndex StartIdx,
   } while (I != MBB->end());
 }
 
+void UserLabel::insertDebugLabel(MachineBasicBlock *MBB, SlotIndex Idx,
+                                 LiveIntervals &LIS,
+                                 const TargetInstrInfo &TII) {
+  MachineBasicBlock::iterator I = findInsertLocation(MBB, Idx, LIS);
+  ++NumInsertedDebugLabels;
+  BuildMI(*MBB, I, getDebugLoc(), TII.get(TargetOpcode::DBG_LABEL))
+      .addMetadata(Label);
+}
+
 void UserValue::emitDebugValues(VirtRegMap *VRM, LiveIntervals &LIS,
                                 const TargetInstrInfo &TII,
                                 const TargetRegisterInfo &TRI,
@@ -1295,16 +1391,31 @@ void UserValue::emitDebugValues(VirtRegMap *VRM, LiveIntervals &LIS,
   }
 }
 
+void UserLabel::emitDebugLabel(LiveIntervals &LIS, const TargetInstrInfo &TII) {
+  LLVM_DEBUG(dbgs() << "\t" << loc);
+  MachineFunction::iterator MBB = LIS.getMBBFromIndex(loc)->getIterator();
+
+  LLVM_DEBUG(dbgs() << ' ' << printMBBReference(*MBB));
+  insertDebugLabel(&*MBB, loc, LIS, TII);
+
+  LLVM_DEBUG(dbgs() << '\n');
+}
+
 void LDVImpl::emitDebugValues(VirtRegMap *VRM) {
   LLVM_DEBUG(dbgs() << "********** EMITTING LIVE DEBUG VARIABLES **********\n");
   if (!MF)
     return;
   const TargetInstrInfo *TII = MF->getSubtarget().getInstrInfo();
   SpillOffsetMap SpillOffsets;
-  for (unsigned i = 0, e = userValues.size(); i != e; ++i) {
-    LLVM_DEBUG(userValues[i]->print(dbgs(), TRI));
-    userValues[i]->rewriteLocations(*VRM, *MF, *TII, *TRI, SpillOffsets);
-    userValues[i]->emitDebugValues(VRM, *LIS, *TII, *TRI, SpillOffsets);
+  for (auto &userValue : userValues) {
+    LLVM_DEBUG(userValue->print(dbgs(), TRI));
+    userValue->rewriteLocations(*VRM, *MF, *TII, *TRI, SpillOffsets);
+    userValue->emitDebugValues(VRM, *LIS, *TII, *TRI, SpillOffsets);
+  }
+  LLVM_DEBUG(dbgs() << "********** EMITTING LIVE DEBUG LABELS **********\n");
+  for (auto &userLabel : userLabels) {
+    LLVM_DEBUG(userLabel->print(dbgs(), TRI));
+    userLabel->emitDebugLabel(*LIS, *TII);
   }
   EmitDone = true;
 }
diff --git a/lib/CodeGen/LiveDebugVariables.h b/lib/CodeGen/LiveDebugVariables.h
index 0060399c2b04526f837a7fca1b2ab63778a198cd..0cbe10c6a42274cdc10bce6aaee0b1b4b038418c 100644
--- a/lib/CodeGen/LiveDebugVariables.h
+++ b/lib/CodeGen/LiveDebugVariables.h
@@ -1,9 +1,8 @@
 //===- LiveDebugVariables.h - Tracking debug info variables -----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/LiveInterval.cpp b/lib/CodeGen/LiveInterval.cpp
index 2340b6abd87c8f44e92799f17ece00b91111d663..821680e756c725786b40f563232ac80c2a10858a 100644
--- a/lib/CodeGen/LiveInterval.cpp
+++ b/lib/CodeGen/LiveInterval.cpp
@@ -1,9 +1,8 @@
 //===- LiveInterval.cpp - Live Interval Representation --------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/LiveIntervalUnion.cpp b/lib/CodeGen/LiveIntervalUnion.cpp
index 36428e0335f945c38c52502368b02f78a4ed1029..43fa8f2d7157a7ee1da3929c2357b9c2e1118dbb 100644
--- a/lib/CodeGen/LiveIntervalUnion.cpp
+++ b/lib/CodeGen/LiveIntervalUnion.cpp
@@ -1,9 +1,8 @@
 //===- LiveIntervalUnion.cpp - Live interval union data structure ---------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/LiveIntervals.cpp b/lib/CodeGen/LiveIntervals.cpp
index 471775f8706b8580638ed5a0aa52e84d972a5711..8706bdb09fc5a1749c3607a2ccaf1a7686869167 100644
--- a/lib/CodeGen/LiveIntervals.cpp
+++ b/lib/CodeGen/LiveIntervals.cpp
@@ -1,9 +1,8 @@
 //===- LiveIntervals.cpp - Live Interval Analysis -------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/LivePhysRegs.cpp b/lib/CodeGen/LivePhysRegs.cpp
index 619643acb6d37853988ef386997b18045ee74520..cd3d248ac8788dcfd231f444052762c829690b1d 100644
--- a/lib/CodeGen/LivePhysRegs.cpp
+++ b/lib/CodeGen/LivePhysRegs.cpp
@@ -1,9 +1,8 @@
 //===--- LivePhysRegs.cpp - Live Physical Register Set --------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/LiveRangeCalc.cpp b/lib/CodeGen/LiveRangeCalc.cpp
index 70e135ab1aff98dfe631a2d2b62d61da1b39279d..0020221f4d6b7010a6fa1b800009d930ffa39a62 100644
--- a/lib/CodeGen/LiveRangeCalc.cpp
+++ b/lib/CodeGen/LiveRangeCalc.cpp
@@ -1,9 +1,8 @@
 //===- LiveRangeCalc.cpp - Calculate live ranges --------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/LiveRangeCalc.h b/lib/CodeGen/LiveRangeCalc.h
index 9f226b154a670bfda72089e9ee512ab08003a6a3..11aea5a3b016ec21095755c6156d83cd5797a279 100644
--- a/lib/CodeGen/LiveRangeCalc.h
+++ b/lib/CodeGen/LiveRangeCalc.h
@@ -1,9 +1,8 @@
 //===- LiveRangeCalc.h - Calculate live ranges ------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/LiveRangeEdit.cpp b/lib/CodeGen/LiveRangeEdit.cpp
index 8dfe8b68c3af0348f440fd432a69a595549d255d..860217796ead8a6306343a498dea8fb498d60084 100644
--- a/lib/CodeGen/LiveRangeEdit.cpp
+++ b/lib/CodeGen/LiveRangeEdit.cpp
@@ -1,9 +1,8 @@
 //===-- LiveRangeEdit.cpp - Basic tools for editing a register live range -===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/LiveRangeShrink.cpp b/lib/CodeGen/LiveRangeShrink.cpp
index f75d513c89f53971c6c4cce05b70a924254c3ad7..8818f1ce0ad9c0d70ed63a167a835e6c73f7bc76 100644
--- a/lib/CodeGen/LiveRangeShrink.cpp
+++ b/lib/CodeGen/LiveRangeShrink.cpp
@@ -1,9 +1,8 @@
 //===- LiveRangeShrink.cpp - Move instructions to shrink live range -------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 ///===---------------------------------------------------------------------===//
 ///
diff --git a/lib/CodeGen/LiveRangeUtils.h b/lib/CodeGen/LiveRangeUtils.h
index bd57609c3d849eceffc099de3787b74240682260..0e6bfeb0d4a59eb7fff3bf2c5d99c5a81cecf7dd 100644
--- a/lib/CodeGen/LiveRangeUtils.h
+++ b/lib/CodeGen/LiveRangeUtils.h
@@ -1,9 +1,8 @@
 //===-- LiveRangeUtils.h - Live Range modification utilities ----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/LiveRegMatrix.cpp b/lib/CodeGen/LiveRegMatrix.cpp
index e72977b02675acd67dff4b73a37f0252e349e987..ce99e5535c2553bd22aef5fb34ce1533fc03b1ae 100644
--- a/lib/CodeGen/LiveRegMatrix.cpp
+++ b/lib/CodeGen/LiveRegMatrix.cpp
@@ -1,9 +1,8 @@
 //===- LiveRegMatrix.cpp - Track register interference --------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/LiveRegUnits.cpp b/lib/CodeGen/LiveRegUnits.cpp
index c226813854922e847eed6b51563895483037a0dd..6afb7fb7aa11556411c2748b880c1364eb37131b 100644
--- a/lib/CodeGen/LiveRegUnits.cpp
+++ b/lib/CodeGen/LiveRegUnits.cpp
@@ -1,9 +1,8 @@
 //===- LiveRegUnits.cpp - Register Unit Set -------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -126,13 +125,15 @@ void LiveRegUnits::addPristines(const MachineFunction &MF) {
 
 void LiveRegUnits::addLiveOuts(const MachineBasicBlock &MBB) {
   const MachineFunction &MF = *MBB.getParent();
-  if (!MBB.succ_empty()) {
-    addPristines(MF);
-    // To get the live-outs we simply merge the live-ins of all successors.
-    for (const MachineBasicBlock *Succ : MBB.successors())
-      addBlockLiveIns(*this, *Succ);
-  } else if (MBB.isReturnBlock()) {
-    // For the return block: Add all callee saved registers.
+
+  addPristines(MF);
+
+  // To get the live-outs we simply merge the live-ins of all successors.
+  for (const MachineBasicBlock *Succ : MBB.successors())
+    addBlockLiveIns(*this, *Succ);
+
+  // For the return block: Add all callee saved registers.
+  if (MBB.isReturnBlock()) {
     const MachineFrameInfo &MFI = MF.getFrameInfo();
     if (MFI.isCalleeSavedInfoValid())
       addCalleeSavedRegs(*this, MF);
diff --git a/lib/CodeGen/LiveStacks.cpp b/lib/CodeGen/LiveStacks.cpp
index 80ecfdb7a507ef0a7ac044cb354ebd9555f48157..f55977d727238cbd057d9e2c9307478a603eaaf0 100644
--- a/lib/CodeGen/LiveStacks.cpp
+++ b/lib/CodeGen/LiveStacks.cpp
@@ -1,9 +1,8 @@
 //===-- LiveStacks.cpp - Live Stack Slot Analysis -------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/LiveVariables.cpp b/lib/CodeGen/LiveVariables.cpp
index 0b92eab83806a03335e3db0d3e1d154287182183..fd0ea35aadaca968746a7a84aa3053bb22d26dd4 100644
--- a/lib/CodeGen/LiveVariables.cpp
+++ b/lib/CodeGen/LiveVariables.cpp
@@ -1,9 +1,8 @@
 //===-- LiveVariables.cpp - Live Variable Analysis for Machine Code -------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/LocalStackSlotAllocation.cpp b/lib/CodeGen/LocalStackSlotAllocation.cpp
index 795028e9792983f7ca9de2b10f6f1da70eb7e719..bddd0c7732cdc3769b6a106a39f016c3e4e20037 100644
--- a/lib/CodeGen/LocalStackSlotAllocation.cpp
+++ b/lib/CodeGen/LocalStackSlotAllocation.cpp
@@ -1,9 +1,8 @@
 //===- LocalStackSlotAllocation.cpp - Pre-allocate locals to stack slots --===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/LoopTraversal.cpp b/lib/CodeGen/LoopTraversal.cpp
index a02d10e09d7d986aec095536b34d0193bb7c95a3..9490dfc40a82c7745248abe2fc81d4f6b5fb6223 100644
--- a/lib/CodeGen/LoopTraversal.cpp
+++ b/lib/CodeGen/LoopTraversal.cpp
@@ -1,9 +1,8 @@
 //===- LoopTraversal.cpp - Optimal basic block traversal order --*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/CodeGen/LowLevelType.cpp b/lib/CodeGen/LowLevelType.cpp
index 1c682e72fa49145aaf795fb67f0b271683d6d9f0..ca0daa14fedf6d9c7779382e0057033a2e5648fd 100644
--- a/lib/CodeGen/LowLevelType.cpp
+++ b/lib/CodeGen/LowLevelType.cpp
@@ -1,9 +1,8 @@
 //===-- llvm/CodeGen/LowLevelType.cpp -------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/LowerEmuTLS.cpp b/lib/CodeGen/LowerEmuTLS.cpp
index 36c1d358a9bd556dcc793f7f8e80aa17db4fb8f6..c8cf6abda4fcc08aa3e70c9883925363523a2f52 100644
--- a/lib/CodeGen/LowerEmuTLS.cpp
+++ b/lib/CodeGen/LowerEmuTLS.cpp
@@ -1,9 +1,8 @@
 //===- LowerEmuTLS.cpp - Add __emutls_[vt].* variables --------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/MIRCanonicalizerPass.cpp b/lib/CodeGen/MIRCanonicalizerPass.cpp
index f17c23619ed58b7c7a5184c4e0bcbf2e8abe8e29..e7db863e1f9f827d99d55096875ef10f9fdbd1be 100644
--- a/lib/CodeGen/MIRCanonicalizerPass.cpp
+++ b/lib/CodeGen/MIRCanonicalizerPass.cpp
@@ -1,9 +1,8 @@
 //===-------------- MIRCanonicalizer.cpp - MIR Canonicalizer --------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/MIRParser/LLVMBuild.txt b/lib/CodeGen/MIRParser/LLVMBuild.txt
index 6269b005a9853d3d148cc82d90da82f915a7de13..3c3e2f99e4f8db45208c552ff2ff64d62bbe2d06 100644
--- a/lib/CodeGen/MIRParser/LLVMBuild.txt
+++ b/lib/CodeGen/MIRParser/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/CodeGen/MIRParser/LLVMBuild.txt --------------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/CodeGen/MIRParser/MILexer.cpp b/lib/CodeGen/MIRParser/MILexer.cpp
index 265877c2f5b42a073e36498714bb0a0f8c31cdab..c274e282f59a485666619eace874e0d3f45ad1b7 100644
--- a/lib/CodeGen/MIRParser/MILexer.cpp
+++ b/lib/CodeGen/MIRParser/MILexer.cpp
@@ -1,9 +1,8 @@
 //===- MILexer.cpp - Machine instructions lexer implementation ------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/MIRParser/MILexer.h b/lib/CodeGen/MIRParser/MILexer.h
index ceff79087d814c1b7691054eefff22a30fdccddb..1fccf98633b61cba206669c1ccb39c2256db2285 100644
--- a/lib/CodeGen/MIRParser/MILexer.h
+++ b/lib/CodeGen/MIRParser/MILexer.h
@@ -1,9 +1,8 @@
 //===- MILexer.h - Lexer for machine instructions ---------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/MIRParser/MIParser.cpp b/lib/CodeGen/MIRParser/MIParser.cpp
index 6f2d8bb53ac82b848809447b49c8ef7d270ff6e6..2ee4f78259779c092f6448cde45954cc724c96bf 100644
--- a/lib/CodeGen/MIRParser/MIParser.cpp
+++ b/lib/CodeGen/MIRParser/MIParser.cpp
@@ -1,9 +1,8 @@
 //===- MIParser.cpp - Machine instructions parser implementation ----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -1341,6 +1340,19 @@ bool MIParser::parseIRConstant(StringRef::iterator Loc, const Constant *&C) {
   return false;
 }
 
+// See LLT implemntation for bit size limits.
+static bool verifyScalarSize(uint64_t Size) {
+  return Size != 0 && isUInt<16>(Size);
+}
+
+static bool verifyVectorElementCount(uint64_t NumElts) {
+  return NumElts != 0 && isUInt<16>(NumElts);
+}
+
+static bool verifyAddrSpace(uint64_t AddrSpace) {
+  return isUInt<24>(AddrSpace);
+}
+
 bool MIParser::parseLowLevelType(StringRef::iterator Loc, LLT &Ty) {
   if (Token.range().front() == 's' || Token.range().front() == 'p') {
     StringRef SizeStr = Token.range().drop_front();
@@ -1349,12 +1361,19 @@ bool MIParser::parseLowLevelType(StringRef::iterator Loc, LLT &Ty) {
   }
 
   if (Token.range().front() == 's') {
-    Ty = LLT::scalar(APSInt(Token.range().drop_front()).getZExtValue());
+    auto ScalarSize = APSInt(Token.range().drop_front()).getZExtValue();
+    if (!verifyScalarSize(ScalarSize))
+      return error("invalid size for scalar type");
+
+    Ty = LLT::scalar(ScalarSize);
     lex();
     return false;
   } else if (Token.range().front() == 'p') {
     const DataLayout &DL = MF.getDataLayout();
-    unsigned AS = APSInt(Token.range().drop_front()).getZExtValue();
+    uint64_t AS = APSInt(Token.range().drop_front()).getZExtValue();
+    if (!verifyAddrSpace(AS))
+      return error("invalid address space number");
+
     Ty = LLT::pointer(AS, DL.getPointerSizeInBits(AS));
     lex();
     return false;
@@ -1369,6 +1388,9 @@ bool MIParser::parseLowLevelType(StringRef::iterator Loc, LLT &Ty) {
   if (Token.isNot(MIToken::IntegerLiteral))
     return error(Loc, "expected <M x sN> or <M x pA> for vector type");
   uint64_t NumElements = Token.integerValue().getZExtValue();
+  if (!verifyVectorElementCount(NumElements))
+    return error("invalid number of vector elements");
+
   lex();
 
   if (Token.isNot(MIToken::Identifier) || Token.stringValue() != "x")
@@ -1381,11 +1403,17 @@ bool MIParser::parseLowLevelType(StringRef::iterator Loc, LLT &Ty) {
   if (SizeStr.size() == 0 || !llvm::all_of(SizeStr, isdigit))
     return error("expected integers after 's'/'p' type character");
 
-  if (Token.range().front() == 's')
-    Ty = LLT::scalar(APSInt(Token.range().drop_front()).getZExtValue());
-  else if (Token.range().front() == 'p') {
+  if (Token.range().front() == 's') {
+    auto ScalarSize = APSInt(Token.range().drop_front()).getZExtValue();
+    if (!verifyScalarSize(ScalarSize))
+      return error("invalid size for scalar type");
+    Ty = LLT::scalar(ScalarSize);
+  } else if (Token.range().front() == 'p') {
     const DataLayout &DL = MF.getDataLayout();
-    unsigned AS = APSInt(Token.range().drop_front()).getZExtValue();
+    uint64_t AS = APSInt(Token.range().drop_front()).getZExtValue();
+    if (!verifyAddrSpace(AS))
+      return error("invalid address space number");
+
     Ty = LLT::pointer(AS, DL.getPointerSizeInBits(AS));
   } else
     return error(Loc, "expected <M x sN> or <M x pA> for vector type");
@@ -2325,6 +2353,10 @@ bool MIParser::parseAlignment(unsigned &Alignment) {
   if (getUnsigned(Alignment))
     return true;
   lex();
+
+  if (!isPowerOf2_32(Alignment))
+    return error("expected a power-of-2 literal after 'align'");
+
   return false;
 }
 
diff --git a/lib/CodeGen/MIRParser/MIParser.h b/lib/CodeGen/MIRParser/MIParser.h
index b06ceb21b740728cba84ac705a523df5b224a908..cc9f83c260157a1bc763ee4b398230b098f8712c 100644
--- a/lib/CodeGen/MIRParser/MIParser.h
+++ b/lib/CodeGen/MIRParser/MIParser.h
@@ -1,9 +1,8 @@
 //===- MIParser.h - Machine Instructions Parser -----------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/MIRParser/MIRParser.cpp b/lib/CodeGen/MIRParser/MIRParser.cpp
index 00da92a92ec62326093102cd50b966ded9a426fc..2fc53d78290139295f5938054ce3c3c46af6b342 100644
--- a/lib/CodeGen/MIRParser/MIRParser.cpp
+++ b/lib/CodeGen/MIRParser/MIRParser.cpp
@@ -1,9 +1,8 @@
 //===- MIRParser.cpp - MIR serialization format parser implementation -----===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -271,8 +270,9 @@ bool MIRParserImpl::parseMachineFunctions(Module &M, MachineModuleInfo &MMI) {
 /// Create an empty function with the given name.
 static Function *createDummyFunction(StringRef Name, Module &M) {
   auto &Context = M.getContext();
-  Function *F = cast<Function>(M.getOrInsertFunction(
-      Name, FunctionType::get(Type::getVoidTy(Context), false)));
+  Function *F =
+      Function::Create(FunctionType::get(Type::getVoidTy(Context), false),
+                       Function::ExternalLinkage, Name, M);
   BasicBlock *BB = BasicBlock::Create(Context, "entry", F);
   new UnreachableInst(Context, BB);
   return F;
diff --git a/lib/CodeGen/MIRPrinter.cpp b/lib/CodeGen/MIRPrinter.cpp
index d9dcc428943f19cb1a93cce21fd3979cf59543fe..79392569a799203210b8fbfcd6c5c897673f09c1 100644
--- a/lib/CodeGen/MIRPrinter.cpp
+++ b/lib/CodeGen/MIRPrinter.cpp
@@ -1,9 +1,8 @@
 //===- MIRPrinter.cpp - MIR serialization format printer ------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/MIRPrintingPass.cpp b/lib/CodeGen/MIRPrintingPass.cpp
index 1a8427430ea099e35d0cca49b16885df3d70d863..e032fffd658c843ce034568b7b23b8bd4c4bd6e5 100644
--- a/lib/CodeGen/MIRPrintingPass.cpp
+++ b/lib/CodeGen/MIRPrintingPass.cpp
@@ -1,9 +1,8 @@
 //===- MIRPrintingPass.cpp - Pass that prints out using the MIR format ----===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/MachineBasicBlock.cpp b/lib/CodeGen/MachineBasicBlock.cpp
index 03771bc5dae10471e0154bd0a7b461b8e865e83b..c3e9c185be9a4f1a9a09021f8efbdbf71e82fdf5 100644
--- a/lib/CodeGen/MachineBasicBlock.cpp
+++ b/lib/CodeGen/MachineBasicBlock.cpp
@@ -1,9 +1,8 @@
 //===-- llvm/CodeGen/MachineBasicBlock.cpp ----------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -133,8 +132,12 @@ void ilist_traits<MachineInstr>::transferNodesFromList(ilist_traits &FromList,
                                                        instr_iterator First,
                                                        instr_iterator Last) {
   assert(Parent->getParent() == FromList.Parent->getParent() &&
-        "MachineInstr parent mismatch!");
-  assert(this != &FromList && "Called without a real transfer...");
+         "cannot transfer MachineInstrs between MachineFunctions");
+
+  // If it's within the same BB, there's nothing to do.
+  if (this == &FromList)
+    return;
+
   assert(Parent != FromList.Parent && "Two lists have the same parent?");
 
   // If splicing between two blocks within the same function, just update the
diff --git a/lib/CodeGen/MachineBlockFrequencyInfo.cpp b/lib/CodeGen/MachineBlockFrequencyInfo.cpp
index 3459a9f71a73c0506cc448e16bbc23ef2127da20..53a35b7e89c2dff193ff1a8f341176d1126947dd 100644
--- a/lib/CodeGen/MachineBlockFrequencyInfo.cpp
+++ b/lib/CodeGen/MachineBlockFrequencyInfo.cpp
@@ -1,9 +1,8 @@
 //===- MachineBlockFrequencyInfo.cpp - MBB Frequency Analysis -------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/MachineBlockPlacement.cpp b/lib/CodeGen/MachineBlockPlacement.cpp
index 624d3365b4fd3f628434af5cc487ea7aeb145f7d..a246717cad615308d62621f49a087bff962954fe 100644
--- a/lib/CodeGen/MachineBlockPlacement.cpp
+++ b/lib/CodeGen/MachineBlockPlacement.cpp
@@ -1,9 +1,8 @@
 //===- MachineBlockPlacement.cpp - Basic Block Code Layout optimization ---===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -316,7 +315,7 @@ class MachineBlockPlacement : public MachineFunctionPass {
   /// A type for a block filter set.
   using BlockFilterSet = SmallSetVector<const MachineBasicBlock *, 16>;
 
-  /// Pair struct containing basic block and taildup profitiability
+  /// Pair struct containing basic block and taildup profitability
   struct BlockAndTailDupResult {
     MachineBasicBlock *BB;
     bool ShouldTailDup;
@@ -452,6 +451,8 @@ class MachineBlockPlacement : public MachineFunctionPass {
 
   void buildChain(const MachineBasicBlock *BB, BlockChain &Chain,
                   BlockFilterSet *BlockFilter = nullptr);
+  bool canMoveBottomBlockToTop(const MachineBasicBlock *BottomBlock,
+                               const MachineBasicBlock *OldTop);
   MachineBasicBlock *findBestLoopTop(
       const MachineLoop &L, const BlockFilterSet &LoopBlockSet);
   MachineBasicBlock *findBestLoopExit(
@@ -1757,6 +1758,39 @@ void MachineBlockPlacement::buildChain(
                     << getBlockName(*Chain.begin()) << "\n");
 }
 
+// If bottom of block BB has only one successor OldTop, in most cases it is
+// profitable to move it before OldTop, except the following case:
+//
+//     -->OldTop<-
+//     |    .    |
+//     |    .    |
+//     |    .    |
+//     ---Pred   |
+//          |    |
+//         BB-----
+//
+// If BB is moved before OldTop, Pred needs a taken branch to BB, and it can't
+// layout the other successor below it, so it can't reduce taken branch.
+// In this case we keep its original layout.
+bool
+MachineBlockPlacement::canMoveBottomBlockToTop(
+    const MachineBasicBlock *BottomBlock,
+    const MachineBasicBlock *OldTop) {
+  if (BottomBlock->pred_size() != 1)
+    return true;
+  MachineBasicBlock *Pred = *BottomBlock->pred_begin();
+  if (Pred->succ_size() != 2)
+    return true;
+
+  MachineBasicBlock *OtherBB = *Pred->succ_begin();
+  if (OtherBB == BottomBlock)
+    OtherBB = *Pred->succ_rbegin();
+  if (OtherBB == OldTop)
+    return false;
+
+  return true;
+}
+
 /// Find the best loop top block for layout.
 ///
 /// Look for a block which is strictly better than the loop header for laying
@@ -1801,6 +1835,9 @@ MachineBlockPlacement::findBestLoopTop(const MachineLoop &L,
     if (Pred->succ_size() > 1)
       continue;
 
+    if (!canMoveBottomBlockToTop(Pred, L.getHeader()))
+      continue;
+
     BlockFrequency PredFreq = MBFI->getBlockFreq(Pred);
     if (!BestPred || PredFreq > BestPredFreq ||
         (!(PredFreq < BestPredFreq) &&
diff --git a/lib/CodeGen/MachineBranchProbabilityInfo.cpp b/lib/CodeGen/MachineBranchProbabilityInfo.cpp
index e4952aaaba06fbf384d0ccd2e3e3886ed9865710..d2277ce517462fd506d880cee9ac5c5850b770fd 100644
--- a/lib/CodeGen/MachineBranchProbabilityInfo.cpp
+++ b/lib/CodeGen/MachineBranchProbabilityInfo.cpp
@@ -1,9 +1,8 @@
 //===- MachineBranchProbabilityInfo.cpp - Machine Branch Probability Info -===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/MachineCSE.cpp b/lib/CodeGen/MachineCSE.cpp
index 6ee8571c28aab6453aa7570081cb44473958b520..f09288ec15c5c90ad867672236a550157183a7eb 100644
--- a/lib/CodeGen/MachineCSE.cpp
+++ b/lib/CodeGen/MachineCSE.cpp
@@ -1,9 +1,8 @@
 //===- MachineCSE.cpp - Machine Common Subexpression Elimination Pass -----===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/MachineCombiner.cpp b/lib/CodeGen/MachineCombiner.cpp
index 0c6efff7bb40ea8ada12b4944895c92235e23301..f35f11308173358db5b470dd55d6d2a3fea24744 100644
--- a/lib/CodeGen/MachineCombiner.cpp
+++ b/lib/CodeGen/MachineCombiner.cpp
@@ -1,9 +1,8 @@
 //===---- MachineCombiner.cpp - Instcombining on SSA form machine code ----===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -231,6 +230,8 @@ unsigned MachineCombiner::getLatency(MachineInstr *Root, MachineInstr *NewRoot,
     // Get the first instruction that uses MO
     MachineRegisterInfo::reg_iterator RI = MRI->reg_begin(MO.getReg());
     RI++;
+    if (RI == MRI->reg_end())
+      continue;
     MachineInstr *UseMO = RI->getParent();
     unsigned LatencyOp = 0;
     if (UseMO && BlockTrace.isDepInTrace(*Root, *UseMO)) {
@@ -557,16 +558,13 @@ bool MachineCombiner::combineInstructions(MachineBasicBlock *MBB) {
         continue;
 
       LLVM_DEBUG(if (dump_intrs) {
-        dbgs() << "\tFor the Pattern (" << (int)P << ") these instructions could be removed\n";
-        for (auto const *InstrPtr : DelInstrs) {
-          dbgs() << "\t\t" << STI->getSchedInfoStr(*InstrPtr) << ": ";
+        dbgs() << "\tFor the Pattern (" << (int)P
+               << ") these instructions could be removed\n";
+        for (auto const *InstrPtr : DelInstrs)
           InstrPtr->print(dbgs(), false, false, false, TII);
-        }
         dbgs() << "\tThese instructions could replace the removed ones\n";
-        for (auto const *InstrPtr : InsInstrs) {
-          dbgs() << "\t\t" << STI->getSchedInfoStr(*InstrPtr) << ": ";
+        for (auto const *InstrPtr : InsInstrs)
           InstrPtr->print(dbgs(), false, false, false, TII);
-        }
       });
 
       bool SubstituteAlways = false;
diff --git a/lib/CodeGen/MachineCopyPropagation.cpp b/lib/CodeGen/MachineCopyPropagation.cpp
index 19879fe890078c51a6881c677b1823d2ce0425de..9fc12ac89e12e371d9d08a91585c5faca47e3658 100644
--- a/lib/CodeGen/MachineCopyPropagation.cpp
+++ b/lib/CodeGen/MachineCopyPropagation.cpp
@@ -1,9 +1,8 @@
 //===- MachineCopyPropagation.cpp - Machine Copy Propagation Pass ---------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/MachineDominanceFrontier.cpp b/lib/CodeGen/MachineDominanceFrontier.cpp
index b559e4e513a6f7ad46490035d3ec6eef43dd561e..6704298c17d6ef354052e0299ecf9eaf9a437a71 100644
--- a/lib/CodeGen/MachineDominanceFrontier.cpp
+++ b/lib/CodeGen/MachineDominanceFrontier.cpp
@@ -1,9 +1,8 @@
 //===- MachineDominanceFrontier.cpp ---------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/CodeGen/MachineDominators.cpp b/lib/CodeGen/MachineDominators.cpp
index 6b28026264563e021df31f71948a407110a6be95..1dfba8638c2271abc34fae562ef8823957f4189b 100644
--- a/lib/CodeGen/MachineDominators.cpp
+++ b/lib/CodeGen/MachineDominators.cpp
@@ -1,9 +1,8 @@
 //===- MachineDominators.cpp - Machine Dominator Calculation --------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/MachineFrameInfo.cpp b/lib/CodeGen/MachineFrameInfo.cpp
index 0b316871dbdff0f781141c23e91378330557b6a5..d75ba83ffd045c9c1c491ca847ac8aa2d1d87bee 100644
--- a/lib/CodeGen/MachineFrameInfo.cpp
+++ b/lib/CodeGen/MachineFrameInfo.cpp
@@ -1,9 +1,8 @@
 //===-- MachineFrameInfo.cpp ---------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/MachineFunction.cpp b/lib/CodeGen/MachineFunction.cpp
index 3ded00b70fd43f3ac0405ea47c5c18d01f9d345b..84a20762d62b5f196cfd4fb163bf5b1ad62bbc8e 100644
--- a/lib/CodeGen/MachineFunction.cpp
+++ b/lib/CodeGen/MachineFunction.cpp
@@ -1,9 +1,8 @@
 //===- MachineFunction.cpp ------------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -139,12 +138,12 @@ MachineFunction::MachineFunction(const Function &F,
   init();
 }
 
-void MachineFunction::handleInsertion(const MachineInstr &MI) {
+void MachineFunction::handleInsertion(MachineInstr &MI) {
   if (TheDelegate)
     TheDelegate->MF_HandleInsertion(MI);
 }
 
-void MachineFunction::handleRemoval(const MachineInstr &MI) {
+void MachineFunction::handleRemoval(MachineInstr &MI) {
   if (TheDelegate)
     TheDelegate->MF_HandleRemoval(MI);
 }
@@ -396,19 +395,18 @@ MachineMemOperand *MachineFunction::getMachineMemOperand(
 MachineMemOperand *
 MachineFunction::getMachineMemOperand(const MachineMemOperand *MMO,
                                       int64_t Offset, uint64_t Size) {
-  if (MMO->getValue())
-    return new (Allocator)
-               MachineMemOperand(MachinePointerInfo(MMO->getValue(),
-                                                    MMO->getOffset()+Offset),
-                                 MMO->getFlags(), Size, MMO->getBaseAlignment(),
-                                 AAMDNodes(), nullptr, MMO->getSyncScopeID(),
-                                 MMO->getOrdering(), MMO->getFailureOrdering());
+  const MachinePointerInfo &PtrInfo = MMO->getPointerInfo();
+
+  // If there is no pointer value, the offset isn't tracked so we need to adjust
+  // the base alignment.
+  unsigned Align = PtrInfo.V.isNull()
+                       ? MinAlign(MMO->getBaseAlignment(), Offset)
+                       : MMO->getBaseAlignment();
+
   return new (Allocator)
-             MachineMemOperand(MachinePointerInfo(MMO->getPseudoValue(),
-                                                  MMO->getOffset()+Offset),
-                               MMO->getFlags(), Size, MMO->getBaseAlignment(),
-                               AAMDNodes(), nullptr, MMO->getSyncScopeID(),
-                               MMO->getOrdering(), MMO->getFailureOrdering());
+      MachineMemOperand(PtrInfo.getWithOffset(Offset), MMO->getFlags(), Size,
+                        Align, AAMDNodes(), nullptr, MMO->getSyncScopeID(),
+                        MMO->getOrdering(), MMO->getFailureOrdering());
 }
 
 MachineMemOperand *
diff --git a/lib/CodeGen/MachineFunctionPass.cpp b/lib/CodeGen/MachineFunctionPass.cpp
index 5db4e299fa701bdae32c3cd3bc4c4fb37d286291..0da4cf3fc90c85e6c03af5e216b6811c5d38f967 100644
--- a/lib/CodeGen/MachineFunctionPass.cpp
+++ b/lib/CodeGen/MachineFunctionPass.cpp
@@ -1,9 +1,8 @@
 //===-- MachineFunctionPass.cpp -------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/MachineFunctionPrinterPass.cpp b/lib/CodeGen/MachineFunctionPrinterPass.cpp
index 9c96ba748778ba18fc06bf87e9dd7303038f798c..0ea8975cc74c894d6930dc408d71fce81fcbcf2f 100644
--- a/lib/CodeGen/MachineFunctionPrinterPass.cpp
+++ b/lib/CodeGen/MachineFunctionPrinterPass.cpp
@@ -1,9 +1,8 @@
 //===-- MachineFunctionPrinterPass.cpp ------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/MachineInstr.cpp b/lib/CodeGen/MachineInstr.cpp
index 764a84c7e1327ed25c0b77d1b5f9da8e1339e9d4..0b8e9caa09b7db00094ffa473d6552db097e3a59 100644
--- a/lib/CodeGen/MachineInstr.cpp
+++ b/lib/CodeGen/MachineInstr.cpp
@@ -1,9 +1,8 @@
 //===- lib/CodeGen/MachineInstr.cpp ---------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -26,6 +25,7 @@
 #include "llvm/Analysis/MemoryLocation.h"
 #include "llvm/CodeGen/GlobalISel/RegisterBank.h"
 #include "llvm/CodeGen/MachineBasicBlock.h"
+#include "llvm/CodeGen/MachineFrameInfo.h"
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineInstrBuilder.h"
 #include "llvm/CodeGen/MachineInstrBundle.h"
@@ -50,9 +50,9 @@
 #include "llvm/IR/Metadata.h"
 #include "llvm/IR/Module.h"
 #include "llvm/IR/ModuleSlotTracker.h"
+#include "llvm/IR/Operator.h"
 #include "llvm/IR/Type.h"
 #include "llvm/IR/Value.h"
-#include "llvm/IR/Operator.h"
 #include "llvm/MC/MCInstrDesc.h"
 #include "llvm/MC/MCRegisterInfo.h"
 #include "llvm/MC/MCSymbol.h"
@@ -225,12 +225,13 @@ void MachineInstr::addOperand(MachineFunction &MF, const MachineOperand &Op) {
   }
 
 #ifndef NDEBUG
-  bool isMetaDataOp = Op.getType() == MachineOperand::MO_Metadata;
+  bool isDebugOp = Op.getType() == MachineOperand::MO_Metadata ||
+                   Op.getType() == MachineOperand::MO_MCSymbol;
   // OpNo now points as the desired insertion point.  Unless this is a variadic
   // instruction, only implicit regs are allowed beyond MCID->getNumOperands().
   // RegMask operands go between the explicit and implicit operands.
   assert((isImpReg || Op.isRegMask() || MCID->isVariadic() ||
-          OpNo < MCID->getNumOperands() || isMetaDataOp) &&
+          OpNo < MCID->getNumOperands() || isDebugOp) &&
          "Trying to add an operand to a machine instr that is already done!");
 #endif
 
@@ -518,39 +519,46 @@ uint16_t MachineInstr::mergeFlagsWith(const MachineInstr &Other) const {
   return getFlags() | Other.getFlags();
 }
 
-void MachineInstr::copyIRFlags(const Instruction &I) {
+uint16_t MachineInstr::copyFlagsFromInstruction(const Instruction &I) {
+  uint16_t MIFlags = 0;
   // Copy the wrapping flags.
   if (const OverflowingBinaryOperator *OB =
           dyn_cast<OverflowingBinaryOperator>(&I)) {
     if (OB->hasNoSignedWrap())
-      setFlag(MachineInstr::MIFlag::NoSWrap);
+      MIFlags |= MachineInstr::MIFlag::NoSWrap;
     if (OB->hasNoUnsignedWrap())
-      setFlag(MachineInstr::MIFlag::NoUWrap);
+      MIFlags |= MachineInstr::MIFlag::NoUWrap;
   }
 
   // Copy the exact flag.
   if (const PossiblyExactOperator *PE = dyn_cast<PossiblyExactOperator>(&I))
     if (PE->isExact())
-      setFlag(MachineInstr::MIFlag::IsExact);
+      MIFlags |= MachineInstr::MIFlag::IsExact;
 
   // Copy the fast-math flags.
   if (const FPMathOperator *FP = dyn_cast<FPMathOperator>(&I)) {
     const FastMathFlags Flags = FP->getFastMathFlags();
     if (Flags.noNaNs())
-      setFlag(MachineInstr::MIFlag::FmNoNans);
+      MIFlags |= MachineInstr::MIFlag::FmNoNans;
     if (Flags.noInfs())
-      setFlag(MachineInstr::MIFlag::FmNoInfs);
+      MIFlags |= MachineInstr::MIFlag::FmNoInfs;
     if (Flags.noSignedZeros())
-      setFlag(MachineInstr::MIFlag::FmNsz);
+      MIFlags |= MachineInstr::MIFlag::FmNsz;
     if (Flags.allowReciprocal())
-      setFlag(MachineInstr::MIFlag::FmArcp);
+      MIFlags |= MachineInstr::MIFlag::FmArcp;
     if (Flags.allowContract())
-      setFlag(MachineInstr::MIFlag::FmContract);
+      MIFlags |= MachineInstr::MIFlag::FmContract;
     if (Flags.approxFunc())
-      setFlag(MachineInstr::MIFlag::FmAfn);
+      MIFlags |= MachineInstr::MIFlag::FmAfn;
     if (Flags.allowReassoc())
-      setFlag(MachineInstr::MIFlag::FmReassoc);
+      MIFlags |= MachineInstr::MIFlag::FmReassoc;
   }
+
+  return MIFlags;
+}
+
+void MachineInstr::copyIRFlags(const Instruction &I) {
+  Flags = copyFlagsFromInstruction(I);
 }
 
 bool MachineInstr::hasPropertyInBundle(uint64_t Mask, QueryType Type) const {
@@ -1283,8 +1291,10 @@ bool MachineInstr::hasOrderedMemoryRef() const {
     return true;
 
   // Check if any of our memory operands are ordered.
+  // TODO: This should probably be be isUnordered (see D57601), but the callers
+  // need audited and test cases written to be sure.
   return llvm::any_of(memoperands(), [](const MachineMemOperand *MMO) {
-    return !MMO->isUnordered();
+    return MMO->isVolatile() || MMO->isAtomic();
   });
 }
 
@@ -1305,6 +1315,8 @@ bool MachineInstr::isDereferenceableInvariantLoad(AliasAnalysis *AA) const {
 
   for (MachineMemOperand *MMO : memoperands()) {
     if (MMO->isVolatile()) return false;
+    // TODO: Figure out whether isAtomic is really necessary (see D57601).
+    if (MMO->isAtomic()) return false;
     if (MMO->isStore()) return false;
     if (MMO->isInvariant() && MMO->isDereferenceable())
       continue;
@@ -2100,3 +2112,54 @@ void MachineInstr::changeDebugValuesDefReg(unsigned Reg) {
   for (auto *DBI : DbgValues)
     DBI->getOperand(0).setReg(Reg);
 }
+
+using MMOList = SmallVector<const MachineMemOperand *, 2>;
+
+static unsigned getSpillSlotSize(MMOList &Accesses,
+                                 const MachineFrameInfo &MFI) {
+  unsigned Size = 0;
+  for (auto A : Accesses)
+    if (MFI.isSpillSlotObjectIndex(
+            cast<FixedStackPseudoSourceValue>(A->getPseudoValue())
+                ->getFrameIndex()))
+      Size += A->getSize();
+  return Size;
+}
+
+Optional<unsigned>
+MachineInstr::getSpillSize(const TargetInstrInfo *TII) const {
+  int FI;
+  if (TII->isStoreToStackSlotPostFE(*this, FI)) {
+    const MachineFrameInfo &MFI = getMF()->getFrameInfo();
+    if (MFI.isSpillSlotObjectIndex(FI))
+      return (*memoperands_begin())->getSize();
+  }
+  return None;
+}
+
+Optional<unsigned>
+MachineInstr::getFoldedSpillSize(const TargetInstrInfo *TII) const {
+  MMOList Accesses;
+  if (TII->hasStoreToStackSlot(*this, Accesses))
+    return getSpillSlotSize(Accesses, getMF()->getFrameInfo());
+  return None;
+}
+
+Optional<unsigned>
+MachineInstr::getRestoreSize(const TargetInstrInfo *TII) const {
+  int FI;
+  if (TII->isLoadFromStackSlotPostFE(*this, FI)) {
+    const MachineFrameInfo &MFI = getMF()->getFrameInfo();
+    if (MFI.isSpillSlotObjectIndex(FI))
+      return (*memoperands_begin())->getSize();
+  }
+  return None;
+}
+
+Optional<unsigned>
+MachineInstr::getFoldedRestoreSize(const TargetInstrInfo *TII) const {
+  MMOList Accesses;
+  if (TII->hasLoadFromStackSlot(*this, Accesses))
+    return getSpillSlotSize(Accesses, getMF()->getFrameInfo());
+  return None;
+}
diff --git a/lib/CodeGen/MachineInstrBundle.cpp b/lib/CodeGen/MachineInstrBundle.cpp
index ae378cc8c4647668aa1260222b0574e1b7ab6ef7..32e266e9401ef4bd21e7edb403ddda8e724db2e5 100644
--- a/lib/CodeGen/MachineInstrBundle.cpp
+++ b/lib/CodeGen/MachineInstrBundle.cpp
@@ -1,9 +1,8 @@
 //===-- lib/CodeGen/MachineInstrBundle.cpp --------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/CodeGen/MachineLICM.cpp b/lib/CodeGen/MachineLICM.cpp
index 58fd1f23842060aebc302c18bc22af7c923b3176..1107e609c2580deeec7b0484c3cde0aca561aae9 100644
--- a/lib/CodeGen/MachineLICM.cpp
+++ b/lib/CodeGen/MachineLICM.cpp
@@ -1,9 +1,8 @@
 //===- MachineLICM.cpp - Machine Loop Invariant Code Motion Pass ----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/MachineLoopInfo.cpp b/lib/CodeGen/MachineLoopInfo.cpp
index 2bce59235057ed07c13462b2a43b7f2701ed3110..3b8b430d1b0f0dc4c73e83ee2462d052a6af0b18 100644
--- a/lib/CodeGen/MachineLoopInfo.cpp
+++ b/lib/CodeGen/MachineLoopInfo.cpp
@@ -1,9 +1,8 @@
 //===- MachineLoopInfo.cpp - Natural Loop Calculator ----------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/MachineModuleInfo.cpp b/lib/CodeGen/MachineModuleInfo.cpp
index 6ef8de88f8b117484dcaf0d6fd2df4864c4cd7cc..8e03b59607dbf918140598f3da2c38e09f6fdaed 100644
--- a/lib/CodeGen/MachineModuleInfo.cpp
+++ b/lib/CodeGen/MachineModuleInfo.cpp
@@ -1,9 +1,8 @@
 //===-- llvm/CodeGen/MachineModuleInfo.cpp ----------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -206,7 +205,7 @@ MachineModuleInfo::~MachineModuleInfo() = default;
 bool MachineModuleInfo::doInitialization(Module &M) {
   ObjFileMMI = nullptr;
   CurCallSite = 0;
-  UsesVAFloatArgument = UsesMorestackAddr = false;
+  UsesMSVCFloatingPoint = UsesMorestackAddr = false;
   HasSplitStack = HasNosplitStack = false;
   AddrLabelSymbols = nullptr;
   TheModule = &M;
@@ -328,22 +327,3 @@ char FreeMachineFunction::ID;
 FunctionPass *llvm::createFreeMachineFunctionPass() {
   return new FreeMachineFunction();
 }
-
-//===- MMI building helpers -----------------------------------------------===//
-
-void llvm::computeUsesVAFloatArgument(const CallInst &I,
-                                      MachineModuleInfo &MMI) {
-  FunctionType *FT =
-      cast<FunctionType>(I.getCalledValue()->getType()->getContainedType(0));
-  if (FT->isVarArg() && !MMI.usesVAFloatArgument()) {
-    for (unsigned i = 0, e = I.getNumArgOperands(); i != e; ++i) {
-      Type *T = I.getArgOperand(i)->getType();
-      for (auto i : post_order(T)) {
-        if (i->isFloatingPointTy()) {
-          MMI.setUsesVAFloatArgument(true);
-          return;
-        }
-      }
-    }
-  }
-}
diff --git a/lib/CodeGen/MachineModuleInfoImpls.cpp b/lib/CodeGen/MachineModuleInfoImpls.cpp
index 7b4f64bfe60d071f282cd20d50b43033784f7cff..16d24880ebe49f9013410e83af96274b63c5eaf6 100644
--- a/lib/CodeGen/MachineModuleInfoImpls.cpp
+++ b/lib/CodeGen/MachineModuleInfoImpls.cpp
@@ -1,9 +1,8 @@
 //===- llvm/CodeGen/MachineModuleInfoImpls.cpp ----------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/MachineOperand.cpp b/lib/CodeGen/MachineOperand.cpp
index 05e51e1873cf90a65f1ef6b5da9706340f51e4bc..3ff7a3c252ab7d67b245adcce136e0cc6e5cffc9 100644
--- a/lib/CodeGen/MachineOperand.cpp
+++ b/lib/CodeGen/MachineOperand.cpp
@@ -1,9 +1,8 @@
 //===- lib/CodeGen/MachineOperand.cpp -------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -994,7 +993,7 @@ MachineMemOperand::MachineMemOperand(MachinePointerInfo ptrinfo, Flags f,
   assert((PtrInfo.V.isNull() || PtrInfo.V.is<const PseudoSourceValue *>() ||
           isa<PointerType>(PtrInfo.V.get<const Value *>()->getType())) &&
          "invalid pointer value");
-  assert(getBaseAlignment() == a && "Alignment is not a power of 2!");
+  assert(getBaseAlignment() == a && a != 0 && "Alignment is not a power of 2!");
   assert((isLoad() || isStore()) && "Not a load/store!");
 
   AtomicInfo.SSID = static_cast<unsigned>(SSID);
diff --git a/lib/CodeGen/MachineOptimizationRemarkEmitter.cpp b/lib/CodeGen/MachineOptimizationRemarkEmitter.cpp
index 906d5560d56845ec4786b9ad203a30eef83074e1..27db9106b3374cbd3fad71bbdd75abd29989b9e0 100644
--- a/lib/CodeGen/MachineOptimizationRemarkEmitter.cpp
+++ b/lib/CodeGen/MachineOptimizationRemarkEmitter.cpp
@@ -1,9 +1,8 @@
 ///===- MachineOptimizationRemarkEmitter.cpp - Opt Diagnostic -*- C++ -*---===//
 ///
-///                     The LLVM Compiler Infrastructure
-///
-/// This file is distributed under the University of Illinois Open Source
-/// License. See LICENSE.TXT for details.
+/// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+/// See https://llvm.org/LICENSE.txt for license information.
+/// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ///
 ///===---------------------------------------------------------------------===//
 /// \file
diff --git a/lib/CodeGen/MachineOutliner.cpp b/lib/CodeGen/MachineOutliner.cpp
index ad96c0e579e43665c2b301009b8c0b6aad305665..8c0626092bbed19f99356edbefeba8c47ca14a26 100644
--- a/lib/CodeGen/MachineOutliner.cpp
+++ b/lib/CodeGen/MachineOutliner.cpp
@@ -1,9 +1,8 @@
 //===---- MachineOutliner.cpp - Outline instructions -----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
@@ -1105,9 +1104,9 @@ MachineOutliner::createOutlinedFunction(Module &M, OutlinedFunction &OF,
 
   // Create the function using an IR-level function.
   LLVMContext &C = M.getContext();
-  Function *F = dyn_cast<Function>(
-      M.getOrInsertFunction(NameStream.str(), Type::getVoidTy(C)));
-  assert(F && "Function was null!");
+  Function *F =
+      Function::Create(FunctionType::get(Type::getVoidTy(C), false),
+                       Function::ExternalLinkage, NameStream.str(), M);
 
   // NOTE: If this is linkonceodr, then we can take advantage of linker deduping
   // which gives us better results when we outline from linkonceodr functions.
diff --git a/lib/CodeGen/MachinePipeliner.cpp b/lib/CodeGen/MachinePipeliner.cpp
index de8c55585d3fdfdd37593c3b9c0918e9f4e1982d..1e729d4851aba651d345efa1693abd0c0a8db7b7 100644
--- a/lib/CodeGen/MachinePipeliner.cpp
+++ b/lib/CodeGen/MachinePipeliner.cpp
@@ -1,42 +1,13 @@
 //===- MachinePipeliner.cpp - Machine Software Pipeliner Pass -------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 // An implementation of the Swing Modulo Scheduling (SMS) software pipeliner.
 //
-// Software pipelining (SWP) is an instruction scheduling technique for loops
-// that overlap loop iterations and exploits ILP via a compiler transformation.
-//
-// Swing Modulo Scheduling is an implementation of software pipelining
-// that generates schedules that are near optimal in terms of initiation
-// interval, register requirements, and stage count. See the papers:
-//
-// "Swing Modulo Scheduling: A Lifetime-Sensitive Approach", by J. Llosa,
-// A. Gonzalez, E. Ayguade, and M. Valero. In PACT '96 Proceedings of the 1996
-// Conference on Parallel Architectures and Compilation Techiniques.
-//
-// "Lifetime-Sensitive Modulo Scheduling in a Production Environment", by J.
-// Llosa, E. Ayguade, A. Gonzalez, M. Valero, and J. Eckhardt. In IEEE
-// Transactions on Computers, Vol. 50, No. 3, 2001.
-//
-// "An Implementation of Swing Modulo Scheduling With Extensions for
-// Superblocks", by T. Lattner, Master's Thesis, University of Illinois at
-// Urbana-Chambpain, 2005.
-//
-//
-// The SMS algorithm consists of three main steps after computing the minimal
-// initiation interval (MII).
-// 1) Analyze the dependence graph and compute information about each
-//    instruction in the graph.
-// 2) Order the nodes (instructions) by priority based upon the heuristics
-//    described in the algorithm.
-// 3) Attempt to schedule the nodes in the specified order using the MII.
-//
 // This SMS implementation is a target-independent back-end pass. When enabled,
 // the pass runs just prior to the register allocation pass, while the machine
 // IR is in SSA form. If software pipelining is successful, then the original
@@ -83,13 +54,11 @@
 #include "llvm/CodeGen/MachineLoopInfo.h"
 #include "llvm/CodeGen/MachineMemOperand.h"
 #include "llvm/CodeGen/MachineOperand.h"
+#include "llvm/CodeGen/MachinePipeliner.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
-#include "llvm/CodeGen/RegisterClassInfo.h"
 #include "llvm/CodeGen/RegisterPressure.h"
 #include "llvm/CodeGen/ScheduleDAG.h"
-#include "llvm/CodeGen/ScheduleDAGInstrs.h"
 #include "llvm/CodeGen/ScheduleDAGMutation.h"
-#include "llvm/CodeGen/TargetInstrInfo.h"
 #include "llvm/CodeGen/TargetOpcodes.h"
 #include "llvm/CodeGen/TargetRegisterInfo.h"
 #include "llvm/CodeGen/TargetSubtargetInfo.h"
@@ -171,575 +140,15 @@ static cl::opt<bool> SwpIgnoreRecMII("pipeliner-ignore-recmii",
                                      cl::ReallyHidden, cl::init(false),
                                      cl::ZeroOrMore, cl::desc("Ignore RecMII"));
 
+namespace llvm {
+
 // A command line option to enable the CopyToPhi DAG mutation.
-static cl::opt<bool>
+cl::opt<bool>
     SwpEnableCopyToPhi("pipeliner-enable-copytophi", cl::ReallyHidden,
                        cl::init(true), cl::ZeroOrMore,
                        cl::desc("Enable CopyToPhi DAG Mutation"));
 
-namespace {
-
-class NodeSet;
-class SMSchedule;
-
-/// The main class in the implementation of the target independent
-/// software pipeliner pass.
-class MachinePipeliner : public MachineFunctionPass {
-public:
-  MachineFunction *MF = nullptr;
-  const MachineLoopInfo *MLI = nullptr;
-  const MachineDominatorTree *MDT = nullptr;
-  const InstrItineraryData *InstrItins;
-  const TargetInstrInfo *TII = nullptr;
-  RegisterClassInfo RegClassInfo;
-
-#ifndef NDEBUG
-  static int NumTries;
-#endif
-
-  /// Cache the target analysis information about the loop.
-  struct LoopInfo {
-    MachineBasicBlock *TBB = nullptr;
-    MachineBasicBlock *FBB = nullptr;
-    SmallVector<MachineOperand, 4> BrCond;
-    MachineInstr *LoopInductionVar = nullptr;
-    MachineInstr *LoopCompare = nullptr;
-  };
-  LoopInfo LI;
-
-  static char ID;
-
-  MachinePipeliner() : MachineFunctionPass(ID) {
-    initializeMachinePipelinerPass(*PassRegistry::getPassRegistry());
-  }
-
-  bool runOnMachineFunction(MachineFunction &MF) override;
-
-  void getAnalysisUsage(AnalysisUsage &AU) const override {
-    AU.addRequired<AAResultsWrapperPass>();
-    AU.addPreserved<AAResultsWrapperPass>();
-    AU.addRequired<MachineLoopInfo>();
-    AU.addRequired<MachineDominatorTree>();
-    AU.addRequired<LiveIntervals>();
-    MachineFunctionPass::getAnalysisUsage(AU);
-  }
-
-private:
-  void preprocessPhiNodes(MachineBasicBlock &B);
-  bool canPipelineLoop(MachineLoop &L);
-  bool scheduleLoop(MachineLoop &L);
-  bool swingModuloScheduler(MachineLoop &L);
-};
-
-/// This class builds the dependence graph for the instructions in a loop,
-/// and attempts to schedule the instructions using the SMS algorithm.
-class SwingSchedulerDAG : public ScheduleDAGInstrs {
-  MachinePipeliner &Pass;
-  /// The minimum initiation interval between iterations for this schedule.
-  unsigned MII = 0;
-  /// Set to true if a valid pipelined schedule is found for the loop.
-  bool Scheduled = false;
-  MachineLoop &Loop;
-  LiveIntervals &LIS;
-  const RegisterClassInfo &RegClassInfo;
-
-  /// A toplogical ordering of the SUnits, which is needed for changing
-  /// dependences and iterating over the SUnits.
-  ScheduleDAGTopologicalSort Topo;
-
-  struct NodeInfo {
-    int ASAP = 0;
-    int ALAP = 0;
-    int ZeroLatencyDepth = 0;
-    int ZeroLatencyHeight = 0;
-
-    NodeInfo() = default;
-  };
-  /// Computed properties for each node in the graph.
-  std::vector<NodeInfo> ScheduleInfo;
-
-  enum OrderKind { BottomUp = 0, TopDown = 1 };
-  /// Computed node ordering for scheduling.
-  SetVector<SUnit *> NodeOrder;
-
-  using NodeSetType = SmallVector<NodeSet, 8>;
-  using ValueMapTy = DenseMap<unsigned, unsigned>;
-  using MBBVectorTy = SmallVectorImpl<MachineBasicBlock *>;
-  using InstrMapTy = DenseMap<MachineInstr *, MachineInstr *>;
-
-  /// Instructions to change when emitting the final schedule.
-  DenseMap<SUnit *, std::pair<unsigned, int64_t>> InstrChanges;
-
-  /// We may create a new instruction, so remember it because it
-  /// must be deleted when the pass is finished.
-  SmallPtrSet<MachineInstr *, 4> NewMIs;
-
-  /// Ordered list of DAG postprocessing steps.
-  std::vector<std::unique_ptr<ScheduleDAGMutation>> Mutations;
-
-  /// Helper class to implement Johnson's circuit finding algorithm.
-  class Circuits {
-    std::vector<SUnit> &SUnits;
-    SetVector<SUnit *> Stack;
-    BitVector Blocked;
-    SmallVector<SmallPtrSet<SUnit *, 4>, 10> B;
-    SmallVector<SmallVector<int, 4>, 16> AdjK;
-    // Node to Index from ScheduleDAGTopologicalSort
-    std::vector<int> *Node2Idx;
-    unsigned NumPaths;
-    static unsigned MaxPaths;
-
-  public:
-    Circuits(std::vector<SUnit> &SUs, ScheduleDAGTopologicalSort &Topo)
-        : SUnits(SUs), Blocked(SUs.size()), B(SUs.size()), AdjK(SUs.size()) {
-      Node2Idx = new std::vector<int>(SUs.size());
-      unsigned Idx = 0;
-      for (const auto &NodeNum : Topo)
-        Node2Idx->at(NodeNum) = Idx++;
-    }
-
-    ~Circuits() { delete Node2Idx; }
-
-    /// Reset the data structures used in the circuit algorithm.
-    void reset() {
-      Stack.clear();
-      Blocked.reset();
-      B.assign(SUnits.size(), SmallPtrSet<SUnit *, 4>());
-      NumPaths = 0;
-    }
-
-    void createAdjacencyStructure(SwingSchedulerDAG *DAG);
-    bool circuit(int V, int S, NodeSetType &NodeSets, bool HasBackedge = false);
-    void unblock(int U);
-  };
-
-  struct CopyToPhiMutation : public ScheduleDAGMutation {
-    void apply(ScheduleDAGInstrs *DAG) override;
-  };
-
-public:
-  SwingSchedulerDAG(MachinePipeliner &P, MachineLoop &L, LiveIntervals &lis,
-                    const RegisterClassInfo &rci)
-      : ScheduleDAGInstrs(*P.MF, P.MLI, false), Pass(P), Loop(L), LIS(lis),
-        RegClassInfo(rci), Topo(SUnits, &ExitSU) {
-    P.MF->getSubtarget().getSMSMutations(Mutations);
-    if (SwpEnableCopyToPhi)
-      Mutations.push_back(llvm::make_unique<CopyToPhiMutation>());
-  }
-
-  void schedule() override;
-  void finishBlock() override;
-
-  /// Return true if the loop kernel has been scheduled.
-  bool hasNewSchedule() { return Scheduled; }
-
-  /// Return the earliest time an instruction may be scheduled.
-  int getASAP(SUnit *Node) { return ScheduleInfo[Node->NodeNum].ASAP; }
-
-  /// Return the latest time an instruction my be scheduled.
-  int getALAP(SUnit *Node) { return ScheduleInfo[Node->NodeNum].ALAP; }
-
-  /// The mobility function, which the number of slots in which
-  /// an instruction may be scheduled.
-  int getMOV(SUnit *Node) { return getALAP(Node) - getASAP(Node); }
-
-  /// The depth, in the dependence graph, for a node.
-  unsigned getDepth(SUnit *Node) { return Node->getDepth(); }
-
-  /// The maximum unweighted length of a path from an arbitrary node to the
-  /// given node in which each edge has latency 0
-  int getZeroLatencyDepth(SUnit *Node) {
-    return ScheduleInfo[Node->NodeNum].ZeroLatencyDepth;
-  }
-
-  /// The height, in the dependence graph, for a node.
-  unsigned getHeight(SUnit *Node) { return Node->getHeight(); }
-
-  /// The maximum unweighted length of a path from the given node to an
-  /// arbitrary node in which each edge has latency 0
-  int getZeroLatencyHeight(SUnit *Node) {
-    return ScheduleInfo[Node->NodeNum].ZeroLatencyHeight;
-  }
-
-  /// Return true if the dependence is a back-edge in the data dependence graph.
-  /// Since the DAG doesn't contain cycles, we represent a cycle in the graph
-  /// using an anti dependence from a Phi to an instruction.
-  bool isBackedge(SUnit *Source, const SDep &Dep) {
-    if (Dep.getKind() != SDep::Anti)
-      return false;
-    return Source->getInstr()->isPHI() || Dep.getSUnit()->getInstr()->isPHI();
-  }
-
-  bool isLoopCarriedDep(SUnit *Source, const SDep &Dep, bool isSucc = true);
-
-  /// The distance function, which indicates that operation V of iteration I
-  /// depends on operations U of iteration I-distance.
-  unsigned getDistance(SUnit *U, SUnit *V, const SDep &Dep) {
-    // Instructions that feed a Phi have a distance of 1. Computing larger
-    // values for arrays requires data dependence information.
-    if (V->getInstr()->isPHI() && Dep.getKind() == SDep::Anti)
-      return 1;
-    return 0;
-  }
-
-  /// Set the Minimum Initiation Interval for this schedule attempt.
-  void setMII(unsigned mii) { MII = mii; }
-
-  void applyInstrChange(MachineInstr *MI, SMSchedule &Schedule);
-
-  void fixupRegisterOverlaps(std::deque<SUnit *> &Instrs);
-
-  /// Return the new base register that was stored away for the changed
-  /// instruction.
-  unsigned getInstrBaseReg(SUnit *SU) {
-    DenseMap<SUnit *, std::pair<unsigned, int64_t>>::iterator It =
-        InstrChanges.find(SU);
-    if (It != InstrChanges.end())
-      return It->second.first;
-    return 0;
-  }
-
-  void addMutation(std::unique_ptr<ScheduleDAGMutation> Mutation) {
-    Mutations.push_back(std::move(Mutation));
-  }
-
-  static bool classof(const ScheduleDAGInstrs *DAG) { return true; }
-
-private:
-  void addLoopCarriedDependences(AliasAnalysis *AA);
-  void updatePhiDependences();
-  void changeDependences();
-  unsigned calculateResMII();
-  unsigned calculateRecMII(NodeSetType &RecNodeSets);
-  void findCircuits(NodeSetType &NodeSets);
-  void fuseRecs(NodeSetType &NodeSets);
-  void removeDuplicateNodes(NodeSetType &NodeSets);
-  void computeNodeFunctions(NodeSetType &NodeSets);
-  void registerPressureFilter(NodeSetType &NodeSets);
-  void colocateNodeSets(NodeSetType &NodeSets);
-  void checkNodeSets(NodeSetType &NodeSets);
-  void groupRemainingNodes(NodeSetType &NodeSets);
-  void addConnectedNodes(SUnit *SU, NodeSet &NewSet,
-                         SetVector<SUnit *> &NodesAdded);
-  void computeNodeOrder(NodeSetType &NodeSets);
-  void checkValidNodeOrder(const NodeSetType &Circuits) const;
-  bool schedulePipeline(SMSchedule &Schedule);
-  void generatePipelinedLoop(SMSchedule &Schedule);
-  void generateProlog(SMSchedule &Schedule, unsigned LastStage,
-                      MachineBasicBlock *KernelBB, ValueMapTy *VRMap,
-                      MBBVectorTy &PrologBBs);
-  void generateEpilog(SMSchedule &Schedule, unsigned LastStage,
-                      MachineBasicBlock *KernelBB, ValueMapTy *VRMap,
-                      MBBVectorTy &EpilogBBs, MBBVectorTy &PrologBBs);
-  void generateExistingPhis(MachineBasicBlock *NewBB, MachineBasicBlock *BB1,
-                            MachineBasicBlock *BB2, MachineBasicBlock *KernelBB,
-                            SMSchedule &Schedule, ValueMapTy *VRMap,
-                            InstrMapTy &InstrMap, unsigned LastStageNum,
-                            unsigned CurStageNum, bool IsLast);
-  void generatePhis(MachineBasicBlock *NewBB, MachineBasicBlock *BB1,
-                    MachineBasicBlock *BB2, MachineBasicBlock *KernelBB,
-                    SMSchedule &Schedule, ValueMapTy *VRMap,
-                    InstrMapTy &InstrMap, unsigned LastStageNum,
-                    unsigned CurStageNum, bool IsLast);
-  void removeDeadInstructions(MachineBasicBlock *KernelBB,
-                              MBBVectorTy &EpilogBBs);
-  void splitLifetimes(MachineBasicBlock *KernelBB, MBBVectorTy &EpilogBBs,
-                      SMSchedule &Schedule);
-  void addBranches(MBBVectorTy &PrologBBs, MachineBasicBlock *KernelBB,
-                   MBBVectorTy &EpilogBBs, SMSchedule &Schedule,
-                   ValueMapTy *VRMap);
-  bool computeDelta(MachineInstr &MI, unsigned &Delta);
-  void updateMemOperands(MachineInstr &NewMI, MachineInstr &OldMI,
-                         unsigned Num);
-  MachineInstr *cloneInstr(MachineInstr *OldMI, unsigned CurStageNum,
-                           unsigned InstStageNum);
-  MachineInstr *cloneAndChangeInstr(MachineInstr *OldMI, unsigned CurStageNum,
-                                    unsigned InstStageNum,
-                                    SMSchedule &Schedule);
-  void updateInstruction(MachineInstr *NewMI, bool LastDef,
-                         unsigned CurStageNum, unsigned InstrStageNum,
-                         SMSchedule &Schedule, ValueMapTy *VRMap);
-  MachineInstr *findDefInLoop(unsigned Reg);
-  unsigned getPrevMapVal(unsigned StageNum, unsigned PhiStage, unsigned LoopVal,
-                         unsigned LoopStage, ValueMapTy *VRMap,
-                         MachineBasicBlock *BB);
-  void rewritePhiValues(MachineBasicBlock *NewBB, unsigned StageNum,
-                        SMSchedule &Schedule, ValueMapTy *VRMap,
-                        InstrMapTy &InstrMap);
-  void rewriteScheduledInstr(MachineBasicBlock *BB, SMSchedule &Schedule,
-                             InstrMapTy &InstrMap, unsigned CurStageNum,
-                             unsigned PhiNum, MachineInstr *Phi,
-                             unsigned OldReg, unsigned NewReg,
-                             unsigned PrevReg = 0);
-  bool canUseLastOffsetValue(MachineInstr *MI, unsigned &BasePos,
-                             unsigned &OffsetPos, unsigned &NewBase,
-                             int64_t &NewOffset);
-  void postprocessDAG();
-};
-
-/// A NodeSet contains a set of SUnit DAG nodes with additional information
-/// that assigns a priority to the set.
-class NodeSet {
-  SetVector<SUnit *> Nodes;
-  bool HasRecurrence = false;
-  unsigned RecMII = 0;
-  int MaxMOV = 0;
-  unsigned MaxDepth = 0;
-  unsigned Colocate = 0;
-  SUnit *ExceedPressure = nullptr;
-  unsigned Latency = 0;
-
-public:
-  using iterator = SetVector<SUnit *>::const_iterator;
-
-  NodeSet() = default;
-  NodeSet(iterator S, iterator E) : Nodes(S, E), HasRecurrence(true) {
-    Latency = 0;
-    for (unsigned i = 0, e = Nodes.size(); i < e; ++i)
-      for (const SDep &Succ : Nodes[i]->Succs)
-        if (Nodes.count(Succ.getSUnit()))
-          Latency += Succ.getLatency();
-  }
-
-  bool insert(SUnit *SU) { return Nodes.insert(SU); }
-
-  void insert(iterator S, iterator E) { Nodes.insert(S, E); }
-
-  template <typename UnaryPredicate> bool remove_if(UnaryPredicate P) {
-    return Nodes.remove_if(P);
-  }
-
-  unsigned count(SUnit *SU) const { return Nodes.count(SU); }
-
-  bool hasRecurrence() { return HasRecurrence; };
-
-  unsigned size() const { return Nodes.size(); }
-
-  bool empty() const { return Nodes.empty(); }
-
-  SUnit *getNode(unsigned i) const { return Nodes[i]; };
-
-  void setRecMII(unsigned mii) { RecMII = mii; };
-
-  void setColocate(unsigned c) { Colocate = c; };
-
-  void setExceedPressure(SUnit *SU) { ExceedPressure = SU; }
-
-  bool isExceedSU(SUnit *SU) { return ExceedPressure == SU; }
-
-  int compareRecMII(NodeSet &RHS) { return RecMII - RHS.RecMII; }
-
-  int getRecMII() { return RecMII; }
-
-  /// Summarize node functions for the entire node set.
-  void computeNodeSetInfo(SwingSchedulerDAG *SSD) {
-    for (SUnit *SU : *this) {
-      MaxMOV = std::max(MaxMOV, SSD->getMOV(SU));
-      MaxDepth = std::max(MaxDepth, SSD->getDepth(SU));
-    }
-  }
-
-  unsigned getLatency() { return Latency; }
-
-  unsigned getMaxDepth() { return MaxDepth; }
-
-  void clear() {
-    Nodes.clear();
-    RecMII = 0;
-    HasRecurrence = false;
-    MaxMOV = 0;
-    MaxDepth = 0;
-    Colocate = 0;
-    ExceedPressure = nullptr;
-  }
-
-  operator SetVector<SUnit *> &() { return Nodes; }
-
-  /// Sort the node sets by importance. First, rank them by recurrence MII,
-  /// then by mobility (least mobile done first), and finally by depth.
-  /// Each node set may contain a colocate value which is used as the first
-  /// tie breaker, if it's set.
-  bool operator>(const NodeSet &RHS) const {
-    if (RecMII == RHS.RecMII) {
-      if (Colocate != 0 && RHS.Colocate != 0 && Colocate != RHS.Colocate)
-        return Colocate < RHS.Colocate;
-      if (MaxMOV == RHS.MaxMOV)
-        return MaxDepth > RHS.MaxDepth;
-      return MaxMOV < RHS.MaxMOV;
-    }
-    return RecMII > RHS.RecMII;
-  }
-
-  bool operator==(const NodeSet &RHS) const {
-    return RecMII == RHS.RecMII && MaxMOV == RHS.MaxMOV &&
-           MaxDepth == RHS.MaxDepth;
-  }
-
-  bool operator!=(const NodeSet &RHS) const { return !operator==(RHS); }
-
-  iterator begin() { return Nodes.begin(); }
-  iterator end() { return Nodes.end(); }
-
-  void print(raw_ostream &os) const {
-    os << "Num nodes " << size() << " rec " << RecMII << " mov " << MaxMOV
-       << " depth " << MaxDepth << " col " << Colocate << "\n";
-    for (const auto &I : Nodes)
-      os << "   SU(" << I->NodeNum << ") " << *(I->getInstr());
-    os << "\n";
-  }
-
-#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
-  LLVM_DUMP_METHOD void dump() const { print(dbgs()); }
-#endif
-};
-
-/// This class represents the scheduled code.  The main data structure is a
-/// map from scheduled cycle to instructions.  During scheduling, the
-/// data structure explicitly represents all stages/iterations.   When
-/// the algorithm finshes, the schedule is collapsed into a single stage,
-/// which represents instructions from different loop iterations.
-///
-/// The SMS algorithm allows negative values for cycles, so the first cycle
-/// in the schedule is the smallest cycle value.
-class SMSchedule {
-private:
-  /// Map from execution cycle to instructions.
-  DenseMap<int, std::deque<SUnit *>> ScheduledInstrs;
-
-  /// Map from instruction to execution cycle.
-  std::map<SUnit *, int> InstrToCycle;
-
-  /// Map for each register and the max difference between its uses and def.
-  /// The first element in the pair is the max difference in stages. The
-  /// second is true if the register defines a Phi value and loop value is
-  /// scheduled before the Phi.
-  std::map<unsigned, std::pair<unsigned, bool>> RegToStageDiff;
-
-  /// Keep track of the first cycle value in the schedule.  It starts
-  /// as zero, but the algorithm allows negative values.
-  int FirstCycle = 0;
-
-  /// Keep track of the last cycle value in the schedule.
-  int LastCycle = 0;
-
-  /// The initiation interval (II) for the schedule.
-  int InitiationInterval = 0;
-
-  /// Target machine information.
-  const TargetSubtargetInfo &ST;
-
-  /// Virtual register information.
-  MachineRegisterInfo &MRI;
-
-  std::unique_ptr<DFAPacketizer> Resources;
-
-public:
-  SMSchedule(MachineFunction *mf)
-      : ST(mf->getSubtarget()), MRI(mf->getRegInfo()),
-        Resources(ST.getInstrInfo()->CreateTargetScheduleState(ST)) {}
-
-  void reset() {
-    ScheduledInstrs.clear();
-    InstrToCycle.clear();
-    RegToStageDiff.clear();
-    FirstCycle = 0;
-    LastCycle = 0;
-    InitiationInterval = 0;
-  }
-
-  /// Set the initiation interval for this schedule.
-  void setInitiationInterval(int ii) { InitiationInterval = ii; }
-
-  /// Return the first cycle in the completed schedule.  This
-  /// can be a negative value.
-  int getFirstCycle() const { return FirstCycle; }
-
-  /// Return the last cycle in the finalized schedule.
-  int getFinalCycle() const { return FirstCycle + InitiationInterval - 1; }
-
-  /// Return the cycle of the earliest scheduled instruction in the dependence
-  /// chain.
-  int earliestCycleInChain(const SDep &Dep);
-
-  /// Return the cycle of the latest scheduled instruction in the dependence
-  /// chain.
-  int latestCycleInChain(const SDep &Dep);
-
-  void computeStart(SUnit *SU, int *MaxEarlyStart, int *MinLateStart,
-                    int *MinEnd, int *MaxStart, int II, SwingSchedulerDAG *DAG);
-  bool insert(SUnit *SU, int StartCycle, int EndCycle, int II);
-
-  /// Iterators for the cycle to instruction map.
-  using sched_iterator = DenseMap<int, std::deque<SUnit *>>::iterator;
-  using const_sched_iterator =
-      DenseMap<int, std::deque<SUnit *>>::const_iterator;
-
-  /// Return true if the instruction is scheduled at the specified stage.
-  bool isScheduledAtStage(SUnit *SU, unsigned StageNum) {
-    return (stageScheduled(SU) == (int)StageNum);
-  }
-
-  /// Return the stage for a scheduled instruction.  Return -1 if
-  /// the instruction has not been scheduled.
-  int stageScheduled(SUnit *SU) const {
-    std::map<SUnit *, int>::const_iterator it = InstrToCycle.find(SU);
-    if (it == InstrToCycle.end())
-      return -1;
-    return (it->second - FirstCycle) / InitiationInterval;
-  }
-
-  /// Return the cycle for a scheduled instruction. This function normalizes
-  /// the first cycle to be 0.
-  unsigned cycleScheduled(SUnit *SU) const {
-    std::map<SUnit *, int>::const_iterator it = InstrToCycle.find(SU);
-    assert(it != InstrToCycle.end() && "Instruction hasn't been scheduled.");
-    return (it->second - FirstCycle) % InitiationInterval;
-  }
-
-  /// Return the maximum stage count needed for this schedule.
-  unsigned getMaxStageCount() {
-    return (LastCycle - FirstCycle) / InitiationInterval;
-  }
-
-  /// Return the max. number of stages/iterations that can occur between a
-  /// register definition and its uses.
-  unsigned getStagesForReg(int Reg, unsigned CurStage) {
-    std::pair<unsigned, bool> Stages = RegToStageDiff[Reg];
-    if (CurStage > getMaxStageCount() && Stages.first == 0 && Stages.second)
-      return 1;
-    return Stages.first;
-  }
-
-  /// The number of stages for a Phi is a little different than other
-  /// instructions. The minimum value computed in RegToStageDiff is 1
-  /// because we assume the Phi is needed for at least 1 iteration.
-  /// This is not the case if the loop value is scheduled prior to the
-  /// Phi in the same stage.  This function returns the number of stages
-  /// or iterations needed between the Phi definition and any uses.
-  unsigned getStagesForPhi(int Reg) {
-    std::pair<unsigned, bool> Stages = RegToStageDiff[Reg];
-    if (Stages.second)
-      return Stages.first;
-    return Stages.first - 1;
-  }
-
-  /// Return the instructions that are scheduled at the specified cycle.
-  std::deque<SUnit *> &getInstructions(int cycle) {
-    return ScheduledInstrs[cycle];
-  }
-
-  bool isValidSchedule(SwingSchedulerDAG *SSD);
-  void finalizeSchedule(SwingSchedulerDAG *SSD);
-  void orderDependence(SwingSchedulerDAG *SSD, SUnit *SU,
-                       std::deque<SUnit *> &Insts);
-  bool isLoopCarried(SwingSchedulerDAG *SSD, MachineInstr &Phi);
-  bool isLoopCarriedDefOfUse(SwingSchedulerDAG *SSD, MachineInstr *Def,
-                             MachineOperand &MO);
-  void print(raw_ostream &os) const;
-  void dump() const;
-};
-
-} // end anonymous namespace
+} // end namespace llvm
 
 unsigned SwingSchedulerDAG::Circuits::MaxPaths = 5;
 char MachinePipeliner::ID = 0;
@@ -801,8 +210,11 @@ bool MachinePipeliner::scheduleLoop(MachineLoop &L) {
   }
 #endif
 
-  if (!canPipelineLoop(L))
+  setPragmaPipelineOptions(L);
+  if (!canPipelineLoop(L)) {
+    LLVM_DEBUG(dbgs() << "\n!!! Can not pipeline loop.\n");
     return Changed;
+  }
 
   ++NumTrytoPipeline;
 
@@ -811,6 +223,50 @@ bool MachinePipeliner::scheduleLoop(MachineLoop &L) {
   return Changed;
 }
 
+void MachinePipeliner::setPragmaPipelineOptions(MachineLoop &L) {
+  MachineBasicBlock *LBLK = L.getTopBlock();
+
+  if (LBLK == nullptr)
+    return;
+
+  const BasicBlock *BBLK = LBLK->getBasicBlock();
+  if (BBLK == nullptr)
+    return;
+
+  const Instruction *TI = BBLK->getTerminator();
+  if (TI == nullptr)
+    return;
+
+  MDNode *LoopID = TI->getMetadata(LLVMContext::MD_loop);
+  if (LoopID == nullptr)
+    return;
+
+  assert(LoopID->getNumOperands() > 0 && "requires atleast one operand");
+  assert(LoopID->getOperand(0) == LoopID && "invalid loop");
+
+  for (unsigned i = 1, e = LoopID->getNumOperands(); i < e; ++i) {
+    MDNode *MD = dyn_cast<MDNode>(LoopID->getOperand(i));
+
+    if (MD == nullptr)
+      continue;
+
+    MDString *S = dyn_cast<MDString>(MD->getOperand(0));
+
+    if (S == nullptr)
+      continue;
+
+    if (S->getString() == "llvm.loop.pipeline.initiationinterval") {
+      assert(MD->getNumOperands() == 2 &&
+             "Pipeline initiation interval hint metadata should have two operands.");
+      II_setByPragma =
+          mdconst::extract<ConstantInt>(MD->getOperand(1))->getZExtValue();
+      assert(II_setByPragma >= 1 && "Pipeline initiation interval must be positive.");
+    } else if (S->getString() == "llvm.loop.pipeline.disable") {
+      disabledByPragma = true;
+    }
+  }
+}
+
 /// Return true if the loop can be software pipelined.  The algorithm is
 /// restricted to loops with a single basic block.  Make sure that the
 /// branch in the loop can be analyzed.
@@ -818,6 +274,9 @@ bool MachinePipeliner::canPipelineLoop(MachineLoop &L) {
   if (L.getNumBlocks() != 1)
     return false;
 
+  if (disabledByPragma)
+    return false;
+
   // Check if the branch can't be understood because we can't do pipelining
   // if that's the case.
   LI.TBB = nullptr;
@@ -876,7 +335,8 @@ void MachinePipeliner::preprocessPhiNodes(MachineBasicBlock &B) {
 bool MachinePipeliner::swingModuloScheduler(MachineLoop &L) {
   assert(L.getBlocks().size() == 1 && "SMS works on single blocks only.");
 
-  SwingSchedulerDAG SMS(*this, L, getAnalysis<LiveIntervals>(), RegClassInfo);
+  SwingSchedulerDAG SMS(*this, L, getAnalysis<LiveIntervals>(), RegClassInfo,
+                        II_setByPragma);
 
   MachineBasicBlock *MBB = L.getHeader();
   // The kernel should not include any terminator instructions.  These
@@ -899,6 +359,20 @@ bool MachinePipeliner::swingModuloScheduler(MachineLoop &L) {
   return SMS.hasNewSchedule();
 }
 
+void SwingSchedulerDAG::setMII(unsigned ResMII, unsigned RecMII) {
+  if (II_setByPragma > 0)
+    MII = II_setByPragma;
+  else
+    MII = std::max(ResMII, RecMII);
+}
+
+void SwingSchedulerDAG::setMAX_II() {
+  if (II_setByPragma > 0)
+    MAX_II = II_setByPragma;
+  else
+    MAX_II = MII + 10;
+}
+
 /// We override the schedule function in ScheduleDAGInstrs to implement the
 /// scheduling part of the Swing Modulo Scheduling algorithm.
 void SwingSchedulerDAG::schedule() {
@@ -925,9 +399,11 @@ void SwingSchedulerDAG::schedule() {
   if (SwpIgnoreRecMII)
     RecMII = 0;
 
-  MII = std::max(ResMII, RecMII);
-  LLVM_DEBUG(dbgs() << "MII = " << MII << " (rec=" << RecMII
-                    << ", res=" << ResMII << ")\n");
+  setMII(ResMII, RecMII);
+  setMAX_II();
+
+  LLVM_DEBUG(dbgs() << "MII = " << MII << " MAX_II = " << MAX_II
+                    << " (rec=" << RecMII << ", res=" << ResMII << ")\n");
 
   // Can't schedule a loop without a valid MII.
   if (MII == 0)
@@ -1557,7 +1033,7 @@ void SwingSchedulerDAG::Circuits::createAdjacencyStructure(
       }
     }
   }
-  // Add back-eges in the adjacency matrix for the output dependences.
+  // Add back-edges in the adjacency matrix for the output dependences.
   for (auto &OD : OutputDeps)
     if (!Added.test(OD.second)) {
       AdjK[OD.first].push_back(OD.second);
@@ -2107,7 +1583,7 @@ void SwingSchedulerDAG::groupRemainingNodes(NodeSetType &NodeSets) {
   }
 }
 
-/// Add the node to the set, and add all is its connected nodes to the set.
+/// Add the node to the set, and add all of its connected nodes to the set.
 void SwingSchedulerDAG::addConnectedNodes(SUnit *SU, NodeSet &NewSet,
                                           SetVector<SUnit *> &NodesAdded) {
   NewSet.insert(SU);
@@ -2335,8 +1811,9 @@ bool SwingSchedulerDAG::schedulePipeline(SMSchedule &Schedule) {
     return false;
 
   bool scheduleFound = false;
+  unsigned II = 0;
   // Keep increasing II until a valid schedule is found.
-  for (unsigned II = MII; II < MII + 10 && !scheduleFound; ++II) {
+  for (II = MII; II <= MAX_II && !scheduleFound; ++II) {
     Schedule.reset();
     Schedule.setInitiationInterval(II);
     LLVM_DEBUG(dbgs() << "Try to schedule with " << II << "\n");
@@ -2408,7 +1885,8 @@ bool SwingSchedulerDAG::schedulePipeline(SMSchedule &Schedule) {
       scheduleFound = Schedule.isValidSchedule(this);
   }
 
-  LLVM_DEBUG(dbgs() << "Schedule Found? " << scheduleFound << "\n");
+  LLVM_DEBUG(dbgs() << "Schedule Found? " << scheduleFound << " (II=" << II
+                    << ")\n");
 
   if (scheduleFound)
     Schedule.finalizeSchedule(this);
@@ -2773,7 +2251,7 @@ void SwingSchedulerDAG::generateExistingPhis(
       else if (PrologStage >= AccessStage + StageDiff + np &&
                VRMap[PrologStage - StageDiff - np].count(LoopVal) != 0)
         PhiOp1 = VRMap[PrologStage - StageDiff - np][LoopVal];
-      // Check if the Phi has already been scheduled, but the loop intruction
+      // Check if the Phi has already been scheduled, but the loop instruction
       // is either another Phi, or doesn't occur in the loop.
       else if (PrologStage >= AccessStage + StageDiff + np) {
         // If the Phi references another Phi, we need to examine the other
@@ -3288,7 +2766,9 @@ void SwingSchedulerDAG::updateMemOperands(MachineInstr &NewMI,
     return;
   SmallVector<MachineMemOperand *, 2> NewMMOs;
   for (MachineMemOperand *MMO : NewMI.memoperands()) {
-    if (MMO->isVolatile() || (MMO->isInvariant() && MMO->isDereferenceable()) ||
+    // TODO: Figure out whether isAtomic is really necessary (see D57601).
+    if (MMO->isVolatile() || MMO->isAtomic() ||
+        (MMO->isInvariant() && MMO->isDereferenceable()) ||
         (!MMO->getValue())) {
       NewMMOs.push_back(MMO);
       continue;
@@ -4300,6 +3780,14 @@ void SMSchedule::finalizeSchedule(SwingSchedulerDAG *SSD) {
   LLVM_DEBUG(dump(););
 }
 
+void NodeSet::print(raw_ostream &os) const {
+  os << "Num nodes " << size() << " rec " << RecMII << " mov " << MaxMOV
+     << " depth " << MaxDepth << " col " << Colocate << "\n";
+  for (const auto &I : Nodes)
+    os << "   SU(" << I->NodeNum << ") " << *(I->getInstr());
+  os << "\n";
+}
+
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
 /// Print the schedule information to the given output.
 void SMSchedule::print(raw_ostream &os) const {
@@ -4318,4 +3806,9 @@ void SMSchedule::print(raw_ostream &os) const {
 
 /// Utility function used for debugging to print the schedule.
 LLVM_DUMP_METHOD void SMSchedule::dump() const { print(dbgs()); }
+LLVM_DUMP_METHOD void NodeSet::dump() const { print(dbgs()); }
+
 #endif
+
+
+
diff --git a/lib/CodeGen/MachinePostDominators.cpp b/lib/CodeGen/MachinePostDominators.cpp
index 488377998cb31a080869da0fb1f094eb08623f32..7f220ed1fd8f33c98158ce6636f2b8178adf20dc 100644
--- a/lib/CodeGen/MachinePostDominators.cpp
+++ b/lib/CodeGen/MachinePostDominators.cpp
@@ -1,9 +1,8 @@
 //===- MachinePostDominators.cpp -Machine Post Dominator Calculation ------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/MachineRegionInfo.cpp b/lib/CodeGen/MachineRegionInfo.cpp
index 2619d8f78276d150cc701861ea5b049aa053fafa..2961d456be0d323499bd7a7a1379c831a6884124 100644
--- a/lib/CodeGen/MachineRegionInfo.cpp
+++ b/lib/CodeGen/MachineRegionInfo.cpp
@@ -1,9 +1,8 @@
 //===- lib/Codegen/MachineRegionInfo.cpp ----------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/CodeGen/MachineRegisterInfo.cpp b/lib/CodeGen/MachineRegisterInfo.cpp
index 6e5ca45d5e5eaef127c22361a9238b61e7a202f1..a632286c3a1ee537017064d98351b7bb5910acf9 100644
--- a/lib/CodeGen/MachineRegisterInfo.cpp
+++ b/lib/CodeGen/MachineRegisterInfo.cpp
@@ -1,9 +1,8 @@
 //===- lib/Codegen/MachineRegisterInfo.cpp --------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/MachineSSAUpdater.cpp b/lib/CodeGen/MachineSSAUpdater.cpp
index 542491eabbf29c6cbc33b7177a6916f816a4df48..e8b42047b49f5952cb0d528674cd90bce35cd179 100644
--- a/lib/CodeGen/MachineSSAUpdater.cpp
+++ b/lib/CodeGen/MachineSSAUpdater.cpp
@@ -1,9 +1,8 @@
 //===- MachineSSAUpdater.cpp - Unstructured SSA Update Tool ---------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/MachineScheduler.cpp b/lib/CodeGen/MachineScheduler.cpp
index ec11586b08ead6fb3326cbcbe2449e03afa07865..66d3a281d42dbb6c5c81f347a315b65e191295ac 100644
--- a/lib/CodeGen/MachineScheduler.cpp
+++ b/lib/CodeGen/MachineScheduler.cpp
@@ -1,9 +1,8 @@
 //===- MachineScheduler.cpp - Machine Instruction Scheduler ---------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -2460,13 +2459,13 @@ static unsigned computeRemLatency(SchedBoundary &CurrZone) {
 }
 
 /// Returns true if the current cycle plus remaning latency is greater than
-/// the cirtical path in the scheduling region.
+/// the critical path in the scheduling region.
 bool GenericSchedulerBase::shouldReduceLatency(const CandPolicy &Policy,
                                                SchedBoundary &CurrZone,
                                                bool ComputeRemLatency,
                                                unsigned &RemLatency) const {
   // The current cycle is already greater than the critical path, so we are
-  // already latnecy limited and don't need to compute the remaining latency.
+  // already latency limited and don't need to compute the remaining latency.
   if (CurrZone.getCurrCycle() > Rem.CriticalPath)
     return true;
 
diff --git a/lib/CodeGen/MachineSink.cpp b/lib/CodeGen/MachineSink.cpp
index cdc597db640166ed5973436a9520e5240aef470b..4e88fc0f7993791a57a3109166e4f7e8a32728ae 100644
--- a/lib/CodeGen/MachineSink.cpp
+++ b/lib/CodeGen/MachineSink.cpp
@@ -1,9 +1,8 @@
 //===- MachineSink.cpp - Sinking for machine instructions -----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/MachineTraceMetrics.cpp b/lib/CodeGen/MachineTraceMetrics.cpp
index e62ed3094651e9113271b2ca10c661c00c54c515..f9505df4e7f45a1802f42e47ba77be4662497c36 100644
--- a/lib/CodeGen/MachineTraceMetrics.cpp
+++ b/lib/CodeGen/MachineTraceMetrics.cpp
@@ -1,9 +1,8 @@
 //===- lib/CodeGen/MachineTraceMetrics.cpp --------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/CodeGen/MachineVerifier.cpp b/lib/CodeGen/MachineVerifier.cpp
index f4804e734f612e99de4920f21cb947e65467ed73..280283f939f720a92e9568b324228369992a1fa5 100644
--- a/lib/CodeGen/MachineVerifier.cpp
+++ b/lib/CodeGen/MachineVerifier.cpp
@@ -1,9 +1,8 @@
 //===- MachineVerifier.cpp - Machine Code Verifier ------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -231,6 +230,9 @@ namespace {
     void visitMachineFunctionBefore();
     void visitMachineBasicBlockBefore(const MachineBasicBlock *MBB);
     void visitMachineBundleBefore(const MachineInstr *MI);
+
+    bool verifyVectorElementMatch(LLT Ty0, LLT Ty1, const MachineInstr *MI);
+    void verifyPreISelGenericInstruction(const MachineInstr *MI);
     void visitMachineInstrBefore(const MachineInstr *MI);
     void visitMachineOperand(const MachineOperand *MO, unsigned MONum);
     void visitMachineInstrAfter(const MachineInstr *MI);
@@ -250,6 +252,7 @@ namespace {
     void report_context(const LiveRange::Segment &S) const;
     void report_context(const VNInfo &VNI) const;
     void report_context(SlotIndex Pos) const;
+    void report_context(MCPhysReg PhysReg) const;
     void report_context_liverange(const LiveRange &LR) const;
     void report_context_lanemask(LaneBitmask LaneMask) const;
     void report_context_vreg(unsigned VReg) const;
@@ -540,6 +543,10 @@ void MachineVerifier::report_context_liverange(const LiveRange &LR) const {
   errs() << "- liverange:   " << LR << '\n';
 }
 
+void MachineVerifier::report_context(MCPhysReg PReg) const {
+  errs() << "- p. register: " << printReg(PReg, TRI) << '\n';
+}
+
 void MachineVerifier::report_context_vreg(unsigned VReg) const {
   errs() << "- v. register: " << printReg(VReg, TRI) << '\n';
 }
@@ -619,6 +626,7 @@ MachineVerifier::visitMachineBasicBlockBefore(const MachineBasicBlock *MBB) {
       if (isAllocatable(LI.PhysReg) && !MBB->isEHPad() &&
           MBB->getIterator() != MBB->getParent()->begin()) {
         report("MBB has allocatable live-in, but isn't entry or landing-pad.", MBB);
+        report_context(LI.PhysReg);
       }
     }
   }
@@ -677,7 +685,7 @@ MachineVerifier::visitMachineBasicBlockBefore(const MachineBasicBlock *MBB) {
         // out the bottom of the function.
       } else if (MBB->succ_size() == LandingPadSuccs.size()) {
         // It's possible that the block legitimately ends with a noreturn
-        // call or an unreachable, in which case it won't actuall fall
+        // call or an unreachable, in which case it won't actually fall
         // out of the block.
       } else if (MBB->succ_size() != 1+LandingPadSuccs.size()) {
         report("MBB exits via unconditional fall-through but doesn't have "
@@ -883,109 +891,150 @@ void MachineVerifier::verifyInlineAsm(const MachineInstr *MI) {
   }
 }
 
-void MachineVerifier::visitMachineInstrBefore(const MachineInstr *MI) {
-  const MCInstrDesc &MCID = MI->getDesc();
-  if (MI->getNumOperands() < MCID.getNumOperands()) {
-    report("Too few operands", MI);
-    errs() << MCID.getNumOperands() << " operands expected, but "
-           << MI->getNumOperands() << " given.\n";
+/// Check that types are consistent when two operands need to have the same
+/// number of vector elements.
+/// \return true if the types are valid.
+bool MachineVerifier::verifyVectorElementMatch(LLT Ty0, LLT Ty1,
+                                               const MachineInstr *MI) {
+  if (Ty0.isVector() != Ty1.isVector()) {
+    report("operand types must be all-vector or all-scalar", MI);
+    // Generally we try to report as many issues as possible at once, but in
+    // this case it's not clear what should we be comparing the size of the
+    // scalar with: the size of the whole vector or its lane. Instead of
+    // making an arbitrary choice and emitting not so helpful message, let's
+    // avoid the extra noise and stop here.
+    return false;
   }
 
-  if (MI->isPHI()) {
-    if (MF->getProperties().hasProperty(
-            MachineFunctionProperties::Property::NoPHIs))
-      report("Found PHI instruction with NoPHIs property set", MI);
+  if (Ty0.isVector() && Ty0.getNumElements() != Ty1.getNumElements()) {
+    report("operand types must preserve number of vector elements", MI);
+    return false;
+  }
 
-    if (FirstNonPHI)
-      report("Found PHI instruction after non-PHI", MI);
-  } else if (FirstNonPHI == nullptr)
-    FirstNonPHI = MI;
+  return true;
+}
 
-  // Check the tied operands.
-  if (MI->isInlineAsm())
-    verifyInlineAsm(MI);
+void MachineVerifier::verifyPreISelGenericInstruction(const MachineInstr *MI) {
+  if (isFunctionSelected)
+    report("Unexpected generic instruction in a Selected function", MI);
 
-  // Check the MachineMemOperands for basic consistency.
-  for (MachineInstr::mmo_iterator I = MI->memoperands_begin(),
-                                  E = MI->memoperands_end();
+  const MCInstrDesc &MCID = MI->getDesc();
+  unsigned NumOps = MI->getNumOperands();
+
+  // Check types.
+  SmallVector<LLT, 4> Types;
+  for (unsigned I = 0, E = std::min(MCID.getNumOperands(), NumOps);
        I != E; ++I) {
-    if ((*I)->isLoad() && !MI->mayLoad())
-      report("Missing mayLoad flag", MI);
-    if ((*I)->isStore() && !MI->mayStore())
-      report("Missing mayStore flag", MI);
-  }
+    if (!MCID.OpInfo[I].isGenericType())
+      continue;
+    // Generic instructions specify type equality constraints between some of
+    // their operands. Make sure these are consistent.
+    size_t TypeIdx = MCID.OpInfo[I].getGenericTypeIndex();
+    Types.resize(std::max(TypeIdx + 1, Types.size()));
+
+    const MachineOperand *MO = &MI->getOperand(I);
+    if (!MO->isReg()) {
+      report("generic instruction must use register operands", MI);
+      continue;
+    }
 
-  // Debug values must not have a slot index.
-  // Other instructions must have one, unless they are inside a bundle.
-  if (LiveInts) {
-    bool mapped = !LiveInts->isNotInMIMap(*MI);
-    if (MI->isDebugInstr()) {
-      if (mapped)
-        report("Debug instruction has a slot index", MI);
-    } else if (MI->isInsideBundle()) {
-      if (mapped)
-        report("Instruction inside bundle has a slot index", MI);
+    LLT OpTy = MRI->getType(MO->getReg());
+    // Don't report a type mismatch if there is no actual mismatch, only a
+    // type missing, to reduce noise:
+    if (OpTy.isValid()) {
+      // Only the first valid type for a type index will be printed: don't
+      // overwrite it later so it's always clear which type was expected:
+      if (!Types[TypeIdx].isValid())
+        Types[TypeIdx] = OpTy;
+      else if (Types[TypeIdx] != OpTy)
+        report("Type mismatch in generic instruction", MO, I, OpTy);
     } else {
-      if (!mapped)
-        report("Missing slot index", MI);
+      // Generic instructions must have types attached to their operands.
+      report("Generic instruction is missing a virtual register type", MO, I);
     }
   }
 
-  if (isPreISelGenericOpcode(MCID.getOpcode())) {
-    if (isFunctionSelected)
-      report("Unexpected generic instruction in a Selected function", MI);
-
-    // Check types.
-    SmallVector<LLT, 4> Types;
-    for (unsigned I = 0; I < MCID.getNumOperands(); ++I) {
-      if (!MCID.OpInfo[I].isGenericType())
-        continue;
-      // Generic instructions specify type equality constraints between some of
-      // their operands. Make sure these are consistent.
-      size_t TypeIdx = MCID.OpInfo[I].getGenericTypeIndex();
-      Types.resize(std::max(TypeIdx + 1, Types.size()));
-
-      const MachineOperand *MO = &MI->getOperand(I);
-      LLT OpTy = MRI->getType(MO->getReg());
-      // Don't report a type mismatch if there is no actual mismatch, only a
-      // type missing, to reduce noise:
-      if (OpTy.isValid()) {
-        // Only the first valid type for a type index will be printed: don't
-        // overwrite it later so it's always clear which type was expected:
-        if (!Types[TypeIdx].isValid())
-          Types[TypeIdx] = OpTy;
-        else if (Types[TypeIdx] != OpTy)
-          report("Type mismatch in generic instruction", MO, I, OpTy);
-      } else {
-        // Generic instructions must have types attached to their operands.
-        report("Generic instruction is missing a virtual register type", MO, I);
-      }
-    }
-
-    // Generic opcodes must not have physical register operands.
-    for (unsigned I = 0; I < MI->getNumOperands(); ++I) {
-      const MachineOperand *MO = &MI->getOperand(I);
-      if (MO->isReg() && TargetRegisterInfo::isPhysicalRegister(MO->getReg()))
-        report("Generic instruction cannot have physical register", MO, I);
-    }
+  // Generic opcodes must not have physical register operands.
+  for (unsigned I = 0; I < MI->getNumOperands(); ++I) {
+    const MachineOperand *MO = &MI->getOperand(I);
+    if (MO->isReg() && TargetRegisterInfo::isPhysicalRegister(MO->getReg()))
+      report("Generic instruction cannot have physical register", MO, I);
   }
 
+  // Avoid out of bounds in checks below. This was already reported earlier.
+  if (MI->getNumOperands() < MCID.getNumOperands())
+    return;
+
   StringRef ErrorInfo;
   if (!TII->verifyInstruction(*MI, ErrorInfo))
     report(ErrorInfo.data(), MI);
 
   // Verify properties of various specific instruction types
-  switch(MI->getOpcode()) {
-  default:
+  switch (MI->getOpcode()) {
+  case TargetOpcode::G_CONSTANT:
+  case TargetOpcode::G_FCONSTANT: {
+    if (MI->getNumOperands() < MCID.getNumOperands())
+      break;
+
+    LLT DstTy = MRI->getType(MI->getOperand(0).getReg());
+    if (DstTy.isVector())
+      report("Instruction cannot use a vector result type", MI);
+
+    if (MI->getOpcode() == TargetOpcode::G_CONSTANT) {
+      if (!MI->getOperand(1).isCImm()) {
+        report("G_CONSTANT operand must be cimm", MI);
+        break;
+      }
+
+      const ConstantInt *CI = MI->getOperand(1).getCImm();
+      if (CI->getBitWidth() != DstTy.getSizeInBits())
+        report("inconsistent constant size", MI);
+    } else {
+      if (!MI->getOperand(1).isFPImm()) {
+        report("G_FCONSTANT operand must be fpimm", MI);
+        break;
+      }
+      const ConstantFP *CF = MI->getOperand(1).getFPImm();
+
+      if (APFloat::getSizeInBits(CF->getValueAPF().getSemantics()) !=
+          DstTy.getSizeInBits()) {
+        report("inconsistent constant size", MI);
+      }
+    }
+
     break;
+  }
   case TargetOpcode::G_LOAD:
   case TargetOpcode::G_STORE:
+  case TargetOpcode::G_ZEXTLOAD:
+  case TargetOpcode::G_SEXTLOAD: {
+    LLT ValTy = MRI->getType(MI->getOperand(0).getReg());
+    LLT PtrTy = MRI->getType(MI->getOperand(1).getReg());
+    if (!PtrTy.isPointer())
+      report("Generic memory instruction must access a pointer", MI);
+
     // Generic loads and stores must have a single MachineMemOperand
     // describing that access.
-    if (!MI->hasOneMemOperand())
+    if (!MI->hasOneMemOperand()) {
       report("Generic instruction accessing memory must have one mem operand",
              MI);
+    } else {
+      const MachineMemOperand &MMO = **MI->memoperands_begin();
+      if (MI->getOpcode() == TargetOpcode::G_ZEXTLOAD ||
+          MI->getOpcode() == TargetOpcode::G_SEXTLOAD) {
+        if (MMO.getSize() * 8 >= ValTy.getSizeInBits())
+          report("Generic extload must have a narrower memory type", MI);
+      } else if (MI->getOpcode() == TargetOpcode::G_LOAD) {
+        if (MMO.getSize() > (ValTy.getSizeInBits() + 7) / 8)
+          report("load memory size cannot exceed result size", MI);
+      } else if (MI->getOpcode() == TargetOpcode::G_STORE) {
+        if ((ValTy.getSizeInBits() + 7) / 8 < MMO.getSize())
+          report("store memory size cannot exceed value size", MI);
+      }
+    }
+
     break;
+  }
   case TargetOpcode::G_PHI: {
     LLT DstTy = MRI->getType(MI->getOperand(0).getReg());
     if (!DstTy.isValid() ||
@@ -1003,6 +1052,70 @@ void MachineVerifier::visitMachineInstrBefore(const MachineInstr *MI) {
              MI);
     break;
   }
+  case TargetOpcode::G_BITCAST: {
+    LLT DstTy = MRI->getType(MI->getOperand(0).getReg());
+    LLT SrcTy = MRI->getType(MI->getOperand(1).getReg());
+    if (!DstTy.isValid() || !SrcTy.isValid())
+      break;
+
+    if (SrcTy.isPointer() != DstTy.isPointer())
+      report("bitcast cannot convert between pointers and other types", MI);
+
+    if (SrcTy.getSizeInBits() != DstTy.getSizeInBits())
+      report("bitcast sizes must match", MI);
+    break;
+  }
+  case TargetOpcode::G_INTTOPTR:
+  case TargetOpcode::G_PTRTOINT:
+  case TargetOpcode::G_ADDRSPACE_CAST: {
+    LLT DstTy = MRI->getType(MI->getOperand(0).getReg());
+    LLT SrcTy = MRI->getType(MI->getOperand(1).getReg());
+    if (!DstTy.isValid() || !SrcTy.isValid())
+      break;
+
+    verifyVectorElementMatch(DstTy, SrcTy, MI);
+
+    DstTy = DstTy.getScalarType();
+    SrcTy = SrcTy.getScalarType();
+
+    if (MI->getOpcode() == TargetOpcode::G_INTTOPTR) {
+      if (!DstTy.isPointer())
+        report("inttoptr result type must be a pointer", MI);
+      if (SrcTy.isPointer())
+        report("inttoptr source type must not be a pointer", MI);
+    } else if (MI->getOpcode() == TargetOpcode::G_PTRTOINT) {
+      if (!SrcTy.isPointer())
+        report("ptrtoint source type must be a pointer", MI);
+      if (DstTy.isPointer())
+        report("ptrtoint result type must not be a pointer", MI);
+    } else {
+      assert(MI->getOpcode() == TargetOpcode::G_ADDRSPACE_CAST);
+      if (!SrcTy.isPointer() || !DstTy.isPointer())
+        report("addrspacecast types must be pointers", MI);
+      else {
+        if (SrcTy.getAddressSpace() == DstTy.getAddressSpace())
+          report("addrspacecast must convert different address spaces", MI);
+      }
+    }
+
+    break;
+  }
+  case TargetOpcode::G_GEP: {
+    LLT DstTy = MRI->getType(MI->getOperand(0).getReg());
+    LLT PtrTy = MRI->getType(MI->getOperand(1).getReg());
+    LLT OffsetTy = MRI->getType(MI->getOperand(2).getReg());
+    if (!DstTy.isValid() || !PtrTy.isValid() || !OffsetTy.isValid())
+      break;
+
+    if (!PtrTy.getScalarType().isPointer())
+      report("gep first operand must be a pointer", MI);
+
+    if (OffsetTy.getScalarType().isPointer())
+      report("gep offset operand must not be a pointer", MI);
+
+    // TODO: Is the offset allowed to be a scalar with a vector?
+    break;
+  }
   case TargetOpcode::G_SEXT:
   case TargetOpcode::G_ZEXT:
   case TargetOpcode::G_ANYEXT:
@@ -1015,30 +1128,18 @@ void MachineVerifier::visitMachineInstrBefore(const MachineInstr *MI) {
     // instructions aren't guaranteed to have the right number of operands or
     // types attached to them at this point
     assert(MCID.getNumOperands() == 2 && "Expected 2 operands G_*{EXT,TRUNC}");
-    if (MI->getNumOperands() < MCID.getNumOperands())
-      break;
     LLT DstTy = MRI->getType(MI->getOperand(0).getReg());
     LLT SrcTy = MRI->getType(MI->getOperand(1).getReg());
     if (!DstTy.isValid() || !SrcTy.isValid())
       break;
 
-    LLT DstElTy = DstTy.isVector() ? DstTy.getElementType() : DstTy;
-    LLT SrcElTy = SrcTy.isVector() ? SrcTy.getElementType() : SrcTy;
+    LLT DstElTy = DstTy.getScalarType();
+    LLT SrcElTy = SrcTy.getScalarType();
     if (DstElTy.isPointer() || SrcElTy.isPointer())
       report("Generic extend/truncate can not operate on pointers", MI);
 
-    if (DstTy.isVector() != SrcTy.isVector()) {
-      report("Generic extend/truncate must be all-vector or all-scalar", MI);
-      // Generally we try to report as many issues as possible at once, but in
-      // this case it's not clear what should we be comparing the size of the
-      // scalar with: the size of the whole vector or its lane. Instead of
-      // making an arbitrary choice and emitting not so helpful message, let's
-      // avoid the extra noise and stop here.
-      break;
-    }
-    if (DstTy.isVector() && DstTy.getNumElements() != SrcTy.getNumElements())
-      report("Generic vector extend/truncate must preserve number of lanes",
-             MI);
+    verifyVectorElementMatch(DstTy, SrcTy, MI);
+
     unsigned DstSize = DstElTy.getSizeInBits();
     unsigned SrcSize = SrcElTy.getSizeInBits();
     switch (MI->getOpcode()) {
@@ -1055,6 +1156,17 @@ void MachineVerifier::visitMachineInstrBefore(const MachineInstr *MI) {
     }
     break;
   }
+  case TargetOpcode::G_SELECT: {
+    LLT SelTy = MRI->getType(MI->getOperand(0).getReg());
+    LLT CondTy = MRI->getType(MI->getOperand(1).getReg());
+    if (!SelTy.isValid() || !CondTy.isValid())
+      break;
+
+    // Scalar condition select on a vector is valid.
+    if (CondTy.isVector())
+      verifyVectorElementMatch(SelTy, CondTy, MI);
+    break;
+  }
   case TargetOpcode::G_MERGE_VALUES: {
     // G_MERGE_VALUES should only be used to merge scalars into a larger scalar,
     // e.g. s2N = MERGE sN, sN
@@ -1138,6 +1250,103 @@ void MachineVerifier::visitMachineInstrBefore(const MachineInstr *MI) {
       report("G_CONCAT_VECTOR num dest and source elements should match", MI);
     break;
   }
+  case TargetOpcode::G_ICMP:
+  case TargetOpcode::G_FCMP: {
+    LLT DstTy = MRI->getType(MI->getOperand(0).getReg());
+    LLT SrcTy = MRI->getType(MI->getOperand(2).getReg());
+
+    if ((DstTy.isVector() != SrcTy.isVector()) ||
+        (DstTy.isVector() && DstTy.getNumElements() != SrcTy.getNumElements()))
+      report("Generic vector icmp/fcmp must preserve number of lanes", MI);
+
+    break;
+  }
+  case TargetOpcode::G_EXTRACT: {
+    const MachineOperand &SrcOp = MI->getOperand(1);
+    if (!SrcOp.isReg()) {
+      report("extract source must be a register", MI);
+      break;
+    }
+
+    const MachineOperand &OffsetOp = MI->getOperand(2);
+    if (!OffsetOp.isImm()) {
+      report("extract offset must be a constant", MI);
+      break;
+    }
+
+    unsigned DstSize = MRI->getType(MI->getOperand(0).getReg()).getSizeInBits();
+    unsigned SrcSize = MRI->getType(SrcOp.getReg()).getSizeInBits();
+    if (SrcSize == DstSize)
+      report("extract source must be larger than result", MI);
+
+    if (DstSize + OffsetOp.getImm() > SrcSize)
+      report("extract reads past end of register", MI);
+    break;
+  }
+  default:
+    break;
+  }
+}
+
+void MachineVerifier::visitMachineInstrBefore(const MachineInstr *MI) {
+  const MCInstrDesc &MCID = MI->getDesc();
+  if (MI->getNumOperands() < MCID.getNumOperands()) {
+    report("Too few operands", MI);
+    errs() << MCID.getNumOperands() << " operands expected, but "
+           << MI->getNumOperands() << " given.\n";
+  }
+
+  if (MI->isPHI()) {
+    if (MF->getProperties().hasProperty(
+            MachineFunctionProperties::Property::NoPHIs))
+      report("Found PHI instruction with NoPHIs property set", MI);
+
+    if (FirstNonPHI)
+      report("Found PHI instruction after non-PHI", MI);
+  } else if (FirstNonPHI == nullptr)
+    FirstNonPHI = MI;
+
+  // Check the tied operands.
+  if (MI->isInlineAsm())
+    verifyInlineAsm(MI);
+
+  // Check the MachineMemOperands for basic consistency.
+  for (MachineInstr::mmo_iterator I = MI->memoperands_begin(),
+                                  E = MI->memoperands_end();
+       I != E; ++I) {
+    if ((*I)->isLoad() && !MI->mayLoad())
+      report("Missing mayLoad flag", MI);
+    if ((*I)->isStore() && !MI->mayStore())
+      report("Missing mayStore flag", MI);
+  }
+
+  // Debug values must not have a slot index.
+  // Other instructions must have one, unless they are inside a bundle.
+  if (LiveInts) {
+    bool mapped = !LiveInts->isNotInMIMap(*MI);
+    if (MI->isDebugInstr()) {
+      if (mapped)
+        report("Debug instruction has a slot index", MI);
+    } else if (MI->isInsideBundle()) {
+      if (mapped)
+        report("Instruction inside bundle has a slot index", MI);
+    } else {
+      if (!mapped)
+        report("Missing slot index", MI);
+    }
+  }
+
+  if (isPreISelGenericOpcode(MCID.getOpcode())) {
+    verifyPreISelGenericInstruction(MI);
+    return;
+  }
+
+  StringRef ErrorInfo;
+  if (!TII->verifyInstruction(*MI, ErrorInfo))
+    report(ErrorInfo.data(), MI);
+
+  // Verify properties of various specific instruction types
+  switch (MI->getOpcode()) {
   case TargetOpcode::COPY: {
     if (foundErrors)
       break;
@@ -1187,7 +1396,8 @@ void MachineVerifier::visitMachineInstrBefore(const MachineInstr *MI) {
     VerifyStackMapConstant(VarStart + StatepointOpers::NumDeoptOperandsOffset);
 
     // TODO: verify we have properly encoded deopt arguments
-  };
+    break;
+  }
 }
 
 void
@@ -1350,7 +1560,7 @@ MachineVerifier::visitMachineOperand(const MachineOperand *MO, unsigned MONum) {
           return;
         }
         if (SubIdx)  {
-          report("Generic virtual register does not subregister index", MO,
+          report("Generic virtual register does not allow subregister index", MO,
                  MONum);
           return;
         }
diff --git a/lib/CodeGen/MacroFusion.cpp b/lib/CodeGen/MacroFusion.cpp
index 82b6d642c73b0cc68deff8e22e6186727b07fc3e..c7b8bf9fff68a7df7482c40c98af207b5ad87bba 100644
--- a/lib/CodeGen/MacroFusion.cpp
+++ b/lib/CodeGen/MacroFusion.cpp
@@ -1,9 +1,8 @@
 //===- MacroFusion.cpp - Macro Fusion -------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/OptimizePHIs.cpp b/lib/CodeGen/OptimizePHIs.cpp
index 770f6c5b0403020590152ae11162e080a5bf3687..c70b62252139e7ccc8474bd801f12d58235422f5 100644
--- a/lib/CodeGen/OptimizePHIs.cpp
+++ b/lib/CodeGen/OptimizePHIs.cpp
@@ -1,9 +1,8 @@
 //===- OptimizePHIs.cpp - Optimize machine instruction PHIs ---------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -182,11 +181,12 @@ bool OptimizePHIs::OptimizeBB(MachineBasicBlock &MBB) {
       if (!MRI->constrainRegClass(SingleValReg, MRI->getRegClass(OldReg)))
         continue;
 
-      // for the case SingleValReg taken from copy instr
-      MRI->clearKillFlags(SingleValReg);
-
       MRI->replaceRegWith(OldReg, SingleValReg);
       MI->eraseFromParent();
+
+      // The kill flags on OldReg and SingleValReg may no longer be correct.
+      MRI->clearKillFlags(SingleValReg);
+
       ++NumPHICycles;
       Changed = true;
       continue;
diff --git a/lib/CodeGen/PHIElimination.cpp b/lib/CodeGen/PHIElimination.cpp
index b9801c6fd97be0344383bdd3bd2249df6700cef2..948a5835438c8b390b3f99b83fd6e6c2e4059894 100644
--- a/lib/CodeGen/PHIElimination.cpp
+++ b/lib/CodeGen/PHIElimination.cpp
@@ -1,9 +1,8 @@
 //===- PhiElimination.cpp - Eliminate PHI nodes by inserting copies -------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/PHIEliminationUtils.cpp b/lib/CodeGen/PHIEliminationUtils.cpp
index 4e67ff2e5088d25fd846582c47a388759d4c5d46..3a2cdaf3bd3cb8aef2a58ba084c99d09867b5c05 100644
--- a/lib/CodeGen/PHIEliminationUtils.cpp
+++ b/lib/CodeGen/PHIEliminationUtils.cpp
@@ -1,9 +1,8 @@
 //===-- PHIEliminationUtils.cpp - Helper functions for PHI elimination ----===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/CodeGen/PHIEliminationUtils.h b/lib/CodeGen/PHIEliminationUtils.h
index b997d7ac5f4fe8f971c75f19d3769b01b581765e..0ff3a41f47d30ef21fc72c68840a4e0aa5846aac 100644
--- a/lib/CodeGen/PHIEliminationUtils.h
+++ b/lib/CodeGen/PHIEliminationUtils.h
@@ -1,9 +1,8 @@
 //=- PHIEliminationUtils.h - Helper functions for PHI elimination -*- C++ -*-=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/CodeGen/ParallelCG.cpp b/lib/CodeGen/ParallelCG.cpp
index bc3f2a6e9b5a21ea61a9153fe1d44fcfb81435ae..e4c73658cb4f7e8c2dd2bdf50f63f2cec29dfe3e 100644
--- a/lib/CodeGen/ParallelCG.cpp
+++ b/lib/CodeGen/ParallelCG.cpp
@@ -1,9 +1,8 @@
 //===-- ParallelCG.cpp ----------------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/PatchableFunction.cpp b/lib/CodeGen/PatchableFunction.cpp
index afb4b0a7e174e162222ce13e106d7279ec3d80df..a3fa1b0ad8ed332df72eb4805334053e8a3f7cb1 100644
--- a/lib/CodeGen/PatchableFunction.cpp
+++ b/lib/CodeGen/PatchableFunction.cpp
@@ -1,9 +1,8 @@
 //===-- PatchableFunction.cpp - Patchable prologues for LLVM -------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/PeepholeOptimizer.cpp b/lib/CodeGen/PeepholeOptimizer.cpp
index 1d058ccfb633569d56104e425a3cf4f0366a6ffd..f5277490ae61f974fba728d0c5c5b55a633990ba 100644
--- a/lib/CodeGen/PeepholeOptimizer.cpp
+++ b/lib/CodeGen/PeepholeOptimizer.cpp
@@ -1,9 +1,8 @@
 //===- PeepholeOptimizer.cpp - Peephole Optimizations ---------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/PostRAHazardRecognizer.cpp b/lib/CodeGen/PostRAHazardRecognizer.cpp
index f9d4a9746e4162b099580043d944f5ad9ad443c9..0a3838617bc5605babf95e1f6fc69f95107ed688 100644
--- a/lib/CodeGen/PostRAHazardRecognizer.cpp
+++ b/lib/CodeGen/PostRAHazardRecognizer.cpp
@@ -1,9 +1,8 @@
 //===----- PostRAHazardRecognizer.cpp - hazard recognizer -----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/PostRASchedulerList.cpp b/lib/CodeGen/PostRASchedulerList.cpp
index dd0a5fe1b39d9eb64c2025c1951b878469481b8a..5bea9f2893c91731505c170025128568e9474fe8 100644
--- a/lib/CodeGen/PostRASchedulerList.cpp
+++ b/lib/CodeGen/PostRASchedulerList.cpp
@@ -1,9 +1,8 @@
 //===----- SchedulePostRAList.cpp - list scheduler ------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/PreISelIntrinsicLowering.cpp b/lib/CodeGen/PreISelIntrinsicLowering.cpp
index b0e9ac03612d10c45cc6e51ff9e7f5cf7e206183..2752e186875c1288a5a4949957096396ac741a23 100644
--- a/lib/CodeGen/PreISelIntrinsicLowering.cpp
+++ b/lib/CodeGen/PreISelIntrinsicLowering.cpp
@@ -1,9 +1,8 @@
 //===- PreISelIntrinsicLowering.cpp - Pre-ISel intrinsic lowering pass ----===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -45,7 +44,7 @@ static bool lowerLoadRelative(Function &F) {
     Value *OffsetPtr =
         B.CreateGEP(Int8Ty, CI->getArgOperand(0), CI->getArgOperand(1));
     Value *OffsetPtrI32 = B.CreateBitCast(OffsetPtr, Int32PtrTy);
-    Value *OffsetI32 = B.CreateAlignedLoad(OffsetPtrI32, 4);
+    Value *OffsetI32 = B.CreateAlignedLoad(Int32Ty, OffsetPtrI32, 4);
 
     Value *ResultPtr = B.CreateGEP(Int8Ty, CI->getArgOperand(0), OffsetI32);
 
@@ -65,9 +64,9 @@ static bool lowerObjCCall(Function &F, const char *NewFn,
   // If we haven't already looked up this function, check to see if the
   // program already contains a function with this name.
   Module *M = F.getParent();
-  Constant* FCache = M->getOrInsertFunction(NewFn, F.getFunctionType());
+  FunctionCallee FCache = M->getOrInsertFunction(NewFn, F.getFunctionType());
 
-  if (Function* Fn = dyn_cast<Function>(FCache)) {
+  if (Function *Fn = dyn_cast<Function>(FCache.getCallee())) {
     Fn->setLinkage(F.getLinkage());
     if (setNonLazyBind && !Fn->isWeakForLinker()) {
       // If we have Native ARC, set nonlazybind attribute for these APIs for
diff --git a/lib/CodeGen/ProcessImplicitDefs.cpp b/lib/CodeGen/ProcessImplicitDefs.cpp
index 7e9b4af12ee9a639dab7e3e8512cbe0bf888a690..b38987ad1c90456832da296990bc01c8dd052cd5 100644
--- a/lib/CodeGen/ProcessImplicitDefs.cpp
+++ b/lib/CodeGen/ProcessImplicitDefs.cpp
@@ -1,9 +1,8 @@
 //===---------------------- ProcessImplicitDefs.cpp -----------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/CodeGen/PrologEpilogInserter.cpp b/lib/CodeGen/PrologEpilogInserter.cpp
index 23754e487a18112e00d87c0009759dce6a07865e..f2b1802fff2d4126e30409d02948d9be03fbcf60 100644
--- a/lib/CodeGen/PrologEpilogInserter.cpp
+++ b/lib/CodeGen/PrologEpilogInserter.cpp
@@ -1,9 +1,8 @@
 //===- PrologEpilogInserter.cpp - Insert Prolog/Epilog code in function ---===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -169,6 +168,46 @@ void PEI::getAnalysisUsage(AnalysisUsage &AU) const {
 /// StackObjSet - A set of stack object indexes
 using StackObjSet = SmallSetVector<int, 8>;
 
+using SavedDbgValuesMap =
+    SmallDenseMap<MachineBasicBlock *, SmallVector<MachineInstr *, 4>, 4>;
+
+/// Stash DBG_VALUEs that describe parameters and which are placed at the start
+/// of the block. Later on, after the prologue code has been emitted, the
+/// stashed DBG_VALUEs will be reinserted at the start of the block.
+static void stashEntryDbgValues(MachineBasicBlock &MBB,
+                                SavedDbgValuesMap &EntryDbgValues) {
+  SmallVector<const MachineInstr *, 4> FrameIndexValues;
+
+  for (auto &MI : MBB) {
+    if (!MI.isDebugInstr())
+      break;
+    if (!MI.isDebugValue() || !MI.getDebugVariable()->isParameter())
+      continue;
+    if (MI.getOperand(0).isFI()) {
+      // We can only emit valid locations for frame indices after the frame
+      // setup, so do not stash away them.
+      FrameIndexValues.push_back(&MI);
+      continue;
+    }
+    const DILocalVariable *Var = MI.getDebugVariable();
+    const DIExpression *Expr = MI.getDebugExpression();
+    auto Overlaps = [Var, Expr](const MachineInstr *DV) {
+      return Var == DV->getDebugVariable() &&
+             Expr->fragmentsOverlap(DV->getDebugExpression());
+    };
+    // See if the debug value overlaps with any preceding debug value that will
+    // not be stashed. If that is the case, then we can't stash this value, as
+    // we would then reorder the values at reinsertion.
+    if (llvm::none_of(FrameIndexValues, Overlaps))
+      EntryDbgValues[&MBB].push_back(&MI);
+  }
+
+  // Remove stashed debug values from the block.
+  if (EntryDbgValues.count(&MBB))
+    for (auto *MI : EntryDbgValues[&MBB])
+      MI->removeFromParent();
+}
+
 /// runOnMachineFunction - Insert prolog/epilog code and replace abstract
 /// frame indexes with appropriate references.
 bool PEI::runOnMachineFunction(MachineFunction &MF) {
@@ -192,6 +231,11 @@ bool PEI::runOnMachineFunction(MachineFunction &MF) {
   // place all spills in the entry block, all restores in return blocks.
   calculateSaveRestoreBlocks(MF);
 
+  // Stash away DBG_VALUEs that should not be moved by insertion of prolog code.
+  SavedDbgValuesMap EntryDbgValues;
+  for (MachineBasicBlock *SaveBlock : SaveBlocks)
+    stashEntryDbgValues(*SaveBlock, EntryDbgValues);
+
   // Handle CSR spilling and restoring, for targets that need it.
   if (MF.getTarget().usesPhysRegsForPEI())
     spillCalleeSavedRegs(MF);
@@ -211,6 +255,10 @@ bool PEI::runOnMachineFunction(MachineFunction &MF) {
   if (!F.hasFnAttribute(Attribute::Naked))
     insertPrologEpilogCode(MF);
 
+  // Reinsert stashed debug values at the start of the entry blocks.
+  for (auto &I : EntryDbgValues)
+    I.first->insert(I.first->begin(), I.second.begin(), I.second.end());
+
   // Replace all MO_FrameIndex operands with physical register references
   // and actual offsets.
   //
diff --git a/lib/CodeGen/PseudoSourceValue.cpp b/lib/CodeGen/PseudoSourceValue.cpp
index 6ca8d86e3f8e12d6875189fce124139ced1d1633..da3ef4b771f316c21411ec4a0c09d5813335e260 100644
--- a/lib/CodeGen/PseudoSourceValue.cpp
+++ b/lib/CodeGen/PseudoSourceValue.cpp
@@ -1,9 +1,8 @@
 //===-- llvm/CodeGen/PseudoSourceValue.cpp ----------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/ReachingDefAnalysis.cpp b/lib/CodeGen/ReachingDefAnalysis.cpp
index a9f0a9387297842b386c8df267f294965e3c96a2..f05c97ad621e2550c13af895b104e37b84bfe650 100644
--- a/lib/CodeGen/ReachingDefAnalysis.cpp
+++ b/lib/CodeGen/ReachingDefAnalysis.cpp
@@ -1,9 +1,8 @@
 //===---- ReachingDefAnalysis.cpp - Reaching Def Analysis ---*- C++ -*-----===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/CodeGen/RegAllocBase.cpp b/lib/CodeGen/RegAllocBase.cpp
index bc28a054c680a4e47ea990bb37f359656d6de969..0df951dc99793a72a3d6e47236ba30a02f1b2126 100644
--- a/lib/CodeGen/RegAllocBase.cpp
+++ b/lib/CodeGen/RegAllocBase.cpp
@@ -1,9 +1,8 @@
 //===- RegAllocBase.cpp - Register Allocator Base Class -------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/RegAllocBase.h b/lib/CodeGen/RegAllocBase.h
index 686ffc36e0493e1305ce8063d22b2eb39e044ec2..6a7cc5ba430841df829430addc6744f8d4c18b8c 100644
--- a/lib/CodeGen/RegAllocBase.h
+++ b/lib/CodeGen/RegAllocBase.h
@@ -1,9 +1,8 @@
 //===- RegAllocBase.h - basic regalloc interface and driver -----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/RegAllocBasic.cpp b/lib/CodeGen/RegAllocBasic.cpp
index daeff3fc3963fe2681088120946720273662cd20..46f6946f70031143b293118b7179e35044614835 100644
--- a/lib/CodeGen/RegAllocBasic.cpp
+++ b/lib/CodeGen/RegAllocBasic.cpp
@@ -1,9 +1,8 @@
 //===-- RegAllocBasic.cpp - Basic Register Allocator ----------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/RegAllocFast.cpp b/lib/CodeGen/RegAllocFast.cpp
index eb3a4e481f5d37d4ba9e8d8b22c79b1d1020ef43..ad080e9c2900923e2e518449da2dcac824994eb3 100644
--- a/lib/CodeGen/RegAllocFast.cpp
+++ b/lib/CodeGen/RegAllocFast.cpp
@@ -1,9 +1,8 @@
 //===- RegAllocFast.cpp - A fast register allocator for debug code --------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/RegAllocGreedy.cpp b/lib/CodeGen/RegAllocGreedy.cpp
index f244028534f23b2937264219d7e9ed5d244acced..9437469485941ef8956a055fabd10d5f0218338f 100644
--- a/lib/CodeGen/RegAllocGreedy.cpp
+++ b/lib/CodeGen/RegAllocGreedy.cpp
@@ -1,9 +1,8 @@
 //===- RegAllocGreedy.cpp - greedy register allocator ---------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -318,7 +317,7 @@ class RAGreedy : public MachineFunctionPass,
 
     /// Track new eviction.
     /// The Evictor vreg has evicted the Evictee vreg from Physreg.
-    /// \param PhysReg The phisical register Evictee was evicted from.
+    /// \param PhysReg The physical register Evictee was evicted from.
     /// \param Evictor The evictor Vreg that evicted Evictee.
     /// \param Evictee The evictee Vreg.
     void addEviction(unsigned PhysReg, unsigned Evictor, unsigned Evictee) {
diff --git a/lib/CodeGen/RegAllocPBQP.cpp b/lib/CodeGen/RegAllocPBQP.cpp
index c19001c8403df3ab4a9691906a19acc2d83a48d9..7a5a6c148ed47dfe39a23f0f7d4046eec940aac6 100644
--- a/lib/CodeGen/RegAllocPBQP.cpp
+++ b/lib/CodeGen/RegAllocPBQP.cpp
@@ -1,9 +1,8 @@
 //===- RegAllocPBQP.cpp ---- PBQP Register Allocator ----------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/RegUsageInfoCollector.cpp b/lib/CodeGen/RegUsageInfoCollector.cpp
index 66c7c5cd7dbf7137a8775973c8d7b35f416c01ef..01c8fcbc64ce41fa249eda570e4c90f4511ae167 100644
--- a/lib/CodeGen/RegUsageInfoCollector.cpp
+++ b/lib/CodeGen/RegUsageInfoCollector.cpp
@@ -1,9 +1,8 @@
 //===-- RegUsageInfoCollector.cpp - Register Usage Information Collector --===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
diff --git a/lib/CodeGen/RegUsageInfoPropagate.cpp b/lib/CodeGen/RegUsageInfoPropagate.cpp
index 256de295821d0e9f2c813603d31854282780f187..fc4be82d215e5354922287224ad852f56bd2f39d 100644
--- a/lib/CodeGen/RegUsageInfoPropagate.cpp
+++ b/lib/CodeGen/RegUsageInfoPropagate.cpp
@@ -1,9 +1,8 @@
 //=--- RegUsageInfoPropagate.cpp - Register Usage Informartion Propagation --=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
diff --git a/lib/CodeGen/RegisterClassInfo.cpp b/lib/CodeGen/RegisterClassInfo.cpp
index add8faec97d4a3382910f716a167cab2c91d7234..3d6687ef7b80b2c51f756bd1d323a6b2f060eb61 100644
--- a/lib/CodeGen/RegisterClassInfo.cpp
+++ b/lib/CodeGen/RegisterClassInfo.cpp
@@ -1,9 +1,8 @@
 //===- RegisterClassInfo.cpp - Dynamic Register Class Info ----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/RegisterCoalescer.cpp b/lib/CodeGen/RegisterCoalescer.cpp
index 771d50e7f4023397d106be7e4c17efda652c0996..976234db1033eed7405efc04e9a7ea49f89da03e 100644
--- a/lib/CodeGen/RegisterCoalescer.cpp
+++ b/lib/CodeGen/RegisterCoalescer.cpp
@@ -1,9 +1,8 @@
 //===- RegisterCoalescer.cpp - Generic Register Coalescing Interface ------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -176,7 +175,7 @@ namespace {
     /// If one def has many copy like uses, and those copy uses are all
     /// rematerialized, the live interval update needed for those
     /// rematerializations will be delayed and done all at once instead
-    /// of being done multiple times. This is to save compile cost becuase
+    /// of being done multiple times. This is to save compile cost because
     /// live interval update is costly.
     void lateLiveIntervalUpdate();
 
@@ -1910,6 +1909,13 @@ bool RegisterCoalescer::joinCopy(MachineInstr *CopyMI, bool &Again) {
     }
     LI.removeEmptySubRanges();
   }
+
+  // CP.getSrcReg()'s live interval has been merged into CP.getDstReg's live
+  // interval. Since CP.getSrcReg() is in ToBeUpdated set and its live interval
+  // is not up-to-date, need to update the merged live interval here.
+  if (ToBeUpdated.count(CP.getSrcReg()))
+    ShrinkMainRange = true;
+
   if (ShrinkMainRange) {
     LiveInterval &LI = LIS->getInterval(CP.getDstReg());
     shrinkToUses(&LI);
@@ -2501,8 +2507,10 @@ JoinVals::analyzeValue(unsigned ValNo, JoinVals &Other) {
         // We normally expect IMPLICIT_DEF values to be live only until the end
         // of their block. If the value is really live longer and gets pruned in
         // another block, this flag is cleared again.
+        //
+        // Clearing the valid lanes is deferred until it is sure this can be
+        // erased.
         V.ErasableImplicitDef = true;
-        V.ValidLanes &= ~V.WriteLanes;
       }
     }
   }
@@ -2557,20 +2565,25 @@ JoinVals::analyzeValue(unsigned ValNo, JoinVals &Other) {
   Other.computeAssignment(V.OtherVNI->id, *this);
   Val &OtherV = Other.Vals[V.OtherVNI->id];
 
-  // Check if OtherV is an IMPLICIT_DEF that extends beyond its basic block.
-  // This shouldn't normally happen, but ProcessImplicitDefs can leave such
-  // IMPLICIT_DEF instructions behind, and there is nothing wrong with it
-  // technically.
-  //
-  // When it happens, treat that IMPLICIT_DEF as a normal value, and don't try
-  // to erase the IMPLICIT_DEF instruction.
-  if (OtherV.ErasableImplicitDef && DefMI &&
-      DefMI->getParent() != Indexes->getMBBFromIndex(V.OtherVNI->def)) {
-    LLVM_DEBUG(dbgs() << "IMPLICIT_DEF defined at " << V.OtherVNI->def
-                      << " extends into "
-                      << printMBBReference(*DefMI->getParent())
-                      << ", keeping it.\n");
-    OtherV.ErasableImplicitDef = false;
+  if (OtherV.ErasableImplicitDef) {
+    // Check if OtherV is an IMPLICIT_DEF that extends beyond its basic block.
+    // This shouldn't normally happen, but ProcessImplicitDefs can leave such
+    // IMPLICIT_DEF instructions behind, and there is nothing wrong with it
+    // technically.
+    //
+    // When it happens, treat that IMPLICIT_DEF as a normal value, and don't try
+    // to erase the IMPLICIT_DEF instruction.
+    if (DefMI &&
+        DefMI->getParent() != Indexes->getMBBFromIndex(V.OtherVNI->def)) {
+      LLVM_DEBUG(dbgs() << "IMPLICIT_DEF defined at " << V.OtherVNI->def
+                 << " extends into "
+                 << printMBBReference(*DefMI->getParent())
+                 << ", keeping it.\n");
+      OtherV.ErasableImplicitDef = false;
+    } else {
+      // We deferred clearing these lanes in case we needed to save them
+      OtherV.ValidLanes &= ~OtherV.WriteLanes;
+    }
   }
 
   // Allow overlapping PHI values. Any real interference would show up in a
@@ -2613,6 +2626,12 @@ JoinVals::analyzeValue(unsigned ValNo, JoinVals &Other) {
     return CR_Erase;
   }
 
+  // The remaining checks apply to the lanes, which aren't tracked here.  This
+  // was already decided to be OK via the following CR_Replace condition.
+  // CR_Replace.
+  if (SubRangeJoin)
+    return CR_Replace;
+
   // If the lanes written by this instruction were all undef in OtherVNI, it is
   // still safe to join the live ranges. This can't be done with a simple value
   // mapping, though - OtherVNI will map to multiple values:
@@ -2694,8 +2713,18 @@ void JoinVals::computeAssignment(unsigned ValNo, JoinVals &Other) {
     Val &OtherV = Other.Vals[V.OtherVNI->id];
     // We cannot erase an IMPLICIT_DEF if we don't have valid values for all
     // its lanes.
-    if ((OtherV.WriteLanes & ~V.ValidLanes).any() && TrackSubRegLiveness)
+    if (OtherV.ErasableImplicitDef &&
+        TrackSubRegLiveness &&
+        (OtherV.WriteLanes & ~V.ValidLanes).any()) {
+      LLVM_DEBUG(dbgs() << "Cannot erase implicit_def with missing values\n");
+
       OtherV.ErasableImplicitDef = false;
+      // The valid lanes written by the implicit_def were speculatively cleared
+      // before, so make this more conservative. It may be better to track this,
+      // I haven't found a testcase where it matters.
+      OtherV.ValidLanes = LaneBitmask::getAll();
+    }
+
     OtherV.Pruned = true;
     LLVM_FALLTHROUGH;
   }
diff --git a/lib/CodeGen/RegisterCoalescer.h b/lib/CodeGen/RegisterCoalescer.h
index 1a46f6d053e64856912e7d706926d9ab668f1f9b..f505d46cd33801629c8051840ba2d6f16b560e65 100644
--- a/lib/CodeGen/RegisterCoalescer.h
+++ b/lib/CodeGen/RegisterCoalescer.h
@@ -1,9 +1,8 @@
 //===- RegisterCoalescer.h - Register Coalescing Interface ------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/RegisterPressure.cpp b/lib/CodeGen/RegisterPressure.cpp
index 1099e468e8850724e94bcce8188b95e6e38bac06..dafd95fa7d56f81fee2ab3a06faad6c8a2ecf6f2 100644
--- a/lib/CodeGen/RegisterPressure.cpp
+++ b/lib/CodeGen/RegisterPressure.cpp
@@ -1,9 +1,8 @@
 //===- RegisterPressure.cpp - Dynamic Register Pressure -------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/RegisterScavenging.cpp b/lib/CodeGen/RegisterScavenging.cpp
index 3660586c135813a4dd48257b751d102185f2fe9c..d00b55171b6e0c796fb89a82c22c08bd2241a91c 100644
--- a/lib/CodeGen/RegisterScavenging.cpp
+++ b/lib/CodeGen/RegisterScavenging.cpp
@@ -1,9 +1,8 @@
 //===- RegisterScavenging.cpp - Machine register scavenging ---------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/RegisterUsageInfo.cpp b/lib/CodeGen/RegisterUsageInfo.cpp
index 6b9880a8913f9f135c1e0590175d2e41a94e5819..6858d7233bc5fb53688018ae8df60f52dbee9365 100644
--- a/lib/CodeGen/RegisterUsageInfo.cpp
+++ b/lib/CodeGen/RegisterUsageInfo.cpp
@@ -1,9 +1,8 @@
 //===- RegisterUsageInfo.cpp - Register Usage Information Storage ---------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
diff --git a/lib/CodeGen/RenameIndependentSubregs.cpp b/lib/CodeGen/RenameIndependentSubregs.cpp
index 156d1c81c23834055587f0efb2d71bde1dbe91b9..22cff48c305124d994b805aef9b5d1fc485e30a6 100644
--- a/lib/CodeGen/RenameIndependentSubregs.cpp
+++ b/lib/CodeGen/RenameIndependentSubregs.cpp
@@ -1,9 +1,8 @@
 //===-- RenameIndependentSubregs.cpp - Live Interval Analysis -------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/ResetMachineFunctionPass.cpp b/lib/CodeGen/ResetMachineFunctionPass.cpp
index a02302e6ff993703134660c98fd11fce27a21ccf..2e5ff23b4f81aeb7a246f18f82fe2e913ea3b32d 100644
--- a/lib/CodeGen/ResetMachineFunctionPass.cpp
+++ b/lib/CodeGen/ResetMachineFunctionPass.cpp
@@ -1,9 +1,8 @@
 //===-- ResetMachineFunctionPass.cpp - Reset Machine Function ----*- C++ -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/lib/CodeGen/SafeStack.cpp b/lib/CodeGen/SafeStack.cpp
index 7b1c7fe49e4aaf3f8c5f987942d2648fc697cc59..f792fc6db912a7756bedd57f7eca7e0142b0af42 100644
--- a/lib/CodeGen/SafeStack.cpp
+++ b/lib/CodeGen/SafeStack.cpp
@@ -1,9 +1,8 @@
 //===- SafeStack.cpp - Safe Stack Insertion -------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -324,11 +323,8 @@ bool SafeStack::IsSafeStackAlloca(const Value *AllocaPtr, uint64_t AllocaSize) {
       case Instruction::Invoke: {
         ImmutableCallSite CS(I);
 
-        if (const IntrinsicInst *II = dyn_cast<IntrinsicInst>(I)) {
-          if (II->getIntrinsicID() == Intrinsic::lifetime_start ||
-              II->getIntrinsicID() == Intrinsic::lifetime_end)
-            continue;
-        }
+        if (I->isLifetimeStartOrEnd())
+          continue;
 
         if (const MemIntrinsic *MI = dyn_cast<MemIntrinsic>(I)) {
           if (!IsMemIntrinsicSafe(MI, UI, AllocaPtr, AllocaSize)) {
@@ -375,7 +371,7 @@ Value *SafeStack::getStackGuard(IRBuilder<> &IRB, Function &F) {
   if (!StackGuardVar)
     StackGuardVar =
         F.getParent()->getOrInsertGlobal("__stack_chk_guard", StackPtrTy);
-  return IRB.CreateLoad(StackGuardVar, "StackGuard");
+  return IRB.CreateLoad(StackPtrTy, StackGuardVar, "StackGuard");
 }
 
 void SafeStack::findInsts(Function &F,
@@ -456,7 +452,8 @@ SafeStack::createStackRestorePoints(IRBuilder<> &IRB, Function &F,
     ++NumUnsafeStackRestorePoints;
 
     IRB.SetInsertPoint(I->getNextNode());
-    Value *CurrentTop = DynamicTop ? IRB.CreateLoad(DynamicTop) : StaticTop;
+    Value *CurrentTop =
+        DynamicTop ? IRB.CreateLoad(StackPtrTy, DynamicTop) : StaticTop;
     IRB.CreateStore(CurrentTop, UnsafeStackPtr);
   }
 
@@ -465,7 +462,7 @@ SafeStack::createStackRestorePoints(IRBuilder<> &IRB, Function &F,
 
 void SafeStack::checkStackGuard(IRBuilder<> &IRB, Function &F, ReturnInst &RI,
                                 AllocaInst *StackGuardSlot, Value *StackGuard) {
-  Value *V = IRB.CreateLoad(StackGuardSlot);
+  Value *V = IRB.CreateLoad(StackPtrTy, StackGuardSlot);
   Value *Cmp = IRB.CreateICmpNE(StackGuard, V);
 
   auto SuccessProb = BranchProbabilityInfo::getBranchProbStackProtector(true);
@@ -478,8 +475,8 @@ void SafeStack::checkStackGuard(IRBuilder<> &IRB, Function &F, ReturnInst &RI,
                                 /* Unreachable */ true, Weights);
   IRBuilder<> IRBFail(CheckTerm);
   // FIXME: respect -fsanitize-trap / -ftrap-function here?
-  Constant *StackChkFail = F.getParent()->getOrInsertFunction(
-      "__stack_chk_fail", IRB.getVoidTy());
+  FunctionCallee StackChkFail =
+      F.getParent()->getOrInsertFunction("__stack_chk_fail", IRB.getVoidTy());
   IRBFail.CreateCall(StackChkFail, {});
 }
 
@@ -553,7 +550,7 @@ Value *SafeStack::moveStaticAllocasToUnsafeStack(
 
   if (StackGuardSlot) {
     unsigned Offset = SSL.getObjectOffset(StackGuardSlot);
-    Value *Off = IRB.CreateGEP(BasePointer, // BasePointer is i8*
+    Value *Off = IRB.CreateGEP(Int8Ty, BasePointer, // BasePointer is i8*
                                ConstantInt::get(Int32Ty, -Offset));
     Value *NewAI =
         IRB.CreateBitCast(Off, StackGuardSlot->getType(), "StackGuardSlot");
@@ -572,7 +569,7 @@ Value *SafeStack::moveStaticAllocasToUnsafeStack(
     if (Size == 0)
       Size = 1; // Don't create zero-sized stack objects.
 
-    Value *Off = IRB.CreateGEP(BasePointer, // BasePointer is i8*
+    Value *Off = IRB.CreateGEP(Int8Ty, BasePointer, // BasePointer is i8*
                                ConstantInt::get(Int32Ty, -Offset));
     Value *NewArg = IRB.CreateBitCast(Off, Arg->getType(),
                                      Arg->getName() + ".unsafe-byval");
@@ -612,7 +609,7 @@ Value *SafeStack::moveStaticAllocasToUnsafeStack(
         InsertBefore = User;
 
       IRBuilder<> IRBUser(InsertBefore);
-      Value *Off = IRBUser.CreateGEP(BasePointer, // BasePointer is i8*
+      Value *Off = IRBUser.CreateGEP(Int8Ty, BasePointer, // BasePointer is i8*
                                      ConstantInt::get(Int32Ty, -Offset));
       Value *Replacement = IRBUser.CreateBitCast(Off, AI->getType(), Name);
 
@@ -640,7 +637,7 @@ Value *SafeStack::moveStaticAllocasToUnsafeStack(
   IRB.SetInsertPoint(BasePointer->getNextNode());
 
   Value *StaticTop =
-      IRB.CreateGEP(BasePointer, ConstantInt::get(Int32Ty, -FrameSize),
+      IRB.CreateGEP(Int8Ty, BasePointer, ConstantInt::get(Int32Ty, -FrameSize),
                     "unsafe_stack_static_top");
   IRB.CreateStore(StaticTop, UnsafeStackPtr);
   return StaticTop;
@@ -663,7 +660,8 @@ void SafeStack::moveDynamicAllocasToUnsafeStack(
     uint64_t TySize = DL.getTypeAllocSize(Ty);
     Value *Size = IRB.CreateMul(ArraySize, ConstantInt::get(IntPtrTy, TySize));
 
-    Value *SP = IRB.CreatePtrToInt(IRB.CreateLoad(UnsafeStackPtr), IntPtrTy);
+    Value *SP = IRB.CreatePtrToInt(IRB.CreateLoad(StackPtrTy, UnsafeStackPtr),
+                                   IntPtrTy);
     SP = IRB.CreateSub(SP, Size);
 
     // Align the SP value to satisfy the AllocaInst, type and stack alignments.
@@ -701,7 +699,7 @@ void SafeStack::moveDynamicAllocasToUnsafeStack(
 
       if (II->getIntrinsicID() == Intrinsic::stacksave) {
         IRBuilder<> IRB(II);
-        Instruction *LI = IRB.CreateLoad(UnsafeStackPtr);
+        Instruction *LI = IRB.CreateLoad(StackPtrTy, UnsafeStackPtr);
         LI->takeName(II);
         II->replaceAllUsesWith(LI);
         II->eraseFromParent();
@@ -786,7 +784,7 @@ bool SafeStack::run() {
   if (DISubprogram *SP = F.getSubprogram())
     IRB.SetCurrentDebugLocation(DebugLoc::get(SP->getScopeLine(), 0, SP));
   if (SafeStackUsePointerAddress) {
-    Value *Fn = F.getParent()->getOrInsertFunction(
+    FunctionCallee Fn = F.getParent()->getOrInsertFunction(
         "__safestack_pointer_address", StackPtrTy->getPointerTo(0));
     UnsafeStackPtr = IRB.CreateCall(Fn);
   } else {
@@ -796,7 +794,7 @@ bool SafeStack::run() {
   // Load the current stack pointer (we'll also use it as a base pointer).
   // FIXME: use a dedicated register for it ?
   Instruction *BasePointer =
-      IRB.CreateLoad(UnsafeStackPtr, false, "unsafe_stack_ptr");
+      IRB.CreateLoad(StackPtrTy, UnsafeStackPtr, false, "unsafe_stack_ptr");
   assert(BasePointer->getType() == StackPtrTy);
 
   AllocaInst *StackGuardSlot = nullptr;
diff --git a/lib/CodeGen/SafeStackColoring.cpp b/lib/CodeGen/SafeStackColoring.cpp
index b1914b5d3804d66a2c7cd1341a036d0f9342a928..04a5c4b6d892fb7febe69b22b9a81df761c54b1b 100644
--- a/lib/CodeGen/SafeStackColoring.cpp
+++ b/lib/CodeGen/SafeStackColoring.cpp
@@ -1,9 +1,8 @@
 //===- SafeStackColoring.cpp - SafeStack frame coloring -------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -46,11 +45,10 @@ const StackColoring::LiveRange &StackColoring::getLiveRange(AllocaInst *AI) {
 }
 
 bool StackColoring::readMarker(Instruction *I, bool *IsStart) {
-  auto *II = dyn_cast<IntrinsicInst>(I);
-  if (!II || (II->getIntrinsicID() != Intrinsic::lifetime_start &&
-              II->getIntrinsicID() != Intrinsic::lifetime_end))
+  if (!I->isLifetimeStartOrEnd())
     return false;
 
+  auto *II = cast<IntrinsicInst>(I);
   *IsStart = II->getIntrinsicID() == Intrinsic::lifetime_start;
   return true;
 }
diff --git a/lib/CodeGen/SafeStackColoring.h b/lib/CodeGen/SafeStackColoring.h
index 902e63ebeb7e189ba3940bfaa929c9fd6e26eb09..b696b1b6baed155d01fdb1172dc4e372c1d1ac69 100644
--- a/lib/CodeGen/SafeStackColoring.h
+++ b/lib/CodeGen/SafeStackColoring.h
@@ -1,9 +1,8 @@
 //===- SafeStackColoring.h - SafeStack frame coloring ----------*- C++ -*--===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/CodeGen/SafeStackLayout.cpp b/lib/CodeGen/SafeStackLayout.cpp
index 07b6a5d1883b4e38ee3d6f8a0ae601827826d538..09964866e4d3b4ed19dc4419dd19e53d66123226 100644
--- a/lib/CodeGen/SafeStackLayout.cpp
+++ b/lib/CodeGen/SafeStackLayout.cpp
@@ -1,9 +1,8 @@
 //===- SafeStackLayout.cpp - SafeStack frame layout -----------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/CodeGen/SafeStackLayout.h b/lib/CodeGen/SafeStackLayout.h
index ac531d800f6e7aa89a5af3c986c84f064e3e344a..349d9a8b595c41ddccdc0546c7ccd04b77a4d2c4 100644
--- a/lib/CodeGen/SafeStackLayout.h
+++ b/lib/CodeGen/SafeStackLayout.h
@@ -1,9 +1,8 @@
 //===- SafeStackLayout.h - SafeStack frame layout --------------*- C++ -*--===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/CodeGen/ScalarizeMaskedMemIntrin.cpp b/lib/CodeGen/ScalarizeMaskedMemIntrin.cpp
index 2684f92b3a9318192860d36142cef1e697b68d6f..103ba7661d9a65e2766fa75cb23b8be313ba504c 100644
--- a/lib/CodeGen/ScalarizeMaskedMemIntrin.cpp
+++ b/lib/CodeGen/ScalarizeMaskedMemIntrin.cpp
@@ -1,10 +1,9 @@
 //===- ScalarizeMaskedMemIntrin.cpp - Scalarize unsupported masked mem ----===//
 //                                    instrinsics
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -144,7 +143,7 @@ static void scalarizeMaskedLoad(CallInst *CI) {
 
   // Short-cut if the mask is all-true.
   if (isa<Constant>(Mask) && cast<Constant>(Mask)->isAllOnesValue()) {
-    Value *NewI = Builder.CreateAlignedLoad(Ptr, AlignVal);
+    Value *NewI = Builder.CreateAlignedLoad(VecType, Ptr, AlignVal);
     CI->replaceAllUsesWith(NewI);
     CI->eraseFromParent();
     return;
@@ -167,7 +166,7 @@ static void scalarizeMaskedLoad(CallInst *CI) {
         continue;
       Value *Gep =
           Builder.CreateInBoundsGEP(EltTy, FirstEltPtr, Builder.getInt32(Idx));
-      LoadInst *Load = Builder.CreateAlignedLoad(Gep, AlignVal);
+      LoadInst *Load = Builder.CreateAlignedLoad(EltTy, Gep, AlignVal);
       VResult =
           Builder.CreateInsertElement(VResult, Load, Builder.getInt32(Idx));
     }
@@ -199,7 +198,7 @@ static void scalarizeMaskedLoad(CallInst *CI) {
 
     Value *Gep =
         Builder.CreateInBoundsGEP(EltTy, FirstEltPtr, Builder.getInt32(Idx));
-    LoadInst *Load = Builder.CreateAlignedLoad(Gep, AlignVal);
+    LoadInst *Load = Builder.CreateAlignedLoad(EltTy, Gep, AlignVal);
     Value *NewVResult = Builder.CreateInsertElement(VResult, Load,
                                                     Builder.getInt32(Idx));
 
@@ -367,6 +366,7 @@ static void scalarizeMaskedGather(CallInst *CI) {
   Value *Src0 = CI->getArgOperand(3);
 
   VectorType *VecType = cast<VectorType>(CI->getType());
+  Type *EltTy = VecType->getElementType();
 
   IRBuilder<> Builder(CI->getContext());
   Instruction *InsertPt = CI;
@@ -388,7 +388,7 @@ static void scalarizeMaskedGather(CallInst *CI) {
       Value *Ptr = Builder.CreateExtractElement(Ptrs, Builder.getInt32(Idx),
                                                 "Ptr" + Twine(Idx));
       LoadInst *Load =
-          Builder.CreateAlignedLoad(Ptr, AlignVal, "Load" + Twine(Idx));
+          Builder.CreateAlignedLoad(EltTy, Ptr, AlignVal, "Load" + Twine(Idx));
       VResult = Builder.CreateInsertElement(
           VResult, Load, Builder.getInt32(Idx), "Res" + Twine(Idx));
     }
@@ -419,7 +419,7 @@ static void scalarizeMaskedGather(CallInst *CI) {
     Value *Ptr = Builder.CreateExtractElement(Ptrs, Builder.getInt32(Idx),
                                               "Ptr" + Twine(Idx));
     LoadInst *Load =
-        Builder.CreateAlignedLoad(Ptr, AlignVal, "Load" + Twine(Idx));
+        Builder.CreateAlignedLoad(EltTy, Ptr, AlignVal, "Load" + Twine(Idx));
     Value *NewVResult = Builder.CreateInsertElement(VResult, Load,
                                                     Builder.getInt32(Idx),
                                                     "Res" + Twine(Idx));
diff --git a/lib/CodeGen/ScheduleDAG.cpp b/lib/CodeGen/ScheduleDAG.cpp
index 6c135b3d69d66ec37be8b974b2eba8ec0722d178..8ebe8fa6bd63c07ef9aa9b392d826fead749d982 100644
--- a/lib/CodeGen/ScheduleDAG.cpp
+++ b/lib/CodeGen/ScheduleDAG.cpp
@@ -1,9 +1,8 @@
 //===- ScheduleDAG.cpp - Implement the ScheduleDAG class ------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/ScheduleDAGInstrs.cpp b/lib/CodeGen/ScheduleDAGInstrs.cpp
index 99406ed1496a7cb49806d7d4c7fa836756d53f5f..3e68e09c44e3b499f77f841242ede33369c4a160 100644
--- a/lib/CodeGen/ScheduleDAGInstrs.cpp
+++ b/lib/CodeGen/ScheduleDAGInstrs.cpp
@@ -1,9 +1,8 @@
 //===---- ScheduleDAGInstrs.cpp - MachineInstr Rescheduling ---------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -132,7 +131,8 @@ static bool getUnderlyingObjectsForInstr(const MachineInstr *MI,
                                          const DataLayout &DL) {
   auto allMMOsOkay = [&]() {
     for (const MachineMemOperand *MMO : MI->memoperands()) {
-      if (MMO->isVolatile())
+      // TODO: Figure out whether isAtomic is really necessary (see D57601).
+      if (MMO->isVolatile() || MMO->isAtomic())
         return false;
 
       if (const PseudoSourceValue *PSV = MMO->getPseudoValue()) {
diff --git a/lib/CodeGen/ScheduleDAGPrinter.cpp b/lib/CodeGen/ScheduleDAGPrinter.cpp
index ff2085aae8653126d77998e6fa131f96ea409193..8d04711f07c64962799344d9698048a15e89e268 100644
--- a/lib/CodeGen/ScheduleDAGPrinter.cpp
+++ b/lib/CodeGen/ScheduleDAGPrinter.cpp
@@ -1,9 +1,8 @@
 //===-- ScheduleDAGPrinter.cpp - Implement ScheduleDAG::viewGraph() -------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/ScoreboardHazardRecognizer.cpp b/lib/CodeGen/ScoreboardHazardRecognizer.cpp
index 4301372179b8e28198748a103db10c93541a5fc5..a9fda56f2dacd5d91d1f103821f55c1653f42ce5 100644
--- a/lib/CodeGen/ScoreboardHazardRecognizer.cpp
+++ b/lib/CodeGen/ScoreboardHazardRecognizer.cpp
@@ -1,9 +1,8 @@
 //===- ScoreboardHazardRecognizer.cpp - Scheduler Support -----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index 7a22caf9c8b1e1ec3e4b67bcc7e52eb5bc798da4..c2404f963ce8ec747cdd5ea31addf043fa387d07 100644
--- a/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -1,9 +1,8 @@
 //===- DAGCombiner.cpp - Implement a DAG node combiner --------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -293,6 +292,8 @@ namespace {
     SDValue visitADD(SDNode *N);
     SDValue visitADDLike(SDValue N0, SDValue N1, SDNode *LocReference);
     SDValue visitSUB(SDNode *N);
+    SDValue visitADDSAT(SDNode *N);
+    SDValue visitSUBSAT(SDNode *N);
     SDValue visitADDC(SDNode *N);
     SDValue visitUADDO(SDNode *N);
     SDValue visitUADDOLike(SDValue N0, SDValue N1, SDNode *N);
@@ -478,7 +479,7 @@ namespace {
 
     /// Walk up chain skipping non-aliasing memory nodes,
     /// looking for aliasing nodes and adding them to the Aliases vector.
-    void GatherAllAliases(SDNode *N, SDValue OriginalChain,
+    void GatherAllAliases(LSBaseSDNode *N, SDValue OriginalChain,
                           SmallVectorImpl<SDValue> &Aliases);
 
     /// Return true if there is any possibility that the two addresses overlap.
@@ -486,7 +487,7 @@ namespace {
 
     /// Walk up chain skipping non-aliasing memory nodes, looking for a better
     /// chain (aliasing node.)
-    SDValue FindBetterChain(SDNode *N, SDValue Chain);
+    SDValue FindBetterChain(LSBaseSDNode *N, SDValue Chain);
 
     /// Try to replace a store and any possibly adjacent stores on
     /// consecutive chains with better chains. Return true only if St is
@@ -915,9 +916,11 @@ static bool isConstantOrConstantVector(SDValue N, bool NoOpaques = false) {
 
 // Determines if a BUILD_VECTOR is composed of all-constants possibly mixed with
 // undef's.
-static bool isAnyConstantBuildVector(const SDNode *N) {
-  return ISD::isBuildVectorOfConstantSDNodes(N) ||
-         ISD::isBuildVectorOfConstantFPSDNodes(N);
+static bool isAnyConstantBuildVector(SDValue V, bool NoOpaques = false) {
+  if (V.getOpcode() != ISD::BUILD_VECTOR)
+    return false;
+  return isConstantOrConstantVector(V, NoOpaques) ||
+         ISD::isBuildVectorOfConstantFPSDNodes(V.getNode());
 }
 
 SDValue DAGCombiner::ReassociateOps(unsigned Opc, const SDLoc &DL, SDValue N0,
@@ -1484,6 +1487,10 @@ SDValue DAGCombiner::visit(SDNode *N) {
   case ISD::MERGE_VALUES:       return visitMERGE_VALUES(N);
   case ISD::ADD:                return visitADD(N);
   case ISD::SUB:                return visitSUB(N);
+  case ISD::SADDSAT:
+  case ISD::UADDSAT:            return visitADDSAT(N);
+  case ISD::SSUBSAT:
+  case ISD::USUBSAT:            return visitSUBSAT(N);
   case ISD::ADDC:               return visitADDC(N);
   case ISD::UADDO:              return visitUADDO(N);
   case ISD::SUBC:               return visitSUBC(N);
@@ -2108,6 +2115,18 @@ SDValue DAGCombiner::visitADD(SDNode *N) {
   if (N0.getOpcode() == ISD::SUB && N1 == N0.getOperand(1))
     return N0.getOperand(0);
 
+  // fold ((A-B)+(C-A)) -> (C-B)
+  if (N0.getOpcode() == ISD::SUB && N1.getOpcode() == ISD::SUB &&
+      N0.getOperand(0) == N1.getOperand(1))
+    return DAG.getNode(ISD::SUB, DL, VT, N1.getOperand(0),
+                       N0.getOperand(1));
+
+  // fold ((A-B)+(B-C)) -> (A-C)
+  if (N0.getOpcode() == ISD::SUB && N1.getOpcode() == ISD::SUB &&
+      N0.getOperand(1) == N1.getOperand(0))
+    return DAG.getNode(ISD::SUB, DL, VT, N0.getOperand(0),
+                       N1.getOperand(1));
+
   // fold (A+(B-(A+C))) to (B-C)
   if (N1.getOpcode() == ISD::SUB && N1.getOperand(1).getOpcode() == ISD::ADD &&
       N0 == N1.getOperand(1).getOperand(0))
@@ -2140,6 +2159,18 @@ SDValue DAGCombiner::visitADD(SDNode *N) {
                          DAG.getNode(ISD::ADD, SDLoc(N1), VT, N01, N11));
   }
 
+  // fold (add (umax X, C), -C) --> (usubsat X, C)
+  if (N0.getOpcode() == ISD::UMAX && hasOperation(ISD::USUBSAT, VT)) {
+    auto MatchUSUBSAT = [](ConstantSDNode *Max, ConstantSDNode *Op) {
+      return (!Max && !Op) ||
+             (Max && Op && Max->getAPIntValue() == (-Op->getAPIntValue()));
+    };
+    if (ISD::matchBinaryPredicate(N0.getOperand(1), N1, MatchUSUBSAT,
+                                  /*AllowUndefs*/ true))
+      return DAG.getNode(ISD::USUBSAT, DL, VT, N0.getOperand(0),
+                         N0.getOperand(1));
+  }
+
   if (SDValue V = foldAddSubBoolOfMaskedVal(N, DAG))
     return V;
 
@@ -2168,6 +2199,49 @@ SDValue DAGCombiner::visitADD(SDNode *N) {
   return SDValue();
 }
 
+SDValue DAGCombiner::visitADDSAT(SDNode *N) {
+  unsigned Opcode = N->getOpcode();
+  SDValue N0 = N->getOperand(0);
+  SDValue N1 = N->getOperand(1);
+  EVT VT = N0.getValueType();
+  SDLoc DL(N);
+
+  // fold vector ops
+  if (VT.isVector()) {
+    // TODO SimplifyVBinOp
+
+    // fold (add_sat x, 0) -> x, vector edition
+    if (ISD::isBuildVectorAllZeros(N1.getNode()))
+      return N0;
+    if (ISD::isBuildVectorAllZeros(N0.getNode()))
+      return N1;
+  }
+
+  // fold (add_sat x, undef) -> -1
+  if (N0.isUndef() || N1.isUndef())
+    return DAG.getAllOnesConstant(DL, VT);
+
+  if (DAG.isConstantIntBuildVectorOrConstantInt(N0)) {
+    // canonicalize constant to RHS
+    if (!DAG.isConstantIntBuildVectorOrConstantInt(N1))
+      return DAG.getNode(Opcode, DL, VT, N1, N0);
+    // fold (add_sat c1, c2) -> c3
+    return DAG.FoldConstantArithmetic(Opcode, DL, VT, N0.getNode(),
+                                      N1.getNode());
+  }
+
+  // fold (add_sat x, 0) -> x
+  if (isNullConstant(N1))
+    return N0;
+
+  // If it cannot overflow, transform into an add.
+  if (Opcode == ISD::UADDSAT)
+    if (DAG.computeOverflowKind(N0, N1) == SelectionDAG::OFK_Never)
+      return DAG.getNode(ISD::ADD, DL, VT, N0, N1);
+
+  return SDValue();
+}
+
 static SDValue getAsCarry(const TargetLowering &TLI, SDValue V) {
   bool Masked = false;
 
@@ -2206,6 +2280,23 @@ static SDValue getAsCarry(const TargetLowering &TLI, SDValue V) {
   return SDValue();
 }
 
+/// Given the operands of an add/sub operation, see if the 2nd operand is a
+/// masked 0/1 whose source operand is actually known to be 0/-1. If so, invert
+/// the opcode and bypass the mask operation.
+static SDValue foldAddSubMasked1(bool IsAdd, SDValue N0, SDValue N1,
+                                 SelectionDAG &DAG, const SDLoc &DL) {
+  if (N1.getOpcode() != ISD::AND || !isOneOrOneSplat(N1->getOperand(1)))
+    return SDValue();
+
+  EVT VT = N0.getValueType();
+  if (DAG.ComputeNumSignBits(N1.getOperand(0)) != VT.getScalarSizeInBits())
+    return SDValue();
+
+  // add N0, (and (AssertSext X, i1), 1) --> sub N0, X
+  // sub N0, (and (AssertSext X, i1), 1) --> add N0, X
+  return DAG.getNode(IsAdd ? ISD::SUB : ISD::ADD, DL, VT, N0, N1.getOperand(0));
+}
+
 SDValue DAGCombiner::visitADDLike(SDValue N0, SDValue N1, SDNode *LocReference) {
   EVT VT = N0.getValueType();
   SDLoc DL(LocReference);
@@ -2218,21 +2309,15 @@ SDValue DAGCombiner::visitADDLike(SDValue N0, SDValue N1, SDNode *LocReference)
                                    N1.getOperand(0).getOperand(1),
                                    N1.getOperand(1)));
 
-  if (N1.getOpcode() == ISD::AND) {
-    SDValue AndOp0 = N1.getOperand(0);
-    unsigned NumSignBits = DAG.ComputeNumSignBits(AndOp0);
-    unsigned DestBits = VT.getScalarSizeInBits();
-
-    // (add z, (and (sbbl x, x), 1)) -> (sub z, (sbbl x, x))
-    // and similar xforms where the inner op is either ~0 or 0.
-    if (NumSignBits == DestBits && isOneOrOneSplat(N1->getOperand(1)))
-      return DAG.getNode(ISD::SUB, DL, VT, N0, AndOp0);
-  }
+  if (SDValue V = foldAddSubMasked1(true, N0, N1, DAG, DL))
+    return V;
 
-  // add (sext i1), X -> sub X, (zext i1)
+  // If the target's bool is represented as 0/1, prefer to make this 'sub 0/1'
+  // rather than 'add 0/-1' (the zext should get folded).
+  // add (sext i1 Y), X --> sub X, (zext i1 Y)
   if (N0.getOpcode() == ISD::SIGN_EXTEND &&
-      N0.getOperand(0).getValueType() == MVT::i1 &&
-      !TLI.isOperationLegal(ISD::SIGN_EXTEND, MVT::i1)) {
+      N0.getOperand(0).getScalarValueSizeInBits() == 1 &&
+      TLI.getBooleanContents(VT) == TargetLowering::ZeroOrOneBooleanContent) {
     SDValue ZExt = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, N0.getOperand(0));
     return DAG.getNode(ISD::SUB, DL, VT, N1, ZExt);
   }
@@ -2677,6 +2762,20 @@ SDValue DAGCombiner::visitSUB(SDNode *N) {
   if (SDValue V = foldAddSubOfSignBit(N, DAG))
     return V;
 
+  if (SDValue V = foldAddSubMasked1(false, N0, N1, DAG, SDLoc(N)))
+    return V;
+
+  // If the target's bool is represented as 0/-1, prefer to make this 'add 0/-1'
+  // rather than 'sub 0/1' (the sext should get folded).
+  // sub X, (zext i1 Y) --> add X, (sext i1 Y)
+  if (N1.getOpcode() == ISD::ZERO_EXTEND &&
+      N1.getOperand(0).getScalarValueSizeInBits() == 1 &&
+      TLI.getBooleanContents(VT) ==
+          TargetLowering::ZeroOrNegativeOneBooleanContent) {
+    SDValue SExt = DAG.getNode(ISD::SIGN_EXTEND, DL, VT, N1.getOperand(0));
+    return DAG.getNode(ISD::ADD, DL, VT, N0, SExt);
+  }
+
   // fold Y = sra (X, size(X)-1); sub (xor (X, Y), Y) -> (abs X)
   if (TLI.isOperationLegalOrCustom(ISD::ABS, VT)) {
     if (N0.getOpcode() == ISD::XOR && N1.getOpcode() == ISD::SRA) {
@@ -2730,6 +2829,43 @@ SDValue DAGCombiner::visitSUB(SDNode *N) {
   return SDValue();
 }
 
+SDValue DAGCombiner::visitSUBSAT(SDNode *N) {
+  SDValue N0 = N->getOperand(0);
+  SDValue N1 = N->getOperand(1);
+  EVT VT = N0.getValueType();
+  SDLoc DL(N);
+
+  // fold vector ops
+  if (VT.isVector()) {
+    // TODO SimplifyVBinOp
+
+    // fold (sub_sat x, 0) -> x, vector edition
+    if (ISD::isBuildVectorAllZeros(N1.getNode()))
+      return N0;
+  }
+
+  // fold (sub_sat x, undef) -> 0
+  if (N0.isUndef() || N1.isUndef())
+    return DAG.getConstant(0, DL, VT);
+
+  // fold (sub_sat x, x) -> 0
+  if (N0 == N1)
+    return DAG.getConstant(0, DL, VT);
+
+  if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
+      DAG.isConstantIntBuildVectorOrConstantInt(N1)) {
+    // fold (sub_sat c1, c2) -> c3
+    return DAG.FoldConstantArithmetic(N->getOpcode(), DL, VT, N0.getNode(),
+                                      N1.getNode());
+  }
+
+  // fold (sub_sat x, 0) -> x
+  if (isNullConstant(N1))
+    return N0;
+
+  return SDValue();
+}
+
 SDValue DAGCombiner::visitSUBC(SDNode *N) {
   SDValue N0 = N->getOperand(0);
   SDValue N1 = N->getOperand(1);
@@ -3161,8 +3297,19 @@ SDValue DAGCombiner::visitSDIV(SDNode *N) {
   if (DAG.SignBitIsZero(N1) && DAG.SignBitIsZero(N0))
     return DAG.getNode(ISD::UDIV, DL, N1.getValueType(), N0, N1);
 
-  if (SDValue V = visitSDIVLike(N0, N1, N))
+  if (SDValue V = visitSDIVLike(N0, N1, N)) {
+    // If the corresponding remainder node exists, update its users with
+    // (Dividend - (Quotient * Divisor).
+    if (SDNode *RemNode = DAG.getNodeIfExists(ISD::SREM, N->getVTList(),
+                                              { N0, N1 })) {
+      SDValue Mul = DAG.getNode(ISD::MUL, DL, VT, V, N1);
+      SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N0, Mul);
+      AddToWorklist(Mul.getNode());
+      AddToWorklist(Sub.getNode());
+      CombineTo(RemNode, Sub);
+    }
     return V;
+  }
 
   // sdiv, srem -> sdivrem
   // If the divisor is constant, then return DIVREM only if isIntDivCheap() is
@@ -3288,8 +3435,19 @@ SDValue DAGCombiner::visitUDIV(SDNode *N) {
   if (SDValue NewSel = foldBinOpIntoSelect(N))
     return NewSel;
 
-  if (SDValue V = visitUDIVLike(N0, N1, N))
+  if (SDValue V = visitUDIVLike(N0, N1, N)) {
+    // If the corresponding remainder node exists, update its users with
+    // (Dividend - (Quotient * Divisor).
+    if (SDNode *RemNode = DAG.getNodeIfExists(ISD::UREM, N->getVTList(),
+                                              { N0, N1 })) {
+      SDValue Mul = DAG.getNode(ISD::MUL, DL, VT, V, N1);
+      SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N0, Mul);
+      AddToWorklist(Mul.getNode());
+      AddToWorklist(Sub.getNode());
+      CombineTo(RemNode, Sub);
+    }
     return V;
+  }
 
   // sdiv, srem -> sdivrem
   // If the divisor is constant, then return DIVREM only if isIntDivCheap() is
@@ -3408,6 +3566,11 @@ SDValue DAGCombiner::visitREM(SDNode *N) {
     SDValue OptimizedDiv =
         isSigned ? visitSDIVLike(N0, N1, N) : visitUDIVLike(N0, N1, N);
     if (OptimizedDiv.getNode()) {
+      // If the equivalent Div node also exists, update its users.
+      unsigned DivOpcode = isSigned ? ISD::SDIV : ISD::UDIV;
+      if (SDNode *DivNode = DAG.getNodeIfExists(DivOpcode, N->getVTList(),
+                                                { N0, N1 }))
+        CombineTo(DivNode, OptimizedDiv);
       SDValue Mul = DAG.getNode(ISD::MUL, DL, VT, OptimizedDiv, N1);
       SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N0, Mul);
       AddToWorklist(OptimizedDiv.getNode());
@@ -3731,13 +3894,12 @@ SDValue DAGCombiner::hoistLogicOpWithSameOpcodeHands(SDNode *N) {
     if (!N0.hasOneUse() && !N1.hasOneUse())
       return SDValue();
     // We need matching integer source types.
-    // Do not hoist logic op inside of a vector extend, since it may combine
-    // into a vsetcc.
-    // TODO: Should the vector check apply to truncate though?
-    if (VT.isVector() || XVT != Y.getValueType())
+    if (XVT != Y.getValueType())
       return SDValue();
-    // Don't create an illegal op during or after legalization.
-    if (LegalOperations && !TLI.isOperationLegal(LogicOpcode, XVT))
+    // Don't create an illegal op during or after legalization. Don't ever
+    // create an unsupported vector op.
+    if ((VT.isVector() || LegalOperations) &&
+        !TLI.isOperationLegalOrCustom(LogicOpcode, XVT))
       return SDValue();
     // Avoid infinite looping with PromoteIntBinOp.
     // TODO: Should we apply desirable/legal constraints to all opcodes?
@@ -3961,6 +4123,32 @@ SDValue DAGCombiner::foldLogicOfSetCCs(bool IsAnd, SDValue N0, SDValue N1,
       SDValue Zero = DAG.getConstant(0, DL, OpVT);
       return DAG.getSetCC(DL, VT, Or, Zero, CC1);
     }
+
+    // Turn compare of constants whose difference is 1 bit into add+and+setcc.
+    if ((IsAnd && CC1 == ISD::SETNE) || (!IsAnd && CC1 == ISD::SETEQ)) {
+      // Match a shared variable operand and 2 non-opaque constant operands.
+      ConstantSDNode *C0 = isConstOrConstSplat(LR);
+      ConstantSDNode *C1 = isConstOrConstSplat(RR);
+      if (LL == RL && C0 && C1 && !C0->isOpaque() && !C1->isOpaque()) {
+        // Canonicalize larger constant as C0.
+        if (C1->getAPIntValue().ugt(C0->getAPIntValue()))
+          std::swap(C0, C1);
+
+        // The difference of the constants must be a single bit.
+        const APInt &C0Val = C0->getAPIntValue();
+        const APInt &C1Val = C1->getAPIntValue();
+        if ((C0Val - C1Val).isPowerOf2()) {
+          // and/or (setcc X, C0, ne), (setcc X, C1, ne/eq) -->
+          // setcc ((add X, -C1), ~(C0 - C1)), 0, ne/eq
+          SDValue OffsetC = DAG.getConstant(-C1Val, DL, OpVT);
+          SDValue Add = DAG.getNode(ISD::ADD, DL, OpVT, LL, OffsetC);
+          SDValue MaskC = DAG.getConstant(~(C0Val - C1Val), DL, OpVT);
+          SDValue And = DAG.getNode(ISD::AND, DL, OpVT, Add, MaskC);
+          SDValue Zero = DAG.getConstant(0, DL, OpVT);
+          return DAG.getSetCC(DL, VT, And, Zero, CC0);
+        }
+      }
+    }
   }
 
   // Canonicalize equivalent operands to LL == RL.
@@ -4610,9 +4798,8 @@ SDValue DAGCombiner::visitAND(SDNode *N) {
     if (SDValue Res = ReduceLoadWidth(N)) {
       LoadSDNode *LN0 = N0->getOpcode() == ISD::ANY_EXTEND
         ? cast<LoadSDNode>(N0.getOperand(0)) : cast<LoadSDNode>(N0);
-
       AddToWorklist(N);
-      CombineTo(LN0, Res, Res.getValue(1));
+      DAG.ReplaceAllUsesOfValueWith(SDValue(LN0, 0), Res);
       return SDValue(N, 0);
     }
   }
@@ -5175,12 +5362,12 @@ SDValue DAGCombiner::visitOR(SDNode *N) {
     return ROR;
 
   // Canonicalize (or (and X, c1), c2) -> (and (or X, c2), c1|c2)
-  // iff (c1 & c2) != 0.
-  auto MatchIntersect = [](ConstantSDNode *LHS, ConstantSDNode *RHS) {
-    return LHS->getAPIntValue().intersects(RHS->getAPIntValue());
+  // iff (c1 & c2) != 0 or c1/c2 are undef.
+  auto MatchIntersect = [](ConstantSDNode *C1, ConstantSDNode *C2) {
+    return !C1 || !C2 || C1->getAPIntValue().intersects(C2->getAPIntValue());
   };
   if (N0.getOpcode() == ISD::AND && N0.getNode()->hasOneUse() &&
-      ISD::matchBinaryPredicate(N0.getOperand(1), N1, MatchIntersect)) {
+      ISD::matchBinaryPredicate(N0.getOperand(1), N1, MatchIntersect, true)) {
     if (SDValue COR = DAG.FoldConstantArithmetic(
             ISD::OR, SDLoc(N1), VT, N1.getNode(), N0.getOperand(1).getNode())) {
       SDValue IOR = DAG.getNode(ISD::OR, SDLoc(N0), VT, N0.getOperand(0), N1);
@@ -5387,8 +5574,7 @@ static bool matchRotateSub(SDValue Pos, SDValue Neg, unsigned EltSize,
   unsigned MaskLoBits = 0;
   if (Neg.getOpcode() == ISD::AND && isPowerOf2_64(EltSize)) {
     if (ConstantSDNode *NegC = isConstOrConstSplat(Neg.getOperand(1))) {
-      KnownBits Known;
-      DAG.computeKnownBits(Neg.getOperand(0), Known);
+      KnownBits Known = DAG.computeKnownBits(Neg.getOperand(0));
       unsigned Bits = Log2_64(EltSize);
       if (NegC->getAPIntValue().getActiveBits() <= Bits &&
           ((NegC->getAPIntValue() | Known.Zero).countTrailingOnes() >= Bits)) {
@@ -5410,8 +5596,7 @@ static bool matchRotateSub(SDValue Pos, SDValue Neg, unsigned EltSize,
   // Pos'.  The truncation is redundant for the purpose of the equality.
   if (MaskLoBits && Pos.getOpcode() == ISD::AND) {
     if (ConstantSDNode *PosC = isConstOrConstSplat(Pos.getOperand(1))) {
-      KnownBits Known;
-      DAG.computeKnownBits(Pos.getOperand(0), Known);
+      KnownBits Known = DAG.computeKnownBits(Pos.getOperand(0));
       if (PosC->getAPIntValue().getActiveBits() <= MaskLoBits &&
           ((PosC->getAPIntValue() | Known.Zero).countTrailingOnes() >=
            MaskLoBits))
@@ -6518,7 +6703,8 @@ SDValue DAGCombiner::visitSHL(SDNode *N) {
   //                               (and (srl x, (sub c1, c2), MASK)
   // Only fold this if the inner shift has no other uses -- if it does, folding
   // this will increase the total number of instructions.
-  if (N1C && N0.getOpcode() == ISD::SRL && N0.hasOneUse()) {
+  if (N1C && N0.getOpcode() == ISD::SRL && N0.hasOneUse() &&
+      TLI.shouldFoldShiftPairToMask(N, Level)) {
     if (ConstantSDNode *N0C1 = isConstOrConstSplat(N0.getOperand(1))) {
       uint64_t c1 = N0C1->getZExtValue();
       if (c1 < OpSizeInBits) {
@@ -6866,8 +7052,7 @@ SDValue DAGCombiner::visitSRL(SDNode *N) {
   // fold (srl (ctlz x), "5") -> x  iff x has one bit set (the low bit).
   if (N1C && N0.getOpcode() == ISD::CTLZ &&
       N1C->getAPIntValue() == Log2_32(OpSizeInBits)) {
-    KnownBits Known;
-    DAG.computeKnownBits(N0.getOperand(0), Known);
+    KnownBits Known = DAG.computeKnownBits(N0.getOperand(0));
 
     // If any of the input bits are KnownOne, then the input couldn't be all
     // zeros, thus the result of the srl will always be zero.
@@ -6968,13 +7153,48 @@ SDValue DAGCombiner::visitFunnelShift(SDNode *N) {
             N2, APInt(N2.getScalarValueSizeInBits(), BitWidth - 1)))
       return IsFSHL ? N0 : N1;
 
-  // fold (fsh* N0, N1, c) -> (fsh* N0, N1, c % BitWidth)
+  auto IsUndefOrZero = [](SDValue V) {
+    return V.isUndef() || isNullOrNullSplat(V, /*AllowUndefs*/ true);
+  };
+
   if (ConstantSDNode *Cst = isConstOrConstSplat(N2)) {
+    EVT ShAmtTy = N2.getValueType();
+
+    // fold (fsh* N0, N1, c) -> (fsh* N0, N1, c % BitWidth)
     if (Cst->getAPIntValue().uge(BitWidth)) {
       uint64_t RotAmt = Cst->getAPIntValue().urem(BitWidth);
       return DAG.getNode(N->getOpcode(), SDLoc(N), VT, N0, N1,
-                         DAG.getConstant(RotAmt, SDLoc(N), N2.getValueType()));
+                         DAG.getConstant(RotAmt, SDLoc(N), ShAmtTy));
     }
+
+    unsigned ShAmt = Cst->getZExtValue();
+    if (ShAmt == 0)
+      return IsFSHL ? N0 : N1;
+
+    // fold fshl(undef_or_zero, N1, C) -> lshr(N1, BW-C)
+    // fold fshr(undef_or_zero, N1, C) -> lshr(N1, C)
+    // fold fshl(N0, undef_or_zero, C) -> shl(N0, C)
+    // fold fshr(N0, undef_or_zero, C) -> shl(N0, BW-C)
+    if (IsUndefOrZero(N0))
+      return DAG.getNode(ISD::SRL, SDLoc(N), VT, N1,
+                         DAG.getConstant(IsFSHL ? BitWidth - ShAmt : ShAmt,
+                                         SDLoc(N), ShAmtTy));
+    if (IsUndefOrZero(N1))
+      return DAG.getNode(ISD::SHL, SDLoc(N), VT, N0,
+                         DAG.getConstant(IsFSHL ? ShAmt : BitWidth - ShAmt,
+                                         SDLoc(N), ShAmtTy));
+  }
+
+  // fold fshr(undef_or_zero, N1, N2) -> lshr(N1, N2)
+  // fold fshl(N0, undef_or_zero, N2) -> shl(N0, N2)
+  // iff We know the shift amount is in range.
+  // TODO: when is it worth doing SUB(BW, N2) as well?
+  if (isPowerOf2_32(BitWidth)) {
+    APInt ModuloBits(N2.getScalarValueSizeInBits(), BitWidth - 1);
+    if (IsUndefOrZero(N0) && !IsFSHL && DAG.MaskedValueIsZero(N2, ~ModuloBits))
+      return DAG.getNode(ISD::SRL, SDLoc(N), VT, N1, N2);
+    if (IsUndefOrZero(N1) && IsFSHL && DAG.MaskedValueIsZero(N2, ~ModuloBits))
+      return DAG.getNode(ISD::SHL, SDLoc(N), VT, N0, N2);
   }
 
   // fold (fshl N0, N0, N2) -> (rotl N0, N2)
@@ -6985,6 +7205,10 @@ SDValue DAGCombiner::visitFunnelShift(SDNode *N) {
   if (N0 == N1 && hasOperation(RotOpc, VT))
     return DAG.getNode(RotOpc, SDLoc(N), VT, N0, N2);
 
+  // Simplify, based on bits shifted out of N0/N1.
+  if (SimplifyDemandedBits(SDValue(N, 0)))
+    return SDValue(N, 0);
+
   return SDValue();
 }
 
@@ -7096,11 +7320,14 @@ SDValue DAGCombiner::visitCTPOP(SDNode *N) {
 
 // FIXME: This should be checking for no signed zeros on individual operands, as
 // well as no nans.
-static bool isLegalToCombineMinNumMaxNum(SelectionDAG &DAG, SDValue LHS, SDValue RHS) {
+static bool isLegalToCombineMinNumMaxNum(SelectionDAG &DAG, SDValue LHS,
+                                         SDValue RHS,
+                                         const TargetLowering &TLI) {
   const TargetOptions &Options = DAG.getTarget().Options;
   EVT VT = LHS.getValueType();
 
   return Options.NoSignedZerosFPMath && VT.isFloatingPoint() &&
+         TLI.isProfitableToCombineMinNumMaxNum(VT) &&
          DAG.isKnownNeverNaN(LHS) && DAG.isKnownNeverNaN(RHS);
 }
 
@@ -7307,6 +7534,9 @@ SDValue DAGCombiner::visitSELECT(SDNode *N) {
       if (normalizeToSequence || !InnerSelect.use_empty())
         return DAG.getNode(ISD::SELECT, DL, N1.getValueType(), Cond0,
                            InnerSelect, N2);
+      // Cleanup on failure.
+      if (InnerSelect.use_empty())
+        recursivelyDeleteUnusedNodes(InnerSelect.getNode());
     }
     // select (or Cond0, Cond1), X, Y -> select Cond0, X, (select Cond1, X, Y)
     if (N0->getOpcode() == ISD::OR && N0->hasOneUse()) {
@@ -7317,6 +7547,9 @@ SDValue DAGCombiner::visitSELECT(SDNode *N) {
       if (normalizeToSequence || !InnerSelect.use_empty())
         return DAG.getNode(ISD::SELECT, DL, N1.getValueType(), Cond0, N1,
                            InnerSelect);
+      // Cleanup on failure.
+      if (InnerSelect.use_empty())
+        recursivelyDeleteUnusedNodes(InnerSelect.getNode());
     }
 
     // select Cond0, (select Cond1, X, Y), Y -> select (and Cond0, Cond1), X, Y
@@ -7370,7 +7603,7 @@ SDValue DAGCombiner::visitSELECT(SDNode *N) {
     // select (fcmp gt x, y), x, y -> fmaxnum x, y
     //
     // This is OK if we don't care what happens if either operand is a NaN.
-    if (N0.hasOneUse() && isLegalToCombineMinNumMaxNum(DAG, N1, N2))
+    if (N0.hasOneUse() && isLegalToCombineMinNumMaxNum(DAG, N1, N2, TLI))
       if (SDValue FMinMax = combineMinNumMaxNum(DL, VT, Cond0, Cond1, N1, N2,
                                                 CC, TLI, DAG))
         return FMinMax;
@@ -7877,7 +8110,8 @@ SDValue DAGCombiner::visitVSELECT(SDNode *N) {
     // NaN.
     //
     EVT VT = N->getValueType(0);
-    if (N0.hasOneUse() && isLegalToCombineMinNumMaxNum(DAG, N0.getOperand(0), N0.getOperand(1))) {
+    if (N0.hasOneUse() && isLegalToCombineMinNumMaxNum(
+                              DAG, N0.getOperand(0), N0.getOperand(1), TLI)) {
       ISD::CondCode CC = cast<CondCodeSDNode>(N0.getOperand(2))->get();
       if (SDValue FMinMax = combineMinNumMaxNum(
             DL, VT, N0.getOperand(0), N0.getOperand(1), N1, N2, CC, TLI, DAG))
@@ -8064,10 +8298,15 @@ static SDValue tryToFoldExtendOfConstant(SDNode *N, const TargetLowering &TLI,
   unsigned NumElts = VT.getVectorNumElements();
   SDLoc DL(N);
 
-  for (unsigned i=0; i != NumElts; ++i) {
-    SDValue Op = N0->getOperand(i);
-    if (Op->isUndef()) {
-      Elts.push_back(DAG.getUNDEF(SVT));
+  // For zero-extensions, UNDEF elements still guarantee to have the upper
+  // bits set to zero.
+  bool IsZext =
+      Opcode == ISD::ZERO_EXTEND || Opcode == ISD::ZERO_EXTEND_VECTOR_INREG;
+
+  for (unsigned i = 0; i != NumElts; ++i) {
+    SDValue Op = N0.getOperand(i);
+    if (Op.isUndef()) {
+      Elts.push_back(IsZext ? DAG.getConstant(0, DL, SVT) : DAG.getUNDEF(SVT));
       continue;
     }
 
@@ -8271,6 +8510,9 @@ SDValue DAGCombiner::CombineExtLoad(SDNode *N) {
 SDValue DAGCombiner::CombineZExtLogicopShiftLoad(SDNode *N) {
   assert(N->getOpcode() == ISD::ZERO_EXTEND);
   EVT VT = N->getValueType(0);
+  EVT OrigVT = N->getOperand(0).getValueType();
+  if (TLI.isZExtFree(OrigVT, VT))
+    return SDValue();
 
   // and/or/xor
   SDValue N0 = N->getOperand(0);
@@ -8334,6 +8576,10 @@ SDValue DAGCombiner::CombineZExtLogicopShiftLoad(SDNode *N) {
                                 Load->getValueType(0), ExtLoad);
     CombineTo(Load, Trunc, ExtLoad.getValue(1));
   }
+
+  // N0 is dead at this point.
+  recursivelyDeleteUnusedNodes(N0.getNode());
+
   return SDValue(N,0); // Return N so it doesn't get rechecked!
 }
 
@@ -8393,19 +8639,21 @@ static SDValue tryToFoldExtOfExtload(SelectionDAG &DAG, DAGCombiner &Combiner,
                                                    : ISD::isZEXTLoad(N0Node);
   if ((!isAExtLoad && !ISD::isEXTLoad(N0Node)) ||
       !ISD::isUNINDEXEDLoad(N0Node) || !N0.hasOneUse())
-    return {};
+    return SDValue();
 
   LoadSDNode *LN0 = cast<LoadSDNode>(N0);
   EVT MemVT = LN0->getMemoryVT();
   if ((LegalOperations || LN0->isVolatile() || VT.isVector()) &&
       !TLI.isLoadExtLegal(ExtLoadType, VT, MemVT))
-    return {};
+    return SDValue();
 
   SDValue ExtLoad =
       DAG.getExtLoad(ExtLoadType, SDLoc(LN0), VT, LN0->getChain(),
                      LN0->getBasePtr(), MemVT, LN0->getMemOperand());
   Combiner.CombineTo(N, ExtLoad);
   DAG.ReplaceAllUsesOfValueWith(SDValue(LN0, 1), ExtLoad.getValue(1));
+  if (LN0->use_empty())
+    Combiner.recursivelyDeleteUnusedNodes(LN0);
   return SDValue(N, 0); // Return N so it doesn't get rechecked!
 }
 
@@ -8443,6 +8691,7 @@ static SDValue tryToFoldExtOfLoad(SelectionDAG &DAG, DAGCombiner &Combiner,
   Combiner.CombineTo(N, ExtLoad);
   if (NoReplaceTrunc) {
     DAG.ReplaceAllUsesOfValueWith(SDValue(LN0, 1), ExtLoad.getValue(1));
+    Combiner.recursivelyDeleteUnusedNodes(LN0);
   } else {
     SDValue Trunc =
         DAG.getNode(ISD::TRUNCATE, SDLoc(N0), N0.getValueType(), ExtLoad);
@@ -8699,7 +8948,7 @@ static bool isTruncateOf(SelectionDAG &DAG, SDValue N, SDValue &Op,
                          KnownBits &Known) {
   if (N->getOpcode() == ISD::TRUNCATE) {
     Op = N->getOperand(0);
-    DAG.computeKnownBits(Op, Known);
+    Known = DAG.computeKnownBits(Op);
     return true;
   }
 
@@ -8719,7 +8968,7 @@ static bool isTruncateOf(SelectionDAG &DAG, SDValue N, SDValue &Op,
   else
     return false;
 
-  DAG.computeKnownBits(Op, Known);
+  Known = DAG.computeKnownBits(Op);
 
   return (Known.Zero | 1).isAllOnesValue();
 }
@@ -9046,6 +9295,7 @@ SDValue DAGCombiner::visitANY_EXTEND(SDNode *N) {
       CombineTo(N, ExtLoad);
       if (NoReplaceTrunc) {
         DAG.ReplaceAllUsesOfValueWith(SDValue(LN0, 1), ExtLoad.getValue(1));
+        recursivelyDeleteUnusedNodes(LN0);
       } else {
         SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SDLoc(N0),
                                     N0.getValueType(), ExtLoad);
@@ -9069,6 +9319,7 @@ SDValue DAGCombiner::visitANY_EXTEND(SDNode *N) {
                                        MemVT, LN0->getMemOperand());
       CombineTo(N, ExtLoad);
       DAG.ReplaceAllUsesOfValueWith(SDValue(LN0, 1), ExtLoad.getValue(1));
+      recursivelyDeleteUnusedNodes(LN0);
       return SDValue(N, 0);   // Return N so it doesn't get rechecked!
     }
   }
@@ -9369,18 +9620,15 @@ SDValue DAGCombiner::ReduceLoadWidth(SDNode *N) {
     if (DAG.getDataLayout().isBigEndian())
       ShAmt = AdjustBigEndianShift(ShAmt);
 
-    // We're using a shifted mask, so the load now has an offset. This means we
-    // now need to shift right the mask to match the new load and then shift
-    // right the result of the AND.
-    const APInt &Mask = cast<ConstantSDNode>(N->getOperand(1))->getAPIntValue();
-    APInt ShiftedMask = Mask.lshr(ShAmt);
-    DAG.UpdateNodeOperands(N, Result, DAG.getConstant(ShiftedMask, DL, VT));
+    // We're using a shifted mask, so the load now has an offset. This means
+    // that data has been loaded into the lower bytes than it would have been
+    // before, so we need to shl the loaded data into the correct position in the
+    // register.
     SDValue ShiftC = DAG.getConstant(ShAmt, DL, VT);
-    SDValue Shifted = DAG.getNode(ISD::SHL, DL, VT, SDValue(N, 0),
-                                  ShiftC);
-    DAG.ReplaceAllUsesOfValueWith(SDValue(N, 0), Shifted);
-    DAG.UpdateNodeOperands(Shifted.getNode(), SDValue(N, 0), ShiftC);
+    Result = DAG.getNode(ISD::SHL, DL, VT, Result, ShiftC);
+    DAG.ReplaceAllUsesOfValueWith(SDValue(N, 0), Result);
   }
+
   // Return the new loaded value.
   return Result;
 }
@@ -9412,12 +9660,15 @@ SDValue DAGCombiner::visitSIGN_EXTEND_INREG(SDNode *N) {
 
   // fold (sext_in_reg (sext x)) -> (sext x)
   // fold (sext_in_reg (aext x)) -> (sext x)
-  // if x is small enough.
+  // if x is small enough or if we know that x has more than 1 sign bit and the
+  // sign_extend_inreg is extending from one of them.
   if (N0.getOpcode() == ISD::SIGN_EXTEND || N0.getOpcode() == ISD::ANY_EXTEND) {
     SDValue N00 = N0.getOperand(0);
-    if (N00.getScalarValueSizeInBits() <= EVTBits &&
+    unsigned N00Bits = N00.getScalarValueSizeInBits();
+    if ((N00Bits <= EVTBits ||
+         (N00Bits - DAG.ComputeNumSignBits(N00)) < EVTBits) &&
         (!LegalOperations || TLI.isOperationLegal(ISD::SIGN_EXTEND, VT)))
-      return DAG.getNode(ISD::SIGN_EXTEND, SDLoc(N), VT, N00, N1);
+      return DAG.getNode(ISD::SIGN_EXTEND, SDLoc(N), VT, N00);
   }
 
   // fold (sext_in_reg (*_extend_vector_inreg x)) -> (sext_vector_inreg x)
@@ -9640,8 +9891,7 @@ SDValue DAGCombiner::visitTRUNCATE(SDNode *N) {
       (!LegalOperations || TLI.isOperationLegalOrCustom(ISD::SHL, VT)) &&
       TLI.isTypeDesirableForOp(ISD::SHL, VT)) {
     SDValue Amt = N0.getOperand(1);
-    KnownBits Known;
-    DAG.computeKnownBits(Amt, Known);
+    KnownBits Known = DAG.computeKnownBits(Amt);
     unsigned Size = VT.getScalarSizeInBits();
     if (Known.getBitWidth() - Known.countMinLeadingZeros() <= Log2_32(Size)) {
       SDLoc SL(N);
@@ -9823,8 +10073,7 @@ SDValue DAGCombiner::visitTRUNCATE(SDNode *N) {
   // because targets may prefer a wider type during later combines and invert
   // this transform.
   switch (N0.getOpcode()) {
-  // TODO: Add case for ADD - that will likely require a change in logic here
-  // or target-specific changes to avoid regressions.
+  case ISD::ADD:
   case ISD::SUB:
   case ISD::MUL:
   case ISD::AND:
@@ -10156,7 +10405,7 @@ SDValue DAGCombiner::visitBITCAST(SDNode *N) {
   // float vectors bitcast to integer vectors) into shuffles.
   // bitcast(shuffle(bitcast(s0),bitcast(s1))) -> shuffle(s0,s1)
   if (Level < AfterLegalizeDAG && TLI.isTypeLegal(VT) && VT.isVector() &&
-      N0->getOpcode() == ISD::VECTOR_SHUFFLE &&
+      N0->getOpcode() == ISD::VECTOR_SHUFFLE && N0.hasOneUse() &&
       VT.getVectorNumElements() >= N0.getValueType().getVectorNumElements() &&
       !(VT.getVectorNumElements() % N0.getValueType().getVectorNumElements())) {
     ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(N0);
@@ -11724,18 +11973,24 @@ SDValue DAGCombiner::visitFPOW(SDNode *N) {
     return DAG.getNode(ISD::FCBRT, SDLoc(N), VT, N->getOperand(0), Flags);
   }
 
-  // Try to convert x ** (1/4) into square roots.
+  // Try to convert x ** (1/4) and x ** (3/4) into square roots.
   // x ** (1/2) is canonicalized to sqrt, so we do not bother with that case.
   // TODO: This could be extended (using a target hook) to handle smaller
   // power-of-2 fractional exponents.
-  if (ExponentC->getValueAPF().isExactlyValue(0.25)) {
+  bool ExponentIs025 = ExponentC->getValueAPF().isExactlyValue(0.25);
+  bool ExponentIs075 = ExponentC->getValueAPF().isExactlyValue(0.75);
+  if (ExponentIs025 || ExponentIs075) {
     // pow(-0.0, 0.25) = +0.0; sqrt(sqrt(-0.0)) = -0.0.
     // pow(-inf, 0.25) = +inf; sqrt(sqrt(-inf)) =  NaN.
+    // pow(-0.0, 0.75) = +0.0; sqrt(-0.0) * sqrt(sqrt(-0.0)) = +0.0.
+    // pow(-inf, 0.75) = +inf; sqrt(-inf) * sqrt(sqrt(-inf)) =  NaN.
     // For regular numbers, rounding may cause the results to differ.
     // Therefore, we require { nsz ninf afn } for this transform.
     // TODO: We could select out the special cases if we don't have nsz/ninf.
     SDNodeFlags Flags = N->getFlags();
-    if (!Flags.hasNoSignedZeros() || !Flags.hasNoInfs() ||
+
+    // We only need no signed zeros for the 0.25 case.
+    if ((!Flags.hasNoSignedZeros() && ExponentIs025) || !Flags.hasNoInfs() ||
         !Flags.hasApproximateFuncs())
       return SDValue();
 
@@ -11751,7 +12006,11 @@ SDValue DAGCombiner::visitFPOW(SDNode *N) {
     // pow(X, 0.25) --> sqrt(sqrt(X))
     SDLoc DL(N);
     SDValue Sqrt = DAG.getNode(ISD::FSQRT, DL, VT, N->getOperand(0), Flags);
-    return DAG.getNode(ISD::FSQRT, DL, VT, Sqrt, Flags);
+    SDValue SqrtSqrt = DAG.getNode(ISD::FSQRT, DL, VT, Sqrt, Flags);
+    if (ExponentIs025)
+      return SqrtSqrt;
+    // pow(X, 0.75) --> sqrt(X) * sqrt(sqrt(X))
+    return DAG.getNode(ISD::FMUL, DL, VT, Sqrt, SqrtSqrt, Flags);
   }
 
   return SDValue();
@@ -12539,7 +12798,13 @@ bool DAGCombiner::CombineToPreIndexedLoadStore(SDNode *N) {
   // Check #2.
   if (!isLoad) {
     SDValue Val = cast<StoreSDNode>(N)->getValue();
-    if (Val == BasePtr || BasePtr.getNode()->isPredecessorOf(Val.getNode()))
+
+    // Would require a copy.
+    if (Val == BasePtr)
+      return false;
+
+    // Would create a cycle.
+    if (Val == Ptr || Ptr->isPredecessorOf(Val.getNode()))
       return false;
   }
 
@@ -13076,7 +13341,7 @@ SDValue DAGCombiner::visitLOAD(SDNode *N) {
 
   if (LD->isUnindexed()) {
     // Walk up chain skipping non-aliasing memory nodes.
-    SDValue BetterChain = FindBetterChain(N, Chain);
+    SDValue BetterChain = FindBetterChain(LD, Chain);
 
     // If there is a better chain.
     if (Chain != BetterChain) {
@@ -15502,7 +15767,9 @@ SDValue DAGCombiner::scalarizeExtractedVectorLoad(SDNode *EVE, EVT InVecVT,
     Offset = DAG.getNode(
         ISD::MUL, DL, PtrType, Offset,
         DAG.getConstant(VecEltVT.getStoreSize(), DL, PtrType));
-    MPI = OriginalLoad->getPointerInfo();
+    // Discard the pointer info except the address space because the memory
+    // operand can't represent this new access since the offset is variable.
+    MPI = MachinePointerInfo(OriginalLoad->getPointerInfo().getAddrSpace());
   }
   NewPtr = DAG.getNode(ISD::ADD, DL, PtrType, NewPtr, Offset);
 
@@ -15550,6 +15817,40 @@ SDValue DAGCombiner::scalarizeExtractedVectorLoad(SDNode *EVE, EVT InVecVT,
   return SDValue(EVE, 0);
 }
 
+/// Transform a vector binary operation into a scalar binary operation by moving
+/// the math/logic after an extract element of a vector.
+static SDValue scalarizeExtractedBinop(SDNode *ExtElt, SelectionDAG &DAG,
+                                       bool LegalOperations) {
+  SDValue Vec = ExtElt->getOperand(0);
+  SDValue Index = ExtElt->getOperand(1);
+  auto *IndexC = dyn_cast<ConstantSDNode>(Index);
+  if (!IndexC || !ISD::isBinaryOp(Vec.getNode()) || !Vec.hasOneUse())
+    return SDValue();
+
+  // Targets may want to avoid this to prevent an expensive register transfer.
+  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
+  if (!TLI.shouldScalarizeBinop(Vec))
+    return SDValue();
+
+  // Extracting an element of a vector constant is constant-folded, so this
+  // transform is just replacing a vector op with a scalar op while moving the
+  // extract.
+  SDValue Op0 = Vec.getOperand(0);
+  SDValue Op1 = Vec.getOperand(1);
+  if (isAnyConstantBuildVector(Op0, true) ||
+      isAnyConstantBuildVector(Op1, true)) {
+    // extractelt (binop X, C), IndexC --> binop (extractelt X, IndexC), C'
+    // extractelt (binop C, X), IndexC --> binop C', (extractelt X, IndexC)
+    SDLoc DL(ExtElt);
+    EVT VT = ExtElt->getValueType(0);
+    SDValue Ext0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Op0, Index);
+    SDValue Ext1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Op1, Index);
+    return DAG.getNode(Vec.getOpcode(), DL, VT, Ext0, Ext1);
+  }
+
+  return SDValue();
+}
+
 SDValue DAGCombiner::visitEXTRACT_VECTOR_ELT(SDNode *N) {
   SDValue VecOp = N->getOperand(0);
   SDValue Index = N->getOperand(1);
@@ -15640,6 +15941,9 @@ SDValue DAGCombiner::visitEXTRACT_VECTOR_ELT(SDNode *N) {
     }
   }
 
+  if (SDValue BO = scalarizeExtractedBinop(N, DAG, LegalOperations))
+    return BO;
+
   // Transform: (EXTRACT_VECTOR_ELT( VECTOR_SHUFFLE )) -> EXTRACT_VECTOR_ELT.
   // We only perform this optimization before the op legalization phase because
   // we may introduce new vector instructions which are not backed by TD
@@ -15732,14 +16036,13 @@ SDValue DAGCombiner::visitEXTRACT_VECTOR_ELT(SDNode *N) {
     ExtVT = BCVT.getVectorElementType();
   }
 
-  // (vextract (vN[if]M load $addr), i) -> ([if]M load $addr + i * size)
+  // extract (vector load $addr), i --> load $addr + i * size
   if (!LegalOperations && !IndexC && VecOp.hasOneUse() &&
       ISD::isNormalLoad(VecOp.getNode()) &&
-      !N->getOperand(1)->hasPredecessor(VecOp.getNode())) {
-    SDValue Index = N->getOperand(1);
-    if (auto *OrigLoad = dyn_cast<LoadSDNode>(VecOp))
-      if (!OrigLoad->isVolatile())
-        return scalarizeExtractedVectorLoad(N, VecVT, Index, OrigLoad);
+      !Index->hasPredecessor(VecOp.getNode())) {
+    auto *VecLoad = dyn_cast<LoadSDNode>(VecOp);
+    if (VecLoad && !VecLoad->isVolatile())
+      return scalarizeExtractedVectorLoad(N, VecVT, Index, VecLoad);
   }
 
   // Perform only after legalization to ensure build_vector / vector_shuffle
@@ -16041,6 +16344,84 @@ SDValue DAGCombiner::createBuildVecShuffle(const SDLoc &DL, SDNode *N,
   return Shuffle;
 }
 
+static SDValue reduceBuildVecToShuffleWithZero(SDNode *BV, SelectionDAG &DAG) {
+  assert(BV->getOpcode() == ISD::BUILD_VECTOR && "Expected build vector");
+
+  // First, determine where the build vector is not undef.
+  // TODO: We could extend this to handle zero elements as well as undefs.
+  int NumBVOps = BV->getNumOperands();
+  int ZextElt = -1;
+  for (int i = 0; i != NumBVOps; ++i) {
+    SDValue Op = BV->getOperand(i);
+    if (Op.isUndef())
+      continue;
+    if (ZextElt == -1)
+      ZextElt = i;
+    else
+      return SDValue();
+  }
+  // Bail out if there's no non-undef element.
+  if (ZextElt == -1)
+    return SDValue();
+
+  // The build vector contains some number of undef elements and exactly
+  // one other element. That other element must be a zero-extended scalar
+  // extracted from a vector at a constant index to turn this into a shuffle.
+  // Also, require that the build vector does not implicitly truncate/extend
+  // its elements.
+  // TODO: This could be enhanced to allow ANY_EXTEND as well as ZERO_EXTEND.
+  EVT VT = BV->getValueType(0);
+  SDValue Zext = BV->getOperand(ZextElt);
+  if (Zext.getOpcode() != ISD::ZERO_EXTEND || !Zext.hasOneUse() ||
+      Zext.getOperand(0).getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
+      !isa<ConstantSDNode>(Zext.getOperand(0).getOperand(1)) ||
+      Zext.getValueSizeInBits() != VT.getScalarSizeInBits())
+    return SDValue();
+
+  // The zero-extend must be a multiple of the source size, and we must be
+  // building a vector of the same size as the source of the extract element.
+  SDValue Extract = Zext.getOperand(0);
+  unsigned DestSize = Zext.getValueSizeInBits();
+  unsigned SrcSize = Extract.getValueSizeInBits();
+  if (DestSize % SrcSize != 0 ||
+      Extract.getOperand(0).getValueSizeInBits() != VT.getSizeInBits())
+    return SDValue();
+
+  // Create a shuffle mask that will combine the extracted element with zeros
+  // and undefs.
+  int ZextRatio = DestSize / SrcSize;
+  int NumMaskElts = NumBVOps * ZextRatio;
+  SmallVector<int, 32> ShufMask(NumMaskElts, -1);
+  for (int i = 0; i != NumMaskElts; ++i) {
+    if (i / ZextRatio == ZextElt) {
+      // The low bits of the (potentially translated) extracted element map to
+      // the source vector. The high bits map to zero. We will use a zero vector
+      // as the 2nd source operand of the shuffle, so use the 1st element of
+      // that vector (mask value is number-of-elements) for the high bits.
+      if (i % ZextRatio == 0)
+        ShufMask[i] = Extract.getConstantOperandVal(1);
+      else
+        ShufMask[i] = NumMaskElts;
+    }
+
+    // Undef elements of the build vector remain undef because we initialize
+    // the shuffle mask with -1.
+  }
+
+  // Turn this into a shuffle with zero if that's legal.
+  EVT VecVT = Extract.getOperand(0).getValueType();
+  if (!DAG.getTargetLoweringInfo().isShuffleMaskLegal(ShufMask, VecVT))
+    return SDValue();
+
+  // buildvec undef, ..., (zext (extractelt V, IndexC)), undef... -->
+  // bitcast (shuffle V, ZeroVec, VectorMask)
+  SDLoc DL(BV);
+  SDValue ZeroVec = DAG.getConstant(0, DL, VecVT);
+  SDValue Shuf = DAG.getVectorShuffle(VecVT, DL, Extract.getOperand(0), ZeroVec,
+                                      ShufMask);
+  return DAG.getBitcast(VT, Shuf);
+}
+
 // Check to see if this is a BUILD_VECTOR of a bunch of EXTRACT_VECTOR_ELT
 // operations. If the types of the vectors we're extracting from allow it,
 // turn this into a vector_shuffle node.
@@ -16052,6 +16433,9 @@ SDValue DAGCombiner::reduceBuildVecToShuffle(SDNode *N) {
   if (!isTypeLegal(VT))
     return SDValue();
 
+  if (SDValue V = reduceBuildVecToShuffleWithZero(N, DAG))
+    return V;
+
   // May only combine to shuffle after legalize if shuffle is legal.
   if (LegalOperations && !TLI.isOperationLegal(ISD::VECTOR_SHUFFLE, VT))
     return SDValue();
@@ -16091,7 +16475,7 @@ SDValue DAGCombiner::reduceBuildVecToShuffle(SDNode *N) {
       return SDValue();
     SDValue ExtractedFromVec = Op.getOperand(0);
 
-    APInt ExtractIdx = cast<ConstantSDNode>(Op.getOperand(1))->getAPIntValue();
+    const APInt &ExtractIdx = Op.getConstantOperandAPInt(1);
     if (ExtractIdx.uge(ExtractedFromVec.getValueType().getVectorNumElements()))
       return SDValue();
 
@@ -16839,15 +17223,28 @@ SDValue DAGCombiner::visitEXTRACT_SUBVECTOR(SDNode* N) {
     if (SDValue NarrowLoad = narrowExtractedVectorLoad(N, DAG))
       return NarrowLoad;
 
+  // Combine an extract of an extract into a single extract_subvector.
+  // ext (ext X, C), 0 --> ext X, C
+  if (isNullConstant(N->getOperand(1)) &&
+      V.getOpcode() == ISD::EXTRACT_SUBVECTOR && V.hasOneUse() &&
+      isa<ConstantSDNode>(V.getOperand(1))) {
+    if (TLI.isExtractSubvectorCheap(NVT, V.getOperand(0).getValueType(),
+                                    V.getConstantOperandVal(1)) &&
+        TLI.isOperationLegalOrCustom(ISD::EXTRACT_SUBVECTOR, NVT)) {
+      return DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N), NVT, V.getOperand(0),
+                         V.getOperand(1));
+    }
+  }
+
   // Combine:
   //    (extract_subvec (concat V1, V2, ...), i)
   // Into:
   //    Vi if possible
   // Only operand 0 is checked as 'concat' assumes all inputs of the same
   // type.
-  if (V->getOpcode() == ISD::CONCAT_VECTORS &&
+  if (V.getOpcode() == ISD::CONCAT_VECTORS &&
       isa<ConstantSDNode>(N->getOperand(1)) &&
-      V->getOperand(0).getValueType() == NVT) {
+      V.getOperand(0).getValueType() == NVT) {
     unsigned Idx = N->getConstantOperandVal(1);
     unsigned NumElems = NVT.getVectorNumElements();
     assert((Idx % NumElems) == 0 &&
@@ -16858,9 +17255,9 @@ SDValue DAGCombiner::visitEXTRACT_SUBVECTOR(SDNode* N) {
   V = peekThroughBitcasts(V);
 
   // If the input is a build vector. Try to make a smaller build vector.
-  if (V->getOpcode() == ISD::BUILD_VECTOR) {
+  if (V.getOpcode() == ISD::BUILD_VECTOR) {
     if (auto *Idx = dyn_cast<ConstantSDNode>(N->getOperand(1))) {
-      EVT InVT = V->getValueType(0);
+      EVT InVT = V.getValueType();
       unsigned ExtractSize = NVT.getSizeInBits();
       unsigned EltSize = InVT.getScalarSizeInBits();
       // Only do this if we won't split any elements.
@@ -16893,16 +17290,16 @@ SDValue DAGCombiner::visitEXTRACT_SUBVECTOR(SDNode* N) {
     }
   }
 
-  if (V->getOpcode() == ISD::INSERT_SUBVECTOR) {
+  if (V.getOpcode() == ISD::INSERT_SUBVECTOR) {
     // Handle only simple case where vector being inserted and vector
     // being extracted are of same size.
-    EVT SmallVT = V->getOperand(1).getValueType();
+    EVT SmallVT = V.getOperand(1).getValueType();
     if (!NVT.bitsEq(SmallVT))
       return SDValue();
 
     // Only handle cases where both indexes are constants.
-    ConstantSDNode *ExtIdx = dyn_cast<ConstantSDNode>(N->getOperand(1));
-    ConstantSDNode *InsIdx = dyn_cast<ConstantSDNode>(V->getOperand(2));
+    auto *ExtIdx = dyn_cast<ConstantSDNode>(N->getOperand(1));
+    auto *InsIdx = dyn_cast<ConstantSDNode>(V.getOperand(2));
 
     if (InsIdx && ExtIdx) {
       // Combine:
@@ -16912,11 +17309,11 @@ SDValue DAGCombiner::visitEXTRACT_SUBVECTOR(SDNode* N) {
       //    otherwise => (extract_subvec V1, ExtIdx)
       if (InsIdx->getZExtValue() * SmallVT.getScalarSizeInBits() ==
           ExtIdx->getZExtValue() * NVT.getScalarSizeInBits())
-        return DAG.getBitcast(NVT, V->getOperand(1));
+        return DAG.getBitcast(NVT, V.getOperand(1));
       return DAG.getNode(
           ISD::EXTRACT_SUBVECTOR, SDLoc(N), NVT,
-          DAG.getBitcast(N->getOperand(0).getValueType(), V->getOperand(0)),
-          N->getOperand(1));
+          DAG.getBitcast(N->getOperand(0).getValueType(), V.getOperand(0)),
+                         N->getOperand(1));
     }
   }
 
@@ -17026,8 +17423,8 @@ static SDValue combineShuffleOfScalars(ShuffleVectorSDNode *SVN,
     if (!N1->hasOneUse())
       return SDValue();
 
-    bool N0AnyConst = isAnyConstantBuildVector(N0.getNode());
-    bool N1AnyConst = isAnyConstantBuildVector(N1.getNode());
+    bool N0AnyConst = isAnyConstantBuildVector(N0);
+    bool N1AnyConst = isAnyConstantBuildVector(N1);
     if (N0AnyConst && !N1AnyConst && !ISD::isBuildVectorAllZeros(N0.getNode()))
       return SDValue();
     if (!N0AnyConst && N1AnyConst && !ISD::isBuildVectorAllZeros(N1.getNode()))
@@ -17093,8 +17490,7 @@ static SDValue combineShuffleOfScalars(ShuffleVectorSDNode *SVN,
 static SDValue combineShuffleToVectorExtend(ShuffleVectorSDNode *SVN,
                                             SelectionDAG &DAG,
                                             const TargetLowering &TLI,
-                                            bool LegalOperations,
-                                            bool LegalTypes) {
+                                            bool LegalOperations) {
   EVT VT = SVN->getValueType(0);
   bool IsBigEndian = DAG.getDataLayout().isBigEndian();
 
@@ -17130,7 +17526,9 @@ static SDValue combineShuffleToVectorExtend(ShuffleVectorSDNode *SVN,
 
     EVT OutSVT = EVT::getIntegerVT(*DAG.getContext(), EltSizeInBits * Scale);
     EVT OutVT = EVT::getVectorVT(*DAG.getContext(), OutSVT, NumElts / Scale);
-    if (!LegalTypes || TLI.isTypeLegal(OutVT))
+    // Never create an illegal type. Only create unsupported operations if we
+    // are pre-legalization.
+    if (TLI.isTypeLegal(OutVT))
       if (!LegalOperations ||
           TLI.isOperationLegalOrCustom(ISD::ANY_EXTEND_VECTOR_INREG, OutVT))
         return DAG.getBitcast(VT,
@@ -17440,7 +17838,7 @@ SDValue DAGCombiner::visitVECTOR_SHUFFLE(SDNode *N) {
     return SDValue(N, 0);
 
   // Match shuffles that can be converted to any_vector_extend_in_reg.
-  if (SDValue V = combineShuffleToVectorExtend(SVN, DAG, TLI, LegalOperations, LegalTypes))
+  if (SDValue V = combineShuffleToVectorExtend(SVN, DAG, TLI, LegalOperations))
     return V;
 
   // Combine "truncate_vector_in_reg" style shuffles.
@@ -17478,7 +17876,7 @@ SDValue DAGCombiner::visitVECTOR_SHUFFLE(SDNode *N) {
           NewMask.push_back(M < 0 ? -1 : Scale * M + s);
       return NewMask;
     };
-    
+
     SDValue BC0 = peekThroughOneUseBitcasts(N0);
     if (BC0.getOpcode() == ISD::VECTOR_SHUFFLE && BC0.hasOneUse()) {
       EVT SVT = VT.getScalarType();
@@ -17769,6 +18167,14 @@ SDValue DAGCombiner::visitINSERT_SUBVECTOR(SDNode *N) {
     return DAG.getNode(ISD::INSERT_SUBVECTOR, SDLoc(N), VT, N0.getOperand(0),
                        N1, N2);
 
+  // Eliminate an intermediate insert into an undef vector:
+  // insert_subvector undef, (insert_subvector undef, X, 0), N2 -->
+  // insert_subvector undef, X, N2
+  if (N0.isUndef() && N1.getOpcode() == ISD::INSERT_SUBVECTOR &&
+      N1.getOperand(0).isUndef() && isNullConstant(N1.getOperand(2)))
+    return DAG.getNode(ISD::INSERT_SUBVECTOR, SDLoc(N), VT, N0,
+                       N1.getOperand(1), N2);
+
   if (!isa<ConstantSDNode>(N2))
     return SDValue();
 
@@ -17935,6 +18341,7 @@ SDValue DAGCombiner::SimplifyVBinOp(SDNode *N) {
   SDValue LHS = N->getOperand(0);
   SDValue RHS = N->getOperand(1);
   SDValue Ops[] = {LHS, RHS};
+  EVT VT = N->getValueType(0);
 
   // See if we can constant fold the vector operation.
   if (SDValue Fold = DAG.FoldConstantVectorArithmetic(
@@ -17952,7 +18359,6 @@ SDValue DAGCombiner::SimplifyVBinOp(SDNode *N) {
     ShuffleVectorSDNode *SVN1 = cast<ShuffleVectorSDNode>(RHS);
 
     if (SVN0->getMask().equals(SVN1->getMask())) {
-      EVT VT = N->getValueType(0);
       SDValue UndefVector = LHS.getOperand(1);
       SDValue NewBinOp = DAG.getNode(N->getOpcode(), SDLoc(N), VT,
                                      LHS.getOperand(0), RHS.getOperand(0),
@@ -17963,6 +18369,29 @@ SDValue DAGCombiner::SimplifyVBinOp(SDNode *N) {
     }
   }
 
+  // The following pattern is likely to emerge with vector reduction ops. Moving
+  // the binary operation ahead of insertion may allow using a narrower vector
+  // instruction that has better performance than the wide version of the op:
+  // VBinOp (ins undef, X, Z), (ins undef, Y, Z) --> ins VecC, (VBinOp X, Y), Z
+  if (LHS.getOpcode() == ISD::INSERT_SUBVECTOR && LHS.getOperand(0).isUndef() &&
+      RHS.getOpcode() == ISD::INSERT_SUBVECTOR && RHS.getOperand(0).isUndef() &&
+      LHS.getOperand(2) == RHS.getOperand(2) &&
+      (LHS.hasOneUse() || RHS.hasOneUse())) {
+    SDValue X = LHS.getOperand(1);
+    SDValue Y = RHS.getOperand(1);
+    SDValue Z = LHS.getOperand(2);
+    EVT NarrowVT = X.getValueType();
+    if (NarrowVT == Y.getValueType() &&
+        TLI.isOperationLegalOrCustomOrPromote(N->getOpcode(), NarrowVT)) {
+      // (binop undef, undef) may not return undef, so compute that result.
+      SDLoc DL(N);
+      SDValue VecC = DAG.getNode(N->getOpcode(), DL, VT, DAG.getUNDEF(VT),
+                                 DAG.getUNDEF(VT));
+      SDValue NarrowBO = DAG.getNode(N->getOpcode(), DL, NarrowVT, X, Y);
+      return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, VecC, NarrowBO, Z);
+    }
+  }
+
   return SDValue();
 }
 
@@ -18892,13 +19321,13 @@ bool DAGCombiner::isAlias(LSBaseSDNode *Op0, LSBaseSDNode *Op1) const {
 
 /// Walk up chain skipping non-aliasing memory nodes,
 /// looking for aliasing nodes and adding them to the Aliases vector.
-void DAGCombiner::GatherAllAliases(SDNode *N, SDValue OriginalChain,
+void DAGCombiner::GatherAllAliases(LSBaseSDNode *N, SDValue OriginalChain,
                                    SmallVectorImpl<SDValue> &Aliases) {
   SmallVector<SDValue, 8> Chains;     // List of chains to visit.
   SmallPtrSet<SDNode *, 16> Visited;  // Visited node set.
 
   // Get alias information for node.
-  bool IsLoad = isa<LoadSDNode>(N) && !cast<LSBaseSDNode>(N)->isVolatile();
+  bool IsLoad = isa<LoadSDNode>(N) && !N->isVolatile();
 
   // Starting off.
   Chains.push_back(OriginalChain);
@@ -18939,7 +19368,7 @@ void DAGCombiner::GatherAllAliases(SDNode *N, SDValue OriginalChain,
 
       // If chain is alias then stop here.
       if (!(IsLoad && IsOpLoad) &&
-          isAlias(cast<LSBaseSDNode>(N), cast<LSBaseSDNode>(Chain.getNode()))) {
+          isAlias(N, cast<LSBaseSDNode>(Chain.getNode()))) {
         Aliases.push_back(Chain);
       } else {
         // Look further up the chain.
@@ -18979,7 +19408,7 @@ void DAGCombiner::GatherAllAliases(SDNode *N, SDValue OriginalChain,
 
 /// Walk up chain skipping non-aliasing memory nodes, looking for a better chain
 /// (aliasing node.)
-SDValue DAGCombiner::FindBetterChain(SDNode *N, SDValue OldChain) {
+SDValue DAGCombiner::FindBetterChain(LSBaseSDNode *N, SDValue OldChain) {
   if (OptLevel == CodeGenOpt::None)
     return OldChain;
 
@@ -19001,10 +19430,12 @@ SDValue DAGCombiner::FindBetterChain(SDNode *N, SDValue OldChain) {
   return DAG.getNode(ISD::TokenFactor, SDLoc(N), MVT::Other, Aliases);
 }
 
+namespace {
 // TODO: Replace with with std::monostate when we move to C++17.
 struct UnitT { } Unit;
 bool operator==(const UnitT &, const UnitT &) { return true; }
 bool operator!=(const UnitT &, const UnitT &) { return false; }
+} // namespace
 
 // This function tries to collect a bunch of potentially interesting
 // nodes to improve the chains of, all at once. This might seem
@@ -19115,7 +19546,7 @@ bool DAGCombiner::parallelizeChainedStores(StoreSDNode *St) {
   if (AddNewChain)
     TFOps.insert(TFOps.begin(), NewChain);
 
-  SDValue TF = DAG.getNode(ISD::TokenFactor, SDLoc(STChain), MVT::Other, TFOps);
+  SDValue TF = DAG.getTokenFactor(SDLoc(STChain), TFOps);
   CombineTo(St, TF);
 
   AddToWorklist(STChain);
diff --git a/lib/CodeGen/SelectionDAG/FastISel.cpp b/lib/CodeGen/SelectionDAG/FastISel.cpp
index a9a3c44ea0c9ca6d62af1f5f84331bf8ba343761..30294ae159538696b7d2b5150dc03e7a629701bf 100644
--- a/lib/CodeGen/SelectionDAG/FastISel.cpp
+++ b/lib/CodeGen/SelectionDAG/FastISel.cpp
@@ -1,9 +1,8 @@
 //===- FastISel.cpp - Implementation of the FastISel class ----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -1304,9 +1303,6 @@ bool FastISel::selectCall(const User *I) {
     return true;
   }
 
-  MachineModuleInfo &MMI = FuncInfo.MF->getMMI();
-  computeUsesVAFloatArgument(*Call, MMI);
-
   // Handle intrinsic function calls.
   if (const auto *II = dyn_cast<IntrinsicInst>(Call))
     return selectIntrinsicCall(II);
diff --git a/lib/CodeGen/SelectionDAG/FunctionLoweringInfo.cpp b/lib/CodeGen/SelectionDAG/FunctionLoweringInfo.cpp
index fba728625b071bbc19e24655ee0b43dd4a34511e..4a465e8e9ca220a379f5106595b5e7a809e4fc49 100644
--- a/lib/CodeGen/SelectionDAG/FunctionLoweringInfo.cpp
+++ b/lib/CodeGen/SelectionDAG/FunctionLoweringInfo.cpp
@@ -1,9 +1,8 @@
 //===-- FunctionLoweringInfo.cpp ------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -343,6 +342,7 @@ void FunctionLoweringInfo::clear() {
   LiveOutRegInfo.clear();
   VisitedBBs.clear();
   ArgDbgValues.clear();
+  DescribedArgs.clear();
   ByValArgFrameIndexMap.clear();
   RegFixups.clear();
   RegsWithFixups.clear();
diff --git a/lib/CodeGen/SelectionDAG/InstrEmitter.cpp b/lib/CodeGen/SelectionDAG/InstrEmitter.cpp
index 6a6114677cc2b5ab22fff63661417d10792aca6f..5cbc9e2a88b063390f4368e6d6e5fb2551d44239 100644
--- a/lib/CodeGen/SelectionDAG/InstrEmitter.cpp
+++ b/lib/CodeGen/SelectionDAG/InstrEmitter.cpp
@@ -1,9 +1,8 @@
 //==--- InstrEmitter.cpp - Emit MachineInstrs for the SelectionDAG class ---==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -1049,14 +1048,18 @@ EmitSpecialNode(SDNode *Node, bool IsClone, bool IsCloned,
     break;
   }
 
-  case ISD::INLINEASM: {
+  case ISD::INLINEASM:
+  case ISD::INLINEASM_BR: {
     unsigned NumOps = Node->getNumOperands();
     if (Node->getOperand(NumOps-1).getValueType() == MVT::Glue)
       --NumOps;  // Ignore the glue operand.
 
     // Create the inline asm machine instruction.
-    MachineInstrBuilder MIB = BuildMI(*MF, Node->getDebugLoc(),
-                                      TII->get(TargetOpcode::INLINEASM));
+    unsigned TgtOpc = Node->getOpcode() == ISD::INLINEASM_BR
+                          ? TargetOpcode::INLINEASM_BR
+                          : TargetOpcode::INLINEASM;
+    MachineInstrBuilder MIB =
+        BuildMI(*MF, Node->getDebugLoc(), TII->get(TgtOpc));
 
     // Add the asm string as an external symbol operand.
     SDValue AsmStrV = Node->getOperand(InlineAsm::Op_AsmString);
diff --git a/lib/CodeGen/SelectionDAG/InstrEmitter.h b/lib/CodeGen/SelectionDAG/InstrEmitter.h
index 701b6368690b9ed6fc8fc696b0a51b1b2ea1c58d..3188c2678f1af9b2b4c0151cb00d03da4f01fdbd 100644
--- a/lib/CodeGen/SelectionDAG/InstrEmitter.h
+++ b/lib/CodeGen/SelectionDAG/InstrEmitter.h
@@ -1,9 +1,8 @@
 //===- InstrEmitter.h - Emit MachineInstrs for the SelectionDAG -*- C++ -*--==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/SelectionDAG/LLVMBuild.txt b/lib/CodeGen/SelectionDAG/LLVMBuild.txt
index 81d2e000a2e80720972e26f3cbf1fba126d8713e..81503d5967c8f786f98dc30e3600647ae181a153 100644
--- a/lib/CodeGen/SelectionDAG/LLVMBuild.txt
+++ b/lib/CodeGen/SelectionDAG/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/CodeGen/SelectionDAG/LLVMBuild.txt -----------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
index 9d6a6939fbabafc0fcc1e3ed1b0eed83cab6610b..b6b96deed3266cd93d15807c21c69d00dc022ec7 100644
--- a/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
+++ b/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
@@ -1,9 +1,8 @@
 //===- LegalizeDAG.cpp - Implement SelectionDAG::Legalize -----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -1128,7 +1127,8 @@ void SelectionDAGLegalize::LegalizeOp(SDNode *Node) {
     Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
     break;
   }
-  case ISD::SMULFIX: {
+  case ISD::SMULFIX:
+  case ISD::UMULFIX: {
     unsigned Scale = Node->getConstantOperandVal(2);
     Action = TLI.getFixedPointOperationAction(Node->getOpcode(),
                                               Node->getValueType(0), Scale);
@@ -2645,6 +2645,10 @@ bool SelectionDAGLegalize::ExpandNode(SDNode *Node) {
   SDValue Tmp1, Tmp2, Tmp3, Tmp4;
   bool NeedInvert;
   switch (Node->getOpcode()) {
+  case ISD::ABS:
+    if (TLI.expandABS(Node, Tmp1, DAG))
+      Results.push_back(Tmp1);
+    break;
   case ISD::CTPOP:
     if (TLI.expandCTPOP(Node, Tmp1, DAG))
       Results.push_back(Tmp1);
@@ -3283,14 +3287,13 @@ bool SelectionDAGLegalize::ExpandNode(SDNode *Node) {
   case ISD::SADDSAT:
   case ISD::UADDSAT:
   case ISD::SSUBSAT:
-  case ISD::USUBSAT: {
-    Results.push_back(TLI.getExpandedSaturationAdditionSubtraction(Node, DAG));
+  case ISD::USUBSAT:
+    Results.push_back(TLI.expandAddSubSat(Node, DAG));
     break;
-  }
-  case ISD::SMULFIX: {
-    Results.push_back(TLI.getExpandedFixedPointMultiplication(Node, DAG));
+  case ISD::SMULFIX:
+  case ISD::UMULFIX:
+    Results.push_back(TLI.expandFixedPointMul(Node, DAG));
     break;
-  }
   case ISD::SADDO:
   case ISD::SSUBO: {
     SDValue LHS = Node->getOperand(0);
@@ -3420,7 +3423,7 @@ bool SelectionDAGLegalize::ExpandNode(SDNode *Node) {
       // being a legal type for the architecture and thus has to be split to
       // two arguments.
       SDValue Ret;
-      if(DAG.getDataLayout().isLittleEndian()) {
+      if (DAG.getDataLayout().isLittleEndian()) {
         // Halves of WideVT are packed into registers in different order
         // depending on platform endianness. This is usually handled by
         // the C calling convention, but we can't defer to it in
@@ -3433,8 +3436,14 @@ bool SelectionDAGLegalize::ExpandNode(SDNode *Node) {
       }
       assert(Ret.getOpcode() == ISD::MERGE_VALUES &&
              "Ret value is a collection of constituent nodes holding result.");
-      BottomHalf = Ret.getOperand(0);
-      TopHalf = Ret.getOperand(1);
+      if (DAG.getDataLayout().isLittleEndian()) {
+        // Same as above.
+        BottomHalf = Ret.getOperand(0);
+        TopHalf = Ret.getOperand(1);
+      } else {
+        BottomHalf = Ret.getOperand(1);
+        TopHalf = Ret.getOperand(0);
+      }
     }
 
     if (isSigned) {
@@ -3683,10 +3692,7 @@ bool SelectionDAGLegalize::ExpandNode(SDNode *Node) {
     (void)Legalized;
     assert(Legalized && "Can't legalize BR_CC with legal condition!");
 
-    // If we expanded the SETCC by inverting the condition code, then wrap
-    // the existing SETCC in a NOT to restore the intended condition.
-    if (NeedInvert)
-      Tmp4 = DAG.getNOT(dl, Tmp4, Tmp4->getValueType(0));
+    assert(!NeedInvert && "Don't know how to invert BR_CC!");
 
     // If we expanded the SETCC by swapping LHS and RHS, create a new BR_CC
     // node.
@@ -4533,6 +4539,24 @@ void SelectionDAGLegalize::PromoteNode(SDNode *Node) {
     Results.push_back(CvtVec);
     break;
   }
+  case ISD::ATOMIC_SWAP: {
+    AtomicSDNode *AM = cast<AtomicSDNode>(Node);
+    SDLoc SL(Node);
+    SDValue CastVal = DAG.getNode(ISD::BITCAST, SL, NVT, AM->getVal());
+    assert(NVT.getSizeInBits() == OVT.getSizeInBits() &&
+           "unexpected promotion type");
+    assert(AM->getMemoryVT().getSizeInBits() == NVT.getSizeInBits() &&
+           "unexpected atomic_swap with illegal type");
+
+    SDValue NewAtomic
+      = DAG.getAtomic(ISD::ATOMIC_SWAP, SL, NVT,
+                      DAG.getVTList(NVT, MVT::Other),
+                      { AM->getChain(), AM->getBasePtr(), CastVal },
+                      AM->getMemOperand());
+    Results.push_back(DAG.getNode(ISD::BITCAST, SL, OVT, NewAtomic));
+    Results.push_back(NewAtomic.getValue(1));
+    break;
+  }
   }
 
   // Replace the original node with the legalized result.
diff --git a/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp b/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp
index 866744c397b03ac2ea7e41059dd5fb7e6bf8c0dd..ae00d69ad0eba7a2f4d4b2939e0ffddf8614d542 100644
--- a/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp
+++ b/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp
@@ -1,9 +1,8 @@
 //===-------- LegalizeFloatTypes.cpp - Legalization of float types --------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -104,6 +103,7 @@ bool DAGTypeLegalizer::SoftenFloatResult(SDNode *N, unsigned ResNo) {
     case ISD::FSUB:        R = SoftenFloatRes_FSUB(N); break;
     case ISD::FTRUNC:      R = SoftenFloatRes_FTRUNC(N); break;
     case ISD::LOAD:        R = SoftenFloatRes_LOAD(N, ResNo); break;
+    case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break;
     case ISD::SELECT:      R = SoftenFloatRes_SELECT(N, ResNo); break;
     case ISD::SELECT_CC:   R = SoftenFloatRes_SELECT_CC(N, ResNo); break;
     case ISD::SINT_TO_FP:
@@ -440,6 +440,15 @@ SDValue DAGTypeLegalizer::SoftenFloatRes_FNEG(SDNode *N, unsigned ResNo) {
     return SDValue(N, ResNo);
   EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
   SDLoc dl(N);
+
+  EVT FloatVT = N->getValueType(ResNo);
+  if (FloatVT == MVT::f32 || FloatVT == MVT::f64 || FloatVT == MVT::f128) {
+    // Expand Y = FNEG(X) -> Y = X ^ sign mask
+    APInt SignMask = APInt::getSignMask(NVT.getSizeInBits());
+    return DAG.getNode(ISD::XOR, dl, NVT, GetSoftenedFloat(N->getOperand(0)),
+                       DAG.getConstant(SignMask, dl, NVT));
+  }
+
   // Expand Y = FNEG(X) -> Y = SUB -0.0, X
   SDValue Ops[2] = { DAG.getConstantFP(-0.0, dl, N->getValueType(0)),
                      GetSoftenedFloat(N->getOperand(0)) };
@@ -1748,6 +1757,8 @@ static ISD::NodeType GetPromotionOpcode(EVT OpVT, EVT RetVT) {
 }
 
 bool DAGTypeLegalizer::PromoteFloatOperand(SDNode *N, unsigned OpNo) {
+  LLVM_DEBUG(dbgs() << "Promote float operand " << OpNo << ": "; N->dump(&DAG);
+             dbgs() << "\n");
   SDValue R = SDValue();
 
   if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) {
@@ -1762,6 +1773,10 @@ bool DAGTypeLegalizer::PromoteFloatOperand(SDNode *N, unsigned OpNo) {
   // a part of PromoteFloatResult.
   switch (N->getOpcode()) {
     default:
+  #ifndef NDEBUG
+      dbgs() << "PromoteFloatOperand Op #" << OpNo << ": ";
+      N->dump(&DAG); dbgs() << "\n";
+  #endif
       llvm_unreachable("Do not know how to promote this operator's operand!");
 
     case ISD::BITCAST:    R = PromoteFloatOp_BITCAST(N, OpNo); break;
@@ -1872,6 +1887,8 @@ SDValue DAGTypeLegalizer::PromoteFloatOp_STORE(SDNode *N, unsigned OpNo) {
 //===----------------------------------------------------------------------===//
 
 void DAGTypeLegalizer::PromoteFloatResult(SDNode *N, unsigned ResNo) {
+  LLVM_DEBUG(dbgs() << "Promote float result " << ResNo << ": "; N->dump(&DAG);
+             dbgs() << "\n");
   SDValue R = SDValue();
 
   switch (N->getOpcode()) {
@@ -1880,6 +1897,10 @@ void DAGTypeLegalizer::PromoteFloatResult(SDNode *N, unsigned ResNo) {
     case ISD::FP16_TO_FP:
     case ISD::FP_TO_FP16:
     default:
+#ifndef NDEBUG
+      dbgs() << "PromoteFloatResult #" << ResNo << ": ";
+      N->dump(&DAG); dbgs() << "\n";
+#endif
       llvm_unreachable("Do not know how to promote this operator's result!");
 
     case ISD::BITCAST:    R = PromoteFloatRes_BITCAST(N); break;
@@ -1932,7 +1953,7 @@ void DAGTypeLegalizer::PromoteFloatResult(SDNode *N, unsigned ResNo) {
     case ISD::SINT_TO_FP:
     case ISD::UINT_TO_FP: R = PromoteFloatRes_XINT_TO_FP(N); break;
     case ISD::UNDEF:      R = PromoteFloatRes_UNDEF(N); break;
-
+    case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break;
   }
 
   if (R.getNode())
@@ -2143,9 +2164,9 @@ SDValue DAGTypeLegalizer::PromoteFloatRes_SELECT_CC(SDNode *N) {
   SDValue TrueVal = GetPromotedFloat(N->getOperand(2));
   SDValue FalseVal = GetPromotedFloat(N->getOperand(3));
 
-  return DAG.getNode(ISD::SELECT_CC, SDLoc(N), N->getValueType(0),
-                     N->getOperand(0), N->getOperand(1), TrueVal, FalseVal,
-                     N->getOperand(4));
+  return DAG.getNode(ISD::SELECT_CC, SDLoc(N),
+                     TrueVal.getNode()->getValueType(0), N->getOperand(0),
+                     N->getOperand(1), TrueVal, FalseVal, N->getOperand(4));
 }
 
 // Construct a SDNode that transforms the SINT or UINT operand to the promoted
@@ -2166,3 +2187,29 @@ SDValue DAGTypeLegalizer::PromoteFloatRes_UNDEF(SDNode *N) {
                                                N->getValueType(0)));
 }
 
+SDValue DAGTypeLegalizer::BitcastToInt_ATOMIC_SWAP(SDNode *N) {
+  EVT VT = N->getValueType(0);
+  EVT NFPVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
+
+  AtomicSDNode *AM = cast<AtomicSDNode>(N);
+  SDLoc SL(N);
+
+  SDValue CastVal = BitConvertToInteger(AM->getVal());
+  EVT CastVT = CastVal.getValueType();
+
+  SDValue NewAtomic
+    = DAG.getAtomic(ISD::ATOMIC_SWAP, SL, CastVT,
+                    DAG.getVTList(CastVT, MVT::Other),
+                    { AM->getChain(), AM->getBasePtr(), CastVal },
+                    AM->getMemOperand());
+
+  SDValue ResultCast = DAG.getNode(GetPromotionOpcode(VT, NFPVT), SL, NFPVT,
+                                   NewAtomic);
+  // Legalize the chain result by replacing uses of the old value chain with the
+  // new one
+  ReplaceValueWith(SDValue(N, 1), NewAtomic.getValue(1));
+
+  return ResultCast;
+
+}
+
diff --git a/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp b/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
index 25fa2a0a4af1f6489f217db622178daa5c874b07..aee4194b86e7f1030d093a93d6b7f668a190e352 100644
--- a/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
+++ b/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
@@ -1,9 +1,8 @@
 //===----- LegalizeIntegerTypes.cpp - Legalization of integer types -------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -140,6 +139,8 @@ void DAGTypeLegalizer::PromoteIntegerResult(SDNode *N, unsigned ResNo) {
   case ISD::SMULO:
   case ISD::UMULO:       Res = PromoteIntRes_XMULO(N, ResNo); break;
 
+  case ISD::ADDE:
+  case ISD::SUBE:
   case ISD::ADDCARRY:
   case ISD::SUBCARRY:    Res = PromoteIntRes_ADDSUBCARRY(N, ResNo); break;
 
@@ -147,7 +148,8 @@ void DAGTypeLegalizer::PromoteIntegerResult(SDNode *N, unsigned ResNo) {
   case ISD::UADDSAT:
   case ISD::SSUBSAT:
   case ISD::USUBSAT:     Res = PromoteIntRes_ADDSUBSAT(N); break;
-  case ISD::SMULFIX:     Res = PromoteIntRes_SMULFIX(N); break;
+  case ISD::SMULFIX:
+  case ISD::UMULFIX:     Res = PromoteIntRes_MULFIX(N); break;
 
   case ISD::ATOMIC_LOAD:
     Res = PromoteIntRes_Atomic0(cast<AtomicSDNode>(N)); break;
@@ -439,8 +441,26 @@ SDValue DAGTypeLegalizer::PromoteIntRes_CTTZ(SDNode *N) {
 SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_VECTOR_ELT(SDNode *N) {
   SDLoc dl(N);
   EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
-  return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NVT, N->getOperand(0),
-                     N->getOperand(1));
+
+  SDValue Op0 = N->getOperand(0);
+  SDValue Op1 = N->getOperand(1);
+
+  // If the input also needs to be promoted, do that first so we can get a
+  // get a good idea for the output type.
+  if (TLI.getTypeAction(*DAG.getContext(), Op0.getValueType())
+      == TargetLowering::TypePromoteInteger) {
+    SDValue In = GetPromotedInteger(Op0);
+
+    // If the new type is larger than NVT, use it. We probably won't need to
+    // promote it again.
+    EVT SVT = In.getValueType().getScalarType();
+    if (SVT.bitsGE(NVT)) {
+      SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, SVT, In, Op1);
+      return DAG.getAnyExtOrTrunc(Ext, dl, NVT);
+    }
+  }
+
+  return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NVT, Op0, Op1);
 }
 
 SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_XINT(SDNode *N) {
@@ -535,7 +555,7 @@ SDValue DAGTypeLegalizer::PromoteIntRes_MLOAD(MaskedLoadSDNode *N) {
   SDLoc dl(N);
   SDValue Res = DAG.getMaskedLoad(NVT, dl, N->getChain(), N->getBasePtr(),
                                   N->getMask(), ExtPassThru, N->getMemoryVT(),
-                                  N->getMemOperand(), ISD::SEXTLOAD);
+                                  N->getMemOperand(), ISD::EXTLOAD);
   // Legalize the chain result - switch anything that used the old chain to
   // use the new one.
   ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
@@ -626,11 +646,17 @@ SDValue DAGTypeLegalizer::PromoteIntRes_ADDSUBSAT(SDNode *N) {
   return DAG.getNode(ShiftOp, dl, PromotedType, Result, ShiftAmount);
 }
 
-SDValue DAGTypeLegalizer::PromoteIntRes_SMULFIX(SDNode *N) {
+SDValue DAGTypeLegalizer::PromoteIntRes_MULFIX(SDNode *N) {
   // Can just promote the operands then continue with operation.
   SDLoc dl(N);
-  SDValue Op1Promoted = SExtPromotedInteger(N->getOperand(0));
-  SDValue Op2Promoted = SExtPromotedInteger(N->getOperand(1));
+  SDValue Op1Promoted, Op2Promoted;
+  if (N->getOpcode() == ISD::SMULFIX) {
+    Op1Promoted = SExtPromotedInteger(N->getOperand(0));
+    Op2Promoted = SExtPromotedInteger(N->getOperand(1));
+  } else {
+    Op1Promoted = ZExtPromotedInteger(N->getOperand(0));
+    Op2Promoted = ZExtPromotedInteger(N->getOperand(1));
+  }
   EVT PromotedType = Op1Promoted.getValueType();
   return DAG.getNode(N->getOpcode(), dl, PromotedType, Op1Promoted, Op2Promoted,
                      N->getOperand(2));
@@ -855,7 +881,7 @@ SDValue DAGTypeLegalizer::PromoteIntRes_UADDSUBO(SDNode *N, unsigned ResNo) {
 
   // Calculate the overflow flag: zero extend the arithmetic result from
   // the original type.
-  SDValue Ofl = DAG.getZeroExtendInReg(Res, dl, OVT);
+  SDValue Ofl = DAG.getZeroExtendInReg(Res, dl, OVT.getScalarType());
   // Overflowed if and only if this is not equal to Res.
   Ofl = DAG.getSetCC(dl, N->getValueType(1), Ofl, Res, ISD::SETNE);
 
@@ -865,6 +891,9 @@ SDValue DAGTypeLegalizer::PromoteIntRes_UADDSUBO(SDNode *N, unsigned ResNo) {
   return Res;
 }
 
+// Handle promotion for the ADDE/SUBE/ADDCARRY/SUBCARRY nodes. Notice that
+// the third operand of ADDE/SUBE nodes is carry flag, which differs from 
+// the ADDCARRY/SUBCARRY nodes in that the third operand is carry Boolean.
 SDValue DAGTypeLegalizer::PromoteIntRes_ADDSUBCARRY(SDNode *N, unsigned ResNo) {
   if (ResNo == 1)
     return PromoteIntRes_Overflow(N);
@@ -1068,7 +1097,10 @@ bool DAGTypeLegalizer::PromoteIntegerOperand(SDNode *N, unsigned OpNo) {
 
   case ISD::PREFETCH: Res = PromoteIntOp_PREFETCH(N, OpNo); break;
 
-  case ISD::SMULFIX: Res = PromoteIntOp_SMULFIX(N); break;
+  case ISD::SMULFIX:
+  case ISD::UMULFIX: Res = PromoteIntOp_MULFIX(N); break;
+
+  case ISD::FPOWI: Res = PromoteIntOp_FPOWI(N); break;
   }
 
   // If the result is null, the sub-method took care of registering results etc.
@@ -1428,7 +1460,7 @@ SDValue DAGTypeLegalizer::PromoteIntOp_ADDSUBCARRY(SDNode *N, unsigned OpNo) {
   return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, Carry), 0);
 }
 
-SDValue DAGTypeLegalizer::PromoteIntOp_SMULFIX(SDNode *N) {
+SDValue DAGTypeLegalizer::PromoteIntOp_MULFIX(SDNode *N) {
   SDValue Op2 = ZExtPromotedInteger(N->getOperand(2));
   return SDValue(
       DAG.UpdateNodeOperands(N, N->getOperand(0), N->getOperand(1), Op2), 0);
@@ -1452,6 +1484,11 @@ SDValue DAGTypeLegalizer::PromoteIntOp_PREFETCH(SDNode *N, unsigned OpNo) {
                  0);
 }
 
+SDValue DAGTypeLegalizer::PromoteIntOp_FPOWI(SDNode *N) {
+  SDValue Op = SExtPromotedInteger(N->getOperand(1));
+  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Op), 0);
+}
+
 //===----------------------------------------------------------------------===//
 //  Integer Result Expansion
 //===----------------------------------------------------------------------===//
@@ -1476,7 +1513,8 @@ void DAGTypeLegalizer::ExpandIntegerResult(SDNode *N, unsigned ResNo) {
     dbgs() << "ExpandIntegerResult #" << ResNo << ": ";
     N->dump(&DAG); dbgs() << "\n";
 #endif
-    llvm_unreachable("Do not know how to expand the result of this operator!");
+    report_fatal_error("Do not know how to expand the result of this "
+                       "operator!");
 
   case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, ResNo, Lo, Hi); break;
   case ISD::SELECT:       SplitRes_SELECT(N, Lo, Hi); break;
@@ -1590,7 +1628,8 @@ void DAGTypeLegalizer::ExpandIntegerResult(SDNode *N, unsigned ResNo) {
   case ISD::UADDSAT:
   case ISD::SSUBSAT:
   case ISD::USUBSAT: ExpandIntRes_ADDSUBSAT(N, Lo, Hi); break;
-  case ISD::SMULFIX: ExpandIntRes_SMULFIX(N, Lo, Hi); break;
+  case ISD::SMULFIX:
+  case ISD::UMULFIX: ExpandIntRes_MULFIX(N, Lo, Hi); break;
   }
 
   // If Lo/Hi is null, the sub-method took care of registering results etc.
@@ -1711,8 +1750,7 @@ ExpandShiftWithKnownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) {
   SDLoc dl(N);
 
   APInt HighBitMask = APInt::getHighBitsSet(ShBits, ShBits - Log2_32(NVTBits));
-  KnownBits Known;
-  DAG.computeKnownBits(N->getOperand(1), Known);
+  KnownBits Known = DAG.computeKnownBits(N->getOperand(1));
 
   // If we don't know anything about the high bits, exit.
   if (((Known.Zero|Known.One) & HighBitMask) == 0)
@@ -2555,12 +2593,16 @@ void DAGTypeLegalizer::ExpandIntRes_READCYCLECOUNTER(SDNode *N, SDValue &Lo,
 
 void DAGTypeLegalizer::ExpandIntRes_ADDSUBSAT(SDNode *N, SDValue &Lo,
                                               SDValue &Hi) {
-  SDValue Result = TLI.getExpandedSaturationAdditionSubtraction(N, DAG);
+  SDValue Result = TLI.expandAddSubSat(N, DAG);
   SplitInteger(Result, Lo, Hi);
 }
 
-void DAGTypeLegalizer::ExpandIntRes_SMULFIX(SDNode *N, SDValue &Lo,
-                                            SDValue &Hi) {
+void DAGTypeLegalizer::ExpandIntRes_MULFIX(SDNode *N, SDValue &Lo,
+                                           SDValue &Hi) {
+  assert(
+      (N->getOpcode() == ISD::SMULFIX || N->getOpcode() == ISD::UMULFIX) &&
+      "Expected operand to be signed or unsigned fixed point multiplication");
+
   SDLoc dl(N);
   EVT VT = N->getValueType(0);
   SDValue LHS = N->getOperand(0);
@@ -2578,10 +2620,12 @@ void DAGTypeLegalizer::ExpandIntRes_SMULFIX(SDNode *N, SDValue &Lo,
   GetExpandedInteger(RHS, RL, RH);
   SmallVector<SDValue, 4> Result;
 
-  if (!TLI.expandMUL_LOHI(ISD::SMUL_LOHI, VT, dl, LHS, RHS, Result, NVT, DAG,
+  bool Signed = N->getOpcode() == ISD::SMULFIX;
+  unsigned LoHiOp = Signed ? ISD::SMUL_LOHI : ISD::UMUL_LOHI;
+  if (!TLI.expandMUL_LOHI(LoHiOp, VT, dl, LHS, RHS, Result, NVT, DAG,
                           TargetLowering::MulExpansionKind::OnlyLegalOrCustom,
                           LL, LH, RL, RH)) {
-    report_fatal_error("Unable to expand SMUL_FIX using SMUL_LOHI.");
+    report_fatal_error("Unable to expand MUL_FIX using MUL_LOHI.");
     return;
   }
 
@@ -2642,9 +2686,16 @@ void DAGTypeLegalizer::ExpandIntRes_SMULFIX(SDNode *N, SDValue &Lo,
     Hi = DAG.getNode(ISD::SRL, dl, NVT, ResultHL, SRLAmnt);
     Hi = DAG.getNode(ISD::OR, dl, NVT, Hi,
                      DAG.getNode(ISD::SHL, dl, NVT, ResultHH, SHLAmnt));
+  } else if (Scale == VTSize) {
+    assert(
+        !Signed &&
+        "Only unsigned types can have a scale equal to the operand bit width");
+
+    Lo = ResultHL;
+    Hi = ResultHH;
   } else {
-    llvm_unreachable(
-        "Expected the scale to be less than the width of the operands");
+    llvm_unreachable("Expected the scale to be less than or equal to the width "
+                     "of the operands");
   }
 }
 
@@ -2743,11 +2794,15 @@ void DAGTypeLegalizer::ExpandIntRes_Shift(SDNode *N,
   }
 
   // Next check to see if the target supports this SHL_PARTS operation or if it
-  // will custom expand it.
+  // will custom expand it. Don't lower this to SHL_PARTS when we optimise for
+  // size, but create a libcall instead.
   EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
   TargetLowering::LegalizeAction Action = TLI.getOperationAction(PartsOpc, NVT);
-  if ((Action == TargetLowering::Legal && TLI.isTypeLegal(NVT)) ||
-      Action == TargetLowering::Custom) {
+  const bool LegalOrCustom =
+    (Action == TargetLowering::Legal && TLI.isTypeLegal(NVT)) ||
+    Action == TargetLowering::Custom;
+
+  if (LegalOrCustom && TLI.shouldExpandShift(DAG, N)) {
     // Expand the subcomponents.
     SDValue LHSL, LHSH;
     GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
@@ -3145,7 +3200,7 @@ bool DAGTypeLegalizer::ExpandIntegerOperand(SDNode *N, unsigned OpNo) {
     dbgs() << "ExpandIntegerOperand Op #" << OpNo << ": ";
     N->dump(&DAG); dbgs() << "\n";
   #endif
-    llvm_unreachable("Do not know how to expand this operator's operand!");
+    report_fatal_error("Do not know how to expand this operator's operand!");
 
   case ISD::BITCAST:           Res = ExpandOp_BITCAST(N); break;
   case ISD::BR_CC:             Res = ExpandIntOp_BR_CC(N); break;
diff --git a/lib/CodeGen/SelectionDAG/LegalizeTypes.cpp b/lib/CodeGen/SelectionDAG/LegalizeTypes.cpp
index a9f144c06e9ada31b806ef02bd3dbd73e5d44d5b..03ec64d59332a2b4ac24fbe062579c4537853de4 100644
--- a/lib/CodeGen/SelectionDAG/LegalizeTypes.cpp
+++ b/lib/CodeGen/SelectionDAG/LegalizeTypes.cpp
@@ -1,9 +1,8 @@
 //===-- LegalizeTypes.cpp - Common code for DAG type legalizer ------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/SelectionDAG/LegalizeTypes.h b/lib/CodeGen/SelectionDAG/LegalizeTypes.h
index 032000f6cb799bbb6bc87ddedb9296105faff342..ace46e91358ff85bc34ff059a7dd696876f2c81e 100644
--- a/lib/CodeGen/SelectionDAG/LegalizeTypes.h
+++ b/lib/CodeGen/SelectionDAG/LegalizeTypes.h
@@ -1,9 +1,8 @@
 //===-- LegalizeTypes.h - DAG Type Legalizer class definition ---*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -345,7 +344,7 @@ private:
   SDValue PromoteIntRes_VAARG(SDNode *N);
   SDValue PromoteIntRes_XMULO(SDNode *N, unsigned ResNo);
   SDValue PromoteIntRes_ADDSUBSAT(SDNode *N);
-  SDValue PromoteIntRes_SMULFIX(SDNode *N);
+  SDValue PromoteIntRes_MULFIX(SDNode *N);
   SDValue PromoteIntRes_FLT_ROUNDS(SDNode *N);
 
   // Integer Operand Promotion.
@@ -379,7 +378,8 @@ private:
   SDValue PromoteIntOp_ADDSUBCARRY(SDNode *N, unsigned OpNo);
   SDValue PromoteIntOp_FRAMERETURNADDR(SDNode *N);
   SDValue PromoteIntOp_PREFETCH(SDNode *N, unsigned OpNo);
-  SDValue PromoteIntOp_SMULFIX(SDNode *N);
+  SDValue PromoteIntOp_MULFIX(SDNode *N);
+  SDValue PromoteIntOp_FPOWI(SDNode *N);
 
   void PromoteSetCCOperands(SDValue &LHS,SDValue &RHS, ISD::CondCode Code);
 
@@ -435,7 +435,7 @@ private:
   void ExpandIntRes_UADDSUBO          (SDNode *N, SDValue &Lo, SDValue &Hi);
   void ExpandIntRes_XMULO             (SDNode *N, SDValue &Lo, SDValue &Hi);
   void ExpandIntRes_ADDSUBSAT         (SDNode *N, SDValue &Lo, SDValue &Hi);
-  void ExpandIntRes_SMULFIX           (SDNode *N, SDValue &Lo, SDValue &Hi);
+  void ExpandIntRes_MULFIX            (SDNode *N, SDValue &Lo, SDValue &Hi);
 
   void ExpandIntRes_ATOMIC_LOAD       (SDNode *N, SDValue &Lo, SDValue &Hi);
 
@@ -640,6 +640,7 @@ private:
   SDValue PromoteFloatRes_SELECT_CC(SDNode *N);
   SDValue PromoteFloatRes_UnaryOp(SDNode *N);
   SDValue PromoteFloatRes_UNDEF(SDNode *N);
+  SDValue BitcastToInt_ATOMIC_SWAP(SDNode *N);
   SDValue PromoteFloatRes_XINT_TO_FP(SDNode *N);
 
   bool PromoteFloatOperand(SDNode *N, unsigned OpNo);
@@ -673,6 +674,7 @@ private:
   SDValue ScalarizeVecRes_TernaryOp(SDNode *N);
   SDValue ScalarizeVecRes_UnaryOp(SDNode *N);
   SDValue ScalarizeVecRes_StrictFPOp(SDNode *N);
+  SDValue ScalarizeVecRes_OverflowOp(SDNode *N, unsigned ResNo);
   SDValue ScalarizeVecRes_InregOp(SDNode *N);
   SDValue ScalarizeVecRes_VecInregOp(SDNode *N);
 
@@ -691,7 +693,7 @@ private:
   SDValue ScalarizeVecRes_UNDEF(SDNode *N);
   SDValue ScalarizeVecRes_VECTOR_SHUFFLE(SDNode *N);
 
-  SDValue ScalarizeVecRes_SMULFIX(SDNode *N);
+  SDValue ScalarizeVecRes_MULFIX(SDNode *N);
 
   // Vector Operand Scalarization: <1 x ty> -> ty.
   bool ScalarizeVectorOperand(SDNode *N, unsigned OpNo);
@@ -727,8 +729,10 @@ private:
   void SplitVecRes_InregOp(SDNode *N, SDValue &Lo, SDValue &Hi);
   void SplitVecRes_ExtVecInRegOp(SDNode *N, SDValue &Lo, SDValue &Hi);
   void SplitVecRes_StrictFPOp(SDNode *N, SDValue &Lo, SDValue &Hi);
+  void SplitVecRes_OverflowOp(SDNode *N, unsigned ResNo,
+                              SDValue &Lo, SDValue &Hi);
 
-  void SplitVecRes_SMULFIX(SDNode *N, SDValue &Lo, SDValue &Hi);
+  void SplitVecRes_MULFIX(SDNode *N, SDValue &Lo, SDValue &Hi);
 
   void SplitVecRes_BITCAST(SDNode *N, SDValue &Lo, SDValue &Hi);
   void SplitVecRes_BUILD_VECTOR(SDNode *N, SDValue &Lo, SDValue &Hi);
@@ -808,6 +812,7 @@ private:
   SDValue WidenVecRes_Binary(SDNode *N);
   SDValue WidenVecRes_BinaryCanTrap(SDNode *N);
   SDValue WidenVecRes_StrictFP(SDNode *N);
+  SDValue WidenVecRes_OverflowOp(SDNode *N, unsigned ResNo);
   SDValue WidenVecRes_Convert(SDNode *N);
   SDValue WidenVecRes_FCOPYSIGN(SDNode *N);
   SDValue WidenVecRes_POWI(SDNode *N);
diff --git a/lib/CodeGen/SelectionDAG/LegalizeTypesGeneric.cpp b/lib/CodeGen/SelectionDAG/LegalizeTypesGeneric.cpp
index b9d370441c3e709a4f9415b5ddba18bfc1ee4d88..943f63f46c472504ea0cd074efbc44fa69710bbd 100644
--- a/lib/CodeGen/SelectionDAG/LegalizeTypesGeneric.cpp
+++ b/lib/CodeGen/SelectionDAG/LegalizeTypesGeneric.cpp
@@ -1,9 +1,8 @@
 //===-------- LegalizeTypesGeneric.cpp - Generic type legalization --------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp b/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
index 4df02c6a6bc5f35e45bcfe1a772c86c194a5a5fd..5d080e06d7598812168a2c4c7b1a87a2b58669ac 100644
--- a/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
+++ b/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
@@ -1,9 +1,8 @@
 //===- LegalizeVectorOps.cpp - Implement SelectionDAG::LegalizeVectors ----===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -117,6 +116,12 @@ class VectorLegalizer {
   /// the remaining lanes, finally bitcasting to the proper type.
   SDValue ExpandZERO_EXTEND_VECTOR_INREG(SDValue Op);
 
+  /// Implement expand-based legalization of ABS vector operations.
+  /// If following expanding is legal/custom then do it:
+  /// (ABS x) --> (XOR (ADD x, (SRA x, sizeof(x)-1)), (SRA x, sizeof(x)-1))
+  /// else unroll the operation.
+  SDValue ExpandABS(SDValue Op);
+
   /// Expand bswap of vectors into a shuffle if legal.
   SDValue ExpandBSWAP(SDValue Op);
 
@@ -135,6 +140,8 @@ class VectorLegalizer {
   SDValue ExpandFunnelShift(SDValue Op);
   SDValue ExpandROT(SDValue Op);
   SDValue ExpandFMINNUM_FMAXNUM(SDValue Op);
+  SDValue ExpandAddSubSat(SDValue Op);
+  SDValue ExpandFixedPointMul(SDValue Op);
   SDValue ExpandStrictFPOp(SDValue Op);
 
   /// Implements vector promotion.
@@ -351,8 +358,11 @@ SDValue VectorLegalizer::LegalizeOp(SDValue Op) {
   case ISD::SHL:
   case ISD::SRA:
   case ISD::SRL:
+  case ISD::FSHL:
+  case ISD::FSHR:
   case ISD::ROTL:
   case ISD::ROTR:
+  case ISD::ABS:
   case ISD::BSWAP:
   case ISD::BITREVERSE:
   case ISD::CTLZ:
@@ -415,7 +425,8 @@ SDValue VectorLegalizer::LegalizeOp(SDValue Op) {
   case ISD::USUBSAT:
     Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
     break;
-  case ISD::SMULFIX: {
+  case ISD::SMULFIX:
+  case ISD::UMULFIX: {
     unsigned Scale = Node->getConstantOperandVal(2);
     Action = TLI.getFixedPointOperationAction(Node->getOpcode(),
                                               Node->getValueType(0), Scale);
@@ -747,6 +758,8 @@ SDValue VectorLegalizer::Expand(SDValue Op) {
     return ExpandFSUB(Op);
   case ISD::SETCC:
     return UnrollVSETCC(Op);
+  case ISD::ABS:
+    return ExpandABS(Op);
   case ISD::BITREVERSE:
     return ExpandBITREVERSE(Op);
   case ISD::CTPOP:
@@ -766,6 +779,14 @@ SDValue VectorLegalizer::Expand(SDValue Op) {
   case ISD::FMINNUM:
   case ISD::FMAXNUM:
     return ExpandFMINNUM_FMAXNUM(Op);
+  case ISD::USUBSAT:
+  case ISD::SSUBSAT:
+  case ISD::UADDSAT:
+  case ISD::SADDSAT:
+    return ExpandAddSubSat(Op);
+  case ISD::SMULFIX:
+  case ISD::UMULFIX:
+    return ExpandFixedPointMul(Op);
   case ISD::STRICT_FADD:
   case ISD::STRICT_FSUB:
   case ISD::STRICT_FMUL:
@@ -1062,6 +1083,16 @@ SDValue VectorLegalizer::ExpandVSELECT(SDValue Op) {
   return DAG.getNode(ISD::BITCAST, DL, Op.getValueType(), Val);
 }
 
+SDValue VectorLegalizer::ExpandABS(SDValue Op) {
+  // Attempt to expand using TargetLowering.
+  SDValue Result;
+  if (TLI.expandABS(Op.getNode(), Result, DAG))
+    return Result;
+
+  // Otherwise go ahead and unroll.
+  return DAG.UnrollVectorOp(Op.getNode());
+}
+
 SDValue VectorLegalizer::ExpandFP_TO_UINT(SDValue Op) {
   // Attempt to expand using TargetLowering.
   SDValue Result;
@@ -1185,6 +1216,18 @@ SDValue VectorLegalizer::ExpandFMINNUM_FMAXNUM(SDValue Op) {
   return DAG.UnrollVectorOp(Op.getNode());
 }
 
+SDValue VectorLegalizer::ExpandAddSubSat(SDValue Op) {
+  if (SDValue Expanded = TLI.expandAddSubSat(Op.getNode(), DAG))
+    return Expanded;
+  return DAG.UnrollVectorOp(Op.getNode());
+}
+
+SDValue VectorLegalizer::ExpandFixedPointMul(SDValue Op) {
+  if (SDValue Expanded = TLI.expandFixedPointMul(Op.getNode(), DAG))
+    return Expanded;
+  return DAG.UnrollVectorOp(Op.getNode());
+}
+
 SDValue VectorLegalizer::ExpandStrictFPOp(SDValue Op) {
   EVT VT = Op.getValueType();
   EVT EltVT = VT.getVectorElementType();
diff --git a/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp b/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
index f367e9358576ab5e37d8e64f9f5d0b45ca75c513..f5ebb56ef2d4f7ac2af2eb056d4f31ba938e92d1 100644
--- a/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
+++ b/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
@@ -1,9 +1,8 @@
 //===------- LegalizeVectorTypes.cpp - Legalization of vector types -------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -172,8 +171,17 @@ void DAGTypeLegalizer::ScalarizeVectorResult(SDNode *N, unsigned ResNo) {
   case ISD::STRICT_FTRUNC:
     R = ScalarizeVecRes_StrictFPOp(N);
     break;
+  case ISD::UADDO:
+  case ISD::SADDO:
+  case ISD::USUBO:
+  case ISD::SSUBO:
+  case ISD::UMULO:
+  case ISD::SMULO:
+    R = ScalarizeVecRes_OverflowOp(N, ResNo);
+    break;
   case ISD::SMULFIX:
-    R = ScalarizeVecRes_SMULFIX(N);
+  case ISD::UMULFIX:
+    R = ScalarizeVecRes_MULFIX(N);
     break;
   }
 
@@ -197,7 +205,7 @@ SDValue DAGTypeLegalizer::ScalarizeVecRes_TernaryOp(SDNode *N) {
                      Op0.getValueType(), Op0, Op1, Op2);
 }
 
-SDValue DAGTypeLegalizer::ScalarizeVecRes_SMULFIX(SDNode *N) {
+SDValue DAGTypeLegalizer::ScalarizeVecRes_MULFIX(SDNode *N) {
   SDValue Op0 = GetScalarizedVector(N->getOperand(0));
   SDValue Op1 = GetScalarizedVector(N->getOperand(1));
   SDValue Op2 = N->getOperand(2);
@@ -235,6 +243,43 @@ SDValue DAGTypeLegalizer::ScalarizeVecRes_StrictFPOp(SDNode *N) {
   return Result;
 }
 
+SDValue DAGTypeLegalizer::ScalarizeVecRes_OverflowOp(SDNode *N,
+                                                     unsigned ResNo) {
+  SDLoc DL(N);
+  EVT ResVT = N->getValueType(0);
+  EVT OvVT = N->getValueType(1);
+
+  SDValue ScalarLHS, ScalarRHS;
+  if (getTypeAction(ResVT) == TargetLowering::TypeScalarizeVector) {
+    ScalarLHS = GetScalarizedVector(N->getOperand(0));
+    ScalarRHS = GetScalarizedVector(N->getOperand(1));
+  } else {
+    SmallVector<SDValue, 1> ElemsLHS, ElemsRHS;
+    DAG.ExtractVectorElements(N->getOperand(0), ElemsLHS);
+    DAG.ExtractVectorElements(N->getOperand(1), ElemsRHS);
+    ScalarLHS = ElemsLHS[0];
+    ScalarRHS = ElemsRHS[0];
+  }
+
+  SDVTList ScalarVTs = DAG.getVTList(
+      ResVT.getVectorElementType(), OvVT.getVectorElementType());
+  SDNode *ScalarNode = DAG.getNode(
+      N->getOpcode(), DL, ScalarVTs, ScalarLHS, ScalarRHS).getNode();
+
+  // Replace the other vector result not being explicitly scalarized here.
+  unsigned OtherNo = 1 - ResNo;
+  EVT OtherVT = N->getValueType(OtherNo);
+  if (getTypeAction(OtherVT) == TargetLowering::TypeScalarizeVector) {
+    SetScalarizedVector(SDValue(N, OtherNo), SDValue(ScalarNode, OtherNo));
+  } else {
+    SDValue OtherVal = DAG.getNode(
+        ISD::SCALAR_TO_VECTOR, DL, OtherVT, SDValue(ScalarNode, OtherNo));
+    ReplaceValueWith(SDValue(N, OtherNo), OtherVal);
+  }
+
+  return SDValue(ScalarNode, ResNo);
+}
+
 SDValue DAGTypeLegalizer::ScalarizeVecRes_MERGE_VALUES(SDNode *N,
                                                        unsigned ResNo) {
   SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
@@ -859,8 +904,17 @@ void DAGTypeLegalizer::SplitVectorResult(SDNode *N, unsigned ResNo) {
   case ISD::STRICT_FTRUNC:
     SplitVecRes_StrictFPOp(N, Lo, Hi);
     break;
+  case ISD::UADDO:
+  case ISD::SADDO:
+  case ISD::USUBO:
+  case ISD::SSUBO:
+  case ISD::UMULO:
+  case ISD::SMULO:
+    SplitVecRes_OverflowOp(N, ResNo, Lo, Hi);
+    break;
   case ISD::SMULFIX:
-    SplitVecRes_SMULFIX(N, Lo, Hi);
+  case ISD::UMULFIX:
+    SplitVecRes_MULFIX(N, Lo, Hi);
     break;
   }
 
@@ -899,8 +953,7 @@ void DAGTypeLegalizer::SplitVecRes_TernaryOp(SDNode *N, SDValue &Lo,
                    Op0Hi, Op1Hi, Op2Hi);
 }
 
-void DAGTypeLegalizer::SplitVecRes_SMULFIX(SDNode *N, SDValue &Lo,
-                                           SDValue &Hi) {
+void DAGTypeLegalizer::SplitVecRes_MULFIX(SDNode *N, SDValue &Lo, SDValue &Hi) {
   SDValue LHSLo, LHSHi;
   GetSplitVector(N->getOperand(0), LHSLo, LHSHi);
   SDValue RHSLo, RHSHi;
@@ -1205,6 +1258,47 @@ void DAGTypeLegalizer::SplitVecRes_StrictFPOp(SDNode *N, SDValue &Lo,
   ReplaceValueWith(SDValue(N, 1), Chain);
 }
 
+void DAGTypeLegalizer::SplitVecRes_OverflowOp(SDNode *N, unsigned ResNo,
+                                              SDValue &Lo, SDValue &Hi) {
+  SDLoc dl(N);
+  EVT ResVT = N->getValueType(0);
+  EVT OvVT = N->getValueType(1);
+  EVT LoResVT, HiResVT, LoOvVT, HiOvVT;
+  std::tie(LoResVT, HiResVT) = DAG.GetSplitDestVTs(ResVT);
+  std::tie(LoOvVT, HiOvVT) = DAG.GetSplitDestVTs(OvVT);
+
+  SDValue LoLHS, HiLHS, LoRHS, HiRHS;
+  if (getTypeAction(ResVT) == TargetLowering::TypeSplitVector) {
+    GetSplitVector(N->getOperand(0), LoLHS, HiLHS);
+    GetSplitVector(N->getOperand(1), LoRHS, HiRHS);
+  } else {
+    std::tie(LoLHS, HiLHS) = DAG.SplitVectorOperand(N, 0);
+    std::tie(LoRHS, HiRHS) = DAG.SplitVectorOperand(N, 1);
+  }
+
+  unsigned Opcode = N->getOpcode();
+  SDVTList LoVTs = DAG.getVTList(LoResVT, LoOvVT);
+  SDVTList HiVTs = DAG.getVTList(HiResVT, HiOvVT);
+  SDNode *LoNode = DAG.getNode(Opcode, dl, LoVTs, LoLHS, LoRHS).getNode();
+  SDNode *HiNode = DAG.getNode(Opcode, dl, HiVTs, HiLHS, HiRHS).getNode();
+
+  Lo = SDValue(LoNode, ResNo);
+  Hi = SDValue(HiNode, ResNo);
+
+  // Replace the other vector result not being explicitly split here.
+  unsigned OtherNo = 1 - ResNo;
+  EVT OtherVT = N->getValueType(OtherNo);
+  if (getTypeAction(OtherVT) == TargetLowering::TypeSplitVector) {
+    SetSplitVector(SDValue(N, OtherNo),
+                   SDValue(LoNode, OtherNo), SDValue(HiNode, OtherNo));
+  } else {
+    SDValue OtherVal = DAG.getNode(
+        ISD::CONCAT_VECTORS, dl, OtherVT,
+        SDValue(LoNode, OtherNo), SDValue(HiNode, OtherNo));
+    ReplaceValueWith(SDValue(N, OtherNo), OtherVal);
+  }
+}
+
 void DAGTypeLegalizer::SplitVecRes_INSERT_VECTOR_ELT(SDNode *N, SDValue &Lo,
                                                      SDValue &Hi) {
   SDValue Vec = N->getOperand(0);
@@ -1920,7 +2014,6 @@ SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
 
   if (isa<ConstantSDNode>(Idx)) {
     uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
-    assert(IdxVal < VecVT.getVectorNumElements() && "Invalid vector index!");
 
     SDValue Lo, Hi;
     GetSplitVector(Vec, Lo, Hi);
@@ -2472,6 +2565,15 @@ void DAGTypeLegalizer::WidenVectorResult(SDNode *N, unsigned ResNo) {
     Res = WidenVecRes_StrictFP(N);
     break;
 
+  case ISD::UADDO:
+  case ISD::SADDO:
+  case ISD::USUBO:
+  case ISD::SSUBO:
+  case ISD::UMULO:
+  case ISD::SMULO:
+    Res = WidenVecRes_OverflowOp(N, ResNo);
+    break;
+
   case ISD::FCOPYSIGN:
     Res = WidenVecRes_FCOPYSIGN(N);
     break;
@@ -2846,6 +2948,58 @@ SDValue DAGTypeLegalizer::WidenVecRes_StrictFP(SDNode *N) {
   return CollectOpsToWiden(DAG, TLI, ConcatOps, ConcatEnd, VT, MaxVT, WidenVT);
 }
 
+SDValue DAGTypeLegalizer::WidenVecRes_OverflowOp(SDNode *N, unsigned ResNo) {
+  SDLoc DL(N);
+  EVT ResVT = N->getValueType(0);
+  EVT OvVT = N->getValueType(1);
+  EVT WideResVT, WideOvVT;
+  SDValue WideLHS, WideRHS;
+
+  // TODO: This might result in a widen/split loop.
+  if (ResNo == 0) {
+    WideResVT = TLI.getTypeToTransformTo(*DAG.getContext(), ResVT);
+    WideOvVT = EVT::getVectorVT(
+        *DAG.getContext(), OvVT.getVectorElementType(),
+        WideResVT.getVectorNumElements());
+
+    WideLHS = GetWidenedVector(N->getOperand(0));
+    WideRHS = GetWidenedVector(N->getOperand(1));
+  } else {
+    WideOvVT = TLI.getTypeToTransformTo(*DAG.getContext(), OvVT);
+    WideResVT = EVT::getVectorVT(
+        *DAG.getContext(), ResVT.getVectorElementType(),
+        WideOvVT.getVectorNumElements());
+
+    SDValue Zero = DAG.getConstant(
+        0, DL, TLI.getVectorIdxTy(DAG.getDataLayout()));
+    WideLHS = DAG.getNode(
+        ISD::INSERT_SUBVECTOR, DL, WideResVT, DAG.getUNDEF(WideResVT),
+        N->getOperand(0), Zero);
+    WideRHS = DAG.getNode(
+        ISD::INSERT_SUBVECTOR, DL, WideResVT, DAG.getUNDEF(WideResVT),
+        N->getOperand(1), Zero);
+  }
+
+  SDVTList WideVTs = DAG.getVTList(WideResVT, WideOvVT);
+  SDNode *WideNode = DAG.getNode(
+      N->getOpcode(), DL, WideVTs, WideLHS, WideRHS).getNode();
+
+  // Replace the other vector result not being explicitly widened here.
+  unsigned OtherNo = 1 - ResNo;
+  EVT OtherVT = N->getValueType(OtherNo);
+  if (getTypeAction(OtherVT) == TargetLowering::TypeWidenVector) {
+    SetWidenedVector(SDValue(N, OtherNo), SDValue(WideNode, OtherNo));
+  } else {
+    SDValue Zero = DAG.getConstant(
+        0, DL, TLI.getVectorIdxTy(DAG.getDataLayout()));
+    SDValue OtherVal = DAG.getNode(
+        ISD::EXTRACT_SUBVECTOR, DL, OtherVT, SDValue(WideNode, OtherNo), Zero);
+    ReplaceValueWith(SDValue(N, OtherNo), OtherVal);
+  }
+
+  return SDValue(WideNode, ResNo);
+}
+
 SDValue DAGTypeLegalizer::WidenVecRes_Convert(SDNode *N) {
   SDValue InOp = N->getOperand(0);
   SDLoc DL(N);
diff --git a/lib/CodeGen/SelectionDAG/ResourcePriorityQueue.cpp b/lib/CodeGen/SelectionDAG/ResourcePriorityQueue.cpp
index 7f369c746d24e555608ac84c283af8223c76efdb..34660e3a48ec5ebc37a38e5a34b9718d08982c17 100644
--- a/lib/CodeGen/SelectionDAG/ResourcePriorityQueue.cpp
+++ b/lib/CodeGen/SelectionDAG/ResourcePriorityQueue.cpp
@@ -1,9 +1,8 @@
 //===- ResourcePriorityQueue.cpp - A DFA-oriented priority queue -*- C++ -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -85,6 +84,7 @@ ResourcePriorityQueue::numberRCValPredInSU(SUnit *SU, unsigned RCId) {
       case ISD::CopyFromReg:    NumberDeps++;  break;
       case ISD::CopyToReg:      break;
       case ISD::INLINEASM:      break;
+      case ISD::INLINEASM_BR:   break;
     }
     if (!ScegN->isMachineOpcode())
       continue;
@@ -121,6 +121,7 @@ unsigned ResourcePriorityQueue::numberRCValSuccInSU(SUnit *SU,
       case ISD::CopyFromReg:    break;
       case ISD::CopyToReg:      NumberDeps++;  break;
       case ISD::INLINEASM:      break;
+      case ISD::INLINEASM_BR:   break;
     }
     if (!ScegN->isMachineOpcode())
       continue;
@@ -446,6 +447,7 @@ int ResourcePriorityQueue::SUSchedulingCost(SUnit *SU) {
         break;
 
       case ISD::INLINEASM:
+      case ISD::INLINEASM_BR:
         ResCount += PriorityThree;
         break;
       }
@@ -548,6 +550,7 @@ void ResourcePriorityQueue::initNumRegDefsLeft(SUnit *SU) {
           NodeNumDefs++;
           break;
         case ISD::INLINEASM:
+        case ISD::INLINEASM_BR:
           NodeNumDefs++;
           break;
       }
diff --git a/lib/CodeGen/SelectionDAG/SDNodeDbgValue.h b/lib/CodeGen/SelectionDAG/SDNodeDbgValue.h
index f7566b246f323d13aa606026d04e507d03069922..65b9d017fc5cdf786ad5cb67dbbce96f0bf0bcac 100644
--- a/lib/CodeGen/SelectionDAG/SDNodeDbgValue.h
+++ b/lib/CodeGen/SelectionDAG/SDNodeDbgValue.h
@@ -1,9 +1,8 @@
 //===-- llvm/CodeGen/SDNodeDbgValue.h - SelectionDAG dbg_value --*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -136,7 +135,8 @@ public:
   /// dbg.addr is emitted twice.
   void clearIsEmitted() { Emitted = false; }
 
-  LLVM_DUMP_METHOD void dump(raw_ostream &OS) const;
+  LLVM_DUMP_METHOD void dump() const;
+  LLVM_DUMP_METHOD void print(raw_ostream &OS) const;
 };
 
 /// Holds the information from a dbg_label node through SDISel.
diff --git a/lib/CodeGen/SelectionDAG/ScheduleDAGFast.cpp b/lib/CodeGen/SelectionDAG/ScheduleDAGFast.cpp
index 90e109b022fd5fee6110d95f9217db9ae7f21ecf..2cb850fa1a3d55dad83a9c107b4334984bbb847b 100644
--- a/lib/CodeGen/SelectionDAG/ScheduleDAGFast.cpp
+++ b/lib/CodeGen/SelectionDAG/ScheduleDAGFast.cpp
@@ -1,9 +1,8 @@
 //===----- ScheduleDAGFast.cpp - Fast poor list scheduler -----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -480,7 +479,8 @@ bool ScheduleDAGFast::DelayForLiveRegsBottomUp(SUnit *SU,
   }
 
   for (SDNode *Node = SU->getNode(); Node; Node = Node->getGluedNode()) {
-    if (Node->getOpcode() == ISD::INLINEASM) {
+    if (Node->getOpcode() == ISD::INLINEASM ||
+        Node->getOpcode() == ISD::INLINEASM_BR) {
       // Inline asm can clobber physical defs.
       unsigned NumOps = Node->getNumOperands();
       if (Node->getOperand(NumOps-1).getValueType() == MVT::Glue)
diff --git a/lib/CodeGen/SelectionDAG/ScheduleDAGRRList.cpp b/lib/CodeGen/SelectionDAG/ScheduleDAGRRList.cpp
index 8d75b8133a30263638f5c1fa40e4ddda5267b1f1..75d9eb265aed17813eb481f3ef888c6d62bf6622 100644
--- a/lib/CodeGen/SelectionDAG/ScheduleDAGRRList.cpp
+++ b/lib/CodeGen/SelectionDAG/ScheduleDAGRRList.cpp
@@ -1,9 +1,8 @@
 //===- ScheduleDAGRRList.cpp - Reg pressure reduction list scheduler ------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -709,6 +708,7 @@ void ScheduleDAGRRList::EmitNode(SUnit *SU) {
     // removed.
     return;
   case ISD::INLINEASM:
+  case ISD::INLINEASM_BR:
     // For inline asm, clear the pipeline state.
     HazardRec->Reset();
     return;
@@ -1348,7 +1348,8 @@ DelayForLiveRegsBottomUp(SUnit *SU, SmallVectorImpl<unsigned> &LRegs) {
   }
 
   for (SDNode *Node = SU->getNode(); Node; Node = Node->getGluedNode()) {
-    if (Node->getOpcode() == ISD::INLINEASM) {
+    if (Node->getOpcode() == ISD::INLINEASM ||
+        Node->getOpcode() == ISD::INLINEASM_BR) {
       // Inline asm can clobber physical defs.
       unsigned NumOps = Node->getNumOperands();
       if (Node->getOperand(NumOps-1).getValueType() == MVT::Glue)
@@ -2939,6 +2940,29 @@ void RegReductionPQBase::PrescheduleNodesWithMultipleUses() {
             (cast<RegisterSDNode>(N->getOperand(1))->getReg()))
         continue;
 
+    SDNode *PredFrameSetup = nullptr;
+    for (const SDep &Pred : SU.Preds)
+      if (Pred.isCtrl() && Pred.getSUnit()) {
+        // Find the predecessor which is not data dependence.
+        SDNode *PredND = Pred.getSUnit()->getNode();
+
+        // If PredND is FrameSetup, we should not pre-scheduled the node,
+        // or else, when bottom up scheduling, ADJCALLSTACKDOWN and
+        // ADJCALLSTACKUP may hold CallResource too long and make other
+        // calls can't be scheduled. If there's no other available node
+        // to schedule, the schedular will try to rename the register by
+        // creating copy to avoid the conflict which will fail because
+        // CallResource is not a real physical register.
+        if (PredND && PredND->isMachineOpcode() &&
+            (PredND->getMachineOpcode() == TII->getCallFrameSetupOpcode())) {
+          PredFrameSetup = PredND;
+          break;
+        }
+      }
+    // Skip the node has FrameSetup parent.
+    if (PredFrameSetup != nullptr)
+      continue;
+
     // Locate the single data predecessor.
     SUnit *PredSU = nullptr;
     for (const SDep &Pred : SU.Preds)
diff --git a/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp b/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp
index e258f0a218a538cfa0fde5a1cfe5888808abdb1e..559265a3ac8e95e6dfa1c340d90e66e1966342ab 100644
--- a/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp
+++ b/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp
@@ -1,9 +1,8 @@
 //===--- ScheduleDAGSDNodes.cpp - Implement the ScheduleDAGSDNodes class --===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -741,28 +740,27 @@ ProcessSDDbgValues(SDNode *N, SelectionDAG *DAG, InstrEmitter &Emitter,
 static void
 ProcessSourceNode(SDNode *N, SelectionDAG *DAG, InstrEmitter &Emitter,
                   DenseMap<SDValue, unsigned> &VRBaseMap,
-                  SmallVectorImpl<std::pair<unsigned, MachineInstr*> > &Orders,
-                  SmallSet<unsigned, 8> &Seen) {
+                  SmallVectorImpl<std::pair<unsigned, MachineInstr *>> &Orders,
+                  SmallSet<unsigned, 8> &Seen, MachineInstr *NewInsn) {
   unsigned Order = N->getIROrder();
-  if (!Order || !Seen.insert(Order).second) {
+  if (!Order || Seen.count(Order)) {
     // Process any valid SDDbgValues even if node does not have any order
     // assigned.
     ProcessSDDbgValues(N, DAG, Emitter, Orders, VRBaseMap, 0);
     return;
   }
 
-  MachineBasicBlock *BB = Emitter.getBlock();
-  auto IP = Emitter.getInsertPos();
-  if (IP == BB->begin() || BB->back().isPHI() ||
-      // Fast-isel may have inserted some instructions, in which case the
-      // BB->back().isPHI() test will not fire when we want it to.
-      std::prev(IP)->isPHI()) {
-    // Did not insert any instruction.
-    Orders.push_back({Order, (MachineInstr *)nullptr});
-    return;
+  // If a new instruction was generated for this Order number, record it.
+  // Otherwise, leave this order number unseen: we will either find later
+  // instructions for it, or leave it unseen if there were no instructions at
+  // all.
+  if (NewInsn) {
+    Seen.insert(Order);
+    Orders.push_back({Order, NewInsn});
   }
 
-  Orders.push_back({Order, &*std::prev(IP)});
+  // Even if no instruction was generated, a Value may have become defined via
+  // earlier nodes. Try to process them now.
   ProcessSDDbgValues(N, DAG, Emitter, Orders, VRBaseMap, Order);
 }
 
@@ -815,6 +813,37 @@ EmitSchedule(MachineBasicBlock::iterator &InsertPos) {
   SmallSet<unsigned, 8> Seen;
   bool HasDbg = DAG->hasDebugValues();
 
+  // Emit a node, and determine where its first instruction is for debuginfo.
+  // Zero, one, or multiple instructions can be created when emitting a node.
+  auto EmitNode =
+      [&](SDNode *Node, bool IsClone, bool IsCloned,
+          DenseMap<SDValue, unsigned> &VRBaseMap) -> MachineInstr * {
+    // Fetch instruction prior to this, or end() if nonexistant.
+    auto GetPrevInsn = [&](MachineBasicBlock::iterator I) {
+      if (I == BB->begin())
+        return BB->end();
+      else
+        return std::prev(Emitter.getInsertPos());
+    };
+
+    MachineBasicBlock::iterator Before = GetPrevInsn(Emitter.getInsertPos());
+    Emitter.EmitNode(Node, IsClone, IsCloned, VRBaseMap);
+    MachineBasicBlock::iterator After = GetPrevInsn(Emitter.getInsertPos());
+
+    // If the iterator did not change, no instructions were inserted.
+    if (Before == After)
+      return nullptr;
+
+    if (Before == BB->end()) {
+      // There were no prior instructions; the new ones must start at the
+      // beginning of the block.
+      return &Emitter.getBlock()->instr_front();
+    } else {
+      // Return first instruction after the pre-existing instructions.
+      return &*std::next(Before);
+    }
+  };
+
   // If this is the first BB, emit byval parameter dbg_value's.
   if (HasDbg && BB->getParent()->begin() == MachineFunction::iterator(BB)) {
     SDDbgInfo::DbgIterator PDI = DAG->ByvalParmDbgBegin();
@@ -851,18 +880,18 @@ EmitSchedule(MachineBasicBlock::iterator &InsertPos) {
       GluedNodes.push_back(N);
     while (!GluedNodes.empty()) {
       SDNode *N = GluedNodes.back();
-      Emitter.EmitNode(N, SU->OrigNode != SU, SU->isCloned, VRBaseMap);
+      auto NewInsn = EmitNode(N, SU->OrigNode != SU, SU->isCloned, VRBaseMap);
       // Remember the source order of the inserted instruction.
       if (HasDbg)
-        ProcessSourceNode(N, DAG, Emitter, VRBaseMap, Orders, Seen);
+        ProcessSourceNode(N, DAG, Emitter, VRBaseMap, Orders, Seen, NewInsn);
       GluedNodes.pop_back();
     }
-    Emitter.EmitNode(SU->getNode(), SU->OrigNode != SU, SU->isCloned,
-                     VRBaseMap);
+    auto NewInsn =
+        EmitNode(SU->getNode(), SU->OrigNode != SU, SU->isCloned, VRBaseMap);
     // Remember the source order of the inserted instruction.
     if (HasDbg)
-      ProcessSourceNode(SU->getNode(), DAG, Emitter, VRBaseMap, Orders,
-                        Seen);
+      ProcessSourceNode(SU->getNode(), DAG, Emitter, VRBaseMap, Orders, Seen,
+                        NewInsn);
   }
 
   // Insert all the dbg_values which have not already been inserted in source
@@ -887,8 +916,7 @@ EmitSchedule(MachineBasicBlock::iterator &InsertPos) {
       unsigned Order = Orders[i].first;
       MachineInstr *MI = Orders[i].second;
       // Insert all SDDbgValue's whose order(s) are before "Order".
-      if (!MI)
-        continue;
+      assert(MI);
       for (; DI != DE; ++DI) {
         if ((*DI)->getOrder() < LastOrder || (*DI)->getOrder() >= Order)
           break;
diff --git a/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.h b/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.h
index 3fa7ad8957258cdaa4c820563ef19105e64810c1..5163b4fa4fd38451a984377e7f4a581b0fadc7ff 100644
--- a/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.h
+++ b/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.h
@@ -1,9 +1,8 @@
 //===---- ScheduleDAGSDNodes.h - SDNode Scheduling --------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/SelectionDAG/ScheduleDAGVLIW.cpp b/lib/CodeGen/SelectionDAG/ScheduleDAGVLIW.cpp
index 416061475b1a81148350b665d65f1bdd48734a2d..ab06b55b49fdda9404458ed8f02abb3948edfb1e 100644
--- a/lib/CodeGen/SelectionDAG/ScheduleDAGVLIW.cpp
+++ b/lib/CodeGen/SelectionDAG/ScheduleDAGVLIW.cpp
@@ -1,9 +1,8 @@
 //===- ScheduleDAGVLIW.cpp - SelectionDAG list scheduler for VLIW -*- C++ -*-=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index d69e50d9007be7ae2d2a00aa773b764684a2b39b..79b699751dc8f41a0e7f061f65634220955c192c 100644
--- a/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -1,9 +1,8 @@
 //===- SelectionDAG.cpp - Implement the SelectionDAG data structures ------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -87,6 +86,8 @@ static SDVTList makeVTList(const EVT *VTs, unsigned NumVTs) {
 void SelectionDAG::DAGUpdateListener::NodeDeleted(SDNode*, SDNode*) {}
 void SelectionDAG::DAGUpdateListener::NodeUpdated(SDNode*) {}
 
+void SelectionDAG::DAGNodeDeletedListener::anchor() {}
+
 #define DEBUG_TYPE "selectiondag"
 
 static cl::opt<bool> EnableMemCpyDAGOpt("enable-memcpy-dag-opt",
@@ -269,15 +270,24 @@ bool ISD::allOperandsUndef(const SDNode *N) {
 }
 
 bool ISD::matchUnaryPredicate(SDValue Op,
-                              std::function<bool(ConstantSDNode *)> Match) {
+                              std::function<bool(ConstantSDNode *)> Match,
+                              bool AllowUndefs) {
+  // FIXME: Add support for scalar UNDEF cases?
   if (auto *Cst = dyn_cast<ConstantSDNode>(Op))
     return Match(Cst);
 
+  // FIXME: Add support for vector UNDEF cases?
   if (ISD::BUILD_VECTOR != Op.getOpcode())
     return false;
 
   EVT SVT = Op.getValueType().getScalarType();
   for (unsigned i = 0, e = Op.getNumOperands(); i != e; ++i) {
+    if (AllowUndefs && Op.getOperand(i).isUndef()) {
+      if (!Match(nullptr))
+        return false;
+      continue;
+    }
+
     auto *Cst = dyn_cast<ConstantSDNode>(Op.getOperand(i));
     if (!Cst || Cst->getValueType(0) != SVT || !Match(Cst))
       return false;
@@ -287,26 +297,33 @@ bool ISD::matchUnaryPredicate(SDValue Op,
 
 bool ISD::matchBinaryPredicate(
     SDValue LHS, SDValue RHS,
-    std::function<bool(ConstantSDNode *, ConstantSDNode *)> Match) {
+    std::function<bool(ConstantSDNode *, ConstantSDNode *)> Match,
+    bool AllowUndefs) {
   if (LHS.getValueType() != RHS.getValueType())
     return false;
 
+  // TODO: Add support for scalar UNDEF cases?
   if (auto *LHSCst = dyn_cast<ConstantSDNode>(LHS))
     if (auto *RHSCst = dyn_cast<ConstantSDNode>(RHS))
       return Match(LHSCst, RHSCst);
 
+  // TODO: Add support for vector UNDEF cases?
   if (ISD::BUILD_VECTOR != LHS.getOpcode() ||
       ISD::BUILD_VECTOR != RHS.getOpcode())
     return false;
 
   EVT SVT = LHS.getValueType().getScalarType();
   for (unsigned i = 0, e = LHS.getNumOperands(); i != e; ++i) {
-    auto *LHSCst = dyn_cast<ConstantSDNode>(LHS.getOperand(i));
-    auto *RHSCst = dyn_cast<ConstantSDNode>(RHS.getOperand(i));
-    if (!LHSCst || !RHSCst)
+    SDValue LHSOp = LHS.getOperand(i);
+    SDValue RHSOp = RHS.getOperand(i);
+    bool LHSUndef = AllowUndefs && LHSOp.isUndef();
+    bool RHSUndef = AllowUndefs && RHSOp.isUndef();
+    auto *LHSCst = dyn_cast<ConstantSDNode>(LHSOp);
+    auto *RHSCst = dyn_cast<ConstantSDNode>(RHSOp);
+    if ((!LHSCst && !LHSUndef) || (!RHSCst && !RHSUndef))
       return false;
-    if (LHSCst->getValueType(0) != SVT ||
-        LHSCst->getValueType(0) != RHSCst->getValueType(0))
+    if (LHSOp.getValueType() != SVT ||
+        LHSOp.getValueType() != RHSOp.getValueType())
       return false;
     if (!Match(LHSCst, RHSCst))
       return false;
@@ -2102,6 +2119,15 @@ SDValue SelectionDAG::GetDemandedBits(SDValue V, const APInt &Mask) {
       return getNode(ISD::ANY_EXTEND, SDLoc(V), V.getValueType(), DemandedSrc);
     break;
   }
+  case ISD::SIGN_EXTEND_INREG:
+    EVT ExVT = cast<VTSDNode>(V.getOperand(1))->getVT();
+    unsigned ExVTBits = ExVT.getScalarSizeInBits();
+
+    // If none of the extended bits are demanded, eliminate the sextinreg.
+    if (Mask.getActiveBits() <= ExVTBits)
+      return V.getOperand(0);
+
+    break;
   }
   return SDValue();
 }
@@ -2800,7 +2826,15 @@ KnownBits SelectionDAG::computeKnownBits(SDValue Op, const APInt &DemandedElts,
     Known.Zero.setBitsFrom(InVT.getScalarSizeInBits());
     break;
   }
-  // TODO ISD::SIGN_EXTEND_VECTOR_INREG
+  case ISD::SIGN_EXTEND_VECTOR_INREG: {
+    EVT InVT = Op.getOperand(0).getValueType();
+    APInt InDemandedElts = DemandedElts.zextOrSelf(InVT.getVectorNumElements());
+    Known = computeKnownBits(Op.getOperand(0), InDemandedElts, Depth + 1);
+    // If the sign bit is known to be zero or one, then sext will extend
+    // it to the top bits, else it will just zext.
+    Known = Known.sext(BitWidth);
+    break;
+  }
   case ISD::SIGN_EXTEND: {
     Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
     // If the sign bit is known to be zero or one, then sext will extend
@@ -3171,11 +3205,9 @@ SelectionDAG::OverflowKind SelectionDAG::computeOverflowKind(SDValue N0,
   if (isNullConstant(N1))
     return OFK_Never;
 
-  KnownBits N1Known;
-  computeKnownBits(N1, N1Known);
+  KnownBits N1Known = computeKnownBits(N1);
   if (N1Known.Zero.getBoolValue()) {
-    KnownBits N0Known;
-    computeKnownBits(N0, N0Known);
+    KnownBits N0Known = computeKnownBits(N0);
 
     bool overflow;
     (void)(~N0Known.Zero).uadd_ov(~N1Known.Zero, overflow);
@@ -3189,8 +3221,7 @@ SelectionDAG::OverflowKind SelectionDAG::computeOverflowKind(SDValue N0,
     return OFK_Never;
 
   if (N1.getOpcode() == ISD::UMUL_LOHI && N1.getResNo() == 1) {
-    KnownBits N0Known;
-    computeKnownBits(N0, N0Known);
+    KnownBits N0Known = computeKnownBits(N0);
 
     if ((~N0Known.Zero & 0x01) == ~N0Known.Zero)
       return OFK_Never;
@@ -3661,7 +3692,7 @@ unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, const APInt &DemandedElts,
     }
     return ComputeNumSignBits(Src, Depth + 1);
   }
-  case ISD::CONCAT_VECTORS:
+  case ISD::CONCAT_VECTORS: {
     // Determine the minimum number of sign bits across all demanded
     // elts of the input vectors. Early out if the result is already 1.
     Tmp = std::numeric_limits<unsigned>::max();
@@ -3679,6 +3710,40 @@ unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, const APInt &DemandedElts,
     assert(Tmp <= VTBits && "Failed to determine minimum sign bits");
     return Tmp;
   }
+  case ISD::INSERT_SUBVECTOR: {
+    // If we know the element index, demand any elements from the subvector and
+    // the remainder from the src its inserted into, otherwise demand them all.
+    SDValue Src = Op.getOperand(0);
+    SDValue Sub = Op.getOperand(1);
+    auto *SubIdx = dyn_cast<ConstantSDNode>(Op.getOperand(2));
+    unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
+    if (SubIdx && SubIdx->getAPIntValue().ule(NumElts - NumSubElts)) {
+      Tmp = std::numeric_limits<unsigned>::max();
+      uint64_t Idx = SubIdx->getZExtValue();
+      APInt DemandedSubElts = DemandedElts.extractBits(NumSubElts, Idx);
+      if (!!DemandedSubElts) {
+        Tmp = ComputeNumSignBits(Sub, DemandedSubElts, Depth + 1);
+        if (Tmp == 1) return 1; // early-out
+      }
+      APInt SubMask = APInt::getBitsSet(NumElts, Idx, Idx + NumSubElts);
+      APInt DemandedSrcElts = DemandedElts & ~SubMask;
+      if (!!DemandedSrcElts) {
+        Tmp2 = ComputeNumSignBits(Src, DemandedSrcElts, Depth + 1);
+        Tmp = std::min(Tmp, Tmp2);
+      }
+      assert(Tmp <= VTBits && "Failed to determine minimum sign bits");
+      return Tmp;
+    }
+
+    // Not able to determine the index so just assume worst case.
+    Tmp = ComputeNumSignBits(Sub, Depth + 1);
+    if (Tmp == 1) return 1; // early-out
+    Tmp2 = ComputeNumSignBits(Src, Depth + 1);
+    Tmp = std::min(Tmp, Tmp2);
+    assert(Tmp <= VTBits && "Failed to determine minimum sign bits");
+    return Tmp;
+  }
+  }
 
   // If we are looking at the loaded value of the SDNode.
   if (Op.getResNo() == 0) {
@@ -4422,6 +4487,10 @@ static std::pair<APInt, bool> FoldValue(unsigned Opcode, const APInt &C1,
   case ISD::SMAX: return std::make_pair(C1.sge(C2) ? C1 : C2, true);
   case ISD::UMIN: return std::make_pair(C1.ule(C2) ? C1 : C2, true);
   case ISD::UMAX: return std::make_pair(C1.uge(C2) ? C1 : C2, true);
+  case ISD::SADDSAT: return std::make_pair(C1.sadd_sat(C2), true);
+  case ISD::UADDSAT: return std::make_pair(C1.uadd_sat(C2), true);
+  case ISD::SSUBSAT: return std::make_pair(C1.ssub_sat(C2), true);
+  case ISD::USUBSAT: return std::make_pair(C1.usub_sat(C2), true);
   case ISD::UDIV:
     if (!C2.getBoolValue())
       break;
@@ -4443,13 +4512,13 @@ static std::pair<APInt, bool> FoldValue(unsigned Opcode, const APInt &C1,
 }
 
 SDValue SelectionDAG::FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL,
-                                             EVT VT, const ConstantSDNode *Cst1,
-                                             const ConstantSDNode *Cst2) {
-  if (Cst1->isOpaque() || Cst2->isOpaque())
+                                             EVT VT, const ConstantSDNode *C1,
+                                             const ConstantSDNode *C2) {
+  if (C1->isOpaque() || C2->isOpaque())
     return SDValue();
 
-  std::pair<APInt, bool> Folded = FoldValue(Opcode, Cst1->getAPIntValue(),
-                                            Cst2->getAPIntValue());
+  std::pair<APInt, bool> Folded = FoldValue(Opcode, C1->getAPIntValue(),
+                                            C2->getAPIntValue());
   if (!Folded.second)
     return SDValue();
   return getConstant(Folded.first, DL, VT);
@@ -4462,16 +4531,16 @@ SDValue SelectionDAG::FoldSymbolOffset(unsigned Opcode, EVT VT,
     return SDValue();
   if (!TLI->isOffsetFoldingLegal(GA))
     return SDValue();
-  const ConstantSDNode *Cst2 = dyn_cast<ConstantSDNode>(N2);
-  if (!Cst2)
+  auto *C2 = dyn_cast<ConstantSDNode>(N2);
+  if (!C2)
     return SDValue();
-  int64_t Offset = Cst2->getSExtValue();
+  int64_t Offset = C2->getSExtValue();
   switch (Opcode) {
   case ISD::ADD: break;
   case ISD::SUB: Offset = -uint64_t(Offset); break;
   default: return SDValue();
   }
-  return getGlobalAddress(GA->getGlobal(), SDLoc(Cst2), VT,
+  return getGlobalAddress(GA->getGlobal(), SDLoc(C2), VT,
                           GA->getOffset() + uint64_t(Offset));
 }
 
@@ -4501,21 +4570,20 @@ bool SelectionDAG::isUndef(unsigned Opcode, ArrayRef<SDValue> Ops) {
 }
 
 SDValue SelectionDAG::FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL,
-                                             EVT VT, SDNode *Cst1,
-                                             SDNode *Cst2) {
+                                             EVT VT, SDNode *N1, SDNode *N2) {
   // If the opcode is a target-specific ISD node, there's nothing we can
   // do here and the operand rules may not line up with the below, so
   // bail early.
   if (Opcode >= ISD::BUILTIN_OP_END)
     return SDValue();
 
-  if (isUndef(Opcode, {SDValue(Cst1, 0), SDValue(Cst2, 0)}))
+  if (isUndef(Opcode, {SDValue(N1, 0), SDValue(N2, 0)}))
     return getUNDEF(VT);
 
   // Handle the case of two scalars.
-  if (const ConstantSDNode *Scalar1 = dyn_cast<ConstantSDNode>(Cst1)) {
-    if (const ConstantSDNode *Scalar2 = dyn_cast<ConstantSDNode>(Cst2)) {
-      SDValue Folded = FoldConstantArithmetic(Opcode, DL, VT, Scalar1, Scalar2);
+  if (auto *C1 = dyn_cast<ConstantSDNode>(N1)) {
+    if (auto *C2 = dyn_cast<ConstantSDNode>(N2)) {
+      SDValue Folded = FoldConstantArithmetic(Opcode, DL, VT, C1, C2);
       assert((!Folded || !VT.isVector()) &&
              "Can't fold vectors ops with scalar operands");
       return Folded;
@@ -4523,19 +4591,19 @@ SDValue SelectionDAG::FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL,
   }
 
   // fold (add Sym, c) -> Sym+c
-  if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(Cst1))
-    return FoldSymbolOffset(Opcode, VT, GA, Cst2);
+  if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(N1))
+    return FoldSymbolOffset(Opcode, VT, GA, N2);
   if (TLI->isCommutativeBinOp(Opcode))
-    if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(Cst2))
-      return FoldSymbolOffset(Opcode, VT, GA, Cst1);
+    if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(N2))
+      return FoldSymbolOffset(Opcode, VT, GA, N1);
 
   // For vectors, extract each constant element and fold them individually.
   // Either input may be an undef value.
-  auto *BV1 = dyn_cast<BuildVectorSDNode>(Cst1);
-  if (!BV1 && !Cst1->isUndef())
+  auto *BV1 = dyn_cast<BuildVectorSDNode>(N1);
+  if (!BV1 && !N1->isUndef())
     return SDValue();
-  auto *BV2 = dyn_cast<BuildVectorSDNode>(Cst2);
-  if (!BV2 && !Cst2->isUndef())
+  auto *BV2 = dyn_cast<BuildVectorSDNode>(N2);
+  if (!BV2 && !N2->isUndef())
     return SDValue();
   // If both operands are undef, that's handled the same way as scalars.
   if (!BV1 && !BV2)
@@ -4761,6 +4829,10 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
   case ISD::SMAX:
   case ISD::UMIN:
   case ISD::UMAX:
+  case ISD::SADDSAT:
+  case ISD::SSUBSAT:
+  case ISD::UADDSAT:
+  case ISD::USUBSAT:
     assert(VT.isInteger() && "This operator does not apply to FP types!");
     assert(N1.getValueType() == N2.getValueType() &&
            N1.getValueType() == VT && "Binary operator types must match!");
@@ -5108,6 +5180,8 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
       case ISD::SDIV:
       case ISD::UREM:
       case ISD::SREM:
+      case ISD::SSUBSAT:
+      case ISD::USUBSAT:
         return getConstant(0, DL, VT);    // fold op(undef, arg2) -> 0
       }
     }
@@ -5131,8 +5205,12 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
       return getUNDEF(VT);       // fold op(arg1, undef) -> undef
     case ISD::MUL:
     case ISD::AND:
+    case ISD::SSUBSAT:
+    case ISD::USUBSAT:
       return getConstant(0, DL, VT);  // fold op(arg1, undef) -> 0
     case ISD::OR:
+    case ISD::SADDSAT:
+    case ISD::UADDSAT:
       return getAllOnesConstant(DL, VT);
     }
   }
@@ -6342,35 +6420,6 @@ SDValue SelectionDAG::getAtomicCmpSwap(unsigned Opcode, const SDLoc &dl,
   return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO);
 }
 
-SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT,
-                                SDValue Chain, SDValue Ptr, SDValue Val,
-                                const Value *PtrVal, unsigned Alignment,
-                                AtomicOrdering Ordering,
-                                SyncScope::ID SSID) {
-  if (Alignment == 0)  // Ensure that codegen never sees alignment 0
-    Alignment = getEVTAlignment(MemVT);
-
-  MachineFunction &MF = getMachineFunction();
-  // An atomic store does not load. An atomic load does not store.
-  // (An atomicrmw obviously both loads and stores.)
-  // For now, atomics are considered to be volatile always, and they are
-  // chained as such.
-  // FIXME: Volatile isn't really correct; we should keep track of atomic
-  // orderings in the memoperand.
-  auto Flags = MachineMemOperand::MOVolatile;
-  if (Opcode != ISD::ATOMIC_STORE)
-    Flags |= MachineMemOperand::MOLoad;
-  if (Opcode != ISD::ATOMIC_LOAD)
-    Flags |= MachineMemOperand::MOStore;
-
-  MachineMemOperand *MMO =
-    MF.getMachineMemOperand(MachinePointerInfo(PtrVal), Flags,
-                            MemVT.getStoreSize(), Alignment, AAMDNodes(),
-                            nullptr, SSID, Ordering);
-
-  return getAtomic(Opcode, dl, MemVT, Chain, Ptr, Val, MMO);
-}
-
 SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT,
                                 SDValue Chain, SDValue Ptr, SDValue Val,
                                 MachineMemOperand *MMO) {
@@ -6385,6 +6434,8 @@ SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT,
           Opcode == ISD::ATOMIC_LOAD_MAX ||
           Opcode == ISD::ATOMIC_LOAD_UMIN ||
           Opcode == ISD::ATOMIC_LOAD_UMAX ||
+          Opcode == ISD::ATOMIC_LOAD_FADD ||
+          Opcode == ISD::ATOMIC_LOAD_FSUB ||
           Opcode == ISD::ATOMIC_SWAP ||
           Opcode == ISD::ATOMIC_STORE) &&
          "Invalid Atomic Op");
@@ -6967,11 +7018,11 @@ SDValue SelectionDAG::simplifyShift(SDValue X, SDValue Y) {
     return X;
 
   // shift X, C >= bitwidth(X) --> undef
-  // All vector elements must be too big to avoid partial undefs.
+  // All vector elements must be too big (or undef) to avoid partial undefs.
   auto isShiftTooBig = [X](ConstantSDNode *Val) {
-    return Val->getAPIntValue().uge(X.getScalarValueSizeInBits());
+    return !Val || Val->getAPIntValue().uge(X.getScalarValueSizeInBits());
   };
-  if (ISD::matchUnaryPredicate(Y, isShiftTooBig))
+  if (ISD::matchUnaryPredicate(Y, isShiftTooBig, true))
     return getUNDEF(X.getValueType());
 
   return SDValue();
@@ -8442,6 +8493,32 @@ SDValue SelectionDAG::makeEquivalentMemoryOrdering(LoadSDNode *OldLoad,
   return TokenFactor;
 }
 
+SDValue SelectionDAG::getSymbolFunctionGlobalAddress(SDValue Op,
+                                                     Function **OutFunction) {
+  assert(isa<ExternalSymbolSDNode>(Op) && "Node should be an ExternalSymbol");
+
+  auto *Symbol = cast<ExternalSymbolSDNode>(Op)->getSymbol();
+  auto *Module = MF->getFunction().getParent();
+  auto *Function = Module->getFunction(Symbol);
+
+  if (OutFunction != nullptr)
+      *OutFunction = Function;
+
+  if (Function != nullptr) {
+    auto PtrTy = TLI->getPointerTy(getDataLayout(), Function->getAddressSpace());
+    return getGlobalAddress(Function, SDLoc(Op), PtrTy);
+  }
+
+  std::string ErrorStr;
+  raw_string_ostream ErrorFormatter(ErrorStr);
+
+  ErrorFormatter << "Undefined external symbol ";
+  ErrorFormatter << '"' << Symbol << '"';
+  ErrorFormatter.flush();
+
+  report_fatal_error(ErrorStr);
+}
+
 //===----------------------------------------------------------------------===//
 //                              SDNode Class
 //===----------------------------------------------------------------------===//
@@ -8516,9 +8593,9 @@ ConstantFPSDNode *llvm::isConstOrConstSplatFP(SDValue N, bool AllowUndefs) {
   return nullptr;
 }
 
-bool llvm::isNullOrNullSplat(SDValue N) {
+bool llvm::isNullOrNullSplat(SDValue N, bool AllowUndefs) {
   // TODO: may want to use peekThroughBitcast() here.
-  ConstantSDNode *C = isConstOrConstSplat(N);
+  ConstantSDNode *C = isConstOrConstSplat(N, AllowUndefs);
   return C && C->isNullValue();
 }
 
@@ -9160,8 +9237,7 @@ SDNode *SelectionDAG::isConstantFPBuildVectorOrConstantFP(SDValue N) {
 
 void SelectionDAG::createOperands(SDNode *Node, ArrayRef<SDValue> Vals) {
   assert(!Node->OperandList && "Node already has operands");
-  assert(std::numeric_limits<decltype(SDNode::NumOperands)>::max() >
-             Vals.size() &&
+  assert(SDNode::getMaxNumOperands() >= Vals.size() &&
          "too many operands to fit into SDNode");
   SDUse *Ops = OperandRecycler.allocate(
       ArrayRecycler<SDUse>::Capacity::get(Vals.size()), OperandAllocator);
@@ -9181,6 +9257,19 @@ void SelectionDAG::createOperands(SDNode *Node, ArrayRef<SDValue> Vals) {
   checkForCycles(Node);
 }
 
+SDValue SelectionDAG::getTokenFactor(const SDLoc &DL,
+                                     SmallVectorImpl<SDValue> &Vals) {
+  size_t Limit = SDNode::getMaxNumOperands();
+  while (Vals.size() > Limit) {
+    unsigned SliceIdx = Vals.size() - Limit;
+    auto ExtractedTFs = ArrayRef<SDValue>(Vals).slice(SliceIdx, Limit);
+    SDValue NewTF = getNode(ISD::TokenFactor, DL, MVT::Other, ExtractedTFs);
+    Vals.erase(Vals.begin() + SliceIdx, Vals.end());
+    Vals.emplace_back(NewTF);
+  }
+  return getNode(ISD::TokenFactor, DL, MVT::Other, Vals);
+}
+
 #ifndef NDEBUG
 static void checkForCyclesHelper(const SDNode *N,
                                  SmallPtrSetImpl<const SDNode*> &Visited,
diff --git a/lib/CodeGen/SelectionDAG/SelectionDAGAddressAnalysis.cpp b/lib/CodeGen/SelectionDAG/SelectionDAGAddressAnalysis.cpp
index 488bac1a9a80013b17c4fd6eb9ef0a09129f2a3f..5666c1293e968041b06a8ca4fbd008cad29543b4 100644
--- a/lib/CodeGen/SelectionDAG/SelectionDAGAddressAnalysis.cpp
+++ b/lib/CodeGen/SelectionDAG/SelectionDAGAddressAnalysis.cpp
@@ -1,9 +1,8 @@
 //==- llvm/CodeGen/SelectionDAGAddressAnalysis.cpp - DAG Address Analysis --==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -60,17 +59,22 @@ bool BaseIndexOffset::equalBaseIndex(const BaseIndexOffset &Other,
 
     const MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
 
-    // Match non-equal FrameIndexes - If both frame indices are fixed
-    // we know their relative offsets and can compare them. Otherwise
-    // we must be conservative.
+    // Match FrameIndexes.
     if (auto *A = dyn_cast<FrameIndexSDNode>(Base))
-      if (auto *B = dyn_cast<FrameIndexSDNode>(Other.Base))
+      if (auto *B = dyn_cast<FrameIndexSDNode>(Other.Base)) {
+        // Equal FrameIndexes - offsets are directly comparable.
+        if (A->getIndex() == B->getIndex())
+          return true;
+        // Non-equal FrameIndexes - If both frame indices are fixed
+        // we know their relative offsets and can compare them. Otherwise
+        // we must be conservative.
         if (MFI.isFixedObjectIndex(A->getIndex()) &&
             MFI.isFixedObjectIndex(B->getIndex())) {
           Off += MFI.getObjectOffset(B->getIndex()) -
                  MFI.getObjectOffset(A->getIndex());
           return true;
         }
+      }
   }
   return false;
 }
@@ -178,3 +182,21 @@ BaseIndexOffset BaseIndexOffset::match(const LSBaseSDNode *N,
   }
   return BaseIndexOffset(Base, Index, Offset, IsIndexSignExt);
 }
+
+
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+
+LLVM_DUMP_METHOD void BaseIndexOffset::dump() const {
+  print(dbgs());
+}
+
+void BaseIndexOffset::print(raw_ostream& OS) const {
+  OS << "BaseIndexOffset base=[";
+  Base->print(OS);
+  OS << "] index=[";
+  if (Index)
+    Index->print(OS);
+  OS << "] offset=" << Offset;
+}
+
+#endif
diff --git a/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index c42d2491243553ff306423622fdf4f7a34b2ed2c..edb05fb6bcf2096a49e0f9f03fd457ff9688b04b 100644
--- a/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -1,9 +1,8 @@
 //===- SelectionDAGBuilder.cpp - Selection-DAG building -------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -109,6 +108,7 @@
 #include "llvm/Target/TargetIntrinsicInfo.h"
 #include "llvm/Target/TargetMachine.h"
 #include "llvm/Target/TargetOptions.h"
+#include "llvm/Transforms/Utils/Local.h"
 #include <algorithm>
 #include <cassert>
 #include <cstddef>
@@ -1032,19 +1032,7 @@ SDValue SelectionDAGBuilder::getRoot() {
   }
 
   // Otherwise, we have to make a token factor node.
-  // If we have >= 2^16 loads then split across multiple token factors as
-  // there's a 64k limit on the number of SDNode operands.
-  SDValue Root;
-  size_t Limit = (1 << 16) - 1;
-  while (PendingLoads.size() > Limit) {
-    unsigned SliceIdx = PendingLoads.size() - Limit;
-    auto ExtractedTFs = ArrayRef<SDValue>(PendingLoads).slice(SliceIdx, Limit);
-    SDValue NewTF =
-        DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other, ExtractedTFs);
-    PendingLoads.erase(PendingLoads.begin() + SliceIdx, PendingLoads.end());
-    PendingLoads.emplace_back(NewTF);
-  }
-  Root = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other, PendingLoads);
+  SDValue Root = DAG.getTokenFactor(getCurSDLoc(), PendingLoads);
   PendingLoads.clear();
   DAG.setRoot(Root);
   return Root;
@@ -1144,6 +1132,13 @@ void SelectionDAGBuilder::dropDanglingDebugInfo(const DILocalVariable *Variable,
 
   for (auto &DDIMI : DanglingDebugInfoMap) {
     DanglingDebugInfoVector &DDIV = DDIMI.second;
+
+    // If debug info is to be dropped, run it through final checks to see
+    // whether it can be salvaged.
+    for (auto &DDI : DDIV)
+      if (isMatchingDbgValue(DDI))
+        salvageUnresolvedDbgValue(DDI);
+
     DDIV.erase(remove_if(DDIV, isMatchingDbgValue), DDIV.end());
   }
 }
@@ -1169,6 +1164,12 @@ void SelectionDAGBuilder::resolveDanglingDebugInfo(const Value *V,
            "Expected inlined-at fields to agree");
     SDDbgValue *SDV;
     if (Val.getNode()) {
+      // FIXME: I doubt that it is correct to resolve a dangling DbgValue as a
+      // FuncArgumentDbgValue (it would be hoisted to the function entry, and if
+      // we couldn't resolve it directly when examining the DbgValue intrinsic
+      // in the first place we should not be more successful here). Unless we
+      // have some test case that prove this to be correct we should avoid
+      // calling EmitFuncArgumentDbgValue here.
       if (!EmitFuncArgumentDbgValue(V, Variable, Expr, dl, false, Val)) {
         LLVM_DEBUG(dbgs() << "Resolve dangling debug info [order="
                           << DbgSDNodeOrder << "] for:\n  " << *DI << "\n");
@@ -1186,12 +1187,173 @@ void SelectionDAGBuilder::resolveDanglingDebugInfo(const Value *V,
       } else
         LLVM_DEBUG(dbgs() << "Resolved dangling debug info for " << *DI
                           << "in EmitFuncArgumentDbgValue\n");
-    } else
+    } else {
       LLVM_DEBUG(dbgs() << "Dropping debug info for " << *DI << "\n");
+      auto Undef =
+          UndefValue::get(DDI.getDI()->getVariableLocation()->getType());
+      auto SDV =
+          DAG.getConstantDbgValue(Variable, Expr, Undef, dl, DbgSDNodeOrder);
+      DAG.AddDbgValue(SDV, nullptr, false);
+    }
   }
   DDIV.clear();
 }
 
+void SelectionDAGBuilder::salvageUnresolvedDbgValue(DanglingDebugInfo &DDI) {
+  Value *V = DDI.getDI()->getValue();
+  DILocalVariable *Var = DDI.getDI()->getVariable();
+  DIExpression *Expr = DDI.getDI()->getExpression();
+  DebugLoc DL = DDI.getdl();
+  DebugLoc InstDL = DDI.getDI()->getDebugLoc();
+  unsigned SDOrder = DDI.getSDNodeOrder();
+
+  // Currently we consider only dbg.value intrinsics -- we tell the salvager
+  // that DW_OP_stack_value is desired.
+  assert(isa<DbgValueInst>(DDI.getDI()));
+  bool StackValue = true;
+
+  // Can this Value can be encoded without any further work?
+  if (handleDebugValue(V, Var, Expr, DL, InstDL, SDOrder))
+    return;
+
+  // Attempt to salvage back through as many instructions as possible. Bail if
+  // a non-instruction is seen, such as a constant expression or global
+  // variable. FIXME: Further work could recover those too.
+  while (isa<Instruction>(V)) {
+    Instruction &VAsInst = *cast<Instruction>(V);
+    DIExpression *NewExpr = salvageDebugInfoImpl(VAsInst, Expr, StackValue);
+
+    // If we cannot salvage any further, and haven't yet found a suitable debug
+    // expression, bail out.
+    if (!NewExpr)
+      break;
+
+    // New value and expr now represent this debuginfo.
+    V = VAsInst.getOperand(0);
+    Expr = NewExpr;
+
+    // Some kind of simplification occurred: check whether the operand of the
+    // salvaged debug expression can be encoded in this DAG.
+    if (handleDebugValue(V, Var, Expr, DL, InstDL, SDOrder)) {
+      LLVM_DEBUG(dbgs() << "Salvaged debug location info for:\n  "
+                        << DDI.getDI() << "\nBy stripping back to:\n  " << V);
+      return;
+    }
+  }
+
+  // This was the final opportunity to salvage this debug information, and it
+  // couldn't be done. Place an undef DBG_VALUE at this location to terminate
+  // any earlier variable location.
+  auto Undef = UndefValue::get(DDI.getDI()->getVariableLocation()->getType());
+  auto SDV = DAG.getConstantDbgValue(Var, Expr, Undef, DL, SDNodeOrder);
+  DAG.AddDbgValue(SDV, nullptr, false);
+
+  LLVM_DEBUG(dbgs() << "Dropping debug value info for:\n  " << DDI.getDI()
+                    << "\n");
+  LLVM_DEBUG(dbgs() << "  Last seen at:\n    " << *DDI.getDI()->getOperand(0)
+                    << "\n");
+}
+
+bool SelectionDAGBuilder::handleDebugValue(const Value *V, DILocalVariable *Var,
+                                           DIExpression *Expr, DebugLoc dl,
+                                           DebugLoc InstDL, unsigned Order) {
+  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
+  SDDbgValue *SDV;
+  if (isa<ConstantInt>(V) || isa<ConstantFP>(V) || isa<UndefValue>(V) ||
+      isa<ConstantPointerNull>(V)) {
+    SDV = DAG.getConstantDbgValue(Var, Expr, V, dl, SDNodeOrder);
+    DAG.AddDbgValue(SDV, nullptr, false);
+    return true;
+  }
+
+  // If the Value is a frame index, we can create a FrameIndex debug value
+  // without relying on the DAG at all.
+  if (const AllocaInst *AI = dyn_cast<AllocaInst>(V)) {
+    auto SI = FuncInfo.StaticAllocaMap.find(AI);
+    if (SI != FuncInfo.StaticAllocaMap.end()) {
+      auto SDV =
+          DAG.getFrameIndexDbgValue(Var, Expr, SI->second,
+                                    /*IsIndirect*/ false, dl, SDNodeOrder);
+      // Do not attach the SDNodeDbgValue to an SDNode: this variable location
+      // is still available even if the SDNode gets optimized out.
+      DAG.AddDbgValue(SDV, nullptr, false);
+      return true;
+    }
+  }
+
+  // Do not use getValue() in here; we don't want to generate code at
+  // this point if it hasn't been done yet.
+  SDValue N = NodeMap[V];
+  if (!N.getNode() && isa<Argument>(V)) // Check unused arguments map.
+    N = UnusedArgNodeMap[V];
+  if (N.getNode()) {
+    if (EmitFuncArgumentDbgValue(V, Var, Expr, dl, false, N))
+      return true;
+    SDV = getDbgValue(N, Var, Expr, dl, SDNodeOrder);
+    DAG.AddDbgValue(SDV, N.getNode(), false);
+    return true;
+  }
+
+  // Special rules apply for the first dbg.values of parameter variables in a
+  // function. Identify them by the fact they reference Argument Values, that
+  // they're parameters, and they are parameters of the current function. We
+  // need to let them dangle until they get an SDNode.
+  bool IsParamOfFunc = isa<Argument>(V) && Var->isParameter() &&
+                       !InstDL.getInlinedAt();
+  if (!IsParamOfFunc) {
+    // The value is not used in this block yet (or it would have an SDNode).
+    // We still want the value to appear for the user if possible -- if it has
+    // an associated VReg, we can refer to that instead.
+    auto VMI = FuncInfo.ValueMap.find(V);
+    if (VMI != FuncInfo.ValueMap.end()) {
+      unsigned Reg = VMI->second;
+      // If this is a PHI node, it may be split up into several MI PHI nodes
+      // (in FunctionLoweringInfo::set).
+      RegsForValue RFV(V->getContext(), TLI, DAG.getDataLayout(), Reg,
+                       V->getType(), None);
+      if (RFV.occupiesMultipleRegs()) {
+        unsigned Offset = 0;
+        unsigned BitsToDescribe = 0;
+        if (auto VarSize = Var->getSizeInBits())
+          BitsToDescribe = *VarSize;
+        if (auto Fragment = Expr->getFragmentInfo())
+          BitsToDescribe = Fragment->SizeInBits;
+        for (auto RegAndSize : RFV.getRegsAndSizes()) {
+          unsigned RegisterSize = RegAndSize.second;
+          // Bail out if all bits are described already.
+          if (Offset >= BitsToDescribe)
+            break;
+          unsigned FragmentSize = (Offset + RegisterSize > BitsToDescribe)
+              ? BitsToDescribe - Offset
+              : RegisterSize;
+          auto FragmentExpr = DIExpression::createFragmentExpression(
+              Expr, Offset, FragmentSize);
+          if (!FragmentExpr)
+              continue;
+          SDV = DAG.getVRegDbgValue(Var, *FragmentExpr, RegAndSize.first,
+                                    false, dl, SDNodeOrder);
+          DAG.AddDbgValue(SDV, nullptr, false);
+          Offset += RegisterSize;
+        }
+      } else {
+        SDV = DAG.getVRegDbgValue(Var, Expr, Reg, false, dl, SDNodeOrder);
+        DAG.AddDbgValue(SDV, nullptr, false);
+      }
+      return true;
+    }
+  }
+
+  return false;
+}
+
+void SelectionDAGBuilder::resolveOrClearDbgInfo() {
+  // Try to fixup any remaining dangling debug info -- and drop it if we can't.
+  for (auto &Pair : DanglingDebugInfoMap)
+    for (auto &DDI : Pair.getSecond())
+      salvageUnresolvedDbgValue(DDI);
+  clearDanglingDebugInfo();
+}
+
 /// getCopyFromRegs - If there was virtual register allocated for the value V
 /// emit CopyFromReg of the specified type Ty. Return empty SDValue() otherwise.
 SDValue SelectionDAGBuilder::getCopyFromRegs(const Value *V, Type *Ty) {
@@ -1469,6 +1631,36 @@ void SelectionDAGBuilder::visitCleanupPad(const CleanupPadInst &CPI) {
   }
 }
 
+// For wasm, there's alwyas a single catch pad attached to a catchswitch, and
+// the control flow always stops at the single catch pad, as it does for a
+// cleanup pad. In case the exception caught is not of the types the catch pad
+// catches, it will be rethrown by a rethrow.
+static void findWasmUnwindDestinations(
+    FunctionLoweringInfo &FuncInfo, const BasicBlock *EHPadBB,
+    BranchProbability Prob,
+    SmallVectorImpl<std::pair<MachineBasicBlock *, BranchProbability>>
+        &UnwindDests) {
+  while (EHPadBB) {
+    const Instruction *Pad = EHPadBB->getFirstNonPHI();
+    if (isa<CleanupPadInst>(Pad)) {
+      // Stop on cleanup pads.
+      UnwindDests.emplace_back(FuncInfo.MBBMap[EHPadBB], Prob);
+      UnwindDests.back().first->setIsEHScopeEntry();
+      break;
+    } else if (auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Pad)) {
+      // Add the catchpad handlers to the possible destinations. We don't
+      // continue to the unwind destination of the catchswitch for wasm.
+      for (const BasicBlock *CatchPadBB : CatchSwitch->handlers()) {
+        UnwindDests.emplace_back(FuncInfo.MBBMap[CatchPadBB], Prob);
+        UnwindDests.back().first->setIsEHScopeEntry();
+      }
+      break;
+    } else {
+      continue;
+    }
+  }
+}
+
 /// When an invoke or a cleanupret unwinds to the next EH pad, there are
 /// many places it could ultimately go. In the IR, we have a single unwind
 /// destination, but in the machine CFG, we enumerate all the possible blocks.
@@ -1489,6 +1681,11 @@ static void findUnwindDestinations(
   bool IsWasmCXX = Personality == EHPersonality::Wasm_CXX;
   bool IsSEH = isAsynchronousEHPersonality(Personality);
 
+  if (IsWasmCXX) {
+    findWasmUnwindDestinations(FuncInfo, EHPadBB, Prob, UnwindDests);
+    return;
+  }
+
   while (EHPadBB) {
     const Instruction *Pad = EHPadBB->getFirstNonPHI();
     BasicBlock *NewEHPadBB = nullptr;
@@ -1501,8 +1698,7 @@ static void findUnwindDestinations(
       // personalities.
       UnwindDests.emplace_back(FuncInfo.MBBMap[EHPadBB], Prob);
       UnwindDests.back().first->setIsEHScopeEntry();
-      if (!IsWasmCXX)
-        UnwindDests.back().first->setIsEHFuncletEntry();
+      UnwindDests.back().first->setIsEHFuncletEntry();
       break;
     } else if (auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Pad)) {
       // Add the catchpad handlers to the possible destinations.
@@ -2262,27 +2458,26 @@ void SelectionDAGBuilder::visitSPDescriptorParent(StackProtectorDescriptor &SPD,
     GuardVal = TLI.emitStackGuardXorFP(DAG, GuardVal, dl);
 
   // Retrieve guard check function, nullptr if instrumentation is inlined.
-  if (const Value *GuardCheck = TLI.getSSPStackGuardCheck(M)) {
+  if (const Function *GuardCheckFn = TLI.getSSPStackGuardCheck(M)) {
     // The target provides a guard check function to validate the guard value.
     // Generate a call to that function with the content of the guard slot as
     // argument.
-    auto *Fn = cast<Function>(GuardCheck);
-    FunctionType *FnTy = Fn->getFunctionType();
+    FunctionType *FnTy = GuardCheckFn->getFunctionType();
     assert(FnTy->getNumParams() == 1 && "Invalid function signature");
 
     TargetLowering::ArgListTy Args;
     TargetLowering::ArgListEntry Entry;
     Entry.Node = GuardVal;
     Entry.Ty = FnTy->getParamType(0);
-    if (Fn->hasAttribute(1, Attribute::AttrKind::InReg))
+    if (GuardCheckFn->hasAttribute(1, Attribute::AttrKind::InReg))
       Entry.IsInReg = true;
     Args.push_back(Entry);
 
     TargetLowering::CallLoweringInfo CLI(DAG);
     CLI.setDebugLoc(getCurSDLoc())
-      .setChain(DAG.getEntryNode())
-      .setCallee(Fn->getCallingConv(), FnTy->getReturnType(),
-                 getValue(GuardCheck), std::move(Args));
+        .setChain(DAG.getEntryNode())
+        .setCallee(GuardCheckFn->getCallingConv(), FnTy->getReturnType(),
+                   getValue(GuardCheckFn), std::move(Args));
 
     std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);
     DAG.setRoot(Result.second);
@@ -2528,6 +2723,35 @@ void SelectionDAGBuilder::visitInvoke(const InvokeInst &I) {
   InvokeMBB->normalizeSuccProbs();
 
   // Drop into normal successor.
+  DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other, getControlRoot(),
+                          DAG.getBasicBlock(Return)));
+}
+
+void SelectionDAGBuilder::visitCallBr(const CallBrInst &I) {
+  MachineBasicBlock *CallBrMBB = FuncInfo.MBB;
+
+  // Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't
+  // have to do anything here to lower funclet bundles.
+  assert(!I.hasOperandBundlesOtherThan(
+             {LLVMContext::OB_deopt, LLVMContext::OB_funclet}) &&
+         "Cannot lower callbrs with arbitrary operand bundles yet!");
+
+  assert(isa<InlineAsm>(I.getCalledValue()) &&
+         "Only know how to handle inlineasm callbr");
+  visitInlineAsm(&I);
+
+  // Retrieve successors.
+  MachineBasicBlock *Return = FuncInfo.MBBMap[I.getDefaultDest()];
+
+  // Update successor info.
+  addSuccessorWithProb(CallBrMBB, Return);
+  for (unsigned i = 0, e = I.getNumIndirectDests(); i < e; ++i) {
+    MachineBasicBlock *Target = FuncInfo.MBBMap[I.getIndirectDest(i)];
+    addSuccessorWithProb(CallBrMBB, Target);
+  }
+  CallBrMBB->normalizeSuccProbs();
+
+  // Drop into default successor.
   DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(),
                           MVT::Other, getControlRoot(),
                           DAG.getBasicBlock(Return)));
@@ -3694,8 +3918,11 @@ void SelectionDAGBuilder::visitLoad(const LoadInst &I) {
   if (isVolatile || NumValues > MaxParallelChains)
     // Serialize volatile loads with other side effects.
     Root = getRoot();
-  else if (AA && AA->pointsToConstantMemory(MemoryLocation(
-               SV, DAG.getDataLayout().getTypeStoreSize(Ty), AAInfo))) {
+  else if (AA &&
+           AA->pointsToConstantMemory(MemoryLocation(
+               SV,
+               LocationSize::precise(DAG.getDataLayout().getTypeStoreSize(Ty)),
+               AAInfo))) {
     // Do not serialize (non-volatile) loads of constant memory with anything.
     Root = DAG.getEntryNode();
     ConstantMemory = true;
@@ -3806,9 +4033,12 @@ void SelectionDAGBuilder::visitLoadFromSwiftError(const LoadInst &I) {
   Type *Ty = I.getType();
   AAMDNodes AAInfo;
   I.getAAMetadata(AAInfo);
-  assert((!AA || !AA->pointsToConstantMemory(MemoryLocation(
-             SV, DAG.getDataLayout().getTypeStoreSize(Ty), AAInfo))) &&
-         "load_from_swift_error should not be constant memory");
+  assert(
+      (!AA ||
+       !AA->pointsToConstantMemory(MemoryLocation(
+           SV, LocationSize::precise(DAG.getDataLayout().getTypeStoreSize(Ty)),
+           AAInfo))) &&
+      "load_from_swift_error should not be constant memory");
 
   SmallVector<EVT, 4> ValueVTs;
   SmallVector<uint64_t, 4> Offsets;
@@ -4095,8 +4325,12 @@ void SelectionDAGBuilder::visitMaskedLoad(const CallInst &I, bool IsExpanding) {
   const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range);
 
   // Do not serialize masked loads of constant memory with anything.
-  bool AddToChain = !AA || !AA->pointsToConstantMemory(MemoryLocation(
-      PtrOperand, DAG.getDataLayout().getTypeStoreSize(I.getType()), AAInfo));
+  bool AddToChain =
+      !AA || !AA->pointsToConstantMemory(MemoryLocation(
+                 PtrOperand,
+                 LocationSize::precise(
+                     DAG.getDataLayout().getTypeStoreSize(I.getType())),
+                 AAInfo));
   SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode();
 
   MachineMemOperand *MMO =
@@ -4137,10 +4371,12 @@ void SelectionDAGBuilder::visitMaskedGather(const CallInst &I) {
   const Value *BasePtr = Ptr;
   bool UniformBase = getUniformBase(BasePtr, Base, Index, Scale, this);
   bool ConstantMemory = false;
-  if (UniformBase &&
-      AA && AA->pointsToConstantMemory(MemoryLocation(
-          BasePtr, DAG.getDataLayout().getTypeStoreSize(I.getType()),
-          AAInfo))) {
+  if (UniformBase && AA &&
+      AA->pointsToConstantMemory(
+          MemoryLocation(BasePtr,
+                         LocationSize::precise(
+                             DAG.getDataLayout().getTypeStoreSize(I.getType())),
+                         AAInfo))) {
     // Do not serialize (non-volatile) loads of constant memory with anything.
     Root = DAG.getEntryNode();
     ConstantMemory = true;
@@ -4205,20 +4441,34 @@ void SelectionDAGBuilder::visitAtomicRMW(const AtomicRMWInst &I) {
   case AtomicRMWInst::Min:  NT = ISD::ATOMIC_LOAD_MIN; break;
   case AtomicRMWInst::UMax: NT = ISD::ATOMIC_LOAD_UMAX; break;
   case AtomicRMWInst::UMin: NT = ISD::ATOMIC_LOAD_UMIN; break;
+  case AtomicRMWInst::FAdd: NT = ISD::ATOMIC_LOAD_FADD; break;
+  case AtomicRMWInst::FSub: NT = ISD::ATOMIC_LOAD_FSUB; break;
   }
-  AtomicOrdering Order = I.getOrdering();
+  AtomicOrdering Ordering = I.getOrdering();
   SyncScope::ID SSID = I.getSyncScopeID();
 
   SDValue InChain = getRoot();
 
+  auto MemVT = getValue(I.getValOperand()).getSimpleValueType();
+  auto Alignment = DAG.getEVTAlignment(MemVT);
+
+  // For now, atomics are considered to be volatile always, and they are
+  // chained as such.
+  // FIXME: Volatile isn't really correct; we should keep track of atomic
+  // orderings in the memoperand.
+  auto Flags = MachineMemOperand::MOVolatile |
+    MachineMemOperand::MOLoad |  MachineMemOperand::MOStore;
+
+  MachineFunction &MF = DAG.getMachineFunction();
+  MachineMemOperand *MMO =
+    MF.getMachineMemOperand(MachinePointerInfo(I.getPointerOperand()), Flags,
+                            MemVT.getStoreSize(), Alignment, AAMDNodes(),
+                            nullptr, SSID, Ordering);
+
   SDValue L =
-    DAG.getAtomic(NT, dl,
-                  getValue(I.getValOperand()).getSimpleValueType(),
-                  InChain,
-                  getValue(I.getPointerOperand()),
-                  getValue(I.getValOperand()),
-                  I.getPointerOperand(),
-                  /* Alignment=*/ 0, Order, SSID);
+    DAG.getAtomic(NT, dl, MemVT, InChain,
+                  getValue(I.getPointerOperand()), getValue(I.getValOperand()),
+                  MMO);
 
   SDValue OutChain = L.getValue(1);
 
@@ -4276,7 +4526,7 @@ void SelectionDAGBuilder::visitAtomicLoad(const LoadInst &I) {
 void SelectionDAGBuilder::visitAtomicStore(const StoreInst &I) {
   SDLoc dl = getCurSDLoc();
 
-  AtomicOrdering Order = I.getOrdering();
+  AtomicOrdering Ordering = I.getOrdering();
   SyncScope::ID SSID = I.getSyncScopeID();
 
   SDValue InChain = getRoot();
@@ -4288,13 +4538,22 @@ void SelectionDAGBuilder::visitAtomicStore(const StoreInst &I) {
   if (I.getAlignment() < VT.getStoreSize())
     report_fatal_error("Cannot generate unaligned atomic store");
 
+  // For now, atomics are considered to be volatile always, and they are
+  // chained as such.
+  // FIXME: Volatile isn't really correct; we should keep track of atomic
+  // orderings in the memoperand.
+  auto Flags = MachineMemOperand::MOVolatile |  MachineMemOperand::MOStore;
+
+  MachineFunction &MF = DAG.getMachineFunction();
+  MachineMemOperand *MMO =
+    MF.getMachineMemOperand(MachinePointerInfo(I.getPointerOperand()), Flags,
+                            VT.getStoreSize(), I.getAlignment(), AAMDNodes(),
+                            nullptr, SSID, Ordering);
   SDValue OutChain =
-    DAG.getAtomic(ISD::ATOMIC_STORE, dl, VT,
-                  InChain,
-                  getValue(I.getPointerOperand()),
-                  getValue(I.getValueOperand()),
-                  I.getPointerOperand(), I.getAlignment(),
-                  Order, SSID);
+    DAG.getAtomic(ISD::ATOMIC_STORE, dl, VT, InChain,
+              getValue(I.getPointerOperand()), getValue(I.getValueOperand()),
+              MMO);
+
 
   DAG.setRoot(OutChain);
 }
@@ -4940,6 +5199,71 @@ bool SelectionDAGBuilder::EmitFuncArgumentDbgValue(
   if (!Arg)
     return false;
 
+  if (!IsDbgDeclare) {
+    // ArgDbgValues are hoisted to the beginning of the entry block. So we
+    // should only emit as ArgDbgValue if the dbg.value intrinsic is found in
+    // the entry block.
+    bool IsInEntryBlock = FuncInfo.MBB == &FuncInfo.MF->front();
+    if (!IsInEntryBlock)
+      return false;
+
+    // ArgDbgValues are hoisted to the beginning of the entry block.  So we
+    // should only emit as ArgDbgValue if the dbg.value intrinsic describes a
+    // variable that also is a param.
+    //
+    // Although, if we are at the top of the entry block already, we can still
+    // emit using ArgDbgValue. This might catch some situations when the
+    // dbg.value refers to an argument that isn't used in the entry block, so
+    // any CopyToReg node would be optimized out and the only way to express
+    // this DBG_VALUE is by using the physical reg (or FI) as done in this
+    // method.  ArgDbgValues are hoisted to the beginning of the entry block. So
+    // we should only emit as ArgDbgValue if the Variable is an argument to the
+    // current function, and the dbg.value intrinsic is found in the entry
+    // block.
+    bool VariableIsFunctionInputArg = Variable->isParameter() &&
+        !DL->getInlinedAt();
+    bool IsInPrologue = SDNodeOrder == LowestSDNodeOrder;
+    if (!IsInPrologue && !VariableIsFunctionInputArg)
+      return false;
+
+    // Here we assume that a function argument on IR level only can be used to
+    // describe one input parameter on source level. If we for example have
+    // source code like this
+    //
+    //    struct A { long x, y; };
+    //    void foo(struct A a, long b) {
+    //      ...
+    //      b = a.x;
+    //      ...
+    //    }
+    //
+    // and IR like this
+    //
+    //  define void @foo(i32 %a1, i32 %a2, i32 %b)  {
+    //  entry:
+    //    call void @llvm.dbg.value(metadata i32 %a1, "a", DW_OP_LLVM_fragment
+    //    call void @llvm.dbg.value(metadata i32 %a2, "a", DW_OP_LLVM_fragment
+    //    call void @llvm.dbg.value(metadata i32 %b, "b",
+    //    ...
+    //    call void @llvm.dbg.value(metadata i32 %a1, "b"
+    //    ...
+    //
+    // then the last dbg.value is describing a parameter "b" using a value that
+    // is an argument. But since we already has used %a1 to describe a parameter
+    // we should not handle that last dbg.value here (that would result in an
+    // incorrect hoisting of the DBG_VALUE to the function entry).
+    // Notice that we allow one dbg.value per IR level argument, to accomodate
+    // for the situation with fragments above.
+    if (VariableIsFunctionInputArg) {
+      unsigned ArgNo = Arg->getArgNo();
+      if (ArgNo >= FuncInfo.DescribedArgs.size())
+        FuncInfo.DescribedArgs.resize(ArgNo + 1, false);
+      else if (!IsInPrologue && FuncInfo.DescribedArgs.test(ArgNo))
+        return false;
+      FuncInfo.DescribedArgs.set(ArgNo);
+    }
+  }
+
   MachineFunction &MF = DAG.getMachineFunction();
   const TargetInstrInfo *TII = DAG.getSubtarget().getInstrInfo();
 
@@ -5043,6 +5367,17 @@ SDDbgValue *SelectionDAGBuilder::getDbgValue(SDValue N,
 #  define setjmp_undefined_for_msvc
 #endif
 
+static unsigned FixedPointIntrinsicToOpcode(unsigned Intrinsic) {
+  switch (Intrinsic) {
+  case Intrinsic::smul_fix:
+    return ISD::SMULFIX;
+  case Intrinsic::umul_fix:
+    return ISD::UMULFIX;
+  default:
+    llvm_unreachable("Unhandled fixed point intrinsic");
+  }
+}
+
 /// Lower the call to the specified intrinsic function. If we want to emit this
 /// as a call to a named external function, return the name. Otherwise, lower it
 /// and return null.
@@ -5311,85 +5646,16 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) {
     if (!V)
       return nullptr;
 
-    SDDbgValue *SDV;
-    if (isa<ConstantInt>(V) || isa<ConstantFP>(V) || isa<UndefValue>(V) ||
-        isa<ConstantPointerNull>(V)) {
-      SDV = DAG.getConstantDbgValue(Variable, Expression, V, dl, SDNodeOrder);
-      DAG.AddDbgValue(SDV, nullptr, false);
+    if (handleDebugValue(V, Variable, Expression, DI.getDebugLoc(), dl,
+        SDNodeOrder))
       return nullptr;
-    }
-
-    // Do not use getValue() in here; we don't want to generate code at
-    // this point if it hasn't been done yet.
-    SDValue N = NodeMap[V];
-    if (!N.getNode() && isa<Argument>(V)) // Check unused arguments map.
-      N = UnusedArgNodeMap[V];
-    if (N.getNode()) {
-      if (EmitFuncArgumentDbgValue(V, Variable, Expression, dl, false, N))
-        return nullptr;
-      SDV = getDbgValue(N, Variable, Expression, dl, SDNodeOrder);
-      DAG.AddDbgValue(SDV, N.getNode(), false);
-      return nullptr;
-    }
-
-    // PHI nodes have already been selected, so we should know which VReg that
-    // is assigns to already.
-    if (isa<PHINode>(V)) {
-      auto VMI = FuncInfo.ValueMap.find(V);
-      if (VMI != FuncInfo.ValueMap.end()) {
-        unsigned Reg = VMI->second;
-        // The PHI node may be split up into several MI PHI nodes (in
-        // FunctionLoweringInfo::set).
-        RegsForValue RFV(V->getContext(), TLI, DAG.getDataLayout(), Reg,
-                         V->getType(), None);
-        if (RFV.occupiesMultipleRegs()) {
-          unsigned Offset = 0;
-          unsigned BitsToDescribe = 0;
-          if (auto VarSize = Variable->getSizeInBits())
-            BitsToDescribe = *VarSize;
-          if (auto Fragment = Expression->getFragmentInfo())
-            BitsToDescribe = Fragment->SizeInBits;
-          for (auto RegAndSize : RFV.getRegsAndSizes()) {
-            unsigned RegisterSize = RegAndSize.second;
-            // Bail out if all bits are described already.
-            if (Offset >= BitsToDescribe)
-              break;
-            unsigned FragmentSize = (Offset + RegisterSize > BitsToDescribe)
-                ? BitsToDescribe - Offset
-                : RegisterSize;
-            auto FragmentExpr = DIExpression::createFragmentExpression(
-                Expression, Offset, FragmentSize);
-            if (!FragmentExpr)
-                continue;
-            SDV = DAG.getVRegDbgValue(Variable, *FragmentExpr, RegAndSize.first,
-                                      false, dl, SDNodeOrder);
-            DAG.AddDbgValue(SDV, nullptr, false);
-            Offset += RegisterSize;
-          }
-        } else {
-          SDV = DAG.getVRegDbgValue(Variable, Expression, Reg, false, dl,
-                                    SDNodeOrder);
-          DAG.AddDbgValue(SDV, nullptr, false);
-        }
-        return nullptr;
-      }
-    }
 
-    // TODO: When we get here we will either drop the dbg.value completely, or
-    // we try to move it forward by letting it dangle for awhile. So we should
-    // probably add an extra DbgValue to the DAG here, with a reference to
-    // "noreg", to indicate that we have lost the debug location for the
-    // variable.
+    // TODO: Dangling debug info will eventually either be resolved or produce
+    // an Undef DBG_VALUE. However in the resolution case, a gap may appear
+    // between the original dbg.value location and its resolved DBG_VALUE, which
+    // we should ideally fill with an extra Undef DBG_VALUE.
 
-    if (!V->use_empty() ) {
-      // Do not call getValue(V) yet, as we don't want to generate code.
-      // Remember it for later.
-      DanglingDebugInfoMap[V].emplace_back(&DI, dl, SDNodeOrder);
-      return nullptr;
-    }
-
-    LLVM_DEBUG(dbgs() << "Dropping debug location info for:\n  " << DI << "\n");
-    LLVM_DEBUG(dbgs() << "  Last seen at:\n    " << *V << "\n");
+    DanglingDebugInfoMap[V].emplace_back(&DI, dl, SDNodeOrder);
     return nullptr;
   }
 
@@ -5762,17 +6028,15 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) {
     // avoid the select that is necessary in the general case to filter out
     // the 0-shift possibility that leads to UB.
     if (X == Y && isPowerOf2_32(VT.getScalarSizeInBits())) {
-      // TODO: This should also be done if the operation is custom, but we have
-      // to make sure targets are handling the modulo shift amount as expected.
       auto RotateOpcode = IsFSHL ? ISD::ROTL : ISD::ROTR;
-      if (TLI.isOperationLegal(RotateOpcode, VT)) {
+      if (TLI.isOperationLegalOrCustom(RotateOpcode, VT)) {
         setValue(&I, DAG.getNode(RotateOpcode, sdl, VT, X, Z));
         return nullptr;
       }
 
       // Some targets only rotate one way. Try the opposite direction.
       RotateOpcode = IsFSHL ? ISD::ROTR : ISD::ROTL;
-      if (TLI.isOperationLegal(RotateOpcode, VT)) {
+      if (TLI.isOperationLegalOrCustom(RotateOpcode, VT)) {
         // Negate the shift amount because it is safe to ignore the high bits.
         SDValue NegShAmt = DAG.getNode(ISD::SUB, sdl, VT, Zero, Z);
         setValue(&I, DAG.getNode(RotateOpcode, sdl, VT, X, NegShAmt));
@@ -5832,12 +6096,13 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) {
     setValue(&I, DAG.getNode(ISD::USUBSAT, sdl, Op1.getValueType(), Op1, Op2));
     return nullptr;
   }
-  case Intrinsic::smul_fix: {
+  case Intrinsic::smul_fix:
+  case Intrinsic::umul_fix: {
     SDValue Op1 = getValue(I.getArgOperand(0));
     SDValue Op2 = getValue(I.getArgOperand(1));
     SDValue Op3 = getValue(I.getArgOperand(2));
-    setValue(&I,
-             DAG.getNode(ISD::SMULFIX, sdl, Op1.getValueType(), Op1, Op2, Op3));
+    setValue(&I, DAG.getNode(FixedPointIntrinsicToOpcode(Intrinsic), sdl,
+                             Op1.getValueType(), Op1, Op2, Op3));
     return nullptr;
   }
   case Intrinsic::stacksave: {
@@ -6053,7 +6318,13 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) {
     SDValue Op1 = getValue(I.getArgOperand(0));
     SDValue Op2 = getValue(I.getArgOperand(1));
 
-    SDVTList VTs = DAG.getVTList(Op1.getValueType(), MVT::i1);
+    EVT ResultVT = Op1.getValueType();
+    EVT OverflowVT = MVT::i1;
+    if (ResultVT.isVector())
+      OverflowVT = EVT::getVectorVT(
+          *Context, OverflowVT, ResultVT.getVectorNumElements());
+
+    SDVTList VTs = DAG.getVTList(ResultVT, OverflowVT);
     setValue(&I, DAG.getNode(Op, sdl, VTs, Op1, Op2));
     return nullptr;
   }
@@ -6985,9 +7256,6 @@ void SelectionDAGBuilder::visitCall(const CallInst &I) {
     return;
   }
 
-  MachineModuleInfo &MMI = DAG.getMachineFunction().getMMI();
-  computeUsesVAFloatArgument(I, MMI);
-
   const char *RenameFn = nullptr;
   if (Function *F = I.getCalledFunction()) {
     if (F->isDeclaration()) {
@@ -7333,22 +7601,35 @@ static SDValue getAddressForMemoryInput(SDValue Chain, const SDLoc &Location,
 ///
 ///   OpInfo describes the operand
 ///   RefOpInfo describes the matching operand if any, the operand otherwise
-static void GetRegistersForValue(SelectionDAG &DAG, const TargetLowering &TLI,
-                                 const SDLoc &DL, SDISelAsmOperandInfo &OpInfo,
+static void GetRegistersForValue(SelectionDAG &DAG, const SDLoc &DL,
+                                 SDISelAsmOperandInfo &OpInfo,
                                  SDISelAsmOperandInfo &RefOpInfo) {
   LLVMContext &Context = *DAG.getContext();
+  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
 
   MachineFunction &MF = DAG.getMachineFunction();
   SmallVector<unsigned, 4> Regs;
   const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo();
 
+  // No work to do for memory operations.
+  if (OpInfo.ConstraintType == TargetLowering::C_Memory)
+    return;
+
   // If this is a constraint for a single physreg, or a constraint for a
   // register class, find it.
-  std::pair<unsigned, const TargetRegisterClass *> PhysReg =
-      TLI.getRegForInlineAsmConstraint(&TRI, RefOpInfo.ConstraintCode,
-                                       RefOpInfo.ConstraintVT);
+  unsigned AssignedReg;
+  const TargetRegisterClass *RC;
+  std::tie(AssignedReg, RC) = TLI.getRegForInlineAsmConstraint(
+      &TRI, RefOpInfo.ConstraintCode, RefOpInfo.ConstraintVT);
+  // RC is unset only on failure. Return immediately.
+  if (!RC)
+    return;
+
+  // Get the actual register value type.  This is important, because the user
+  // may have asked for (e.g.) the AX register in i32 type.  We need to
+  // remember that AX is actually i16 to get the right extension.
+  const MVT RegVT = *TRI.legalclasstypes_begin(*RC);
 
-  unsigned NumRegs = 1;
   if (OpInfo.ConstraintVT != MVT::Other) {
     // If this is an FP operand in an integer register (or visa versa), or more
     // generally if the operand value disagrees with the register class we plan
@@ -7358,13 +7639,11 @@ static void GetRegistersForValue(SelectionDAG &DAG, const TargetLowering &TLI,
     // Bitcast for output value is done at the end of visitInlineAsm().
     if ((OpInfo.Type == InlineAsm::isOutput ||
          OpInfo.Type == InlineAsm::isInput) &&
-        PhysReg.second &&
-        !TRI.isTypeLegalForClass(*PhysReg.second, OpInfo.ConstraintVT)) {
+        !TRI.isTypeLegalForClass(*RC, OpInfo.ConstraintVT)) {
       // Try to convert to the first EVT that the reg class contains.  If the
       // types are identical size, use a bitcast to convert (e.g. two differing
       // vector types).  Note: output bitcast is done at the end of
       // visitInlineAsm().
-      MVT RegVT = *TRI.legalclasstypes_begin(*PhysReg.second);
       if (RegVT.getSizeInBits() == OpInfo.ConstraintVT.getSizeInBits()) {
         // Exclude indirect inputs while they are unsupported because the code
         // to perform the load is missing and thus OpInfo.CallOperand still
@@ -7377,15 +7656,13 @@ static void GetRegistersForValue(SelectionDAG &DAG, const TargetLowering &TLI,
         // use the corresponding integer type. This turns an f64 value into
         // i64, which can be passed with two i32 values on a 32-bit machine.
       } else if (RegVT.isInteger() && OpInfo.ConstraintVT.isFloatingPoint()) {
-        RegVT = MVT::getIntegerVT(OpInfo.ConstraintVT.getSizeInBits());
+        MVT VT = MVT::getIntegerVT(OpInfo.ConstraintVT.getSizeInBits());
         if (OpInfo.Type == InlineAsm::isInput)
           OpInfo.CallOperand =
-              DAG.getNode(ISD::BITCAST, DL, RegVT, OpInfo.CallOperand);
-        OpInfo.ConstraintVT = RegVT;
+              DAG.getNode(ISD::BITCAST, DL, VT, OpInfo.CallOperand);
+        OpInfo.ConstraintVT = VT;
       }
     }
-
-    NumRegs = TLI.getNumRegisters(Context, OpInfo.ConstraintVT);
   }
 
   // No need to allocate a matching input constraint since the constraint it's
@@ -7393,59 +7670,38 @@ static void GetRegistersForValue(SelectionDAG &DAG, const TargetLowering &TLI,
   if (OpInfo.isMatchingInputConstraint())
     return;
 
-  MVT RegVT;
   EVT ValueVT = OpInfo.ConstraintVT;
+  if (OpInfo.ConstraintVT == MVT::Other)
+    ValueVT = RegVT;
+
+  // Initialize NumRegs.
+  unsigned NumRegs = 1;
+  if (OpInfo.ConstraintVT != MVT::Other)
+    NumRegs = TLI.getNumRegisters(Context, OpInfo.ConstraintVT);
 
   // If this is a constraint for a specific physical register, like {r17},
   // assign it now.
-  if (unsigned AssignedReg = PhysReg.first) {
-    const TargetRegisterClass *RC = PhysReg.second;
-    if (OpInfo.ConstraintVT == MVT::Other)
-      ValueVT = *TRI.legalclasstypes_begin(*RC);
-
-    // Get the actual register value type.  This is important, because the user
-    // may have asked for (e.g.) the AX register in i32 type.  We need to
-    // remember that AX is actually i16 to get the right extension.
-    RegVT = *TRI.legalclasstypes_begin(*RC);
-
-    // This is an explicit reference to a physical register.
-    Regs.push_back(AssignedReg);
-
-    // If this is an expanded reference, add the rest of the regs to Regs.
-    if (NumRegs != 1) {
-      TargetRegisterClass::iterator I = RC->begin();
-      for (; *I != AssignedReg; ++I)
-        assert(I != RC->end() && "Didn't find reg!");
 
-      // Already added the first reg.
-      --NumRegs; ++I;
-      for (; NumRegs; --NumRegs, ++I) {
-        assert(I != RC->end() && "Ran out of registers to allocate!");
-        Regs.push_back(*I);
-      }
-    }
-
-    OpInfo.AssignedRegs = RegsForValue(Regs, RegVT, ValueVT);
-    return;
-  }
+  // If this associated to a specific register, initialize iterator to correct
+  // place. If virtual, make sure we have enough registers
 
-  // Otherwise, if this was a reference to an LLVM register class, create vregs
-  // for this reference.
-  if (const TargetRegisterClass *RC = PhysReg.second) {
-    RegVT = *TRI.legalclasstypes_begin(*RC);
-    if (OpInfo.ConstraintVT == MVT::Other)
-      ValueVT = RegVT;
+  // Initialize iterator if necessary
+  TargetRegisterClass::iterator I = RC->begin();
+  MachineRegisterInfo &RegInfo = MF.getRegInfo();
 
-    // Create the appropriate number of virtual registers.
-    MachineRegisterInfo &RegInfo = MF.getRegInfo();
-    for (; NumRegs; --NumRegs)
-      Regs.push_back(RegInfo.createVirtualRegister(RC));
+  // Do not check for single registers.
+  if (AssignedReg) {
+      for (; *I != AssignedReg; ++I)
+        assert(I != RC->end() && "AssignedReg should be member of RC");
+  }
 
-    OpInfo.AssignedRegs = RegsForValue(Regs, RegVT, ValueVT);
-    return;
+  for (; NumRegs; --NumRegs, ++I) {
+    assert(I != RC->end() && "Ran out of registers to allocate!");
+    auto R = (AssignedReg) ? *I : RegInfo.createVirtualRegister(RC);
+    Regs.push_back(R);
   }
 
-  // Otherwise, we couldn't allocate enough registers for this.
+  OpInfo.AssignedRegs = RegsForValue(Regs, RegVT, ValueVT);
 }
 
 static unsigned
@@ -7466,21 +7722,6 @@ findMatchingInlineAsmOperand(unsigned OperandNo,
   return CurOp;
 }
 
-/// Fill \p Regs with \p NumRegs new virtual registers of type \p RegVT
-/// \return true if it has succeeded, false otherwise
-static bool createVirtualRegs(SmallVector<unsigned, 4> &Regs, unsigned NumRegs,
-                              MVT RegVT, SelectionDAG &DAG) {
-  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
-  MachineRegisterInfo &RegInfo = DAG.getMachineFunction().getRegInfo();
-  for (unsigned i = 0, e = NumRegs; i != e; ++i) {
-    if (const TargetRegisterClass *RC = TLI.getRegClassFor(RegVT))
-      Regs.push_back(RegInfo.createVirtualRegister(RC));
-    else
-      return false;
-  }
-  return true;
-}
-
 namespace {
 
 class ExtraFlags {
@@ -7530,19 +7771,17 @@ void SelectionDAGBuilder::visitInlineAsm(ImmutableCallSite CS) {
   TargetLowering::AsmOperandInfoVector TargetConstraints = TLI.ParseConstraints(
       DAG.getDataLayout(), DAG.getSubtarget().getRegisterInfo(), CS);
 
-  bool hasMemory = false;
-
-  // Remember the HasSideEffect, AlignStack, AsmDialect, MayLoad and MayStore
+  // First Pass: Calculate HasSideEffects and ExtraFlags (AlignStack,
+  // AsmDialect, MayLoad, MayStore).
+  bool HasSideEffect = IA->hasSideEffects();
   ExtraFlags ExtraInfo(CS);
 
   unsigned ArgNo = 0;   // ArgNo - The argument of the CallInst.
   unsigned ResNo = 0;   // ResNo - The result number of the next output.
-  for (unsigned i = 0, e = TargetConstraints.size(); i != e; ++i) {
-    ConstraintOperands.push_back(SDISelAsmOperandInfo(TargetConstraints[i]));
+  for (auto &T : TargetConstraints) {
+    ConstraintOperands.push_back(SDISelAsmOperandInfo(T));
     SDISelAsmOperandInfo &OpInfo = ConstraintOperands.back();
 
-    MVT OpVT = MVT::Other;
-
     // Compute the value type for each operand.
     if (OpInfo.Type == InlineAsm::isInput ||
         (OpInfo.Type == InlineAsm::isOutput && OpInfo.isIndirect)) {
@@ -7550,61 +7789,58 @@ void SelectionDAGBuilder::visitInlineAsm(ImmutableCallSite CS) {
 
       // Process the call argument. BasicBlocks are labels, currently appearing
       // only in asm's.
-      if (const BasicBlock *BB = dyn_cast<BasicBlock>(OpInfo.CallOperandVal)) {
+      const Instruction *I = CS.getInstruction();
+      if (isa<CallBrInst>(I) &&
+          (ArgNo - 1) >= (cast<CallBrInst>(I)->getNumArgOperands() -
+                          cast<CallBrInst>(I)->getNumIndirectDests())) {
+        const auto *BA = cast<BlockAddress>(OpInfo.CallOperandVal);
+        EVT VT = TLI.getValueType(DAG.getDataLayout(), BA->getType(), true);
+        OpInfo.CallOperand = DAG.getTargetBlockAddress(BA, VT);
+      } else if (const auto *BB = dyn_cast<BasicBlock>(OpInfo.CallOperandVal)) {
         OpInfo.CallOperand = DAG.getBasicBlock(FuncInfo.MBBMap[BB]);
       } else {
         OpInfo.CallOperand = getValue(OpInfo.CallOperandVal);
       }
 
-      OpVT =
+      OpInfo.ConstraintVT =
           OpInfo
               .getCallOperandValEVT(*DAG.getContext(), TLI, DAG.getDataLayout())
               .getSimpleVT();
-    }
-
-    if (OpInfo.Type == InlineAsm::isOutput && !OpInfo.isIndirect) {
+    } else if (OpInfo.Type == InlineAsm::isOutput && !OpInfo.isIndirect) {
       // The return value of the call is this value.  As such, there is no
       // corresponding argument.
       assert(!CS.getType()->isVoidTy() && "Bad inline asm!");
       if (StructType *STy = dyn_cast<StructType>(CS.getType())) {
-        OpVT = TLI.getSimpleValueType(DAG.getDataLayout(),
-                                      STy->getElementType(ResNo));
+        OpInfo.ConstraintVT = TLI.getSimpleValueType(
+            DAG.getDataLayout(), STy->getElementType(ResNo));
       } else {
         assert(ResNo == 0 && "Asm only has one result!");
-        OpVT = TLI.getSimpleValueType(DAG.getDataLayout(), CS.getType());
+        OpInfo.ConstraintVT =
+            TLI.getSimpleValueType(DAG.getDataLayout(), CS.getType());
       }
       ++ResNo;
+    } else {
+      OpInfo.ConstraintVT = MVT::Other;
     }
 
-    OpInfo.ConstraintVT = OpVT;
-
-    if (!hasMemory)
-      hasMemory = OpInfo.hasMemory(TLI);
+    if (!HasSideEffect)
+      HasSideEffect = OpInfo.hasMemory(TLI);
 
     // Determine if this InlineAsm MayLoad or MayStore based on the constraints.
-    // FIXME: Could we compute this on OpInfo rather than TargetConstraints[i]?
-    auto TargetConstraint = TargetConstraints[i];
+    // FIXME: Could we compute this on OpInfo rather than T?
 
     // Compute the constraint code and ConstraintType to use.
-    TLI.ComputeConstraintToUse(TargetConstraint, SDValue());
+    TLI.ComputeConstraintToUse(T, SDValue());
 
-    ExtraInfo.update(TargetConstraint);
+    ExtraInfo.update(T);
   }
 
-  SDValue Chain, Flag;
-
   // We won't need to flush pending loads if this asm doesn't touch
   // memory and is nonvolatile.
-  if (hasMemory || IA->hasSideEffects())
-    Chain = getRoot();
-  else
-    Chain = DAG.getRoot();
-
-  // Second pass over the constraints: compute which constraint option to use
-  // and assign registers to constraints that want a specific physreg.
-  for (unsigned i = 0, e = ConstraintOperands.size(); i != e; ++i) {
-    SDISelAsmOperandInfo &OpInfo = ConstraintOperands[i];
+  SDValue Flag, Chain = (HasSideEffect) ? getRoot() : DAG.getRoot();
 
+  // Second pass over the constraints: compute which constraint option to use.
+  for (SDISelAsmOperandInfo &OpInfo : ConstraintOperands) {
     // If this is an output operand with a matching input operand, look up the
     // matching input. If their types mismatch, e.g. one is an integer, the
     // other is floating point, or their sizes are different, flag it as an
@@ -7639,29 +7875,6 @@ void SelectionDAGBuilder::visitInlineAsm(ImmutableCallSite CS) {
       OpInfo.isIndirect = true;
     }
 
-    // If this constraint is for a specific register, allocate it before
-    // anything else.
-    SDISelAsmOperandInfo &RefOpInfo =
-        OpInfo.isMatchingInputConstraint()
-            ? ConstraintOperands[OpInfo.getMatchedOperand()]
-            : ConstraintOperands[i];
-    if (RefOpInfo.ConstraintType == TargetLowering::C_Register)
-      GetRegistersForValue(DAG, TLI, getCurSDLoc(), OpInfo, RefOpInfo);
-  }
-
-  // Third pass - Loop over all of the operands, assigning virtual or physregs
-  // to register class operands.
-  for (unsigned i = 0, e = ConstraintOperands.size(); i != e; ++i) {
-    SDISelAsmOperandInfo &OpInfo = ConstraintOperands[i];
-    SDISelAsmOperandInfo &RefOpInfo =
-        OpInfo.isMatchingInputConstraint()
-            ? ConstraintOperands[OpInfo.getMatchedOperand()]
-            : ConstraintOperands[i];
-
-    // C_Register operands have already been allocated, Other/Memory don't need
-    // to be.
-    if (RefOpInfo.ConstraintType == TargetLowering::C_RegisterClass)
-      GetRegistersForValue(DAG, TLI, getCurSDLoc(), OpInfo, RefOpInfo);
   }
 
   // AsmNodeOperands - The operands for the ISD::INLINEASM node.
@@ -7681,23 +7894,21 @@ void SelectionDAGBuilder::visitInlineAsm(ImmutableCallSite CS) {
   AsmNodeOperands.push_back(DAG.getTargetConstant(
       ExtraInfo.get(), getCurSDLoc(), TLI.getPointerTy(DAG.getDataLayout())));
 
-  // Loop over all of the inputs, copying the operand values into the
-  // appropriate registers and processing the output regs.
-  RegsForValue RetValRegs;
-
-  // IndirectStoresToEmit - The set of stores to emit after the inline asm node.
-  std::vector<std::pair<RegsForValue, Value *>> IndirectStoresToEmit;
-
-  for (unsigned i = 0, e = ConstraintOperands.size(); i != e; ++i) {
-    SDISelAsmOperandInfo &OpInfo = ConstraintOperands[i];
+  // Third pass: Loop over operands to prepare DAG-level operands.. As part of
+  // this, assign virtual and physical registers for inputs and otput.
+  for (SDISelAsmOperandInfo &OpInfo : ConstraintOperands) {
+    // Assign Registers.
+    SDISelAsmOperandInfo &RefOpInfo =
+        OpInfo.isMatchingInputConstraint()
+            ? ConstraintOperands[OpInfo.getMatchedOperand()]
+            : OpInfo;
+    GetRegistersForValue(DAG, getCurSDLoc(), OpInfo, RefOpInfo);
 
     switch (OpInfo.Type) {
     case InlineAsm::isOutput:
-      if (OpInfo.ConstraintType != TargetLowering::C_RegisterClass &&
-          OpInfo.ConstraintType != TargetLowering::C_Register) {
-        // Memory output, or 'other' output (e.g. 'X' constraint).
-        assert(OpInfo.isIndirect && "Memory output must be indirect operand");
-
+      if (OpInfo.ConstraintType == TargetLowering::C_Memory ||
+          (OpInfo.ConstraintType == TargetLowering::C_Other &&
+           OpInfo.isIndirect)) {
         unsigned ConstraintID =
             TLI.getInlineAsmMemConstraint(OpInfo.ConstraintCode);
         assert(ConstraintID != InlineAsm::Constraint_Unknown &&
@@ -7710,38 +7921,27 @@ void SelectionDAGBuilder::visitInlineAsm(ImmutableCallSite CS) {
                                                         MVT::i32));
         AsmNodeOperands.push_back(OpInfo.CallOperand);
         break;
-      }
-
-      // Otherwise, this is a register or register class output.
+      } else if ((OpInfo.ConstraintType == TargetLowering::C_Other &&
+                  !OpInfo.isIndirect) ||
+                 OpInfo.ConstraintType == TargetLowering::C_Register ||
+                 OpInfo.ConstraintType == TargetLowering::C_RegisterClass) {
+        // Otherwise, this outputs to a register (directly for C_Register /
+        // C_RegisterClass, and a target-defined fashion for C_Other). Find a
+        // register that we can use.
+        if (OpInfo.AssignedRegs.Regs.empty()) {
+          emitInlineAsmError(
+              CS, "couldn't allocate output register for constraint '" +
+                      Twine(OpInfo.ConstraintCode) + "'");
+          return;
+        }
 
-      // Copy the output from the appropriate register.  Find a register that
-      // we can use.
-      if (OpInfo.AssignedRegs.Regs.empty()) {
-        emitInlineAsmError(
-            CS, "couldn't allocate output register for constraint '" +
-                    Twine(OpInfo.ConstraintCode) + "'");
-        return;
+        // Add information to the INLINEASM node to know that this register is
+        // set.
+        OpInfo.AssignedRegs.AddInlineAsmOperands(
+            OpInfo.isEarlyClobber ? InlineAsm::Kind_RegDefEarlyClobber
+                                  : InlineAsm::Kind_RegDef,
+            false, 0, getCurSDLoc(), DAG, AsmNodeOperands);
       }
-
-      // If this is an indirect operand, store through the pointer after the
-      // asm.
-      if (OpInfo.isIndirect) {
-        IndirectStoresToEmit.push_back(std::make_pair(OpInfo.AssignedRegs,
-                                                      OpInfo.CallOperandVal));
-      } else {
-        // This is the result value of the call.
-        assert(!CS.getType()->isVoidTy() && "Bad inline asm!");
-        // Concatenate this output onto the outputs list.
-        RetValRegs.append(OpInfo.AssignedRegs);
-      }
-
-      // Add information to the INLINEASM node to know that this register is
-      // set.
-      OpInfo.AssignedRegs
-          .AddInlineAsmOperands(OpInfo.isEarlyClobber
-                                    ? InlineAsm::Kind_RegDefEarlyClobber
-                                    : InlineAsm::Kind_RegDef,
-                                false, 0, getCurSDLoc(), DAG, AsmNodeOperands);
       break;
 
     case InlineAsm::isInput: {
@@ -7768,9 +7968,13 @@ void SelectionDAGBuilder::visitInlineAsm(ImmutableCallSite CS) {
           MVT RegVT = AsmNodeOperands[CurOp+1].getSimpleValueType();
           SmallVector<unsigned, 4> Regs;
 
-          if (!createVirtualRegs(Regs,
-                                 InlineAsm::getNumOperandRegisters(OpFlag),
-                                 RegVT, DAG)) {
+          if (const TargetRegisterClass *RC = TLI.getRegClassFor(RegVT)) {
+            unsigned NumRegs = InlineAsm::getNumOperandRegisters(OpFlag);
+            MachineRegisterInfo &RegInfo =
+                DAG.getMachineFunction().getRegInfo();
+            for (unsigned i = 0; i != NumRegs; ++i)
+              Regs.push_back(RegInfo.createVirtualRegister(RC));
+          } else {
             emitInlineAsmError(CS, "inline asm error: This value type register "
                                    "class is not natively supported!");
             return;
@@ -7891,98 +8095,118 @@ void SelectionDAGBuilder::visitInlineAsm(ImmutableCallSite CS) {
   AsmNodeOperands[InlineAsm::Op_InputChain] = Chain;
   if (Flag.getNode()) AsmNodeOperands.push_back(Flag);
 
-  Chain = DAG.getNode(ISD::INLINEASM, getCurSDLoc(),
+  unsigned ISDOpc = isa<CallBrInst>(CS.getInstruction()) ? ISD::INLINEASM_BR : ISD::INLINEASM;
+  Chain = DAG.getNode(ISDOpc, getCurSDLoc(),
                       DAG.getVTList(MVT::Other, MVT::Glue), AsmNodeOperands);
   Flag = Chain.getValue(1);
 
-  // If this asm returns a register value, copy the result from that register
-  // and set it as the value of the call.
-  if (!RetValRegs.Regs.empty()) {
-    SDValue Val = RetValRegs.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(),
-                                             Chain, &Flag, CS.getInstruction());
-
-    llvm::Type *CSResultType = CS.getType();
-    unsigned numRet;
-    ArrayRef<Type *> ResultTypes;
-    SmallVector<SDValue, 1> ResultValues(1);
-    if (CSResultType->isSingleValueType()) {
-      numRet = 1;
-      ResultValues[0] = Val;
-      ResultTypes = makeArrayRef(CSResultType);
-    } else {
-      numRet = CSResultType->getNumContainedTypes();
-      assert(Val->getNumOperands() == numRet &&
-             "Mismatch in number of output operands in asm result");
-      ResultTypes = CSResultType->subtypes();
-      ArrayRef<SDUse> ValueUses = Val->ops();
-      ResultValues.resize(numRet);
-      std::transform(ValueUses.begin(), ValueUses.end(), ResultValues.begin(),
-                     [](const SDUse &u) -> SDValue { return u.get(); });
-    }
-    SmallVector<EVT, 1> ResultVTs(numRet);
-    for (unsigned i = 0; i < numRet; i++) {
-      EVT ResultVT = TLI.getValueType(DAG.getDataLayout(), ResultTypes[i]);
-      SDValue Val = ResultValues[i];
-      assert(ResultTypes[i]->isSized() && "Unexpected unsized type");
-      // If the type of the inline asm call site return value is different but
-      // has same size as the type of the asm output bitcast it.  One example
-      // of this is for vectors with different width / number of elements.
-      // This can happen for register classes that can contain multiple
-      // different value types.  The preg or vreg allocated may not have the
-      // same VT as was expected.
-      //
-      // This can also happen for a return value that disagrees with the
-      // register class it is put in, eg. a double in a general-purpose
-      // register on a 32-bit machine.
-      if (ResultVT != Val.getValueType() &&
-          ResultVT.getSizeInBits() == Val.getValueSizeInBits())
-        Val = DAG.getNode(ISD::BITCAST, getCurSDLoc(), ResultVT, Val);
-      else if (ResultVT != Val.getValueType() && ResultVT.isInteger() &&
-               Val.getValueType().isInteger()) {
-        // If a result value was tied to an input value, the computed result
-        // may have a wider width than the expected result.  Extract the
-        // relevant portion.
-        Val = DAG.getNode(ISD::TRUNCATE, getCurSDLoc(), ResultVT, Val);
-      }
+  // Do additional work to generate outputs.
 
-      assert(ResultVT == Val.getValueType() && "Asm result value mismatch!");
-      ResultVTs[i] = ResultVT;
-      ResultValues[i] = Val;
-    }
+  SmallVector<EVT, 1> ResultVTs;
+  SmallVector<SDValue, 1> ResultValues;
+  SmallVector<SDValue, 8> OutChains;
 
-    Val = DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
-                      DAG.getVTList(ResultVTs), ResultValues);
-    setValue(CS.getInstruction(), Val);
-    // Don't need to use this as a chain in this case.
-    if (!IA->hasSideEffects() && !hasMemory && IndirectStoresToEmit.empty())
-      return;
-  }
+  llvm::Type *CSResultType = CS.getType();
+  ArrayRef<Type *> ResultTypes;
+  if (StructType *StructResult = dyn_cast<StructType>(CSResultType))
+    ResultTypes = StructResult->elements();
+  else if (!CSResultType->isVoidTy())
+    ResultTypes = makeArrayRef(CSResultType);
+
+  auto CurResultType = ResultTypes.begin();
+  auto handleRegAssign = [&](SDValue V) {
+    assert(CurResultType != ResultTypes.end() && "Unexpected value");
+    assert((*CurResultType)->isSized() && "Unexpected unsized type");
+    EVT ResultVT = TLI.getValueType(DAG.getDataLayout(), *CurResultType);
+    ++CurResultType;
+    // If the type of the inline asm call site return value is different but has
+    // same size as the type of the asm output bitcast it.  One example of this
+    // is for vectors with different width / number of elements.  This can
+    // happen for register classes that can contain multiple different value
+    // types.  The preg or vreg allocated may not have the same VT as was
+    // expected.
+    //
+    // This can also happen for a return value that disagrees with the register
+    // class it is put in, eg. a double in a general-purpose register on a
+    // 32-bit machine.
+    if (ResultVT != V.getValueType() &&
+        ResultVT.getSizeInBits() == V.getValueSizeInBits())
+      V = DAG.getNode(ISD::BITCAST, getCurSDLoc(), ResultVT, V);
+    else if (ResultVT != V.getValueType() && ResultVT.isInteger() &&
+             V.getValueType().isInteger()) {
+      // If a result value was tied to an input value, the computed result
+      // may have a wider width than the expected result.  Extract the
+      // relevant portion.
+      V = DAG.getNode(ISD::TRUNCATE, getCurSDLoc(), ResultVT, V);
+    }
+    assert(ResultVT == V.getValueType() && "Asm result value mismatch!");
+    ResultVTs.push_back(ResultVT);
+    ResultValues.push_back(V);
+  };
 
-  std::vector<std::pair<SDValue, const Value *>> StoresToEmit;
+  // Deal with output operands.
+  for (SDISelAsmOperandInfo &OpInfo : ConstraintOperands) {
+    if (OpInfo.Type == InlineAsm::isOutput) {
+      SDValue Val;
+      // Skip trivial output operands.
+      if (OpInfo.AssignedRegs.Regs.empty())
+        continue;
 
-  // Process indirect outputs, first output all of the flagged copies out of
-  // physregs.
-  for (unsigned i = 0, e = IndirectStoresToEmit.size(); i != e; ++i) {
-    RegsForValue &OutRegs = IndirectStoresToEmit[i].first;
-    const Value *Ptr = IndirectStoresToEmit[i].second;
-    SDValue OutVal = OutRegs.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(),
-                                             Chain, &Flag, IA);
-    StoresToEmit.push_back(std::make_pair(OutVal, Ptr));
+      switch (OpInfo.ConstraintType) {
+      case TargetLowering::C_Register:
+      case TargetLowering::C_RegisterClass:
+        Val = OpInfo.AssignedRegs.getCopyFromRegs(
+            DAG, FuncInfo, getCurSDLoc(), Chain, &Flag, CS.getInstruction());
+        break;
+      case TargetLowering::C_Other:
+        Val = TLI.LowerAsmOutputForConstraint(Chain, &Flag, getCurSDLoc(),
+                                              OpInfo, DAG);
+        break;
+      case TargetLowering::C_Memory:
+        break; // Already handled.
+      case TargetLowering::C_Unknown:
+        assert(false && "Unexpected unknown constraint");
+      }
+
+      // Indirect output manifest as stores. Record output chains.
+      if (OpInfo.isIndirect) {
+
+        const Value *Ptr = OpInfo.CallOperandVal;
+        assert(Ptr && "Expected value CallOperandVal for indirect asm operand");
+        SDValue Store = DAG.getStore(Chain, getCurSDLoc(), Val, getValue(Ptr),
+                                     MachinePointerInfo(Ptr));
+        OutChains.push_back(Store);
+      } else {
+        // generate CopyFromRegs to associated registers.
+        assert(!CS.getType()->isVoidTy() && "Bad inline asm!");
+        if (Val.getOpcode() == ISD::MERGE_VALUES) {
+          for (const SDValue &V : Val->op_values())
+            handleRegAssign(V);
+        } else
+          handleRegAssign(Val);
+      }
+    }
   }
 
-  // Emit the non-flagged stores from the physregs.
-  SmallVector<SDValue, 8> OutChains;
-  for (unsigned i = 0, e = StoresToEmit.size(); i != e; ++i) {
-    SDValue Val = DAG.getStore(Chain, getCurSDLoc(), StoresToEmit[i].first,
-                               getValue(StoresToEmit[i].second),
-                               MachinePointerInfo(StoresToEmit[i].second));
-    OutChains.push_back(Val);
+  // Set results.
+  if (!ResultValues.empty()) {
+    assert(CurResultType == ResultTypes.end() &&
+           "Mismatch in number of ResultTypes");
+    assert(ResultValues.size() == ResultTypes.size() &&
+           "Mismatch in number of output operands in asm result");
+
+    SDValue V = DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
+                            DAG.getVTList(ResultVTs), ResultValues);
+    setValue(CS.getInstruction(), V);
   }
 
+  // Collect store chains.
   if (!OutChains.empty())
     Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other, OutChains);
 
-  DAG.setRoot(Chain);
+  // Only Update Root if inline assembly has a memory effect.
+  if (ResultValues.empty() || HasSideEffect || !OutChains.empty())
+    DAG.setRoot(Chain);
 }
 
 void SelectionDAGBuilder::emitInlineAsmError(ImmutableCallSite CS,
@@ -8084,7 +8308,7 @@ SDValue SelectionDAGBuilder::lowerRangeToAssertZExt(SelectionDAG &DAG,
 /// convention or require stack pointer adjustment. Only a subset of the
 /// intrinsic's operands need to participate in the calling convention.
 void SelectionDAGBuilder::populateCallLoweringInfo(
-    TargetLowering::CallLoweringInfo &CLI, ImmutableCallSite CS,
+    TargetLowering::CallLoweringInfo &CLI, const CallBase *Call,
     unsigned ArgIdx, unsigned NumArgs, SDValue Callee, Type *ReturnTy,
     bool IsPatchPoint) {
   TargetLowering::ArgListTy Args;
@@ -8094,21 +8318,21 @@ void SelectionDAGBuilder::populateCallLoweringInfo(
   // Attributes for args start at offset 1, after the return attribute.
   for (unsigned ArgI = ArgIdx, ArgE = ArgIdx + NumArgs;
        ArgI != ArgE; ++ArgI) {
-    const Value *V = CS->getOperand(ArgI);
+    const Value *V = Call->getOperand(ArgI);
 
     assert(!V->getType()->isEmptyTy() && "Empty type passed to intrinsic.");
 
     TargetLowering::ArgListEntry Entry;
     Entry.Node = getValue(V);
     Entry.Ty = V->getType();
-    Entry.setAttributes(&CS, ArgI);
+    Entry.setAttributes(Call, ArgI);
     Args.push_back(Entry);
   }
 
   CLI.setDebugLoc(getCurSDLoc())
       .setChain(getRoot())
-      .setCallee(CS.getCallingConv(), ReturnTy, Callee, std::move(Args))
-      .setDiscardResult(CS->use_empty())
+      .setCallee(Call->getCallingConv(), ReturnTy, Callee, std::move(Args))
+      .setDiscardResult(Call->use_empty())
       .setIsPatchPoint(IsPatchPoint);
 }
 
@@ -8252,8 +8476,8 @@ void SelectionDAGBuilder::visitPatchpoint(ImmutableCallSite CS,
     IsAnyRegCC ? Type::getVoidTy(*DAG.getContext()) : CS->getType();
 
   TargetLowering::CallLoweringInfo CLI(DAG);
-  populateCallLoweringInfo(CLI, CS, NumMetaOpers, NumCallArgs, Callee, ReturnTy,
-                           true);
+  populateCallLoweringInfo(CLI, cast<CallBase>(CS.getInstruction()),
+                           NumMetaOpers, NumCallArgs, Callee, ReturnTy, true);
   std::pair<SDValue, SDValue> Result = lowerInvokable(CLI, EHPadBB);
 
   SDNode *CallEnd = Result.second.getNode();
diff --git a/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h b/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h
index 5f9cdb69daf72d0d017ace630f7f6f7bed163951..e84214dbf5edf16fe8f0c1fd36c07ab77e1fc966 100644
--- a/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h
+++ b/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h
@@ -1,9 +1,8 @@
 //===- SelectionDAGBuilder.h - Selection-DAG building -----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -47,6 +46,7 @@ class AtomicRMWInst;
 class BasicBlock;
 class BranchInst;
 class CallInst;
+class CallBrInst;
 class CatchPadInst;
 class CatchReturnInst;
 class CatchSwitchInst;
@@ -91,16 +91,16 @@ class Value;
 /// implementation that is parameterized by a TargetLowering object.
 ///
 class SelectionDAGBuilder {
-  /// CurInst - The current instruction being visited
+  /// The current instruction being visited.
   const Instruction *CurInst = nullptr;
 
   DenseMap<const Value*, SDValue> NodeMap;
 
-  /// UnusedArgNodeMap - Maps argument value for unused arguments. This is used
+  /// Maps argument value for unused arguments. This is used
   /// to preserve debug information for incoming arguments.
   DenseMap<const Value*, SDValue> UnusedArgNodeMap;
 
-  /// DanglingDebugInfo - Helper type for DanglingDebugInfoMap.
+  /// Helper type for DanglingDebugInfoMap.
   class DanglingDebugInfo {
     const DbgValueInst* DI = nullptr;
     DebugLoc dl;
@@ -116,18 +116,17 @@ class SelectionDAGBuilder {
     unsigned getSDNodeOrder() { return SDNodeOrder; }
   };
 
-  /// DanglingDebugInfoVector - Helper type for DanglingDebugInfoMap.
+  /// Helper type for DanglingDebugInfoMap.
   typedef std::vector<DanglingDebugInfo> DanglingDebugInfoVector;
 
-  /// DanglingDebugInfoMap - Keeps track of dbg_values for which we have not
-  /// yet seen the referent.  We defer handling these until we do see it.
+  /// Keeps track of dbg_values for which we have not yet seen the referent.
+  /// We defer handling these until we do see it.
   DenseMap<const Value*, DanglingDebugInfoVector> DanglingDebugInfoMap;
 
 public:
-  /// PendingLoads - Loads are not emitted to the program immediately.  We bunch
-  /// them up and then emit token factor nodes when possible.  This allows us to
-  /// get simple disambiguation between loads without worrying about alias
-  /// analysis.
+  /// Loads are not emitted to the program immediately.  We bunch them up and
+  /// then emit token factor nodes when possible.  This allows us to get simple
+  /// disambiguation between loads without worrying about alias analysis.
   SmallVector<SDValue, 8> PendingLoads;
 
   /// State used while lowering a statepoint sequence (gc_statepoint,
@@ -135,15 +134,14 @@ public:
   StatepointLoweringState StatepointLowering;
 
 private:
-  /// PendingExports - CopyToReg nodes that copy values to virtual registers
-  /// for export to other blocks need to be emitted before any terminator
-  /// instruction, but they have no other ordering requirements. We bunch them
-  /// up and the emit a single tokenfactor for them just before terminator
-  /// instructions.
+  /// CopyToReg nodes that copy values to virtual registers for export to other
+  /// blocks need to be emitted before any terminator instruction, but they have
+  /// no other ordering requirements. We bunch them up and the emit a single
+  /// tokenfactor for them just before terminator instructions.
   SmallVector<SDValue, 8> PendingExports;
 
-  /// SDNodeOrder - A unique monotonically increasing number used to order the
-  /// SDNodes we create.
+  /// A unique monotonically increasing number used to order the SDNodes we
+  /// create.
   unsigned SDNodeOrder;
 
   enum CaseClusterKind {
@@ -222,29 +220,29 @@ private:
   /// Sort Clusters and merge adjacent cases.
   void sortAndRangeify(CaseClusterVector &Clusters);
 
-  /// CaseBlock - This structure is used to communicate between
-  /// SelectionDAGBuilder and SDISel for the code generation of additional basic
-  /// blocks needed by multi-case switch statements.
+  /// This structure is used to communicate between SelectionDAGBuilder and
+  /// SDISel for the code generation of additional basic blocks needed by
+  /// multi-case switch statements.
   struct CaseBlock {
-    // CC - the condition code to use for the case block's setcc node
+    // The condition code to use for the case block's setcc node.
     ISD::CondCode CC;
 
-    // CmpLHS/CmpRHS/CmpMHS - The LHS/MHS/RHS of the comparison to emit.
+    // The LHS/MHS/RHS of the comparison to emit.
     // Emit by default LHS op RHS. MHS is used for range comparisons:
     // If MHS is not null: (LHS <= MHS) and (MHS <= RHS).
     const Value *CmpLHS, *CmpMHS, *CmpRHS;
 
-    // TrueBB/FalseBB - the block to branch to if the setcc is true/false.
+    // The block to branch to if the setcc is true/false.
     MachineBasicBlock *TrueBB, *FalseBB;
 
-    // ThisBB - the block into which to emit the code for the setcc and branches
+    // The block into which to emit the code for the setcc and branches.
     MachineBasicBlock *ThisBB;
 
     /// The debug location of the instruction this CaseBlock was
     /// produced from.
     SDLoc DL;
 
-    // TrueProb/FalseProb - branch weights.
+    // Branch weights.
     BranchProbability TrueProb, FalseProb;
 
     CaseBlock(ISD::CondCode cc, const Value *cmplhs, const Value *cmprhs,
@@ -259,14 +257,14 @@ private:
   };
 
   struct JumpTable {
-    /// Reg - the virtual register containing the index of the jump table entry
-    //. to jump to.
+    /// The virtual register containing the index of the jump table entry
+    /// to jump to.
     unsigned Reg;
-    /// JTI - the JumpTableIndex for this jump table in the function.
+    /// The JumpTableIndex for this jump table in the function.
     unsigned JTI;
-    /// MBB - the MBB into which to emit the code for the indirect jump.
+    /// The MBB into which to emit the code for the indirect jump.
     MachineBasicBlock *MBB;
-    /// Default - the MBB of the default bb, which is a successor of the range
+    /// The MBB of the default bb, which is a successor of the range
     /// check MBB.  This is when updating PHI nodes in successors.
     MachineBasicBlock *Default;
 
@@ -588,16 +586,16 @@ public:
   AliasAnalysis *AA = nullptr;
   const TargetLibraryInfo *LibInfo;
 
-  /// SwitchCases - Vector of CaseBlock structures used to communicate
-  /// SwitchInst code generation information.
+  /// Vector of CaseBlock structures used to communicate SwitchInst code
+  /// generation information.
   std::vector<CaseBlock> SwitchCases;
 
-  /// JTCases - Vector of JumpTable structures used to communicate
-  /// SwitchInst code generation information.
+  /// Vector of JumpTable structures used to communicate SwitchInst code
+  /// generation information.
   std::vector<JumpTableBlock> JTCases;
 
-  /// BitTestCases - Vector of BitTestBlock structures used to communicate
-  /// SwitchInst code generation information.
+  /// Vector of BitTestBlock structures used to communicate SwitchInst code
+  /// generation information.
   std::vector<BitTestBlock> BitTestCases;
 
   /// A StackProtectorDescriptor structure used to communicate stack protector
@@ -608,19 +606,17 @@ public:
   // PHI nodes.
   DenseMap<const Constant *, unsigned> ConstantsOut;
 
-  /// FuncInfo - Information about the function as a whole.
-  ///
+  /// Information about the function as a whole.
   FunctionLoweringInfo &FuncInfo;
 
-  /// GFI - Garbage collection metadata for the function.
+  /// Garbage collection metadata for the function.
   GCFunctionInfo *GFI;
 
-  /// LPadToCallSiteMap - Map a landing pad to the call site indexes.
+  /// Map a landing pad to the call site indexes.
   DenseMap<MachineBasicBlock *, SmallVector<unsigned, 4>> LPadToCallSiteMap;
 
-  /// HasTailCall - This is set to true if a call in the current
-  /// block has been translated as a tail call. In this case,
-  /// no subsequent DAG nodes should be created.
+  /// This is set to true if a call in the current block has been translated as
+  /// a tail call. In this case, no subsequent DAG nodes should be created.
   bool HasTailCall = false;
 
   LLVMContext *Context;
@@ -670,20 +666,34 @@ public:
 
   void visit(unsigned Opcode, const User &I);
 
-  /// getCopyFromRegs - If there was virtual register allocated for the value V
-  /// emit CopyFromReg of the specified type Ty. Return empty SDValue() otherwise.
+  /// If there was virtual register allocated for the value V emit CopyFromReg
+  /// of the specified type Ty. Return empty SDValue() otherwise.
   SDValue getCopyFromRegs(const Value *V, Type *Ty);
 
   /// If we have dangling debug info that describes \p Variable, or an
   /// overlapping part of variable considering the \p Expr, then this method
-  /// weill drop that debug info as it isn't valid any longer.
+  /// will drop that debug info as it isn't valid any longer.
   void dropDanglingDebugInfo(const DILocalVariable *Variable,
                              const DIExpression *Expr);
 
-  // resolveDanglingDebugInfo - if we saw an earlier dbg_value referring to V,
-  // generate the debug data structures now that we've seen its definition.
+  /// If we saw an earlier dbg_value referring to V, generate the debug data
+  /// structures now that we've seen its definition.
   void resolveDanglingDebugInfo(const Value *V, SDValue Val);
 
+  /// For the given dangling debuginfo record, perform last-ditch efforts to
+  /// resolve the debuginfo to something that is represented in this DAG. If
+  /// this cannot be done, produce an Undef debug value record.
+  void salvageUnresolvedDbgValue(DanglingDebugInfo &DDI);
+
+  /// For a given Value, attempt to create and record a SDDbgValue in the
+  /// SelectionDAG.
+  bool handleDebugValue(const Value *V, DILocalVariable *Var,
+                        DIExpression *Expr, DebugLoc CurDL,
+                        DebugLoc InstDL, unsigned Order);
+
+  /// Evict any dangling debug information, attempting to salvage it first.
+  void resolveOrClearDbgInfo();
+
   SDValue getValue(const Value *V);
   bool findValue(const Value *V) const;
 
@@ -733,7 +743,7 @@ public:
                                  SDValue Op);
 
   void populateCallLoweringInfo(TargetLowering::CallLoweringInfo &CLI,
-                                ImmutableCallSite CS, unsigned ArgIdx,
+                                const CallBase *Call, unsigned ArgIdx,
                                 unsigned NumArgs, SDValue Callee,
                                 Type *ReturnTy, bool IsPatchPoint);
 
@@ -741,7 +751,7 @@ public:
   lowerInvokable(TargetLowering::CallLoweringInfo &CLI,
                  const BasicBlock *EHPadBB = nullptr);
 
-  /// UpdateSplitBlock - When an MBB was split during scheduling, update the
+  /// When an MBB was split during scheduling, update the
   /// references that need to refer to the last resulting block.
   void UpdateSplitBlock(MachineBasicBlock *First, MachineBasicBlock *Last);
 
@@ -797,13 +807,13 @@ public:
   void LowerStatepoint(ImmutableStatepoint ISP,
                        const BasicBlock *EHPadBB = nullptr);
 
-  void LowerCallSiteWithDeoptBundle(ImmutableCallSite CS, SDValue Callee,
+  void LowerCallSiteWithDeoptBundle(const CallBase *Call, SDValue Callee,
                                     const BasicBlock *EHPadBB);
 
   void LowerDeoptimizeCall(const CallInst *CI);
   void LowerDeoptimizingReturn();
 
-  void LowerCallSiteWithDeoptBundleImpl(ImmutableCallSite CS, SDValue Callee,
+  void LowerCallSiteWithDeoptBundleImpl(const CallBase *Call, SDValue Callee,
                                         const BasicBlock *EHPadBB,
                                         bool VarArgDisallowed,
                                         bool ForceVoidReturnTy);
@@ -852,6 +862,7 @@ public:
 private:
   // These all get lowered before this pass.
   void visitInvoke(const InvokeInst &I);
+  void visitCallBr(const CallBrInst &I);
   void visitResume(const ResumeInst &I);
 
   void visitUnary(const User &I, unsigned Opcode);
@@ -984,7 +995,7 @@ private:
                           unsigned DbgSDNodeOrder);
 };
 
-/// RegsForValue - This struct represents the registers (physical or virtual)
+/// This struct represents the registers (physical or virtual)
 /// that a particular set of values is assigned, and the type information about
 /// the value. The most common situation is to represent one value at a time,
 /// but struct or array values are handled element-wise as multiple values.  The
diff --git a/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp b/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
index 43df2abb674bebb53530040b91156b783c121efb..490b2f9957e8d54f0634336eb154ee0f4059c9e7 100644
--- a/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
+++ b/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
@@ -1,9 +1,8 @@
 //===- SelectionDAGDumper.cpp - Implement SelectionDAG::dump() ------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -96,6 +95,7 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const {
   case ISD::ATOMIC_LOAD_MAX:            return "AtomicLoadMax";
   case ISD::ATOMIC_LOAD_UMIN:           return "AtomicLoadUMin";
   case ISD::ATOMIC_LOAD_UMAX:           return "AtomicLoadUMax";
+  case ISD::ATOMIC_LOAD_FADD:           return "AtomicLoadFAdd";
   case ISD::ATOMIC_LOAD:                return "AtomicLoad";
   case ISD::ATOMIC_STORE:               return "AtomicStore";
   case ISD::PCMARKER:                   return "PCMarker";
@@ -145,6 +145,8 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const {
     unsigned IID = cast<ConstantSDNode>(getOperand(OpNo))->getZExtValue();
     if (IID < Intrinsic::num_intrinsics)
       return Intrinsic::getName((Intrinsic::ID)IID, None);
+    else if (!G)
+      return "Unknown intrinsic";
     else if (const TargetIntrinsicInfo *TII = G->getTarget().getIntrinsicInfo())
       return TII->getName(IID);
     llvm_unreachable("Invalid intrinsic ID");
@@ -170,6 +172,7 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const {
   case ISD::UNDEF:                      return "undef";
   case ISD::MERGE_VALUES:               return "merge_values";
   case ISD::INLINEASM:                  return "inlineasm";
+  case ISD::INLINEASM_BR:               return "inlineasm_br";
   case ISD::EH_LABEL:                   return "eh_label";
   case ISD::HANDLENODE:                 return "handlenode";
 
@@ -298,6 +301,7 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const {
   case ISD::SSUBSAT:                    return "ssubsat";
   case ISD::USUBSAT:                    return "usubsat";
   case ISD::SMULFIX:                    return "smulfix";
+  case ISD::UMULFIX:                    return "umulfix";
 
   // Conversion operators.
   case ISD::SIGN_EXTEND:                return "sign_extend";
@@ -649,6 +653,36 @@ void SDNode::print_details(raw_ostream &OS, const SelectionDAG *G) const {
     if (*AM)
       OS << ", " << AM;
 
+    OS << ">";
+  } else if (const MaskedLoadSDNode *MLd = dyn_cast<MaskedLoadSDNode>(this)) {
+    OS << "<";
+
+    printMemOperand(OS, *MLd->getMemOperand(), G);
+
+    bool doExt = true;
+    switch (MLd->getExtensionType()) {
+    default: doExt = false; break;
+    case ISD::EXTLOAD:  OS << ", anyext"; break;
+    case ISD::SEXTLOAD: OS << ", sext"; break;
+    case ISD::ZEXTLOAD: OS << ", zext"; break;
+    }
+    if (doExt)
+      OS << " from " << MLd->getMemoryVT().getEVTString();
+
+    if (MLd->isExpandingLoad())
+      OS << ", expanding";
+
+    OS << ">";
+  } else if (const MaskedStoreSDNode *MSt = dyn_cast<MaskedStoreSDNode>(this)) {
+    OS << "<";
+    printMemOperand(OS, *MSt->getMemOperand(), G);
+
+    if (MSt->isTruncatingStore())
+      OS << ", trunc to " << MSt->getMemoryVT().getEVTString();
+
+    if (MSt->isCompressingStore())
+      OS << ", compressing";
+
     OS << ">";
   } else if (const MemSDNode* M = dyn_cast<MemSDNode>(this)) {
     OS << "<";
@@ -684,45 +718,63 @@ void SDNode::print_details(raw_ostream &OS, const SelectionDAG *G) const {
     if (getNodeId() != -1)
       OS << " [ID=" << getNodeId() << ']';
     if (!(isa<ConstantSDNode>(this) || (isa<ConstantFPSDNode>(this))))
-      OS << "# D:" << isDivergent();
-
-    if (!G)
-      return;
-
-    DILocation *L = getDebugLoc();
-    if (!L)
-      return;
-
-    if (auto *Scope = L->getScope())
-      OS << Scope->getFilename();
-    else
-      OS << "<unknown>";
-    OS << ':' << L->getLine();
-    if (unsigned C = L->getColumn())
-      OS << ':' << C;
-
-    for (SDDbgValue *Dbg : G->GetDbgValues(this)) {
-      if (Dbg->getKind() != SDDbgValue::SDNODE || Dbg->isInvalidated())
-        continue;
-      Dbg->dump(OS);
-    }
+      OS << " # D:" << isDivergent();
+
+    if (G && !G->GetDbgValues(this).empty()) {
+      OS << " [NoOfDbgValues=" << G->GetDbgValues(this).size() << ']';
+      for (SDDbgValue *Dbg : G->GetDbgValues(this))
+        if (!Dbg->isInvalidated())
+          Dbg->print(OS);
+    } else if (getHasDebugValue())
+      OS << " [NoOfDbgValues>0]";
   }
 }
 
-LLVM_DUMP_METHOD void SDDbgValue::dump(raw_ostream &OS) const {
- OS << " DbgVal";
- if (kind==SDNODE)
-   OS << '(' << u.s.ResNo << ')';
- OS << ":\"" << Var->getName() << '"';
+LLVM_DUMP_METHOD void SDDbgValue::print(raw_ostream &OS) const {
+  OS << " DbgVal(Order=" << getOrder() << ')';
+  if (isInvalidated()) OS << "(Invalidated)";
+  if (isEmitted()) OS << "(Emitted)";
+  switch (getKind()) {
+  case SDNODE:
+    if (getSDNode())
+      OS << "(SDNODE=" << PrintNodeId(*getSDNode()) << ':' <<  getResNo() << ')';
+    else
+      OS << "(SDNODE)";
+    break;
+  case CONST:
+    OS << "(CONST)";
+    break;
+  case FRAMEIX:
+    OS << "(FRAMEIX=" << getFrameIx() << ')';
+    break;
+  case VREG:
+    OS << "(VREG=" << getVReg() << ')';
+    break;
+  }
+  if (isIndirect()) OS << "(Indirect)";
+  OS << ":\"" << Var->getName() << '"';
 #ifndef NDEBUG
- if (Expr->getNumElements())
-   Expr->dump();
+  if (Expr->getNumElements())
+    Expr->dump();
 #endif
 }
 
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+LLVM_DUMP_METHOD void SDDbgValue::dump() const {
+  if (isInvalidated())
+    return;
+  print(dbgs());
+  dbgs() << "\n";
+}
+#endif
+
 /// Return true if this node is so simple that we should just print it inline
 /// if it appears as an operand.
-static bool shouldPrintInline(const SDNode &Node) {
+static bool shouldPrintInline(const SDNode &Node, const SelectionDAG *G) {
+  // Avoid lots of cluttering when inline printing nodes with associated
+  // DbgValues in verbose mode.
+  if (VerboseDAGDumping && G && !G->GetDbgValues(&Node).empty())
+    return false;
   if (Node.getOpcode() == ISD::EntryToken)
     return false;
   return Node.getNumOperands() == 0;
@@ -731,7 +783,7 @@ static bool shouldPrintInline(const SDNode &Node) {
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
 static void DumpNodes(const SDNode *N, unsigned indent, const SelectionDAG *G) {
   for (const SDValue &Op : N->op_values()) {
-    if (shouldPrintInline(*Op.getNode()))
+    if (shouldPrintInline(*Op.getNode(), G))
       continue;
     if (Op.getNode()->hasOneUse())
       DumpNodes(Op.getNode(), indent+2, G);
@@ -748,12 +800,24 @@ LLVM_DUMP_METHOD void SelectionDAG::dump() const {
        I != E; ++I) {
     const SDNode *N = &*I;
     if (!N->hasOneUse() && N != getRoot().getNode() &&
-        (!shouldPrintInline(*N) || N->use_empty()))
+        (!shouldPrintInline(*N, this) || N->use_empty()))
       DumpNodes(N, 2, this);
   }
 
   if (getRoot().getNode()) DumpNodes(getRoot().getNode(), 2, this);
-  dbgs() << "\n\n";
+  dbgs() << "\n";
+
+  if (VerboseDAGDumping) {
+    if (DbgBegin() != DbgEnd())
+      dbgs() << "SDDbgValues:\n";
+    for (auto *Dbg : make_range(DbgBegin(), DbgEnd()))
+      Dbg->dump();
+    if (ByvalParmDbgBegin() != ByvalParmDbgEnd())
+      dbgs() << "Byval SDDbgValues:\n";
+    for (auto *Dbg : make_range(ByvalParmDbgBegin(), ByvalParmDbgEnd()))
+      Dbg->dump();
+  }
+  dbgs() << "\n";
 }
 #endif
 
@@ -769,7 +833,7 @@ static bool printOperand(raw_ostream &OS, const SelectionDAG *G,
   if (!Value.getNode()) {
     OS << "<null>";
     return false;
-  } else if (shouldPrintInline(*Value.getNode())) {
+  } else if (shouldPrintInline(*Value.getNode(), G)) {
     OS << Value->getOperationName(G) << ':';
     Value->print_types(OS, G);
     Value->print_details(OS, G);
diff --git a/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp b/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
index 7d341dae210b1e532b6cfc3654dc50345e2ddc50..173919378ae35dba00344171a511b1a5eed08ca9 100644
--- a/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
+++ b/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
@@ -1,9 +1,8 @@
 //===- SelectionDAGISel.cpp - Implement the SelectionDAGISel class --------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -42,6 +41,7 @@
 #include "llvm/CodeGen/MachineInstr.h"
 #include "llvm/CodeGen/MachineInstrBuilder.h"
 #include "llvm/CodeGen/MachineMemOperand.h"
+#include "llvm/CodeGen/MachineModuleInfo.h"
 #include "llvm/CodeGen/MachineOperand.h"
 #include "llvm/CodeGen/MachinePassRegistry.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
@@ -63,6 +63,7 @@
 #include "llvm/IR/Dominators.h"
 #include "llvm/IR/Function.h"
 #include "llvm/IR/InlineAsm.h"
+#include "llvm/IR/InstIterator.h"
 #include "llvm/IR/InstrTypes.h"
 #include "llvm/IR/Instruction.h"
 #include "llvm/IR/Instructions.h"
@@ -379,6 +380,30 @@ static void SplitCriticalSideEffectEdges(Function &Fn, DominatorTree *DT,
   }
 }
 
+static void computeUsesMSVCFloatingPoint(const Triple &TT, const Function &F,
+                                         MachineModuleInfo &MMI) {
+  // Only needed for MSVC
+  if (!TT.isKnownWindowsMSVCEnvironment())
+    return;
+
+  // If it's already set, nothing to do.
+  if (MMI.usesMSVCFloatingPoint())
+    return;
+
+  for (const Instruction &I : instructions(F)) {
+    if (I.getType()->isFPOrFPVectorTy()) {
+      MMI.setUsesMSVCFloatingPoint(true);
+      return;
+    }
+    for (const auto &Op : I.operands()) {
+      if (Op->getType()->isFPOrFPVectorTy()) {
+        MMI.setUsesMSVCFloatingPoint(true);
+        return;
+      }
+    }
+  }
+}
+
 bool SelectionDAGISel::runOnMachineFunction(MachineFunction &mf) {
   // If we already selected that function, we do not need to run SDISel.
   if (mf.getProperties().hasProperty(
@@ -590,6 +615,9 @@ bool SelectionDAGISel::runOnMachineFunction(MachineFunction &mf) {
   // Determine if there is a call to setjmp in the machine function.
   MF->setExposesReturnsTwice(Fn.callsFunctionThatReturnsTwice());
 
+  // Determine if floating point is used for msvc
+  computeUsesMSVCFloatingPoint(TM.getTargetTriple(), Fn, MF->getMMI());
+
   // Replace forward-declared registers with the registers containing
   // the desired value.
   MachineRegisterInfo &MRI = MF->getRegInfo();
@@ -663,6 +691,7 @@ void SelectionDAGISel::SelectBasicBlock(BasicBlock::const_iterator Begin,
   // Make sure the root of the DAG is up-to-date.
   CurDAG->setRoot(SDB->getControlRoot());
   HadTailCall = SDB->HasTailCall;
+  SDB->resolveOrClearDbgInfo();
   SDB->clear();
 
   // Final step, emit the lowered DAG as machine code.
@@ -704,7 +733,7 @@ void SelectionDAGISel::ComputeLiveOutVRegInfo() {
       continue;
 
     unsigned NumSignBits = CurDAG->ComputeNumSignBits(Src);
-    CurDAG->computeKnownBits(Src, Known);
+    Known = CurDAG->computeKnownBits(Src);
     FuncInfo->AddLiveOutRegInfo(DestReg, NumSignBits, Known);
   } while (!Worklist.empty());
 }
@@ -2211,9 +2240,7 @@ bool SelectionDAGISel::CheckOrMask(SDValue LHS, ConstantSDNode *RHS,
   // Otherwise, the DAG Combiner may have proven that the value coming in is
   // either already zero or is not demanded.  Check for known zero input bits.
   APInt NeededMask = DesiredMask & ~ActualMask;
-
-  KnownBits Known;
-  CurDAG->computeKnownBits(LHS, Known);
+  KnownBits Known = CurDAG->computeKnownBits(LHS);
 
   // If all the missing bits in the or are already known to be set, match!
   if (NeededMask.isSubsetOf(Known.One))
@@ -2415,14 +2442,14 @@ bool SelectionDAGISel::IsLegalToFold(SDValue N, SDNode *U, SDNode *Root,
   return !findNonImmUse(Root, N.getNode(), U, IgnoreChains);
 }
 
-void SelectionDAGISel::Select_INLINEASM(SDNode *N) {
+void SelectionDAGISel::Select_INLINEASM(SDNode *N, bool Branch) {
   SDLoc DL(N);
 
   std::vector<SDValue> Ops(N->op_begin(), N->op_end());
   SelectInlineAsmMemoryOperands(Ops, DL);
 
   const EVT VTs[] = {MVT::Other, MVT::Glue};
-  SDValue New = CurDAG->getNode(ISD::INLINEASM, DL, VTs, Ops);
+  SDValue New = CurDAG->getNode(Branch ? ISD::INLINEASM_BR : ISD::INLINEASM, DL, VTs, Ops);
   New->setNodeId(-1);
   ReplaceUses(N, New.getNode());
   CurDAG->RemoveDeadNode(N);
@@ -2972,7 +2999,9 @@ void SelectionDAGISel::SelectCodeCommon(SDNode *NodeToMatch,
     CurDAG->RemoveDeadNode(NodeToMatch);
     return;
   case ISD::INLINEASM:
-    Select_INLINEASM(NodeToMatch);
+  case ISD::INLINEASM_BR:
+    Select_INLINEASM(NodeToMatch,
+                     NodeToMatch->getOpcode() == ISD::INLINEASM_BR);
     return;
   case ISD::READ_REGISTER:
     Select_READ_REGISTER(NodeToMatch);
diff --git a/lib/CodeGen/SelectionDAG/SelectionDAGPrinter.cpp b/lib/CodeGen/SelectionDAG/SelectionDAGPrinter.cpp
index 3b19bff4743d12c0452e6eb3b68495d113f34c3d..cdc09d59f6a4929f72102b94e2bfb88aeceed219 100644
--- a/lib/CodeGen/SelectionDAG/SelectionDAGPrinter.cpp
+++ b/lib/CodeGen/SelectionDAG/SelectionDAGPrinter.cpp
@@ -1,9 +1,8 @@
 //===-- SelectionDAGPrinter.cpp - Implement SelectionDAG::viewGraph() -----===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/SelectionDAG/SelectionDAGTargetInfo.cpp b/lib/CodeGen/SelectionDAG/SelectionDAGTargetInfo.cpp
index 3a283bc5fdc0cfbb282368353e99f2d068d0caba..3a2df6f60593a90ae7968def0355334cdc9cfc63 100644
--- a/lib/CodeGen/SelectionDAG/SelectionDAGTargetInfo.cpp
+++ b/lib/CodeGen/SelectionDAG/SelectionDAGTargetInfo.cpp
@@ -1,9 +1,8 @@
 //===- SelectionDAGTargetInfo.cpp - SelectionDAG Info ---------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/SelectionDAG/StatepointLowering.cpp b/lib/CodeGen/SelectionDAG/StatepointLowering.cpp
index 90a1b350fc942e1bbb426066c7c71c3ecd83d3b8..6c72a9c225a54589e6b3bd5681e6b22efce53db9 100644
--- a/lib/CodeGen/SelectionDAG/StatepointLowering.cpp
+++ b/lib/CodeGen/SelectionDAG/StatepointLowering.cpp
@@ -1,9 +1,8 @@
 //===- StatepointLowering.cpp - SDAGBuilder's statepoint code -------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -367,10 +366,6 @@ spillIncomingStatepointValue(SDValue Incoming, SDValue Chain,
     // We use TargetFrameIndex so that isel will not select it into LEA
     Loc = Builder.DAG.getTargetFrameIndex(Index, Builder.getFrameIndexTy());
 
-    // TODO: We can create TokenFactor node instead of
-    //       chaining stores one after another, this may allow
-    //       a bit more optimal scheduling for them
-
 #ifndef NDEBUG
     // Right now we always allocate spill slots that are of the same
     // size as the value we're about to spill (the size of spillee can
@@ -399,6 +394,9 @@ spillIncomingStatepointValue(SDValue Incoming, SDValue Chain,
 static void lowerIncomingStatepointValue(SDValue Incoming, bool LiveInOnly,
                                          SmallVectorImpl<SDValue> &Ops,
                                          SelectionDAGBuilder &Builder) {
+  // Note: We know all of these spills are independent, but don't bother to
+  // exploit that chain wise.  DAGCombine will happily do so as needed, so
+  // doing it here would be a small compile time win at most.
   SDValue Chain = Builder.getRoot();
 
   if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Incoming)) {
@@ -799,7 +797,7 @@ SDValue SelectionDAGBuilder::LowerAsSTATEPOINT(
 void
 SelectionDAGBuilder::LowerStatepoint(ImmutableStatepoint ISP,
                                      const BasicBlock *EHPadBB /*= nullptr*/) {
-  assert(ISP.getCallSite().getCallingConv() != CallingConv::AnyReg &&
+  assert(ISP.getCall()->getCallingConv() != CallingConv::AnyReg &&
          "anyregcc is not supported on statepoints!");
 
 #ifndef NDEBUG
@@ -832,7 +830,7 @@ SelectionDAGBuilder::LowerStatepoint(ImmutableStatepoint ISP,
   }
 
   StatepointLoweringInfo SI(DAG);
-  populateCallLoweringInfo(SI.CLI, ISP.getCallSite(),
+  populateCallLoweringInfo(SI.CLI, ISP.getCall(),
                            ImmutableStatepoint::CallArgsBeginPos,
                            ISP.getNumCallArgs(), ActualCallee,
                            ISP.getActualReturnType(), false /* IsPatchPoint */);
@@ -859,7 +857,7 @@ SelectionDAGBuilder::LowerStatepoint(ImmutableStatepoint ISP,
   const GCResultInst *GCResult = ISP.getGCResult();
   Type *RetTy = ISP.getActualReturnType();
   if (!RetTy->isVoidTy() && GCResult) {
-    if (GCResult->getParent() != ISP.getCallSite().getParent()) {
+    if (GCResult->getParent() != ISP.getCall()->getParent()) {
       // Result value will be used in a different basic block so we need to
       // export it now.  Default exporting mechanism will not work here because
       // statepoint call has a different type than the actual call. It means
@@ -871,7 +869,7 @@ SelectionDAGBuilder::LowerStatepoint(ImmutableStatepoint ISP,
       unsigned Reg = FuncInfo.CreateRegs(RetTy);
       RegsForValue RFV(*DAG.getContext(), DAG.getTargetLoweringInfo(),
                        DAG.getDataLayout(), Reg, RetTy,
-                       ISP.getCallSite().getCallingConv());
+                       ISP.getCall()->getCallingConv());
       SDValue Chain = DAG.getEntryNode();
 
       RFV.getCopyToRegs(ReturnValue, DAG, getCurSDLoc(), Chain, nullptr);
@@ -891,22 +889,22 @@ SelectionDAGBuilder::LowerStatepoint(ImmutableStatepoint ISP,
 }
 
 void SelectionDAGBuilder::LowerCallSiteWithDeoptBundleImpl(
-    ImmutableCallSite CS, SDValue Callee, const BasicBlock *EHPadBB,
+    const CallBase *Call, SDValue Callee, const BasicBlock *EHPadBB,
     bool VarArgDisallowed, bool ForceVoidReturnTy) {
   StatepointLoweringInfo SI(DAG);
-  unsigned ArgBeginIndex = CS.arg_begin() - CS.getInstruction()->op_begin();
+  unsigned ArgBeginIndex = Call->arg_begin() - Call->op_begin();
   populateCallLoweringInfo(
-      SI.CLI, CS, ArgBeginIndex, CS.getNumArgOperands(), Callee,
-      ForceVoidReturnTy ? Type::getVoidTy(*DAG.getContext()) : CS.getType(),
+      SI.CLI, Call, ArgBeginIndex, Call->getNumArgOperands(), Callee,
+      ForceVoidReturnTy ? Type::getVoidTy(*DAG.getContext()) : Call->getType(),
       false);
   if (!VarArgDisallowed)
-    SI.CLI.IsVarArg = CS.getFunctionType()->isVarArg();
+    SI.CLI.IsVarArg = Call->getFunctionType()->isVarArg();
 
-  auto DeoptBundle = *CS.getOperandBundle(LLVMContext::OB_deopt);
+  auto DeoptBundle = *Call->getOperandBundle(LLVMContext::OB_deopt);
 
   unsigned DefaultID = StatepointDirectives::DeoptBundleStatepointID;
 
-  auto SD = parseStatepointDirectivesFromAttrs(CS.getAttributes());
+  auto SD = parseStatepointDirectivesFromAttrs(Call->getAttributes());
   SI.ID = SD.StatepointID.getValueOr(DefaultID);
   SI.NumPatchBytes = SD.NumPatchBytes.getValueOr(0);
 
@@ -918,15 +916,14 @@ void SelectionDAGBuilder::LowerCallSiteWithDeoptBundleImpl(
   // NB! The GC arguments are deliberately left empty.
 
   if (SDValue ReturnVal = LowerAsSTATEPOINT(SI)) {
-    const Instruction *Inst = CS.getInstruction();
-    ReturnVal = lowerRangeToAssertZExt(DAG, *Inst, ReturnVal);
-    setValue(Inst, ReturnVal);
+    ReturnVal = lowerRangeToAssertZExt(DAG, *Call, ReturnVal);
+    setValue(Call, ReturnVal);
   }
 }
 
 void SelectionDAGBuilder::LowerCallSiteWithDeoptBundle(
-    ImmutableCallSite CS, SDValue Callee, const BasicBlock *EHPadBB) {
-  LowerCallSiteWithDeoptBundleImpl(CS, Callee, EHPadBB,
+    const CallBase *Call, SDValue Callee, const BasicBlock *EHPadBB) {
+  LowerCallSiteWithDeoptBundleImpl(Call, Callee, EHPadBB,
                                    /* VarArgDisallowed = */ false,
                                    /* ForceVoidReturnTy  = */ false);
 }
@@ -986,11 +983,11 @@ void SelectionDAGBuilder::visitGCRelocate(const GCRelocateInst &Relocate) {
   }
 
   SDValue SpillSlot =
-      DAG.getTargetFrameIndex(*DerivedPtrLocation, getFrameIndexTy());
+    DAG.getTargetFrameIndex(*DerivedPtrLocation, getFrameIndexTy());
 
-  // Be conservative: flush all pending loads
-  // TODO: Probably we can be less restrictive on this,
-  // it may allow more scheduling opportunities.
+  // Note: We know all of these reloads are independent, but don't bother to
+  // exploit that chain wise.  DAGCombine will happily do so as needed, so
+  // doing it here would be a small compile time win at most.
   SDValue Chain = getRoot();
 
   SDValue SpillLoad =
@@ -1000,7 +997,6 @@ void SelectionDAGBuilder::visitGCRelocate(const GCRelocateInst &Relocate) {
                   MachinePointerInfo::getFixedStack(DAG.getMachineFunction(),
                                                     *DerivedPtrLocation));
 
-  // Again, be conservative, don't emit pending loads
   DAG.setRoot(SpillLoad.getValue(1));
 
   assert(SpillLoad.getNode());
diff --git a/lib/CodeGen/SelectionDAG/StatepointLowering.h b/lib/CodeGen/SelectionDAG/StatepointLowering.h
index 372c82a359f6b46ed81916defed6ceb9f2b111c8..a4fdce515b9e4a9dfd0fafee969e88de0e7874ae 100644
--- a/lib/CodeGen/SelectionDAG/StatepointLowering.h
+++ b/lib/CodeGen/SelectionDAG/StatepointLowering.h
@@ -1,9 +1,8 @@
 //===- StatepointLowering.h - SDAGBuilder's statepoint code ---*- C++ -*---===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/lib/CodeGen/SelectionDAG/TargetLowering.cpp
index 4c551d5b231ff8f56b0fd6fac193a2e9ee1a0ce9..405b596bf9d56a88ffa05d43d4c6a746816eb8f3 100644
--- a/lib/CodeGen/SelectionDAG/TargetLowering.cpp
+++ b/lib/CodeGen/SelectionDAG/TargetLowering.cpp
@@ -1,9 +1,8 @@
 //===-- TargetLowering.cpp - Implement the TargetLowering class -----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -100,19 +99,19 @@ bool TargetLowering::parametersInCSRMatch(const MachineRegisterInfo &MRI,
 
 /// Set CallLoweringInfo attribute flags based on a call instruction
 /// and called function attributes.
-void TargetLoweringBase::ArgListEntry::setAttributes(ImmutableCallSite *CS,
+void TargetLoweringBase::ArgListEntry::setAttributes(const CallBase *Call,
                                                      unsigned ArgIdx) {
-  IsSExt = CS->paramHasAttr(ArgIdx, Attribute::SExt);
-  IsZExt = CS->paramHasAttr(ArgIdx, Attribute::ZExt);
-  IsInReg = CS->paramHasAttr(ArgIdx, Attribute::InReg);
-  IsSRet = CS->paramHasAttr(ArgIdx, Attribute::StructRet);
-  IsNest = CS->paramHasAttr(ArgIdx, Attribute::Nest);
-  IsByVal = CS->paramHasAttr(ArgIdx, Attribute::ByVal);
-  IsInAlloca = CS->paramHasAttr(ArgIdx, Attribute::InAlloca);
-  IsReturned = CS->paramHasAttr(ArgIdx, Attribute::Returned);
-  IsSwiftSelf = CS->paramHasAttr(ArgIdx, Attribute::SwiftSelf);
-  IsSwiftError = CS->paramHasAttr(ArgIdx, Attribute::SwiftError);
-  Alignment  = CS->getParamAlignment(ArgIdx);
+  IsSExt = Call->paramHasAttr(ArgIdx, Attribute::SExt);
+  IsZExt = Call->paramHasAttr(ArgIdx, Attribute::ZExt);
+  IsInReg = Call->paramHasAttr(ArgIdx, Attribute::InReg);
+  IsSRet = Call->paramHasAttr(ArgIdx, Attribute::StructRet);
+  IsNest = Call->paramHasAttr(ArgIdx, Attribute::Nest);
+  IsByVal = Call->paramHasAttr(ArgIdx, Attribute::ByVal);
+  IsInAlloca = Call->paramHasAttr(ArgIdx, Attribute::InAlloca);
+  IsReturned = Call->paramHasAttr(ArgIdx, Attribute::Returned);
+  IsSwiftSelf = Call->paramHasAttr(ArgIdx, Attribute::SwiftSelf);
+  IsSwiftError = Call->paramHasAttr(ArgIdx, Attribute::SwiftError);
+  Alignment = Call->getParamAlignment(ArgIdx);
 }
 
 /// Generate a libcall taking the given operands as arguments and returning a
@@ -431,56 +430,6 @@ bool TargetLowering::ShrinkDemandedOp(SDValue Op, unsigned BitWidth,
   return false;
 }
 
-bool
-TargetLowering::SimplifyDemandedBits(SDNode *User, unsigned OpIdx,
-                                     const APInt &DemandedBits,
-                                     DAGCombinerInfo &DCI,
-                                     TargetLoweringOpt &TLO) const {
-  SDValue Op = User->getOperand(OpIdx);
-  KnownBits Known;
-
-  if (!SimplifyDemandedBits(Op, DemandedBits, Known, TLO, 0, true))
-    return false;
-
-
-  // Old will not always be the same as Op.  For example:
-  //
-  // Demanded = 0xffffff
-  // Op = i64 truncate (i32 and x, 0xffffff)
-  // In this case simplify demand bits will want to replace the 'and' node
-  // with the value 'x', which will give us:
-  // Old = i32 and x, 0xffffff
-  // New = x
-  if (TLO.Old.hasOneUse()) {
-    // For the one use case, we just commit the change.
-    DCI.CommitTargetLoweringOpt(TLO);
-    return true;
-  }
-
-  // If Old has more than one use then it must be Op, because the
-  // AssumeSingleUse flag is not propogated to recursive calls of
-  // SimplifyDemanded bits, so the only node with multiple use that
-  // it will attempt to combine will be Op.
-  assert(TLO.Old == Op);
-
-  SmallVector <SDValue, 4> NewOps;
-  for (unsigned i = 0, e = User->getNumOperands(); i != e; ++i) {
-    if (i == OpIdx) {
-      NewOps.push_back(TLO.New);
-      continue;
-    }
-    NewOps.push_back(User->getOperand(i));
-  }
-  User = TLO.DAG.UpdateNodeOperands(User, NewOps);
-  // Op has less users now, so we may be able to perform additional combines
-  // with it.
-  DCI.AddToWorklist(Op.getNode());
-  // User's operands have been updated, so we may be able to do new combines
-  // with it.
-  DCI.AddToWorklist(User);
-  return true;
-}
-
 bool TargetLowering::SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits,
                                           DAGCombinerInfo &DCI) const {
   SelectionDAG &DAG = DCI.DAG;
@@ -550,7 +499,7 @@ bool TargetLowering::SimplifyDemandedBits(
     if (Depth != 0) {
       // If not at the root, Just compute the Known bits to
       // simplify things downstream.
-      TLO.DAG.computeKnownBits(Op, Known, DemandedElts, Depth);
+      Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
       return false;
     }
     // If this is the root being simplified, allow it to have multiple uses,
@@ -666,9 +615,8 @@ bool TargetLowering::SimplifyDemandedBits(
     // simplify the LHS, here we're using information from the LHS to simplify
     // the RHS.
     if (ConstantSDNode *RHSC = isConstOrConstSplat(Op1)) {
-      KnownBits LHSKnown;
       // Do not increment Depth here; that can cause an infinite loop.
-      TLO.DAG.computeKnownBits(Op0, LHSKnown, DemandedElts, Depth);
+      KnownBits LHSKnown = TLO.DAG.computeKnownBits(Op0, DemandedElts, Depth);
       // If the LHS already has zeros where RHSC does, this 'and' is dead.
       if ((LHSKnown.Zero & DemandedBits) ==
           (~RHSC->getAPIntValue() & DemandedBits))
@@ -1077,6 +1025,45 @@ bool TargetLowering::SimplifyDemandedBits(
     }
     break;
   }
+  case ISD::FSHL:
+  case ISD::FSHR: {
+    SDValue Op0 = Op.getOperand(0);
+    SDValue Op1 = Op.getOperand(1);
+    SDValue Op2 = Op.getOperand(2);
+    bool IsFSHL = (Op.getOpcode() == ISD::FSHL);
+
+    if (ConstantSDNode *SA = isConstOrConstSplat(Op2)) {
+      unsigned Amt = SA->getAPIntValue().urem(BitWidth);
+
+      // For fshl, 0-shift returns the 1st arg.
+      // For fshr, 0-shift returns the 2nd arg.
+      if (Amt == 0) {
+        if (SimplifyDemandedBits(IsFSHL ? Op0 : Op1, DemandedBits, DemandedElts,
+                                 Known, TLO, Depth + 1))
+          return true;
+        break;
+      }
+
+      // fshl: (Op0 << Amt) | (Op1 >> (BW - Amt))
+      // fshr: (Op0 << (BW - Amt)) | (Op1 >> Amt)
+      APInt Demanded0 = DemandedBits.lshr(IsFSHL ? Amt : (BitWidth - Amt));
+      APInt Demanded1 = DemandedBits << (IsFSHL ? (BitWidth - Amt) : Amt);
+      if (SimplifyDemandedBits(Op0, Demanded0, DemandedElts, Known2, TLO,
+                               Depth + 1))
+        return true;
+      if (SimplifyDemandedBits(Op1, Demanded1, DemandedElts, Known, TLO,
+                               Depth + 1))
+        return true;
+
+      Known2.One <<= (IsFSHL ? Amt : (BitWidth - Amt));
+      Known2.Zero <<= (IsFSHL ? Amt : (BitWidth - Amt));
+      Known.One.lshrInPlace(IsFSHL ? (BitWidth - Amt) : Amt);
+      Known.Zero.lshrInPlace(IsFSHL ? (BitWidth - Amt) : Amt);
+      Known.One |= Known2.One;
+      Known.Zero |= Known2.Zero;
+    }
+    break;
+  }
   case ISD::SIGN_EXTEND_INREG: {
     SDValue Op0 = Op.getOperand(0);
     EVT ExVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
@@ -1381,7 +1368,7 @@ bool TargetLowering::SimplifyDemandedBits(
     // If this is a bitcast, let computeKnownBits handle it.  Only do this on a
     // recursive call where Known may be useful to the caller.
     if (Depth > 0) {
-      TLO.DAG.computeKnownBits(Op, Known, Depth);
+      Known = TLO.DAG.computeKnownBits(Op, Depth);
       return false;
     }
     break;
@@ -1442,7 +1429,7 @@ bool TargetLowering::SimplifyDemandedBits(
     }
 
     // Just use computeKnownBits to compute output bits.
-    TLO.DAG.computeKnownBits(Op, Known, DemandedElts, Depth);
+    Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
     break;
   }
 
@@ -1485,6 +1472,53 @@ bool TargetLowering::SimplifyDemandedVectorElts(SDValue Op,
   return Simplified;
 }
 
+/// Given a vector binary operation and known undefined elements for each input
+/// operand, compute whether each element of the output is undefined.
+static APInt getKnownUndefForVectorBinop(SDValue BO, SelectionDAG &DAG,
+                                         const APInt &UndefOp0,
+                                         const APInt &UndefOp1) {
+  EVT VT = BO.getValueType();
+  assert(ISD::isBinaryOp(BO.getNode()) && VT.isVector() && "Vector binop only");
+
+  EVT EltVT = VT.getVectorElementType();
+  unsigned NumElts = VT.getVectorNumElements();
+  assert(UndefOp0.getBitWidth() == NumElts &&
+         UndefOp1.getBitWidth() == NumElts && "Bad type for undef analysis");
+
+  auto getUndefOrConstantElt = [&](SDValue V, unsigned Index,
+                                   const APInt &UndefVals) {
+    if (UndefVals[Index])
+      return DAG.getUNDEF(EltVT);
+
+    if (auto *BV = dyn_cast<BuildVectorSDNode>(V)) {
+      // Try hard to make sure that the getNode() call is not creating temporary
+      // nodes. Ignore opaque integers because they do not constant fold.
+      SDValue Elt = BV->getOperand(Index);
+      auto *C = dyn_cast<ConstantSDNode>(Elt);
+      if (isa<ConstantFPSDNode>(Elt) || Elt.isUndef() || (C && !C->isOpaque()))
+        return Elt;
+    }
+
+    return SDValue();
+  };
+
+  APInt KnownUndef = APInt::getNullValue(NumElts);
+  for (unsigned i = 0; i != NumElts; ++i) {
+    // If both inputs for this element are either constant or undef and match
+    // the element type, compute the constant/undef result for this element of
+    // the vector.
+    // TODO: Ideally we would use FoldConstantArithmetic() here, but that does
+    // not handle FP constants. The code within getNode() should be refactored
+    // to avoid the danger of creating a bogus temporary node here.
+    SDValue C0 = getUndefOrConstantElt(BO.getOperand(0), i, UndefOp0);
+    SDValue C1 = getUndefOrConstantElt(BO.getOperand(1), i, UndefOp1);
+    if (C0 && C1 && C0.getValueType() == EltVT && C1.getValueType() == EltVT)
+      if (DAG.getNode(BO.getOpcode(), SDLoc(BO), EltVT, C0, C1).isUndef())
+        KnownUndef.setBit(i);
+  }
+  return KnownUndef;
+}
+
 bool TargetLowering::SimplifyDemandedVectorElts(
     SDValue Op, const APInt &DemandedEltMask, APInt &KnownUndef,
     APInt &KnownZero, TargetLoweringOpt &TLO, unsigned Depth,
@@ -1848,8 +1882,18 @@ bool TargetLowering::SimplifyDemandedVectorElts(
       return true;
     KnownZero = SrcZero.zextOrTrunc(NumElts);
     KnownUndef = SrcUndef.zextOrTrunc(NumElts);
+
+    if (Op.getOpcode() == ISD::ZERO_EXTEND_VECTOR_INREG) {
+      // zext(undef) upper bits are guaranteed to be zero.
+      if (DemandedElts.isSubsetOf(KnownUndef))
+        return TLO.CombineTo(Op, TLO.DAG.getConstant(0, SDLoc(Op), VT));
+      KnownUndef.clearAllBits();
+    }
     break;
   }
+
+  // TODO: There are more binop opcodes that could be handled here - MUL, MIN,
+  // MAX, saturated math, etc.
   case ISD::OR:
   case ISD::XOR:
   case ISD::ADD:
@@ -1859,15 +1903,17 @@ bool TargetLowering::SimplifyDemandedVectorElts(
   case ISD::FMUL:
   case ISD::FDIV:
   case ISD::FREM: {
-    APInt SrcUndef, SrcZero;
-    if (SimplifyDemandedVectorElts(Op.getOperand(1), DemandedElts, SrcUndef,
-                                   SrcZero, TLO, Depth + 1))
+    APInt UndefRHS, ZeroRHS;
+    if (SimplifyDemandedVectorElts(Op.getOperand(1), DemandedElts, UndefRHS,
+                                   ZeroRHS, TLO, Depth + 1))
       return true;
-    if (SimplifyDemandedVectorElts(Op.getOperand(0), DemandedElts, KnownUndef,
-                                   KnownZero, TLO, Depth + 1))
+    APInt UndefLHS, ZeroLHS;
+    if (SimplifyDemandedVectorElts(Op.getOperand(0), DemandedElts, UndefLHS,
+                                   ZeroLHS, TLO, Depth + 1))
       return true;
-    KnownZero &= SrcZero;
-    KnownUndef &= SrcUndef;
+
+    KnownZero = ZeroLHS & ZeroRHS;
+    KnownUndef = getKnownUndefForVectorBinop(Op, TLO.DAG, UndefLHS, UndefRHS);
     break;
   }
   case ISD::AND: {
@@ -1881,6 +1927,8 @@ bool TargetLowering::SimplifyDemandedVectorElts(
 
     // If either side has a zero element, then the result element is zero, even
     // if the other is an UNDEF.
+    // TODO: Extend getKnownUndefForVectorBinop to also deal with known zeros
+    // and then handle 'and' nodes with the rest of the binop opcodes.
     KnownZero |= SrcZero;
     KnownUndef &= SrcUndef;
     KnownUndef &= ~KnownZero;
@@ -1892,6 +1940,13 @@ bool TargetLowering::SimplifyDemandedVectorElts(
     if (SimplifyDemandedVectorElts(Op.getOperand(0), DemandedElts, KnownUndef,
                                    KnownZero, TLO, Depth + 1))
       return true;
+
+    if (Op.getOpcode() == ISD::ZERO_EXTEND) {
+      // zext(undef) upper bits are guaranteed to be zero.
+      if (DemandedElts.isSubsetOf(KnownUndef))
+        return TLO.CombineTo(Op, TLO.DAG.getConstant(0, SDLoc(Op), VT));
+      KnownUndef.clearAllBits();
+    }
     break;
   default: {
     if (Op.getOpcode() >= ISD::BUILTIN_OP_END) {
@@ -2081,10 +2136,9 @@ bool TargetLowering::isExtendedTrueVal(const ConstantSDNode *N, EVT VT,
 
 /// This helper function of SimplifySetCC tries to optimize the comparison when
 /// either operand of the SetCC node is a bitwise-and instruction.
-SDValue TargetLowering::simplifySetCCWithAnd(EVT VT, SDValue N0, SDValue N1,
-                                             ISD::CondCode Cond,
-                                             DAGCombinerInfo &DCI,
-                                             const SDLoc &DL) const {
+SDValue TargetLowering::foldSetCCWithAnd(EVT VT, SDValue N0, SDValue N1,
+                                         ISD::CondCode Cond, const SDLoc &DL,
+                                         DAGCombinerInfo &DCI) const {
   // Match these patterns in any of their permutations:
   // (X & Y) == Y
   // (X & Y) != Y
@@ -2237,6 +2291,49 @@ SDValue TargetLowering::optimizeSetCCOfSignedTruncationCheck(
   return T2;
 }
 
+/// Try to fold an equality comparison with a {add/sub/xor} binary operation as
+/// the 1st operand (N0). Callers are expected to swap the N0/N1 parameters to
+/// handle the commuted versions of these patterns.
+SDValue TargetLowering::foldSetCCWithBinOp(EVT VT, SDValue N0, SDValue N1,
+                                           ISD::CondCode Cond, const SDLoc &DL,
+                                           DAGCombinerInfo &DCI) const {
+  unsigned BOpcode = N0.getOpcode();
+  assert((BOpcode == ISD::ADD || BOpcode == ISD::SUB || BOpcode == ISD::XOR) &&
+         "Unexpected binop");
+  assert((Cond == ISD::SETEQ || Cond == ISD::SETNE) && "Unexpected condcode");
+
+  // (X + Y) == X --> Y == 0
+  // (X - Y) == X --> Y == 0
+  // (X ^ Y) == X --> Y == 0
+  SelectionDAG &DAG = DCI.DAG;
+  EVT OpVT = N0.getValueType();
+  SDValue X = N0.getOperand(0);
+  SDValue Y = N0.getOperand(1);
+  if (X == N1)
+    return DAG.getSetCC(DL, VT, Y, DAG.getConstant(0, DL, OpVT), Cond);
+
+  if (Y != N1)
+    return SDValue();
+
+  // (X + Y) == Y --> X == 0
+  // (X ^ Y) == Y --> X == 0
+  if (BOpcode == ISD::ADD || BOpcode == ISD::XOR)
+    return DAG.getSetCC(DL, VT, X, DAG.getConstant(0, DL, OpVT), Cond);
+
+  // The shift would not be valid if the operands are boolean (i1).
+  if (!N0.hasOneUse() || OpVT.getScalarSizeInBits() == 1)
+    return SDValue();
+
+  // (X - Y) == Y --> X == Y << 1
+  EVT ShiftVT = getShiftAmountTy(OpVT, DAG.getDataLayout(),
+                                 !DCI.isBeforeLegalize());
+  SDValue One = DAG.getConstant(1, DL, ShiftVT);
+  SDValue YShl1 = DAG.getNode(ISD::SHL, DL, N1.getValueType(), Y, One);
+  if (!DCI.isCalledByLegalizer())
+    DCI.AddToWorklist(YShl1.getNode());
+  return DAG.getSetCC(DL, VT, X, YShl1, Cond);
+}
+
 /// Try to simplify a setcc built with the specified operands and cc. If it is
 /// unable to simplify it, return a null SDValue.
 SDValue TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1,
@@ -3006,62 +3103,21 @@ SDValue TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1,
           LegalRHSImm = isLegalICmpImmediate(RHSC->getSExtValue());
       }
 
-      // Simplify (X+Z) == X -->  Z == 0
+      // (X+Y) == X --> Y == 0 and similar folds.
       // Don't do this if X is an immediate that can fold into a cmp
-      // instruction and X+Z has other uses. It could be an induction variable
+      // instruction and X+Y has other uses. It could be an induction variable
       // chain, and the transform would increase register pressure.
-      if (!LegalRHSImm || N0.getNode()->hasOneUse()) {
-        if (N0.getOperand(0) == N1)
-          return DAG.getSetCC(dl, VT, N0.getOperand(1),
-                              DAG.getConstant(0, dl, N0.getValueType()), Cond);
-        if (N0.getOperand(1) == N1) {
-          if (isCommutativeBinOp(N0.getOpcode()))
-            return DAG.getSetCC(dl, VT, N0.getOperand(0),
-                                DAG.getConstant(0, dl, N0.getValueType()),
-                                Cond);
-          if (N0.getNode()->hasOneUse()) {
-            assert(N0.getOpcode() == ISD::SUB && "Unexpected operation!");
-            auto &DL = DAG.getDataLayout();
-            // (Z-X) == X  --> Z == X<<1
-            SDValue SH = DAG.getNode(
-                ISD::SHL, dl, N1.getValueType(), N1,
-                DAG.getConstant(1, dl,
-                                getShiftAmountTy(N1.getValueType(), DL,
-                                                 !DCI.isBeforeLegalize())));
-            if (!DCI.isCalledByLegalizer())
-              DCI.AddToWorklist(SH.getNode());
-            return DAG.getSetCC(dl, VT, N0.getOperand(0), SH, Cond);
-          }
-        }
-      }
+      if (!LegalRHSImm || N0.hasOneUse())
+        if (SDValue V = foldSetCCWithBinOp(VT, N0, N1, Cond, dl, DCI))
+          return V;
     }
 
     if (N1.getOpcode() == ISD::ADD || N1.getOpcode() == ISD::SUB ||
-        N1.getOpcode() == ISD::XOR) {
-      // Simplify  X == (X+Z) -->  Z == 0
-      if (N1.getOperand(0) == N0)
-        return DAG.getSetCC(dl, VT, N1.getOperand(1),
-                        DAG.getConstant(0, dl, N1.getValueType()), Cond);
-      if (N1.getOperand(1) == N0) {
-        if (isCommutativeBinOp(N1.getOpcode()))
-          return DAG.getSetCC(dl, VT, N1.getOperand(0),
-                          DAG.getConstant(0, dl, N1.getValueType()), Cond);
-        if (N1.getNode()->hasOneUse()) {
-          assert(N1.getOpcode() == ISD::SUB && "Unexpected operation!");
-          auto &DL = DAG.getDataLayout();
-          // X == (Z-X)  --> X<<1 == Z
-          SDValue SH = DAG.getNode(
-              ISD::SHL, dl, N1.getValueType(), N0,
-              DAG.getConstant(1, dl, getShiftAmountTy(N0.getValueType(), DL,
-                                                      !DCI.isBeforeLegalize())));
-          if (!DCI.isCalledByLegalizer())
-            DCI.AddToWorklist(SH.getNode());
-          return DAG.getSetCC(dl, VT, SH, N1.getOperand(0), Cond);
-        }
-      }
-    }
+        N1.getOpcode() == ISD::XOR)
+      if (SDValue V = foldSetCCWithBinOp(VT, N1, N0, Cond, dl, DCI))
+        return V;
 
-    if (SDValue V = simplifySetCCWithAnd(VT, N0, N1, Cond, DCI, dl))
+    if (SDValue V = foldSetCCWithAnd(VT, N0, N1, Cond, dl, DCI))
       return V;
   }
 
@@ -3215,6 +3271,12 @@ const char *TargetLowering::LowerXConstraint(EVT ConstraintVT) const{
   return nullptr;
 }
 
+SDValue TargetLowering::LowerAsmOutputForConstraint(
+    SDValue &Chain, SDValue *Flag, SDLoc DL, const AsmOperandInfo &OpInfo,
+    SelectionDAG &DAG) const {
+  return SDValue();
+}
+
 /// Lower the specified operand into the Ops vector.
 /// If it is invalid, don't add anything to Ops.
 void TargetLowering::LowerAsmOperandForConstraint(SDValue Op,
@@ -3228,7 +3290,8 @@ void TargetLowering::LowerAsmOperandForConstraint(SDValue Op,
   switch (ConstraintLetter) {
   default: break;
   case 'X':     // Allows any operand; labels (basic block) use this.
-    if (Op.getOpcode() == ISD::BasicBlock) {
+    if (Op.getOpcode() == ISD::BasicBlock ||
+        Op.getOpcode() == ISD::TargetBlockAddress) {
       Ops.push_back(Op);
       return;
     }
@@ -3715,6 +3778,9 @@ void TargetLowering::ComputeConstraintToUse(AsmOperandInfo &OpInfo,
       return;
     }
 
+    if (Op.getNode() && Op.getOpcode() == ISD::TargetBlockAddress)
+      return;
+
     // Otherwise, try to resolve it to something we know about by looking at
     // the actual operand type.
     if (const char *Repl = LowerXConstraint(OpInfo.ConstraintVT)) {
@@ -4357,7 +4423,7 @@ bool TargetLowering::expandFP_TO_SINT(SDNode *Node, SDValue &Result,
 
   // Expand f32 -> i64 conversion
   // This algorithm comes from compiler-rt's implementation of fixsfdi:
-  // https://github.com/llvm-mirror/compiler-rt/blob/master/lib/builtins/fixsfdi.c
+  // https://github.com/llvm/llvm-project/blob/master/compiler-rt/lib/builtins/fixsfdi.c
   unsigned SrcEltBits = SrcVT.getScalarSizeInBits();
   EVT IntVT = SrcVT.changeTypeToInteger();
   EVT IntShVT = getShiftAmountTy(IntVT, DAG.getDataLayout());
@@ -4752,6 +4818,27 @@ bool TargetLowering::expandCTTZ(SDNode *Node, SDValue &Result,
   return true;
 }
 
+bool TargetLowering::expandABS(SDNode *N, SDValue &Result,
+                               SelectionDAG &DAG) const {
+  SDLoc dl(N);
+  EVT VT = N->getValueType(0);
+  EVT ShVT = getShiftAmountTy(VT, DAG.getDataLayout());
+  SDValue Op = N->getOperand(0);
+
+  // Only expand vector types if we have the appropriate vector operations.
+  if (VT.isVector() && (!isOperationLegalOrCustom(ISD::SRA, VT) ||
+                        !isOperationLegalOrCustom(ISD::ADD, VT) ||
+                        !isOperationLegalOrCustomOrPromote(ISD::XOR, VT)))
+    return false;
+
+  SDValue Shift =
+      DAG.getNode(ISD::SRA, dl, VT, Op,
+                  DAG.getConstant(VT.getScalarSizeInBits() - 1, dl, ShVT));
+  SDValue Add = DAG.getNode(ISD::ADD, dl, VT, Op, Shift);
+  Result = DAG.getNode(ISD::XOR, dl, VT, Add, Shift);
+  return true;
+}
+
 SDValue TargetLowering::scalarizeVectorLoad(LoadSDNode *LD,
                                             SelectionDAG &DAG) const {
   SDLoc SL(LD);
@@ -5291,9 +5378,30 @@ SDValue TargetLowering::lowerCmpEqZeroToCtlzSrl(SDValue Op,
   return SDValue();
 }
 
-SDValue TargetLowering::getExpandedSaturationAdditionSubtraction(
-    SDNode *Node, SelectionDAG &DAG) const {
+SDValue TargetLowering::expandAddSubSat(SDNode *Node, SelectionDAG &DAG) const {
   unsigned Opcode = Node->getOpcode();
+  SDValue LHS = Node->getOperand(0);
+  SDValue RHS = Node->getOperand(1);
+  EVT VT = LHS.getValueType();
+  SDLoc dl(Node);
+
+  assert(LHS.getValueType().isInteger() && "Expected operands to be integers");
+  assert(RHS.getValueType().isInteger() && "Expected operands to be integers");
+  assert(LHS.getValueType() == RHS.getValueType() &&
+         "Expected both operands to be the same type");
+
+  // usub.sat(a, b) -> umax(a, b) - b
+  if (Opcode == ISD::USUBSAT && isOperationLegalOrCustom(ISD::UMAX, VT)) {
+    SDValue Max = DAG.getNode(ISD::UMAX, dl, VT, LHS, RHS);
+    return DAG.getNode(ISD::SUB, dl, VT, Max, RHS);
+  }
+
+  if (Opcode == ISD::UADDSAT && isOperationLegalOrCustom(ISD::UMIN, VT)) {
+    SDValue InvRHS = DAG.getNOT(dl, RHS, VT);
+    SDValue Min = DAG.getNode(ISD::UMIN, dl, VT, LHS, InvRHS);
+    return DAG.getNode(ISD::ADD, dl, VT, Min, RHS);
+  }
+
   unsigned OverflowOp;
   switch (Opcode) {
   case ISD::SADDSAT:
@@ -5312,21 +5420,8 @@ SDValue TargetLowering::getExpandedSaturationAdditionSubtraction(
     llvm_unreachable("Expected method to receive signed or unsigned saturation "
                      "addition or subtraction node.");
   }
-  assert(Node->getNumOperands() == 2 && "Expected node to have 2 operands.");
-
-  SDLoc dl(Node);
-  SDValue LHS = Node->getOperand(0);
-  SDValue RHS = Node->getOperand(1);
-  assert(LHS.getValueType().isScalarInteger() &&
-         "Expected operands to be integers. Vector of int arguments should "
-         "already be unrolled.");
-  assert(RHS.getValueType().isScalarInteger() &&
-         "Expected operands to be integers. Vector of int arguments should "
-         "already be unrolled.");
-  assert(LHS.getValueType() == RHS.getValueType() &&
-         "Expected both operands to be the same type");
 
-  unsigned BitWidth = LHS.getValueSizeInBits();
+  unsigned BitWidth = LHS.getScalarValueSizeInBits();
   EVT ResultType = LHS.getValueType();
   EVT BoolVT =
       getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), ResultType);
@@ -5360,53 +5455,59 @@ SDValue TargetLowering::getExpandedSaturationAdditionSubtraction(
 }
 
 SDValue
-TargetLowering::getExpandedFixedPointMultiplication(SDNode *Node,
-                                                    SelectionDAG &DAG) const {
-  assert(Node->getOpcode() == ISD::SMULFIX && "Expected opcode to be SMULFIX.");
-  assert(Node->getNumOperands() == 3 &&
-         "Expected signed fixed point multiplication to have 3 operands.");
+TargetLowering::expandFixedPointMul(SDNode *Node, SelectionDAG &DAG) const {
+  assert((Node->getOpcode() == ISD::SMULFIX ||
+          Node->getOpcode() == ISD::UMULFIX) &&
+         "Expected opcode to be SMULFIX or UMULFIX.");
 
   SDLoc dl(Node);
   SDValue LHS = Node->getOperand(0);
   SDValue RHS = Node->getOperand(1);
-  assert(LHS.getValueType().isScalarInteger() &&
-         "Expected operands to be integers. Vector of int arguments should "
-         "already be unrolled.");
-  assert(RHS.getValueType().isScalarInteger() &&
-         "Expected operands to be integers. Vector of int arguments should "
-         "already be unrolled.");
-  assert(LHS.getValueType() == RHS.getValueType() &&
-         "Expected both operands to be the same type");
-
-  unsigned Scale = Node->getConstantOperandVal(2);
   EVT VT = LHS.getValueType();
-  assert(Scale < VT.getScalarSizeInBits() &&
-         "Expected scale to be less than the number of bits.");
+  unsigned Scale = Node->getConstantOperandVal(2);
 
-  if (!Scale)
+  // [us]mul.fix(a, b, 0) -> mul(a, b)
+  if (!Scale) {
+    if (VT.isVector() && !isOperationLegalOrCustom(ISD::MUL, VT))
+      return SDValue();
     return DAG.getNode(ISD::MUL, dl, VT, LHS, RHS);
+  }
+
+  unsigned VTSize = VT.getScalarSizeInBits();
+  bool Signed = Node->getOpcode() == ISD::SMULFIX;
+
+  assert(((Signed && Scale < VTSize) || (!Signed && Scale <= VTSize)) &&
+         "Expected scale to be less than the number of bits if signed or at "
+         "most the number of bits if unsigned.");
+  assert(LHS.getValueType() == RHS.getValueType() &&
+         "Expected both operands to be the same type");
 
   // Get the upper and lower bits of the result.
   SDValue Lo, Hi;
-  if (isOperationLegalOrCustom(ISD::SMUL_LOHI, VT)) {
-    SDValue Result =
-        DAG.getNode(ISD::SMUL_LOHI, dl, DAG.getVTList(VT, VT), LHS, RHS);
+  unsigned LoHiOp = Signed ? ISD::SMUL_LOHI : ISD::UMUL_LOHI;
+  unsigned HiOp = Signed ? ISD::MULHS : ISD::MULHU;
+  if (isOperationLegalOrCustom(LoHiOp, VT)) {
+    SDValue Result = DAG.getNode(LoHiOp, dl, DAG.getVTList(VT, VT), LHS, RHS);
     Lo = Result.getValue(0);
     Hi = Result.getValue(1);
-  } else if (isOperationLegalOrCustom(ISD::MULHS, VT)) {
+  } else if (isOperationLegalOrCustom(HiOp, VT)) {
     Lo = DAG.getNode(ISD::MUL, dl, VT, LHS, RHS);
-    Hi = DAG.getNode(ISD::MULHS, dl, VT, LHS, RHS);
+    Hi = DAG.getNode(HiOp, dl, VT, LHS, RHS);
+  } else if (VT.isVector()) {
+    return SDValue();
   } else {
-    report_fatal_error("Unable to expand signed fixed point multiplication.");
+    report_fatal_error("Unable to expand fixed point multiplication.");
   }
 
+  if (Scale == VTSize)
+    // Result is just the top half since we'd be shifting by the width of the
+    // operand.
+    return Hi;
+
   // The result will need to be shifted right by the scale since both operands
   // are scaled. The result is given to us in 2 halves, so we only want part of
   // both in the result.
   EVT ShiftTy = getShiftAmountTy(VT, DAG.getDataLayout());
-  Lo = DAG.getNode(ISD::SRL, dl, VT, Lo, DAG.getConstant(Scale, dl, ShiftTy));
-  Hi = DAG.getNode(
-      ISD::SHL, dl, VT, Hi,
-      DAG.getConstant(VT.getScalarSizeInBits() - Scale, dl, ShiftTy));
-  return DAG.getNode(ISD::OR, dl, VT, Lo, Hi);
+  return DAG.getNode(ISD::FSHR, dl, VT, Hi, Lo,
+                     DAG.getConstant(Scale, dl, ShiftTy));
 }
diff --git a/lib/CodeGen/ShadowStackGCLowering.cpp b/lib/CodeGen/ShadowStackGCLowering.cpp
index 3e12b32b12d4fc64d247362afddf0e29151446eb..17a4d76c4c8026681f2a1903f5b4b98710c7192d 100644
--- a/lib/CodeGen/ShadowStackGCLowering.cpp
+++ b/lib/CodeGen/ShadowStackGCLowering.cpp
@@ -1,9 +1,8 @@
 //===- ShadowStackGCLowering.cpp - Custom lowering for shadow-stack gc ----===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -313,7 +312,8 @@ bool ShadowStackGCLowering::runOnFunction(Function &F) {
   AtEntry.SetInsertPoint(IP->getParent(), IP);
 
   // Initialize the map pointer and load the current head of the shadow stack.
-  Instruction *CurrentHead = AtEntry.CreateLoad(Head, "gc_currhead");
+  Instruction *CurrentHead =
+      AtEntry.CreateLoad(StackEntryTy->getPointerTo(), Head, "gc_currhead");
   Instruction *EntryMapPtr = CreateGEP(Context, AtEntry, ConcreteStackEntryTy,
                                        StackEntry, 0, 1, "gc_frame.map");
   AtEntry.CreateStore(FrameMap, EntryMapPtr);
@@ -354,7 +354,8 @@ bool ShadowStackGCLowering::runOnFunction(Function &F) {
     Instruction *EntryNextPtr2 =
         CreateGEP(Context, *AtExit, ConcreteStackEntryTy, StackEntry, 0, 0,
                   "gc_frame.next");
-    Value *SavedHead = AtExit->CreateLoad(EntryNextPtr2, "gc_savedhead");
+    Value *SavedHead = AtExit->CreateLoad(StackEntryTy->getPointerTo(),
+                                          EntryNextPtr2, "gc_savedhead");
     AtExit->CreateStore(SavedHead, Head);
   }
 
diff --git a/lib/CodeGen/ShrinkWrap.cpp b/lib/CodeGen/ShrinkWrap.cpp
index d3454ca6ba6ae992ee19971c00f82cfed3dce714..3306100792fffecbfdadfcda58aa6339f98f2a78 100644
--- a/lib/CodeGen/ShrinkWrap.cpp
+++ b/lib/CodeGen/ShrinkWrap.cpp
@@ -1,9 +1,8 @@
 //===- ShrinkWrap.cpp - Compute safe point for prolog/epilog insertion ----===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/SjLjEHPrepare.cpp b/lib/CodeGen/SjLjEHPrepare.cpp
index 5d2669f5ae92b2c01d795a8a44238b4ae5e25b25..23e5ce0acae8a2834c64267ca1e50a2d7c27fa1c 100644
--- a/lib/CodeGen/SjLjEHPrepare.cpp
+++ b/lib/CodeGen/SjLjEHPrepare.cpp
@@ -1,9 +1,8 @@
 //===- SjLjEHPrepare.cpp - Eliminate Invoke & Unwind instructions ---------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -40,15 +39,15 @@ class SjLjEHPrepare : public FunctionPass {
   Type *doubleUnderDataTy;
   Type *doubleUnderJBufTy;
   Type *FunctionContextTy;
-  Constant *RegisterFn;
-  Constant *UnregisterFn;
-  Constant *BuiltinSetupDispatchFn;
-  Constant *FrameAddrFn;
-  Constant *StackAddrFn;
-  Constant *StackRestoreFn;
-  Constant *LSDAAddrFn;
-  Constant *CallSiteFn;
-  Constant *FuncCtxFn;
+  FunctionCallee RegisterFn;
+  FunctionCallee UnregisterFn;
+  Function *BuiltinSetupDispatchFn;
+  Function *FrameAddrFn;
+  Function *StackAddrFn;
+  Function *StackRestoreFn;
+  Function *LSDAAddrFn;
+  Function *CallSiteFn;
+  Function *FuncCtxFn;
   AllocaInst *FuncCtx;
 
 public:
@@ -190,14 +189,16 @@ Value *SjLjEHPrepare::setupFunctionContext(Function &F,
         Builder.CreateConstGEP2_32(FunctionContextTy, FuncCtx, 0, 2, "__data");
 
     // The exception values come back in context->__data[0].
+    Type *Int32Ty = Type::getInt32Ty(F.getContext());
     Value *ExceptionAddr = Builder.CreateConstGEP2_32(doubleUnderDataTy, FCData,
                                                       0, 0, "exception_gep");
-    Value *ExnVal = Builder.CreateLoad(ExceptionAddr, true, "exn_val");
+    Value *ExnVal = Builder.CreateLoad(Int32Ty, ExceptionAddr, true, "exn_val");
     ExnVal = Builder.CreateIntToPtr(ExnVal, Builder.getInt8PtrTy());
 
     Value *SelectorAddr = Builder.CreateConstGEP2_32(doubleUnderDataTy, FCData,
                                                      0, 1, "exn_selector_gep");
-    Value *SelVal = Builder.CreateLoad(SelectorAddr, true, "exn_selector_val");
+    Value *SelVal =
+        Builder.CreateLoad(Int32Ty, SelectorAddr, true, "exn_selector_val");
 
     substituteLPadValues(LPI, ExnVal, SelVal);
   }
diff --git a/lib/CodeGen/SlotIndexes.cpp b/lib/CodeGen/SlotIndexes.cpp
index fccbb8ec91cb9494669e4638770f3845ea99d033..a55e74c4c063cc944a495f6a2ee36db957a6840d 100644
--- a/lib/CodeGen/SlotIndexes.cpp
+++ b/lib/CodeGen/SlotIndexes.cpp
@@ -1,9 +1,8 @@
 //===-- SlotIndexes.cpp - Slot Indexes Pass  ------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/CodeGen/SpillPlacement.cpp b/lib/CodeGen/SpillPlacement.cpp
index f6786b30b21cc59c5973702b4ea3f8aff5342ce0..11452fdb747ac92bcbf1d687924d3b52cf89f011 100644
--- a/lib/CodeGen/SpillPlacement.cpp
+++ b/lib/CodeGen/SpillPlacement.cpp
@@ -1,9 +1,8 @@
 //===- SpillPlacement.cpp - Optimal Spill Code Placement ------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/SpillPlacement.h b/lib/CodeGen/SpillPlacement.h
index aa3ac444e0da9d783ab717d0d20ed3e2290569c5..aa0e07ef92e342eacbd408f9b31a24ddf072d760 100644
--- a/lib/CodeGen/SpillPlacement.h
+++ b/lib/CodeGen/SpillPlacement.h
@@ -1,9 +1,8 @@
 //===- SpillPlacement.h - Optimal Spill Code Placement ---------*- C++ -*--===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/Spiller.h b/lib/CodeGen/Spiller.h
index 330ee81342b6fabdbb98685826f08eb15bdd6130..66dabf78f87335bb536baf656d291cfed0de5e77 100644
--- a/lib/CodeGen/Spiller.h
+++ b/lib/CodeGen/Spiller.h
@@ -1,9 +1,8 @@
 //===- llvm/CodeGen/Spiller.h - Spiller -------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/CodeGen/SplitKit.cpp b/lib/CodeGen/SplitKit.cpp
index d639f44753015049f688fb3bdf99cbf326799371..0bdbdce30d0355c2771f4e65c1fcf1a7f786a068 100644
--- a/lib/CodeGen/SplitKit.cpp
+++ b/lib/CodeGen/SplitKit.cpp
@@ -1,9 +1,8 @@
 //===- SplitKit.cpp - Toolkit for splitting live ranges -------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/SplitKit.h b/lib/CodeGen/SplitKit.h
index bcc8f8cf18bc12749870fda53ebd382696ba789d..86ad3811e3ad27c18ab5d38b8646b6ab630066e6 100644
--- a/lib/CodeGen/SplitKit.h
+++ b/lib/CodeGen/SplitKit.h
@@ -1,9 +1,8 @@
 //===- SplitKit.h - Toolkit for splitting live ranges -----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/StackColoring.cpp b/lib/CodeGen/StackColoring.cpp
index eb8552915e2a053d6240bd22151eed974a149d9a..30e70660cda15bb1d288f398218af36644782f5e 100644
--- a/lib/CodeGen/StackColoring.cpp
+++ b/lib/CodeGen/StackColoring.cpp
@@ -1,9 +1,8 @@
 //===- StackColoring.cpp --------------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/StackMapLivenessAnalysis.cpp b/lib/CodeGen/StackMapLivenessAnalysis.cpp
index 00cf8070be5ec90de4040671951fca3b47d0997e..fb2abf3daa7fda5c31881cc0f136b7ac7ba5ea5e 100644
--- a/lib/CodeGen/StackMapLivenessAnalysis.cpp
+++ b/lib/CodeGen/StackMapLivenessAnalysis.cpp
@@ -1,9 +1,8 @@
 //===-- StackMapLivenessAnalysis.cpp - StackMap live Out Analysis ----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/StackMaps.cpp b/lib/CodeGen/StackMaps.cpp
index 0676fa2421e8006dae1fde55392ab07d695db78e..ae9401b897001ce2644d75d7d026ce566aef7d44 100644
--- a/lib/CodeGen/StackMaps.cpp
+++ b/lib/CodeGen/StackMaps.cpp
@@ -1,9 +1,8 @@
 //===- StackMaps.cpp ------------------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/CodeGen/StackProtector.cpp b/lib/CodeGen/StackProtector.cpp
index dcf37ca76b204bf51c285194f470328e8244f4d0..34942bc1b6c33138a2ef0f314aeed6b377c735f3 100644
--- a/lib/CodeGen/StackProtector.cpp
+++ b/lib/CodeGen/StackProtector.cpp
@@ -1,9 +1,8 @@
 //===- StackProtector.cpp - Stack Protector Insertion ---------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -157,14 +156,6 @@ bool StackProtector::ContainsProtectableArray(Type *Ty, bool &IsLarge,
   return NeedsProtector;
 }
 
-static bool isLifetimeInst(const Instruction *I) {
-  if (const auto Intrinsic = dyn_cast<IntrinsicInst>(I)) {
-    const auto Id = Intrinsic->getIntrinsicID();
-    return Id == Intrinsic::lifetime_start || Id == Intrinsic::lifetime_end;
-  }
-  return false;
-}
-
 bool StackProtector::HasAddressTaken(const Instruction *AI) {
   for (const User *U : AI->users()) {
     if (const StoreInst *SI = dyn_cast<StoreInst>(U)) {
@@ -175,7 +166,7 @@ bool StackProtector::HasAddressTaken(const Instruction *AI) {
         return true;
     } else if (const CallInst *CI = dyn_cast<CallInst>(U)) {
       // Ignore intrinsics that are not calls. TODO: Use isLoweredToCall().
-      if (!isa<DbgInfoIntrinsic>(CI) && !isLifetimeInst(CI))
+      if (!isa<DbgInfoIntrinsic>(CI) && !CI->isLifetimeStartOrEnd())
         return true;
     } else if (isa<InvokeInst>(U)) {
       return true;
@@ -331,7 +322,7 @@ static Value *getStackGuard(const TargetLoweringBase *TLI, Module *M,
                             IRBuilder<> &B,
                             bool *SupportsSelectionDAGSP = nullptr) {
   if (Value *Guard = TLI->getIRStackGuard(B))
-    return B.CreateLoad(Guard, true, "StackGuard");
+    return B.CreateLoad(B.getInt8PtrTy(), Guard, true, "StackGuard");
 
   // Use SelectionDAG SSP handling, since there isn't an IR guard.
   //
@@ -422,15 +413,14 @@ bool StackProtector::InsertStackProtectors() {
     // Generate epilogue instrumentation. The epilogue intrumentation can be
     // function-based or inlined depending on which mechanism the target is
     // providing.
-    if (Value* GuardCheck = TLI->getSSPStackGuardCheck(*M)) {
+    if (Function *GuardCheck = TLI->getSSPStackGuardCheck(*M)) {
       // Generate the function-based epilogue instrumentation.
       // The target provides a guard check function, generate a call to it.
       IRBuilder<> B(RI);
-      LoadInst *Guard = B.CreateLoad(AI, true, "Guard");
+      LoadInst *Guard = B.CreateLoad(B.getInt8PtrTy(), AI, true, "Guard");
       CallInst *Call = B.CreateCall(GuardCheck, {Guard});
-      llvm::Function *Function = cast<llvm::Function>(GuardCheck);
-      Call->setAttributes(Function->getAttributes());
-      Call->setCallingConv(Function->getCallingConv());
+      Call->setAttributes(GuardCheck->getAttributes());
+      Call->setCallingConv(GuardCheck->getCallingConv());
     } else {
       // Generate the epilogue with inline instrumentation.
       // If we do not support SelectionDAG based tail calls, generate IR level
@@ -482,7 +472,7 @@ bool StackProtector::InsertStackProtectors() {
       // Generate the stack protector instructions in the old basic block.
       IRBuilder<> B(BB);
       Value *Guard = getStackGuard(TLI, M, B);
-      LoadInst *LI2 = B.CreateLoad(AI, true);
+      LoadInst *LI2 = B.CreateLoad(B.getInt8PtrTy(), AI, true);
       Value *Cmp = B.CreateICmpEQ(Guard, LI2);
       auto SuccessProb =
           BranchProbabilityInfo::getBranchProbStackProtector(true);
@@ -508,14 +498,13 @@ BasicBlock *StackProtector::CreateFailBB() {
   IRBuilder<> B(FailBB);
   B.SetCurrentDebugLocation(DebugLoc::get(0, 0, F->getSubprogram()));
   if (Trip.isOSOpenBSD()) {
-    Constant *StackChkFail =
-        M->getOrInsertFunction("__stack_smash_handler",
-                               Type::getVoidTy(Context),
-                               Type::getInt8PtrTy(Context));
+    FunctionCallee StackChkFail = M->getOrInsertFunction(
+        "__stack_smash_handler", Type::getVoidTy(Context),
+        Type::getInt8PtrTy(Context));
 
     B.CreateCall(StackChkFail, B.CreateGlobalStringPtr(F->getName(), "SSH"));
   } else {
-    Constant *StackChkFail =
+    FunctionCallee StackChkFail =
         M->getOrInsertFunction("__stack_chk_fail", Type::getVoidTy(Context));
 
     B.CreateCall(StackChkFail, {});
diff --git a/lib/CodeGen/StackSlotColoring.cpp b/lib/CodeGen/StackSlotColoring.cpp
index d8c6a249e4da1f33c69e7f547b9496ca657aad97..60893f490703c9d6e3d3fffdc4559b9dc64ca848 100644
--- a/lib/CodeGen/StackSlotColoring.cpp
+++ b/lib/CodeGen/StackSlotColoring.cpp
@@ -1,9 +1,8 @@
 //===- StackSlotColoring.cpp - Stack slot coloring pass. ------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/TailDuplication.cpp b/lib/CodeGen/TailDuplication.cpp
index 25cd7802264ea22a1f863732a6232199ffe6f70c..ba348b4a9d41b5e86223a3e36d104b7de61600f6 100644
--- a/lib/CodeGen/TailDuplication.cpp
+++ b/lib/CodeGen/TailDuplication.cpp
@@ -1,9 +1,8 @@
 //===- TailDuplication.cpp - Duplicate blocks into predecessors' tails ----===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/TailDuplicator.cpp b/lib/CodeGen/TailDuplicator.cpp
index b118c176a89744f03afb73a0a63ca990b4a210c9..d3cd6c6713ac31895358bd3e3882acf22011b424 100644
--- a/lib/CodeGen/TailDuplicator.cpp
+++ b/lib/CodeGen/TailDuplicator.cpp
@@ -1,9 +1,8 @@
 //===- TailDuplicator.cpp - Duplicate blocks into predecessors' tails -----===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/TargetFrameLoweringImpl.cpp b/lib/CodeGen/TargetFrameLoweringImpl.cpp
index f0cfa2fbe4fddf3dc54b0a41b487141e2fa68ea5..9c4483cb240d70d88ed7734e15e1d3c76d016ba6 100644
--- a/lib/CodeGen/TargetFrameLoweringImpl.cpp
+++ b/lib/CodeGen/TargetFrameLoweringImpl.cpp
@@ -1,9 +1,8 @@
 //===- TargetFrameLoweringImpl.cpp - Implement target frame interface ------==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -30,12 +29,6 @@ using namespace llvm;
 
 TargetFrameLowering::~TargetFrameLowering() = default;
 
-/// The default implementation just looks at attribute "no-frame-pointer-elim".
-bool TargetFrameLowering::noFramePointerElim(const MachineFunction &MF) const {
-  auto Attr = MF.getFunction().getFnAttribute("no-frame-pointer-elim");
-  return Attr.getValueAsString() == "true";
-}
-
 bool TargetFrameLowering::enableCalleeSaveSkip(const MachineFunction &MF) const {
   assert(MF.getFunction().hasFnAttribute(Attribute::NoReturn) &&
          MF.getFunction().hasFnAttribute(Attribute::NoUnwind) &&
diff --git a/lib/CodeGen/TargetInstrInfo.cpp b/lib/CodeGen/TargetInstrInfo.cpp
index 2a17af391105cb8ac7fc7f1bd9f0f575218e16d7..b3aeaf8272598b44e62fa40a175f467a4caf6182 100644
--- a/lib/CodeGen/TargetInstrInfo.cpp
+++ b/lib/CodeGen/TargetInstrInfo.cpp
@@ -1,9 +1,8 @@
 //===-- TargetInstrInfo.cpp - Target Instruction Information --------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/TargetLoweringBase.cpp b/lib/CodeGen/TargetLoweringBase.cpp
index e8619037564245319d06ff5f430490875ee83c7c..7eeea33a842722731b76de75effceda0f9e13634 100644
--- a/lib/CodeGen/TargetLoweringBase.cpp
+++ b/lib/CodeGen/TargetLoweringBase.cpp
@@ -1,9 +1,8 @@
 //===- TargetLoweringBase.cpp - Implement the TargetLoweringBase class ----===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -583,6 +582,14 @@ void TargetLoweringBase::initActions() {
   std::fill(std::begin(TargetDAGCombineArray),
             std::end(TargetDAGCombineArray), 0);
 
+  for (MVT VT : MVT::fp_valuetypes()) {
+    MVT IntVT = MVT::getIntegerVT(VT.getSizeInBits());
+    if (IntVT.isValid()) {
+      setOperationAction(ISD::ATOMIC_SWAP, VT, Promote);
+      AddPromotedToType(ISD::ATOMIC_SWAP, VT, IntVT);
+    }
+  }
+
   // Set default actions for various operations.
   for (MVT VT : MVT::all_valuetypes()) {
     // Default all indexed load / store to expand.
@@ -617,6 +624,7 @@ void TargetLoweringBase::initActions() {
     setOperationAction(ISD::SSUBSAT, VT, Expand);
     setOperationAction(ISD::USUBSAT, VT, Expand);
     setOperationAction(ISD::SMULFIX, VT, Expand);
+    setOperationAction(ISD::UMULFIX, VT, Expand);
 
     // Overflow operations default to expand
     setOperationAction(ISD::SADDO, VT, Expand);
@@ -1447,6 +1455,7 @@ int TargetLoweringBase::InstructionOpcodeToISD(unsigned Opcode) const {
   case Switch:         return 0;
   case IndirectBr:     return 0;
   case Invoke:         return 0;
+  case CallBr:         return 0;
   case Resume:         return 0;
   case Unreachable:    return 0;
   case CleanupRet:     return 0;
@@ -1580,8 +1589,8 @@ Value *TargetLoweringBase::getSafeStackPointerLocation(IRBuilder<> &IRB) const {
   // thread's unsafe stack pointer.
   Module *M = IRB.GetInsertBlock()->getParent()->getParent();
   Type *StackPtrTy = Type::getInt8PtrTy(M->getContext());
-  Value *Fn = M->getOrInsertFunction("__safestack_pointer_address",
-                                     StackPtrTy->getPointerTo(0));
+  FunctionCallee Fn = M->getOrInsertFunction("__safestack_pointer_address",
+                                             StackPtrTy->getPointerTo(0));
   return IRB.CreateCall(Fn);
 }
 
@@ -1656,7 +1665,7 @@ Value *TargetLoweringBase::getSDagStackGuard(const Module &M) const {
   return M.getNamedValue("__stack_chk_guard");
 }
 
-Value *TargetLoweringBase::getSSPStackGuardCheck(const Module &M) const {
+Function *TargetLoweringBase::getSSPStackGuardCheck(const Module &M) const {
   return nullptr;
 }
 
diff --git a/lib/CodeGen/TargetLoweringObjectFileImpl.cpp b/lib/CodeGen/TargetLoweringObjectFileImpl.cpp
index cb2fe691d7021a7ec07907eb6168cdefec40fee4..a7cdcee8cd61e6e9a306f9d3108ef59976dfa8e0 100644
--- a/lib/CodeGen/TargetLoweringObjectFileImpl.cpp
+++ b/lib/CodeGen/TargetLoweringObjectFileImpl.cpp
@@ -1,9 +1,8 @@
 //===- llvm/CodeGen/TargetLoweringObjectFileImpl.cpp - Object File Info ---===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/TargetOptionsImpl.cpp b/lib/CodeGen/TargetOptionsImpl.cpp
index 853e71d0efa514eb61f501cf62f434ab2b6e0862..039748d817caad60021755aac9ccd64c5f8fb504 100644
--- a/lib/CodeGen/TargetOptionsImpl.cpp
+++ b/lib/CodeGen/TargetOptionsImpl.cpp
@@ -1,9 +1,8 @@
 //===-- TargetOptionsImpl.cpp - Options that apply to all targets ----------==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -23,15 +22,34 @@ using namespace llvm;
 /// DisableFramePointerElim - This returns true if frame pointer elimination
 /// optimization should be disabled for the given machine function.
 bool TargetOptions::DisableFramePointerElim(const MachineFunction &MF) const {
-  // Check to see if we should eliminate all frame pointers.
-  if (MF.getSubtarget().getFrameLowering()->noFramePointerElim(MF))
+  // Check to see if the target want to forcably keep frame pointer.
+  if (MF.getSubtarget().getFrameLowering()->keepFramePointer(MF))
     return true;
 
-  // Check to see if we should eliminate non-leaf frame pointers.
-  if (MF.getFunction().hasFnAttribute("no-frame-pointer-elim-non-leaf"))
-    return MF.getFrameInfo().hasCalls();
+  const Function &F = MF.getFunction();
+
+  // TODO: Remove support for old `fp elim` function attributes after fully
+  //       migrate to use "frame-pointer"
+  if (!F.hasFnAttribute("frame-pointer")) {
+    // Check to see if we should eliminate all frame pointers.
+    if (F.getFnAttribute("no-frame-pointer-elim").getValueAsString() == "true")
+      return true;
+
+    // Check to see if we should eliminate non-leaf frame pointers.
+    if (F.hasFnAttribute("no-frame-pointer-elim-non-leaf"))
+      return MF.getFrameInfo().hasCalls();
 
-  return false;
+    return false;
+  }
+
+  StringRef FP = F.getFnAttribute("frame-pointer").getValueAsString();
+  if (FP == "all")
+    return true;
+  if (FP == "non-leaf")
+    return MF.getFrameInfo().hasCalls();
+  if (FP == "none")
+    return false;
+  llvm_unreachable("unknown frame pointer flag");
 }
 
 /// HonorSignDependentRoundingFPMath - Return true if the codegen must assume
diff --git a/lib/CodeGen/TargetPassConfig.cpp b/lib/CodeGen/TargetPassConfig.cpp
index defb165fe0f29adb628e8cae798a197ec16dc8a7..85c0011911412e484e7403ea6091a66f39c3113c 100644
--- a/lib/CodeGen/TargetPassConfig.cpp
+++ b/lib/CodeGen/TargetPassConfig.cpp
@@ -1,9 +1,8 @@
 //===- TargetPassConfig.cpp - Target independent code generation passes ---===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -755,22 +754,33 @@ void TargetPassConfig::addISelPrepare() {
 bool TargetPassConfig::addCoreISelPasses() {
   // Enable FastISel with -fast-isel, but allow that to be overridden.
   TM->setO0WantsFastISel(EnableFastISelOption != cl::BOU_FALSE);
-  if (EnableFastISelOption == cl::BOU_TRUE ||
-      (TM->getOptLevel() == CodeGenOpt::None && TM->getO0WantsFastISel() &&
-       !TM->Options.EnableGlobalISel)) {
+
+  // Determine an instruction selector.
+  enum class SelectorType { SelectionDAG, FastISel, GlobalISel };
+  SelectorType Selector;
+
+  if (EnableFastISelOption == cl::BOU_TRUE)
+    Selector = SelectorType::FastISel;
+  else if (EnableGlobalISelOption == cl::BOU_TRUE ||
+           (TM->Options.EnableGlobalISel &&
+            EnableGlobalISelOption != cl::BOU_FALSE))
+    Selector = SelectorType::GlobalISel;
+  else if (TM->getOptLevel() == CodeGenOpt::None && TM->getO0WantsFastISel())
+    Selector = SelectorType::FastISel;
+  else
+    Selector = SelectorType::SelectionDAG;
+
+  // Set consistently TM->Options.EnableFastISel and EnableGlobalISel.
+  if (Selector == SelectorType::FastISel) {
     TM->setFastISel(true);
     TM->setGlobalISel(false);
-  }
-
-  // Ask the target for an instruction selector.
-  // Explicitly enabling fast-isel should override implicitly enabled
-  // global-isel.
-  if (EnableGlobalISelOption == cl::BOU_TRUE ||
-      (EnableGlobalISelOption == cl::BOU_UNSET &&
-       TM->Options.EnableGlobalISel && EnableFastISelOption != cl::BOU_TRUE)) {
-    TM->setGlobalISel(true);
+  } else if (Selector == SelectorType::GlobalISel) {
     TM->setFastISel(false);
+    TM->setGlobalISel(true);
+  }
 
+  // Add instruction selector passes.
+  if (Selector == SelectorType::GlobalISel) {
     SaveAndRestore<bool> SavedAddingMachinePasses(AddingMachinePasses, true);
     if (addIRTranslator())
       return true;
@@ -1210,3 +1220,7 @@ bool TargetPassConfig::isGlobalISelAbortEnabled() const {
 bool TargetPassConfig::reportDiagnosticWhenGlobalISelFallback() const {
   return TM->Options.GlobalISelAbort == GlobalISelAbortMode::DisableWithDiag;
 }
+
+bool TargetPassConfig::isGISelCSEEnabled() const {
+  return getOptLevel() != CodeGenOpt::Level::None;
+}
diff --git a/lib/CodeGen/TargetRegisterInfo.cpp b/lib/CodeGen/TargetRegisterInfo.cpp
index 661dc18f7a85ef1beb3fdb800232ebab821aee88..264999804a10a7347d9cfe449cd22c943006c2c9 100644
--- a/lib/CodeGen/TargetRegisterInfo.cpp
+++ b/lib/CodeGen/TargetRegisterInfo.cpp
@@ -1,9 +1,8 @@
 //==- TargetRegisterInfo.cpp - Target Register Information Implementation --==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/TargetSchedule.cpp b/lib/CodeGen/TargetSchedule.cpp
index 3cff31ad4933588135cf9efd0a4a9d947d5016ed..195279719ad47c06febfea864722e6826211a397 100644
--- a/lib/CodeGen/TargetSchedule.cpp
+++ b/lib/CodeGen/TargetSchedule.cpp
@@ -1,9 +1,8 @@
 //===- llvm/Target/TargetSchedule.cpp - Sched Machine Model ---------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/TargetSubtargetInfo.cpp b/lib/CodeGen/TargetSubtargetInfo.cpp
index fa29c05fd6c2accf79daf4e0fd0208bc188a9604..7b29b68597c5a9170071d0d17e759df04618c27c 100644
--- a/lib/CodeGen/TargetSubtargetInfo.cpp
+++ b/lib/CodeGen/TargetSubtargetInfo.cpp
@@ -1,9 +1,8 @@
 //===- TargetSubtargetInfo.cpp - General Target Information ----------------==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -12,14 +11,6 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/CodeGen/TargetSubtargetInfo.h"
-#include "llvm/ADT/Optional.h"
-#include "llvm/CodeGen/MachineInstr.h"
-#include "llvm/CodeGen/TargetInstrInfo.h"
-#include "llvm/CodeGen/TargetSchedule.h"
-#include "llvm/MC/MCInst.h"
-#include "llvm/Support/Format.h"
-#include "llvm/Support/raw_ostream.h"
-#include <string>
 
 using namespace llvm;
 
@@ -67,50 +58,4 @@ bool TargetSubtargetInfo::useAA() const {
   return false;
 }
 
-static std::string createSchedInfoStr(unsigned Latency, double RThroughput) {
-  static const char *SchedPrefix = " sched: [";
-  std::string Comment;
-  raw_string_ostream CS(Comment);
-  if (RThroughput != 0.0)
-    CS << SchedPrefix << Latency << format(":%2.2f", RThroughput)
-       << "]";
-  else
-    CS << SchedPrefix << Latency << ":?]";
-  CS.flush();
-  return Comment;
-}
-
-/// Returns string representation of scheduler comment
-std::string TargetSubtargetInfo::getSchedInfoStr(const MachineInstr &MI) const {
-  if (MI.isPseudo() || MI.isTerminator())
-    return std::string();
-  // We don't cache TSchedModel because it depends on TargetInstrInfo
-  // that could be changed during the compilation
-  TargetSchedModel TSchedModel;
-  TSchedModel.init(this);
-  unsigned Latency = TSchedModel.computeInstrLatency(&MI);
-  double RThroughput = TSchedModel.computeReciprocalThroughput(&MI);
-  return createSchedInfoStr(Latency, RThroughput);
-}
-
-/// Returns string representation of scheduler comment
-std::string TargetSubtargetInfo::getSchedInfoStr(MCInst const &MCI) const {
-  // We don't cache TSchedModel because it depends on TargetInstrInfo
-  // that could be changed during the compilation
-  TargetSchedModel TSchedModel;
-  TSchedModel.init(this);
-  unsigned Latency;
-  if (TSchedModel.hasInstrSchedModel())
-    Latency = TSchedModel.computeInstrLatency(MCI);
-  else if (TSchedModel.hasInstrItineraries()) {
-    auto *ItinData = TSchedModel.getInstrItineraries();
-    Latency = ItinData->getStageLatency(
-        getInstrInfo()->get(MCI.getOpcode()).getSchedClass());
-  } else
-    return std::string();
-  double RThroughput = TSchedModel.computeReciprocalThroughput(MCI);
-  return createSchedInfoStr(Latency, RThroughput);
-}
-
-void TargetSubtargetInfo::mirFileLoaded(MachineFunction &MF) const {
-}
+void TargetSubtargetInfo::mirFileLoaded(MachineFunction &MF) const { }
diff --git a/lib/CodeGen/TwoAddressInstructionPass.cpp b/lib/CodeGen/TwoAddressInstructionPass.cpp
index 2e2fe72e5395017ab033c2bbc87ed6ed1eb57822..737d01eaa17d50940d54900bdb82a8a98a42ff68 100644
--- a/lib/CodeGen/TwoAddressInstructionPass.cpp
+++ b/lib/CodeGen/TwoAddressInstructionPass.cpp
@@ -1,9 +1,8 @@
 //===- TwoAddressInstructionPass.cpp - Two-Address instruction pass -------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -929,9 +928,12 @@ rescheduleMIBelowKill(MachineBasicBlock::iterator &mi,
   MachineBasicBlock::iterator Begin = MI;
   MachineBasicBlock::iterator AfterMI = std::next(Begin);
   MachineBasicBlock::iterator End = AfterMI;
-  while (End->isCopy() &&
-         regOverlapsSet(Defs, End->getOperand(1).getReg(), TRI)) {
-    Defs.push_back(End->getOperand(0).getReg());
+  while (End != MBB->end()) {
+    End = skipDebugInstructionsForward(End, MBB->end());
+    if (End->isCopy() && regOverlapsSet(Defs, End->getOperand(1).getReg(), TRI))
+      Defs.push_back(End->getOperand(0).getReg());
+    else
+      break;
     ++End;
   }
 
diff --git a/lib/CodeGen/UnreachableBlockElim.cpp b/lib/CodeGen/UnreachableBlockElim.cpp
index 5288ca6727742d724cee4f93f6a65ad094027d4f..9da7eb32c9369f206345f7b847c88d31acfb8443 100644
--- a/lib/CodeGen/UnreachableBlockElim.cpp
+++ b/lib/CodeGen/UnreachableBlockElim.cpp
@@ -1,9 +1,8 @@
 //===-- UnreachableBlockElim.cpp - Remove unreachable blocks for codegen --===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/ValueTypes.cpp b/lib/CodeGen/ValueTypes.cpp
index adb7075de651988198223a05f2983d980294df3a..83b31d3aacda146e5239ade298caac314de5b478 100644
--- a/lib/CodeGen/ValueTypes.cpp
+++ b/lib/CodeGen/ValueTypes.cpp
@@ -1,9 +1,8 @@
 //===----------- ValueTypes.cpp - Implementation of EVT methods -----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/CodeGen/VirtRegMap.cpp b/lib/CodeGen/VirtRegMap.cpp
index ed7bef667e77e82a69910d6202f66decfbc40917..d7f213998268177787cdebba05c9a583179f4841 100644
--- a/lib/CodeGen/VirtRegMap.cpp
+++ b/lib/CodeGen/VirtRegMap.cpp
@@ -1,9 +1,8 @@
 //===- llvm/CodeGen/VirtRegMap.cpp - Virtual Register Map -----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/CodeGen/WasmEHPrepare.cpp b/lib/CodeGen/WasmEHPrepare.cpp
index e5002eb95346dbe6228ab2498bfd68e4cb6080a2..e9986f566c37b98df897be4dd78f3de6c544ab3a 100644
--- a/lib/CodeGen/WasmEHPrepare.cpp
+++ b/lib/CodeGen/WasmEHPrepare.cpp
@@ -1,14 +1,14 @@
 //===-- WasmEHPrepare - Prepare excepton handling for WebAssembly --------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 // This transformation is designed for use by code generators which use
-// WebAssembly exception handling scheme.
+// WebAssembly exception handling scheme. This currently supports C++
+// exceptions.
 //
 // WebAssembly exception handling uses Windows exception IR for the middle level
 // representation. This pass does the following transformation for every
@@ -23,53 +23,20 @@
 //
 // - After:
 //   catchpad ...
-//   exn = wasm.catch(0); // 0 is a tag for C++
-//   wasm.landingpad.index(index);
+//   exn = wasm.extract.exception();
 //   // Only add below in case it's not a single catch (...)
+//   wasm.landingpad.index(index);
 //   __wasm_lpad_context.lpad_index = index;
 //   __wasm_lpad_context.lsda = wasm.lsda();
 //   _Unwind_CallPersonality(exn);
-//   int selector = __wasm.landingpad_context.selector;
+//   selector = __wasm.landingpad_context.selector;
 //   ...
 //
-// Also, does the following for a cleanuppad block with a call to
-// __clang_call_terminate():
-// - Before:
-//   cleanuppad ...
-//   exn = wasm.get.exception();
-//   __clang_call_terminate(exn);
-//
-// - After:
-//   cleanuppad ...
-//   exn = wasm.catch(0); // 0 is a tag for C++
-//   __clang_call_terminate(exn);
-//
-//
-// * Background: WebAssembly EH instructions
-// WebAssembly's try and catch instructions are structured as follows:
-// try
-//   instruction*
-// catch (C++ tag)
-//   instruction*
-// ...
-// catch_all
-//   instruction*
-// try_end
-//
-// A catch instruction in WebAssembly does not correspond to a C++ catch clause.
-// In WebAssembly, there is a single catch instruction for all C++ exceptions.
-// There can be more catch instructions for exceptions in other languages, but
-// they are not generated for now. catch_all catches all exceptions including
-// foreign exceptions (e.g. JavaScript). We turn catchpads into catch (C++ tag)
-// and cleanuppads into catch_all, with one exception: cleanuppad with a call to
-// __clang_call_terminate should be both in catch (C++ tag) and catch_all.
-//
 //
 // * Background: Direct personality function call
 // In WebAssembly EH, the VM is responsible for unwinding the stack once an
 // exception is thrown. After the stack is unwound, the control flow is
-// transfered to WebAssembly 'catch' instruction, which returns a caught
-// exception object.
+// transfered to WebAssembly 'catch' instruction.
 //
 // Unwinding the stack is not done by libunwind but the VM, so the personality
 // function in libcxxabi cannot be called from libunwind during the unwinding
@@ -138,18 +105,19 @@ class WasmEHPrepare : public FunctionPass {
   Value *SelectorField = nullptr;  // selector
 
   Function *ThrowF = nullptr;           // wasm.throw() intrinsic
-  Function *CatchF = nullptr;           // wasm.catch.extract() intrinsic
+  Function *RethrowF = nullptr;         // wasm.rethrow() intrinsic
   Function *LPadIndexF = nullptr;       // wasm.landingpad.index() intrinsic
   Function *LSDAF = nullptr;            // wasm.lsda() intrinsic
   Function *GetExnF = nullptr;          // wasm.get.exception() intrinsic
+  Function *ExtractExnF = nullptr;      // wasm.extract.exception() intrinsic
   Function *GetSelectorF = nullptr;     // wasm.get.ehselector() intrinsic
-  Function *CallPersonalityF = nullptr; // _Unwind_CallPersonality() wrapper
-  Function *ClangCallTermF = nullptr;   // __clang_call_terminate() function
+  FunctionCallee CallPersonalityF =
+      nullptr;                          // _Unwind_CallPersonality() wrapper
 
   bool prepareEHPads(Function &F);
   bool prepareThrows(Function &F);
 
-  void prepareEHPad(BasicBlock *BB, unsigned Index);
+  void prepareEHPad(BasicBlock *BB, bool NeedLSDA, unsigned Index = 0);
   void prepareTerminateCleanupPad(BasicBlock *BB);
 
 public:
@@ -209,25 +177,29 @@ bool WasmEHPrepare::prepareThrows(Function &F) {
 
   // wasm.throw() intinsic, which will be lowered to wasm 'throw' instruction.
   ThrowF = Intrinsic::getDeclaration(&M, Intrinsic::wasm_throw);
-
-  // Insert an unreachable instruction after a call to @llvm.wasm.throw and
-  // delete all following instructions within the BB, and delete all the dead
-  // children of the BB as well.
-  for (User *U : ThrowF->users()) {
-    // A call to @llvm.wasm.throw() is only generated from
-    // __builtin_wasm_throw() builtin call within libcxxabi, and cannot be an
-    // InvokeInst.
-    auto *ThrowI = cast<CallInst>(U);
-    if (ThrowI->getFunction() != &F)
-      continue;
-    Changed = true;
-    auto *BB = ThrowI->getParent();
-    SmallVector<BasicBlock *, 4> Succs(succ_begin(BB), succ_end(BB));
-    auto &InstList = BB->getInstList();
-    InstList.erase(std::next(BasicBlock::iterator(ThrowI)), InstList.end());
-    IRB.SetInsertPoint(BB);
-    IRB.CreateUnreachable();
-    eraseDeadBBsAndChildren(Succs);
+  // wasm.rethrow() intinsic, which will be lowered to wasm 'rethrow'
+  // instruction.
+  RethrowF = Intrinsic::getDeclaration(&M, Intrinsic::wasm_rethrow);
+
+  // Insert an unreachable instruction after a call to @llvm.wasm.throw /
+  // @llvm.wasm.rethrow and delete all following instructions within the BB, and
+  // delete all the dead children of the BB as well.
+  for (auto L : {ThrowF->users(), RethrowF->users()}) {
+    for (User *U : L) {
+      // A call to @llvm.wasm.throw() is only generated from __cxa_throw()
+      // builtin call within libcxxabi, and cannot be an InvokeInst.
+      auto *ThrowI = cast<CallInst>(U);
+      if (ThrowI->getFunction() != &F)
+        continue;
+      Changed = true;
+      auto *BB = ThrowI->getParent();
+      SmallVector<BasicBlock *, 4> Succs(succ_begin(BB), succ_end(BB));
+      auto &InstList = BB->getInstList();
+      InstList.erase(std::next(BasicBlock::iterator(ThrowI)), InstList.end());
+      IRB.SetInsertPoint(BB);
+      IRB.CreateUnreachable();
+      eraseDeadBBsAndChildren(Succs);
+    }
   }
 
   return Changed;
@@ -263,8 +235,6 @@ bool WasmEHPrepare::prepareEHPads(Function &F) {
   SelectorField = IRB.CreateConstGEP2_32(LPadContextTy, LPadContextGV, 0, 2,
                                          "selector_gep");
 
-  // wasm.catch() intinsic, which will be lowered to wasm 'catch' instruction.
-  CatchF = Intrinsic::getDeclaration(&M, Intrinsic::wasm_catch);
   // wasm.landingpad.index() intrinsic, which is to specify landingpad index
   LPadIndexF = Intrinsic::getDeclaration(&M, Intrinsic::wasm_landingpad_index);
   // wasm.lsda() intrinsic. Returns the address of LSDA table for the current
@@ -275,14 +245,18 @@ bool WasmEHPrepare::prepareEHPads(Function &F) {
   GetExnF = Intrinsic::getDeclaration(&M, Intrinsic::wasm_get_exception);
   GetSelectorF = Intrinsic::getDeclaration(&M, Intrinsic::wasm_get_ehselector);
 
-  // _Unwind_CallPersonality() wrapper function, which calls the personality
-  CallPersonalityF = cast<Function>(M.getOrInsertFunction(
-      "_Unwind_CallPersonality", IRB.getInt32Ty(), IRB.getInt8PtrTy()));
-  CallPersonalityF->setDoesNotThrow();
+  // wasm.extract.exception() is the same as wasm.get.exception() but it does
+  // not take a token argument. This will be lowered down to EXTRACT_EXCEPTION
+  // pseudo instruction in instruction selection, which will be expanded using
+  // 'br_on_exn' instruction later.
+  ExtractExnF =
+      Intrinsic::getDeclaration(&M, Intrinsic::wasm_extract_exception);
 
-  // __clang_call_terminate() function, which is inserted by clang in case a
-  // cleanup throws
-  ClangCallTermF = M.getFunction("__clang_call_terminate");
+  // _Unwind_CallPersonality() wrapper function, which calls the personality
+  CallPersonalityF = M.getOrInsertFunction(
+      "_Unwind_CallPersonality", IRB.getInt32Ty(), IRB.getInt8PtrTy());
+  if (Function *F = dyn_cast<Function>(CallPersonalityF.getCallee()))
+    F->setDoesNotThrow();
 
   unsigned Index = 0;
   for (auto *BB : CatchPads) {
@@ -290,60 +264,52 @@ bool WasmEHPrepare::prepareEHPads(Function &F) {
     // In case of a single catch (...), we don't need to emit LSDA
     if (CPI->getNumArgOperands() == 1 &&
         cast<Constant>(CPI->getArgOperand(0))->isNullValue())
-      prepareEHPad(BB, -1);
+      prepareEHPad(BB, false);
     else
-      prepareEHPad(BB, Index++);
+      prepareEHPad(BB, true, Index++);
   }
 
-  if (!ClangCallTermF)
-    return !CatchPads.empty();
-
-  // Cleanuppads will turn into catch_all later, but cleanuppads with a call to
-  // __clang_call_terminate() is a special case. __clang_call_terminate() takes
-  // an exception object, so we have to duplicate call in both 'catch <C++ tag>'
-  // and 'catch_all' clauses. Here we only insert a call to catch; the
-  // duplication will be done later. In catch_all, the exception object will be
-  // set to null.
+  // Cleanup pads don't need LSDA.
   for (auto *BB : CleanupPads)
-    for (auto &I : *BB)
-      if (auto *CI = dyn_cast<CallInst>(&I))
-        if (CI->getCalledValue() == ClangCallTermF)
-          prepareEHPad(BB, -1);
+    prepareEHPad(BB, false);
 
   return true;
 }
 
-void WasmEHPrepare::prepareEHPad(BasicBlock *BB, unsigned Index) {
+// Prepare an EH pad for Wasm EH handling. If NeedLSDA is false, Index is
+// ignored.
+void WasmEHPrepare::prepareEHPad(BasicBlock *BB, bool NeedLSDA,
+                                 unsigned Index) {
   assert(BB->isEHPad() && "BB is not an EHPad!");
   IRBuilder<> IRB(BB->getContext());
-
   IRB.SetInsertPoint(&*BB->getFirstInsertionPt());
-  // The argument to wasm.catch() is the tag for C++ exceptions, which we set to
-  // 0 for this module.
-  // Pseudocode: void *exn = wasm.catch(0);
-  Instruction *Exn = IRB.CreateCall(CatchF, IRB.getInt32(0), "exn");
-  // Replace the return value of wasm.get.exception() with the return value from
-  // wasm.catch().
+
   auto *FPI = cast<FuncletPadInst>(BB->getFirstNonPHI());
   Instruction *GetExnCI = nullptr, *GetSelectorCI = nullptr;
   for (auto &U : FPI->uses()) {
     if (auto *CI = dyn_cast<CallInst>(U.getUser())) {
       if (CI->getCalledValue() == GetExnF)
         GetExnCI = CI;
-      else if (CI->getCalledValue() == GetSelectorF)
+      if (CI->getCalledValue() == GetSelectorF)
         GetSelectorCI = CI;
     }
   }
 
-  assert(GetExnCI && "wasm.get.exception() call does not exist");
-  GetExnCI->replaceAllUsesWith(Exn);
+  // Cleanup pads w/o __clang_call_terminate call do not have any of
+  // wasm.get.exception() or wasm.get.ehselector() calls. We need to do nothing.
+  if (!GetExnCI) {
+    assert(!GetSelectorCI &&
+           "wasm.get.ehselector() cannot exist w/o wasm.get.exception()");
+    return;
+  }
+
+  Instruction *ExtractExnCI = IRB.CreateCall(ExtractExnF, {}, "exn");
+  GetExnCI->replaceAllUsesWith(ExtractExnCI);
   GetExnCI->eraseFromParent();
 
   // In case it is a catchpad with single catch (...) or a cleanuppad, we don't
   // need to call personality function because we don't need a selector.
-  if (FPI->getNumArgOperands() == 0 ||
-      (FPI->getNumArgOperands() == 1 &&
-       cast<Constant>(FPI->getArgOperand(0))->isNullValue())) {
+  if (!NeedLSDA) {
     if (GetSelectorCI) {
       assert(GetSelectorCI->use_empty() &&
              "wasm.get.ehselector() still has uses!");
@@ -351,7 +317,7 @@ void WasmEHPrepare::prepareEHPad(BasicBlock *BB, unsigned Index) {
     }
     return;
   }
-  IRB.SetInsertPoint(Exn->getNextNode());
+  IRB.SetInsertPoint(ExtractExnCI->getNextNode());
 
   // This is to create a map of <landingpad EH label, landingpad index> in
   // SelectionDAGISel, which is to be used in EHStreamer to emit LSDA tables.
@@ -373,12 +339,13 @@ void WasmEHPrepare::prepareEHPad(BasicBlock *BB, unsigned Index) {
     IRB.CreateStore(IRB.CreateCall(LSDAF), LSDAField);
 
   // Pseudocode: _Unwind_CallPersonality(exn);
-  CallInst *PersCI =
-      IRB.CreateCall(CallPersonalityF, Exn, OperandBundleDef("funclet", CPI));
+  CallInst *PersCI = IRB.CreateCall(CallPersonalityF, ExtractExnCI,
+                                    OperandBundleDef("funclet", CPI));
   PersCI->setDoesNotThrow();
 
   // Pseudocode: int selector = __wasm.landingpad_context.selector;
-  Instruction *Selector = IRB.CreateLoad(SelectorField, "selector");
+  Instruction *Selector =
+      IRB.CreateLoad(IRB.getInt32Ty(), SelectorField, "selector");
 
   // Replace the return value from wasm.get.ehselector() with the selector value
   // loaded from __wasm_lpad_context.selector.
@@ -388,15 +355,15 @@ void WasmEHPrepare::prepareEHPad(BasicBlock *BB, unsigned Index) {
 }
 
 void llvm::calculateWasmEHInfo(const Function *F, WasmEHFuncInfo &EHInfo) {
+  // If an exception is not caught by a catchpad (i.e., it is a foreign
+  // exception), it will unwind to its parent catchswitch's unwind destination.
+  // We don't record an unwind destination for cleanuppads because every
+  // exception should be caught by it.
   for (const auto &BB : *F) {
     if (!BB.isEHPad())
       continue;
     const Instruction *Pad = BB.getFirstNonPHI();
 
-    // If an exception is not caught by a catchpad (i.e., it is a foreign
-    // exception), it will unwind to its parent catchswitch's unwind
-    // destination. We don't record an unwind destination for cleanuppads
-    // because every exception should be caught by it.
     if (const auto *CatchPad = dyn_cast<CatchPadInst>(Pad)) {
       const auto *UnwindBB = CatchPad->getCatchSwitch()->getUnwindDest();
       if (!UnwindBB)
diff --git a/lib/CodeGen/WinEHPrepare.cpp b/lib/CodeGen/WinEHPrepare.cpp
index 6a15240fa6e0ad401338ea15bfa1071b4f3d5d19..d97d8e1dec57d06a704f3c09a65ab0b4846b8b90 100644
--- a/lib/CodeGen/WinEHPrepare.cpp
+++ b/lib/CodeGen/WinEHPrepare.cpp
@@ -1,9 +1,8 @@
 //===-- WinEHPrepare - Prepare exception handling for code generation ---===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -1080,7 +1079,8 @@ AllocaInst *WinEHPrepare::insertPHILoads(PHINode *PN, Function &F) {
     SpillSlot = new AllocaInst(PN->getType(), DL->getAllocaAddrSpace(), nullptr,
                                Twine(PN->getName(), ".wineh.spillslot"),
                                &F.getEntryBlock().front());
-    Value *V = new LoadInst(SpillSlot, Twine(PN->getName(), ".wineh.reload"),
+    Value *V = new LoadInst(PN->getType(), SpillSlot,
+                            Twine(PN->getName(), ".wineh.reload"),
                             &*PHIBlock->getFirstInsertionPt());
     PN->replaceAllUsesWith(V);
     return SpillSlot;
@@ -1222,13 +1222,15 @@ void WinEHPrepare::replaceUseWithLoad(Value *V, Use &U, AllocaInst *&SpillSlot,
     Value *&Load = Loads[IncomingBlock];
     // Insert the load into the predecessor block
     if (!Load)
-      Load = new LoadInst(SpillSlot, Twine(V->getName(), ".wineh.reload"),
+      Load = new LoadInst(V->getType(), SpillSlot,
+                          Twine(V->getName(), ".wineh.reload"),
                           /*Volatile=*/false, IncomingBlock->getTerminator());
 
     U.set(Load);
   } else {
     // Reload right before the old use.
-    auto *Load = new LoadInst(SpillSlot, Twine(V->getName(), ".wineh.reload"),
+    auto *Load = new LoadInst(V->getType(), SpillSlot,
+                              Twine(V->getName(), ".wineh.reload"),
                               /*Volatile=*/false, UsingInst);
     U.set(Load);
   }
diff --git a/lib/CodeGen/XRayInstrumentation.cpp b/lib/CodeGen/XRayInstrumentation.cpp
index 32a7457c20609bfe6b0f5f93a1e2722d151e69df..463f5185c2a77b963feacd52085dc5feaff9e993 100644
--- a/lib/CodeGen/XRayInstrumentation.cpp
+++ b/lib/CodeGen/XRayInstrumentation.cpp
@@ -1,9 +1,8 @@
 //===- XRayInstrumentation.cpp - Adds XRay instrumentation to functions. --===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/DebugInfo/CodeView/AppendingTypeTableBuilder.cpp b/lib/DebugInfo/CodeView/AppendingTypeTableBuilder.cpp
index 8828671d9be9de99df3cb85abb8e26618a87222f..323349869156ed78eb9d80488f68b1cc942bb7f8 100644
--- a/lib/DebugInfo/CodeView/AppendingTypeTableBuilder.cpp
+++ b/lib/DebugInfo/CodeView/AppendingTypeTableBuilder.cpp
@@ -1,9 +1,8 @@
 //===- AppendingTypeTableBuilder.cpp --------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/CodeView/CVSymbolVisitor.cpp b/lib/DebugInfo/CodeView/CVSymbolVisitor.cpp
index cbcaa569282818a8e4b3bc5617792b53241d932d..7d163f9f50f0702810b7c4dac98bbe7b6a509b7f 100644
--- a/lib/DebugInfo/CodeView/CVSymbolVisitor.cpp
+++ b/lib/DebugInfo/CodeView/CVSymbolVisitor.cpp
@@ -1,9 +1,8 @@
 //===- CVSymbolVisitor.cpp --------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/CodeView/CVTypeVisitor.cpp b/lib/DebugInfo/CodeView/CVTypeVisitor.cpp
index a4182a3b2fa15c76e9e36009f622e037c9fcaf4a..1a6f0c3e2af8672f0b238a51544802dbffd44cfb 100644
--- a/lib/DebugInfo/CodeView/CVTypeVisitor.cpp
+++ b/lib/DebugInfo/CodeView/CVTypeVisitor.cpp
@@ -1,9 +1,8 @@
 //===- CVTypeVisitor.cpp ----------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/CodeView/CodeViewError.cpp b/lib/DebugInfo/CodeView/CodeViewError.cpp
index 2a9753add311e6d43f859a4a2a784b1983d6e5b8..69390c708f59901b4a0d23b5147dee78036a8652 100644
--- a/lib/DebugInfo/CodeView/CodeViewError.cpp
+++ b/lib/DebugInfo/CodeView/CodeViewError.cpp
@@ -1,9 +1,8 @@
 //===- CodeViewError.cpp - Error extensions for CodeView --------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -14,6 +13,7 @@
 using namespace llvm;
 using namespace llvm::codeview;
 
+namespace {
 // FIXME: This class is only here to support the transition to llvm::Error. It
 // will be removed once this transition is complete. Clients should prefer to
 // deal with the Error value directly, rather than converting to error_code.
@@ -39,6 +39,7 @@ public:
     llvm_unreachable("Unrecognized cv_error_code");
   }
 };
+} // namespace
 
 static llvm::ManagedStatic<CodeViewErrorCategory> CodeViewErrCategory;
 const std::error_category &llvm::codeview::CVErrorCategory() {
diff --git a/lib/DebugInfo/CodeView/CodeViewRecordIO.cpp b/lib/DebugInfo/CodeView/CodeViewRecordIO.cpp
index 4fc14480578e673e8ac8b5e4a653c1a01ae3b1dd..92c795ab8fa6abfaf58079943a9a51fc7f3a0eaf 100644
--- a/lib/DebugInfo/CodeView/CodeViewRecordIO.cpp
+++ b/lib/DebugInfo/CodeView/CodeViewRecordIO.cpp
@@ -1,9 +1,8 @@
 //===- CodeViewRecordIO.cpp -------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/CodeView/DebugChecksumsSubsection.cpp b/lib/DebugInfo/CodeView/DebugChecksumsSubsection.cpp
index 0f155a95d60703bb456c838f6cf55f1c79cc64e5..3d28bac00c449cfe41655a7b1c244f86be14b2d1 100644
--- a/lib/DebugInfo/CodeView/DebugChecksumsSubsection.cpp
+++ b/lib/DebugInfo/CodeView/DebugChecksumsSubsection.cpp
@@ -1,9 +1,8 @@
 //===- DebugChecksumsSubsection.cpp ---------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/CodeView/DebugCrossExSubsection.cpp b/lib/DebugInfo/CodeView/DebugCrossExSubsection.cpp
index cef27787cfd109b11ad376a140e46bebf6db48cd..b23410409f8821645847747dde6bb696e8c70c67 100644
--- a/lib/DebugInfo/CodeView/DebugCrossExSubsection.cpp
+++ b/lib/DebugInfo/CodeView/DebugCrossExSubsection.cpp
@@ -1,9 +1,8 @@
 //===- DebugCrossExSubsection.cpp -----------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/CodeView/DebugCrossImpSubsection.cpp b/lib/DebugInfo/CodeView/DebugCrossImpSubsection.cpp
index 4001741f560a6c4ce5fe23ab054ea4127477788c..dbadafd3aaf3d3a5b281091dbe7e70bae8a7d602 100644
--- a/lib/DebugInfo/CodeView/DebugCrossImpSubsection.cpp
+++ b/lib/DebugInfo/CodeView/DebugCrossImpSubsection.cpp
@@ -1,9 +1,8 @@
 //===- DebugCrossImpSubsection.cpp ----------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/CodeView/DebugFrameDataSubsection.cpp b/lib/DebugInfo/CodeView/DebugFrameDataSubsection.cpp
index 5881bf177a55765b0c127c910c14cc40e2688708..be8c32d5b294f63ca4aef1069287430e8bc132bc 100644
--- a/lib/DebugInfo/CodeView/DebugFrameDataSubsection.cpp
+++ b/lib/DebugInfo/CodeView/DebugFrameDataSubsection.cpp
@@ -1,9 +1,8 @@
 //===- DebugFrameDataSubsection.cpp -----------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/CodeView/DebugInlineeLinesSubsection.cpp b/lib/DebugInfo/CodeView/DebugInlineeLinesSubsection.cpp
index 077c103a615b8b1f8aa8f162805c8f4532f2e6f2..48ec7e4ecdd647931e664b677ddc2b4bebed8cf2 100644
--- a/lib/DebugInfo/CodeView/DebugInlineeLinesSubsection.cpp
+++ b/lib/DebugInfo/CodeView/DebugInlineeLinesSubsection.cpp
@@ -1,9 +1,8 @@
 //===- DebugInlineeLinesSubsection.cpp ------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/CodeView/DebugLinesSubsection.cpp b/lib/DebugInfo/CodeView/DebugLinesSubsection.cpp
index 57ad40819fbc45165c69926f82b4a2378a6ccf65..ea16c0a6c6714774e475a39b97bdea6fa34b0436 100644
--- a/lib/DebugInfo/CodeView/DebugLinesSubsection.cpp
+++ b/lib/DebugInfo/CodeView/DebugLinesSubsection.cpp
@@ -1,9 +1,8 @@
 //===- DebugLinesSubsection.cpp -------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/CodeView/DebugStringTableSubsection.cpp b/lib/DebugInfo/CodeView/DebugStringTableSubsection.cpp
index 9b251f5931b3e62f429c4d5be322f36d63cc3cea..63342749918d8e0407a257b735b1bc43590e23f1 100644
--- a/lib/DebugInfo/CodeView/DebugStringTableSubsection.cpp
+++ b/lib/DebugInfo/CodeView/DebugStringTableSubsection.cpp
@@ -1,9 +1,8 @@
 //===- DebugStringTableSubsection.cpp - CodeView String Table -------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/CodeView/DebugSubsection.cpp b/lib/DebugInfo/CodeView/DebugSubsection.cpp
index 67b428bfa713349be7f70dc9dbb0b0caff211aa6..3f93463fe6d644a982fc89acac3e30647327761f 100644
--- a/lib/DebugInfo/CodeView/DebugSubsection.cpp
+++ b/lib/DebugInfo/CodeView/DebugSubsection.cpp
@@ -1,9 +1,8 @@
 //===- DebugSubsection.cpp -----------------------------------*- C++-*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/CodeView/DebugSubsectionRecord.cpp b/lib/DebugInfo/CodeView/DebugSubsectionRecord.cpp
index 55f343c11e7fe2412036d7d3da505917d8b5c6e8..0f704f286ee9db1110342814d8e1e5f90b09c12a 100644
--- a/lib/DebugInfo/CodeView/DebugSubsectionRecord.cpp
+++ b/lib/DebugInfo/CodeView/DebugSubsectionRecord.cpp
@@ -1,9 +1,8 @@
 //===- DebugSubsectionRecord.cpp ------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/CodeView/DebugSubsectionVisitor.cpp b/lib/DebugInfo/CodeView/DebugSubsectionVisitor.cpp
index 9b824333369be31b38741eac83ea3b03db87e109..7968b6a2d757577fc2ea624805f62a03c74d4894 100644
--- a/lib/DebugInfo/CodeView/DebugSubsectionVisitor.cpp
+++ b/lib/DebugInfo/CodeView/DebugSubsectionVisitor.cpp
@@ -1,9 +1,8 @@
 //===- DebugSubsectionVisitor.cpp -------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/CodeView/DebugSymbolRVASubsection.cpp b/lib/DebugInfo/CodeView/DebugSymbolRVASubsection.cpp
index 60fbf9d747b289775d378ef0380f7d19a7ef04fc..52328967357b02801f11c3339f79aec0bef524ce 100644
--- a/lib/DebugInfo/CodeView/DebugSymbolRVASubsection.cpp
+++ b/lib/DebugInfo/CodeView/DebugSymbolRVASubsection.cpp
@@ -1,9 +1,8 @@
 //===- DebugSymbolRVASubsection.cpp ---------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/CodeView/DebugSymbolsSubsection.cpp b/lib/DebugInfo/CodeView/DebugSymbolsSubsection.cpp
index dc8ba8c929aed3c934cab73a26199ad3e5addcdb..060eed7eafa0e1ad97a17bbf357e518611e781d5 100644
--- a/lib/DebugInfo/CodeView/DebugSymbolsSubsection.cpp
+++ b/lib/DebugInfo/CodeView/DebugSymbolsSubsection.cpp
@@ -1,9 +1,8 @@
 //===- DebugSymbolsSubsection.cpp -------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/CodeView/EnumTables.cpp b/lib/DebugInfo/CodeView/EnumTables.cpp
index ef4e42f79ebc9f71650dad82f1f2b4a6367ac929..2308c355136319f1af2f21b1c75c93f18f83eead 100644
--- a/lib/DebugInfo/CodeView/EnumTables.cpp
+++ b/lib/DebugInfo/CodeView/EnumTables.cpp
@@ -1,9 +1,8 @@
 //===- EnumTables.cpp - Enum to string conversion tables ------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/CodeView/Formatters.cpp b/lib/DebugInfo/CodeView/Formatters.cpp
index b8d89c76da3b6fbfb442e0fa658b78bb4f2878ea..a7a8c7ff82bf317ba3b12cb878c37497179f45ed 100644
--- a/lib/DebugInfo/CodeView/Formatters.cpp
+++ b/lib/DebugInfo/CodeView/Formatters.cpp
@@ -1,9 +1,8 @@
 //===- Formatters.cpp -----------------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/CodeView/GlobalTypeTableBuilder.cpp b/lib/DebugInfo/CodeView/GlobalTypeTableBuilder.cpp
index e76f9e12f0af7749b5e1c320aa47bc864c0fa03c..b0438bfa004e93d7999ef15a2f47685b8c4af1f8 100644
--- a/lib/DebugInfo/CodeView/GlobalTypeTableBuilder.cpp
+++ b/lib/DebugInfo/CodeView/GlobalTypeTableBuilder.cpp
@@ -1,9 +1,8 @@
 //===- GlobalTypeTableBuilder.cpp -----------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/CodeView/LLVMBuild.txt b/lib/DebugInfo/CodeView/LLVMBuild.txt
index f799a9eeae965002c358fdd95793c0d581027133..c3696fd20a4b2e782038a9258ca0d76f39f336ae 100644
--- a/lib/DebugInfo/CodeView/LLVMBuild.txt
+++ b/lib/DebugInfo/CodeView/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/DebugInfo/CodeView/LLVMBuild.txt -------------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/DebugInfo/CodeView/LazyRandomTypeCollection.cpp b/lib/DebugInfo/CodeView/LazyRandomTypeCollection.cpp
index ddcad8c631d7484c1978824d455d8b9bd8f2618c..dc1253b7a39fe10c5118e3b1cde39e8391a5bf33 100644
--- a/lib/DebugInfo/CodeView/LazyRandomTypeCollection.cpp
+++ b/lib/DebugInfo/CodeView/LazyRandomTypeCollection.cpp
@@ -1,9 +1,8 @@
 //===- LazyRandomTypeCollection.cpp ---------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/CodeView/Line.cpp b/lib/DebugInfo/CodeView/Line.cpp
index 4cb766b5fd2632fd0638bc5e37abb7e1c782adf8..53adc8cac511511e6966cb0a60bd1ba73a90b067 100644
--- a/lib/DebugInfo/CodeView/Line.cpp
+++ b/lib/DebugInfo/CodeView/Line.cpp
@@ -1,9 +1,8 @@
 //===-- Line.cpp ----------------------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/CodeView/MergingTypeTableBuilder.cpp b/lib/DebugInfo/CodeView/MergingTypeTableBuilder.cpp
index 8aee4aa2e2aee7e1406d1ee12966490705ad4ad9..c3a17b8f35eabe809647c2915f7d0e3aeafceadf 100644
--- a/lib/DebugInfo/CodeView/MergingTypeTableBuilder.cpp
+++ b/lib/DebugInfo/CodeView/MergingTypeTableBuilder.cpp
@@ -1,9 +1,8 @@
 //===- MergingTypeTableBuilder.cpp ----------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/CodeView/RecordName.cpp b/lib/DebugInfo/CodeView/RecordName.cpp
index d868ae237a44ff3ee2c5ce91d765075432f13072..cfaad1581159399486fd185ca54b4a07085d1c61 100644
--- a/lib/DebugInfo/CodeView/RecordName.cpp
+++ b/lib/DebugInfo/CodeView/RecordName.cpp
@@ -1,9 +1,8 @@
 //===- RecordName.cpp ----------------------------------------- *- C++ --*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/CodeView/RecordSerialization.cpp b/lib/DebugInfo/CodeView/RecordSerialization.cpp
index bff9a619a846139642c16f69eb4c0b5c1c162e23..e7f032f9c6702f37428c77195b525c3948c1a2bd 100644
--- a/lib/DebugInfo/CodeView/RecordSerialization.cpp
+++ b/lib/DebugInfo/CodeView/RecordSerialization.cpp
@@ -1,9 +1,8 @@
 //===-- RecordSerialization.cpp -------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/DebugInfo/CodeView/StringsAndChecksums.cpp b/lib/DebugInfo/CodeView/StringsAndChecksums.cpp
index 85d9dbb8c7dfed31d308a32ee13808b9644c5cae..9e204eec8604e15300ae3c24b83f548373735520 100644
--- a/lib/DebugInfo/CodeView/StringsAndChecksums.cpp
+++ b/lib/DebugInfo/CodeView/StringsAndChecksums.cpp
@@ -1,9 +1,8 @@
 //===- StringsAndChecksums.cpp --------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/CodeView/SymbolDumper.cpp b/lib/DebugInfo/CodeView/SymbolDumper.cpp
index 04e0bab745d3efdc5d7965ad46804d1b19f3a995..39cc18a19171475f49a84af6b92f35c01dddd56e 100644
--- a/lib/DebugInfo/CodeView/SymbolDumper.cpp
+++ b/lib/DebugInfo/CodeView/SymbolDumper.cpp
@@ -1,9 +1,8 @@
 //===-- SymbolDumper.cpp - CodeView symbol info dumper ----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/CodeView/SymbolRecordHelpers.cpp b/lib/DebugInfo/CodeView/SymbolRecordHelpers.cpp
index 01746138ad1f2df923ff1b1e721f0b140b3352c2..51a5a9e9243eae5f0288e60080829dd8fec6a3a9 100644
--- a/lib/DebugInfo/CodeView/SymbolRecordHelpers.cpp
+++ b/lib/DebugInfo/CodeView/SymbolRecordHelpers.cpp
@@ -1,9 +1,8 @@
 //===- SymbolRecordHelpers.cpp ----------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/CodeView/SymbolRecordMapping.cpp b/lib/DebugInfo/CodeView/SymbolRecordMapping.cpp
index 2af8205cebc35991731f09d6a3fe32536e2cf325..36d77965daf62555151896e24a0ad514a451e1c1 100644
--- a/lib/DebugInfo/CodeView/SymbolRecordMapping.cpp
+++ b/lib/DebugInfo/CodeView/SymbolRecordMapping.cpp
@@ -1,9 +1,8 @@
 //===- SymbolRecordMapping.cpp -----------------------------------*- C++-*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/CodeView/SymbolSerializer.cpp b/lib/DebugInfo/CodeView/SymbolSerializer.cpp
index 0071ecc856853d8aa7f5d06551ce39e9180dd3cd..de9bb42b17987336f85559369a117ef6cb720769 100644
--- a/lib/DebugInfo/CodeView/SymbolSerializer.cpp
+++ b/lib/DebugInfo/CodeView/SymbolSerializer.cpp
@@ -1,9 +1,8 @@
 //===- SymbolSerializer.cpp -----------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/CodeView/TypeDumpVisitor.cpp b/lib/DebugInfo/CodeView/TypeDumpVisitor.cpp
index f5d3bea43a14799e1ddc3c15e53914dd956667a1..56406bfec0921320ad49aefa4ed6159a7888f535 100644
--- a/lib/DebugInfo/CodeView/TypeDumpVisitor.cpp
+++ b/lib/DebugInfo/CodeView/TypeDumpVisitor.cpp
@@ -1,9 +1,8 @@
 //===-- TypeDumpVisitor.cpp - CodeView type info dumper ----------*- C++-*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/CodeView/TypeHashing.cpp b/lib/DebugInfo/CodeView/TypeHashing.cpp
index 826faef35875816c5efe0310b736d0e9c964cd2d..2dbc11a84f0b03cace938e2a8618e31ce17070da 100644
--- a/lib/DebugInfo/CodeView/TypeHashing.cpp
+++ b/lib/DebugInfo/CodeView/TypeHashing.cpp
@@ -1,9 +1,8 @@
 //===- TypeHashing.cpp -------------------------------------------*- C++-*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -55,10 +54,16 @@ GloballyHashedType::hashType(ArrayRef<uint8_t> RecordData,
         reinterpret_cast<const TypeIndex *>(RefData.data()), Ref.Count);
     for (TypeIndex TI : Indices) {
       ArrayRef<uint8_t> BytesToHash;
-      if (TI.isSimple() || TI.isNoneType() || TI.toArrayIndex() >= Prev.size()) {
+      if (TI.isSimple() || TI.isNoneType()) {
         const uint8_t *IndexBytes = reinterpret_cast<const uint8_t *>(&TI);
         BytesToHash = makeArrayRef(IndexBytes, sizeof(TypeIndex));
       } else {
+        if (TI.toArrayIndex() >= Prev.size() ||
+            Prev[TI.toArrayIndex()].empty()) {
+          // There are references to yet-unhashed records. Suspend hashing for
+          // this record until all the other records are processed.
+          return {};
+        }
         BytesToHash = Prev[TI.toArrayIndex()].Hash;
       }
       S.update(BytesToHash);
diff --git a/lib/DebugInfo/CodeView/TypeIndex.cpp b/lib/DebugInfo/CodeView/TypeIndex.cpp
index 332d67470da50dd12360bdc067dc1a2bdda5a8e8..604d342448d3fa7aa9e6679d911203ebc3e122f5 100644
--- a/lib/DebugInfo/CodeView/TypeIndex.cpp
+++ b/lib/DebugInfo/CodeView/TypeIndex.cpp
@@ -1,9 +1,8 @@
 //===-- TypeIndex.cpp - CodeView type index ---------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/CodeView/TypeIndexDiscovery.cpp b/lib/DebugInfo/CodeView/TypeIndexDiscovery.cpp
index 839ab6f0a70582c7376846e784b11d5aeaff3074..b01511e06e795ad38bf43d2a37d8a02ce0e70208 100644
--- a/lib/DebugInfo/CodeView/TypeIndexDiscovery.cpp
+++ b/lib/DebugInfo/CodeView/TypeIndexDiscovery.cpp
@@ -1,9 +1,8 @@
 //===- TypeIndexDiscovery.cpp -----------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 #include "llvm/DebugInfo/CodeView/TypeIndexDiscovery.h"
diff --git a/lib/DebugInfo/CodeView/TypeRecordHelpers.cpp b/lib/DebugInfo/CodeView/TypeRecordHelpers.cpp
index 2a66474cf5b6647af3a3017b6da3883a064da153..8e632f3be4609ad1b967eb58f7accabbda64f936 100644
--- a/lib/DebugInfo/CodeView/TypeRecordHelpers.cpp
+++ b/lib/DebugInfo/CodeView/TypeRecordHelpers.cpp
@@ -1,9 +1,8 @@
 //===- TypeRecordHelpers.cpp ------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/CodeView/TypeRecordMapping.cpp b/lib/DebugInfo/CodeView/TypeRecordMapping.cpp
index 3203ff64d3b1e210427c081501500cb3b40e1dcc..5a9f6c616ac47208fbc436280555593c3529798e 100644
--- a/lib/DebugInfo/CodeView/TypeRecordMapping.cpp
+++ b/lib/DebugInfo/CodeView/TypeRecordMapping.cpp
@@ -1,9 +1,8 @@
 //===- TypeRecordMapping.cpp ------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/CodeView/TypeStreamMerger.cpp b/lib/DebugInfo/CodeView/TypeStreamMerger.cpp
index 803818226e54b7b450ed19723c962886d0652019..aba0e96d606e7129de2c575af6be44985a24a187 100644
--- a/lib/DebugInfo/CodeView/TypeStreamMerger.cpp
+++ b/lib/DebugInfo/CodeView/TypeStreamMerger.cpp
@@ -1,9 +1,8 @@
 //===-- TypeStreamMerger.cpp ------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -77,8 +76,7 @@ public:
   // Local hashing entry points
   Error mergeTypesAndIds(MergingTypeTableBuilder &DestIds,
                          MergingTypeTableBuilder &DestTypes,
-                         const CVTypeArray &IdsAndTypes,
-                         Optional<EndPrecompRecord> &EP);
+                         const CVTypeArray &IdsAndTypes, Optional<uint32_t> &S);
   Error mergeIdRecords(MergingTypeTableBuilder &Dest,
                        ArrayRef<TypeIndex> TypeSourceToDest,
                        const CVTypeArray &Ids);
@@ -90,14 +88,14 @@ public:
                          GlobalTypeTableBuilder &DestTypes,
                          const CVTypeArray &IdsAndTypes,
                          ArrayRef<GloballyHashedType> Hashes,
-                         Optional<EndPrecompRecord> &EP);
+                         Optional<uint32_t> &S);
   Error mergeIdRecords(GlobalTypeTableBuilder &Dest,
                        ArrayRef<TypeIndex> TypeSourceToDest,
                        const CVTypeArray &Ids,
                        ArrayRef<GloballyHashedType> Hashes);
   Error mergeTypeRecords(GlobalTypeTableBuilder &Dest, const CVTypeArray &Types,
                          ArrayRef<GloballyHashedType> Hashes,
-                         Optional<EndPrecompRecord> &EP);
+                         Optional<uint32_t> &S);
 
 private:
   Error doit(const CVTypeArray &Types);
@@ -197,7 +195,7 @@ private:
   /// its type indices.
   SmallVector<uint8_t, 256> RemapStorage;
 
-  Optional<EndPrecompRecord> EndPrecomp; 
+  Optional<uint32_t> PCHSignature;
 };
 
 } // end anonymous namespace
@@ -275,12 +273,12 @@ Error TypeStreamMerger::mergeIdRecords(MergingTypeTableBuilder &Dest,
 Error TypeStreamMerger::mergeTypesAndIds(MergingTypeTableBuilder &DestIds,
                                          MergingTypeTableBuilder &DestTypes,
                                          const CVTypeArray &IdsAndTypes,
-                                         Optional<EndPrecompRecord> &EP) {
+                                         Optional<uint32_t> &S) {
   DestIdStream = &DestIds;
   DestTypeStream = &DestTypes;
   UseGlobalHashes = false;
   auto Err = doit(IdsAndTypes);
-  EP = EndPrecomp;
+  S = PCHSignature;
   return Err;
 }
 
@@ -288,12 +286,12 @@ Error TypeStreamMerger::mergeTypesAndIds(MergingTypeTableBuilder &DestIds,
 Error TypeStreamMerger::mergeTypeRecords(GlobalTypeTableBuilder &Dest,
                                          const CVTypeArray &Types,
                                          ArrayRef<GloballyHashedType> Hashes,
-                                         Optional<EndPrecompRecord> &EP) {
+                                         Optional<uint32_t> &S) {
   DestGlobalTypeStream = &Dest;
   UseGlobalHashes = true;
   GlobalHashes = Hashes;
   auto Err = doit(Types);
-  EP = EndPrecomp;
+  S = PCHSignature;
   return Err;
 }
 
@@ -313,13 +311,13 @@ Error TypeStreamMerger::mergeTypesAndIds(GlobalTypeTableBuilder &DestIds,
                                          GlobalTypeTableBuilder &DestTypes,
                                          const CVTypeArray &IdsAndTypes,
                                          ArrayRef<GloballyHashedType> Hashes,
-                                         Optional<EndPrecompRecord> &EP) {
+                                         Optional<uint32_t> &S) {
   DestGlobalIdStream = &DestIds;
   DestGlobalTypeStream = &DestTypes;
   UseGlobalHashes = true;
   GlobalHashes = Hashes;
   auto Err = doit(IdsAndTypes);
-  EP = EndPrecomp;
+  S = PCHSignature;
   return Err;
 }
 
@@ -445,28 +443,27 @@ Error llvm::codeview::mergeIdRecords(MergingTypeTableBuilder &Dest,
 Error llvm::codeview::mergeTypeAndIdRecords(
     MergingTypeTableBuilder &DestIds, MergingTypeTableBuilder &DestTypes,
     SmallVectorImpl<TypeIndex> &SourceToDest, const CVTypeArray &IdsAndTypes,
-    Optional<EndPrecompRecord> &EndPrecomp) {
+    Optional<uint32_t> &PCHSignature) {
   TypeStreamMerger M(SourceToDest);
-  return M.mergeTypesAndIds(DestIds, DestTypes, IdsAndTypes, EndPrecomp);
+  return M.mergeTypesAndIds(DestIds, DestTypes, IdsAndTypes, PCHSignature);
 }
 
 Error llvm::codeview::mergeTypeAndIdRecords(
     GlobalTypeTableBuilder &DestIds, GlobalTypeTableBuilder &DestTypes,
     SmallVectorImpl<TypeIndex> &SourceToDest, const CVTypeArray &IdsAndTypes,
-    ArrayRef<GloballyHashedType> Hashes,
-    Optional<EndPrecompRecord> &EndPrecomp) {
+    ArrayRef<GloballyHashedType> Hashes, Optional<uint32_t> &PCHSignature) {
   TypeStreamMerger M(SourceToDest);
   return M.mergeTypesAndIds(DestIds, DestTypes, IdsAndTypes, Hashes,
-                            EndPrecomp);
+                            PCHSignature);
 }
 
 Error llvm::codeview::mergeTypeRecords(GlobalTypeTableBuilder &Dest,
                                        SmallVectorImpl<TypeIndex> &SourceToDest,
                                        const CVTypeArray &Types,
                                        ArrayRef<GloballyHashedType> Hashes,
-                                       Optional<EndPrecompRecord> &EndPrecomp) {
+                                       Optional<uint32_t> &PCHSignature) {
   TypeStreamMerger M(SourceToDest);
-  return M.mergeTypeRecords(Dest, Types, Hashes, EndPrecomp);
+  return M.mergeTypeRecords(Dest, Types, Hashes, PCHSignature);
 }
 
 Error llvm::codeview::mergeIdRecords(GlobalTypeTableBuilder &Dest,
@@ -483,11 +480,13 @@ Expected<bool> TypeStreamMerger::shouldRemapType(const CVType &Type) {
   // signature, through EndPrecompRecord. This is done here for performance
   // reasons, to avoid re-parsing the Types stream.
   if (Type.kind() == LF_ENDPRECOMP) {
-    assert(!EndPrecomp);
-    EndPrecomp.emplace();
+    EndPrecompRecord EP;
     if (auto EC = TypeDeserializer::deserializeAs(const_cast<CVType &>(Type),
-                                                  EndPrecomp.getValue()))
+                                                  EP))
       return joinErrors(std::move(EC), errorCorruptRecord());
+    if (PCHSignature.hasValue())
+      return errorCorruptRecord();
+    PCHSignature.emplace(EP.getSignature());
     return false;
   }
   return true;
diff --git a/lib/DebugInfo/CodeView/TypeTableCollection.cpp b/lib/DebugInfo/CodeView/TypeTableCollection.cpp
index cf951baa5111ca9b68bbdab99d98938e005a2663..3434012eb6f18983225fadb7526cc218db2e14d1 100644
--- a/lib/DebugInfo/CodeView/TypeTableCollection.cpp
+++ b/lib/DebugInfo/CodeView/TypeTableCollection.cpp
@@ -1,9 +1,8 @@
 //===- TypeTableCollection.cpp -------------------------------- *- C++ --*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/DWARF/DWARFAbbreviationDeclaration.cpp b/lib/DebugInfo/DWARF/DWARFAbbreviationDeclaration.cpp
index f49ab40fad9a9493d76a6831a20922d677604c24..28354dafb84a9e6a9590c4b7dead6d5bea3f285d 100644
--- a/lib/DebugInfo/DWARF/DWARFAbbreviationDeclaration.cpp
+++ b/lib/DebugInfo/DWARF/DWARFAbbreviationDeclaration.cpp
@@ -1,9 +1,8 @@
 //===- DWARFAbbreviationDeclaration.cpp -----------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/DWARF/DWARFAcceleratorTable.cpp b/lib/DebugInfo/DWARF/DWARFAcceleratorTable.cpp
index 54daf34ff253e0dd64f9db4ebc17eb3d912e758c..1d8f23499bc0449e4f03b0dfba193cb537c4b7d2 100644
--- a/lib/DebugInfo/DWARF/DWARFAcceleratorTable.cpp
+++ b/lib/DebugInfo/DWARF/DWARFAcceleratorTable.cpp
@@ -1,9 +1,8 @@
 //===- DWARFAcceleratorTable.cpp ------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/DWARF/DWARFAddressRange.cpp b/lib/DebugInfo/DWARF/DWARFAddressRange.cpp
index 86c8d19c02f450dda0ecab949e9ba51d77cf61ab..ef6da08d34aa7c01ec13f05b2dc6f124a4cd4d52 100644
--- a/lib/DebugInfo/DWARF/DWARFAddressRange.cpp
+++ b/lib/DebugInfo/DWARF/DWARFAddressRange.cpp
@@ -1,9 +1,8 @@
 //===- DWARFDebugAranges.cpp ------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/DWARF/DWARFCompileUnit.cpp b/lib/DebugInfo/DWARF/DWARFCompileUnit.cpp
index 00a23b3898fa5a16fcc3030d8b93ad18ae8db3cb..74cce42466ddecbb9958cba01f3947d76f682769 100644
--- a/lib/DebugInfo/DWARF/DWARFCompileUnit.cpp
+++ b/lib/DebugInfo/DWARF/DWARFCompileUnit.cpp
@@ -1,9 +1,8 @@
 //===-- DWARFCompileUnit.cpp ----------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/DWARF/DWARFContext.cpp b/lib/DebugInfo/DWARF/DWARFContext.cpp
index e6620ee3dd1d16a342b64ba0b2e525fdb6d3a0a4..d06b737b810202646bb89781273ff437d6190905 100644
--- a/lib/DebugInfo/DWARF/DWARFContext.cpp
+++ b/lib/DebugInfo/DWARF/DWARFContext.cpp
@@ -1,9 +1,8 @@
 //===- DWARFContext.cpp ---------------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/DWARF/DWARFDataExtractor.cpp b/lib/DebugInfo/DWARF/DWARFDataExtractor.cpp
index 03e3174613962ade66aa14cc14b29c38986d0878..73663f868f048333404abe6ae05378e578496591 100644
--- a/lib/DebugInfo/DWARF/DWARFDataExtractor.cpp
+++ b/lib/DebugInfo/DWARF/DWARFDataExtractor.cpp
@@ -1,9 +1,8 @@
 //===- DWARFDataExtractor.cpp ---------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/DWARF/DWARFDebugAbbrev.cpp b/lib/DebugInfo/DWARF/DWARFDebugAbbrev.cpp
index 4830c36a8ee791209e558571c8e3b5d2995e4bb0..f46545ff3d214d9ccf836391025f7a23a9d1b62d 100644
--- a/lib/DebugInfo/DWARF/DWARFDebugAbbrev.cpp
+++ b/lib/DebugInfo/DWARF/DWARFDebugAbbrev.cpp
@@ -1,9 +1,8 @@
 //===- DWARFDebugAbbrev.cpp -----------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/DWARF/DWARFDebugAddr.cpp b/lib/DebugInfo/DWARF/DWARFDebugAddr.cpp
index 22759bfac26c859d3b3b5f329f4685de95bb5dc7..6ab2a2675cce1e74d1ffc8980e9b0699597135a5 100644
--- a/lib/DebugInfo/DWARF/DWARFDebugAddr.cpp
+++ b/lib/DebugInfo/DWARF/DWARFDebugAddr.cpp
@@ -1,9 +1,8 @@
 //===- DWARFDebugAddr.cpp -------------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/DWARF/DWARFDebugArangeSet.cpp b/lib/DebugInfo/DWARF/DWARFDebugArangeSet.cpp
index b9ef6905912adf3684f753f1a2803488bfb75db3..6551b61accb8741511d7b6cb217d9b33c42dfb74 100644
--- a/lib/DebugInfo/DWARF/DWARFDebugArangeSet.cpp
+++ b/lib/DebugInfo/DWARF/DWARFDebugArangeSet.cpp
@@ -1,9 +1,8 @@
 //===- DWARFDebugArangeSet.cpp --------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/DWARF/DWARFDebugAranges.cpp b/lib/DebugInfo/DWARF/DWARFDebugAranges.cpp
index c26db9ab96e0ac061ae70fc8b4ff3a40babe36de..d4714b4fb3a9b0afcc6be5abc3a85f3da1f03c13 100644
--- a/lib/DebugInfo/DWARF/DWARFDebugAranges.cpp
+++ b/lib/DebugInfo/DWARF/DWARFDebugAranges.cpp
@@ -1,9 +1,8 @@
 //===- DWARFDebugAranges.cpp ----------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -12,6 +11,7 @@
 #include "llvm/DebugInfo/DWARF/DWARFContext.h"
 #include "llvm/DebugInfo/DWARF/DWARFDebugArangeSet.h"
 #include "llvm/Support/DataExtractor.h"
+#include "llvm/Support/WithColor.h"
 #include <algorithm>
 #include <cassert>
 #include <cstdint>
@@ -53,10 +53,12 @@ void DWARFDebugAranges::generate(DWARFContext *CTX) {
   for (const auto &CU : CTX->compile_units()) {
     uint32_t CUOffset = CU->getOffset();
     if (ParsedCUOffsets.insert(CUOffset).second) {
-      DWARFAddressRangesVector CURanges;
-      CU->collectAddressRanges(CURanges);
-      for (const auto &R : CURanges)
-        appendRange(CUOffset, R.LowPC, R.HighPC);
+      Expected<DWARFAddressRangesVector> CURanges = CU->collectAddressRanges();
+      if (!CURanges)
+        WithColor::error() << toString(CURanges.takeError()) << '\n';
+      else
+        for (const auto &R : *CURanges)
+          appendRange(CUOffset, R.LowPC, R.HighPC);
     }
   }
 
diff --git a/lib/DebugInfo/DWARF/DWARFDebugFrame.cpp b/lib/DebugInfo/DWARF/DWARFDebugFrame.cpp
index 6349a6efb8c21ee43398d63dfdc9a1f998b3d455..154e466e91113427aca1f817ada8d838c5af3f18 100644
--- a/lib/DebugInfo/DWARF/DWARFDebugFrame.cpp
+++ b/lib/DebugInfo/DWARF/DWARFDebugFrame.cpp
@@ -1,9 +1,8 @@
 //===- DWARFDebugFrame.h - Parsing of .debug_frame ------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -446,6 +445,11 @@ void DWARFDebugFrame::parse(DWARFDataExtractor Data) {
               StartAugmentationOffset = Offset;
               EndAugmentationOffset = Offset +
                 static_cast<uint32_t>(*AugmentationLength);
+              break;
+            case 'B':
+              // B-Key is used for signing functions associated with this
+              // augmentation string
+              break;
           }
         }
 
diff --git a/lib/DebugInfo/DWARF/DWARFDebugInfoEntry.cpp b/lib/DebugInfo/DWARF/DWARFDebugInfoEntry.cpp
index 976bc4651ae696b5c7cdd81ad618c28625ceee88..d8a755e90df4bf39d17714e59813817fe71acae0 100644
--- a/lib/DebugInfo/DWARF/DWARFDebugInfoEntry.cpp
+++ b/lib/DebugInfo/DWARF/DWARFDebugInfoEntry.cpp
@@ -1,9 +1,8 @@
 //===- DWARFDebugInfoEntry.cpp --------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/DWARF/DWARFDebugLine.cpp b/lib/DebugInfo/DWARF/DWARFDebugLine.cpp
index 1d621ff244f3dea97e57688857b1c73fbf656c4c..ac9b2e0d28fd6e7e00c5d99253fc68906019412e 100644
--- a/lib/DebugInfo/DWARF/DWARFDebugLine.cpp
+++ b/lib/DebugInfo/DWARF/DWARFDebugLine.cpp
@@ -1,9 +1,8 @@
 //===- DWARFDebugLine.cpp -------------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -818,8 +817,8 @@ Error DWARFDebugLine::LineTable::parse(
       State.Row.Address += AddrOffset;
 
       if (OS) {
-        *OS << "address += " << ((uint32_t)AdjustOpcode)
-            << ",  line += " << LineOffset << "\n";
+        *OS << "address += " << AddrOffset << ",  line += " << LineOffset
+            << "\n";
         OS->indent(12);
         State.Row.dump(*OS);
       }
diff --git a/lib/DebugInfo/DWARF/DWARFDebugLoc.cpp b/lib/DebugInfo/DWARF/DWARFDebugLoc.cpp
index f8b5ff6ec8fb60344cc9d07142fdb361b9587e32..54e0e8e18fab0d211b05f2da322035a2d3c29fcc 100644
--- a/lib/DebugInfo/DWARF/DWARFDebugLoc.cpp
+++ b/lib/DebugInfo/DWARF/DWARFDebugLoc.cpp
@@ -1,9 +1,8 @@
 //===- DWARFDebugLoc.cpp --------------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -184,7 +183,8 @@ DWARFDebugLoclists::parseOneLocationList(DataExtractor Data, unsigned *Offset,
     }
 
     if (Kind != dwarf::DW_LLE_base_address) {
-      unsigned Bytes = Data.getU16(Offset);
+      unsigned Bytes =
+          Version >= 5 ? Data.getULEB128(Offset) : Data.getU16(Offset);
       // A single location description describing the location of the object...
       StringRef str = Data.getData().substr(*Offset, Bytes);
       *Offset += Bytes;
diff --git a/lib/DebugInfo/DWARF/DWARFDebugMacro.cpp b/lib/DebugInfo/DWARF/DWARFDebugMacro.cpp
index 6d789c3027a5ac0fd2ddc58fe8e23a06b6528429..3317a778cc7018a5687a53c717420cc17e1812bb 100644
--- a/lib/DebugInfo/DWARF/DWARFDebugMacro.cpp
+++ b/lib/DebugInfo/DWARF/DWARFDebugMacro.cpp
@@ -1,9 +1,8 @@
 //===- DWARFDebugMacro.cpp ------------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/DWARF/DWARFDebugPubTable.cpp b/lib/DebugInfo/DWARF/DWARFDebugPubTable.cpp
index abd1ad59a9c18d4d1e9f22ae7f38fe275279d12c..963ec64f5e91c532cc4fa06c2ec83c23aa33577a 100644
--- a/lib/DebugInfo/DWARF/DWARFDebugPubTable.cpp
+++ b/lib/DebugInfo/DWARF/DWARFDebugPubTable.cpp
@@ -1,9 +1,8 @@
 //===- DWARFDebugPubTable.cpp ---------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/DWARF/DWARFDebugRangeList.cpp b/lib/DebugInfo/DWARF/DWARFDebugRangeList.cpp
index dfb913000a46219904866f4995ed13b22c7d5e35..31193b563a54699b8103960182fe6fb4dda52d6b 100644
--- a/lib/DebugInfo/DWARF/DWARFDebugRangeList.cpp
+++ b/lib/DebugInfo/DWARF/DWARFDebugRangeList.cpp
@@ -1,9 +1,8 @@
 //===- DWARFDebugRangesList.cpp -------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/DWARF/DWARFDebugRnglists.cpp b/lib/DebugInfo/DWARF/DWARFDebugRnglists.cpp
index 60c6eb30857fa303e1b62e382f6868d39b7c394f..2e4babac6c7f4fce7ffce67ab7e3ef6b5c6774d3 100644
--- a/lib/DebugInfo/DWARF/DWARFDebugRnglists.cpp
+++ b/lib/DebugInfo/DWARF/DWARFDebugRnglists.cpp
@@ -1,9 +1,8 @@
 //===- DWARFDebugRnglists.cpp ---------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/DWARF/DWARFDie.cpp b/lib/DebugInfo/DWARF/DWARFDie.cpp
index 551e292fb0323d35ecd11296a01bbe837edf048b..b1aa77d1f236d6e68a8c993a1ba11017b7099843 100644
--- a/lib/DebugInfo/DWARF/DWARFDie.cpp
+++ b/lib/DebugInfo/DWARF/DWARFDie.cpp
@@ -1,9 +1,8 @@
 //===- DWARFDie.cpp -------------------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -71,15 +70,7 @@ static void dumpRanges(const DWARFObject &Obj, raw_ostream &OS,
     OS.indent(Indent);
     R.dump(OS, AddressSize);
 
-    if (SectionNames.empty() || R.SectionIndex == -1ULL)
-      continue;
-
-    StringRef Name = SectionNames[R.SectionIndex].Name;
-    OS << " \"" << Name << '\"';
-
-    // Print section index if name is not unique.
-    if (!SectionNames[R.SectionIndex].IsNameUnique)
-      OS << format(" [%" PRIu64 "]", R.SectionIndex);
+    DWARFFormValue::dumpAddressSection(Obj, OS, DumpOpts, R.SectionIndex);
   }
 }
 
@@ -154,6 +145,52 @@ static void dumpTypeTagName(raw_ostream &OS, dwarf::Tag T) {
   OS << TagStr.substr(7, TagStr.size() - 12) << " ";
 }
 
+static void dumpArrayType(raw_ostream &OS, const DWARFDie &D) {
+  Optional<uint64_t> Bound;
+  for (const DWARFDie &C : D.children())
+    if (C.getTag() == DW_TAG_subrange_type) {
+      Optional<uint64_t> LB;
+      Optional<uint64_t> Count;
+      Optional<uint64_t> UB;
+      Optional<unsigned> DefaultLB;
+      if (Optional<DWARFFormValue> L = C.find(DW_AT_lower_bound))
+        LB = L->getAsUnsignedConstant();
+      if (Optional<DWARFFormValue> CountV = C.find(DW_AT_count))
+        Count = CountV->getAsUnsignedConstant();
+      if (Optional<DWARFFormValue> UpperV = C.find(DW_AT_upper_bound))
+        UB = UpperV->getAsUnsignedConstant();
+      if (Optional<DWARFFormValue> LV =
+              D.getDwarfUnit()->getUnitDIE().find(DW_AT_language))
+        if (Optional<uint64_t> LC = LV->getAsUnsignedConstant())
+          if ((DefaultLB =
+                   LanguageLowerBound(static_cast<dwarf::SourceLanguage>(*LC))))
+            if (LB && *LB == *DefaultLB)
+              LB = None;
+      if (!LB && !Count && !UB)
+        OS << "[]";
+      else if (!LB && (Count || UB) && DefaultLB)
+        OS << '[' << (Count ? *Count : *UB - *DefaultLB + 1) << ']';
+      else {
+        OS << "[[";
+        if (LB)
+          OS << *LB;
+        else
+          OS << '?';
+        OS << ", ";
+        if (Count)
+          if (LB)
+            OS << *LB + *Count;
+          else
+            OS << "? + " << *Count;
+        else if (UB)
+          OS << *UB + 1;
+        else
+          OS << '?';
+        OS << ")]";
+      }
+    }
+}
+
 /// Recursively dump the DIE type name when applicable.
 static void dumpTypeName(raw_ostream &OS, const DWARFDie &D) {
   if (!D.isValid())
@@ -201,24 +238,7 @@ static void dumpTypeName(raw_ostream &OS, const DWARFDie &D) {
     break;
   }
   case DW_TAG_array_type: {
-    Optional<uint64_t> Bound;
-    for (const DWARFDie &C : D.children())
-      if (C.getTag() == DW_TAG_subrange_type) {
-        OS << '[';
-        uint64_t LowerBound = 0;
-        if (Optional<DWARFFormValue> L = C.find(DW_AT_lower_bound))
-          if (Optional<uint64_t> LB = L->getAsUnsignedConstant()) {
-            LowerBound = *LB;
-            OS << LowerBound << '-';
-          }
-        if (Optional<DWARFFormValue> CountV = C.find(DW_AT_count)) {
-          if (Optional<uint64_t> C = CountV->getAsUnsignedConstant())
-            OS << (*C + LowerBound);
-        } else if (Optional<DWARFFormValue> UpperV = C.find(DW_AT_upper_bound))
-          if (Optional<uint64_t> U = UpperV->getAsUnsignedConstant())
-            OS << *U;
-        OS << ']';
-      }
+    dumpArrayType(OS, D);
     break;
   }
   case DW_TAG_pointer_type:
@@ -324,10 +344,9 @@ static void dumpAttribute(raw_ostream &OS, const DWARFDie &Die,
     const DWARFObject &Obj = Die.getDwarfUnit()->getContext().getDWARFObj();
     // For DW_FORM_rnglistx we need to dump the offset separately, since
     // we have only dumped the index so far.
-    Optional<DWARFFormValue> Value = Die.find(DW_AT_ranges);
-    if (Value && Value->getForm() == DW_FORM_rnglistx)
+    if (formValue.getForm() == DW_FORM_rnglistx)
       if (auto RangeListOffset =
-              U->getRnglistOffset(*Value->getAsSectionOffset())) {
+              U->getRnglistOffset(*formValue.getAsSectionOffset())) {
         DWARFFormValue FV(dwarf::DW_FORM_sec_offset);
         FV.setUValue(*RangeListOffset);
         FV.dump(OS, DumpOpts);
@@ -446,13 +465,13 @@ Optional<uint64_t> DWARFDie::getHighPC(uint64_t LowPC) const {
 bool DWARFDie::getLowAndHighPC(uint64_t &LowPC, uint64_t &HighPC,
                                uint64_t &SectionIndex) const {
   auto F = find(DW_AT_low_pc);
-  auto LowPcAddr = toAddress(F);
+  auto LowPcAddr = toSectionedAddress(F);
   if (!LowPcAddr)
     return false;
-  if (auto HighPcAddr = getHighPC(*LowPcAddr)) {
-    LowPC = *LowPcAddr;
+  if (auto HighPcAddr = getHighPC(LowPcAddr->Address)) {
+    LowPC = LowPcAddr->Address;
     HighPC = *HighPcAddr;
-    SectionIndex = F->getSectionIndex();
+    SectionIndex = LowPcAddr->SectionIndex;
     return true;
   }
   return false;
diff --git a/lib/DebugInfo/DWARF/DWARFExpression.cpp b/lib/DebugInfo/DWARF/DWARFExpression.cpp
index 2df4456053fb13e03edd753df709aad2146a67c2..920fa183e4526b34811b5920c160f4394b1a27b6 100644
--- a/lib/DebugInfo/DWARF/DWARFExpression.cpp
+++ b/lib/DebugInfo/DWARF/DWARFExpression.cpp
@@ -1,9 +1,8 @@
 //===-- DWARFExpression.cpp -----------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/DWARF/DWARFFormValue.cpp b/lib/DebugInfo/DWARF/DWARFFormValue.cpp
index 9226dcad39a9438f35651c6fae5acd300978867b..ffcd25709e6b0cd83add10b264ba5f9d23b5e575 100644
--- a/lib/DebugInfo/DWARF/DWARFFormValue.cpp
+++ b/lib/DebugInfo/DWARF/DWARFFormValue.cpp
@@ -1,9 +1,8 @@
 //===- DWARFFormValue.cpp -------------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -331,6 +330,29 @@ bool DWARFFormValue::extractValue(const DWARFDataExtractor &Data,
   return true;
 }
 
+void DWARFFormValue::dumpSectionedAddress(raw_ostream &OS,
+                                          DIDumpOptions DumpOpts,
+                                          SectionedAddress SA) const {
+  OS << format("0x%016" PRIx64, SA.Address);
+  dumpAddressSection(U->getContext().getDWARFObj(), OS, DumpOpts,
+                     SA.SectionIndex);
+}
+
+void DWARFFormValue::dumpAddressSection(const DWARFObject &Obj, raw_ostream &OS,
+                                        DIDumpOptions DumpOpts,
+                                        uint64_t SectionIndex) {
+  if (!DumpOpts.Verbose || SectionIndex == -1ULL)
+    return;
+  ArrayRef<SectionName> SectionNames = Obj.getSectionNames();
+  const auto &SecRef = SectionNames[SectionIndex];
+
+  OS << " \"" << SecRef.Name << '\"';
+
+  // Print section index if name is not unique.
+  if (!SecRef.IsNameUnique)
+    OS << format(" [%" PRIu64 "]", SectionIndex);
+}
+
 void DWARFFormValue::dump(raw_ostream &OS, DIDumpOptions DumpOpts) const {
   uint64_t UValue = Value.uval;
   bool CURelativeOffset = false;
@@ -339,7 +361,7 @@ void DWARFFormValue::dump(raw_ostream &OS, DIDumpOptions DumpOpts) const {
                             : nulls();
   switch (Form) {
   case DW_FORM_addr:
-    AddrOS << format("0x%016" PRIx64, UValue);
+    dumpSectionedAddress(AddrOS, DumpOpts, {Value.uval, Value.SectionIndex});
     break;
   case DW_FORM_addrx:
   case DW_FORM_addrx1:
@@ -347,11 +369,13 @@ void DWARFFormValue::dump(raw_ostream &OS, DIDumpOptions DumpOpts) const {
   case DW_FORM_addrx3:
   case DW_FORM_addrx4:
   case DW_FORM_GNU_addr_index: {
-    AddrOS << format(" indexed (%8.8x) address = ", (uint32_t)UValue);
+    Optional<SectionedAddress> A = U->getAddrOffsetSectionItem(UValue);
+    if (!A || DumpOpts.Verbose)
+      AddrOS << format("indexed (%8.8x) address = ", (uint32_t)UValue);
     if (U == nullptr)
       OS << "<invalid dwarf unit>";
-    else if (Optional<SectionedAddress> A = U->getAddrOffsetSectionItem(UValue))
-      AddrOS << format("0x%016" PRIx64, A->Address);
+    else if (A)
+      dumpSectionedAddress(AddrOS, DumpOpts, *A);
     else
       OS << "<no .debug_addr section>";
     break;
@@ -443,7 +467,7 @@ void DWARFFormValue::dump(raw_ostream &OS, DIDumpOptions DumpOpts) const {
   case DW_FORM_strx4:
   case DW_FORM_GNU_str_index:
     if (DumpOpts.Verbose)
-      OS << format(" indexed (%8.8x) string = ", (uint32_t)UValue);
+      OS << format("indexed (%8.8x) string = ", (uint32_t)UValue);
     dumpString(OS);
     break;
   case DW_FORM_GNU_strp_alt:
diff --git a/lib/DebugInfo/DWARF/DWARFGdbIndex.cpp b/lib/DebugInfo/DWARF/DWARFGdbIndex.cpp
index 1abd931e3b8b307c1d27f8c67c3e9a31c715d4bd..6b5357f23f6c2cf7099b3c5b3c38d1339e27334d 100644
--- a/lib/DebugInfo/DWARF/DWARFGdbIndex.cpp
+++ b/lib/DebugInfo/DWARF/DWARFGdbIndex.cpp
@@ -1,9 +1,8 @@
 //===- DWARFGdbIndex.cpp --------------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/DWARF/DWARFListTable.cpp b/lib/DebugInfo/DWARF/DWARFListTable.cpp
index 462c036d73ada4a188f5d4fc03d755241da435b2..32bb6d329ae5e243fc539f4143a70b4fc196fc31 100644
--- a/lib/DebugInfo/DWARF/DWARFListTable.cpp
+++ b/lib/DebugInfo/DWARF/DWARFListTable.cpp
@@ -1,9 +1,8 @@
 //===- DWARFListTable.cpp ---------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/DWARF/DWARFTypeUnit.cpp b/lib/DebugInfo/DWARF/DWARFTypeUnit.cpp
index 00be75e1a94db53c1f434ad42772218297a44ff0..844920ba5b119d9d6da3e5844884da8be6c9fd97 100644
--- a/lib/DebugInfo/DWARF/DWARFTypeUnit.cpp
+++ b/lib/DebugInfo/DWARF/DWARFTypeUnit.cpp
@@ -1,9 +1,8 @@
 //===- DWARFTypeUnit.cpp --------------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/DWARF/DWARFUnit.cpp b/lib/DebugInfo/DWARF/DWARFUnit.cpp
index 48900e4b7a28947f85d42488029d0b97ad7b8abb..78543c53acc579e82dbfe60bf082e167e1c1ceea 100644
--- a/lib/DebugInfo/DWARF/DWARFUnit.cpp
+++ b/lib/DebugInfo/DWARF/DWARFUnit.cpp
@@ -1,9 +1,8 @@
 //===- DWARFUnit.cpp ------------------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -566,42 +565,18 @@ DWARFUnit::findRnglistFromIndex(uint32_t Index) {
                              "missing or invalid range list table");
 }
 
-void DWARFUnit::collectAddressRanges(DWARFAddressRangesVector &CURanges) {
+Expected<DWARFAddressRangesVector> DWARFUnit::collectAddressRanges() {
   DWARFDie UnitDie = getUnitDIE();
   if (!UnitDie)
-    return;
+    return createStringError(errc::invalid_argument, "No unit DIE");
+
   // First, check if unit DIE describes address ranges for the whole unit.
   auto CUDIERangesOrError = UnitDie.getAddressRanges();
-  if (CUDIERangesOrError) {
-    if (!CUDIERangesOrError.get().empty()) {
-      CURanges.insert(CURanges.end(), CUDIERangesOrError.get().begin(),
-                      CUDIERangesOrError.get().end());
-      return;
-    }
-  } else
-    WithColor::error() << "decoding address ranges: "
-                       << toString(CUDIERangesOrError.takeError()) << '\n';
-
-  // This function is usually called if there in no .debug_aranges section
-  // in order to produce a compile unit level set of address ranges that
-  // is accurate. If the DIEs weren't parsed, then we don't want all dies for
-  // all compile units to stay loaded when they weren't needed. So we can end
-  // up parsing the DWARF and then throwing them all away to keep memory usage
-  // down.
-  const bool ClearDIEs = extractDIEsIfNeeded(false) > 1;
-  getUnitDIE().collectChildrenAddressRanges(CURanges);
-
-  // Collect address ranges from DIEs in .dwo if necessary.
-  bool DWOCreated = parseDWO();
-  if (DWO)
-    DWO->collectAddressRanges(CURanges);
-  if (DWOCreated)
-    DWO.reset();
-
-  // Keep memory down by clearing DIEs if this generate function
-  // caused them to be parsed.
-  if (ClearDIEs)
-    clearDIEs(true);
+  if (!CUDIERangesOrError)
+    return createStringError(errc::invalid_argument,
+                             "decoding address ranges: %s",
+                             toString(CUDIERangesOrError.takeError()).c_str());
+  return *CUDIERangesOrError;
 }
 
 void DWARFUnit::updateAddressDieMap(DWARFDie Die) {
diff --git a/lib/DebugInfo/DWARF/DWARFUnitIndex.cpp b/lib/DebugInfo/DWARF/DWARFUnitIndex.cpp
index 84b6c4b81817cad8a6d2cc227dfb361d67a34cd7..8225a26d64c870fe89a4fba772826c2dd7c0fdca 100644
--- a/lib/DebugInfo/DWARF/DWARFUnitIndex.cpp
+++ b/lib/DebugInfo/DWARF/DWARFUnitIndex.cpp
@@ -1,9 +1,8 @@
 //===- DWARFUnitIndex.cpp -------------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/DWARF/DWARFVerifier.cpp b/lib/DebugInfo/DWARF/DWARFVerifier.cpp
index f8370178b6279dc798152333b56cad687d68cf19..0eef0f6a82bf5a05fd04238c21b4f7fc7cbed106 100644
--- a/lib/DebugInfo/DWARF/DWARFVerifier.cpp
+++ b/lib/DebugInfo/DWARF/DWARFVerifier.cpp
@@ -1,9 +1,8 @@
 //===- DWARFVerifier.cpp --------------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 #include "llvm/DebugInfo/DWARF/DWARFVerifier.h"
diff --git a/lib/DebugInfo/DWARF/LLVMBuild.txt b/lib/DebugInfo/DWARF/LLVMBuild.txt
index a21276244ea75caaf44265c0aabac5b50094fe80..182be10f0b4ca3877210752ab54321fc23a120f9 100644
--- a/lib/DebugInfo/DWARF/LLVMBuild.txt
+++ b/lib/DebugInfo/DWARF/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/DebugInfo/DWARF/LLVMBuild.txt ----------------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/DebugInfo/LLVMBuild.txt b/lib/DebugInfo/LLVMBuild.txt
index 86503e13a03ad36e7b689ca285958efb4ac4b1bf..486cedaf6c2e2cff94ad7e26d420e43600ce04c0 100644
--- a/lib/DebugInfo/LLVMBuild.txt
+++ b/lib/DebugInfo/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/DebugInfo/LLVMBuild.txt ----------------------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/DebugInfo/MSF/LLVMBuild.txt b/lib/DebugInfo/MSF/LLVMBuild.txt
index 391547c949bed28449ead542dd707b1c51a9bdba..ca492315080aaa29f7a61d7a15874f69555974c1 100644
--- a/lib/DebugInfo/MSF/LLVMBuild.txt
+++ b/lib/DebugInfo/MSF/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/DebugInfo/MSF/LLVMBuild.txt -------------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/DebugInfo/MSF/MSFBuilder.cpp b/lib/DebugInfo/MSF/MSFBuilder.cpp
index 71609919558afeafeb7993e7fab7c496e404ce88..c6fe764ab7e0eadad925c8f2d60a61ac91b5a43f 100644
--- a/lib/DebugInfo/MSF/MSFBuilder.cpp
+++ b/lib/DebugInfo/MSF/MSFBuilder.cpp
@@ -1,9 +1,8 @@
 //===- MSFBuilder.cpp -----------------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/MSF/MSFCommon.cpp b/lib/DebugInfo/MSF/MSFCommon.cpp
index d398304375acd0f859ade7a8083653a313601451..fb4f0700059c4b8f45d7bb66844b5a0461c7dce1 100644
--- a/lib/DebugInfo/MSF/MSFCommon.cpp
+++ b/lib/DebugInfo/MSF/MSFCommon.cpp
@@ -1,9 +1,8 @@
 //===- MSFCommon.cpp - Common types and functions for MSF files -----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/MSF/MSFError.cpp b/lib/DebugInfo/MSF/MSFError.cpp
index bfac6bebba3faeb7ed3f74befd90363de66d2c79..b368b802c5643a82a1d26cbb7887a97643f76186 100644
--- a/lib/DebugInfo/MSF/MSFError.cpp
+++ b/lib/DebugInfo/MSF/MSFError.cpp
@@ -1,9 +1,8 @@
 //===- MSFError.cpp - Error extensions for MSF files ------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -14,6 +13,7 @@
 using namespace llvm;
 using namespace llvm::msf;
 
+namespace {
 // FIXME: This class is only here to support the transition to llvm::Error. It
 // will be removed once this transition is complete. Clients should prefer to
 // deal with the Error value directly, rather than converting to error_code.
@@ -39,6 +39,7 @@ public:
     llvm_unreachable("Unrecognized msf_error_code");
   }
 };
+} // namespace
 
 static llvm::ManagedStatic<MSFErrorCategory> MSFCategory;
 const std::error_category &llvm::msf::MSFErrCategory() { return *MSFCategory; }
diff --git a/lib/DebugInfo/MSF/MappedBlockStream.cpp b/lib/DebugInfo/MSF/MappedBlockStream.cpp
index dec28eb3069729e8459eecad6214f00f17586db4..df925771f0d94d64c7be5c73a967d4b1a67fcc53 100644
--- a/lib/DebugInfo/MSF/MappedBlockStream.cpp
+++ b/lib/DebugInfo/MSF/MappedBlockStream.cpp
@@ -1,9 +1,8 @@
 //===- MappedBlockStream.cpp - Reads stream data from an MSF file ---------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/PDB/DIA/DIADataStream.cpp b/lib/DebugInfo/PDB/DIA/DIADataStream.cpp
index 6a10513fad972bc58135b4ec1898e0cf5d9e57bd..8a806f298d0fbc31a315216e0bbaa3cf46e7d4ce 100644
--- a/lib/DebugInfo/PDB/DIA/DIADataStream.cpp
+++ b/lib/DebugInfo/PDB/DIA/DIADataStream.cpp
@@ -1,9 +1,8 @@
 //===- DIADataStream.cpp - DIA implementation of IPDBDataStream -*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/PDB/DIA/DIAEnumDebugStreams.cpp b/lib/DebugInfo/PDB/DIA/DIAEnumDebugStreams.cpp
index d2451f13e6cb2a697b5726e11dc38b7d62a851b3..e4cb4daf94b1453e9613c1be8431dbb8dd395c13 100644
--- a/lib/DebugInfo/PDB/DIA/DIAEnumDebugStreams.cpp
+++ b/lib/DebugInfo/PDB/DIA/DIAEnumDebugStreams.cpp
@@ -1,9 +1,8 @@
 //==- DIAEnumDebugStreams.cpp - DIA Debug Stream Enumerator impl -*- C++ -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/PDB/DIA/DIAEnumFrameData.cpp b/lib/DebugInfo/PDB/DIA/DIAEnumFrameData.cpp
index f873f3525df59b1cd0ff1bcdb781b3cc5a5c45c2..8a181b448a270f1de7a8f2b9b69568688907310f 100644
--- a/lib/DebugInfo/PDB/DIA/DIAEnumFrameData.cpp
+++ b/lib/DebugInfo/PDB/DIA/DIAEnumFrameData.cpp
@@ -1,9 +1,8 @@
 //==- DIAEnumFrameData.cpp ---------------------------------------*- C++ -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/PDB/DIA/DIAEnumInjectedSources.cpp b/lib/DebugInfo/PDB/DIA/DIAEnumInjectedSources.cpp
index 6c361b81e33de111fb375c1b4ff15364dab9c4ca..7226ab2ba0a043050bb8d344f7c5b6ff5aded48c 100644
--- a/lib/DebugInfo/PDB/DIA/DIAEnumInjectedSources.cpp
+++ b/lib/DebugInfo/PDB/DIA/DIAEnumInjectedSources.cpp
@@ -1,9 +1,8 @@
 //==- DIAEnumSourceFiles.cpp - DIA Source File Enumerator impl ---*- C++ -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/PDB/DIA/DIAEnumLineNumbers.cpp b/lib/DebugInfo/PDB/DIA/DIAEnumLineNumbers.cpp
index 0820d9dc7c9f6e7f37d8b37600b9abc950256760..6f1d7733fb2deddb73ac4ff5f893a00a715d8e82 100644
--- a/lib/DebugInfo/PDB/DIA/DIAEnumLineNumbers.cpp
+++ b/lib/DebugInfo/PDB/DIA/DIAEnumLineNumbers.cpp
@@ -1,9 +1,8 @@
 //==- DIAEnumLineNumbers.cpp - DIA Line Number Enumerator impl ---*- C++ -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/PDB/DIA/DIAEnumSectionContribs.cpp b/lib/DebugInfo/PDB/DIA/DIAEnumSectionContribs.cpp
index 90c857aa5713586abbdf2673d7a2ea223c8fc94c..4f9b232a024aa2ddcd2747f48205c04db26c7939 100644
--- a/lib/DebugInfo/PDB/DIA/DIAEnumSectionContribs.cpp
+++ b/lib/DebugInfo/PDB/DIA/DIAEnumSectionContribs.cpp
@@ -1,9 +1,8 @@
 //==- DIAEnumSectionContribs.cpp ---------------------------------*- C++ -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/PDB/DIA/DIAEnumSourceFiles.cpp b/lib/DebugInfo/PDB/DIA/DIAEnumSourceFiles.cpp
index 06595e7ec1c8901441e486b75944e15429a729fd..943e9e1b4d580bf4f407b0be761084f6f3d7351f 100644
--- a/lib/DebugInfo/PDB/DIA/DIAEnumSourceFiles.cpp
+++ b/lib/DebugInfo/PDB/DIA/DIAEnumSourceFiles.cpp
@@ -1,9 +1,8 @@
 //==- DIAEnumSourceFiles.cpp - DIA Source File Enumerator impl ---*- C++ -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/PDB/DIA/DIAEnumSymbols.cpp b/lib/DebugInfo/PDB/DIA/DIAEnumSymbols.cpp
index 48bc32767e6c6a29518c41920cc50b20827ef769..5153596d52ae00dae31fa4b65356f8b3651bf033 100644
--- a/lib/DebugInfo/PDB/DIA/DIAEnumSymbols.cpp
+++ b/lib/DebugInfo/PDB/DIA/DIAEnumSymbols.cpp
@@ -1,9 +1,8 @@
 //==- DIAEnumSymbols.cpp - DIA Symbol Enumerator impl ------------*- C++ -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/PDB/DIA/DIAEnumTables.cpp b/lib/DebugInfo/PDB/DIA/DIAEnumTables.cpp
index 6fa096156d4810b7aecc69239977a2afb5f676c2..335b575d654239b007298fb7b091aed3160e2a24 100644
--- a/lib/DebugInfo/PDB/DIA/DIAEnumTables.cpp
+++ b/lib/DebugInfo/PDB/DIA/DIAEnumTables.cpp
@@ -1,9 +1,8 @@
 //===- DIAEnumTables.cpp - DIA Table Enumerator Impl ------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/PDB/DIA/DIAFrameData.cpp b/lib/DebugInfo/PDB/DIA/DIAFrameData.cpp
index 533cce7923c0d8089e3ac9d6bfd34a645ab49dde..7975156b1abde66b1d6f4e8be5dc4fc8afdfed0a 100644
--- a/lib/DebugInfo/PDB/DIA/DIAFrameData.cpp
+++ b/lib/DebugInfo/PDB/DIA/DIAFrameData.cpp
@@ -1,9 +1,8 @@
 //===- DIAFrameData.cpp - DIA impl. of IPDBFrameData -------------- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/PDB/DIA/DIAInjectedSource.cpp b/lib/DebugInfo/PDB/DIA/DIAInjectedSource.cpp
index 1d642f221d79574955fc74affec0b3417375ef03..211b4e11ac46258945b18d5219586ab7edc67125 100644
--- a/lib/DebugInfo/PDB/DIA/DIAInjectedSource.cpp
+++ b/lib/DebugInfo/PDB/DIA/DIAInjectedSource.cpp
@@ -1,9 +1,8 @@
 //===- DIAInjectedSource.cpp - DIA impl for IPDBInjectedSource --*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/PDB/DIA/DIALineNumber.cpp b/lib/DebugInfo/PDB/DIA/DIALineNumber.cpp
index b19be6b595abe0933ead2b18ef80e98831054fcf..3af02ea36c7ba44d711965046d885d7266e189a1 100644
--- a/lib/DebugInfo/PDB/DIA/DIALineNumber.cpp
+++ b/lib/DebugInfo/PDB/DIA/DIALineNumber.cpp
@@ -1,9 +1,8 @@
 //===- DIALineNumber.cpp - DIA implementation of IPDBLineNumber -*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/PDB/DIA/DIARawSymbol.cpp b/lib/DebugInfo/PDB/DIA/DIARawSymbol.cpp
index cd4d00a13b18ba0acf9bf11f22961eed3de31258..a8ae076e1d6c2c0082d170294bfceafda56ab3ef 100644
--- a/lib/DebugInfo/PDB/DIA/DIARawSymbol.cpp
+++ b/lib/DebugInfo/PDB/DIA/DIARawSymbol.cpp
@@ -1,9 +1,8 @@
 //===- DIARawSymbol.cpp - DIA implementation of IPDBRawSymbol ---*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/PDB/DIA/DIASectionContrib.cpp b/lib/DebugInfo/PDB/DIA/DIASectionContrib.cpp
index 8e233ca151617db68a73a35ea5c68f2084eb37db..e2d928f2c4b2032f4fb728c4edae1844808ae7f8 100644
--- a/lib/DebugInfo/PDB/DIA/DIASectionContrib.cpp
+++ b/lib/DebugInfo/PDB/DIA/DIASectionContrib.cpp
@@ -1,9 +1,8 @@
 //===- DIASectionContrib.cpp - DIA impl. of IPDBSectionContrib ---- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/PDB/DIA/DIASession.cpp b/lib/DebugInfo/PDB/DIA/DIASession.cpp
index bd375e172ac09819c4d9fb995bfbb12b70151a30..4e0b8587c61351933ed31cd8b508e4df2733ea6b 100644
--- a/lib/DebugInfo/PDB/DIA/DIASession.cpp
+++ b/lib/DebugInfo/PDB/DIA/DIASession.cpp
@@ -1,9 +1,8 @@
 //===- DIASession.cpp - DIA implementation of IPDBSession -------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 #include "llvm/DebugInfo/PDB/DIA/DIASession.h"
diff --git a/lib/DebugInfo/PDB/DIA/DIASourceFile.cpp b/lib/DebugInfo/PDB/DIA/DIASourceFile.cpp
index d3e408166a874039364b64e5c1d76e55cee2d25e..21e757c3a0604b06405853742998893d9f86eb06 100644
--- a/lib/DebugInfo/PDB/DIA/DIASourceFile.cpp
+++ b/lib/DebugInfo/PDB/DIA/DIASourceFile.cpp
@@ -1,9 +1,8 @@
 //===- DIASourceFile.cpp - DIA implementation of IPDBSourceFile -*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/PDB/DIA/DIATable.cpp b/lib/DebugInfo/PDB/DIA/DIATable.cpp
index 6017081b2cb61c380787f3853a15b1406001cecd..33d74abd740e0fe3b4acf1ea7f50b55476637c4a 100644
--- a/lib/DebugInfo/PDB/DIA/DIATable.cpp
+++ b/lib/DebugInfo/PDB/DIA/DIATable.cpp
@@ -1,9 +1,8 @@
 //===- DIATable.cpp - DIA implementation of IPDBTable -----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/PDB/GenericError.cpp b/lib/DebugInfo/PDB/GenericError.cpp
index 256952073e88a0fc6dee89a680f4c95ff5e3a2e5..70dc094c42ecf66103ed069c4112cb1ef4af5daf 100644
--- a/lib/DebugInfo/PDB/GenericError.cpp
+++ b/lib/DebugInfo/PDB/GenericError.cpp
@@ -1,9 +1,8 @@
 //===- Error.cpp - system_error extensions for PDB --------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -14,6 +13,7 @@
 using namespace llvm;
 using namespace llvm::pdb;
 
+namespace {
 // FIXME: This class is only here to support the transition to llvm::Error. It
 // will be removed once this transition is complete. Clients should prefer to
 // deal with the Error value directly, rather than converting to error_code.
@@ -40,6 +40,7 @@ public:
     llvm_unreachable("Unrecognized generic_error_code");
   }
 };
+} // namespace
 
 static llvm::ManagedStatic<PDBErrorCategory> PDBCategory;
 const std::error_category &llvm::pdb::PDBErrCategory() { return *PDBCategory; }
diff --git a/lib/DebugInfo/PDB/IPDBSourceFile.cpp b/lib/DebugInfo/PDB/IPDBSourceFile.cpp
index 8cb1fbef51f402ff3b3dde9a84a5b7acea21f86a..113ee04bab952a89681e30cd7725b93c297d9b52 100644
--- a/lib/DebugInfo/PDB/IPDBSourceFile.cpp
+++ b/lib/DebugInfo/PDB/IPDBSourceFile.cpp
@@ -1,9 +1,8 @@
 //===- IPDBSourceFile.cpp - base interface for a PDB source file ----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/PDB/LLVMBuild.txt b/lib/DebugInfo/PDB/LLVMBuild.txt
index d9af656b51167cb19c3106c8204cd89336a363a6..4bd42a2c1590f8aa2b3b3fa6fe2e32f1689a5ea7 100644
--- a/lib/DebugInfo/PDB/LLVMBuild.txt
+++ b/lib/DebugInfo/PDB/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/DebugInfo/PDB/LLVMBuild.txt ------------------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/DebugInfo/PDB/Native/DbiModuleDescriptor.cpp b/lib/DebugInfo/PDB/Native/DbiModuleDescriptor.cpp
index 931ac7bb81db8950690463c93d3ac9d981427390..5095efcdee3cf20be8320efcc723760ff8e61ad5 100644
--- a/lib/DebugInfo/PDB/Native/DbiModuleDescriptor.cpp
+++ b/lib/DebugInfo/PDB/Native/DbiModuleDescriptor.cpp
@@ -1,9 +1,8 @@
 //===- DbiModuleDescriptor.cpp - PDB module information -------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/PDB/Native/DbiModuleDescriptorBuilder.cpp b/lib/DebugInfo/PDB/Native/DbiModuleDescriptorBuilder.cpp
index ab93efc839a9a101ff68c0595228466a64601ac0..922bea765c71bac013381e0202eacbb64ce555b1 100644
--- a/lib/DebugInfo/PDB/Native/DbiModuleDescriptorBuilder.cpp
+++ b/lib/DebugInfo/PDB/Native/DbiModuleDescriptorBuilder.cpp
@@ -1,9 +1,8 @@
 //===- DbiModuleDescriptorBuilder.cpp - PDB Mod Info Creation ---*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/PDB/Native/DbiModuleList.cpp b/lib/DebugInfo/PDB/Native/DbiModuleList.cpp
index eea70b229c67694db80e38e6f94cde3fa236d072..5cf014e881cde1fc226df108d2835ab4b01860a2 100644
--- a/lib/DebugInfo/PDB/Native/DbiModuleList.cpp
+++ b/lib/DebugInfo/PDB/Native/DbiModuleList.cpp
@@ -1,9 +1,8 @@
 //===- DbiModuleList.cpp - PDB module information list --------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/PDB/Native/DbiStream.cpp b/lib/DebugInfo/PDB/Native/DbiStream.cpp
index 60ac17b655a7bca8db32f062606672704405c910..ea15bcd01f5f6a38e2e4f40e96083e2fb7d29e3a 100644
--- a/lib/DebugInfo/PDB/Native/DbiStream.cpp
+++ b/lib/DebugInfo/PDB/Native/DbiStream.cpp
@@ -1,9 +1,8 @@
 //===- DbiStream.cpp - PDB Dbi Stream (Stream 3) Access -------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -127,8 +126,10 @@ Error DbiStream::reload(PDBFile *Pdb) {
     return EC;
   if (auto EC = initializeSectionMapData())
     return EC;
-  if (auto EC = initializeFpoRecords(Pdb))
+  if (auto EC = initializeOldFpoRecords(Pdb))
     return EC;
+  if (auto EC = initializeNewFpoRecords(Pdb))
+     return EC;
 
   if (Reader.bytesRemaining() > 0)
     return make_error<RawError>(raw_error_code::corrupt_file,
@@ -201,8 +202,16 @@ FixedStreamArray<object::coff_section> DbiStream::getSectionHeaders() const {
   return SectionHeaders;
 }
 
-FixedStreamArray<object::FpoData> DbiStream::getFpoRecords() {
-  return FpoRecords;
+bool DbiStream::hasOldFpoRecords() const { return OldFpoStream != nullptr; }
+
+FixedStreamArray<object::FpoData> DbiStream::getOldFpoRecords() const {
+  return OldFpoRecords;
+}
+
+bool DbiStream::hasNewFpoRecords() const { return NewFpoStream != nullptr; }
+
+const DebugFrameDataSubsectionRef &DbiStream::getNewFpoRecords() const {
+  return NewFpoRecords;
 }
 
 const DbiModuleList &DbiStream::modules() const { return Modules; }
@@ -247,22 +256,15 @@ Error DbiStream::initializeSectionContributionData() {
 
 // Initializes this->SectionHeaders.
 Error DbiStream::initializeSectionHeadersData(PDBFile *Pdb) {
-  if (!Pdb)
-    return Error::success();
-
-  if (DbgStreams.size() == 0)
-    return Error::success();
+  Expected<std::unique_ptr<msf::MappedBlockStream>> ExpectedStream =
+      createIndexedStreamForHeaderType(Pdb, DbgHeaderType::SectionHdr);
+  if (auto EC = ExpectedStream.takeError())
+    return EC;
 
-  uint32_t StreamNum = getDebugStreamIndex(DbgHeaderType::SectionHdr);
-  if (StreamNum == kInvalidStreamIndex)
+  auto &SHS = *ExpectedStream;
+  if (!SHS)
     return Error::success();
 
-  if (StreamNum >= Pdb->getNumStreams())
-    return make_error<RawError>(raw_error_code::no_stream);
-
-  auto SHS = MappedBlockStream::createIndexedStream(
-      Pdb->getMsfLayout(), Pdb->getMsfBuffer(), StreamNum, Pdb->getAllocator());
-
   size_t StreamLen = SHS->getLength();
   if (StreamLen % sizeof(object::coff_section))
     return make_error<RawError>(raw_error_code::corrupt_file,
@@ -279,39 +281,69 @@ Error DbiStream::initializeSectionHeadersData(PDBFile *Pdb) {
 }
 
 // Initializes this->Fpos.
-Error DbiStream::initializeFpoRecords(PDBFile *Pdb) {
-  if (!Pdb)
-    return Error::success();
-
-  if (DbgStreams.size() == 0)
-    return Error::success();
-
-  uint32_t StreamNum = getDebugStreamIndex(DbgHeaderType::NewFPO);
+Error DbiStream::initializeOldFpoRecords(PDBFile *Pdb) {
+  Expected<std::unique_ptr<msf::MappedBlockStream>> ExpectedStream =
+      createIndexedStreamForHeaderType(Pdb, DbgHeaderType::FPO);
+  if (auto EC = ExpectedStream.takeError())
+    return EC;
 
-  // This means there is no FPO data.
-  if (StreamNum == kInvalidStreamIndex)
+  auto &FS = *ExpectedStream;
+  if (!FS)
     return Error::success();
 
-  if (StreamNum >= Pdb->getNumStreams())
-    return make_error<RawError>(raw_error_code::no_stream);
-
-  auto FS = MappedBlockStream::createIndexedStream(
-      Pdb->getMsfLayout(), Pdb->getMsfBuffer(), StreamNum, Pdb->getAllocator());
-
   size_t StreamLen = FS->getLength();
   if (StreamLen % sizeof(object::FpoData))
     return make_error<RawError>(raw_error_code::corrupt_file,
-                                "Corrupted New FPO stream.");
+                                "Corrupted Old FPO stream.");
 
   size_t NumRecords = StreamLen / sizeof(object::FpoData);
   BinaryStreamReader Reader(*FS);
-  if (auto EC = Reader.readArray(FpoRecords, NumRecords))
+  if (auto EC = Reader.readArray(OldFpoRecords, NumRecords))
     return make_error<RawError>(raw_error_code::corrupt_file,
-                                "Corrupted New FPO stream.");
-  FpoStream = std::move(FS);
+                                "Corrupted Old FPO stream.");
+  OldFpoStream = std::move(FS);
+  return Error::success();
+}
+
+Error DbiStream::initializeNewFpoRecords(PDBFile *Pdb) {
+  Expected<std::unique_ptr<msf::MappedBlockStream>> ExpectedStream =
+      createIndexedStreamForHeaderType(Pdb, DbgHeaderType::NewFPO);
+  if (auto EC = ExpectedStream.takeError())
+    return EC;
+
+  auto &FS = *ExpectedStream;
+  if (!FS)
+    return Error::success();
+
+  if (auto EC = NewFpoRecords.initialize(*FS))
+    return EC;
+
+  NewFpoStream = std::move(FS);
   return Error::success();
 }
 
+Expected<std::unique_ptr<msf::MappedBlockStream>>
+DbiStream::createIndexedStreamForHeaderType(PDBFile *Pdb,
+                                            DbgHeaderType Type) const {
+  if (!Pdb)
+    return nullptr;
+
+  if (DbgStreams.empty())
+    return nullptr;
+
+  uint32_t StreamNum = getDebugStreamIndex(Type);
+
+  // This means there is no such stream
+  if (StreamNum == kInvalidStreamIndex)
+    return nullptr;
+
+  if (StreamNum >= Pdb->getNumStreams())
+    return make_error<RawError>(raw_error_code::no_stream);
+
+  return MappedBlockStream::createIndexedStream(
+      Pdb->getMsfLayout(), Pdb->getMsfBuffer(), StreamNum, Pdb->getAllocator());
+}
+
 BinarySubstreamRef DbiStream::getSectionContributionData() const {
   return SecContrSubstream;
 }
diff --git a/lib/DebugInfo/PDB/Native/DbiStreamBuilder.cpp b/lib/DebugInfo/PDB/Native/DbiStreamBuilder.cpp
index 094216ea800a0a0a51c4c3587e68c75cb022b1e7..b7ade0072ee5a884809967c0b21a1295d05fa542 100644
--- a/lib/DebugInfo/PDB/Native/DbiStreamBuilder.cpp
+++ b/lib/DebugInfo/PDB/Native/DbiStreamBuilder.cpp
@@ -1,9 +1,8 @@
 //===- DbiStreamBuilder.cpp - PDB Dbi Stream Creation -----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/PDB/Native/EnumTables.cpp b/lib/DebugInfo/PDB/Native/EnumTables.cpp
index b3837dc72e5b2b0fdd98b6d10c026795bfd42fa7..f5125393695bc139b51ccca6e4cf335538026d49 100644
--- a/lib/DebugInfo/PDB/Native/EnumTables.cpp
+++ b/lib/DebugInfo/PDB/Native/EnumTables.cpp
@@ -1,9 +1,8 @@
 //===- EnumTables.cpp - Enum to string conversion tables --------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/PDB/Native/GSIStreamBuilder.cpp b/lib/DebugInfo/PDB/Native/GSIStreamBuilder.cpp
index 57da7003da2b82cb486bb000a22fca72575ec7fa..a52b7e0c4e476c55b761c8202bfc4bade2ab9076 100644
--- a/lib/DebugInfo/PDB/Native/GSIStreamBuilder.cpp
+++ b/lib/DebugInfo/PDB/Native/GSIStreamBuilder.cpp
@@ -1,9 +1,8 @@
 //===- DbiStreamBuilder.cpp - PDB Dbi Stream Creation -----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/PDB/Native/GlobalsStream.cpp b/lib/DebugInfo/PDB/Native/GlobalsStream.cpp
index e363195668219bd01c83f3c9f62ff1ca98d94920..f27d60f468156420a853aab1a04002f0d34b03bd 100644
--- a/lib/DebugInfo/PDB/Native/GlobalsStream.cpp
+++ b/lib/DebugInfo/PDB/Native/GlobalsStream.cpp
@@ -1,9 +1,8 @@
 //===- GlobalsStream.cpp - PDB Index of Symbols by Name ---------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/DebugInfo/PDB/Native/Hash.cpp b/lib/DebugInfo/PDB/Native/Hash.cpp
index 61188ece2dcb8598daf4e2e7a2cbe8c76681f44c..b5c139ecbec0798e13180bed5d5bf9b5babb8d93 100644
--- a/lib/DebugInfo/PDB/Native/Hash.cpp
+++ b/lib/DebugInfo/PDB/Native/Hash.cpp
@@ -1,9 +1,8 @@
 //===- Hash.cpp - PDB Hash Functions --------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/PDB/Native/HashTable.cpp b/lib/DebugInfo/PDB/Native/HashTable.cpp
index cfabc9cd1ad82b149bf6835f495d57ef7b8b0bb9..dfdcdf1f4eafe2a66af8720c2d12b0fe2452e83a 100644
--- a/lib/DebugInfo/PDB/Native/HashTable.cpp
+++ b/lib/DebugInfo/PDB/Native/HashTable.cpp
@@ -1,9 +1,8 @@
 //===- HashTable.cpp - PDB Hash Table -------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/PDB/Native/InfoStream.cpp b/lib/DebugInfo/PDB/Native/InfoStream.cpp
index 973a520ffca95dc613019a968c2186639d569a7c..f41bb32d69af8c480af3232097c57d8d28519c58 100644
--- a/lib/DebugInfo/PDB/Native/InfoStream.cpp
+++ b/lib/DebugInfo/PDB/Native/InfoStream.cpp
@@ -1,9 +1,8 @@
 //===- InfoStream.cpp - PDB Info Stream (Stream 1) Access -------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/PDB/Native/InfoStreamBuilder.cpp b/lib/DebugInfo/PDB/Native/InfoStreamBuilder.cpp
index 3b5a2accdba6506ace55cc906db49516586e1640..42daa7cae7997c9667e43410e898c26d9c38e525 100644
--- a/lib/DebugInfo/PDB/Native/InfoStreamBuilder.cpp
+++ b/lib/DebugInfo/PDB/Native/InfoStreamBuilder.cpp
@@ -1,9 +1,8 @@
 //===- InfoStreamBuilder.cpp - PDB Info Stream Creation ---------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/PDB/Native/ModuleDebugStream.cpp b/lib/DebugInfo/PDB/Native/ModuleDebugStream.cpp
index 8c97f4a012f03d61fde19ae78fce6deab7006946..649ff7c56adcdd042c0e6463e7d62e47d3ce3330 100644
--- a/lib/DebugInfo/PDB/Native/ModuleDebugStream.cpp
+++ b/lib/DebugInfo/PDB/Native/ModuleDebugStream.cpp
@@ -1,9 +1,8 @@
 //===- ModuleDebugStream.cpp - PDB Module Info Stream Access --------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/PDB/Native/NamedStreamMap.cpp b/lib/DebugInfo/PDB/Native/NamedStreamMap.cpp
index a4eaed90837d3be89f139c6a7fcc814587bf7260..a9965ff8d0663cf2bb2e837a10f84f70c7a0cda5 100644
--- a/lib/DebugInfo/PDB/Native/NamedStreamMap.cpp
+++ b/lib/DebugInfo/PDB/Native/NamedStreamMap.cpp
@@ -1,9 +1,8 @@
 //===- NamedStreamMap.cpp - PDB Named Stream Map --------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/PDB/Native/NativeCompilandSymbol.cpp b/lib/DebugInfo/PDB/Native/NativeCompilandSymbol.cpp
index efa70b0e7bd8b18c3816ce77819275f716be1a42..39ae84acba20218f137e9b9e1680ec437da8a281 100644
--- a/lib/DebugInfo/PDB/Native/NativeCompilandSymbol.cpp
+++ b/lib/DebugInfo/PDB/Native/NativeCompilandSymbol.cpp
@@ -1,9 +1,8 @@
 //===- NativeCompilandSymbol.cpp - Native impl for compilands ---*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/PDB/Native/NativeEnumGlobals.cpp b/lib/DebugInfo/PDB/Native/NativeEnumGlobals.cpp
index 6eece3df2db3470e56b0757aebe70764d0d6b75c..54646867bc5ff28ed994d8e268128098037f003f 100644
--- a/lib/DebugInfo/PDB/Native/NativeEnumGlobals.cpp
+++ b/lib/DebugInfo/PDB/Native/NativeEnumGlobals.cpp
@@ -1,9 +1,8 @@
 //==- NativeEnumGlobals.cpp - Native Global Enumerator impl ------*- C++ -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/PDB/Native/NativeEnumModules.cpp b/lib/DebugInfo/PDB/Native/NativeEnumModules.cpp
index 6e4d56443a07280c58efda27563a56fc7c23270a..c6621924b5160b5403052eb5d56304bb88dfd310 100644
--- a/lib/DebugInfo/PDB/Native/NativeEnumModules.cpp
+++ b/lib/DebugInfo/PDB/Native/NativeEnumModules.cpp
@@ -1,9 +1,8 @@
 //==- NativeEnumModules.cpp - Native Symbol Enumerator impl ------*- C++ -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/PDB/Native/NativeEnumTypes.cpp b/lib/DebugInfo/PDB/Native/NativeEnumTypes.cpp
index 288a9128147abebc080bdb251892ff4b462f3ad8..ac217df1ee48c675159e2ff95f67370029694534 100644
--- a/lib/DebugInfo/PDB/Native/NativeEnumTypes.cpp
+++ b/lib/DebugInfo/PDB/Native/NativeEnumTypes.cpp
@@ -1,9 +1,8 @@
 //==- NativeEnumTypes.cpp - Native Type Enumerator impl ----------*- C++ -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/PDB/Native/NativeExeSymbol.cpp b/lib/DebugInfo/PDB/Native/NativeExeSymbol.cpp
index 6dde5d08a500bb628b915699b0d2c51590c48257..3f393409129b105608a4db430dd9a98ebc4e37e2 100644
--- a/lib/DebugInfo/PDB/Native/NativeExeSymbol.cpp
+++ b/lib/DebugInfo/PDB/Native/NativeExeSymbol.cpp
@@ -1,9 +1,8 @@
 //===- NativeExeSymbol.cpp - native impl for PDBSymbolExe -------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/PDB/Native/NativeRawSymbol.cpp b/lib/DebugInfo/PDB/Native/NativeRawSymbol.cpp
index 62950cb3e52a659faaa18dd24229fd0814a77196..8e43cf24495a1ae33fb9f0baf290a0fd65650c4d 100644
--- a/lib/DebugInfo/PDB/Native/NativeRawSymbol.cpp
+++ b/lib/DebugInfo/PDB/Native/NativeRawSymbol.cpp
@@ -1,9 +1,8 @@
 //===- NativeRawSymbol.cpp - Native implementation of IPDBRawSymbol -------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/PDB/Native/NativeSession.cpp b/lib/DebugInfo/PDB/Native/NativeSession.cpp
index 7807e312365c9acb0d76f6a1fc6b9859b76d109c..5fb2ea3fec5db2859b17f9aedac7db232e6e25fc 100644
--- a/lib/DebugInfo/PDB/Native/NativeSession.cpp
+++ b/lib/DebugInfo/PDB/Native/NativeSession.cpp
@@ -1,9 +1,8 @@
 //===- NativeSession.cpp - Native implementation of IPDBSession -*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/PDB/Native/NativeSymbolEnumerator.cpp b/lib/DebugInfo/PDB/Native/NativeSymbolEnumerator.cpp
index 6ebb8cae3a6563f6468c1871cd90b6bc910205f0..704c1254afbfda8dfcc0976c5b28a2e7d6d54adc 100644
--- a/lib/DebugInfo/PDB/Native/NativeSymbolEnumerator.cpp
+++ b/lib/DebugInfo/PDB/Native/NativeSymbolEnumerator.cpp
@@ -1,9 +1,8 @@
 //===- NativeSymbolEnumerator.cpp - info about enumerators ------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/PDB/Native/NativeTypeArray.cpp b/lib/DebugInfo/PDB/Native/NativeTypeArray.cpp
index a52561728a98d82a709a14941d4266b645c4b7c5..80d455ad66e9585964a53c0b3ae115bf5be52652 100644
--- a/lib/DebugInfo/PDB/Native/NativeTypeArray.cpp
+++ b/lib/DebugInfo/PDB/Native/NativeTypeArray.cpp
@@ -1,9 +1,8 @@
 //===- NativeTypeArray.cpp - info about arrays ------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/PDB/Native/NativeTypeBuiltin.cpp b/lib/DebugInfo/PDB/Native/NativeTypeBuiltin.cpp
index 7b0f13f3c07501084c22149ce82c93125ceeb118..a08663aa91bad4ba3618bd85479e609b50bb1538 100644
--- a/lib/DebugInfo/PDB/Native/NativeTypeBuiltin.cpp
+++ b/lib/DebugInfo/PDB/Native/NativeTypeBuiltin.cpp
@@ -1,9 +1,8 @@
 //===- NativeTypeBuiltin.cpp -------------------------------------- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/PDB/Native/NativeTypeEnum.cpp b/lib/DebugInfo/PDB/Native/NativeTypeEnum.cpp
index 37176fe083b97c7c137f473919e8550deb22a06e..9f5e86281a23a028f48db0561a57de4d00fce552 100644
--- a/lib/DebugInfo/PDB/Native/NativeTypeEnum.cpp
+++ b/lib/DebugInfo/PDB/Native/NativeTypeEnum.cpp
@@ -1,9 +1,8 @@
 //===- NativeTypeEnum.cpp - info about enum type ----------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/PDB/Native/NativeTypeFunctionSig.cpp b/lib/DebugInfo/PDB/Native/NativeTypeFunctionSig.cpp
index a9590fffdb87ee3c20f5d0bf62bdcf36ae611b7d..405303469c18c4e260ea6a5c89cfdc835dc6a763 100644
--- a/lib/DebugInfo/PDB/Native/NativeTypeFunctionSig.cpp
+++ b/lib/DebugInfo/PDB/Native/NativeTypeFunctionSig.cpp
@@ -1,9 +1,8 @@
 //===- NativeTypeFunctionSig.cpp - info about function signature -*- C++-*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/PDB/Native/NativeTypePointer.cpp b/lib/DebugInfo/PDB/Native/NativeTypePointer.cpp
index bd8ecb6c4007b5178daca0fa377b80f8285cdbe6..32dcfc235954146e51e3d23ab1eae981898e3ab2 100644
--- a/lib/DebugInfo/PDB/Native/NativeTypePointer.cpp
+++ b/lib/DebugInfo/PDB/Native/NativeTypePointer.cpp
@@ -1,9 +1,8 @@
 //===- NativeTypePointer.cpp - info about pointer type ----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/PDB/Native/NativeTypeUDT.cpp b/lib/DebugInfo/PDB/Native/NativeTypeUDT.cpp
index 3abf91dcc6a38394d853cc49a816a98941d951ee..be67846c0b24678881a1b0d6561923ca2d3eab65 100644
--- a/lib/DebugInfo/PDB/Native/NativeTypeUDT.cpp
+++ b/lib/DebugInfo/PDB/Native/NativeTypeUDT.cpp
@@ -1,9 +1,8 @@
 //===- NativeTypeUDT.cpp - info about class/struct type ---------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/PDB/Native/PDBFile.cpp b/lib/DebugInfo/PDB/Native/PDBFile.cpp
index a1f8786ff12fbf2f1be94e7a1cdf28f66ed0ccfd..96fc509d6909cf17e94640023bb57d4748686ba4 100644
--- a/lib/DebugInfo/PDB/Native/PDBFile.cpp
+++ b/lib/DebugInfo/PDB/Native/PDBFile.cpp
@@ -1,9 +1,8 @@
 //===- PDBFile.cpp - Low level interface to a PDB file ----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/PDB/Native/PDBFileBuilder.cpp b/lib/DebugInfo/PDB/Native/PDBFileBuilder.cpp
index e0ceb7499ee53d8fac8cd87d27c5065fd804705b..84eb4fbbfa631f4d1593d42646dd32e8456fe534 100644
--- a/lib/DebugInfo/PDB/Native/PDBFileBuilder.cpp
+++ b/lib/DebugInfo/PDB/Native/PDBFileBuilder.cpp
@@ -1,9 +1,8 @@
 //===- PDBFileBuilder.cpp - PDB File Creation -------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/PDB/Native/PDBStringTable.cpp b/lib/DebugInfo/PDB/Native/PDBStringTable.cpp
index afeea32043dd652c9d8e3457dad3a77d4a71d77d..2be1656e06bbda2a7dab7acbe0418bdaa7c69d44 100644
--- a/lib/DebugInfo/PDB/Native/PDBStringTable.cpp
+++ b/lib/DebugInfo/PDB/Native/PDBStringTable.cpp
@@ -1,9 +1,8 @@
 //===- PDBStringTable.cpp - PDB String Table ---------------------*- C++-*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/PDB/Native/PDBStringTableBuilder.cpp b/lib/DebugInfo/PDB/Native/PDBStringTableBuilder.cpp
index d9dcabf3d958588e152df3bc71079752cac75598..b2ae1188bb170b944e5f42543fbe93df57904ef3 100644
--- a/lib/DebugInfo/PDB/Native/PDBStringTableBuilder.cpp
+++ b/lib/DebugInfo/PDB/Native/PDBStringTableBuilder.cpp
@@ -1,9 +1,8 @@
 //===- PDBStringTableBuilder.cpp - PDB String Table -------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/PDB/Native/PublicsStream.cpp b/lib/DebugInfo/PDB/Native/PublicsStream.cpp
index f6466eb80464d3f0f8c8f51ebdbd5f4166db72a3..a33bf03bf8fb312e116a5864b81f9b09fc80790b 100644
--- a/lib/DebugInfo/PDB/Native/PublicsStream.cpp
+++ b/lib/DebugInfo/PDB/Native/PublicsStream.cpp
@@ -1,9 +1,8 @@
 //===- PublicsStream.cpp - PDB Public Symbol Stream -----------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/DebugInfo/PDB/Native/RawError.cpp b/lib/DebugInfo/PDB/Native/RawError.cpp
index dec9797088f23e3ba5ab9fb41865ba42c40e0392..ed6cf08396755a32da98777d664e81e819fd5514 100644
--- a/lib/DebugInfo/PDB/Native/RawError.cpp
+++ b/lib/DebugInfo/PDB/Native/RawError.cpp
@@ -5,6 +5,7 @@
 using namespace llvm;
 using namespace llvm::pdb;
 
+namespace {
 // FIXME: This class is only here to support the transition to llvm::Error. It
 // will be removed once this transition is complete. Clients should prefer to
 // deal with the Error value directly, rather than converting to error_code.
@@ -44,6 +45,7 @@ public:
     llvm_unreachable("Unrecognized raw_error_code");
   }
 };
+} // namespace
 
 static llvm::ManagedStatic<RawErrorCategory> RawCategory;
 const std::error_category &llvm::pdb::RawErrCategory() { return *RawCategory; }
diff --git a/lib/DebugInfo/PDB/Native/SymbolStream.cpp b/lib/DebugInfo/PDB/Native/SymbolStream.cpp
index 2d8d04ceca4d59d2f6d83e723ee99287787d7ab8..003840b6e67e8b60a46607a486288fc7d7100661 100644
--- a/lib/DebugInfo/PDB/Native/SymbolStream.cpp
+++ b/lib/DebugInfo/PDB/Native/SymbolStream.cpp
@@ -1,9 +1,8 @@
 //===- SymbolStream.cpp - PDB Symbol Stream Access ------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/PDB/Native/TpiHashing.cpp b/lib/DebugInfo/PDB/Native/TpiHashing.cpp
index 18708826ffc786e47449a6829d5838c738b00037..b21b82bf76fd3848da468890dee38018f9d74dbc 100644
--- a/lib/DebugInfo/PDB/Native/TpiHashing.cpp
+++ b/lib/DebugInfo/PDB/Native/TpiHashing.cpp
@@ -1,9 +1,8 @@
 //===- TpiHashing.cpp -----------------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/PDB/Native/TpiStream.cpp b/lib/DebugInfo/PDB/Native/TpiStream.cpp
index f234d446e6a099246e159fb0590c007aaffe3ea2..25dc593c9f6593644adfd3751436c5e60d2c6282 100644
--- a/lib/DebugInfo/PDB/Native/TpiStream.cpp
+++ b/lib/DebugInfo/PDB/Native/TpiStream.cpp
@@ -1,9 +1,8 @@
 //===- TpiStream.cpp - PDB Type Info (TPI) Stream 2 Access ----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/PDB/Native/TpiStreamBuilder.cpp b/lib/DebugInfo/PDB/Native/TpiStreamBuilder.cpp
index 8dd30018028e7a29271fbb407aabee0ed3149f91..6b308453c2dede8156a20271d470929e52dc946f 100644
--- a/lib/DebugInfo/PDB/Native/TpiStreamBuilder.cpp
+++ b/lib/DebugInfo/PDB/Native/TpiStreamBuilder.cpp
@@ -1,9 +1,8 @@
 //===- TpiStreamBuilder.cpp -   -------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -77,7 +76,7 @@ Error TpiStreamBuilder::finalize() {
   H->HashStreamIndex = HashStreamIndex;
   H->HashAuxStreamIndex = kInvalidStreamIndex;
   H->HashKeySize = sizeof(ulittle32_t);
-  H->NumHashBuckets = MinTpiHashBuckets;
+  H->NumHashBuckets = MaxTpiHashBuckets - 1;
 
   // Recall that hash values go into a completely different stream identified by
   // the `HashStreamIndex` field of the `TpiStreamHeader`.  Therefore, the data
@@ -130,7 +129,7 @@ Error TpiStreamBuilder::finalizeMsfLayout() {
     ulittle32_t *H = Allocator.Allocate<ulittle32_t>(TypeHashes.size());
     MutableArrayRef<ulittle32_t> HashBuffer(H, TypeHashes.size());
     for (uint32_t I = 0; I < TypeHashes.size(); ++I) {
-      HashBuffer[I] = TypeHashes[I] % MinTpiHashBuckets;
+      HashBuffer[I] = TypeHashes[I] % (MaxTpiHashBuckets - 1);
     }
     ArrayRef<uint8_t> Bytes(
         reinterpret_cast<const uint8_t *>(HashBuffer.data()),
@@ -153,9 +152,12 @@ Error TpiStreamBuilder::commit(const msf::MSFLayout &Layout,
   if (auto EC = Writer.writeObject(*Header))
     return EC;
 
-  for (auto Rec : TypeRecords)
+  for (auto Rec : TypeRecords) {
+    assert(!Rec.empty()); // An empty record will not write anything, but it
+                          // would shift all offsets from here on.
     if (auto EC = Writer.writeBytes(Rec))
       return EC;
+  }
 
   if (HashStreamIndex != kInvalidStreamIndex) {
     auto HVS = WritableMappedBlockStream::createIndexedStream(
diff --git a/lib/DebugInfo/PDB/PDB.cpp b/lib/DebugInfo/PDB/PDB.cpp
index fc1ad8bcd7cd0a764a3b6ae45723d8ce8e6410f1..e7b968cb7beace84408566de16448269e09ce744 100644
--- a/lib/DebugInfo/PDB/PDB.cpp
+++ b/lib/DebugInfo/PDB/PDB.cpp
@@ -1,9 +1,8 @@
 //===- PDB.cpp - base header file for creating a PDB reader ---------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/PDB/PDBContext.cpp b/lib/DebugInfo/PDB/PDBContext.cpp
index df0feac2bc40a78ba7bb7e96ba4f6d9df8d327ef..6d030d4aedab3c4871b32c4312c80abdb597d871 100644
--- a/lib/DebugInfo/PDB/PDBContext.cpp
+++ b/lib/DebugInfo/PDB/PDBContext.cpp
@@ -1,9 +1,8 @@
 //===-- PDBContext.cpp ------------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===/
 
diff --git a/lib/DebugInfo/PDB/PDBExtras.cpp b/lib/DebugInfo/PDB/PDBExtras.cpp
index 0d8af232cd92dea65b2279d31e9b8a62ea62e365..c67ad0b3b9866b9cb262c74b4947ac765d9bdd9e 100644
--- a/lib/DebugInfo/PDB/PDBExtras.cpp
+++ b/lib/DebugInfo/PDB/PDBExtras.cpp
@@ -1,9 +1,8 @@
 //===- PDBExtras.cpp - helper functions and classes for PDBs --------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/PDB/PDBInterfaceAnchors.cpp b/lib/DebugInfo/PDB/PDBInterfaceAnchors.cpp
index 951909295d13e36102037193da3231905768224c..8eb3311b09e3518463238fad7fbcef5d58e7cb03 100644
--- a/lib/DebugInfo/PDB/PDBInterfaceAnchors.cpp
+++ b/lib/DebugInfo/PDB/PDBInterfaceAnchors.cpp
@@ -1,9 +1,8 @@
 //===- PDBInterfaceAnchors.h - defines class anchor funcions ----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 // Class anchors are necessary per the LLVM Coding style guide, to ensure that
diff --git a/lib/DebugInfo/PDB/PDBSymDumper.cpp b/lib/DebugInfo/PDB/PDBSymDumper.cpp
index 2f819312e54e665f878ee81b39da7ca47c31c497..0956a32f4a4976149ef3b648c37f07b4b47766eb 100644
--- a/lib/DebugInfo/PDB/PDBSymDumper.cpp
+++ b/lib/DebugInfo/PDB/PDBSymDumper.cpp
@@ -1,9 +1,8 @@
 //===- PDBSymDumper.cpp - ---------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/PDB/PDBSymbol.cpp b/lib/DebugInfo/PDB/PDBSymbol.cpp
index d492edafdafe55959b9173fdc742ac8633d87fc3..34c8ac41d45b4aac78f80ab9da91c662084505bf 100644
--- a/lib/DebugInfo/PDB/PDBSymbol.cpp
+++ b/lib/DebugInfo/PDB/PDBSymbol.cpp
@@ -1,9 +1,8 @@
 //===- PDBSymbol.cpp - base class for user-facing symbol types --*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/PDB/PDBSymbolAnnotation.cpp b/lib/DebugInfo/PDB/PDBSymbolAnnotation.cpp
index cb1a9bee80243488346612a1dabe2ca330463ef5..0fa83efb7ae0e7d11c6f4333a117a6c14f133c90 100644
--- a/lib/DebugInfo/PDB/PDBSymbolAnnotation.cpp
+++ b/lib/DebugInfo/PDB/PDBSymbolAnnotation.cpp
@@ -1,9 +1,8 @@
 //===- PDBSymbolAnnotation.cpp - --------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/PDB/PDBSymbolBlock.cpp b/lib/DebugInfo/PDB/PDBSymbolBlock.cpp
index 13eec9734d02dfcb52ed043109c89284d84b9ca9..9452282a8817b59a2514855a86746dfbc5710fd4 100644
--- a/lib/DebugInfo/PDB/PDBSymbolBlock.cpp
+++ b/lib/DebugInfo/PDB/PDBSymbolBlock.cpp
@@ -1,9 +1,8 @@
 //===- PDBSymbolBlock.cpp - -------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/PDB/PDBSymbolCompiland.cpp b/lib/DebugInfo/PDB/PDBSymbolCompiland.cpp
index bbc5e6dd2a171dad2152e403a67a1c48f6d91ad1..f7819b8e5d07638908c61f331747d76d747fd7e7 100644
--- a/lib/DebugInfo/PDB/PDBSymbolCompiland.cpp
+++ b/lib/DebugInfo/PDB/PDBSymbolCompiland.cpp
@@ -1,9 +1,8 @@
 //===- PDBSymbolCompiland.cpp - compiland details ---------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/PDB/PDBSymbolCompilandDetails.cpp b/lib/DebugInfo/PDB/PDBSymbolCompilandDetails.cpp
index bdd8535a3ef3eb8263005cdeb1dd3db899ceb9f4..0d86dfe1e632a0030298d60d7494ad8c44574fe3 100644
--- a/lib/DebugInfo/PDB/PDBSymbolCompilandDetails.cpp
+++ b/lib/DebugInfo/PDB/PDBSymbolCompilandDetails.cpp
@@ -1,9 +1,8 @@
 //===- PDBSymbolCompilandDetails.cpp - compiland details --------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/PDB/PDBSymbolCompilandEnv.cpp b/lib/DebugInfo/PDB/PDBSymbolCompilandEnv.cpp
index f88df2df6be482a98475dd62b95c69b901014c0f..61f119405fd954cc55aba577b68e4803f3ef1751 100644
--- a/lib/DebugInfo/PDB/PDBSymbolCompilandEnv.cpp
+++ b/lib/DebugInfo/PDB/PDBSymbolCompilandEnv.cpp
@@ -1,9 +1,8 @@
 //===- PDBSymbolCompilandEnv.cpp - compiland env variables ------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/PDB/PDBSymbolCustom.cpp b/lib/DebugInfo/PDB/PDBSymbolCustom.cpp
index 10a21806adb6535c102a7e7c17aa8c891c718f37..6c9a4aa76c3d7b9dd4e7ce63f304385394fcaf09 100644
--- a/lib/DebugInfo/PDB/PDBSymbolCustom.cpp
+++ b/lib/DebugInfo/PDB/PDBSymbolCustom.cpp
@@ -1,9 +1,8 @@
 //===- PDBSymbolCustom.cpp - compiler-specific types ------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/PDB/PDBSymbolData.cpp b/lib/DebugInfo/PDB/PDBSymbolData.cpp
index 7de94670bcb3f774acc2047d356de29e9b7ead1d..d2b82111ccd587af1b6ff8ad1140d6a591cd62f1 100644
--- a/lib/DebugInfo/PDB/PDBSymbolData.cpp
+++ b/lib/DebugInfo/PDB/PDBSymbolData.cpp
@@ -1,9 +1,8 @@
 //===- PDBSymbolData.cpp - PDB data (e.g. variable) accessors ---*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/PDB/PDBSymbolExe.cpp b/lib/DebugInfo/PDB/PDBSymbolExe.cpp
index eb409412af59c2dd1baa0e386203b31f856dac24..c85756c43e4792f03d74a093c1a5cac1f43cde46 100644
--- a/lib/DebugInfo/PDB/PDBSymbolExe.cpp
+++ b/lib/DebugInfo/PDB/PDBSymbolExe.cpp
@@ -1,9 +1,8 @@
 //===- PDBSymbolExe.cpp - ---------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/PDB/PDBSymbolFunc.cpp b/lib/DebugInfo/PDB/PDBSymbolFunc.cpp
index 75063cb3e7f8a2becfa8f3675f21e932dae56151..7c3ba981fd6b6dd9c759f08c926d5db4ed959174 100644
--- a/lib/DebugInfo/PDB/PDBSymbolFunc.cpp
+++ b/lib/DebugInfo/PDB/PDBSymbolFunc.cpp
@@ -1,9 +1,8 @@
 //===- PDBSymbolFunc.cpp - --------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/PDB/PDBSymbolFuncDebugEnd.cpp b/lib/DebugInfo/PDB/PDBSymbolFuncDebugEnd.cpp
index af8aafa7be96f4413e6009469c019ed347600c37..66433dc17b49c3714a06dec34f87674aa18e3cb1 100644
--- a/lib/DebugInfo/PDB/PDBSymbolFuncDebugEnd.cpp
+++ b/lib/DebugInfo/PDB/PDBSymbolFuncDebugEnd.cpp
@@ -1,9 +1,8 @@
 //===- PDBSymbolFuncDebugEnd.cpp - ------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/PDB/PDBSymbolFuncDebugStart.cpp b/lib/DebugInfo/PDB/PDBSymbolFuncDebugStart.cpp
index 77b510873beac720e7ab474cbb05bbbebbf43a42..fe32c93c01211e41201d0713188fecee20976da7 100644
--- a/lib/DebugInfo/PDB/PDBSymbolFuncDebugStart.cpp
+++ b/lib/DebugInfo/PDB/PDBSymbolFuncDebugStart.cpp
@@ -1,9 +1,8 @@
 //===- PDBSymbolFuncDebugStart.cpp - ----------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/PDB/PDBSymbolLabel.cpp b/lib/DebugInfo/PDB/PDBSymbolLabel.cpp
index c802b97925e612774cc6385e085ec915dee0ae45..1fffe69a0c830dfc4828fb408c92a0d4a060179f 100644
--- a/lib/DebugInfo/PDB/PDBSymbolLabel.cpp
+++ b/lib/DebugInfo/PDB/PDBSymbolLabel.cpp
@@ -1,9 +1,8 @@
 //===- PDBSymbolLabel.cpp - -------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/PDB/PDBSymbolPublicSymbol.cpp b/lib/DebugInfo/PDB/PDBSymbolPublicSymbol.cpp
index a2dd2ab92dd9da62c88d8a6526cc588fc6dc39d6..08697683f6415efd7826379dae6c1998c0089ca6 100644
--- a/lib/DebugInfo/PDB/PDBSymbolPublicSymbol.cpp
+++ b/lib/DebugInfo/PDB/PDBSymbolPublicSymbol.cpp
@@ -1,9 +1,8 @@
 //===- PDBSymbolPublicSymbol.cpp - ------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/PDB/PDBSymbolThunk.cpp b/lib/DebugInfo/PDB/PDBSymbolThunk.cpp
index d227e3a7a60cbd5f92a2be147a88024d6c552830..6483858183e5353a9d807643e9e15036ccd0c588 100644
--- a/lib/DebugInfo/PDB/PDBSymbolThunk.cpp
+++ b/lib/DebugInfo/PDB/PDBSymbolThunk.cpp
@@ -1,9 +1,8 @@
 //===- PDBSymbolThunk.cpp - -------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/PDB/PDBSymbolTypeArray.cpp b/lib/DebugInfo/PDB/PDBSymbolTypeArray.cpp
index a2064d1ac1ebb79049eba171db0774549c3cfc34..a0d521abe43f36eead96db7ac4005143ab88af53 100644
--- a/lib/DebugInfo/PDB/PDBSymbolTypeArray.cpp
+++ b/lib/DebugInfo/PDB/PDBSymbolTypeArray.cpp
@@ -1,9 +1,8 @@
 //===- PDBSymbolTypeArray.cpp - ---------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/PDB/PDBSymbolTypeBaseClass.cpp b/lib/DebugInfo/PDB/PDBSymbolTypeBaseClass.cpp
index f0376c05557f4b12cd68f34917bf4d4f01c935d7..08467059b5e14d17b17b3c4461297e814bbc9a54 100644
--- a/lib/DebugInfo/PDB/PDBSymbolTypeBaseClass.cpp
+++ b/lib/DebugInfo/PDB/PDBSymbolTypeBaseClass.cpp
@@ -1,9 +1,8 @@
 //===- PDBSymbolTypeBaseClass.cpp - -----------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/PDB/PDBSymbolTypeBuiltin.cpp b/lib/DebugInfo/PDB/PDBSymbolTypeBuiltin.cpp
index a9f59e5f9d4d9e1bd3a1bc7bb4a88a3c68995c17..a0dd9ef601c0b78fff3eef40af840e4eaf9cdc6c 100644
--- a/lib/DebugInfo/PDB/PDBSymbolTypeBuiltin.cpp
+++ b/lib/DebugInfo/PDB/PDBSymbolTypeBuiltin.cpp
@@ -1,9 +1,8 @@
 //===- PDBSymbolTypeBuiltin.cpp - ------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/PDB/PDBSymbolTypeCustom.cpp b/lib/DebugInfo/PDB/PDBSymbolTypeCustom.cpp
index cfb347fbac55b0d03373879fb0b7976a2e78a449..6723894c90ea80f4293520bd9ceae3444899b209 100644
--- a/lib/DebugInfo/PDB/PDBSymbolTypeCustom.cpp
+++ b/lib/DebugInfo/PDB/PDBSymbolTypeCustom.cpp
@@ -1,9 +1,8 @@
 //===- PDBSymbolTypeCustom.cpp - --------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/PDB/PDBSymbolTypeDimension.cpp b/lib/DebugInfo/PDB/PDBSymbolTypeDimension.cpp
index 4eb48997635ae560670464d8fd2fb7d391276203..4a25a391f278e9905f0c390e0560e2e5ec5c0d0c 100644
--- a/lib/DebugInfo/PDB/PDBSymbolTypeDimension.cpp
+++ b/lib/DebugInfo/PDB/PDBSymbolTypeDimension.cpp
@@ -1,10 +1,9 @@
 //===- PDBSymbolTypeDimension.cpp - --------------------------------*- C++
 //-*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/PDB/PDBSymbolTypeEnum.cpp b/lib/DebugInfo/PDB/PDBSymbolTypeEnum.cpp
index 2e88d9eb284a77e8ad995a929e3fe9c98632b34a..b9fdf6aec81143dc0324288020e3dad813b810a5 100644
--- a/lib/DebugInfo/PDB/PDBSymbolTypeEnum.cpp
+++ b/lib/DebugInfo/PDB/PDBSymbolTypeEnum.cpp
@@ -1,9 +1,8 @@
 //===- PDBSymbolTypeEnum.cpp - --------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/PDB/PDBSymbolTypeFriend.cpp b/lib/DebugInfo/PDB/PDBSymbolTypeFriend.cpp
index 00d2d51aa8a718078c02b81854a75877fb2b4f8b..4ffea42cbb0a7596894d68848e4a964b3ba48844 100644
--- a/lib/DebugInfo/PDB/PDBSymbolTypeFriend.cpp
+++ b/lib/DebugInfo/PDB/PDBSymbolTypeFriend.cpp
@@ -1,9 +1,8 @@
 //===- PDBSymbolTypeFriend.cpp - --------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/PDB/PDBSymbolTypeFunctionArg.cpp b/lib/DebugInfo/PDB/PDBSymbolTypeFunctionArg.cpp
index 0399e110d5926f141ffb4893ccf9f77e423f4ad9..683e93548fb148525df09622213f0d7775ca8d52 100644
--- a/lib/DebugInfo/PDB/PDBSymbolTypeFunctionArg.cpp
+++ b/lib/DebugInfo/PDB/PDBSymbolTypeFunctionArg.cpp
@@ -1,9 +1,8 @@
 //===- PDBSymbolTypeFunctionArg.cpp - --------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/PDB/PDBSymbolTypeFunctionSig.cpp b/lib/DebugInfo/PDB/PDBSymbolTypeFunctionSig.cpp
index c0564d3941dd2911b64898824c502573f7fccc07..292320a6fe6d20d0abc4a50214e164a583d8ac8c 100644
--- a/lib/DebugInfo/PDB/PDBSymbolTypeFunctionSig.cpp
+++ b/lib/DebugInfo/PDB/PDBSymbolTypeFunctionSig.cpp
@@ -1,9 +1,8 @@
 //===- PDBSymbolTypeFunctionSig.cpp - --------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/PDB/PDBSymbolTypeManaged.cpp b/lib/DebugInfo/PDB/PDBSymbolTypeManaged.cpp
index 1faaf9c67a2c5dd43f4ed7ff0e171ad5e93ad32d..e80e6c716572ac2ddeb3fbbebc8d39c53639ee92 100644
--- a/lib/DebugInfo/PDB/PDBSymbolTypeManaged.cpp
+++ b/lib/DebugInfo/PDB/PDBSymbolTypeManaged.cpp
@@ -1,9 +1,8 @@
 //===- PDBSymboTypelManaged.cpp - ------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/PDB/PDBSymbolTypePointer.cpp b/lib/DebugInfo/PDB/PDBSymbolTypePointer.cpp
index cf5a369116a9248610ab2479b879d30d8f966636..462fc315359b5b4554e242477177322f1bd8d1f3 100644
--- a/lib/DebugInfo/PDB/PDBSymbolTypePointer.cpp
+++ b/lib/DebugInfo/PDB/PDBSymbolTypePointer.cpp
@@ -1,9 +1,8 @@
 //===- PDBSymbolTypePointer.cpp -----------------------------------*- C++ -===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/PDB/PDBSymbolTypeTypedef.cpp b/lib/DebugInfo/PDB/PDBSymbolTypeTypedef.cpp
index 1838f1612b49e3f729334692de509cacb95acd49..70749d9bf5f51ecbd70c7ece00c9c457f8ca44ba 100644
--- a/lib/DebugInfo/PDB/PDBSymbolTypeTypedef.cpp
+++ b/lib/DebugInfo/PDB/PDBSymbolTypeTypedef.cpp
@@ -1,9 +1,8 @@
 //===- PDBSymbolTypeTypedef.cpp ---------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/PDB/PDBSymbolTypeUDT.cpp b/lib/DebugInfo/PDB/PDBSymbolTypeUDT.cpp
index 2f5222f34fe4852e3c15020a6f69077155ea61f4..d302c29a3bec351d32e2badd560544a6e455c4f2 100644
--- a/lib/DebugInfo/PDB/PDBSymbolTypeUDT.cpp
+++ b/lib/DebugInfo/PDB/PDBSymbolTypeUDT.cpp
@@ -1,9 +1,8 @@
 //===- PDBSymbolTypeUDT.cpp - --------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/PDB/PDBSymbolTypeVTable.cpp b/lib/DebugInfo/PDB/PDBSymbolTypeVTable.cpp
index 0262f91e833681546eb99a0dbeaa30e8d0d99788..4e2a45116d512e72bc0aa12a5c0a1acfe58591ca 100644
--- a/lib/DebugInfo/PDB/PDBSymbolTypeVTable.cpp
+++ b/lib/DebugInfo/PDB/PDBSymbolTypeVTable.cpp
@@ -1,9 +1,8 @@
 //===- PDBSymbolTypeVTable.cpp - --------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/PDB/PDBSymbolTypeVTableShape.cpp b/lib/DebugInfo/PDB/PDBSymbolTypeVTableShape.cpp
index 16c3a36069810ef67bd301d2817e90d63941a702..78957620e0836d7b58cf99fc41e7eb7744e92557 100644
--- a/lib/DebugInfo/PDB/PDBSymbolTypeVTableShape.cpp
+++ b/lib/DebugInfo/PDB/PDBSymbolTypeVTableShape.cpp
@@ -1,9 +1,8 @@
 //===- PDBSymbolTypeVTableShape.cpp - ---------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/PDB/PDBSymbolUnknown.cpp b/lib/DebugInfo/PDB/PDBSymbolUnknown.cpp
index 7bcf9457a2b64d15c861be1a96b0792c62823b11..650d01183171b3cf55b2309746d1e15aeb1d7c85 100644
--- a/lib/DebugInfo/PDB/PDBSymbolUnknown.cpp
+++ b/lib/DebugInfo/PDB/PDBSymbolUnknown.cpp
@@ -1,9 +1,8 @@
 //===- PDBSymbolUnknown.cpp - -----------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/PDB/PDBSymbolUsingNamespace.cpp b/lib/DebugInfo/PDB/PDBSymbolUsingNamespace.cpp
index ecf2126f88026fd0d21d6f8b38c9f29b361b1a65..74afbdb18086553169b06b7cefb5c91ee675c853 100644
--- a/lib/DebugInfo/PDB/PDBSymbolUsingNamespace.cpp
+++ b/lib/DebugInfo/PDB/PDBSymbolUsingNamespace.cpp
@@ -1,9 +1,8 @@
 //===- PDBSymbolUsingNamespace.cpp - ------------------- --------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/PDB/UDTLayout.cpp b/lib/DebugInfo/PDB/UDTLayout.cpp
index 5f4390bbaf12ba042f50ae133a70e6b7f4f8e2b4..acb1599480b0e5b02a48ad8d08d07acffa0f4aea 100644
--- a/lib/DebugInfo/PDB/UDTLayout.cpp
+++ b/lib/DebugInfo/PDB/UDTLayout.cpp
@@ -1,9 +1,8 @@
 //===- UDTLayout.cpp ------------------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/DebugInfo/Symbolize/DIPrinter.cpp b/lib/DebugInfo/Symbolize/DIPrinter.cpp
index c1e2536d6e209eb0e54c140635dd2c321505ad25..18fe9bc610b765561f21a55a0874c47a7f51b474 100644
--- a/lib/DebugInfo/Symbolize/DIPrinter.cpp
+++ b/lib/DebugInfo/Symbolize/DIPrinter.cpp
@@ -1,9 +1,8 @@
 //===- lib/DebugInfo/Symbolize/DIPrinter.cpp ------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -19,6 +18,7 @@
 #include "llvm/Support/Format.h"
 #include "llvm/Support/LineIterator.h"
 #include "llvm/Support/MemoryBuffer.h"
+#include "llvm/Support/Path.h"
 #include "llvm/Support/raw_ostream.h"
 #include <algorithm>
 #include <cmath>
@@ -78,6 +78,8 @@ void DIPrinter::print(const DILineInfo &Info, bool Inlined) {
   std::string Filename = Info.FileName;
   if (Filename == kDILineInfoBadString)
     Filename = kBadString;
+  else if (Basenames)
+    Filename = llvm::sys::path::filename(Filename);
   if (!Verbose) {
     OS << Filename << ":" << Info.Line << ":" << Info.Column << "\n";
     printContext(Filename, Info.Line);
diff --git a/lib/DebugInfo/Symbolize/LLVMBuild.txt b/lib/DebugInfo/Symbolize/LLVMBuild.txt
index 06498a817fb749bd2eaa7645010eb0c694afebc9..70972ba9a825d139f5ed6e758973d07fb80948e8 100644
--- a/lib/DebugInfo/Symbolize/LLVMBuild.txt
+++ b/lib/DebugInfo/Symbolize/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/DebugInfo/Symbolize/LLVMBuild.txt ------------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/DebugInfo/Symbolize/SymbolizableObjectFile.cpp b/lib/DebugInfo/Symbolize/SymbolizableObjectFile.cpp
index 08be524ab464599373f14be20a8f66f28b9dc99c..21f017e768cb9ca05ddf614d9d4502dbc21be8ac 100644
--- a/lib/DebugInfo/Symbolize/SymbolizableObjectFile.cpp
+++ b/lib/DebugInfo/Symbolize/SymbolizableObjectFile.cpp
@@ -1,9 +1,8 @@
 //===- SymbolizableObjectFile.cpp -----------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/DebugInfo/Symbolize/SymbolizableObjectFile.h b/lib/DebugInfo/Symbolize/SymbolizableObjectFile.h
index 216cca8de4f5d4b184c43f416c457f4b1000ec4f..69da0f3e1450234709290944f4cf21b2ff055aa0 100644
--- a/lib/DebugInfo/Symbolize/SymbolizableObjectFile.h
+++ b/lib/DebugInfo/Symbolize/SymbolizableObjectFile.h
@@ -1,9 +1,8 @@
 //===- SymbolizableObjectFile.h ---------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/DebugInfo/Symbolize/Symbolize.cpp b/lib/DebugInfo/Symbolize/Symbolize.cpp
index 59a85d6c3fcf66acf7edefcb05aec8d287121337..73fe9780f522da1698c9da4dc5748768eb14cb82 100644
--- a/lib/DebugInfo/Symbolize/Symbolize.cpp
+++ b/lib/DebugInfo/Symbolize/Symbolize.cpp
@@ -1,9 +1,8 @@
 //===-- LLVMSymbolize.cpp -------------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -17,7 +16,6 @@
 
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/BinaryFormat/COFF.h"
-#include "llvm/Config/config.h"
 #include "llvm/DebugInfo/DWARF/DWARFContext.h"
 #include "llvm/DebugInfo/PDB/PDB.h"
 #include "llvm/DebugInfo/PDB/PDBContext.h"
@@ -34,7 +32,6 @@
 #include "llvm/Support/Path.h"
 #include <algorithm>
 #include <cassert>
-#include <cstdlib>
 #include <cstring>
 
 #if defined(_MSC_VER)
@@ -168,37 +165,40 @@ bool checkFileCRC(StringRef Path, uint32_t CRCHash) {
 
 bool findDebugBinary(const std::string &OrigPath,
                      const std::string &DebuglinkName, uint32_t CRCHash,
+                     const std::string &FallbackDebugPath,
                      std::string &Result) {
-  std::string OrigRealPath = OrigPath;
-#if defined(HAVE_REALPATH)
-  if (char *RP = realpath(OrigPath.c_str(), nullptr)) {
-    OrigRealPath = RP;
-    free(RP);
-  }
-#endif
-  SmallString<16> OrigDir(OrigRealPath);
+  SmallString<16> OrigDir(OrigPath);
   llvm::sys::path::remove_filename(OrigDir);
   SmallString<16> DebugPath = OrigDir;
-  // Try /path/to/original_binary/debuglink_name
+  // Try relative/path/to/original_binary/debuglink_name
   llvm::sys::path::append(DebugPath, DebuglinkName);
   if (checkFileCRC(DebugPath, CRCHash)) {
     Result = DebugPath.str();
     return true;
   }
-  // Try /path/to/original_binary/.debug/debuglink_name
+  // Try relative/path/to/original_binary/.debug/debuglink_name
   DebugPath = OrigDir;
   llvm::sys::path::append(DebugPath, ".debug", DebuglinkName);
   if (checkFileCRC(DebugPath, CRCHash)) {
     Result = DebugPath.str();
     return true;
   }
+  // Make the path absolute so that lookups will go to
+  // "/usr/lib/debug/full/path/to/debug", not
+  // "/usr/lib/debug/to/debug"
+  llvm::sys::fs::make_absolute(OrigDir);
+  if (!FallbackDebugPath.empty()) {
+    // Try <FallbackDebugPath>/absolute/path/to/original_binary/debuglink_name
+    DebugPath = FallbackDebugPath;
+  } else {
 #if defined(__NetBSD__)
-  // Try /usr/libdata/debug/path/to/original_binary/debuglink_name
-  DebugPath = "/usr/libdata/debug";
+    // Try /usr/libdata/debug/absolute/path/to/original_binary/debuglink_name
+    DebugPath = "/usr/libdata/debug";
 #else
-  // Try /usr/lib/debug/path/to/original_binary/debuglink_name
-  DebugPath = "/usr/lib/debug";
+    // Try /usr/lib/debug/absolute/path/to/original_binary/debuglink_name
+    DebugPath = "/usr/lib/debug";
 #endif
+  }
   llvm::sys::path::append(DebugPath, llvm::sys::path::relative_path(OrigDir),
                           DebuglinkName);
   if (checkFileCRC(DebugPath, CRCHash)) {
@@ -284,7 +284,8 @@ ObjectFile *LLVMSymbolizer::lookUpDebuglinkObject(const std::string &Path,
   std::string DebugBinaryPath;
   if (!getGNUDebuglinkContents(Obj, DebuglinkName, CRCHash))
     return nullptr;
-  if (!findDebugBinary(Path, DebuglinkName, CRCHash, DebugBinaryPath))
+  if (!findDebugBinary(Path, DebuglinkName, CRCHash, Opts.FallbackDebugPath,
+                       DebugBinaryPath))
     return nullptr;
   auto DbgObjOrErr = getOrCreateObject(DebugBinaryPath, ArchName);
   if (!DbgObjOrErr) {
diff --git a/lib/Demangle/CMakeLists.txt b/lib/Demangle/CMakeLists.txt
index a681af979e45a07b087a7870e7915e2b6b6fdfa3..cde1a4c5c28f56d378d89db8dfc082e31e2c3a9b 100644
--- a/lib/Demangle/CMakeLists.txt
+++ b/lib/Demangle/CMakeLists.txt
@@ -1,4 +1,5 @@
 add_llvm_library(LLVMDemangle
+  Demangle.cpp
   ItaniumDemangle.cpp
   MicrosoftDemangle.cpp
   MicrosoftDemangleNodes.cpp
diff --git a/lib/Demangle/Demangle.cpp b/lib/Demangle/Demangle.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..5f921537b9bd7c8ef1fe5ce736084180aae579c0
--- /dev/null
+++ b/lib/Demangle/Demangle.cpp
@@ -0,0 +1,36 @@
+//===-- Demangle.cpp - Common demangling functions ------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file This file contains definitions of common demangling functions.
+///
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Demangle/Demangle.h"
+#include <cstdlib>
+
+static bool isItaniumEncoding(const std::string &MangledName) {
+  size_t Pos = MangledName.find_first_not_of('_');
+  // A valid Itanium encoding requires 1-4 leading underscores, followed by 'Z'.
+  return Pos > 0 && Pos <= 4 && MangledName[Pos] == 'Z';
+}
+
+std::string llvm::demangle(const std::string &MangledName) {
+  char *Demangled;
+  if (isItaniumEncoding(MangledName))
+    Demangled = itaniumDemangle(MangledName.c_str(), nullptr, nullptr, nullptr);
+  else
+    Demangled =
+        microsoftDemangle(MangledName.c_str(), nullptr, nullptr, nullptr);
+
+  if (!Demangled)
+    return MangledName;
+
+  std::string Ret = Demangled;
+  free(Demangled);
+  return Ret;
+}
diff --git a/lib/Demangle/ItaniumDemangle.cpp b/lib/Demangle/ItaniumDemangle.cpp
index b2de0be2b70c5f2a5e418bd5502f0291d94070e3..5c99c70e3cc634450555292ee7334711f3863a70 100644
--- a/lib/Demangle/ItaniumDemangle.cpp
+++ b/lib/Demangle/ItaniumDemangle.cpp
@@ -1,9 +1,8 @@
 //===------------------------- ItaniumDemangle.cpp ------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is dual licensed under the MIT and the University of Illinois Open
-// Source Licenses. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Demangle/LLVMBuild.txt b/lib/Demangle/LLVMBuild.txt
index 725a3fc52121f66121bcdcb8b0d17c77ecaddc11..6fe47a21314a99ad5d553c573dcb6f57535dca46 100644
--- a/lib/Demangle/LLVMBuild.txt
+++ b/lib/Demangle/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/Support/LLVMBuild.txt ------------------------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Demangle/MicrosoftDemangle.cpp b/lib/Demangle/MicrosoftDemangle.cpp
index cca7bf95a7e4afdb9115fe52334b874461ac3d70..c75175c9f9ca95fe2a31ec6cd9c832687626920a 100644
--- a/lib/Demangle/MicrosoftDemangle.cpp
+++ b/lib/Demangle/MicrosoftDemangle.cpp
@@ -1,9 +1,8 @@
 //===- MicrosoftDemangle.cpp ----------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is dual licensed under the MIT and the University of Illinois Open
-// Source Licenses. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -18,7 +17,7 @@
 #include "llvm/Demangle/Demangle.h"
 #include "llvm/Demangle/MicrosoftDemangleNodes.h"
 
-#include "llvm/Demangle/Compiler.h"
+#include "llvm/Demangle/DemangleConfig.h"
 #include "llvm/Demangle/StringView.h"
 #include "llvm/Demangle/Utility.h"
 
@@ -265,7 +264,7 @@ Demangler::demangleSpecialTableSymbolNode(StringView &MangledName,
     NI->Name = "`RTTI Complete Object Locator'";
     break;
   default:
-    LLVM_BUILTIN_UNREACHABLE;
+    DEMANGLE_UNREACHABLE;
   }
   QualifiedNameNode *QN = demangleNameScopeChain(MangledName, NI);
   SpecialTableSymbolNode *STSN = Arena.alloc<SpecialTableSymbolNode>();
@@ -632,7 +631,7 @@ translateIntrinsicFunctionCode(char CH, FunctionIdentifierCodeGroup Group) {
   case FunctionIdentifierCodeGroup::DoubleUnder:
     return DoubleUnder[Index];
   }
-  LLVM_BUILTIN_UNREACHABLE;
+  DEMANGLE_UNREACHABLE;
 }
 
 IdentifierNode *
@@ -1188,7 +1187,7 @@ Demangler::demangleStringLiteral(StringView &MangledName) {
   switch (MangledName.popFront()) {
   case '1':
     IsWcharT = true;
-    LLVM_FALLTHROUGH;
+    DEMANGLE_FALLTHROUGH;
   case '0':
     break;
   default:
@@ -1255,7 +1254,7 @@ Demangler::demangleStringLiteral(StringView &MangledName) {
       Result->Char = CharKind::Char32;
       break;
     default:
-      LLVM_BUILTIN_UNREACHABLE;
+      DEMANGLE_UNREACHABLE;
     }
     const unsigned NumChars = BytesDecoded / CharBytes;
     for (unsigned CharIndex = 0; CharIndex < NumChars; ++CharIndex) {
@@ -1681,11 +1680,14 @@ TypeNode *Demangler::demangleType(StringView &MangledName,
   return Ty;
 }
 
-void Demangler::demangleThrowSpecification(StringView &MangledName) {
+bool Demangler::demangleThrowSpecification(StringView &MangledName) {
+  if (MangledName.consumeFront("_E"))
+    return true;
   if (MangledName.consumeFront('Z'))
-    return;
+    return false;
 
   Error = true;
+  return false;
 }
 
 FunctionSignatureNode *Demangler::demangleFunctionType(StringView &MangledName,
@@ -1709,7 +1711,7 @@ FunctionSignatureNode *Demangler::demangleFunctionType(StringView &MangledName,
 
   FTy->Params = demangleFunctionParameterList(MangledName);
 
-  demangleThrowSpecification(MangledName);
+  FTy->IsNoexcept = demangleThrowSpecification(MangledName);
 
   return FTy;
 }
@@ -2080,15 +2082,15 @@ Demangler::demangleTemplateParameterList(StringView &MangledName) {
       case 'J':
         TPRN->ThunkOffsets[TPRN->ThunkOffsetCount++] =
             demangleSigned(MangledName);
-        LLVM_FALLTHROUGH;
+        DEMANGLE_FALLTHROUGH;
       case 'I':
         TPRN->ThunkOffsets[TPRN->ThunkOffsetCount++] =
             demangleSigned(MangledName);
-        LLVM_FALLTHROUGH;
+        DEMANGLE_FALLTHROUGH;
       case 'H':
         TPRN->ThunkOffsets[TPRN->ThunkOffsetCount++] =
             demangleSigned(MangledName);
-        LLVM_FALLTHROUGH;
+        DEMANGLE_FALLTHROUGH;
       case '1':
         break;
       default:
@@ -2114,13 +2116,13 @@ Demangler::demangleTemplateParameterList(StringView &MangledName) {
       case 'G':
         TPRN->ThunkOffsets[TPRN->ThunkOffsetCount++] =
             demangleSigned(MangledName);
-        LLVM_FALLTHROUGH;
+        DEMANGLE_FALLTHROUGH;
       case 'F':
         TPRN->ThunkOffsets[TPRN->ThunkOffsetCount++] =
             demangleSigned(MangledName);
         TPRN->ThunkOffsets[TPRN->ThunkOffsetCount++] =
             demangleSigned(MangledName);
-        LLVM_FALLTHROUGH;
+        DEMANGLE_FALLTHROUGH;
       case '0':
         break;
       default:
diff --git a/lib/Demangle/MicrosoftDemangleNodes.cpp b/lib/Demangle/MicrosoftDemangleNodes.cpp
index a588dca5b7ba338bdf5498965099833b47d886c0..e19d041b90ec38d271cd40cead09808defa52ec6 100644
--- a/lib/Demangle/MicrosoftDemangleNodes.cpp
+++ b/lib/Demangle/MicrosoftDemangleNodes.cpp
@@ -1,9 +1,8 @@
 //===- MicrosoftDemangle.cpp ----------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is dual licensed under the MIT and the University of Illinois Open
-// Source Licenses. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -12,7 +11,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/Demangle/MicrosoftDemangleNodes.h"
-#include "llvm/Demangle/Compiler.h"
+#include "llvm/Demangle/DemangleConfig.h"
 #include "llvm/Demangle/Utility.h"
 #include <cctype>
 #include <string>
@@ -114,10 +113,10 @@ static void outputCallingConvention(OutputStream &OS, CallingConv CC) {
   }
 }
 
-std::string Node::toString() const {
+std::string Node::toString(OutputFlags Flags) const {
   OutputStream OS;
   initializeOutputStream(nullptr, nullptr, OS, 1024);
-  this->output(OS, llvm::ms_demangle::OF_Default);
+  this->output(OS, Flags);
   OS << '\0';
   return {OS.getBuffer()};
 }
@@ -423,6 +422,9 @@ void FunctionSignatureNode::outputPost(OutputStream &OS,
   if (Quals & Q_Unaligned)
     OS << " __unaligned";
 
+  if (IsNoexcept)
+    OS << " noexcept";
+
   if (RefQualifier == FunctionRefQualifier::Reference)
     OS << " &";
   else if (RefQualifier == FunctionRefQualifier::RValueReference)
@@ -510,13 +512,15 @@ void PointerTypeNode::outputPost(OutputStream &OS, OutputFlags Flags) const {
 }
 
 void TagTypeNode::outputPre(OutputStream &OS, OutputFlags Flags) const {
-  switch (Tag) {
-    OUTPUT_ENUM_CLASS_VALUE(TagKind, Class, "class");
-    OUTPUT_ENUM_CLASS_VALUE(TagKind, Struct, "struct");
-    OUTPUT_ENUM_CLASS_VALUE(TagKind, Union, "union");
-    OUTPUT_ENUM_CLASS_VALUE(TagKind, Enum, "enum");
+  if (!(Flags & OF_NoTagSpecifier)) {
+    switch (Tag) {
+      OUTPUT_ENUM_CLASS_VALUE(TagKind, Class, "class");
+      OUTPUT_ENUM_CLASS_VALUE(TagKind, Struct, "struct");
+      OUTPUT_ENUM_CLASS_VALUE(TagKind, Union, "union");
+      OUTPUT_ENUM_CLASS_VALUE(TagKind, Enum, "enum");
+    }
+    OS << " ";
   }
-  OS << " ";
   QualifiedName->output(OS, Flags);
   outputQualifiers(OS, Quals, true, false);
 }
diff --git a/lib/ExecutionEngine/ExecutionEngine.cpp b/lib/ExecutionEngine/ExecutionEngine.cpp
index ae96c7f5955fef2cd7f73006129cf714b772734b..c2ea0f07f85e701320f7a30b2f698bc249e2bb3f 100644
--- a/lib/ExecutionEngine/ExecutionEngine.cpp
+++ b/lib/ExecutionEngine/ExecutionEngine.cpp
@@ -1,9 +1,8 @@
 //===-- ExecutionEngine.cpp - Common Implementation shared by EEs ---------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/ExecutionEngine/ExecutionEngineBindings.cpp b/lib/ExecutionEngine/ExecutionEngineBindings.cpp
index 3be4bec566a0bd5c6da6c9fe9b5c4d981cbad610..c741fe2b3778374e4c4c7e319ebaafa80ab5723d 100644
--- a/lib/ExecutionEngine/ExecutionEngineBindings.cpp
+++ b/lib/ExecutionEngine/ExecutionEngineBindings.cpp
@@ -1,9 +1,8 @@
 //===-- ExecutionEngineBindings.cpp - C bindings for EEs ------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/ExecutionEngine/GDBRegistrationListener.cpp b/lib/ExecutionEngine/GDBRegistrationListener.cpp
index 8204f5a90268bb313dc2c1500ebf0341e9db9511..08d20156a590aa5df4573d430257c74badd23450 100644
--- a/lib/ExecutionEngine/GDBRegistrationListener.cpp
+++ b/lib/ExecutionEngine/GDBRegistrationListener.cpp
@@ -1,9 +1,8 @@
 //===----- GDBRegistrationListener.cpp - Registers objects with GDB -------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/ExecutionEngine/IntelJITEvents/IntelJITEventListener.cpp b/lib/ExecutionEngine/IntelJITEvents/IntelJITEventListener.cpp
index e9051c198506ed7d12125495036466ab0f2bbb0a..4e8360649bc04f951635ccc20fe9d4dfc9f4192a 100644
--- a/lib/ExecutionEngine/IntelJITEvents/IntelJITEventListener.cpp
+++ b/lib/ExecutionEngine/IntelJITEvents/IntelJITEventListener.cpp
@@ -1,9 +1,8 @@
 //===-- IntelJITEventListener.cpp - Tell Intel profiler about JITed code --===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/ExecutionEngine/IntelJITEvents/IntelJITEventsWrapper.h b/lib/ExecutionEngine/IntelJITEvents/IntelJITEventsWrapper.h
index 777d0f179cb5a96f3f47dae909d3c0d8330b18a9..68699c6a2200c9057fcec576704a381bcda9b5b3 100644
--- a/lib/ExecutionEngine/IntelJITEvents/IntelJITEventsWrapper.h
+++ b/lib/ExecutionEngine/IntelJITEvents/IntelJITEventsWrapper.h
@@ -1,9 +1,8 @@
 //===-- IntelJITEventsWrapper.h - Intel JIT Events API Wrapper --*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/ExecutionEngine/IntelJITEvents/LLVMBuild.txt b/lib/ExecutionEngine/IntelJITEvents/LLVMBuild.txt
index 8584500d64ad356c392ff2c67fd546ff94a43c90..2b1aa1b3915289acbd5bd5311910187648d0988b 100644
--- a/lib/ExecutionEngine/IntelJITEvents/LLVMBuild.txt
+++ b/lib/ExecutionEngine/IntelJITEvents/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/ExecutionEngine/JITProfileAmplifier/LLVMBuild.txt --*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/ExecutionEngine/IntelJITEvents/ittnotify_config.h b/lib/ExecutionEngine/IntelJITEvents/ittnotify_config.h
index 61d8cc75d9f201f6164cbcc5278e4230a742db49..16ce672150cc25e60eca46c5291c96a55c141c9c 100644
--- a/lib/ExecutionEngine/IntelJITEvents/ittnotify_config.h
+++ b/lib/ExecutionEngine/IntelJITEvents/ittnotify_config.h
@@ -1,9 +1,8 @@
 /*===-- ittnotify_config.h - JIT Profiling API internal config-----*- C -*-===*
  *
- *                     The LLVM Compiler Infrastructure
- *
- * This file is distributed under the University of Illinois Open Source
- * License. See LICENSE.TXT for details.
+ * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+ * See https://llvm.org/LICENSE.txt for license information.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
  *
  *===----------------------------------------------------------------------===*
  *
diff --git a/lib/ExecutionEngine/IntelJITEvents/ittnotify_types.h b/lib/ExecutionEngine/IntelJITEvents/ittnotify_types.h
index 5df752f66f107359929587aea0874468010b5d82..15008fe93e607bfa62fe7e7d24387af62b539dc0 100644
--- a/lib/ExecutionEngine/IntelJITEvents/ittnotify_types.h
+++ b/lib/ExecutionEngine/IntelJITEvents/ittnotify_types.h
@@ -1,9 +1,8 @@
 /*===-- ittnotify_types.h - JIT Profiling API internal types--------*- C -*-===*
  *
- *                     The LLVM Compiler Infrastructure
- *
- * This file is distributed under the University of Illinois Open Source
- * License. See LICENSE.TXT for details.
+ * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+ * See https://llvm.org/LICENSE.txt for license information.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
  *
  *===----------------------------------------------------------------------===*
  *
diff --git a/lib/ExecutionEngine/IntelJITEvents/jitprofiling.c b/lib/ExecutionEngine/IntelJITEvents/jitprofiling.c
index bc8fea1487490a49375760f0014c0c636b90951c..074e0735628a5767e0ebfc505cd0243100ac4df7 100644
--- a/lib/ExecutionEngine/IntelJITEvents/jitprofiling.c
+++ b/lib/ExecutionEngine/IntelJITEvents/jitprofiling.c
@@ -1,9 +1,8 @@
 /*===-- jitprofiling.c - JIT (Just-In-Time) Profiling API----------*- C -*-===*
  *
- *                     The LLVM Compiler Infrastructure
- *
- * This file is distributed under the University of Illinois Open Source
- * License. See LICENSE.TXT for details.
+ * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+ * See https://llvm.org/LICENSE.txt for license information.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
  *
  *===----------------------------------------------------------------------===*
  *
diff --git a/lib/ExecutionEngine/IntelJITEvents/jitprofiling.h b/lib/ExecutionEngine/IntelJITEvents/jitprofiling.h
index efd2b1a33f7534881e4b67da700bd8f987ea0b2b..ba627b430ff1272101a65bd8c41b2af123cf743e 100644
--- a/lib/ExecutionEngine/IntelJITEvents/jitprofiling.h
+++ b/lib/ExecutionEngine/IntelJITEvents/jitprofiling.h
@@ -1,9 +1,8 @@
 /*===-- jitprofiling.h - JIT Profiling API-------------------------*- C -*-===*
  *
- *                     The LLVM Compiler Infrastructure
- *
- * This file is distributed under the University of Illinois Open Source
- * License. See LICENSE.TXT for details.
+ * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+ * See https://llvm.org/LICENSE.txt for license information.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
  *
  *===----------------------------------------------------------------------===*
  *
diff --git a/lib/ExecutionEngine/Interpreter/Execution.cpp b/lib/ExecutionEngine/Interpreter/Execution.cpp
index 39cf6d4a32a35e5ac459691611d8d7edb8504929..4c1cf95b37f8c356e5e3f3081c342fe5fff68c7a 100644
--- a/lib/ExecutionEngine/Interpreter/Execution.cpp
+++ b/lib/ExecutionEngine/Interpreter/Execution.cpp
@@ -1,9 +1,8 @@
 //===-- Execution.cpp - Implement code to simulate the program ------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -1778,17 +1777,14 @@ void Interpreter::visitExtractElementInst(ExtractElementInst &I) {
 
 void Interpreter::visitInsertElementInst(InsertElementInst &I) {
   ExecutionContext &SF = ECStack.back();
-  Type *Ty = I.getType();
-
-  if(!(Ty->isVectorTy()) )
-    llvm_unreachable("Unhandled dest type for insertelement instruction");
+  VectorType *Ty = cast<VectorType>(I.getType());
 
   GenericValue Src1 = getOperandValue(I.getOperand(0), SF);
   GenericValue Src2 = getOperandValue(I.getOperand(1), SF);
   GenericValue Src3 = getOperandValue(I.getOperand(2), SF);
   GenericValue Dest;
 
-  Type *TyContained = Ty->getContainedType(0);
+  Type *TyContained = Ty->getElementType();
 
   const unsigned indx = unsigned(Src3.IntVal.getZExtValue());
   Dest.AggregateVal = Src1.AggregateVal;
@@ -1814,9 +1810,7 @@ void Interpreter::visitInsertElementInst(InsertElementInst &I) {
 void Interpreter::visitShuffleVectorInst(ShuffleVectorInst &I){
   ExecutionContext &SF = ECStack.back();
 
-  Type *Ty = I.getType();
-  if(!(Ty->isVectorTy()))
-    llvm_unreachable("Unhandled dest type for shufflevector instruction");
+  VectorType *Ty = cast<VectorType>(I.getType());
 
   GenericValue Src1 = getOperandValue(I.getOperand(0), SF);
   GenericValue Src2 = getOperandValue(I.getOperand(1), SF);
@@ -1827,7 +1821,7 @@ void Interpreter::visitShuffleVectorInst(ShuffleVectorInst &I){
   // bytecode can't contain different types for src1 and src2 for a
   // shufflevector instruction.
 
-  Type *TyContained = Ty->getContainedType(0);
+  Type *TyContained = Ty->getElementType();
   unsigned src1Size = (unsigned)Src1.AggregateVal.size();
   unsigned src2Size = (unsigned)Src2.AggregateVal.size();
   unsigned src3Size = (unsigned)Src3.AggregateVal.size();
diff --git a/lib/ExecutionEngine/Interpreter/ExternalFunctions.cpp b/lib/ExecutionEngine/Interpreter/ExternalFunctions.cpp
index 044d9b7f27a793ea235c91b163965f76e2d696cd..c3a2ccc582c91852665cbc6d5a5ad81e5b531713 100644
--- a/lib/ExecutionEngine/Interpreter/ExternalFunctions.cpp
+++ b/lib/ExecutionEngine/Interpreter/ExternalFunctions.cpp
@@ -1,9 +1,8 @@
 //===-- ExternalFunctions.cpp - Implement External Functions --------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -103,8 +102,9 @@ static ExFunc lookupFunction(const Function *F) {
   // composite function name should be.
   std::string ExtName = "lle_";
   FunctionType *FT = F->getFunctionType();
-  for (unsigned i = 0, e = FT->getNumContainedTypes(); i != e; ++i)
-    ExtName += getTypeID(FT->getContainedType(i));
+  ExtName += getTypeID(FT->getReturnType());
+  for (Type *T : FT->params())
+    ExtName += getTypeID(T);
   ExtName += ("_" + F->getName()).str();
 
   sys::ScopedLock Writer(*FunctionsLock);
diff --git a/lib/ExecutionEngine/Interpreter/Interpreter.cpp b/lib/ExecutionEngine/Interpreter/Interpreter.cpp
index 9818adfff82ed84fb3e4f1de8693a574bb5b6d84..5727f7adb49c3473a696d8b8348a96262d618848 100644
--- a/lib/ExecutionEngine/Interpreter/Interpreter.cpp
+++ b/lib/ExecutionEngine/Interpreter/Interpreter.cpp
@@ -1,9 +1,8 @@
 //===- Interpreter.cpp - Top-Level LLVM Interpreter Implementation --------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/ExecutionEngine/Interpreter/Interpreter.h b/lib/ExecutionEngine/Interpreter/Interpreter.h
index 33542e7e43add705b942b787660e4caa01173477..898f1d34ec1d36e12b73e54d1984c0fc49c65054 100644
--- a/lib/ExecutionEngine/Interpreter/Interpreter.h
+++ b/lib/ExecutionEngine/Interpreter/Interpreter.h
@@ -1,9 +1,8 @@
 //===-- Interpreter.h ------------------------------------------*- C++ -*--===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/ExecutionEngine/Interpreter/LLVMBuild.txt b/lib/ExecutionEngine/Interpreter/LLVMBuild.txt
index 5af77e54725306b91d908df5e842f0f7c5ce084b..3debc32b0313af25e5113a4f65ddd216c0462d35 100644
--- a/lib/ExecutionEngine/Interpreter/LLVMBuild.txt
+++ b/lib/ExecutionEngine/Interpreter/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/ExecutionEngine/Interpreter/LLVMBuild.txt ----------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/ExecutionEngine/LLVMBuild.txt b/lib/ExecutionEngine/LLVMBuild.txt
index b6e1bda6a514e89600058266639d254f20bcff31..4a568e8cf045bc1bef74895a60dbea7190926da6 100644
--- a/lib/ExecutionEngine/LLVMBuild.txt
+++ b/lib/ExecutionEngine/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/ExecutionEngine/LLVMBuild.txt ----------------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/ExecutionEngine/MCJIT/LLVMBuild.txt b/lib/ExecutionEngine/MCJIT/LLVMBuild.txt
index 922cd0ddf15acdb94a89ddaa57f3893b8de322e7..a20b6d808dfca6e3e9d95766dedf551c2a85c228 100644
--- a/lib/ExecutionEngine/MCJIT/LLVMBuild.txt
+++ b/lib/ExecutionEngine/MCJIT/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/ExecutionEngine/MCJIT/LLVMBuild.txt ----------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/ExecutionEngine/MCJIT/MCJIT.cpp b/lib/ExecutionEngine/MCJIT/MCJIT.cpp
index ffc6707e1488c517c5168b3d1a3f9a06c19af22a..08815b7a80aef0d3672912ae256fd21a849e760a 100644
--- a/lib/ExecutionEngine/MCJIT/MCJIT.cpp
+++ b/lib/ExecutionEngine/MCJIT/MCJIT.cpp
@@ -1,9 +1,8 @@
 //===-- MCJIT.cpp - MC-based Just-in-Time Compiler ------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/ExecutionEngine/MCJIT/MCJIT.h b/lib/ExecutionEngine/MCJIT/MCJIT.h
index 1119e138720f43138ee206649a452672445aeaee..77097fc0d17e40d56aa698ba15581d705224bdd6 100644
--- a/lib/ExecutionEngine/MCJIT/MCJIT.h
+++ b/lib/ExecutionEngine/MCJIT/MCJIT.h
@@ -1,9 +1,8 @@
 //===-- MCJIT.h - Class definition for the MCJIT ----------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/ExecutionEngine/OProfileJIT/LLVMBuild.txt b/lib/ExecutionEngine/OProfileJIT/LLVMBuild.txt
index ea10028631890bfa506bf86353f9ab5b9f8047a0..9084fea3635a32d0d861b2133fd51b9b899e0a20 100644
--- a/lib/ExecutionEngine/OProfileJIT/LLVMBuild.txt
+++ b/lib/ExecutionEngine/OProfileJIT/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/ExecutionEngine/OProfileJIT/LLVMBuild.txt ----------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/ExecutionEngine/OProfileJIT/OProfileJITEventListener.cpp b/lib/ExecutionEngine/OProfileJIT/OProfileJITEventListener.cpp
index 21af6b585c41e7d55a2ebab94767815e7634291f..2ad9d24555f3dbe312e694b820275e4786aa87c9 100644
--- a/lib/ExecutionEngine/OProfileJIT/OProfileJITEventListener.cpp
+++ b/lib/ExecutionEngine/OProfileJIT/OProfileJITEventListener.cpp
@@ -1,9 +1,8 @@
 //===-- OProfileJITEventListener.cpp - Tell OProfile about JITted code ----===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/ExecutionEngine/OProfileJIT/OProfileWrapper.cpp b/lib/ExecutionEngine/OProfileJIT/OProfileWrapper.cpp
index b473ac3faf4c5135bac7ab5530abd9fd6b32268a..1a2667736926fe94cfae0d714a9559db9ccf946d 100644
--- a/lib/ExecutionEngine/OProfileJIT/OProfileWrapper.cpp
+++ b/lib/ExecutionEngine/OProfileJIT/OProfileWrapper.cpp
@@ -1,9 +1,8 @@
 //===-- OProfileWrapper.cpp - OProfile JIT API Wrapper implementation -----===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/ExecutionEngine/Orc/CompileOnDemandLayer.cpp b/lib/ExecutionEngine/Orc/CompileOnDemandLayer.cpp
index 241eb3600da748463aa8c7e461240153e581740e..99bf53bc3afa88cb116eb4408de523490b6ccae5 100644
--- a/lib/ExecutionEngine/Orc/CompileOnDemandLayer.cpp
+++ b/lib/ExecutionEngine/Orc/CompileOnDemandLayer.cpp
@@ -1,9 +1,8 @@
 //===----- CompileOnDemandLayer.cpp - Lazily emit IR on first call --------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/ExecutionEngine/Orc/Core.cpp b/lib/ExecutionEngine/Orc/Core.cpp
index 73c0bcdf7d28dab061dfc670cea5574ca2cfef0b..99504585d5bcb2192fd390051b3fe3035d84238d 100644
--- a/lib/ExecutionEngine/Orc/Core.cpp
+++ b/lib/ExecutionEngine/Orc/Core.cpp
@@ -1,9 +1,8 @@
 //===--- Core.cpp - Core ORC APIs (MaterializationUnit, JITDylib, etc.) ---===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/ExecutionEngine/Orc/ExecutionUtils.cpp b/lib/ExecutionEngine/Orc/ExecutionUtils.cpp
index 7c3c50b4d6e534549a1b4b3a274fe63c1a42613e..0ef85fd6857e27ec1f668ab5536b449914d438e6 100644
--- a/lib/ExecutionEngine/Orc/ExecutionUtils.cpp
+++ b/lib/ExecutionEngine/Orc/ExecutionUtils.cpp
@@ -1,9 +1,8 @@
 //===---- ExecutionUtils.cpp - Utilities for executing functions in Orc ---===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/ExecutionEngine/Orc/IRCompileLayer.cpp b/lib/ExecutionEngine/Orc/IRCompileLayer.cpp
index d952d1be70dab1e587b93b52a82d5602e1b337ca..81dfc02f55b2fe5bd2dfe168cb6dd7bc93332d4f 100644
--- a/lib/ExecutionEngine/Orc/IRCompileLayer.cpp
+++ b/lib/ExecutionEngine/Orc/IRCompileLayer.cpp
@@ -1,9 +1,8 @@
 //===--------------- IRCompileLayer.cpp - IR Compiling Layer --------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/ExecutionEngine/Orc/IRTransformLayer.cpp b/lib/ExecutionEngine/Orc/IRTransformLayer.cpp
index 7bc0d696e3acaba058381ef5fcb9b83ed500ea8f..e3519284613e0cc1dd744fb3cf2a43fe678bd8cf 100644
--- a/lib/ExecutionEngine/Orc/IRTransformLayer.cpp
+++ b/lib/ExecutionEngine/Orc/IRTransformLayer.cpp
@@ -1,9 +1,8 @@
 //===-------------- IRTransformLayer.cpp - IR Transform Layer -------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/ExecutionEngine/Orc/IndirectionUtils.cpp b/lib/ExecutionEngine/Orc/IndirectionUtils.cpp
index 82000ec5b32bc4e708217a601c6ed472ee81126f..0b52ba813abecb02b604fd31561fb0628fda1143 100644
--- a/lib/ExecutionEngine/Orc/IndirectionUtils.cpp
+++ b/lib/ExecutionEngine/Orc/IndirectionUtils.cpp
@@ -1,9 +1,8 @@
 //===---- IndirectionUtils.cpp - Utilities for call indirection in Orc ----===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -238,11 +237,11 @@ void makeStub(Function &F, Value &ImplPointer) {
   Module &M = *F.getParent();
   BasicBlock *EntryBlock = BasicBlock::Create(M.getContext(), "entry", &F);
   IRBuilder<> Builder(EntryBlock);
-  LoadInst *ImplAddr = Builder.CreateLoad(&ImplPointer);
+  LoadInst *ImplAddr = Builder.CreateLoad(F.getType(), &ImplPointer);
   std::vector<Value*> CallArgs;
   for (auto &A : F.args())
     CallArgs.push_back(&A);
-  CallInst *Call = Builder.CreateCall(ImplAddr, CallArgs);
+  CallInst *Call = Builder.CreateCall(F.getFunctionType(), ImplAddr, CallArgs);
   Call->setTailCall();
   Call->setAttributes(F.getAttributes());
   if (F.getReturnType()->isVoidTy())
diff --git a/lib/ExecutionEngine/Orc/JITTargetMachineBuilder.cpp b/lib/ExecutionEngine/Orc/JITTargetMachineBuilder.cpp
index 4af09d196ff9a0fca7d56da03f2831a03e49fb6b..df23547a9de3ab0345ac57f2de2ba9d9c3e85dbb 100644
--- a/lib/ExecutionEngine/Orc/JITTargetMachineBuilder.cpp
+++ b/lib/ExecutionEngine/Orc/JITTargetMachineBuilder.cpp
@@ -1,9 +1,8 @@
 //===----- JITTargetMachineBuilder.cpp - Build TargetMachines for JIT -----===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/ExecutionEngine/Orc/LLJIT.cpp b/lib/ExecutionEngine/Orc/LLJIT.cpp
index e2089f9106bd67ecd286e299b1e3b96027d4a598..62027314621efb79e3a80b07b949c27f3e2e17bd 100644
--- a/lib/ExecutionEngine/Orc/LLJIT.cpp
+++ b/lib/ExecutionEngine/Orc/LLJIT.cpp
@@ -1,9 +1,8 @@
 //===--------- LLJIT.cpp - An ORC-based JIT for compiling LLVM IR ---------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/ExecutionEngine/Orc/LLVMBuild.txt b/lib/ExecutionEngine/Orc/LLVMBuild.txt
index 3f8fe9545fcbad0f366e00771d70b2fe6db9594f..c5af0475c9ccfe50874365d062a6a26076121bc7 100644
--- a/lib/ExecutionEngine/Orc/LLVMBuild.txt
+++ b/lib/ExecutionEngine/Orc/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/ExecutionEngine/Orc/LLVMBuild.txt ----------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/ExecutionEngine/Orc/Layer.cpp b/lib/ExecutionEngine/Orc/Layer.cpp
index 11af76825e9fd09d2daef003f96871cfe32391ee..73470bf18c55a86efbc4657ae4df4d1b251b2883 100644
--- a/lib/ExecutionEngine/Orc/Layer.cpp
+++ b/lib/ExecutionEngine/Orc/Layer.cpp
@@ -1,9 +1,8 @@
 //===-------------------- Layer.cpp - Layer interfaces --------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/ExecutionEngine/Orc/LazyReexports.cpp b/lib/ExecutionEngine/Orc/LazyReexports.cpp
index 55f4a7c5afcec2f8ed2d76e2e301343f9561ad71..c2565812d5174d4e5067b54c196d8d3b488b96b2 100644
--- a/lib/ExecutionEngine/Orc/LazyReexports.cpp
+++ b/lib/ExecutionEngine/Orc/LazyReexports.cpp
@@ -1,9 +1,8 @@
 //===---------- LazyReexports.cpp - Utilities for lazy reexports ----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/ExecutionEngine/Orc/Legacy.cpp b/lib/ExecutionEngine/Orc/Legacy.cpp
index ddb72544b770b63ea898c0eaf5c9f3d1c6a152e9..9fd07d0072ff815e36b2506c591caa7d6706993d 100644
--- a/lib/ExecutionEngine/Orc/Legacy.cpp
+++ b/lib/ExecutionEngine/Orc/Legacy.cpp
@@ -1,9 +1,8 @@
 //===------- Legacy.cpp - Adapters for ExecutionEngine API interop --------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/ExecutionEngine/Orc/NullResolver.cpp b/lib/ExecutionEngine/Orc/NullResolver.cpp
index 922fc6f021ce972821c8d0e4b959cf5c472a852a..5b4345b870bb8dc4543fd78263ea6fe90f039c16 100644
--- a/lib/ExecutionEngine/Orc/NullResolver.cpp
+++ b/lib/ExecutionEngine/Orc/NullResolver.cpp
@@ -1,9 +1,8 @@
 //===---------- NullResolver.cpp - Reject symbol lookup requests ----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/ExecutionEngine/Orc/ObjectTransformLayer.cpp b/lib/ExecutionEngine/Orc/ObjectTransformLayer.cpp
index 825f532047360a948affd515573bc804a224d769..815517321b76bea840c318af96692730149d6d64 100644
--- a/lib/ExecutionEngine/Orc/ObjectTransformLayer.cpp
+++ b/lib/ExecutionEngine/Orc/ObjectTransformLayer.cpp
@@ -1,9 +1,8 @@
 //===---------- ObjectTransformLayer.cpp - Object Transform Layer ---------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/ExecutionEngine/Orc/OrcABISupport.cpp b/lib/ExecutionEngine/Orc/OrcABISupport.cpp
index cf335df0e7c52ecb2d79d384437c18684ae64592..4bf52864107b1404df57a46020d040f8f74f027a 100644
--- a/lib/ExecutionEngine/Orc/OrcABISupport.cpp
+++ b/lib/ExecutionEngine/Orc/OrcABISupport.cpp
@@ -1,9 +1,8 @@
 //===------------- OrcABISupport.cpp - ABI specific support code ----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -537,7 +536,8 @@ Error OrcI386::emitIndirectStubsBlock(IndirectStubsInfo &StubsInfo,
   return Error::success();
 }
 
-void OrcMips32_Base::writeResolverCode(uint8_t *ResolverMem, JITReentryFn ReentryFn,
+void OrcMips32_Base::writeResolverCode(uint8_t *ResolverMem,
+                                       JITReentryFn ReentryFn,
                                        void *CallbackMgr, bool isBigEndian) {
 
   const uint32_t ResolverCode[] = {
@@ -604,51 +604,42 @@ void OrcMips32_Base::writeResolverCode(uint8_t *ResolverMem, JITReentryFn Reentr
       0x8fb1001c,                    // 0xd4: lw $s1,28($sp)
       0x8fb00018,                    // 0xd8: lw $s0,24($sp)
       0x8fa70014,                    // 0xdc: lw $a3,20($sp)
-      0x8fa70014,                    // 0xe0: lw $a3,20($sp)
-      0x8fa60010,                    // 0xe4: lw $a2,16($sp)
-      0x8fa5000c,                    // 0xe8: lw $a1,12($sp)
-      0x8fa40008,                    // 0xec: lw $a0,8($sp)
-      0x27bd0068,                    // 0xf4: addiu $sp,$sp,104
-      0x0300f825,                    // 0xf8: move $ra, $t8
-      0x00000000                     // 0xfc: jr $v0/v1
+      0x8fa60010,                    // 0xe0: lw $a2,16($sp)
+      0x8fa5000c,                    // 0xe4: lw $a1,12($sp)
+      0x8fa40008,                    // 0xe8: lw $a0,8($sp)
+      0x27bd0068,                    // 0xec: addiu $sp,$sp,104
+      0x0300f825,                    // 0xf0: move $ra, $t8
+      0x03200008,                    // 0xf4: jr $t9
+      0x00000000,                    // 0xf8: move $t9, $v0/v1
   };
 
-
-  const unsigned ReentryFnAddrOffset = 0x7c;  // JIT re-entry fn addr lui
+  const unsigned ReentryFnAddrOffset = 0x7c;   // JIT re-entry fn addr lui
   const unsigned CallbackMgrAddrOffset = 0x6c; // Callback manager addr lui
-  const unsigned offsett = 0xfc;
+  const unsigned Offsett = 0xf8;
 
   memcpy(ResolverMem, ResolverCode, sizeof(ResolverCode));
 
-  //Depending on endian return value will be in v0 or v1.
-  uint32_t JumpV0 = 0x00400008;
-  uint32_t JumpV1 = 0x00600008;
-
-  if(isBigEndian == true)
-     memcpy(ResolverMem + offsett, &JumpV1,
-         sizeof(JumpV1));
-  else
-     memcpy(ResolverMem + offsett, &JumpV0,
-         sizeof(JumpV0));
+  // Depending on endian return value will be in v0 or v1.
+  uint32_t MoveVxT9 = isBigEndian ? 0x0060c825 : 0x0040c825;
+  memcpy(ResolverMem + Offsett, &MoveVxT9, sizeof(MoveVxT9));
 
   uint64_t CallMgrAddr = reinterpret_cast<uint64_t>(CallbackMgr);
   uint32_t CallMgrLUi = 0x3c040000 | (((CallMgrAddr + 0x8000) >> 16) & 0xFFFF);
   uint32_t CallMgrADDiu = 0x24840000 | ((CallMgrAddr) & 0xFFFF);
-  memcpy(ResolverMem + CallbackMgrAddrOffset, &CallMgrLUi,
-         sizeof(CallMgrLUi));
-  memcpy(ResolverMem + (CallbackMgrAddrOffset + 4), &CallMgrADDiu,
+  memcpy(ResolverMem + CallbackMgrAddrOffset, &CallMgrLUi, sizeof(CallMgrLUi));
+  memcpy(ResolverMem + CallbackMgrAddrOffset + 4, &CallMgrADDiu,
          sizeof(CallMgrADDiu));
 
   uint64_t ReentryAddr = reinterpret_cast<uint64_t>(ReentryFn);
   uint32_t ReentryLUi = 0x3c190000 | (((ReentryAddr + 0x8000) >> 16) & 0xFFFF);
   uint32_t ReentryADDiu = 0x27390000 | ((ReentryAddr) & 0xFFFF);
-  memcpy(ResolverMem + ReentryFnAddrOffset, &ReentryLUi,
-         sizeof(ReentryLUi));
-  memcpy(ResolverMem + (ReentryFnAddrOffset + 4), &ReentryADDiu,
+  memcpy(ResolverMem + ReentryFnAddrOffset, &ReentryLUi, sizeof(ReentryLUi));
+  memcpy(ResolverMem + ReentryFnAddrOffset + 4, &ReentryADDiu,
          sizeof(ReentryADDiu));
 }
 
-void OrcMips32_Base::writeTrampolines(uint8_t *TrampolineMem, void *ResolverAddr,
+void OrcMips32_Base::writeTrampolines(uint8_t *TrampolineMem,
+                                      void *ResolverAddr,
                                       unsigned NumTrampolines) {
 
   uint32_t *Trampolines = reinterpret_cast<uint32_t *>(TrampolineMem);
@@ -662,7 +653,6 @@ void OrcMips32_Base::writeTrampolines(uint8_t *TrampolineMem, void *ResolverAddr
     Trampolines[5 * I + 3] = 0x0320f809;                           // jalr $t9
     Trampolines[5 * I + 4] = 0x00000000;                           // nop
   }
-
 }
 
 Error OrcMips32_Base::emitIndirectStubsBlock(IndirectStubsInfo &StubsInfo,
@@ -769,7 +759,7 @@ void OrcMips64::writeResolverCode(uint8_t *ResolverMem, JITReentryFn ReentryFn,
       0xffb700a8,                     // 0x58: sd s7,168(sp)
       0xffb800b0,                     // 0x5c: sd t8,176(sp)
       0xffb900b8,                     // 0x60: sd t9,184(sp)
-      0xffbe00c0,                     // 0x64: sd s8,192(sp)
+      0xffbe00c0,                     // 0x64: sd fp,192(sp)
       0xffbf00c8,                     // 0x68: sd ra,200(sp)
 
       // Callback manager addr.
@@ -793,7 +783,7 @@ void OrcMips64::writeResolverCode(uint8_t *ResolverMem, JITReentryFn ReentryFn,
       0x0320f809,                     // 0xa4: jalr $t9
       0x00000000,                     // 0xa8: nop
       0xdfbf00c8,                     // 0xac: ld ra, 200(sp)
-      0xdfbe00c0,                     // 0xb0: ld s8, 192(sp)
+      0xdfbe00c0,                     // 0xb0: ld fp, 192(sp)
       0xdfb900b8,                     // 0xb4: ld t9, 184(sp)
       0xdfb800b0,                     // 0xb8: ld t8, 176(sp)
       0xdfb700a8,                     // 0xbc: ld s7, 168(sp)
@@ -819,7 +809,8 @@ void OrcMips64::writeResolverCode(uint8_t *ResolverMem, JITReentryFn ReentryFn,
       0xdfa30008,                     // 0x10c: ld v1, 8(sp)
       0x67bd00d0,                     // 0x110: daddiu $sp,$sp,208
       0x0300f825,                     // 0x114: move $ra, $t8
-      0x00400008                      // 0x118: jr $v0
+      0x03200008,                     // 0x118: jr $t9
+      0x0040c825,                     // 0x11c: move $t9, $v0
   };
 
   const unsigned ReentryFnAddrOffset = 0x8c;   // JIT re-entry fn addr lui
@@ -827,21 +818,19 @@ void OrcMips64::writeResolverCode(uint8_t *ResolverMem, JITReentryFn ReentryFn,
 
   memcpy(ResolverMem, ResolverCode, sizeof(ResolverCode));
 
-
   uint64_t CallMgrAddr = reinterpret_cast<uint64_t>(CallbackMgr);
 
   uint32_t CallMgrLUi =
       0x3c040000 | (((CallMgrAddr + 0x800080008000) >> 48) & 0xFFFF);
   uint32_t CallMgrDADDiu =
-      0x64840000 | (((CallMgrAddr + 0x80008000)  >> 32) & 0xFFFF);
+      0x64840000 | (((CallMgrAddr + 0x80008000) >> 32) & 0xFFFF);
   uint32_t CallMgrDSLL = 0x00042438;
   uint32_t CallMgrDADDiu2 =
       0x64840000 | ((((CallMgrAddr + 0x8000) >> 16) & 0xFFFF));
   uint32_t CallMgrDSLL2 = 0x00042438;
-  uint32_t CallMgrDADDiu3 = 0x64840000 | ((CallMgrAddr) & 0xFFFF);
+  uint32_t CallMgrDADDiu3 = 0x64840000 | ((CallMgrAddr)&0xFFFF);
 
-  memcpy(ResolverMem + CallbackMgrAddrOffset, &CallMgrLUi,
-         sizeof(CallMgrLUi));
+  memcpy(ResolverMem + CallbackMgrAddrOffset, &CallMgrLUi, sizeof(CallMgrLUi));
   memcpy(ResolverMem + (CallbackMgrAddrOffset + 4), &CallMgrDADDiu,
          sizeof(CallMgrDADDiu));
   memcpy(ResolverMem + (CallbackMgrAddrOffset + 8), &CallMgrDSLL,
@@ -868,10 +857,9 @@ void OrcMips64::writeResolverCode(uint8_t *ResolverMem, JITReentryFn ReentryFn,
 
   uint32_t ReentryDSLL2 = 0x0019cc38;
 
-  uint32_t ReentryDADDiu3 = 0x67390000 | ((ReentryAddr) & 0xFFFF);
+  uint32_t ReentryDADDiu3 = 0x67390000 | ((ReentryAddr)&0xFFFF);
 
-  memcpy(ResolverMem + ReentryFnAddrOffset, &ReentryLUi,
-         sizeof(ReentryLUi));
+  memcpy(ResolverMem + ReentryFnAddrOffset, &ReentryLUi, sizeof(ReentryLUi));
   memcpy(ResolverMem + (ReentryFnAddrOffset + 4), &ReentryDADDiu,
          sizeof(ReentryDADDiu));
   memcpy(ResolverMem + (ReentryFnAddrOffset + 8), &ReentryDSLL,
@@ -906,7 +894,6 @@ void OrcMips64::writeTrampolines(uint8_t *TrampolineMem, void *ResolverAddr,
     Trampolines[10 * I + 8] = 0x00000000;                            // nop
     Trampolines[10 * I + 9] = 0x00000000;                            // nop
   }
-
 }
 
 Error OrcMips64::emitIndirectStubsBlock(IndirectStubsInfo &StubsInfo,
diff --git a/lib/ExecutionEngine/Orc/OrcCBindings.cpp b/lib/ExecutionEngine/Orc/OrcCBindings.cpp
index 6dea64a6e78f52c5db248e07e8ea8ef05a5cebfd..28c8479abba448be2154b87ec39c9f7a6af90c66 100644
--- a/lib/ExecutionEngine/Orc/OrcCBindings.cpp
+++ b/lib/ExecutionEngine/Orc/OrcCBindings.cpp
@@ -1,9 +1,8 @@
 //===----------- OrcCBindings.cpp - C bindings for the Orc APIs -----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/ExecutionEngine/Orc/OrcCBindingsStack.h b/lib/ExecutionEngine/Orc/OrcCBindingsStack.h
index 817a4b89bfb0517e99a30c70589f94035124b56b..b0464380b31ea768ced0740605526b47607efe6b 100644
--- a/lib/ExecutionEngine/Orc/OrcCBindingsStack.h
+++ b/lib/ExecutionEngine/Orc/OrcCBindingsStack.h
@@ -1,9 +1,8 @@
 //===- OrcCBindingsStack.h - Orc JIT stack for C bindings -----*- C++ -*---===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/ExecutionEngine/Orc/OrcError.cpp b/lib/ExecutionEngine/Orc/OrcError.cpp
index f4102b359a6b5f8680de3002e99904797f41579b..e6e9a095319ca5e0c9bd4bd91f543bc74fc5b14e 100644
--- a/lib/ExecutionEngine/Orc/OrcError.cpp
+++ b/lib/ExecutionEngine/Orc/OrcError.cpp
@@ -1,9 +1,8 @@
 //===---------------- OrcError.cpp - Error codes for ORC ------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/ExecutionEngine/Orc/OrcMCJITReplacement.cpp b/lib/ExecutionEngine/Orc/OrcMCJITReplacement.cpp
index 617bc2fc64b500a1a84532a5eb780f1429ae0104..7c56ecc36bedd1badd0641ef5292222a025c289e 100644
--- a/lib/ExecutionEngine/Orc/OrcMCJITReplacement.cpp
+++ b/lib/ExecutionEngine/Orc/OrcMCJITReplacement.cpp
@@ -1,9 +1,8 @@
 //===-------- OrcMCJITReplacement.cpp - Orc-based MCJIT replacement -------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/ExecutionEngine/Orc/OrcMCJITReplacement.h b/lib/ExecutionEngine/Orc/OrcMCJITReplacement.h
index 36e7e83a8babfbb0774be78019c6f5147d504475..9d8694ee6b653d5b428213b2e3ff51b15e8c01e6 100644
--- a/lib/ExecutionEngine/Orc/OrcMCJITReplacement.h
+++ b/lib/ExecutionEngine/Orc/OrcMCJITReplacement.h
@@ -1,9 +1,8 @@
 //===- OrcMCJITReplacement.h - Orc based MCJIT replacement ------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/ExecutionEngine/Orc/RPCUtils.cpp b/lib/ExecutionEngine/Orc/RPCUtils.cpp
index 2a7ab5ca81807df318704b5f482c25f6522e89e2..367b3639f84119f7aa7ee59b0ee6f19bad7e4ebc 100644
--- a/lib/ExecutionEngine/Orc/RPCUtils.cpp
+++ b/lib/ExecutionEngine/Orc/RPCUtils.cpp
@@ -1,9 +1,8 @@
 //===--------------- RPCUtils.cpp - RPCUtils implementation ---------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.cpp b/lib/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.cpp
index 299d76183cd41a2542055742e441e4edfd767f4c..29c5445b9d1d57b5cd35228f8bcd9111733c7e5d 100644
--- a/lib/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.cpp
+++ b/lib/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.cpp
@@ -1,9 +1,8 @@
 //===-- RTDyldObjectLinkingLayer.cpp - RuntimeDyld backed ORC ObjectLayer -===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/ExecutionEngine/Orc/ThreadSafeModule.cpp b/lib/ExecutionEngine/Orc/ThreadSafeModule.cpp
index 9525b168fbd3fd706313d35d41c1728f1e3209d3..4cb7376758a71147e1842b61abc76354c1a0a0e5 100644
--- a/lib/ExecutionEngine/Orc/ThreadSafeModule.cpp
+++ b/lib/ExecutionEngine/Orc/ThreadSafeModule.cpp
@@ -1,10 +1,9 @@
 //===-- ThreadSafeModule.cpp - Thread safe Module, Context, and Utilities
 //h-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/ExecutionEngine/PerfJITEvents/LLVMBuild.txt b/lib/ExecutionEngine/PerfJITEvents/LLVMBuild.txt
index bba12c2da007068b320630a677282addac953a84..c83bfa852dcdfc0888ca872ccfaf42ee2a2cc033 100644
--- a/lib/ExecutionEngine/PerfJITEvents/LLVMBuild.txt
+++ b/lib/ExecutionEngine/PerfJITEvents/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/ExecutionEngine/PerfJITEvents/LLVMBuild.txt ----------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/ExecutionEngine/PerfJITEvents/PerfJITEventListener.cpp b/lib/ExecutionEngine/PerfJITEvents/PerfJITEventListener.cpp
index f195d028299865aeea3850bab1cc0e38a7f2e5f6..892a10782ae8dec41d8ab7887dcb9a1ff073b22b 100644
--- a/lib/ExecutionEngine/PerfJITEvents/PerfJITEventListener.cpp
+++ b/lib/ExecutionEngine/PerfJITEvents/PerfJITEventListener.cpp
@@ -1,9 +1,8 @@
 //===-- PerfJITEventListener.cpp - Tell Linux's perf about JITted code ----===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/ExecutionEngine/RuntimeDyld/JITSymbol.cpp b/lib/ExecutionEngine/RuntimeDyld/JITSymbol.cpp
index 0553c217c2a23573d9021098cd5e80fdce35e44d..4e2d0f422f39bc48e3eb38966807ba3807b5c4e4 100644
--- a/lib/ExecutionEngine/RuntimeDyld/JITSymbol.cpp
+++ b/lib/ExecutionEngine/RuntimeDyld/JITSymbol.cpp
@@ -1,9 +1,8 @@
 //===----------- JITSymbol.cpp - JITSymbol class implementation -----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/ExecutionEngine/RuntimeDyld/LLVMBuild.txt b/lib/ExecutionEngine/RuntimeDyld/LLVMBuild.txt
index 8bd5621918916053e8dbf3c851ed0271f9c92b10..bb7766e745e6e8bd8b3032f6485cb89817a14789 100644
--- a/lib/ExecutionEngine/RuntimeDyld/LLVMBuild.txt
+++ b/lib/ExecutionEngine/RuntimeDyld/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/ExecutionEngine/RuntimeDyld/LLVMBuild.txt ----------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/ExecutionEngine/RuntimeDyld/RTDyldMemoryManager.cpp b/lib/ExecutionEngine/RuntimeDyld/RTDyldMemoryManager.cpp
index 75d4c2b5134e12ffeb4a1023f2f59dca0b302d2c..0a19f7e6c5293c250c0fe3be78e8af375df2dc1c 100644
--- a/lib/ExecutionEngine/RuntimeDyld/RTDyldMemoryManager.cpp
+++ b/lib/ExecutionEngine/RuntimeDyld/RTDyldMemoryManager.cpp
@@ -1,9 +1,8 @@
 //===-- RTDyldMemoryManager.cpp - Memory manager for MC-JIT -----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyld.cpp b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyld.cpp
index 53cb782c55c404bb051680b3c0e831bf4c158093..589cdcbcb0d12490ff87da9c344c790482f7e895 100644
--- a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyld.cpp
+++ b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyld.cpp
@@ -1,9 +1,8 @@
 //===-- RuntimeDyld.cpp - Run-time dynamic linker for MC-JIT ----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -382,6 +381,25 @@ RuntimeDyldImpl::loadObjectImpl(const object::ObjectFile &Obj) {
       Checker->registerStubMap(Obj.getFileName(), SectionID, Stubs);
   }
 
+  // Process remaining sections
+  if (ProcessAllSections) {
+    LLVM_DEBUG(dbgs() << "Process remaining sections:\n");
+    for (section_iterator SI = Obj.section_begin(), SE = Obj.section_end();
+         SI != SE; ++SI) {
+
+      /* Ignore already loaded sections */
+      if (LocalSections.find(*SI) != LocalSections.end())
+        continue;
+
+      bool IsCode = SI->isText();
+      if (auto SectionIDOrErr =
+              findOrEmitSection(Obj, *SI, IsCode, LocalSections))
+        LLVM_DEBUG(dbgs() << "\tSectionID: " << (*SectionIDOrErr) << "\n");
+      else
+        return SectionIDOrErr.takeError();
+    }
+  }
+
   // Give the subclasses a chance to tie-up any loose ends.
   if (auto Err = finalizeLoad(Obj, LocalSections))
     return std::move(Err);
@@ -725,6 +743,11 @@ RuntimeDyldImpl::emitSection(const ObjectFile &Obj,
   bool IsReadOnly = isReadOnlyData(Section);
   uint64_t DataSize = Section.getSize();
 
+  // An alignment of 0 (at least with ELF) is identical to an alignment of 1,
+  // while being more "polite".  Other formats do not support 0-aligned sections
+  // anyway, so we should guarantee that the alignment is always at least 1.
+  Alignment = std::max(1u, Alignment);
+
   StringRef Name;
   if (auto EC = Section.getName(Name))
     return errorCodeToError(EC);
diff --git a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldCOFF.cpp b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldCOFF.cpp
index 340ddaab186da07273def89ffa7d17caa3bf5ed8..d4e3b0ba7670ab486380a8e1b677d827f0989f22 100644
--- a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldCOFF.cpp
+++ b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldCOFF.cpp
@@ -1,9 +1,8 @@
 //===-- RuntimeDyldCOFF.cpp - Run-time dynamic linker for MC-JIT -*- C++ -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldCOFF.h b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldCOFF.h
index 729a358fa0eae0116f02decd9c3c1259dd06f65a..4efd18a2e6c57046a6b240bd5e9ff264128c9e5d 100644
--- a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldCOFF.h
+++ b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldCOFF.h
@@ -1,9 +1,8 @@
 //===-- RuntimeDyldCOFF.h - Run-time dynamic linker for MC-JIT ---*- C++ -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldChecker.cpp b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldChecker.cpp
index 6eb6256080ff4ced15a3d747e76c2489cc74b7d9..642c381aaf912b5e0d5089a8aed9aee87be21aba 100644
--- a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldChecker.cpp
+++ b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldChecker.cpp
@@ -1,9 +1,8 @@
 //===--- RuntimeDyldChecker.cpp - RuntimeDyld tester framework --*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldCheckerImpl.h b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldCheckerImpl.h
index 6da1a68d06d622bb6d0fb44735c0d6401d479f5d..988fac28719790dea0207c10959496b6d32815c1 100644
--- a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldCheckerImpl.h
+++ b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldCheckerImpl.h
@@ -1,9 +1,8 @@
 //===-- RuntimeDyldCheckerImpl.h -- RuntimeDyld test framework --*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.cpp b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.cpp
index 226ee715e18bc768a1d48e572265195c9f94e2b4..f00c6d16b079564e8f553c238d1f0dcb24f50282 100644
--- a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.cpp
+++ b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.cpp
@@ -1,9 +1,8 @@
 //===-- RuntimeDyldELF.cpp - Run-time dynamic linker for MC-JIT -*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.h b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.h
index f37bd0bbaea657eccdb78aade0daef32259624e9..ec4809fd93b9780bfc9ec8e1c189234ae403d78c 100644
--- a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.h
+++ b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.h
@@ -1,9 +1,8 @@
 //===-- RuntimeDyldELF.h - Run-time dynamic linker for MC-JIT ---*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldImpl.h b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldImpl.h
index 4c650e09ac1f509cd4035a94ea5ee96c632cc9bb..a4f1c55ca3946ab997e52671c6c39fa617c7fd89 100644
--- a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldImpl.h
+++ b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldImpl.h
@@ -1,9 +1,8 @@
 //===-- RuntimeDyldImpl.h - Run-time dynamic linker for MC-JIT --*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldMachO.cpp b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldMachO.cpp
index d47fcd45be8859fa67e164d4a46c491f83f9e2ad..202c3ca1c507b9325820b607cfc2a0977210bd23 100644
--- a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldMachO.cpp
+++ b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldMachO.cpp
@@ -1,9 +1,8 @@
 //===-- RuntimeDyldMachO.cpp - Run-time dynamic linker for MC-JIT -*- C++ -*-=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldMachO.h b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldMachO.h
index d71ca4e54953bd85fee61e86786a094a2b87e6cc..650e7b79fbb8e1be62c8daef294637c3f7ac070b 100644
--- a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldMachO.h
+++ b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldMachO.h
@@ -1,9 +1,8 @@
 //===-- RuntimeDyldMachO.h - Run-time dynamic linker for MC-JIT ---*- C++ -*-=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFI386.h b/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFI386.h
index dd65051edad7bea760f15c7ea08b9dad774aac75..d43aec688610a5b6b4ebe4a4e77494932754187d 100644
--- a/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFI386.h
+++ b/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFI386.h
@@ -1,9 +1,8 @@
 //===--- RuntimeDyldCOFFI386.h --- COFF/X86_64 specific code ---*- C++ --*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -34,10 +33,10 @@ public:
 
   unsigned getStubAlignment() override { return 1; }
 
-  Expected<relocation_iterator>
+  Expected<object::relocation_iterator>
   processRelocationRef(unsigned SectionID,
-                       relocation_iterator RelI,
-                       const ObjectFile &Obj,
+                       object::relocation_iterator RelI,
+                       const object::ObjectFile &Obj,
                        ObjSectionToIDMap &ObjSectionToID,
                        StubMap &Stubs) override {
 
diff --git a/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFThumb.h b/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFThumb.h
index 8723dd0fd0ea5140fb8ba80d48dcb9d3f915cadb..24b77d5056917570a2e7f667d1964bb6ca16e7cd 100644
--- a/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFThumb.h
+++ b/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFThumb.h
@@ -1,9 +1,8 @@
 //===--- RuntimeDyldCOFFThumb.h --- COFF/Thumb specific code ---*- C++ --*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -22,9 +21,10 @@
 
 namespace llvm {
 
-static bool isThumbFunc(symbol_iterator Symbol, const ObjectFile &Obj,
-                        section_iterator Section) {
-  Expected<SymbolRef::Type> SymTypeOrErr = Symbol->getType();
+static bool isThumbFunc(object::symbol_iterator Symbol,
+                        const object::ObjectFile &Obj,
+                        object::section_iterator Section) {
+  Expected<object::SymbolRef::Type> SymTypeOrErr = Symbol->getType();
   if (!SymTypeOrErr) {
     std::string Buf;
     raw_string_ostream OS(Buf);
@@ -33,12 +33,14 @@ static bool isThumbFunc(symbol_iterator Symbol, const ObjectFile &Obj,
     report_fatal_error(Buf);
   }
 
-  if (*SymTypeOrErr != SymbolRef::ST_Function)
+  if (*SymTypeOrErr != object::SymbolRef::ST_Function)
     return false;
 
   // We check the IMAGE_SCN_MEM_16BIT flag in the section of the symbol to tell
   // if it's thumb or not
-  return cast<COFFObjectFile>(Obj).getCOFFSection(*Section)->Characteristics &
+  return cast<object::COFFObjectFile>(Obj)
+             .getCOFFSection(*Section)
+             ->Characteristics &
          COFF::IMAGE_SCN_MEM_16BIT;
 }
 
@@ -54,10 +56,10 @@ public:
 
   unsigned getStubAlignment() override { return 1; }
 
-  Expected<relocation_iterator>
+  Expected<object::relocation_iterator>
   processRelocationRef(unsigned SectionID,
-                       relocation_iterator RelI,
-                       const ObjectFile &Obj,
+                       object::relocation_iterator RelI,
+                       const object::ObjectFile &Obj,
                        ObjSectionToIDMap &ObjSectionToID,
                        StubMap &Stubs) override {
     auto Symbol = RelI->getSymbol();
diff --git a/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFX86_64.h b/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFX86_64.h
index aee5f6dc374693c86b35c93a6135804ca554c118..07a3e7bd9811baae21e4047dd0b5e7feb0d5f561 100644
--- a/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFX86_64.h
+++ b/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFX86_64.h
@@ -1,9 +1,8 @@
 //===-- RuntimeDyldCOFFX86_64.h --- COFF/X86_64 specific code ---*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -187,21 +186,21 @@ public:
     return std::make_tuple(Offset, RelType, Addend);
   }
 
-  Expected<relocation_iterator>
+  Expected<object::relocation_iterator>
   processRelocationRef(unsigned SectionID,
-                       relocation_iterator RelI,
-                       const ObjectFile &Obj,
+                       object::relocation_iterator RelI,
+                       const object::ObjectFile &Obj,
                        ObjSectionToIDMap &ObjSectionToID,
                        StubMap &Stubs) override {
     // If possible, find the symbol referred to in the relocation,
     // and the section that contains it.
-    symbol_iterator Symbol = RelI->getSymbol();
+    object::symbol_iterator Symbol = RelI->getSymbol();
     if (Symbol == Obj.symbol_end())
       report_fatal_error("Unknown symbol in relocation");
     auto SectionOrError = Symbol->getSection();
     if (!SectionOrError)
       return SectionOrError.takeError();
-    section_iterator SecI = *SectionOrError;
+    object::section_iterator SecI = *SectionOrError;
     // If there is no section, this must be an external reference.
     const bool IsExtern = SecI == Obj.section_end();
 
@@ -280,11 +279,11 @@ public:
     UnregisteredEHFrameSections.clear();
   }
 
-  Error finalizeLoad(const ObjectFile &Obj,
+  Error finalizeLoad(const object::ObjectFile &Obj,
                      ObjSectionToIDMap &SectionMap) override {
     // Look for and record the EH frame section IDs.
     for (const auto &SectionPair : SectionMap) {
-      const SectionRef &Section = SectionPair.first;
+      const object::SectionRef &Section = SectionPair.first;
       StringRef Name;
       if (auto EC = Section.getName(Name))
         return errorCodeToError(EC);
diff --git a/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldELFMips.cpp b/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldELFMips.cpp
index 3a166b40af2d5267053914d1fd0e66caecc74126..17cbe612fb43eb0a4affccc51c0cee7ea34418be 100644
--- a/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldELFMips.cpp
+++ b/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldELFMips.cpp
@@ -1,9 +1,8 @@
 //===-- RuntimeDyldELFMips.cpp ---- ELF/Mips specific code. -----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldELFMips.h b/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldELFMips.h
index f53b9e6bd75a51bf2eecb0432028f2aef2387b58..14fb36f070f8ed95060ae3489b5f9286deb16fe5 100644
--- a/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldELFMips.h
+++ b/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldELFMips.h
@@ -1,9 +1,8 @@
 //===-- RuntimeDyldELFMips.h ---- ELF/Mips specific code. -------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOAArch64.h b/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOAArch64.h
index 2a619c549cfa41b3ac3f57f283f0dc4abd10f93f..c994e3032f0ab2f3655c6fe2aa077d2433f4ee08 100644
--- a/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOAArch64.h
+++ b/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOAArch64.h
@@ -1,9 +1,8 @@
 //===-- RuntimeDyldMachOAArch64.h -- MachO/AArch64 specific code. -*- C++ -*-=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOARM.h b/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOARM.h
index ab7cd2bdae15e74e63fafb2499e2fb83b3462071..acc347a47621ced25de9f13b82553115adbb7c85 100644
--- a/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOARM.h
+++ b/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOARM.h
@@ -1,9 +1,8 @@
 //===----- RuntimeDyldMachOARM.h ---- MachO/ARM specific code. ----*- C++ -*-=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOI386.h b/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOI386.h
index d384d70b8b0f1042497a7bad613cd14c36ca907d..4d216e8e0ca0a9bbe1c2942a60097ec2fee4987a 100644
--- a/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOI386.h
+++ b/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOI386.h
@@ -1,9 +1,8 @@
 //===---- RuntimeDyldMachOI386.h ---- MachO/I386 specific code. ---*- C++ -*-=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOX86_64.h b/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOX86_64.h
index 9732ea6a0cd2ff5a6f44098bf7b12fa03bbe29ac..0700f9af0d6e749f5ea1d3fd065aaf74c20c96c3 100644
--- a/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOX86_64.h
+++ b/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOX86_64.h
@@ -1,9 +1,8 @@
 //===-- RuntimeDyldMachOX86_64.h ---- MachO/X86_64 specific code. -*- C++ -*-=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/ExecutionEngine/SectionMemoryManager.cpp b/lib/ExecutionEngine/SectionMemoryManager.cpp
index 05ab4a074e37d8d7cffecdfcd4ff26da5d6cdffc..ba277796dd53fc7c25ee1b45576ab3e8da718f1f 100644
--- a/lib/ExecutionEngine/SectionMemoryManager.cpp
+++ b/lib/ExecutionEngine/SectionMemoryManager.cpp
@@ -1,9 +1,8 @@
 //===- SectionMemoryManager.cpp - Memory manager for MCJIT/RtDyld *- C++ -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -243,7 +242,11 @@ public:
   allocateMappedMemory(SectionMemoryManager::AllocationPurpose Purpose,
                        size_t NumBytes, const sys::MemoryBlock *const NearBlock,
                        unsigned Flags, std::error_code &EC) override {
-    return sys::Memory::allocateMappedMemory(NumBytes, NearBlock, Flags, EC);
+    // allocateMappedMemory calls mmap(2). We round up a request size
+    // to page size to get extra space for free.
+    static const size_t PageSize = sys::Process::getPageSize();
+    size_t ReqBytes = (NumBytes + PageSize - 1) & ~(PageSize - 1);
+    return sys::Memory::allocateMappedMemory(ReqBytes, NearBlock, Flags, EC);
   }
 
   std::error_code protectMappedMemory(const sys::MemoryBlock &Block,
diff --git a/lib/ExecutionEngine/TargetSelect.cpp b/lib/ExecutionEngine/TargetSelect.cpp
index 9626b8d3ffa3826c0fb5fcaf8d57b985b241cabf..0d9c6cfa090876777eca11253f78fce0c8e24815 100644
--- a/lib/ExecutionEngine/TargetSelect.cpp
+++ b/lib/ExecutionEngine/TargetSelect.cpp
@@ -1,9 +1,8 @@
 //===-- TargetSelect.cpp - Target Chooser Code ----------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/FuzzMutate/FuzzerCLI.cpp b/lib/FuzzMutate/FuzzerCLI.cpp
index a70dad37dfcff7e20c260ee1ef3d39c0f875e59b..63d31c035390a065ad4520cadb53016b531763aa 100644
--- a/lib/FuzzMutate/FuzzerCLI.cpp
+++ b/lib/FuzzMutate/FuzzerCLI.cpp
@@ -1,9 +1,8 @@
 //===-- FuzzerCLI.cpp -----------------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/FuzzMutate/IRMutator.cpp b/lib/FuzzMutate/IRMutator.cpp
index 40e402cdadefb2edeaf9571395635d7e9eba5512..2fc65981f1db1014b57a9e70b931cbde4c989759 100644
--- a/lib/FuzzMutate/IRMutator.cpp
+++ b/lib/FuzzMutate/IRMutator.cpp
@@ -1,9 +1,8 @@
 //===-- IRMutator.cpp -----------------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/FuzzMutate/LLVMBuild.txt b/lib/FuzzMutate/LLVMBuild.txt
index 1c533c91d4190476cd40ef81334dc8d17e8825bd..4430be4ee065aee3b75058871fe5902a1fa3f690 100644
--- a/lib/FuzzMutate/LLVMBuild.txt
+++ b/lib/FuzzMutate/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/FuzzMutate/LLVMBuild.txt ---------------------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/FuzzMutate/OpDescriptor.cpp b/lib/FuzzMutate/OpDescriptor.cpp
index 1c5d8f606aea2de83fdaf462fb509dfbacc294c5..67d44be8b699bdf6dfae534f4ce7d61b6179070b 100644
--- a/lib/FuzzMutate/OpDescriptor.cpp
+++ b/lib/FuzzMutate/OpDescriptor.cpp
@@ -1,9 +1,8 @@
 //===-- OpDescriptor.cpp --------------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/FuzzMutate/Operations.cpp b/lib/FuzzMutate/Operations.cpp
index b842f6d64fb1d7c55c883d4080cad59768d53345..cf55d09caf7e63d7debc5a892c85782e869879b7 100644
--- a/lib/FuzzMutate/Operations.cpp
+++ b/lib/FuzzMutate/Operations.cpp
@@ -1,9 +1,8 @@
 //===-- Operations.cpp ----------------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/FuzzMutate/RandomIRBuilder.cpp b/lib/FuzzMutate/RandomIRBuilder.cpp
index 3371845355580d23d8214088cc2cdb6070f67f96..1295714839e873f596818c0087c1fb78ed73de57 100644
--- a/lib/FuzzMutate/RandomIRBuilder.cpp
+++ b/lib/FuzzMutate/RandomIRBuilder.cpp
@@ -1,9 +1,8 @@
 //===-- RandomIRBuilder.cpp -----------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -54,7 +53,8 @@ Value *RandomIRBuilder::newSource(BasicBlock &BB, ArrayRef<Instruction *> Insts,
       IP = ++I->getIterator();
       assert(IP != BB.end() && "guaranteed by the findPointer");
     }
-    auto *NewLoad = new LoadInst(Ptr, "L", &*IP);
+    auto *NewLoad = new LoadInst(
+        cast<PointerType>(Ptr->getType())->getElementType(), Ptr, "L", &*IP);
 
     // Only sample this load if it really matches the descriptor
     if (Pred.matches(Srcs, NewLoad))
diff --git a/lib/IR/AbstractCallSite.cpp b/lib/IR/AbstractCallSite.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..b7a81030f41cade00f5b38e92a0e7b824f37ebbf
--- /dev/null
+++ b/lib/IR/AbstractCallSite.cpp
@@ -0,0 +1,134 @@
+//===-- AbstractCallSite.cpp - Implementation of abstract call sites ------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements abstract call sites which unify the interface for
+// direct, indirect, and callback call sites.
+//
+// For more information see:
+// https://llvm.org/devmtg/2018-10/talk-abstracts.html#talk20
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ADT/Statistic.h"
+#include "llvm/ADT/StringSwitch.h"
+#include "llvm/IR/CallSite.h"
+#include "llvm/Support/Debug.h"
+
+using namespace llvm;
+
+#define DEBUG_TYPE "abstract-call-sites"
+
+STATISTIC(NumCallbackCallSites, "Number of callback call sites created");
+STATISTIC(NumDirectAbstractCallSites,
+          "Number of direct abstract call sites created");
+STATISTIC(NumInvalidAbstractCallSitesUnknownUse,
+          "Number of invalid abstract call sites created (unknown use)");
+STATISTIC(NumInvalidAbstractCallSitesUnknownCallee,
+          "Number of invalid abstract call sites created (unknown callee)");
+STATISTIC(NumInvalidAbstractCallSitesNoCallback,
+          "Number of invalid abstract call sites created (no callback)");
+
+/// Create an abstract call site from a use.
+AbstractCallSite::AbstractCallSite(const Use *U) : CS(U->getUser()) {
+
+  // First handle unknown users.
+  if (!CS) {
+
+    // If the use is actually in a constant cast expression which itself
+    // has only one use, we look through the constant cast expression.
+    // This happens by updating the use @p U to the use of the constant
+    // cast expression and afterwards re-initializing CS accordingly.
+    if (ConstantExpr *CE = dyn_cast<ConstantExpr>(U->getUser()))
+      if (CE->getNumUses() == 1 && CE->isCast()) {
+        U = &*CE->use_begin();
+        CS = CallSite(U->getUser());
+      }
+
+    if (!CS) {
+      NumInvalidAbstractCallSitesUnknownUse++;
+      return;
+    }
+  }
+
+  // Then handle direct or indirect calls. Thus, if U is the callee of the
+  // call site CS it is not a callback and we are done.
+  if (CS.isCallee(U)) {
+    NumDirectAbstractCallSites++;
+    return;
+  }
+
+  // If we cannot identify the broker function we cannot create a callback and
+  // invalidate the abstract call site.
+  Function *Callee = CS.getCalledFunction();
+  if (!Callee) {
+    NumInvalidAbstractCallSitesUnknownCallee++;
+    CS = CallSite();
+    return;
+  }
+
+  MDNode *CallbackMD = Callee->getMetadata(LLVMContext::MD_callback);
+  if (!CallbackMD) {
+    NumInvalidAbstractCallSitesNoCallback++;
+    CS = CallSite();
+    return;
+  }
+
+  unsigned UseIdx = CS.getArgumentNo(U);
+  MDNode *CallbackEncMD = nullptr;
+  for (const MDOperand &Op : CallbackMD->operands()) {
+    MDNode *OpMD = cast<MDNode>(Op.get());
+    auto *CBCalleeIdxAsCM = cast<ConstantAsMetadata>(OpMD->getOperand(0));
+    uint64_t CBCalleeIdx =
+        cast<ConstantInt>(CBCalleeIdxAsCM->getValue())->getZExtValue();
+    if (CBCalleeIdx != UseIdx)
+      continue;
+    CallbackEncMD = OpMD;
+    break;
+  }
+
+  if (!CallbackEncMD) {
+    NumInvalidAbstractCallSitesNoCallback++;
+    CS = CallSite();
+    return;
+  }
+
+  NumCallbackCallSites++;
+
+  assert(CallbackEncMD->getNumOperands() >= 2 && "Incomplete !callback metadata");
+
+  unsigned NumCallOperands = CS.getNumArgOperands();
+  // Skip the var-arg flag at the end when reading the metadata.
+  for (unsigned u = 0, e = CallbackEncMD->getNumOperands() - 1; u < e; u++) {
+    Metadata *OpAsM = CallbackEncMD->getOperand(u).get();
+    auto *OpAsCM = cast<ConstantAsMetadata>(OpAsM);
+    assert(OpAsCM->getType()->isIntegerTy(64) &&
+           "Malformed !callback metadata");
+
+    int64_t Idx = cast<ConstantInt>(OpAsCM->getValue())->getSExtValue();
+    assert(-1 <= Idx && Idx <= NumCallOperands &&
+           "Out-of-bounds !callback metadata index");
+
+    CI.ParameterEncoding.push_back(Idx);
+  }
+
+  if (!Callee->isVarArg())
+    return;
+
+  Metadata *VarArgFlagAsM =
+      CallbackEncMD->getOperand(CallbackEncMD->getNumOperands() - 1).get();
+  auto *VarArgFlagAsCM = cast<ConstantAsMetadata>(VarArgFlagAsM);
+  assert(VarArgFlagAsCM->getType()->isIntegerTy(1) &&
+         "Malformed !callback metadata var-arg flag");
+
+  if (VarArgFlagAsCM->getValue()->isNullValue())
+    return;
+
+  // Add all variadic arguments at the end.
+  for (unsigned u = Callee->arg_size(); u < NumCallOperands; u++)
+    CI.ParameterEncoding.push_back(u);
+}
diff --git a/lib/IR/AsmWriter.cpp b/lib/IR/AsmWriter.cpp
index 36f4f3aa8764b9b350ea45fbdf7c9f6080b47f31..af3db5186a261e47cecc5a953d00eba918429365 100644
--- a/lib/IR/AsmWriter.cpp
+++ b/lib/IR/AsmWriter.cpp
@@ -1,9 +1,8 @@
 //===- AsmWriter.cpp - Printing LLVM as an assembly file ------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -36,7 +35,6 @@
 #include "llvm/IR/Attributes.h"
 #include "llvm/IR/BasicBlock.h"
 #include "llvm/IR/CFG.h"
-#include "llvm/IR/CallSite.h"
 #include "llvm/IR/CallingConv.h"
 #include "llvm/IR/Comdat.h"
 #include "llvm/IR/Constant.h"
@@ -998,9 +996,9 @@ void SlotTracker::processFunction() {
 
       // We allow direct calls to any llvm.foo function here, because the
       // target may not be linked into the optimizer.
-      if (auto CS = ImmutableCallSite(&I)) {
+      if (const auto *Call = dyn_cast<CallBase>(&I)) {
         // Add all the call attributes to the table.
-        AttributeSet Attrs = CS.getAttributes().getFnAttributes();
+        AttributeSet Attrs = Call->getAttributes().getFnAttributes();
         if (Attrs.hasAttributes())
           CreateAttributeSetSlot(Attrs);
       }
@@ -2359,7 +2357,7 @@ public:
 
   void writeOperand(const Value *Op, bool PrintType);
   void writeParamOperand(const Value *Operand, AttributeSet Attrs);
-  void writeOperandBundles(ImmutableCallSite CS);
+  void writeOperandBundles(const CallBase *Call);
   void writeSyncScope(const LLVMContext &Context,
                       SyncScope::ID SSID);
   void writeAtomic(const LLVMContext &Context,
@@ -2510,15 +2508,15 @@ void AssemblyWriter::writeParamOperand(const Value *Operand,
   WriteAsOperandInternal(Out, Operand, &TypePrinter, &Machine, TheModule);
 }
 
-void AssemblyWriter::writeOperandBundles(ImmutableCallSite CS) {
-  if (!CS.hasOperandBundles())
+void AssemblyWriter::writeOperandBundles(const CallBase *Call) {
+  if (!Call->hasOperandBundles())
     return;
 
   Out << " [ ";
 
   bool FirstBundle = true;
-  for (unsigned i = 0, e = CS.getNumOperandBundles(); i != e; ++i) {
-    OperandBundleUse BU = CS.getOperandBundleAt(i);
+  for (unsigned i = 0, e = Call->getNumOperandBundles(); i != e; ++i) {
+    OperandBundleUse BU = Call->getOperandBundleAt(i);
 
     if (!FirstBundle)
       Out << ", ";
@@ -3838,6 +3836,51 @@ void AssemblyWriter::printInstruction(const Instruction &I) {
     writeOperand(II->getNormalDest(), true);
     Out << " unwind ";
     writeOperand(II->getUnwindDest(), true);
+  } else if (const CallBrInst *CBI = dyn_cast<CallBrInst>(&I)) {
+    Operand = CBI->getCalledValue();
+    FunctionType *FTy = CBI->getFunctionType();
+    Type *RetTy = FTy->getReturnType();
+    const AttributeList &PAL = CBI->getAttributes();
+
+    // Print the calling convention being used.
+    if (CBI->getCallingConv() != CallingConv::C) {
+      Out << " ";
+      PrintCallingConv(CBI->getCallingConv(), Out);
+    }
+
+    if (PAL.hasAttributes(AttributeList::ReturnIndex))
+      Out << ' ' << PAL.getAsString(AttributeList::ReturnIndex);
+
+    // If possible, print out the short form of the callbr instruction. We can
+    // only do this if the first argument is a pointer to a nonvararg function,
+    // and if the return type is not a pointer to a function.
+    //
+    Out << ' ';
+    TypePrinter.print(FTy->isVarArg() ? FTy : RetTy, Out);
+    Out << ' ';
+    writeOperand(Operand, false);
+    Out << '(';
+    for (unsigned op = 0, Eop = CBI->getNumArgOperands(); op < Eop; ++op) {
+      if (op)
+        Out << ", ";
+      writeParamOperand(CBI->getArgOperand(op), PAL.getParamAttributes(op));
+    }
+
+    Out << ')';
+    if (PAL.hasAttributes(AttributeList::FunctionIndex))
+      Out << " #" << Machine.getAttributeGroupSlot(PAL.getFnAttributes());
+
+    writeOperandBundles(CBI);
+
+    Out << "\n          to ";
+    writeOperand(CBI->getDefaultDest(), true);
+    Out << " [";
+    for (unsigned i = 0, e = CBI->getNumIndirectDests(); i != e; ++i) {
+      if (i != 0)
+        Out << ", ";
+      writeOperand(CBI->getIndirectDest(i), true);
+    }
+    Out << ']';
   } else if (const AllocaInst *AI = dyn_cast<AllocaInst>(&I)) {
     Out << ' ';
     if (AI->isUsedWithInAlloca())
diff --git a/lib/IR/AttributeImpl.h b/lib/IR/AttributeImpl.h
index bb0c072e47814b7a825a7ee57b1cf9afacff40c0..8ebcb04a565d1b0db7a1aa5a8d64c985810ec25b 100644
--- a/lib/IR/AttributeImpl.h
+++ b/lib/IR/AttributeImpl.h
@@ -1,9 +1,8 @@
 //===- AttributeImpl.h - Attribute Internals --------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
diff --git a/lib/IR/Attributes.cpp b/lib/IR/Attributes.cpp
index ff46debb7a9e574681eac51f06272733af1b4bcc..a474d5740fea7e381c2bf8b80c524e54670cc5e5 100644
--- a/lib/IR/Attributes.cpp
+++ b/lib/IR/Attributes.cpp
@@ -1,9 +1,8 @@
 //===- Attributes.cpp - Implement AttributesList --------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/IR/AutoUpgrade.cpp b/lib/IR/AutoUpgrade.cpp
index 39e29a2a093c3bec882b7503419ad24b2ac7c41b..8ebe55a1ea364fd93391f59a338eeb9c37026e9a 100644
--- a/lib/IR/AutoUpgrade.cpp
+++ b/lib/IR/AutoUpgrade.cpp
@@ -1,9 +1,8 @@
 //===-- AutoUpgrade.cpp - Implement auto-upgrade helper functions ---------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -77,10 +76,18 @@ static bool ShouldUpgradeX86Intrinsic(Function *F, StringRef Name) {
       Name == "addcarry.u64" || // Added in 8.0
       Name == "subborrow.u32" || // Added in 8.0
       Name == "subborrow.u64" || // Added in 8.0
+      Name.startswith("sse2.padds.") || // Added in 8.0
+      Name.startswith("sse2.psubs.") || // Added in 8.0
       Name.startswith("sse2.paddus.") || // Added in 8.0
       Name.startswith("sse2.psubus.") || // Added in 8.0
+      Name.startswith("avx2.padds.") || // Added in 8.0
+      Name.startswith("avx2.psubs.") || // Added in 8.0
       Name.startswith("avx2.paddus.") || // Added in 8.0
       Name.startswith("avx2.psubus.") || // Added in 8.0
+      Name.startswith("avx512.padds.") || // Added in 8.0
+      Name.startswith("avx512.psubs.") || // Added in 8.0
+      Name.startswith("avx512.mask.padds.") || // Added in 8.0
+      Name.startswith("avx512.mask.psubs.") || // Added in 8.0
       Name.startswith("avx512.mask.paddus.") || // Added in 8.0
       Name.startswith("avx512.mask.psubus.") || // Added in 8.0
       Name=="ssse3.pabs.b.128" || // Added in 6.0
@@ -191,14 +198,14 @@ static bool ShouldUpgradeX86Intrinsic(Function *F, StringRef Name) {
       Name.startswith("avx512.mask.pmull.") || // Added in 4.0
       Name.startswith("avx512.mask.cvtdq2pd.") || // Added in 4.0
       Name.startswith("avx512.mask.cvtudq2pd.") || // Added in 4.0
-      Name == "avx512.mask.cvtudq2ps.128" || // Added in 7.0
-      Name == "avx512.mask.cvtudq2ps.256" || // Added in 7.0
-      Name == "avx512.mask.cvtqq2pd.128" || // Added in 7.0
-      Name == "avx512.mask.cvtqq2pd.256" || // Added in 7.0
-      Name == "avx512.mask.cvtuqq2pd.128" || // Added in 7.0
-      Name == "avx512.mask.cvtuqq2pd.256" || // Added in 7.0
-      Name == "avx512.mask.cvtdq2ps.128" || // Added in 7.0
-      Name == "avx512.mask.cvtdq2ps.256" || // Added in 7.0
+      Name.startswith("avx512.mask.cvtudq2ps.") || // Added in 7.0 updated 9.0
+      Name.startswith("avx512.mask.cvtqq2pd.") || // Added in 7.0 updated 9.0
+      Name.startswith("avx512.mask.cvtuqq2pd.") || // Added in 7.0 updated 9.0
+      Name.startswith("avx512.mask.cvtdq2ps.") || // Added in 7.0 updated 9.0
+      Name == "avx512.mask.cvtqq2ps.256" || // Added in 9.0
+      Name == "avx512.mask.cvtqq2ps.512" || // Added in 9.0
+      Name == "avx512.mask.cvtuqq2ps.256" || // Added in 9.0
+      Name == "avx512.mask.cvtuqq2ps.512" || // Added in 9.0
       Name == "avx512.mask.cvtpd2dq.256" || // Added in 7.0
       Name == "avx512.mask.cvtpd2ps.256" || // Added in 7.0
       Name == "avx512.mask.cvttpd2dq.256" || // Added in 7.0
@@ -277,6 +284,12 @@ static bool ShouldUpgradeX86Intrinsic(Function *F, StringRef Name) {
       Name.startswith("avx512.mask.dbpsadbw.") || // Added in 7.0
       Name.startswith("avx512.mask.vpshld.") || // Added in 7.0
       Name.startswith("avx512.mask.vpshrd.") || // Added in 7.0
+      Name.startswith("avx512.mask.vpshldv.") || // Added in 8.0
+      Name.startswith("avx512.mask.vpshrdv.") || // Added in 8.0
+      Name.startswith("avx512.maskz.vpshldv.") || // Added in 8.0
+      Name.startswith("avx512.maskz.vpshrdv.") || // Added in 8.0
+      Name.startswith("avx512.vpshld.") || // Added in 8.0
+      Name.startswith("avx512.vpshrd.") || // Added in 8.0
       Name.startswith("avx512.mask.add.p") || // Added in 7.0. 128/256 in 4.0
       Name.startswith("avx512.mask.sub.p") || // Added in 7.0. 128/256 in 4.0
       Name.startswith("avx512.mask.mul.p") || // Added in 7.0. 128/256 in 4.0
@@ -284,12 +297,13 @@ static bool ShouldUpgradeX86Intrinsic(Function *F, StringRef Name) {
       Name.startswith("avx512.mask.max.p") || // Added in 7.0. 128/256 in 5.0
       Name.startswith("avx512.mask.min.p") || // Added in 7.0. 128/256 in 5.0
       Name.startswith("avx512.mask.fpclass.p") || // Added in 7.0
-      Name.startswith("avx512.mask.prorv.") || // Added in 7.0
-      Name.startswith("avx512.mask.pror.") || // Added in 7.0
-      Name.startswith("avx512.mask.prolv.") || // Added in 7.0
-      Name.startswith("avx512.mask.prol.") || // Added in 7.0
-      Name.startswith("avx512.mask.padds.") || // Added in 8.0
-      Name.startswith("avx512.mask.psubs.") || // Added in 8.0
+      Name.startswith("avx512.mask.vpshufbitqmb.") || // Added in 8.0
+      Name.startswith("avx512.mask.pmultishift.qb.") || // Added in 8.0
+      Name.startswith("avx512.mask.conflict.") || // Added in 9.0
+      Name == "avx512.mask.pmov.qd.256" || // Added in 9.0
+      Name == "avx512.mask.pmov.qd.512" || // Added in 9.0
+      Name == "avx512.mask.pmov.wb.256" || // Added in 9.0
+      Name == "avx512.mask.pmov.wb.512" || // Added in 9.0
       Name == "sse.cvtsi2ss" || // Added in 7.0
       Name == "sse.cvtsi642ss" || // Added in 7.0
       Name == "sse2.cvtsi2sd" || // Added in 7.0
@@ -328,6 +342,16 @@ static bool ShouldUpgradeX86Intrinsic(Function *F, StringRef Name) {
       Name.startswith("avx512.mask.load.") || // Added in 3.9
       Name.startswith("avx512.mask.expand.load.") || // Added in 7.0
       Name.startswith("avx512.mask.compress.store.") || // Added in 7.0
+      Name.startswith("avx512.mask.expand.b") || // Added in 9.0
+      Name.startswith("avx512.mask.expand.w") || // Added in 9.0
+      Name.startswith("avx512.mask.expand.d") || // Added in 9.0
+      Name.startswith("avx512.mask.expand.q") || // Added in 9.0
+      Name.startswith("avx512.mask.expand.p") || // Added in 9.0
+      Name.startswith("avx512.mask.compress.b") || // Added in 9.0
+      Name.startswith("avx512.mask.compress.w") || // Added in 9.0
+      Name.startswith("avx512.mask.compress.d") || // Added in 9.0
+      Name.startswith("avx512.mask.compress.q") || // Added in 9.0
+      Name.startswith("avx512.mask.compress.p") || // Added in 9.0
       Name == "sse42.crc32.64.8" || // Added in 3.4
       Name.startswith("avx.vbroadcast.s") || // Added in 3.5
       Name.startswith("avx512.vbroadcast.s") || // Added in 7.0
@@ -352,8 +376,14 @@ static bool ShouldUpgradeX86Intrinsic(Function *F, StringRef Name) {
       Name == "xop.vpcmov.256" || // Added in 5.0
       Name.startswith("avx512.mask.move.s") || // Added in 4.0
       Name.startswith("avx512.cvtmask2") || // Added in 5.0
-      (Name.startswith("xop.vpcom") && // Added in 3.2
-       F->arg_size() == 2) ||
+      Name.startswith("xop.vpcom") || // Added in 3.2, Updated in 9.0
+      Name.startswith("xop.vprot") || // Added in 8.0
+      Name.startswith("avx512.prol") || // Added in 8.0
+      Name.startswith("avx512.pror") || // Added in 8.0
+      Name.startswith("avx512.mask.prorv.") || // Added in 8.0
+      Name.startswith("avx512.mask.pror.") ||  // Added in 8.0
+      Name.startswith("avx512.mask.prolv.") || // Added in 8.0
+      Name.startswith("avx512.mask.prol.") ||  // Added in 8.0
       Name.startswith("avx512.ptestm") || //Added in 6.0
       Name.startswith("avx512.ptestnm") || //Added in 6.0
       Name.startswith("sse2.pavg") || // Added in 6.0
@@ -452,6 +482,11 @@ static bool UpgradeX86IntrinsicFunction(Function *F, StringRef Name,
     }
   }
 
+  if (Name == "seh.recoverfp") {
+    NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::eh_recoverfp);
+    return true;
+  }
+
   return false;
 }
 
@@ -708,7 +743,7 @@ static bool UpgradeIntrinsicFunction1(Function *F, Function *&NewFn) {
     // address space.
     if (Name.startswith("objectsize.")) {
       Type *Tys[2] = { F->getReturnType(), F->arg_begin()->getType() };
-      if (F->arg_size() == 2 ||
+      if (F->arg_size() == 2 || F->arg_size() == 3 ||
           F->getName() != Intrinsic::getName(Intrinsic::objectsize, Tys)) {
         rename(F);
         NewFn = Intrinsic::getDeclaration(F->getParent(), Intrinsic::objectsize,
@@ -924,29 +959,106 @@ static Value *UpgradeX86ALIGNIntrinsics(IRBuilder<> &Builder, Value *Op0,
   return EmitX86Select(Builder, Mask, Align, Passthru);
 }
 
+static Value *UpgradeX86VPERMT2Intrinsics(IRBuilder<> &Builder, CallInst &CI,
+                                          bool ZeroMask, bool IndexForm) {
+  Type *Ty = CI.getType();
+  unsigned VecWidth = Ty->getPrimitiveSizeInBits();
+  unsigned EltWidth = Ty->getScalarSizeInBits();
+  bool IsFloat = Ty->isFPOrFPVectorTy();
+  Intrinsic::ID IID;
+  if (VecWidth == 128 && EltWidth == 32 && IsFloat)
+    IID = Intrinsic::x86_avx512_vpermi2var_ps_128;
+  else if (VecWidth == 128 && EltWidth == 32 && !IsFloat)
+    IID = Intrinsic::x86_avx512_vpermi2var_d_128;
+  else if (VecWidth == 128 && EltWidth == 64 && IsFloat)
+    IID = Intrinsic::x86_avx512_vpermi2var_pd_128;
+  else if (VecWidth == 128 && EltWidth == 64 && !IsFloat)
+    IID = Intrinsic::x86_avx512_vpermi2var_q_128;
+  else if (VecWidth == 256 && EltWidth == 32 && IsFloat)
+    IID = Intrinsic::x86_avx512_vpermi2var_ps_256;
+  else if (VecWidth == 256 && EltWidth == 32 && !IsFloat)
+    IID = Intrinsic::x86_avx512_vpermi2var_d_256;
+  else if (VecWidth == 256 && EltWidth == 64 && IsFloat)
+    IID = Intrinsic::x86_avx512_vpermi2var_pd_256;
+  else if (VecWidth == 256 && EltWidth == 64 && !IsFloat)
+    IID = Intrinsic::x86_avx512_vpermi2var_q_256;
+  else if (VecWidth == 512 && EltWidth == 32 && IsFloat)
+    IID = Intrinsic::x86_avx512_vpermi2var_ps_512;
+  else if (VecWidth == 512 && EltWidth == 32 && !IsFloat)
+    IID = Intrinsic::x86_avx512_vpermi2var_d_512;
+  else if (VecWidth == 512 && EltWidth == 64 && IsFloat)
+    IID = Intrinsic::x86_avx512_vpermi2var_pd_512;
+  else if (VecWidth == 512 && EltWidth == 64 && !IsFloat)
+    IID = Intrinsic::x86_avx512_vpermi2var_q_512;
+  else if (VecWidth == 128 && EltWidth == 16)
+    IID = Intrinsic::x86_avx512_vpermi2var_hi_128;
+  else if (VecWidth == 256 && EltWidth == 16)
+    IID = Intrinsic::x86_avx512_vpermi2var_hi_256;
+  else if (VecWidth == 512 && EltWidth == 16)
+    IID = Intrinsic::x86_avx512_vpermi2var_hi_512;
+  else if (VecWidth == 128 && EltWidth == 8)
+    IID = Intrinsic::x86_avx512_vpermi2var_qi_128;
+  else if (VecWidth == 256 && EltWidth == 8)
+    IID = Intrinsic::x86_avx512_vpermi2var_qi_256;
+  else if (VecWidth == 512 && EltWidth == 8)
+    IID = Intrinsic::x86_avx512_vpermi2var_qi_512;
+  else
+    llvm_unreachable("Unexpected intrinsic");
+
+  Value *Args[] = { CI.getArgOperand(0) , CI.getArgOperand(1),
+                    CI.getArgOperand(2) };
+
+  // If this isn't index form we need to swap operand 0 and 1.
+  if (!IndexForm)
+    std::swap(Args[0], Args[1]);
+
+  Value *V = Builder.CreateCall(Intrinsic::getDeclaration(CI.getModule(), IID),
+                                Args);
+  Value *PassThru = ZeroMask ? ConstantAggregateZero::get(Ty)
+                             : Builder.CreateBitCast(CI.getArgOperand(1),
+                                                     Ty);
+  return EmitX86Select(Builder, CI.getArgOperand(3), V, PassThru);
+}
+
 static Value *UpgradeX86AddSubSatIntrinsics(IRBuilder<> &Builder, CallInst &CI,
-                                            bool IsAddition) {
+                                            bool IsSigned, bool IsAddition) {
+  Type *Ty = CI.getType();
   Value *Op0 = CI.getOperand(0);
   Value *Op1 = CI.getOperand(1);
 
-  // Collect vector elements and type data.
-  Type *ResultType = CI.getType();
-
-  Value *Res;
-  if (IsAddition) {
-    // ADDUS: a > (a+b) ? ~0 : (a+b)
-    // If Op0 > Add, overflow occured.
-    Value *Add = Builder.CreateAdd(Op0, Op1);
-    Value *ICmp = Builder.CreateICmp(ICmpInst::ICMP_UGT, Op0, Add);
-    Value *Max = llvm::Constant::getAllOnesValue(ResultType);
-    Res = Builder.CreateSelect(ICmp, Max, Add);
-  } else {
-    // SUBUS: max(a, b) - b
-    Value *ICmp = Builder.CreateICmp(ICmpInst::ICMP_UGT, Op0, Op1);
-    Value *Select = Builder.CreateSelect(ICmp, Op0, Op1);
-    Res = Builder.CreateSub(Select, Op1);
+  Intrinsic::ID IID =
+      IsSigned ? (IsAddition ? Intrinsic::sadd_sat : Intrinsic::ssub_sat)
+               : (IsAddition ? Intrinsic::uadd_sat : Intrinsic::usub_sat);
+  Function *Intrin = Intrinsic::getDeclaration(CI.getModule(), IID, Ty);
+  Value *Res = Builder.CreateCall(Intrin, {Op0, Op1});
+
+  if (CI.getNumArgOperands() == 4) { // For masked intrinsics.
+    Value *VecSrc = CI.getOperand(2);
+    Value *Mask = CI.getOperand(3);
+    Res = EmitX86Select(Builder, Mask, Res, VecSrc);
+  }
+  return Res;
+}
+
+static Value *upgradeX86Rotate(IRBuilder<> &Builder, CallInst &CI,
+                               bool IsRotateRight) {
+  Type *Ty = CI.getType();
+  Value *Src = CI.getArgOperand(0);
+  Value *Amt = CI.getArgOperand(1);
+
+  // Amount may be scalar immediate, in which case create a splat vector.
+  // Funnel shifts amounts are treated as modulo and types are all power-of-2 so
+  // we only care about the lowest log2 bits anyway.
+  if (Amt->getType() != Ty) {
+    unsigned NumElts = Ty->getVectorNumElements();
+    Amt = Builder.CreateIntCast(Amt, Ty->getScalarType(), false);
+    Amt = Builder.CreateVectorSplat(NumElts, Amt);
   }
 
+  Intrinsic::ID IID = IsRotateRight ? Intrinsic::fshr : Intrinsic::fshl;
+  Function *Intrin = Intrinsic::getDeclaration(CI.getModule(), IID, Ty);
+  Value *Res = Builder.CreateCall(Intrin, {Src, Src, Amt});
+
   if (CI.getNumArgOperands() == 4) { // For masked intrinsics.
     Value *VecSrc = CI.getOperand(2);
     Value *Mask = CI.getOperand(3);
@@ -955,6 +1067,79 @@ static Value *UpgradeX86AddSubSatIntrinsics(IRBuilder<> &Builder, CallInst &CI,
   return Res;
 }
 
+static Value *upgradeX86vpcom(IRBuilder<> &Builder, CallInst &CI, unsigned Imm,
+                              bool IsSigned) {
+  Type *Ty = CI.getType();
+  Value *LHS = CI.getArgOperand(0);
+  Value *RHS = CI.getArgOperand(1);
+
+  CmpInst::Predicate Pred;
+  switch (Imm) {
+  case 0x0:
+    Pred = IsSigned ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT;
+    break;
+  case 0x1:
+    Pred = IsSigned ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE;
+    break;
+  case 0x2:
+    Pred = IsSigned ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT;
+    break;
+  case 0x3:
+    Pred = IsSigned ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE;
+    break;
+  case 0x4:
+    Pred = ICmpInst::ICMP_EQ;
+    break;
+  case 0x5:
+    Pred = ICmpInst::ICMP_NE;
+    break;
+  case 0x6:
+    return Constant::getNullValue(Ty); // FALSE
+  case 0x7:
+    return Constant::getAllOnesValue(Ty); // TRUE
+  default:
+    llvm_unreachable("Unknown XOP vpcom/vpcomu predicate");
+  }
+
+  Value *Cmp = Builder.CreateICmp(Pred, LHS, RHS);
+  Value *Ext = Builder.CreateSExt(Cmp, Ty);
+  return Ext;
+}
+
+static Value *upgradeX86ConcatShift(IRBuilder<> &Builder, CallInst &CI,
+                                    bool IsShiftRight, bool ZeroMask) {
+  Type *Ty = CI.getType();
+  Value *Op0 = CI.getArgOperand(0);
+  Value *Op1 = CI.getArgOperand(1);
+  Value *Amt = CI.getArgOperand(2);
+
+  if (IsShiftRight)
+    std::swap(Op0, Op1);
+
+  // Amount may be scalar immediate, in which case create a splat vector.
+  // Funnel shifts amounts are treated as modulo and types are all power-of-2 so
+  // we only care about the lowest log2 bits anyway.
+  if (Amt->getType() != Ty) {
+    unsigned NumElts = Ty->getVectorNumElements();
+    Amt = Builder.CreateIntCast(Amt, Ty->getScalarType(), false);
+    Amt = Builder.CreateVectorSplat(NumElts, Amt);
+  }
+
+  Intrinsic::ID IID = IsShiftRight ? Intrinsic::fshr : Intrinsic::fshl;
+  Function *Intrin = Intrinsic::getDeclaration(CI.getModule(), IID, Ty);
+  Value *Res = Builder.CreateCall(Intrin, {Op0, Op1, Amt});
+
+  unsigned NumArgs = CI.getNumArgOperands();
+  if (NumArgs >= 4) { // For masked intrinsics.
+    Value *VecSrc = NumArgs == 5 ? CI.getArgOperand(3) :
+                    ZeroMask     ? ConstantAggregateZero::get(CI.getType()) :
+                                   CI.getArgOperand(0);
+    Value *Mask = CI.getOperand(NumArgs - 1);
+    Res = EmitX86Select(Builder, Mask, Res, VecSrc);
+  }
+  return Res;
+}
+
 static Value *UpgradeMaskedStore(IRBuilder<> &Builder,
                                  Value *Ptr, Value *Data, Value *Mask,
                                  bool Aligned) {
@@ -978,16 +1163,16 @@ static Value *UpgradeMaskedStore(IRBuilder<> &Builder,
 static Value *UpgradeMaskedLoad(IRBuilder<> &Builder,
                                 Value *Ptr, Value *Passthru, Value *Mask,
                                 bool Aligned) {
+  Type *ValTy = Passthru->getType();
   // Cast the pointer to the right type.
-  Ptr = Builder.CreateBitCast(Ptr,
-                             llvm::PointerType::getUnqual(Passthru->getType()));
+  Ptr = Builder.CreateBitCast(Ptr, llvm::PointerType::getUnqual(ValTy));
   unsigned Align =
     Aligned ? cast<VectorType>(Passthru->getType())->getBitWidth() / 8 : 1;
 
   // If the mask is all ones just emit a regular store.
   if (const auto *C = dyn_cast<Constant>(Mask))
     if (C->isAllOnesValue())
-      return Builder.CreateAlignedLoad(Ptr, Align);
+      return Builder.CreateAlignedLoad(ValTy, Ptr, Align);
 
   // Convert the mask from an integer type to a vector of i1.
   unsigned NumElts = Passthru->getType()->getVectorNumElements();
@@ -1321,136 +1506,28 @@ static bool upgradeAVX512MaskToSelect(StringRef Name, IRBuilder<> &Builder,
       IID = Intrinsic::x86_avx512_dbpsadbw_512;
     else
       llvm_unreachable("Unexpected intrinsic");
-  } else if (Name.startswith("vpshld.")) {
-    if (VecWidth == 128 && Name[7] == 'q')
-      IID = Intrinsic::x86_avx512_vpshld_q_128;
-    else if (VecWidth == 128 && Name[7] == 'd')
-      IID = Intrinsic::x86_avx512_vpshld_d_128;
-    else if (VecWidth == 128 && Name[7] == 'w')
-      IID = Intrinsic::x86_avx512_vpshld_w_128;
-    else if (VecWidth == 256 && Name[7] == 'q')
-      IID = Intrinsic::x86_avx512_vpshld_q_256;
-    else if (VecWidth == 256 && Name[7] == 'd')
-      IID = Intrinsic::x86_avx512_vpshld_d_256;
-    else if (VecWidth == 256 && Name[7] == 'w')
-      IID = Intrinsic::x86_avx512_vpshld_w_256;
-    else if (VecWidth == 512 && Name[7] == 'q')
-      IID = Intrinsic::x86_avx512_vpshld_q_512;
-    else if (VecWidth == 512 && Name[7] == 'd')
-      IID = Intrinsic::x86_avx512_vpshld_d_512;
-    else if (VecWidth == 512 && Name[7] == 'w')
-      IID = Intrinsic::x86_avx512_vpshld_w_512;
-    else
-      llvm_unreachable("Unexpected intrinsic");
-  } else if (Name.startswith("vpshrd.")) {
-    if (VecWidth == 128 && Name[7] == 'q')
-      IID = Intrinsic::x86_avx512_vpshrd_q_128;
-    else if (VecWidth == 128 && Name[7] == 'd')
-      IID = Intrinsic::x86_avx512_vpshrd_d_128;
-    else if (VecWidth == 128 && Name[7] == 'w')
-      IID = Intrinsic::x86_avx512_vpshrd_w_128;
-    else if (VecWidth == 256 && Name[7] == 'q')
-      IID = Intrinsic::x86_avx512_vpshrd_q_256;
-    else if (VecWidth == 256 && Name[7] == 'd')
-      IID = Intrinsic::x86_avx512_vpshrd_d_256;
-    else if (VecWidth == 256 && Name[7] == 'w')
-      IID = Intrinsic::x86_avx512_vpshrd_w_256;
-    else if (VecWidth == 512 && Name[7] == 'q')
-      IID = Intrinsic::x86_avx512_vpshrd_q_512;
-    else if (VecWidth == 512 && Name[7] == 'd')
-      IID = Intrinsic::x86_avx512_vpshrd_d_512;
-    else if (VecWidth == 512 && Name[7] == 'w')
-      IID = Intrinsic::x86_avx512_vpshrd_w_512;
-    else
-      llvm_unreachable("Unexpected intrinsic");
-  } else if (Name.startswith("prorv.")) {
-    if (VecWidth == 128 && EltWidth == 32)
-      IID = Intrinsic::x86_avx512_prorv_d_128;
-    else if (VecWidth == 256 && EltWidth == 32)
-      IID = Intrinsic::x86_avx512_prorv_d_256;
-    else if (VecWidth == 512 && EltWidth == 32)
-      IID = Intrinsic::x86_avx512_prorv_d_512;
-    else if (VecWidth == 128 && EltWidth == 64)
-      IID = Intrinsic::x86_avx512_prorv_q_128;
-    else if (VecWidth == 256 && EltWidth == 64)
-      IID = Intrinsic::x86_avx512_prorv_q_256;
-    else if (VecWidth == 512 && EltWidth == 64)
-      IID = Intrinsic::x86_avx512_prorv_q_512;
-    else
-      llvm_unreachable("Unexpected intrinsic");
-  } else if (Name.startswith("prolv.")) {
-    if (VecWidth == 128 && EltWidth == 32)
-      IID = Intrinsic::x86_avx512_prolv_d_128;
-    else if (VecWidth == 256 && EltWidth == 32)
-      IID = Intrinsic::x86_avx512_prolv_d_256;
-    else if (VecWidth == 512 && EltWidth == 32)
-      IID = Intrinsic::x86_avx512_prolv_d_512;
-    else if (VecWidth == 128 && EltWidth == 64)
-      IID = Intrinsic::x86_avx512_prolv_q_128;
-    else if (VecWidth == 256 && EltWidth == 64)
-      IID = Intrinsic::x86_avx512_prolv_q_256;
-    else if (VecWidth == 512 && EltWidth == 64)
-      IID = Intrinsic::x86_avx512_prolv_q_512;
-    else
-      llvm_unreachable("Unexpected intrinsic");
-  } else if (Name.startswith("pror.")) {
-    if (VecWidth == 128 && EltWidth == 32)
-      IID = Intrinsic::x86_avx512_pror_d_128;
-    else if (VecWidth == 256 && EltWidth == 32)
-      IID = Intrinsic::x86_avx512_pror_d_256;
-    else if (VecWidth == 512 && EltWidth == 32)
-      IID = Intrinsic::x86_avx512_pror_d_512;
-    else if (VecWidth == 128 && EltWidth == 64)
-      IID = Intrinsic::x86_avx512_pror_q_128;
-    else if (VecWidth == 256 && EltWidth == 64)
-      IID = Intrinsic::x86_avx512_pror_q_256;
-    else if (VecWidth == 512 && EltWidth == 64)
-      IID = Intrinsic::x86_avx512_pror_q_512;
-    else
-      llvm_unreachable("Unexpected intrinsic");
-  } else if (Name.startswith("prol.")) {
-    if (VecWidth == 128 && EltWidth == 32)
-      IID = Intrinsic::x86_avx512_prol_d_128;
-    else if (VecWidth == 256 && EltWidth == 32)
-      IID = Intrinsic::x86_avx512_prol_d_256;
-    else if (VecWidth == 512 && EltWidth == 32)
-      IID = Intrinsic::x86_avx512_prol_d_512;
-    else if (VecWidth == 128 && EltWidth == 64)
-      IID = Intrinsic::x86_avx512_prol_q_128;
-    else if (VecWidth == 256 && EltWidth == 64)
-      IID = Intrinsic::x86_avx512_prol_q_256;
-    else if (VecWidth == 512 && EltWidth == 64)
-      IID = Intrinsic::x86_avx512_prol_q_512;
-    else
-      llvm_unreachable("Unexpected intrinsic");
-  } else if (Name.startswith("padds.")) {
-    if (VecWidth == 128 && EltWidth == 8)
-      IID = Intrinsic::x86_sse2_padds_b;
-    else if (VecWidth == 256 && EltWidth == 8)
-      IID = Intrinsic::x86_avx2_padds_b;
-    else if (VecWidth == 512 && EltWidth == 8)
-      IID = Intrinsic::x86_avx512_padds_b_512;
-    else if (VecWidth == 128 && EltWidth == 16)
-      IID = Intrinsic::x86_sse2_padds_w;
-    else if (VecWidth == 256 && EltWidth == 16)
-      IID = Intrinsic::x86_avx2_padds_w;
-    else if (VecWidth == 512 && EltWidth == 16)
-      IID = Intrinsic::x86_avx512_padds_w_512;
+  } else if (Name.startswith("pmultishift.qb.")) {
+    if (VecWidth == 128)
+      IID = Intrinsic::x86_avx512_pmultishift_qb_128;
+    else if (VecWidth == 256)
+      IID = Intrinsic::x86_avx512_pmultishift_qb_256;
+    else if (VecWidth == 512)
+      IID = Intrinsic::x86_avx512_pmultishift_qb_512;
     else
       llvm_unreachable("Unexpected intrinsic");
-  } else if (Name.startswith("psubs.")) {
-    if (VecWidth == 128 && EltWidth == 8)
-      IID = Intrinsic::x86_sse2_psubs_b;
-    else if (VecWidth == 256 && EltWidth == 8)
-      IID = Intrinsic::x86_avx2_psubs_b;
-    else if (VecWidth == 512 && EltWidth == 8)
-      IID = Intrinsic::x86_avx512_psubs_b_512;
-    else if (VecWidth == 128 && EltWidth == 16)
-      IID = Intrinsic::x86_sse2_psubs_w;
-    else if (VecWidth == 256 && EltWidth == 16)
-      IID = Intrinsic::x86_avx2_psubs_w;
-    else if (VecWidth == 512 && EltWidth == 16)
-      IID = Intrinsic::x86_avx512_psubs_w_512;
+  } else if (Name.startswith("conflict.")) {
+    if (Name[9] == 'd' && VecWidth == 128)
+      IID = Intrinsic::x86_avx512_conflict_d_128;
+    else if (Name[9] == 'd' && VecWidth == 256)
+      IID = Intrinsic::x86_avx512_conflict_d_256;
+    else if (Name[9] == 'd' && VecWidth == 512)
+      IID = Intrinsic::x86_avx512_conflict_d_512;
+    else if (Name[9] == 'q' && VecWidth == 128)
+      IID = Intrinsic::x86_avx512_conflict_q_128;
+    else if (Name[9] == 'q' && VecWidth == 256)
+      IID = Intrinsic::x86_avx512_conflict_q_256;
+    else if (Name[9] == 'q' && VecWidth == 512)
+      IID = Intrinsic::x86_avx512_conflict_q_512;
     else
       llvm_unreachable("Unexpected intrinsic");
   } else
@@ -1764,6 +1841,20 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) {
       // "avx512.mask.pcmpeq." or "avx512.mask.pcmpgt."
       bool CmpEq = Name[16] == 'e';
       Rep = upgradeMaskedCompare(Builder, *CI, CmpEq ? 0 : 6, true);
+    } else if (IsX86 && Name.startswith("avx512.mask.vpshufbitqmb.")) {
+      Type *OpTy = CI->getArgOperand(0)->getType();
+      unsigned VecWidth = OpTy->getPrimitiveSizeInBits();
+      Intrinsic::ID IID;
+      switch (VecWidth) {
+      default: llvm_unreachable("Unexpected intrinsic");
+      case 128: IID = Intrinsic::x86_avx512_vpshufbitqmb_128; break;
+      case 256: IID = Intrinsic::x86_avx512_vpshufbitqmb_256; break;
+      case 512: IID = Intrinsic::x86_avx512_vpshufbitqmb_512; break;
+      }
+
+      Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),
+                               { CI->getOperand(0), CI->getArgOperand(1) });
+      Rep = ApplyX86MaskOn1BitsVec(Builder, Rep, CI->getArgOperand(2));
     } else if (IsX86 && Name.startswith("avx512.mask.fpclass.p")) {
       Type *OpTy = CI->getArgOperand(0)->getType();
       unsigned VecWidth = OpTy->getPrimitiveSizeInBits();
@@ -1894,38 +1985,47 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) {
                          Name == "avx.cvtdq2.ps.256" ||
                          Name.startswith("avx512.mask.cvtdq2pd.") ||
                          Name.startswith("avx512.mask.cvtudq2pd.") ||
-                         Name == "avx512.mask.cvtdq2ps.128" ||
-                         Name == "avx512.mask.cvtdq2ps.256" ||
-                         Name == "avx512.mask.cvtudq2ps.128" ||
-                         Name == "avx512.mask.cvtudq2ps.256" ||
-                         Name == "avx512.mask.cvtqq2pd.128" ||
-                         Name == "avx512.mask.cvtqq2pd.256" ||
-                         Name == "avx512.mask.cvtuqq2pd.128" ||
-                         Name == "avx512.mask.cvtuqq2pd.256" ||
+                         Name.startswith("avx512.mask.cvtdq2ps.") ||
+                         Name.startswith("avx512.mask.cvtudq2ps.") ||
+                         Name.startswith("avx512.mask.cvtqq2pd.") ||
+                         Name.startswith("avx512.mask.cvtuqq2pd.") ||
+                         Name == "avx512.mask.cvtqq2ps.256" ||
+                         Name == "avx512.mask.cvtqq2ps.512" ||
+                         Name == "avx512.mask.cvtuqq2ps.256" ||
+                         Name == "avx512.mask.cvtuqq2ps.512" ||
                          Name == "sse2.cvtps2pd" ||
                          Name == "avx.cvt.ps2.pd.256" ||
                          Name == "avx512.mask.cvtps2pd.128" ||
                          Name == "avx512.mask.cvtps2pd.256")) {
       Type *DstTy = CI->getType();
       Rep = CI->getArgOperand(0);
+      Type *SrcTy = Rep->getType();
 
       unsigned NumDstElts = DstTy->getVectorNumElements();
-      if (NumDstElts < Rep->getType()->getVectorNumElements()) {
+      if (NumDstElts < SrcTy->getVectorNumElements()) {
         assert(NumDstElts == 2 && "Unexpected vector size");
         uint32_t ShuffleMask[2] = { 0, 1 };
         Rep = Builder.CreateShuffleVector(Rep, Rep, ShuffleMask);
       }
 
-      bool IsPS2PD = (StringRef::npos != Name.find("ps2"));
+      bool IsPS2PD = SrcTy->getVectorElementType()->isFloatTy();
       bool IsUnsigned = (StringRef::npos != Name.find("cvtu"));
       if (IsPS2PD)
         Rep = Builder.CreateFPExt(Rep, DstTy, "cvtps2pd");
-      else if (IsUnsigned)
-        Rep = Builder.CreateUIToFP(Rep, DstTy, "cvt");
-      else
-        Rep = Builder.CreateSIToFP(Rep, DstTy, "cvt");
+      else if (CI->getNumArgOperands() == 4 &&
+               (!isa<ConstantInt>(CI->getArgOperand(3)) ||
+                cast<ConstantInt>(CI->getArgOperand(3))->getZExtValue() != 4)) {
+        Intrinsic::ID IID = IsUnsigned ? Intrinsic::x86_avx512_uitofp_round
+                                       : Intrinsic::x86_avx512_sitofp_round;
+        Function *F = Intrinsic::getDeclaration(CI->getModule(), IID,
+                                                { DstTy, SrcTy });
+        Rep = Builder.CreateCall(F, { Rep, CI->getArgOperand(3) });
+      } else {
+        Rep = IsUnsigned ? Builder.CreateUIToFP(Rep, DstTy, "cvt")
+                         : Builder.CreateSIToFP(Rep, DstTy, "cvt");
+      }
 
-      if (CI->getNumArgOperands() == 3)
+      if (CI->getNumArgOperands() >= 3)
         Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,
                             CI->getArgOperand(1));
     } else if (IsX86 && (Name.startswith("avx512.mask.loadu."))) {
@@ -1966,58 +2066,79 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) {
                                                 Intrinsic::masked_compressstore,
                                                 ResultTy);
       Rep = Builder.CreateCall(CSt, { CI->getArgOperand(1), Ptr, MaskVec });
+    } else if (IsX86 && (Name.startswith("avx512.mask.compress.") ||
+                         Name.startswith("avx512.mask.expand."))) {
+      Type *ResultTy = CI->getType();
+
+      Value *MaskVec = getX86MaskVec(Builder, CI->getArgOperand(2),
+                                     ResultTy->getVectorNumElements());
+
+      bool IsCompress = Name[12] == 'c';
+      Intrinsic::ID IID = IsCompress ? Intrinsic::x86_avx512_mask_compress
+                                     : Intrinsic::x86_avx512_mask_expand;
+      Function *Intr = Intrinsic::getDeclaration(F->getParent(), IID, ResultTy);
+      Rep = Builder.CreateCall(Intr, { CI->getOperand(0), CI->getOperand(1),
+                                       MaskVec });
     } else if (IsX86 && Name.startswith("xop.vpcom")) {
-      Intrinsic::ID intID;
-      if (Name.endswith("ub"))
-        intID = Intrinsic::x86_xop_vpcomub;
-      else if (Name.endswith("uw"))
-        intID = Intrinsic::x86_xop_vpcomuw;
-      else if (Name.endswith("ud"))
-        intID = Intrinsic::x86_xop_vpcomud;
-      else if (Name.endswith("uq"))
-        intID = Intrinsic::x86_xop_vpcomuq;
-      else if (Name.endswith("b"))
-        intID = Intrinsic::x86_xop_vpcomb;
-      else if (Name.endswith("w"))
-        intID = Intrinsic::x86_xop_vpcomw;
-      else if (Name.endswith("d"))
-        intID = Intrinsic::x86_xop_vpcomd;
-      else if (Name.endswith("q"))
-        intID = Intrinsic::x86_xop_vpcomq;
+      bool IsSigned;
+      if (Name.endswith("ub") || Name.endswith("uw") || Name.endswith("ud") ||
+          Name.endswith("uq"))
+        IsSigned = false;
+      else if (Name.endswith("b") || Name.endswith("w") || Name.endswith("d") ||
+               Name.endswith("q"))
+        IsSigned = true;
       else
         llvm_unreachable("Unknown suffix");
 
-      Name = Name.substr(9); // strip off "xop.vpcom"
       unsigned Imm;
-      if (Name.startswith("lt"))
-        Imm = 0;
-      else if (Name.startswith("le"))
-        Imm = 1;
-      else if (Name.startswith("gt"))
-        Imm = 2;
-      else if (Name.startswith("ge"))
-        Imm = 3;
-      else if (Name.startswith("eq"))
-        Imm = 4;
-      else if (Name.startswith("ne"))
-        Imm = 5;
-      else if (Name.startswith("false"))
-        Imm = 6;
-      else if (Name.startswith("true"))
-        Imm = 7;
-      else
-        llvm_unreachable("Unknown condition");
+      if (CI->getNumArgOperands() == 3) {
+        Imm = cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
+      } else {
+        Name = Name.substr(9); // strip off "xop.vpcom"
+        if (Name.startswith("lt"))
+          Imm = 0;
+        else if (Name.startswith("le"))
+          Imm = 1;
+        else if (Name.startswith("gt"))
+          Imm = 2;
+        else if (Name.startswith("ge"))
+          Imm = 3;
+        else if (Name.startswith("eq"))
+          Imm = 4;
+        else if (Name.startswith("ne"))
+          Imm = 5;
+        else if (Name.startswith("false"))
+          Imm = 6;
+        else if (Name.startswith("true"))
+          Imm = 7;
+        else
+          llvm_unreachable("Unknown condition");
+      }
 
-      Function *VPCOM = Intrinsic::getDeclaration(F->getParent(), intID);
-      Rep =
-          Builder.CreateCall(VPCOM, {CI->getArgOperand(0), CI->getArgOperand(1),
-                                     Builder.getInt8(Imm)});
+      Rep = upgradeX86vpcom(Builder, *CI, Imm, IsSigned);
     } else if (IsX86 && Name.startswith("xop.vpcmov")) {
       Value *Sel = CI->getArgOperand(2);
       Value *NotSel = Builder.CreateNot(Sel);
       Value *Sel0 = Builder.CreateAnd(CI->getArgOperand(0), Sel);
       Value *Sel1 = Builder.CreateAnd(CI->getArgOperand(1), NotSel);
       Rep = Builder.CreateOr(Sel0, Sel1);
+    } else if (IsX86 && (Name.startswith("xop.vprot") ||
+                         Name.startswith("avx512.prol") ||
+                         Name.startswith("avx512.mask.prol"))) {
+      Rep = upgradeX86Rotate(Builder, *CI, false);
+    } else if (IsX86 && (Name.startswith("avx512.pror") ||
+                         Name.startswith("avx512.mask.pror"))) {
+      Rep = upgradeX86Rotate(Builder, *CI, true);
+    } else if (IsX86 && (Name.startswith("avx512.vpshld.") ||
+                         Name.startswith("avx512.mask.vpshld") ||
+                         Name.startswith("avx512.maskz.vpshld"))) {
+      bool ZeroMask = Name[11] == 'z';
+      Rep = upgradeX86ConcatShift(Builder, *CI, false, ZeroMask);
+    } else if (IsX86 && (Name.startswith("avx512.vpshrd.") ||
+                         Name.startswith("avx512.mask.vpshrd") ||
+                         Name.startswith("avx512.maskz.vpshrd"))) {
+      bool ZeroMask = Name[11] == 'z';
+      Rep = upgradeX86ConcatShift(Builder, *CI, true, ZeroMask);
     } else if (IsX86 && Name == "sse42.crc32.64.8") {
       Function *CRC32 = Intrinsic::getDeclaration(F->getParent(),
                                                Intrinsic::x86_sse42_crc32_32_8);
@@ -2063,6 +2184,14 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) {
       if (CI->getNumArgOperands() == 3)
         Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,
                             CI->getArgOperand(1));
+    } else if (Name == "avx512.mask.pmov.qd.256" ||
+               Name == "avx512.mask.pmov.qd.512" ||
+               Name == "avx512.mask.pmov.wb.256" ||
+               Name == "avx512.mask.pmov.wb.512") {
+      Type *Ty = CI->getArgOperand(1)->getType();
+      Rep = Builder.CreateTrunc(CI->getArgOperand(0), Ty);
+      Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,
+                          CI->getArgOperand(1));
     } else if (IsX86 && (Name.startswith("avx.vbroadcastf128") ||
                          Name == "avx2.vbroadcasti128")) {
       // Replace vbroadcastf128/vbroadcasti128 with a vector load+shuffle.
@@ -2071,7 +2200,7 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) {
       Type *VT = VectorType::get(EltTy, NumSrcElts);
       Value *Op = Builder.CreatePointerCast(CI->getArgOperand(0),
                                             PointerType::getUnqual(VT));
-      Value *Load = Builder.CreateAlignedLoad(Op, 1);
+      Value *Load = Builder.CreateAlignedLoad(VT, Op, 1);
       if (NumSrcElts == 2)
         Rep = Builder.CreateShuffleVector(Load, UndefValue::get(Load->getType()),
                                           { 0, 1, 0, 1 });
@@ -2129,6 +2258,16 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) {
       if (CI->getNumArgOperands() == 3)
         Rep = EmitX86Select(Builder, CI->getArgOperand(2), Rep,
                             CI->getArgOperand(1));
+    } else if (IsX86 && (Name.startswith("sse2.padds.") ||
+                         Name.startswith("sse2.psubs.") ||
+                         Name.startswith("avx2.padds.") ||
+                         Name.startswith("avx2.psubs.") ||
+                         Name.startswith("avx512.padds.") ||
+                         Name.startswith("avx512.psubs.") ||
+                         Name.startswith("avx512.mask.padds.") ||
+                         Name.startswith("avx512.mask.psubs."))) {
+      bool IsAdd = Name.contains(".padds");
+      Rep = UpgradeX86AddSubSatIntrinsics(Builder, *CI, true, IsAdd);
     } else if (IsX86 && (Name.startswith("sse2.paddus.") ||
                          Name.startswith("sse2.psubus.") ||
                          Name.startswith("avx2.paddus.") ||
@@ -2136,7 +2275,7 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) {
                          Name.startswith("avx512.mask.paddus.") ||
                          Name.startswith("avx512.mask.psubus."))) {
       bool IsAdd = Name.contains(".paddus");
-      Rep = UpgradeX86AddSubSatIntrinsics(Builder, *CI, IsAdd);
+      Rep = UpgradeX86AddSubSatIntrinsics(Builder, *CI, false, IsAdd);
     } else if (IsX86 && Name.startswith("avx512.mask.palignr.")) {
       Rep = UpgradeX86ALIGNIntrinsics(Builder, CI->getArgOperand(0),
                                       CI->getArgOperand(1),
@@ -2454,24 +2593,8 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) {
 
       Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
                           CI->getArgOperand(2));
-    } else if (IsX86 && Name.startswith("avx512.mask.pand.")) {
-      Rep = Builder.CreateAnd(CI->getArgOperand(0), CI->getArgOperand(1));
-      Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
-                          CI->getArgOperand(2));
-    } else if (IsX86 && Name.startswith("avx512.mask.pandn.")) {
-      Rep = Builder.CreateAnd(Builder.CreateNot(CI->getArgOperand(0)),
-                              CI->getArgOperand(1));
-      Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
-                          CI->getArgOperand(2));
-    } else if (IsX86 && Name.startswith("avx512.mask.por.")) {
-      Rep = Builder.CreateOr(CI->getArgOperand(0), CI->getArgOperand(1));
-      Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
-                          CI->getArgOperand(2));
-    } else if (IsX86 && Name.startswith("avx512.mask.pxor.")) {
-      Rep = Builder.CreateXor(CI->getArgOperand(0), CI->getArgOperand(1));
-      Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
-                          CI->getArgOperand(2));
-    } else if (IsX86 && Name.startswith("avx512.mask.and.")) {
+    } else if (IsX86 && (Name.startswith("avx512.mask.and.") ||
+                         Name.startswith("avx512.mask.pand."))) {
       VectorType *FTy = cast<VectorType>(CI->getType());
       VectorType *ITy = VectorType::getInteger(FTy);
       Rep = Builder.CreateAnd(Builder.CreateBitCast(CI->getArgOperand(0), ITy),
@@ -2479,7 +2602,8 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) {
       Rep = Builder.CreateBitCast(Rep, FTy);
       Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
                           CI->getArgOperand(2));
-    } else if (IsX86 && Name.startswith("avx512.mask.andn.")) {
+    } else if (IsX86 && (Name.startswith("avx512.mask.andn.") ||
+                         Name.startswith("avx512.mask.pandn."))) {
       VectorType *FTy = cast<VectorType>(CI->getType());
       VectorType *ITy = VectorType::getInteger(FTy);
       Rep = Builder.CreateNot(Builder.CreateBitCast(CI->getArgOperand(0), ITy));
@@ -2488,7 +2612,8 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) {
       Rep = Builder.CreateBitCast(Rep, FTy);
       Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
                           CI->getArgOperand(2));
-    } else if (IsX86 && Name.startswith("avx512.mask.or.")) {
+    } else if (IsX86 && (Name.startswith("avx512.mask.or.") ||
+                         Name.startswith("avx512.mask.por."))) {
       VectorType *FTy = cast<VectorType>(CI->getType());
       VectorType *ITy = VectorType::getInteger(FTy);
       Rep = Builder.CreateOr(Builder.CreateBitCast(CI->getArgOperand(0), ITy),
@@ -2496,7 +2621,8 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) {
       Rep = Builder.CreateBitCast(Rep, FTy);
       Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
                           CI->getArgOperand(2));
-    } else if (IsX86 && Name.startswith("avx512.mask.xor.")) {
+    } else if (IsX86 && (Name.startswith("avx512.mask.xor.") ||
+                         Name.startswith("avx512.mask.pxor."))) {
       VectorType *FTy = cast<VectorType>(CI->getType());
       VectorType *ITy = VectorType::getInteger(FTy);
       Rep = Builder.CreateXor(Builder.CreateBitCast(CI->getArgOperand(0), ITy),
@@ -2580,26 +2706,16 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) {
       }
       Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
                           CI->getArgOperand(2));
-    } else if (IsX86 && Name.startswith("avx512.mask.max.p") &&
+    } else if (IsX86 && (Name.startswith("avx512.mask.max.p") ||
+                         Name.startswith("avx512.mask.min.p")) &&
                Name.drop_front(18) == ".512") {
-      Intrinsic::ID IID;
-      if (Name[17] == 's')
-        IID = Intrinsic::x86_avx512_max_ps_512;
-      else
-        IID = Intrinsic::x86_avx512_max_pd_512;
-
-      Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),
-                               { CI->getArgOperand(0), CI->getArgOperand(1),
-                                 CI->getArgOperand(4) });
-      Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep,
-                          CI->getArgOperand(2));
-    } else if (IsX86 && Name.startswith("avx512.mask.min.p") &&
-               Name.drop_front(18) == ".512") {
-      Intrinsic::ID IID;
-      if (Name[17] == 's')
-        IID = Intrinsic::x86_avx512_min_ps_512;
-      else
-        IID = Intrinsic::x86_avx512_min_pd_512;
+      bool IsDouble = Name[17] == 'd';
+      bool IsMin = Name[13] == 'i';
+      static const Intrinsic::ID MinMaxTbl[2][2] = {
+        { Intrinsic::x86_avx512_max_ps_512, Intrinsic::x86_avx512_max_pd_512 },
+        { Intrinsic::x86_avx512_min_ps_512, Intrinsic::x86_avx512_min_pd_512 }
+      };
+      Intrinsic::ID IID = MinMaxTbl[IsMin][IsDouble];
 
       Rep = Builder.CreateCall(Intrinsic::getDeclaration(F->getParent(), IID),
                                { CI->getArgOperand(0), CI->getArgOperand(1),
@@ -2830,7 +2946,7 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) {
       // Convert the type of the pointer to a pointer to the stored type.
       Value *BC =
           Builder.CreateBitCast(Ptr, PointerType::getUnqual(VTy), "cast");
-      LoadInst *LI = Builder.CreateAlignedLoad(BC, VTy->getBitWidth() / 8);
+      LoadInst *LI = Builder.CreateAlignedLoad(VTy, BC, VTy->getBitWidth() / 8);
       LI->setMetadata(M->getMDKindID("nontemporal"), Node);
       Rep = LI;
     } else if (IsX86 &&
@@ -3143,62 +3259,7 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) {
                          Name.startswith("avx512.maskz.vpermt2var."))) {
       bool ZeroMask = Name[11] == 'z';
       bool IndexForm = Name[17] == 'i';
-      unsigned VecWidth = CI->getType()->getPrimitiveSizeInBits();
-      unsigned EltWidth = CI->getType()->getScalarSizeInBits();
-      bool IsFloat = CI->getType()->isFPOrFPVectorTy();
-      Intrinsic::ID IID;
-      if (VecWidth == 128 && EltWidth == 32 && IsFloat)
-        IID = Intrinsic::x86_avx512_vpermi2var_ps_128;
-      else if (VecWidth == 128 && EltWidth == 32 && !IsFloat)
-        IID = Intrinsic::x86_avx512_vpermi2var_d_128;
-      else if (VecWidth == 128 && EltWidth == 64 && IsFloat)
-        IID = Intrinsic::x86_avx512_vpermi2var_pd_128;
-      else if (VecWidth == 128 && EltWidth == 64 && !IsFloat)
-        IID = Intrinsic::x86_avx512_vpermi2var_q_128;
-      else if (VecWidth == 256 && EltWidth == 32 && IsFloat)
-        IID = Intrinsic::x86_avx512_vpermi2var_ps_256;
-      else if (VecWidth == 256 && EltWidth == 32 && !IsFloat)
-        IID = Intrinsic::x86_avx512_vpermi2var_d_256;
-      else if (VecWidth == 256 && EltWidth == 64 && IsFloat)
-        IID = Intrinsic::x86_avx512_vpermi2var_pd_256;
-      else if (VecWidth == 256 && EltWidth == 64 && !IsFloat)
-        IID = Intrinsic::x86_avx512_vpermi2var_q_256;
-      else if (VecWidth == 512 && EltWidth == 32 && IsFloat)
-        IID = Intrinsic::x86_avx512_vpermi2var_ps_512;
-      else if (VecWidth == 512 && EltWidth == 32 && !IsFloat)
-        IID = Intrinsic::x86_avx512_vpermi2var_d_512;
-      else if (VecWidth == 512 && EltWidth == 64 && IsFloat)
-        IID = Intrinsic::x86_avx512_vpermi2var_pd_512;
-      else if (VecWidth == 512 && EltWidth == 64 && !IsFloat)
-        IID = Intrinsic::x86_avx512_vpermi2var_q_512;
-      else if (VecWidth == 128 && EltWidth == 16)
-        IID = Intrinsic::x86_avx512_vpermi2var_hi_128;
-      else if (VecWidth == 256 && EltWidth == 16)
-        IID = Intrinsic::x86_avx512_vpermi2var_hi_256;
-      else if (VecWidth == 512 && EltWidth == 16)
-        IID = Intrinsic::x86_avx512_vpermi2var_hi_512;
-      else if (VecWidth == 128 && EltWidth == 8)
-        IID = Intrinsic::x86_avx512_vpermi2var_qi_128;
-      else if (VecWidth == 256 && EltWidth == 8)
-        IID = Intrinsic::x86_avx512_vpermi2var_qi_256;
-      else if (VecWidth == 512 && EltWidth == 8)
-        IID = Intrinsic::x86_avx512_vpermi2var_qi_512;
-      else
-        llvm_unreachable("Unexpected intrinsic");
-
-      Value *Args[] = { CI->getArgOperand(0) , CI->getArgOperand(1),
-                        CI->getArgOperand(2) };
-
-      // If this isn't index form we need to swap operand 0 and 1.
-      if (!IndexForm)
-        std::swap(Args[0], Args[1]);
-
-      Rep = Builder.CreateCall(Intrinsic::getDeclaration(CI->getModule(), IID),
-                               Args);
-      Value *PassThru = ZeroMask ? ConstantAggregateZero::get(CI->getType())
-                                 : Builder.CreateBitCast(CI->getArgOperand(1),
-                                                         CI->getType());
-      Rep = EmitX86Select(Builder, CI->getArgOperand(3), Rep, PassThru);
+      Rep = UpgradeX86VPERMT2Intrinsics(Builder, *CI, ZeroMask, IndexForm);
     } else if (IsX86 && (Name.startswith("avx512.mask.vpdpbusd.") ||
                          Name.startswith("avx512.maskz.vpdpbusd.") ||
                          Name.startswith("avx512.mask.vpdpbusds.") ||
@@ -3399,8 +3460,10 @@ void llvm::UpgradeIntrinsicCall(CallInst *CI, Function *NewFn) {
     Value *NullIsUnknownSize = CI->getNumArgOperands() == 2
                                    ? Builder.getFalse()
                                    : CI->getArgOperand(2);
+    Value *Dynamic =
+        CI->getNumArgOperands() < 4 ? Builder.getFalse() : CI->getArgOperand(3);
     NewCall = Builder.CreateCall(
-        NewFn, {CI->getArgOperand(0), CI->getArgOperand(1), NullIsUnknownSize});
+        NewFn, {CI->getArgOperand(0), CI->getArgOperand(1), NullIsUnknownSize, Dynamic});
     break;
   }
 
diff --git a/lib/IR/BasicBlock.cpp b/lib/IR/BasicBlock.cpp
index e9eb686c1e139536819d9df21d6c7156cc4e19ea..18e2fd898f72b3a7c75e8eb21eb83b70dc548993 100644
--- a/lib/IR/BasicBlock.cpp
+++ b/lib/IR/BasicBlock.cpp
@@ -1,9 +1,8 @@
 //===-- BasicBlock.cpp - Implement BasicBlock related methods -------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -206,10 +205,8 @@ const Instruction* BasicBlock::getFirstNonPHIOrDbgOrLifetime() const {
     if (isa<PHINode>(I) || isa<DbgInfoIntrinsic>(I))
       continue;
 
-    if (auto *II = dyn_cast<IntrinsicInst>(&I))
-      if (II->getIntrinsicID() == Intrinsic::lifetime_start ||
-          II->getIntrinsicID() == Intrinsic::lifetime_end)
-        continue;
+    if (I.isLifetimeStartOrEnd())
+      continue;
 
     return &I;
   }
@@ -302,7 +299,7 @@ iterator_range<BasicBlock::phi_iterator> BasicBlock::phis() {
 /// called while the predecessor still refers to this block.
 ///
 void BasicBlock::removePredecessor(BasicBlock *Pred,
-                                   bool DontDeleteUselessPHIs) {
+                                   bool KeepOneInputPHIs) {
   assert((hasNUsesOrMore(16)||// Reduce cost of this assertion for complex CFGs.
           find(pred_begin(this), pred_end(this), Pred) != pred_end(this)) &&
          "removePredecessor: BB is not a predecessor!");
@@ -333,11 +330,11 @@ void BasicBlock::removePredecessor(BasicBlock *Pred,
   }
 
   // <= Two predecessors BEFORE I remove one?
-  if (max_idx <= 2 && !DontDeleteUselessPHIs) {
+  if (max_idx <= 2 && !KeepOneInputPHIs) {
     // Yup, loop through and nuke the PHI nodes
     while (PHINode *PN = dyn_cast<PHINode>(&front())) {
       // Remove the predecessor first.
-      PN->removeIncomingValue(Pred, !DontDeleteUselessPHIs);
+      PN->removeIncomingValue(Pred, !KeepOneInputPHIs);
 
       // If the PHI _HAD_ two uses, replace PHI node with its now *single* value
       if (max_idx == 2) {
@@ -362,7 +359,7 @@ void BasicBlock::removePredecessor(BasicBlock *Pred,
       // If all incoming values to the Phi are the same, we can replace the Phi
       // with that value.
       Value* PNV = nullptr;
-      if (!DontDeleteUselessPHIs && (PNV = PN->hasConstantValue()))
+      if (!KeepOneInputPHIs && (PNV = PN->hasConstantValue()))
         if (PNV != PN) {
           PN->replaceAllUsesWith(PNV);
           PN->eraseFromParent();
diff --git a/lib/IR/CMakeLists.txt b/lib/IR/CMakeLists.txt
index 2586f9872897002ce8d44025d05cae609f07c4c5..2ea01b8a7679d74dcaff6ac34c8f464787b12847 100644
--- a/lib/IR/CMakeLists.txt
+++ b/lib/IR/CMakeLists.txt
@@ -3,6 +3,7 @@ tablegen(LLVM AttributesCompatFunc.inc -gen-attrs)
 add_public_tablegen_target(AttributeCompatFuncTableGen)
 
 add_llvm_library(LLVMCore
+  AbstractCallSite.cpp
   AsmWriter.cpp
   Attributes.cpp
   AutoUpgrade.cpp
@@ -21,7 +22,6 @@ add_llvm_library(LLVMCore
   DiagnosticInfo.cpp
   DiagnosticPrinter.cpp
   Dominators.cpp
-  DomTreeUpdater.cpp
   Function.cpp
   GVMaterializer.cpp
   Globals.cpp
diff --git a/lib/IR/Comdat.cpp b/lib/IR/Comdat.cpp
index 3b1f7d62cdae628898ae4bbbfa0710731bbb9669..c9f715daf4573a738901902f7a5fce1e1b79f79a 100644
--- a/lib/IR/Comdat.cpp
+++ b/lib/IR/Comdat.cpp
@@ -1,9 +1,8 @@
 //===- Comdat.cpp - Implement Metadata classes ----------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/IR/ConstantFold.cpp b/lib/IR/ConstantFold.cpp
index 826199ee054e38df2d3c060b2cef45ce7a594658..748a2ce475f35cb575d30f99f98038a41ea47337 100644
--- a/lib/IR/ConstantFold.cpp
+++ b/lib/IR/ConstantFold.cpp
@@ -1,9 +1,8 @@
 //===- ConstantFold.cpp - LLVM constant folder ----------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -27,7 +26,6 @@
 #include "llvm/IR/GlobalAlias.h"
 #include "llvm/IR/GlobalVariable.h"
 #include "llvm/IR/Instructions.h"
-#include "llvm/IR/Module.h"
 #include "llvm/IR/Operator.h"
 #include "llvm/IR/PatternMatch.h"
 #include "llvm/Support/ErrorHandling.h"
@@ -1078,10 +1076,10 @@ Constant *llvm::ConstantFoldBinaryInstruction(unsigned Opcode, Constant *C1,
             isa<GlobalValue>(CE1->getOperand(0))) {
           GlobalValue *GV = cast<GlobalValue>(CE1->getOperand(0));
 
-          unsigned GVAlign =
-              GV->getParent()
-                  ? GV->getPointerAlignment(GV->getParent()->getDataLayout())
-                  : 0;
+          // Functions are at least 4-byte aligned.
+          unsigned GVAlign = GV->getAlignment();
+          if (isa<Function>(GV))
+            GVAlign = std::max(GVAlign, 4U);
 
           if (GVAlign > 1) {
             unsigned DstWidth = CI2->getType()->getBitWidth();
@@ -2073,7 +2071,7 @@ Constant *llvm::ConstantFoldGetElementPtr(Type *PointeeTy, Constant *C,
   if (Idxs.empty()) return C;
 
   Type *GEPTy = GetElementPtrInst::getGEPReturnType(
-      C, makeArrayRef((Value *const *)Idxs.data(), Idxs.size()));
+      PointeeTy, C, makeArrayRef((Value *const *)Idxs.data(), Idxs.size()));
 
   if (isa<UndefValue>(C))
     return UndefValue::get(GEPTy);
diff --git a/lib/IR/ConstantFold.h b/lib/IR/ConstantFold.h
index 2d8de1132b964d26348f86cbbe9afb62d6b413be..b57153bf00b01196fa7157ca4134b3c298a29798 100644
--- a/lib/IR/ConstantFold.h
+++ b/lib/IR/ConstantFold.h
@@ -1,9 +1,8 @@
 //===-- ConstantFolding.h - Internal Constant Folding Interface -*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/IR/ConstantRange.cpp b/lib/IR/ConstantRange.cpp
index 39a0b13c4e0c005d8ec2a3d914830d633ecfb484..bab83fcddf736cda27cc79ef0d1424e62a8a03a3 100644
--- a/lib/IR/ConstantRange.cpp
+++ b/lib/IR/ConstantRange.cpp
@@ -1,9 +1,8 @@
 //===- ConstantRange.cpp - ConstantRange implementation -------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/IR/Constants.cpp b/lib/IR/Constants.cpp
index df09d13d3ebaa8d6f2c5e205a42864e3f3ade446..00d6cc7ea23ff2f8fc4c3b7e801f0337856b316b 100644
--- a/lib/IR/Constants.cpp
+++ b/lib/IR/Constants.cpp
@@ -1,9 +1,8 @@
 //===-- Constants.cpp - Implement Constant nodes --------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -1999,9 +1998,8 @@ Constant *ConstantExpr::getGetElementPtr(Type *Ty, Constant *C,
   if (!Ty)
     Ty = cast<PointerType>(C->getType()->getScalarType())->getElementType();
   else
-    assert(
-        Ty ==
-        cast<PointerType>(C->getType()->getScalarType())->getContainedType(0u));
+    assert(Ty ==
+           cast<PointerType>(C->getType()->getScalarType())->getElementType());
 
   if (Constant *FC =
           ConstantFoldGetElementPtr(Ty, C, InBounds, InRangeIndex, Idxs))
diff --git a/lib/IR/ConstantsContext.h b/lib/IR/ConstantsContext.h
index eac17139708471dc357b1ef54d0e20eb04ac88ca..7614dab9f15d983039196f0179f9af860d353959 100644
--- a/lib/IR/ConstantsContext.h
+++ b/lib/IR/ConstantsContext.h
@@ -1,9 +1,8 @@
 //===-- ConstantsContext.h - Constants-related Context Interals -*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/IR/Core.cpp b/lib/IR/Core.cpp
index a3065733c81b280fcce5cf2b0e978afa43c839ed..1b8ad4823f92134e2ba13322863f8b993b376870 100644
--- a/lib/IR/Core.cpp
+++ b/lib/IR/Core.cpp
@@ -1,9 +1,8 @@
 //===-- Core.cpp ----------------------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -15,7 +14,6 @@
 #include "llvm-c/Core.h"
 #include "llvm/ADT/StringSwitch.h"
 #include "llvm/IR/Attributes.h"
-#include "llvm/IR/CallSite.h"
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/DebugInfoMetadata.h"
 #include "llvm/IR/DerivedTypes.h"
@@ -108,6 +106,14 @@ void LLVMContextSetYieldCallback(LLVMContextRef C, LLVMYieldCallback Callback,
   unwrap(C)->setYieldCallback(YieldCallback, OpaqueHandle);
 }
 
+LLVMBool LLVMContextShouldDiscardValueNames(LLVMContextRef C) {
+  return unwrap(C)->shouldDiscardValueNames();
+}
+
+void LLVMContextSetDiscardValueNames(LLVMContextRef C, LLVMBool Discard) {
+  unwrap(C)->setDiscardValueNames(Discard);
+}
+
 void LLVMContextDispose(LLVMContextRef C) {
   delete unwrap(C);
 }
@@ -1610,17 +1616,21 @@ LLVMValueRef LLVMConstGEP(LLVMValueRef ConstantVal,
                           LLVMValueRef *ConstantIndices, unsigned NumIndices) {
   ArrayRef<Constant *> IdxList(unwrap<Constant>(ConstantIndices, NumIndices),
                                NumIndices);
-  return wrap(ConstantExpr::getGetElementPtr(
-      nullptr, unwrap<Constant>(ConstantVal), IdxList));
+  Constant *Val = unwrap<Constant>(ConstantVal);
+  Type *Ty =
+      cast<PointerType>(Val->getType()->getScalarType())->getElementType();
+  return wrap(ConstantExpr::getGetElementPtr(Ty, Val, IdxList));
 }
 
 LLVMValueRef LLVMConstInBoundsGEP(LLVMValueRef ConstantVal,
                                   LLVMValueRef *ConstantIndices,
                                   unsigned NumIndices) {
-  Constant* Val = unwrap<Constant>(ConstantVal);
   ArrayRef<Constant *> IdxList(unwrap<Constant>(ConstantIndices, NumIndices),
                                NumIndices);
-  return wrap(ConstantExpr::getInBoundsGetElementPtr(nullptr, Val, IdxList));
+  Constant *Val = unwrap<Constant>(ConstantVal);
+  Type *Ty =
+      cast<PointerType>(Val->getType()->getScalarType())->getElementType();
+  return wrap(ConstantExpr::getInBoundsGetElementPtr(Ty, Val, IdxList));
 }
 
 LLVMValueRef LLVMConstTrunc(LLVMValueRef ConstantVal, LLVMTypeRef ToType) {
@@ -2453,6 +2463,71 @@ void LLVMSetParamAlignment(LLVMValueRef Arg, unsigned align) {
   A->addAttr(Attribute::getWithAlignment(A->getContext(), align));
 }
 
+/*--.. Operations on ifuncs ................................................--*/
+
+LLVMValueRef LLVMAddGlobalIFunc(LLVMModuleRef M,
+                                const char *Name, size_t NameLen,
+                                LLVMTypeRef Ty, unsigned AddrSpace,
+                                LLVMValueRef Resolver) {
+  return wrap(GlobalIFunc::create(unwrap(Ty), AddrSpace,
+                                  GlobalValue::ExternalLinkage,
+                                  StringRef(Name, NameLen),
+                                  unwrap<Constant>(Resolver), unwrap(M)));
+}
+
+LLVMValueRef LLVMGetNamedGlobalIFunc(LLVMModuleRef M,
+                                     const char *Name, size_t NameLen) {
+  return wrap(unwrap(M)->getNamedIFunc(StringRef(Name, NameLen)));
+}
+
+LLVMValueRef LLVMGetFirstGlobalIFunc(LLVMModuleRef M) {
+  Module *Mod = unwrap(M);
+  Module::ifunc_iterator I = Mod->ifunc_begin();
+  if (I == Mod->ifunc_end())
+    return nullptr;
+  return wrap(&*I);
+}
+
+LLVMValueRef LLVMGetLastGlobalIFunc(LLVMModuleRef M) {
+  Module *Mod = unwrap(M);
+  Module::ifunc_iterator I = Mod->ifunc_end();
+  if (I == Mod->ifunc_begin())
+    return nullptr;
+  return wrap(&*--I);
+}
+
+LLVMValueRef LLVMGetNextGlobalIFunc(LLVMValueRef IFunc) {
+  GlobalIFunc *GIF = unwrap<GlobalIFunc>(IFunc);
+  Module::ifunc_iterator I(GIF);
+  if (++I == GIF->getParent()->ifunc_end())
+    return nullptr;
+  return wrap(&*I);
+}
+
+LLVMValueRef LLVMGetPreviousGlobalIFunc(LLVMValueRef IFunc) {
+  GlobalIFunc *GIF = unwrap<GlobalIFunc>(IFunc);
+  Module::ifunc_iterator I(GIF);
+  if (I == GIF->getParent()->ifunc_begin())
+    return nullptr;
+  return wrap(&*--I);
+}
+
+LLVMValueRef LLVMGetGlobalIFuncResolver(LLVMValueRef IFunc) {
+  return wrap(unwrap<GlobalIFunc>(IFunc)->getResolver());
+}
+
+void LLVMSetGlobalIFuncResolver(LLVMValueRef IFunc, LLVMValueRef Resolver) {
+  unwrap<GlobalIFunc>(IFunc)->setResolver(unwrap<Constant>(Resolver));
+}
+
+void LLVMEraseGlobalIFunc(LLVMValueRef IFunc) {
+  unwrap<GlobalIFunc>(IFunc)->eraseFromParent();
+}
+
+void LLVMRemoveGlobalIFunc(LLVMValueRef IFunc) {
+  unwrap<GlobalIFunc>(IFunc)->removeFromParent();
+}
+
 /*--.. Operations on basic blocks ..........................................--*/
 
 LLVMValueRef LLVMBasicBlockAsValue(LLVMBasicBlockRef BB) {
@@ -2525,6 +2600,11 @@ LLVMBasicBlockRef LLVMGetPreviousBasicBlock(LLVMBasicBlockRef BB) {
   return wrap(&*--I);
 }
 
+LLVMBasicBlockRef LLVMCreateBasicBlockInContext(LLVMContextRef C,
+                                                const char *Name) {
+  return wrap(llvm::BasicBlock::Create(*unwrap(C), Name));
+}
+
 LLVMBasicBlockRef LLVMAppendBasicBlockInContext(LLVMContextRef C,
                                                 LLVMValueRef FnRef,
                                                 const char *Name) {
@@ -2648,43 +2728,43 @@ unsigned LLVMGetNumArgOperands(LLVMValueRef Instr) {
   if (FuncletPadInst *FPI = dyn_cast<FuncletPadInst>(unwrap(Instr))) {
     return FPI->getNumArgOperands();
   }
-  return CallSite(unwrap<Instruction>(Instr)).getNumArgOperands();
+  return unwrap<CallBase>(Instr)->getNumArgOperands();
 }
 
 /*--.. Call and invoke instructions ........................................--*/
 
 unsigned LLVMGetInstructionCallConv(LLVMValueRef Instr) {
-  return CallSite(unwrap<Instruction>(Instr)).getCallingConv();
+  return unwrap<CallBase>(Instr)->getCallingConv();
 }
 
 void LLVMSetInstructionCallConv(LLVMValueRef Instr, unsigned CC) {
-  return CallSite(unwrap<Instruction>(Instr))
-    .setCallingConv(static_cast<CallingConv::ID>(CC));
+  return unwrap<CallBase>(Instr)->setCallingConv(
+      static_cast<CallingConv::ID>(CC));
 }
 
 void LLVMSetInstrParamAlignment(LLVMValueRef Instr, unsigned index,
                                 unsigned align) {
-  CallSite Call = CallSite(unwrap<Instruction>(Instr));
+  auto *Call = unwrap<CallBase>(Instr);
   Attribute AlignAttr = Attribute::getWithAlignment(Call->getContext(), align);
-  Call.addAttribute(index, AlignAttr);
+  Call->addAttribute(index, AlignAttr);
 }
 
 void LLVMAddCallSiteAttribute(LLVMValueRef C, LLVMAttributeIndex Idx,
                               LLVMAttributeRef A) {
-  CallSite(unwrap<Instruction>(C)).addAttribute(Idx, unwrap(A));
+  unwrap<CallBase>(C)->addAttribute(Idx, unwrap(A));
 }
 
 unsigned LLVMGetCallSiteAttributeCount(LLVMValueRef C,
                                        LLVMAttributeIndex Idx) {
-  auto CS = CallSite(unwrap<Instruction>(C));
-  auto AS = CS.getAttributes().getAttributes(Idx);
+  auto *Call = unwrap<CallBase>(C);
+  auto AS = Call->getAttributes().getAttributes(Idx);
   return AS.getNumAttributes();
 }
 
 void LLVMGetCallSiteAttributes(LLVMValueRef C, LLVMAttributeIndex Idx,
                                LLVMAttributeRef *Attrs) {
-  auto CS = CallSite(unwrap<Instruction>(C));
-  auto AS = CS.getAttributes().getAttributes(Idx);
+  auto *Call = unwrap<CallBase>(C);
+  auto AS = Call->getAttributes().getAttributes(Idx);
   for (auto A : AS)
     *Attrs++ = wrap(A);
 }
@@ -2692,30 +2772,32 @@ void LLVMGetCallSiteAttributes(LLVMValueRef C, LLVMAttributeIndex Idx,
 LLVMAttributeRef LLVMGetCallSiteEnumAttribute(LLVMValueRef C,
                                               LLVMAttributeIndex Idx,
                                               unsigned KindID) {
-  return wrap(CallSite(unwrap<Instruction>(C))
-    .getAttribute(Idx, (Attribute::AttrKind)KindID));
+  return wrap(
+      unwrap<CallBase>(C)->getAttribute(Idx, (Attribute::AttrKind)KindID));
 }
 
 LLVMAttributeRef LLVMGetCallSiteStringAttribute(LLVMValueRef C,
                                                 LLVMAttributeIndex Idx,
                                                 const char *K, unsigned KLen) {
-  return wrap(CallSite(unwrap<Instruction>(C))
-    .getAttribute(Idx, StringRef(K, KLen)));
+  return wrap(unwrap<CallBase>(C)->getAttribute(Idx, StringRef(K, KLen)));
 }
 
 void LLVMRemoveCallSiteEnumAttribute(LLVMValueRef C, LLVMAttributeIndex Idx,
                                      unsigned KindID) {
-  CallSite(unwrap<Instruction>(C))
-    .removeAttribute(Idx, (Attribute::AttrKind)KindID);
+  unwrap<CallBase>(C)->removeAttribute(Idx, (Attribute::AttrKind)KindID);
 }
 
 void LLVMRemoveCallSiteStringAttribute(LLVMValueRef C, LLVMAttributeIndex Idx,
                                        const char *K, unsigned KLen) {
-  CallSite(unwrap<Instruction>(C)).removeAttribute(Idx, StringRef(K, KLen));
+  unwrap<CallBase>(C)->removeAttribute(Idx, StringRef(K, KLen));
 }
 
 LLVMValueRef LLVMGetCalledValue(LLVMValueRef Instr) {
-  return wrap(CallSite(unwrap<Instruction>(Instr)).getCalledValue());
+  return wrap(unwrap<CallBase>(Instr)->getCalledValue());
+}
+
+LLVMTypeRef LLVMGetCalledFunctionType(LLVMValueRef Instr) {
+  return wrap(unwrap<CallBase>(Instr)->getFunctionType());
 }
 
 /*--.. Operations on call instructions (only) ..............................--*/
@@ -2961,9 +3043,22 @@ LLVMValueRef LLVMBuildInvoke(LLVMBuilderRef B, LLVMValueRef Fn,
                              LLVMValueRef *Args, unsigned NumArgs,
                              LLVMBasicBlockRef Then, LLVMBasicBlockRef Catch,
                              const char *Name) {
-  return wrap(unwrap(B)->CreateInvoke(unwrap(Fn), unwrap(Then), unwrap(Catch),
-                                      makeArrayRef(unwrap(Args), NumArgs),
-                                      Name));
+  Value *V = unwrap(Fn);
+  FunctionType *FnT =
+      cast<FunctionType>(cast<PointerType>(V->getType())->getElementType());
+
+  return wrap(
+      unwrap(B)->CreateInvoke(FnT, unwrap(Fn), unwrap(Then), unwrap(Catch),
+                              makeArrayRef(unwrap(Args), NumArgs), Name));
+}
+
+LLVMValueRef LLVMBuildInvoke2(LLVMBuilderRef B, LLVMTypeRef Ty, LLVMValueRef Fn,
+                              LLVMValueRef *Args, unsigned NumArgs,
+                              LLVMBasicBlockRef Then, LLVMBasicBlockRef Catch,
+                              const char *Name) {
+  return wrap(unwrap(B)->CreateInvoke(
+      unwrap<FunctionType>(Ty), unwrap(Fn), unwrap(Then), unwrap(Catch),
+      makeArrayRef(unwrap(Args), NumArgs), Name));
 }
 
 LLVMValueRef LLVMBuildLandingPad(LLVMBuilderRef B, LLVMTypeRef Ty,
@@ -3319,7 +3414,15 @@ LLVMValueRef LLVMBuildFree(LLVMBuilderRef B, LLVMValueRef PointerVal) {
 
 LLVMValueRef LLVMBuildLoad(LLVMBuilderRef B, LLVMValueRef PointerVal,
                            const char *Name) {
-  return wrap(unwrap(B)->CreateLoad(unwrap(PointerVal), Name));
+  Value *V = unwrap(PointerVal);
+  PointerType *Ty = cast<PointerType>(V->getType());
+
+  return wrap(unwrap(B)->CreateLoad(Ty->getElementType(), V, Name));
+}
+
+LLVMValueRef LLVMBuildLoad2(LLVMBuilderRef B, LLVMTypeRef Ty,
+                            LLVMValueRef PointerVal, const char *Name) {
+  return wrap(unwrap(B)->CreateLoad(unwrap(Ty), unwrap(PointerVal), Name));
 }
 
 LLVMValueRef LLVMBuildStore(LLVMBuilderRef B, LLVMValueRef Val,
@@ -3374,20 +3477,50 @@ LLVMValueRef LLVMBuildGEP(LLVMBuilderRef B, LLVMValueRef Pointer,
                           LLVMValueRef *Indices, unsigned NumIndices,
                           const char *Name) {
   ArrayRef<Value *> IdxList(unwrap(Indices), NumIndices);
-  return wrap(unwrap(B)->CreateGEP(nullptr, unwrap(Pointer), IdxList, Name));
+  Value *Val = unwrap(Pointer);
+  Type *Ty =
+      cast<PointerType>(Val->getType()->getScalarType())->getElementType();
+  return wrap(unwrap(B)->CreateGEP(Ty, Val, IdxList, Name));
+}
+
+LLVMValueRef LLVMBuildGEP2(LLVMBuilderRef B, LLVMTypeRef Ty,
+                           LLVMValueRef Pointer, LLVMValueRef *Indices,
+                           unsigned NumIndices, const char *Name) {
+  ArrayRef<Value *> IdxList(unwrap(Indices), NumIndices);
+  return wrap(unwrap(B)->CreateGEP(unwrap(Ty), unwrap(Pointer), IdxList, Name));
 }
 
 LLVMValueRef LLVMBuildInBoundsGEP(LLVMBuilderRef B, LLVMValueRef Pointer,
                                   LLVMValueRef *Indices, unsigned NumIndices,
                                   const char *Name) {
   ArrayRef<Value *> IdxList(unwrap(Indices), NumIndices);
+  Value *Val = unwrap(Pointer);
+  Type *Ty =
+      cast<PointerType>(Val->getType()->getScalarType())->getElementType();
+  return wrap(unwrap(B)->CreateInBoundsGEP(Ty, Val, IdxList, Name));
+}
+
+LLVMValueRef LLVMBuildInBoundsGEP2(LLVMBuilderRef B, LLVMTypeRef Ty,
+                                   LLVMValueRef Pointer, LLVMValueRef *Indices,
+                                   unsigned NumIndices, const char *Name) {
+  ArrayRef<Value *> IdxList(unwrap(Indices), NumIndices);
   return wrap(
-      unwrap(B)->CreateInBoundsGEP(nullptr, unwrap(Pointer), IdxList, Name));
+      unwrap(B)->CreateInBoundsGEP(unwrap(Ty), unwrap(Pointer), IdxList, Name));
 }
 
 LLVMValueRef LLVMBuildStructGEP(LLVMBuilderRef B, LLVMValueRef Pointer,
                                 unsigned Idx, const char *Name) {
-  return wrap(unwrap(B)->CreateStructGEP(nullptr, unwrap(Pointer), Idx, Name));
+  Value *Val = unwrap(Pointer);
+  Type *Ty =
+      cast<PointerType>(Val->getType()->getScalarType())->getElementType();
+  return wrap(unwrap(B)->CreateStructGEP(Ty, Val, Idx, Name));
+}
+
+LLVMValueRef LLVMBuildStructGEP2(LLVMBuilderRef B, LLVMTypeRef Ty,
+                                 LLVMValueRef Pointer, unsigned Idx,
+                                 const char *Name) {
+  return wrap(
+      unwrap(B)->CreateStructGEP(unwrap(Ty), unwrap(Pointer), Idx, Name));
 }
 
 LLVMValueRef LLVMBuildGlobalString(LLVMBuilderRef B, const char *Str,
@@ -3529,6 +3662,13 @@ LLVMValueRef LLVMBuildPointerCast(LLVMBuilderRef B, LLVMValueRef Val,
   return wrap(unwrap(B)->CreatePointerCast(unwrap(Val), unwrap(DestTy), Name));
 }
 
+LLVMValueRef LLVMBuildIntCast2(LLVMBuilderRef B, LLVMValueRef Val,
+                               LLVMTypeRef DestTy, LLVMBool IsSigned,
+                               const char *Name) {
+  return wrap(
+      unwrap(B)->CreateIntCast(unwrap(Val), unwrap(DestTy), IsSigned, Name));
+}
+
 LLVMValueRef LLVMBuildIntCast(LLVMBuilderRef B, LLVMValueRef Val,
                               LLVMTypeRef DestTy, const char *Name) {
   return wrap(unwrap(B)->CreateIntCast(unwrap(Val), unwrap(DestTy),
@@ -3565,9 +3705,20 @@ LLVMValueRef LLVMBuildPhi(LLVMBuilderRef B, LLVMTypeRef Ty, const char *Name) {
 LLVMValueRef LLVMBuildCall(LLVMBuilderRef B, LLVMValueRef Fn,
                            LLVMValueRef *Args, unsigned NumArgs,
                            const char *Name) {
-  return wrap(unwrap(B)->CreateCall(unwrap(Fn),
-                                    makeArrayRef(unwrap(Args), NumArgs),
-                                    Name));
+  Value *V = unwrap(Fn);
+  FunctionType *FnT =
+      cast<FunctionType>(cast<PointerType>(V->getType())->getElementType());
+
+  return wrap(unwrap(B)->CreateCall(FnT, unwrap(Fn),
+                                    makeArrayRef(unwrap(Args), NumArgs), Name));
+}
+
+LLVMValueRef LLVMBuildCall2(LLVMBuilderRef B, LLVMTypeRef Ty, LLVMValueRef Fn,
+                            LLVMValueRef *Args, unsigned NumArgs,
+                            const char *Name) {
+  FunctionType *FTy = unwrap<FunctionType>(Ty);
+  return wrap(unwrap(B)->CreateCall(FTy, unwrap(Fn),
+                                    makeArrayRef(unwrap(Args), NumArgs), Name));
 }
 
 LLVMValueRef LLVMBuildSelect(LLVMBuilderRef B, LLVMValueRef If,
diff --git a/lib/IR/DIBuilder.cpp b/lib/IR/DIBuilder.cpp
index d29759f567918a8a5eca38e7a5df44aa3a4da6b4..c248740b2411b5c3d986518982c8df94b154ee64 100644
--- a/lib/IR/DIBuilder.cpp
+++ b/lib/IR/DIBuilder.cpp
@@ -1,9 +1,8 @@
 //===--- DIBuilder.cpp - Debug Information Builder ------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -504,11 +503,11 @@ DISubroutineType *DIBuilder::createSubroutineType(DITypeRefArray ParameterTypes,
 DICompositeType *DIBuilder::createEnumerationType(
     DIScope *Scope, StringRef Name, DIFile *File, unsigned LineNumber,
     uint64_t SizeInBits, uint32_t AlignInBits, DINodeArray Elements,
-    DIType *UnderlyingType, StringRef UniqueIdentifier, bool IsFixed) {
+    DIType *UnderlyingType, StringRef UniqueIdentifier, bool IsScoped) {
   auto *CTy = DICompositeType::get(
       VMContext, dwarf::DW_TAG_enumeration_type, Name, File, LineNumber,
       getNonCompileUnitScope(Scope), UnderlyingType, SizeInBits, AlignInBits, 0,
-      IsFixed ? DINode::FlagFixedEnum : DINode::FlagZero, Elements, 0, nullptr,
+      IsScoped ? DINode::FlagEnumClass : DINode::FlagZero, Elements, 0, nullptr,
       nullptr, UniqueIdentifier);
   AllEnumTypes.push_back(CTy);
   trackIfUnresolved(CTy);
diff --git a/lib/IR/DataLayout.cpp b/lib/IR/DataLayout.cpp
index 18d502a52ca7a36a1af470b01ede9abab7c19233..2644214a49c0d168b694a516b30d7350c553d5d4 100644
--- a/lib/IR/DataLayout.cpp
+++ b/lib/IR/DataLayout.cpp
@@ -1,9 +1,8 @@
 //===- DataLayout.cpp - Data size & alignment routines ---------------------==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -635,6 +634,14 @@ unsigned DataLayout::getPointerSize(unsigned AS) const {
   return I->TypeByteWidth;
 }
 
+unsigned DataLayout::getMaxPointerSize() const {
+  unsigned MaxPointerSize = 0;
+  for (auto &P : Pointers)
+    MaxPointerSize = std::max(MaxPointerSize, P.TypeByteWidth);
+
+  return MaxPointerSize;
+}
+
 unsigned DataLayout::getPointerTypeSizeInBits(Type *Ty) const {
   assert(Ty->isPtrOrPtrVectorTy() &&
          "This should only be called with a pointer or pointer vector type");
diff --git a/lib/IR/DebugInfo.cpp b/lib/IR/DebugInfo.cpp
index d1ff54571782d10d89b0129e1593fc7bc144cd0f..435685d42619cbde0b112165ca71e666c2c88bfa 100644
--- a/lib/IR/DebugInfo.cpp
+++ b/lib/IR/DebugInfo.cpp
@@ -1,9 +1,8 @@
 //===- DebugInfo.cpp - Debug Information Helper Classes -------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -697,8 +696,9 @@ void Instruction::applyMergedLocation(const DILocation *LocA,
 
 static unsigned map_from_llvmDWARFsourcelanguage(LLVMDWARFSourceLanguage lang) {
   switch (lang) {
-#define HANDLE_DW_LANG(ID, NAME, VERSION, VENDOR) \
-case LLVMDWARFSourceLanguage##NAME: return ID;
+#define HANDLE_DW_LANG(ID, NAME, LOWER_BOUND, VERSION, VENDOR)                 \
+  case LLVMDWARFSourceLanguage##NAME:                                          \
+    return ID;
 #include "llvm/BinaryFormat/Dwarf.def"
 #undef HANDLE_DW_LANG
   }
diff --git a/lib/IR/DebugInfoMetadata.cpp b/lib/IR/DebugInfoMetadata.cpp
index 3b702ce4798b99ddf10e077565c30079480a9f9e..f772276613c87706085688d60533ec107ee0cb30 100644
--- a/lib/IR/DebugInfoMetadata.cpp
+++ b/lib/IR/DebugInfoMetadata.cpp
@@ -1,9 +1,8 @@
 //===- DebugInfoMetadata.cpp - Implement debug info metadata --------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -20,6 +19,8 @@
 #include "llvm/IR/Function.h"
 #include "llvm/IR/Instructions.h"
 
+#include <numeric>
+
 using namespace llvm;
 
 DILocation::DILocation(LLVMContext &C, StorageType Storage, unsigned Line,
@@ -113,6 +114,47 @@ const DILocation *DILocation::getMergedLocation(const DILocation *LocA,
   return DILocation::get(Result->getContext(), 0, 0, S, L);
 }
 
+Optional<unsigned> DILocation::encodeDiscriminator(unsigned BD, unsigned DF, unsigned CI) {
+  SmallVector<unsigned, 3> Components = {BD, DF, CI};
+  uint64_t RemainingWork = 0U;
+  // We use RemainingWork to figure out if we have no remaining components to
+  // encode. For example: if BD != 0 but DF == 0 && CI == 0, we don't need to
+  // encode anything for the latter 2.
+  // Since any of the input components is at most 32 bits, their sum will be
+  // less than 34 bits, and thus RemainingWork won't overflow.
+  RemainingWork = std::accumulate(Components.begin(), Components.end(), RemainingWork);
+
+  int I = 0;
+  unsigned Ret = 0;
+  unsigned NextBitInsertionIndex = 0;
+  while (RemainingWork > 0) {
+    unsigned C = Components[I++];
+    RemainingWork -= C;
+    unsigned EC = encodeComponent(C);
+    Ret |= (EC << NextBitInsertionIndex);
+    NextBitInsertionIndex += encodingBits(C);
+  }
+
+  // Encoding may be unsuccessful because of overflow. We determine success by
+  // checking equivalence of components before & after encoding. Alternatively,
+  // we could determine Success during encoding, but the current alternative is
+  // simpler.
+  unsigned TBD, TDF, TCI = 0;
+  decodeDiscriminator(Ret, TBD, TDF, TCI);
+  if (TBD == BD && TDF == DF && TCI == CI)
+    return Ret;
+  return None;
+}
+
+void DILocation::decodeDiscriminator(unsigned D, unsigned &BD, unsigned &DF,
+                                     unsigned &CI) {
+  BD = getUnsignedFromPrefixEncoding(D);
+  DF = getUnsignedFromPrefixEncoding(getNextComponentInDiscriminator(D));
+  CI = getUnsignedFromPrefixEncoding(
+      getNextComponentInDiscriminator(getNextComponentInDiscriminator(D)));
+}
+
+
 DINode::DIFlags DINode::getFlag(StringRef Flag) {
   return StringSwitch<DIFlags>(Flag)
 #define HANDLE_DI_FLAG(ID, NAME) .Case("DIFlag" #NAME, Flag##NAME)
@@ -886,6 +928,24 @@ bool DIExpression::extractIfOffset(int64_t &Offset) const {
   return false;
 }
 
+const DIExpression *DIExpression::extractAddressClass(const DIExpression *Expr,
+                                                      unsigned &AddrClass) {
+  const unsigned PatternSize = 4;
+  if (Expr->Elements.size() >= PatternSize &&
+      Expr->Elements[PatternSize - 4] == dwarf::DW_OP_constu &&
+      Expr->Elements[PatternSize - 2] == dwarf::DW_OP_swap &&
+      Expr->Elements[PatternSize - 1] == dwarf::DW_OP_xderef) {
+    AddrClass = Expr->Elements[PatternSize - 3];
+
+    if (Expr->Elements.size() == PatternSize)
+      return nullptr;
+    return DIExpression::get(Expr->getContext(),
+                             makeArrayRef(&*Expr->Elements.begin(),
+                                          Expr->Elements.size() - PatternSize));
+  }
+  return Expr;
+}
+
 DIExpression *DIExpression::prepend(const DIExpression *Expr, bool DerefBefore,
                                     int64_t Offset, bool DerefAfter,
                                     bool StackValue) {
diff --git a/lib/IR/DebugLoc.cpp b/lib/IR/DebugLoc.cpp
index 10ec98ac7e6c8ee259ee8c64694ff5f3bd7824e2..14d1396f154339f71e2536cca8c623738362cd10 100644
--- a/lib/IR/DebugLoc.cpp
+++ b/lib/IR/DebugLoc.cpp
@@ -1,9 +1,8 @@
 //===-- DebugLoc.cpp - Implement DebugLoc class ---------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/IR/DiagnosticHandler.cpp b/lib/IR/DiagnosticHandler.cpp
index 8f972785cf91859e58e3eb0c10adc49ec12f677a..2fe63480389477c03018333c00780f55cd03ca50 100644
--- a/lib/IR/DiagnosticHandler.cpp
+++ b/lib/IR/DiagnosticHandler.cpp
@@ -1,9 +1,8 @@
 //===- DiagnosticHandler.h - DiagnosticHandler class for LLVM -------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/IR/DiagnosticInfo.cpp b/lib/IR/DiagnosticInfo.cpp
index 3c6665bbcb7eac8e2c4689516ab706cc5a41a879..14bee35dc2929ec6060c63c8e6fdc1cdb44c8151 100644
--- a/lib/IR/DiagnosticInfo.cpp
+++ b/lib/IR/DiagnosticInfo.cpp
@@ -1,9 +1,8 @@
 //===- llvm/Support/DiagnosticInfo.cpp - Diagnostic Definitions -*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -104,6 +103,11 @@ void DiagnosticInfoPGOProfile::print(DiagnosticPrinter &DP) const {
   DP << getMsg();
 }
 
+void DiagnosticInfo::anchor() {}
+void DiagnosticInfoStackSize::anchor() {}
+void DiagnosticInfoWithLocationBase::anchor() {}
+void DiagnosticInfoIROptimization::anchor() {}
+
 DiagnosticLocation::DiagnosticLocation(const DebugLoc &DL) {
   if (!DL)
     return;
@@ -366,6 +370,9 @@ std::string DiagnosticInfoOptimizationBase::getMsg() const {
   return OS.str();
 }
 
+void OptimizationRemarkAnalysisFPCommute::anchor() {}
+void OptimizationRemarkAnalysisAliasing::anchor() {}
+
 namespace llvm {
 namespace yaml {
 
@@ -431,11 +438,33 @@ template <> struct MappingTraits<DiagnosticLocation> {
   static const bool flow = true;
 };
 
+/// Helper struct for multiline string block literals. Use this type to preserve
+/// newlines in strings.
+struct StringBlockVal {
+  StringRef Value;
+  StringBlockVal(const std::string &Value) : Value(Value) {}
+};
+
+template <> struct BlockScalarTraits<StringBlockVal> {
+  static void output(const StringBlockVal &S, void *Ctx, raw_ostream &OS) {
+    return ScalarTraits<StringRef>::output(S.Value, Ctx, OS);
+  }
+
+  static StringRef input(StringRef Scalar, void *Ctx, StringBlockVal &S) {
+    return ScalarTraits<StringRef>::input(Scalar, Ctx, S.Value);
+  }
+};
+
 // Implement this as a mapping for now to get proper quotation for the value.
 template <> struct MappingTraits<DiagnosticInfoOptimizationBase::Argument> {
   static void mapping(IO &io, DiagnosticInfoOptimizationBase::Argument &A) {
     assert(io.outputting() && "input not yet implemented");
-    io.mapRequired(A.Key.data(), A.Val);
+    // Emit a string block scalar for multiline strings, to preserve newlines.
+    if (StringRef(A.Val).count('\n') > 1) {
+      StringBlockVal S(A.Val);
+      io.mapRequired(A.Key.data(), S);
+    } else
+      io.mapRequired(A.Key.data(), A.Val);
     if (A.Loc.isValid())
       io.mapOptional("DebugLoc", A.Loc);
   }
diff --git a/lib/IR/DiagnosticPrinter.cpp b/lib/IR/DiagnosticPrinter.cpp
index ee2df9e24f939cbaea3d9ac9b9568d35bde61514..496bd18e78e27356b2429c9e4501080c84322dc5 100644
--- a/lib/IR/DiagnosticPrinter.cpp
+++ b/lib/IR/DiagnosticPrinter.cpp
@@ -1,9 +1,8 @@
 //===- llvm/Support/DiagnosticInfo.cpp - Diagnostic Definitions -*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/IR/Dominators.cpp b/lib/IR/Dominators.cpp
index cf9f5759ba5339231191864f23393cc130151a36..910a41050b941c0eb2bf338759dafc6f4f435a19 100644
--- a/lib/IR/Dominators.cpp
+++ b/lib/IR/Dominators.cpp
@@ -1,9 +1,8 @@
 //===- Dominators.cpp - Dominator Calculation -----------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/IR/Function.cpp b/lib/IR/Function.cpp
index ec094812ceb2f3c61a68225896535506f48fe003..574ca88e1c90a7d319b57e2826d365b7d2d82051 100644
--- a/lib/IR/Function.cpp
+++ b/lib/IR/Function.cpp
@@ -1,9 +1,8 @@
 //===- Function.cpp - Implement the Global object classes -----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -24,7 +23,6 @@
 #include "llvm/IR/Argument.h"
 #include "llvm/IR/Attributes.h"
 #include "llvm/IR/BasicBlock.h"
-#include "llvm/IR/CallSite.h"
 #include "llvm/IR/Constant.h"
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/DerivedTypes.h"
@@ -950,10 +948,9 @@ static Type *DecodeFixedType(ArrayRef<Intrinsic::IITDescriptor> &Infos,
   case IITDescriptor::SameVecWidthArgument: {
     Type *EltTy = DecodeFixedType(Infos, Tys, Context);
     Type *Ty = Tys[D.getArgumentNumber()];
-    if (VectorType *VTy = dyn_cast<VectorType>(Ty)) {
+    if (auto *VTy = dyn_cast<VectorType>(Ty))
       return VectorType::get(EltTy, VTy->getNumElements());
-    }
-    llvm_unreachable("unhandled");
+    return EltTy;
   }
   case IITDescriptor::PtrToArgument: {
     Type *Ty = Tys[D.getArgumentNumber()];
@@ -1021,9 +1018,10 @@ bool Intrinsic::isLeaf(ID id) {
 Function *Intrinsic::getDeclaration(Module *M, ID id, ArrayRef<Type*> Tys) {
   // There can never be multiple globals with the same name of different types,
   // because intrinsics must be a specific type.
-  return
-    cast<Function>(M->getOrInsertFunction(getName(id, Tys),
-                                          getType(M->getContext(), id, Tys)));
+  return cast<Function>(
+      M->getOrInsertFunction(getName(id, Tys),
+                             getType(M->getContext(), id, Tys))
+          .getCallee());
 }
 
 // This defines the "Intrinsic::getIntrinsicForGCCBuiltin()" method.
@@ -1084,19 +1082,19 @@ bool Intrinsic::matchIntrinsicType(Type *Ty, ArrayRef<Intrinsic::IITDescriptor>
       if (D.getArgumentNumber() < ArgTys.size())
         return Ty != ArgTys[D.getArgumentNumber()];
 
-          // Otherwise, if this is the first instance of an argument, record it and
-          // verify the "Any" kind.
-          assert(D.getArgumentNumber() == ArgTys.size() && "Table consistency error");
-          ArgTys.push_back(Ty);
+      // Otherwise, if this is the first instance of an argument, record it and
+      // verify the "Any" kind.
+      assert(D.getArgumentNumber() == ArgTys.size() && "Table consistency error");
+      ArgTys.push_back(Ty);
 
-          switch (D.getArgumentKind()) {
-            case IITDescriptor::AK_Any:        return false; // Success
-            case IITDescriptor::AK_AnyInteger: return !Ty->isIntOrIntVectorTy();
-            case IITDescriptor::AK_AnyFloat:   return !Ty->isFPOrFPVectorTy();
-            case IITDescriptor::AK_AnyVector:  return !isa<VectorType>(Ty);
-            case IITDescriptor::AK_AnyPointer: return !isa<PointerType>(Ty);
-          }
-          llvm_unreachable("all argument kinds not covered");
+      switch (D.getArgumentKind()) {
+        case IITDescriptor::AK_Any:        return false; // Success
+        case IITDescriptor::AK_AnyInteger: return !Ty->isIntOrIntVectorTy();
+        case IITDescriptor::AK_AnyFloat:   return !Ty->isFPOrFPVectorTy();
+        case IITDescriptor::AK_AnyVector:  return !isa<VectorType>(Ty);
+        case IITDescriptor::AK_AnyPointer: return !isa<PointerType>(Ty);
+      }
+      llvm_unreachable("all argument kinds not covered");
 
     case IITDescriptor::ExtendArgument: {
       // This may only be used when referring to a previous vector argument.
@@ -1137,15 +1135,19 @@ bool Intrinsic::matchIntrinsicType(Type *Ty, ArrayRef<Intrinsic::IITDescriptor>
     case IITDescriptor::SameVecWidthArgument: {
       if (D.getArgumentNumber() >= ArgTys.size())
         return true;
-      VectorType * ReferenceType =
-        dyn_cast<VectorType>(ArgTys[D.getArgumentNumber()]);
-      VectorType *ThisArgType = dyn_cast<VectorType>(Ty);
-      if (!ThisArgType || !ReferenceType ||
-          (ReferenceType->getVectorNumElements() !=
-           ThisArgType->getVectorNumElements()))
+      auto *ReferenceType = dyn_cast<VectorType>(ArgTys[D.getArgumentNumber()]);
+      auto *ThisArgType = dyn_cast<VectorType>(Ty);
+      // Both must be vectors of the same number of elements or neither.
+      if ((ReferenceType != nullptr) != (ThisArgType != nullptr))
         return true;
-      return matchIntrinsicType(ThisArgType->getVectorElementType(),
-                                Infos, ArgTys);
+      Type *EltTy = Ty;
+      if (ThisArgType) {
+        if (ReferenceType->getVectorNumElements() !=
+            ThisArgType->getVectorNumElements())
+          return true;
+        EltTy = ThisArgType->getVectorElementType();
+      }
+      return matchIntrinsicType(EltTy, Infos, ArgTys);
     }
     case IITDescriptor::PtrToArgument: {
       if (D.getArgumentNumber() >= ArgTys.size())
@@ -1257,13 +1259,13 @@ bool Function::hasAddressTaken(const User* *PutOffender) const {
     const User *FU = U.getUser();
     if (isa<BlockAddress>(FU))
       continue;
-    if (!isa<CallInst>(FU) && !isa<InvokeInst>(FU)) {
+    const auto *Call = dyn_cast<CallBase>(FU);
+    if (!Call) {
       if (PutOffender)
         *PutOffender = FU;
       return true;
     }
-    ImmutableCallSite CS(cast<Instruction>(FU));
-    if (!CS.isCallee(&U)) {
+    if (!Call->isCallee(&U)) {
       if (PutOffender)
         *PutOffender = FU;
       return true;
@@ -1289,12 +1291,10 @@ bool Function::isDefTriviallyDead() const {
 /// callsFunctionThatReturnsTwice - Return true if the function has a call to
 /// setjmp or other function that gcc recognizes as "returning twice".
 bool Function::callsFunctionThatReturnsTwice() const {
-  for (const_inst_iterator
-         I = inst_begin(this), E = inst_end(this); I != E; ++I) {
-    ImmutableCallSite CS(&*I);
-    if (CS && CS.hasFnAttr(Attribute::ReturnsTwice))
-      return true;
-  }
+  for (const Instruction &I : instructions(this))
+    if (const auto *Call = dyn_cast<CallBase>(&I))
+      if (Call->hasFnAttr(Attribute::ReturnsTwice))
+        return true;
 
   return false;
 }
diff --git a/lib/IR/GVMaterializer.cpp b/lib/IR/GVMaterializer.cpp
index 706926d1b9818d0cd1f9eaac24481929a81477ec..35397309a103d9c11527191e965b20239ea276b3 100644
--- a/lib/IR/GVMaterializer.cpp
+++ b/lib/IR/GVMaterializer.cpp
@@ -1,9 +1,8 @@
 //===-- GVMaterializer.cpp - Base implementation for GV materializers -----===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/IR/Globals.cpp b/lib/IR/Globals.cpp
index cbd6450a20c9b138c9c222a0778497606bb4e5ff..b2f5640026f9f5a831802386698685b2896cb9b9 100644
--- a/lib/IR/Globals.cpp
+++ b/lib/IR/Globals.cpp
@@ -1,9 +1,8 @@
 //===-- Globals.cpp - Implement the GlobalValue & GlobalVariable class ----===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/IR/IRBuilder.cpp b/lib/IR/IRBuilder.cpp
index a98189956770970f1780f9f27cd729255b9da899..a7290cb8c0f02529913ca946192aa873c312561c 100644
--- a/lib/IR/IRBuilder.cpp
+++ b/lib/IR/IRBuilder.cpp
@@ -1,9 +1,8 @@
 //===- IRBuilder.cpp - Builder for LLVM Instrs ----------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -72,7 +71,7 @@ Value *IRBuilderBase::getCastedInt8PtrValue(Value *Ptr) {
   return BCI;
 }
 
-static CallInst *createCallHelper(Value *Callee, ArrayRef<Value *> Ops,
+static CallInst *createCallHelper(Function *Callee, ArrayRef<Value *> Ops,
                                   IRBuilderBase *Builder,
                                   const Twine &Name = "",
                                   Instruction *FMFSource = nullptr) {
@@ -84,7 +83,7 @@ static CallInst *createCallHelper(Value *Callee, ArrayRef<Value *> Ops,
   return CI;
 }
 
-static InvokeInst *createInvokeHelper(Value *Invokee, BasicBlock *NormalDest,
+static InvokeInst *createInvokeHelper(Function *Invokee, BasicBlock *NormalDest,
                                       BasicBlock *UnwindDest,
                                       ArrayRef<Value *> Ops,
                                       IRBuilderBase *Builder,
@@ -105,7 +104,7 @@ CreateMemSet(Value *Ptr, Value *Val, Value *Size, unsigned Align,
   Value *Ops[] = {Ptr, Val, Size, getInt1(isVolatile)};
   Type *Tys[] = { Ptr->getType(), Size->getType() };
   Module *M = BB->getParent()->getParent();
-  Value *TheFn = Intrinsic::getDeclaration(M, Intrinsic::memset, Tys);
+  Function *TheFn = Intrinsic::getDeclaration(M, Intrinsic::memset, Tys);
 
   CallInst *CI = createCallHelper(TheFn, Ops, this);
 
@@ -135,7 +134,7 @@ CallInst *IRBuilderBase::CreateElementUnorderedAtomicMemSet(
   Value *Ops[] = {Ptr, Val, Size, getInt32(ElementSize)};
   Type *Tys[] = {Ptr->getType(), Size->getType()};
   Module *M = BB->getParent()->getParent();
-  Value *TheFn = Intrinsic::getDeclaration(
+  Function *TheFn = Intrinsic::getDeclaration(
       M, Intrinsic::memset_element_unordered_atomic, Tys);
 
   CallInst *CI = createCallHelper(TheFn, Ops, this);
@@ -167,7 +166,7 @@ CreateMemCpy(Value *Dst, unsigned DstAlign, Value *Src, unsigned SrcAlign,
   Value *Ops[] = {Dst, Src, Size, getInt1(isVolatile)};
   Type *Tys[] = { Dst->getType(), Src->getType(), Size->getType() };
   Module *M = BB->getParent()->getParent();
-  Value *TheFn = Intrinsic::getDeclaration(M, Intrinsic::memcpy, Tys);
+  Function *TheFn = Intrinsic::getDeclaration(M, Intrinsic::memcpy, Tys);
 
   CallInst *CI = createCallHelper(TheFn, Ops, this);
 
@@ -208,7 +207,7 @@ CallInst *IRBuilderBase::CreateElementUnorderedAtomicMemCpy(
   Value *Ops[] = {Dst, Src, Size, getInt32(ElementSize)};
   Type *Tys[] = {Dst->getType(), Src->getType(), Size->getType()};
   Module *M = BB->getParent()->getParent();
-  Value *TheFn = Intrinsic::getDeclaration(
+  Function *TheFn = Intrinsic::getDeclaration(
       M, Intrinsic::memcpy_element_unordered_atomic, Tys);
 
   CallInst *CI = createCallHelper(TheFn, Ops, this);
@@ -247,7 +246,7 @@ CreateMemMove(Value *Dst, unsigned DstAlign, Value *Src, unsigned SrcAlign,
   Value *Ops[] = {Dst, Src, Size, getInt1(isVolatile)};
   Type *Tys[] = { Dst->getType(), Src->getType(), Size->getType() };
   Module *M = BB->getParent()->getParent();
-  Value *TheFn = Intrinsic::getDeclaration(M, Intrinsic::memmove, Tys);
+  Function *TheFn = Intrinsic::getDeclaration(M, Intrinsic::memmove, Tys);
 
   CallInst *CI = createCallHelper(TheFn, Ops, this);
 
@@ -284,7 +283,7 @@ CallInst *IRBuilderBase::CreateElementUnorderedAtomicMemMove(
   Value *Ops[] = {Dst, Src, Size, getInt32(ElementSize)};
   Type *Tys[] = {Dst->getType(), Src->getType(), Size->getType()};
   Module *M = BB->getParent()->getParent();
-  Value *TheFn = Intrinsic::getDeclaration(
+  Function *TheFn = Intrinsic::getDeclaration(
       M, Intrinsic::memmove_element_unordered_atomic, Tys);
 
   CallInst *CI = createCallHelper(TheFn, Ops, this);
@@ -409,8 +408,8 @@ CallInst *IRBuilderBase::CreateLifetimeStart(Value *Ptr, ConstantInt *Size) {
            "lifetime.start requires the size to be an i64");
   Value *Ops[] = { Size, Ptr };
   Module *M = BB->getParent()->getParent();
-  Value *TheFn = Intrinsic::getDeclaration(M, Intrinsic::lifetime_start,
-                                           { Ptr->getType() });
+  Function *TheFn =
+      Intrinsic::getDeclaration(M, Intrinsic::lifetime_start, {Ptr->getType()});
   return createCallHelper(TheFn, Ops, this);
 }
 
@@ -425,8 +424,8 @@ CallInst *IRBuilderBase::CreateLifetimeEnd(Value *Ptr, ConstantInt *Size) {
            "lifetime.end requires the size to be an i64");
   Value *Ops[] = { Size, Ptr };
   Module *M = BB->getParent()->getParent();
-  Value *TheFn = Intrinsic::getDeclaration(M, Intrinsic::lifetime_end,
-                                           { Ptr->getType() });
+  Function *TheFn =
+      Intrinsic::getDeclaration(M, Intrinsic::lifetime_end, {Ptr->getType()});
   return createCallHelper(TheFn, Ops, this);
 }
 
@@ -445,7 +444,7 @@ CallInst *IRBuilderBase::CreateInvariantStart(Value *Ptr, ConstantInt *Size) {
   // Fill in the single overloaded type: memory object type.
   Type *ObjectPtr[1] = {Ptr->getType()};
   Module *M = BB->getParent()->getParent();
-  Value *TheFn =
+  Function *TheFn =
       Intrinsic::getDeclaration(M, Intrinsic::invariant_start, ObjectPtr);
   return createCallHelper(TheFn, Ops, this);
 }
@@ -456,7 +455,7 @@ CallInst *IRBuilderBase::CreateAssumption(Value *Cond) {
 
   Value *Ops[] = { Cond };
   Module *M = BB->getParent()->getParent();
-  Value *FnAssume = Intrinsic::getDeclaration(M, Intrinsic::assume);
+  Function *FnAssume = Intrinsic::getDeclaration(M, Intrinsic::assume);
   return createCallHelper(FnAssume, Ops, this);
 }
 
@@ -508,7 +507,7 @@ CallInst *IRBuilderBase::CreateMaskedIntrinsic(Intrinsic::ID Id,
                                                ArrayRef<Type *> OverloadedTypes,
                                                const Twine &Name) {
   Module *M = BB->getParent()->getParent();
-  Value *TheFn = Intrinsic::getDeclaration(M, Id, OverloadedTypes);
+  Function *TheFn = Intrinsic::getDeclaration(M, Id, OverloadedTypes);
   return createCallHelper(TheFn, Ops, this, Name);
 }
 
@@ -709,7 +708,7 @@ CallInst *IRBuilderBase::CreateGCResult(Instruction *Statepoint,
  Intrinsic::ID ID = Intrinsic::experimental_gc_result;
  Module *M = BB->getParent()->getParent();
  Type *Types[] = {ResultType};
- Value *FnGCResult = Intrinsic::getDeclaration(M, ID, Types);
+ Function *FnGCResult = Intrinsic::getDeclaration(M, ID, Types);
 
  Value *Args[] = {Statepoint};
  return createCallHelper(FnGCResult, Args, this, Name);
@@ -722,8 +721,8 @@ CallInst *IRBuilderBase::CreateGCRelocate(Instruction *Statepoint,
                                          const Twine &Name) {
  Module *M = BB->getParent()->getParent();
  Type *Types[] = {ResultType};
- Value *FnGCRelocate =
-   Intrinsic::getDeclaration(M, Intrinsic::experimental_gc_relocate, Types);
+ Function *FnGCRelocate =
+     Intrinsic::getDeclaration(M, Intrinsic::experimental_gc_relocate, Types);
 
  Value *Args[] = {Statepoint,
                   getInt32(BaseOffset),
diff --git a/lib/IR/IRPrintingPasses.cpp b/lib/IR/IRPrintingPasses.cpp
index 43010220b9f3cac88bfbf46d4552a4d82eb97b44..35b06135a8280a1355449fa234a4e51c6c7ea148 100644
--- a/lib/IR/IRPrintingPasses.cpp
+++ b/lib/IR/IRPrintingPasses.cpp
@@ -1,9 +1,8 @@
 //===--- IRPrintingPasses.cpp - Module and Function printing passes -------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/IR/InlineAsm.cpp b/lib/IR/InlineAsm.cpp
index 4623f69bd9a36d5c5906005917f9da14888cdf8c..99da7caaccf09ad1ee6e85271024449feb0bdc29 100644
--- a/lib/IR/InlineAsm.cpp
+++ b/lib/IR/InlineAsm.cpp
@@ -1,9 +1,8 @@
 //===- InlineAsm.cpp - Implement the InlineAsm class ----------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/IR/Instruction.cpp b/lib/IR/Instruction.cpp
index f077957969fa5019998401f986556c245ac89781..32f3bfa66b1c8a4a045b7b78ba99d74842b72334 100644
--- a/lib/IR/Instruction.cpp
+++ b/lib/IR/Instruction.cpp
@@ -1,9 +1,8 @@
 //===-- Instruction.cpp - Implement the Instruction class -----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -302,6 +301,7 @@ const char *Instruction::getOpcodeName(unsigned OpCode) {
   case CatchRet: return "catchret";
   case CatchPad: return "catchpad";
   case CatchSwitch: return "catchswitch";
+  case CallBr: return "callbr";
 
   // Standard unary operators...
   case FNeg: return "fneg";
@@ -406,6 +406,10 @@ static bool haveSameSpecialState(const Instruction *I1, const Instruction *I2,
     return CI->getCallingConv() == cast<InvokeInst>(I2)->getCallingConv() &&
            CI->getAttributes() == cast<InvokeInst>(I2)->getAttributes() &&
            CI->hasIdenticalOperandBundleSchema(*cast<InvokeInst>(I2));
+  if (const CallBrInst *CI = dyn_cast<CallBrInst>(I1))
+    return CI->getCallingConv() == cast<CallBrInst>(I2)->getCallingConv() &&
+           CI->getAttributes() == cast<CallBrInst>(I2)->getAttributes() &&
+           CI->hasIdenticalOperandBundleSchema(*cast<CallBrInst>(I2));
   if (const InsertValueInst *IVI = dyn_cast<InsertValueInst>(I1))
     return IVI->getIndices() == cast<InsertValueInst>(I2)->getIndices();
   if (const ExtractValueInst *EVI = dyn_cast<ExtractValueInst>(I1))
@@ -516,9 +520,9 @@ bool Instruction::mayReadFromMemory() const {
   case Instruction::CatchRet:
     return true;
   case Instruction::Call:
-    return !cast<CallInst>(this)->doesNotAccessMemory();
   case Instruction::Invoke:
-    return !cast<InvokeInst>(this)->doesNotAccessMemory();
+  case Instruction::CallBr:
+    return !cast<CallBase>(this)->doesNotAccessMemory();
   case Instruction::Store:
     return !cast<StoreInst>(this)->isUnordered();
   }
@@ -536,9 +540,9 @@ bool Instruction::mayWriteToMemory() const {
   case Instruction::CatchRet:
     return true;
   case Instruction::Call:
-    return !cast<CallInst>(this)->onlyReadsMemory();
   case Instruction::Invoke:
-    return !cast<InvokeInst>(this)->onlyReadsMemory();
+  case Instruction::CallBr:
+    return !cast<CallBase>(this)->onlyReadsMemory();
   case Instruction::Load:
     return !cast<LoadInst>(this)->isUnordered();
   }
@@ -598,6 +602,14 @@ bool Instruction::isSafeToRemove() const {
          !this->isTerminator();
 }
 
+bool Instruction::isLifetimeStartOrEnd() const {
+  auto II = dyn_cast<IntrinsicInst>(this);
+  if (!II)
+    return false;
+  Intrinsic::ID ID = II->getIntrinsicID();
+  return ID == Intrinsic::lifetime_start || ID == Intrinsic::lifetime_end;
+}
+
 const Instruction *Instruction::getNextNonDebugInstruction() const {
   for (const Instruction *I = getNextNode(); I; I = I->getNextNode())
     if (!isa<DbgInfoIntrinsic>(I))
@@ -767,8 +779,8 @@ void Instruction::updateProfWeight(uint64_t S, uint64_t T) {
 }
 
 void Instruction::setProfWeight(uint64_t W) {
-  assert((isa<CallInst>(this) || isa<InvokeInst>(this)) &&
-         "Can only set weights for call and invoke instrucitons");
+  assert(isa<CallBase>(this) &&
+         "Can only set weights for call like instructions");
   SmallVector<uint32_t, 1> Weights;
   Weights.push_back(W);
   MDBuilder MDB(getContext());
diff --git a/lib/IR/Instructions.cpp b/lib/IR/Instructions.cpp
index 8405bbde3220d19540963f1678a477d10ed6bd0a..766c41188ff8c82565d3e56830be374703106286 100644
--- a/lib/IR/Instructions.cpp
+++ b/lib/IR/Instructions.cpp
@@ -1,9 +1,8 @@
 //===- Instructions.cpp - Implement the LLVM instructions -----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -27,6 +26,7 @@
 #include "llvm/IR/Function.h"
 #include "llvm/IR/InstrTypes.h"
 #include "llvm/IR/Instruction.h"
+#include "llvm/IR/Intrinsics.h"
 #include "llvm/IR/LLVMContext.h"
 #include "llvm/IR/Metadata.h"
 #include "llvm/IR/Module.h"
@@ -254,6 +254,56 @@ void LandingPadInst::addClause(Constant *Val) {
 //                        CallBase Implementation
 //===----------------------------------------------------------------------===//
 
+Function *CallBase::getCaller() { return getParent()->getParent(); }
+
+unsigned CallBase::getNumSubclassExtraOperandsDynamic() const {
+  assert(getOpcode() == Instruction::CallBr && "Unexpected opcode!");
+  return cast<CallBrInst>(this)->getNumIndirectDests() + 1;
+}
+
+bool CallBase::isIndirectCall() const {
+  const Value *V = getCalledValue();
+  if (isa<Function>(V) || isa<Constant>(V))
+    return false;
+  if (const CallInst *CI = dyn_cast<CallInst>(this))
+    if (CI->isInlineAsm())
+      return false;
+  return true;
+}
+
+/// Tests if this call site must be tail call optimized. Only a CallInst can
+/// be tail call optimized.
+bool CallBase::isMustTailCall() const {
+  if (auto *CI = dyn_cast<CallInst>(this))
+    return CI->isMustTailCall();
+  return false;
+}
+
+/// Tests if this call site is marked as a tail call.
+bool CallBase::isTailCall() const {
+  if (auto *CI = dyn_cast<CallInst>(this))
+    return CI->isTailCall();
+  return false;
+}
+
+Intrinsic::ID CallBase::getIntrinsicID() const {
+  if (auto *F = getCalledFunction())
+    return F->getIntrinsicID();
+  return Intrinsic::not_intrinsic;
+}
+
+bool CallBase::isReturnNonNull() const {
+  if (hasRetAttr(Attribute::NonNull))
+    return true;
+
+  if (getDereferenceableBytes(AttributeList::ReturnIndex) > 0 &&
+           !NullPointerIsDefined(getCaller(),
+                                 getType()->getPointerAddressSpace()))
+    return true;
+
+  return false;
+}
+
 Value *CallBase::getReturnedArgOperand() const {
   unsigned Index;
 
@@ -357,9 +407,8 @@ void CallInst::init(FunctionType *FTy, Value *Func, ArrayRef<Value *> Args,
   setName(NameStr);
 }
 
-void CallInst::init(Value *Func, const Twine &NameStr) {
-  FTy =
-      cast<FunctionType>(cast<PointerType>(Func->getType())->getElementType());
+void CallInst::init(FunctionType *FTy, Value *Func, const Twine &NameStr) {
+  this->FTy = FTy;
   assert(getNumOperands() == 1 && "NumOperands not set up?");
   setCalledOperand(Func);
 
@@ -368,22 +417,18 @@ void CallInst::init(Value *Func, const Twine &NameStr) {
   setName(NameStr);
 }
 
-CallInst::CallInst(Value *Func, const Twine &Name, Instruction *InsertBefore)
-    : CallBase(cast<FunctionType>(
-                   cast<PointerType>(Func->getType())->getElementType())
-                   ->getReturnType(),
-               Instruction::Call, OperandTraits<CallBase>::op_end(this) - 1, 1,
-               InsertBefore) {
-  init(Func, Name);
+CallInst::CallInst(FunctionType *Ty, Value *Func, const Twine &Name,
+                   Instruction *InsertBefore)
+    : CallBase(Ty->getReturnType(), Instruction::Call,
+               OperandTraits<CallBase>::op_end(this) - 1, 1, InsertBefore) {
+  init(Ty, Func, Name);
 }
 
-CallInst::CallInst(Value *Func, const Twine &Name, BasicBlock *InsertAtEnd)
-    : CallBase(cast<FunctionType>(
-                   cast<PointerType>(Func->getType())->getElementType())
-                   ->getReturnType(),
-               Instruction::Call, OperandTraits<CallBase>::op_end(this) - 1, 1,
-               InsertAtEnd) {
-  init(Func, Name);
+CallInst::CallInst(FunctionType *Ty, Value *Func, const Twine &Name,
+                   BasicBlock *InsertAtEnd)
+    : CallBase(Ty->getReturnType(), Instruction::Call,
+               OperandTraits<CallBase>::op_end(this) - 1, 1, InsertAtEnd) {
+  init(Ty, Func, Name);
 }
 
 CallInst::CallInst(const CallInst &CI)
@@ -403,8 +448,8 @@ CallInst *CallInst::Create(CallInst *CI, ArrayRef<OperandBundleDef> OpB,
                            Instruction *InsertPt) {
   std::vector<Value *> Args(CI->arg_begin(), CI->arg_end());
 
-  auto *NewCI = CallInst::Create(CI->getCalledValue(), Args, OpB, CI->getName(),
-                                 InsertPt);
+  auto *NewCI = CallInst::Create(CI->getFunctionType(), CI->getCalledValue(),
+                                 Args, OpB, CI->getName(), InsertPt);
   NewCI->setTailCallKind(CI->getTailCallKind());
   NewCI->setCallingConv(CI->getCallingConv());
   NewCI->SubclassOptionalData = CI->SubclassOptionalData;
@@ -477,7 +522,7 @@ static Instruction *createMalloc(Instruction *InsertBefore,
   BasicBlock *BB = InsertBefore ? InsertBefore->getParent() : InsertAtEnd;
   Module *M = BB->getParent()->getParent();
   Type *BPTy = Type::getInt8PtrTy(BB->getContext());
-  Value *MallocFunc = MallocF;
+  FunctionCallee MallocFunc = MallocF;
   if (!MallocFunc)
     // prototype malloc as "void *malloc(size_t)"
     MallocFunc = M->getOrInsertFunction("malloc", BPTy, IntPtrTy);
@@ -501,7 +546,7 @@ static Instruction *createMalloc(Instruction *InsertBefore,
     }
   }
   MCall->setTailCall();
-  if (Function *F = dyn_cast<Function>(MallocFunc)) {
+  if (Function *F = dyn_cast<Function>(MallocFunc.getCallee())) {
     MCall->setCallingConv(F->getCallingConv());
     if (!F->returnDoesNotAlias())
       F->setReturnDoesNotAlias();
@@ -574,7 +619,7 @@ static Instruction *createFree(Value *Source,
   Type *VoidTy = Type::getVoidTy(M->getContext());
   Type *IntPtrTy = Type::getInt8PtrTy(M->getContext());
   // prototype free as "void free(void*)"
-  Value *FreeFunc = M->getOrInsertFunction("free", VoidTy, IntPtrTy);
+  FunctionCallee FreeFunc = M->getOrInsertFunction("free", VoidTy, IntPtrTy);
   CallInst *Result = nullptr;
   Value *PtrCast = Source;
   if (InsertBefore) {
@@ -587,7 +632,7 @@ static Instruction *createFree(Value *Source,
     Result = CallInst::Create(FreeFunc, PtrCast, Bundles, "");
   }
   Result->setTailCall();
-  if (Function *F = dyn_cast<Function>(FreeFunc))
+  if (Function *F = dyn_cast<Function>(FreeFunc.getCallee()))
     Result->setCallingConv(F->getCallingConv());
 
   return Result;
@@ -671,9 +716,9 @@ InvokeInst *InvokeInst::Create(InvokeInst *II, ArrayRef<OperandBundleDef> OpB,
                                Instruction *InsertPt) {
   std::vector<Value *> Args(II->arg_begin(), II->arg_end());
 
-  auto *NewII = InvokeInst::Create(II->getCalledValue(), II->getNormalDest(),
-                                   II->getUnwindDest(), Args, OpB,
-                                   II->getName(), InsertPt);
+  auto *NewII = InvokeInst::Create(II->getFunctionType(), II->getCalledValue(),
+                                   II->getNormalDest(), II->getUnwindDest(),
+                                   Args, OpB, II->getName(), InsertPt);
   NewII->setCallingConv(II->getCallingConv());
   NewII->SubclassOptionalData = II->SubclassOptionalData;
   NewII->setAttributes(II->getAttributes());
@@ -686,6 +731,76 @@ LandingPadInst *InvokeInst::getLandingPadInst() const {
   return cast<LandingPadInst>(getUnwindDest()->getFirstNonPHI());
 }
 
+//===----------------------------------------------------------------------===//
+//                        CallBrInst Implementation
+//===----------------------------------------------------------------------===//
+
+void CallBrInst::init(FunctionType *FTy, Value *Fn, BasicBlock *Fallthrough,
+                      ArrayRef<BasicBlock *> IndirectDests,
+                      ArrayRef<Value *> Args,
+                      ArrayRef<OperandBundleDef> Bundles,
+                      const Twine &NameStr) {
+  this->FTy = FTy;
+
+  assert((int)getNumOperands() ==
+             ComputeNumOperands(Args.size(), IndirectDests.size(),
+                                CountBundleInputs(Bundles)) &&
+         "NumOperands not set up?");
+  NumIndirectDests = IndirectDests.size();
+  setDefaultDest(Fallthrough);
+  for (unsigned i = 0; i != NumIndirectDests; ++i)
+    setIndirectDest(i, IndirectDests[i]);
+  setCalledOperand(Fn);
+
+#ifndef NDEBUG
+  assert(((Args.size() == FTy->getNumParams()) ||
+          (FTy->isVarArg() && Args.size() > FTy->getNumParams())) &&
+         "Calling a function with bad signature");
+
+  for (unsigned i = 0, e = Args.size(); i != e; i++)
+    assert((i >= FTy->getNumParams() ||
+            FTy->getParamType(i) == Args[i]->getType()) &&
+           "Calling a function with a bad signature!");
+#endif
+
+  std::copy(Args.begin(), Args.end(), op_begin());
+
+  auto It = populateBundleOperandInfos(Bundles, Args.size());
+  (void)It;
+  assert(It + 2 + IndirectDests.size() == op_end() && "Should add up!");
+
+  setName(NameStr);
+}
+
+CallBrInst::CallBrInst(const CallBrInst &CBI)
+    : CallBase(CBI.Attrs, CBI.FTy, CBI.getType(), Instruction::CallBr,
+               OperandTraits<CallBase>::op_end(this) - CBI.getNumOperands(),
+               CBI.getNumOperands()) {
+  setCallingConv(CBI.getCallingConv());
+  std::copy(CBI.op_begin(), CBI.op_end(), op_begin());
+  std::copy(CBI.bundle_op_info_begin(), CBI.bundle_op_info_end(),
+            bundle_op_info_begin());
+  SubclassOptionalData = CBI.SubclassOptionalData;
+  NumIndirectDests = CBI.NumIndirectDests;
+}
+
+CallBrInst *CallBrInst::Create(CallBrInst *CBI, ArrayRef<OperandBundleDef> OpB,
+                               Instruction *InsertPt) {
+  std::vector<Value *> Args(CBI->arg_begin(), CBI->arg_end());
+
+  auto *NewCBI = CallBrInst::Create(CBI->getFunctionType(),
+                                    CBI->getCalledValue(),
+                                    CBI->getDefaultDest(),
+                                    CBI->getIndirectDests(),
+                                    Args, OpB, CBI->getName(), InsertPt);
+  NewCBI->setCallingConv(CBI->getCallingConv());
+  NewCBI->SubclassOptionalData = CBI->SubclassOptionalData;
+  NewCBI->setAttributes(CBI->getAttributes());
+  NewCBI->setDebugLoc(CBI->getDebugLoc());
+  NewCBI->NumIndirectDests = CBI->NumIndirectDests;
+  return NewCBI;
+}
+
 //===----------------------------------------------------------------------===//
 //                        ReturnInst Implementation
 //===----------------------------------------------------------------------===//
@@ -1112,28 +1227,30 @@ void LoadInst::AssertOK() {
          "Alignment required for atomic load");
 }
 
-LoadInst::LoadInst(Value *Ptr, const Twine &Name, Instruction *InsertBef)
-    : LoadInst(Ptr, Name, /*isVolatile=*/false, InsertBef) {}
+LoadInst::LoadInst(Type *Ty, Value *Ptr, const Twine &Name,
+                   Instruction *InsertBef)
+    : LoadInst(Ty, Ptr, Name, /*isVolatile=*/false, InsertBef) {}
 
-LoadInst::LoadInst(Value *Ptr, const Twine &Name, BasicBlock *InsertAE)
-    : LoadInst(Ptr, Name, /*isVolatile=*/false, InsertAE) {}
+LoadInst::LoadInst(Type *Ty, Value *Ptr, const Twine &Name,
+                   BasicBlock *InsertAE)
+    : LoadInst(Ty, Ptr, Name, /*isVolatile=*/false, InsertAE) {}
 
 LoadInst::LoadInst(Type *Ty, Value *Ptr, const Twine &Name, bool isVolatile,
                    Instruction *InsertBef)
     : LoadInst(Ty, Ptr, Name, isVolatile, /*Align=*/0, InsertBef) {}
 
-LoadInst::LoadInst(Value *Ptr, const Twine &Name, bool isVolatile,
+LoadInst::LoadInst(Type *Ty, Value *Ptr, const Twine &Name, bool isVolatile,
                    BasicBlock *InsertAE)
-    : LoadInst(Ptr, Name, isVolatile, /*Align=*/0, InsertAE) {}
+    : LoadInst(Ty, Ptr, Name, isVolatile, /*Align=*/0, InsertAE) {}
 
 LoadInst::LoadInst(Type *Ty, Value *Ptr, const Twine &Name, bool isVolatile,
                    unsigned Align, Instruction *InsertBef)
     : LoadInst(Ty, Ptr, Name, isVolatile, Align, AtomicOrdering::NotAtomic,
                SyncScope::System, InsertBef) {}
 
-LoadInst::LoadInst(Value *Ptr, const Twine &Name, bool isVolatile,
+LoadInst::LoadInst(Type *Ty, Value *Ptr, const Twine &Name, bool isVolatile,
                    unsigned Align, BasicBlock *InsertAE)
-    : LoadInst(Ptr, Name, isVolatile, Align, AtomicOrdering::NotAtomic,
+    : LoadInst(Ty, Ptr, Name, isVolatile, Align, AtomicOrdering::NotAtomic,
                SyncScope::System, InsertAE) {}
 
 LoadInst::LoadInst(Type *Ty, Value *Ptr, const Twine &Name, bool isVolatile,
@@ -1148,12 +1265,11 @@ LoadInst::LoadInst(Type *Ty, Value *Ptr, const Twine &Name, bool isVolatile,
   setName(Name);
 }
 
-LoadInst::LoadInst(Value *Ptr, const Twine &Name, bool isVolatile,
-                   unsigned Align, AtomicOrdering Order,
-                   SyncScope::ID SSID,
+LoadInst::LoadInst(Type *Ty, Value *Ptr, const Twine &Name, bool isVolatile,
+                   unsigned Align, AtomicOrdering Order, SyncScope::ID SSID,
                    BasicBlock *InsertAE)
-  : UnaryInstruction(cast<PointerType>(Ptr->getType())->getElementType(),
-                     Load, Ptr, InsertAE) {
+    : UnaryInstruction(Ty, Load, Ptr, InsertAE) {
+  assert(Ty == cast<PointerType>(Ptr->getType())->getElementType());
   setVolatile(isVolatile);
   setAlignment(Align);
   setAtomic(Order, SSID);
@@ -1161,48 +1277,6 @@ LoadInst::LoadInst(Value *Ptr, const Twine &Name, bool isVolatile,
   setName(Name);
 }
 
-LoadInst::LoadInst(Value *Ptr, const char *Name, Instruction *InsertBef)
-  : UnaryInstruction(cast<PointerType>(Ptr->getType())->getElementType(),
-                     Load, Ptr, InsertBef) {
-  setVolatile(false);
-  setAlignment(0);
-  setAtomic(AtomicOrdering::NotAtomic);
-  AssertOK();
-  if (Name && Name[0]) setName(Name);
-}
-
-LoadInst::LoadInst(Value *Ptr, const char *Name, BasicBlock *InsertAE)
-  : UnaryInstruction(cast<PointerType>(Ptr->getType())->getElementType(),
-                     Load, Ptr, InsertAE) {
-  setVolatile(false);
-  setAlignment(0);
-  setAtomic(AtomicOrdering::NotAtomic);
-  AssertOK();
-  if (Name && Name[0]) setName(Name);
-}
-
-LoadInst::LoadInst(Type *Ty, Value *Ptr, const char *Name, bool isVolatile,
-                   Instruction *InsertBef)
-    : UnaryInstruction(Ty, Load, Ptr, InsertBef) {
-  assert(Ty == cast<PointerType>(Ptr->getType())->getElementType());
-  setVolatile(isVolatile);
-  setAlignment(0);
-  setAtomic(AtomicOrdering::NotAtomic);
-  AssertOK();
-  if (Name && Name[0]) setName(Name);
-}
-
-LoadInst::LoadInst(Value *Ptr, const char *Name, bool isVolatile,
-                   BasicBlock *InsertAE)
-  : UnaryInstruction(cast<PointerType>(Ptr->getType())->getElementType(),
-                     Load, Ptr, InsertAE) {
-  setVolatile(isVolatile);
-  setAlignment(0);
-  setAtomic(AtomicOrdering::NotAtomic);
-  AssertOK();
-  if (Name && Name[0]) setName(Name);
-}
-
 void LoadInst::setAlignment(unsigned Align) {
   assert((Align & (Align-1)) == 0 && "Alignment is not a power of 2!");
   assert(Align <= MaximumAlignment &&
@@ -1423,6 +1497,10 @@ StringRef AtomicRMWInst::getOperationName(BinOp Op) {
     return "umax";
   case AtomicRMWInst::UMin:
     return "umin";
+  case AtomicRMWInst::FAdd:
+    return "fadd";
+  case AtomicRMWInst::FSub:
+    return "fsub";
   case AtomicRMWInst::BAD_BINOP:
     return "<invalid operation>";
   }
@@ -3853,7 +3931,7 @@ AllocaInst *AllocaInst::cloneImpl() const {
 }
 
 LoadInst *LoadInst::cloneImpl() const {
-  return new LoadInst(getOperand(0), Twine(), isVolatile(),
+  return new LoadInst(getType(), getOperand(0), Twine(), isVolatile(),
                       getAlignment(), getOrdering(), getSyncScopeID());
 }
 
@@ -3993,6 +4071,14 @@ InvokeInst *InvokeInst::cloneImpl() const {
   return new(getNumOperands()) InvokeInst(*this);
 }
 
+CallBrInst *CallBrInst::cloneImpl() const {
+  if (hasOperandBundles()) {
+    unsigned DescriptorBytes = getNumOperandBundles() * sizeof(BundleOpInfo);
+    return new (getNumOperands(), DescriptorBytes) CallBrInst(*this);
+  }
+  return new (getNumOperands()) CallBrInst(*this);
+}
+
 ResumeInst *ResumeInst::cloneImpl() const { return new (1) ResumeInst(*this); }
 
 CleanupReturnInst *CleanupReturnInst::cloneImpl() const {
diff --git a/lib/IR/IntrinsicInst.cpp b/lib/IR/IntrinsicInst.cpp
index df3a38ac147feb08fbf98a9bb4d229ff2ebc0995..8d371aabb21860f4cb4052918e9773a41efe3458 100644
--- a/lib/IR/IntrinsicInst.cpp
+++ b/lib/IR/IntrinsicInst.cpp
@@ -1,9 +1,8 @@
 //===-- InstrinsicInst.cpp - Intrinsic Instruction Wrappers ---------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/IR/LLVMBuild.txt b/lib/IR/LLVMBuild.txt
index 71368abfd8748438a8b3d8e0b6758ab3f5944da9..b3dcd413f41d31309801487241e89b468d297cf0 100644
--- a/lib/IR/LLVMBuild.txt
+++ b/lib/IR/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/IR/LLVMBuild.txt -----------------------------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/IR/LLVMContext.cpp b/lib/IR/LLVMContext.cpp
index 62d9e387162ef1d91183ae0874db33f209b64457..1cea21461f46e40e166e4794d1bea1276ca9bc6a 100644
--- a/lib/IR/LLVMContext.cpp
+++ b/lib/IR/LLVMContext.cpp
@@ -1,9 +1,8 @@
 //===-- LLVMContext.cpp - Implement LLVMContext ---------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -61,6 +60,8 @@ LLVMContext::LLVMContext() : pImpl(new LLVMContextImpl(*this)) {
     {MD_associated, "associated"},
     {MD_callees, "callees"},
     {MD_irr_loop, "irr_loop"},
+    {MD_access_group, "llvm.access.group"},
+    {MD_callback, "callback"},
   };
 
   for (auto &MDKind : MDKinds) {
diff --git a/lib/IR/LLVMContextImpl.cpp b/lib/IR/LLVMContextImpl.cpp
index 3c34ca55c22451197276941e8d5d9283f2dc97c8..c6ab2c6f213a1d0e028d52eafad6cd9efb5e8370 100644
--- a/lib/IR/LLVMContextImpl.cpp
+++ b/lib/IR/LLVMContextImpl.cpp
@@ -1,9 +1,8 @@
 //===- LLVMContextImpl.cpp - Implement LLVMContextImpl --------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/IR/LLVMContextImpl.h b/lib/IR/LLVMContextImpl.h
index 2d120869860a0eb02500ab5f036d02262b909d93..f7038ace035d6b4b1c958e026fe8d0dec099452e 100644
--- a/lib/IR/LLVMContextImpl.h
+++ b/lib/IR/LLVMContextImpl.h
@@ -1,9 +1,8 @@
 //===- LLVMContextImpl.h - The LLVMContextImpl opaque class -----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/IR/LegacyPassManager.cpp b/lib/IR/LegacyPassManager.cpp
index 01d14f17bba5a4066eccd5b7c6c96cbbdb6b680e..98b7d9b76aaaad9b3ee8281bc72b31ac1a18ba2c 100644
--- a/lib/IR/LegacyPassManager.cpp
+++ b/lib/IR/LegacyPassManager.cpp
@@ -1,9 +1,8 @@
 //===- LegacyPassManager.cpp - LLVM Pass Infrastructure Implementation ----===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/IR/MDBuilder.cpp b/lib/IR/MDBuilder.cpp
index 3fa541f1b5351dd5e89d0d8bd3556a2eba39c797..f65ac31ca5ae7144435aaae5dd78e7694addaaed 100644
--- a/lib/IR/MDBuilder.cpp
+++ b/lib/IR/MDBuilder.cpp
@@ -1,9 +1,8 @@
 //===---- llvm/MDBuilder.cpp - Builder for LLVM metadata ------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -107,6 +106,52 @@ MDNode *MDBuilder::createCallees(ArrayRef<Function *> Callees) {
   return MDNode::get(Context, Ops);
 }
 
+MDNode *MDBuilder::createCallbackEncoding(unsigned CalleeArgNo,
+                                          ArrayRef<int> Arguments,
+                                          bool VarArgArePassed) {
+  SmallVector<Metadata *, 4> Ops;
+
+  Type *Int64 = Type::getInt64Ty(Context);
+  Ops.push_back(createConstant(ConstantInt::get(Int64, CalleeArgNo)));
+
+  for (int ArgNo : Arguments)
+    Ops.push_back(createConstant(ConstantInt::get(Int64, ArgNo, true)));
+
+  Type *Int1 = Type::getInt1Ty(Context);
+  Ops.push_back(createConstant(ConstantInt::get(Int1, VarArgArePassed)));
+
+  return MDNode::get(Context, Ops);
+}
+
+MDNode *MDBuilder::mergeCallbackEncodings(MDNode *ExistingCallbacks,
+                                          MDNode *NewCB) {
+  if (!ExistingCallbacks)
+    return MDNode::get(Context, {NewCB});
+
+  auto *NewCBCalleeIdxAsCM = cast<ConstantAsMetadata>(NewCB->getOperand(0));
+  uint64_t NewCBCalleeIdx =
+      cast<ConstantInt>(NewCBCalleeIdxAsCM->getValue())->getZExtValue();
+  (void)NewCBCalleeIdx;
+
+  SmallVector<Metadata *, 4> Ops;
+  unsigned NumExistingOps = ExistingCallbacks->getNumOperands();
+  Ops.resize(NumExistingOps + 1);
+
+  for (unsigned u = 0; u < NumExistingOps; u++) {
+    Ops[u] = ExistingCallbacks->getOperand(u);
+
+    auto *OldCBCalleeIdxAsCM = cast<ConstantAsMetadata>(Ops[u]);
+    uint64_t OldCBCalleeIdx =
+      cast<ConstantInt>(OldCBCalleeIdxAsCM->getValue())->getZExtValue();
+    (void)OldCBCalleeIdx;
+    assert(NewCBCalleeIdx != OldCBCalleeIdx &&
+           "Cannot map a callback callee index twice!");
+  }
+
+  Ops[NumExistingOps] = NewCB;
+  return MDNode::get(Context, Ops);
+}
+
 MDNode *MDBuilder::createAnonymousAARoot(StringRef Name, MDNode *Extra) {
   // To ensure uniqueness the root node is self-referential.
   auto Dummy = MDNode::getTemporary(Context, None);
diff --git a/lib/IR/Mangler.cpp b/lib/IR/Mangler.cpp
index be3086cfcf05d5f599de80b48d431b900b3029a9..0d7833a7e8b9e07bfd3aa6f79956d59f5ba29a7e 100644
--- a/lib/IR/Mangler.cpp
+++ b/lib/IR/Mangler.cpp
@@ -1,9 +1,8 @@
 //===-- Mangler.cpp - Self-contained c/asm llvm name mangler --------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/IR/Metadata.cpp b/lib/IR/Metadata.cpp
index 5536c2497f1e986859933cf9fd5a90f5367d93a8..70ffa652bfb2eeee102d82e89f288f3188888614 100644
--- a/lib/IR/Metadata.cpp
+++ b/lib/IR/Metadata.cpp
@@ -1,9 +1,8 @@
 //===- Metadata.cpp - Implement Metadata classes --------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/IR/MetadataImpl.h b/lib/IR/MetadataImpl.h
index b9137460bd203142f219a2e32c7cf1ca03aa6b4f..b4188dd7d3ee46e2da4f43091d15277807214e04 100644
--- a/lib/IR/MetadataImpl.h
+++ b/lib/IR/MetadataImpl.h
@@ -1,9 +1,8 @@
 //===- MetadataImpl.h - Helpers for implementing metadata -----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/IR/Module.cpp b/lib/IR/Module.cpp
index 70a16cbbf24210b45a2c11e9fa4aaebdd173fd8f..b6dd7ab70a8b81795df710c2bb1cde687d447352 100644
--- a/lib/IR/Module.cpp
+++ b/lib/IR/Module.cpp
@@ -1,9 +1,8 @@
 //===- Module.cpp - Implement the Module class ----------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -141,8 +140,8 @@ void Module::getOperandBundleTags(SmallVectorImpl<StringRef> &Result) const {
 // it.  This is nice because it allows most passes to get away with not handling
 // the symbol table directly for this common task.
 //
-Constant *Module::getOrInsertFunction(StringRef Name, FunctionType *Ty,
-                                      AttributeList AttributeList) {
+FunctionCallee Module::getOrInsertFunction(StringRef Name, FunctionType *Ty,
+                                           AttributeList AttributeList) {
   // See if we have a definition for the specified function already.
   GlobalValue *F = getNamedValue(Name);
   if (!F) {
@@ -152,21 +151,20 @@ Constant *Module::getOrInsertFunction(StringRef Name, FunctionType *Ty,
     if (!New->isIntrinsic())       // Intrinsics get attrs set on construction
       New->setAttributes(AttributeList);
     FunctionList.push_back(New);
-    return New;                    // Return the new prototype.
+    return {Ty, New}; // Return the new prototype.
   }
 
   // If the function exists but has the wrong type, return a bitcast to the
   // right type.
   auto *PTy = PointerType::get(Ty, F->getAddressSpace());
   if (F->getType() != PTy)
-    return ConstantExpr::getBitCast(F, PTy);
+    return {Ty, ConstantExpr::getBitCast(F, PTy)};
 
   // Otherwise, we just found the existing function or a prototype.
-  return F;
+  return {Ty, F};
 }
 
-Constant *Module::getOrInsertFunction(StringRef Name,
-                                      FunctionType *Ty) {
+FunctionCallee Module::getOrInsertFunction(StringRef Name, FunctionType *Ty) {
   return getOrInsertFunction(Name, Ty, AttributeList());
 }
 
@@ -203,16 +201,14 @@ GlobalVariable *Module::getGlobalVariable(StringRef Name,
 ///      with a constantexpr cast to the right type.
 ///   3. Finally, if the existing global is the correct declaration, return the
 ///      existing global.
-Constant *Module::getOrInsertGlobal(StringRef Name, Type *Ty) {
+Constant *Module::getOrInsertGlobal(
+    StringRef Name, Type *Ty,
+    function_ref<GlobalVariable *()> CreateGlobalCallback) {
   // See if we have a definition for the specified global already.
   GlobalVariable *GV = dyn_cast_or_null<GlobalVariable>(getNamedValue(Name));
-  if (!GV) {
-    // Nope, add it
-    GlobalVariable *New =
-      new GlobalVariable(*this, Ty, false, GlobalVariable::ExternalLinkage,
-                         nullptr, Name);
-     return New;                    // Return the new declaration.
-  }
+  if (!GV)
+    GV = CreateGlobalCallback();
+  assert(GV && "The CreateGlobalCallback is expected to create a global");
 
   // If the variable exists but has the wrong type, return a bitcast to the
   // right type.
@@ -225,6 +221,14 @@ Constant *Module::getOrInsertGlobal(StringRef Name, Type *Ty) {
   return GV;
 }
 
+// Overload to construct a global variable using its constructor's defaults.
+Constant *Module::getOrInsertGlobal(StringRef Name, Type *Ty) {
+  return getOrInsertGlobal(Name, Ty, [&] {
+    return new GlobalVariable(*this, Ty, false, GlobalVariable::ExternalLinkage,
+                              nullptr, Name);
+  });
+}
+
 //===----------------------------------------------------------------------===//
 // Methods for easy access to the global variables in the module.
 //
diff --git a/lib/IR/ModuleSummaryIndex.cpp b/lib/IR/ModuleSummaryIndex.cpp
index 46b88cd31779a2097a8d6ddb846568b271ab9f8c..5279cd63c84e23f530c10f2540684fb031080dc1 100644
--- a/lib/IR/ModuleSummaryIndex.cpp
+++ b/lib/IR/ModuleSummaryIndex.cpp
@@ -1,9 +1,8 @@
 //===-- ModuleSummaryIndex.cpp - Module Summary Index ---------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/IR/Operator.cpp b/lib/IR/Operator.cpp
index 5b4c7524b672e40491fbd8617e19e53a2ee4c307..8ba68674d50ed91b72a094f460f5ad9faacc4a7f 100644
--- a/lib/IR/Operator.cpp
+++ b/lib/IR/Operator.cpp
@@ -1,9 +1,8 @@
 //===-- Operator.cpp - Implement the LLVM operators -----------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/IR/OptBisect.cpp b/lib/IR/OptBisect.cpp
index c79e1fc2b0b4f039f375ed379a3f6f550a2762fa..6f4d954d82221807cabed343635d45ce114c3449 100644
--- a/lib/IR/OptBisect.cpp
+++ b/lib/IR/OptBisect.cpp
@@ -1,9 +1,8 @@
 //===- llvm/IR/OptBisect/Bisect.cpp - LLVM Bisect support -----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/IR/Pass.cpp b/lib/IR/Pass.cpp
index a1dc17882493ff9a297451edf0be4eca641cbeeb..e9a0c68495b7046463443191dd8877bf7d3a0e92 100644
--- a/lib/IR/Pass.cpp
+++ b/lib/IR/Pass.cpp
@@ -1,9 +1,8 @@
 //===- Pass.cpp - LLVM Pass Infrastructure Implementation -----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/IR/PassInstrumentation.cpp b/lib/IR/PassInstrumentation.cpp
index 5aa2bc6d895eddc823f13b6835f24e41009b2809..49cc6ec04d90c1afec26a271345c9fd2d5b63ff9 100644
--- a/lib/IR/PassInstrumentation.cpp
+++ b/lib/IR/PassInstrumentation.cpp
@@ -1,9 +1,8 @@
 //===- PassInstrumentation.cpp - Pass Instrumentation interface -*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/lib/IR/PassManager.cpp b/lib/IR/PassManager.cpp
index 47fdfedfdde815836eec739c08e7f3f3354b8150..cde9b873795eb674172473f3ae94bdb35e596b8b 100644
--- a/lib/IR/PassManager.cpp
+++ b/lib/IR/PassManager.cpp
@@ -1,9 +1,8 @@
 //===- PassManager.cpp - Infrastructure for managing & running IR passes --===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/IR/PassRegistry.cpp b/lib/IR/PassRegistry.cpp
index b0f1a9928725f5787adb95468e261b7a1bc9eeb9..92c188b1189830b9b19dbeb4a273d0d67b4258e2 100644
--- a/lib/IR/PassRegistry.cpp
+++ b/lib/IR/PassRegistry.cpp
@@ -1,9 +1,8 @@
 //===- PassRegistry.cpp - Pass Registration Implementation ----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/IR/PassTimingInfo.cpp b/lib/IR/PassTimingInfo.cpp
index 40b3977ecbd9b92dbaea277803af952d4c591e2f..57efdca423fc9ccf5c1f95d2f9c141341fe146be 100644
--- a/lib/IR/PassTimingInfo.cpp
+++ b/lib/IR/PassTimingInfo.cpp
@@ -1,9 +1,8 @@
 //===- PassTimingInfo.cpp - LLVM Pass Timing Implementation ---------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/IR/ProfileSummary.cpp b/lib/IR/ProfileSummary.cpp
index 491fe834df9ab2cf7e0a818fbacd8bc4d0665703..d8596924719bdae9b8768ed372da084ff86fa42a 100644
--- a/lib/IR/ProfileSummary.cpp
+++ b/lib/IR/ProfileSummary.cpp
@@ -1,9 +1,8 @@
 //=-- Profilesummary.cpp - Profile summary support --------------------------=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/IR/SafepointIRVerifier.cpp b/lib/IR/SafepointIRVerifier.cpp
index 3596b31dd25abee549979222a5569e28b23f4f92..217f10e864d15f2fbd70ed57ddc88ac01e77c0d7 100644
--- a/lib/IR/SafepointIRVerifier.cpp
+++ b/lib/IR/SafepointIRVerifier.cpp
@@ -1,9 +1,8 @@
 //===-- SafepointIRVerifier.cpp - Verify gc.statepoint invariants ---------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -257,7 +256,7 @@ static bool containsGCPtrType(Type *Ty) {
   if (ArrayType *AT = dyn_cast<ArrayType>(Ty))
     return containsGCPtrType(AT->getElementType());
   if (StructType *ST = dyn_cast<StructType>(Ty))
-    return llvm::any_of(ST->subtypes(), containsGCPtrType);
+    return llvm::any_of(ST->elements(), containsGCPtrType);
   return false;
 }
 
diff --git a/lib/IR/Statepoint.cpp b/lib/IR/Statepoint.cpp
index 18efee2177c343591e578f6ebca6371c7a8a7f13..fce89b42e9bf6775e645d02f9c993630c51a4fe1 100644
--- a/lib/IR/Statepoint.cpp
+++ b/lib/IR/Statepoint.cpp
@@ -1,9 +1,8 @@
 //===-- IR/Statepoint.cpp -- gc.statepoint utilities ---  -----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -18,21 +17,15 @@
 
 using namespace llvm;
 
-static const Function *getCalledFunction(ImmutableCallSite CS) {
-  if (!CS.getInstruction())
-    return nullptr;
-  return CS.getCalledFunction();
-}
-
-bool llvm::isStatepoint(ImmutableCallSite CS) {
-  if (auto *F = getCalledFunction(CS))
+bool llvm::isStatepoint(const CallBase *Call) {
+  if (auto *F = Call->getCalledFunction())
     return F->getIntrinsicID() == Intrinsic::experimental_gc_statepoint;
   return false;
 }
 
 bool llvm::isStatepoint(const Value *V) {
-  if (auto CS = ImmutableCallSite(V))
-    return isStatepoint(CS);
+  if (auto *Call = dyn_cast<CallBase>(V))
+    return isStatepoint(Call);
   return false;
 }
 
@@ -40,23 +33,21 @@ bool llvm::isStatepoint(const Value &V) {
   return isStatepoint(&V);
 }
 
-bool llvm::isGCRelocate(ImmutableCallSite CS) {
-  return CS.getInstruction() && isa<GCRelocateInst>(CS.getInstruction());
+bool llvm::isGCRelocate(const CallBase *Call) {
+  return isa<GCRelocateInst>(Call);
 }
 
 bool llvm::isGCRelocate(const Value *V) {
-  if (auto CS = ImmutableCallSite(V))
-    return isGCRelocate(CS);
+  if (auto *Call = dyn_cast<CallBase>(V))
+    return isGCRelocate(Call);
   return false;
 }
 
-bool llvm::isGCResult(ImmutableCallSite CS) {
-  return CS.getInstruction() && isa<GCResultInst>(CS.getInstruction());
-}
+bool llvm::isGCResult(const CallBase *Call) { return isa<GCResultInst>(Call); }
 
 bool llvm::isGCResult(const Value *V) {
-  if (auto CS = ImmutableCallSite(V))
-    return isGCResult(CS);
+  if (auto *Call = dyn_cast<CallBase>(V))
+    return isGCResult(Call);
   return false;
 }
 
diff --git a/lib/IR/SymbolTableListTraitsImpl.h b/lib/IR/SymbolTableListTraitsImpl.h
index d4ad1eba33c6c0960cd7687ae9639df9b5058811..f399c823d6fb0bf7638fc0bd30f9bcf398352493 100644
--- a/lib/IR/SymbolTableListTraitsImpl.h
+++ b/lib/IR/SymbolTableListTraitsImpl.h
@@ -1,9 +1,8 @@
 //===-- llvm/SymbolTableListTraitsImpl.h - Implementation ------*- C++ -*--===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -84,7 +83,8 @@ void SymbolTableListTraits<ValueSubClass>::transferNodesFromList(
     SymbolTableListTraits &L2, iterator first, iterator last) {
   // We only have to do work here if transferring instructions between BBs
   ItemParentClass *NewIP = getListOwner(), *OldIP = L2.getListOwner();
-  assert(NewIP != OldIP && "Expected different list owners");
+  if (NewIP == OldIP)
+    return;
 
   // We only have to update symbol table entries if we are transferring the
   // instructions to a different symtab object...
diff --git a/lib/IR/Type.cpp b/lib/IR/Type.cpp
index 0fb079c5ab73956d46d30ec3ed3d67e059bfaa14..c839648a03ed3eec39465eeeba694f8ba8f5d0e2 100644
--- a/lib/IR/Type.cpp
+++ b/lib/IR/Type.cpp
@@ -1,9 +1,8 @@
 //===- Type.cpp - Implement the Type class --------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/IR/TypeFinder.cpp b/lib/IR/TypeFinder.cpp
index e9af78c71bfd0f76676ab815f20c814bc280f43e..2e2c194860cd9c23cd8d69250c7432cb3096f55e 100644
--- a/lib/IR/TypeFinder.cpp
+++ b/lib/IR/TypeFinder.cpp
@@ -1,9 +1,8 @@
 //===- TypeFinder.cpp - Implement the TypeFinder class --------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/IR/Use.cpp b/lib/IR/Use.cpp
index cae845d99fe58fe3a943b4d38566f3339c7af805..18c61757ee8483981bd3bd9421bc7ecf860f90df 100644
--- a/lib/IR/Use.cpp
+++ b/lib/IR/Use.cpp
@@ -1,9 +1,8 @@
 //===-- Use.cpp - Implement the Use class ---------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/IR/User.cpp b/lib/IR/User.cpp
index 041593f20b57467f014926619cf21cabe3e04a69..33a3686c94a132ced51eca7d3ee81aec73d30212 100644
--- a/lib/IR/User.cpp
+++ b/lib/IR/User.cpp
@@ -1,9 +1,8 @@
 //===-- User.cpp - Implement the User class -------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/IR/Value.cpp b/lib/IR/Value.cpp
index dc8af6b68e7bf4deb2a9f945af79ba8ee5138dd5..38eed76fe452c4ddeae4d37fd3d9dfd9390f4b15 100644
--- a/lib/IR/Value.cpp
+++ b/lib/IR/Value.cpp
@@ -1,9 +1,8 @@
 //===-- Value.cpp - Implement the Value class -----------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -16,7 +15,6 @@
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/SetVector.h"
-#include "llvm/IR/CallSite.h"
 #include "llvm/IR/Constant.h"
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/DataLayout.h"
@@ -59,7 +57,8 @@ Value::Value(Type *ty, unsigned scid)
   // FIXME: Why isn't this in the subclass gunk??
   // Note, we cannot call isa<CallInst> before the CallInst has been
   // constructed.
-  if (SubclassID == Instruction::Call || SubclassID == Instruction::Invoke)
+  if (SubclassID == Instruction::Call || SubclassID == Instruction::Invoke ||
+      SubclassID == Instruction::CallBr)
     assert((VTy->isFirstClassType() || VTy->isVoidTy() || VTy->isStructTy()) &&
            "invalid CallInst type!");
   else if (SubclassID != BasicBlockVal &&
@@ -503,8 +502,8 @@ static const Value *stripPointerCastsAndOffsets(const Value *V) {
         return V;
       V = GA->getAliasee();
     } else {
-      if (auto CS = ImmutableCallSite(V)) {
-        if (const Value *RV = CS.getReturnedArgOperand()) {
+      if (const auto *Call = dyn_cast<CallBase>(V)) {
+        if (const Value *RV = Call->getReturnedArgOperand()) {
           V = RV;
           continue;
         }
@@ -512,9 +511,9 @@ static const Value *stripPointerCastsAndOffsets(const Value *V) {
         // but it can't be marked with returned attribute, that's why it needs
         // special case.
         if (StripKind == PSK_ZeroIndicesAndAliasesAndInvariantGroups &&
-            (CS.getIntrinsicID() == Intrinsic::launder_invariant_group ||
-             CS.getIntrinsicID() == Intrinsic::strip_invariant_group)) {
-          V = CS.getArgOperand(0);
+            (Call->getIntrinsicID() == Intrinsic::launder_invariant_group ||
+             Call->getIntrinsicID() == Intrinsic::strip_invariant_group)) {
+          V = Call->getArgOperand(0);
           continue;
         }
       }
@@ -573,8 +572,8 @@ Value::stripAndAccumulateInBoundsConstantOffsets(const DataLayout &DL,
     } else if (auto *GA = dyn_cast<GlobalAlias>(V)) {
       V = GA->getAliasee();
     } else {
-      if (auto CS = ImmutableCallSite(V))
-        if (const Value *RV = CS.getReturnedArgOperand()) {
+      if (const auto *Call = dyn_cast<CallBase>(V))
+        if (const Value *RV = Call->getReturnedArgOperand()) {
           V = RV;
           continue;
         }
@@ -608,10 +607,11 @@ uint64_t Value::getPointerDereferenceableBytes(const DataLayout &DL,
       DerefBytes = A->getDereferenceableOrNullBytes();
       CanBeNull = true;
     }
-  } else if (auto CS = ImmutableCallSite(this)) {
-    DerefBytes = CS.getDereferenceableBytes(AttributeList::ReturnIndex);
+  } else if (const auto *Call = dyn_cast<CallBase>(this)) {
+    DerefBytes = Call->getDereferenceableBytes(AttributeList::ReturnIndex);
     if (DerefBytes == 0) {
-      DerefBytes = CS.getDereferenceableOrNullBytes(AttributeList::ReturnIndex);
+      DerefBytes =
+          Call->getDereferenceableOrNullBytes(AttributeList::ReturnIndex);
       CanBeNull = true;
     }
   } else if (const LoadInst *LI = dyn_cast<LoadInst>(this)) {
@@ -683,8 +683,8 @@ unsigned Value::getPointerAlignment(const DataLayout &DL) const {
       if (AllocatedType->isSized())
         Align = DL.getPrefTypeAlignment(AllocatedType);
     }
-  } else if (auto CS = ImmutableCallSite(this))
-    Align = CS.getAttributes().getRetAlignment();
+  } else if (const auto *Call = dyn_cast<CallBase>(this))
+    Align = Call->getAttributes().getRetAlignment();
   else if (const LoadInst *LI = dyn_cast<LoadInst>(this))
     if (MDNode *MD = LI->getMetadata(LLVMContext::MD_align)) {
       ConstantInt *CI = mdconst::extract<ConstantInt>(MD->getOperand(0));
@@ -931,7 +931,7 @@ void ValueHandleBase::ValueIsRAUWd(Value *Old, Value *New) {
                << Old->getName() << " to " << *New->getType() << " %"
                << New->getName() << "\n";
         llvm_unreachable(
-            "A weak tracking value handle still pointed to the  old value!\n");
+            "A weak tracking value handle still pointed to the old value!\n");
       default:
         break;
       }
diff --git a/lib/IR/ValueSymbolTable.cpp b/lib/IR/ValueSymbolTable.cpp
index f4bea560404383be2b7f232e9c48e27ced15a4aa..417ec045071d1a4c15c8f400ae8d6a077b566a30 100644
--- a/lib/IR/ValueSymbolTable.cpp
+++ b/lib/IR/ValueSymbolTable.cpp
@@ -1,9 +1,8 @@
 //===- ValueSymbolTable.cpp - Implement the ValueSymbolTable class --------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/IR/Verifier.cpp b/lib/IR/Verifier.cpp
index 7fd6df3e6e001f304205ef3db1d7ef774691e817..4a39ff40fd180df8816e3bdbf841d21dc340fcaa 100644
--- a/lib/IR/Verifier.cpp
+++ b/lib/IR/Verifier.cpp
@@ -1,9 +1,8 @@
 //===-- Verifier.cpp - Implement the Module Verifier -----------------------==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -65,7 +64,6 @@
 #include "llvm/IR/Attributes.h"
 #include "llvm/IR/BasicBlock.h"
 #include "llvm/IR/CFG.h"
-#include "llvm/IR/CallSite.h"
 #include "llvm/IR/CallingConv.h"
 #include "llvm/IR/Comdat.h"
 #include "llvm/IR/Constant.h"
@@ -140,21 +138,20 @@ private:
   }
 
   void Write(const Value *V) {
-    if (!V)
-      return;
+    if (V)
+      Write(*V);
+  }
+
+  void Write(const Value &V) {
     if (isa<Instruction>(V)) {
-      V->print(*OS, MST);
+      V.print(*OS, MST);
       *OS << '\n';
     } else {
-      V->printAsOperand(*OS, true, MST);
+      V.printAsOperand(*OS, true, MST);
       *OS << '\n';
     }
   }
 
-  void Write(ImmutableCallSite CS) {
-    Write(CS.getInstruction());
-  }
-
   void Write(const Metadata *MD) {
     if (!MD)
       return;
@@ -448,6 +445,7 @@ private:
   void visitBitCastInst(BitCastInst &I);
   void visitAddrSpaceCastInst(AddrSpaceCastInst &I);
   void visitPHINode(PHINode &PN);
+  void visitCallBase(CallBase &Call);
   void visitUnaryOperator(UnaryOperator &U);
   void visitBinaryOperator(BinaryOperator &B);
   void visitICmpInst(ICmpInst &IC);
@@ -468,10 +466,11 @@ private:
   void visitReturnInst(ReturnInst &RI);
   void visitSwitchInst(SwitchInst &SI);
   void visitIndirectBrInst(IndirectBrInst &BI);
+  void visitCallBrInst(CallBrInst &CBI);
   void visitSelectInst(SelectInst &SI);
   void visitUserOp1(Instruction &I);
   void visitUserOp2(Instruction &I) { visitUserOp1(I); }
-  void visitIntrinsicCallSite(Intrinsic::ID ID, CallSite CS);
+  void visitIntrinsicCall(Intrinsic::ID ID, CallBase &Call);
   void visitConstrainedFPIntrinsic(ConstrainedFPIntrinsic &FPI);
   void visitDbgIntrinsic(StringRef Kind, DbgVariableIntrinsic &DII);
   void visitDbgLabelIntrinsic(StringRef Kind, DbgLabelInst &DLI);
@@ -491,8 +490,7 @@ private:
   void visitCatchSwitchInst(CatchSwitchInst &CatchSwitch);
   void visitCleanupReturnInst(CleanupReturnInst &CRI);
 
-  void verifyCallSite(CallSite CS);
-  void verifySwiftErrorCallSite(CallSite CS, const Value *SwiftErrorVal);
+  void verifySwiftErrorCall(CallBase &Call, const Value *SwiftErrorVal);
   void verifySwiftErrorValue(const Value *SwiftErrorVal);
   void verifyMustTailCall(CallInst &CI);
   bool performTypeCheck(Intrinsic::ID ID, Function *F, Type *Ty, int VT,
@@ -507,7 +505,7 @@ private:
 
   void visitConstantExprsRecursively(const Constant *EntryC);
   void visitConstantExpr(const ConstantExpr *CE);
-  void verifyStatepoint(ImmutableCallSite CS);
+  void verifyStatepoint(const CallBase &Call);
   void verifyFrameRecoverIndices();
   void verifySiblingFuncletUnwinds();
 
@@ -1243,6 +1241,8 @@ void Verifier::visitDILocalVariable(const DILocalVariable &N) {
   AssertDI(N.getTag() == dwarf::DW_TAG_variable, "invalid tag", &N);
   AssertDI(N.getRawScope() && isa<DILocalScope>(N.getRawScope()),
            "local variable requires a valid scope", &N, N.getRawScope());
+  if (auto Ty = N.getType())
+    AssertDI(!isa<DISubroutineType>(Ty), "invalid type", &N, N.getType());
 }
 
 void Verifier::visitDILabel(const DILabel &N) {
@@ -1893,134 +1893,136 @@ bool Verifier::verifyAttributeCount(AttributeList Attrs, unsigned Params) {
 }
 
 /// Verify that statepoint intrinsic is well formed.
-void Verifier::verifyStatepoint(ImmutableCallSite CS) {
-  assert(CS.getCalledFunction() &&
-         CS.getCalledFunction()->getIntrinsicID() ==
-           Intrinsic::experimental_gc_statepoint);
-
-  const Instruction &CI = *CS.getInstruction();
+void Verifier::verifyStatepoint(const CallBase &Call) {
+  assert(Call.getCalledFunction() &&
+         Call.getCalledFunction()->getIntrinsicID() ==
+             Intrinsic::experimental_gc_statepoint);
 
-  Assert(!CS.doesNotAccessMemory() && !CS.onlyReadsMemory() &&
-         !CS.onlyAccessesArgMemory(),
+  Assert(!Call.doesNotAccessMemory() && !Call.onlyReadsMemory() &&
+             !Call.onlyAccessesArgMemory(),
          "gc.statepoint must read and write all memory to preserve "
          "reordering restrictions required by safepoint semantics",
-         &CI);
+         Call);
 
-  const Value *IDV = CS.getArgument(0);
+  const Value *IDV = Call.getArgOperand(0);
   Assert(isa<ConstantInt>(IDV), "gc.statepoint ID must be a constant integer",
-         &CI);
+         Call);
 
-  const Value *NumPatchBytesV = CS.getArgument(1);
+  const Value *NumPatchBytesV = Call.getArgOperand(1);
   Assert(isa<ConstantInt>(NumPatchBytesV),
          "gc.statepoint number of patchable bytes must be a constant integer",
-         &CI);
+         Call);
   const int64_t NumPatchBytes =
       cast<ConstantInt>(NumPatchBytesV)->getSExtValue();
   assert(isInt<32>(NumPatchBytes) && "NumPatchBytesV is an i32!");
-  Assert(NumPatchBytes >= 0, "gc.statepoint number of patchable bytes must be "
-                             "positive",
-         &CI);
+  Assert(NumPatchBytes >= 0,
+         "gc.statepoint number of patchable bytes must be "
+         "positive",
+         Call);
 
-  const Value *Target = CS.getArgument(2);
+  const Value *Target = Call.getArgOperand(2);
   auto *PT = dyn_cast<PointerType>(Target->getType());
   Assert(PT && PT->getElementType()->isFunctionTy(),
-         "gc.statepoint callee must be of function pointer type", &CI, Target);
+         "gc.statepoint callee must be of function pointer type", Call, Target);
   FunctionType *TargetFuncType = cast<FunctionType>(PT->getElementType());
 
-  const Value *NumCallArgsV = CS.getArgument(3);
+  const Value *NumCallArgsV = Call.getArgOperand(3);
   Assert(isa<ConstantInt>(NumCallArgsV),
          "gc.statepoint number of arguments to underlying call "
          "must be constant integer",
-         &CI);
+         Call);
   const int NumCallArgs = cast<ConstantInt>(NumCallArgsV)->getZExtValue();
   Assert(NumCallArgs >= 0,
          "gc.statepoint number of arguments to underlying call "
          "must be positive",
-         &CI);
+         Call);
   const int NumParams = (int)TargetFuncType->getNumParams();
   if (TargetFuncType->isVarArg()) {
     Assert(NumCallArgs >= NumParams,
-           "gc.statepoint mismatch in number of vararg call args", &CI);
+           "gc.statepoint mismatch in number of vararg call args", Call);
 
     // TODO: Remove this limitation
     Assert(TargetFuncType->getReturnType()->isVoidTy(),
            "gc.statepoint doesn't support wrapping non-void "
            "vararg functions yet",
-           &CI);
+           Call);
   } else
     Assert(NumCallArgs == NumParams,
-           "gc.statepoint mismatch in number of call args", &CI);
+           "gc.statepoint mismatch in number of call args", Call);
 
-  const Value *FlagsV = CS.getArgument(4);
+  const Value *FlagsV = Call.getArgOperand(4);
   Assert(isa<ConstantInt>(FlagsV),
-         "gc.statepoint flags must be constant integer", &CI);
+         "gc.statepoint flags must be constant integer", Call);
   const uint64_t Flags = cast<ConstantInt>(FlagsV)->getZExtValue();
   Assert((Flags & ~(uint64_t)StatepointFlags::MaskAll) == 0,
-         "unknown flag used in gc.statepoint flags argument", &CI);
+         "unknown flag used in gc.statepoint flags argument", Call);
 
   // Verify that the types of the call parameter arguments match
   // the type of the wrapped callee.
-  AttributeList Attrs = CS.getAttributes();
+  AttributeList Attrs = Call.getAttributes();
   for (int i = 0; i < NumParams; i++) {
     Type *ParamType = TargetFuncType->getParamType(i);
-    Type *ArgType = CS.getArgument(5 + i)->getType();
+    Type *ArgType = Call.getArgOperand(5 + i)->getType();
     Assert(ArgType == ParamType,
            "gc.statepoint call argument does not match wrapped "
            "function type",
-           &CI);
+           Call);
 
     if (TargetFuncType->isVarArg()) {
       AttributeSet ArgAttrs = Attrs.getParamAttributes(5 + i);
       Assert(!ArgAttrs.hasAttribute(Attribute::StructRet),
-             "Attribute 'sret' cannot be used for vararg call arguments!", &CI);
+             "Attribute 'sret' cannot be used for vararg call arguments!",
+             Call);
     }
   }
 
   const int EndCallArgsInx = 4 + NumCallArgs;
 
-  const Value *NumTransitionArgsV = CS.getArgument(EndCallArgsInx+1);
+  const Value *NumTransitionArgsV = Call.getArgOperand(EndCallArgsInx + 1);
   Assert(isa<ConstantInt>(NumTransitionArgsV),
          "gc.statepoint number of transition arguments "
          "must be constant integer",
-         &CI);
+         Call);
   const int NumTransitionArgs =
       cast<ConstantInt>(NumTransitionArgsV)->getZExtValue();
   Assert(NumTransitionArgs >= 0,
-         "gc.statepoint number of transition arguments must be positive", &CI);
+         "gc.statepoint number of transition arguments must be positive", Call);
   const int EndTransitionArgsInx = EndCallArgsInx + 1 + NumTransitionArgs;
 
-  const Value *NumDeoptArgsV = CS.getArgument(EndTransitionArgsInx+1);
+  const Value *NumDeoptArgsV = Call.getArgOperand(EndTransitionArgsInx + 1);
   Assert(isa<ConstantInt>(NumDeoptArgsV),
          "gc.statepoint number of deoptimization arguments "
          "must be constant integer",
-         &CI);
+         Call);
   const int NumDeoptArgs = cast<ConstantInt>(NumDeoptArgsV)->getZExtValue();
-  Assert(NumDeoptArgs >= 0, "gc.statepoint number of deoptimization arguments "
-                            "must be positive",
-         &CI);
+  Assert(NumDeoptArgs >= 0,
+         "gc.statepoint number of deoptimization arguments "
+         "must be positive",
+         Call);
 
   const int ExpectedNumArgs =
       7 + NumCallArgs + NumTransitionArgs + NumDeoptArgs;
-  Assert(ExpectedNumArgs <= (int)CS.arg_size(),
-         "gc.statepoint too few arguments according to length fields", &CI);
+  Assert(ExpectedNumArgs <= (int)Call.arg_size(),
+         "gc.statepoint too few arguments according to length fields", Call);
 
   // Check that the only uses of this gc.statepoint are gc.result or
   // gc.relocate calls which are tied to this statepoint and thus part
   // of the same statepoint sequence
-  for (const User *U : CI.users()) {
-    const CallInst *Call = dyn_cast<const CallInst>(U);
-    Assert(Call, "illegal use of statepoint token", &CI, U);
-    if (!Call) continue;
-    Assert(isa<GCRelocateInst>(Call) || isa<GCResultInst>(Call),
+  for (const User *U : Call.users()) {
+    const CallInst *UserCall = dyn_cast<const CallInst>(U);
+    Assert(UserCall, "illegal use of statepoint token", Call, U);
+    if (!UserCall)
+      continue;
+    Assert(isa<GCRelocateInst>(UserCall) || isa<GCResultInst>(UserCall),
            "gc.result or gc.relocate are the only value uses "
            "of a gc.statepoint",
-           &CI, U);
-    if (isa<GCResultInst>(Call)) {
-      Assert(Call->getArgOperand(0) == &CI,
-             "gc.result connected to wrong gc.statepoint", &CI, Call);
+           Call, U);
+    if (isa<GCResultInst>(UserCall)) {
+      Assert(UserCall->getArgOperand(0) == &Call,
+             "gc.result connected to wrong gc.statepoint", Call, UserCall);
     } else if (isa<GCRelocateInst>(Call)) {
-      Assert(Call->getArgOperand(0) == &CI,
-             "gc.relocate connected to wrong gc.statepoint", &CI, Call);
+      Assert(UserCall->getArgOperand(0) == &Call,
+             "gc.relocate connected to wrong gc.statepoint", Call, UserCall);
     }
   }
 
@@ -2449,6 +2451,26 @@ void Verifier::visitIndirectBrInst(IndirectBrInst &BI) {
   visitTerminator(BI);
 }
 
+void Verifier::visitCallBrInst(CallBrInst &CBI) {
+  Assert(CBI.isInlineAsm(), "Callbr is currently only used for asm-goto!",
+         &CBI);
+  Assert(CBI.getType()->isVoidTy(), "Callbr return value is not supported!",
+         &CBI);
+  for (unsigned i = 0, e = CBI.getNumSuccessors(); i != e; ++i)
+    Assert(CBI.getSuccessor(i)->getType()->isLabelTy(),
+           "Callbr successors must all have pointer type!", &CBI);
+  for (unsigned i = 0, e = CBI.getNumOperands(); i != e; ++i) {
+    Assert(i >= CBI.getNumArgOperands() || !isa<BasicBlock>(CBI.getOperand(i)),
+           "Using an unescaped label as a callbr argument!", &CBI);
+    if (isa<BasicBlock>(CBI.getOperand(i)))
+      for (unsigned j = i + 1; j != e; ++j)
+        Assert(CBI.getOperand(i) != CBI.getOperand(j),
+               "Duplicate callbr destination!", &CBI);
+  }
+
+  visitTerminator(CBI);
+}
+
 void Verifier::visitSelectInst(SelectInst &SI) {
   Assert(!SelectInst::areInvalidOperands(SI.getOperand(0), SI.getOperand(1),
                                          SI.getOperand(2)),
@@ -2745,77 +2767,79 @@ void Verifier::visitPHINode(PHINode &PN) {
   visitInstruction(PN);
 }
 
-void Verifier::verifyCallSite(CallSite CS) {
-  Instruction *I = CS.getInstruction();
-
-  Assert(CS.getCalledValue()->getType()->isPointerTy(),
-         "Called function must be a pointer!", I);
-  PointerType *FPTy = cast<PointerType>(CS.getCalledValue()->getType());
+void Verifier::visitCallBase(CallBase &Call) {
+  Assert(Call.getCalledValue()->getType()->isPointerTy(),
+         "Called function must be a pointer!", Call);
+  PointerType *FPTy = cast<PointerType>(Call.getCalledValue()->getType());
 
   Assert(FPTy->getElementType()->isFunctionTy(),
-         "Called function is not pointer to function type!", I);
+         "Called function is not pointer to function type!", Call);
 
-  Assert(FPTy->getElementType() == CS.getFunctionType(),
-         "Called function is not the same type as the call!", I);
+  Assert(FPTy->getElementType() == Call.getFunctionType(),
+         "Called function is not the same type as the call!", Call);
 
-  FunctionType *FTy = CS.getFunctionType();
+  FunctionType *FTy = Call.getFunctionType();
 
   // Verify that the correct number of arguments are being passed
   if (FTy->isVarArg())
-    Assert(CS.arg_size() >= FTy->getNumParams(),
-           "Called function requires more parameters than were provided!", I);
+    Assert(Call.arg_size() >= FTy->getNumParams(),
+           "Called function requires more parameters than were provided!",
+           Call);
   else
-    Assert(CS.arg_size() == FTy->getNumParams(),
-           "Incorrect number of arguments passed to called function!", I);
+    Assert(Call.arg_size() == FTy->getNumParams(),
+           "Incorrect number of arguments passed to called function!", Call);
 
   // Verify that all arguments to the call match the function type.
   for (unsigned i = 0, e = FTy->getNumParams(); i != e; ++i)
-    Assert(CS.getArgument(i)->getType() == FTy->getParamType(i),
+    Assert(Call.getArgOperand(i)->getType() == FTy->getParamType(i),
            "Call parameter type does not match function signature!",
-           CS.getArgument(i), FTy->getParamType(i), I);
+           Call.getArgOperand(i), FTy->getParamType(i), Call);
 
-  AttributeList Attrs = CS.getAttributes();
+  AttributeList Attrs = Call.getAttributes();
 
-  Assert(verifyAttributeCount(Attrs, CS.arg_size()),
-         "Attribute after last parameter!", I);
+  Assert(verifyAttributeCount(Attrs, Call.arg_size()),
+         "Attribute after last parameter!", Call);
 
   if (Attrs.hasAttribute(AttributeList::FunctionIndex, Attribute::Speculatable)) {
     // Don't allow speculatable on call sites, unless the underlying function
     // declaration is also speculatable.
-    Function *Callee
-      = dyn_cast<Function>(CS.getCalledValue()->stripPointerCasts());
+    Function *Callee =
+        dyn_cast<Function>(Call.getCalledValue()->stripPointerCasts());
     Assert(Callee && Callee->isSpeculatable(),
-           "speculatable attribute may not apply to call sites", I);
+           "speculatable attribute may not apply to call sites", Call);
   }
 
   // Verify call attributes.
-  verifyFunctionAttrs(FTy, Attrs, I);
+  verifyFunctionAttrs(FTy, Attrs, &Call);
 
   // Conservatively check the inalloca argument.
   // We have a bug if we can find that there is an underlying alloca without
   // inalloca.
-  if (CS.hasInAllocaArgument()) {
-    Value *InAllocaArg = CS.getArgument(FTy->getNumParams() - 1);
+  if (Call.hasInAllocaArgument()) {
+    Value *InAllocaArg = Call.getArgOperand(FTy->getNumParams() - 1);
     if (auto AI = dyn_cast<AllocaInst>(InAllocaArg->stripInBoundsOffsets()))
       Assert(AI->isUsedWithInAlloca(),
-             "inalloca argument for call has mismatched alloca", AI, I);
+             "inalloca argument for call has mismatched alloca", AI, Call);
   }
 
   // For each argument of the callsite, if it has the swifterror argument,
   // make sure the underlying alloca/parameter it comes from has a swifterror as
   // well.
   for (unsigned i = 0, e = FTy->getNumParams(); i != e; ++i)
-    if (CS.paramHasAttr(i, Attribute::SwiftError)) {
-      Value *SwiftErrorArg = CS.getArgument(i);
+    if (Call.paramHasAttr(i, Attribute::SwiftError)) {
+      Value *SwiftErrorArg = Call.getArgOperand(i);
       if (auto AI = dyn_cast<AllocaInst>(SwiftErrorArg->stripInBoundsOffsets())) {
         Assert(AI->isSwiftError(),
-               "swifterror argument for call has mismatched alloca", AI, I);
+               "swifterror argument for call has mismatched alloca", AI, Call);
         continue;
       }
       auto ArgI = dyn_cast<Argument>(SwiftErrorArg);
-      Assert(ArgI, "swifterror argument should come from an alloca or parameter", SwiftErrorArg, I);
+      Assert(ArgI,
+             "swifterror argument should come from an alloca or parameter",
+             SwiftErrorArg, Call);
       Assert(ArgI->hasSwiftErrorAttr(),
-             "swifterror argument for call has mismatched parameter", ArgI, I);
+             "swifterror argument for call has mismatched parameter", ArgI,
+             Call);
     }
 
   if (FTy->isVarArg()) {
@@ -2831,95 +2855,97 @@ void Verifier::verifyCallSite(CallSite CS) {
     }
 
     // Check attributes on the varargs part.
-    for (unsigned Idx = FTy->getNumParams(); Idx < CS.arg_size(); ++Idx) {
-      Type *Ty = CS.getArgument(Idx)->getType();
+    for (unsigned Idx = FTy->getNumParams(); Idx < Call.arg_size(); ++Idx) {
+      Type *Ty = Call.getArgOperand(Idx)->getType();
       AttributeSet ArgAttrs = Attrs.getParamAttributes(Idx);
-      verifyParameterAttrs(ArgAttrs, Ty, I);
+      verifyParameterAttrs(ArgAttrs, Ty, &Call);
 
       if (ArgAttrs.hasAttribute(Attribute::Nest)) {
-        Assert(!SawNest, "More than one parameter has attribute nest!", I);
+        Assert(!SawNest, "More than one parameter has attribute nest!", Call);
         SawNest = true;
       }
 
       if (ArgAttrs.hasAttribute(Attribute::Returned)) {
         Assert(!SawReturned, "More than one parameter has attribute returned!",
-               I);
+               Call);
         Assert(Ty->canLosslesslyBitCastTo(FTy->getReturnType()),
                "Incompatible argument and return types for 'returned' "
                "attribute",
-               I);
+               Call);
         SawReturned = true;
       }
 
       // Statepoint intrinsic is vararg but the wrapped function may be not.
       // Allow sret here and check the wrapped function in verifyStatepoint.
-      if (CS.getCalledFunction() == nullptr ||
-          CS.getCalledFunction()->getIntrinsicID() !=
-            Intrinsic::experimental_gc_statepoint)
+      if (!Call.getCalledFunction() ||
+          Call.getCalledFunction()->getIntrinsicID() !=
+              Intrinsic::experimental_gc_statepoint)
         Assert(!ArgAttrs.hasAttribute(Attribute::StructRet),
-               "Attribute 'sret' cannot be used for vararg call arguments!", I);
+               "Attribute 'sret' cannot be used for vararg call arguments!",
+               Call);
 
       if (ArgAttrs.hasAttribute(Attribute::InAlloca))
-        Assert(Idx == CS.arg_size() - 1, "inalloca isn't on the last argument!",
-               I);
+        Assert(Idx == Call.arg_size() - 1,
+               "inalloca isn't on the last argument!", Call);
     }
   }
 
   // Verify that there's no metadata unless it's a direct call to an intrinsic.
-  if (CS.getCalledFunction() == nullptr ||
-      !CS.getCalledFunction()->getName().startswith("llvm.")) {
+  if (!Call.getCalledFunction() ||
+      !Call.getCalledFunction()->getName().startswith("llvm.")) {
     for (Type *ParamTy : FTy->params()) {
       Assert(!ParamTy->isMetadataTy(),
-             "Function has metadata parameter but isn't an intrinsic", I);
+             "Function has metadata parameter but isn't an intrinsic", Call);
       Assert(!ParamTy->isTokenTy(),
-             "Function has token parameter but isn't an intrinsic", I);
+             "Function has token parameter but isn't an intrinsic", Call);
     }
   }
 
   // Verify that indirect calls don't return tokens.
-  if (CS.getCalledFunction() == nullptr)
+  if (!Call.getCalledFunction())
     Assert(!FTy->getReturnType()->isTokenTy(),
            "Return type cannot be token for indirect call!");
 
-  if (Function *F = CS.getCalledFunction())
+  if (Function *F = Call.getCalledFunction())
     if (Intrinsic::ID ID = (Intrinsic::ID)F->getIntrinsicID())
-      visitIntrinsicCallSite(ID, CS);
+      visitIntrinsicCall(ID, Call);
 
   // Verify that a callsite has at most one "deopt", at most one "funclet" and
   // at most one "gc-transition" operand bundle.
   bool FoundDeoptBundle = false, FoundFuncletBundle = false,
        FoundGCTransitionBundle = false;
-  for (unsigned i = 0, e = CS.getNumOperandBundles(); i < e; ++i) {
-    OperandBundleUse BU = CS.getOperandBundleAt(i);
+  for (unsigned i = 0, e = Call.getNumOperandBundles(); i < e; ++i) {
+    OperandBundleUse BU = Call.getOperandBundleAt(i);
     uint32_t Tag = BU.getTagID();
     if (Tag == LLVMContext::OB_deopt) {
-      Assert(!FoundDeoptBundle, "Multiple deopt operand bundles", I);
+      Assert(!FoundDeoptBundle, "Multiple deopt operand bundles", Call);
       FoundDeoptBundle = true;
     } else if (Tag == LLVMContext::OB_gc_transition) {
       Assert(!FoundGCTransitionBundle, "Multiple gc-transition operand bundles",
-             I);
+             Call);
       FoundGCTransitionBundle = true;
     } else if (Tag == LLVMContext::OB_funclet) {
-      Assert(!FoundFuncletBundle, "Multiple funclet operand bundles", I);
+      Assert(!FoundFuncletBundle, "Multiple funclet operand bundles", Call);
       FoundFuncletBundle = true;
       Assert(BU.Inputs.size() == 1,
-             "Expected exactly one funclet bundle operand", I);
+             "Expected exactly one funclet bundle operand", Call);
       Assert(isa<FuncletPadInst>(BU.Inputs.front()),
              "Funclet bundle operands should correspond to a FuncletPadInst",
-             I);
+             Call);
     }
   }
 
   // Verify that each inlinable callsite of a debug-info-bearing function in a
   // debug-info-bearing function has a debug location attached to it. Failure to
   // do so causes assertion failures when the inliner sets up inline scope info.
-  if (I->getFunction()->getSubprogram() && CS.getCalledFunction() &&
-      CS.getCalledFunction()->getSubprogram())
-    AssertDI(I->getDebugLoc(), "inlinable function call in a function with "
-                               "debug info must have a !dbg location",
-             I);
+  if (Call.getFunction()->getSubprogram() && Call.getCalledFunction() &&
+      Call.getCalledFunction()->getSubprogram())
+    AssertDI(Call.getDebugLoc(),
+             "inlinable function call in a function with "
+             "debug info must have a !dbg location",
+             Call);
 
-  visitInstruction(*I);
+  visitInstruction(Call);
 }
 
 /// Two types are "congruent" if they are identical, or if they are both pointer
@@ -3014,14 +3040,14 @@ void Verifier::verifyMustTailCall(CallInst &CI) {
 }
 
 void Verifier::visitCallInst(CallInst &CI) {
-  verifyCallSite(&CI);
+  visitCallBase(CI);
 
   if (CI.isMustTailCall())
     verifyMustTailCall(CI);
 }
 
 void Verifier::visitInvokeInst(InvokeInst &II) {
-  verifyCallSite(&II);
+  visitCallBase(II);
 
   // Verify that the first non-PHI instruction of the unwind destination is an
   // exception handling instruction.
@@ -3330,16 +3356,15 @@ void Verifier::visitStoreInst(StoreInst &SI) {
 }
 
 /// Check that SwiftErrorVal is used as a swifterror argument in CS.
-void Verifier::verifySwiftErrorCallSite(CallSite CS,
-                                        const Value *SwiftErrorVal) {
+void Verifier::verifySwiftErrorCall(CallBase &Call,
+                                    const Value *SwiftErrorVal) {
   unsigned Idx = 0;
-  for (CallSite::arg_iterator I = CS.arg_begin(), E = CS.arg_end();
-       I != E; ++I, ++Idx) {
+  for (auto I = Call.arg_begin(), E = Call.arg_end(); I != E; ++I, ++Idx) {
     if (*I == SwiftErrorVal) {
-      Assert(CS.paramHasAttr(Idx, Attribute::SwiftError),
+      Assert(Call.paramHasAttr(Idx, Attribute::SwiftError),
              "swifterror value when used in a callsite should be marked "
              "with swifterror attribute",
-              SwiftErrorVal, CS);
+             SwiftErrorVal, Call);
     }
   }
 }
@@ -3358,10 +3383,8 @@ void Verifier::verifySwiftErrorValue(const Value *SwiftErrorVal) {
       Assert(StoreI->getOperand(1) == SwiftErrorVal,
              "swifterror value should be the second operand when used "
              "by stores", SwiftErrorVal, U);
-    if (auto CallI = dyn_cast<CallInst>(U))
-      verifySwiftErrorCallSite(const_cast<CallInst*>(CallI), SwiftErrorVal);
-    if (auto II = dyn_cast<InvokeInst>(U))
-      verifySwiftErrorCallSite(const_cast<InvokeInst*>(II), SwiftErrorVal);
+    if (auto *Call = dyn_cast<CallBase>(U))
+      verifySwiftErrorCall(*const_cast<CallBase *>(Call), SwiftErrorVal);
   }
 }
 
@@ -3428,10 +3451,22 @@ void Verifier::visitAtomicRMWInst(AtomicRMWInst &RMWI) {
   PointerType *PTy = dyn_cast<PointerType>(RMWI.getOperand(0)->getType());
   Assert(PTy, "First atomicrmw operand must be a pointer.", &RMWI);
   Type *ElTy = PTy->getElementType();
-  Assert(ElTy->isIntegerTy(), "atomicrmw " +
-         AtomicRMWInst::getOperationName(Op) +
-         " operand must have integer type!",
-         &RMWI, ElTy);
+  if (Op == AtomicRMWInst::Xchg) {
+    Assert(ElTy->isIntegerTy() || ElTy->isFloatingPointTy(), "atomicrmw " +
+           AtomicRMWInst::getOperationName(Op) +
+           " operand must have integer or floating point type!",
+           &RMWI, ElTy);
+  } else if (AtomicRMWInst::isFPOperation(Op)) {
+    Assert(ElTy->isFloatingPointTy(), "atomicrmw " +
+           AtomicRMWInst::getOperationName(Op) +
+           " operand must have floating point type!",
+           &RMWI, ElTy);
+  } else {
+    Assert(ElTy->isIntegerTy(), "atomicrmw " +
+           AtomicRMWInst::getOperationName(Op) +
+           " operand must have integer type!",
+           &RMWI, ElTy);
+  }
   checkAtomicMemAccessSize(ElTy, &RMWI);
   Assert(ElTy == RMWI.getOperand(1)->getType(),
          "Argument value type does not match pointer operand type!", &RMWI,
@@ -4078,8 +4113,8 @@ void Verifier::visitInstruction(Instruction &I) {
 }
 
 /// Allow intrinsics to be verified in different ways.
-void Verifier::visitIntrinsicCallSite(Intrinsic::ID ID, CallSite CS) {
-  Function *IF = CS.getCalledFunction();
+void Verifier::visitIntrinsicCall(Intrinsic::ID ID, CallBase &Call) {
+  Function *IF = Call.getCalledFunction();
   Assert(IF->isDeclaration(), "Intrinsic functions should never be defined!",
          IF);
 
@@ -4125,15 +4160,15 @@ void Verifier::visitIntrinsicCallSite(Intrinsic::ID ID, CallSite CS) {
 
   // If the intrinsic takes MDNode arguments, verify that they are either global
   // or are local to *this* function.
-  for (Value *V : CS.args())
+  for (Value *V : Call.args())
     if (auto *MD = dyn_cast<MetadataAsValue>(V))
-      visitMetadataAsValue(*MD, CS.getCaller());
+      visitMetadataAsValue(*MD, Call.getCaller());
 
   switch (ID) {
   default:
     break;
   case Intrinsic::coro_id: {
-    auto *InfoArg = CS.getArgOperand(3)->stripPointerCasts();
+    auto *InfoArg = Call.getArgOperand(3)->stripPointerCasts();
     if (isa<ConstantPointerNull>(InfoArg))
       break;
     auto *GV = dyn_cast<GlobalVariable>(InfoArg);
@@ -4148,10 +4183,10 @@ void Verifier::visitIntrinsicCallSite(Intrinsic::ID ID, CallSite CS) {
   }
   case Intrinsic::ctlz:  // llvm.ctlz
   case Intrinsic::cttz:  // llvm.cttz
-    Assert(isa<ConstantInt>(CS.getArgOperand(1)),
+    Assert(isa<ConstantInt>(Call.getArgOperand(1)),
            "is_zero_undef argument of bit counting intrinsics must be a "
            "constant int",
-           CS);
+           Call);
     break;
   case Intrinsic::experimental_constrained_fadd:
   case Intrinsic::experimental_constrained_fsub:
@@ -4177,59 +4212,58 @@ void Verifier::visitIntrinsicCallSite(Intrinsic::ID ID, CallSite CS) {
   case Intrinsic::experimental_constrained_floor:
   case Intrinsic::experimental_constrained_round:
   case Intrinsic::experimental_constrained_trunc:
-    visitConstrainedFPIntrinsic(
-        cast<ConstrainedFPIntrinsic>(*CS.getInstruction()));
+    visitConstrainedFPIntrinsic(cast<ConstrainedFPIntrinsic>(Call));
     break;
   case Intrinsic::dbg_declare: // llvm.dbg.declare
-    Assert(isa<MetadataAsValue>(CS.getArgOperand(0)),
-           "invalid llvm.dbg.declare intrinsic call 1", CS);
-    visitDbgIntrinsic("declare", cast<DbgVariableIntrinsic>(*CS.getInstruction()));
+    Assert(isa<MetadataAsValue>(Call.getArgOperand(0)),
+           "invalid llvm.dbg.declare intrinsic call 1", Call);
+    visitDbgIntrinsic("declare", cast<DbgVariableIntrinsic>(Call));
     break;
   case Intrinsic::dbg_addr: // llvm.dbg.addr
-    visitDbgIntrinsic("addr", cast<DbgVariableIntrinsic>(*CS.getInstruction()));
+    visitDbgIntrinsic("addr", cast<DbgVariableIntrinsic>(Call));
     break;
   case Intrinsic::dbg_value: // llvm.dbg.value
-    visitDbgIntrinsic("value", cast<DbgVariableIntrinsic>(*CS.getInstruction()));
+    visitDbgIntrinsic("value", cast<DbgVariableIntrinsic>(Call));
     break;
   case Intrinsic::dbg_label: // llvm.dbg.label
-    visitDbgLabelIntrinsic("label", cast<DbgLabelInst>(*CS.getInstruction()));
+    visitDbgLabelIntrinsic("label", cast<DbgLabelInst>(Call));
     break;
   case Intrinsic::memcpy:
   case Intrinsic::memmove:
   case Intrinsic::memset: {
-    const auto *MI = cast<MemIntrinsic>(CS.getInstruction());
+    const auto *MI = cast<MemIntrinsic>(&Call);
     auto IsValidAlignment = [&](unsigned Alignment) -> bool {
       return Alignment == 0 || isPowerOf2_32(Alignment);
     };
     Assert(IsValidAlignment(MI->getDestAlignment()),
            "alignment of arg 0 of memory intrinsic must be 0 or a power of 2",
-           CS);
+           Call);
     if (const auto *MTI = dyn_cast<MemTransferInst>(MI)) {
       Assert(IsValidAlignment(MTI->getSourceAlignment()),
              "alignment of arg 1 of memory intrinsic must be 0 or a power of 2",
-             CS);
+             Call);
     }
-    Assert(isa<ConstantInt>(CS.getArgOperand(3)),
+    Assert(isa<ConstantInt>(Call.getArgOperand(3)),
            "isvolatile argument of memory intrinsics must be a constant int",
-           CS);
+           Call);
     break;
   }
   case Intrinsic::memcpy_element_unordered_atomic:
   case Intrinsic::memmove_element_unordered_atomic:
   case Intrinsic::memset_element_unordered_atomic: {
-    const auto *AMI = cast<AtomicMemIntrinsic>(CS.getInstruction());
+    const auto *AMI = cast<AtomicMemIntrinsic>(&Call);
 
     ConstantInt *ElementSizeCI =
         dyn_cast<ConstantInt>(AMI->getRawElementSizeInBytes());
     Assert(ElementSizeCI,
            "element size of the element-wise unordered atomic memory "
            "intrinsic must be a constant int",
-           CS);
+           Call);
     const APInt &ElementSizeVal = ElementSizeCI->getValue();
     Assert(ElementSizeVal.isPowerOf2(),
            "element size of the element-wise atomic memory intrinsic "
            "must be a power of 2",
-           CS);
+           Call);
 
     if (auto *LengthCI = dyn_cast<ConstantInt>(AMI->getLength())) {
       uint64_t Length = LengthCI->getZExtValue();
@@ -4237,7 +4271,7 @@ void Verifier::visitIntrinsicCallSite(Intrinsic::ID ID, CallSite CS) {
       Assert((Length % ElementSize) == 0,
              "constant length must be a multiple of the element size in the "
              "element-wise atomic memory intrinsic",
-             CS);
+             Call);
     }
 
     auto IsValidAlignment = [&](uint64_t Alignment) {
@@ -4245,11 +4279,11 @@ void Verifier::visitIntrinsicCallSite(Intrinsic::ID ID, CallSite CS) {
     };
     uint64_t DstAlignment = AMI->getDestAlignment();
     Assert(IsValidAlignment(DstAlignment),
-           "incorrect alignment of the destination argument", CS);
+           "incorrect alignment of the destination argument", Call);
     if (const auto *AMT = dyn_cast<AtomicMemTransferInst>(AMI)) {
       uint64_t SrcAlignment = AMT->getSourceAlignment();
       Assert(IsValidAlignment(SrcAlignment),
-             "incorrect alignment of the source argument", CS);
+             "incorrect alignment of the source argument", Call);
     }
     break;
   }
@@ -4258,76 +4292,76 @@ void Verifier::visitIntrinsicCallSite(Intrinsic::ID ID, CallSite CS) {
   case Intrinsic::gcread:
     if (ID == Intrinsic::gcroot) {
       AllocaInst *AI =
-        dyn_cast<AllocaInst>(CS.getArgOperand(0)->stripPointerCasts());
-      Assert(AI, "llvm.gcroot parameter #1 must be an alloca.", CS);
-      Assert(isa<Constant>(CS.getArgOperand(1)),
-             "llvm.gcroot parameter #2 must be a constant.", CS);
+          dyn_cast<AllocaInst>(Call.getArgOperand(0)->stripPointerCasts());
+      Assert(AI, "llvm.gcroot parameter #1 must be an alloca.", Call);
+      Assert(isa<Constant>(Call.getArgOperand(1)),
+             "llvm.gcroot parameter #2 must be a constant.", Call);
       if (!AI->getAllocatedType()->isPointerTy()) {
-        Assert(!isa<ConstantPointerNull>(CS.getArgOperand(1)),
+        Assert(!isa<ConstantPointerNull>(Call.getArgOperand(1)),
                "llvm.gcroot parameter #1 must either be a pointer alloca, "
                "or argument #2 must be a non-null constant.",
-               CS);
+               Call);
       }
     }
 
-    Assert(CS.getParent()->getParent()->hasGC(),
-           "Enclosing function does not use GC.", CS);
+    Assert(Call.getParent()->getParent()->hasGC(),
+           "Enclosing function does not use GC.", Call);
     break;
   case Intrinsic::init_trampoline:
-    Assert(isa<Function>(CS.getArgOperand(1)->stripPointerCasts()),
+    Assert(isa<Function>(Call.getArgOperand(1)->stripPointerCasts()),
            "llvm.init_trampoline parameter #2 must resolve to a function.",
-           CS);
+           Call);
     break;
   case Intrinsic::prefetch:
-    Assert(isa<ConstantInt>(CS.getArgOperand(1)) &&
-               isa<ConstantInt>(CS.getArgOperand(2)) &&
-               cast<ConstantInt>(CS.getArgOperand(1))->getZExtValue() < 2 &&
-               cast<ConstantInt>(CS.getArgOperand(2))->getZExtValue() < 4,
-           "invalid arguments to llvm.prefetch", CS);
+    Assert(isa<ConstantInt>(Call.getArgOperand(1)) &&
+               isa<ConstantInt>(Call.getArgOperand(2)) &&
+               cast<ConstantInt>(Call.getArgOperand(1))->getZExtValue() < 2 &&
+               cast<ConstantInt>(Call.getArgOperand(2))->getZExtValue() < 4,
+           "invalid arguments to llvm.prefetch", Call);
     break;
   case Intrinsic::stackprotector:
-    Assert(isa<AllocaInst>(CS.getArgOperand(1)->stripPointerCasts()),
-           "llvm.stackprotector parameter #2 must resolve to an alloca.", CS);
+    Assert(isa<AllocaInst>(Call.getArgOperand(1)->stripPointerCasts()),
+           "llvm.stackprotector parameter #2 must resolve to an alloca.", Call);
     break;
   case Intrinsic::lifetime_start:
   case Intrinsic::lifetime_end:
   case Intrinsic::invariant_start:
-    Assert(isa<ConstantInt>(CS.getArgOperand(0)),
+    Assert(isa<ConstantInt>(Call.getArgOperand(0)),
            "size argument of memory use markers must be a constant integer",
-           CS);
+           Call);
     break;
   case Intrinsic::invariant_end:
-    Assert(isa<ConstantInt>(CS.getArgOperand(1)),
-           "llvm.invariant.end parameter #2 must be a constant integer", CS);
+    Assert(isa<ConstantInt>(Call.getArgOperand(1)),
+           "llvm.invariant.end parameter #2 must be a constant integer", Call);
     break;
 
   case Intrinsic::localescape: {
-    BasicBlock *BB = CS.getParent();
+    BasicBlock *BB = Call.getParent();
     Assert(BB == &BB->getParent()->front(),
-           "llvm.localescape used outside of entry block", CS);
+           "llvm.localescape used outside of entry block", Call);
     Assert(!SawFrameEscape,
-           "multiple calls to llvm.localescape in one function", CS);
-    for (Value *Arg : CS.args()) {
+           "multiple calls to llvm.localescape in one function", Call);
+    for (Value *Arg : Call.args()) {
       if (isa<ConstantPointerNull>(Arg))
         continue; // Null values are allowed as placeholders.
       auto *AI = dyn_cast<AllocaInst>(Arg->stripPointerCasts());
       Assert(AI && AI->isStaticAlloca(),
-             "llvm.localescape only accepts static allocas", CS);
+             "llvm.localescape only accepts static allocas", Call);
     }
-    FrameEscapeInfo[BB->getParent()].first = CS.getNumArgOperands();
+    FrameEscapeInfo[BB->getParent()].first = Call.getNumArgOperands();
     SawFrameEscape = true;
     break;
   }
   case Intrinsic::localrecover: {
-    Value *FnArg = CS.getArgOperand(0)->stripPointerCasts();
+    Value *FnArg = Call.getArgOperand(0)->stripPointerCasts();
     Function *Fn = dyn_cast<Function>(FnArg);
     Assert(Fn && !Fn->isDeclaration(),
            "llvm.localrecover first "
            "argument must be function defined in this module",
-           CS);
-    auto *IdxArg = dyn_cast<ConstantInt>(CS.getArgOperand(2));
+           Call);
+    auto *IdxArg = dyn_cast<ConstantInt>(Call.getArgOperand(2));
     Assert(IdxArg, "idx argument of llvm.localrecover must be a constant int",
-           CS);
+           Call);
     auto &Entry = FrameEscapeInfo[Fn];
     Entry.second = unsigned(
         std::max(uint64_t(Entry.second), IdxArg->getLimitedValue(~0U) + 1));
@@ -4335,45 +4369,46 @@ void Verifier::visitIntrinsicCallSite(Intrinsic::ID ID, CallSite CS) {
   }
 
   case Intrinsic::experimental_gc_statepoint:
-    Assert(!CS.isInlineAsm(),
-           "gc.statepoint support for inline assembly unimplemented", CS);
-    Assert(CS.getParent()->getParent()->hasGC(),
-           "Enclosing function does not use GC.", CS);
+    if (auto *CI = dyn_cast<CallInst>(&Call))
+      Assert(!CI->isInlineAsm(),
+             "gc.statepoint support for inline assembly unimplemented", CI);
+    Assert(Call.getParent()->getParent()->hasGC(),
+           "Enclosing function does not use GC.", Call);
 
-    verifyStatepoint(CS);
+    verifyStatepoint(Call);
     break;
   case Intrinsic::experimental_gc_result: {
-    Assert(CS.getParent()->getParent()->hasGC(),
-           "Enclosing function does not use GC.", CS);
+    Assert(Call.getParent()->getParent()->hasGC(),
+           "Enclosing function does not use GC.", Call);
     // Are we tied to a statepoint properly?
-    CallSite StatepointCS(CS.getArgOperand(0));
+    const auto *StatepointCall = dyn_cast<CallBase>(Call.getArgOperand(0));
     const Function *StatepointFn =
-      StatepointCS.getInstruction() ? StatepointCS.getCalledFunction() : nullptr;
+        StatepointCall ? StatepointCall->getCalledFunction() : nullptr;
     Assert(StatepointFn && StatepointFn->isDeclaration() &&
                StatepointFn->getIntrinsicID() ==
                    Intrinsic::experimental_gc_statepoint,
-           "gc.result operand #1 must be from a statepoint", CS,
-           CS.getArgOperand(0));
+           "gc.result operand #1 must be from a statepoint", Call,
+           Call.getArgOperand(0));
 
     // Assert that result type matches wrapped callee.
-    const Value *Target = StatepointCS.getArgument(2);
+    const Value *Target = StatepointCall->getArgOperand(2);
     auto *PT = cast<PointerType>(Target->getType());
     auto *TargetFuncType = cast<FunctionType>(PT->getElementType());
-    Assert(CS.getType() == TargetFuncType->getReturnType(),
-           "gc.result result type does not match wrapped callee", CS);
+    Assert(Call.getType() == TargetFuncType->getReturnType(),
+           "gc.result result type does not match wrapped callee", Call);
     break;
   }
   case Intrinsic::experimental_gc_relocate: {
-    Assert(CS.getNumArgOperands() == 3, "wrong number of arguments", CS);
+    Assert(Call.getNumArgOperands() == 3, "wrong number of arguments", Call);
 
-    Assert(isa<PointerType>(CS.getType()->getScalarType()),
-           "gc.relocate must return a pointer or a vector of pointers", CS);
+    Assert(isa<PointerType>(Call.getType()->getScalarType()),
+           "gc.relocate must return a pointer or a vector of pointers", Call);
 
     // Check that this relocate is correctly tied to the statepoint
 
     // This is case for relocate on the unwinding path of an invoke statepoint
     if (LandingPadInst *LandingPad =
-          dyn_cast<LandingPadInst>(CS.getArgOperand(0))) {
+            dyn_cast<LandingPadInst>(Call.getArgOperand(0))) {
 
       const BasicBlock *InvokeBB =
           LandingPad->getParent()->getUniquePredecessor();
@@ -4386,161 +4421,160 @@ void Verifier::visitIntrinsicCallSite(Intrinsic::ID ID, CallSite CS) {
              InvokeBB);
       Assert(isStatepoint(InvokeBB->getTerminator()),
              "gc relocate should be linked to a statepoint", InvokeBB);
-    }
-    else {
+    } else {
       // In all other cases relocate should be tied to the statepoint directly.
       // This covers relocates on a normal return path of invoke statepoint and
       // relocates of a call statepoint.
-      auto Token = CS.getArgOperand(0);
+      auto Token = Call.getArgOperand(0);
       Assert(isa<Instruction>(Token) && isStatepoint(cast<Instruction>(Token)),
-             "gc relocate is incorrectly tied to the statepoint", CS, Token);
+             "gc relocate is incorrectly tied to the statepoint", Call, Token);
     }
 
     // Verify rest of the relocate arguments.
-
-    ImmutableCallSite StatepointCS(
-        cast<GCRelocateInst>(*CS.getInstruction()).getStatepoint());
+    const CallBase &StatepointCall =
+        *cast<CallBase>(cast<GCRelocateInst>(Call).getStatepoint());
 
     // Both the base and derived must be piped through the safepoint.
-    Value* Base = CS.getArgOperand(1);
+    Value *Base = Call.getArgOperand(1);
     Assert(isa<ConstantInt>(Base),
-           "gc.relocate operand #2 must be integer offset", CS);
+           "gc.relocate operand #2 must be integer offset", Call);
 
-    Value* Derived = CS.getArgOperand(2);
+    Value *Derived = Call.getArgOperand(2);
     Assert(isa<ConstantInt>(Derived),
-           "gc.relocate operand #3 must be integer offset", CS);
+           "gc.relocate operand #3 must be integer offset", Call);
 
     const int BaseIndex = cast<ConstantInt>(Base)->getZExtValue();
     const int DerivedIndex = cast<ConstantInt>(Derived)->getZExtValue();
     // Check the bounds
-    Assert(0 <= BaseIndex && BaseIndex < (int)StatepointCS.arg_size(),
-           "gc.relocate: statepoint base index out of bounds", CS);
-    Assert(0 <= DerivedIndex && DerivedIndex < (int)StatepointCS.arg_size(),
-           "gc.relocate: statepoint derived index out of bounds", CS);
+    Assert(0 <= BaseIndex && BaseIndex < (int)StatepointCall.arg_size(),
+           "gc.relocate: statepoint base index out of bounds", Call);
+    Assert(0 <= DerivedIndex && DerivedIndex < (int)StatepointCall.arg_size(),
+           "gc.relocate: statepoint derived index out of bounds", Call);
 
     // Check that BaseIndex and DerivedIndex fall within the 'gc parameters'
     // section of the statepoint's argument.
-    Assert(StatepointCS.arg_size() > 0,
+    Assert(StatepointCall.arg_size() > 0,
            "gc.statepoint: insufficient arguments");
-    Assert(isa<ConstantInt>(StatepointCS.getArgument(3)),
+    Assert(isa<ConstantInt>(StatepointCall.getArgOperand(3)),
            "gc.statement: number of call arguments must be constant integer");
     const unsigned NumCallArgs =
-        cast<ConstantInt>(StatepointCS.getArgument(3))->getZExtValue();
-    Assert(StatepointCS.arg_size() > NumCallArgs + 5,
+        cast<ConstantInt>(StatepointCall.getArgOperand(3))->getZExtValue();
+    Assert(StatepointCall.arg_size() > NumCallArgs + 5,
            "gc.statepoint: mismatch in number of call arguments");
-    Assert(isa<ConstantInt>(StatepointCS.getArgument(NumCallArgs + 5)),
+    Assert(isa<ConstantInt>(StatepointCall.getArgOperand(NumCallArgs + 5)),
            "gc.statepoint: number of transition arguments must be "
            "a constant integer");
     const int NumTransitionArgs =
-        cast<ConstantInt>(StatepointCS.getArgument(NumCallArgs + 5))
+        cast<ConstantInt>(StatepointCall.getArgOperand(NumCallArgs + 5))
             ->getZExtValue();
     const int DeoptArgsStart = 4 + NumCallArgs + 1 + NumTransitionArgs + 1;
-    Assert(isa<ConstantInt>(StatepointCS.getArgument(DeoptArgsStart)),
+    Assert(isa<ConstantInt>(StatepointCall.getArgOperand(DeoptArgsStart)),
            "gc.statepoint: number of deoptimization arguments must be "
            "a constant integer");
     const int NumDeoptArgs =
-        cast<ConstantInt>(StatepointCS.getArgument(DeoptArgsStart))
+        cast<ConstantInt>(StatepointCall.getArgOperand(DeoptArgsStart))
             ->getZExtValue();
     const int GCParamArgsStart = DeoptArgsStart + 1 + NumDeoptArgs;
-    const int GCParamArgsEnd = StatepointCS.arg_size();
+    const int GCParamArgsEnd = StatepointCall.arg_size();
     Assert(GCParamArgsStart <= BaseIndex && BaseIndex < GCParamArgsEnd,
            "gc.relocate: statepoint base index doesn't fall within the "
            "'gc parameters' section of the statepoint call",
-           CS);
+           Call);
     Assert(GCParamArgsStart <= DerivedIndex && DerivedIndex < GCParamArgsEnd,
            "gc.relocate: statepoint derived index doesn't fall within the "
            "'gc parameters' section of the statepoint call",
-           CS);
+           Call);
 
     // Relocated value must be either a pointer type or vector-of-pointer type,
     // but gc_relocate does not need to return the same pointer type as the
     // relocated pointer. It can be casted to the correct type later if it's
     // desired. However, they must have the same address space and 'vectorness'
-    GCRelocateInst &Relocate = cast<GCRelocateInst>(*CS.getInstruction());
+    GCRelocateInst &Relocate = cast<GCRelocateInst>(Call);
     Assert(Relocate.getDerivedPtr()->getType()->isPtrOrPtrVectorTy(),
-           "gc.relocate: relocated value must be a gc pointer", CS);
+           "gc.relocate: relocated value must be a gc pointer", Call);
 
-    auto ResultType = CS.getType();
+    auto ResultType = Call.getType();
     auto DerivedType = Relocate.getDerivedPtr()->getType();
     Assert(ResultType->isVectorTy() == DerivedType->isVectorTy(),
            "gc.relocate: vector relocates to vector and pointer to pointer",
-           CS);
+           Call);
     Assert(
         ResultType->getPointerAddressSpace() ==
             DerivedType->getPointerAddressSpace(),
         "gc.relocate: relocating a pointer shouldn't change its address space",
-        CS);
+        Call);
     break;
   }
   case Intrinsic::eh_exceptioncode:
   case Intrinsic::eh_exceptionpointer: {
-    Assert(isa<CatchPadInst>(CS.getArgOperand(0)),
-           "eh.exceptionpointer argument must be a catchpad", CS);
+    Assert(isa<CatchPadInst>(Call.getArgOperand(0)),
+           "eh.exceptionpointer argument must be a catchpad", Call);
     break;
   }
   case Intrinsic::masked_load: {
-    Assert(CS.getType()->isVectorTy(), "masked_load: must return a vector", CS);
+    Assert(Call.getType()->isVectorTy(), "masked_load: must return a vector",
+           Call);
 
-    Value *Ptr = CS.getArgOperand(0);
-    //Value *Alignment = CS.getArgOperand(1);
-    Value *Mask = CS.getArgOperand(2);
-    Value *PassThru = CS.getArgOperand(3);
-    Assert(Mask->getType()->isVectorTy(),
-           "masked_load: mask must be vector", CS);
+    Value *Ptr = Call.getArgOperand(0);
+    // Value *Alignment = Call.getArgOperand(1);
+    Value *Mask = Call.getArgOperand(2);
+    Value *PassThru = Call.getArgOperand(3);
+    Assert(Mask->getType()->isVectorTy(), "masked_load: mask must be vector",
+           Call);
 
     // DataTy is the overloaded type
     Type *DataTy = cast<PointerType>(Ptr->getType())->getElementType();
-    Assert(DataTy == CS.getType(),
-           "masked_load: return must match pointer type", CS);
+    Assert(DataTy == Call.getType(),
+           "masked_load: return must match pointer type", Call);
     Assert(PassThru->getType() == DataTy,
-           "masked_load: pass through and data type must match", CS);
+           "masked_load: pass through and data type must match", Call);
     Assert(Mask->getType()->getVectorNumElements() ==
-           DataTy->getVectorNumElements(),
-           "masked_load: vector mask must be same length as data", CS);
+               DataTy->getVectorNumElements(),
+           "masked_load: vector mask must be same length as data", Call);
     break;
   }
   case Intrinsic::masked_store: {
-    Value *Val = CS.getArgOperand(0);
-    Value *Ptr = CS.getArgOperand(1);
-    //Value *Alignment = CS.getArgOperand(2);
-    Value *Mask = CS.getArgOperand(3);
-    Assert(Mask->getType()->isVectorTy(),
-           "masked_store: mask must be vector", CS);
+    Value *Val = Call.getArgOperand(0);
+    Value *Ptr = Call.getArgOperand(1);
+    // Value *Alignment = Call.getArgOperand(2);
+    Value *Mask = Call.getArgOperand(3);
+    Assert(Mask->getType()->isVectorTy(), "masked_store: mask must be vector",
+           Call);
 
     // DataTy is the overloaded type
     Type *DataTy = cast<PointerType>(Ptr->getType())->getElementType();
     Assert(DataTy == Val->getType(),
-           "masked_store: storee must match pointer type", CS);
+           "masked_store: storee must match pointer type", Call);
     Assert(Mask->getType()->getVectorNumElements() ==
-           DataTy->getVectorNumElements(),
-           "masked_store: vector mask must be same length as data", CS);
+               DataTy->getVectorNumElements(),
+           "masked_store: vector mask must be same length as data", Call);
     break;
   }
 
   case Intrinsic::experimental_guard: {
-    Assert(CS.isCall(), "experimental_guard cannot be invoked", CS);
-    Assert(CS.countOperandBundlesOfType(LLVMContext::OB_deopt) == 1,
+    Assert(isa<CallInst>(Call), "experimental_guard cannot be invoked", Call);
+    Assert(Call.countOperandBundlesOfType(LLVMContext::OB_deopt) == 1,
            "experimental_guard must have exactly one "
            "\"deopt\" operand bundle");
     break;
   }
 
   case Intrinsic::experimental_deoptimize: {
-    Assert(CS.isCall(), "experimental_deoptimize cannot be invoked", CS);
-    Assert(CS.countOperandBundlesOfType(LLVMContext::OB_deopt) == 1,
+    Assert(isa<CallInst>(Call), "experimental_deoptimize cannot be invoked",
+           Call);
+    Assert(Call.countOperandBundlesOfType(LLVMContext::OB_deopt) == 1,
            "experimental_deoptimize must have exactly one "
            "\"deopt\" operand bundle");
-    Assert(CS.getType() == CS.getInstruction()->getFunction()->getReturnType(),
+    Assert(Call.getType() == Call.getFunction()->getReturnType(),
            "experimental_deoptimize return type must match caller return type");
 
-    if (CS.isCall()) {
-      auto *DeoptCI = CS.getInstruction();
-      auto *RI = dyn_cast<ReturnInst>(DeoptCI->getNextNode());
+    if (isa<CallInst>(Call)) {
+      auto *RI = dyn_cast<ReturnInst>(Call.getNextNode());
       Assert(RI,
              "calls to experimental_deoptimize must be followed by a return");
 
-      if (!CS.getType()->isVoidTy() && RI)
-        Assert(RI->getReturnValue() == DeoptCI,
+      if (!Call.getType()->isVoidTy() && RI)
+        Assert(RI->getReturnValue() == &Call,
                "calls to experimental_deoptimize must be followed by a return "
                "of the value computed by experimental_deoptimize");
     }
@@ -4551,8 +4585,8 @@ void Verifier::visitIntrinsicCallSite(Intrinsic::ID ID, CallSite CS) {
   case Intrinsic::uadd_sat:
   case Intrinsic::ssub_sat:
   case Intrinsic::usub_sat: {
-    Value *Op1 = CS.getArgOperand(0);
-    Value *Op2 = CS.getArgOperand(1);
+    Value *Op1 = Call.getArgOperand(0);
+    Value *Op2 = Call.getArgOperand(1);
     Assert(Op1->getType()->isIntOrIntVectorTy(),
            "first operand of [us][add|sub]_sat must be an int type or vector "
            "of ints");
@@ -4561,22 +4595,31 @@ void Verifier::visitIntrinsicCallSite(Intrinsic::ID ID, CallSite CS) {
            "of ints");
     break;
   }
-  case Intrinsic::smul_fix: {
-    Value *Op1 = CS.getArgOperand(0);
-    Value *Op2 = CS.getArgOperand(1);
+  case Intrinsic::smul_fix:
+  case Intrinsic::umul_fix: {
+    Value *Op1 = Call.getArgOperand(0);
+    Value *Op2 = Call.getArgOperand(1);
     Assert(Op1->getType()->isIntOrIntVectorTy(),
-           "first operand of smul_fix must be an int type or vector "
+           "first operand of [us]mul_fix must be an int type or vector "
            "of ints");
     Assert(Op2->getType()->isIntOrIntVectorTy(),
-           "second operand of smul_fix must be an int type or vector "
+           "second operand of [us]mul_fix must be an int type or vector "
            "of ints");
 
-    auto *Op3 = dyn_cast<ConstantInt>(CS.getArgOperand(2));
-    Assert(Op3, "third argument of smul_fix must be a constant integer");
+    auto *Op3 = dyn_cast<ConstantInt>(Call.getArgOperand(2));
+    Assert(Op3, "third argument of [us]mul_fix must be a constant integer");
     Assert(Op3->getType()->getBitWidth() <= 32,
-           "third argument of smul_fix must fit within 32 bits");
-    Assert(Op3->getZExtValue() < Op1->getType()->getScalarSizeInBits(),
-           "the scale of smul_fix must be less than the width of the operands");
+           "third argument of [us]mul_fix must fit within 32 bits");
+
+    if (ID == Intrinsic::smul_fix) {
+      Assert(
+          Op3->getZExtValue() < Op1->getType()->getScalarSizeInBits(),
+          "the scale of smul_fix must be less than the width of the operands");
+    } else {
+      Assert(Op3->getZExtValue() <= Op1->getType()->getScalarSizeInBits(),
+             "the scale of umul_fix must be less than or equal to the width of "
+             "the operands");
+    }
     break;
   }
   };
diff --git a/lib/IRReader/IRReader.cpp b/lib/IRReader/IRReader.cpp
index 36bbf719bb61087f6997bcd5236ec17c9b1818e4..7ca6c2fca52a2959e194726a9c322f0e5c38e34a 100644
--- a/lib/IRReader/IRReader.cpp
+++ b/lib/IRReader/IRReader.cpp
@@ -1,9 +1,8 @@
 //===---- IRReader.cpp - Reader for LLVM IR files -------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -30,9 +29,9 @@ static const char *const TimeIRParsingGroupDescription = "LLVM IR Parsing";
 static const char *const TimeIRParsingName = "parse";
 static const char *const TimeIRParsingDescription = "Parse IR";
 
-static std::unique_ptr<Module>
-getLazyIRModule(std::unique_ptr<MemoryBuffer> Buffer, SMDiagnostic &Err,
-                LLVMContext &Context, bool ShouldLazyLoadMetadata) {
+std::unique_ptr<Module>
+llvm::getLazyIRModule(std::unique_ptr<MemoryBuffer> Buffer, SMDiagnostic &Err,
+                      LLVMContext &Context, bool ShouldLazyLoadMetadata) {
   if (isBitcode((const unsigned char *)Buffer->getBufferStart(),
                 (const unsigned char *)Buffer->getBufferEnd())) {
     Expected<std::unique_ptr<Module>> ModuleOrErr = getOwningLazyBitcodeModule(
diff --git a/lib/IRReader/LLVMBuild.txt b/lib/IRReader/LLVMBuild.txt
index b7bc74d6164987040b8d0e537be3759ad5201f9f..1e5fc763639682f8e8b9deafde7460e544452eac 100644
--- a/lib/IRReader/LLVMBuild.txt
+++ b/lib/IRReader/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/IRReader/LLVMBuild.txt -----------------------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/LLVMBuild.txt b/lib/LLVMBuild.txt
index d87bf797161bea615d6f2402bda9f3d9849f1f05..838624b02a0bd5a6a38a39a712c00431bb6fbc65 100644
--- a/lib/LLVMBuild.txt
+++ b/lib/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/LLVMBuild.txt --------------------------------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/LTO/Caching.cpp b/lib/LTO/Caching.cpp
index 089e77e742eb037e69d226e1ac0b339e26f4a8c4..9971a28aed497a5db285be8f3a6a9c761e7f8073 100644
--- a/lib/LTO/Caching.cpp
+++ b/lib/LTO/Caching.cpp
@@ -1,9 +1,8 @@
 //===-Caching.cpp - LLVM Link Time Optimizer Cache Handling ---------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/LTO/LLVMBuild.txt b/lib/LTO/LLVMBuild.txt
index a1993314e36906894822676b6b6ec68161ad675a..ed2b150ddaf992254581ebd754e47461aec9f238 100644
--- a/lib/LTO/LLVMBuild.txt
+++ b/lib/LTO/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/LTO/LLVMBuild.txt ----------------------------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/LTO/LTO.cpp b/lib/LTO/LTO.cpp
index 08924fb92dd6a4cb4ee516d88a23b27da5f002f5..23a4a4011f7fa05a5447f53fd18ea357a25e18c0 100644
--- a/lib/LTO/LTO.cpp
+++ b/lib/LTO/LTO.cpp
@@ -1,9 +1,8 @@
 //===-LTO.cpp - LLVM Link Time Optimizer ----------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -546,6 +545,15 @@ Error LTO::addModule(InputFile &Input, unsigned ModI,
   if (!LTOInfo)
     return LTOInfo.takeError();
 
+  if (EnableSplitLTOUnit.hasValue()) {
+    // If only some modules were split, flag this in the index so that
+    // we can skip or error on optimizations that need consistently split
+    // modules (whole program devirt and lower type tests).
+    if (EnableSplitLTOUnit.getValue() != LTOInfo->EnableSplitLTOUnit)
+      ThinLTO.CombinedIndex.setPartiallySplitLTOUnits();
+  } else
+    EnableSplitLTOUnit = LTOInfo->EnableSplitLTOUnit;
+
   BitcodeModule BM = Input.Mods[ModI];
   auto ModSyms = Input.module_symbols(ModI);
   addModuleToGlobalRes(ModSyms, {ResI, ResE},
@@ -693,8 +701,12 @@ LTO::addRegularLTO(BitcodeModule BM, ArrayRef<InputFile::Symbol> Syms,
       }
 
       // Set the 'local' flag based on the linker resolution for this symbol.
-      if (Res.FinalDefinitionInLinkageUnit)
+      if (Res.FinalDefinitionInLinkageUnit) {
         GV->setDSOLocal(true);
+        if (GV->hasDLLImportStorageClass())
+          GV->setDLLStorageClass(GlobalValue::DLLStorageClassTypes::
+                                 DefaultStorageClass);
+      }
     }
     // Common resolution: collect the maximum size/alignment over all commons.
     // We also record if we see an instance of a common as prevailing, so that
diff --git a/lib/LTO/LTOBackend.cpp b/lib/LTO/LTOBackend.cpp
index 926c419e34a84c35a3b9470e7ed29758c420e39e..02e51fbfa35734d2bce29c9508a816bebe10bde8 100644
--- a/lib/LTO/LTOBackend.cpp
+++ b/lib/LTO/LTOBackend.cpp
@@ -1,9 +1,8 @@
 //===-LTOBackend.cpp - LLVM Link Time Optimizer Backend -------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/LTO/LTOCodeGenerator.cpp b/lib/LTO/LTOCodeGenerator.cpp
index 3b63bbc7e25685c70973e1488cf7d2a932ba05e4..1b1de2ba187fa2de0bea52e876b4e817baa1eac8 100644
--- a/lib/LTO/LTOCodeGenerator.cpp
+++ b/lib/LTO/LTOCodeGenerator.cpp
@@ -1,9 +1,8 @@
 //===-LTOCodeGenerator.cpp - LLVM Link Time Optimizer ---------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/LTO/LTOModule.cpp b/lib/LTO/LTOModule.cpp
index 0d40d49dbe394e3281164a1ef999d269c6933162..eb46b31d211e5843a829a078beaba74358f66eb3 100644
--- a/lib/LTO/LTOModule.cpp
+++ b/lib/LTO/LTOModule.cpp
@@ -1,9 +1,8 @@
 //===-- LTOModule.cpp - LLVM Link Time Optimizer --------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/LTO/SummaryBasedOptimizations.cpp b/lib/LTO/SummaryBasedOptimizations.cpp
index 8b1abb78462b1ab9044bae7e72c8f31a98f4cd67..e919fd530fb012c71caf65632a00675954347058 100644
--- a/lib/LTO/SummaryBasedOptimizations.cpp
+++ b/lib/LTO/SummaryBasedOptimizations.cpp
@@ -1,9 +1,8 @@
 //==-SummaryBasedOptimizations.cpp - Optimizations based on ThinLTO summary-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -60,21 +59,27 @@ void llvm::computeSyntheticCounts(ModuleSummaryIndex &Index) {
       return UINT64_C(0);
     }
   };
-  auto AddToEntryCount = [](ValueInfo V, uint64_t New) {
+  auto AddToEntryCount = [](ValueInfo V, Scaled64 New) {
     if (!V.getSummaryList().size())
       return;
     for (auto &GVS : V.getSummaryList()) {
       auto S = GVS.get()->getBaseObject();
       auto *F = cast<FunctionSummary>(S);
-      F->setEntryCount(SaturatingAdd(F->entryCount(), New));
+      F->setEntryCount(
+          SaturatingAdd(F->entryCount(), New.template toInt<uint64_t>()));
     }
   };
 
+  auto GetProfileCount = [&](ValueInfo V, FunctionSummary::EdgeTy &Edge) {
+    auto RelFreq = GetCallSiteRelFreq(Edge);
+    Scaled64 EC(GetEntryCount(V), 0);
+    return RelFreq * EC;
+  };
   // After initializing the counts in initializeCounts above, the counts have to
   // be propagated across the combined callgraph.
   // SyntheticCountsUtils::propagate takes care of this propagation on any
   // callgraph that specialized GraphTraits.
-  SyntheticCountsUtils<ModuleSummaryIndex *>::propagate(
-      &Index, GetCallSiteRelFreq, GetEntryCount, AddToEntryCount);
+  SyntheticCountsUtils<ModuleSummaryIndex *>::propagate(&Index, GetProfileCount,
+                                                        AddToEntryCount);
   Index.setHasSyntheticEntryCounts();
 }
diff --git a/lib/LTO/ThinLTOCodeGenerator.cpp b/lib/LTO/ThinLTOCodeGenerator.cpp
index d9ec68fe3eb50a3234d16e2e40d9715df5e9730f..557afd6c36030d6428e81a1679ecbb5173ce1de8 100644
--- a/lib/LTO/ThinLTOCodeGenerator.cpp
+++ b/lib/LTO/ThinLTOCodeGenerator.cpp
@@ -1,9 +1,8 @@
 //===-ThinLTOCodeGenerator.cpp - LLVM Link Time Optimizer -----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/LTO/UpdateCompilerUsed.cpp b/lib/LTO/UpdateCompilerUsed.cpp
index 00482dee6e106f3e7d213625156b6d1e57b1f206..6434f902088d1fd45240e201658398e778b85086 100644
--- a/lib/LTO/UpdateCompilerUsed.cpp
+++ b/lib/LTO/UpdateCompilerUsed.cpp
@@ -1,9 +1,8 @@
 //==-LTOInternalize.cpp - LLVM Link Time Optimizer Internalization Utility -==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/LineEditor/LLVMBuild.txt b/lib/LineEditor/LLVMBuild.txt
index 35f4361e0694702315c52288424e3ee6b9f539d1..55f9675efcb460abdd6016a0e9eafe40f42b0ade 100644
--- a/lib/LineEditor/LLVMBuild.txt
+++ b/lib/LineEditor/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/LineEditor/LLVMBuild.txt ---------------------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/LineEditor/LineEditor.cpp b/lib/LineEditor/LineEditor.cpp
index 533a928b2dfdcfb732de6976f2ca7fa332952c15..57e62bd64871e5ba6a469b678bfb42d625e7fdbe 100644
--- a/lib/LineEditor/LineEditor.cpp
+++ b/lib/LineEditor/LineEditor.cpp
@@ -1,9 +1,8 @@
 //===-- LineEditor.cpp - line editor --------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Linker/IRMover.cpp b/lib/Linker/IRMover.cpp
index afbc57abfcc08da06c0afe775e550353af5ce4cb..70d5262fd618ef4eee7cfe2b17bcb0f2dc037319 100644
--- a/lib/Linker/IRMover.cpp
+++ b/lib/Linker/IRMover.cpp
@@ -1,9 +1,8 @@
 //===- lib/Linker/IRMover.cpp ---------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Linker/LLVMBuild.txt b/lib/Linker/LLVMBuild.txt
index 0bb26d0c2aeac382b9a31401b02e138369983cc2..a642af828b6e0fd8ae248d46b195ba9881f57021 100644
--- a/lib/Linker/LLVMBuild.txt
+++ b/lib/Linker/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/Linker/LLVMBuild.txt -------------------------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Linker/LinkDiagnosticInfo.h b/lib/Linker/LinkDiagnosticInfo.h
index d91f19c69aacf21e5192dacf71c398ac029ec8cb..30c16abaf509074372161f035b25e54a9bf6c953 100644
--- a/lib/Linker/LinkDiagnosticInfo.h
+++ b/lib/Linker/LinkDiagnosticInfo.h
@@ -1,9 +1,8 @@
 //===- LinkDiagnosticInfo.h -------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Linker/LinkModules.cpp b/lib/Linker/LinkModules.cpp
index 25f31a3401a6b94146101fac15143c948dff0d92..a18f4cc25bcc2ad4e8661cf19f60c4f28703e70c 100644
--- a/lib/Linker/LinkModules.cpp
+++ b/lib/Linker/LinkModules.cpp
@@ -1,9 +1,8 @@
 //===- lib/Linker/LinkModules.cpp - Module Linker Implementation ----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/MC/ConstantPools.cpp b/lib/MC/ConstantPools.cpp
index 18277a225640459eedcceb76b28c016ab8795d2c..8cba6b3281a53b9a01b1f320d9af51b02d52eb49 100644
--- a/lib/MC/ConstantPools.cpp
+++ b/lib/MC/ConstantPools.cpp
@@ -1,9 +1,8 @@
 //===- ConstantPools.cpp - ConstantPool class -----------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/MC/ELFObjectWriter.cpp b/lib/MC/ELFObjectWriter.cpp
index 89f3b30cddd6f44087340e3ab40d84c4e51ee1a1..9cae28128c96ebf9f1f0e0d3cbc3731d1adfc615 100644
--- a/lib/MC/ELFObjectWriter.cpp
+++ b/lib/MC/ELFObjectWriter.cpp
@@ -1,9 +1,8 @@
 //===- lib/MC/ELFObjectWriter.cpp - ELF File Writer -----------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -1275,14 +1274,20 @@ void ELFObjectWriter::executePostLayoutBinding(MCAssembler &Asm,
     if (!Symbol.isUndefined() && !Rest.startswith("@@@"))
       continue;
 
-    // FIXME: produce a better error message.
+    // FIXME: Get source locations for these errors or diagnose them earlier.
     if (Symbol.isUndefined() && Rest.startswith("@@") &&
-        !Rest.startswith("@@@"))
-      report_fatal_error("A @@ version cannot be undefined");
+        !Rest.startswith("@@@")) {
+      Asm.getContext().reportError(SMLoc(), "versioned symbol " + AliasName +
+                                                " must be defined");
+      continue;
+    }
 
-    if (Renames.count(&Symbol) && Renames[&Symbol] != Alias)
-      report_fatal_error(llvm::Twine("Multiple symbol versions defined for ") +
-                         Symbol.getName());
+    if (Renames.count(&Symbol) && Renames[&Symbol] != Alias) {
+      Asm.getContext().reportError(
+          SMLoc(), llvm::Twine("multiple symbol versions defined for ") +
+                       Symbol.getName());
+      continue;
+    }
 
     Renames.insert(std::make_pair(&Symbol, Alias));
   }
diff --git a/lib/MC/LLVMBuild.txt b/lib/MC/LLVMBuild.txt
index e3ee59aa7764c261e118306ed699404e414f14f3..61911e5106c63fed55d53bb53ef41b005e64ca92 100644
--- a/lib/MC/LLVMBuild.txt
+++ b/lib/MC/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/MC/LLVMBuild.txt -----------------------------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/MC/MCAsmBackend.cpp b/lib/MC/MCAsmBackend.cpp
index 92d3a8a2645fba46fe6dfa59c1ccbe0bcfe37a2e..ea794cc91feb9ada9e9e5297ba8fa24a62653bb0 100644
--- a/lib/MC/MCAsmBackend.cpp
+++ b/lib/MC/MCAsmBackend.cpp
@@ -1,9 +1,8 @@
 //===- MCAsmBackend.cpp - Target MC Assembly Backend ----------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/MC/MCAsmInfo.cpp b/lib/MC/MCAsmInfo.cpp
index 30f22d2d68f4e9b873c5a5aa75561b4242950a7d..dd16945f7f2e6d92328656d531cc386485ffbf03 100644
--- a/lib/MC/MCAsmInfo.cpp
+++ b/lib/MC/MCAsmInfo.cpp
@@ -1,9 +1,8 @@
 //===- MCAsmInfo.cpp - Asm Info -------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/MC/MCAsmInfoCOFF.cpp b/lib/MC/MCAsmInfoCOFF.cpp
index d8fb875b67c6a6c0a1a157901f0a957373380459..9f19d163f57bc5fb8968f16446043d30f1640c87 100644
--- a/lib/MC/MCAsmInfoCOFF.cpp
+++ b/lib/MC/MCAsmInfoCOFF.cpp
@@ -1,9 +1,8 @@
 //===- MCAsmInfoCOFF.cpp - COFF asm properties ----------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -25,7 +24,7 @@ MCAsmInfoCOFF::MCAsmInfoCOFF() {
   COMMDirectiveAlignmentIsInBytes = false;
   LCOMMDirectiveAlignmentType = LCOMM::ByteAlignment;
   HasDotTypeDotSizeDirective = false;
-  HasSingleParameterDotFile = false;
+  HasSingleParameterDotFile = true;
   WeakRefDirective = "\t.weak\t";
   HasLinkOnceDirective = true;
 
diff --git a/lib/MC/MCAsmInfoDarwin.cpp b/lib/MC/MCAsmInfoDarwin.cpp
index c74840982fb75d19fae52b492531ed6f4601f82a..62bc5b8c94182d3c87d123dc679dbfa5d634eab6 100644
--- a/lib/MC/MCAsmInfoDarwin.cpp
+++ b/lib/MC/MCAsmInfoDarwin.cpp
@@ -1,9 +1,8 @@
 //===- MCAsmInfoDarwin.cpp - Darwin asm properties ------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/MC/MCAsmInfoELF.cpp b/lib/MC/MCAsmInfoELF.cpp
index b0dc43c6c868fb759e78390effde44bc110419a6..a5e8aff7f129d220b4505990170cafa3fae69b9d 100644
--- a/lib/MC/MCAsmInfoELF.cpp
+++ b/lib/MC/MCAsmInfoELF.cpp
@@ -1,9 +1,8 @@
 //===- MCAsmInfoELF.cpp - ELF asm properties ------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/MC/MCAsmInfoWasm.cpp b/lib/MC/MCAsmInfoWasm.cpp
index d448664baa1430fa50ceeb641ffa2016c249b004..ce6ec7ef211e5199c7939f0cedce4e311f97e9b4 100644
--- a/lib/MC/MCAsmInfoWasm.cpp
+++ b/lib/MC/MCAsmInfoWasm.cpp
@@ -1,9 +1,8 @@
 //===-- MCAsmInfoWasm.cpp - Wasm asm properties -----------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -19,6 +18,7 @@ void MCAsmInfoWasm::anchor() {}
 
 MCAsmInfoWasm::MCAsmInfoWasm() {
   HasIdentDirective = true;
+  HasNoDeadStrip = true;
   WeakRefDirective = "\t.weak\t";
   PrivateGlobalPrefix = ".L";
   PrivateLabelPrefix = ".L";
diff --git a/lib/MC/MCAsmMacro.cpp b/lib/MC/MCAsmMacro.cpp
index 7e89c03c6c6b4c9603a401a7c05ea99085d1bb09..ba4fb7d4f387852e67721da4eb485fd0e23bb034 100644
--- a/lib/MC/MCAsmMacro.cpp
+++ b/lib/MC/MCAsmMacro.cpp
@@ -1,9 +1,8 @@
 //===- MCAsmMacro.h - Assembly Macros ---------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/MC/MCAsmStreamer.cpp b/lib/MC/MCAsmStreamer.cpp
index fd10c6901b732899bf4c266f66e430f6df74ab25..f5e40f5c604d284ba3cbb541909208302e11d895 100644
--- a/lib/MC/MCAsmStreamer.cpp
+++ b/lib/MC/MCAsmStreamer.cpp
@@ -1,9 +1,8 @@
 //===- lib/MC/MCAsmStreamer.cpp - Text Assembly Output ----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -108,10 +107,7 @@ public:
   void AddComment(const Twine &T, bool EOL = true) override;
 
   /// Add a comment showing the encoding of an instruction.
-  /// If PrintSchedInfo is true, then the comment sched:[x:y] will be added to
-  /// the output if supported by the target.
-  void AddEncodingComment(const MCInst &Inst, const MCSubtargetInfo &,
-                          bool PrintSchedInfo);
+  void AddEncodingComment(const MCInst &Inst, const MCSubtargetInfo &);
 
   /// Return a raw_ostream that comments can be written to.
   /// Unlike AddComment, you are required to terminate comments with \n if you
@@ -266,6 +262,7 @@ public:
   void EmitCVFPOData(const MCSymbol *ProcSym, SMLoc L) override;
 
   void EmitIdent(StringRef IdentString) override;
+  void EmitCFIBKeyFrame() override;
   void EmitCFISections(bool EH, bool Debug) override;
   void EmitCFIDefCfa(int64_t Register, int64_t Offset) override;
   void EmitCFIDefCfaOffset(int64_t Offset) override;
@@ -311,8 +308,7 @@ public:
   void emitCGProfileEntry(const MCSymbolRefExpr *From,
                           const MCSymbolRefExpr *To, uint64_t Count) override;
 
-  void EmitInstruction(const MCInst &Inst, const MCSubtargetInfo &STI,
-                       bool PrintSchedInfo) override;
+  void EmitInstruction(const MCInst &Inst, const MCSubtargetInfo &STI) override;
 
   void EmitBundleAlignMode(unsigned AlignPow2) override;
   void EmitBundleLock(bool AlignToEnd) override;
@@ -540,11 +536,14 @@ void MCAsmStreamer::EmitVersionMin(MCVersionMinType Type, unsigned Major,
 
 static const char *getPlatformName(MachO::PlatformType Type) {
   switch (Type) {
-  case MachO::PLATFORM_MACOS:    return "macos";
-  case MachO::PLATFORM_IOS:      return "ios";
-  case MachO::PLATFORM_TVOS:     return "tvos";
-  case MachO::PLATFORM_WATCHOS:  return "watchos";
-  case MachO::PLATFORM_BRIDGEOS: return "bridgeos";
+  case MachO::PLATFORM_MACOS:            return "macos";
+  case MachO::PLATFORM_IOS:              return "ios";
+  case MachO::PLATFORM_TVOS:             return "tvos";
+  case MachO::PLATFORM_WATCHOS:          return "watchos";
+  case MachO::PLATFORM_BRIDGEOS:         return "bridgeos";
+  case MachO::PLATFORM_IOSSIMULATOR:     return "iossimulator";
+  case MachO::PLATFORM_TVOSSIMULATOR:    return "tvossimulator";
+  case MachO::PLATFORM_WATCHOSSIMULATOR: return "watchossimulator";
   }
   llvm_unreachable("Invalid Mach-O platform type");
 }
@@ -653,6 +652,9 @@ bool MCAsmStreamer::EmitSymbolAttribute(MCSymbol *Symbol,
       // .weak_reference
   case MCSA_WeakReference:  OS << MAI->getWeakRefDirective(); break;
   case MCSA_WeakDefAutoPrivate: OS << "\t.weak_def_can_be_hidden\t"; break;
+  case MCSA_Cold:
+    // Assemblers currently do not support a .cold directive.
+    return false;
   }
 
   Symbol->print(OS, MAI);
@@ -1599,6 +1601,12 @@ void MCAsmStreamer::EmitCFIReturnColumn(int64_t Register) {
   EmitEOL();
 }
 
+void MCAsmStreamer::EmitCFIBKeyFrame() {
+  MCStreamer::EmitCFIBKeyFrame();
+  OS << "\t.cfi_b_key_frame";
+  EmitEOL();
+}
+
 void MCAsmStreamer::EmitWinCFIStartProc(const MCSymbol *Symbol, SMLoc Loc) {
   MCStreamer::EmitWinCFIStartProc(Symbol, Loc);
 
@@ -1727,8 +1735,7 @@ void MCAsmStreamer::emitCGProfileEntry(const MCSymbolRefExpr *From,
 }
 
 void MCAsmStreamer::AddEncodingComment(const MCInst &Inst,
-                                       const MCSubtargetInfo &STI,
-                                       bool PrintSchedInfo) {
+                                       const MCSubtargetInfo &STI) {
   raw_ostream &OS = GetCommentOS();
   SmallString<256> Code;
   SmallVector<MCFixup, 4> Fixups;
@@ -1807,11 +1814,7 @@ void MCAsmStreamer::AddEncodingComment(const MCInst &Inst,
       }
     }
   }
-  OS << "]";
-  // If we are not going to add fixup or schedule comments after this point
-  // then we have to end the current comment line with "\n".
-  if (Fixups.size() || !PrintSchedInfo)
-    OS << "\n";
+  OS << "]\n";
 
   for (unsigned i = 0, e = Fixups.size(); i != e; ++i) {
     MCFixup &F = Fixups[i];
@@ -1823,18 +1826,15 @@ void MCAsmStreamer::AddEncodingComment(const MCInst &Inst,
 }
 
 void MCAsmStreamer::EmitInstruction(const MCInst &Inst,
-                                    const MCSubtargetInfo &STI,
-                                    bool PrintSchedInfo) {
+                                    const MCSubtargetInfo &STI) {
   assert(getCurrentSectionOnly() &&
          "Cannot emit contents before setting section!");
 
   // Show the encoding in a comment if we have a code emitter.
-  AddEncodingComment(Inst, STI, PrintSchedInfo);
+  AddEncodingComment(Inst, STI);
 
   // Show the MCInst if enabled.
   if (ShowInst) {
-    if (PrintSchedInfo)
-      GetCommentOS() << "\n";
     Inst.dump_pretty(GetCommentOS(), InstPrinter.get(), "\n ");
     GetCommentOS() << "\n";
   }
@@ -1844,12 +1844,6 @@ void MCAsmStreamer::EmitInstruction(const MCInst &Inst,
   else
     InstPrinter->printInst(&Inst, OS, "", STI);
 
-  if (PrintSchedInfo) {
-    std::string SI = STI.getSchedInfoStr(Inst);
-    if (!SI.empty())
-      GetCommentOS() << SI;
-  }
-
   StringRef Comments = CommentToEmit;
   if (Comments.size() && Comments.back() != '\n')
     GetCommentOS() << "\n";
diff --git a/lib/MC/MCAssembler.cpp b/lib/MC/MCAssembler.cpp
index cde6a93a164729636b8671cd48634bbc028219aa..c4f4d4c2870e1e10821707284d1d10b8cfac9ad6 100644
--- a/lib/MC/MCAssembler.cpp
+++ b/lib/MC/MCAssembler.cpp
@@ -1,9 +1,8 @@
 //===- lib/MC/MCAssembler.cpp - Assembler Backend Implementation ----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -323,6 +322,13 @@ uint64_t MCAssembler::computeFragmentSize(const MCAsmLayout &Layout,
     const MCAlignFragment &AF = cast<MCAlignFragment>(F);
     unsigned Offset = Layout.getFragmentOffset(&AF);
     unsigned Size = OffsetToAlignment(Offset, AF.getAlignment());
+
+    // Insert extra Nops for code alignment if the target define
+    // shouldInsertExtraNopBytesForCodeAlign target hook.
+    if (AF.getParent()->UseCodeAlign() && AF.hasEmitNops() &&
+        getBackend().shouldInsertExtraNopBytesForCodeAlign(AF, Size))
+      return Size;
+
     // If we are padding with nops, force the padding to be larger than the
     // minimum nop size.
     if (Size > 0 && AF.hasEmitNops()) {
@@ -805,7 +811,8 @@ void MCAssembler::layout(MCAsmLayout &Layout) {
       if (isa<MCEncodedFragment>(&Frag) &&
           isa<MCCompactEncodedInstFragment>(&Frag))
         continue;
-      if (!isa<MCEncodedFragment>(&Frag) && !isa<MCCVDefRangeFragment>(&Frag))
+      if (!isa<MCEncodedFragment>(&Frag) && !isa<MCCVDefRangeFragment>(&Frag) &&
+          !isa<MCAlignFragment>(&Frag))
         continue;
       ArrayRef<MCFixup> Fixups;
       MutableArrayRef<char> Contents;
@@ -826,6 +833,13 @@ void MCAssembler::layout(MCAsmLayout &Layout) {
       } else if (auto *FragWithFixups = dyn_cast<MCDwarfLineAddrFragment>(&Frag)) {
         Fixups = FragWithFixups->getFixups();
         Contents = FragWithFixups->getContents();
+      } else if (auto *AF = dyn_cast<MCAlignFragment>(&Frag)) {
+        // Insert fixup type for code alignment if the target define
+        // shouldInsertFixupForCodeAlign target hook.
+        if (Sec.UseCodeAlign() && AF->hasEmitNops()) {
+          getBackend().shouldInsertFixupForCodeAlign(*this, Layout, *AF);
+        }
+        continue;
       } else
         llvm_unreachable("Unknown fragment with fixups!");
       for (const MCFixup &Fixup : Fixups) {
diff --git a/lib/MC/MCCodeEmitter.cpp b/lib/MC/MCCodeEmitter.cpp
index ca69478ed10d3f3287d9de47ba3fe2d2e1b4974d..0d114f12d58c53053f870ea424166549308ef16c 100644
--- a/lib/MC/MCCodeEmitter.cpp
+++ b/lib/MC/MCCodeEmitter.cpp
@@ -1,9 +1,8 @@
 //===- MCCodeEmitter.cpp - Instruction Encoding ---------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/MC/MCCodePadder.cpp b/lib/MC/MCCodePadder.cpp
index 57547814e595fbc7fb244b9b1f2335382af304be..27a62f95a529482b0e09b52b13d7462f0995c969 100644
--- a/lib/MC/MCCodePadder.cpp
+++ b/lib/MC/MCCodePadder.cpp
@@ -1,9 +1,8 @@
 //===- MCCodePadder.cpp - Target MC Code Padder ---------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/MC/MCCodeView.cpp b/lib/MC/MCCodeView.cpp
index 978ac789c31e118afca58925c40b0b9115b04d13..1a71b542bd06d075ae3ecdf5a2475688952a88c2 100644
--- a/lib/MC/MCCodeView.cpp
+++ b/lib/MC/MCCodeView.cpp
@@ -1,9 +1,8 @@
 //===- MCCodeView.h - Machine Code CodeView support -------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/MC/MCContext.cpp b/lib/MC/MCContext.cpp
index fab517075c5ac6aef8d7f65fbd94d7910676a205..d7f95df726ff22f1ca71de5ce8632c284ef4ef79 100644
--- a/lib/MC/MCContext.cpp
+++ b/lib/MC/MCContext.cpp
@@ -1,9 +1,8 @@
 //===- lib/MC/MCContext.cpp - Machine Code Context ------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/MC/MCDisassembler/Disassembler.cpp b/lib/MC/MCDisassembler/Disassembler.cpp
index ad0a39991c536effad1de502e94900eff90f6274..fd3db8edd74dbe55f0fcff323bd3cefe9c7513fd 100644
--- a/lib/MC/MCDisassembler/Disassembler.cpp
+++ b/lib/MC/MCDisassembler/Disassembler.cpp
@@ -1,9 +1,8 @@
 //===-- lib/MC/Disassembler.cpp - Disassembler Public C Interface ---------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/MC/MCDisassembler/Disassembler.h b/lib/MC/MCDisassembler/Disassembler.h
index f638fdc781d780875e5a08489683d496f2b866e3..d3a2dbb6da5dfc7b0deaa785d2592f93fd40dc44 100644
--- a/lib/MC/MCDisassembler/Disassembler.h
+++ b/lib/MC/MCDisassembler/Disassembler.h
@@ -1,9 +1,8 @@
 //===------------- Disassembler.h - LLVM Disassembler -----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/MC/MCDisassembler/LLVMBuild.txt b/lib/MC/MCDisassembler/LLVMBuild.txt
index e4876cd18fff6d1fdba039de6ba6c6e1581963a3..a3640c74b9e445cd20ead6a09c4b79a63f82f30f 100644
--- a/lib/MC/MCDisassembler/LLVMBuild.txt
+++ b/lib/MC/MCDisassembler/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/MC/MCDisassembler/LLVMBuild.txt --------------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/MC/MCDisassembler/MCDisassembler.cpp b/lib/MC/MCDisassembler/MCDisassembler.cpp
index 2f1275d00b861f3db7a4c87273a915d486453b4f..063f7e706024be32fa018b71be2c96a099528b75 100644
--- a/lib/MC/MCDisassembler/MCDisassembler.cpp
+++ b/lib/MC/MCDisassembler/MCDisassembler.cpp
@@ -1,13 +1,14 @@
 //===- MCDisassembler.cpp - Disassembler interface ------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #include "llvm/MC/MCDisassembler/MCDisassembler.h"
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/StringRef.h"
 #include "llvm/Support/raw_ostream.h"
 #include <algorithm>
 
@@ -15,6 +16,13 @@ using namespace llvm;
 
 MCDisassembler::~MCDisassembler() = default;
 
+MCDisassembler::DecodeStatus MCDisassembler::onSymbolStart(
+    StringRef Name, uint64_t &Size, ArrayRef<uint8_t> Bytes, uint64_t Address,
+    raw_ostream &VStream, raw_ostream &CStream) const {
+  Size = 0;
+  return MCDisassembler::Success;
+}
+
 bool MCDisassembler::tryAddingSymbolicOperand(MCInst &Inst, int64_t Value,
                                               uint64_t Address, bool IsBranch,
                                               uint64_t Offset,
diff --git a/lib/MC/MCDisassembler/MCExternalSymbolizer.cpp b/lib/MC/MCDisassembler/MCExternalSymbolizer.cpp
index 1969c5dc66abc3f08559a10e39ff3e34dc786fd3..7befef86303cbfdd181ccd17f59ed5b50761faad 100644
--- a/lib/MC/MCDisassembler/MCExternalSymbolizer.cpp
+++ b/lib/MC/MCDisassembler/MCExternalSymbolizer.cpp
@@ -1,9 +1,8 @@
 //===-- MCExternalSymbolizer.cpp - External symbolizer --------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/MC/MCDisassembler/MCRelocationInfo.cpp b/lib/MC/MCDisassembler/MCRelocationInfo.cpp
index 8f932a3f0d48771f5322ac508ffb48f15c6cba7f..64e216e0051daf8b7bfca7086a8e4fff651d1e7f 100644
--- a/lib/MC/MCDisassembler/MCRelocationInfo.cpp
+++ b/lib/MC/MCDisassembler/MCRelocationInfo.cpp
@@ -1,9 +1,8 @@
 //===-- MCRelocationInfo.cpp ----------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/MC/MCDisassembler/MCSymbolizer.cpp b/lib/MC/MCDisassembler/MCSymbolizer.cpp
index 78e611e3ddda947585fd5f254b4c28f1558f4e35..8214a196afb117eedff50d5b8f765fdce4e18f31 100644
--- a/lib/MC/MCDisassembler/MCSymbolizer.cpp
+++ b/lib/MC/MCDisassembler/MCSymbolizer.cpp
@@ -1,9 +1,8 @@
 //===-- llvm/MC/MCSymbolizer.cpp - MCSymbolizer class ---------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/MC/MCDwarf.cpp b/lib/MC/MCDwarf.cpp
index 9791fb5724b1365219dbce1f2c3abaf419cf9558..2cab55f15bcc4711be4192e08155e1c373901e71 100644
--- a/lib/MC/MCDwarf.cpp
+++ b/lib/MC/MCDwarf.cpp
@@ -1,9 +1,8 @@
 //===- lib/MC/MCDwarf.cpp - MCDwarf implementation ------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -1565,9 +1564,8 @@ const MCSymbol &FrameEmitterImpl::EmitCIE(const MCDwarfFrameInfo &Frame) {
   uint8_t CIEVersion = getCIEVersion(IsEH, context.getDwarfVersion());
   Streamer.EmitIntValue(CIEVersion, 1);
 
-  // Augmentation String
-  SmallString<8> Augmentation;
   if (IsEH) {
+    SmallString<8> Augmentation;
     Augmentation += "z";
     if (Frame.Personality)
       Augmentation += "P";
@@ -1576,6 +1574,8 @@ const MCSymbol &FrameEmitterImpl::EmitCIE(const MCDwarfFrameInfo &Frame) {
     Augmentation += "R";
     if (Frame.IsSignalFrame)
       Augmentation += "S";
+    if (Frame.IsBKeyFrame)
+      Augmentation += "B";
     Streamer.EmitBytes(Augmentation);
   }
   Streamer.EmitIntValue(0, 1);
@@ -1730,25 +1730,28 @@ namespace {
 
 struct CIEKey {
   static const CIEKey getEmptyKey() {
-    return CIEKey(nullptr, 0, -1, false, false, static_cast<unsigned>(INT_MAX));
+    return CIEKey(nullptr, 0, -1, false, false, static_cast<unsigned>(INT_MAX),
+                  false);
   }
 
   static const CIEKey getTombstoneKey() {
-    return CIEKey(nullptr, -1, 0, false, false, static_cast<unsigned>(INT_MAX));
+    return CIEKey(nullptr, -1, 0, false, false, static_cast<unsigned>(INT_MAX),
+                  false);
   }
 
   CIEKey(const MCSymbol *Personality, unsigned PersonalityEncoding,
          unsigned LSDAEncoding, bool IsSignalFrame, bool IsSimple,
-         unsigned RAReg)
+         unsigned RAReg, bool IsBKeyFrame)
       : Personality(Personality), PersonalityEncoding(PersonalityEncoding),
         LsdaEncoding(LSDAEncoding), IsSignalFrame(IsSignalFrame),
-        IsSimple(IsSimple), RAReg(RAReg) {}
+        IsSimple(IsSimple), RAReg(RAReg), IsBKeyFrame(IsBKeyFrame) {}
 
   explicit CIEKey(const MCDwarfFrameInfo &Frame)
       : Personality(Frame.Personality),
         PersonalityEncoding(Frame.PersonalityEncoding),
         LsdaEncoding(Frame.LsdaEncoding), IsSignalFrame(Frame.IsSignalFrame),
-        IsSimple(Frame.IsSimple), RAReg(Frame.RAReg) {}
+        IsSimple(Frame.IsSimple), RAReg(Frame.RAReg),
+        IsBKeyFrame(Frame.IsBKeyFrame) {}
 
   const MCSymbol *Personality;
   unsigned PersonalityEncoding;
@@ -1756,6 +1759,7 @@ struct CIEKey {
   bool IsSignalFrame;
   bool IsSimple;
   unsigned RAReg;
+  bool IsBKeyFrame;
 };
 
 } // end anonymous namespace
@@ -1767,9 +1771,9 @@ template <> struct DenseMapInfo<CIEKey> {
   static CIEKey getTombstoneKey() { return CIEKey::getTombstoneKey(); }
 
   static unsigned getHashValue(const CIEKey &Key) {
-    return static_cast<unsigned>(
-        hash_combine(Key.Personality, Key.PersonalityEncoding, Key.LsdaEncoding,
-                     Key.IsSignalFrame, Key.IsSimple, Key.RAReg));
+    return static_cast<unsigned>(hash_combine(
+        Key.Personality, Key.PersonalityEncoding, Key.LsdaEncoding,
+        Key.IsSignalFrame, Key.IsSimple, Key.RAReg, Key.IsBKeyFrame));
   }
 
   static bool isEqual(const CIEKey &LHS, const CIEKey &RHS) {
@@ -1777,8 +1781,8 @@ template <> struct DenseMapInfo<CIEKey> {
            LHS.PersonalityEncoding == RHS.PersonalityEncoding &&
            LHS.LsdaEncoding == RHS.LsdaEncoding &&
            LHS.IsSignalFrame == RHS.IsSignalFrame &&
-           LHS.IsSimple == RHS.IsSimple &&
-           LHS.RAReg == RHS.RAReg;
+           LHS.IsSimple == RHS.IsSimple && LHS.RAReg == RHS.RAReg &&
+           LHS.IsBKeyFrame == RHS.IsBKeyFrame;
   }
 };
 
diff --git a/lib/MC/MCELFObjectTargetWriter.cpp b/lib/MC/MCELFObjectTargetWriter.cpp
index ff53dd7299c12daa73a71173eaa00864abd33654..9ad7f8c123a27d707a16312397b67649cf0d4f33 100644
--- a/lib/MC/MCELFObjectTargetWriter.cpp
+++ b/lib/MC/MCELFObjectTargetWriter.cpp
@@ -1,9 +1,8 @@
 //===-- MCELFObjectTargetWriter.cpp - ELF Target Writer Subclass ----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/MC/MCELFStreamer.cpp b/lib/MC/MCELFStreamer.cpp
index 95b48e6abc740f8022ee96566f84a03a0d789d6d..6fe16abd5a3a692555509af77f11825772e327ab 100644
--- a/lib/MC/MCELFStreamer.cpp
+++ b/lib/MC/MCELFStreamer.cpp
@@ -1,9 +1,8 @@
 //===- lib/MC/MCELFStreamer.cpp - ELF Object Output -----------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -202,6 +201,7 @@ bool MCELFStreamer::EmitSymbolAttribute(MCSymbol *S, MCSymbolAttr Attribute) {
   // In the future it might be worth trying to make these operations more well
   // defined.
   switch (Attribute) {
+  case MCSA_Cold:
   case MCSA_LazyReference:
   case MCSA_Reference:
   case MCSA_SymbolResolver:
diff --git a/lib/MC/MCExpr.cpp b/lib/MC/MCExpr.cpp
index 3c022199145fbafe128f4ce4368cc546c969486f..7ca935f248d0d05f2b1260fa0b63c12fa0f5b89a 100644
--- a/lib/MC/MCExpr.cpp
+++ b/lib/MC/MCExpr.cpp
@@ -1,9 +1,8 @@
 //===- MCExpr.cpp - Assembly Level Expression Implementation --------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -559,6 +558,11 @@ static void AttemptToFoldSymbolOffsetDifference(
   if (Asm->isThumbFunc(&SA))
     Addend |= 1;
 
+  // If symbol is labeled as micromips, we set low-bit to ensure
+  // correct offset in .gcc_except_table
+  if (Asm->getBackend().isMicroMips(&SA))
+    Addend |= 1;
+
   // Clear the symbol expr pointers to indicate we have folded these
   // operands.
   A = B = nullptr;
diff --git a/lib/MC/MCFragment.cpp b/lib/MC/MCFragment.cpp
index d22b117972bf230c38d62ab70f110de947673203..ae5bd65507bc92bb439906631c3ec9eb87493f43 100644
--- a/lib/MC/MCFragment.cpp
+++ b/lib/MC/MCFragment.cpp
@@ -1,9 +1,8 @@
 //===- lib/MC/MCFragment.cpp - Assembler Fragment Implementation ----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/MC/MCInst.cpp b/lib/MC/MCInst.cpp
index 64f111fc7114847f4a39c5a60b16b7349185d8a2..f6f6edee5822e0c04172b5d15a4af7ea496120c0 100644
--- a/lib/MC/MCInst.cpp
+++ b/lib/MC/MCInst.cpp
@@ -1,9 +1,8 @@
 //===- lib/MC/MCInst.cpp - MCInst implementation --------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/MC/MCInstPrinter.cpp b/lib/MC/MCInstPrinter.cpp
index 9296fcedb72b5ae6ffe3b6434b69f3213679b81d..ab8773e7bbb2fa1ed5e18407ac428cd5e453518e 100644
--- a/lib/MC/MCInstPrinter.cpp
+++ b/lib/MC/MCInstPrinter.cpp
@@ -1,9 +1,8 @@
 //===- MCInstPrinter.cpp - Convert an MCInst to target assembly syntax ----===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/MC/MCInstrAnalysis.cpp b/lib/MC/MCInstrAnalysis.cpp
index 8223f3a5c66fd2dd59db74f9f3af160faf076fcc..eca87f940bf5103c69e483927a18432cdf2295fe 100644
--- a/lib/MC/MCInstrAnalysis.cpp
+++ b/lib/MC/MCInstrAnalysis.cpp
@@ -1,9 +1,8 @@
 //===- MCInstrAnalysis.cpp - InstrDesc target hooks -----------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/MC/MCInstrDesc.cpp b/lib/MC/MCInstrDesc.cpp
index 53cba864a85d7b893b6d6406dd115e7119fc70a4..d54aeba89edc5f85d7ec25319c00e855c84bd5cc 100644
--- a/lib/MC/MCInstrDesc.cpp
+++ b/lib/MC/MCInstrDesc.cpp
@@ -1,9 +1,8 @@
 //===------ llvm/MC/MCInstrDesc.cpp- Instruction Descriptors --------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/MC/MCLabel.cpp b/lib/MC/MCLabel.cpp
index c376c83274efe423b8b76a24733cdd99cc6c826e..66ee73c5bbb3e740a1afb55cf1bfb667df885139 100644
--- a/lib/MC/MCLabel.cpp
+++ b/lib/MC/MCLabel.cpp
@@ -1,9 +1,8 @@
 //===- lib/MC/MCLabel.cpp - MCLabel implementation ------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/MC/MCLinkerOptimizationHint.cpp b/lib/MC/MCLinkerOptimizationHint.cpp
index 2f8581470ea6ba758f110a133057d0779e3c0ffd..9ab321872b1135f791cb1c79436e757ccaf72e70 100644
--- a/lib/MC/MCLinkerOptimizationHint.cpp
+++ b/lib/MC/MCLinkerOptimizationHint.cpp
@@ -1,9 +1,8 @@
 //===- llvm/MC/MCLinkerOptimizationHint.cpp ----- LOH handling ------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/MC/MCMachOStreamer.cpp b/lib/MC/MCMachOStreamer.cpp
index b30317e74672bb3198a1f6aa8efad4e61411fcf9..613f255a4ea47cfb9d19ce96288f07a6989010a2 100644
--- a/lib/MC/MCMachOStreamer.cpp
+++ b/lib/MC/MCMachOStreamer.cpp
@@ -1,9 +1,8 @@
 //===- MCMachOStreamer.cpp - MachO Streamer -------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -387,6 +386,10 @@ bool MCMachOStreamer::EmitSymbolAttribute(MCSymbol *Sym,
     Symbol->setWeakDefinition();
     Symbol->setWeakReference();
     break;
+
+  case MCSA_Cold:
+    Symbol->setCold();
+    break;
   }
 
   return true;
diff --git a/lib/MC/MCMachObjectTargetWriter.cpp b/lib/MC/MCMachObjectTargetWriter.cpp
index 8809a3c320f837773b54f23c72ea04d8a98fe905..a57b8a7ac0ff39f0f03cc46cbdf603803e5f505a 100644
--- a/lib/MC/MCMachObjectTargetWriter.cpp
+++ b/lib/MC/MCMachObjectTargetWriter.cpp
@@ -1,9 +1,8 @@
 //===- MCMachObjectTargetWriter.cpp - Mach-O Target Writer Subclass -------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/MC/MCNullStreamer.cpp b/lib/MC/MCNullStreamer.cpp
index 4e97e7550bcbfc34bcaa4dea08b9336307a819cb..8452317c8c6b71aa0073dee0d893773726c99848 100644
--- a/lib/MC/MCNullStreamer.cpp
+++ b/lib/MC/MCNullStreamer.cpp
@@ -1,9 +1,8 @@
 //===- lib/MC/MCNullStreamer.cpp - Dummy Streamer Implementation ----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/MC/MCObjectFileInfo.cpp b/lib/MC/MCObjectFileInfo.cpp
index 9e35355d06e0a50e4825fd1f927a6e13b266012c..d9f1b2cb0594aefa20bbd9d02d569a38e270b1f1 100644
--- a/lib/MC/MCObjectFileInfo.cpp
+++ b/lib/MC/MCObjectFileInfo.cpp
@@ -1,9 +1,8 @@
 //===-- MCObjectFileInfo.cpp - Object File Information --------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/MC/MCObjectStreamer.cpp b/lib/MC/MCObjectStreamer.cpp
index 6ec705bdddb73451aa0169e6df9847a7881b14ef..1587d8498666f8ebe952274e07275cc823b2e90a 100644
--- a/lib/MC/MCObjectStreamer.cpp
+++ b/lib/MC/MCObjectStreamer.cpp
@@ -1,9 +1,8 @@
 //===- lib/MC/MCObjectStreamer.cpp - Object File MCStreamer Interface -----===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -315,7 +314,7 @@ bool MCObjectStreamer::mayHaveInstructions(MCSection &Sec) const {
 }
 
 void MCObjectStreamer::EmitInstruction(const MCInst &Inst,
-                                       const MCSubtargetInfo &STI, bool) {
+                                       const MCSubtargetInfo &STI) {
   getAssembler().getBackend().handleCodePaddingInstructionBegin(Inst);
   EmitInstructionImpl(Inst, STI);
   getAssembler().getBackend().handleCodePaddingInstructionEnd(Inst);
diff --git a/lib/MC/MCObjectWriter.cpp b/lib/MC/MCObjectWriter.cpp
index 98ac48a23f91c5cbe4090ed6f3c7eafe873205db..a058bbe0ba0b8b475446d783a8a46997be8c61ec 100644
--- a/lib/MC/MCObjectWriter.cpp
+++ b/lib/MC/MCObjectWriter.cpp
@@ -1,9 +1,8 @@
 //===- lib/MC/MCObjectWriter.cpp - MCObjectWriter implementation ----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/MC/MCParser/AsmLexer.cpp b/lib/MC/MCParser/AsmLexer.cpp
index 2b0d20f9b8e2efc10507323dc1cd80edcb7ce613..88a131efe10ea8686e09fdfda853022f10856fe9 100644
--- a/lib/MC/MCParser/AsmLexer.cpp
+++ b/lib/MC/MCParser/AsmLexer.cpp
@@ -1,9 +1,8 @@
 //===- AsmLexer.cpp - Lexer for Assembly Files ----------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/MC/MCParser/AsmParser.cpp b/lib/MC/MCParser/AsmParser.cpp
index aa07eee44916483c4e6b5fbb27268d1d010241c6..a66a4eb29afc6864f5e8167ec71905de86449c12 100644
--- a/lib/MC/MCParser/AsmParser.cpp
+++ b/lib/MC/MCParser/AsmParser.cpp
@@ -1,9 +1,8 @@
 //===- AsmParser.cpp - Parser for Assembly Files --------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -159,9 +158,10 @@ private:
   /// The values from the last parsed cpp hash file line comment if any.
   struct CppHashInfoTy {
     StringRef Filename;
-    int64_t LineNumber = 0;
+    int64_t LineNumber;
     SMLoc Loc;
-    unsigned Buf = 0;
+    unsigned Buf;
+    CppHashInfoTy() : Filename(), LineNumber(0), Loc(), Buf(0) {}
   };
   CppHashInfoTy CppHashInfo;
 
@@ -426,6 +426,7 @@ private:
     DK_WEAK_DEFINITION,
     DK_WEAK_REFERENCE,
     DK_WEAK_DEF_CAN_BE_HIDDEN,
+    DK_COLD,
     DK_COMM,
     DK_COMMON,
     DK_LCOMM,
@@ -495,6 +496,7 @@ private:
     DK_CFI_UNDEFINED,
     DK_CFI_REGISTER,
     DK_CFI_WINDOW_SAVE,
+    DK_CFI_B_KEY_FRAME,
     DK_MACROS_ON,
     DK_MACROS_OFF,
     DK_ALTMACRO,
@@ -898,6 +900,12 @@ bool AsmParser::Run(bool NoInitialTextSection, bool NoFinalize) {
       eatToEndOfStatement();
   }
 
+  // Make sure we get proper DWARF even for empty files.
+  (void)enabledGenDwarfForAssembly();
+
+  getTargetParser().onEndOfFile();
+  printPendingErrors();
+
   // All errors should have been emitted.
   assert(!hasPendingError() && "unexpected error from parseStatement");
 
@@ -1979,6 +1987,8 @@ bool AsmParser::parseStatement(ParseStatementInfo &Info,
       return parseDirectiveSymbolAttribute(MCSA_WeakReference);
     case DK_WEAK_DEF_CAN_BE_HIDDEN:
       return parseDirectiveSymbolAttribute(MCSA_WeakDefAutoPrivate);
+    case DK_COLD:
+      return parseDirectiveSymbolAttribute(MCSA_Cold);
     case DK_COMM:
     case DK_COMMON:
       return parseDirectiveComm(/*IsLocal=*/false);
@@ -3359,9 +3369,13 @@ bool AsmParser::parseDirectiveFile(SMLoc DirectiveLoc) {
     }
   }
 
-  if (FileNumber == -1)
-    getStreamer().EmitFileDirective(Filename);
-  else {
+  if (FileNumber == -1) {
+    // Ignore the directive if there is no number and the target doesn't support
+    // numberless .file directives. This allows some portability of assembler
+    // between different object file formats.
+    if (getContext().getAsmInfo()->hasSingleParameterDotFile())
+      getStreamer().EmitFileDirective(Filename);
+  } else {
     // In case there is a -g option as well as debug info from directive .file,
     // we turn off the -g option, directly use the existing debug info instead.
     // Also reset any implicit ".file 0" for the assembler source.
@@ -5028,9 +5042,9 @@ bool AsmParser::parseDirectiveIfdef(SMLoc DirectiveLoc, bool expect_defined) {
     MCSymbol *Sym = getContext().lookupSymbol(Name);
 
     if (expect_defined)
-      TheCondState.CondMet = (Sym && !Sym->isUndefined());
+      TheCondState.CondMet = (Sym && !Sym->isUndefined(false));
     else
-      TheCondState.CondMet = (!Sym || Sym->isUndefined());
+      TheCondState.CondMet = (!Sym || Sym->isUndefined(false));
     TheCondState.Ignore = !TheCondState.CondMet;
   }
 
@@ -5216,6 +5230,7 @@ void AsmParser::initializeDirectiveKindMap() {
   DirectiveKindMap[".weak_definition"] = DK_WEAK_DEFINITION;
   DirectiveKindMap[".weak_reference"] = DK_WEAK_REFERENCE;
   DirectiveKindMap[".weak_def_can_be_hidden"] = DK_WEAK_DEF_CAN_BE_HIDDEN;
+  DirectiveKindMap[".cold"] = DK_COLD;
   DirectiveKindMap[".comm"] = DK_COMM;
   DirectiveKindMap[".common"] = DK_COMMON;
   DirectiveKindMap[".lcomm"] = DK_LCOMM;
@@ -5293,6 +5308,7 @@ void AsmParser::initializeDirectiveKindMap() {
   DirectiveKindMap[".cfi_undefined"] = DK_CFI_UNDEFINED;
   DirectiveKindMap[".cfi_register"] = DK_CFI_REGISTER;
   DirectiveKindMap[".cfi_window_save"] = DK_CFI_WINDOW_SAVE;
+  DirectiveKindMap[".cfi_b_key_frame"] = DK_CFI_B_KEY_FRAME;
   DirectiveKindMap[".macros_on"] = DK_MACROS_ON;
   DirectiveKindMap[".macros_off"] = DK_MACROS_OFF;
   DirectiveKindMap[".macro"] = DK_MACRO;
diff --git a/lib/MC/MCParser/COFFAsmParser.cpp b/lib/MC/MCParser/COFFAsmParser.cpp
index 388304a72395c22f60982f3944b2c56e5b50b60c..1217ea99e4656e38f0dad1a10e49b181be3af48b 100644
--- a/lib/MC/MCParser/COFFAsmParser.cpp
+++ b/lib/MC/MCParser/COFFAsmParser.cpp
@@ -1,9 +1,8 @@
 //===- COFFAsmParser.cpp - COFF Assembly Parser ---------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/MC/MCParser/DarwinAsmParser.cpp b/lib/MC/MCParser/DarwinAsmParser.cpp
index 07926d6c1c9ab7840f17c93f8e7fcce75fed1e44..09d41db06c60f98b5caa9a4fb8b3c9b1b64ca516 100644
--- a/lib/MC/MCParser/DarwinAsmParser.cpp
+++ b/lib/MC/MCParser/DarwinAsmParser.cpp
@@ -1,9 +1,8 @@
 //===- DarwinAsmParser.cpp - Darwin (Mach-O) Assembly Parser --------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -1148,7 +1147,10 @@ static Triple::OSType getOSTypeFromPlatform(MachO::PlatformType Type) {
   case MachO::PLATFORM_IOS:     return Triple::IOS;
   case MachO::PLATFORM_TVOS:    return Triple::TvOS;
   case MachO::PLATFORM_WATCHOS: return Triple::WatchOS;
-  case MachO::PLATFORM_BRIDGEOS: /* silence warning */break;
+  case MachO::PLATFORM_BRIDGEOS:         /* silence warning */ break;
+  case MachO::PLATFORM_IOSSIMULATOR:     /* silence warning */ break;
+  case MachO::PLATFORM_TVOSSIMULATOR:    /* silence warning */ break;
+  case MachO::PLATFORM_WATCHOSSIMULATOR: /* silence warning */ break;
   }
   llvm_unreachable("Invalid mach-o platform type");
 }
diff --git a/lib/MC/MCParser/ELFAsmParser.cpp b/lib/MC/MCParser/ELFAsmParser.cpp
index d568f7a71eeb5dca8f4e9293e2c0678be636d706..9e5b54a6881303587ca132a344fd8ef90f5448fc 100644
--- a/lib/MC/MCParser/ELFAsmParser.cpp
+++ b/lib/MC/MCParser/ELFAsmParser.cpp
@@ -1,9 +1,8 @@
 //===- ELFAsmParser.cpp - ELF Assembly Parser -----------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/MC/MCParser/LLVMBuild.txt b/lib/MC/MCParser/LLVMBuild.txt
index bcb0febf332369489e4f0e6141f1f5563c4edd41..52b0d6c4a5094c52fa25711a3b398475893746d1 100644
--- a/lib/MC/MCParser/LLVMBuild.txt
+++ b/lib/MC/MCParser/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/MC/MCParser/LLVMBuild.txt --------------------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/MC/MCParser/MCAsmLexer.cpp b/lib/MC/MCParser/MCAsmLexer.cpp
index 10960fc696334870f353a07c9a4b6e990f41c305..497055bc1760a37f288295b2657e7a90f0de264c 100644
--- a/lib/MC/MCParser/MCAsmLexer.cpp
+++ b/lib/MC/MCParser/MCAsmLexer.cpp
@@ -1,9 +1,8 @@
 //===- MCAsmLexer.cpp - Abstract Asm Lexer Interface ----------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/MC/MCParser/MCAsmParser.cpp b/lib/MC/MCParser/MCAsmParser.cpp
index efedcdc5a314d10e4c1882f0ad9095a136538bb7..41a1ee555d6fbf24826197d6fafe60cff1fa86c0 100644
--- a/lib/MC/MCParser/MCAsmParser.cpp
+++ b/lib/MC/MCParser/MCAsmParser.cpp
@@ -1,9 +1,8 @@
 //===-- MCAsmParser.cpp - Abstract Asm Parser Interface -------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/MC/MCParser/MCAsmParserExtension.cpp b/lib/MC/MCParser/MCAsmParserExtension.cpp
index 031f473dc5fe4bac44daf53786138c8bddbcca88..18d18f0cf6ed5e3b412ee7c56077c4beb991a6c2 100644
--- a/lib/MC/MCParser/MCAsmParserExtension.cpp
+++ b/lib/MC/MCParser/MCAsmParserExtension.cpp
@@ -1,9 +1,8 @@
 //===- MCAsmParserExtension.cpp - Asm Parser Hooks ------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/MC/MCParser/MCTargetAsmParser.cpp b/lib/MC/MCParser/MCTargetAsmParser.cpp
index a0c06c9d501894cbf43c0125310aea9855dac8bc..940f26d4750b4324f178fd07ed677e85e411ddd9 100644
--- a/lib/MC/MCParser/MCTargetAsmParser.cpp
+++ b/lib/MC/MCParser/MCTargetAsmParser.cpp
@@ -1,9 +1,8 @@
 //===-- MCTargetAsmParser.cpp - Target Assembly Parser --------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/MC/MCParser/WasmAsmParser.cpp b/lib/MC/MCParser/WasmAsmParser.cpp
index 93bb0cb3c72e476360da73d8e054f8bb8bcc0af2..17d86aa0d64a2025415afca205d7f56b3388a31a 100644
--- a/lib/MC/MCParser/WasmAsmParser.cpp
+++ b/lib/MC/MCParser/WasmAsmParser.cpp
@@ -1,9 +1,8 @@
 //===- WasmAsmParser.cpp - Wasm Assembly Parser -----------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 // --
 //
@@ -22,6 +21,7 @@
 #include "llvm/MC/MCParser/MCAsmLexer.h"
 #include "llvm/MC/MCParser/MCAsmParser.h"
 #include "llvm/MC/MCParser/MCAsmParserExtension.h"
+#include "llvm/MC/MCSectionWasm.h"
 #include "llvm/MC/MCStreamer.h"
 #include "llvm/MC/MCSymbol.h"
 #include "llvm/MC/MCSymbolWasm.h"
@@ -32,8 +32,8 @@ using namespace llvm;
 namespace {
 
 class WasmAsmParser : public MCAsmParserExtension {
-  MCAsmParser *Parser;
-  MCAsmLexer *Lexer;
+  MCAsmParser *Parser = nullptr;
+  MCAsmLexer *Lexer = nullptr;
 
   template<bool (WasmAsmParser::*HandlerMethod)(StringRef, SMLoc)>
   void addDirectiveHandler(StringRef Directive) {
@@ -44,9 +44,7 @@ class WasmAsmParser : public MCAsmParserExtension {
   }
 
 public:
-  WasmAsmParser() : Parser(nullptr), Lexer(nullptr) {
-    BracketExpressionsSupported = true;
-  }
+  WasmAsmParser() { BracketExpressionsSupported = true; }
 
   void Initialize(MCAsmParser &P) override {
     Parser = &P;
@@ -60,19 +58,20 @@ public:
     addDirectiveHandler<&WasmAsmParser::parseDirectiveType>(".type");
   }
 
-  bool Error(const StringRef &msg, const AsmToken &tok) {
-    return Parser->Error(tok.getLoc(), msg + tok.getString());
+  bool error(const StringRef &Msg, const AsmToken &Tok) {
+    return Parser->Error(Tok.getLoc(), Msg + Tok.getString());
   }
 
-  bool IsNext(AsmToken::TokenKind Kind) {
-    auto ok = Lexer->is(Kind);
-    if (ok) Lex();
-    return ok;
+  bool isNext(AsmToken::TokenKind Kind) {
+    auto Ok = Lexer->is(Kind);
+    if (Ok)
+      Lex();
+    return Ok;
   }
 
-  bool Expect(AsmToken::TokenKind Kind, const char *KindName) {
-    if (!IsNext(Kind))
-      return Error(std::string("Expected ") + KindName + ", instead got: ",
+  bool expect(AsmToken::TokenKind Kind, const char *KindName) {
+    if (!isNext(Kind))
+      return error(std::string("Expected ") + KindName + ", instead got: ",
                    Lexer->getTok());
     return false;
   }
@@ -83,8 +82,19 @@ public:
   }
 
   bool parseSectionDirective(StringRef, SMLoc) {
-    // FIXME: .section currently no-op.
-    while (Lexer->isNot(AsmToken::EndOfStatement)) Parser->Lex();
+    StringRef Name;
+    if (Parser->parseIdentifier(Name))
+      return TokError("expected identifier in directive");
+    // FIXME: currently requiring this very fixed format.
+    if (expect(AsmToken::Comma, ",") || expect(AsmToken::String, "string") ||
+        expect(AsmToken::Comma, ",") || expect(AsmToken::At, "@") ||
+        expect(AsmToken::EndOfStatement, "eol"))
+      return true;
+    // This is done automatically by the assembler for text sections currently,
+    // so we don't need to emit that here. This is what it would do (and may
+    // be needed later for other section types):
+    // auto WS = getContext().getWasmSection(Name, SectionKind::getText());
+    // getStreamer().SwitchSection(WS);
     return false;
   }
 
@@ -95,17 +105,17 @@ public:
     if (Parser->parseIdentifier(Name))
       return TokError("expected identifier in directive");
     auto Sym = getContext().getOrCreateSymbol(Name);
-    if (Lexer->isNot(AsmToken::Comma))
-      return TokError("unexpected token in directive");
-    Lex();
+    if (expect(AsmToken::Comma, ","))
+      return true;
     const MCExpr *Expr;
     if (Parser->parseExpression(Expr))
       return true;
-    if (Lexer->isNot(AsmToken::EndOfStatement))
-      return TokError("unexpected token in directive");
-    Lex();
-    // MCWasmStreamer implements this.
-    getStreamer().emitELFSize(Sym, Expr);
+    if (expect(AsmToken::EndOfStatement, "eol"))
+      return true;
+    // This is done automatically by the assembler for functions currently,
+    // so we don't need to emit that here. This is what it would do:
+    (void)Sym;
+    // getStreamer().emitELFSize(Sym, Expr);
     return false;
   }
 
@@ -113,24 +123,24 @@ public:
     // This could be the start of a function, check if followed by
     // "label,@function"
     if (!Lexer->is(AsmToken::Identifier))
-      return Error("Expected label after .type directive, got: ",
+      return error("Expected label after .type directive, got: ",
                    Lexer->getTok());
     auto WasmSym = cast<MCSymbolWasm>(
                      getStreamer().getContext().getOrCreateSymbol(
                        Lexer->getTok().getString()));
     Lex();
-    if (!(IsNext(AsmToken::Comma) && IsNext(AsmToken::At) &&
+    if (!(isNext(AsmToken::Comma) && isNext(AsmToken::At) &&
           Lexer->is(AsmToken::Identifier)))
-      return Error("Expected label,@type declaration, got: ", Lexer->getTok());
+      return error("Expected label,@type declaration, got: ", Lexer->getTok());
     auto TypeName = Lexer->getTok().getString();
     if (TypeName == "function")
       WasmSym->setType(wasm::WASM_SYMBOL_TYPE_FUNCTION);
     else if (TypeName == "global")
       WasmSym->setType(wasm::WASM_SYMBOL_TYPE_GLOBAL);
     else
-      return Error("Unknown WASM symbol type: ", Lexer->getTok());
+      return error("Unknown WASM symbol type: ", Lexer->getTok());
     Lex();
-    return Expect(AsmToken::EndOfStatement, "EOL");
+    return expect(AsmToken::EndOfStatement, "EOL");
   }
 };
 
diff --git a/lib/MC/MCRegisterInfo.cpp b/lib/MC/MCRegisterInfo.cpp
index 5abae5379867807b62c6e936d8702aea1f75884c..4273b876b7bb0cbfe60f95102807ad71e56f1ec0 100644
--- a/lib/MC/MCRegisterInfo.cpp
+++ b/lib/MC/MCRegisterInfo.cpp
@@ -1,9 +1,8 @@
 //===- MC/MCRegisterInfo.cpp - Target Register Description ----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/MC/MCSchedule.cpp b/lib/MC/MCSchedule.cpp
index 929bd7f6046cc3e19dc45a8df25c08281f9c5e8e..1fc5ec5e975f8424f384deb9dc8658230ab2ea85 100644
--- a/lib/MC/MCSchedule.cpp
+++ b/lib/MC/MCSchedule.cpp
@@ -1,9 +1,8 @@
 //===- MCSchedule.cpp - Scheduling ------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -150,3 +149,19 @@ MCSchedModel::getReciprocalThroughput(unsigned SchedClass,
   // that it can execute at the maximum default issue width.
   return 1.0 / DefaultIssueWidth;
 }
+
+unsigned
+MCSchedModel::getForwardingDelayCycles(ArrayRef<MCReadAdvanceEntry> Entries,
+                                       unsigned WriteResourceID) {
+  if (Entries.empty())
+    return 0;
+
+  int DelayCycles = 0;
+  for (const MCReadAdvanceEntry &E : Entries) {
+    if (E.WriteResourceID != WriteResourceID)
+      continue;
+    DelayCycles = std::min(DelayCycles, E.Cycles);
+  }
+
+  return std::abs(DelayCycles);
+}
diff --git a/lib/MC/MCSection.cpp b/lib/MC/MCSection.cpp
index d4f11d10136a6af42b2db969fdd47b21142e40c1..2c892ab8160815a44cf081c9c9e118d552b9099d 100644
--- a/lib/MC/MCSection.cpp
+++ b/lib/MC/MCSection.cpp
@@ -1,9 +1,8 @@
 //===- lib/MC/MCSection.cpp - Machine Code Section Representation ---------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/MC/MCSectionCOFF.cpp b/lib/MC/MCSectionCOFF.cpp
index c861963eec8a47511d4ad9bf92996d1dcc213f8a..f0c06f70bd734693731079426931069334af28e8 100644
--- a/lib/MC/MCSectionCOFF.cpp
+++ b/lib/MC/MCSectionCOFF.cpp
@@ -1,9 +1,8 @@
 //===- lib/MC/MCSectionCOFF.cpp - COFF Code Section Representation --------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -15,8 +14,6 @@
 
 using namespace llvm;
 
-MCSectionCOFF::~MCSectionCOFF() = default; // anchor.
-
 // ShouldOmitSectionDirective - Decides whether a '.section' directive
 // should be printed before the section name
 bool MCSectionCOFF::ShouldOmitSectionDirective(StringRef Name,
diff --git a/lib/MC/MCSectionELF.cpp b/lib/MC/MCSectionELF.cpp
index 7ee1694ebbf71dd8f09be6386683a6110f91fea2..e3c24efa93f31f864565f9d75a9f9e9f5384876c 100644
--- a/lib/MC/MCSectionELF.cpp
+++ b/lib/MC/MCSectionELF.cpp
@@ -1,9 +1,8 @@
 //===- lib/MC/MCSectionELF.cpp - ELF Code Section Representation ----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -18,8 +17,6 @@
 
 using namespace llvm;
 
-MCSectionELF::~MCSectionELF() = default; // anchor.
-
 // Decides whether a '.section' directive
 // should be printed before the section name.
 bool MCSectionELF::ShouldOmitSectionDirective(StringRef Name,
diff --git a/lib/MC/MCSectionMachO.cpp b/lib/MC/MCSectionMachO.cpp
index f40237231a2f7a162c0e3bb80b3bff0c810c4684..0fd89dcbe5fa7b268ff6cfe6248fa2bf457e4ebd 100644
--- a/lib/MC/MCSectionMachO.cpp
+++ b/lib/MC/MCSectionMachO.cpp
@@ -1,9 +1,8 @@
 //===- lib/MC/MCSectionMachO.cpp - MachO Code Section Representation ------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/MC/MCSectionWasm.cpp b/lib/MC/MCSectionWasm.cpp
index 626027a24f97b264e8019ea01acdf643a126f6fa..164ded9a1f8877c7b347443a6b120671d3e03088 100644
--- a/lib/MC/MCSectionWasm.cpp
+++ b/lib/MC/MCSectionWasm.cpp
@@ -1,9 +1,8 @@
 //===- lib/MC/MCSectionWasm.cpp - Wasm Code Section Representation --------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -15,11 +14,9 @@
 
 using namespace llvm;
 
-MCSectionWasm::~MCSectionWasm() {} // anchor.
-
 // Decides whether a '.section' directive
 // should be printed before the section name.
-bool MCSectionWasm::ShouldOmitSectionDirective(StringRef Name,
+bool MCSectionWasm::shouldOmitSectionDirective(StringRef Name,
                                                const MCAsmInfo &MAI) const {
   return MAI.shouldOmitSectionDirective(Name);
 }
@@ -51,7 +48,7 @@ void MCSectionWasm::PrintSwitchToSection(const MCAsmInfo &MAI, const Triple &T,
                                          raw_ostream &OS,
                                          const MCExpr *Subsection) const {
 
-  if (ShouldOmitSectionDirective(SectionName, MAI)) {
+  if (shouldOmitSectionDirective(SectionName, MAI)) {
     OS << '\t' << getSectionName();
     if (Subsection) {
       OS << '\t';
diff --git a/lib/MC/MCStreamer.cpp b/lib/MC/MCStreamer.cpp
index 85e69f6f663e0283a47c3d2e01a2b452d34daa7a..554cce1214adcb111c4d1037219015559f54b9c4 100644
--- a/lib/MC/MCStreamer.cpp
+++ b/lib/MC/MCStreamer.cpp
@@ -1,9 +1,8 @@
 //===- lib/MC/MCStreamer.cpp - Streaming Machine Code Output --------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -221,6 +220,13 @@ void MCStreamer::emitDwarfFile0Directive(StringRef Directory,
                                       Source);
 }
 
+void MCStreamer::EmitCFIBKeyFrame() {
+  MCDwarfFrameInfo *CurFrame = getCurrentDwarfFrameInfo();
+  if (!CurFrame)
+    return;
+  CurFrame->IsBKeyFrame = true;
+}
+
 void MCStreamer::EmitDwarfLocDirective(unsigned FileNo, unsigned Line,
                                        unsigned Column, unsigned Flags,
                                        unsigned Isa,
@@ -858,13 +864,11 @@ void MCStreamer::EmitWinCFIEndProlog(SMLoc Loc) {
   CurFrame->PrologEnd = Label;
 }
 
-void MCStreamer::EmitCOFFSafeSEH(MCSymbol const *Symbol) {
-}
+void MCStreamer::EmitCOFFSafeSEH(MCSymbol const *Symbol) {}
 
 void MCStreamer::EmitCOFFSymbolIndex(MCSymbol const *Symbol) {}
 
-void MCStreamer::EmitCOFFSectionIndex(MCSymbol const *Symbol) {
-}
+void MCStreamer::EmitCOFFSectionIndex(MCSymbol const *Symbol) {}
 
 void MCStreamer::EmitCOFFSecRel32(MCSymbol const *Symbol, uint64_t Offset) {}
 
@@ -874,9 +878,12 @@ void MCStreamer::EmitCOFFImgRel32(MCSymbol const *Symbol, int64_t Offset) {}
 /// the specified string in the output .s file.  This capability is
 /// indicated by the hasRawTextSupport() predicate.
 void MCStreamer::EmitRawTextImpl(StringRef String) {
-  errs() << "EmitRawText called on an MCStreamer that doesn't support it, "
-  " something must not be fully mc'ized\n";
-  abort();
+  // This is not llvm_unreachable for the sake of out of tree backend
+  // developers who may not have assembly streamers and should serve as a
+  // reminder to not accidentally call EmitRawText in the absence of such.
+  report_fatal_error("EmitRawText called on an MCStreamer that doesn't support "
+                     "it (target backend is likely missing an AsmStreamer "
+                     "implementation)");
 }
 
 void MCStreamer::EmitRawText(const Twine &T) {
@@ -910,8 +917,9 @@ void MCStreamer::EmitAssignment(MCSymbol *Symbol, const MCExpr *Value) {
     TS->emitAssignment(Symbol, Value);
 }
 
-void MCTargetStreamer::prettyPrintAsm(MCInstPrinter &InstPrinter, raw_ostream &OS,
-                              const MCInst &Inst, const MCSubtargetInfo &STI) {
+void MCTargetStreamer::prettyPrintAsm(MCInstPrinter &InstPrinter,
+                                      raw_ostream &OS, const MCInst &Inst,
+                                      const MCSubtargetInfo &STI) {
   InstPrinter.printInst(&Inst, OS, "", STI);
 }
 
@@ -944,8 +952,7 @@ void MCStreamer::visitUsedExpr(const MCExpr &Expr) {
   }
 }
 
-void MCStreamer::EmitInstruction(const MCInst &Inst, const MCSubtargetInfo &STI,
-                                 bool) {
+void MCStreamer::EmitInstruction(const MCInst &Inst, const MCSubtargetInfo &) {
   // Scan for values.
   for (unsigned i = Inst.getNumOperands(); i--;)
     if (Inst.getOperand(i).isExpr())
diff --git a/lib/MC/MCSubtargetInfo.cpp b/lib/MC/MCSubtargetInfo.cpp
index f6167826fae24b53a032b1b72ae7a77bff60c55b..540af9a9d478d9555bd5923fbfeeb495350ec8d0 100644
--- a/lib/MC/MCSubtargetInfo.cpp
+++ b/lib/MC/MCSubtargetInfo.cpp
@@ -1,9 +1,8 @@
 //===- MCSubtargetInfo.cpp - Subtarget Information ------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/MC/MCSymbol.cpp b/lib/MC/MCSymbol.cpp
index 5502c658f56544bcef00f3e53daa643f35eb9e55..67cab9a92722acd7d4188fe8dc86bafe3d55bde2 100644
--- a/lib/MC/MCSymbol.cpp
+++ b/lib/MC/MCSymbol.cpp
@@ -1,9 +1,8 @@
 //===- lib/MC/MCSymbol.cpp - MCSymbol implementation ----------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/MC/MCSymbolELF.cpp b/lib/MC/MCSymbolELF.cpp
index 12c724f6b1ee3be3ffb42a532bd6a2eb190a02d4..3931f1e572178628bfec7115ba1ddc55c06cc2db 100644
--- a/lib/MC/MCSymbolELF.cpp
+++ b/lib/MC/MCSymbolELF.cpp
@@ -1,9 +1,8 @@
 //===- lib/MC/MCSymbolELF.cpp ---------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/MC/MCTargetOptions.cpp b/lib/MC/MCTargetOptions.cpp
index b85e53db5d616af8bd97193b3fa3fea038e90110..a072928269c704c04f6bf4ead35b1ea818bc369a 100644
--- a/lib/MC/MCTargetOptions.cpp
+++ b/lib/MC/MCTargetOptions.cpp
@@ -1,9 +1,8 @@
 //===- lib/MC/MCTargetOptions.cpp - MC Target Options ---------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/MC/MCValue.cpp b/lib/MC/MCValue.cpp
index 7e03913aa680a3734810088355b8b81568683a48..81da47b2eced2085b30aac4704bddeb368379d3e 100644
--- a/lib/MC/MCValue.cpp
+++ b/lib/MC/MCValue.cpp
@@ -1,9 +1,8 @@
 //===- lib/MC/MCValue.cpp - MCValue implementation ------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/MC/MCWasmObjectTargetWriter.cpp b/lib/MC/MCWasmObjectTargetWriter.cpp
index 59082a160caf05f3b5dde8e5ab8895a8175fb574..e46257823e3437c223ba1b6ca140819ccdf46942 100644
--- a/lib/MC/MCWasmObjectTargetWriter.cpp
+++ b/lib/MC/MCWasmObjectTargetWriter.cpp
@@ -1,9 +1,8 @@
 //===-- MCWasmObjectTargetWriter.cpp - Wasm Target Writer Subclass --------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -11,8 +10,8 @@
 
 using namespace llvm;
 
-MCWasmObjectTargetWriter::MCWasmObjectTargetWriter(bool Is64Bit_)
-    : Is64Bit(Is64Bit_) {}
+MCWasmObjectTargetWriter::MCWasmObjectTargetWriter(bool Is64Bit)
+    : Is64Bit(Is64Bit) {}
 
 // Pin the vtable to this object file
 MCWasmObjectTargetWriter::~MCWasmObjectTargetWriter() = default;
diff --git a/lib/MC/MCWasmStreamer.cpp b/lib/MC/MCWasmStreamer.cpp
index d2a152058b90dfb024f195a0559f80960ac2cc52..86fa72197855bdf8f5f16600c8ec0930c8fc397f 100644
--- a/lib/MC/MCWasmStreamer.cpp
+++ b/lib/MC/MCWasmStreamer.cpp
@@ -1,9 +1,8 @@
 //===- lib/MC/MCWasmStreamer.cpp - Wasm Object Output ---------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -35,15 +34,15 @@
 
 using namespace llvm;
 
-MCWasmStreamer::~MCWasmStreamer() {}
+MCWasmStreamer::~MCWasmStreamer() = default; // anchor.
 
 void MCWasmStreamer::mergeFragment(MCDataFragment *DF, MCDataFragment *EF) {
   flushPendingLabels(DF, DF->getContents().size());
 
-  for (unsigned i = 0, e = EF->getFixups().size(); i != e; ++i) {
-    EF->getFixups()[i].setOffset(EF->getFixups()[i].getOffset() +
+  for (unsigned I = 0, E = EF->getFixups().size(); I != E; ++I) {
+    EF->getFixups()[I].setOffset(EF->getFixups()[I].getOffset() +
                                  DF->getContents().size());
-    DF->getFixups().push_back(EF->getFixups()[i]);
+    DF->getFixups().push_back(EF->getFixups()[I]);
   }
   if (DF->getSubtargetInfo() == nullptr && EF->getSubtargetInfo())
     DF->setHasInstructions(*EF->getSubtargetInfo());
@@ -119,6 +118,11 @@ bool MCWasmStreamer::EmitSymbolAttribute(MCSymbol *S, MCSymbolAttr Attribute) {
     break;
 
   case MCSA_ELF_TypeObject:
+  case MCSA_Cold:
+    break;
+
+  case MCSA_NoDeadStrip:
+    Symbol->setExported();
     break;
 
   default:
@@ -179,9 +183,9 @@ void MCWasmStreamer::EmitInstToData(const MCInst &Inst,
   MCDataFragment *DF = getOrCreateDataFragment();
 
   // Add the fixups and data.
-  for (unsigned i = 0, e = Fixups.size(); i != e; ++i) {
-    Fixups[i].setOffset(Fixups[i].getOffset() + DF->getContents().size());
-    DF->getFixups().push_back(Fixups[i]);
+  for (unsigned I = 0, E = Fixups.size(); I != E; ++I) {
+    Fixups[I].setOffset(Fixups[I].getOffset() + DF->getContents().size());
+    DF->getFixups().push_back(Fixups[I]);
   }
   DF->setHasInstructions(STI);
   DF->getContents().append(Code.begin(), Code.end());
diff --git a/lib/MC/MCWin64EH.cpp b/lib/MC/MCWin64EH.cpp
index 0724b109e1a1da6ab89a43e7c4d3561e6470bc7a..93c115970e55d806e18fe6cf50e9546be6bac1ef 100644
--- a/lib/MC/MCWin64EH.cpp
+++ b/lib/MC/MCWin64EH.cpp
@@ -1,9 +1,8 @@
 //===- lib/MC/MCWin64EH.cpp - MCWin64EH implementation --------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -453,6 +452,38 @@ static void ARM64EmitUnwindCode(MCStreamer &streamer, const MCSymbol *begin,
   }
 }
 
+// Returns the epilog symbol of an epilog with the exact same unwind code
+// sequence, if it exists.  Otherwise, returns nulltpr.
+// EpilogInstrs - Unwind codes for the current epilog.
+// Epilogs - Epilogs that potentialy match the current epilog.
+static MCSymbol*
+FindMatchingEpilog(const std::vector<WinEH::Instruction>& EpilogInstrs,
+                   const std::vector<MCSymbol *>& Epilogs,
+                   const WinEH::FrameInfo *info) {
+  for (auto *EpilogStart : Epilogs) {
+    auto InstrsIter = info->EpilogMap.find(EpilogStart);
+    assert(InstrsIter != info->EpilogMap.end() &&
+           "Epilog not found in EpilogMap");
+    const auto &Instrs = InstrsIter->second;
+
+    if (Instrs.size() != EpilogInstrs.size())
+      continue;
+
+    bool Match = true;
+    for (unsigned i = 0; i < Instrs.size(); ++i)
+      if (Instrs[i].Operation != EpilogInstrs[i].Operation ||
+          Instrs[i].Offset != EpilogInstrs[i].Offset ||
+          Instrs[i].Register != EpilogInstrs[i].Register) {
+         Match = false;
+         break;
+      }
+
+    if (Match)
+      return EpilogStart;
+  }
+  return nullptr;
+}
+
 // Populate the .xdata section.  The format of .xdata on ARM64 is documented at
 // https://docs.microsoft.com/en-us/cpp/build/arm64-exception-handling
 static void ARM64EmitUnwindInfo(MCStreamer &streamer, WinEH::FrameInfo *info) {
@@ -477,12 +508,28 @@ static void ARM64EmitUnwindInfo(MCStreamer &streamer, WinEH::FrameInfo *info) {
 
   // Process epilogs.
   MapVector<MCSymbol *, uint32_t> EpilogInfo;
+  // Epilogs processed so far.
+  std::vector<MCSymbol *> AddedEpilogs;
+
   for (auto &I : info->EpilogMap) {
     MCSymbol *EpilogStart = I.first;
     auto &EpilogInstrs = I.second;
     uint32_t CodeBytes = ARM64CountOfUnwindCodes(EpilogInstrs);
-    EpilogInfo[EpilogStart] = TotalCodeBytes;
-    TotalCodeBytes += CodeBytes;
+
+    MCSymbol* MatchingEpilog =
+      FindMatchingEpilog(EpilogInstrs, AddedEpilogs, info);
+    if (MatchingEpilog) {
+      assert(EpilogInfo.find(MatchingEpilog) != EpilogInfo.end() &&
+             "Duplicate epilog not found");
+      EpilogInfo[EpilogStart] = EpilogInfo.lookup(MatchingEpilog);
+      // Clear the unwind codes in the EpilogMap, so that they don't get output
+      // in the logic below.
+      EpilogInstrs.clear();
+    } else {
+      EpilogInfo[EpilogStart] = TotalCodeBytes;
+      TotalCodeBytes += CodeBytes;
+      AddedEpilogs.push_back(EpilogStart);
+    }
   }
 
   // Code Words, Epilog count, E, X, Vers, Function Length
diff --git a/lib/MC/MCWinCOFFStreamer.cpp b/lib/MC/MCWinCOFFStreamer.cpp
index 7b1dc7abf708aa35ae1bd0895bb47afab3d78150..08f513f13cfedd72f8a1c176966666d3e88eb6d6 100644
--- a/lib/MC/MCWinCOFFStreamer.cpp
+++ b/lib/MC/MCWinCOFFStreamer.cpp
@@ -1,9 +1,8 @@
 //===- llvm/MC/MCWinCOFFStreamer.cpp --------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/MC/MCWinEH.cpp b/lib/MC/MCWinEH.cpp
index a5d0f5a2cb7509dd88021b84881ae4c6fa1460ad..e58a0b2cf6540cd445b85387fd460c522dbf44ec 100644
--- a/lib/MC/MCWinEH.cpp
+++ b/lib/MC/MCWinEH.cpp
@@ -1,9 +1,8 @@
 //===- lib/MC/MCWinEH.cpp - Windows EH implementation ---------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/MC/MachObjectWriter.cpp b/lib/MC/MachObjectWriter.cpp
index 2fa65658ccfacfec0db3928d16586ee18f3be9e8..395273e44f6f8e4f2f071f24da57d28a965d2ada 100644
--- a/lib/MC/MachObjectWriter.cpp
+++ b/lib/MC/MachObjectWriter.cpp
@@ -1,9 +1,8 @@
 //===- lib/MC/MachObjectWriter.cpp - Mach-O File Writer -------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/MC/StringTableBuilder.cpp b/lib/MC/StringTableBuilder.cpp
index de40a7728d3fcfb6775b30e91ecf828cb6b01fcf..bd51aa413fbb32d40149754ad2493e46e696adcd 100644
--- a/lib/MC/StringTableBuilder.cpp
+++ b/lib/MC/StringTableBuilder.cpp
@@ -1,9 +1,8 @@
 //===- StringTableBuilder.cpp - String table building utility -------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/MC/SubtargetFeature.cpp b/lib/MC/SubtargetFeature.cpp
index b69af24b531e699f1585b959033d825820eb8c21..793f00d46eda9d6bcf59ae91d4f4c46959f803b4 100644
--- a/lib/MC/SubtargetFeature.cpp
+++ b/lib/MC/SubtargetFeature.cpp
@@ -1,9 +1,8 @@
 //===- SubtargetFeature.cpp - CPU characteristics Implementation ----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/MC/WasmObjectWriter.cpp b/lib/MC/WasmObjectWriter.cpp
index 8a369204cd6c85aed7061960acf04f2d4d247fe7..600928c05bc1cf882fd99dcda4cd86cb8a704292 100644
--- a/lib/MC/WasmObjectWriter.cpp
+++ b/lib/MC/WasmObjectWriter.cpp
@@ -1,9 +1,8 @@
 //===- lib/MC/WasmObjectWriter.cpp - Wasm File Writer ---------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -41,7 +40,7 @@ namespace {
 
 // Went we ceate the indirect function table we start at 1, so that there is
 // and emtpy slot at 0 and therefore calling a null function pointer will trap.
-static const uint32_t kInitialTableOffset = 1;
+static const uint32_t InitialTableOffset = 1;
 
 // For patching purposes, we need to remember where each section starts, both
 // for patching up the section size field, and for patching up references to
@@ -61,7 +60,7 @@ struct SectionBookkeeping {
 // TODO: Consider using wasm::WasmSignature directly instead.
 struct WasmSignature {
   // Support empty and tombstone instances, needed by DenseMap.
-  enum { Plain, Empty, Tombstone } State;
+  enum { Plain, Empty, Tombstone } State = Plain;
 
   // The return types of the function.
   SmallVector<wasm::ValType, 1> Returns;
@@ -69,8 +68,6 @@ struct WasmSignature {
   // The parameter types of the function.
   SmallVector<wasm::ValType, 4> Params;
 
-  WasmSignature() : State(Plain) {}
-
   bool operator==(const WasmSignature &Other) const {
     return State == Other.State && Returns == Other.Returns &&
            Params == Other.Params;
@@ -151,11 +148,11 @@ struct WasmRelocationEntry {
 
   bool hasAddend() const {
     switch (Type) {
-    case wasm::R_WEBASSEMBLY_MEMORY_ADDR_LEB:
-    case wasm::R_WEBASSEMBLY_MEMORY_ADDR_SLEB:
-    case wasm::R_WEBASSEMBLY_MEMORY_ADDR_I32:
-    case wasm::R_WEBASSEMBLY_FUNCTION_OFFSET_I32:
-    case wasm::R_WEBASSEMBLY_SECTION_OFFSET_I32:
+    case wasm::R_WASM_MEMORY_ADDR_LEB:
+    case wasm::R_WASM_MEMORY_ADDR_SLEB:
+    case wasm::R_WASM_MEMORY_ADDR_I32:
+    case wasm::R_WASM_FUNCTION_OFFSET_I32:
+    case wasm::R_WASM_SECTION_OFFSET_I32:
       return true;
     default:
       return false;
@@ -173,7 +170,7 @@ struct WasmRelocationEntry {
 #endif
 };
 
-static const uint32_t INVALID_INDEX = -1;
+static const uint32_t InvalidIndex = -1;
 
 struct WasmCustomSection {
 
@@ -185,7 +182,7 @@ struct WasmCustomSection {
 
   WasmCustomSection(StringRef Name, MCSectionWasm *Section)
       : Name(Name), Section(Section), OutputContentsOffset(0),
-        OutputIndex(INVALID_INDEX) {}
+        OutputIndex(InvalidIndex) {}
 };
 
 #if !defined(NDEBUG)
@@ -195,6 +192,33 @@ raw_ostream &operator<<(raw_ostream &OS, const WasmRelocationEntry &Rel) {
 }
 #endif
 
+// Write X as an (unsigned) LEB value at offset Offset in Stream, padded
+// to allow patching.
+static void writePatchableLEB(raw_pwrite_stream &Stream, uint32_t X,
+                              uint64_t Offset) {
+  uint8_t Buffer[5];
+  unsigned SizeLen = encodeULEB128(X, Buffer, 5);
+  assert(SizeLen == 5);
+  Stream.pwrite((char *)Buffer, SizeLen, Offset);
+}
+
+// Write X as an signed LEB value at offset Offset in Stream, padded
+// to allow patching.
+static void writePatchableSLEB(raw_pwrite_stream &Stream, int32_t X,
+                               uint64_t Offset) {
+  uint8_t Buffer[5];
+  unsigned SizeLen = encodeSLEB128(X, Buffer, 5);
+  assert(SizeLen == 5);
+  Stream.pwrite((char *)Buffer, SizeLen, Offset);
+}
+
+// Write X as a plain integer value at offset Offset in Stream.
+static void writeI32(raw_pwrite_stream &Stream, uint32_t X, uint64_t Offset) {
+  uint8_t Buffer[4];
+  support::endian::write32le(Buffer, X);
+  Stream.pwrite((char *)Buffer, sizeof(Buffer), Offset);
+}
+
 class WasmObjectWriter : public MCObjectWriter {
   support::endian::Writer W;
 
@@ -224,6 +248,7 @@ class WasmObjectWriter : public MCObjectWriter {
   // Stores output data (index, relocations, content offset) for custom
   // section.
   std::vector<WasmCustomSection> CustomSections;
+  std::unique_ptr<WasmCustomSection> ProducersSection;
   // Relocations for fixing up references in the custom sections.
   DenseMap<const MCSectionWasm *, std::vector<WasmRelocationEntry>>
       CustomSectionsRelocations;
@@ -255,8 +280,6 @@ public:
                    raw_pwrite_stream &OS)
       : W(OS, support::little), TargetObjectWriter(std::move(MOTW)) {}
 
-  ~WasmObjectWriter() override;
-
 private:
   void reset() override {
     CodeRelocations.clear();
@@ -265,6 +288,8 @@ private:
     WasmIndices.clear();
     TableIndices.clear();
     DataLocations.clear();
+    CustomSections.clear();
+    ProducersSection.reset();
     CustomSectionsRelocations.clear();
     SignatureIndices.clear();
     Signatures.clear();
@@ -311,7 +336,8 @@ private:
       ArrayRef<wasm::WasmSymbolInfo> SymbolInfos,
       ArrayRef<std::pair<uint16_t, uint32_t>> InitFuncs,
       const std::map<StringRef, std::vector<WasmComdatEntry>> &Comdats);
-  void writeCustomSections(const MCAssembler &Asm, const MCAsmLayout &Layout);
+  void writeCustomSection(WasmCustomSection &CustomSection,
+                          const MCAssembler &Asm, const MCAsmLayout &Layout);
   void writeCustomRelocSections();
   void
   updateCustomSectionRelocations(const SmallVector<WasmFunction, 4> &Functions,
@@ -330,8 +356,6 @@ private:
 
 } // end anonymous namespace
 
-WasmObjectWriter::~WasmObjectWriter() {}
-
 // Write out a section header and a patchable section size field.
 void WasmObjectWriter::startSection(SectionBookkeeping &Section,
                                     unsigned SectionId) {
@@ -342,7 +366,7 @@ void WasmObjectWriter::startSection(SectionBookkeeping &Section,
 
   // The section size. We don't know the size yet, so reserve enough space
   // for any 32-bit value; we'll patch it later.
-  encodeULEB128(UINT32_MAX, W.OS);
+  encodeULEB128(0, W.OS, 5);
 
   // The position where the section starts, for measuring its size.
   Section.ContentsOffset = W.OS.tell();
@@ -368,7 +392,13 @@ void WasmObjectWriter::startCustomSection(SectionBookkeeping &Section,
 // Now that the section is complete and we know how big it is, patch up the
 // section size field at the start of the section.
 void WasmObjectWriter::endSection(SectionBookkeeping &Section) {
-  uint64_t Size = W.OS.tell() - Section.PayloadOffset;
+  uint64_t Size = W.OS.tell();
+  // /dev/null doesn't support seek/tell and can report offset of 0.
+  // Simply skip this patching in that case.
+  if (!Size)
+    return;
+
+  Size -= Section.PayloadOffset;
   if (uint32_t(Size) != Size)
     report_fatal_error("section size does not fit in a uint32_t");
 
@@ -376,11 +406,8 @@ void WasmObjectWriter::endSection(SectionBookkeeping &Section) {
 
   // Write the final section size to the payload_len field, which follows
   // the section id byte.
-  uint8_t Buffer[16];
-  unsigned SizeLen = encodeULEB128(Size, Buffer, 5);
-  assert(SizeLen == 5);
-  static_cast<raw_pwrite_stream &>(W.OS).pwrite((char *)Buffer, SizeLen,
-                                                Section.SizeOffset);
+  writePatchableLEB(static_cast<raw_pwrite_stream &>(W.OS), Size,
+                    Section.SizeOffset);
 }
 
 // Emit the Wasm header.
@@ -486,8 +513,8 @@ void WasmObjectWriter::recordRelocation(MCAssembler &Asm,
   // Absolute offset within a section or a function.
   // Currently only supported for for metadata sections.
   // See: test/MC/WebAssembly/blockaddress.ll
-  if (Type == wasm::R_WEBASSEMBLY_FUNCTION_OFFSET_I32 ||
-      Type == wasm::R_WEBASSEMBLY_SECTION_OFFSET_I32) {
+  if (Type == wasm::R_WASM_FUNCTION_OFFSET_I32 ||
+      Type == wasm::R_WASM_SECTION_OFFSET_I32) {
     if (!FixupSection.getKind().isMetadata())
       report_fatal_error("relocations for function or section offsets are "
                          "only supported in metadata sections");
@@ -505,9 +532,9 @@ void WasmObjectWriter::recordRelocation(MCAssembler &Asm,
     SymA = cast<MCSymbolWasm>(SectionSymbol);
   }
 
-  // Relocation other than R_WEBASSEMBLY_TYPE_INDEX_LEB are required to be
+  // Relocation other than R_WASM_TYPE_INDEX_LEB are required to be
   // against a named symbol.
-  if (Type != wasm::R_WEBASSEMBLY_TYPE_INDEX_LEB) {
+  if (Type != wasm::R_WASM_TYPE_INDEX_LEB) {
     if (SymA->getName().empty())
       report_fatal_error("relocations against un-named temporaries are not yet "
                          "supported by wasm");
@@ -529,34 +556,7 @@ void WasmObjectWriter::recordRelocation(MCAssembler &Asm,
   }
 }
 
-// Write X as an (unsigned) LEB value at offset Offset in Stream, padded
-// to allow patching.
-static void WritePatchableLEB(raw_pwrite_stream &Stream, uint32_t X,
-                              uint64_t Offset) {
-  uint8_t Buffer[5];
-  unsigned SizeLen = encodeULEB128(X, Buffer, 5);
-  assert(SizeLen == 5);
-  Stream.pwrite((char *)Buffer, SizeLen, Offset);
-}
-
-// Write X as an signed LEB value at offset Offset in Stream, padded
-// to allow patching.
-static void WritePatchableSLEB(raw_pwrite_stream &Stream, int32_t X,
-                               uint64_t Offset) {
-  uint8_t Buffer[5];
-  unsigned SizeLen = encodeSLEB128(X, Buffer, 5);
-  assert(SizeLen == 5);
-  Stream.pwrite((char *)Buffer, SizeLen, Offset);
-}
-
-// Write X as a plain integer value at offset Offset in Stream.
-static void WriteI32(raw_pwrite_stream &Stream, uint32_t X, uint64_t Offset) {
-  uint8_t Buffer[4];
-  support::endian::write32le(Buffer, X);
-  Stream.pwrite((char *)Buffer, sizeof(Buffer), Offset);
-}
-
-static const MCSymbolWasm *ResolveSymbol(const MCSymbolWasm &Symbol) {
+static const MCSymbolWasm *resolveSymbol(const MCSymbolWasm &Symbol) {
   if (Symbol.isVariable()) {
     const MCExpr *Expr = Symbol.getVariableValue();
     auto *Inner = cast<MCSymbolRefExpr>(Expr);
@@ -572,35 +572,35 @@ static const MCSymbolWasm *ResolveSymbol(const MCSymbolWasm &Symbol) {
 uint32_t
 WasmObjectWriter::getProvisionalValue(const WasmRelocationEntry &RelEntry) {
   switch (RelEntry.Type) {
-  case wasm::R_WEBASSEMBLY_TABLE_INDEX_SLEB:
-  case wasm::R_WEBASSEMBLY_TABLE_INDEX_I32: {
+  case wasm::R_WASM_TABLE_INDEX_SLEB:
+  case wasm::R_WASM_TABLE_INDEX_I32: {
     // Provisional value is table address of the resolved symbol itself
-    const MCSymbolWasm *Sym = ResolveSymbol(*RelEntry.Symbol);
+    const MCSymbolWasm *Sym = resolveSymbol(*RelEntry.Symbol);
     assert(Sym->isFunction());
     return TableIndices[Sym];
   }
-  case wasm::R_WEBASSEMBLY_TYPE_INDEX_LEB:
+  case wasm::R_WASM_TYPE_INDEX_LEB:
     // Provisional value is same as the index
     return getRelocationIndexValue(RelEntry);
-  case wasm::R_WEBASSEMBLY_FUNCTION_INDEX_LEB:
-  case wasm::R_WEBASSEMBLY_GLOBAL_INDEX_LEB:
-  case wasm::R_WEBASSEMBLY_EVENT_INDEX_LEB:
+  case wasm::R_WASM_FUNCTION_INDEX_LEB:
+  case wasm::R_WASM_GLOBAL_INDEX_LEB:
+  case wasm::R_WASM_EVENT_INDEX_LEB:
     // Provisional value is function/global/event Wasm index
     if (!WasmIndices.count(RelEntry.Symbol))
       report_fatal_error("symbol not found in wasm index space: " +
                          RelEntry.Symbol->getName());
     return WasmIndices[RelEntry.Symbol];
-  case wasm::R_WEBASSEMBLY_FUNCTION_OFFSET_I32:
-  case wasm::R_WEBASSEMBLY_SECTION_OFFSET_I32: {
+  case wasm::R_WASM_FUNCTION_OFFSET_I32:
+  case wasm::R_WASM_SECTION_OFFSET_I32: {
     const auto &Section =
         static_cast<const MCSectionWasm &>(RelEntry.Symbol->getSection());
     return Section.getSectionOffset() + RelEntry.Addend;
   }
-  case wasm::R_WEBASSEMBLY_MEMORY_ADDR_LEB:
-  case wasm::R_WEBASSEMBLY_MEMORY_ADDR_I32:
-  case wasm::R_WEBASSEMBLY_MEMORY_ADDR_SLEB: {
+  case wasm::R_WASM_MEMORY_ADDR_LEB:
+  case wasm::R_WASM_MEMORY_ADDR_I32:
+  case wasm::R_WASM_MEMORY_ADDR_SLEB: {
     // Provisional value is address of the global
-    const MCSymbolWasm *Sym = ResolveSymbol(*RelEntry.Symbol);
+    const MCSymbolWasm *Sym = resolveSymbol(*RelEntry.Symbol);
     // For undefined symbols, use zero
     if (!Sym->isDefined())
       return 0;
@@ -654,7 +654,7 @@ static void addData(SmallVectorImpl<char> &DataBytes,
 
 uint32_t
 WasmObjectWriter::getRelocationIndexValue(const WasmRelocationEntry &RelEntry) {
-  if (RelEntry.Type == wasm::R_WEBASSEMBLY_TYPE_INDEX_LEB) {
+  if (RelEntry.Type == wasm::R_WASM_TYPE_INDEX_LEB) {
     if (!TypeIndices.count(RelEntry.Symbol))
       report_fatal_error("symbol not found in type index space: " +
                          RelEntry.Symbol->getName());
@@ -678,22 +678,22 @@ void WasmObjectWriter::applyRelocations(
     uint32_t Value = getProvisionalValue(RelEntry);
 
     switch (RelEntry.Type) {
-    case wasm::R_WEBASSEMBLY_FUNCTION_INDEX_LEB:
-    case wasm::R_WEBASSEMBLY_TYPE_INDEX_LEB:
-    case wasm::R_WEBASSEMBLY_GLOBAL_INDEX_LEB:
-    case wasm::R_WEBASSEMBLY_MEMORY_ADDR_LEB:
-    case wasm::R_WEBASSEMBLY_EVENT_INDEX_LEB:
-      WritePatchableLEB(Stream, Value, Offset);
+    case wasm::R_WASM_FUNCTION_INDEX_LEB:
+    case wasm::R_WASM_TYPE_INDEX_LEB:
+    case wasm::R_WASM_GLOBAL_INDEX_LEB:
+    case wasm::R_WASM_MEMORY_ADDR_LEB:
+    case wasm::R_WASM_EVENT_INDEX_LEB:
+      writePatchableLEB(Stream, Value, Offset);
       break;
-    case wasm::R_WEBASSEMBLY_TABLE_INDEX_I32:
-    case wasm::R_WEBASSEMBLY_MEMORY_ADDR_I32:
-    case wasm::R_WEBASSEMBLY_FUNCTION_OFFSET_I32:
-    case wasm::R_WEBASSEMBLY_SECTION_OFFSET_I32:
-      WriteI32(Stream, Value, Offset);
+    case wasm::R_WASM_TABLE_INDEX_I32:
+    case wasm::R_WASM_MEMORY_ADDR_I32:
+    case wasm::R_WASM_FUNCTION_OFFSET_I32:
+    case wasm::R_WASM_SECTION_OFFSET_I32:
+      writeI32(Stream, Value, Offset);
       break;
-    case wasm::R_WEBASSEMBLY_TABLE_INDEX_SLEB:
-    case wasm::R_WEBASSEMBLY_MEMORY_ADDR_SLEB:
-      WritePatchableSLEB(Stream, Value, Offset);
+    case wasm::R_WASM_TABLE_INDEX_SLEB:
+    case wasm::R_WASM_MEMORY_ADDR_SLEB:
+      writePatchableSLEB(Stream, Value, Offset);
       break;
     default:
       llvm_unreachable("invalid relocation type");
@@ -848,7 +848,7 @@ void WasmObjectWriter::writeElemSection(ArrayRef<uint32_t> TableElems) {
 
   // init expr for starting offset
   W.OS << char(wasm::WASM_OPCODE_I32_CONST);
-  encodeSLEB128(kInitialTableOffset, W.OS);
+  encodeSLEB128(InitialTableOffset, W.OS);
   W.OS << char(wasm::WASM_OPCODE_END);
 
   encodeULEB128(TableElems.size(), W.OS);
@@ -982,7 +982,8 @@ void WasmObjectWriter::writeLinkingMetaDataSection(
       case wasm::WASM_SYMBOL_TYPE_GLOBAL:
       case wasm::WASM_SYMBOL_TYPE_EVENT:
         encodeULEB128(Sym.ElementIndex, W.OS);
-        if ((Sym.Flags & wasm::WASM_SYMBOL_UNDEFINED) == 0)
+        if ((Sym.Flags & wasm::WASM_SYMBOL_UNDEFINED) == 0 ||
+            (Sym.Flags & wasm::WASM_SYMBOL_EXPLICIT_NAME) != 0)
           writeString(Sym.Name);
         break;
       case wasm::WASM_SYMBOL_TYPE_DATA:
@@ -1045,25 +1046,24 @@ void WasmObjectWriter::writeLinkingMetaDataSection(
   endSection(Section);
 }
 
-void WasmObjectWriter::writeCustomSections(const MCAssembler &Asm,
-                                           const MCAsmLayout &Layout) {
-  for (auto &CustomSection : CustomSections) {
-    SectionBookkeeping Section;
-    auto *Sec = CustomSection.Section;
-    startCustomSection(Section, CustomSection.Name);
+void WasmObjectWriter::writeCustomSection(WasmCustomSection &CustomSection,
+                                          const MCAssembler &Asm,
+                                          const MCAsmLayout &Layout) {
+  SectionBookkeeping Section;
+  auto *Sec = CustomSection.Section;
+  startCustomSection(Section, CustomSection.Name);
 
-    Sec->setSectionOffset(W.OS.tell() - Section.ContentsOffset);
-    Asm.writeSectionData(W.OS, Sec, Layout);
+  Sec->setSectionOffset(W.OS.tell() - Section.ContentsOffset);
+  Asm.writeSectionData(W.OS, Sec, Layout);
 
-    CustomSection.OutputContentsOffset = Section.ContentsOffset;
-    CustomSection.OutputIndex = Section.Index;
+  CustomSection.OutputContentsOffset = Section.ContentsOffset;
+  CustomSection.OutputIndex = Section.Index;
 
-    endSection(Section);
+  endSection(Section);
 
-    // Apply fixups.
-    auto &Relocations = CustomSectionsRelocations[CustomSection.Section];
-    applyRelocations(Relocations, CustomSection.OutputContentsOffset);
-  }
+  // Apply fixups.
+  auto &Relocations = CustomSectionsRelocations[CustomSection.Section];
+  applyRelocations(Relocations, CustomSection.OutputContentsOffset);
 }
 
 uint32_t WasmObjectWriter::getFunctionType(const MCSymbolWasm &Symbol) {
@@ -1082,7 +1082,7 @@ void WasmObjectWriter::registerFunctionType(const MCSymbolWasm &Symbol) {
   assert(Symbol.isFunction());
 
   WasmSignature S;
-  const MCSymbolWasm *ResolvedSym = ResolveSymbol(Symbol);
+  const MCSymbolWasm *ResolvedSym = resolveSymbol(Symbol);
   if (auto *Sig = ResolvedSym->getSignature()) {
     S.Returns = Sig->Returns;
     S.Params = Sig->Params;
@@ -1159,24 +1159,24 @@ uint64_t WasmObjectWriter::writeObject(MCAssembler &Asm,
   // For now, always emit the memory import, since loads and stores are not
   // valid without it. In the future, we could perhaps be more clever and omit
   // it if there are no loads or stores.
-  MCSymbolWasm *MemorySym =
+  auto *MemorySym =
       cast<MCSymbolWasm>(Ctx.getOrCreateSymbol("__linear_memory"));
   wasm::WasmImport MemImport;
-  MemImport.Module = MemorySym->getModuleName();
-  MemImport.Field = MemorySym->getName();
+  MemImport.Module = MemorySym->getImportModule();
+  MemImport.Field = MemorySym->getImportName();
   MemImport.Kind = wasm::WASM_EXTERNAL_MEMORY;
   Imports.push_back(MemImport);
 
   // For now, always emit the table section, since indirect calls are not
   // valid without it. In the future, we could perhaps be more clever and omit
   // it if there are no indirect calls.
-  MCSymbolWasm *TableSym =
+  auto *TableSym =
       cast<MCSymbolWasm>(Ctx.getOrCreateSymbol("__indirect_function_table"));
   wasm::WasmImport TableImport;
-  TableImport.Module = TableSym->getModuleName();
-  TableImport.Field = TableSym->getName();
+  TableImport.Module = TableSym->getImportModule();
+  TableImport.Field = TableSym->getImportName();
   TableImport.Kind = wasm::WASM_EXTERNAL_TABLE;
-  TableImport.Table.ElemType = wasm::WASM_TYPE_ANYFUNC;
+  TableImport.Table.ElemType = wasm::WASM_TYPE_FUNCREF;
   Imports.push_back(TableImport);
 
   // Populate SignatureIndices, and Imports and WasmIndices for undefined
@@ -1200,8 +1200,8 @@ uint64_t WasmObjectWriter::writeObject(MCAssembler &Asm,
     if (!WS.isDefined() && !WS.isComdat()) {
       if (WS.isFunction()) {
         wasm::WasmImport Import;
-        Import.Module = WS.getModuleName();
-        Import.Field = WS.getName();
+        Import.Module = WS.getImportModule();
+        Import.Field = WS.getImportName();
         Import.Kind = wasm::WASM_EXTERNAL_FUNCTION;
         Import.SigIndex = getFunctionType(WS);
         Imports.push_back(Import);
@@ -1211,8 +1211,8 @@ uint64_t WasmObjectWriter::writeObject(MCAssembler &Asm,
           report_fatal_error("undefined global symbol cannot be weak");
 
         wasm::WasmImport Import;
-        Import.Module = WS.getModuleName();
-        Import.Field = WS.getName();
+        Import.Module = WS.getImportModule();
+        Import.Field = WS.getImportName();
         Import.Kind = wasm::WASM_EXTERNAL_GLOBAL;
         Import.Global = WS.getGlobalType();
         Imports.push_back(Import);
@@ -1222,8 +1222,8 @@ uint64_t WasmObjectWriter::writeObject(MCAssembler &Asm,
           report_fatal_error("undefined event symbol cannot be weak");
 
         wasm::WasmImport Import;
-        Import.Module = WS.getModuleName();
-        Import.Field = WS.getName();
+        Import.Module = WS.getImportModule();
+        Import.Field = WS.getImportName();
         Import.Kind = wasm::WASM_EXTERNAL_EVENT;
         Import.Event.Attribute = wasm::WASM_EVENT_ATTRIBUTE_EXCEPTION;
         Import.Event.SigIndex = getEventType(WS);
@@ -1256,7 +1256,7 @@ uint64_t WasmObjectWriter::writeObject(MCAssembler &Asm,
       Segment.Offset = DataSize;
       Segment.Section = &Section;
       addData(Segment.Data, Section);
-      Segment.Alignment = Section.getAlignment();
+      Segment.Alignment = Log2_32(Section.getAlignment());
       Segment.Flags = 0;
       DataSize += Segment.Data.size();
       Section.setSegmentIndex(SegmentIndex);
@@ -1282,6 +1282,13 @@ uint64_t WasmObjectWriter::writeObject(MCAssembler &Asm,
           report_fatal_error("section name and begin symbol should match: " +
                              Twine(SectionName));
       }
+
+      // Separate out the producers section
+      if (Name == "producers") {
+        ProducersSection = llvm::make_unique<WasmCustomSection>(Name, &Section);
+        continue;
+      }
+
       CustomSections.emplace_back(Name, &Section);
     }
   }
@@ -1313,7 +1320,7 @@ uint64_t WasmObjectWriter::writeObject(MCAssembler &Asm,
           report_fatal_error(
               "function sections must contain one function each");
 
-        if (WS.getSize() == 0)
+        if (WS.getSize() == nullptr)
           report_fatal_error(
               "function symbols must have a size set with .size");
 
@@ -1410,7 +1417,7 @@ uint64_t WasmObjectWriter::writeObject(MCAssembler &Asm,
 
     // Find the target symbol of this weak alias and export that index
     const auto &WS = static_cast<const MCSymbolWasm &>(S);
-    const MCSymbolWasm *ResolvedSym = ResolveSymbol(WS);
+    const MCSymbolWasm *ResolvedSym = resolveSymbol(WS);
     LLVM_DEBUG(dbgs() << WS.getName() << ": weak alias of '" << *ResolvedSym
                       << "'\n");
 
@@ -1434,7 +1441,7 @@ uint64_t WasmObjectWriter::writeObject(MCAssembler &Asm,
   for (const MCSymbol &S : Asm.symbols()) {
     const auto &WS = static_cast<const MCSymbolWasm &>(S);
     if (!isInSymtab(WS)) {
-      WS.setIndex(INVALID_INDEX);
+      WS.setIndex(InvalidIndex);
       continue;
     }
     LLVM_DEBUG(dbgs() << "adding to symtab: " << WS << "\n");
@@ -1448,6 +1455,10 @@ uint64_t WasmObjectWriter::writeObject(MCAssembler &Asm,
       Flags |= wasm::WASM_SYMBOL_BINDING_LOCAL;
     if (WS.isUndefined())
       Flags |= wasm::WASM_SYMBOL_UNDEFINED;
+    if (WS.isExported())
+      Flags |= wasm::WASM_SYMBOL_EXPORTED;
+    if (WS.getName() != WS.getImportName())
+      Flags |= wasm::WASM_SYMBOL_EXPLICIT_NAME;
 
     wasm::WasmSymbolInfo Info;
     Info.Name = WS.getName();
@@ -1469,13 +1480,13 @@ uint64_t WasmObjectWriter::writeObject(MCAssembler &Asm,
       // Functions referenced by a relocation need to put in the table.  This is
       // purely to make the object file's provisional values readable, and is
       // ignored by the linker, which re-calculates the relocations itself.
-      if (Rel.Type != wasm::R_WEBASSEMBLY_TABLE_INDEX_I32 &&
-          Rel.Type != wasm::R_WEBASSEMBLY_TABLE_INDEX_SLEB)
+      if (Rel.Type != wasm::R_WASM_TABLE_INDEX_I32 &&
+          Rel.Type != wasm::R_WASM_TABLE_INDEX_SLEB)
         return;
       assert(Rel.Symbol->isFunction());
-      const MCSymbolWasm &WS = *ResolveSymbol(*Rel.Symbol);
+      const MCSymbolWasm &WS = *resolveSymbol(*Rel.Symbol);
       uint32_t FunctionIndex = WasmIndices.find(&WS)->second;
-      uint32_t TableIndex = TableElems.size() + kInitialTableOffset;
+      uint32_t TableIndex = TableElems.size() + InitialTableOffset;
       if (TableIndices.try_emplace(&WS, TableIndex).second) {
         LLVM_DEBUG(dbgs() << "  -> adding " << WS.getName()
                           << " to table: " << TableIndex << "\n");
@@ -1534,10 +1545,10 @@ uint64_t WasmObjectWriter::writeObject(MCAssembler &Asm,
     const auto &DataFrag = cast<MCDataFragment>(Frag);
     const SmallVectorImpl<char> &Contents = DataFrag.getContents();
     for (const uint8_t *
-             p = (const uint8_t *)Contents.data(),
-            *end = (const uint8_t *)Contents.data() + Contents.size();
-         p != end; ++p) {
-      if (*p != 0)
+             P = (const uint8_t *)Contents.data(),
+            *End = (const uint8_t *)Contents.data() + Contents.size();
+         P != End; ++P) {
+      if (*P != 0)
         report_fatal_error("non-symbolic data in .init_array section");
     }
     for (const MCFixup &Fixup : DataFrag.getFixups()) {
@@ -1549,7 +1560,7 @@ uint64_t WasmObjectWriter::writeObject(MCAssembler &Asm,
         report_fatal_error("fixups in .init_array should be symbol references");
       if (Sym->getKind() != MCSymbolRefExpr::VK_WebAssembly_FUNCTION)
         report_fatal_error("symbols in .init_array should be for functions");
-      if (Sym->getSymbol().getIndex() == INVALID_INDEX)
+      if (Sym->getSymbol().getIndex() == InvalidIndex)
         report_fatal_error("symbols in .init_array should exist in symbtab");
       InitFuncs.push_back(
           std::make_pair(Priority, Sym->getSymbol().getIndex()));
@@ -1570,11 +1581,14 @@ uint64_t WasmObjectWriter::writeObject(MCAssembler &Asm,
   writeElemSection(TableElems);
   writeCodeSection(Asm, Layout, Functions);
   writeDataSection();
-  writeCustomSections(Asm, Layout);
+  for (auto &CustomSection : CustomSections)
+    writeCustomSection(CustomSection, Asm, Layout);
   writeLinkingMetaDataSection(SymbolInfos, InitFuncs, Comdats);
   writeRelocSection(CodeSectionIndex, "CODE", CodeRelocations);
   writeRelocSection(DataSectionIndex, "DATA", DataRelocations);
   writeCustomRelocSections();
+  if (ProducersSection)
+    writeCustomSection(*ProducersSection, Asm, Layout);
 
   // TODO: Translate the .comment section to the output.
   return W.OS.tell() - StartOffset;
diff --git a/lib/MC/WinCOFFObjectWriter.cpp b/lib/MC/WinCOFFObjectWriter.cpp
index b774852eabe6b8d895ba2c2044d0ea6707a823c3..1dfb1cf440f34a073626bb08bb0c06fe85330cc8 100644
--- a/lib/MC/WinCOFFObjectWriter.cpp
+++ b/lib/MC/WinCOFFObjectWriter.cpp
@@ -1,9 +1,8 @@
 //===- llvm/MC/WinCOFFObjectWriter.cpp ------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/MCA/Context.cpp b/lib/MCA/Context.cpp
index c1b197dfe2e6744fc53e2c9b05283e7da9406083..18489ccdf1fb71c5f20c78f3220fddad8e50cd47 100644
--- a/lib/MCA/Context.cpp
+++ b/lib/MCA/Context.cpp
@@ -1,9 +1,8 @@
 //===---------------------------- Context.cpp -------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/lib/MCA/HWEventListener.cpp b/lib/MCA/HWEventListener.cpp
index 4a0e5b1754dd593761557726cf1fcfaab97af9f6..58b2e03292227117583da68e16415a62602ccf3e 100644
--- a/lib/MCA/HWEventListener.cpp
+++ b/lib/MCA/HWEventListener.cpp
@@ -1,9 +1,8 @@
 //===----------------------- HWEventListener.cpp ----------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/lib/MCA/HardwareUnits/HardwareUnit.cpp b/lib/MCA/HardwareUnits/HardwareUnit.cpp
index edd32b9c0c1ab9bbbb3d9e49e18145ee7058274b..69f793796ec7f183353cee99510e75152e220ad9 100644
--- a/lib/MCA/HardwareUnits/HardwareUnit.cpp
+++ b/lib/MCA/HardwareUnits/HardwareUnit.cpp
@@ -1,9 +1,8 @@
 //===------------------------- HardwareUnit.cpp -----------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/lib/MCA/HardwareUnits/LSUnit.cpp b/lib/MCA/HardwareUnits/LSUnit.cpp
index 8895eb392b60ae92dd17aa377f4b9f03efb077f7..2adc20f8255a99c36411c773f19b06ff728ee419 100644
--- a/lib/MCA/HardwareUnits/LSUnit.cpp
+++ b/lib/MCA/HardwareUnits/LSUnit.cpp
@@ -1,9 +1,8 @@
 //===----------------------- LSUnit.cpp --------------------------*- C++-*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/lib/MCA/HardwareUnits/RegisterFile.cpp b/lib/MCA/HardwareUnits/RegisterFile.cpp
index 22977e5ded65b054ce6be3c4207ac05d1e98a6b2..3621d182b3e63d764bc433fab3ac35f6f1f6a15d 100644
--- a/lib/MCA/HardwareUnits/RegisterFile.cpp
+++ b/lib/MCA/HardwareUnits/RegisterFile.cpp
@@ -1,9 +1,8 @@
 //===--------------------- RegisterFile.cpp ---------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
@@ -402,7 +401,7 @@ void RegisterFile::collectWrites(const ReadState &RS,
 }
 
 void RegisterFile::addRegisterRead(ReadState &RS,
-                                   SmallVectorImpl<WriteRef> &Defs) const {
+                                   const MCSubtargetInfo &STI) const {
   unsigned RegID = RS.getRegisterID();
   const RegisterRenamingInfo &RRI = RegisterMappings[RegID].second;
   RS.setPRF(RRI.IndexPlusCost.first);
@@ -411,8 +410,23 @@ void RegisterFile::addRegisterRead(ReadState &RS,
 
   if (ZeroRegisters[RS.getRegisterID()])
     RS.setReadZero();
-  collectWrites(RS, Defs);
-  RS.setDependentWrites(Defs.size());
+
+  SmallVector<WriteRef, 4> DependentWrites;
+  collectWrites(RS, DependentWrites);
+  RS.setDependentWrites(DependentWrites.size());
+
+  // We know that this read depends on all the writes in DependentWrites.
+  // For each write, check if we have ReadAdvance information, and use it
+  // to figure out in how many cycles this read becomes available.
+  const ReadDescriptor &RD = RS.getDescriptor();
+  const MCSchedModel &SM = STI.getSchedModel();
+  const MCSchedClassDesc *SC = SM.getSchedClassDesc(RD.SchedClassID);
+  for (WriteRef &WR : DependentWrites) {
+    WriteState &WS = *WR.getWriteState();
+    unsigned WriteResID = WS.getWriteResourceID();
+    int ReadAdvance = STI.getReadAdvanceCycles(SC, RD.UseIndex, WriteResID);
+    WS.addUser(&RS, ReadAdvance);
+  }
 }
 
 unsigned RegisterFile::isAvailable(ArrayRef<unsigned> Regs) const {
diff --git a/lib/MCA/HardwareUnits/ResourceManager.cpp b/lib/MCA/HardwareUnits/ResourceManager.cpp
index e76543c53e14560dc71a9f6ce8cb5db594d7bdf0..21e0271c852503825ff198968733b42a07e07e0e 100644
--- a/lib/MCA/HardwareUnits/ResourceManager.cpp
+++ b/lib/MCA/HardwareUnits/ResourceManager.cpp
@@ -1,9 +1,8 @@
 //===--------------------- ResourceManager.cpp ------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
@@ -24,29 +23,35 @@ namespace mca {
 #define DEBUG_TYPE "llvm-mca"
 ResourceStrategy::~ResourceStrategy() = default;
 
+// Returns the index of the highest bit set. For resource masks, the position of
+// the highest bit set can be used to construct a resource mask identifier.
+static unsigned getResourceStateIndex(uint64_t Mask) {
+  return std::numeric_limits<uint64_t>::digits - countLeadingZeros(Mask);
+}
+
+static uint64_t selectImpl(uint64_t CandidateMask,
+                           uint64_t &NextInSequenceMask) {
+  // The upper bit set in CandidateMask identifies our next candidate resource.
+  CandidateMask = 1ULL << (getResourceStateIndex(CandidateMask) - 1);
+  NextInSequenceMask &= (CandidateMask | (CandidateMask - 1));
+  return CandidateMask;
+}
+
 uint64_t DefaultResourceStrategy::select(uint64_t ReadyMask) {
   // This method assumes that ReadyMask cannot be zero.
   uint64_t CandidateMask = ReadyMask & NextInSequenceMask;
-  if (CandidateMask) {
-    CandidateMask = PowerOf2Floor(CandidateMask);
-    NextInSequenceMask &= (CandidateMask | (CandidateMask - 1));
-    return CandidateMask;
-  }
+  if (CandidateMask)
+    return selectImpl(CandidateMask, NextInSequenceMask);
 
   NextInSequenceMask = ResourceUnitMask ^ RemovedFromNextInSequence;
   RemovedFromNextInSequence = 0;
   CandidateMask = ReadyMask & NextInSequenceMask;
-
-  if (CandidateMask) {
-    CandidateMask = PowerOf2Floor(CandidateMask);
-    NextInSequenceMask &= (CandidateMask | (CandidateMask - 1));
-    return CandidateMask;
-  }
+  if (CandidateMask)
+    return selectImpl(CandidateMask, NextInSequenceMask);
 
   NextInSequenceMask = ResourceUnitMask;
-  CandidateMask = PowerOf2Floor(ReadyMask & NextInSequenceMask);
-  NextInSequenceMask &= (CandidateMask | (CandidateMask - 1));
-  return CandidateMask;
+  CandidateMask = ReadyMask & NextInSequenceMask;
+  return selectImpl(CandidateMask, NextInSequenceMask);
 }
 
 void DefaultResourceStrategy::used(uint64_t Mask) {
@@ -66,11 +71,13 @@ void DefaultResourceStrategy::used(uint64_t Mask) {
 ResourceState::ResourceState(const MCProcResourceDesc &Desc, unsigned Index,
                              uint64_t Mask)
     : ProcResourceDescIndex(Index), ResourceMask(Mask),
-      BufferSize(Desc.BufferSize), IsAGroup(countPopulation(ResourceMask)>1) {
-  if (IsAGroup)
-    ResourceSizeMask = ResourceMask ^ PowerOf2Floor(ResourceMask);
-  else
+      BufferSize(Desc.BufferSize), IsAGroup(countPopulation(ResourceMask) > 1) {
+  if (IsAGroup) {
+    ResourceSizeMask =
+        ResourceMask ^ 1ULL << (getResourceStateIndex(ResourceMask) - 1);
+  } else {
     ResourceSizeMask = (1ULL << Desc.NumUnits) - 1;
+  }
   ReadyMask = ResourceSizeMask;
   AvailableSlots = BufferSize == -1 ? 0U : static_cast<unsigned>(BufferSize);
   Unavailable = false;
@@ -91,19 +98,15 @@ ResourceStateEvent ResourceState::isBufferAvailable() const {
 
 #ifndef NDEBUG
 void ResourceState::dump() const {
-  dbgs() << "MASK=" << format_hex(ResourceMask, 8)
-         << ", SZMASK=" << format_hex(ResourceSizeMask, 8)
-         << ", RDYMASK=" << format_hex(ReadyMask, 8)
+  dbgs() << "MASK=" << format_hex(ResourceMask, 16)
+         << ", SZMASK=" << format_hex(ResourceSizeMask, 16)
+         << ", RDYMASK=" << format_hex(ReadyMask, 16)
          << ", BufferSize=" << BufferSize
          << ", AvailableSlots=" << AvailableSlots
          << ", Reserved=" << Unavailable << '\n';
 }
 #endif
 
-static unsigned getResourceStateIndex(uint64_t Mask) {
-  return std::numeric_limits<uint64_t>::digits - countLeadingZeros(Mask);
-}
-
 static std::unique_ptr<ResourceStrategy>
 getStrategyFor(const ResourceState &RS) {
   if (RS.isAResourceGroup() || RS.getNumUnits() > 1)
@@ -111,10 +114,13 @@ getStrategyFor(const ResourceState &RS) {
   return std::unique_ptr<ResourceStrategy>(nullptr);
 }
 
-ResourceManager::ResourceManager(const MCSchedModel &SM) {
+ResourceManager::ResourceManager(const MCSchedModel &SM)
+    : Resources(SM.getNumProcResourceKinds()),
+      Strategies(SM.getNumProcResourceKinds()),
+      Resource2Groups(SM.getNumProcResourceKinds(), 0),
+      ProcResID2Mask(SM.getNumProcResourceKinds()), ProcResUnitMask(0),
+      ReservedResourceGroups(0) {
   computeProcResourceMasks(SM, ProcResID2Mask);
-  Resources.resize(SM.getNumProcResourceKinds());
-  Strategies.resize(SM.getNumProcResourceKinds());
 
   for (unsigned I = 0, E = SM.getNumProcResourceKinds(); I < E; ++I) {
     uint64_t Mask = ProcResID2Mask[I];
@@ -123,6 +129,28 @@ ResourceManager::ResourceManager(const MCSchedModel &SM) {
         llvm::make_unique<ResourceState>(*SM.getProcResource(I), I, Mask);
     Strategies[Index] = getStrategyFor(*Resources[Index]);
   }
+
+  for (unsigned I = 0, E = SM.getNumProcResourceKinds(); I < E; ++I) {
+    uint64_t Mask = ProcResID2Mask[I];
+    unsigned Index = getResourceStateIndex(Mask);
+    const ResourceState &RS = *Resources[Index];
+    if (!RS.isAResourceGroup()) {
+      ProcResUnitMask |= Mask;
+      continue;
+    }
+
+    uint64_t GroupMaskIdx = 1ULL << (Index - 1);
+    Mask -= GroupMaskIdx;
+    while (Mask) {
+      // Extract lowest set isolated bit.
+      uint64_t Unit = Mask & (-Mask);
+      unsigned IndexUnit = getResourceStateIndex(Unit);
+      Resource2Groups[IndexUnit] |= GroupMaskIdx;
+      Mask ^= Unit;
+    }
+  }
+
+  AvailableProcResUnits = ProcResUnitMask;
 }
 
 void ResourceManager::setCustomStrategyImpl(std::unique_ptr<ResourceStrategy> S,
@@ -146,6 +174,7 @@ unsigned ResourceManager::getNumUnits(uint64_t ResourceID) const {
 // Second, is the specific sub-resource ID.
 ResourceRef ResourceManager::selectPipe(uint64_t ResourceID) {
   unsigned Index = getResourceStateIndex(ResourceID);
+  assert(Index < Resources.size() && "Invalid resource use!");
   ResourceState &RS = *Resources[Index];
   assert(RS.isReady() && "No available units to select!");
 
@@ -175,34 +204,38 @@ void ResourceManager::use(const ResourceRef &RR) {
   if (RS.isReady())
     return;
 
-  // Notify to other resources that RR.first is no longer available.
-  for (std::unique_ptr<ResourceState> &Res : Resources) {
-    ResourceState &Current = *Res;
-    if (!Current.isAResourceGroup() || Current.getResourceMask() == RR.first)
-      continue;
-
-    if (Current.containsResource(RR.first)) {
-      unsigned Index = getResourceStateIndex(Current.getResourceMask());
-      Current.markSubResourceAsUsed(RR.first);
-      Strategies[Index]->used(RR.first);
-    }
+  AvailableProcResUnits ^= RR.first;
+
+  // Notify groups that RR.first is no longer available.
+  uint64_t Users = Resource2Groups[RSID];
+  while (Users) {
+    // Extract lowest set isolated bit.
+    unsigned GroupIndex = getResourceStateIndex(Users & (-Users));
+    ResourceState &CurrentUser = *Resources[GroupIndex];
+    CurrentUser.markSubResourceAsUsed(RR.first);
+    Strategies[GroupIndex]->used(RR.first);
+    // Reset lowest set bit.
+    Users &= Users - 1;
   }
 }
 
 void ResourceManager::release(const ResourceRef &RR) {
-  ResourceState &RS = *Resources[getResourceStateIndex(RR.first)];
+  unsigned RSID = getResourceStateIndex(RR.first);
+  ResourceState &RS = *Resources[RSID];
   bool WasFullyUsed = !RS.isReady();
   RS.releaseSubResource(RR.second);
   if (!WasFullyUsed)
     return;
 
-  for (std::unique_ptr<ResourceState> &Res : Resources) {
-    ResourceState &Current = *Res;
-    if (!Current.isAResourceGroup() || Current.getResourceMask() == RR.first)
-      continue;
+  AvailableProcResUnits ^= RR.first;
 
-    if (Current.containsResource(RR.first))
-      Current.releaseSubResource(RR.first);
+  // Notify groups that RR.first is now available again.
+  uint64_t Users = Resource2Groups[RSID];
+  while (Users) {
+    unsigned GroupIndex = getResourceStateIndex(Users & (-Users));
+    ResourceState &CurrentUser = *Resources[GroupIndex];
+    CurrentUser.releaseSubResource(RR.first);
+    Users &= Users - 1;
   }
 }
 
@@ -236,31 +269,19 @@ void ResourceManager::releaseBuffers(ArrayRef<uint64_t> Buffers) {
     Resources[getResourceStateIndex(R)]->releaseBuffer();
 }
 
-bool ResourceManager::canBeIssued(const InstrDesc &Desc) const {
-  return all_of(
-      Desc.Resources, [&](const std::pair<uint64_t, const ResourceUsage> &E) {
-        unsigned NumUnits = E.second.isReserved() ? 0U : E.second.NumUnits;
-        unsigned Index = getResourceStateIndex(E.first);
-        return Resources[Index]->isReady(NumUnits);
-      });
-}
+uint64_t ResourceManager::checkAvailability(const InstrDesc &Desc) const {
+  uint64_t BusyResourceMask = 0;
+  for (const std::pair<uint64_t, const ResourceUsage> &E : Desc.Resources) {
+    unsigned NumUnits = E.second.isReserved() ? 0U : E.second.NumUnits;
+    unsigned Index = getResourceStateIndex(E.first);
+    if (!Resources[Index]->isReady(NumUnits))
+      BusyResourceMask |= E.first;
+  }
 
-// Returns true if all resources are in-order, and there is at least one
-// resource which is a dispatch hazard (BufferSize = 0).
-bool ResourceManager::mustIssueImmediately(const InstrDesc &Desc) const {
-  if (!canBeIssued(Desc))
-    return false;
-  bool AllInOrderResources = all_of(Desc.Buffers, [&](uint64_t BufferMask) {
-    unsigned Index = getResourceStateIndex(BufferMask);
-    const ResourceState &Resource = *Resources[Index];
-    return Resource.isInOrder() || Resource.isADispatchHazard();
-  });
-  if (!AllInOrderResources)
-    return false;
-
-  return any_of(Desc.Buffers, [&](uint64_t BufferMask) {
-    return Resources[getResourceStateIndex(BufferMask)]->isADispatchHazard();
-  });
+  BusyResourceMask &= ProcResUnitMask;
+  if (BusyResourceMask)
+    return BusyResourceMask;
+  return Desc.UsedProcResGroups & ReservedResourceGroups;
 }
 
 void ResourceManager::issueInstruction(
@@ -278,9 +299,6 @@ void ResourceManager::issueInstruction(
       ResourceRef Pipe = selectPipe(R.first);
       use(Pipe);
       BusyResources[Pipe] += CS.size();
-      // Replace the resource mask with a valid processor resource index.
-      const ResourceState &RS = *Resources[getResourceStateIndex(Pipe.first)];
-      Pipe.first = RS.getProcResourceID();
       Pipes.emplace_back(std::pair<ResourceRef, ResourceCycles>(
           Pipe, ResourceCycles(CS.size())));
     } else {
@@ -315,13 +333,17 @@ void ResourceManager::cycleEvent(SmallVectorImpl<ResourceRef> &ResourcesFreed) {
 
 void ResourceManager::reserveResource(uint64_t ResourceID) {
   ResourceState &Resource = *Resources[getResourceStateIndex(ResourceID)];
-  assert(!Resource.isReserved());
+  assert(Resource.isAResourceGroup() && !Resource.isReserved() &&
+         "Unexpected resource found!");
   Resource.setReserved();
+  ReservedResourceGroups ^= PowerOf2Floor(ResourceID);
 }
 
 void ResourceManager::releaseResource(uint64_t ResourceID) {
   ResourceState &Resource = *Resources[getResourceStateIndex(ResourceID)];
   Resource.clearReserved();
+  if (Resource.isAResourceGroup())
+    ReservedResourceGroups ^= PowerOf2Floor(ResourceID);
 }
 
 } // namespace mca
diff --git a/lib/MCA/HardwareUnits/RetireControlUnit.cpp b/lib/MCA/HardwareUnits/RetireControlUnit.cpp
index de9f24552c38be4190bd364fc3f7b17868ae53c8..068c5062ccdf19f095603c14227ec5f970da334d 100644
--- a/lib/MCA/HardwareUnits/RetireControlUnit.cpp
+++ b/lib/MCA/HardwareUnits/RetireControlUnit.cpp
@@ -1,9 +1,8 @@
 //===---------------------- RetireControlUnit.cpp ---------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/lib/MCA/HardwareUnits/Scheduler.cpp b/lib/MCA/HardwareUnits/Scheduler.cpp
index 3924ac599105aa3e803f7361568613c9c0d3f81a..a7a6ed9570f4f93ec71b7f133ffa6ace3657a82d 100644
--- a/lib/MCA/HardwareUnits/Scheduler.cpp
+++ b/lib/MCA/HardwareUnits/Scheduler.cpp
@@ -1,9 +1,8 @@
 //===--------------------- Scheduler.cpp ------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -97,15 +96,15 @@ void Scheduler::issueInstruction(
   // other dependent instructions. Dependent instructions may be issued during
   // this same cycle if operands have ReadAdvance entries.  Promote those
   // instructions to the ReadySet and notify the caller that those are ready.
-  if (HasDependentUsers)
+  if (HasDependentUsers && promoteToPendingSet())
     promoteToReadySet(ReadyInstructions);
 }
 
-void Scheduler::promoteToReadySet(SmallVectorImpl<InstRef> &Ready) {
+bool Scheduler::promoteToReadySet(SmallVectorImpl<InstRef> &Ready) {
   // Scan the set of waiting instructions and promote them to the
-  // ready queue if operands are all ready.
-  unsigned RemovedElements = 0;
-  for (auto I = WaitSet.begin(), E = WaitSet.end(); I != E;) {
+  // ready set if operands are all ready.
+  unsigned PromotedElements = 0;
+  for (auto I = PendingSet.begin(), E = PendingSet.end(); I != E;) {
     InstRef &IR = *I;
     if (!IR)
       break;
@@ -114,7 +113,7 @@ void Scheduler::promoteToReadySet(SmallVectorImpl<InstRef> &Ready) {
     // a transition in state using method 'update()'.
     Instruction &IS = *IR.getInstruction();
     if (!IS.isReady())
-      IS.update();
+      IS.updatePending();
 
     // Check if there are still unsolved data dependencies.
     if (!isReady(IR)) {
@@ -122,15 +121,49 @@ void Scheduler::promoteToReadySet(SmallVectorImpl<InstRef> &Ready) {
       continue;
     }
 
+    LLVM_DEBUG(dbgs() << "[SCHEDULER]: Instruction #" << IR
+                      << " promoted to the READY set.\n");
+
     Ready.emplace_back(IR);
     ReadySet.emplace_back(IR);
 
+    IR.invalidate();
+    ++PromotedElements;
+    std::iter_swap(I, E - PromotedElements);
+  }
+
+  PendingSet.resize(PendingSet.size() - PromotedElements);
+  return PromotedElements;
+}
+
+bool Scheduler::promoteToPendingSet() {
+  // Scan the set of waiting instructions and promote them to the
+  // pending set if operands are all ready.
+  unsigned RemovedElements = 0;
+  for (auto I = WaitSet.begin(), E = WaitSet.end(); I != E;) {
+    InstRef &IR = *I;
+    if (!IR)
+      break;
+
+    // Check if this instruction is now ready. In case, force
+    // a transition in state using method 'update()'.
+    Instruction &IS = *IR.getInstruction();
+    if (IS.isDispatched() && !IS.updateDispatched()) {
+      ++I;
+      continue;
+    }
+    LLVM_DEBUG(dbgs() << "[SCHEDULER]: Instruction #" << IR
+                      << " promoted to the PENDING set.\n");
+
+    PendingSet.emplace_back(IR);
+
     IR.invalidate();
     ++RemovedElements;
     std::iter_swap(I, E - RemovedElements);
   }
 
   WaitSet.resize(WaitSet.size() - RemovedElements);
+  return RemovedElements;
 }
 
 InstRef Scheduler::select() {
@@ -140,7 +173,9 @@ InstRef Scheduler::select() {
     if (QueueIndex == ReadySet.size() ||
         Strategy->compare(IR, ReadySet[QueueIndex])) {
       const InstrDesc &D = IR.getInstruction()->getDesc();
-      if (Resources->canBeIssued(D))
+      uint64_t BusyResourceMask = Resources->checkAvailability(D);
+      BusyResourceUnits |= BusyResourceMask;
+      if (!BusyResourceMask)
         QueueIndex = I;
     }
   }
@@ -192,18 +227,26 @@ void Scheduler::cycleEvent(SmallVectorImpl<ResourceRef> &Freed,
 
   updateIssuedSet(Executed);
 
+  for (InstRef &IR : PendingSet)
+    IR.getInstruction()->cycleEvent();
+
   for (InstRef &IR : WaitSet)
     IR.getInstruction()->cycleEvent();
 
+  promoteToPendingSet();
   promoteToReadySet(Ready);
+
+  BusyResourceUnits = 0;
 }
 
 bool Scheduler::mustIssueImmediately(const InstRef &IR) const {
+  const InstrDesc &Desc = IR.getInstruction()->getDesc();
+  if (Desc.isZeroLatency())
+    return true;
   // Instructions that use an in-order dispatch/issue processor resource must be
   // issued immediately to the pipeline(s). Any other in-order buffered
   // resources (i.e. BufferSize=1) is consumed.
-  const InstrDesc &Desc = IR.getInstruction()->getDesc();
-  return Desc.isZeroLatency() || Resources->mustIssueImmediately(Desc);
+  return Desc.MustIssueImmediately;
 }
 
 void Scheduler::dispatch(const InstRef &IR) {
@@ -215,6 +258,13 @@ void Scheduler::dispatch(const InstRef &IR) {
   if (IsMemOp)
     LSU.dispatch(IR);
 
+  if (IR.getInstruction()->isPending()) {
+    LLVM_DEBUG(dbgs() << "[SCHEDULER] Adding #" << IR
+                      << " to the PendingSet\n");
+    PendingSet.push_back(IR);
+    return;
+  }
+
   if (!isReady(IR)) {
     LLVM_DEBUG(dbgs() << "[SCHEDULER] Adding #" << IR << " to the WaitSet\n");
     WaitSet.push_back(IR);
diff --git a/lib/MCA/InstrBuilder.cpp b/lib/MCA/InstrBuilder.cpp
index 8d501dc6b15abb422234e5d4e4f7c718d0c14e7f..e10efb28f77531e03ccb5f4c0032621cd749411d 100644
--- a/lib/MCA/InstrBuilder.cpp
+++ b/lib/MCA/InstrBuilder.cpp
@@ -1,9 +1,8 @@
 //===--------------------- InstrBuilder.cpp ---------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
@@ -31,6 +30,8 @@ InstrBuilder::InstrBuilder(const llvm::MCSubtargetInfo &sti,
                            const llvm::MCInstrAnalysis *mcia)
     : STI(sti), MCII(mcii), MRI(mri), MCIA(mcia), FirstCallInst(true),
       FirstReturnInst(true) {
+  const MCSchedModel &SM = STI.getSchedModel();
+  ProcResourceMasks.resize(SM.getNumProcResourceKinds());
   computeProcResourceMasks(STI.getSchedModel(), ProcResourceMasks);
 }
 
@@ -59,12 +60,20 @@ static void initializeUsedResources(InstrDesc &ID,
   unsigned NumProcResources = SM.getNumProcResourceKinds();
   APInt Buffers(NumProcResources, 0);
 
+  bool AllInOrderResources = true;
+  bool AnyDispatchHazards = false;
   for (unsigned I = 0, E = SCDesc.NumWriteProcResEntries; I < E; ++I) {
     const MCWriteProcResEntry *PRE = STI.getWriteProcResBegin(&SCDesc) + I;
     const MCProcResourceDesc &PR = *SM.getProcResource(PRE->ProcResourceIdx);
     uint64_t Mask = ProcResourceMasks[PRE->ProcResourceIdx];
-    if (PR.BufferSize != -1)
+    if (PR.BufferSize < 0) {
+      AllInOrderResources = false;
+    } else {
       Buffers.setBit(PRE->ProcResourceIdx);
+      AnyDispatchHazards |= (PR.BufferSize == 0);
+      AllInOrderResources &= (PR.BufferSize <= 1);
+    }
+
     CycleSegment RCy(0, PRE->Cycles, false);
     Worklist.emplace_back(ResourcePlusCycles(Mask, ResourceUsage(RCy)));
     if (PR.SuperIdx) {
@@ -73,6 +82,8 @@ static void initializeUsedResources(InstrDesc &ID,
     }
   }
 
+  ID.MustIssueImmediately = AllInOrderResources && AnyDispatchHazards;
+
   // Sort elements by mask popcount, so that we prioritize resource units over
   // resource groups, and smaller groups over larger groups.
   sort(Worklist, [](const ResourcePlusCycles &A, const ResourcePlusCycles &B) {
@@ -86,14 +97,14 @@ static void initializeUsedResources(InstrDesc &ID,
   });
 
   uint64_t UsedResourceUnits = 0;
+  uint64_t UsedResourceGroups = 0;
 
   // Remove cycles contributed by smaller resources.
   for (unsigned I = 0, E = Worklist.size(); I < E; ++I) {
     ResourcePlusCycles &A = Worklist[I];
     if (!A.second.size()) {
-      A.second.NumUnits = 0;
-      A.second.setReserved();
-      ID.Resources.emplace_back(A);
+      assert(countPopulation(A.first) > 1 && "Expected a group!");
+      UsedResourceGroups |= PowerOf2Floor(A.first);
       continue;
     }
 
@@ -116,6 +127,9 @@ static void initializeUsedResources(InstrDesc &ID,
     }
   }
 
+  ID.UsedProcResUnits = UsedResourceUnits;
+  ID.UsedProcResGroups = UsedResourceGroups;
+
   // A SchedWrite may specify a number of cycles in which a resource group
   // is reserved. For example (on target x86; cpu Haswell):
   //
@@ -168,9 +182,14 @@ static void initializeUsedResources(InstrDesc &ID,
 
   LLVM_DEBUG({
     for (const std::pair<uint64_t, ResourceUsage> &R : ID.Resources)
-      dbgs() << "\t\tMask=" << R.first << ", cy=" << R.second.size() << '\n';
+      dbgs() << "\t\tResource Mask=" << format_hex(R.first, 16) << ", "
+             << "Reserved=" << R.second.isReserved() << ", "
+             << "#Units=" << R.second.NumUnits << ", "
+             << "cy=" << R.second.size() << '\n';
     for (const uint64_t R : ID.Buffers)
-      dbgs() << "\t\tBuffer Mask=" << R << '\n';
+      dbgs() << "\t\tBuffer Mask=" << format_hex(R, 16) << '\n';
+    dbgs()   << "\t\t Used Units=" << format_hex(ID.UsedProcResUnits, 16) << '\n';
+    dbgs()   << "\t\tUsed Groups=" << format_hex(ID.UsedProcResGroups, 16) << '\n';
   });
 }
 
@@ -514,9 +533,13 @@ InstrBuilder::createInstrDescImpl(const MCInst &MCI) {
         MCI);
   }
 
+  LLVM_DEBUG(dbgs() << "\n\t\tOpcode Name= " << MCII.getName(Opcode) << '\n');
+  LLVM_DEBUG(dbgs() << "\t\tSchedClassID=" << SchedClassID << '\n');
+
   // Create a new empty descriptor.
   std::unique_ptr<InstrDesc> ID = llvm::make_unique<InstrDesc>();
   ID->NumMicroOps = SCDesc.NumMicroOps;
+  ID->SchedClassID = SchedClassID;
 
   if (MCDesc.isCall() && FirstCallInst) {
     // We don't correctly model calls.
diff --git a/lib/MCA/Instruction.cpp b/lib/MCA/Instruction.cpp
index 057e95ca9990fdf967cda1c741c5e8e2ce453db7..1722ce08f691954d2cdae6e95930261220a5c55e 100644
--- a/lib/MCA/Instruction.cpp
+++ b/lib/MCA/Instruction.cpp
@@ -1,9 +1,8 @@
 //===--------------------- Instruction.cpp ----------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -65,11 +64,7 @@ void WriteState::addUser(ReadState *User, int ReadAdvance) {
     return;
   }
 
-  if (llvm::find_if(Users, [&User](const std::pair<ReadState *, int> &Use) {
-        return Use.first == User;
-      }) == Users.end()) {
-    Users.emplace_back(User, ReadAdvance);
-  }
+  Users.emplace_back(User, ReadAdvance);
 }
 
 void WriteState::addUser(WriteState *User) {
@@ -128,7 +123,7 @@ void WriteRef::dump() const {
 
 void Instruction::dispatch(unsigned RCUToken) {
   assert(Stage == IS_INVALID);
-  Stage = IS_AVAILABLE;
+  Stage = IS_DISPATCHED;
   RCUTokenID = RCUToken;
 
   // Check if input operands are already available.
@@ -156,30 +151,49 @@ void Instruction::forceExecuted() {
   Stage = IS_EXECUTED;
 }
 
-void Instruction::update() {
-  assert(isDispatched() && "Unexpected instruction stage found!");
+bool Instruction::updatePending() {
+  assert(isPending() && "Unexpected instruction stage found!");
 
   if (!all_of(getUses(), [](const ReadState &Use) { return Use.isReady(); }))
-    return;
+    return false;
+
+  // A partial register write cannot complete before a dependent write.
+  if (!all_of(getDefs(), [](const WriteState &Def) { return Def.isReady(); }))
+    return false;
+
+  Stage = IS_READY;
+  return true;
+}
+
+bool Instruction::updateDispatched() {
+  assert(isDispatched() && "Unexpected instruction stage found!");
+
+  if (!all_of(getUses(), [](const ReadState &Use) {
+        return Use.isPending() || Use.isReady();
+      }))
+    return false;
 
   // A partial register write cannot complete before a dependent write.
-  auto IsDefReady = [&](const WriteState &Def) {
-    if (!Def.getDependentWrite()) {
-      unsigned CyclesLeft = Def.getDependentWriteCyclesLeft();
-      return !CyclesLeft || CyclesLeft < getLatency();
-    }
+  if (!all_of(getDefs(),
+              [](const WriteState &Def) { return !Def.getDependentWrite(); }))
     return false;
-  };
 
-  if (all_of(getDefs(), IsDefReady))
-    Stage = IS_READY;
+  Stage = IS_PENDING;
+  return true;
+}
+
+void Instruction::update() {
+  if (isDispatched())
+    updateDispatched();
+  if (isPending())
+    updatePending();
 }
 
 void Instruction::cycleEvent() {
   if (isReady())
     return;
 
-  if (isDispatched()) {
+  if (isDispatched() || isPending()) {
     for (ReadState &Use : getUses())
       Use.cycleEvent();
 
diff --git a/lib/MCA/LLVMBuild.txt b/lib/MCA/LLVMBuild.txt
index 75f3a9e2229931cccd2e5b67ac1d8d19c4a4367c..ccbd3a2ae7c5a822bf82bea74ad74f8ff3611738 100644
--- a/lib/MCA/LLVMBuild.txt
+++ b/lib/MCA/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./tools/llvm-mca/lib/LLVMBuild.txt -----------------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/MCA/Pipeline.cpp b/lib/MCA/Pipeline.cpp
index fd97ea624b839c4be09a0357710017973b987e84..5361f08f15479b806bc0c80b54f7fe9a2d1730a5 100644
--- a/lib/MCA/Pipeline.cpp
+++ b/lib/MCA/Pipeline.cpp
@@ -1,9 +1,8 @@
 //===--------------------- Pipeline.cpp -------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
@@ -83,13 +82,13 @@ void Pipeline::appendStage(std::unique_ptr<Stage> S) {
 }
 
 void Pipeline::notifyCycleBegin() {
-  LLVM_DEBUG(dbgs() << "[E] Cycle begin: " << Cycles << '\n');
+  LLVM_DEBUG(dbgs() << "\n[E] Cycle begin: " << Cycles << '\n');
   for (HWEventListener *Listener : Listeners)
     Listener->onCycleBegin();
 }
 
 void Pipeline::notifyCycleEnd() {
-  LLVM_DEBUG(dbgs() << "[E] Cycle end: " << Cycles << "\n\n");
+  LLVM_DEBUG(dbgs() << "[E] Cycle end: " << Cycles << "\n");
   for (HWEventListener *Listener : Listeners)
     Listener->onCycleEnd();
 }
diff --git a/lib/MCA/Stages/DispatchStage.cpp b/lib/MCA/Stages/DispatchStage.cpp
index 7fb4eb6a1c0ef85b3b1a6a03157a2df1197f3cfe..a67b4612167193ca8beb95bc4ab8e6294bcd8a3e 100644
--- a/lib/MCA/Stages/DispatchStage.cpp
+++ b/lib/MCA/Stages/DispatchStage.cpp
@@ -1,9 +1,8 @@
 //===--------------------- DispatchStage.cpp --------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
@@ -63,27 +62,6 @@ bool DispatchStage::canDispatch(const InstRef &IR) const {
   return checkRCU(IR) && checkPRF(IR) && checkNextStage(IR);
 }
 
-void DispatchStage::updateRAWDependencies(ReadState &RS,
-                                          const MCSubtargetInfo &STI) {
-  SmallVector<WriteRef, 4> DependentWrites;
-
-  // Collect all the dependent writes, and update RS internal state.
-  PRF.addRegisterRead(RS, DependentWrites);
-
-  // We know that this read depends on all the writes in DependentWrites.
-  // For each write, check if we have ReadAdvance information, and use it
-  // to figure out in how many cycles this read becomes available.
-  const ReadDescriptor &RD = RS.getDescriptor();
-  const MCSchedModel &SM = STI.getSchedModel();
-  const MCSchedClassDesc *SC = SM.getSchedClassDesc(RD.SchedClassID);
-  for (WriteRef &WR : DependentWrites) {
-    WriteState &WS = *WR.getWriteState();
-    unsigned WriteResID = WS.getWriteResourceID();
-    int ReadAdvance = STI.getReadAdvanceCycles(SC, RD.UseIndex, WriteResID);
-    WS.addUser(&RS, ReadAdvance);
-  }
-}
-
 Error DispatchStage::dispatch(InstRef IR) {
   assert(!CarryOver && "Cannot dispatch another instruction!");
   Instruction &IS = *IR.getInstruction();
@@ -122,7 +100,7 @@ Error DispatchStage::dispatch(InstRef IR) {
   // eliminated at register renaming stage.
   if (!IsEliminated) {
     for (ReadState &RS : IS.getUses())
-      updateRAWDependencies(RS, STI);
+      PRF.addRegisterRead(RS, STI);
   }
 
   // By default, a dependency-breaking zero-idiom is expected to be optimized
diff --git a/lib/MCA/Stages/EntryStage.cpp b/lib/MCA/Stages/EntryStage.cpp
index 3325bb36f5afe91e9bcf9f04064f2029fe8975c0..2028b9e2b765c2a19553b7c0e42db8f31d069b35 100644
--- a/lib/MCA/Stages/EntryStage.cpp
+++ b/lib/MCA/Stages/EntryStage.cpp
@@ -1,9 +1,8 @@
 //===---------------------- EntryStage.cpp ----------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/lib/MCA/Stages/ExecuteStage.cpp b/lib/MCA/Stages/ExecuteStage.cpp
index 17f7ff7259a3e963bc5e003c011f417ab130c4e6..9e0bd26e6d029526746bbf009e62c0b61782e9fb 100644
--- a/lib/MCA/Stages/ExecuteStage.cpp
+++ b/lib/MCA/Stages/ExecuteStage.cpp
@@ -1,9 +1,8 @@
 //===---------------------- ExecuteStage.cpp --------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
@@ -57,6 +56,7 @@ Error ExecuteStage::issueInstruction(InstRef &IR) {
   HWS.issueInstruction(IR, Used, Ready);
 
   notifyReservedOrReleasedBuffers(IR, /* Reserved */ false);
+
   notifyInstructionIssued(IR, Used);
   if (IR.getInstruction()->isExecuted()) {
     notifyInstructionExecuted(IR);
@@ -184,15 +184,21 @@ void ExecuteStage::notifyResourceAvailable(const ResourceRef &RR) const {
 
 void ExecuteStage::notifyInstructionIssued(
     const InstRef &IR,
-    ArrayRef<std::pair<ResourceRef, ResourceCycles>> Used) const {
+    MutableArrayRef<std::pair<ResourceRef, ResourceCycles>> Used) const {
   LLVM_DEBUG({
     dbgs() << "[E] Instruction Issued: #" << IR << '\n';
     for (const std::pair<ResourceRef, ResourceCycles> &Resource : Used) {
+      assert(Resource.second.getDenominator() == 1 && "Invalid cycles!");
       dbgs() << "[E] Resource Used: [" << Resource.first.first << '.'
              << Resource.first.second << "], ";
-      dbgs() << "cycles: " << Resource.second << '\n';
+      dbgs() << "cycles: " << Resource.second.getNumerator() << '\n';
     }
   });
+
+  // Replace resource masks with valid resource processor IDs.
+  for (std::pair<ResourceRef, ResourceCycles> &Use : Used)
+    Use.first.first = HWS.getResourceID(Use.first.first);
+
   notifyEvent<HWInstructionEvent>(HWInstructionIssuedEvent(IR, Used));
 }
 
diff --git a/lib/MCA/Stages/InstructionTables.cpp b/lib/MCA/Stages/InstructionTables.cpp
index f918c183aa5a0486452a4cd5dac3f80deb99f1f3..adeefb45ec2d9691293d6bd82a921b541725f0a4 100644
--- a/lib/MCA/Stages/InstructionTables.cpp
+++ b/lib/MCA/Stages/InstructionTables.cpp
@@ -1,9 +1,8 @@
 //===--------------------- InstructionTables.cpp ----------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/lib/MCA/Stages/RetireStage.cpp b/lib/MCA/Stages/RetireStage.cpp
index d6bcc518662f5323e87c2a21c93a237e328b667f..e1789dd7fa2aa7c4edd9ba4947c72b71f7ce5436 100644
--- a/lib/MCA/Stages/RetireStage.cpp
+++ b/lib/MCA/Stages/RetireStage.cpp
@@ -1,9 +1,8 @@
 //===---------------------- RetireStage.cpp ---------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/lib/MCA/Stages/Stage.cpp b/lib/MCA/Stages/Stage.cpp
index 38191645e736dab45a0543e5cf7800c74b9aaaed..ed512ac9711c54ad948296c8b6f7ff7fcb077ae4 100644
--- a/lib/MCA/Stages/Stage.cpp
+++ b/lib/MCA/Stages/Stage.cpp
@@ -1,9 +1,8 @@
 //===---------------------- Stage.cpp ---------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/lib/MCA/Support.cpp b/lib/MCA/Support.cpp
index 3271bc6bf5b4f103fcd046cd5c48547dccc96f5e..ce1f0f6f211b510f57d4482dec41b68e3d38ad7d 100644
--- a/lib/MCA/Support.cpp
+++ b/lib/MCA/Support.cpp
@@ -1,9 +1,8 @@
 //===--------------------- Support.cpp --------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
@@ -19,13 +18,34 @@
 namespace llvm {
 namespace mca {
 
+#define DEBUG_TYPE "llvm-mca"
+
+ResourceCycles &ResourceCycles::operator+=(const ResourceCycles &RHS) {
+  if (Denominator == RHS.Denominator)
+    Numerator += RHS.Numerator;
+  else {
+    // Create a common denominator for LHS and RHS by calculating the least
+    // common multiple from the GCD.
+    unsigned GCD = GreatestCommonDivisor64(Denominator, RHS.Denominator);
+    unsigned LCM = (Denominator * RHS.Denominator) / GCD;
+    unsigned LHSNumerator = Numerator * (LCM / Denominator);
+    unsigned RHSNumerator = RHS.Numerator * (LCM / RHS.Denominator);
+    Numerator = LHSNumerator + RHSNumerator;
+    Denominator = LCM;
+  }
+  return *this;
+}
+
 void computeProcResourceMasks(const MCSchedModel &SM,
-                              SmallVectorImpl<uint64_t> &Masks) {
+                              MutableArrayRef<uint64_t> Masks) {
   unsigned ProcResourceID = 0;
 
+  assert(Masks.size() == SM.getNumProcResourceKinds() &&
+         "Invalid number of elements");
+  // Resource at index 0 is the 'InvalidUnit'. Set an invalid mask for it.
+  Masks[0] = 0;
+
   // Create a unique bitmask for every processor resource unit.
-  // Skip resource at index 0, since it always references 'InvalidUnit'.
-  Masks.resize(SM.getNumProcResourceKinds());
   for (unsigned I = 1, E = SM.getNumProcResourceKinds(); I < E; ++I) {
     const MCProcResourceDesc &Desc = *SM.getProcResource(I);
     if (Desc.SubUnitsIdxBegin)
@@ -46,6 +66,17 @@ void computeProcResourceMasks(const MCSchedModel &SM,
     }
     ProcResourceID++;
   }
+
+#ifndef NDEBUG
+  LLVM_DEBUG(dbgs() << "\nProcessor resource masks:"
+                    << "\n");
+  for (unsigned I = 0, E = SM.getNumProcResourceKinds(); I < E; ++I) {
+    const MCProcResourceDesc &Desc = *SM.getProcResource(I);
+    LLVM_DEBUG(dbgs() << '[' << format_decimal(I,2) << "] " << " - "
+                      << format_hex(Masks[I],16) << " - "
+                      << Desc.Name << '\n');
+  }
+#endif
 }
 
 double computeBlockRThroughput(const MCSchedModel &SM, unsigned DispatchWidth,
diff --git a/lib/Object/Archive.cpp b/lib/Object/Archive.cpp
index 8ec115a5566cadfbf07bb961b3ac715e396afbb4..49e66f46ab3f249eff64ff27233297d9201c5dd3 100644
--- a/lib/Object/Archive.cpp
+++ b/lib/Object/Archive.cpp
@@ -1,9 +1,8 @@
 //===- Archive.cpp - ar File Format implementation ------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -512,7 +511,7 @@ Expected<MemoryBufferRef> Archive::Child::getMemoryBufferRef() const {
   StringRef Name = NameOrErr.get();
   Expected<StringRef> Buf = getBuffer();
   if (!Buf)
-    return Buf.takeError();
+    return createFileError(Name, Buf.takeError());
   return MemoryBufferRef(*Buf, Name);
 }
 
@@ -779,19 +778,18 @@ Archive::child_iterator Archive::child_begin(Error &Err,
     return child_end();
 
   if (SkipInternal)
-    return child_iterator(Child(this, FirstRegularData,
-                                FirstRegularStartOfFile),
-                          &Err);
+    return child_iterator::itr(
+        Child(this, FirstRegularData, FirstRegularStartOfFile), Err);
 
   const char *Loc = Data.getBufferStart() + strlen(Magic);
   Child C(this, Loc, &Err);
   if (Err)
     return child_end();
-  return child_iterator(C, &Err);
+  return child_iterator::itr(C, Err);
 }
 
 Archive::child_iterator Archive::child_end() const {
-  return child_iterator(Child(nullptr, nullptr, nullptr), nullptr);
+  return child_iterator::end(Child(nullptr, nullptr, nullptr));
 }
 
 StringRef Archive::Symbol::getName() const {
diff --git a/lib/Object/ArchiveWriter.cpp b/lib/Object/ArchiveWriter.cpp
index 767205390e0782bf655af0882c1751126b364461..d27233ec2f4d4ea08dd7abcc110e44d97d5dbc72 100644
--- a/lib/Object/ArchiveWriter.cpp
+++ b/lib/Object/ArchiveWriter.cpp
@@ -1,9 +1,8 @@
 //===- ArchiveWriter.cpp - ar File Format implementation --------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -250,6 +249,7 @@ static void addToStringTable(raw_ostream &Out, StringRef ArcName,
 
 static void printMemberHeader(raw_ostream &Out, uint64_t Pos,
                               raw_ostream &StringTable,
+                              StringMap<uint64_t> &MemberNames,
                               object::Archive::Kind Kind, bool Thin,
                               StringRef ArcName, const NewArchiveMember &M,
                               sys::TimePoint<std::chrono::seconds> ModTime,
@@ -262,8 +262,18 @@ static void printMemberHeader(raw_ostream &Out, uint64_t Pos,
     return printGNUSmallMemberHeader(Out, M.MemberName, ModTime, M.UID, M.GID,
                                      M.Perms, Size);
   Out << '/';
-  uint64_t NamePos = StringTable.tell();
-  addToStringTable(StringTable, ArcName, M, Thin);
+  uint64_t NamePos;
+  if (Thin) {
+    NamePos = StringTable.tell();
+    addToStringTable(StringTable, ArcName, M, Thin);
+  } else {
+    auto Insertion = MemberNames.insert({M.MemberName, uint64_t(0)});
+    if (Insertion.second) {
+      Insertion.first->second = StringTable.tell();
+      addToStringTable(StringTable, ArcName, M, Thin);
+    }
+    NamePos = Insertion.first->second;
+  }
   printWithSpacePadding(Out, NamePos, 15);
   printRestOfMemberHeader(Out, ModTime, M.UID, M.GID, M.Perms, Size);
 }
@@ -433,6 +443,11 @@ computeMemberData(raw_ostream &StringTable, raw_ostream &SymNames,
   std::vector<MemberData> Ret;
   bool HasObject = false;
 
+  // Deduplicate long member names in the string table and reuse earlier name
+  // offsets. This especially saves space for COFF Import libraries where all
+  // members have the same name.
+  StringMap<uint64_t> MemberNames;
+
   // UniqueTimestamps is a special case to improve debugging on Darwin:
   //
   // The Darwin linker does not link debug info into the final
@@ -505,8 +520,8 @@ computeMemberData(raw_ostream &StringTable, raw_ostream &SymNames,
       ModTime = sys::toTimePoint(FilenameCount[M.MemberName]++);
     else
       ModTime = M.ModTime;
-    printMemberHeader(Out, Pos, StringTable, Kind, Thin, ArcName, M, ModTime,
-                      Buf.getBufferSize() + MemberPadding);
+    printMemberHeader(Out, Pos, StringTable, MemberNames, Kind, Thin, ArcName,
+                      M, ModTime, Buf.getBufferSize() + MemberPadding);
     Out.flush();
 
     Expected<std::vector<unsigned>> Symbols =
diff --git a/lib/Object/Binary.cpp b/lib/Object/Binary.cpp
index d7c25921ec36ec2f47fd0329b3aeb6748944778f..87b2c1278c0694b5474f3e82504d28723838ece5 100644
--- a/lib/Object/Binary.cpp
+++ b/lib/Object/Binary.cpp
@@ -1,9 +1,8 @@
 //===- Binary.cpp - A generic binary file ---------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -88,7 +87,8 @@ Expected<std::unique_ptr<Binary>> object::createBinary(MemoryBufferRef Buffer,
 
 Expected<OwningBinary<Binary>> object::createBinary(StringRef Path) {
   ErrorOr<std::unique_ptr<MemoryBuffer>> FileOrErr =
-      MemoryBuffer::getFileOrSTDIN(Path);
+      MemoryBuffer::getFileOrSTDIN(Path, /*FileSize=*/-1,
+                                   /*RequiresNullTerminator=*/false);
   if (std::error_code EC = FileOrErr.getError())
     return errorCodeToError(EC);
   std::unique_ptr<MemoryBuffer> &Buffer = FileOrErr.get();
diff --git a/lib/Object/COFFImportFile.cpp b/lib/Object/COFFImportFile.cpp
index dc11cc4bcffe64b7157d187a4dea675adcb7a080..e8d2edd2d2b42f80a8656efb3b3059866e57f6d3 100644
--- a/lib/Object/COFFImportFile.cpp
+++ b/lib/Object/COFFImportFile.cpp
@@ -1,9 +1,8 @@
 //===- COFFImportFile.cpp - COFF short import file implementation ---------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Object/COFFModuleDefinition.cpp b/lib/Object/COFFModuleDefinition.cpp
index c703071b86e06447024eb40b6c14bd106b939b33..64d4cf0efda2d2d2b61e145bb8041847b0324eeb 100644
--- a/lib/Object/COFFModuleDefinition.cpp
+++ b/lib/Object/COFFModuleDefinition.cpp
@@ -1,9 +1,8 @@
 //===--- COFFModuleDefinition.cpp - Simple DEF parser ---------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Object/COFFObjectFile.cpp b/lib/Object/COFFObjectFile.cpp
index e17f0e2783e1e838c728cbb2d33a9283ca7e9c78..7839c65516b4d70d1879a7f2fcc4129e03e4637e 100644
--- a/lib/Object/COFFObjectFile.cpp
+++ b/lib/Object/COFFObjectFile.cpp
@@ -1,9 +1,8 @@
 //===- COFFObjectFile.cpp - COFF object file implementation ---------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -1065,6 +1064,16 @@ COFFObjectFile::getSymbolAuxData(COFFSymbolRef Symbol) const {
   return makeArrayRef(Aux, Symbol.getNumberOfAuxSymbols() * SymbolSize);
 }
 
+uint32_t COFFObjectFile::getSymbolIndex(COFFSymbolRef Symbol) const {
+  uintptr_t Offset =
+      reinterpret_cast<uintptr_t>(Symbol.getRawPtr()) - getSymbolTable();
+  assert(Offset % getSymbolTableEntrySize() == 0 &&
+         "Symbol did not point to the beginning of a symbol");
+  size_t Index = Offset / getSymbolTableEntrySize();
+  assert(Index < getNumberOfSymbols());
+  return Index;
+}
+
 std::error_code COFFObjectFile::getSectionName(const coff_section *Sec,
                                                StringRef &Res) const {
   StringRef Name;
@@ -1227,6 +1236,7 @@ StringRef COFFObjectFile::getRelocationTypeName(uint16_t Type) const {
     LLVM_COFF_SWITCH_RELOC_TYPE_NAME(IMAGE_REL_ARM_TOKEN);
     LLVM_COFF_SWITCH_RELOC_TYPE_NAME(IMAGE_REL_ARM_BLX24);
     LLVM_COFF_SWITCH_RELOC_TYPE_NAME(IMAGE_REL_ARM_BLX11);
+    LLVM_COFF_SWITCH_RELOC_TYPE_NAME(IMAGE_REL_ARM_REL32);
     LLVM_COFF_SWITCH_RELOC_TYPE_NAME(IMAGE_REL_ARM_SECTION);
     LLVM_COFF_SWITCH_RELOC_TYPE_NAME(IMAGE_REL_ARM_SECREL);
     LLVM_COFF_SWITCH_RELOC_TYPE_NAME(IMAGE_REL_ARM_MOV32A);
@@ -1234,6 +1244,7 @@ StringRef COFFObjectFile::getRelocationTypeName(uint16_t Type) const {
     LLVM_COFF_SWITCH_RELOC_TYPE_NAME(IMAGE_REL_ARM_BRANCH20T);
     LLVM_COFF_SWITCH_RELOC_TYPE_NAME(IMAGE_REL_ARM_BRANCH24T);
     LLVM_COFF_SWITCH_RELOC_TYPE_NAME(IMAGE_REL_ARM_BLX23T);
+    LLVM_COFF_SWITCH_RELOC_TYPE_NAME(IMAGE_REL_ARM_PAIR);
     default:
       return "Unknown";
     }
@@ -1257,6 +1268,7 @@ StringRef COFFObjectFile::getRelocationTypeName(uint16_t Type) const {
     LLVM_COFF_SWITCH_RELOC_TYPE_NAME(IMAGE_REL_ARM64_ADDR64);
     LLVM_COFF_SWITCH_RELOC_TYPE_NAME(IMAGE_REL_ARM64_BRANCH19);
     LLVM_COFF_SWITCH_RELOC_TYPE_NAME(IMAGE_REL_ARM64_BRANCH14);
+    LLVM_COFF_SWITCH_RELOC_TYPE_NAME(IMAGE_REL_ARM64_REL32);
     default:
       return "Unknown";
     }
diff --git a/lib/Object/Decompressor.cpp b/lib/Object/Decompressor.cpp
index 53f084d7620e7419a28920b85cf76a95852e33a2..ec15e6f69ada77442f17654ff73d79098bbe479d 100644
--- a/lib/Object/Decompressor.cpp
+++ b/lib/Object/Decompressor.cpp
@@ -1,9 +1,8 @@
 //===-- Decompressor.cpp --------------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Object/ELF.cpp b/lib/Object/ELF.cpp
index cf8313f88f9394ed4a68b74295f9662714ed9da2..0cf928651cd11fd5ef470268473e14e87116ac5f 100644
--- a/lib/Object/ELF.cpp
+++ b/lib/Object/ELF.cpp
@@ -1,9 +1,8 @@
 //===- ELF.cpp - ELF object file implementation ---------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Object/ELFObjectFile.cpp b/lib/Object/ELFObjectFile.cpp
index 9fb3a55ac7b1855d531316489d366c1d4753e45c..15e09f8116b0896a2f0199ca3925a4aa891015ab 100644
--- a/lib/Object/ELFObjectFile.cpp
+++ b/lib/Object/ELFObjectFile.cpp
@@ -1,9 +1,8 @@
 //===- ELFObjectFile.cpp - ELF object file implementation -----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Object/Error.cpp b/lib/Object/Error.cpp
index 7d43a84f3e0e069e625b5e8c35d53a47a822f568..f2a009000c58de4879fe907f0cb5c7a12f1bdf31 100644
--- a/lib/Object/Error.cpp
+++ b/lib/Object/Error.cpp
@@ -1,9 +1,8 @@
 //===- Error.cpp - system_error extensions for Object -----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -57,6 +56,7 @@ std::string _object_error_category::message(int EV) const {
                    "defined.");
 }
 
+void BinaryError::anchor() {}
 char BinaryError::ID = 0;
 char GenericBinaryError::ID = 0;
 
diff --git a/lib/Object/IRObjectFile.cpp b/lib/Object/IRObjectFile.cpp
index 1ecb26d60bcea34a9d8b96831cad528541b6259a..5001ac336be0f8a4be6625c2a6d08cff3e5d88ee 100644
--- a/lib/Object/IRObjectFile.cpp
+++ b/lib/Object/IRObjectFile.cpp
@@ -1,9 +1,8 @@
 //===- IRObjectFile.cpp - IR object file implementation ---------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Object/IRSymtab.cpp b/lib/Object/IRSymtab.cpp
index 344d565349c09361d58f8a0101022c61ac673e3e..e80eaf1db328fd31f74e8672532d405d9c5e7a2a 100644
--- a/lib/Object/IRSymtab.cpp
+++ b/lib/Object/IRSymtab.cpp
@@ -1,9 +1,8 @@
 //===- IRSymtab.cpp - implementation of IR symbol tables ------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Object/LLVMBuild.txt b/lib/Object/LLVMBuild.txt
index 687713bab6a2903caa860758c23fede1d382b520..605f1fbab7b2a8b68994813b65b6d937900ba262 100644
--- a/lib/Object/LLVMBuild.txt
+++ b/lib/Object/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/Object/LLVMBuild.txt -------------------------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Object/MachOObjectFile.cpp b/lib/Object/MachOObjectFile.cpp
index 86f29365273eb6173849fc63ee3b6e1c403835b2..69e69bdcf93e6fbd96a2373b3a544247c1f3934b 100644
--- a/lib/Object/MachOObjectFile.cpp
+++ b/lib/Object/MachOObjectFile.cpp
@@ -1,9 +1,8 @@
 //===- MachOObjectFile.cpp - Mach-O object file binding -------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -2242,9 +2241,18 @@ uint8_t MachOObjectFile::getRelocationLength(DataRefImpl Rel) const {
 // one of the two following forms:
 //      libFoo.A.dylib
 //      libFoo.dylib
+//
 // The library may have a suffix trailing the name Foo of the form:
 //      libFoo_profile.A.dylib
 //      libFoo_profile.dylib
+// These dyld image suffixes are separated from the short name by a '_'
+// character. Because the '_' character is commonly used to separate words in
+// filenames guessLibraryShortName() cannot reliably separate a dylib's short
+// name from an arbitrary image suffix; imagine if both the short name and the
+// suffix contains an '_' character! To better deal with this ambiguity,
+// guessLibraryShortName() will recognize only "_debug" and "_profile" as valid
+// Suffix values. Calling code needs to be tolerant of guessLibraryShortName()
+// guessing incorrectly.
 //
 // The Name of the dynamic library is also recognized as a library name if it
 // has the following form:
@@ -2252,7 +2260,6 @@ uint8_t MachOObjectFile::getRelocationLength(DataRefImpl Rel) const {
 //
 // If the Name of the dynamic library is none of the forms above then a NULL
 // StringRef is returned.
-//
 StringRef MachOObjectFile::guessLibraryShortName(StringRef Name,
                                                  bool &isFramework,
                                                  StringRef &Suffix) {
@@ -2272,7 +2279,10 @@ StringRef MachOObjectFile::guessLibraryShortName(StringRef Name,
   Idx = Foo.rfind('_');
   if (Idx != Foo.npos && Foo.size() >= 2) {
     Suffix = Foo.slice(Idx, Foo.npos);
-    Foo = Foo.slice(0, Idx);
+    if (Suffix != "_debug" && Suffix != "_profile")
+      Suffix = StringRef();
+    else
+      Foo = Foo.slice(0, Idx);
   }
 
   // First look for the form Foo.framework/Foo
@@ -2333,10 +2343,14 @@ guess_library:
   else
     b = b+1;
   // ignore any suffix after an underbar like Foo_profile.A.dylib
-  Idx = Name.find('_', b);
+  Idx = Name.rfind('_');
   if (Idx != Name.npos && Idx != b) {
     Lib = Name.slice(b, Idx);
     Suffix = Name.slice(Idx, a);
+    if (Suffix != "_debug" && Suffix != "_profile") {
+      Suffix = StringRef();
+      Lib = Name.slice(b, a);
+    }
   }
   else
     Lib = Name.slice(b, a);
@@ -2438,7 +2452,7 @@ basic_symbol_iterator MachOObjectFile::symbol_end() const {
   return basic_symbol_iterator(SymbolRef(DRI, this));
 }
 
-basic_symbol_iterator MachOObjectFile::getSymbolByIndex(unsigned Index) const {
+symbol_iterator MachOObjectFile::getSymbolByIndex(unsigned Index) const {
   MachO::symtab_command Symtab = getSymtabLoadCommand();
   if (!SymtabLoadCmd || Index >= Symtab.nsyms)
     report_fatal_error("Requested symbol index is out of range.");
diff --git a/lib/Object/MachOUniversal.cpp b/lib/Object/MachOUniversal.cpp
index 309708e9b37c895775e83c639b182069e790ccf5..b3f0993412c67de55528e03559195cf73c1feb81 100644
--- a/lib/Object/MachOUniversal.cpp
+++ b/lib/Object/MachOUniversal.cpp
@@ -1,9 +1,8 @@
 //===- MachOUniversal.cpp - Mach-O universal binary -------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Object/ModuleSymbolTable.cpp b/lib/Object/ModuleSymbolTable.cpp
index 33ce7d8109fb8a74083dd6a85205df2603d45346..d1e0ce5edae1c1436c561007814cbcfa0b90636d 100644
--- a/lib/Object/ModuleSymbolTable.cpp
+++ b/lib/Object/ModuleSymbolTable.cpp
@@ -1,9 +1,8 @@
 //===- ModuleSymbolTable.cpp - symbol table for in-memory IR --------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Object/Object.cpp b/lib/Object/Object.cpp
index f5de2e1d5ce23b8cae7d7ff4d956ad589d61b22f..5032509a47c75008424d8a94c661e48346715cf8 100644
--- a/lib/Object/Object.cpp
+++ b/lib/Object/Object.cpp
@@ -1,9 +1,8 @@
 //===- Object.cpp - C bindings to the object file library--------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Object/ObjectFile.cpp b/lib/Object/ObjectFile.cpp
index cf63b89adc122a516d69a8adc37f368b8b027404..441ef54bd84297b92bc632da439f0ab1452e92c1 100644
--- a/lib/Object/ObjectFile.cpp
+++ b/lib/Object/ObjectFile.cpp
@@ -1,9 +1,8 @@
 //===- ObjectFile.cpp - File format independent object file ---------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Object/RecordStreamer.cpp b/lib/Object/RecordStreamer.cpp
index 1f57867dd21abf993f77ec4ab5b4d9956f769e6c..f39a6c28ed50ccaa1823f3d842c778bb8d301e8e 100644
--- a/lib/Object/RecordStreamer.cpp
+++ b/lib/Object/RecordStreamer.cpp
@@ -1,9 +1,8 @@
 //===-- RecordStreamer.cpp - Record asm defined and used symbols ----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -83,7 +82,7 @@ RecordStreamer::const_iterator RecordStreamer::begin() {
 RecordStreamer::const_iterator RecordStreamer::end() { return Symbols.end(); }
 
 void RecordStreamer::EmitInstruction(const MCInst &Inst,
-                                     const MCSubtargetInfo &STI, bool) {
+                                     const MCSubtargetInfo &STI) {
   MCStreamer::EmitInstruction(Inst, STI);
 }
 
diff --git a/lib/Object/RecordStreamer.h b/lib/Object/RecordStreamer.h
index 3d5ae59b58fe71d4ecac2aa56cef335a21b11fb1..c8b75bcc6d1dfad126cd0404306dbc2459bc184e 100644
--- a/lib/Object/RecordStreamer.h
+++ b/lib/Object/RecordStreamer.h
@@ -1,9 +1,8 @@
 //===- RecordStreamer.h - Record asm defined and used symbols ---*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -47,8 +46,7 @@ private:
 public:
   RecordStreamer(MCContext &Context, const Module &M);
 
-  void EmitInstruction(const MCInst &Inst, const MCSubtargetInfo &STI,
-                       bool) override;
+  void EmitInstruction(const MCInst &Inst, const MCSubtargetInfo &STI) override;
   void EmitLabel(MCSymbol *Symbol, SMLoc Loc = SMLoc()) override;
   void EmitAssignment(MCSymbol *Symbol, const MCExpr *Value) override;
   bool EmitSymbolAttribute(MCSymbol *Symbol, MCSymbolAttr Attribute) override;
@@ -56,6 +54,15 @@ public:
                     unsigned ByteAlignment, SMLoc Loc = SMLoc()) override;
   void EmitCommonSymbol(MCSymbol *Symbol, uint64_t Size,
                         unsigned ByteAlignment) override;
+
+  // Ignore COFF-specific directives; we do not need any information from them,
+  // but the default implementation of these methods crashes, so we override
+  // them with versions that do nothing.
+  void BeginCOFFSymbolDef(const MCSymbol *Symbol) override {}
+  void EmitCOFFSymbolStorageClass(int StorageClass) override {}
+  void EmitCOFFSymbolType(int Type) override {}
+  void EndCOFFSymbolDef() override {}
+
   /// Record .symver aliases for later processing.
   void emitELFSymverDirective(StringRef AliasName,
                               const MCSymbol *Aliasee) override;
diff --git a/lib/Object/SymbolSize.cpp b/lib/Object/SymbolSize.cpp
index 004fb1b075464b1cefd8b770d9c72246de686bd5..bdf4dc55cf3cc6395411ec956143464ee0281967 100644
--- a/lib/Object/SymbolSize.cpp
+++ b/lib/Object/SymbolSize.cpp
@@ -1,9 +1,8 @@
 //===- SymbolSize.cpp -----------------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Object/SymbolicFile.cpp b/lib/Object/SymbolicFile.cpp
index 3e998a2682b815528e08566683d7a57b658615f9..8ee8256771074a5c17df13ff19ac54b936bc1d43 100644
--- a/lib/Object/SymbolicFile.cpp
+++ b/lib/Object/SymbolicFile.cpp
@@ -1,9 +1,8 @@
 //===- SymbolicFile.cpp - Interface that only provides symbols ------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Object/WasmObjectFile.cpp b/lib/Object/WasmObjectFile.cpp
index 34e4d192e590a5f4b42287ac7b1ee16b3c6b89f7..e5b6a9b6e35b12188e509f0eddad60b90bf5f7ab 100644
--- a/lib/Object/WasmObjectFile.cpp
+++ b/lib/Object/WasmObjectFile.cpp
@@ -1,15 +1,15 @@
 //===- WasmObjectFile.cpp - Wasm object file implementation ---------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/SmallSet.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/StringSet.h"
 #include "llvm/ADT/Triple.h"
@@ -131,24 +131,24 @@ static int64_t readLEB128(WasmObjectFile::ReadContext &Ctx) {
 }
 
 static uint8_t readVaruint1(WasmObjectFile::ReadContext &Ctx) {
-  int64_t result = readLEB128(Ctx);
-  if (result > VARUINT1_MAX || result < 0)
+  int64_t Result = readLEB128(Ctx);
+  if (Result > VARUINT1_MAX || Result < 0)
     report_fatal_error("LEB is outside Varuint1 range");
-  return result;
+  return Result;
 }
 
 static int32_t readVarint32(WasmObjectFile::ReadContext &Ctx) {
-  int64_t result = readLEB128(Ctx);
-  if (result > INT32_MAX || result < INT32_MIN)
+  int64_t Result = readLEB128(Ctx);
+  if (Result > INT32_MAX || Result < INT32_MIN)
     report_fatal_error("LEB is outside Varint32 range");
-  return result;
+  return Result;
 }
 
 static uint32_t readVaruint32(WasmObjectFile::ReadContext &Ctx) {
-  uint64_t result = readULEB128(Ctx);
-  if (result > UINT32_MAX)
+  uint64_t Result = readULEB128(Ctx);
+  if (Result > UINT32_MAX)
     report_fatal_error("LEB is outside Varuint32 range");
-  return result;
+  return Result;
 }
 
 static int64_t readVarint64(WasmObjectFile::ReadContext &Ctx) {
@@ -176,7 +176,7 @@ static Error readInitExpr(wasm::WasmInitExpr &Expr,
   case wasm::WASM_OPCODE_F64_CONST:
     Expr.Value.Float64 = readFloat64(Ctx);
     break;
-  case wasm::WASM_OPCODE_GET_GLOBAL:
+  case wasm::WASM_OPCODE_GLOBAL_GET:
     Expr.Value.Global = readULEB128(Ctx);
     break;
   default:
@@ -255,7 +255,7 @@ WasmObjectFile::WasmObjectFile(MemoryBufferRef Buffer, Error &Err)
   }
 
   ReadContext Ctx;
-  Ctx.Start = getPtr(0);
+  Ctx.Start = reinterpret_cast<const uint8_t *>(getData().data());
   Ctx.Ptr = Ctx.Start + 4;
   Ctx.End = Ctx.Start + getData().size();
 
@@ -418,17 +418,17 @@ Error WasmObjectFile::parseLinkingSection(ReadContext &Ctx) {
       if (Count > DataSegments.size())
         return make_error<GenericBinaryError>("Too many segment names",
                                               object_error::parse_failed);
-      for (uint32_t i = 0; i < Count; i++) {
-        DataSegments[i].Data.Name = readString(Ctx);
-        DataSegments[i].Data.Alignment = readVaruint32(Ctx);
-        DataSegments[i].Data.Flags = readVaruint32(Ctx);
+      for (uint32_t I = 0; I < Count; I++) {
+        DataSegments[I].Data.Name = readString(Ctx);
+        DataSegments[I].Data.Alignment = readVaruint32(Ctx);
+        DataSegments[I].Data.Flags = readVaruint32(Ctx);
       }
       break;
     }
     case wasm::WASM_INIT_FUNCS: {
       uint32_t Count = readVaruint32(Ctx);
       LinkingData.InitFunctions.reserve(Count);
-      for (uint32_t i = 0; i < Count; i++) {
+      for (uint32_t I = 0; I < Count; I++) {
         wasm::WasmInitFunc Init;
         Init.Priority = readVaruint32(Ctx);
         Init.Symbol = readVaruint32(Ctx);
@@ -505,9 +505,13 @@ Error WasmObjectFile::parseLinkingSectionSymtab(ReadContext &Ctx) {
           Function.SymbolName = Info.Name;
       } else {
         wasm::WasmImport &Import = *ImportedFunctions[Info.ElementIndex];
+        if ((Info.Flags & wasm::WASM_SYMBOL_EXPLICIT_NAME) != 0)
+          Info.Name = readString(Ctx);
+        else
+          Info.Name = Import.Field;
         Signature = &Signatures[Import.SigIndex];
-        Info.Name = Import.Field;
-        Info.Module = Import.Module;
+        Info.ImportName = Import.Field;
+        Info.ImportModule = Import.Module;
       }
       break;
 
@@ -530,8 +534,13 @@ Error WasmObjectFile::parseLinkingSectionSymtab(ReadContext &Ctx) {
           Global.SymbolName = Info.Name;
       } else {
         wasm::WasmImport &Import = *ImportedGlobals[Info.ElementIndex];
-        Info.Name = Import.Field;
+        if ((Info.Flags & wasm::WASM_SYMBOL_EXPLICIT_NAME) != 0)
+          Info.Name = readString(Ctx);
+        else
+          Info.Name = Import.Field;
         GlobalType = &Import.Global;
+        Info.ImportName = Import.Field;
+        Info.ImportModule = Import.Module;
       }
       break;
 
@@ -585,9 +594,14 @@ Error WasmObjectFile::parseLinkingSectionSymtab(ReadContext &Ctx) {
 
       } else {
         wasm::WasmImport &Import = *ImportedEvents[Info.ElementIndex];
+        if ((Info.Flags & wasm::WASM_SYMBOL_EXPLICIT_NAME) != 0)
+          Info.Name = readString(Ctx);
+        else
+          Info.Name = Import.Field;
         EventType = &Import.Event;
         Signature = &Signatures[EventType->SigIndex];
-        Info.Name = Import.Field;
+        Info.ImportName = Import.Field;
+        Info.ImportModule = Import.Module;
       }
       break;
     }
@@ -659,6 +673,47 @@ Error WasmObjectFile::parseLinkingSectionComdat(ReadContext &Ctx) {
   return Error::success();
 }
 
+Error WasmObjectFile::parseProducersSection(ReadContext &Ctx) {
+  llvm::SmallSet<StringRef, 3> FieldsSeen;
+  uint32_t Fields = readVaruint32(Ctx);
+  for (size_t I = 0; I < Fields; ++I) {
+    StringRef FieldName = readString(Ctx);
+    if (!FieldsSeen.insert(FieldName).second)
+      return make_error<GenericBinaryError>(
+          "Producers section does not have unique fields",
+          object_error::parse_failed);
+    std::vector<std::pair<std::string, std::string>> *ProducerVec = nullptr;
+    if (FieldName == "language") {
+      ProducerVec = &ProducerInfo.Languages;
+    } else if (FieldName == "processed-by") {
+      ProducerVec = &ProducerInfo.Tools;
+    } else if (FieldName == "sdk") {
+      ProducerVec = &ProducerInfo.SDKs;
+    } else {
+      return make_error<GenericBinaryError>(
+          "Producers section field is not named one of language, processed-by, "
+          "or sdk",
+          object_error::parse_failed);
+    }
+    uint32_t ValueCount = readVaruint32(Ctx);
+    llvm::SmallSet<StringRef, 8> ProducersSeen;
+    for (size_t J = 0; J < ValueCount; ++J) {
+      StringRef Name = readString(Ctx);
+      StringRef Version = readString(Ctx);
+      if (!ProducersSeen.insert(Name).second) {
+        return make_error<GenericBinaryError>(
+            "Producers section contains repeated producer",
+            object_error::parse_failed);
+      }
+      ProducerVec->emplace_back(Name, Version);
+    }
+  }
+  if (Ctx.Ptr != Ctx.End)
+    return make_error<GenericBinaryError>("Producers section ended prematurely",
+                                          object_error::parse_failed);
+  return Error::success();
+}
+
 Error WasmObjectFile::parseRelocSection(StringRef Name, ReadContext &Ctx) {
   uint32_t SectionIndex = readVaruint32(Ctx);
   if (SectionIndex >= Sections.size())
@@ -678,43 +733,43 @@ Error WasmObjectFile::parseRelocSection(StringRef Name, ReadContext &Ctx) {
     PreviousOffset = Reloc.Offset;
     Reloc.Index = readVaruint32(Ctx);
     switch (Reloc.Type) {
-    case wasm::R_WEBASSEMBLY_FUNCTION_INDEX_LEB:
-    case wasm::R_WEBASSEMBLY_TABLE_INDEX_SLEB:
-    case wasm::R_WEBASSEMBLY_TABLE_INDEX_I32:
+    case wasm::R_WASM_FUNCTION_INDEX_LEB:
+    case wasm::R_WASM_TABLE_INDEX_SLEB:
+    case wasm::R_WASM_TABLE_INDEX_I32:
       if (!isValidFunctionSymbol(Reloc.Index))
         return make_error<GenericBinaryError>("Bad relocation function index",
                                               object_error::parse_failed);
       break;
-    case wasm::R_WEBASSEMBLY_TYPE_INDEX_LEB:
+    case wasm::R_WASM_TYPE_INDEX_LEB:
       if (Reloc.Index >= Signatures.size())
         return make_error<GenericBinaryError>("Bad relocation type index",
                                               object_error::parse_failed);
       break;
-    case wasm::R_WEBASSEMBLY_GLOBAL_INDEX_LEB:
+    case wasm::R_WASM_GLOBAL_INDEX_LEB:
       if (!isValidGlobalSymbol(Reloc.Index))
         return make_error<GenericBinaryError>("Bad relocation global index",
                                               object_error::parse_failed);
       break;
-    case wasm::R_WEBASSEMBLY_EVENT_INDEX_LEB:
+    case wasm::R_WASM_EVENT_INDEX_LEB:
       if (!isValidEventSymbol(Reloc.Index))
         return make_error<GenericBinaryError>("Bad relocation event index",
                                               object_error::parse_failed);
       break;
-    case wasm::R_WEBASSEMBLY_MEMORY_ADDR_LEB:
-    case wasm::R_WEBASSEMBLY_MEMORY_ADDR_SLEB:
-    case wasm::R_WEBASSEMBLY_MEMORY_ADDR_I32:
+    case wasm::R_WASM_MEMORY_ADDR_LEB:
+    case wasm::R_WASM_MEMORY_ADDR_SLEB:
+    case wasm::R_WASM_MEMORY_ADDR_I32:
       if (!isValidDataSymbol(Reloc.Index))
         return make_error<GenericBinaryError>("Bad relocation data index",
                                               object_error::parse_failed);
       Reloc.Addend = readVarint32(Ctx);
       break;
-    case wasm::R_WEBASSEMBLY_FUNCTION_OFFSET_I32:
+    case wasm::R_WASM_FUNCTION_OFFSET_I32:
       if (!isValidFunctionSymbol(Reloc.Index))
         return make_error<GenericBinaryError>("Bad relocation function index",
                                               object_error::parse_failed);
       Reloc.Addend = readVarint32(Ctx);
       break;
-    case wasm::R_WEBASSEMBLY_SECTION_OFFSET_I32:
+    case wasm::R_WASM_SECTION_OFFSET_I32:
       if (!isValidSectionSymbol(Reloc.Index))
         return make_error<GenericBinaryError>("Bad relocation section index",
                                               object_error::parse_failed);
@@ -730,10 +785,10 @@ Error WasmObjectFile::parseRelocSection(StringRef Name, ReadContext &Ctx) {
     // also shouldn't overlap a function/element boundary, but we don't bother
     // to check that.
     uint64_t Size = 5;
-    if (Reloc.Type == wasm::R_WEBASSEMBLY_TABLE_INDEX_I32 ||
-        Reloc.Type == wasm::R_WEBASSEMBLY_MEMORY_ADDR_I32 ||
-        Reloc.Type == wasm::R_WEBASSEMBLY_SECTION_OFFSET_I32 ||
-        Reloc.Type == wasm::R_WEBASSEMBLY_FUNCTION_OFFSET_I32)
+    if (Reloc.Type == wasm::R_WASM_TABLE_INDEX_I32 ||
+        Reloc.Type == wasm::R_WASM_MEMORY_ADDR_I32 ||
+        Reloc.Type == wasm::R_WASM_SECTION_OFFSET_I32 ||
+        Reloc.Type == wasm::R_WASM_FUNCTION_OFFSET_I32)
       Size = 4;
     if (Reloc.Offset + Size > EndOffset)
       return make_error<GenericBinaryError>("Bad relocation offset",
@@ -757,6 +812,9 @@ Error WasmObjectFile::parseCustomSection(WasmSection &Sec, ReadContext &Ctx) {
   } else if (Sec.Name == "linking") {
     if (Error Err = parseLinkingSection(Ctx))
       return Err;
+  } else if (Sec.Name == "producers") {
+    if (Error Err = parseProducersSection(Ctx))
+      return Err;
   } else if (Sec.Name.startswith("reloc.")) {
     if (Error Err = parseRelocSection(Sec.Name, Ctx))
       return Err;
@@ -799,7 +857,7 @@ Error WasmObjectFile::parseTypeSection(ReadContext &Ctx) {
 Error WasmObjectFile::parseImportSection(ReadContext &Ctx) {
   uint32_t Count = readVaruint32(Ctx);
   Imports.reserve(Count);
-  for (uint32_t i = 0; i < Count; i++) {
+  for (uint32_t I = 0; I < Count; I++) {
     wasm::WasmImport Im;
     Im.Module = readString(Ctx);
     Im.Field = readString(Ctx);
@@ -819,7 +877,7 @@ Error WasmObjectFile::parseImportSection(ReadContext &Ctx) {
       break;
     case wasm::WASM_EXTERNAL_TABLE:
       Im.Table = readTable(Ctx);
-      if (Im.Table.ElemType != wasm::WASM_TYPE_ANYFUNC)
+      if (Im.Table.ElemType != wasm::WASM_TYPE_FUNCREF)
         return make_error<GenericBinaryError>("Invalid table element type",
                                               object_error::parse_failed);
       break;
@@ -862,7 +920,7 @@ Error WasmObjectFile::parseTableSection(ReadContext &Ctx) {
   Tables.reserve(Count);
   while (Count--) {
     Tables.push_back(readTable(Ctx));
-    if (Tables.back().ElemType != wasm::WASM_TYPE_ANYFUNC) {
+    if (Tables.back().ElemType != wasm::WASM_TYPE_FUNCREF) {
       return make_error<GenericBinaryError>("Invalid table element type",
                                             object_error::parse_failed);
     }
@@ -925,7 +983,7 @@ Error WasmObjectFile::parseEventSection(ReadContext &Ctx) {
 Error WasmObjectFile::parseExportSection(ReadContext &Ctx) {
   uint32_t Count = readVaruint32(Ctx);
   Exports.reserve(Count);
-  for (uint32_t i = 0; i < Count; i++) {
+  for (uint32_t I = 0; I < Count; I++) {
     wasm::WasmExport Ex;
     Ex.Name = readString(Ctx);
     Ex.Kind = readUint8(Ctx);
@@ -1010,6 +1068,12 @@ wasm::WasmFunction &WasmObjectFile::getDefinedFunction(uint32_t Index) {
   return Functions[Index - NumImportedFunctions];
 }
 
+const wasm::WasmFunction &
+WasmObjectFile::getDefinedFunction(uint32_t Index) const {
+  assert(isDefinedFunctionIndex(Index));
+  return Functions[Index - NumImportedFunctions];
+}
+
 wasm::WasmGlobal &WasmObjectFile::getDefinedGlobal(uint32_t Index) {
   assert(isDefinedGlobalIndex(Index));
   return Globals[Index - NumImportedGlobals];
@@ -1123,15 +1187,11 @@ Error WasmObjectFile::parseDataSection(ReadContext &Ctx) {
   return Error::success();
 }
 
-const uint8_t *WasmObjectFile::getPtr(size_t Offset) const {
-  return reinterpret_cast<const uint8_t *>(getData().data() + Offset);
-}
-
 const wasm::WasmObjectHeader &WasmObjectFile::getHeader() const {
   return Header;
 }
 
-void WasmObjectFile::moveSymbolNext(DataRefImpl &Symb) const { Symb.d.a++; }
+void WasmObjectFile::moveSymbolNext(DataRefImpl &Symb) const { Symb.d.b++; }
 
 uint32_t WasmObjectFile::getSymbolFlags(DataRefImpl Symb) const {
   uint32_t Result = SymbolRef::SF_None;
@@ -1153,18 +1213,20 @@ uint32_t WasmObjectFile::getSymbolFlags(DataRefImpl Symb) const {
 
 basic_symbol_iterator WasmObjectFile::symbol_begin() const {
   DataRefImpl Ref;
-  Ref.d.a = 0;
+  Ref.d.a = 1; // Arbitrary non-zero value so that Ref.p is non-null
+  Ref.d.b = 0; // Symbol index
   return BasicSymbolRef(Ref, this);
 }
 
 basic_symbol_iterator WasmObjectFile::symbol_end() const {
   DataRefImpl Ref;
-  Ref.d.a = Symbols.size();
+  Ref.d.a = 1; // Arbitrary non-zero value so that Ref.p is non-null
+  Ref.d.b = Symbols.size(); // Symbol index
   return BasicSymbolRef(Ref, this);
 }
 
 const WasmSymbol &WasmObjectFile::getWasmSymbol(const DataRefImpl &Symb) const {
-  return Symbols[Symb.d.a];
+  return Symbols[Symb.d.b];
 }
 
 const WasmSymbol &WasmObjectFile::getWasmSymbol(const SymbolRef &Symb) const {
@@ -1176,7 +1238,12 @@ Expected<StringRef> WasmObjectFile::getSymbolName(DataRefImpl Symb) const {
 }
 
 Expected<uint64_t> WasmObjectFile::getSymbolAddress(DataRefImpl Symb) const {
-  return getSymbolValue(Symb);
+  auto &Sym = getWasmSymbol(Symb);
+  if (Sym.Info.Kind == wasm::WASM_SYMBOL_TYPE_FUNCTION &&
+      isDefinedFunctionIndex(Sym.Info.ElementIndex))
+    return getDefinedFunction(Sym.Info.ElementIndex).CodeSectionOffset;
+  else
+    return getSymbolValue(Symb);
 }
 
 uint64_t WasmObjectFile::getWasmSymbolValue(const WasmSymbol &Sym) const {
@@ -1362,11 +1429,11 @@ uint64_t WasmObjectFile::getRelocationOffset(DataRefImpl Ref) const {
 
 symbol_iterator WasmObjectFile::getRelocationSymbol(DataRefImpl Ref) const {
   const wasm::WasmRelocation &Rel = getWasmRelocation(Ref);
-  if (Rel.Type == wasm::R_WEBASSEMBLY_TYPE_INDEX_LEB)
+  if (Rel.Type == wasm::R_WASM_TYPE_INDEX_LEB)
     return symbol_end();
   DataRefImpl Sym;
-  Sym.d.a = Rel.Index;
-  Sym.d.b = 0;
+  Sym.d.a = 1;
+  Sym.d.b = Rel.Index;
   return symbol_iterator(SymbolRef(Sym, this));
 }
 
diff --git a/lib/Object/WindowsResource.cpp b/lib/Object/WindowsResource.cpp
index 65413dd8bea178170721258532099723448cb4ab..53fe4d5e280d4f0291f6f4067dfd3c6ae8ba3488 100644
--- a/lib/Object/WindowsResource.cpp
+++ b/lib/Object/WindowsResource.cpp
@@ -1,9 +1,8 @@
 //===-- WindowsResource.cpp -------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/ObjectYAML/COFFYAML.cpp b/lib/ObjectYAML/COFFYAML.cpp
index 9351ef96beb2e0861e2d8c7e73a94568d58c51d5..f206e31b55079dd87df432edf134b885d4dbaa9d 100644
--- a/lib/ObjectYAML/COFFYAML.cpp
+++ b/lib/ObjectYAML/COFFYAML.cpp
@@ -1,9 +1,8 @@
 //===- COFFYAML.cpp - COFF YAMLIO implementation --------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -188,6 +187,7 @@ void ScalarEnumerationTraits<COFF::RelocationTypesARM>::enumeration(
   ECase(IMAGE_REL_ARM_TOKEN);
   ECase(IMAGE_REL_ARM_BLX24);
   ECase(IMAGE_REL_ARM_BLX11);
+  ECase(IMAGE_REL_ARM_REL32);
   ECase(IMAGE_REL_ARM_SECTION);
   ECase(IMAGE_REL_ARM_SECREL);
   ECase(IMAGE_REL_ARM_MOV32A);
@@ -195,6 +195,7 @@ void ScalarEnumerationTraits<COFF::RelocationTypesARM>::enumeration(
   ECase(IMAGE_REL_ARM_BRANCH20T);
   ECase(IMAGE_REL_ARM_BRANCH24T);
   ECase(IMAGE_REL_ARM_BLX23T);
+  ECase(IMAGE_REL_ARM_PAIR);
 }
 
 void ScalarEnumerationTraits<COFF::RelocationTypesARM64>::enumeration(
@@ -216,6 +217,7 @@ void ScalarEnumerationTraits<COFF::RelocationTypesARM64>::enumeration(
   ECase(IMAGE_REL_ARM64_ADDR64);
   ECase(IMAGE_REL_ARM64_BRANCH19);
   ECase(IMAGE_REL_ARM64_BRANCH14);
+  ECase(IMAGE_REL_ARM64_REL32);
 }
 
 void ScalarEnumerationTraits<COFF::WindowsSubsystem>::enumeration(
@@ -407,7 +409,8 @@ struct NDLLCharacteristics {
 void MappingTraits<COFFYAML::Relocation>::mapping(IO &IO,
                                                   COFFYAML::Relocation &Rel) {
   IO.mapRequired("VirtualAddress", Rel.VirtualAddress);
-  IO.mapRequired("SymbolName", Rel.SymbolName);
+  IO.mapOptional("SymbolName", Rel.SymbolName, StringRef());
+  IO.mapOptional("SymbolTableIndex", Rel.SymbolTableIndex);
 
   COFF::header &H = *static_cast<COFF::header *>(IO.getContext());
   if (H.Machine == COFF::IMAGE_FILE_MACHINE_I386) {
diff --git a/lib/ObjectYAML/CodeViewYAMLDebugSections.cpp b/lib/ObjectYAML/CodeViewYAMLDebugSections.cpp
index 4deeae878013aca2e8446a32633d46e009b052ae..eeebb694589b28d34a030c8d5da02a3ebac8d18f 100644
--- a/lib/ObjectYAML/CodeViewYAMLDebugSections.cpp
+++ b/lib/ObjectYAML/CodeViewYAMLDebugSections.cpp
@@ -1,9 +1,8 @@
 //===- CodeViewYAMLDebugSections.cpp - CodeView YAMLIO debug sections -----===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/ObjectYAML/CodeViewYAMLSymbols.cpp b/lib/ObjectYAML/CodeViewYAMLSymbols.cpp
index 713e9a710e945a88fc14a73e3f04530bdf10574b..d6c9f10de014286f558d2d336185f2f2456708ba 100644
--- a/lib/ObjectYAML/CodeViewYAMLSymbols.cpp
+++ b/lib/ObjectYAML/CodeViewYAMLSymbols.cpp
@@ -1,9 +1,8 @@
 //===- CodeViewYAMLSymbols.cpp - CodeView YAMLIO Symbol implementation ----===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/ObjectYAML/CodeViewYAMLTypeHashing.cpp b/lib/ObjectYAML/CodeViewYAMLTypeHashing.cpp
index ed117059560fd2db5fc116c0ce14e895d0243842..e921ae1e7d8d37bb8ad983c6dae80829a2c9c9ff 100644
--- a/lib/ObjectYAML/CodeViewYAMLTypeHashing.cpp
+++ b/lib/ObjectYAML/CodeViewYAMLTypeHashing.cpp
@@ -1,9 +1,8 @@
 //===- CodeViewYAMLTypeHashing.cpp - CodeView YAMLIO type hashing ---------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/ObjectYAML/CodeViewYAMLTypes.cpp b/lib/ObjectYAML/CodeViewYAMLTypes.cpp
index 791b115dc4929b7b7027363d28862dbddb42065c..4d37cbb8455e8d93eca0bb9f42fc938ad288a9c6 100644
--- a/lib/ObjectYAML/CodeViewYAMLTypes.cpp
+++ b/lib/ObjectYAML/CodeViewYAMLTypes.cpp
@@ -1,9 +1,8 @@
 //===- CodeViewYAMLTypes.cpp - CodeView YAMLIO types implementation -------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/ObjectYAML/DWARFEmitter.cpp b/lib/ObjectYAML/DWARFEmitter.cpp
index f23fa12376004fac812475b44a34bb5ba60e445d..2ae66997cf597d0775bb23666f37ecabc32dae9a 100644
--- a/lib/ObjectYAML/DWARFEmitter.cpp
+++ b/lib/ObjectYAML/DWARFEmitter.cpp
@@ -1,9 +1,8 @@
 //===- DWARFEmitter - Convert YAML to DWARF binary data -------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
diff --git a/lib/ObjectYAML/DWARFVisitor.cpp b/lib/ObjectYAML/DWARFVisitor.cpp
index e6114c85ac0b9ac5a7797ce3003b94d8aa0168bd..ecb5967ac532d768428a0bf3db9d12c66e3d1e94 100644
--- a/lib/ObjectYAML/DWARFVisitor.cpp
+++ b/lib/ObjectYAML/DWARFVisitor.cpp
@@ -1,9 +1,8 @@
 //===--- DWARFVisitor.cpp ---------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/ObjectYAML/DWARFVisitor.h b/lib/ObjectYAML/DWARFVisitor.h
index 5489031dc33196e28faa6d1d25c8a03879ba3f39..50e88aa7a26bf137d71360608c0b40e86dc8e376 100644
--- a/lib/ObjectYAML/DWARFVisitor.h
+++ b/lib/ObjectYAML/DWARFVisitor.h
@@ -1,9 +1,8 @@
 //===--- DWARFVisitor.h -----------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/ObjectYAML/DWARFYAML.cpp b/lib/ObjectYAML/DWARFYAML.cpp
index d6c09e1a35d733186c50a0c45498f8aa2eca0b50..bb3b1422eb62d3551b4d650b24740f107e9d66a4 100644
--- a/lib/ObjectYAML/DWARFYAML.cpp
+++ b/lib/ObjectYAML/DWARFYAML.cpp
@@ -1,9 +1,8 @@
 //===- DWARFYAML.cpp - DWARF YAMLIO implementation ------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/ObjectYAML/ELFYAML.cpp b/lib/ObjectYAML/ELFYAML.cpp
index b9f52432fc802b3accf8ed864c6dd17908e9e31b..cec56771e1d8840154768abb97c43445cee0faf9 100644
--- a/lib/ObjectYAML/ELFYAML.cpp
+++ b/lib/ObjectYAML/ELFYAML.cpp
@@ -1,9 +1,8 @@
 //===- ELFYAML.cpp - ELF YAMLIO implementation ----------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -573,6 +572,7 @@ void ScalarEnumerationTraits<ELFYAML::ELF_STT>::enumeration(
   ECase(STT_TLS);
   ECase(STT_GNU_IFUNC);
 #undef ECase
+  IO.enumFallback<Hex8>(Value);
 }
 
 void ScalarEnumerationTraits<ELFYAML::ELF_STV>::enumeration(
@@ -661,6 +661,32 @@ void ScalarEnumerationTraits<ELFYAML::ELF_REL>::enumeration(
   IO.enumFallback<Hex32>(Value);
 }
 
+void ScalarEnumerationTraits<ELFYAML::ELF_DYNTAG>::enumeration(
+    IO &IO, ELFYAML::ELF_DYNTAG &Value) {
+  assert(IO.getContext() && "The IO context is not initialized");
+
+// TODO: For simplicity we do not handle target specific flags. They are
+// still supported and will be shown as a raw numeric values in the output.
+#define MIPS_DYNAMIC_TAG(name, value)
+#define HEXAGON_DYNAMIC_TAG(name, value)
+#define PPC64_DYNAMIC_TAG(name, value)
+// Also ignore marker tags such as DT_HIOS (maps to DT_VERNEEDNUM), etc.
+#define DYNAMIC_TAG_MARKER(name, value)
+
+#define STRINGIFY(X) (#X)
+#define DYNAMIC_TAG(X, Y) IO.enumCase(Value, STRINGIFY(DT_##X), ELF::DT_##X);
+#include "llvm/BinaryFormat/DynamicTags.def"
+
+#undef MIPS_DYNAMIC_TAG
+#undef HEXAGON_DYNAMIC_TAG
+#undef PPC64_DYNAMIC_TAG
+#undef DYNAMIC_TAG_MARKER
+#undef STRINGIFY
+#undef DYNAMIC_TAG
+
+  IO.enumFallback<Hex64>(Value);
+}
+
 void ScalarEnumerationTraits<ELFYAML::MIPS_AFL_REG>::enumeration(
     IO &IO, ELFYAML::MIPS_AFL_REG &Value) {
 #define ECase(X) IO.enumCase(Value, #X, Mips::AFL_##X)
@@ -753,6 +779,7 @@ void MappingTraits<ELFYAML::FileHeader>::mapping(IO &IO,
   IO.mapRequired("Class", FileHdr.Class);
   IO.mapRequired("Data", FileHdr.Data);
   IO.mapOptional("OSABI", FileHdr.OSABI, ELFYAML::ELF_ELFOSABI(0));
+  IO.mapOptional("ABIVersion", FileHdr.ABIVersion, Hex8(0));
   IO.mapRequired("Type", FileHdr.Type);
   IO.mapRequired("Machine", FileHdr.Machine);
   IO.mapOptional("Flags", FileHdr.Flags, ELFYAML::ELF_EF(0));
@@ -827,7 +854,11 @@ static void commonSectionMapping(IO &IO, ELFYAML::Section &Section) {
   IO.mapOptional("Link", Section.Link, StringRef());
   IO.mapOptional("AddressAlign", Section.AddressAlign, Hex64(0));
   IO.mapOptional("EntSize", Section.EntSize);
-  IO.mapOptional("Info", Section.Info, StringRef());
+}
+
+static void sectionMapping(IO &IO, ELFYAML::DynamicSection &Section) {
+  commonSectionMapping(IO, Section);
+  IO.mapOptional("Entries", Section.Entries);
 }
 
 static void sectionMapping(IO &IO, ELFYAML::RawContentSection &Section) {
@@ -843,12 +874,14 @@ static void sectionMapping(IO &IO, ELFYAML::NoBitsSection &Section) {
 
 static void sectionMapping(IO &IO, ELFYAML::RelocationSection &Section) {
   commonSectionMapping(IO, Section);
+  IO.mapOptional("Info", Section.RelocatableSec, StringRef());
   IO.mapOptional("Relocations", Section.Relocations);
 }
 
-static void groupSectionMapping(IO &IO, ELFYAML::Group &group) {
-  commonSectionMapping(IO, group);
-  IO.mapRequired("Members", group.Members);
+static void groupSectionMapping(IO &IO, ELFYAML::Group &Group) {
+  commonSectionMapping(IO, Group);
+  IO.mapOptional("Info", Group.Signature, StringRef());
+  IO.mapRequired("Members", Group.Members);
 }
 
 void MappingTraits<ELFYAML::SectionOrType>::mapping(
@@ -890,6 +923,11 @@ void MappingTraits<std::unique_ptr<ELFYAML::Section>>::mapping(
     IO.mapRequired("Type", sectionType);
 
   switch (sectionType) {
+  case ELF::SHT_DYNAMIC:
+    if (!IO.outputting())
+      Section.reset(new ELFYAML::DynamicSection());
+    sectionMapping(IO, *cast<ELFYAML::DynamicSection>(Section.get()));
+    break;
   case ELF::SHT_REL:
   case ELF::SHT_RELA:
     if (!IO.outputting())
@@ -951,6 +989,14 @@ struct NormalizedMips64RelType {
 
 } // end anonymous namespace
 
+void MappingTraits<ELFYAML::DynamicEntry>::mapping(IO &IO,
+                                                   ELFYAML::DynamicEntry &Rel) {
+  assert(IO.getContext() && "The IO context is not initialized");
+
+  IO.mapRequired("Tag", Rel.Tag);
+  IO.mapRequired("Value", Rel.Val);
+}
+
 void MappingTraits<ELFYAML::Relocation>::mapping(IO &IO,
                                                  ELFYAML::Relocation &Rel) {
   const auto *Object = static_cast<ELFYAML::Object *>(IO.getContext());
diff --git a/lib/ObjectYAML/LLVMBuild.txt b/lib/ObjectYAML/LLVMBuild.txt
index 44657e916a9151629770dca6ec673ddf58f12d47..66a508d334f36ab1fd57c88a1d21ffde344d4e52 100644
--- a/lib/ObjectYAML/LLVMBuild.txt
+++ b/lib/ObjectYAML/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===------------------------------------------------------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 
diff --git a/lib/ObjectYAML/MachOYAML.cpp b/lib/ObjectYAML/MachOYAML.cpp
index e00a4ea93074ae43ce5f013ff75e9116f3460379..d12f12cf44350b2b8b457a085dd9a34573a04ac6 100644
--- a/lib/ObjectYAML/MachOYAML.cpp
+++ b/lib/ObjectYAML/MachOYAML.cpp
@@ -1,9 +1,8 @@
 //===- MachOYAML.cpp - MachO YAMLIO implementation ------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/ObjectYAML/ObjectYAML.cpp b/lib/ObjectYAML/ObjectYAML.cpp
index 850c1a5a06c025a2d94654c7fb72347d7500cc89..42353972103f6ad092097c0fa45e5fb9119964b3 100644
--- a/lib/ObjectYAML/ObjectYAML.cpp
+++ b/lib/ObjectYAML/ObjectYAML.cpp
@@ -1,9 +1,8 @@
 //===- ObjectYAML.cpp - YAML utilities for object files -------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/ObjectYAML/WasmYAML.cpp b/lib/ObjectYAML/WasmYAML.cpp
index b97803340c0d9e4cf07fbb353f7629304db1ad13..def4d58a1016428b70c9137b1941e6a2a545a257 100644
--- a/lib/ObjectYAML/WasmYAML.cpp
+++ b/lib/ObjectYAML/WasmYAML.cpp
@@ -1,9 +1,8 @@
 //===- WasmYAML.cpp - Wasm YAMLIO implementation --------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -74,6 +73,14 @@ static void sectionMapping(IO &IO, WasmYAML::LinkingSection &Section) {
   IO.mapOptional("Comdats", Section.Comdats);
 }
 
+static void sectionMapping(IO &IO, WasmYAML::ProducersSection &Section) {
+  commonSectionMapping(IO, Section);
+  IO.mapRequired("Name", Section.Name);
+  IO.mapOptional("Languages", Section.Languages);
+  IO.mapOptional("Tools", Section.Tools);
+  IO.mapOptional("SDKs", Section.SDKs);
+}
+
 static void sectionMapping(IO &IO, WasmYAML::CustomSection &Section) {
   commonSectionMapping(IO, Section);
   IO.mapRequired("Name", Section.Name);
@@ -169,6 +176,10 @@ void MappingTraits<std::unique_ptr<WasmYAML::Section>>::mapping(
       if (!IO.outputting())
         Section.reset(new WasmYAML::NameSection());
       sectionMapping(IO, *cast<WasmYAML::NameSection>(Section.get()));
+    } else if (SectionName == "producers") {
+      if (!IO.outputting())
+        Section.reset(new WasmYAML::ProducersSection());
+      sectionMapping(IO, *cast<WasmYAML::ProducersSection>(Section.get()));
     } else {
       if (!IO.outputting())
         Section.reset(new WasmYAML::CustomSection(SectionName));
@@ -293,6 +304,12 @@ void MappingTraits<WasmYAML::NameEntry>::mapping(
   IO.mapRequired("Name", NameEntry.Name);
 }
 
+void MappingTraits<WasmYAML::ProducerEntry>::mapping(
+    IO &IO, WasmYAML::ProducerEntry &ProducerEntry) {
+  IO.mapRequired("Name", ProducerEntry.Name);
+  IO.mapRequired("Version", ProducerEntry.Version);
+}
+
 void MappingTraits<WasmYAML::SegmentInfo>::mapping(
     IO &IO, WasmYAML::SegmentInfo &SegmentInfo) {
   IO.mapRequired("Index", SegmentInfo.Index);
@@ -377,7 +394,7 @@ void MappingTraits<wasm::WasmInitExpr>::mapping(IO &IO,
   case wasm::WASM_OPCODE_F64_CONST:
     IO.mapRequired("Value", Expr.Value.Float64);
     break;
-  case wasm::WASM_OPCODE_GET_GLOBAL:
+  case wasm::WASM_OPCODE_GLOBAL_GET:
     IO.mapRequired("Index", Expr.Value.Global);
     break;
   }
@@ -469,6 +486,7 @@ void ScalarBitSetTraits<WasmYAML::SymbolFlags>::bitset(
   // BCaseMask(VISIBILITY_MASK, VISIBILITY_DEFAULT);
   BCaseMask(VISIBILITY_MASK, VISIBILITY_HIDDEN);
   BCaseMask(UNDEFINED, UNDEFINED);
+  BCaseMask(EXPORTED, EXPORTED);
 #undef BCaseMask
 }
 
@@ -491,7 +509,7 @@ void ScalarEnumerationTraits<WasmYAML::ValueType>::enumeration(
   ECase(F32);
   ECase(F64);
   ECase(V128);
-  ECase(ANYFUNC);
+  ECase(FUNCREF);
   ECase(FUNC);
   ECase(NORESULT);
 #undef ECase
@@ -516,14 +534,14 @@ void ScalarEnumerationTraits<WasmYAML::Opcode>::enumeration(
   ECase(I64_CONST);
   ECase(F64_CONST);
   ECase(F32_CONST);
-  ECase(GET_GLOBAL);
+  ECase(GLOBAL_GET);
 #undef ECase
 }
 
 void ScalarEnumerationTraits<WasmYAML::TableType>::enumeration(
     IO &IO, WasmYAML::TableType &Type) {
 #define ECase(X) IO.enumCase(Type, #X, wasm::WASM_TYPE_##X);
-  ECase(ANYFUNC);
+  ECase(FUNCREF);
 #undef ECase
 }
 
diff --git a/lib/ObjectYAML/YAML.cpp b/lib/ObjectYAML/YAML.cpp
index 67b5764eadaa42ba6ce460b72da978a2b3f4fa17..6eba16e36c2a21627f2d3ae9139b0791d541d3a0 100644
--- a/lib/ObjectYAML/YAML.cpp
+++ b/lib/ObjectYAML/YAML.cpp
@@ -1,9 +1,8 @@
 //===- YAML.cpp - YAMLIO utilities for object files -----------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -32,7 +31,7 @@ StringRef yaml::ScalarTraits<yaml::BinaryRef>::input(StringRef Scalar, void *,
   // TODO: Can we improve YAMLIO to permit a more accurate diagnostic here?
   // (e.g. a caret pointing to the offending character).
   for (unsigned I = 0, N = Scalar.size(); I != N; ++I)
-    if (!isxdigit(Scalar[I]))
+    if (!llvm::isHexDigit(Scalar[I]))
       return "BinaryRef hex string must contain only hex digits.";
   Val = yaml::BinaryRef(Scalar);
   return {};
@@ -44,8 +43,9 @@ void yaml::BinaryRef::writeAsBinary(raw_ostream &OS) const {
     return;
   }
   for (unsigned I = 0, N = Data.size(); I != N; I += 2) {
-    uint8_t Byte;
-    StringRef((const char *)&Data[I],  2).getAsInteger(16, Byte);
+    uint8_t Byte = llvm::hexDigitValue(Data[I]);
+    Byte <<= 4;
+    Byte |= llvm::hexDigitValue(Data[I + 1]);
     OS.write(Byte);
   }
 }
diff --git a/lib/OptRemarks/LLVMBuild.txt b/lib/OptRemarks/LLVMBuild.txt
index 4c1032296dcb7f0a82ea79f39c99e32fb6066739..a2962690d75cbecc144b0504128ba13953269161 100644
--- a/lib/OptRemarks/LLVMBuild.txt
+++ b/lib/OptRemarks/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/OptRemarks/LLVMBuild.txt ---------------------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/OptRemarks/OptRemarksParser.cpp b/lib/OptRemarks/OptRemarksParser.cpp
index 0478d2bfbfa600521bc9385cc79d614638c748c0..18c0df27993ba219690e54eed670e6e7ce458538 100644
--- a/lib/OptRemarks/OptRemarksParser.cpp
+++ b/lib/OptRemarks/OptRemarksParser.cpp
@@ -1,9 +1,8 @@
 //===- OptRemarksParser.cpp -----------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is dual licensed under the MIT and the University of Illinois Open
-// Source Licenses. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Option/Arg.cpp b/lib/Option/Arg.cpp
index 4ce40e3ab26c6128f1dd3b5733cc9f826c58d9a2..420279b0bcac6c79e7195add636f5d5da94cf27e 100644
--- a/lib/Option/Arg.cpp
+++ b/lib/Option/Arg.cpp
@@ -1,9 +1,8 @@
 //===- Arg.cpp - Argument Implementations ---------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Option/ArgList.cpp b/lib/Option/ArgList.cpp
index 8a7d59d2436600e7f96cfc327432fde6477e6a8e..36e2f45fe391d0e0929909364b2051deb3765e1a 100644
--- a/lib/Option/ArgList.cpp
+++ b/lib/Option/ArgList.cpp
@@ -1,9 +1,8 @@
 //===- ArgList.cpp - Argument List Management -----------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Option/LLVMBuild.txt b/lib/Option/LLVMBuild.txt
index 0b78cf20c05fe53e4aa4d99efe8f6fe331488112..3443492e661b23eaba323750e275027cb5608a93 100644
--- a/lib/Option/LLVMBuild.txt
+++ b/lib/Option/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/Option/LLVMBuild.txt -------------------------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Option/OptTable.cpp b/lib/Option/OptTable.cpp
index 312ff78087590dc5f8de465bfa8d11511e9dc6b8..c0a1861a8d539bb69a1d1204bfe7b3d86e48835f 100644
--- a/lib/Option/OptTable.cpp
+++ b/lib/Option/OptTable.cpp
@@ -1,9 +1,8 @@
 //===- OptTable.cpp - Option Table Implementation -------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Option/Option.cpp b/lib/Option/Option.cpp
index f9d8a5e54043f7bb3ddcea1c5d2c372e4a252799..1e3b5452e3cf56444fd7274fa0e8669648cb9ef9 100644
--- a/lib/Option/Option.cpp
+++ b/lib/Option/Option.cpp
@@ -1,9 +1,8 @@
 //===- Option.cpp - Abstract Driver Options -------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Passes/LLVMBuild.txt b/lib/Passes/LLVMBuild.txt
index d6e9bf1fcf3959736feb64909fb1079de958db29..c310e2b8b20904b2b7e07dcaed05ef22a26feed9 100644
--- a/lib/Passes/LLVMBuild.txt
+++ b/lib/Passes/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/Passes/LLVMBuild.txt -------------------------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Passes/PassBuilder.cpp b/lib/Passes/PassBuilder.cpp
index 4da8f549d9bf9c5d6c7fa8fbaee4606255e6bcd2..8e62c1d0b690a4d6855e59e9533ba2986a81b352 100644
--- a/lib/Passes/PassBuilder.cpp
+++ b/lib/Passes/PassBuilder.cpp
@@ -1,9 +1,8 @@
 //===- Parsing, selection, and construction of pass pipelines -------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
@@ -88,11 +87,14 @@
 #include "llvm/Transforms/IPO/SyntheticCountsPropagation.h"
 #include "llvm/Transforms/IPO/WholeProgramDevirt.h"
 #include "llvm/Transforms/InstCombine/InstCombine.h"
+#include "llvm/Transforms/Instrumentation.h"
 #include "llvm/Transforms/Instrumentation/BoundsChecking.h"
 #include "llvm/Transforms/Instrumentation/CGProfile.h"
 #include "llvm/Transforms/Instrumentation/ControlHeightReduction.h"
 #include "llvm/Transforms/Instrumentation/GCOVProfiler.h"
 #include "llvm/Transforms/Instrumentation/InstrProfiling.h"
+#include "llvm/Transforms/Instrumentation/MemorySanitizer.h"
+#include "llvm/Transforms/Instrumentation/ThreadSanitizer.h"
 #include "llvm/Transforms/Instrumentation/PGOInstrumentation.h"
 #include "llvm/Transforms/Scalar/ADCE.h"
 #include "llvm/Transforms/Scalar/AlignmentFromAssumptions.h"
@@ -131,6 +133,7 @@
 #include "llvm/Transforms/Scalar/LowerAtomic.h"
 #include "llvm/Transforms/Scalar/LowerExpectIntrinsic.h"
 #include "llvm/Transforms/Scalar/LowerGuardIntrinsic.h"
+#include "llvm/Transforms/Scalar/LowerWidenableCondition.h"
 #include "llvm/Transforms/Scalar/MakeGuardsExplicit.h"
 #include "llvm/Transforms/Scalar/MemCpyOptimizer.h"
 #include "llvm/Transforms/Scalar/MergedLoadStoreMotion.h"
@@ -151,6 +154,7 @@
 #include "llvm/Transforms/Scalar/WarnMissedTransforms.h"
 #include "llvm/Transforms/Utils/AddDiscriminators.h"
 #include "llvm/Transforms/Utils/BreakCriticalEdges.h"
+#include "llvm/Transforms/Utils/CanonicalizeAliases.h"
 #include "llvm/Transforms/Utils/EntryExitInstrumenter.h"
 #include "llvm/Transforms/Utils/LCSSA.h"
 #include "llvm/Transforms/Utils/LibCallsShrinkWrap.h"
@@ -207,6 +211,8 @@ static cl::opt<bool>
 
 extern cl::opt<bool> EnableHotColdSplit;
 
+extern cl::opt<bool> FlattenedProfileUsed;
+
 static bool isOptimizingForSize(PassBuilder::OptimizationLevel Level) {
   switch (Level) {
   case PassBuilder::O0:
@@ -583,6 +589,32 @@ PassBuilder::buildModuleSimplificationPipeline(OptimizationLevel Level,
                                                bool DebugLogging) {
   ModulePassManager MPM(DebugLogging);
 
+  bool HasSampleProfile = PGOOpt && !PGOOpt->SampleProfileFile.empty();
+
+  // In ThinLTO mode, when flattened profile is used, all the available
+  // profile information will be annotated in PreLink phase so there is
+  // no need to load the profile again in PostLink.
+  bool LoadSampleProfile =
+      HasSampleProfile &&
+      !(FlattenedProfileUsed && Phase == ThinLTOPhase::PostLink);
+
+  // During the ThinLTO backend phase we perform early indirect call promotion
+  // here, before globalopt. Otherwise imported available_externally functions
+  // look unreferenced and are removed. If we are going to load the sample
+  // profile then defer until later.
+  // TODO: See if we can move later and consolidate with the location where
+  // we perform ICP when we are loading a sample profile.
+  // TODO: We pass HasSampleProfile (whether there was a sample profile file
+  // passed to the compile) to the SamplePGO flag of ICP. This is used to
+  // determine whether the new direct calls are annotated with prof metadata.
+  // Ideally this should be determined from whether the IR is annotated with
+  // sample profile, and not whether the a sample profile was provided on the
+  // command line. E.g. for flattened profiles where we will not be reloading
+  // the sample profile in the ThinLTO backend, we ideally shouldn't have to
+  // provide the sample profile file.
+  if (Phase == ThinLTOPhase::PostLink && !LoadSampleProfile)
+    MPM.addPass(PGOIndirectCallPromotion(true /* InLTO */, HasSampleProfile));
+
   // Do basic inference of function attributes from known properties of system
   // libraries and other oracles.
   MPM.addPass(InferFunctionAttrsPass());
@@ -603,12 +635,11 @@ PassBuilder::buildModuleSimplificationPipeline(OptimizationLevel Level,
   // More details about SamplePGO design can be found in:
   // https://research.google.com/pubs/pub45290.html
   // FIXME: revisit how SampleProfileLoad/Inliner/ICP is structured.
-  if (PGOOpt && !PGOOpt->SampleProfileFile.empty() &&
-      Phase == ThinLTOPhase::PostLink)
+  if (LoadSampleProfile)
     EarlyFPM.addPass(InstCombinePass());
   MPM.addPass(createModuleToFunctionPassAdaptor(std::move(EarlyFPM)));
 
-  if (PGOOpt && !PGOOpt->SampleProfileFile.empty()) {
+  if (LoadSampleProfile) {
     // Annotate sample profile right after early FPM to ensure freshness of
     // the debug info.
     MPM.addPass(SampleProfileLoaderPass(PGOOpt->SampleProfileFile,
@@ -622,7 +653,7 @@ PassBuilder::buildModuleSimplificationPipeline(OptimizationLevel Level,
       // imported available_externally functions look unreferenced and are
       // removed.
       MPM.addPass(PGOIndirectCallPromotion(Phase == ThinLTOPhase::PostLink,
-                                           true));
+                                           true /* SamplePGO */));
   }
 
   // Interprocedural constant propagation now that basic cleanup has occurred
@@ -671,6 +702,14 @@ PassBuilder::buildModuleSimplificationPipeline(OptimizationLevel Level,
   if (EnableSyntheticCounts && !PGOOpt)
     MPM.addPass(SyntheticCountsPropagation());
 
+  // Split out cold code. Splitting is done before inlining because 1) the most
+  // common kinds of cold regions can (a) be found before inlining and (b) do
+  // not grow after inlining, and 2) inhibiting inlining of cold code improves
+  // code size & compile time. Split after Mem2Reg to make code model estimates
+  // more accurate, but before InstCombine to allow it to clean things up.
+  if (EnableHotColdSplit && Phase != ThinLTOPhase::PostLink)
+    MPM.addPass(HotColdSplittingPass());
+
   // Require the GlobalsAA analysis for the module so we can query it within
   // the CGSCC pipeline.
   MPM.addPass(RequireAnalysisPass<GlobalsAA, Module>());
@@ -714,11 +753,6 @@ PassBuilder::buildModuleSimplificationPipeline(OptimizationLevel Level,
   MainCGPipeline.addPass(createCGSCCToFunctionPassAdaptor(
       buildFunctionSimplificationPipeline(Level, Phase, DebugLogging)));
 
-  // We only want to do hot cold splitting once for ThinLTO, during the
-  // post-link ThinLTO.
-  if (EnableHotColdSplit && Phase != ThinLTOPhase::PreLink)
-    MPM.addPass(HotColdSplittingPass());
-
   for (auto &C : CGSCCOptimizerLateEPCallbacks)
     C(MainCGPipeline, Level);
 
@@ -977,15 +1011,6 @@ ModulePassManager PassBuilder::buildThinLTODefaultPipeline(
   // Force any function attributes we want the rest of the pipeline to observe.
   MPM.addPass(ForceFunctionAttrsPass());
 
-  // During the ThinLTO backend phase we perform early indirect call promotion
-  // here, before globalopt. Otherwise imported available_externally functions
-  // look unreferenced and are removed.
-  // FIXME: move this into buildModuleSimplificationPipeline to merge the logic
-  //        with SamplePGO.
-  if (!PGOOpt || PGOOpt->SampleProfileFile.empty())
-    MPM.addPass(PGOIndirectCallPromotion(true /* InLTO */,
-                                         false /* SamplePGO */));
-
   // Add the core simplification pipeline.
   MPM.addPass(buildModuleSimplificationPipeline(Level, ThinLTOPhase::PostLink,
                                                 DebugLogging));
@@ -1239,6 +1264,119 @@ static Optional<int> parseDevirtPassName(StringRef Name) {
   return Count;
 }
 
+static bool checkParametrizedPassName(StringRef Name, StringRef PassName) {
+  if (!Name.consume_front(PassName))
+    return false;
+  // normal pass name w/o parameters == default parameters
+  if (Name.empty())
+    return true;
+  return Name.startswith("<") && Name.endswith(">");
+}
+
+namespace {
+
+/// This performs customized parsing of pass name with parameters.
+///
+/// We do not need parametrization of passes in textual pipeline very often,
+/// yet on a rare occasion ability to specify parameters right there can be
+/// useful.
+///
+/// \p Name - parameterized specification of a pass from a textual pipeline
+/// is a string in a form of :
+///      PassName '<' parameter-list '>'
+///
+/// Parameter list is being parsed by the parser callable argument, \p Parser,
+/// It takes a string-ref of parameters and returns either StringError or a
+/// parameter list in a form of a custom parameters type, all wrapped into
+/// Expected<> template class.
+///
+template <typename ParametersParseCallableT>
+auto parsePassParameters(ParametersParseCallableT &&Parser, StringRef Name,
+                         StringRef PassName) -> decltype(Parser(StringRef{})) {
+  using ParametersT = typename decltype(Parser(StringRef{}))::value_type;
+
+  StringRef Params = Name;
+  if (!Params.consume_front(PassName)) {
+    assert(false &&
+           "unable to strip pass name from parametrized pass specification");
+  }
+  if (Params.empty())
+    return ParametersT{};
+  if (!Params.consume_front("<") || !Params.consume_back(">")) {
+    assert(false && "invalid format for parametrized pass name");
+  }
+
+  Expected<ParametersT> Result = Parser(Params);
+  assert((Result || Result.template errorIsA<StringError>()) &&
+         "Pass parameter parser can only return StringErrors.");
+  return std::move(Result);
+}
+
+/// Parser of parameters for LoopUnroll pass.
+Expected<LoopUnrollOptions> parseLoopUnrollOptions(StringRef Params) {
+  LoopUnrollOptions UnrollOpts;
+  while (!Params.empty()) {
+    StringRef ParamName;
+    std::tie(ParamName, Params) = Params.split(';');
+    int OptLevel = StringSwitch<int>(ParamName)
+                       .Case("O0", 0)
+                       .Case("O1", 1)
+                       .Case("O2", 2)
+                       .Case("O3", 3)
+                       .Default(-1);
+    if (OptLevel >= 0) {
+      UnrollOpts.setOptLevel(OptLevel);
+      continue;
+    }
+
+    bool Enable = !ParamName.consume_front("no-");
+    if (ParamName == "partial") {
+      UnrollOpts.setPartial(Enable);
+    } else if (ParamName == "peeling") {
+      UnrollOpts.setPeeling(Enable);
+    } else if (ParamName == "runtime") {
+      UnrollOpts.setRuntime(Enable);
+    } else if (ParamName == "upperbound") {
+      UnrollOpts.setUpperBound(Enable);
+    } else {
+      return make_error<StringError>(
+          formatv("invalid LoopUnrollPass parameter '{0}' ", ParamName).str(),
+          inconvertibleErrorCode());
+    }
+  }
+  return UnrollOpts;
+}
+
+Expected<MemorySanitizerOptions> parseMSanPassOptions(StringRef Params) {
+  MemorySanitizerOptions Result;
+  while (!Params.empty()) {
+    StringRef ParamName;
+    std::tie(ParamName, Params) = Params.split(';');
+
+    if (ParamName == "recover") {
+      Result.Recover = true;
+    } else if (ParamName == "kernel") {
+      Result.Kernel = true;
+    } else if (ParamName.consume_front("track-origins=")) {
+      if (ParamName.getAsInteger(0, Result.TrackOrigins))
+        return make_error<StringError>(
+            formatv("invalid argument to MemorySanitizer pass track-origins "
+                    "parameter: '{0}' ",
+                    ParamName)
+                .str(),
+            inconvertibleErrorCode());
+    } else {
+      return make_error<StringError>(
+          formatv("invalid MemorySanitizer pass parameter '{0}' ", ParamName)
+              .str(),
+          inconvertibleErrorCode());
+    }
+  }
+  return Result;
+}
+
+} // namespace
+
 /// Tests whether a pass name starts with a valid prefix for a default pipeline
 /// alias.
 static bool startsWithDefaultPipelineAliasPrefix(StringRef Name) {
@@ -1334,6 +1472,9 @@ static bool isFunctionPassName(StringRef Name, CallbacksT &Callbacks) {
 #define FUNCTION_PASS(NAME, CREATE_PASS)                                       \
   if (Name == NAME)                                                            \
     return true;
+#define FUNCTION_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER)                   \
+  if (checkParametrizedPassName(Name, NAME))                                   \
+    return true;
 #define FUNCTION_ANALYSIS(NAME, CREATE_PASS)                                   \
   if (Name == "require<" NAME ">" || Name == "invalidate<" NAME ">")           \
     return true;
@@ -1671,6 +1812,14 @@ Error PassBuilder::parseFunctionPass(FunctionPassManager &FPM,
     FPM.addPass(CREATE_PASS);                                                  \
     return Error::success();                                                   \
   }
+#define FUNCTION_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER)                   \
+  if (checkParametrizedPassName(Name, NAME)) {                                 \
+    auto Params = parsePassParameters(PARSER, Name, NAME);                     \
+    if (!Params)                                                               \
+      return Params.takeError();                                               \
+    FPM.addPass(CREATE_PASS(Params.get()));                                    \
+    return Error::success();                                                   \
+  }
 #define FUNCTION_ANALYSIS(NAME, CREATE_PASS)                                   \
   if (Name == "require<" NAME ">") {                                           \
     FPM.addPass(                                                               \
diff --git a/lib/Passes/PassPlugin.cpp b/lib/Passes/PassPlugin.cpp
index bf38fdb842e7f70f931b20f81caf082b216094b5..ceefa25a703b6e38d22d3b8c6d5586c1bbd990a4 100644
--- a/lib/Passes/PassPlugin.cpp
+++ b/lib/Passes/PassPlugin.cpp
@@ -1,9 +1,8 @@
 //===- lib/Passes/PassPluginLoader.cpp - Load Plugins for New PM Passes ---===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Passes/PassRegistry.def b/lib/Passes/PassRegistry.def
index 97f0d577b30d2ba45e4733ed94cb6e2ac889cbec..ac193f5cae14cbb4e3ed3df117b40ce72e723bf6 100644
--- a/lib/Passes/PassRegistry.def
+++ b/lib/Passes/PassRegistry.def
@@ -1,9 +1,8 @@
 //===- PassRegistry.def - Registry of passes --------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -42,6 +41,7 @@ MODULE_ALIAS_ANALYSIS("globals-aa", GlobalsAA())
 #endif
 MODULE_PASS("always-inline", AlwaysInlinerPass())
 MODULE_PASS("called-value-propagation", CalledValuePropagationPass())
+MODULE_PASS("canonicalize-aliases", CanonicalizeAliasesPass())
 MODULE_PASS("cg-profile", CGProfilePass())
 MODULE_PASS("constmerge", ConstantMergePass())
 MODULE_PASS("cross-dso-cfi", CrossDSOCFIPass())
@@ -177,6 +177,7 @@ FUNCTION_PASS("libcalls-shrinkwrap", LibCallsShrinkWrapPass())
 FUNCTION_PASS("loweratomic", LowerAtomicPass())
 FUNCTION_PASS("lower-expect", LowerExpectIntrinsicPass())
 FUNCTION_PASS("lower-guard-intrinsic", LowerGuardIntrinsicPass())
+FUNCTION_PASS("lower-widenable-condition", LowerWidenableConditionPass())
 FUNCTION_PASS("guard-widening", GuardWideningPass())
 FUNCTION_PASS("gvn", GVN())
 FUNCTION_PASS("load-store-vectorizer", LoadStoreVectorizerPass())
@@ -200,6 +201,7 @@ FUNCTION_PASS("print", PrintFunctionPass(dbgs()))
 FUNCTION_PASS("print<assumptions>", AssumptionPrinterPass(dbgs()))
 FUNCTION_PASS("print<block-freq>", BlockFrequencyPrinterPass(dbgs()))
 FUNCTION_PASS("print<branch-prob>", BranchProbabilityPrinterPass(dbgs()))
+FUNCTION_PASS("print<da>", DependenceAnalysisPrinterPass(dbgs()))
 FUNCTION_PASS("print<domtree>", DominatorTreePrinterPass(dbgs()))
 FUNCTION_PASS("print<postdomtree>", PostDominatorTreePrinterPass(dbgs()))
 FUNCTION_PASS("print<demanded-bits>", DemandedBitsPrinterPass(dbgs()))
@@ -221,8 +223,6 @@ FUNCTION_PASS("spec-phis", SpeculateAroundPHIsPass())
 FUNCTION_PASS("sroa", SROA())
 FUNCTION_PASS("tailcallelim", TailCallElimPass())
 FUNCTION_PASS("unreachableblockelim", UnreachableBlockElimPass())
-FUNCTION_PASS("unroll", LoopUnrollPass())
-FUNCTION_PASS("unroll<peeling;no-runtime>",LoopUnrollPass(LoopUnrollOptions().setPeeling(true).setRuntime(false)))
 FUNCTION_PASS("verify", VerifierPass())
 FUNCTION_PASS("verify<domtree>", DominatorTreeVerifierPass())
 FUNCTION_PASS("verify<loops>", LoopVerifierPass())
@@ -231,8 +231,25 @@ FUNCTION_PASS("verify<regions>", RegionInfoVerifierPass())
 FUNCTION_PASS("view-cfg", CFGViewerPass())
 FUNCTION_PASS("view-cfg-only", CFGOnlyViewerPass())
 FUNCTION_PASS("transform-warning", WarnMissedTransformationsPass())
+FUNCTION_PASS("msan", MemorySanitizerPass({}))
+FUNCTION_PASS("tsan", ThreadSanitizerPass())
 #undef FUNCTION_PASS
 
+#ifndef FUNCTION_PASS_WITH_PARAMS
+#define FUNCTION_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER)
+#endif
+FUNCTION_PASS_WITH_PARAMS("unroll",
+                           [](LoopUnrollOptions Opts) {
+                             return LoopUnrollPass(Opts);
+                           },
+                           parseLoopUnrollOptions)
+FUNCTION_PASS_WITH_PARAMS("msan",
+                           [](MemorySanitizerOptions Opts) {
+                             return MemorySanitizerPass(Opts);
+                           },
+                           parseMSanPassOptions)
+#undef FUNCTION_PASS_WITH_PARAMS
+
 #ifndef LOOP_ANALYSIS
 #define LOOP_ANALYSIS(NAME, CREATE_PASS)
 #endif
diff --git a/lib/Passes/StandardInstrumentations.cpp b/lib/Passes/StandardInstrumentations.cpp
index 765ffe626620108a47adcc1ab77788d66b6876a1..5cf0ca8e28f69e1e4d158d3916774a540e6439db 100644
--- a/lib/Passes/StandardInstrumentations.cpp
+++ b/lib/Passes/StandardInstrumentations.cpp
@@ -1,9 +1,8 @@
 //===- Standard pass instrumentations handling ----------------*- C++ -*--===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
@@ -14,6 +13,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/Passes/StandardInstrumentations.h"
+#include "llvm/ADT/Optional.h"
 #include "llvm/Analysis/CallGraphSCCPass.h"
 #include "llvm/Analysis/LazyCallGraph.h"
 #include "llvm/Analysis/LoopInfo.h"
@@ -28,112 +28,215 @@
 using namespace llvm;
 
 namespace {
-namespace PrintIR {
 
-//===----------------------------------------------------------------------===//
-// IR-printing instrumentation
-//===----------------------------------------------------------------------===//
+/// Extracting Module out of \p IR unit. Also fills a textual description
+/// of \p IR for use in header when printing.
+Optional<std::pair<const Module *, std::string>> unwrapModule(Any IR) {
+  if (any_isa<const Module *>(IR))
+    return std::make_pair(any_cast<const Module *>(IR), std::string());
 
-/// Generic IR-printing helper that unpacks a pointer to IRUnit wrapped into
-/// llvm::Any and does actual print job.
-void unwrapAndPrint(StringRef Banner, Any IR) {
-  SmallString<40> Extra{"\n"};
-  const Module *M = nullptr;
-  if (any_isa<const Module *>(IR)) {
-    M = any_cast<const Module *>(IR);
-  } else if (any_isa<const Function *>(IR)) {
+  if (any_isa<const Function *>(IR)) {
     const Function *F = any_cast<const Function *>(IR);
     if (!llvm::isFunctionInPrintList(F->getName()))
-      return;
-    if (!llvm::forcePrintModuleIR()) {
-      dbgs() << Banner << Extra << static_cast<const Value &>(*F);
-      return;
-    }
-    M = F->getParent();
-    Extra = formatv(" (function: {0})\n", F->getName());
-  } else if (any_isa<const LazyCallGraph::SCC *>(IR)) {
+      return None;
+    const Module *M = F->getParent();
+    return std::make_pair(M, formatv(" (function: {0})", F->getName()).str());
+  }
+
+  if (any_isa<const LazyCallGraph::SCC *>(IR)) {
     const LazyCallGraph::SCC *C = any_cast<const LazyCallGraph::SCC *>(IR);
-    if (!llvm::forcePrintModuleIR()) {
-      Extra = formatv(" (scc: {0})\n", C->getName());
-      bool BannerPrinted = false;
-      for (const LazyCallGraph::Node &N : *C) {
-        const Function &F = N.getFunction();
-        if (!F.isDeclaration() && isFunctionInPrintList(F.getName())) {
-          if (!BannerPrinted) {
-            dbgs() << Banner << Extra;
-            BannerPrinted = true;
-          }
-          F.print(dbgs());
-        }
-      }
-      return;
-    }
     for (const LazyCallGraph::Node &N : *C) {
       const Function &F = N.getFunction();
       if (!F.isDeclaration() && isFunctionInPrintList(F.getName())) {
-        M = F.getParent();
-        break;
+        const Module *M = F.getParent();
+        return std::make_pair(M, formatv(" (scc: {0})", C->getName()).str());
       }
     }
-    if (!M)
-      return;
-    Extra = formatv(" (for scc: {0})\n", C->getName());
-  } else if (any_isa<const Loop *>(IR)) {
+    return None;
+  }
+
+  if (any_isa<const Loop *>(IR)) {
     const Loop *L = any_cast<const Loop *>(IR);
     const Function *F = L->getHeader()->getParent();
     if (!isFunctionInPrintList(F->getName()))
-      return;
-    if (!llvm::forcePrintModuleIR()) {
-      llvm::printLoop(const_cast<Loop &>(*L), dbgs(), Banner);
-      return;
-    }
-    M = F->getParent();
-    {
-      std::string LoopName;
-      raw_string_ostream ss(LoopName);
-      L->getHeader()->printAsOperand(ss, false);
-      Extra = formatv(" (loop: {0})\n", ss.str());
+      return None;
+    const Module *M = F->getParent();
+    std::string LoopName;
+    raw_string_ostream ss(LoopName);
+    L->getHeader()->printAsOperand(ss, false);
+    return std::make_pair(M, formatv(" (loop: {0})", ss.str()).str());
+  }
+
+  llvm_unreachable("Unknown IR unit");
+}
+
+void printIR(const Module *M, StringRef Banner, StringRef Extra = StringRef()) {
+  dbgs() << Banner << Extra << "\n";
+  M->print(dbgs(), nullptr, false);
+}
+void printIR(const Function *F, StringRef Banner,
+             StringRef Extra = StringRef()) {
+  if (!llvm::isFunctionInPrintList(F->getName()))
+    return;
+  dbgs() << Banner << Extra << "\n" << static_cast<const Value &>(*F);
+}
+void printIR(const LazyCallGraph::SCC *C, StringRef Banner,
+             StringRef Extra = StringRef()) {
+  bool BannerPrinted = false;
+  for (const LazyCallGraph::Node &N : *C) {
+    const Function &F = N.getFunction();
+    if (!F.isDeclaration() && llvm::isFunctionInPrintList(F.getName())) {
+      if (!BannerPrinted) {
+        dbgs() << Banner << Extra << "\n";
+        BannerPrinted = true;
+      }
+      F.print(dbgs());
     }
   }
-  if (M) {
-    dbgs() << Banner << Extra;
-    M->print(dbgs(), nullptr, false);
-  } else {
-    llvm_unreachable("Unknown wrapped IR type");
+}
+void printIR(const Loop *L, StringRef Banner) {
+  const Function *F = L->getHeader()->getParent();
+  if (!llvm::isFunctionInPrintList(F->getName()))
+    return;
+  llvm::printLoop(const_cast<Loop &>(*L), dbgs(), Banner);
+}
+
+/// Generic IR-printing helper that unpacks a pointer to IRUnit wrapped into
+/// llvm::Any and does actual print job.
+void unwrapAndPrint(Any IR, StringRef Banner, bool ForceModule = false) {
+  if (ForceModule) {
+    if (auto UnwrappedModule = unwrapModule(IR))
+      printIR(UnwrappedModule->first, Banner, UnwrappedModule->second);
+    return;
+  }
+
+  if (any_isa<const Module *>(IR)) {
+    const Module *M = any_cast<const Module *>(IR);
+    assert(M && "module should be valid for printing");
+    printIR(M, Banner);
+    return;
+  }
+
+  if (any_isa<const Function *>(IR)) {
+    const Function *F = any_cast<const Function *>(IR);
+    assert(F && "function should be valid for printing");
+    printIR(F, Banner);
+    return;
+  }
+
+  if (any_isa<const LazyCallGraph::SCC *>(IR)) {
+    const LazyCallGraph::SCC *C = any_cast<const LazyCallGraph::SCC *>(IR);
+    assert(C && "scc should be valid for printing");
+    std::string Extra = formatv(" (scc: {0})", C->getName());
+    printIR(C, Banner, Extra);
+    return;
+  }
+
+  if (any_isa<const Loop *>(IR)) {
+    const Loop *L = any_cast<const Loop *>(IR);
+    assert(L && "Loop should be valid for printing");
+    printIR(L, Banner);
+    return;
   }
+  llvm_unreachable("Unknown wrapped IR type");
 }
 
-bool printBeforePass(StringRef PassID, Any IR) {
-  if (!llvm::shouldPrintBeforePass(PassID))
-    return true;
+} // namespace
 
+PrintIRInstrumentation::~PrintIRInstrumentation() {
+  assert(ModuleDescStack.empty() && "ModuleDescStack is not empty at exit");
+}
+
+void PrintIRInstrumentation::pushModuleDesc(StringRef PassID, Any IR) {
+  assert(StoreModuleDesc);
+  const Module *M = nullptr;
+  std::string Extra;
+  if (auto UnwrappedModule = unwrapModule(IR))
+    std::tie(M, Extra) = UnwrappedModule.getValue();
+  ModuleDescStack.emplace_back(M, Extra, PassID);
+}
+
+PrintIRInstrumentation::PrintModuleDesc
+PrintIRInstrumentation::popModuleDesc(StringRef PassID) {
+  assert(!ModuleDescStack.empty() && "empty ModuleDescStack");
+  PrintModuleDesc ModuleDesc = ModuleDescStack.pop_back_val();
+  assert(std::get<2>(ModuleDesc).equals(PassID) && "malformed ModuleDescStack");
+  return ModuleDesc;
+}
+
+bool PrintIRInstrumentation::printBeforePass(StringRef PassID, Any IR) {
   if (PassID.startswith("PassManager<") || PassID.contains("PassAdaptor<"))
     return true;
 
+  // Saving Module for AfterPassInvalidated operations.
+  // Note: here we rely on a fact that we do not change modules while
+  // traversing the pipeline, so the latest captured module is good
+  // for all print operations that has not happen yet.
+  if (StoreModuleDesc && llvm::shouldPrintAfterPass(PassID))
+    pushModuleDesc(PassID, IR);
+
+  if (!llvm::shouldPrintBeforePass(PassID))
+    return true;
+
   SmallString<20> Banner = formatv("*** IR Dump Before {0} ***", PassID);
-  unwrapAndPrint(Banner, IR);
+  unwrapAndPrint(IR, Banner, llvm::forcePrintModuleIR());
   return true;
 }
 
-void printAfterPass(StringRef PassID, Any IR) {
+void PrintIRInstrumentation::printAfterPass(StringRef PassID, Any IR) {
+  if (PassID.startswith("PassManager<") || PassID.contains("PassAdaptor<"))
+    return;
+
   if (!llvm::shouldPrintAfterPass(PassID))
     return;
 
+  if (StoreModuleDesc)
+    popModuleDesc(PassID);
+
+  SmallString<20> Banner = formatv("*** IR Dump After {0} ***", PassID);
+  unwrapAndPrint(IR, Banner, llvm::forcePrintModuleIR());
+}
+
+void PrintIRInstrumentation::printAfterPassInvalidated(StringRef PassID) {
+  if (!StoreModuleDesc || !llvm::shouldPrintAfterPass(PassID))
+    return;
+
   if (PassID.startswith("PassManager<") || PassID.contains("PassAdaptor<"))
     return;
 
-  SmallString<20> Banner = formatv("*** IR Dump After {0} ***", PassID);
-  unwrapAndPrint(Banner, IR);
-  return;
+  const Module *M;
+  std::string Extra;
+  StringRef StoredPassID;
+  std::tie(M, Extra, StoredPassID) = popModuleDesc(PassID);
+  // Additional filtering (e.g. -filter-print-func) can lead to module
+  // printing being skipped.
+  if (!M)
+    return;
+
+  SmallString<20> Banner =
+      formatv("*** IR Dump After {0} *** invalidated: ", PassID);
+  printIR(M, Banner, Extra);
+}
+
+void PrintIRInstrumentation::registerCallbacks(
+    PassInstrumentationCallbacks &PIC) {
+  // BeforePass callback is not just for printing, it also saves a Module
+  // for later use in AfterPassInvalidated.
+  StoreModuleDesc = llvm::forcePrintModuleIR() && llvm::shouldPrintAfterPass();
+  if (llvm::shouldPrintBeforePass() || StoreModuleDesc)
+    PIC.registerBeforePassCallback(
+        [this](StringRef P, Any IR) { return this->printBeforePass(P, IR); });
+
+  if (llvm::shouldPrintAfterPass()) {
+    PIC.registerAfterPassCallback(
+        [this](StringRef P, Any IR) { this->printAfterPass(P, IR); });
+    PIC.registerAfterPassInvalidatedCallback(
+        [this](StringRef P) { this->printAfterPassInvalidated(P); });
+  }
 }
-} // namespace PrintIR
-} // namespace
 
 void StandardInstrumentations::registerCallbacks(
     PassInstrumentationCallbacks &PIC) {
-  if (llvm::shouldPrintBeforePass())
-    PIC.registerBeforePassCallback(PrintIR::printBeforePass);
-  if (llvm::shouldPrintAfterPass())
-    PIC.registerAfterPassCallback(PrintIR::printAfterPass);
+  PrintIR.registerCallbacks(PIC);
   TimePasses.registerCallbacks(PIC);
 }
diff --git a/lib/ProfileData/Coverage/CoverageMapping.cpp b/lib/ProfileData/Coverage/CoverageMapping.cpp
index b2dde3406a63c320bd308ee097132911244b5938..e122e5c7b5d5b944042bc42a82d7f4140e680d29 100644
--- a/lib/ProfileData/Coverage/CoverageMapping.cpp
+++ b/lib/ProfileData/Coverage/CoverageMapping.cpp
@@ -1,9 +1,8 @@
 //===- CoverageMapping.cpp - Code coverage mapping support ----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/ProfileData/Coverage/CoverageMappingReader.cpp b/lib/ProfileData/Coverage/CoverageMappingReader.cpp
index ee48256bc2e572693e413745294fa7b5fb6c8710..af09a611570938b881eb8c63bd888c12b22abd3b 100644
--- a/lib/ProfileData/Coverage/CoverageMappingReader.cpp
+++ b/lib/ProfileData/Coverage/CoverageMappingReader.cpp
@@ -1,9 +1,8 @@
 //===- CoverageMappingReader.cpp - Code coverage mapping reader -----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/ProfileData/Coverage/CoverageMappingWriter.cpp b/lib/ProfileData/Coverage/CoverageMappingWriter.cpp
index bb3f4f854e0480619e4f967d0fc07fd3e8dd38cd..0ef6744e64f5565aef51151161f70b9d4491d172 100644
--- a/lib/ProfileData/Coverage/CoverageMappingWriter.cpp
+++ b/lib/ProfileData/Coverage/CoverageMappingWriter.cpp
@@ -1,9 +1,8 @@
 //===- CoverageMappingWriter.cpp - Code coverage mapping writer -----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/ProfileData/Coverage/LLVMBuild.txt b/lib/ProfileData/Coverage/LLVMBuild.txt
index fc8284b0ef38f4d9af1a4ffab073e0eb59e6d7c5..21a9bf0b873763ad44183c3e07c2626bc78f714b 100644
--- a/lib/ProfileData/Coverage/LLVMBuild.txt
+++ b/lib/ProfileData/Coverage/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/ProfileData/Coverage/LLVMBuild.txt -----------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/ProfileData/GCOV.cpp b/lib/ProfileData/GCOV.cpp
index b687346a2c054a964aec6cebda24f988841fa6a2..41ea7555b9f096c4cc7d14bc1f4ebdb55790c560 100644
--- a/lib/ProfileData/GCOV.cpp
+++ b/lib/ProfileData/GCOV.cpp
@@ -1,9 +1,8 @@
 //===- GCOV.cpp - LLVM coverage tool --------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/ProfileData/InstrProf.cpp b/lib/ProfileData/InstrProf.cpp
index 544a77ec20a5d6d9b9ed48ec65cd88cbeb5280d7..8a2ff7769f16f2083d1ec4a80c4b5eb37d4dd04d 100644
--- a/lib/ProfileData/InstrProf.cpp
+++ b/lib/ProfileData/InstrProf.cpp
@@ -1,9 +1,8 @@
 //===- InstrProf.cpp - Instrumented profiling format support --------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -252,11 +251,12 @@ static StringRef stripDirPrefix(StringRef PathNameStr, uint32_t NumPrefix) {
 // data, its original linkage must be non-internal.
 std::string getPGOFuncName(const Function &F, bool InLTO, uint64_t Version) {
   if (!InLTO) {
-    StringRef FileName = (StaticFuncFullModulePrefix
-                              ? F.getParent()->getName()
-                              : sys::path::filename(F.getParent()->getName()));
-    if (StaticFuncFullModulePrefix && StaticFuncStripDirNamePrefix != 0)
-      FileName = stripDirPrefix(FileName, StaticFuncStripDirNamePrefix);
+    StringRef FileName(F.getParent()->getSourceFileName());
+    uint32_t StripLevel = StaticFuncFullModulePrefix ? 0 : (uint32_t)-1;
+    if (StripLevel < StaticFuncStripDirNamePrefix)
+      StripLevel = StaticFuncStripDirNamePrefix;
+    if (StripLevel)
+      FileName = stripDirPrefix(FileName, StripLevel);
     return getPGOFuncName(F.getName(), F.getLinkage(), FileName, Version);
   }
 
@@ -1011,4 +1011,21 @@ void getMemOPSizeRangeFromOption(StringRef MemOPSizeRange, int64_t &RangeStart,
   assert(RangeLast >= RangeStart);
 }
 
+// Create the variable for the profile file name.
+void createProfileFileNameVar(Module &M, StringRef InstrProfileOutput) {
+  if (InstrProfileOutput.empty())
+    return;
+  Constant *ProfileNameConst =
+      ConstantDataArray::getString(M.getContext(), InstrProfileOutput, true);
+  GlobalVariable *ProfileNameVar = new GlobalVariable(
+      M, ProfileNameConst->getType(), true, GlobalValue::WeakAnyLinkage,
+      ProfileNameConst, INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_NAME_VAR));
+  Triple TT(M.getTargetTriple());
+  if (TT.supportsCOMDAT()) {
+    ProfileNameVar->setLinkage(GlobalValue::ExternalLinkage);
+    ProfileNameVar->setComdat(M.getOrInsertComdat(
+        StringRef(INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_NAME_VAR))));
+  }
+}
+
 } // end namespace llvm
diff --git a/lib/ProfileData/InstrProfReader.cpp b/lib/ProfileData/InstrProfReader.cpp
index eaf0eb04bfbfe2adff00b48865b34ed61ae28abe..5919efa72d3c67e7ee7a2b58959f6a93d5ad1c8d 100644
--- a/lib/ProfileData/InstrProfReader.cpp
+++ b/lib/ProfileData/InstrProfReader.cpp
@@ -1,9 +1,8 @@
 //===- InstrProfReader.cpp - Instrumented profiling reader ----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/ProfileData/InstrProfWriter.cpp b/lib/ProfileData/InstrProfWriter.cpp
index 18b9deec158f4113a2c2c1e882c80a7852ac2bcc..c025cd38121ed8feeb1bd7b624e51aec5ac025b8 100644
--- a/lib/ProfileData/InstrProfWriter.cpp
+++ b/lib/ProfileData/InstrProfWriter.cpp
@@ -1,9 +1,8 @@
 //===- InstrProfWriter.cpp - Instrumented profiling writer ----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/ProfileData/LLVMBuild.txt b/lib/ProfileData/LLVMBuild.txt
index b3d749feda61e81cad326a40502d743ae94073f2..335c2260a00294785e857306f1a72fa40b2023bc 100644
--- a/lib/ProfileData/LLVMBuild.txt
+++ b/lib/ProfileData/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/ProfileData/LLVMBuild.txt --------------------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/ProfileData/ProfileSummaryBuilder.cpp b/lib/ProfileData/ProfileSummaryBuilder.cpp
index 3a8462fd9b0db7912ff4b48b0a0011c43570e584..35ac5b90cd1d3bac23379829adc255da342c7790 100644
--- a/lib/ProfileData/ProfileSummaryBuilder.cpp
+++ b/lib/ProfileData/ProfileSummaryBuilder.cpp
@@ -1,9 +1,8 @@
 //=-- ProfilesummaryBuilder.cpp - Profile summary computation ---------------=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/ProfileData/SampleProf.cpp b/lib/ProfileData/SampleProf.cpp
index 1a124415f17979a25eef70c6b4ccba7c4433a824..e17865cd15a474daa81018cc1d39c30e2cf8b3b7 100644
--- a/lib/ProfileData/SampleProf.cpp
+++ b/lib/ProfileData/SampleProf.cpp
@@ -1,9 +1,8 @@
 //=-- SampleProf.cpp - Sample profiling format support --------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/ProfileData/SampleProfReader.cpp b/lib/ProfileData/SampleProfReader.cpp
index a68d1e9d3ab06262106d1b0beaca99499c749c89..2e43c9599c86ae6970aea94fcf22aeb8497191ae 100644
--- a/lib/ProfileData/SampleProfReader.cpp
+++ b/lib/ProfileData/SampleProfReader.cpp
@@ -1,9 +1,8 @@
 //===- SampleProfReader.cpp - Read LLVM sample profile data ---------------===//
 //
-//                      The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/ProfileData/SampleProfWriter.cpp b/lib/ProfileData/SampleProfWriter.cpp
index b1c669ec31c467b1ced467dc745e291c34b6b452..8135febdf86037ed8f6b88f76e7f4f5e5e2bf8bf 100644
--- a/lib/ProfileData/SampleProfWriter.cpp
+++ b/lib/ProfileData/SampleProfWriter.cpp
@@ -1,9 +1,8 @@
 //===- SampleProfWriter.cpp - Write LLVM sample profile data --------------===//
 //
-//                      The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Support/AArch64TargetParser.cpp b/lib/Support/AArch64TargetParser.cpp
index e897137df680fcfc99f344223b3ecf31c152790f..faf8385c745f2b5f074fb9134ce4474a3e7ff442 100644
--- a/lib/Support/AArch64TargetParser.cpp
+++ b/lib/Support/AArch64TargetParser.cpp
@@ -1,9 +1,8 @@
 //===-- AArch64TargetParser - Parser for AArch64 features -------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Support/AMDGPUMetadata.cpp b/lib/Support/AMDGPUMetadata.cpp
index a04bfc2ea2999d517ae7b6537e87dedf46de88ab..4d5ce34dff39218a7ae282956856ddd62c5d2882 100644
--- a/lib/Support/AMDGPUMetadata.cpp
+++ b/lib/Support/AMDGPUMetadata.cpp
@@ -1,9 +1,8 @@
 //===--- AMDGPUMetadata.cpp -------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Support/APFloat.cpp b/lib/Support/APFloat.cpp
index e9e429c8031b65f5efb0ef0a0340a53b4e5ae248..3ebed5b5ab8c2d7a6ea15f2b31a6925e520e1c18 100644
--- a/lib/Support/APFloat.cpp
+++ b/lib/Support/APFloat.cpp
@@ -1,9 +1,8 @@
 //===-- APFloat.cpp - Implement APFloat class -----------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Support/APInt.cpp b/lib/Support/APInt.cpp
index a5f4f98c489ae4f44822a99d17ea1a682c18f351..5ed176f1b627d667a8c52bf1fcff7a832fc2523f 100644
--- a/lib/Support/APInt.cpp
+++ b/lib/Support/APInt.cpp
@@ -1,9 +1,8 @@
 //===-- APInt.cpp - Implement APInt class ---------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Support/APSInt.cpp b/lib/Support/APSInt.cpp
index 46c0f70ff66bc3dcc1df8ad738651f903cc8e3f7..d2dc860f7384f964e2d8e9317693d1f13025e432 100644
--- a/lib/Support/APSInt.cpp
+++ b/lib/Support/APSInt.cpp
@@ -1,9 +1,8 @@
 //===-- llvm/ADT/APSInt.cpp - Arbitrary Precision Signed Int ---*- C++ -*--===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Support/ARMAttributeParser.cpp b/lib/Support/ARMAttributeParser.cpp
index 1f98ac2f40ba24dcb0232012f5d50b72abbe516a..0b7db2f2f7205ab3a7d54bf9f129a31c9efc8f49 100644
--- a/lib/Support/ARMAttributeParser.cpp
+++ b/lib/Support/ARMAttributeParser.cpp
@@ -1,9 +1,8 @@
 //===--- ARMAttributeParser.cpp - ARM Attribute Information Printer -------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Support/ARMBuildAttrs.cpp b/lib/Support/ARMBuildAttrs.cpp
index 8f18e9eb24edaca5f9e61ab023e30897c814bbb7..f5fb64bb7ba6926555d45902a06c56fef053514f 100644
--- a/lib/Support/ARMBuildAttrs.cpp
+++ b/lib/Support/ARMBuildAttrs.cpp
@@ -1,9 +1,8 @@
 //===-- ARMBuildAttrs.cpp - ARM Build Attributes --------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Support/ARMTargetParser.cpp b/lib/Support/ARMTargetParser.cpp
index 07294b0c09a3031bc138072eb56d46c122994c66..7b8ec2f339065775ab31b5345bfa88a2cb6acbce 100644
--- a/lib/Support/ARMTargetParser.cpp
+++ b/lib/Support/ARMTargetParser.cpp
@@ -1,9 +1,8 @@
 //===-- ARMTargetParser - Parser for ARM target features --------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Support/ARMWinEH.cpp b/lib/Support/ARMWinEH.cpp
index 03c150f1150b166ffe8886d6d83403d2727e9b3c..831f95cd4b0b143bcfbe179684cceb087a080f2d 100644
--- a/lib/Support/ARMWinEH.cpp
+++ b/lib/Support/ARMWinEH.cpp
@@ -1,9 +1,8 @@
 //===-- ARMWinEH.cpp - Windows on ARM EH Support Functions ------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Support/Allocator.cpp b/lib/Support/Allocator.cpp
index f48edac0598ce13ece81ec075fbfbb9b0ca35a10..718d3fc0d8e1ce514585b2caa9db38177d521aa5 100644
--- a/lib/Support/Allocator.cpp
+++ b/lib/Support/Allocator.cpp
@@ -1,9 +1,8 @@
 //===--- Allocator.cpp - Simple memory allocation abstraction -------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Support/Atomic.cpp b/lib/Support/Atomic.cpp
index 7328a93052ccbd92f975101ad16c0ba3d14a9763..f6865405c2b8500a01a4b3650f9b49a0629ba076 100644
--- a/lib/Support/Atomic.cpp
+++ b/lib/Support/Atomic.cpp
@@ -1,9 +1,8 @@
 //===-- Atomic.cpp - Atomic Operations --------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Support/BinaryStreamError.cpp b/lib/Support/BinaryStreamError.cpp
index cdc811d78d63d603d90e216bd5c285db27639d6e..f22523f09ac80d1a36868c216198be337686c727 100644
--- a/lib/Support/BinaryStreamError.cpp
+++ b/lib/Support/BinaryStreamError.cpp
@@ -1,9 +1,8 @@
 //===- BinaryStreamError.cpp - Error extensions for streams -----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Support/BinaryStreamReader.cpp b/lib/Support/BinaryStreamReader.cpp
index e00527f2519e1c1a40bbd064c01946f20eceb3ae..6f691bf4dcdfa34f76f7fbf733d9b8077b8cc321 100644
--- a/lib/Support/BinaryStreamReader.cpp
+++ b/lib/Support/BinaryStreamReader.cpp
@@ -1,9 +1,8 @@
 //===- BinaryStreamReader.cpp - Reads objects from a binary stream --------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Support/BinaryStreamRef.cpp b/lib/Support/BinaryStreamRef.cpp
index bdc0f54bf25ad558259c2abb2035cb08efcf071b..6bcc504ffad5fa8763de554930462e63e7193e9c 100644
--- a/lib/Support/BinaryStreamRef.cpp
+++ b/lib/Support/BinaryStreamRef.cpp
@@ -1,9 +1,8 @@
 //===- BinaryStreamRef.cpp - ----------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Support/BinaryStreamWriter.cpp b/lib/Support/BinaryStreamWriter.cpp
index bfad1280b9297d229e3b23d1d4da4743a424ae0b..929968bf2e44fdd8d97837ab791a45a15ed4580a 100644
--- a/lib/Support/BinaryStreamWriter.cpp
+++ b/lib/Support/BinaryStreamWriter.cpp
@@ -1,9 +1,8 @@
 //===- BinaryStreamWriter.cpp - Writes objects to a BinaryStream ----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Support/BlockFrequency.cpp b/lib/Support/BlockFrequency.cpp
index 34fcbde23a28e9ea5ededf47eb0d5699c2457e5c..2b63294f3789e4c4a4db8ad5112fa09e4bdd2ac2 100644
--- a/lib/Support/BlockFrequency.cpp
+++ b/lib/Support/BlockFrequency.cpp
@@ -1,9 +1,8 @@
 //====--------------- lib/Support/BlockFrequency.cpp -----------*- C++ -*-====//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Support/BranchProbability.cpp b/lib/Support/BranchProbability.cpp
index 31dee9561f4941e2630bc04f9a76877bad8a00a2..bb608da16c174d2ca1f5af6e54b7c8db93ab15c4 100644
--- a/lib/Support/BranchProbability.cpp
+++ b/lib/Support/BranchProbability.cpp
@@ -1,9 +1,8 @@
 //===-------------- lib/Support/BranchProbability.cpp -----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Support/BuryPointer.cpp b/lib/Support/BuryPointer.cpp
index 6c988b4a0ab263d79e76e5ba397647859cfe7e1c..435f89010d410ac09ea0b200a3f9aa18b232ef6d 100644
--- a/lib/Support/BuryPointer.cpp
+++ b/lib/Support/BuryPointer.cpp
@@ -1,9 +1,8 @@
 //===- BuryPointer.cpp - Memory Manipulation/Leak ---------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Support/CMakeLists.txt b/lib/Support/CMakeLists.txt
index 2a6810672b1da92a5146df21f5b761197ba1f29f..7e791594587e43aea1de92a85dea4b14d558eb99 100644
--- a/lib/Support/CMakeLists.txt
+++ b/lib/Support/CMakeLists.txt
@@ -106,6 +106,7 @@ add_llvm_library(LLVMSupport
   MemoryBuffer.cpp
   MD5.cpp
   NativeFormatting.cpp
+  Optional.cpp
   Options.cpp
   Parallel.cpp
   PluginLoader.cpp
diff --git a/lib/Support/COM.cpp b/lib/Support/COM.cpp
index 97cd085853b05032009a59f4a527d9fca1fb0dfd..f37b95ba865121266c4923dd470aea18d750d1f9 100644
--- a/lib/Support/COM.cpp
+++ b/lib/Support/COM.cpp
@@ -1,9 +1,8 @@
 //===-- COM.cpp - Implement COM utility classes -----------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Support/CachePruning.cpp b/lib/Support/CachePruning.cpp
index a0aa6024b3ed26b91640e02c1e699d886e7c7b95..7f97d1afbd3a4cf757d0951ce1fc9feb2578b88b 100644
--- a/lib/Support/CachePruning.cpp
+++ b/lib/Support/CachePruning.cpp
@@ -1,9 +1,8 @@
 //===-CachePruning.cpp - LLVM Cache Directory Pruning ---------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Support/Chrono.cpp b/lib/Support/Chrono.cpp
index a2626a89eb637abed8d7d86727aec3209b3834d6..8c28d45d8822828cf96abdae24b6498e79c3fefe 100644
--- a/lib/Support/Chrono.cpp
+++ b/lib/Support/Chrono.cpp
@@ -1,9 +1,8 @@
 //===- Support/Chrono.cpp - Utilities for Timing Manipulation ---*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Support/CodeGenCoverage.cpp b/lib/Support/CodeGenCoverage.cpp
index 811020e3254a05209096695bd2cbef5e56d764b9..f39eb7533b4379c08b405a9f4012736d68973956 100644
--- a/lib/Support/CodeGenCoverage.cpp
+++ b/lib/Support/CodeGenCoverage.cpp
@@ -1,9 +1,8 @@
 //===- lib/Support/CodeGenCoverage.cpp -------------------------------------==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/lib/Support/CommandLine.cpp b/lib/Support/CommandLine.cpp
index cb2a2e557fa982488973d5edc879bb3314ca177e..b0c92b7c72c45d5334176b4f7f43976bf4544d83 100644
--- a/lib/Support/CommandLine.cpp
+++ b/lib/Support/CommandLine.cpp
@@ -1,9 +1,8 @@
 //===-- CommandLine.cpp - Command line parser implementation --------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -426,12 +425,17 @@ Option *CommandLineParser::LookupOption(SubCommand &Sub, StringRef &Arg,
     return I != Sub.OptionsMap.end() ? I->second : nullptr;
   }
 
-  // If the argument before the = is a valid option name, we match.  If not,
-  // return Arg unmolested.
+  // If the argument before the = is a valid option name and the option allows
+  // non-prefix form (ie is not AlwaysPrefix), we match.  If not, signal match
+  // failure by returning nullptr.
   auto I = Sub.OptionsMap.find(Arg.substr(0, EqualPos));
   if (I == Sub.OptionsMap.end())
     return nullptr;
 
+  auto O = I->second;
+  if (O->getFormattingFlag() == cl::AlwaysPrefix)
+    return nullptr;
+
   Value = Arg.substr(EqualPos + 1);
   Arg = Arg.substr(0, EqualPos);
   return I->second;
@@ -539,7 +543,9 @@ static inline bool ProvideOption(Option *Handler, StringRef ArgName,
   switch (Handler->getValueExpectedFlag()) {
   case ValueRequired:
     if (!Value.data()) { // No value specified?
-      if (i + 1 >= argc)
+      // If no other argument or the option only supports prefix form, we
+      // cannot look at the next argument.
+      if (i + 1 >= argc || Handler->getFormattingFlag() == cl::AlwaysPrefix)
         return Handler->error("requires a value!");
       // Steal the next argument, like for '-o filename'
       assert(argv && "null check");
@@ -597,7 +603,8 @@ static inline bool isGrouping(const Option *O) {
   return O->getFormattingFlag() == cl::Grouping;
 }
 static inline bool isPrefixedOrGrouping(const Option *O) {
-  return isGrouping(O) || O->getFormattingFlag() == cl::Prefix;
+  return isGrouping(O) || O->getFormattingFlag() == cl::Prefix ||
+         O->getFormattingFlag() == cl::AlwaysPrefix;
 }
 
 // getOptionPred - Check to see if there are any options that satisfy the
@@ -647,7 +654,8 @@ HandlePrefixedOrGroupedOption(StringRef &Arg, StringRef &Value,
   // If the option is a prefixed option, then the value is simply the
   // rest of the name...  so fall through to later processing, by
   // setting up the argument name flags and value fields.
-  if (PGOpt->getFormattingFlag() == cl::Prefix) {
+  if (PGOpt->getFormattingFlag() == cl::Prefix ||
+      PGOpt->getFormattingFlag() == cl::AlwaysPrefix) {
     Value = Arg.substr(Length);
     Arg = Arg.substr(0, Length);
     assert(OptionsMap.count(Arg) && OptionsMap.find(Arg)->second == PGOpt);
@@ -1470,17 +1478,23 @@ static StringRef getValueStr(const Option &O, StringRef DefaultMsg) {
   return O.ValueStr;
 }
 
+static StringRef ArgPrefix = "  -";
+static StringRef ArgHelpPrefix = " - ";
+static size_t ArgPrefixesSize = ArgPrefix.size() + ArgHelpPrefix.size();
+
 //===----------------------------------------------------------------------===//
 // cl::alias class implementation
 //
 
 // Return the width of the option tag for printing...
-size_t alias::getOptionWidth() const { return ArgStr.size() + 6; }
+size_t alias::getOptionWidth() const { return ArgStr.size() + ArgPrefixesSize; }
 
 void Option::printHelpStr(StringRef HelpStr, size_t Indent,
-                                 size_t FirstLineIndentedBy) {
+                          size_t FirstLineIndentedBy) {
+  assert(Indent >= FirstLineIndentedBy);
   std::pair<StringRef, StringRef> Split = HelpStr.split('\n');
-  outs().indent(Indent - FirstLineIndentedBy) << " - " << Split.first << "\n";
+  outs().indent(Indent - FirstLineIndentedBy)
+      << ArgHelpPrefix << Split.first << "\n";
   while (!Split.second.empty()) {
     Split = Split.second.split('\n');
     outs().indent(Indent) << Split.first << "\n";
@@ -1489,8 +1503,8 @@ void Option::printHelpStr(StringRef HelpStr, size_t Indent,
 
 // Print out the option for the alias.
 void alias::printOptionInfo(size_t GlobalWidth) const {
-  outs() << "  -" << ArgStr;
-  printHelpStr(HelpStr, GlobalWidth, ArgStr.size() + 6);
+  outs() << ArgPrefix << ArgStr;
+  printHelpStr(HelpStr, GlobalWidth, ArgStr.size() + ArgPrefixesSize);
 }
 
 //===----------------------------------------------------------------------===//
@@ -1511,7 +1525,7 @@ size_t basic_parser_impl::getOptionWidth(const Option &O) const {
     Len += getValueStr(O, ValName).size() + FormattingLen;
   }
 
-  return Len + 6;
+  return Len + ArgPrefixesSize;
 }
 
 // printOptionInfo - Print out information about this option.  The
@@ -1519,7 +1533,7 @@ size_t basic_parser_impl::getOptionWidth(const Option &O) const {
 //
 void basic_parser_impl::printOptionInfo(const Option &O,
                                         size_t GlobalWidth) const {
-  outs() << "  -" << O.ArgStr;
+  outs() << ArgPrefix << O.ArgStr;
 
   auto ValName = getValueName();
   if (!ValName.empty()) {
@@ -1535,7 +1549,7 @@ void basic_parser_impl::printOptionInfo(const Option &O,
 
 void basic_parser_impl::printOptionName(const Option &O,
                                         size_t GlobalWidth) const {
-  outs() << "  -" << O.ArgStr;
+  outs() << ArgPrefix << O.ArgStr;
   outs().indent(GlobalWidth - O.ArgStr.size());
 }
 
@@ -1643,12 +1657,28 @@ unsigned generic_parser_base::findOption(StringRef Name) {
   return e;
 }
 
+static StringRef EqValue = "=<value>";
+static StringRef EmptyOption = "<empty>";
+static StringRef OptionPrefix = "    =";
+static size_t OptionPrefixesSize = OptionPrefix.size() + ArgHelpPrefix.size();
+
+static bool shouldPrintOption(StringRef Name, StringRef Description,
+                              const Option &O) {
+  return O.getValueExpectedFlag() != ValueOptional || !Name.empty() ||
+         !Description.empty();
+}
+
 // Return the width of the option tag for printing...
 size_t generic_parser_base::getOptionWidth(const Option &O) const {
   if (O.hasArgStr()) {
-    size_t Size = O.ArgStr.size() + 6;
-    for (unsigned i = 0, e = getNumOptions(); i != e; ++i)
-      Size = std::max(Size, getOption(i).size() + 8);
+    size_t Size = O.ArgStr.size() + ArgPrefixesSize + EqValue.size();
+    for (unsigned i = 0, e = getNumOptions(); i != e; ++i) {
+      StringRef Name = getOption(i);
+      if (!shouldPrintOption(Name, getDescription(i), O))
+        continue;
+      size_t NameSize = Name.empty() ? EmptyOption.size() : Name.size();
+      Size = std::max(Size, NameSize + OptionPrefixesSize);
+    }
     return Size;
   } else {
     size_t BaseSize = 0;
@@ -1664,13 +1694,38 @@ size_t generic_parser_base::getOptionWidth(const Option &O) const {
 void generic_parser_base::printOptionInfo(const Option &O,
                                           size_t GlobalWidth) const {
   if (O.hasArgStr()) {
-    outs() << "  -" << O.ArgStr;
-    Option::printHelpStr(O.HelpStr, GlobalWidth, O.ArgStr.size() + 6);
+    // When the value is optional, first print a line just describing the
+    // option without values.
+    if (O.getValueExpectedFlag() == ValueOptional) {
+      for (unsigned i = 0, e = getNumOptions(); i != e; ++i) {
+        if (getOption(i).empty()) {
+          outs() << ArgPrefix << O.ArgStr;
+          Option::printHelpStr(O.HelpStr, GlobalWidth,
+                               O.ArgStr.size() + ArgPrefixesSize);
+          break;
+        }
+      }
+    }
 
+    outs() << ArgPrefix << O.ArgStr << EqValue;
+    Option::printHelpStr(O.HelpStr, GlobalWidth,
+                         O.ArgStr.size() + EqValue.size() + ArgPrefixesSize);
     for (unsigned i = 0, e = getNumOptions(); i != e; ++i) {
-      size_t NumSpaces = GlobalWidth - getOption(i).size() - 8;
-      outs() << "    =" << getOption(i);
-      outs().indent(NumSpaces) << " -   " << getDescription(i) << '\n';
+      StringRef OptionName = getOption(i);
+      StringRef Description = getDescription(i);
+      if (!shouldPrintOption(OptionName, Description, O))
+        continue;
+      assert(GlobalWidth >= OptionName.size() + OptionPrefixesSize);
+      size_t NumSpaces = GlobalWidth - OptionName.size() - OptionPrefixesSize;
+      outs() << OptionPrefix << OptionName;
+      if (OptionName.empty()) {
+        outs() << EmptyOption;
+        assert(NumSpaces >= EmptyOption.size());
+        NumSpaces -= EmptyOption.size();
+      }
+      if (!Description.empty())
+        outs().indent(NumSpaces) << ArgHelpPrefix << "  " << Description;
+      outs() << '\n';
     }
   } else {
     if (!O.HelpStr.empty())
diff --git a/lib/Support/Compression.cpp b/lib/Support/Compression.cpp
index 95261d4aad239b84cde6e85d9554db1e72e97d78..97d5ffaadf8273dd42955fb570282253c1027e3f 100644
--- a/lib/Support/Compression.cpp
+++ b/lib/Support/Compression.cpp
@@ -1,9 +1,8 @@
 //===--- Compression.cpp - Compression implementation ---------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Support/ConvertUTF.cpp b/lib/Support/ConvertUTF.cpp
index 8f02fae4f55848823199e7c5111a14110ed19cc5..e24a918c5c898206dc8b401c6214e4e9c652e9a9 100644
--- a/lib/Support/ConvertUTF.cpp
+++ b/lib/Support/ConvertUTF.cpp
@@ -1,9 +1,8 @@
 /*===--- ConvertUTF.c - Universal Character Names conversions ---------------===
  *
- *                     The LLVM Compiler Infrastructure
- *
- * This file is distributed under the University of Illinois Open Source
- * License. See LICENSE.TXT for details.
+ * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+ * See https://llvm.org/LICENSE.txt for license information.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
  *
  *===------------------------------------------------------------------------=*/
 /*
diff --git a/lib/Support/ConvertUTFWrapper.cpp b/lib/Support/ConvertUTFWrapper.cpp
index 6cb4f63762500de7b2bb8be6cf25604a7087fb92..eb4ead6b46b4500799903e338bcaaa91a9c66df7 100644
--- a/lib/Support/ConvertUTFWrapper.cpp
+++ b/lib/Support/ConvertUTFWrapper.cpp
@@ -1,9 +1,8 @@
 //===-- ConvertUTFWrapper.cpp - Wrap ConvertUTF.h with clang data types -----===
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Support/CrashRecoveryContext.cpp b/lib/Support/CrashRecoveryContext.cpp
index be4b5c3e01c30212ae7459762d1f122d344bfc1f..c2459256f8feb83a3313c69ee8a04995e327d1ba 100644
--- a/lib/Support/CrashRecoveryContext.cpp
+++ b/lib/Support/CrashRecoveryContext.cpp
@@ -1,9 +1,8 @@
 //===--- CrashRecoveryContext.cpp - Crash Recovery ------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Support/DAGDeltaAlgorithm.cpp b/lib/Support/DAGDeltaAlgorithm.cpp
index bd9f98b0b82d185d328750920c306e0198fbb784..4675fe3a9401037836d33965c51dd3a3530d8a97 100644
--- a/lib/Support/DAGDeltaAlgorithm.cpp
+++ b/lib/Support/DAGDeltaAlgorithm.cpp
@@ -1,9 +1,8 @@
 //===--- DAGDeltaAlgorithm.cpp - A DAG Minimization Algorithm --*- C++ -*--===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //===----------------------------------------------------------------------===//
 //
 // The algorithm we use attempts to exploit the dependency information by
diff --git a/lib/Support/DJB.cpp b/lib/Support/DJB.cpp
index 905dcf1b7e81a0ee55e917b35562fc24b3784509..a20dfc08e35f43dfabeba7246b27d325852638b0 100644
--- a/lib/Support/DJB.cpp
+++ b/lib/Support/DJB.cpp
@@ -1,9 +1,8 @@
 //===-- Support/DJB.cpp ---DJB Hash -----------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Support/DataExtractor.cpp b/lib/Support/DataExtractor.cpp
index 0199b300ba72dbf1ec1554125a5351688c33a8b4..6328d779b3d21f51a22842593ac1dc22d2fe5c04 100644
--- a/lib/Support/DataExtractor.cpp
+++ b/lib/Support/DataExtractor.cpp
@@ -1,9 +1,8 @@
 //===-- DataExtractor.cpp -------------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Support/Debug.cpp b/lib/Support/Debug.cpp
index 1a70017fee32cfe6898a6eb9462088255fe11219..737cd576ed80f7a52e26c02397e13df2cc498b09 100644
--- a/lib/Support/Debug.cpp
+++ b/lib/Support/Debug.cpp
@@ -1,9 +1,8 @@
 //===-- Debug.cpp - An easy way to add debug output to your code ----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Support/DeltaAlgorithm.cpp b/lib/Support/DeltaAlgorithm.cpp
index 50ea4e9ce0c61224e7809cd420975d9065a1e192..6aee69f4340547be4e7ca8f7afac3c21830c8206 100644
--- a/lib/Support/DeltaAlgorithm.cpp
+++ b/lib/Support/DeltaAlgorithm.cpp
@@ -1,9 +1,8 @@
 //===--- DeltaAlgorithm.cpp - A Set Minimization Algorithm -----*- C++ -*--===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //===----------------------------------------------------------------------===//
 
 #include "llvm/ADT/DeltaAlgorithm.h"
diff --git a/lib/Support/DynamicLibrary.cpp b/lib/Support/DynamicLibrary.cpp
index 530e92d99a900881bf0874e6663c572a2e35fca5..d23716016fb29bb2c65f338036b8f21f433d1d10 100644
--- a/lib/Support/DynamicLibrary.cpp
+++ b/lib/Support/DynamicLibrary.cpp
@@ -1,9 +1,8 @@
 //===-- DynamicLibrary.cpp - Runtime link/load libraries --------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Support/Errno.cpp b/lib/Support/Errno.cpp
index 2149f21281d3726e5afa86f622426e6361bcec64..485378dfdc983748fcc5866b84d9ad504078f336 100644
--- a/lib/Support/Errno.cpp
+++ b/lib/Support/Errno.cpp
@@ -1,9 +1,8 @@
 //===- Errno.cpp - errno support --------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Support/Error.cpp b/lib/Support/Error.cpp
index 0b0be8d48fa7a72b9088293b08b598ef6de4c929..72bc08af2ddb51de3bc530a92982142487524d4d 100644
--- a/lib/Support/Error.cpp
+++ b/lib/Support/Error.cpp
@@ -1,9 +1,8 @@
 //===----- lib/Support/Error.cpp - Error and associated utilities ---------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -54,6 +53,7 @@ namespace llvm {
 void ErrorInfoBase::anchor() {}
 char ErrorInfoBase::ID = 0;
 char ErrorList::ID = 0;
+void ECError::anchor() {}
 char ECError::ID = 0;
 char StringError::ID = 0;
 char FileError::ID = 0;
diff --git a/lib/Support/ErrorHandling.cpp b/lib/Support/ErrorHandling.cpp
index 21712c5c039e1f3b5907c2a02ff391ff660f1099..27b48670ac00898f19c1039eb9bd625f0ed4e93a 100644
--- a/lib/Support/ErrorHandling.cpp
+++ b/lib/Support/ErrorHandling.cpp
@@ -1,9 +1,8 @@
 //===- lib/Support/ErrorHandling.cpp - Callbacks for errors ---------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Support/FileCheck.cpp b/lib/Support/FileCheck.cpp
index 37986c96c0817f600ef34b16aecd5b93643f28ea..5f6ffab894bda32b199ecec0eb1c3137ff71a062 100644
--- a/lib/Support/FileCheck.cpp
+++ b/lib/Support/FileCheck.cpp
@@ -1,9 +1,8 @@
 //===- FileCheck.cpp - Check that File's Contents match what is expected --===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -901,16 +900,24 @@ static void PrintMatch(bool ExpectedMatch, const SourceMgr &SM,
                        StringMap<StringRef> &VariableTable, size_t MatchPos,
                        size_t MatchLen, const FileCheckRequest &Req,
                        std::vector<FileCheckDiag> *Diags) {
+  bool PrintDiag = true;
   if (ExpectedMatch) {
     if (!Req.Verbose)
       return;
     if (!Req.VerboseVerbose && Pat.getCheckTy() == Check::CheckEOF)
       return;
+    // Due to their verbosity, we don't print verbose diagnostics here if we're
+    // gathering them for a different rendering, but we always print other
+    // diagnostics.
+    PrintDiag = !Diags;
   }
   SMRange MatchRange = ProcessMatchResult(
       ExpectedMatch ? FileCheckDiag::MatchFoundAndExpected
                     : FileCheckDiag::MatchFoundButExcluded,
       SM, Loc, Pat.getCheckTy(), Buffer, MatchPos, MatchLen, Diags);
+  if (!PrintDiag)
+    return;
+
   std::string Message = formatv("{0}: {1} string found in input",
                                 Pat.getCheckTy().getDescription(Prefix),
                                 (ExpectedMatch ? "expected" : "excluded"))
@@ -941,27 +948,37 @@ static void PrintNoMatch(bool ExpectedMatch, const SourceMgr &SM,
                          StringRef Buffer, StringMap<StringRef> &VariableTable,
                          bool VerboseVerbose,
                          std::vector<FileCheckDiag> *Diags) {
-  if (!ExpectedMatch && !VerboseVerbose)
+  bool PrintDiag = true;
+  if (!ExpectedMatch) {
+    if (!VerboseVerbose)
+      return;
+    // Due to their verbosity, we don't print verbose diagnostics here if we're
+    // gathering them for a different rendering, but we always print other
+    // diagnostics.
+    PrintDiag = !Diags;
+  }
+
+  // If the current position is at the end of a line, advance to the start of
+  // the next line.
+  Buffer = Buffer.substr(Buffer.find_first_not_of(" \t\n\r"));
+  SMRange SearchRange = ProcessMatchResult(
+      ExpectedMatch ? FileCheckDiag::MatchNoneButExpected
+                    : FileCheckDiag::MatchNoneAndExcluded,
+      SM, Loc, Pat.getCheckTy(), Buffer, 0, Buffer.size(), Diags);
+  if (!PrintDiag)
     return;
 
-  // Otherwise, we have an error, emit an error message.
+  // Print "not found" diagnostic.
   std::string Message = formatv("{0}: {1} string not found in input",
                                 Pat.getCheckTy().getDescription(Prefix),
                                 (ExpectedMatch ? "expected" : "excluded"))
                             .str();
   if (Pat.getCount() > 1)
     Message += formatv(" ({0} out of {1})", MatchedCount, Pat.getCount()).str();
-
   SM.PrintMessage(
       Loc, ExpectedMatch ? SourceMgr::DK_Error : SourceMgr::DK_Remark, Message);
 
-  // Print the "scanning from here" line.  If the current position is at the
-  // end of a line, advance to the start of the next line.
-  Buffer = Buffer.substr(Buffer.find_first_not_of(" \t\n\r"));
-  SMRange SearchRange = ProcessMatchResult(
-      ExpectedMatch ? FileCheckDiag::MatchNoneButExpected
-                    : FileCheckDiag::MatchNoneAndExcluded,
-      SM, Loc, Pat.getCheckTy(), Buffer, 0, Buffer.size(), Diags);
+  // Print the "scanning from here" line.
   SM.PrintMessage(SearchRange.Start, SourceMgr::DK_Note, "scanning from here");
 
   // Allow the pattern to print additional information if desired.
@@ -1276,13 +1293,17 @@ FileCheckString::CheckDag(const SourceMgr &SM, StringRef Buffer,
         break;
       }
       if (Req.VerboseVerbose) {
-        SMLoc OldStart = SMLoc::getFromPointer(Buffer.data() + MI->Pos);
-        SMLoc OldEnd = SMLoc::getFromPointer(Buffer.data() + MI->End);
-        SMRange OldRange(OldStart, OldEnd);
-        SM.PrintMessage(OldStart, SourceMgr::DK_Note,
-                        "match discarded, overlaps earlier DAG match here",
-                        {OldRange});
-        if (Diags)
+        // Due to their verbosity, we don't print verbose diagnostics here if
+        // we're gathering them for a different rendering, but we always print
+        // other diagnostics.
+        if (!Diags) {
+          SMLoc OldStart = SMLoc::getFromPointer(Buffer.data() + MI->Pos);
+          SMLoc OldEnd = SMLoc::getFromPointer(Buffer.data() + MI->End);
+          SMRange OldRange(OldStart, OldEnd);
+          SM.PrintMessage(OldStart, SourceMgr::DK_Note,
+                          "match discarded, overlaps earlier DAG match here",
+                          {OldRange});
+        } else
           Diags->rbegin()->MatchTy = FileCheckDiag::MatchFoundButDiscarded;
       }
       MatchPos = MI->End;
diff --git a/lib/Support/FileOutputBuffer.cpp b/lib/Support/FileOutputBuffer.cpp
index b8223126227df9c0d090afe6918735473a2b1a86..b0b817b4bdd03bf9b753654f6eda5f4f1cd29163 100644
--- a/lib/Support/FileOutputBuffer.cpp
+++ b/lib/Support/FileOutputBuffer.cpp
@@ -1,9 +1,8 @@
 //===- FileOutputBuffer.cpp - File Output Buffer ----------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -88,6 +87,12 @@ public:
   size_t getBufferSize() const override { return Buffer.size(); }
 
   Error commit() override {
+    if (FinalPath == "-") {
+      llvm::outs() << StringRef((const char *)Buffer.base(), Buffer.size());
+      llvm::outs().flush();
+      return Error::success();
+    }
+
     using namespace sys::fs;
     int FD;
     std::error_code EC;
@@ -115,40 +120,38 @@ createInMemoryBuffer(StringRef Path, size_t Size, unsigned Mode) {
   return llvm::make_unique<InMemoryBuffer>(Path, MB, Mode);
 }
 
-static Expected<std::unique_ptr<OnDiskBuffer>>
-createOnDiskBuffer(StringRef Path, size_t Size, bool InitExisting,
-                   unsigned Mode) {
+static Expected<std::unique_ptr<FileOutputBuffer>>
+createOnDiskBuffer(StringRef Path, size_t Size, unsigned Mode) {
   Expected<fs::TempFile> FileOrErr =
       fs::TempFile::create(Path + ".tmp%%%%%%%", Mode);
   if (!FileOrErr)
     return FileOrErr.takeError();
   fs::TempFile File = std::move(*FileOrErr);
 
-  if (InitExisting) {
-    if (auto EC = sys::fs::copy_file(Path, File.FD))
-      return errorCodeToError(EC);
-  } else {
 #ifndef _WIN32
-    // On Windows, CreateFileMapping (the mmap function on Windows)
-    // automatically extends the underlying file. We don't need to
-    // extend the file beforehand. _chsize (ftruncate on Windows) is
-    // pretty slow just like it writes specified amount of bytes,
-    // so we should avoid calling that function.
-    if (auto EC = fs::resize_file(File.FD, Size)) {
-      consumeError(File.discard());
-      return errorCodeToError(EC);
-    }
-#endif
+  // On Windows, CreateFileMapping (the mmap function on Windows)
+  // automatically extends the underlying file. We don't need to
+  // extend the file beforehand. _chsize (ftruncate on Windows) is
+  // pretty slow just like it writes specified amount of bytes,
+  // so we should avoid calling that function.
+  if (auto EC = fs::resize_file(File.FD, Size)) {
+    consumeError(File.discard());
+    return errorCodeToError(EC);
   }
+#endif
 
   // Mmap it.
   std::error_code EC;
   auto MappedFile = llvm::make_unique<fs::mapped_file_region>(
       File.FD, fs::mapped_file_region::readwrite, Size, 0, EC);
+
+  // mmap(2) can fail if the underlying filesystem does not support it.
+  // If that happens, we fall back to in-memory buffer as the last resort.
   if (EC) {
     consumeError(File.discard());
-    return errorCodeToError(EC);
+    return createInMemoryBuffer(Path, Size, Mode);
   }
+
   return llvm::make_unique<OnDiskBuffer>(Path, std::move(File),
                                          std::move(MappedFile));
 }
@@ -156,6 +159,10 @@ createOnDiskBuffer(StringRef Path, size_t Size, bool InitExisting,
 // Create an instance of FileOutputBuffer.
 Expected<std::unique_ptr<FileOutputBuffer>>
 FileOutputBuffer::create(StringRef Path, size_t Size, unsigned Flags) {
+  // Handle "-" as stdout just like llvm::raw_ostream does.
+  if (Path == "-")
+    return createInMemoryBuffer("-", Size, /*Mode=*/0);
+
   unsigned Mode = fs::all_read | fs::all_write;
   if (Flags & F_executable)
     Mode |= fs::all_exe;
@@ -163,15 +170,6 @@ FileOutputBuffer::create(StringRef Path, size_t Size, unsigned Flags) {
   fs::file_status Stat;
   fs::status(Path, Stat);
 
-  if ((Flags & F_modify) && Size == size_t(-1)) {
-    if (Stat.type() == fs::file_type::regular_file)
-      Size = Stat.getSize();
-    else if (Stat.type() == fs::file_type::file_not_found)
-      return errorCodeToError(errc::no_such_file_or_directory);
-    else
-      return errorCodeToError(errc::invalid_argument);
-  }
-
   // Usually, we want to create OnDiskBuffer to create a temporary file in
   // the same directory as the destination file and atomically replaces it
   // by rename(2).
@@ -186,7 +184,7 @@ FileOutputBuffer::create(StringRef Path, size_t Size, unsigned Flags) {
   case fs::file_type::regular_file:
   case fs::file_type::file_not_found:
   case fs::file_type::status_error:
-    return createOnDiskBuffer(Path, Size, !!(Flags & F_modify), Mode);
+    return createOnDiskBuffer(Path, Size, Mode);
   default:
     return createInMemoryBuffer(Path, Size, Mode);
   }
diff --git a/lib/Support/FileUtilities.cpp b/lib/Support/FileUtilities.cpp
index 39dbefff5b7083467f5ee52bd410f35613110b0b..62eb7bfda195d93c84502ec948e4b934c2db7afd 100644
--- a/lib/Support/FileUtilities.cpp
+++ b/lib/Support/FileUtilities.cpp
@@ -1,9 +1,8 @@
 //===- Support/FileUtilities.cpp - File System Utilities ------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Support/FoldingSet.cpp b/lib/Support/FoldingSet.cpp
index ee69a64ac97bda472ac5940fc62e7833a56c0238..ce6f196e1060f7d857d5466034997293e66244c4 100644
--- a/lib/Support/FoldingSet.cpp
+++ b/lib/Support/FoldingSet.cpp
@@ -1,9 +1,8 @@
 //===-- Support/FoldingSet.cpp - Uniquing Hash Set --------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Support/FormatVariadic.cpp b/lib/Support/FormatVariadic.cpp
index 6dd133e6c50a2c7e643189525a77d84dd5d7e056..f9e89f69b528cd5a8975d96233efa30b3c33b02e 100644
--- a/lib/Support/FormatVariadic.cpp
+++ b/lib/Support/FormatVariadic.cpp
@@ -1,9 +1,8 @@
 //===- FormatVariadic.cpp - Format string parsing and analysis ----*-C++-*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //===----------------------------------------------------------------------===//
 
 #include "llvm/Support/FormatVariadic.h"
@@ -152,3 +151,5 @@ formatv_object_base::parseFormatString(StringRef Fmt) {
   }
   return Replacements;
 }
+
+void detail::format_adapter::anchor() { }
diff --git a/lib/Support/FormattedStream.cpp b/lib/Support/FormattedStream.cpp
index b0cb06c1daa2f1dfc390412ce68f3b00d884da1e..4eb747038bb9ee8a81753cfd9b891f6616839f7b 100644
--- a/lib/Support/FormattedStream.cpp
+++ b/lib/Support/FormattedStream.cpp
@@ -1,9 +1,8 @@
 //===-- llvm/Support/FormattedStream.cpp - Formatted streams ----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Support/GlobPattern.cpp b/lib/Support/GlobPattern.cpp
index 4ea110301f16de42e17b29945a802663c0751b94..6011be86d77fba053e9b27672e13a3eb1158dd88 100644
--- a/lib/Support/GlobPattern.cpp
+++ b/lib/Support/GlobPattern.cpp
@@ -1,9 +1,8 @@
 //===-- GlobPattern.cpp - Glob pattern matcher implementation -------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Support/GraphWriter.cpp b/lib/Support/GraphWriter.cpp
index 9335daffc3e2a3cb583f35e5cc973b6996389571..c689a81925d4cf5174d76dc8008d12379ba0706e 100644
--- a/lib/Support/GraphWriter.cpp
+++ b/lib/Support/GraphWriter.cpp
@@ -1,9 +1,8 @@
 //===- GraphWriter.cpp - Implements GraphWriter support routines ----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Support/Hashing.cpp b/lib/Support/Hashing.cpp
index 7de25cec73715fb1f7af361a0f4eea2756053599..1b20a670434f12b624b9d1b7f24c1ce615f7dd39 100644
--- a/lib/Support/Hashing.cpp
+++ b/lib/Support/Hashing.cpp
@@ -1,9 +1,8 @@
 //===-------------- lib/Support/Hashing.cpp -------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Support/Host.cpp b/lib/Support/Host.cpp
index d5a688c7fb9b0b1ca5aed00d54122117c564c1af..35bc97222410963c0cefd691f8ddb31a5865b3d1 100644
--- a/lib/Support/Host.cpp
+++ b/lib/Support/Host.cpp
@@ -1,9 +1,8 @@
 //===-- Host.cpp - Implement OS Host Concept --------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -331,7 +330,19 @@ StringRef sys::detail::getHostCPUNameForBPF() {
 #if !defined(__linux__) || !defined(__x86_64__)
   return "generic";
 #else
-  uint8_t insns[40] __attribute__ ((aligned (8))) =
+  uint8_t v3_insns[40] __attribute__ ((aligned (8))) =
+      /* BPF_MOV64_IMM(BPF_REG_0, 0) */
+    { 0xb7, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
+      /* BPF_MOV64_IMM(BPF_REG_2, 1) */
+      0xb7, 0x2, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0,
+      /* BPF_JMP32_REG(BPF_JLT, BPF_REG_0, BPF_REG_2, 1) */
+      0xae, 0x20, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0,
+      /* BPF_MOV64_IMM(BPF_REG_0, 1) */
+      0xb7, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0,
+      /* BPF_EXIT_INSN() */
+      0x95, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 };
+
+  uint8_t v2_insns[40] __attribute__ ((aligned (8))) =
       /* BPF_MOV64_IMM(BPF_REG_0, 0) */
     { 0xb7, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
       /* BPF_MOV64_IMM(BPF_REG_2, 1) */
@@ -356,10 +367,23 @@ StringRef sys::detail::getHostCPUNameForBPF() {
   } attr = {};
   attr.prog_type = 1; /* BPF_PROG_TYPE_SOCKET_FILTER */
   attr.insn_cnt = 5;
-  attr.insns = (uint64_t)insns;
+  attr.insns = (uint64_t)v3_insns;
   attr.license = (uint64_t)"DUMMY";
 
-  int fd = syscall(321 /* __NR_bpf */, 5 /* BPF_PROG_LOAD */, &attr, sizeof(attr));
+  int fd = syscall(321 /* __NR_bpf */, 5 /* BPF_PROG_LOAD */, &attr,
+                   sizeof(attr));
+  if (fd >= 0) {
+    close(fd);
+    return "v3";
+  }
+
+  /* Clear the whole attr in case its content changed by syscall. */
+  memset(&attr, 0, sizeof(attr));
+  attr.prog_type = 1; /* BPF_PROG_TYPE_SOCKET_FILTER */
+  attr.insn_cnt = 5;
+  attr.insns = (uint64_t)v2_insns;
+  attr.license = (uint64_t)"DUMMY";
+  fd = syscall(321 /* __NR_bpf */, 5 /* BPF_PROG_LOAD */, &attr, sizeof(attr));
   if (fd >= 0) {
     close(fd);
     return "v2";
@@ -1235,6 +1259,7 @@ bool sys::getHostCPUFeatures(StringMap<bool> &Features) {
 
   Features["cmov"]   = (EDX >> 15) & 1;
   Features["mmx"]    = (EDX >> 23) & 1;
+  Features["fxsr"]   = (EDX >> 24) & 1;
   Features["sse"]    = (EDX >> 25) & 1;
   Features["sse2"]   = (EDX >> 26) & 1;
 
@@ -1298,6 +1323,7 @@ bool sys::getHostCPUFeatures(StringMap<bool> &Features) {
   Features["bmi2"]       = HasLeaf7 && ((EBX >>  8) & 1);
   Features["invpcid"]    = HasLeaf7 && ((EBX >> 10) & 1);
   Features["rtm"]        = HasLeaf7 && ((EBX >> 11) & 1);
+  Features["mpx"]        = HasLeaf7 && ((EBX >> 14) & 1);
   // AVX512 is only supported if the OS supports the context save for it.
   Features["avx512f"]    = HasLeaf7 && ((EBX >> 16) & 1) && HasAVX512Save;
   Features["avx512dq"]   = HasLeaf7 && ((EBX >> 17) & 1) && HasAVX512Save;
diff --git a/lib/Support/InitLLVM.cpp b/lib/Support/InitLLVM.cpp
index c008d0455c99787b0fbac916c479ce760b286b3c..eeb3fd3b670da2ea1361b6db4615f71d9a2c5f6f 100644
--- a/lib/Support/InitLLVM.cpp
+++ b/lib/Support/InitLLVM.cpp
@@ -1,9 +1,8 @@
 //===-- InitLLVM.cpp -----------------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Support/IntEqClasses.cpp b/lib/Support/IntEqClasses.cpp
index cb6e3a19e8d32492c42b9b4f2e1210b86a4ee5e1..4a976dcefc65fc24f61599b949d7c8016260fa91 100644
--- a/lib/Support/IntEqClasses.cpp
+++ b/lib/Support/IntEqClasses.cpp
@@ -1,9 +1,8 @@
 //===-- llvm/ADT/IntEqClasses.cpp - Equivalence Classes of Integers -------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Support/IntervalMap.cpp b/lib/Support/IntervalMap.cpp
index e11a7f2eb843ee0e7a507148bf1cdb8c0c1e1578..f15c7c9403c36f2b6415470a6381250784172b3c 100644
--- a/lib/Support/IntervalMap.cpp
+++ b/lib/Support/IntervalMap.cpp
@@ -1,9 +1,8 @@
 //===- lib/Support/IntervalMap.cpp - A sorted interval map ----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Support/ItaniumManglingCanonicalizer.cpp b/lib/Support/ItaniumManglingCanonicalizer.cpp
index e55dcd7618095cb7dea033fbf5381efdfde2057d..da6514f7170bb3d84ccd06efd537050e4dece50f 100644
--- a/lib/Support/ItaniumManglingCanonicalizer.cpp
+++ b/lib/Support/ItaniumManglingCanonicalizer.cpp
@@ -1,9 +1,8 @@
 //===----------------- ItaniumManglingCanonicalizer.cpp -------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is dual licensed under the MIT and the University of Illinois Open
-// Source Licenses. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -22,6 +21,7 @@ using namespace llvm;
 using llvm::itanium_demangle::ForwardTemplateReference;
 using llvm::itanium_demangle::Node;
 using llvm::itanium_demangle::NodeKind;
+using llvm::itanium_demangle::StringView;
 
 namespace {
 struct FoldingSetNodeIDBuilder {
diff --git a/lib/Support/JSON.cpp b/lib/Support/JSON.cpp
index d468013fb94a53e0e5e8927685cefeca6ce2399e..790b28f6e316b72c294d8a326446158dd0c4996b 100644
--- a/lib/Support/JSON.cpp
+++ b/lib/Support/JSON.cpp
@@ -1,9 +1,8 @@
 //=== JSON.cpp - JSON value, parsing and serialization - C++ -----------*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===---------------------------------------------------------------------===//
 
@@ -182,6 +181,12 @@ bool operator==(const Value &L, const Value &R) {
   case Value::Boolean:
     return *L.getAsBoolean() == *R.getAsBoolean();
   case Value::Number:
+    // Workaround for https://gcc.gnu.org/bugzilla/show_bug.cgi?id=323
+    // The same integer must convert to the same double, per the standard.
+    // However we see 64-vs-80-bit precision comparisons with gcc-7 -O3 -m32.
+    // So we avoid floating point promotion for exact comparisons.
+    if (L.Type == Value::T_Integer || R.Type == Value::T_Integer)
+      return L.getAsInteger() == R.getAsInteger();
     return *L.getAsNumber() == *R.getAsNumber();
   case Value::String:
     return *L.getAsString() == *R.getAsString();
diff --git a/lib/Support/JamCRC.cpp b/lib/Support/JamCRC.cpp
index 17c55f565e08227b005cadaa9687145f32fed01d..e043a3c33c28837d046ba966153f7460124f52a5 100644
--- a/lib/Support/JamCRC.cpp
+++ b/lib/Support/JamCRC.cpp
@@ -1,9 +1,8 @@
 //===-- JamCRC.cpp - Cyclic Redundancy Check --------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Support/KnownBits.cpp b/lib/Support/KnownBits.cpp
index ac790ebed3522c3dd082b4dc0118965a99ef794e..9988314fabb9b0cc6af8dcc9956a43359c5213c9 100644
--- a/lib/Support/KnownBits.cpp
+++ b/lib/Support/KnownBits.cpp
@@ -1,9 +1,8 @@
 //===-- KnownBits.cpp - Stores known zeros/ones ---------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Support/LEB128.cpp b/lib/Support/LEB128.cpp
index 449626f2d451ca3c4a9d6c234f4c8e0a473f469c..d41b673e9c8a5adf11a9f9856433ab072b693dd8 100644
--- a/lib/Support/LEB128.cpp
+++ b/lib/Support/LEB128.cpp
@@ -1,9 +1,8 @@
 //===- LEB128.cpp - LEB128 utility functions implementation -----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Support/LLVMBuild.txt b/lib/Support/LLVMBuild.txt
index 4c75139704153412b02646fdaa85e07eac046b2b..0fa389be0707735e346562acaaa76ba824e0d677 100644
--- a/lib/Support/LLVMBuild.txt
+++ b/lib/Support/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/Support/LLVMBuild.txt ------------------------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Support/LineIterator.cpp b/lib/Support/LineIterator.cpp
index 5baa1a37f385df9f8e099726dea3601b6107f41d..164436a2c48ed531785e69a21341264ac9300d0f 100644
--- a/lib/Support/LineIterator.cpp
+++ b/lib/Support/LineIterator.cpp
@@ -1,9 +1,8 @@
 //===- LineIterator.cpp - Implementation of line iteration ----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Support/LockFileManager.cpp b/lib/Support/LockFileManager.cpp
index c166230ba3a3cc207f12e3ef3219d5a7b730229b..10181192afbd8da9eccb4069808ac98a49ded660 100644
--- a/lib/Support/LockFileManager.cpp
+++ b/lib/Support/LockFileManager.cpp
@@ -1,9 +1,8 @@
 //===--- LockFileManager.cpp - File-level Locking Utility------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Support/LowLevelType.cpp b/lib/Support/LowLevelType.cpp
index cb2187405d6bac7f7a31e6d1ff38b8e077797b70..cffcfff7c735e17e71f271fdacdfe6712ed477cc 100644
--- a/lib/Support/LowLevelType.cpp
+++ b/lib/Support/LowLevelType.cpp
@@ -1,9 +1,8 @@
 //===-- llvm/Support/LowLevelType.cpp -------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Support/ManagedStatic.cpp b/lib/Support/ManagedStatic.cpp
index 74f71a3850270b1c4ef9f12c14304e30ca8e6986..28ceb1a70e42ff0487dea416c4839aaaf77a71ea 100644
--- a/lib/Support/ManagedStatic.cpp
+++ b/lib/Support/ManagedStatic.cpp
@@ -1,9 +1,8 @@
 //===-- ManagedStatic.cpp - Static Global wrapper -------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Support/MathExtras.cpp b/lib/Support/MathExtras.cpp
index ba0924540cebc33829c104ab0401ff1b2b4a7ee5..87c7101c424b22168f8a2cac75f49e4868bfa1b0 100644
--- a/lib/Support/MathExtras.cpp
+++ b/lib/Support/MathExtras.cpp
@@ -1,9 +1,8 @@
 //===-- MathExtras.cpp - Implement the MathExtras header --------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Support/Memory.cpp b/lib/Support/Memory.cpp
index c245eedd2c161f6308f7d4cd9647980dd02a4ce0..a9d33759dd6fda2a7c348261bbe9078dc928205c 100644
--- a/lib/Support/Memory.cpp
+++ b/lib/Support/Memory.cpp
@@ -1,9 +1,8 @@
 //===- Memory.cpp - Memory Handling Support ---------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Support/MemoryBuffer.cpp b/lib/Support/MemoryBuffer.cpp
index ef9159bac284021b2ec1fadd39b893c3d258d840..3f13d4d3552adec7ccafee43e67b32ad0520d021 100644
--- a/lib/Support/MemoryBuffer.cpp
+++ b/lib/Support/MemoryBuffer.cpp
@@ -1,9 +1,8 @@
 //===--- MemoryBuffer.cpp - Memory Buffer implementation ------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Support/Mutex.cpp b/lib/Support/Mutex.cpp
index 7138c7a4b984c3de7d49aec5627242df6ffb495f..69b7b8126ab1cef22ad3c6880601cfaec4b81603 100644
--- a/lib/Support/Mutex.cpp
+++ b/lib/Support/Mutex.cpp
@@ -1,9 +1,8 @@
 //===- Mutex.cpp - Mutual Exclusion Lock ------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Support/NativeFormatting.cpp b/lib/Support/NativeFormatting.cpp
index 85b4bfb81568dd1bda1f1e971cf743a84b1e15e5..3731e0c5635991d2ac373118f4d56a8bec3b1d49 100644
--- a/lib/Support/NativeFormatting.cpp
+++ b/lib/Support/NativeFormatting.cpp
@@ -1,9 +1,8 @@
 //===- NativeFormatting.cpp - Low level formatting helpers -------*- C++-*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Support/Optional.cpp b/lib/Support/Optional.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..2425739c845d39830335630d9af23a8920d0421d
--- /dev/null
+++ b/lib/Support/Optional.cpp
@@ -0,0 +1,14 @@
+//===- Optional.cpp - Optional values ---------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ADT/Optional.h"
+#include "llvm/Support/raw_ostream.h"
+
+llvm::raw_ostream &llvm::operator<<(raw_ostream &OS, NoneType) {
+  return OS << "None";
+}
diff --git a/lib/Support/Options.cpp b/lib/Support/Options.cpp
index 71258450efa6896506fe7ba05310fb15a77d18c2..770b7381c20ee5d5c84f375ec771800b6920981a 100644
--- a/lib/Support/Options.cpp
+++ b/lib/Support/Options.cpp
@@ -1,9 +1,8 @@
 //===- llvm/Support/Options.cpp - Debug options support ---------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Support/Parallel.cpp b/lib/Support/Parallel.cpp
index 1844003b9d3dcbfd26a90eae350e7b4a5967cf7f..d40fce67e9a05a555aa9da4a5ed57b08b8656448 100644
--- a/lib/Support/Parallel.cpp
+++ b/lib/Support/Parallel.cpp
@@ -1,9 +1,8 @@
 //===- llvm/Support/Parallel.cpp - Parallel algorithms --------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Support/Path.cpp b/lib/Support/Path.cpp
index a3e6941bd62277d69c41ae9807d8ffc2fea3739d..8f580c66d2442d0a42744137b006b073b6d800e9 100644
--- a/lib/Support/Path.cpp
+++ b/lib/Support/Path.cpp
@@ -1,9 +1,8 @@
 //===-- Path.cpp - Implement OS Path Concept ------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -849,9 +848,8 @@ getPotentiallyUniqueTempFileName(const Twine &Prefix, StringRef Suffix,
   return createTemporaryFile(Prefix, Suffix, Dummy, ResultPath, FS_Name);
 }
 
-static std::error_code make_absolute(const Twine &current_directory,
-                                     SmallVectorImpl<char> &path,
-                                     bool use_current_directory) {
+void make_absolute(const Twine &current_directory,
+                   SmallVectorImpl<char> &path) {
   StringRef p(path.data(), path.size());
 
   bool rootDirectory = path::has_root_directory(p);
@@ -860,14 +858,11 @@ static std::error_code make_absolute(const Twine &current_directory,
 
   // Already absolute.
   if (rootName && rootDirectory)
-    return std::error_code();
+    return;
 
   // All of the following conditions will need the current directory.
   SmallString<128> current_dir;
-  if (use_current_directory)
-    current_directory.toVector(current_dir);
-  else if (std::error_code ec = current_path(current_dir))
-    return ec;
+  current_directory.toVector(current_dir);
 
   // Relative path. Prepend the current directory.
   if (!rootName && !rootDirectory) {
@@ -875,7 +870,7 @@ static std::error_code make_absolute(const Twine &current_directory,
     path::append(current_dir, p);
     // Set path to the result.
     path.swap(current_dir);
-    return std::error_code();
+    return;
   }
 
   if (!rootName && rootDirectory) {
@@ -884,7 +879,7 @@ static std::error_code make_absolute(const Twine &current_directory,
     path::append(curDirRootName, p);
     // Set path to the result.
     path.swap(curDirRootName);
-    return std::error_code();
+    return;
   }
 
   if (rootName && !rootDirectory) {
@@ -896,20 +891,23 @@ static std::error_code make_absolute(const Twine &current_directory,
     SmallString<128> res;
     path::append(res, pRootName, bRootDirectory, bRelativePath, pRelativePath);
     path.swap(res);
-    return std::error_code();
+    return;
   }
 
   llvm_unreachable("All rootName and rootDirectory combinations should have "
                    "occurred above!");
 }
 
-std::error_code make_absolute(const Twine &current_directory,
-                              SmallVectorImpl<char> &path) {
-  return make_absolute(current_directory, path, true);
-}
-
 std::error_code make_absolute(SmallVectorImpl<char> &path) {
-  return make_absolute(Twine(), path, false);
+  if (path::is_absolute(path))
+    return {};
+
+  SmallString<128> current_dir;
+  if (std::error_code ec = current_path(current_dir))
+    return ec;
+
+  make_absolute(current_dir, path);
+  return {};
 }
 
 std::error_code create_directories(const Twine &Path, bool IgnoreExisting,
diff --git a/lib/Support/PluginLoader.cpp b/lib/Support/PluginLoader.cpp
index 358137f08f5f86fbb1eecc9adf2ecfff905b1927..6fe195ffda7ac8fff5c97d15c6a8a7c928fca96b 100644
--- a/lib/Support/PluginLoader.cpp
+++ b/lib/Support/PluginLoader.cpp
@@ -1,9 +1,8 @@
 //===-- PluginLoader.cpp - Implement -load command line option ------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Support/PrettyStackTrace.cpp b/lib/Support/PrettyStackTrace.cpp
index 206de91ae2393642bfbe6a979951da81b6bcfdc3..6e0b35b5e532e5e36f25523305fa176c7bc991a7 100644
--- a/lib/Support/PrettyStackTrace.cpp
+++ b/lib/Support/PrettyStackTrace.cpp
@@ -1,9 +1,8 @@
 //===- PrettyStackTrace.cpp - Pretty Crash Handling -----------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Support/Process.cpp b/lib/Support/Process.cpp
index f32355aefbb7585940322c1f638b3db8d180568f..5b647100815976f4fc09add93bb48a557c762b76 100644
--- a/lib/Support/Process.cpp
+++ b/lib/Support/Process.cpp
@@ -1,9 +1,8 @@
 //===-- Process.cpp - Implement OS Process Concept --------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Support/Program.cpp b/lib/Support/Program.cpp
index 63cdcdaabee904c0c5c36855de95d3785cbf4e60..0a9363c59fc683093a04108d4ef0b3bb502c4bdb 100644
--- a/lib/Support/Program.cpp
+++ b/lib/Support/Program.cpp
@@ -1,9 +1,8 @@
 //===-- Program.cpp - Implement OS Program Concept --------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Support/RWMutex.cpp b/lib/Support/RWMutex.cpp
index 8b6d74e49f31a7a8013efa244f6693b473510741..7ce856b716c6081957ea34df45bf9398afa74cc8 100644
--- a/lib/Support/RWMutex.cpp
+++ b/lib/Support/RWMutex.cpp
@@ -1,9 +1,8 @@
 //===- RWMutex.cpp - Reader/Writer Mutual Exclusion Lock --------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Support/RandomNumberGenerator.cpp b/lib/Support/RandomNumberGenerator.cpp
index df0d87fab021b558cf58ac095a0e985bcf096b69..72bbdbf1e57e5d86d3d7a8378f78689f04ef88c5 100644
--- a/lib/Support/RandomNumberGenerator.cpp
+++ b/lib/Support/RandomNumberGenerator.cpp
@@ -1,9 +1,8 @@
 //===-- RandomNumberGenerator.cpp - Implement RNG class -------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Support/Regex.cpp b/lib/Support/Regex.cpp
index 48caab1315264d9fdb92c881b6dc3e9d368d2cea..4c1b0703802446de66af74a3d8595922d7ac5acf 100644
--- a/lib/Support/Regex.cpp
+++ b/lib/Support/Regex.cpp
@@ -1,9 +1,8 @@
 //===-- Regex.cpp - Regular Expression matcher implementation -------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Support/SHA1.cpp b/lib/Support/SHA1.cpp
index 3007a78d5e22b206fa8d47ad323e74bc8b871a4c..47a5f07fbe7b1b44b18a17bd854bdc2b6b943972 100644
--- a/lib/Support/SHA1.cpp
+++ b/lib/Support/SHA1.cpp
@@ -1,9 +1,8 @@
 //====- SHA1.cpp - Private copy of the SHA1 implementation ---*- C++ -* ======//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Support/ScaledNumber.cpp b/lib/Support/ScaledNumber.cpp
index 807c9fa521decfef749a327a09fdfc4b18da9445..54d4cc33410b17653e33b0bc8282a31d67552140 100644
--- a/lib/Support/ScaledNumber.cpp
+++ b/lib/Support/ScaledNumber.cpp
@@ -1,9 +1,8 @@
 //==- lib/Support/ScaledNumber.cpp - Support for scaled numbers -*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Support/Signals.cpp b/lib/Support/Signals.cpp
index 333f492d4589489840567abc58db7699c18b6994..570b18f9a207b57e47dcc87946c33510b24216a1 100644
--- a/lib/Support/Signals.cpp
+++ b/lib/Support/Signals.cpp
@@ -1,9 +1,8 @@
 //===- Signals.cpp - Signal Handling support --------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Support/SmallPtrSet.cpp b/lib/Support/SmallPtrSet.cpp
index fed4a17d66350630f7563940ee6dc8c5cb5cf0f8..f60464c8e756170e3371bc527c54f3f783c149cc 100644
--- a/lib/Support/SmallPtrSet.cpp
+++ b/lib/Support/SmallPtrSet.cpp
@@ -1,9 +1,8 @@
 //===- llvm/ADT/SmallPtrSet.cpp - 'Normally small' pointer set ------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Support/SmallVector.cpp b/lib/Support/SmallVector.cpp
index 1070c6672edcec422476b15a0837a1cdf6bc157c..36f0a81f6b00deb0c3fa8b3e816e95dc7ddda95b 100644
--- a/lib/Support/SmallVector.cpp
+++ b/lib/Support/SmallVector.cpp
@@ -1,9 +1,8 @@
 //===- llvm/ADT/SmallVector.cpp - 'Normally small' vectors ----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Support/SourceMgr.cpp b/lib/Support/SourceMgr.cpp
index a55ad881d01271490839974aea50b40152137ca8..401f6aec4caa00eec26c0cdde06a77339f3a8adc 100644
--- a/lib/Support/SourceMgr.cpp
+++ b/lib/Support/SourceMgr.cpp
@@ -1,9 +1,8 @@
 //===- SourceMgr.cpp - Manager for Simple Source Buffers & Diagnostics ----===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Support/SpecialCaseList.cpp b/lib/Support/SpecialCaseList.cpp
index bf807e66e02c19886d47f91d832bedd055544fc7..96e09f9552bbcef20911603517aedd61ab9c136f 100644
--- a/lib/Support/SpecialCaseList.cpp
+++ b/lib/Support/SpecialCaseList.cpp
@@ -1,9 +1,8 @@
 //===-- SpecialCaseList.cpp - special case list for sanitizers ------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Support/Statistic.cpp b/lib/Support/Statistic.cpp
index d57300a75d1d24b3ac647288a392b03f2314558a..0d8b5b2e0abcc075d0ad26912de13f7234944ab9 100644
--- a/lib/Support/Statistic.cpp
+++ b/lib/Support/Statistic.cpp
@@ -1,9 +1,8 @@
 //===-- Statistic.cpp - Easy way to expose stats information --------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Support/StringExtras.cpp b/lib/Support/StringExtras.cpp
index 386d74a4798353d52e80a8e98c99b4a39ada71b9..bf28b2be56578a85f438b454c0b8833a62c24a05 100644
--- a/lib/Support/StringExtras.cpp
+++ b/lib/Support/StringExtras.cpp
@@ -1,9 +1,8 @@
 //===-- StringExtras.cpp - Implement the StringExtras header --------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Support/StringMap.cpp b/lib/Support/StringMap.cpp
index c1f707ce50a5cee8431b36725a0d34529b2e71b8..6b5ea020dd46dbba6a6f936c764bbf839ec981c6 100644
--- a/lib/Support/StringMap.cpp
+++ b/lib/Support/StringMap.cpp
@@ -1,9 +1,8 @@
 //===--- StringMap.cpp - String Hash table map implementation -------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Support/StringPool.cpp b/lib/Support/StringPool.cpp
index c591857c415d9b1782b14c864f654824a3422139..82351017b8ccac59f71634600f823ab817f7ec29 100644
--- a/lib/Support/StringPool.cpp
+++ b/lib/Support/StringPool.cpp
@@ -1,9 +1,8 @@
 //===-- StringPool.cpp - Interned string pool -----------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Support/StringRef.cpp b/lib/Support/StringRef.cpp
index f0349260e22f69d28509bc85c3ca3e6f75ca40e5..4bafc4ec7181920e6df1aa0ed59bec15caa7e3ff 100644
--- a/lib/Support/StringRef.cpp
+++ b/lib/Support/StringRef.cpp
@@ -1,9 +1,8 @@
 //===-- StringRef.cpp - Lightweight String References ---------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Support/StringSaver.cpp b/lib/Support/StringSaver.cpp
index bf0ac8de98216ff758d695ddd5d8c5c1e3c0bd28..f7ccfb97ea79877ad8db036ed1cfc0db8bbc08dd 100644
--- a/lib/Support/StringSaver.cpp
+++ b/lib/Support/StringSaver.cpp
@@ -1,9 +1,8 @@
 //===-- StringSaver.cpp ---------------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Support/SymbolRemappingReader.cpp b/lib/Support/SymbolRemappingReader.cpp
index 264c890ce8f12eb6d6151668840b59511e4b9a49..1caf0947216ea9011dbbd8d80c6a06fbd85fcc0f 100644
--- a/lib/Support/SymbolRemappingReader.cpp
+++ b/lib/Support/SymbolRemappingReader.cpp
@@ -1,9 +1,8 @@
 //===- SymbolRemappingReader.cpp - Read symbol remapping file -------------===//
 //
-//                      The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Support/SystemUtils.cpp b/lib/Support/SystemUtils.cpp
index 7fa6ae3f61990ccb43e0e2258377a58c7491bb33..47e0c72ec7c139da03da43a45062f46817160f45 100644
--- a/lib/Support/SystemUtils.cpp
+++ b/lib/Support/SystemUtils.cpp
@@ -1,9 +1,8 @@
 //===- SystemUtils.cpp - Utilities for low-level system tasks -------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Support/TarWriter.cpp b/lib/Support/TarWriter.cpp
index 5b4d554befe4c07e611c4cc49f9553b18f3199db..6136e9219767231b3f4f1bde422d671f87cc20a2 100644
--- a/lib/Support/TarWriter.cpp
+++ b/lib/Support/TarWriter.cpp
@@ -1,9 +1,8 @@
 //===-- TarWriter.cpp - Tar archive file creator --------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Support/TargetParser.cpp b/lib/Support/TargetParser.cpp
index bdc0dc52c5e2334b4d4fb6d8e68618eb38457013..76401ac0a8502fccee52db7f5c054f153542e39a 100644
--- a/lib/Support/TargetParser.cpp
+++ b/lib/Support/TargetParser.cpp
@@ -1,9 +1,8 @@
 //===-- TargetParser - Parser for target features ---------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Support/TargetRegistry.cpp b/lib/Support/TargetRegistry.cpp
index bb63891cd71375580ad173e3f431fdd55903f1f8..1f9c3bbf8229469c8e02eaa7164125ceb6276971 100644
--- a/lib/Support/TargetRegistry.cpp
+++ b/lib/Support/TargetRegistry.cpp
@@ -1,9 +1,8 @@
 //===--- TargetRegistry.cpp - Target registration -------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Support/ThreadLocal.cpp b/lib/Support/ThreadLocal.cpp
index f6e4a652302cd5a7c7eb878acc6e39cf1fef8c6c..44e6223cf17b690f2985d7de26d8479817af760a 100644
--- a/lib/Support/ThreadLocal.cpp
+++ b/lib/Support/ThreadLocal.cpp
@@ -1,9 +1,8 @@
 //===- ThreadLocal.cpp - Thread Local Data ----------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Support/ThreadPool.cpp b/lib/Support/ThreadPool.cpp
index d0212ca13467bbd1808616ce0f65ff9176f99ea1..40982d777914d346accb294d782725a3abd2cb16 100644
--- a/lib/Support/ThreadPool.cpp
+++ b/lib/Support/ThreadPool.cpp
@@ -1,9 +1,8 @@
 //==-- llvm/Support/ThreadPool.cpp - A ThreadPool implementation -*- C++ -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Support/Threading.cpp b/lib/Support/Threading.cpp
index fcb1030e1ab42910caa23f5aece170d36ef853ec..e5899a60f4dbcf366e6885bb36b85a97670d0d7a 100644
--- a/lib/Support/Threading.cpp
+++ b/lib/Support/Threading.cpp
@@ -1,9 +1,8 @@
 //===-- llvm/Support/Threading.cpp- Control multithreading mode --*- C++ -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Support/Timer.cpp b/lib/Support/Timer.cpp
index 82f5810dd107ec4a3294a33a543ec696786370f8..0e634b392b5cfcfa112493b6a2f0105a3054c500 100644
--- a/lib/Support/Timer.cpp
+++ b/lib/Support/Timer.cpp
@@ -1,9 +1,8 @@
 //===-- Timer.cpp - Interval Timing Support -------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Support/ToolOutputFile.cpp b/lib/Support/ToolOutputFile.cpp
index e12d9e824f7edbac06309543ba26e9aed7c29f3b..ed3a247f011558f042dd231c51cd1e4149e39df7 100644
--- a/lib/Support/ToolOutputFile.cpp
+++ b/lib/Support/ToolOutputFile.cpp
@@ -1,9 +1,8 @@
 //===--- ToolOutputFile.cpp - Implement the ToolOutputFile class --------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Support/TrigramIndex.cpp b/lib/Support/TrigramIndex.cpp
index 721763c885252e9a18757af18e5ced0dadc982cc..94810b56db8ecd1973272624c43a6bb6d33e8628 100644
--- a/lib/Support/TrigramIndex.cpp
+++ b/lib/Support/TrigramIndex.cpp
@@ -1,9 +1,8 @@
 //===-- TrigramIndex.cpp - a heuristic for SpecialCaseList ----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Support/Triple.cpp b/lib/Support/Triple.cpp
index 4471fd05181eaf58bd4390d5c110169cd68acada..ab9fcccd5c4bff75cce15d5af6b9f742cff47824 100644
--- a/lib/Support/Triple.cpp
+++ b/lib/Support/Triple.cpp
@@ -1,9 +1,8 @@
 //===--- Triple.cpp - Target triple helper class --------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -35,7 +34,6 @@ StringRef Triple::getArchTypeName(ArchType Kind) {
   case mips64:         return "mips64";
   case mips64el:       return "mips64el";
   case msp430:         return "msp430";
-  case nios2:          return "nios2";
   case ppc64:          return "powerpc64";
   case ppc64le:        return "powerpc64le";
   case ppc:            return "powerpc";
@@ -102,8 +100,6 @@ StringRef Triple::getArchTypePrefix(ArchType Kind) {
   case mips64:
   case mips64el:    return "mips";
 
-  case nios2:       return "nios2";
-
   case hexagon:     return "hexagon";
 
   case amdgcn:      return "amdgcn";
@@ -211,6 +207,7 @@ StringRef Triple::getOSTypeName(OSType Kind) {
   case AMDPAL: return "amdpal";
   case HermitCore: return "hermit";
   case Hurd: return "hurd";
+  case WASI: return "wasi";
   }
 
   llvm_unreachable("Invalid OSType");
@@ -273,7 +270,6 @@ Triple::ArchType Triple::getArchTypeForLLVMName(StringRef Name) {
     .Case("mips64", mips64)
     .Case("mips64el", mips64el)
     .Case("msp430", msp430)
-    .Case("nios2", nios2)
     .Case("ppc64", ppc64)
     .Case("ppc32", ppc)
     .Case("ppc", ppc)
@@ -408,7 +404,6 @@ static Triple::ArchType parseArch(StringRef ArchName) {
            "mips64r6", "mipsn32r6", Triple::mips64)
     .Cases("mips64el", "mipsn32el", "mipsisa64r6el", "mips64r6el",
            "mipsn32r6el", Triple::mips64el)
-    .Case("nios2", Triple::nios2)
     .Case("r600", Triple::r600)
     .Case("amdgcn", Triple::amdgcn)
     .Case("riscv32", Triple::riscv32)
@@ -510,6 +505,7 @@ static Triple::OSType parseOS(StringRef OSName) {
     .StartsWith("amdpal", Triple::AMDPAL)
     .StartsWith("hermit", Triple::HermitCore)
     .StartsWith("hurd", Triple::Hurd)
+    .StartsWith("wasi", Triple::WASI)
     .Default(Triple::UnknownOS);
 }
 
@@ -665,7 +661,6 @@ static Triple::ObjectFormatType getDefaultFormat(const Triple &T) {
   case Triple::mips64el:
   case Triple::mipsel:
   case Triple::msp430:
-  case Triple::nios2:
   case Triple::nvptx:
   case Triple::nvptx64:
   case Triple::ppc64le:
@@ -1223,7 +1218,6 @@ static unsigned getArchPointerBitWidth(llvm::Triple::ArchType Arch) {
   case llvm::Triple::le32:
   case llvm::Triple::mips:
   case llvm::Triple::mipsel:
-  case llvm::Triple::nios2:
   case llvm::Triple::nvptx:
   case llvm::Triple::ppc:
   case llvm::Triple::r600:
@@ -1308,7 +1302,6 @@ Triple Triple::get32BitArchVariant() const {
   case Triple::le32:
   case Triple::mips:
   case Triple::mipsel:
-  case Triple::nios2:
   case Triple::nvptx:
   case Triple::ppc:
   case Triple::r600:
@@ -1357,7 +1350,6 @@ Triple Triple::get64BitArchVariant() const {
   case Triple::kalimba:
   case Triple::lanai:
   case Triple::msp430:
-  case Triple::nios2:
   case Triple::r600:
   case Triple::tce:
   case Triple::tcele:
@@ -1429,7 +1421,6 @@ Triple Triple::getBigEndianArchVariant() const {
   case Triple::le32:
   case Triple::le64:
   case Triple::msp430:
-  case Triple::nios2:
   case Triple::nvptx64:
   case Triple::nvptx:
   case Triple::r600:
@@ -1516,7 +1507,6 @@ bool Triple::isLittleEndian() const {
   case Triple::mips64el:
   case Triple::mipsel:
   case Triple::msp430:
-  case Triple::nios2:
   case Triple::nvptx64:
   case Triple::nvptx:
   case Triple::ppc64le:
diff --git a/lib/Support/Twine.cpp b/lib/Support/Twine.cpp
index 4726c8ab74940ca5b6d6317debabf2e6992f56ff..fbbcd8848f1cd26a3e273bc1eb2fe6e971b22ed3 100644
--- a/lib/Support/Twine.cpp
+++ b/lib/Support/Twine.cpp
@@ -1,9 +1,8 @@
 //===-- Twine.cpp - Fast Temporary String Concatenation -------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Support/Unicode.cpp b/lib/Support/Unicode.cpp
index b719bd826dc108b2bdf75f98f786d12ac7508c76..4d195069682bb34b2797ac181038d3eb886322a7 100644
--- a/lib/Support/Unicode.cpp
+++ b/lib/Support/Unicode.cpp
@@ -1,9 +1,8 @@
 //===- llvm/Support/Unicode.cpp - Unicode character properties  -*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Support/Unix/COM.inc b/lib/Support/Unix/COM.inc
index 5b71de74ebf3813cefa3df3247fbd2d4f313115b..03a690ac37667be8bff62d5b90537608cdea14f8 100644
--- a/lib/Support/Unix/COM.inc
+++ b/lib/Support/Unix/COM.inc
@@ -1,9 +1,8 @@
 //===- llvm/Support/Unix/COM.inc - Unix COM Implementation -----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Support/Unix/DynamicLibrary.inc b/lib/Support/Unix/DynamicLibrary.inc
index 029451f347e8c8b89b94a12dd0ad186dd16973f0..a2a379963de034b0435676ac50af20695e280978 100644
--- a/lib/Support/Unix/DynamicLibrary.inc
+++ b/lib/Support/Unix/DynamicLibrary.inc
@@ -1,9 +1,8 @@
 //===- Unix/DynamicLibrary.cpp - Unix DL Implementation ---------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Support/Unix/Host.inc b/lib/Support/Unix/Host.inc
index b65f84bf4444756995795168649f2c534e81e56b..4ae261aa8cfed57615ce208749c6f4bd3d16b928 100644
--- a/lib/Support/Unix/Host.inc
+++ b/lib/Support/Unix/Host.inc
@@ -1,9 +1,8 @@
 //===- llvm/Support/Unix/Host.inc -------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Support/Unix/Memory.inc b/lib/Support/Unix/Memory.inc
index adbfff2f59a5f16ceac62365278bb4cd12f7e841..3c4d324a8bbff6af0594b41337ac0f1382b9fbe4 100644
--- a/lib/Support/Unix/Memory.inc
+++ b/lib/Support/Unix/Memory.inc
@@ -1,9 +1,8 @@
 //===- Unix/Memory.cpp - Generic UNIX System Configuration ------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -92,9 +91,6 @@ Memory::allocateMappedMemory(size_t NumBytes,
   if (NumBytes == 0)
     return MemoryBlock();
 
-  static const size_t PageSize = Process::getPageSize();
-  const size_t NumPages = (NumBytes+PageSize-1)/PageSize;
-
   int fd = -1;
 
   int MMFlags = MAP_PRIVATE |
@@ -114,11 +110,12 @@ Memory::allocateMappedMemory(size_t NumBytes,
   // Use any near hint and the page size to set a page-aligned starting address
   uintptr_t Start = NearBlock ? reinterpret_cast<uintptr_t>(NearBlock->base()) +
                                       NearBlock->size() : 0;
+  static const size_t PageSize = Process::getPageSize();
   if (Start && Start % PageSize)
     Start += PageSize - Start % PageSize;
 
-  void *Addr = ::mmap(reinterpret_cast<void*>(Start), PageSize*NumPages,
-                      Protect, MMFlags, fd, 0);
+  void *Addr = ::mmap(reinterpret_cast<void *>(Start), NumBytes, Protect,
+                      MMFlags, fd, 0);
   if (Addr == MAP_FAILED) {
     if (NearBlock) //Try again without a near hint
       return allocateMappedMemory(NumBytes, nullptr, PFlags, EC);
@@ -129,7 +126,7 @@ Memory::allocateMappedMemory(size_t NumBytes,
 
   MemoryBlock Result;
   Result.Address = Addr;
-  Result.Size = NumPages*PageSize;
+  Result.Size = NumBytes;
 
   // Rely on protectMappedMemory to invalidate instruction cache.
   if (PFlags & MF_EXEC) {
diff --git a/lib/Support/Unix/Mutex.inc b/lib/Support/Unix/Mutex.inc
index fe6b1704145745ac3b49e7118f3c9b664fea401e..2c982b38d6ffe15a121eb2e8cc082bf273dcc8a8 100644
--- a/lib/Support/Unix/Mutex.inc
+++ b/lib/Support/Unix/Mutex.inc
@@ -1,9 +1,8 @@
 //===- llvm/Support/Unix/Mutex.inc - Unix Mutex Implementation ---*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Support/Unix/Path.inc b/lib/Support/Unix/Path.inc
index d7cc0d627d09e894f6fb380bf18946576c526bd9..5eba86d2077d9da6d50dad4ed516c2564f600efb 100644
--- a/lib/Support/Unix/Path.inc
+++ b/lib/Support/Unix/Path.inc
@@ -1,9 +1,8 @@
 //===- llvm/Support/Unix/Path.inc - Unix Path Implementation ----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Support/Unix/Process.inc b/lib/Support/Unix/Process.inc
index 3185f45a3a61b3a6b59528fa15e38ea3df1513fb..7c834f9711469405c932827868e5163eaa2c56b2 100644
--- a/lib/Support/Unix/Process.inc
+++ b/lib/Support/Unix/Process.inc
@@ -1,9 +1,8 @@
 //===- Unix/Process.cpp - Unix Process Implementation --------- -*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -33,10 +32,7 @@
 #if HAVE_SIGNAL_H
 #include <signal.h>
 #endif
-// DragonFlyBSD, and OpenBSD have deprecated <malloc.h> for
-// <stdlib.h> instead. Unix.h includes this for us already.
-#if defined(HAVE_MALLOC_H) && !defined(__DragonFly__) && \
-    !defined(__OpenBSD__) 
+#if defined(HAVE_MALLINFO)
 #include <malloc.h>
 #endif
 #if defined(HAVE_MALLCTL)
diff --git a/lib/Support/Unix/Program.inc b/lib/Support/Unix/Program.inc
index d0abc3763e821beffecc44a5af03448a9a209d44..dce592f5923ab62e708de93c628b4f25eba7fe29 100644
--- a/lib/Support/Unix/Program.inc
+++ b/lib/Support/Unix/Program.inc
@@ -1,9 +1,8 @@
 //===- llvm/Support/Unix/Program.cpp -----------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Support/Unix/RWMutex.inc b/lib/Support/Unix/RWMutex.inc
index 85a104334a27c970f288c30af2d0b0e442ac9c7e..8b47dfa0f85cd5714741df4affe8f26be700c067 100644
--- a/lib/Support/Unix/RWMutex.inc
+++ b/lib/Support/Unix/RWMutex.inc
@@ -1,9 +1,8 @@
 //= llvm/Support/Unix/RWMutex.inc - Unix Reader/Writer Mutual Exclusion Lock  =//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Support/Unix/Signals.inc b/lib/Support/Unix/Signals.inc
index ad88d5e969068e6802d0e9b359d8c4101667f1f5..51e15071aa418cb0e02ed17acc6992eb32fb5738 100644
--- a/lib/Support/Unix/Signals.inc
+++ b/lib/Support/Unix/Signals.inc
@@ -1,9 +1,8 @@
 //===- Signals.cpp - Generic Unix Signals Implementation -----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Support/Unix/ThreadLocal.inc b/lib/Support/Unix/ThreadLocal.inc
index a6564f0fa281dd26a42dcd63bcfb97237bae7183..a402ae980424e37c656b34ed2a4366c4fae4fef5 100644
--- a/lib/Support/Unix/ThreadLocal.inc
+++ b/lib/Support/Unix/ThreadLocal.inc
@@ -1,9 +1,8 @@
 //=== llvm/Support/Unix/ThreadLocal.inc - Unix Thread Local Data -*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Support/Unix/Threading.inc b/lib/Support/Unix/Threading.inc
index 2d49ce1ad747a0ef1f628ac93872a3812d1d604e..767f62640bffee81585b5cba7c8bbf5d5868a34c 100644
--- a/lib/Support/Unix/Threading.inc
+++ b/lib/Support/Unix/Threading.inc
@@ -1,9 +1,8 @@
 //===- Unix/Threading.inc - Unix Threading Implementation ----- -*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -202,6 +201,12 @@ void llvm::get_thread_name(SmallVectorImpl<char> &Name) {
   char buf[len];
   ::pthread_getname_np(::pthread_self(), buf, len);
 
+  Name.append(buf, buf + strlen(buf));
+#elif defined(__OpenBSD__)
+  constexpr uint32_t len = get_max_thread_name_length_impl();
+  char buf[len];
+  ::pthread_get_name_np(::pthread_self(), buf, len);
+
   Name.append(buf, buf + strlen(buf));
 #elif defined(__linux__)
 #if HAVE_PTHREAD_GETNAME_NP
diff --git a/lib/Support/Unix/Unix.h b/lib/Support/Unix/Unix.h
index 0c5d4de556d50ce12c31de878372d6fdcc441c7d..20d70bd2fcee81a0378d4ee190dfdfc3197c6015 100644
--- a/lib/Support/Unix/Unix.h
+++ b/lib/Support/Unix/Unix.h
@@ -1,9 +1,8 @@
 //===- llvm/Support/Unix/Unix.h - Common Unix Include File -------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Support/Unix/Watchdog.inc b/lib/Support/Unix/Watchdog.inc
index f4253391d95281b2079860a80749780d9088e545..b363ef7795603924508e15f826ff112101089bea 100644
--- a/lib/Support/Unix/Watchdog.inc
+++ b/lib/Support/Unix/Watchdog.inc
@@ -1,9 +1,8 @@
 //===--- Unix/Watchdog.inc - Unix Watchdog Implementation -------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Support/Valgrind.cpp b/lib/Support/Valgrind.cpp
index 8d852a67c0756b1b4cf6dc2d66c1879e0dc2b9e4..886cb6ba3311104b54ddc01f97d8f833d36c3c93 100644
--- a/lib/Support/Valgrind.cpp
+++ b/lib/Support/Valgrind.cpp
@@ -1,9 +1,8 @@
 //===-- Valgrind.cpp - Implement Valgrind communication ---------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Support/VersionTuple.cpp b/lib/Support/VersionTuple.cpp
index 3f219bfbedfa9df827a1d2ea6a69a4cd833f1f6c..60b59424fbb49ab66318665d322197a1a90476ce 100644
--- a/lib/Support/VersionTuple.cpp
+++ b/lib/Support/VersionTuple.cpp
@@ -1,9 +1,8 @@
 //===- VersionTuple.cpp - Version Number Handling ---------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Support/VirtualFileSystem.cpp b/lib/Support/VirtualFileSystem.cpp
index 24581944fa81ef82a22f25b14f42658fc8150b44..afa027520311954b3a4185ad0ff54ff657d8ad8b 100644
--- a/lib/Support/VirtualFileSystem.cpp
+++ b/lib/Support/VirtualFileSystem.cpp
@@ -1,9 +1,8 @@
 //===- VirtualFileSystem.cpp - Virtual File System Layer ------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -128,7 +127,8 @@ std::error_code FileSystem::makeAbsolute(SmallVectorImpl<char> &Path) const {
   if (!WorkingDir)
     return WorkingDir.getError();
 
-  return llvm::sys::fs::make_absolute(WorkingDir.get(), Path);
+  llvm::sys::fs::make_absolute(WorkingDir.get(), Path);
+  return {};
 }
 
 std::error_code FileSystem::getRealPath(const Twine &Path,
@@ -472,6 +472,8 @@ directory_iterator OverlayFileSystem::dir_begin(const Twine &Dir,
       std::make_shared<OverlayFSDirIterImpl>(Dir, *this, EC));
 }
 
+void ProxyFileSystem::anchor() {}
+
 namespace llvm {
 namespace vfs {
 
@@ -941,84 +943,12 @@ std::error_code InMemoryFileSystem::isLocal(const Twine &Path, bool &Result) {
 // RedirectingFileSystem implementation
 //===-----------------------------------------------------------------------===/
 
-namespace {
-
-enum EntryKind { EK_Directory, EK_File };
-
-/// A single file or directory in the VFS.
-class Entry {
-  EntryKind Kind;
-  std::string Name;
-
-public:
-  Entry(EntryKind K, StringRef Name) : Kind(K), Name(Name) {}
-  virtual ~Entry() = default;
-
-  StringRef getName() const { return Name; }
-  EntryKind getKind() const { return Kind; }
-};
-
-class RedirectingDirectoryEntry : public Entry {
-  std::vector<std::unique_ptr<Entry>> Contents;
-  Status S;
-
-public:
-  RedirectingDirectoryEntry(StringRef Name,
-                            std::vector<std::unique_ptr<Entry>> Contents,
-                            Status S)
-      : Entry(EK_Directory, Name), Contents(std::move(Contents)),
-        S(std::move(S)) {}
-  RedirectingDirectoryEntry(StringRef Name, Status S)
-      : Entry(EK_Directory, Name), S(std::move(S)) {}
-
-  Status getStatus() { return S; }
-
-  void addContent(std::unique_ptr<Entry> Content) {
-    Contents.push_back(std::move(Content));
-  }
-
-  Entry *getLastContent() const { return Contents.back().get(); }
-
-  using iterator = decltype(Contents)::iterator;
-
-  iterator contents_begin() { return Contents.begin(); }
-  iterator contents_end() { return Contents.end(); }
-
-  static bool classof(const Entry *E) { return E->getKind() == EK_Directory; }
-};
-
-class RedirectingFileEntry : public Entry {
-public:
-  enum NameKind { NK_NotSet, NK_External, NK_Virtual };
-
-private:
-  std::string ExternalContentsPath;
-  NameKind UseName;
-
-public:
-  RedirectingFileEntry(StringRef Name, StringRef ExternalContentsPath,
-                       NameKind UseName)
-      : Entry(EK_File, Name), ExternalContentsPath(ExternalContentsPath),
-        UseName(UseName) {}
-
-  StringRef getExternalContentsPath() const { return ExternalContentsPath; }
-
-  /// whether to use the external path as the name for this file.
-  bool useExternalName(bool GlobalUseExternalName) const {
-    return UseName == NK_NotSet ? GlobalUseExternalName
-                                : (UseName == NK_External);
-  }
-
-  NameKind getUseName() const { return UseName; }
-
-  static bool classof(const Entry *E) { return E->getKind() == EK_File; }
-};
-
 // FIXME: reuse implementation common with OverlayFSDirIterImpl as these
 // iterators are conceptually similar.
-class VFSFromYamlDirIterImpl : public llvm::vfs::detail::DirIterImpl {
+class llvm::vfs::VFSFromYamlDirIterImpl
+    : public llvm::vfs::detail::DirIterImpl {
   std::string Dir;
-  RedirectingDirectoryEntry::iterator Current, End;
+  RedirectingFileSystem::RedirectingDirectoryEntry::iterator Current, End;
 
   // To handle 'fallthrough' mode we need to iterate at first through
   // RedirectingDirectoryEntry and then through ExternalFS. These operations are
@@ -1048,216 +978,92 @@ class VFSFromYamlDirIterImpl : public llvm::vfs::detail::DirIterImpl {
   /// @}
 
 public:
-  VFSFromYamlDirIterImpl(const Twine &Path,
-                         RedirectingDirectoryEntry::iterator Begin,
-                         RedirectingDirectoryEntry::iterator End,
-                         bool IterateExternalFS, FileSystem &ExternalFS,
-                         std::error_code &EC);
+  VFSFromYamlDirIterImpl(
+      const Twine &Path,
+      RedirectingFileSystem::RedirectingDirectoryEntry::iterator Begin,
+      RedirectingFileSystem::RedirectingDirectoryEntry::iterator End,
+      bool IterateExternalFS, FileSystem &ExternalFS, std::error_code &EC);
 
   std::error_code increment() override;
 };
 
-/// A virtual file system parsed from a YAML file.
-///
-/// Currently, this class allows creating virtual directories and mapping
-/// virtual file paths to existing external files, available in \c ExternalFS.
-///
-/// The basic structure of the parsed file is:
-/// \verbatim
-/// {
-///   'version': <version number>,
-///   <optional configuration>
-///   'roots': [
-///              <directory entries>
-///            ]
-/// }
-/// \endverbatim
-///
-/// All configuration options are optional.
-///   'case-sensitive': <boolean, default=true>
-///   'use-external-names': <boolean, default=true>
-///   'overlay-relative': <boolean, default=false>
-///   'fallthrough': <boolean, default=true>
-///
-/// Virtual directories are represented as
-/// \verbatim
-/// {
-///   'type': 'directory',
-///   'name': <string>,
-///   'contents': [ <file or directory entries> ]
-/// }
-/// \endverbatim
-///
-/// The default attributes for virtual directories are:
-/// \verbatim
-/// MTime = now() when created
-/// Perms = 0777
-/// User = Group = 0
-/// Size = 0
-/// UniqueID = unspecified unique value
-/// \endverbatim
-///
-/// Re-mapped files are represented as
-/// \verbatim
-/// {
-///   'type': 'file',
-///   'name': <string>,
-///   'use-external-name': <boolean> # Optional
-///   'external-contents': <path to external file>
-/// }
-/// \endverbatim
-///
-/// and inherit their attributes from the external contents.
-///
-/// In both cases, the 'name' field may contain multiple path components (e.g.
-/// /path/to/file). However, any directory that contains more than one child
-/// must be uniquely represented by a directory entry.
-class RedirectingFileSystem : public vfs::FileSystem {
-  friend class RedirectingFileSystemParser;
-
-  /// The root(s) of the virtual file system.
-  std::vector<std::unique_ptr<Entry>> Roots;
-
-  /// The file system to use for external references.
-  IntrusiveRefCntPtr<FileSystem> ExternalFS;
-
-  /// If IsRelativeOverlay is set, this represents the directory
-  /// path that should be prefixed to each 'external-contents' entry
-  /// when reading from YAML files.
-  std::string ExternalContentsPrefixDir;
-
-  /// @name Configuration
-  /// @{
-
-  /// Whether to perform case-sensitive comparisons.
-  ///
-  /// Currently, case-insensitive matching only works correctly with ASCII.
-  bool CaseSensitive = true;
-
-  /// IsRelativeOverlay marks whether a ExternalContentsPrefixDir path must
-  /// be prefixed in every 'external-contents' when reading from YAML files.
-  bool IsRelativeOverlay = false;
-
-  /// Whether to use to use the value of 'external-contents' for the
-  /// names of files.  This global value is overridable on a per-file basis.
-  bool UseExternalNames = true;
-
-  /// Whether to attempt a file lookup in external file system after it wasn't
-  /// found in VFS.
-  bool IsFallthrough = true;
-  /// @}
-
-  /// Virtual file paths and external files could be canonicalized without "..",
-  /// "." and "./" in their paths. FIXME: some unittests currently fail on
-  /// win32 when using remove_dots and remove_leading_dotslash on paths.
-  bool UseCanonicalizedPaths =
-#ifdef _WIN32
-      false;
-#else
-      true;
-#endif
-
-private:
-  RedirectingFileSystem(IntrusiveRefCntPtr<FileSystem> ExternalFS)
-      : ExternalFS(std::move(ExternalFS)) {}
-
-  /// Looks up the path <tt>[Start, End)</tt> in \p From, possibly
-  /// recursing into the contents of \p From if it is a directory.
-  ErrorOr<Entry *> lookupPath(sys::path::const_iterator Start,
-                              sys::path::const_iterator End, Entry *From) const;
-
-  /// Get the status of a given an \c Entry.
-  ErrorOr<Status> status(const Twine &Path, Entry *E);
-
-public:
-  /// Looks up \p Path in \c Roots.
-  ErrorOr<Entry *> lookupPath(const Twine &Path) const;
-
-  /// Parses \p Buffer, which is expected to be in YAML format and
-  /// returns a virtual file system representing its contents.
-  static RedirectingFileSystem *
-  create(std::unique_ptr<MemoryBuffer> Buffer,
-         SourceMgr::DiagHandlerTy DiagHandler, StringRef YAMLFilePath,
-         void *DiagContext, IntrusiveRefCntPtr<FileSystem> ExternalFS);
+llvm::ErrorOr<std::string>
+RedirectingFileSystem::getCurrentWorkingDirectory() const {
+  return ExternalFS->getCurrentWorkingDirectory();
+}
 
-  ErrorOr<Status> status(const Twine &Path) override;
-  ErrorOr<std::unique_ptr<File>> openFileForRead(const Twine &Path) override;
+std::error_code
+RedirectingFileSystem::setCurrentWorkingDirectory(const Twine &Path) {
+  return ExternalFS->setCurrentWorkingDirectory(Path);
+}
 
-  std::error_code getRealPath(const Twine &Path,
-                              SmallVectorImpl<char> &Output) const override;
+std::error_code RedirectingFileSystem::isLocal(const Twine &Path,
+                                               bool &Result) {
+  return ExternalFS->isLocal(Path, Result);
+}
 
-  llvm::ErrorOr<std::string> getCurrentWorkingDirectory() const override {
-    return ExternalFS->getCurrentWorkingDirectory();
+directory_iterator RedirectingFileSystem::dir_begin(const Twine &Dir,
+                                                    std::error_code &EC) {
+  ErrorOr<RedirectingFileSystem::Entry *> E = lookupPath(Dir);
+  if (!E) {
+    EC = E.getError();
+    if (IsFallthrough && EC == errc::no_such_file_or_directory)
+      return ExternalFS->dir_begin(Dir, EC);
+    return {};
   }
-
-  std::error_code setCurrentWorkingDirectory(const Twine &Path) override {
-    return ExternalFS->setCurrentWorkingDirectory(Path);
+  ErrorOr<Status> S = status(Dir, *E);
+  if (!S) {
+    EC = S.getError();
+    return {};
   }
-
-  std::error_code isLocal(const Twine &Path, bool &Result) override {
-    return ExternalFS->isLocal(Path, Result);
+  if (!S->isDirectory()) {
+    EC = std::error_code(static_cast<int>(errc::not_a_directory),
+                         std::system_category());
+    return {};
   }
 
-  directory_iterator dir_begin(const Twine &Dir, std::error_code &EC) override {
-    ErrorOr<Entry *> E = lookupPath(Dir);
-    if (!E) {
-      EC = E.getError();
-      if (IsFallthrough && EC == errc::no_such_file_or_directory)
-        return ExternalFS->dir_begin(Dir, EC);
-      return {};
-    }
-    ErrorOr<Status> S = status(Dir, *E);
-    if (!S) {
-      EC = S.getError();
-      return {};
-    }
-    if (!S->isDirectory()) {
-      EC = std::error_code(static_cast<int>(errc::not_a_directory),
-                           std::system_category());
-      return {};
-    }
-
-    auto *D = cast<RedirectingDirectoryEntry>(*E);
-    return directory_iterator(std::make_shared<VFSFromYamlDirIterImpl>(
-        Dir, D->contents_begin(), D->contents_end(),
-        /*IterateExternalFS=*/IsFallthrough, *ExternalFS, EC));
-  }
+  auto *D = cast<RedirectingFileSystem::RedirectingDirectoryEntry>(*E);
+  return directory_iterator(std::make_shared<VFSFromYamlDirIterImpl>(
+      Dir, D->contents_begin(), D->contents_end(),
+      /*IterateExternalFS=*/IsFallthrough, *ExternalFS, EC));
+}
 
-  void setExternalContentsPrefixDir(StringRef PrefixDir) {
-    ExternalContentsPrefixDir = PrefixDir.str();
-  }
+void RedirectingFileSystem::setExternalContentsPrefixDir(StringRef PrefixDir) {
+  ExternalContentsPrefixDir = PrefixDir.str();
+}
 
-  StringRef getExternalContentsPrefixDir() const {
-    return ExternalContentsPrefixDir;
-  }
+StringRef RedirectingFileSystem::getExternalContentsPrefixDir() const {
+  return ExternalContentsPrefixDir;
+}
 
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
-  LLVM_DUMP_METHOD void dump() const {
-    for (const auto &Root : Roots)
-      dumpEntry(Root.get());
-  }
+LLVM_DUMP_METHOD void RedirectingFileSystem::dump() const {
+  for (const auto &Root : Roots)
+    dumpEntry(Root.get());
+}
 
-  LLVM_DUMP_METHOD void dumpEntry(Entry *E, int NumSpaces = 0) const {
-    StringRef Name = E->getName();
-    for (int i = 0, e = NumSpaces; i < e; ++i)
-      dbgs() << " ";
-    dbgs() << "'" << Name.str().c_str() << "'"
-           << "\n";
+LLVM_DUMP_METHOD void
+RedirectingFileSystem::dumpEntry(RedirectingFileSystem::Entry *E,
+                                 int NumSpaces) const {
+  StringRef Name = E->getName();
+  for (int i = 0, e = NumSpaces; i < e; ++i)
+    dbgs() << " ";
+  dbgs() << "'" << Name.str().c_str() << "'"
+         << "\n";
 
-    if (E->getKind() == EK_Directory) {
-      auto *DE = dyn_cast<RedirectingDirectoryEntry>(E);
-      assert(DE && "Should be a directory");
+  if (E->getKind() == RedirectingFileSystem::EK_Directory) {
+    auto *DE = dyn_cast<RedirectingFileSystem::RedirectingDirectoryEntry>(E);
+    assert(DE && "Should be a directory");
 
-      for (std::unique_ptr<Entry> &SubEntry :
-           llvm::make_range(DE->contents_begin(), DE->contents_end()))
-        dumpEntry(SubEntry.get(), NumSpaces + 2);
-    }
+    for (std::unique_ptr<Entry> &SubEntry :
+         llvm::make_range(DE->contents_begin(), DE->contents_end()))
+      dumpEntry(SubEntry.get(), NumSpaces + 2);
   }
+}
 #endif
-};
 
 /// A helper class to hold the common YAML parsing state.
-class RedirectingFileSystemParser {
+class llvm::vfs::RedirectingFileSystemParser {
   yaml::Stream &Stream;
 
   void error(yaml::Node *N, const Twine &Msg) { Stream.printError(N, Msg); }
@@ -1332,8 +1138,9 @@ class RedirectingFileSystemParser {
     return true;
   }
 
-  Entry *lookupOrCreateEntry(RedirectingFileSystem *FS, StringRef Name,
-                             Entry *ParentEntry = nullptr) {
+  RedirectingFileSystem::Entry *
+  lookupOrCreateEntry(RedirectingFileSystem *FS, StringRef Name,
+                      RedirectingFileSystem::Entry *ParentEntry = nullptr) {
     if (!ParentEntry) { // Look for a existent root
       for (const auto &Root : FS->Roots) {
         if (Name.equals(Root->getName())) {
@@ -1342,20 +1149,24 @@ class RedirectingFileSystemParser {
         }
       }
     } else { // Advance to the next component
-      auto *DE = dyn_cast<RedirectingDirectoryEntry>(ParentEntry);
-      for (std::unique_ptr<Entry> &Content :
+      auto *DE = dyn_cast<RedirectingFileSystem::RedirectingDirectoryEntry>(
+          ParentEntry);
+      for (std::unique_ptr<RedirectingFileSystem::Entry> &Content :
            llvm::make_range(DE->contents_begin(), DE->contents_end())) {
-        auto *DirContent = dyn_cast<RedirectingDirectoryEntry>(Content.get());
+        auto *DirContent =
+            dyn_cast<RedirectingFileSystem::RedirectingDirectoryEntry>(
+                Content.get());
         if (DirContent && Name.equals(Content->getName()))
           return DirContent;
       }
     }
 
     // ... or create a new one
-    std::unique_ptr<Entry> E = llvm::make_unique<RedirectingDirectoryEntry>(
-        Name,
-        Status("", getNextVirtualUniqueID(), std::chrono::system_clock::now(),
-               0, 0, 0, file_type::directory_file, sys::fs::all_all));
+    std::unique_ptr<RedirectingFileSystem::Entry> E =
+        llvm::make_unique<RedirectingFileSystem::RedirectingDirectoryEntry>(
+            Name, Status("", getNextVirtualUniqueID(),
+                         std::chrono::system_clock::now(), 0, 0, 0,
+                         file_type::directory_file, sys::fs::all_all));
 
     if (!ParentEntry) { // Add a new root to the overlay
       FS->Roots.push_back(std::move(E));
@@ -1363,42 +1174,47 @@ class RedirectingFileSystemParser {
       return ParentEntry;
     }
 
-    auto *DE = dyn_cast<RedirectingDirectoryEntry>(ParentEntry);
+    auto *DE =
+        dyn_cast<RedirectingFileSystem::RedirectingDirectoryEntry>(ParentEntry);
     DE->addContent(std::move(E));
     return DE->getLastContent();
   }
 
-  void uniqueOverlayTree(RedirectingFileSystem *FS, Entry *SrcE,
-                         Entry *NewParentE = nullptr) {
+  void uniqueOverlayTree(RedirectingFileSystem *FS,
+                         RedirectingFileSystem::Entry *SrcE,
+                         RedirectingFileSystem::Entry *NewParentE = nullptr) {
     StringRef Name = SrcE->getName();
     switch (SrcE->getKind()) {
-    case EK_Directory: {
-      auto *DE = dyn_cast<RedirectingDirectoryEntry>(SrcE);
+    case RedirectingFileSystem::EK_Directory: {
+      auto *DE =
+          dyn_cast<RedirectingFileSystem::RedirectingDirectoryEntry>(SrcE);
       assert(DE && "Must be a directory");
       // Empty directories could be present in the YAML as a way to
       // describe a file for a current directory after some of its subdir
       // is parsed. This only leads to redundant walks, ignore it.
       if (!Name.empty())
         NewParentE = lookupOrCreateEntry(FS, Name, NewParentE);
-      for (std::unique_ptr<Entry> &SubEntry :
+      for (std::unique_ptr<RedirectingFileSystem::Entry> &SubEntry :
            llvm::make_range(DE->contents_begin(), DE->contents_end()))
         uniqueOverlayTree(FS, SubEntry.get(), NewParentE);
       break;
     }
-    case EK_File: {
-      auto *FE = dyn_cast<RedirectingFileEntry>(SrcE);
+    case RedirectingFileSystem::EK_File: {
+      auto *FE = dyn_cast<RedirectingFileSystem::RedirectingFileEntry>(SrcE);
       assert(FE && "Must be a file");
       assert(NewParentE && "Parent entry must exist");
-      auto *DE = dyn_cast<RedirectingDirectoryEntry>(NewParentE);
-      DE->addContent(llvm::make_unique<RedirectingFileEntry>(
-          Name, FE->getExternalContentsPath(), FE->getUseName()));
+      auto *DE = dyn_cast<RedirectingFileSystem::RedirectingDirectoryEntry>(
+          NewParentE);
+      DE->addContent(
+          llvm::make_unique<RedirectingFileSystem::RedirectingFileEntry>(
+              Name, FE->getExternalContentsPath(), FE->getUseName()));
       break;
     }
     }
   }
 
-  std::unique_ptr<Entry> parseEntry(yaml::Node *N, RedirectingFileSystem *FS,
-                                    bool IsRootEntry) {
+  std::unique_ptr<RedirectingFileSystem::Entry>
+  parseEntry(yaml::Node *N, RedirectingFileSystem *FS, bool IsRootEntry) {
     auto *M = dyn_cast<yaml::MappingNode>(N);
     if (!M) {
       error(N, "expected mapping node for file or directory entry");
@@ -1416,12 +1232,14 @@ class RedirectingFileSystemParser {
     DenseMap<StringRef, KeyStatus> Keys(std::begin(Fields), std::end(Fields));
 
     bool HasContents = false; // external or otherwise
-    std::vector<std::unique_ptr<Entry>> EntryArrayContents;
+    std::vector<std::unique_ptr<RedirectingFileSystem::Entry>>
+        EntryArrayContents;
     std::string ExternalContentsPath;
     std::string Name;
     yaml::Node *NameValueNode;
-    auto UseExternalName = RedirectingFileEntry::NK_NotSet;
-    EntryKind Kind;
+    auto UseExternalName =
+        RedirectingFileSystem::RedirectingFileEntry::NK_NotSet;
+    RedirectingFileSystem::EntryKind Kind;
 
     for (auto &I : *M) {
       StringRef Key;
@@ -1454,9 +1272,9 @@ class RedirectingFileSystemParser {
         if (!parseScalarString(I.getValue(), Value, Buffer))
           return nullptr;
         if (Value == "file")
-          Kind = EK_File;
+          Kind = RedirectingFileSystem::EK_File;
         else if (Value == "directory")
-          Kind = EK_Directory;
+          Kind = RedirectingFileSystem::EK_Directory;
         else {
           error(I.getValue(), "unknown value for 'type'");
           return nullptr;
@@ -1476,7 +1294,7 @@ class RedirectingFileSystemParser {
         }
 
         for (auto &I : *Contents) {
-          if (std::unique_ptr<Entry> E =
+          if (std::unique_ptr<RedirectingFileSystem::Entry> E =
                   parseEntry(&I, FS, /*IsRootEntry*/ false))
             EntryArrayContents.push_back(std::move(E));
           else
@@ -1513,8 +1331,9 @@ class RedirectingFileSystemParser {
         bool Val;
         if (!parseScalarBool(I.getValue(), Val))
           return nullptr;
-        UseExternalName = Val ? RedirectingFileEntry::NK_External
-                              : RedirectingFileEntry::NK_Virtual;
+        UseExternalName =
+            Val ? RedirectingFileSystem::RedirectingFileEntry::NK_External
+                : RedirectingFileSystem::RedirectingFileEntry::NK_Virtual;
       } else {
         llvm_unreachable("key missing from Keys");
       }
@@ -1532,8 +1351,9 @@ class RedirectingFileSystemParser {
       return nullptr;
 
     // check invalid configuration
-    if (Kind == EK_Directory &&
-        UseExternalName != RedirectingFileEntry::NK_NotSet) {
+    if (Kind == RedirectingFileSystem::EK_Directory &&
+        UseExternalName !=
+            RedirectingFileSystem::RedirectingFileEntry::NK_NotSet) {
       error(N, "'use-external-name' is not supported for directories");
       return nullptr;
     }
@@ -1554,17 +1374,19 @@ class RedirectingFileSystemParser {
     // Get the last component
     StringRef LastComponent = sys::path::filename(Trimmed);
 
-    std::unique_ptr<Entry> Result;
+    std::unique_ptr<RedirectingFileSystem::Entry> Result;
     switch (Kind) {
-    case EK_File:
-      Result = llvm::make_unique<RedirectingFileEntry>(
+    case RedirectingFileSystem::EK_File:
+      Result = llvm::make_unique<RedirectingFileSystem::RedirectingFileEntry>(
           LastComponent, std::move(ExternalContentsPath), UseExternalName);
       break;
-    case EK_Directory:
-      Result = llvm::make_unique<RedirectingDirectoryEntry>(
-          LastComponent, std::move(EntryArrayContents),
-          Status("", getNextVirtualUniqueID(), std::chrono::system_clock::now(),
-                 0, 0, 0, file_type::directory_file, sys::fs::all_all));
+    case RedirectingFileSystem::EK_Directory:
+      Result =
+          llvm::make_unique<RedirectingFileSystem::RedirectingDirectoryEntry>(
+              LastComponent, std::move(EntryArrayContents),
+              Status("", getNextVirtualUniqueID(),
+                     std::chrono::system_clock::now(), 0, 0, 0,
+                     file_type::directory_file, sys::fs::all_all));
       break;
     }
 
@@ -1576,12 +1398,14 @@ class RedirectingFileSystemParser {
     for (sys::path::reverse_iterator I = sys::path::rbegin(Parent),
                                      E = sys::path::rend(Parent);
          I != E; ++I) {
-      std::vector<std::unique_ptr<Entry>> Entries;
+      std::vector<std::unique_ptr<RedirectingFileSystem::Entry>> Entries;
       Entries.push_back(std::move(Result));
-      Result = llvm::make_unique<RedirectingDirectoryEntry>(
-          *I, std::move(Entries),
-          Status("", getNextVirtualUniqueID(), std::chrono::system_clock::now(),
-                 0, 0, 0, file_type::directory_file, sys::fs::all_all));
+      Result =
+          llvm::make_unique<RedirectingFileSystem::RedirectingDirectoryEntry>(
+              *I, std::move(Entries),
+              Status("", getNextVirtualUniqueID(),
+                     std::chrono::system_clock::now(), 0, 0, 0,
+                     file_type::directory_file, sys::fs::all_all));
     }
     return Result;
   }
@@ -1607,7 +1431,7 @@ public:
     };
 
     DenseMap<StringRef, KeyStatus> Keys(std::begin(Fields), std::end(Fields));
-    std::vector<std::unique_ptr<Entry>> RootEntries;
+    std::vector<std::unique_ptr<RedirectingFileSystem::Entry>> RootEntries;
 
     // Parse configuration and 'roots'
     for (auto &I : *Top) {
@@ -1627,7 +1451,7 @@ public:
         }
 
         for (auto &I : *Roots) {
-          if (std::unique_ptr<Entry> E =
+          if (std::unique_ptr<RedirectingFileSystem::Entry> E =
                   parseEntry(&I, FS, /*IsRootEntry*/ true))
             RootEntries.push_back(std::move(E));
           else
@@ -1684,8 +1508,6 @@ public:
   }
 };
 
-} // namespace
-
 RedirectingFileSystem *
 RedirectingFileSystem::create(std::unique_ptr<MemoryBuffer> Buffer,
                               SourceMgr::DiagHandlerTy DiagHandler,
@@ -1729,7 +1551,8 @@ RedirectingFileSystem::create(std::unique_ptr<MemoryBuffer> Buffer,
   return FS.release();
 }
 
-ErrorOr<Entry *> RedirectingFileSystem::lookupPath(const Twine &Path_) const {
+ErrorOr<RedirectingFileSystem::Entry *>
+RedirectingFileSystem::lookupPath(const Twine &Path_) const {
   SmallString<256> Path;
   Path_.toVector(Path);
 
@@ -1751,17 +1574,18 @@ ErrorOr<Entry *> RedirectingFileSystem::lookupPath(const Twine &Path_) const {
   sys::path::const_iterator Start = sys::path::begin(Path);
   sys::path::const_iterator End = sys::path::end(Path);
   for (const auto &Root : Roots) {
-    ErrorOr<Entry *> Result = lookupPath(Start, End, Root.get());
+    ErrorOr<RedirectingFileSystem::Entry *> Result =
+        lookupPath(Start, End, Root.get());
     if (Result || Result.getError() != llvm::errc::no_such_file_or_directory)
       return Result;
   }
   return make_error_code(llvm::errc::no_such_file_or_directory);
 }
 
-ErrorOr<Entry *>
+ErrorOr<RedirectingFileSystem::Entry *>
 RedirectingFileSystem::lookupPath(sys::path::const_iterator Start,
                                   sys::path::const_iterator End,
-                                  Entry *From) const {
+                                  RedirectingFileSystem::Entry *From) const {
 #ifndef _WIN32
   assert(!isTraversalComponent(*Start) &&
          !isTraversalComponent(From->getName()) &&
@@ -1790,13 +1614,14 @@ RedirectingFileSystem::lookupPath(sys::path::const_iterator Start,
     }
   }
 
-  auto *DE = dyn_cast<RedirectingDirectoryEntry>(From);
+  auto *DE = dyn_cast<RedirectingFileSystem::RedirectingDirectoryEntry>(From);
   if (!DE)
     return make_error_code(llvm::errc::not_a_directory);
 
-  for (const std::unique_ptr<Entry> &DirEntry :
+  for (const std::unique_ptr<RedirectingFileSystem::Entry> &DirEntry :
        llvm::make_range(DE->contents_begin(), DE->contents_end())) {
-    ErrorOr<Entry *> Result = lookupPath(Start, End, DirEntry.get());
+    ErrorOr<RedirectingFileSystem::Entry *> Result =
+        lookupPath(Start, End, DirEntry.get());
     if (Result || Result.getError() != llvm::errc::no_such_file_or_directory)
       return Result;
   }
@@ -1812,9 +1637,10 @@ static Status getRedirectedFileStatus(const Twine &Path, bool UseExternalNames,
   return S;
 }
 
-ErrorOr<Status> RedirectingFileSystem::status(const Twine &Path, Entry *E) {
+ErrorOr<Status> RedirectingFileSystem::status(const Twine &Path,
+                                              RedirectingFileSystem::Entry *E) {
   assert(E != nullptr);
-  if (auto *F = dyn_cast<RedirectingFileEntry>(E)) {
+  if (auto *F = dyn_cast<RedirectingFileSystem::RedirectingFileEntry>(E)) {
     ErrorOr<Status> S = ExternalFS->status(F->getExternalContentsPath());
     assert(!S || S->getName() == F->getExternalContentsPath());
     if (S)
@@ -1822,13 +1648,13 @@ ErrorOr<Status> RedirectingFileSystem::status(const Twine &Path, Entry *E) {
                                      *S);
     return S;
   } else { // directory
-    auto *DE = cast<RedirectingDirectoryEntry>(E);
+    auto *DE = cast<RedirectingFileSystem::RedirectingDirectoryEntry>(E);
     return Status::copyWithNewName(DE->getStatus(), Path.str());
   }
 }
 
 ErrorOr<Status> RedirectingFileSystem::status(const Twine &Path) {
-  ErrorOr<Entry *> Result = lookupPath(Path);
+  ErrorOr<RedirectingFileSystem::Entry *> Result = lookupPath(Path);
   if (!Result) {
     if (IsFallthrough &&
         Result.getError() == llvm::errc::no_such_file_or_directory) {
@@ -1866,7 +1692,7 @@ public:
 
 ErrorOr<std::unique_ptr<File>>
 RedirectingFileSystem::openFileForRead(const Twine &Path) {
-  ErrorOr<Entry *> E = lookupPath(Path);
+  ErrorOr<RedirectingFileSystem::Entry *> E = lookupPath(Path);
   if (!E) {
     if (IsFallthrough &&
         E.getError() == llvm::errc::no_such_file_or_directory) {
@@ -1875,7 +1701,7 @@ RedirectingFileSystem::openFileForRead(const Twine &Path) {
     return E.getError();
   }
 
-  auto *F = dyn_cast<RedirectingFileEntry>(*E);
+  auto *F = dyn_cast<RedirectingFileSystem::RedirectingFileEntry>(*E);
   if (!F) // FIXME: errc::not_a_file?
     return make_error_code(llvm::errc::invalid_argument);
 
@@ -1897,7 +1723,7 @@ RedirectingFileSystem::openFileForRead(const Twine &Path) {
 std::error_code
 RedirectingFileSystem::getRealPath(const Twine &Path,
                                    SmallVectorImpl<char> &Output) const {
-  ErrorOr<Entry *> Result = lookupPath(Path);
+  ErrorOr<RedirectingFileSystem::Entry *> Result = lookupPath(Path);
   if (!Result) {
     if (IsFallthrough &&
         Result.getError() == llvm::errc::no_such_file_or_directory) {
@@ -1906,7 +1732,8 @@ RedirectingFileSystem::getRealPath(const Twine &Path,
     return Result.getError();
   }
 
-  if (auto *F = dyn_cast<RedirectingFileEntry>(*Result)) {
+  if (auto *F =
+          dyn_cast<RedirectingFileSystem::RedirectingFileEntry>(*Result)) {
     return ExternalFS->getRealPath(F->getExternalContentsPath(), Output);
   }
   // Even if there is a directory entry, fall back to ExternalFS if allowed,
@@ -1925,13 +1752,14 @@ vfs::getVFSFromYAML(std::unique_ptr<MemoryBuffer> Buffer,
                                        std::move(ExternalFS));
 }
 
-static void getVFSEntries(Entry *SrcE, SmallVectorImpl<StringRef> &Path,
+static void getVFSEntries(RedirectingFileSystem::Entry *SrcE,
+                          SmallVectorImpl<StringRef> &Path,
                           SmallVectorImpl<YAMLVFSEntry> &Entries) {
   auto Kind = SrcE->getKind();
-  if (Kind == EK_Directory) {
-    auto *DE = dyn_cast<RedirectingDirectoryEntry>(SrcE);
+  if (Kind == RedirectingFileSystem::EK_Directory) {
+    auto *DE = dyn_cast<RedirectingFileSystem::RedirectingDirectoryEntry>(SrcE);
     assert(DE && "Must be a directory");
-    for (std::unique_ptr<Entry> &SubEntry :
+    for (std::unique_ptr<RedirectingFileSystem::Entry> &SubEntry :
          llvm::make_range(DE->contents_begin(), DE->contents_end())) {
       Path.push_back(SubEntry->getName());
       getVFSEntries(SubEntry.get(), Path, Entries);
@@ -1940,8 +1768,8 @@ static void getVFSEntries(Entry *SrcE, SmallVectorImpl<StringRef> &Path,
     return;
   }
 
-  assert(Kind == EK_File && "Must be a EK_File");
-  auto *FE = dyn_cast<RedirectingFileEntry>(SrcE);
+  assert(Kind == RedirectingFileSystem::EK_File && "Must be a EK_File");
+  auto *FE = dyn_cast<RedirectingFileSystem::RedirectingFileEntry>(SrcE);
   assert(FE && "Must be a file");
   SmallString<128> VPath;
   for (auto &Comp : Path)
@@ -1958,7 +1786,7 @@ void vfs::collectVFSFromYAML(std::unique_ptr<MemoryBuffer> Buffer,
   RedirectingFileSystem *VFS = RedirectingFileSystem::create(
       std::move(Buffer), DiagHandler, YAMLFilePath, DiagContext,
       std::move(ExternalFS));
-  ErrorOr<Entry *> RootE = VFS->lookupPath("/");
+  ErrorOr<RedirectingFileSystem::Entry *> RootE = VFS->lookupPath("/");
   if (!RootE)
     return;
   SmallVector<StringRef, 8> Components;
@@ -2134,9 +1962,10 @@ void YAMLVFSWriter::write(llvm::raw_ostream &OS) {
 }
 
 VFSFromYamlDirIterImpl::VFSFromYamlDirIterImpl(
-    const Twine &_Path, RedirectingDirectoryEntry::iterator Begin,
-    RedirectingDirectoryEntry::iterator End, bool IterateExternalFS,
-    FileSystem &ExternalFS, std::error_code &EC)
+    const Twine &_Path,
+    RedirectingFileSystem::RedirectingDirectoryEntry::iterator Begin,
+    RedirectingFileSystem::RedirectingDirectoryEntry::iterator End,
+    bool IterateExternalFS, FileSystem &ExternalFS, std::error_code &EC)
     : Dir(_Path.str()), Current(Begin), End(End),
       IterateExternalFS(IterateExternalFS), ExternalFS(ExternalFS) {
   EC = incrementImpl(/*IsFirstTime=*/true);
@@ -2176,10 +2005,10 @@ std::error_code VFSFromYamlDirIterImpl::incrementContent(bool IsFirstTime) {
     llvm::sys::path::append(PathStr, (*Current)->getName());
     sys::fs::file_type Type;
     switch ((*Current)->getKind()) {
-    case EK_Directory:
+    case RedirectingFileSystem::EK_Directory:
       Type = sys::fs::file_type::directory_file;
       break;
-    case EK_File:
+    case RedirectingFileSystem::EK_File:
       Type = sys::fs::file_type::regular_file;
       break;
     }
diff --git a/lib/Support/Watchdog.cpp b/lib/Support/Watchdog.cpp
index be55e3122e70e804c90febf57ea437ac55dc0a88..246f3dc7a0cad37c90500677af5ccd82a7a3b3db 100644
--- a/lib/Support/Watchdog.cpp
+++ b/lib/Support/Watchdog.cpp
@@ -1,9 +1,8 @@
 //===---- Watchdog.cpp - Implement Watchdog ---------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Support/Windows/COM.inc b/lib/Support/Windows/COM.inc
index 54f3ecf28ec259da649bd8cdfc2da0e61e104b59..002182bc3939422e8921c9bfa58387a04bdaddb7 100644
--- a/lib/Support/Windows/COM.inc
+++ b/lib/Support/Windows/COM.inc
@@ -1,9 +1,8 @@
 //==- llvm/Support/Windows/COM.inc - Windows COM Implementation -*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Support/Windows/DynamicLibrary.inc b/lib/Support/Windows/DynamicLibrary.inc
index 1d47f0848a6dfac5e0b295b5fa344264dcd1a81a..71b206c4cf9ee6e3680f5b9b532f1c7afeb0e485 100644
--- a/lib/Support/Windows/DynamicLibrary.inc
+++ b/lib/Support/Windows/DynamicLibrary.inc
@@ -1,9 +1,8 @@
 //===- Win32/DynamicLibrary.cpp - Win32 DL Implementation -------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Support/Windows/Host.inc b/lib/Support/Windows/Host.inc
index 58c4dc5d678ff994bc3ba247f3bbc48fea396139..21b947f26df3eb0b7ad62c805391cc466612910f 100644
--- a/lib/Support/Windows/Host.inc
+++ b/lib/Support/Windows/Host.inc
@@ -1,9 +1,8 @@
 //===- llvm/Support/Win32/Host.inc ------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Support/Windows/Memory.inc b/lib/Support/Windows/Memory.inc
index 318e65aaa9ee71e8c92a298eab3fda1b95d86943..c9340c1e7b0fb895c244188571e4ebc5ec6dee02 100644
--- a/lib/Support/Windows/Memory.inc
+++ b/lib/Support/Windows/Memory.inc
@@ -1,9 +1,8 @@
 //===- Win32/Memory.cpp - Win32 Memory Implementation -----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Support/Windows/Mutex.inc b/lib/Support/Windows/Mutex.inc
index 0af145ec9a4e6c6c61dcf8fd98d9aa39912e744c..b55b14febf2cd23a4aa4448255c28069ae5ce530 100644
--- a/lib/Support/Windows/Mutex.inc
+++ b/lib/Support/Windows/Mutex.inc
@@ -1,9 +1,8 @@
 //===- llvm/Support/Win32/Mutex.inc - Win32 Mutex Implementation -*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Support/Windows/Path.inc b/lib/Support/Windows/Path.inc
index d34aa763124c9aaced0f36459df1564979b44ba7..012e5ce84068bce1686e2420060d3e0653396b5c 100644
--- a/lib/Support/Windows/Path.inc
+++ b/lib/Support/Windows/Path.inc
@@ -1,9 +1,8 @@
 //===- llvm/Support/Windows/Path.inc - Windows Path Impl --------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Support/Windows/Process.inc b/lib/Support/Windows/Process.inc
index 2b2d7923143438b33defe04ac9173655b28c92ef..b3c9aa0b83cb8d2dbd84582ced5e28915ffa6559 100644
--- a/lib/Support/Windows/Process.inc
+++ b/lib/Support/Windows/Process.inc
@@ -1,9 +1,8 @@
 //===- Win32/Process.cpp - Win32 Process Implementation ------- -*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Support/Windows/Program.inc b/lib/Support/Windows/Program.inc
index c037956603f268f4fd67618fd03a83b69109a39e..0f54e59ee55b62cb4732dd4dc7743a28b8f1247c 100644
--- a/lib/Support/Windows/Program.inc
+++ b/lib/Support/Windows/Program.inc
@@ -1,9 +1,8 @@
 //===- Win32/Program.cpp - Win32 Program Implementation ------- -*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Support/Windows/RWMutex.inc b/lib/Support/Windows/RWMutex.inc
index 5eb9351eee5268435ed702e60049ccd78a04bd4a..8df9bc394160338ee3d4a76c0fe4ad48f3a27379 100644
--- a/lib/Support/Windows/RWMutex.inc
+++ b/lib/Support/Windows/RWMutex.inc
@@ -1,9 +1,8 @@
 //= llvm/Support/Win32/Mutex.inc - Win32 Reader/Writer Mutual Exclusion Lock  =//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Support/Windows/Signals.inc b/lib/Support/Windows/Signals.inc
index 41eb5e593aa5ca5a404a0143b5a51aba52ed3f7f..01dc0574d9deb464232f7be7fd601720b21bc12b 100644
--- a/lib/Support/Windows/Signals.inc
+++ b/lib/Support/Windows/Signals.inc
@@ -1,9 +1,8 @@
 //===- Win32/Signals.cpp - Win32 Signals Implementation ---------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Support/Windows/ThreadLocal.inc b/lib/Support/Windows/ThreadLocal.inc
index 8be1c3ecfbb908baf27fb4d221d47f602114e94d..1e0ed955e9abee4b6b58fd15035bd45d34c8c6c0 100644
--- a/lib/Support/Windows/ThreadLocal.inc
+++ b/lib/Support/Windows/ThreadLocal.inc
@@ -1,9 +1,8 @@
 //= llvm/Support/Win32/ThreadLocal.inc - Win32 Thread Local Data -*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Support/Windows/Threading.inc b/lib/Support/Windows/Threading.inc
index decb48887af22a7a08f2495c3713766315d4098a..c53624c0dc9d09cb2d65e3d1bb70c33b571a4738 100644
--- a/lib/Support/Windows/Threading.inc
+++ b/lib/Support/Windows/Threading.inc
@@ -1,9 +1,8 @@
 //===- Windows/Threading.inc - Win32 Threading Implementation - -*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -14,7 +13,7 @@
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/Twine.h"
 
-#include "Windows/WindowsSupport.h"
+#include "WindowsSupport.h"
 #include <process.h>
 
 // Windows will at times define MemoryFence.
diff --git a/lib/Support/Windows/Watchdog.inc b/lib/Support/Windows/Watchdog.inc
index fab2bdf2a94118a34922a9fbb9dc68cc93c98b94..a362c999de7691911dc98d1665a272afc04699b5 100644
--- a/lib/Support/Windows/Watchdog.inc
+++ b/lib/Support/Windows/Watchdog.inc
@@ -1,9 +1,8 @@
 //===--- Windows/Watchdog.inc - Windows Watchdog Implementation -*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Support/Windows/WindowsSupport.h b/lib/Support/Windows/WindowsSupport.h
index 979cc5d0139048d881ef63ae4d1210dfef2dec9e..fed9b2f462ef7d4c1c1087fadec6d7e417164dff 100644
--- a/lib/Support/Windows/WindowsSupport.h
+++ b/lib/Support/Windows/WindowsSupport.h
@@ -1,9 +1,8 @@
 //===- WindowsSupport.h - Common Windows Include File -----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Support/WithColor.cpp b/lib/Support/WithColor.cpp
index cf4c10956f218fba4ef020c9061ceb164dcd2703..345dd9cf39492762a28ad06efbc3bcc37f0b5d3f 100644
--- a/lib/Support/WithColor.cpp
+++ b/lib/Support/WithColor.cpp
@@ -1,9 +1,8 @@
 //===- WithColor.cpp ------------------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Support/YAMLParser.cpp b/lib/Support/YAMLParser.cpp
index 9ef1410b99a5bd82354a0979c5412c1c169abb9b..9b2fe9c4418a156834dd7da1e1ecffac796abeaa 100644
--- a/lib/Support/YAMLParser.cpp
+++ b/lib/Support/YAMLParser.cpp
@@ -1,9 +1,8 @@
 //===- YAMLParser.cpp - Simple YAML parser --------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Support/YAMLTraits.cpp b/lib/Support/YAMLTraits.cpp
index b9bbee7883c61b5c9f29c72f0e55aefd6e1c71f6..4b5bf6ad30d36c68d65ebfd5a0317b3978126daa 100644
--- a/lib/Support/YAMLTraits.cpp
+++ b/lib/Support/YAMLTraits.cpp
@@ -1,9 +1,8 @@
 //===- lib/Support/YAMLTraits.cpp -----------------------------------------===//
 //
-//                             The LLVM Linker
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Support/circular_raw_ostream.cpp b/lib/Support/circular_raw_ostream.cpp
index e768f17cd00d459f4ca056519abe6b279db4e3ed..acd230704ff8de45de7ba6cfb47a357cc8905c99 100644
--- a/lib/Support/circular_raw_ostream.cpp
+++ b/lib/Support/circular_raw_ostream.cpp
@@ -1,9 +1,8 @@
 //===- circular_raw_ostream.cpp - Implement circular_raw_ostream ----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Support/raw_os_ostream.cpp b/lib/Support/raw_os_ostream.cpp
index 44f2325d7f8a4b6d369c0274344c6d14afb1eb1f..81f0d739696ef877c8012067f6f8eaecd576bc79 100644
--- a/lib/Support/raw_os_ostream.cpp
+++ b/lib/Support/raw_os_ostream.cpp
@@ -1,9 +1,8 @@
 //===--- raw_os_ostream.cpp - Implement the raw_os_ostream class ----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Support/raw_ostream.cpp b/lib/Support/raw_ostream.cpp
index 81df38f341c620124df491afa87d29eda97c09c6..4124121b86b1ff970f2c1223634b23b05f0f8acd 100644
--- a/lib/Support/raw_ostream.cpp
+++ b/lib/Support/raw_ostream.cpp
@@ -1,9 +1,8 @@
 //===--- raw_ostream.cpp - Implement the raw_ostream classes --------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -914,3 +913,5 @@ void raw_null_ostream::pwrite_impl(const char *Ptr, size_t Size,
                                    uint64_t Offset) {}
 
 void raw_pwrite_stream::anchor() {}
+
+void buffer_ostream::anchor() {}
diff --git a/lib/TableGen/Error.cpp b/lib/TableGen/Error.cpp
index e6171c71efc000639359f07c401e8b47ba08170d..7523b32ca0e525d038be7bf8e58f99a5b6557d15 100644
--- a/lib/TableGen/Error.cpp
+++ b/lib/TableGen/Error.cpp
@@ -1,9 +1,8 @@
 //===- Error.cpp - tblgen error handling helper routines --------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/TableGen/JSONBackend.cpp b/lib/TableGen/JSONBackend.cpp
index 36cb2208a294fc09ec27edcc11a1225737dd0da5..196644cda66712ba89fe408613e3c7adce9e148c 100644
--- a/lib/TableGen/JSONBackend.cpp
+++ b/lib/TableGen/JSONBackend.cpp
@@ -1,9 +1,8 @@
 //===- JSONBackend.cpp - Generate a JSON dump of all records. -*- C++ -*-=====//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/TableGen/LLVMBuild.txt b/lib/TableGen/LLVMBuild.txt
index 54cedfd5918b93f005e3cd77c2632f050b4a6b4a..99a8b8349f3c49f7d2af276d7e7895e011e0495a 100644
--- a/lib/TableGen/LLVMBuild.txt
+++ b/lib/TableGen/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/TableGen/LLVMBuild.txt -----------------------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/TableGen/Main.cpp b/lib/TableGen/Main.cpp
index df4088ad2f867ec9787a9a20a39e17812a2f4464..bcd39584e450ec71f3e87e76c0853ac3b765fc88 100644
--- a/lib/TableGen/Main.cpp
+++ b/lib/TableGen/Main.cpp
@@ -1,9 +1,8 @@
 //===- Main.cpp - Top-Level TableGen implementation -----------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -100,23 +99,39 @@ int llvm::TableGenMain(char *argv0, TableGenMainFn *MainFn) {
   if (Parser.ParseFile())
     return 1;
 
-  std::error_code EC;
-  ToolOutputFile Out(OutputFilename, EC, sys::fs::F_Text);
-  if (EC)
-    return reportError(argv0, "error opening " + OutputFilename + ":" +
-                                  EC.message() + "\n");
+  // Write output to memory.
+  std::string OutString;
+  raw_string_ostream Out(OutString);
+  if (MainFn(Out, Records))
+    return 1;
+
+  // Always write the depfile, even if the main output hasn't changed.
+  // If it's missing, Ninja considers the output dirty.  If this was below
+  // the early exit below and someone deleted the .inc.d file but not the .inc
+  // file, tablegen would never write the depfile.
   if (!DependFilename.empty()) {
     if (int Ret = createDependencyFile(Parser, argv0))
       return Ret;
   }
 
-  if (MainFn(Out.os(), Records))
-    return 1;
+  // Only updates the real output file if there are any differences.
+  // This prevents recompilation of all the files depending on it if there
+  // aren't any.
+  if (auto ExistingOrErr = MemoryBuffer::getFile(OutputFilename))
+    if (std::move(ExistingOrErr.get())->getBuffer() == Out.str())
+      return 0;
+
+  std::error_code EC;
+  ToolOutputFile OutFile(OutputFilename, EC, sys::fs::F_Text);
+  if (EC)
+    return reportError(argv0, "error opening " + OutputFilename + ":" +
+                                  EC.message() + "\n");
+  OutFile.os() << Out.str();
 
   if (ErrorsPrinted > 0)
     return reportError(argv0, Twine(ErrorsPrinted) + " errors.\n");
 
   // Declare success.
-  Out.keep();
+  OutFile.keep();
   return 0;
 }
diff --git a/lib/TableGen/Record.cpp b/lib/TableGen/Record.cpp
index cf1685a2e8c2c1e1fd40ae5f98e6ac57b7479c3d..ab3dfce5e84f625fb23d2abe6a61acb354d2dc00 100644
--- a/lib/TableGen/Record.cpp
+++ b/lib/TableGen/Record.cpp
@@ -1,9 +1,8 @@
 //===- Record.cpp - Record implementation ---------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -1694,6 +1693,137 @@ Init *FieldInit::Fold(Record *CurRec) const {
   return const_cast<FieldInit *>(this);
 }
 
+static void ProfileCondOpInit(FoldingSetNodeID &ID,
+                             ArrayRef<Init *> CondRange,
+                             ArrayRef<Init *> ValRange,
+                             const RecTy *ValType) {
+  assert(CondRange.size() == ValRange.size() &&
+         "Number of conditions and values must match!");
+  ID.AddPointer(ValType);
+  ArrayRef<Init *>::iterator Case = CondRange.begin();
+  ArrayRef<Init *>::iterator Val = ValRange.begin();
+
+  while (Case != CondRange.end()) {
+    ID.AddPointer(*Case++);
+    ID.AddPointer(*Val++);
+  }
+}
+
+void CondOpInit::Profile(FoldingSetNodeID &ID) const {
+  ProfileCondOpInit(ID,
+      makeArrayRef(getTrailingObjects<Init *>(), NumConds),
+      makeArrayRef(getTrailingObjects<Init *>() + NumConds, NumConds),
+      ValType);
+}
+
+CondOpInit *
+CondOpInit::get(ArrayRef<Init *> CondRange,
+                ArrayRef<Init *> ValRange, RecTy *Ty) {
+  assert(CondRange.size() == ValRange.size() &&
+         "Number of conditions and values must match!");
+
+  static FoldingSet<CondOpInit> ThePool;
+  FoldingSetNodeID ID;
+  ProfileCondOpInit(ID, CondRange, ValRange, Ty);
+
+  void *IP = nullptr;
+  if (CondOpInit *I = ThePool.FindNodeOrInsertPos(ID, IP))
+    return I;
+
+  void *Mem = Allocator.Allocate(totalSizeToAlloc<Init *>(2*CondRange.size()),
+                                 alignof(BitsInit));
+  CondOpInit *I = new(Mem) CondOpInit(CondRange.size(), Ty);
+
+  std::uninitialized_copy(CondRange.begin(), CondRange.end(),
+                          I->getTrailingObjects<Init *>());
+  std::uninitialized_copy(ValRange.begin(), ValRange.end(),
+                          I->getTrailingObjects<Init *>()+CondRange.size());
+  ThePool.InsertNode(I, IP);
+  return I;
+}
+
+Init *CondOpInit::resolveReferences(Resolver &R) const {
+  SmallVector<Init*, 4> NewConds;
+  bool Changed = false;
+  for (const Init *Case : getConds()) {
+    Init *NewCase = Case->resolveReferences(R);
+    NewConds.push_back(NewCase);
+    Changed |= NewCase != Case;
+  }
+
+  SmallVector<Init*, 4> NewVals;
+  for (const Init *Val : getVals()) {
+    Init *NewVal = Val->resolveReferences(R);
+    NewVals.push_back(NewVal);
+    Changed |= NewVal != Val;
+  }
+
+  if (Changed)
+    return (CondOpInit::get(NewConds, NewVals,
+            getValType()))->Fold(R.getCurrentRecord());
+
+  return const_cast<CondOpInit *>(this);
+}
+
+Init *CondOpInit::Fold(Record *CurRec) const {
+  for ( unsigned i = 0; i < NumConds; ++i) {
+    Init *Cond = getCond(i);
+    Init *Val = getVal(i);
+
+    if (IntInit *CondI = dyn_cast_or_null<IntInit>(
+            Cond->convertInitializerTo(IntRecTy::get()))) {
+      if (CondI->getValue())
+        return Val->convertInitializerTo(getValType());
+    } else
+     return const_cast<CondOpInit *>(this);
+  }
+
+  PrintFatalError(CurRec->getLoc(),
+                  CurRec->getName() +
+                  " does not have any true condition in:" +
+                  this->getAsString());
+  return nullptr;
+}
+
+bool CondOpInit::isConcrete() const {
+  for (const Init *Case : getConds())
+    if (!Case->isConcrete())
+      return false;
+
+  for (const Init *Val : getVals())
+    if (!Val->isConcrete())
+      return false;
+
+  return true;
+}
+
+bool CondOpInit::isComplete() const {
+  for (const Init *Case : getConds())
+    if (!Case->isComplete())
+      return false;
+
+  for (const Init *Val : getVals())
+    if (!Val->isConcrete())
+      return false;
+
+  return true;
+}
+
+std::string CondOpInit::getAsString() const {
+  std::string Result = "!cond(";
+  for (unsigned i = 0; i < getNumConds(); i++) {
+    Result += getCond(i)->getAsString() + ": ";
+    Result += getVal(i)->getAsString();
+    if (i != getNumConds()-1)
+      Result += ", ";
+  }
+  return Result + ")";
+}
+
+Init *CondOpInit::getBit(unsigned Bit) const {
+  return VarBitInit::get(const_cast<CondOpInit *>(this), Bit);
+}
+
 static void ProfileDagInit(FoldingSetNodeID &ID, Init *V, StringInit *VN,
                            ArrayRef<Init *> ArgRange,
                            ArrayRef<StringInit *> NameRange) {
diff --git a/lib/TableGen/SetTheory.cpp b/lib/TableGen/SetTheory.cpp
index 733e0aeef6234814fb430b18327ef3e926533100..a870e41d58f8da0251924744bd303d16c77da1aa 100644
--- a/lib/TableGen/SetTheory.cpp
+++ b/lib/TableGen/SetTheory.cpp
@@ -1,9 +1,8 @@
 //===- SetTheory.cpp - Generate ordered sets from DAG expressions ---------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/TableGen/StringMatcher.cpp b/lib/TableGen/StringMatcher.cpp
index 2c4d1f33997d21205e782e6fa122c3b2827507b6..2fca068893f3ac5bb959b48428753c6c36e45618 100644
--- a/lib/TableGen/StringMatcher.cpp
+++ b/lib/TableGen/StringMatcher.cpp
@@ -1,9 +1,8 @@
 //===- StringMatcher.cpp - Generate a matcher for input strings -----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/TableGen/TGLexer.cpp b/lib/TableGen/TGLexer.cpp
index 16aeee561075c54a2f103d8fc185305bdc1a55e6..dbbffda7b3510398d8944836e8ef67daf1e56c09 100644
--- a/lib/TableGen/TGLexer.cpp
+++ b/lib/TableGen/TGLexer.cpp
@@ -1,9 +1,8 @@
 //===- TGLexer.cpp - Lexer for TableGen -----------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -545,6 +544,7 @@ tgtok::TokKind TGLexer::LexExclaim() {
     .Case("ge", tgtok::XGe)
     .Case("gt", tgtok::XGt)
     .Case("if", tgtok::XIf)
+    .Case("cond", tgtok::XCond)
     .Case("isa", tgtok::XIsA)
     .Case("head", tgtok::XHead)
     .Case("tail", tgtok::XTail)
diff --git a/lib/TableGen/TGLexer.h b/lib/TableGen/TGLexer.h
index e9980b36b97bb0bfd29a643a6b9289bcbb4aff1f..0cb9ae3f567cfbdca2ee78e7a4b406b433157567 100644
--- a/lib/TableGen/TGLexer.h
+++ b/lib/TableGen/TGLexer.h
@@ -1,9 +1,8 @@
 //===- TGLexer.h - Lexer for TableGen Files ---------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -51,7 +50,7 @@ namespace tgtok {
 
     // !keywords.
     XConcat, XADD, XAND, XOR, XSRA, XSRL, XSHL, XListConcat, XStrConcat, XCast,
-    XSubst, XForEach, XFoldl, XHead, XTail, XSize, XEmpty, XIf, XEq, XIsA, XDag,
+    XSubst, XForEach, XFoldl, XHead, XTail, XSize, XEmpty, XIf, XCond, XEq, XIsA, XDag,
     XNe, XLe, XLt, XGe, XGt,
 
     // Integer value.
diff --git a/lib/TableGen/TGParser.cpp b/lib/TableGen/TGParser.cpp
index 1d1f3603c83c9aa81d2324b56cc75bd779ebeb3d..22b4b16bc39a1ad10ea22905624dcb03f78e6f46 100644
--- a/lib/TableGen/TGParser.cpp
+++ b/lib/TableGen/TGParser.cpp
@@ -1,9 +1,8 @@
 //===- TGParser.cpp - Parser for TableGen Files ---------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -1445,6 +1444,9 @@ Init *TGParser::ParseOperation(Record *CurRec, RecTy *ItemType) {
     return (TernOpInit::get(Code, LHS, MHS, RHS, Type))->Fold(CurRec);
   }
 
+  case tgtok::XCond:
+    return ParseOperationCond(CurRec, ItemType);
+
   case tgtok::XFoldl: {
     // Value ::= !foldl '(' Id ',' Id ',' Value ',' Value ',' Value ')'
     Lex.Lex(); // eat the operation
@@ -1603,6 +1605,91 @@ RecTy *TGParser::ParseOperatorType() {
   return Type;
 }
 
+Init *TGParser::ParseOperationCond(Record *CurRec, RecTy *ItemType) {
+  Lex.Lex();  // eat the operation 'cond'
+
+  if (Lex.getCode() != tgtok::l_paren) {
+     TokError("expected '(' after !cond operator");
+     return nullptr;
+  }
+  Lex.Lex();  // eat the '('
+
+  // Parse through '[Case: Val,]+'
+  SmallVector<Init *, 4> Case;
+  SmallVector<Init *, 4> Val;
+  while (true) {
+    if (Lex.getCode() == tgtok::r_paren) {
+      Lex.Lex(); // eat the ')'
+      break;
+    }
+
+    Init *V = ParseValue(CurRec);
+    if (!V)
+      return nullptr;
+    Case.push_back(V);
+
+    if (Lex.getCode() != tgtok::colon) {
+      TokError("expected ':'  following a condition in !cond operator");
+      return nullptr;
+    }
+    Lex.Lex(); // eat the ':'
+
+    V = ParseValue(CurRec, ItemType);
+    if (!V)
+      return nullptr;
+    Val.push_back(V);
+
+    if (Lex.getCode() == tgtok::r_paren) {
+      Lex.Lex(); // eat the ')'
+      break;
+    }
+
+    if (Lex.getCode() != tgtok::comma) {
+      TokError("expected ',' or ')' following a value in !cond operator");
+      return nullptr;
+    }
+    Lex.Lex();  // eat the ','
+  }
+
+  if (Case.size() < 1) {
+    TokError("there should be at least 1 'condition : value' in the !cond operator");
+    return nullptr;
+  }
+
+  // resolve type
+  RecTy *Type = nullptr;
+  for (Init *V : Val) {
+    RecTy *VTy = nullptr;
+    if (TypedInit *Vt = dyn_cast<TypedInit>(V))
+      VTy = Vt->getType();
+    if (BitsInit *Vbits = dyn_cast<BitsInit>(V))
+      VTy = BitsRecTy::get(Vbits->getNumBits());
+    if (isa<BitInit>(V))
+      VTy = BitRecTy::get();
+
+    if (Type == nullptr) {
+      if (!isa<UnsetInit>(V))
+        Type = VTy;
+    } else {
+      if (!isa<UnsetInit>(V)) {
+        RecTy *RType = resolveTypes(Type, VTy);
+        if (!RType) {
+          TokError(Twine("inconsistent types '") + Type->getAsString() +
+                         "' and '" + VTy->getAsString() + "' for !cond");
+          return nullptr;
+        }
+        Type = RType;
+      }
+    }
+  }
+
+  if (!Type) {
+    TokError("could not determine type for !cond from its arguments");
+    return nullptr;
+  }
+  return CondOpInit::get(Case, Val, Type)->Fold(CurRec);
+}
+
 /// ParseSimpleValue - Parse a tblgen value.  This returns null on error.
 ///
 ///   SimpleValue ::= IDValue
@@ -1621,6 +1708,7 @@ RecTy *TGParser::ParseOperatorType() {
 ///   SimpleValue ::= SRLTOK '(' Value ',' Value ')'
 ///   SimpleValue ::= LISTCONCATTOK '(' Value ',' Value ')'
 ///   SimpleValue ::= STRCONCATTOK '(' Value ',' Value ')'
+///   SimpleValue ::= COND '(' [Value ':' Value,]+ ')'
 ///
 Init *TGParser::ParseSimpleValue(Record *CurRec, RecTy *ItemType,
                                  IDParseMode Mode) {
@@ -1933,6 +2021,7 @@ Init *TGParser::ParseSimpleValue(Record *CurRec, RecTy *ItemType,
   case tgtok::XListConcat:
   case tgtok::XStrConcat:   // Value ::= !binop '(' Value ',' Value ')'
   case tgtok::XIf:
+  case tgtok::XCond:
   case tgtok::XFoldl:
   case tgtok::XForEach:
   case tgtok::XSubst: {  // Value ::= !ternop '(' Value ',' Value ',' Value ')'
diff --git a/lib/TableGen/TGParser.h b/lib/TableGen/TGParser.h
index e3849043513b58106ecb71d8d4768d1121a2e20d..38b782461bd56ef12fc22e4011d0ae5b3e8fc763 100644
--- a/lib/TableGen/TGParser.h
+++ b/lib/TableGen/TGParser.h
@@ -1,9 +1,8 @@
 //===- TGParser.h - Parser for TableGen Files -------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -194,6 +193,7 @@ private:  // Parser methods.
   bool ParseRangePiece(SmallVectorImpl<unsigned> &Ranges);
   RecTy *ParseType();
   Init *ParseOperation(Record *CurRec, RecTy *ItemType);
+  Init *ParseOperationCond(Record *CurRec, RecTy *ItemType);
   RecTy *ParseOperatorType();
   Init *ParseObjectName(MultiClass *CurMultiClass);
   Record *ParseClassID();
diff --git a/lib/TableGen/TableGenBackend.cpp b/lib/TableGen/TableGenBackend.cpp
index 77ed8414b15f978eaba530c0f0351b14eec2184d..e11b28e8cff99c49793fe9bcea7b82765466c075 100644
--- a/lib/TableGen/TableGenBackend.cpp
+++ b/lib/TableGen/TableGenBackend.cpp
@@ -1,9 +1,8 @@
 //===- TableGenBackend.cpp - Utilities for TableGen Backends ----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AArch64/AArch64.h b/lib/Target/AArch64/AArch64.h
index c36d9354f3ba6f234fab670da883f96ec374a37d..2e63e261c489acbaa0f985a93601be15d7d877bf 100644
--- a/lib/Target/AArch64/AArch64.h
+++ b/lib/Target/AArch64/AArch64.h
@@ -1,9 +1,8 @@
 //==-- AArch64.h - Top-level interface for AArch64  --------------*- C++ -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AArch64/AArch64.td b/lib/Target/AArch64/AArch64.td
index a13f6c848816f79c29ac3b5301d29cd57b464dda..766d365bbb59318421f6267267e501b9a98d02c1 100644
--- a/lib/Target/AArch64/AArch64.td
+++ b/lib/Target/AArch64/AArch64.td
@@ -1,9 +1,8 @@
 //=- AArch64.td - Describe the AArch64 Target Machine --------*- tablegen -*-=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -127,7 +126,7 @@ def FeatureStrictAlign : SubtargetFeature<"strict-align",
                                           "Disallow all unaligned memory "
                                           "access">;
 
-foreach i = {1-7,18,20} in
+foreach i = {1-7,9-15,18,20-28} in
     def FeatureReserveX#i : SubtargetFeature<"reserve-x"#i, "ReserveXRegister["#i#"]", "true",
                                              "Reserve X"#i#", making it unavailable "
                                              "as a GPR">;
@@ -188,14 +187,18 @@ def FeatureFuseAES : SubtargetFeature<
     "fuse-aes", "HasFuseAES", "true",
     "CPU fuses AES crypto operations">;
 
-def FeatureFuseCryptoEOR : SubtargetFeature<
-    "fuse-crypto-eor", "HasFuseCryptoEOR", "true",
-    "CPU fuses AES/PMULL and EOR operations">;
+def FeatureFuseArithmeticLogic : SubtargetFeature<
+    "fuse-arith-logic", "HasFuseArithmeticLogic", "true",
+    "CPU fuses arithmetic and logic operations">;
 
 def FeatureFuseCCSelect : SubtargetFeature<
     "fuse-csel", "HasFuseCCSelect", "true",
     "CPU fuses conditional select operations">;
 
+def FeatureFuseCryptoEOR : SubtargetFeature<
+    "fuse-crypto-eor", "HasFuseCryptoEOR", "true",
+    "CPU fuses AES/PMULL and EOR operations">;
+
 def FeatureFuseLiterals : SubtargetFeature<
     "fuse-literals", "HasFuseLiterals", "true",
     "CPU fuses literal generation operations">;
@@ -306,14 +309,14 @@ def FeatureFRInt3264 : SubtargetFeature<"fptoint", "HasFRInt3264", "true",
 def FeatureSpecRestrict : SubtargetFeature<"specrestrict", "HasSpecRestrict",
   "true", "Enable architectural speculation restriction" >;
 
+def FeatureSB : SubtargetFeature<"sb", "HasSB",
+  "true", "Enable v8.5 Speculation Barrier" >;
+
 def FeatureSSBS : SubtargetFeature<"ssbs", "HasSSBS",
   "true", "Enable Speculative Store Bypass Safe bit" >;
 
-def FeatureSpecCtrl : SubtargetFeature<"specctrl", "HasSpecCtrl", "true",
-  "Enable speculation control barrier" >;
-
-def FeaturePredCtrl : SubtargetFeature<"predctrl", "HasPredCtrl", "true",
-  "Enable execution and data prediction invalidation instructions" >;
+def FeaturePredRes : SubtargetFeature<"predres", "HasPredRes", "true",
+  "Enable v8.5a execution and data prediction invalidation instructions" >;
 
 def FeatureCacheDeepPersist : SubtargetFeature<"ccdp", "HasCCDP",
     "true", "Enable v8.5 Cache Clean to Point of Deep Persistence" >;
@@ -352,7 +355,7 @@ def HasV8_4aOps : SubtargetFeature<"v8.4a", "HasV8_4aOps", "true",
 def HasV8_5aOps : SubtargetFeature<
   "v8.5a", "HasV8_5aOps", "true", "Support ARM v8.5a instructions",
   [HasV8_4aOps, FeatureAltFPCmp, FeatureFRInt3264, FeatureSpecRestrict,
-   FeatureSSBS, FeatureSpecCtrl, FeaturePredCtrl, FeatureCacheDeepPersist,
+   FeatureSSBS, FeatureSB, FeaturePredRes, FeatureCacheDeepPersist,
    FeatureBranchTargetId]
 >;
 
@@ -391,6 +394,7 @@ include "AArch64SchedFalkor.td"
 include "AArch64SchedKryo.td"
 include "AArch64SchedExynosM1.td"
 include "AArch64SchedExynosM3.td"
+include "AArch64SchedExynosM4.td"
 include "AArch64SchedThunderX.td"
 include "AArch64SchedThunderX2T99.td"
 
@@ -540,6 +544,26 @@ def ProcExynosM3 : SubtargetFeature<"exynosm3", "ARMProcFamily", "ExynosM3",
                                      FeaturePredictableSelectIsExpensive,
                                      FeatureZCZeroingFP]>;
 
+def ProcExynosM4 : SubtargetFeature<"exynosm4", "ARMProcFamily", "ExynosM3",
+                                    "Samsung Exynos-M4 processors",
+                                    [HasV8_2aOps,
+                                     FeatureArithmeticBccFusion,
+                                     FeatureArithmeticCbzFusion,
+                                     FeatureCrypto,
+                                     FeatureDotProd,
+                                     FeatureExynosCheapAsMoveHandling,
+                                     FeatureForce32BitJumpTables,
+                                     FeatureFP16FML,
+                                     FeatureFuseAddress,
+                                     FeatureFuseAES,
+                                     FeatureFuseArithmeticLogic,
+                                     FeatureFuseCCSelect,
+                                     FeatureFuseLiterals,
+                                     FeatureLSLFast,
+                                     FeaturePerfMon,
+                                     FeaturePostRAScheduler,
+                                     FeatureZCZeroing]>;
+
 def ProcKryo    : SubtargetFeature<"kryo", "ARMProcFamily", "Kryo",
                                    "Qualcomm Kryo processors", [
                                    FeatureCRC,
@@ -677,7 +701,7 @@ def : ProcessorModel<"cyclone", CycloneModel, [ProcCyclone]>;
 def : ProcessorModel<"exynos-m1", ExynosM1Model, [ProcExynosM1]>;
 def : ProcessorModel<"exynos-m2", ExynosM1Model, [ProcExynosM2]>;
 def : ProcessorModel<"exynos-m3", ExynosM3Model, [ProcExynosM3]>;
-def : ProcessorModel<"exynos-m4", ExynosM3Model, [ProcExynosM3]>;
+def : ProcessorModel<"exynos-m4", ExynosM4Model, [ProcExynosM4]>;
 def : ProcessorModel<"falkor", FalkorModel, [ProcFalkor]>;
 def : ProcessorModel<"saphira", FalkorModel, [ProcSaphira]>;
 def : ProcessorModel<"kryo", KryoModel, [ProcKryo]>;
@@ -691,6 +715,9 @@ def : ProcessorModel<"thunderx2t99", ThunderX2T99Model, [ProcThunderX2T99]>;
 // FIXME: HiSilicon TSV110 is currently modeled as a Cortex-A57.
 def : ProcessorModel<"tsv110", CortexA57Model, [ProcTSV110]>;
 
+// Alias for the latest Apple processor model supported by LLVM.
+def : ProcessorModel<"apple-latest", CycloneModel, [ProcCyclone]>;
+
 //===----------------------------------------------------------------------===//
 // Assembly parser
 //===----------------------------------------------------------------------===//
diff --git a/lib/Target/AArch64/AArch64A53Fix835769.cpp b/lib/Target/AArch64/AArch64A53Fix835769.cpp
index 30232afaf0248b0ab475f9c1af6a2ec412d53b60..e80fe2cada09c15716124a75c6dc4b9774e96934 100644
--- a/lib/Target/AArch64/AArch64A53Fix835769.cpp
+++ b/lib/Target/AArch64/AArch64A53Fix835769.cpp
@@ -1,9 +1,8 @@
 //===-- AArch64A53Fix835769.cpp -------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 // This pass changes code to work around Cortex-A53 erratum 835769.
diff --git a/lib/Target/AArch64/AArch64A57FPLoadBalancing.cpp b/lib/Target/AArch64/AArch64A57FPLoadBalancing.cpp
index 452fbd3488b0b5e772e0f4eef73b4f604e656df4..92c8c4955d506c3731bc1773451ce59ff5569f43 100644
--- a/lib/Target/AArch64/AArch64A57FPLoadBalancing.cpp
+++ b/lib/Target/AArch64/AArch64A57FPLoadBalancing.cpp
@@ -1,9 +1,8 @@
 //===-- AArch64A57FPLoadBalancing.cpp - Balance FP ops statically on A57---===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 // For best-case performance on Cortex-A57, we should try to use a balanced
diff --git a/lib/Target/AArch64/AArch64AdvSIMDScalarPass.cpp b/lib/Target/AArch64/AArch64AdvSIMDScalarPass.cpp
index 22b0c1e3b4715ec525dd638804e37855f38a8459..89404463e1f07e3b321a665329e2e829fe69cb12 100644
--- a/lib/Target/AArch64/AArch64AdvSIMDScalarPass.cpp
+++ b/lib/Target/AArch64/AArch64AdvSIMDScalarPass.cpp
@@ -1,9 +1,8 @@
 //===-- AArch64AdvSIMDScalar.cpp - Replace dead defs w/ zero reg --===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 // When profitable, replace GPR targeting i64 instructions with their
diff --git a/lib/Target/AArch64/AArch64AsmPrinter.cpp b/lib/Target/AArch64/AArch64AsmPrinter.cpp
index 828fbb2de7ee7600e9349bfbacd7fe8a6207ac84..61196ba2d85f4f2d39dd1c6bf048cc243146a945 100644
--- a/lib/Target/AArch64/AArch64AsmPrinter.cpp
+++ b/lib/Target/AArch64/AArch64AsmPrinter.cpp
@@ -1,9 +1,8 @@
 //===- AArch64AsmPrinter.cpp - AArch64 LLVM assembly writer ---------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -29,6 +28,7 @@
 #include "llvm/ADT/Triple.h"
 #include "llvm/ADT/Twine.h"
 #include "llvm/BinaryFormat/COFF.h"
+#include "llvm/BinaryFormat/ELF.h"
 #include "llvm/CodeGen/AsmPrinter.h"
 #include "llvm/CodeGen/MachineBasicBlock.h"
 #include "llvm/CodeGen/MachineFunction.h"
@@ -44,6 +44,7 @@
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCInst.h"
 #include "llvm/MC/MCInstBuilder.h"
+#include "llvm/MC/MCSectionELF.h"
 #include "llvm/MC/MCStreamer.h"
 #include "llvm/MC/MCSymbol.h"
 #include "llvm/Support/Casting.h"
@@ -96,6 +97,10 @@ public:
   void LowerPATCHABLE_FUNCTION_EXIT(const MachineInstr &MI);
   void LowerPATCHABLE_TAIL_CALL(const MachineInstr &MI);
 
+  std::map<std::pair<unsigned, uint32_t>, MCSymbol *> HwasanMemaccessSymbols;
+  void LowerHWASAN_CHECK_MEMACCESS(const MachineInstr &MI);
+  void EmitHwasanMemaccessSymbols(Module &M);
+
   void EmitSled(const MachineInstr &MI, SledKind Kind);
 
   /// tblgen'erated driver function for lowering simple MI->MC
@@ -230,7 +235,109 @@ void AArch64AsmPrinter::EmitSled(const MachineInstr &MI, SledKind Kind)
   recordSled(CurSled, MI, Kind);
 }
 
+void AArch64AsmPrinter::LowerHWASAN_CHECK_MEMACCESS(const MachineInstr &MI) {
+  unsigned Reg = MI.getOperand(0).getReg();
+  uint32_t AccessInfo = MI.getOperand(1).getImm();
+  MCSymbol *&Sym = HwasanMemaccessSymbols[{Reg, AccessInfo}];
+  if (!Sym) {
+    // FIXME: Make this work on non-ELF.
+    if (!TM.getTargetTriple().isOSBinFormatELF())
+      report_fatal_error("llvm.hwasan.check.memaccess only supported on ELF");
+
+    std::string SymName = "__hwasan_check_x" + utostr(Reg - AArch64::X0) + "_" +
+                          utostr(AccessInfo);
+    Sym = OutContext.getOrCreateSymbol(SymName);
+  }
+
+  EmitToStreamer(*OutStreamer,
+                 MCInstBuilder(AArch64::BL)
+                     .addExpr(MCSymbolRefExpr::create(Sym, OutContext)));
+}
+
+void AArch64AsmPrinter::EmitHwasanMemaccessSymbols(Module &M) {
+  if (HwasanMemaccessSymbols.empty())
+    return;
+
+  const Triple &TT = TM.getTargetTriple();
+  assert(TT.isOSBinFormatELF());
+  std::unique_ptr<MCSubtargetInfo> STI(
+      TM.getTarget().createMCSubtargetInfo(TT.str(), "", ""));
+
+  MCSymbol *HwasanTagMismatchSym =
+      OutContext.getOrCreateSymbol("__hwasan_tag_mismatch");
+
+  for (auto &P : HwasanMemaccessSymbols) {
+    unsigned Reg = P.first.first;
+    uint32_t AccessInfo = P.first.second;
+    MCSymbol *Sym = P.second;
+
+    OutStreamer->SwitchSection(OutContext.getELFSection(
+        ".text.hot", ELF::SHT_PROGBITS,
+        ELF::SHF_EXECINSTR | ELF::SHF_ALLOC | ELF::SHF_GROUP, 0,
+        Sym->getName()));
+
+    OutStreamer->EmitSymbolAttribute(Sym, MCSA_ELF_TypeFunction);
+    OutStreamer->EmitSymbolAttribute(Sym, MCSA_Weak);
+    OutStreamer->EmitSymbolAttribute(Sym, MCSA_Hidden);
+    OutStreamer->EmitLabel(Sym);
+
+    OutStreamer->EmitInstruction(MCInstBuilder(AArch64::UBFMXri)
+                                     .addReg(AArch64::X16)
+                                     .addReg(Reg)
+                                     .addImm(4)
+                                     .addImm(55),
+                                 *STI);
+    OutStreamer->EmitInstruction(MCInstBuilder(AArch64::LDRBBroX)
+                                     .addReg(AArch64::W16)
+                                     .addReg(AArch64::X9)
+                                     .addReg(AArch64::X16)
+                                     .addImm(0)
+                                     .addImm(0),
+                                 *STI);
+    OutStreamer->EmitInstruction(MCInstBuilder(AArch64::UBFMXri)
+                                     .addReg(AArch64::X17)
+                                     .addReg(Reg)
+                                     .addImm(56)
+                                     .addImm(63),
+                                 *STI);
+    OutStreamer->EmitInstruction(MCInstBuilder(AArch64::SUBSWrs)
+                                     .addReg(AArch64::WZR)
+                                     .addReg(AArch64::W16)
+                                     .addReg(AArch64::W17)
+                                     .addImm(0),
+                                 *STI);
+    MCSymbol *HandleMismatchSym = OutContext.createTempSymbol();
+    OutStreamer->EmitInstruction(
+        MCInstBuilder(AArch64::Bcc)
+            .addImm(AArch64CC::NE)
+            .addExpr(MCSymbolRefExpr::create(HandleMismatchSym, OutContext)),
+        *STI);
+    OutStreamer->EmitInstruction(
+        MCInstBuilder(AArch64::RET).addReg(AArch64::LR), *STI);
+
+    OutStreamer->EmitLabel(HandleMismatchSym);
+    if (Reg != AArch64::X0)
+      OutStreamer->EmitInstruction(MCInstBuilder(AArch64::ORRXrs)
+                                       .addReg(AArch64::X0)
+                                       .addReg(AArch64::XZR)
+                                       .addReg(Reg)
+                                       .addImm(0),
+                                   *STI);
+    OutStreamer->EmitInstruction(MCInstBuilder(AArch64::MOVZXi)
+                                     .addReg(AArch64::X1)
+                                     .addImm(AccessInfo)
+                                     .addImm(0),
+                                 *STI);
+    OutStreamer->EmitInstruction(
+        MCInstBuilder(AArch64::B)
+            .addExpr(MCSymbolRefExpr::create(HwasanTagMismatchSym, OutContext)),
+        *STI);
+  }
+}
+
 void AArch64AsmPrinter::EmitEndOfAsmFile(Module &M) {
+  EmitHwasanMemaccessSymbols(M);
+
   const Triple &TT = TM.getTargetTriple();
   if (TT.isOSBinFormatMachO()) {
     // Funny Darwin hack: This flag tells the linker that no global symbols
@@ -471,9 +578,18 @@ void AArch64AsmPrinter::EmitJumpTableInfo() {
   const std::vector<MachineJumpTableEntry> &JT = MJTI->getJumpTables();
   if (JT.empty()) return;
 
+  const Function &F = MF->getFunction();
   const TargetLoweringObjectFile &TLOF = getObjFileLowering();
-  MCSection *ReadOnlySec = TLOF.getSectionForJumpTable(MF->getFunction(), TM);
-  OutStreamer->SwitchSection(ReadOnlySec);
+  bool JTInDiffSection =
+      !STI->isTargetCOFF() ||
+      !TLOF.shouldPutJumpTableInFunctionSection(
+          MJTI->getEntryKind() == MachineJumpTableInfo::EK_LabelDifference32,
+          F);
+  if (JTInDiffSection) {
+      // Drop it in the readonly section.
+      MCSection *ReadOnlySec = TLOF.getSectionForJumpTable(F, TM);
+      OutStreamer->SwitchSection(ReadOnlySec);
+  }
 
   auto AFI = MF->getInfo<AArch64FunctionInfo>();
   for (unsigned JTI = 0, e = JT.size(); JTI != e; ++JTI) {
@@ -694,6 +810,34 @@ void AArch64AsmPrinter::EmitInstruction(const MachineInstr *MI) {
   switch (MI->getOpcode()) {
   default:
     break;
+    case AArch64::MOVMCSym: {
+    unsigned DestReg = MI->getOperand(0).getReg();
+    const MachineOperand &MO_Sym = MI->getOperand(1);
+    MachineOperand Hi_MOSym(MO_Sym), Lo_MOSym(MO_Sym);
+    MCOperand Hi_MCSym, Lo_MCSym;
+
+    Hi_MOSym.setTargetFlags(AArch64II::MO_G1 | AArch64II::MO_S);
+    Lo_MOSym.setTargetFlags(AArch64II::MO_G0 | AArch64II::MO_NC);
+
+    MCInstLowering.lowerOperand(Hi_MOSym, Hi_MCSym);
+    MCInstLowering.lowerOperand(Lo_MOSym, Lo_MCSym);
+
+    MCInst MovZ;
+    MovZ.setOpcode(AArch64::MOVZXi);
+    MovZ.addOperand(MCOperand::createReg(DestReg));
+    MovZ.addOperand(Hi_MCSym);
+    MovZ.addOperand(MCOperand::createImm(16));
+    EmitToStreamer(*OutStreamer, MovZ);
+
+    MCInst MovK;
+    MovK.setOpcode(AArch64::MOVKXi);
+    MovK.addOperand(MCOperand::createReg(DestReg));
+    MovK.addOperand(MCOperand::createReg(DestReg));
+    MovK.addOperand(Lo_MCSym);
+    MovK.addOperand(MCOperand::createImm(0));
+    EmitToStreamer(*OutStreamer, MovK);
+    return;
+  }
   case AArch64::MOVIv2d_ns:
     // If the target has <rdar://problem/16473581>, lower this
     // instruction to movi.16b instead.
@@ -716,6 +860,19 @@ void AArch64AsmPrinter::EmitInstruction(const MachineInstr *MI) {
       OutStreamer->EmitRawText(StringRef(OS.str()));
     }
     return;
+
+  case AArch64::EMITBKEY: {
+      ExceptionHandling ExceptionHandlingType = MAI->getExceptionHandlingType();
+      if (ExceptionHandlingType != ExceptionHandling::DwarfCFI &&
+          ExceptionHandlingType != ExceptionHandling::ARM)
+        return;
+
+      if (needsCFIMoves() == CFI_M_None)
+        return;
+
+      OutStreamer->EmitCFIBKeyFrame();
+      return;
+    }
   }
 
   // Tail calls use pseudo instructions so they have the proper code-gen
@@ -843,6 +1000,10 @@ void AArch64AsmPrinter::EmitInstruction(const MachineInstr *MI) {
     LowerPATCHABLE_TAIL_CALL(*MI);
     return;
 
+  case AArch64::HWASAN_CHECK_MEMACCESS:
+    LowerHWASAN_CHECK_MEMACCESS(*MI);
+    return;
+
   case AArch64::SEH_StackAlloc:
     TS->EmitARM64WinCFIAllocStack(MI->getOperand(0).getImm());
     return;
diff --git a/lib/Target/AArch64/AArch64BranchTargets.cpp b/lib/Target/AArch64/AArch64BranchTargets.cpp
index da70a624c5be5fb8c371acd31d5dcdbb058dc435..6fa3a462bc71a2268c20bda568269136c5eb9a83 100644
--- a/lib/Target/AArch64/AArch64BranchTargets.cpp
+++ b/lib/Target/AArch64/AArch64BranchTargets.cpp
@@ -1,9 +1,8 @@
 //===-- AArch64BranchTargets.cpp -- Harden code using v8.5-A BTI extension -==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AArch64/AArch64CallLowering.cpp b/lib/Target/AArch64/AArch64CallLowering.cpp
index 5980e5684e891342faef0993f5e4c44fd78b09a8..7f8cb7f5e6ff238a16249da745891dc04532e359 100644
--- a/lib/Target/AArch64/AArch64CallLowering.cpp
+++ b/lib/Target/AArch64/AArch64CallLowering.cpp
@@ -1,9 +1,8 @@
 //===--- AArch64CallLowering.cpp - Call lowering --------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
@@ -86,9 +85,10 @@ struct IncomingArgHandler : public CallLowering::ValueHandler {
 
   void assignValueToAddress(unsigned ValVReg, unsigned Addr, uint64_t Size,
                             MachinePointerInfo &MPO, CCValAssign &VA) override {
+    // FIXME: Get alignment
     auto MMO = MIRBuilder.getMF().getMachineMemOperand(
         MPO, MachineMemOperand::MOLoad | MachineMemOperand::MOInvariant, Size,
-        0);
+        1);
     MIRBuilder.buildLoad(ValVReg, Addr, *MMO);
   }
 
@@ -162,7 +162,7 @@ struct OutgoingArgHandler : public CallLowering::ValueHandler {
                     .getReg();
     }
     auto MMO = MIRBuilder.getMF().getMachineMemOperand(
-        MPO, MachineMemOperand::MOStore, Size, 0);
+        MPO, MachineMemOperand::MOStore, Size, 1);
     MIRBuilder.buildStore(ValVReg, Addr, *MMO);
   }
 
diff --git a/lib/Target/AArch64/AArch64CallLowering.h b/lib/Target/AArch64/AArch64CallLowering.h
index 1c2bd6a4de5d96b2bc0e3f679b7b29e40fc1933b..76337567c7a502ad4756db9a3f572bf666b8fb9b 100644
--- a/lib/Target/AArch64/AArch64CallLowering.h
+++ b/lib/Target/AArch64/AArch64CallLowering.h
@@ -1,9 +1,8 @@
 //===- AArch64CallLowering.h - Call lowering --------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
diff --git a/lib/Target/AArch64/AArch64CallingConvention.cpp b/lib/Target/AArch64/AArch64CallingConvention.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..02538a187611fb99e7c8fd09fb511871f15b2794
--- /dev/null
+++ b/lib/Target/AArch64/AArch64CallingConvention.cpp
@@ -0,0 +1,134 @@
+//=== AArch64CallingConvention.cpp - AArch64 CC impl ------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains the table-generated and custom routines for the AArch64
+// Calling Convention.
+//
+//===----------------------------------------------------------------------===//
+
+#include "AArch64CallingConvention.h"
+#include "AArch64.h"
+#include "AArch64InstrInfo.h"
+#include "AArch64Subtarget.h"
+#include "llvm/CodeGen/CallingConvLower.h"
+#include "llvm/CodeGen/TargetInstrInfo.h"
+#include "llvm/IR/CallingConv.h"
+using namespace llvm;
+
+static const MCPhysReg XRegList[] = {AArch64::X0, AArch64::X1, AArch64::X2,
+                                     AArch64::X3, AArch64::X4, AArch64::X5,
+                                     AArch64::X6, AArch64::X7};
+static const MCPhysReg HRegList[] = {AArch64::H0, AArch64::H1, AArch64::H2,
+                                     AArch64::H3, AArch64::H4, AArch64::H5,
+                                     AArch64::H6, AArch64::H7};
+static const MCPhysReg SRegList[] = {AArch64::S0, AArch64::S1, AArch64::S2,
+                                     AArch64::S3, AArch64::S4, AArch64::S5,
+                                     AArch64::S6, AArch64::S7};
+static const MCPhysReg DRegList[] = {AArch64::D0, AArch64::D1, AArch64::D2,
+                                     AArch64::D3, AArch64::D4, AArch64::D5,
+                                     AArch64::D6, AArch64::D7};
+static const MCPhysReg QRegList[] = {AArch64::Q0, AArch64::Q1, AArch64::Q2,
+                                     AArch64::Q3, AArch64::Q4, AArch64::Q5,
+                                     AArch64::Q6, AArch64::Q7};
+
+static bool finishStackBlock(SmallVectorImpl<CCValAssign> &PendingMembers,
+                             MVT LocVT, ISD::ArgFlagsTy &ArgFlags,
+                             CCState &State, unsigned SlotAlign) {
+  unsigned Size = LocVT.getSizeInBits() / 8;
+  unsigned StackAlign =
+      State.getMachineFunction().getDataLayout().getStackAlignment();
+  unsigned Align = std::min(ArgFlags.getOrigAlign(), StackAlign);
+
+  for (auto &It : PendingMembers) {
+    It.convertToMem(State.AllocateStack(Size, std::max(Align, SlotAlign)));
+    State.addLoc(It);
+    SlotAlign = 1;
+  }
+
+  // All pending members have now been allocated
+  PendingMembers.clear();
+  return true;
+}
+
+/// The Darwin variadic PCS places anonymous arguments in 8-byte stack slots. An
+/// [N x Ty] type must still be contiguous in memory though.
+static bool CC_AArch64_Custom_Stack_Block(
+      unsigned &ValNo, MVT &ValVT, MVT &LocVT, CCValAssign::LocInfo &LocInfo,
+      ISD::ArgFlagsTy &ArgFlags, CCState &State) {
+  SmallVectorImpl<CCValAssign> &PendingMembers = State.getPendingLocs();
+
+  // Add the argument to the list to be allocated once we know the size of the
+  // block.
+  PendingMembers.push_back(
+      CCValAssign::getPending(ValNo, ValVT, LocVT, LocInfo));
+
+  if (!ArgFlags.isInConsecutiveRegsLast())
+    return true;
+
+  return finishStackBlock(PendingMembers, LocVT, ArgFlags, State, 8);
+}
+
+/// Given an [N x Ty] block, it should be passed in a consecutive sequence of
+/// registers. If no such sequence is available, mark the rest of the registers
+/// of that type as used and place the argument on the stack.
+static bool CC_AArch64_Custom_Block(unsigned &ValNo, MVT &ValVT, MVT &LocVT,
+                                    CCValAssign::LocInfo &LocInfo,
+                                    ISD::ArgFlagsTy &ArgFlags, CCState &State) {
+  // Try to allocate a contiguous block of registers, each of the correct
+  // size to hold one member.
+  ArrayRef<MCPhysReg> RegList;
+  if (LocVT.SimpleTy == MVT::i64)
+    RegList = XRegList;
+  else if (LocVT.SimpleTy == MVT::f16)
+    RegList = HRegList;
+  else if (LocVT.SimpleTy == MVT::f32 || LocVT.is32BitVector())
+    RegList = SRegList;
+  else if (LocVT.SimpleTy == MVT::f64 || LocVT.is64BitVector())
+    RegList = DRegList;
+  else if (LocVT.SimpleTy == MVT::f128 || LocVT.is128BitVector())
+    RegList = QRegList;
+  else {
+    // Not an array we want to split up after all.
+    return false;
+  }
+
+  SmallVectorImpl<CCValAssign> &PendingMembers = State.getPendingLocs();
+
+  // Add the argument to the list to be allocated once we know the size of the
+  // block.
+  PendingMembers.push_back(
+      CCValAssign::getPending(ValNo, ValVT, LocVT, LocInfo));
+
+  if (!ArgFlags.isInConsecutiveRegsLast())
+    return true;
+
+  unsigned RegResult = State.AllocateRegBlock(RegList, PendingMembers.size());
+  if (RegResult) {
+    for (auto &It : PendingMembers) {
+      It.convertToReg(RegResult);
+      State.addLoc(It);
+      ++RegResult;
+    }
+    PendingMembers.clear();
+    return true;
+  }
+
+  // Mark all regs in the class as unavailable
+  for (auto Reg : RegList)
+    State.AllocateReg(Reg);
+
+  const AArch64Subtarget &Subtarget = static_cast<const AArch64Subtarget &>(
+      State.getMachineFunction().getSubtarget());
+  unsigned SlotAlign = Subtarget.isTargetDarwin() ? 1 : 8;
+
+  return finishStackBlock(PendingMembers, LocVT, ArgFlags, State, SlotAlign);
+}
+
+// TableGen provides definitions of the calling convention analysis entry
+// points.
+#include "AArch64GenCallingConv.inc"
diff --git a/lib/Target/AArch64/AArch64CallingConvention.h b/lib/Target/AArch64/AArch64CallingConvention.h
index 461c01318d4e17e661d748b786875dbd654c5c67..13cc0c583fd243e829b91d56073ce29c53ce8bd5 100644
--- a/lib/Target/AArch64/AArch64CallingConvention.h
+++ b/lib/Target/AArch64/AArch64CallingConvention.h
@@ -1,139 +1,45 @@
-//=== AArch64CallingConv.h - Custom Calling Convention Routines -*- C++ -*-===//
+//=== AArch64CallingConvention.h - AArch64 CC entry points ------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
-// This file contains the custom routines for the AArch64 Calling Convention
-// that aren't done by tablegen.
+// This file declares the entry points for AArch64 calling convention analysis.
 //
 //===----------------------------------------------------------------------===//
 
 #ifndef LLVM_LIB_TARGET_AARCH64_AARCH64CALLINGCONVENTION_H
 #define LLVM_LIB_TARGET_AARCH64_AARCH64CALLINGCONVENTION_H
 
-#include "AArch64.h"
-#include "AArch64InstrInfo.h"
-#include "AArch64Subtarget.h"
 #include "llvm/CodeGen/CallingConvLower.h"
-#include "llvm/CodeGen/TargetInstrInfo.h"
-#include "llvm/IR/CallingConv.h"
-
-namespace {
-using namespace llvm;
-
-static const MCPhysReg XRegList[] = {AArch64::X0, AArch64::X1, AArch64::X2,
-                                     AArch64::X3, AArch64::X4, AArch64::X5,
-                                     AArch64::X6, AArch64::X7};
-static const MCPhysReg HRegList[] = {AArch64::H0, AArch64::H1, AArch64::H2,
-                                     AArch64::H3, AArch64::H4, AArch64::H5,
-                                     AArch64::H6, AArch64::H7};
-static const MCPhysReg SRegList[] = {AArch64::S0, AArch64::S1, AArch64::S2,
-                                     AArch64::S3, AArch64::S4, AArch64::S5,
-                                     AArch64::S6, AArch64::S7};
-static const MCPhysReg DRegList[] = {AArch64::D0, AArch64::D1, AArch64::D2,
-                                     AArch64::D3, AArch64::D4, AArch64::D5,
-                                     AArch64::D6, AArch64::D7};
-static const MCPhysReg QRegList[] = {AArch64::Q0, AArch64::Q1, AArch64::Q2,
-                                     AArch64::Q3, AArch64::Q4, AArch64::Q5,
-                                     AArch64::Q6, AArch64::Q7};
-
-static bool finishStackBlock(SmallVectorImpl<CCValAssign> &PendingMembers,
-                             MVT LocVT, ISD::ArgFlagsTy &ArgFlags,
-                             CCState &State, unsigned SlotAlign) {
-  unsigned Size = LocVT.getSizeInBits() / 8;
-  unsigned StackAlign =
-      State.getMachineFunction().getDataLayout().getStackAlignment();
-  unsigned Align = std::min(ArgFlags.getOrigAlign(), StackAlign);
-
-  for (auto &It : PendingMembers) {
-    It.convertToMem(State.AllocateStack(Size, std::max(Align, SlotAlign)));
-    State.addLoc(It);
-    SlotAlign = 1;
-  }
-
-  // All pending members have now been allocated
-  PendingMembers.clear();
-  return true;
-}
-
-/// The Darwin variadic PCS places anonymous arguments in 8-byte stack slots. An
-/// [N x Ty] type must still be contiguous in memory though.
-static bool CC_AArch64_Custom_Stack_Block(
-      unsigned &ValNo, MVT &ValVT, MVT &LocVT, CCValAssign::LocInfo &LocInfo,
-      ISD::ArgFlagsTy &ArgFlags, CCState &State) {
-  SmallVectorImpl<CCValAssign> &PendingMembers = State.getPendingLocs();
-
-  // Add the argument to the list to be allocated once we know the size of the
-  // block.
-  PendingMembers.push_back(
-      CCValAssign::getPending(ValNo, ValVT, LocVT, LocInfo));
-
-  if (!ArgFlags.isInConsecutiveRegsLast())
-    return true;
-
-  return finishStackBlock(PendingMembers, LocVT, ArgFlags, State, 8);
-}
-
-/// Given an [N x Ty] block, it should be passed in a consecutive sequence of
-/// registers. If no such sequence is available, mark the rest of the registers
-/// of that type as used and place the argument on the stack.
-static bool CC_AArch64_Custom_Block(unsigned &ValNo, MVT &ValVT, MVT &LocVT,
-                                    CCValAssign::LocInfo &LocInfo,
-                                    ISD::ArgFlagsTy &ArgFlags, CCState &State) {
-  // Try to allocate a contiguous block of registers, each of the correct
-  // size to hold one member.
-  ArrayRef<MCPhysReg> RegList;
-  if (LocVT.SimpleTy == MVT::i64)
-    RegList = XRegList;
-  else if (LocVT.SimpleTy == MVT::f16)
-    RegList = HRegList;
-  else if (LocVT.SimpleTy == MVT::f32 || LocVT.is32BitVector())
-    RegList = SRegList;
-  else if (LocVT.SimpleTy == MVT::f64 || LocVT.is64BitVector())
-    RegList = DRegList;
-  else if (LocVT.SimpleTy == MVT::f128 || LocVT.is128BitVector())
-    RegList = QRegList;
-  else {
-    // Not an array we want to split up after all.
-    return false;
-  }
-
-  SmallVectorImpl<CCValAssign> &PendingMembers = State.getPendingLocs();
-
-  // Add the argument to the list to be allocated once we know the size of the
-  // block.
-  PendingMembers.push_back(
-      CCValAssign::getPending(ValNo, ValVT, LocVT, LocInfo));
-
-  if (!ArgFlags.isInConsecutiveRegsLast())
-    return true;
-
-  unsigned RegResult = State.AllocateRegBlock(RegList, PendingMembers.size());
-  if (RegResult) {
-    for (auto &It : PendingMembers) {
-      It.convertToReg(RegResult);
-      State.addLoc(It);
-      ++RegResult;
-    }
-    PendingMembers.clear();
-    return true;
-  }
-
-  // Mark all regs in the class as unavailable
-  for (auto Reg : RegList)
-    State.AllocateReg(Reg);
-
-  const AArch64Subtarget &Subtarget = static_cast<const AArch64Subtarget &>(
-      State.getMachineFunction().getSubtarget());
-  unsigned SlotAlign = Subtarget.isTargetDarwin() ? 1 : 8;
-
-  return finishStackBlock(PendingMembers, LocVT, ArgFlags, State, SlotAlign);
-}
 
-}
+namespace llvm {
+bool CC_AArch64_AAPCS(unsigned ValNo, MVT ValVT, MVT LocVT,
+                      CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags,
+                      CCState &State);
+bool CC_AArch64_DarwinPCS_VarArg(unsigned ValNo, MVT ValVT, MVT LocVT,
+                                 CCValAssign::LocInfo LocInfo,
+                                 ISD::ArgFlagsTy ArgFlags, CCState &State);
+bool CC_AArch64_DarwinPCS(unsigned ValNo, MVT ValVT, MVT LocVT,
+                          CCValAssign::LocInfo LocInfo,
+                          ISD::ArgFlagsTy ArgFlags, CCState &State);
+bool CC_AArch64_Win64_VarArg(unsigned ValNo, MVT ValVT, MVT LocVT,
+                             CCValAssign::LocInfo LocInfo,
+                             ISD::ArgFlagsTy ArgFlags, CCState &State);
+bool CC_AArch64_WebKit_JS(unsigned ValNo, MVT ValVT, MVT LocVT,
+                          CCValAssign::LocInfo LocInfo,
+                          ISD::ArgFlagsTy ArgFlags, CCState &State);
+bool CC_AArch64_GHC(unsigned ValNo, MVT ValVT, MVT LocVT,
+                    CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags,
+                    CCState &State);
+bool RetCC_AArch64_AAPCS(unsigned ValNo, MVT ValVT, MVT LocVT,
+                         CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags,
+                         CCState &State);
+bool RetCC_AArch64_WebKit_JS(unsigned ValNo, MVT ValVT, MVT LocVT,
+                             CCValAssign::LocInfo LocInfo,
+                             ISD::ArgFlagsTy ArgFlags, CCState &State);
+} // namespace llvm
 
 #endif
diff --git a/lib/Target/AArch64/AArch64CallingConvention.td b/lib/Target/AArch64/AArch64CallingConvention.td
index 5db941e9dac72b651ded275d04ec4b91bf31b01b..e164dcbf63bb6541f9be081c05575e42bd240b9a 100644
--- a/lib/Target/AArch64/AArch64CallingConvention.td
+++ b/lib/Target/AArch64/AArch64CallingConvention.td
@@ -1,9 +1,8 @@
 //=- AArch64CallingConv.td - Calling Conventions for AArch64 -*- tablegen -*-=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -22,6 +21,7 @@ class CCIfBigEndian<CCAction A> :
 // ARM AAPCS64 Calling Convention
 //===----------------------------------------------------------------------===//
 
+let Entry = 1 in
 def CC_AArch64_AAPCS : CallingConv<[
   CCIfType<[iPTR], CCBitConvertToType<i64>>,
   CCIfType<[v2f32], CCBitConvertToType<v2i32>>,
@@ -89,6 +89,7 @@ def CC_AArch64_AAPCS : CallingConv<[
            CCAssignToStack<16, 16>>
 ]>;
 
+let Entry = 1 in
 def RetCC_AArch64_AAPCS : CallingConv<[
   CCIfType<[iPTR], CCBitConvertToType<i64>>,
   CCIfType<[v2f32], CCBitConvertToType<v2i32>>,
@@ -122,6 +123,7 @@ def RetCC_AArch64_AAPCS : CallingConv<[
 ]>;
 
 // Vararg functions on windows pass floats in integer registers
+let Entry = 1 in
 def CC_AArch64_Win64_VarArg : CallingConv<[
   CCIfType<[f16, f32], CCPromoteToType<f64>>,
   CCIfType<[f64], CCBitConvertToType<i64>>,
@@ -133,6 +135,7 @@ def CC_AArch64_Win64_VarArg : CallingConv<[
 // from the standard one at this level:
 //     + i128s (i.e. split i64s) don't need even registers.
 //     + Stack slots are sized as needed rather than being at least 64-bit.
+let Entry = 1 in
 def CC_AArch64_DarwinPCS : CallingConv<[
   CCIfType<[iPTR], CCBitConvertToType<i64>>,
   CCIfType<[v2f32], CCBitConvertToType<v2i32>>,
@@ -189,6 +192,7 @@ def CC_AArch64_DarwinPCS : CallingConv<[
            CCAssignToStack<16, 16>>
 ]>;
 
+let Entry = 1 in
 def CC_AArch64_DarwinPCS_VarArg : CallingConv<[
   CCIfType<[iPTR], CCBitConvertToType<i64>>,
   CCIfType<[v2f32], CCBitConvertToType<v2i32>>,
@@ -213,6 +217,7 @@ def CC_AArch64_DarwinPCS_VarArg : CallingConv<[
 // in register and the remaining arguments on stack. We allow 32bit stack slots,
 // so that WebKit can write partial values in the stack and define the other
 // 32bit quantity as undef.
+let Entry = 1 in
 def CC_AArch64_WebKit_JS : CallingConv<[
   // Handle i1, i8, i16, i32, and i64 passing in register X0 (W0).
   CCIfType<[i1, i8, i16], CCPromoteToType<i32>>,
@@ -224,6 +229,7 @@ def CC_AArch64_WebKit_JS : CallingConv<[
   CCIfType<[i64, f64], CCAssignToStack<8, 8>>
 ]>;
 
+let Entry = 1 in
 def RetCC_AArch64_WebKit_JS : CallingConv<[
   CCIfType<[i32], CCAssignToRegWithShadow<[W0, W1, W2, W3, W4, W5, W6, W7],
                                           [X0, X1, X2, X3, X4, X5, X6, X7]>>,
@@ -257,6 +263,7 @@ def RetCC_AArch64_WebKit_JS : CallingConv<[
 // The AArch64 register mapping is under the heading "The ARMv8/AArch64 ABI
 // register mapping".
 
+let Entry = 1 in
 def CC_AArch64_GHC : CallingConv<[
   CCIfType<[iPTR], CCBitConvertToType<i64>>,
 
diff --git a/lib/Target/AArch64/AArch64CleanupLocalDynamicTLSPass.cpp b/lib/Target/AArch64/AArch64CleanupLocalDynamicTLSPass.cpp
index b88fba4452a18877363020f4a65915e6bb5d7a9d..688bd1b28e855a8e4022fca495e01da2228f9f75 100644
--- a/lib/Target/AArch64/AArch64CleanupLocalDynamicTLSPass.cpp
+++ b/lib/Target/AArch64/AArch64CleanupLocalDynamicTLSPass.cpp
@@ -1,9 +1,8 @@
 //===-- AArch64CleanupLocalDynamicTLSPass.cpp ---------------------*- C++ -*-=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AArch64/AArch64CollectLOH.cpp b/lib/Target/AArch64/AArch64CollectLOH.cpp
index 720323f81d29ca4ce047a59dc5792c659c1f63a2..9f324b4332093977d5632f1c24e4b2d43d5e13be 100644
--- a/lib/Target/AArch64/AArch64CollectLOH.cpp
+++ b/lib/Target/AArch64/AArch64CollectLOH.cpp
@@ -1,9 +1,8 @@
 //===---------- AArch64CollectLOH.cpp - AArch64 collect LOH pass --*- C++ -*-=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AArch64/AArch64CompressJumpTables.cpp b/lib/Target/AArch64/AArch64CompressJumpTables.cpp
index 0924a27e2586567ade5ff0dc8cb8e89265f17772..bcb6e4a49ccc24c8dad383f5fd454aaf4f8199f5 100644
--- a/lib/Target/AArch64/AArch64CompressJumpTables.cpp
+++ b/lib/Target/AArch64/AArch64CompressJumpTables.cpp
@@ -1,9 +1,8 @@
 //==-- AArch64CompressJumpTables.cpp - Compress jump tables for AArch64 --====//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 // This pass looks at the basic blocks each jump-table refers to and works out
 // whether they can be emitted in a compressed form (with 8 or 16-bit
diff --git a/lib/Target/AArch64/AArch64CondBrTuning.cpp b/lib/Target/AArch64/AArch64CondBrTuning.cpp
index 5ae787409ae8fab47bb42d7f9d57e88058375f77..453132e0966994048714149bad9ffe70b4d762f0 100644
--- a/lib/Target/AArch64/AArch64CondBrTuning.cpp
+++ b/lib/Target/AArch64/AArch64CondBrTuning.cpp
@@ -1,9 +1,8 @@
 //===-- AArch64CondBrTuning.cpp --- Conditional branch tuning for AArch64 -===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/lib/Target/AArch64/AArch64ConditionOptimizer.cpp b/lib/Target/AArch64/AArch64ConditionOptimizer.cpp
index 5064762b9f779100b5948f84bcee7f3bfa4c9539..a6efb115ed4448871c56c569e7cdd6c265bbf991 100644
--- a/lib/Target/AArch64/AArch64ConditionOptimizer.cpp
+++ b/lib/Target/AArch64/AArch64ConditionOptimizer.cpp
@@ -1,9 +1,8 @@
 //=- AArch64ConditionOptimizer.cpp - Remove useless comparisons for AArch64 -=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AArch64/AArch64ConditionalCompares.cpp b/lib/Target/AArch64/AArch64ConditionalCompares.cpp
index 8176b6fb269d2f762aab544c8494dc24cc20e6a5..b136222b21b67f30325def3bc880cd0eb9d58510 100644
--- a/lib/Target/AArch64/AArch64ConditionalCompares.cpp
+++ b/lib/Target/AArch64/AArch64ConditionalCompares.cpp
@@ -1,9 +1,8 @@
 //===-- AArch64ConditionalCompares.cpp --- CCMP formation for AArch64 -----===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AArch64/AArch64DeadRegisterDefinitionsPass.cpp b/lib/Target/AArch64/AArch64DeadRegisterDefinitionsPass.cpp
index 2ba10d25e9397d08f30fb6f8c3461258d7feffa1..c0888badb234c8c26900640796058e09822c1989 100644
--- a/lib/Target/AArch64/AArch64DeadRegisterDefinitionsPass.cpp
+++ b/lib/Target/AArch64/AArch64DeadRegisterDefinitionsPass.cpp
@@ -1,9 +1,8 @@
 //==-- AArch64DeadRegisterDefinitions.cpp - Replace dead defs w/ zero reg --==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file When allowed by the instruction, replace a dead definition of a GPR
@@ -55,8 +54,6 @@ public:
     AU.setPreservesCFG();
     MachineFunctionPass::getAnalysisUsage(AU);
   }
-
-  bool shouldSkip(const MachineInstr &MI, const MachineFunction &MF) const;
 };
 char AArch64DeadRegisterDefinitions::ID = 0;
 } // end anonymous namespace
@@ -71,63 +68,6 @@ static bool usesFrameIndex(const MachineInstr &MI) {
   return false;
 }
 
-bool
-AArch64DeadRegisterDefinitions::shouldSkip(const MachineInstr &MI,
-                                           const MachineFunction &MF) const {
-  if (!MF.getSubtarget<AArch64Subtarget>().hasLSE())
-    return false;
-
-#define CASE_AARCH64_ATOMIC_(PREFIX) \
-  case AArch64::PREFIX##X: \
-  case AArch64::PREFIX##W: \
-  case AArch64::PREFIX##H: \
-  case AArch64::PREFIX##B
-
-  for (const MachineMemOperand *MMO : MI.memoperands()) {
-    if (MMO->isAtomic()) {
-      unsigned Opcode = MI.getOpcode();
-      switch (Opcode) {
-      default:
-        return false;
-        break;
-
-      CASE_AARCH64_ATOMIC_(LDADDA):
-      CASE_AARCH64_ATOMIC_(LDADDAL):
-
-      CASE_AARCH64_ATOMIC_(LDCLRA):
-      CASE_AARCH64_ATOMIC_(LDCLRAL):
-
-      CASE_AARCH64_ATOMIC_(LDEORA):
-      CASE_AARCH64_ATOMIC_(LDEORAL):
-
-      CASE_AARCH64_ATOMIC_(LDSETA):
-      CASE_AARCH64_ATOMIC_(LDSETAL):
-
-      CASE_AARCH64_ATOMIC_(LDSMAXA):
-      CASE_AARCH64_ATOMIC_(LDSMAXAL):
-
-      CASE_AARCH64_ATOMIC_(LDSMINA):
-      CASE_AARCH64_ATOMIC_(LDSMINAL):
-
-      CASE_AARCH64_ATOMIC_(LDUMAXA):
-      CASE_AARCH64_ATOMIC_(LDUMAXAL):
-
-      CASE_AARCH64_ATOMIC_(LDUMINA):
-      CASE_AARCH64_ATOMIC_(LDUMINAL):
-
-      CASE_AARCH64_ATOMIC_(SWPA):
-      CASE_AARCH64_ATOMIC_(SWPAL):
-        return true;
-        break;
-                                                                    }
-    }
-  }
-
-#undef CASE_AARCH64_ATOMIC_
-
-  return false;
-}
-
 void AArch64DeadRegisterDefinitions::processMachineBasicBlock(
     MachineBasicBlock &MBB) {
   const MachineFunction &MF = *MBB.getParent();
@@ -148,9 +88,8 @@ void AArch64DeadRegisterDefinitions::processMachineBasicBlock(
       continue;
     }
 
-    if (shouldSkip(MI, MF)) {
-      LLVM_DEBUG(dbgs() << "    Ignoring, Atomic instruction with acquire "
-                           "semantics using WZR/XZR\n");
+    if (atomicBarrierDroppedOnZero(MI.getOpcode())) {
+      LLVM_DEBUG(dbgs() << "    Ignoring, semantics change with xzr/wzr.\n");
       continue;
     }
 
diff --git a/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp b/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp
index f7190d58fbf92992aa8990c9d2613a5e77a04e3b..2d0b52b67e570cd2e8b168f03818942b4e3fc1ec 100644
--- a/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp
+++ b/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp
@@ -1,9 +1,8 @@
 //===- AArch64ExpandPseudoInsts.cpp - Expand pseudo instructions ----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AArch64/AArch64FalkorHWPFFix.cpp b/lib/Target/AArch64/AArch64FalkorHWPFFix.cpp
index bc9a5ca97fea3a3ca7e994b7400b972152e95851..c7edb848945ec98480388ae85982910b7a50a0fc 100644
--- a/lib/Target/AArch64/AArch64FalkorHWPFFix.cpp
+++ b/lib/Target/AArch64/AArch64FalkorHWPFFix.cpp
@@ -1,9 +1,8 @@
 //===- AArch64FalkorHWPFFix.cpp - Avoid HW prefetcher pitfalls on Falkor --===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file For Falkor, we want to avoid HW prefetcher instruction tag collisions
diff --git a/lib/Target/AArch64/AArch64FastISel.cpp b/lib/Target/AArch64/AArch64FastISel.cpp
index 47550cabb9f0dac8a899c02d2af7f42715e0a2b8..8b033ee63766077334de2153ee2abf54a6d1902b 100644
--- a/lib/Target/AArch64/AArch64FastISel.cpp
+++ b/lib/Target/AArch64/AArch64FastISel.cpp
@@ -1,9 +1,8 @@
 //===- AArch6464FastISel.cpp - AArch64 FastISel implementation ------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -305,8 +304,6 @@ public:
 
 } // end anonymous namespace
 
-#include "AArch64GenCallingConv.inc"
-
 /// Check if the sign-/zero-extend will be a noop.
 static bool isIntExtFree(const Instruction *I) {
   assert((isa<ZExtInst>(I) || isa<SExtInst>(I)) &&
@@ -408,10 +405,9 @@ unsigned AArch64FastISel::materializeFP(const ConstantFP *CFP, MVT VT) {
   bool Is64Bit = (VT == MVT::f64);
   // This checks to see if we can use FMOV instructions to materialize
   // a constant, otherwise we have to materialize via the constant pool.
-  if (TLI.isFPImmLegal(Val, VT)) {
-    int Imm =
-        Is64Bit ? AArch64_AM::getFP64Imm(Val) : AArch64_AM::getFP32Imm(Val);
-    assert((Imm != -1) && "Cannot encode floating-point constant.");
+  int Imm =
+      Is64Bit ? AArch64_AM::getFP64Imm(Val) : AArch64_AM::getFP32Imm(Val);
+  if (Imm != -1) {
     unsigned Opc = Is64Bit ? AArch64::FMOVDi : AArch64::FMOVSi;
     return fastEmitInst_i(Opc, TLI.getRegClassFor(VT), Imm);
   }
@@ -5172,10 +5168,6 @@ bool AArch64FastISel::fastSelectInstruction(const Instruction *I) {
     return selectAtomicCmpXchg(cast<AtomicCmpXchgInst>(I));
   }
 
-  // Silence warnings.
-  (void)&CC_AArch64_DarwinPCS_VarArg;
-  (void)&CC_AArch64_Win64_VarArg;
-
   // fall-back to target-independent instruction selection.
   return selectOperator(I, I->getOpcode());
 }
diff --git a/lib/Target/AArch64/AArch64FrameLowering.cpp b/lib/Target/AArch64/AArch64FrameLowering.cpp
index 800dd449b49f2d5fb84077e6c93549eb19290584..446e1928d26ff6426edca339982374da1ca103ed 100644
--- a/lib/Target/AArch64/AArch64FrameLowering.cpp
+++ b/lib/Target/AArch64/AArch64FrameLowering.cpp
@@ -1,9 +1,8 @@
 //===- AArch64FrameLowering.cpp - AArch64 Frame Lowering -------*- C++ -*-====//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -816,10 +815,15 @@ void AArch64FrameLowering::emitPrologue(MachineFunction &MF,
   DebugLoc DL;
 
   if (ShouldSignReturnAddress(MF)) {
-    BuildMI(
-        MBB, MBBI, DL,
-        TII->get(ShouldSignWithAKey(MF) ? AArch64::PACIASP : AArch64::PACIBSP))
-        .setMIFlag(MachineInstr::FrameSetup);
+    if (ShouldSignWithAKey(MF))
+      BuildMI(MBB, MBBI, DL, TII->get(AArch64::PACIASP))
+          .setMIFlag(MachineInstr::FrameSetup);
+    else {
+      BuildMI(MBB, MBBI, DL, TII->get(AArch64::EMITBKEY))
+          .setMIFlag(MachineInstr::FrameSetup);
+      BuildMI(MBB, MBBI, DL, TII->get(AArch64::PACIBSP))
+          .setMIFlag(MachineInstr::FrameSetup);
+    }
 
     unsigned CFIIndex =
         MF.addFrameInst(MCCFIInstruction::createNegateRAState(nullptr));
@@ -914,6 +918,19 @@ void AArch64FrameLowering::emitPrologue(MachineFunction &MF,
     if (NeedsWinCFI)
       BuildMI(MBB, MBBI, DL, TII->get(AArch64::SEH_PrologEnd))
           .setMIFlag(MachineInstr::FrameSetup);
+
+    // SEH funclets are passed the frame pointer in X1.  If the parent
+    // function uses the base register, then the base register is used
+    // directly, and is not retrieved from X1.
+    if (F.hasPersonalityFn()) {
+      EHPersonality Per = classifyEHPersonality(F.getPersonalityFn());
+      if (isAsynchronousEHPersonality(Per)) {
+        BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::COPY), AArch64::FP)
+            .addReg(AArch64::X1).setMIFlag(MachineInstr::FrameSetup);
+        MBB.addLiveIn(AArch64::X1);
+      }
+    }
+
     return;
   }
 
@@ -1448,19 +1465,44 @@ int AArch64FrameLowering::getFrameIndexReference(const MachineFunction &MF,
   return resolveFrameIndexReference(MF, FI, FrameReg);
 }
 
+int AArch64FrameLowering::getNonLocalFrameIndexReference(
+  const MachineFunction &MF, int FI) const {
+  return getSEHFrameIndexOffset(MF, FI);
+}
+
+static int getFPOffset(const MachineFunction &MF, int FI) {
+  const auto &MFI = MF.getFrameInfo();
+  const auto *AFI = MF.getInfo<AArch64FunctionInfo>();
+  const auto &Subtarget = MF.getSubtarget<AArch64Subtarget>();
+  bool IsWin64 =
+      Subtarget.isCallingConvWin64(MF.getFunction().getCallingConv());
+  unsigned FixedObject = IsWin64 ? alignTo(AFI->getVarArgsGPRSize(), 16) : 0;
+  return MFI.getObjectOffset(FI) + FixedObject + 16;
+}
+
+static int getStackOffset(const MachineFunction &MF, int FI) {
+  const auto &MFI = MF.getFrameInfo();
+  return MFI.getObjectOffset(FI) + MFI.getStackSize();
+}
+
+int AArch64FrameLowering::getSEHFrameIndexOffset(const MachineFunction &MF,
+                                                 int FI) const {
+  const auto *RegInfo = static_cast<const AArch64RegisterInfo *>(
+      MF.getSubtarget().getRegisterInfo());
+  return RegInfo->getLocalAddressRegister(MF) == AArch64::FP ?
+         getFPOffset(MF, FI) : getStackOffset(MF, FI);
+}
+
 int AArch64FrameLowering::resolveFrameIndexReference(const MachineFunction &MF,
                                                      int FI, unsigned &FrameReg,
                                                      bool PreferFP) const {
-  const MachineFrameInfo &MFI = MF.getFrameInfo();
-  const AArch64RegisterInfo *RegInfo = static_cast<const AArch64RegisterInfo *>(
+  const auto &MFI = MF.getFrameInfo();
+  const auto *RegInfo = static_cast<const AArch64RegisterInfo *>(
       MF.getSubtarget().getRegisterInfo());
-  const AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
-  const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
-  bool IsWin64 =
-      Subtarget.isCallingConvWin64(MF.getFunction().getCallingConv());
-  unsigned FixedObject = IsWin64 ? alignTo(AFI->getVarArgsGPRSize(), 16) : 0;
-  int FPOffset = MFI.getObjectOffset(FI) + FixedObject + 16;
-  int Offset = MFI.getObjectOffset(FI) + MFI.getStackSize();
+  const auto *AFI = MF.getInfo<AArch64FunctionInfo>();
+  const auto &Subtarget = MF.getSubtarget<AArch64Subtarget>();
+  int FPOffset = getFPOffset(MF, FI);
+  int Offset = getStackOffset(MF, FI);
   bool isFixed = MFI.isFixedObjectIndex(FI);
   bool isCSR = !isFixed && MFI.getObjectOffset(FI) >=
                                -((int)AFI->getCalleeSavedStackSize());
@@ -1512,6 +1554,7 @@ int AArch64FrameLowering::resolveFrameIndexReference(const MachineFunction &MF,
         // Funclets access the locals contained in the parent's stack frame
         // via the frame pointer, so we have to use the FP in the parent
         // function.
+        (void) Subtarget;
         assert(
             Subtarget.isCallingConvWin64(MF.getFunction().getCallingConv()) &&
             "Funclets should only be present on Win64");
@@ -1754,8 +1797,8 @@ bool AArch64FrameLowering::spillCalleeSavedRegisters(
           static_cast<char>(unsigned(dwarf::DW_OP_breg18)),
           static_cast<char>(-8) & 0x7f, // addend (sleb128)
       };
-      unsigned CFIIndex =
-          MF.addFrameInst(MCCFIInstruction::createEscape(nullptr, CFIInst));
+      unsigned CFIIndex = MF.addFrameInst(MCCFIInstruction::createEscape(
+          nullptr, StringRef(CFIInst, sizeof(CFIInst))));
       BuildMI(MBB, MI, DL, TII.get(AArch64::CFI_INSTRUCTION))
           .addCFIIndex(CFIIndex)
           .setMIFlag(MachineInstr::FrameSetup);
diff --git a/lib/Target/AArch64/AArch64FrameLowering.h b/lib/Target/AArch64/AArch64FrameLowering.h
index 0d0385acf46e6155e27acb7793a007bfdc46994f..f4bbeb4bdaef8434107d71b57993b02c9bf16fac 100644
--- a/lib/Target/AArch64/AArch64FrameLowering.h
+++ b/lib/Target/AArch64/AArch64FrameLowering.h
@@ -1,9 +1,8 @@
 //==-- AArch64FrameLowering.h - TargetFrameLowering for AArch64 --*- C++ -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -79,6 +78,9 @@ public:
   int getFrameIndexReferencePreferSP(const MachineFunction &MF, int FI,
                                      unsigned &FrameReg,
                                      bool IgnoreSPUpdates) const override;
+  int getNonLocalFrameIndexReference(const MachineFunction &MF,
+                               int FI) const override;
+  int getSEHFrameIndexOffset(const MachineFunction &MF, int FI) const;
 
 private:
   bool shouldCombineCSRLocalStackBump(MachineFunction &MF,
diff --git a/lib/Target/AArch64/AArch64GenRegisterBankInfo.def b/lib/Target/AArch64/AArch64GenRegisterBankInfo.def
index 37720cbd32bbbc5d6404d027fd6423238bb9bdf4..fb5b8a01156e4c21cc5eadd025f61a9e02d237c9 100644
--- a/lib/Target/AArch64/AArch64GenRegisterBankInfo.def
+++ b/lib/Target/AArch64/AArch64GenRegisterBankInfo.def
@@ -1,9 +1,8 @@
 //===- AArch64GenRegisterBankInfo.def ----------------------------*- C++ -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp b/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
index 80ad23ba69a4f7eacabcf9a1c4b83b4eb29b8cae..038089162d5fca2e037772acc33c53a352176768 100644
--- a/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
+++ b/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
@@ -1,9 +1,8 @@
 //===-- AArch64ISelDAGToDAG.cpp - A dag to dag inst selector for AArch64 --===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -2087,8 +2086,7 @@ static bool isBitfieldPositioningOp(SelectionDAG *CurDAG, SDValue Op,
   (void)BitWidth;
   assert(BitWidth == 32 || BitWidth == 64);
 
-  KnownBits Known;
-  CurDAG->computeKnownBits(Op, Known);
+  KnownBits Known = CurDAG->computeKnownBits(Op);
 
   // Non-zero in the sense that they're not provably zero, which is the key
   // point if we want to use this value
@@ -2167,8 +2165,7 @@ static bool tryBitfieldInsertOpFromOrAndImm(SDNode *N, SelectionDAG *CurDAG) {
 
   // Compute the Known Zero for the AND as this allows us to catch more general
   // cases than just looking for AND with imm.
-  KnownBits Known;
-  CurDAG->computeKnownBits(And, Known);
+  KnownBits Known = CurDAG->computeKnownBits(And);
 
   // Non-zero in the sense that they're not provably zero, which is the key
   // point if we want to use this value.
@@ -2309,8 +2306,7 @@ static bool tryBitfieldInsertOpFromOr(SDNode *N, const APInt &UsefulBits,
     // This allows to catch more general case than just looking for
     // AND with imm. Indeed, simplify-demanded-bits may have removed
     // the AND instruction because it proves it was useless.
-    KnownBits Known;
-    CurDAG->computeKnownBits(OrOpd1Val, Known);
+    KnownBits Known = CurDAG->computeKnownBits(OrOpd1Val);
 
     // Check if there is enough room for the second operand to appear
     // in the first one
diff --git a/lib/Target/AArch64/AArch64ISelLowering.cpp b/lib/Target/AArch64/AArch64ISelLowering.cpp
index cc10c9688e14f38ee3d99790334c245b2be4850a..020fd9ff0237a4d7f06c05b63d863896d628b15c 100644
--- a/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -1,9 +1,8 @@
 //===-- AArch64ISelLowering.cpp - AArch64 DAG Lowering Implementation  ----===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -55,9 +54,11 @@
 #include "llvm/IR/IRBuilder.h"
 #include "llvm/IR/Instruction.h"
 #include "llvm/IR/Instructions.h"
+#include "llvm/IR/IntrinsicInst.h"
 #include "llvm/IR/Intrinsics.h"
 #include "llvm/IR/Module.h"
 #include "llvm/IR/OperandTraits.h"
+#include "llvm/IR/PatternMatch.h"
 #include "llvm/IR/Type.h"
 #include "llvm/IR/Use.h"
 #include "llvm/IR/Value.h"
@@ -87,6 +88,7 @@
 #include <vector>
 
 using namespace llvm;
+using namespace llvm::PatternMatch;
 
 #define DEBUG_TYPE "aarch64-lower"
 
@@ -333,36 +335,38 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
     setOperationAction(ISD::FCOPYSIGN, MVT::f16, Promote);
 
   setOperationAction(ISD::FREM,    MVT::f16,   Promote);
-  setOperationAction(ISD::FREM,    MVT::v4f16, Promote);
-  setOperationAction(ISD::FREM,    MVT::v8f16, Promote);
+  setOperationAction(ISD::FREM,    MVT::v4f16, Expand);
+  setOperationAction(ISD::FREM,    MVT::v8f16, Expand);
   setOperationAction(ISD::FPOW,    MVT::f16,   Promote);
-  setOperationAction(ISD::FPOW,    MVT::v4f16, Promote);
-  setOperationAction(ISD::FPOW,    MVT::v8f16, Promote);
+  setOperationAction(ISD::FPOW,    MVT::v4f16, Expand);
+  setOperationAction(ISD::FPOW,    MVT::v8f16, Expand);
   setOperationAction(ISD::FPOWI,   MVT::f16,   Promote);
+  setOperationAction(ISD::FPOWI,   MVT::v4f16, Expand);
+  setOperationAction(ISD::FPOWI,   MVT::v8f16, Expand);
   setOperationAction(ISD::FCOS,    MVT::f16,   Promote);
-  setOperationAction(ISD::FCOS,    MVT::v4f16, Promote);
-  setOperationAction(ISD::FCOS,    MVT::v8f16, Promote);
+  setOperationAction(ISD::FCOS,    MVT::v4f16, Expand);
+  setOperationAction(ISD::FCOS,    MVT::v8f16, Expand);
   setOperationAction(ISD::FSIN,    MVT::f16,   Promote);
-  setOperationAction(ISD::FSIN,    MVT::v4f16, Promote);
-  setOperationAction(ISD::FSIN,    MVT::v8f16, Promote);
+  setOperationAction(ISD::FSIN,    MVT::v4f16, Expand);
+  setOperationAction(ISD::FSIN,    MVT::v8f16, Expand);
   setOperationAction(ISD::FSINCOS, MVT::f16,   Promote);
-  setOperationAction(ISD::FSINCOS, MVT::v4f16, Promote);
-  setOperationAction(ISD::FSINCOS, MVT::v8f16, Promote);
+  setOperationAction(ISD::FSINCOS, MVT::v4f16, Expand);
+  setOperationAction(ISD::FSINCOS, MVT::v8f16, Expand);
   setOperationAction(ISD::FEXP,    MVT::f16,   Promote);
-  setOperationAction(ISD::FEXP,    MVT::v4f16, Promote);
-  setOperationAction(ISD::FEXP,    MVT::v8f16, Promote);
+  setOperationAction(ISD::FEXP,    MVT::v4f16, Expand);
+  setOperationAction(ISD::FEXP,    MVT::v8f16, Expand);
   setOperationAction(ISD::FEXP2,   MVT::f16,   Promote);
-  setOperationAction(ISD::FEXP2,   MVT::v4f16, Promote);
-  setOperationAction(ISD::FEXP2,   MVT::v8f16, Promote);
+  setOperationAction(ISD::FEXP2,   MVT::v4f16, Expand);
+  setOperationAction(ISD::FEXP2,   MVT::v8f16, Expand);
   setOperationAction(ISD::FLOG,    MVT::f16,   Promote);
-  setOperationAction(ISD::FLOG,    MVT::v4f16, Promote);
-  setOperationAction(ISD::FLOG,    MVT::v8f16, Promote);
+  setOperationAction(ISD::FLOG,    MVT::v4f16, Expand);
+  setOperationAction(ISD::FLOG,    MVT::v8f16, Expand);
   setOperationAction(ISD::FLOG2,   MVT::f16,   Promote);
-  setOperationAction(ISD::FLOG2,   MVT::v4f16, Promote);
-  setOperationAction(ISD::FLOG2,   MVT::v8f16, Promote);
+  setOperationAction(ISD::FLOG2,   MVT::v4f16, Expand);
+  setOperationAction(ISD::FLOG2,   MVT::v8f16, Expand);
   setOperationAction(ISD::FLOG10,  MVT::f16,   Promote);
-  setOperationAction(ISD::FLOG10,  MVT::v4f16, Promote);
-  setOperationAction(ISD::FLOG10,  MVT::v8f16, Promote);
+  setOperationAction(ISD::FLOG10,  MVT::v4f16, Expand);
+  setOperationAction(ISD::FLOG10,  MVT::v8f16, Expand);
 
   if (!Subtarget->hasFullFP16()) {
     setOperationAction(ISD::SELECT,      MVT::f16,  Promote);
@@ -677,15 +681,6 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
     setOperationAction(ISD::CTLZ,       MVT::v1i64, Expand);
     setOperationAction(ISD::CTLZ,       MVT::v2i64, Expand);
 
-    setOperationAction(ISD::CTTZ,       MVT::v2i8,  Expand);
-    setOperationAction(ISD::CTTZ,       MVT::v4i16, Expand);
-    setOperationAction(ISD::CTTZ,       MVT::v2i32, Expand);
-    setOperationAction(ISD::CTTZ,       MVT::v1i64, Expand);
-    setOperationAction(ISD::CTTZ,       MVT::v16i8, Expand);
-    setOperationAction(ISD::CTTZ,       MVT::v8i16, Expand);
-    setOperationAction(ISD::CTTZ,       MVT::v4i32, Expand);
-    setOperationAction(ISD::CTTZ,       MVT::v2i64, Expand);
-
     // AArch64 doesn't have MUL.2d:
     setOperationAction(ISD::MUL, MVT::v2i64, Expand);
     // Custom handling for some quad-vector types to detect MULL.
@@ -724,6 +719,7 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
       setOperationAction(ISD::UMUL_LOHI, VT, Expand);
 
       setOperationAction(ISD::BSWAP, VT, Expand);
+      setOperationAction(ISD::CTTZ, VT, Expand);
 
       for (MVT InnerVT : MVT::vector_valuetypes()) {
         setTruncStoreAction(VT, InnerVT, Expand);
@@ -993,8 +989,8 @@ void AArch64TargetLowering::computeKnownBitsForTargetNode(
     break;
   case AArch64ISD::CSEL: {
     KnownBits Known2;
-    DAG.computeKnownBits(Op->getOperand(0), Known, Depth + 1);
-    DAG.computeKnownBits(Op->getOperand(1), Known2, Depth + 1);
+    Known = DAG.computeKnownBits(Op->getOperand(0), Depth + 1);
+    Known2 = DAG.computeKnownBits(Op->getOperand(1), Depth + 1);
     Known.Zero &= Known2.Zero;
     Known.One &= Known2.One;
     break;
@@ -2716,9 +2712,19 @@ SDValue AArch64TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
     EVT PtrVT = getPointerTy(DAG.getDataLayout());
     return DAG.getNode(AArch64ISD::THREAD_POINTER, dl, PtrVT);
   }
-  case Intrinsic::aarch64_neon_abs:
-    return DAG.getNode(ISD::ABS, dl, Op.getValueType(),
-                       Op.getOperand(1));
+  case Intrinsic::aarch64_neon_abs: {
+    EVT Ty = Op.getValueType();
+    if (Ty == MVT::i64) {
+      SDValue Result = DAG.getNode(ISD::BITCAST, dl, MVT::v1i64,
+                                   Op.getOperand(1));
+      Result = DAG.getNode(ISD::ABS, dl, MVT::v1i64, Result);
+      return DAG.getNode(ISD::BITCAST, dl, MVT::i64, Result);
+    } else if (Ty.isVector() && Ty.isInteger() && isTypeLegal(Ty)) {
+      return DAG.getNode(ISD::ABS, dl, Ty, Op.getOperand(1));
+    } else {
+      report_fatal_error("Unexpected type for AArch64 NEON intrinic");
+    }
+  }
   case Intrinsic::aarch64_neon_smax:
     return DAG.getNode(ISD::SMAX, dl, Op.getValueType(),
                        Op.getOperand(1), Op.getOperand(2));
@@ -2731,6 +2737,28 @@ SDValue AArch64TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
   case Intrinsic::aarch64_neon_umin:
     return DAG.getNode(ISD::UMIN, dl, Op.getValueType(),
                        Op.getOperand(1), Op.getOperand(2));
+
+  case Intrinsic::localaddress: {
+    const auto &MF = DAG.getMachineFunction();
+    const auto *RegInfo = Subtarget->getRegisterInfo();
+    unsigned Reg = RegInfo->getLocalAddressRegister(MF);
+    return DAG.getCopyFromReg(DAG.getEntryNode(), dl, Reg,
+                              Op.getSimpleValueType());
+  }
+
+  case Intrinsic::eh_recoverfp: {
+    // FIXME: This needs to be implemented to correctly handle highly aligned
+    // stack objects. For now we simply return the incoming FP. Refer D53541
+    // for more details.
+    SDValue FnOp = Op.getOperand(1);
+    SDValue IncomingFPOp = Op.getOperand(2);
+    GlobalAddressSDNode *GSD = dyn_cast<GlobalAddressSDNode>(FnOp);
+    auto *Fn = dyn_cast_or_null<Function>(GSD ? GSD->getGlobal() : nullptr);
+    if (!Fn)
+      report_fatal_error(
+          "llvm.eh.recoverfp must take a function as the first argument");
+    return IncomingFPOp;
+  }
   }
 }
 
@@ -2933,8 +2961,6 @@ SDValue AArch64TargetLowering::LowerOperation(SDValue Op,
 //                      Calling Convention Implementation
 //===----------------------------------------------------------------------===//
 
-#include "AArch64GenCallingConv.inc"
-
 /// Selects the correct CCAssignFn for a given CallingConvention value.
 CCAssignFn *AArch64TargetLowering::CCAssignFnForCall(CallingConv::ID CC,
                                                      bool IsVarArg) const {
@@ -5185,50 +5211,20 @@ SDValue AArch64TargetLowering::LowerSPONENTRY(SDValue Op,
   return DAG.getFrameIndex(FI, VT);
 }
 
+#define GET_REGISTER_MATCHER
+#include "AArch64GenAsmMatcher.inc"
+
 // FIXME? Maybe this could be a TableGen attribute on some registers and
 // this table could be generated automatically from RegInfo.
 unsigned AArch64TargetLowering::getRegisterByName(const char* RegName, EVT VT,
                                                   SelectionDAG &DAG) const {
-  unsigned Reg = StringSwitch<unsigned>(RegName)
-                       .Case("sp", AArch64::SP)
-                       .Case("x1", AArch64::X1)
-                       .Case("w1", AArch64::W1)
-                       .Case("x2", AArch64::X2)
-                       .Case("w2", AArch64::W2)
-                       .Case("x3", AArch64::X3)
-                       .Case("w3", AArch64::W3)
-                       .Case("x4", AArch64::X4)
-                       .Case("w4", AArch64::W4)
-                       .Case("x5", AArch64::X5)
-                       .Case("w5", AArch64::W5)
-                       .Case("x6", AArch64::X6)
-                       .Case("w6", AArch64::W6)
-                       .Case("x7", AArch64::X7)
-                       .Case("w7", AArch64::W7)
-                       .Case("x18", AArch64::X18)
-                       .Case("w18", AArch64::W18)
-                       .Case("x20", AArch64::X20)
-                       .Case("w20", AArch64::W20)
-                       .Default(0);
-  if (((Reg == AArch64::X1 || Reg == AArch64::W1) &&
-      !Subtarget->isXRegisterReserved(1)) ||
-      ((Reg == AArch64::X2 || Reg == AArch64::W2) &&
-      !Subtarget->isXRegisterReserved(2)) ||
-      ((Reg == AArch64::X3 || Reg == AArch64::W3) &&
-      !Subtarget->isXRegisterReserved(3)) ||
-      ((Reg == AArch64::X4 || Reg == AArch64::W4) &&
-      !Subtarget->isXRegisterReserved(4)) ||
-      ((Reg == AArch64::X5 || Reg == AArch64::W5) &&
-      !Subtarget->isXRegisterReserved(5)) ||
-      ((Reg == AArch64::X6 || Reg == AArch64::W6) &&
-      !Subtarget->isXRegisterReserved(6)) ||
-      ((Reg == AArch64::X7 || Reg == AArch64::W7) &&
-      !Subtarget->isXRegisterReserved(7)) ||
-      ((Reg == AArch64::X18 || Reg == AArch64::W18) &&
-      !Subtarget->isXRegisterReserved(18)) ||
-      ((Reg == AArch64::X20 || Reg == AArch64::W20) &&
-      !Subtarget->isXRegisterReserved(20)))
-    Reg = 0;
+  unsigned Reg = MatchRegisterName(RegName);
+  if (AArch64::X1 <= Reg && Reg <= AArch64::X28) {
+    const MCRegisterInfo *MRI = Subtarget->getRegisterInfo();
+    unsigned DwarfRegNum = MRI->getDwarfRegNum(Reg, false);
+    if (!Subtarget->isXRegisterReserved(DwarfRegNum))
+      Reg = 0;
+  }
   if (Reg)
     return Reg;
   report_fatal_error(Twine("Invalid register name \""
@@ -5387,34 +5383,30 @@ bool AArch64TargetLowering::isOffsetFoldingLegal(
 }
 
 bool AArch64TargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT) const {
-  // We can materialize #0.0 as fmov $Rd, XZR for 64-bit and 32-bit cases.
-  // FIXME: We should be able to handle f128 as well with a clever lowering.
-  if (Imm.isPosZero() && (VT == MVT::f64 || VT == MVT::f32 ||
-                          (VT == MVT::f16 && Subtarget->hasFullFP16()))) {
-    LLVM_DEBUG(dbgs() << "Legal " << VT.getEVTString() << " imm value: 0\n");
-    return true;
-  }
-
   bool IsLegal = false;
-  SmallString<128> ImmStrVal;
-  Imm.toString(ImmStrVal);
-
+  // We can materialize #0.0 as fmov $Rd, XZR for 64-bit, 32-bit cases, and
+  // 16-bit case when target has full fp16 support.
+  // FIXME: We should be able to handle f128 as well with a clever lowering.
+  const APInt ImmInt = Imm.bitcastToAPInt();
   if (VT == MVT::f64)
-    IsLegal = AArch64_AM::getFP64Imm(Imm) != -1;
+    IsLegal = AArch64_AM::getFP64Imm(ImmInt) != -1 || Imm.isPosZero();
   else if (VT == MVT::f32)
-    IsLegal = AArch64_AM::getFP32Imm(Imm) != -1;
+    IsLegal = AArch64_AM::getFP32Imm(ImmInt) != -1 || Imm.isPosZero();
   else if (VT == MVT::f16 && Subtarget->hasFullFP16())
-    IsLegal = AArch64_AM::getFP16Imm(Imm) != -1;
-
-  if (IsLegal) {
-    LLVM_DEBUG(dbgs() << "Legal " << VT.getEVTString()
-                      << " imm value: " << ImmStrVal << "\n");
-    return true;
-  }
-
-  LLVM_DEBUG(dbgs() << "Illegal " << VT.getEVTString()
-                    << " imm value: " << ImmStrVal << "\n");
-  return false;
+    IsLegal = AArch64_AM::getFP16Imm(ImmInt) != -1 || Imm.isPosZero();
+  // TODO: fmov h0, w0 is also legal, however on't have an isel pattern to
+  //       generate that fmov.
+
+  // If we can not materialize in immediate field for fmov, check if the
+  // value can be encoded as the immediate operand of a logical instruction.
+  // The immediate value will be created with either MOVZ, MOVN, or ORR.
+  if (!IsLegal && (VT == MVT::f64 || VT == MVT::f32))
+    IsLegal = AArch64_AM::isAnyMOVWMovAlias(ImmInt.getZExtValue(),
+                                            VT.getSizeInBits());
+
+  LLVM_DEBUG(dbgs() << (IsLegal ? "Legal " : "Illegal ") << VT.getEVTString()
+                    << " imm value: "; Imm.dump(););
+  return IsLegal;
 }
 
 //===----------------------------------------------------------------------===//
@@ -8243,6 +8235,110 @@ bool AArch64TargetLowering::isExtFreeImpl(const Instruction *Ext) const {
   return true;
 }
 
+/// Check if both Op1 and Op2 are shufflevector extracts of either the lower
+/// or upper half of the vector elements.
+static bool areExtractShuffleVectors(Value *Op1, Value *Op2) {
+  auto areTypesHalfed = [](Value *FullV, Value *HalfV) {
+    auto *FullVT = cast<VectorType>(FullV->getType());
+    auto *HalfVT = cast<VectorType>(HalfV->getType());
+    return FullVT->getBitWidth() == 2 * HalfVT->getBitWidth();
+  };
+
+  auto extractHalf = [](Value *FullV, Value *HalfV) {
+    auto *FullVT = cast<VectorType>(FullV->getType());
+    auto *HalfVT = cast<VectorType>(HalfV->getType());
+    return FullVT->getNumElements() == 2 * HalfVT->getNumElements();
+  };
+
+  Constant *M1, *M2;
+  Value *S1Op1, *S2Op1;
+  if (!match(Op1, m_ShuffleVector(m_Value(S1Op1), m_Undef(), m_Constant(M1))) ||
+      !match(Op2, m_ShuffleVector(m_Value(S2Op1), m_Undef(), m_Constant(M2))))
+    return false;
+
+  // Check that the operands are half as wide as the result and we extract
+  // half of the elements of the input vectors.
+  if (!areTypesHalfed(S1Op1, Op1) || !areTypesHalfed(S2Op1, Op2) ||
+      !extractHalf(S1Op1, Op1) || !extractHalf(S2Op1, Op2))
+    return false;
+
+  // Check the mask extracts either the lower or upper half of vector
+  // elements.
+  int M1Start = -1;
+  int M2Start = -1;
+  int NumElements = cast<VectorType>(Op1->getType())->getNumElements() * 2;
+  if (!ShuffleVectorInst::isExtractSubvectorMask(M1, NumElements, M1Start) ||
+      !ShuffleVectorInst::isExtractSubvectorMask(M2, NumElements, M2Start) ||
+      M1Start != M2Start || (M1Start != 0 && M2Start != (NumElements / 2)))
+    return false;
+
+  return true;
+}
+
+/// Check if Ext1 and Ext2 are extends of the same type, doubling the bitwidth
+/// of the vector elements.
+static bool areExtractExts(Value *Ext1, Value *Ext2) {
+  auto areExtDoubled = [](Instruction *Ext) {
+    return Ext->getType()->getScalarSizeInBits() ==
+           2 * Ext->getOperand(0)->getType()->getScalarSizeInBits();
+  };
+
+  if (!match(Ext1, m_ZExtOrSExt(m_Value())) ||
+      !match(Ext2, m_ZExtOrSExt(m_Value())) ||
+      !areExtDoubled(cast<Instruction>(Ext1)) ||
+      !areExtDoubled(cast<Instruction>(Ext2)))
+    return false;
+
+  return true;
+}
+
+/// Check if sinking \p I's operands to I's basic block is profitable, because
+/// the operands can be folded into a target instruction, e.g.
+/// shufflevectors extracts and/or sext/zext can be folded into (u,s)subl(2).
+bool AArch64TargetLowering::shouldSinkOperands(
+    Instruction *I, SmallVectorImpl<Use *> &Ops) const {
+  if (!I->getType()->isVectorTy())
+    return false;
+
+  if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(I)) {
+    switch (II->getIntrinsicID()) {
+    case Intrinsic::aarch64_neon_umull:
+      if (!areExtractShuffleVectors(II->getOperand(0), II->getOperand(1)))
+        return false;
+      Ops.push_back(&II->getOperandUse(0));
+      Ops.push_back(&II->getOperandUse(1));
+      return true;
+    default:
+      return false;
+    }
+  }
+
+  switch (I->getOpcode()) {
+  case Instruction::Sub:
+  case Instruction::Add: {
+    if (!areExtractExts(I->getOperand(0), I->getOperand(1)))
+      return false;
+
+    // If the exts' operands extract either the lower or upper elements, we
+    // can sink them too.
+    auto Ext1 = cast<Instruction>(I->getOperand(0));
+    auto Ext2 = cast<Instruction>(I->getOperand(1));
+    if (areExtractShuffleVectors(Ext1, Ext2)) {
+      Ops.push_back(&Ext1->getOperandUse(0));
+      Ops.push_back(&Ext2->getOperandUse(0));
+    }
+
+    Ops.push_back(&I->getOperandUse(0));
+    Ops.push_back(&I->getOperandUse(1));
+
+    return true;
+  }
+  default:
+    return false;
+  }
+  return false;
+}
+
 bool AArch64TargetLowering::hasPairedLoad(EVT LoadedType,
                                           unsigned &RequiredAligment) const {
   if (!LoadedType.isSimple() ||
@@ -8365,8 +8461,9 @@ bool AArch64TargetLowering::lowerInterleavedLoad(
     // If we're generating more than one load, compute the base address of
     // subsequent loads as an offset from the previous.
     if (LoadCount > 0)
-      BaseAddr = Builder.CreateConstGEP1_32(
-          BaseAddr, VecTy->getVectorNumElements() * Factor);
+      BaseAddr =
+          Builder.CreateConstGEP1_32(VecTy->getVectorElementType(), BaseAddr,
+                                     VecTy->getVectorNumElements() * Factor);
 
     CallInst *LdN = Builder.CreateCall(
         LdNFunc, Builder.CreateBitCast(BaseAddr, PtrTy), "ldN");
@@ -8528,7 +8625,8 @@ bool AArch64TargetLowering::lowerInterleavedStore(StoreInst *SI,
     // If we generating more than one store, we compute the base address of
     // subsequent stores as an offset from the previous.
     if (StoreCount > 0)
-      BaseAddr = Builder.CreateConstGEP1_32(BaseAddr, LaneLen * Factor);
+      BaseAddr = Builder.CreateConstGEP1_32(SubVecTy->getVectorElementType(),
+                                            BaseAddr, LaneLen * Factor);
 
     Ops.push_back(Builder.CreateBitCast(BaseAddr, PtrTy));
     Builder.CreateCall(StNFunc, Ops);
@@ -9586,12 +9684,13 @@ static SDValue tryExtendDUPToExtractHigh(SDValue N, SelectionDAG &DAG) {
                      DAG.getConstant(NumElems, dl, MVT::i64));
 }
 
-static bool isEssentiallyExtractSubvector(SDValue N) {
-  if (N.getOpcode() == ISD::EXTRACT_SUBVECTOR)
-    return true;
-
-  return N.getOpcode() == ISD::BITCAST &&
-         N.getOperand(0).getOpcode() == ISD::EXTRACT_SUBVECTOR;
+static bool isEssentiallyExtractHighSubvector(SDValue N) {
+  if (N.getOpcode() == ISD::BITCAST)
+    N = N.getOperand(0);
+  if (N.getOpcode() != ISD::EXTRACT_SUBVECTOR)
+    return false;
+  return cast<ConstantSDNode>(N.getOperand(1))->getAPIntValue() ==
+         N.getOperand(0).getValueType().getVectorNumElements() / 2;
 }
 
 /// Helper structure to keep track of ISD::SET_CC operands.
@@ -9758,13 +9857,13 @@ static SDValue performAddSubLongCombine(SDNode *N,
 
   // It's not worth doing if at least one of the inputs isn't already an
   // extract, but we don't know which it'll be so we have to try both.
-  if (isEssentiallyExtractSubvector(LHS.getOperand(0))) {
+  if (isEssentiallyExtractHighSubvector(LHS.getOperand(0))) {
     RHS = tryExtendDUPToExtractHigh(RHS.getOperand(0), DAG);
     if (!RHS.getNode())
       return SDValue();
 
     RHS = DAG.getNode(ExtType, SDLoc(N), VT, RHS);
-  } else if (isEssentiallyExtractSubvector(RHS.getOperand(0))) {
+  } else if (isEssentiallyExtractHighSubvector(RHS.getOperand(0))) {
     LHS = tryExtendDUPToExtractHigh(LHS.getOperand(0), DAG);
     if (!LHS.getNode())
       return SDValue();
@@ -9797,11 +9896,11 @@ static SDValue tryCombineLongOpWithDup(unsigned IID, SDNode *N,
   // Either node could be a DUP, but it's not worth doing both of them (you'd
   // just as well use the non-high version) so look for a corresponding extract
   // operation on the other "wing".
-  if (isEssentiallyExtractSubvector(LHS)) {
+  if (isEssentiallyExtractHighSubvector(LHS)) {
     RHS = tryExtendDUPToExtractHigh(RHS, DAG);
     if (!RHS.getNode())
       return SDValue();
-  } else if (isEssentiallyExtractSubvector(RHS)) {
+  } else if (isEssentiallyExtractHighSubvector(RHS)) {
     LHS = tryExtendDUPToExtractHigh(LHS, DAG);
     if (!LHS.getNode())
       return SDValue();
@@ -10053,6 +10152,7 @@ static SDValue performExtendCombine(SDNode *N,
 
 static SDValue splitStoreSplat(SelectionDAG &DAG, StoreSDNode &St,
                                SDValue SplatVal, unsigned NumVecElts) {
+  assert(!St.isTruncatingStore() && "cannot split truncating vector store");
   unsigned OrigAlignment = St.getAlignment();
   unsigned EltOffset = SplatVal.getValueType().getSizeInBits() / 8;
 
@@ -10127,6 +10227,11 @@ static SDValue replaceZeroVectorStore(SelectionDAG &DAG, StoreSDNode &St) {
   if (!StVal.hasOneUse())
     return SDValue();
 
+  // If the store is truncating then it's going down to i16 or smaller, which
+  // means it can be implemented in a single store anyway.
+  if (St.isTruncatingStore())
+    return SDValue();
+
   // If the immediate offset of the address operand is too large for the stp
   // instruction, then bail out.
   if (DAG.isBaseWithConstantOffset(St.getBasePtr())) {
@@ -10177,6 +10282,11 @@ static SDValue replaceSplatVectorStore(SelectionDAG &DAG, StoreSDNode &St) {
   if (NumVecElts != 4 && NumVecElts != 2)
     return SDValue();
 
+  // If the store is truncating then it's going down to i16 or smaller, which
+  // means it can be implemented in a single store anyway.
+  if (St.isTruncatingStore())
+    return SDValue();
+
   // Check that this is a splat.
   // Make sure that each of the relevant vector element locations are inserted
   // to, i.e. 0 and 1 for v2i64 and 0, 1, 2, 3 for v4i32.
@@ -11550,6 +11660,9 @@ AArch64TargetLowering::shouldExpandAtomicLoadInIR(LoadInst *LI) const {
 // For the real atomic operations, we have ldxr/stxr up to 128 bits,
 TargetLowering::AtomicExpansionKind
 AArch64TargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const {
+  if (AI->isFloatingPointOperation())
+    return AtomicExpansionKind::CmpXChg;
+
   unsigned Size = AI->getType()->getPrimitiveSizeInBits();
   if (Size > 128) return AtomicExpansionKind::None;
   // Nand not supported in LSE.
@@ -11604,9 +11717,13 @@ Value *AArch64TargetLowering::emitLoadLinked(IRBuilder<> &Builder, Value *Addr,
       IsAcquire ? Intrinsic::aarch64_ldaxr : Intrinsic::aarch64_ldxr;
   Function *Ldxr = Intrinsic::getDeclaration(M, Int, Tys);
 
-  return Builder.CreateTruncOrBitCast(
-      Builder.CreateCall(Ldxr, Addr),
-      cast<PointerType>(Addr->getType())->getElementType());
+  Type *EltTy = cast<PointerType>(Addr->getType())->getElementType();
+
+  const DataLayout &DL = M->getDataLayout();
+  IntegerType *IntEltTy = Builder.getIntNTy(DL.getTypeSizeInBits(EltTy));
+  Value *Trunc = Builder.CreateTrunc(Builder.CreateCall(Ldxr, Addr), IntEltTy);
+
+  return Builder.CreateBitCast(Trunc, EltTy);
 }
 
 void AArch64TargetLowering::emitAtomicCmpXchgNoStoreLLBalance(
@@ -11641,6 +11758,10 @@ Value *AArch64TargetLowering::emitStoreConditional(IRBuilder<> &Builder,
   Type *Tys[] = { Addr->getType() };
   Function *Stxr = Intrinsic::getDeclaration(M, Int, Tys);
 
+  const DataLayout &DL = M->getDataLayout();
+  IntegerType *IntValTy = Builder.getIntNTy(DL.getTypeSizeInBits(Val->getType()));
+  Val = Builder.CreateBitCast(Val, IntValTy);
+
   return Builder.CreateCall(Stxr,
                             {Builder.CreateZExtOrBitCast(
                                  Val, Stxr->getFunctionType()->getParamType(0)),
@@ -11662,8 +11783,9 @@ static Value *UseTlsOffset(IRBuilder<> &IRB, unsigned Offset) {
   Function *ThreadPointerFunc =
       Intrinsic::getDeclaration(M, Intrinsic::thread_pointer);
   return IRB.CreatePointerCast(
-      IRB.CreateConstGEP1_32(IRB.CreateCall(ThreadPointerFunc), Offset),
-      Type::getInt8PtrTy(IRB.getContext())->getPointerTo(0));
+      IRB.CreateConstGEP1_32(IRB.getInt8Ty(), IRB.CreateCall(ThreadPointerFunc),
+                             Offset),
+      IRB.getInt8PtrTy()->getPointerTo(0));
 }
 
 Value *AArch64TargetLowering::getIRStackGuard(IRBuilder<> &IRB) const {
@@ -11689,12 +11811,13 @@ void AArch64TargetLowering::insertSSPDeclarations(Module &M) const {
                         Type::getInt8PtrTy(M.getContext()));
 
     // MSVC CRT has a function to validate security cookie.
-    auto *SecurityCheckCookie = cast<Function>(
-        M.getOrInsertFunction("__security_check_cookie",
-                              Type::getVoidTy(M.getContext()),
-                              Type::getInt8PtrTy(M.getContext())));
-    SecurityCheckCookie->setCallingConv(CallingConv::Win64);
-    SecurityCheckCookie->addAttribute(1, Attribute::AttrKind::InReg);
+    FunctionCallee SecurityCheckCookie = M.getOrInsertFunction(
+        "__security_check_cookie", Type::getVoidTy(M.getContext()),
+        Type::getInt8PtrTy(M.getContext()));
+    if (Function *F = dyn_cast<Function>(SecurityCheckCookie.getCallee())) {
+      F->setCallingConv(CallingConv::Win64);
+      F->addAttribute(1, Attribute::AttrKind::InReg);
+    }
     return;
   }
   TargetLowering::insertSSPDeclarations(M);
@@ -11707,7 +11830,7 @@ Value *AArch64TargetLowering::getSDagStackGuard(const Module &M) const {
   return TargetLowering::getSDagStackGuard(M);
 }
 
-Value *AArch64TargetLowering::getSSPStackGuardCheck(const Module &M) const {
+Function *AArch64TargetLowering::getSSPStackGuardCheck(const Module &M) const {
   // MSVC CRT has a function to validate security cookie.
   if (Subtarget->getTargetTriple().isWindowsMSVCEnvironment())
     return M.getFunction("__security_check_cookie");
diff --git a/lib/Target/AArch64/AArch64ISelLowering.h b/lib/Target/AArch64/AArch64ISelLowering.h
index ffc4cc3ef53474064faf487c305a37f442005574..05b653af77b4af3bf9aec96862aaf6efff63638a 100644
--- a/lib/Target/AArch64/AArch64ISelLowering.h
+++ b/lib/Target/AArch64/AArch64ISelLowering.h
@@ -1,9 +1,8 @@
 //==-- AArch64ISelLowering.h - AArch64 DAG Lowering Interface ----*- C++ -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -328,6 +327,9 @@ public:
   bool isZExtFree(EVT VT1, EVT VT2) const override;
   bool isZExtFree(SDValue Val, EVT VT2) const override;
 
+  bool shouldSinkOperands(Instruction *I,
+                          SmallVectorImpl<Use *> &Ops) const override;
+
   bool hasPairedLoad(EVT LoadedType, unsigned &RequiredAligment) const override;
 
   unsigned getMaxSupportedInterleaveFactor() const override { return 4; }
@@ -409,7 +411,7 @@ public:
 
   void insertSSPDeclarations(Module &M) const override;
   Value *getSDagStackGuard(const Module &M) const override;
-  Value *getSSPStackGuardCheck(const Module &M) const override;
+  Function *getSSPStackGuardCheck(const Module &M) const override;
 
   /// If the target has a standard location for the unsafe stack pointer,
   /// returns the address of that location. Otherwise, returns nullptr.
@@ -470,6 +472,12 @@ public:
     return VT.getSizeInBits() >= 64; // vector 'bic'
   }
 
+  bool shouldExpandShift(SelectionDAG &DAG, SDNode *N) const override {
+    if (DAG.getMachineFunction().getFunction().optForMinSize())
+      return false;
+    return true;
+  }
+
   bool shouldTransformSignedTruncationCheck(EVT XVT,
                                             unsigned KeptBits) const override {
     // For vectors, we don't have a preference..
diff --git a/lib/Target/AArch64/AArch64InstrAtomics.td b/lib/Target/AArch64/AArch64InstrAtomics.td
index 35cd7735ceb76e006fbfa79174648f4335cff4bb..e22cb44d81aeaf0e26d92e81c4642f255d5329e1 100644
--- a/lib/Target/AArch64/AArch64InstrAtomics.td
+++ b/lib/Target/AArch64/AArch64InstrAtomics.td
@@ -1,9 +1,8 @@
 //=- AArch64InstrAtomics.td - AArch64 Atomic codegen support -*- tablegen -*-=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AArch64/AArch64InstrFormats.td b/lib/Target/AArch64/AArch64InstrFormats.td
index 9061ed4f9f54a6f275685ba66656cdd6072fe629..a5428788417f452f39b3764b63f6bbe000591142 100644
--- a/lib/Target/AArch64/AArch64InstrFormats.td
+++ b/lib/Target/AArch64/AArch64InstrFormats.td
@@ -1,9 +1,8 @@
 //===- AArch64InstrFormats.td - AArch64 Instruction Formats --*- tblgen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -1818,6 +1817,14 @@ multiclass Shift<bits<2> shift_type, string asm, SDNode OpNode> {
 
   def : Pat<(i32 (OpNode GPR32:$Rn, (i64 (sext GPR32:$Rm)))),
             (!cast<Instruction>(NAME # "Wr") GPR32:$Rn, GPR32:$Rm)>;
+
+  def : Pat<(i64 (OpNode GPR64:$Rn, (i64 (sext GPR32:$Rm)))),
+            (!cast<Instruction>(NAME # "Xr") GPR64:$Rn,
+                (SUBREG_TO_REG (i32 0), GPR32:$Rm, sub_32))>;
+
+  def : Pat<(i64 (OpNode GPR64:$Rn, (i64 (zext GPR32:$Rm)))),
+            (!cast<Instruction>(NAME # "Xr") GPR64:$Rn,
+                (SUBREG_TO_REG (i32 0), GPR32:$Rm, sub_32))>;
 }
 
 class ShiftAlias<string asm, Instruction inst, RegisterClass regtype>
diff --git a/lib/Target/AArch64/AArch64InstrInfo.cpp b/lib/Target/AArch64/AArch64InstrInfo.cpp
index 10464ea57bb9915f45248e99736565cb41329934..f5d86c2e54aefb9ac5b7c17b2793224b3f284177 100644
--- a/lib/Target/AArch64/AArch64InstrInfo.cpp
+++ b/lib/Target/AArch64/AArch64InstrInfo.cpp
@@ -1,9 +1,8 @@
 //===- AArch64InstrInfo.cpp - AArch64 Instruction Information -------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -965,6 +964,11 @@ bool AArch64InstrInfo::isSchedulingBoundary(const MachineInstr &MI,
   if (TargetInstrInfo::isSchedulingBoundary(MI, MBB, MF))
     return true;
   switch (MI.getOpcode()) {
+  case AArch64::HINT:
+    // CSDB hints are scheduling barriers.
+    if (MI.getOperand(0).getImm() == 0x14)
+      return true;
+    break;
   case AArch64::DSB:
   case AArch64::ISB:
     // DSB and ISB also are scheduling barriers.
@@ -2287,6 +2291,31 @@ void AArch64InstrInfo::copyPhysRegTuple(MachineBasicBlock &MBB,
   }
 }
 
+void AArch64InstrInfo::copyGPRRegTuple(MachineBasicBlock &MBB,
+                                       MachineBasicBlock::iterator I,
+                                       DebugLoc DL, unsigned DestReg,
+                                       unsigned SrcReg, bool KillSrc,
+                                       unsigned Opcode, unsigned ZeroReg,
+                                       llvm::ArrayRef<unsigned> Indices) const {
+  const TargetRegisterInfo *TRI = &getRegisterInfo();
+  unsigned NumRegs = Indices.size();
+
+#ifndef NDEBUG
+  uint16_t DestEncoding = TRI->getEncodingValue(DestReg);
+  uint16_t SrcEncoding = TRI->getEncodingValue(SrcReg);
+  assert(DestEncoding % NumRegs == 0 && SrcEncoding % NumRegs == 0 &&
+         "GPR reg sequences should not be able to overlap");
+#endif
+
+  for (unsigned SubReg = 0; SubReg != NumRegs; ++SubReg) {
+    const MachineInstrBuilder MIB = BuildMI(MBB, I, DL, get(Opcode));
+    AddSubReg(MIB, DestReg, Indices[SubReg], RegState::Define, TRI);
+    MIB.addReg(ZeroReg);
+    AddSubReg(MIB, SrcReg, Indices[SubReg], getKillRegState(KillSrc), TRI);
+    MIB.addImm(0);
+  }
+}
+
 void AArch64InstrInfo::copyPhysReg(MachineBasicBlock &MBB,
                                    MachineBasicBlock::iterator I,
                                    const DebugLoc &DL, unsigned DestReg,
@@ -2426,6 +2455,22 @@ void AArch64InstrInfo::copyPhysReg(MachineBasicBlock &MBB,
     return;
   }
 
+  if (AArch64::XSeqPairsClassRegClass.contains(DestReg) &&
+      AArch64::XSeqPairsClassRegClass.contains(SrcReg)) {
+    static const unsigned Indices[] = {AArch64::sube64, AArch64::subo64};
+    copyGPRRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, AArch64::ORRXrs,
+                    AArch64::XZR, Indices);
+    return;
+  }
+
+  if (AArch64::WSeqPairsClassRegClass.contains(DestReg) &&
+      AArch64::WSeqPairsClassRegClass.contains(SrcReg)) {
+    static const unsigned Indices[] = {AArch64::sube32, AArch64::subo32};
+    copyGPRRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, AArch64::ORRWrs,
+                    AArch64::WZR, Indices);
+    return;
+  }
+
   if (AArch64::FPR128RegClass.contains(DestReg) &&
       AArch64::FPR128RegClass.contains(SrcReg)) {
     if (Subtarget.hasNEON()) {
@@ -4746,7 +4791,8 @@ AArch64InstrInfo::getSerializableBitmaskMachineOperandTargetFlags() const {
   static const std::pair<unsigned, const char *> TargetFlags[] = {
       {MO_COFFSTUB, "aarch64-coffstub"},
       {MO_GOT, "aarch64-got"},   {MO_NC, "aarch64-nc"},
-      {MO_TLS, "aarch64-tls"},   {MO_DLLIMPORT, "aarch64-dllimport"}};
+      {MO_S, "aarch64-s"},       {MO_TLS, "aarch64-tls"},
+      {MO_DLLIMPORT, "aarch64-dllimport"}};
   return makeArrayRef(TargetFlags);
 }
 
@@ -5325,6 +5371,14 @@ AArch64InstrInfo::getOutliningType(MachineBasicBlock::iterator &MIT,
       MI.modifiesRegister(AArch64::W30, &getRegisterInfo()))
     return outliner::InstrType::Illegal;
 
+  // Don't outline BTI instructions, because that will prevent the outlining
+  // site from being indirectly callable.
+  if (MI.getOpcode() == AArch64::HINT) {
+    int64_t Imm = MI.getOperand(0).getImm();
+    if (Imm == 32 || Imm == 34 || Imm == 36 || Imm == 38)
+      return outliner::InstrType::Illegal;
+  }
+
   return outliner::InstrType::Legal;
 }
 
diff --git a/lib/Target/AArch64/AArch64InstrInfo.h b/lib/Target/AArch64/AArch64InstrInfo.h
index 9954669d567508d6ce83cb9d17d00dee077c1a91..537a812264ac6a290024e6deb7e6f015b8903df7 100644
--- a/lib/Target/AArch64/AArch64InstrInfo.h
+++ b/lib/Target/AArch64/AArch64InstrInfo.h
@@ -1,9 +1,8 @@
 //===- AArch64InstrInfo.h - AArch64 Instruction Information -----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -122,6 +121,10 @@ public:
                         const DebugLoc &DL, unsigned DestReg, unsigned SrcReg,
                         bool KillSrc, unsigned Opcode,
                         llvm::ArrayRef<unsigned> Indices) const;
+  void copyGPRRegTuple(MachineBasicBlock &MBB, MachineBasicBlock::iterator I,
+                       DebugLoc DL, unsigned DestReg, unsigned SrcReg,
+                       bool KillSrc, unsigned Opcode, unsigned ZeroReg,
+                       llvm::ArrayRef<unsigned> Indices) const;
   void copyPhysReg(MachineBasicBlock &MBB, MachineBasicBlock::iterator I,
                    const DebugLoc &DL, unsigned DestReg, unsigned SrcReg,
                    bool KillSrc) const override;
diff --git a/lib/Target/AArch64/AArch64InstrInfo.td b/lib/Target/AArch64/AArch64InstrInfo.td
index ee43b55aed93508ae319f977e53c1195b8f96303..acfec340563ce542a64b99471f3620ffbbf7a31b 100644
--- a/lib/Target/AArch64/AArch64InstrInfo.td
+++ b/lib/Target/AArch64/AArch64InstrInfo.td
@@ -1,9 +1,8 @@
 //=- AArch64InstrInfo.td - Describe the AArch64 Instructions -*- tablegen -*-=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -114,10 +113,10 @@ def HasAltNZCV       : Predicate<"Subtarget->hasAlternativeNZCV()">,
                        AssemblerPredicate<"FeatureAltFPCmp", "altnzcv">;
 def HasFRInt3264     : Predicate<"Subtarget->hasFRInt3264()">,
                        AssemblerPredicate<"FeatureFRInt3264", "frint3264">;
-def HasSpecCtrl      : Predicate<"Subtarget->hasSpecCtrl()">,
-                       AssemblerPredicate<"FeatureSpecCtrl", "specctrl">;
-def HasPredCtrl      : Predicate<"Subtarget->hasPredCtrl()">,
-                       AssemblerPredicate<"FeaturePredCtrl", "predctrl">;
+def HasSB            : Predicate<"Subtarget->hasSB()">,
+                       AssemblerPredicate<"FeatureSB", "sb">;
+def HasPredRes      : Predicate<"Subtarget->hasPredRes()">,
+                       AssemblerPredicate<"FeaturePredRes", "predres">;
 def HasCCDP          : Predicate<"Subtarget->hasCCDP()">,
                        AssemblerPredicate<"FeatureCacheDeepPersist", "ccdp">;
 def HasBTI           : Predicate<"Subtarget->hasBTI()">,
@@ -133,6 +132,10 @@ def UseNegativeImmediates
     : Predicate<"false">, AssemblerPredicate<"!FeatureNoNegativeImmediates",
                                              "NegativeImmediates">;
 
+def AArch64LocalRecover : SDNode<"ISD::LOCAL_RECOVER",
+                                  SDTypeProfile<1, 1, [SDTCisSameAs<0, 1>,
+                                                       SDTCisInt<1>]>>;
+
 
 //===----------------------------------------------------------------------===//
 // AArch64-specific DAG Nodes.
@@ -520,6 +523,14 @@ def SPACE : Pseudo<(outs GPR64:$Rd), (ins i32imm:$size, GPR64:$Rn),
                    [(set GPR64:$Rd, (int_aarch64_space imm:$size, GPR64:$Rn))]>,
             Sched<[]>;
 
+let hasSideEffects = 1, isCodeGenOnly = 1 in {
+  def SpeculationSafeValueX
+      : Pseudo<(outs GPR64:$dst), (ins GPR64:$src), []>, Sched<[]>;
+  def SpeculationSafeValueW
+      : Pseudo<(outs GPR32:$dst), (ins GPR32:$src), []>, Sched<[]>;
+}
+
+
 //===----------------------------------------------------------------------===//
 // System instructions.
 //===----------------------------------------------------------------------===//
@@ -733,7 +744,7 @@ def AXFLAG : PstateWriteSimple<(ins), "axflag", "">, Sched<[WriteSys]> {
 def SB : SimpleSystemI<0, (ins), "sb", "">, Sched<[]> {
   let Inst{20-5} = 0b0001100110000111;
   let Unpredictable{11-8} = 0b1111;
-  let Predicates = [HasSpecCtrl];
+  let Predicates = [HasSB];
   let hasSideEffects = 1;
 }
 
@@ -752,6 +763,13 @@ def MSRpstateImm4 : MSRpstateImm0_15;
 def MOVbaseTLS : Pseudo<(outs GPR64:$dst), (ins),
                        [(set GPR64:$dst, AArch64threadpointer)]>, Sched<[WriteSys]>;
 
+let Uses = [ X9 ], Defs = [ X16, X17, LR, NZCV ] in {
+def HWASAN_CHECK_MEMACCESS : Pseudo<
+  (outs), (ins GPR64noip:$ptr, i32imm:$accessinfo),
+  [(int_hwasan_check_memaccess X9, GPR64noip:$ptr, (i32 imm:$accessinfo))]>,
+  Sched<[]>;
+}
+
 // The cycle counter PMC register is PMCCNTR_EL0.
 let Predicates = [HasPerfMon] in
 def : Pat<(readcyclecounter), (MRS 0xdce8)>;
@@ -1619,6 +1637,10 @@ def TLSDESCCALL : Pseudo<(outs), (ins i64imm:$sym), []>, Sched<[]> {
   let AsmString = ".tlsdesccall $sym";
 }
 
+// Pseudo instruction to tell the streamer to emit a 'B' character into the
+// augmentation string.
+def EMITBKEY : Pseudo<(outs), (ins), []>, Sched<[]> {}
+
 // FIXME: maybe the scratch register used shouldn't be fixed to X1?
 // FIXME: can "hasSideEffects be dropped?
 let isCall = 1, Defs = [LR, X0, X1], hasSideEffects = 1,
@@ -6789,5 +6811,8 @@ def : Pat<(AArch64tcret tglobaladdr:$dst, (i32 timm:$FPDiff)),
 def : Pat<(AArch64tcret texternalsym:$dst, (i32 timm:$FPDiff)),
           (TCRETURNdi texternalsym:$dst, imm:$FPDiff)>;
 
+def MOVMCSym : Pseudo<(outs GPR64:$dst), (ins i64imm:$sym), []>, Sched<[]>;
+def : Pat<(i64 (AArch64LocalRecover mcsym:$sym)), (MOVMCSym mcsym:$sym)>;
+
 include "AArch64InstrAtomics.td"
 include "AArch64SVEInstrInfo.td"
diff --git a/lib/Target/AArch64/AArch64InstructionSelector.cpp b/lib/Target/AArch64/AArch64InstructionSelector.cpp
index 6cbfb6ab161b25e4ce8526fd7007fcc5e6eb78ed..e1352c366e8c1d4574f5eae0cd473dd69eb08aa3 100644
--- a/lib/Target/AArch64/AArch64InstructionSelector.cpp
+++ b/lib/Target/AArch64/AArch64InstructionSelector.cpp
@@ -1,9 +1,8 @@
 //===- AArch64InstructionSelector.cpp ----------------------------*- C++ -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
@@ -73,6 +72,8 @@ private:
                           MachineBasicBlock::iterator MBBI,
                           MachineRegisterInfo &MRI) const;
   bool selectBuildVector(MachineInstr &I, MachineRegisterInfo &MRI) const;
+  bool selectMergeValues(MachineInstr &I, MachineRegisterInfo &MRI) const;
+  bool selectUnmergeValues(MachineInstr &I, MachineRegisterInfo &MRI) const;
 
   ComplexRendererFns selectArithImmed(MachineOperand &Root) const;
 
@@ -176,6 +177,70 @@ getRegClassForTypeOnBank(LLT Ty, const RegisterBank &RB,
   return nullptr;
 }
 
+/// Given a register bank, and size in bits, return the smallest register class
+/// that can represent that combination.
+static const TargetRegisterClass *
+getMinClassForRegBank(const RegisterBank &RB, unsigned SizeInBits,
+                      bool GetAllRegSet = false) {
+  unsigned RegBankID = RB.getID();
+
+  if (RegBankID == AArch64::GPRRegBankID) {
+    if (SizeInBits <= 32)
+      return GetAllRegSet ? &AArch64::GPR32allRegClass
+                          : &AArch64::GPR32RegClass;
+    if (SizeInBits == 64)
+      return GetAllRegSet ? &AArch64::GPR64allRegClass
+                          : &AArch64::GPR64RegClass;
+  }
+
+  if (RegBankID == AArch64::FPRRegBankID) {
+    switch (SizeInBits) {
+    default:
+      return nullptr;
+    case 8:
+      return &AArch64::FPR8RegClass;
+    case 16:
+      return &AArch64::FPR16RegClass;
+    case 32:
+      return &AArch64::FPR32RegClass;
+    case 64:
+      return &AArch64::FPR64RegClass;
+    case 128:
+      return &AArch64::FPR128RegClass;
+    }
+  }
+
+  return nullptr;
+}
+
+/// Returns the correct subregister to use for a given register class.
+static bool getSubRegForClass(const TargetRegisterClass *RC,
+                              const TargetRegisterInfo &TRI, unsigned &SubReg) {
+  switch (TRI.getRegSizeInBits(*RC)) {
+  case 8:
+    SubReg = AArch64::bsub;
+    break;
+  case 16:
+    SubReg = AArch64::hsub;
+    break;
+  case 32:
+    if (RC == &AArch64::GPR32RegClass)
+      SubReg = AArch64::sub_32;
+    else
+      SubReg = AArch64::ssub;
+    break;
+  case 64:
+    SubReg = AArch64::dsub;
+    break;
+  default:
+    LLVM_DEBUG(
+        dbgs() << "Couldn't find appropriate subregister for register class.");
+    return false;
+  }
+
+  return true;
+}
+
 /// Check whether \p I is a currently unsupported binary operation:
 /// - it has an unsized type
 /// - an operand is not a vreg
@@ -331,107 +396,181 @@ static unsigned selectLoadStoreUIOp(unsigned GenericOpc, unsigned RegBankID,
   return GenericOpc;
 }
 
-static bool selectFP16CopyFromGPR32(MachineInstr &I, const TargetInstrInfo &TII,
-                                    MachineRegisterInfo &MRI, unsigned SrcReg) {
-  // Copies from gpr32 to fpr16 need to use a sub-register copy.
-  unsigned CopyReg = MRI.createVirtualRegister(&AArch64::FPR32RegClass);
-  BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(AArch64::COPY))
-      .addDef(CopyReg)
-      .addUse(SrcReg);
-  unsigned SubRegCopy = MRI.createVirtualRegister(&AArch64::FPR16RegClass);
-  BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(TargetOpcode::COPY))
-      .addDef(SubRegCopy)
-      .addUse(CopyReg, 0, AArch64::hsub);
-
-  MachineOperand &RegOp = I.getOperand(1);
-  RegOp.setReg(SubRegCopy);
-  return true;
-}
-
-static bool selectCopy(MachineInstr &I, const TargetInstrInfo &TII,
-                       MachineRegisterInfo &MRI, const TargetRegisterInfo &TRI,
-                       const RegisterBankInfo &RBI) {
-
-  unsigned DstReg = I.getOperand(0).getReg();
-  unsigned SrcReg = I.getOperand(1).getReg();
-
-  if (TargetRegisterInfo::isPhysicalRegister(DstReg)) {
-    if (TRI.getRegClass(AArch64::FPR16RegClassID)->contains(DstReg) &&
-        !TargetRegisterInfo::isPhysicalRegister(SrcReg)) {
-      const RegisterBank &RegBank = *RBI.getRegBank(SrcReg, MRI, TRI);
-      const TargetRegisterClass *SrcRC = getRegClassForTypeOnBank(
-          MRI.getType(SrcReg), RegBank, RBI, /* GetAllRegSet */ true);
-      if (SrcRC == &AArch64::GPR32allRegClass)
-        return selectFP16CopyFromGPR32(I, TII, MRI, SrcReg);
-    }
-    assert(I.isCopy() && "Generic operators do not allow physical registers");
-    return true;
-  }
-
-  const RegisterBank &RegBank = *RBI.getRegBank(DstReg, MRI, TRI);
-  const unsigned DstSize = MRI.getType(DstReg).getSizeInBits();
-  (void)DstSize;
+#ifndef NDEBUG
+/// Helper function that verifies that we have a valid copy at the end of
+/// selectCopy. Verifies that the source and dest have the expected sizes and
+/// then returns true.
+static bool isValidCopy(const MachineInstr &I, const RegisterBank &DstBank,
+                        const MachineRegisterInfo &MRI,
+                        const TargetRegisterInfo &TRI,
+                        const RegisterBankInfo &RBI) {
+  const unsigned DstReg = I.getOperand(0).getReg();
+  const unsigned SrcReg = I.getOperand(1).getReg();
+  const unsigned DstSize = RBI.getSizeInBits(DstReg, MRI, TRI);
   const unsigned SrcSize = RBI.getSizeInBits(SrcReg, MRI, TRI);
-  (void)SrcSize;
-  assert((!TargetRegisterInfo::isPhysicalRegister(SrcReg) || I.isCopy()) &&
-         "No phys reg on generic operators");
+
+  // Make sure the size of the source and dest line up.
   assert(
       (DstSize == SrcSize ||
        // Copies are a mean to setup initial types, the number of
        // bits may not exactly match.
-       (TargetRegisterInfo::isPhysicalRegister(SrcReg) &&
-        DstSize <= RBI.getSizeInBits(SrcReg, MRI, TRI)) ||
+       (TargetRegisterInfo::isPhysicalRegister(SrcReg) && DstSize <= SrcSize) ||
        // Copies are a mean to copy bits around, as long as we are
        // on the same register class, that's fine. Otherwise, that
        // means we need some SUBREG_TO_REG or AND & co.
        (((DstSize + 31) / 32 == (SrcSize + 31) / 32) && DstSize > SrcSize)) &&
       "Copy with different width?!");
-  assert((DstSize <= 64 || RegBank.getID() == AArch64::FPRRegBankID) &&
+
+  // Check the size of the destination.
+  assert((DstSize <= 64 || DstBank.getID() == AArch64::FPRRegBankID) &&
          "GPRs cannot get more than 64-bit width values");
 
-  const TargetRegisterClass *RC = getRegClassForTypeOnBank(
-      MRI.getType(DstReg), RegBank, RBI, /* GetAllRegSet */ true);
-  if (!RC) {
-    LLVM_DEBUG(dbgs() << "Unexpected bitcast size " << DstSize << '\n');
+  return true;
+}
+#endif
+
+/// Helper function for selectCopy. Inserts a subregister copy from
+/// \p *From to \p *To, linking it up to \p I.
+///
+/// e.g, given I = "Dst = COPY SrcReg", we'll transform that into
+///
+/// CopyReg (From class) = COPY SrcReg
+/// SubRegCopy (To class) = COPY CopyReg:SubReg
+/// Dst = COPY SubRegCopy
+static bool selectSubregisterCopy(MachineInstr &I, const TargetInstrInfo &TII,
+                                  MachineRegisterInfo &MRI,
+                                  const RegisterBankInfo &RBI, unsigned SrcReg,
+                                  const TargetRegisterClass *From,
+                                  const TargetRegisterClass *To,
+                                  unsigned SubReg) {
+  unsigned CopyReg = MRI.createVirtualRegister(From);
+  BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(AArch64::COPY), CopyReg)
+      .addUse(SrcReg);
+  unsigned SubRegCopy = MRI.createVirtualRegister(To);
+  BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(TargetOpcode::COPY),
+          SubRegCopy)
+      .addUse(CopyReg, 0, SubReg);
+  MachineOperand &RegOp = I.getOperand(1);
+  RegOp.setReg(SubRegCopy);
+
+  // It's possible that the destination register won't be constrained. Make
+  // sure that happens.
+  if (!TargetRegisterInfo::isPhysicalRegister(I.getOperand(0).getReg()))
+    RBI.constrainGenericRegister(I.getOperand(0).getReg(), *To, MRI);
+
+  return true;
+}
+
+static bool selectCopy(MachineInstr &I, const TargetInstrInfo &TII,
+                       MachineRegisterInfo &MRI, const TargetRegisterInfo &TRI,
+                       const RegisterBankInfo &RBI) {
+
+  unsigned DstReg = I.getOperand(0).getReg();
+  unsigned SrcReg = I.getOperand(1).getReg();
+  const RegisterBank &DstRegBank = *RBI.getRegBank(DstReg, MRI, TRI);
+  const RegisterBank &SrcRegBank = *RBI.getRegBank(SrcReg, MRI, TRI);
+  const TargetRegisterClass *DstRC = getMinClassForRegBank(
+      DstRegBank, RBI.getSizeInBits(DstReg, MRI, TRI), true);
+  if (!DstRC) {
+    LLVM_DEBUG(dbgs() << "Unexpected dest size "
+                      << RBI.getSizeInBits(DstReg, MRI, TRI) << '\n');
     return false;
   }
 
-  if (!TargetRegisterInfo::isPhysicalRegister(SrcReg)) {
-    const RegClassOrRegBank &RegClassOrBank = MRI.getRegClassOrRegBank(SrcReg);
-    const TargetRegisterClass *SrcRC =
-        RegClassOrBank.dyn_cast<const TargetRegisterClass *>();
-    const RegisterBank *RB = nullptr;
+  // A couple helpers below, for making sure that the copy we produce is valid.
+
+  // Set to true if we insert a SUBREG_TO_REG. If we do this, then we don't want
+  // to verify that the src and dst are the same size, since that's handled by
+  // the SUBREG_TO_REG.
+  bool KnownValid = false;
+
+  // Returns true, or asserts if something we don't expect happens. Instead of
+  // returning true, we return isValidCopy() to ensure that we verify the
+  // result.
+  auto CheckCopy = [&]() {
+    // If we have a bitcast or something, we can't have physical registers.
+    assert(
+        (I.isCopy() ||
+         (!TargetRegisterInfo::isPhysicalRegister(I.getOperand(0).getReg()) &&
+          !TargetRegisterInfo::isPhysicalRegister(I.getOperand(1).getReg()))) &&
+        "No phys reg on generic operator!");
+    assert(KnownValid || isValidCopy(I, DstRegBank, MRI, TRI, RBI));
+    return true;
+  };
+
+  // Is this a copy? If so, then we may need to insert a subregister copy, or
+  // a SUBREG_TO_REG.
+  if (I.isCopy()) {
+    // Yes. Check if there's anything to fix up.
+    const TargetRegisterClass *SrcRC = getMinClassForRegBank(
+        SrcRegBank, RBI.getSizeInBits(SrcReg, MRI, TRI), true);
     if (!SrcRC) {
-      RB = RegClassOrBank.get<const RegisterBank *>();
-      SrcRC = getRegClassForTypeOnBank(MRI.getType(SrcReg), *RB, RBI, true);
+      LLVM_DEBUG(dbgs() << "Couldn't determine source register class\n");
+      return false;
     }
-    // Copies from fpr16 to gpr32 need to use SUBREG_TO_REG.
-    if (RC == &AArch64::GPR32allRegClass && SrcRC == &AArch64::FPR16RegClass) {
-      unsigned PromoteReg = MRI.createVirtualRegister(&AArch64::FPR32RegClass);
-      BuildMI(*I.getParent(), I, I.getDebugLoc(),
-              TII.get(AArch64::SUBREG_TO_REG))
-          .addDef(PromoteReg)
-          .addImm(0)
-          .addUse(SrcReg)
-          .addImm(AArch64::hsub);
-      MachineOperand &RegOp = I.getOperand(1);
-      RegOp.setReg(PromoteReg);
-    } else if (RC == &AArch64::FPR16RegClass &&
-               SrcRC == &AArch64::GPR32allRegClass) {
-      selectFP16CopyFromGPR32(I, TII, MRI, SrcReg);
+
+    // Is this a cross-bank copy?
+    if (DstRegBank.getID() != SrcRegBank.getID()) {
+      // If we're doing a cross-bank copy on different-sized registers, we need
+      // to do a bit more work.
+      unsigned SrcSize = TRI.getRegSizeInBits(*SrcRC);
+      unsigned DstSize = TRI.getRegSizeInBits(*DstRC);
+
+      if (SrcSize > DstSize) {
+        // We're doing a cross-bank copy into a smaller register. We need a
+        // subregister copy. First, get a register class that's on the same bank
+        // as the destination, but the same size as the source.
+        const TargetRegisterClass *SubregRC =
+            getMinClassForRegBank(DstRegBank, SrcSize, true);
+        assert(SubregRC && "Didn't get a register class for subreg?");
+
+        // Get the appropriate subregister for the destination.
+        unsigned SubReg = 0;
+        if (!getSubRegForClass(DstRC, TRI, SubReg)) {
+          LLVM_DEBUG(dbgs() << "Couldn't determine subregister for copy.\n");
+          return false;
+        }
+
+        // Now, insert a subregister copy using the new register class.
+        selectSubregisterCopy(I, TII, MRI, RBI, SrcReg, SubregRC, DstRC,
+                              SubReg);
+        return CheckCopy();
+      }
+
+      else if (DstRegBank.getID() == AArch64::GPRRegBankID && DstSize == 32 &&
+               SrcSize == 16) {
+        // Special case for FPR16 to GPR32.
+        // FIXME: This can probably be generalized like the above case.
+        unsigned PromoteReg =
+            MRI.createVirtualRegister(&AArch64::FPR32RegClass);
+        BuildMI(*I.getParent(), I, I.getDebugLoc(),
+                TII.get(AArch64::SUBREG_TO_REG), PromoteReg)
+            .addImm(0)
+            .addUse(SrcReg)
+            .addImm(AArch64::hsub);
+        MachineOperand &RegOp = I.getOperand(1);
+        RegOp.setReg(PromoteReg);
+
+        // Promise that the copy is implicitly validated by the SUBREG_TO_REG.
+        KnownValid = true;
+      }
     }
+
+    // If the destination is a physical register, then there's nothing to
+    // change, so we're done.
+    if (TargetRegisterInfo::isPhysicalRegister(DstReg))
+      return CheckCopy();
   }
 
-  // No need to constrain SrcReg. It will get constrained when
-  // we hit another of its use or its defs.
-  // Copies do not have constraints.
-  if (!RBI.constrainGenericRegister(DstReg, *RC, MRI)) {
+  // No need to constrain SrcReg. It will get constrained when we hit another
+  // of its use or its defs. Copies do not have constraints.
+  if (!RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) {
     LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode())
                       << " operand\n");
     return false;
   }
   I.setDesc(TII.get(AArch64::COPY));
-  return true;
+  return CheckCopy();
 }
 
 static unsigned selectFPConvOpc(unsigned GenericOpc, LLT DstTy, LLT SrcTy) {
@@ -701,7 +840,7 @@ void AArch64InstructionSelector::materializeLargeCMVal(
     constrainSelectedInstRegOperands(*MovI, TII, TRI, RBI);
     return DstReg;
   };
-  unsigned DstReg = BuildMovK(MovZ->getOperand(0).getReg(),
+  unsigned DstReg = BuildMovK(MovZ.getReg(0),
                               AArch64II::MO_G1 | AArch64II::MO_NC, 16, 0);
   DstReg = BuildMovK(DstReg, AArch64II::MO_G2 | AArch64II::MO_NC, 32, 0);
   BuildMovK(DstReg, AArch64II::MO_G3, 48, I.getOperand(0).getReg());
@@ -1553,6 +1692,10 @@ bool AArch64InstructionSelector::select(MachineInstr &I,
   }
   case TargetOpcode::G_BUILD_VECTOR:
     return selectBuildVector(I, MRI);
+  case TargetOpcode::G_MERGE_VALUES:
+    return selectMergeValues(I, MRI);
+  case TargetOpcode::G_UNMERGE_VALUES:
+    return selectUnmergeValues(I, MRI);
   }
 
   return false;
@@ -1581,6 +1724,8 @@ bool AArch64InstructionSelector::emitScalarToVector(
   };
 
   switch (DstTy.getElementType().getSizeInBits()) {
+  case 16:
+    return BuildFn(AArch64::hsub);
   case 32:
     return BuildFn(AArch64::ssub);
   case 64:
@@ -1590,6 +1735,184 @@ bool AArch64InstructionSelector::emitScalarToVector(
   }
 }
 
+bool AArch64InstructionSelector::selectMergeValues(
+    MachineInstr &I, MachineRegisterInfo &MRI) const {
+  assert(I.getOpcode() == TargetOpcode::G_MERGE_VALUES && "unexpected opcode");
+  const LLT DstTy = MRI.getType(I.getOperand(0).getReg());
+  const LLT SrcTy = MRI.getType(I.getOperand(1).getReg());
+  assert(!DstTy.isVector() && !SrcTy.isVector() && "invalid merge operation");
+
+  // At the moment we only support merging two s32s into an s64.
+  if (I.getNumOperands() != 3)
+    return false;
+  if (DstTy.getSizeInBits() != 64 || SrcTy.getSizeInBits() != 32)
+    return false;
+  const RegisterBank &RB = *RBI.getRegBank(I.getOperand(1).getReg(), MRI, TRI);
+  if (RB.getID() != AArch64::GPRRegBankID)
+    return false;
+
+  auto *DstRC = &AArch64::GPR64RegClass;
+  unsigned SubToRegDef = MRI.createVirtualRegister(DstRC);
+  MachineInstr &SubRegMI = *BuildMI(*I.getParent(), I, I.getDebugLoc(),
+                                    TII.get(TargetOpcode::SUBREG_TO_REG))
+                                .addDef(SubToRegDef)
+                                .addImm(0)
+                                .addUse(I.getOperand(1).getReg())
+                                .addImm(AArch64::sub_32);
+  unsigned SubToRegDef2 = MRI.createVirtualRegister(DstRC);
+  // Need to anyext the second scalar before we can use bfm
+  MachineInstr &SubRegMI2 = *BuildMI(*I.getParent(), I, I.getDebugLoc(),
+                                    TII.get(TargetOpcode::SUBREG_TO_REG))
+                                .addDef(SubToRegDef2)
+                                .addImm(0)
+                                .addUse(I.getOperand(2).getReg())
+                                .addImm(AArch64::sub_32);
+  MachineInstr &BFM =
+      *BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(AArch64::BFMXri))
+           .addDef(I.getOperand(0).getReg())
+           .addUse(SubToRegDef)
+           .addUse(SubToRegDef2)
+           .addImm(32)
+           .addImm(31);
+  constrainSelectedInstRegOperands(SubRegMI, TII, TRI, RBI);
+  constrainSelectedInstRegOperands(SubRegMI2, TII, TRI, RBI);
+  constrainSelectedInstRegOperands(BFM, TII, TRI, RBI);
+  I.eraseFromParent();
+  return true;
+}
+
+bool AArch64InstructionSelector::selectUnmergeValues(
+    MachineInstr &I, MachineRegisterInfo &MRI) const {
+  assert(I.getOpcode() == TargetOpcode::G_UNMERGE_VALUES &&
+         "unexpected opcode");
+
+  // TODO: Handle unmerging into GPRs and from scalars to scalars.
+  if (RBI.getRegBank(I.getOperand(0).getReg(), MRI, TRI)->getID() !=
+          AArch64::FPRRegBankID ||
+      RBI.getRegBank(I.getOperand(1).getReg(), MRI, TRI)->getID() !=
+          AArch64::FPRRegBankID) {
+    LLVM_DEBUG(dbgs() << "Unmerging vector-to-gpr and scalar-to-scalar "
+                         "currently unsupported.\n");
+    return false;
+  }
+
+  // The last operand is the vector source register, and every other operand is
+  // a register to unpack into.
+  unsigned NumElts = I.getNumOperands() - 1;
+  unsigned SrcReg = I.getOperand(NumElts).getReg();
+  const LLT NarrowTy = MRI.getType(I.getOperand(0).getReg());
+  const LLT WideTy = MRI.getType(SrcReg);
+  (void)WideTy;
+  assert(WideTy.isVector() && "can only unmerge from vector types!");
+  assert(WideTy.getSizeInBits() > NarrowTy.getSizeInBits() &&
+         "source register size too small!");
+
+  // TODO: Handle unmerging into vectors.
+  if (!NarrowTy.isScalar()) {
+    LLVM_DEBUG(dbgs() << "Vector-to-vector unmerges not supported yet.\n");
+    return false;
+  }
+
+  // Choose a lane copy opcode and subregister based off of the size of the
+  // vector's elements.
+  unsigned CopyOpc = 0;
+  unsigned ExtractSubReg = 0;
+  switch (NarrowTy.getSizeInBits()) {
+  case 16:
+    CopyOpc = AArch64::CPYi16;
+    ExtractSubReg = AArch64::hsub;
+    break;
+  case 32:
+    CopyOpc = AArch64::CPYi32;
+    ExtractSubReg = AArch64::ssub;
+    break;
+  case 64:
+    CopyOpc = AArch64::CPYi64;
+    ExtractSubReg = AArch64::dsub;
+    break;
+  default:
+    // Unknown size, bail out.
+    LLVM_DEBUG(dbgs() << "NarrowTy had unsupported size.\n");
+    return false;
+  }
+
+  // Set up for the lane copies.
+  MachineBasicBlock &MBB = *I.getParent();
+
+  // Stores the registers we'll be copying from.
+  SmallVector<unsigned, 4> InsertRegs;
+
+  // We'll use the first register twice, so we only need NumElts-1 registers.
+  unsigned NumInsertRegs = NumElts - 1;
+
+  // If our elements fit into exactly 128 bits, then we can copy from the source
+  // directly. Otherwise, we need to do a bit of setup with some subregister
+  // inserts.
+  if (NarrowTy.getSizeInBits() * NumElts == 128) {
+    InsertRegs = SmallVector<unsigned, 4>(NumInsertRegs, SrcReg);
+  } else {
+    // No. We have to perform subregister inserts. For each insert, create an
+    // implicit def and a subregister insert, and save the register we create.
+    for (unsigned Idx = 0; Idx < NumInsertRegs; ++Idx) {
+      unsigned ImpDefReg = MRI.createVirtualRegister(&AArch64::FPR128RegClass);
+      MachineInstr &ImpDefMI =
+          *BuildMI(MBB, I, I.getDebugLoc(), TII.get(TargetOpcode::IMPLICIT_DEF),
+                   ImpDefReg);
+
+      // Now, create the subregister insert from SrcReg.
+      unsigned InsertReg = MRI.createVirtualRegister(&AArch64::FPR128RegClass);
+      MachineInstr &InsMI =
+          *BuildMI(MBB, I, I.getDebugLoc(),
+                   TII.get(TargetOpcode::INSERT_SUBREG), InsertReg)
+               .addUse(ImpDefReg)
+               .addUse(SrcReg)
+               .addImm(AArch64::dsub);
+
+      constrainSelectedInstRegOperands(ImpDefMI, TII, TRI, RBI);
+      constrainSelectedInstRegOperands(InsMI, TII, TRI, RBI);
+
+      // Save the register so that we can copy from it after.
+      InsertRegs.push_back(InsertReg);
+    }
+  }
+
+  // Now that we've created any necessary subregister inserts, we can
+  // create the copies.
+  //
+  // Perform the first copy separately as a subregister copy.
+  unsigned CopyTo = I.getOperand(0).getReg();
+  MachineInstr &FirstCopy =
+      *BuildMI(MBB, I, I.getDebugLoc(), TII.get(TargetOpcode::COPY), CopyTo)
+           .addUse(InsertRegs[0], 0, ExtractSubReg);
+  constrainSelectedInstRegOperands(FirstCopy, TII, TRI, RBI);
+
+  // Now, perform the remaining copies as vector lane copies.
+  unsigned LaneIdx = 1;
+  for (unsigned InsReg : InsertRegs) {
+    unsigned CopyTo = I.getOperand(LaneIdx).getReg();
+    MachineInstr &CopyInst =
+        *BuildMI(MBB, I, I.getDebugLoc(), TII.get(CopyOpc), CopyTo)
+             .addUse(InsReg)
+             .addImm(LaneIdx);
+    constrainSelectedInstRegOperands(CopyInst, TII, TRI, RBI);
+    ++LaneIdx;
+  }
+
+  // Separately constrain the first copy's destination. Because of the
+  // limitation in constrainOperandRegClass, we can't guarantee that this will
+  // actually be constrained. So, do it ourselves using the second operand.
+  const TargetRegisterClass *RC =
+      MRI.getRegClassOrNull(I.getOperand(1).getReg());
+  if (!RC) {
+    LLVM_DEBUG(dbgs() << "Couldn't constrain copy destination.\n");
+    return false;
+  }
+
+  RBI.constrainGenericRegister(CopyTo, *RC, MRI);
+  I.eraseFromParent();
+  return true;
+}
+
 bool AArch64InstructionSelector::selectBuildVector(
     MachineInstr &I, MachineRegisterInfo &MRI) const {
   assert(I.getOpcode() == TargetOpcode::G_BUILD_VECTOR);
@@ -1598,7 +1921,7 @@ bool AArch64InstructionSelector::selectBuildVector(
   const LLT DstTy = MRI.getType(I.getOperand(0).getReg());
   const LLT EltTy = MRI.getType(I.getOperand(1).getReg());
   unsigned EltSize = EltTy.getSizeInBits();
-  if (EltSize < 32 || EltSize > 64)
+  if (EltSize < 16 || EltSize > 64)
     return false; // Don't support all element types yet.
   const RegisterBank &RB = *RBI.getRegBank(I.getOperand(1).getReg(), MRI, TRI);
   unsigned Opc;
@@ -1612,7 +1935,10 @@ bool AArch64InstructionSelector::selectBuildVector(
       SubregIdx = AArch64::dsub;
     }
   } else {
-    if (EltSize == 32) {
+    if (EltSize == 16) {
+      Opc = AArch64::INSvi16lane;
+      SubregIdx = AArch64::hsub;
+    } else if (EltSize == 32) {
       Opc = AArch64::INSvi32lane;
       SubregIdx = AArch64::ssub;
     } else {
@@ -1621,21 +1947,24 @@ bool AArch64InstructionSelector::selectBuildVector(
     }
   }
 
-  if (EltSize * DstTy.getNumElements() != 128)
-    return false; // Don't handle unpacked vectors yet.
-
   unsigned DstVec = 0;
-  const TargetRegisterClass *DstRC = getRegClassForTypeOnBank(
-      DstTy, RBI.getRegBank(AArch64::FPRRegBankID), RBI);
-  emitScalarToVector(DstVec, DstTy, DstRC, I.getOperand(1).getReg(),
-                     *I.getParent(), I.getIterator(), MRI);
-  for (unsigned i = 2, e = DstTy.getSizeInBits() / EltSize + 1; i < e; ++i) {
+
+  const TargetRegisterClass *DstRC = &AArch64::FPR128RegClass;
+  if (!emitScalarToVector(DstVec, DstTy, DstRC, I.getOperand(1).getReg(),
+                          *I.getParent(), I.getIterator(), MRI))
+    return false;
+
+  unsigned DstSize = DstTy.getSizeInBits();
+
+  // Keep track of the last MI we inserted. Later on, we might be able to save
+  // a copy using it.
+  MachineInstr *PrevMI = nullptr;
+  for (unsigned i = 2, e = DstSize / EltSize + 1; i < e; ++i) {
     unsigned InsDef;
-    // For the last insert re-use the dst reg of the G_BUILD_VECTOR.
-    if (i + 1 < e)
-      InsDef = MRI.createVirtualRegister(DstRC);
-    else
-      InsDef = I.getOperand(0).getReg();
+
+    // Note that if we don't do a subregister copy, we end up making one more
+    // of these than we need.
+    InsDef = MRI.createVirtualRegister(DstRC);
     unsigned LaneIdx = i - 1;
     if (RB.getID() == AArch64::FPRRegBankID) {
       unsigned ImpDef = MRI.createVirtualRegister(DstRC);
@@ -1660,6 +1989,7 @@ bool AArch64InstructionSelector::selectBuildVector(
       constrainSelectedInstRegOperands(InsSubMI, TII, TRI, RBI);
       constrainSelectedInstRegOperands(InsEltMI, TII, TRI, RBI);
       DstVec = InsDef;
+      PrevMI = &InsEltMI;
     } else {
       MachineInstr &InsMI =
           *BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opc))
@@ -1669,8 +1999,53 @@ bool AArch64InstructionSelector::selectBuildVector(
                .addUse(I.getOperand(i).getReg());
       constrainSelectedInstRegOperands(InsMI, TII, TRI, RBI);
       DstVec = InsDef;
+      PrevMI = &InsMI;
     }
   }
+
+  // If DstTy's size in bits is less than 128, then emit a subregister copy
+  // from DstVec to the last register we've defined.
+  if (DstSize < 128) {
+    unsigned SubReg = 0;
+
+    // Helper lambda to decide on a register class and subregister for the
+    // subregister copy.
+    auto GetRegInfoForCopy = [&SubReg,
+                              &DstSize]() -> const TargetRegisterClass * {
+      switch (DstSize) {
+      default:
+        LLVM_DEBUG(dbgs() << "Unknown destination size (" << DstSize << ")\n");
+        return nullptr;
+      case 32:
+        SubReg = AArch64::ssub;
+        return &AArch64::FPR32RegClass;
+      case 64:
+        SubReg = AArch64::dsub;
+        return &AArch64::FPR64RegClass;
+      }
+    };
+
+    const TargetRegisterClass *RC = GetRegInfoForCopy();
+    if (!RC)
+      return false;
+
+    unsigned Reg = MRI.createVirtualRegister(RC);
+    unsigned DstReg = I.getOperand(0).getReg();
+
+    BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(TargetOpcode::COPY),
+            DstReg)
+        .addUse(DstVec, 0, SubReg);
+    MachineOperand &RegOp = I.getOperand(1);
+    RegOp.setReg(Reg);
+    RBI.constrainGenericRegister(DstReg, *RC, MRI);
+  } else {
+    // We don't need a subregister copy. Save a copy by re-using the
+    // destination register on the final insert.
+    assert(PrevMI && "PrevMI was null?");
+    PrevMI->getOperand(0).setReg(I.getOperand(0).getReg());
+    constrainSelectedInstRegOperands(*PrevMI, TII, TRI, RBI);
+  }
+
   I.eraseFromParent();
   return true;
 }
diff --git a/lib/Target/AArch64/AArch64LegalizerInfo.cpp b/lib/Target/AArch64/AArch64LegalizerInfo.cpp
index 5d63f0c244ea32f2f57b5adabc4f0353b3f39258..34ce38896dacfd85e99c308a0afa15739aab8806 100644
--- a/lib/Target/AArch64/AArch64LegalizerInfo.cpp
+++ b/lib/Target/AArch64/AArch64LegalizerInfo.cpp
@@ -1,9 +1,8 @@
 //===- AArch64LegalizerInfo.cpp ----------------------------------*- C++ -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
@@ -24,6 +23,7 @@
 
 using namespace llvm;
 using namespace LegalizeActions;
+using namespace LegalizeMutations;
 using namespace LegalityPredicates;
 
 AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST) {
@@ -48,9 +48,21 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST) {
   const LLT v2s64 = LLT::vector(2, 64);
 
   getActionDefinitionsBuilder(G_IMPLICIT_DEF)
-      .legalFor({p0, s1, s8, s16, s32, s64, v2s64})
-      .clampScalar(0, s1, s64)
-      .widenScalarToNextPow2(0, 8);
+    .legalFor({p0, s1, s8, s16, s32, s64, v2s64})
+    .clampScalar(0, s1, s64)
+    .widenScalarToNextPow2(0, 8)
+    .fewerElementsIf(
+      [=](const LegalityQuery &Query) {
+        return Query.Types[0].isVector() &&
+          (Query.Types[0].getElementType() != s64 ||
+           Query.Types[0].getNumElements() != 2);
+      },
+      [=](const LegalityQuery &Query) {
+        LLT EltTy = Query.Types[0].getElementType();
+        if (EltTy == s64)
+          return std::make_pair(0, LLT::vector(2, 64));
+        return std::make_pair(0, EltTy);
+      });
 
   getActionDefinitionsBuilder(G_PHI)
       .legalFor({p0, s16, s32, s64})
@@ -62,7 +74,7 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST) {
       .clampScalar(0, s16, s64)
       .widenScalarToNextPow2(0);
 
-  getActionDefinitionsBuilder({G_ADD, G_SUB, G_MUL, G_AND, G_OR, G_XOR, G_SHL})
+  getActionDefinitionsBuilder({G_ADD, G_SUB, G_MUL, G_AND, G_OR, G_XOR})
       .legalFor({s32, s64, v2s32, v4s32, v2s64})
       .clampScalar(0, s32, s64)
       .widenScalarToNextPow2(0)
@@ -70,17 +82,34 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST) {
       .clampNumElements(0, v2s64, v2s64)
       .moreElementsToNextPow2(0);
 
+  getActionDefinitionsBuilder(G_SHL)
+    .legalFor({{s32, s32}, {s64, s64},
+               {v2s32, v2s32}, {v4s32, v4s32}, {v2s64, v2s64}})
+    .clampScalar(1, s32, s64)
+    .clampScalar(0, s32, s64)
+    .widenScalarToNextPow2(0)
+    .clampNumElements(0, v2s32, v4s32)
+    .clampNumElements(0, v2s64, v2s64)
+    .moreElementsToNextPow2(0)
+    .minScalarSameAs(1, 0);
+
   getActionDefinitionsBuilder(G_GEP)
       .legalFor({{p0, s64}})
       .clampScalar(1, s64, s64);
 
   getActionDefinitionsBuilder(G_PTR_MASK).legalFor({p0});
 
-  getActionDefinitionsBuilder({G_LSHR, G_ASHR, G_SDIV, G_UDIV})
+  getActionDefinitionsBuilder({G_SDIV, G_UDIV})
       .legalFor({s32, s64})
       .clampScalar(0, s32, s64)
       .widenScalarToNextPow2(0);
 
+  getActionDefinitionsBuilder({G_LSHR, G_ASHR})
+    .legalFor({{s32, s32}, {s64, s64}})
+    .clampScalar(1, s32, s64)
+    .clampScalar(0, s32, s64)
+    .minScalarSameAs(1, 0);
+
   getActionDefinitionsBuilder({G_SREM, G_UREM})
       .lowerFor({s1, s8, s16, s32, s64});
 
@@ -92,11 +121,38 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST) {
   getActionDefinitionsBuilder({G_UADDE, G_USUBE, G_SADDO, G_SSUBO})
       .legalFor({{s32, s1}, {s64, s1}});
 
-  getActionDefinitionsBuilder({G_FADD, G_FSUB, G_FMA, G_FMUL, G_FDIV})
-      .legalFor({s32, s64});
+  getActionDefinitionsBuilder({G_FADD, G_FSUB, G_FMA, G_FMUL, G_FDIV, G_FNEG})
+    .legalFor({s32, s64, v2s64, v4s32, v2s32});
 
   getActionDefinitionsBuilder({G_FREM, G_FPOW}).libcallFor({s32, s64});
 
+  getActionDefinitionsBuilder({G_FCEIL, G_FABS, G_FSQRT, G_FFLOOR})
+      // If we don't have full FP16 support, then scalarize the elements of
+      // vectors containing fp16 types.
+      .fewerElementsIf(
+          [=, &ST](const LegalityQuery &Query) {
+            const auto &Ty = Query.Types[0];
+            return Ty.isVector() && Ty.getElementType() == s16 &&
+                   !ST.hasFullFP16();
+          },
+          [=](const LegalityQuery &Query) { return std::make_pair(0, s16); })
+      // If we don't have full FP16 support, then widen s16 to s32 if we
+      // encounter it.
+      .widenScalarIf(
+          [=, &ST](const LegalityQuery &Query) {
+            return Query.Types[0] == s16 && !ST.hasFullFP16();
+          },
+          [=](const LegalityQuery &Query) { return std::make_pair(0, s32); })
+      .legalFor({s16, s32, s64, v2s32, v4s32, v2s64, v2s16, v4s16, v8s16});
+
+  getActionDefinitionsBuilder(
+      {G_FCOS, G_FSIN, G_FLOG10, G_FLOG, G_FLOG2, G_FEXP})
+      // We need a call for these, so we always need to scalarize.
+      .scalarize(0)
+      // Regardless of FP16 support, widen 16-bit elements to 32-bits.
+      .minScalar(0, s32)
+      .libcallFor({s32, s64, v2s32, v4s32, v2s64});
+
   getActionDefinitionsBuilder(G_INSERT)
       .unsupportedIf([=](const LegalityQuery &Query) {
         return Query.Types[0].getSizeInBits() <= Query.Types[1].getSizeInBits();
@@ -169,7 +225,7 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST) {
       .lowerIf([=](const LegalityQuery &Query) {
         return Query.Types[0].getSizeInBits() != Query.MMODescrs[0].SizeInBits;
       })
-      .clampNumElements(0, v2s32, v2s32)
+      .clampMaxNumElements(0, s32, 2)
       .clampMaxNumElements(0, s64, 1);
 
   getActionDefinitionsBuilder(G_STORE)
@@ -188,7 +244,7 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST) {
         return Query.Types[0].isScalar() &&
                Query.Types[0].getSizeInBits() != Query.MMODescrs[0].SizeInBits;
       })
-      .clampNumElements(0, v2s32, v2s32)
+      .clampMaxNumElements(0, s32, 2)
       .clampMaxNumElements(0, s64, 1);
 
   // Constants
@@ -218,20 +274,20 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST) {
 
   // FP conversions
   getActionDefinitionsBuilder(G_FPTRUNC).legalFor(
-      {{s16, s32}, {s16, s64}, {s32, s64}});
+      {{s16, s32}, {s16, s64}, {s32, s64}, {v4s16, v4s32}, {v2s32, v2s64}});
   getActionDefinitionsBuilder(G_FPEXT).legalFor(
-      {{s32, s16}, {s64, s16}, {s64, s32}});
+      {{s32, s16}, {s64, s16}, {s64, s32}, {v4s32, v4s16}, {v2s64, v2s32}});
 
   // Conversions
   getActionDefinitionsBuilder({G_FPTOSI, G_FPTOUI})
-      .legalForCartesianProduct({s32, s64})
+      .legalForCartesianProduct({s32, s64, v2s64, v4s32, v2s32})
       .clampScalar(0, s32, s64)
       .widenScalarToNextPow2(0)
       .clampScalar(1, s32, s64)
       .widenScalarToNextPow2(1);
 
   getActionDefinitionsBuilder({G_SITOFP, G_UITOFP})
-      .legalForCartesianProduct({s32, s64})
+      .legalForCartesianProduct({s32, s64, v2s64, v4s32, v2s32})
       .clampScalar(1, s32, s64)
       .widenScalarToNextPow2(1)
       .clampScalar(0, s32, s64)
@@ -313,11 +369,6 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST) {
       }
       return false;
     };
-    auto scalarize =
-        [](const LegalityQuery &Query, unsigned TypeIdx) {
-          const LLT &Ty = Query.Types[TypeIdx];
-          return std::make_pair(TypeIdx, Ty.getElementType());
-        };
 
     // FIXME: This rule is horrible, but specifies the same as what we had
     // before with the particularly strange definitions removed (e.g.
@@ -331,10 +382,10 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST) {
         // Break up vectors with weird elements into scalars
         .fewerElementsIf(
             [=](const LegalityQuery &Query) { return notValidElt(Query, 0); },
-            [=](const LegalityQuery &Query) { return scalarize(Query, 0); })
+            scalarize(0))
         .fewerElementsIf(
             [=](const LegalityQuery &Query) { return notValidElt(Query, 1); },
-            [=](const LegalityQuery &Query) { return scalarize(Query, 1); })
+            scalarize(1))
         // Clamp the big scalar to s8-s512 and make it either a power of 2, 192,
         // or 384.
         .clampScalar(BigTyIdx, s8, s512)
@@ -375,16 +426,8 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST) {
           return BigTy.getSizeInBits() % LitTy.getSizeInBits() == 0;
         })
         // Any vectors left are the wrong size. Scalarize them.
-        .fewerElementsIf([](const LegalityQuery &Query) { return true; },
-                         [](const LegalityQuery &Query) {
-                           return std::make_pair(
-                               0, Query.Types[0].getElementType());
-                         })
-        .fewerElementsIf([](const LegalityQuery &Query) { return true; },
-                         [](const LegalityQuery &Query) {
-                           return std::make_pair(
-                               1, Query.Types[1].getElementType());
-                         });
+      .scalarize(0)
+      .scalarize(1);
   }
 
   getActionDefinitionsBuilder(G_EXTRACT_VECTOR_ELT)
@@ -399,7 +442,11 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST) {
       });
 
   getActionDefinitionsBuilder(G_BUILD_VECTOR)
-      .legalFor({{v4s32, s32}, {v2s64, s64}})
+      .legalFor({{v4s16, s16},
+                 {v8s16, s16},
+                 {v2s32, s32},
+                 {v4s32, s32},
+                 {v2s64, s64}})
       .clampNumElements(0, v4s32, v4s32)
       .clampNumElements(0, v2s64, v2s64)
 
@@ -454,7 +501,7 @@ bool AArch64LegalizerInfo::legalizeVaArg(MachineInstr &MI,
     auto AlignMinus1 = MIRBuilder.buildConstant(IntPtrTy, Align - 1);
 
     unsigned ListTmp = MRI.createGenericVirtualRegister(PtrTy);
-    MIRBuilder.buildGEP(ListTmp, List, AlignMinus1->getOperand(0).getReg());
+    MIRBuilder.buildGEP(ListTmp, List, AlignMinus1.getReg(0));
 
     DstPtr = MRI.createGenericVirtualRegister(PtrTy);
     MIRBuilder.buildPtrMask(DstPtr, ListTmp, Log2_64(Align));
diff --git a/lib/Target/AArch64/AArch64LegalizerInfo.h b/lib/Target/AArch64/AArch64LegalizerInfo.h
index 77e8bdc7623c9ba2a1617631cfa9eeb9eca60d9e..c5979d8bbe5826b4b4842ad261230ed319eba5eb 100644
--- a/lib/Target/AArch64/AArch64LegalizerInfo.h
+++ b/lib/Target/AArch64/AArch64LegalizerInfo.h
@@ -1,9 +1,8 @@
 //===- AArch64LegalizerInfo --------------------------------------*- C++ -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp b/lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp
index aa732a99469cf3945fa52c0fd47821ea13bd9f30..b5fe5d6a62ef8d95397d5c987649618ef839eb20 100644
--- a/lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp
+++ b/lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp
@@ -1,9 +1,8 @@
 //===- AArch64LoadStoreOptimizer.cpp - AArch64 load/store opt. pass -------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AArch64/AArch64MCInstLower.cpp b/lib/Target/AArch64/AArch64MCInstLower.cpp
index 07295989ec8e29d888aa3f902cee13fa665fc02b..e7d4a2789a28fdf8db75a4c06252ae160c070402 100644
--- a/lib/Target/AArch64/AArch64MCInstLower.cpp
+++ b/lib/Target/AArch64/AArch64MCInstLower.cpp
@@ -1,9 +1,8 @@
 //==-- AArch64MCInstLower.cpp - Convert AArch64 MachineInstr to an MCInst --==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -189,20 +188,51 @@ MCOperand AArch64MCInstLower::lowerSymbolOperandELF(const MachineOperand &MO,
 
 MCOperand AArch64MCInstLower::lowerSymbolOperandCOFF(const MachineOperand &MO,
                                                      MCSymbol *Sym) const {
-  AArch64MCExpr::VariantKind RefKind = AArch64MCExpr::VK_NONE;
+  uint32_t RefFlags = 0;
+
   if (MO.getTargetFlags() & AArch64II::MO_TLS) {
     if ((MO.getTargetFlags() & AArch64II::MO_FRAGMENT) == AArch64II::MO_PAGEOFF)
-      RefKind = AArch64MCExpr::VK_SECREL_LO12;
+      RefFlags |= AArch64MCExpr::VK_SECREL_LO12;
     else if ((MO.getTargetFlags() & AArch64II::MO_FRAGMENT) ==
              AArch64II::MO_HI12)
-      RefKind = AArch64MCExpr::VK_SECREL_HI12;
+      RefFlags |= AArch64MCExpr::VK_SECREL_HI12;
+
+  } else if (MO.getTargetFlags() & AArch64II::MO_S) {
+    RefFlags |= AArch64MCExpr::VK_SABS;
+  } else {
+    RefFlags |= AArch64MCExpr::VK_ABS;
   }
+
+  if ((MO.getTargetFlags() & AArch64II::MO_FRAGMENT) == AArch64II::MO_G3)
+    RefFlags |= AArch64MCExpr::VK_G3;
+  else if ((MO.getTargetFlags() & AArch64II::MO_FRAGMENT) == AArch64II::MO_G2)
+    RefFlags |= AArch64MCExpr::VK_G2;
+  else if ((MO.getTargetFlags() & AArch64II::MO_FRAGMENT) == AArch64II::MO_G1)
+    RefFlags |= AArch64MCExpr::VK_G1;
+  else if ((MO.getTargetFlags() & AArch64II::MO_FRAGMENT) == AArch64II::MO_G0)
+    RefFlags |= AArch64MCExpr::VK_G0;
+
+  // FIXME: Currently we only set VK_NC for MO_G3/MO_G2/MO_G1/MO_G0. This is
+  // because setting VK_NC for others would mean setting their respective
+  // RefFlags correctly.  We should do this in a separate patch.
+  if (MO.getTargetFlags() & AArch64II::MO_NC) {
+    auto MOFrag = (MO.getTargetFlags() & AArch64II::MO_FRAGMENT);
+    if (MOFrag == AArch64II::MO_G3 || MOFrag == AArch64II::MO_G2 ||
+        MOFrag == AArch64II::MO_G1 || MOFrag == AArch64II::MO_G0)
+      RefFlags |= AArch64MCExpr::VK_NC;
+  }
+
   const MCExpr *Expr =
       MCSymbolRefExpr::create(Sym, MCSymbolRefExpr::VK_None, Ctx);
   if (!MO.isJTI() && MO.getOffset())
     Expr = MCBinaryExpr::createAdd(
         Expr, MCConstantExpr::create(MO.getOffset(), Ctx), Ctx);
+
+  auto RefKind = static_cast<AArch64MCExpr::VariantKind>(RefFlags);
+  assert(RefKind != AArch64MCExpr::VK_INVALID &&
+         "Invalid relocation requested");
   Expr = AArch64MCExpr::create(Expr, RefKind, Ctx);
+
   return MCOperand::createExpr(Expr);
 }
 
diff --git a/lib/Target/AArch64/AArch64MCInstLower.h b/lib/Target/AArch64/AArch64MCInstLower.h
index aa30fe1fa7078f8484acd8c9cd50f88764553d48..8f3148a98410b8b9529032b3541290ce865b8cc9 100644
--- a/lib/Target/AArch64/AArch64MCInstLower.h
+++ b/lib/Target/AArch64/AArch64MCInstLower.h
@@ -1,9 +1,8 @@
 //===-- AArch64MCInstLower.h - Lower MachineInstr to MCInst ---------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/AArch64/AArch64MachineFunctionInfo.h b/lib/Target/AArch64/AArch64MachineFunctionInfo.h
index 5183e7d3c0d0974f4d680e48e06eeb4128191082..5ab511ee88a54e4605184e5d37dd997cc89aa219 100644
--- a/lib/Target/AArch64/AArch64MachineFunctionInfo.h
+++ b/lib/Target/AArch64/AArch64MachineFunctionInfo.h
@@ -1,9 +1,8 @@
 //=- AArch64MachineFunctionInfo.h - AArch64 machine function info -*- C++ -*-=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AArch64/AArch64MacroFusion.cpp b/lib/Target/AArch64/AArch64MacroFusion.cpp
index fb8a339dc4d8eb478fce6ae7c6d4389dcb43d7ef..9a2103579a6ab355d41faeac70a81a95d9b98e8f 100644
--- a/lib/Target/AArch64/AArch64MacroFusion.cpp
+++ b/lib/Target/AArch64/AArch64MacroFusion.cpp
@@ -1,9 +1,8 @@
 //===- AArch64MacroFusion.cpp - AArch64 Macro Fusion ----------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -270,7 +269,107 @@ static bool isCCSelectPair(const MachineInstr *FirstMI,
   return false;
 }
 
-/// Check if the instr pair, FirstMI and SecondMI, should be fused
+// Arithmetic and logic.
+static bool isArithmeticLogicPair(const MachineInstr *FirstMI,
+                                  const MachineInstr &SecondMI) {
+  if (AArch64InstrInfo::hasShiftedReg(SecondMI))
+    return false;
+
+  switch (SecondMI.getOpcode()) {
+  // Arithmetic
+  case AArch64::ADDWrr:
+  case AArch64::ADDXrr:
+  case AArch64::SUBWrr:
+  case AArch64::SUBXrr:
+  case AArch64::ADDWrs:
+  case AArch64::ADDXrs:
+  case AArch64::SUBWrs:
+  case AArch64::SUBXrs:
+  // Logic
+  case AArch64::ANDWrr:
+  case AArch64::ANDXrr:
+  case AArch64::BICWrr:
+  case AArch64::BICXrr:
+  case AArch64::EONWrr:
+  case AArch64::EONXrr:
+  case AArch64::EORWrr:
+  case AArch64::EORXrr:
+  case AArch64::ORNWrr:
+  case AArch64::ORNXrr:
+  case AArch64::ORRWrr:
+  case AArch64::ORRXrr:
+  case AArch64::ANDWrs:
+  case AArch64::ANDXrs:
+  case AArch64::BICWrs:
+  case AArch64::BICXrs:
+  case AArch64::EONWrs:
+  case AArch64::EONXrs:
+  case AArch64::EORWrs:
+  case AArch64::EORXrs:
+  case AArch64::ORNWrs:
+  case AArch64::ORNXrs:
+  case AArch64::ORRWrs:
+  case AArch64::ORRXrs:
+    // Assume the 1st instr to be a wildcard if it is unspecified.
+    if (FirstMI == nullptr)
+      return true;
+
+    // Arithmetic
+    switch (FirstMI->getOpcode()) {
+    case AArch64::ADDWrr:
+    case AArch64::ADDXrr:
+    case AArch64::ADDSWrr:
+    case AArch64::ADDSXrr:
+    case AArch64::SUBWrr:
+    case AArch64::SUBXrr:
+    case AArch64::SUBSWrr:
+    case AArch64::SUBSXrr:
+      return true;
+    case AArch64::ADDWrs:
+    case AArch64::ADDXrs:
+    case AArch64::ADDSWrs:
+    case AArch64::ADDSXrs:
+    case AArch64::SUBWrs:
+    case AArch64::SUBXrs:
+    case AArch64::SUBSWrs:
+    case AArch64::SUBSXrs:
+      return !AArch64InstrInfo::hasShiftedReg(*FirstMI);
+    }
+    break;
+
+  // Arithmetic, setting flags.
+  case AArch64::ADDSWrr:
+  case AArch64::ADDSXrr:
+  case AArch64::SUBSWrr:
+  case AArch64::SUBSXrr:
+  case AArch64::ADDSWrs:
+  case AArch64::ADDSXrs:
+  case AArch64::SUBSWrs:
+  case AArch64::SUBSXrs:
+    // Assume the 1st instr to be a wildcard if it is unspecified.
+    if (FirstMI == nullptr)
+      return true;
+
+    // Arithmetic, not setting flags.
+    switch (FirstMI->getOpcode()) {
+    case AArch64::ADDWrr:
+    case AArch64::ADDXrr:
+    case AArch64::SUBWrr:
+    case AArch64::SUBXrr:
+      return true;
+    case AArch64::ADDWrs:
+    case AArch64::ADDXrs:
+    case AArch64::SUBWrs:
+    case AArch64::SUBXrs:
+      return !AArch64InstrInfo::hasShiftedReg(*FirstMI);
+    }
+    break;
+  }
+
+  return false;
+}
+
+/// \brief Check if the instr pair, FirstMI and SecondMI, should be fused
 /// together. Given SecondMI, when FirstMI is unspecified, then check if
 /// SecondMI may be part of a fused pair at all.
 static bool shouldScheduleAdjacent(const TargetInstrInfo &TII,
@@ -295,6 +394,8 @@ static bool shouldScheduleAdjacent(const TargetInstrInfo &TII,
     return true;
   if (ST.hasFuseCCSelect() && isCCSelectPair(FirstMI, SecondMI))
     return true;
+  if (ST.hasFuseArithmeticLogic() && isArithmeticLogicPair(FirstMI, SecondMI))
+    return true;
 
   return false;
 }
diff --git a/lib/Target/AArch64/AArch64MacroFusion.h b/lib/Target/AArch64/AArch64MacroFusion.h
index 32d90d4c40d6fa8a56497b33a78e41a22c34bb21..4e7ccbe4baab01e83e3614cef2f4543016f94e88 100644
--- a/lib/Target/AArch64/AArch64MacroFusion.h
+++ b/lib/Target/AArch64/AArch64MacroFusion.h
@@ -1,9 +1,8 @@
 //===- AArch64MacroFusion.h - AArch64 Macro Fusion ------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AArch64/AArch64PBQPRegAlloc.cpp b/lib/Target/AArch64/AArch64PBQPRegAlloc.cpp
index ccf646575296beb9f0d6f50dfd727f8b8bc571d7..aff861aae6be3ec16a89fa9ac92f962fdbf00c84 100644
--- a/lib/Target/AArch64/AArch64PBQPRegAlloc.cpp
+++ b/lib/Target/AArch64/AArch64PBQPRegAlloc.cpp
@@ -1,9 +1,8 @@
 //===-- AArch64PBQPRegAlloc.cpp - AArch64 specific PBQP constraints -------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 // This file contains the AArch64 / Cortex-A57 specific register allocation
diff --git a/lib/Target/AArch64/AArch64PBQPRegAlloc.h b/lib/Target/AArch64/AArch64PBQPRegAlloc.h
index b99c1d1d6b3e2d4e6b4bd148c86832ddc6460cfb..5ea91b4a1967b9cb321c60dee5c85e7196a1f353 100644
--- a/lib/Target/AArch64/AArch64PBQPRegAlloc.h
+++ b/lib/Target/AArch64/AArch64PBQPRegAlloc.h
@@ -1,9 +1,8 @@
 //==- AArch64PBQPRegAlloc.h - AArch64 specific PBQP constraints --*- C++ -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/AArch64/AArch64PerfectShuffle.h b/lib/Target/AArch64/AArch64PerfectShuffle.h
index 9e9eec48c555a25bfacf4bf064e303096b755697..f443cd03935ce9c457767866622ec6bbf3f6913d 100644
--- a/lib/Target/AArch64/AArch64PerfectShuffle.h
+++ b/lib/Target/AArch64/AArch64PerfectShuffle.h
@@ -1,9 +1,8 @@
 //===-- AArch64PerfectShuffle.h - AdvSIMD Perfect Shuffle Table -----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AArch64/AArch64PfmCounters.td b/lib/Target/AArch64/AArch64PfmCounters.td
index 16ba3e4282a0d0fd9adb232a634b95b274b77da6..b1d1664e3f1b1aad5f8029e559a9522ba6027bb3 100644
--- a/lib/Target/AArch64/AArch64PfmCounters.td
+++ b/lib/Target/AArch64/AArch64PfmCounters.td
@@ -1,9 +1,8 @@
 //===-- AArch64PfmCounters.td - AArch64 Hardware Counters --*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AArch64/AArch64PreLegalizerCombiner.cpp b/lib/Target/AArch64/AArch64PreLegalizerCombiner.cpp
index 5022b6221ab99029caa1e54089da7928480ce2ff..cffe7dbfe6632c3d7f7243ea3b06655b45aa05e4 100644
--- a/lib/Target/AArch64/AArch64PreLegalizerCombiner.cpp
+++ b/lib/Target/AArch64/AArch64PreLegalizerCombiner.cpp
@@ -1,9 +1,8 @@
 //=== lib/CodeGen/GlobalISel/AArch64PreLegalizerCombiner.cpp --------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -88,7 +87,7 @@ bool AArch64PreLegalizerCombiner::runOnMachineFunction(MachineFunction &MF) {
   auto *TPC = &getAnalysis<TargetPassConfig>();
   AArch64PreLegalizerCombinerInfo PCInfo;
   Combiner C(PCInfo, TPC);
-  return C.combineMachineInstrs(MF);
+  return C.combineMachineInstrs(MF, /*CSEInfo*/ nullptr);
 }
 
 char AArch64PreLegalizerCombiner::ID = 0;
diff --git a/lib/Target/AArch64/AArch64PromoteConstant.cpp b/lib/Target/AArch64/AArch64PromoteConstant.cpp
index 01d8a35bbc238cc2dacf54e8a3d7971b992bf7b9..a594ecb71fc9b10b15cbcf934914bb09ea649a68 100644
--- a/lib/Target/AArch64/AArch64PromoteConstant.cpp
+++ b/lib/Target/AArch64/AArch64PromoteConstant.cpp
@@ -1,9 +1,8 @@
 //==- AArch64PromoteConstant.cpp - Promote constant to global for AArch64 --==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -494,7 +493,8 @@ void AArch64PromoteConstant::insertDefinitions(Function &F,
   for (const auto &IPI : InsertPts) {
     // Create the load of the global variable.
     IRBuilder<> Builder(IPI.first);
-    LoadInst *LoadedCst = Builder.CreateLoad(&PromotedGV);
+    LoadInst *LoadedCst =
+        Builder.CreateLoad(PromotedGV.getValueType(), &PromotedGV);
     LLVM_DEBUG(dbgs() << "**********\n");
     LLVM_DEBUG(dbgs() << "New def: ");
     LLVM_DEBUG(LoadedCst->print(dbgs()));
diff --git a/lib/Target/AArch64/AArch64RedundantCopyElimination.cpp b/lib/Target/AArch64/AArch64RedundantCopyElimination.cpp
index fcb0b36a9f6de9674d88c9615836fda9101e4bf3..a6af09cd17cdb1396ed3b5cdb5336643903ee6a8 100644
--- a/lib/Target/AArch64/AArch64RedundantCopyElimination.cpp
+++ b/lib/Target/AArch64/AArch64RedundantCopyElimination.cpp
@@ -1,9 +1,8 @@
 //=- AArch64RedundantCopyElimination.cpp - Remove useless copy for AArch64 -=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 // This pass removes unnecessary copies/moves in BBs based on a dominating
 // condition.
diff --git a/lib/Target/AArch64/AArch64RegisterBankInfo.cpp b/lib/Target/AArch64/AArch64RegisterBankInfo.cpp
index c497669f937f821eee967af3d5bae6fb22706643..08cf6859f05d4a94fbfd62d55d100e94854ecf6c 100644
--- a/lib/Target/AArch64/AArch64RegisterBankInfo.cpp
+++ b/lib/Target/AArch64/AArch64RegisterBankInfo.cpp
@@ -1,9 +1,8 @@
 //===- AArch64RegisterBankInfo.cpp ----------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
@@ -243,12 +242,17 @@ const RegisterBank &AArch64RegisterBankInfo::getRegBankFromRegClass(
   case AArch64::GPR32RegClassID:
   case AArch64::GPR32spRegClassID:
   case AArch64::GPR32sponlyRegClassID:
+  case AArch64::GPR32argRegClassID:
   case AArch64::GPR32allRegClassID:
   case AArch64::GPR64commonRegClassID:
   case AArch64::GPR64RegClassID:
   case AArch64::GPR64spRegClassID:
   case AArch64::GPR64sponlyRegClassID:
+  case AArch64::GPR64argRegClassID:
   case AArch64::GPR64allRegClassID:
+  case AArch64::GPR64noipRegClassID:
+  case AArch64::GPR64common_and_GPR64noipRegClassID:
+  case AArch64::GPR64noip_and_tcGPR64RegClassID:
   case AArch64::tcGPR64RegClassID:
   case AArch64::WSeqPairsClassRegClassID:
   case AArch64::XSeqPairsClassRegClassID:
@@ -389,6 +393,17 @@ static bool isPreISelGenericFloatingPointOpcode(unsigned Opc) {
   case TargetOpcode::G_FCONSTANT:
   case TargetOpcode::G_FPEXT:
   case TargetOpcode::G_FPTRUNC:
+  case TargetOpcode::G_FCEIL:
+  case TargetOpcode::G_FFLOOR:
+  case TargetOpcode::G_FNEG:
+  case TargetOpcode::G_FCOS:
+  case TargetOpcode::G_FSIN:
+  case TargetOpcode::G_FLOG10:
+  case TargetOpcode::G_FLOG:
+  case TargetOpcode::G_FLOG2:
+  case TargetOpcode::G_FSQRT:
+  case TargetOpcode::G_FABS:
+  case TargetOpcode::G_FEXP:
     return true;
   }
   return false;
@@ -562,10 +577,14 @@ AArch64RegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
   switch (Opc) {
   case TargetOpcode::G_SITOFP:
   case TargetOpcode::G_UITOFP:
+    if (MRI.getType(MI.getOperand(0).getReg()).isVector())
+      break;
     OpRegBankIdx = {PMI_FirstFPR, PMI_FirstGPR};
     break;
   case TargetOpcode::G_FPTOSI:
   case TargetOpcode::G_FPTOUI:
+    if (MRI.getType(MI.getOperand(0).getReg()).isVector())
+      break;
     OpRegBankIdx = {PMI_FirstGPR, PMI_FirstFPR};
     break;
   case TargetOpcode::G_FCMP:
@@ -632,6 +651,66 @@ AArch64RegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
         OpRegBankIdx[0] = PMI_FirstFPR;
       break;
     }
+    break;
+  case TargetOpcode::G_UNMERGE_VALUES: {
+    // If the first operand belongs to a FPR register bank, then make sure that
+    // we preserve that.
+    if (OpRegBankIdx[0] != PMI_FirstGPR)
+      break;
+
+    // Helper lambda that returns true if MI has floating point constraints.
+    auto HasFPConstraints = [&TRI, &MRI, this](MachineInstr &MI) {
+      unsigned Op = MI.getOpcode();
+
+      // Do we have an explicit floating point instruction?
+      if (isPreISelGenericFloatingPointOpcode(Op))
+        return true;
+
+      // No. Check if we have a copy-like instruction. If we do, then we could
+      // still be fed by floating point instructions.
+      if (Op != TargetOpcode::COPY && !MI.isPHI())
+        return false;
+
+      // MI is copy-like. Return true if it's using an FPR.
+      return getRegBank(MI.getOperand(0).getReg(), MRI, TRI) ==
+             &AArch64::FPRRegBank;
+    };
+
+    LLT SrcTy = MRI.getType(MI.getOperand(MI.getNumOperands()-1).getReg());
+    // UNMERGE into scalars from a vector should always use FPR.
+    // Likewise if any of the uses are FP instructions.
+    if (SrcTy.isVector() ||
+        any_of(MRI.use_instructions(MI.getOperand(0).getReg()),
+               [&](MachineInstr &MI) { return HasFPConstraints(MI); })) {
+      // Set the register bank of every operand to FPR.
+      for (unsigned Idx = 0, NumOperands = MI.getNumOperands();
+           Idx < NumOperands; ++Idx)
+        OpRegBankIdx[Idx] = PMI_FirstFPR;
+    }
+    break;
+  }
+
+  case TargetOpcode::G_BUILD_VECTOR:
+    // If the first source operand belongs to a FPR register bank, then make
+    // sure that we preserve that.
+    if (OpRegBankIdx[1] != PMI_FirstGPR)
+      break;
+    unsigned VReg = MI.getOperand(1).getReg();
+    if (!VReg)
+      break;
+
+    // Get the instruction that defined the source operand reg, and check if
+    // it's a floating point operation.
+    MachineInstr *DefMI = MRI.getVRegDef(VReg);
+    unsigned DefOpc = DefMI->getOpcode();
+    if (isPreISelGenericFloatingPointOpcode(DefOpc)) {
+      // Have a floating point op.
+      // Make sure every operand gets mapped to a FPR register class.
+      unsigned NumOperands = MI.getNumOperands();
+      for (unsigned Idx = 0; Idx < NumOperands; ++Idx)
+        OpRegBankIdx[Idx] = PMI_FirstFPR;
+    }
+    break;
   }
 
   // Finally construct the computed mapping.
diff --git a/lib/Target/AArch64/AArch64RegisterBankInfo.h b/lib/Target/AArch64/AArch64RegisterBankInfo.h
index 008221dbef58a9e981007b3101a0addc3cc19842..98a51c339360ed71a6ebb8dbeb1af3a5ddcb4dba 100644
--- a/lib/Target/AArch64/AArch64RegisterBankInfo.h
+++ b/lib/Target/AArch64/AArch64RegisterBankInfo.h
@@ -1,9 +1,8 @@
 //===- AArch64RegisterBankInfo -----------------------------------*- C++ -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/lib/Target/AArch64/AArch64RegisterBanks.td b/lib/Target/AArch64/AArch64RegisterBanks.td
index eee584708f69ab6e6aceb2df2ee34b526c76e803..7bbd992890d189e97fdaf2aca3e431e3b77b5f46 100644
--- a/lib/Target/AArch64/AArch64RegisterBanks.td
+++ b/lib/Target/AArch64/AArch64RegisterBanks.td
@@ -1,9 +1,8 @@
 //=- AArch64RegisterBank.td - Describe the AArch64 Banks -----*- tablegen -*-=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AArch64/AArch64RegisterInfo.cpp b/lib/Target/AArch64/AArch64RegisterInfo.cpp
index 96ae45ae3d0dcd7d074fc4294ee823c3ed8a7d21..54d43431620c4cdfbeb594b1f323c16ed01f22c1 100644
--- a/lib/Target/AArch64/AArch64RegisterInfo.cpp
+++ b/lib/Target/AArch64/AArch64RegisterInfo.cpp
@@ -1,9 +1,8 @@
 //===- AArch64RegisterInfo.cpp - AArch64 Register Information -------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -217,11 +216,8 @@ bool AArch64RegisterInfo::isReservedReg(const MachineFunction &MF,
 }
 
 bool AArch64RegisterInfo::isAnyArgRegReserved(const MachineFunction &MF) const {
-  // FIXME: Get the list of argument registers from TableGen.
-  static const MCPhysReg GPRArgRegs[] = { AArch64::X0, AArch64::X1, AArch64::X2,
-                                          AArch64::X3, AArch64::X4, AArch64::X5,
-                                          AArch64::X6, AArch64::X7 };
-  return std::any_of(std::begin(GPRArgRegs), std::end(GPRArgRegs),
+  return std::any_of(std::begin(*AArch64::GPR64argRegClass.MC),
+                     std::end(*AArch64::GPR64argRegClass.MC),
                      [this, &MF](MCPhysReg r){return isReservedReg(MF, r);});
 }
 
@@ -464,8 +460,16 @@ void AArch64RegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator II,
     return;
   }
 
+  if (MI.getOpcode() == TargetOpcode::LOCAL_ESCAPE) {
+    MachineOperand &FI = MI.getOperand(FIOperandNum);
+    Offset = TFI->getNonLocalFrameIndexReference(MF, FrameIndex);
+    FI.ChangeToImmediate(Offset);
+    return;
+  }
+
   // Modify MI as necessary to handle as much of 'Offset' as possible
-  Offset = TFI->resolveFrameIndexReference(MF, FrameIndex, FrameReg);
+  Offset = TFI->getFrameIndexReference(MF, FrameIndex, FrameReg);
+
   if (rewriteAArch64FrameIndex(MI, FIOperandNum, FrameReg, Offset, TII))
     return;
 
@@ -519,3 +523,13 @@ unsigned AArch64RegisterInfo::getRegPressureLimit(const TargetRegisterClass *RC,
     return 16;
   }
 }
+
+unsigned AArch64RegisterInfo::getLocalAddressRegister(
+  const MachineFunction &MF) const {
+  const auto &MFI = MF.getFrameInfo();
+  if (!MF.hasEHFunclets() && !MFI.hasVarSizedObjects())
+    return AArch64::SP;
+  else if (needsStackRealignment(MF))
+    return getBaseRegister();
+  return getFrameRegister(MF);
+}
diff --git a/lib/Target/AArch64/AArch64RegisterInfo.h b/lib/Target/AArch64/AArch64RegisterInfo.h
index c4153228a7c0467fd3ede037bfd0208c1543d0f3..3d83fcefef051f59c54ca93972d1d41a5d4167ef 100644
--- a/lib/Target/AArch64/AArch64RegisterInfo.h
+++ b/lib/Target/AArch64/AArch64RegisterInfo.h
@@ -1,9 +1,8 @@
 //==- AArch64RegisterInfo.h - AArch64 Register Information Impl --*- C++ -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -122,6 +121,8 @@ public:
   bool trackLivenessAfterRegAlloc(const MachineFunction&) const override {
     return true;
   }
+
+  unsigned getLocalAddressRegister(const MachineFunction &MF) const;
 };
 
 } // end namespace llvm
diff --git a/lib/Target/AArch64/AArch64RegisterInfo.td b/lib/Target/AArch64/AArch64RegisterInfo.td
index d3710cea0687ec53fac2d72382ce8b9eefe041ed..61fc0795c2428bd0f4ce6768a626ad3433accb7e 100644
--- a/lib/Target/AArch64/AArch64RegisterInfo.td
+++ b/lib/Target/AArch64/AArch64RegisterInfo.td
@@ -1,9 +1,8 @@
 //=- AArch64RegisterInfo.td - Describe the AArch64 Registers -*- tablegen -*-=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -188,6 +187,10 @@ def GPR64z : RegisterOperand<GPR64> {
   let GIZeroRegister = XZR;
 }
 
+// GPR argument registers.
+def GPR32arg : RegisterClass<"AArch64", [i32], 32, (sequence "W%u", 0, 7)>;
+def GPR64arg : RegisterClass<"AArch64", [i64], 64, (sequence "X%u", 0, 7)>;
+
 // GPR register classes which include WZR/XZR AND SP/WSP. This is not a
 // constraint used by any instructions, it is used as a common super-class.
 def GPR32all : RegisterClass<"AArch64", [i32], 32, (add GPR32common, WZR, WSP)>;
@@ -206,6 +209,11 @@ def tcGPR64 : RegisterClass<"AArch64", [i64], 64, (sub GPR64common, X19, X20, X2
 // BTI-protected function.
 def rtcGPR64 : RegisterClass<"AArch64", [i64], 64, (add X16, X17)>;
 
+// Register set that excludes registers that are reserved for procedure calls.
+// This is used for pseudo-instructions that are actually implemented using a
+// procedure call.
+def GPR64noip : RegisterClass<"AArch64", [i64], 64, (sub GPR64, X16, X17, LR)>;
+
 // GPR register classes for post increment amount of vector load/store that
 // has alternate printing when Rm=31 and prints a constant immediate value
 // equal to the total number of bytes transferred.
@@ -649,10 +657,12 @@ def FPR128Op : RegisterOperand<FPR128, "printOperand"> {
 // ARMv8.1a atomic CASP register operands
 
 
-def WSeqPairs : RegisterTuples<[sube32, subo32], 
-                               [(rotl GPR32, 0), (rotl GPR32, 1)]>;
-def XSeqPairs : RegisterTuples<[sube64, subo64], 
-                               [(rotl GPR64, 0), (rotl GPR64, 1)]>;
+def WSeqPairs : RegisterTuples<[sube32, subo32],
+                               [(decimate (rotl GPR32, 0), 2),
+                                (decimate (rotl GPR32, 1), 2)]>;
+def XSeqPairs : RegisterTuples<[sube64, subo64],
+                               [(decimate (rotl GPR64, 0), 2),
+                                (decimate (rotl GPR64, 1), 2)]>;
 
 def WSeqPairsClass   : RegisterClass<"AArch64", [untyped], 32, 
                                      (add WSeqPairs)>{
diff --git a/lib/Target/AArch64/AArch64SIMDInstrOpt.cpp b/lib/Target/AArch64/AArch64SIMDInstrOpt.cpp
index af555f6d2266b774a9614b5500ba70df9ac89468..854670079e40d1dd1799b80c395dac907aefc1f8 100644
--- a/lib/Target/AArch64/AArch64SIMDInstrOpt.cpp
+++ b/lib/Target/AArch64/AArch64SIMDInstrOpt.cpp
@@ -1,8 +1,7 @@
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AArch64/AArch64SVEInstrInfo.td b/lib/Target/AArch64/AArch64SVEInstrInfo.td
index 0fde68011e8614327c5711917a93955a384acae9..655caa3deb53f03424909d6ddf95729da2321328 100644
--- a/lib/Target/AArch64/AArch64SVEInstrInfo.td
+++ b/lib/Target/AArch64/AArch64SVEInstrInfo.td
@@ -1,9 +1,8 @@
 //=- AArch64SVEInstrInfo.td -  AArch64 SVE Instructions -*- tablegen -*-----=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AArch64/AArch64SchedA53.td b/lib/Target/AArch64/AArch64SchedA53.td
index f253a4f3e25a23cbfa69bb4b6a59c6a982d6fb0d..3c2abbc33cfc826fac8a8ca903532b5cdf22d9d5 100644
--- a/lib/Target/AArch64/AArch64SchedA53.td
+++ b/lib/Target/AArch64/AArch64SchedA53.td
@@ -1,9 +1,8 @@
 //==- AArch64SchedA53.td - Cortex-A53 Scheduling Definitions -*- tablegen -*-=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AArch64/AArch64SchedA57.td b/lib/Target/AArch64/AArch64SchedA57.td
index ade03f23f8c794c06fd8f6bcef906c073d730fbd..b10f80b90459dcb27bec9ef430ab54cb241e9281 100644
--- a/lib/Target/AArch64/AArch64SchedA57.td
+++ b/lib/Target/AArch64/AArch64SchedA57.td
@@ -1,9 +1,8 @@
 //=- AArch64SchedA57.td - ARM Cortex-A57 Scheduling Defs -----*- tablegen -*-=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AArch64/AArch64SchedA57WriteRes.td b/lib/Target/AArch64/AArch64SchedA57WriteRes.td
index 55005e1d9ed1e5f10bfbb108b344ba83f36417b1..987ed3c4ebfb3ddec470d2af079b9ba5d3182172 100644
--- a/lib/Target/AArch64/AArch64SchedA57WriteRes.td
+++ b/lib/Target/AArch64/AArch64SchedA57WriteRes.td
@@ -1,9 +1,8 @@
 //=- AArch64SchedA57WriteRes.td - ARM Cortex-A57 Write Res ---*- tablegen -*-=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AArch64/AArch64SchedCyclone.td b/lib/Target/AArch64/AArch64SchedCyclone.td
index 7a474ba8ef9bca378103e9bcd9503fd9d738c86e..9ee247a4b753d82652fb587b26022d7d809b4690 100644
--- a/lib/Target/AArch64/AArch64SchedCyclone.td
+++ b/lib/Target/AArch64/AArch64SchedCyclone.td
@@ -1,9 +1,8 @@
 //=- AArch64SchedCyclone.td - Cyclone Scheduling Definitions -*- tablegen -*-=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AArch64/AArch64SchedExynosM1.td b/lib/Target/AArch64/AArch64SchedExynosM1.td
index 7f1c2d4c7648281cc6bfb308608c68ce15476540..afc8629fba0a3352f3b634d2ec7666ec61e88536 100644
--- a/lib/Target/AArch64/AArch64SchedExynosM1.td
+++ b/lib/Target/AArch64/AArch64SchedExynosM1.td
@@ -1,9 +1,8 @@
 //=- AArch64SchedExynosM1.td - Samsung Exynos M1 Sched Defs --*- tablegen -*-=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -78,9 +77,8 @@ def M1WriteAC : SchedWriteRes<[M1UnitALU,
 def M1WriteAD : SchedWriteRes<[M1UnitALU,
                                M1UnitC]>   { let Latency = 2;
                                              let NumMicroOps = 2; }
-def M1WriteAX : SchedWriteVariant<[SchedVar<ExynosExtPred, [M1WriteA1]>,
-                                   SchedVar<NoSchedPred,   [M1WriteAA]>]>;
-def M1WriteAY : SchedWriteVariant<[SchedVar<ExynosShiftPred, [M1WriteA1]>,
+def M1WriteAX : SchedWriteVariant<[SchedVar<ExynosArithPred, [M1WriteA1]>,
+                                   SchedVar<ExynosLogicPred, [M1WriteA1]>,
                                    SchedVar<NoSchedPred,     [M1WriteAA]>]>;
 def M1WriteC1 : SchedWriteRes<[M1UnitC]>   { let Latency = 1; }
 def M1WriteC2 : SchedWriteRes<[M1UnitC]>   { let Latency = 2; }
@@ -430,7 +428,7 @@ def : InstRW<[M1WriteAD], (instregex "^TBN?Z[WX]")>;
 
 // Arithmetic and logical integer instructions.
 def : InstRW<[M1WriteAX], (instregex ".+rx(64)?$")>;
-def : InstRW<[M1WriteAY], (instregex ".+rs$")>;
+def : InstRW<[M1WriteAX], (instregex ".+rs$")>;
 
 // Move instructions.
 def : InstRW<[M1WriteCOPY], (instrs COPY)>;
diff --git a/lib/Target/AArch64/AArch64SchedExynosM3.td b/lib/Target/AArch64/AArch64SchedExynosM3.td
index fd19ff84f1e010d40b16fdc6917d003bc3fb985b..b1cac9791a7543f8e701644aa901b09ddc4c8d13 100644
--- a/lib/Target/AArch64/AArch64SchedExynosM3.td
+++ b/lib/Target/AArch64/AArch64SchedExynosM3.td
@@ -1,9 +1,8 @@
 //=- AArch64SchedExynosM3.td - Samsung Exynos M3 Sched Defs --*- tablegen -*-=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -108,6 +107,8 @@ def M3UnitNSHF : ProcResGroup<[M3UnitNSHF0,
 
 def M3WriteZ0 : SchedWriteRes<[]> { let Latency = 0;
                                     let NumMicroOps = 1; }
+def M3WriteZ1 : SchedWriteRes<[]> { let Latency = 1;
+                                    let NumMicroOps = 0; }
 
 def M3WriteA1 : SchedWriteRes<[M3UnitALU]> { let Latency = 1; }
 def M3WriteAA : SchedWriteRes<[M3UnitALU]> { let Latency = 2;
@@ -125,18 +126,18 @@ def M3WriteAD : SchedWriteRes<[M3UnitALU,
 def M3WriteC1 : SchedWriteRes<[M3UnitC]>   { let Latency = 1; }
 def M3WriteC2 : SchedWriteRes<[M3UnitC]>   { let Latency = 2; }
 def M3WriteAU : SchedWriteVariant<[SchedVar<IsCopyIdiomPred, [M3WriteZ0]>,
-                                   SchedVar<ExynosShiftPred, [M3WriteA1]>,
+                                   SchedVar<ExynosArithPred, [M3WriteA1]>,
+                                   SchedVar<ExynosLogicPred, [M3WriteA1]>,
                                    SchedVar<NoSchedPred,     [M3WriteAA]>]>;
 def M3WriteAV : SchedWriteVariant<[SchedVar<IsCopyIdiomPred, [M3WriteZ0]>,
                                    SchedVar<NoSchedPred,     [M3WriteAA]>]>;
 def M3WriteAW : SchedWriteVariant<[SchedVar<IsZeroIdiomPred, [M3WriteZ0]>,
                                    SchedVar<NoSchedPred,     [M3WriteAA]>]>;
-def M3WriteAX : SchedWriteVariant<[SchedVar<ExynosExtPred, [M3WriteA1]>,
-                                   SchedVar<NoSchedPred,   [M3WriteAA]>]>;
+def M3WriteAX : SchedWriteVariant<[SchedVar<ExynosArithPred, [M3WriteA1]>,
+                                   SchedVar<ExynosLogicPred, [M3WriteA1]>,
+                                   SchedVar<NoSchedPred,     [M3WriteAA]>]>;
 def M3WriteAY : SchedWriteVariant<[SchedVar<ExynosRotateRightImmPred, [M3WriteA1]>,
                                    SchedVar<NoSchedPred,              [M3WriteAA]>]>;
-def M3WriteAZ : SchedWriteVariant<[SchedVar<ExynosShiftPred, [M3WriteA1]>,
-                                   SchedVar<NoSchedPred,     [M3WriteAA]>]>;
 
 def M3WriteB1 : SchedWriteRes<[M3UnitB]> { let Latency = 1; }
 def M3WriteBX : SchedWriteVariant<[SchedVar<ExynosBranchLinkLRPred, [M3WriteAC]>,
@@ -162,8 +163,8 @@ def M3WriteLE : SchedWriteRes<[M3UnitA,
                                            let NumMicroOps = 2; }
 def M3WriteLH : SchedWriteRes<[]>        { let Latency = 5;
                                            let NumMicroOps = 0; }
-def M3WriteLX : SchedWriteVariant<[SchedVar<ScaledIdxPred, [M3WriteL5]>,
-                                   SchedVar<NoSchedPred,   [M3WriteL4]>]>;
+def M3WriteLX : SchedWriteVariant<[SchedVar<ExynosScaledIdxPred, [M3WriteL5]>,
+                                   SchedVar<NoSchedPred,         [M3WriteL4]>]>;
 
 def M3WriteS1 : SchedWriteRes<[M3UnitS]>   { let Latency = 1; }
 def M3WriteSA : SchedWriteRes<[M3UnitA,
@@ -174,21 +175,21 @@ def M3WriteSB : SchedWriteRes<[M3UnitA,
                                M3UnitS]>   { let Latency = 2;
                                              let NumMicroOps = 2; }
 
-def M3ReadAdrBase : SchedReadVariant<[SchedVar<ScaledIdxPred, [ReadDefault]>,
-                                      SchedVar<NoSchedPred,   [ReadDefault]>]>;
+def M3ReadAdrBase : SchedReadVariant<[SchedVar<ExynosScaledIdxPred, [ReadDefault]>,
+                                      SchedVar<NoSchedPred,         [ReadDefault]>]>;
 
 // Branch instructions.
 def : SchedAlias<WriteBr, M3WriteZ0>;
-def : WriteRes<WriteBrReg, [M3UnitC]> { let Latency = 1; }
+def : SchedAlias<WriteBrReg, M3WriteC1>;
 
 // Arithmetic and logical integer instructions.
-def : WriteRes<WriteI,     [M3UnitALU]> { let Latency = 1; }
-def : WriteRes<WriteISReg, [M3UnitALU]> { let Latency = 1; }
-def : WriteRes<WriteIEReg, [M3UnitALU]> { let Latency = 1; }
-def : WriteRes<WriteIS,    [M3UnitALU]> { let Latency = 1; }
+def : SchedAlias<WriteI,     M3WriteA1>;
+def : SchedAlias<WriteISReg, M3WriteA1>;
+def : SchedAlias<WriteIEReg, M3WriteA1>;
+def : SchedAlias<WriteIS,    M3WriteA1>;
 
 // Move instructions.
-def : WriteRes<WriteImm, [M3UnitALU]> { let Latency = 1; }
+def : SchedAlias<WriteImm, M3WriteA1>;
 
 // Divide and multiply instructions.
 def : WriteRes<WriteID32, [M3UnitC,
@@ -205,8 +206,7 @@ def : WriteRes<WriteIM64, [M3UnitC]>  { let Latency = 4;
 def : SchedAlias<WriteExtr, M3WriteAY>;
 
 // Addressing modes.
-def : WriteRes<WriteAdr, []> { let Latency = 1;
-                               let NumMicroOps = 0; }
+def : SchedAlias<WriteAdr,    M3WriteZ1>;
 def : SchedAlias<ReadAdrBase, M3ReadAdrBase>;
 
 // Load instructions.
@@ -483,10 +483,10 @@ def : InstRW<[M3WriteC1], (instregex "^CBN?Z[WX]")>;
 def : InstRW<[M3WriteAD], (instregex "^TBN?Z[WX]")>;
 
 // Arithmetic and logical integer instructions.
-def : InstRW<[M3WriteAZ], (instregex "^(ADD|AND|BIC|EON|EOR|ORN|SUB)[WX]rs$")>;
+def : InstRW<[M3WriteAX], (instregex "^(ADD|AND|BIC|EON|EOR|ORN|SUB)[WX]rs$")>;
 def : InstRW<[M3WriteAU], (instrs ORRWrs, ORRXrs)>;
 def : InstRW<[M3WriteAX], (instregex "^(ADD|SUB)S?[WX]rx(64)?$")>;
-def : InstRW<[M3WriteAZ], (instregex "^(ADD|AND|BIC|SUB)S[WX]rs$")>;
+def : InstRW<[M3WriteAX], (instregex "^(ADD|AND|BIC|SUB)S[WX]rs$")>;
 def : InstRW<[M3WriteAV], (instrs ADDWri, ADDXri)>;
 def : InstRW<[M3WriteAW], (instrs ORRWri, ORRXri)>;
 
@@ -669,108 +669,108 @@ def : InstRW<[M3WriteNSHF1], (instregex "^(TRN|UZP|ZIP)[12]v")>;
 // ASIMD load instructions.
 def : InstRW<[M3WriteL5],   (instregex "LD1Onev(8b|4h|2s|1d)$")>;
 def : InstRW<[M3WriteL5,
-              WriteAdr],    (instregex "LD1Onev(8b|4h|2s|1d)_POST")>;
+              M3WriteA1],   (instregex "LD1Onev(8b|4h|2s|1d)_POST")>;
 def : InstRW<[M3WriteL5],   (instregex "LD1Onev(16b|8h|4s|2d)$")>;
 def : InstRW<[M3WriteL5,
-              WriteAdr],    (instregex "LD1Onev(16b|8h|4s|2d)_POST")>;
+              M3WriteA1],   (instregex "LD1Onev(16b|8h|4s|2d)_POST")>;
 
 def : InstRW<[M3WriteVLDA], (instregex "LD1Twov(8b|4h|2s|1d)$")>;
 def : InstRW<[M3WriteVLDA,
-              WriteAdr],    (instregex "LD1Twov(8b|4h|2s|1d)_POST")>;
+              M3WriteA1],   (instregex "LD1Twov(8b|4h|2s|1d)_POST")>;
 def : InstRW<[M3WriteVLDA], (instregex "LD1Twov(16b|8h|4s|2d)$")>;
 def : InstRW<[M3WriteVLDA,
-              WriteAdr],    (instregex "LD1Twov(16b|8h|4s|2d)_POST")>;
+              M3WriteA1],   (instregex "LD1Twov(16b|8h|4s|2d)_POST")>;
 
 def : InstRW<[M3WriteVLDB], (instregex "LD1Threev(8b|4h|2s|1d)$")>;
 def : InstRW<[M3WriteVLDB,
-              WriteAdr],    (instregex "LD1Threev(8b|4h|2s|1d)_POST")>;
+              M3WriteA1],   (instregex "LD1Threev(8b|4h|2s|1d)_POST")>;
 def : InstRW<[M3WriteVLDB], (instregex "LD1Threev(16b|8h|4s|2d)$")>;
 def : InstRW<[M3WriteVLDB,
-              WriteAdr],    (instregex "LD1Threev(16b|8h|4s|2d)_POST")>;
+              M3WriteA1],   (instregex "LD1Threev(16b|8h|4s|2d)_POST")>;
 
 def : InstRW<[M3WriteVLDC], (instregex "LD1Fourv(8b|4h|2s|1d)$")>;
 def : InstRW<[M3WriteVLDC,
-              WriteAdr],    (instregex "LD1Fourv(8b|4h|2s|1d)_POST")>;
+              M3WriteA1],   (instregex "LD1Fourv(8b|4h|2s|1d)_POST")>;
 def : InstRW<[M3WriteVLDC], (instregex "LD1Fourv(16b|8h|4s|2d)$")>;
 def : InstRW<[M3WriteVLDC,
-              WriteAdr],    (instregex "LD1Fourv(16b|8h|4s|2d)_POST")>;
+              M3WriteA1],   (instregex "LD1Fourv(16b|8h|4s|2d)_POST")>;
 
 def : InstRW<[M3WriteVLDD], (instregex "LD1i(8|16|32)$")>;
 def : InstRW<[M3WriteVLDD,
-              WriteAdr],    (instregex "LD1i(8|16|32)_POST")>;
+              M3WriteA1],   (instregex "LD1i(8|16|32)_POST")>;
 def : InstRW<[M3WriteVLDE], (instregex "LD1i(64)$")>;
 def : InstRW<[M3WriteVLDE,
-              WriteAdr],    (instregex "LD1i(64)_POST")>;
+              M3WriteA1],   (instregex "LD1i(64)_POST")>;
 
 def : InstRW<[M3WriteL5],   (instregex "LD1Rv(8b|4h|2s|1d)$")>;
 def : InstRW<[M3WriteL5,
-              WriteAdr],    (instregex "LD1Rv(8b|4h|2s|1d)_POST")>;
+              M3WriteA1],   (instregex "LD1Rv(8b|4h|2s|1d)_POST")>;
 def : InstRW<[M3WriteL5],   (instregex "LD1Rv(16b|8h|4s|2d)$")>;
 def : InstRW<[M3WriteL5,
-              WriteAdr],    (instregex "LD1Rv(16b|8h|4s|2d)_POST")>;
+              M3WriteA1],   (instregex "LD1Rv(16b|8h|4s|2d)_POST")>;
 
 def : InstRW<[M3WriteVLDF], (instregex "LD2Twov(8b|4h|2s)$")>;
 def : InstRW<[M3WriteVLDF,
-              WriteAdr],    (instregex "LD2Twov(8b|4h|2s)_POST")>;
+              M3WriteA1],   (instregex "LD2Twov(8b|4h|2s)_POST")>;
 def : InstRW<[M3WriteVLDF], (instregex "LD2Twov(16b|8h|4s|2d)$")>;
 def : InstRW<[M3WriteVLDF,
-              WriteAdr],    (instregex "LD2Twov(16b|8h|4s|2d)_POST")>;
+              M3WriteA1],   (instregex "LD2Twov(16b|8h|4s|2d)_POST")>;
 
 def : InstRW<[M3WriteVLDG], (instregex "LD2i(8|16|32)$")>;
 def : InstRW<[M3WriteVLDG,
-              WriteAdr],    (instregex "LD2i(8|16|32)_POST")>;
+              M3WriteA1],   (instregex "LD2i(8|16|32)_POST")>;
 def : InstRW<[M3WriteVLDH], (instregex "LD2i(64)$")>;
 def : InstRW<[M3WriteVLDH,
-              WriteAdr],    (instregex "LD2i(64)_POST")>;
+              M3WriteA1],   (instregex "LD2i(64)_POST")>;
 
 def : InstRW<[M3WriteVLDA], (instregex "LD2Rv(8b|4h|2s|1d)$")>;
 def : InstRW<[M3WriteVLDA,
-              WriteAdr],    (instregex "LD2Rv(8b|4h|2s|1d)_POST")>;
+              M3WriteA1],   (instregex "LD2Rv(8b|4h|2s|1d)_POST")>;
 def : InstRW<[M3WriteVLDA], (instregex "LD2Rv(16b|8h|4s|2d)$")>;
 def : InstRW<[M3WriteVLDA,
-              WriteAdr],    (instregex "LD2Rv(16b|8h|4s|2d)_POST")>;
+              M3WriteA1],   (instregex "LD2Rv(16b|8h|4s|2d)_POST")>;
 
 def : InstRW<[M3WriteVLDI], (instregex "LD3Threev(8b|4h|2s)$")>;
 def : InstRW<[M3WriteVLDI,
-              WriteAdr],    (instregex "LD3Threev(8b|4h|2s)_POST")>;
+              M3WriteA1],   (instregex "LD3Threev(8b|4h|2s)_POST")>;
 def : InstRW<[M3WriteVLDI], (instregex "LD3Threev(16b|8h|4s|2d)$")>;
 def : InstRW<[M3WriteVLDI,
-              WriteAdr],    (instregex "LD3Threev(16b|8h|4s|2d)_POST")>;
+              M3WriteA1],   (instregex "LD3Threev(16b|8h|4s|2d)_POST")>;
 
 def : InstRW<[M3WriteVLDJ], (instregex "LD3i(8|16|32)$")>;
 def : InstRW<[M3WriteVLDJ,
-              WriteAdr],    (instregex "LD3i(8|16|32)_POST")>;
+              M3WriteA1],   (instregex "LD3i(8|16|32)_POST")>;
 def : InstRW<[M3WriteVLDL], (instregex "LD3i(64)$")>;
 def : InstRW<[M3WriteVLDL,
-              WriteAdr],    (instregex "LD3i(64)_POST")>;
+              M3WriteA1],   (instregex "LD3i(64)_POST")>;
 
 def : InstRW<[M3WriteVLDB], (instregex "LD3Rv(8b|4h|2s|1d)$")>;
 def : InstRW<[M3WriteVLDB,
-              WriteAdr],    (instregex "LD3Rv(8b|4h|2s|1d)_POST")>;
+              M3WriteA1],   (instregex "LD3Rv(8b|4h|2s|1d)_POST")>;
 def : InstRW<[M3WriteVLDB], (instregex "LD3Rv(16b|8h|4s|2d)$")>;
 def : InstRW<[M3WriteVLDB,
-              WriteAdr],    (instregex "LD3Rv(16b|8h|4s|2d)_POST")>;
+              M3WriteA1],   (instregex "LD3Rv(16b|8h|4s|2d)_POST")>;
 
 def : InstRW<[M3WriteVLDN], (instregex "LD4Fourv(8b|4h|2s)$")>;
 def : InstRW<[M3WriteVLDN,
-              WriteAdr],    (instregex "LD4Fourv(8b|4h|2s)_POST")>;
+              M3WriteA1],   (instregex "LD4Fourv(8b|4h|2s)_POST")>;
 def : InstRW<[M3WriteVLDN], (instregex "LD4Fourv(16b|8h|4s|2d)$")>;
 def : InstRW<[M3WriteVLDN,
-              WriteAdr],    (instregex "LD4Fourv(16b|8h|4s|2d)_POST")>;
+              M3WriteA1],   (instregex "LD4Fourv(16b|8h|4s|2d)_POST")>;
 
 def : InstRW<[M3WriteVLDK], (instregex "LD4i(8|16|32)$")>;
 def : InstRW<[M3WriteVLDK,
-              WriteAdr],    (instregex "LD4i(8|16|32)_POST")>;
+              M3WriteA1],   (instregex "LD4i(8|16|32)_POST")>;
 def : InstRW<[M3WriteVLDM], (instregex "LD4i(64)$")>;
 def : InstRW<[M3WriteVLDM,
-              WriteAdr],    (instregex "LD4i(64)_POST")>;
+              M3WriteA1],   (instregex "LD4i(64)_POST")>;
 
 def : InstRW<[M3WriteVLDC], (instregex "LD4Rv(8b|4h|2s|1d)$")>;
 def : InstRW<[M3WriteVLDC,
-              WriteAdr],    (instregex "LD4Rv(8b|4h|2s|1d)_POST")>;
+              M3WriteA1],   (instregex "LD4Rv(8b|4h|2s|1d)_POST")>;
 def : InstRW<[M3WriteVLDC], (instregex "LD4Rv(16b|8h|4s|2d)$")>;
 def : InstRW<[M3WriteVLDC,
-              WriteAdr],    (instregex "LD4Rv(16b|8h|4s|2d)_POST")>;
+              M3WriteA1],   (instregex "LD4Rv(16b|8h|4s|2d)_POST")>;
 
 // ASIMD store instructions.
 def : InstRW<[WriteVST],    (instregex "ST1Onev(8b|4h|2s|1d)$")>;
diff --git a/lib/Target/AArch64/AArch64SchedExynosM4.td b/lib/Target/AArch64/AArch64SchedExynosM4.td
new file mode 100644
index 0000000000000000000000000000000000000000..ebf35c3cdc582b97cc507a01a3cfa270a0c9862a
--- /dev/null
+++ b/lib/Target/AArch64/AArch64SchedExynosM4.td
@@ -0,0 +1,1003 @@
+//=- AArch64SchedExynosM4.td - Samsung Exynos M4 Sched Defs --*- tablegen -*-=//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines the machine model for the Samsung Exynos M4 to support
+// instruction scheduling and other instruction cost heuristics.
+//
+//===----------------------------------------------------------------------===//
+
+//===----------------------------------------------------------------------===//
+// The Exynos-M4 is an advanced superscalar microprocessor with a 6-wide
+// in-order stage for decode and dispatch and a wider issue stage.
+// The execution units and loads and stores are out-of-order.
+
+def ExynosM4Model : SchedMachineModel {
+  let IssueWidth            =   6; // Up to 6 uops per cycle.
+  let MicroOpBufferSize     = 228; // ROB size.
+  let LoopMicroOpBufferSize =  48; // Based on the instruction queue size.
+  let LoadLatency           =   4; // Optimistic load cases.
+  let MispredictPenalty     =  16; // Minimum branch misprediction penalty.
+  let CompleteModel         =   1; // Use the default model otherwise.
+
+  list<Predicate> UnsupportedFeatures = [HasSVE];
+}
+
+//===----------------------------------------------------------------------===//
+// Define each kind of processor resource and number available on the Exynos-M4.
+
+let SchedModel = ExynosM4Model in {
+
+def M4UnitA  : ProcResource<2>; // Simple integer
+def M4UnitC  : ProcResource<2>; // Simple and complex integer
+let Super =  M4UnitC, BufferSize = 1 in
+def M4UnitD  : ProcResource<1>; // Integer division (inside C0, serialized)
+let Super =  M4UnitC in
+def M4UnitE  : ProcResource<1>; // CRC (inside C0)
+def M4UnitB  : ProcResource<2>; // Branch
+def M4UnitL0 : ProcResource<1>; // Load
+def M4UnitS0 : ProcResource<1>; // Store
+def M4PipeLS : ProcResource<1>; // Load/Store
+let Super = M4PipeLS in {
+  def M4UnitL1 : ProcResource<1>;
+  def M4UnitS1 : ProcResource<1>;
+}
+def M4PipeF0 : ProcResource<1>; // FP #0
+let Super = M4PipeF0 in {
+  def M4UnitFMAC0 : ProcResource<1>; // FP multiplication
+  def M4UnitFADD0 : ProcResource<1>; // Simple FP
+  def M4UnitFCVT0 : ProcResource<1>; // FP conversion
+  def M4UnitNALU0 : ProcResource<1>; // Simple vector
+  def M4UnitNHAD  : ProcResource<1>; // Horizontal vector
+  def M4UnitNMSC  : ProcResource<1>; // FP and vector miscellanea
+  def M4UnitNMUL0 : ProcResource<1>; // Vector multiplication
+  def M4UnitNSHT0 : ProcResource<1>; // Vector shifting
+  def M4UnitNSHF0 : ProcResource<1>; // Vector shuffling
+  def M4UnitNCRY0 : ProcResource<1>; // Cryptographic
+}
+def M4PipeF1 : ProcResource<1>; // FP #1
+let Super = M4PipeF1 in {
+  def M4UnitFMAC1 : ProcResource<1>; // FP multiplication
+  def M4UnitFADD1 : ProcResource<1>; // Simple FP
+  def M4UnitFDIV0 : ProcResource<2>; // FP division (serialized)
+  def M4UnitFSQR0 : ProcResource<2>; // FP square root (serialized)
+  def M4UnitFST0  : ProcResource<1>; // FP store
+  def M4UnitNALU1 : ProcResource<1>; // Simple vector
+  def M4UnitNSHT1 : ProcResource<1>; // Vector shifting
+  def M4UnitNSHF1 : ProcResource<1>; // Vector shuffling
+}
+def M4PipeF2 : ProcResource<1>; // FP #2
+let Super = M4PipeF2 in {
+  def M4UnitFMAC2 : ProcResource<1>; // FP multiplication
+  def M4UnitFADD2 : ProcResource<1>; // Simple FP
+  def M4UnitFCVT1 : ProcResource<1>; // FP conversion
+  def M4UnitFDIV1 : ProcResource<2>; // FP division (serialized)
+  def M4UnitFSQR1 : ProcResource<2>; // FP square root (serialized)
+  def M4UnitFST1  : ProcResource<1>; // FP store
+  def M4UnitNALU2 : ProcResource<1>; // Simple vector
+  def M4UnitNMUL1 : ProcResource<1>; // Vector multiplication
+  def M4UnitNSHT2 : ProcResource<1>; // Vector shifting
+  def M4UnitNCRY1 : ProcResource<1>; // Cryptographic
+}
+
+def M4UnitALU   : ProcResGroup<[M4UnitA,
+                                M4UnitC]>;
+def M4UnitL     : ProcResGroup<[M4UnitL0,
+                                M4UnitL1]>;
+def M4UnitS     : ProcResGroup<[M4UnitS0,
+                                M4UnitS1]>;
+def M4UnitFMAC  : ProcResGroup<[M4UnitFMAC0,
+                                M4UnitFMAC1,
+                                M4UnitFMAC2]>;
+def M4UnitFMACH : ProcResGroup<[M4UnitFMAC0,
+                                M4UnitFMAC1]>;
+def M4UnitFADD  : ProcResGroup<[M4UnitFADD0,
+                                M4UnitFADD1,
+                                M4UnitFADD2]>;
+def M4UnitFADDH : ProcResGroup<[M4UnitFADD0,
+                                M4UnitFADD1]>;
+def M4UnitFCVT  : ProcResGroup<[M4UnitFCVT0,
+                                M4UnitFCVT1]>;
+def M4UnitFCVTH : ProcResGroup<[M4UnitFCVT0]>;
+def M4UnitFDIV  : ProcResGroup<[M4UnitFDIV0,
+                                M4UnitFDIV1]>;
+def M4UnitFDIVH : ProcResGroup<[M4UnitFDIV0]>;
+def M4UnitFSQR  : ProcResGroup<[M4UnitFSQR0,
+                                M4UnitFSQR1]>;
+def M4UnitFSQRH : ProcResGroup<[M4UnitFSQR0]>;
+def M4UnitFST   : ProcResGroup<[M4UnitFST0,
+                                M4UnitFST1]>;
+def M4UnitNALU  : ProcResGroup<[M4UnitNALU0,
+                                M4UnitNALU1,
+                                M4UnitNALU2]>;
+def M4UnitNALUH : ProcResGroup<[M4UnitNALU0,
+                                M4UnitNALU1]>;
+def M4UnitNMUL  : ProcResGroup<[M4UnitNMUL0,
+                                M4UnitNMUL1]>;
+def M4UnitNSHT  : ProcResGroup<[M4UnitNSHT0,
+                                M4UnitNSHT1,
+                                M4UnitNSHT2]>;
+def M4UnitNSHF  : ProcResGroup<[M4UnitNSHF0,
+                                M4UnitNSHF1]>;
+def M4UnitNSHFH : ProcResGroup<[M4UnitNSHF0]>;
+def M4UnitNCRY  : ProcResGroup<[M4UnitNCRY0,
+                                M4UnitNCRY1]>;
+
+//===----------------------------------------------------------------------===//
+// Resources details.
+
+def M4WriteZ0 : SchedWriteRes<[]> { let Latency = 0; }
+def M4WriteZ1 : SchedWriteRes<[]> { let Latency = 1;
+                                    let NumMicroOps = 0; }
+def M4WriteZ4 : SchedWriteRes<[]> { let Latency = 4;
+                                    let NumMicroOps = 0; }
+
+def M4WriteA1 : SchedWriteRes<[M4UnitALU]> { let Latency = 1; }
+def M4WriteA2 : SchedWriteRes<[M4UnitALU]> { let Latency = 2; }
+def M4WriteAA : SchedWriteRes<[M4UnitALU]> { let Latency = 2;
+                                             let ResourceCycles = [2]; }
+def M4WriteAB : SchedWriteRes<[M4UnitALU,
+                               M4UnitC]>   { let Latency = 2;
+                                             let NumMicroOps = 2; }
+def M4WriteAC : SchedWriteRes<[M4UnitALU,
+                               M4UnitALU,
+                               M4UnitC]>   { let Latency = 3;
+                                             let NumMicroOps = 3; }
+def M4WriteAD : SchedWriteRes<[M4UnitALU,
+                               M4UnitC]>   { let Latency = 2;
+                                             let NumMicroOps = 2; }
+def M4WriteAF : SchedWriteRes<[M4UnitALU]> { let Latency = 2;
+                                             let NumMicroOps = 2; }
+def M4WriteAU : SchedWriteVariant<[SchedVar<IsCopyIdiomPred,   [M4WriteZ0]>,
+                                   SchedVar<ExynosArithPred,   [M4WriteA1]>,
+                                   SchedVar<ExynosLogicExPred, [M4WriteA1]>,
+                                   SchedVar<NoSchedPred,       [M4WriteAA]>]>;
+def M4WriteAV : SchedWriteVariant<[SchedVar<ExynosResetPred, [M4WriteZ0]>,
+                                   SchedVar<NoSchedPred,     [M4WriteAA]>]>;
+def M4WriteAX : SchedWriteVariant<[SchedVar<ExynosArithPred,   [M4WriteA1]>,
+                                   SchedVar<ExynosLogicExPred, [M4WriteA1]>,
+                                   SchedVar<NoSchedPred,       [M4WriteAA]>]>;
+def M4WriteAY : SchedWriteVariant<[SchedVar<ExynosRotateRightImmPred, [M4WriteA1]>,
+                                   SchedVar<NoSchedPred,              [M4WriteAF]>]>;
+
+def M4WriteB1 : SchedWriteRes<[M4UnitB]> { let Latency = 1; }
+def M4WriteBX : SchedWriteVariant<[SchedVar<ExynosBranchLinkLRPred, [M4WriteAC]>,
+                                   SchedVar<NoSchedPred,            [M4WriteAB]>]>;
+
+def M4WriteC1 : SchedWriteRes<[M4UnitC]> { let Latency = 1; }
+def M4WriteC3 : SchedWriteRes<[M4UnitC]> { let Latency = 3; }
+def M4WriteCA : SchedWriteRes<[M4UnitC]> { let Latency = 4;
+                                           let ResourceCycles = [2]; }
+
+def M4WriteD12 : SchedWriteRes<[M4UnitD]> { let Latency = 12; }
+def M4WriteD21 : SchedWriteRes<[M4UnitD]> { let Latency = 21; }
+
+def M4WriteE2 : SchedWriteRes<[M4UnitE]> { let Latency = 2; }
+
+def M4WriteL4 : SchedWriteRes<[M4UnitL]> { let Latency = 4; }
+def M4WriteL5 : SchedWriteRes<[M4UnitL]> { let Latency = 5; }
+def M4WriteLA : SchedWriteRes<[M4UnitL,
+                               M4UnitL]> { let Latency = 5;
+                                           let NumMicroOps = 1; }
+def M4WriteLB : SchedWriteRes<[M4UnitA,
+                               M4UnitL]> { let Latency = 5;
+                                           let NumMicroOps = 2; }
+def M4WriteLC : SchedWriteRes<[M4UnitA,
+                               M4UnitL,
+                               M4UnitL]> { let Latency = 5;
+                                           let NumMicroOps = 2; }
+def M4WriteLD : SchedWriteRes<[M4UnitA,
+                               M4UnitL]> { let Latency = 4;
+                                           let NumMicroOps = 2; }
+def M4WriteLE : SchedWriteRes<[M4UnitA,
+                               M4UnitL]> { let Latency = 6;
+                                           let NumMicroOps = 2; }
+def M4WriteLH : SchedWriteRes<[]>        { let Latency = 5;
+                                           let NumMicroOps = 0; }
+def M4WriteLX : SchedWriteVariant<[SchedVar<ScaledIdxPred, [M4WriteL5]>,
+                                   SchedVar<NoSchedPred,   [M4WriteL4]>]>;
+
+def M4WriteS1 : SchedWriteRes<[M4UnitS]>  { let Latency = 1; }
+def M4WriteSA : SchedWriteRes<[M4UnitS0]> { let Latency = 3; }
+def M4WriteSB : SchedWriteRes<[M4UnitA,
+                               M4UnitS]>  { let Latency = 2;
+                                            let NumMicroOps = 1; }
+def M4WriteSX : SchedWriteVariant<[SchedVar<ExynosScaledIdxPred, [M4WriteSB]>,
+                                   SchedVar<NoSchedPred,         [M4WriteS1]>]>;
+
+def M4ReadAdrBase : SchedReadVariant<[SchedVar<
+                                        MCSchedPredicate<
+                                          CheckAny<
+                                            [ScaledIdxFn,
+                                             ExynosScaledIdxFn]>>, [ReadDefault]>,
+                                      SchedVar<NoSchedPred,        [ReadDefault]>]>;
+
+def M4WriteNEONA   : SchedWriteRes<[M4UnitNSHF,
+                                    M4UnitFADD]>  { let Latency = 3;
+                                                    let NumMicroOps = 2; }
+def M4WriteNEONB   : SchedWriteRes<[M4UnitNALU,
+                                    M4UnitS0]>    { let Latency = 5;
+                                                    let NumMicroOps = 2; }
+def M4WriteNEOND   : SchedWriteRes<[M4UnitNSHF,
+                                    M4UnitFST]>   { let Latency = 6;
+                                                    let NumMicroOps = 2; }
+def M4WriteNEONH   : SchedWriteRes<[M4UnitNALU,
+                                    M4UnitS0]>    { let Latency = 5;
+                                                    let NumMicroOps = 2; }
+def M4WriteNEONI   : SchedWriteRes<[M4UnitNSHF,
+                                    M4UnitS0]>    { let Latency = 2;
+                                                    let NumMicroOps = 2; }
+def M4WriteNEONJ   : SchedWriteRes<[M4UnitNMSC,
+                                    M4UnitS0]>    { let Latency = 4; }
+def M4WriteNEONK   : SchedWriteRes<[M4UnitNSHF,
+                                    M4UnitNMSC,
+                                    M4UnitS0]>    { let Latency = 5;
+                                                    let NumMicroOps = 2; }
+def M4WriteNEONL   : SchedWriteRes<[M4UnitNMUL]>  { let Latency = 3; }
+def M4WriteNEONM   : SchedWriteRes<[M4UnitNMUL]>  { let Latency = 3; }
+def M4WriteNEONN   : SchedWriteRes<[M4UnitNMSC,
+                                    M4UnitNMSC]>  { let Latency = 5;
+                                                    let NumMicroOps = 2; }
+def M4WriteNEONO   : SchedWriteRes<[M4UnitNMSC,
+                                    M4UnitNMSC,
+                                    M4UnitNMSC]>  { let Latency = 8;
+                                                    let NumMicroOps = 3; }
+def M4WriteNEONP   : SchedWriteRes<[M4UnitNSHF,
+                                    M4UnitNMSC]>  { let Latency = 4;
+                                                    let NumMicroOps = 2; }
+def M4WriteNEONQ   : SchedWriteRes<[M4UnitNMSC,
+                                    M4UnitC]>     { let Latency = 3;
+                                                    let NumMicroOps = 1; }
+def M4WriteNEONR   : SchedWriteRes<[M4UnitFCVT0,
+                                    M4UnitS0]>    { let Latency = 4;
+                                                    let NumMicroOps = 1; }
+def M4WriteNEONV   : SchedWriteRes<[M4UnitFDIV,
+                                    M4UnitFDIV]>  { let Latency = 7;
+                                                    let ResourceCycles = [6, 6]; }
+def M4WriteNEONVH  : SchedWriteRes<[M4UnitFDIVH,
+                                    M4UnitFDIVH]> { let Latency = 7;
+                                                    let ResourceCycles = [6, 6]; }
+def M4WriteNEONW   : SchedWriteRes<[M4UnitFDIV,
+                                    M4UnitFDIV]>  { let Latency = 12;
+                                                    let ResourceCycles = [9, 9]; }
+def M4WriteNEONX   : SchedWriteRes<[M4UnitFSQR,
+                                    M4UnitFSQR]>  { let Latency = 8;
+                                                    let ResourceCycles = [7, 7]; }
+def M4WriteNEONXH  : SchedWriteRes<[M4UnitFSQRH,
+                                    M4UnitFSQRH]> { let Latency = 7;
+                                                    let ResourceCycles = [6, 6]; }
+def M4WriteNEONY   : SchedWriteRes<[M4UnitFSQR,
+                                    M4UnitFSQR]>  { let Latency = 12;
+                                                    let ResourceCycles = [9, 9]; }
+def M4WriteNEONZ   : SchedWriteVariant<[SchedVar<ExynosQFormPred, [M4WriteNEONO]>,
+                                        SchedVar<NoSchedPred,     [M4WriteNEONN]>]>;
+
+def M4WriteFADD2   : SchedWriteRes<[M4UnitFADD]>  { let Latency = 2; }
+def M4WriteFADD2H  : SchedWriteRes<[M4UnitFADDH]> { let Latency = 2; }
+
+def M4WriteFCVT2   : SchedWriteRes<[M4UnitFCVT]>  { let Latency = 2; }
+def M4WriteFCVT2A  : SchedWriteRes<[M4UnitFCVT0]> { let Latency = 2; }
+def M4WriteFCVT2H  : SchedWriteRes<[M4UnitFCVTH]> { let Latency = 2; }
+def M4WriteFCVT3   : SchedWriteRes<[M4UnitFCVT]>  { let Latency = 3; }
+def M4WriteFCVT3A  : SchedWriteRes<[M4UnitFCVT0]> { let Latency = 3; }
+def M4WriteFCVT3H  : SchedWriteRes<[M4UnitFCVTH]> { let Latency = 3; }
+def M4WriteFCVT4   : SchedWriteRes<[M4UnitFCVT]>  { let Latency = 4; }
+def M4WriteFCVT4A  : SchedWriteRes<[M4UnitFCVT0]> { let Latency = 4; }
+def M4WriteFCVT6A  : SchedWriteRes<[M4UnitFCVT0]> { let Latency = 6; }
+
+def M4WriteFDIV7   : SchedWriteRes<[M4UnitFDIV]>  { let Latency = 7;
+                                                    let ResourceCycles = [6]; }
+def M4WriteFDIV7H  : SchedWriteRes<[M4UnitFDIVH]> { let Latency = 7;
+                                                    let ResourceCycles = [6]; }
+def M4WriteFDIV12  : SchedWriteRes<[M4UnitFDIV]>  { let Latency = 12;
+                                                    let ResourceCycles = [9]; }
+
+def M4WriteFMAC2H  : SchedWriteRes<[M4UnitFMACH]> { let Latency = 2; }
+def M4WriteFMAC3H  : SchedWriteRes<[M4UnitFMACH]> { let Latency = 3; }
+def M4WriteFMAC3   : SchedWriteRes<[M4UnitFMAC]>  { let Latency = 3; }
+def M4WriteFMAC4   : SchedWriteRes<[M4UnitFMAC]>  { let Latency = 4; }
+def M4WriteFMAC4H  : SchedWriteRes<[M4UnitFMACH]> { let Latency = 4; }
+def M4WriteFMAC5   : SchedWriteRes<[M4UnitFMAC]>  { let Latency = 5; }
+
+def M4WriteFSQR7H  : SchedWriteRes<[M4UnitFSQRH]> { let Latency = 7;
+                                                    let ResourceCycles = [6]; }
+def M4WriteFSQR8   : SchedWriteRes<[M4UnitFSQR]>  { let Latency = 8;
+                                                    let ResourceCycles = [7]; }
+def M4WriteFSQR12  : SchedWriteRes<[M4UnitFSQR]>  { let Latency = 12;
+                                                    let ResourceCycles = [9]; }
+
+def M4WriteNALU1   : SchedWriteRes<[M4UnitNALU]>  { let Latency = 1; }
+def M4WriteNALU1H  : SchedWriteRes<[M4UnitNALUH]> { let Latency = 1; }
+
+def M4WriteNCRY1   : SchedWriteRes<[M4UnitNCRY]>  { let Latency = 1; }
+def M4WriteNCRY1A  : SchedWriteRes<[M4UnitNCRY0]> { let Latency = 1; }
+def M4WriteNCRY3A  : SchedWriteRes<[M4UnitNCRY0]> { let Latency = 3; }
+def M4WriteNCRY5A  : SchedWriteRes<[M4UnitNCRY]>  { let Latency = 5; }
+
+def M4WriteNHAD1   : SchedWriteRes<[M4UnitNHAD]>  { let Latency = 1; }
+def M4WriteNHAD3   : SchedWriteRes<[M4UnitNHAD]>  { let Latency = 3; }
+
+def M4WriteNMSC1   : SchedWriteRes<[M4UnitNMSC]>  { let Latency = 1; }
+def M4WriteNMSC2   : SchedWriteRes<[M4UnitNMSC]>  { let Latency = 2; }
+def M4WriteNMSC3   : SchedWriteRes<[M4UnitNMSC]>  { let Latency = 3; }
+
+def M4WriteNMUL3   : SchedWriteRes<[M4UnitNMUL]>  { let Latency = 3; }
+
+def M4WriteNSHF1   : SchedWriteRes<[M4UnitNSHF]>  { let Latency = 1; }
+def M4WriteNSHF1H  : SchedWriteRes<[M4UnitNSHFH]> { let Latency = 1; }
+def M4WriteNSHF3   : SchedWriteRes<[M4UnitNSHF]>  { let Latency = 3; }
+def M4WriteNSHFA   : SchedWriteRes<[M4UnitNSHF]>  { let Latency = 1;
+                                                    let ResourceCycles = [2]; }
+def M4WriteNSHFB   : SchedWriteRes<[M4UnitNSHF]>  { let Latency = 2;
+                                                    let NumMicroOps = 2;
+                                                    let ResourceCycles = [2]; }
+def M4WriteNSHFC   : SchedWriteRes<[M4UnitNSHF]>  { let Latency = 3;
+                                                    let NumMicroOps = 3;
+                                                    let ResourceCycles = [4]; }
+def M4WriteNSHFD   : SchedWriteRes<[M4UnitNSHF]>  { let Latency = 4;
+                                                    let NumMicroOps = 4;
+                                                    let ResourceCycles = [4]; }
+
+def M4WriteNSHT1   : SchedWriteRes<[M4UnitNSHT]>  { let Latency = 1; }
+def M4WriteNSHT2   : SchedWriteRes<[M4UnitNSHT]>  { let Latency = 2; }
+def M4WriteNSHT3   : SchedWriteRes<[M4UnitNSHT]>  { let Latency = 3; }
+def M4WriteNSHT4A  : SchedWriteRes<[M4UnitNSHT1]> { let Latency = 4; }
+
+def M4WriteVLDA    : SchedWriteRes<[M4UnitL,
+                                    M4UnitL]>     { let Latency = 5;
+                                                    let NumMicroOps = 2; }
+def M4WriteVLDB    : SchedWriteRes<[M4UnitL,
+                                    M4UnitL,
+                                    M4UnitL]>     { let Latency = 6;
+                                                    let NumMicroOps = 3; }
+def M4WriteVLDC    : SchedWriteRes<[M4UnitL,
+                                    M4UnitL,
+                                    M4UnitL,
+                                    M4UnitL]>     { let Latency = 6;
+                                                    let NumMicroOps = 4; }
+def M4WriteVLDD    : SchedWriteRes<[M4UnitL,
+                                    M4UnitNSHF]>  { let Latency = 6;
+                                                    let NumMicroOps = 2;
+                                                    let ResourceCycles = [2, 1]; }
+def M4WriteVLDF    : SchedWriteRes<[M4UnitL,
+                                    M4UnitL]>     { let Latency = 10;
+                                                    let NumMicroOps = 2;
+                                                    let ResourceCycles = [3, 3]; }
+def M4WriteVLDG    : SchedWriteRes<[M4UnitL,
+                                    M4UnitNSHF,
+                                    M4UnitNSHF]>  { let Latency = 6;
+                                                    let NumMicroOps = 3;
+                                                    let ResourceCycles = [2, 1, 1]; }
+def M4WriteVLDI    : SchedWriteRes<[M4UnitL,
+                                    M4UnitL,
+                                    M4UnitL]>     { let Latency = 12;
+                                                    let NumMicroOps = 3;
+                                                    let ResourceCycles = [3, 3, 3]; }
+def M4WriteVLDJ    : SchedWriteRes<[M4UnitL,
+                                    M4UnitNSHF,
+                                    M4UnitNSHF,
+                                    M4UnitNSHF]>  { let Latency = 7;
+                                                    let NumMicroOps = 4;
+                                                    let ResourceCycles = [3, 1, 1, 1]; }
+def M4WriteVLDK    : SchedWriteRes<[M4UnitL,
+                                    M4UnitNSHF,
+                                    M4UnitNSHF,
+                                    M4UnitNSHF,
+                                    M4UnitNSHF]>  { let Latency = 7;
+                                                    let NumMicroOps = 5;
+                                                    let ResourceCycles = [3, 1, 1, 1, 1]; }
+def M4WriteVLDL    : SchedWriteRes<[M4UnitL,
+                                    M4UnitNSHF,
+                                    M4UnitNSHF,
+                                    M4UnitL,
+                                    M4UnitNSHF]>  { let Latency = 7;
+                                                    let NumMicroOps = 5;
+                                                    let ResourceCycles = [3, 1, 1, 6, 1]; }
+def M4WriteVLDM    : SchedWriteRes<[M4UnitL,
+                                    M4UnitNSHF,
+                                    M4UnitNSHF,
+                                    M4UnitL,
+                                    M4UnitNSHF,
+                                    M4UnitNSHF]>  { let Latency = 7;
+                                                    let NumMicroOps = 6;
+                                                    let ResourceCycles = [3, 1, 1, 3, 1, 1]; }
+def M4WriteVLDN    : SchedWriteRes<[M4UnitL,
+                                    M4UnitL,
+                                    M4UnitL,
+                                    M4UnitL]>     { let Latency = 14;
+                                                    let NumMicroOps = 4;
+                                                    let ResourceCycles = [3, 3, 3, 3]; }
+
+def M4WriteVST1    : SchedWriteRes<[M4UnitS,
+                                    M4UnitFST]>  { let Latency = 1;
+                                                   let NumMicroOps = 1; }
+def M4WriteVSTA    : WriteSequence<[WriteVST], 2>;
+def M4WriteVSTB    : WriteSequence<[WriteVST], 3>;
+def M4WriteVSTC    : WriteSequence<[WriteVST], 4>;
+def M4WriteVSTD    : SchedWriteRes<[M4UnitS,
+                                    M4UnitFST]>   { let Latency = 2; }
+def M4WriteVSTE    : SchedWriteRes<[M4UnitS,
+                                    M4UnitFST,
+                                    M4UnitS,
+                                    M4UnitFST]>   { let Latency = 2;
+                                                    let NumMicroOps = 2; }
+def M4WriteVSTF    : SchedWriteRes<[M4UnitNSHF,
+                                    M4UnitS,
+                                    M4UnitFST,
+                                    M4UnitS,
+                                    M4UnitFST]>   { let Latency = 4;
+                                                    let NumMicroOps = 4;
+                                                    let ResourceCycles = [1, 2, 1, 2, 1]; }
+def M4WriteVSTG    : SchedWriteRes<[M4UnitNSHF,
+                                    M4UnitNSHF,
+                                    M4UnitNSHF,
+                                    M4UnitS,
+                                    M4UnitFST,
+                                    M4UnitS,
+                                    M4UnitFST,
+                                    M4UnitS,
+                                    M4UnitFST]>   { let Latency = 5;
+                                                    let NumMicroOps = 6;
+                                                    let ResourceCycles = [1, 1, 1, 2, 1, 2, 1, 2, 1]; }
+def M4WriteVSTI    : SchedWriteRes<[M4UnitNSHF,
+                                    M4UnitNSHF,
+                                    M4UnitNSHF,
+                                    M4UnitNSHF,
+                                    M4UnitS,
+                                    M4UnitFST,
+                                    M4UnitS,
+                                    M4UnitFST,
+                                    M4UnitS,
+                                    M4UnitFST,
+                                    M4UnitS,
+                                    M4UnitFST]>   { let Latency = 8;
+                                                    let NumMicroOps = 5;
+                                                    let ResourceCycles = [1, 1, 1, 1, 2, 1, 2, 1, 2, 1, 2, 1]; }
+def M4WriteVSTJ    : SchedWriteRes<[M4UnitA,
+                                    M4UnitS,
+                                    M4UnitFST]>   { let Latency = 1;
+                                                    let NumMicroOps = 2; }
+def M4WriteVSTK    : SchedWriteRes<[M4UnitA,
+                                    M4UnitS,
+                                    M4UnitFST]>   { let Latency = 3;
+                                                    let NumMicroOps = 2; }
+def M4WriteVSTL    : SchedWriteRes<[M4UnitNSHF,
+                                    M4UnitNSHF,
+                                    M4UnitS,
+                                    M4UnitFST,
+                                    M4UnitS,
+                                    M4UnitFST]>   { let Latency = 4;
+                                                    let NumMicroOps = 4;
+                                                    let ResourceCycles = [1, 1, 2, 1, 2, 1]; }
+
+// Special cases.
+def M4WriteCOPY    : SchedWriteVariant<[SchedVar<ExynosFPPred, [M4WriteNALU1]>,
+                                        SchedVar<NoSchedPred,  [M4WriteZ0]>]>;
+def M4WriteMOVI    : SchedWriteVariant<[SchedVar<IsZeroFPIdiomPred, [M4WriteZ0]>,
+                                        SchedVar<NoSchedPred,       [M4WriteNALU1]>]>;
+def M4WriteMULL    : SchedWriteVariant<[SchedVar<ExynosLongVectorUpperPred, [M4WriteNEONM]>,
+                                        SchedVar<NoSchedPred,               [M4WriteNMUL3]>]>;
+
+// Fast forwarding.
+def M4ReadAESM1    : SchedReadAdvance<+1, [M4WriteNCRY1]>;
+def M4ReadFMACM1   : SchedReadAdvance<+1, [M4WriteFMAC4,
+                                           M4WriteFMAC4H,
+                                           M4WriteFMAC5]>;
+def M4ReadNMULM1   : SchedReadAdvance<+1, [M4WriteNMUL3]>;
+def M4ReadMULLP2   : SchedReadAdvance<-2, [M4WriteNEONM]>;
+
+//===----------------------------------------------------------------------===//
+// Coarse scheduling model.
+
+// Branch instructions.
+def : SchedAlias<WriteBr,    M4WriteZ0>;
+def : SchedAlias<WriteBrReg, M4WriteC1>;
+
+// Arithmetic and logical integer instructions.
+def : SchedAlias<WriteI,     M4WriteA1>;
+def : SchedAlias<WriteIEReg, M4WriteAA>; // FIXME: M4WriteAX crashes TableGen.
+def : SchedAlias<WriteISReg, M4WriteAA>; // FIXME: M4WriteAX crashes TableGen.
+def : SchedAlias<WriteIS,    M4WriteA1>;
+
+// Move instructions.
+def : SchedAlias<WriteImm, M4WriteA1>;
+
+// Divide and multiply instructions.
+def : SchedAlias<WriteID32, M4WriteD12>;
+def : SchedAlias<WriteID64, M4WriteD21>;
+def : SchedAlias<WriteIM32, M4WriteC3>;
+def : SchedAlias<WriteIM64, M4WriteCA>;
+
+// Miscellaneous instructions.
+def : SchedAlias<WriteExtr, M4WriteAY>;
+
+// Addressing modes.
+def : SchedAlias<WriteAdr,    M4WriteZ1>;
+def : SchedAlias<ReadAdrBase, M4ReadAdrBase>;
+
+// Load instructions.
+def : SchedAlias<WriteLD,    M4WriteL4>;
+def : SchedAlias<WriteLDHi,  M4WriteZ4>;
+def : SchedAlias<WriteLDIdx, M4WriteLX>;
+
+// Store instructions.
+def : SchedAlias<WriteST,    M4WriteS1>;
+def : SchedAlias<WriteSTP,   M4WriteS1>;
+def : SchedAlias<WriteSTX,   M4WriteS1>;
+def : SchedAlias<WriteSTIdx, M4WriteSX>;
+
+// FP data instructions.
+def : SchedAlias<WriteF,    M4WriteFADD2>;
+def : SchedAlias<WriteFCmp, M4WriteNMSC2>;
+def : SchedAlias<WriteFDiv, M4WriteFDIV12>;
+def : SchedAlias<WriteFMul, M4WriteFMAC3>;
+
+// FP miscellaneous instructions.
+def : SchedAlias<WriteFCvt,  M4WriteFCVT2>;
+def : SchedAlias<WriteFImm,  M4WriteNALU1>;
+def : SchedAlias<WriteFCopy, M4WriteCOPY>;
+
+// FP load instructions.
+def : SchedAlias<WriteVLD, M4WriteL5>;
+
+// FP store instructions.
+def : SchedAlias<WriteVST, M4WriteVST1>;
+
+// ASIMD FP instructions.
+def : SchedAlias<WriteV, M4WriteNALU1>;
+
+// Other miscellaneous instructions.
+def : WriteRes<WriteAtomic,  []> { let Unsupported = 1; }
+def : WriteRes<WriteBarrier, []> { let Latency = 1; }
+def : WriteRes<WriteHint,    []> { let Latency = 1; }
+def : WriteRes<WriteSys,     []> { let Latency = 1; }
+
+//===----------------------------------------------------------------------===//
+// Generic fast forwarding.
+
+// TODO: Add FP register forwarding rules.
+
+def : ReadAdvance<ReadI,       0>;
+def : ReadAdvance<ReadISReg,   0>;
+def : ReadAdvance<ReadIEReg,   0>;
+def : ReadAdvance<ReadIM,      0>;
+// TODO: The forwarding for 32 bits actually saves 2 cycles.
+def : ReadAdvance<ReadIMA,     3, [WriteIM32, WriteIM64]>;
+def : ReadAdvance<ReadID,      0>;
+def : ReadAdvance<ReadExtrHi,  0>;
+def : ReadAdvance<ReadAdrBase, 0>;
+def : ReadAdvance<ReadVLD,     0>;
+
+//===----------------------------------------------------------------------===//
+// Finer scheduling model.
+
+// Branch instructions
+def : InstRW<[M4WriteB1], (instrs Bcc)>;
+def : InstRW<[M4WriteAF], (instrs BL)>;
+def : InstRW<[M4WriteBX], (instrs BLR)>;
+def : InstRW<[M4WriteC1], (instregex "^CBN?Z[WX]")>;
+def : InstRW<[M4WriteAD], (instregex "^TBN?Z[WX]")>;
+
+// Arithmetic and logical integer instructions.
+def : InstRW<[M4WriteAX], (instregex "^(ADD|AND|BIC|EON|EOR|ORN|SUB)[WX]rs$")>;
+def : InstRW<[M4WriteAU], (instrs ORRWrs, ORRXrs)>;
+def : InstRW<[M4WriteAX], (instregex "^(ADD|AND|BIC|SUB)S[WX]rs$")>;
+def : InstRW<[M4WriteAX], (instregex "^(ADD|SUB)S?[WX]rx(64)?$")>;
+def : InstRW<[M4WriteAV], (instrs ADDWri, ADDXri, ORRWri, ORRXri)>;
+
+// Move instructions.
+def : InstRW<[M4WriteCOPY], (instrs COPY)>;
+def : InstRW<[M4WriteZ0],   (instrs ADR, ADRP)>;
+def : InstRW<[M4WriteZ0],   (instregex "^MOV[NZ][WX]i")>;
+
+// Divide and multiply instructions.
+
+// Miscellaneous instructions.
+
+// Load instructions.
+def : InstRW<[M4WriteLD,
+              WriteLDHi,
+              WriteAdr],    (instregex "^LDP(SW|W|X)(post|pre)")>;
+def : InstRW<[M4WriteL5,
+              ReadAdrBase], (instregex "^LDR(BB|SBW|SBX|HH|SHW|SHX|SW|W|X)roW")>;
+def : InstRW<[WriteLDIdx,
+              ReadAdrBase], (instregex "^LDR(BB|SBW|SBX|HH|SHW|SHX|SW|W|X)roX")>;
+def : InstRW<[M4WriteL5,
+              ReadAdrBase], (instrs PRFMroW)>;
+def : InstRW<[WriteLDIdx,
+              ReadAdrBase], (instrs PRFMroX)>;
+
+// Store instructions.
+def : InstRW<[M4WriteSB,
+              ReadAdrBase], (instregex "^STR(BB|HH|W|X)roW")>;
+def : InstRW<[WriteST,
+              ReadAdrBase], (instregex "^STR(BB|HH|W|X)roX")>;
+
+// FP data instructions.
+def : InstRW<[M4WriteNSHF1H], (instrs FABSHr)>;
+def : InstRW<[M4WriteNSHF1],  (instregex "^FABS[SD]r")>;
+def : InstRW<[M4WriteFADD2H], (instregex "^F(ADD|SUB)Hrr")>;
+def : InstRW<[M4WriteFADD2],  (instregex "^F(ADD|SUB)[SD]rr")>;
+def : InstRW<[M4WriteFADD2H], (instregex "^FADDPv.i16")>;
+def : InstRW<[M4WriteFADD2],  (instregex "^FADDPv.i(32|64)")>;
+def : InstRW<[M4WriteNEONQ],  (instregex "^FCCMPE?[HSD]rr")>;
+def : InstRW<[M4WriteNMSC2],  (instregex "^FCMPE?[HSD]r[ir]")>;
+def : InstRW<[M4WriteNMSC1],  (instregex "^F(AC|CM)(EQ|GE|GT|LE|LT)(16|32|64|v1)")>;
+def : InstRW<[M4WriteFDIV7H], (instrs FDIVHrr)>;
+def : InstRW<[M4WriteFDIV7],  (instrs FDIVSrr)>;
+def : InstRW<[M4WriteFDIV12], (instrs FDIVDrr)>;
+def : InstRW<[M4WriteNMSC1],  (instregex "^F(MAX|MIN)(NM)?[HSD]rr")>;
+def : InstRW<[M4WriteFMAC3H], (instregex "^FN?MULHrr")>;
+def : InstRW<[M4WriteFMAC3],  (instregex "^FN?MUL[SD]rr")>;
+def : InstRW<[M4WriteFMAC3H], (instrs FMULX16)>;
+def : InstRW<[M4WriteFMAC3],  (instregex "^FMULX(32|64)")>;
+def : InstRW<[M4WriteFMAC4H,
+              M4ReadFMACM1],  (instregex "^FN?M(ADD|SUB)Hrrr")>;
+def : InstRW<[M4WriteFMAC4,
+              M4ReadFMACM1],  (instregex "^FN?M(ADD|SUB)[SD]rrr")>;
+def : InstRW<[M4WriteNALU1H], (instrs FNEGHr)>;
+def : InstRW<[M4WriteNALU1],  (instregex "^FNEG[SD]r")>;
+def : InstRW<[M4WriteFCVT3A], (instregex "^FRINT.+r")>;
+def : InstRW<[M4WriteNEONH],  (instregex "^FCSEL[HSD]rrr")>;
+def : InstRW<[M4WriteFSQR7H], (instrs FSQRTHr)>;
+def : InstRW<[M4WriteFSQR8],  (instrs FSQRTSr)>;
+def : InstRW<[M4WriteFSQR12], (instrs FSQRTDr)>;
+
+// FP miscellaneous instructions.
+def : InstRW<[M4WriteFCVT2H], (instregex "^FCVTH[SD]r")>;
+def : InstRW<[M4WriteFCVT2H], (instregex "^FCVT[SD]Hr")>;
+def : InstRW<[M4WriteFCVT2],  (instregex "^FCVT[SD][SD]r")>;
+def : InstRW<[M4WriteFCVT6A], (instregex "^[SU]CVTF[SU][XW][HSD]ri")>;
+def : InstRW<[M4WriteNEONR],  (instregex "^FCVT[AMNPZ][SU][SU][XW][HSD]r")>;
+def : InstRW<[M4WriteNALU1],  (instregex "^FMOV[HSD][ir]")>;
+def : InstRW<[M4WriteSA],     (instregex "^FMOV[WX][HSD]r")>;
+def : InstRW<[M4WriteNEONJ],  (instregex "^FMOV[HSD][WX]r")>;
+def : InstRW<[M4WriteNEONI],  (instregex "^FMOVXDHighr")>;
+def : InstRW<[M4WriteNEONK],  (instregex "^FMOVDXHighr")>;
+def : InstRW<[M4WriteFCVT3H], (instregex "^F(RECP|RSQRT)Ev1f16")>;
+def : InstRW<[M4WriteFCVT3],  (instregex "^F(RECP|RSQRT)Ev1i(32|64)")>;
+def : InstRW<[M4WriteNMSC1],  (instregex "^FRECPXv1")>;
+def : InstRW<[M4WriteFMAC4H,
+              M4ReadFMACM1],  (instregex "^F(RECP|RSQRT)S16")>;
+def : InstRW<[M4WriteFMAC4,
+              M4ReadFMACM1],  (instregex "^F(RECP|RSQRT)S(32|64)")>;
+
+// FP load instructions.
+def : InstRW<[WriteVLD],    (instregex "^LDR[SDQ]l")>;
+def : InstRW<[WriteVLD],    (instregex "^LDUR[BHSDQ]i")>;
+def : InstRW<[WriteVLD,
+              WriteAdr],    (instregex "^LDR[BHSDQ](post|pre)")>;
+def : InstRW<[WriteVLD],    (instregex "^LDR[BHSDQ]ui")>;
+def : InstRW<[M4WriteLE,
+              ReadAdrBase], (instregex "^LDR[BHSDQ]roW")>;
+def : InstRW<[WriteVLD,
+              ReadAdrBase], (instregex "^LDR[BHSD]roX")>;
+def : InstRW<[M4WriteLE,
+              ReadAdrBase], (instrs LDRQroX)>;
+def : InstRW<[WriteVLD,
+              M4WriteLH],   (instregex "^LDN?P[SD]i")>;
+def : InstRW<[M4WriteLA,
+              M4WriteLH],   (instregex "^LDN?PQi")>;
+def : InstRW<[M4WriteL5,
+              M4WriteLH,
+              WriteAdr],    (instregex "^LDP[SD]post")>;
+def : InstRW<[M4WriteLB,
+              M4WriteLH,
+              WriteAdr],    (instrs LDPQpost)>;
+def : InstRW<[M4WriteLB,
+              M4WriteLH,
+              WriteAdr],    (instregex "^LDP[SD]pre")>;
+def : InstRW<[M4WriteLC,
+              M4WriteLH,
+              WriteAdr],    (instrs LDPQpre)>;
+
+// FP store instructions.
+def : InstRW<[WriteVST],    (instregex "^STUR[BHSDQ]i")>;
+def : InstRW<[WriteVST,
+              WriteAdr],    (instregex "^STR[BHSDQ](post|pre)")>;
+def : InstRW<[WriteVST],    (instregex "^STR[BHSDQ]ui")>;
+def : InstRW<[M4WriteVSTJ,
+              ReadAdrBase], (instregex "^STR[BHSD]roW")>;
+def : InstRW<[M4WriteVSTK,
+              ReadAdrBase], (instrs STRQroW)>;
+def : InstRW<[WriteVST,
+              ReadAdrBase], (instregex "^STR[BHSD]roX")>;
+def : InstRW<[M4WriteVSTK,
+              ReadAdrBase], (instrs STRQroX)>;
+def : InstRW<[WriteVST],    (instregex "^STN?P[SD]i")>;
+def : InstRW<[M4WriteVSTA], (instregex "^STN?PQi")>;
+def : InstRW<[WriteVST,
+              WriteAdr],    (instregex "^STP[SD](post|pre)")>;
+def : InstRW<[M4WriteVSTJ,
+              WriteAdr],    (instregex "^STPQ(post|pre)")>;
+
+// ASIMD instructions.
+def : InstRW<[M4WriteNHAD1],  (instregex "^[SU]ABDL?v")>;
+def : InstRW<[M4WriteNHAD3],  (instregex "^[SU]ABAL?v")>;
+def : InstRW<[M4WriteNMSC1],  (instregex "^ABSv")>;
+def : InstRW<[M4WriteNALU1],  (instregex "^(ADD|NEG|SUB)v")>;
+def : InstRW<[M4WriteNHAD3],  (instregex "^[SU]?ADDL?Pv")>;
+def : InstRW<[M4WriteNHAD3],  (instregex "^[SU]H(ADD|SUB)v")>;
+def : InstRW<[M4WriteNHAD3],  (instregex "^[SU](ADD|SUB)[LW]v")>;
+def : InstRW<[M4WriteNHAD3],  (instregex "^R?(ADD|SUB)HN2?v")>;
+def : InstRW<[M4WriteNHAD3],  (instregex "^[SU]Q(ADD|SUB)v")>;
+def : InstRW<[M4WriteNHAD3],  (instregex "^(SU|US)QADDv")>;
+def : InstRW<[M4WriteNHAD3],  (instregex "^[SU]RHADDv")>;
+def : InstRW<[M4WriteNMSC1],  (instregex "^SQ(ABS|NEG)v")>;
+def : InstRW<[M4WriteNHAD3],  (instregex "^[SU]?ADDL?Vv")>;
+def : InstRW<[M4WriteNMSC1],  (instregex "^CM(EQ|GE|GT|HI|HS|LE|LT)v")>;
+def : InstRW<[M4WriteNALU1],  (instregex "^CMTSTv")>;
+def : InstRW<[M4WriteNALU1],  (instregex "^(AND|BIC|EOR|NOT|ORN|ORR)v")>;
+def : InstRW<[M4WriteNMSC1],  (instregex "^[SU](MIN|MAX)v")>;
+def : InstRW<[M4WriteNMSC2],  (instregex "^[SU](MIN|MAX)Pv")>;
+def : InstRW<[M4WriteNHAD3],  (instregex "^[SU](MIN|MAX)Vv")>;
+def : InstRW<[M4WriteNMUL3],  (instregex "^(SQR?D)?MULH?v")>;
+def : InstRW<[M4WriteNMUL3,
+              M4ReadNMULM1],  (instregex "^ML[AS]v")>;
+def : InstRW<[M4WriteNMUL3],  (instregex "^SQRDML[AS]H")>;
+def : InstRW<[M4WriteMULL,
+              M4ReadMULLP2],  (instregex "^(S|U|SQD)ML[AS]Lv")>;
+def : InstRW<[M4WriteMULL,
+              M4ReadMULLP2],  (instregex "^(S|U|SQD)MULLv")>;
+def : InstRW<[M4WriteNMUL3],  (instregex "^[SU]DOT(lane)?v")>;
+def : InstRW<[M4WriteNHAD3],  (instregex "^[SU]ADALPv")>;
+def : InstRW<[M4WriteNSHT4A], (instregex "^[SU]R?SRA[dv]")>;
+def : InstRW<[M4WriteNSHT1],  (instregex "^SHL[dv]")>;
+def : InstRW<[M4WriteNSHT1],  (instregex "^S[LR]I[dv]")>;
+def : InstRW<[M4WriteNSHT1],  (instregex "^[SU]SH[LR][dv]")>;
+def : InstRW<[M4WriteNSHT2],  (instregex "^[SU]?SHLLv")>;
+def : InstRW<[M4WriteNSHT4A], (instregex "^[SU]?Q?R?SHRU?N[bhsv]")>;
+def : InstRW<[M4WriteNSHT4A], (instregex "^[SU]RSH[LR][dv]")>;
+def : InstRW<[M4WriteNSHT4A], (instregex "^[SU]QR?SHLU?[bhsdv]")>;
+
+// ASIMD FP instructions.
+def : InstRW<[M4WriteNSHF1H], (instregex "^FABSv.f16")>;
+def : InstRW<[M4WriteNSHF1],  (instregex "^FABSv.f(32|64)")>;
+def : InstRW<[M4WriteFADD2H], (instregex "^F(ABD|ADD|SUB)v.f16")>;
+def : InstRW<[M4WriteFADD2],  (instregex "^F(ABD|ADD|SUB)v.f(32|64)")>;
+def : InstRW<[M4WriteFADD2H], (instregex "^FADDPv.f16")>;
+def : InstRW<[M4WriteFADD2],  (instregex "^FADDPv.f(32|64)")>;
+def : InstRW<[M4WriteNMSC1],  (instregex "^F(AC|CM)(EQ|GE|GT|LE|LT)v[^1]")>;
+def : InstRW<[M4WriteFCVT2],  (instregex "^FCVT(L|N|XN)v")>;
+def : InstRW<[M4WriteFCVT2A], (instregex "^FCVT[AMNPZ][SU]v")>;
+def : InstRW<[M4WriteFCVT2H], (instregex "^[SU]CVTFv.[fi]16")>;
+def : InstRW<[M4WriteFCVT2],  (instregex "^[SU]CVTFv.[fi](32|64)")>;
+def : InstRW<[M4WriteFDIV7H], (instrs FDIVv4f16)>;
+def : InstRW<[M4WriteNEONVH], (instrs FDIVv8f16)>;
+def : InstRW<[M4WriteFDIV7],  (instrs FDIVv2f32)>;
+def : InstRW<[M4WriteNEONV],  (instrs FDIVv4f32)>;
+def : InstRW<[M4WriteNEONW],  (instrs FDIVv2f64)>;
+def : InstRW<[M4WriteNMSC1],  (instregex "^F(MAX|MIN)(NM)?v")>;
+def : InstRW<[M4WriteNMSC2],  (instregex "^F(MAX|MIN)(NM)?Pv")>;
+def : InstRW<[M4WriteNEONZ],  (instregex "^F(MAX|MIN)(NM)?Vv")>;
+def : InstRW<[M4WriteFMAC2H], (instregex "^FMULX?v.[fi]16")>;
+def : InstRW<[M4WriteFMAC3],  (instregex "^FMULX?v.[fi](32|64)")>;
+def : InstRW<[M4WriteFMAC4H,
+              M4ReadFMACM1],  (instregex "^FML[AS]v.[fi]16")>;
+def : InstRW<[M4WriteFMAC4,
+              M4ReadFMACM1],  (instregex "^FML[AS]v.[fi](32|64)")>;
+def : InstRW<[M4WriteNALU1H], (instregex "^FNEGv.f16")>;
+def : InstRW<[M4WriteNALU1],  (instregex "^FNEGv.f(32|64)")>;
+def : InstRW<[M4WriteFCVT3A], (instregex "^FRINT[AIMNPXZ]v")>;
+def : InstRW<[M4WriteFSQR7H], (instrs FSQRTv4f16)>;
+def : InstRW<[M4WriteNEONXH], (instrs FSQRTv8f16)>;
+def : InstRW<[M4WriteFSQR8],  (instrs FSQRTv2f32)>;
+def : InstRW<[M4WriteNEONX],  (instrs FSQRTv4f32)>;
+def : InstRW<[M4WriteNEONY],  (instrs FSQRTv2f64)>;
+
+// ASIMD miscellaneous instructions.
+def : InstRW<[M4WriteNALU1],  (instregex "^RBITv")>;
+def : InstRW<[M4WriteNALU1],  (instregex "^(BIF|BIT|BSL)v")>;
+def : InstRW<[M4WriteNALU1],  (instregex "^CL[STZ]v")>;
+def : InstRW<[M4WriteNEONB],  (instregex "^DUPv.+gpr")>;
+def : InstRW<[M4WriteNSHF1],  (instregex "^CPY")>;
+def : InstRW<[M4WriteNSHF1],  (instregex "^DUPv.+lane")>;
+def : InstRW<[M4WriteNSHF1],  (instregex "^EXTv")>;
+def : InstRW<[M4WriteNSHT4A], (instregex "^XTNv")>;
+def : InstRW<[M4WriteNSHT4A], (instregex "^[SU]?QXTU?Nv")>;
+def : InstRW<[M4WriteNEONB],  (instregex "^INSv.+gpr")>;
+def : InstRW<[M4WriteNSHF1],  (instregex "^INSv.+lane")>;
+def : InstRW<[M4WriteMOVI],   (instregex "^(MOV|MVN)I")>;
+def : InstRW<[M4WriteNALU1H], (instregex "^FMOVv.f16")>;
+def : InstRW<[M4WriteNALU1],  (instregex "^FMOVv.f(32|64)")>;
+def : InstRW<[M4WriteFCVT3H], (instregex "^F(RECP|RSQRT)Ev[248]f16")>;
+def : InstRW<[M4WriteFCVT3],  (instregex "^F(RECP|RSQRT)Ev[248]f(32|64)")>;
+def : InstRW<[M4WriteFCVT3],  (instregex "^U(RECP|RSQRT)Ev[24]i32")>;
+def : InstRW<[M4WriteFMAC4H,
+              M4ReadFMACM1],  (instregex "^F(RECP|RSQRT)Sv.f16")>;
+def : InstRW<[M4WriteFMAC4,
+              M4ReadFMACM1],  (instregex "^F(RECP|RSQRT)Sv.f(32|64)")>;
+def : InstRW<[M4WriteNSHF1],  (instregex "^REV(16|32|64)v")>;
+def : InstRW<[M4WriteNSHFA],  (instregex "^TB[LX]v(8|16)i8One")>;
+def : InstRW<[M4WriteNSHFB],  (instregex "^TB[LX]v(8|16)i8Two")>;
+def : InstRW<[M4WriteNSHFC],  (instregex "^TB[LX]v(8|16)i8Three")>;
+def : InstRW<[M4WriteNSHFD],  (instregex "^TB[LX]v(8|16)i8Four")>;
+def : InstRW<[M4WriteNEONP],  (instregex "^[SU]MOVv")>;
+def : InstRW<[M4WriteNSHF1],  (instregex "^(TRN|UZP|ZIP)[12]v")>;
+
+// ASIMD load instructions.
+def : InstRW<[WriteVLD],    (instregex "LD1Onev(8b|4h|2s|1d)$")>;
+def : InstRW<[WriteVLD,
+              M4WriteA1],   (instregex "LD1Onev(8b|4h|2s|1d)_POST$")>;
+def : InstRW<[WriteVLD],    (instregex "LD1Onev(16b|8h|4s|2d)$")>;
+def : InstRW<[WriteVLD,
+              M4WriteA1],   (instregex "LD1Onev(16b|8h|4s|2d)_POST$")>;
+
+def : InstRW<[M4WriteVLDA], (instregex "LD1Twov(8b|4h|2s|1d)$")>;
+def : InstRW<[M4WriteVLDA,
+              M4WriteA1],   (instregex "LD1Twov(8b|4h|2s|1d)_POST$")>;
+def : InstRW<[M4WriteVLDA], (instregex "LD1Twov(16b|8h|4s|2d)$")>;
+def : InstRW<[M4WriteVLDA,
+              M4WriteA1],   (instregex "LD1Twov(16b|8h|4s|2d)_POST$")>;
+
+def : InstRW<[M4WriteVLDB], (instregex "LD1Threev(8b|4h|2s|1d)$")>;
+def : InstRW<[M4WriteVLDB,
+              M4WriteA1],   (instregex "LD1Threev(8b|4h|2s|1d)_POST$")>;
+def : InstRW<[M4WriteVLDB], (instregex "LD1Threev(16b|8h|4s|2d)$")>;
+def : InstRW<[M4WriteVLDB,
+              M4WriteA1],   (instregex "LD1Threev(16b|8h|4s|2d)_POST$")>;
+
+def : InstRW<[M4WriteVLDC], (instregex "LD1Fourv(8b|4h|2s|1d)$")>;
+def : InstRW<[M4WriteVLDC,
+              M4WriteA1],   (instregex "LD1Fourv(8b|4h|2s|1d)_POST$")>;
+def : InstRW<[M4WriteVLDC], (instregex "LD1Fourv(16b|8h|4s|2d)$")>;
+def : InstRW<[M4WriteVLDC,
+              M4WriteA1],   (instregex "LD1Fourv(16b|8h|4s|2d)_POST$")>;
+
+def : InstRW<[M4WriteVLDD], (instregex "LD1i(8|16|32|64)$")>;
+def : InstRW<[M4WriteVLDD,
+              M4WriteA1],   (instregex "LD1i(8|16|32|64)_POST$")>;
+
+def : InstRW<[WriteVLD],    (instregex "LD1Rv(8b|4h|2s|1d)$")>;
+def : InstRW<[WriteVLD,
+              M4WriteA1],   (instregex "LD1Rv(8b|4h|2s|1d)_POST$")>;
+def : InstRW<[WriteVLD],    (instregex "LD1Rv(16b|8h|4s|2d)$")>;
+def : InstRW<[WriteVLD,
+              M4WriteA1],   (instregex "LD1Rv(16b|8h|4s|2d)_POST$")>;
+
+def : InstRW<[M4WriteVLDF], (instregex "LD2Twov(8b|4h|2s)$")>;
+def : InstRW<[M4WriteVLDF,
+              M4WriteA1],   (instregex "LD2Twov(8b|4h|2s)_POST$")>;
+def : InstRW<[M4WriteVLDF], (instregex "LD2Twov(16b|8h|4s|2d)$")>;
+def : InstRW<[M4WriteVLDF,
+              M4WriteA1],   (instregex "LD2Twov(16b|8h|4s|2d)_POST$")>;
+
+def : InstRW<[M4WriteVLDG], (instregex "LD2i(8|16|32|64)$")>;
+def : InstRW<[M4WriteVLDG,
+              M4WriteA1],   (instregex "LD2i(8|16|32|64)_POST$")>;
+
+def : InstRW<[M4WriteVLDA], (instregex "LD2Rv(8b|4h|2s|1d)$")>;
+def : InstRW<[M4WriteVLDA,
+              M4WriteA1],   (instregex "LD2Rv(8b|4h|2s|1d)_POST$")>;
+def : InstRW<[M4WriteVLDA], (instregex "LD2Rv(16b|8h|4s|2d)$")>;
+def : InstRW<[M4WriteVLDA,
+              M4WriteA1],   (instregex "LD2Rv(16b|8h|4s|2d)_POST$")>;
+
+def : InstRW<[M4WriteVLDI], (instregex "LD3Threev(8b|4h|2s)$")>;
+def : InstRW<[M4WriteVLDI,
+              M4WriteA1],   (instregex "LD3Threev(8b|4h|2s)_POST$")>;
+def : InstRW<[M4WriteVLDI], (instregex "LD3Threev(16b|8h|4s|2d)$")>;
+def : InstRW<[M4WriteVLDI,
+              M4WriteA1],   (instregex "LD3Threev(16b|8h|4s|2d)_POST$")>;
+
+def : InstRW<[M4WriteVLDJ], (instregex "LD3i(8|16|32)$")>;
+def : InstRW<[M4WriteVLDJ,
+              M4WriteA1],   (instregex "LD3i(8|16|32)_POST$")>;
+def : InstRW<[M4WriteVLDL], (instregex "LD3i64$")>;
+def : InstRW<[M4WriteVLDL,
+              M4WriteA1],   (instregex "LD3i64_POST$")>;
+
+def : InstRW<[M4WriteVLDB], (instregex "LD3Rv(8b|4h|2s|1d)$")>;
+def : InstRW<[M4WriteVLDB,
+              M4WriteA1],   (instregex "LD3Rv(8b|4h|2s|1d)_POST$")>;
+def : InstRW<[M4WriteVLDB], (instregex "LD3Rv(16b|8h|4s|2d)$")>;
+def : InstRW<[M4WriteVLDB,
+              M4WriteA1],   (instregex "LD3Rv(16b|8h|4s|2d)_POST$")>;
+
+def : InstRW<[M4WriteVLDN], (instregex "LD4Fourv(8b|4h|2s)$")>;
+def : InstRW<[M4WriteVLDN,
+              M4WriteA1],   (instregex "LD4Fourv(8b|4h|2s)_POST$")>;
+def : InstRW<[M4WriteVLDN], (instregex "LD4Fourv(16b|8h|4s|2d)$")>;
+def : InstRW<[M4WriteVLDN,
+              M4WriteA1],   (instregex "LD4Fourv(16b|8h|4s|2d)_POST$")>;
+
+def : InstRW<[M4WriteVLDK], (instregex "LD4i(8|16|32)$")>;
+def : InstRW<[M4WriteVLDK,
+              M4WriteA1],   (instregex "LD4i(8|16|32)_POST$")>;
+def : InstRW<[M4WriteVLDM], (instregex "LD4i64$")>;
+def : InstRW<[M4WriteVLDM,
+              M4WriteA1],   (instregex "LD4i64_POST$")>;
+
+def : InstRW<[M4WriteVLDC], (instregex "LD4Rv(8b|4h|2s|1d)$")>;
+def : InstRW<[M4WriteVLDC,
+              M4WriteA1],   (instregex "LD4Rv(8b|4h|2s|1d)_POST$")>;
+def : InstRW<[M4WriteVLDC], (instregex "LD4Rv(16b|8h|4s|2d)$")>;
+def : InstRW<[M4WriteVLDC,
+              M4WriteA1],   (instregex "LD4Rv(16b|8h|4s|2d)_POST$")>;
+
+// ASIMD store instructions.
+def : InstRW<[WriteVST],    (instregex "ST1Onev(8b|4h|2s|1d)$")>;
+def : InstRW<[WriteVST,
+              M4WriteA1],   (instregex "ST1Onev(8b|4h|2s|1d)_POST$")>;
+def : InstRW<[WriteVST],    (instregex "ST1Onev(16b|8h|4s|2d)$")>;
+def : InstRW<[WriteVST,
+              M4WriteA1],   (instregex "ST1Onev(16b|8h|4s|2d)_POST$")>;
+
+def : InstRW<[M4WriteVSTA], (instregex "ST1Twov(8b|4h|2s|1d)$")>;
+def : InstRW<[M4WriteVSTA,
+              M4WriteA1],   (instregex "ST1Twov(8b|4h|2s|1d)_POST$")>;
+def : InstRW<[M4WriteVSTA], (instregex "ST1Twov(16b|8h|4s|2d)$")>;
+def : InstRW<[M4WriteVSTA,
+              M4WriteA1],   (instregex "ST1Twov(16b|8h|4s|2d)_POST$")>;
+
+def : InstRW<[M4WriteVSTB], (instregex "ST1Threev(8b|4h|2s|1d)$")>;
+def : InstRW<[M4WriteVSTB,
+              M4WriteA1],   (instregex "ST1Threev(8b|4h|2s|1d)_POST$")>;
+def : InstRW<[M4WriteVSTB], (instregex "ST1Threev(16b|8h|4s|2d)$")>;
+def : InstRW<[M4WriteVSTB,
+              M4WriteA1],   (instregex "ST1Threev(16b|8h|4s|2d)_POST$")>;
+
+def : InstRW<[M4WriteVSTC], (instregex "ST1Fourv(8b|4h|2s|1d)$")>;
+def : InstRW<[M4WriteVSTC,
+              M4WriteA1],   (instregex "ST1Fourv(8b|4h|2s|1d)_POST$")>;
+def : InstRW<[M4WriteVSTC], (instregex "ST1Fourv(16b|8h|4s|2d)$")>;
+def : InstRW<[M4WriteVSTC,
+              M4WriteA1],   (instregex "ST1Fourv(16b|8h|4s|2d)_POST$")>;
+
+def : InstRW<[WriteVST],    (instregex "ST1i(8|16|32|64)$")>;
+def : InstRW<[WriteVST,
+              M4WriteA1],   (instregex "ST1i(8|16|32|64)_POST$")>;
+
+def : InstRW<[M4WriteVSTD], (instregex "ST2Twov(8b|4h|2s)$")>;
+def : InstRW<[M4WriteVSTD,
+              M4WriteA1],   (instregex "ST2Twov(8b|4h|2s)_POST$")>;
+def : InstRW<[M4WriteVSTE], (instregex "ST2Twov(16b|8h|4s|2d)$")>;
+def : InstRW<[M4WriteVSTE,
+              M4WriteA1],   (instregex "ST2Twov(16b|8h|4s|2d)_POST$")>;
+
+def : InstRW<[M4WriteVSTD], (instregex "ST2i(8|16|32|64)$")>;
+def : InstRW<[M4WriteVSTD,
+              M4WriteA1],   (instregex "ST2i(8|16|32|64)_POST$")>;
+
+def : InstRW<[M4WriteVSTF], (instregex "ST3Threev(8b|4h|2s)$")>;
+def : InstRW<[M4WriteVSTF,
+              M4WriteA1],   (instregex "ST3Threev(8b|4h|2s)_POST$")>;
+def : InstRW<[M4WriteVSTG], (instregex "ST3Threev(16b|8h|4s|2d)$")>;
+def : InstRW<[M4WriteVSTG,
+              M4WriteA1],   (instregex "ST3Threev(16b|8h|4s|2d)_POST$")>;
+
+def : InstRW<[M4WriteVSTE], (instregex "ST3i(8|16|32|64)$")>;
+def : InstRW<[M4WriteVSTE,
+              M4WriteA1],   (instregex "ST3i(8|16|32|64)_POST$")>;
+
+def : InstRW<[M4WriteVSTL], (instregex "ST4Fourv(8b|4h|2s)$")>;
+def : InstRW<[M4WriteVSTL,
+              M4WriteA1],   (instregex "ST4Fourv(8b|4h|2s)_POST$")>;
+def : InstRW<[M4WriteVSTI], (instregex "ST4Fourv(16b|8h|4s|2d)$")>;
+def : InstRW<[M4WriteVSTI,
+              M4WriteA1],   (instregex "ST4Fourv(16b|8h|4s|2d)_POST$")>;
+
+def : InstRW<[M4WriteVSTE], (instregex "ST4i(8|16|32|64)$")>;
+def : InstRW<[M4WriteVSTE,
+              M4WriteA1],   (instregex "ST4i(8|16|32|64)_POST$")>;
+
+// Cryptography instructions.
+def : InstRW<[M4WriteNCRY1],  (instregex "^AES[DE]")>;
+def : InstRW<[M4WriteNCRY1,
+              M4ReadAESM1],   (instregex "^AESI?MC")>;
+def : InstRW<[M4WriteNCRY1A], (instregex "^PMULv")>;
+def : InstRW<[M4WriteNCRY1A], (instregex "^PMULLv(1|8)i")>;
+def : InstRW<[M4WriteNCRY3A], (instregex "^PMULLv(2|16)i")>;
+def : InstRW<[M4WriteNCRY1A], (instregex "^SHA1([CHMP]|SU[01])")>;
+def : InstRW<[M4WriteNCRY1A], (instrs SHA256SU0rr)>;
+def : InstRW<[M4WriteNCRY5A], (instrs SHA256SU1rrr)>;
+def : InstRW<[M4WriteNCRY5A], (instrs SHA256H2rrr)>;
+
+// CRC instructions.
+def : InstRW<[M4WriteE2], (instregex "^CRC32C?[BHWX]rr$")>;
+
+} // SchedModel = ExynosM4Model
diff --git a/lib/Target/AArch64/AArch64SchedFalkor.td b/lib/Target/AArch64/AArch64SchedFalkor.td
index 84825458e47c013df97907cd85cad5564a8cf071..c5d3287ee1fad2b96ee53bb36c28b88f4e07294c 100644
--- a/lib/Target/AArch64/AArch64SchedFalkor.td
+++ b/lib/Target/AArch64/AArch64SchedFalkor.td
@@ -1,9 +1,8 @@
 //==- AArch64SchedFalkor.td - Falkor Scheduling Definitions -*- tablegen -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AArch64/AArch64SchedFalkorDetails.td b/lib/Target/AArch64/AArch64SchedFalkorDetails.td
index ff14e639d1a57cd02bb0152a4a8b103d6a697c21..697a0f69c58cbf8510e93ddf15c3f3f5e85e7be1 100644
--- a/lib/Target/AArch64/AArch64SchedFalkorDetails.td
+++ b/lib/Target/AArch64/AArch64SchedFalkorDetails.td
@@ -1,9 +1,8 @@
 //==- AArch64SchedFalkorDetails.td - Falkor Scheduling Defs -*- tablegen -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AArch64/AArch64SchedKryo.td b/lib/Target/AArch64/AArch64SchedKryo.td
index 68de3e077c9645c2eaeb4a196386137550ac08c3..25e27c32243677e6cc067b51b070073a10c15d92 100644
--- a/lib/Target/AArch64/AArch64SchedKryo.td
+++ b/lib/Target/AArch64/AArch64SchedKryo.td
@@ -1,9 +1,8 @@
 //==- AArch64SchedKryo.td - Qualcomm Kryo Scheduling Defs ---*- tablegen -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AArch64/AArch64SchedKryoDetails.td b/lib/Target/AArch64/AArch64SchedKryoDetails.td
index cf4cdabb8cbfc57431d5d68f59a7703317eb98e6..4c60992e6351a28840b24e28666dad4e394aacf0 100644
--- a/lib/Target/AArch64/AArch64SchedKryoDetails.td
+++ b/lib/Target/AArch64/AArch64SchedKryoDetails.td
@@ -1,9 +1,8 @@
 //=- AArch64SchedKryoDetails.td - QC Kryo Scheduling Defs ----*- tablegen -*-=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AArch64/AArch64SchedPredExynos.td b/lib/Target/AArch64/AArch64SchedPredExynos.td
index f8533d18022af3108248f201451a0301b66f34b0..150bff0e1a1dad645076abbd2f48387d1dc7ba06 100644
--- a/lib/Target/AArch64/AArch64SchedPredExynos.td
+++ b/lib/Target/AArch64/AArch64SchedPredExynos.td
@@ -1,9 +1,8 @@
 //===- AArch64SchedPredExynos.td - AArch64 Sched Preds -----*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -30,24 +29,61 @@ def ExynosBranchLinkLRPred : MCSchedPredicate<
                                CheckAll<[CheckOpcode<[BLR]>,
                                          CheckRegOperand<0, LR>]>>;
 
-// Identify arithmetic and logic instructions without or with limited extension.
-def ExynosExtFn   : TIIPredicate<
-                      "isExynosExtFast",
-                      MCOpcodeSwitchStatement<
-                        [MCOpcodeSwitchCase<
-                           IsArithExtOp.ValidOpcodes,
-                           MCReturnStatement<
-                             CheckAny<[CheckExtBy0,
-                                       CheckAll<
-                                         [CheckAny<
-                                           [CheckExtUXTW,
-                                            CheckExtUXTX]>,
-                                          CheckAny<
-                                            [CheckExtBy1,
-                                             CheckExtBy2,
-                                             CheckExtBy3]>]>]>>>],
-                        MCReturnStatement<FalsePred>>>;
-def ExynosExtPred : MCSchedPredicate<ExynosExtFn>;
+// Identify arithmetic instructions without or with limited extension or shift.
+def ExynosArithFn   : TIIPredicate<
+                        "isExynosArithFast",
+                        MCOpcodeSwitchStatement<
+                          [MCOpcodeSwitchCase<
+                             IsArithExtOp.ValidOpcodes,
+                             MCReturnStatement<
+                               CheckAny<[CheckExtBy0,
+                                         CheckAll<
+                                           [CheckAny<
+                                             [CheckExtUXTW,
+                                              CheckExtUXTX]>,
+                                            CheckAny<
+                                              [CheckExtBy1,
+                                               CheckExtBy2,
+                                               CheckExtBy3]>]>]>>>,
+                           MCOpcodeSwitchCase<
+                             IsArithShiftOp.ValidOpcodes,
+                             MCReturnStatement<ExynosCheckShift>>,
+                           MCOpcodeSwitchCase<
+                             IsArithUnshiftOp.ValidOpcodes,
+                             MCReturnStatement<TruePred>>],
+                          MCReturnStatement<FalsePred>>>;
+def ExynosArithPred : MCSchedPredicate<ExynosArithFn>;
+
+// Identify logic instructions with limited shift.
+def ExynosLogicFn   : TIIPredicate<
+                        "isExynosLogicFast",
+                        MCOpcodeSwitchStatement<
+                          [MCOpcodeSwitchCase<
+                             IsLogicShiftOp.ValidOpcodes,
+                             MCReturnStatement<ExynosCheckShift>>,
+                           MCOpcodeSwitchCase<
+                             IsLogicUnshiftOp.ValidOpcodes,
+                             MCReturnStatement<TruePred>>],
+                          MCReturnStatement<FalsePred>>>;
+def ExynosLogicPred : MCSchedPredicate<ExynosLogicFn>;
+
+// Identify more logic instructions with limited shift.
+def ExynosLogicExFn   : TIIPredicate<
+                          "isExynosLogicExFast",
+                          MCOpcodeSwitchStatement<
+                            [MCOpcodeSwitchCase<
+                               IsLogicShiftOp.ValidOpcodes,
+                               MCReturnStatement<
+                                 CheckAny<
+                                   [ExynosCheckShift,
+                                    CheckAll<
+                                     [CheckShiftLSL,
+                                      CheckShiftBy8]>]>>>,
+                             MCOpcodeSwitchCase<
+                               IsLogicUnshiftOp.ValidOpcodes,
+                               MCReturnStatement<TruePred>>],
+                            MCReturnStatement<FalsePred>>>;
+def ExynosLogicExPred : MCSchedPredicate<ExynosLogicExFn>;
 
 // Identify a load or store using the register offset addressing mode
 // with a scaled non-extended register.
@@ -88,12 +124,19 @@ def ExynosResetFn   : TIIPredicate<
                              [ADR, ADRP,
                               MOVNWi, MOVNXi,
                               MOVZWi, MOVZXi],
-                             MCReturnStatement<TruePred>>],
+                             MCReturnStatement<TruePred>>,
+                           MCOpcodeSwitchCase<
+                             [ORRWri, ORRXri],
+                             MCReturnStatement<
+                               CheckAll<
+                                 [CheckIsRegOperand<1>,
+                                  CheckAny<
+                                    [CheckRegOperand<1, WZR>,
+                                     CheckRegOperand<1, XZR>]>]>>>],
                           MCReturnStatement<
                             CheckAny<
                               [IsCopyIdiomFn,
-                               IsZeroFPIdiomFn,
-                               IsZeroIdiomFn]>>>>;
+                               IsZeroFPIdiomFn]>>>>;
 def ExynosResetPred : MCSchedPredicate<ExynosResetFn>;
 
 // Identify EXTR as the alias for ROR (immediate).
@@ -101,33 +144,7 @@ def ExynosRotateRightImmPred : MCSchedPredicate<
                                  CheckAll<[CheckOpcode<[EXTRWrri, EXTRXrri]>,
                                            CheckSameRegOperand<1, 2>]>>;
 
-// Identify arithmetic and logic instructions with limited shift.
-def ExynosShiftFn   : TIIPredicate<
-                        "isExynosShiftFast",
-                        MCOpcodeSwitchStatement<
-                          [MCOpcodeSwitchCase<
-                             IsArithLogicShiftOp.ValidOpcodes,
-                             MCReturnStatement<ExynosCheckShift>>],
-                          MCReturnStatement<FalsePred>>>;
-def ExynosShiftPred : MCSchedPredicate<ExynosShiftFn>;
-
-// Identify more arithmetic and logic instructions with limited shift.
-def ExynosShiftExFn   : TIIPredicate<
-                          "isExynosShiftExFast",
-                          MCOpcodeSwitchStatement<
-                            [MCOpcodeSwitchCase<
-                               IsArithLogicShiftOp.ValidOpcodes,
-                               MCReturnStatement<
-                                 CheckAny<
-                                   [CheckAll<
-                                     [CheckShiftLSL,
-                                      CheckShiftBy8]>,
-                                    ExynosCheckShift]>>>],
-                            MCReturnStatement<FalsePred>>>;
-def ExynosShiftExPred : MCSchedPredicate<ExynosShiftExFn>;
-
-
-// Identify arithmetic and logic immediate instructions.
+// Identify cheap arithmetic and logic immediate instructions.
 def ExynosCheapFn : TIIPredicate<
                       "isExynosCheapAsMove",
                       MCOpcodeSwitchStatement<
@@ -136,4 +153,4 @@ def ExynosCheapFn : TIIPredicate<
                            MCReturnStatement<TruePred>>],
                         MCReturnStatement<
                           CheckAny<
-                            [ExynosExtFn, ExynosResetFn, ExynosShiftFn]>>>>;
+                            [ExynosArithFn, ExynosResetFn, ExynosLogicFn]>>>>;
diff --git a/lib/Target/AArch64/AArch64SchedPredicates.td b/lib/Target/AArch64/AArch64SchedPredicates.td
index a48f1dcffaffadd70758855a6735d01ee177af32..db15b026c3d80ce191028bc84b8aa1870c7f252a 100644
--- a/lib/Target/AArch64/AArch64SchedPredicates.td
+++ b/lib/Target/AArch64/AArch64SchedPredicates.td
@@ -1,9 +1,8 @@
 //===- AArch64SchedPredicates.td - AArch64 Sched Preds -----*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -182,6 +181,92 @@ def IsArithLogicShiftOp    : CheckOpcode<!listconcat(IsArithShiftOp.ValidOpcodes
 def IsArithLogicUnshiftOp  : CheckOpcode<!listconcat(IsArithUnshiftOp.ValidOpcodes,
                                                      IsLogicUnshiftOp.ValidOpcodes)>;
 
+// Identify whether an instruction is an ASIMD
+// load using the post index addressing mode.
+def IsLoadASIMDPostOp      : CheckOpcode<[LD1Onev8b_POST, LD1Onev4h_POST, LD1Onev2s_POST, LD1Onev1d_POST,
+                                          LD1Onev16b_POST, LD1Onev8h_POST, LD1Onev4s_POST, LD1Onev2d_POST,
+                                          LD1Twov8b_POST, LD1Twov4h_POST, LD1Twov2s_POST, LD1Twov1d_POST,
+                                          LD1Twov16b_POST, LD1Twov8h_POST, LD1Twov4s_POST, LD1Twov2d_POST,
+                                          LD1Threev8b_POST, LD1Threev4h_POST, LD1Threev2s_POST, LD1Threev1d_POST,
+                                          LD1Threev16b_POST, LD1Threev8h_POST, LD1Threev4s_POST, LD1Threev2d_POST,
+                                          LD1Fourv8b_POST, LD1Fourv4h_POST, LD1Fourv2s_POST, LD1Fourv1d_POST,
+                                          LD1Fourv16b_POST, LD1Fourv8h_POST, LD1Fourv4s_POST, LD1Fourv2d_POST,
+                                          LD1i8_POST, LD1i16_POST, LD1i32_POST, LD1i64_POST,
+                                          LD1Rv8b_POST, LD1Rv4h_POST, LD1Rv2s_POST, LD1Rv1d_POST,
+                                          LD1Rv16b_POST, LD1Rv8h_POST, LD1Rv4s_POST, LD1Rv2d_POST,
+                                          LD2Twov8b_POST, LD2Twov4h_POST, LD2Twov2s_POST,
+                                          LD2Twov16b_POST, LD2Twov8h_POST, LD2Twov4s_POST, LD2Twov2d_POST,
+                                          LD2i8_POST, LD2i16_POST, LD2i32_POST, LD2i64_POST,
+                                          LD2Rv8b_POST, LD2Rv4h_POST, LD2Rv2s_POST, LD2Rv1d_POST,
+                                          LD2Rv16b_POST, LD2Rv8h_POST, LD2Rv4s_POST, LD2Rv2d_POST,
+                                          LD3Threev8b_POST, LD3Threev4h_POST, LD3Threev2s_POST,
+                                          LD3Threev16b_POST, LD3Threev8h_POST, LD3Threev4s_POST, LD3Threev2d_POST,
+                                          LD3i8_POST, LD3i16_POST, LD3i32_POST, LD3i64_POST,
+                                          LD3Rv8b_POST, LD3Rv4h_POST, LD3Rv2s_POST, LD3Rv1d_POST,
+                                          LD3Rv16b_POST, LD3Rv8h_POST, LD3Rv4s_POST, LD3Rv2d_POST,
+                                          LD4Fourv8b_POST, LD4Fourv4h_POST, LD4Fourv2s_POST,
+                                          LD4Fourv16b_POST, LD4Fourv8h_POST, LD4Fourv4s_POST, LD4Fourv2d_POST,
+                                          LD4i8_POST, LD4i16_POST, LD4i32_POST, LD4i64_POST,
+                                          LD4Rv8b_POST, LD4Rv4h_POST, LD4Rv2s_POST, LD4Rv1d_POST,
+                                          LD4Rv16b_POST, LD4Rv8h_POST, LD4Rv4s_POST, LD4Rv2d_POST]>;
+
+// Identify whether an instruction is an ASIMD
+// store using the post index addressing mode.
+def IsStoreASIMDPostOp     : CheckOpcode<[ST1Onev8b_POST, ST1Onev4h_POST, ST1Onev2s_POST, ST1Onev1d_POST,
+                                          ST1Onev16b_POST, ST1Onev8h_POST, ST1Onev4s_POST, ST1Onev2d_POST,
+                                          ST1Twov8b_POST, ST1Twov4h_POST, ST1Twov2s_POST, ST1Twov1d_POST,
+                                          ST1Twov16b_POST, ST1Twov8h_POST, ST1Twov4s_POST, ST1Twov2d_POST,
+                                          ST1Threev8b_POST, ST1Threev4h_POST, ST1Threev2s_POST, ST1Threev1d_POST,
+                                          ST1Threev16b_POST, ST1Threev8h_POST, ST1Threev4s_POST, ST1Threev2d_POST,
+                                          ST1Fourv8b_POST, ST1Fourv4h_POST, ST1Fourv2s_POST, ST1Fourv1d_POST,
+                                          ST1Fourv16b_POST, ST1Fourv8h_POST, ST1Fourv4s_POST, ST1Fourv2d_POST,
+                                          ST1i8_POST, ST1i16_POST, ST1i32_POST, ST1i64_POST,
+                                          ST2Twov8b_POST, ST2Twov4h_POST, ST2Twov2s_POST,
+                                          ST2Twov16b_POST, ST2Twov8h_POST, ST2Twov4s_POST, ST2Twov2d_POST,
+                                          ST2i8_POST, ST2i16_POST, ST2i32_POST, ST2i64_POST,
+                                          ST3Threev8b_POST, ST3Threev4h_POST, ST3Threev2s_POST,
+                                          ST3Threev16b_POST, ST3Threev8h_POST, ST3Threev4s_POST, ST3Threev2d_POST,
+                                          ST3i8_POST, ST3i16_POST, ST3i32_POST, ST3i64_POST,
+                                          ST4Fourv8b_POST, ST4Fourv4h_POST, ST4Fourv2s_POST,
+                                          ST4Fourv16b_POST, ST4Fourv8h_POST, ST4Fourv4s_POST, ST4Fourv2d_POST,
+                                          ST4i8_POST, ST4i16_POST, ST4i32_POST, ST4i64_POST]>;
+
+// Identify whether an instruction is an ASIMD load
+// or store using the post index addressing mode.
+def IsLoadStoreASIMDPostOp : CheckOpcode<!listconcat(IsLoadASIMDPostOp.ValidOpcodes,
+                                                     IsStoreASIMDPostOp.ValidOpcodes)>;
+
+// Identify whether an instruction is a load
+// using the register offset addressing mode.
+def IsLoadRegOffsetOp      : CheckOpcode<[PRFMroW, PRFMroX,
+                                          LDRBBroW, LDRBBroX,
+                                          LDRSBWroW, LDRSBWroX, LDRSBXroW, LDRSBXroX,
+                                          LDRHHroW, LDRHHroX,
+                                          LDRSHWroW, LDRSHWroX, LDRSHXroW, LDRSHXroX,
+                                          LDRWroW, LDRWroX,
+                                          LDRSWroW, LDRSWroX,
+                                          LDRXroW, LDRXroX,
+                                          LDRBroW, LDRBroX,
+                                          LDRHroW, LDRHroX,
+                                          LDRSroW, LDRSroX,
+                                          LDRDroW, LDRDroX]>;
+
+// Identify whether an instruction is a load
+// using the register offset addressing mode.
+def IsStoreRegOffsetOp     : CheckOpcode<[STRBBroW, STRBBroX,
+                                          STRHHroW, STRHHroX,
+                                          STRWroW, STRWroX,
+                                          STRXroW, STRXroX,
+                                          STRBroW, STRBroX,
+                                          STRHroW, STRHroX,
+                                          STRSroW, STRSroX,
+                                          STRDroW, STRDroX]>;
+
+// Identify whether an instruction is a load or
+// store using the register offset addressing mode.
+def IsLoadStoreRegOffsetOp : CheckOpcode<!listconcat(IsLoadRegOffsetOp.ValidOpcodes,
+                                                     IsStoreRegOffsetOp.ValidOpcodes)>;
+
 // Identify whether an instruction whose result is a long vector
 // operates on the upper half of the input registers.
 def IsLongVectorUpperOp    : CheckOpcode<[FCVTLv8i16, FCVTLv4i32,
@@ -235,37 +320,6 @@ def IsLongVectorUpperOp    : CheckOpcode<[FCVTLv8i16, FCVTLv4i32,
                                           USUBWv16i8_v8i16, USUBWv8i16_v4i32, USUBWv4i32_v2i64,
                                           XTNv16i8, XTNv8i16, XTNv4i32]>;
 
-// Identify whether an instruction is a load
-// using the register offset addressing mode.
-def IsLoadRegOffsetOp      : CheckOpcode<[PRFMroW, PRFMroX,
-                                          LDRBBroW, LDRBBroX,
-                                          LDRSBWroW, LDRSBWroX, LDRSBXroW, LDRSBXroX,
-                                          LDRHHroW, LDRHHroX,
-                                          LDRSHWroW, LDRSHWroX, LDRSHXroW, LDRSHXroX,
-                                          LDRWroW, LDRWroX,
-                                          LDRSWroW, LDRSWroX,
-                                          LDRXroW, LDRXroX,
-                                          LDRBroW, LDRBroX,
-                                          LDRHroW, LDRHroX,
-                                          LDRSroW, LDRSroX,
-                                          LDRDroW, LDRDroX]>;
-
-// Identify whether an instruction is a load
-// using the register offset addressing mode.
-def IsStoreRegOffsetOp     : CheckOpcode<[STRBBroW, STRBBroX,
-                                          STRHHroW, STRHHroX,
-                                          STRWroW, STRWroX,
-                                          STRXroW, STRXroX,
-                                          STRBroW, STRBroX,
-                                          STRHroW, STRHroX,
-                                          STRSroW, STRSroX,
-                                          STRDroW, STRDroX]>;
-
-// Identify whether an instruction is a load or
-// store using the register offset addressing mode.
-def IsLoadStoreRegOffsetOp : CheckOpcode<!listconcat(IsLoadRegOffsetOp.ValidOpcodes,
-                                                     IsStoreRegOffsetOp.ValidOpcodes)>;
-
 // Target predicates.
 
 // Identify an instruction that effectively transfers a register to another.
@@ -295,9 +349,7 @@ def IsCopyIdiomFn     : TIIPredicate<"isCopyIdiom",
                                                CheckIsRegOperand<2>,
                                                CheckAny<
                                                  [CheckRegOperand<1, WZR>,
-                                                  CheckRegOperand<1, XZR>,
-                                                  CheckRegOperand<2, WZR>,
-                                                  CheckRegOperand<2, XZR>]>,
+                                                  CheckRegOperand<1, XZR>]>,
                                                CheckShiftBy0]>>>],
                                        MCReturnStatement<FalsePred>>>;
 def IsCopyIdiomPred   : MCSchedPredicate<IsCopyIdiomFn>;
diff --git a/lib/Target/AArch64/AArch64SchedThunderX.td b/lib/Target/AArch64/AArch64SchedThunderX.td
index fbbd3850d0fded4b7ab48e051654fd2ec21cef3c..7411d9d9273b54fd4251323648d911bac1f7214d 100644
--- a/lib/Target/AArch64/AArch64SchedThunderX.td
+++ b/lib/Target/AArch64/AArch64SchedThunderX.td
@@ -1,9 +1,8 @@
 //==- AArch64SchedThunderX.td - Cavium ThunderX T8X Scheduling Definitions -*- tablegen -*-=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AArch64/AArch64SchedThunderX2T99.td b/lib/Target/AArch64/AArch64SchedThunderX2T99.td
index bee3392b6d3b8974513d95ecca9b244386ca3ed0..d08bc9048921900ad5c83005302e1f9683ae2ec5 100644
--- a/lib/Target/AArch64/AArch64SchedThunderX2T99.td
+++ b/lib/Target/AArch64/AArch64SchedThunderX2T99.td
@@ -1,9 +1,8 @@
 //=- AArch64SchedThunderX2T99.td - Cavium ThunderX T99 ---*- tablegen -*-=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AArch64/AArch64Schedule.td b/lib/Target/AArch64/AArch64Schedule.td
index f55ba4d42fce18b95b7149a7e44115ee667cd5ee..49c0c17822361cd59344af5950638e3a449e87f9 100644
--- a/lib/Target/AArch64/AArch64Schedule.td
+++ b/lib/Target/AArch64/AArch64Schedule.td
@@ -1,9 +1,8 @@
 //==-- AArch64Schedule.td - AArch64 Scheduling Definitions -*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/AArch64/AArch64SelectionDAGInfo.cpp b/lib/Target/AArch64/AArch64SelectionDAGInfo.cpp
index a719d47618e5b58cce195d933c2931c472582c9b..953d7387f2e05cf20a8b746324ae29533e9a3634 100644
--- a/lib/Target/AArch64/AArch64SelectionDAGInfo.cpp
+++ b/lib/Target/AArch64/AArch64SelectionDAGInfo.cpp
@@ -1,9 +1,8 @@
 //===-- AArch64SelectionDAGInfo.cpp - AArch64 SelectionDAG Info -----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AArch64/AArch64SelectionDAGInfo.h b/lib/Target/AArch64/AArch64SelectionDAGInfo.h
index 7e4f11091226d080308753c55e446a4d2f10316b..9d386128fdccd3e7de5b68236e81cd0b50284df1 100644
--- a/lib/Target/AArch64/AArch64SelectionDAGInfo.h
+++ b/lib/Target/AArch64/AArch64SelectionDAGInfo.h
@@ -1,9 +1,8 @@
 //===-- AArch64SelectionDAGInfo.h - AArch64 SelectionDAG Info ---*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AArch64/AArch64SpeculationHardening.cpp b/lib/Target/AArch64/AArch64SpeculationHardening.cpp
index 1f8ef5ee6eae58a1454e05f5cf2e3e309154fa7b..3087e6ce441db96450d4ae8132ec247d3fe769a6 100644
--- a/lib/Target/AArch64/AArch64SpeculationHardening.cpp
+++ b/lib/Target/AArch64/AArch64SpeculationHardening.cpp
@@ -1,9 +1,8 @@
 //===- AArch64SpeculationHardening.cpp - Harden Against Missspeculation  --===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -16,17 +15,15 @@
 // execution".
 // This pass is aimed at mitigating against SpectreV1-style vulnarabilities.
 //
-// At the moment, it implements the tracking of miss-speculation of control
-// flow into a taint register, but doesn't implement a mechanism yet to then
-// use that taint register to mask of vulnerable data in registers (something
-// for a follow-on improvement). Possible strategies to mask out vulnerable
-// data that can be implemented on top of this are:
-// - speculative load hardening to automatically mask of data loaded
-//   in registers.
-// - using intrinsics to mask of data in registers as indicated by the
-//   programmer (see https://lwn.net/Articles/759423/).
+// It also implements speculative load hardening, i.e. using the taint register
+// to automatically mask off loaded data.
+//
+// As a possible follow-on improvement, also an intrinsics-based approach as
+// explained at https://lwn.net/Articles/759423/ could be implemented on top of
+// the current design.
 //
-// For AArch64, the following implementation choices are made below.
+// For AArch64, the following implementation choices are made to implement the
+// tracking of control flow miss-speculation into a taint register:
 // Some of these are different than the implementation choices made in
 // the similar pass implemented in X86SpeculativeLoadHardening.cpp, as
 // the instruction set characteristics result in different trade-offs.
@@ -65,6 +62,24 @@
 // - On function call boundaries, the miss-speculation state is transferred from
 //   the taint register X16 to be encoded in the SP register as value 0.
 //
+// For the aspect of automatically hardening loads, using the taint register,
+// (a.k.a. speculative load hardening, see
+//  https://llvm.org/docs/SpeculativeLoadHardening.html), the following
+// implementation choices are made for AArch64:
+//   - Many of the optimizations described at
+//     https://llvm.org/docs/SpeculativeLoadHardening.html to harden fewer
+//     loads haven't been implemented yet - but for some of them there are
+//     FIXMEs in the code.
+//   - loads that load into general purpose (X or W) registers get hardened by
+//     masking the loaded data. For loads that load into other registers, the
+//     address loaded from gets hardened. It is expected that hardening the
+//     loaded data may be more efficient; but masking data in registers other
+//     than X or W is not easy and may result in being slower than just
+//     hardening the X address register loaded from.
+//   - On AArch64, CSDB instructions are inserted between the masking of the
+//     register and its first use, to ensure there's no non-control-flow
+//     speculation that might undermine the hardening mechanism.
+//
 // Future extensions/improvements could be:
 // - Implement this functionality using full speculation barriers, akin to the
 //   x86-slh-lfence option. This may be more useful for the intrinsics-based
@@ -87,6 +102,7 @@
 #include "llvm/CodeGen/MachineInstrBuilder.h"
 #include "llvm/CodeGen/MachineOperand.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/CodeGen/RegisterScavenging.h"
 #include "llvm/IR/DebugLoc.h"
 #include "llvm/Pass.h"
 #include "llvm/Support/CodeGen.h"
@@ -99,6 +115,10 @@ using namespace llvm;
 
 #define AARCH64_SPECULATION_HARDENING_NAME "AArch64 speculation hardening pass"
 
+cl::opt<bool> HardenLoads("aarch64-slh-loads", cl::Hidden,
+                          cl::desc("Sanitize loads from memory."),
+                          cl::init(true));
+
 namespace {
 
 class AArch64SpeculationHardening : public MachineFunctionPass {
@@ -120,20 +140,39 @@ public:
 
 private:
   unsigned MisspeculatingTaintReg;
+  unsigned MisspeculatingTaintReg32Bit;
   bool UseControlFlowSpeculationBarrier;
+  BitVector RegsNeedingCSDBBeforeUse;
+  BitVector RegsAlreadyMasked;
 
   bool functionUsesHardeningRegister(MachineFunction &MF) const;
-  bool instrumentControlFlow(MachineBasicBlock &MBB);
+  bool instrumentControlFlow(MachineBasicBlock &MBB,
+                             bool &UsesFullSpeculationBarrier);
   bool endsWithCondControlFlow(MachineBasicBlock &MBB, MachineBasicBlock *&TBB,
                                MachineBasicBlock *&FBB,
                                AArch64CC::CondCode &CondCode) const;
   void insertTrackingCode(MachineBasicBlock &SplitEdgeBB,
                           AArch64CC::CondCode &CondCode, DebugLoc DL) const;
-  void insertSPToRegTaintPropagation(MachineBasicBlock *MBB,
+  void insertSPToRegTaintPropagation(MachineBasicBlock &MBB,
                                      MachineBasicBlock::iterator MBBI) const;
-  void insertRegToSPTaintPropagation(MachineBasicBlock *MBB,
+  void insertRegToSPTaintPropagation(MachineBasicBlock &MBB,
                                      MachineBasicBlock::iterator MBBI,
                                      unsigned TmpReg) const;
+  void insertFullSpeculationBarrier(MachineBasicBlock &MBB,
+                                    MachineBasicBlock::iterator MBBI,
+                                    DebugLoc DL) const;
+
+  bool slhLoads(MachineBasicBlock &MBB);
+  bool makeGPRSpeculationSafe(MachineBasicBlock &MBB,
+                              MachineBasicBlock::iterator MBBI,
+                              MachineInstr &MI, unsigned Reg);
+  bool lowerSpeculationSafeValuePseudos(MachineBasicBlock &MBB,
+                                        bool UsesFullSpeculationBarrier);
+  bool expandSpeculationSafeValue(MachineBasicBlock &MBB,
+                                  MachineBasicBlock::iterator MBBI,
+                                  bool UsesFullSpeculationBarrier);
+  bool insertCSDB(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
+                  DebugLoc DL);
 };
 
 } // end anonymous namespace
@@ -174,15 +213,19 @@ bool AArch64SpeculationHardening::endsWithCondControlFlow(
   return true;
 }
 
+void AArch64SpeculationHardening::insertFullSpeculationBarrier(
+    MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
+    DebugLoc DL) const {
+  // A full control flow speculation barrier consists of (DSB SYS + ISB)
+  BuildMI(MBB, MBBI, DL, TII->get(AArch64::DSB)).addImm(0xf);
+  BuildMI(MBB, MBBI, DL, TII->get(AArch64::ISB)).addImm(0xf);
+}
+
 void AArch64SpeculationHardening::insertTrackingCode(
     MachineBasicBlock &SplitEdgeBB, AArch64CC::CondCode &CondCode,
     DebugLoc DL) const {
   if (UseControlFlowSpeculationBarrier) {
-    // insert full control flow speculation barrier (DSB SYS + ISB)
-    BuildMI(SplitEdgeBB, SplitEdgeBB.begin(), DL, TII->get(AArch64::ISB))
-        .addImm(0xf);
-    BuildMI(SplitEdgeBB, SplitEdgeBB.begin(), DL, TII->get(AArch64::DSB))
-        .addImm(0xf);
+    insertFullSpeculationBarrier(SplitEdgeBB, SplitEdgeBB.begin(), DL);
   } else {
     BuildMI(SplitEdgeBB, SplitEdgeBB.begin(), DL, TII->get(AArch64::CSELXr))
         .addDef(MisspeculatingTaintReg)
@@ -194,7 +237,7 @@ void AArch64SpeculationHardening::insertTrackingCode(
 }
 
 bool AArch64SpeculationHardening::instrumentControlFlow(
-    MachineBasicBlock &MBB) {
+    MachineBasicBlock &MBB, bool &UsesFullSpeculationBarrier) {
   LLVM_DEBUG(dbgs() << "Instrument control flow tracking on MBB: " << MBB);
 
   bool Modified = false;
@@ -230,55 +273,105 @@ bool AArch64SpeculationHardening::instrumentControlFlow(
   }
 
   // Perform correct code generation around function calls and before returns.
-  {
-    SmallVector<MachineInstr *, 4> ReturnInstructions;
-    SmallVector<MachineInstr *, 4> CallInstructions;
+  // The below variables record the return/terminator instructions and the call
+  // instructions respectively; including which register is available as a
+  // temporary register just before the recorded instructions.
+  SmallVector<std::pair<MachineInstr *, unsigned>, 4> ReturnInstructions;
+  SmallVector<std::pair<MachineInstr *, unsigned>, 4> CallInstructions;
+  // if a temporary register is not available for at least one of the
+  // instructions for which we need to transfer taint to the stack pointer, we
+  // need to insert a full speculation barrier.
+  // TmpRegisterNotAvailableEverywhere tracks that condition.
+  bool TmpRegisterNotAvailableEverywhere = false;
+
+  RegScavenger RS;
+  RS.enterBasicBlock(MBB);
+
+  for (MachineBasicBlock::iterator I = MBB.begin(); I != MBB.end(); I++) {
+    MachineInstr &MI = *I;
+    if (!MI.isReturn() && !MI.isCall())
+      continue;
+
+    // The RegScavenger represents registers available *after* the MI
+    // instruction pointed to by RS.getCurrentPosition().
+    // We need to have a register that is available *before* the MI is executed.
+    if (I != MBB.begin())
+      RS.forward(std::prev(I));
+    // FIXME: The below just finds *a* unused register. Maybe code could be
+    // optimized more if this looks for the register that isn't used for the
+    // longest time around this place, to enable more scheduling freedom. Not
+    // sure if that would actually result in a big performance difference
+    // though. Maybe RegisterScavenger::findSurvivorBackwards has some logic
+    // already to do this - but it's unclear if that could easily be used here.
+    unsigned TmpReg = RS.FindUnusedReg(&AArch64::GPR64commonRegClass);
+    LLVM_DEBUG(dbgs() << "RS finds "
+                      << ((TmpReg == 0) ? "no register " : "register ");
+               if (TmpReg != 0) dbgs() << printReg(TmpReg, TRI) << " ";
+               dbgs() << "to be available at MI " << MI);
+    if (TmpReg == 0)
+      TmpRegisterNotAvailableEverywhere = true;
+    if (MI.isReturn())
+      ReturnInstructions.push_back({&MI, TmpReg});
+    else if (MI.isCall())
+      CallInstructions.push_back({&MI, TmpReg});
+  }
 
-    for (MachineInstr &MI : MBB) {
-      if (MI.isReturn())
-        ReturnInstructions.push_back(&MI);
-      else if (MI.isCall())
-        CallInstructions.push_back(&MI);
+  if (TmpRegisterNotAvailableEverywhere) {
+    // When a temporary register is not available everywhere in this basic
+    // basic block where a propagate-taint-to-sp operation is needed, just
+    // emit a full speculation barrier at the start of this basic block, which
+    // renders the taint/speculation tracking in this basic block unnecessary.
+    insertFullSpeculationBarrier(MBB, MBB.begin(),
+                                 (MBB.begin())->getDebugLoc());
+    UsesFullSpeculationBarrier = true;
+    Modified = true;
+  } else {
+    for (auto MI_Reg : ReturnInstructions) {
+      assert(MI_Reg.second != 0);
+      LLVM_DEBUG(
+          dbgs()
+          << " About to insert Reg to SP taint propagation with temp register "
+          << printReg(MI_Reg.second, TRI)
+          << " on instruction: " << *MI_Reg.first);
+      insertRegToSPTaintPropagation(MBB, MI_Reg.first, MI_Reg.second);
+      Modified = true;
     }
 
-    Modified |=
-        (ReturnInstructions.size() > 0) || (CallInstructions.size() > 0);
-
-    for (MachineInstr *Return : ReturnInstructions)
-      insertRegToSPTaintPropagation(Return->getParent(), Return, AArch64::X17);
-    for (MachineInstr *Call : CallInstructions) {
+    for (auto MI_Reg : CallInstructions) {
+      assert(MI_Reg.second != 0);
+      LLVM_DEBUG(dbgs() << " About to insert Reg to SP and back taint "
+                           "propagation with temp register "
+                        << printReg(MI_Reg.second, TRI)
+                        << " around instruction: " << *MI_Reg.first);
       // Just after the call:
-      MachineBasicBlock::iterator i = Call;
-      i++;
-      insertSPToRegTaintPropagation(Call->getParent(), i);
+      insertSPToRegTaintPropagation(
+          MBB, std::next((MachineBasicBlock::iterator)MI_Reg.first));
       // Just before the call:
-      insertRegToSPTaintPropagation(Call->getParent(), Call, AArch64::X17);
+      insertRegToSPTaintPropagation(MBB, MI_Reg.first, MI_Reg.second);
+      Modified = true;
     }
   }
-
   return Modified;
 }
 
 void AArch64SpeculationHardening::insertSPToRegTaintPropagation(
-    MachineBasicBlock *MBB, MachineBasicBlock::iterator MBBI) const {
+    MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI) const {
   // If full control flow speculation barriers are used, emit a control flow
   // barrier to block potential miss-speculation in flight coming in to this
   // function.
   if (UseControlFlowSpeculationBarrier) {
-    // insert full control flow speculation barrier (DSB SYS + ISB)
-    BuildMI(*MBB, MBBI, DebugLoc(), TII->get(AArch64::DSB)).addImm(0xf);
-    BuildMI(*MBB, MBBI, DebugLoc(), TII->get(AArch64::ISB)).addImm(0xf);
+    insertFullSpeculationBarrier(MBB, MBBI, DebugLoc());
     return;
   }
 
   // CMP   SP, #0   === SUBS   xzr, SP, #0
-  BuildMI(*MBB, MBBI, DebugLoc(), TII->get(AArch64::SUBSXri))
+  BuildMI(MBB, MBBI, DebugLoc(), TII->get(AArch64::SUBSXri))
       .addDef(AArch64::XZR)
       .addUse(AArch64::SP)
       .addImm(0)
       .addImm(0); // no shift
   // CSETM x16, NE  === CSINV  x16, xzr, xzr, EQ
-  BuildMI(*MBB, MBBI, DebugLoc(), TII->get(AArch64::CSINVXr))
+  BuildMI(MBB, MBBI, DebugLoc(), TII->get(AArch64::CSINVXr))
       .addDef(MisspeculatingTaintReg)
       .addUse(AArch64::XZR)
       .addUse(AArch64::XZR)
@@ -286,7 +379,7 @@ void AArch64SpeculationHardening::insertSPToRegTaintPropagation(
 }
 
 void AArch64SpeculationHardening::insertRegToSPTaintPropagation(
-    MachineBasicBlock *MBB, MachineBasicBlock::iterator MBBI,
+    MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
     unsigned TmpReg) const {
   // If full control flow speculation barriers are used, there will not be
   // miss-speculation when returning from this function, and therefore, also
@@ -295,19 +388,19 @@ void AArch64SpeculationHardening::insertRegToSPTaintPropagation(
     return;
 
   // mov   Xtmp, SP  === ADD  Xtmp, SP, #0
-  BuildMI(*MBB, MBBI, DebugLoc(), TII->get(AArch64::ADDXri))
+  BuildMI(MBB, MBBI, DebugLoc(), TII->get(AArch64::ADDXri))
       .addDef(TmpReg)
       .addUse(AArch64::SP)
       .addImm(0)
       .addImm(0); // no shift
   // and   Xtmp, Xtmp, TaintReg === AND Xtmp, Xtmp, TaintReg, #0
-  BuildMI(*MBB, MBBI, DebugLoc(), TII->get(AArch64::ANDXrs))
+  BuildMI(MBB, MBBI, DebugLoc(), TII->get(AArch64::ANDXrs))
       .addDef(TmpReg, RegState::Renamable)
       .addUse(TmpReg, RegState::Kill | RegState::Renamable)
       .addUse(MisspeculatingTaintReg, RegState::Kill)
       .addImm(0);
   // mov   SP, Xtmp === ADD SP, Xtmp, #0
-  BuildMI(*MBB, MBBI, DebugLoc(), TII->get(AArch64::ADDXri))
+  BuildMI(MBB, MBBI, DebugLoc(), TII->get(AArch64::ADDXri))
       .addDef(AArch64::SP)
       .addUse(TmpReg, RegState::Kill)
       .addImm(0)
@@ -330,34 +423,274 @@ bool AArch64SpeculationHardening::functionUsesHardeningRegister(
   return false;
 }
 
+// Make GPR register Reg speculation-safe by putting it through the
+// SpeculationSafeValue pseudo instruction, if we can't prove that
+// the value in the register has already been hardened.
+bool AArch64SpeculationHardening::makeGPRSpeculationSafe(
+    MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, MachineInstr &MI,
+    unsigned Reg) {
+  assert(AArch64::GPR32allRegClass.contains(Reg) ||
+         AArch64::GPR64allRegClass.contains(Reg));
+
+  // Loads cannot directly load a value into the SP (nor WSP).
+  // Therefore, if Reg is SP or WSP, it is because the instruction loads from
+  // the stack through the stack pointer.
+  //
+  // Since the stack pointer is never dynamically controllable, don't harden it.
+  if (Reg == AArch64::SP || Reg == AArch64::WSP)
+    return false;
+
+  // Do not harden the register again if already hardened before.
+  if (RegsAlreadyMasked[Reg])
+    return false;
+
+  const bool Is64Bit = AArch64::GPR64allRegClass.contains(Reg);
+  LLVM_DEBUG(dbgs() << "About to harden register : " << Reg << "\n");
+  BuildMI(MBB, MBBI, MI.getDebugLoc(),
+          TII->get(Is64Bit ? AArch64::SpeculationSafeValueX
+                           : AArch64::SpeculationSafeValueW))
+      .addDef(Reg)
+      .addUse(Reg);
+  RegsAlreadyMasked.set(Reg);
+  return true;
+}
+
+bool AArch64SpeculationHardening::slhLoads(MachineBasicBlock &MBB) {
+  bool Modified = false;
+
+  LLVM_DEBUG(dbgs() << "slhLoads running on MBB: " << MBB);
+
+  RegsAlreadyMasked.reset();
+
+  MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end();
+  MachineBasicBlock::iterator NextMBBI;
+  for (; MBBI != E; MBBI = NextMBBI) {
+    MachineInstr &MI = *MBBI;
+    NextMBBI = std::next(MBBI);
+    // Only harden loaded values or addresses used in loads.
+    if (!MI.mayLoad())
+      continue;
+
+    LLVM_DEBUG(dbgs() << "About to harden: " << MI);
+
+    // For general purpose register loads, harden the registers loaded into.
+    // For other loads, harden the address loaded from.
+    // Masking the loaded value is expected to result in less performance
+    // overhead, as the load can still execute speculatively in comparison to
+    // when the address loaded from gets masked. However, masking is only
+    // easy to do efficiently on GPR registers, so for loads into non-GPR
+    // registers (e.g. floating point loads), mask the address loaded from.
+    bool AllDefsAreGPR = llvm::all_of(MI.defs(), [&](MachineOperand &Op) {
+      return Op.isReg() && (AArch64::GPR32allRegClass.contains(Op.getReg()) ||
+                            AArch64::GPR64allRegClass.contains(Op.getReg()));
+    });
+    // FIXME: it might be a worthwhile optimization to not mask loaded
+    // values if all the registers involved in address calculation are already
+    // hardened, leading to this load not able to execute on a miss-speculated
+    // path.
+    bool HardenLoadedData = AllDefsAreGPR;
+    bool HardenAddressLoadedFrom = !HardenLoadedData;
+
+    // First remove registers from AlreadyMaskedRegisters if their value is
+    // updated by this instruction - it makes them contain a new value that is
+    // not guaranteed to already have been masked.
+    for (MachineOperand Op : MI.defs())
+      for (MCRegAliasIterator AI(Op.getReg(), TRI, true); AI.isValid(); ++AI)
+        RegsAlreadyMasked.reset(*AI);
+
+    // FIXME: loads from the stack with an immediate offset from the stack
+    // pointer probably shouldn't be hardened, which could result in a
+    // significant optimization. See section "Don’t check loads from
+    // compile-time constant stack offsets", in
+    // https://llvm.org/docs/SpeculativeLoadHardening.html
+
+    if (HardenLoadedData)
+      for (auto Def : MI.defs()) {
+        if (Def.isDead())
+          // Do not mask a register that is not used further.
+          continue;
+        // FIXME: For pre/post-increment addressing modes, the base register
+        // used in address calculation is also defined by this instruction.
+        // It might be a worthwhile optimization to not harden that
+        // base register increment/decrement when the increment/decrement is
+        // an immediate.
+        Modified |= makeGPRSpeculationSafe(MBB, NextMBBI, MI, Def.getReg());
+      }
+
+    if (HardenAddressLoadedFrom)
+      for (auto Use : MI.uses()) {
+        if (!Use.isReg())
+          continue;
+        unsigned Reg = Use.getReg();
+        // Some loads of floating point data have implicit defs/uses on a
+        // super register of that floating point data. Some examples:
+        // $s0 = LDRSui $sp, 22, implicit-def $q0
+        // $q0 = LD1i64 $q0, 1, renamable $x0
+        // We need to filter out these uses for non-GPR register which occur
+        // because the load partially fills a non-GPR register with the loaded
+        // data. Just skipping all non-GPR registers is safe (for now) as all
+        // AArch64 load instructions only use GPR registers to perform the
+        // address calculation. FIXME: However that might change once we can
+        // produce SVE gather instructions.
+        if (!(AArch64::GPR32allRegClass.contains(Reg) ||
+              AArch64::GPR64allRegClass.contains(Reg)))
+          continue;
+        Modified |= makeGPRSpeculationSafe(MBB, MBBI, MI, Reg);
+      }
+  }
+  return Modified;
+}
+
+/// \brief If MBBI references a pseudo instruction that should be expanded
+/// here, do the expansion and return true. Otherwise return false.
+bool AArch64SpeculationHardening::expandSpeculationSafeValue(
+    MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
+    bool UsesFullSpeculationBarrier) {
+  MachineInstr &MI = *MBBI;
+  unsigned Opcode = MI.getOpcode();
+  bool Is64Bit = true;
+
+  switch (Opcode) {
+  default:
+    break;
+  case AArch64::SpeculationSafeValueW:
+    Is64Bit = false;
+    LLVM_FALLTHROUGH;
+  case AArch64::SpeculationSafeValueX:
+    // Just remove the SpeculationSafe pseudo's if control flow
+    // miss-speculation isn't happening because we're already inserting barriers
+    // to guarantee that.
+    if (!UseControlFlowSpeculationBarrier && !UsesFullSpeculationBarrier) {
+      unsigned DstReg = MI.getOperand(0).getReg();
+      unsigned SrcReg = MI.getOperand(1).getReg();
+      // Mark this register and all its aliasing registers as needing to be
+      // value speculation hardened before its next use, by using a CSDB
+      // barrier instruction.
+      for (MachineOperand Op : MI.defs())
+        for (MCRegAliasIterator AI(Op.getReg(), TRI, true); AI.isValid(); ++AI)
+          RegsNeedingCSDBBeforeUse.set(*AI);
+
+      // Mask off with taint state.
+      BuildMI(MBB, MBBI, MI.getDebugLoc(),
+              Is64Bit ? TII->get(AArch64::ANDXrs) : TII->get(AArch64::ANDWrs))
+          .addDef(DstReg)
+          .addUse(SrcReg, RegState::Kill)
+          .addUse(Is64Bit ? MisspeculatingTaintReg
+                          : MisspeculatingTaintReg32Bit)
+          .addImm(0);
+    }
+    MI.eraseFromParent();
+    return true;
+  }
+  return false;
+}
+
+bool AArch64SpeculationHardening::insertCSDB(MachineBasicBlock &MBB,
+                                             MachineBasicBlock::iterator MBBI,
+                                             DebugLoc DL) {
+  assert(!UseControlFlowSpeculationBarrier && "No need to insert CSDBs when "
+                                              "control flow miss-speculation "
+                                              "is already blocked");
+  // insert data value speculation barrier (CSDB)
+  BuildMI(MBB, MBBI, DL, TII->get(AArch64::HINT)).addImm(0x14);
+  RegsNeedingCSDBBeforeUse.reset();
+  return true;
+}
+
+bool AArch64SpeculationHardening::lowerSpeculationSafeValuePseudos(
+    MachineBasicBlock &MBB, bool UsesFullSpeculationBarrier) {
+  bool Modified = false;
+
+  RegsNeedingCSDBBeforeUse.reset();
+
+  // The following loop iterates over all instructions in the basic block,
+  // and performs 2 operations:
+  // 1. Insert a CSDB at this location if needed.
+  // 2. Expand the SpeculationSafeValuePseudo if the current instruction is
+  // one.
+  //
+  // The insertion of the CSDB is done as late as possible (i.e. just before
+  // the use of a masked register), in the hope that that will reduce the
+  // total number of CSDBs in a block when there are multiple masked registers
+  // in the block.
+  MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end();
+  DebugLoc DL;
+  while (MBBI != E) {
+    MachineInstr &MI = *MBBI;
+    DL = MI.getDebugLoc();
+    MachineBasicBlock::iterator NMBBI = std::next(MBBI);
+
+    // First check if a CSDB needs to be inserted due to earlier registers
+    // that were masked and that are used by the next instruction.
+    // Also emit the barrier on any potential control flow changes.
+    bool NeedToEmitBarrier = false;
+    if (RegsNeedingCSDBBeforeUse.any() && (MI.isCall() || MI.isTerminator()))
+      NeedToEmitBarrier = true;
+    if (!NeedToEmitBarrier)
+      for (MachineOperand Op : MI.uses())
+        if (Op.isReg() && RegsNeedingCSDBBeforeUse[Op.getReg()]) {
+          NeedToEmitBarrier = true;
+          break;
+        }
+
+    if (NeedToEmitBarrier && !UsesFullSpeculationBarrier)
+      Modified |= insertCSDB(MBB, MBBI, DL);
+
+    Modified |=
+        expandSpeculationSafeValue(MBB, MBBI, UsesFullSpeculationBarrier);
+
+    MBBI = NMBBI;
+  }
+
+  if (RegsNeedingCSDBBeforeUse.any() && !UsesFullSpeculationBarrier)
+    Modified |= insertCSDB(MBB, MBBI, DL);
+
+  return Modified;
+}
+
 bool AArch64SpeculationHardening::runOnMachineFunction(MachineFunction &MF) {
   if (!MF.getFunction().hasFnAttribute(Attribute::SpeculativeLoadHardening))
     return false;
 
   MisspeculatingTaintReg = AArch64::X16;
+  MisspeculatingTaintReg32Bit = AArch64::W16;
   TII = MF.getSubtarget().getInstrInfo();
   TRI = MF.getSubtarget().getRegisterInfo();
+  RegsNeedingCSDBBeforeUse.resize(TRI->getNumRegs());
+  RegsAlreadyMasked.resize(TRI->getNumRegs());
+  UseControlFlowSpeculationBarrier = functionUsesHardeningRegister(MF);
+
   bool Modified = false;
 
-  UseControlFlowSpeculationBarrier = functionUsesHardeningRegister(MF);
+  // Step 1: Enable automatic insertion of SpeculationSafeValue.
+  if (HardenLoads) {
+    LLVM_DEBUG(
+        dbgs() << "***** AArch64SpeculationHardening - automatic insertion of "
+                  "SpeculationSafeValue intrinsics *****\n");
+    for (auto &MBB : MF)
+      Modified |= slhLoads(MBB);
+  }
 
-  // Instrument control flow speculation tracking, if requested.
+  // 2. Add instrumentation code to function entry and exits.
   LLVM_DEBUG(
       dbgs()
       << "***** AArch64SpeculationHardening - track control flow *****\n");
 
-  // 1. Add instrumentation code to function entry and exits.
   SmallVector<MachineBasicBlock *, 2> EntryBlocks;
   EntryBlocks.push_back(&MF.front());
   for (const LandingPadInfo &LPI : MF.getLandingPads())
     EntryBlocks.push_back(LPI.LandingPadBlock);
   for (auto Entry : EntryBlocks)
     insertSPToRegTaintPropagation(
-        Entry, Entry->SkipPHIsLabelsAndDebug(Entry->begin()));
+        *Entry, Entry->SkipPHIsLabelsAndDebug(Entry->begin()));
 
-  // 2. Add instrumentation code to every basic block.
-  for (auto &MBB : MF)
-    Modified |= instrumentControlFlow(MBB);
+  // 3. Add instrumentation code to every basic block.
+  for (auto &MBB : MF) {
+    bool UsesFullSpeculationBarrier = false;
+    Modified |= instrumentControlFlow(MBB, UsesFullSpeculationBarrier);
+    Modified |=
+        lowerSpeculationSafeValuePseudos(MBB, UsesFullSpeculationBarrier);
+  }
 
   return Modified;
 }
diff --git a/lib/Target/AArch64/AArch64StorePairSuppress.cpp b/lib/Target/AArch64/AArch64StorePairSuppress.cpp
index d5643d384283340e0674c344a0e7aad498b568c7..27ddd7af2714f14ceb7e22310594033246150e63 100644
--- a/lib/Target/AArch64/AArch64StorePairSuppress.cpp
+++ b/lib/Target/AArch64/AArch64StorePairSuppress.cpp
@@ -1,9 +1,8 @@
 //===--- AArch64StorePairSuppress.cpp --- Suppress store pair formation ---===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AArch64/AArch64Subtarget.cpp b/lib/Target/AArch64/AArch64Subtarget.cpp
index dd30d25b2b505de77c9beb0b727c62f6ed8d01e2..e22ce44ecb991dffa67c8e85ff13256d644a0487 100644
--- a/lib/Target/AArch64/AArch64Subtarget.cpp
+++ b/lib/Target/AArch64/AArch64Subtarget.cpp
@@ -1,9 +1,8 @@
 //===-- AArch64Subtarget.cpp - AArch64 Subtarget Information ----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AArch64/AArch64Subtarget.h b/lib/Target/AArch64/AArch64Subtarget.h
index 9b27e0a0e8fdf8428ac8783d9bf98f79c8ae1c17..4ae14bd1335732073833e1cae16beaee5edbe96d 100644
--- a/lib/Target/AArch64/AArch64Subtarget.h
+++ b/lib/Target/AArch64/AArch64Subtarget.h
@@ -1,9 +1,8 @@
 //===--- AArch64Subtarget.h - Define Subtarget for the AArch64 -*- C++ -*--===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -126,9 +125,9 @@ protected:
   bool HasAlternativeNZCV = false;
   bool HasFRInt3264 = false;
   bool HasSpecRestrict = false;
-  bool HasSpecCtrl = false;
   bool HasSSBS = false;
-  bool HasPredCtrl = false;
+  bool HasSB = false;
+  bool HasPredRes = false;
   bool HasCCDP = false;
   bool HasBTI = false;
   bool HasRandGen = false;
@@ -166,8 +165,9 @@ protected:
   bool HasArithmeticCbzFusion = false;
   bool HasFuseAddress = false;
   bool HasFuseAES = false;
-  bool HasFuseCryptoEOR = false;
+  bool HasFuseArithmeticLogic = false;
   bool HasFuseCCSelect = false;
+  bool HasFuseCryptoEOR = false;
   bool HasFuseLiterals = false;
   bool DisableLatencySchedHeuristic = false;
   bool UseRSqrt = false;
@@ -311,14 +311,16 @@ public:
   bool hasArithmeticCbzFusion() const { return HasArithmeticCbzFusion; }
   bool hasFuseAddress() const { return HasFuseAddress; }
   bool hasFuseAES() const { return HasFuseAES; }
-  bool hasFuseCryptoEOR() const { return HasFuseCryptoEOR; }
+  bool hasFuseArithmeticLogic() const { return HasFuseArithmeticLogic; }
   bool hasFuseCCSelect() const { return HasFuseCCSelect; }
+  bool hasFuseCryptoEOR() const { return HasFuseCryptoEOR; }
   bool hasFuseLiterals() const { return HasFuseLiterals; }
 
   /// Return true if the CPU supports any kind of instruction fusion.
   bool hasFusion() const {
     return hasArithmeticBccFusion() || hasArithmeticCbzFusion() ||
-           hasFuseAES() || hasFuseCCSelect() || hasFuseLiterals();
+           hasFuseAES() || hasFuseArithmeticLogic() ||
+           hasFuseCCSelect() || hasFuseLiterals();
   }
 
   bool useRSqrt() const { return UseRSqrt; }
@@ -355,9 +357,9 @@ public:
   bool hasAlternativeNZCV() const { return HasAlternativeNZCV; }
   bool hasFRInt3264() const { return HasFRInt3264; }
   bool hasSpecRestrict() const { return HasSpecRestrict; }
-  bool hasSpecCtrl() const { return HasSpecCtrl; }
   bool hasSSBS() const { return HasSSBS; }
-  bool hasPredCtrl() const { return HasPredCtrl; }
+  bool hasSB() const { return HasSB; }
+  bool hasPredRes() const { return HasPredRes; }
   bool hasCCDP() const { return HasCCDP; }
   bool hasBTI() const { return HasBTI; }
   bool hasRandGen() const { return HasRandGen; }
diff --git a/lib/Target/AArch64/AArch64SystemOperands.td b/lib/Target/AArch64/AArch64SystemOperands.td
index 60d48e4d99d750e41b2d04efe20459e82ef9c91b..30c4822a61080a0e5c0aa7baff8a158d4cb22778 100644
--- a/lib/Target/AArch64/AArch64SystemOperands.td
+++ b/lib/Target/AArch64/AArch64SystemOperands.td
@@ -1,9 +1,8 @@
 //===- AArch64SystemOperands.td ----------------------------*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -501,7 +500,7 @@ class PRCTX<string name, bits<4> crm> : SearchableTable {
   code Requires = [{ {} }];
 }
 
-let Requires = [{ {AArch64::FeaturePredCtrl} }] in {
+let Requires = [{ {AArch64::FeaturePredRes} }] in {
 def : PRCTX<"RCTX", 0b0011>;
 }
 
diff --git a/lib/Target/AArch64/AArch64TargetMachine.cpp b/lib/Target/AArch64/AArch64TargetMachine.cpp
index 4e016525f7e4e6bc1fea1dad2b68c2b58904541b..d213f20755f8f5ae17dddc212ef2842458f323ab 100644
--- a/lib/Target/AArch64/AArch64TargetMachine.cpp
+++ b/lib/Target/AArch64/AArch64TargetMachine.cpp
@@ -1,9 +1,8 @@
 //===-- AArch64TargetMachine.cpp - Define TargetMachine for AArch64 -------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -209,8 +208,8 @@ static std::string computeDataLayout(const Triple &TT,
 
 static Reloc::Model getEffectiveRelocModel(const Triple &TT,
                                            Optional<Reloc::Model> RM) {
-  // AArch64 Darwin is always PIC.
-  if (TT.isOSDarwin())
+  // AArch64 Darwin and Windows are always PIC.
+  if (TT.isOSDarwin() || TT.isOSWindows())
     return Reloc::PIC_;
   // On ELF platforms the default static relocation model has a smart enough
   // linker to cope with referencing external symbols defined in a shared
diff --git a/lib/Target/AArch64/AArch64TargetMachine.h b/lib/Target/AArch64/AArch64TargetMachine.h
index 8d28a5e30ebfaa0610f9b37ed597d906985e2329..5264efb89b9c5a8f3e4edb576123c566b7b1c588 100644
--- a/lib/Target/AArch64/AArch64TargetMachine.h
+++ b/lib/Target/AArch64/AArch64TargetMachine.h
@@ -1,9 +1,8 @@
 //==-- AArch64TargetMachine.h - Define TargetMachine for AArch64 -*- C++ -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AArch64/AArch64TargetObjectFile.cpp b/lib/Target/AArch64/AArch64TargetObjectFile.cpp
index 8ae72a7ddb573654c6b6d499dd11ab8bd85c8e12..1c3d5d0743ad0de34099dedb0a60cab59d56a78c 100644
--- a/lib/Target/AArch64/AArch64TargetObjectFile.cpp
+++ b/lib/Target/AArch64/AArch64TargetObjectFile.cpp
@@ -1,9 +1,8 @@
 //===-- AArch64TargetObjectFile.cpp - AArch64 Object Info -----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/AArch64/AArch64TargetObjectFile.h b/lib/Target/AArch64/AArch64TargetObjectFile.h
index 9077eb7902fd468add5edcdb84c6a96c490ea9be..7ead363d42fe72c618517a33f87d355e4171d7cf 100644
--- a/lib/Target/AArch64/AArch64TargetObjectFile.h
+++ b/lib/Target/AArch64/AArch64TargetObjectFile.h
@@ -1,9 +1,8 @@
 //===-- AArch64TargetObjectFile.h - AArch64 Object Info -*- C++ ---------*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index a256cb7c9215645cdbf6e3568eee015da4df010e..50624a82def93302cb047e60c3b5af63ae6c903b 100644
--- a/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -1,9 +1,8 @@
 //===-- AArch64TargetTransformInfo.cpp - AArch64 specific TTI -------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/AArch64/AArch64TargetTransformInfo.h b/lib/Target/AArch64/AArch64TargetTransformInfo.h
index 08c1a8924220fd00085ffad86f1bd75db99774d0..67c3707ec5c43de5c5e47d54f5c0c3a104a96c90 100644
--- a/lib/Target/AArch64/AArch64TargetTransformInfo.h
+++ b/lib/Target/AArch64/AArch64TargetTransformInfo.h
@@ -1,9 +1,8 @@
 //===- AArch64TargetTransformInfo.h - AArch64 specific TTI ------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp b/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp
index edf361bc399cce8321b1b0d985f1f1563c44c082..66cb9ae572e2655dac883f696b7bc299dbade68f 100644
--- a/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp
+++ b/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp
@@ -1,9 +1,8 @@
 //==- AArch64AsmParser.cpp - Parse AArch64 assembly to MCInst instructions -==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -165,6 +164,7 @@ private:
                       OperandVector &Operands);
 
   bool parseDirectiveArch(SMLoc L);
+  bool parseDirectiveArchExtension(SMLoc L);
   bool parseDirectiveCPU(SMLoc L);
   bool parseDirectiveInst(SMLoc L);
 
@@ -176,6 +176,7 @@ private:
   bool parseDirectiveReq(StringRef Name, SMLoc L);
   bool parseDirectiveUnreq(SMLoc L);
   bool parseDirectiveCFINegateRAState();
+  bool parseDirectiveCFIBKeyFrame();
 
   bool validateInstruction(MCInst &Inst, SMLoc &IDLoc,
                            SmallVectorImpl<SMLoc> &Loc);
@@ -2824,12 +2825,13 @@ static const struct Extension {
     {"simd", {AArch64::FeatureNEON}},
     {"ras", {AArch64::FeatureRAS}},
     {"lse", {AArch64::FeatureLSE}},
-    {"predctrl", {AArch64::FeaturePredCtrl}},
+    {"predres", {AArch64::FeaturePredRes}},
     {"ccdp", {AArch64::FeatureCacheDeepPersist}},
     {"mte", {AArch64::FeatureMTE}},
     {"tlb-rmi", {AArch64::FeatureTLB_RMI}},
     {"pan-rwv", {AArch64::FeaturePAN_RWV}},
     {"ccpp", {AArch64::FeatureCCPP}},
+    {"sve", {AArch64::FeatureSVE}},
     // FIXME: Unsupported extensions
     {"pan", {}},
     {"lor", {}},
@@ -5030,6 +5032,10 @@ bool AArch64AsmParser::ParseDirective(AsmToken DirectiveID) {
     parseDirectiveInst(Loc);
   else if (IDVal == ".cfi_negate_ra_state")
     parseDirectiveCFINegateRAState();
+  else if (IDVal == ".cfi_b_key_frame")
+    parseDirectiveCFIBKeyFrame();
+  else if (IDVal == ".arch_extension")
+    parseDirectiveArchExtension(Loc);
   else if (IsMachO) {
     if (IDVal == MCLOHDirectiveName())
       parseDirectiveLOH(IDVal, Loc);
@@ -5150,6 +5156,50 @@ bool AArch64AsmParser::parseDirectiveArch(SMLoc L) {
   return false;
 }
 
+/// parseDirectiveArchExtension
+///   ::= .arch_extension [no]feature
+bool AArch64AsmParser::parseDirectiveArchExtension(SMLoc L) {
+  MCAsmParser &Parser = getParser();
+
+  if (getLexer().isNot(AsmToken::Identifier))
+    return Error(getLexer().getLoc(), "expected architecture extension name");
+
+  const AsmToken &Tok = Parser.getTok();
+  StringRef Name = Tok.getString();
+  SMLoc ExtLoc = Tok.getLoc();
+  Lex();
+
+  if (parseToken(AsmToken::EndOfStatement,
+                 "unexpected token in '.arch_extension' directive"))
+    return true;
+
+  bool EnableFeature = true;
+  if (Name.startswith_lower("no")) {
+    EnableFeature = false;
+    Name = Name.substr(2);
+  }
+
+  MCSubtargetInfo &STI = copySTI();
+  FeatureBitset Features = STI.getFeatureBits();
+  for (const auto &Extension : ExtensionMap) {
+    if (Extension.Name != Name)
+      continue;
+
+    if (Extension.Features.none())
+      return Error(ExtLoc, "unsupported architectural extension: " + Name);
+
+    FeatureBitset ToggleFeatures = EnableFeature
+                                       ? (~Features & Extension.Features)
+                                       : (Features & Extension.Features);
+    uint64_t Features =
+        ComputeAvailableFeatures(STI.ToggleFeature(ToggleFeatures));
+    setAvailableFeatures(Features);
+    return false;
+  }
+
+  return Error(ExtLoc, "unknown architectural extension: " + Name);
+}
+
 static SMLoc incrementLoc(SMLoc L, int Offset) {
   return SMLoc::getFromPointer(L.getPointer() + Offset);
 }
@@ -5410,6 +5460,16 @@ bool AArch64AsmParser::parseDirectiveCFINegateRAState() {
   return false;
 }
 
+/// parseDirectiveCFIBKeyFrame
+/// ::= .cfi_b_key
+bool AArch64AsmParser::parseDirectiveCFIBKeyFrame() {
+  if (parseToken(AsmToken::EndOfStatement,
+                 "unexpected token in '.cfi_b_key_frame'"))
+    return true;
+  getStreamer().EmitCFIBKeyFrame();
+  return false;
+}
+
 bool
 AArch64AsmParser::classifySymbolRef(const MCExpr *Expr,
                                     AArch64MCExpr::VariantKind &ELFRefKind,
@@ -5434,10 +5494,16 @@ AArch64AsmParser::classifySymbolRef(const MCExpr *Expr,
   // Check that it looks like a symbol + an addend
   MCValue Res;
   bool Relocatable = Expr->evaluateAsRelocatable(Res, nullptr, nullptr);
-  if (!Relocatable || !Res.getSymA() || Res.getSymB())
+  if (!Relocatable || Res.getSymB())
+    return false;
+
+  // Treat expressions with an ELFRefKind (like ":abs_g1:3", or
+  // ":abs_g1:x" where x is constant) as symbolic even if there is no symbol.
+  if (!Res.getSymA() && ELFRefKind == AArch64MCExpr::VK_INVALID)
     return false;
 
-  DarwinRefKind = Res.getSymA()->getKind();
+  if (Res.getSymA())
+    DarwinRefKind = Res.getSymA()->getKind();
   Addend = Res.getConstant();
 
   // It's some symbol reference + a constant addend, but really
diff --git a/lib/Target/AArch64/AsmParser/LLVMBuild.txt b/lib/Target/AArch64/AsmParser/LLVMBuild.txt
index 11eb9d55f615e7e7aae143d84d994f8d4adfb24f..d98c05c959bdc325c67d91ade99bf5d6a1690357 100644
--- a/lib/Target/AArch64/AsmParser/LLVMBuild.txt
+++ b/lib/Target/AArch64/AsmParser/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/Target/AArch64/AsmParser/LLVMBuild.txt ---------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Target/AArch64/CMakeLists.txt b/lib/Target/AArch64/CMakeLists.txt
index 7778882d4915f3b338f6918e3e3ee97c9e8888ce..e219f5f3b67d5063e7e4c735635796402688bf07 100644
--- a/lib/Target/AArch64/CMakeLists.txt
+++ b/lib/Target/AArch64/CMakeLists.txt
@@ -24,6 +24,7 @@ add_llvm_target(AArch64CodeGen
   AArch64AdvSIMDScalarPass.cpp
   AArch64AsmPrinter.cpp
   AArch64BranchTargets.cpp
+  AArch64CallingConvention.cpp
   AArch64CallLowering.cpp
   AArch64CleanupLocalDynamicTLSPass.cpp
   AArch64CollectLOH.cpp
diff --git a/lib/Target/AArch64/Disassembler/AArch64Disassembler.cpp b/lib/Target/AArch64/Disassembler/AArch64Disassembler.cpp
index 4102f1eb5cc12c6afdacebd52182058db2888844..dc1cf368f8e9cea129ca9081a7962d6cfe679ae9 100644
--- a/lib/Target/AArch64/Disassembler/AArch64Disassembler.cpp
+++ b/lib/Target/AArch64/Disassembler/AArch64Disassembler.cpp
@@ -1,9 +1,8 @@
 //===- AArch64Disassembler.cpp - Disassembler for AArch64 -----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -1779,8 +1778,8 @@ static DecodeStatus DecodeGPRSeqPairsClassRegisterClass(MCInst &Inst,
   if (RegNo & 0x1)
     return Fail;
 
-  unsigned Register = AArch64MCRegisterClasses[RegClassID].getRegister(RegNo);
-  Inst.addOperand(MCOperand::createReg(Register));
+  unsigned Reg = AArch64MCRegisterClasses[RegClassID].getRegister(RegNo / 2);
+  Inst.addOperand(MCOperand::createReg(Reg));
   return Success;
 }
 
diff --git a/lib/Target/AArch64/Disassembler/AArch64Disassembler.h b/lib/Target/AArch64/Disassembler/AArch64Disassembler.h
index bc2f7f181699cf83a3849de3b6137cdb7b36de9f..2ba5a695701f43f4785f78ac9e80a8077ebdffa2 100644
--- a/lib/Target/AArch64/Disassembler/AArch64Disassembler.h
+++ b/lib/Target/AArch64/Disassembler/AArch64Disassembler.h
@@ -1,9 +1,8 @@
 //===- AArch64Disassembler.h - Disassembler for AArch64 ---------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AArch64/Disassembler/AArch64ExternalSymbolizer.cpp b/lib/Target/AArch64/Disassembler/AArch64ExternalSymbolizer.cpp
index 342655a29b1ddf11d55f1e7b44842664ff91a49e..3f815ac8c3d05392fa40fe4d9875da3e2677456f 100644
--- a/lib/Target/AArch64/Disassembler/AArch64ExternalSymbolizer.cpp
+++ b/lib/Target/AArch64/Disassembler/AArch64ExternalSymbolizer.cpp
@@ -1,9 +1,8 @@
 //===- AArch64ExternalSymbolizer.cpp - Symbolizer for AArch64 ---*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/AArch64/Disassembler/AArch64ExternalSymbolizer.h b/lib/Target/AArch64/Disassembler/AArch64ExternalSymbolizer.h
index 49e8449637971e66d9607f1757a7a6cc34a9c7ac..dc72331660ccd2f87b9ce6094a60868f7fe6fc0b 100644
--- a/lib/Target/AArch64/Disassembler/AArch64ExternalSymbolizer.h
+++ b/lib/Target/AArch64/Disassembler/AArch64ExternalSymbolizer.h
@@ -1,9 +1,8 @@
 //===- AArch64ExternalSymbolizer.h - Symbolizer for AArch64 -----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AArch64/Disassembler/LLVMBuild.txt b/lib/Target/AArch64/Disassembler/LLVMBuild.txt
index 73665eb5701a7225b9ea388db62c834c09454f95..aefa0aa728e06273d05c423b67b932b32e624b6d 100644
--- a/lib/Target/AArch64/Disassembler/LLVMBuild.txt
+++ b/lib/Target/AArch64/Disassembler/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/Target/AArch64/Disassembler/LLVMBuild.txt ------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Target/AArch64/InstPrinter/AArch64InstPrinter.cpp b/lib/Target/AArch64/InstPrinter/AArch64InstPrinter.cpp
index dcf2dd2511492485ed6dad89624c8d3d4f0d7810..d302da740697ab8e8a5063df3029b08fa2ab1ae3 100644
--- a/lib/Target/AArch64/InstPrinter/AArch64InstPrinter.cpp
+++ b/lib/Target/AArch64/InstPrinter/AArch64InstPrinter.cpp
@@ -1,9 +1,8 @@
 //==-- AArch64InstPrinter.cpp - Convert AArch64 MCInst to assembly syntax --==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -70,6 +69,11 @@ void AArch64InstPrinter::printInst(const MCInst *MI, raw_ostream &O,
       return;
     }
 
+  if (atomicBarrierDroppedOnZero(Opcode) &&
+      (MI->getOperand(0).getReg() == AArch64::XZR ||
+       MI->getOperand(0).getReg() == AArch64::WZR))
+    printAnnotation(O, "acquire semantics dropped since destination is zero");
+
   // SBFM/UBFM should print to a nicer aliased form if possible.
   if (Opcode == AArch64::SBFMXri || Opcode == AArch64::SBFMWri ||
       Opcode == AArch64::UBFMXri || Opcode == AArch64::UBFMWri) {
diff --git a/lib/Target/AArch64/InstPrinter/AArch64InstPrinter.h b/lib/Target/AArch64/InstPrinter/AArch64InstPrinter.h
index 4e9982f5b7be9604fc1bda6e13138ea80f851613..5e48d8e2f8ec320639c8fe16fb8cd478171d13ab 100644
--- a/lib/Target/AArch64/InstPrinter/AArch64InstPrinter.h
+++ b/lib/Target/AArch64/InstPrinter/AArch64InstPrinter.h
@@ -1,9 +1,8 @@
 //===-- AArch64InstPrinter.h - Convert AArch64 MCInst to assembly syntax --===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AArch64/InstPrinter/LLVMBuild.txt b/lib/Target/AArch64/InstPrinter/LLVMBuild.txt
index a13e842cdd3bf8d6cebfc91654f9d025a0f9a191..880d9925383f4be1008c1dd11899f1ac61d2cce0 100644
--- a/lib/Target/AArch64/InstPrinter/LLVMBuild.txt
+++ b/lib/Target/AArch64/InstPrinter/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/Target/AArch64/InstPrinter/LLVMBuild.txt -------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Target/AArch64/LLVMBuild.txt b/lib/Target/AArch64/LLVMBuild.txt
index 0196c505ba3cc3250551418f08650d2664c51943..93bcbc7715fe5cc18730f438d4971c2c530a15a6 100644
--- a/lib/Target/AArch64/LLVMBuild.txt
+++ b/lib/Target/AArch64/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/Target/AArch64/LLVMBuild.txt -------------------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Target/AArch64/MCTargetDesc/AArch64AddressingModes.h b/lib/Target/AArch64/MCTargetDesc/AArch64AddressingModes.h
index 688ca755d0b5775ec97c20a64ccd8f84c4363c92..05a909f1780a0e0db469fe000a7cc8cece8f0b3f 100644
--- a/lib/Target/AArch64/MCTargetDesc/AArch64AddressingModes.h
+++ b/lib/Target/AArch64/MCTargetDesc/AArch64AddressingModes.h
@@ -1,9 +1,8 @@
 //===- AArch64AddressingModes.h - AArch64 Addressing Modes ------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AArch64/MCTargetDesc/AArch64AsmBackend.cpp b/lib/Target/AArch64/MCTargetDesc/AArch64AsmBackend.cpp
index f21d8b9539f4c21769aa9c437ba3d68b13e8d9c6..4df8acbb66512f085a75b371d0482882684e4e63 100644
--- a/lib/Target/AArch64/MCTargetDesc/AArch64AsmBackend.cpp
+++ b/lib/Target/AArch64/MCTargetDesc/AArch64AsmBackend.cpp
@@ -1,9 +1,8 @@
 //===-- AArch64AsmBackend.cpp - AArch64 Assembler Backend -----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -110,11 +109,11 @@ static unsigned getFixupKindNumBytes(unsigned Kind) {
   case FK_Data_1:
     return 1;
 
-  case AArch64::fixup_aarch64_movw:
   case FK_Data_2:
   case FK_SecRel_2:
     return 2;
 
+  case AArch64::fixup_aarch64_movw:
   case AArch64::fixup_aarch64_pcrel_branch14:
   case AArch64::fixup_aarch64_add_imm12:
   case AArch64::fixup_aarch64_ldst_imm12_scale1:
@@ -145,9 +144,9 @@ static unsigned AdrImmBits(unsigned Value) {
   return (hi19 << 5) | (lo2 << 29);
 }
 
-static uint64_t adjustFixupValue(const MCFixup &Fixup, uint64_t Value,
-                                 MCContext &Ctx, const Triple &TheTriple,
-                                 bool IsResolved) {
+static uint64_t adjustFixupValue(const MCFixup &Fixup, const MCValue &Target,
+                                 uint64_t Value, MCContext &Ctx,
+                                 const Triple &TheTriple, bool IsResolved) {
   unsigned Kind = Fixup.getKind();
   int64_t SignedValue = static_cast<int64_t>(Value);
   switch (Kind) {
@@ -215,10 +214,79 @@ static uint64_t adjustFixupValue(const MCFixup &Fixup, uint64_t Value,
     if (Value & 0xf)
       Ctx.reportError(Fixup.getLoc(), "fixup must be 16-byte aligned");
     return Value >> 4;
-  case AArch64::fixup_aarch64_movw:
-    Ctx.reportError(Fixup.getLoc(),
-                    "no resolvable MOVZ/MOVK fixups supported yet");
+  case AArch64::fixup_aarch64_movw: {
+    AArch64MCExpr::VariantKind RefKind =
+        static_cast<AArch64MCExpr::VariantKind>(Target.getRefKind());
+    if (AArch64MCExpr::getSymbolLoc(RefKind) != AArch64MCExpr::VK_ABS &&
+        AArch64MCExpr::getSymbolLoc(RefKind) != AArch64MCExpr::VK_SABS) {
+      // VK_GOTTPREL, VK_TPREL, VK_DTPREL are movw fixups, but they can't
+      // ever be resolved in the assembler.
+      Ctx.reportError(Fixup.getLoc(),
+                      "relocation for a thread-local variable points to an "
+                      "absolute symbol");
+      return Value;
+    }
+
+    if (!IsResolved) {
+      // FIXME: Figure out when this can actually happen, and verify our
+      // behavior.
+      Ctx.reportError(Fixup.getLoc(), "unresolved movw fixup not yet "
+                                      "implemented");
+      return Value;
+    }
+
+    if (AArch64MCExpr::getSymbolLoc(RefKind) == AArch64MCExpr::VK_SABS) {
+      switch (AArch64MCExpr::getAddressFrag(RefKind)) {
+      case AArch64MCExpr::VK_G0:
+        break;
+      case AArch64MCExpr::VK_G1:
+        SignedValue = SignedValue >> 16;
+        break;
+      case AArch64MCExpr::VK_G2:
+        SignedValue = SignedValue >> 32;
+        break;
+      case AArch64MCExpr::VK_G3:
+        SignedValue = SignedValue >> 48;
+        break;
+      default:
+        llvm_unreachable("Variant kind doesn't correspond to fixup");
+      }
+
+    } else {
+      switch (AArch64MCExpr::getAddressFrag(RefKind)) {
+      case AArch64MCExpr::VK_G0:
+        break;
+      case AArch64MCExpr::VK_G1:
+        Value = Value >> 16;
+        break;
+      case AArch64MCExpr::VK_G2:
+        Value = Value >> 32;
+        break;
+      case AArch64MCExpr::VK_G3:
+        Value = Value >> 48;
+        break;
+      default:
+        llvm_unreachable("Variant kind doesn't correspond to fixup");
+      }
+    }
+
+    if (RefKind & AArch64MCExpr::VK_NC) {
+      Value &= 0xFFFF;
+    }
+    else if (RefKind & AArch64MCExpr::VK_SABS) {
+      if (SignedValue > 0xFFFF || SignedValue < -0xFFFF)
+        Ctx.reportError(Fixup.getLoc(), "fixup value out of range");
+
+      // Invert the negative immediate because it will feed into a MOVN.
+      if (SignedValue < 0)
+        SignedValue = ~SignedValue;
+      Value = static_cast<uint64_t>(SignedValue);
+    }
+    else if (Value > 0xFFFF) {
+      Ctx.reportError(Fixup.getLoc(), "fixup value out of range");
+    }
     return Value;
+  }
   case AArch64::fixup_aarch64_pcrel_branch14:
     // Signed 16-bit immediate
     if (SignedValue > 32767 || SignedValue < -32768)
@@ -295,8 +363,9 @@ void AArch64AsmBackend::applyFixup(const MCAssembler &Asm, const MCFixup &Fixup,
     return; // Doesn't change encoding.
   MCFixupKindInfo Info = getFixupKindInfo(Fixup.getKind());
   MCContext &Ctx = Asm.getContext();
+  int64_t SignedValue = static_cast<int64_t>(Value);
   // Apply any target-specific value adjustments.
-  Value = adjustFixupValue(Fixup, Value, Ctx, TheTriple, IsResolved);
+  Value = adjustFixupValue(Fixup, Target, Value, Ctx, TheTriple, IsResolved);
 
   // Shift the value into position.
   Value <<= Info.TargetOffset;
@@ -323,6 +392,19 @@ void AArch64AsmBackend::applyFixup(const MCAssembler &Asm, const MCFixup &Fixup,
       Data[Offset + Idx] |= uint8_t((Value >> (i * 8)) & 0xff);
     }
   }
+
+  // FIXME: getFixupKindInfo() and getFixupKindNumBytes() could be fixed to
+  // handle this more cleanly. This may affect the output of -show-mc-encoding.
+  AArch64MCExpr::VariantKind RefKind =
+    static_cast<AArch64MCExpr::VariantKind>(Target.getRefKind());
+  if (RefKind & AArch64MCExpr::VK_SABS) {
+    // If the immediate is negative, generate MOVN else MOVZ.
+    // (Bit 30 = 0) ==> MOVN, (Bit 30 = 1) ==> MOVZ.
+    if (SignedValue < 0)
+      Data[Offset + 3] &= ~(1 << 6);
+    else
+      Data[Offset + 3] |= (1 << 6);
+  }
 }
 
 bool AArch64AsmBackend::mayNeedRelaxation(const MCInst &Inst,
diff --git a/lib/Target/AArch64/MCTargetDesc/AArch64ELFObjectWriter.cpp b/lib/Target/AArch64/MCTargetDesc/AArch64ELFObjectWriter.cpp
index 2ccd7cef8bef230302eac7feab92264cb4b82852..d7361809e6d37c8b601353e854478e9b0a16934b 100644
--- a/lib/Target/AArch64/MCTargetDesc/AArch64ELFObjectWriter.cpp
+++ b/lib/Target/AArch64/MCTargetDesc/AArch64ELFObjectWriter.cpp
@@ -1,9 +1,8 @@
 //===-- AArch64ELFObjectWriter.cpp - AArch64 ELF Writer -------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AArch64/MCTargetDesc/AArch64ELFStreamer.cpp b/lib/Target/AArch64/MCTargetDesc/AArch64ELFStreamer.cpp
index 9a7e34b0aeb10f14b798a2b70b5a54ed01fe4268..c33f7e957b54a6c8a11e4e8341a34c14aefc2274 100644
--- a/lib/Target/AArch64/MCTargetDesc/AArch64ELFStreamer.cpp
+++ b/lib/Target/AArch64/MCTargetDesc/AArch64ELFStreamer.cpp
@@ -1,9 +1,8 @@
 //===- lib/MC/AArch64ELFStreamer.cpp - ELF Object Output for AArch64 ------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -103,8 +102,8 @@ public:
   /// This function is the one used to emit instruction data into the ELF
   /// streamer. We override it to add the appropriate mapping symbol if
   /// necessary.
-  void EmitInstruction(const MCInst &Inst, const MCSubtargetInfo &STI,
-                       bool) override {
+  void EmitInstruction(const MCInst &Inst,
+                       const MCSubtargetInfo &STI) override {
     EmitA64MappingSymbol();
     MCELFStreamer::EmitInstruction(Inst, STI);
   }
diff --git a/lib/Target/AArch64/MCTargetDesc/AArch64ELFStreamer.h b/lib/Target/AArch64/MCTargetDesc/AArch64ELFStreamer.h
index d5b009ec30d18b026cae8205ef4d3946d6a345ab..25c609ee1496b3a652c1b9f54e27fa2eb81b33dd 100644
--- a/lib/Target/AArch64/MCTargetDesc/AArch64ELFStreamer.h
+++ b/lib/Target/AArch64/MCTargetDesc/AArch64ELFStreamer.h
@@ -1,9 +1,8 @@
 //===-- AArch64ELFStreamer.h - ELF Streamer for AArch64 ---------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AArch64/MCTargetDesc/AArch64FixupKinds.h b/lib/Target/AArch64/MCTargetDesc/AArch64FixupKinds.h
index 4293dcba955efc90790be0c2a785a8b701c161ae..fe8043fe5ec0c406bb0c44539f995340a6dadb4a 100644
--- a/lib/Target/AArch64/MCTargetDesc/AArch64FixupKinds.h
+++ b/lib/Target/AArch64/MCTargetDesc/AArch64FixupKinds.h
@@ -1,9 +1,8 @@
 //===-- AArch64FixupKinds.h - AArch64 Specific Fixup Entries ----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/AArch64/MCTargetDesc/AArch64MCAsmInfo.cpp b/lib/Target/AArch64/MCTargetDesc/AArch64MCAsmInfo.cpp
index 58e4a9c9a9e9aa7bdc5a0a6b070ed57750acd63f..728e91572e1c2e8e58c180f14abe7013840b8bb4 100644
--- a/lib/Target/AArch64/MCTargetDesc/AArch64MCAsmInfo.cpp
+++ b/lib/Target/AArch64/MCTargetDesc/AArch64MCAsmInfo.cpp
@@ -1,9 +1,8 @@
 //===-- AArch64MCAsmInfo.cpp - AArch64 asm properties ---------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AArch64/MCTargetDesc/AArch64MCAsmInfo.h b/lib/Target/AArch64/MCTargetDesc/AArch64MCAsmInfo.h
index e8570b1c28873df1f2bee81f674c1c945c7d732f..36ae92afc8c1214491093047709f54f8b83acd60 100644
--- a/lib/Target/AArch64/MCTargetDesc/AArch64MCAsmInfo.h
+++ b/lib/Target/AArch64/MCTargetDesc/AArch64MCAsmInfo.h
@@ -1,9 +1,8 @@
 //=====-- AArch64MCAsmInfo.h - AArch64 asm properties ---------*- C++ -*--====//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AArch64/MCTargetDesc/AArch64MCCodeEmitter.cpp b/lib/Target/AArch64/MCTargetDesc/AArch64MCCodeEmitter.cpp
index 41cad48f7aea921e2deebeed4a5235998b214146..fb852da93a96d71cc778fe73a99d54d703750e5b 100644
--- a/lib/Target/AArch64/MCTargetDesc/AArch64MCCodeEmitter.cpp
+++ b/lib/Target/AArch64/MCTargetDesc/AArch64MCCodeEmitter.cpp
@@ -1,9 +1,8 @@
 //=- AArch64/AArch64MCCodeEmitter.cpp - Convert AArch64 code to machine code-=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AArch64/MCTargetDesc/AArch64MCExpr.cpp b/lib/Target/AArch64/MCTargetDesc/AArch64MCExpr.cpp
index 729486b1020c995a7578eb96a949d4fdf8efa67b..20676ebb61a28fabc1f5c78254f0b7f2ab4a2379 100644
--- a/lib/Target/AArch64/MCTargetDesc/AArch64MCExpr.cpp
+++ b/lib/Target/AArch64/MCTargetDesc/AArch64MCExpr.cpp
@@ -1,9 +1,8 @@
 //===-- AArch64MCExpr.cpp - AArch64 specific MC expression classes --------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AArch64/MCTargetDesc/AArch64MCExpr.h b/lib/Target/AArch64/MCTargetDesc/AArch64MCExpr.h
index b6bf254d383589993590863f936c39d10988af86..246060ace034b1bf4f4f31b35aceb6160a46510f 100644
--- a/lib/Target/AArch64/MCTargetDesc/AArch64MCExpr.h
+++ b/lib/Target/AArch64/MCTargetDesc/AArch64MCExpr.h
@@ -1,9 +1,8 @@
 //=--- AArch64MCExpr.h - AArch64 specific MC expression classes ---*- C++ -*-=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AArch64/MCTargetDesc/AArch64MCTargetDesc.cpp b/lib/Target/AArch64/MCTargetDesc/AArch64MCTargetDesc.cpp
index 0f8198ba4e9b679ae5f1105c193ecf099f902656..564d492f06554a266deca9bbb9d28a7ff723b358 100644
--- a/lib/Target/AArch64/MCTargetDesc/AArch64MCTargetDesc.cpp
+++ b/lib/Target/AArch64/MCTargetDesc/AArch64MCTargetDesc.cpp
@@ -1,9 +1,8 @@
 //===-- AArch64MCTargetDesc.cpp - AArch64 Target Descriptions ---*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AArch64/MCTargetDesc/AArch64MCTargetDesc.h b/lib/Target/AArch64/MCTargetDesc/AArch64MCTargetDesc.h
index 0f22f69bd5b0a9a954183a5cc0dea29a7fa6f9b5..4a690c66278733ae7a30308ae1b876deae8c8717 100644
--- a/lib/Target/AArch64/MCTargetDesc/AArch64MCTargetDesc.h
+++ b/lib/Target/AArch64/MCTargetDesc/AArch64MCTargetDesc.h
@@ -1,9 +1,8 @@
 //===-- AArch64MCTargetDesc.h - AArch64 Target Descriptions -----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AArch64/MCTargetDesc/AArch64MachObjectWriter.cpp b/lib/Target/AArch64/MCTargetDesc/AArch64MachObjectWriter.cpp
index 1021cdeeb3be4c4e74928d7cd6c3866671b9058b..e8d9e3d1f723111489993bf475bc3d96f1cb5dbc 100644
--- a/lib/Target/AArch64/MCTargetDesc/AArch64MachObjectWriter.cpp
+++ b/lib/Target/AArch64/MCTargetDesc/AArch64MachObjectWriter.cpp
@@ -1,9 +1,8 @@
 //===-- AArch64MachObjectWriter.cpp - ARM Mach Object Writer --------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/AArch64/MCTargetDesc/AArch64TargetStreamer.cpp b/lib/Target/AArch64/MCTargetDesc/AArch64TargetStreamer.cpp
index a6b8d963bef905f67e9b9a90bd74dca36c8e80a4..ab9519f75ee2649dbd0a86e04c2b66e52d5e1782 100644
--- a/lib/Target/AArch64/MCTargetDesc/AArch64TargetStreamer.cpp
+++ b/lib/Target/AArch64/MCTargetDesc/AArch64TargetStreamer.cpp
@@ -1,9 +1,8 @@
 //===- AArch64TargetStreamer.cpp - AArch64TargetStreamer class ------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AArch64/MCTargetDesc/AArch64TargetStreamer.h b/lib/Target/AArch64/MCTargetDesc/AArch64TargetStreamer.h
index 73fb9baea3e335a3f378e25a8714783589e6ffb7..3a0c5d8318dd5feeb1fd6b76849d7a84282039dc 100644
--- a/lib/Target/AArch64/MCTargetDesc/AArch64TargetStreamer.h
+++ b/lib/Target/AArch64/MCTargetDesc/AArch64TargetStreamer.h
@@ -1,9 +1,8 @@
 //===-- AArch64TargetStreamer.h - AArch64 Target Streamer ------*- C++ -*--===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/AArch64/MCTargetDesc/AArch64WinCOFFObjectWriter.cpp b/lib/Target/AArch64/MCTargetDesc/AArch64WinCOFFObjectWriter.cpp
index 7ea7d5f2a20e659680f0c8c2f0da84e5347572d0..a45880a074272104e5ae22ffabba7c3a901e24d1 100644
--- a/lib/Target/AArch64/MCTargetDesc/AArch64WinCOFFObjectWriter.cpp
+++ b/lib/Target/AArch64/MCTargetDesc/AArch64WinCOFFObjectWriter.cpp
@@ -1,9 +1,8 @@
 //= AArch64WinCOFFObjectWriter.cpp - AArch64 Windows COFF Object Writer C++ =//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===---------------------------------------------------------------------===//
 
diff --git a/lib/Target/AArch64/MCTargetDesc/AArch64WinCOFFStreamer.cpp b/lib/Target/AArch64/MCTargetDesc/AArch64WinCOFFStreamer.cpp
index b828ab832e9d38637533e7e60edf06270d59cd3d..37c6fbb03908128bfc9cb3d6f98181e4f9861f71 100644
--- a/lib/Target/AArch64/MCTargetDesc/AArch64WinCOFFStreamer.cpp
+++ b/lib/Target/AArch64/MCTargetDesc/AArch64WinCOFFStreamer.cpp
@@ -1,9 +1,8 @@
 //===-- AArch64WinCOFFStreamer.cpp - ARM Target WinCOFF Streamer ----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/AArch64/MCTargetDesc/AArch64WinCOFFStreamer.h b/lib/Target/AArch64/MCTargetDesc/AArch64WinCOFFStreamer.h
index ed265a876ab3cf906a8c2bea29c08cfca267b5b0..8c0656652eed2afc3b4600020df54c23c94ae8c5 100644
--- a/lib/Target/AArch64/MCTargetDesc/AArch64WinCOFFStreamer.h
+++ b/lib/Target/AArch64/MCTargetDesc/AArch64WinCOFFStreamer.h
@@ -1,9 +1,8 @@
 //===-- AArch64WinCOFFStreamer.h - WinCOFF Streamer for AArch64 -*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AArch64/MCTargetDesc/LLVMBuild.txt b/lib/Target/AArch64/MCTargetDesc/LLVMBuild.txt
index 70cff0b704f759bb5131b2d11871e248087c94b3..cf37da5afac9e816151a7983aa60f1b2481dd75c 100644
--- a/lib/Target/AArch64/MCTargetDesc/LLVMBuild.txt
+++ b/lib/Target/AArch64/MCTargetDesc/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/Target/AArch64/MCTargetDesc/LLVMBuild.txt ------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Target/AArch64/SVEInstrFormats.td b/lib/Target/AArch64/SVEInstrFormats.td
index 23a65b345badd43a5afd0b91efebc50c7c706b3f..3a3efaa96ce600bf75458f72735d9d6cb32f0695 100644
--- a/lib/Target/AArch64/SVEInstrFormats.td
+++ b/lib/Target/AArch64/SVEInstrFormats.td
@@ -1,9 +1,8 @@
 //=-- SVEInstrFormats.td -  AArch64 SVE Instruction classes -*- tablegen -*--=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AArch64/TargetInfo/AArch64TargetInfo.cpp b/lib/Target/AArch64/TargetInfo/AArch64TargetInfo.cpp
index 8fb161574c5b63cdd7510c0aa80a523db9569356..e710f46393549b7a485c7b6a459019695cf2034b 100644
--- a/lib/Target/AArch64/TargetInfo/AArch64TargetInfo.cpp
+++ b/lib/Target/AArch64/TargetInfo/AArch64TargetInfo.cpp
@@ -1,9 +1,8 @@
 //===-- AArch64TargetInfo.cpp - AArch64 Target Implementation -----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/AArch64/TargetInfo/LLVMBuild.txt b/lib/Target/AArch64/TargetInfo/LLVMBuild.txt
index 93c5407bb1f1fc4aa8c0d5ff7adacf308fc869f3..31114a136a9a9212a78f97edb3c012e0eb7649a2 100644
--- a/lib/Target/AArch64/TargetInfo/LLVMBuild.txt
+++ b/lib/Target/AArch64/TargetInfo/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/Target/AArch64/TargetInfo/LLVMBuild.txt --------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Target/AArch64/Utils/AArch64BaseInfo.cpp b/lib/Target/AArch64/Utils/AArch64BaseInfo.cpp
index c88155db7037200f55ef227154d2d5eceb4266aa..7bb075c36e7982aae91974afcd4ebd1f419dbb53 100644
--- a/lib/Target/AArch64/Utils/AArch64BaseInfo.cpp
+++ b/lib/Target/AArch64/Utils/AArch64BaseInfo.cpp
@@ -1,9 +1,8 @@
 //===-- AArch64BaseInfo.cpp - AArch64 Base encoding information------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AArch64/Utils/AArch64BaseInfo.h b/lib/Target/AArch64/Utils/AArch64BaseInfo.h
index b8d0977ae547eb5d76645f328bed458d77ce0a27..e5e2fc2cb0df4b9b5b7efb4b50bb17e9f0d0478a 100644
--- a/lib/Target/AArch64/Utils/AArch64BaseInfo.h
+++ b/lib/Target/AArch64/Utils/AArch64BaseInfo.h
@@ -1,9 +1,8 @@
 //===-- AArch64BaseInfo.h - Top level definitions for AArch64 ---*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -186,6 +185,49 @@ static inline unsigned getDRegFromBReg(unsigned Reg) {
   return Reg;
 }
 
+static inline bool atomicBarrierDroppedOnZero(unsigned Opcode) {
+  switch (Opcode) {
+  case AArch64::LDADDAB:   case AArch64::LDADDAH:
+  case AArch64::LDADDAW:   case AArch64::LDADDAX:
+  case AArch64::LDADDALB:  case AArch64::LDADDALH:
+  case AArch64::LDADDALW:  case AArch64::LDADDALX:
+  case AArch64::LDCLRAB:   case AArch64::LDCLRAH:
+  case AArch64::LDCLRAW:   case AArch64::LDCLRAX:
+  case AArch64::LDCLRALB:  case AArch64::LDCLRALH:
+  case AArch64::LDCLRALW:  case AArch64::LDCLRALX:
+  case AArch64::LDEORAB:   case AArch64::LDEORAH:
+  case AArch64::LDEORAW:   case AArch64::LDEORAX:
+  case AArch64::LDEORALB:  case AArch64::LDEORALH:
+  case AArch64::LDEORALW:  case AArch64::LDEORALX:
+  case AArch64::LDSETAB:   case AArch64::LDSETAH:
+  case AArch64::LDSETAW:   case AArch64::LDSETAX:
+  case AArch64::LDSETALB:  case AArch64::LDSETALH:
+  case AArch64::LDSETALW:  case AArch64::LDSETALX:
+  case AArch64::LDSMAXAB:  case AArch64::LDSMAXAH:
+  case AArch64::LDSMAXAW:  case AArch64::LDSMAXAX:
+  case AArch64::LDSMAXALB: case AArch64::LDSMAXALH:
+  case AArch64::LDSMAXALW: case AArch64::LDSMAXALX:
+  case AArch64::LDSMINAB:  case AArch64::LDSMINAH:
+  case AArch64::LDSMINAW:  case AArch64::LDSMINAX:
+  case AArch64::LDSMINALB: case AArch64::LDSMINALH:
+  case AArch64::LDSMINALW: case AArch64::LDSMINALX:
+  case AArch64::LDUMAXAB:  case AArch64::LDUMAXAH:
+  case AArch64::LDUMAXAW:  case AArch64::LDUMAXAX:
+  case AArch64::LDUMAXALB: case AArch64::LDUMAXALH:
+  case AArch64::LDUMAXALW: case AArch64::LDUMAXALX:
+  case AArch64::LDUMINAB:  case AArch64::LDUMINAH:
+  case AArch64::LDUMINAW:  case AArch64::LDUMINAX:
+  case AArch64::LDUMINALB: case AArch64::LDUMINALH:
+  case AArch64::LDUMINALW: case AArch64::LDUMINALX:
+  case AArch64::SWPAB:     case AArch64::SWPAH:
+  case AArch64::SWPAW:     case AArch64::SWPAX:
+  case AArch64::SWPALB:    case AArch64::SWPALH:
+  case AArch64::SWPALW:    case AArch64::SWPALX:
+    return true;
+  }
+  return false;
+}
+
 namespace AArch64CC {
 
 // The CondCodes constants map directly to the 4-bit encoding of the condition
@@ -581,6 +623,10 @@ namespace AArch64II {
     /// to the symbol is for an import stub.  This is used for DLL import
     /// storage class indication on Windows.
     MO_DLLIMPORT = 0x80,
+
+    /// MO_S - Indicates that the bits of the symbol operand represented by
+    /// MO_G0 etc are signed.
+    MO_S = 0x100,
   };
 } // end namespace AArch64II
 
diff --git a/lib/Target/AArch64/Utils/LLVMBuild.txt b/lib/Target/AArch64/Utils/LLVMBuild.txt
index bcefeb672f76ca816e3f1337da8a2fce977c3445..4a994a739732d8efac940a8e3786aa476a919b61 100644
--- a/lib/Target/AArch64/Utils/LLVMBuild.txt
+++ b/lib/Target/AArch64/Utils/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/Target/AArch64/Utils/LLVMBuild.txt ----------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Target/AArch64/select-bswap.mir b/lib/Target/AArch64/select-bswap.mir
new file mode 100644
index 0000000000000000000000000000000000000000..13dfa99e3fa25c71c24ec4c29cea4590ded142a5
--- /dev/null
+++ b/lib/Target/AArch64/select-bswap.mir
@@ -0,0 +1,38 @@
+# RUN: llc -verify-machineinstrs -mtriple aarch64--- \
+# RUN: -run-pass=instruction-select -mattr=+fullfp16 -global-isel %s -o - \
+# RUN: | FileCheck %s
+...
+---
+name:            bswap_s32
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    ; CHECK-LABEL: name:            bswap_s32
+    ; CHECK: [[COPY:%[0-9]+]]:gpr32 = COPY $w0
+    ; CHECK: [[REVWr:%[0-9]+]]:gpr32 = REVWr [[COPY]]
+    ; CHECK: $w0 = COPY [[REVWr]]
+    liveins: $s0
+    %0:gpr(s32) = COPY $w0
+    %1:gpr(s32) = G_BSWAP %0
+    $w0 = COPY %1(s32)
+
+...
+---
+name:            bswap_s64
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    ; CHECK-LABEL: name:            bswap_s64
+    ; CHECK: [[COPY:%[0-9]+]]:gpr64 = COPY $x0
+    ; CHECK: [[REVXr:%[0-9]+]]:gpr64 = REVXr [[COPY]]
+    ; CHECK: $x0 = COPY [[REVXr]]
+    liveins: $x0
+    %0:gpr(s64) = COPY $d0
+    %1:gpr(s64) = G_BSWAP %0
+    $x0 = COPY %1(s64)
+
+...
diff --git a/lib/Target/AMDGPU/AMDGPU.h b/lib/Target/AMDGPU/AMDGPU.h
index d26397a4271a0709c91925d70c636b289ec9b827..344c3f4fbf7d8eb4b1f66b977cdf53412f69bff6 100644
--- a/lib/Target/AMDGPU/AMDGPU.h
+++ b/lib/Target/AMDGPU/AMDGPU.h
@@ -1,9 +1,8 @@
 //===-- AMDGPU.h - MachineFunction passes hw codegen --------------*- C++ -*-=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 /// \file
 //===----------------------------------------------------------------------===//
@@ -43,6 +42,7 @@ FunctionPass *createSIFoldOperandsPass();
 FunctionPass *createSIPeepholeSDWAPass();
 FunctionPass *createSILowerI1CopiesPass();
 FunctionPass *createSIFixupVectorISelPass();
+FunctionPass *createSIAddIMGInitPass();
 FunctionPass *createSIShrinkInstructionsPass();
 FunctionPass *createSILoadStoreOptimizerPass();
 FunctionPass *createSIWholeQuadModePass();
@@ -158,6 +158,9 @@ extern char &AMDGPUSimplifyLibCallsID;
 void initializeAMDGPUUseNativeCallsPass(PassRegistry &);
 extern char &AMDGPUUseNativeCallsID;
 
+void initializeSIAddIMGInitPass(PassRegistry &);
+extern char &SIAddIMGInitID;
+
 void initializeAMDGPUPerfHintAnalysisPass(PassRegistry &);
 extern char &AMDGPUPerfHintAnalysisID;
 
@@ -250,7 +253,7 @@ namespace AMDGPUAS {
 
     FLAT_ADDRESS = 0,     ///< Address space for flat memory.
     GLOBAL_ADDRESS = 1,   ///< Address space for global memory (RAT0, VTX0).
-    REGION_ADDRESS = 2,   ///< Address space for region memory.
+    REGION_ADDRESS = 2,   ///< Address space for region memory. (GDS)
 
     CONSTANT_ADDRESS = 4, ///< Address space for constant memory (VTX2)
     LOCAL_ADDRESS = 3,    ///< Address space for local memory.
diff --git a/lib/Target/AMDGPU/AMDGPU.td b/lib/Target/AMDGPU/AMDGPU.td
index 0aacedf24e32b84b07c189882b9abd19e6a7d028..0b0c125e73f38ddaf82a3d9f12241c094ae3e7ff 100644
--- a/lib/Target/AMDGPU/AMDGPU.td
+++ b/lib/Target/AMDGPU/AMDGPU.td
@@ -1,9 +1,8 @@
 //===-- AMDGPU.td - AMDGPU Tablegen files --------*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===------------------------------------------------------------===//
 
@@ -267,7 +266,19 @@ def FeatureUnpackedD16VMem : SubtargetFeature<"unpacked-d16-vmem",
 def FeatureDLInsts : SubtargetFeature<"dl-insts",
   "HasDLInsts",
   "true",
-  "Has deep learning instructions"
+  "Has v_fmac_f32 and v_xnor_b32 instructions"
+>;
+
+def FeatureDot1Insts : SubtargetFeature<"dot1-insts",
+  "HasDot1Insts",
+  "true",
+  "Has v_dot4_i32_i8 and v_dot8_i32_i4 instructions"
+>;
+
+def FeatureDot2Insts : SubtargetFeature<"dot2-insts",
+  "HasDot2Insts",
+  "true",
+  "Has v_dot2_f32_f16, v_dot2_i32_i16, v_dot2_u32_u16, v_dot4_u32_u8, v_dot8_u32_u4 instructions"
 >;
 
 def FeatureSRAMECC : SubtargetFeature<"sram-ecc",
@@ -371,6 +382,16 @@ def FeatureEnableDS128 : SubtargetFeature<"enable-ds128",
   "Use ds_{read|write}_b128"
 >;
 
+// Sparse texture support requires that all result registers are zeroed when
+// PRTStrictNull is set to true. This feature is turned on for all architectures
+// but is enabled as a feature in case there are situations where PRTStrictNull
+// is disabled by the driver.
+def FeatureEnablePRTStrictNull : SubtargetFeature<"enable-prt-strict-null",
+  "EnablePRTStrictNull",
+  "true",
+  "Enable zeroing of result registers for sparse texture fetches"
+>;
+
 // Unless +-flat-for-global is specified, turn on FlatForGlobal for
 // all OS-es on VI and newer hardware to avoid assertion failures due
 // to missing ADDR64 variants of MUBUF instructions.
@@ -409,31 +430,25 @@ def FeatureDisable : SubtargetFeature<"",
   "Dummy feature to disable assembler instructions"
 >;
 
-def FeatureGCN : SubtargetFeature<"gcn",
-  "IsGCN",
-  "true",
-  "GCN or newer GPU"
->;
-
 class GCNSubtargetFeatureGeneration <string Value,
                                   list<SubtargetFeature> Implies> :
         SubtargetFeatureGeneration <Value, "GCNSubtarget", Implies>;
 
 def FeatureSouthernIslands : GCNSubtargetFeatureGeneration<"SOUTHERN_ISLANDS",
   [FeatureFP64, FeatureLocalMemorySize32768, FeatureMIMG_R128,
-  FeatureWavefrontSize64, FeatureGCN,
+  FeatureWavefrontSize64,
   FeatureLDSBankCount32, FeatureMovrel, FeatureTrigReducedRange]
 >;
 
 def FeatureSeaIslands : GCNSubtargetFeatureGeneration<"SEA_ISLANDS",
   [FeatureFP64, FeatureLocalMemorySize65536, FeatureMIMG_R128,
-  FeatureWavefrontSize64, FeatureGCN, FeatureFlatAddressSpace,
+  FeatureWavefrontSize64, FeatureFlatAddressSpace,
   FeatureCIInsts, FeatureMovrel, FeatureTrigReducedRange]
 >;
 
 def FeatureVolcanicIslands : GCNSubtargetFeatureGeneration<"VOLCANIC_ISLANDS",
   [FeatureFP64, FeatureLocalMemorySize65536, FeatureMIMG_R128,
-   FeatureWavefrontSize64, FeatureFlatAddressSpace, FeatureGCN,
+   FeatureWavefrontSize64, FeatureFlatAddressSpace,
    FeatureGCN3Encoding, FeatureCIInsts, FeatureVIInsts, Feature16BitInsts,
    FeatureSMemRealTime, FeatureVGPRIndexMode, FeatureMovrel,
    FeatureScalarStores, FeatureInv2PiInlineImm,
@@ -444,7 +459,7 @@ def FeatureVolcanicIslands : GCNSubtargetFeatureGeneration<"VOLCANIC_ISLANDS",
 
 def FeatureGFX9 : GCNSubtargetFeatureGeneration<"GFX9",
   [FeatureFP64, FeatureLocalMemorySize65536,
-   FeatureWavefrontSize64, FeatureFlatAddressSpace, FeatureGCN,
+   FeatureWavefrontSize64, FeatureFlatAddressSpace,
    FeatureGCN3Encoding, FeatureCIInsts, FeatureVIInsts, Feature16BitInsts,
    FeatureSMemRealTime, FeatureScalarStores, FeatureInv2PiInlineImm,
    FeatureApertureRegs, FeatureGFX9Insts, FeatureVOP3P, FeatureVGPRIndexMode,
@@ -455,57 +470,50 @@ def FeatureGFX9 : GCNSubtargetFeatureGeneration<"GFX9",
   ]
 >;
 
-class SubtargetFeatureISAVersion <int Major, int Minor, int Stepping,
-                                  list<SubtargetFeature> Implies>
-                                 : SubtargetFeature <
-  "isaver"#Major#"."#Minor#"."#Stepping,
-  "IsaVersion",
-  "ISAVersion"#Major#"_"#Minor#"_"#Stepping,
-  "Instruction set version number",
-  Implies
->;
+class FeatureSet<list<SubtargetFeature> Features_> {
+  list<SubtargetFeature> Features = Features_;
+}
 
-def FeatureISAVersion6_0_0 : SubtargetFeatureISAVersion <6,0,0,
-  [FeatureSouthernIslands,
+def FeatureISAVersion6_0_0 : FeatureSet<[FeatureSouthernIslands,
    FeatureFastFMAF32,
    HalfRate64Ops,
    FeatureLDSBankCount32,
    FeatureCodeObjectV3]>;
 
-def FeatureISAVersion6_0_1 : SubtargetFeatureISAVersion <6,0,1,
+def FeatureISAVersion6_0_1 : FeatureSet<
   [FeatureSouthernIslands,
    FeatureLDSBankCount32,
    FeatureCodeObjectV3]>;
 
-def FeatureISAVersion7_0_0 : SubtargetFeatureISAVersion <7,0,0,
+def FeatureISAVersion7_0_0 : FeatureSet<
   [FeatureSeaIslands,
    FeatureLDSBankCount32,
    FeatureCodeObjectV3]>;
 
-def FeatureISAVersion7_0_1 : SubtargetFeatureISAVersion <7,0,1,
+def FeatureISAVersion7_0_1 : FeatureSet<
   [FeatureSeaIslands,
    HalfRate64Ops,
    FeatureLDSBankCount32,
    FeatureFastFMAF32,
    FeatureCodeObjectV3]>;
 
-def FeatureISAVersion7_0_2 : SubtargetFeatureISAVersion <7,0,2,
+def FeatureISAVersion7_0_2 : FeatureSet<
   [FeatureSeaIslands,
    FeatureLDSBankCount16,
    FeatureFastFMAF32,
    FeatureCodeObjectV3]>;
 
-def FeatureISAVersion7_0_3 : SubtargetFeatureISAVersion <7,0,3,
+def FeatureISAVersion7_0_3 : FeatureSet<
   [FeatureSeaIslands,
    FeatureLDSBankCount16,
    FeatureCodeObjectV3]>;
 
-def FeatureISAVersion7_0_4 : SubtargetFeatureISAVersion <7,0,4,
+def FeatureISAVersion7_0_4 : FeatureSet<
   [FeatureSeaIslands,
    FeatureLDSBankCount32,
    FeatureCodeObjectV3]>;
 
-def FeatureISAVersion8_0_1 : SubtargetFeatureISAVersion <8,0,1,
+def FeatureISAVersion8_0_1 : FeatureSet<
   [FeatureVolcanicIslands,
    FeatureFastFMAF32,
    HalfRate64Ops,
@@ -514,54 +522,56 @@ def FeatureISAVersion8_0_1 : SubtargetFeatureISAVersion <8,0,1,
    FeatureUnpackedD16VMem,
    FeatureCodeObjectV3]>;
 
-def FeatureISAVersion8_0_2 : SubtargetFeatureISAVersion <8,0,2,
+def FeatureISAVersion8_0_2 : FeatureSet<
   [FeatureVolcanicIslands,
    FeatureLDSBankCount32,
    FeatureSGPRInitBug,
    FeatureUnpackedD16VMem,
    FeatureCodeObjectV3]>;
 
-def FeatureISAVersion8_0_3 : SubtargetFeatureISAVersion <8,0,3,
+def FeatureISAVersion8_0_3 : FeatureSet<
   [FeatureVolcanicIslands,
    FeatureLDSBankCount32,
    FeatureUnpackedD16VMem,
    FeatureCodeObjectV3]>;
 
-def FeatureISAVersion8_1_0 : SubtargetFeatureISAVersion <8,1,0,
+def FeatureISAVersion8_1_0 : FeatureSet<
   [FeatureVolcanicIslands,
    FeatureLDSBankCount16,
    FeatureXNACK,
    FeatureCodeObjectV3]>;
 
-def FeatureISAVersion9_0_0 : SubtargetFeatureISAVersion <9,0,0,
+def FeatureISAVersion9_0_0 : FeatureSet<
   [FeatureGFX9,
    FeatureMadMixInsts,
    FeatureLDSBankCount32,
    FeatureCodeObjectV3]>;
 
-def FeatureISAVersion9_0_2 : SubtargetFeatureISAVersion <9,0,2,
+def FeatureISAVersion9_0_2 : FeatureSet<
   [FeatureGFX9,
    FeatureMadMixInsts,
    FeatureLDSBankCount32,
    FeatureXNACK,
    FeatureCodeObjectV3]>;
 
-def FeatureISAVersion9_0_4 : SubtargetFeatureISAVersion <9,0,4,
+def FeatureISAVersion9_0_4 : FeatureSet<
   [FeatureGFX9,
    FeatureLDSBankCount32,
    FeatureFmaMixInsts,
    FeatureCodeObjectV3]>;
 
-def FeatureISAVersion9_0_6 : SubtargetFeatureISAVersion <9,0,6,
+def FeatureISAVersion9_0_6 : FeatureSet<
   [FeatureGFX9,
    HalfRate64Ops,
    FeatureFmaMixInsts,
    FeatureLDSBankCount32,
    FeatureDLInsts,
+   FeatureDot1Insts,
+   FeatureDot2Insts,
    FeatureSRAMECC,
    FeatureCodeObjectV3]>;
 
-def FeatureISAVersion9_0_9 : SubtargetFeatureISAVersion <9,0,9,
+def FeatureISAVersion9_0_9 : FeatureSet<
   [FeatureGFX9,
    FeatureMadMixInsts,
    FeatureLDSBankCount32,
@@ -756,6 +766,12 @@ def HasFmaMixInsts : Predicate<"Subtarget->hasFmaMixInsts()">,
 def HasDLInsts : Predicate<"Subtarget->hasDLInsts()">,
   AssemblerPredicate<"FeatureDLInsts">;
 
+def HasDot1Insts : Predicate<"Subtarget->hasDot1Insts()">,
+  AssemblerPredicate<"FeatureDot1Insts">;
+
+def HasDot2Insts : Predicate<"Subtarget->hasDot2Insts()">,
+  AssemblerPredicate<"FeatureDot2Insts">;
+
 
 def EnableLateCFGStructurize : Predicate<
   "EnableLateStructurizeCFG">;
@@ -764,7 +780,6 @@ def EnableLateCFGStructurize : Predicate<
 include "SISchedule.td"
 include "GCNProcessors.td"
 include "AMDGPUInstrInfo.td"
-include "SIIntrinsics.td"
 include "AMDGPURegisterInfo.td"
 include "AMDGPURegisterBanks.td"
 include "AMDGPUInstructions.td"
diff --git a/lib/Target/AMDGPU/AMDGPUAliasAnalysis.cpp b/lib/Target/AMDGPU/AMDGPUAliasAnalysis.cpp
index 73709ba13643ee06673f64a380b5c93f2d29de4e..629332150cc524277a708d4b38d679c43241ccef 100644
--- a/lib/Target/AMDGPU/AMDGPUAliasAnalysis.cpp
+++ b/lib/Target/AMDGPU/AMDGPUAliasAnalysis.cpp
@@ -1,9 +1,8 @@
 //===- AMDGPUAliasAnalysis ------------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/lib/Target/AMDGPU/AMDGPUAliasAnalysis.h b/lib/Target/AMDGPU/AMDGPUAliasAnalysis.h
index d76c9fc481995de8ea655e3f7b8a743acf1c640b..0e01a4698af3694e027e1e6ad36b73c4bb3d6c0d 100644
--- a/lib/Target/AMDGPU/AMDGPUAliasAnalysis.h
+++ b/lib/Target/AMDGPU/AMDGPUAliasAnalysis.h
@@ -1,9 +1,8 @@
 //===- AMDGPUAliasAnalysis --------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/lib/Target/AMDGPU/AMDGPUAlwaysInlinePass.cpp b/lib/Target/AMDGPU/AMDGPUAlwaysInlinePass.cpp
index fc65430b745f3f246bf6b436d4cb362d346306b4..4c1dbd4c53046f44dd4ace78f9d529e553b52892 100644
--- a/lib/Target/AMDGPU/AMDGPUAlwaysInlinePass.cpp
+++ b/lib/Target/AMDGPU/AMDGPUAlwaysInlinePass.cpp
@@ -1,9 +1,8 @@
 //===-- AMDGPUAlwaysInlinePass.cpp - Promote Allocas ----------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AMDGPU/AMDGPUAnnotateKernelFeatures.cpp b/lib/Target/AMDGPU/AMDGPUAnnotateKernelFeatures.cpp
index 896ac9c87779eda62b38ac0ac7db32ea4d12d7cc..96e7603106574a55411432bd8d36bbafc66113a6 100644
--- a/lib/Target/AMDGPU/AMDGPUAnnotateKernelFeatures.cpp
+++ b/lib/Target/AMDGPU/AMDGPUAnnotateKernelFeatures.cpp
@@ -1,9 +1,8 @@
 //===- AMDGPUAnnotateKernelFeaturesPass.cpp -------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AMDGPU/AMDGPUAnnotateUniformValues.cpp b/lib/Target/AMDGPU/AMDGPUAnnotateUniformValues.cpp
index 4e0cc736baddf2872ad4514e5d7df70562c9beab..71121ade0a49bae3b499329efb4b27665c973e49 100644
--- a/lib/Target/AMDGPU/AMDGPUAnnotateUniformValues.cpp
+++ b/lib/Target/AMDGPU/AMDGPUAnnotateUniformValues.cpp
@@ -1,9 +1,8 @@
 //===-- AMDGPUAnnotateUniformValues.cpp - ---------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -14,7 +13,6 @@
 //===----------------------------------------------------------------------===//
 
 #include "AMDGPU.h"
-#include "AMDGPUIntrinsicInfo.h"
 #include "llvm/ADT/SetVector.h"
 #include "llvm/Analysis/LegacyDivergenceAnalysis.h"
 #include "llvm/Analysis/LoopInfo.h"
@@ -117,14 +115,8 @@ bool AMDGPUAnnotateUniformValues::isClobberedInFunction(LoadInst * Load) {
 }
 
 void AMDGPUAnnotateUniformValues::visitBranchInst(BranchInst &I) {
-  if (I.isUnconditional())
-    return;
-
-  Value *Cond = I.getCondition();
-  if (!DA->isUniform(Cond))
-    return;
-
-  setUniformMetadata(I.getParent()->getTerminator());
+  if (DA->isUniform(&I))
+    setUniformMetadata(I.getParent()->getTerminator());
 }
 
 void AMDGPUAnnotateUniformValues::visitLoadInst(LoadInst &I) {
diff --git a/lib/Target/AMDGPU/AMDGPUArgumentUsageInfo.cpp b/lib/Target/AMDGPU/AMDGPUArgumentUsageInfo.cpp
index 7465cf22b5a4f2314440b4f3f8e05798c109d7c4..81c3356ab9ef3ebb7d5e20accc3d01b691795e32 100644
--- a/lib/Target/AMDGPU/AMDGPUArgumentUsageInfo.cpp
+++ b/lib/Target/AMDGPU/AMDGPUArgumentUsageInfo.cpp
@@ -1,9 +1,8 @@
 //===----------------------------------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/AMDGPU/AMDGPUArgumentUsageInfo.h b/lib/Target/AMDGPU/AMDGPUArgumentUsageInfo.h
index f0e6d1b83f15d04c2416b10aa42e3d14e0d90792..277f3616031f251ef763af414e0146333b968d31 100644
--- a/lib/Target/AMDGPU/AMDGPUArgumentUsageInfo.h
+++ b/lib/Target/AMDGPU/AMDGPUArgumentUsageInfo.h
@@ -1,9 +1,8 @@
 //==- AMDGPUArgumentrUsageInfo.h - Function Arg Usage Info -------*- C++ -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp b/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp
index 2ded7cdb648994026a4990431e5479dc25130b16..1b62b3f1d09399c4cc2627ade6ff62471385e244 100644
--- a/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp
+++ b/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp
@@ -1,9 +1,8 @@
 //===-- AMDGPUAsmPrinter.cpp - AMDGPU assembly printer  -------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -100,7 +99,7 @@ extern "C" void LLVMInitializeAMDGPUAsmPrinter() {
 AMDGPUAsmPrinter::AMDGPUAsmPrinter(TargetMachine &TM,
                                    std::unique_ptr<MCStreamer> Streamer)
   : AsmPrinter(TM, std::move(Streamer)) {
-    if (IsaInfo::hasCodeObjectV3(getSTI()))
+    if (IsaInfo::hasCodeObjectV3(getGlobalSTI()))
       HSAMetadataStream.reset(new MetadataStreamerV3());
     else
       HSAMetadataStream.reset(new MetadataStreamerV2());
@@ -110,7 +109,7 @@ StringRef AMDGPUAsmPrinter::getPassName() const {
   return "AMDGPU Assembly Printer";
 }
 
-const MCSubtargetInfo* AMDGPUAsmPrinter::getSTI() const {
+const MCSubtargetInfo *AMDGPUAsmPrinter::getGlobalSTI() const {
   return TM.getMCSubtargetInfo();
 }
 
@@ -121,10 +120,10 @@ AMDGPUTargetStreamer* AMDGPUAsmPrinter::getTargetStreamer() const {
 }
 
 void AMDGPUAsmPrinter::EmitStartOfAsmFile(Module &M) {
-  if (IsaInfo::hasCodeObjectV3(getSTI())) {
+  if (IsaInfo::hasCodeObjectV3(getGlobalSTI())) {
     std::string ExpectedTarget;
     raw_string_ostream ExpectedTargetOS(ExpectedTarget);
-    IsaInfo::streamIsaVersion(getSTI(), ExpectedTargetOS);
+    IsaInfo::streamIsaVersion(getGlobalSTI(), ExpectedTargetOS);
 
     getTargetStreamer()->EmitDirectiveAMDGCNTarget(ExpectedTarget);
   }
@@ -139,7 +138,7 @@ void AMDGPUAsmPrinter::EmitStartOfAsmFile(Module &M) {
   if (TM.getTargetTriple().getOS() == Triple::AMDPAL)
     readPALMetadata(M);
 
-  if (IsaInfo::hasCodeObjectV3(getSTI()))
+  if (IsaInfo::hasCodeObjectV3(getGlobalSTI()))
     return;
 
   // HSA emits NT_AMDGPU_HSA_CODE_OBJECT_VERSION for code objects v2.
@@ -147,7 +146,7 @@ void AMDGPUAsmPrinter::EmitStartOfAsmFile(Module &M) {
     getTargetStreamer()->EmitDirectiveHSACodeObjectVersion(2, 1);
 
   // HSA and PAL emit NT_AMDGPU_HSA_ISA for code objects v2.
-  IsaVersion Version = getIsaVersion(getSTI()->getCPU());
+  IsaVersion Version = getIsaVersion(getGlobalSTI()->getCPU());
   getTargetStreamer()->EmitDirectiveHSACodeObjectISA(
       Version.Major, Version.Minor, Version.Stepping, "AMD", "AMDGPU");
 }
@@ -157,11 +156,11 @@ void AMDGPUAsmPrinter::EmitEndOfAsmFile(Module &M) {
   if (!getTargetStreamer())
     return;
 
-  if (!IsaInfo::hasCodeObjectV3(getSTI())) {
+  if (!IsaInfo::hasCodeObjectV3(getGlobalSTI())) {
     // Emit ISA Version (NT_AMD_AMDGPU_ISA).
     std::string ISAVersionString;
     raw_string_ostream ISAVersionStream(ISAVersionString);
-    IsaInfo::streamIsaVersion(getSTI(), ISAVersionStream);
+    IsaInfo::streamIsaVersion(getGlobalSTI(), ISAVersionStream);
     getTargetStreamer()->EmitISAVersion(ISAVersionStream.str());
   }
 
@@ -173,7 +172,7 @@ void AMDGPUAsmPrinter::EmitEndOfAsmFile(Module &M) {
     assert(Success && "Malformed HSA Metadata");
   }
 
-  if (!IsaInfo::hasCodeObjectV3(getSTI())) {
+  if (!IsaInfo::hasCodeObjectV3(getGlobalSTI())) {
     // Emit PAL Metadata (NT_AMD_AMDGPU_PAL_METADATA).
     if (TM.getTargetTriple().getOS() == Triple::AMDPAL) {
       // Copy the PAL metadata from the map where we collected it into a vector,
@@ -225,7 +224,8 @@ void AMDGPUAsmPrinter::EmitFunctionBodyEnd() {
   const SIMachineFunctionInfo &MFI = *MF->getInfo<SIMachineFunctionInfo>();
   if (!MFI.isEntryFunction())
     return;
-  if (!IsaInfo::hasCodeObjectV3(getSTI()) ||
+
+  if (!IsaInfo::hasCodeObjectV3(getGlobalSTI()) ||
       TM.getTargetTriple().getOS() != Triple::AMDHSA)
     return;
 
@@ -243,23 +243,25 @@ void AMDGPUAsmPrinter::EmitFunctionBodyEnd() {
   if (ReadOnlySection.getAlignment() < 64)
     ReadOnlySection.setAlignment(64);
 
+  const MCSubtargetInfo &STI = MF->getSubtarget();
+
   SmallString<128> KernelName;
   getNameWithPrefix(KernelName, &MF->getFunction());
   getTargetStreamer()->EmitAmdhsaKernelDescriptor(
-      *getSTI(), KernelName, getAmdhsaKernelDescriptor(*MF, CurrentProgramInfo),
+      STI, KernelName, getAmdhsaKernelDescriptor(*MF, CurrentProgramInfo),
       CurrentProgramInfo.NumVGPRsForWavesPerEU,
       CurrentProgramInfo.NumSGPRsForWavesPerEU -
-          IsaInfo::getNumExtraSGPRs(getSTI(),
+          IsaInfo::getNumExtraSGPRs(&STI,
                                     CurrentProgramInfo.VCCUsed,
                                     CurrentProgramInfo.FlatUsed),
       CurrentProgramInfo.VCCUsed, CurrentProgramInfo.FlatUsed,
-      hasXNACK(*getSTI()));
+      hasXNACK(STI));
 
   Streamer.PopSection();
 }
 
 void AMDGPUAsmPrinter::EmitFunctionEntryLabel() {
-  if (IsaInfo::hasCodeObjectV3(getSTI()) &&
+  if (IsaInfo::hasCodeObjectV3(getGlobalSTI()) &&
       TM.getTargetTriple().getOS() == Triple::AMDHSA) {
     AsmPrinter::EmitFunctionEntryLabel();
     return;
@@ -273,8 +275,7 @@ void AMDGPUAsmPrinter::EmitFunctionEntryLabel() {
     getTargetStreamer()->EmitAMDGPUSymbolType(
         SymbolName, ELF::STT_AMDGPU_HSA_KERNEL);
   }
-  const GCNSubtarget &STI = MF->getSubtarget<GCNSubtarget>();
-  if (STI.dumpCode()) {
+  if (STM.dumpCode()) {
     // Disassemble function name label to text.
     DisasmLines.push_back(MF->getName().str() + ":");
     DisasmLineMaxLen = std::max(DisasmLineMaxLen, DisasmLines.back().size());
@@ -687,6 +688,9 @@ AMDGPUAsmPrinter::SIFunctionResourceInfo AMDGPUAsmPrinter::analyzeResourceUsage(
         case AMDGPU::XNACK_MASK_HI:
           llvm_unreachable("xnack_mask registers should not be used");
 
+        case AMDGPU::LDS_DIRECT:
+          llvm_unreachable("lds_direct register should not be used");
+
         case AMDGPU::TBA:
         case AMDGPU::TBA_LO:
         case AMDGPU::TBA_HI:
@@ -767,8 +771,7 @@ AMDGPUAsmPrinter::SIFunctionResourceInfo AMDGPUAsmPrinter::analyzeResourceUsage(
 
           // 48 SGPRs - vcc, - flat_scr, -xnack
           int MaxSGPRGuess =
-              47 - IsaInfo::getNumExtraSGPRs(getSTI(), true,
-                                             ST.hasFlatAddressSpace());
+            47 - IsaInfo::getNumExtraSGPRs(&ST, true, ST.hasFlatAddressSpace());
           MaxSGPR = std::max(MaxSGPR, MaxSGPRGuess);
           MaxVGPR = std::max(MaxVGPR, 23);
 
@@ -832,7 +835,7 @@ void AMDGPUAsmPrinter::getSIProgramInfo(SIProgramInfo &ProgInfo,
   // duplicated in part in AMDGPUAsmParser::calculateGPRBlocks, and could be
   // unified.
   unsigned ExtraSGPRs = IsaInfo::getNumExtraSGPRs(
-      getSTI(), ProgInfo.VCCUsed, ProgInfo.FlatUsed);
+      &STM, ProgInfo.VCCUsed, ProgInfo.FlatUsed);
 
   // Check the addressable register limit before we add ExtraSGPRs.
   if (STM.getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS &&
@@ -1144,7 +1147,7 @@ void AMDGPUAsmPrinter::getAmdKernelCode(amd_kernel_code_t &Out,
   const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
   const GCNSubtarget &STM = MF.getSubtarget<GCNSubtarget>();
 
-  AMDGPU::initDefaultAMDKernelCodeT(Out, getSTI());
+  AMDGPU::initDefaultAMDKernelCodeT(Out, &STM);
 
   Out.compute_pgm_resource_registers =
       CurrentProgramInfo.ComputePGMRSrc1 |
diff --git a/lib/Target/AMDGPU/AMDGPUAsmPrinter.h b/lib/Target/AMDGPU/AMDGPUAsmPrinter.h
index 167ac4b21e1e20feb3dbcc39d168e2853d21d545..796fa815c89e8925509973309eb4dd0417b6b8e2 100644
--- a/lib/Target/AMDGPU/AMDGPUAsmPrinter.h
+++ b/lib/Target/AMDGPU/AMDGPUAsmPrinter.h
@@ -1,9 +1,8 @@
 //===-- AMDGPUAsmPrinter.h - Print AMDGPU assembly code ---------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -95,7 +94,7 @@ public:
 
   StringRef getPassName() const override;
 
-  const MCSubtargetInfo* getSTI() const;
+  const MCSubtargetInfo* getGlobalSTI() const;
 
   AMDGPUTargetStreamer* getTargetStreamer() const;
 
diff --git a/lib/Target/AMDGPU/AMDGPUAtomicOptimizer.cpp b/lib/Target/AMDGPU/AMDGPUAtomicOptimizer.cpp
index 644e4fd558badf42e524570e0428ef0ceaf2c654..3210df461f568be9916c803a1775450314eefc13 100644
--- a/lib/Target/AMDGPU/AMDGPUAtomicOptimizer.cpp
+++ b/lib/Target/AMDGPU/AMDGPUAtomicOptimizer.cpp
@@ -1,9 +1,8 @@
 //===-- AMDGPUAtomicOptimizer.cpp -----------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -31,6 +30,7 @@ namespace {
 enum DPP_CTRL {
   DPP_ROW_SR1 = 0x111,
   DPP_ROW_SR2 = 0x112,
+  DPP_ROW_SR3 = 0x113,
   DPP_ROW_SR4 = 0x114,
   DPP_ROW_SR8 = 0x118,
   DPP_WF_SR1 = 0x138,
@@ -212,8 +212,6 @@ void AMDGPUAtomicOptimizer::optimizeAtomic(Instruction &I,
                                            Instruction::BinaryOps Op,
                                            unsigned ValIdx,
                                            bool ValDivergent) const {
-  LLVMContext &Context = I.getContext();
-
   // Start building just before the instruction.
   IRBuilder<> B(&I);
 
@@ -251,20 +249,17 @@ void AMDGPUAtomicOptimizer::optimizeAtomic(Instruction &I,
   Value *const V = I.getOperand(ValIdx);
 
   // We need to know how many lanes are active within the wavefront, and we do
-  // this by getting the exec register, which tells us all the lanes that are
-  // active.
-  MDNode *const RegName =
-      llvm::MDNode::get(Context, llvm::MDString::get(Context, "exec"));
-  Value *const Metadata = llvm::MetadataAsValue::get(Context, RegName);
-  CallInst *const Exec =
-      B.CreateIntrinsic(Intrinsic::read_register, {B.getInt64Ty()}, {Metadata});
-  setConvergent(Exec);
+  // this by doing a ballot of active lanes.
+  CallInst *const Ballot =
+      B.CreateIntrinsic(Intrinsic::amdgcn_icmp, {B.getInt32Ty()},
+                        {B.getInt32(1), B.getInt32(0), B.getInt32(33)});
+  setConvergent(Ballot);
 
   // We need to know how many lanes are active within the wavefront that are
   // below us. If we counted each lane linearly starting from 0, a lane is
   // below us only if its associated index was less than ours. We do this by
   // using the mbcnt intrinsic.
-  Value *const BitCast = B.CreateBitCast(Exec, VecTy);
+  Value *const BitCast = B.CreateBitCast(Ballot, VecTy);
   Value *const ExtractLo = B.CreateExtractElement(BitCast, B.getInt32(0));
   Value *const ExtractHi = B.CreateExtractElement(BitCast, B.getInt32(1));
   CallInst *const PartialMbcnt = B.CreateIntrinsic(
@@ -280,44 +275,43 @@ void AMDGPUAtomicOptimizer::optimizeAtomic(Instruction &I,
   // If we have a divergent value in each lane, we need to combine the value
   // using DPP.
   if (ValDivergent) {
+    Value *const Identity = B.getIntN(TyBitWidth, 0);
+
     // First we need to set all inactive invocations to 0, so that they can
     // correctly contribute to the final result.
-    CallInst *const SetInactive = B.CreateIntrinsic(
-        Intrinsic::amdgcn_set_inactive, Ty, {V, B.getIntN(TyBitWidth, 0)});
+    CallInst *const SetInactive =
+        B.CreateIntrinsic(Intrinsic::amdgcn_set_inactive, Ty, {V, Identity});
     setConvergent(SetInactive);
-    NewV = SetInactive;
 
-    const unsigned Iters = 6;
-    const unsigned DPPCtrl[Iters] = {DPP_ROW_SR1,     DPP_ROW_SR2,
-                                     DPP_ROW_SR4,     DPP_ROW_SR8,
-                                     DPP_ROW_BCAST15, DPP_ROW_BCAST31};
-    const unsigned RowMask[Iters] = {0xf, 0xf, 0xf, 0xf, 0xa, 0xc};
-
-    // This loop performs an inclusive scan across the wavefront, with all lanes
+    CallInst *const FirstDPP =
+        B.CreateIntrinsic(Intrinsic::amdgcn_update_dpp, Ty,
+                          {Identity, SetInactive, B.getInt32(DPP_WF_SR1),
+                           B.getInt32(0xf), B.getInt32(0xf), B.getFalse()});
+    setConvergent(FirstDPP);
+    NewV = FirstDPP;
+
+    const unsigned Iters = 7;
+    const unsigned DPPCtrl[Iters] = {
+        DPP_ROW_SR1, DPP_ROW_SR2,     DPP_ROW_SR3,    DPP_ROW_SR4,
+        DPP_ROW_SR8, DPP_ROW_BCAST15, DPP_ROW_BCAST31};
+    const unsigned RowMask[Iters] = {0xf, 0xf, 0xf, 0xf, 0xf, 0xa, 0xc};
+    const unsigned BankMask[Iters] = {0xf, 0xf, 0xf, 0xe, 0xc, 0xf, 0xf};
+
+    // This loop performs an exclusive scan across the wavefront, with all lanes
     // active (by using the WWM intrinsic).
     for (unsigned Idx = 0; Idx < Iters; Idx++) {
-      CallInst *const DPP = B.CreateIntrinsic(Intrinsic::amdgcn_mov_dpp, Ty,
-                                              {NewV, B.getInt32(DPPCtrl[Idx]),
-                                               B.getInt32(RowMask[Idx]),
-                                               B.getInt32(0xf), B.getFalse()});
+      Value *const UpdateValue = Idx < 3 ? FirstDPP : NewV;
+      CallInst *const DPP = B.CreateIntrinsic(
+          Intrinsic::amdgcn_update_dpp, Ty,
+          {Identity, UpdateValue, B.getInt32(DPPCtrl[Idx]),
+           B.getInt32(RowMask[Idx]), B.getInt32(BankMask[Idx]), B.getFalse()});
       setConvergent(DPP);
-      Value *const WWM = B.CreateIntrinsic(Intrinsic::amdgcn_wwm, Ty, DPP);
 
-      NewV = B.CreateBinOp(Op, NewV, WWM);
-      NewV = B.CreateIntrinsic(Intrinsic::amdgcn_wwm, Ty, NewV);
+      NewV = B.CreateBinOp(Op, NewV, DPP);
     }
 
-    // NewV has returned the inclusive scan of V, but for the lane offset we
-    // require an exclusive scan. We do this by shifting the values from the
-    // entire wavefront right by 1, and by setting the bound_ctrl (last argument
-    // to the intrinsic below) to true, we can guarantee that 0 will be shifted
-    // into the 0'th invocation.
-    CallInst *const DPP =
-        B.CreateIntrinsic(Intrinsic::amdgcn_mov_dpp, {Ty},
-                          {NewV, B.getInt32(DPP_WF_SR1), B.getInt32(0xf),
-                           B.getInt32(0xf), B.getTrue()});
-    setConvergent(DPP);
-    LaneOffset = B.CreateIntrinsic(Intrinsic::amdgcn_wwm, Ty, DPP);
+    LaneOffset = B.CreateIntrinsic(Intrinsic::amdgcn_wwm, Ty, NewV);
+    NewV = B.CreateBinOp(Op, NewV, SetInactive);
 
     // Read the value from the last lane, which has accumlated the values of
     // each active lane in the wavefront. This will be our new value with which
@@ -345,9 +339,12 @@ void AMDGPUAtomicOptimizer::optimizeAtomic(Instruction &I,
     } else {
       llvm_unreachable("Unhandled atomic bit width");
     }
+
+    // Finally mark the readlanes in the WWM section.
+    NewV = B.CreateIntrinsic(Intrinsic::amdgcn_wwm, Ty, NewV);
   } else {
     // Get the total number of active lanes we have by using popcount.
-    Instruction *const Ctpop = B.CreateUnaryIntrinsic(Intrinsic::ctpop, Exec);
+    Instruction *const Ctpop = B.CreateUnaryIntrinsic(Intrinsic::ctpop, Ballot);
     Value *const CtpopCast = B.CreateIntCast(Ctpop, Ty, false);
 
     // Calculate the new value we will be contributing to the atomic operation
diff --git a/lib/Target/AMDGPU/AMDGPUCallLowering.cpp b/lib/Target/AMDGPU/AMDGPUCallLowering.cpp
index daef37f9c21fb72a942e4c141aed761a76f1e59b..20dbcf309555ed28ae7c6d969cfb761a1fb1ee1d 100644
--- a/lib/Target/AMDGPU/AMDGPUCallLowering.cpp
+++ b/lib/Target/AMDGPU/AMDGPUCallLowering.cpp
@@ -1,9 +1,8 @@
 //===-- llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp - Call lowering -----===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
diff --git a/lib/Target/AMDGPU/AMDGPUCallLowering.h b/lib/Target/AMDGPU/AMDGPUCallLowering.h
index ed859716218efc30acdac37df31576e5b9ffa757..84a512e62f2f9a151fb538e781082e8bddc9259a 100644
--- a/lib/Target/AMDGPU/AMDGPUCallLowering.h
+++ b/lib/Target/AMDGPU/AMDGPUCallLowering.h
@@ -1,9 +1,8 @@
 //===- lib/Target/AMDGPU/AMDGPUCallLowering.h - Call lowering -*- C++ -*---===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
diff --git a/lib/Target/AMDGPU/AMDGPUCallingConv.td b/lib/Target/AMDGPU/AMDGPUCallingConv.td
index 367f120b5fa6bd3bf8292cdee396a30e64af1f17..b89ea1d5ea46a6f9787d6b4ba2494df991240102 100644
--- a/lib/Target/AMDGPU/AMDGPUCallingConv.td
+++ b/lib/Target/AMDGPU/AMDGPUCallingConv.td
@@ -1,9 +1,8 @@
 //===---- AMDCallingConv.td - Calling Conventions for Radeon GPUs ---------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp b/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp
index 4dc1e67c573d3d97297eaf3ccb8c227636ec05ce..759820753255a077f19c66c91a1d32bc84eec36c 100644
--- a/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp
+++ b/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp
@@ -1,9 +1,8 @@
 //===-- AMDGPUCodeGenPrepare.cpp ------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -807,7 +806,7 @@ bool AMDGPUCodeGenPrepare::visitLoadInst(LoadInst &I) {
     Type *I32Ty = Builder.getInt32Ty();
     Type *PT = PointerType::get(I32Ty, I.getPointerAddressSpace());
     Value *BitCast= Builder.CreateBitCast(I.getPointerOperand(), PT);
-    LoadInst *WidenLoad = Builder.CreateLoad(BitCast);
+    LoadInst *WidenLoad = Builder.CreateLoad(I32Ty, BitCast);
     WidenLoad->copyMetadata(I);
 
     // If we have range metadata, we need to convert the type, and not make
diff --git a/lib/Target/AMDGPU/AMDGPUFeatures.td b/lib/Target/AMDGPU/AMDGPUFeatures.td
index 3c7d8a8fc550979c1bdbd0df45f73c6b49b802b5..59637883b12e4dd7d6f76c49e66356f1a2a57046 100644
--- a/lib/Target/AMDGPU/AMDGPUFeatures.td
+++ b/lib/Target/AMDGPU/AMDGPUFeatures.td
@@ -1,9 +1,8 @@
 //===-- AMDGPUFeatures.td - AMDGPU Feature Definitions -----*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/AMDGPU/AMDGPUFixFunctionBitcasts.cpp b/lib/Target/AMDGPU/AMDGPUFixFunctionBitcasts.cpp
index 6e2a981d33968889370d2062eaaf8cedde700457..9ba04d113c7021d03e895dd2e0ea66937774fec9 100644
--- a/lib/Target/AMDGPU/AMDGPUFixFunctionBitcasts.cpp
+++ b/lib/Target/AMDGPU/AMDGPUFixFunctionBitcasts.cpp
@@ -1,9 +1,8 @@
 //===-- AMDGPUFixFunctionBitcasts.cpp - Fix function bitcasts -------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
diff --git a/lib/Target/AMDGPU/AMDGPUFrameLowering.cpp b/lib/Target/AMDGPU/AMDGPUFrameLowering.cpp
index e32ca9653b3a1f314d2b0cbba9dcbd98bcd9a638..e807977363639ce1f169d101407f69760080725a 100644
--- a/lib/Target/AMDGPU/AMDGPUFrameLowering.cpp
+++ b/lib/Target/AMDGPU/AMDGPUFrameLowering.cpp
@@ -1,9 +1,8 @@
 //===----------------------- AMDGPUFrameLowering.cpp ----------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //==-----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AMDGPU/AMDGPUFrameLowering.h b/lib/Target/AMDGPU/AMDGPUFrameLowering.h
index ee836bf8a631090b6a0fa7fff6820f62b8f9012a..48b64488303e9135ab789361a00c172bd1318f67 100644
--- a/lib/Target/AMDGPU/AMDGPUFrameLowering.h
+++ b/lib/Target/AMDGPU/AMDGPUFrameLowering.h
@@ -1,9 +1,8 @@
 //===--------------------- AMDGPUFrameLowering.h ----------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AMDGPU/AMDGPUGISel.td b/lib/Target/AMDGPU/AMDGPUGISel.td
index 59bb2a16e0f3475c507d30a2a47355f799442a3a..7b010b02aec11bb6d0ddcc07f422523362054495 100644
--- a/lib/Target/AMDGPU/AMDGPUGISel.td
+++ b/lib/Target/AMDGPU/AMDGPUGISel.td
@@ -1,9 +1,8 @@
 //===-- AMDGPUGIsel.td - AMDGPU GlobalISel Patterns---------*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 // This files contains patterns that should only be used by GlobalISel.  For
diff --git a/lib/Target/AMDGPU/AMDGPUGenRegisterBankInfo.def b/lib/Target/AMDGPU/AMDGPUGenRegisterBankInfo.def
index 3a58c6c6a29fe19b6c2a446eacdf61e26c03a2f0..f2daf47df5da270d60ed1ea7c3d7d003fec412e9 100644
--- a/lib/Target/AMDGPU/AMDGPUGenRegisterBankInfo.def
+++ b/lib/Target/AMDGPU/AMDGPUGenRegisterBankInfo.def
@@ -1,9 +1,8 @@
 //===- AMDGPUGenRegisterBankInfo.def -----------------------------*- C++ -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
@@ -16,34 +15,38 @@ namespace AMDGPU {
 
 enum PartialMappingIdx {
   None = - 1,
-  PM_SGPR1  = 0,
-  PM_SGPR16 = 4,
-  PM_SGPR32 = 5,
-  PM_SGPR64 = 6,
-  PM_SGPR128 = 7,
-  PM_SGPR256 = 8,
-  PM_SGPR512 = 9,
-  PM_VGPR1  = 10,
-  PM_VGPR16 = 14,
-  PM_VGPR32 = 15,
-  PM_VGPR64 = 16,
-  PM_VGPR128 = 17,
-  PM_VGPR256 = 18,
-  PM_VGPR512 = 19,
-  PM_SGPR96 = 20,
-  PM_VGPR96 = 21
+  PM_SGPR1  = 2,
+  PM_SGPR16 = 6,
+  PM_SGPR32 = 7,
+  PM_SGPR64 = 8,
+  PM_SGPR128 = 9,
+  PM_SGPR256 = 10,
+  PM_SGPR512 = 11,
+  PM_VGPR1  = 12,
+  PM_VGPR16 = 16,
+  PM_VGPR32 = 17,
+  PM_VGPR64 = 18,
+  PM_VGPR128 = 19,
+  PM_VGPR256 = 20,
+  PM_VGPR512 = 21,
+  PM_SGPR96 = 22,
+  PM_VGPR96 = 23
 };
 
 const RegisterBankInfo::PartialMapping PartMappings[] {
   // StartIdx, Length, RegBank
   {0, 1,  SCCRegBank},
+  {0, 1,  VCCRegBank},
+
+  {0, 1,  SGPRRegBank}, // SGPR begin
   {0, 16, SGPRRegBank},
   {0, 32, SGPRRegBank},
   {0, 64, SGPRRegBank},
   {0, 128, SGPRRegBank},
   {0, 256, SGPRRegBank},
   {0, 512, SGPRRegBank},
-  {0, 1,  SGPRRegBank},
+
+  {0, 1,  VGPRRegBank}, // VGPR begin
   {0, 16, VGPRRegBank},
   {0, 32, VGPRRegBank},
   {0, 64, VGPRRegBank},
@@ -55,33 +58,65 @@ const RegisterBankInfo::PartialMapping PartMappings[] {
 };
 
 const RegisterBankInfo::ValueMapping ValMappings[] {
+  // SCC
   {&PartMappings[0], 1},
-  {nullptr, 0},
-  {nullptr, 0},
-  {nullptr, 0},
+
+  // VCC
   {&PartMappings[1], 1},
+
+  // SGPRs
   {&PartMappings[2], 1},
+  {nullptr, 0}, // Illegal power of 2 sizes
+  {nullptr, 0},
+  {nullptr, 0},
   {&PartMappings[3], 1},
   {&PartMappings[4], 1},
   {&PartMappings[5], 1},
   {&PartMappings[6], 1},
   {&PartMappings[7], 1},
+  {&PartMappings[8], 1},
+
+    // VGPRs
+  {&PartMappings[9], 1},
   {nullptr, 0},
   {nullptr, 0},
   {nullptr, 0},
-  {&PartMappings[8], 1},
-  {&PartMappings[9], 1},
   {&PartMappings[10], 1},
   {&PartMappings[11], 1},
   {&PartMappings[12], 1},
   {&PartMappings[13], 1},
   {&PartMappings[14], 1},
-  {&PartMappings[15], 1}
+  {&PartMappings[15], 1},
+  {&PartMappings[16], 1},
+  {&PartMappings[17], 1}
+};
+
+const RegisterBankInfo::PartialMapping SGPROnly64BreakDown[] {
+     /*32-bit op*/ {0, 32, SGPRRegBank},
+   /*2x32-bit op*/ {0, 32, SGPRRegBank},
+                   {32, 32, SGPRRegBank},
+/*<2x32-bit> op*/  {0, 64, SGPRRegBank},
+
+    /*32-bit op*/  {0, 32, VGPRRegBank},
+  /*2x32-bit op*/  {0, 32, VGPRRegBank},
+                   {32, 32, VGPRRegBank},
+};
+
+
+// For some instructions which can operate 64-bit only for the scalar version.
+const RegisterBankInfo::ValueMapping ValMappingsSGPR64OnlyVGPR32[] {
+  /*32-bit sgpr*/     {&SGPROnly64BreakDown[0], 1},
+  /*2 x 32-bit sgpr*/ {&SGPROnly64BreakDown[1], 2},
+  /*64-bit sgpr */    {&SGPROnly64BreakDown[3], 1},
+
+  /*32-bit vgpr*/     {&SGPROnly64BreakDown[4], 1},
+  /*2 x 32-bit vgpr*/ {&SGPROnly64BreakDown[5], 2}
 };
 
 enum ValueMappingIdx {
-  SGPRStartIdx = 0,
-  VGPRStartIdx = 10
+  SCCStartIdx = 0,
+  SGPRStartIdx = 2,
+  VGPRStartIdx = 12
 };
 
 const RegisterBankInfo::ValueMapping *getValueMapping(unsigned BankID,
@@ -89,18 +124,41 @@ const RegisterBankInfo::ValueMapping *getValueMapping(unsigned BankID,
   unsigned Idx;
   switch (Size) {
   case 1:
-    Idx = BankID == AMDGPU::SCCRegBankID ? PM_SGPR1 : PM_VGPR1;
+    if (BankID == AMDGPU::SCCRegBankID)
+      return &ValMappings[0];
+    if (BankID == AMDGPU::VCCRegBankID)
+      return &ValMappings[1];
+
+    // 1-bit values not from a compare etc.
+    Idx = BankID == AMDGPU::SGPRRegBankID ? PM_SGPR1 : PM_VGPR1;
     break;
   case 96:
+    assert(BankID != AMDGPU::VCCRegBankID);
     Idx = BankID == AMDGPU::SGPRRegBankID ? PM_SGPR96 : PM_VGPR96;
     break;
   default:
+    assert(BankID != AMDGPU::VCCRegBankID);
     Idx = BankID == AMDGPU::VGPRRegBankID ? VGPRStartIdx : SGPRStartIdx;
     Idx += Log2_32_Ceil(Size);
     break;
   }
+
+  assert(Log2_32_Ceil(Size) == Log2_32_Ceil(ValMappings[Idx].BreakDown->Length));
+  assert(BankID == ValMappings[Idx].BreakDown->RegBank->getID());
+
   return &ValMappings[Idx];
 }
 
+const RegisterBankInfo::ValueMapping *getValueMappingSGPR64Only(unsigned BankID,
+                                                                unsigned Size) {
+  assert(Size == 64);
+
+  if (BankID == AMDGPU::VGPRRegBankID)
+    return &ValMappingsSGPR64OnlyVGPR32[4];
+
+  assert(BankID == AMDGPU::SGPRRegBankID);
+  return &ValMappingsSGPR64OnlyVGPR32[2];
+}
+
 } // End AMDGPU namespace.
 } // End llvm namespace.
diff --git a/lib/Target/AMDGPU/AMDGPUHSAMetadataStreamer.cpp b/lib/Target/AMDGPU/AMDGPUHSAMetadataStreamer.cpp
index c38b0e61558b3d2a2464c776b6dfb43dcb1f3aee..15cee69e935a96cf7885e78b313dfbb5d211c542 100644
--- a/lib/Target/AMDGPU/AMDGPUHSAMetadataStreamer.cpp
+++ b/lib/Target/AMDGPU/AMDGPUHSAMetadataStreamer.cpp
@@ -1,9 +1,8 @@
 //===--- AMDGPUHSAMetadataStreamer.cpp --------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AMDGPU/AMDGPUHSAMetadataStreamer.h b/lib/Target/AMDGPU/AMDGPUHSAMetadataStreamer.h
index afc09baf952d6e7c869cd9435600da4a14bf3ed6..5835ed715a66caf2894505795022e53a3224d08f 100644
--- a/lib/Target/AMDGPU/AMDGPUHSAMetadataStreamer.h
+++ b/lib/Target/AMDGPU/AMDGPUHSAMetadataStreamer.h
@@ -1,9 +1,8 @@
 //===--- AMDGPUHSAMetadataStreamer.h ----------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp b/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
index 025e2de742d6e7ffe77194c5e3be654c0186c1d1..2cdd691fc109e2c8f38e5c4c825168174b965af8 100644
--- a/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
+++ b/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
@@ -1,9 +1,8 @@
 //===-- AMDGPUISelDAGToDAG.cpp - A dag to dag inst selector for AMDGPU ----===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //==-----------------------------------------------------------------------===//
 //
@@ -107,12 +106,13 @@ private:
 
   MachineSDNode *buildSMovImm64(SDLoc &DL, uint64_t Val, EVT VT) const;
 
-  SDNode *glueCopyToM0(SDNode *N) const;
+  SDNode *glueCopyToM0LDSInit(SDNode *N) const;
+  SDNode *glueCopyToM0(SDNode *N, SDValue Val) const;
 
   const TargetRegisterClass *getOperandRegClass(SDNode *N, unsigned OpNo) const;
   virtual bool SelectADDRVTX_READ(SDValue Addr, SDValue &Base, SDValue &Offset);
   virtual bool SelectADDRIndirect(SDValue Addr, SDValue &Base, SDValue &Offset);
-  bool isDSOffsetLegal(const SDValue &Base, unsigned Offset,
+  bool isDSOffsetLegal(SDValue Base, unsigned Offset,
                        unsigned OffsetBits) const;
   bool SelectDS1Addr1Offset(SDValue Ptr, SDValue &Base, SDValue &Offset) const;
   bool SelectDS64Bit4ByteAligned(SDValue Ptr, SDValue &Base, SDValue &Offset0,
@@ -210,6 +210,7 @@ private:
   void SelectBRCOND(SDNode *N);
   void SelectFMAD_FMA(SDNode *N);
   void SelectATOMIC_CMP_SWAP(SDNode *N);
+  void SelectINTRINSIC_W_CHAIN(SDNode *N);
 
 protected:
   // Include the pieces autogenerated from the target description.
@@ -340,29 +341,32 @@ const TargetRegisterClass *AMDGPUDAGToDAGISel::getOperandRegClass(SDNode *N,
   }
 }
 
-SDNode *AMDGPUDAGToDAGISel::glueCopyToM0(SDNode *N) const {
-  if (cast<MemSDNode>(N)->getAddressSpace() != AMDGPUAS::LOCAL_ADDRESS ||
-      !Subtarget->ldsRequiresM0Init())
-    return N;
-
+SDNode *AMDGPUDAGToDAGISel::glueCopyToM0(SDNode *N, SDValue Val) const {
   const SITargetLowering& Lowering =
-      *static_cast<const SITargetLowering*>(getTargetLowering());
+    *static_cast<const SITargetLowering*>(getTargetLowering());
 
   // Write max value to m0 before each load operation
 
   SDValue M0 = Lowering.copyToM0(*CurDAG, CurDAG->getEntryNode(), SDLoc(N),
-                                 CurDAG->getTargetConstant(-1, SDLoc(N), MVT::i32));
+                                 Val);
 
   SDValue Glue = M0.getValue(1);
 
   SmallVector <SDValue, 8> Ops;
-  for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) {
-     Ops.push_back(N->getOperand(i));
-  }
+  for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i)
+    Ops.push_back(N->getOperand(i));
+
   Ops.push_back(Glue);
   return CurDAG->MorphNodeTo(N, N->getOpcode(), N->getVTList(), Ops);
 }
 
+SDNode *AMDGPUDAGToDAGISel::glueCopyToM0LDSInit(SDNode *N) const {
+  if (cast<MemSDNode>(N)->getAddressSpace() != AMDGPUAS::LOCAL_ADDRESS ||
+      !Subtarget->ldsRequiresM0Init())
+    return N;
+  return glueCopyToM0(N, CurDAG->getTargetConstant(-1, SDLoc(N), MVT::i32));
+}
+
 MachineSDNode *AMDGPUDAGToDAGISel::buildSMovImm64(SDLoc &DL, uint64_t Imm,
                                                   EVT VT) const {
   SDNode *Lo = CurDAG->getMachineNode(
@@ -470,10 +474,10 @@ void AMDGPUDAGToDAGISel::Select(SDNode *N) {
 
   if (isa<AtomicSDNode>(N) ||
       (Opc == AMDGPUISD::ATOMIC_INC || Opc == AMDGPUISD::ATOMIC_DEC ||
-       Opc == AMDGPUISD::ATOMIC_LOAD_FADD ||
+       Opc == ISD::ATOMIC_LOAD_FADD ||
        Opc == AMDGPUISD::ATOMIC_LOAD_FMIN ||
        Opc == AMDGPUISD::ATOMIC_LOAD_FMAX))
-    N = glueCopyToM0(N);
+    N = glueCopyToM0LDSInit(N);
 
   switch (Opc) {
   default:
@@ -571,7 +575,7 @@ void AMDGPUDAGToDAGISel::Select(SDNode *N) {
   case ISD::STORE:
   case ISD::ATOMIC_LOAD:
   case ISD::ATOMIC_STORE: {
-    N = glueCopyToM0(N);
+    N = glueCopyToM0LDSInit(N);
     break;
   }
 
@@ -649,6 +653,12 @@ void AMDGPUDAGToDAGISel::Select(SDNode *N) {
       SelectCode(N);
       return;
     }
+
+    break;
+  }
+  case ISD::INTRINSIC_W_CHAIN: {
+    SelectINTRINSIC_W_CHAIN(N);
+    return;
   }
   }
 
@@ -829,7 +839,7 @@ void AMDGPUDAGToDAGISel::SelectMAD_64_32(SDNode *N) {
   CurDAG->SelectNodeTo(N, Opc, N->getVTList(), Ops);
 }
 
-bool AMDGPUDAGToDAGISel::isDSOffsetLegal(const SDValue &Base, unsigned Offset,
+bool AMDGPUDAGToDAGISel::isDSOffsetLegal(SDValue Base, unsigned Offset,
                                          unsigned OffsetBits) const {
   if ((OffsetBits == 16 && !isUInt<16>(Offset)) ||
       (OffsetBits == 8 && !isUInt<8>(Offset)))
@@ -1454,9 +1464,13 @@ bool AMDGPUDAGToDAGISel::SelectMOVRELOffset(SDValue Index,
     ConstantSDNode *C1 = cast<ConstantSDNode>(N1);
 
     // (add n0, c0)
-    Base = N0;
-    Offset = CurDAG->getTargetConstant(C1->getZExtValue(), DL, MVT::i32);
-    return true;
+    // Don't peel off the offset (c0) if doing so could possibly lead
+    // the base (n0) to be negative.
+    if (C1->getSExtValue() <= 0 || CurDAG->SignBitIsZero(N0)) {
+      Base = N0;
+      Offset = CurDAG->getTargetConstant(C1->getZExtValue(), DL, MVT::i32);
+      return true;
+    }
   }
 
   if (isa<ConstantSDNode>(Index))
@@ -1757,6 +1771,52 @@ void AMDGPUDAGToDAGISel::SelectATOMIC_CMP_SWAP(SDNode *N) {
   CurDAG->RemoveDeadNode(N);
 }
 
+void AMDGPUDAGToDAGISel::SelectINTRINSIC_W_CHAIN(SDNode *N) {
+  unsigned IntrID = cast<ConstantSDNode>(N->getOperand(1))->getZExtValue();
+  if ((IntrID != Intrinsic::amdgcn_ds_append &&
+       IntrID != Intrinsic::amdgcn_ds_consume) ||
+      N->getValueType(0) != MVT::i32) {
+    SelectCode(N);
+    return;
+  }
+
+  // The address is assumed to be uniform, so if it ends up in a VGPR, it will
+  // be copied to an SGPR with readfirstlane.
+  unsigned Opc = IntrID == Intrinsic::amdgcn_ds_append ?
+    AMDGPU::DS_APPEND : AMDGPU::DS_CONSUME;
+
+  SDValue Chain = N->getOperand(0);
+  SDValue Ptr = N->getOperand(2);
+  MemIntrinsicSDNode *M = cast<MemIntrinsicSDNode>(N);
+  bool IsGDS = M->getAddressSpace() == AMDGPUAS::REGION_ADDRESS;
+
+  SDValue Offset;
+  if (CurDAG->isBaseWithConstantOffset(Ptr)) {
+    SDValue PtrBase = Ptr.getOperand(0);
+    SDValue PtrOffset = Ptr.getOperand(1);
+
+    const APInt &OffsetVal = cast<ConstantSDNode>(PtrOffset)->getAPIntValue();
+    if (isDSOffsetLegal(PtrBase, OffsetVal.getZExtValue(), 16)) {
+      N = glueCopyToM0(N, PtrBase);
+      Offset = CurDAG->getTargetConstant(OffsetVal, SDLoc(), MVT::i32);
+    }
+  }
+
+  if (!Offset) {
+    N = glueCopyToM0(N, Ptr);
+    Offset = CurDAG->getTargetConstant(0, SDLoc(), MVT::i32);
+  }
+
+  SDValue Ops[] = {
+    Offset,
+    CurDAG->getTargetConstant(IsGDS, SDLoc(), MVT::i32),
+    Chain,
+    N->getOperand(N->getNumOperands() - 1) // New glue
+  };
+
+  CurDAG->SelectNodeTo(N, Opc, N->getVTList(), Ops);
+}
+
 bool AMDGPUDAGToDAGISel::SelectVOP3ModsImpl(SDValue In, SDValue &Src,
                                             unsigned &Mods) const {
   Mods = 0;
diff --git a/lib/Target/AMDGPU/AMDGPUISelLowering.cpp b/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
index f5894c9022a6e5e3c142785cb4e640f99fcaec72..1fc44b671366cd563440c87f45b6f1d8eea5b7de 100644
--- a/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
+++ b/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
@@ -1,9 +1,8 @@
 //===-- AMDGPUISelLowering.cpp - AMDGPU Common DAG lowering functions -----===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -21,7 +20,6 @@
 #include "AMDGPU.h"
 #include "AMDGPUCallLowering.h"
 #include "AMDGPUFrameLowering.h"
-#include "AMDGPUIntrinsicInfo.h"
 #include "AMDGPURegisterInfo.h"
 #include "AMDGPUSubtarget.h"
 #include "AMDGPUTargetMachine.h"
@@ -128,10 +126,8 @@ EVT AMDGPUTargetLowering::getEquivalentMemType(LLVMContext &Ctx, EVT VT) {
 }
 
 unsigned AMDGPUTargetLowering::numBitsUnsigned(SDValue Op, SelectionDAG &DAG) {
-  KnownBits Known;
   EVT VT = Op.getValueType();
-  DAG.computeKnownBits(Op, Known);
-
+  KnownBits Known = DAG.computeKnownBits(Op);
   return VT.getSizeInBits() - Known.countMinLeadingZeros();
 }
 
@@ -594,6 +590,7 @@ static bool hasSourceMods(const SDNode *N) {
   case ISD::FDIV:
   case ISD::FREM:
   case ISD::INLINEASM:
+  case ISD::INLINEASM_BR:
   case AMDGPUISD::INTERP_P1:
   case AMDGPUISD::INTERP_P2:
   case AMDGPUISD::DIV_SCALE:
@@ -2719,21 +2716,33 @@ static bool isI24(SDValue Op, SelectionDAG &DAG) {
     AMDGPUTargetLowering::numBitsSigned(Op, DAG) < 24;
 }
 
-static bool simplifyI24(SDNode *Node24, unsigned OpIdx,
-                        TargetLowering::DAGCombinerInfo &DCI) {
-
+static SDValue simplifyI24(SDNode *Node24,
+                           TargetLowering::DAGCombinerInfo &DCI) {
   SelectionDAG &DAG = DCI.DAG;
-  SDValue Op = Node24->getOperand(OpIdx);
+  SDValue LHS = Node24->getOperand(0);
+  SDValue RHS = Node24->getOperand(1);
+
+  APInt Demanded = APInt::getLowBitsSet(LHS.getValueSizeInBits(), 24);
+
+  // First try to simplify using GetDemandedBits which allows the operands to
+  // have other uses, but will only perform simplifications that involve
+  // bypassing some nodes for this user.
+  SDValue DemandedLHS = DAG.GetDemandedBits(LHS, Demanded);
+  SDValue DemandedRHS = DAG.GetDemandedBits(RHS, Demanded);
+  if (DemandedLHS || DemandedRHS)
+    return DAG.getNode(Node24->getOpcode(), SDLoc(Node24), Node24->getVTList(),
+                       DemandedLHS ? DemandedLHS : LHS,
+                       DemandedRHS ? DemandedRHS : RHS);
+
+  // Now try SimplifyDemandedBits which can simplify the nodes used by our
+  // operands if this node is the only user.
   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
-  EVT VT = Op.getValueType();
-
-  APInt Demanded = APInt::getLowBitsSet(VT.getSizeInBits(), 24);
-  APInt KnownZero, KnownOne;
-  TargetLowering::TargetLoweringOpt TLO(DAG, true, true);
-  if (TLI.SimplifyDemandedBits(Node24, OpIdx, Demanded, DCI, TLO))
-    return true;
+  if (TLI.SimplifyDemandedBits(LHS, Demanded, DCI))
+    return SDValue(Node24, 0);
+  if (TLI.SimplifyDemandedBits(RHS, Demanded, DCI))
+    return SDValue(Node24, 0);
 
-  return false;
+  return SDValue();
 }
 
 template <typename IntTy>
@@ -2970,8 +2979,7 @@ SDValue AMDGPUTargetLowering::performShlCombine(SDNode *N,
     // shl (ext x) => zext (shl x), if shift does not overflow int
     if (VT != MVT::i64)
       break;
-    KnownBits Known;
-    DAG.computeKnownBits(X, Known);
+    KnownBits Known = DAG.computeKnownBits(X);
     unsigned LZ = Known.countMinLeadingZeros();
     if (LZ < RHSVal)
       break;
@@ -3081,7 +3089,7 @@ SDValue AMDGPUTargetLowering::performTruncateCombine(
   SDValue Src = N->getOperand(0);
 
   // vt1 (truncate (bitcast (build_vector vt0:x, ...))) -> vt1 (bitcast vt0:x)
-  if (Src.getOpcode() == ISD::BITCAST) {
+  if (Src.getOpcode() == ISD::BITCAST && !VT.isVector()) {
     SDValue Vec = Src.getOperand(0);
     if (Vec.getOpcode() == ISD::BUILD_VECTOR) {
       SDValue Elt0 = Vec.getOperand(0);
@@ -3130,8 +3138,7 @@ SDValue AMDGPUTargetLowering::performTruncateCombine(
          Src.getOpcode() == ISD::SRA ||
          Src.getOpcode() == ISD::SHL)) {
       SDValue Amt = Src.getOperand(1);
-      KnownBits Known;
-      DAG.computeKnownBits(Amt, Known);
+      KnownBits Known = DAG.computeKnownBits(Amt);
       unsigned Size = VT.getScalarSizeInBits();
       if ((Known.isConstant() && Known.getConstant().ule(Size)) ||
           (Known.getBitWidth() - Known.countMinLeadingZeros() <= Log2_32(Size))) {
@@ -3283,8 +3290,8 @@ SDValue AMDGPUTargetLowering::performMulLoHi24Combine(
   SelectionDAG &DAG = DCI.DAG;
 
   // Simplify demanded bits before splitting into multiple users.
-  if (simplifyI24(N, 0, DCI) || simplifyI24(N, 1, DCI))
-    return SDValue();
+  if (SDValue V = simplifyI24(N, DCI))
+    return V;
 
   SDValue N0 = N->getOperand(0);
   SDValue N1 = N->getOperand(1);
@@ -3870,9 +3877,8 @@ SDValue AMDGPUTargetLowering::PerformDAGCombine(SDNode *N,
   case AMDGPUISD::MUL_U24:
   case AMDGPUISD::MULHI_I24:
   case AMDGPUISD::MULHI_U24: {
-    // If the first call to simplify is successfull, then N may end up being
-    // deleted, so we shouldn't call simplifyI24 again.
-    simplifyI24(N, 0, DCI) || simplifyI24(N, 1, DCI);
+    if (SDValue V = simplifyI24(N, DCI))
+      return V;
     return SDValue();
   }
   case AMDGPUISD::MUL_LOHI_I24:
@@ -4178,6 +4184,9 @@ const char* AMDGPUTargetLowering::getTargetNodeName(unsigned Opcode) const {
   NODE_NAME_CASE(INTERP_MOV)
   NODE_NAME_CASE(INTERP_P1)
   NODE_NAME_CASE(INTERP_P2)
+  NODE_NAME_CASE(INTERP_P1LL_F16)
+  NODE_NAME_CASE(INTERP_P1LV_F16)
+  NODE_NAME_CASE(INTERP_P2_F16)
   NODE_NAME_CASE(STORE_MSKOR)
   NODE_NAME_CASE(LOAD_CONSTANT)
   NODE_NAME_CASE(TBUFFER_STORE_FORMAT)
@@ -4185,10 +4194,10 @@ const char* AMDGPUTargetLowering::getTargetNodeName(unsigned Opcode) const {
   NODE_NAME_CASE(TBUFFER_STORE_FORMAT_D16)
   NODE_NAME_CASE(TBUFFER_LOAD_FORMAT)
   NODE_NAME_CASE(TBUFFER_LOAD_FORMAT_D16)
+  NODE_NAME_CASE(DS_ORDERED_COUNT)
   NODE_NAME_CASE(ATOMIC_CMP_SWAP)
   NODE_NAME_CASE(ATOMIC_INC)
   NODE_NAME_CASE(ATOMIC_DEC)
-  NODE_NAME_CASE(ATOMIC_LOAD_FADD)
   NODE_NAME_CASE(ATOMIC_LOAD_FMIN)
   NODE_NAME_CASE(ATOMIC_LOAD_FMAX)
   NODE_NAME_CASE(BUFFER_LOAD)
@@ -4294,33 +4303,42 @@ void AMDGPUTargetLowering::computeKnownBitsForTargetNode(
   }
   case AMDGPUISD::MUL_U24:
   case AMDGPUISD::MUL_I24: {
-    KnownBits LHSKnown, RHSKnown;
-    DAG.computeKnownBits(Op.getOperand(0), LHSKnown, Depth + 1);
-    DAG.computeKnownBits(Op.getOperand(1), RHSKnown, Depth + 1);
-
+    KnownBits LHSKnown = DAG.computeKnownBits(Op.getOperand(0), Depth + 1);
+    KnownBits RHSKnown = DAG.computeKnownBits(Op.getOperand(1), Depth + 1);
     unsigned TrailZ = LHSKnown.countMinTrailingZeros() +
                       RHSKnown.countMinTrailingZeros();
     Known.Zero.setLowBits(std::min(TrailZ, 32u));
 
-    unsigned LHSValBits = 32 - std::max(LHSKnown.countMinSignBits(), 8u);
-    unsigned RHSValBits = 32 - std::max(RHSKnown.countMinSignBits(), 8u);
-    unsigned MaxValBits = std::min(LHSValBits + RHSValBits, 32u);
-    if (MaxValBits >= 32)
-      break;
+    // Truncate to 24 bits.
+    LHSKnown = LHSKnown.trunc(24);
+    RHSKnown = RHSKnown.trunc(24);
+
     bool Negative = false;
     if (Opc == AMDGPUISD::MUL_I24) {
-      bool LHSNegative = !!(LHSKnown.One  & (1 << 23));
-      bool LHSPositive = !!(LHSKnown.Zero & (1 << 23));
-      bool RHSNegative = !!(RHSKnown.One  & (1 << 23));
-      bool RHSPositive = !!(RHSKnown.Zero & (1 << 23));
+      unsigned LHSValBits = 24 - LHSKnown.countMinSignBits();
+      unsigned RHSValBits = 24 - RHSKnown.countMinSignBits();
+      unsigned MaxValBits = std::min(LHSValBits + RHSValBits, 32u);
+      if (MaxValBits >= 32)
+        break;
+      bool LHSNegative = LHSKnown.isNegative();
+      bool LHSPositive = LHSKnown.isNonNegative();
+      bool RHSNegative = RHSKnown.isNegative();
+      bool RHSPositive = RHSKnown.isNonNegative();
       if ((!LHSNegative && !LHSPositive) || (!RHSNegative && !RHSPositive))
         break;
       Negative = (LHSNegative && RHSPositive) || (LHSPositive && RHSNegative);
-    }
-    if (Negative)
-      Known.One.setHighBits(32 - MaxValBits);
-    else
+      if (Negative)
+        Known.One.setHighBits(32 - MaxValBits);
+      else
+        Known.Zero.setHighBits(32 - MaxValBits);
+    } else {
+      unsigned LHSValBits = 24 - LHSKnown.countMinLeadingZeros();
+      unsigned RHSValBits = 24 - RHSKnown.countMinLeadingZeros();
+      unsigned MaxValBits = std::min(LHSValBits + RHSValBits, 32u);
+      if (MaxValBits >= 32)
+        break;
       Known.Zero.setHighBits(32 - MaxValBits);
+    }
     break;
   }
   case AMDGPUISD::PERM: {
@@ -4328,9 +4346,8 @@ void AMDGPUTargetLowering::computeKnownBitsForTargetNode(
     if (!CMask)
       return;
 
-    KnownBits LHSKnown, RHSKnown;
-    DAG.computeKnownBits(Op.getOperand(0), LHSKnown, Depth + 1);
-    DAG.computeKnownBits(Op.getOperand(1), RHSKnown, Depth + 1);
+    KnownBits LHSKnown = DAG.computeKnownBits(Op.getOperand(0), Depth + 1);
+    KnownBits RHSKnown = DAG.computeKnownBits(Op.getOperand(1), Depth + 1);
     unsigned Sel = CMask->getZExtValue();
 
     for (unsigned I = 0; I < 32; I += 8) {
@@ -4504,7 +4521,12 @@ bool AMDGPUTargetLowering::isKnownNeverNaNForTargetNode(SDValue Op,
 
 TargetLowering::AtomicExpansionKind
 AMDGPUTargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *RMW) const {
-  if (RMW->getOperation() == AtomicRMWInst::Nand)
+  switch (RMW->getOperation()) {
+  case AtomicRMWInst::Nand:
+  case AtomicRMWInst::FAdd:
+  case AtomicRMWInst::FSub:
     return AtomicExpansionKind::CmpXChg;
-  return AtomicExpansionKind::None;
+  default:
+    return AtomicExpansionKind::None;
+  }
 }
diff --git a/lib/Target/AMDGPU/AMDGPUISelLowering.h b/lib/Target/AMDGPU/AMDGPUISelLowering.h
index 0d22cb2e3e20bcba352aafb7216ce88bc8ecaab3..359e16cfa5617715d519abb644f410ea3ad83f6e 100644
--- a/lib/Target/AMDGPU/AMDGPUISelLowering.h
+++ b/lib/Target/AMDGPU/AMDGPUISelLowering.h
@@ -1,9 +1,8 @@
 //===-- AMDGPUISelLowering.h - AMDGPU Lowering Interface --------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -212,14 +211,14 @@ public:
 
   const char* getTargetNodeName(unsigned Opcode) const override;
 
-  // FIXME: Turn off MergeConsecutiveStores() before Instruction Selection
-  // for AMDGPU.
-  // A commit ( git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@319036
-  // 91177308-0d34-0410-b5e6-96231b3b80d8 ) turned on
-  // MergeConsecutiveStores() before Instruction Selection for all targets.
-  // Enough AMDGPU compiles go into an infinite loop ( MergeConsecutiveStores()
-  // merges two stores; LegalizeStoreOps() un-merges; MergeConsecutiveStores()
-  // re-merges, etc. ) to warrant turning it off for now.
+  // FIXME: Turn off MergeConsecutiveStores() before Instruction Selection for
+  // AMDGPU.  Commit r319036,
+  // (https://github.com/llvm/llvm-project/commit/db77e57ea86d941a4262ef60261692f4cb6893e6)
+  // turned on MergeConsecutiveStores() before Instruction Selection for all
+  // targets.  Enough AMDGPU compiles go into an infinite loop (
+  // MergeConsecutiveStores() merges two stores; LegalizeStoreOps() un-merges;
+  // MergeConsecutiveStores() re-merges, etc. ) to warrant turning it off for
+  // now.
   bool mergeStoresAfterLegalization() const override { return false; }
 
   bool isFsqrtCheap(SDValue Operand, SelectionDAG &DAG) const override {
@@ -463,6 +462,9 @@ enum NodeType : unsigned {
   INTERP_MOV,
   INTERP_P1,
   INTERP_P2,
+  INTERP_P1LL_F16,
+  INTERP_P1LV_F16,
+  INTERP_P2_F16,
   PC_ADD_REL_OFFSET,
   KILL,
   DUMMY_CHAIN,
@@ -474,10 +476,10 @@ enum NodeType : unsigned {
   TBUFFER_STORE_FORMAT_D16,
   TBUFFER_LOAD_FORMAT,
   TBUFFER_LOAD_FORMAT_D16,
+  DS_ORDERED_COUNT,
   ATOMIC_CMP_SWAP,
   ATOMIC_INC,
   ATOMIC_DEC,
-  ATOMIC_LOAD_FADD,
   ATOMIC_LOAD_FMIN,
   ATOMIC_LOAD_FMAX,
   BUFFER_LOAD,
diff --git a/lib/Target/AMDGPU/AMDGPUInline.cpp b/lib/Target/AMDGPU/AMDGPUInline.cpp
index 945c9acd379a5f3ba39d1803441f5c50504dfcb8..0ad78e036a18a078c4970b4baf312dba3cd6387d 100644
--- a/lib/Target/AMDGPU/AMDGPUInline.cpp
+++ b/lib/Target/AMDGPU/AMDGPUInline.cpp
@@ -1,9 +1,8 @@
 //===- AMDGPUInline.cpp - Code to perform simple function inlining --------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -182,9 +181,10 @@ InlineCost AMDGPUInliner::getInlineCost(CallSite CS) {
     return llvm::InlineCost::getNever("incompatible");
 
   if (CS.hasFnAttr(Attribute::AlwaysInline)) {
-    if (isInlineViable(*Callee))
+    auto IsViable = isInlineViable(*Callee);
+    if (IsViable)
       return llvm::InlineCost::getAlways("alwaysinline viable");
-    return llvm::InlineCost::getNever("alwaysinline unviable");
+    return llvm::InlineCost::getNever(IsViable.message);
   }
 
   if (isWrapperOnlyCall(CS))
diff --git a/lib/Target/AMDGPU/AMDGPUInstrInfo.cpp b/lib/Target/AMDGPU/AMDGPUInstrInfo.cpp
index 07aa7c2cc8ada5be034a6b07fba41a9a4a54be01..9951cbf2326e35ee94d8eabc19d5dc7a45013161 100644
--- a/lib/Target/AMDGPU/AMDGPUInstrInfo.cpp
+++ b/lib/Target/AMDGPU/AMDGPUInstrInfo.cpp
@@ -1,9 +1,8 @@
 //===-- AMDGPUInstrInfo.cpp - Base class for AMD GPU InstrInfo ------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AMDGPU/AMDGPUInstrInfo.h b/lib/Target/AMDGPU/AMDGPUInstrInfo.h
index 2f8166da0d3375caed68b685a03dd45c64b5bea2..698189e14c21eeb32bc3838a2c316a2f00a1f6e7 100644
--- a/lib/Target/AMDGPU/AMDGPUInstrInfo.h
+++ b/lib/Target/AMDGPU/AMDGPUInstrInfo.h
@@ -1,9 +1,8 @@
 //===-- AMDGPUInstrInfo.h - AMDGPU Instruction Information ------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AMDGPU/AMDGPUInstrInfo.td b/lib/Target/AMDGPU/AMDGPUInstrInfo.td
index 82644be2656384cd2add9d63a505037e5dc66322..ba924b1f63a2714a447fc2e395458200f70ff902 100644
--- a/lib/Target/AMDGPU/AMDGPUInstrInfo.td
+++ b/lib/Target/AMDGPU/AMDGPUInstrInfo.td
@@ -1,9 +1,8 @@
 //===-- AMDGPUInstrInfo.td - AMDGPU DAG nodes --------------*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -70,8 +69,6 @@ def AMDGPUAddeSubeOp : SDTypeProfile<2, 3,
   [SDTCisSameAs<0, 2>, SDTCisSameAs<0, 3>, SDTCisVT<0, i32>, SDTCisVT<1, i1>, SDTCisVT<4, i1>]
 >;
 
-def SDT_AMDGPUTCRET : SDTypeProfile<0, 2, [SDTCisPtrTy<0>]>;
-
 //===----------------------------------------------------------------------===//
 // AMDGPU DAG Nodes
 //
@@ -96,7 +93,8 @@ def AMDGPUcall : SDNode<"AMDGPUISD::CALL",
   SDNPVariadic]
 >;
 
-def AMDGPUtc_return: SDNode<"AMDGPUISD::TC_RETURN", SDT_AMDGPUTCRET,
+def AMDGPUtc_return: SDNode<"AMDGPUISD::TC_RETURN",
+  SDTypeProfile<0, 3, [SDTCisPtrTy<0>]>,
   [SDNPHasChain, SDNPOptInGlue, SDNPVariadic]
 >;
 
@@ -370,6 +368,17 @@ def AMDGPUinterp_p2 : SDNode<"AMDGPUISD::INTERP_P2",
                       SDTypeProfile<1, 4, [SDTCisFP<0>]>,
                       [SDNPInGlue]>;
 
+def AMDGPUinterp_p1ll_f16 : SDNode<"AMDGPUISD::INTERP_P1LL_F16",
+                            SDTypeProfile<1, 7, [SDTCisFP<0>]>,
+                            [SDNPInGlue, SDNPOutGlue]>;
+
+def AMDGPUinterp_p1lv_f16 : SDNode<"AMDGPUISD::INTERP_P1LV_F16",
+                            SDTypeProfile<1, 9, [SDTCisFP<0>]>,
+                            [SDNPInGlue, SDNPOutGlue]>;
+
+def AMDGPUinterp_p2_f16 : SDNode<"AMDGPUISD::INTERP_P2_F16",
+                          SDTypeProfile<1, 8, [SDTCisFP<0>]>,
+                          [SDNPInGlue]>;
 
 def AMDGPUkill : SDNode<"AMDGPUISD::KILL", AMDGPUKillSDT,
   [SDNPHasChain, SDNPSideEffect]>;
diff --git a/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp b/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
index 8eb49d49b2e086c93d8a194bca47cc2933c01cbd..3e21edd2408519158b7900b020eb4aedafaf6119 100644
--- a/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
+++ b/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
@@ -1,9 +1,8 @@
 //===- AMDGPUInstructionSelector.cpp ----------------------------*- C++ -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/lib/Target/AMDGPU/AMDGPUInstructionSelector.h b/lib/Target/AMDGPU/AMDGPUInstructionSelector.h
index 449431adc561a7fac9bc450498719b0e3001f34c..a7a040dd21b5d8ec84c788cddc94c5a84abcb364 100644
--- a/lib/Target/AMDGPU/AMDGPUInstructionSelector.h
+++ b/lib/Target/AMDGPU/AMDGPUInstructionSelector.h
@@ -1,9 +1,8 @@
 //===- AMDGPUInstructionSelector --------------------------------*- C++ -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/lib/Target/AMDGPU/AMDGPUInstructions.td b/lib/Target/AMDGPU/AMDGPUInstructions.td
index 282d1c118333629717d6eecbd8f4c0791b5e5193..bdbfdaf1dc129954c117276c0aede9ef9eb2f71b 100644
--- a/lib/Target/AMDGPU/AMDGPUInstructions.td
+++ b/lib/Target/AMDGPU/AMDGPUInstructions.td
@@ -1,9 +1,8 @@
 //===-- AMDGPUInstructions.td - Common instruction defs ---*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -66,12 +65,8 @@ class ILFormat<dag outs, dag ins, string asmstr, list<dag> pattern>
 
 def TruePredicate : Predicate<"true">;
 
-// Exists to help track down where SubtargetPredicate isn't set rather
-// than letting tablegen crash with an unhelpful error.
-def InvalidPred : Predicate<"predicate not set on instruction or pattern">;
-
 class PredicateControl {
-  Predicate SubtargetPredicate = InvalidPred;
+  Predicate SubtargetPredicate = TruePredicate;
   list<Predicate> AssemblerPredicates = [];
   Predicate AssemblerPredicate = TruePredicate;
   list<Predicate> OtherPredicates = [];
@@ -626,9 +621,7 @@ class Extract_Element <ValueType sub_type, ValueType vec_type, int sub_idx,
   : AMDGPUPat<
   (sub_type (extractelt vec_type:$src, sub_idx)),
   (EXTRACT_SUBREG $src, sub_reg)
-> {
-  let SubtargetPredicate = TruePredicate;
-}
+>;
 
 /* Insert element pattern */
 class Insert_Element <ValueType elem_type, ValueType vec_type,
@@ -636,9 +629,7 @@ class Insert_Element <ValueType elem_type, ValueType vec_type,
   : AMDGPUPat <
   (insertelt vec_type:$vec, elem_type:$elem, sub_idx),
   (INSERT_SUBREG $vec, $elem, sub_reg)
-> {
-  let SubtargetPredicate = TruePredicate;
-}
+>;
 
 // XXX: Convert to new syntax and use COPY_TO_REG, once the DFAPacketizer
 // can handle COPY instructions.
@@ -842,6 +833,7 @@ def cvt_flr_i32_f32 : PatFrag <
   [{ (void)N; return TM.Options.NoNaNsFPMath; }]
 >;
 
+let AddedComplexity = 2 in {
 class IMad24Pat<Instruction Inst, bit HasClamp = 0> : AMDGPUPat <
   (add (AMDGPUmul_i24 i32:$src0, i32:$src1), i32:$src2),
   !if(HasClamp, (Inst $src0, $src1, $src2, (i1 0)),
@@ -853,6 +845,7 @@ class UMad24Pat<Instruction Inst, bit HasClamp = 0> : AMDGPUPat <
   !if(HasClamp, (Inst $src0, $src1, $src2, (i1 0)),
                 (Inst $src0, $src1, $src2))
 >;
+} // AddedComplexity.
 
 class RcpPat<Instruction RcpInst, ValueType vt> : AMDGPUPat <
   (fdiv FP_ONE, vt:$src),
diff --git a/lib/Target/AMDGPU/AMDGPUIntrinsicInfo.cpp b/lib/Target/AMDGPU/AMDGPUIntrinsicInfo.cpp
deleted file mode 100644
index 02108ca3ddd783cbe3d355139faec0b2cad4c878..0000000000000000000000000000000000000000
--- a/lib/Target/AMDGPU/AMDGPUIntrinsicInfo.cpp
+++ /dev/null
@@ -1,103 +0,0 @@
-//===- AMDGPUIntrinsicInfo.cpp - AMDGPU Intrinsic Information ---*- C++ -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//==-----------------------------------------------------------------------===//
-//
-/// \file
-/// AMDGPU Implementation of the IntrinsicInfo class.
-//
-//===-----------------------------------------------------------------------===//
-
-#include "AMDGPUIntrinsicInfo.h"
-#include "AMDGPUSubtarget.h"
-#include "llvm/IR/DerivedTypes.h"
-#include "llvm/IR/Intrinsics.h"
-#include "llvm/IR/Module.h"
-
-using namespace llvm;
-
-AMDGPUIntrinsicInfo::AMDGPUIntrinsicInfo()
-    : TargetIntrinsicInfo() {}
-
-static const char *const IntrinsicNameTable[] = {
-#define GET_INTRINSIC_NAME_TABLE
-#include "AMDGPUGenIntrinsicImpl.inc"
-#undef GET_INTRINSIC_NAME_TABLE
-};
-
-namespace {
-#define GET_INTRINSIC_ATTRIBUTES
-#include "AMDGPUGenIntrinsicImpl.inc"
-#undef GET_INTRINSIC_ATTRIBUTES
-}
-
-StringRef AMDGPUIntrinsicInfo::getName(unsigned IntrID,
-                                       ArrayRef<Type *> Tys) const {
-  if (IntrID < Intrinsic::num_intrinsics)
-    return StringRef();
-
-  assert(IntrID < SIIntrinsic::num_AMDGPU_intrinsics &&
-         "Invalid intrinsic ID");
-
-  return IntrinsicNameTable[IntrID - Intrinsic::num_intrinsics];
-}
-
-std::string AMDGPUIntrinsicInfo::getName(unsigned IntrID, Type **Tys,
-                                         unsigned NumTys) const {
-  return getName(IntrID, makeArrayRef(Tys, NumTys)).str();
-}
-
-FunctionType *AMDGPUIntrinsicInfo::getType(LLVMContext &Context, unsigned ID,
-                                           ArrayRef<Type*> Tys) const {
-  // FIXME: Re-use Intrinsic::getType machinery
-  llvm_unreachable("unhandled intrinsic");
-}
-
-unsigned AMDGPUIntrinsicInfo::lookupName(const char *NameData,
-                                         unsigned Len) const {
-  StringRef Name(NameData, Len);
-  if (!Name.startswith("llvm."))
-    return 0; // All intrinsics start with 'llvm.'
-
-  // Look for a name match in our table.  If the intrinsic is not overloaded,
-  // require an exact match. If it is overloaded, require a prefix match. The
-  // AMDGPU enum enum starts at Intrinsic::num_intrinsics.
-  int Idx = Intrinsic::lookupLLVMIntrinsicByName(IntrinsicNameTable, Name);
-  if (Idx >= 0) {
-    bool IsPrefixMatch = Name.size() > strlen(IntrinsicNameTable[Idx]);
-    return IsPrefixMatch == isOverloaded(Idx + 1)
-               ? Intrinsic::num_intrinsics + Idx
-               : 0;
-  }
-
-  return 0;
-}
-
-bool AMDGPUIntrinsicInfo::isOverloaded(unsigned id) const {
-// Overload Table
-#define GET_INTRINSIC_OVERLOAD_TABLE
-#include "AMDGPUGenIntrinsicImpl.inc"
-#undef GET_INTRINSIC_OVERLOAD_TABLE
-}
-
-Function *AMDGPUIntrinsicInfo::getDeclaration(Module *M, unsigned IntrID,
-                                              ArrayRef<Type *> Tys) const {
-  FunctionType *FTy = getType(M->getContext(), IntrID, Tys);
-  Function *F
-    = cast<Function>(M->getOrInsertFunction(getName(IntrID, Tys), FTy));
-
-  AttributeList AS =
-      getAttributes(M->getContext(), static_cast<SIIntrinsic::ID>(IntrID));
-  F->setAttributes(AS);
-  return F;
-}
-
-Function *AMDGPUIntrinsicInfo::getDeclaration(Module *M, unsigned IntrID,
-                                              Type **Tys,
-                                              unsigned NumTys) const {
-  return getDeclaration(M, IntrID, makeArrayRef(Tys, NumTys));
-}
diff --git a/lib/Target/AMDGPU/AMDGPUIntrinsicInfo.h b/lib/Target/AMDGPU/AMDGPUIntrinsicInfo.h
deleted file mode 100644
index a1a094dded23d007a277f3a68152a56dbbbb0e60..0000000000000000000000000000000000000000
--- a/lib/Target/AMDGPU/AMDGPUIntrinsicInfo.h
+++ /dev/null
@@ -1,58 +0,0 @@
-//===- AMDGPUIntrinsicInfo.h - AMDGPU Intrinsic Information ------*- C++ -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//==-----------------------------------------------------------------------===//
-//
-/// \file
-/// Interface for the AMDGPU Implementation of the Intrinsic Info class.
-//
-//===-----------------------------------------------------------------------===//
-#ifndef LLVM_LIB_TARGET_AMDGPU_AMDGPUINTRINSICINFO_H
-#define LLVM_LIB_TARGET_AMDGPU_AMDGPUINTRINSICINFO_H
-
-#include "llvm/IR/Intrinsics.h"
-#include "llvm/Target/TargetIntrinsicInfo.h"
-
-namespace llvm {
-class TargetMachine;
-
-namespace SIIntrinsic {
-enum ID {
-  last_non_AMDGPU_intrinsic = Intrinsic::num_intrinsics - 1,
-#define GET_INTRINSIC_ENUM_VALUES
-#include "AMDGPUGenIntrinsicEnums.inc"
-#undef GET_INTRINSIC_ENUM_VALUES
-      , num_AMDGPU_intrinsics
-};
-
-} // end namespace AMDGPUIntrinsic
-
-class AMDGPUIntrinsicInfo final : public TargetIntrinsicInfo {
-public:
-  AMDGPUIntrinsicInfo();
-
-  StringRef getName(unsigned IntrId, ArrayRef<Type *> Tys = None) const;
-
-  std::string getName(unsigned IntrId, Type **Tys = nullptr,
-                      unsigned NumTys = 0) const override;
-
-  unsigned lookupName(const char *Name, unsigned Len) const override;
-  bool isOverloaded(unsigned IID) const override;
-  Function *getDeclaration(Module *M, unsigned ID,
-                           Type **Tys = nullptr,
-                           unsigned NumTys = 0) const override;
-
-  Function *getDeclaration(Module *M, unsigned ID,
-                           ArrayRef<Type *> = None) const;
-
-  FunctionType *getType(LLVMContext &Context, unsigned ID,
-                        ArrayRef<Type*> Tys = None) const;
-};
-
-} // end namespace llvm
-
-#endif
diff --git a/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
index 62fe0f3a8b91aa38c3e651307940f2667f7fdbf3..ecf8e1b732e97e6b5dcdf73ec3b10b64f7589172 100644
--- a/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
+++ b/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
@@ -1,9 +1,8 @@
 //===- AMDGPULegalizerInfo.cpp -----------------------------------*- C++ -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
@@ -15,6 +14,9 @@
 #include "AMDGPU.h"
 #include "AMDGPULegalizerInfo.h"
 #include "AMDGPUTargetMachine.h"
+#include "SIMachineFunctionInfo.h"
+
+#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
 #include "llvm/CodeGen/TargetOpcodes.h"
 #include "llvm/CodeGen/ValueTypes.h"
 #include "llvm/IR/DerivedTypes.h"
@@ -23,6 +25,36 @@
 
 using namespace llvm;
 using namespace LegalizeActions;
+using namespace LegalizeMutations;
+using namespace LegalityPredicates;
+
+
+static LegalityPredicate isMultiple32(unsigned TypeIdx,
+                                      unsigned MaxSize = 512) {
+  return [=](const LegalityQuery &Query) {
+    const LLT Ty = Query.Types[TypeIdx];
+    const LLT EltTy = Ty.getScalarType();
+    return Ty.getSizeInBits() <= MaxSize && EltTy.getSizeInBits() % 32 == 0;
+  };
+}
+
+static LegalityPredicate isSmallOddVector(unsigned TypeIdx) {
+  return [=](const LegalityQuery &Query) {
+    const LLT Ty = Query.Types[TypeIdx];
+    return Ty.isVector() &&
+           Ty.getNumElements() % 2 != 0 &&
+           Ty.getElementType().getSizeInBits() < 32;
+  };
+}
+
+static LegalizeMutation oneMoreElement(unsigned TypeIdx) {
+  return [=](const LegalityQuery &Query) {
+    const LLT Ty = Query.Types[TypeIdx];
+    const LLT EltTy = Ty.getElementType();
+    return std::make_pair(TypeIdx, LLT::vector(Ty.getNumElements() + 1, EltTy));
+  };
+}
+
 
 AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST,
                                          const GCNTargetMachine &TM) {
@@ -33,12 +65,48 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST,
   };
 
   const LLT S1 = LLT::scalar(1);
-  const LLT V2S16 = LLT::vector(2, 16);
-
+  const LLT S8 = LLT::scalar(8);
+  const LLT S16 = LLT::scalar(16);
   const LLT S32 = LLT::scalar(32);
   const LLT S64 = LLT::scalar(64);
+  const LLT S128 = LLT::scalar(128);
+  const LLT S256 = LLT::scalar(256);
   const LLT S512 = LLT::scalar(512);
 
+  const LLT V2S16 = LLT::vector(2, 16);
+  const LLT V4S16 = LLT::vector(4, 16);
+  const LLT V8S16 = LLT::vector(8, 16);
+
+  const LLT V2S32 = LLT::vector(2, 32);
+  const LLT V3S32 = LLT::vector(3, 32);
+  const LLT V4S32 = LLT::vector(4, 32);
+  const LLT V5S32 = LLT::vector(5, 32);
+  const LLT V6S32 = LLT::vector(6, 32);
+  const LLT V7S32 = LLT::vector(7, 32);
+  const LLT V8S32 = LLT::vector(8, 32);
+  const LLT V9S32 = LLT::vector(9, 32);
+  const LLT V10S32 = LLT::vector(10, 32);
+  const LLT V11S32 = LLT::vector(11, 32);
+  const LLT V12S32 = LLT::vector(12, 32);
+  const LLT V13S32 = LLT::vector(13, 32);
+  const LLT V14S32 = LLT::vector(14, 32);
+  const LLT V15S32 = LLT::vector(15, 32);
+  const LLT V16S32 = LLT::vector(16, 32);
+
+  const LLT V2S64 = LLT::vector(2, 64);
+  const LLT V3S64 = LLT::vector(3, 64);
+  const LLT V4S64 = LLT::vector(4, 64);
+  const LLT V5S64 = LLT::vector(5, 64);
+  const LLT V6S64 = LLT::vector(6, 64);
+  const LLT V7S64 = LLT::vector(7, 64);
+  const LLT V8S64 = LLT::vector(8, 64);
+
+  std::initializer_list<LLT> AllS32Vectors =
+    {V2S32, V3S32, V4S32, V5S32, V6S32, V7S32, V8S32,
+     V9S32, V10S32, V11S32, V12S32, V13S32, V14S32, V15S32, V16S32};
+  std::initializer_list<LLT> AllS64Vectors =
+    {V2S64, V3S64, V4S64, V5S64, V6S64, V7S64, V8S64};
+
   const LLT GlobalPtr = GetAddrSpacePtr(AMDGPUAS::GLOBAL_ADDRESS);
   const LLT ConstantPtr = GetAddrSpacePtr(AMDGPUAS::CONSTANT_ADDRESS);
   const LLT LocalPtr = GetAddrSpacePtr(AMDGPUAS::LOCAL_ADDRESS);
@@ -55,74 +123,127 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST,
     PrivatePtr
   };
 
-  setAction({G_ADD, S32}, Legal);
-  setAction({G_ASHR, S32}, Legal);
-  setAction({G_SUB, S32}, Legal);
-  setAction({G_MUL, S32}, Legal);
-  setAction({G_AND, S32}, Legal);
-  setAction({G_OR, S32}, Legal);
-  setAction({G_XOR, S32}, Legal);
-
-  setAction({G_BITCAST, V2S16}, Legal);
-  setAction({G_BITCAST, 1, S32}, Legal);
-
-  setAction({G_BITCAST, S32}, Legal);
-  setAction({G_BITCAST, 1, V2S16}, Legal);
-
-  getActionDefinitionsBuilder(G_FCONSTANT)
-    .legalFor({S32, S64});
+  setAction({G_BRCOND, S1}, Legal);
+
+  getActionDefinitionsBuilder({G_ADD, G_SUB, G_MUL, G_UMULH, G_SMULH})
+    .legalFor({S32})
+    .clampScalar(0, S32, S32)
+    .scalarize(0);
+
+  // Report legal for any types we can handle anywhere. For the cases only legal
+  // on the SALU, RegBankSelect will be able to re-legalize.
+  getActionDefinitionsBuilder({G_AND, G_OR, G_XOR})
+    .legalFor({S32, S1, S64, V2S32, V2S16, V4S16})
+    .clampScalar(0, S32, S64)
+    .scalarize(0);
+
+  getActionDefinitionsBuilder({G_UADDO, G_SADDO, G_USUBO, G_SSUBO,
+                               G_UADDE, G_SADDE, G_USUBE, G_SSUBE})
+    .legalFor({{S32, S1}})
+    .clampScalar(0, S32, S32);
+
+  getActionDefinitionsBuilder(G_BITCAST)
+    .legalForCartesianProduct({S32, V2S16})
+    .legalForCartesianProduct({S64, V2S32, V4S16})
+    .legalForCartesianProduct({V2S64, V4S32})
+    // Don't worry about the size constraint.
+    .legalIf(all(isPointer(0), isPointer(1)));
+
+  if (ST.has16BitInsts()) {
+    getActionDefinitionsBuilder(G_FCONSTANT)
+      .legalFor({S32, S64, S16})
+      .clampScalar(0, S16, S64);
+  } else {
+    getActionDefinitionsBuilder(G_FCONSTANT)
+      .legalFor({S32, S64})
+      .clampScalar(0, S32, S64);
+  }
 
-  // G_IMPLICIT_DEF is a no-op so we can make it legal for any value type that
-  // can fit in a register.
-  // FIXME: We need to legalize several more operations before we can add
-  // a test case for size > 512.
   getActionDefinitionsBuilder(G_IMPLICIT_DEF)
-    .legalIf([=](const LegalityQuery &Query) {
-        return Query.Types[0].getSizeInBits() <= 512;
-    })
-    .clampScalar(0, S1, S512);
+    .legalFor({S1, S32, S64, V2S32, V4S32, V2S16, V4S16, GlobalPtr,
+               ConstantPtr, LocalPtr, FlatPtr, PrivatePtr})
+    .moreElementsIf(isSmallOddVector(0), oneMoreElement(0))
+    .clampScalarOrElt(0, S32, S512)
+    .legalIf(isMultiple32(0))
+    .widenScalarToNextPow2(0, 32);
 
-  getActionDefinitionsBuilder(G_CONSTANT)
-    .legalFor({S1, S32, S64});
 
   // FIXME: i1 operands to intrinsics should always be legal, but other i1
   // values may not be legal.  We need to figure out how to distinguish
   // between these two scenarios.
-  setAction({G_CONSTANT, S1}, Legal);
+  getActionDefinitionsBuilder(G_CONSTANT)
+    .legalFor({S1, S32, S64, GlobalPtr,
+               LocalPtr, ConstantPtr, PrivatePtr, FlatPtr })
+    .clampScalar(0, S32, S64)
+    .widenScalarToNextPow2(0)
+    .legalIf(isPointer(0));
 
   setAction({G_FRAME_INDEX, PrivatePtr}, Legal);
 
-  getActionDefinitionsBuilder(
-    { G_FADD, G_FMUL, G_FNEG, G_FABS, G_FMA})
+  auto &FPOpActions = getActionDefinitionsBuilder(
+    { G_FADD, G_FMUL, G_FNEG, G_FABS, G_FMA, G_FCANONICALIZE})
     .legalFor({S32, S64});
 
-  // Use actual fsub instruction
-  setAction({G_FSUB, S32}, Legal);
-
-  // Must use fadd + fneg
-  setAction({G_FSUB, S64}, Lower);
-
-  setAction({G_FCMP, S1}, Legal);
-  setAction({G_FCMP, 1, S32}, Legal);
-  setAction({G_FCMP, 1, S64}, Legal);
-
-  setAction({G_ZEXT, S64}, Legal);
-  setAction({G_ZEXT, 1, S32}, Legal);
-
-  setAction({G_SEXT, S64}, Legal);
-  setAction({G_SEXT, 1, S32}, Legal);
-
-  setAction({G_ANYEXT, S64}, Legal);
-  setAction({G_ANYEXT, 1, S32}, Legal);
-
-  setAction({G_FPTOSI, S32}, Legal);
-  setAction({G_FPTOSI, 1, S32}, Legal);
+  if (ST.has16BitInsts()) {
+    if (ST.hasVOP3PInsts())
+      FPOpActions.legalFor({S16, V2S16});
+    else
+      FPOpActions.legalFor({S16});
+  }
 
-  setAction({G_SITOFP, S32}, Legal);
-  setAction({G_SITOFP, 1, S32}, Legal);
+  if (ST.hasVOP3PInsts())
+    FPOpActions.clampMaxNumElements(0, S16, 2);
+  FPOpActions
+    .scalarize(0)
+    .clampScalar(0, ST.has16BitInsts() ? S16 : S32, S64);
+
+  if (ST.has16BitInsts()) {
+    getActionDefinitionsBuilder(G_FSQRT)
+      .legalFor({S32, S64, S16})
+      .scalarize(0)
+      .clampScalar(0, S16, S64);
+  } else {
+    getActionDefinitionsBuilder(G_FSQRT)
+      .legalFor({S32, S64})
+      .scalarize(0)
+      .clampScalar(0, S32, S64);
+  }
 
-  setAction({G_FPTOUI, S32}, Legal);
-  setAction({G_FPTOUI, 1, S32}, Legal);
+  getActionDefinitionsBuilder(G_FPTRUNC)
+    .legalFor({{S32, S64}, {S16, S32}})
+    .scalarize(0);
+
+  getActionDefinitionsBuilder(G_FPEXT)
+    .legalFor({{S64, S32}, {S32, S16}})
+    .lowerFor({{S64, S16}}) // FIXME: Implement
+    .scalarize(0);
+
+  getActionDefinitionsBuilder(G_FSUB)
+      // Use actual fsub instruction
+      .legalFor({S32})
+      // Must use fadd + fneg
+      .lowerFor({S64, S16, V2S16})
+      .scalarize(0)
+      .clampScalar(0, S32, S64);
+
+  getActionDefinitionsBuilder({G_SEXT, G_ZEXT, G_ANYEXT})
+    .legalFor({{S64, S32}, {S32, S16}, {S64, S16},
+               {S32, S1}, {S64, S1}, {S16, S1},
+               // FIXME: Hack
+               {S32, S8}, {S128, S32}, {S128, S64}, {S32, LLT::scalar(24)}})
+    .scalarize(0);
+
+  getActionDefinitionsBuilder({G_SITOFP, G_UITOFP})
+    .legalFor({{S32, S32}, {S64, S32}})
+    .scalarize(0);
+
+  getActionDefinitionsBuilder({G_FPTOSI, G_FPTOUI})
+    .legalFor({{S32, S32}, {S32, S64}})
+    .scalarize(0);
+
+  getActionDefinitionsBuilder({G_INTRINSIC_TRUNC, G_INTRINSIC_ROUND})
+    .legalFor({S32, S64})
+    .scalarize(0);
 
   for (LLT PtrTy : AddrSpaces) {
     LLT IdxTy = LLT::scalar(PtrTy.getSizeInBits());
@@ -130,35 +251,141 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST,
     setAction({G_GEP, 1, IdxTy}, Legal);
   }
 
+  // FIXME: When RegBankSelect inserts copies, it will only create new registers
+  // with scalar types. This means we can end up with G_LOAD/G_STORE/G_GEP
+  // instruction with scalar types for their pointer operands. In assert builds,
+  // the instruction selector will assert if it sees a generic instruction which
+  // isn't legal, so we need to tell it that scalar types are legal for pointer
+  // operands
+  setAction({G_GEP, S64}, Legal);
+
   setAction({G_BLOCK_ADDR, CodePtr}, Legal);
 
-  setAction({G_ICMP, S1}, Legal);
-  setAction({G_ICMP, 1, S32}, Legal);
+  getActionDefinitionsBuilder(G_ICMP)
+    .legalForCartesianProduct(
+      {S1}, {S32, S64, GlobalPtr, LocalPtr, ConstantPtr, PrivatePtr, FlatPtr})
+    .legalFor({{S1, S32}, {S1, S64}})
+    .widenScalarToNextPow2(1)
+    .clampScalar(1, S32, S64)
+    .scalarize(0)
+    .legalIf(all(typeIs(0, S1), isPointer(1)));
+
+  getActionDefinitionsBuilder(G_FCMP)
+    .legalFor({{S1, S32}, {S1, S64}})
+    .widenScalarToNextPow2(1)
+    .clampScalar(1, S32, S64)
+    .scalarize(0);
+
+  // FIXME: fexp, flog2, flog10 needs to be custom lowered.
+  getActionDefinitionsBuilder({G_FPOW, G_FEXP, G_FEXP2,
+                               G_FLOG, G_FLOG2, G_FLOG10})
+    .legalFor({S32})
+    .scalarize(0);
+
+  // The 64-bit versions produce 32-bit results, but only on the SALU.
+  getActionDefinitionsBuilder({G_CTLZ, G_CTLZ_ZERO_UNDEF,
+                               G_CTTZ, G_CTTZ_ZERO_UNDEF,
+                               G_CTPOP})
+    .legalFor({{S32, S32}, {S32, S64}})
+    .clampScalar(0, S32, S32)
+    .clampScalar(1, S32, S64);
+  // TODO: Scalarize
+
+  // TODO: Expand for > s32
+  getActionDefinitionsBuilder(G_BSWAP)
+    .legalFor({S32})
+    .clampScalar(0, S32, S32)
+    .scalarize(0);
+
+
+  auto smallerThan = [](unsigned TypeIdx0, unsigned TypeIdx1) {
+    return [=](const LegalityQuery &Query) {
+      return Query.Types[TypeIdx0].getSizeInBits() <
+             Query.Types[TypeIdx1].getSizeInBits();
+    };
+  };
 
-  setAction({G_CTLZ, S32}, Legal);
-  setAction({G_CTLZ_ZERO_UNDEF, S32}, Legal);
-  setAction({G_CTTZ, S32}, Legal);
-  setAction({G_CTTZ_ZERO_UNDEF, S32}, Legal);
-  setAction({G_BSWAP, S32}, Legal);
-  setAction({G_CTPOP, S32}, Legal);
+  auto greaterThan = [](unsigned TypeIdx0, unsigned TypeIdx1) {
+    return [=](const LegalityQuery &Query) {
+      return Query.Types[TypeIdx0].getSizeInBits() >
+             Query.Types[TypeIdx1].getSizeInBits();
+    };
+  };
 
   getActionDefinitionsBuilder(G_INTTOPTR)
-    .legalIf([](const LegalityQuery &Query) {
-      return true;
-    });
+    // List the common cases
+    .legalForCartesianProduct({GlobalPtr, ConstantPtr, FlatPtr}, {S64})
+    .legalForCartesianProduct({LocalPtr, PrivatePtr}, {S32})
+    .scalarize(0)
+    // Accept any address space as long as the size matches
+    .legalIf(sameSize(0, 1))
+    .widenScalarIf(smallerThan(1, 0),
+      [](const LegalityQuery &Query) {
+        return std::make_pair(1, LLT::scalar(Query.Types[0].getSizeInBits()));
+      })
+    .narrowScalarIf(greaterThan(1, 0),
+      [](const LegalityQuery &Query) {
+        return std::make_pair(1, LLT::scalar(Query.Types[0].getSizeInBits()));
+      });
 
   getActionDefinitionsBuilder(G_PTRTOINT)
-    .legalIf([](const LegalityQuery &Query) {
-      return true;
-    });
+    // List the common cases
+    .legalForCartesianProduct({GlobalPtr, ConstantPtr, FlatPtr}, {S64})
+    .legalForCartesianProduct({LocalPtr, PrivatePtr}, {S32})
+    .scalarize(0)
+    // Accept any address space as long as the size matches
+    .legalIf(sameSize(0, 1))
+    .widenScalarIf(smallerThan(0, 1),
+      [](const LegalityQuery &Query) {
+        return std::make_pair(0, LLT::scalar(Query.Types[1].getSizeInBits()));
+      })
+    .narrowScalarIf(
+      greaterThan(0, 1),
+      [](const LegalityQuery &Query) {
+        return std::make_pair(0, LLT::scalar(Query.Types[1].getSizeInBits()));
+      });
+
+  if (ST.hasFlatAddressSpace()) {
+    getActionDefinitionsBuilder(G_ADDRSPACE_CAST)
+      .scalarize(0)
+      .custom();
+  }
 
   getActionDefinitionsBuilder({G_LOAD, G_STORE})
+    .narrowScalarIf([](const LegalityQuery &Query) {
+        unsigned Size = Query.Types[0].getSizeInBits();
+        unsigned MemSize = Query.MMODescrs[0].SizeInBits;
+        return (Size > 32 && MemSize < Size);
+      },
+      [](const LegalityQuery &Query) {
+        return std::make_pair(0, LLT::scalar(32));
+      })
+    .fewerElementsIf([=, &ST](const LegalityQuery &Query) {
+        unsigned MemSize = Query.MMODescrs[0].SizeInBits;
+        return (MemSize == 96) &&
+               Query.Types[0].isVector() &&
+               ST.getGeneration() < AMDGPUSubtarget::SEA_ISLANDS;
+      },
+      [=](const LegalityQuery &Query) {
+        return std::make_pair(0, V2S32);
+      })
     .legalIf([=, &ST](const LegalityQuery &Query) {
         const LLT &Ty0 = Query.Types[0];
 
+        unsigned Size = Ty0.getSizeInBits();
+        unsigned MemSize = Query.MMODescrs[0].SizeInBits;
+        if (Size < 32 || (Size > 32 && MemSize < Size))
+          return false;
+
+        if (Ty0.isVector() && Size != MemSize)
+          return false;
+
         // TODO: Decompose private loads into 4-byte components.
         // TODO: Illegal flat loads on SI
-        switch (Ty0.getSizeInBits()) {
+        switch (MemSize) {
+        case 8:
+        case 16:
+          return Size == 32;
         case 32:
         case 64:
         case 128:
@@ -174,79 +401,394 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST,
         default:
           return false;
         }
-      });
-
-
-
-  setAction({G_SELECT, S32}, Legal);
-  setAction({G_SELECT, 1, S1}, Legal);
+      })
+    .clampScalar(0, S32, S64);
+
+
+  auto &ExtLoads = getActionDefinitionsBuilder({G_SEXTLOAD, G_ZEXTLOAD})
+    .legalForTypesWithMemSize({
+        {S32, GlobalPtr, 8},
+        {S32, GlobalPtr, 16},
+        {S32, LocalPtr, 8},
+        {S32, LocalPtr, 16},
+        {S32, PrivatePtr, 8},
+        {S32, PrivatePtr, 16}});
+  if (ST.hasFlatAddressSpace()) {
+    ExtLoads.legalForTypesWithMemSize({{S32, FlatPtr, 8},
+                                       {S32, FlatPtr, 16}});
+  }
 
-  setAction({G_SHL, S32}, Legal);
+  ExtLoads.clampScalar(0, S32, S32)
+          .widenScalarToNextPow2(0)
+          .unsupportedIfMemSizeNotPow2()
+          .lower();
+
+  auto &Atomics = getActionDefinitionsBuilder(
+    {G_ATOMICRMW_XCHG, G_ATOMICRMW_ADD, G_ATOMICRMW_SUB,
+     G_ATOMICRMW_AND, G_ATOMICRMW_OR, G_ATOMICRMW_XOR,
+     G_ATOMICRMW_MAX, G_ATOMICRMW_MIN, G_ATOMICRMW_UMAX,
+     G_ATOMICRMW_UMIN, G_ATOMIC_CMPXCHG})
+    .legalFor({{S32, GlobalPtr}, {S32, LocalPtr},
+               {S64, GlobalPtr}, {S64, LocalPtr}});
+  if (ST.hasFlatAddressSpace()) {
+    Atomics.legalFor({{S32, FlatPtr}, {S64, FlatPtr}});
+  }
 
+  // TODO: Pointer types, any 32-bit or 64-bit vector
+  getActionDefinitionsBuilder(G_SELECT)
+    .legalForCartesianProduct({S32, S64, V2S32, V2S16, V4S16,
+          GlobalPtr, LocalPtr, FlatPtr, PrivatePtr,
+          LLT::vector(2, LocalPtr), LLT::vector(2, PrivatePtr)}, {S1})
+    .clampScalar(0, S32, S64)
+    .fewerElementsIf(
+      [=](const LegalityQuery &Query) {
+        if (Query.Types[1].isVector())
+          return true;
 
-  // FIXME: When RegBankSelect inserts copies, it will only create new
-  // registers with scalar types.  This means we can end up with
-  // G_LOAD/G_STORE/G_GEP instruction with scalar types for their pointer
-  // operands.  In assert builds, the instruction selector will assert
-  // if it sees a generic instruction which isn't legal, so we need to
-  // tell it that scalar types are legal for pointer operands
-  setAction({G_GEP, S64}, Legal);
+        LLT Ty = Query.Types[0];
+
+        // FIXME: Hack until odd splits handled
+        return Ty.isVector() &&
+          (Ty.getScalarSizeInBits() > 32 || Ty.getNumElements() % 2 != 0);
+      },
+      scalarize(0))
+    // FIXME: Handle 16-bit vectors better
+    .fewerElementsIf(
+      [=](const LegalityQuery &Query) {
+        return Query.Types[0].isVector() &&
+               Query.Types[0].getElementType().getSizeInBits() < 32;},
+      scalarize(0))
+    .scalarize(1)
+    .clampMaxNumElements(0, S32, 2)
+    .clampMaxNumElements(0, LocalPtr, 2)
+    .clampMaxNumElements(0, PrivatePtr, 2)
+    .legalIf(all(isPointer(0), typeIs(1, S1)));
+
+  // TODO: Only the low 4/5/6 bits of the shift amount are observed, so we can
+  // be more flexible with the shift amount type.
+  auto &Shifts = getActionDefinitionsBuilder({G_SHL, G_LSHR, G_ASHR})
+    .legalFor({{S32, S32}, {S64, S32}});
+  if (ST.has16BitInsts()) {
+    if (ST.hasVOP3PInsts()) {
+      Shifts.legalFor({{S16, S32}, {S16, S16}, {V2S16, V2S16}})
+            .clampMaxNumElements(0, S16, 2);
+    } else
+      Shifts.legalFor({{S16, S32}, {S16, S16}});
+
+    Shifts.clampScalar(1, S16, S32);
+    Shifts.clampScalar(0, S16, S64);
+    Shifts.widenScalarToNextPow2(0, 16);
+  } else {
+    // Make sure we legalize the shift amount type first, as the general
+    // expansion for the shifted type will produce much worse code if it hasn't
+    // been truncated already.
+    Shifts.clampScalar(1, S32, S32);
+    Shifts.clampScalar(0, S32, S64);
+    Shifts.widenScalarToNextPow2(0, 32);
+  }
+  Shifts.scalarize(0);
 
   for (unsigned Op : {G_EXTRACT_VECTOR_ELT, G_INSERT_VECTOR_ELT}) {
+    unsigned VecTypeIdx = Op == G_EXTRACT_VECTOR_ELT ? 1 : 0;
+    unsigned EltTypeIdx = Op == G_EXTRACT_VECTOR_ELT ? 0 : 1;
+    unsigned IdxTypeIdx = 2;
+
     getActionDefinitionsBuilder(Op)
       .legalIf([=](const LegalityQuery &Query) {
-          const LLT &VecTy = Query.Types[1];
-          const LLT &IdxTy = Query.Types[2];
+          const LLT &VecTy = Query.Types[VecTypeIdx];
+          const LLT &IdxTy = Query.Types[IdxTypeIdx];
           return VecTy.getSizeInBits() % 32 == 0 &&
             VecTy.getSizeInBits() <= 512 &&
             IdxTy.getSizeInBits() == 32;
-        });
+        })
+      .clampScalar(EltTypeIdx, S32, S64)
+      .clampScalar(VecTypeIdx, S32, S64)
+      .clampScalar(IdxTypeIdx, S32, S32);
   }
 
+  getActionDefinitionsBuilder(G_EXTRACT_VECTOR_ELT)
+    .unsupportedIf([=](const LegalityQuery &Query) {
+        const LLT &EltTy = Query.Types[1].getElementType();
+        return Query.Types[0] != EltTy;
+      });
+
   // FIXME: Doesn't handle extract of illegal sizes.
   getActionDefinitionsBuilder({G_EXTRACT, G_INSERT})
-    .legalIf([=](const LegalityQuery &Query) {
+      .legalIf([=](const LegalityQuery &Query) {
         const LLT &Ty0 = Query.Types[0];
         const LLT &Ty1 = Query.Types[1];
-        return (Ty0.getSizeInBits() % 32 == 0) &&
-               (Ty1.getSizeInBits() % 32 == 0);
-      });
-
+        return (Ty0.getSizeInBits() % 16 == 0) &&
+               (Ty1.getSizeInBits() % 16 == 0);
+      })
+      .widenScalarIf(
+          [=](const LegalityQuery &Query) {
+            const LLT Ty1 = Query.Types[1];
+            return (Ty1.getScalarSizeInBits() < 16);
+          },
+          LegalizeMutations::widenScalarOrEltToNextPow2(1, 16));
+
+  // TODO: vectors of pointers
   getActionDefinitionsBuilder(G_BUILD_VECTOR)
+      .legalForCartesianProduct(AllS32Vectors, {S32})
+      .legalForCartesianProduct(AllS64Vectors, {S64})
+      .clampNumElements(0, V16S32, V16S32)
+      .clampNumElements(0, V2S64, V8S64)
+      .minScalarSameAs(1, 0)
+      // FIXME: Sort of a hack to make progress on other legalizations.
       .legalIf([=](const LegalityQuery &Query) {
-        const LLT &VecTy = Query.Types[0];
-        const LLT &ScalarTy = Query.Types[1];
-        return VecTy.getSizeInBits() % 32 == 0 &&
-               ScalarTy.getSizeInBits() % 32 == 0 &&
-               VecTy.getSizeInBits() <= 512;
+        return Query.Types[0].getScalarSizeInBits() <= 32 ||
+               Query.Types[0].getScalarSizeInBits() == 64;
       });
+
+  // TODO: Support any combination of v2s32
+  getActionDefinitionsBuilder(G_CONCAT_VECTORS)
+    .legalFor({{V4S32, V2S32},
+               {V8S32, V2S32},
+               {V8S32, V4S32},
+               {V4S64, V2S64},
+               {V4S16, V2S16},
+               {V8S16, V2S16},
+               {V8S16, V4S16},
+               {LLT::vector(4, LocalPtr), LLT::vector(2, LocalPtr)},
+               {LLT::vector(4, PrivatePtr), LLT::vector(2, PrivatePtr)}});
+
   // Merge/Unmerge
   for (unsigned Op : {G_MERGE_VALUES, G_UNMERGE_VALUES}) {
     unsigned BigTyIdx = Op == G_MERGE_VALUES ? 0 : 1;
     unsigned LitTyIdx = Op == G_MERGE_VALUES ? 1 : 0;
 
+    auto notValidElt = [=](const LegalityQuery &Query, unsigned TypeIdx) {
+      const LLT &Ty = Query.Types[TypeIdx];
+      if (Ty.isVector()) {
+        const LLT &EltTy = Ty.getElementType();
+        if (EltTy.getSizeInBits() < 8 || EltTy.getSizeInBits() > 64)
+          return true;
+        if (!isPowerOf2_32(EltTy.getSizeInBits()))
+          return true;
+      }
+      return false;
+    };
+
     getActionDefinitionsBuilder(Op)
+      .widenScalarToNextPow2(LitTyIdx, /*Min*/ 16)
+      // Clamp the little scalar to s8-s256 and make it a power of 2. It's not
+      // worth considering the multiples of 64 since 2*192 and 2*384 are not
+      // valid.
+      .clampScalar(LitTyIdx, S16, S256)
+      .widenScalarToNextPow2(LitTyIdx, /*Min*/ 32)
+
+      // Break up vectors with weird elements into scalars
+      .fewerElementsIf(
+        [=](const LegalityQuery &Query) { return notValidElt(Query, 0); },
+        scalarize(0))
+      .fewerElementsIf(
+        [=](const LegalityQuery &Query) { return notValidElt(Query, 1); },
+        scalarize(1))
+      .clampScalar(BigTyIdx, S32, S512)
+      .widenScalarIf(
+        [=](const LegalityQuery &Query) {
+          const LLT &Ty = Query.Types[BigTyIdx];
+          return !isPowerOf2_32(Ty.getSizeInBits()) &&
+                 Ty.getSizeInBits() % 16 != 0;
+        },
+        [=](const LegalityQuery &Query) {
+          // Pick the next power of 2, or a multiple of 64 over 128.
+          // Whichever is smaller.
+          const LLT &Ty = Query.Types[BigTyIdx];
+          unsigned NewSizeInBits = 1 << Log2_32_Ceil(Ty.getSizeInBits() + 1);
+          if (NewSizeInBits >= 256) {
+            unsigned RoundedTo = alignTo<64>(Ty.getSizeInBits() + 1);
+            if (RoundedTo < NewSizeInBits)
+              NewSizeInBits = RoundedTo;
+          }
+          return std::make_pair(BigTyIdx, LLT::scalar(NewSizeInBits));
+        })
       .legalIf([=](const LegalityQuery &Query) {
           const LLT &BigTy = Query.Types[BigTyIdx];
           const LLT &LitTy = Query.Types[LitTyIdx];
-          return BigTy.getSizeInBits() % 32 == 0 &&
-                 LitTy.getSizeInBits() % 32 == 0 &&
+
+          if (BigTy.isVector() && BigTy.getSizeInBits() < 32)
+            return false;
+          if (LitTy.isVector() && LitTy.getSizeInBits() < 32)
+            return false;
+
+          return BigTy.getSizeInBits() % 16 == 0 &&
+                 LitTy.getSizeInBits() % 16 == 0 &&
                  BigTy.getSizeInBits() <= 512;
         })
       // Any vectors left are the wrong size. Scalarize them.
-      .fewerElementsIf([](const LegalityQuery &Query) { return true; },
-                       [](const LegalityQuery &Query) {
-                         return std::make_pair(
-                           0, Query.Types[0].getElementType());
-                       })
-      .fewerElementsIf([](const LegalityQuery &Query) { return true; },
-                       [](const LegalityQuery &Query) {
-                         return std::make_pair(
-                           1, Query.Types[1].getElementType());
-                       });
-
+      .scalarize(0)
+      .scalarize(1);
   }
 
   computeTables();
   verify(*ST.getInstrInfo());
 }
+
+bool AMDGPULegalizerInfo::legalizeCustom(MachineInstr &MI,
+                                         MachineRegisterInfo &MRI,
+                                         MachineIRBuilder &MIRBuilder,
+                                         GISelChangeObserver &Observer) const {
+  switch (MI.getOpcode()) {
+  case TargetOpcode::G_ADDRSPACE_CAST:
+    return legalizeAddrSpaceCast(MI, MRI, MIRBuilder);
+  default:
+    return false;
+  }
+
+  llvm_unreachable("expected switch to return");
+}
+
+unsigned AMDGPULegalizerInfo::getSegmentAperture(
+  unsigned AS,
+  MachineRegisterInfo &MRI,
+  MachineIRBuilder &MIRBuilder) const {
+  MachineFunction &MF = MIRBuilder.getMF();
+  const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
+  const LLT S32 = LLT::scalar(32);
+
+  if (ST.hasApertureRegs()) {
+    // FIXME: Use inline constants (src_{shared, private}_base) instead of
+    // getreg.
+    unsigned Offset = AS == AMDGPUAS::LOCAL_ADDRESS ?
+        AMDGPU::Hwreg::OFFSET_SRC_SHARED_BASE :
+        AMDGPU::Hwreg::OFFSET_SRC_PRIVATE_BASE;
+    unsigned WidthM1 = AS == AMDGPUAS::LOCAL_ADDRESS ?
+        AMDGPU::Hwreg::WIDTH_M1_SRC_SHARED_BASE :
+        AMDGPU::Hwreg::WIDTH_M1_SRC_PRIVATE_BASE;
+    unsigned Encoding =
+        AMDGPU::Hwreg::ID_MEM_BASES << AMDGPU::Hwreg::ID_SHIFT_ |
+        Offset << AMDGPU::Hwreg::OFFSET_SHIFT_ |
+        WidthM1 << AMDGPU::Hwreg::WIDTH_M1_SHIFT_;
+
+    unsigned ShiftAmt = MRI.createGenericVirtualRegister(S32);
+    unsigned ApertureReg = MRI.createGenericVirtualRegister(S32);
+    unsigned GetReg = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
+
+    MIRBuilder.buildInstr(AMDGPU::S_GETREG_B32)
+      .addDef(GetReg)
+      .addImm(Encoding);
+    MRI.setType(GetReg, S32);
+
+    MIRBuilder.buildConstant(ShiftAmt, WidthM1 + 1);
+    MIRBuilder.buildInstr(TargetOpcode::G_SHL)
+      .addDef(ApertureReg)
+      .addUse(GetReg)
+      .addUse(ShiftAmt);
+
+    return ApertureReg;
+  }
+
+  unsigned QueuePtr = MRI.createGenericVirtualRegister(
+    LLT::pointer(AMDGPUAS::CONSTANT_ADDRESS, 64));
+
+  // FIXME: Placeholder until we can track the input registers.
+  MIRBuilder.buildConstant(QueuePtr, 0xdeadbeef);
+
+  // Offset into amd_queue_t for group_segment_aperture_base_hi /
+  // private_segment_aperture_base_hi.
+  uint32_t StructOffset = (AS == AMDGPUAS::LOCAL_ADDRESS) ? 0x40 : 0x44;
+
+  // FIXME: Don't use undef
+  Value *V = UndefValue::get(PointerType::get(
+                               Type::getInt8Ty(MF.getFunction().getContext()),
+                               AMDGPUAS::CONSTANT_ADDRESS));
+
+  MachinePointerInfo PtrInfo(V, StructOffset);
+  MachineMemOperand *MMO = MF.getMachineMemOperand(
+    PtrInfo,
+    MachineMemOperand::MOLoad |
+    MachineMemOperand::MODereferenceable |
+    MachineMemOperand::MOInvariant,
+    4,
+    MinAlign(64, StructOffset));
+
+  unsigned LoadResult = MRI.createGenericVirtualRegister(S32);
+  unsigned LoadAddr = AMDGPU::NoRegister;
+
+  MIRBuilder.materializeGEP(LoadAddr, QueuePtr, LLT::scalar(64), StructOffset);
+  MIRBuilder.buildLoad(LoadResult, LoadAddr, *MMO);
+  return LoadResult;
+}
+
+bool AMDGPULegalizerInfo::legalizeAddrSpaceCast(
+  MachineInstr &MI, MachineRegisterInfo &MRI,
+  MachineIRBuilder &MIRBuilder) const {
+  MachineFunction &MF = MIRBuilder.getMF();
+
+  MIRBuilder.setInstr(MI);
+
+  unsigned Dst = MI.getOperand(0).getReg();
+  unsigned Src = MI.getOperand(1).getReg();
+
+  LLT DstTy = MRI.getType(Dst);
+  LLT SrcTy = MRI.getType(Src);
+  unsigned DestAS = DstTy.getAddressSpace();
+  unsigned SrcAS = SrcTy.getAddressSpace();
+
+  // TODO: Avoid reloading from the queue ptr for each cast, or at least each
+  // vector element.
+  assert(!DstTy.isVector());
+
+  const AMDGPUTargetMachine &TM
+    = static_cast<const AMDGPUTargetMachine &>(MF.getTarget());
+
+  const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
+  if (ST.getTargetLowering()->isNoopAddrSpaceCast(SrcAS, DestAS)) {
+    MI.setDesc(MIRBuilder.getTII().get(TargetOpcode::G_BITCAST));
+    return true;
+  }
+
+  if (SrcAS == AMDGPUAS::FLAT_ADDRESS) {
+    assert(DestAS == AMDGPUAS::LOCAL_ADDRESS ||
+           DestAS == AMDGPUAS::PRIVATE_ADDRESS);
+    unsigned NullVal = TM.getNullPointerValue(DestAS);
+
+    unsigned SegmentNullReg = MRI.createGenericVirtualRegister(DstTy);
+    unsigned FlatNullReg = MRI.createGenericVirtualRegister(SrcTy);
+
+    MIRBuilder.buildConstant(SegmentNullReg, NullVal);
+    MIRBuilder.buildConstant(FlatNullReg, 0);
+
+    unsigned PtrLo32 = MRI.createGenericVirtualRegister(DstTy);
+
+    // Extract low 32-bits of the pointer.
+    MIRBuilder.buildExtract(PtrLo32, Src, 0);
+
+    unsigned CmpRes = MRI.createGenericVirtualRegister(LLT::scalar(1));
+    MIRBuilder.buildICmp(CmpInst::ICMP_NE, CmpRes, Src, FlatNullReg);
+    MIRBuilder.buildSelect(Dst, CmpRes, PtrLo32, SegmentNullReg);
+
+    MI.eraseFromParent();
+    return true;
+  }
+
+  assert(SrcAS == AMDGPUAS::LOCAL_ADDRESS ||
+         SrcAS == AMDGPUAS::PRIVATE_ADDRESS);
+
+  unsigned FlatNullReg = MRI.createGenericVirtualRegister(DstTy);
+  unsigned SegmentNullReg = MRI.createGenericVirtualRegister(SrcTy);
+  MIRBuilder.buildConstant(SegmentNullReg, TM.getNullPointerValue(SrcAS));
+  MIRBuilder.buildConstant(FlatNullReg, TM.getNullPointerValue(DestAS));
+
+  unsigned ApertureReg = getSegmentAperture(DestAS, MRI, MIRBuilder);
+
+  unsigned CmpRes = MRI.createGenericVirtualRegister(LLT::scalar(1));
+  MIRBuilder.buildICmp(CmpInst::ICMP_NE, CmpRes, Src, SegmentNullReg);
+
+  unsigned BuildPtr = MRI.createGenericVirtualRegister(DstTy);
+
+  // Coerce the type of the low half of the result so we can use merge_values.
+  unsigned SrcAsInt = MRI.createGenericVirtualRegister(LLT::scalar(32));
+  MIRBuilder.buildInstr(TargetOpcode::G_PTRTOINT)
+    .addDef(SrcAsInt)
+    .addUse(Src);
+
+  // TODO: Should we allow mismatched types but matching sizes in merges to
+  // avoid the ptrtoint?
+  MIRBuilder.buildMerge(BuildPtr, {SrcAsInt, ApertureReg});
+  MIRBuilder.buildSelect(Dst, CmpRes, BuildPtr, FlatNullReg);
+
+  MI.eraseFromParent();
+  return true;
+}
diff --git a/lib/Target/AMDGPU/AMDGPULegalizerInfo.h b/lib/Target/AMDGPU/AMDGPULegalizerInfo.h
index 1cbd37c42c4b30d454626c56878d993745246d8d..65fb9caa4904a5d5c4adea6ab7ef224f8142ea2f 100644
--- a/lib/Target/AMDGPU/AMDGPULegalizerInfo.h
+++ b/lib/Target/AMDGPU/AMDGPULegalizerInfo.h
@@ -1,9 +1,8 @@
 //===- AMDGPULegalizerInfo ---------------------------------------*- C++ -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
@@ -28,6 +27,17 @@ class AMDGPULegalizerInfo : public LegalizerInfo {
 public:
   AMDGPULegalizerInfo(const GCNSubtarget &ST,
                       const GCNTargetMachine &TM);
+
+  bool legalizeCustom(MachineInstr &MI, MachineRegisterInfo &MRI,
+                      MachineIRBuilder &MIRBuilder,
+                      GISelChangeObserver &Observer) const override;
+
+  unsigned getSegmentAperture(unsigned AddrSpace,
+                              MachineRegisterInfo &MRI,
+                              MachineIRBuilder &MIRBuilder) const;
+
+  bool legalizeAddrSpaceCast(MachineInstr &MI, MachineRegisterInfo &MRI,
+                             MachineIRBuilder &MIRBuilder) const;
 };
 } // End llvm namespace.
 #endif
diff --git a/lib/Target/AMDGPU/AMDGPULibCalls.cpp b/lib/Target/AMDGPU/AMDGPULibCalls.cpp
index 14e8800426911aa5f24454217aa0201a3525d3a8..581e229b4a0954b842c35965eb75c3173195cf32 100644
--- a/lib/Target/AMDGPU/AMDGPULibCalls.cpp
+++ b/lib/Target/AMDGPU/AMDGPULibCalls.cpp
@@ -1,9 +1,8 @@
 //===- AMDGPULibCalls.cpp -------------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -73,7 +72,7 @@ private:
 
   // Return a pointer (pointer expr) to the function if function defintion with
   // "FuncName" exists. It may create a new function prototype in pre-link mode.
-  Constant *getFunction(Module *M, const FuncInfo& fInfo);
+  FunctionCallee getFunction(Module *M, const FuncInfo &fInfo);
 
   // Replace a normal function with its native version.
   bool replaceWithNative(CallInst *CI, const FuncInfo &FInfo);
@@ -140,7 +139,7 @@ private:
   // Insert an Alloc instruction.
   AllocaInst* insertAlloca(CallInst * UI, IRBuilder<> &B, const char *prefix);
   // Get a scalar native builtin signle argument FP function
-  Constant* getNativeFunction(Module* M, const FuncInfo &FInfo);
+  FunctionCallee getNativeFunction(Module *M, const FuncInfo &FInfo);
 
 protected:
   CallInst *CI;
@@ -217,19 +216,19 @@ INITIALIZE_PASS(AMDGPUUseNativeCalls, "amdgpu-usenative",
                 false, false)
 
 template <typename IRB>
-static CallInst *CreateCallEx(IRB &B, Value *Callee, Value *Arg,
+static CallInst *CreateCallEx(IRB &B, FunctionCallee Callee, Value *Arg,
                               const Twine &Name = "") {
   CallInst *R = B.CreateCall(Callee, Arg, Name);
-  if (Function* F = dyn_cast<Function>(Callee))
+  if (Function *F = dyn_cast<Function>(Callee.getCallee()))
     R->setCallingConv(F->getCallingConv());
   return R;
 }
 
 template <typename IRB>
-static CallInst *CreateCallEx2(IRB &B, Value *Callee, Value *Arg1, Value *Arg2,
-                               const Twine &Name = "") {
+static CallInst *CreateCallEx2(IRB &B, FunctionCallee Callee, Value *Arg1,
+                               Value *Arg2, const Twine &Name = "") {
   CallInst *R = B.CreateCall(Callee, {Arg1, Arg2}, Name);
-  if (Function* F = dyn_cast<Function>(Callee))
+  if (Function *F = dyn_cast<Function>(Callee.getCallee()))
     R->setCallingConv(F->getCallingConv());
   return R;
 }
@@ -472,7 +471,7 @@ static inline AMDGPULibFunc::EType getArgType(const AMDGPULibFunc& FInfo) {
   return (AMDGPULibFunc::EType)FInfo.getLeads()[0].ArgType;
 }
 
-Constant *AMDGPULibCalls::getFunction(Module *M, const FuncInfo& fInfo) {
+FunctionCallee AMDGPULibCalls::getFunction(Module *M, const FuncInfo &fInfo) {
   // If we are doing PreLinkOpt, the function is external. So it is safe to
   // use getOrInsertFunction() at this stage.
 
@@ -519,11 +518,11 @@ bool AMDGPULibCalls::sincosUseNative(CallInst *aCI, const FuncInfo &FInfo) {
 
     nf.setPrefix(AMDGPULibFunc::NATIVE);
     nf.setId(AMDGPULibFunc::EI_SIN);
-    Constant *sinExpr = getFunction(M, nf);
+    FunctionCallee sinExpr = getFunction(M, nf);
 
     nf.setPrefix(AMDGPULibFunc::NATIVE);
     nf.setId(AMDGPULibFunc::EI_COS);
-    Constant *cosExpr = getFunction(M, nf);
+    FunctionCallee cosExpr = getFunction(M, nf);
     if (sinExpr && cosExpr) {
       Value *sinval = CallInst::Create(sinExpr, opr0, "splitsin", aCI);
       Value *cosval = CallInst::Create(cosExpr, opr0, "splitcos", aCI);
@@ -555,7 +554,7 @@ bool AMDGPULibCalls::useNative(CallInst *aCI) {
     return sincosUseNative(aCI, FInfo);
 
   FInfo.setPrefix(AMDGPULibFunc::NATIVE);
-  Constant *F = getFunction(aCI->getModule(), FInfo);
+  FunctionCallee F = getFunction(aCI->getModule(), FInfo);
   if (!F)
     return false;
 
@@ -613,7 +612,7 @@ bool AMDGPULibCalls::fold_read_write_pipe(CallInst *CI, IRBuilder<> &B,
   auto *FTy = FunctionType::get(Callee->getReturnType(),
                                 ArrayRef<Type *>(ArgTys), false);
   AMDGPULibFunc NewLibFunc(Name, FTy);
-  auto *F = AMDGPULibFunc::getOrInsertFunction(M, NewLibFunc);
+  FunctionCallee F = AMDGPULibFunc::getOrInsertFunction(M, NewLibFunc);
   if (!F)
     return false;
 
@@ -795,7 +794,7 @@ bool AMDGPULibCalls::replaceWithNative(CallInst *CI, const FuncInfo &FInfo) {
 
   AMDGPULibFunc nf = FInfo;
   nf.setPrefix(AMDGPULibFunc::NATIVE);
-  if (Constant *FPExpr = getFunction(M, nf)) {
+  if (FunctionCallee FPExpr = getFunction(M, nf)) {
     LLVM_DEBUG(dbgs() << "AMDIC: " << *CI << " ---> ");
 
     CI->setCalledFunction(FPExpr);
@@ -934,9 +933,10 @@ bool AMDGPULibCalls::fold_pow(CallInst *CI, IRBuilder<> &B,
   if (CF && (CF->isExactlyValue(0.5) || CF->isExactlyValue(-0.5))) {
     // pow[r](x, [-]0.5) = sqrt(x)
     bool issqrt = CF->isExactlyValue(0.5);
-    if (Constant *FPExpr = getFunction(M,
-        AMDGPULibFunc(issqrt ? AMDGPULibFunc::EI_SQRT
-                             : AMDGPULibFunc::EI_RSQRT, FInfo))) {
+    if (FunctionCallee FPExpr =
+            getFunction(M, AMDGPULibFunc(issqrt ? AMDGPULibFunc::EI_SQRT
+                                                : AMDGPULibFunc::EI_RSQRT,
+                                         FInfo))) {
       LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> "
                         << FInfo.getName().c_str() << "(" << *opr0 << ")\n");
       Value *nval = CreateCallEx(B,FPExpr, opr0, issqrt ? "__pow2sqrt"
@@ -1003,8 +1003,8 @@ bool AMDGPULibCalls::fold_pow(CallInst *CI, IRBuilder<> &B,
 
   // powr ---> exp2(y * log2(x))
   // pown/pow ---> powr(fabs(x), y) | (x & ((int)y << 31))
-  Constant *ExpExpr = getFunction(M, AMDGPULibFunc(AMDGPULibFunc::EI_EXP2,
-                                                   FInfo));
+  FunctionCallee ExpExpr =
+      getFunction(M, AMDGPULibFunc(AMDGPULibFunc::EI_EXP2, FInfo));
   if (!ExpExpr)
     return false;
 
@@ -1090,8 +1090,8 @@ bool AMDGPULibCalls::fold_pow(CallInst *CI, IRBuilder<> &B,
 
   Value *nval;
   if (needabs) {
-    Constant *AbsExpr = getFunction(M, AMDGPULibFunc(AMDGPULibFunc::EI_FABS,
-                                                     FInfo));
+    FunctionCallee AbsExpr =
+        getFunction(M, AMDGPULibFunc(AMDGPULibFunc::EI_FABS, FInfo));
     if (!AbsExpr)
       return false;
     nval = CreateCallEx(B, AbsExpr, opr0, "__fabs");
@@ -1099,8 +1099,8 @@ bool AMDGPULibCalls::fold_pow(CallInst *CI, IRBuilder<> &B,
     nval = cnval ? cnval : opr0;
   }
   if (needlog) {
-    Constant *LogExpr = getFunction(M, AMDGPULibFunc(AMDGPULibFunc::EI_LOG2,
-                                                     FInfo));
+    FunctionCallee LogExpr =
+        getFunction(M, AMDGPULibFunc(AMDGPULibFunc::EI_LOG2, FInfo));
     if (!LogExpr)
       return false;
     nval = CreateCallEx(B,LogExpr, nval, "__log2");
@@ -1159,8 +1159,8 @@ bool AMDGPULibCalls::fold_rootn(CallInst *CI, IRBuilder<> &B,
     std::vector<const Type*> ParamsTys;
     ParamsTys.push_back(opr0->getType());
     Module *M = CI->getModule();
-    if (Constant *FPExpr = getFunction(M, AMDGPULibFunc(AMDGPULibFunc::EI_SQRT,
-                                                        FInfo))) {
+    if (FunctionCallee FPExpr =
+            getFunction(M, AMDGPULibFunc(AMDGPULibFunc::EI_SQRT, FInfo))) {
       LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> sqrt(" << *opr0 << ")\n");
       Value *nval = CreateCallEx(B,FPExpr, opr0, "__rootn2sqrt");
       replaceCall(nval);
@@ -1168,8 +1168,8 @@ bool AMDGPULibCalls::fold_rootn(CallInst *CI, IRBuilder<> &B,
     }
   } else if (ci_opr1 == 3) { // rootn(x, 3) = cbrt(x)
     Module *M = CI->getModule();
-    if (Constant *FPExpr = getFunction(M, AMDGPULibFunc(AMDGPULibFunc::EI_CBRT,
-                                                        FInfo))) {
+    if (FunctionCallee FPExpr =
+            getFunction(M, AMDGPULibFunc(AMDGPULibFunc::EI_CBRT, FInfo))) {
       LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> cbrt(" << *opr0 << ")\n");
       Value *nval = CreateCallEx(B,FPExpr, opr0, "__rootn2cbrt");
       replaceCall(nval);
@@ -1186,8 +1186,8 @@ bool AMDGPULibCalls::fold_rootn(CallInst *CI, IRBuilder<> &B,
     std::vector<const Type*> ParamsTys;
     ParamsTys.push_back(opr0->getType());
     Module *M = CI->getModule();
-    if (Constant *FPExpr = getFunction(M, AMDGPULibFunc(AMDGPULibFunc::EI_RSQRT,
-                                                        FInfo))) {
+    if (FunctionCallee FPExpr =
+            getFunction(M, AMDGPULibFunc(AMDGPULibFunc::EI_RSQRT, FInfo))) {
       LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> rsqrt(" << *opr0
                         << ")\n");
       Value *nval = CreateCallEx(B,FPExpr, opr0, "__rootn2rsqrt");
@@ -1243,7 +1243,8 @@ bool AMDGPULibCalls::fold_fma_mad(CallInst *CI, IRBuilder<> &B,
 }
 
 // Get a scalar native builtin signle argument FP function
-Constant* AMDGPULibCalls::getNativeFunction(Module* M, const FuncInfo& FInfo) {
+FunctionCallee AMDGPULibCalls::getNativeFunction(Module *M,
+                                                 const FuncInfo &FInfo) {
   if (getArgType(FInfo) == AMDGPULibFunc::F64 || !HasNative(FInfo.getId()))
     return nullptr;
   FuncInfo nf = FInfo;
@@ -1256,8 +1257,8 @@ bool AMDGPULibCalls::fold_sqrt(CallInst *CI, IRBuilder<> &B,
                                const FuncInfo &FInfo) {
   if (getArgType(FInfo) == AMDGPULibFunc::F32 && (getVecSize(FInfo) == 1) &&
       (FInfo.getPrefix() != AMDGPULibFunc::NATIVE)) {
-    if (Constant *FPExpr = getNativeFunction(
-        CI->getModule(), AMDGPULibFunc(AMDGPULibFunc::EI_SQRT, FInfo))) {
+    if (FunctionCallee FPExpr = getNativeFunction(
+            CI->getModule(), AMDGPULibFunc(AMDGPULibFunc::EI_SQRT, FInfo))) {
       Value *opr0 = CI->getArgOperand(0);
       LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> "
                         << "sqrt(" << *opr0 << ")\n");
@@ -1334,7 +1335,7 @@ bool AMDGPULibCalls::fold_sincos(CallInst *CI, IRBuilder<> &B,
   // function.
   AMDGPULibFunc nf(AMDGPULibFunc::EI_SINCOS, fInfo);
   nf.getLeads()[0].PtrKind = AMDGPULibFunc::getEPtrKindFromAddrSpace(AMDGPUAS::FLAT_ADDRESS);
-  Function *Fsincos = dyn_cast_or_null<Function>(getFunction(M, nf));
+  FunctionCallee Fsincos = getFunction(M, nf);
   if (!Fsincos) return false;
 
   BasicBlock::iterator ItOld = B.GetInsertPoint();
@@ -1342,7 +1343,7 @@ bool AMDGPULibCalls::fold_sincos(CallInst *CI, IRBuilder<> &B,
   B.SetInsertPoint(UI);
 
   Value *P = Alloc;
-  Type *PTy = Fsincos->getFunctionType()->getParamType(1);
+  Type *PTy = Fsincos.getFunctionType()->getParamType(1);
   // The allocaInst allocates the memory in private address space. This need
   // to be bitcasted to point to the address space of cos pointer type.
   // In OpenCL 2.0 this is generic, while in 1.2 that is private.
@@ -1356,12 +1357,12 @@ bool AMDGPULibCalls::fold_sincos(CallInst *CI, IRBuilder<> &B,
   if (!isSin) { // CI->cos, UI->sin
     B.SetInsertPoint(&*ItOld);
     UI->replaceAllUsesWith(&*Call);
-    Instruction *Reload = B.CreateLoad(Alloc);
+    Instruction *Reload = B.CreateLoad(Alloc->getAllocatedType(), Alloc);
     CI->replaceAllUsesWith(Reload);
     UI->eraseFromParent();
     CI->eraseFromParent();
   } else { // CI->sin, UI->cos
-    Instruction *Reload = B.CreateLoad(Alloc);
+    Instruction *Reload = B.CreateLoad(Alloc->getAllocatedType(), Alloc);
     UI->replaceAllUsesWith(Reload);
     CI->replaceAllUsesWith(Call);
     UI->eraseFromParent();
diff --git a/lib/Target/AMDGPU/AMDGPULibFunc.cpp b/lib/Target/AMDGPU/AMDGPULibFunc.cpp
index 4fc3fe0f105b0823a4a5d926ce9576950f87a74e..09322594dc8e0a896adcdc0e910a0e10c734ced2 100644
--- a/lib/Target/AMDGPU/AMDGPULibFunc.cpp
+++ b/lib/Target/AMDGPU/AMDGPULibFunc.cpp
@@ -1,9 +1,8 @@
 //===-- AMDGPULibFunc.cpp -------------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -961,8 +960,8 @@ Function *AMDGPULibFunc::getFunction(Module *M, const AMDGPULibFunc &fInfo) {
   return nullptr;
 }
 
-Function *AMDGPULibFunc::getOrInsertFunction(Module *M,
-                                             const AMDGPULibFunc &fInfo) {
+FunctionCallee AMDGPULibFunc::getOrInsertFunction(Module *M,
+                                                  const AMDGPULibFunc &fInfo) {
   std::string const FuncName = fInfo.mangle();
   Function *F = dyn_cast_or_null<Function>(
     M->getValueSymbolTable().lookup(FuncName));
@@ -988,7 +987,7 @@ Function *AMDGPULibFunc::getOrInsertFunction(Module *M,
     }
   }
 
-  Constant *C = nullptr;
+  FunctionCallee C;
   if (hasPtr) {
     // Do not set extra attributes for functions with pointer arguments.
     C = M->getOrInsertFunction(FuncName, FuncTy);
@@ -1002,7 +1001,7 @@ Function *AMDGPULibFunc::getOrInsertFunction(Module *M,
     C = M->getOrInsertFunction(FuncName, FuncTy, Attr);
   }
 
-  return cast<Function>(C);
+  return C;
 }
 
 bool UnmangledFuncInfo::lookup(StringRef Name, ID &Id) {
diff --git a/lib/Target/AMDGPU/AMDGPULibFunc.h b/lib/Target/AMDGPU/AMDGPULibFunc.h
index fe062384800a7eb0f45b8e20793c3dc5fbc75424..2354ed7df2059b55bc256c31046b173db4790a14 100644
--- a/lib/Target/AMDGPU/AMDGPULibFunc.h
+++ b/lib/Target/AMDGPU/AMDGPULibFunc.h
@@ -1,9 +1,8 @@
 //===-- AMDGPULibFunc.h ----------------------------------------*- C++ -*--===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -394,8 +393,8 @@ public:
   }
   static Function *getFunction(llvm::Module *M, const AMDGPULibFunc &fInfo);
 
-  static Function *getOrInsertFunction(llvm::Module *M,
-                                       const AMDGPULibFunc &fInfo);
+  static FunctionCallee getOrInsertFunction(llvm::Module *M,
+                                            const AMDGPULibFunc &fInfo);
   static bool parse(StringRef MangledName, AMDGPULibFunc &Ptr);
 
 private:
diff --git a/lib/Target/AMDGPU/AMDGPULowerIntrinsics.cpp b/lib/Target/AMDGPU/AMDGPULowerIntrinsics.cpp
index 2cec8fe532833df9aea3deb7e16201d523fbc348..15032969890e2bfc3f8daf4d78a5108b32965e9f 100644
--- a/lib/Target/AMDGPU/AMDGPULowerIntrinsics.cpp
+++ b/lib/Target/AMDGPU/AMDGPULowerIntrinsics.cpp
@@ -1,9 +1,8 @@
 //===-- AMDGPULowerIntrinsics.cpp -----------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/AMDGPU/AMDGPULowerKernelArguments.cpp b/lib/Target/AMDGPU/AMDGPULowerKernelArguments.cpp
index 743dc7a0d00b97fb3b119cb6502687fe35aaafa8..b1abe1001e98833d64a992c81542bd16eb05f814 100644
--- a/lib/Target/AMDGPU/AMDGPULowerKernelArguments.cpp
+++ b/lib/Target/AMDGPU/AMDGPULowerKernelArguments.cpp
@@ -1,9 +1,8 @@
 //===-- AMDGPULowerKernelArguments.cpp ------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -110,7 +109,8 @@ bool AMDGPULowerKernelArguments::runOnFunction(Function &F) {
       // modes on SI to know the high bits are 0 so pointer adds don't wrap. We
       // can't represent this with range metadata because it's only allowed for
       // integer types.
-      if (PT->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS &&
+      if ((PT->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS ||
+           PT->getAddressSpace() == AMDGPUAS::REGION_ADDRESS) &&
           ST.getGeneration() == AMDGPUSubtarget::SOUTHERN_ISLANDS)
         continue;
 
@@ -132,6 +132,7 @@ bool AMDGPULowerKernelArguments::runOnFunction(Function &F) {
                                       KernArgBaseAlign);
 
     Value *ArgPtr;
+    Type *AdjustedArgTy;
     if (DoShiftOpt) { // FIXME: Handle aggregate types
       // Since we don't have sub-dword scalar loads, avoid doing an extload by
       // loading earlier than the argument address, and extracting the relevant
@@ -139,30 +140,27 @@ bool AMDGPULowerKernelArguments::runOnFunction(Function &F) {
       //
       // Additionally widen any sub-dword load to i32 even if suitably aligned,
       // so that CSE between different argument loads works easily.
-
       ArgPtr = Builder.CreateConstInBoundsGEP1_64(
-        KernArgSegment,
-        AlignDownOffset,
-        Arg.getName() + ".kernarg.offset.align.down");
-      ArgPtr = Builder.CreateBitCast(ArgPtr,
-                                     Builder.getInt32Ty()->getPointerTo(AS),
-                                     ArgPtr->getName() + ".cast");
+          Builder.getInt8Ty(), KernArgSegment, AlignDownOffset,
+          Arg.getName() + ".kernarg.offset.align.down");
+      AdjustedArgTy = Builder.getInt32Ty();
     } else {
       ArgPtr = Builder.CreateConstInBoundsGEP1_64(
-        KernArgSegment,
-        EltOffset,
-        Arg.getName() + ".kernarg.offset");
-      ArgPtr = Builder.CreateBitCast(ArgPtr, ArgTy->getPointerTo(AS),
-                                     ArgPtr->getName() + ".cast");
+          Builder.getInt8Ty(), KernArgSegment, EltOffset,
+          Arg.getName() + ".kernarg.offset");
+      AdjustedArgTy = ArgTy;
     }
 
     if (IsV3 && Size >= 32) {
       V4Ty = VectorType::get(VT->getVectorElementType(), 4);
       // Use the hack that clang uses to avoid SelectionDAG ruining v3 loads
-      ArgPtr = Builder.CreateBitCast(ArgPtr, V4Ty->getPointerTo(AS));
+      AdjustedArgTy = V4Ty;
     }
 
-    LoadInst *Load = Builder.CreateAlignedLoad(ArgPtr, AdjustedAlign);
+    ArgPtr = Builder.CreateBitCast(ArgPtr, AdjustedArgTy->getPointerTo(AS),
+                                   ArgPtr->getName() + ".cast");
+    LoadInst *Load =
+        Builder.CreateAlignedLoad(AdjustedArgTy, ArgPtr, AdjustedAlign);
     Load->setMetadata(LLVMContext::MD_invariant_load, MDNode::get(Ctx, {}));
 
     MDBuilder MDB(Ctx);
diff --git a/lib/Target/AMDGPU/AMDGPULowerKernelAttributes.cpp b/lib/Target/AMDGPU/AMDGPULowerKernelAttributes.cpp
index a43dcef4cf0b9beeec9a29f8268c5b7c6e217000..00e12f808783fa7c56a5bfb168bfbeff269b2aa4 100644
--- a/lib/Target/AMDGPU/AMDGPULowerKernelAttributes.cpp
+++ b/lib/Target/AMDGPU/AMDGPULowerKernelAttributes.cpp
@@ -1,9 +1,8 @@
 //===-- AMDGPULowerKernelAttributes.cpp ------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AMDGPU/AMDGPUMCInstLower.cpp b/lib/Target/AMDGPU/AMDGPUMCInstLower.cpp
index f6bdbf5e9be2caa3290b4a1ffd4d19c8f191cc5f..24674f0e59d6084abb70632926bff1c314181fd5 100644
--- a/lib/Target/AMDGPU/AMDGPUMCInstLower.cpp
+++ b/lib/Target/AMDGPU/AMDGPUMCInstLower.cpp
@@ -1,9 +1,8 @@
 //===- AMDGPUMCInstLower.cpp - Lower AMDGPU MachineInstr to an MCInst -----===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AMDGPU/AMDGPUMachineCFGStructurizer.cpp b/lib/Target/AMDGPU/AMDGPUMachineCFGStructurizer.cpp
index 6f44e2dbb2d580e4b904664164c407b987ff873f..2374909570587ab83ba0abf3144ddbfad6399fdb 100644
--- a/lib/Target/AMDGPU/AMDGPUMachineCFGStructurizer.cpp
+++ b/lib/Target/AMDGPU/AMDGPUMachineCFGStructurizer.cpp
@@ -1,9 +1,8 @@
 //===- AMDGPUMachineCFGStructurizer.cpp - Machine code if conversion pass. ===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AMDGPU/AMDGPUMachineFunction.cpp b/lib/Target/AMDGPU/AMDGPUMachineFunction.cpp
index 13b4b50149cedc2d70ac34d0ee585967fe2cfffd..05d5e2ec40076c637eb49c51f3ad2a7e26727043 100644
--- a/lib/Target/AMDGPU/AMDGPUMachineFunction.cpp
+++ b/lib/Target/AMDGPU/AMDGPUMachineFunction.cpp
@@ -1,9 +1,8 @@
 //===-- AMDGPUMachineFunctionInfo.cpp ---------------------------------------=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/AMDGPU/AMDGPUMachineFunction.h b/lib/Target/AMDGPU/AMDGPUMachineFunction.h
index 8d6b871bc03ea444d6a8f5a44dece3659f2cb5a8..52987e2fa4111c30934e4748f84449b248cea664 100644
--- a/lib/Target/AMDGPU/AMDGPUMachineFunction.h
+++ b/lib/Target/AMDGPU/AMDGPUMachineFunction.h
@@ -1,9 +1,8 @@
 //===-- AMDGPUMachineFunctionInfo.h -------------------------------*- C++ -*-=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/AMDGPU/AMDGPUMachineModuleInfo.cpp b/lib/Target/AMDGPU/AMDGPUMachineModuleInfo.cpp
index 7b9f673c418ced9bc7832955aef4a0cab1cafeef..1870c9ca5fa9e16200723f386c2c58a3e396ee56 100644
--- a/lib/Target/AMDGPU/AMDGPUMachineModuleInfo.cpp
+++ b/lib/Target/AMDGPU/AMDGPUMachineModuleInfo.cpp
@@ -1,9 +1,8 @@
 //===--- AMDGPUMachineModuleInfo.cpp ----------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AMDGPU/AMDGPUMachineModuleInfo.h b/lib/Target/AMDGPU/AMDGPUMachineModuleInfo.h
index 1219ab26fb69ed025bad1d4eb895c19a8d3428f3..e6d4268a66e15f596bdb7ae5bca2b5d8f72502dc 100644
--- a/lib/Target/AMDGPU/AMDGPUMachineModuleInfo.h
+++ b/lib/Target/AMDGPU/AMDGPUMachineModuleInfo.h
@@ -1,9 +1,8 @@
 //===--- AMDGPUMachineModuleInfo.h ------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AMDGPU/AMDGPUMacroFusion.cpp b/lib/Target/AMDGPU/AMDGPUMacroFusion.cpp
index 5e0b7d4290220ad0ac854b1bb71d21407628add1..8c11230f411a90f8b391f783a95eac7c610b0fbc 100644
--- a/lib/Target/AMDGPU/AMDGPUMacroFusion.cpp
+++ b/lib/Target/AMDGPU/AMDGPUMacroFusion.cpp
@@ -1,9 +1,8 @@
 //===--- AMDGPUMacroFusion.cpp - AMDGPU Macro Fusion ----------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AMDGPU/AMDGPUMacroFusion.h b/lib/Target/AMDGPU/AMDGPUMacroFusion.h
index 844958580a65b338c24e4bb19a8c37db03fb5c15..da4b3cf8bc2487d354ea3dedc279b9f61be6fa29 100644
--- a/lib/Target/AMDGPU/AMDGPUMacroFusion.h
+++ b/lib/Target/AMDGPU/AMDGPUMacroFusion.h
@@ -1,9 +1,8 @@
 //===- AMDGPUMacroFusion.h - AMDGPU Macro Fusion ----------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/AMDGPU/AMDGPUOpenCLEnqueuedBlockLowering.cpp b/lib/Target/AMDGPU/AMDGPUOpenCLEnqueuedBlockLowering.cpp
index 7bd8533a0ccf799276e6a49341e1095f8577327b..50871e1a0f1509bd2d4b07ee8dc85e398b95b109 100644
--- a/lib/Target/AMDGPU/AMDGPUOpenCLEnqueuedBlockLowering.cpp
+++ b/lib/Target/AMDGPU/AMDGPUOpenCLEnqueuedBlockLowering.cpp
@@ -1,9 +1,8 @@
 //===- AMDGPUOpenCLEnqueuedBlockLowering.cpp - Lower enqueued block -------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AMDGPU/AMDGPUPTNote.h b/lib/Target/AMDGPU/AMDGPUPTNote.h
index 2feff14d34a15864df2e444cc71ec2ade690101a..8b69f51c1a0d9314ac82b3e63f143854585caf40 100644
--- a/lib/Target/AMDGPU/AMDGPUPTNote.h
+++ b/lib/Target/AMDGPU/AMDGPUPTNote.h
@@ -1,9 +1,8 @@
 //===-- AMDGPUNoteType.h - AMDGPU ELF PT_NOTE section info-------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AMDGPU/AMDGPUPerfHintAnalysis.cpp b/lib/Target/AMDGPU/AMDGPUPerfHintAnalysis.cpp
index e53a8fe7c074db81a7af5ecca4f219ac44f75219..e9ebb912b08bad371cdf8519e49256d4982c5002 100644
--- a/lib/Target/AMDGPU/AMDGPUPerfHintAnalysis.cpp
+++ b/lib/Target/AMDGPU/AMDGPUPerfHintAnalysis.cpp
@@ -1,9 +1,8 @@
 //===- AMDGPUPerfHintAnalysis.cpp - analysis of functions memory traffic --===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AMDGPU/AMDGPUPerfHintAnalysis.h b/lib/Target/AMDGPU/AMDGPUPerfHintAnalysis.h
index be7f37cb6815264a37f6060068c575553ccca48f..88b9c96a914829affc366c91cd751794ed3a6eb7 100644
--- a/lib/Target/AMDGPU/AMDGPUPerfHintAnalysis.h
+++ b/lib/Target/AMDGPU/AMDGPUPerfHintAnalysis.h
@@ -1,9 +1,8 @@
 //===- AMDGPUPerfHintAnalysis.h - analysis of functions memory traffic ----===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp b/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp
index 5d087c0991844ae19ad0732c72f124734aa1111f..a2d62c73404c1672380bb79fb3d8e5a298a3b60f 100644
--- a/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp
+++ b/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp
@@ -1,9 +1,8 @@
 //===-- AMDGPUPromoteAlloca.cpp - Promote Allocas -------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -245,11 +244,11 @@ AMDGPUPromoteAlloca::getLocalSizeYZ(IRBuilder<> &Builder) {
   // We could do a single 64-bit load here, but it's likely that the basic
   // 32-bit and extract sequence is already present, and it is probably easier
   // to CSE this. The loads should be mergable later anyway.
-  Value *GEPXY = Builder.CreateConstInBoundsGEP1_64(CastDispatchPtr, 1);
-  LoadInst *LoadXY = Builder.CreateAlignedLoad(GEPXY, 4);
+  Value *GEPXY = Builder.CreateConstInBoundsGEP1_64(I32Ty, CastDispatchPtr, 1);
+  LoadInst *LoadXY = Builder.CreateAlignedLoad(I32Ty, GEPXY, 4);
 
-  Value *GEPZU = Builder.CreateConstInBoundsGEP1_64(CastDispatchPtr, 2);
-  LoadInst *LoadZU = Builder.CreateAlignedLoad(GEPZU, 4);
+  Value *GEPZU = Builder.CreateConstInBoundsGEP1_64(I32Ty, CastDispatchPtr, 2);
+  LoadInst *LoadZU = Builder.CreateAlignedLoad(I32Ty, GEPZU, 4);
 
   MDNode *MD = MDNode::get(Mod->getContext(), None);
   LoadXY->setMetadata(LLVMContext::MD_invariant_load, MD);
@@ -427,7 +426,7 @@ static bool tryPromoteAllocaToVector(AllocaInst *Alloca) {
       Value *Index = calculateVectorIndex(Ptr, GEPVectorIdx);
 
       Value *BitCast = Builder.CreateBitCast(Alloca, VecPtrTy);
-      Value *VecValue = Builder.CreateLoad(BitCast);
+      Value *VecValue = Builder.CreateLoad(VectorTy, BitCast);
       Value *ExtractElement = Builder.CreateExtractElement(VecValue, Index);
       Inst->replaceAllUsesWith(ExtractElement);
       Inst->eraseFromParent();
@@ -442,7 +441,7 @@ static bool tryPromoteAllocaToVector(AllocaInst *Alloca) {
       Value *Ptr = SI->getPointerOperand();
       Value *Index = calculateVectorIndex(Ptr, GEPVectorIdx);
       Value *BitCast = Builder.CreateBitCast(Alloca, VecPtrTy);
-      Value *VecValue = Builder.CreateLoad(BitCast);
+      Value *VecValue = Builder.CreateLoad(VectorTy, BitCast);
       Value *NewVecValue = Builder.CreateInsertElement(VecValue,
                                                        SI->getValueOperand(),
                                                        Index);
@@ -919,7 +918,8 @@ bool AMDGPUPromoteAlloca::handleAlloca(AllocaInst &I, bool SufficientLDS) {
       );
 
       CallInst *NewCall = Builder.CreateCall(
-          ObjectSize, {Src, Intr->getOperand(1), Intr->getOperand(2)});
+          ObjectSize,
+          {Src, Intr->getOperand(1), Intr->getOperand(2), Intr->getOperand(3)});
       Intr->replaceAllUsesWith(NewCall);
       Intr->eraseFromParent();
       continue;
diff --git a/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp b/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
index e53c02202df5bb5c101f29502ff941a0bf7db503..bb7a492fd8ab902cb946a5878da5c275cbc6088e 100644
--- a/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
+++ b/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
@@ -1,9 +1,8 @@
 //===- AMDGPURegisterBankInfo.cpp -------------------------------*- C++ -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
@@ -17,6 +16,7 @@
 #include "SIMachineFunctionInfo.h"
 #include "SIRegisterInfo.h"
 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
+#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
 #include "llvm/CodeGen/GlobalISel/RegisterBank.h"
 #include "llvm/CodeGen/GlobalISel/RegisterBankInfo.h"
 #include "llvm/CodeGen/TargetRegisterInfo.h"
@@ -74,18 +74,48 @@ unsigned AMDGPURegisterBankInfo::copyCost(const RegisterBank &Dst,
                                           const RegisterBank &Src,
                                           unsigned Size) const {
   if (Dst.getID() == AMDGPU::SGPRRegBankID &&
-      Src.getID() == AMDGPU::VGPRRegBankID)
+      Src.getID() == AMDGPU::VGPRRegBankID) {
     return std::numeric_limits<unsigned>::max();
+  }
 
   // SGPRRegBank with size 1 is actually vcc or another 64-bit sgpr written by
   // the valu.
   if (Size == 1 && Dst.getID() == AMDGPU::SCCRegBankID &&
-      Src.getID() == AMDGPU::SGPRRegBankID)
+      (Src.getID() == AMDGPU::SGPRRegBankID ||
+       Src.getID() == AMDGPU::VGPRRegBankID ||
+       Src.getID() == AMDGPU::VCCRegBankID))
+    return std::numeric_limits<unsigned>::max();
+
+  if (Dst.getID() == AMDGPU::SCCRegBankID &&
+      Src.getID() == AMDGPU::VCCRegBankID)
     return std::numeric_limits<unsigned>::max();
 
   return RegisterBankInfo::copyCost(Dst, Src, Size);
 }
 
+unsigned AMDGPURegisterBankInfo::getBreakDownCost(
+  const ValueMapping &ValMapping,
+  const RegisterBank *CurBank) const {
+  // Currently we should only see rewrites of defs since copies from VGPR to
+  // SGPR are illegal.
+  assert(CurBank == nullptr && "shouldn't see already assigned bank");
+
+  assert(ValMapping.NumBreakDowns == 2 &&
+         ValMapping.BreakDown[0].Length == 32 &&
+         ValMapping.BreakDown[0].StartIdx == 0 &&
+         ValMapping.BreakDown[1].Length == 32 &&
+         ValMapping.BreakDown[1].StartIdx == 32 &&
+         ValMapping.BreakDown[0].RegBank == ValMapping.BreakDown[1].RegBank);
+
+  // 32-bit extract of a 64-bit value is just access of a subregister, so free.
+  // TODO: Cost of 0 hits assert, though it's not clear it's what we really
+  // want.
+
+  // TODO: 32-bit insert to a 64-bit SGPR may incur a non-free copy due to SGPR
+  // alignment restrictions, but this probably isn't important.
+  return 1;
+}
+
 const RegisterBank &AMDGPURegisterBankInfo::getRegBankFromRegClass(
     const TargetRegisterClass &RC) const {
 
@@ -105,6 +135,48 @@ AMDGPURegisterBankInfo::getInstrAlternativeMappings(
 
   InstructionMappings AltMappings;
   switch (MI.getOpcode()) {
+  case TargetOpcode::G_AND:
+  case TargetOpcode::G_OR:
+  case TargetOpcode::G_XOR: {
+    unsigned Size = getSizeInBits(MI.getOperand(0).getReg(), MRI, *TRI);
+    if (Size != 64)
+      break;
+
+    const InstructionMapping &SSMapping = getInstructionMapping(
+      1, 1, getOperandsMapping(
+        {AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size),
+         AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size),
+         AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size)}),
+      3); // Num Operands
+    AltMappings.push_back(&SSMapping);
+
+    const InstructionMapping &VVMapping = getInstructionMapping(
+      2, 2, getOperandsMapping(
+        {AMDGPU::getValueMappingSGPR64Only(AMDGPU::VGPRRegBankID, Size),
+         AMDGPU::getValueMappingSGPR64Only(AMDGPU::VGPRRegBankID, Size),
+         AMDGPU::getValueMappingSGPR64Only(AMDGPU::VGPRRegBankID, Size)}),
+      3); // Num Operands
+    AltMappings.push_back(&VVMapping);
+
+    const InstructionMapping &SVMapping = getInstructionMapping(
+      3, 3, getOperandsMapping(
+        {AMDGPU::getValueMappingSGPR64Only(AMDGPU::VGPRRegBankID, Size),
+         AMDGPU::getValueMappingSGPR64Only(AMDGPU::SGPRRegBankID, Size),
+         AMDGPU::getValueMappingSGPR64Only(AMDGPU::VGPRRegBankID, Size)}),
+      3); // Num Operands
+    AltMappings.push_back(&SVMapping);
+
+    // SGPR in LHS is slightly preferrable, so make it VS more expnesive than
+    // SV.
+    const InstructionMapping &VSMapping = getInstructionMapping(
+      3, 4, getOperandsMapping(
+        {AMDGPU::getValueMappingSGPR64Only(AMDGPU::VGPRRegBankID, Size),
+         AMDGPU::getValueMappingSGPR64Only(AMDGPU::VGPRRegBankID, Size),
+         AMDGPU::getValueMappingSGPR64Only(AMDGPU::SGPRRegBankID, Size)}),
+      3); // Num Operands
+    AltMappings.push_back(&VSMapping);
+    break;
+  }
   case TargetOpcode::G_LOAD: {
     unsigned Size = getSizeInBits(MI.getOperand(0).getReg(), MRI, *TRI);
     // FIXME: Should we be hard coding the size for these mappings?
@@ -145,7 +217,7 @@ AMDGPURegisterBankInfo::getInstrAlternativeMappings(
     AltMappings.push_back(&SSMapping);
 
     const InstructionMapping &SVMapping = getInstructionMapping(2, 1,
-      getOperandsMapping({AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, 1),
+      getOperandsMapping({AMDGPU::getValueMapping(AMDGPU::VCCRegBankID, 1),
                           nullptr, // Predicate operand.
                           AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size),
                           AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Size)}),
@@ -153,7 +225,7 @@ AMDGPURegisterBankInfo::getInstrAlternativeMappings(
     AltMappings.push_back(&SVMapping);
 
     const InstructionMapping &VSMapping = getInstructionMapping(3, 1,
-      getOperandsMapping({AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, 1),
+      getOperandsMapping({AMDGPU::getValueMapping(AMDGPU::VCCRegBankID, 1),
                           nullptr, // Predicate operand.
                           AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Size),
                           AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size)}),
@@ -161,7 +233,7 @@ AMDGPURegisterBankInfo::getInstrAlternativeMappings(
     AltMappings.push_back(&VSMapping);
 
     const InstructionMapping &VVMapping = getInstructionMapping(4, 1,
-      getOperandsMapping({AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, 1),
+      getOperandsMapping({AMDGPU::getValueMapping(AMDGPU::VCCRegBankID, 1),
                           nullptr, // Predicate operand.
                           AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Size),
                           AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Size)}),
@@ -180,30 +252,55 @@ AMDGPURegisterBankInfo::getInstrAlternativeMappings(
       4); // Num Operands
     AltMappings.push_back(&SSMapping);
 
-    const InstructionMapping &SVMapping = getInstructionMapping(2, 1,
+    const InstructionMapping &VVMapping = getInstructionMapping(2, 1,
       getOperandsMapping({AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Size),
-                          AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, 1),
-                          AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size),
+                          AMDGPU::getValueMapping(AMDGPU::VCCRegBankID, 1),
+                          AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Size),
                           AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Size)}),
       4); // Num Operands
-    AltMappings.push_back(&SVMapping);
+    AltMappings.push_back(&VVMapping);
 
-    const InstructionMapping &VSMapping = getInstructionMapping(2, 1,
-      getOperandsMapping({AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Size),
-                          AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, 1),
-                          AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Size),
-                          AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size)}),
-      4); // Num Operands
-    AltMappings.push_back(&VSMapping);
+    return AltMappings;
+  }
+  case TargetOpcode::G_UADDE:
+  case TargetOpcode::G_USUBE:
+  case TargetOpcode::G_SADDE:
+  case TargetOpcode::G_SSUBE: {
+    unsigned Size = getSizeInBits(MI.getOperand(0).getReg(), MRI, *TRI);
+    const InstructionMapping &SSMapping = getInstructionMapping(1, 1,
+      getOperandsMapping(
+        {AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size),
+         AMDGPU::getValueMapping(AMDGPU::SCCRegBankID, 1),
+         AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size),
+         AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size),
+         AMDGPU::getValueMapping(AMDGPU::SCCRegBankID, 1)}),
+      5); // Num Operands
+    AltMappings.push_back(&SSMapping);
 
     const InstructionMapping &VVMapping = getInstructionMapping(2, 1,
       getOperandsMapping({AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Size),
-                          AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, 1),
+                          AMDGPU::getValueMapping(AMDGPU::VCCRegBankID, 1),
                           AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Size),
-                          AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Size)}),
-      4); // Num Operands
+                          AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Size),
+                          AMDGPU::getValueMapping(AMDGPU::VCCRegBankID, 1)}),
+      5); // Num Operands
     AltMappings.push_back(&VVMapping);
-
+    return AltMappings;
+  }
+  case AMDGPU::G_BRCOND: {
+    assert(MRI.getType(MI.getOperand(0).getReg()).getSizeInBits() == 1);
+
+    const InstructionMapping &SMapping = getInstructionMapping(
+      1, 1, getOperandsMapping(
+        {AMDGPU::getValueMapping(AMDGPU::SCCRegBankID, 1), nullptr}),
+      2); // Num Operands
+    AltMappings.push_back(&SMapping);
+
+    const InstructionMapping &VMapping = getInstructionMapping(
+      1, 1, getOperandsMapping(
+        {AMDGPU::getValueMapping(AMDGPU::VCCRegBankID, 1), nullptr }),
+      2); // Num Operands
+    AltMappings.push_back(&VMapping);
     return AltMappings;
   }
   default:
@@ -212,8 +309,85 @@ AMDGPURegisterBankInfo::getInstrAlternativeMappings(
   return RegisterBankInfo::getInstrAlternativeMappings(MI);
 }
 
+void AMDGPURegisterBankInfo::split64BitValueForMapping(
+  MachineIRBuilder &B,
+  SmallVector<unsigned, 2> &Regs,
+  unsigned Reg) const {
+  LLT S32 = LLT::scalar(32);
+  MachineRegisterInfo *MRI = B.getMRI();
+  unsigned LoLHS = MRI->createGenericVirtualRegister(S32);
+  unsigned HiLHS = MRI->createGenericVirtualRegister(S32);
+  const RegisterBank *Bank = getRegBank(Reg, *MRI, *TRI);
+  MRI->setRegBank(LoLHS, *Bank);
+  MRI->setRegBank(HiLHS, *Bank);
+
+  Regs.push_back(LoLHS);
+  Regs.push_back(HiLHS);
+
+  B.buildInstr(AMDGPU::G_UNMERGE_VALUES)
+    .addDef(LoLHS)
+    .addDef(HiLHS)
+    .addUse(Reg);
+}
+
 void AMDGPURegisterBankInfo::applyMappingImpl(
     const OperandsMapper &OpdMapper) const {
+  MachineInstr &MI = OpdMapper.getMI();
+  unsigned Opc = MI.getOpcode();
+  MachineRegisterInfo &MRI = OpdMapper.getMRI();
+  switch (Opc) {
+  case AMDGPU::G_AND:
+  case AMDGPU::G_OR:
+  case AMDGPU::G_XOR: {
+    // 64-bit and is only available on the SALU, so split into 2 32-bit ops if
+    // there is a VGPR input.
+    unsigned DstReg = MI.getOperand(0).getReg();
+    if (MRI.getType(DstReg).getSizeInBits() != 64)
+      break;
+
+    SmallVector<unsigned, 2> DefRegs(OpdMapper.getVRegs(0));
+    SmallVector<unsigned, 2> Src0Regs(OpdMapper.getVRegs(1));
+    SmallVector<unsigned, 2> Src1Regs(OpdMapper.getVRegs(2));
+
+    // All inputs are SGPRs, nothing special to do.
+    if (DefRegs.empty()) {
+      assert(Src0Regs.empty() && Src1Regs.empty());
+      break;
+    }
+
+    assert(DefRegs.size() == 2);
+    assert(Src0Regs.size() == Src1Regs.size() &&
+           (Src0Regs.empty() || Src0Regs.size() == 2));
+
+    // Depending on where the source registers came from, the generic code may
+    // have decided to split the inputs already or not. If not, we still need to
+    // extract the values.
+    MachineIRBuilder B(MI);
+
+    if (Src0Regs.empty())
+      split64BitValueForMapping(B, Src0Regs, MI.getOperand(1).getReg());
+
+    if (Src1Regs.empty())
+      split64BitValueForMapping(B, Src1Regs, MI.getOperand(2).getReg());
+
+    B.buildInstr(Opc)
+      .addDef(DefRegs[0])
+      .addUse(Src0Regs[0])
+      .addUse(Src1Regs[0]);
+
+    B.buildInstr(Opc)
+      .addDef(DefRegs[1])
+      .addUse(Src0Regs[1])
+      .addUse(Src1Regs[1]);
+
+    MRI.setRegBank(DstReg, getRegBank(AMDGPU::VGPRRegBankID));
+    MI.eraseFromParent();
+    return;
+  }
+  default:
+    break;
+  }
+
   return applyDefaultMapping(OpdMapper);
 }
 
@@ -232,9 +406,13 @@ bool AMDGPURegisterBankInfo::isSALUMapping(const MachineInstr &MI) const {
     if (!MI.getOperand(i).isReg())
       continue;
     unsigned Reg = MI.getOperand(i).getReg();
-    const RegisterBank *Bank = getRegBank(Reg, MRI, *TRI);
-    if (Bank && Bank->getID() != AMDGPU::SGPRRegBankID)
-      return false;
+    if (const RegisterBank *Bank = getRegBank(Reg, MRI, *TRI)) {
+      if (Bank->getID() == AMDGPU::VGPRRegBankID)
+        return false;
+
+      assert(Bank->getID() == AMDGPU::SGPRRegBankID ||
+             Bank->getID() == AMDGPU::SCCRegBankID);
+    }
   }
   return true;
 }
@@ -247,7 +425,8 @@ AMDGPURegisterBankInfo::getDefaultMappingSOP(const MachineInstr &MI) const {
 
   for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
     unsigned Size = getSizeInBits(MI.getOperand(i).getReg(), MRI, *TRI);
-    OpdsMapping[i] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size);
+    unsigned BankID = Size == 1 ? AMDGPU::SCCRegBankID : AMDGPU::SGPRRegBankID;
+    OpdsMapping[i] = AMDGPU::getValueMapping(BankID, Size);
   }
   return getInstructionMapping(1, 1, getOperandsMapping(OpdsMapping),
                                MI.getNumOperands());
@@ -268,12 +447,32 @@ AMDGPURegisterBankInfo::getDefaultMappingVOP(const MachineInstr &MI) const {
 
   unsigned Reg1 = MI.getOperand(OpdIdx).getReg();
   unsigned Size1 = getSizeInBits(Reg1, MRI, *TRI);
-  unsigned Bank1 = getRegBankID(Reg1, MRI, *TRI);
+
+  unsigned DefaultBankID = Size1 == 1 ?
+    AMDGPU::VCCRegBankID : AMDGPU::VGPRRegBankID;
+  unsigned Bank1 = getRegBankID(Reg1, MRI, *TRI, DefaultBankID);
+
   OpdsMapping[OpdIdx++] = AMDGPU::getValueMapping(Bank1, Size1);
 
   for (unsigned e = MI.getNumOperands(); OpdIdx != e; ++OpdIdx) {
     unsigned Size = getSizeInBits(MI.getOperand(OpdIdx).getReg(), MRI, *TRI);
-    OpdsMapping[OpdIdx] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Size);
+    unsigned BankID = Size == 1 ? AMDGPU::VCCRegBankID : AMDGPU::VGPRRegBankID;
+    OpdsMapping[OpdIdx] = AMDGPU::getValueMapping(BankID, Size);
+  }
+
+  return getInstructionMapping(1, 1, getOperandsMapping(OpdsMapping),
+                               MI.getNumOperands());
+}
+
+const RegisterBankInfo::InstructionMapping &
+AMDGPURegisterBankInfo::getDefaultMappingAllVGPR(const MachineInstr &MI) const {
+  const MachineFunction &MF = *MI.getParent()->getParent();
+  const MachineRegisterInfo &MRI = MF.getRegInfo();
+  SmallVector<const ValueMapping*, 8> OpdsMapping(MI.getNumOperands());
+
+  for (unsigned I = 0, E = MI.getNumOperands(); I != E; ++I) {
+    unsigned Size = getSizeInBits(MI.getOperand(I).getReg(), MRI, *TRI);
+    OpdsMapping[I] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Size);
   }
 
   return getInstructionMapping(1, 1, getOperandsMapping(OpdsMapping),
@@ -342,13 +541,53 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
   switch (MI.getOpcode()) {
   default:
     return getInvalidInstructionMapping();
+
+  case AMDGPU::G_AND:
+  case AMDGPU::G_OR:
+  case AMDGPU::G_XOR: {
+    unsigned Size = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
+    if (Size == 1) {
+      OpdsMapping[0] = OpdsMapping[1] =
+        OpdsMapping[2] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size);
+      break;
+    }
+
+    if (Size == 64) {
+
+      if (isSALUMapping(MI)) {
+        OpdsMapping[0] = getValueMappingSGPR64Only(AMDGPU::SGPRRegBankID, Size);
+        OpdsMapping[1] = OpdsMapping[2] = OpdsMapping[0];
+      } else {
+        OpdsMapping[0] = getValueMappingSGPR64Only(AMDGPU::VGPRRegBankID, Size);
+        unsigned Bank1 = getRegBankID(MI.getOperand(1).getReg(), MRI, *TRI/*, DefaultBankID*/);
+        OpdsMapping[1] = AMDGPU::getValueMapping(Bank1, Size);
+
+        unsigned Bank2 = getRegBankID(MI.getOperand(2).getReg(), MRI, *TRI/*, DefaultBankID*/);
+        OpdsMapping[2] = AMDGPU::getValueMapping(Bank2, Size);
+      }
+
+      break;
+    }
+
+    LLVM_FALLTHROUGH;
+  }
+
   case AMDGPU::G_ADD:
   case AMDGPU::G_SUB:
   case AMDGPU::G_MUL:
-  case AMDGPU::G_AND:
-  case AMDGPU::G_OR:
-  case AMDGPU::G_XOR:
   case AMDGPU::G_SHL:
+  case AMDGPU::G_LSHR:
+  case AMDGPU::G_ASHR:
+  case AMDGPU::G_UADDO:
+  case AMDGPU::G_SADDO:
+  case AMDGPU::G_USUBO:
+  case AMDGPU::G_SSUBO:
+  case AMDGPU::G_UADDE:
+  case AMDGPU::G_SADDE:
+  case AMDGPU::G_USUBE:
+  case AMDGPU::G_SSUBE:
+  case AMDGPU::G_UMULH:
+  case AMDGPU::G_SMULH:
     if (isSALUMapping(MI))
       return getDefaultMappingSOP(MI);
     LLVM_FALLTHROUGH;
@@ -359,6 +598,15 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
   case AMDGPU::G_FPTOUI:
   case AMDGPU::G_FMUL:
   case AMDGPU::G_FMA:
+  case AMDGPU::G_FSQRT:
+  case AMDGPU::G_SITOFP:
+  case AMDGPU::G_UITOFP:
+  case AMDGPU::G_FPTRUNC:
+  case AMDGPU::G_FPEXT:
+  case AMDGPU::G_FEXP2:
+  case AMDGPU::G_FLOG2:
+  case AMDGPU::G_INTRINSIC_TRUNC:
+  case AMDGPU::G_INTRINSIC_ROUND:
     return getDefaultMappingVOP(MI);
   case AMDGPU::G_IMPLICIT_DEF: {
     unsigned Size = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
@@ -457,7 +705,7 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
   case AMDGPU::G_FCMP: {
     unsigned Size = MRI.getType(MI.getOperand(2).getReg()).getSizeInBits();
     unsigned Op2Bank = getRegBankID(MI.getOperand(2).getReg(), MRI, *TRI);
-    OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, 1);
+    OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::VCCRegBankID, 1);
     OpdsMapping[1] = nullptr; // Predicate Operand.
     OpdsMapping[2] = AMDGPU::getValueMapping(Op2Bank, Size);
     OpdsMapping[3] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Size);
@@ -497,7 +745,7 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
     unsigned Op3Bank = getRegBankID(MI.getOperand(3).getReg(), MRI, *TRI);
     unsigned Op0Bank = Op2Bank == AMDGPU::SGPRRegBankID &&
                        Op3Bank == AMDGPU::SGPRRegBankID ?
-                       AMDGPU::SCCRegBankID : AMDGPU::VGPRRegBankID;
+                       AMDGPU::SCCRegBankID : AMDGPU::VCCRegBankID;
     OpdsMapping[0] = AMDGPU::getValueMapping(Op0Bank, 1);
     OpdsMapping[1] = nullptr; // Predicate Operand.
     OpdsMapping[2] = AMDGPU::getValueMapping(Op2Bank, Size);
@@ -545,6 +793,18 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
 
     break;
   }
+  case AMDGPU::G_UNMERGE_VALUES: {
+    unsigned Bank = isSALUMapping(MI) ? AMDGPU::SGPRRegBankID :
+      AMDGPU::VGPRRegBankID;
+
+    // Op1 and Dst should use the same register bank.
+    // FIXME: Shouldn't this be the default? Why do we need to handle this?
+    for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
+      unsigned Size = getSizeInBits(MI.getOperand(i).getReg(), MRI, *TRI);
+      OpdsMapping[i] = AMDGPU::getValueMapping(Bank, Size);
+    }
+    break;
+  }
   case AMDGPU::G_INTRINSIC: {
     switch (MI.getOperand(1).getIntrinsicID()) {
     default:
@@ -558,6 +818,12 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
       OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size);
       break;
     }
+    case Intrinsic::amdgcn_wqm_vote: {
+      unsigned Size = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
+      OpdsMapping[0] = OpdsMapping[2]
+        = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size);
+      break;
+    }
     }
     break;
   }
@@ -604,7 +870,7 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
                     Op2Bank == AMDGPU::SGPRRegBankID &&
                     Op3Bank == AMDGPU::SGPRRegBankID;
     unsigned Bank = SGPRSrcs ? AMDGPU::SGPRRegBankID : AMDGPU::VGPRRegBankID;
-    Op1Bank = SGPRSrcs ? AMDGPU::SCCRegBankID : AMDGPU::SGPRRegBankID;
+    Op1Bank = SGPRSrcs ? AMDGPU::SCCRegBankID : AMDGPU::VCCRegBankID;
     OpdsMapping[0] = AMDGPU::getValueMapping(Bank, Size);
     OpdsMapping[1] = AMDGPU::getValueMapping(Op1Bank, 1);
     OpdsMapping[2] = AMDGPU::getValueMapping(Bank, Size);
@@ -614,8 +880,33 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
 
   case AMDGPU::G_LOAD:
     return getInstrMappingForLoad(MI);
+
+  case AMDGPU::G_ATOMICRMW_XCHG:
+  case AMDGPU::G_ATOMICRMW_ADD:
+  case AMDGPU::G_ATOMICRMW_SUB:
+  case AMDGPU::G_ATOMICRMW_AND:
+  case AMDGPU::G_ATOMICRMW_OR:
+  case AMDGPU::G_ATOMICRMW_XOR:
+  case AMDGPU::G_ATOMICRMW_MAX:
+  case AMDGPU::G_ATOMICRMW_MIN:
+  case AMDGPU::G_ATOMICRMW_UMAX:
+  case AMDGPU::G_ATOMICRMW_UMIN:
+  case AMDGPU::G_ATOMIC_CMPXCHG: {
+    return getDefaultMappingAllVGPR(MI);
+  }
+  case AMDGPU::G_BRCOND: {
+    unsigned Bank = getRegBankID(MI.getOperand(0).getReg(), MRI, *TRI,
+                                 AMDGPU::SGPRRegBankID);
+    assert(MRI.getType(MI.getOperand(0).getReg()).getSizeInBits() == 1);
+    if (Bank != AMDGPU::SCCRegBankID)
+      Bank = AMDGPU::VCCRegBankID;
+
+    OpdsMapping[0] = AMDGPU::getValueMapping(Bank, 1);
+    break;
+  }
   }
 
   return getInstructionMapping(1, 1, getOperandsMapping(OpdsMapping),
                                MI.getNumOperands());
 }
+
diff --git a/lib/Target/AMDGPU/AMDGPURegisterBankInfo.h b/lib/Target/AMDGPU/AMDGPURegisterBankInfo.h
index d48a665898735b3cf73102c1e7976fab26a8d8fb..9cb7a7f8a23c3b952080431fdc7341bbce668e53 100644
--- a/lib/Target/AMDGPU/AMDGPURegisterBankInfo.h
+++ b/lib/Target/AMDGPU/AMDGPURegisterBankInfo.h
@@ -1,9 +1,8 @@
 //===- AMDGPURegisterBankInfo -----------------------------------*- C++ -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
@@ -22,6 +21,7 @@
 
 namespace llvm {
 
+class MachineIRBuilder;
 class SIRegisterInfo;
 class TargetRegisterInfo;
 
@@ -46,15 +46,26 @@ class AMDGPURegisterBankInfo : public AMDGPUGenRegisterBankInfo {
                         const TargetRegisterInfo &TRI,
                         unsigned Default = AMDGPU::VGPRRegBankID) const;
 
+  /// Split 64-bit value \p Reg into two 32-bit halves and populate them into \p
+  /// Regs. This appropriately sets the regbank of the new registers.
+  void split64BitValueForMapping(MachineIRBuilder &B,
+                                 SmallVector<unsigned, 2> &Regs,
+                                 unsigned Reg) const;
+
   bool isSALUMapping(const MachineInstr &MI) const;
   const InstructionMapping &getDefaultMappingSOP(const MachineInstr &MI) const;
   const InstructionMapping &getDefaultMappingVOP(const MachineInstr &MI) const;
+  const InstructionMapping &getDefaultMappingAllVGPR(
+    const MachineInstr &MI) const;
 public:
   AMDGPURegisterBankInfo(const TargetRegisterInfo &TRI);
 
   unsigned copyCost(const RegisterBank &A, const RegisterBank &B,
                     unsigned Size) const override;
 
+  unsigned getBreakDownCost(const ValueMapping &ValMapping,
+                            const RegisterBank *CurBank = nullptr) const override;
+
   const RegisterBank &
   getRegBankFromRegClass(const TargetRegisterClass &RC) const override;
 
diff --git a/lib/Target/AMDGPU/AMDGPURegisterBanks.td b/lib/Target/AMDGPU/AMDGPURegisterBanks.td
index 7f7f75f656479a6b47e3e4a1a121b39473cb81bb..3fdc3bc357eff110ddc8c677b1083a2f12146419 100644
--- a/lib/Target/AMDGPU/AMDGPURegisterBanks.td
+++ b/lib/Target/AMDGPU/AMDGPURegisterBanks.td
@@ -1,9 +1,8 @@
 //=- AMDGPURegisterBank.td - Describe the AMDGPU Banks -------*- tablegen -*-=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -15,4 +14,7 @@ def VGPRRegBank : RegisterBank<"VGPR",
   [VGPR_32, VReg_64, VReg_96, VReg_128, VReg_256, VReg_512]
 >;
 
-def SCCRegBank : RegisterBank <"SCC", [SCC_CLASS ]>;
+def SCCRegBank : RegisterBank <"SCC", [SCC_CLASS]>;
+
+// It is helpful to distinguish conditions from ordinary SGPRs.
+def VCCRegBank : RegisterBank <"VCC", [SReg_64]>;
diff --git a/lib/Target/AMDGPU/AMDGPURegisterInfo.cpp b/lib/Target/AMDGPU/AMDGPURegisterInfo.cpp
index 50f859addc2b77f00e90782443a79beba30b3cc6..60f23f7d778b4d9331f8332e394291c6d2dbaab1 100644
--- a/lib/Target/AMDGPU/AMDGPURegisterInfo.cpp
+++ b/lib/Target/AMDGPU/AMDGPURegisterInfo.cpp
@@ -1,9 +1,8 @@
 //===-- AMDGPURegisterInfo.cpp - AMDGPU Register Information -------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AMDGPU/AMDGPURegisterInfo.h b/lib/Target/AMDGPU/AMDGPURegisterInfo.h
index 922d974f2ebd68ee0f738b8d41968626f06c0370..3453a8c1b0b396fc063dee50bcd3c3b23b57c4eb 100644
--- a/lib/Target/AMDGPU/AMDGPURegisterInfo.h
+++ b/lib/Target/AMDGPU/AMDGPURegisterInfo.h
@@ -1,9 +1,8 @@
 //===-- AMDGPURegisterInfo.h - AMDGPURegisterInfo Interface -*- C++ -*-----===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AMDGPU/AMDGPURegisterInfo.td b/lib/Target/AMDGPU/AMDGPURegisterInfo.td
index ceabae524414615f47fa9f66ea98cd2690d95db0..465be5259b644f232fe430ffad6999f7135f685e 100644
--- a/lib/Target/AMDGPU/AMDGPURegisterInfo.td
+++ b/lib/Target/AMDGPU/AMDGPURegisterInfo.td
@@ -1,9 +1,8 @@
 //===-- AMDGPURegisterInfo.td - AMDGPU register info -------*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AMDGPU/AMDGPURewriteOutArguments.cpp b/lib/Target/AMDGPU/AMDGPURewriteOutArguments.cpp
index a861762a8c9e3f563196b40bec6677d25b25966d..4f095087a57f4cc9f8390fc0915e11cb29fa2914 100644
--- a/lib/Target/AMDGPU/AMDGPURewriteOutArguments.cpp
+++ b/lib/Target/AMDGPU/AMDGPURewriteOutArguments.cpp
@@ -1,9 +1,8 @@
 //===- AMDGPURewriteOutArgumentsPass.cpp - Create struct returns ----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -163,7 +162,7 @@ bool AMDGPURewriteOutArguments::checkArgumentUses(Value &Arg) const {
       // some casts between structs and non-structs, but we can't bitcast
       // directly between them.  directly bitcast between them.  Blender uses
       // some casts that look like { <3 x float> }* to <4 x float>*
-      if ((SrcEltTy->isStructTy() && (SrcEltTy->getNumContainedTypes() != 1)))
+      if ((SrcEltTy->isStructTy() && (SrcEltTy->getStructNumElements() != 1)))
         return false;
 
       // Clang emits OpenCL 3-vector type accesses with a bitcast to the
@@ -401,8 +400,8 @@ bool AMDGPURewriteOutArguments::runOnFunction(Function &F) {
       if (Val->getType() != EltTy) {
         Type *EffectiveEltTy = EltTy;
         if (StructType *CT = dyn_cast<StructType>(EltTy)) {
-          assert(CT->getNumContainedTypes() == 1);
-          EffectiveEltTy = CT->getContainedType(0);
+          assert(CT->getNumElements() == 1);
+          EffectiveEltTy = CT->getElementType(0);
         }
 
         if (DL->getTypeSizeInBits(EffectiveEltTy) !=
diff --git a/lib/Target/AMDGPU/AMDGPUSearchableTables.td b/lib/Target/AMDGPU/AMDGPUSearchableTables.td
index 9dbd7751b4d8856209b6de06e454a73ad710cdc7..916b1efffd05b302f53b88e0cb245eb7ee655f1e 100644
--- a/lib/Target/AMDGPU/AMDGPUSearchableTables.td
+++ b/lib/Target/AMDGPU/AMDGPUSearchableTables.td
@@ -1,9 +1,8 @@
 //===-- AMDGPUSearchableTables.td - ------------------------*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -49,6 +48,8 @@ def : SourceOfDivergence<int_amdgcn_workitem_id_z>;
 def : SourceOfDivergence<int_amdgcn_interp_mov>;
 def : SourceOfDivergence<int_amdgcn_interp_p1>;
 def : SourceOfDivergence<int_amdgcn_interp_p2>;
+def : SourceOfDivergence<int_amdgcn_interp_p1_f16>;
+def : SourceOfDivergence<int_amdgcn_interp_p2_f16>;
 def : SourceOfDivergence<int_amdgcn_mbcnt_hi>;
 def : SourceOfDivergence<int_amdgcn_mbcnt_lo>;
 def : SourceOfDivergence<int_r600_read_tidig_x>;
@@ -72,6 +73,8 @@ def : SourceOfDivergence<int_amdgcn_buffer_atomic_xor>;
 def : SourceOfDivergence<int_amdgcn_buffer_atomic_cmpswap>;
 def : SourceOfDivergence<int_amdgcn_ps_live>;
 def : SourceOfDivergence<int_amdgcn_ds_swizzle>;
+def : SourceOfDivergence<int_amdgcn_ds_ordered_add>;
+def : SourceOfDivergence<int_amdgcn_ds_ordered_swap>;
 
 foreach intr = AMDGPUImageDimAtomicIntrinsics in
 def : SourceOfDivergence<intr>;
diff --git a/lib/Target/AMDGPU/AMDGPUSubtarget.cpp b/lib/Target/AMDGPU/AMDGPUSubtarget.cpp
index f1acd72b03a2086eebd069e2052466bade548a05..cdfbfb5154e81000563bda3ca6b1981904372511 100644
--- a/lib/Target/AMDGPU/AMDGPUSubtarget.cpp
+++ b/lib/Target/AMDGPU/AMDGPUSubtarget.cpp
@@ -1,9 +1,8 @@
 //===-- AMDGPUSubtarget.cpp - AMDGPU Subtarget Information ----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -74,6 +73,9 @@ GCNSubtarget::initializeSubtargetDependencies(const Triple &TT,
   // We want to be able to turn these off, but making this a subtarget feature
   // for SI has the unhelpful behavior that it unsets everything else if you
   // disable it.
+  //
+  // Similarly we want enable-prt-strict-null to be on by default and not to
+  // unset everything else if it is disabled
 
   SmallString<256> FullFS("+promote-alloca,+dx10-clamp,+load-store-opt,");
 
@@ -89,6 +91,8 @@ GCNSubtarget::initializeSubtargetDependencies(const Triple &TT,
     FullFS += "-fp32-denormals,";
   }
 
+  FullFS += "+enable-prt-strict-null,"; // This is overridden by a disable in FS
+
   FullFS += FS;
 
   ParseSubtargetFeatures(GPU, FullFS);
@@ -119,6 +123,10 @@ GCNSubtarget::initializeSubtargetDependencies(const Triple &TT,
       HasMovrel = true;
   }
 
+  // Don't crash on invalid devices.
+  if (WavefrontSize == 0)
+    WavefrontSize = 64;
+
   HasFminFmaxLegacy = getGeneration() < AMDGPUSubtarget::VOLCANIC_ISLANDS;
 
   return *this;
@@ -148,7 +156,6 @@ GCNSubtarget::GCNSubtarget(const Triple &TT, StringRef GPU, StringRef FS,
     AMDGPUSubtarget(TT),
     TargetTriple(TT),
     Gen(SOUTHERN_ISLANDS),
-    IsaVersion(ISAVersion0_0_0),
     InstrItins(getInstrItineraryForCPU(GPU)),
     LDSBankCount(0),
     MaxPrivateElementSize(0),
@@ -175,6 +182,7 @@ GCNSubtarget::GCNSubtarget(const Triple &TT, StringRef GPU, StringRef FS,
     EnableUnsafeDSOffsetFolding(false),
     EnableSIScheduler(false),
     EnableDS128(false),
+    EnablePRTStrictNull(false),
     DumpCode(false),
 
     FP64(false),
@@ -198,6 +206,8 @@ GCNSubtarget::GCNSubtarget(const Triple &TT, StringRef GPU, StringRef FS,
     HasDPP(false),
     HasR128A16(false),
     HasDLInsts(false),
+    HasDot1Insts(false),
+    HasDot2Insts(false),
     EnableSRAMECC(false),
     FlatAddressSpace(false),
     FlatInstOffsets(false),
diff --git a/lib/Target/AMDGPU/AMDGPUSubtarget.h b/lib/Target/AMDGPU/AMDGPUSubtarget.h
index 886aca42b6cb1eb3a8a1bbe9747b423b61f69906..789fd2af4b91ff98804876364a7b0b5209291bdc 100644
--- a/lib/Target/AMDGPU/AMDGPUSubtarget.h
+++ b/lib/Target/AMDGPU/AMDGPUSubtarget.h
@@ -1,9 +1,8 @@
 //=====-- AMDGPUSubtarget.h - Define Subtarget for AMDGPU ------*- C++ -*-====//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //==-----------------------------------------------------------------------===//
 //
@@ -246,26 +245,6 @@ public:
 class GCNSubtarget : public AMDGPUGenSubtargetInfo,
                      public AMDGPUSubtarget {
 public:
-  enum {
-    ISAVersion0_0_0,
-    ISAVersion6_0_0,
-    ISAVersion6_0_1,
-    ISAVersion7_0_0,
-    ISAVersion7_0_1,
-    ISAVersion7_0_2,
-    ISAVersion7_0_3,
-    ISAVersion7_0_4,
-    ISAVersion8_0_1,
-    ISAVersion8_0_2,
-    ISAVersion8_0_3,
-    ISAVersion8_1_0,
-    ISAVersion9_0_0,
-    ISAVersion9_0_2,
-    ISAVersion9_0_4,
-    ISAVersion9_0_6,
-    ISAVersion9_0_9,
-  };
-
   enum TrapHandlerAbi {
     TrapHandlerAbiNone = 0,
     TrapHandlerAbiHsa = 1
@@ -297,7 +276,6 @@ protected:
   // Basic subtarget description.
   Triple TargetTriple;
   unsigned Gen;
-  unsigned IsaVersion;
   InstrItineraryData InstrItins;
   int LDSBankCount;
   unsigned MaxPrivateElementSize;
@@ -326,6 +304,7 @@ protected:
   bool EnableUnsafeDSOffsetFolding;
   bool EnableSIScheduler;
   bool EnableDS128;
+  bool EnablePRTStrictNull;
   bool DumpCode;
 
   // Subtarget statically properties set by tablegen
@@ -353,6 +332,8 @@ protected:
   bool HasDPP;
   bool HasR128A16;
   bool HasDLInsts;
+  bool HasDot1Insts;
+  bool HasDot2Insts;
   bool EnableSRAMECC;
   bool FlatAddressSpace;
   bool FlatInstOffsets;
@@ -576,6 +557,12 @@ public:
     return getGeneration() < AMDGPUSubtarget::GFX9;
   }
 
+  /// \returns If target requires PRT Struct NULL support (zero result registers
+  /// for sparse texture support).
+  bool usePRTStrictNull() const {
+    return EnablePRTStrictNull;
+  }
+
   bool hasAutoWaitcntBeforeBarrier() const {
     return AutoWaitcntBeforeBarrier;
   }
@@ -680,6 +667,14 @@ public:
     return HasDLInsts;
   }
 
+  bool hasDot1Insts() const {
+    return HasDot1Insts;
+  }
+
+  bool hasDot2Insts() const {
+    return HasDot2Insts;
+  }
+
   bool isSRAMECCEnabled() const {
     return EnableSRAMECC;
   }
@@ -780,6 +775,9 @@ public:
     return HasScalarAtomics;
   }
 
+  bool hasLDSFPAtomics() const {
+    return VIInsts;
+  }
 
   bool hasDPP() const {
     return HasDPP;
@@ -817,6 +815,11 @@ public:
     return getGeneration() != AMDGPUSubtarget::SOUTHERN_ISLANDS;
   }
 
+  // \returns true if the subtarget supports DWORDX3 load/store instructions.
+  bool hasDwordx3LoadStores() const {
+    return CIInsts;
+  }
+
   bool hasSMovFedHazard() const {
     return getGeneration() >= AMDGPUSubtarget::GFX9;
   }
diff --git a/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
index 70d365f4ad75d51734bd338cf3882868785e1251..f9bd500281080afdcc24cb120365dfdc922f73d0 100644
--- a/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
+++ b/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
@@ -1,9 +1,8 @@
 //===-- AMDGPUTargetMachine.cpp - TargetMachine for hw codegen targets-----===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -109,7 +108,7 @@ static cl::opt<bool> EnableSDWAPeephole(
 static cl::opt<bool> EnableDPPCombine(
   "amdgpu-dpp-combine",
   cl::desc("Enable DPP combiner"),
-  cl::init(false));
+  cl::init(true));
 
 // Enable address space based alias analysis
 static cl::opt<bool> EnableAMDGPUAliasAnalysis("enable-amdgpu-aa", cl::Hidden,
@@ -825,6 +824,7 @@ bool GCNPassConfig::addInstSelector() {
   addPass(&SIFixSGPRCopiesID);
   addPass(createSILowerI1CopiesPass());
   addPass(createSIFixupVectorISelPass());
+  addPass(createSIAddIMGInitPass());
   return false;
 }
 
diff --git a/lib/Target/AMDGPU/AMDGPUTargetMachine.h b/lib/Target/AMDGPU/AMDGPUTargetMachine.h
index 62fbe71d19023a68f7827eda4a080a5b8dac2d91..b8805f08bc0a81d31442a00857a14b0a1d2d89a3 100644
--- a/lib/Target/AMDGPU/AMDGPUTargetMachine.h
+++ b/lib/Target/AMDGPU/AMDGPUTargetMachine.h
@@ -1,9 +1,8 @@
 //===-- AMDGPUTargetMachine.h - AMDGPU TargetMachine Interface --*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -15,7 +14,6 @@
 #ifndef LLVM_LIB_TARGET_AMDGPU_AMDGPUTARGETMACHINE_H
 #define LLVM_LIB_TARGET_AMDGPU_AMDGPUTARGETMACHINE_H
 
-#include "AMDGPUIntrinsicInfo.h"
 #include "AMDGPUSubtarget.h"
 #include "llvm/ADT/Optional.h"
 #include "llvm/ADT/StringMap.h"
@@ -95,7 +93,6 @@ public:
 
 class GCNTargetMachine final : public AMDGPUTargetMachine {
 private:
-  AMDGPUIntrinsicInfo IntrinsicInfo;
   mutable StringMap<std::unique_ptr<GCNSubtarget>> SubtargetMap;
 
 public:
@@ -110,10 +107,6 @@ public:
 
   TargetTransformInfo getTargetTransformInfo(const Function &F) override;
 
-  const AMDGPUIntrinsicInfo *getIntrinsicInfo() const override {
-    return &IntrinsicInfo;
-  }
-
   bool useIPRA() const override {
     return true;
   }
diff --git a/lib/Target/AMDGPU/AMDGPUTargetObjectFile.cpp b/lib/Target/AMDGPU/AMDGPUTargetObjectFile.cpp
index 7a720717cc800857a67e01da6e8423ad1e2fd2f4..6569980d2c75c1090a572a80f98d48aff3806bc3 100644
--- a/lib/Target/AMDGPU/AMDGPUTargetObjectFile.cpp
+++ b/lib/Target/AMDGPU/AMDGPUTargetObjectFile.cpp
@@ -1,9 +1,8 @@
 //===-- AMDGPUHSATargetObjectFile.cpp - AMDGPU Object Files ---------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -34,7 +33,7 @@ MCSection *AMDGPUTargetObjectFile::getExplicitSectionGlobal(
     const GlobalObject *GO, SectionKind SK, const TargetMachine &TM) const {
   // Set metadata access for the explicit section
   StringRef SectionName = GO->getSection();
-  if (SectionName.startswith(".AMDGPU.metadata."))
+  if (SectionName.startswith(".AMDGPU.comment."))
     SK = SectionKind::getMetadata();
 
   return TargetLoweringObjectFileELF::getExplicitSectionGlobal(GO, SK, TM);
diff --git a/lib/Target/AMDGPU/AMDGPUTargetObjectFile.h b/lib/Target/AMDGPU/AMDGPUTargetObjectFile.h
index a4ae1a2c18c26bdbdace155b7923ca9be4aa83e0..819bebb7932d752b8f0b5394abec251385f61e8f 100644
--- a/lib/Target/AMDGPU/AMDGPUTargetObjectFile.h
+++ b/lib/Target/AMDGPU/AMDGPUTargetObjectFile.h
@@ -1,9 +1,8 @@
 //===-- AMDGPUTargetObjectFile.h - AMDGPU  Object Info ----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
diff --git a/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp b/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
index 11e4ba4b5010dc3655d1014dfd81596241488845..626c4f35f17b33f71f8f868f33377b02b7728fee 100644
--- a/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
+++ b/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
@@ -1,9 +1,8 @@
 //===- AMDGPUTargetTransformInfo.cpp - AMDGPU specific TTI pass -----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -308,6 +307,8 @@ bool GCNTTIImpl::getTgtMemIntrinsic(IntrinsicInst *Inst,
   switch (Inst->getIntrinsicID()) {
   case Intrinsic::amdgcn_atomic_inc:
   case Intrinsic::amdgcn_atomic_dec:
+  case Intrinsic::amdgcn_ds_ordered_add:
+  case Intrinsic::amdgcn_ds_ordered_swap:
   case Intrinsic::amdgcn_ds_fadd:
   case Intrinsic::amdgcn_ds_fmin:
   case Intrinsic::amdgcn_ds_fmax: {
@@ -577,6 +578,8 @@ bool GCNTTIImpl::isAlwaysUniform(const Value *V) const {
       return false;
     case Intrinsic::amdgcn_readfirstlane:
     case Intrinsic::amdgcn_readlane:
+    case Intrinsic::amdgcn_icmp:
+    case Intrinsic::amdgcn_fcmp:
       return true;
     }
   }
diff --git a/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h b/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h
index 397c5c6fa6fbe4fe510f3d9e098494cbf7a26d15..e5840062cb81ab1c58373c9e8669b509f78113af 100644
--- a/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h
+++ b/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h
@@ -1,9 +1,8 @@
 //===- AMDGPUTargetTransformInfo.h - AMDGPU specific TTI --------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -85,6 +84,7 @@ class GCNTTIImpl final : public BasicTTIImplBase<GCNTTIImpl> {
     AMDGPU::FeatureSGPRInitBug,
     AMDGPU::FeatureXNACK,
     AMDGPU::FeatureTrapHandler,
+    AMDGPU::FeatureCodeObjectV3,
 
     // Perf-tuning features
     AMDGPU::FeatureFastFMAF32,
diff --git a/lib/Target/AMDGPU/AMDGPUUnifyDivergentExitNodes.cpp b/lib/Target/AMDGPU/AMDGPUUnifyDivergentExitNodes.cpp
index ced3f6f567e2fce16b9d11ab7487b31a5ae6ac1b..6550dc362b0de15e2553a580b00a0231e4f949fa 100644
--- a/lib/Target/AMDGPU/AMDGPUUnifyDivergentExitNodes.cpp
+++ b/lib/Target/AMDGPU/AMDGPUUnifyDivergentExitNodes.cpp
@@ -1,9 +1,8 @@
 //===- AMDGPUUnifyDivergentExitNodes.cpp ----------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AMDGPU/AMDGPUUnifyMetadata.cpp b/lib/Target/AMDGPU/AMDGPUUnifyMetadata.cpp
index 1f6d9234c1ed9dcaff1794556770e90025a3f076..d4401a22a1adb850b2a799089ebc36c9b2a0a01b 100644
--- a/lib/Target/AMDGPU/AMDGPUUnifyMetadata.cpp
+++ b/lib/Target/AMDGPU/AMDGPUUnifyMetadata.cpp
@@ -1,9 +1,8 @@
 //===- AMDGPUUnifyMetadata.cpp - Unify OpenCL metadata --------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AMDGPU/AMDILCFGStructurizer.cpp b/lib/Target/AMDGPU/AMDILCFGStructurizer.cpp
index 11cd49e5b3dcf9a9b44a886d7740cf473a02d97e..12f2e9519c9ecca0470c0ba2e03b9cee6d4af1af 100644
--- a/lib/Target/AMDGPU/AMDILCFGStructurizer.cpp
+++ b/lib/Target/AMDGPU/AMDILCFGStructurizer.cpp
@@ -1,9 +1,8 @@
 //===- AMDILCFGStructurizer.cpp - CFG Structurizer ------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //==-----------------------------------------------------------------------===//
 
diff --git a/lib/Target/AMDGPU/AMDKernelCodeT.h b/lib/Target/AMDGPU/AMDKernelCodeT.h
index 289642aaa2d07a156569838e74d6210f9888af0d..9a2cb95bf9634512fc545601d1b39d2df7e7e34e 100644
--- a/lib/Target/AMDGPU/AMDKernelCodeT.h
+++ b/lib/Target/AMDGPU/AMDKernelCodeT.h
@@ -1,9 +1,8 @@
 //===-- AMDGPUKernelCodeT.h - Print AMDGPU assembly code ---------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file AMDKernelCodeT.h
diff --git a/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp b/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
index 3f9af27a2e5e1d2fec59755ae1896ff42b2abf50..0f0731bfc22f0dc46a413dd7416891c832c8e59e 100644
--- a/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
+++ b/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
@@ -1,9 +1,8 @@
 //===- AMDGPUAsmParser.cpp - Parse SI asm to MCInst instructions ----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -924,6 +923,10 @@ public:
         MCSymbol *Sym =
             Ctx.getOrCreateSymbol(Twine(".amdgcn.gfx_generation_number"));
         Sym->setVariableValue(MCConstantExpr::create(ISA.Major, Ctx));
+        Sym = Ctx.getOrCreateSymbol(Twine(".amdgcn.gfx_generation_minor"));
+        Sym->setVariableValue(MCConstantExpr::create(ISA.Minor, Ctx));
+        Sym = Ctx.getOrCreateSymbol(Twine(".amdgcn.gfx_generation_stepping"));
+        Sym->setVariableValue(MCConstantExpr::create(ISA.Stepping, Ctx));
       } else {
         MCSymbol *Sym =
             Ctx.getOrCreateSymbol(Twine(".option.machine_version_major"));
@@ -1084,6 +1087,7 @@ private:
   OperandMatchResultTy parseExpTgtImpl(StringRef Str, uint8_t &Val);
 
   bool validateInstruction(const MCInst &Inst, const SMLoc &IDLoc);
+  bool validateSOPLiteral(const MCInst &Inst) const;
   bool validateConstantBusLimitations(const MCInst &Inst);
   bool validateEarlyClobberLimitations(const MCInst &Inst);
   bool validateIntClampSupported(const MCInst &Inst);
@@ -1091,6 +1095,7 @@ private:
   bool validateMIMGGatherDMask(const MCInst &Inst);
   bool validateMIMGDataSize(const MCInst &Inst);
   bool validateMIMGD16(const MCInst &Inst);
+  bool validateLdsDirect(const MCInst &Inst);
   bool usesConstantBus(const MCInst &Inst, unsigned OpIdx);
   bool isInlineConstant(const MCInst &Inst, unsigned OpIdx) const;
   unsigned findImplicitSGPRReadInVOP(const MCInst &Inst) const;
@@ -1595,6 +1600,8 @@ static unsigned getSpecialRegForName(StringRef RegName) {
     .Case("vcc", AMDGPU::VCC)
     .Case("flat_scratch", AMDGPU::FLAT_SCR)
     .Case("xnack_mask", AMDGPU::XNACK_MASK)
+    .Case("lds_direct", AMDGPU::LDS_DIRECT)
+    .Case("src_lds_direct", AMDGPU::LDS_DIRECT)
     .Case("m0", AMDGPU::M0)
     .Case("scc", AMDGPU::SCC)
     .Case("tba", AMDGPU::TBA)
@@ -2461,8 +2468,131 @@ bool AMDGPUAsmParser::validateMIMGD16(const MCInst &Inst) {
   return true;
 }
 
+bool AMDGPUAsmParser::validateLdsDirect(const MCInst &Inst) {
+
+  using namespace SIInstrFlags;
+  const unsigned Opcode = Inst.getOpcode();
+  const MCInstrDesc &Desc = MII.get(Opcode);
+
+  // lds_direct register is defined so that it can be used
+  // with 9-bit operands only. Ignore encodings which do not accept these.
+  if ((Desc.TSFlags & (VOP1 | VOP2 | VOP3 | VOPC | VOP3P | SIInstrFlags::SDWA)) == 0)
+    return true;
+
+  const int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0);
+  const int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src1);
+  const int Src2Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src2);
+
+  const int SrcIndices[] = { Src1Idx, Src2Idx };
+
+  // lds_direct cannot be specified as either src1 or src2.
+  for (int SrcIdx : SrcIndices) {
+    if (SrcIdx == -1) break;
+    const MCOperand &Src = Inst.getOperand(SrcIdx);
+    if (Src.isReg() && Src.getReg() == LDS_DIRECT) {
+      return false;
+    }
+  }
+
+  if (Src0Idx == -1)
+    return true;
+
+  const MCOperand &Src = Inst.getOperand(Src0Idx);
+  if (!Src.isReg() || Src.getReg() != LDS_DIRECT)
+    return true;
+
+  // lds_direct is specified as src0. Check additional limitations.
+
+  // FIXME: This is a workaround for bug 37943
+  // which allows 64-bit VOP3 opcodes use 32-bit operands.
+  if (AMDGPU::getRegOperandSize(getMRI(), Desc, Src0Idx) != 4)
+    return false;
+
+  // Documentation does not disable lds_direct for SDWA, but SP3 assembler does.
+  // FIXME: This inconsistence needs to be investigated further.
+  if (Desc.TSFlags & SIInstrFlags::SDWA)
+    return false;
+
+  // The following opcodes do not accept lds_direct which is explicitly stated
+  // in AMD documentation. However SP3 disables lds_direct for most other 'rev'
+  // opcodes as well (e.g. for v_subrev_u32 but not for v_subrev_f32).
+  // FIXME: This inconsistence needs to be investigated further.
+  switch (Opcode) {
+  case AMDGPU::V_LSHLREV_B32_e32_si:
+  case AMDGPU::V_LSHLREV_B32_e64_si:
+  case AMDGPU::V_LSHLREV_B16_e32_vi:
+  case AMDGPU::V_LSHLREV_B16_e64_vi:
+  case AMDGPU::V_LSHLREV_B32_e32_vi:
+  case AMDGPU::V_LSHLREV_B32_e64_vi:
+  case AMDGPU::V_LSHLREV_B64_vi:
+  case AMDGPU::V_LSHRREV_B32_e32_si:
+  case AMDGPU::V_LSHRREV_B32_e64_si:
+  case AMDGPU::V_LSHRREV_B16_e32_vi:
+  case AMDGPU::V_LSHRREV_B16_e64_vi:
+  case AMDGPU::V_LSHRREV_B32_e32_vi:
+  case AMDGPU::V_LSHRREV_B32_e64_vi:
+  case AMDGPU::V_LSHRREV_B64_vi:
+  case AMDGPU::V_ASHRREV_I32_e64_si:
+  case AMDGPU::V_ASHRREV_I32_e32_si:
+  case AMDGPU::V_ASHRREV_I16_e32_vi:
+  case AMDGPU::V_ASHRREV_I16_e64_vi:
+  case AMDGPU::V_ASHRREV_I32_e32_vi:
+  case AMDGPU::V_ASHRREV_I32_e64_vi:
+  case AMDGPU::V_ASHRREV_I64_vi:
+  case AMDGPU::V_PK_LSHLREV_B16_vi:
+  case AMDGPU::V_PK_LSHRREV_B16_vi:
+  case AMDGPU::V_PK_ASHRREV_I16_vi:
+    return false;
+  default:
+    return true;
+  }
+}
+
+bool AMDGPUAsmParser::validateSOPLiteral(const MCInst &Inst) const {
+  unsigned Opcode = Inst.getOpcode();
+  const MCInstrDesc &Desc = MII.get(Opcode);
+  if (!(Desc.TSFlags & (SIInstrFlags::SOP2 | SIInstrFlags::SOPC)))
+    return true;
+
+  const int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0);
+  const int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src1);
+
+  const int OpIndices[] = { Src0Idx, Src1Idx };
+
+  unsigned NumLiterals = 0;
+  uint32_t LiteralValue;
+
+  for (int OpIdx : OpIndices) {
+    if (OpIdx == -1) break;
+
+    const MCOperand &MO = Inst.getOperand(OpIdx);
+    if (MO.isImm() &&
+        // Exclude special imm operands (like that used by s_set_gpr_idx_on)
+        AMDGPU::isSISrcOperand(Desc, OpIdx) &&
+        !isInlineConstant(Inst, OpIdx)) {
+      uint32_t Value = static_cast<uint32_t>(MO.getImm());
+      if (NumLiterals == 0 || LiteralValue != Value) {
+        LiteralValue = Value;
+        ++NumLiterals;
+      }
+    }
+  }
+
+  return NumLiterals <= 1;
+}
+
 bool AMDGPUAsmParser::validateInstruction(const MCInst &Inst,
                                           const SMLoc &IDLoc) {
+  if (!validateLdsDirect(Inst)) {
+    Error(IDLoc,
+      "invalid use of lds_direct");
+    return false;
+  }
+  if (!validateSOPLiteral(Inst)) {
+    Error(IDLoc,
+      "only one literal operand is allowed");
+    return false;
+  }
   if (!validateConstantBusLimitations(Inst)) {
     Error(IDLoc,
       "invalid operand (violates constant bus restrictions)");
diff --git a/lib/Target/AMDGPU/AsmParser/LLVMBuild.txt b/lib/Target/AMDGPU/AsmParser/LLVMBuild.txt
index dab0c6f585af1d6a2d7f13a5741ece4a1d53063a..adc5fd426307475b6329f1e9a2ea5b50f54ae3c6 100644
--- a/lib/Target/AMDGPU/AsmParser/LLVMBuild.txt
+++ b/lib/Target/AMDGPU/AsmParser/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/Target/AMDGPU/AsmParser/LLVMBuild.txt -------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Target/AMDGPU/BUFInstructions.td b/lib/Target/AMDGPU/BUFInstructions.td
index 51c2abeac2ffb07f0b495a81a48cbbf7113e7bd2..6acd7eb44bdfb429bc5045e140ccaef3607919f4 100644
--- a/lib/Target/AMDGPU/BUFInstructions.td
+++ b/lib/Target/AMDGPU/BUFInstructions.td
@@ -1,9 +1,8 @@
 //===-- BUFInstructions.td - Buffer Instruction Defintions ----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/AMDGPU/CMakeLists.txt b/lib/Target/AMDGPU/CMakeLists.txt
index 7d1219914823d558b2d4dd28ffcd74cc8fee1ec3..442946c4543e1c6a0b2fc4561920e2bfd4bb6dbb 100644
--- a/lib/Target/AMDGPU/CMakeLists.txt
+++ b/lib/Target/AMDGPU/CMakeLists.txt
@@ -6,8 +6,6 @@ tablegen(LLVM AMDGPUGenCallingConv.inc -gen-callingconv)
 tablegen(LLVM AMDGPUGenDAGISel.inc -gen-dag-isel)
 tablegen(LLVM AMDGPUGenDisassemblerTables.inc -gen-disassembler)
 tablegen(LLVM AMDGPUGenInstrInfo.inc -gen-instr-info)
-tablegen(LLVM AMDGPUGenIntrinsicEnums.inc -gen-tgt-intrinsic-enums)
-tablegen(LLVM AMDGPUGenIntrinsicImpl.inc -gen-tgt-intrinsic-impl)
 tablegen(LLVM AMDGPUGenMCCodeEmitter.inc -gen-emitter)
 tablegen(LLVM AMDGPUGenMCPseudoLowering.inc -gen-pseudo-lowering)
 tablegen(LLVM AMDGPUGenRegisterBank.inc -gen-register-bank)
@@ -45,7 +43,6 @@ add_llvm_target(AMDGPUCodeGen
   AMDGPUHSAMetadataStreamer.cpp
   AMDGPUInstrInfo.cpp
   AMDGPUInstructionSelector.cpp
-  AMDGPUIntrinsicInfo.cpp
   AMDGPUISelDAGToDAG.cpp
   AMDGPUISelLowering.cpp
   AMDGPULegalizerInfo.cpp
@@ -93,6 +90,7 @@ add_llvm_target(AMDGPUCodeGen
   R600OptimizeVectorRegisters.cpp
   R600Packetizer.cpp
   R600RegisterInfo.cpp
+  SIAddIMGInit.cpp
   SIAnnotateControlFlow.cpp
   SIDebuggerInsertNops.cpp
   SIFixSGPRCopies.cpp
diff --git a/lib/Target/AMDGPU/CaymanInstructions.td b/lib/Target/AMDGPU/CaymanInstructions.td
index ae40c6387982bdab32a49bcd9add8dd695edf8d8..1a526675164a039627b5b8529836e8955015daa9 100644
--- a/lib/Target/AMDGPU/CaymanInstructions.td
+++ b/lib/Target/AMDGPU/CaymanInstructions.td
@@ -1,9 +1,8 @@
 //===-- CaymanInstructions.td - CM Instruction defs  -------*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AMDGPU/DSInstructions.td b/lib/Target/AMDGPU/DSInstructions.td
index 31d2ebef481d20430213c207eeb718e5fc1d057d..c099a29d1c8c35e1860c1e2edb3f629e84447253 100644
--- a/lib/Target/AMDGPU/DSInstructions.td
+++ b/lib/Target/AMDGPU/DSInstructions.td
@@ -1,9 +1,8 @@
 //===-- DSInstructions.td - DS Instruction Defintions ---------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -11,8 +10,6 @@ class DS_Pseudo <string opName, dag outs, dag ins, string asmOps, list<dag> patt
   InstSI <outs, ins, "", pattern>,
   SIMCInstr <opName, SIEncodingFamily.NONE> {
 
-  let SubtargetPredicate = isGCN;
-
   let LGKM_CNT = 1;
   let DS = 1;
   let Size = 8;
@@ -817,6 +814,11 @@ defm : DSAtomicRetPat_mc<DS_MAX_RTN_U64, i64, "atomic_load_umax_local">;
 
 defm : DSAtomicCmpXChg_mc<DS_CMPST_RTN_B64, i64, "atomic_cmp_swap_local">;
 
+def : Pat <
+  (SIds_ordered_count i32:$value, i16:$offset),
+  (DS_ORDERED_COUNT $value, (as_i16imm $offset))
+>;
+
 //===----------------------------------------------------------------------===//
 // Real instructions
 //===----------------------------------------------------------------------===//
diff --git a/lib/Target/AMDGPU/Disassembler/AMDGPUDisassembler.cpp b/lib/Target/AMDGPU/Disassembler/AMDGPUDisassembler.cpp
index f3de903f21b2aeaab8833a4bae1487aff266c973..9ae1bcd44ec71cddf93b1a7c6f302aa07569f03f 100644
--- a/lib/Target/AMDGPU/Disassembler/AMDGPUDisassembler.cpp
+++ b/lib/Target/AMDGPU/Disassembler/AMDGPUDisassembler.cpp
@@ -1,9 +1,8 @@
 //===- AMDGPUDisassembler.cpp - Disassembler for AMDGPU ISA ---------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -782,6 +781,7 @@ MCOperand AMDGPUDisassembler::decodeSpecialReg32(unsigned Val) const {
     // ToDo: no support for execz register
   case 252: break;
   case 253: return createRegOperand(SCC);
+  case 254: return createRegOperand(LDS_DIRECT);
   default: break;
   }
   return errOperand(Val, "unknown operand encoding " + Twine(Val));
diff --git a/lib/Target/AMDGPU/Disassembler/AMDGPUDisassembler.h b/lib/Target/AMDGPU/Disassembler/AMDGPUDisassembler.h
index 75cfc5e112820aea42928744b18c54bb87b7e98d..4e1c75ff57b3030d383f5e624fa03f4d42435337 100644
--- a/lib/Target/AMDGPU/Disassembler/AMDGPUDisassembler.h
+++ b/lib/Target/AMDGPU/Disassembler/AMDGPUDisassembler.h
@@ -1,9 +1,8 @@
 //===- AMDGPUDisassembler.hpp - Disassembler for AMDGPU ISA -----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AMDGPU/Disassembler/LLVMBuild.txt b/lib/Target/AMDGPU/Disassembler/LLVMBuild.txt
index c9005f8a7884fada8ce56d3bb394ab680800eee3..4163e3800297b1f55cf367fe3403e0398356a969 100644
--- a/lib/Target/AMDGPU/Disassembler/LLVMBuild.txt
+++ b/lib/Target/AMDGPU/Disassembler/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/Target/AMDGPU/Disassembler/LLVMBuild.txt ------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Target/AMDGPU/EvergreenInstructions.td b/lib/Target/AMDGPU/EvergreenInstructions.td
index 944f4ffe598d41972d8260ce3caa90e5b842edc4..0550092ce1d6e6c422270b70a41ea4ceb3418f86 100644
--- a/lib/Target/AMDGPU/EvergreenInstructions.td
+++ b/lib/Target/AMDGPU/EvergreenInstructions.td
@@ -1,9 +1,8 @@
 //===-- EvergreenInstructions.td - EG Instruction defs  ----*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AMDGPU/FLATInstructions.td b/lib/Target/AMDGPU/FLATInstructions.td
index 44040d352e6a86ef5e6fa979cb6d815c4a5e8872..2179b21e0a6aed11f056376009024a70a431133b 100644
--- a/lib/Target/AMDGPU/FLATInstructions.td
+++ b/lib/Target/AMDGPU/FLATInstructions.td
@@ -1,9 +1,8 @@
 //===-- FLATInstructions.td - FLAT Instruction Defintions -----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/AMDGPU/GCNDPPCombine.cpp b/lib/Target/AMDGPU/GCNDPPCombine.cpp
index 56071d0d23744dfd0e887e8b6f13cff377cc5c9f..430cf480eb190973b3572ebf335ab5e1fe8311d0 100644
--- a/lib/Target/AMDGPU/GCNDPPCombine.cpp
+++ b/lib/Target/AMDGPU/GCNDPPCombine.cpp
@@ -1,37 +1,40 @@
 //=======- GCNDPPCombine.cpp - optimization for DPP instructions ---==========//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 // The pass combines V_MOV_B32_dpp instruction with its VALU uses as a DPP src0
-// operand.If any of the use instruction cannot be combined with the mov the
+// operand. If any of the use instruction cannot be combined with the mov the
 // whole sequence is reverted.
 //
 // $old = ...
 // $dpp_value = V_MOV_B32_dpp $old, $vgpr_to_be_read_from_other_lane,
-//                            dpp_controls..., $bound_ctrl
-// $res = VALU $dpp_value, ...
+//                            dpp_controls..., $row_mask, $bank_mask, $bound_ctrl
+// $res = VALU $dpp_value [, src1]
 //
 // to
 //
-// $res = VALU_DPP $folded_old, $vgpr_to_be_read_from_other_lane, ...,
-//                 dpp_controls..., $folded_bound_ctrl
+// $res = VALU_DPP $combined_old, $vgpr_to_be_read_from_other_lane, [src1,]
+//                 dpp_controls..., $row_mask, $bank_mask, $combined_bound_ctrl
 //
 // Combining rules :
 //
-// $bound_ctrl is DPP_BOUND_ZERO, $old is any
-// $bound_ctrl is DPP_BOUND_OFF, $old is 0
+// if $row_mask and $bank_mask are fully enabled (0xF) and
+//    $bound_ctrl==DPP_BOUND_ZERO or $old==0
+// -> $combined_old = undef,
+//    $combined_bound_ctrl = DPP_BOUND_ZERO
 //
-// ->$folded_old = undef, $folded_bound_ctrl = DPP_BOUND_ZERO
-// $bound_ctrl is DPP_BOUND_OFF, $old is undef
+// if the VALU op is binary and
+//    $bound_ctrl==DPP_BOUND_OFF and
+//    $old==identity value (immediate) for the VALU op
+// -> $combined_old = src1,
+//    $combined_bound_ctrl = DPP_BOUND_OFF
 //
-// ->$folded_old = undef, $folded_bound_ctrl = DPP_BOUND_OFF
-// $bound_ctrl is DPP_BOUND_OFF, $old is foldable
+// Othervise cancel.
 //
-// ->$folded_old = folded value, $folded_bound_ctrl = DPP_BOUND_OFF
+// The mov_dpp instruction should recide in the same BB as all it's uses
 //===----------------------------------------------------------------------===//
 
 #include "AMDGPU.h"
@@ -67,20 +70,16 @@ class GCNDPPCombine : public MachineFunctionPass {
 
   MachineOperand *getOldOpndValue(MachineOperand &OldOpnd) const;
 
-  RegSubRegPair foldOldOpnd(MachineInstr &OrigMI,
-                            RegSubRegPair OldOpndVGPR,
-                            MachineOperand &OldOpndValue) const;
-
   MachineInstr *createDPPInst(MachineInstr &OrigMI,
                               MachineInstr &MovMI,
-                              RegSubRegPair OldOpndVGPR,
+                              RegSubRegPair CombOldVGPR,
                               MachineOperand *OldOpnd,
-                              bool BoundCtrlZero) const;
+                              bool CombBCZ) const;
 
   MachineInstr *createDPPInst(MachineInstr &OrigMI,
                               MachineInstr &MovMI,
-                              RegSubRegPair OldOpndVGPR,
-                              bool BoundCtrlZero) const;
+                              RegSubRegPair CombOldVGPR,
+                              bool CombBCZ) const;
 
   bool hasNoImmOrEqual(MachineInstr &MI,
                        unsigned OpndName,
@@ -153,8 +152,8 @@ MachineOperand *GCNDPPCombine::getOldOpndValue(MachineOperand &OldOpnd) const {
 
 MachineInstr *GCNDPPCombine::createDPPInst(MachineInstr &OrigMI,
                                            MachineInstr &MovMI,
-                                           RegSubRegPair OldOpndVGPR,
-                                           bool BoundCtrlZero) const {
+                                           RegSubRegPair CombOldVGPR,
+                                           bool CombBCZ) const {
   assert(MovMI.getOpcode() == AMDGPU::V_MOV_B32_dpp);
   assert(TII->getNamedOperand(MovMI, AMDGPU::OpName::vdst)->getReg() ==
          TII->getNamedOperand(OrigMI, AMDGPU::OpName::src0)->getReg());
@@ -178,9 +177,15 @@ MachineInstr *GCNDPPCombine::createDPPInst(MachineInstr &OrigMI,
     const int OldIdx = AMDGPU::getNamedOperandIdx(DPPOp, AMDGPU::OpName::old);
     if (OldIdx != -1) {
       assert(OldIdx == NumOperands);
-      assert(isOfRegClass(OldOpndVGPR, AMDGPU::VGPR_32RegClass, *MRI));
-      DPPInst.addReg(OldOpndVGPR.Reg, 0, OldOpndVGPR.SubReg);
+      assert(isOfRegClass(CombOldVGPR, AMDGPU::VGPR_32RegClass, *MRI));
+      DPPInst.addReg(CombOldVGPR.Reg, 0, CombOldVGPR.SubReg);
       ++NumOperands;
+    } else {
+      // TODO: this discards MAC/FMA instructions for now, let's add it later
+      LLVM_DEBUG(dbgs() << "  failed: no old operand in DPP instruction,"
+                           " TBD\n");
+      Fail = true;
+      break;
     }
 
     if (auto *Mod0 = TII->getNamedOperand(OrigMI,
@@ -199,6 +204,7 @@ MachineInstr *GCNDPPCombine::createDPPInst(MachineInstr &OrigMI,
       break;
     }
     DPPInst.add(*Src0);
+    DPPInst->getOperand(NumOperands).setIsKill(false);
     ++NumOperands;
 
     if (auto *Mod1 = TII->getNamedOperand(OrigMI,
@@ -231,7 +237,7 @@ MachineInstr *GCNDPPCombine::createDPPInst(MachineInstr &OrigMI,
     DPPInst.add(*TII->getNamedOperand(MovMI, AMDGPU::OpName::dpp_ctrl));
     DPPInst.add(*TII->getNamedOperand(MovMI, AMDGPU::OpName::row_mask));
     DPPInst.add(*TII->getNamedOperand(MovMI, AMDGPU::OpName::bank_mask));
-    DPPInst.addImm(BoundCtrlZero ? 1 : 0);
+    DPPInst.addImm(CombBCZ ? 1 : 0);
   } while (false);
 
   if (Fail) {
@@ -242,64 +248,68 @@ MachineInstr *GCNDPPCombine::createDPPInst(MachineInstr &OrigMI,
   return DPPInst.getInstr();
 }
 
-GCNDPPCombine::RegSubRegPair
-GCNDPPCombine::foldOldOpnd(MachineInstr &OrigMI,
-                           RegSubRegPair OldOpndVGPR,
-                           MachineOperand &OldOpndValue) const {
-  assert(OldOpndValue.isImm());
-  switch (OrigMI.getOpcode()) {
+static bool isIdentityValue(unsigned OrigMIOp, MachineOperand *OldOpnd) {
+  assert(OldOpnd->isImm());
+  switch (OrigMIOp) {
   default: break;
+  case AMDGPU::V_ADD_U32_e32:
+  case AMDGPU::V_ADD_I32_e32:
+  case AMDGPU::V_OR_B32_e32:
+  case AMDGPU::V_SUBREV_U32_e32:
+  case AMDGPU::V_SUBREV_I32_e32:
   case AMDGPU::V_MAX_U32_e32:
-    if (OldOpndValue.getImm() == std::numeric_limits<uint32_t>::max())
-      return OldOpndVGPR;
+  case AMDGPU::V_XOR_B32_e32:
+    if (OldOpnd->getImm() == 0)
+      return true;
     break;
-  case AMDGPU::V_MAX_I32_e32:
-    if (OldOpndValue.getImm() == std::numeric_limits<int32_t>::max())
-      return OldOpndVGPR;
+  case AMDGPU::V_AND_B32_e32:
+  case AMDGPU::V_MIN_U32_e32:
+    if (static_cast<uint32_t>(OldOpnd->getImm()) ==
+        std::numeric_limits<uint32_t>::max())
+      return true;
     break;
   case AMDGPU::V_MIN_I32_e32:
-    if (OldOpndValue.getImm() == std::numeric_limits<int32_t>::min())
-      return OldOpndVGPR;
+    if (static_cast<int32_t>(OldOpnd->getImm()) ==
+        std::numeric_limits<int32_t>::max())
+      return true;
+    break;
+  case AMDGPU::V_MAX_I32_e32:
+    if (static_cast<int32_t>(OldOpnd->getImm()) ==
+        std::numeric_limits<int32_t>::min())
+      return true;
     break;
-
   case AMDGPU::V_MUL_I32_I24_e32:
   case AMDGPU::V_MUL_U32_U24_e32:
-    if (OldOpndValue.getImm() == 1) {
-      auto *Src1 = TII->getNamedOperand(OrigMI, AMDGPU::OpName::src1);
-      assert(Src1 && Src1->isReg());
-      return getRegSubRegPair(*Src1);
-    }
+    if (OldOpnd->getImm() == 1)
+      return true;
     break;
   }
-  return RegSubRegPair();
+  return false;
 }
 
-// Cases to combine:
-//  $bound_ctrl is DPP_BOUND_ZERO, $old is any
-//  $bound_ctrl is DPP_BOUND_OFF, $old is 0
-//  -> $old = undef, $bound_ctrl = DPP_BOUND_ZERO
-
-//  $bound_ctrl is DPP_BOUND_OFF, $old is undef
-//  -> $old = undef, $bound_ctrl = DPP_BOUND_OFF
-
-//  $bound_ctrl is DPP_BOUND_OFF, $old is foldable
-//  -> $old = folded value, $bound_ctrl = DPP_BOUND_OFF
-
 MachineInstr *GCNDPPCombine::createDPPInst(MachineInstr &OrigMI,
                                            MachineInstr &MovMI,
-                                           RegSubRegPair OldOpndVGPR,
+                                           RegSubRegPair CombOldVGPR,
                                            MachineOperand *OldOpndValue,
-                                           bool BoundCtrlZero) const {
-  assert(OldOpndVGPR.Reg);
-  if (!BoundCtrlZero && OldOpndValue) {
-    assert(OldOpndValue->isImm());
-    OldOpndVGPR = foldOldOpnd(OrigMI, OldOpndVGPR, *OldOpndValue);
-    if (!OldOpndVGPR.Reg) {
-      LLVM_DEBUG(dbgs() << "  failed: old immediate cannot be folded\n");
+                                           bool CombBCZ) const {
+  assert(CombOldVGPR.Reg);
+  if (!CombBCZ && OldOpndValue && OldOpndValue->isImm()) {
+    auto *Src1 = TII->getNamedOperand(OrigMI, AMDGPU::OpName::src1);
+    if (!Src1 || !Src1->isReg()) {
+      LLVM_DEBUG(dbgs() << "  failed: no src1 or it isn't a register\n");
+      return nullptr;
+    }
+    if (!isIdentityValue(OrigMI.getOpcode(), OldOpndValue)) {
+      LLVM_DEBUG(dbgs() << "  failed: old immediate ins't an identity\n");
+      return nullptr;
+    }
+    CombOldVGPR = getRegSubRegPair(*Src1);
+    if (!isOfRegClass(CombOldVGPR, AMDGPU::VGPR_32RegClass, *MRI)) {
+      LLVM_DEBUG(dbgs() << "  failed: src1 isn't a VGPR32 register\n");
       return nullptr;
     }
   }
-  return createDPPInst(OrigMI, MovMI, OldOpndVGPR, BoundCtrlZero);
+  return createDPPInst(OrigMI, MovMI, CombOldVGPR, CombBCZ);
 }
 
 // returns true if MI doesn't have OpndName immediate operand or the
@@ -316,31 +326,64 @@ bool GCNDPPCombine::hasNoImmOrEqual(MachineInstr &MI, unsigned OpndName,
 
 bool GCNDPPCombine::combineDPPMov(MachineInstr &MovMI) const {
   assert(MovMI.getOpcode() == AMDGPU::V_MOV_B32_dpp);
+  LLVM_DEBUG(dbgs() << "\nDPP combine: " << MovMI);
+
+  auto *DstOpnd = TII->getNamedOperand(MovMI, AMDGPU::OpName::vdst);
+  assert(DstOpnd && DstOpnd->isReg());
+  auto DPPMovReg = DstOpnd->getReg();
+  if (!isEXECMaskConstantBetweenDefAndUses(DPPMovReg, *MRI)) {
+    LLVM_DEBUG(dbgs() << "  failed: EXEC mask should remain the same"
+                         " for all uses\n");
+    return false;
+  }
+
+  auto *RowMaskOpnd = TII->getNamedOperand(MovMI, AMDGPU::OpName::row_mask);
+  assert(RowMaskOpnd && RowMaskOpnd->isImm());
+  auto *BankMaskOpnd = TII->getNamedOperand(MovMI, AMDGPU::OpName::bank_mask);
+  assert(BankMaskOpnd && BankMaskOpnd->isImm());
+  const bool MaskAllLanes = RowMaskOpnd->getImm() == 0xF &&
+                            BankMaskOpnd->getImm() == 0xF;
+
   auto *BCZOpnd = TII->getNamedOperand(MovMI, AMDGPU::OpName::bound_ctrl);
   assert(BCZOpnd && BCZOpnd->isImm());
-  bool BoundCtrlZero = 0 != BCZOpnd->getImm();
-
-  LLVM_DEBUG(dbgs() << "\nDPP combine: " << MovMI);
+  bool BoundCtrlZero = BCZOpnd->getImm();
 
   auto *OldOpnd = TII->getNamedOperand(MovMI, AMDGPU::OpName::old);
   assert(OldOpnd && OldOpnd->isReg());
-  auto OldOpndVGPR = getRegSubRegPair(*OldOpnd);
-  auto *OldOpndValue = getOldOpndValue(*OldOpnd);
+
+  auto * const OldOpndValue = getOldOpndValue(*OldOpnd);
+  // OldOpndValue is either undef (IMPLICIT_DEF) or immediate or something else
+  // We could use: assert(!OldOpndValue || OldOpndValue->isImm())
+  // but the third option is used to distinguish undef from non-immediate
+  // to reuse IMPLICIT_DEF instruction later
   assert(!OldOpndValue || OldOpndValue->isImm() || OldOpndValue == OldOpnd);
-  if (OldOpndValue) {
-    if (BoundCtrlZero) {
-      OldOpndVGPR.Reg = AMDGPU::NoRegister; // should be undef, ignore old opnd
-      OldOpndValue = nullptr;
-    } else {
-      if (!OldOpndValue->isImm()) {
-        LLVM_DEBUG(dbgs() << "  failed: old operand isn't an imm or undef\n");
-        return false;
-      }
-      if (OldOpndValue->getImm() == 0) {
-        OldOpndVGPR.Reg = AMDGPU::NoRegister; // should be undef
-        OldOpndValue = nullptr;
-        BoundCtrlZero = true;
+
+  bool CombBCZ = false;
+
+  if (MaskAllLanes && BoundCtrlZero) { // [1]
+    CombBCZ = true;
+  } else {
+    if (!OldOpndValue || !OldOpndValue->isImm()) {
+      LLVM_DEBUG(dbgs() << "  failed: the DPP mov isn't combinable\n");
+      return false;
+    }
+
+    if (OldOpndValue->getParent()->getParent() != MovMI.getParent()) {
+      LLVM_DEBUG(dbgs() <<
+        "  failed: old reg def and mov should be in the same BB\n");
+      return false;
+    }
+
+    if (OldOpndValue->getImm() == 0) {
+      if (MaskAllLanes) {
+        assert(!BoundCtrlZero); // by check [1]
+        CombBCZ = true;
       }
+    } else if (BoundCtrlZero) {
+      assert(!MaskAllLanes); // by check [1]
+      LLVM_DEBUG(dbgs() <<
+        "  failed: old!=0 and bctrl:0 and not all lanes isn't combinable\n");
+      return false;
     }
   }
 
@@ -348,25 +391,28 @@ bool GCNDPPCombine::combineDPPMov(MachineInstr &MovMI) const {
     if (!OldOpndValue)
       dbgs() << "undef";
     else
-      dbgs() << OldOpndValue->getImm();
-    dbgs() << ", bound_ctrl=" << BoundCtrlZero << '\n');
-
-  std::vector<MachineInstr*> OrigMIs, DPPMIs;
-  if (!OldOpndVGPR.Reg) { // OldOpndVGPR = undef
-    OldOpndVGPR = RegSubRegPair(
+      dbgs() << *OldOpndValue;
+    dbgs() << ", bound_ctrl=" << CombBCZ << '\n');
+
+  SmallVector<MachineInstr*, 4> OrigMIs, DPPMIs;
+  auto CombOldVGPR = getRegSubRegPair(*OldOpnd);
+  // try to reuse previous old reg if its undefined (IMPLICIT_DEF)
+  if (CombBCZ && OldOpndValue) { // CombOldVGPR should be undef
+    CombOldVGPR = RegSubRegPair(
       MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass));
     auto UndefInst = BuildMI(*MovMI.getParent(), MovMI, MovMI.getDebugLoc(),
-                             TII->get(AMDGPU::IMPLICIT_DEF), OldOpndVGPR.Reg);
+                             TII->get(AMDGPU::IMPLICIT_DEF), CombOldVGPR.Reg);
     DPPMIs.push_back(UndefInst.getInstr());
   }
 
   OrigMIs.push_back(&MovMI);
   bool Rollback = true;
-  for (auto &Use : MRI->use_nodbg_operands(
-       TII->getNamedOperand(MovMI, AMDGPU::OpName::vdst)->getReg())) {
+  for (auto &Use : MRI->use_nodbg_operands(DPPMovReg)) {
     Rollback = true;
 
     auto &OrigMI = *Use.getParent();
+    LLVM_DEBUG(dbgs() << "  try: " << OrigMI);
+
     auto OrigOp = OrigMI.getOpcode();
     if (TII->isVOP3(OrigOp)) {
       if (!TII->hasVALU32BitEncoding(OrigOp)) {
@@ -389,8 +435,8 @@ bool GCNDPPCombine::combineDPPMov(MachineInstr &MovMI) const {
 
     LLVM_DEBUG(dbgs() << "  combining: " << OrigMI);
     if (&Use == TII->getNamedOperand(OrigMI, AMDGPU::OpName::src0)) {
-      if (auto *DPPInst = createDPPInst(OrigMI, MovMI, OldOpndVGPR,
-                                        OldOpndValue, BoundCtrlZero)) {
+      if (auto *DPPInst = createDPPInst(OrigMI, MovMI, CombOldVGPR,
+                                        OldOpndValue, CombBCZ)) {
         DPPMIs.push_back(DPPInst);
         Rollback = false;
       }
@@ -401,8 +447,8 @@ bool GCNDPPCombine::combineDPPMov(MachineInstr &MovMI) const {
       BB->insert(OrigMI, NewMI);
       if (TII->commuteInstruction(*NewMI)) {
         LLVM_DEBUG(dbgs() << "  commuted:  " << *NewMI);
-        if (auto *DPPInst = createDPPInst(*NewMI, MovMI, OldOpndVGPR,
-                                          OldOpndValue, BoundCtrlZero)) {
+        if (auto *DPPInst = createDPPInst(*NewMI, MovMI, CombOldVGPR,
+                                          OldOpndValue, CombBCZ)) {
           DPPMIs.push_back(DPPInst);
           Rollback = false;
         }
diff --git a/lib/Target/AMDGPU/GCNHazardRecognizer.cpp b/lib/Target/AMDGPU/GCNHazardRecognizer.cpp
index c6396de89c4f6e6dbbd456b390ceecc6cab589c2..4af8a457a0e5dfe02d7786280ead0d901ce64156 100644
--- a/lib/Target/AMDGPU/GCNHazardRecognizer.cpp
+++ b/lib/Target/AMDGPU/GCNHazardRecognizer.cpp
@@ -1,9 +1,8 @@
 //===-- GCNHazardRecognizers.cpp - GCN Hazard Recognizer Impls ------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -38,6 +37,7 @@ using namespace llvm;
 //===----------------------------------------------------------------------===//
 
 GCNHazardRecognizer::GCNHazardRecognizer(const MachineFunction &MF) :
+  IsHazardRecognizerMode(false),
   CurrCycleInstr(nullptr),
   MF(MF),
   ST(MF.getSubtarget<GCNSubtarget>()),
@@ -88,14 +88,28 @@ static bool isSMovRel(unsigned Opcode) {
   }
 }
 
-static bool isSendMsgTraceDataOrGDS(const MachineInstr &MI) {
+static bool isSendMsgTraceDataOrGDS(const SIInstrInfo &TII,
+                                    const MachineInstr &MI) {
+  if (TII.isAlwaysGDS(MI.getOpcode()))
+    return true;
+
   switch (MI.getOpcode()) {
   case AMDGPU::S_SENDMSG:
   case AMDGPU::S_SENDMSGHALT:
   case AMDGPU::S_TTRACEDATA:
     return true;
+  // These DS opcodes don't support GDS.
+  case AMDGPU::DS_NOP:
+  case AMDGPU::DS_PERMUTE_B32:
+  case AMDGPU::DS_BPERMUTE_B32:
+    return false;
   default:
-    // TODO: GDS
+    if (TII.isDS(MI.getOpcode())) {
+      int GDS = AMDGPU::getNamedOperandIdx(MI.getOpcode(),
+                                           AMDGPU::OpName::gds);
+      if (MI.getOperand(GDS).getImm())
+        return true;
+    }
     return false;
   }
 }
@@ -145,7 +159,7 @@ GCNHazardRecognizer::getHazardType(SUnit *SU, int Stalls) {
       checkReadM0Hazards(MI) > 0)
     return NoopHazard;
 
-  if (ST.hasReadM0SendMsgHazard() && isSendMsgTraceDataOrGDS(*MI) &&
+  if (ST.hasReadM0SendMsgHazard() && isSendMsgTraceDataOrGDS(TII, *MI) &&
       checkReadM0Hazards(MI) > 0)
     return NoopHazard;
 
@@ -159,10 +173,19 @@ GCNHazardRecognizer::getHazardType(SUnit *SU, int Stalls) {
 }
 
 unsigned GCNHazardRecognizer::PreEmitNoops(SUnit *SU) {
-  return PreEmitNoops(SU->getInstr());
+  IsHazardRecognizerMode = false;
+  return PreEmitNoopsCommon(SU->getInstr());
 }
 
 unsigned GCNHazardRecognizer::PreEmitNoops(MachineInstr *MI) {
+  IsHazardRecognizerMode = true;
+  CurrCycleInstr = MI;
+  unsigned W = PreEmitNoopsCommon(MI);
+  CurrCycleInstr = nullptr;
+  return W;
+}
+
+unsigned GCNHazardRecognizer::PreEmitNoopsCommon(MachineInstr *MI) {
   int WaitStates = std::max(0, checkAnyInstHazards(MI));
 
   if (SIInstrInfo::isSMRD(*MI))
@@ -199,7 +222,7 @@ unsigned GCNHazardRecognizer::PreEmitNoops(MachineInstr *MI) {
                                            isSMovRel(MI->getOpcode())))
     return std::max(WaitStates, checkReadM0Hazards(MI));
 
-  if (ST.hasReadM0SendMsgHazard() && isSendMsgTraceDataOrGDS(*MI))
+  if (ST.hasReadM0SendMsgHazard() && isSendMsgTraceDataOrGDS(TII, *MI))
     return std::max(WaitStates, checkReadM0Hazards(MI));
 
   return WaitStates;
@@ -218,7 +241,7 @@ void GCNHazardRecognizer::AdvanceCycle() {
   // Do not track non-instructions which do not affect the wait states.
   // If included, these instructions can lead to buffer overflow such that
   // detectable hazards are missed.
-  if (CurrCycleInstr->getOpcode() == AMDGPU::IMPLICIT_DEF)
+  if (CurrCycleInstr->isImplicitDef())
     return;
   else if (CurrCycleInstr->isDebugInstr())
     return;
@@ -252,41 +275,109 @@ void GCNHazardRecognizer::RecedeCycle() {
 // Helper Functions
 //===----------------------------------------------------------------------===//
 
-int GCNHazardRecognizer::getWaitStatesSince(
-    function_ref<bool(MachineInstr *)> IsHazard) {
+typedef function_ref<bool(MachineInstr *, int WaitStates)> IsExpiredFn;
+
+// Returns a minimum wait states since \p I walking all predecessors.
+// Only scans until \p IsExpired does not return true.
+// Can only be run in a hazard recognizer mode.
+static int getWaitStatesSince(GCNHazardRecognizer::IsHazardFn IsHazard,
+                              MachineBasicBlock *MBB,
+                              MachineBasicBlock::reverse_instr_iterator I,
+                              int WaitStates,
+                              IsExpiredFn IsExpired,
+                              DenseSet<const MachineBasicBlock *> &Visited) {
+
+  for (auto E = MBB->rend() ; I != E; ++I) {
+    if (IsHazard(&*I))
+      return WaitStates;
+
+    if (I->isInlineAsm() || I->isImplicitDef() || I->isDebugInstr())
+      continue;
+
+    WaitStates += SIInstrInfo::getNumWaitStates(*I);
+
+    if (IsExpired(&*I, WaitStates))
+      return std::numeric_limits<int>::max();
+  }
+
+  int MinWaitStates = WaitStates;
+  bool Found = false;
+  for (MachineBasicBlock *Pred : MBB->predecessors()) {
+    if (!Visited.insert(Pred).second)
+      continue;
+
+    int W = getWaitStatesSince(IsHazard, Pred, Pred->instr_rbegin(),
+                               WaitStates, IsExpired, Visited);
+
+    if (W == std::numeric_limits<int>::max())
+      continue;
+
+    MinWaitStates = Found ? std::min(MinWaitStates, W) : W;
+    if (IsExpired(nullptr, MinWaitStates))
+      return MinWaitStates;
+
+    Found = true;
+  }
+
+  if (Found)
+    return MinWaitStates;
+
+  return std::numeric_limits<int>::max();
+}
+
+static int getWaitStatesSince(GCNHazardRecognizer::IsHazardFn IsHazard,
+                              MachineInstr *MI,
+                              IsExpiredFn IsExpired) {
+  DenseSet<const MachineBasicBlock *> Visited;
+  return getWaitStatesSince(IsHazard, MI->getParent(),
+                            std::next(MI->getReverseIterator()),
+                            0, IsExpired, Visited);
+}
+
+int GCNHazardRecognizer::getWaitStatesSince(IsHazardFn IsHazard, int Limit) {
+  if (IsHazardRecognizerMode) {
+    auto IsExpiredFn = [Limit] (MachineInstr *, int WaitStates) {
+      return WaitStates >= Limit;
+    };
+    return ::getWaitStatesSince(IsHazard, CurrCycleInstr, IsExpiredFn);
+  }
+
   int WaitStates = 0;
   for (MachineInstr *MI : EmittedInstrs) {
     if (MI) {
       if (IsHazard(MI))
         return WaitStates;
 
-      unsigned Opcode = MI->getOpcode();
-      if (Opcode == AMDGPU::INLINEASM)
+      if (MI->isInlineAsm())
         continue;
     }
     ++WaitStates;
+
+    if (WaitStates >= Limit)
+      break;
   }
   return std::numeric_limits<int>::max();
 }
 
-int GCNHazardRecognizer::getWaitStatesSinceDef(
-    unsigned Reg, function_ref<bool(MachineInstr *)> IsHazardDef) {
+int GCNHazardRecognizer::getWaitStatesSinceDef(unsigned Reg,
+                                               IsHazardFn IsHazardDef,
+                                               int Limit) {
   const SIRegisterInfo *TRI = ST.getRegisterInfo();
 
   auto IsHazardFn = [IsHazardDef, TRI, Reg] (MachineInstr *MI) {
     return IsHazardDef(MI) && MI->modifiesRegister(Reg, TRI);
   };
 
-  return getWaitStatesSince(IsHazardFn);
+  return getWaitStatesSince(IsHazardFn, Limit);
 }
 
-int GCNHazardRecognizer::getWaitStatesSinceSetReg(
-    function_ref<bool(MachineInstr *)> IsHazard) {
+int GCNHazardRecognizer::getWaitStatesSinceSetReg(IsHazardFn IsHazard,
+                                                  int Limit) {
   auto IsHazardFn = [IsHazard] (MachineInstr *MI) {
     return isSSetReg(MI->getOpcode()) && IsHazard(MI);
   };
 
-  return getWaitStatesSince(IsHazardFn);
+  return getWaitStatesSince(IsHazardFn, Limit);
 }
 
 //===----------------------------------------------------------------------===//
@@ -384,7 +475,8 @@ int GCNHazardRecognizer::checkSMRDHazards(MachineInstr *SMRD) {
     if (!Use.isReg())
       continue;
     int WaitStatesNeededForUse =
-        SmrdSgprWaitStates - getWaitStatesSinceDef(Use.getReg(), IsHazardDefFn);
+        SmrdSgprWaitStates - getWaitStatesSinceDef(Use.getReg(), IsHazardDefFn,
+                                                   SmrdSgprWaitStates);
     WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
 
     // This fixes what appears to be undocumented hardware behavior in SI where
@@ -397,7 +489,8 @@ int GCNHazardRecognizer::checkSMRDHazards(MachineInstr *SMRD) {
     if (IsBufferSMRD) {
       int WaitStatesNeededForUse =
         SmrdSgprWaitStates - getWaitStatesSinceDef(Use.getReg(),
-                                                   IsBufferHazardDefFn);
+                                                   IsBufferHazardDefFn,
+                                                   SmrdSgprWaitStates);
       WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
     }
   }
@@ -421,7 +514,8 @@ int GCNHazardRecognizer::checkVMEMHazards(MachineInstr* VMEM) {
       continue;
 
     int WaitStatesNeededForUse =
-        VmemSgprWaitStates - getWaitStatesSinceDef(Use.getReg(), IsHazardDefFn);
+        VmemSgprWaitStates - getWaitStatesSinceDef(Use.getReg(), IsHazardDefFn,
+                                                   VmemSgprWaitStates);
     WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
   }
   return WaitStatesNeeded;
@@ -441,13 +535,16 @@ int GCNHazardRecognizer::checkDPPHazards(MachineInstr *DPP) {
     if (!Use.isReg() || !TRI->isVGPR(MF.getRegInfo(), Use.getReg()))
       continue;
     int WaitStatesNeededForUse =
-        DppVgprWaitStates - getWaitStatesSinceDef(Use.getReg());
+        DppVgprWaitStates - getWaitStatesSinceDef(Use.getReg(),
+                              [](MachineInstr *) { return true; },
+                              DppVgprWaitStates);
     WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
   }
 
   WaitStatesNeeded = std::max(
       WaitStatesNeeded,
-      DppExecWaitStates - getWaitStatesSinceDef(AMDGPU::EXEC, IsHazardDefFn));
+      DppExecWaitStates - getWaitStatesSinceDef(AMDGPU::EXEC, IsHazardDefFn,
+                                                DppExecWaitStates));
 
   return WaitStatesNeeded;
 }
@@ -459,7 +556,8 @@ int GCNHazardRecognizer::checkDivFMasHazards(MachineInstr *DivFMas) {
   // instruction.
   const int DivFMasWaitStates = 4;
   auto IsHazardDefFn = [TII] (MachineInstr *MI) { return TII->isVALU(*MI); };
-  int WaitStatesNeeded = getWaitStatesSinceDef(AMDGPU::VCC, IsHazardDefFn);
+  int WaitStatesNeeded = getWaitStatesSinceDef(AMDGPU::VCC, IsHazardDefFn,
+                                               DivFMasWaitStates);
 
   return DivFMasWaitStates - WaitStatesNeeded;
 }
@@ -472,7 +570,7 @@ int GCNHazardRecognizer::checkGetRegHazards(MachineInstr *GetRegInstr) {
   auto IsHazardFn = [TII, GetRegHWReg] (MachineInstr *MI) {
     return GetRegHWReg == getHWReg(TII, *MI);
   };
-  int WaitStatesNeeded = getWaitStatesSinceSetReg(IsHazardFn);
+  int WaitStatesNeeded = getWaitStatesSinceSetReg(IsHazardFn, GetRegWaitStates);
 
   return GetRegWaitStates - WaitStatesNeeded;
 }
@@ -486,7 +584,7 @@ int GCNHazardRecognizer::checkSetRegHazards(MachineInstr *SetRegInstr) {
   auto IsHazardFn = [TII, HWReg] (MachineInstr *MI) {
     return HWReg == getHWReg(TII, *MI);
   };
-  int WaitStatesNeeded = getWaitStatesSinceSetReg(IsHazardFn);
+  int WaitStatesNeeded = getWaitStatesSinceSetReg(IsHazardFn, SetRegWaitStates);
   return SetRegWaitStates - WaitStatesNeeded;
 }
 
@@ -557,7 +655,7 @@ int GCNHazardRecognizer::checkVALUHazardsHelper(const MachineOperand &Def,
     TRI->regsOverlap(MI->getOperand(DataIdx).getReg(), Reg);
   };
   int WaitStatesNeededForDef =
-    VALUWaitStates - getWaitStatesSince(IsHazardFn);
+    VALUWaitStates - getWaitStatesSince(IsHazardFn, VALUWaitStates);
   WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForDef);
 
   return WaitStatesNeeded;
@@ -622,7 +720,8 @@ int GCNHazardRecognizer::checkRWLaneHazards(MachineInstr *RWLane) {
   };
 
   const int RWLaneWaitStates = 4;
-  int WaitStatesSince = getWaitStatesSinceDef(LaneSelectReg, IsHazardFn);
+  int WaitStatesSince = getWaitStatesSinceDef(LaneSelectReg, IsHazardFn,
+                                              RWLaneWaitStates);
   return RWLaneWaitStates - WaitStatesSince;
 }
 
@@ -637,7 +736,7 @@ int GCNHazardRecognizer::checkRFEHazards(MachineInstr *RFE) {
   auto IsHazardFn = [TII] (MachineInstr *MI) {
     return getHWReg(TII, *MI) == AMDGPU::Hwreg::ID_TRAPSTS;
   };
-  int WaitStatesNeeded = getWaitStatesSinceSetReg(IsHazardFn);
+  int WaitStatesNeeded = getWaitStatesSinceSetReg(IsHazardFn, RFEWaitStates);
   return RFEWaitStates - WaitStatesNeeded;
 }
 
@@ -661,7 +760,8 @@ int GCNHazardRecognizer::checkAnyInstHazards(MachineInstr *MI) {
       return MI->getOpcode() == AMDGPU::S_MOV_FED_B32;
     };
     int WaitStatesNeededForUse =
-        MovFedWaitStates - getWaitStatesSinceDef(Use.getReg(), IsHazardFn);
+        MovFedWaitStates - getWaitStatesSinceDef(Use.getReg(), IsHazardFn,
+                                                 MovFedWaitStates);
     WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
   }
 
@@ -674,5 +774,6 @@ int GCNHazardRecognizer::checkReadM0Hazards(MachineInstr *MI) {
   auto IsHazardFn = [TII] (MachineInstr *MI) {
     return TII->isSALU(*MI);
   };
-  return SMovRelWaitStates - getWaitStatesSinceDef(AMDGPU::M0, IsHazardFn);
+  return SMovRelWaitStates - getWaitStatesSinceDef(AMDGPU::M0, IsHazardFn,
+                                                   SMovRelWaitStates);
 }
diff --git a/lib/Target/AMDGPU/GCNHazardRecognizer.h b/lib/Target/AMDGPU/GCNHazardRecognizer.h
index ca17e7cb6018f4e4dad0ca6d0f130aa23596c941..2a6a2deed2c1e4d03264839f6ca9f4e89d03d1a6 100644
--- a/lib/Target/AMDGPU/GCNHazardRecognizer.h
+++ b/lib/Target/AMDGPU/GCNHazardRecognizer.h
@@ -1,9 +1,8 @@
 //===-- GCNHazardRecognizers.h - GCN Hazard Recognizers ---------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -31,6 +30,13 @@ class SIRegisterInfo;
 class GCNSubtarget;
 
 class GCNHazardRecognizer final : public ScheduleHazardRecognizer {
+public:
+  typedef function_ref<bool(MachineInstr *)> IsHazardFn;
+
+private:
+  // Distinguish if we are called from scheduler or hazard recognizer
+  bool IsHazardRecognizerMode;
+
   // This variable stores the instruction that has been emitted this cycle. It
   // will be added to EmittedInstrs, when AdvanceCycle() or RecedeCycle() is
   // called.
@@ -54,11 +60,9 @@ class GCNHazardRecognizer final : public ScheduleHazardRecognizer {
 
   void addClauseInst(const MachineInstr &MI);
 
-  int getWaitStatesSince(function_ref<bool(MachineInstr *)> IsHazard);
-  int getWaitStatesSinceDef(unsigned Reg,
-                            function_ref<bool(MachineInstr *)> IsHazardDef =
-                                [](MachineInstr *) { return true; });
-  int getWaitStatesSinceSetReg(function_ref<bool(MachineInstr *)> IsHazard);
+  int getWaitStatesSince(IsHazardFn IsHazard, int Limit);
+  int getWaitStatesSinceDef(unsigned Reg, IsHazardFn IsHazardDef, int Limit);
+  int getWaitStatesSinceSetReg(IsHazardFn IsHazard, int Limit);
 
   int checkSoftClauseHazards(MachineInstr *SMEM);
   int checkSMRDHazards(MachineInstr *SMRD);
@@ -85,6 +89,7 @@ public:
   void EmitNoop() override;
   unsigned PreEmitNoops(SUnit *SU) override;
   unsigned PreEmitNoops(MachineInstr *) override;
+  unsigned PreEmitNoopsCommon(MachineInstr *);
   void AdvanceCycle() override;
   void RecedeCycle() override;
 };
diff --git a/lib/Target/AMDGPU/GCNILPSched.cpp b/lib/Target/AMDGPU/GCNILPSched.cpp
index d62dc8d86781c4f62239d5f9afb710e651c44698..1eb617640c32a34cdd750e3aa48c1dce59ffb4be 100644
--- a/lib/Target/AMDGPU/GCNILPSched.cpp
+++ b/lib/Target/AMDGPU/GCNILPSched.cpp
@@ -1,9 +1,8 @@
 //===---------------------------- GCNILPSched.cpp - -----------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AMDGPU/GCNIterativeScheduler.cpp b/lib/Target/AMDGPU/GCNIterativeScheduler.cpp
index 8e4cc391dc21cf56b6f4260ba82a63a566e090cd..3525174223bdc317c8142a7ad13ff856f0610b31 100644
--- a/lib/Target/AMDGPU/GCNIterativeScheduler.cpp
+++ b/lib/Target/AMDGPU/GCNIterativeScheduler.cpp
@@ -1,9 +1,8 @@
 //===- GCNIterativeScheduler.cpp ------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/AMDGPU/GCNIterativeScheduler.h b/lib/Target/AMDGPU/GCNIterativeScheduler.h
index 14ef5147f32af146d6b8be8cdf11c1f16f20c505..e6f83914af5ba3c92047de8ad96d9bf1e0dc1216 100644
--- a/lib/Target/AMDGPU/GCNIterativeScheduler.h
+++ b/lib/Target/AMDGPU/GCNIterativeScheduler.h
@@ -1,9 +1,8 @@
 //===- GCNIterativeScheduler.h - GCN Scheduler ------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/AMDGPU/GCNMinRegStrategy.cpp b/lib/Target/AMDGPU/GCNMinRegStrategy.cpp
index ec6bcae335551eade682c53de71d6fc170decf37..c469cf290e264215d9863de53f1c42895b4e65b2 100644
--- a/lib/Target/AMDGPU/GCNMinRegStrategy.cpp
+++ b/lib/Target/AMDGPU/GCNMinRegStrategy.cpp
@@ -1,9 +1,8 @@
 //===- GCNMinRegStrategy.cpp ----------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/AMDGPU/GCNProcessors.td b/lib/Target/AMDGPU/GCNProcessors.td
index b8142a4e4ff888f159f450c1b864cc671132de0a..fbbf1dea64c6286d697d03a20fe522c537b675c2 100644
--- a/lib/Target/AMDGPU/GCNProcessors.td
+++ b/lib/Target/AMDGPU/GCNProcessors.td
@@ -1,163 +1,162 @@
 //===-- GCNProcessors.td - GCN Processor definitions ----------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 // The code produced for "generic" is only useful for tests and cannot
 // reasonably be expected to execute on any particular target.
 def : ProcessorModel<"generic", NoSchedModel,
-  [FeatureGCN, FeatureWavefrontSize64]
+  [FeatureWavefrontSize64]
 >;
 
-//===----------------------------------------------------------------------===//
+//===------------------------------------------------------------===//
 // GCN GFX6 (Southern Islands (SI)).
-//===----------------------------------------------------------------------===//
+//===------------------------------------------------------------===//
 
 def : ProcessorModel<"gfx600", SIFullSpeedModel,
-  [FeatureISAVersion6_0_0]
+  FeatureISAVersion6_0_0.Features
 >;
 
 def : ProcessorModel<"tahiti", SIFullSpeedModel,
-  [FeatureISAVersion6_0_0]
+  FeatureISAVersion6_0_0.Features
 >;
 
 def : ProcessorModel<"gfx601", SIQuarterSpeedModel,
-  [FeatureISAVersion6_0_1]
+  FeatureISAVersion6_0_1.Features
 >;
 
 def : ProcessorModel<"hainan", SIQuarterSpeedModel,
-  [FeatureISAVersion6_0_1]
+  FeatureISAVersion6_0_1.Features
 >;
 
 def : ProcessorModel<"oland", SIQuarterSpeedModel,
-  [FeatureISAVersion6_0_1]
+  FeatureISAVersion6_0_1.Features
 >;
 
 def : ProcessorModel<"pitcairn", SIQuarterSpeedModel,
-  [FeatureISAVersion6_0_1]
+  FeatureISAVersion6_0_1.Features
 >;
 
 def : ProcessorModel<"verde", SIQuarterSpeedModel,
-  [FeatureISAVersion6_0_1]
+  FeatureISAVersion6_0_1.Features
 >;
 
-//===----------------------------------------------------------------------===//
+//===------------------------------------------------------------===//
 // GCN GFX7 (Sea Islands (CI)).
-//===----------------------------------------------------------------------===//
+//===------------------------------------------------------------===//
 
 def : ProcessorModel<"gfx700", SIQuarterSpeedModel,
-  [FeatureISAVersion7_0_0]
+  FeatureISAVersion7_0_0.Features
 >;
 
 def : ProcessorModel<"kaveri", SIQuarterSpeedModel,
-  [FeatureISAVersion7_0_0]
+  FeatureISAVersion7_0_0.Features
 >;
 
 def : ProcessorModel<"gfx701", SIFullSpeedModel,
-  [FeatureISAVersion7_0_1]
+  FeatureISAVersion7_0_1.Features
 >;
 
 def : ProcessorModel<"hawaii", SIFullSpeedModel,
-  [FeatureISAVersion7_0_1]
+  FeatureISAVersion7_0_1.Features
 >;
 
 def : ProcessorModel<"gfx702", SIQuarterSpeedModel,
-  [FeatureISAVersion7_0_2]
+  FeatureISAVersion7_0_2.Features
 >;
 
 def : ProcessorModel<"gfx703", SIQuarterSpeedModel,
-  [FeatureISAVersion7_0_3]
+  FeatureISAVersion7_0_3.Features
 >;
 
 def : ProcessorModel<"kabini", SIQuarterSpeedModel,
-  [FeatureISAVersion7_0_3]
+  FeatureISAVersion7_0_3.Features
 >;
 
 def : ProcessorModel<"mullins", SIQuarterSpeedModel,
-  [FeatureISAVersion7_0_3]
+  FeatureISAVersion7_0_3.Features
 >;
 
 def : ProcessorModel<"gfx704", SIQuarterSpeedModel,
-  [FeatureISAVersion7_0_4]
+  FeatureISAVersion7_0_4.Features
 >;
 
 def : ProcessorModel<"bonaire", SIQuarterSpeedModel,
-  [FeatureISAVersion7_0_4]
+  FeatureISAVersion7_0_4.Features
 >;
 
-//===----------------------------------------------------------------------===//
+//===------------------------------------------------------------===//
 // GCN GFX8 (Volcanic Islands (VI)).
-//===----------------------------------------------------------------------===//
+//===------------------------------------------------------------===//
 
 def : ProcessorModel<"gfx801", SIQuarterSpeedModel,
-  [FeatureISAVersion8_0_1]
+  FeatureISAVersion8_0_1.Features
 >;
 
 def : ProcessorModel<"carrizo", SIQuarterSpeedModel,
-  [FeatureISAVersion8_0_1]
+  FeatureISAVersion8_0_1.Features
 >;
 
 def : ProcessorModel<"gfx802", SIQuarterSpeedModel,
-  [FeatureISAVersion8_0_2]
+  FeatureISAVersion8_0_2.Features
 >;
 
 def : ProcessorModel<"iceland", SIQuarterSpeedModel,
-  [FeatureISAVersion8_0_2]
+  FeatureISAVersion8_0_2.Features
 >;
 
 def : ProcessorModel<"tonga", SIQuarterSpeedModel,
-  [FeatureISAVersion8_0_2]
+  FeatureISAVersion8_0_2.Features
 >;
 
 def : ProcessorModel<"gfx803", SIQuarterSpeedModel,
-  [FeatureISAVersion8_0_3]
+  FeatureISAVersion8_0_3.Features
 >;
 
 def : ProcessorModel<"fiji", SIQuarterSpeedModel,
-  [FeatureISAVersion8_0_3]
+  FeatureISAVersion8_0_3.Features
 >;
 
 def : ProcessorModel<"polaris10", SIQuarterSpeedModel,
-  [FeatureISAVersion8_0_3]
+  FeatureISAVersion8_0_3.Features
 >;
 
 def : ProcessorModel<"polaris11", SIQuarterSpeedModel,
-  [FeatureISAVersion8_0_3]
+  FeatureISAVersion8_0_3.Features
 >;
 
 def : ProcessorModel<"gfx810", SIQuarterSpeedModel,
-  [FeatureISAVersion8_1_0]
+  FeatureISAVersion8_1_0.Features
 >;
 
 def : ProcessorModel<"stoney", SIQuarterSpeedModel,
-  [FeatureISAVersion8_1_0]
+  FeatureISAVersion8_1_0.Features
 >;
 
-//===----------------------------------------------------------------------===//
+//===------------------------------------------------------------===//
 // GCN GFX9.
-//===----------------------------------------------------------------------===//
+//===------------------------------------------------------------===//
 
 def : ProcessorModel<"gfx900", SIQuarterSpeedModel,
-  [FeatureISAVersion9_0_0]
+  FeatureISAVersion9_0_0.Features
 >;
 
 def : ProcessorModel<"gfx902", SIQuarterSpeedModel,
-  [FeatureISAVersion9_0_2]
+  FeatureISAVersion9_0_2.Features
 >;
 
 def : ProcessorModel<"gfx904", SIQuarterSpeedModel,
-  [FeatureISAVersion9_0_4]
+  FeatureISAVersion9_0_4.Features
 >;
 
 def : ProcessorModel<"gfx906", SIQuarterSpeedModel,
-  [FeatureISAVersion9_0_6]
+  FeatureISAVersion9_0_6.Features
 >;
 
 def : ProcessorModel<"gfx909", SIQuarterSpeedModel,
-  [FeatureISAVersion9_0_9]
+  FeatureISAVersion9_0_9.Features
 >;
 
diff --git a/lib/Target/AMDGPU/GCNRegPressure.cpp b/lib/Target/AMDGPU/GCNRegPressure.cpp
index 3d8cacc4f02ce629b967e45a10c231aa9f39650d..1c6081772d58adbb370081cb2a8d487429daae60 100644
--- a/lib/Target/AMDGPU/GCNRegPressure.cpp
+++ b/lib/Target/AMDGPU/GCNRegPressure.cpp
@@ -1,9 +1,8 @@
 //===- GCNRegPressure.cpp -------------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/AMDGPU/GCNRegPressure.h b/lib/Target/AMDGPU/GCNRegPressure.h
index 357d3b7b23340077be12be17234b0ce04fe72232..a061e3c746677b7e17e64749bf303194cc299ead 100644
--- a/lib/Target/AMDGPU/GCNRegPressure.h
+++ b/lib/Target/AMDGPU/GCNRegPressure.h
@@ -1,9 +1,8 @@
 //===- GCNRegPressure.h -----------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/AMDGPU/GCNSchedStrategy.cpp b/lib/Target/AMDGPU/GCNSchedStrategy.cpp
index f09b7f6cff2264994c8d6e89d7ff6044a547e57a..226ea7967bef09c7322d7e735b45aacefcfd8144 100644
--- a/lib/Target/AMDGPU/GCNSchedStrategy.cpp
+++ b/lib/Target/AMDGPU/GCNSchedStrategy.cpp
@@ -1,9 +1,8 @@
 //===-- GCNSchedStrategy.cpp - GCN Scheduler Strategy ---------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AMDGPU/GCNSchedStrategy.h b/lib/Target/AMDGPU/GCNSchedStrategy.h
index 3ac6af89cb9bfa4fd8ba5169a6f30960ba4e8efb..8ff52a753b949927c5554ab416fafd67432d65d8 100644
--- a/lib/Target/AMDGPU/GCNSchedStrategy.h
+++ b/lib/Target/AMDGPU/GCNSchedStrategy.h
@@ -1,9 +1,8 @@
 //===-- GCNSchedStrategy.h - GCN Scheduler Strategy -*- C++ -*-------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AMDGPU/InstPrinter/AMDGPUInstPrinter.cpp b/lib/Target/AMDGPU/InstPrinter/AMDGPUInstPrinter.cpp
index fab0f87dfcbea5fc4d48855f169e7feee45148ea..3871cfd1674d1ccd30bbf78c65019eb501c8a3d4 100644
--- a/lib/Target/AMDGPU/InstPrinter/AMDGPUInstPrinter.cpp
+++ b/lib/Target/AMDGPU/InstPrinter/AMDGPUInstPrinter.cpp
@@ -1,9 +1,8 @@
 //===-- AMDGPUInstPrinter.cpp - AMDGPU MC Inst -> ASM ---------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 // \file
 //===----------------------------------------------------------------------===//
@@ -269,6 +268,9 @@ void AMDGPUInstPrinter::printRegOperand(unsigned RegNo, raw_ostream &O,
   case AMDGPU::XNACK_MASK:
     O << "xnack_mask";
     return;
+  case AMDGPU::LDS_DIRECT:
+    O << "src_lds_direct";
+    return;
   case AMDGPU::VCC_LO:
     O << "vcc_lo";
     return;
@@ -491,7 +493,7 @@ void AMDGPUInstPrinter::printImmediate64(uint64_t Imm,
     O << "-4.0";
   else if (Imm == 0x3fc45f306dc9c882 &&
            STI.getFeatureBits()[AMDGPU::FeatureInv2PiInlineImm])
-  O << "0.15915494";
+    O << "0.15915494309189532";
   else {
     assert(isUInt<32>(Imm) || Imm == 0x3fc45f306dc9c882);
 
diff --git a/lib/Target/AMDGPU/InstPrinter/AMDGPUInstPrinter.h b/lib/Target/AMDGPU/InstPrinter/AMDGPUInstPrinter.h
index 0ba74ca0f3e19db21521541f57ddd81b043787ba..e0c44399d494c1f8167bed3203d5dee6e4efa80d 100644
--- a/lib/Target/AMDGPU/InstPrinter/AMDGPUInstPrinter.h
+++ b/lib/Target/AMDGPU/InstPrinter/AMDGPUInstPrinter.h
@@ -1,9 +1,8 @@
 //===-- AMDGPUInstPrinter.h - AMDGPU MC Inst -> ASM interface ---*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AMDGPU/InstPrinter/LLVMBuild.txt b/lib/Target/AMDGPU/InstPrinter/LLVMBuild.txt
index 30c2670316c88bd29a02753a734a7514c35e100e..15f59461fbd077c9bb8d37cb08ff11f1301a7d9d 100644
--- a/lib/Target/AMDGPU/InstPrinter/LLVMBuild.txt
+++ b/lib/Target/AMDGPU/InstPrinter/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/Target/AMDGPU/InstPrinter/LLVMBuild.txt -----------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Target/AMDGPU/LLVMBuild.txt b/lib/Target/AMDGPU/LLVMBuild.txt
index e591d756a545e25d914a76efefec29d0f4626232..2f661af14f963ce008873af608a5098bcdbbec68 100644
--- a/lib/Target/AMDGPU/LLVMBuild.txt
+++ b/lib/Target/AMDGPU/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/Target/AMDGPU/LLVMBuild.txt ------------------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Target/AMDGPU/MCTargetDesc/AMDGPUAsmBackend.cpp b/lib/Target/AMDGPU/MCTargetDesc/AMDGPUAsmBackend.cpp
index abc88c02adca17980f69f1871f0493c3e66d3017..529b014995ed4e3082211f2b816cfdb67119059a 100644
--- a/lib/Target/AMDGPU/MCTargetDesc/AMDGPUAsmBackend.cpp
+++ b/lib/Target/AMDGPU/MCTargetDesc/AMDGPUAsmBackend.cpp
@@ -1,9 +1,8 @@
 //===-- AMDGPUAsmBackend.cpp - AMDGPU Assembler Backend -------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 /// \file
 //===----------------------------------------------------------------------===//
diff --git a/lib/Target/AMDGPU/MCTargetDesc/AMDGPUELFObjectWriter.cpp b/lib/Target/AMDGPU/MCTargetDesc/AMDGPUELFObjectWriter.cpp
index 07bef9103c0d8551ec23cdaf7f21300101074aa3..d0fe4df5d8e889a364f3962e56a97a6b44e57889 100644
--- a/lib/Target/AMDGPU/MCTargetDesc/AMDGPUELFObjectWriter.cpp
+++ b/lib/Target/AMDGPU/MCTargetDesc/AMDGPUELFObjectWriter.cpp
@@ -1,9 +1,8 @@
 //===- AMDGPUELFObjectWriter.cpp - AMDGPU ELF Writer ----------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -46,11 +45,9 @@ unsigned AMDGPUELFObjectWriter::getRelocType(MCContext &Ctx,
   if (const auto *SymA = Target.getSymA()) {
     // SCRATCH_RSRC_DWORD[01] is a special global variable that represents
     // the scratch buffer.
-    if (SymA->getSymbol().getName() == "SCRATCH_RSRC_DWORD0")
+    if (SymA->getSymbol().getName() == "SCRATCH_RSRC_DWORD0" ||
+        SymA->getSymbol().getName() == "SCRATCH_RSRC_DWORD1")
       return ELF::R_AMDGPU_ABS32_LO;
-
-    if (SymA->getSymbol().getName() == "SCRATCH_RSRC_DWORD1")
-      return ELF::R_AMDGPU_ABS32_HI;
   }
 
   switch (Target.getAccessVariant()) {
diff --git a/lib/Target/AMDGPU/MCTargetDesc/AMDGPUELFStreamer.cpp b/lib/Target/AMDGPU/MCTargetDesc/AMDGPUELFStreamer.cpp
index c627a08e74631b3662ba7a32d8a09a4bdc1c7101..40437d8fa1a480d5d211ee1e71ab9eb0993954d1 100644
--- a/lib/Target/AMDGPU/MCTargetDesc/AMDGPUELFStreamer.cpp
+++ b/lib/Target/AMDGPU/MCTargetDesc/AMDGPUELFStreamer.cpp
@@ -1,9 +1,8 @@
 //===-------- AMDGPUELFStreamer.cpp - ELF Object Output -------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/AMDGPU/MCTargetDesc/AMDGPUELFStreamer.h b/lib/Target/AMDGPU/MCTargetDesc/AMDGPUELFStreamer.h
index 41e9063a759e88f9bf4e1d72a57da9fd8b4dddfb..9fbf53c944ef575a401e1264dea2c36e05cd75e6 100644
--- a/lib/Target/AMDGPU/MCTargetDesc/AMDGPUELFStreamer.h
+++ b/lib/Target/AMDGPU/MCTargetDesc/AMDGPUELFStreamer.h
@@ -1,9 +1,8 @@
 //===-------- AMDGPUELFStreamer.h - ELF Object Output -----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AMDGPU/MCTargetDesc/AMDGPUFixupKinds.h b/lib/Target/AMDGPU/MCTargetDesc/AMDGPUFixupKinds.h
index 20c1adfbc6b99dfe9dbb82d99196b8e16c3477a1..d49bb196ab3a183bd2b3e508d6d519930a06bee3 100644
--- a/lib/Target/AMDGPU/MCTargetDesc/AMDGPUFixupKinds.h
+++ b/lib/Target/AMDGPU/MCTargetDesc/AMDGPUFixupKinds.h
@@ -1,9 +1,8 @@
 //===-- AMDGPUFixupKinds.h - AMDGPU Specific Fixup Entries ------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCAsmInfo.cpp b/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCAsmInfo.cpp
index 2364e7b7b5fb685e158ca7262e42704fce08c5ba..2f3dbdc96f7952d0ba813e1e552c8b897ad44867 100644
--- a/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCAsmInfo.cpp
+++ b/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCAsmInfo.cpp
@@ -1,9 +1,8 @@
 //===-- MCTargetDesc/AMDGPUMCAsmInfo.cpp - Assembly Info ------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 /// \file
 //===----------------------------------------------------------------------===//
diff --git a/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCAsmInfo.h b/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCAsmInfo.h
index 8cb33a3179cd8018a80e7f2a389a55bf7664892e..a3c069ce970eec9f3e8aef97fa7d636edc6a0fec 100644
--- a/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCAsmInfo.h
+++ b/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCAsmInfo.h
@@ -1,9 +1,8 @@
 //===-- MCTargetDesc/AMDGPUMCAsmInfo.h - AMDGPU MCAsm Interface -*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCCodeEmitter.cpp b/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCCodeEmitter.cpp
index cae7a7a6c7e7c0089aa12cba8010570af014e93a..f3d945cc076400519d9c9e14ef125455388632eb 100644
--- a/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCCodeEmitter.cpp
+++ b/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCCodeEmitter.cpp
@@ -1,9 +1,8 @@
 //===-- AMDGPUCodeEmitter.cpp - AMDGPU Code Emitter interface -------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCCodeEmitter.h b/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCCodeEmitter.h
index dcc10a032afe1ec969276c2d878671864c6c523b..e720289ff7990789acf340371b3d802b7c73aca0 100644
--- a/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCCodeEmitter.h
+++ b/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCCodeEmitter.h
@@ -1,9 +1,8 @@
 //===-- AMDGPUCodeEmitter.h - AMDGPU Code Emitter interface -----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCTargetDesc.cpp b/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCTargetDesc.cpp
index c579c7d60e167471bda9f4a37d1871e8790a5ec8..9cb77058a4a490890aec80bf0ce1e96006671b11 100644
--- a/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCTargetDesc.cpp
+++ b/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCTargetDesc.cpp
@@ -1,9 +1,8 @@
 //===-- AMDGPUMCTargetDesc.cpp - AMDGPU Target Descriptions ---------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCTargetDesc.h b/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCTargetDesc.h
index f3628d96d6e9a89fcd9eff60f0b64028e653552e..a8b1421270e08326c26d9d170900e2b97b6ceecd 100644
--- a/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCTargetDesc.h
+++ b/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCTargetDesc.h
@@ -1,9 +1,8 @@
 //===-- AMDGPUMCTargetDesc.h - AMDGPU Target Descriptions -----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.cpp b/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.cpp
index c17fe126546ce97b3cd28cac59fd6f97114f3eca..5499fa988ef6556df9a62aa311fb2be6ba06d3ea 100644
--- a/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.cpp
+++ b/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.cpp
@@ -1,9 +1,8 @@
 //===-- AMDGPUTargetStreamer.cpp - Mips Target Streamer Methods -----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.h b/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.h
index 9a807c804f9ff1cddc1a5593900146aef938b411..c7a58397aa401b5cdd73807e363745a4e63ae627 100644
--- a/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.h
+++ b/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.h
@@ -1,9 +1,8 @@
 //===-- AMDGPUTargetStreamer.h - AMDGPU Target Streamer --------*- C++ -*--===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/AMDGPU/MCTargetDesc/LLVMBuild.txt b/lib/Target/AMDGPU/MCTargetDesc/LLVMBuild.txt
index bc910a470d72a412ed799b1d0b7bd8a4027b3773..11ff266a84a7d6e7b4a1e44c20d3edf8a6c35cde 100644
--- a/lib/Target/AMDGPU/MCTargetDesc/LLVMBuild.txt
+++ b/lib/Target/AMDGPU/MCTargetDesc/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/Target/AMDGPU/MCTargetDesc/LLVMBuild.txt -------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Target/AMDGPU/MCTargetDesc/R600MCCodeEmitter.cpp b/lib/Target/AMDGPU/MCTargetDesc/R600MCCodeEmitter.cpp
index 28d4bc1829e26c6ddf1422b33b1c42b6feab0f7f..57f5e4c2adee544dad380a90c62ccecb1da77265 100644
--- a/lib/Target/AMDGPU/MCTargetDesc/R600MCCodeEmitter.cpp
+++ b/lib/Target/AMDGPU/MCTargetDesc/R600MCCodeEmitter.cpp
@@ -1,9 +1,8 @@
 //===- R600MCCodeEmitter.cpp - Code Emitter for R600->Cayman GPU families -===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AMDGPU/MCTargetDesc/R600MCTargetDesc.cpp b/lib/Target/AMDGPU/MCTargetDesc/R600MCTargetDesc.cpp
index 1c99a708e5ac352de56b0099aa6cd0070a575b2a..a4809af29daa5d15aabe6fb10cd512a6deeba240 100644
--- a/lib/Target/AMDGPU/MCTargetDesc/R600MCTargetDesc.cpp
+++ b/lib/Target/AMDGPU/MCTargetDesc/R600MCTargetDesc.cpp
@@ -1,9 +1,8 @@
 //===-- R600MCTargetDesc.cpp - R600 Target Descriptions -------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AMDGPU/MCTargetDesc/SIMCCodeEmitter.cpp b/lib/Target/AMDGPU/MCTargetDesc/SIMCCodeEmitter.cpp
index 36913bd0427498eaaaac2848a36d1b9b8bf3195c..205e9b3b5849485a6e38948b98c9f34c64045cee 100644
--- a/lib/Target/AMDGPU/MCTargetDesc/SIMCCodeEmitter.cpp
+++ b/lib/Target/AMDGPU/MCTargetDesc/SIMCCodeEmitter.cpp
@@ -1,9 +1,8 @@
 //===-- SIMCCodeEmitter.cpp - SI Code Emitter -----------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AMDGPU/MIMGInstructions.td b/lib/Target/AMDGPU/MIMGInstructions.td
index 1462682e761cbdb47a651497967481be9dabb37b..4071adcf2009ed9435679034e7d4a68d9ce28898 100644
--- a/lib/Target/AMDGPU/MIMGInstructions.td
+++ b/lib/Target/AMDGPU/MIMGInstructions.td
@@ -1,9 +1,8 @@
 //===-- MIMGInstructions.td - MIMG Instruction Defintions -----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -29,6 +28,7 @@ class MIMGBaseOpcode {
   bit Atomic = 0;
   bit AtomicX2 = 0; // (f)cmpswap
   bit Sampler = 0;
+  bit Gather4 = 0;
   bits<8> NumExtraArgs = 0;
   bit Gradients = 0;
   bit Coordinates = 1;
@@ -43,7 +43,7 @@ def MIMGBaseOpcode : GenericEnum {
 def MIMGBaseOpcodesTable : GenericTable {
   let FilterClass = "MIMGBaseOpcode";
   let CppTypeName = "MIMGBaseOpcodeInfo";
-  let Fields = ["BaseOpcode", "Store", "Atomic", "AtomicX2", "Sampler",
+  let Fields = ["BaseOpcode", "Store", "Atomic", "AtomicX2", "Sampler", "Gather4",
                 "NumExtraArgs", "Gradients", "Coordinates", "LodOrClampOrMip",
                 "HasD16"];
   GenericEnum TypeOf_BaseOpcode = MIMGBaseOpcode;
@@ -101,7 +101,6 @@ class MIMG <dag outs, string dns = "">
   let UseNamedOperandTable = 1;
   let hasSideEffects = 0; // XXX ????
 
-  let SubtargetPredicate = isGCN;
   let DecoderNamespace = dns;
   let isAsmParserOnly = !if(!eq(dns,""), 1, 0);
   let AsmMatchConverter = "cvtMIMG";
@@ -179,6 +178,8 @@ multiclass MIMG_NoSampler <bits<7> op, string asm, bit has_d16, bit mip = 0,
     defm _V3 : MIMG_NoSampler_Src_Helper <op, asm, VReg_96, 0>;
     let VDataDwords = 4 in
     defm _V4 : MIMG_NoSampler_Src_Helper <op, asm, VReg_128, 0>;
+    let VDataDwords = 8 in
+    defm _V8 : MIMG_NoSampler_Src_Helper <op, asm, VReg_256, 0>;
   }
 }
 
@@ -411,6 +412,8 @@ multiclass MIMG_Sampler <bits<7> op, AMDGPUSampleVariant sample, bit wqm = 0,
     defm _V3 : MIMG_Sampler_Src_Helper<op, asm, sample, VReg_96>;
     let VDataDwords = 4 in
     defm _V4 : MIMG_Sampler_Src_Helper<op, asm, sample, VReg_128>;
+    let VDataDwords = 8 in
+    defm _V8 : MIMG_Sampler_Src_Helper<op, asm, sample, VReg_256>;
   }
 }
 
@@ -421,6 +424,7 @@ multiclass MIMG_Gather <bits<7> op, AMDGPUSampleVariant sample, bit wqm = 0,
                         string asm = "image_gather4"#sample.LowerCaseMod> {
   def "" : MIMG_Sampler_BaseOpcode<sample> {
     let HasD16 = 1;
+    let Gather4 = 1;
   }
 
   let BaseOpcode = !cast<MIMGBaseOpcode>(NAME), WQM = wqm,
@@ -429,6 +433,8 @@ multiclass MIMG_Gather <bits<7> op, AMDGPUSampleVariant sample, bit wqm = 0,
     defm _V2 : MIMG_Sampler_Src_Helper<op, asm, sample, VReg_64>; /* for packed D16 only */
     let VDataDwords = 4 in
     defm _V4 : MIMG_Sampler_Src_Helper<op, asm, sample, VReg_128, 1>;
+    let VDataDwords = 8 in
+    defm _V8 : MIMG_Sampler_Src_Helper<op, asm, sample, VReg_256>;
   }
 }
 
diff --git a/lib/Target/AMDGPU/R600.td b/lib/Target/AMDGPU/R600.td
index 5c9c1c1ed504a1809c67262ff861f3e0eb53873d..1d11da96947457ccafb41978a30d91588320a9cc 100644
--- a/lib/Target/AMDGPU/R600.td
+++ b/lib/Target/AMDGPU/R600.td
@@ -1,9 +1,8 @@
 //===-- R600.td - R600 Tablegen files ----------------------*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/AMDGPU/R600AsmPrinter.cpp b/lib/Target/AMDGPU/R600AsmPrinter.cpp
index 68f8c30775b82e02015dd22811b35c0f425d9d93..3fb18862fca85611d97367637bfd8d57ece4556c 100644
--- a/lib/Target/AMDGPU/R600AsmPrinter.cpp
+++ b/lib/Target/AMDGPU/R600AsmPrinter.cpp
@@ -1,9 +1,8 @@
 //===-- R600AsmPrinter.cpp - R600 Assebly printer  ------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AMDGPU/R600AsmPrinter.h b/lib/Target/AMDGPU/R600AsmPrinter.h
index 079fc707b03c7408eb50eee15af2d88033a4b0d0..0da9526d716ea2cec6e034a4fc2281acaef4d2a9 100644
--- a/lib/Target/AMDGPU/R600AsmPrinter.h
+++ b/lib/Target/AMDGPU/R600AsmPrinter.h
@@ -1,9 +1,8 @@
 //===-- R600AsmPrinter.h - Print R600 assembly code -------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AMDGPU/R600ClauseMergePass.cpp b/lib/Target/AMDGPU/R600ClauseMergePass.cpp
index 0c62d6a4b3d98db9d61800f88864c50331f56b43..290a960ae9017ea04a061da2a86ca5efa5037369 100644
--- a/lib/Target/AMDGPU/R600ClauseMergePass.cpp
+++ b/lib/Target/AMDGPU/R600ClauseMergePass.cpp
@@ -1,9 +1,8 @@
 //===-- R600ClauseMergePass - Merge consecutive CF_ALU -------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AMDGPU/R600ControlFlowFinalizer.cpp b/lib/Target/AMDGPU/R600ControlFlowFinalizer.cpp
index a19020276f354b16085f3a9bba569ccf772bc8ab..8098b81d1ea27d9c821da7e919a749e60441af66 100644
--- a/lib/Target/AMDGPU/R600ControlFlowFinalizer.cpp
+++ b/lib/Target/AMDGPU/R600ControlFlowFinalizer.cpp
@@ -1,9 +1,8 @@
 //===- R600ControlFlowFinalizer.cpp - Finalize Control Flow Inst ----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AMDGPU/R600Defines.h b/lib/Target/AMDGPU/R600Defines.h
index 0d33d82e8e0f6e53d5e6e758425f4b08baf635fb..d72534908dcf05aa05a5ea4cff4a82b866019235 100644
--- a/lib/Target/AMDGPU/R600Defines.h
+++ b/lib/Target/AMDGPU/R600Defines.h
@@ -1,9 +1,8 @@
 //===-- R600Defines.h - R600 Helper Macros ----------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 /// \file
 //===----------------------------------------------------------------------===//
diff --git a/lib/Target/AMDGPU/R600EmitClauseMarkers.cpp b/lib/Target/AMDGPU/R600EmitClauseMarkers.cpp
index 679cf18d2c20b970f1828686bc164919126c004c..b97e3c8b8dd7bfbe3174d410a0b346a06a70cd7d 100644
--- a/lib/Target/AMDGPU/R600EmitClauseMarkers.cpp
+++ b/lib/Target/AMDGPU/R600EmitClauseMarkers.cpp
@@ -1,9 +1,8 @@
 //===-- R600EmitClauseMarkers.cpp - Emit CF_ALU ---------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AMDGPU/R600ExpandSpecialInstrs.cpp b/lib/Target/AMDGPU/R600ExpandSpecialInstrs.cpp
index b924ff019dd12e415d5ee80b5dbb26f6e45a1582..c6e8a060d8a02f7cfcc5bcc3552ff30467ba3cd1 100644
--- a/lib/Target/AMDGPU/R600ExpandSpecialInstrs.cpp
+++ b/lib/Target/AMDGPU/R600ExpandSpecialInstrs.cpp
@@ -1,9 +1,8 @@
 //===- R600ExpandSpecialInstrs.cpp - Expand special instructions ----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AMDGPU/R600FrameLowering.cpp b/lib/Target/AMDGPU/R600FrameLowering.cpp
index 37787b3c5f7292cbc2a76cb94cb84e4256e4a7e9..d9aa9ebe878d80c30ba5928c7648ad87f82e4d16 100644
--- a/lib/Target/AMDGPU/R600FrameLowering.cpp
+++ b/lib/Target/AMDGPU/R600FrameLowering.cpp
@@ -1,9 +1,8 @@
 //===----------------------- R600FrameLowering.cpp ------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //==-----------------------------------------------------------------------===//
 
diff --git a/lib/Target/AMDGPU/R600FrameLowering.h b/lib/Target/AMDGPU/R600FrameLowering.h
index fe367d73682fb759a4236a5c575307ad614047ab..950e238f497952b11fb35fc33b52097966a5992e 100644
--- a/lib/Target/AMDGPU/R600FrameLowering.h
+++ b/lib/Target/AMDGPU/R600FrameLowering.h
@@ -1,9 +1,8 @@
 //===--------------------- R600FrameLowering.h ------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/AMDGPU/R600ISelLowering.cpp b/lib/Target/AMDGPU/R600ISelLowering.cpp
index e2a0f05d2b34d4d0e0fc3110e2a481285a8f8a7b..8da15a2d62341b2f3916c43bb49926d10284a6e8 100644
--- a/lib/Target/AMDGPU/R600ISelLowering.cpp
+++ b/lib/Target/AMDGPU/R600ISelLowering.cpp
@@ -1,9 +1,8 @@
 //===-- R600ISelLowering.cpp - R600 DAG Lowering Implementation -----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AMDGPU/R600ISelLowering.h b/lib/Target/AMDGPU/R600ISelLowering.h
index 767c3c7bd5bfe3fb688c47824a0393a9387b86dc..d4204fb0cd54e8f0e986d826742b314c1bc4cb43 100644
--- a/lib/Target/AMDGPU/R600ISelLowering.h
+++ b/lib/Target/AMDGPU/R600ISelLowering.h
@@ -1,9 +1,8 @@
 //===-- R600ISelLowering.h - R600 DAG Lowering Interface -*- C++ -*--------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AMDGPU/R600InstrFormats.td b/lib/Target/AMDGPU/R600InstrFormats.td
index 687a9affa13825f9b51c7d5737275a8d22bbb031..f62e6313b148a46c4c6b8db05b9d89461b14491d 100644
--- a/lib/Target/AMDGPU/R600InstrFormats.td
+++ b/lib/Target/AMDGPU/R600InstrFormats.td
@@ -1,9 +1,8 @@
 //===-- R600InstrFormats.td - R600 Instruction Encodings ------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AMDGPU/R600InstrInfo.cpp b/lib/Target/AMDGPU/R600InstrInfo.cpp
index 9cc3e5f3c314c929b50f9a6defef686d7af14cbf..bc21d912db10834d81dadf2e442d8cdfaa143e63 100644
--- a/lib/Target/AMDGPU/R600InstrInfo.cpp
+++ b/lib/Target/AMDGPU/R600InstrInfo.cpp
@@ -1,9 +1,8 @@
 //===-- R600InstrInfo.cpp - R600 Instruction Information ------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AMDGPU/R600InstrInfo.h b/lib/Target/AMDGPU/R600InstrInfo.h
index e6e34dc125f4d465ca0434a8d657854a7e4bf3f9..00d96c9676aac42790c643f2669dff4b158d6971 100644
--- a/lib/Target/AMDGPU/R600InstrInfo.h
+++ b/lib/Target/AMDGPU/R600InstrInfo.h
@@ -1,9 +1,8 @@
 //===-- R600InstrInfo.h - R600 Instruction Info Interface -------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AMDGPU/R600Instructions.td b/lib/Target/AMDGPU/R600Instructions.td
index 10e87375522241321555131e6a671ef6d7f38491..dcbedbd8cfd6834fd97b383619feb8979b52fee9 100644
--- a/lib/Target/AMDGPU/R600Instructions.td
+++ b/lib/Target/AMDGPU/R600Instructions.td
@@ -1,9 +1,8 @@
 //===-- R600Instructions.td - R600 Instruction defs  -------*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AMDGPU/R600MachineFunctionInfo.cpp b/lib/Target/AMDGPU/R600MachineFunctionInfo.cpp
index 3ca319c6c6c2204322103d00da01691064e253e7..65011a9eadf837beb87afa11188124067924dd70 100644
--- a/lib/Target/AMDGPU/R600MachineFunctionInfo.cpp
+++ b/lib/Target/AMDGPU/R600MachineFunctionInfo.cpp
@@ -1,9 +1,8 @@
 //===-- R600MachineFunctionInfo.cpp - R600 Machine Function Info-*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 /// \file
 //===----------------------------------------------------------------------===//
diff --git a/lib/Target/AMDGPU/R600MachineFunctionInfo.h b/lib/Target/AMDGPU/R600MachineFunctionInfo.h
index 29ac0920f997252bf7405105752a123ae0e54747..6a5ac9023329af019078f9c351db27c6940146a4 100644
--- a/lib/Target/AMDGPU/R600MachineFunctionInfo.h
+++ b/lib/Target/AMDGPU/R600MachineFunctionInfo.h
@@ -1,9 +1,8 @@
 //===-- R600MachineFunctionInfo.h - R600 Machine Function Info ----*- C++ -*-=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AMDGPU/R600MachineScheduler.cpp b/lib/Target/AMDGPU/R600MachineScheduler.cpp
index 7769a35aadcee6ad114ef25c4881a84dc1b5cc0e..34267a909b5e6f58825509241a9dfd64400c3033 100644
--- a/lib/Target/AMDGPU/R600MachineScheduler.cpp
+++ b/lib/Target/AMDGPU/R600MachineScheduler.cpp
@@ -1,9 +1,8 @@
 //===-- R600MachineScheduler.cpp - R600 Scheduler Interface -*- C++ -*-----===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AMDGPU/R600MachineScheduler.h b/lib/Target/AMDGPU/R600MachineScheduler.h
index 8a9a8d3d1e238f1e6f24d1cdec9f87266c077574..bc66f2ef59074fdcaf50792fc731f7fd1925e480 100644
--- a/lib/Target/AMDGPU/R600MachineScheduler.h
+++ b/lib/Target/AMDGPU/R600MachineScheduler.h
@@ -1,9 +1,8 @@
 //===-- R600MachineScheduler.h - R600 Scheduler Interface -*- C++ -*-------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AMDGPU/R600OpenCLImageTypeLoweringPass.cpp b/lib/Target/AMDGPU/R600OpenCLImageTypeLoweringPass.cpp
index 7de5e2c9577decf2f6bc6a047e0573641d02755c..1fe92d2269d3ebefe6273470bd4b3f8611ae8082 100644
--- a/lib/Target/AMDGPU/R600OpenCLImageTypeLoweringPass.cpp
+++ b/lib/Target/AMDGPU/R600OpenCLImageTypeLoweringPass.cpp
@@ -1,9 +1,8 @@
 //===- R600OpenCLImageTypeLoweringPass.cpp ------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AMDGPU/R600OptimizeVectorRegisters.cpp b/lib/Target/AMDGPU/R600OptimizeVectorRegisters.cpp
index 692451cb8fe08d735ba724f4b93836338149b294..ff21176a38fd78951d182c29ff4624054145d4a1 100644
--- a/lib/Target/AMDGPU/R600OptimizeVectorRegisters.cpp
+++ b/lib/Target/AMDGPU/R600OptimizeVectorRegisters.cpp
@@ -1,9 +1,8 @@
 //===- R600MergeVectorRegisters.cpp ---------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AMDGPU/R600Packetizer.cpp b/lib/Target/AMDGPU/R600Packetizer.cpp
index 612c62b514fd7c5745c8996876119d9b7b9d64db..768a4cfc38b6875b997b5e8e4a2abb648d0e28fd 100644
--- a/lib/Target/AMDGPU/R600Packetizer.cpp
+++ b/lib/Target/AMDGPU/R600Packetizer.cpp
@@ -1,9 +1,8 @@
 //===----- R600Packetizer.cpp - VLIW packetizer ---------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AMDGPU/R600Processors.td b/lib/Target/AMDGPU/R600Processors.td
index f39b3dc1bfd46c953bce33963027dadcec44664c..12892840ecc0363f81642d55740156ba82057dcb 100644
--- a/lib/Target/AMDGPU/R600Processors.td
+++ b/lib/Target/AMDGPU/R600Processors.td
@@ -1,9 +1,8 @@
 //===-- R600Processors.td - R600 Processor definitions --------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/AMDGPU/R600RegisterInfo.cpp b/lib/Target/AMDGPU/R600RegisterInfo.cpp
index 38933e7616a0e4ac2ec532571e7435a9ffbc9b60..1b9e399cd327e39cf64d450c742554f5c8ff4abd 100644
--- a/lib/Target/AMDGPU/R600RegisterInfo.cpp
+++ b/lib/Target/AMDGPU/R600RegisterInfo.cpp
@@ -1,9 +1,8 @@
 //===-- R600RegisterInfo.cpp - R600 Register Information ------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AMDGPU/R600RegisterInfo.h b/lib/Target/AMDGPU/R600RegisterInfo.h
index c4c77172b299b75aa20a16b6c235dc3ffdf419c9..d5a26ead031eb899a476615a2446be6a558d8ecd 100644
--- a/lib/Target/AMDGPU/R600RegisterInfo.h
+++ b/lib/Target/AMDGPU/R600RegisterInfo.h
@@ -1,9 +1,8 @@
 //===-- R600RegisterInfo.h - R600 Register Info Interface ------*- C++ -*--===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AMDGPU/R600Schedule.td b/lib/Target/AMDGPU/R600Schedule.td
index 70fb46c1a7d66fb5a4e5c1ef8dd19b49d5c43d99..c998fe8481933fd56033d4d645fc50b4f437b647 100644
--- a/lib/Target/AMDGPU/R600Schedule.td
+++ b/lib/Target/AMDGPU/R600Schedule.td
@@ -1,9 +1,8 @@
 //===-- R600Schedule.td - R600 Scheduling definitions ------*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AMDGPU/R700Instructions.td b/lib/Target/AMDGPU/R700Instructions.td
index 613a0d729bb338e3ac5822ac46d46dfafe46814e..9c9a03209ec2b67d780f84f0102d409b57545e00 100644
--- a/lib/Target/AMDGPU/R700Instructions.td
+++ b/lib/Target/AMDGPU/R700Instructions.td
@@ -1,9 +1,8 @@
 //===-- R700Instructions.td - R700 Instruction defs  -------*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AMDGPU/SIAddIMGInit.cpp b/lib/Target/AMDGPU/SIAddIMGInit.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..f8094e35816c19b0f64055662918e71642e1cde1
--- /dev/null
+++ b/lib/Target/AMDGPU/SIAddIMGInit.cpp
@@ -0,0 +1,180 @@
+//===-- SIAddIMGInit.cpp - Add any required IMG inits ---------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+/// \file
+/// Any MIMG instructions that use tfe or lwe require an initialization of the
+/// result register that will be written in the case of a memory access failure
+/// The required code is also added to tie this init code to the result of the
+/// img instruction
+///
+//===----------------------------------------------------------------------===//
+//
+
+#include "AMDGPU.h"
+#include "AMDGPUSubtarget.h"
+#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
+#include "SIInstrInfo.h"
+#include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/CodeGen/MachineInstrBuilder.h"
+#include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/IR/Function.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Target/TargetMachine.h"
+
+#define DEBUG_TYPE "si-img-init"
+
+using namespace llvm;
+
+namespace {
+
+class SIAddIMGInit : public MachineFunctionPass {
+public:
+  static char ID;
+
+public:
+  SIAddIMGInit() : MachineFunctionPass(ID) {
+    initializeSIAddIMGInitPass(*PassRegistry::getPassRegistry());
+  }
+
+  bool runOnMachineFunction(MachineFunction &MF) override;
+
+  void getAnalysisUsage(AnalysisUsage &AU) const override {
+    AU.setPreservesCFG();
+    MachineFunctionPass::getAnalysisUsage(AU);
+  }
+};
+
+} // End anonymous namespace.
+
+INITIALIZE_PASS(SIAddIMGInit, DEBUG_TYPE, "SI Add IMG Init", false, false)
+
+char SIAddIMGInit::ID = 0;
+
+char &llvm::SIAddIMGInitID = SIAddIMGInit::ID;
+
+FunctionPass *llvm::createSIAddIMGInitPass() { return new SIAddIMGInit(); }
+
+bool SIAddIMGInit::runOnMachineFunction(MachineFunction &MF) {
+  MachineRegisterInfo &MRI = MF.getRegInfo();
+  const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
+  const SIInstrInfo *TII = ST.getInstrInfo();
+  const SIRegisterInfo *RI = ST.getRegisterInfo();
+  bool Changed = false;
+
+  for (MachineFunction::iterator BI = MF.begin(), BE = MF.end(); BI != BE;
+       ++BI) {
+    MachineBasicBlock &MBB = *BI;
+    MachineBasicBlock::iterator I, Next;
+    for (I = MBB.begin(); I != MBB.end(); I = Next) {
+      Next = std::next(I);
+      MachineInstr &MI = *I;
+
+      auto Opcode = MI.getOpcode();
+      if (TII->isMIMG(Opcode) && !MI.mayStore()) {
+        MachineOperand *TFE = TII->getNamedOperand(MI, AMDGPU::OpName::tfe);
+        MachineOperand *LWE = TII->getNamedOperand(MI, AMDGPU::OpName::lwe);
+        MachineOperand *D16 = TII->getNamedOperand(MI, AMDGPU::OpName::d16);
+
+        // Check for instructions that don't have tfe or lwe fields
+        // There shouldn't be any at this point.
+        assert( (TFE && LWE) && "Expected tfe and lwe operands in instruction");
+
+        unsigned TFEVal = TFE->getImm();
+        unsigned LWEVal = LWE->getImm();
+        unsigned D16Val = D16 ? D16->getImm() : 0;
+
+        if (TFEVal || LWEVal) {
+          // At least one of TFE or LWE are non-zero
+          // We have to insert a suitable initialization of the result value and
+          // tie this to the dest of the image instruction.
+
+          const DebugLoc &DL = MI.getDebugLoc();
+
+          int DstIdx =
+              AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::vdata);
+
+          // Calculate which dword we have to initialize to 0.
+          MachineOperand *MO_Dmask =
+              TII->getNamedOperand(MI, AMDGPU::OpName::dmask);
+
+          // check that dmask operand is found.
+          assert(MO_Dmask && "Expected dmask operand in instruction");
+
+          unsigned dmask = MO_Dmask->getImm();
+          // Determine the number of active lanes taking into account the
+          // Gather4 special case
+          unsigned ActiveLanes =
+              TII->isGather4(Opcode) ? 4 : countPopulation(dmask);
+
+          // Subreg indices are counted from 1
+          // When D16 then we want next whole VGPR after write data.
+          static_assert(AMDGPU::sub0 == 1 && AMDGPU::sub4 == 5, "Subreg indices different from expected");
+
+          bool Packed = !ST.hasUnpackedD16VMem();
+
+          unsigned InitIdx =
+              D16Val && Packed ? ((ActiveLanes + 1) >> 1) + 1 : ActiveLanes + 1;
+
+          // Abandon attempt if the dst size isn't large enough
+          // - this is in fact an error but this is picked up elsewhere and
+          // reported correctly.
+          uint32_t DstSize =
+              RI->getRegSizeInBits(*TII->getOpRegClass(MI, DstIdx)) / 32;
+          if (DstSize < InitIdx)
+            continue;
+
+          // Create a register for the intialization value.
+          unsigned PrevDst =
+              MRI.createVirtualRegister(TII->getOpRegClass(MI, DstIdx));
+          unsigned NewDst = 0; // Final initialized value will be in here
+
+          // If PRTStrictNull feature is enabled (the default) then initialize
+          // all the result registers to 0, otherwise just the error indication
+          // register (VGPRn+1)
+          unsigned SizeLeft = ST.usePRTStrictNull() ? InitIdx : 1;
+          unsigned CurrIdx = ST.usePRTStrictNull() ? 1 : InitIdx;
+
+          if (DstSize == 1) {
+            // In this case we can just initialize the result directly
+            BuildMI(MBB, MI, DL, TII->get(AMDGPU::V_MOV_B32_e32), PrevDst)
+                .addImm(0);
+            NewDst = PrevDst;
+          } else {
+            BuildMI(MBB, MI, DL, TII->get(AMDGPU::IMPLICIT_DEF), PrevDst);
+            for (; SizeLeft; SizeLeft--, CurrIdx++) {
+              NewDst =
+                  MRI.createVirtualRegister(TII->getOpRegClass(MI, DstIdx));
+              // Initialize dword
+              unsigned SubReg =
+                  MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
+              BuildMI(MBB, MI, DL, TII->get(AMDGPU::V_MOV_B32_e32), SubReg)
+                  .addImm(0);
+              // Insert into the super-reg
+              BuildMI(MBB, I, DL, TII->get(TargetOpcode::INSERT_SUBREG), NewDst)
+                  .addReg(PrevDst)
+                  .addReg(SubReg)
+                  .addImm(CurrIdx);
+
+              PrevDst = NewDst;
+            }
+          }
+
+          // Add as an implicit operand
+          MachineInstrBuilder(MF, MI).addReg(NewDst, RegState::Implicit);
+
+          // Tie the just added implicit operand to the dst
+          MI.tieOperands(DstIdx, MI.getNumOperands() - 1);
+
+          Changed = true;
+        }
+      }
+    }
+  }
+
+  return Changed;
+}
diff --git a/lib/Target/AMDGPU/SIAnnotateControlFlow.cpp b/lib/Target/AMDGPU/SIAnnotateControlFlow.cpp
index 90f430d5ca4c6c16bb946951db76dee704c68025..456422889481bad5d8d2835ca1123cf994df234e 100644
--- a/lib/Target/AMDGPU/SIAnnotateControlFlow.cpp
+++ b/lib/Target/AMDGPU/SIAnnotateControlFlow.cpp
@@ -1,9 +1,8 @@
 //===- SIAnnotateControlFlow.cpp ------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -155,7 +154,7 @@ bool SIAnnotateControlFlow::doInitialization(Module &M) {
 /// Is the branch condition uniform or did the StructurizeCFG pass
 /// consider it as such?
 bool SIAnnotateControlFlow::isUniform(BranchInst *T) {
-  return DA->isUniform(T->getCondition()) ||
+  return DA->isUniform(T) ||
          T->getMetadata("structurizecfg.uniform") != nullptr;
 }
 
diff --git a/lib/Target/AMDGPU/SIDebuggerInsertNops.cpp b/lib/Target/AMDGPU/SIDebuggerInsertNops.cpp
index 7e884ad93a233c68950e720500d6049a7c3ed9cd..e30da738831b695e88e4f23e65ad92850e108c2b 100644
--- a/lib/Target/AMDGPU/SIDebuggerInsertNops.cpp
+++ b/lib/Target/AMDGPU/SIDebuggerInsertNops.cpp
@@ -1,9 +1,8 @@
 //===--- SIDebuggerInsertNops.cpp - Inserts nops for debugger usage -------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AMDGPU/SIDefines.h b/lib/Target/AMDGPU/SIDefines.h
index 7f6abc34cff3a2f7685fb10f65a8c31be47be4f5..bb9d3a0d2d5748be08ee02458f46a79050b89e8a 100644
--- a/lib/Target/AMDGPU/SIDefines.h
+++ b/lib/Target/AMDGPU/SIDefines.h
@@ -1,9 +1,8 @@
 //===-- SIDefines.h - SI Helper Macros ----------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 /// \file
 //===----------------------------------------------------------------------===//
diff --git a/lib/Target/AMDGPU/SIFixSGPRCopies.cpp b/lib/Target/AMDGPU/SIFixSGPRCopies.cpp
index 809f5bab46932c9e2683fa77f7c2ecbf9a3aeda4..94b1e636c7b1534bfb8a55804c0183665bc12f4a 100644
--- a/lib/Target/AMDGPU/SIFixSGPRCopies.cpp
+++ b/lib/Target/AMDGPU/SIFixSGPRCopies.cpp
@@ -1,9 +1,8 @@
 //===- SIFixSGPRCopies.cpp - Remove potential VGPR => SGPR copies ---------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AMDGPU/SIFixVGPRCopies.cpp b/lib/Target/AMDGPU/SIFixVGPRCopies.cpp
index 15ba78edf91953e38292eb8602610b59dcacc708..29484668a01d9f17e31e69cb0b52def4aef63253 100644
--- a/lib/Target/AMDGPU/SIFixVGPRCopies.cpp
+++ b/lib/Target/AMDGPU/SIFixVGPRCopies.cpp
@@ -1,9 +1,8 @@
 //===-- SIFixVGPRCopies.cpp - Fix VGPR Copies after regalloc --------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AMDGPU/SIFixWWMLiveness.cpp b/lib/Target/AMDGPU/SIFixWWMLiveness.cpp
index 7761418c533648b5a106da035df36c678ae6d623..01c76327aab102e9c44c5d3111bcc62066dd6d00 100644
--- a/lib/Target/AMDGPU/SIFixWWMLiveness.cpp
+++ b/lib/Target/AMDGPU/SIFixWWMLiveness.cpp
@@ -1,9 +1,8 @@
 //===-- SIFixWWMLiveness.cpp - Fix WWM live intervals ---------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AMDGPU/SIFixupVectorISel.cpp b/lib/Target/AMDGPU/SIFixupVectorISel.cpp
index ee39eb04d8316c6af5eb8006114cf935170b9298..43023b3df70cb95be63a8dea452af5fc7d9c0538 100644
--- a/lib/Target/AMDGPU/SIFixupVectorISel.cpp
+++ b/lib/Target/AMDGPU/SIFixupVectorISel.cpp
@@ -1,9 +1,8 @@
 //===-- SIFixupVectorISel.cpp - Fixup post ISel vector issues -------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 /// \file
 /// SIFixupVectorISel pass cleans up post ISEL Vector issues.
diff --git a/lib/Target/AMDGPU/SIFoldOperands.cpp b/lib/Target/AMDGPU/SIFoldOperands.cpp
index bd0bc734060c8e3573b1c99a008a20ce57d4b6c9..9cc24ce915b1d5f9d5d92cc7d6dabd51ca4123ef 100644
--- a/lib/Target/AMDGPU/SIFoldOperands.cpp
+++ b/lib/Target/AMDGPU/SIFoldOperands.cpp
@@ -1,9 +1,8 @@
 //===-- SIFoldOperands.cpp - Fold operands --- ----------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 /// \file
 //===----------------------------------------------------------------------===//
@@ -854,13 +853,17 @@ void SIFoldOperands::foldInstOperand(MachineInstr &MI,
     }
   } else {
     // Folding register.
+    SmallVector <MachineRegisterInfo::use_iterator, 4> UsesToProcess;
     for (MachineRegisterInfo::use_iterator
            Use = MRI->use_begin(Dst.getReg()), E = MRI->use_end();
          Use != E; ++Use) {
-      MachineInstr *UseMI = Use->getParent();
+      UsesToProcess.push_back(Use);
+    }
+    for (auto U : UsesToProcess) {
+      MachineInstr *UseMI = U->getParent();
 
-      foldOperand(OpToFold, UseMI, Use.getOperandNo(),
-                  FoldList, CopiesToReplace);
+      foldOperand(OpToFold, UseMI, U.getOperandNo(),
+        FoldList, CopiesToReplace);
     }
   }
 
diff --git a/lib/Target/AMDGPU/SIFormMemoryClauses.cpp b/lib/Target/AMDGPU/SIFormMemoryClauses.cpp
index aa976d5141f8617c983d1d33302793d1eebec614..2cbed99beb2ffe19168f4993bd4f0362675a9b7d 100644
--- a/lib/Target/AMDGPU/SIFormMemoryClauses.cpp
+++ b/lib/Target/AMDGPU/SIFormMemoryClauses.cpp
@@ -1,9 +1,8 @@
 //===-- SIFormMemoryClauses.cpp -------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -119,6 +118,17 @@ static bool isValidClauseInst(const MachineInstr &MI, bool IsVMEMClause) {
     return false;
   if (!IsVMEMClause && !isSMEMClauseInst(MI))
     return false;
+  // If this is a load instruction where the result has been coalesced with an operand, then we cannot clause it.
+  for (const MachineOperand &ResMO : MI.defs()) {
+    unsigned ResReg = ResMO.getReg();
+    for (const MachineOperand &MO : MI.uses()) {
+      if (!MO.isReg() || MO.isDef())
+        continue;
+      if (MO.getReg() == ResReg)
+        return false;
+    }
+    break; // Only check the first def.
+  }
   return true;
 }
 
diff --git a/lib/Target/AMDGPU/SIFrameLowering.cpp b/lib/Target/AMDGPU/SIFrameLowering.cpp
index e4633c88e18ffdd7689f642ada81ca1ea1bb7dfc..536884704006ea5448e3582efb3de3c9c54f63e4 100644
--- a/lib/Target/AMDGPU/SIFrameLowering.cpp
+++ b/lib/Target/AMDGPU/SIFrameLowering.cpp
@@ -1,9 +1,8 @@
 //===----------------------- SIFrameLowering.cpp --------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //==-----------------------------------------------------------------------===//
 
@@ -421,11 +420,13 @@ void SIFrameLowering::emitEntryFunctionScratchSetup(const GCNSubtarget &ST,
                                        MachineMemOperand::MOLoad |
                                        MachineMemOperand::MOInvariant |
                                        MachineMemOperand::MODereferenceable,
-                                       0, 0);
+                                       16, 4);
     unsigned Offset = Fn.getCallingConv() == CallingConv::AMDGPU_CS ? 16 : 0;
+    const GCNSubtarget &Subtarget = MF.getSubtarget<GCNSubtarget>();
+    unsigned EncodedOffset = AMDGPU::getSMRDEncodedOffset(Subtarget, Offset);
     BuildMI(MBB, I, DL, LoadDwordX4, ScratchRsrcReg)
       .addReg(Rsrc01)
-      .addImm(Offset) // offset
+      .addImm(EncodedOffset) // offset
       .addImm(0) // glc
       .addReg(ScratchRsrcReg, RegState::ImplicitDefine)
       .addMemOperand(MMO);
@@ -462,7 +463,7 @@ void SIFrameLowering::emitEntryFunctionScratchSetup(const GCNSubtarget &ST,
                                            MachineMemOperand::MOLoad |
                                            MachineMemOperand::MOInvariant |
                                            MachineMemOperand::MODereferenceable,
-                                           0, 0);
+                                           8, 4);
         BuildMI(MBB, I, DL, LoadDwordX2, Rsrc01)
           .addReg(MFI->getImplicitBufferPtrUserSGPR())
           .addImm(0) // offset
diff --git a/lib/Target/AMDGPU/SIFrameLowering.h b/lib/Target/AMDGPU/SIFrameLowering.h
index 2f35b3631cdc37e27847d4b4e006edc2e220aa4c..a8795e126f6979bae5bc0adb164de4948b9362ac 100644
--- a/lib/Target/AMDGPU/SIFrameLowering.h
+++ b/lib/Target/AMDGPU/SIFrameLowering.h
@@ -1,9 +1,8 @@
 //===--------------------- SIFrameLowering.h --------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/AMDGPU/SIISelLowering.cpp b/lib/Target/AMDGPU/SIISelLowering.cpp
index 12d0bc528ca570c13034dd724b2487065bfc9c79..17332edcbcea9f5c25d8d26fa1ceb9f953e9087f 100644
--- a/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -1,9 +1,8 @@
 //===-- SIISelLowering.cpp - SI DAG Lowering Implementation ---------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -19,7 +18,6 @@
 
 #include "SIISelLowering.h"
 #include "AMDGPU.h"
-#include "AMDGPUIntrinsicInfo.h"
 #include "AMDGPUSubtarget.h"
 #include "AMDGPUTargetMachine.h"
 #include "SIDefines.h"
@@ -216,6 +214,7 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM,
 
   setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::v2f16, Custom);
   setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::v4f16, Custom);
+  setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::v8f16, Custom);
   setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::Other, Custom);
 
   setOperationAction(ISD::INTRINSIC_VOID, MVT::Other, Custom);
@@ -700,6 +699,7 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM,
   setTargetDAGCombine(ISD::ATOMIC_LOAD_MAX);
   setTargetDAGCombine(ISD::ATOMIC_LOAD_UMIN);
   setTargetDAGCombine(ISD::ATOMIC_LOAD_UMAX);
+  setTargetDAGCombine(ISD::ATOMIC_LOAD_FADD);
 
   setSchedulingPreference(Sched::RegPressure);
 
@@ -813,6 +813,47 @@ unsigned SITargetLowering::getVectorTypeBreakdownForCallingConv(
     Context, CC, VT, IntermediateVT, NumIntermediates, RegisterVT);
 }
 
+static MVT memVTFromAggregate(Type *Ty) {
+  // Only limited forms of aggregate type currently expected.
+  assert(Ty->isStructTy() && "Expected struct type");
+
+
+  Type *ElementType = nullptr;
+  unsigned NumElts;
+  if (Ty->getContainedType(0)->isVectorTy()) {
+    VectorType *VecComponent = cast<VectorType>(Ty->getContainedType(0));
+    ElementType = VecComponent->getElementType();
+    NumElts = VecComponent->getNumElements();
+  } else {
+    ElementType = Ty->getContainedType(0);
+    NumElts = 1;
+  }
+
+  assert((Ty->getContainedType(1) && Ty->getContainedType(1)->isIntegerTy(32)) && "Expected int32 type");
+
+  // Calculate the size of the memVT type from the aggregate
+  unsigned Pow2Elts = 0;
+  unsigned ElementSize;
+  switch (ElementType->getTypeID()) {
+    default:
+      llvm_unreachable("Unknown type!");
+    case Type::IntegerTyID:
+      ElementSize = cast<IntegerType>(ElementType)->getBitWidth();
+      break;
+    case Type::HalfTyID:
+      ElementSize = 16;
+      break;
+    case Type::FloatTyID:
+      ElementSize = 32;
+      break;
+  }
+  unsigned AdditionalElts = ElementSize == 16 ? 2 : 1;
+  Pow2Elts = 1 << Log2_32_Ceil(NumElts + AdditionalElts);
+
+  return MVT::getVectorVT(MVT::getVT(ElementType, false),
+                          Pow2Elts);
+}
+
 bool SITargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
                                           const CallInst &CI,
                                           MachineFunction &MF,
@@ -840,7 +881,12 @@ bool SITargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
     Info.flags = MachineMemOperand::MODereferenceable;
     if (Attr.hasFnAttribute(Attribute::ReadOnly)) {
       Info.opc = ISD::INTRINSIC_W_CHAIN;
-      Info.memVT = MVT::getVT(CI.getType());
+      Info.memVT = MVT::getVT(CI.getType(), true);
+      if (Info.memVT == MVT::Other) {
+        // Some intrinsics return an aggregate type - special case to work out
+        // the correct memVT
+        Info.memVT = memVTFromAggregate(CI.getType());
+      }
       Info.flags |= MachineMemOperand::MOLoad;
     } else if (Attr.hasFnAttribute(Attribute::WriteOnly)) {
       Info.opc = ISD::INTRINSIC_VOID;
@@ -863,6 +909,8 @@ bool SITargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
   switch (IntrID) {
   case Intrinsic::amdgcn_atomic_inc:
   case Intrinsic::amdgcn_atomic_dec:
+  case Intrinsic::amdgcn_ds_ordered_add:
+  case Intrinsic::amdgcn_ds_ordered_swap:
   case Intrinsic::amdgcn_ds_fadd:
   case Intrinsic::amdgcn_ds_fmin:
   case Intrinsic::amdgcn_ds_fmax: {
@@ -878,7 +926,20 @@ bool SITargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
 
     return true;
   }
+  case Intrinsic::amdgcn_ds_append:
+  case Intrinsic::amdgcn_ds_consume: {
+    Info.opc = ISD::INTRINSIC_W_CHAIN;
+    Info.memVT = MVT::getVT(CI.getType());
+    Info.ptrVal = CI.getOperand(0);
+    Info.align = 0;
+    Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore;
+
+    const ConstantInt *Vol = dyn_cast<ConstantInt>(CI.getOperand(1));
+    if (!Vol || !Vol->isZero())
+      Info.flags |= MachineMemOperand::MOVolatile;
 
+    return true;
+  }
   default:
     return false;
   }
@@ -890,6 +951,8 @@ bool SITargetLowering::getAddrModeArguments(IntrinsicInst *II,
   switch (II->getIntrinsicID()) {
   case Intrinsic::amdgcn_atomic_inc:
   case Intrinsic::amdgcn_atomic_dec:
+  case Intrinsic::amdgcn_ds_ordered_add:
+  case Intrinsic::amdgcn_ds_ordered_swap:
   case Intrinsic::amdgcn_ds_fadd:
   case Intrinsic::amdgcn_ds_fmin:
   case Intrinsic::amdgcn_ds_fmax: {
@@ -1154,7 +1217,8 @@ EVT SITargetLowering::getOptimalMemOpType(uint64_t Size, unsigned DstAlign,
 static bool isFlatGlobalAddrSpace(unsigned AS) {
   return AS == AMDGPUAS::GLOBAL_ADDRESS ||
          AS == AMDGPUAS::FLAT_ADDRESS ||
-         AS == AMDGPUAS::CONSTANT_ADDRESS;
+         AS == AMDGPUAS::CONSTANT_ADDRESS ||
+         AS > AMDGPUAS::MAX_AMDGPU_ADDRESS;
 }
 
 bool SITargetLowering::isNoopAddrSpaceCast(unsigned SrcAS,
@@ -1928,7 +1992,8 @@ SDValue SITargetLowering::LowerFormalArguments(
       auto *ParamTy =
         dyn_cast<PointerType>(FType->getParamType(Ins[i].getOrigArgIndex()));
       if (Subtarget->getGeneration() == AMDGPUSubtarget::SOUTHERN_ISLANDS &&
-          ParamTy && ParamTy->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS) {
+          ParamTy && (ParamTy->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS ||
+                      ParamTy->getAddressSpace() == AMDGPUAS::REGION_ADDRESS)) {
         // On SI local pointers are just offsets into LDS, so they are always
         // less than 16-bits.  On CI and newer they could potentially be
         // real pointers, so we can't guarantee their size.
@@ -2647,6 +2712,11 @@ SDValue SITargetLowering::LowerCall(CallLoweringInfo &CLI,
   std::vector<SDValue> Ops;
   Ops.push_back(Chain);
   Ops.push_back(Callee);
+  // Add a redundant copy of the callee global which will not be legalized, as
+  // we need direct access to the callee later.
+  GlobalAddressSDNode *GSD = cast<GlobalAddressSDNode>(Callee);
+  const GlobalValue *GV = GSD->getGlobal();
+  Ops.push_back(DAG.getTargetGlobalAddress(GV, DL, MVT::i64));
 
   if (IsTailCall) {
     // Each tail call may have to adjust the stack by a different amount, so
@@ -2867,7 +2937,7 @@ static MachineBasicBlock::iterator emitLoadM0FromVGPRLoop(
 
   // Update EXEC, switch all done bits to 0 and all todo bits to 1.
   MachineInstr *InsertPt =
-    BuildMI(LoopBB, I, DL, TII->get(AMDGPU::S_XOR_B64), AMDGPU::EXEC)
+    BuildMI(LoopBB, I, DL, TII->get(AMDGPU::S_XOR_B64_term), AMDGPU::EXEC)
     .addReg(AMDGPU::EXEC)
     .addReg(NewExec);
 
@@ -3410,34 +3480,16 @@ MachineBasicBlock *SITargetLowering::EmitInstrWithCustomInserter(
         .addReg(Info->getFrameOffsetReg(), RegState::Implicit);
     return BB;
   }
-  case AMDGPU::SI_CALL_ISEL:
-  case AMDGPU::SI_TCRETURN_ISEL: {
+  case AMDGPU::SI_CALL_ISEL: {
     const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
     const DebugLoc &DL = MI.getDebugLoc();
-    unsigned ReturnAddrReg = TII->getRegisterInfo().getReturnAddressReg(*MF);
 
-    MachineRegisterInfo &MRI = MF->getRegInfo();
-    unsigned GlobalAddrReg = MI.getOperand(0).getReg();
-    MachineInstr *PCRel = MRI.getVRegDef(GlobalAddrReg);
-    assert(PCRel->getOpcode() == AMDGPU::SI_PC_ADD_REL_OFFSET);
-
-    const GlobalValue *G = PCRel->getOperand(1).getGlobal();
+    unsigned ReturnAddrReg = TII->getRegisterInfo().getReturnAddressReg(*MF);
 
     MachineInstrBuilder MIB;
-    if (MI.getOpcode() == AMDGPU::SI_CALL_ISEL) {
-      MIB = BuildMI(*BB, MI, DL, TII->get(AMDGPU::SI_CALL), ReturnAddrReg)
-        .add(MI.getOperand(0))
-        .addGlobalAddress(G);
-    } else {
-      MIB = BuildMI(*BB, MI, DL, TII->get(AMDGPU::SI_TCRETURN))
-        .add(MI.getOperand(0))
-        .addGlobalAddress(G);
+    MIB = BuildMI(*BB, MI, DL, TII->get(AMDGPU::SI_CALL), ReturnAddrReg);
 
-      // There is an additional imm operand for tcreturn, but it should be in the
-      // right place already.
-    }
-
-    for (unsigned I = 1, E = MI.getNumOperands(); I != E; ++I)
+    for (unsigned I = 0, E = MI.getNumOperands(); I != E; ++I)
       MIB.add(MI.getOperand(I));
 
     MIB.cloneMemRefs(MI);
@@ -3944,7 +3996,10 @@ bool SITargetLowering::shouldEmitFixup(const GlobalValue *GV) const {
 }
 
 bool SITargetLowering::shouldEmitGOTReloc(const GlobalValue *GV) const {
-  return (GV->getType()->getAddressSpace() == AMDGPUAS::GLOBAL_ADDRESS ||
+  // FIXME: Either avoid relying on address space here or change the default
+  // address space for functions to avoid the explicit check.
+  return (GV->getValueType()->isFunctionTy() ||
+          GV->getType()->getAddressSpace() == AMDGPUAS::GLOBAL_ADDRESS ||
           GV->getType()->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS ||
           GV->getType()->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS_32BIT) &&
          !shouldEmitFixup(GV) &&
@@ -4305,12 +4360,12 @@ SDValue SITargetLowering::lowerINSERT_VECTOR_ELT(SDValue Op,
   MVT IntVT = MVT::getIntegerVT(VecSize);
 
   // Avoid stack access for dynamic indexing.
-  SDValue Val = InsVal;
-  if (InsVal.getValueType() == MVT::f16)
-      Val = DAG.getNode(ISD::BITCAST, SL, MVT::i16, InsVal);
-
   // v_bfi_b32 (v_bfm_b32 16, (shl idx, 16)), val, vec
-  SDValue ExtVal = DAG.getNode(ISD::ZERO_EXTEND, SL, IntVT, Val);
+
+  // Create a congruent vector with the target value in each element so that
+  // the required element can be masked and ORed into the target vector.
+  SDValue ExtVal = DAG.getNode(ISD::BITCAST, SL, IntVT,
+                               DAG.getSplatBuildVector(VecVT, SL, InsVal));
 
   assert(isPowerOf2_32(EltSize));
   SDValue ScaleFactor = DAG.getConstant(Log2_32(EltSize), SL, MVT::i32);
@@ -4613,6 +4668,109 @@ static bool parseCachePolicy(SDValue CachePolicy, SelectionDAG &DAG,
   return Value == 0;
 }
 
+// Re-construct the required return value for a image load intrinsic.
+// This is more complicated due to the optional use TexFailCtrl which means the required
+// return type is an aggregate
+static SDValue constructRetValue(SelectionDAG &DAG,
+                                 MachineSDNode *Result,
+                                 ArrayRef<EVT> ResultTypes,
+                                 bool IsTexFail, bool Unpacked, bool IsD16,
+                                 int DMaskPop, int NumVDataDwords,
+                                 const SDLoc &DL, LLVMContext &Context) {
+  // Determine the required return type. This is the same regardless of IsTexFail flag
+  EVT ReqRetVT = ResultTypes[0];
+  EVT ReqRetEltVT = ReqRetVT.isVector() ? ReqRetVT.getVectorElementType() : ReqRetVT;
+  int ReqRetNumElts = ReqRetVT.isVector() ? ReqRetVT.getVectorNumElements() : 1;
+  EVT AdjEltVT = Unpacked && IsD16 ? MVT::i32 : ReqRetEltVT;
+  EVT AdjVT = Unpacked ? ReqRetNumElts > 1 ? EVT::getVectorVT(Context, AdjEltVT, ReqRetNumElts)
+                                           : AdjEltVT
+                       : ReqRetVT;
+
+  // Extract data part of the result
+  // Bitcast the result to the same type as the required return type
+  int NumElts;
+  if (IsD16 && !Unpacked)
+    NumElts = NumVDataDwords << 1;
+  else
+    NumElts = NumVDataDwords;
+
+  EVT CastVT = NumElts > 1 ? EVT::getVectorVT(Context, AdjEltVT, NumElts)
+                           : AdjEltVT;
+
+  // Special case for v8f16. Rather than add support for this, use v4i32 to
+  // extract the data elements
+  bool V8F16Special = false;
+  if (CastVT == MVT::v8f16) {
+    CastVT = MVT::v4i32;
+    DMaskPop >>= 1;
+    ReqRetNumElts >>= 1;
+    V8F16Special = true;
+    AdjVT = MVT::v2i32;
+  }
+
+  SDValue N = SDValue(Result, 0);
+  SDValue CastRes = DAG.getNode(ISD::BITCAST, DL, CastVT, N);
+
+  // Iterate over the result
+  SmallVector<SDValue, 4> BVElts;
+
+  if (CastVT.isVector()) {
+    DAG.ExtractVectorElements(CastRes, BVElts, 0, DMaskPop);
+  } else {
+    BVElts.push_back(CastRes);
+  }
+  int ExtraElts = ReqRetNumElts - DMaskPop;
+  while(ExtraElts--)
+    BVElts.push_back(DAG.getUNDEF(AdjEltVT));
+
+  SDValue PreTFCRes;
+  if (ReqRetNumElts > 1) {
+    SDValue NewVec = DAG.getBuildVector(AdjVT, DL, BVElts);
+    if (IsD16 && Unpacked)
+      PreTFCRes = adjustLoadValueTypeImpl(NewVec, ReqRetVT, DL, DAG, Unpacked);
+    else
+      PreTFCRes = NewVec;
+  } else {
+    PreTFCRes = BVElts[0];
+  }
+
+  if (V8F16Special)
+    PreTFCRes = DAG.getNode(ISD::BITCAST, DL, MVT::v4f16, PreTFCRes);
+
+  if (!IsTexFail) {
+    if (Result->getNumValues() > 1)
+      return DAG.getMergeValues({PreTFCRes, SDValue(Result, 1)}, DL);
+    else
+      return PreTFCRes;
+  }
+
+  // Extract the TexFail result and insert into aggregate return
+  SmallVector<SDValue, 1> TFCElt;
+  DAG.ExtractVectorElements(N, TFCElt, DMaskPop, 1);
+  SDValue TFCRes = DAG.getNode(ISD::BITCAST, DL, ResultTypes[1], TFCElt[0]);
+  return DAG.getMergeValues({PreTFCRes, TFCRes, SDValue(Result, 1)}, DL);
+}
+
+static bool parseTexFail(SDValue TexFailCtrl, SelectionDAG &DAG, SDValue *TFE,
+                         SDValue *LWE, bool &IsTexFail) {
+  auto TexFailCtrlConst = dyn_cast<ConstantSDNode>(TexFailCtrl.getNode());
+  if (!TexFailCtrlConst)
+    return false;
+
+  uint64_t Value = TexFailCtrlConst->getZExtValue();
+  if (Value) {
+    IsTexFail = true;
+  }
+
+  SDLoc DL(TexFailCtrlConst);
+  *TFE = DAG.getTargetConstant((Value & 0x1) ? 1 : 0, DL, MVT::i32);
+  Value &= ~(uint64_t)0x1;
+  *LWE = DAG.getTargetConstant((Value & 0x2) ? 1 : 0, DL, MVT::i32);
+  Value &= ~(uint64_t)0x2;
+
+  return Value == 0;
+}
+
 SDValue SITargetLowering::lowerImage(SDValue Op,
                                      const AMDGPU::ImageDimIntrinsicInfo *Intr,
                                      SelectionDAG &DAG) const {
@@ -4626,13 +4784,17 @@ SDValue SITargetLowering::lowerImage(SDValue Op,
       AMDGPU::getMIMGLZMappingInfo(Intr->BaseOpcode);
   unsigned IntrOpcode = Intr->BaseOpcode;
 
-  SmallVector<EVT, 2> ResultTypes(Op->value_begin(), Op->value_end());
+  SmallVector<EVT, 3> ResultTypes(Op->value_begin(), Op->value_end());
+  SmallVector<EVT, 3> OrigResultTypes(Op->value_begin(), Op->value_end());
   bool IsD16 = false;
   bool IsA16 = false;
   SDValue VData;
   int NumVDataDwords;
+  bool AdjustRetType = false;
+
   unsigned AddrIdx; // Index of first address argument
   unsigned DMask;
+  unsigned DMaskLanes = 0;
 
   if (BaseOpcode->Atomic) {
     VData = Op.getOperand(2);
@@ -4655,7 +4817,12 @@ SDValue SITargetLowering::lowerImage(SDValue Op,
       AddrIdx = 3;
     }
   } else {
-    unsigned DMaskIdx;
+    unsigned DMaskIdx = BaseOpcode->Store ? 3 : isa<MemSDNode>(Op) ? 2 : 1;
+    auto DMaskConst = dyn_cast<ConstantSDNode>(Op.getOperand(DMaskIdx));
+    if (!DMaskConst)
+      return Op;
+    DMask = DMaskConst->getZExtValue();
+    DMaskLanes = BaseOpcode->Gather4 ? 4 : countPopulation(DMask);
 
     if (BaseOpcode->Store) {
       VData = Op.getOperand(2);
@@ -4671,37 +4838,32 @@ SDValue SITargetLowering::lowerImage(SDValue Op,
       }
 
       NumVDataDwords = (VData.getValueType().getSizeInBits() + 31) / 32;
-      DMaskIdx = 3;
     } else {
-      MVT LoadVT = Op.getSimpleValueType();
+      // Work out the num dwords based on the dmask popcount and underlying type
+      // and whether packing is supported.
+      MVT LoadVT = ResultTypes[0].getSimpleVT();
       if (LoadVT.getScalarType() == MVT::f16) {
         if (Subtarget->getGeneration() < AMDGPUSubtarget::VOLCANIC_ISLANDS ||
             !BaseOpcode->HasD16)
           return Op; // D16 is unsupported for this instruction
 
         IsD16 = true;
-        if (LoadVT.isVector() && Subtarget->hasUnpackedD16VMem())
-          ResultTypes[0] = (LoadVT == MVT::v2f16) ? MVT::v2i32 : MVT::v4i32;
       }
 
-      NumVDataDwords = (ResultTypes[0].getSizeInBits() + 31) / 32;
-      DMaskIdx = isa<MemSDNode>(Op) ? 2 : 1;
-    }
+      // Confirm that the return type is large enough for the dmask specified
+      if ((LoadVT.isVector() && LoadVT.getVectorNumElements() < DMaskLanes) ||
+          (!LoadVT.isVector() && DMaskLanes > 1))
+          return Op;
 
-    auto DMaskConst = dyn_cast<ConstantSDNode>(Op.getOperand(DMaskIdx));
-    if (!DMaskConst)
-      return Op;
+      if (IsD16 && !Subtarget->hasUnpackedD16VMem())
+        NumVDataDwords = (DMaskLanes + 1) / 2;
+      else
+        NumVDataDwords = DMaskLanes;
 
-    AddrIdx = DMaskIdx + 1;
-    DMask = DMaskConst->getZExtValue();
-    if (!DMask && !BaseOpcode->Store) {
-      // Eliminate no-op loads. Stores with dmask == 0 are *not* no-op: they
-      // store the channels' default values.
-      SDValue Undef = DAG.getUNDEF(Op.getValueType());
-      if (isa<MemSDNode>(Op))
-        return DAG.getMergeValues({Undef, Op.getOperand(0)}, DL);
-      return Undef;
+      AdjustRetType = true;
     }
+
+    AddrIdx = DMaskIdx + 1;
   }
 
   unsigned NumGradients = BaseOpcode->Gradients ? DimInfo->NumGradients : 0;
@@ -4780,11 +4942,53 @@ SDValue SITargetLowering::lowerImage(SDValue Op,
     CtrlIdx = AddrIdx + NumVAddrs + 3;
   }
 
+  SDValue TFE;
+  SDValue LWE;
   SDValue TexFail = Op.getOperand(CtrlIdx);
-  auto TexFailConst = dyn_cast<ConstantSDNode>(TexFail.getNode());
-  if (!TexFailConst || TexFailConst->getZExtValue() != 0)
+  bool IsTexFail = false;
+  if (!parseTexFail(TexFail, DAG, &TFE, &LWE, IsTexFail))
     return Op;
 
+  if (IsTexFail) {
+    if (!DMaskLanes) {
+      // Expecting to get an error flag since TFC is on - and dmask is 0
+      // Force dmask to be at least 1 otherwise the instruction will fail
+      DMask = 0x1;
+      DMaskLanes = 1;
+      NumVDataDwords = 1;
+    }
+    NumVDataDwords += 1;
+    AdjustRetType = true;
+  }
+
+  // Has something earlier tagged that the return type needs adjusting
+  // This happens if the instruction is a load or has set TexFailCtrl flags
+  if (AdjustRetType) {
+    // NumVDataDwords reflects the true number of dwords required in the return type
+    if (DMaskLanes == 0 && !BaseOpcode->Store) {
+      // This is a no-op load. This can be eliminated
+      SDValue Undef = DAG.getUNDEF(Op.getValueType());
+      if (isa<MemSDNode>(Op))
+        return DAG.getMergeValues({Undef, Op.getOperand(0)}, DL);
+      return Undef;
+    }
+
+    // Have to use a power of 2 number of dwords
+    NumVDataDwords = 1 << Log2_32_Ceil(NumVDataDwords);
+
+    EVT NewVT = NumVDataDwords > 1 ?
+                  EVT::getVectorVT(*DAG.getContext(), MVT::f32, NumVDataDwords)
+                : MVT::f32;
+
+    ResultTypes[0] = NewVT;
+    if (ResultTypes.size() == 3) {
+      // Original result was aggregate type used for TexFailCtrl results
+      // The actual instruction returns as a vector type which has now been
+      // created. Remove the aggregate result.
+      ResultTypes.erase(&ResultTypes[1]);
+    }
+  }
+
   SDValue GLC;
   SDValue SLC;
   if (BaseOpcode->Atomic) {
@@ -4809,8 +5013,8 @@ SDValue SITargetLowering::lowerImage(SDValue Op,
   Ops.push_back(SLC);
   Ops.push_back(IsA16 &&  // a16 or r128
                 ST->hasFeature(AMDGPU::FeatureR128A16) ? True : False);
-  Ops.push_back(False); // tfe
-  Ops.push_back(False); // lwe
+  Ops.push_back(TFE); // tfe
+  Ops.push_back(LWE); // lwe
   Ops.push_back(DimInfo->DA ? True : False);
   if (BaseOpcode->HasD16)
     Ops.push_back(IsD16 ? True : False);
@@ -4838,11 +5042,12 @@ SDValue SITargetLowering::lowerImage(SDValue Op,
     SmallVector<SDValue, 1> Elt;
     DAG.ExtractVectorElements(SDValue(NewNode, 0), Elt, 0, 1);
     return DAG.getMergeValues({Elt[0], SDValue(NewNode, 1)}, DL);
-  } else if (IsD16 && !BaseOpcode->Store) {
-    MVT LoadVT = Op.getSimpleValueType();
-    SDValue Adjusted = adjustLoadValueTypeImpl(
-        SDValue(NewNode, 0), LoadVT, DL, DAG, Subtarget->hasUnpackedD16VMem());
-    return DAG.getMergeValues({Adjusted, SDValue(NewNode, 1)}, DL);
+  } else if (!BaseOpcode->Store) {
+    return constructRetValue(DAG, NewNode,
+                             OrigResultTypes, IsTexFail,
+                             Subtarget->hasUnpackedD16VMem(), IsD16,
+                             DMaskLanes, NumVDataDwords, DL,
+                             *DAG.getContext());
   }
 
   return SDValue(NewNode, 0);
@@ -5066,12 +5271,6 @@ SDValue SITargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
     return loadInputValue(DAG, &AMDGPU::VGPR_32RegClass, MVT::i32,
                           SDLoc(DAG.getEntryNode()),
                           MFI->getArgInfo().WorkItemIDZ);
-  case SIIntrinsic::SI_load_const: {
-    SDValue Load =
-        lowerSBuffer(MVT::i32, DL, Op.getOperand(1), Op.getOperand(2),
-                     DAG.getTargetConstant(0, DL, MVT::i1), DAG);
-    return DAG.getNode(ISD::BITCAST, DL, MVT::f32, Load);
-  }
   case Intrinsic::amdgcn_s_buffer_load: {
     unsigned Cache = cast<ConstantSDNode>(Op.getOperand(3))->getZExtValue();
     return lowerSBuffer(VT, DL, Op.getOperand(1), Op.getOperand(2),
@@ -5098,6 +5297,59 @@ SDValue SITargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
                        Op.getOperand(2), Op.getOperand(3), Op.getOperand(4),
                        Glue);
   }
+  case Intrinsic::amdgcn_interp_p1_f16: {
+    SDValue M0 = copyToM0(DAG, DAG.getEntryNode(), DL, Op.getOperand(5));
+    SDValue Glue = M0.getValue(1);
+    if (getSubtarget()->getLDSBankCount() == 16) {
+      // 16 bank LDS
+      SDValue S = DAG.getNode(AMDGPUISD::INTERP_MOV, DL, MVT::f32,
+                              DAG.getConstant(2, DL, MVT::i32), // P0
+                              Op.getOperand(2), // Attrchan
+                              Op.getOperand(3), // Attr
+                              Glue);
+      SDValue Ops[] = {
+        Op.getOperand(1), // Src0
+        Op.getOperand(2), // Attrchan
+        Op.getOperand(3), // Attr
+        DAG.getConstant(0, DL, MVT::i32), // $src0_modifiers
+        S, // Src2 - holds two f16 values selected by high
+        DAG.getConstant(0, DL, MVT::i32), // $src2_modifiers
+        Op.getOperand(4), // high
+        DAG.getConstant(0, DL, MVT::i1), // $clamp
+        DAG.getConstant(0, DL, MVT::i32) // $omod
+      };
+      return DAG.getNode(AMDGPUISD::INTERP_P1LV_F16, DL, MVT::f32, Ops);
+    } else {
+      // 32 bank LDS
+      SDValue Ops[] = {
+        Op.getOperand(1), // Src0
+        Op.getOperand(2), // Attrchan
+        Op.getOperand(3), // Attr
+        DAG.getConstant(0, DL, MVT::i32), // $src0_modifiers
+        Op.getOperand(4), // high
+        DAG.getConstant(0, DL, MVT::i1), // $clamp
+        DAG.getConstant(0, DL, MVT::i32), // $omod
+        Glue
+      };
+      return DAG.getNode(AMDGPUISD::INTERP_P1LL_F16, DL, MVT::f32, Ops);
+    }
+  }
+  case Intrinsic::amdgcn_interp_p2_f16: {
+    SDValue M0 = copyToM0(DAG, DAG.getEntryNode(), DL, Op.getOperand(6));
+    SDValue Glue = SDValue(M0.getNode(), 1);
+    SDValue Ops[] = {
+      Op.getOperand(2), // Src0
+      Op.getOperand(3), // Attrchan
+      Op.getOperand(4), // Attr
+      DAG.getConstant(0, DL, MVT::i32), // $src0_modifiers
+      Op.getOperand(1), // Src2
+      DAG.getConstant(0, DL, MVT::i32), // $src2_modifiers
+      Op.getOperand(5), // high
+      DAG.getConstant(0, DL, MVT::i1), // $clamp
+      Glue
+    };
+    return DAG.getNode(AMDGPUISD::INTERP_P2_F16, DL, MVT::f16, Ops);
+  }
   case Intrinsic::amdgcn_sin:
     return DAG.getNode(AMDGPUISD::SIN_HW, DL, VT, Op.getOperand(1));
 
@@ -5158,6 +5410,11 @@ SDValue SITargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
                        Denominator, Numerator);
   }
   case Intrinsic::amdgcn_icmp: {
+    // There is a Pat that handles this variant, so return it as-is.
+    if (Op.getOperand(1).getValueType() == MVT::i1 &&
+        Op.getConstantOperandVal(2) == 0 &&
+        Op.getConstantOperandVal(3) == ICmpInst::Predicate::ICMP_NE)
+      return Op;
     return lowerICMPIntrinsic(*this, Op.getNode(), DAG);
   }
   case Intrinsic::amdgcn_fcmp: {
@@ -5236,9 +5493,78 @@ SDValue SITargetLowering::LowerINTRINSIC_W_CHAIN(SDValue Op,
   SDLoc DL(Op);
 
   switch (IntrID) {
+  case Intrinsic::amdgcn_ds_ordered_add:
+  case Intrinsic::amdgcn_ds_ordered_swap: {
+    MemSDNode *M = cast<MemSDNode>(Op);
+    SDValue Chain = M->getOperand(0);
+    SDValue M0 = M->getOperand(2);
+    SDValue Value = M->getOperand(3);
+    unsigned OrderedCountIndex = M->getConstantOperandVal(7);
+    unsigned WaveRelease = M->getConstantOperandVal(8);
+    unsigned WaveDone = M->getConstantOperandVal(9);
+    unsigned ShaderType;
+    unsigned Instruction;
+
+    switch (IntrID) {
+    case Intrinsic::amdgcn_ds_ordered_add:
+      Instruction = 0;
+      break;
+    case Intrinsic::amdgcn_ds_ordered_swap:
+      Instruction = 1;
+      break;
+    }
+
+    if (WaveDone && !WaveRelease)
+      report_fatal_error("ds_ordered_count: wave_done requires wave_release");
+
+    switch (DAG.getMachineFunction().getFunction().getCallingConv()) {
+    case CallingConv::AMDGPU_CS:
+    case CallingConv::AMDGPU_KERNEL:
+      ShaderType = 0;
+      break;
+    case CallingConv::AMDGPU_PS:
+      ShaderType = 1;
+      break;
+    case CallingConv::AMDGPU_VS:
+      ShaderType = 2;
+      break;
+    case CallingConv::AMDGPU_GS:
+      ShaderType = 3;
+      break;
+    default:
+      report_fatal_error("ds_ordered_count unsupported for this calling conv");
+    }
+
+    unsigned Offset0 = OrderedCountIndex << 2;
+    unsigned Offset1 = WaveRelease | (WaveDone << 1) | (ShaderType << 2) |
+                       (Instruction << 4);
+    unsigned Offset = Offset0 | (Offset1 << 8);
+
+    SDValue Ops[] = {
+      Chain,
+      Value,
+      DAG.getTargetConstant(Offset, DL, MVT::i16),
+      copyToM0(DAG, Chain, DL, M0).getValue(1), // Glue
+    };
+    return DAG.getMemIntrinsicNode(AMDGPUISD::DS_ORDERED_COUNT, DL,
+                                   M->getVTList(), Ops, M->getMemoryVT(),
+                                   M->getMemOperand());
+  }
+  case Intrinsic::amdgcn_ds_fadd: {
+    MemSDNode *M = cast<MemSDNode>(Op);
+    unsigned Opc;
+    switch (IntrID) {
+    case Intrinsic::amdgcn_ds_fadd:
+      Opc = ISD::ATOMIC_LOAD_FADD;
+      break;
+    }
+
+    return DAG.getAtomic(Opc, SDLoc(Op), M->getMemoryVT(),
+                         M->getOperand(0), M->getOperand(2), M->getOperand(3),
+                         M->getMemOperand());
+  }
   case Intrinsic::amdgcn_atomic_inc:
   case Intrinsic::amdgcn_atomic_dec:
-  case Intrinsic::amdgcn_ds_fadd:
   case Intrinsic::amdgcn_ds_fmin:
   case Intrinsic::amdgcn_ds_fmax: {
     MemSDNode *M = cast<MemSDNode>(Op);
@@ -5250,9 +5576,6 @@ SDValue SITargetLowering::LowerINTRINSIC_W_CHAIN(SDValue Op,
     case Intrinsic::amdgcn_atomic_dec:
       Opc = AMDGPUISD::ATOMIC_DEC;
       break;
-    case Intrinsic::amdgcn_ds_fadd:
-      Opc = AMDGPUISD::ATOMIC_LOAD_FADD;
-      break;
     case Intrinsic::amdgcn_ds_fmin:
       Opc = AMDGPUISD::ATOMIC_LOAD_FMIN;
       break;
@@ -6005,11 +6328,13 @@ std::pair<SDValue, SDValue> SITargetLowering::splitBufferOffsets(
   const unsigned MaxImm = 4095;
   SDValue N0 = Offset;
   ConstantSDNode *C1 = nullptr;
-  if (N0.getOpcode() == ISD::ADD) {
-    if ((C1 = dyn_cast<ConstantSDNode>(N0.getOperand(1))))
-      N0 = N0.getOperand(0);
-  } else if ((C1 = dyn_cast<ConstantSDNode>(N0)))
+
+  if ((C1 = dyn_cast<ConstantSDNode>(N0)))
     N0 = SDValue();
+  else if (DAG.isBaseWithConstantOffset(N0)) {
+    C1 = cast<ConstantSDNode>(N0.getOperand(1));
+    N0 = N0.getOperand(0);
+  }
 
   if (C1) {
     unsigned ImmOffset = C1->getZExtValue();
@@ -8383,7 +8708,7 @@ SDValue SITargetLowering::performFMACombine(SDNode *N,
   EVT VT = N->getValueType(0);
   SDLoc SL(N);
 
-  if (!Subtarget->hasDLInsts() || VT != MVT::f32)
+  if (!Subtarget->hasDot2Insts() || VT != MVT::f32)
     return SDValue();
 
   // FMA((F32)S0.x, (F32)S1. x, FMA((F32)S0.y, (F32)S1.y, (F32)z)) ->
@@ -8584,8 +8909,7 @@ SDValue SITargetLowering::performCvtF32UByteNCombine(SDNode *N,
   TargetLowering::TargetLoweringOpt TLO(DAG, !DCI.isBeforeLegalize(),
                                         !DCI.isBeforeLegalizeOps());
   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
-  if (TLI.ShrinkDemandedConstant(Src, Demanded, TLO) ||
-      TLI.SimplifyDemandedBits(Src, Demanded, Known, TLO)) {
+  if (TLI.SimplifyDemandedBits(Src, Demanded, Known, TLO)) {
     DCI.CommitTargetLoweringOpt(TLO);
   }
 
@@ -8670,11 +8994,11 @@ SDValue SITargetLowering::PerformDAGCombine(SDNode *N,
   case ISD::ATOMIC_LOAD_MAX:
   case ISD::ATOMIC_LOAD_UMIN:
   case ISD::ATOMIC_LOAD_UMAX:
+  case ISD::ATOMIC_LOAD_FADD:
   case AMDGPUISD::ATOMIC_INC:
   case AMDGPUISD::ATOMIC_DEC:
-  case AMDGPUISD::ATOMIC_LOAD_FADD:
   case AMDGPUISD::ATOMIC_LOAD_FMIN:
-  case AMDGPUISD::ATOMIC_LOAD_FMAX:  // TODO: Target mem intrinsics.
+  case AMDGPUISD::ATOMIC_LOAD_FMAX: // TODO: Target mem intrinsics.
     if (DCI.isBeforeLegalize())
       break;
     return performMemSDNodeCombine(cast<MemSDNode>(N), DCI);
@@ -8752,6 +9076,7 @@ static unsigned SubIdx2Lane(unsigned Idx) {
   case AMDGPU::sub1: return 1;
   case AMDGPU::sub2: return 2;
   case AMDGPU::sub3: return 3;
+  case AMDGPU::sub4: return 4; // Possible with TFE/LWE
   }
 }
 
@@ -8765,11 +9090,16 @@ SDNode *SITargetLowering::adjustWritemask(MachineSDNode *&Node,
   if (D16Idx >= 0 && Node->getConstantOperandVal(D16Idx))
     return Node; // not implemented for D16
 
-  SDNode *Users[4] = { nullptr };
+  SDNode *Users[5] = { nullptr };
   unsigned Lane = 0;
   unsigned DmaskIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::dmask) - 1;
   unsigned OldDmask = Node->getConstantOperandVal(DmaskIdx);
   unsigned NewDmask = 0;
+  unsigned TFEIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::tfe) - 1;
+  unsigned LWEIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::lwe) - 1;
+  bool UsesTFC = (Node->getConstantOperandVal(TFEIdx) ||
+                  Node->getConstantOperandVal(LWEIdx)) ? 1 : 0;
+  unsigned TFCLane = 0;
   bool HasChain = Node->getNumValues() > 1;
 
   if (OldDmask == 0) {
@@ -8777,6 +9107,12 @@ SDNode *SITargetLowering::adjustWritemask(MachineSDNode *&Node,
     return Node;
   }
 
+  unsigned OldBitsSet = countPopulation(OldDmask);
+  // Work out which is the TFE/LWE lane if that is enabled.
+  if (UsesTFC) {
+    TFCLane = OldBitsSet;
+  }
+
   // Try to figure out the used register components
   for (SDNode::use_iterator I = Node->use_begin(), E = Node->use_end();
        I != E; ++I) {
@@ -8796,28 +9132,49 @@ SDNode *SITargetLowering::adjustWritemask(MachineSDNode *&Node,
     // set, etc.
     Lane = SubIdx2Lane(I->getConstantOperandVal(1));
 
-    // Set which texture component corresponds to the lane.
-    unsigned Comp;
-    for (unsigned i = 0, Dmask = OldDmask; (i <= Lane) && (Dmask != 0); i++) {
-      Comp = countTrailingZeros(Dmask);
-      Dmask &= ~(1 << Comp);
-    }
+    // Check if the use is for the TFE/LWE generated result at VGPRn+1.
+    if (UsesTFC && Lane == TFCLane) {
+      Users[Lane] = *I;
+    } else {
+      // Set which texture component corresponds to the lane.
+      unsigned Comp;
+      for (unsigned i = 0, Dmask = OldDmask; (i <= Lane) && (Dmask != 0); i++) {
+        Comp = countTrailingZeros(Dmask);
+        Dmask &= ~(1 << Comp);
+      }
 
-    // Abort if we have more than one user per component
-    if (Users[Lane])
-      return Node;
+      // Abort if we have more than one user per component.
+      if (Users[Lane])
+        return Node;
 
-    Users[Lane] = *I;
-    NewDmask |= 1 << Comp;
+      Users[Lane] = *I;
+      NewDmask |= 1 << Comp;
+    }
   }
 
+  // Don't allow 0 dmask, as hardware assumes one channel enabled.
+  bool NoChannels = !NewDmask;
+  if (NoChannels) {
+    // If the original dmask has one channel - then nothing to do
+    if (OldBitsSet == 1)
+      return Node;
+    // Use an arbitrary dmask - required for the instruction to work
+    NewDmask = 1;
+  }
   // Abort if there's no change
   if (NewDmask == OldDmask)
     return Node;
 
   unsigned BitsSet = countPopulation(NewDmask);
 
-  int NewOpcode = AMDGPU::getMaskedMIMGOp(Node->getMachineOpcode(), BitsSet);
+  // Check for TFE or LWE - increase the number of channels by one to account
+  // for the extra return value
+  // This will need adjustment for D16 if this is also included in
+  // adjustWriteMask (this function) but at present D16 are excluded.
+  unsigned NewChannels = BitsSet + UsesTFC;
+
+  int NewOpcode =
+      AMDGPU::getMaskedMIMGOp(Node->getMachineOpcode(), NewChannels);
   assert(NewOpcode != -1 &&
          NewOpcode != static_cast<int>(Node->getMachineOpcode()) &&
          "failed to find equivalent MIMG op");
@@ -8830,8 +9187,9 @@ SDNode *SITargetLowering::adjustWritemask(MachineSDNode *&Node,
 
   MVT SVT = Node->getValueType(0).getVectorElementType().getSimpleVT();
 
-  MVT ResultVT = BitsSet == 1 ?
-    SVT : MVT::getVectorVT(SVT, BitsSet == 3 ? 4 : BitsSet);
+  MVT ResultVT = NewChannels == 1 ?
+    SVT : MVT::getVectorVT(SVT, NewChannels == 3 ? 4 :
+                           NewChannels == 5 ? 8 : NewChannels);
   SDVTList NewVTList = HasChain ?
     DAG.getVTList(ResultVT, MVT::Other) : DAG.getVTList(ResultVT);
 
@@ -8845,7 +9203,7 @@ SDNode *SITargetLowering::adjustWritemask(MachineSDNode *&Node,
     DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), SDValue(NewNode, 1));
   }
 
-  if (BitsSet == 1) {
+  if (NewChannels == 1) {
     assert(Node->hasNUsesOfValue(1, 0));
     SDNode *Copy = DAG.getMachineNode(TargetOpcode::COPY,
                                       SDLoc(Node), Users[Lane]->getValueType(0),
@@ -8855,19 +9213,24 @@ SDNode *SITargetLowering::adjustWritemask(MachineSDNode *&Node,
   }
 
   // Update the users of the node with the new indices
-  for (unsigned i = 0, Idx = AMDGPU::sub0; i < 4; ++i) {
+  for (unsigned i = 0, Idx = AMDGPU::sub0; i < 5; ++i) {
     SDNode *User = Users[i];
-    if (!User)
-      continue;
-
-    SDValue Op = DAG.getTargetConstant(Idx, SDLoc(User), MVT::i32);
-    DAG.UpdateNodeOperands(User, SDValue(NewNode, 0), Op);
+    if (!User) {
+      // Handle the special case of NoChannels. We set NewDmask to 1 above, but
+      // Users[0] is still nullptr because channel 0 doesn't really have a use.
+      if (i || !NoChannels)
+        continue;
+    } else {
+      SDValue Op = DAG.getTargetConstant(Idx, SDLoc(User), MVT::i32);
+      DAG.UpdateNodeOperands(User, SDValue(NewNode, 0), Op);
+    }
 
     switch (Idx) {
     default: break;
     case AMDGPU::sub0: Idx = AMDGPU::sub1; break;
     case AMDGPU::sub1: Idx = AMDGPU::sub2; break;
     case AMDGPU::sub2: Idx = AMDGPU::sub3; break;
+    case AMDGPU::sub3: Idx = AMDGPU::sub4; break;
     }
   }
 
@@ -8996,6 +9359,51 @@ SDNode *SITargetLowering::PostISelFolding(MachineSDNode *Node,
     Ops.push_back(ImpDef.getValue(1));
     return DAG.getMachineNode(Opcode, SDLoc(Node), Node->getVTList(), Ops);
   }
+  case AMDGPU::FLAT_LOAD_UBYTE_D16_HI:
+  case AMDGPU::FLAT_LOAD_SBYTE_D16_HI:
+  case AMDGPU::FLAT_LOAD_SHORT_D16_HI:
+  case AMDGPU::GLOBAL_LOAD_UBYTE_D16_HI:
+  case AMDGPU::GLOBAL_LOAD_SBYTE_D16_HI:
+  case AMDGPU::GLOBAL_LOAD_SHORT_D16_HI:
+  case AMDGPU::DS_READ_U16_D16_HI:
+  case AMDGPU::DS_READ_I8_D16_HI:
+  case AMDGPU::DS_READ_U8_D16_HI:
+  case AMDGPU::BUFFER_LOAD_SHORT_D16_HI_OFFSET:
+  case AMDGPU::BUFFER_LOAD_UBYTE_D16_HI_OFFSET:
+  case AMDGPU::BUFFER_LOAD_SBYTE_D16_HI_OFFSET:
+  case AMDGPU::BUFFER_LOAD_SHORT_D16_HI_OFFEN:
+  case AMDGPU::BUFFER_LOAD_UBYTE_D16_HI_OFFEN:
+  case AMDGPU::BUFFER_LOAD_SBYTE_D16_HI_OFFEN: {
+    // For these loads that write to the HI part of a register,
+    // we should chain them to the op that writes to the LO part
+    // of the register to maintain the order.
+    unsigned NumOps = Node->getNumOperands();
+    SDValue OldChain = Node->getOperand(NumOps-1);
+
+    if (OldChain.getValueType() != MVT::Other)
+      break;
+
+    // Look for the chain to replace to.
+    SDValue Lo = Node->getOperand(NumOps-2);
+    SDNode *LoNode = Lo.getNode();
+    if (LoNode->getNumValues() == 1 ||
+        LoNode->getValueType(LoNode->getNumValues() - 1) != MVT::Other)
+      break;
+
+    SDValue NewChain = Lo.getValue(LoNode->getNumValues() - 1);
+    if (NewChain == OldChain) // Already replaced.
+      break;
+
+    SmallVector<SDValue, 16> Ops;
+    for (unsigned I = 0; I < NumOps-1; ++I)
+      Ops.push_back(Node->getOperand(I));
+    // Repalce the Chain.
+    Ops.push_back(NewChain);
+    MachineSDNode *NewNode = DAG.getMachineNode(Opcode, SDLoc(Node),
+                                                Node->getVTList(), Ops);
+    DAG.setNodeMemRefs(NewNode, Node->memoperands());
+    return NewNode;
+  }
   default:
     break;
   }
@@ -9289,7 +9697,8 @@ static bool isCopyFromRegOfInlineAsm(const SDNode *N) {
   do {
     // Follow the chain until we find an INLINEASM node.
     N = N->getOperand(0).getNode();
-    if (N->getOpcode() == ISD::INLINEASM)
+    if (N->getOpcode() == ISD::INLINEASM ||
+        N->getOpcode() == ISD::INLINEASM_BR)
       return true;
   } while (N->getOpcode() == ISD::CopyFromReg);
   return false;
@@ -9382,3 +9791,29 @@ bool SITargetLowering::isKnownNeverNaNForTargetNode(SDValue Op,
   return AMDGPUTargetLowering::isKnownNeverNaNForTargetNode(Op, DAG,
                                                             SNaN, Depth);
 }
+
+TargetLowering::AtomicExpansionKind
+SITargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *RMW) const {
+  switch (RMW->getOperation()) {
+  case AtomicRMWInst::FAdd: {
+    Type *Ty = RMW->getType();
+
+    // We don't have a way to support 16-bit atomics now, so just leave them
+    // as-is.
+    if (Ty->isHalfTy())
+      return AtomicExpansionKind::None;
+
+    if (!Ty->isFloatTy())
+      return AtomicExpansionKind::CmpXChg;
+
+    // TODO: Do have these for flat. Older targets also had them for buffers.
+    unsigned AS = RMW->getPointerAddressSpace();
+    return (AS == AMDGPUAS::LOCAL_ADDRESS && Subtarget->hasLDSFPAtomics()) ?
+      AtomicExpansionKind::None : AtomicExpansionKind::CmpXChg;
+  }
+  default:
+    break;
+  }
+
+  return AMDGPUTargetLowering::shouldExpandAtomicRMWInIR(RMW);
+}
diff --git a/lib/Target/AMDGPU/SIISelLowering.h b/lib/Target/AMDGPU/SIISelLowering.h
index bcef519ee6635ddb6b1d2f12096cce80a7ef57ef..62758ddbdccb018bfecb09a80a155f355829cf38 100644
--- a/lib/Target/AMDGPU/SIISelLowering.h
+++ b/lib/Target/AMDGPU/SIISelLowering.h
@@ -1,9 +1,8 @@
 //===-- SIISelLowering.h - SI DAG Lowering Interface ------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -352,6 +351,7 @@ public:
                                     const SelectionDAG &DAG,
                                     bool SNaN = false,
                                     unsigned Depth = 0) const override;
+  AtomicExpansionKind shouldExpandAtomicRMWInIR(AtomicRMWInst *) const override;
 };
 
 } // End namespace llvm
diff --git a/lib/Target/AMDGPU/SIInsertSkips.cpp b/lib/Target/AMDGPU/SIInsertSkips.cpp
index ba21a5ce1293a0d139ddf2089ba53e9fc1b06e66..9539deeb7d9c370baa13ab4b9c1c75101c51fd8b 100644
--- a/lib/Target/AMDGPU/SIInsertSkips.cpp
+++ b/lib/Target/AMDGPU/SIInsertSkips.cpp
@@ -1,9 +1,8 @@
 //===-- SIInsertSkips.cpp - Use predicates for control flow ---------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AMDGPU/SIInsertWaitcnts.cpp b/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
index 79674046fce1bcf9dfba54254655664670bf5e55..c3abac11150ce9a87ad488e25bc60191237d1c96 100644
--- a/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
+++ b/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
@@ -1,9 +1,8 @@
 //===- SIInsertWaitcnts.cpp - Insert Wait Instructions --------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -536,10 +535,13 @@ void WaitcntBrackets::updateByEvent(const SIInstrInfo *TII,
             CurrScore);
       }
       if (Inst.mayStore()) {
-        setExpScore(
-            &Inst, TII, TRI, MRI,
-            AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::data0),
-            CurrScore);
+        if (AMDGPU::getNamedOperandIdx(Inst.getOpcode(),
+                                       AMDGPU::OpName::data0) != -1) {
+          setExpScore(
+              &Inst, TII, TRI, MRI,
+              AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::data0),
+              CurrScore);
+        }
         if (AMDGPU::getNamedOperandIdx(Inst.getOpcode(),
                                        AMDGPU::OpName::data1) != -1) {
           setExpScore(&Inst, TII, TRI, MRI,
@@ -1093,7 +1095,8 @@ void SIInsertWaitcnts::updateEventWaitcntAfter(MachineInstr &Inst,
   // bracket and the destination operand scores.
   // TODO: Use the (TSFlags & SIInstrFlags::LGKM_CNT) property everywhere.
   if (TII->isDS(Inst) && TII->usesLGKM_CNT(Inst)) {
-    if (TII->hasModifiersSet(Inst, AMDGPU::OpName::gds)) {
+    if (TII->isAlwaysGDS(Inst.getOpcode()) ||
+        TII->hasModifiersSet(Inst, AMDGPU::OpName::gds)) {
       ScoreBrackets->updateByEvent(TII, TRI, MRI, GDS_ACCESS, Inst);
       ScoreBrackets->updateByEvent(TII, TRI, MRI, GDS_GPR_LOCK, Inst);
     } else {
@@ -1216,6 +1219,9 @@ bool WaitcntBrackets::merge(const WaitcntBrackets &Other) {
       StrictDom = true;
   }
 
+  VgprUB = std::max(getMaxVGPR(), Other.getMaxVGPR());
+  SgprUB = std::max(getMaxSGPR(), Other.getMaxSGPR());
+
   return StrictDom;
 }
 
diff --git a/lib/Target/AMDGPU/SIInstrFormats.td b/lib/Target/AMDGPU/SIInstrFormats.td
index 65ffc27b8b6084068c8b5f0da64ed079f98a5b91..06503ac7dcbdbc9e764d8c0c6f08c4036a305b46 100644
--- a/lib/Target/AMDGPU/SIInstrFormats.td
+++ b/lib/Target/AMDGPU/SIInstrFormats.td
@@ -1,9 +1,8 @@
 //===-- SIInstrFormats.td - SI Instruction Encodings ----------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -11,9 +10,6 @@
 //
 //===----------------------------------------------------------------------===//
 
-def isGCN : Predicate<"Subtarget->getGeneration() "
-                      ">= AMDGPUSubtarget::SOUTHERN_ISLANDS">,
-            AssemblerPredicate<"FeatureGCN">;
 def isSI : Predicate<"Subtarget->getGeneration() "
                       "== AMDGPUSubtarget::SOUTHERN_ISLANDS">,
            AssemblerPredicate<"FeatureSouthernIslands">;
@@ -22,8 +18,6 @@ def isSI : Predicate<"Subtarget->getGeneration() "
 class InstSI <dag outs, dag ins, string asm = "",
               list<dag> pattern = []> :
   AMDGPUInst<outs, ins, asm, pattern>, GCNPredicateControl {
-  let SubtargetPredicate = isGCN;
-
   // Low bits - basic encoding information.
   field bit SALU = 0;
   field bit VALU = 0;
diff --git a/lib/Target/AMDGPU/SIInstrInfo.cpp b/lib/Target/AMDGPU/SIInstrInfo.cpp
index b7c4eed621124654b9a6de824f9a9e6eb0fe44f0..0e3048792aa9933eabdecf8af7dbe3e9510e88d5 100644
--- a/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -1,9 +1,8 @@
 //===- SIInstrInfo.cpp - SI Instruction Information  ----------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -14,7 +13,6 @@
 
 #include "SIInstrInfo.h"
 #include "AMDGPU.h"
-#include "AMDGPUIntrinsicInfo.h"
 #include "AMDGPUSubtarget.h"
 #include "GCNHazardRecognizer.h"
 #include "SIDefines.h"
@@ -142,7 +140,8 @@ bool SIInstrInfo::isReallyTriviallyReMaterializable(const MachineInstr &MI,
   case AMDGPU::V_MOV_B32_e32:
   case AMDGPU::V_MOV_B32_e64:
   case AMDGPU::V_MOV_B64_PSEUDO:
-    return true;
+    // No implicit operands.
+    return MI.getNumOperands() == MI.getDesc().getNumOperands();
   default:
     return false;
   }
@@ -277,6 +276,11 @@ bool SIInstrInfo::getMemOperandWithOffset(MachineInstr &LdSt,
     if (OffsetImm) {
       // Normal, single offset LDS instruction.
       BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::addr);
+      // TODO: ds_consume/ds_append use M0 for the base address. Is it safe to
+      // report that here?
+      if (!BaseOp)
+        return false;
+
       Offset = OffsetImm->getImm();
       assert(BaseOp->isReg() && "getMemOperandWithOffset only supports base "
                                 "operands of type register.");
@@ -1154,7 +1158,7 @@ void SIInstrInfo::insertReturn(MachineBasicBlock &MBB) const {
   }
 }
 
-unsigned SIInstrInfo::getNumWaitStates(const MachineInstr &MI) const {
+unsigned SIInstrInfo::getNumWaitStates(const MachineInstr &MI) {
   switch (MI.getOpcode()) {
   default: return 1; // FIXME: Do wait states equal cycles?
 
@@ -2390,6 +2394,16 @@ bool SIInstrInfo::isSchedulingBoundary(const MachineInstr &MI,
          changesVGPRIndexingMode(MI);
 }
 
+bool SIInstrInfo::isAlwaysGDS(uint16_t Opcode) const {
+  return Opcode == AMDGPU::DS_ORDERED_COUNT ||
+         Opcode == AMDGPU::DS_GWS_INIT ||
+         Opcode == AMDGPU::DS_GWS_SEMA_V ||
+         Opcode == AMDGPU::DS_GWS_SEMA_BR ||
+         Opcode == AMDGPU::DS_GWS_SEMA_P ||
+         Opcode == AMDGPU::DS_GWS_SEMA_RELEASE_ALL ||
+         Opcode == AMDGPU::DS_GWS_BARRIER;
+}
+
 bool SIInstrInfo::hasUnwantedEffectsWhenEXECEmpty(const MachineInstr &MI) const {
   unsigned Opcode = MI.getOpcode();
 
@@ -2403,7 +2417,8 @@ bool SIInstrInfo::hasUnwantedEffectsWhenEXECEmpty(const MachineInstr &MI) const
   //       EXEC = 0, but checking for that case here seems not worth it
   //       given the typical code patterns.
   if (Opcode == AMDGPU::S_SENDMSG || Opcode == AMDGPU::S_SENDMSGHALT ||
-      Opcode == AMDGPU::EXP || Opcode == AMDGPU::EXP_DONE)
+      Opcode == AMDGPU::EXP || Opcode == AMDGPU::EXP_DONE ||
+      Opcode == AMDGPU::DS_ORDERED_COUNT)
     return true;
 
   if (MI.isInlineAsm())
@@ -2972,6 +2987,42 @@ bool SIInstrInfo::verifyInstruction(const MachineInstr &MI,
     }
   }
 
+  // Verify MIMG
+  if (isMIMG(MI.getOpcode()) && !MI.mayStore()) {
+    // Ensure that the return type used is large enough for all the options
+    // being used TFE/LWE require an extra result register.
+    const MachineOperand *DMask = getNamedOperand(MI, AMDGPU::OpName::dmask);
+    if (DMask) {
+      uint64_t DMaskImm = DMask->getImm();
+      uint32_t RegCount =
+          isGather4(MI.getOpcode()) ? 4 : countPopulation(DMaskImm);
+      const MachineOperand *TFE = getNamedOperand(MI, AMDGPU::OpName::tfe);
+      const MachineOperand *LWE = getNamedOperand(MI, AMDGPU::OpName::lwe);
+      const MachineOperand *D16 = getNamedOperand(MI, AMDGPU::OpName::d16);
+
+      // Adjust for packed 16 bit values
+      if (D16 && D16->getImm() && !ST.hasUnpackedD16VMem())
+        RegCount >>= 1;
+
+      // Adjust if using LWE or TFE
+      if ((LWE && LWE->getImm()) || (TFE && TFE->getImm()))
+        RegCount += 1;
+
+      const uint32_t DstIdx =
+          AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::vdata);
+      const MachineOperand &Dst = MI.getOperand(DstIdx);
+      if (Dst.isReg()) {
+        const TargetRegisterClass *DstRC = getOpRegClass(MI, DstIdx);
+        uint32_t DstSize = RI.getRegSizeInBits(*DstRC) / 32;
+        if (RegCount > DstSize) {
+          ErrInfo = "MIMG instruction returns too many registers for dst "
+                    "register class";
+          return false;
+        }
+      }
+    }
+  }
+
   // Verify VOP*. Ignore multiple sgpr operands on writelane.
   if (Desc.getOpcode() != AMDGPU::V_WRITELANE_B32
       && (isVOP1(MI) || isVOP2(MI) || isVOP3(MI) || isVOPC(MI) || isSDWA(MI))) {
@@ -3227,18 +3278,6 @@ const TargetRegisterClass *SIInstrInfo::getOpRegClass(const MachineInstr &MI,
   return RI.getRegClass(RCID);
 }
 
-bool SIInstrInfo::canReadVGPR(const MachineInstr &MI, unsigned OpNo) const {
-  switch (MI.getOpcode()) {
-  case AMDGPU::COPY:
-  case AMDGPU::REG_SEQUENCE:
-  case AMDGPU::PHI:
-  case AMDGPU::INSERT_SUBREG:
-    return RI.hasVGPRs(getOpRegClass(MI, 0));
-  default:
-    return RI.hasVGPRs(getOpRegClass(MI, OpNo));
-  }
-}
-
 void SIInstrInfo::legalizeOpWithMove(MachineInstr &MI, unsigned OpIdx) const {
   MachineBasicBlock::iterator I = MI;
   MachineBasicBlock *MBB = MI.getParent();
@@ -4907,7 +4946,23 @@ void SIInstrInfo::addUsersToMoveToVALUWorklist(
   for (MachineRegisterInfo::use_iterator I = MRI.use_begin(DstReg),
          E = MRI.use_end(); I != E;) {
     MachineInstr &UseMI = *I->getParent();
-    if (!canReadVGPR(UseMI, I.getOperandNo())) {
+
+    unsigned OpNo = 0;
+
+    switch (UseMI.getOpcode()) {
+    case AMDGPU::COPY:
+    case AMDGPU::WQM:
+    case AMDGPU::WWM:
+    case AMDGPU::REG_SEQUENCE:
+    case AMDGPU::PHI:
+    case AMDGPU::INSERT_SUBREG:
+      break;
+    default:
+      OpNo = I.getOperandNo();
+      break;
+    }
+
+    if (!RI.hasVGPRs(getOpRegClass(UseMI, OpNo))) {
       Worklist.insert(&UseMI);
 
       do {
@@ -5258,7 +5313,8 @@ unsigned SIInstrInfo::getInstSizeInBytes(const MachineInstr &MI) const {
     return 0;
   case TargetOpcode::BUNDLE:
     return getInstBundleSize(MI);
-  case TargetOpcode::INLINEASM: {
+  case TargetOpcode::INLINEASM:
+  case TargetOpcode::INLINEASM_BR: {
     const MachineFunction *MF = MI.getParent()->getParent();
     const char *AsmStr = MI.getOperand(0).getSymbolName();
     return getInlineAsmLength(AsmStr, *MF->getTarget().getMCAsmInfo());
@@ -5591,3 +5647,29 @@ MachineInstr *llvm::getVRegSubRegDef(const TargetInstrInfo::RegSubRegPair &P,
   }
   return nullptr;
 }
+
+bool llvm::isEXECMaskConstantBetweenDefAndUses(unsigned VReg,
+                                               MachineRegisterInfo &MRI) {
+  assert(MRI.isSSA() && "Must be run on SSA");
+  auto *TRI = MRI.getTargetRegisterInfo();
+
+  auto *DefI = MRI.getVRegDef(VReg);
+  auto *BB = DefI->getParent();
+
+  DenseSet<MachineInstr*> Uses;
+  for (auto &Use : MRI.use_nodbg_operands(VReg)) {
+    auto *I = Use.getParent();
+    if (I->getParent() != BB)
+      return false;
+    Uses.insert(I);
+  }
+
+  auto E = BB->end();
+  for (auto I = std::next(DefI->getIterator()); I != E; ++I) {
+    Uses.erase(&*I);
+    // don't check the last use
+    if (Uses.empty() || I->modifiesRegister(AMDGPU::EXEC, TRI))
+      break;
+  }
+  return Uses.empty();
+}
diff --git a/lib/Target/AMDGPU/SIInstrInfo.h b/lib/Target/AMDGPU/SIInstrInfo.h
index 5b1a05f3785ec722553790ae753668bee63b8f31..b96c40def00419b962ee4b211a74d22cabd4393a 100644
--- a/lib/Target/AMDGPU/SIInstrInfo.h
+++ b/lib/Target/AMDGPU/SIInstrInfo.h
@@ -1,9 +1,8 @@
 //===- SIInstrInfo.h - SI Instruction Info Interface ------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -450,6 +449,8 @@ public:
     return get(Opcode).TSFlags & SIInstrFlags::DS;
   }
 
+  bool isAlwaysGDS(uint16_t Opcode) const;
+
   static bool isMIMG(const MachineInstr &MI) {
     return MI.getDesc().TSFlags & SIInstrFlags::MIMG;
   }
@@ -761,10 +762,6 @@ public:
     return RI.getRegSizeInBits(*getOpRegClass(MI, OpNo)) / 8;
   }
 
-  /// \returns true if it is legal for the operand at index \p OpNo
-  /// to read a VGPR.
-  bool canReadVGPR(const MachineInstr &MI, unsigned OpNo) const;
-
   /// Legalize the \p OpIndex operand of this instruction by inserting
   /// a MOV.  For example:
   /// ADD_I32_e32 VGPR0, 15
@@ -836,7 +833,7 @@ public:
   void insertReturn(MachineBasicBlock &MBB) const;
   /// Return the number of wait states that result from executing this
   /// instruction.
-  unsigned getNumWaitStates(const MachineInstr &MI) const;
+  static unsigned getNumWaitStates(const MachineInstr &MI);
 
   /// Returns the operand named \p Op.  If \p MI does not have an
   /// operand named \c Op, this function returns nullptr.
@@ -956,6 +953,12 @@ TargetInstrInfo::RegSubRegPair getRegSequenceSubReg(MachineInstr &MI,
 MachineInstr *getVRegSubRegDef(const TargetInstrInfo::RegSubRegPair &P,
                                MachineRegisterInfo &MRI);
 
+/// \brief Return true if EXEC mask isnt' changed between the def and
+/// all uses of VReg. Currently if def and uses are in different BBs -
+/// simply return false. Should be run on SSA.
+bool isEXECMaskConstantBetweenDefAndUses(unsigned VReg,
+                                         MachineRegisterInfo &MRI);
+
 namespace AMDGPU {
 
   LLVM_READONLY
diff --git a/lib/Target/AMDGPU/SIInstrInfo.td b/lib/Target/AMDGPU/SIInstrInfo.td
index 13afa4d4974bf18f1b592f51e0e61a452c63ce1a..8438fc016be3fc871dff3e58c500aa0c9bc8a49e 100644
--- a/lib/Target/AMDGPU/SIInstrInfo.td
+++ b/lib/Target/AMDGPU/SIInstrInfo.td
@@ -1,9 +1,8 @@
 //===-- SIInstrInfo.td - SI Instruction Infos -------------*- tablegen -*--===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 def isCI : Predicate<"Subtarget->getGeneration() "
@@ -45,6 +44,11 @@ def SIsbuffer_load : SDNode<"AMDGPUISD::SBUFFER_LOAD",
   [SDNPMayLoad, SDNPMemOperand]
 >;
 
+def SIds_ordered_count : SDNode<"AMDGPUISD::DS_ORDERED_COUNT",
+  SDTypeProfile<1, 2, [SDTCisVT<0, i32>, SDTCisVT<1, i32>, SDTCisVT<2, i16>]>,
+  [SDNPMayLoad, SDNPMayStore, SDNPMemOperand, SDNPHasChain, SDNPInGlue]
+>;
+
 def SIatomic_inc : SDNode<"AMDGPUISD::ATOMIC_INC", SDTAtomic2,
   [SDNPMayLoad, SDNPMayStore, SDNPMemOperand, SDNPHasChain]
 >;
@@ -57,10 +61,6 @@ def SDTAtomic2_f32 : SDTypeProfile<1, 2, [
   SDTCisSameAs<0,2>, SDTCisFP<0>, SDTCisPtrTy<1>
 ]>;
 
-def SIatomic_fadd : SDNode<"AMDGPUISD::ATOMIC_LOAD_FADD", SDTAtomic2_f32,
-  [SDNPMayLoad, SDNPMayStore, SDNPMemOperand, SDNPHasChain]
->;
-
 def SIatomic_fmin : SDNode<"AMDGPUISD::ATOMIC_LOAD_FMIN", SDTAtomic2_f32,
   [SDNPMayLoad, SDNPMayStore, SDNPMemOperand, SDNPHasChain]
 >;
@@ -228,7 +228,7 @@ defm atomic_dec_global : global_binary_atomic_op<SIatomic_dec>;
 
 def atomic_inc_local : local_binary_atomic_op<SIatomic_inc>;
 def atomic_dec_local : local_binary_atomic_op<SIatomic_dec>;
-def atomic_load_fadd_local : local_binary_atomic_op<SIatomic_fadd>;
+def atomic_load_fadd_local : local_binary_atomic_op<atomic_load_fadd>;
 def atomic_load_fmin_local : local_binary_atomic_op<SIatomic_fmin>;
 def atomic_load_fmax_local : local_binary_atomic_op<SIatomic_fmax>;
 
@@ -424,7 +424,7 @@ defm atomic_load_xor : SIAtomicM0Glue2 <"LOAD_XOR">;
 defm atomic_load_umin : SIAtomicM0Glue2 <"LOAD_UMIN">;
 defm atomic_load_umax : SIAtomicM0Glue2 <"LOAD_UMAX">;
 defm atomic_swap : SIAtomicM0Glue2 <"SWAP">;
-defm atomic_load_fadd : SIAtomicM0Glue2 <"LOAD_FADD", 1, SDTAtomic2_f32>;
+defm atomic_load_fadd : SIAtomicM0Glue2 <"LOAD_FADD", 0, SDTAtomic2_f32>;
 defm atomic_load_fmin : SIAtomicM0Glue2 <"LOAD_FMIN", 1, SDTAtomic2_f32>;
 defm atomic_load_fmax : SIAtomicM0Glue2 <"LOAD_FMAX", 1, SDTAtomic2_f32>;
 
diff --git a/lib/Target/AMDGPU/SIInstructions.td b/lib/Target/AMDGPU/SIInstructions.td
index 5f78a10727503525fd54b3f06fb0c6a1486da9b1..88acebb89696131592e8c1257cd726db40d3c126 100644
--- a/lib/Target/AMDGPU/SIInstructions.td
+++ b/lib/Target/AMDGPU/SIInstructions.td
@@ -1,9 +1,8 @@
 //===-- SIInstructions.td - SI Instruction Defintions ---------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 // This file was originally auto-generated from a GPU register header file and
@@ -12,7 +11,7 @@
 //===----------------------------------------------------------------------===//
 
 class GCNPat<dag pattern, dag result> : Pat<pattern, result>, GCNPredicateControl {
-  let SubtargetPredicate = isGCN;
+
 }
 
 include "SOPInstructions.td"
@@ -374,7 +373,8 @@ def SI_RETURN : SPseudoInstSI <
 // This version is only needed so we can fill in the output regiter in
 // the custom inserter.
 def SI_CALL_ISEL : SPseudoInstSI <
-  (outs), (ins SSrc_b64:$src0), [(AMDGPUcall i64:$src0)]> {
+  (outs), (ins SSrc_b64:$src0, unknown:$callee),
+  [(AMDGPUcall i64:$src0, tglobaladdr:$callee)]> {
   let Size = 4;
   let isCall = 1;
   let SchedRW = [WriteBranch];
@@ -392,20 +392,9 @@ def SI_CALL : SPseudoInstSI <
 }
 
 // Tail call handling pseudo
-def SI_TCRETURN_ISEL : SPseudoInstSI<(outs),
-  (ins SSrc_b64:$src0, i32imm:$fpdiff),
-  [(AMDGPUtc_return i64:$src0, i32:$fpdiff)]> {
-  let isCall = 1;
-  let isTerminator = 1;
-  let isReturn = 1;
-  let isBarrier = 1;
-  let SchedRW = [WriteBranch];
-  let usesCustomInserter = 1;
-}
-
-def SI_TCRETURN : SPseudoInstSI <
-  (outs),
-  (ins SSrc_b64:$src0, unknown:$callee, i32imm:$fpdiff)> {
+def SI_TCRETURN : SPseudoInstSI <(outs),
+  (ins SSrc_b64:$src0, unknown:$callee, i32imm:$fpdiff),
+  [(AMDGPUtc_return i64:$src0, tglobaladdr:$callee, i32:$fpdiff)]> {
   let Size = 4;
   let isCall = 1;
   let isTerminator = 1;
@@ -583,11 +572,16 @@ def : Pat <
 
   // TODO: we could add more variants for other types of conditionals
 
+def : Pat <
+  (int_amdgcn_icmp i1:$src, (i1 0), (i32 33)),
+  (COPY $src) // Return the SGPRs representing i1 src
+>;
+
 //===----------------------------------------------------------------------===//
 // VOP1 Patterns
 //===----------------------------------------------------------------------===//
 
-let SubtargetPredicate = isGCN, OtherPredicates = [UnsafeFPMath] in {
+let OtherPredicates = [UnsafeFPMath] in {
 
 //def : RcpPat<V_RCP_F64_e32, f64>;
 //defm : RsqPat<V_RSQ_F64_e32, f64>;
@@ -610,7 +604,7 @@ def : GCNPat <
   (V_FRACT_F64_e64 $mods, $x, DSTCLAMP.NONE, DSTOMOD.NONE)
 >;
 
-} // End SubtargetPredicate = isGCN, OtherPredicates = [UnsafeFPMath]
+} // End OtherPredicates = [UnsafeFPMath]
 
 
 // f16_to_fp patterns
@@ -813,8 +807,6 @@ def : Pat <
   (v2f16 (EXTRACT_SUBREG v4f16:$vec, sub1))
 >;
 
-let SubtargetPredicate = isGCN in {
-
 // FIXME: Why do only some of these type combinations for SReg and
 // VReg?
 // 16-bit bitcast
@@ -852,6 +844,8 @@ def : BitConvert <f64, v2f32, VReg_64>;
 def : BitConvert <v2f32, f64, VReg_64>;
 def : BitConvert <f64, v2i32, VReg_64>;
 def : BitConvert <v2i32, f64, VReg_64>;
+def : BitConvert <v4i16, v4f16, VReg_64>;
+def : BitConvert <v4f16, v4i16, VReg_64>;
 
 // FIXME: Make SGPR
 def : BitConvert <v2i32, v4f16, VReg_64>;
@@ -895,8 +889,6 @@ def : BitConvert <v8f32, v8i32, VReg_256>;
 def : BitConvert <v16i32, v16f32, VReg_512>;
 def : BitConvert <v16f32, v16i32, VReg_512>;
 
-} // End SubtargetPredicate = isGCN
-
 /********** =================== **********/
 /********** Src & Dst modifiers **********/
 /********** =================== **********/
@@ -1108,9 +1100,7 @@ def : GCNPat <
 /********** Intrinsic Patterns **********/
 /********** ================== **********/
 
-let SubtargetPredicate = isGCN in {
 def : POW_Common <V_LOG_F32_e32, V_EXP_F32_e32, V_MUL_LEGACY_F32_e32>;
-}
 
 def : GCNPat <
   (i32 (sext i1:$src0)),
@@ -1137,8 +1127,6 @@ def : GCNPat <
 // VOP3 Patterns
 //===----------------------------------------------------------------------===//
 
-let SubtargetPredicate = isGCN in {
-
 def : IMad24Pat<V_MAD_I32_I24, 1>;
 def : UMad24Pat<V_MAD_U32_U24, 1>;
 
@@ -1146,8 +1134,6 @@ def : UMad24Pat<V_MAD_U32_U24, 1>;
 defm : BFIPatterns <V_BFI_B32, S_MOV_B32, SReg_64>;
 def : ROTRPattern <V_ALIGNBIT_B32>;
 
-}
-
 def : GCNPat<(i32 (trunc (srl i64:$src0, (and i32:$src1, (i32 31))))),
           (V_ALIGNBIT_B32 (i32 (EXTRACT_SUBREG (i64 $src0), sub1)),
                           (i32 (EXTRACT_SUBREG (i64 $src0), sub0)), $src1)>;
@@ -1615,8 +1601,6 @@ multiclass BFMPatterns <ValueType vt, InstSI BFM, InstSI MOV> {
   >;
 }
 
-let SubtargetPredicate = isGCN in {
-
 defm : BFMPatterns <i32, S_BFM_B32, S_MOV_B32>;
 // FIXME: defm : BFMPatterns <i64, S_BFM_B64, S_MOV_B64>;
 
@@ -1626,8 +1610,6 @@ defm : SHA256MaPattern <V_BFI_B32, V_XOR_B32_e64, SReg_64>;
 defm : IntMed3Pat<V_MED3_I32, smin, smax, smin_oneuse, smax_oneuse>;
 defm : IntMed3Pat<V_MED3_U32, umin, umax, umin_oneuse, umax_oneuse>;
 
-}
-
 // This matches 16 permutations of
 // max(min(x, y), min(max(x, y), z))
 class FPMed3Pat<ValueType vt,
diff --git a/lib/Target/AMDGPU/SIIntrinsics.td b/lib/Target/AMDGPU/SIIntrinsics.td
deleted file mode 100644
index e51ff4b4bc50efd42aa9eb5a9e99a4b69a7d39ee..0000000000000000000000000000000000000000
--- a/lib/Target/AMDGPU/SIIntrinsics.td
+++ /dev/null
@@ -1,19 +0,0 @@
-//===-- SIIntrinsics.td - SI Intrinsic defs ----------------*- tablegen -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// Backend internal SI Intrinsic Definitions. User code should not
-// directly use these.
-//
-//===----------------------------------------------------------------------===//
-
-
-let TargetPrefix = "SI", isTarget = 1 in {
-  def int_SI_load_const : Intrinsic <[llvm_float_ty], [llvm_anyint_ty, llvm_i32_ty], [IntrNoMem]>;
-
-} // End TargetPrefix = "SI", isTarget = 1
diff --git a/lib/Target/AMDGPU/SILoadStoreOptimizer.cpp b/lib/Target/AMDGPU/SILoadStoreOptimizer.cpp
index 52bbe5c03450b0137d8d8fab666f0abc3b13bcaf..6fc587bf0d642e8eec27685aedc72ad384d413da 100644
--- a/lib/Target/AMDGPU/SILoadStoreOptimizer.cpp
+++ b/lib/Target/AMDGPU/SILoadStoreOptimizer.cpp
@@ -1,9 +1,8 @@
 //===- SILoadStoreOptimizer.cpp -------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -160,7 +159,7 @@ private:
   bool OptimizeAgain;
 
   static bool offsetsCanBeCombined(CombineInfo &CI);
-  static bool widthsFit(const CombineInfo &CI);
+  static bool widthsFit(const GCNSubtarget &STM, const CombineInfo &CI);
   static unsigned getNewOpcode(const CombineInfo &CI);
   static std::pair<unsigned, unsigned> getSubRegIdxs(const CombineInfo &CI);
   const TargetRegisterClass *getTargetRegisterClass(const CombineInfo &CI);
@@ -367,11 +366,12 @@ bool SILoadStoreOptimizer::offsetsCanBeCombined(CombineInfo &CI) {
   return false;
 }
 
-bool SILoadStoreOptimizer::widthsFit(const CombineInfo &CI) {
+bool SILoadStoreOptimizer::widthsFit(const GCNSubtarget &STM,
+                                     const CombineInfo &CI) {
   const unsigned Width = (CI.Width0 + CI.Width1);
   switch (CI.InstClass) {
   default:
-    return Width <= 4;
+    return (Width <= 4) && (STM.hasDwordx3LoadStores() || (Width != 3));
   case S_BUFFER_LOAD_IMM:
     switch (Width) {
     default:
@@ -645,7 +645,7 @@ bool SILoadStoreOptimizer::findMatchingInst(CombineInfo &CI) {
       // We also need to go through the list of instructions that we plan to
       // move and make sure they are all safe to move down past the merged
       // instruction.
-      if (widthsFit(CI) && offsetsCanBeCombined(CI))
+      if (widthsFit(*STM, CI) && offsetsCanBeCombined(CI))
         if (canMoveInstsAcrossMemOp(*MBBI, CI.InstsToMove, TII, AA))
           return true;
     }
diff --git a/lib/Target/AMDGPU/SILowerControlFlow.cpp b/lib/Target/AMDGPU/SILowerControlFlow.cpp
index 1aa1feebbdae654304bc2535313822c098e24e85..20042801637b01a035d6a068467b45f9c0beeb1d 100644
--- a/lib/Target/AMDGPU/SILowerControlFlow.cpp
+++ b/lib/Target/AMDGPU/SILowerControlFlow.cpp
@@ -1,9 +1,8 @@
 //===-- SILowerControlFlow.cpp - Use predicates for control flow ----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AMDGPU/SILowerI1Copies.cpp b/lib/Target/AMDGPU/SILowerI1Copies.cpp
index eb038bb5d5fcf5561439bf19ddec9dd65f58a89d..25536176971f7c1199755a21ee94e9dbe3b5c015 100644
--- a/lib/Target/AMDGPU/SILowerI1Copies.cpp
+++ b/lib/Target/AMDGPU/SILowerI1Copies.cpp
@@ -1,9 +1,8 @@
 //===-- SILowerI1Copies.cpp - Lower I1 Copies -----------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AMDGPU/SIMachineFunctionInfo.cpp b/lib/Target/AMDGPU/SIMachineFunctionInfo.cpp
index 181cc41bd5ff743eedcabac51500599af1afd11c..22dc4c139ba8f1d9e0fa1e511436982c152cecc5 100644
--- a/lib/Target/AMDGPU/SIMachineFunctionInfo.cpp
+++ b/lib/Target/AMDGPU/SIMachineFunctionInfo.cpp
@@ -1,9 +1,8 @@
 //===- SIMachineFunctionInfo.cpp - SI Machine Function Info ---------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/AMDGPU/SIMachineFunctionInfo.h b/lib/Target/AMDGPU/SIMachineFunctionInfo.h
index ef91d1e43075f40f6581ce42c6d0ecc94ff9efe6..79be2760e2bbb900fe203301c2125df1ebd0e781 100644
--- a/lib/Target/AMDGPU/SIMachineFunctionInfo.h
+++ b/lib/Target/AMDGPU/SIMachineFunctionInfo.h
@@ -1,9 +1,8 @@
 //==- SIMachineFunctionInfo.h - SIMachineFunctionInfo interface --*- C++ -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AMDGPU/SIMachineScheduler.cpp b/lib/Target/AMDGPU/SIMachineScheduler.cpp
index fb7e670068fe61201caf3dae4acb8258ac5646bc..22fa16632c9704c466b37a00613c90a7a840da0f 100644
--- a/lib/Target/AMDGPU/SIMachineScheduler.cpp
+++ b/lib/Target/AMDGPU/SIMachineScheduler.cpp
@@ -1,9 +1,8 @@
 //===-- SIMachineScheduler.cpp - SI Scheduler Interface -------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AMDGPU/SIMachineScheduler.h b/lib/Target/AMDGPU/SIMachineScheduler.h
index 0ce68ac6a8971d096dec8d0146c1e7886b333f83..c28a7be4d03a54f47d28df71b8a940fc6333fb38 100644
--- a/lib/Target/AMDGPU/SIMachineScheduler.h
+++ b/lib/Target/AMDGPU/SIMachineScheduler.h
@@ -1,9 +1,8 @@
 //===-- SIMachineScheduler.h - SI Scheduler Interface -----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AMDGPU/SIMemoryLegalizer.cpp b/lib/Target/AMDGPU/SIMemoryLegalizer.cpp
index b4a4e9e33133dfb5373300e5b6c2395acf6e40f1..1080332d6e4a7e314996690355bf1e7ac59dd552 100644
--- a/lib/Target/AMDGPU/SIMemoryLegalizer.cpp
+++ b/lib/Target/AMDGPU/SIMemoryLegalizer.cpp
@@ -1,9 +1,8 @@
 //===- SIMemoryLegalizer.cpp ----------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AMDGPU/SIModeRegister.cpp b/lib/Target/AMDGPU/SIModeRegister.cpp
index 883fd308f2f4bd64c622fb69729117e336bb5dde..197398a169b92164fb5002c009c286cb5f5197ed 100644
--- a/lib/Target/AMDGPU/SIModeRegister.cpp
+++ b/lib/Target/AMDGPU/SIModeRegister.cpp
@@ -1,9 +1,8 @@
 //===-- SIModeRegister.cpp - Mode Register --------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/lib/Target/AMDGPU/SIOptimizeExecMasking.cpp b/lib/Target/AMDGPU/SIOptimizeExecMasking.cpp
index ebcad30a1866aab38392feac4ebbd31447c51266..c73066fe8d6140ca1f37e1d6c53b0bb5e76e4062 100644
--- a/lib/Target/AMDGPU/SIOptimizeExecMasking.cpp
+++ b/lib/Target/AMDGPU/SIOptimizeExecMasking.cpp
@@ -1,9 +1,8 @@
 //===-- SIOptimizeExecMasking.cpp -----------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/AMDGPU/SIOptimizeExecMaskingPreRA.cpp b/lib/Target/AMDGPU/SIOptimizeExecMaskingPreRA.cpp
index c671fed34bdf14bfcc27dc34183f4889f238ca56..adf23a3480582956518c8c3c9cd72c3d4667ea6b 100644
--- a/lib/Target/AMDGPU/SIOptimizeExecMaskingPreRA.cpp
+++ b/lib/Target/AMDGPU/SIOptimizeExecMaskingPreRA.cpp
@@ -1,9 +1,8 @@
 //===-- SIOptimizeExecMaskingPreRA.cpp ------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AMDGPU/SIPeepholeSDWA.cpp b/lib/Target/AMDGPU/SIPeepholeSDWA.cpp
index 2d43d5d05ef6437a0f49038141978d1218646b5f..547fad16d8446d7fc058b304e53c192e171a762b 100644
--- a/lib/Target/AMDGPU/SIPeepholeSDWA.cpp
+++ b/lib/Target/AMDGPU/SIPeepholeSDWA.cpp
@@ -1,9 +1,8 @@
 //===- SIPeepholeSDWA.cpp - Peephole optimization for SDWA instructions ---===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AMDGPU/SIProgramInfo.h b/lib/Target/AMDGPU/SIProgramInfo.h
index 383f6b57580833ff1877a61773e8876761a3a6ca..279067baf6cf2e94bcc0a96379381edd085a1157 100644
--- a/lib/Target/AMDGPU/SIProgramInfo.h
+++ b/lib/Target/AMDGPU/SIProgramInfo.h
@@ -1,9 +1,8 @@
 //===--- SIProgramInfo.h ----------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AMDGPU/SIRegisterInfo.cpp b/lib/Target/AMDGPU/SIRegisterInfo.cpp
index 97cfde2b2354128982007949da015f1d03000fe0..fdbafd9b4b166762b041fa053a15cc3339a82e42 100644
--- a/lib/Target/AMDGPU/SIRegisterInfo.cpp
+++ b/lib/Target/AMDGPU/SIRegisterInfo.cpp
@@ -1,9 +1,8 @@
 //===-- SIRegisterInfo.cpp - SI Register Information ---------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -164,6 +163,9 @@ BitVector SIRegisterInfo::getReservedRegs(const MachineFunction &MF) const {
   // Reserve xnack_mask registers - support is not implemented in Codegen.
   reserveRegisterTuples(Reserved, AMDGPU::XNACK_MASK);
 
+  // Reserve lds_direct register - support is not implemented in Codegen.
+  reserveRegisterTuples(Reserved, AMDGPU::LDS_DIRECT);
+
   // Reserve Trap Handler registers - support is not implemented in Codegen.
   reserveRegisterTuples(Reserved, AMDGPU::TBA);
   reserveRegisterTuples(Reserved, AMDGPU::TMA);
diff --git a/lib/Target/AMDGPU/SIRegisterInfo.h b/lib/Target/AMDGPU/SIRegisterInfo.h
index b82fefde47e1344d7b21638328793fca2ef2b8b6..bfdc1ef9645decb428c38cb0f7f973397479809d 100644
--- a/lib/Target/AMDGPU/SIRegisterInfo.h
+++ b/lib/Target/AMDGPU/SIRegisterInfo.h
@@ -1,9 +1,8 @@
 //===-- SIRegisterInfo.h - SI Register Info Interface ----------*- C++ -*--===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AMDGPU/SIRegisterInfo.td b/lib/Target/AMDGPU/SIRegisterInfo.td
index 80bc180ba6eea5a2dbafeb60858a96c8bfbb9594..84751d1cbfe2da298c77c71c10a8ee7a77c55859 100644
--- a/lib/Target/AMDGPU/SIRegisterInfo.td
+++ b/lib/Target/AMDGPU/SIRegisterInfo.td
@@ -1,9 +1,8 @@
 //===-- SIRegisterInfo.td - SI Register defs ---------------*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -76,6 +75,8 @@ def SRC_SHARED_LIMIT : SIReg<"src_shared_limit", 236>;
 def SRC_PRIVATE_BASE : SIReg<"src_private_base", 237>;
 def SRC_PRIVATE_LIMIT : SIReg<"src_private_limit", 238>;
 
+def LDS_DIRECT : SIReg <"lds_direct", 254>;
+
 def XNACK_MASK_LO : SIReg<"xnack_mask_lo", 104>;
 def XNACK_MASK_HI : SIReg<"xnack_mask_hi", 105>;
 
@@ -410,6 +411,12 @@ def Pseudo_SReg_128 : RegisterClass<"AMDGPU", [v4i32, v2i64, v2f64], 32,
   let CopyCost = -1;
 }
 
+def LDS_DIRECT_CLASS : RegisterClass<"AMDGPU", [i32, f32, i16, f16, v2i16, v2f16], 32,
+  (add LDS_DIRECT)> {
+  let isAllocatable = 0;
+  let CopyCost = -1;
+}
+
 // Subset of SReg_32 without M0 for SMRD instructions and alike.
 // See comments in SIInstructions.td for more info.
 def SReg_32_XM0_XEXEC : RegisterClass<"AMDGPU", [i32, f32, i16, f16, v2i16, v2f16], 32,
@@ -459,15 +466,15 @@ def SReg_64 : RegisterClass<"AMDGPU", [v2i32, i64, v2f32, f64, i1, v4i16, v4f16]
 // Requires 2 s_mov_b64 to copy
 let CopyCost = 2 in {
 
-def SGPR_128 : RegisterClass<"AMDGPU", [v4i32, v4f32, v16i8, v2i64], 32, (add SGPR_128Regs)> {
+def SGPR_128 : RegisterClass<"AMDGPU", [v4i32, v4f32, v2i64], 32, (add SGPR_128Regs)> {
   let AllocationPriority = 10;
 }
 
-def TTMP_128 : RegisterClass<"AMDGPU", [v4i32, v4f32, v16i8, v2i64], 32, (add TTMP_128Regs)> {
+def TTMP_128 : RegisterClass<"AMDGPU", [v4i32, v4f32, v2i64], 32, (add TTMP_128Regs)> {
   let isAllocatable = 0;
 }
 
-def SReg_128 : RegisterClass<"AMDGPU", [v4i32, v4f32, v16i8, v2i64, v2f64], 32,
+def SReg_128 : RegisterClass<"AMDGPU", [v4i32, v4f32, v2i64, v2f64], 32,
   (add SGPR_128, TTMP_128)> {
   let AllocationPriority = 10;
 }
@@ -546,7 +553,7 @@ def VReg_1 : RegisterClass<"AMDGPU", [i1], 32, (add VGPR_32)> {
 }
 
 def VS_32 : RegisterClass<"AMDGPU", [i32, f32, i16, f16, v2i16, v2f16], 32,
-                          (add VGPR_32, SReg_32)> {
+                          (add VGPR_32, SReg_32, LDS_DIRECT_CLASS)> {
   let isAllocatable = 0;
 }
 
diff --git a/lib/Target/AMDGPU/SISchedule.td b/lib/Target/AMDGPU/SISchedule.td
index 7af69cb6a46db6803c8a3db5f01ac11ebf12b465..4c75a16ce3a4e80ec932f68dbaa3dd54efa7a8d2 100644
--- a/lib/Target/AMDGPU/SISchedule.td
+++ b/lib/Target/AMDGPU/SISchedule.td
@@ -1,9 +1,8 @@
 //===-- SISchedule.td - SI Scheduling definitons -------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AMDGPU/SIShrinkInstructions.cpp b/lib/Target/AMDGPU/SIShrinkInstructions.cpp
index 6ad7dd0e3a7ce9cb764b8bfc61f1fd3b45887ecc..2b1264d6f6313d69924f528685e913c312649e43 100644
--- a/lib/Target/AMDGPU/SIShrinkInstructions.cpp
+++ b/lib/Target/AMDGPU/SIShrinkInstructions.cpp
@@ -1,9 +1,8 @@
 //===-- SIShrinkInstructions.cpp - Shrink Instructions --------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 /// The pass tries to use the 32-bit encoding for instructions when possible.
 //===----------------------------------------------------------------------===//
diff --git a/lib/Target/AMDGPU/SIWholeQuadMode.cpp b/lib/Target/AMDGPU/SIWholeQuadMode.cpp
index 879726b1528c9d6741165c887a5355b6ccb77c81..dae7d455d826aae1044decda72cbb8fd55e58889 100644
--- a/lib/Target/AMDGPU/SIWholeQuadMode.cpp
+++ b/lib/Target/AMDGPU/SIWholeQuadMode.cpp
@@ -1,9 +1,8 @@
 //===-- SIWholeQuadMode.cpp - enter and suspend whole quad mode -----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AMDGPU/SMInstructions.td b/lib/Target/AMDGPU/SMInstructions.td
index 8a063e1a4867328022854a2f8fb4d83add936fae..db9cc72a9d050d12e1b123dd0e0f7d76663663d0 100644
--- a/lib/Target/AMDGPU/SMInstructions.td
+++ b/lib/Target/AMDGPU/SMInstructions.td
@@ -1,9 +1,8 @@
 //===---- SMInstructions.td - Scalar Memory Instruction Defintions --------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -34,7 +33,6 @@ class SM_Pseudo <string opName, dag outs, dag ins, string asmOps, list<dag> patt
   let hasSideEffects = 0;
   let UseNamedOperandTable = 1;
   let SchedRW = [WriteSMEM];
-  let SubtargetPredicate = isGCN;
 
   string Mnemonic = opName;
   string AsmOperands = asmOps;
diff --git a/lib/Target/AMDGPU/SOPInstructions.td b/lib/Target/AMDGPU/SOPInstructions.td
index ca5e981ac5c25c1f1f6c77a10d5c4e9d8a206795..1cb9f27dd021624188da8068345829ac637f8f00 100644
--- a/lib/Target/AMDGPU/SOPInstructions.td
+++ b/lib/Target/AMDGPU/SOPInstructions.td
@@ -1,9 +1,8 @@
 //===-- SOPInstructions.td - SOP Instruction Defintions -------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -26,7 +25,6 @@ class SOP_Pseudo<string opName, dag outs, dag ins, string asmOps,
 
   let isPseudo = 1;
   let isCodeGenOnly = 1;
-  let SubtargetPredicate = isGCN;
 
   string Mnemonic = opName;
   string AsmOperands = asmOps;
@@ -302,6 +300,8 @@ class SOP2_Real<bits<7> op, SOP_Pseudo ps> :
   // copy relevant pseudo op flags
   let SubtargetPredicate = ps.SubtargetPredicate;
   let AsmMatchConverter  = ps.AsmMatchConverter;
+  let UseNamedOperandTable = ps.UseNamedOperandTable;
+  let TSFlags = ps.TSFlags;
 
   // encoding
   bits<7> sdst;
@@ -555,7 +555,6 @@ class SOPK_Pseudo <string opName, dag outs, dag ins,
   SIMCInstr<opName, SIEncodingFamily.NONE> {
   let isPseudo = 1;
   let isCodeGenOnly = 1;
-  let SubtargetPredicate = isGCN;
   let mayLoad = 0;
   let mayStore = 0;
   let hasSideEffects = 0;
@@ -756,7 +755,6 @@ class SOPC <bits<7> op, dag outs, dag ins, string asm,
   let Defs = [SCC];
   let SchedRW = [WriteSALU];
   let UseNamedOperandTable = 1;
-  let SubtargetPredicate = isGCN;
 }
 
 class SOPC_Base <bits<7> op, RegisterOperand rc0, RegisterOperand rc1,
@@ -854,7 +852,6 @@ class SOPP <bits<7> op, dag ins, string asm, list<dag> pattern = []> :
   let SchedRW = [WriteSALU];
 
   let UseNamedOperandTable = 1;
-  let SubtargetPredicate = isGCN;
 }
 
 
diff --git a/lib/Target/AMDGPU/TargetInfo/AMDGPUTargetInfo.cpp b/lib/Target/AMDGPU/TargetInfo/AMDGPUTargetInfo.cpp
index e4c442db3016fd3fb9fb3816973d629ee2363d36..3d412830681871cb0dfdac450bce29408bbd82d8 100644
--- a/lib/Target/AMDGPU/TargetInfo/AMDGPUTargetInfo.cpp
+++ b/lib/Target/AMDGPU/TargetInfo/AMDGPUTargetInfo.cpp
@@ -1,9 +1,8 @@
 //===-- TargetInfo/AMDGPUTargetInfo.cpp - TargetInfo for AMDGPU -----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AMDGPU/TargetInfo/LLVMBuild.txt b/lib/Target/AMDGPU/TargetInfo/LLVMBuild.txt
index 291317fa072f82112bdd9de0bf25355af2fb22b9..1b1848c5c447fb91c9ba5b55669cb01ac506b1e8 100644
--- a/lib/Target/AMDGPU/TargetInfo/LLVMBuild.txt
+++ b/lib/Target/AMDGPU/TargetInfo/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/Target/AMDGPU/TargetInfo/LLVMBuild.txt --------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Target/AMDGPU/Utils/AMDGPUAsmUtils.cpp b/lib/Target/AMDGPU/Utils/AMDGPUAsmUtils.cpp
index 9eb4c6513cce923f37dbdb10594f3fe717d904af..67aa535e903a859bc45b740e5c9c0dc98bfb4474 100644
--- a/lib/Target/AMDGPU/Utils/AMDGPUAsmUtils.cpp
+++ b/lib/Target/AMDGPU/Utils/AMDGPUAsmUtils.cpp
@@ -1,9 +1,8 @@
 //===-- AMDGPUAsmUtils.cpp - AsmParser/InstPrinter common -----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 #include "AMDGPUAsmUtils.h"
diff --git a/lib/Target/AMDGPU/Utils/AMDGPUAsmUtils.h b/lib/Target/AMDGPU/Utils/AMDGPUAsmUtils.h
index ebb2be22b48795ae04134d7ac3d25d05f869d564..9265d6015278be8c52e2734c19a694b2ef3206b3 100644
--- a/lib/Target/AMDGPU/Utils/AMDGPUAsmUtils.h
+++ b/lib/Target/AMDGPU/Utils/AMDGPUAsmUtils.h
@@ -1,9 +1,8 @@
 //===-- AMDGPUAsmUtils.h - AsmParser/InstPrinter common ---------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp b/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp
index 54c866bdc63ce060b036a09d86f226cbc02a0bf6..e1727338599b5874254bddc041ae66d96a1b0f0b 100644
--- a/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp
+++ b/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp
@@ -1,9 +1,8 @@
 //===- AMDGPUBaseInfo.cpp - AMDGPU Base encoding information --------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -230,7 +229,8 @@ unsigned getEUsPerCU(const MCSubtargetInfo *STI) {
 
 unsigned getMaxWorkGroupsPerCU(const MCSubtargetInfo *STI,
                                unsigned FlatWorkGroupSize) {
-  if (!STI->getFeatureBits().test(FeatureGCN))
+  assert(FlatWorkGroupSize != 0);
+  if (STI->getTargetTriple().getArch() != Triple::amdgcn)
     return 8;
   unsigned N = getWavesPerWorkGroup(STI, FlatWorkGroupSize);
   if (N == 1)
diff --git a/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h b/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h
index 1ef7da328d31a85658b0a4ee3df9f613c7a34106..174fec47986475e4aa4d7f5067d996397ff50621 100644
--- a/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h
+++ b/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h
@@ -1,9 +1,8 @@
 //===- AMDGPUBaseInfo.h - Top level definitions for AMDGPU ------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -184,6 +183,7 @@ struct MIMGBaseOpcodeInfo {
   bool Atomic;
   bool AtomicX2;
   bool Sampler;
+  bool Gather4;
 
   uint8_t NumExtraArgs;
   bool Gradients;
diff --git a/lib/Target/AMDGPU/Utils/AMDKernelCodeTInfo.h b/lib/Target/AMDGPU/Utils/AMDKernelCodeTInfo.h
index 82ffdef8e674a6974d59c631627fe689304a2000..8efe6f6741ee597452dcb16be0ce55d0b9c6cc4c 100644
--- a/lib/Target/AMDGPU/Utils/AMDKernelCodeTInfo.h
+++ b/lib/Target/AMDGPU/Utils/AMDKernelCodeTInfo.h
@@ -1,9 +1,8 @@
 //===--------------------- AMDKernelCodeTInfo.h ---------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AMDGPU/Utils/AMDKernelCodeTUtils.cpp b/lib/Target/AMDGPU/Utils/AMDKernelCodeTUtils.cpp
index 20059f4a1ed7f63e84f60f46a9e7677cf29346ef..443e2cc45ac07ed1038c7e6c19002b630ce96d0e 100644
--- a/lib/Target/AMDGPU/Utils/AMDKernelCodeTUtils.cpp
+++ b/lib/Target/AMDGPU/Utils/AMDKernelCodeTUtils.cpp
@@ -1,9 +1,8 @@
 //===- AMDKernelCodeTUtils.cpp --------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AMDGPU/Utils/AMDKernelCodeTUtils.h b/lib/Target/AMDGPU/Utils/AMDKernelCodeTUtils.h
index ef9f9bdb6bcb1b606d9695fd810a601c51e369fd..a87325a78df37931ea036281da548adf5419d07e 100644
--- a/lib/Target/AMDGPU/Utils/AMDKernelCodeTUtils.h
+++ b/lib/Target/AMDGPU/Utils/AMDKernelCodeTUtils.h
@@ -1,9 +1,8 @@
 //===- AMDGPUKernelCodeTUtils.h - helpers for amd_kernel_code_t -*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AMDGPU/Utils/LLVMBuild.txt b/lib/Target/AMDGPU/Utils/LLVMBuild.txt
index 2453bc546b99fc7284a817c68a3b1d88411e8bff..f59a47846c091eb10845f23d270ce438edba3eca 100644
--- a/lib/Target/AMDGPU/Utils/LLVMBuild.txt
+++ b/lib/Target/AMDGPU/Utils/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/Target/AMDGPU/Utils/LLVMBuild.txt ------------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Target/AMDGPU/VIInstrFormats.td b/lib/Target/AMDGPU/VIInstrFormats.td
index 1fd1c1e215279c42b06a1775c1f88a77e391703b..bd65a495fa727822760dcb247ccb5a979761c84f 100644
--- a/lib/Target/AMDGPU/VIInstrFormats.td
+++ b/lib/Target/AMDGPU/VIInstrFormats.td
@@ -1,9 +1,8 @@
 //===-- VIInstrFormats.td - VI Instruction Encodings ----------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AMDGPU/VIInstructions.td b/lib/Target/AMDGPU/VIInstructions.td
index b45c8fc9c7d59501518ffec22fe4b756b5e68eec..ec7d8875a746e8bc4453153c93490c493d11b90d 100644
--- a/lib/Target/AMDGPU/VIInstructions.td
+++ b/lib/Target/AMDGPU/VIInstructions.td
@@ -1,9 +1,8 @@
 //===-- VIInstructions.td - VI Instruction Defintions ---------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 // Instruction definitions for VI and newer.
diff --git a/lib/Target/AMDGPU/VOP1Instructions.td b/lib/Target/AMDGPU/VOP1Instructions.td
index 68446ab79720af70c856845bcefaffd1f19ca642..15006155d9b01d58c1030e751fbea9706a8aad6b 100644
--- a/lib/Target/AMDGPU/VOP1Instructions.td
+++ b/lib/Target/AMDGPU/VOP1Instructions.td
@@ -1,9 +1,8 @@
 //===-- VOP1Instructions.td - Vector Instruction Defintions ---------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -48,7 +47,6 @@ class VOP1_Pseudo <string opName, VOPProfile P, list<dag> pattern=[], bit VOP1On
   let mayLoad = 0;
   let mayStore = 0;
   let hasSideEffects = 0;
-  let SubtargetPredicate = isGCN;
 
   let VOP1 = 1;
   let VALU = 1;
@@ -156,7 +154,6 @@ def V_READFIRSTLANE_B32 :
   let mayLoad = 0;
   let mayStore = 0;
   let hasSideEffects = 0;
-  let SubtargetPredicate = isGCN;
 
   let VOP1 = 1;
   let VALU = 1;
diff --git a/lib/Target/AMDGPU/VOP2Instructions.td b/lib/Target/AMDGPU/VOP2Instructions.td
index e3fd7b5f9fadd9f645fabeab78b30a3d749f9e6d..ee165515ce7766a24ba83c4c3ae7abb0f449f2ff 100644
--- a/lib/Target/AMDGPU/VOP2Instructions.td
+++ b/lib/Target/AMDGPU/VOP2Instructions.td
@@ -1,9 +1,8 @@
 //===-- VOP2Instructions.td - Vector Instruction Defintions ---------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -69,7 +68,6 @@ class VOP2_Pseudo <string opName, VOPProfile P, list<dag> pattern=[], string suf
   let mayLoad = 0;
   let mayStore = 0;
   let hasSideEffects = 0;
-  let SubtargetPredicate = isGCN;
 
   let VOP2 = 1;
   let VALU = 1;
@@ -393,8 +391,6 @@ def VOP_WRITELANE : VOPProfile<[i32, i32, i32, i32]> {
 // VOP2 Instructions
 //===----------------------------------------------------------------------===//
 
-let SubtargetPredicate = isGCN, Predicates = [isGCN] in {
-
 defm V_CNDMASK_B32 : VOP2eInst <"v_cndmask_b32", VOP2e_I32_I32_I32_I1>;
 def V_MADMK_F32 : VOP2_Pseudo <"v_madmk_f32", VOP_MADMK_F32, []>;
 
@@ -472,7 +468,6 @@ defm V_CVT_PKRTZ_F16_F32 : VOP2Inst <"v_cvt_pkrtz_f16_f32", VOP_NO_EXT<VOP_V2F16
 defm V_CVT_PK_U16_U32 : VOP2Inst <"v_cvt_pk_u16_u32", VOP_NO_EXT<VOP_V2I16_I32_I32>, AMDGPUpk_u16_u32>;
 defm V_CVT_PK_I16_I32 : VOP2Inst <"v_cvt_pk_i16_i32", VOP_NO_EXT<VOP_V2I16_I32_I32>, AMDGPUpk_i16_i32>;
 
-} // End SubtargetPredicate = isGCN, Predicates = [isGCN]
 
 def : GCNPat<
     (AMDGPUadde i32:$src0, i32:$src1, i1:$src2),
diff --git a/lib/Target/AMDGPU/VOP3Instructions.td b/lib/Target/AMDGPU/VOP3Instructions.td
index 4b8c1f208a0eda54003b5fb238f2c55e92279994..babd0417ecc0892a77b8dde62206dc826bea42f9 100644
--- a/lib/Target/AMDGPU/VOP3Instructions.td
+++ b/lib/Target/AMDGPU/VOP3Instructions.td
@@ -1,9 +1,8 @@
 //===-- VOP3Instructions.td - Vector Instruction Defintions ---------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -463,7 +462,14 @@ def V_MAD_I16 : VOP3Inst <"v_mad_i16", VOP3_Profile<VOP_I16_I16_I16_I16, VOP3_CL
 let FPDPRounding = 1 in {
 def V_MAD_F16 : VOP3Inst <"v_mad_f16", VOP3_Profile<VOP_F16_F16_F16_F16>, fmad>;
 let Uses = [M0, EXEC] in {
-def V_INTERP_P2_F16 : VOP3Interp <"v_interp_p2_f16", VOP3_INTERP16<[f16, f32, i32, f32]>>;
+def V_INTERP_P2_F16 : VOP3Interp <"v_interp_p2_f16", VOP3_INTERP16<[f16, f32, i32, f32]>,
+       [(set f16:$vdst, (AMDGPUinterp_p2_f16 f32:$src0, (i32 imm:$attrchan),
+                                                        (i32 imm:$attr),
+                                                        (i32 imm:$src0_modifiers),
+                                                        (f32 VRegSrc_32:$src2),
+                                                        (i32 imm:$src2_modifiers),
+                                                        (i1 imm:$high),
+                                                        (i1 imm:$clamp)))]>;
 } // End Uses = [M0, EXEC]
 } // End FPDPRounding = 1
 } // End renamedInGFX9 = 1
@@ -478,8 +484,22 @@ def V_INTERP_P2_F16_gfx9 : VOP3Interp <"v_interp_p2_f16_gfx9", VOP3_INTERP16<[f1
 } // End SubtargetPredicate = isGFX9
 
 let Uses = [M0, EXEC], FPDPRounding = 1 in {
-def V_INTERP_P1LL_F16 : VOP3Interp <"v_interp_p1ll_f16", VOP3_INTERP16<[f32, f32, i32, untyped]>>;
-def V_INTERP_P1LV_F16 : VOP3Interp <"v_interp_p1lv_f16", VOP3_INTERP16<[f32, f32, i32, f16]>>;
+def V_INTERP_P1LL_F16 : VOP3Interp <"v_interp_p1ll_f16", VOP3_INTERP16<[f32, f32, i32, untyped]>,
+       [(set f32:$vdst, (AMDGPUinterp_p1ll_f16 f32:$src0, (i32 imm:$attrchan),
+                                                          (i32 imm:$attr),
+                                                          (i32 imm:$src0_modifiers),
+                                                          (i1 imm:$high),
+                                                          (i1 imm:$clamp),
+                                                          (i32 imm:$omod)))]>;
+def V_INTERP_P1LV_F16 : VOP3Interp <"v_interp_p1lv_f16", VOP3_INTERP16<[f32, f32, i32, f16]>,
+       [(set f32:$vdst, (AMDGPUinterp_p1lv_f16 f32:$src0, (i32 imm:$attrchan),
+                                                          (i32 imm:$attr),
+                                                          (i32 imm:$src0_modifiers),
+                                                          (f32 VRegSrc_32:$src2),
+                                                          (i32 imm:$src2_modifiers),
+                                                          (i1 imm:$high),
+                                                          (i1 imm:$clamp),
+                                                          (i32 imm:$omod)))]>;
 } // End Uses = [M0, EXEC], FPDPRounding = 1
 
 } // End SubtargetPredicate = Has16BitInsts, isCommutable = 1
diff --git a/lib/Target/AMDGPU/VOP3PInstructions.td b/lib/Target/AMDGPU/VOP3PInstructions.td
index 0d25a86da3266480619df72c85e6d6f0610ca7da..be17f1371081b614ed4d28d7357bd1dcc8339fb9 100644
--- a/lib/Target/AMDGPU/VOP3PInstructions.td
+++ b/lib/Target/AMDGPU/VOP3PInstructions.td
@@ -1,9 +1,8 @@
 //===-- VOP3PInstructions.td - Vector Instruction Defintions --------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -239,29 +238,39 @@ class UDot2Pat<Instruction Inst> : GCNPat <
        (AMDGPUmul_u24_oneuse (and i32:$src0, (i32 65535)),
                              (and i32:$src1, (i32 65535)))
    ),
-  (Inst (i32 8), $src0, (i32 8), $src1, (i32 8), $src2, (i1 0))
->;
+  (Inst (i32 8), $src0, (i32 8), $src1, (i32 8), $src2, (i1 0))> {
+  let SubtargetPredicate = !cast<VOP_Pseudo>(Inst).SubtargetPredicate;
+}
 
 class SDot2Pat<Instruction Inst> : GCNPat <
   (add (add_oneuse (AMDGPUmul_i24_oneuse (sra i32:$src0, (i32 16)),
                                          (sra i32:$src1, (i32 16))), i32:$src2),
        (AMDGPUmul_i24_oneuse (sext_inreg i32:$src0, i16),
                              (sext_inreg i32:$src1, i16))),
-  (Inst (i32 8), $src0, (i32 8), $src1, (i32 8), $src2, (i1 0))
->;
+  (Inst (i32 8), $src0, (i32 8), $src1, (i32 8), $src2, (i1 0))> {
+  let SubtargetPredicate = !cast<VOP_Pseudo>(Inst).SubtargetPredicate;
+}
 
-let SubtargetPredicate = HasDLInsts in {
+let SubtargetPredicate = HasDot2Insts in {
 
 def V_DOT2_F32_F16 : VOP3PInst<"v_dot2_f32_f16", VOP3_Profile<VOP_F32_V2F16_V2F16_F32>>;
 def V_DOT2_I32_I16 : VOP3PInst<"v_dot2_i32_i16", VOP3_Profile<VOP_I32_V2I16_V2I16_I32>>;
 def V_DOT2_U32_U16 : VOP3PInst<"v_dot2_u32_u16", VOP3_Profile<VOP_I32_V2I16_V2I16_I32>>;
-def V_DOT4_I32_I8  : VOP3PInst<"v_dot4_i32_i8", VOP3_Profile<VOP_I32_I32_I32_I32, VOP3_PACKED>>;
 def V_DOT4_U32_U8  : VOP3PInst<"v_dot4_u32_u8", VOP3_Profile<VOP_I32_I32_I32_I32, VOP3_PACKED>>;
-def V_DOT8_I32_I4  : VOP3PInst<"v_dot8_i32_i4", VOP3_Profile<VOP_I32_I32_I32_I32, VOP3_PACKED>>;
 def V_DOT8_U32_U4  : VOP3PInst<"v_dot8_u32_u4", VOP3_Profile<VOP_I32_I32_I32_I32, VOP3_PACKED>>;
 
+} // End SubtargetPredicate = HasDot2Insts
+
+let SubtargetPredicate = HasDot1Insts in {
+
+def V_DOT4_I32_I8  : VOP3PInst<"v_dot4_i32_i8", VOP3_Profile<VOP_I32_I32_I32_I32, VOP3_PACKED>>;
+def V_DOT8_I32_I4  : VOP3PInst<"v_dot8_i32_i4", VOP3_Profile<VOP_I32_I32_I32_I32, VOP3_PACKED>>;
+
+} // End SubtargetPredicate = HasDot1Insts
+
 multiclass DotPats<SDPatternOperator dot_op,
                    VOP3PInst dot_inst> {
+  let SubtargetPredicate = dot_inst.SubtargetPredicate in
   def : GCNPat <
     (dot_op (dot_inst.Pfl.Src0VT (VOP3PMods0 dot_inst.Pfl.Src0VT:$src0, i32:$src0_modifiers)),
             (dot_inst.Pfl.Src1VT (VOP3PMods dot_inst.Pfl.Src1VT:$src1, i32:$src1_modifiers)),
@@ -281,12 +290,14 @@ def : UDot2Pat<V_DOT2_U32_U16>;
 def : SDot2Pat<V_DOT2_I32_I16>;
 
 foreach Type = ["U", "I"] in
+  let SubtargetPredicate = !cast<VOP_Pseudo>("V_DOT4_"#Type#"32_"#Type#8).SubtargetPredicate in
   def : GCNPat <
     !cast<dag>(!foldl((i32 i32:$src2), [0, 1, 2, 3], lhs, y,
                       (add_oneuse lhs, (!cast<PatFrag>("Mul"#Type#"_Elt"#y) i32:$src0, i32:$src1)))),
     (!cast<VOP3PInst>("V_DOT4_"#Type#"32_"#Type#8) (i32 8), $src0, (i32 8), $src1, (i32 8), $src2, (i1 0))>;
 
 foreach Type = ["U", "I"] in
+  let SubtargetPredicate = !cast<VOP_Pseudo>("V_DOT8_"#Type#"32_"#Type#4).SubtargetPredicate in
   def : GCNPat <
     !cast<dag>(!foldl((add_oneuse i32:$src2, (!cast<PatFrag>("Mul"#Type#"0_4bit") i32:$src0, i32:$src1)),
                       [1, 2, 3, 4, 5, 6, 7], lhs, y,
@@ -296,14 +307,13 @@ foreach Type = ["U", "I"] in
 // Different variants of dot8 code-gen dag patterns are not generated through table-gen due to a huge increase
 // in the compile time. Directly handle the pattern generated by the FE here.
 foreach Type = ["U", "I"] in
+  let SubtargetPredicate = !cast<VOP_Pseudo>("V_DOT8_"#Type#"32_"#Type#4).SubtargetPredicate in
   def : GCNPat <
     !cast<dag>(!foldl((add_oneuse i32:$src2, (!cast<PatFrag>("Mul"#Type#"0_4bit") i32:$src0, i32:$src1)),
                       [7, 1, 2, 3, 4, 5, 6], lhs, y,
                       (NonACAdd_oneuse lhs, (!cast<PatFrag>("Mul"#Type#y#"_4bit") i32:$src0, i32:$src1)))),
     (!cast<VOP3PInst>("V_DOT8_"#Type#"32_"#Type#4) (i32 8), $src0, (i32 8), $src1, (i32 8), $src2, (i1 0))>;
 
-} // End SubtargetPredicate = HasDLInsts
-
 multiclass VOP3P_Real_vi<bits<10> op> {
   def _vi : VOP3P_Real<!cast<VOP3_Pseudo>(NAME), SIEncodingFamily.VI>,
             VOP3Pe <op, !cast<VOP3_Pseudo>(NAME).Pfl> {
@@ -352,14 +362,19 @@ defm V_FMA_MIXHI_F16 : VOP3P_Real_vi <0x3a2>;
 }
 
 
-let SubtargetPredicate = HasDLInsts in {
+let SubtargetPredicate = HasDot2Insts in {
 
 defm V_DOT2_F32_F16 : VOP3P_Real_vi <0x3a3>;
 defm V_DOT2_I32_I16 : VOP3P_Real_vi <0x3a6>;
 defm V_DOT2_U32_U16 : VOP3P_Real_vi <0x3a7>;
-defm V_DOT4_I32_I8  : VOP3P_Real_vi <0x3a8>;
 defm V_DOT4_U32_U8  : VOP3P_Real_vi <0x3a9>;
-defm V_DOT8_I32_I4  : VOP3P_Real_vi <0x3aa>;
 defm V_DOT8_U32_U4  : VOP3P_Real_vi <0x3ab>;
 
-} // End SubtargetPredicate = HasDLInsts
+} // End SubtargetPredicate = HasDot2Insts
+
+let SubtargetPredicate = HasDot1Insts in {
+
+defm V_DOT4_I32_I8  : VOP3P_Real_vi <0x3a8>;
+defm V_DOT8_I32_I4  : VOP3P_Real_vi <0x3aa>;
+
+} // End SubtargetPredicate = HasDot1Insts
diff --git a/lib/Target/AMDGPU/VOPCInstructions.td b/lib/Target/AMDGPU/VOPCInstructions.td
index 091cac8cd35ca295f5a71f16ae40f8300ba1131c..9d6106108553774a4ecdc038ed06308737d1f1a0 100644
--- a/lib/Target/AMDGPU/VOPCInstructions.td
+++ b/lib/Target/AMDGPU/VOPCInstructions.td
@@ -1,9 +1,8 @@
 //===-- VOPCInstructions.td - Vector Instruction Defintions ---------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -83,8 +82,6 @@ class VOPC_Pseudo <string opName, VOPC_Profile P, list<dag> pattern=[]> :
   let Uses = [EXEC];
   let Defs = [VCC];
 
-  let SubtargetPredicate = isGCN;
-
   VOPProfile Pfl = P;
 }
 
diff --git a/lib/Target/AMDGPU/VOPInstructions.td b/lib/Target/AMDGPU/VOPInstructions.td
index 7de7d90d27b3add0bb70891242e856098492d38a..e056196c572d5cd5cce7759f242a2b3de7b88986 100644
--- a/lib/Target/AMDGPU/VOPInstructions.td
+++ b/lib/Target/AMDGPU/VOPInstructions.td
@@ -1,9 +1,8 @@
 //===-- VOPInstructions.td - Vector Instruction Defintions ----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -100,7 +99,6 @@ class VOP3_Pseudo <string opName, VOPProfile P, list<dag> pattern = [],
   let mayLoad = 0;
   let mayStore = 0;
   let hasSideEffects = 0;
-  let SubtargetPredicate = isGCN;
 
   // Because SGPRs may be allowed if there are multiple operands, we
   // need a post-isel hook to insert copies in order to avoid
diff --git a/lib/Target/ARC/ARC.h b/lib/Target/ARC/ARC.h
index 65f6ed67eb5bf47b0ae7be72782a4733d5d88588..cce58656c317330c6700252ac526694aa6d41458 100644
--- a/lib/Target/ARC/ARC.h
+++ b/lib/Target/ARC/ARC.h
@@ -1,9 +1,8 @@
 //===- ARC.h - Top-level interface for ARC representation -------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/ARC/ARC.td b/lib/Target/ARC/ARC.td
index 6635630c62a310c13f36837ce6ae4d5a24067da9..846f1bb6735e19088022665aec957cc646da4953 100644
--- a/lib/Target/ARC/ARC.td
+++ b/lib/Target/ARC/ARC.td
@@ -1,9 +1,8 @@
 //===- ARC.td - Describe the ARC Target Machine ------------*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/ARC/ARCAsmPrinter.cpp b/lib/Target/ARC/ARCAsmPrinter.cpp
index 8c13da0484fdab78a8835236ed7eb73cce8dee7d..487360ebc13c7f41dbf94d062fd2179b53b17a8a 100644
--- a/lib/Target/ARC/ARCAsmPrinter.cpp
+++ b/lib/Target/ARC/ARCAsmPrinter.cpp
@@ -1,9 +1,8 @@
 //===- ARCAsmPrinter.cpp - ARC LLVM assembly writer -------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/ARC/ARCBranchFinalize.cpp b/lib/Target/ARC/ARCBranchFinalize.cpp
index 3b410fa383b7dbd7463d4eab142d7351ea7fa001..633c081b3137298cecf41a945b7256bcc3913ff2 100644
--- a/lib/Target/ARC/ARCBranchFinalize.cpp
+++ b/lib/Target/ARC/ARCBranchFinalize.cpp
@@ -1,9 +1,8 @@
 //===- ARCBranchFinalize.cpp - ARC conditional branches ---------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/ARC/ARCCallingConv.td b/lib/Target/ARC/ARCCallingConv.td
index b7d37bc2a41fe5a7ce54de54078e5a639a3b8a7c..098e03e36bca43461a1b187b61bb952caa9d954b 100644
--- a/lib/Target/ARC/ARCCallingConv.td
+++ b/lib/Target/ARC/ARCCallingConv.td
@@ -1,9 +1,8 @@
 //===- ARCCallingConv.td - Calling Conventions for ARC -----*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 // This describes the calling conventions for ARC architecture.
diff --git a/lib/Target/ARC/ARCExpandPseudos.cpp b/lib/Target/ARC/ARCExpandPseudos.cpp
index 3177735c0529903bf7fef4cbbcb2da565c41ca0c..a1646d17605feb0c57d4e565806a772aaeb9c94e 100644
--- a/lib/Target/ARC/ARCExpandPseudos.cpp
+++ b/lib/Target/ARC/ARCExpandPseudos.cpp
@@ -1,9 +1,8 @@
 //===- ARCExpandPseudosPass - ARC expand pseudo loads -----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/ARC/ARCFrameLowering.cpp b/lib/Target/ARC/ARCFrameLowering.cpp
index ca59cb2baaa70589bb897308de964c79cee1bdf8..047da85298c148f779df84206dc3f62c0bed38a9 100644
--- a/lib/Target/ARC/ARCFrameLowering.cpp
+++ b/lib/Target/ARC/ARCFrameLowering.cpp
@@ -1,9 +1,8 @@
 //===- ARCFrameLowering.cpp - ARC Frame Information -------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/ARC/ARCFrameLowering.h b/lib/Target/ARC/ARCFrameLowering.h
index c042bec016cac17ccb1d7e2474e4fa08602a6ce4..41b559d1676108f2d485cb51eb69c1c7981c6310 100644
--- a/lib/Target/ARC/ARCFrameLowering.h
+++ b/lib/Target/ARC/ARCFrameLowering.h
@@ -1,9 +1,8 @@
 //===- ARCFrameLowering.h - Define frame lowering for ARC -------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/ARC/ARCISelDAGToDAG.cpp b/lib/Target/ARC/ARCISelDAGToDAG.cpp
index 8dbd3d5bf036ef3221569495aa46b565e95192a0..f639c4e6f0ffe894b9a46634484d9fc9ee2b9202 100644
--- a/lib/Target/ARC/ARCISelDAGToDAG.cpp
+++ b/lib/Target/ARC/ARCISelDAGToDAG.cpp
@@ -1,9 +1,8 @@
 //===- ARCISelDAGToDAG.cpp - ARC dag to dag inst selector -------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/ARC/ARCISelLowering.cpp b/lib/Target/ARC/ARCISelLowering.cpp
index bf98af801406a9b48e499530fd60a0c916f6f7fb..847d23f0abdbc005d7ee6a1ed810aad1b8ea923b 100644
--- a/lib/Target/ARC/ARCISelLowering.cpp
+++ b/lib/Target/ARC/ARCISelLowering.cpp
@@ -1,9 +1,8 @@
 //===- ARCISelLowering.cpp - ARC DAG Lowering Impl --------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/ARC/ARCISelLowering.h b/lib/Target/ARC/ARCISelLowering.h
index fec01b13a866b38a6eb4d52db7786e80ed1df043..4b72bfdaee9c4f4b98fe684a18aadf3e9a78416f 100644
--- a/lib/Target/ARC/ARCISelLowering.h
+++ b/lib/Target/ARC/ARCISelLowering.h
@@ -1,9 +1,8 @@
 //===- ARCISelLowering.h - ARC DAG Lowering Interface -----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/ARC/ARCInstrFormats.td b/lib/Target/ARC/ARCInstrFormats.td
index 0a49b83ef16a383b5fa4a6a00d70a0ce581d2f7f..abacf9100c7bfde95389a515725a182b332807d1 100644
--- a/lib/Target/ARC/ARCInstrFormats.td
+++ b/lib/Target/ARC/ARCInstrFormats.td
@@ -1,9 +1,8 @@
 //===- ARCInstrFormats.td - ARC Instruction Formats --------*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/ARC/ARCInstrInfo.cpp b/lib/Target/ARC/ARCInstrInfo.cpp
index a8084f16893bdfce2ced9746c9f9665773982be5..d71db50a57f5b83e8ffe896f22183880b7dcce73 100644
--- a/lib/Target/ARC/ARCInstrInfo.cpp
+++ b/lib/Target/ARC/ARCInstrInfo.cpp
@@ -1,9 +1,8 @@
 //===- ARCInstrInfo.cpp - ARC Instruction Information -----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -389,7 +388,7 @@ unsigned ARCInstrInfo::insertBranch(MachineBasicBlock &MBB,
 }
 
 unsigned ARCInstrInfo::getInstSizeInBytes(const MachineInstr &MI) const {
-  if (MI.getOpcode() == TargetOpcode::INLINEASM) {
+  if (MI.isInlineAsm()) {
     const MachineFunction *MF = MI.getParent()->getParent();
     const char *AsmStr = MI.getOperand(0).getSymbolName();
     return getInlineAsmLength(AsmStr, *MF->getTarget().getMCAsmInfo());
diff --git a/lib/Target/ARC/ARCInstrInfo.h b/lib/Target/ARC/ARCInstrInfo.h
index f965dd4ff7f8f8f6a43d62aad8e7e93d4837623a..dd30c40d1d3889a2093968b84d214de8eb6deed9 100644
--- a/lib/Target/ARC/ARCInstrInfo.h
+++ b/lib/Target/ARC/ARCInstrInfo.h
@@ -1,9 +1,8 @@
 //===- ARCInstrInfo.h - ARC Instruction Information -------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/ARC/ARCInstrInfo.td b/lib/Target/ARC/ARCInstrInfo.td
index 525098c4ff66bd16c5f1b84bdb6ca6c887d2da66..7ccddd249365c074d0b68e3d30ae2a8b35c9945a 100644
--- a/lib/Target/ARC/ARCInstrInfo.td
+++ b/lib/Target/ARC/ARCInstrInfo.td
@@ -1,9 +1,8 @@
 //===- ARCInstrInfo.td - Target Description for ARC --------*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/ARC/ARCMCInstLower.cpp b/lib/Target/ARC/ARCMCInstLower.cpp
index 43b087a5720423bae17f3ca89736d9e194526985..62462b77eccf94b3b062f31bd256921ec001d6c1 100644
--- a/lib/Target/ARC/ARCMCInstLower.cpp
+++ b/lib/Target/ARC/ARCMCInstLower.cpp
@@ -1,9 +1,8 @@
 //===- ARCMCInstLower.cpp - ARC MachineInstr to MCInst ----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
diff --git a/lib/Target/ARC/ARCMCInstLower.h b/lib/Target/ARC/ARCMCInstLower.h
index 9a698f26334aa5705581172c45efa9a9f592e9af..24a7f68c695de885a8c7877f02a78ae033cca2a0 100644
--- a/lib/Target/ARC/ARCMCInstLower.h
+++ b/lib/Target/ARC/ARCMCInstLower.h
@@ -1,9 +1,8 @@
 //===- ARCMCInstLower.h - Lower MachineInstr to MCInst ----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/ARC/ARCMachineFunctionInfo.cpp b/lib/Target/ARC/ARCMachineFunctionInfo.cpp
index 7672f8d2c6dd5bad62dda980031f87aa6f5ee103..9cd9661ae24552bcfc57e30ba1f9d19087d943c5 100644
--- a/lib/Target/ARC/ARCMachineFunctionInfo.cpp
+++ b/lib/Target/ARC/ARCMachineFunctionInfo.cpp
@@ -1,9 +1,8 @@
 //===- ARCMachineFunctionInfo.cpp - ARC machine func info -------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/ARC/ARCMachineFunctionInfo.h b/lib/Target/ARC/ARCMachineFunctionInfo.h
index 95ad294e366854fd2bcc0b7d9b1c8b97d2a994a9..31aa5b93246cfed4bdce8b807bb8076b7ef12205 100644
--- a/lib/Target/ARC/ARCMachineFunctionInfo.h
+++ b/lib/Target/ARC/ARCMachineFunctionInfo.h
@@ -1,9 +1,8 @@
 //===- ARCMachineFunctionInfo.h - ARC machine function info -----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/ARC/ARCRegisterInfo.cpp b/lib/Target/ARC/ARCRegisterInfo.cpp
index 38ea3c93a2d46bca4134ddc3bf21f84a0729fe6d..d479bd4eb5bc4458a6615f6405002b8d8aae42d9 100644
--- a/lib/Target/ARC/ARCRegisterInfo.cpp
+++ b/lib/Target/ARC/ARCRegisterInfo.cpp
@@ -1,9 +1,8 @@
 //===- ARCRegisterInfo.cpp - ARC Register Information -----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/ARC/ARCRegisterInfo.h b/lib/Target/ARC/ARCRegisterInfo.h
index 53abae3ac7a55cf0d6bea8bbd020c414f3d15e0b..b710e995d5a35055e9b85c22a60b3392a6534a59 100644
--- a/lib/Target/ARC/ARCRegisterInfo.h
+++ b/lib/Target/ARC/ARCRegisterInfo.h
@@ -1,9 +1,8 @@
 //===- ARCRegisterInfo.h - ARC Register Information Impl --------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/ARC/ARCRegisterInfo.td b/lib/Target/ARC/ARCRegisterInfo.td
index 6d8d1b3dfd25684d12b7968ea29a3cad0532b2d2..4b6744ad73da11b4ebdcbcfe9a633272c9482f7b 100644
--- a/lib/Target/ARC/ARCRegisterInfo.td
+++ b/lib/Target/ARC/ARCRegisterInfo.td
@@ -1,9 +1,8 @@
 //===- ARCRegisterInfo.td - ARC Register defs --------------*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/ARC/ARCSubtarget.cpp b/lib/Target/ARC/ARCSubtarget.cpp
index 2107a27bf786f4873729a6abd0c911888c37e173..bce2dbd2eaa6bcdba489bc03773328cd8534301a 100644
--- a/lib/Target/ARC/ARCSubtarget.cpp
+++ b/lib/Target/ARC/ARCSubtarget.cpp
@@ -1,9 +1,8 @@
 //===- ARCSubtarget.cpp - ARC Subtarget Information -------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/ARC/ARCSubtarget.h b/lib/Target/ARC/ARCSubtarget.h
index 631d846f3c9cb6142f0d7524e85bcf1a38d734ab..0be797f753d5446c93cb5cd7aacb34b21f50f57e 100644
--- a/lib/Target/ARC/ARCSubtarget.h
+++ b/lib/Target/ARC/ARCSubtarget.h
@@ -1,9 +1,8 @@
 //===- ARCSubtarget.h - Define Subtarget for the ARC ------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/ARC/ARCTargetMachine.cpp b/lib/Target/ARC/ARCTargetMachine.cpp
index 6f5bbd3b4ef3690584a115112dbcf80821d73e1f..2476d409828e149b830a12520d86b925e60e0f28 100644
--- a/lib/Target/ARC/ARCTargetMachine.cpp
+++ b/lib/Target/ARC/ARCTargetMachine.cpp
@@ -1,9 +1,8 @@
 //===- ARCTargetMachine.cpp - Define TargetMachine for ARC ------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/ARC/ARCTargetMachine.h b/lib/Target/ARC/ARCTargetMachine.h
index 18117e3409af325590688905808a39ab596ff9da..c5e8c3f2936d9d81bd3e23d129474871409791e7 100644
--- a/lib/Target/ARC/ARCTargetMachine.h
+++ b/lib/Target/ARC/ARCTargetMachine.h
@@ -1,9 +1,8 @@
 //===- ARCTargetMachine.h - Define TargetMachine for ARC --------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/ARC/ARCTargetStreamer.h b/lib/Target/ARC/ARCTargetStreamer.h
index 29fdfda661a4335dafe7cc4196a0d8133a0c9f45..abe89673316f5deb4808233fc7d69833b8fb5293 100644
--- a/lib/Target/ARC/ARCTargetStreamer.h
+++ b/lib/Target/ARC/ARCTargetStreamer.h
@@ -1,9 +1,8 @@
 //===- ARCTargetStreamer.h - ARC Target Streamer ----------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/ARC/ARCTargetTransformInfo.h b/lib/Target/ARC/ARCTargetTransformInfo.h
index 20a83d5ae4c7229086a4d6e91f1c0775d8dc768c..3e34008902b5cd8501a0729f11f0336ed3e828f7 100644
--- a/lib/Target/ARC/ARCTargetTransformInfo.h
+++ b/lib/Target/ARC/ARCTargetTransformInfo.h
@@ -1,9 +1,8 @@
 //===- ARCTargetTransformInfo.h - ARC specific TTI --------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 // \file
diff --git a/lib/Target/ARC/Disassembler/ARCDisassembler.cpp b/lib/Target/ARC/Disassembler/ARCDisassembler.cpp
index 3fc5a033dd5d74a7bae9020e9afa79b3125bc9e2..deabeb69f6937c0fb117f32f945591113721d60f 100644
--- a/lib/Target/ARC/Disassembler/ARCDisassembler.cpp
+++ b/lib/Target/ARC/Disassembler/ARCDisassembler.cpp
@@ -1,9 +1,8 @@
 //===- ARCDisassembler.cpp - Disassembler for ARC ---------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
diff --git a/lib/Target/ARC/Disassembler/LLVMBuild.txt b/lib/Target/ARC/Disassembler/LLVMBuild.txt
index fd77f0ba08f09bb3666265b89cdb016398b7a8a6..9fb3c5a15db1c7d1f21bb79cfc4b63acbcfb11e1 100644
--- a/lib/Target/ARC/Disassembler/LLVMBuild.txt
+++ b/lib/Target/ARC/Disassembler/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/Target/ARC/Disassembler/LLVMBuild.txt ------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Target/ARC/InstPrinter/ARCInstPrinter.cpp b/lib/Target/ARC/InstPrinter/ARCInstPrinter.cpp
index 9c820c2fc595d970ed190e0425b35f445dcdbc96..e3e0ea48995781bf325b125d0ba392655cb502b0 100644
--- a/lib/Target/ARC/InstPrinter/ARCInstPrinter.cpp
+++ b/lib/Target/ARC/InstPrinter/ARCInstPrinter.cpp
@@ -1,9 +1,8 @@
 //===- ARCInstPrinter.cpp - ARC MCInst to assembly syntax -------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/ARC/InstPrinter/ARCInstPrinter.h b/lib/Target/ARC/InstPrinter/ARCInstPrinter.h
index bb3898a67cefa69f495327bcc826bc4560d7ab58..5ea58407f9ed158d6ec89c6545d6429228ac3976 100644
--- a/lib/Target/ARC/InstPrinter/ARCInstPrinter.h
+++ b/lib/Target/ARC/InstPrinter/ARCInstPrinter.h
@@ -1,9 +1,8 @@
 //===- ARCInstPrinter.h - Convert ARC MCInst to assembly syntax -*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
diff --git a/lib/Target/ARC/InstPrinter/LLVMBuild.txt b/lib/Target/ARC/InstPrinter/LLVMBuild.txt
index 6718b48652c49028f0afa8d1906ae0d83bde25c6..5133da1ef12130581186b396a9e750048a25f6f9 100644
--- a/lib/Target/ARC/InstPrinter/LLVMBuild.txt
+++ b/lib/Target/ARC/InstPrinter/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/Target/ARC/InstPrinter/LLVMBuild.txt -------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Target/ARC/LLVMBuild.txt b/lib/Target/ARC/LLVMBuild.txt
index 4889f1141fb115042f8ca941094e12b87d5a7ead..a8113f65e3a93631dd7404ffb6f9a06f0bb4c363 100644
--- a/lib/Target/ARC/LLVMBuild.txt
+++ b/lib/Target/ARC/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/Target/ARC/LLVMBuild.txt -------------------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Target/ARC/MCTargetDesc/ARCInfo.h b/lib/Target/ARC/MCTargetDesc/ARCInfo.h
index 401b4c5e66133a4a07fc411a70e84c9528ea49b7..57a77631a1fb5ab37157e2e26a78e03853edbf97 100644
--- a/lib/Target/ARC/MCTargetDesc/ARCInfo.h
+++ b/lib/Target/ARC/MCTargetDesc/ARCInfo.h
@@ -1,9 +1,8 @@
 //===- ARCInfo.h - Additional ARC Info --------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/ARC/MCTargetDesc/ARCMCAsmInfo.cpp b/lib/Target/ARC/MCTargetDesc/ARCMCAsmInfo.cpp
index 5d3fb52cfb450b97b4296fb3535b8da8619ba3e1..10f93e292e9bf9a42742b0aaa0cd6d202397687b 100644
--- a/lib/Target/ARC/MCTargetDesc/ARCMCAsmInfo.cpp
+++ b/lib/Target/ARC/MCTargetDesc/ARCMCAsmInfo.cpp
@@ -1,9 +1,8 @@
 //===- ARCMCAsmInfo.cpp - ARC asm properties --------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/ARC/MCTargetDesc/ARCMCAsmInfo.h b/lib/Target/ARC/MCTargetDesc/ARCMCAsmInfo.h
index 997a370fee8d60fa9c9a14997b5f94bb2ca7b5fd..a086bd88d459c721511e1a6822c813e96e5e360f 100644
--- a/lib/Target/ARC/MCTargetDesc/ARCMCAsmInfo.h
+++ b/lib/Target/ARC/MCTargetDesc/ARCMCAsmInfo.h
@@ -1,9 +1,8 @@
 //===- ARCMCAsmInfo.h - ARC asm properties ----------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/ARC/MCTargetDesc/ARCMCTargetDesc.cpp b/lib/Target/ARC/MCTargetDesc/ARCMCTargetDesc.cpp
index 17be15f730de3600f90536a14d0f2485c0cf8ac5..ae753334ddb7633584f5192bb0f2afc7743d8f7a 100644
--- a/lib/Target/ARC/MCTargetDesc/ARCMCTargetDesc.cpp
+++ b/lib/Target/ARC/MCTargetDesc/ARCMCTargetDesc.cpp
@@ -1,9 +1,8 @@
 //===- ARCMCTargetDesc.cpp - ARC Target Descriptions ------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/ARC/MCTargetDesc/ARCMCTargetDesc.h b/lib/Target/ARC/MCTargetDesc/ARCMCTargetDesc.h
index dd152a6a34f936a50c890f64196985b31603f945..8d14c714831864c4e76d39e9a7adbc6f27821880 100644
--- a/lib/Target/ARC/MCTargetDesc/ARCMCTargetDesc.h
+++ b/lib/Target/ARC/MCTargetDesc/ARCMCTargetDesc.h
@@ -1,9 +1,8 @@
 //===- ARCMCTargetDesc.h - ARC Target Descriptions --------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/ARC/MCTargetDesc/LLVMBuild.txt b/lib/Target/ARC/MCTargetDesc/LLVMBuild.txt
index 9ee33b2d0b686aecbf370e4cb391a7141b0a875f..6abf96e066e08c61f2bef48dd07701a588e25a15 100644
--- a/lib/Target/ARC/MCTargetDesc/LLVMBuild.txt
+++ b/lib/Target/ARC/MCTargetDesc/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/Target/ARC/MCTargetDesc/LLVMBuild.txt ------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Target/ARC/TargetInfo/ARCTargetInfo.cpp b/lib/Target/ARC/TargetInfo/ARCTargetInfo.cpp
index 460b0a9f3e9b2773bf029aeb7864ecc6a0e37603..8c1ec588c8ec83873b3e3862a3e195985ac9779e 100644
--- a/lib/Target/ARC/TargetInfo/ARCTargetInfo.cpp
+++ b/lib/Target/ARC/TargetInfo/ARCTargetInfo.cpp
@@ -1,9 +1,8 @@
 //===- ARCTargetInfo.cpp - ARC Target Implementation ----------- *- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/ARC/TargetInfo/LLVMBuild.txt b/lib/Target/ARC/TargetInfo/LLVMBuild.txt
index bf3520348791a2acbbf092e6cb80f521aa2240c8..d2543e5245858bcfe07a318a730ac5a5e8bfef20 100644
--- a/lib/Target/ARC/TargetInfo/LLVMBuild.txt
+++ b/lib/Target/ARC/TargetInfo/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/Target/ARC/TargetInfo/LLVMBuild.txt --------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Target/ARM/A15SDOptimizer.cpp b/lib/Target/ARM/A15SDOptimizer.cpp
index be88fe4ddb1449ef4fcb28c9b4aeeeaeaaca59e4..fb238bfc9cbc5f04584ede518421368a456ae5a5 100644
--- a/lib/Target/ARM/A15SDOptimizer.cpp
+++ b/lib/Target/ARM/A15SDOptimizer.cpp
@@ -1,9 +1,8 @@
 //=== A15SDOptimizerPass.cpp - Optimize DPR and SPR register accesses on A15==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/ARM/ARM.h b/lib/Target/ARM/ARM.h
index b5cc45c5cc94a89c7f1504c67d6bfa9473781a57..f87337bfe16c4b3f9f2d25810ed2977b757bde76 100644
--- a/lib/Target/ARM/ARM.h
+++ b/lib/Target/ARM/ARM.h
@@ -1,9 +1,8 @@
 //===-- ARM.h - Top-level interface for ARM representation ------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/ARM/ARM.td b/lib/Target/ARM/ARM.td
index b71a09828bce0273fe12577ceaf88e4d218ced37..d7bad51698f1c3e955cb9e6bc26adf277f144b06 100644
--- a/lib/Target/ARM/ARM.td
+++ b/lib/Target/ARM/ARM.td
@@ -1,9 +1,8 @@
 //===-- ARM.td - Describe the ARM Target Machine -----------*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -365,8 +364,8 @@ def FeatureUseAA : SubtargetFeature<"use-aa", "UseAA", "true",
 
 // Armv8.5-A extensions
 
-def FeatureSpecCtrl : SubtargetFeature<"specctrl", "HasSpecCtrl", "true",
-  "Enable speculation control barrier" >;
+def FeatureSB       : SubtargetFeature<"sb", "HasSB", "true",
+  "Enable v8.5a Speculation Barrier" >;
 
 //===----------------------------------------------------------------------===//
 // ARM architecture class
@@ -459,7 +458,7 @@ def HasV8_4aOps   : SubtargetFeature<"v8.4a", "HasV8_4aOps", "true",
 
 def HasV8_5aOps   : SubtargetFeature<"v8.5a", "HasV8_5aOps", "true",
                                    "Support ARM v8.5a instructions",
-                                   [HasV8_4aOps, FeatureSpecCtrl]>;
+                                   [HasV8_4aOps, FeatureSB]>;
 
 //===----------------------------------------------------------------------===//
 // ARM Processor subtarget features.
@@ -1078,7 +1077,9 @@ def : ProcessorModel<"cyclone",     SwiftModel,         [ARMv8a, ProcSwift,
 def : ProcNoItin<"exynos-m1",                           [ARMv8a, ProcExynos]>;
 def : ProcNoItin<"exynos-m2",                           [ARMv8a, ProcExynos]>;
 def : ProcNoItin<"exynos-m3",                           [ARMv8a, ProcExynos]>;
-def : ProcNoItin<"exynos-m4",                           [ARMv8a, ProcExynos]>;
+def : ProcNoItin<"exynos-m4",                           [ARMv82a, ProcExynos,
+                                                         FeatureFullFP16,
+                                                         FeatureDotProd]>;
 
 def : ProcNoItin<"kryo",                                [ARMv8a, ProcKryo,
                                                          FeatureHWDivThumb,
diff --git a/lib/Target/ARM/ARMAsmPrinter.cpp b/lib/Target/ARM/ARMAsmPrinter.cpp
index b7cd3a0c2dae8f6a2d316866dc390a8be7f01d1d..5a33c2028070d973673a8cb1188729d6a1a93fcc 100644
--- a/lib/Target/ARM/ARMAsmPrinter.cpp
+++ b/lib/Target/ARM/ARMAsmPrinter.cpp
@@ -1,9 +1,8 @@
 //===-- ARMAsmPrinter.cpp - Print machine code to an ARM .s file ----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/ARM/ARMAsmPrinter.h b/lib/Target/ARM/ARMAsmPrinter.h
index 0ba4bc05d6f7719f1e65f133e4009d3d9622b485..92c3a4ea5ffcc3d7bfdcaf2beac389143494d17f 100644
--- a/lib/Target/ARM/ARMAsmPrinter.h
+++ b/lib/Target/ARM/ARMAsmPrinter.h
@@ -1,9 +1,8 @@
 //===-- ARMAsmPrinter.h - ARM implementation of AsmPrinter ------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/ARM/ARMBaseInstrInfo.cpp b/lib/Target/ARM/ARMBaseInstrInfo.cpp
index bbebed59c8512ffa32346204210941e233385bfb..9c8fed0e223d11db9e761bdfa310200c9d7322a9 100644
--- a/lib/Target/ARM/ARMBaseInstrInfo.cpp
+++ b/lib/Target/ARM/ARMBaseInstrInfo.cpp
@@ -1,9 +1,8 @@
 //===-- ARMBaseInstrInfo.cpp - ARM Instruction Information ----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -2266,7 +2265,7 @@ bool llvm::tryFoldSPUpdateIntoPushPop(const ARMSubtarget &Subtarget,
                                       unsigned NumBytes) {
   // This optimisation potentially adds lots of load and store
   // micro-operations, it's only really a great benefit to code-size.
-  if (!MF.getFunction().optForMinSize())
+  if (!Subtarget.optForMinSize())
     return false;
 
   // If only one register is pushed/popped, LLVM can use an LDR/STR
@@ -2620,20 +2619,16 @@ inline static ARMCC::CondCodes getCmpToAddCondition(ARMCC::CondCodes CC) {
 inline static bool isRedundantFlagInstr(const MachineInstr *CmpI,
                                         unsigned SrcReg, unsigned SrcReg2,
                                         int ImmValue, const MachineInstr *OI) {
-  if ((CmpI->getOpcode() == ARM::CMPrr ||
-       CmpI->getOpcode() == ARM::t2CMPrr) &&
-      (OI->getOpcode() == ARM::SUBrr ||
-       OI->getOpcode() == ARM::t2SUBrr) &&
+  if ((CmpI->getOpcode() == ARM::CMPrr || CmpI->getOpcode() == ARM::t2CMPrr) &&
+      (OI->getOpcode() == ARM::SUBrr || OI->getOpcode() == ARM::t2SUBrr) &&
       ((OI->getOperand(1).getReg() == SrcReg &&
         OI->getOperand(2).getReg() == SrcReg2) ||
        (OI->getOperand(1).getReg() == SrcReg2 &&
         OI->getOperand(2).getReg() == SrcReg)))
     return true;
 
-  if ((CmpI->getOpcode() == ARM::CMPri ||
-       CmpI->getOpcode() == ARM::t2CMPri) &&
-      (OI->getOpcode() == ARM::SUBri ||
-       OI->getOpcode() == ARM::t2SUBri) &&
+  if ((CmpI->getOpcode() == ARM::CMPri || CmpI->getOpcode() == ARM::t2CMPri) &&
+      (OI->getOpcode() == ARM::SUBri || OI->getOpcode() == ARM::t2SUBri) &&
       OI->getOperand(1).getReg() == SrcReg &&
       OI->getOperand(2).getImm() == ImmValue)
     return true;
diff --git a/lib/Target/ARM/ARMBaseInstrInfo.h b/lib/Target/ARM/ARMBaseInstrInfo.h
index de1f307083babc538e079d89f599850d6c58092f..24c11e43793610b326217918e0927d9fba352c99 100644
--- a/lib/Target/ARM/ARMBaseInstrInfo.h
+++ b/lib/Target/ARM/ARMBaseInstrInfo.h
@@ -1,9 +1,8 @@
 //===-- ARMBaseInstrInfo.h - ARM Base Instruction Information ---*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/ARM/ARMBaseRegisterInfo.cpp b/lib/Target/ARM/ARMBaseRegisterInfo.cpp
index 02b3daf3c6fdfe289efe5b470cd2c1cae0c4ae73..1870e4c0b7f0832bb65b28de0f42d84b8c3fef87 100644
--- a/lib/Target/ARM/ARMBaseRegisterInfo.cpp
+++ b/lib/Target/ARM/ARMBaseRegisterInfo.cpp
@@ -1,9 +1,8 @@
 //===-- ARMBaseRegisterInfo.cpp - ARM Register Information ----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/ARM/ARMBaseRegisterInfo.h b/lib/Target/ARM/ARMBaseRegisterInfo.h
index 45d29ebc0bd362e4fc07736811a4fb1f7c3e272f..308a0345fbd9e283b372dcca9124a6e4737eee7a 100644
--- a/lib/Target/ARM/ARMBaseRegisterInfo.h
+++ b/lib/Target/ARM/ARMBaseRegisterInfo.h
@@ -1,9 +1,8 @@
 //===-- ARMBaseRegisterInfo.h - ARM Register Information Impl ---*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/ARM/ARMBasicBlockInfo.h b/lib/Target/ARM/ARMBasicBlockInfo.h
index e0cb0aa676a6f7fec71d8f6f7f284c6e26c10f70..4eb4106378a2303d6fbc4f3749e9070d2b7e9e8a 100644
--- a/lib/Target/ARM/ARMBasicBlockInfo.h
+++ b/lib/Target/ARM/ARMBasicBlockInfo.h
@@ -1,9 +1,8 @@
 //===-- ARMBasicBlockInfo.h - Basic Block Information -----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/ARM/ARMCallLowering.cpp b/lib/Target/ARM/ARMCallLowering.cpp
index 8e80c32bcf89183425809fe85ac320f4255535d3..def7c5ccb2edb8f7f0d981af0aa029064e749f02 100644
--- a/lib/Target/ARM/ARMCallLowering.cpp
+++ b/lib/Target/ARM/ARMCallLowering.cpp
@@ -1,9 +1,8 @@
 //===- llvm/lib/Target/ARM/ARMCallLowering.cpp - Call lowering ------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -132,7 +131,7 @@ struct OutgoingValueHandler : public CallLowering::ValueHandler {
     unsigned ExtReg = extendRegister(ValVReg, VA);
     auto MMO = MIRBuilder.getMF().getMachineMemOperand(
         MPO, MachineMemOperand::MOStore, VA.getLocVT().getStoreSize(),
-        /* Alignment */ 0);
+        /* Alignment */ 1);
     MIRBuilder.buildStore(ExtReg, Addr, *MMO);
   }
 
@@ -332,11 +331,11 @@ struct IncomingValueHandler : public CallLowering::ValueHandler {
       assert(MRI.getType(ValVReg).isScalar() && "Only scalars supported atm");
 
       auto LoadVReg = MRI.createGenericVirtualRegister(LLT::scalar(32));
-      buildLoad(LoadVReg, Addr, Size, /* Alignment */ 0, MPO);
+      buildLoad(LoadVReg, Addr, Size, /* Alignment */ 1, MPO);
       MIRBuilder.buildTrunc(ValVReg, LoadVReg);
     } else {
       // If the value is not extended, a simple load will suffice.
-      buildLoad(ValVReg, Addr, Size, /* Alignment */ 0, MPO);
+      buildLoad(ValVReg, Addr, Size, /* Alignment */ 1, MPO);
     }
   }
 
@@ -540,19 +539,19 @@ bool ARMCallLowering::lowerCall(MachineIRBuilder &MIRBuilder,
 
   // Create the call instruction so we can add the implicit uses of arg
   // registers, but don't insert it yet.
-  bool isDirect = !Callee.isReg();
-  auto CallOpcode = getCallOpcode(STI, isDirect);
+  bool IsDirect = !Callee.isReg();
+  auto CallOpcode = getCallOpcode(STI, IsDirect);
   auto MIB = MIRBuilder.buildInstrNoInsert(CallOpcode);
 
-  bool isThumb = STI.isThumb();
-  if (isThumb)
+  bool IsThumb = STI.isThumb();
+  if (IsThumb)
     MIB.add(predOps(ARMCC::AL));
 
   MIB.add(Callee);
-  if (!isDirect) {
+  if (!IsDirect) {
     auto CalleeReg = Callee.getReg();
     if (CalleeReg && !TRI->isPhysicalRegister(CalleeReg)) {
-      unsigned CalleeIdx = isThumb ? 2 : 0;
+      unsigned CalleeIdx = IsThumb ? 2 : 0;
       MIB->getOperand(CalleeIdx).setReg(constrainOperandRegClass(
           MF, *TRI, MRI, *STI.getInstrInfo(), *STI.getRegBankInfo(),
           *MIB.getInstr(), MIB->getDesc(), Callee, CalleeIdx));
@@ -561,13 +560,14 @@ bool ARMCallLowering::lowerCall(MachineIRBuilder &MIRBuilder,
 
   MIB.addRegMask(TRI->getCallPreservedMask(MF, CallConv));
 
+  bool IsVarArg = false;
   SmallVector<ArgInfo, 8> ArgInfos;
   for (auto Arg : OrigArgs) {
     if (!isSupportedType(DL, TLI, Arg.Ty))
       return false;
 
     if (!Arg.IsFixed)
-      return false;
+      IsVarArg = true;
 
     if (Arg.Flags.isByVal())
       return false;
@@ -581,7 +581,7 @@ bool ARMCallLowering::lowerCall(MachineIRBuilder &MIRBuilder,
       MIRBuilder.buildUnmerge(Regs, Arg.Reg);
   }
 
-  auto ArgAssignFn = TLI.CCAssignFnForCall(CallConv, /*IsVarArg=*/false);
+  auto ArgAssignFn = TLI.CCAssignFnForCall(CallConv, IsVarArg);
   OutgoingValueHandler ArgHandler(MIRBuilder, MRI, MIB, ArgAssignFn);
   if (!handleAssignments(MIRBuilder, ArgInfos, ArgHandler))
     return false;
@@ -600,7 +600,7 @@ bool ARMCallLowering::lowerCall(MachineIRBuilder &MIRBuilder,
                         SplitRegs.push_back(Reg);
                       });
 
-    auto RetAssignFn = TLI.CCAssignFnForReturn(CallConv, /*IsVarArg=*/false);
+    auto RetAssignFn = TLI.CCAssignFnForReturn(CallConv, IsVarArg);
     CallReturnHandler RetHandler(MIRBuilder, MRI, MIB, RetAssignFn);
     if (!handleAssignments(MIRBuilder, ArgInfos, RetHandler))
       return false;
diff --git a/lib/Target/ARM/ARMCallLowering.h b/lib/Target/ARM/ARMCallLowering.h
index 45a988a2f00e9003ea01047bfc32cf33ccab2aea..d395a42179cdca7d9457ce937bbef95a8d891f00 100644
--- a/lib/Target/ARM/ARMCallLowering.h
+++ b/lib/Target/ARM/ARMCallLowering.h
@@ -1,9 +1,8 @@
 //===- llvm/lib/Target/ARM/ARMCallLowering.h - Call lowering ----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/ARM/ARMCallingConv.cpp b/lib/Target/ARM/ARMCallingConv.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..5ede7c67f7c2ea3443386805e02c469c1acabd4e
--- /dev/null
+++ b/lib/Target/ARM/ARMCallingConv.cpp
@@ -0,0 +1,284 @@
+//=== ARMCallingConv.cpp - ARM Custom CC Routines ---------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains the custom routines for the ARM Calling Convention that
+// aren't done by tablegen, and includes the table generated implementations.
+//
+//===----------------------------------------------------------------------===//
+
+#include "ARM.h"
+#include "ARMCallingConv.h"
+#include "ARMSubtarget.h"
+#include "ARMRegisterInfo.h"
+using namespace llvm;
+
+// APCS f64 is in register pairs, possibly split to stack
+static bool f64AssignAPCS(unsigned &ValNo, MVT &ValVT, MVT &LocVT,
+                          CCValAssign::LocInfo &LocInfo,
+                          CCState &State, bool CanFail) {
+  static const MCPhysReg RegList[] = { ARM::R0, ARM::R1, ARM::R2, ARM::R3 };
+
+  // Try to get the first register.
+  if (unsigned Reg = State.AllocateReg(RegList))
+    State.addLoc(CCValAssign::getCustomReg(ValNo, ValVT, Reg, LocVT, LocInfo));
+  else {
+    // For the 2nd half of a v2f64, do not fail.
+    if (CanFail)
+      return false;
+
+    // Put the whole thing on the stack.
+    State.addLoc(CCValAssign::getCustomMem(ValNo, ValVT,
+                                           State.AllocateStack(8, 4),
+                                           LocVT, LocInfo));
+    return true;
+  }
+
+  // Try to get the second register.
+  if (unsigned Reg = State.AllocateReg(RegList))
+    State.addLoc(CCValAssign::getCustomReg(ValNo, ValVT, Reg, LocVT, LocInfo));
+  else
+    State.addLoc(CCValAssign::getCustomMem(ValNo, ValVT,
+                                           State.AllocateStack(4, 4),
+                                           LocVT, LocInfo));
+  return true;
+}
+
+static bool CC_ARM_APCS_Custom_f64(unsigned &ValNo, MVT &ValVT, MVT &LocVT,
+                                   CCValAssign::LocInfo &LocInfo,
+                                   ISD::ArgFlagsTy &ArgFlags,
+                                   CCState &State) {
+  if (!f64AssignAPCS(ValNo, ValVT, LocVT, LocInfo, State, true))
+    return false;
+  if (LocVT == MVT::v2f64 &&
+      !f64AssignAPCS(ValNo, ValVT, LocVT, LocInfo, State, false))
+    return false;
+  return true;  // we handled it
+}
+
+// AAPCS f64 is in aligned register pairs
+static bool f64AssignAAPCS(unsigned &ValNo, MVT &ValVT, MVT &LocVT,
+                           CCValAssign::LocInfo &LocInfo,
+                           CCState &State, bool CanFail) {
+  static const MCPhysReg HiRegList[] = { ARM::R0, ARM::R2 };
+  static const MCPhysReg LoRegList[] = { ARM::R1, ARM::R3 };
+  static const MCPhysReg ShadowRegList[] = { ARM::R0, ARM::R1 };
+  static const MCPhysReg GPRArgRegs[] = { ARM::R0, ARM::R1, ARM::R2, ARM::R3 };
+
+  unsigned Reg = State.AllocateReg(HiRegList, ShadowRegList);
+  if (Reg == 0) {
+
+    // If we had R3 unallocated only, now we still must to waste it.
+    Reg = State.AllocateReg(GPRArgRegs);
+    assert((!Reg || Reg == ARM::R3) && "Wrong GPRs usage for f64");
+
+    // For the 2nd half of a v2f64, do not just fail.
+    if (CanFail)
+      return false;
+
+    // Put the whole thing on the stack.
+    State.addLoc(CCValAssign::getCustomMem(ValNo, ValVT,
+                                           State.AllocateStack(8, 8),
+                                           LocVT, LocInfo));
+    return true;
+  }
+
+  unsigned i;
+  for (i = 0; i < 2; ++i)
+    if (HiRegList[i] == Reg)
+      break;
+
+  unsigned T = State.AllocateReg(LoRegList[i]);
+  (void)T;
+  assert(T == LoRegList[i] && "Could not allocate register");
+
+  State.addLoc(CCValAssign::getCustomReg(ValNo, ValVT, Reg, LocVT, LocInfo));
+  State.addLoc(CCValAssign::getCustomReg(ValNo, ValVT, LoRegList[i],
+                                         LocVT, LocInfo));
+  return true;
+}
+
+static bool CC_ARM_AAPCS_Custom_f64(unsigned &ValNo, MVT &ValVT, MVT &LocVT,
+                                    CCValAssign::LocInfo &LocInfo,
+                                    ISD::ArgFlagsTy &ArgFlags,
+                                    CCState &State) {
+  if (!f64AssignAAPCS(ValNo, ValVT, LocVT, LocInfo, State, true))
+    return false;
+  if (LocVT == MVT::v2f64 &&
+      !f64AssignAAPCS(ValNo, ValVT, LocVT, LocInfo, State, false))
+    return false;
+  return true;  // we handled it
+}
+
+static bool f64RetAssign(unsigned &ValNo, MVT &ValVT, MVT &LocVT,
+                         CCValAssign::LocInfo &LocInfo, CCState &State) {
+  static const MCPhysReg HiRegList[] = { ARM::R0, ARM::R2 };
+  static const MCPhysReg LoRegList[] = { ARM::R1, ARM::R3 };
+
+  unsigned Reg = State.AllocateReg(HiRegList, LoRegList);
+  if (Reg == 0)
+    return false; // we didn't handle it
+
+  unsigned i;
+  for (i = 0; i < 2; ++i)
+    if (HiRegList[i] == Reg)
+      break;
+
+  State.addLoc(CCValAssign::getCustomReg(ValNo, ValVT, Reg, LocVT, LocInfo));
+  State.addLoc(CCValAssign::getCustomReg(ValNo, ValVT, LoRegList[i],
+                                         LocVT, LocInfo));
+  return true;
+}
+
+static bool RetCC_ARM_APCS_Custom_f64(unsigned &ValNo, MVT &ValVT, MVT &LocVT,
+                                      CCValAssign::LocInfo &LocInfo,
+                                      ISD::ArgFlagsTy &ArgFlags,
+                                      CCState &State) {
+  if (!f64RetAssign(ValNo, ValVT, LocVT, LocInfo, State))
+    return false;
+  if (LocVT == MVT::v2f64 && !f64RetAssign(ValNo, ValVT, LocVT, LocInfo, State))
+    return false;
+  return true;  // we handled it
+}
+
+static bool RetCC_ARM_AAPCS_Custom_f64(unsigned &ValNo, MVT &ValVT, MVT &LocVT,
+                                       CCValAssign::LocInfo &LocInfo,
+                                       ISD::ArgFlagsTy &ArgFlags,
+                                       CCState &State) {
+  return RetCC_ARM_APCS_Custom_f64(ValNo, ValVT, LocVT, LocInfo, ArgFlags,
+                                   State);
+}
+
+static const MCPhysReg RRegList[] = { ARM::R0,  ARM::R1,  ARM::R2,  ARM::R3 };
+
+static const MCPhysReg SRegList[] = { ARM::S0,  ARM::S1,  ARM::S2,  ARM::S3,
+                                      ARM::S4,  ARM::S5,  ARM::S6,  ARM::S7,
+                                      ARM::S8,  ARM::S9,  ARM::S10, ARM::S11,
+                                      ARM::S12, ARM::S13, ARM::S14,  ARM::S15 };
+static const MCPhysReg DRegList[] = { ARM::D0, ARM::D1, ARM::D2, ARM::D3,
+                                      ARM::D4, ARM::D5, ARM::D6, ARM::D7 };
+static const MCPhysReg QRegList[] = { ARM::Q0, ARM::Q1, ARM::Q2, ARM::Q3 };
+
+
+// Allocate part of an AAPCS HFA or HVA. We assume that each member of the HA
+// has InConsecutiveRegs set, and that the last member also has
+// InConsecutiveRegsLast set. We must process all members of the HA before
+// we can allocate it, as we need to know the total number of registers that
+// will be needed in order to (attempt to) allocate a contiguous block.
+static bool CC_ARM_AAPCS_Custom_Aggregate(unsigned &ValNo, MVT &ValVT,
+                                          MVT &LocVT,
+                                          CCValAssign::LocInfo &LocInfo,
+                                          ISD::ArgFlagsTy &ArgFlags,
+                                          CCState &State) {
+  SmallVectorImpl<CCValAssign> &PendingMembers = State.getPendingLocs();
+
+  // AAPCS HFAs must have 1-4 elements, all of the same type
+  if (PendingMembers.size() > 0)
+    assert(PendingMembers[0].getLocVT() == LocVT);
+
+  // Add the argument to the list to be allocated once we know the size of the
+  // aggregate. Store the type's required alignmnent as extra info for later: in
+  // the [N x i64] case all trace has been removed by the time we actually get
+  // to do allocation.
+  PendingMembers.push_back(CCValAssign::getPending(ValNo, ValVT, LocVT, LocInfo,
+                                                   ArgFlags.getOrigAlign()));
+
+  if (!ArgFlags.isInConsecutiveRegsLast())
+    return true;
+
+  // Try to allocate a contiguous block of registers, each of the correct
+  // size to hold one member.
+  auto &DL = State.getMachineFunction().getDataLayout();
+  unsigned StackAlign = DL.getStackAlignment();
+  unsigned Align = std::min(PendingMembers[0].getExtraInfo(), StackAlign);
+
+  ArrayRef<MCPhysReg> RegList;
+  switch (LocVT.SimpleTy) {
+  case MVT::i32: {
+    RegList = RRegList;
+    unsigned RegIdx = State.getFirstUnallocated(RegList);
+
+    // First consume all registers that would give an unaligned object. Whether
+    // we go on stack or in regs, no-one will be using them in future.
+    unsigned RegAlign = alignTo(Align, 4) / 4;
+    while (RegIdx % RegAlign != 0 && RegIdx < RegList.size())
+      State.AllocateReg(RegList[RegIdx++]);
+
+    break;
+  }
+  case MVT::f16:
+  case MVT::f32:
+    RegList = SRegList;
+    break;
+  case MVT::v4f16:
+  case MVT::f64:
+    RegList = DRegList;
+    break;
+  case MVT::v8f16:
+  case MVT::v2f64:
+    RegList = QRegList;
+    break;
+  default:
+    llvm_unreachable("Unexpected member type for block aggregate");
+    break;
+  }
+
+  unsigned RegResult = State.AllocateRegBlock(RegList, PendingMembers.size());
+  if (RegResult) {
+    for (SmallVectorImpl<CCValAssign>::iterator It = PendingMembers.begin();
+         It != PendingMembers.end(); ++It) {
+      It->convertToReg(RegResult);
+      State.addLoc(*It);
+      ++RegResult;
+    }
+    PendingMembers.clear();
+    return true;
+  }
+
+  // Register allocation failed, we'll be needing the stack
+  unsigned Size = LocVT.getSizeInBits() / 8;
+  if (LocVT == MVT::i32 && State.getNextStackOffset() == 0) {
+    // If nothing else has used the stack until this point, a non-HFA aggregate
+    // can be split between regs and stack.
+    unsigned RegIdx = State.getFirstUnallocated(RegList);
+    for (auto &It : PendingMembers) {
+      if (RegIdx >= RegList.size())
+        It.convertToMem(State.AllocateStack(Size, Size));
+      else
+        It.convertToReg(State.AllocateReg(RegList[RegIdx++]));
+
+      State.addLoc(It);
+    }
+    PendingMembers.clear();
+    return true;
+  } else if (LocVT != MVT::i32)
+    RegList = SRegList;
+
+  // Mark all regs as unavailable (AAPCS rule C.2.vfp for VFP, C.6 for core)
+  for (auto Reg : RegList)
+    State.AllocateReg(Reg);
+
+  // After the first item has been allocated, the rest are packed as tightly as
+  // possible. (E.g. an incoming i64 would have starting Align of 8, but we'll
+  // be allocating a bunch of i32 slots).
+  unsigned RestAlign = std::min(Align, Size);
+
+  for (auto &It : PendingMembers) {
+    It.convertToMem(State.AllocateStack(Size, Align));
+    State.addLoc(It);
+    Align = RestAlign;
+  }
+
+  // All pending members have now been allocated
+  PendingMembers.clear();
+
+  // This will be allocated by the last member of the aggregate
+  return true;
+}
+
+// Include the table generated calling convention implementations.
+#include "ARMGenCallingConv.inc"
diff --git a/lib/Target/ARM/ARMCallingConv.h b/lib/Target/ARM/ARMCallingConv.h
index 543165de38d0406c34762dff1607dbb2731dae99..615634551d904d1aba94d10e0a7b63744a480823 100644
--- a/lib/Target/ARM/ARMCallingConv.h
+++ b/lib/Target/ARM/ARMCallingConv.h
@@ -1,292 +1,50 @@
 //=== ARMCallingConv.h - ARM Custom Calling Convention Routines -*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
-// This file contains the custom routines for the ARM Calling Convention that
-// aren't done by tablegen.
+// This file declares the entry points for ARM calling convention analysis.
 //
 //===----------------------------------------------------------------------===//
 
 #ifndef LLVM_LIB_TARGET_ARM_ARMCALLINGCONV_H
 #define LLVM_LIB_TARGET_ARM_ARMCALLINGCONV_H
 
-#include "ARM.h"
-#include "ARMBaseInstrInfo.h"
-#include "ARMSubtarget.h"
 #include "llvm/CodeGen/CallingConvLower.h"
-#include "llvm/CodeGen/TargetInstrInfo.h"
-#include "llvm/IR/CallingConv.h"
 
 namespace llvm {
 
-// APCS f64 is in register pairs, possibly split to stack
-static bool f64AssignAPCS(unsigned &ValNo, MVT &ValVT, MVT &LocVT,
-                          CCValAssign::LocInfo &LocInfo,
-                          CCState &State, bool CanFail) {
-  static const MCPhysReg RegList[] = { ARM::R0, ARM::R1, ARM::R2, ARM::R3 };
-
-  // Try to get the first register.
-  if (unsigned Reg = State.AllocateReg(RegList))
-    State.addLoc(CCValAssign::getCustomReg(ValNo, ValVT, Reg, LocVT, LocInfo));
-  else {
-    // For the 2nd half of a v2f64, do not fail.
-    if (CanFail)
-      return false;
-
-    // Put the whole thing on the stack.
-    State.addLoc(CCValAssign::getCustomMem(ValNo, ValVT,
-                                           State.AllocateStack(8, 4),
-                                           LocVT, LocInfo));
-    return true;
-  }
-
-  // Try to get the second register.
-  if (unsigned Reg = State.AllocateReg(RegList))
-    State.addLoc(CCValAssign::getCustomReg(ValNo, ValVT, Reg, LocVT, LocInfo));
-  else
-    State.addLoc(CCValAssign::getCustomMem(ValNo, ValVT,
-                                           State.AllocateStack(4, 4),
-                                           LocVT, LocInfo));
-  return true;
-}
-
-static bool CC_ARM_APCS_Custom_f64(unsigned &ValNo, MVT &ValVT, MVT &LocVT,
-                                   CCValAssign::LocInfo &LocInfo,
-                                   ISD::ArgFlagsTy &ArgFlags,
-                                   CCState &State) {
-  if (!f64AssignAPCS(ValNo, ValVT, LocVT, LocInfo, State, true))
-    return false;
-  if (LocVT == MVT::v2f64 &&
-      !f64AssignAPCS(ValNo, ValVT, LocVT, LocInfo, State, false))
-    return false;
-  return true;  // we handled it
-}
-
-// AAPCS f64 is in aligned register pairs
-static bool f64AssignAAPCS(unsigned &ValNo, MVT &ValVT, MVT &LocVT,
-                           CCValAssign::LocInfo &LocInfo,
-                           CCState &State, bool CanFail) {
-  static const MCPhysReg HiRegList[] = { ARM::R0, ARM::R2 };
-  static const MCPhysReg LoRegList[] = { ARM::R1, ARM::R3 };
-  static const MCPhysReg ShadowRegList[] = { ARM::R0, ARM::R1 };
-  static const MCPhysReg GPRArgRegs[] = { ARM::R0, ARM::R1, ARM::R2, ARM::R3 };
-
-  unsigned Reg = State.AllocateReg(HiRegList, ShadowRegList);
-  if (Reg == 0) {
-
-    // If we had R3 unallocated only, now we still must to waste it.
-    Reg = State.AllocateReg(GPRArgRegs);
-    assert((!Reg || Reg == ARM::R3) && "Wrong GPRs usage for f64");
-
-    // For the 2nd half of a v2f64, do not just fail.
-    if (CanFail)
-      return false;
-
-    // Put the whole thing on the stack.
-    State.addLoc(CCValAssign::getCustomMem(ValNo, ValVT,
-                                           State.AllocateStack(8, 8),
-                                           LocVT, LocInfo));
-    return true;
-  }
-
-  unsigned i;
-  for (i = 0; i < 2; ++i)
-    if (HiRegList[i] == Reg)
-      break;
-
-  unsigned T = State.AllocateReg(LoRegList[i]);
-  (void)T;
-  assert(T == LoRegList[i] && "Could not allocate register");
-
-  State.addLoc(CCValAssign::getCustomReg(ValNo, ValVT, Reg, LocVT, LocInfo));
-  State.addLoc(CCValAssign::getCustomReg(ValNo, ValVT, LoRegList[i],
-                                         LocVT, LocInfo));
-  return true;
-}
-
-static bool CC_ARM_AAPCS_Custom_f64(unsigned &ValNo, MVT &ValVT, MVT &LocVT,
-                                    CCValAssign::LocInfo &LocInfo,
-                                    ISD::ArgFlagsTy &ArgFlags,
-                                    CCState &State) {
-  if (!f64AssignAAPCS(ValNo, ValVT, LocVT, LocInfo, State, true))
-    return false;
-  if (LocVT == MVT::v2f64 &&
-      !f64AssignAAPCS(ValNo, ValVT, LocVT, LocInfo, State, false))
-    return false;
-  return true;  // we handled it
-}
-
-static bool f64RetAssign(unsigned &ValNo, MVT &ValVT, MVT &LocVT,
-                         CCValAssign::LocInfo &LocInfo, CCState &State) {
-  static const MCPhysReg HiRegList[] = { ARM::R0, ARM::R2 };
-  static const MCPhysReg LoRegList[] = { ARM::R1, ARM::R3 };
-
-  unsigned Reg = State.AllocateReg(HiRegList, LoRegList);
-  if (Reg == 0)
-    return false; // we didn't handle it
-
-  unsigned i;
-  for (i = 0; i < 2; ++i)
-    if (HiRegList[i] == Reg)
-      break;
-
-  State.addLoc(CCValAssign::getCustomReg(ValNo, ValVT, Reg, LocVT, LocInfo));
-  State.addLoc(CCValAssign::getCustomReg(ValNo, ValVT, LoRegList[i],
-                                         LocVT, LocInfo));
-  return true;
-}
-
-static bool RetCC_ARM_APCS_Custom_f64(unsigned &ValNo, MVT &ValVT, MVT &LocVT,
-                                      CCValAssign::LocInfo &LocInfo,
-                                      ISD::ArgFlagsTy &ArgFlags,
-                                      CCState &State) {
-  if (!f64RetAssign(ValNo, ValVT, LocVT, LocInfo, State))
-    return false;
-  if (LocVT == MVT::v2f64 && !f64RetAssign(ValNo, ValVT, LocVT, LocInfo, State))
-    return false;
-  return true;  // we handled it
-}
-
-static bool RetCC_ARM_AAPCS_Custom_f64(unsigned &ValNo, MVT &ValVT, MVT &LocVT,
-                                       CCValAssign::LocInfo &LocInfo,
-                                       ISD::ArgFlagsTy &ArgFlags,
-                                       CCState &State) {
-  return RetCC_ARM_APCS_Custom_f64(ValNo, ValVT, LocVT, LocInfo, ArgFlags,
-                                   State);
-}
-
-static const MCPhysReg RRegList[] = { ARM::R0,  ARM::R1,  ARM::R2,  ARM::R3 };
-
-static const MCPhysReg SRegList[] = { ARM::S0,  ARM::S1,  ARM::S2,  ARM::S3,
-                                      ARM::S4,  ARM::S5,  ARM::S6,  ARM::S7,
-                                      ARM::S8,  ARM::S9,  ARM::S10, ARM::S11,
-                                      ARM::S12, ARM::S13, ARM::S14,  ARM::S15 };
-static const MCPhysReg DRegList[] = { ARM::D0, ARM::D1, ARM::D2, ARM::D3,
-                                      ARM::D4, ARM::D5, ARM::D6, ARM::D7 };
-static const MCPhysReg QRegList[] = { ARM::Q0, ARM::Q1, ARM::Q2, ARM::Q3 };
-
-
-// Allocate part of an AAPCS HFA or HVA. We assume that each member of the HA
-// has InConsecutiveRegs set, and that the last member also has
-// InConsecutiveRegsLast set. We must process all members of the HA before
-// we can allocate it, as we need to know the total number of registers that
-// will be needed in order to (attempt to) allocate a contiguous block.
-static bool CC_ARM_AAPCS_Custom_Aggregate(unsigned &ValNo, MVT &ValVT,
-                                          MVT &LocVT,
-                                          CCValAssign::LocInfo &LocInfo,
-                                          ISD::ArgFlagsTy &ArgFlags,
-                                          CCState &State) {
-  SmallVectorImpl<CCValAssign> &PendingMembers = State.getPendingLocs();
-
-  // AAPCS HFAs must have 1-4 elements, all of the same type
-  if (PendingMembers.size() > 0)
-    assert(PendingMembers[0].getLocVT() == LocVT);
-
-  // Add the argument to the list to be allocated once we know the size of the
-  // aggregate. Store the type's required alignmnent as extra info for later: in
-  // the [N x i64] case all trace has been removed by the time we actually get
-  // to do allocation.
-  PendingMembers.push_back(CCValAssign::getPending(ValNo, ValVT, LocVT, LocInfo,
-                                                   ArgFlags.getOrigAlign()));
-
-  if (!ArgFlags.isInConsecutiveRegsLast())
-    return true;
-
-  // Try to allocate a contiguous block of registers, each of the correct
-  // size to hold one member.
-  auto &DL = State.getMachineFunction().getDataLayout();
-  unsigned StackAlign = DL.getStackAlignment();
-  unsigned Align = std::min(PendingMembers[0].getExtraInfo(), StackAlign);
-
-  ArrayRef<MCPhysReg> RegList;
-  switch (LocVT.SimpleTy) {
-  case MVT::i32: {
-    RegList = RRegList;
-    unsigned RegIdx = State.getFirstUnallocated(RegList);
-
-    // First consume all registers that would give an unaligned object. Whether
-    // we go on stack or in regs, no-one will be using them in future.
-    unsigned RegAlign = alignTo(Align, 4) / 4;
-    while (RegIdx % RegAlign != 0 && RegIdx < RegList.size())
-      State.AllocateReg(RegList[RegIdx++]);
-
-    break;
-  }
-  case MVT::f16:
-  case MVT::f32:
-    RegList = SRegList;
-    break;
-  case MVT::v4f16:
-  case MVT::f64:
-    RegList = DRegList;
-    break;
-  case MVT::v8f16:
-  case MVT::v2f64:
-    RegList = QRegList;
-    break;
-  default:
-    llvm_unreachable("Unexpected member type for block aggregate");
-    break;
-  }
-
-  unsigned RegResult = State.AllocateRegBlock(RegList, PendingMembers.size());
-  if (RegResult) {
-    for (SmallVectorImpl<CCValAssign>::iterator It = PendingMembers.begin();
-         It != PendingMembers.end(); ++It) {
-      It->convertToReg(RegResult);
-      State.addLoc(*It);
-      ++RegResult;
-    }
-    PendingMembers.clear();
-    return true;
-  }
-
-  // Register allocation failed, we'll be needing the stack
-  unsigned Size = LocVT.getSizeInBits() / 8;
-  if (LocVT == MVT::i32 && State.getNextStackOffset() == 0) {
-    // If nothing else has used the stack until this point, a non-HFA aggregate
-    // can be split between regs and stack.
-    unsigned RegIdx = State.getFirstUnallocated(RegList);
-    for (auto &It : PendingMembers) {
-      if (RegIdx >= RegList.size())
-        It.convertToMem(State.AllocateStack(Size, Size));
-      else
-        It.convertToReg(State.AllocateReg(RegList[RegIdx++]));
-
-      State.addLoc(It);
-    }
-    PendingMembers.clear();
-    return true;
-  } else if (LocVT != MVT::i32)
-    RegList = SRegList;
-
-  // Mark all regs as unavailable (AAPCS rule C.2.vfp for VFP, C.6 for core)
-  for (auto Reg : RegList)
-    State.AllocateReg(Reg);
-
-  // After the first item has been allocated, the rest are packed as tightly as
-  // possible. (E.g. an incoming i64 would have starting Align of 8, but we'll
-  // be allocating a bunch of i32 slots).
-  unsigned RestAlign = std::min(Align, Size);
-
-  for (auto &It : PendingMembers) {
-    It.convertToMem(State.AllocateStack(Size, Align));
-    State.addLoc(It);
-    Align = RestAlign;
-  }
-
-  // All pending members have now been allocated
-  PendingMembers.clear();
-
-  // This will be allocated by the last member of the aggregate
-  return true;
-}
-
-} // End llvm namespace
+bool CC_ARM_AAPCS(unsigned ValNo, MVT ValVT, MVT LocVT,
+                  CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags,
+                  CCState &State);
+bool CC_ARM_AAPCS_VFP(unsigned ValNo, MVT ValVT, MVT LocVT,
+                      CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags,
+                      CCState &State);
+bool CC_ARM_APCS(unsigned ValNo, MVT ValVT, MVT LocVT,
+                 CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags,
+                 CCState &State);
+bool CC_ARM_APCS_GHC(unsigned ValNo, MVT ValVT, MVT LocVT,
+                     CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags,
+                     CCState &State);
+bool FastCC_ARM_APCS(unsigned ValNo, MVT ValVT, MVT LocVT,
+                     CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags,
+                     CCState &State);
+bool RetCC_ARM_AAPCS(unsigned ValNo, MVT ValVT, MVT LocVT,
+                     CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags,
+                     CCState &State);
+bool RetCC_ARM_AAPCS_VFP(unsigned ValNo, MVT ValVT, MVT LocVT,
+                         CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags,
+                         CCState &State);
+bool RetCC_ARM_APCS(unsigned ValNo, MVT ValVT, MVT LocVT,
+                    CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags,
+                    CCState &State);
+bool RetFastCC_ARM_APCS(unsigned ValNo, MVT ValVT, MVT LocVT,
+                        CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags,
+                        CCState &State);
+
+} // namespace llvm
 
 #endif
diff --git a/lib/Target/ARM/ARMCallingConv.td b/lib/Target/ARM/ARMCallingConv.td
index f173e423f3e4170eb386ee8d30705cd3a3d424d5..ff5a5e28711ca3de7002dbcd85dc58dd79b61cc2 100644
--- a/lib/Target/ARM/ARMCallingConv.td
+++ b/lib/Target/ARM/ARMCallingConv.td
@@ -1,9 +1,8 @@
 //===-- ARMCallingConv.td - Calling Conventions for ARM ----*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 // This describes the calling conventions for ARM architecture.
@@ -16,6 +15,7 @@ class CCIfAlign<string Align, CCAction A>:
 //===----------------------------------------------------------------------===//
 // ARM APCS Calling Convention
 //===----------------------------------------------------------------------===//
+let Entry = 1 in
 def CC_ARM_APCS : CallingConv<[
 
   // Handles byval parameters.
@@ -44,6 +44,7 @@ def CC_ARM_APCS : CallingConv<[
   CCIfType<[v2f64], CCAssignToStack<16, 4>>
 ]>;
 
+let Entry = 1 in
 def RetCC_ARM_APCS : CallingConv<[
   CCIfType<[i1, i8, i16], CCPromoteToType<i32>>,
   CCIfType<[f32], CCBitConvertToType<i32>>,
@@ -67,6 +68,7 @@ def RetCC_ARM_APCS : CallingConv<[
 //===----------------------------------------------------------------------===//
 // ARM APCS Calling Convention for FastCC (when VFP2 or later is available)
 //===----------------------------------------------------------------------===//
+let Entry = 1 in
 def FastCC_ARM_APCS : CallingConv<[
   // Handle all vector types as either f64 or v2f64.
   CCIfType<[v1i64, v2i32, v4i16, v8i8, v2f32], CCBitConvertToType<f64>>,
@@ -86,6 +88,7 @@ def FastCC_ARM_APCS : CallingConv<[
   CCDelegateTo<CC_ARM_APCS>
 ]>;
 
+let Entry = 1 in
 def RetFastCC_ARM_APCS : CallingConv<[
   // Handle all vector types as either f64 or v2f64.
   CCIfType<[v1i64, v2i32, v4i16, v8i8, v2f32], CCBitConvertToType<f64>>,
@@ -102,6 +105,7 @@ def RetFastCC_ARM_APCS : CallingConv<[
 // ARM APCS Calling Convention for GHC
 //===----------------------------------------------------------------------===//
 
+let Entry = 1 in
 def CC_ARM_APCS_GHC : CallingConv<[
   // Handle all vector types as either f64 or v2f64.
   CCIfType<[v1i64, v2i32, v4i16, v8i8, v2f32], CCBitConvertToType<f64>>,
@@ -152,6 +156,7 @@ def RetCC_ARM_AAPCS_Common : CallingConv<[
 // ARM AAPCS (EABI) Calling Convention
 //===----------------------------------------------------------------------===//
 
+let Entry = 1 in
 def CC_ARM_AAPCS : CallingConv<[
   // Handles byval parameters.
   CCIfByVal<CCPassByVal<4, 4>>,
@@ -174,6 +179,7 @@ def CC_ARM_AAPCS : CallingConv<[
   CCDelegateTo<CC_ARM_AAPCS_Common>
 ]>;
 
+let Entry = 1 in
 def RetCC_ARM_AAPCS : CallingConv<[
   // Handle all vector types as either f64 or v2f64.
   CCIfType<[v1i64, v2i32, v4i16, v4f16, v8i8, v2f32], CCBitConvertToType<f64>>,
@@ -196,6 +202,7 @@ def RetCC_ARM_AAPCS : CallingConv<[
 // Also used for FastCC (when VFP2 or later is available)
 //===----------------------------------------------------------------------===//
 
+let Entry = 1 in
 def CC_ARM_AAPCS_VFP : CallingConv<[
   // Handles byval parameters.
   CCIfByVal<CCPassByVal<4, 4>>,
@@ -220,6 +227,7 @@ def CC_ARM_AAPCS_VFP : CallingConv<[
   CCDelegateTo<CC_ARM_AAPCS_Common>
 ]>;
 
+let Entry = 1 in
 def RetCC_ARM_AAPCS_VFP : CallingConv<[
   // Handle all vector types as either f64 or v2f64.
   CCIfType<[v1i64, v2i32, v4i16, v4f16, v8i8, v2f32], CCBitConvertToType<f64>>,
diff --git a/lib/Target/ARM/ARMCodeGenPrepare.cpp b/lib/Target/ARM/ARMCodeGenPrepare.cpp
index b631c2bc687b05cd3e63f3a94dbf31489242598d..bfa836cd7f7e4f9ff39870af8b0d73e1ceeb36fb 100644
--- a/lib/Target/ARM/ARMCodeGenPrepare.cpp
+++ b/lib/Target/ARM/ARMCodeGenPrepare.cpp
@@ -1,9 +1,8 @@
 //===----- ARMCodeGenPrepare.cpp ------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -114,7 +113,11 @@ class IRPromoter {
   SmallPtrSet<Value*, 8> Promoted;
   Module *M = nullptr;
   LLVMContext &Ctx;
+  // The type we promote to: always i32
   IntegerType *ExtTy = nullptr;
+  // The type of the value that the search began from, either i8 or i16.
+  // This defines the max range of the values that we allow in the promoted
+  // tree.
   IntegerType *OrigTy = nullptr;
   SmallPtrSetImpl<Value*> *Visited;
   SmallPtrSetImpl<Value*> *Sources;
@@ -327,7 +330,7 @@ bool ARMCodeGenPrepare::isSafeOverflow(Instruction *I) {
   // - (255 >= 254) == (0xFFFFFFFF >= 254) == true
   //
   // To demonstrate why we can't handle increasing values:
-  // 
+  //
   // %add = add i8 %a, 2
   // %cmp = icmp ult i8 %add, 127
   //
@@ -605,7 +608,7 @@ void IRPromoter::PromoteTree() {
 
     if (!shouldPromote(I) || SafeToPromote->count(I) || NewInsts.count(I))
       continue;
-  
+
     assert(EnableDSP && "DSP intrinisc insertion not enabled!");
 
     // Replace unsafe instructions with appropriate intrinsic calls.
@@ -686,10 +689,10 @@ void IRPromoter::Cleanup() {
   // Some zexts will now have become redundant, along with their trunc
   // operands, so remove them
   for (auto V : *Visited) {
-    if (!isa<CastInst>(V))
+    if (!isa<ZExtInst>(V))
       continue;
 
-    auto ZExt = cast<CastInst>(V);
+    auto ZExt = cast<ZExtInst>(V);
     if (ZExt->getDestTy() != ExtTy)
       continue;
 
@@ -701,9 +704,11 @@ void IRPromoter::Cleanup() {
       continue;
     }
 
-    // For any truncs that we insert to handle zexts, we can replace the
-    // result of the zext with the input to the trunc.
-    if (NewInsts.count(Src) && isa<ZExtInst>(V) && isa<TruncInst>(Src)) {
+    // Unless they produce a value that is narrower than ExtTy, we can
+    // replace the result of the zext with the input of a newly inserted
+    // trunc.
+    if (NewInsts.count(Src) && isa<TruncInst>(Src) &&
+        Src->getType() == OrigTy) {
       auto *Trunc = cast<TruncInst>(Src);
       assert(Trunc->getOperand(0)->getType() == ExtTy &&
              "expected inserted trunc to be operating on i32");
diff --git a/lib/Target/ARM/ARMComputeBlockSize.cpp b/lib/Target/ARM/ARMComputeBlockSize.cpp
index b263e9d86c42c20048f875c2957ea7483a764732..828fc6568380c4c717a942a5a9782d8808edc16d 100644
--- a/lib/Target/ARM/ARMComputeBlockSize.cpp
+++ b/lib/Target/ARM/ARMComputeBlockSize.cpp
@@ -1,9 +1,8 @@
 //===--- ARMComputeBlockSize.cpp - Compute machine block sizes ------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/ARM/ARMConstantIslandPass.cpp b/lib/Target/ARM/ARMConstantIslandPass.cpp
index 5e97c4cb35e3636ab8f1ded5839c0145d83b91d0..5107df85a603e95aa5f8bc2426fd10a5170e5ae4 100644
--- a/lib/Target/ARM/ARMConstantIslandPass.cpp
+++ b/lib/Target/ARM/ARMConstantIslandPass.cpp
@@ -1,9 +1,8 @@
 //===- ARMConstantIslandPass.cpp - ARM constant islands -------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/ARM/ARMConstantPoolValue.cpp b/lib/Target/ARM/ARMConstantPoolValue.cpp
index 236c4fab2a5c37e33cda1aabc4ae6fbc63a63503..3bdb0e1ef62d9331e49fc9b78c30b9217ea2dd2f 100644
--- a/lib/Target/ARM/ARMConstantPoolValue.cpp
+++ b/lib/Target/ARM/ARMConstantPoolValue.cpp
@@ -1,9 +1,8 @@
 //===- ARMConstantPoolValue.cpp - ARM constantpool value ------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/ARM/ARMConstantPoolValue.h b/lib/Target/ARM/ARMConstantPoolValue.h
index 55194ed945325fd30c16bfcc4a40f19559acba9e..660b7fc88d82ec450f394ef65a9359cdfcf52dcc 100644
--- a/lib/Target/ARM/ARMConstantPoolValue.h
+++ b/lib/Target/ARM/ARMConstantPoolValue.h
@@ -1,9 +1,8 @@
 //===- ARMConstantPoolValue.h - ARM constantpool value ----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/ARM/ARMExpandPseudoInsts.cpp b/lib/Target/ARM/ARMExpandPseudoInsts.cpp
index eecd0a10dc7d5b6b089538e45fea485dcf1e0505..680142c228899fa7d602677bd520bdfba9235558 100644
--- a/lib/Target/ARM/ARMExpandPseudoInsts.cpp
+++ b/lib/Target/ARM/ARMExpandPseudoInsts.cpp
@@ -1,9 +1,8 @@
 //===-- ARMExpandPseudoInsts.cpp - Expand pseudo instructions -------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/ARM/ARMFastISel.cpp b/lib/Target/ARM/ARMFastISel.cpp
index a50abfdbee446188465f9c215fdf1725a218a9b6..cd01b70b378b4d72a34cc8d7ef867621d8a26970 100644
--- a/lib/Target/ARM/ARMFastISel.cpp
+++ b/lib/Target/ARM/ARMFastISel.cpp
@@ -1,9 +1,8 @@
 //===- ARMFastISel.cpp - ARM FastISel implementation ----------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -245,8 +244,6 @@ class ARMFastISel final : public FastISel {
 
 } // end anonymous namespace
 
-#include "ARMGenCallingConv.inc"
-
 // DefinesOptionalPredicate - This is different from DefinesPredicate in that
 // we don't care about implicit defs here, just places we'll need to add a
 // default CCReg argument. Sets CPSR if we're setting CPSR instead of CCR.
@@ -500,7 +497,7 @@ unsigned ARMFastISel::ARMMaterializeInt(const Constant *C, MVT VT) {
   }
 
   unsigned ResultReg = 0;
-  if (Subtarget->useMovt(*FuncInfo.MF))
+  if (Subtarget->useMovt())
     ResultReg = fastEmit_i(VT, VT, ISD::Constant, CI->getZExtValue());
 
   if (ResultReg)
@@ -558,7 +555,7 @@ unsigned ARMFastISel::ARMMaterializeGV(const GlobalValue *GV, MVT VT) {
   bool IsPositionIndependent = isPositionIndependent();
   // Use movw+movt when possible, it avoids constant pool entries.
   // Non-darwin targets only support static movt relocations in FastISel.
-  if (Subtarget->useMovt(*FuncInfo.MF) &&
+  if (Subtarget->useMovt() &&
       (Subtarget->isTargetMachO() || !IsPositionIndependent)) {
     unsigned Opc;
     unsigned char TF = 0;
diff --git a/lib/Target/ARM/ARMFeatures.h b/lib/Target/ARM/ARMFeatures.h
index 8c0df4c2cbf9498a888ebb9a5453e4c7190ca455..5cd7006c22fc36cbb1dae8ef981eb7d2d02a625e 100644
--- a/lib/Target/ARM/ARMFeatures.h
+++ b/lib/Target/ARM/ARMFeatures.h
@@ -1,9 +1,8 @@
 //===-- ARMFeatures.h - Checks for ARM instruction features -----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/ARM/ARMFrameLowering.cpp b/lib/Target/ARM/ARMFrameLowering.cpp
index 2417166ca962149fc429675f4934228763cac98f..cee6f867526880d75666fa86e682f8025f241cfb 100644
--- a/lib/Target/ARM/ARMFrameLowering.cpp
+++ b/lib/Target/ARM/ARMFrameLowering.cpp
@@ -1,9 +1,8 @@
 //===- ARMFrameLowering.cpp - ARM Frame Information -----------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -79,12 +78,11 @@ ARMFrameLowering::ARMFrameLowering(const ARMSubtarget &sti)
     : TargetFrameLowering(StackGrowsDown, sti.getStackAlignment(), 0, 4),
       STI(sti) {}
 
-bool ARMFrameLowering::noFramePointerElim(const MachineFunction &MF) const {
+bool ARMFrameLowering::keepFramePointer(const MachineFunction &MF) const {
   // iOS always has a FP for backtracking, force other targets to keep their FP
   // when doing FastISel. The emitted code is currently superior, and in cases
   // like test-suite's lencod FastISel isn't quite correct when FP is eliminated.
-  return TargetFrameLowering::noFramePointerElim(MF) ||
-         MF.getSubtarget<ARMSubtarget>().useFastISel();
+  return MF.getSubtarget<ARMSubtarget>().useFastISel();
 }
 
 /// Returns true if the target can safely skip saving callee-saved registers
diff --git a/lib/Target/ARM/ARMFrameLowering.h b/lib/Target/ARM/ARMFrameLowering.h
index e994cab28fe7783b14dc8b349b8c66e3a1bb7966..7544ca3c38d6ef9492c1b3d225e1b8f9376b5bac 100644
--- a/lib/Target/ARM/ARMFrameLowering.h
+++ b/lib/Target/ARM/ARMFrameLowering.h
@@ -1,9 +1,8 @@
 //===- ARMTargetFrameLowering.h - Define frame lowering for ARM -*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -42,7 +41,7 @@ public:
                                   std::vector<CalleeSavedInfo> &CSI,
                                   const TargetRegisterInfo *TRI) const override;
 
-  bool noFramePointerElim(const MachineFunction &MF) const override;
+  bool keepFramePointer(const MachineFunction &MF) const override;
 
   bool enableCalleeSaveSkip(const MachineFunction &MF) const override;
 
diff --git a/lib/Target/ARM/ARMHazardRecognizer.cpp b/lib/Target/ARM/ARMHazardRecognizer.cpp
index d5dacbe08770163dd7e9aae1b12c5c1ea65028e9..0fa32a0abeffc7a8bc95dfbd15e867138d1f140c 100644
--- a/lib/Target/ARM/ARMHazardRecognizer.cpp
+++ b/lib/Target/ARM/ARMHazardRecognizer.cpp
@@ -1,9 +1,8 @@
 //===-- ARMHazardRecognizer.cpp - ARM postra hazard recognizer ------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/ARM/ARMHazardRecognizer.h b/lib/Target/ARM/ARMHazardRecognizer.h
index ccf09db69937ba3567ac1a5cc418793002e09ff7..b5ac694e01f72d179f1e876bfd26a5d59ef4e1ea 100644
--- a/lib/Target/ARM/ARMHazardRecognizer.h
+++ b/lib/Target/ARM/ARMHazardRecognizer.h
@@ -1,9 +1,8 @@
 //===-- ARMHazardRecognizer.h - ARM Hazard Recognizers ----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/ARM/ARMISelDAGToDAG.cpp b/lib/Target/ARM/ARMISelDAGToDAG.cpp
index 8e0e82388251ae6499998fd6c32cc2d8c875cfa3..f765334577db7b9f961dd69a238095f357f0499b 100644
--- a/lib/Target/ARM/ARMISelDAGToDAG.cpp
+++ b/lib/Target/ARM/ARMISelDAGToDAG.cpp
@@ -1,9 +1,8 @@
 //===-- ARMISelDAGToDAG.cpp - A dag to dag inst selector for ARM ----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -131,6 +130,7 @@ public:
 
   // Thumb Addressing Modes:
   bool SelectThumbAddrModeRR(SDValue N, SDValue &Base, SDValue &Offset);
+  bool SelectThumbAddrModeRRSext(SDValue N, SDValue &Base, SDValue &Offset);
   bool SelectThumbAddrModeImm5S(SDValue N, unsigned Scale, SDValue &Base,
                                 SDValue &OffImm);
   bool SelectThumbAddrModeImm5S1(SDValue N, SDValue &Base,
@@ -452,8 +452,10 @@ unsigned ARMDAGToDAGISel::ConstantMaterializationCost(unsigned Val) const {
   if (Subtarget->isThumb()) {
     if (Val <= 255) return 1;                               // MOV
     if (Subtarget->hasV6T2Ops() &&
-        (Val <= 0xffff || ARM_AM::getT2SOImmValSplatVal(Val) != -1))
-      return 1; // MOVW
+        (Val <= 0xffff ||                                   // MOV
+         ARM_AM::getT2SOImmVal(Val) != -1 ||                // MOVW
+         ARM_AM::getT2SOImmVal(~Val) != -1))                // MVN
+      return 1;
     if (Val <= 510) return 2;                               // MOV + ADDi8
     if (~Val <= 255) return 2;                              // MOV + MVN
     if (ARM_AM::isThumbImmShiftedVal(Val)) return 2;        // MOV + LSL
@@ -463,7 +465,7 @@ unsigned ARMDAGToDAGISel::ConstantMaterializationCost(unsigned Val) const {
     if (Subtarget->hasV6T2Ops() && Val <= 0xffff) return 1; // MOVW
     if (ARM_AM::isSOImmTwoPartVal(Val)) return 2;           // two instrs
   }
-  if (Subtarget->useMovt(*MF)) return 2; // MOVW + MOVT
+  if (Subtarget->useMovt()) return 2; // MOVW + MOVT
   return 3; // Literal pool load
 }
 
@@ -1033,8 +1035,22 @@ bool ARMDAGToDAGISel::SelectAddrModePC(SDValue N,
 //                         Thumb Addressing Modes
 //===----------------------------------------------------------------------===//
 
-bool ARMDAGToDAGISel::SelectThumbAddrModeRR(SDValue N,
-                                            SDValue &Base, SDValue &Offset){
+static bool shouldUseZeroOffsetLdSt(SDValue N) {
+  // Negative numbers are difficult to materialise in thumb1. If we are
+  // selecting the add of a negative, instead try to select ri with a zero
+  // offset, so create the add node directly which will become a sub.
+  if (N.getOpcode() != ISD::ADD)
+    return false;
+
+  // Look for an imm which is not legal for ld/st, but is legal for sub.
+  if (auto C = dyn_cast<ConstantSDNode>(N.getOperand(1)))
+    return C->getSExtValue() < 0 && C->getSExtValue() >= -255;
+
+  return false;
+}
+
+bool ARMDAGToDAGISel::SelectThumbAddrModeRRSext(SDValue N, SDValue &Base,
+                                                SDValue &Offset) {
   if (N.getOpcode() != ISD::ADD && !CurDAG->isBaseWithConstantOffset(N)) {
     ConstantSDNode *NC = dyn_cast<ConstantSDNode>(N);
     if (!NC || !NC->isNullValue())
@@ -1049,9 +1065,22 @@ bool ARMDAGToDAGISel::SelectThumbAddrModeRR(SDValue N,
   return true;
 }
 
+bool ARMDAGToDAGISel::SelectThumbAddrModeRR(SDValue N, SDValue &Base,
+                                            SDValue &Offset) {
+  if (shouldUseZeroOffsetLdSt(N))
+    return false; // Select ri instead
+  return SelectThumbAddrModeRRSext(N, Base, Offset);
+}
+
 bool
 ARMDAGToDAGISel::SelectThumbAddrModeImm5S(SDValue N, unsigned Scale,
                                           SDValue &Base, SDValue &OffImm) {
+  if (shouldUseZeroOffsetLdSt(N)) {
+    Base = N;
+    OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32);
+    return true;
+  }
+
   if (!CurDAG->isBaseWithConstantOffset(N)) {
     if (N.getOpcode() == ISD::ADD) {
       return false; // We want to select register offset instead
@@ -2586,6 +2615,7 @@ void ARMDAGToDAGISel::Select(SDNode *N) {
       return;
     break;
   case ISD::INLINEASM:
+  case ISD::INLINEASM_BR:
     if (tryInlineAsm(N))
       return;
     break;
@@ -4290,7 +4320,7 @@ bool ARMDAGToDAGISel::tryInlineAsm(SDNode *N){
   if (!Changed)
     return false;
 
-  SDValue New = CurDAG->getNode(ISD::INLINEASM, SDLoc(N),
+  SDValue New = CurDAG->getNode(N->getOpcode(), SDLoc(N),
       CurDAG->getVTList(MVT::Other, MVT::Glue), AsmNodeOperands);
   New->setNodeId(-1);
   ReplaceNode(N, New.getNode());
diff --git a/lib/Target/ARM/ARMISelLowering.cpp b/lib/Target/ARM/ARMISelLowering.cpp
index 8d5cf0a2c1c2180bc9172f705791628724464eac..01ee6a33c7cc208c54de3e99be41571a42a637e4 100644
--- a/lib/Target/ARM/ARMISelLowering.cpp
+++ b/lib/Target/ARM/ARMISelLowering.cpp
@@ -1,9 +1,8 @@
 //===- ARMISelLowering.cpp - ARM DAG Lowering Implementation --------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -1177,6 +1176,8 @@ ARMTargetLowering::ARMTargetLowering(const TargetMachine &TM,
 
   if (Subtarget->hasV6Ops())
     setTargetDAGCombine(ISD::SRL);
+  if (Subtarget->isThumb1Only())
+    setTargetDAGCombine(ISD::SHL);
 
   setStackPointerRegisterToSaveRestore(ARM::SP);
 
@@ -1590,8 +1591,6 @@ static void FPCCToARMCC(ISD::CondCode CC, ARMCC::CondCodes &CondCode,
 //                      Calling Convention Implementation
 //===----------------------------------------------------------------------===//
 
-#include "ARMGenCallingConv.inc"
-
 /// getEffectiveCallingConv - Get the effective calling convention, taking into
 /// account presence of floating point hardware and calling convention
 /// limitations, such as support for variadic functions.
@@ -2070,7 +2069,7 @@ ARMTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
     auto *GV = cast<GlobalAddressSDNode>(Callee)->getGlobal();
     auto *BB = CLI.CS.getParent();
     bool PreferIndirect =
-        Subtarget->isThumb() && MF.getFunction().optForMinSize() &&
+        Subtarget->isThumb() && Subtarget->optForMinSize() &&
         count_if(GV->users(), [&BB](const User *U) {
           return isa<Instruction>(U) && cast<Instruction>(U)->getParent() == BB;
         }) > 2;
@@ -2142,7 +2141,7 @@ ARMTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
       CallOpc = ARMISD::CALL_NOLINK;
     else if (doesNotRet && isDirect && Subtarget->hasRetAddrStack() &&
              // Emit regular call when code size is the priority
-             !MF.getFunction().optForMinSize())
+             !Subtarget->optForMinSize())
       // "mov lr, pc; b _foo" to avoid confusing the RSP
       CallOpc = ARMISD::CALL_NOLINK;
     else
@@ -3225,7 +3224,7 @@ SDValue ARMTargetLowering::LowerGlobalAddressELF(SDValue Op,
   } else if (Subtarget->isRWPI() && !IsRO) {
     // SB-relative.
     SDValue RelAddr;
-    if (Subtarget->useMovt(DAG.getMachineFunction())) {
+    if (Subtarget->useMovt()) {
       ++NumMovwMovt;
       SDValue G = DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0, ARMII::MO_SBREL);
       RelAddr = DAG.getNode(ARMISD::Wrapper, dl, PtrVT, G);
@@ -3245,7 +3244,7 @@ SDValue ARMTargetLowering::LowerGlobalAddressELF(SDValue Op,
 
   // If we have T2 ops, we can materialize the address directly via movt/movw
   // pair. This is always cheaper.
-  if (Subtarget->useMovt(DAG.getMachineFunction())) {
+  if (Subtarget->useMovt()) {
     ++NumMovwMovt;
     // FIXME: Once remat is capable of dealing with instructions with register
     // operands, expand this into two nodes.
@@ -3268,7 +3267,7 @@ SDValue ARMTargetLowering::LowerGlobalAddressDarwin(SDValue Op,
   SDLoc dl(Op);
   const GlobalValue *GV = cast<GlobalAddressSDNode>(Op)->getGlobal();
 
-  if (Subtarget->useMovt(DAG.getMachineFunction()))
+  if (Subtarget->useMovt())
     ++NumMovwMovt;
 
   // FIXME: Once remat is capable of dealing with instructions with register
@@ -3288,7 +3287,7 @@ SDValue ARMTargetLowering::LowerGlobalAddressDarwin(SDValue Op,
 SDValue ARMTargetLowering::LowerGlobalAddressWindows(SDValue Op,
                                                      SelectionDAG &DAG) const {
   assert(Subtarget->isTargetWindows() && "non-Windows COFF is not supported");
-  assert(Subtarget->useMovt(DAG.getMachineFunction()) &&
+  assert(Subtarget->useMovt() &&
          "Windows on ARM expects to use movw/movt");
   assert(!Subtarget->isROPI() && !Subtarget->isRWPI() &&
          "ROPI/RWPI not currently supported for Windows");
@@ -7809,8 +7808,7 @@ ARMTargetLowering::BuildSDIVPow2(SDNode *N, const APInt &Divisor,
     return SDValue();
 
   const auto &ST = static_cast<const ARMSubtarget&>(DAG.getSubtarget());
-  const auto &MF = DAG.getMachineFunction();
-  const bool MinSize = MF.getFunction().optForMinSize();
+  const bool MinSize = ST.optForMinSize();
   const bool HasDivide = ST.isThumb() ? ST.hasDivideInThumbMode()
                                       : ST.hasDivideInARMMode();
 
@@ -8980,7 +8978,7 @@ ARMTargetLowering::EmitStructByval(MachineInstr &MI,
 
   // Load an immediate to varEnd.
   unsigned varEnd = MRI.createVirtualRegister(TRC);
-  if (Subtarget->useMovt(*MF)) {
+  if (Subtarget->useMovt()) {
     unsigned Vtmp = varEnd;
     if ((LoopSize & 0xFFFF0000) != 0)
       Vtmp = MRI.createVirtualRegister(TRC);
@@ -10419,11 +10417,28 @@ ARMTargetLowering::isDesirableToCommuteWithShift(const SDNode *N,
   if (Level == BeforeLegalizeTypes)
     return true;
 
-  if (Subtarget->isThumb() && Subtarget->isThumb1Only())
+  if (N->getOpcode() != ISD::SHL)
     return true;
 
-  if (N->getOpcode() != ISD::SHL)
+  if (Subtarget->isThumb1Only()) {
+    // Avoid making expensive immediates by commuting shifts. (This logic
+    // only applies to Thumb1 because ARM and Thumb2 immediates can be shifted
+    // for free.)
+    if (N->getOpcode() != ISD::SHL)
+      return true;
+    SDValue N1 = N->getOperand(0);
+    if (N1->getOpcode() != ISD::ADD && N1->getOpcode() != ISD::AND &&
+        N1->getOpcode() != ISD::OR && N1->getOpcode() != ISD::XOR)
+      return true;
+    if (auto *Const = dyn_cast<ConstantSDNode>(N1->getOperand(1))) {
+      if (Const->getAPIntValue().ult(256))
+        return false;
+      if (N1->getOpcode() == ISD::ADD && Const->getAPIntValue().slt(0) &&
+          Const->getAPIntValue().sgt(-256))
+        return false;
+    }
     return true;
+  }
 
   // Turn off commute-with-shift transform after legalization, so it doesn't
   // conflict with PerformSHLSimplify.  (We could try to detect when
@@ -10432,6 +10447,18 @@ ARMTargetLowering::isDesirableToCommuteWithShift(const SDNode *N,
   return false;
 }
 
+bool
+ARMTargetLowering::shouldFoldShiftPairToMask(const SDNode *N,
+                                             CombineLevel Level) const {
+  if (!Subtarget->isThumb1Only())
+    return true;
+
+  if (Level == BeforeLegalizeTypes)
+    return true;
+
+  return false;
+}
+
 static SDValue PerformSHLSimplify(SDNode *N,
                                 TargetLowering::DAGCombinerInfo &DCI,
                                 const ARMSubtarget *ST) {
@@ -10735,6 +10762,12 @@ static SDValue CombineANDShift(SDNode *N,
   if (!C2 || C2 >= 32)
     return SDValue();
 
+  // Clear irrelevant bits in the mask.
+  if (LeftShift)
+    C1 &= (-1U << C2);
+  else
+    C1 &= (-1U >> C2);
+
   SelectionDAG &DAG = DCI.DAG;
   SDLoc DL(N);
 
@@ -10742,9 +10775,7 @@ static SDValue CombineANDShift(SDNode *N,
   // "(and (srl x, c2) c1)", where c1 is a shifted mask. Try to
   // transform to a pair of shifts, to save materializing c1.
 
-  // First pattern: right shift, and c1+1 is a power of two.
-  // FIXME: Also check reversed pattern (left shift, and ~c1+1 is a power
-  // of two).
+  // First pattern: right shift, then mask off leading bits.
   // FIXME: Use demanded bits?
   if (!LeftShift && isMask_32(C1)) {
     uint32_t C3 = countLeadingZeros(C1);
@@ -10756,13 +10787,23 @@ static SDValue CombineANDShift(SDNode *N,
     }
   }
 
-  // Second pattern: left shift, and (c1>>c2)+1 is a power of two.
-  // FIXME: Also check reversed pattern (right shift, and ~(c1<<c2)+1
-  // is a power of two).
+  // First pattern, reversed: left shift, then mask off trailing bits.
+  if (LeftShift && isMask_32(~C1)) {
+    uint32_t C3 = countTrailingZeros(C1);
+    if (C2 < C3) {
+      SDValue SHL = DAG.getNode(ISD::SRL, DL, MVT::i32, N0->getOperand(0),
+                                DAG.getConstant(C3 - C2, DL, MVT::i32));
+      return DAG.getNode(ISD::SHL, DL, MVT::i32, SHL,
+                         DAG.getConstant(C3, DL, MVT::i32));
+    }
+  }
+
+  // Second pattern: left shift, then mask off leading bits.
   // FIXME: Use demanded bits?
   if (LeftShift && isShiftedMask_32(C1)) {
+    uint32_t Trailing = countTrailingZeros(C1);
     uint32_t C3 = countLeadingZeros(C1);
-    if (C2 + C3 < 32 && C1 == ((-1U << (C2 + C3)) >> C3)) {
+    if (Trailing == C2 && C2 + C3 < 32) {
       SDValue SHL = DAG.getNode(ISD::SHL, DL, MVT::i32, N0->getOperand(0),
                                 DAG.getConstant(C2 + C3, DL, MVT::i32));
       return DAG.getNode(ISD::SRL, DL, MVT::i32, SHL,
@@ -10770,6 +10811,19 @@ static SDValue CombineANDShift(SDNode *N,
     }
   }
 
+  // Second pattern, reversed: right shift, then mask off trailing bits.
+  // FIXME: Handle other patterns of known/demanded bits.
+  if (!LeftShift && isShiftedMask_32(C1)) {
+    uint32_t Leading = countLeadingZeros(C1);
+    uint32_t C3 = countTrailingZeros(C1);
+    if (Leading == C2 && C2 + C3 < 32) {
+      SDValue SHL = DAG.getNode(ISD::SRL, DL, MVT::i32, N0->getOperand(0),
+                                DAG.getConstant(C2 + C3, DL, MVT::i32));
+      return DAG.getNode(ISD::SHL, DL, MVT::i32, SHL,
+                         DAG.getConstant(C3, DL, MVT::i32));
+    }
+  }
+
   // FIXME: Transform "(and (shl x, c2) c1)" ->
   // "(shl (and x, c1>>c2), c2)" if "c1 >> c2" is a cheaper immediate than
   // c1.
@@ -12381,8 +12435,10 @@ static SDValue PerformIntrinsicCombine(SDNode *N, SelectionDAG &DAG) {
 /// combining instead of DAG legalizing because the build_vectors for 64-bit
 /// vector element shift counts are generally not legal, and it is hard to see
 /// their values after they get legalized to loads from a constant pool.
-static SDValue PerformShiftCombine(SDNode *N, SelectionDAG &DAG,
+static SDValue PerformShiftCombine(SDNode *N,
+                                   TargetLowering::DAGCombinerInfo &DCI,
                                    const ARMSubtarget *ST) {
+  SelectionDAG &DAG = DCI.DAG;
   EVT VT = N->getValueType(0);
   if (N->getOpcode() == ISD::SRL && VT == MVT::i32 && ST->hasV6Ops()) {
     // Canonicalize (srl (bswap x), 16) to (rotr (bswap x), 16) if the high
@@ -12397,6 +12453,40 @@ static SDValue PerformShiftCombine(SDNode *N, SelectionDAG &DAG,
     }
   }
 
+  if (ST->isThumb1Only() && N->getOpcode() == ISD::SHL && VT == MVT::i32 &&
+      N->getOperand(0)->getOpcode() == ISD::AND &&
+      N->getOperand(0)->hasOneUse()) {
+    if (DCI.isBeforeLegalize() || DCI.isCalledByLegalizer())
+      return SDValue();
+    // Look for the pattern (shl (and x, AndMask), ShiftAmt). This doesn't
+    // usually show up because instcombine prefers to canonicalize it to
+    // (and (shl x, ShiftAmt) (shl AndMask, ShiftAmt)), but the shift can come
+    // out of GEP lowering in some cases.
+    SDValue N0 = N->getOperand(0);
+    ConstantSDNode *ShiftAmtNode = dyn_cast<ConstantSDNode>(N->getOperand(1));
+    if (!ShiftAmtNode)
+      return SDValue();
+    uint32_t ShiftAmt = static_cast<uint32_t>(ShiftAmtNode->getZExtValue());
+    ConstantSDNode *AndMaskNode = dyn_cast<ConstantSDNode>(N0->getOperand(1));
+    if (!AndMaskNode)
+      return SDValue();
+    uint32_t AndMask = static_cast<uint32_t>(AndMaskNode->getZExtValue());
+    // Don't transform uxtb/uxth.
+    if (AndMask == 255 || AndMask == 65535)
+      return SDValue();
+    if (isMask_32(AndMask)) {
+      uint32_t MaskedBits = countLeadingZeros(AndMask);
+      if (MaskedBits > ShiftAmt) {
+        SDLoc DL(N);
+        SDValue SHL = DAG.getNode(ISD::SHL, DL, MVT::i32, N0->getOperand(0),
+                                  DAG.getConstant(MaskedBits, DL, MVT::i32));
+        return DAG.getNode(
+            ISD::SRL, DL, MVT::i32, SHL,
+            DAG.getConstant(MaskedBits - ShiftAmt, DL, MVT::i32));
+      }
+    }
+  }
+
   // Nothing to be done for scalar shifts.
   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
   if (!VT.isVector() || !TLI.isTypeLegal(VT))
@@ -12537,8 +12627,7 @@ SDValue ARMTargetLowering::PerformCMOVToBFICombine(SDNode *CMOV, SelectionDAG &D
 
   // Lastly, can we determine that the bits defined by OrCI
   // are zero in Y?
-  KnownBits Known;
-  DAG.computeKnownBits(Y, Known);
+  KnownBits Known = DAG.computeKnownBits(Y);
   if ((OrCI & Known.Zero) != OrCI)
     return SDValue();
 
@@ -12768,8 +12857,7 @@ ARMTargetLowering::PerformCMOVCombine(SDNode *N, SelectionDAG &DAG) const {
   }
 
   if (Res.getNode()) {
-    KnownBits Known;
-    DAG.computeKnownBits(SDValue(N,0), Known);
+    KnownBits Known = DAG.computeKnownBits(SDValue(N,0));
     // Capture demanded bits information that would be otherwise lost.
     if (Known.Zero == 0xfffffffe)
       Res = DAG.getNode(ISD::AssertZext, dl, MVT::i32, Res,
@@ -12817,7 +12905,8 @@ SDValue ARMTargetLowering::PerformDAGCombine(SDNode *N,
   case ISD::INTRINSIC_WO_CHAIN: return PerformIntrinsicCombine(N, DCI.DAG);
   case ISD::SHL:
   case ISD::SRA:
-  case ISD::SRL:        return PerformShiftCombine(N, DCI.DAG, Subtarget);
+  case ISD::SRL:
+    return PerformShiftCombine(N, DCI, Subtarget);
   case ISD::SIGN_EXTEND:
   case ISD::ZERO_EXTEND:
   case ISD::ANY_EXTEND: return PerformExtendCombine(N, DCI.DAG, Subtarget);
@@ -13560,12 +13649,11 @@ void ARMTargetLowering::computeKnownBitsForTargetNode(const SDValue Op,
     break;
   case ARMISD::CMOV: {
     // Bits are known zero/one if known on the LHS and RHS.
-    DAG.computeKnownBits(Op.getOperand(0), Known, Depth+1);
+    Known = DAG.computeKnownBits(Op.getOperand(0), Depth+1);
     if (Known.isUnknown())
       return;
 
-    KnownBits KnownRHS;
-    DAG.computeKnownBits(Op.getOperand(1), KnownRHS, Depth+1);
+    KnownBits KnownRHS = DAG.computeKnownBits(Op.getOperand(1), Depth+1);
     Known.Zero &= KnownRHS.Zero;
     Known.One  &= KnownRHS.One;
     return;
@@ -13587,7 +13675,7 @@ void ARMTargetLowering::computeKnownBitsForTargetNode(const SDValue Op,
   case ARMISD::BFI: {
     // Conservatively, we can recurse down the first operand
     // and just mask out all affected bits.
-    DAG.computeKnownBits(Op.getOperand(0), Known, Depth + 1);
+    Known = DAG.computeKnownBits(Op.getOperand(0), Depth + 1);
 
     // The operand to BFI is already a mask suitable for removing the bits it
     // sets.
@@ -13597,6 +13685,33 @@ void ARMTargetLowering::computeKnownBitsForTargetNode(const SDValue Op,
     Known.One &= Mask;
     return;
   }
+  case ARMISD::VGETLANEs:
+  case ARMISD::VGETLANEu: {
+    const SDValue &SrcSV = Op.getOperand(0);
+    EVT VecVT = SrcSV.getValueType();
+    assert(VecVT.isVector() && "VGETLANE expected a vector type");
+    const unsigned NumSrcElts = VecVT.getVectorNumElements();
+    ConstantSDNode *Pos = cast<ConstantSDNode>(Op.getOperand(1).getNode());
+    assert(Pos->getAPIntValue().ult(NumSrcElts) &&
+           "VGETLANE index out of bounds");
+    unsigned Idx = Pos->getZExtValue();
+    APInt DemandedElt = APInt::getOneBitSet(NumSrcElts, Idx);
+    Known = DAG.computeKnownBits(SrcSV, DemandedElt, Depth + 1);
+
+    EVT VT = Op.getValueType();
+    const unsigned DstSz = VT.getScalarSizeInBits();
+    const unsigned SrcSz = VecVT.getVectorElementType().getSizeInBits();
+    assert(SrcSz == Known.getBitWidth());
+    assert(DstSz > SrcSz);
+    if (Op.getOpcode() == ARMISD::VGETLANEs)
+      Known = Known.sext(DstSz);
+    else {
+      Known = Known.zext(DstSz);
+      Known.Zero.setBitsFrom(SrcSz);
+    }
+    assert(DstSz == Known.getBitWidth());
+    break;
+  }
   }
 }
 
@@ -14527,6 +14642,9 @@ ARMTargetLowering::shouldExpandAtomicLoadInIR(LoadInst *LI) const {
 // and up to 64 bits on the non-M profiles
 TargetLowering::AtomicExpansionKind
 ARMTargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const {
+  if (AI->isFloatingPointOperation())
+    return AtomicExpansionKind::CmpXChg;
+
   unsigned Size = AI->getType()->getPrimitiveSizeInBits();
   bool hasAtomicRMW = !Subtarget->isThumb() || Subtarget->hasV8MBaselineOps();
   return (Size <= (Subtarget->isMClass() ? 32U : 64U) && hasAtomicRMW)
@@ -14595,6 +14713,10 @@ bool ARMTargetLowering::isCheapToSpeculateCtlz() const {
   return Subtarget->hasV6T2Ops();
 }
 
+bool ARMTargetLowering::shouldExpandShift(SelectionDAG &DAG, SDNode *N) const {
+  return !Subtarget->optForMinSize();
+}
+
 Value *ARMTargetLowering::emitLoadLinked(IRBuilder<> &Builder, Value *Addr,
                                          AtomicOrdering Ord) const {
   Module *M = Builder.GetInsertBlock()->getParent()->getParent();
@@ -14787,8 +14909,9 @@ bool ARMTargetLowering::lowerInterleavedLoad(
     // If we're generating more than one load, compute the base address of
     // subsequent loads as an offset from the previous.
     if (LoadCount > 0)
-      BaseAddr = Builder.CreateConstGEP1_32(
-          BaseAddr, VecTy->getVectorNumElements() * Factor);
+      BaseAddr =
+          Builder.CreateConstGEP1_32(VecTy->getVectorElementType(), BaseAddr,
+                                     VecTy->getVectorNumElements() * Factor);
 
     SmallVector<Value *, 2> Ops;
     Ops.push_back(Builder.CreateBitCast(BaseAddr, Int8Ptr));
@@ -14927,7 +15050,8 @@ bool ARMTargetLowering::lowerInterleavedStore(StoreInst *SI,
     // If we generating more than one store, we compute the base address of
     // subsequent stores as an offset from the previous.
     if (StoreCount > 0)
-      BaseAddr = Builder.CreateConstGEP1_32(BaseAddr, LaneLen * Factor);
+      BaseAddr = Builder.CreateConstGEP1_32(SubVecTy->getVectorElementType(),
+                                            BaseAddr, LaneLen * Factor);
 
     SmallVector<Value *, 6> Ops;
     Ops.push_back(Builder.CreateBitCast(BaseAddr, Int8Ptr));
diff --git a/lib/Target/ARM/ARMISelLowering.h b/lib/Target/ARM/ARMISelLowering.h
index ccee7fa258ab4978bc7b3f8f4b4a5f3d2bd848b4..992eaa5309810b5e1b4fede89619e15e650ffef1 100644
--- a/lib/Target/ARM/ARMISelLowering.h
+++ b/lib/Target/ARM/ARMISelLowering.h
@@ -1,9 +1,8 @@
 //===- ARMISelLowering.h - ARM DAG Lowering Interface -----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -568,6 +567,8 @@ class VectorType;
       return HasStandaloneRem;
     }
 
+    bool shouldExpandShift(SelectionDAG &DAG, SDNode *N) const override;
+
     CCAssignFn *CCAssignFnForCall(CallingConv::ID CC, bool isVarArg) const;
     CCAssignFn *CCAssignFnForReturn(CallingConv::ID CC, bool isVarArg) const;
 
@@ -593,6 +594,8 @@ class VectorType;
     bool isDesirableToCommuteWithShift(const SDNode *N,
                                        CombineLevel Level) const override;
 
+    bool shouldFoldShiftPairToMask(const SDNode *N,
+                                   CombineLevel Level) const override;
   protected:
     std::pair<const TargetRegisterClass *, uint8_t>
     findRepresentativeClass(const TargetRegisterInfo *TRI,
diff --git a/lib/Target/ARM/ARMInstrFormats.td b/lib/Target/ARM/ARMInstrFormats.td
index 0df48ba612993aa56e3398f0713c647bbb2128ca..87764391171aece2838b2ec50323239f71e2bc46 100644
--- a/lib/Target/ARM/ARMInstrFormats.td
+++ b/lib/Target/ARM/ARMInstrFormats.td
@@ -1,9 +1,8 @@
 //===-- ARMInstrFormats.td - ARM Instruction Formats -------*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/ARM/ARMInstrInfo.cpp b/lib/Target/ARM/ARMInstrInfo.cpp
index bcc31f5fa4cc705d204b21825da313c4d578e7d9..388c889349b7012cddca4a9a138c6735da9fff18 100644
--- a/lib/Target/ARM/ARMInstrInfo.cpp
+++ b/lib/Target/ARM/ARMInstrInfo.cpp
@@ -1,9 +1,8 @@
 //===-- ARMInstrInfo.cpp - ARM Instruction Information --------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -95,7 +94,7 @@ void ARMInstrInfo::expandLoadStackGuard(MachineBasicBlock::iterator MI) const {
   const ARMSubtarget &Subtarget = MF.getSubtarget<ARMSubtarget>();
   const TargetMachine &TM = MF.getTarget();
 
-  if (!Subtarget.useMovt(MF)) {
+  if (!Subtarget.useMovt()) {
     if (TM.isPositionIndependent())
       expandLoadStackGuardBase(MI, ARM::LDRLIT_ga_pcrel, ARM::LDRi12);
     else
diff --git a/lib/Target/ARM/ARMInstrInfo.h b/lib/Target/ARM/ARMInstrInfo.h
index c87fb97448c91358673adf0354a37b133bf4140e..042b53f0f8c310808114d0571ca60fa948894121 100644
--- a/lib/Target/ARM/ARMInstrInfo.h
+++ b/lib/Target/ARM/ARMInstrInfo.h
@@ -1,9 +1,8 @@
 //===-- ARMInstrInfo.h - ARM Instruction Information ------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/ARM/ARMInstrInfo.td b/lib/Target/ARM/ARMInstrInfo.td
index 488af7bf12c76a62724244eafb96784fde21aa81..6555590fc5d7269e2157477275edd4811c97a295 100644
--- a/lib/Target/ARM/ARMInstrInfo.td
+++ b/lib/Target/ARM/ARMInstrInfo.td
@@ -1,9 +1,8 @@
 //===- ARMInstrInfo.td - Target Description for ARM Target -*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -355,14 +354,14 @@ def UseNegativeImmediates :
 
 // FIXME: Eventually this will be just "hasV6T2Ops".
 let RecomputePerFunction = 1 in {
-  def UseMovt          : Predicate<"Subtarget->useMovt(*MF)">;
-  def DontUseMovt      : Predicate<"!Subtarget->useMovt(*MF)">;
-  def UseMovtInPic     : Predicate<"Subtarget->useMovt(*MF) && Subtarget->allowPositionIndependentMovt()">;
-  def DontUseMovtInPic : Predicate<"!Subtarget->useMovt(*MF) || !Subtarget->allowPositionIndependentMovt()">;
+  def UseMovt          : Predicate<"Subtarget->useMovt()">;
+  def DontUseMovt      : Predicate<"!Subtarget->useMovt()">;
+  def UseMovtInPic     : Predicate<"Subtarget->useMovt() && Subtarget->allowPositionIndependentMovt()">;
+  def DontUseMovtInPic : Predicate<"!Subtarget->useMovt() || !Subtarget->allowPositionIndependentMovt()">;
 
   def UseFPVMLx: Predicate<"((Subtarget->useFPVMLx() &&"
                            "  TM.Options.AllowFPOpFusion != FPOpFusion::Fast) ||"
-                           "MF->getFunction().optForMinSize())">;
+                           "Subtarget->optForMinSize())">;
 }
 def UseMulOps        : Predicate<"Subtarget->useMulOps()">;
 
@@ -395,8 +394,8 @@ let RecomputePerFunction = 1 in {
 def GenExecuteOnly : Predicate<"Subtarget->genExecuteOnly()">;
 
 // Armv8.5-A extensions
-def HasSpecCtrl      : Predicate<"Subtarget->hasSpecCtrl()">,
-                       AssemblerPredicate<"FeatureSpecCtrl", "specctrl">;
+def HasSB            : Predicate<"Subtarget->hasSB()">,
+                       AssemblerPredicate<"FeatureSB", "sb">;
 
 //===----------------------------------------------------------------------===//
 // ARM Flag Definitions.
@@ -426,24 +425,22 @@ def imm16_31 : ImmLeaf<i32, [{
 
 // sext_16_node predicate - True if the SDNode is sign-extended 16 or more bits.
 def sext_16_node : PatLeaf<(i32 GPR:$a), [{
-  if (CurDAG->ComputeNumSignBits(SDValue(N,0)) >= 17)
-    return true;
-
-  if (N->getOpcode() != ISD::SRA)
-    return false;
-  if (N->getOperand(0).getOpcode() != ISD::SHL)
-    return false;
-
-  auto *ShiftVal = dyn_cast<ConstantSDNode>(N->getOperand(1));
-  if (!ShiftVal || ShiftVal->getZExtValue() != 16)
-    return false;
+  return CurDAG->ComputeNumSignBits(SDValue(N,0)) >= 17;
+}]>;
 
-  ShiftVal = dyn_cast<ConstantSDNode>(N->getOperand(0)->getOperand(1));
-  if (!ShiftVal || ShiftVal->getZExtValue() != 16)
-    return false;
+def sext_bottom_16 : PatFrag<(ops node:$a),
+                             (sext_inreg node:$a, i16)>;
+def sext_top_16 : PatFrag<(ops node:$a),
+                          (i32 (sra node:$a, (i32 16)))>;
 
-  return true;
-}]>;
+def bb_mul : PatFrag<(ops node:$a, node:$b),
+                     (mul (sext_bottom_16 node:$a), (sext_bottom_16 node:$b))>;
+def bt_mul : PatFrag<(ops node:$a, node:$b),
+                     (mul (sext_bottom_16 node:$a), (sra node:$b, (i32 16)))>;
+def tb_mul : PatFrag<(ops node:$a, node:$b),
+                     (mul (sra node:$a, (i32 16)), (sext_bottom_16 node:$b))>;
+def tt_mul : PatFrag<(ops node:$a, node:$b),
+                     (mul (sra node:$a, (i32 16)), (sra node:$b, (i32 16)))>;
 
 /// Split a 32-bit immediate into two 16 bit parts.
 def hi16 : SDNodeXForm<imm, [{
@@ -721,10 +718,22 @@ def mod_imm_neg : Operand<i32>, PatLeaf<(imm), [{
 
 /// arm_i32imm - True for +V6T2, or when isSOImmTwoParVal()
 def arm_i32imm : PatLeaf<(imm), [{
-  if (Subtarget->useMovt(*MF))
+  if (Subtarget->useMovt())
     return true;
   return ARM_AM::isSOImmTwoPartVal((unsigned)N->getZExtValue());
-}]>;
+}]> {
+  // Ideally this would be an IntImmLeaf, but then we wouldn't have access to
+  // the MachineFunction.
+  let GISelPredicateCode = [{
+    if (STI.useMovt())
+      return true;
+
+    const auto &MO = MI.getOperand(1);
+    if (!MO.isCImm())
+      return false;
+    return ARM_AM::isSOImmTwoPartVal(MO.getCImm()->getZExtValue());
+  }];
+}
 
 /// imm0_1 predicate - Immediate in the range [0,1].
 def Imm0_1AsmOperand: ImmAsmOperand<0,1> { let Name = "Imm0_1"; }
@@ -4241,29 +4250,25 @@ def SMMLSR : AMul2Ia <0b0111010, 0b1111, (outs GPR:$Rd),
 multiclass AI_smul<string opc> {
   def BB : AMulxyI<0b0001011, 0b00, (outs GPR:$Rd), (ins GPR:$Rn, GPR:$Rm),
               IIC_iMUL16, !strconcat(opc, "bb"), "\t$Rd, $Rn, $Rm",
-              [(set GPR:$Rd, (mul (sext_inreg GPR:$Rn, i16),
-                                      (sext_inreg GPR:$Rm, i16)))]>,
+              [(set GPR:$Rd, (bb_mul GPR:$Rn, GPR:$Rm))]>,
            Requires<[IsARM, HasV5TE]>,
            Sched<[WriteMUL16, ReadMUL, ReadMUL]>;
 
   def BT : AMulxyI<0b0001011, 0b10, (outs GPR:$Rd), (ins GPR:$Rn, GPR:$Rm),
               IIC_iMUL16, !strconcat(opc, "bt"), "\t$Rd, $Rn, $Rm",
-              [(set GPR:$Rd, (mul (sext_inreg GPR:$Rn, i16),
-                                      (sra GPR:$Rm, (i32 16))))]>,
+              [(set GPR:$Rd, (bt_mul GPR:$Rn, GPR:$Rm))]>,
            Requires<[IsARM, HasV5TE]>,
            Sched<[WriteMUL16, ReadMUL, ReadMUL]>;
 
   def TB : AMulxyI<0b0001011, 0b01, (outs GPR:$Rd), (ins GPR:$Rn, GPR:$Rm),
               IIC_iMUL16, !strconcat(opc, "tb"), "\t$Rd, $Rn, $Rm",
-              [(set GPR:$Rd, (mul (sra GPR:$Rn, (i32 16)),
-                                      (sext_inreg GPR:$Rm, i16)))]>,
+              [(set GPR:$Rd, (tb_mul GPR:$Rn, GPR:$Rm))]>,
            Requires<[IsARM, HasV5TE]>,
            Sched<[WriteMUL16, ReadMUL, ReadMUL]>;
 
   def TT : AMulxyI<0b0001011, 0b11, (outs GPR:$Rd), (ins GPR:$Rn, GPR:$Rm),
               IIC_iMUL16, !strconcat(opc, "tt"), "\t$Rd, $Rn, $Rm",
-              [(set GPR:$Rd, (mul (sra GPR:$Rn, (i32 16)),
-                                      (sra GPR:$Rm, (i32 16))))]>,
+              [(set GPR:$Rd, (tt_mul GPR:$Rn, GPR:$Rm))]>,
             Requires<[IsARM, HasV5TE]>,
            Sched<[WriteMUL16, ReadMUL, ReadMUL]>;
 
@@ -4287,35 +4292,31 @@ multiclass AI_smla<string opc> {
               (ins GPRnopc:$Rn, GPRnopc:$Rm, GPR:$Ra),
               IIC_iMAC16, !strconcat(opc, "bb"), "\t$Rd, $Rn, $Rm, $Ra",
               [(set GPRnopc:$Rd, (add GPR:$Ra,
-                               (mul (sext_inreg GPRnopc:$Rn, i16),
-                                       (sext_inreg GPRnopc:$Rm, i16))))]>,
+                                      (bb_mul GPRnopc:$Rn, GPRnopc:$Rm)))]>,
            Requires<[IsARM, HasV5TE, UseMulOps]>,
            Sched<[WriteMAC16, ReadMUL, ReadMUL, ReadMAC]>;
 
   def BT : AMulxyIa<0b0001000, 0b10, (outs GPRnopc:$Rd),
               (ins GPRnopc:$Rn, GPRnopc:$Rm, GPR:$Ra),
               IIC_iMAC16, !strconcat(opc, "bt"), "\t$Rd, $Rn, $Rm, $Ra",
-              [(set GPRnopc:$Rd,
-                    (add GPR:$Ra, (mul (sext_inreg GPRnopc:$Rn, i16),
-                                          (sra GPRnopc:$Rm, (i32 16)))))]>,
+              [(set GPRnopc:$Rd, (add GPR:$Ra,
+                                      (bt_mul GPRnopc:$Rn, GPRnopc:$Rm)))]>,
            Requires<[IsARM, HasV5TE, UseMulOps]>,
            Sched<[WriteMAC16, ReadMUL, ReadMUL, ReadMAC]>;
 
   def TB : AMulxyIa<0b0001000, 0b01, (outs GPRnopc:$Rd),
               (ins GPRnopc:$Rn, GPRnopc:$Rm, GPR:$Ra),
               IIC_iMAC16, !strconcat(opc, "tb"), "\t$Rd, $Rn, $Rm, $Ra",
-              [(set GPRnopc:$Rd,
-                    (add GPR:$Ra, (mul (sra GPRnopc:$Rn, (i32 16)),
-                                          (sext_inreg GPRnopc:$Rm, i16))))]>,
+              [(set GPRnopc:$Rd, (add GPR:$Ra,
+                                      (tb_mul GPRnopc:$Rn, GPRnopc:$Rm)))]>,
            Requires<[IsARM, HasV5TE, UseMulOps]>,
            Sched<[WriteMAC16, ReadMUL, ReadMUL, ReadMAC]>;
 
   def TT : AMulxyIa<0b0001000, 0b11, (outs GPRnopc:$Rd),
               (ins GPRnopc:$Rn, GPRnopc:$Rm, GPR:$Ra),
               IIC_iMAC16, !strconcat(opc, "tt"), "\t$Rd, $Rn, $Rm, $Ra",
-             [(set GPRnopc:$Rd,
-                   (add GPR:$Ra, (mul (sra GPRnopc:$Rn, (i32 16)),
-                                         (sra GPRnopc:$Rm, (i32 16)))))]>,
+             [(set GPRnopc:$Rd, (add GPR:$Ra,
+                                     (tt_mul GPRnopc:$Rn, GPRnopc:$Rm)))]>,
             Requires<[IsARM, HasV5TE, UseMulOps]>,
             Sched<[WriteMAC16, ReadMUL, ReadMUL, ReadMAC]>;
 
@@ -4895,7 +4896,7 @@ def TSB : AInoP<(outs), (ins tsb_opt:$opt), MiscFrm, NoItinerary,
 
 // Armv8.5-A speculation barrier
 def SB : AInoP<(outs), (ins), MiscFrm, NoItinerary, "sb", "", []>,
-         Requires<[IsARM, HasSpecCtrl]>, Sched<[]> {
+         Requires<[IsARM, HasSB]>, Sched<[]> {
   let Inst{31-0} = 0xf57ff070;
   let Unpredictable = 0x000fff0f;
   let hasSideEffects = 1;
@@ -4908,7 +4909,7 @@ let usesCustomInserter = 1, Defs = [CPSR] in {
   def ABS : ARMPseudoInst<(outs GPR:$dst), (ins GPR:$src), 8, NoItinerary, []>;
 }
 
-let usesCustomInserter = 1 in {
+let usesCustomInserter = 1, Defs = [CPSR] in {
     def COPY_STRUCT_BYVAL_I32 : PseudoInst<
       (outs), (ins GPR:$dst, GPR:$src, i32imm:$size, i32imm:$alignment),
       NoItinerary,
@@ -5816,26 +5817,21 @@ def : ARMPat<(extloadi16 addrmodepc:$addr), (PICLDRH addrmodepc:$addr)>;
 
 // smul* and smla*
 def : ARMV5TEPat<(mul sext_16_node:$a, sext_16_node:$b),
-                 (SMULBB GPR:$a, GPR:$b)>,
-      Sched<[WriteMUL32, ReadMUL, ReadMUL]>;
-def : ARMV5TEPat<(mul sext_16_node:$a, (sra GPR:$b, (i32 16))),
-                 (SMULBT GPR:$a, GPR:$b)>,
-      Sched<[WriteMUL32, ReadMUL, ReadMUL]>;
-def : ARMV5TEPat<(mul (sra GPR:$a, (i32 16)), sext_16_node:$b),
-                (SMULTB GPR:$a, GPR:$b)>,
-      Sched<[WriteMUL32, ReadMUL, ReadMUL]>;
-def : ARMV5MOPat<(add GPR:$acc,
-                      (mul sext_16_node:$a, sext_16_node:$b)),
-                 (SMLABB GPR:$a, GPR:$b, GPR:$acc)>,
-      Sched<[WriteMUL32, ReadMUL, ReadMUL]>;
-def : ARMV5MOPat<(add GPR:$acc,
-                      (mul sext_16_node:$a, (sra GPR:$b, (i32 16)))),
-                 (SMLABT GPR:$a, GPR:$b, GPR:$acc)>,
-      Sched<[WriteMUL32, ReadMUL, ReadMUL]>;
-def : ARMV5MOPat<(add GPR:$acc,
-                      (mul (sra GPR:$a, (i32 16)), sext_16_node:$b)),
-                 (SMLATB GPR:$a, GPR:$b, GPR:$acc)>,
-      Sched<[WriteMUL32, ReadMUL, ReadMUL]>;
+                 (SMULBB GPR:$a, GPR:$b)>;
+def : ARMV5TEPat<(mul sext_16_node:$a, (sext_bottom_16 GPR:$b)),
+                 (SMULBB GPR:$a, GPR:$b)>;
+def : ARMV5TEPat<(mul sext_16_node:$a, (sext_top_16 GPR:$b)),
+                 (SMULBT GPR:$a, GPR:$b)>;
+def : ARMV5TEPat<(mul (sext_top_16 GPR:$a), sext_16_node:$b),
+                 (SMULTB GPR:$a, GPR:$b)>;
+def : ARMV5MOPat<(add GPR:$acc, (mul sext_16_node:$a, sext_16_node:$b)),
+                 (SMLABB GPR:$a, GPR:$b, GPR:$acc)>;
+def : ARMV5MOPat<(add GPR:$acc, (mul sext_16_node:$a, (sext_bottom_16 GPR:$b))),
+                 (SMLABB GPR:$a, GPR:$b, GPR:$acc)>;
+def : ARMV5MOPat<(add GPR:$acc, (mul sext_16_node:$a, (sext_top_16 GPR:$b))),
+                 (SMLABT GPR:$a, GPR:$b, GPR:$acc)>;
+def : ARMV5MOPat<(add GPR:$acc, (mul (sext_top_16 GPR:$a), sext_16_node:$b)),
+                 (SMLATB GPR:$a, GPR:$b, GPR:$acc)>;
 
 def : ARMV5TEPat<(int_arm_smulbb GPR:$a, GPR:$b),
                  (SMULBB GPR:$a, GPR:$b)>;
diff --git a/lib/Target/ARM/ARMInstrNEON.td b/lib/Target/ARM/ARMInstrNEON.td
index 96986e74415b1f91cb05dbfe9ca14c1d5b39a6e6..90aa574cbde2b35fa84ddbc1cdef4876a6ed57c2 100644
--- a/lib/Target/ARM/ARMInstrNEON.td
+++ b/lib/Target/ARM/ARMInstrNEON.td
@@ -1,9 +1,8 @@
 //===-- ARMInstrNEON.td - NEON support for ARM -------------*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/ARM/ARMInstrThumb.td b/lib/Target/ARM/ARMInstrThumb.td
index b20b34eaa6a90b28ca3ca7fc2641f656b96c05c7..1035c4b8c48b17eb3dcc3b810d796f3fe4f8df82 100644
--- a/lib/Target/ARM/ARMInstrThumb.td
+++ b/lib/Target/ARM/ARMInstrThumb.td
@@ -1,9 +1,8 @@
 //===-- ARMInstrThumb.td - Thumb support for ARM -----------*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -188,6 +187,19 @@ def t_addrmode_rr : MemOperand,
   let MIOperandInfo = (ops tGPR:$base, tGPR:$offsreg);
 }
 
+// t_addrmode_rr_sext := reg + reg
+//
+// This is similar to t_addrmode_rr, but uses different heuristics for
+// ldrsb/ldrsh.
+def t_addrmode_rr_sext : MemOperand,
+                    ComplexPattern<i32, 2, "SelectThumbAddrModeRRSext", []> {
+  let EncoderMethod = "getThumbAddrModeRegRegOpValue";
+  let PrintMethod = "printThumbAddrModeRROperand";
+  let DecoderMethod = "DecodeThumbAddrModeRR";
+  let ParserMatchClass = t_addrmode_rr_asm_operand;
+  let MIOperandInfo = (ops tGPR:$base, tGPR:$offsreg);
+}
+
 // t_addrmode_rrs := reg + reg
 //
 // We use separate scaled versions because the Select* functions need
@@ -732,17 +744,17 @@ defm tLDRH : thumb_ld_rr_ri_enc<0b101, 0b1000, t_addrmode_rr,
 
 let AddedComplexity = 10 in
 def tLDRSB :                    // A8.6.80
-  T1pILdStEncode<0b011, (outs tGPR:$Rt), (ins t_addrmode_rr:$addr),
+  T1pILdStEncode<0b011, (outs tGPR:$Rt), (ins t_addrmode_rr_sext:$addr),
                  AddrModeT1_1, IIC_iLoad_bh_r,
                  "ldrsb", "\t$Rt, $addr",
-                 [(set tGPR:$Rt, (sextloadi8 t_addrmode_rr:$addr))]>;
+                 [(set tGPR:$Rt, (sextloadi8 t_addrmode_rr_sext:$addr))]>;
 
 let AddedComplexity = 10 in
 def tLDRSH :                    // A8.6.84
-  T1pILdStEncode<0b111, (outs tGPR:$Rt), (ins t_addrmode_rr:$addr),
+  T1pILdStEncode<0b111, (outs tGPR:$Rt), (ins t_addrmode_rr_sext:$addr),
                  AddrModeT1_2, IIC_iLoad_bh_r,
                  "ldrsh", "\t$Rt, $addr",
-                 [(set tGPR:$Rt, (sextloadi16 t_addrmode_rr:$addr))]>;
+                 [(set tGPR:$Rt, (sextloadi16 t_addrmode_rr_sext:$addr))]>;
 
 
 def tSTRspi : T1pIs<(outs), (ins tGPR:$Rt, t_addrmode_sp:$addr), IIC_iStore_i,
diff --git a/lib/Target/ARM/ARMInstrThumb2.td b/lib/Target/ARM/ARMInstrThumb2.td
index 4130dbd884572d341b80177bc0c67058b835b6a2..a38980cc7964948bb102030056d6ac1dffba3ff8 100644
--- a/lib/Target/ARM/ARMInstrThumb2.td
+++ b/lib/Target/ARM/ARMInstrThumb2.td
@@ -1,9 +1,8 @@
 //===-- ARMInstrThumb2.td - Thumb2 support for ARM ---------*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -2148,6 +2147,11 @@ def : T2Pat<(add        GPR:$src, imm0_4095_neg:$imm),
 def : T2Pat<(add        GPR:$src, imm0_65535_neg:$imm),
             (t2SUBrr    GPR:$src, (t2MOVi16 (imm_neg_XFORM imm:$imm)))>;
 
+// Do the same for v8m targets since they support movw with a 16-bit value.
+def : T1Pat<(add tGPR:$src, imm0_65535_neg:$imm),
+             (tSUBrr tGPR:$src, (t2MOVi16 (imm_neg_XFORM imm:$imm)))>,
+             Requires<[HasV8MBaseline]>;
+
 let AddedComplexity = 1 in
 def : T2Pat<(ARMaddc    rGPR:$src, imm1_255_neg:$imm),
             (t2SUBSri   rGPR:$src, imm1_255_neg:$imm)>;
@@ -2732,28 +2736,25 @@ class T2ThreeRegSMUL<bits<3> op22_20, bits<2> op5_4, string opc,
 }
 
 def t2SMULBB : T2ThreeRegSMUL<0b001, 0b00, "smulbb",
-             [(set rGPR:$Rd, (mul (sext_inreg rGPR:$Rn, i16),
-                                   (sext_inreg rGPR:$Rm, i16)))]>;
+             [(set rGPR:$Rd, (bb_mul rGPR:$Rn, rGPR:$Rm))]>;
 def t2SMULBT : T2ThreeRegSMUL<0b001, 0b01, "smulbt",
-             [(set rGPR:$Rd, (mul (sext_inreg rGPR:$Rn, i16),
-                                   (sra rGPR:$Rm, (i32 16))))]>;
+             [(set rGPR:$Rd, (bt_mul rGPR:$Rn, rGPR:$Rm))]>;
 def t2SMULTB : T2ThreeRegSMUL<0b001, 0b10, "smultb",
-             [(set rGPR:$Rd, (mul (sra rGPR:$Rn, (i32 16)),
-                                   (sext_inreg rGPR:$Rm, i16)))]>;
+             [(set rGPR:$Rd, (tb_mul rGPR:$Rn, rGPR:$Rm))]>;
 def t2SMULTT : T2ThreeRegSMUL<0b001, 0b11, "smultt",
-             [(set rGPR:$Rd, (mul (sra rGPR:$Rn, (i32 16)),
-                                   (sra rGPR:$Rm, (i32 16))))]>;
+             [(set rGPR:$Rd, (tt_mul rGPR:$Rn, rGPR:$Rm))]>;
 def t2SMULWB : T2ThreeRegSMUL<0b011, 0b00, "smulwb",
              [(set rGPR:$Rd, (ARMsmulwb rGPR:$Rn, rGPR:$Rm))]>;
 def t2SMULWT : T2ThreeRegSMUL<0b011, 0b01, "smulwt",
              [(set rGPR:$Rd, (ARMsmulwt rGPR:$Rn, rGPR:$Rm))]>;
 
-def : Thumb2DSPPat<(mul sext_16_node:$Rm, sext_16_node:$Rn),
-                   (t2SMULBB rGPR:$Rm, rGPR:$Rn)>;
-def : Thumb2DSPPat<(mul sext_16_node:$Rn, (sra rGPR:$Rm, (i32 16))),
+def : Thumb2DSPPat<(mul sext_16_node:$Rn, (sext_bottom_16 rGPR:$Rm)),
+                   (t2SMULBB rGPR:$Rn, rGPR:$Rm)>;
+def : Thumb2DSPPat<(mul sext_16_node:$Rn, (sext_top_16 rGPR:$Rm)),
                    (t2SMULBT rGPR:$Rn, rGPR:$Rm)>;
-def : Thumb2DSPPat<(mul (sra rGPR:$Rn, (i32 16)), sext_16_node:$Rm),
+def : Thumb2DSPPat<(mul (sext_top_16 rGPR:$Rn), sext_16_node:$Rm),
                    (t2SMULTB rGPR:$Rn, rGPR:$Rm)>;
+
 def : Thumb2DSPPat<(int_arm_smulbb rGPR:$Rn, rGPR:$Rm),
                    (t2SMULBB rGPR:$Rn, rGPR:$Rm)>;
 def : Thumb2DSPPat<(int_arm_smulbt rGPR:$Rn, rGPR:$Rm),
@@ -2781,18 +2782,13 @@ class T2FourRegSMLA<bits<3> op22_20, bits<2> op5_4, string opc,
 }
 
 def t2SMLABB : T2FourRegSMLA<0b001, 0b00, "smlabb",
-             [(set rGPR:$Rd, (add rGPR:$Ra,
-                               (mul (sext_inreg rGPR:$Rn, i16),
-                                     (sext_inreg rGPR:$Rm, i16))))]>;
+             [(set rGPR:$Rd, (add rGPR:$Ra, (bb_mul rGPR:$Rn, rGPR:$Rm)))]>;
 def t2SMLABT : T2FourRegSMLA<0b001, 0b01, "smlabt",
-             [(set rGPR:$Rd, (add rGPR:$Ra, (mul (sext_inreg rGPR:$Rn, i16),
-                                                 (sra rGPR:$Rm, (i32 16)))))]>;
+             [(set rGPR:$Rd, (add rGPR:$Ra, (bt_mul rGPR:$Rn, rGPR:$Rm)))]>;
 def t2SMLATB : T2FourRegSMLA<0b001, 0b10, "smlatb",
-             [(set rGPR:$Rd, (add rGPR:$Ra, (mul (sra rGPR:$Rn, (i32 16)),
-                                                (sext_inreg rGPR:$Rm, i16))))]>;
+             [(set rGPR:$Rd, (add rGPR:$Ra, (tb_mul rGPR:$Rn, rGPR:$Rm)))]>;
 def t2SMLATT : T2FourRegSMLA<0b001, 0b11, "smlatt",
-             [(set rGPR:$Rd, (add rGPR:$Ra, (mul (sra rGPR:$Rn, (i32 16)),
-                                                 (sra rGPR:$Rm, (i32 16)))))]>;
+             [(set rGPR:$Rd, (add rGPR:$Ra, (tt_mul rGPR:$Rn, rGPR:$Rm)))]>;
 def t2SMLAWB : T2FourRegSMLA<0b011, 0b00, "smlawb",
              [(set rGPR:$Rd, (add rGPR:$Ra, (ARMsmulwb rGPR:$Rn, rGPR:$Rm)))]>;
 def t2SMLAWT : T2FourRegSMLA<0b011, 0b01, "smlawt",
@@ -2800,11 +2796,14 @@ def t2SMLAWT : T2FourRegSMLA<0b011, 0b01, "smlawt",
 
 def : Thumb2DSPMulPat<(add rGPR:$Ra, (mul sext_16_node:$Rn, sext_16_node:$Rm)),
                       (t2SMLABB rGPR:$Rn, rGPR:$Rm, rGPR:$Ra)>;
-def : Thumb2DSPMulPat<(add rGPR:$Ra,
-                        (mul sext_16_node:$Rn, (sra rGPR:$Rm, (i32 16)))),
+def : Thumb2DSPMulPat<(add rGPR:$Ra, (mul sext_16_node:$Rn, 
+                                          (sext_bottom_16 rGPR:$Rm))),
+                      (t2SMLABB rGPR:$Rn, rGPR:$Rm, rGPR:$Ra)>;
+def : Thumb2DSPMulPat<(add rGPR:$Ra, (mul sext_16_node:$Rn,
+                                          (sext_top_16 rGPR:$Rm))),
                       (t2SMLABT rGPR:$Rn, rGPR:$Rm, rGPR:$Ra)>;
-def : Thumb2DSPMulPat<(add rGPR:$Ra,
-                        (mul (sra rGPR:$Rn, (i32 16)), sext_16_node:$Rm)),
+def : Thumb2DSPMulPat<(add rGPR:$Ra, (mul (sext_top_16 rGPR:$Rn),
+                                          sext_16_node:$Rm)),
                       (t2SMLATB rGPR:$Rn, rGPR:$Rm, rGPR:$Ra)>;
 
 def : Thumb2DSPPat<(int_arm_smlabb GPR:$a, GPR:$b, GPR:$acc),
@@ -3239,7 +3238,7 @@ def t2TSB : T2I<(outs), (ins tsb_opt:$opt), NoItinerary,
 
 // Armv8.5-A speculation barrier
 def t2SB : Thumb2XI<(outs), (ins), AddrModeNone, 4, NoItinerary, "sb", "", []>,
-           Requires<[IsThumb2, HasSpecCtrl]>, Sched<[]> {
+           Requires<[IsThumb2, HasSB]>, Sched<[]> {
   let Inst{31-0} = 0xf3bf8f70;
   let Unpredictable = 0x000f2f0f;
   let hasSideEffects = 1;
diff --git a/lib/Target/ARM/ARMInstrVFP.td b/lib/Target/ARM/ARMInstrVFP.td
index b58730c452f741a06b41893ce9f3e5ee177e7ab8..8773c1606073b04b8e4d07183150d4504a0244b9 100644
--- a/lib/Target/ARM/ARMInstrVFP.td
+++ b/lib/Target/ARM/ARMInstrVFP.td
@@ -1,9 +1,8 @@
 //===-- ARMInstrVFP.td - VFP support for ARM ---------------*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/ARM/ARMInstructionSelector.cpp b/lib/Target/ARM/ARMInstructionSelector.cpp
index 293e734c97cd6a7c600077151d46816515d9cf9d..262300ebb625fce1ed50ce5118bb3b217dad6824 100644
--- a/lib/Target/ARM/ARMInstructionSelector.cpp
+++ b/lib/Target/ARM/ARMInstructionSelector.cpp
@@ -1,9 +1,8 @@
 //===- ARMInstructionSelector.cpp ----------------------------*- C++ -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
@@ -98,6 +97,15 @@ private:
     unsigned STORE8;
     unsigned LOAD8;
 
+    // Used for G_ICMP
+    unsigned CMPrr;
+    unsigned MOVi;
+    unsigned MOVCCi;
+
+    // Used for G_SELECT
+    unsigned CMPri;
+    unsigned MOVCCr;
+
     OpcodeCache(const ARMSubtarget &STI);
   } const Opcodes;
 
@@ -285,6 +293,13 @@ ARMInstructionSelector::OpcodeCache::OpcodeCache(const ARMSubtarget &STI) {
 
   STORE_OPCODE(STORE8, STRBi12);
   STORE_OPCODE(LOAD8, LDRBi12);
+
+  STORE_OPCODE(CMPrr, CMPrr);
+  STORE_OPCODE(MOVi, MOVi);
+  STORE_OPCODE(MOVCCi, MOVCCi);
+
+  STORE_OPCODE(CMPri, CMPri);
+  STORE_OPCODE(MOVCCr, MOVCCr);
 #undef MAP_OPCODE
 }
 
@@ -408,10 +423,11 @@ getComparePreds(CmpInst::Predicate Pred) {
 }
 
 struct ARMInstructionSelector::CmpConstants {
-  CmpConstants(unsigned CmpOpcode, unsigned FlagsOpcode, unsigned OpRegBank,
-               unsigned OpSize)
+  CmpConstants(unsigned CmpOpcode, unsigned FlagsOpcode, unsigned SelectOpcode,
+               unsigned OpRegBank, unsigned OpSize)
       : ComparisonOpcode(CmpOpcode), ReadFlagsOpcode(FlagsOpcode),
-        OperandRegBankID(OpRegBank), OperandSize(OpSize) {}
+        SelectResultOpcode(SelectOpcode), OperandRegBankID(OpRegBank),
+        OperandSize(OpSize) {}
 
   // The opcode used for performing the comparison.
   const unsigned ComparisonOpcode;
@@ -420,6 +436,9 @@ struct ARMInstructionSelector::CmpConstants {
   // ARM::INSTRUCTION_LIST_END if we don't need to read the flags.
   const unsigned ReadFlagsOpcode;
 
+  // The opcode used for materializing the result of the comparison.
+  const unsigned SelectResultOpcode;
+
   // The assumed register bank ID for the operands.
   const unsigned OperandRegBankID;
 
@@ -439,7 +458,7 @@ struct ARMInstructionSelector::InsertInfo {
 
 void ARMInstructionSelector::putConstant(InsertInfo I, unsigned DestReg,
                                          unsigned Constant) const {
-  (void)BuildMI(I.MBB, I.InsertBefore, I.DbgLoc, TII.get(ARM::MOVi))
+  (void)BuildMI(I.MBB, I.InsertBefore, I.DbgLoc, TII.get(Opcodes.MOVi))
       .addDef(DestReg)
       .addImm(Constant)
       .add(predOps(ARMCC::AL))
@@ -542,7 +561,8 @@ bool ARMInstructionSelector::insertComparison(CmpConstants Helper, InsertInfo I,
   }
 
   // Select either 1 or the previous result based on the value of the flags.
-  auto Mov1I = BuildMI(I.MBB, I.InsertBefore, I.DbgLoc, TII.get(ARM::MOVCCi))
+  auto Mov1I = BuildMI(I.MBB, I.InsertBefore, I.DbgLoc,
+                       TII.get(Helper.SelectResultOpcode))
                    .addDef(ResReg)
                    .addUse(PrevRes)
                    .addImm(1)
@@ -569,7 +589,7 @@ bool ARMInstructionSelector::selectGlobal(MachineInstrBuilder &MIB,
   auto &MBB = *MIB->getParent();
   auto &MF = *MBB.getParent();
 
-  bool UseMovt = STI.useMovt(MF);
+  bool UseMovt = STI.useMovt();
 
   unsigned Size = TM.getPointerSize(0);
   unsigned Alignment = 4;
@@ -684,7 +704,7 @@ bool ARMInstructionSelector::selectSelect(MachineInstrBuilder &MIB,
   auto CondReg = MIB->getOperand(1).getReg();
   assert(validReg(MRI, CondReg, 1, ARM::GPRRegBankID) &&
          "Unsupported types for select operation");
-  auto CmpI = BuildMI(MBB, InsertBefore, DbgLoc, TII.get(ARM::CMPri))
+  auto CmpI = BuildMI(MBB, InsertBefore, DbgLoc, TII.get(Opcodes.CMPri))
                   .addUse(CondReg)
                   .addImm(0)
                   .add(predOps(ARMCC::AL));
@@ -699,7 +719,7 @@ bool ARMInstructionSelector::selectSelect(MachineInstrBuilder &MIB,
   assert(validOpRegPair(MRI, ResReg, TrueReg, 32, ARM::GPRRegBankID) &&
          validOpRegPair(MRI, TrueReg, FalseReg, 32, ARM::GPRRegBankID) &&
          "Unsupported types for select operation");
-  auto Mov1I = BuildMI(MBB, InsertBefore, DbgLoc, TII.get(ARM::MOVCCr))
+  auto Mov1I = BuildMI(MBB, InsertBefore, DbgLoc, TII.get(Opcodes.MOVCCr))
                    .addDef(ResReg)
                    .addUse(TrueReg)
                    .addUse(FalseReg)
@@ -900,8 +920,8 @@ bool ARMInstructionSelector::select(MachineInstr &I,
   case G_SELECT:
     return selectSelect(MIB, MRI);
   case G_ICMP: {
-    CmpConstants Helper(ARM::CMPrr, ARM::INSTRUCTION_LIST_END,
-                        ARM::GPRRegBankID, 32);
+    CmpConstants Helper(Opcodes.CMPrr, ARM::INSTRUCTION_LIST_END,
+                        Opcodes.MOVCCi, ARM::GPRRegBankID, 32);
     return selectCmp(Helper, MIB, MRI);
   }
   case G_FCMP: {
@@ -920,7 +940,7 @@ bool ARMInstructionSelector::select(MachineInstr &I,
     }
 
     CmpConstants Helper(Size == 32 ? ARM::VCMPS : ARM::VCMPD, ARM::FMSTAT,
-                        ARM::FPRRegBankID, Size);
+                        Opcodes.MOVCCi, ARM::FPRRegBankID, Size);
     return selectCmp(Helper, MIB, MRI);
   }
   case G_LSHR:
@@ -931,7 +951,7 @@ bool ARMInstructionSelector::select(MachineInstr &I,
     return selectShift(ARM_AM::ShiftOpc::lsl, MIB);
   }
   case G_GEP:
-    I.setDesc(TII.get(ARM::ADDrr));
+    I.setDesc(TII.get(STI.isThumb2() ? ARM::t2ADDrr : ARM::ADDrr));
     MIB.add(predOps(ARMCC::AL)).add(condCodeOp());
     break;
   case G_FRAME_INDEX:
diff --git a/lib/Target/ARM/ARMLegalizerInfo.cpp b/lib/Target/ARM/ARMLegalizerInfo.cpp
index 4a0c24d58474c73e2478a52744ef0a36831a6ad0..10320e59897187d684f59e59e3cda7c37e68fad1 100644
--- a/lib/Target/ARM/ARMLegalizerInfo.cpp
+++ b/lib/Target/ARM/ARMLegalizerInfo.cpp
@@ -1,9 +1,8 @@
 //===- ARMLegalizerInfo.cpp --------------------------------------*- C++ -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
@@ -89,6 +88,31 @@ ARMLegalizerInfo::ARMLegalizerInfo(const ARMSubtarget &ST) {
       .legalFor({s32})
       .minScalar(0, s32);
 
+  getActionDefinitionsBuilder({G_ASHR, G_LSHR, G_SHL})
+    .legalFor({{s32, s32}})
+    .clampScalar(1, s32, s32);
+
+  bool HasHWDivide = (!ST.isThumb() && ST.hasDivideInARMMode()) ||
+                     (ST.isThumb() && ST.hasDivideInThumbMode());
+  if (HasHWDivide)
+    getActionDefinitionsBuilder({G_SDIV, G_UDIV})
+        .legalFor({s32})
+        .clampScalar(0, s32, s32);
+  else
+    getActionDefinitionsBuilder({G_SDIV, G_UDIV})
+        .libcallFor({s32})
+        .clampScalar(0, s32, s32);
+
+  for (unsigned Op : {G_SREM, G_UREM}) {
+    setLegalizeScalarToDifferentSizeStrategy(Op, 0, widen_8_16);
+    if (HasHWDivide)
+      setAction({Op, s32}, Lower);
+    else if (AEABI(ST))
+      setAction({Op, s32}, Custom);
+    else
+      setAction({Op, s32}, Libcall);
+  }
+
   getActionDefinitionsBuilder(G_INTTOPTR).legalFor({{p0, s32}});
   getActionDefinitionsBuilder(G_PTRTOINT).legalFor({{s32, p0}});
 
@@ -96,6 +120,13 @@ ARMLegalizerInfo::ARMLegalizerInfo(const ARMSubtarget &ST) {
       .legalFor({s32, p0})
       .clampScalar(0, s32, s32);
 
+  getActionDefinitionsBuilder(G_ICMP)
+      .legalForCartesianProduct({s1}, {s32, p0})
+      .minScalar(1, s32);
+
+  getActionDefinitionsBuilder(G_SELECT).legalForCartesianProduct({s32, p0},
+                                                                 {s1});
+
   // We're keeping these builders around because we'll want to add support for
   // floating point to them.
   auto &LoadStoreBuilder =
@@ -107,6 +138,8 @@ ARMLegalizerInfo::ARMLegalizerInfo(const ARMSubtarget &ST) {
               {s32, p0, 32},
               {p0, p0, 32}});
 
+  getActionDefinitionsBuilder(G_GEP).legalFor({{p0, s32}});
+
   if (ST.isThumb()) {
     // FIXME: merge with the code for non-Thumb.
     computeTables();
@@ -117,54 +150,28 @@ ARMLegalizerInfo::ARMLegalizerInfo(const ARMSubtarget &ST) {
   getActionDefinitionsBuilder(G_GLOBAL_VALUE).legalFor({p0});
   getActionDefinitionsBuilder(G_FRAME_INDEX).legalFor({p0});
 
-  if (ST.hasDivideInARMMode())
-    getActionDefinitionsBuilder({G_SDIV, G_UDIV})
-        .legalFor({s32})
-        .clampScalar(0, s32, s32);
-  else
-    getActionDefinitionsBuilder({G_SDIV, G_UDIV})
-        .libcallFor({s32})
-        .clampScalar(0, s32, s32);
-
-  for (unsigned Op : {G_SREM, G_UREM}) {
-    setLegalizeScalarToDifferentSizeStrategy(Op, 0, widen_8_16);
-    if (ST.hasDivideInARMMode())
-      setAction({Op, s32}, Lower);
-    else if (AEABI(ST))
-      setAction({Op, s32}, Custom);
-    else
-      setAction({Op, s32}, Libcall);
-  }
-
-  getActionDefinitionsBuilder({G_ASHR, G_LSHR, G_SHL}).legalFor({s32});
-
   if (ST.hasV5TOps()) {
     getActionDefinitionsBuilder(G_CTLZ)
-        .legalFor({s32})
+        .legalFor({s32, s32})
+        .clampScalar(1, s32, s32)
         .clampScalar(0, s32, s32);
     getActionDefinitionsBuilder(G_CTLZ_ZERO_UNDEF)
-        .lowerFor({s32})
+        .lowerFor({s32, s32})
+        .clampScalar(1, s32, s32)
         .clampScalar(0, s32, s32);
   } else {
     getActionDefinitionsBuilder(G_CTLZ_ZERO_UNDEF)
-        .libcallFor({s32})
+        .libcallFor({s32, s32})
+        .clampScalar(1, s32, s32)
         .clampScalar(0, s32, s32);
     getActionDefinitionsBuilder(G_CTLZ)
-        .lowerFor({s32})
+        .lowerFor({s32, s32})
+        .clampScalar(1, s32, s32)
         .clampScalar(0, s32, s32);
   }
 
-  getActionDefinitionsBuilder(G_GEP).legalFor({{p0, s32}});
-
-  getActionDefinitionsBuilder(G_SELECT).legalForCartesianProduct({s32, p0},
-                                                                 {s1});
-
   getActionDefinitionsBuilder(G_BRCOND).legalFor({s1});
 
-  getActionDefinitionsBuilder(G_ICMP)
-      .legalForCartesianProduct({s1}, {s32, p0})
-      .minScalar(1, s32);
-
   // We're keeping these builders around because we'll want to add support for
   // floating point to them.
   auto &PhiBuilder =
diff --git a/lib/Target/ARM/ARMLegalizerInfo.h b/lib/Target/ARM/ARMLegalizerInfo.h
index 527bf87f1093043191f1e36951b48ab7e3118c5e..e95f8cf76103f2be1780e504a89494eb7459165b 100644
--- a/lib/Target/ARM/ARMLegalizerInfo.h
+++ b/lib/Target/ARM/ARMLegalizerInfo.h
@@ -1,9 +1,8 @@
 //===- ARMLegalizerInfo ------------------------------------------*- C++ -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/lib/Target/ARM/ARMLoadStoreOptimizer.cpp b/lib/Target/ARM/ARMLoadStoreOptimizer.cpp
index 6da7430a8e513522404710709ca038f1feee7060..132030d0393d94253b51d9d43532481a182d2394 100644
--- a/lib/Target/ARM/ARMLoadStoreOptimizer.cpp
+++ b/lib/Target/ARM/ARMLoadStoreOptimizer.cpp
@@ -1,9 +1,8 @@
 //===- ARMLoadStoreOptimizer.cpp - ARM load / store opt. pass -------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -1287,7 +1286,7 @@ bool ARMLoadStoreOpt::MergeBaseUpdateLSMultiple(MachineInstr *MI) {
       // can still change to a writeback form as that will save us 2 bytes
       // of code size. It can create WAW hazards though, so only do it if
       // we're minimizing code size.
-      if (!MBB.getParent()->getFunction().optForMinSize() || !BaseKill)
+      if (!STI->optForMinSize() || !BaseKill)
         return false;
 
       bool HighRegsUsed = false;
@@ -2048,6 +2047,11 @@ char ARMPreAllocLoadStoreOpt::ID = 0;
 INITIALIZE_PASS(ARMPreAllocLoadStoreOpt, "arm-prera-ldst-opt",
                 ARM_PREALLOC_LOAD_STORE_OPT_NAME, false, false)
 
+// Limit the number of instructions to be rescheduled.
+// FIXME: tune this limit, and/or come up with some better heuristics.
+static cl::opt<unsigned> InstReorderLimit("arm-prera-ldst-opt-reorder-limit",
+                                          cl::init(8), cl::Hidden);
+
 bool ARMPreAllocLoadStoreOpt::runOnMachineFunction(MachineFunction &Fn) {
   if (AssumeMisalignedLoadStores || skipFunction(Fn.getFunction()))
     return false;
@@ -2223,7 +2227,7 @@ bool ARMPreAllocLoadStoreOpt::RescheduleOps(MachineBasicBlock *MBB,
       }
 
       // Don't try to reschedule too many instructions.
-      if (NumMove == 8) // FIXME: Tune this limit.
+      if (NumMove == InstReorderLimit)
         break;
 
       // Found a mergable instruction; save information about it.
@@ -2351,10 +2355,13 @@ ARMPreAllocLoadStoreOpt::RescheduleLoadStoreInstrs(MachineBasicBlock *MBB) {
   bool RetVal = false;
 
   DenseMap<MachineInstr*, unsigned> MI2LocMap;
-  DenseMap<unsigned, SmallVector<MachineInstr *, 4>> Base2LdsMap;
-  DenseMap<unsigned, SmallVector<MachineInstr *, 4>> Base2StsMap;
-  SmallVector<unsigned, 4> LdBases;
-  SmallVector<unsigned, 4> StBases;
+  using MapIt = DenseMap<unsigned, SmallVector<MachineInstr *, 4>>::iterator;
+  using Base2InstMap = DenseMap<unsigned, SmallVector<MachineInstr *, 4>>;
+  using BaseVec = SmallVector<unsigned, 4>;
+  Base2InstMap Base2LdsMap;
+  Base2InstMap Base2StsMap;
+  BaseVec LdBases;
+  BaseVec StBases;
 
   unsigned Loc = 0;
   MachineBasicBlock::iterator MBBI = MBB->begin();
@@ -2381,41 +2388,28 @@ ARMPreAllocLoadStoreOpt::RescheduleLoadStoreInstrs(MachineBasicBlock *MBB) {
       bool isLd = isLoadSingle(Opc);
       unsigned Base = MI.getOperand(1).getReg();
       int Offset = getMemoryOpOffset(MI);
-
       bool StopHere = false;
-      if (isLd) {
-        DenseMap<unsigned, SmallVector<MachineInstr *, 4>>::iterator BI =
-          Base2LdsMap.find(Base);
-        if (BI != Base2LdsMap.end()) {
-          for (unsigned i = 0, e = BI->second.size(); i != e; ++i) {
-            if (Offset == getMemoryOpOffset(*BI->second[i])) {
-              StopHere = true;
-              break;
-            }
-          }
-          if (!StopHere)
-            BI->second.push_back(&MI);
-        } else {
-          Base2LdsMap[Base].push_back(&MI);
-          LdBases.push_back(Base);
+      auto FindBases = [&] (Base2InstMap &Base2Ops, BaseVec &Bases) {
+        MapIt BI = Base2Ops.find(Base);
+        if (BI == Base2Ops.end()) {
+          Base2Ops[Base].push_back(&MI);
+          Bases.push_back(Base);
+          return;
         }
-      } else {
-        DenseMap<unsigned, SmallVector<MachineInstr *, 4>>::iterator BI =
-          Base2StsMap.find(Base);
-        if (BI != Base2StsMap.end()) {
-          for (unsigned i = 0, e = BI->second.size(); i != e; ++i) {
-            if (Offset == getMemoryOpOffset(*BI->second[i])) {
-              StopHere = true;
-              break;
-            }
+        for (unsigned i = 0, e = BI->second.size(); i != e; ++i) {
+          if (Offset == getMemoryOpOffset(*BI->second[i])) {
+            StopHere = true;
+            break;
           }
-          if (!StopHere)
-            BI->second.push_back(&MI);
-        } else {
-          Base2StsMap[Base].push_back(&MI);
-          StBases.push_back(Base);
         }
-      }
+        if (!StopHere)
+          BI->second.push_back(&MI);
+      };
+
+      if (isLd)
+        FindBases(Base2LdsMap, LdBases);
+      else
+        FindBases(Base2StsMap, StBases);
 
       if (StopHere) {
         // Found a duplicate (a base+offset combination that's seen earlier).
diff --git a/lib/Target/ARM/ARMMCInstLower.cpp b/lib/Target/ARM/ARMMCInstLower.cpp
index 48b02d40b246619a12221bc4cfaa92c1f6b985f8..90c5ad025e5670c0bd335508eced0766ba261017 100644
--- a/lib/Target/ARM/ARMMCInstLower.cpp
+++ b/lib/Target/ARM/ARMMCInstLower.cpp
@@ -1,9 +1,8 @@
 //===-- ARMMCInstLower.cpp - Convert ARM MachineInstr to an MCInst --------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/ARM/ARMMachineFunctionInfo.cpp b/lib/Target/ARM/ARMMachineFunctionInfo.cpp
index e25d36b576169a540c923d82be86d9c8d77124bb..3b676ca4c883f5a6f85dac1352330af0d5bcacab 100644
--- a/lib/Target/ARM/ARMMachineFunctionInfo.cpp
+++ b/lib/Target/ARM/ARMMachineFunctionInfo.cpp
@@ -1,9 +1,8 @@
 //===-- ARMMachineFunctionInfo.cpp - ARM machine function info ------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/ARM/ARMMachineFunctionInfo.h b/lib/Target/ARM/ARMMachineFunctionInfo.h
index 91310e81e398b1c0ba99094f1841d03a3d67bd45..cad6e8e38256af3866debce723ab63fabbad818a 100644
--- a/lib/Target/ARM/ARMMachineFunctionInfo.h
+++ b/lib/Target/ARM/ARMMachineFunctionInfo.h
@@ -1,9 +1,8 @@
 //===-- ARMMachineFunctionInfo.h - ARM machine function info ----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/ARM/ARMMacroFusion.cpp b/lib/Target/ARM/ARMMacroFusion.cpp
index df1da9d8e4744ac2a4fad78a64dac00fa8d0d5ef..38bf28ba8219b9016524d5d15d3188ab4ff39727 100644
--- a/lib/Target/ARM/ARMMacroFusion.cpp
+++ b/lib/Target/ARM/ARMMacroFusion.cpp
@@ -1,9 +1,8 @@
 //===- ARMMacroFusion.cpp - ARM Macro Fusion ----------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/ARM/ARMMacroFusion.h b/lib/Target/ARM/ARMMacroFusion.h
index b3abd7b593a12960f5937979913052f6041bfde7..4896a4a2544dbf5d0379a6d1a6a4af64b7a5fc47 100644
--- a/lib/Target/ARM/ARMMacroFusion.h
+++ b/lib/Target/ARM/ARMMacroFusion.h
@@ -1,9 +1,8 @@
 //===- ARMMacroFusion.h - ARM Macro Fusion ------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/ARM/ARMOptimizeBarriersPass.cpp b/lib/Target/ARM/ARMOptimizeBarriersPass.cpp
index cff4a256100d91989042bd8985df1512f1501cb4..348895da713f65c70d2149eff3fbe4cda0355496 100644
--- a/lib/Target/ARM/ARMOptimizeBarriersPass.cpp
+++ b/lib/Target/ARM/ARMOptimizeBarriersPass.cpp
@@ -1,10 +1,9 @@
 //===-- ARMOptimizeBarriersPass - two DMBs without a memory access in between,
 //removed one -===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===------------------------------------------------------------------------------------------===//
 
diff --git a/lib/Target/ARM/ARMParallelDSP.cpp b/lib/Target/ARM/ARMParallelDSP.cpp
index fc3258914f92b4069f2a90db48e67f1e733a1ae0..9730c32577e91c1e17de8397859b627de2a4325f 100644
--- a/lib/Target/ARM/ARMParallelDSP.cpp
+++ b/lib/Target/ARM/ARMParallelDSP.cpp
@@ -1,9 +1,8 @@
 //===- ParallelDSP.cpp - Parallel DSP Pass --------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -689,12 +688,12 @@ bool ARMParallelDSP::MatchSMLAD(Function &F) {
 }
 
 static LoadInst *CreateLoadIns(IRBuilder<NoFolder> &IRB, LoadInst &BaseLoad,
-                               const Type *LoadTy) {
+                               Type *LoadTy) {
   const unsigned AddrSpace = BaseLoad.getPointerAddressSpace();
 
   Value *VecPtr = IRB.CreateBitCast(BaseLoad.getPointerOperand(),
                                     LoadTy->getPointerTo(AddrSpace));
-  return IRB.CreateAlignedLoad(VecPtr, BaseLoad.getAlignment());
+  return IRB.CreateAlignedLoad(LoadTy, VecPtr, BaseLoad.getAlignment());
 }
 
 Instruction *ARMParallelDSP::CreateSMLADCall(LoadInst *VecLd0, LoadInst *VecLd1,
@@ -710,7 +709,7 @@ Instruction *ARMParallelDSP::CreateSMLADCall(LoadInst *VecLd0, LoadInst *VecLd1,
                               ++BasicBlock::iterator(InsertAfter));
 
   // Replace the reduction chain with an intrinsic call
-  const Type *Ty = IntegerType::get(M->getContext(), 32);
+  Type *Ty = IntegerType::get(M->getContext(), 32);
   LoadInst *NewLd0 = CreateLoadIns(Builder, VecLd0[0], Ty);
   LoadInst *NewLd1 = CreateLoadIns(Builder, VecLd1[0], Ty);
   Value* Args[] = { NewLd0, NewLd1, Acc };
diff --git a/lib/Target/ARM/ARMPerfectShuffle.h b/lib/Target/ARM/ARMPerfectShuffle.h
index 3ff0bee7e5bf8d033c9a5882103f3a3f8540ffe4..d519490c9c572af6d4a8cab749ece4fd303b7b48 100644
--- a/lib/Target/ARM/ARMPerfectShuffle.h
+++ b/lib/Target/ARM/ARMPerfectShuffle.h
@@ -1,9 +1,8 @@
 //===-- ARMPerfectShuffle.h - NEON Perfect Shuffle Table --------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/ARM/ARMRegisterBankInfo.cpp b/lib/Target/ARM/ARMRegisterBankInfo.cpp
index 4f28f2dafc70ab4e31c978443a1d28191fe250cf..082de6698c2f9cf1367c76e1a599ca6e613e21b2 100644
--- a/lib/Target/ARM/ARMRegisterBankInfo.cpp
+++ b/lib/Target/ARM/ARMRegisterBankInfo.cpp
@@ -1,9 +1,8 @@
 //===- ARMRegisterBankInfo.cpp -----------------------------------*- C++ -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/lib/Target/ARM/ARMRegisterBankInfo.h b/lib/Target/ARM/ARMRegisterBankInfo.h
index 9650b358f3194bb66f6407000770cce7027d7832..1961f7af49bbd60732276fee2c1340c8c4e6b380 100644
--- a/lib/Target/ARM/ARMRegisterBankInfo.h
+++ b/lib/Target/ARM/ARMRegisterBankInfo.h
@@ -1,9 +1,8 @@
 //===- ARMRegisterBankInfo ---------------------------------------*- C++ -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/lib/Target/ARM/ARMRegisterBanks.td b/lib/Target/ARM/ARMRegisterBanks.td
index 6e3834da3bb579900e832902c8c20d9c3921cdfa..e4ebf793f9b0e54453e597face6afddcb40d7bb2 100644
--- a/lib/Target/ARM/ARMRegisterBanks.td
+++ b/lib/Target/ARM/ARMRegisterBanks.td
@@ -1,9 +1,8 @@
 //=- ARMRegisterBank.td - Describe the AArch64 Banks ---------*- tablegen -*-=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/ARM/ARMRegisterInfo.cpp b/lib/Target/ARM/ARMRegisterInfo.cpp
index e6e8cdf965e23e451171145e0dc07758a028a6cf..6649750bb3883c09cc5731f6c718f497e7aa5453 100644
--- a/lib/Target/ARM/ARMRegisterInfo.cpp
+++ b/lib/Target/ARM/ARMRegisterInfo.cpp
@@ -1,9 +1,8 @@
 //===-- ARMRegisterInfo.cpp - ARM Register Information --------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/ARM/ARMRegisterInfo.h b/lib/Target/ARM/ARMRegisterInfo.h
index e2e650e4af930f562156a97e93c5b6ac8809a781..87c0f322d3b3aa806ae8a42c797489bf2081469f 100644
--- a/lib/Target/ARM/ARMRegisterInfo.h
+++ b/lib/Target/ARM/ARMRegisterInfo.h
@@ -1,9 +1,8 @@
 //===-- ARMRegisterInfo.h - ARM Register Information Impl -------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/ARM/ARMRegisterInfo.td b/lib/Target/ARM/ARMRegisterInfo.td
index dc56186cb54a11dcf80b7a87e8377bf9f05e32a3..668d374ef16098d69132eadd6c02f302790c3b5a 100644
--- a/lib/Target/ARM/ARMRegisterInfo.td
+++ b/lib/Target/ARM/ARMRegisterInfo.td
@@ -1,9 +1,8 @@
 //===-- ARMRegisterInfo.td - ARM Register defs -------------*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -302,7 +301,7 @@ def SPR : RegisterClass<"ARM", [f32], 32, (sequence "S%u", 0, 31)> {
                         (decimate (rotl SPR, 1), 4),
                         (decimate (rotl SPR, 1), 2))];
   let AltOrderSelect = [{
-    return 1 + MF.getSubtarget<ARMSubtarget>().useStride4VFPs(MF);
+    return 1 + MF.getSubtarget<ARMSubtarget>().useStride4VFPs();
   }];
   let DiagnosticString = "operand must be a register in range [s0, s31]";
 }
@@ -314,7 +313,7 @@ def HPR : RegisterClass<"ARM", [f16], 32, (sequence "S%u", 0, 31)> {
                         (decimate (rotl HPR, 1), 4),
                         (decimate (rotl HPR, 1), 2))];
   let AltOrderSelect = [{
-    return 1 + MF.getSubtarget<ARMSubtarget>().useStride4VFPs(MF);
+    return 1 + MF.getSubtarget<ARMSubtarget>().useStride4VFPs();
   }];
   let DiagnosticString = "operand must be a register in range [s0, s31]";
 }
@@ -336,7 +335,7 @@ def DPR : RegisterClass<"ARM", [f64, v8i8, v4i16, v2i32, v1i64, v2f32, v4f16], 6
   let AltOrders = [(rotl DPR, 16),
                    (add (decimate (rotl DPR, 16), 2), (rotl DPR, 16))];
   let AltOrderSelect = [{
-    return 1 + MF.getSubtarget<ARMSubtarget>().useStride4VFPs(MF);
+    return 1 + MF.getSubtarget<ARMSubtarget>().useStride4VFPs();
   }];
   let DiagnosticType = "DPR";
 }
diff --git a/lib/Target/ARM/ARMSchedule.td b/lib/Target/ARM/ARMSchedule.td
index ed5a3a7bb696622101530e68eabbb98576080dbd..4e848406694b774ea891c6ceeae330b0d010b1da 100644
--- a/lib/Target/ARM/ARMSchedule.td
+++ b/lib/Target/ARM/ARMSchedule.td
@@ -1,9 +1,8 @@
 //===-- ARMSchedule.td - ARM Scheduling Definitions --------*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //===----------------------------------------------------------------------===//
diff --git a/lib/Target/ARM/ARMScheduleA57.td b/lib/Target/ARM/ARMScheduleA57.td
index 63f975ba6e39e1b63eb9fe9afef4c075377a377e..6db6eda317abf224d39afce0499cbdf0d77904a2 100644
--- a/lib/Target/ARM/ARMScheduleA57.td
+++ b/lib/Target/ARM/ARMScheduleA57.td
@@ -1,9 +1,8 @@
 //=- ARMScheduleA57.td - ARM Cortex-A57 Scheduling Defs -----*- tablegen -*-=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/ARM/ARMScheduleA57WriteRes.td b/lib/Target/ARM/ARMScheduleA57WriteRes.td
index 670717dc7c13886c3378528e793c7853c5612f7b..5ba61503686ecf92fccd298106ca2cbd81e18ebf 100644
--- a/lib/Target/ARM/ARMScheduleA57WriteRes.td
+++ b/lib/Target/ARM/ARMScheduleA57WriteRes.td
@@ -1,9 +1,8 @@
 //=- ARMScheduleA57WriteRes.td - ARM Cortex-A57 Write Res ---*- tablegen -*-=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/ARM/ARMScheduleA8.td b/lib/Target/ARM/ARMScheduleA8.td
index ba380cba100f75361e9724258cbc4d3e1e3b5285..1be0ee4334a8346adc29adc1c56f41ee202bec07 100644
--- a/lib/Target/ARM/ARMScheduleA8.td
+++ b/lib/Target/ARM/ARMScheduleA8.td
@@ -1,9 +1,8 @@
 //=- ARMScheduleA8.td - ARM Cortex-A8 Scheduling Definitions -*- tablegen -*-=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/ARM/ARMScheduleA9.td b/lib/Target/ARM/ARMScheduleA9.td
index fc301c58926991df01add9b66e268e100882db36..21d32bde47105ffe2c592860c3d951c2282567b1 100644
--- a/lib/Target/ARM/ARMScheduleA9.td
+++ b/lib/Target/ARM/ARMScheduleA9.td
@@ -1,9 +1,8 @@
 //=- ARMScheduleA9.td - ARM Cortex-A9 Scheduling Definitions -*- tablegen -*-=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/ARM/ARMScheduleM3.td b/lib/Target/ARM/ARMScheduleM3.td
index 93f8299f9bd042cadd91a7bed43b3a5079fe9698..325e28977ee1bec9d87638c15e9dac7189ff4b81 100644
--- a/lib/Target/ARM/ARMScheduleM3.td
+++ b/lib/Target/ARM/ARMScheduleM3.td
@@ -1,9 +1,8 @@
 //=- ARMScheduleM3.td - ARM Cortex-M3 Scheduling Definitions -*- tablegen -*-=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/ARM/ARMScheduleR52.td b/lib/Target/ARM/ARMScheduleR52.td
index 11bce45161b3eae380e44c0633e1703c473ce440..d1cbf754b5a1bf0f0f8c97c3e583ce1cb927ba24 100644
--- a/lib/Target/ARM/ARMScheduleR52.td
+++ b/lib/Target/ARM/ARMScheduleR52.td
@@ -1,9 +1,8 @@
 //==- ARMScheduleR52.td - Cortex-R52 Scheduling Definitions -*- tablegen -*-=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/ARM/ARMScheduleSwift.td b/lib/Target/ARM/ARMScheduleSwift.td
index 87984648139b42b957664b74a8ea3f0446e74cec..00a44599b1b2f9d886dc447c52721ccd7c2fec5b 100644
--- a/lib/Target/ARM/ARMScheduleSwift.td
+++ b/lib/Target/ARM/ARMScheduleSwift.td
@@ -1,9 +1,8 @@
 //=- ARMScheduleSwift.td - Swift Scheduling Definitions -*- tablegen -*----===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/ARM/ARMScheduleV6.td b/lib/Target/ARM/ARMScheduleV6.td
index 57d0bfb650493a731e6e6768b7a0527cb5e2ffa4..9b86097329c06fe97876fc93b1a5f66e179f78f1 100644
--- a/lib/Target/ARM/ARMScheduleV6.td
+++ b/lib/Target/ARM/ARMScheduleV6.td
@@ -1,9 +1,8 @@
 //===-- ARMScheduleV6.td - ARM v6 Scheduling Definitions ---*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/ARM/ARMSelectionDAGInfo.cpp b/lib/Target/ARM/ARMSelectionDAGInfo.cpp
index 4d685158e2588def11d8094d43154578a25b6433..332e4e703ed8591282f3885b11eefa9b9da3bc9f 100644
--- a/lib/Target/ARM/ARMSelectionDAGInfo.cpp
+++ b/lib/Target/ARM/ARMSelectionDAGInfo.cpp
@@ -1,9 +1,8 @@
 //===-- ARMSelectionDAGInfo.cpp - ARM SelectionDAG Info -------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -171,7 +170,7 @@ SDValue ARMSelectionDAGInfo::EmitTargetCodeForMemcpy(
 
   // Code size optimisation: do not inline memcpy if expansion results in
   // more instructions than the libary call.
-  if (NumMEMCPYs > 1 && DAG.getMachineFunction().getFunction().optForMinSize()) {
+  if (NumMEMCPYs > 1 && Subtarget.optForMinSize()) {
     return SDValue();
   }
 
diff --git a/lib/Target/ARM/ARMSelectionDAGInfo.h b/lib/Target/ARM/ARMSelectionDAGInfo.h
index 2ddb42c95397664dcf82c7d4d54c9195b6c3ccc6..b8a86ae7310fafbafcfac6afca75e99fa252a379 100644
--- a/lib/Target/ARM/ARMSelectionDAGInfo.h
+++ b/lib/Target/ARM/ARMSelectionDAGInfo.h
@@ -1,9 +1,8 @@
 //===-- ARMSelectionDAGInfo.h - ARM SelectionDAG Info -----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/ARM/ARMSubtarget.cpp b/lib/Target/ARM/ARMSubtarget.cpp
index b1d0761e323155d419326fb57b1631c9cc593d1e..692585ea79e70f6fad68d2ba71432fe046db66e8 100644
--- a/lib/Target/ARM/ARMSubtarget.cpp
+++ b/lib/Target/ARM/ARMSubtarget.cpp
@@ -1,9 +1,8 @@
 //===-- ARMSubtarget.cpp - ARM Subtarget Information ----------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -93,10 +92,12 @@ ARMFrameLowering *ARMSubtarget::initializeFrameLowering(StringRef CPU,
 
 ARMSubtarget::ARMSubtarget(const Triple &TT, const std::string &CPU,
                            const std::string &FS,
-                           const ARMBaseTargetMachine &TM, bool IsLittle)
+                           const ARMBaseTargetMachine &TM, bool IsLittle,
+                           bool MinSize)
     : ARMGenSubtargetInfo(TT, CPU, FS), UseMulOps(UseFusedMulOps),
-      CPUString(CPU), IsLittle(IsLittle), TargetTriple(TT), Options(TM.Options),
-      TM(TM), FrameLowering(initializeFrameLowering(CPU, FS)),
+      CPUString(CPU), OptMinSize(MinSize), IsLittle(IsLittle),
+      TargetTriple(TT), Options(TM.Options), TM(TM),
+      FrameLowering(initializeFrameLowering(CPU, FS)),
       // At this point initializeSubtargetDependencies has been called so
       // we can query directly.
       InstrInfo(isThumb1Only()
@@ -374,20 +375,20 @@ bool ARMSubtarget::enablePostRAScheduler() const {
 
 bool ARMSubtarget::enableAtomicExpand() const { return hasAnyDataBarrier(); }
 
-bool ARMSubtarget::useStride4VFPs(const MachineFunction &MF) const {
+bool ARMSubtarget::useStride4VFPs() const {
   // For general targets, the prologue can grow when VFPs are allocated with
   // stride 4 (more vpush instructions). But WatchOS uses a compact unwind
   // format which it's more important to get right.
   return isTargetWatchABI() ||
-         (useWideStrideVFP() && !MF.getFunction().optForMinSize());
+         (useWideStrideVFP() && !OptMinSize);
 }
 
-bool ARMSubtarget::useMovt(const MachineFunction &MF) const {
+bool ARMSubtarget::useMovt() const {
   // NOTE Windows on ARM needs to use mov.w/mov.t pairs to materialise 32-bit
   // immediates as it is inherently position independent, and may be out of
   // range otherwise.
   return !NoMovt && hasV8MBaselineOps() &&
-         (isTargetWindows() || !MF.getFunction().optForMinSize() || genExecuteOnly());
+         (isTargetWindows() || !OptMinSize || genExecuteOnly());
 }
 
 bool ARMSubtarget::useFastISel() const {
diff --git a/lib/Target/ARM/ARMSubtarget.h b/lib/Target/ARM/ARMSubtarget.h
index b06c35ff1acf8cd257a6ec3a16138b98d25ff871..d2936d3ca72435efe8823f1c2aee4fcb5dcca34a 100644
--- a/lib/Target/ARM/ARMSubtarget.h
+++ b/lib/Target/ARM/ARMSubtarget.h
@@ -1,9 +1,8 @@
 //===-- ARMSubtarget.h - Define Subtarget for the ARM ----------*- C++ -*--===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -417,7 +416,7 @@ protected:
   bool UseSjLjEH = false;
 
   /// Has speculation barrier
-  bool HasSpecCtrl = false;
+  bool HasSB = false;
 
   /// Implicitly convert an instruction to a different one if its immediates
   /// cannot be encoded. For example, ADD r0, r1, #FFFFFFFF -> SUB r0, r1, #1.
@@ -446,6 +445,10 @@ protected:
   /// What alignment is preferred for loop bodies, in log2(bytes).
   unsigned PrefLoopAlignment = 0;
 
+  /// OptMinSize - True if we're optimising for minimum code size, equal to
+  /// the function attribute.
+  bool OptMinSize = false;
+
   /// IsLittle - The target is Little Endian
   bool IsLittle;
 
@@ -468,7 +471,8 @@ public:
   /// of the specified triple.
   ///
   ARMSubtarget(const Triple &TT, const std::string &CPU, const std::string &FS,
-               const ARMBaseTargetMachine &TM, bool IsLittle);
+               const ARMBaseTargetMachine &TM, bool IsLittle,
+               bool MinSize = false);
 
   /// getMaxInlineSizeThreshold - Returns the maximum memset / memcpy size
   /// that still makes it profitable to inline the call.
@@ -628,7 +632,7 @@ public:
   bool hasDSP() const { return HasDSP; }
   bool useNaClTrap() const { return UseNaClTrap; }
   bool useSjLjEH() const { return UseSjLjEH; }
-  bool hasSpecCtrl() const { return HasSpecCtrl; }
+  bool hasSB() const { return HasSB; }
   bool genLongCalls() const { return GenLongCalls; }
   bool genExecuteOnly() const { return GenExecuteOnly; }
 
@@ -710,6 +714,7 @@ public:
   bool disablePostRAScheduler() const { return DisablePostRAScheduler; }
   bool useSoftFloat() const { return UseSoftFloat; }
   bool isThumb() const { return InThumbMode; }
+  bool optForMinSize() const { return OptMinSize; }
   bool isThumb1Only() const { return InThumbMode && !HasThumb2; }
   bool isThumb2() const { return InThumbMode && HasThumb2; }
   bool hasThumb2() const { return HasThumb2; }
@@ -736,9 +741,9 @@ public:
            isThumb1Only();
   }
 
-  bool useStride4VFPs(const MachineFunction &MF) const;
+  bool useStride4VFPs() const;
 
-  bool useMovt(const MachineFunction &MF) const;
+  bool useMovt() const;
 
   bool supportsTailCall() const { return SupportsTailCall; }
 
diff --git a/lib/Target/ARM/ARMSystemRegister.td b/lib/Target/ARM/ARMSystemRegister.td
index ad1d37168e087a296d2216986f61d456f9c4dda9..f21c7f0246f9145d4366bdcd9198277c39cdf005 100644
--- a/lib/Target/ARM/ARMSystemRegister.td
+++ b/lib/Target/ARM/ARMSystemRegister.td
@@ -1,9 +1,8 @@
 //===-- ARMSystemRegister.td - ARM Register defs -------------*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/ARM/ARMTargetMachine.cpp b/lib/Target/ARM/ARMTargetMachine.cpp
index ec02c840d5e17e75b5e2962cc38b1a9f996258dd..9954eee2e5f4dcd9f0b3e3b1da574e6cb933ce5d 100644
--- a/lib/Target/ARM/ARMTargetMachine.cpp
+++ b/lib/Target/ARM/ARMTargetMachine.cpp
@@ -1,9 +1,8 @@
 //===-- ARMTargetMachine.cpp - Define TargetMachine for ARM ---------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -264,13 +263,20 @@ ARMBaseTargetMachine::getSubtargetImpl(const Function &F) const {
   if (SoftFloat)
     FS += FS.empty() ? "+soft-float" : ",+soft-float";
 
-  auto &I = SubtargetMap[CPU + FS];
+  // Use the optminsize to identify the subtarget, but don't use it in the
+  // feature string.
+  std::string Key = CPU + FS;
+  if (F.optForMinSize())
+    Key += "+minsize";
+
+  auto &I = SubtargetMap[Key];
   if (!I) {
     // This needs to be done before we create a new subtarget since any
     // creation will depend on the TM and the code generation flags on the
     // function that reside in TargetOptions.
     resetTargetOptions(F);
-    I = llvm::make_unique<ARMSubtarget>(TargetTriple, CPU, FS, *this, isLittle);
+    I = llvm::make_unique<ARMSubtarget>(TargetTriple, CPU, FS, *this, isLittle,
+                                        F.optForMinSize());
 
     if (!I->isThumb() && !I->hasARMOps())
       F.getContext().emitError("Function '" + F.getName() + "' uses ARM "
diff --git a/lib/Target/ARM/ARMTargetMachine.h b/lib/Target/ARM/ARMTargetMachine.h
index 2c791998e702d3b490d51cecfd4de141bc3d1a6e..cb8650d8139becf5c5c224ccb152e88b1288361e 100644
--- a/lib/Target/ARM/ARMTargetMachine.h
+++ b/lib/Target/ARM/ARMTargetMachine.h
@@ -1,9 +1,8 @@
 //===-- ARMTargetMachine.h - Define TargetMachine for ARM -------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/ARM/ARMTargetObjectFile.cpp b/lib/Target/ARM/ARMTargetObjectFile.cpp
index 9c13359cba715807fbd2eaa14e0cf33e40e7ca12..891329d3f2979fb06f0ea16b75f26a3fdad2aef0 100644
--- a/lib/Target/ARM/ARMTargetObjectFile.cpp
+++ b/lib/Target/ARM/ARMTargetObjectFile.cpp
@@ -1,9 +1,8 @@
 //===-- llvm/Target/ARMTargetObjectFile.cpp - ARM Object Info Impl --------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/ARM/ARMTargetObjectFile.h b/lib/Target/ARM/ARMTargetObjectFile.h
index 0dc0882809c091bfbfe96dd020c3de4a9b711917..7b15dcc61f566aaf081fa9c6cc4648fee9b40a51 100644
--- a/lib/Target/ARM/ARMTargetObjectFile.h
+++ b/lib/Target/ARM/ARMTargetObjectFile.h
@@ -1,9 +1,8 @@
 //===-- llvm/Target/ARMTargetObjectFile.h - ARM Object Info -----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/ARM/ARMTargetTransformInfo.cpp b/lib/Target/ARM/ARMTargetTransformInfo.cpp
index f72bb8632eb78a5826e8ce775bfd8939430c3abb..1d1d92c99a33f4f8da9fc02cc65771b0a9b91419 100644
--- a/lib/Target/ARM/ARMTargetTransformInfo.cpp
+++ b/lib/Target/ARM/ARMTargetTransformInfo.cpp
@@ -1,9 +1,8 @@
 //===- ARMTargetTransformInfo.cpp - ARM specific TTI ----------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -107,9 +106,13 @@ int ARMTTIImpl::getIntImmCost(unsigned Opcode, unsigned Idx, const APInt &Imm,
       Idx == 1)
     return 0;
 
-  if (Opcode == Instruction::And)
-      // Conversion to BIC is free, and means we can use ~Imm instead.
-      return std::min(getIntImmCost(Imm, Ty), getIntImmCost(~Imm, Ty));
+  if (Opcode == Instruction::And) {
+    // UXTB/UXTH
+    if (Imm == 255 || Imm == 65535)
+      return 0;
+    // Conversion to BIC is free, and means we can use ~Imm instead.
+    return std::min(getIntImmCost(Imm, Ty), getIntImmCost(~Imm, Ty));
+  }
 
   if (Opcode == Instruction::Add)
     // Conversion to SUB is free, and means we can use -Imm instead.
diff --git a/lib/Target/ARM/ARMTargetTransformInfo.h b/lib/Target/ARM/ARMTargetTransformInfo.h
index 2dd143d48a15c5a661c374f42e567a35091f0c8f..90842643c36aa81baad0d248c495b419bc12f188 100644
--- a/lib/Target/ARM/ARMTargetTransformInfo.h
+++ b/lib/Target/ARM/ARMTargetTransformInfo.h
@@ -1,9 +1,8 @@
 //===- ARMTargetTransformInfo.h - ARM specific TTI --------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -94,6 +93,12 @@ public:
 
   bool enableInterleavedAccessVectorization() { return true; }
 
+  bool shouldFavorBackedgeIndex(const Loop *L) const {
+    if (L->getHeader()->getParent()->optForSize())
+      return false;
+    return ST->isMClass() && ST->isThumb2() && L->getNumBlocks() == 1;
+  }
+
   /// Floating-point computation using ARMv8 AArch32 Advanced
   /// SIMD instructions remains unchanged from ARMv7. Only AArch64 SIMD
   /// is IEEE-754 compliant, but it's not covered in this target.
diff --git a/lib/Target/ARM/AsmParser/ARMAsmParser.cpp b/lib/Target/ARM/AsmParser/ARMAsmParser.cpp
index 3832b0112b87b627a41abbad7821dbb43f883b59..5682c2a6547da6518e9e446607ba938622346fc0 100644
--- a/lib/Target/ARM/AsmParser/ARMAsmParser.cpp
+++ b/lib/Target/ARM/AsmParser/ARMAsmParser.cpp
@@ -1,9 +1,8 @@
 //===- ARMAsmParser.cpp - Parse ARM assembly to MCInst instructions -------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/ARM/AsmParser/LLVMBuild.txt b/lib/Target/ARM/AsmParser/LLVMBuild.txt
index 19058aeadb956c886dbfb857ded878c07c55daae..0f1c1b4fc553978ca49f2c7ee903c8d822adc03e 100644
--- a/lib/Target/ARM/AsmParser/LLVMBuild.txt
+++ b/lib/Target/ARM/AsmParser/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/Target/ARM/AsmParser/LLVMBuild.txt -----------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Target/ARM/CMakeLists.txt b/lib/Target/ARM/CMakeLists.txt
index d635c0add577abbec7970de8dba2d729dacc5506..2b198827fccbbb9b79ae46f6eb45ac6b68a71d2d 100644
--- a/lib/Target/ARM/CMakeLists.txt
+++ b/lib/Target/ARM/CMakeLists.txt
@@ -22,6 +22,7 @@ add_llvm_target(ARMCodeGen
   ARMAsmPrinter.cpp
   ARMBaseInstrInfo.cpp
   ARMBaseRegisterInfo.cpp
+  ARMCallingConv.cpp
   ARMCallLowering.cpp
   ARMCodeGenPrepare.cpp
   ARMConstantIslandPass.cpp
diff --git a/lib/Target/ARM/Disassembler/ARMDisassembler.cpp b/lib/Target/ARM/Disassembler/ARMDisassembler.cpp
index 61bec04678dd4a0619301eeee543a2daa9ee602b..1ac1ade2f7e283d1a055bc6f72c2c1f1b393b0ae 100644
--- a/lib/Target/ARM/Disassembler/ARMDisassembler.cpp
+++ b/lib/Target/ARM/Disassembler/ARMDisassembler.cpp
@@ -1,9 +1,8 @@
 //===- ARMDisassembler.cpp - Disassembler for ARM/Thumb ISA ---------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/ARM/Disassembler/LLVMBuild.txt b/lib/Target/ARM/Disassembler/LLVMBuild.txt
index 48eef05e4f2d34740256af74c1283331f2396c67..9a5c3a8a195cf3d977152b0dd2235dfafe02519a 100644
--- a/lib/Target/ARM/Disassembler/LLVMBuild.txt
+++ b/lib/Target/ARM/Disassembler/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/Target/ARM/Disassembler/LLVMBuild.txt --------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Target/ARM/InstPrinter/ARMInstPrinter.cpp b/lib/Target/ARM/InstPrinter/ARMInstPrinter.cpp
index 2f84719c4c4f35635f960f1e2ac46634766e9d7c..ee909f46af23f45d4a0bf9f9de76447b1e147dc8 100644
--- a/lib/Target/ARM/InstPrinter/ARMInstPrinter.cpp
+++ b/lib/Target/ARM/InstPrinter/ARMInstPrinter.cpp
@@ -1,9 +1,8 @@
 //===-- ARMInstPrinter.cpp - Convert ARM MCInst to assembly syntax --------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/ARM/InstPrinter/ARMInstPrinter.h b/lib/Target/ARM/InstPrinter/ARMInstPrinter.h
index afc8515136bc985539469952abc994c82d27bafa..b226633b0ef5539a212e51bf78617c0532204d50 100644
--- a/lib/Target/ARM/InstPrinter/ARMInstPrinter.h
+++ b/lib/Target/ARM/InstPrinter/ARMInstPrinter.h
@@ -1,9 +1,8 @@
 //===- ARMInstPrinter.h - Convert ARM MCInst to assembly syntax -*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/ARM/InstPrinter/LLVMBuild.txt b/lib/Target/ARM/InstPrinter/LLVMBuild.txt
index 75d700715a5266e49836e9567d9c35ca8ce1ed02..544d8ef9e42b6c2b2aec58e0245f65f18edfd2b9 100644
--- a/lib/Target/ARM/InstPrinter/LLVMBuild.txt
+++ b/lib/Target/ARM/InstPrinter/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/Target/ARM/InstPrinter/LLVMBuild.txt ---------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Target/ARM/LICENSE.TXT b/lib/Target/ARM/LICENSE.TXT
deleted file mode 100644
index 68afea12ed44a79db2b4126fef910771d5b68673..0000000000000000000000000000000000000000
--- a/lib/Target/ARM/LICENSE.TXT
+++ /dev/null
@@ -1,47 +0,0 @@
-ARM Limited
-
-Software Grant License Agreement ("Agreement")
-
-Except for the license granted herein to you, ARM Limited ("ARM") reserves all
-right, title, and interest in and to the Software (defined below).
-
-Definition
-
-"Software" means the code and documentation as well as any original work of
-authorship, including any modifications or additions to an existing work, that
-is intentionally submitted by ARM to llvm.org (http://llvm.org) ("LLVM") for
-inclusion in, or documentation of, any of the products owned or managed by LLVM
-(the "Work"). For the purposes of this definition, "submitted" means any form of
-electronic, verbal, or written communication sent to LLVM or its
-representatives, including but not limited to communication on electronic
-mailing lists, source code control systems, and issue tracking systems that are
-managed by, or on behalf of, LLVM for the purpose of discussing and improving
-the Work, but excluding communication that is conspicuously marked otherwise.
-
-1. Grant of Copyright License. Subject to the terms and conditions of this
-   Agreement, ARM hereby grants to you and to recipients of the Software
-   distributed by LLVM a perpetual, worldwide, non-exclusive, no-charge,
-   royalty-free, irrevocable copyright license to reproduce, prepare derivative
-   works of, publicly display, publicly perform, sublicense, and distribute the
-   Software and such derivative works.
-
-2. Grant of Patent License. Subject to the terms and conditions of this
-   Agreement, ARM hereby grants you and to recipients of the Software
-   distributed by LLVM a perpetual, worldwide, non-exclusive, no-charge,
-   royalty-free, irrevocable (except as stated in this section) patent license
-   to make, have made, use, offer to sell, sell, import, and otherwise transfer
-   the Work, where such license applies only to those patent claims licensable
-   by ARM that are necessarily infringed by ARM's Software alone or by
-   combination of the Software with the Work to which such Software was
-   submitted. If any entity institutes patent litigation against ARM or any
-   other entity (including a cross-claim or counterclaim in a lawsuit) alleging
-   that ARM's Software, or the Work to which ARM has contributed constitutes
-   direct or contributory patent infringement, then any patent licenses granted
-   to that entity under this Agreement for the Software or Work shall terminate
-   as of the date such litigation is filed.
-
-Unless required by applicable law or agreed to in writing, the software is
-provided on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
-either express or implied, including, without limitation, any warranties or
-conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
-PARTICULAR PURPOSE.
diff --git a/lib/Target/ARM/LLVMBuild.txt b/lib/Target/ARM/LLVMBuild.txt
index 78d28427f3d995d2ae2074a6a5974ad951023e6e..57b942ac91eb73d122c376ab9d30425e7c82d455 100644
--- a/lib/Target/ARM/LLVMBuild.txt
+++ b/lib/Target/ARM/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/Target/ARM/LLVMBuild.txt ---------------------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Target/ARM/MCTargetDesc/ARMAddressingModes.h b/lib/Target/ARM/MCTargetDesc/ARMAddressingModes.h
index e1ea5964cf674ab0ea6a9f0cdc0601e15694232d..c70c2cf8f07872a53b30bfc960d1289044cefed7 100644
--- a/lib/Target/ARM/MCTargetDesc/ARMAddressingModes.h
+++ b/lib/Target/ARM/MCTargetDesc/ARMAddressingModes.h
@@ -1,9 +1,8 @@
 //===-- ARMAddressingModes.h - ARM Addressing Modes -------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/ARM/MCTargetDesc/ARMAsmBackend.cpp b/lib/Target/ARM/MCTargetDesc/ARMAsmBackend.cpp
index c2a07d4ddceffc5c87d749a3c98118bc809e708f..b846ca0699b77e1ab3a3ee1415c32f9b1dd93481 100644
--- a/lib/Target/ARM/MCTargetDesc/ARMAsmBackend.cpp
+++ b/lib/Target/ARM/MCTargetDesc/ARMAsmBackend.cpp
@@ -1,9 +1,8 @@
 //===-- ARMAsmBackend.cpp - ARM Assembler Backend -------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/ARM/MCTargetDesc/ARMAsmBackend.h b/lib/Target/ARM/MCTargetDesc/ARMAsmBackend.h
index 88c476bf65f4ec23db6fa89349dbf7ad60a452ba..52abc2f20e0692ea15b67a211f035a0392922d08 100644
--- a/lib/Target/ARM/MCTargetDesc/ARMAsmBackend.h
+++ b/lib/Target/ARM/MCTargetDesc/ARMAsmBackend.h
@@ -1,9 +1,8 @@
 //===-- ARMAsmBackend.h - ARM Assembler Backend -----------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/ARM/MCTargetDesc/ARMAsmBackendDarwin.h b/lib/Target/ARM/MCTargetDesc/ARMAsmBackendDarwin.h
index de1bfaf203e47368fb82840a2252c72ebad0a8c0..87e56940f46db55549f5f768aa870f181c1b9ca0 100644
--- a/lib/Target/ARM/MCTargetDesc/ARMAsmBackendDarwin.h
+++ b/lib/Target/ARM/MCTargetDesc/ARMAsmBackendDarwin.h
@@ -1,9 +1,8 @@
 //===-- ARMAsmBackendDarwin.h   ARM Asm Backend Darwin ----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/ARM/MCTargetDesc/ARMAsmBackendELF.h b/lib/Target/ARM/MCTargetDesc/ARMAsmBackendELF.h
index 86a583b19cf7a922fc95e5c72754320f9a6d6d0f..5d735114d44146b13651f904172ed9d102eb125b 100644
--- a/lib/Target/ARM/MCTargetDesc/ARMAsmBackendELF.h
+++ b/lib/Target/ARM/MCTargetDesc/ARMAsmBackendELF.h
@@ -1,9 +1,8 @@
 //===-- ARMAsmBackendELF.h  ARM Asm Backend ELF -----------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/ARM/MCTargetDesc/ARMAsmBackendWinCOFF.h b/lib/Target/ARM/MCTargetDesc/ARMAsmBackendWinCOFF.h
index 553922d20f4345ed8dca98b45f74e78f97bd01b9..8cd7a4a00ead3fc67286647f4a5c7f02de26aee6 100644
--- a/lib/Target/ARM/MCTargetDesc/ARMAsmBackendWinCOFF.h
+++ b/lib/Target/ARM/MCTargetDesc/ARMAsmBackendWinCOFF.h
@@ -1,9 +1,8 @@
 //===-- ARMAsmBackendWinCOFF.h - ARM Asm Backend WinCOFF --------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/ARM/MCTargetDesc/ARMBaseInfo.h b/lib/Target/ARM/MCTargetDesc/ARMBaseInfo.h
index 33c32d5464af6b998fd5e6c417c1feda56559603..21a7f2dfb33d11666e38ead3db2e22071bff2560 100644
--- a/lib/Target/ARM/MCTargetDesc/ARMBaseInfo.h
+++ b/lib/Target/ARM/MCTargetDesc/ARMBaseInfo.h
@@ -1,9 +1,8 @@
 //===-- ARMBaseInfo.h - Top level definitions for ARM -------- --*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/ARM/MCTargetDesc/ARMELFObjectWriter.cpp b/lib/Target/ARM/MCTargetDesc/ARMELFObjectWriter.cpp
index b8ba7584911bb79010bf704f7f059e2e623f3ffd..a93202b82022bdb9e05e534e6071afc5a23a7a4d 100644
--- a/lib/Target/ARM/MCTargetDesc/ARMELFObjectWriter.cpp
+++ b/lib/Target/ARM/MCTargetDesc/ARMELFObjectWriter.cpp
@@ -1,9 +1,8 @@
 //===-- ARMELFObjectWriter.cpp - ARM ELF Writer ---------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/ARM/MCTargetDesc/ARMELFStreamer.cpp b/lib/Target/ARM/MCTargetDesc/ARMELFStreamer.cpp
index d3744fffac3228da7787f75c07e9b7622288224a..f51fbdcd84da7c62bdca9ee72c6c6b307e12a9e4 100644
--- a/lib/Target/ARM/MCTargetDesc/ARMELFStreamer.cpp
+++ b/lib/Target/ARM/MCTargetDesc/ARMELFStreamer.cpp
@@ -1,9 +1,8 @@
 //===- lib/MC/ARMELFStreamer.cpp - ELF Object Output for ARM --------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -485,8 +484,8 @@ public:
   /// This function is the one used to emit instruction data into the ELF
   /// streamer. We override it to add the appropriate mapping symbol if
   /// necessary.
-  void EmitInstruction(const MCInst &Inst, const MCSubtargetInfo &STI,
-                       bool) override {
+  void EmitInstruction(const MCInst &Inst,
+                       const MCSubtargetInfo &STI) override {
     if (IsThumb)
       EmitThumbMappingSymbol();
     else
diff --git a/lib/Target/ARM/MCTargetDesc/ARMFixupKinds.h b/lib/Target/ARM/MCTargetDesc/ARMFixupKinds.h
index 831589ba0581a42ccbc9b66d8df983ad36bd818c..533bf6d4df99b4c83e34d241a434b45328ceb877 100644
--- a/lib/Target/ARM/MCTargetDesc/ARMFixupKinds.h
+++ b/lib/Target/ARM/MCTargetDesc/ARMFixupKinds.h
@@ -1,9 +1,8 @@
 //===-- ARMFixupKinds.h - ARM Specific Fixup Entries ------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/ARM/MCTargetDesc/ARMMCAsmInfo.cpp b/lib/Target/ARM/MCTargetDesc/ARMMCAsmInfo.cpp
index 3ee63ac374b3392c6ba7a61eef7e4c708584226f..d30d15df3d001925c710891784c1dfc7d98cedc2 100644
--- a/lib/Target/ARM/MCTargetDesc/ARMMCAsmInfo.cpp
+++ b/lib/Target/ARM/MCTargetDesc/ARMMCAsmInfo.cpp
@@ -1,9 +1,8 @@
 //===-- ARMMCAsmInfo.cpp - ARM asm properties -----------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/ARM/MCTargetDesc/ARMMCAsmInfo.h b/lib/Target/ARM/MCTargetDesc/ARMMCAsmInfo.h
index 5e548162bec61b8c5e7edb1a78489c0e117ae223..55d7b299674d39447be2e19042848834d9d9cb69 100644
--- a/lib/Target/ARM/MCTargetDesc/ARMMCAsmInfo.h
+++ b/lib/Target/ARM/MCTargetDesc/ARMMCAsmInfo.h
@@ -1,9 +1,8 @@
 //===-- ARMMCAsmInfo.h - ARM asm properties --------------------*- C++ -*--===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/ARM/MCTargetDesc/ARMMCCodeEmitter.cpp b/lib/Target/ARM/MCTargetDesc/ARMMCCodeEmitter.cpp
index b37b8073548f44d4221d855a0a58a643c84528b8..5dd8a6957693304c892a1d15bc6309b013f4a36b 100644
--- a/lib/Target/ARM/MCTargetDesc/ARMMCCodeEmitter.cpp
+++ b/lib/Target/ARM/MCTargetDesc/ARMMCCodeEmitter.cpp
@@ -1,9 +1,8 @@
 //===-- ARM/ARMMCCodeEmitter.cpp - Convert ARM code to machine code -------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/ARM/MCTargetDesc/ARMMCExpr.cpp b/lib/Target/ARM/MCTargetDesc/ARMMCExpr.cpp
index 306f068312f538569cc1eb9e394f00c00a5f065c..fbad05fb1759fece1fccf417c93e750ebc2e7910 100644
--- a/lib/Target/ARM/MCTargetDesc/ARMMCExpr.cpp
+++ b/lib/Target/ARM/MCTargetDesc/ARMMCExpr.cpp
@@ -1,9 +1,8 @@
 //===-- ARMMCExpr.cpp - ARM specific MC expression classes ----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/ARM/MCTargetDesc/ARMMCExpr.h b/lib/Target/ARM/MCTargetDesc/ARMMCExpr.h
index 75dde8008fca0613978e56b601fa829971fcc23a..033a43288f3eed33ddbc126aafc12ddf5c75af01 100644
--- a/lib/Target/ARM/MCTargetDesc/ARMMCExpr.h
+++ b/lib/Target/ARM/MCTargetDesc/ARMMCExpr.h
@@ -1,9 +1,8 @@
 //===-- ARMMCExpr.h - ARM specific MC expression classes --------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/ARM/MCTargetDesc/ARMMCTargetDesc.cpp b/lib/Target/ARM/MCTargetDesc/ARMMCTargetDesc.cpp
index 46434007a8545ff0d0ee565b8945c5048f99024c..9f2672f912f86e5b581a506125d6ee37b4442774 100644
--- a/lib/Target/ARM/MCTargetDesc/ARMMCTargetDesc.cpp
+++ b/lib/Target/ARM/MCTargetDesc/ARMMCTargetDesc.cpp
@@ -1,9 +1,8 @@
 //===-- ARMMCTargetDesc.cpp - ARM Target Descriptions ---------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/ARM/MCTargetDesc/ARMMCTargetDesc.h b/lib/Target/ARM/MCTargetDesc/ARMMCTargetDesc.h
index 3ee004592ac6ce1685da61dcfa611e1cc857efc0..4966b4d94aba21c8bda136a4ea9dda1b6d48266e 100644
--- a/lib/Target/ARM/MCTargetDesc/ARMMCTargetDesc.h
+++ b/lib/Target/ARM/MCTargetDesc/ARMMCTargetDesc.h
@@ -1,9 +1,8 @@
 //===-- ARMMCTargetDesc.h - ARM Target Descriptions -------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/ARM/MCTargetDesc/ARMMachORelocationInfo.cpp b/lib/Target/ARM/MCTargetDesc/ARMMachORelocationInfo.cpp
index 6259c98321f4e72959812e29770ea22fcba70354..886b7e7bc84e8d96e6d0f63c2608ccbaca93fd13 100644
--- a/lib/Target/ARM/MCTargetDesc/ARMMachORelocationInfo.cpp
+++ b/lib/Target/ARM/MCTargetDesc/ARMMachORelocationInfo.cpp
@@ -1,9 +1,8 @@
 //===- ARMMachORelocationInfo.cpp -----------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/ARM/MCTargetDesc/ARMMachObjectWriter.cpp b/lib/Target/ARM/MCTargetDesc/ARMMachObjectWriter.cpp
index 0ced8195790d6c5ba78cd0fde12f08e9648b31c4..c49885023cb27816755af0573b430d4ba0f7ec9f 100644
--- a/lib/Target/ARM/MCTargetDesc/ARMMachObjectWriter.cpp
+++ b/lib/Target/ARM/MCTargetDesc/ARMMachObjectWriter.cpp
@@ -1,9 +1,8 @@
 //===-- ARMMachObjectWriter.cpp - ARM Mach Object Writer ------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/ARM/MCTargetDesc/ARMTargetStreamer.cpp b/lib/Target/ARM/MCTargetDesc/ARMTargetStreamer.cpp
index 91836cff95c832609b6ffd283b0a0bc7fcdc2dd7..8f9c66507a47a6c41002987457d53a2128162a34 100644
--- a/lib/Target/ARM/MCTargetDesc/ARMTargetStreamer.cpp
+++ b/lib/Target/ARM/MCTargetDesc/ARMTargetStreamer.cpp
@@ -1,9 +1,8 @@
 //===- ARMTargetStreamer.cpp - ARMTargetStreamer class --*- C++ -*---------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/ARM/MCTargetDesc/ARMUnwindOpAsm.cpp b/lib/Target/ARM/MCTargetDesc/ARMUnwindOpAsm.cpp
index d3ab83bbccbc264d8db78d68276bc0ef0a624065..38667d686b851badd2c5fd7b0b315719877df280 100644
--- a/lib/Target/ARM/MCTargetDesc/ARMUnwindOpAsm.cpp
+++ b/lib/Target/ARM/MCTargetDesc/ARMUnwindOpAsm.cpp
@@ -1,9 +1,8 @@
 //===-- ARMUnwindOpAsm.cpp - ARM Unwind Opcodes Assembler -------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/ARM/MCTargetDesc/ARMUnwindOpAsm.h b/lib/Target/ARM/MCTargetDesc/ARMUnwindOpAsm.h
index a7bfbdf4938e9da074a9ca2dd12053928c8bf8a4..c3134c04b33a0d61e1978dd03b8d8b969e1c5f84 100644
--- a/lib/Target/ARM/MCTargetDesc/ARMUnwindOpAsm.h
+++ b/lib/Target/ARM/MCTargetDesc/ARMUnwindOpAsm.h
@@ -1,9 +1,8 @@
 //===-- ARMUnwindOpAsm.h - ARM Unwind Opcodes Assembler ---------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/ARM/MCTargetDesc/ARMWinCOFFObjectWriter.cpp b/lib/Target/ARM/MCTargetDesc/ARMWinCOFFObjectWriter.cpp
index 30cbde1ca71fa2fd569d7f3c260a38075ca02a6c..054a95dd1e12308ab4f76e100dd4418ed30ed573 100644
--- a/lib/Target/ARM/MCTargetDesc/ARMWinCOFFObjectWriter.cpp
+++ b/lib/Target/ARM/MCTargetDesc/ARMWinCOFFObjectWriter.cpp
@@ -1,9 +1,8 @@
 //===-- ARMWinCOFFObjectWriter.cpp - ARM Windows COFF Object Writer -- C++ -==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/ARM/MCTargetDesc/ARMWinCOFFStreamer.cpp b/lib/Target/ARM/MCTargetDesc/ARMWinCOFFStreamer.cpp
index 32cb3dcdcad8a6a8cd1f16a9c02f339411bc8151..2e816bea5e918e9279030f2b83ec656e936ba08f 100644
--- a/lib/Target/ARM/MCTargetDesc/ARMWinCOFFStreamer.cpp
+++ b/lib/Target/ARM/MCTargetDesc/ARMWinCOFFStreamer.cpp
@@ -1,9 +1,8 @@
 //===-- ARMWinCOFFStreamer.cpp - ARM Target WinCOFF Streamer ----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/ARM/MCTargetDesc/LLVMBuild.txt b/lib/Target/ARM/MCTargetDesc/LLVMBuild.txt
index db8fc925edb894a3bbaabc38a13fd2c7a643e2b1..fd99351f80352596fdea2616c5f4af0f8b9a8a1d 100644
--- a/lib/Target/ARM/MCTargetDesc/LLVMBuild.txt
+++ b/lib/Target/ARM/MCTargetDesc/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/Target/ARM/MCTargetDesc/LLVMBuild.txt --------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Target/ARM/MLxExpansionPass.cpp b/lib/Target/ARM/MLxExpansionPass.cpp
index 7f03e1463c1d6ea40b15c22c76c0c5e4f25d1b3f..4b25986b90a7550a11455e99a2083c12c936d0e1 100644
--- a/lib/Target/ARM/MLxExpansionPass.cpp
+++ b/lib/Target/ARM/MLxExpansionPass.cpp
@@ -1,9 +1,8 @@
 //===-- MLxExpansionPass.cpp - Expand MLx instrs to avoid hazards ---------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/ARM/TargetInfo/ARMTargetInfo.cpp b/lib/Target/ARM/TargetInfo/ARMTargetInfo.cpp
index b0491a4108a62eb7b0683838b123ead008d0f9bf..c75bd361aceef305719b9eaf018972c8a302ea1a 100644
--- a/lib/Target/ARM/TargetInfo/ARMTargetInfo.cpp
+++ b/lib/Target/ARM/TargetInfo/ARMTargetInfo.cpp
@@ -1,9 +1,8 @@
 //===-- ARMTargetInfo.cpp - ARM Target Implementation ---------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/ARM/TargetInfo/LLVMBuild.txt b/lib/Target/ARM/TargetInfo/LLVMBuild.txt
index cce6cc7695592deae25ab9e95cee782c1bcfebf0..e635aee253edc8a7c2610babc543d48ca3d6d370 100644
--- a/lib/Target/ARM/TargetInfo/LLVMBuild.txt
+++ b/lib/Target/ARM/TargetInfo/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/Target/ARM/TargetInfo/LLVMBuild.txt ----------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Target/ARM/Thumb1FrameLowering.cpp b/lib/Target/ARM/Thumb1FrameLowering.cpp
index 5c745e112b2e1f9a25f2c3a9311373eb76443e44..077f6eeff137893e416230b8f0a4edac5aefd8cc 100644
--- a/lib/Target/ARM/Thumb1FrameLowering.cpp
+++ b/lib/Target/ARM/Thumb1FrameLowering.cpp
@@ -1,9 +1,8 @@
 //===- Thumb1FrameLowering.cpp - Thumb1 Frame Information -----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/ARM/Thumb1FrameLowering.h b/lib/Target/ARM/Thumb1FrameLowering.h
index a4d6451ccf125d8b1096beeefc51b763eda6092c..61af48712b6cf64c9b49838cdd943c8c7b4bb554 100644
--- a/lib/Target/ARM/Thumb1FrameLowering.h
+++ b/lib/Target/ARM/Thumb1FrameLowering.h
@@ -1,9 +1,8 @@
 //===- Thumb1FrameLowering.h - Thumb1-specific frame info stuff ---*- C++ -*-=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/ARM/Thumb1InstrInfo.cpp b/lib/Target/ARM/Thumb1InstrInfo.cpp
index 11aa285fc9390b6db3cdb7157a8677b03f86808e..f57d93a2e83df154ebe242d0d50c4cfca4bb62ec 100644
--- a/lib/Target/ARM/Thumb1InstrInfo.cpp
+++ b/lib/Target/ARM/Thumb1InstrInfo.cpp
@@ -1,9 +1,8 @@
 //===-- Thumb1InstrInfo.cpp - Thumb-1 Instruction Information -------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/ARM/Thumb1InstrInfo.h b/lib/Target/ARM/Thumb1InstrInfo.h
index 9f04a3ed262fbdbaabba72d06cf8efb2cae47e06..bc433e7a7a931377c51efe5c5220783985e16fe3 100644
--- a/lib/Target/ARM/Thumb1InstrInfo.h
+++ b/lib/Target/ARM/Thumb1InstrInfo.h
@@ -1,9 +1,8 @@
 //===-- Thumb1InstrInfo.h - Thumb-1 Instruction Information -----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/ARM/Thumb2ITBlockPass.cpp b/lib/Target/ARM/Thumb2ITBlockPass.cpp
index e0a5f7f04fa92e39e9c9e9fbf49c99a94a74625a..70c07f50b032aea35a54f512ebdb29dd13897ae1 100644
--- a/lib/Target/ARM/Thumb2ITBlockPass.cpp
+++ b/lib/Target/ARM/Thumb2ITBlockPass.cpp
@@ -1,9 +1,8 @@
 //===-- Thumb2ITBlockPass.cpp - Insert Thumb-2 IT blocks ------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/ARM/Thumb2InstrInfo.cpp b/lib/Target/ARM/Thumb2InstrInfo.cpp
index d567d333904922f285b765ba6581decab970d920..1ed35fb3d3833af5e36203360504db665fbca7d2 100644
--- a/lib/Target/ARM/Thumb2InstrInfo.cpp
+++ b/lib/Target/ARM/Thumb2InstrInfo.cpp
@@ -1,9 +1,8 @@
 //===- Thumb2InstrInfo.cpp - Thumb-2 Instruction Information --------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/ARM/Thumb2InstrInfo.h b/lib/Target/ARM/Thumb2InstrInfo.h
index c834ba73bfea7392fb7dddf3a2186e72743fb0ca..c5aca82acc26799fe0f8b5340fba27b1ae922879 100644
--- a/lib/Target/ARM/Thumb2InstrInfo.h
+++ b/lib/Target/ARM/Thumb2InstrInfo.h
@@ -1,9 +1,8 @@
 //===-- Thumb2InstrInfo.h - Thumb-2 Instruction Information -----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/ARM/Thumb2SizeReduction.cpp b/lib/Target/ARM/Thumb2SizeReduction.cpp
index 2f510c52640e5e97f4da0a30dec0b7aa1e31a537..be9c1ebd69f9c08f450a8918819ddf616338aa24 100644
--- a/lib/Target/ARM/Thumb2SizeReduction.cpp
+++ b/lib/Target/ARM/Thumb2SizeReduction.cpp
@@ -1,9 +1,8 @@
 //===-- Thumb2SizeReduction.cpp - Thumb2 code size reduction pass -*- C++ -*-=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -122,6 +121,7 @@ namespace {
   { ARM::t2SUBSrr,ARM::tSUBrr,  0,             0,   0,   1,   0,  2,0, 0,0,0 },
   { ARM::t2SXTB,  ARM::tSXTB,   0,             0,   0,   1,   0,  1,0, 0,1,0 },
   { ARM::t2SXTH,  ARM::tSXTH,   0,             0,   0,   1,   0,  1,0, 0,1,0 },
+  { ARM::t2TEQrr, ARM::tEOR,    0,             0,   0,   1,   0,  2,0, 0,1,0 },
   { ARM::t2TSTrr, ARM::tTST,    0,             0,   0,   1,   0,  2,0, 0,0,0 },
   { ARM::t2UXTB,  ARM::tUXTB,   0,             0,   0,   1,   0,  1,0, 0,1,0 },
   { ARM::t2UXTH,  ARM::tUXTH,   0,             0,   0,   1,   0,  1,0, 0,1,0 },
@@ -453,7 +453,7 @@ Thumb2SizeReduce::ReduceLoadStore(MachineBasicBlock &MBB, MachineInstr *MI,
     break;
   case ARM::t2LDR_POST:
   case ARM::t2STR_POST: {
-    if (!MBB.getParent()->getFunction().optForMinSize())
+    if (!MinimizeSize)
       return false;
 
     if (!MI->hasOneMemOperand() ||
@@ -717,6 +717,16 @@ Thumb2SizeReduce::ReduceSpecial(MachineBasicBlock &MBB, MachineInstr *MI,
       return true;
     return ReduceToNarrow(MBB, MI, Entry, LiveCPSR, IsSelfLoop);
   }
+  case ARM::t2TEQrr: {
+    unsigned PredReg = 0;
+    // Can only convert to eors if we're not in an IT block.
+    if (getInstrPredicate(*MI, PredReg) != ARMCC::AL)
+      break;
+    // TODO if Operand 0 is not killed but Operand 1 is, then we could write
+    // to Op1 instead.
+    if (MI->getOperand(0).isKill())
+      return ReduceToNarrow(MBB, MI, Entry, LiveCPSR, IsSelfLoop);
+  }
   }
   return false;
 }
@@ -903,9 +913,24 @@ Thumb2SizeReduce::ReduceToNarrow(MachineBasicBlock &MBB, MachineInstr *MI,
   // Add the 16-bit instruction.
   DebugLoc dl = MI->getDebugLoc();
   MachineInstrBuilder MIB = BuildMI(MBB, MI, dl, NewMCID);
-  MIB.add(MI->getOperand(0));
-  if (NewMCID.hasOptionalDef())
-    MIB.add(HasCC ? t1CondCodeOp(CCDead) : condCodeOp());
+
+  // TEQ is special in that it doesn't define a register but we're converting
+  // it into an EOR which does. So add the first operand as a def and then
+  // again as a use.
+  if (MCID.getOpcode() == ARM::t2TEQrr) {
+    MIB.add(MI->getOperand(0));
+    MIB->getOperand(0).setIsKill(false);
+    MIB->getOperand(0).setIsDef(true);
+    MIB->getOperand(0).setIsDead(true);
+
+    if (NewMCID.hasOptionalDef())
+      MIB.add(HasCC ? t1CondCodeOp(CCDead) : condCodeOp());
+    MIB.add(MI->getOperand(0));
+  } else {
+    MIB.add(MI->getOperand(0));
+    if (NewMCID.hasOptionalDef())
+      MIB.add(HasCC ? t1CondCodeOp(CCDead) : condCodeOp());
+  }
 
   // Transfer the rest of operands.
   unsigned NumOps = MCID.getNumOperands();
@@ -1103,7 +1128,7 @@ bool Thumb2SizeReduce::runOnMachineFunction(MachineFunction &MF) {
 
   // Optimizing / minimizing size? Minimizing size implies optimizing for size.
   OptimizeSize = MF.getFunction().optForSize();
-  MinimizeSize = MF.getFunction().optForMinSize();
+  MinimizeSize = STI->optForMinSize();
 
   BlockInfo.clear();
   BlockInfo.resize(MF.getNumBlockIDs());
diff --git a/lib/Target/ARM/ThumbRegisterInfo.cpp b/lib/Target/ARM/ThumbRegisterInfo.cpp
index e4bdd40fb7435756050bb67a880ecc9ebe1595b4..e485c79d3f3d960f7db24ac64db319ba76b28659 100644
--- a/lib/Target/ARM/ThumbRegisterInfo.cpp
+++ b/lib/Target/ARM/ThumbRegisterInfo.cpp
@@ -1,9 +1,8 @@
 //===-- ThumbRegisterInfo.cpp - Thumb-1 Register Information -------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/ARM/ThumbRegisterInfo.h b/lib/Target/ARM/ThumbRegisterInfo.h
index 75c3fe9ae8ad57abb640764e95111bfbd69ce899..10017a7a0c6abeab44c86f0daec9a001fab1b6fb 100644
--- a/lib/Target/ARM/ThumbRegisterInfo.h
+++ b/lib/Target/ARM/ThumbRegisterInfo.h
@@ -1,9 +1,8 @@
 //===- ThumbRegisterInfo.h - Thumb Register Information Impl -*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/ARM/Utils/ARMBaseInfo.cpp b/lib/Target/ARM/Utils/ARMBaseInfo.cpp
index 534f78c6d4d2c1a9fa21eaf1592c8c4c7e205d9b..4ace61cccd0f66447375ec16a10d1b58bc49b55c 100644
--- a/lib/Target/ARM/Utils/ARMBaseInfo.cpp
+++ b/lib/Target/ARM/Utils/ARMBaseInfo.cpp
@@ -1,9 +1,8 @@
 //===-- ARMBaseInfo.cpp - ARM Base encoding information------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/ARM/Utils/ARMBaseInfo.h b/lib/Target/ARM/Utils/ARMBaseInfo.h
index f32d8223f53cee25371e110def5b2ac100536e51..bd946dce8f1e78e9f2f4343c8169fc01f2838c56 100644
--- a/lib/Target/ARM/Utils/ARMBaseInfo.h
+++ b/lib/Target/ARM/Utils/ARMBaseInfo.h
@@ -1,9 +1,8 @@
 //===-- ARMBaseInfo.h - Top level definitions for ARM ---*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/ARM/Utils/LLVMBuild.txt b/lib/Target/ARM/Utils/LLVMBuild.txt
index bbbaef6332135cdb78d047632ce5eddb99205d29..004f7768d4df86f3e4f3ba2f690e35379107120d 100644
--- a/lib/Target/ARM/Utils/LLVMBuild.txt
+++ b/lib/Target/ARM/Utils/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/Target/ARM/Utils/LLVMBuild.txt ----------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Target/AVR/AVR.h b/lib/Target/AVR/AVR.h
index 48327fd377b21e47bba00572e2780f509af1ce07..f0746d73c95f477f92433a14d1e88bbb848e212c 100644
--- a/lib/Target/AVR/AVR.h
+++ b/lib/Target/AVR/AVR.h
@@ -1,9 +1,8 @@
 //===-- AVR.h - Top-level interface for AVR representation ------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AVR/AVR.td b/lib/Target/AVR/AVR.td
index d03b983aa70bef4f5627976a4a1c2874792372f0..53768f99df3b492783be36e205679b282382fce5 100644
--- a/lib/Target/AVR/AVR.td
+++ b/lib/Target/AVR/AVR.td
@@ -1,9 +1,8 @@
 //===-- AVR.td - Describe the AVR Target Machine ----------*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===---------------------------------------------------------------------===//
 // This is the top level entry point for the AVR target.
diff --git a/lib/Target/AVR/AVRAsmPrinter.cpp b/lib/Target/AVR/AVRAsmPrinter.cpp
index f9a6e77387b25afd8856e9696b4918f1ada1d8cb..f96f58860c77b0fbcb5d1341e9e931d66a141f41 100644
--- a/lib/Target/AVR/AVRAsmPrinter.cpp
+++ b/lib/Target/AVR/AVRAsmPrinter.cpp
@@ -1,9 +1,8 @@
 //===-- AVRAsmPrinter.cpp - AVR LLVM assembly writer ----------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AVR/AVRCallingConv.td b/lib/Target/AVR/AVRCallingConv.td
index 68dbce02706ff6c790b9f39222cef4817f2e2b56..213e35fca66d1c7b3cbf18f7c49bde83c94d1c1d 100644
--- a/lib/Target/AVR/AVRCallingConv.td
+++ b/lib/Target/AVR/AVRCallingConv.td
@@ -1,9 +1,8 @@
 //===-- AVRCallingConv.td - Calling Conventions for AVR ----*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 // This describes the calling conventions for AVR architecture.
diff --git a/lib/Target/AVR/AVRExpandPseudoInsts.cpp b/lib/Target/AVR/AVRExpandPseudoInsts.cpp
index 536a54759c777f9aa499da14abbdc33bf969a52e..c45b2d0e39c1a4f8771e75825ace2bbf4015a976 100644
--- a/lib/Target/AVR/AVRExpandPseudoInsts.cpp
+++ b/lib/Target/AVR/AVRExpandPseudoInsts.cpp
@@ -1,9 +1,8 @@
 //===-- AVRExpandPseudoInsts.cpp - Expand pseudo instructions -------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -583,8 +582,8 @@ bool AVRExpandPseudo::expand<AVR::LDWRdPtr>(Block &MBB, BlockIt MBBI) {
   unsigned TmpReg = 0; // 0 for no temporary register
   unsigned SrcReg = MI.getOperand(1).getReg();
   bool SrcIsKill = MI.getOperand(1).isKill();
-  OpLo = AVR::LDRdPtrPi;
-  OpHi = AVR::LDRdPtr;
+  OpLo = AVR::LDRdPtr;
+  OpHi = AVR::LDDRdPtrQ;
   TRI->splitReg(DstReg, DstLoReg, DstHiReg);
 
   // Use a temporary register if src and dst registers are the same.
@@ -597,8 +596,7 @@ bool AVRExpandPseudo::expand<AVR::LDWRdPtr>(Block &MBB, BlockIt MBBI) {
   // Load low byte.
   auto MIBLO = buildMI(MBB, MBBI, OpLo)
     .addReg(CurDstLoReg, RegState::Define)
-    .addReg(SrcReg, RegState::Define)
-    .addReg(SrcReg);
+    .addReg(SrcReg, RegState::Define);
 
   // Push low byte onto stack if necessary.
   if (TmpReg)
@@ -607,7 +605,8 @@ bool AVRExpandPseudo::expand<AVR::LDWRdPtr>(Block &MBB, BlockIt MBBI) {
   // Load high byte.
   auto MIBHI = buildMI(MBB, MBBI, OpHi)
     .addReg(CurDstHiReg, RegState::Define)
-    .addReg(SrcReg, getKillRegState(SrcIsKill));
+    .addReg(SrcReg, getKillRegState(SrcIsKill))
+    .addImm(1);
 
   if (TmpReg) {
     // Move the high byte into the final destination.
diff --git a/lib/Target/AVR/AVRFrameLowering.cpp b/lib/Target/AVR/AVRFrameLowering.cpp
index 3b73223657727958c5cc90351bed5e948af9b4ea..f39575b0afeaa65fce0961a42de820563ed1df1e 100644
--- a/lib/Target/AVR/AVRFrameLowering.cpp
+++ b/lib/Target/AVR/AVRFrameLowering.cpp
@@ -1,9 +1,8 @@
 //===-- AVRFrameLowering.cpp - AVR Frame Information ----------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AVR/AVRFrameLowering.h b/lib/Target/AVR/AVRFrameLowering.h
index a0ba6c9512765417bac62ccbcbe4f4851c670489..a7658438232a4686dbedd4d8c6a50b58edab6ea0 100644
--- a/lib/Target/AVR/AVRFrameLowering.h
+++ b/lib/Target/AVR/AVRFrameLowering.h
@@ -1,9 +1,8 @@
 //===-- AVRFrameLowering.h - Define frame lowering for AVR ------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/AVR/AVRISelDAGToDAG.cpp b/lib/Target/AVR/AVRISelDAGToDAG.cpp
index 85abf42eaa6778a0dc2c2289c753c8cd691c74ef..5cb4441c438007704a4a007715b7cf7c7c53b1c1 100644
--- a/lib/Target/AVR/AVRISelDAGToDAG.cpp
+++ b/lib/Target/AVR/AVRISelDAGToDAG.cpp
@@ -1,9 +1,8 @@
 //===-- AVRISelDAGToDAG.cpp - A dag to dag inst selector for AVR ----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AVR/AVRISelLowering.cpp b/lib/Target/AVR/AVRISelLowering.cpp
index 57fc978b54bb61aec6ee39e867d549200239054e..e95a41472bff42d057968efa0b722f0e6b053229 100644
--- a/lib/Target/AVR/AVRISelLowering.cpp
+++ b/lib/Target/AVR/AVRISelLowering.cpp
@@ -1,9 +1,8 @@
 //===-- AVRISelLowering.cpp - AVR DAG Lowering Implementation -------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -26,19 +25,21 @@
 
 #include "AVR.h"
 #include "AVRMachineFunctionInfo.h"
+#include "AVRSubtarget.h"
 #include "AVRTargetMachine.h"
 #include "MCTargetDesc/AVRMCTargetDesc.h"
 
 namespace llvm {
 
-AVRTargetLowering::AVRTargetLowering(AVRTargetMachine &tm)
-    : TargetLowering(tm) {
+AVRTargetLowering::AVRTargetLowering(const AVRTargetMachine &TM,
+                                     const AVRSubtarget &STI)
+    : TargetLowering(TM), Subtarget(STI) {
   // Set up the register classes.
   addRegisterClass(MVT::i8, &AVR::GPR8RegClass);
   addRegisterClass(MVT::i16, &AVR::DREGSRegClass);
 
   // Compute derived properties from the register classes.
-  computeRegisterProperties(tm.getSubtargetImpl()->getRegisterInfo());
+  computeRegisterProperties(Subtarget.getRegisterInfo());
 
   setBooleanContents(ZeroOrOneBooleanContent);
   setBooleanVectorContents(ZeroOrOneBooleanContent);
@@ -163,6 +164,13 @@ AVRTargetLowering::AVRTargetLowering(AVRTargetMachine &tm)
   setOperationAction(ISD::SMUL_LOHI, MVT::i16, Expand);
   setOperationAction(ISD::UMUL_LOHI, MVT::i16, Expand);
 
+  // Expand multiplications to libcalls when there is
+  // no hardware MUL.
+  if (!Subtarget.supportsMultiplication()) {
+    setOperationAction(ISD::SMUL_LOHI, MVT::i8, Expand);
+    setOperationAction(ISD::UMUL_LOHI, MVT::i8, Expand);
+  }
+
   for (MVT VT : MVT::integer_valuetypes()) {
     setOperationAction(ISD::MULHS, VT, Expand);
     setOperationAction(ISD::MULHU, VT, Expand);
@@ -1270,8 +1278,7 @@ SDValue AVRTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
   }
 
   // Add a register mask operand representing the call-preserved registers.
-  const AVRTargetMachine &TM = (const AVRTargetMachine &)getTargetMachine();
-  const TargetRegisterInfo *TRI = TM.getSubtargetImpl()->getRegisterInfo();
+  const TargetRegisterInfo *TRI = Subtarget.getRegisterInfo();
   const uint32_t *Mask =
       TRI->getCallPreservedMask(DAG.getMachineFunction(), CallConv);
   assert(Mask && "Missing call preserved mask for calling convention");
@@ -1433,8 +1440,7 @@ MachineBasicBlock *AVRTargetLowering::insertShift(MachineInstr &MI,
   bool HasRepeatedOperand = false;
   MachineFunction *F = BB->getParent();
   MachineRegisterInfo &RI = F->getRegInfo();
-  const AVRTargetMachine &TM = (const AVRTargetMachine &)getTargetMachine();
-  const TargetInstrInfo &TII = *TM.getSubtargetImpl()->getInstrInfo();
+  const TargetInstrInfo &TII = *Subtarget.getInstrInfo();
   DebugLoc dl = MI.getDebugLoc();
 
   switch (MI.getOpcode()) {
@@ -1574,8 +1580,7 @@ static bool isCopyMulResult(MachineBasicBlock::iterator const &I) {
 // it, but it works for now.
 MachineBasicBlock *AVRTargetLowering::insertMul(MachineInstr &MI,
                                                 MachineBasicBlock *BB) const {
-  const AVRTargetMachine &TM = (const AVRTargetMachine &)getTargetMachine();
-  const TargetInstrInfo &TII = *TM.getSubtargetImpl()->getInstrInfo();
+  const TargetInstrInfo &TII = *Subtarget.getInstrInfo();
   MachineBasicBlock::iterator I(MI);
   ++I; // in any case insert *after* the mul instruction
   if (isCopyMulResult(I))
@@ -1629,6 +1634,15 @@ AVRTargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
 
   MachineFunction *MF = MBB->getParent();
   const BasicBlock *LLVM_BB = MBB->getBasicBlock();
+  MachineBasicBlock *FallThrough = MBB->getFallThrough();
+
+  // If the current basic block falls through to another basic block,
+  // we must insert an unconditional branch to the fallthrough destination
+  // if we are to insert basic blocks at the prior fallthrough point.
+  if (FallThrough != nullptr) {
+    BuildMI(MBB, dl, TII.get(AVR::RJMPk)).addMBB(FallThrough);
+  }
+
   MachineBasicBlock *trueMBB = MF->CreateMachineBasicBlock(LLVM_BB);
   MachineBasicBlock *falseMBB = MF->CreateMachineBasicBlock(LLVM_BB);
 
@@ -1838,9 +1852,6 @@ std::pair<unsigned, const TargetRegisterClass *>
 AVRTargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
                                                 StringRef Constraint,
                                                 MVT VT) const {
-  auto STI = static_cast<const AVRTargetMachine &>(this->getTargetMachine())
-                 .getSubtargetImpl();
-
   // We only support i8 and i16.
   //
   //:FIXME: remove this assert for now since it gets sometimes executed
@@ -1884,8 +1895,8 @@ AVRTargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
     }
   }
 
-  return TargetLowering::getRegForInlineAsmConstraint(STI->getRegisterInfo(),
-                                                      Constraint, VT);
+  return TargetLowering::getRegForInlineAsmConstraint(
+      Subtarget.getRegisterInfo(), Constraint, VT);
 }
 
 void AVRTargetLowering::LowerAsmOperandForConstraint(SDValue Op,
diff --git a/lib/Target/AVR/AVRISelLowering.h b/lib/Target/AVR/AVRISelLowering.h
index c90c65c81f700672bf9e089c95929156a7ca9c52..7e4496324a1dfb42e57cbe668a9bd9832fdc7fd9 100644
--- a/lib/Target/AVR/AVRISelLowering.h
+++ b/lib/Target/AVR/AVRISelLowering.h
@@ -1,9 +1,8 @@
 //===-- AVRISelLowering.h - AVR DAG Lowering Interface ----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -64,12 +63,14 @@ enum NodeType {
 
 } // end of namespace AVRISD
 
+class AVRSubtarget;
 class AVRTargetMachine;
 
 /// Performs target lowering for the AVR.
 class AVRTargetLowering : public TargetLowering {
 public:
-  explicit AVRTargetLowering(AVRTargetMachine &TM);
+  explicit AVRTargetLowering(const AVRTargetMachine &TM,
+                             const AVRSubtarget &STI);
 
 public:
   MVT getScalarShiftAmountTy(const DataLayout &, EVT LHSTy) const override {
@@ -164,6 +165,10 @@ private:
                           const SDLoc &dl, SelectionDAG &DAG,
                           SmallVectorImpl<SDValue> &InVals) const;
 
+protected:
+
+  const AVRSubtarget &Subtarget;
+
 private:
   MachineBasicBlock *insertShift(MachineInstr &MI, MachineBasicBlock *BB) const;
   MachineBasicBlock *insertMul(MachineInstr &MI, MachineBasicBlock *BB) const;
diff --git a/lib/Target/AVR/AVRInstrFormats.td b/lib/Target/AVR/AVRInstrFormats.td
index ce5e606f97877a754081b8d0a5dadc156651bc3b..347e683cd47f51b535fd32824d4db463be911118 100644
--- a/lib/Target/AVR/AVRInstrFormats.td
+++ b/lib/Target/AVR/AVRInstrFormats.td
@@ -1,9 +1,8 @@
 //===-- AVRInstrInfo.td - AVR Instruction Formats ----------*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AVR/AVRInstrInfo.cpp b/lib/Target/AVR/AVRInstrInfo.cpp
index 0c32334167f06be62239e3516777ef86411e1b85..ba7a95e92c5c5c04e419a6a2dd7fde9d54a9bac1 100644
--- a/lib/Target/AVR/AVRInstrInfo.cpp
+++ b/lib/Target/AVR/AVRInstrInfo.cpp
@@ -1,9 +1,8 @@
 //===-- AVRInstrInfo.cpp - AVR Instruction Information --------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -488,7 +487,8 @@ unsigned AVRInstrInfo::getInstSizeInBytes(const MachineInstr &MI) const {
   case TargetOpcode::KILL:
   case TargetOpcode::DBG_VALUE:
     return 0;
-  case TargetOpcode::INLINEASM: {
+  case TargetOpcode::INLINEASM:
+  case TargetOpcode::INLINEASM_BR: {
     const MachineFunction &MF = *MI.getParent()->getParent();
     const AVRTargetMachine &TM = static_cast<const AVRTargetMachine&>(MF.getTarget());
     const AVRSubtarget &STI = MF.getSubtarget<AVRSubtarget>();
diff --git a/lib/Target/AVR/AVRInstrInfo.h b/lib/Target/AVR/AVRInstrInfo.h
index 354edcec3466a9126e8fdf0ca2ae2517d45c58ec..ba74af325474a8bbd6614aac147c41ada42c0d9a 100644
--- a/lib/Target/AVR/AVRInstrInfo.h
+++ b/lib/Target/AVR/AVRInstrInfo.h
@@ -1,9 +1,8 @@
 //===-- AVRInstrInfo.h - AVR Instruction Information ------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AVR/AVRInstrInfo.td b/lib/Target/AVR/AVRInstrInfo.td
index 5720af7d8df6167c046997d2064f74ecaca9142b..c458fe7de068e4811264f2f20c2bda97c68f7a00 100644
--- a/lib/Target/AVR/AVRInstrInfo.td
+++ b/lib/Target/AVR/AVRInstrInfo.td
@@ -1,9 +1,8 @@
 //===-- AVRInstrInfo.td - AVR Instruction defs -------------*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -90,6 +89,22 @@ def imm0_63_neg : PatLeaf<(imm),
 
 def uimm6 : PatLeaf<(imm), [{ return isUInt<6>(N->getZExtValue()); }]>;
 
+// imm_com8_XFORM - Return the complement of a imm_com8 value
+def imm_com8_XFORM : SDNodeXForm<imm, [{
+  return CurDAG->getTargetConstant(~((uint8_t)N->getZExtValue()), SDLoc(N),
+                                   MVT::i8);
+}]>;
+
+// imm_com8 - Match an immediate that is a complement
+// of a 8-bit immediate.
+// Note: this pattern doesn't require an encoder method and such, as it's
+// only used on aliases (Pat<> and InstAlias<>). The actual encoding
+// is handled by the destination instructions, which use imm_com8.
+def imm_com8_asmoperand : AsmOperandClass { let Name = "ImmCom8"; }
+def imm_com8 : Operand<i8> {
+  let ParserMatchClass = imm_com8_asmoperand;
+}
+
 def ioaddr_XFORM : SDNodeXForm<imm,
 [{
   return CurDAG->getTargetConstant(uint8_t(N->getZExtValue()) - 0x20, SDLoc(N), MVT::i8);
@@ -157,13 +172,6 @@ def memspi : Operand<iPTR>
   let MIOperandInfo = (ops GPRSP, i16imm);
 }
 
-def imm_com8 : Operand<i8>
-{
-  let EncoderMethod = "encodeComplement";
-
-  let MIOperandInfo = (ops i8imm);
-}
-
 def relbrtarget_7 : Operand<OtherVT>
 {
     let PrintMethod   = "printPCRelImm";
@@ -1729,20 +1737,7 @@ def BLD : FRdB<0b00,
                "bld\t$rd, $b",
                []>;
 
-// Set/clear bit in register operations.
-let Constraints = "$src = $rd",
-Defs = [SREG] in
-{
-  // CBR Rd, K
-  // Alias for `ANDI Rd, COM(K)` where COM(K) is the complement of K.
-  // FIXME: This uses the 'complement' encoder. We need it to also use the
-  // imm_ldi8 encoder. This will cause no fixups to be created on this instruction.
-  def CBRRdK : FRdK<0b0111,
-                    (outs LD8:$rd),
-                    (ins LD8:$src, imm_com8:$k),
-                    "cbr\t$rd, $k",
-                    []>;
-}
+def CBR : InstAlias<"cbr\t$rd, $k", (ANDIRdK LD8:$rd, imm_com8:$k), 0>;
 
 // CLR Rd
 // Alias for EOR Rd, Rd
diff --git a/lib/Target/AVR/AVRMCInstLower.cpp b/lib/Target/AVR/AVRMCInstLower.cpp
index dfefd09bc4b86ade6ee0e48b57d52af1501e3b6e..49a318762b63853cfaab1fd1484835d74dd61d3c 100644
--- a/lib/Target/AVR/AVRMCInstLower.cpp
+++ b/lib/Target/AVR/AVRMCInstLower.cpp
@@ -1,9 +1,8 @@
 //===-- AVRMCInstLower.cpp - Convert AVR MachineInstr to an MCInst --------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AVR/AVRMCInstLower.h b/lib/Target/AVR/AVRMCInstLower.h
index 2e2d1014485e88fdc31f217ca1f796a04ea59104..5e0f42ac16a751f1cd12eab054bd62d251da4b41 100644
--- a/lib/Target/AVR/AVRMCInstLower.h
+++ b/lib/Target/AVR/AVRMCInstLower.h
@@ -1,9 +1,8 @@
 //===-- AVRMCInstLower.h - Lower MachineInstr to MCInst ---------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/AVR/AVRMachineFunctionInfo.h b/lib/Target/AVR/AVRMachineFunctionInfo.h
index cf0c73576301d9672128782bf3333305c875a2cb..5226e30491c35560a71caa45f33b418b6c843a49 100644
--- a/lib/Target/AVR/AVRMachineFunctionInfo.h
+++ b/lib/Target/AVR/AVRMachineFunctionInfo.h
@@ -1,9 +1,8 @@
 //===-- AVRMachineFuctionInfo.h - AVR machine function info -----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AVR/AVRRegisterInfo.cpp b/lib/Target/AVR/AVRRegisterInfo.cpp
index 808a85e459c1bbaee5f703e2811ed6f013e1056c..8dc31fe066b96b34d1101e5c5c953e6b0a312f2b 100644
--- a/lib/Target/AVR/AVRRegisterInfo.cpp
+++ b/lib/Target/AVR/AVRRegisterInfo.cpp
@@ -1,9 +1,8 @@
 //===-- AVRRegisterInfo.cpp - AVR Register Information --------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AVR/AVRRegisterInfo.h b/lib/Target/AVR/AVRRegisterInfo.h
index 104b336b9c486214a28bfb9bdde1c39d70fc19d0..2365039dbe32acdccf44aaf4959bf5184bcdab77 100644
--- a/lib/Target/AVR/AVRRegisterInfo.h
+++ b/lib/Target/AVR/AVRRegisterInfo.h
@@ -1,9 +1,8 @@
 //===-- AVRRegisterInfo.h - AVR Register Information Impl -------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AVR/AVRRegisterInfo.td b/lib/Target/AVR/AVRRegisterInfo.td
index d55252bcac46d1f91d01e1c0cbb02b52c121eddc..e20f69beabe7dc93614ca91dc3ef6fa3fff4061b 100644
--- a/lib/Target/AVR/AVRRegisterInfo.td
+++ b/lib/Target/AVR/AVRRegisterInfo.td
@@ -1,9 +1,8 @@
 //===-- AVRRegisterInfo.td - AVR Register defs -------------*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/AVR/AVRRelaxMemOperations.cpp b/lib/Target/AVR/AVRRelaxMemOperations.cpp
index fdb09897eda89d050f2228af8dc16c4a2d432ab4..6be901743e827fd2a3535f952573d5f8b2648c97 100644
--- a/lib/Target/AVR/AVRRelaxMemOperations.cpp
+++ b/lib/Target/AVR/AVRRelaxMemOperations.cpp
@@ -1,9 +1,8 @@
 //===-- AVRRelaxMemOperations.cpp - Relax out of range loads/stores -------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AVR/AVRSelectionDAGInfo.h b/lib/Target/AVR/AVRSelectionDAGInfo.h
index 6474c8779330ff036ad9825d28550d14c308b5eb..3e7bd57f10cf070fed9da75b834bb5b0b10398a4 100644
--- a/lib/Target/AVR/AVRSelectionDAGInfo.h
+++ b/lib/Target/AVR/AVRSelectionDAGInfo.h
@@ -1,9 +1,8 @@
 //===-- AVRSelectionDAGInfo.h - AVR SelectionDAG Info -----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AVR/AVRSubtarget.cpp b/lib/Target/AVR/AVRSubtarget.cpp
index 556d69ec523413ef2e9926142cac1c7f3b73387d..6a41036fdd6c892eaec101b8cf0e97ddfe01fadf 100644
--- a/lib/Target/AVR/AVRSubtarget.cpp
+++ b/lib/Target/AVR/AVRSubtarget.cpp
@@ -1,9 +1,8 @@
 //===-- AVRSubtarget.cpp - AVR Subtarget Information ----------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -29,9 +28,9 @@
 namespace llvm {
 
 AVRSubtarget::AVRSubtarget(const Triple &TT, const std::string &CPU,
-                           const std::string &FS, AVRTargetMachine &TM)
+                           const std::string &FS, const AVRTargetMachine &TM)
     : AVRGenSubtargetInfo(TT, CPU, FS), InstrInfo(), FrameLowering(),
-      TLInfo(TM), TSInfo(),
+      TLInfo(TM, initializeSubtargetDependencies(CPU, FS, TM)), TSInfo(),
 
       // Subtarget features
       m_hasSRAM(false), m_hasJMPCALL(false), m_hasIJMPCALL(false),
@@ -44,4 +43,12 @@ AVRSubtarget::AVRSubtarget(const Triple &TT, const std::string &CPU,
   ParseSubtargetFeatures(CPU, FS);
 }
 
+AVRSubtarget &
+AVRSubtarget::initializeSubtargetDependencies(StringRef CPU, StringRef FS,
+                                              const TargetMachine &TM) {
+  // Parse features string.
+  ParseSubtargetFeatures(CPU, FS);
+  return *this;
+}
+
 } // end of namespace llvm
diff --git a/lib/Target/AVR/AVRSubtarget.h b/lib/Target/AVR/AVRSubtarget.h
index fa26738da1900e1b5e1427b9c7b89551572a96a9..da9289af7c8d720dc7866d94538ac4f863490b7e 100644
--- a/lib/Target/AVR/AVRSubtarget.h
+++ b/lib/Target/AVR/AVRSubtarget.h
@@ -1,9 +1,8 @@
 //===-- AVRSubtarget.h - Define Subtarget for the AVR -----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -37,7 +36,7 @@ public:
   //! \param FS  The feature string.
   //! \param TM  The target machine.
   AVRSubtarget(const Triple &TT, const std::string &CPU, const std::string &FS,
-               AVRTargetMachine &TM);
+               const AVRTargetMachine &TM);
 
   const AVRInstrInfo *getInstrInfo() const override { return &InstrInfo; }
   const TargetFrameLowering *getFrameLowering() const override { return &FrameLowering; }
@@ -49,6 +48,9 @@ public:
   /// \note Definition of function is auto generated by `tblgen`.
   void ParseSubtargetFeatures(StringRef CPU, StringRef FS);
 
+  AVRSubtarget &initializeSubtargetDependencies(StringRef CPU, StringRef FS,
+                                                const TargetMachine &TM);
+
   // Subtarget feature getters.
   // See AVR.td for details.
   bool hasSRAM() const { return m_hasSRAM; }
diff --git a/lib/Target/AVR/AVRTargetMachine.cpp b/lib/Target/AVR/AVRTargetMachine.cpp
index 9828cdab68c313465e7be17f073892b3000adfb8..fa1a117311f4f9f0ca3a19a018839665d48294e7 100644
--- a/lib/Target/AVR/AVRTargetMachine.cpp
+++ b/lib/Target/AVR/AVRTargetMachine.cpp
@@ -1,9 +1,8 @@
 //===-- AVRTargetMachine.cpp - Define TargetMachine for AVR ---------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AVR/AVRTargetMachine.h b/lib/Target/AVR/AVRTargetMachine.h
index ffcf4350d45a13946868f48ae69725101432db9f..f9015c8741ea858b7e139c2b9916f8aae7c06575 100644
--- a/lib/Target/AVR/AVRTargetMachine.h
+++ b/lib/Target/AVR/AVRTargetMachine.h
@@ -1,9 +1,8 @@
 //===-- AVRTargetMachine.h - Define TargetMachine for AVR -------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AVR/AVRTargetObjectFile.cpp b/lib/Target/AVR/AVRTargetObjectFile.cpp
index 0cebb0f043f927fd3a2db12cf3856d05e2e64cb2..980096a098351fd0a91f5b236b0a84d0e77ef79f 100644
--- a/lib/Target/AVR/AVRTargetObjectFile.cpp
+++ b/lib/Target/AVR/AVRTargetObjectFile.cpp
@@ -1,9 +1,8 @@
 //===-- AVRTargetObjectFile.cpp - AVR Object Files ------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/AVR/AVRTargetObjectFile.h b/lib/Target/AVR/AVRTargetObjectFile.h
index ba91036fd64cff072aef0677681697c521a3bbc3..53d8510d9a21db88ba219cd379593ad61908fca4 100644
--- a/lib/Target/AVR/AVRTargetObjectFile.h
+++ b/lib/Target/AVR/AVRTargetObjectFile.h
@@ -1,9 +1,8 @@
 //===-- AVRTargetObjectFile.h - AVR Object Info -----------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/AVR/AsmParser/AVRAsmParser.cpp b/lib/Target/AVR/AsmParser/AVRAsmParser.cpp
index f2bb59265271b4625115e9a051cfbd47453e5c71..d667b6800b37a0a7b11f489956121db9ff77ed18 100644
--- a/lib/Target/AVR/AsmParser/AVRAsmParser.cpp
+++ b/lib/Target/AVR/AsmParser/AVRAsmParser.cpp
@@ -1,9 +1,8 @@
 //===---- AVRAsmParser.cpp - Parse AVR assembly to MCInst instructions ----===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -160,6 +159,22 @@ public:
     addExpr(Inst, getImm());
   }
 
+  void addImmCom8Operands(MCInst &Inst, unsigned N) const {
+    assert(N == 1 && "Invalid number of operands!");
+    // The operand is actually a imm8, but we have its bitwise
+    // negation in the assembly source, so twiddle it here.
+    const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getImm());
+    Inst.addOperand(MCOperand::createImm(~(uint8_t)CE->getValue()));
+  }
+
+  bool isImmCom8() const {
+    if (!isImm()) return false;
+    const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getImm());
+    if (!CE) return false;
+    int64_t Value = CE->getValue();
+    return isUInt<8>(Value);
+  }
+
   bool isReg() const { return Kind == k_Register; }
   bool isImm() const { return Kind == k_Immediate; }
   bool isToken() const { return Kind == k_Token; }
diff --git a/lib/Target/AVR/AsmParser/LLVMBuild.txt b/lib/Target/AVR/AsmParser/LLVMBuild.txt
index 047a1b25d3097e298e385170ac5a213a2a292594..0eb4eeb85d40909b099e662fb88490c17548899c 100644
--- a/lib/Target/AVR/AsmParser/LLVMBuild.txt
+++ b/lib/Target/AVR/AsmParser/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/Target/AVR/AsmParser/LLVMBuild.txt -----------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Target/AVR/Disassembler/AVRDisassembler.cpp b/lib/Target/AVR/Disassembler/AVRDisassembler.cpp
index e69accfa9393b15116892bbc3c92e4fe87135e39..ca554056f070d06735b8b30b6e5467db285178d7 100644
--- a/lib/Target/AVR/Disassembler/AVRDisassembler.cpp
+++ b/lib/Target/AVR/Disassembler/AVRDisassembler.cpp
@@ -1,9 +1,8 @@
 //===- AVRDisassembler.cpp - Disassembler for AVR ---------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AVR/Disassembler/LLVMBuild.txt b/lib/Target/AVR/Disassembler/LLVMBuild.txt
index e97babd2a11df1e50a29388040c66a9d0fef4ffc..86b5e0b7816caca553f67a69022f11a179798a8c 100644
--- a/lib/Target/AVR/Disassembler/LLVMBuild.txt
+++ b/lib/Target/AVR/Disassembler/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/Target/AVR/Disassembler/LLVMBuild.txt --------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Target/AVR/InstPrinter/AVRInstPrinter.cpp b/lib/Target/AVR/InstPrinter/AVRInstPrinter.cpp
index 0f34b8e18ff963086834ee30c3913c793ae01461..88ce9a25680e2b0447c9b87b36a71a656d18c348 100644
--- a/lib/Target/AVR/InstPrinter/AVRInstPrinter.cpp
+++ b/lib/Target/AVR/InstPrinter/AVRInstPrinter.cpp
@@ -1,9 +1,8 @@
 //===-- AVRInstPrinter.cpp - Convert AVR MCInst to assembly syntax --------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AVR/InstPrinter/AVRInstPrinter.h b/lib/Target/AVR/InstPrinter/AVRInstPrinter.h
index c9f65b922745924366b919db0ad351ae5782bbab..5b758a7503c9f69716c8802f910d92fee2a5203a 100644
--- a/lib/Target/AVR/InstPrinter/AVRInstPrinter.h
+++ b/lib/Target/AVR/InstPrinter/AVRInstPrinter.h
@@ -1,9 +1,8 @@
 //===- AVRInstPrinter.h - Convert AVR MCInst to assembly syntax -*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AVR/InstPrinter/LLVMBuild.txt b/lib/Target/AVR/InstPrinter/LLVMBuild.txt
index 9f87c97d1989856be0e6691fb234f6e0033c5ffb..67203ff9bb31686909550845402aa72992788e95 100644
--- a/lib/Target/AVR/InstPrinter/LLVMBuild.txt
+++ b/lib/Target/AVR/InstPrinter/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/Target/AVR/InstPrinter/LLVMBuild.txt ----------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Target/AVR/LLVMBuild.txt b/lib/Target/AVR/LLVMBuild.txt
index daed4260843aac83c2941d08a2450d914620320e..359ac1ae4b9fc126d97478ad7195e3f7d02cb646 100644
--- a/lib/Target/AVR/LLVMBuild.txt
+++ b/lib/Target/AVR/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/Target/AVR/LLVMBuild.txt ---------------------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Target/AVR/MCTargetDesc/AVRAsmBackend.cpp b/lib/Target/AVR/MCTargetDesc/AVRAsmBackend.cpp
index f81a57dd71e31358fa9cc520017eda255fba1627..e92b16c8ee9d61c63b4f6a33ff295a45b97423d7 100644
--- a/lib/Target/AVR/MCTargetDesc/AVRAsmBackend.cpp
+++ b/lib/Target/AVR/MCTargetDesc/AVRAsmBackend.cpp
@@ -1,9 +1,8 @@
 //===-- AVRAsmBackend.cpp - AVR Asm Backend  ------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AVR/MCTargetDesc/AVRAsmBackend.h b/lib/Target/AVR/MCTargetDesc/AVRAsmBackend.h
index d48077c3ab8ef3b9aae58060aa1242603a136637..1e713db38145dee4d20bebf2ea4e9f7ebbbd147a 100644
--- a/lib/Target/AVR/MCTargetDesc/AVRAsmBackend.h
+++ b/lib/Target/AVR/MCTargetDesc/AVRAsmBackend.h
@@ -1,9 +1,8 @@
 //===-- AVRAsmBackend.h - AVR Asm Backend  --------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AVR/MCTargetDesc/AVRELFObjectWriter.cpp b/lib/Target/AVR/MCTargetDesc/AVRELFObjectWriter.cpp
index 4a921a1601a9da1efac02d71393fdbd2cb191068..6025e4b2437ca574deabc44133d6e025b22bd234 100644
--- a/lib/Target/AVR/MCTargetDesc/AVRELFObjectWriter.cpp
+++ b/lib/Target/AVR/MCTargetDesc/AVRELFObjectWriter.cpp
@@ -1,9 +1,8 @@
 //===-- AVRELFObjectWriter.cpp - AVR ELF Writer ---------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/AVR/MCTargetDesc/AVRELFStreamer.h b/lib/Target/AVR/MCTargetDesc/AVRELFStreamer.h
index e5df6cc34e402babb3a76e9a9d4b6074589d3c69..461f1660c952cb724196260a7d0dcadb6f245040 100644
--- a/lib/Target/AVR/MCTargetDesc/AVRELFStreamer.h
+++ b/lib/Target/AVR/MCTargetDesc/AVRELFStreamer.h
@@ -1,9 +1,8 @@
 //===----- AVRELFStreamer.h - AVR Target Streamer --------------*- C++ -*--===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/AVR/MCTargetDesc/AVRFixupKinds.h b/lib/Target/AVR/MCTargetDesc/AVRFixupKinds.h
index cdb0b215bc603d9ac2a99a99c40f4febfc587951..b3504b89e4d33e68194c6c92a59837a2f545eadc 100644
--- a/lib/Target/AVR/MCTargetDesc/AVRFixupKinds.h
+++ b/lib/Target/AVR/MCTargetDesc/AVRFixupKinds.h
@@ -1,9 +1,8 @@
 //===-- AVRFixupKinds.h - AVR Specific Fixup Entries ------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/AVR/MCTargetDesc/AVRMCAsmInfo.cpp b/lib/Target/AVR/MCTargetDesc/AVRMCAsmInfo.cpp
index 535bb012eb07cae173e943f84bdb367f7d3b1613..99b2172c562f5d5216499379253b568116fc8a06 100644
--- a/lib/Target/AVR/MCTargetDesc/AVRMCAsmInfo.cpp
+++ b/lib/Target/AVR/MCTargetDesc/AVRMCAsmInfo.cpp
@@ -1,9 +1,8 @@
 //===-- AVRMCAsmInfo.cpp - AVR asm properties -----------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -24,6 +23,7 @@ AVRMCAsmInfo::AVRMCAsmInfo(const Triple &TT) {
   PrivateGlobalPrefix = ".L";
   UsesELFSectionDirectiveForBSS = true;
   UseIntegratedAssembler = true;
+  SupportsDebugInformation = true;
 }
 
 } // end of namespace llvm
diff --git a/lib/Target/AVR/MCTargetDesc/AVRMCAsmInfo.h b/lib/Target/AVR/MCTargetDesc/AVRMCAsmInfo.h
index cc2207a3cfaece96c89183bbe61d3b55ebf338ba..b2fa18777bc034dcf8148a0ac4fa86a719c86e67 100644
--- a/lib/Target/AVR/MCTargetDesc/AVRMCAsmInfo.h
+++ b/lib/Target/AVR/MCTargetDesc/AVRMCAsmInfo.h
@@ -1,9 +1,8 @@
 //===-- AVRMCAsmInfo.h - AVR asm properties ---------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AVR/MCTargetDesc/AVRMCCodeEmitter.cpp b/lib/Target/AVR/MCTargetDesc/AVRMCCodeEmitter.cpp
index 4dbbce8c205e21cf5acb4039b81060d83606b111..bc0488778685b6b46f00d5ab88182ead3b877339 100644
--- a/lib/Target/AVR/MCTargetDesc/AVRMCCodeEmitter.cpp
+++ b/lib/Target/AVR/MCTargetDesc/AVRMCCodeEmitter.cpp
@@ -1,9 +1,8 @@
 //===-- AVRMCCodeEmitter.cpp - Convert AVR Code to Machine Code -----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AVR/MCTargetDesc/AVRMCCodeEmitter.h b/lib/Target/AVR/MCTargetDesc/AVRMCCodeEmitter.h
index 883abf8db78a93406f01346a8193165691d44db2..2e24d885c1551eec9f1bb598513e46dab8b55fa6 100644
--- a/lib/Target/AVR/MCTargetDesc/AVRMCCodeEmitter.h
+++ b/lib/Target/AVR/MCTargetDesc/AVRMCCodeEmitter.h
@@ -1,9 +1,8 @@
 //===-- AVRMCCodeEmitter.h - Convert AVR Code to Machine Code -------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AVR/MCTargetDesc/AVRMCELFStreamer.cpp b/lib/Target/AVR/MCTargetDesc/AVRMCELFStreamer.cpp
index 861acd47347f47c117c941916cc1479dfe52fc25..d9169f90a765ba801385c9d4a451ec2f7f8411a9 100644
--- a/lib/Target/AVR/MCTargetDesc/AVRMCELFStreamer.cpp
+++ b/lib/Target/AVR/MCTargetDesc/AVRMCELFStreamer.cpp
@@ -1,9 +1,8 @@
 //===--------- AVRMCELFStreamer.cpp - AVR subclass of MCELFStreamer -------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AVR/MCTargetDesc/AVRMCELFStreamer.h b/lib/Target/AVR/MCTargetDesc/AVRMCELFStreamer.h
index 12e805fc7d132855ad28a29ecc04502a84388232..37a610bc4248fab86d1d210c2dba7611d2257ef8 100644
--- a/lib/Target/AVR/MCTargetDesc/AVRMCELFStreamer.h
+++ b/lib/Target/AVR/MCTargetDesc/AVRMCELFStreamer.h
@@ -1,9 +1,8 @@
 //===--------- AVRMCELFStreamer.h - AVR subclass of MCELFStreamer ---------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/AVR/MCTargetDesc/AVRMCExpr.cpp b/lib/Target/AVR/MCTargetDesc/AVRMCExpr.cpp
index d4a67973af7f7faac3a008365866b4721a0e0af9..0a53e534677963b347373f2bb371f1c4f5c82cf3 100644
--- a/lib/Target/AVR/MCTargetDesc/AVRMCExpr.cpp
+++ b/lib/Target/AVR/MCTargetDesc/AVRMCExpr.cpp
@@ -1,9 +1,8 @@
 //===-- AVRMCExpr.cpp - AVR specific MC expression classes ----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/AVR/MCTargetDesc/AVRMCExpr.h b/lib/Target/AVR/MCTargetDesc/AVRMCExpr.h
index a166b094674995e56c7a43219ece7e0df566d281..3b696bab1715de955cd81d9567abde8d7b8db6af 100644
--- a/lib/Target/AVR/MCTargetDesc/AVRMCExpr.h
+++ b/lib/Target/AVR/MCTargetDesc/AVRMCExpr.h
@@ -1,9 +1,8 @@
 //===-- AVRMCExpr.h - AVR specific MC expression classes --------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/AVR/MCTargetDesc/AVRMCTargetDesc.cpp b/lib/Target/AVR/MCTargetDesc/AVRMCTargetDesc.cpp
index 8c39b5f4039e3615027e732ac991f1e9aaf5d958..9791bce0c2976f4d6e12f85d95d547f7c4bbaa7b 100644
--- a/lib/Target/AVR/MCTargetDesc/AVRMCTargetDesc.cpp
+++ b/lib/Target/AVR/MCTargetDesc/AVRMCTargetDesc.cpp
@@ -1,9 +1,8 @@
 //===-- AVRMCTargetDesc.cpp - AVR Target Descriptions ---------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AVR/MCTargetDesc/AVRMCTargetDesc.h b/lib/Target/AVR/MCTargetDesc/AVRMCTargetDesc.h
index a764f15bd0650c655fa6fd51fe4b3368ba49ac81..54f7c003ae79dc04e31ab0de39083c5b5a640665 100644
--- a/lib/Target/AVR/MCTargetDesc/AVRMCTargetDesc.h
+++ b/lib/Target/AVR/MCTargetDesc/AVRMCTargetDesc.h
@@ -1,9 +1,8 @@
 //===-- AVRMCTargetDesc.h - AVR Target Descriptions -------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AVR/MCTargetDesc/AVRTargetStreamer.cpp b/lib/Target/AVR/MCTargetDesc/AVRTargetStreamer.cpp
index 2b45d9adc7e903e4865ba4ff47dd638ab5d57a4b..3487a2bbb8640bf3e6f11e11c7cbcf291e59f4f0 100644
--- a/lib/Target/AVR/MCTargetDesc/AVRTargetStreamer.cpp
+++ b/lib/Target/AVR/MCTargetDesc/AVRTargetStreamer.cpp
@@ -1,9 +1,8 @@
 //===-- AVRTargetStreamer.cpp - AVR Target Streamer Methods ---------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/AVR/MCTargetDesc/AVRTargetStreamer.h b/lib/Target/AVR/MCTargetDesc/AVRTargetStreamer.h
index 815088b0a5de0495312eaaa94b91506885be3fc3..5c4d1a22f6c6ce285f9639cc9d70895f1d83f612 100644
--- a/lib/Target/AVR/MCTargetDesc/AVRTargetStreamer.h
+++ b/lib/Target/AVR/MCTargetDesc/AVRTargetStreamer.h
@@ -1,9 +1,8 @@
 //===-- AVRTargetStreamer.h - AVR Target Streamer --------------*- C++ -*--===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/AVR/MCTargetDesc/LLVMBuild.txt b/lib/Target/AVR/MCTargetDesc/LLVMBuild.txt
index db4d5754b6679ba39c7c27213822b93e8f3993b1..b62e615ef7d525614c4106f575140c87dbeabddf 100644
--- a/lib/Target/AVR/MCTargetDesc/LLVMBuild.txt
+++ b/lib/Target/AVR/MCTargetDesc/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/Target/AVR/MCTargetDesc/LLVMBuild.txt --------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Target/AVR/TargetInfo/AVRTargetInfo.cpp b/lib/Target/AVR/TargetInfo/AVRTargetInfo.cpp
index abe9cf45fcb3869354959efe2ad65df49264f317..cb196e35fc623b3aa0f1ca9511afa22f3352f8c7 100644
--- a/lib/Target/AVR/TargetInfo/AVRTargetInfo.cpp
+++ b/lib/Target/AVR/TargetInfo/AVRTargetInfo.cpp
@@ -1,9 +1,8 @@
 //===-- AVRTargetInfo.cpp - AVR Target Implementation ---------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/AVR/TargetInfo/LLVMBuild.txt b/lib/Target/AVR/TargetInfo/LLVMBuild.txt
index bc6e0ad2ee19f573360505687453bc8da57d2747..64469776328551d034a8d45cb89355162beae925 100644
--- a/lib/Target/AVR/TargetInfo/LLVMBuild.txt
+++ b/lib/Target/AVR/TargetInfo/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/Target/AVR/TargetInfo/LLVMBuild.txt ----------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Target/BPF/AsmParser/BPFAsmParser.cpp b/lib/Target/BPF/AsmParser/BPFAsmParser.cpp
index 8890fb8adf4de0e85688c23c1e8e567461ae3704..6d2183f6f39a51016dcc7cafcee42d4c1a18b043 100644
--- a/lib/Target/BPF/AsmParser/BPFAsmParser.cpp
+++ b/lib/Target/BPF/AsmParser/BPFAsmParser.cpp
@@ -1,9 +1,8 @@
 //===-- BPFAsmParser.cpp - Parse BPF assembly to MCInst instructions --===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/BPF/AsmParser/LLVMBuild.txt b/lib/Target/BPF/AsmParser/LLVMBuild.txt
index 2733da9d4a0221cbcd1b7f95a2cc1c23d88f040d..cd01fb625fd4751bb01b36fa09f11f14f3c1179f 100644
--- a/lib/Target/BPF/AsmParser/LLVMBuild.txt
+++ b/lib/Target/BPF/AsmParser/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/Target/BPF/AsmParser/LLVMBuild.txt ---------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Target/BPF/BPF.h b/lib/Target/BPF/BPF.h
index 9749e369c2c1f5813e787cde8c7b3a0bd82cefa7..6c75fff9c7aa7163cf77ba6d7ae353c5f40196b3 100644
--- a/lib/Target/BPF/BPF.h
+++ b/lib/Target/BPF/BPF.h
@@ -1,9 +1,8 @@
 //===-- BPF.h - Top-level interface for BPF representation ------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/BPF/BPF.td b/lib/Target/BPF/BPF.td
index 877bd15f4f2b2dd8450c0a7fe010807180179854..fad966ff5a1377e425e226b0023775d6aa4842eb 100644
--- a/lib/Target/BPF/BPF.td
+++ b/lib/Target/BPF/BPF.td
@@ -1,9 +1,8 @@
 //===-- BPF.td - Describe the BPF Target Machine -----------*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -21,6 +20,7 @@ class Proc<string Name, list<SubtargetFeature> Features>
 def : Proc<"generic", []>;
 def : Proc<"v1", []>;
 def : Proc<"v2", []>;
+def : Proc<"v3", []>;
 def : Proc<"probe", []>;
 
 def DummyFeature : SubtargetFeature<"dummy", "isDummyMode",
diff --git a/lib/Target/BPF/BPFAsmPrinter.cpp b/lib/Target/BPF/BPFAsmPrinter.cpp
index 705211b486bf72557f1b0388305c7e892482afcb..ab2886682ddea46facb60e7a4cc9e473a6803105 100644
--- a/lib/Target/BPF/BPFAsmPrinter.cpp
+++ b/lib/Target/BPF/BPFAsmPrinter.cpp
@@ -1,9 +1,8 @@
 //===-- BPFAsmPrinter.cpp - BPF LLVM assembly writer ----------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -16,6 +15,7 @@
 #include "BPFInstrInfo.h"
 #include "BPFMCInstLower.h"
 #include "BPFTargetMachine.h"
+#include "BTFDebug.h"
 #include "InstPrinter/BPFInstPrinter.h"
 #include "llvm/CodeGen/AsmPrinter.h"
 #include "llvm/CodeGen/MachineConstantPool.h"
@@ -40,6 +40,7 @@ public:
       : AsmPrinter(TM, std::move(Streamer)) {}
 
   StringRef getPassName() const override { return "BPF Assembly Printer"; }
+  bool doInitialization(Module &M) override;
   void printOperand(const MachineInstr *MI, int OpNum, raw_ostream &O);
   bool PrintAsmOperand(const MachineInstr *MI, unsigned OpNo,
                        unsigned AsmVariant, const char *ExtraCode,
@@ -52,6 +53,18 @@ public:
 };
 } // namespace
 
+bool BPFAsmPrinter::doInitialization(Module &M) {
+  AsmPrinter::doInitialization(M);
+
+  if (MAI->doesSupportDebugInformation()) {
+    Handlers.push_back(HandlerInfo(new BTFDebug(this), "emit",
+                                   "Debug Info Emission", "BTF",
+                                   "BTF Emission"));
+  }
+
+  return false;
+}
+
 void BPFAsmPrinter::printOperand(const MachineInstr *MI, int OpNum,
                                  raw_ostream &O) {
   const MachineOperand &MO = MI->getOperand(OpNum);
diff --git a/lib/Target/BPF/BPFCallingConv.td b/lib/Target/BPF/BPFCallingConv.td
index 637f9752ec42948a659789dbef597bf1f58186b2..ef4ef1930aa8fb48fd184b4d7f870f56f408c35e 100644
--- a/lib/Target/BPF/BPFCallingConv.td
+++ b/lib/Target/BPF/BPFCallingConv.td
@@ -1,9 +1,8 @@
 //===-- BPFCallingConv.td - Calling Conventions BPF --------*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/BPF/BPFFrameLowering.cpp b/lib/Target/BPF/BPFFrameLowering.cpp
index c2806c85f24f84e1ba597151ba56818ebf3bdfbc..8812cfdd86da43c88117e43ef2d7d920e6b73263 100644
--- a/lib/Target/BPF/BPFFrameLowering.cpp
+++ b/lib/Target/BPF/BPFFrameLowering.cpp
@@ -1,9 +1,8 @@
 //===-- BPFFrameLowering.cpp - BPF Frame Information ----------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/BPF/BPFFrameLowering.h b/lib/Target/BPF/BPFFrameLowering.h
index b4ffa0713fa6a8ec63ea14b6679123b3682d8997..2dc6277d2244d3fc63e471f87616a4aa8621f7dc 100644
--- a/lib/Target/BPF/BPFFrameLowering.h
+++ b/lib/Target/BPF/BPFFrameLowering.h
@@ -1,9 +1,8 @@
 //===-- BPFFrameLowering.h - Define frame lowering for BPF -----*- C++ -*--===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/BPF/BPFISelDAGToDAG.cpp b/lib/Target/BPF/BPFISelDAGToDAG.cpp
index 8b9bc08e144f54f2100cc47fd481869e03ec721c..1bd705c5518873dbae2a75a6073751218425ee8a 100644
--- a/lib/Target/BPF/BPFISelDAGToDAG.cpp
+++ b/lib/Target/BPF/BPFISelDAGToDAG.cpp
@@ -1,9 +1,8 @@
 //===-- BPFISelDAGToDAG.cpp - A dag to dag inst selector for BPF ----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/BPF/BPFISelLowering.cpp b/lib/Target/BPF/BPFISelLowering.cpp
index 9272cf692dc9348e2d0ae8df78b55aa5ab15e205..ff69941d26fbd45e4cbb5fbe5a4dc053106654c7 100644
--- a/lib/Target/BPF/BPFISelLowering.cpp
+++ b/lib/Target/BPF/BPFISelLowering.cpp
@@ -1,9 +1,8 @@
 //===-- BPFISelLowering.cpp - BPF DAG Lowering Implementation  ------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -106,7 +105,8 @@ BPFTargetLowering::BPFTargetLowering(const TargetMachine &TM,
 
   if (STI.getHasAlu32()) {
     setOperationAction(ISD::BSWAP, MVT::i32, Promote);
-    setOperationAction(ISD::BR_CC, MVT::i32, Promote);
+    setOperationAction(ISD::BR_CC, MVT::i32,
+                       STI.getHasJmp32() ? Custom : Promote);
   }
 
   setOperationAction(ISD::CTTZ, MVT::i64, Custom);
@@ -163,6 +163,7 @@ BPFTargetLowering::BPFTargetLowering(const TargetMachine &TM,
 
   // CPU/Feature control
   HasAlu32 = STI.getHasAlu32();
+  HasJmp32 = STI.getHasJmp32();
   HasJmpExt = STI.getHasJmpExt();
 }
 
@@ -507,7 +508,7 @@ SDValue BPFTargetLowering::LowerBR_CC(SDValue Op, SelectionDAG &DAG) const {
     NegateCC(LHS, RHS, CC);
 
   return DAG.getNode(BPFISD::BR_CC, DL, Op.getValueType(), Chain, LHS, RHS,
-                     DAG.getConstant(CC, DL, MVT::i64), Dest);
+                     DAG.getConstant(CC, DL, LHS.getValueType()), Dest);
 }
 
 SDValue BPFTargetLowering::LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const {
@@ -677,36 +678,23 @@ BPFTargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
   int CC = MI.getOperand(3).getImm();
   int NewCC;
   switch (CC) {
-  case ISD::SETGT:
-    NewCC = isSelectRROp ? BPF::JSGT_rr : BPF::JSGT_ri;
-    break;
-  case ISD::SETUGT:
-    NewCC = isSelectRROp ? BPF::JUGT_rr : BPF::JUGT_ri;
-    break;
-  case ISD::SETGE:
-    NewCC = isSelectRROp ? BPF::JSGE_rr : BPF::JSGE_ri;
-    break;
-  case ISD::SETUGE:
-    NewCC = isSelectRROp ? BPF::JUGE_rr : BPF::JUGE_ri;
-    break;
-  case ISD::SETEQ:
-    NewCC = isSelectRROp ? BPF::JEQ_rr : BPF::JEQ_ri;
-    break;
-  case ISD::SETNE:
-    NewCC = isSelectRROp ? BPF::JNE_rr : BPF::JNE_ri;
-    break;
-  case ISD::SETLT:
-    NewCC = isSelectRROp ? BPF::JSLT_rr : BPF::JSLT_ri;
-    break;
-  case ISD::SETULT:
-    NewCC = isSelectRROp ? BPF::JULT_rr : BPF::JULT_ri;
-    break;
-  case ISD::SETLE:
-    NewCC = isSelectRROp ? BPF::JSLE_rr : BPF::JSLE_ri;
-    break;
-  case ISD::SETULE:
-    NewCC = isSelectRROp ? BPF::JULE_rr : BPF::JULE_ri;
-    break;
+#define SET_NEWCC(X, Y) \
+  case ISD::X: \
+    if (is32BitCmp && HasJmp32) \
+      NewCC = isSelectRROp ? BPF::Y##_rr_32 : BPF::Y##_ri_32; \
+    else \
+      NewCC = isSelectRROp ? BPF::Y##_rr : BPF::Y##_ri; \
+    break
+  SET_NEWCC(SETGT, JSGT);
+  SET_NEWCC(SETUGT, JUGT);
+  SET_NEWCC(SETGE, JSGE);
+  SET_NEWCC(SETUGE, JUGE);
+  SET_NEWCC(SETEQ, JEQ);
+  SET_NEWCC(SETNE, JNE);
+  SET_NEWCC(SETLT, JSLT);
+  SET_NEWCC(SETULT, JULT);
+  SET_NEWCC(SETLE, JSLE);
+  SET_NEWCC(SETULE, JULE);
   default:
     report_fatal_error("unimplemented select CondCode " + Twine(CC));
   }
@@ -724,13 +712,13 @@ BPFTargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
   //
   // We simply do extension for all situations in this method, but we will
   // try to remove those unnecessary in BPFMIPeephole pass.
-  if (is32BitCmp)
+  if (is32BitCmp && !HasJmp32)
     LHS = EmitSubregExt(MI, BB, LHS, isSignedCmp);
 
   if (isSelectRROp) {
     unsigned RHS = MI.getOperand(2).getReg();
 
-    if (is32BitCmp)
+    if (is32BitCmp && !HasJmp32)
       RHS = EmitSubregExt(MI, BB, RHS, isSignedCmp);
 
     BuildMI(BB, DL, TII.get(NewCC)).addReg(LHS).addReg(RHS).addMBB(Copy1MBB);
diff --git a/lib/Target/BPF/BPFISelLowering.h b/lib/Target/BPF/BPFISelLowering.h
index 0aa8b9ac57ac6e4c3fe9fea28828b4ee302c3a80..2a44165de6910bea935ddd42e0b2da33882de914 100644
--- a/lib/Target/BPF/BPFISelLowering.h
+++ b/lib/Target/BPF/BPFISelLowering.h
@@ -1,9 +1,8 @@
 //===-- BPFISelLowering.h - BPF DAG Lowering Interface ----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -56,6 +55,7 @@ public:
                               MachineBasicBlock *BB) const override;
 
   bool getHasAlu32() const { return HasAlu32; }
+  bool getHasJmp32() const { return HasJmp32; }
   bool getHasJmpExt() const { return HasJmpExt; }
 
   EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context,
@@ -66,6 +66,7 @@ public:
 private:
   // Control Instruction Selection Features
   bool HasAlu32;
+  bool HasJmp32;
   bool HasJmpExt;
 
   SDValue LowerBR_CC(SDValue Op, SelectionDAG &DAG) const;
diff --git a/lib/Target/BPF/BPFInstrFormats.td b/lib/Target/BPF/BPFInstrFormats.td
index 92d4a62fd87557c02c3ed865f22d6f7357b97e6b..9f00dc85d7898c6c5db9167bc29ac9000d306074 100644
--- a/lib/Target/BPF/BPFInstrFormats.td
+++ b/lib/Target/BPF/BPFInstrFormats.td
@@ -1,9 +1,8 @@
 //===-- BPFInstrFormats.td - BPF Instruction Formats -------*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -17,6 +16,7 @@ def BPF_ST    : BPFOpClass<0x2>;
 def BPF_STX   : BPFOpClass<0x3>;
 def BPF_ALU   : BPFOpClass<0x4>;
 def BPF_JMP   : BPFOpClass<0x5>;
+def BPF_JMP32 : BPFOpClass<0x6>;
 def BPF_ALU64 : BPFOpClass<0x7>;
 
 class BPFSrcType<bits<1> val> {
diff --git a/lib/Target/BPF/BPFInstrInfo.cpp b/lib/Target/BPF/BPFInstrInfo.cpp
index 4d47debdaa74a3917ecd0f73b4c2a4951bc04f2e..932f718d549034e315e68a8402c10585e2af0a3c 100644
--- a/lib/Target/BPF/BPFInstrInfo.cpp
+++ b/lib/Target/BPF/BPFInstrInfo.cpp
@@ -1,9 +1,8 @@
 //===-- BPFInstrInfo.cpp - BPF Instruction Information ----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/BPF/BPFInstrInfo.h b/lib/Target/BPF/BPFInstrInfo.h
index fb65a86a6d1897b21ed4b265a241497ad101139e..e4bd757da5608c870e816f1045338e5445fabafc 100644
--- a/lib/Target/BPF/BPFInstrInfo.h
+++ b/lib/Target/BPF/BPFInstrInfo.h
@@ -1,9 +1,8 @@
 //===-- BPFInstrInfo.h - BPF Instruction Information ------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/BPF/BPFInstrInfo.td b/lib/Target/BPF/BPFInstrInfo.td
index aaef5fb706e026e44bd42af0838dc7ef6588d11e..36724b81fc6bea43080249e131748e626badb5eb 100644
--- a/lib/Target/BPF/BPFInstrInfo.td
+++ b/lib/Target/BPF/BPFInstrInfo.td
@@ -1,9 +1,8 @@
 //===-- BPFInstrInfo.td - Target Description for BPF Target ---------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -102,6 +101,26 @@ def BPF_CC_LTU : PatLeaf<(i64 imm),
                          [{return (N->getZExtValue() == ISD::SETULT);}]>;
 def BPF_CC_LEU : PatLeaf<(i64 imm),
                          [{return (N->getZExtValue() == ISD::SETULE);}]>;
+def BPF_CC_EQ_32  : PatLeaf<(i32 imm),
+                         [{return (N->getZExtValue() == ISD::SETEQ);}]>;
+def BPF_CC_NE_32  : PatLeaf<(i32 imm),
+                         [{return (N->getZExtValue() == ISD::SETNE);}]>;
+def BPF_CC_GE_32  : PatLeaf<(i32 imm),
+                         [{return (N->getZExtValue() == ISD::SETGE);}]>;
+def BPF_CC_GT_32  : PatLeaf<(i32 imm),
+                         [{return (N->getZExtValue() == ISD::SETGT);}]>;
+def BPF_CC_GTU_32 : PatLeaf<(i32 imm),
+                         [{return (N->getZExtValue() == ISD::SETUGT);}]>;
+def BPF_CC_GEU_32 : PatLeaf<(i32 imm),
+                         [{return (N->getZExtValue() == ISD::SETUGE);}]>;
+def BPF_CC_LE_32  : PatLeaf<(i32 imm),
+                         [{return (N->getZExtValue() == ISD::SETLE);}]>;
+def BPF_CC_LT_32  : PatLeaf<(i32 imm),
+                         [{return (N->getZExtValue() == ISD::SETLT);}]>;
+def BPF_CC_LTU_32 : PatLeaf<(i32 imm),
+                         [{return (N->getZExtValue() == ISD::SETULT);}]>;
+def BPF_CC_LEU_32 : PatLeaf<(i32 imm),
+                         [{return (N->getZExtValue() == ISD::SETULE);}]>;
 
 // For arithmetic and jump instructions the 8-bit 'code'
 // field is divided into three parts:
@@ -167,23 +186,57 @@ class JMP_RI<BPFJumpOp Opc, string OpcodeStr, PatLeaf Cond>
   let BPFClass = BPF_JMP;
 }
 
-multiclass J<BPFJumpOp Opc, string OpcodeStr, PatLeaf Cond> {
+class JMP_RR_32<BPFJumpOp Opc, string OpcodeStr, PatLeaf Cond>
+    : TYPE_ALU_JMP<Opc.Value, BPF_X.Value,
+                   (outs),
+                   (ins GPR32:$dst, GPR32:$src, brtarget:$BrDst),
+                   "if $dst "#OpcodeStr#" $src goto $BrDst",
+                   [(BPFbrcc i32:$dst, i32:$src, Cond, bb:$BrDst)]> {
+  bits<4> dst;
+  bits<4> src;
+  bits<16> BrDst;
+
+  let Inst{55-52} = src;
+  let Inst{51-48} = dst;
+  let Inst{47-32} = BrDst;
+  let BPFClass = BPF_JMP32;
+}
+
+class JMP_RI_32<BPFJumpOp Opc, string OpcodeStr, PatLeaf Cond>
+    : TYPE_ALU_JMP<Opc.Value, BPF_K.Value,
+                   (outs),
+                   (ins GPR32:$dst, i32imm:$imm, brtarget:$BrDst),
+                   "if $dst "#OpcodeStr#" $imm goto $BrDst",
+                   [(BPFbrcc i32:$dst, i32immSExt32:$imm, Cond, bb:$BrDst)]> {
+  bits<4> dst;
+  bits<16> BrDst;
+  bits<32> imm;
+
+  let Inst{51-48} = dst;
+  let Inst{47-32} = BrDst;
+  let Inst{31-0} = imm;
+  let BPFClass = BPF_JMP32;
+}
+
+multiclass J<BPFJumpOp Opc, string OpcodeStr, PatLeaf Cond, PatLeaf Cond32> {
   def _rr : JMP_RR<Opc, OpcodeStr, Cond>;
   def _ri : JMP_RI<Opc, OpcodeStr, Cond>;
+  def _rr_32 : JMP_RR_32<Opc, OpcodeStr, Cond32>;
+  def _ri_32 : JMP_RI_32<Opc, OpcodeStr, Cond32>;
 }
 
 let isBranch = 1, isTerminator = 1, hasDelaySlot=0 in {
 // cmp+goto instructions
-defm JEQ  : J<BPF_JEQ, "==",  BPF_CC_EQ>;
-defm JUGT : J<BPF_JGT, ">", BPF_CC_GTU>;
-defm JUGE : J<BPF_JGE, ">=", BPF_CC_GEU>;
-defm JNE  : J<BPF_JNE, "!=",  BPF_CC_NE>;
-defm JSGT : J<BPF_JSGT, "s>", BPF_CC_GT>;
-defm JSGE : J<BPF_JSGE, "s>=", BPF_CC_GE>;
-defm JULT : J<BPF_JLT, "<", BPF_CC_LTU>;
-defm JULE : J<BPF_JLE, "<=", BPF_CC_LEU>;
-defm JSLT : J<BPF_JSLT, "s<", BPF_CC_LT>;
-defm JSLE : J<BPF_JSLE, "s<=", BPF_CC_LE>;
+defm JEQ  : J<BPF_JEQ, "==",  BPF_CC_EQ, BPF_CC_EQ_32>;
+defm JUGT : J<BPF_JGT, ">", BPF_CC_GTU, BPF_CC_GTU_32>;
+defm JUGE : J<BPF_JGE, ">=", BPF_CC_GEU, BPF_CC_GEU_32>;
+defm JNE  : J<BPF_JNE, "!=",  BPF_CC_NE, BPF_CC_NE_32>;
+defm JSGT : J<BPF_JSGT, "s>", BPF_CC_GT, BPF_CC_GT_32>;
+defm JSGE : J<BPF_JSGE, "s>=", BPF_CC_GE, BPF_CC_GE_32>;
+defm JULT : J<BPF_JLT, "<", BPF_CC_LTU, BPF_CC_LTU_32>;
+defm JULE : J<BPF_JLE, "<=", BPF_CC_LEU, BPF_CC_LEU_32>;
+defm JSLT : J<BPF_JSLT, "s<", BPF_CC_LT, BPF_CC_LT_32>;
+defm JSLE : J<BPF_JSLE, "s<=", BPF_CC_LE, BPF_CC_LE_32>;
 }
 
 // ALU instructions
diff --git a/lib/Target/BPF/BPFMCInstLower.cpp b/lib/Target/BPF/BPFMCInstLower.cpp
index c8528e867310e52555fe8ea4e96cf8dd1ce41830..846798a63cb7cb9a583fed6b4e13ddd315db7cb9 100644
--- a/lib/Target/BPF/BPFMCInstLower.cpp
+++ b/lib/Target/BPF/BPFMCInstLower.cpp
@@ -1,9 +1,8 @@
 //=-- BPFMCInstLower.cpp - Convert BPF MachineInstr to an MCInst ------------=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/BPF/BPFMCInstLower.h b/lib/Target/BPF/BPFMCInstLower.h
index eac811f4cf8821ed41c59e9cc24616537fb6a392..0622d20814d31d18b76effe9305bbdc553d0be51 100644
--- a/lib/Target/BPF/BPFMCInstLower.h
+++ b/lib/Target/BPF/BPFMCInstLower.h
@@ -1,9 +1,8 @@
 //===-- BPFMCInstLower.h - Lower MachineInstr to MCInst ---------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/BPF/BPFMIChecking.cpp b/lib/Target/BPF/BPFMIChecking.cpp
index 0a311378e7774895e172fad5b6676db6a4f98789..a1e81cb62a979b6729f0e9994987aff3b2a2079e 100644
--- a/lib/Target/BPF/BPFMIChecking.cpp
+++ b/lib/Target/BPF/BPFMIChecking.cpp
@@ -1,9 +1,8 @@
 //===-------------- BPFMIChecking.cpp - MI Checking Legality -------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/BPF/BPFMIPeephole.cpp b/lib/Target/BPF/BPFMIPeephole.cpp
index 9e984d0facfbdfafb01b478315d8cb9a2d771bcb..156ba793e35921b50b3c9eabea11cdd3f26ae849 100644
--- a/lib/Target/BPF/BPFMIPeephole.cpp
+++ b/lib/Target/BPF/BPFMIPeephole.cpp
@@ -1,9 +1,8 @@
 //===-------------- BPFMIPeephole.cpp - MI Peephole Cleanups  -------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/BPF/BPFRegisterInfo.cpp b/lib/Target/BPF/BPFRegisterInfo.cpp
index 635c111131518304f11c3cd37c27f4615ee6fea0..15387b2bf3c445f57a48f6c163d6d5fc53ccdc60 100644
--- a/lib/Target/BPF/BPFRegisterInfo.cpp
+++ b/lib/Target/BPF/BPFRegisterInfo.cpp
@@ -1,9 +1,8 @@
 //===-- BPFRegisterInfo.cpp - BPF Register Information ----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/BPF/BPFRegisterInfo.h b/lib/Target/BPF/BPFRegisterInfo.h
index 4202850e9eb9a3414b2f01d4b13bfa83a58554e5..40d9bb7766106fc095e503987c538aabaa126ed0 100644
--- a/lib/Target/BPF/BPFRegisterInfo.h
+++ b/lib/Target/BPF/BPFRegisterInfo.h
@@ -1,9 +1,8 @@
 //===-- BPFRegisterInfo.h - BPF Register Information Impl -------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/BPF/BPFRegisterInfo.td b/lib/Target/BPF/BPFRegisterInfo.td
index da1d6b505f842c1126195d8a95b49077312499e2..88dec063be70247b2baa64a84aa819c67a477fc3 100644
--- a/lib/Target/BPF/BPFRegisterInfo.td
+++ b/lib/Target/BPF/BPFRegisterInfo.td
@@ -1,9 +1,8 @@
 //===-- BPFRegisterInfo.td - BPF Register defs -------------*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/BPF/BPFSelectionDAGInfo.cpp b/lib/Target/BPF/BPFSelectionDAGInfo.cpp
index 24d5f59bbfd73e8276ed3717381683171c17b456..a711294048ba30638a90b1eaf8d03381f8b1c274 100644
--- a/lib/Target/BPF/BPFSelectionDAGInfo.cpp
+++ b/lib/Target/BPF/BPFSelectionDAGInfo.cpp
@@ -1,9 +1,8 @@
 //===-- BPFSelectionDAGInfo.cpp - BPF SelectionDAG Info -------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/BPF/BPFSelectionDAGInfo.h b/lib/Target/BPF/BPFSelectionDAGInfo.h
index 19d3c5769573620fc5cb09a1a1bf6cbd4488c673..fb88c32ceb0cf0cb4b7906902a0ba264954bc898 100644
--- a/lib/Target/BPF/BPFSelectionDAGInfo.h
+++ b/lib/Target/BPF/BPFSelectionDAGInfo.h
@@ -1,9 +1,8 @@
 //===-- BPFSelectionDAGInfo.h - BPF SelectionDAG Info -----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/BPF/BPFSubtarget.cpp b/lib/Target/BPF/BPFSubtarget.cpp
index 56780bd9d46f71ddc636cbf9433f55bb5acc6a1f..ab3452501b9526b8dbb48d3afc93f57154856c27 100644
--- a/lib/Target/BPF/BPFSubtarget.cpp
+++ b/lib/Target/BPF/BPFSubtarget.cpp
@@ -1,9 +1,8 @@
 //===-- BPFSubtarget.cpp - BPF Subtarget Information ----------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -36,6 +35,7 @@ BPFSubtarget &BPFSubtarget::initializeSubtargetDependencies(StringRef CPU,
 
 void BPFSubtarget::initializeEnvironment() {
   HasJmpExt = false;
+  HasJmp32 = false;
   HasAlu32 = false;
   UseDwarfRIS = false;
 }
@@ -49,6 +49,11 @@ void BPFSubtarget::initSubtargetFeatures(StringRef CPU, StringRef FS) {
     HasJmpExt = true;
     return;
   }
+  if (CPU == "v3") {
+    HasJmpExt = true;
+    HasJmp32 = true;
+    return;
+  }
 }
 
 BPFSubtarget::BPFSubtarget(const Triple &TT, const std::string &CPU,
diff --git a/lib/Target/BPF/BPFSubtarget.h b/lib/Target/BPF/BPFSubtarget.h
index 60e56435fe4ceca233e397ad0e9f775700a912e2..3da6a026ab7ef1c398cb9dc1c45b079ed508e1dc 100644
--- a/lib/Target/BPF/BPFSubtarget.h
+++ b/lib/Target/BPF/BPFSubtarget.h
@@ -1,9 +1,8 @@
 //===-- BPFSubtarget.h - Define Subtarget for the BPF -----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -48,6 +47,10 @@ protected:
   // whether the cpu supports jmp ext
   bool HasJmpExt;
 
+  // whether the cpu supports jmp32 ext.
+  // NOTE: jmp32 is not enabled when alu32 enabled.
+  bool HasJmp32;
+
   // whether the cpu supports alu32 instructions.
   bool HasAlu32;
 
@@ -66,6 +69,7 @@ public:
   // subtarget options.  Definition of function is auto generated by tblgen.
   void ParseSubtargetFeatures(StringRef CPU, StringRef FS);
   bool getHasJmpExt() const { return HasJmpExt; }
+  bool getHasJmp32() const { return HasJmp32; }
   bool getHasAlu32() const { return HasAlu32; }
   bool getUseDwarfRIS() const { return UseDwarfRIS; }
 
diff --git a/lib/Target/BPF/BPFTargetMachine.cpp b/lib/Target/BPF/BPFTargetMachine.cpp
index 350465b118ed227a88f2637fcbe4b0d19c6cafc1..0efeaa2c4626b805517a462ff87b8aa0c5acf7a5 100644
--- a/lib/Target/BPF/BPFTargetMachine.cpp
+++ b/lib/Target/BPF/BPFTargetMachine.cpp
@@ -1,9 +1,8 @@
 //===-- BPFTargetMachine.cpp - Define TargetMachine for BPF ---------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/BPF/BPFTargetMachine.h b/lib/Target/BPF/BPFTargetMachine.h
index a560dd27335a14dd75b592b9f5325af13fe758b8..beac7bd862da20bd8803afe5ac33a2b13c07bdbf 100644
--- a/lib/Target/BPF/BPFTargetMachine.h
+++ b/lib/Target/BPF/BPFTargetMachine.h
@@ -1,9 +1,8 @@
 //===-- BPFTargetMachine.h - Define TargetMachine for BPF --- C++ ---===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/BPF/BTF.def b/lib/Target/BPF/BTF.def
new file mode 100644
index 0000000000000000000000000000000000000000..4069e41fa712038d3e92425f1e21a9a19778eb1c
--- /dev/null
+++ b/lib/Target/BPF/BTF.def
@@ -0,0 +1,32 @@
+//===- BTF.def - BTF definitions --------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Macros for BTF.
+//
+//===----------------------------------------------------------------------===//
+
+#if !defined(HANDLE_BTF_KIND)
+#error "Missing macro definition of HANDLE_BTF_*"
+#endif
+
+HANDLE_BTF_KIND(0, UNKN)
+HANDLE_BTF_KIND(1, INT)
+HANDLE_BTF_KIND(2, PTR)
+HANDLE_BTF_KIND(3, ARRAY)
+HANDLE_BTF_KIND(4, STRUCT)
+HANDLE_BTF_KIND(5, UNION)
+HANDLE_BTF_KIND(6, ENUM)
+HANDLE_BTF_KIND(7, FWD)
+HANDLE_BTF_KIND(8, TYPEDEF)
+HANDLE_BTF_KIND(9, VOLATILE)
+HANDLE_BTF_KIND(10, CONST)
+HANDLE_BTF_KIND(11, RESTRICT)
+HANDLE_BTF_KIND(12, FUNC)
+HANDLE_BTF_KIND(13, FUNC_PROTO)
+
+#undef HANDLE_BTF_KIND
diff --git a/lib/Target/BPF/BTF.h b/lib/Target/BPF/BTF.h
new file mode 100644
index 0000000000000000000000000000000000000000..cbe0c8588917138d9bd2614d98e51bfd67df722c
--- /dev/null
+++ b/lib/Target/BPF/BTF.h
@@ -0,0 +1,208 @@
+//===-- BTF.h --------------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file contains the layout of .BTF and .BTF.ext ELF sections.
+///
+/// The binary layout for .BTF section:
+///   struct Header
+///   Type and Str subsections
+/// The Type subsection is a collection of types with type id starting with 1.
+/// The Str subsection is simply a collection of strings.
+///
+/// The binary layout for .BTF.ext section:
+///   struct ExtHeader
+///   FuncInfo and LineInfo subsections
+/// The FuncInfo subsection is defined as below:
+///   BTFFuncInfo Size
+///   struct SecFuncInfo for ELF section #1
+///   A number of struct BPFFuncInfo for ELF section #1
+///   struct SecFuncInfo for ELF section #2
+///   A number of struct BPFFuncInfo for ELF section #2
+///   ...
+/// The LineInfo subsection is defined as below:
+///   BPFLineInfo Size
+///   struct SecLineInfo for ELF section #1
+///   A number of struct BPFLineInfo for ELF section #1
+///   struct SecLineInfo for ELF section #2
+///   A number of struct BPFLineInfo for ELF section #2
+///   ...
+///
+/// The section formats are also defined at
+///    https://github.com/torvalds/linux/blob/master/include/uapi/linux/btf.h
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_TARGET_BPF_BTF_H
+#define LLVM_LIB_TARGET_BPF_BTF_H
+
+namespace llvm {
+namespace BTF {
+
+enum : uint32_t { MAGIC = 0xeB9F, VERSION = 1 };
+
+/// Sizes in bytes of various things in the BTF format.
+enum {
+  HeaderSize = 24,
+  ExtHeaderSize = 24,
+  CommonTypeSize = 12,
+  BTFArraySize = 12,
+  BTFEnumSize = 8,
+  BTFMemberSize = 12,
+  BTFParamSize = 8,
+  SecFuncInfoSize = 8,
+  SecLineInfoSize = 8,
+  BPFFuncInfoSize = 8,
+  BPFLineInfoSize = 16
+};
+
+/// The .BTF section header definition.
+struct Header {
+  uint16_t Magic;  ///< Magic value
+  uint8_t Version; ///< Version number
+  uint8_t Flags;   ///< Extra flags
+  uint32_t HdrLen; ///< Length of this header
+
+  /// All offsets are in bytes relative to the end of this header.
+  uint32_t TypeOff; ///< Offset of type section
+  uint32_t TypeLen; ///< Length of type section
+  uint32_t StrOff;  ///< Offset of string section
+  uint32_t StrLen;  ///< Length of string section
+};
+
+enum : uint32_t {
+  MAX_VLEN = 0xffff         ///< Max # of struct/union/enum members or func args
+};
+
+enum TypeKinds : uint8_t {
+#define HANDLE_BTF_KIND(ID, NAME) BTF_KIND_##NAME = ID,
+#include "BTF.def"
+};
+
+/// The BTF common type definition. Different kinds may have
+/// additional information after this structure data.
+struct CommonType {
+  /// Type name offset in the string table.
+  uint32_t NameOff;
+
+  /// "Info" bits arrangement:
+  /// Bits  0-15: vlen (e.g. # of struct's members)
+  /// Bits 16-23: unused
+  /// Bits 24-27: kind (e.g. int, ptr, array...etc)
+  /// Bits 28-30: unused
+  /// Bit     31: kind_flag, currently used by
+  ///             struct, union and fwd
+  uint32_t Info;
+
+  /// "Size" is used by INT, ENUM, STRUCT and UNION.
+  /// "Size" tells the size of the type it is describing.
+  ///
+  /// "Type" is used by PTR, TYPEDEF, VOLATILE, CONST, RESTRICT,
+  /// FUNC and FUNC_PROTO.
+  /// "Type" is a type_id referring to another type.
+  union {
+    uint32_t Size;
+    uint32_t Type;
+  };
+};
+
+// For some specific BTF_KIND, "struct CommonType" is immediately
+// followed by extra data.
+
+// BTF_KIND_INT is followed by a u32 and the following
+// is the 32 bits arrangement:
+// BTF_INT_ENCODING(VAL) : (((VAL) & 0x0f000000) >> 24)
+// BTF_INT_OFFSET(VAL) : (((VAL & 0x00ff0000)) >> 16)
+// BTF_INT_BITS(VAL) : ((VAL) & 0x000000ff)
+
+/// Attributes stored in the INT_ENCODING.
+enum : uint8_t { INT_SIGNED = (1 << 0), INT_CHAR = (1 << 1), INT_BOOL = (1 << 2) };
+
+/// BTF_KIND_ENUM is followed by multiple "struct BTFEnum".
+/// The exact number of btf_enum is stored in the vlen (of the
+/// info in "struct CommonType").
+struct BTFEnum {
+  uint32_t NameOff; ///< Enum name offset in the string table
+  int32_t Val;      ///< Enum member value
+};
+
+/// BTF_KIND_ARRAY is followed by one "struct BTFArray".
+struct BTFArray {
+  uint32_t ElemType;  ///< Element type
+  uint32_t IndexType; ///< Index type
+  uint32_t Nelems;    ///< Number of elements for this array
+};
+
+/// BTF_KIND_STRUCT and BTF_KIND_UNION are followed
+/// by multiple "struct BTFMember".  The exact number
+/// of BTFMember is stored in the vlen (of the info in
+/// "struct CommonType").
+///
+/// If the struct/union contains any bitfield member,
+/// the Offset below represents BitOffset (bits 0 - 23)
+/// and BitFieldSize(bits 24 - 31) with BitFieldSize = 0
+/// for non bitfield members. Otherwise, the Offset
+/// represents the BitOffset.
+struct BTFMember {
+  uint32_t NameOff; ///< Member name offset in the string table
+  uint32_t Type;    ///< Member type
+  uint32_t Offset;  ///< BitOffset or BitFieldSize+BitOffset
+};
+
+/// BTF_KIND_FUNC_PROTO are followed by multiple "struct BTFParam".
+/// The exist number of BTFParam is stored in the vlen (of the info
+/// in "struct CommonType").
+struct BTFParam {
+  uint32_t NameOff;
+  uint32_t Type;
+};
+
+/// The .BTF.ext section header definition.
+struct ExtHeader {
+  uint16_t Magic;
+  uint8_t Version;
+  uint8_t Flags;
+  uint32_t HdrLen;
+
+  uint32_t FuncInfoOff; ///< Offset of func info section
+  uint32_t FuncInfoLen; ///< Length of func info section
+  uint32_t LineInfoOff; ///< Offset of line info section
+  uint32_t LineInfoLen; ///< Length of line info section
+};
+
+/// Specifying one function info.
+struct BPFFuncInfo {
+  uint32_t InsnOffset; ///< Byte offset in the section
+  uint32_t TypeId;     ///< Type id referring to .BTF type section
+};
+
+/// Specifying function info's in one section.
+struct SecFuncInfo {
+  uint32_t SecNameOff;  ///< Section name index in the .BTF string table
+  uint32_t NumFuncInfo; ///< Number of func info's in this section
+};
+
+/// Specifying one line info.
+struct BPFLineInfo {
+  uint32_t InsnOffset;  ///< Byte offset in this section
+  uint32_t FileNameOff; ///< File name index in the .BTF string table
+  uint32_t LineOff;     ///< Line index in the .BTF string table
+  uint32_t LineCol;     ///< Line num: line_col >> 10,
+                        ///  col num: line_col & 0x3ff
+};
+
+/// Specifying line info's in one section.
+struct SecLineInfo {
+  uint32_t SecNameOff;  ///< Section name index in the .BTF string tble
+  uint32_t NumLineInfo; ///< Number of line info's in this section
+};
+
+} // End namespace BTF.
+} // End namespace llvm.
+
+#endif
diff --git a/lib/Target/BPF/BTFDebug.cpp b/lib/Target/BPF/BTFDebug.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..ca3a8c5740e2e6bba51c47f5aecc15a600ab8164
--- /dev/null
+++ b/lib/Target/BPF/BTFDebug.cpp
@@ -0,0 +1,758 @@
+//===- BTFDebug.cpp - BTF Generator ---------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains support for writing BTF debug info.
+//
+//===----------------------------------------------------------------------===//
+
+#include "BTFDebug.h"
+#include "llvm/BinaryFormat/ELF.h"
+#include "llvm/CodeGen/AsmPrinter.h"
+#include "llvm/CodeGen/MachineModuleInfo.h"
+#include "llvm/MC/MCContext.h"
+#include "llvm/MC/MCObjectFileInfo.h"
+#include "llvm/MC/MCSectionELF.h"
+#include "llvm/MC/MCStreamer.h"
+#include <fstream>
+#include <sstream>
+
+using namespace llvm;
+
+static const char *BTFKindStr[] = {
+#define HANDLE_BTF_KIND(ID, NAME) "BTF_KIND_" #NAME,
+#include "BTF.def"
+};
+
+/// Emit a BTF common type.
+void BTFTypeBase::emitType(MCStreamer &OS) {
+  OS.AddComment(std::string(BTFKindStr[Kind]) + "(id = " + std::to_string(Id) +
+                ")");
+  OS.EmitIntValue(BTFType.NameOff, 4);
+  OS.AddComment("0x" + Twine::utohexstr(BTFType.Info));
+  OS.EmitIntValue(BTFType.Info, 4);
+  OS.EmitIntValue(BTFType.Size, 4);
+}
+
+BTFTypeDerived::BTFTypeDerived(const DIDerivedType *DTy, unsigned Tag)
+    : DTy(DTy) {
+  switch (Tag) {
+  case dwarf::DW_TAG_pointer_type:
+    Kind = BTF::BTF_KIND_PTR;
+    break;
+  case dwarf::DW_TAG_const_type:
+    Kind = BTF::BTF_KIND_CONST;
+    break;
+  case dwarf::DW_TAG_volatile_type:
+    Kind = BTF::BTF_KIND_VOLATILE;
+    break;
+  case dwarf::DW_TAG_typedef:
+    Kind = BTF::BTF_KIND_TYPEDEF;
+    break;
+  case dwarf::DW_TAG_restrict_type:
+    Kind = BTF::BTF_KIND_RESTRICT;
+    break;
+  default:
+    llvm_unreachable("Unknown DIDerivedType Tag");
+  }
+  BTFType.Info = Kind << 24;
+}
+
+void BTFTypeDerived::completeType(BTFDebug &BDebug) {
+  BTFType.NameOff = BDebug.addString(DTy->getName());
+
+  // The base type for PTR/CONST/VOLATILE could be void.
+  const DIType *ResolvedType = DTy->getBaseType().resolve();
+  if (!ResolvedType) {
+    assert((Kind == BTF::BTF_KIND_PTR || Kind == BTF::BTF_KIND_CONST ||
+            Kind == BTF::BTF_KIND_VOLATILE) &&
+           "Invalid null basetype");
+    BTFType.Type = 0;
+  } else {
+    BTFType.Type = BDebug.getTypeId(ResolvedType);
+  }
+}
+
+void BTFTypeDerived::emitType(MCStreamer &OS) { BTFTypeBase::emitType(OS); }
+
+/// Represent a struct/union forward declaration.
+BTFTypeFwd::BTFTypeFwd(StringRef Name, bool IsUnion) : Name(Name) {
+  Kind = BTF::BTF_KIND_FWD;
+  BTFType.Info = IsUnion << 31 | Kind << 24;
+  BTFType.Type = 0;
+}
+
+void BTFTypeFwd::completeType(BTFDebug &BDebug) {
+  BTFType.NameOff = BDebug.addString(Name);
+}
+
+void BTFTypeFwd::emitType(MCStreamer &OS) { BTFTypeBase::emitType(OS); }
+
+BTFTypeInt::BTFTypeInt(uint32_t Encoding, uint32_t SizeInBits,
+                       uint32_t OffsetInBits, StringRef TypeName)
+    : Name(TypeName) {
+  // Translate IR int encoding to BTF int encoding.
+  uint8_t BTFEncoding;
+  switch (Encoding) {
+  case dwarf::DW_ATE_boolean:
+    BTFEncoding = BTF::INT_BOOL;
+    break;
+  case dwarf::DW_ATE_signed:
+  case dwarf::DW_ATE_signed_char:
+    BTFEncoding = BTF::INT_SIGNED;
+    break;
+  case dwarf::DW_ATE_unsigned:
+  case dwarf::DW_ATE_unsigned_char:
+    BTFEncoding = 0;
+    break;
+  default:
+    llvm_unreachable("Unknown BTFTypeInt Encoding");
+  }
+
+  Kind = BTF::BTF_KIND_INT;
+  BTFType.Info = Kind << 24;
+  BTFType.Size = roundupToBytes(SizeInBits);
+  IntVal = (BTFEncoding << 24) | OffsetInBits << 16 | SizeInBits;
+}
+
+void BTFTypeInt::completeType(BTFDebug &BDebug) {
+  BTFType.NameOff = BDebug.addString(Name);
+}
+
+void BTFTypeInt::emitType(MCStreamer &OS) {
+  BTFTypeBase::emitType(OS);
+  OS.AddComment("0x" + Twine::utohexstr(IntVal));
+  OS.EmitIntValue(IntVal, 4);
+}
+
+BTFTypeEnum::BTFTypeEnum(const DICompositeType *ETy, uint32_t VLen) : ETy(ETy) {
+  Kind = BTF::BTF_KIND_ENUM;
+  BTFType.Info = Kind << 24 | VLen;
+  BTFType.Size = roundupToBytes(ETy->getSizeInBits());
+}
+
+void BTFTypeEnum::completeType(BTFDebug &BDebug) {
+  BTFType.NameOff = BDebug.addString(ETy->getName());
+
+  DINodeArray Elements = ETy->getElements();
+  for (const auto Element : Elements) {
+    const auto *Enum = cast<DIEnumerator>(Element);
+
+    struct BTF::BTFEnum BTFEnum;
+    BTFEnum.NameOff = BDebug.addString(Enum->getName());
+    // BTF enum value is 32bit, enforce it.
+    BTFEnum.Val = static_cast<uint32_t>(Enum->getValue());
+    EnumValues.push_back(BTFEnum);
+  }
+}
+
+void BTFTypeEnum::emitType(MCStreamer &OS) {
+  BTFTypeBase::emitType(OS);
+  for (const auto &Enum : EnumValues) {
+    OS.EmitIntValue(Enum.NameOff, 4);
+    OS.EmitIntValue(Enum.Val, 4);
+  }
+}
+
+BTFTypeArray::BTFTypeArray(const DICompositeType *ATy) : ATy(ATy) {
+  Kind = BTF::BTF_KIND_ARRAY;
+  BTFType.Info = Kind << 24;
+}
+
+/// Represent a BTF array. BTF does not record array dimensions,
+/// so conceptually a BTF array is a one-dimensional array.
+void BTFTypeArray::completeType(BTFDebug &BDebug) {
+  BTFType.NameOff = BDebug.addString(ATy->getName());
+  BTFType.Size = 0;
+
+  auto *BaseType = ATy->getBaseType().resolve();
+  ArrayInfo.ElemType = BDebug.getTypeId(BaseType);
+
+  // The IR does not really have a type for the index.
+  // A special type for array index should have been
+  // created during initial type traversal. Just
+  // retrieve that type id.
+  ArrayInfo.IndexType = BDebug.getArrayIndexTypeId();
+
+  // Get the number of array elements.
+  // If the array size is 0, set the number of elements as 0.
+  // Otherwise, recursively traverse the base types to
+  // find the element size. The number of elements is
+  // the totoal array size in bits divided by
+  // element size in bits.
+  uint64_t ArraySizeInBits = ATy->getSizeInBits();
+  if (!ArraySizeInBits) {
+    ArrayInfo.Nelems = 0;
+  } else {
+    uint32_t BaseTypeSize = BaseType->getSizeInBits();
+    while (!BaseTypeSize) {
+      const auto *DDTy = cast<DIDerivedType>(BaseType);
+      BaseType = DDTy->getBaseType().resolve();
+      assert(BaseType);
+      BaseTypeSize = BaseType->getSizeInBits();
+    }
+    ArrayInfo.Nelems = ATy->getSizeInBits() / BaseTypeSize;
+  }
+}
+
+void BTFTypeArray::emitType(MCStreamer &OS) {
+  BTFTypeBase::emitType(OS);
+  OS.EmitIntValue(ArrayInfo.ElemType, 4);
+  OS.EmitIntValue(ArrayInfo.IndexType, 4);
+  OS.EmitIntValue(ArrayInfo.Nelems, 4);
+}
+
+/// Represent either a struct or a union.
+BTFTypeStruct::BTFTypeStruct(const DICompositeType *STy, bool IsStruct,
+                             bool HasBitField, uint32_t Vlen)
+    : STy(STy), HasBitField(HasBitField) {
+  Kind = IsStruct ? BTF::BTF_KIND_STRUCT : BTF::BTF_KIND_UNION;
+  BTFType.Size = roundupToBytes(STy->getSizeInBits());
+  BTFType.Info = (HasBitField << 31) | (Kind << 24) | Vlen;
+}
+
+void BTFTypeStruct::completeType(BTFDebug &BDebug) {
+  BTFType.NameOff = BDebug.addString(STy->getName());
+
+  // Add struct/union members.
+  const DINodeArray Elements = STy->getElements();
+  for (const auto *Element : Elements) {
+    struct BTF::BTFMember BTFMember;
+    const auto *DDTy = cast<DIDerivedType>(Element);
+
+    BTFMember.NameOff = BDebug.addString(DDTy->getName());
+    if (HasBitField) {
+      uint8_t BitFieldSize = DDTy->isBitField() ? DDTy->getSizeInBits() : 0;
+      BTFMember.Offset = BitFieldSize << 24 | DDTy->getOffsetInBits();
+    } else {
+      BTFMember.Offset = DDTy->getOffsetInBits();
+    }
+    BTFMember.Type = BDebug.getTypeId(DDTy->getBaseType().resolve());
+    Members.push_back(BTFMember);
+  }
+}
+
+void BTFTypeStruct::emitType(MCStreamer &OS) {
+  BTFTypeBase::emitType(OS);
+  for (const auto &Member : Members) {
+    OS.EmitIntValue(Member.NameOff, 4);
+    OS.EmitIntValue(Member.Type, 4);
+    OS.AddComment("0x" + Twine::utohexstr(Member.Offset));
+    OS.EmitIntValue(Member.Offset, 4);
+  }
+}
+
+/// The Func kind represents both subprogram and pointee of function
+/// pointers. If the FuncName is empty, it represents a pointee of function
+/// pointer. Otherwise, it represents a subprogram. The func arg names
+/// are empty for pointee of function pointer case, and are valid names
+/// for subprogram.
+BTFTypeFuncProto::BTFTypeFuncProto(
+    const DISubroutineType *STy, uint32_t VLen,
+    const std::unordered_map<uint32_t, StringRef> &FuncArgNames)
+    : STy(STy), FuncArgNames(FuncArgNames) {
+  Kind = BTF::BTF_KIND_FUNC_PROTO;
+  BTFType.Info = (Kind << 24) | VLen;
+}
+
+void BTFTypeFuncProto::completeType(BTFDebug &BDebug) {
+  DITypeRefArray Elements = STy->getTypeArray();
+  auto RetType = Elements[0].resolve();
+  BTFType.Type = RetType ? BDebug.getTypeId(RetType) : 0;
+  BTFType.NameOff = 0;
+
+  // For null parameter which is typically the last one
+  // to represent the vararg, encode the NameOff/Type to be 0.
+  for (unsigned I = 1, N = Elements.size(); I < N; ++I) {
+    struct BTF::BTFParam Param;
+    auto Element = Elements[I].resolve();
+    if (Element) {
+      Param.NameOff = BDebug.addString(FuncArgNames[I]);
+      Param.Type = BDebug.getTypeId(Element);
+    } else {
+      Param.NameOff = 0;
+      Param.Type = 0;
+    }
+    Parameters.push_back(Param);
+  }
+}
+
+void BTFTypeFuncProto::emitType(MCStreamer &OS) {
+  BTFTypeBase::emitType(OS);
+  for (const auto &Param : Parameters) {
+    OS.EmitIntValue(Param.NameOff, 4);
+    OS.EmitIntValue(Param.Type, 4);
+  }
+}
+
+BTFTypeFunc::BTFTypeFunc(StringRef FuncName, uint32_t ProtoTypeId)
+    : Name(FuncName) {
+  Kind = BTF::BTF_KIND_FUNC;
+  BTFType.Info = Kind << 24;
+  BTFType.Type = ProtoTypeId;
+}
+
+void BTFTypeFunc::completeType(BTFDebug &BDebug) {
+  BTFType.NameOff = BDebug.addString(Name);
+}
+
+void BTFTypeFunc::emitType(MCStreamer &OS) { BTFTypeBase::emitType(OS); }
+
+uint32_t BTFStringTable::addString(StringRef S) {
+  // Check whether the string already exists.
+  for (auto &OffsetM : OffsetToIdMap) {
+    if (Table[OffsetM.second] == S)
+      return OffsetM.first;
+  }
+  // Not find, add to the string table.
+  uint32_t Offset = Size;
+  OffsetToIdMap[Offset] = Table.size();
+  Table.push_back(S);
+  Size += S.size() + 1;
+  return Offset;
+}
+
+BTFDebug::BTFDebug(AsmPrinter *AP)
+    : DebugHandlerBase(AP), OS(*Asm->OutStreamer), SkipInstruction(false),
+      LineInfoGenerated(false), SecNameOff(0), ArrayIndexTypeId(0) {
+  addString("\0");
+}
+
+void BTFDebug::addType(std::unique_ptr<BTFTypeBase> TypeEntry,
+                       const DIType *Ty) {
+  TypeEntry->setId(TypeEntries.size() + 1);
+  DIToIdMap[Ty] = TypeEntry->getId();
+  TypeEntries.push_back(std::move(TypeEntry));
+}
+
+uint32_t BTFDebug::addType(std::unique_ptr<BTFTypeBase> TypeEntry) {
+  TypeEntry->setId(TypeEntries.size() + 1);
+  uint32_t Id = TypeEntry->getId();
+  TypeEntries.push_back(std::move(TypeEntry));
+  return Id;
+}
+
+void BTFDebug::visitBasicType(const DIBasicType *BTy) {
+  // Only int types are supported in BTF.
+  uint32_t Encoding = BTy->getEncoding();
+  if (Encoding != dwarf::DW_ATE_boolean && Encoding != dwarf::DW_ATE_signed &&
+      Encoding != dwarf::DW_ATE_signed_char &&
+      Encoding != dwarf::DW_ATE_unsigned &&
+      Encoding != dwarf::DW_ATE_unsigned_char)
+    return;
+
+  // Create a BTF type instance for this DIBasicType and put it into
+  // DIToIdMap for cross-type reference check.
+  auto TypeEntry = llvm::make_unique<BTFTypeInt>(
+      Encoding, BTy->getSizeInBits(), BTy->getOffsetInBits(), BTy->getName());
+  addType(std::move(TypeEntry), BTy);
+}
+
+/// Handle subprogram or subroutine types.
+void BTFDebug::visitSubroutineType(
+    const DISubroutineType *STy, bool ForSubprog,
+    const std::unordered_map<uint32_t, StringRef> &FuncArgNames,
+    uint32_t &TypeId) {
+  DITypeRefArray Elements = STy->getTypeArray();
+  uint32_t VLen = Elements.size() - 1;
+  if (VLen > BTF::MAX_VLEN)
+    return;
+
+  // Subprogram has a valid non-zero-length name, and the pointee of
+  // a function pointer has an empty name. The subprogram type will
+  // not be added to DIToIdMap as it should not be referenced by
+  // any other types.
+  auto TypeEntry = llvm::make_unique<BTFTypeFuncProto>(STy, VLen, FuncArgNames);
+  if (ForSubprog)
+    TypeId = addType(std::move(TypeEntry)); // For subprogram
+  else
+    addType(std::move(TypeEntry), STy); // For func ptr
+
+  // Visit return type and func arg types.
+  for (const auto Element : Elements) {
+    visitTypeEntry(Element.resolve());
+  }
+}
+
+/// Handle structure/union types.
+void BTFDebug::visitStructType(const DICompositeType *CTy, bool IsStruct) {
+  const DINodeArray Elements = CTy->getElements();
+  uint32_t VLen = Elements.size();
+  if (VLen > BTF::MAX_VLEN)
+    return;
+
+  // Check whether we have any bitfield members or not
+  bool HasBitField = false;
+  for (const auto *Element : Elements) {
+    auto E = cast<DIDerivedType>(Element);
+    if (E->isBitField()) {
+      HasBitField = true;
+      break;
+    }
+  }
+
+  auto TypeEntry =
+      llvm::make_unique<BTFTypeStruct>(CTy, IsStruct, HasBitField, VLen);
+  addType(std::move(TypeEntry), CTy);
+
+  // Visit all struct members.
+  for (const auto *Element : Elements)
+    visitTypeEntry(cast<DIDerivedType>(Element));
+}
+
+void BTFDebug::visitArrayType(const DICompositeType *CTy) {
+  auto TypeEntry = llvm::make_unique<BTFTypeArray>(CTy);
+  addType(std::move(TypeEntry), CTy);
+
+  // The IR does not have a type for array index while BTF wants one.
+  // So create an array index type if there is none.
+  if (!ArrayIndexTypeId) {
+    auto TypeEntry = llvm::make_unique<BTFTypeInt>(dwarf::DW_ATE_unsigned, 32,
+                                                   0, "__ARRAY_SIZE_TYPE__");
+    ArrayIndexTypeId = addType(std::move(TypeEntry));
+  }
+
+  // Visit array element type.
+  visitTypeEntry(CTy->getBaseType().resolve());
+}
+
+void BTFDebug::visitEnumType(const DICompositeType *CTy) {
+  DINodeArray Elements = CTy->getElements();
+  uint32_t VLen = Elements.size();
+  if (VLen > BTF::MAX_VLEN)
+    return;
+
+  auto TypeEntry = llvm::make_unique<BTFTypeEnum>(CTy, VLen);
+  addType(std::move(TypeEntry), CTy);
+  // No need to visit base type as BTF does not encode it.
+}
+
+/// Handle structure/union forward declarations.
+void BTFDebug::visitFwdDeclType(const DICompositeType *CTy, bool IsUnion) {
+  auto TypeEntry = llvm::make_unique<BTFTypeFwd>(CTy->getName(), IsUnion);
+  addType(std::move(TypeEntry), CTy);
+}
+
+/// Handle structure, union, array and enumeration types.
+void BTFDebug::visitCompositeType(const DICompositeType *CTy) {
+  auto Tag = CTy->getTag();
+  if (Tag == dwarf::DW_TAG_structure_type || Tag == dwarf::DW_TAG_union_type) {
+    // Handle forward declaration differently as it does not have members.
+    if (CTy->isForwardDecl())
+      visitFwdDeclType(CTy, Tag == dwarf::DW_TAG_union_type);
+    else
+      visitStructType(CTy, Tag == dwarf::DW_TAG_structure_type);
+  } else if (Tag == dwarf::DW_TAG_array_type)
+    visitArrayType(CTy);
+  else if (Tag == dwarf::DW_TAG_enumeration_type)
+    visitEnumType(CTy);
+}
+
+/// Handle pointer, typedef, const, volatile, restrict and member types.
+void BTFDebug::visitDerivedType(const DIDerivedType *DTy) {
+  unsigned Tag = DTy->getTag();
+
+  if (Tag == dwarf::DW_TAG_pointer_type || Tag == dwarf::DW_TAG_typedef ||
+      Tag == dwarf::DW_TAG_const_type || Tag == dwarf::DW_TAG_volatile_type ||
+      Tag == dwarf::DW_TAG_restrict_type) {
+    auto TypeEntry = llvm::make_unique<BTFTypeDerived>(DTy, Tag);
+    addType(std::move(TypeEntry), DTy);
+  } else if (Tag != dwarf::DW_TAG_member) {
+    return;
+  }
+
+  // Visit base type of pointer, typedef, const, volatile, restrict or
+  // struct/union member.
+  visitTypeEntry(DTy->getBaseType().resolve());
+}
+
+void BTFDebug::visitTypeEntry(const DIType *Ty) {
+  if (!Ty || DIToIdMap.find(Ty) != DIToIdMap.end())
+    return;
+
+  uint32_t TypeId;
+  if (const auto *BTy = dyn_cast<DIBasicType>(Ty))
+    visitBasicType(BTy);
+  else if (const auto *STy = dyn_cast<DISubroutineType>(Ty))
+    visitSubroutineType(STy, false, std::unordered_map<uint32_t, StringRef>(),
+                        TypeId);
+  else if (const auto *CTy = dyn_cast<DICompositeType>(Ty))
+    visitCompositeType(CTy);
+  else if (const auto *DTy = dyn_cast<DIDerivedType>(Ty))
+    visitDerivedType(DTy);
+  else
+    llvm_unreachable("Unknown DIType");
+}
+
+/// Read file contents from the actual file or from the source
+std::string BTFDebug::populateFileContent(const DISubprogram *SP) {
+  auto File = SP->getFile();
+  std::string FileName;
+
+  if (!File->getFilename().startswith("/") && File->getDirectory().size())
+    FileName = File->getDirectory().str() + "/" + File->getFilename().str();
+  else
+    FileName = File->getFilename();
+
+  // No need to populate the contends if it has been populated!
+  if (FileContent.find(FileName) != FileContent.end())
+    return FileName;
+
+  std::vector<std::string> Content;
+  std::string Line;
+  Content.push_back(Line); // Line 0 for empty string
+
+  auto Source = File->getSource();
+  if (Source) {
+    std::istringstream InputString(Source.getValue());
+    while (std::getline(InputString, Line))
+      Content.push_back(Line);
+  } else {
+    std::ifstream InputFile(FileName);
+    while (std::getline(InputFile, Line))
+      Content.push_back(Line);
+  }
+
+  FileContent[FileName] = Content;
+  return FileName;
+}
+
+void BTFDebug::constructLineInfo(const DISubprogram *SP, MCSymbol *Label,
+                                 uint32_t Line, uint32_t Column) {
+  std::string FileName = populateFileContent(SP);
+  BTFLineInfo LineInfo;
+
+  LineInfo.Label = Label;
+  LineInfo.FileNameOff = addString(FileName);
+  // If file content is not available, let LineOff = 0.
+  if (Line < FileContent[FileName].size())
+    LineInfo.LineOff = addString(FileContent[FileName][Line]);
+  else
+    LineInfo.LineOff = 0;
+  LineInfo.LineNum = Line;
+  LineInfo.ColumnNum = Column;
+  LineInfoTable[SecNameOff].push_back(LineInfo);
+}
+
+void BTFDebug::emitCommonHeader() {
+  OS.AddComment("0x" + Twine::utohexstr(BTF::MAGIC));
+  OS.EmitIntValue(BTF::MAGIC, 2);
+  OS.EmitIntValue(BTF::VERSION, 1);
+  OS.EmitIntValue(0, 1);
+}
+
+void BTFDebug::emitBTFSection() {
+  MCContext &Ctx = OS.getContext();
+  OS.SwitchSection(Ctx.getELFSection(".BTF", ELF::SHT_PROGBITS, 0));
+
+  // Emit header.
+  emitCommonHeader();
+  OS.EmitIntValue(BTF::HeaderSize, 4);
+
+  uint32_t TypeLen = 0, StrLen;
+  for (const auto &TypeEntry : TypeEntries)
+    TypeLen += TypeEntry->getSize();
+  StrLen = StringTable.getSize();
+
+  OS.EmitIntValue(0, 4);
+  OS.EmitIntValue(TypeLen, 4);
+  OS.EmitIntValue(TypeLen, 4);
+  OS.EmitIntValue(StrLen, 4);
+
+  // Emit type table.
+  for (const auto &TypeEntry : TypeEntries)
+    TypeEntry->emitType(OS);
+
+  // Emit string table.
+  uint32_t StringOffset = 0;
+  for (const auto &S : StringTable.getTable()) {
+    OS.AddComment("string offset=" + std::to_string(StringOffset));
+    OS.EmitBytes(S);
+    OS.EmitBytes(StringRef("\0", 1));
+    StringOffset += S.size() + 1;
+  }
+}
+
+void BTFDebug::emitBTFExtSection() {
+  MCContext &Ctx = OS.getContext();
+  OS.SwitchSection(Ctx.getELFSection(".BTF.ext", ELF::SHT_PROGBITS, 0));
+
+  // Emit header.
+  emitCommonHeader();
+  OS.EmitIntValue(BTF::ExtHeaderSize, 4);
+
+  // Account for FuncInfo/LineInfo record size as well.
+  uint32_t FuncLen = 4, LineLen = 4;
+  for (const auto &FuncSec : FuncInfoTable) {
+    FuncLen += BTF::SecFuncInfoSize;
+    FuncLen += FuncSec.second.size() * BTF::BPFFuncInfoSize;
+  }
+  for (const auto &LineSec : LineInfoTable) {
+    LineLen += BTF::SecLineInfoSize;
+    LineLen += LineSec.second.size() * BTF::BPFLineInfoSize;
+  }
+
+  OS.EmitIntValue(0, 4);
+  OS.EmitIntValue(FuncLen, 4);
+  OS.EmitIntValue(FuncLen, 4);
+  OS.EmitIntValue(LineLen, 4);
+
+  // Emit func_info table.
+  OS.AddComment("FuncInfo");
+  OS.EmitIntValue(BTF::BPFFuncInfoSize, 4);
+  for (const auto &FuncSec : FuncInfoTable) {
+    OS.AddComment("FuncInfo section string offset=" +
+                  std::to_string(FuncSec.first));
+    OS.EmitIntValue(FuncSec.first, 4);
+    OS.EmitIntValue(FuncSec.second.size(), 4);
+    for (const auto &FuncInfo : FuncSec.second) {
+      Asm->EmitLabelReference(FuncInfo.Label, 4);
+      OS.EmitIntValue(FuncInfo.TypeId, 4);
+    }
+  }
+
+  // Emit line_info table.
+  OS.AddComment("LineInfo");
+  OS.EmitIntValue(BTF::BPFLineInfoSize, 4);
+  for (const auto &LineSec : LineInfoTable) {
+    OS.AddComment("LineInfo section string offset=" +
+                  std::to_string(LineSec.first));
+    OS.EmitIntValue(LineSec.first, 4);
+    OS.EmitIntValue(LineSec.second.size(), 4);
+    for (const auto &LineInfo : LineSec.second) {
+      Asm->EmitLabelReference(LineInfo.Label, 4);
+      OS.EmitIntValue(LineInfo.FileNameOff, 4);
+      OS.EmitIntValue(LineInfo.LineOff, 4);
+      OS.AddComment("Line " + std::to_string(LineInfo.LineNum) + " Col " +
+                    std::to_string(LineInfo.ColumnNum));
+      OS.EmitIntValue(LineInfo.LineNum << 10 | LineInfo.ColumnNum, 4);
+    }
+  }
+}
+
+void BTFDebug::beginFunctionImpl(const MachineFunction *MF) {
+  auto *SP = MF->getFunction().getSubprogram();
+  auto *Unit = SP->getUnit();
+
+  if (Unit->getEmissionKind() == DICompileUnit::NoDebug) {
+    SkipInstruction = true;
+    return;
+  }
+  SkipInstruction = false;
+
+  // Collect all types locally referenced in this function.
+  // Use RetainedNodes so we can collect all argument names
+  // even if the argument is not used.
+  std::unordered_map<uint32_t, StringRef> FuncArgNames;
+  for (const DINode *DN : SP->getRetainedNodes()) {
+    if (const auto *DV = dyn_cast<DILocalVariable>(DN)) {
+      visitTypeEntry(DV->getType().resolve());
+
+      // Collect function arguments for subprogram func type.
+      uint32_t Arg = DV->getArg();
+      if (Arg)
+        FuncArgNames[Arg] = DV->getName();
+    }
+  }
+
+  // Construct subprogram func proto type.
+  uint32_t ProtoTypeId;
+  visitSubroutineType(SP->getType(), true, FuncArgNames, ProtoTypeId);
+
+  // Construct subprogram func type
+  auto FuncTypeEntry =
+      llvm::make_unique<BTFTypeFunc>(SP->getName(), ProtoTypeId);
+  uint32_t FuncTypeId = addType(std::move(FuncTypeEntry));
+
+  // Construct funcinfo and the first lineinfo for the function.
+  MCSymbol *FuncLabel = Asm->getFunctionBegin();
+  BTFFuncInfo FuncInfo;
+  FuncInfo.Label = FuncLabel;
+  FuncInfo.TypeId = FuncTypeId;
+  if (FuncLabel->isInSection()) {
+    MCSection &Section = FuncLabel->getSection();
+    const MCSectionELF *SectionELF = dyn_cast<MCSectionELF>(&Section);
+    assert(SectionELF && "Null section for Function Label");
+    SecNameOff = addString(SectionELF->getSectionName());
+  } else {
+    SecNameOff = addString(".text");
+  }
+  FuncInfoTable[SecNameOff].push_back(FuncInfo);
+}
+
+void BTFDebug::endFunctionImpl(const MachineFunction *MF) {
+  SkipInstruction = false;
+  LineInfoGenerated = false;
+  SecNameOff = 0;
+}
+
+void BTFDebug::beginInstruction(const MachineInstr *MI) {
+  DebugHandlerBase::beginInstruction(MI);
+
+  if (SkipInstruction || MI->isMetaInstruction() ||
+      MI->getFlag(MachineInstr::FrameSetup))
+    return;
+
+  if (MI->isInlineAsm()) {
+    // Count the number of register definitions to find the asm string.
+    unsigned NumDefs = 0;
+    for (; MI->getOperand(NumDefs).isReg() && MI->getOperand(NumDefs).isDef();
+         ++NumDefs)
+      ;
+
+    // Skip this inline asm instruction if the asmstr is empty.
+    const char *AsmStr = MI->getOperand(NumDefs).getSymbolName();
+    if (AsmStr[0] == 0)
+      return;
+  }
+
+  // Skip this instruction if no DebugLoc or the DebugLoc
+  // is the same as the previous instruction.
+  const DebugLoc &DL = MI->getDebugLoc();
+  if (!DL || PrevInstLoc == DL) {
+    // This instruction will be skipped, no LineInfo has
+    // been generated, construct one based on function signature.
+    if (LineInfoGenerated == false) {
+      auto *S = MI->getMF()->getFunction().getSubprogram();
+      MCSymbol *FuncLabel = Asm->getFunctionBegin();
+      constructLineInfo(S, FuncLabel, S->getLine(), 0);
+      LineInfoGenerated = true;
+    }
+
+    return;
+  }
+
+  // Create a temporary label to remember the insn for lineinfo.
+  MCSymbol *LineSym = OS.getContext().createTempSymbol();
+  OS.EmitLabel(LineSym);
+
+  // Construct the lineinfo.
+  auto SP = DL.get()->getScope()->getSubprogram();
+  constructLineInfo(SP, LineSym, DL.getLine(), DL.getCol());
+
+  LineInfoGenerated = true;
+  PrevInstLoc = DL;
+}
+
+void BTFDebug::endModule() {
+  // Collect all types referenced by globals.
+  const Module *M = MMI->getModule();
+  for (const DICompileUnit *CUNode : M->debug_compile_units()) {
+    for (const auto *GVE : CUNode->getGlobalVariables()) {
+      DIGlobalVariable *GV = GVE->getVariable();
+      visitTypeEntry(GV->getType().resolve());
+    }
+  }
+
+  // Complete BTF type cross refereences.
+  for (const auto &TypeEntry : TypeEntries)
+    TypeEntry->completeType(*this);
+
+  // Emit BTF sections.
+  emitBTFSection();
+  emitBTFExtSection();
+}
diff --git a/lib/Target/BPF/BTFDebug.h b/lib/Target/BPF/BTFDebug.h
new file mode 100644
index 0000000000000000000000000000000000000000..5a50b0810ca882e76d7066de64739b393c468882
--- /dev/null
+++ b/lib/Target/BPF/BTFDebug.h
@@ -0,0 +1,284 @@
+//===- BTFDebug.h -----------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file contains support for writing BTF debug info.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_TARGET_BPF_BTFDEBUG_H
+#define LLVM_LIB_TARGET_BPF_BTFDEBUG_H
+
+#include "llvm/ADT/StringMap.h"
+#include "llvm/CodeGen/DebugHandlerBase.h"
+#include <unordered_map>
+#include "BTF.h"
+
+namespace llvm {
+
+class AsmPrinter;
+class BTFDebug;
+class DIType;
+class MCStreamer;
+class MCSymbol;
+class MachineFunction;
+
+/// The base class for BTF type generation.
+class BTFTypeBase {
+protected:
+  uint8_t Kind;
+  uint32_t Id;
+  struct BTF::CommonType BTFType;
+
+public:
+  virtual ~BTFTypeBase() = default;
+  void setId(uint32_t Id) { this->Id = Id; }
+  uint32_t getId() { return Id; }
+  uint32_t roundupToBytes(uint32_t NumBits) { return (NumBits + 7) >> 3; }
+  /// Get the size of this BTF type entry.
+  virtual uint32_t getSize() { return BTF::CommonTypeSize; }
+  /// Complete BTF type generation after all related DebugInfo types
+  /// have been visited so their BTF type id's are available
+  /// for cross referece.
+  virtual void completeType(BTFDebug &BDebug) {}
+  /// Emit types for this BTF type entry.
+  virtual void emitType(MCStreamer &OS);
+};
+
+/// Handle several derived types include pointer, const,
+/// volatile, typedef and restrict.
+class BTFTypeDerived : public BTFTypeBase {
+  const DIDerivedType *DTy;
+
+public:
+  BTFTypeDerived(const DIDerivedType *Ty, unsigned Tag);
+  void completeType(BTFDebug &BDebug);
+  void emitType(MCStreamer &OS);
+};
+
+/// Handle struct or union forward declaration.
+class BTFTypeFwd : public BTFTypeBase {
+  StringRef Name;
+
+public:
+  BTFTypeFwd(StringRef Name, bool IsUnion);
+  void completeType(BTFDebug &BDebug);
+  void emitType(MCStreamer &OS);
+};
+
+/// Handle int type.
+class BTFTypeInt : public BTFTypeBase {
+  StringRef Name;
+  uint32_t IntVal; ///< Encoding, offset, bits
+
+public:
+  BTFTypeInt(uint32_t Encoding, uint32_t SizeInBits, uint32_t OffsetInBits,
+             StringRef TypeName);
+  uint32_t getSize() { return BTFTypeBase::getSize() + sizeof(uint32_t); }
+  void completeType(BTFDebug &BDebug);
+  void emitType(MCStreamer &OS);
+};
+
+/// Handle enumerate type.
+class BTFTypeEnum : public BTFTypeBase {
+  const DICompositeType *ETy;
+  std::vector<struct BTF::BTFEnum> EnumValues;
+
+public:
+  BTFTypeEnum(const DICompositeType *ETy, uint32_t NumValues);
+  uint32_t getSize() {
+    return BTFTypeBase::getSize() + EnumValues.size() * BTF::BTFEnumSize;
+  }
+  void completeType(BTFDebug &BDebug);
+  void emitType(MCStreamer &OS);
+};
+
+/// Handle array type.
+class BTFTypeArray : public BTFTypeBase {
+  const DICompositeType *ATy;
+  struct BTF::BTFArray ArrayInfo;
+
+public:
+  BTFTypeArray(const DICompositeType *ATy);
+  uint32_t getSize() { return BTFTypeBase::getSize() + BTF::BTFArraySize; }
+  void completeType(BTFDebug &BDebug);
+  void emitType(MCStreamer &OS);
+};
+
+/// Handle struct/union type.
+class BTFTypeStruct : public BTFTypeBase {
+  const DICompositeType *STy;
+  bool HasBitField;
+  std::vector<struct BTF::BTFMember> Members;
+
+public:
+  BTFTypeStruct(const DICompositeType *STy, bool IsStruct, bool HasBitField,
+                uint32_t NumMembers);
+  uint32_t getSize() {
+    return BTFTypeBase::getSize() + Members.size() * BTF::BTFMemberSize;
+  }
+  void completeType(BTFDebug &BDebug);
+  void emitType(MCStreamer &OS);
+};
+
+/// Handle function pointer.
+class BTFTypeFuncProto : public BTFTypeBase {
+  const DISubroutineType *STy;
+  std::unordered_map<uint32_t, StringRef> FuncArgNames;
+  std::vector<struct BTF::BTFParam> Parameters;
+
+public:
+  BTFTypeFuncProto(const DISubroutineType *STy, uint32_t NumParams,
+                   const std::unordered_map<uint32_t, StringRef> &FuncArgNames);
+  uint32_t getSize() {
+    return BTFTypeBase::getSize() + Parameters.size() * BTF::BTFParamSize;
+  }
+  void completeType(BTFDebug &BDebug);
+  void emitType(MCStreamer &OS);
+};
+
+/// Handle subprogram
+class BTFTypeFunc : public BTFTypeBase {
+  StringRef Name;
+
+public:
+  BTFTypeFunc(StringRef FuncName, uint32_t ProtoTypeId);
+  uint32_t getSize() { return BTFTypeBase::getSize(); }
+  void completeType(BTFDebug &BDebug);
+  void emitType(MCStreamer &OS);
+};
+
+/// String table.
+class BTFStringTable {
+  /// String table size in bytes.
+  uint32_t Size;
+  /// A mapping from string table offset to the index
+  /// of the Table. It is used to avoid putting
+  /// duplicated strings in the table.
+  std::unordered_map<uint32_t, uint32_t> OffsetToIdMap;
+  /// A vector of strings to represent the string table.
+  std::vector<std::string> Table;
+
+public:
+  BTFStringTable() : Size(0) {}
+  uint32_t getSize() { return Size; }
+  std::vector<std::string> &getTable() { return Table; }
+  /// Add a string to the string table and returns its offset
+  /// in the table.
+  uint32_t addString(StringRef S);
+};
+
+/// Represent one func and its type id.
+struct BTFFuncInfo {
+  const MCSymbol *Label; ///< Func MCSymbol
+  uint32_t TypeId;       ///< Type id referring to .BTF type section
+};
+
+/// Represent one line info.
+struct BTFLineInfo {
+  MCSymbol *Label;      ///< MCSymbol identifying insn for the lineinfo
+  uint32_t FileNameOff; ///< file name offset in the .BTF string table
+  uint32_t LineOff;     ///< line offset in the .BTF string table
+  uint32_t LineNum;     ///< the line number
+  uint32_t ColumnNum;   ///< the column number
+};
+
+/// Collect and emit BTF information.
+class BTFDebug : public DebugHandlerBase {
+  MCStreamer &OS;
+  bool SkipInstruction;
+  bool LineInfoGenerated;
+  uint32_t SecNameOff;
+  uint32_t ArrayIndexTypeId;
+  BTFStringTable StringTable;
+  std::vector<std::unique_ptr<BTFTypeBase>> TypeEntries;
+  std::unordered_map<const DIType *, uint32_t> DIToIdMap;
+  std::unordered_map<uint32_t, std::vector<BTFFuncInfo>> FuncInfoTable;
+  std::unordered_map<uint32_t, std::vector<BTFLineInfo>> LineInfoTable;
+  StringMap<std::vector<std::string>> FileContent;
+
+  /// Add types to TypeEntries.
+  /// @{
+  /// Add types to TypeEntries and DIToIdMap.
+  void addType(std::unique_ptr<BTFTypeBase> TypeEntry, const DIType *Ty);
+  /// Add types to TypeEntries only and return type id.
+  uint32_t addType(std::unique_ptr<BTFTypeBase> TypeEntry);
+  /// @}
+
+  /// IR type visiting functions.
+  /// @{
+  void visitTypeEntry(const DIType *Ty);
+  void visitBasicType(const DIBasicType *BTy);
+  void visitSubroutineType(
+      const DISubroutineType *STy, bool ForSubprog,
+      const std::unordered_map<uint32_t, StringRef> &FuncArgNames,
+      uint32_t &TypeId);
+  void visitFwdDeclType(const DICompositeType *CTy, bool IsUnion);
+  void visitCompositeType(const DICompositeType *CTy);
+  void visitStructType(const DICompositeType *STy, bool IsStruct);
+  void visitArrayType(const DICompositeType *ATy);
+  void visitEnumType(const DICompositeType *ETy);
+  void visitDerivedType(const DIDerivedType *DTy);
+  /// @}
+
+  /// Get the file content for the subprogram. Certain lines of the file
+  /// later may be put into string table and referenced by line info.
+  std::string populateFileContent(const DISubprogram *SP);
+
+  /// Construct a line info.
+  void constructLineInfo(const DISubprogram *SP, MCSymbol *Label, uint32_t Line,
+                         uint32_t Column);
+
+  /// Emit common header of .BTF and .BTF.ext sections.
+  void emitCommonHeader();
+
+  /// Emit the .BTF section.
+  void emitBTFSection();
+
+  /// Emit the .BTF.ext section.
+  void emitBTFExtSection();
+
+protected:
+  /// Gather pre-function debug information.
+  void beginFunctionImpl(const MachineFunction *MF) override;
+
+  /// Post process after all instructions in this function are processed.
+  void endFunctionImpl(const MachineFunction *MF) override;
+
+public:
+  BTFDebug(AsmPrinter *AP);
+
+  /// Get the special array index type id.
+  uint32_t getArrayIndexTypeId() {
+    assert(ArrayIndexTypeId);
+    return ArrayIndexTypeId;
+  }
+
+  /// Add string to the string table.
+  size_t addString(StringRef S) { return StringTable.addString(S); }
+
+  /// Get the type id for a particular DIType.
+  uint32_t getTypeId(const DIType *Ty) {
+    assert(Ty && "Invalid null Type");
+    assert(DIToIdMap.find(Ty) != DIToIdMap.end() &&
+           "DIType not added in the BDIToIdMap");
+    return DIToIdMap[Ty];
+  }
+
+  void setSymbolSize(const MCSymbol *Symbol, uint64_t Size) override {}
+
+  /// Process beginning of an instruction.
+  void beginInstruction(const MachineInstr *MI) override;
+
+  /// Complete all the types and emit the BTF sections.
+  void endModule() override;
+};
+
+} // end namespace llvm
+
+#endif
diff --git a/lib/Target/BPF/CMakeLists.txt b/lib/Target/BPF/CMakeLists.txt
index 5e2ae533ea6c08400cb7660beecbfb3f607f49fd..c18adf859d352f46fc88b6695758cbb0454ccc37 100644
--- a/lib/Target/BPF/CMakeLists.txt
+++ b/lib/Target/BPF/CMakeLists.txt
@@ -25,6 +25,7 @@ add_llvm_target(BPFCodeGen
   BPFTargetMachine.cpp
   BPFMIPeephole.cpp
   BPFMIChecking.cpp
+  BTFDebug.cpp
   )
 
 add_subdirectory(AsmParser)
diff --git a/lib/Target/BPF/Disassembler/BPFDisassembler.cpp b/lib/Target/BPF/Disassembler/BPFDisassembler.cpp
index 9f80b762fe3699e0fabadf3294a05fa229c9d72f..ed09e446e73fd014ea97429662189850d5059a74 100644
--- a/lib/Target/BPF/Disassembler/BPFDisassembler.cpp
+++ b/lib/Target/BPF/Disassembler/BPFDisassembler.cpp
@@ -1,9 +1,8 @@
 //===- BPFDisassembler.cpp - Disassembler for BPF ---------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -40,7 +39,7 @@ public:
     BPF_STX = 0x3,
     BPF_ALU = 0x4,
     BPF_JMP = 0x5,
-    BPF_RES = 0x6,
+    BPF_JMP32 = 0x6,
     BPF_ALU64 = 0x7
   };
 
diff --git a/lib/Target/BPF/Disassembler/LLVMBuild.txt b/lib/Target/BPF/Disassembler/LLVMBuild.txt
index 786074d51c80d223fdf2e92316d9adff869fbf2b..ad1adca5d8bbcc044caf342a33453cec0828d130 100644
--- a/lib/Target/BPF/Disassembler/LLVMBuild.txt
+++ b/lib/Target/BPF/Disassembler/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/Target/BPF/Disassembler/LLVMBuild.txt --------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Target/BPF/InstPrinter/BPFInstPrinter.cpp b/lib/Target/BPF/InstPrinter/BPFInstPrinter.cpp
index 20627da38817ef91bb963d86d64278caf50f7111..91411a745d7f6eae30bd16ecbbaaf2282493ec51 100644
--- a/lib/Target/BPF/InstPrinter/BPFInstPrinter.cpp
+++ b/lib/Target/BPF/InstPrinter/BPFInstPrinter.cpp
@@ -1,9 +1,8 @@
 //===-- BPFInstPrinter.cpp - Convert BPF MCInst to asm syntax -------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/BPF/InstPrinter/BPFInstPrinter.h b/lib/Target/BPF/InstPrinter/BPFInstPrinter.h
index bb0b0d71da539b8c32ee6fcda286b5f550f5bd28..3472423c46665f94dbedc62c759a19faaa3afd7b 100644
--- a/lib/Target/BPF/InstPrinter/BPFInstPrinter.h
+++ b/lib/Target/BPF/InstPrinter/BPFInstPrinter.h
@@ -1,9 +1,8 @@
 //===-- BPFInstPrinter.h - Convert BPF MCInst to asm syntax -------*- C++ -*--//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/BPF/InstPrinter/LLVMBuild.txt b/lib/Target/BPF/InstPrinter/LLVMBuild.txt
index 88a937a1e4e6fc0c070afc04f339dad3edd4bad6..6a5b3f546c80a0ac685671f20d926883e5f639f6 100644
--- a/lib/Target/BPF/InstPrinter/LLVMBuild.txt
+++ b/lib/Target/BPF/InstPrinter/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/Target/BPF/InstPrinter/LLVMBuild.txt ---------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Target/BPF/LLVMBuild.txt b/lib/Target/BPF/LLVMBuild.txt
index 3f9849b5d92272e853284e0009e209554fa07c1b..3cafd1c0acc3fd98c042aa27e732f153cfe4eaa6 100644
--- a/lib/Target/BPF/LLVMBuild.txt
+++ b/lib/Target/BPF/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/Target/BPF/LLVMBuild.txt ---------------------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Target/BPF/MCTargetDesc/BPFAsmBackend.cpp b/lib/Target/BPF/MCTargetDesc/BPFAsmBackend.cpp
index 1822d8688fa262e7f4a31a0048bd6eaf171d56c8..0884469a8b5ca6c57dcf65ff44d2b809fda660a3 100644
--- a/lib/Target/BPF/MCTargetDesc/BPFAsmBackend.cpp
+++ b/lib/Target/BPF/MCTargetDesc/BPFAsmBackend.cpp
@@ -1,9 +1,8 @@
 //===-- BPFAsmBackend.cpp - BPF Assembler Backend -------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/BPF/MCTargetDesc/BPFELFObjectWriter.cpp b/lib/Target/BPF/MCTargetDesc/BPFELFObjectWriter.cpp
index 134e890dfe49e2027531bb2f3b2ab1f8c0c5b3a2..1d7a3ec68a8df9f5f6a55adca9ce8e5efa928a4f 100644
--- a/lib/Target/BPF/MCTargetDesc/BPFELFObjectWriter.cpp
+++ b/lib/Target/BPF/MCTargetDesc/BPFELFObjectWriter.cpp
@@ -1,9 +1,8 @@
 //===-- BPFELFObjectWriter.cpp - BPF ELF Writer ---------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -12,6 +11,7 @@
 #include "llvm/MC/MCELFObjectWriter.h"
 #include "llvm/MC/MCFixup.h"
 #include "llvm/MC/MCObjectWriter.h"
+#include "llvm/MC/MCValue.h"
 #include "llvm/Support/ErrorHandling.h"
 #include <cstdint>
 
@@ -50,6 +50,23 @@ unsigned BPFELFObjectWriter::getRelocType(MCContext &Ctx, const MCValue &Target,
   case FK_Data_8:
     return ELF::R_BPF_64_64;
   case FK_Data_4:
+    // .BTF.ext generates FK_Data_4 relocations for
+    // insn offset by creating temporary labels.
+    // The insn offset is within the code section and
+    // already been fulfilled by applyFixup(). No
+    // further relocation is needed.
+    if (const MCSymbolRefExpr *A = Target.getSymA()) {
+      if (A->getSymbol().isTemporary()) {
+        MCSection &Section = A->getSymbol().getSection();
+        const MCSectionELF *SectionELF = dyn_cast<MCSectionELF>(&Section);
+        assert(SectionELF && "Null section for reloc symbol");
+
+        // The reloc symbol should be in text section.
+        unsigned Flags = SectionELF->getFlags();
+        if ((Flags & ELF::SHF_ALLOC) && (Flags & ELF::SHF_EXECINSTR))
+          return ELF::R_BPF_NONE;
+      }
+    }
     return ELF::R_BPF_64_32;
   }
 }
diff --git a/lib/Target/BPF/MCTargetDesc/BPFMCAsmInfo.h b/lib/Target/BPF/MCTargetDesc/BPFMCAsmInfo.h
index af3ad53152533835a6078a6b728af50849c8c945..04a6a87cebc9e908428449c934a892299f1d77eb 100644
--- a/lib/Target/BPF/MCTargetDesc/BPFMCAsmInfo.h
+++ b/lib/Target/BPF/MCTargetDesc/BPFMCAsmInfo.h
@@ -1,9 +1,8 @@
 //===-- BPFMCAsmInfo.h - BPF asm properties -------------------*- C++ -*--====//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/BPF/MCTargetDesc/BPFMCCodeEmitter.cpp b/lib/Target/BPF/MCTargetDesc/BPFMCCodeEmitter.cpp
index 437f658caf6e21c9e9812383d3f849502530db09..947095b8da7cd2b38c1dd85cf6408f26268cd6d5 100644
--- a/lib/Target/BPF/MCTargetDesc/BPFMCCodeEmitter.cpp
+++ b/lib/Target/BPF/MCTargetDesc/BPFMCCodeEmitter.cpp
@@ -1,9 +1,8 @@
 //===-- BPFMCCodeEmitter.cpp - Convert BPF code to machine code -----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/BPF/MCTargetDesc/BPFMCTargetDesc.cpp b/lib/Target/BPF/MCTargetDesc/BPFMCTargetDesc.cpp
index 834b57527882b8e4d55b59323ef5b1c9dfde1e3f..037587ff7b29fadfdca1639580cfabe66380c9d7 100644
--- a/lib/Target/BPF/MCTargetDesc/BPFMCTargetDesc.cpp
+++ b/lib/Target/BPF/MCTargetDesc/BPFMCTargetDesc.cpp
@@ -1,9 +1,8 @@
 //===-- BPFMCTargetDesc.cpp - BPF Target Descriptions ---------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/BPF/MCTargetDesc/BPFMCTargetDesc.h b/lib/Target/BPF/MCTargetDesc/BPFMCTargetDesc.h
index 6d2f0a1601e61c0582dbe33838a9e974705a2070..4093a4062252124a3b803e869375bf8b09a1e49e 100644
--- a/lib/Target/BPF/MCTargetDesc/BPFMCTargetDesc.h
+++ b/lib/Target/BPF/MCTargetDesc/BPFMCTargetDesc.h
@@ -1,9 +1,8 @@
 //===-- BPFMCTargetDesc.h - BPF Target Descriptions -------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/BPF/MCTargetDesc/LLVMBuild.txt b/lib/Target/BPF/MCTargetDesc/LLVMBuild.txt
index 8bca2e39e899f1ab3112c622dacfe1677e8ea93a..adfd3e7f91c20b2170aec1356d5a46c4d622f6c6 100644
--- a/lib/Target/BPF/MCTargetDesc/LLVMBuild.txt
+++ b/lib/Target/BPF/MCTargetDesc/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/Target/BPF/MCTargetDesc/LLVMBuild.txt --------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Target/BPF/TargetInfo/BPFTargetInfo.cpp b/lib/Target/BPF/TargetInfo/BPFTargetInfo.cpp
index 1f7b8a04d5896f81380c0a77b57779e52fcb4036..be879495fedae25d28d48f07bfd4bb4eefca1992 100644
--- a/lib/Target/BPF/TargetInfo/BPFTargetInfo.cpp
+++ b/lib/Target/BPF/TargetInfo/BPFTargetInfo.cpp
@@ -1,9 +1,8 @@
 //===-- BPFTargetInfo.cpp - BPF Target Implementation ---------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/BPF/TargetInfo/LLVMBuild.txt b/lib/Target/BPF/TargetInfo/LLVMBuild.txt
index b56a8583583a31534287fb42fc6cc5950ff31447..e9e14ad1c43572429ba29306ebe049ef0de54663 100644
--- a/lib/Target/BPF/TargetInfo/LLVMBuild.txt
+++ b/lib/Target/BPF/TargetInfo/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/Target/BPF/TargetInfo/LLVMBuild.txt ----------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Target/Hexagon/AsmParser/HexagonAsmParser.cpp b/lib/Target/Hexagon/AsmParser/HexagonAsmParser.cpp
index 2eb1f0fc8bd94d6ce6b2e7849fde5e6c9278e85c..d5a49a4b43131b5a5b74f0e09bcda97d2b3a0471 100644
--- a/lib/Target/Hexagon/AsmParser/HexagonAsmParser.cpp
+++ b/lib/Target/Hexagon/AsmParser/HexagonAsmParser.cpp
@@ -1,9 +1,8 @@
 //===-- HexagonAsmParser.cpp - Parse Hexagon asm to MCInst instructions----===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/Hexagon/AsmParser/LLVMBuild.txt b/lib/Target/Hexagon/AsmParser/LLVMBuild.txt
index fdd875b619064d21d91b2c70a82321a2b8134475..94f19b1b576d774f69516c38fbbdb817ce97e197 100644
--- a/lib/Target/Hexagon/AsmParser/LLVMBuild.txt
+++ b/lib/Target/Hexagon/AsmParser/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/Target/Hexagon/AsmParser/LLVMBuild.txt --------------*- Conf -*-===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Target/Hexagon/BitTracker.cpp b/lib/Target/Hexagon/BitTracker.cpp
index 69529b0d11622847c9f21a2c9501a92345ae006e..b7e95caf24fbd92469a811fb0cbfafc32461ba76 100644
--- a/lib/Target/Hexagon/BitTracker.cpp
+++ b/lib/Target/Hexagon/BitTracker.cpp
@@ -1,9 +1,8 @@
 //===- BitTracker.cpp -----------------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/Hexagon/BitTracker.h b/lib/Target/Hexagon/BitTracker.h
index 058225c0d812892cf234ed1c2cdddcea5c99cd42..efb21805b8013bbbebff1754bb643cc6f6ee161f 100644
--- a/lib/Target/Hexagon/BitTracker.h
+++ b/lib/Target/Hexagon/BitTracker.h
@@ -1,9 +1,8 @@
 //===- BitTracker.h ---------------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/Hexagon/Disassembler/HexagonDisassembler.cpp b/lib/Target/Hexagon/Disassembler/HexagonDisassembler.cpp
index 428b42eba30da1a4e8e730645e1fb60f5445d67d..01862d2c0907fe5c8f22bfc23b3f675dfbc02abe 100644
--- a/lib/Target/Hexagon/Disassembler/HexagonDisassembler.cpp
+++ b/lib/Target/Hexagon/Disassembler/HexagonDisassembler.cpp
@@ -1,9 +1,8 @@
 //===- HexagonDisassembler.cpp - Disassembler for Hexagon ISA -------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -149,7 +148,7 @@ static DecodeStatus s32_0ImmDecoder(MCInst &MI, unsigned tmp,
                                     uint64_t /*Address*/, const void *Decoder);
 static DecodeStatus brtargetDecoder(MCInst &MI, unsigned tmp, uint64_t Address,
                                     const void *Decoder);
-#include "HexagonDepDecoders.h"
+#include "HexagonDepDecoders.inc"
 #include "HexagonGenDisassemblerTables.inc"
 
 static MCDisassembler *createHexagonDisassembler(const Target &T,
diff --git a/lib/Target/Hexagon/Disassembler/LLVMBuild.txt b/lib/Target/Hexagon/Disassembler/LLVMBuild.txt
index 6c251020f81851e197dfcadbc9866784c410e17a..a26001add427987a06d6371983c0f53f4592de31 100644
--- a/lib/Target/Hexagon/Disassembler/LLVMBuild.txt
+++ b/lib/Target/Hexagon/Disassembler/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===-- ./lib/Target/Hexagon/Disassembler/LLVMBuild.txt ---------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Target/Hexagon/Hexagon.h b/lib/Target/Hexagon/Hexagon.h
index c18492da803b8f54c3989066bdbb79158c09a057..58dadf012da560be94dbafc9c6238525447800f9 100644
--- a/lib/Target/Hexagon/Hexagon.h
+++ b/lib/Target/Hexagon/Hexagon.h
@@ -1,9 +1,8 @@
 //=-- Hexagon.h - Top-level interface for Hexagon representation --*- C++ -*-=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Hexagon/Hexagon.td b/lib/Target/Hexagon/Hexagon.td
index 868353e188329acdcfcebec441cc6c0b2e72065a..26869391c7a316005f54e85a1d0f5bb1c46bb9e7 100644
--- a/lib/Target/Hexagon/Hexagon.td
+++ b/lib/Target/Hexagon/Hexagon.td
@@ -1,9 +1,8 @@
 //===-- Hexagon.td - Describe the Hexagon Target Machine --*- tablegen -*--===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Hexagon/HexagonAsmPrinter.cpp b/lib/Target/Hexagon/HexagonAsmPrinter.cpp
index f44fb16e2d8e0381785559dbab55acc75bf07445..9b3bfda66ee0f4eb99df09b8a16fdb91634a2bff 100644
--- a/lib/Target/Hexagon/HexagonAsmPrinter.cpp
+++ b/lib/Target/Hexagon/HexagonAsmPrinter.cpp
@@ -1,9 +1,8 @@
 //===- HexagonAsmPrinter.cpp - Print machine instrs to Hexagon assembly ---===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Hexagon/HexagonAsmPrinter.h b/lib/Target/Hexagon/HexagonAsmPrinter.h
index d0629d173a65f755b1b36c57c9d6a5cf0c1dfce1..baf0999b30121bd493f2165f6474be3e9c3bf566 100644
--- a/lib/Target/Hexagon/HexagonAsmPrinter.h
+++ b/lib/Target/Hexagon/HexagonAsmPrinter.h
@@ -1,9 +1,8 @@
 //===- HexagonAsmPrinter.h - Print machine code to an Hexagon .s file -----===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Hexagon/HexagonBitSimplify.cpp b/lib/Target/Hexagon/HexagonBitSimplify.cpp
index 1bdebe557a8cefc7c9a5eb8cdf5130af9c21f976..7b75d251ccd3201edf1af2acf6e4d481784d322a 100644
--- a/lib/Target/Hexagon/HexagonBitSimplify.cpp
+++ b/lib/Target/Hexagon/HexagonBitSimplify.cpp
@@ -1,9 +1,8 @@
 //===- HexagonBitSimplify.cpp ---------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/Hexagon/HexagonBitTracker.cpp b/lib/Target/Hexagon/HexagonBitTracker.cpp
index 92b6da871a4c7826a72d84f9e74912ef9c3c2719..ba50faac2cf9a435d16fcaac35fcf6b1a050db07 100644
--- a/lib/Target/Hexagon/HexagonBitTracker.cpp
+++ b/lib/Target/Hexagon/HexagonBitTracker.cpp
@@ -1,9 +1,8 @@
 //===- HexagonBitTracker.cpp ----------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/Hexagon/HexagonBitTracker.h b/lib/Target/Hexagon/HexagonBitTracker.h
index f0b7c9d919501847aafb71c41188819766fd2269..02607d50f6869a25c4bdb2d9f6de76b49b0c3fac 100644
--- a/lib/Target/Hexagon/HexagonBitTracker.h
+++ b/lib/Target/Hexagon/HexagonBitTracker.h
@@ -1,9 +1,8 @@
 //===- HexagonBitTracker.h --------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/Hexagon/HexagonBlockRanges.cpp b/lib/Target/Hexagon/HexagonBlockRanges.cpp
index 48a4505458ae8c0edf8ed1005d2aa61f095e2f29..999150fc8c6e19e697cdfbc82285b1a465f1a806 100644
--- a/lib/Target/Hexagon/HexagonBlockRanges.cpp
+++ b/lib/Target/Hexagon/HexagonBlockRanges.cpp
@@ -1,9 +1,8 @@
 //===- HexagonBlockRanges.cpp ---------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/Hexagon/HexagonBlockRanges.h b/lib/Target/Hexagon/HexagonBlockRanges.h
index 4da5a970a65976f3f5266fcf5f8093a2359304bf..61115e29a70800ca31847a75b5ae3444ec1b6c17 100644
--- a/lib/Target/Hexagon/HexagonBlockRanges.h
+++ b/lib/Target/Hexagon/HexagonBlockRanges.h
@@ -1,9 +1,8 @@
 //===- HexagonBlockRanges.h -------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/Hexagon/HexagonBranchRelaxation.cpp b/lib/Target/Hexagon/HexagonBranchRelaxation.cpp
index 2fa7888dd02bbf9ae65015eebbe2487e4c1dbf7d..ee93739b2c7b148f75f451b7f47c68f3e28149a3 100644
--- a/lib/Target/Hexagon/HexagonBranchRelaxation.cpp
+++ b/lib/Target/Hexagon/HexagonBranchRelaxation.cpp
@@ -1,9 +1,8 @@
 //===--- HexagonBranchRelaxation.cpp - Identify and relax long jumps ------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/Hexagon/HexagonCFGOptimizer.cpp b/lib/Target/Hexagon/HexagonCFGOptimizer.cpp
index a22ac8c9fdf5296cbd572a34336a661fcbef062e..11a455ce43470b04a188c230a6b0de6d3a84b588 100644
--- a/lib/Target/Hexagon/HexagonCFGOptimizer.cpp
+++ b/lib/Target/Hexagon/HexagonCFGOptimizer.cpp
@@ -1,9 +1,8 @@
 //===- HexagonCFGOptimizer.cpp - CFG optimizations ------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/Hexagon/HexagonCallingConv.td b/lib/Target/Hexagon/HexagonCallingConv.td
index ed2f87570d6b7ee47420ec2ea329f09324e313da..5c31a81a1e87181cddfc765c0205a7efd1c57428 100644
--- a/lib/Target/Hexagon/HexagonCallingConv.td
+++ b/lib/Target/Hexagon/HexagonCallingConv.td
@@ -1,9 +1,8 @@
 //===- HexagonCallingConv.td ----------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/Hexagon/HexagonCommonGEP.cpp b/lib/Target/Hexagon/HexagonCommonGEP.cpp
index f315e24eba62bba35072507cb6d8de9890ddb34d..77b51ce2adeca1d5490d7b2440f0941799a6392b 100644
--- a/lib/Target/Hexagon/HexagonCommonGEP.cpp
+++ b/lib/Target/Hexagon/HexagonCommonGEP.cpp
@@ -1,9 +1,8 @@
 //===- HexagonCommonGEP.cpp -----------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/Hexagon/HexagonConstExtenders.cpp b/lib/Target/Hexagon/HexagonConstExtenders.cpp
index ba9f638796eb3c6d7ec2fbcb5a0444330bb9a087..cfed0ecef2720934b3427d40dd7b7d3104ba090d 100644
--- a/lib/Target/Hexagon/HexagonConstExtenders.cpp
+++ b/lib/Target/Hexagon/HexagonConstExtenders.cpp
@@ -1,9 +1,8 @@
 //===- HexagonConstExtenders.cpp ------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/Hexagon/HexagonConstPropagation.cpp b/lib/Target/Hexagon/HexagonConstPropagation.cpp
index fa192391313e20ffae860970b88c39fee888f538..f19a13f9b4c94cebbe301865b9c233d1a806b962 100644
--- a/lib/Target/Hexagon/HexagonConstPropagation.cpp
+++ b/lib/Target/Hexagon/HexagonConstPropagation.cpp
@@ -1,9 +1,8 @@
 //===- HexagonConstPropagation.cpp ----------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/Hexagon/HexagonCopyToCombine.cpp b/lib/Target/Hexagon/HexagonCopyToCombine.cpp
index 28965b69e284161d48b4f87421573c7f6ffca617..2990ad802c4cbe233b1d9b404b37b36aaa9d4306 100644
--- a/lib/Target/Hexagon/HexagonCopyToCombine.cpp
+++ b/lib/Target/Hexagon/HexagonCopyToCombine.cpp
@@ -1,9 +1,8 @@
 //===------- HexagonCopyToCombine.cpp - Hexagon Copy-To-Combine Pass ------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 // This pass replaces transfer instructions by combine instructions.
diff --git a/lib/Target/Hexagon/HexagonDepArch.h b/lib/Target/Hexagon/HexagonDepArch.h
index dff2b2f471d0f1bb83ac53517d0960980a3bb869..529be7ef0ac740dadecf623f4174cb41b29cb207 100644
--- a/lib/Target/Hexagon/HexagonDepArch.h
+++ b/lib/Target/Hexagon/HexagonDepArch.h
@@ -1,9 +1,8 @@
 //===----------------------------------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 // Automatically generated file, please consult code owner before editing.
diff --git a/lib/Target/Hexagon/HexagonDepArch.td b/lib/Target/Hexagon/HexagonDepArch.td
index f1aadae555c8604fed185b243e4d089cf1979d90..115cf2383a7af2051ff81d421576316cc3c10077 100644
--- a/lib/Target/Hexagon/HexagonDepArch.td
+++ b/lib/Target/Hexagon/HexagonDepArch.td
@@ -1,9 +1,8 @@
 //===----------------------------------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 // Automatically generated file, please consult code owner before editing.
diff --git a/lib/Target/Hexagon/HexagonDepDecoders.h b/lib/Target/Hexagon/HexagonDepDecoders.inc
similarity index 92%
rename from lib/Target/Hexagon/HexagonDepDecoders.h
rename to lib/Target/Hexagon/HexagonDepDecoders.inc
index 9f78412f45d24f4b656cdc67ca4633bb453b013d..10068abce7ec8c05464aa6391af7666977b74c6c 100644
--- a/lib/Target/Hexagon/HexagonDepDecoders.h
+++ b/lib/Target/Hexagon/HexagonDepDecoders.inc
@@ -1,9 +1,8 @@
 //===----------------------------------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 // Automatically generated file, please consult code owner before editing.
diff --git a/lib/Target/Hexagon/HexagonDepIICHVX.td b/lib/Target/Hexagon/HexagonDepIICHVX.td
index 9e3dea9f3e9b00c7e4318a084fc8e9f91444f6bb..fefbbfd3f1ac14dbce9cf6a8c9d40e048b642f3d 100644
--- a/lib/Target/Hexagon/HexagonDepIICHVX.td
+++ b/lib/Target/Hexagon/HexagonDepIICHVX.td
@@ -1,9 +1,8 @@
 //===----------------------------------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 // Automatically generated file, please consult code owner before editing.
diff --git a/lib/Target/Hexagon/HexagonDepIICScalar.td b/lib/Target/Hexagon/HexagonDepIICScalar.td
index 9da25952fb1c96e0312b71ab21139a5c96b19b7d..34da0be02d192488149e9399545afa60b0bf1335 100644
--- a/lib/Target/Hexagon/HexagonDepIICScalar.td
+++ b/lib/Target/Hexagon/HexagonDepIICScalar.td
@@ -1,9 +1,8 @@
 //===----------------------------------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 // Automatically generated file, please consult code owner before editing.
diff --git a/lib/Target/Hexagon/HexagonDepITypes.h b/lib/Target/Hexagon/HexagonDepITypes.h
index 81e3971e21d20ae17391759b0a9af940bca4087e..358345e027d81c0d651bc7e9af60b050b328d0dd 100644
--- a/lib/Target/Hexagon/HexagonDepITypes.h
+++ b/lib/Target/Hexagon/HexagonDepITypes.h
@@ -1,9 +1,8 @@
 //===----------------------------------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 // Automatically generated file, please consult code owner before editing.
diff --git a/lib/Target/Hexagon/HexagonDepITypes.td b/lib/Target/Hexagon/HexagonDepITypes.td
index f694062a5232f11e3ae560b34ea0ccb995dcc370..91c02b84b87c062bb947b211d56d0df28d998fd1 100644
--- a/lib/Target/Hexagon/HexagonDepITypes.td
+++ b/lib/Target/Hexagon/HexagonDepITypes.td
@@ -1,9 +1,8 @@
 //===----------------------------------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 // Automatically generated file, please consult code owner before editing.
diff --git a/lib/Target/Hexagon/HexagonDepInstrFormats.td b/lib/Target/Hexagon/HexagonDepInstrFormats.td
index ffe212ef9d97bc4bef363596558b1202cd8edbc6..c08d9a388d3e50d4ba6ecc350c8528a5c1ee6020 100644
--- a/lib/Target/Hexagon/HexagonDepInstrFormats.td
+++ b/lib/Target/Hexagon/HexagonDepInstrFormats.td
@@ -1,9 +1,8 @@
 //===----------------------------------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 // Automatically generated file, please consult code owner before editing.
diff --git a/lib/Target/Hexagon/HexagonDepInstrInfo.td b/lib/Target/Hexagon/HexagonDepInstrInfo.td
index 3ef1c49eb7ee0f4be2c186e03d59b9f9cbae9a26..a49051888c77c7211e95e3d4144f9b47d50a8ee8 100644
--- a/lib/Target/Hexagon/HexagonDepInstrInfo.td
+++ b/lib/Target/Hexagon/HexagonDepInstrInfo.td
@@ -1,9 +1,8 @@
 //===----------------------------------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 // Automatically generated file, please consult code owner before editing.
diff --git a/lib/Target/Hexagon/HexagonDepMapAsm2Intrin.td b/lib/Target/Hexagon/HexagonDepMapAsm2Intrin.td
index 2346fa5726264dfa393ebed107c098b4036dfd68..2ce1419e4790f19a7d8128a2eab3efa2f72a9353 100644
--- a/lib/Target/Hexagon/HexagonDepMapAsm2Intrin.td
+++ b/lib/Target/Hexagon/HexagonDepMapAsm2Intrin.td
@@ -1,9 +1,8 @@
 //===----------------------------------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 // Automatically generated file, please consult code owner before editing.
diff --git a/lib/Target/Hexagon/HexagonDepMappings.td b/lib/Target/Hexagon/HexagonDepMappings.td
index b3132d41b903d607a0111a74781cda51aa9016e4..22ee495b25e6728fbd9e42734c6a5518f36bfcaa 100644
--- a/lib/Target/Hexagon/HexagonDepMappings.td
+++ b/lib/Target/Hexagon/HexagonDepMappings.td
@@ -1,9 +1,8 @@
 //===----------------------------------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 // Automatically generated file, please consult code owner before editing.
diff --git a/lib/Target/Hexagon/HexagonDepOperands.td b/lib/Target/Hexagon/HexagonDepOperands.td
index ef2d4fa4570218fd8e5b5710366fcaf446ee1fe3..fdba7b97125800eb594c6fb77750f0893b2371e5 100644
--- a/lib/Target/Hexagon/HexagonDepOperands.td
+++ b/lib/Target/Hexagon/HexagonDepOperands.td
@@ -1,9 +1,8 @@
 //===----------------------------------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 // Automatically generated file, please consult code owner before editing.
diff --git a/lib/Target/Hexagon/HexagonDepTimingClasses.h b/lib/Target/Hexagon/HexagonDepTimingClasses.h
index 0fd55e8b7997a970e2f0bd9d3d9509558040b24b..b6be74f848bb00439df96eba64d9a6d50d91c901 100644
--- a/lib/Target/Hexagon/HexagonDepTimingClasses.h
+++ b/lib/Target/Hexagon/HexagonDepTimingClasses.h
@@ -1,9 +1,8 @@
 //===----------------------------------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 // Automatically generated file, please consult code owner before editing.
diff --git a/lib/Target/Hexagon/HexagonEarlyIfConv.cpp b/lib/Target/Hexagon/HexagonEarlyIfConv.cpp
index 8e2f5093038e86b336ad08577c60f7055ee86bff..c1f32e54e98ddb286ff3c6f16a60997f15c23a31 100644
--- a/lib/Target/Hexagon/HexagonEarlyIfConv.cpp
+++ b/lib/Target/Hexagon/HexagonEarlyIfConv.cpp
@@ -1,9 +1,8 @@
 //===- HexagonEarlyIfConv.cpp ---------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Hexagon/HexagonExpandCondsets.cpp b/lib/Target/Hexagon/HexagonExpandCondsets.cpp
index 1a762c0c9de737737dce00791ab07de5bc443a3b..2531d790e58e8a296a2bc58dbda6667a0ea64957 100644
--- a/lib/Target/Hexagon/HexagonExpandCondsets.cpp
+++ b/lib/Target/Hexagon/HexagonExpandCondsets.cpp
@@ -1,9 +1,8 @@
 //===- HexagonExpandCondsets.cpp ------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/Hexagon/HexagonFixupHwLoops.cpp b/lib/Target/Hexagon/HexagonFixupHwLoops.cpp
index e9067e2285a89ea1c994a9025f9e07cf4f4346b9..f7edc168de4a3d568c54300b52645a21ac5a46ab 100644
--- a/lib/Target/Hexagon/HexagonFixupHwLoops.cpp
+++ b/lib/Target/Hexagon/HexagonFixupHwLoops.cpp
@@ -1,9 +1,8 @@
 //===---- HexagonFixupHwLoops.cpp - Fixup HW loops too far from LOOPn. ----===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 // The loop start address in the LOOPn instruction is encoded as a distance
 // from the LOOPn instruction itself. If the start address is too far from
diff --git a/lib/Target/Hexagon/HexagonFrameLowering.cpp b/lib/Target/Hexagon/HexagonFrameLowering.cpp
index f5736546a87cca1d7ea38ad48c628e116c66441b..4a47231ddd82ba3a86e7ade635a91ee77feb4cee 100644
--- a/lib/Target/Hexagon/HexagonFrameLowering.cpp
+++ b/lib/Target/Hexagon/HexagonFrameLowering.cpp
@@ -1,9 +1,8 @@
 //===- HexagonFrameLowering.cpp - Define frame lowering -------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //
 //===----------------------------------------------------------------------===//
diff --git a/lib/Target/Hexagon/HexagonFrameLowering.h b/lib/Target/Hexagon/HexagonFrameLowering.h
index d65d870750f871dc1cd60da4af61c5faf28a064d..65e8c76866403dde8bfbe1bda96ad8d13d814385 100644
--- a/lib/Target/Hexagon/HexagonFrameLowering.h
+++ b/lib/Target/Hexagon/HexagonFrameLowering.h
@@ -1,9 +1,8 @@
 //==- HexagonFrameLowering.h - Define frame lowering for Hexagon -*- C++ -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/Hexagon/HexagonGenExtract.cpp b/lib/Target/Hexagon/HexagonGenExtract.cpp
index 08a016b74650f047b9377ccebb4e83b2a195cd1b..3417c74e359bb52236d7f7d1bc77e2c231d55546 100644
--- a/lib/Target/Hexagon/HexagonGenExtract.cpp
+++ b/lib/Target/Hexagon/HexagonGenExtract.cpp
@@ -1,9 +1,8 @@
 //===- HexagonGenExtract.cpp ----------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -211,7 +210,7 @@ bool HexagonGenExtract::convert(Instruction *In) {
   Intrinsic::ID IntId = (BW == 32) ? Intrinsic::hexagon_S2_extractu
                                    : Intrinsic::hexagon_S2_extractup;
   Module *Mod = BB->getParent()->getParent();
-  Value *ExtF = Intrinsic::getDeclaration(Mod, IntId);
+  Function *ExtF = Intrinsic::getDeclaration(Mod, IntId);
   Value *NewIn = IRB.CreateCall(ExtF, {BF, IRB.getInt32(W), IRB.getInt32(SR)});
   if (SL != 0)
     NewIn = IRB.CreateShl(NewIn, SL, CSL->getName());
diff --git a/lib/Target/Hexagon/HexagonGenInsert.cpp b/lib/Target/Hexagon/HexagonGenInsert.cpp
index e3492e7374e9b2f7f06c381f8ca26305b0a5a374..725cfe04df3b98b21d3371fd224a25a6ce71fcdc 100644
--- a/lib/Target/Hexagon/HexagonGenInsert.cpp
+++ b/lib/Target/Hexagon/HexagonGenInsert.cpp
@@ -1,9 +1,8 @@
 //===- HexagonGenInsert.cpp -----------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/Hexagon/HexagonGenMux.cpp b/lib/Target/Hexagon/HexagonGenMux.cpp
index e5af96468af1974beaa3a2a67fe01923be375135..acff247616409fe653d3b2c2f7d07ed54d399029 100644
--- a/lib/Target/Hexagon/HexagonGenMux.cpp
+++ b/lib/Target/Hexagon/HexagonGenMux.cpp
@@ -1,9 +1,8 @@
 //===- HexagonGenMux.cpp --------------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/Hexagon/HexagonGenPredicate.cpp b/lib/Target/Hexagon/HexagonGenPredicate.cpp
index c0d2de90467a1aea0a9995d7d277c13c401755d9..13add87c7b515018b284638cd3f3fb5afa01d35c 100644
--- a/lib/Target/Hexagon/HexagonGenPredicate.cpp
+++ b/lib/Target/Hexagon/HexagonGenPredicate.cpp
@@ -1,9 +1,8 @@
 //===- HexagonGenPredicate.cpp --------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/Hexagon/HexagonHardwareLoops.cpp b/lib/Target/Hexagon/HexagonHardwareLoops.cpp
index 239cf49ca8a2b3d7fd7bf435218ea57028ba80e0..cecbaedb6d70e7ad59d63ec9175d19c2f0844a20 100644
--- a/lib/Target/Hexagon/HexagonHardwareLoops.cpp
+++ b/lib/Target/Hexagon/HexagonHardwareLoops.cpp
@@ -1,9 +1,8 @@
 //===- HexagonHardwareLoops.cpp - Identify and generate hardware loops ----===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Hexagon/HexagonHazardRecognizer.cpp b/lib/Target/Hexagon/HexagonHazardRecognizer.cpp
index 44f1f554c662d2efb0e79bb6b9a95e80c60103d3..e45126bec6efe4943c066089c444128a46bf8d0a 100644
--- a/lib/Target/Hexagon/HexagonHazardRecognizer.cpp
+++ b/lib/Target/Hexagon/HexagonHazardRecognizer.cpp
@@ -1,9 +1,8 @@
 //===-- HexagonHazardRecognizer.cpp - Hexagon Post RA Hazard Recognizer ---===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Hexagon/HexagonHazardRecognizer.h b/lib/Target/Hexagon/HexagonHazardRecognizer.h
index 2874d73ce819ba20cbf12e336dcb61c06e1fab66..53b9cb43b4b6679463d59c28b11c376e2f886822 100644
--- a/lib/Target/Hexagon/HexagonHazardRecognizer.h
+++ b/lib/Target/Hexagon/HexagonHazardRecognizer.h
@@ -1,9 +1,8 @@
 //===--- HexagonHazardRecognizer.h - Hexagon Post RA Hazard Recognizer ----===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 // This file defines the hazard recognizer for scheduling on Hexagon.
diff --git a/lib/Target/Hexagon/HexagonIICHVX.td b/lib/Target/Hexagon/HexagonIICHVX.td
index a804c5a80d032119c7dee283f635c7f4dd9f4324..1d71d60bf581c5e14c4f2e01c44f4ef530a26615 100644
--- a/lib/Target/Hexagon/HexagonIICHVX.td
+++ b/lib/Target/Hexagon/HexagonIICHVX.td
@@ -1,9 +1,8 @@
 //===--- HexagonIICHVX.td -------------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/Hexagon/HexagonIICScalar.td b/lib/Target/Hexagon/HexagonIICScalar.td
index 5fe713346e38984058e25eb7ed0358251f0f8d3a..d37cc3a2cc3efc0edd2766cba4df82127459600d 100644
--- a/lib/Target/Hexagon/HexagonIICScalar.td
+++ b/lib/Target/Hexagon/HexagonIICScalar.td
@@ -1,9 +1,8 @@
 //===--- HexagonIICScalar.td ----------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/Hexagon/HexagonISelDAGToDAG.cpp b/lib/Target/Hexagon/HexagonISelDAGToDAG.cpp
index 470b05bda4c6bc9069755410455caed0459c9558..d34d189ceaf555a3a3a02701436f4b8b52a6ee11 100644
--- a/lib/Target/Hexagon/HexagonISelDAGToDAG.cpp
+++ b/lib/Target/Hexagon/HexagonISelDAGToDAG.cpp
@@ -1,9 +1,8 @@
 //===-- HexagonISelDAGToDAG.cpp - A dag to dag inst selector for Hexagon --===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Hexagon/HexagonISelDAGToDAG.h b/lib/Target/Hexagon/HexagonISelDAGToDAG.h
index f4f09dd4e7581b8d7eb130ddcba60bfdeb84f830..65edb09603b31e5e82e10697305fa1cb6f059234 100644
--- a/lib/Target/Hexagon/HexagonISelDAGToDAG.h
+++ b/lib/Target/Hexagon/HexagonISelDAGToDAG.h
@@ -1,9 +1,8 @@
 //===-- HexagonISelDAGToDAG.h -----------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 // Hexagon specific code to select Hexagon machine instructions for
diff --git a/lib/Target/Hexagon/HexagonISelDAGToDAGHVX.cpp b/lib/Target/Hexagon/HexagonISelDAGToDAGHVX.cpp
index b796e442d4faf4c020c04ef63cd1b91d74e779d1..e7f1c345af1d978ea830108b8fe59d2bee25c174 100644
--- a/lib/Target/Hexagon/HexagonISelDAGToDAGHVX.cpp
+++ b/lib/Target/Hexagon/HexagonISelDAGToDAGHVX.cpp
@@ -1,9 +1,8 @@
 //===-- HexagonISelDAGToDAGHVX.cpp ----------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/Hexagon/HexagonISelLowering.cpp b/lib/Target/Hexagon/HexagonISelLowering.cpp
index b5be507bb97d67984868e024a9a867681450ac5a..c9ee83a249fc10288b458815fa51ae44667e7402 100644
--- a/lib/Target/Hexagon/HexagonISelLowering.cpp
+++ b/lib/Target/Hexagon/HexagonISelLowering.cpp
@@ -1,9 +1,8 @@
 //===-- HexagonISelLowering.cpp - Hexagon DAG Lowering Implementation -----===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -579,7 +578,8 @@ HexagonTargetLowering::LowerINLINEASM(SDValue Op, SelectionDAG &DAG) const {
   const HexagonRegisterInfo &HRI = *Subtarget.getRegisterInfo();
   unsigned LR = HRI.getRARegister();
 
-  if (Op.getOpcode() != ISD::INLINEASM || HMFI.hasClobberLR())
+  if ((Op.getOpcode() != ISD::INLINEASM &&
+       Op.getOpcode() != ISD::INLINEASM_BR) || HMFI.hasClobberLR())
     return Op;
 
   unsigned NumOps = Op.getNumOperands();
@@ -1292,6 +1292,7 @@ HexagonTargetLowering::HexagonTargetLowering(const TargetMachine &TM,
   setOperationAction(ISD::BUILD_PAIR,           MVT::i64,   Expand);
   setOperationAction(ISD::SIGN_EXTEND_INREG,    MVT::i1,    Expand);
   setOperationAction(ISD::INLINEASM,            MVT::Other, Custom);
+  setOperationAction(ISD::INLINEASM_BR,         MVT::Other, Custom);
   setOperationAction(ISD::PREFETCH,             MVT::Other, Custom);
   setOperationAction(ISD::READCYCLECOUNTER,     MVT::i64,   Custom);
   setOperationAction(ISD::INTRINSIC_VOID,       MVT::Other, Custom);
@@ -1359,6 +1360,11 @@ HexagonTargetLowering::HexagonTargetLowering(const TargetMachine &TM,
   setOperationAction(ISD::BSWAP, MVT::i32, Legal);
   setOperationAction(ISD::BSWAP, MVT::i64, Legal);
 
+  setOperationAction(ISD::FSHL, MVT::i32, Legal);
+  setOperationAction(ISD::FSHL, MVT::i64, Legal);
+  setOperationAction(ISD::FSHR, MVT::i32, Legal);
+  setOperationAction(ISD::FSHR, MVT::i64, Legal);
+
   for (unsigned IntExpOp :
        {ISD::SDIV,      ISD::UDIV,      ISD::SREM,      ISD::UREM,
         ISD::SDIVREM,   ISD::UDIVREM,   ISD::ROTL,      ISD::ROTR,
@@ -1538,8 +1544,10 @@ HexagonTargetLowering::HexagonTargetLowering(const TargetMachine &TM,
   // Subtarget-specific operation actions.
   //
   if (Subtarget.hasV60Ops()) {
-    setOperationAction(ISD::ROTL, MVT::i32, Custom);
-    setOperationAction(ISD::ROTL, MVT::i64, Custom);
+    setOperationAction(ISD::ROTL, MVT::i32, Legal);
+    setOperationAction(ISD::ROTL, MVT::i64, Legal);
+    setOperationAction(ISD::ROTR, MVT::i32, Legal);
+    setOperationAction(ISD::ROTR, MVT::i64, Legal);
   }
   if (Subtarget.hasV66Ops()) {
     setOperationAction(ISD::FADD, MVT::f64, Legal);
@@ -1768,11 +1776,8 @@ bool HexagonTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
     // The intrinsic function call is of the form { ElTy, i8* }
     // @llvm.hexagon.L2.loadXX.pbr(i8*, i32). The pointer and memory access type
     // should be derived from ElTy.
-    PointerType *PtrTy = I.getCalledFunction()
-                             ->getReturnType()
-                             ->getContainedType(0)
-                             ->getPointerTo();
-    Info.memVT = MVT::getVT(PtrTy->getElementType());
+    Type *ElTy = I.getCalledFunction()->getReturnType()->getStructElementType(0);
+    Info.memVT = MVT::getVT(ElTy);
     llvm::Value *BasePtrVal = I.getOperand(0);
     Info.ptrVal = getUnderLyingObjectForBrevLdIntr(BasePtrVal);
     // The offset value comes through Modifier register. For now, assume the
@@ -2737,7 +2742,7 @@ HexagonTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
   unsigned Opc = Op.getOpcode();
 
   // Handle INLINEASM first.
-  if (Opc == ISD::INLINEASM)
+  if (Opc == ISD::INLINEASM || Opc == ISD::INLINEASM_BR)
     return LowerINLINEASM(Op, DAG);
 
   if (isHvxOperation(Op)) {
@@ -3107,13 +3112,21 @@ Value *HexagonTargetLowering::emitLoadLinked(IRBuilder<> &Builder, Value *Addr,
       AtomicOrdering Ord) const {
   BasicBlock *BB = Builder.GetInsertBlock();
   Module *M = BB->getParent()->getParent();
-  Type *Ty = cast<PointerType>(Addr->getType())->getElementType();
+  auto PT = cast<PointerType>(Addr->getType());
+  Type *Ty = PT->getElementType();
   unsigned SZ = Ty->getPrimitiveSizeInBits();
   assert((SZ == 32 || SZ == 64) && "Only 32/64-bit atomic loads supported");
   Intrinsic::ID IntID = (SZ == 32) ? Intrinsic::hexagon_L2_loadw_locked
                                    : Intrinsic::hexagon_L4_loadd_locked;
-  Value *Fn = Intrinsic::getDeclaration(M, IntID);
-  return Builder.CreateCall(Fn, Addr, "larx");
+  Function *Fn = Intrinsic::getDeclaration(M, IntID);
+
+  PointerType *NewPtrTy
+    = Builder.getIntNTy(SZ)->getPointerTo(PT->getAddressSpace());
+  Addr = Builder.CreateBitCast(Addr, NewPtrTy);
+
+  Value *Call = Builder.CreateCall(Fn, Addr, "larx");
+
+  return Builder.CreateBitCast(Call, Ty);
 }
 
 /// Perform a store-conditional operation to Addr. Return the status of the
@@ -3124,10 +3137,17 @@ Value *HexagonTargetLowering::emitStoreConditional(IRBuilder<> &Builder,
   Module *M = BB->getParent()->getParent();
   Type *Ty = Val->getType();
   unsigned SZ = Ty->getPrimitiveSizeInBits();
+
+  Type *CastTy = Builder.getIntNTy(SZ);
   assert((SZ == 32 || SZ == 64) && "Only 32/64-bit atomic stores supported");
   Intrinsic::ID IntID = (SZ == 32) ? Intrinsic::hexagon_S2_storew_locked
                                    : Intrinsic::hexagon_S4_stored_locked;
-  Value *Fn = Intrinsic::getDeclaration(M, IntID);
+  Function *Fn = Intrinsic::getDeclaration(M, IntID);
+
+  unsigned AS = Addr->getType()->getPointerAddressSpace();
+  Addr = Builder.CreateBitCast(Addr, CastTy->getPointerTo(AS));
+  Val = Builder.CreateBitCast(Val, CastTy);
+
   Value *Call = Builder.CreateCall(Fn, {Addr, Val}, "stcx");
   Value *Cmp = Builder.CreateICmpEQ(Call, Builder.getInt32(0), "");
   Value *Ext = Builder.CreateZExt(Cmp, Type::getInt32Ty(M->getContext()));
diff --git a/lib/Target/Hexagon/HexagonISelLowering.h b/lib/Target/Hexagon/HexagonISelLowering.h
index 265c37e6ae61ca065472882823575cd27e4345a5..226fffdfe35583898b967634cfa5cc3eb23999a2 100644
--- a/lib/Target/Hexagon/HexagonISelLowering.h
+++ b/lib/Target/Hexagon/HexagonISelLowering.h
@@ -1,9 +1,8 @@
 //===-- HexagonISelLowering.h - Hexagon DAG Lowering Interface --*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Hexagon/HexagonISelLoweringHVX.cpp b/lib/Target/Hexagon/HexagonISelLoweringHVX.cpp
index a6400b5d826685bb00d781acd9e66614cb533005..6ab35c39bd477f50e64f47686d3179bc784ec35c 100644
--- a/lib/Target/Hexagon/HexagonISelLoweringHVX.cpp
+++ b/lib/Target/Hexagon/HexagonISelLoweringHVX.cpp
@@ -1,9 +1,8 @@
 //===-- HexagonISelLoweringHVX.cpp --- Lowering HVX operations ------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/Hexagon/HexagonInstrFormats.td b/lib/Target/Hexagon/HexagonInstrFormats.td
index 2236140d5dd7bfe71cabd52bff26adcd2b25a976..f156de67105932ec0cdca5bc9ea86c827e72fd57 100644
--- a/lib/Target/Hexagon/HexagonInstrFormats.td
+++ b/lib/Target/Hexagon/HexagonInstrFormats.td
@@ -1,9 +1,8 @@
 //==- HexagonInstrFormats.td - Hexagon Instruction Formats --*- tablegen -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/Hexagon/HexagonInstrFormatsV5.td b/lib/Target/Hexagon/HexagonInstrFormatsV5.td
index c8de5cbcc1e0ccd5525a143fe8393ff95bc2f579..68ef2d2d3a8aadc7c4dbf6c39944e9ab1b0b925f 100644
--- a/lib/Target/Hexagon/HexagonInstrFormatsV5.td
+++ b/lib/Target/Hexagon/HexagonInstrFormatsV5.td
@@ -1,9 +1,8 @@
 //==- HexagonInstrFormatsV5.td - Hexagon Instruction Formats --*- tablegen -==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Hexagon/HexagonInstrFormatsV60.td b/lib/Target/Hexagon/HexagonInstrFormatsV60.td
index 1347a655353fa92c82ed7cb5d15de2487b8fc524..86a82183a1ad884f66a8448b09de7f690f65c851 100644
--- a/lib/Target/Hexagon/HexagonInstrFormatsV60.td
+++ b/lib/Target/Hexagon/HexagonInstrFormatsV60.td
@@ -1,9 +1,8 @@
 //==- HexagonInstrFormatsV60.td - Hexagon Instruction Formats -*- tablegen -==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Hexagon/HexagonInstrFormatsV65.td b/lib/Target/Hexagon/HexagonInstrFormatsV65.td
index cddb8777b417175a45b43ffd7751186bb88ad7f9..eaecffe9c89e8501ff58074c8ee24372e7072c3b 100644
--- a/lib/Target/Hexagon/HexagonInstrFormatsV65.td
+++ b/lib/Target/Hexagon/HexagonInstrFormatsV65.td
@@ -1,9 +1,8 @@
 //==- HexagonInstrFormatsV65.td - Hexagon Instruction Formats -*- tablegen -==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Hexagon/HexagonInstrInfo.cpp b/lib/Target/Hexagon/HexagonInstrInfo.cpp
index de0d6c4d9e4e781200b184ed8adb93faea2bd342..b0b029682bf6acd5781a02581528f4ab4b1e83c7 100644
--- a/lib/Target/Hexagon/HexagonInstrInfo.cpp
+++ b/lib/Target/Hexagon/HexagonInstrInfo.cpp
@@ -1,9 +1,8 @@
 //===- HexagonInstrInfo.cpp - Hexagon Instruction Information -------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -2425,7 +2424,7 @@ bool HexagonInstrInfo::isPredicated(unsigned Opcode) const {
 
 bool HexagonInstrInfo::isPredicateLate(unsigned Opcode) const {
   const uint64_t F = get(Opcode).TSFlags;
-  return ~(F >> HexagonII::PredicateLatePos) & HexagonII::PredicateLateMask;
+  return (F >> HexagonII::PredicateLatePos) & HexagonII::PredicateLateMask;
 }
 
 bool HexagonInstrInfo::isPredictedTaken(unsigned Opcode) const {
diff --git a/lib/Target/Hexagon/HexagonInstrInfo.h b/lib/Target/Hexagon/HexagonInstrInfo.h
index 9b840762e88ae8dff8aa60ffd04d0130f1d9fe57..688af2a90895276cdbe4421d817adb0150ecf913 100644
--- a/lib/Target/Hexagon/HexagonInstrInfo.h
+++ b/lib/Target/Hexagon/HexagonInstrInfo.h
@@ -1,9 +1,8 @@
 //===- HexagonInstrInfo.h - Hexagon Instruction Information -----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Hexagon/HexagonIntrinsics.td b/lib/Target/Hexagon/HexagonIntrinsics.td
index 9cab5748bef2292d75e2c4c782064af6c7b4e0e8..cabfd783effa4c9453f19aaf217f0f3a09a03eee 100644
--- a/lib/Target/Hexagon/HexagonIntrinsics.td
+++ b/lib/Target/Hexagon/HexagonIntrinsics.td
@@ -1,9 +1,8 @@
 //===-- HexagonIntrinsics.td - Instruction intrinsics ------*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/Hexagon/HexagonIntrinsicsV5.td b/lib/Target/Hexagon/HexagonIntrinsicsV5.td
index a852394f216076551a649c8c07b97da40a2a6365..44f39a3e9b1633425f4d3ce30e7b2815dd71442c 100644
--- a/lib/Target/Hexagon/HexagonIntrinsicsV5.td
+++ b/lib/Target/Hexagon/HexagonIntrinsicsV5.td
@@ -1,9 +1,8 @@
 //===- HexagonIntrinsicsV5.td - V5 Instruction intrinsics --*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/Hexagon/HexagonIntrinsicsV60.td b/lib/Target/Hexagon/HexagonIntrinsicsV60.td
index 5e5c77b38e8e6668c4bd563bed73ed772e469191..a60c80beb5d66aa1b43fb33cc63abd563805b6bb 100644
--- a/lib/Target/Hexagon/HexagonIntrinsicsV60.td
+++ b/lib/Target/Hexagon/HexagonIntrinsicsV60.td
@@ -1,9 +1,8 @@
 //=- HexagonIntrinsicsV60.td - Target Description for Hexagon -*- tablegen *-=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Hexagon/HexagonLoopIdiomRecognition.cpp b/lib/Target/Hexagon/HexagonLoopIdiomRecognition.cpp
index 702d68fad9bf0832065298ef84a63787f17058b4..ac48e1dc30b067225a8f500ae2f2d340b861f3af 100644
--- a/lib/Target/Hexagon/HexagonLoopIdiomRecognition.cpp
+++ b/lib/Target/Hexagon/HexagonLoopIdiomRecognition.cpp
@@ -1,9 +1,8 @@
 //===- HexagonLoopIdiomRecognition.cpp ------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -1001,6 +1000,7 @@ bool PolynomialMultiplyRecognize::isPromotableTo(Value *Val,
 void PolynomialMultiplyRecognize::promoteTo(Instruction *In,
       IntegerType *DestTy, BasicBlock *LoopB) {
   Type *OrigTy = In->getType();
+  assert(!OrigTy->isVoidTy() && "Invalid instruction to promote");
 
   // Leave boolean values alone.
   if (!In->getType()->isIntegerTy(1))
@@ -1081,7 +1081,8 @@ bool PolynomialMultiplyRecognize::promoteTypes(BasicBlock *LoopB,
   std::transform(LoopB->begin(), LoopB->end(), std::back_inserter(LoopIns),
                  [](Instruction &In) { return &In; });
   for (Instruction *In : LoopIns)
-    promoteTo(In, DestTy, LoopB);
+    if (!In->isTerminator())
+      promoteTo(In, DestTy, LoopB);
 
   // Fix up the PHI nodes in the exit block.
   Instruction *EndI = ExitB->getFirstNonPHI();
@@ -1522,7 +1523,7 @@ Value *PolynomialMultiplyRecognize::generate(BasicBlock::iterator At,
       ParsedValues &PV) {
   IRBuilder<> B(&*At);
   Module *M = At->getParent()->getParent()->getParent();
-  Value *PMF = Intrinsic::getDeclaration(M, Intrinsic::hexagon_M4_pmpyw);
+  Function *PMF = Intrinsic::getDeclaration(M, Intrinsic::hexagon_M4_pmpyw);
 
   Value *P = PV.P, *Q = PV.Q, *P0 = P;
   unsigned IC = PV.IterCount;
@@ -1975,7 +1976,8 @@ mayLoopAccessLocation(Value *Ptr, ModRefInfo Access, Loop *L,
   // If the loop iterates a fixed number of times, we can refine the access
   // size to be exactly the size of the memset, which is (BECount+1)*StoreSize
   if (const SCEVConstant *BECst = dyn_cast<SCEVConstant>(BECount))
-    AccessSize = (BECst->getValue()->getZExtValue() + 1) * StoreSize;
+    AccessSize = LocationSize::precise((BECst->getValue()->getZExtValue() + 1) *
+                                       StoreSize);
 
   // TODO: For this to be really effective, we have to dive into the pointer
   // operand in the store.  Store to &A[i] of 100 will always return may alias
@@ -2251,10 +2253,8 @@ CleanupAndExit:
       Type *Int32PtrTy = Type::getInt32PtrTy(Ctx);
       Type *VoidTy = Type::getVoidTy(Ctx);
       Module *M = Func->getParent();
-      Constant *CF = M->getOrInsertFunction(HexagonVolatileMemcpyName, VoidTy,
-                                            Int32PtrTy, Int32PtrTy, Int32Ty);
-      Function *Fn = cast<Function>(CF);
-      Fn->setLinkage(Function::ExternalLinkage);
+      FunctionCallee Fn = M->getOrInsertFunction(
+          HexagonVolatileMemcpyName, VoidTy, Int32PtrTy, Int32PtrTy, Int32Ty);
 
       const SCEV *OneS = SE->getConstant(Int32Ty, 1);
       const SCEV *BECount32 = SE->getTruncateOrZeroExtend(BECount, Int32Ty);
diff --git a/lib/Target/Hexagon/HexagonMCInstLower.cpp b/lib/Target/Hexagon/HexagonMCInstLower.cpp
index fb5752ade1debd8fa7549c028c31d5cfb6e24ce1..d1a153920e5e0556b5369ddedbb634a47cc954e1 100644
--- a/lib/Target/Hexagon/HexagonMCInstLower.cpp
+++ b/lib/Target/Hexagon/HexagonMCInstLower.cpp
@@ -1,9 +1,8 @@
 //===- HexagonMCInstLower.cpp - Convert Hexagon MachineInstr to an MCInst -===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Hexagon/HexagonMachineFunctionInfo.cpp b/lib/Target/Hexagon/HexagonMachineFunctionInfo.cpp
index 9579c8b6df16f55d8fe5b8928ce0dafe9bac1b67..aabae009d7c3f3c5da742721be8173f6e4b7cae9 100644
--- a/lib/Target/Hexagon/HexagonMachineFunctionInfo.cpp
+++ b/lib/Target/Hexagon/HexagonMachineFunctionInfo.cpp
@@ -1,9 +1,8 @@
 //= HexagonMachineFunctionInfo.cpp - Hexagon machine function info *- C++ -*-=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/Hexagon/HexagonMachineFunctionInfo.h b/lib/Target/Hexagon/HexagonMachineFunctionInfo.h
index d83bcbc41553f1756ad4c0239945209f0d380fec..2961e16cc9dc279e279bcc46d9dbb7a23fc862bf 100644
--- a/lib/Target/Hexagon/HexagonMachineFunctionInfo.h
+++ b/lib/Target/Hexagon/HexagonMachineFunctionInfo.h
@@ -1,9 +1,8 @@
 //=- HexagonMachineFunctionInfo.h - Hexagon machine function info -*- C++ -*-=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/Hexagon/HexagonMachineScheduler.cpp b/lib/Target/Hexagon/HexagonMachineScheduler.cpp
index 908ce24136c766c1108a44438b47ef0f8a24586e..0e6555024303c872ece514fbfb9df65fca54c182 100644
--- a/lib/Target/Hexagon/HexagonMachineScheduler.cpp
+++ b/lib/Target/Hexagon/HexagonMachineScheduler.cpp
@@ -1,9 +1,8 @@
 //===- HexagonMachineScheduler.cpp - MI Scheduler for Hexagon -------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -113,6 +112,7 @@ bool VLIWResourceModel::isResourceAvailable(SUnit *SU, bool IsTop) {
   case TargetOpcode::IMPLICIT_DEF:
   case TargetOpcode::COPY:
   case TargetOpcode::INLINEASM:
+  case TargetOpcode::INLINEASM_BR:
     break;
   }
 
@@ -168,6 +168,7 @@ bool VLIWResourceModel::reserveResources(SUnit *SU, bool IsTop) {
   case TargetOpcode::EH_LABEL:
   case TargetOpcode::COPY:
   case TargetOpcode::INLINEASM:
+  case TargetOpcode::INLINEASM_BR:
     break;
   }
   Packet.push_back(SU);
diff --git a/lib/Target/Hexagon/HexagonMachineScheduler.h b/lib/Target/Hexagon/HexagonMachineScheduler.h
index 585a7858ad2b98d8ea1551128f99a6a2e6d8c256..fb0a7abd339bcaadf1531552f177e06e5659367c 100644
--- a/lib/Target/Hexagon/HexagonMachineScheduler.h
+++ b/lib/Target/Hexagon/HexagonMachineScheduler.h
@@ -1,9 +1,8 @@
 //===- HexagonMachineScheduler.h - Custom Hexagon MI scheduler --*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Hexagon/HexagonMapAsm2IntrinV62.gen.td b/lib/Target/Hexagon/HexagonMapAsm2IntrinV62.gen.td
index b7b0de0efaea3d8bcd1c036f8d9b29c87a56aa54..2fcefe6a4ef6c0abc76d0129364a03340152ef45 100644
--- a/lib/Target/Hexagon/HexagonMapAsm2IntrinV62.gen.td
+++ b/lib/Target/Hexagon/HexagonMapAsm2IntrinV62.gen.td
@@ -1,9 +1,8 @@
 //===--- HexagonMapAsm2IntrinV62.gen.td -----------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/Hexagon/HexagonMapAsm2IntrinV65.gen.td b/lib/Target/Hexagon/HexagonMapAsm2IntrinV65.gen.td
index c29a75e6fe746dae2c8e5a3e87a42498cf6082c8..7293075532c6d247d9c5bfff63031b565a407575 100644
--- a/lib/Target/Hexagon/HexagonMapAsm2IntrinV65.gen.td
+++ b/lib/Target/Hexagon/HexagonMapAsm2IntrinV65.gen.td
@@ -1,9 +1,8 @@
 //===--- HexagonMapAsm2IntrinV65.gen.td -----------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/Hexagon/HexagonNewValueJump.cpp b/lib/Target/Hexagon/HexagonNewValueJump.cpp
index f2a6627c99be9408f5fd0fedc475e86b2da88bc2..db44901ca7064eab6cd3ace99f04b96ca1534d2f 100644
--- a/lib/Target/Hexagon/HexagonNewValueJump.cpp
+++ b/lib/Target/Hexagon/HexagonNewValueJump.cpp
@@ -1,9 +1,8 @@
 //===- HexagonNewValueJump.cpp - Hexagon Backend New Value Jump -----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Hexagon/HexagonOperands.td b/lib/Target/Hexagon/HexagonOperands.td
index 232946ec1579133ba3ddbeb07e34d6158e57371e..212cf03bee67d6f2aada6c7b8ce14dda807bf67c 100644
--- a/lib/Target/Hexagon/HexagonOperands.td
+++ b/lib/Target/Hexagon/HexagonOperands.td
@@ -1,9 +1,8 @@
 //===--- HexagonOperands.td -----------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/Hexagon/HexagonOptAddrMode.cpp b/lib/Target/Hexagon/HexagonOptAddrMode.cpp
index c3a5bd5d57bfeee7c32697458f8dd0035d349179..547da9fd598f605b755f06be6a39c0b32bdff954 100644
--- a/lib/Target/Hexagon/HexagonOptAddrMode.cpp
+++ b/lib/Target/Hexagon/HexagonOptAddrMode.cpp
@@ -1,9 +1,8 @@
 //===- HexagonOptAddrMode.cpp ---------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 // This implements a Hexagon-specific pass to optimize addressing mode for
diff --git a/lib/Target/Hexagon/HexagonOptimizeSZextends.cpp b/lib/Target/Hexagon/HexagonOptimizeSZextends.cpp
index 101de3d8fbee70cda3ac51e6853f593275a21569..d00fc23102a5b4aab40a9e88159ed38c697f9ee8 100644
--- a/lib/Target/Hexagon/HexagonOptimizeSZextends.cpp
+++ b/lib/Target/Hexagon/HexagonOptimizeSZextends.cpp
@@ -1,9 +1,8 @@
 //===- HexagonOptimizeSZextends.cpp - Remove unnecessary argument extends -===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Hexagon/HexagonPatterns.td b/lib/Target/Hexagon/HexagonPatterns.td
index 311b5a702d5a7d5faaf0c980e2b402b995721be5..5c709708f8f91dcd9006a10804af802822d8be96 100644
--- a/lib/Target/Hexagon/HexagonPatterns.td
+++ b/lib/Target/Hexagon/HexagonPatterns.td
@@ -1,9 +1,8 @@
 //==- HexagonPatterns.td - Target Description for Hexagon -*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -177,6 +176,11 @@ def UDEC32: SDNodeXForm<imm, [{
   return CurDAG->getTargetConstant(V-32, SDLoc(N), MVT::i32);
 }]>;
 
+class Subi<int From>: SDNodeXForm<imm,
+  "int32_t V = " # From # " - N->getSExtValue();" #
+  "return CurDAG->getTargetConstant(V, SDLoc(N), MVT::i32);"
+>;
+
 def Log2_32: SDNodeXForm<imm, [{
   uint32_t V = N->getZExtValue();
   return CurDAG->getTargetConstant(Log2_32(V), SDLoc(N), MVT::i32);
@@ -995,10 +999,90 @@ def: OpR_RR_pat<S2_asr_r_p, Sra, i64, I64, I32>;
 def: OpR_RR_pat<S2_lsr_r_p, Srl, i64, I64, I32>;
 def: OpR_RR_pat<S2_asl_r_p, Shl, i64, I64, I32>;
 
-let Predicates = [HasV60] in {
+// Funnel shifts.
+def IsMul8_U3: PatLeaf<(i32 imm), [{
+  uint64_t V = N->getZExtValue();
+  return V % 8 == 0 && isUInt<3>(V / 8);
+}]>;
+
+def Divu8: SDNodeXForm<imm, [{
+  return CurDAG->getTargetConstant(N->getZExtValue() / 8, SDLoc(N), MVT::i32);
+}]>;
+
+// Funnel shift-left.
+def FShl32i: OutPatFrag<(ops node:$Rs, node:$Rt, node:$S),
+  (HiReg (S2_asl_i_p (Combinew $Rs, $Rt), $S))>;
+def FShl32r: OutPatFrag<(ops node:$Rs, node:$Rt, node:$Ru),
+  (HiReg (S2_asl_r_p (Combinew $Rs, $Rt), $Ru))>;
+
+def FShl64i: OutPatFrag<(ops node:$Rs, node:$Rt, node:$S),
+  (S2_lsr_i_p_or (S2_asl_i_p $Rt, $S),  $Rs, (Subi<64> $S))>;
+def FShl64r: OutPatFrag<(ops node:$Rs, node:$Rt, node:$Ru),
+  (S2_lsr_r_p_or (S2_asl_r_p $Rt, $Ru), $Rs, (A2_subri 64, $Ru))>;
+
+// Combined SDNodeXForm: (Divu8 (Subi<64> $S))
+def Divu64_8: SDNodeXForm<imm, [{
+  return CurDAG->getTargetConstant((64 - N->getSExtValue()) / 8,
+                                   SDLoc(N), MVT::i32);
+}]>;
+
+// Special cases:
+let AddedComplexity = 100 in {
+  def: Pat<(fshl I32:$Rs, I32:$Rt, (i32 16)),
+           (A2_combine_hl I32:$Rs, I32:$Rt)>;
+  def: Pat<(fshl I64:$Rs, I64:$Rt, IsMul8_U3:$S),
+           (S2_valignib I64:$Rs, I64:$Rt, (Divu64_8 $S))>;
+}
+
+let Predicates = [HasV60], AddedComplexity = 50 in {
   def: OpR_RI_pat<S6_rol_i_r, Rol, i32, I32, u5_0ImmPred>;
   def: OpR_RI_pat<S6_rol_i_p, Rol, i64, I64, u6_0ImmPred>;
 }
+let AddedComplexity = 30 in {
+  def: Pat<(rotl I32:$Rs, u5_0ImmPred:$S),          (FShl32i $Rs, $Rs, imm:$S)>;
+  def: Pat<(rotl I64:$Rs, u6_0ImmPred:$S),          (FShl64i $Rs, $Rs, imm:$S)>;
+  def: Pat<(fshl I32:$Rs, I32:$Rt, u5_0ImmPred:$S), (FShl32i $Rs, $Rt, imm:$S)>;
+  def: Pat<(fshl I64:$Rs, I64:$Rt, u6_0ImmPred:$S), (FShl64i $Rs, $Rt, imm:$S)>;
+}
+def: Pat<(rotl I32:$Rs, I32:$Rt),           (FShl32r $Rs, $Rs, $Rt)>;
+def: Pat<(rotl I64:$Rs, I32:$Rt),           (FShl64r $Rs, $Rs, $Rt)>;
+def: Pat<(fshl I32:$Rs, I32:$Rt, I32:$Ru),  (FShl32r $Rs, $Rt, $Ru)>;
+def: Pat<(fshl I64:$Rs, I64:$Rt, I32:$Ru),  (FShl64r $Rs, $Rt, $Ru)>;
+
+// Funnel shift-right.
+def FShr32i: OutPatFrag<(ops node:$Rs, node:$Rt, node:$S),
+  (LoReg (S2_lsr_i_p (Combinew $Rs, $Rt), $S))>;
+def FShr32r: OutPatFrag<(ops node:$Rs, node:$Rt, node:$Ru),
+  (LoReg (S2_lsr_r_p (Combinew $Rs, $Rt), $Ru))>;
+
+def FShr64i: OutPatFrag<(ops node:$Rs, node:$Rt, node:$S),
+  (S2_asl_i_p_or (S2_lsr_i_p $Rt, $S),  $Rs, (Subi<64> $S))>;
+def FShr64r: OutPatFrag<(ops node:$Rs, node:$Rt, node:$Ru),
+  (S2_asl_r_p_or (S2_lsr_r_p $Rt, $Ru), $Rs, (A2_subri 64, $Ru))>;
+
+// Special cases:
+let AddedComplexity = 100 in {
+  def: Pat<(fshr I32:$Rs, I32:$Rt, (i32 16)),
+           (A2_combine_hl I32:$Rs, I32:$Rt)>;
+  def: Pat<(fshr I64:$Rs, I64:$Rt, IsMul8_U3:$S),
+           (S2_valignib I64:$Rs, I64:$Rt, (Divu8 $S))>;
+}
+
+let Predicates = [HasV60], AddedComplexity = 50 in {
+  def: Pat<(rotr I32:$Rs, u5_0ImmPred:$S), (S6_rol_i_r I32:$Rs, (Subi<32> $S))>;
+  def: Pat<(rotr I64:$Rs, u6_0ImmPred:$S), (S6_rol_i_p I64:$Rs, (Subi<64> $S))>;
+}
+let AddedComplexity = 30 in {
+  def: Pat<(rotr I32:$Rs, u5_0ImmPred:$S),          (FShr32i $Rs, $Rs, imm:$S)>;
+  def: Pat<(rotr I64:$Rs, u6_0ImmPred:$S),          (FShr64i $Rs, $Rs, imm:$S)>;
+  def: Pat<(fshr I32:$Rs, I32:$Rt, u5_0ImmPred:$S), (FShr32i $Rs, $Rt, imm:$S)>;
+  def: Pat<(fshr I64:$Rs, I64:$Rt, u6_0ImmPred:$S), (FShr64i $Rs, $Rt, imm:$S)>;
+}
+def: Pat<(rotr I32:$Rs, I32:$Rt),           (FShr32r $Rs, $Rs, $Rt)>;
+def: Pat<(rotr I64:$Rs, I32:$Rt),           (FShr64r $Rs, $Rs, $Rt)>;
+def: Pat<(fshr I32:$Rs, I32:$Rt, I32:$Ru),  (FShr32r $Rs, $Rt, $Ru)>;
+def: Pat<(fshr I64:$Rs, I64:$Rt, I32:$Ru),  (FShr64r $Rs, $Rt, $Ru)>;
+
 
 def: Pat<(sra (add (sra I32:$Rs, u5_0ImmPred:$u5), 1), (i32 1)),
          (S2_asr_i_r_rnd I32:$Rs, imm:$u5)>;
@@ -1183,6 +1267,7 @@ def: Pat<(HexagonVASL V2I16:$Rs, I32:$Rt),
 def: Pat<(HexagonVLSR V2I16:$Rs, I32:$Rt),
          (LoReg (S2_lsr_i_vh (ToAext64 $Rs), I32:$Rt))>;
 
+
 // --(9) Arithmetic/bitwise ----------------------------------------------
 //
 
diff --git a/lib/Target/Hexagon/HexagonPatternsV65.td b/lib/Target/Hexagon/HexagonPatternsV65.td
index 50b76847b5632895cc8c2e26aab06a2c2282c6ee..4cd45ecbe1a1f4a80ccbdd12482b69f01d72668c 100644
--- a/lib/Target/Hexagon/HexagonPatternsV65.td
+++ b/lib/Target/Hexagon/HexagonPatternsV65.td
@@ -1,9 +1,8 @@
 //==- HexagonPatternsV65.td -------------------------------*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/Hexagon/HexagonPeephole.cpp b/lib/Target/Hexagon/HexagonPeephole.cpp
index 3c588a89b0da65c3e345e478d94f310c18f9be73..8f761d2d4805c09378eb1e714e30ac792c30d516 100644
--- a/lib/Target/Hexagon/HexagonPeephole.cpp
+++ b/lib/Target/Hexagon/HexagonPeephole.cpp
@@ -1,9 +1,8 @@
 //===-- HexagonPeephole.cpp - Hexagon Peephole Optimiztions ---------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 // This peephole pass optimizes in the following cases.
 // 1. Optimizes redundant sign extends for the following case
diff --git a/lib/Target/Hexagon/HexagonPseudo.td b/lib/Target/Hexagon/HexagonPseudo.td
index b9748c7e189c303dccaa6b977645160fbe4ca12e..50900d8cd25b3380b7a2740d9ce2635d8e2ec2f5 100644
--- a/lib/Target/Hexagon/HexagonPseudo.td
+++ b/lib/Target/Hexagon/HexagonPseudo.td
@@ -1,9 +1,8 @@
 //===--- HexagonPseudo.td -------------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/Hexagon/HexagonRDFOpt.cpp b/lib/Target/Hexagon/HexagonRDFOpt.cpp
index 413bc8edf2b6370803ed93334260dd25175a0d8b..910a17540e6ecd90caeab81ffe1b44375b2fcd52 100644
--- a/lib/Target/Hexagon/HexagonRDFOpt.cpp
+++ b/lib/Target/Hexagon/HexagonRDFOpt.cpp
@@ -1,9 +1,8 @@
 //===- HexagonRDFOpt.cpp --------------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/Hexagon/HexagonRegisterInfo.cpp b/lib/Target/Hexagon/HexagonRegisterInfo.cpp
index 9b8f4e07376fca35520aee0ceca56750ded42cd3..810e4db7fb05c8ca72e53574ece0838836665123 100644
--- a/lib/Target/Hexagon/HexagonRegisterInfo.cpp
+++ b/lib/Target/Hexagon/HexagonRegisterInfo.cpp
@@ -1,9 +1,8 @@
 //===-- HexagonRegisterInfo.cpp - Hexagon Register Information ------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Hexagon/HexagonRegisterInfo.h b/lib/Target/Hexagon/HexagonRegisterInfo.h
index 3e7b63a462f060f2dd0086eacc5179163d635bb9..9b8821aded49bea486524dc8f3f57f2d2264116f 100644
--- a/lib/Target/Hexagon/HexagonRegisterInfo.h
+++ b/lib/Target/Hexagon/HexagonRegisterInfo.h
@@ -1,9 +1,8 @@
 //==- HexagonRegisterInfo.h - Hexagon Register Information Impl --*- C++ -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Hexagon/HexagonRegisterInfo.td b/lib/Target/Hexagon/HexagonRegisterInfo.td
index da90911e2c0514cba15f6b04ff1f2e5a8edf464d..f12189052699e4dd9a650320ceee9693faa47bf4 100644
--- a/lib/Target/Hexagon/HexagonRegisterInfo.td
+++ b/lib/Target/Hexagon/HexagonRegisterInfo.td
@@ -1,9 +1,8 @@
 //===-- HexagonRegisterInfo.td - Hexagon Register defs -----*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/Hexagon/HexagonSchedule.td b/lib/Target/Hexagon/HexagonSchedule.td
index 1024198e9b3ff9f9b631d586aecb504b21cdd12a..0834e90004606629ab04f94e457b73556db2d4a7 100644
--- a/lib/Target/Hexagon/HexagonSchedule.td
+++ b/lib/Target/Hexagon/HexagonSchedule.td
@@ -1,9 +1,8 @@
 //===- HexagonSchedule.td - Hexagon Scheduling Definitions -*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/Hexagon/HexagonScheduleV5.td b/lib/Target/Hexagon/HexagonScheduleV5.td
index 9a893f6dde027c11e14e698a5c2114207c27d6cd..ba0da2c196abcae3820fdaab6cdcaee43f4e2692 100644
--- a/lib/Target/Hexagon/HexagonScheduleV5.td
+++ b/lib/Target/Hexagon/HexagonScheduleV5.td
@@ -1,9 +1,8 @@
 //=-HexagonScheduleV5.td - HexagonV5 Scheduling Definitions --*- tablegen -*-=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/Hexagon/HexagonScheduleV55.td b/lib/Target/Hexagon/HexagonScheduleV55.td
index ca738be5d6ef6266dc017221f55670dbf9f432b7..f88dd5d2056d44f0de6ed09a6cf58c2fb26cd25e 100644
--- a/lib/Target/Hexagon/HexagonScheduleV55.td
+++ b/lib/Target/Hexagon/HexagonScheduleV55.td
@@ -1,9 +1,8 @@
 //=-HexagonScheduleV55.td - HexagonV55 Scheduling Definitions -*- tablegen -*=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/Hexagon/HexagonScheduleV60.td b/lib/Target/Hexagon/HexagonScheduleV60.td
index 861a8d2b0339d7ceb960e10d7050b0e77c11fb00..c6539597a9e7fa8e4038fd570bd8478ca94c4f47 100644
--- a/lib/Target/Hexagon/HexagonScheduleV60.td
+++ b/lib/Target/Hexagon/HexagonScheduleV60.td
@@ -1,9 +1,8 @@
 //=-HexagonScheduleV60.td - HexagonV60 Scheduling Definitions *- tablegen -*-=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/Hexagon/HexagonScheduleV62.td b/lib/Target/Hexagon/HexagonScheduleV62.td
index 1c274191277ca1dbede467254477d3c5ce48ecc0..782d76760992444fb0e787bb7f1e4bad9f25178e 100644
--- a/lib/Target/Hexagon/HexagonScheduleV62.td
+++ b/lib/Target/Hexagon/HexagonScheduleV62.td
@@ -1,9 +1,8 @@
 //=-HexagonScheduleV62.td - HexagonV62 Scheduling Definitions *- tablegen -*-=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Hexagon/HexagonScheduleV65.td b/lib/Target/Hexagon/HexagonScheduleV65.td
index 46a79d521795cef9b050e706ff7f59c09604240c..ac64410e559b20768a20f53158d769ca82837c4a 100644
--- a/lib/Target/Hexagon/HexagonScheduleV65.td
+++ b/lib/Target/Hexagon/HexagonScheduleV65.td
@@ -1,9 +1,8 @@
 //=-HexagonScheduleV65.td - HexagonV65 Scheduling Definitions *- tablegen -*-=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/Hexagon/HexagonScheduleV66.td b/lib/Target/Hexagon/HexagonScheduleV66.td
index 38e3d21d370174133bf010918b22cb1651b35565..56dc59e2a948f4358a80153682acb16e1f88a3df 100644
--- a/lib/Target/Hexagon/HexagonScheduleV66.td
+++ b/lib/Target/Hexagon/HexagonScheduleV66.td
@@ -1,9 +1,8 @@
 //=-HexagonScheduleV66.td - HexagonV66 Scheduling Definitions *- tablegen -*-=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/Hexagon/HexagonSelectionDAGInfo.cpp b/lib/Target/Hexagon/HexagonSelectionDAGInfo.cpp
index 002e87fb32ce5a7e6cc4e91bb2579c3a058a1e34..c5ba7ced4c3088fc7c968fa228b8d5e7aaee3aeb 100644
--- a/lib/Target/Hexagon/HexagonSelectionDAGInfo.cpp
+++ b/lib/Target/Hexagon/HexagonSelectionDAGInfo.cpp
@@ -1,9 +1,8 @@
 //===-- HexagonSelectionDAGInfo.cpp - Hexagon SelectionDAG Info -----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Hexagon/HexagonSelectionDAGInfo.h b/lib/Target/Hexagon/HexagonSelectionDAGInfo.h
index a83a8efb75882b3e16fd363b3da1e01fb3e3035b..af8b8318b059ebbc6fd9d4ded87ff0ce3accca99 100644
--- a/lib/Target/Hexagon/HexagonSelectionDAGInfo.h
+++ b/lib/Target/Hexagon/HexagonSelectionDAGInfo.h
@@ -1,9 +1,8 @@
 //===-- HexagonSelectionDAGInfo.h - Hexagon SelectionDAG Info ---*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Hexagon/HexagonSplitConst32AndConst64.cpp b/lib/Target/Hexagon/HexagonSplitConst32AndConst64.cpp
index 55de25120943b304da4227e170bdb04ac4f624d5..bd4254aea276a6de87466a39fff7a84cbf0a6ed4 100644
--- a/lib/Target/Hexagon/HexagonSplitConst32AndConst64.cpp
+++ b/lib/Target/Hexagon/HexagonSplitConst32AndConst64.cpp
@@ -1,9 +1,8 @@
 //=== HexagonSplitConst32AndConst64.cpp - split CONST32/Const64 into HI/LO ===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Hexagon/HexagonSplitDouble.cpp b/lib/Target/Hexagon/HexagonSplitDouble.cpp
index e018785f24d87a34ad8b56a742c32be645535f8f..e9a4a0733dbc2e1e920a2e798a4a2fd37838556f 100644
--- a/lib/Target/Hexagon/HexagonSplitDouble.cpp
+++ b/lib/Target/Hexagon/HexagonSplitDouble.cpp
@@ -1,9 +1,8 @@
 //===- HexagonSplitDouble.cpp ---------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/Hexagon/HexagonStoreWidening.cpp b/lib/Target/Hexagon/HexagonStoreWidening.cpp
index 61c2121163b87574a0a61879f01518432d755b51..a306c0ddade625190318f4701ff9da9400bee1f3 100644
--- a/lib/Target/Hexagon/HexagonStoreWidening.cpp
+++ b/lib/Target/Hexagon/HexagonStoreWidening.cpp
@@ -1,9 +1,8 @@
 //===- HexagonStoreWidening.cpp -------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 // Replace sequences of "narrow" stores to adjacent memory locations with
diff --git a/lib/Target/Hexagon/HexagonSubtarget.cpp b/lib/Target/Hexagon/HexagonSubtarget.cpp
index 9c77135c2f2f686580d2c12463cec38a7768a25e..7ec63a642b0c028e8ae2bec655765352538c9585 100644
--- a/lib/Target/Hexagon/HexagonSubtarget.cpp
+++ b/lib/Target/Hexagon/HexagonSubtarget.cpp
@@ -1,9 +1,8 @@
 //===- HexagonSubtarget.cpp - Hexagon Subtarget Information ---------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Hexagon/HexagonSubtarget.h b/lib/Target/Hexagon/HexagonSubtarget.h
index 3a5acb53682cd2dee84e6f821745a12dea14b225..007423ef19028998242e6874192f2d062963dcf5 100644
--- a/lib/Target/Hexagon/HexagonSubtarget.h
+++ b/lib/Target/Hexagon/HexagonSubtarget.h
@@ -1,9 +1,8 @@
 //===- HexagonSubtarget.h - Define Subtarget for the Hexagon ----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Hexagon/HexagonTargetMachine.cpp b/lib/Target/Hexagon/HexagonTargetMachine.cpp
index ddfda7e27793947dfa1df82d1565de16c58f2ea8..8851a0819e92b77db29f20552d957bb0430e76ec 100644
--- a/lib/Target/Hexagon/HexagonTargetMachine.cpp
+++ b/lib/Target/Hexagon/HexagonTargetMachine.cpp
@@ -1,9 +1,8 @@
 //===-- HexagonTargetMachine.cpp - Define TargetMachine for Hexagon -------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Hexagon/HexagonTargetMachine.h b/lib/Target/Hexagon/HexagonTargetMachine.h
index a7c6a3437fbc81e3386fbaa976d47ab5728c55c2..7ee4474e90e3fc8b392a14dd57930437471b7bfc 100644
--- a/lib/Target/Hexagon/HexagonTargetMachine.h
+++ b/lib/Target/Hexagon/HexagonTargetMachine.h
@@ -1,9 +1,8 @@
 //=-- HexagonTargetMachine.h - Define TargetMachine for Hexagon ---*- C++ -*-=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Hexagon/HexagonTargetObjectFile.cpp b/lib/Target/Hexagon/HexagonTargetObjectFile.cpp
index 2185bf8eebc66d0fab86786a6e68bd5ae9748e7c..d797ce2f4ee0d9e9e086612a94105993e37239f9 100644
--- a/lib/Target/Hexagon/HexagonTargetObjectFile.cpp
+++ b/lib/Target/Hexagon/HexagonTargetObjectFile.cpp
@@ -1,9 +1,8 @@
 //===-- HexagonTargetObjectFile.cpp ---------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Hexagon/HexagonTargetObjectFile.h b/lib/Target/Hexagon/HexagonTargetObjectFile.h
index 18863630fde23d5a9812604642fa80e63ae4162b..b362825789503cffdf0fac7bbefab3a5f22408f2 100644
--- a/lib/Target/Hexagon/HexagonTargetObjectFile.h
+++ b/lib/Target/Hexagon/HexagonTargetObjectFile.h
@@ -1,9 +1,8 @@
 //===-- HexagonTargetObjectFile.h -----------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/Hexagon/HexagonTargetStreamer.h b/lib/Target/Hexagon/HexagonTargetStreamer.h
index e19c404450e6bc8e57fe5f11285eff12ed3a83f6..c5200b76933e949ff42b23ea00cb3db4355ec55f 100644
--- a/lib/Target/Hexagon/HexagonTargetStreamer.h
+++ b/lib/Target/Hexagon/HexagonTargetStreamer.h
@@ -1,9 +1,8 @@
 //===-- HexagonTargetStreamer.h - Hexagon Target Streamer ------*- C++ -*--===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp b/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp
index c942f645aa88706859acc84210b952c12aaddf5e..c817cb1842de6e70a8af2efa3ee98b59dc70896d 100644
--- a/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp
+++ b/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp
@@ -1,9 +1,8 @@
 //===- HexagonTargetTransformInfo.cpp - Hexagon specific TTI pass ---------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 /// \file
 /// This file implements a TargetTransformInfo analysis pass specific to the
diff --git a/lib/Target/Hexagon/HexagonTargetTransformInfo.h b/lib/Target/Hexagon/HexagonTargetTransformInfo.h
index 5c6f85584ec2f45a1d9d7828cca4f12d29d0473b..27e8fc0190073f60dc6062572030f4854584d129 100644
--- a/lib/Target/Hexagon/HexagonTargetTransformInfo.h
+++ b/lib/Target/Hexagon/HexagonTargetTransformInfo.h
@@ -1,9 +1,8 @@
 //==- HexagonTargetTransformInfo.cpp - Hexagon specific TTI pass -*- C++ -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 /// \file
 /// This file implements a TargetTransformInfo analysis pass specific to the
diff --git a/lib/Target/Hexagon/HexagonVExtract.cpp b/lib/Target/Hexagon/HexagonVExtract.cpp
index 929ac2bd0d938b0eb8a3355a6a4d190e0c33a255..a9692f42e46892e5d05427812f04486a9d3863d2 100644
--- a/lib/Target/Hexagon/HexagonVExtract.cpp
+++ b/lib/Target/Hexagon/HexagonVExtract.cpp
@@ -1,9 +1,8 @@
 //===- HexagonVExtract.cpp ------------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 // This pass will replace multiple occurrences of V6_extractw from the same
diff --git a/lib/Target/Hexagon/HexagonVLIWPacketizer.cpp b/lib/Target/Hexagon/HexagonVLIWPacketizer.cpp
index 722699907ca04aa71411dc5ac2732cbd2900a623..3619e4c239d72ebe5f7f9a9f4cd06d4a83be20bb 100644
--- a/lib/Target/Hexagon/HexagonVLIWPacketizer.cpp
+++ b/lib/Target/Hexagon/HexagonVLIWPacketizer.cpp
@@ -1,9 +1,8 @@
 //===- HexagonPacketizer.cpp - VLIW packetizer ----------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Hexagon/HexagonVLIWPacketizer.h b/lib/Target/Hexagon/HexagonVLIWPacketizer.h
index ca70cf967a46fa994fd12bc34fea78f3efdcdd65..daa86b6f53933d47e6e1b964b30af577578d708c 100644
--- a/lib/Target/Hexagon/HexagonVLIWPacketizer.h
+++ b/lib/Target/Hexagon/HexagonVLIWPacketizer.h
@@ -1,9 +1,8 @@
 //===- HexagonPacketizer.h - VLIW packetizer --------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/Hexagon/HexagonVectorLoopCarriedReuse.cpp b/lib/Target/Hexagon/HexagonVectorLoopCarriedReuse.cpp
index 9d1073346c7239ff0a45ec7ff707deea910de366..b3ef3e487280eaeca216dec1d35b3cfa46c422ce 100644
--- a/lib/Target/Hexagon/HexagonVectorLoopCarriedReuse.cpp
+++ b/lib/Target/Hexagon/HexagonVectorLoopCarriedReuse.cpp
@@ -1,9 +1,8 @@
 //===- HexagonVectorLoopCarriedReuse.cpp ----------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Hexagon/HexagonVectorPrint.cpp b/lib/Target/Hexagon/HexagonVectorPrint.cpp
index 18d2f2f4acde2f5ee6c7312e20a96283cf87a513..65a8dcd75bdca430acd9d000fd90aa0a61bb7548 100644
--- a/lib/Target/Hexagon/HexagonVectorPrint.cpp
+++ b/lib/Target/Hexagon/HexagonVectorPrint.cpp
@@ -1,9 +1,8 @@
 //===- HexagonVectorPrint.cpp - Generate vector printing instructions -----===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Hexagon/LLVMBuild.txt b/lib/Target/Hexagon/LLVMBuild.txt
index 7a27a8c5e10f623945b08bfba8469f8a262cc0c9..3688231b7e050b30b057cec6e60974fd74e5e54e 100644
--- a/lib/Target/Hexagon/LLVMBuild.txt
+++ b/lib/Target/Hexagon/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/Target/Hexagon/LLVMBuild.txt -----------------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Target/Hexagon/MCTargetDesc/HexagonAsmBackend.cpp b/lib/Target/Hexagon/MCTargetDesc/HexagonAsmBackend.cpp
index af1e5429d0c20ac9890386ecfdc50d4c464eee71..57f4860bed60d5430fb15ac9698de6aec4b3ebe0 100644
--- a/lib/Target/Hexagon/MCTargetDesc/HexagonAsmBackend.cpp
+++ b/lib/Target/Hexagon/MCTargetDesc/HexagonAsmBackend.cpp
@@ -1,9 +1,8 @@
 //===-- HexagonAsmBackend.cpp - Hexagon Assembler Backend -----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/Hexagon/MCTargetDesc/HexagonBaseInfo.h b/lib/Target/Hexagon/MCTargetDesc/HexagonBaseInfo.h
index 6543d831390064346a9185984783878e5607226f..3c64893bae45ec65d85045e2b91fadcc6d258cc0 100644
--- a/lib/Target/Hexagon/MCTargetDesc/HexagonBaseInfo.h
+++ b/lib/Target/Hexagon/MCTargetDesc/HexagonBaseInfo.h
@@ -1,9 +1,8 @@
 //===- HexagonBaseInfo.h - Top level definitions for Hexagon ----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Hexagon/MCTargetDesc/HexagonELFObjectWriter.cpp b/lib/Target/Hexagon/MCTargetDesc/HexagonELFObjectWriter.cpp
index e82e6b559f628bbeb2d70156048da4b5ee946df9..a0e0489e90169465c87dd86268fa8fe562cc2c40 100644
--- a/lib/Target/Hexagon/MCTargetDesc/HexagonELFObjectWriter.cpp
+++ b/lib/Target/Hexagon/MCTargetDesc/HexagonELFObjectWriter.cpp
@@ -1,9 +1,8 @@
 //===-- HexagonELFObjectWriter.cpp - Hexagon Target Descriptions ----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/Hexagon/MCTargetDesc/HexagonFixupKinds.h b/lib/Target/Hexagon/MCTargetDesc/HexagonFixupKinds.h
index 347327669ad9cf9fee092fe884c2071f95ea6b3b..8b0ddbcb949f0e4bb78c9f7d57c8a76b1b14b891 100644
--- a/lib/Target/Hexagon/MCTargetDesc/HexagonFixupKinds.h
+++ b/lib/Target/Hexagon/MCTargetDesc/HexagonFixupKinds.h
@@ -1,9 +1,8 @@
 //===-- HexagonFixupKinds.h - Hexagon Specific Fixup Entries --------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/Hexagon/MCTargetDesc/HexagonInstPrinter.cpp b/lib/Target/Hexagon/MCTargetDesc/HexagonInstPrinter.cpp
index 687e79a7dbab9990453f197d18dff64c91ea5e4b..7adddf1f2ac1ea60c5f28d1122a02a84f49046d1 100644
--- a/lib/Target/Hexagon/MCTargetDesc/HexagonInstPrinter.cpp
+++ b/lib/Target/Hexagon/MCTargetDesc/HexagonInstPrinter.cpp
@@ -1,9 +1,8 @@
 //===- HexagonInstPrinter.cpp - Convert Hexagon MCInst to assembly syntax -===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Hexagon/MCTargetDesc/HexagonInstPrinter.h b/lib/Target/Hexagon/MCTargetDesc/HexagonInstPrinter.h
index 17af046ce0908f1c965edefaedb41793beec5e0f..ca32c3c1f50f0713c3cc65e609c2569f3ce91189 100644
--- a/lib/Target/Hexagon/MCTargetDesc/HexagonInstPrinter.h
+++ b/lib/Target/Hexagon/MCTargetDesc/HexagonInstPrinter.h
@@ -1,9 +1,8 @@
 //===-- HexagonInstPrinter.h - Convert Hexagon MCInst to assembly syntax --===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Hexagon/MCTargetDesc/HexagonMCAsmInfo.cpp b/lib/Target/Hexagon/MCTargetDesc/HexagonMCAsmInfo.cpp
index 446b3b2ce668a77af128b30f2cdf78bb7c15fa6c..f3da675623209e57d98f3b78715efa11a4782137 100644
--- a/lib/Target/Hexagon/MCTargetDesc/HexagonMCAsmInfo.cpp
+++ b/lib/Target/Hexagon/MCTargetDesc/HexagonMCAsmInfo.cpp
@@ -1,9 +1,8 @@
 //===-- HexagonMCAsmInfo.cpp - Hexagon asm properties ---------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Hexagon/MCTargetDesc/HexagonMCAsmInfo.h b/lib/Target/Hexagon/MCTargetDesc/HexagonMCAsmInfo.h
index efeff24362343d93e7d4c2af733469bfb5fb86d5..e1f0a26cf858445e14c89fd4e903fd9663f9120d 100644
--- a/lib/Target/Hexagon/MCTargetDesc/HexagonMCAsmInfo.h
+++ b/lib/Target/Hexagon/MCTargetDesc/HexagonMCAsmInfo.h
@@ -1,9 +1,8 @@
 //===-- HexagonTargetAsmInfo.h - Hexagon asm properties --------*- C++ -*--===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Hexagon/MCTargetDesc/HexagonMCChecker.cpp b/lib/Target/Hexagon/MCTargetDesc/HexagonMCChecker.cpp
index 53f3cba052bcc53accb529811a92a52a0311ab09..10041aa8af9e29ddd3dc05850873b34be4bc1894 100644
--- a/lib/Target/Hexagon/MCTargetDesc/HexagonMCChecker.cpp
+++ b/lib/Target/Hexagon/MCTargetDesc/HexagonMCChecker.cpp
@@ -1,9 +1,8 @@
 //===----- HexagonMCChecker.cpp - Instruction bundle checking -------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Hexagon/MCTargetDesc/HexagonMCChecker.h b/lib/Target/Hexagon/MCTargetDesc/HexagonMCChecker.h
index 7577baace20cd6364fe2deec95a1b309519742ce..bc55ade9ccd78e7a3820986c7391190efeef2f1d 100644
--- a/lib/Target/Hexagon/MCTargetDesc/HexagonMCChecker.h
+++ b/lib/Target/Hexagon/MCTargetDesc/HexagonMCChecker.h
@@ -1,9 +1,8 @@
 //===- HexagonMCChecker.h - Instruction bundle checking ---------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Hexagon/MCTargetDesc/HexagonMCCodeEmitter.cpp b/lib/Target/Hexagon/MCTargetDesc/HexagonMCCodeEmitter.cpp
index 3382684803aa52cab2194432da37a1cc64a6aeaa..8c53027bd8472c8a09116e95e13cb9eeca5901f6 100644
--- a/lib/Target/Hexagon/MCTargetDesc/HexagonMCCodeEmitter.cpp
+++ b/lib/Target/Hexagon/MCTargetDesc/HexagonMCCodeEmitter.cpp
@@ -1,9 +1,8 @@
 //===- HexagonMCCodeEmitter.cpp - Hexagon Target Descriptions -------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/Hexagon/MCTargetDesc/HexagonMCCodeEmitter.h b/lib/Target/Hexagon/MCTargetDesc/HexagonMCCodeEmitter.h
index fcea63db23a3dcc5b9f18984535eed387fdd9673..9c60e963e1ff68d438bea634a553c5bebbdd7def 100644
--- a/lib/Target/Hexagon/MCTargetDesc/HexagonMCCodeEmitter.h
+++ b/lib/Target/Hexagon/MCTargetDesc/HexagonMCCodeEmitter.h
@@ -1,9 +1,8 @@
 //===- HexagonMCCodeEmitter.h - Hexagon Target Descriptions -----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
diff --git a/lib/Target/Hexagon/MCTargetDesc/HexagonMCCompound.cpp b/lib/Target/Hexagon/MCTargetDesc/HexagonMCCompound.cpp
index 3eaef9ac7410cb2cdf32672d3c1efbbb810b0dc4..f9c046b7bfa7deb0ef4b0ad4898d315a1108fbb9 100644
--- a/lib/Target/Hexagon/MCTargetDesc/HexagonMCCompound.cpp
+++ b/lib/Target/Hexagon/MCTargetDesc/HexagonMCCompound.cpp
@@ -1,9 +1,8 @@
 //=== HexagonMCCompound.cpp - Hexagon Compound checker  -------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Hexagon/MCTargetDesc/HexagonMCDuplexInfo.cpp b/lib/Target/Hexagon/MCTargetDesc/HexagonMCDuplexInfo.cpp
index f0654d612b4bea8cf28a619d1624fc2a92801f6f..3cbb8600ce7aa809e75f87152674392dc916d661 100644
--- a/lib/Target/Hexagon/MCTargetDesc/HexagonMCDuplexInfo.cpp
+++ b/lib/Target/Hexagon/MCTargetDesc/HexagonMCDuplexInfo.cpp
@@ -1,9 +1,8 @@
 //===- HexagonMCDuplexInfo.cpp - Instruction bundle checking --------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Hexagon/MCTargetDesc/HexagonMCELFStreamer.cpp b/lib/Target/Hexagon/MCTargetDesc/HexagonMCELFStreamer.cpp
index f304bc50530ff57240f698388bc33531f086ee07..f2432883af6fff5ec69f3d850e9156299d8bf095 100644
--- a/lib/Target/Hexagon/MCTargetDesc/HexagonMCELFStreamer.cpp
+++ b/lib/Target/Hexagon/MCTargetDesc/HexagonMCELFStreamer.cpp
@@ -1,9 +1,8 @@
 //=== HexagonMCELFStreamer.cpp - Hexagon subclass of MCELFStreamer -------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -60,7 +59,7 @@ HexagonMCELFStreamer::HexagonMCELFStreamer(
       MCII(createHexagonMCInstrInfo()) {}
 
 void HexagonMCELFStreamer::EmitInstruction(const MCInst &MCB,
-                                           const MCSubtargetInfo &STI, bool) {
+                                           const MCSubtargetInfo &STI) {
   assert(MCB.getOpcode() == Hexagon::BUNDLE);
   assert(HexagonMCInstrInfo::bundleSize(MCB) <= HEXAGON_PACKET_SIZE);
   assert(HexagonMCInstrInfo::bundleSize(MCB) > 0);
diff --git a/lib/Target/Hexagon/MCTargetDesc/HexagonMCELFStreamer.h b/lib/Target/Hexagon/MCTargetDesc/HexagonMCELFStreamer.h
index c02bef8f06f73bcd8be0320c4d259320461c0348..6248bd25d433b95b0bc7c3d431b64b86118ba765 100644
--- a/lib/Target/Hexagon/MCTargetDesc/HexagonMCELFStreamer.h
+++ b/lib/Target/Hexagon/MCTargetDesc/HexagonMCELFStreamer.h
@@ -1,9 +1,8 @@
 //===- HexagonMCELFStreamer.h - Hexagon subclass of MCElfStreamer ---------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -31,8 +30,7 @@ public:
                        std::unique_ptr<MCCodeEmitter> Emitter,
                        MCAssembler *Assembler);
 
-  void EmitInstruction(const MCInst &Inst, const MCSubtargetInfo &STI,
-                       bool) override;
+  void EmitInstruction(const MCInst &Inst, const MCSubtargetInfo &STI) override;
   void EmitSymbol(const MCInst &Inst);
   void HexagonMCEmitLocalCommonSymbol(MCSymbol *Symbol, uint64_t Size,
                                       unsigned ByteAlignment,
diff --git a/lib/Target/Hexagon/MCTargetDesc/HexagonMCExpr.cpp b/lib/Target/Hexagon/MCTargetDesc/HexagonMCExpr.cpp
index f0689252b39688d7b9740a3f7f36084fef46d9dc..1e708ba1bcd3accf2608471b8f32d90b0ca73fc6 100644
--- a/lib/Target/Hexagon/MCTargetDesc/HexagonMCExpr.cpp
+++ b/lib/Target/Hexagon/MCTargetDesc/HexagonMCExpr.cpp
@@ -1,10 +1,9 @@
 //===-- HexagonMCExpr.cpp - Hexagon specific MC expression classes
 //----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/Hexagon/MCTargetDesc/HexagonMCExpr.h b/lib/Target/Hexagon/MCTargetDesc/HexagonMCExpr.h
index acfd996ccf82c74dd8634a51075797ae21a013af..59b1326adf0cccc247e658bbe6c4a2e4452bfc8b 100644
--- a/lib/Target/Hexagon/MCTargetDesc/HexagonMCExpr.h
+++ b/lib/Target/Hexagon/MCTargetDesc/HexagonMCExpr.h
@@ -1,9 +1,8 @@
 //==- HexagonMCExpr.h - Hexagon specific MC expression classes --*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/Hexagon/MCTargetDesc/HexagonMCInstrInfo.cpp b/lib/Target/Hexagon/MCTargetDesc/HexagonMCInstrInfo.cpp
index a11aa92ccbe12e5f38582bdfea90bce5d2b26608..6c0a3b644e9d824723954676ec70415323dc9a11 100644
--- a/lib/Target/Hexagon/MCTargetDesc/HexagonMCInstrInfo.cpp
+++ b/lib/Target/Hexagon/MCTargetDesc/HexagonMCInstrInfo.cpp
@@ -1,9 +1,8 @@
 //===- HexagonMCInstrInfo.cpp - Hexagon sub-class of MCInst ---------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Hexagon/MCTargetDesc/HexagonMCInstrInfo.h b/lib/Target/Hexagon/MCTargetDesc/HexagonMCInstrInfo.h
index d040bea23b6d90dd3c2af46238043af83b0d9740..829f872c453e4c88bbb02ad5342b99874674fc90 100644
--- a/lib/Target/Hexagon/MCTargetDesc/HexagonMCInstrInfo.h
+++ b/lib/Target/Hexagon/MCTargetDesc/HexagonMCInstrInfo.h
@@ -1,9 +1,8 @@
 //===- HexagonMCInstrInfo.cpp - Utility functions on Hexagon MCInsts ------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Hexagon/MCTargetDesc/HexagonMCShuffler.cpp b/lib/Target/Hexagon/MCTargetDesc/HexagonMCShuffler.cpp
index 4281144acaee356ab28bee59f1263670ead0ba0c..dc6f19432a0499b96aee1c483907b268e4bdaefd 100644
--- a/lib/Target/Hexagon/MCTargetDesc/HexagonMCShuffler.cpp
+++ b/lib/Target/Hexagon/MCTargetDesc/HexagonMCShuffler.cpp
@@ -1,9 +1,8 @@
 //===----- HexagonMCShuffler.cpp - MC bundle shuffling --------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Hexagon/MCTargetDesc/HexagonMCShuffler.h b/lib/Target/Hexagon/MCTargetDesc/HexagonMCShuffler.h
index 59658999d24d76b8830151b57675a0df0c9ef0a8..3410c0ddbd84e4b60497b93c90c08c0caffb6b2c 100644
--- a/lib/Target/Hexagon/MCTargetDesc/HexagonMCShuffler.h
+++ b/lib/Target/Hexagon/MCTargetDesc/HexagonMCShuffler.h
@@ -1,9 +1,8 @@
 //===- HexagonMCShuffler.h --------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Hexagon/MCTargetDesc/HexagonMCTargetDesc.cpp b/lib/Target/Hexagon/MCTargetDesc/HexagonMCTargetDesc.cpp
index 92ce7345f3584c9de0d40ce065281f9537f4c5e2..7f68156b2149654dce684bb6618d8ec34c49adb2 100644
--- a/lib/Target/Hexagon/MCTargetDesc/HexagonMCTargetDesc.cpp
+++ b/lib/Target/Hexagon/MCTargetDesc/HexagonMCTargetDesc.cpp
@@ -1,9 +1,8 @@
 //===-- HexagonMCTargetDesc.cpp - Hexagon Target Descriptions -------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Hexagon/MCTargetDesc/HexagonMCTargetDesc.h b/lib/Target/Hexagon/MCTargetDesc/HexagonMCTargetDesc.h
index d6ea664222d34944538bf555562b1ee5170cf81e..1ecc8c1d15cded054c177b6ee02c9c78bdfcaba4 100644
--- a/lib/Target/Hexagon/MCTargetDesc/HexagonMCTargetDesc.h
+++ b/lib/Target/Hexagon/MCTargetDesc/HexagonMCTargetDesc.h
@@ -1,9 +1,8 @@
 //===-- HexagonMCTargetDesc.h - Hexagon Target Descriptions -----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Hexagon/MCTargetDesc/HexagonShuffler.cpp b/lib/Target/Hexagon/MCTargetDesc/HexagonShuffler.cpp
index f4ee2bbfaaaa548ba698e2fe06c9c546ddb19bdf..5514a9803ae43e5baa910ad7fd7cb98a6a10bd8d 100644
--- a/lib/Target/Hexagon/MCTargetDesc/HexagonShuffler.cpp
+++ b/lib/Target/Hexagon/MCTargetDesc/HexagonShuffler.cpp
@@ -1,9 +1,8 @@
 //===- HexagonShuffler.cpp - Instruction bundle shuffling -----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Hexagon/MCTargetDesc/HexagonShuffler.h b/lib/Target/Hexagon/MCTargetDesc/HexagonShuffler.h
index ef50c5bebbfb5e6dbe0e328dc3730a7c9e98be27..fb940db33b2ceecd602315b8c23e1546a9c69772 100644
--- a/lib/Target/Hexagon/MCTargetDesc/HexagonShuffler.h
+++ b/lib/Target/Hexagon/MCTargetDesc/HexagonShuffler.h
@@ -1,9 +1,8 @@
 //===- HexagonShuffler.h - Instruction bundle shuffling ---------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Hexagon/MCTargetDesc/LLVMBuild.txt b/lib/Target/Hexagon/MCTargetDesc/LLVMBuild.txt
index f559a21e3f9ec5476a3bcbcf4345f3bbd1a7e2bc..2e00d013706c0440be6d4eae00334c5e8bfba331 100644
--- a/lib/Target/Hexagon/MCTargetDesc/LLVMBuild.txt
+++ b/lib/Target/Hexagon/MCTargetDesc/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/Target/Hexagon/MCTargetDesc/LLVMBuild.txt ----------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Target/Hexagon/RDFCopy.cpp b/lib/Target/Hexagon/RDFCopy.cpp
index 4339fa2089d994e0c50a72dd98f097f358f2b5a9..7702024f87bda7439867ba4901df6d635a5499ab 100644
--- a/lib/Target/Hexagon/RDFCopy.cpp
+++ b/lib/Target/Hexagon/RDFCopy.cpp
@@ -1,9 +1,8 @@
 //===- RDFCopy.cpp --------------------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Hexagon/RDFCopy.h b/lib/Target/Hexagon/RDFCopy.h
index 7b2e78bdf633d6b6384452a8127d37abb686c778..1450ab8848492b4a856ceedaee51840b690f5f3c 100644
--- a/lib/Target/Hexagon/RDFCopy.h
+++ b/lib/Target/Hexagon/RDFCopy.h
@@ -1,9 +1,8 @@
 //===- RDFCopy.h ------------------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/Hexagon/RDFDeadCode.cpp b/lib/Target/Hexagon/RDFDeadCode.cpp
index 8dcd485d65e938ef892940b8cd99b97aaf3420d4..52178931aa6d271afbae6f456cac81a2b68d527a 100644
--- a/lib/Target/Hexagon/RDFDeadCode.cpp
+++ b/lib/Target/Hexagon/RDFDeadCode.cpp
@@ -1,9 +1,8 @@
 //===--- RDFDeadCode.cpp --------------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Hexagon/RDFDeadCode.h b/lib/Target/Hexagon/RDFDeadCode.h
index 8977e730b8553bc6cab4413b103d2de3eda30acb..7f91977e1d6cd5d62344d89b81133ef7a619be6b 100644
--- a/lib/Target/Hexagon/RDFDeadCode.h
+++ b/lib/Target/Hexagon/RDFDeadCode.h
@@ -1,9 +1,8 @@
 //===--- RDFDeadCode.h ----------------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Hexagon/RDFGraph.cpp b/lib/Target/Hexagon/RDFGraph.cpp
index d8ca08e705050a4f28e6e0da7092c08a6c3170a9..18ceef5b718352af069cfde74e021444a263689c 100644
--- a/lib/Target/Hexagon/RDFGraph.cpp
+++ b/lib/Target/Hexagon/RDFGraph.cpp
@@ -1,9 +1,8 @@
 //===- RDFGraph.cpp -------------------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Hexagon/RDFGraph.h b/lib/Target/Hexagon/RDFGraph.h
index e3abb0e22f76fd3b57344b4ef385c125269fb8a7..e91dff084b8d89471b3ac833d7f0046bbd8b126e 100644
--- a/lib/Target/Hexagon/RDFGraph.h
+++ b/lib/Target/Hexagon/RDFGraph.h
@@ -1,9 +1,8 @@
 //===- RDFGraph.h -----------------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Hexagon/RDFLiveness.cpp b/lib/Target/Hexagon/RDFLiveness.cpp
index 9ff48d25a026f834f831d80c992e4c9b7cf2916e..6a3c82566ba54a77a31584d7b8ee9f5043192d57 100644
--- a/lib/Target/Hexagon/RDFLiveness.cpp
+++ b/lib/Target/Hexagon/RDFLiveness.cpp
@@ -1,9 +1,8 @@
 //===- RDFLiveness.cpp ----------------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Hexagon/RDFLiveness.h b/lib/Target/Hexagon/RDFLiveness.h
index eaeb4ea115b3fc9d25556abbfefeea6464557b7b..a66e7ad751322a97a71bb690a135043957fbefa1 100644
--- a/lib/Target/Hexagon/RDFLiveness.h
+++ b/lib/Target/Hexagon/RDFLiveness.h
@@ -1,9 +1,8 @@
 //===- RDFLiveness.h --------------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Hexagon/RDFRegisters.cpp b/lib/Target/Hexagon/RDFRegisters.cpp
index 9408c5dc3952b660cf23b5192145e41608de1afd..6e0f33695f0eaa681142dd78c4db3d43d06ce6cf 100644
--- a/lib/Target/Hexagon/RDFRegisters.cpp
+++ b/lib/Target/Hexagon/RDFRegisters.cpp
@@ -1,9 +1,8 @@
 //===- RDFRegisters.cpp ---------------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/Hexagon/RDFRegisters.h b/lib/Target/Hexagon/RDFRegisters.h
index 459850d87df1b998226964c7184b3595219f91d2..646233bacda51184e4452ffe514b8a3c7f9ce1ae 100644
--- a/lib/Target/Hexagon/RDFRegisters.h
+++ b/lib/Target/Hexagon/RDFRegisters.h
@@ -1,9 +1,8 @@
 //===- RDFRegisters.h -------------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/Hexagon/TargetInfo/HexagonTargetInfo.cpp b/lib/Target/Hexagon/TargetInfo/HexagonTargetInfo.cpp
index 78e2f2b2ddb3e1bcff7fae2d6c309c05a355f0e7..29d9ce6182940c3f673e3c0f360e4245133e9615 100644
--- a/lib/Target/Hexagon/TargetInfo/HexagonTargetInfo.cpp
+++ b/lib/Target/Hexagon/TargetInfo/HexagonTargetInfo.cpp
@@ -1,9 +1,8 @@
 //===-- HexagonTargetInfo.cpp - Hexagon Target Implementation ------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/Hexagon/TargetInfo/LLVMBuild.txt b/lib/Target/Hexagon/TargetInfo/LLVMBuild.txt
index 095a4b003264b6ed0aef8c6be087e0270864c4bc..6d53a72e40cb7b42d9289e45400f234367db6583 100644
--- a/lib/Target/Hexagon/TargetInfo/LLVMBuild.txt
+++ b/lib/Target/Hexagon/TargetInfo/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/Target/Hexagon/TargetInfo/LLVMBuild.txt ------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Target/LLVMBuild.txt b/lib/Target/LLVMBuild.txt
index 0d899a9c78297434c2818766108d3eb2c7d85dc4..d6a95a3c67133ba5dc353c63605adc6b5a35f793 100644
--- a/lib/Target/LLVMBuild.txt
+++ b/lib/Target/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/Target/LLVMBuild.txt -------------------------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
@@ -30,7 +29,6 @@ subdirectories =
  MSP430
  NVPTX
  Mips
- Nios2
  PowerPC
  RISCV
  Sparc
diff --git a/lib/Target/Lanai/AsmParser/LLVMBuild.txt b/lib/Target/Lanai/AsmParser/LLVMBuild.txt
index 251ddcf63c9a26858a27a649b59552c1cb120be6..6be970109299eea07577f4ba89a91cfa52143319 100644
--- a/lib/Target/Lanai/AsmParser/LLVMBuild.txt
+++ b/lib/Target/Lanai/AsmParser/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/Target/Lanai/AsmParser/LLVMBuild.txt ----------------*- Conf -*-===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Target/Lanai/AsmParser/LanaiAsmParser.cpp b/lib/Target/Lanai/AsmParser/LanaiAsmParser.cpp
index a77b2b8f15cadf3d0cb9f1967f2f768e6df7cf64..e60dd6f7b2c9dbf90b09798172951068cc4e014e 100644
--- a/lib/Target/Lanai/AsmParser/LanaiAsmParser.cpp
+++ b/lib/Target/Lanai/AsmParser/LanaiAsmParser.cpp
@@ -1,9 +1,8 @@
 //===-- LanaiAsmParser.cpp - Parse Lanai assembly to MCInst instructions --===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/Lanai/Disassembler/LLVMBuild.txt b/lib/Target/Lanai/Disassembler/LLVMBuild.txt
index 84417a58e0a0ef0c30dbab5ce564afb1133b3433..8bf3384df5209f318957f70376cfb7b5d6314264 100644
--- a/lib/Target/Lanai/Disassembler/LLVMBuild.txt
+++ b/lib/Target/Lanai/Disassembler/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===-- ./lib/Target/Lanai/Disassembler/LLVMBuild.txt -----------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Target/Lanai/Disassembler/LanaiDisassembler.cpp b/lib/Target/Lanai/Disassembler/LanaiDisassembler.cpp
index 609b650e5d32b9cc2c6e2dd45219edc9d8375fa9..59419e5e563552dde80aa414dad5d2934af5d438 100644
--- a/lib/Target/Lanai/Disassembler/LanaiDisassembler.cpp
+++ b/lib/Target/Lanai/Disassembler/LanaiDisassembler.cpp
@@ -1,9 +1,8 @@
 //===- LanaiDisassembler.cpp - Disassembler for Lanai -----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Lanai/Disassembler/LanaiDisassembler.h b/lib/Target/Lanai/Disassembler/LanaiDisassembler.h
index e0c19e8ea6444614b8d5b9cfb15011c3c39b5691..ae821df303d85ca450060b40d2db2f8792f9f5f3 100644
--- a/lib/Target/Lanai/Disassembler/LanaiDisassembler.h
+++ b/lib/Target/Lanai/Disassembler/LanaiDisassembler.h
@@ -1,9 +1,8 @@
 //===- LanaiDisassembler.cpp - Disassembler for Lanai -----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Lanai/InstPrinter/LLVMBuild.txt b/lib/Target/Lanai/InstPrinter/LLVMBuild.txt
index 8d9768c6017b6107760243aa7886ff7fadd18ca6..59c949d4de55dc559882096719eb5bdf6f495721 100644
--- a/lib/Target/Lanai/InstPrinter/LLVMBuild.txt
+++ b/lib/Target/Lanai/InstPrinter/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===-- ./lib/Target/Lanai/InstPrinter/LLVMBuild.txt ------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Target/Lanai/InstPrinter/LanaiInstPrinter.cpp b/lib/Target/Lanai/InstPrinter/LanaiInstPrinter.cpp
index 2fa411fcfd879c2740f25dbf9910ce7801fc1417..2aadc510462da2aca9484d410386bb523c81d2c7 100644
--- a/lib/Target/Lanai/InstPrinter/LanaiInstPrinter.cpp
+++ b/lib/Target/Lanai/InstPrinter/LanaiInstPrinter.cpp
@@ -1,9 +1,8 @@
 //===-- LanaiInstPrinter.cpp - Convert Lanai MCInst to asm syntax ---------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Lanai/InstPrinter/LanaiInstPrinter.h b/lib/Target/Lanai/InstPrinter/LanaiInstPrinter.h
index 59904fbaa318a01cc78f9621782d73eb75d5d453..252aaa749a86deff11a6acb130f61d3561550456 100644
--- a/lib/Target/Lanai/InstPrinter/LanaiInstPrinter.h
+++ b/lib/Target/Lanai/InstPrinter/LanaiInstPrinter.h
@@ -1,9 +1,8 @@
 //= LanaiInstPrinter.h - Convert Lanai MCInst to asm syntax -------*- C++ -*--//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Lanai/LLVMBuild.txt b/lib/Target/Lanai/LLVMBuild.txt
index cb91ffb61a98876358c2cb1e26102488e7080a96..91c55cd2d9ae960621621b0496066870c5f73373 100644
--- a/lib/Target/Lanai/LLVMBuild.txt
+++ b/lib/Target/Lanai/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/Target/Lanai/LLVMBuild.txt -------------------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Target/Lanai/Lanai.h b/lib/Target/Lanai/Lanai.h
index c1fdf793305b729dd9abc083be249bb44a022b4b..63c521a95da7a08994c3ee005ea5eabd4cb49784 100644
--- a/lib/Target/Lanai/Lanai.h
+++ b/lib/Target/Lanai/Lanai.h
@@ -1,9 +1,8 @@
 //===-- Lanai.h - Top-level interface for Lanai representation --*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Lanai/Lanai.td b/lib/Target/Lanai/Lanai.td
index 73d080457034d3875cfa4943aafa6fbea98a5844..c6d949f42047e3a221cabb5bdcc05fa636905ccd 100644
--- a/lib/Target/Lanai/Lanai.td
+++ b/lib/Target/Lanai/Lanai.td
@@ -1,9 +1,8 @@
 //===- Lanai.td - Describe the Lanai Target Machine --------*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/Lanai/LanaiAluCode.h b/lib/Target/Lanai/LanaiAluCode.h
index d5145694fe4621f14c282eb87ac569fdf06ee961..728332bff00b89dddf7b9bca38e0970d1ab234cd 100644
--- a/lib/Target/Lanai/LanaiAluCode.h
+++ b/lib/Target/Lanai/LanaiAluCode.h
@@ -1,9 +1,8 @@
 //===-- LanaiAluCode.h - ALU operator encoding ----------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Lanai/LanaiAsmPrinter.cpp b/lib/Target/Lanai/LanaiAsmPrinter.cpp
index 607b2a97b29f6d8560162cfca50347c3fa06abfe..0b849680447c9b442cd8cda38156040f14981484 100644
--- a/lib/Target/Lanai/LanaiAsmPrinter.cpp
+++ b/lib/Target/Lanai/LanaiAsmPrinter.cpp
@@ -1,9 +1,8 @@
 //===-- LanaiAsmPrinter.cpp - Lanai LLVM assembly writer ------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Lanai/LanaiCallingConv.td b/lib/Target/Lanai/LanaiCallingConv.td
index 056b329c33c50c4e459f7bd3738225596c1e55be..e2306725290adf0f78ee1c906f55d1f28b6db6d6 100644
--- a/lib/Target/Lanai/LanaiCallingConv.td
+++ b/lib/Target/Lanai/LanaiCallingConv.td
@@ -1,9 +1,8 @@
 //===- LanaiCallingConv.td - Calling Conventions Lanai -------*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Lanai/LanaiDelaySlotFiller.cpp b/lib/Target/Lanai/LanaiDelaySlotFiller.cpp
index ea76a112837334046bd0338f189b0a0f37c987c2..09c63dca23e25fc1204f366115615919fe7f0774 100644
--- a/lib/Target/Lanai/LanaiDelaySlotFiller.cpp
+++ b/lib/Target/Lanai/LanaiDelaySlotFiller.cpp
@@ -1,9 +1,8 @@
 //===-- LanaiDelaySlotFiller.cpp - Lanai delay slot filler ----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Lanai/LanaiFrameLowering.cpp b/lib/Target/Lanai/LanaiFrameLowering.cpp
index 0723668c743e10838187c74aaa35913f6a6f6e49..f220323295bf4a9e0a3367af877a23a73134e8a6 100644
--- a/lib/Target/Lanai/LanaiFrameLowering.cpp
+++ b/lib/Target/Lanai/LanaiFrameLowering.cpp
@@ -1,9 +1,8 @@
 //===-- LanaiFrameLowering.cpp - Lanai Frame Information ------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Lanai/LanaiFrameLowering.h b/lib/Target/Lanai/LanaiFrameLowering.h
index ca690d513fc2dcf65b05df82173f60129a385177..043866112947253fdb8840e52d7824c69c0d35dd 100644
--- a/lib/Target/Lanai/LanaiFrameLowering.h
+++ b/lib/Target/Lanai/LanaiFrameLowering.h
@@ -1,9 +1,8 @@
 //===-- LanaiFrameLowering.h - Define frame lowering for Lanai --*- C++-*--===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Lanai/LanaiISelDAGToDAG.cpp b/lib/Target/Lanai/LanaiISelDAGToDAG.cpp
index 5081cfbe492232a39a9e79ae4d23543d357b4c03..b3e96822184ecc80305237fbcec0577354af9bfa 100644
--- a/lib/Target/Lanai/LanaiISelDAGToDAG.cpp
+++ b/lib/Target/Lanai/LanaiISelDAGToDAG.cpp
@@ -1,9 +1,8 @@
 //===-- LanaiISelDAGToDAG.cpp - A dag to dag inst selector for Lanai ------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Lanai/LanaiISelLowering.cpp b/lib/Target/Lanai/LanaiISelLowering.cpp
index 045a897c4126adcd88bc4537ac977e644b197e17..1ed078bb433f71403dbf794b9b1ade3d4bf22496 100644
--- a/lib/Target/Lanai/LanaiISelLowering.cpp
+++ b/lib/Target/Lanai/LanaiISelLowering.cpp
@@ -1,9 +1,8 @@
 //===-- LanaiISelLowering.cpp - Lanai DAG Lowering Implementation ---------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -1498,8 +1497,8 @@ void LanaiTargetLowering::computeKnownBitsForTargetNode(
     break;
   case LanaiISD::SELECT_CC:
     KnownBits Known2;
-    DAG.computeKnownBits(Op->getOperand(0), Known, Depth + 1);
-    DAG.computeKnownBits(Op->getOperand(1), Known2, Depth + 1);
+    Known = DAG.computeKnownBits(Op->getOperand(0), Depth + 1);
+    Known2 = DAG.computeKnownBits(Op->getOperand(1), Depth + 1);
     Known.Zero &= Known2.Zero;
     Known.One &= Known2.One;
     break;
diff --git a/lib/Target/Lanai/LanaiISelLowering.h b/lib/Target/Lanai/LanaiISelLowering.h
index 0cde633cb41a93237fdecd4fabd1f57b69014a59..e7b5755e9041201603bd00fe1521dfc9cba7fb7d 100644
--- a/lib/Target/Lanai/LanaiISelLowering.h
+++ b/lib/Target/Lanai/LanaiISelLowering.h
@@ -1,9 +1,8 @@
 //===-- LanaiISelLowering.h - Lanai DAG Lowering Interface -....-*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Lanai/LanaiInstrFormats.td b/lib/Target/Lanai/LanaiInstrFormats.td
index 1bb6b3d26a498097f3c49cee8573428fb6866a2b..4101aa912ade47b7e6749b55135d6678e516ae5e 100644
--- a/lib/Target/Lanai/LanaiInstrFormats.td
+++ b/lib/Target/Lanai/LanaiInstrFormats.td
@@ -1,9 +1,8 @@
 //===- LanaiInstrFormats.td - Lanai Instruction Formats ----*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/Lanai/LanaiInstrInfo.cpp b/lib/Target/Lanai/LanaiInstrInfo.cpp
index 196768fdc56a3f01594c4621754329543a78febd..93db67c4e3ebfe288e9af302c41833264eb051ba 100644
--- a/lib/Target/Lanai/LanaiInstrInfo.cpp
+++ b/lib/Target/Lanai/LanaiInstrInfo.cpp
@@ -1,9 +1,8 @@
 //===-- LanaiInstrInfo.cpp - Lanai Instruction Information ------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Lanai/LanaiInstrInfo.h b/lib/Target/Lanai/LanaiInstrInfo.h
index bdcf9a361b5f0a4158384d4aa5d78cfcb5ed06df..feda529610c89db34147d95d6e265c2fae4e7018 100644
--- a/lib/Target/Lanai/LanaiInstrInfo.h
+++ b/lib/Target/Lanai/LanaiInstrInfo.h
@@ -1,9 +1,8 @@
 //===- LanaiInstrInfo.h - Lanai Instruction Information ---------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Lanai/LanaiInstrInfo.td b/lib/Target/Lanai/LanaiInstrInfo.td
index 66192b4a470426f2c080e00f1afbff7580594b44..fcf89a0b52f6b73d6a7cf9189c8998de1f62207e 100644
--- a/lib/Target/Lanai/LanaiInstrInfo.td
+++ b/lib/Target/Lanai/LanaiInstrInfo.td
@@ -1,9 +1,8 @@
 //===-- LanaiInstrInfo.td - Target Description for Lanai Target -----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Lanai/LanaiMCInstLower.cpp b/lib/Target/Lanai/LanaiMCInstLower.cpp
index 90ede6566acf3b5e298bf8a47d996858402d7156..743f4f7c6e2f91110ed980461004f76997ed954c 100644
--- a/lib/Target/Lanai/LanaiMCInstLower.cpp
+++ b/lib/Target/Lanai/LanaiMCInstLower.cpp
@@ -1,9 +1,8 @@
 //=-- LanaiMCInstLower.cpp - Convert Lanai MachineInstr to an MCInst --------=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Lanai/LanaiMCInstLower.h b/lib/Target/Lanai/LanaiMCInstLower.h
index 6d7818d63d875bd2c3321411a5108db8e034ae06..00d3ebb0504555437246bbd3754612f409383444 100644
--- a/lib/Target/Lanai/LanaiMCInstLower.h
+++ b/lib/Target/Lanai/LanaiMCInstLower.h
@@ -1,9 +1,8 @@
 //===-- LanaiMCInstLower.h - Lower MachineInstr to MCInst -------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/Lanai/LanaiMachineFunctionInfo.cpp b/lib/Target/Lanai/LanaiMachineFunctionInfo.cpp
index c72271b677900ee92422af9de94f7baa4cc045b7..7b4e0750ba08a13d598c16f098c81c23db71a878 100644
--- a/lib/Target/Lanai/LanaiMachineFunctionInfo.cpp
+++ b/lib/Target/Lanai/LanaiMachineFunctionInfo.cpp
@@ -1,9 +1,8 @@
 //===-- LanaiMachineFuctionInfo.cpp - Lanai machine function info ---===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/Lanai/LanaiMachineFunctionInfo.h b/lib/Target/Lanai/LanaiMachineFunctionInfo.h
index 3bd9112a9e13497f607d82cbfe3151aadc440983..2c97c619c2462576218ad0d06795fa8ccc6e4f6b 100644
--- a/lib/Target/Lanai/LanaiMachineFunctionInfo.h
+++ b/lib/Target/Lanai/LanaiMachineFunctionInfo.h
@@ -1,9 +1,8 @@
 //===- LanaiMachineFuctionInfo.h - Lanai machine func info -------*- C++ -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Lanai/LanaiMemAluCombiner.cpp b/lib/Target/Lanai/LanaiMemAluCombiner.cpp
index 54500b0e52e364d46e1c8ac65dade159201ac405..ed5b6c4fd4a47238f3f81f71b7639dc2cc61239a 100644
--- a/lib/Target/Lanai/LanaiMemAluCombiner.cpp
+++ b/lib/Target/Lanai/LanaiMemAluCombiner.cpp
@@ -1,9 +1,8 @@
 //===-- LanaiMemAluCombiner.cpp - Pass to combine memory & ALU operations -===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 // Simple pass to combine memory and ALU operations
diff --git a/lib/Target/Lanai/LanaiRegisterInfo.cpp b/lib/Target/Lanai/LanaiRegisterInfo.cpp
index 56a5e0ea2def8ff13d456f8ab040aa216ca77406..d113a82b659f9c4ab45e0eaffa8201e3f80d18cd 100644
--- a/lib/Target/Lanai/LanaiRegisterInfo.cpp
+++ b/lib/Target/Lanai/LanaiRegisterInfo.cpp
@@ -1,9 +1,8 @@
 //===-- LanaiRegisterInfo.cpp - Lanai Register Information ------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Lanai/LanaiRegisterInfo.h b/lib/Target/Lanai/LanaiRegisterInfo.h
index 35f4788b2886e2330b0bddc00c1fde655c12fa2e..8ea3ae187237856080bf85a2f1af5787be61271a 100644
--- a/lib/Target/Lanai/LanaiRegisterInfo.h
+++ b/lib/Target/Lanai/LanaiRegisterInfo.h
@@ -1,9 +1,8 @@
 //===- LanaiRegisterInfo.h - Lanai Register Information Impl ----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Lanai/LanaiRegisterInfo.td b/lib/Target/Lanai/LanaiRegisterInfo.td
index cf8cfe30cce92561938eb6bfb9bc76d6bd18b8cd..5879dfca8d658d1b7628eed4d99ec6cab5c191a9 100644
--- a/lib/Target/Lanai/LanaiRegisterInfo.td
+++ b/lib/Target/Lanai/LanaiRegisterInfo.td
@@ -1,9 +1,8 @@
 //===- LanaiRegisterInfo.td - Lanai Register defs ------------*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //  Declarations that describe the Lanai register file
diff --git a/lib/Target/Lanai/LanaiSchedule.td b/lib/Target/Lanai/LanaiSchedule.td
index 7f931c4be8bb98740a01dd1acd5c4aa33648fe3b..32763c7fdf4927557a3b6aca46a6a5b546dccc1d 100644
--- a/lib/Target/Lanai/LanaiSchedule.td
+++ b/lib/Target/Lanai/LanaiSchedule.td
@@ -1,9 +1,8 @@
 //=-LanaiSchedule.td - Lanai Scheduling Definitions --*- tablegen -*-=========//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/Lanai/LanaiSelectionDAGInfo.cpp b/lib/Target/Lanai/LanaiSelectionDAGInfo.cpp
index b71c30fe3e059dcc60c0c3515f3c734c5720539a..dff87a3e264d9f1bf9c171033a015de0b6492a49 100644
--- a/lib/Target/Lanai/LanaiSelectionDAGInfo.cpp
+++ b/lib/Target/Lanai/LanaiSelectionDAGInfo.cpp
@@ -1,9 +1,8 @@
 //===-- LanaiSelectionDAGInfo.cpp - Lanai SelectionDAG Info -------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Lanai/LanaiSelectionDAGInfo.h b/lib/Target/Lanai/LanaiSelectionDAGInfo.h
index bfd2be2ede09efc34e09d9ff153bba000760a865..c5650a7c1f53beeb5190503e01dd33c6c7c6dd85 100644
--- a/lib/Target/Lanai/LanaiSelectionDAGInfo.h
+++ b/lib/Target/Lanai/LanaiSelectionDAGInfo.h
@@ -1,9 +1,8 @@
 //===-- LanaiSelectionDAGInfo.h - Lanai SelectionDAG Info -----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Lanai/LanaiSubtarget.cpp b/lib/Target/Lanai/LanaiSubtarget.cpp
index 0fa5e82a7a66e9b4576b5a850f4dc79a4d1a1299..9a872c789bcc6262bc687b05ad2147404675def7 100644
--- a/lib/Target/Lanai/LanaiSubtarget.cpp
+++ b/lib/Target/Lanai/LanaiSubtarget.cpp
@@ -1,9 +1,8 @@
 //===- LanaiSubtarget.cpp - Lanai Subtarget Information -----------*- C++ -*-=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Lanai/LanaiSubtarget.h b/lib/Target/Lanai/LanaiSubtarget.h
index 4bfa199202390a686cbd2c86ff33f098d0f32299..116c83a4df91737a9c883a3048f4afcc0c2e82b3 100644
--- a/lib/Target/Lanai/LanaiSubtarget.h
+++ b/lib/Target/Lanai/LanaiSubtarget.h
@@ -1,9 +1,8 @@
 //=====-- LanaiSubtarget.h - Define Subtarget for the Lanai -----*- C++ -*--==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Lanai/LanaiTargetMachine.cpp b/lib/Target/Lanai/LanaiTargetMachine.cpp
index 10bd9e2c65d2f20437d5a62c0c5da0d6c790bf4c..fdf1e368ee4b867ca95048f13703c03e12605e48 100644
--- a/lib/Target/Lanai/LanaiTargetMachine.cpp
+++ b/lib/Target/Lanai/LanaiTargetMachine.cpp
@@ -1,9 +1,8 @@
 //===-- LanaiTargetMachine.cpp - Define TargetMachine for Lanai ---------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Lanai/LanaiTargetMachine.h b/lib/Target/Lanai/LanaiTargetMachine.h
index 0db286ec13e7ea86c877121ae68774612afa1b98..d2ac40007e24d0afcc0e0e5901b6e7409b11670a 100644
--- a/lib/Target/Lanai/LanaiTargetMachine.h
+++ b/lib/Target/Lanai/LanaiTargetMachine.h
@@ -1,9 +1,8 @@
 //===-- LanaiTargetMachine.h - Define TargetMachine for Lanai --- C++ ---===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Lanai/LanaiTargetObjectFile.cpp b/lib/Target/Lanai/LanaiTargetObjectFile.cpp
index 7d165e9c5f8c3d2cafdea4acc56f29c00b3add8e..b0f7c090bb8ec8505a7c3f650999241c93318b6a 100644
--- a/lib/Target/Lanai/LanaiTargetObjectFile.cpp
+++ b/lib/Target/Lanai/LanaiTargetObjectFile.cpp
@@ -1,8 +1,7 @@
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/Lanai/LanaiTargetObjectFile.h b/lib/Target/Lanai/LanaiTargetObjectFile.h
index 99ec1956da4b96111cb1596187b69981b866d1cf..938a1e675b6a4584ce82607a759b4a759e1e67b7 100644
--- a/lib/Target/Lanai/LanaiTargetObjectFile.h
+++ b/lib/Target/Lanai/LanaiTargetObjectFile.h
@@ -1,9 +1,8 @@
 //===-- LanaiTargetObjectFile.h - Lanai Object Info -----------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/Lanai/LanaiTargetTransformInfo.h b/lib/Target/Lanai/LanaiTargetTransformInfo.h
index 3b5a1b88326bff159ebc158d5b572dd3fffeab20..63cc47dedce306b83dcf2f2982805f0eec6a06e6 100644
--- a/lib/Target/Lanai/LanaiTargetTransformInfo.h
+++ b/lib/Target/Lanai/LanaiTargetTransformInfo.h
@@ -1,9 +1,8 @@
 //===-- LanaiTargetTransformInfo.h - Lanai specific TTI ---------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Lanai/MCTargetDesc/LLVMBuild.txt b/lib/Target/Lanai/MCTargetDesc/LLVMBuild.txt
index 05e52ccc18d712565102b764691174e53bf12a29..c705e86ea3380ee304bb2305c743ff68676f8f25 100644
--- a/lib/Target/Lanai/MCTargetDesc/LLVMBuild.txt
+++ b/lib/Target/Lanai/MCTargetDesc/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===-- ./lib/Target/Lanai/MCTargetDesc/LLVMBuild.txt -----------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Target/Lanai/MCTargetDesc/LanaiAsmBackend.cpp b/lib/Target/Lanai/MCTargetDesc/LanaiAsmBackend.cpp
index 82fa93ea5e5e0accea9cac8f60c228ec9ee8ba05..a6ce3d5eb4ff0b661c33ea16a5d535af67ba96bd 100644
--- a/lib/Target/Lanai/MCTargetDesc/LanaiAsmBackend.cpp
+++ b/lib/Target/Lanai/MCTargetDesc/LanaiAsmBackend.cpp
@@ -1,9 +1,8 @@
 //===-- LanaiAsmBackend.cpp - Lanai Assembler Backend ---------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/Lanai/MCTargetDesc/LanaiBaseInfo.h b/lib/Target/Lanai/MCTargetDesc/LanaiBaseInfo.h
index ce7f83509c9b22368928d43ec1d288e4bd8b638c..1bc84014e7367f521aafac9a3058fb10a612ff93 100644
--- a/lib/Target/Lanai/MCTargetDesc/LanaiBaseInfo.h
+++ b/lib/Target/Lanai/MCTargetDesc/LanaiBaseInfo.h
@@ -1,9 +1,8 @@
 //===-- LanaiBaseInfo.h - Top level definitions for Lanai MC ----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Lanai/MCTargetDesc/LanaiELFObjectWriter.cpp b/lib/Target/Lanai/MCTargetDesc/LanaiELFObjectWriter.cpp
index 7676891ef981a7ec6bbede6bd8ebff3e80dec89d..d983ef2fc4f10ab6926b32036e9e01e5648972dc 100644
--- a/lib/Target/Lanai/MCTargetDesc/LanaiELFObjectWriter.cpp
+++ b/lib/Target/Lanai/MCTargetDesc/LanaiELFObjectWriter.cpp
@@ -1,9 +1,8 @@
 //===-- LanaiELFObjectWriter.cpp - Lanai ELF Writer -----------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/Lanai/MCTargetDesc/LanaiFixupKinds.h b/lib/Target/Lanai/MCTargetDesc/LanaiFixupKinds.h
index 9ff8340d2922ec63a9a90d3fac8e2ce74cd42b08..1e692f8d31cb4972c3f92d83081868dea91268a8 100644
--- a/lib/Target/Lanai/MCTargetDesc/LanaiFixupKinds.h
+++ b/lib/Target/Lanai/MCTargetDesc/LanaiFixupKinds.h
@@ -1,9 +1,8 @@
 //===-- LanaiFixupKinds.h - Lanai Specific Fixup Entries --------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/Lanai/MCTargetDesc/LanaiMCAsmInfo.cpp b/lib/Target/Lanai/MCTargetDesc/LanaiMCAsmInfo.cpp
index 7e2705e67b6d770ade4ca0988582959c06104b61..14d3dac26d1f7d105ebc24c162e041f981683a3f 100644
--- a/lib/Target/Lanai/MCTargetDesc/LanaiMCAsmInfo.cpp
+++ b/lib/Target/Lanai/MCTargetDesc/LanaiMCAsmInfo.cpp
@@ -1,9 +1,8 @@
 //===-- LanaiMCAsmInfo.cpp - Lanai asm properties -----------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Lanai/MCTargetDesc/LanaiMCAsmInfo.h b/lib/Target/Lanai/MCTargetDesc/LanaiMCAsmInfo.h
index 3eef0592d2fa9b30fe82309147cc4f5d95543dc3..265af425d03749890aadd699dddb55e93a4a5508 100644
--- a/lib/Target/Lanai/MCTargetDesc/LanaiMCAsmInfo.h
+++ b/lib/Target/Lanai/MCTargetDesc/LanaiMCAsmInfo.h
@@ -1,9 +1,8 @@
 //=====-- LanaiMCAsmInfo.h - Lanai asm properties -----------*- C++ -*--====//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Lanai/MCTargetDesc/LanaiMCCodeEmitter.cpp b/lib/Target/Lanai/MCTargetDesc/LanaiMCCodeEmitter.cpp
index 21f4005aaf8395e348c3b17623a32245956bf996..adfe528f9b528682e65e454f5b1a221c4f5c6327 100644
--- a/lib/Target/Lanai/MCTargetDesc/LanaiMCCodeEmitter.cpp
+++ b/lib/Target/Lanai/MCTargetDesc/LanaiMCCodeEmitter.cpp
@@ -1,9 +1,8 @@
 //===-- LanaiMCCodeEmitter.cpp - Convert Lanai code to machine code -------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Lanai/MCTargetDesc/LanaiMCExpr.cpp b/lib/Target/Lanai/MCTargetDesc/LanaiMCExpr.cpp
index 201c95de07f471c9cdb421ced3ea59f625a906b1..56d5fbf40360fb7b2d6562765862c1aacfca1ba9 100644
--- a/lib/Target/Lanai/MCTargetDesc/LanaiMCExpr.cpp
+++ b/lib/Target/Lanai/MCTargetDesc/LanaiMCExpr.cpp
@@ -1,9 +1,8 @@
 //===-- LanaiMCExpr.cpp - Lanai specific MC expression classes ------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/Lanai/MCTargetDesc/LanaiMCExpr.h b/lib/Target/Lanai/MCTargetDesc/LanaiMCExpr.h
index 5004d541ff70b7b6cd9f0ec2ecea0100d4fb8fa0..c99af32d91028092a12d6fba271c9931de43bbff 100644
--- a/lib/Target/Lanai/MCTargetDesc/LanaiMCExpr.h
+++ b/lib/Target/Lanai/MCTargetDesc/LanaiMCExpr.h
@@ -1,9 +1,8 @@
 //===-- LanaiMCExpr.h - Lanai specific MC expression classes ----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/Lanai/MCTargetDesc/LanaiMCTargetDesc.cpp b/lib/Target/Lanai/MCTargetDesc/LanaiMCTargetDesc.cpp
index ddb01cdd2d8fd9e29c9a79758c3cf646a65b75f5..b6478047f4f68b04f993e8e5d7fa2470d230236b 100644
--- a/lib/Target/Lanai/MCTargetDesc/LanaiMCTargetDesc.cpp
+++ b/lib/Target/Lanai/MCTargetDesc/LanaiMCTargetDesc.cpp
@@ -1,9 +1,8 @@
 //===-- LanaiMCTargetDesc.cpp - Lanai Target Descriptions -----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Lanai/MCTargetDesc/LanaiMCTargetDesc.h b/lib/Target/Lanai/MCTargetDesc/LanaiMCTargetDesc.h
index 2d8828ea4fa98dd6d062c58abefaf9df36396255..96ff9cacdca952eddca5523706e4dfe2e6921fa6 100644
--- a/lib/Target/Lanai/MCTargetDesc/LanaiMCTargetDesc.h
+++ b/lib/Target/Lanai/MCTargetDesc/LanaiMCTargetDesc.h
@@ -1,9 +1,8 @@
 //===-- LanaiMCTargetDesc.h - Lanai Target Descriptions ---------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Lanai/TargetInfo/LLVMBuild.txt b/lib/Target/Lanai/TargetInfo/LLVMBuild.txt
index 9922ec36daae1bff9823c99b52700d5e7a36429b..312e09a88695a69598bf66fa4b5ec436575cd087 100644
--- a/lib/Target/Lanai/TargetInfo/LLVMBuild.txt
+++ b/lib/Target/Lanai/TargetInfo/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/Target/Lanai/TargetInfo/LLVMBuild.txt --------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Target/Lanai/TargetInfo/LanaiTargetInfo.cpp b/lib/Target/Lanai/TargetInfo/LanaiTargetInfo.cpp
index ccf47b08fcff54eca57ec3935e8c3127d4ed15ec..65fba887eea39600964241cd689f581a1b410831 100644
--- a/lib/Target/Lanai/TargetInfo/LanaiTargetInfo.cpp
+++ b/lib/Target/Lanai/TargetInfo/LanaiTargetInfo.cpp
@@ -1,9 +1,8 @@
 //===-- LanaiTargetInfo.cpp - Lanai Target Implementation -----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/MSP430/AsmParser/LLVMBuild.txt b/lib/Target/MSP430/AsmParser/LLVMBuild.txt
index 58f67c07db1849e4d377970f8dd2e99e54fd0cbf..fd0e0a1e2867dc584e2e1c754460f0edb4a68e9f 100644
--- a/lib/Target/MSP430/AsmParser/LLVMBuild.txt
+++ b/lib/Target/MSP430/AsmParser/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- lib/Target/MSP430/AsmParser/LLVMBuild.txt ----------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Target/MSP430/AsmParser/MSP430AsmParser.cpp b/lib/Target/MSP430/AsmParser/MSP430AsmParser.cpp
index 3cc6da2c21aaf8d6912853ff3c46ad152c4f9f27..0c2963172406fe48b69a1c9f06df1e99700798b6 100644
--- a/lib/Target/MSP430/AsmParser/MSP430AsmParser.cpp
+++ b/lib/Target/MSP430/AsmParser/MSP430AsmParser.cpp
@@ -1,9 +1,8 @@
 //===- MSP430AsmParser.cpp - Parse MSP430 assembly to MCInst instructions -===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -497,7 +496,11 @@ bool MSP430AsmParser::ParseOperand(OperandVector &Operands) {
         getLexer().Lex(); // Eat '+'
         return false;
       }
-      Operands.push_back(MSP430Operand::CreateIndReg(RegNo, StartLoc, EndLoc));
+      if (Operands.size() > 1) // Emulate @rd in destination position as 0(rd)
+        Operands.push_back(MSP430Operand::CreateMem(RegNo,
+            MCConstantExpr::create(0, getContext()), StartLoc, EndLoc));
+      else
+        Operands.push_back(MSP430Operand::CreateIndReg(RegNo, StartLoc, EndLoc));
       return false;
     }
     case AsmToken::Hash:
diff --git a/lib/Target/MSP430/Disassembler/LLVMBuild.txt b/lib/Target/MSP430/Disassembler/LLVMBuild.txt
index 8af9cd9c2224ed1b52e28325c8e99cc8c8ffb71f..e93f288e787b158a5c7c433bfbefbb32db6c1a3e 100644
--- a/lib/Target/MSP430/Disassembler/LLVMBuild.txt
+++ b/lib/Target/MSP430/Disassembler/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;====- lib/Target/MSP430/Disassembler/LLVMBuild.txt ------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Target/MSP430/Disassembler/MSP430Disassembler.cpp b/lib/Target/MSP430/Disassembler/MSP430Disassembler.cpp
index 2a66b4ed7f2d150fa2df1e1b081715fe1b17f0c8..37e3cb6ebf50ca7c1964dd686f05e6b4a2b26b2e 100644
--- a/lib/Target/MSP430/Disassembler/MSP430Disassembler.cpp
+++ b/lib/Target/MSP430/Disassembler/MSP430Disassembler.cpp
@@ -1,9 +1,8 @@
 //===-- MSP430Disassembler.cpp - Disassembler for MSP430 ------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -249,6 +248,10 @@ DecodeStatus MSP430Disassembler::getInstructionI(MCInst &MI, uint64_t &Size,
   case amSymbolic:
   case amImmediate:
   case amAbsolute:
+    if (Bytes.size() < (Words + 1) * 2) {
+      Size = 2;
+      return DecodeStatus::Fail;
+    }
     Insn |= (uint64_t)support::endian::read16le(Bytes.data() + 2) << 16;
     ++Words;
     break;
@@ -259,6 +262,10 @@ DecodeStatus MSP430Disassembler::getInstructionI(MCInst &MI, uint64_t &Size,
   case amIndexed:
   case amSymbolic:
   case amAbsolute:
+    if (Bytes.size() < (Words + 1) * 2) {
+      Size = 2;
+      return DecodeStatus::Fail;
+    }
     Insn |= (uint64_t)support::endian::read16le(Bytes.data() + Words * 2)
         << (Words * 16);
     ++Words;
@@ -296,6 +303,10 @@ DecodeStatus MSP430Disassembler::getInstructionII(MCInst &MI, uint64_t &Size,
   case amSymbolic:
   case amImmediate:
   case amAbsolute:
+    if (Bytes.size() < (Words + 1) * 2) {
+      Size = 2;
+      return DecodeStatus::Fail;
+    }
     Insn |= (uint64_t)support::endian::read16le(Bytes.data() + 2) << 16;
     ++Words;
     break;
diff --git a/lib/Target/MSP430/InstPrinter/LLVMBuild.txt b/lib/Target/MSP430/InstPrinter/LLVMBuild.txt
index 37b8c2537fb40a9edf21392f7b11c285655bb9f1..a8c9e970a887ba75bb87300bbf1215cc0f2a216b 100644
--- a/lib/Target/MSP430/InstPrinter/LLVMBuild.txt
+++ b/lib/Target/MSP430/InstPrinter/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/Target/MSP430/InstPrinter/LLVMBuild.txt ------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Target/MSP430/InstPrinter/MSP430InstPrinter.cpp b/lib/Target/MSP430/InstPrinter/MSP430InstPrinter.cpp
index 4d62547bc65bedd46218788f61ffb8fbaa04b3b9..2f3c6ed3c17e22d0c675e339dc8a1d42326cc0d4 100644
--- a/lib/Target/MSP430/InstPrinter/MSP430InstPrinter.cpp
+++ b/lib/Target/MSP430/InstPrinter/MSP430InstPrinter.cpp
@@ -1,9 +1,8 @@
 //===-- MSP430InstPrinter.cpp - Convert MSP430 MCInst to assembly syntax --===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/MSP430/InstPrinter/MSP430InstPrinter.h b/lib/Target/MSP430/InstPrinter/MSP430InstPrinter.h
index cd02c4fa645a2a7cd5583c469f017979afce0e41..a2f527807a43e51e01fe3e050d105a0a5abf4121 100644
--- a/lib/Target/MSP430/InstPrinter/MSP430InstPrinter.h
+++ b/lib/Target/MSP430/InstPrinter/MSP430InstPrinter.h
@@ -1,9 +1,8 @@
 //= MSP430InstPrinter.h - Convert MSP430 MCInst to assembly syntax -*- C++ -*-//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/MSP430/LLVMBuild.txt b/lib/Target/MSP430/LLVMBuild.txt
index 0cbd1851777bba24ead74a35771233debfd21fc0..935f42f565a17ed892bf5a3ba0974c2a578283ac 100644
--- a/lib/Target/MSP430/LLVMBuild.txt
+++ b/lib/Target/MSP430/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/Target/MSP430/LLVMBuild.txt ------------------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Target/MSP430/MCTargetDesc/LLVMBuild.txt b/lib/Target/MSP430/MCTargetDesc/LLVMBuild.txt
index a305b2db8683546971e8b0207ab08aa952ecc607..cc12d654a8e89071baca5eccfc1a46ae42b84b74 100644
--- a/lib/Target/MSP430/MCTargetDesc/LLVMBuild.txt
+++ b/lib/Target/MSP430/MCTargetDesc/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/Target/MSP430/MCTargetDesc/LLVMBuild.txt -----------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Target/MSP430/MCTargetDesc/MSP430AsmBackend.cpp b/lib/Target/MSP430/MCTargetDesc/MSP430AsmBackend.cpp
index bd69a9d8d795a2465ccbc2894a7611448162383a..365e5da74de06df0a365cdcacb5404d6388ca573 100644
--- a/lib/Target/MSP430/MCTargetDesc/MSP430AsmBackend.cpp
+++ b/lib/Target/MSP430/MCTargetDesc/MSP430AsmBackend.cpp
@@ -1,9 +1,8 @@
 //===-- MSP430AsmBackend.cpp - MSP430 Assembler Backend -------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/MSP430/MCTargetDesc/MSP430ELFObjectWriter.cpp b/lib/Target/MSP430/MCTargetDesc/MSP430ELFObjectWriter.cpp
index e47db2400a05345944bb9f6c97aaca4037a318d2..38b7da32c24690725b847550315d3ed2792de666 100644
--- a/lib/Target/MSP430/MCTargetDesc/MSP430ELFObjectWriter.cpp
+++ b/lib/Target/MSP430/MCTargetDesc/MSP430ELFObjectWriter.cpp
@@ -1,9 +1,8 @@
 //===-- MSP430ELFObjectWriter.cpp - MSP430 ELF Writer ---------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/MSP430/MCTargetDesc/MSP430ELFStreamer.cpp b/lib/Target/MSP430/MCTargetDesc/MSP430ELFStreamer.cpp
index 9449cb2780249bcc7f63093db27ac1c3b56c8209..4e054f85ccc39b25011ac28d39dba602dbabdd02 100644
--- a/lib/Target/MSP430/MCTargetDesc/MSP430ELFStreamer.cpp
+++ b/lib/Target/MSP430/MCTargetDesc/MSP430ELFStreamer.cpp
@@ -1,9 +1,8 @@
 //===-- MSP430ELFStreamer.cpp - MSP430 ELF Target Streamer Methods --------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/MSP430/MCTargetDesc/MSP430FixupKinds.h b/lib/Target/MSP430/MCTargetDesc/MSP430FixupKinds.h
index 1eb6a27594235abeacf838876bc21bf0c5595e0d..68e41b0fb87476e55b9b595be0327e7caeade95a 100644
--- a/lib/Target/MSP430/MCTargetDesc/MSP430FixupKinds.h
+++ b/lib/Target/MSP430/MCTargetDesc/MSP430FixupKinds.h
@@ -1,9 +1,8 @@
 //===-- MSP430FixupKinds.h - MSP430 Specific Fixup Entries ------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/MSP430/MCTargetDesc/MSP430MCAsmInfo.cpp b/lib/Target/MSP430/MCTargetDesc/MSP430MCAsmInfo.cpp
index 82e6731ecd7829c862956936358e5a422f53c572..db5a49dd22a7250acf76eeade694a3f732ec5b3a 100644
--- a/lib/Target/MSP430/MCTargetDesc/MSP430MCAsmInfo.cpp
+++ b/lib/Target/MSP430/MCTargetDesc/MSP430MCAsmInfo.cpp
@@ -1,9 +1,8 @@
 //===-- MSP430MCAsmInfo.cpp - MSP430 asm properties -----------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -20,7 +19,9 @@ MSP430MCAsmInfo::MSP430MCAsmInfo(const Triple &TT) {
   CodePointerSize = CalleeSaveStackSlotSize = 2;
 
   CommentString = ";";
+  SeparatorString = "{";
 
   AlignmentIsInBytes = false;
   UsesELFSectionDirectiveForBSS = true;
+  UseIntegratedAssembler = true;
 }
diff --git a/lib/Target/MSP430/MCTargetDesc/MSP430MCAsmInfo.h b/lib/Target/MSP430/MCTargetDesc/MSP430MCAsmInfo.h
index de486ec4b7bd1803dfd8194897cd9784e067509b..93979df037e648a75b07a1428ab098bc7d0a66db 100644
--- a/lib/Target/MSP430/MCTargetDesc/MSP430MCAsmInfo.h
+++ b/lib/Target/MSP430/MCTargetDesc/MSP430MCAsmInfo.h
@@ -1,9 +1,8 @@
 //===-- MSP430MCAsmInfo.h - MSP430 asm properties --------------*- C++ -*--===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/MSP430/MCTargetDesc/MSP430MCCodeEmitter.cpp b/lib/Target/MSP430/MCTargetDesc/MSP430MCCodeEmitter.cpp
index adf2384f6e99515bdae8300b11b4516172e79bc1..cf57e87a073dffc4bc85584be914ce8e034d11cc 100644
--- a/lib/Target/MSP430/MCTargetDesc/MSP430MCCodeEmitter.cpp
+++ b/lib/Target/MSP430/MCTargetDesc/MSP430MCCodeEmitter.cpp
@@ -1,9 +1,8 @@
 //===-- MSP430MCCodeEmitter.cpp - Convert MSP430 code to machine code -----===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -128,7 +127,7 @@ unsigned MSP430MCCodeEmitter::getMemOpValue(const MCInst &MI, unsigned Op,
   const MCOperand &MO2 = MI.getOperand(Op + 1);
   if (MO2.isImm()) {
     Offset += 2;
-    return (MO2.getImm() << 4) | Reg;
+    return ((unsigned)MO2.getImm() << 4) | Reg;
   }
 
   assert(MO2.isExpr() && "Expr operand expected");
diff --git a/lib/Target/MSP430/MCTargetDesc/MSP430MCTargetDesc.cpp b/lib/Target/MSP430/MCTargetDesc/MSP430MCTargetDesc.cpp
index b21145d3904a33e5981058b907bc476e3a74b458..7bb966f5889e48f9bc9710657a7fb178ccb027cc 100644
--- a/lib/Target/MSP430/MCTargetDesc/MSP430MCTargetDesc.cpp
+++ b/lib/Target/MSP430/MCTargetDesc/MSP430MCTargetDesc.cpp
@@ -1,9 +1,8 @@
 //===-- MSP430MCTargetDesc.cpp - MSP430 Target Descriptions ---------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/MSP430/MCTargetDesc/MSP430MCTargetDesc.h b/lib/Target/MSP430/MCTargetDesc/MSP430MCTargetDesc.h
index e484c79c9ee91cc82b26fdc275c8ce02f8ae0f68..d5af8075c177642e29b8b1a509bc906d136106db 100644
--- a/lib/Target/MSP430/MCTargetDesc/MSP430MCTargetDesc.h
+++ b/lib/Target/MSP430/MCTargetDesc/MSP430MCTargetDesc.h
@@ -1,9 +1,8 @@
 //===-- MSP430MCTargetDesc.h - MSP430 Target Descriptions -------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/MSP430/MSP430.h b/lib/Target/MSP430/MSP430.h
index 7a5314a10844b7dba692cff79de1f6c0087600e7..67f35b8034d99b094a136d62872cf39ff56a3e13 100644
--- a/lib/Target/MSP430/MSP430.h
+++ b/lib/Target/MSP430/MSP430.h
@@ -1,9 +1,8 @@
 //==-- MSP430.h - Top-level interface for MSP430 representation --*- C++ -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/MSP430/MSP430.td b/lib/Target/MSP430/MSP430.td
index 8fa99dc13dd5c7fb5e963bc0e4ad436c8d275ab2..38aa30fcf4dd1ee9cda403efe1eb00055c00d11f 100644
--- a/lib/Target/MSP430/MSP430.td
+++ b/lib/Target/MSP430/MSP430.td
@@ -1,9 +1,8 @@
 //===-- MSP430.td - Describe the MSP430 Target Machine -----*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source 
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 // This is the top level entry point for the MSP430 target.
diff --git a/lib/Target/MSP430/MSP430AsmPrinter.cpp b/lib/Target/MSP430/MSP430AsmPrinter.cpp
index f39c21fc8aa2b0c158530a590e57222ce1b974f8..4a3f58d67352abe84758fd4efca147a74602c4eb 100644
--- a/lib/Target/MSP430/MSP430AsmPrinter.cpp
+++ b/lib/Target/MSP430/MSP430AsmPrinter.cpp
@@ -1,9 +1,8 @@
 //===-- MSP430AsmPrinter.cpp - MSP430 LLVM assembly writer ----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -17,6 +16,7 @@
 #include "MSP430InstrInfo.h"
 #include "MSP430MCInstLower.h"
 #include "MSP430TargetMachine.h"
+#include "llvm/BinaryFormat/ELF.h"
 #include "llvm/CodeGen/AsmPrinter.h"
 #include "llvm/CodeGen/MachineConstantPool.h"
 #include "llvm/CodeGen/MachineFunctionPass.h"
@@ -28,6 +28,7 @@
 #include "llvm/IR/Module.h"
 #include "llvm/MC/MCAsmInfo.h"
 #include "llvm/MC/MCInst.h"
+#include "llvm/MC/MCSectionELF.h"
 #include "llvm/MC/MCStreamer.h"
 #include "llvm/MC/MCSymbol.h"
 #include "llvm/Support/TargetRegistry.h"
@@ -44,6 +45,8 @@ namespace {
 
     StringRef getPassName() const override { return "MSP430 Assembly Printer"; }
 
+    bool runOnMachineFunction(MachineFunction &MF) override;
+
     void printOperand(const MachineInstr *MI, int OpNum,
                       raw_ostream &O, const char* Modifier = nullptr);
     void printSrcMemOperand(const MachineInstr *MI, int OpNum,
@@ -55,6 +58,8 @@ namespace {
                                unsigned OpNo, unsigned AsmVariant,
                                const char *ExtraCode, raw_ostream &O) override;
     void EmitInstruction(const MachineInstr *MI) override;
+
+    void EmitInterruptVectorSection(MachineFunction &ISR);
   };
 } // end of anonymous namespace
 
@@ -108,12 +113,12 @@ void MSP430AsmPrinter::printSrcMemOperand(const MachineInstr *MI, int OpNum,
   // Print displacement first
 
   // Imm here is in fact global address - print extra modifier.
-  if (Disp.isImm() && !Base.getReg())
+  if (Disp.isImm() && Base.getReg() == MSP430::SR)
     O << '&';
   printOperand(MI, OpNum+1, O, "nohash");
 
   // Print register base field
-  if (Base.getReg()) {
+  if (Base.getReg() != MSP430::SR && Base.getReg() != MSP430::PC) {
     O << '(';
     printOperand(MI, OpNum, O);
     O << ')';
@@ -153,6 +158,32 @@ void MSP430AsmPrinter::EmitInstruction(const MachineInstr *MI) {
   EmitToStreamer(*OutStreamer, TmpInst);
 }
 
+void MSP430AsmPrinter::EmitInterruptVectorSection(MachineFunction &ISR) {
+  MCSection *Cur = OutStreamer->getCurrentSectionOnly();
+  const auto *F = &ISR.getFunction();
+  assert(F->hasFnAttribute("interrupt") &&
+         "Functions with MSP430_INTR CC should have 'interrupt' attribute");
+  StringRef IVIdx = F->getFnAttribute("interrupt").getValueAsString();
+  MCSection *IV = OutStreamer->getContext().getELFSection(
+    "__interrupt_vector_" + IVIdx,
+    ELF::SHT_PROGBITS, ELF::SHF_ALLOC | ELF::SHF_EXECINSTR);
+  OutStreamer->SwitchSection(IV);
+
+  const MCSymbol *FunctionSymbol = getSymbol(F);
+  OutStreamer->EmitSymbolValue(FunctionSymbol, TM.getProgramPointerSize());
+  OutStreamer->SwitchSection(Cur);
+}
+
+bool MSP430AsmPrinter::runOnMachineFunction(MachineFunction &MF) {
+  // Emit separate section for an interrupt vector if ISR
+  if (MF.getFunction().getCallingConv() == CallingConv::MSP430_INTR)
+    EmitInterruptVectorSection(MF);
+
+  SetupMachineFunction(MF);
+  EmitFunctionBody();
+  return false;
+}
+
 // Force static initialization.
 extern "C" void LLVMInitializeMSP430AsmPrinter() {
   RegisterAsmPrinter<MSP430AsmPrinter> X(getTheMSP430Target());
diff --git a/lib/Target/MSP430/MSP430BranchSelector.cpp b/lib/Target/MSP430/MSP430BranchSelector.cpp
index 2b3495405545114060f868ac8c0bb97b9728620c..45e7c26e4d306c7308b886b1f01e54ed1ece97dc 100644
--- a/lib/Target/MSP430/MSP430BranchSelector.cpp
+++ b/lib/Target/MSP430/MSP430BranchSelector.cpp
@@ -1,9 +1,8 @@
 //===-- MSP430BranchSelector.cpp - Emit long conditional branches ---------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/MSP430/MSP430CallingConv.td b/lib/Target/MSP430/MSP430CallingConv.td
index 0434f8abfbf46679cd004bb39cf293ab8165efc8..49191fa5dd5f862647c3d029f14511773677c228 100644
--- a/lib/Target/MSP430/MSP430CallingConv.td
+++ b/lib/Target/MSP430/MSP430CallingConv.td
@@ -1,9 +1,8 @@
 //==- MSP430CallingConv.td - Calling Conventions for MSP430 -*- tablegen -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 // This describes the calling conventions for MSP430 architecture.
diff --git a/lib/Target/MSP430/MSP430FrameLowering.cpp b/lib/Target/MSP430/MSP430FrameLowering.cpp
index 2421f09fbf59aba2a7e5b151d217aad5eebf63bc..de60ad9bd7e6f070a2be784edb588b96ad3bdbf5 100644
--- a/lib/Target/MSP430/MSP430FrameLowering.cpp
+++ b/lib/Target/MSP430/MSP430FrameLowering.cpp
@@ -1,9 +1,8 @@
 //===-- MSP430FrameLowering.cpp - MSP430 Frame Information ----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/MSP430/MSP430FrameLowering.h b/lib/Target/MSP430/MSP430FrameLowering.h
index 8807101f37cab6004a200700e2c6cbb3d8b33c9e..33ce3c70a2a3b030086265f363ae205ce787c719 100644
--- a/lib/Target/MSP430/MSP430FrameLowering.h
+++ b/lib/Target/MSP430/MSP430FrameLowering.h
@@ -1,9 +1,8 @@
 //==- MSP430FrameLowering.h - Define frame lowering for MSP430 --*- C++ -*--==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/MSP430/MSP430ISelDAGToDAG.cpp b/lib/Target/MSP430/MSP430ISelDAGToDAG.cpp
index 7a1998ad355d1d8615e3bf8c167b4bf0cee58922..23449585505e9c0d1e5d799ead7c27d857518325 100644
--- a/lib/Target/MSP430/MSP430ISelDAGToDAG.cpp
+++ b/lib/Target/MSP430/MSP430ISelDAGToDAG.cpp
@@ -1,9 +1,8 @@
 //===-- MSP430ISelDAGToDAG.cpp - A dag to dag inst selector for MSP430 ----===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/MSP430/MSP430ISelLowering.cpp b/lib/Target/MSP430/MSP430ISelLowering.cpp
index 73c8793aa01c08d337e8b4e7219aecc5fac403a1..fedfb857bd0f8117fd277cfae9dc1425ecf0c277 100644
--- a/lib/Target/MSP430/MSP430ISelLowering.cpp
+++ b/lib/Target/MSP430/MSP430ISelLowering.cpp
@@ -1,9 +1,8 @@
 //===-- MSP430ISelLowering.cpp - MSP430 DAG Lowering Implementation  ------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -95,6 +94,8 @@ MSP430TargetLowering::MSP430TargetLowering(const TargetMachine &TM,
   setOperationAction(ISD::SIGN_EXTEND,      MVT::i16,   Custom);
   setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i8, Expand);
   setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i16, Expand);
+  setOperationAction(ISD::STACKSAVE,        MVT::Other, Expand);
+  setOperationAction(ISD::STACKRESTORE,     MVT::Other, Expand);
 
   setOperationAction(ISD::CTTZ,             MVT::i8,    Expand);
   setOperationAction(ISD::CTTZ,             MVT::i16,   Expand);
@@ -952,15 +953,26 @@ SDValue MSP430TargetLowering::LowerShifts(SDValue Op,
   // Expand the stuff into sequence of shifts.
   SDValue Victim = N->getOperand(0);
 
-  if ((Opc == ISD::SRA || Opc == ISD::SRL) && ShiftAmount >= 8) {
-    // foo >> (8 + N) => sxt(swpb(foo)) >> N
+  if (ShiftAmount >= 8) {
     assert(VT == MVT::i16 && "Can not shift i8 by 8 and more");
-    Victim = DAG.getNode(ISD::BSWAP, dl, VT, Victim);
-    if (Opc == ISD::SRA)
-      Victim = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, VT, Victim,
-                           DAG.getValueType(MVT::i8));
-    else
+    switch(Opc) {
+    default:
+      llvm_unreachable("Unknown shift");
+    case ISD::SHL:
+      // foo << (8 + N) => swpb(zext(foo)) << N
       Victim = DAG.getZeroExtendInReg(Victim, dl, MVT::i8);
+      Victim = DAG.getNode(ISD::BSWAP, dl, VT, Victim);
+      break;
+    case ISD::SRA:
+    case ISD::SRL:
+      // foo >> (8 + N) => sxt(swpb(foo)) >> N
+      Victim = DAG.getNode(ISD::BSWAP, dl, VT, Victim);
+      Victim = (Opc == ISD::SRA)
+                   ? DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, VT, Victim,
+                                 DAG.getValueType(MVT::i8))
+                   : DAG.getZeroExtendInReg(Victim, dl, MVT::i8);
+      break;
+    }
     ShiftAmount -= 8;
   }
 
diff --git a/lib/Target/MSP430/MSP430ISelLowering.h b/lib/Target/MSP430/MSP430ISelLowering.h
index 731bc1406711f307ced5406cd5e1c0b7b7481453..ee6b6316d7a91e596f9a167d07c9dbed61dbd79d 100644
--- a/lib/Target/MSP430/MSP430ISelLowering.h
+++ b/lib/Target/MSP430/MSP430ISelLowering.h
@@ -1,9 +1,8 @@
 //===-- MSP430ISelLowering.h - MSP430 DAG Lowering Interface ----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/MSP430/MSP430InstrFormats.td b/lib/Target/MSP430/MSP430InstrFormats.td
index e2e4503db20c48c35b1a0d43697e5a7b8c4895a8..36f40d6fc89dd4e940fbe6ee7f97ba6e1c432aaf 100644
--- a/lib/Target/MSP430/MSP430InstrFormats.td
+++ b/lib/Target/MSP430/MSP430InstrFormats.td
@@ -1,9 +1,8 @@
 //===-- MSP430InstrFormats.td - MSP430 Instruction Formats -*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source 
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/MSP430/MSP430InstrInfo.cpp b/lib/Target/MSP430/MSP430InstrInfo.cpp
index c136933a51bcfd6395486e4ba26368fa03fcb54f..5c3a3fc692664b864ab112575d4885ab7c48a5ad 100644
--- a/lib/Target/MSP430/MSP430InstrInfo.cpp
+++ b/lib/Target/MSP430/MSP430InstrInfo.cpp
@@ -1,9 +1,8 @@
 //===-- MSP430InstrInfo.cpp - MSP430 Instruction Information --------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -308,7 +307,8 @@ unsigned MSP430InstrInfo::getInstSizeInBytes(const MachineInstr &MI) const {
   case TargetOpcode::KILL:
   case TargetOpcode::DBG_VALUE:
     return 0;
-  case TargetOpcode::INLINEASM: {
+  case TargetOpcode::INLINEASM:
+  case TargetOpcode::INLINEASM_BR: {
     const MachineFunction *MF = MI.getParent()->getParent();
     const TargetInstrInfo &TII = *MF->getSubtarget().getInstrInfo();
     return TII.getInlineAsmLength(MI.getOperand(0).getSymbolName(),
diff --git a/lib/Target/MSP430/MSP430InstrInfo.h b/lib/Target/MSP430/MSP430InstrInfo.h
index fee3bea9b8d692f5a377ce37cdaf396cea83c910..13c50ad23adcfc0ea28599c883d04fda4acf65af 100644
--- a/lib/Target/MSP430/MSP430InstrInfo.h
+++ b/lib/Target/MSP430/MSP430InstrInfo.h
@@ -1,9 +1,8 @@
 //===-- MSP430InstrInfo.h - MSP430 Instruction Information ------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/MSP430/MSP430InstrInfo.td b/lib/Target/MSP430/MSP430InstrInfo.td
index 3ed17374a2d87e70e674853c3dd920258b8c38aa..aaca3504822d7c076ad6e72f1874c4f6a3f7973c 100644
--- a/lib/Target/MSP430/MSP430InstrInfo.td
+++ b/lib/Target/MSP430/MSP430InstrInfo.td
@@ -1,9 +1,8 @@
 //===-- MSP430InstrInfo.td - MSP430 Instruction defs -------*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source 
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -226,7 +225,6 @@ let isCodeGenOnly = 1, usesCustomInserter = 1 in {
 //  Control Flow Instructions...
 //
 
-// FIXME: Provide proper encoding!
 let isReturn = 1, isTerminator = 1, isBarrier = 1 in {
   def RET  : IForm16<0b0100, DstReg, SrcPostInc, 2,
                      (outs), (ins), "ret",  [(MSP430retflag)]> {
@@ -292,6 +290,8 @@ let isCall = 1,
   def CALLm     : II16m<0b101,
                         (outs), (ins memsrc:$src),
                         "call\t$src", [(MSP430call (load addr:$src))]>;
+  def CALLn     : II16n<0b101, (outs), (ins indreg:$rs), "call\t$rs", []>;
+  def CALLp     : II16p<0b101, (outs), (ins postreg:$rs), "call\t$rs", []>;
 }
 
 //===----------------------------------------------------------------------===//
@@ -315,7 +315,6 @@ def PUSH16i : II16i<0b100, (outs), (ins i16imm:$imm), "push\t$imm", []>;
 //===----------------------------------------------------------------------===//
 // Move Instructions
 
-// FIXME: Provide proper encoding!
 let hasSideEffects = 0 in {
 def MOV8rr  : I8rr<0b0100,
                    (outs GR8:$rd), (ins GR8:$rs),
@@ -327,7 +326,6 @@ def MOV16rr : I16rr<0b0100,
                     []>;
 }
 
-// FIXME: Provide proper encoding!
 let isReMaterializable = 1, isAsCheapAsAMove = 1 in {
 def MOV8rc : I8rc<0b0100,
                    (outs GR8:$rd), (ins cg8imm:$imm),
@@ -437,6 +435,11 @@ def MOV16mm : I16mm<0b0100,
                     "mov\t{$src, $dst}",
                     [(store (i16 (load addr:$src)), addr:$dst)]>;
 
+def MOV8mn  : I8mn<0b0100, (outs), (ins memdst:$dst, indreg:$rs),
+                   "mov.b\t{$rs, $dst}", []>;
+def MOV16mn : I16mn<0b0100, (outs), (ins memdst:$dst, indreg:$rs),
+                    "mov\t{$rs, $dst}", []>;
+
 //===----------------------------------------------------------------------===//
 // Arithmetic Instructions
 
@@ -619,11 +622,11 @@ def SETZ : InstAlias<"setz", (BIS16rc SR, 2)>;
 def : Pat<(MSP430rla GR8:$dst),  (ADD8rr  $dst, $dst)>;
 def : Pat<(MSP430rla GR16:$dst), (ADD16rr $dst, $dst)>;
 
+// Format-II (Single Operand) Instruction
+// Register mode
 let Constraints = "$rs = $rd" in {
 
 let Defs = [SR] in {
-
-// FIXME: memory variant!
 def RRA8r :   II8r<0b010,
                    (outs GR8:$rd), (ins GR8:$rs),
                    "rra.b\t$rd",
@@ -646,9 +649,8 @@ def RRC16r : II16r<0b000,
                    "rrc\t$rd",
                    [(set GR16:$rd, (MSP430rrc GR16:$rs)),
                     (implicit SR)]>;
-}
+} // Uses = [SR]
 
-// FIXME: Memory sext's ?
 def SEXT16r : II16r<0b011,
                     (outs GR16:$rd), (ins GR16:$rs),
                     "sxt\t$rd",
@@ -663,7 +665,6 @@ def ZEXT16r : I8rr<0b0100,
                    "mov.b\t{$rs, $rd}",
                    [(set GR16:$rd, (zext (trunc GR16:$rs)))]>;
 
-// FIXME: Memory bitswaps?
 def SWPB16r : II16r<0b001,
                     (outs GR16:$rd), (ins GR16:$rs),
                     "swpb\t$rd",
@@ -671,6 +672,61 @@ def SWPB16r : II16r<0b001,
 
 } // Constraints = "$src = $dst"
 
+// Indexed, indirect register and indirect autoincrement modes
+let Defs = [SR] in {
+def RRA8m  : II8m<0b010,
+                   (outs), (ins memsrc:$src),
+                   "rra.b\t$src",
+                   [(store (MSP430rra (i8 (load addr:$src))), addr:$src),
+                    (implicit SR)]>;
+def RRA16m : II16m<0b010,
+                   (outs), (ins memsrc:$src),
+                   "rra\t$src",
+                   [(store (MSP430rra (i16 (load addr:$src))), addr:$src),
+                    (implicit SR)]>;
+
+def RRA8n  : II8n<0b010, (outs), (ins indreg:$rs), "rra.b\t$rs", []>;
+def RRA16n : II16n<0b010, (outs), (ins indreg:$rs), "rra\t$rs", []>;
+def RRA8p  : II8p<0b010, (outs), (ins postreg:$rs), "rra.b\t$rs", []>;
+def RRA16p : II16p<0b010, (outs), (ins postreg:$rs), "rra\t$rs", []>;
+
+let Uses = [SR] in {
+def RRC8m  : II8m<0b000,
+                   (outs), (ins memsrc:$src),
+                   "rrc.b\t$src",
+                   [(store (MSP430rrc (i8 (load addr:$src))), addr:$src),
+                    (implicit SR)]>;
+def RRC16m : II16m<0b000,
+                   (outs), (ins memsrc:$src),
+                   "rrc\t$src",
+                   [(store (MSP430rrc (i16 (load addr:$src))), addr:$src),
+                    (implicit SR)]>;
+
+def RRC8n  : II8n<0b000, (outs), (ins indreg:$rs), "rrc.b\t$rs", []>;
+def RRC16n : II16n<0b000, (outs), (ins indreg:$rs), "rrc\t$rs", []>;
+def RRC8p  : II8p<0b000, (outs), (ins postreg:$rs), "rrc.b\t$rs", []>;
+def RRC16p : II16p<0b000, (outs), (ins postreg:$rs), "rrc\t$rs", []>;
+
+} // Uses = [SR]
+
+def SEXT16m : II16m<0b011,
+                    (outs), (ins memsrc:$src),
+                    "sxt\t$src",
+                    [(store (sext_inreg (extloadi16i8 addr:$src), i8),
+                             addr:$src),
+                     (implicit SR)]>;
+def SEXT16n : II16n<0b011, (outs), (ins indreg:$rs), "sxt\t$rs", []>;
+def SEXT16p : II16p<0b011, (outs), (ins postreg:$rs), "sxt\t$rs", []>;
+
+} // Defs = [SR]
+
+def SWPB16m : II16m<0b001,
+                   (outs), (ins memsrc:$src),
+                   "swpb\t$src",
+                   [(store (bswap (i16 (load addr:$src))), addr:$src)]>;
+def SWPB16n : II16n<0b001, (outs), (ins indreg:$rs), "swpb\t$rs", []>;
+def SWPB16p : II16p<0b001, (outs), (ins postreg:$rs), "swpb\t$rs", []>;
+
 // Integer comparisons
 let Defs = [SR] in {
 def CMP8rr  : I8rr<0b1001,
@@ -733,6 +789,16 @@ def CMP16rm : I16rm<0b1001,
                     [(MSP430cmp GR16:$rd, (load addr:$src)),
                      (implicit SR)]>;
 
+def CMP8rn  : I8rn<0b1001,
+                   (outs), (ins GR8:$rd, indreg:$rs), "cmp.b\t$rs, $rd", []>;
+def CMP16rn : I16rn<0b1001,
+                    (outs), (ins GR16:$rd, indreg:$rs), "cmp\t$rs, $rd", []>;
+
+def CMP8rp  : I8rp<0b1001,
+                   (outs), (ins GR8:$rd, postreg:$rs), "cmp.b\t$rs, $rd", []>;
+def CMP16rp : I16rp<0b1001,
+                    (outs), (ins GR16:$rd, postreg:$rs), "cmp\t$rs, $rd", []>;
+
 def CMP8mr  : I8mr<0b1001,
                    (outs), (ins memsrc:$dst, GR8:$rs),
                    "cmp.b\t$rs, $dst",
@@ -743,6 +809,25 @@ def CMP16mr : I16mr<0b1001,
                     "cmp\t$rs, $dst",
                     [(MSP430cmp (load addr:$dst), GR16:$rs), 
                      (implicit SR)]>;
+def CMP8mm  : I8mm<0b1001,
+                   (outs), (ins memdst:$dst, memsrc:$src),
+                   "cmp.b\t$src, $dst",
+                   [(MSP430cmp (load addr:$dst), (i8 (load addr:$src))),
+                    (implicit SR)]>;
+def CMP16mm : I16mm<0b1001, (outs), (ins memdst:$dst, memsrc:$src),
+                    "cmp\t$src, $dst",
+                    [(MSP430cmp (load addr:$dst), (i16 (load addr:$src))),
+                     (implicit SR)]>;
+
+def CMP8mn  : I8mn<0b1001, (outs), (ins memsrc:$dst, indreg:$rs),
+                   "cmp.b\t$rs, $dst", []>;
+def CMP16mn : I16mn<0b1001, (outs), (ins memsrc:$dst, indreg:$rs),
+                    "cmp\t$rs, $dst", []>;
+
+def CMP8mp  : I8mp<0b1001, (outs), (ins memsrc:$dst, postreg:$rs),
+                   "cmp.b\t$rs, $dst", []>;
+def CMP16mp : I16mp<0b1001, (outs), (ins memsrc:$dst, postreg:$rs),
+                    "cmp\t$rs, $dst", []>;
 
 // BIT TESTS, just sets condition codes
 // Note that the C condition is set differently than when using CMP.
@@ -791,6 +876,16 @@ def BIT16rm : I16rm<0b1011,
                     [(MSP430cmp (and_su GR16:$rd,  (load addr:$src)), 0),
                      (implicit SR)]>;
 
+def BIT8rn  : I8rn<0b1011, (outs), (ins GR8:$rd, indreg:$rs),
+                   "bit.b\t$rs, $rd", []>;
+def BIT16rn : I16rn<0b1011, (outs), (ins GR16:$rd, indreg:$rs),
+                    "bit\t$rs, $rd", []>;
+
+def BIT8rp  : I8rp<0b1011, (outs), (ins GR8:$rd, postreg:$rs),
+                   "bit.b\t$rs, $rd", []>;
+def BIT16rp : I16rp<0b1011, (outs), (ins GR16:$rd, postreg:$rs),
+                    "bit\t$rs, $rd", []>;
+
 def BIT8mr  : I8mr<0b1011,
                   (outs), (ins memsrc:$dst, GR8:$rs),
                   "bit.b\t$rs, $dst",
@@ -808,7 +903,7 @@ def BIT8mc  : I8mc<0b1011,
                    [(MSP430cmp (and_su (load addr:$dst), (i8 cg8imm:$imm)), 0),
                     (implicit SR)]>;
 def BIT16mc : I16mc<0b1011,
-                    (outs), (ins memsrc:$dst, i16imm:$imm),
+                    (outs), (ins memdst:$dst, cg16imm:$imm),
                     "bit\t$imm, $dst",
                     [(MSP430cmp (and_su (load addr:$dst), (i16 cg16imm:$imm)), 0),
                      (implicit SR)]>;
@@ -838,6 +933,16 @@ def BIT16mm : I16mm<0b1011,
                                         (load addr:$src)),
                                  0),
                      (implicit SR)]>;
+def BIT8mn  : I8mn<0b1011, (outs), (ins memsrc:$dst, indreg:$rs),
+                   "bit.b\t$rs, $dst", []>;
+def BIT16mn : I16mn<0b1011, (outs), (ins memsrc:$dst, indreg:$rs),
+                    "bit\t$rs, $dst", []>;
+
+def BIT8mp  : I8mp<0b1011, (outs), (ins memsrc:$dst, postreg:$rs),
+                   "bit.b\t$rs, $dst", []>;
+def BIT16mp : I16mp<0b1011, (outs), (ins memsrc:$dst, postreg:$rs),
+                    "bit\t$rs, $dst", []>;
+
 } // Defs = [SR]
 
 def TST8r   : InstAlias<"tst.b\t$dst",  (CMP8rc    GR8:$dst,     0)>;
diff --git a/lib/Target/MSP430/MSP430MCInstLower.cpp b/lib/Target/MSP430/MSP430MCInstLower.cpp
index 860c0006f782f534e011cbb469d9a495d48e11d9..1e57f33386e6b5a41308ae6e4cd738ba1e5d4eed 100644
--- a/lib/Target/MSP430/MSP430MCInstLower.cpp
+++ b/lib/Target/MSP430/MSP430MCInstLower.cpp
@@ -1,9 +1,8 @@
 //===-- MSP430MCInstLower.cpp - Convert MSP430 MachineInstr to an MCInst --===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/MSP430/MSP430MCInstLower.h b/lib/Target/MSP430/MSP430MCInstLower.h
index ebd639744bcc9e461bd48e8cc630d990469294ec..910ad4bb12d5493a788f429c9147608af3911861 100644
--- a/lib/Target/MSP430/MSP430MCInstLower.h
+++ b/lib/Target/MSP430/MSP430MCInstLower.h
@@ -1,9 +1,8 @@
 //===-- MSP430MCInstLower.h - Lower MachineInstr to MCInst ------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/MSP430/MSP430MachineFunctionInfo.cpp b/lib/Target/MSP430/MSP430MachineFunctionInfo.cpp
index b442fc03b257c600966d51c9c934b8fa50b64e13..1d3a6d118bd60417b371bbedd6b267c97883882a 100644
--- a/lib/Target/MSP430/MSP430MachineFunctionInfo.cpp
+++ b/lib/Target/MSP430/MSP430MachineFunctionInfo.cpp
@@ -1,9 +1,8 @@
 //===-- MSP430MachineFunctionInfo.cpp - MSP430 machine function info ------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/MSP430/MSP430MachineFunctionInfo.h b/lib/Target/MSP430/MSP430MachineFunctionInfo.h
index fcaa8a1d6c728ee72076574c27bfae7843720094..2b2c8967a749187603c77035ac238a63f05f5046 100644
--- a/lib/Target/MSP430/MSP430MachineFunctionInfo.h
+++ b/lib/Target/MSP430/MSP430MachineFunctionInfo.h
@@ -1,9 +1,8 @@
 //=== MSP430MachineFunctionInfo.h - MSP430 machine function info -*- C++ -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/MSP430/MSP430RegisterInfo.cpp b/lib/Target/MSP430/MSP430RegisterInfo.cpp
index 54e53e19eb548dc6e3f8fb99fa1e644dea9d6381..8815093994b9c0f6939b7386a59b72e61cc168e0 100644
--- a/lib/Target/MSP430/MSP430RegisterInfo.cpp
+++ b/lib/Target/MSP430/MSP430RegisterInfo.cpp
@@ -1,9 +1,8 @@
 //===-- MSP430RegisterInfo.cpp - MSP430 Register Information --------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/MSP430/MSP430RegisterInfo.h b/lib/Target/MSP430/MSP430RegisterInfo.h
index 47a5e147953ec2a6edde384e0e1ba7db6b11c50f..2a1c82fb38dcbfc50201cae73b323525d130c5ec 100644
--- a/lib/Target/MSP430/MSP430RegisterInfo.h
+++ b/lib/Target/MSP430/MSP430RegisterInfo.h
@@ -1,9 +1,8 @@
 //===-- MSP430RegisterInfo.h - MSP430 Register Information Impl -*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/MSP430/MSP430RegisterInfo.td b/lib/Target/MSP430/MSP430RegisterInfo.td
index 1e86bdf34a0b5894c41daf9620e29b138737f75b..11003dba383f410abe5ea3d186901c2503557804 100644
--- a/lib/Target/MSP430/MSP430RegisterInfo.td
+++ b/lib/Target/MSP430/MSP430RegisterInfo.td
@@ -1,9 +1,8 @@
 //===-- MSP430RegisterInfo.td - MSP430 Register defs -------*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source 
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/MSP430/MSP430Subtarget.cpp b/lib/Target/MSP430/MSP430Subtarget.cpp
index 776a9dcb11d4f990f04abea28d1b72c4783bcc19..20168773cd533c06d1c5b0c4105ec7900801d62e 100644
--- a/lib/Target/MSP430/MSP430Subtarget.cpp
+++ b/lib/Target/MSP430/MSP430Subtarget.cpp
@@ -1,9 +1,8 @@
 //===-- MSP430Subtarget.cpp - MSP430 Subtarget Information ----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/MSP430/MSP430Subtarget.h b/lib/Target/MSP430/MSP430Subtarget.h
index 01a428056377249a90598e3f53728b03f5f530dd..ab2b71e3bb1ab1524b1495f6a171c4880818a202 100644
--- a/lib/Target/MSP430/MSP430Subtarget.h
+++ b/lib/Target/MSP430/MSP430Subtarget.h
@@ -1,9 +1,8 @@
 //===-- MSP430Subtarget.h - Define Subtarget for the MSP430 ----*- C++ -*--===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/MSP430/MSP430TargetMachine.cpp b/lib/Target/MSP430/MSP430TargetMachine.cpp
index 9f6ebba75ec66d7f032eaeddee1c9514b20e243b..757dab2f59eff3374b9b424abcca22fa3b3886ef 100644
--- a/lib/Target/MSP430/MSP430TargetMachine.cpp
+++ b/lib/Target/MSP430/MSP430TargetMachine.cpp
@@ -1,9 +1,8 @@
 //===-- MSP430TargetMachine.cpp - Define TargetMachine for MSP430 ---------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/MSP430/MSP430TargetMachine.h b/lib/Target/MSP430/MSP430TargetMachine.h
index 4935b80cfdd99d5a9a596fbffa1c1afe07e730c9..96fbc3ba03779b04803586a7acec84303472a5d9 100644
--- a/lib/Target/MSP430/MSP430TargetMachine.h
+++ b/lib/Target/MSP430/MSP430TargetMachine.h
@@ -1,9 +1,8 @@
 //===-- MSP430TargetMachine.h - Define TargetMachine for MSP430 -*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/MSP430/TargetInfo/LLVMBuild.txt b/lib/Target/MSP430/TargetInfo/LLVMBuild.txt
index ee41ae466744f06671e42a232d2e674392b3b44b..ef994470860a8922f1c6cb00535ccd177e6b81c4 100644
--- a/lib/Target/MSP430/TargetInfo/LLVMBuild.txt
+++ b/lib/Target/MSP430/TargetInfo/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/Target/MSP430/TargetInfo/LLVMBuild.txt -------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Target/MSP430/TargetInfo/MSP430TargetInfo.cpp b/lib/Target/MSP430/TargetInfo/MSP430TargetInfo.cpp
index dfa21f580cb761109ea53c2e43c24660629a7bb8..045a8b9a50fbbd1821a9e0537d3a8caeddc9f988 100644
--- a/lib/Target/MSP430/TargetInfo/MSP430TargetInfo.cpp
+++ b/lib/Target/MSP430/TargetInfo/MSP430TargetInfo.cpp
@@ -1,9 +1,8 @@
 //===-- MSP430TargetInfo.cpp - MSP430 Target Implementation ---------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/Mips/AsmParser/LLVMBuild.txt b/lib/Target/Mips/AsmParser/LLVMBuild.txt
index dd8e3cf6d8cdd334dbab002a952dad4335010654..99703a5a7c857be343d46f53715e5b7093777871 100644
--- a/lib/Target/Mips/AsmParser/LLVMBuild.txt
+++ b/lib/Target/Mips/AsmParser/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/Target/Mips/AsmParser/LLVMBuild.txt ----------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Target/Mips/AsmParser/MipsAsmParser.cpp b/lib/Target/Mips/AsmParser/MipsAsmParser.cpp
index 78dfed68b9228fbab51a90e23ad92613017a2490..6de9e44a059ea1516cd862459276417e30802362 100644
--- a/lib/Target/Mips/AsmParser/MipsAsmParser.cpp
+++ b/lib/Target/Mips/AsmParser/MipsAsmParser.cpp
@@ -1,9 +1,8 @@
 //===-- MipsAsmParser.cpp - Parse Mips assembly to MCInst instructions ----===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -65,10 +64,7 @@ class MCInstrInfo;
 
 } // end namespace llvm
 
-static cl::opt<bool>
-EmitJalrReloc("mips-jalr-reloc", cl::Hidden,
-              cl::desc("MIPS: Emit R_{MICRO}MIPS_JALR relocation with jalr"),
-              cl::init(true));
+extern cl::opt<bool> EmitJalrReloc;
 
 namespace {
 
@@ -1711,14 +1707,23 @@ static const MCInstrDesc &getInstDesc(unsigned Opcode) {
   return MipsInsts[Opcode];
 }
 
-static bool hasShortDelaySlot(unsigned Opcode) {
-  switch (Opcode) {
+static bool hasShortDelaySlot(MCInst &Inst) {
+  switch (Inst.getOpcode()) {
+    case Mips::BEQ_MM:
+    case Mips::BNE_MM:
+    case Mips::BLTZ_MM:
+    case Mips::BGEZ_MM:
+    case Mips::BLEZ_MM:
+    case Mips::BGTZ_MM:
+    case Mips::JRC16_MM:
     case Mips::JALS_MM:
     case Mips::JALRS_MM:
     case Mips::JALRS16_MM:
     case Mips::BGEZALS_MM:
     case Mips::BLTZALS_MM:
       return true;
+    case Mips::J_MM:
+      return !Inst.getOperand(0).isReg();
     default:
       return false;
   }
@@ -2302,7 +2307,7 @@ bool MipsAsmParser::processInstruction(MCInst &Inst, SMLoc IDLoc,
   // If this instruction has a delay slot and .set reorder is active,
   // emit a NOP after it.
   if (FillDelaySlot) {
-    TOut.emitEmptyDelaySlot(hasShortDelaySlot(Inst.getOpcode()), IDLoc, STI);
+    TOut.emitEmptyDelaySlot(hasShortDelaySlot(Inst), IDLoc, STI);
     TOut.emitDirectiveSetReorder();
   }
 
@@ -2314,7 +2319,7 @@ bool MipsAsmParser::processInstruction(MCInst &Inst, SMLoc IDLoc,
       // If .set reorder has been used, we've already emitted a NOP.
       // If .set noreorder has been used, we need to emit a NOP at this point.
       if (!AssemblerOptions.back()->isReorder())
-        TOut.emitEmptyDelaySlot(hasShortDelaySlot(Inst.getOpcode()), IDLoc,
+        TOut.emitEmptyDelaySlot(hasShortDelaySlot(Inst), IDLoc,
                                 STI);
 
       // Load the $gp from the stack.
@@ -2601,7 +2606,7 @@ bool MipsAsmParser::expandJalWithRegs(MCInst &Inst, SMLoc IDLoc,
   // emit a NOP after it.
   const MCInstrDesc &MCID = getInstDesc(JalrInst.getOpcode());
   if (MCID.hasDelaySlot() && AssemblerOptions.back()->isReorder())
-    TOut.emitEmptyDelaySlot(hasShortDelaySlot(JalrInst.getOpcode()), IDLoc,
+    TOut.emitEmptyDelaySlot(hasShortDelaySlot(JalrInst), IDLoc,
                             STI);
 
   return false;
diff --git a/lib/Target/Mips/CMakeLists.txt b/lib/Target/Mips/CMakeLists.txt
index 2cacc0a0870caec9402136916070f090799d4847..b67fb46cf66a1a5d06bea9cd54b55de1931cc5ec 100644
--- a/lib/Target/Mips/CMakeLists.txt
+++ b/lib/Target/Mips/CMakeLists.txt
@@ -44,6 +44,7 @@ add_llvm_target(MipsCodeGen
   MipsModuleISelDAGToDAG.cpp
   MipsOptimizePICCall.cpp
   MipsOs16.cpp
+  MipsPreLegalizerCombiner.cpp
   MipsRegisterBankInfo.cpp
   MipsRegisterInfo.cpp
   MipsSEFrameLowering.cpp
diff --git a/lib/Target/Mips/Disassembler/LLVMBuild.txt b/lib/Target/Mips/Disassembler/LLVMBuild.txt
index 414b4f78480fcee81711ae17cf97c221c863cd08..611bdac35aaf66ede1691e1f6d6c72ba4427e651 100644
--- a/lib/Target/Mips/Disassembler/LLVMBuild.txt
+++ b/lib/Target/Mips/Disassembler/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/Target/Mips/Disassembler/LLVMBuild.txt -------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Target/Mips/Disassembler/MipsDisassembler.cpp b/lib/Target/Mips/Disassembler/MipsDisassembler.cpp
index 27b27ff1e1e25ba1f7a43a4a431b2979c30adfd2..043bc34bbd6c5058e5885afd15b7ffe09e958fb0 100644
--- a/lib/Target/Mips/Disassembler/MipsDisassembler.cpp
+++ b/lib/Target/Mips/Disassembler/MipsDisassembler.cpp
@@ -1,9 +1,8 @@
 //===- MipsDisassembler.cpp - Disassembler for Mips -----------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Mips/InstPrinter/LLVMBuild.txt b/lib/Target/Mips/InstPrinter/LLVMBuild.txt
index 317057b913b1b070eee300645416905fee799619..56fd7085446d0a8172ae3399e8affb4c9e7a5839 100644
--- a/lib/Target/Mips/InstPrinter/LLVMBuild.txt
+++ b/lib/Target/Mips/InstPrinter/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/Target/Mips/InstPrinter/LLVMBuild.txt --------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Target/Mips/InstPrinter/MipsInstPrinter.cpp b/lib/Target/Mips/InstPrinter/MipsInstPrinter.cpp
index 73732a40bb8ad7db61e3de4724c5105d887f8e7d..91084512503fd6d159807b0bfd23869ebf236318 100644
--- a/lib/Target/Mips/InstPrinter/MipsInstPrinter.cpp
+++ b/lib/Target/Mips/InstPrinter/MipsInstPrinter.cpp
@@ -1,9 +1,8 @@
 //===-- MipsInstPrinter.cpp - Convert Mips MCInst to assembly syntax ------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Mips/InstPrinter/MipsInstPrinter.h b/lib/Target/Mips/InstPrinter/MipsInstPrinter.h
index f02443ee21d34399521fc20b8c533e1f0c1f5651..04bacbd6267b8a789a4afff47fa23b6268368444 100644
--- a/lib/Target/Mips/InstPrinter/MipsInstPrinter.h
+++ b/lib/Target/Mips/InstPrinter/MipsInstPrinter.h
@@ -1,9 +1,8 @@
 //=== MipsInstPrinter.h - Convert Mips MCInst to assembly syntax -*- C++ -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Mips/LLVMBuild.txt b/lib/Target/Mips/LLVMBuild.txt
index 4b6851f7af69d7a2a98624b3bc6a2fdaff8966f5..4119010efa9ae4567a5388557b3d7aa0e0c35302 100644
--- a/lib/Target/Mips/LLVMBuild.txt
+++ b/lib/Target/Mips/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/Target/Mips/LLVMBuild.txt --------------------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Target/Mips/MCTargetDesc/LLVMBuild.txt b/lib/Target/Mips/MCTargetDesc/LLVMBuild.txt
index 29f5da691180f07f5778a02e2c72422fce511426..8782383825ad5b3e9a909da2ff710f09bfa186f6 100644
--- a/lib/Target/Mips/MCTargetDesc/LLVMBuild.txt
+++ b/lib/Target/Mips/MCTargetDesc/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/Target/Mips/MCTargetDesc/LLVMBuild.txt -------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Target/Mips/MCTargetDesc/MipsABIFlagsSection.cpp b/lib/Target/Mips/MCTargetDesc/MipsABIFlagsSection.cpp
index 4a2b75b9ae4602736bed1fe28f504000cc74fd8d..fca1149453c9e3a993770523b19c915d9c497a84 100644
--- a/lib/Target/Mips/MCTargetDesc/MipsABIFlagsSection.cpp
+++ b/lib/Target/Mips/MCTargetDesc/MipsABIFlagsSection.cpp
@@ -1,9 +1,8 @@
 //===- MipsABIFlagsSection.cpp - Mips ELF ABI Flags Section ---------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/Mips/MCTargetDesc/MipsABIFlagsSection.h b/lib/Target/Mips/MCTargetDesc/MipsABIFlagsSection.h
index 68bf3829aab51bfce2ae01e3a5c1387cf7ae3e8d..239e55495e9d54e3e91d6ab3ef38e4c830760d7f 100644
--- a/lib/Target/Mips/MCTargetDesc/MipsABIFlagsSection.h
+++ b/lib/Target/Mips/MCTargetDesc/MipsABIFlagsSection.h
@@ -1,9 +1,8 @@
 //===- MipsABIFlagsSection.h - Mips ELF ABI Flags Section -------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/Mips/MCTargetDesc/MipsABIInfo.cpp b/lib/Target/Mips/MCTargetDesc/MipsABIInfo.cpp
index 18d7dd99be34a7da957a68763308d9a1be54fa2f..bdd190fc17c98c6d4958ed640dc55a14f406d5fa 100644
--- a/lib/Target/Mips/MCTargetDesc/MipsABIInfo.cpp
+++ b/lib/Target/Mips/MCTargetDesc/MipsABIInfo.cpp
@@ -1,9 +1,8 @@
 //===---- MipsABIInfo.cpp - Information about MIPS ABI's ------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -15,6 +14,13 @@
 
 using namespace llvm;
 
+// Note: this option is defined here to be visible from libLLVMMipsAsmParser
+//       and libLLVMMipsCodeGen
+cl::opt<bool>
+EmitJalrReloc("mips-jalr-reloc", cl::Hidden,
+              cl::desc("MIPS: Emit R_{MICRO}MIPS_JALR relocation with jalr"),
+              cl::init(true));
+
 namespace {
 static const MCPhysReg O32IntRegs[4] = {Mips::A0, Mips::A1, Mips::A2, Mips::A3};
 
diff --git a/lib/Target/Mips/MCTargetDesc/MipsABIInfo.h b/lib/Target/Mips/MCTargetDesc/MipsABIInfo.h
index 9372a3c2bb1f85a34d71a6fa913ea86930ee2d49..534e6573b63c5eb5c3ff733783662bb1921d016c 100644
--- a/lib/Target/Mips/MCTargetDesc/MipsABIInfo.h
+++ b/lib/Target/Mips/MCTargetDesc/MipsABIInfo.h
@@ -1,9 +1,8 @@
 //===---- MipsABIInfo.h - Information about MIPS ABI's --------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/Mips/MCTargetDesc/MipsAsmBackend.cpp b/lib/Target/Mips/MCTargetDesc/MipsAsmBackend.cpp
index 265d1141cb0b22837bd5576e28a886a603cd3bca..65e2a902182dedec48837b340c630b7351edf498 100644
--- a/lib/Target/Mips/MCTargetDesc/MipsAsmBackend.cpp
+++ b/lib/Target/Mips/MCTargetDesc/MipsAsmBackend.cpp
@@ -1,9 +1,8 @@
 //===-- MipsAsmBackend.cpp - Mips Asm Backend  ----------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Mips/MCTargetDesc/MipsAsmBackend.h b/lib/Target/Mips/MCTargetDesc/MipsAsmBackend.h
index 30359132e92b2642b05f0fe3ba2c6eb36147c4ac..4d7e36995ae4f0d4396a1465023e23078a6328d5 100644
--- a/lib/Target/Mips/MCTargetDesc/MipsAsmBackend.h
+++ b/lib/Target/Mips/MCTargetDesc/MipsAsmBackend.h
@@ -1,9 +1,8 @@
 //===-- MipsAsmBackend.h - Mips Asm Backend  ------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Mips/MCTargetDesc/MipsBaseInfo.h b/lib/Target/Mips/MCTargetDesc/MipsBaseInfo.h
index a90db2384c466422b5558e10695a4059bd60d56d..6d8cb264158f61e0c8485a3ca91b31c20a74f29a 100644
--- a/lib/Target/Mips/MCTargetDesc/MipsBaseInfo.h
+++ b/lib/Target/Mips/MCTargetDesc/MipsBaseInfo.h
@@ -1,9 +1,8 @@
 //===-- MipsBaseInfo.h - Top level definitions for MIPS MC ------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -89,7 +88,10 @@ namespace MipsII {
     MO_GOT_HI16,
     MO_GOT_LO16,
     MO_CALL_HI16,
-    MO_CALL_LO16
+    MO_CALL_LO16,
+
+    /// Helper operand used to generate R_MIPS_JALR
+    MO_JALR
   };
 
   enum {
diff --git a/lib/Target/Mips/MCTargetDesc/MipsELFObjectWriter.cpp b/lib/Target/Mips/MCTargetDesc/MipsELFObjectWriter.cpp
index 0e8de0577a8a93361be17187cfb0cfaf5f410ae7..c11e2ce85e847708cc6f3b126047d80670ffffdc 100644
--- a/lib/Target/Mips/MCTargetDesc/MipsELFObjectWriter.cpp
+++ b/lib/Target/Mips/MCTargetDesc/MipsELFObjectWriter.cpp
@@ -1,9 +1,8 @@
 //===-- MipsELFObjectWriter.cpp - Mips ELF Writer -------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -11,6 +10,7 @@
 #include "MCTargetDesc/MipsMCTargetDesc.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/BinaryFormat/ELF.h"
+#include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCELFObjectWriter.h"
 #include "llvm/MC/MCFixup.h"
 #include "llvm/MC/MCObjectWriter.h"
@@ -225,7 +225,9 @@ unsigned MipsELFObjectWriter::getRelocType(MCContext &Ctx,
   case Mips::fixup_Mips_NONE:
     return ELF::R_MIPS_NONE;
   case FK_Data_1:
-    report_fatal_error("MIPS does not support one byte relocations");
+    Ctx.reportError(Fixup.getLoc(),
+                    "MIPS does not support one byte relocations");
+    return ELF::R_MIPS_NONE;
   case Mips::fixup_Mips_16:
   case FK_Data_2:
     return IsPCRel ? ELF::R_MIPS_PC16 : ELF::R_MIPS_16;
@@ -236,6 +238,10 @@ unsigned MipsELFObjectWriter::getRelocType(MCContext &Ctx,
 
   if (IsPCRel) {
     switch (Kind) {
+    case FK_Data_8:
+      Ctx.reportError(Fixup.getLoc(),
+                      "MIPS does not support 64-bit PC-relative relocations");
+      return ELF::R_MIPS_NONE;
     case Mips::fixup_Mips_Branch_PCRel:
     case Mips::fixup_Mips_PC16:
       return ELF::R_MIPS_PC16;
diff --git a/lib/Target/Mips/MCTargetDesc/MipsELFStreamer.cpp b/lib/Target/Mips/MCTargetDesc/MipsELFStreamer.cpp
index 21b01e8509678ffac52fd1626143667b9edc4c21..1b83e9445fb5c67585ad86d8c719bf547103b15d 100644
--- a/lib/Target/Mips/MCTargetDesc/MipsELFStreamer.cpp
+++ b/lib/Target/Mips/MCTargetDesc/MipsELFStreamer.cpp
@@ -1,9 +1,8 @@
 //===-------- MipsELFStreamer.cpp - ELF Object Output ---------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -35,7 +34,7 @@ MipsELFStreamer::MipsELFStreamer(MCContext &Context,
 }
 
 void MipsELFStreamer::EmitInstruction(const MCInst &Inst,
-                                      const MCSubtargetInfo &STI, bool) {
+                                      const MCSubtargetInfo &STI) {
   MCELFStreamer::EmitInstruction(Inst, STI);
 
   MCContext &Context = getContext();
diff --git a/lib/Target/Mips/MCTargetDesc/MipsELFStreamer.h b/lib/Target/Mips/MCTargetDesc/MipsELFStreamer.h
index 56a0ff96c7bd6e6e10ba4410a4ae35c727aa5a1e..2febfbc69b6fed54e638f43a512859ea58e46733 100644
--- a/lib/Target/Mips/MCTargetDesc/MipsELFStreamer.h
+++ b/lib/Target/Mips/MCTargetDesc/MipsELFStreamer.h
@@ -1,9 +1,8 @@
 //===- MipsELFStreamer.h - ELF Object Output --------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -42,8 +41,7 @@ public:
   /// \p Inst is actually emitted. For example, we can inspect the operands and
   /// gather sufficient information that allows us to reason about the register
   /// usage for the translation unit.
-  void EmitInstruction(const MCInst &Inst, const MCSubtargetInfo &STI,
-                       bool = false) override;
+  void EmitInstruction(const MCInst &Inst, const MCSubtargetInfo &STI) override;
 
   /// Overriding this function allows us to record all labels that should be
   /// marked as microMIPS. Based on this data marking is done in
diff --git a/lib/Target/Mips/MCTargetDesc/MipsFixupKinds.h b/lib/Target/Mips/MCTargetDesc/MipsFixupKinds.h
index eedad16dddc30686de2db04367adcbd13289329d..939afefabe9bf69c1f221c1984d6605edd973f4f 100644
--- a/lib/Target/Mips/MCTargetDesc/MipsFixupKinds.h
+++ b/lib/Target/Mips/MCTargetDesc/MipsFixupKinds.h
@@ -1,9 +1,8 @@
 //===-- MipsFixupKinds.h - Mips Specific Fixup Entries ----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/Mips/MCTargetDesc/MipsMCAsmInfo.cpp b/lib/Target/Mips/MCTargetDesc/MipsMCAsmInfo.cpp
index 1506b4a83649ea6fada9e31198324d9e8ec4cf09..ec78158d387d9d49bad13d714eb734967d446d53 100644
--- a/lib/Target/Mips/MCTargetDesc/MipsMCAsmInfo.cpp
+++ b/lib/Target/Mips/MCTargetDesc/MipsMCAsmInfo.cpp
@@ -1,9 +1,8 @@
 //===-- MipsMCAsmInfo.cpp - Mips Asm Properties ---------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Mips/MCTargetDesc/MipsMCAsmInfo.h b/lib/Target/Mips/MCTargetDesc/MipsMCAsmInfo.h
index d4ccf0349c16a0bd0d580155d5ddb3751664ce8c..867f4d223de42bd84d4d717598608d8e11942fc0 100644
--- a/lib/Target/Mips/MCTargetDesc/MipsMCAsmInfo.h
+++ b/lib/Target/Mips/MCTargetDesc/MipsMCAsmInfo.h
@@ -1,9 +1,8 @@
 //===-- MipsMCAsmInfo.h - Mips Asm Info ------------------------*- C++ -*--===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Mips/MCTargetDesc/MipsMCCodeEmitter.cpp b/lib/Target/Mips/MCTargetDesc/MipsMCCodeEmitter.cpp
index f43a4d980f92aac74211faa9ed6320f9e9ec3537..3d3003a0811f0d7c692eebf78eeb0e3db5bf7aa1 100644
--- a/lib/Target/Mips/MCTargetDesc/MipsMCCodeEmitter.cpp
+++ b/lib/Target/Mips/MCTargetDesc/MipsMCCodeEmitter.cpp
@@ -1,9 +1,8 @@
 //===-- MipsMCCodeEmitter.cpp - Convert Mips Code to Machine Code ---------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -614,8 +613,9 @@ getExprOpValue(const MCExpr *Expr, SmallVectorImpl<MCFixup> &Fixups,
       llvm_unreachable("Unhandled fixup kind!");
       break;
     case MipsMCExpr::MEK_DTPREL:
-      llvm_unreachable("MEK_DTPREL is used for TLS DIEExpr only");
-      break;
+      // MEK_DTPREL is used for marking TLS DIEExpr only
+      // and contains a regular sub-expression.
+      return getExprOpValue(MipsExpr->getSubExpr(), Fixups, STI);
     case MipsMCExpr::MEK_CALL_HI16:
       FixupKind = Mips::fixup_Mips_CALL_HI16;
       break;
diff --git a/lib/Target/Mips/MCTargetDesc/MipsMCCodeEmitter.h b/lib/Target/Mips/MCTargetDesc/MipsMCCodeEmitter.h
index 09d50d4776ba57a463d6e68f1823a78b26187575..ff6e1d62b05f65f03a62c3371fed65e841854990 100644
--- a/lib/Target/Mips/MCTargetDesc/MipsMCCodeEmitter.h
+++ b/lib/Target/Mips/MCTargetDesc/MipsMCCodeEmitter.h
@@ -1,9 +1,8 @@
 //===- MipsMCCodeEmitter.h - Convert Mips Code to Machine Code --*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Mips/MCTargetDesc/MipsMCExpr.cpp b/lib/Target/Mips/MCTargetDesc/MipsMCExpr.cpp
index 99857e083c6ca880964945fb67c0ad3ceeb6022a..680806c4deb2254880621af144543894b54c5521 100644
--- a/lib/Target/Mips/MCTargetDesc/MipsMCExpr.cpp
+++ b/lib/Target/Mips/MCTargetDesc/MipsMCExpr.cpp
@@ -1,9 +1,8 @@
 //===-- MipsMCExpr.cpp - Mips specific MC expression classes --------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -44,8 +43,10 @@ void MipsMCExpr::printImpl(raw_ostream &OS, const MCAsmInfo *MAI) const {
     llvm_unreachable("MEK_None and MEK_Special are invalid");
     break;
   case MEK_DTPREL:
-    llvm_unreachable("MEK_DTPREL is used for TLS DIEExpr only");
-    break;
+    // MEK_DTPREL is used for marking TLS DIEExpr only
+    // and contains a regular sub-expression.
+    getSubExpr()->print(OS, MAI, true);
+    return;
   case MEK_CALL_HI16:
     OS << "%call_hi";
     break;
@@ -161,7 +162,9 @@ MipsMCExpr::evaluateAsRelocatableImpl(MCValue &Res,
     case MEK_Special:
       llvm_unreachable("MEK_None and MEK_Special are invalid");
     case MEK_DTPREL:
-      llvm_unreachable("MEK_DTPREL is used for TLS DIEExpr only");
+      // MEK_DTPREL is used for marking TLS DIEExpr only
+      // and contains a regular sub-expression.
+      return getSubExpr()->evaluateAsRelocatable(Res, Layout, Fixup);
     case MEK_DTPREL_HI:
     case MEK_DTPREL_LO:
     case MEK_GOT:
@@ -249,9 +252,6 @@ void MipsMCExpr::fixELFSymbolsInTLSFixups(MCAssembler &Asm) const {
   case MEK_Special:
     llvm_unreachable("MEK_None and MEK_Special are invalid");
     break;
-  case MEK_DTPREL:
-    llvm_unreachable("MEK_DTPREL is used for TLS DIEExpr only");
-    break;
   case MEK_CALL_HI16:
   case MEK_CALL_LO16:
   case MEK_GOT:
@@ -274,6 +274,7 @@ void MipsMCExpr::fixELFSymbolsInTLSFixups(MCAssembler &Asm) const {
     if (const MipsMCExpr *E = dyn_cast<const MipsMCExpr>(getSubExpr()))
       E->fixELFSymbolsInTLSFixups(Asm);
     break;
+  case MEK_DTPREL:
   case MEK_DTPREL_HI:
   case MEK_DTPREL_LO:
   case MEK_TLSLDM:
diff --git a/lib/Target/Mips/MCTargetDesc/MipsMCExpr.h b/lib/Target/Mips/MCTargetDesc/MipsMCExpr.h
index bf3274ab5d173196bc1a1c01ae1f91d891c08137..edc12e87e9b6354fd01d45fe98b8a7be69566257 100644
--- a/lib/Target/Mips/MCTargetDesc/MipsMCExpr.h
+++ b/lib/Target/Mips/MCTargetDesc/MipsMCExpr.h
@@ -1,9 +1,8 @@
 //===- MipsMCExpr.h - Mips specific MC expression classes -------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/Mips/MCTargetDesc/MipsMCNaCl.h b/lib/Target/Mips/MCTargetDesc/MipsMCNaCl.h
index 988629ed1bca7f0e9727fa8712477f0e2ac43b26..ad5aff6552f64a349645970d650ec6ad902eb9ec 100644
--- a/lib/Target/Mips/MCTargetDesc/MipsMCNaCl.h
+++ b/lib/Target/Mips/MCTargetDesc/MipsMCNaCl.h
@@ -1,9 +1,8 @@
 //===-- MipsMCNaCl.h - NaCl-related declarations --------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/Mips/MCTargetDesc/MipsMCTargetDesc.cpp b/lib/Target/Mips/MCTargetDesc/MipsMCTargetDesc.cpp
index a8cd7b0d9b0398b0959987edab4f49538a0b3fd5..bffdebf84d4fc9d2143a279680bef31ae4c57917 100644
--- a/lib/Target/Mips/MCTargetDesc/MipsMCTargetDesc.cpp
+++ b/lib/Target/Mips/MCTargetDesc/MipsMCTargetDesc.cpp
@@ -1,9 +1,8 @@
 //===-- MipsMCTargetDesc.cpp - Mips Target Descriptions -------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Mips/MCTargetDesc/MipsMCTargetDesc.h b/lib/Target/Mips/MCTargetDesc/MipsMCTargetDesc.h
index 4fc174ab58719b61ff8bd75c51e4c5bf3030a15c..55da1cfca8dcaad2d164db59155eec0648c42725 100644
--- a/lib/Target/Mips/MCTargetDesc/MipsMCTargetDesc.h
+++ b/lib/Target/Mips/MCTargetDesc/MipsMCTargetDesc.h
@@ -1,9 +1,8 @@
 //===-- MipsMCTargetDesc.h - Mips Target Descriptions -----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Mips/MCTargetDesc/MipsNaClELFStreamer.cpp b/lib/Target/Mips/MCTargetDesc/MipsNaClELFStreamer.cpp
index 6bf62ea618b4e277d59cd108194e4f7ca778c2af..c050db8a17fd57b1023df0b966f713f343ed0d60 100644
--- a/lib/Target/Mips/MCTargetDesc/MipsNaClELFStreamer.cpp
+++ b/lib/Target/Mips/MCTargetDesc/MipsNaClELFStreamer.cpp
@@ -1,9 +1,8 @@
 //===-- MipsNaClELFStreamer.cpp - ELF Object Output for Mips NaCl ---------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -144,8 +143,8 @@ private:
 public:
   /// This function is the one used to emit instruction data into the ELF
   /// streamer.  We override it to mask dangerous instructions.
-  void EmitInstruction(const MCInst &Inst, const MCSubtargetInfo &STI,
-                       bool) override {
+  void EmitInstruction(const MCInst &Inst,
+                       const MCSubtargetInfo &STI) override {
     // Sandbox indirect jumps.
     if (isIndirectJump(Inst)) {
       if (PendingCall)
diff --git a/lib/Target/Mips/MCTargetDesc/MipsOptionRecord.cpp b/lib/Target/Mips/MCTargetDesc/MipsOptionRecord.cpp
index 2d84528e7469f2caecf12a48d9af3f5227c6dce3..b4ebb9d18b725c9aa4905d1a118dd03e41408b64 100644
--- a/lib/Target/Mips/MCTargetDesc/MipsOptionRecord.cpp
+++ b/lib/Target/Mips/MCTargetDesc/MipsOptionRecord.cpp
@@ -1,9 +1,8 @@
 //===- MipsOptionRecord.cpp - Abstraction for storing information ---------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/Mips/MCTargetDesc/MipsTargetStreamer.cpp b/lib/Target/Mips/MCTargetDesc/MipsTargetStreamer.cpp
index 1eb21b6cc8261a1c440a27288a752d1e1783e4d3..f00a9fe20efddf95d7b0ec8239c91dce3f2cdeb5 100644
--- a/lib/Target/Mips/MCTargetDesc/MipsTargetStreamer.cpp
+++ b/lib/Target/Mips/MCTargetDesc/MipsTargetStreamer.cpp
@@ -1,9 +1,8 @@
 //===-- MipsTargetStreamer.cpp - Mips Target Streamer Methods -------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -248,7 +247,11 @@ void MipsTargetStreamer::emitEmptyDelaySlot(bool hasShortDelaySlot, SMLoc IDLoc,
 }
 
 void MipsTargetStreamer::emitNop(SMLoc IDLoc, const MCSubtargetInfo *STI) {
-  emitRRI(Mips::SLL, Mips::ZERO, Mips::ZERO, 0, IDLoc, STI);
+  const FeatureBitset &Features = STI->getFeatureBits();
+  if (Features[Mips::FeatureMicroMips])
+    emitRR(Mips::MOVE16_MM, Mips::ZERO, Mips::ZERO, IDLoc, STI);
+  else
+    emitRRI(Mips::SLL, Mips::ZERO, Mips::ZERO, 0, IDLoc, STI);
 }
 
 /// Emit the $gp restore operation for .cprestore.
diff --git a/lib/Target/Mips/MicroMips32r6InstrFormats.td b/lib/Target/Mips/MicroMips32r6InstrFormats.td
index ed5b8dd71a5129149839ad15750e37aeb45cdba5..dbff0f6200f2ad98b1961eb5d7c6a6d9b92cddf9 100644
--- a/lib/Target/Mips/MicroMips32r6InstrFormats.td
+++ b/lib/Target/Mips/MicroMips32r6InstrFormats.td
@@ -1,9 +1,8 @@
 //=- MicroMips32r6InstrFormats.td - Mips32r6 Instruction Formats -*- tablegen -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Mips/MicroMips32r6InstrInfo.td b/lib/Target/Mips/MicroMips32r6InstrInfo.td
index 814918d25e70d7f5508825aa27f21d1f041ff6b0..190e0d2fc64959d59a1a683b6211e463c8c034b9 100644
--- a/lib/Target/Mips/MicroMips32r6InstrInfo.td
+++ b/lib/Target/Mips/MicroMips32r6InstrInfo.td
@@ -1,9 +1,8 @@
 //=- MicroMips32r6InstrInfo.td - MicroMips r6 Instruction Information -*- tablegen -*-=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -460,6 +459,7 @@ class JALRC16_MMR6_DESC_BASE<string opstr, RegisterOperand RO>
   let isCall = 1;
   let hasDelaySlot = 0;
   let Defs = [RA];
+  let hasPostISelHook = 1;
 }
 class JALRC16_MMR6_DESC : JALRC16_MMR6_DESC_BASE<"jalr", GPR32Opnd>;
 
diff --git a/lib/Target/Mips/MicroMipsDSPInstrFormats.td b/lib/Target/Mips/MicroMipsDSPInstrFormats.td
index 0d444dfc9fadecf53aec02faeea7209f42a94950..26b6cf8994ca01352a822675d12149cc5a5294d4 100644
--- a/lib/Target/Mips/MicroMipsDSPInstrFormats.td
+++ b/lib/Target/Mips/MicroMipsDSPInstrFormats.td
@@ -1,9 +1,8 @@
 //===-- MicroMipsDSPInstrFormats.td - Instruction Formats --*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/Mips/MicroMipsDSPInstrInfo.td b/lib/Target/Mips/MicroMipsDSPInstrInfo.td
index 132de6be750d47521984d5ebc5d50e35b283458a..5a12568893aff26ead72fe5edbce54d5f17ab155 100644
--- a/lib/Target/Mips/MicroMipsDSPInstrInfo.td
+++ b/lib/Target/Mips/MicroMipsDSPInstrInfo.td
@@ -1,9 +1,8 @@
 //===- MicroMipsDSPInstrInfo.td - Micromips DSP instructions -*- tablegen *-=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Mips/MicroMipsInstrFPU.td b/lib/Target/Mips/MicroMipsInstrFPU.td
index 1731afc1961f8baa96009ff9eec2a12efc81c06d..e1f25b137ca369dbc7e242cba5313f7cefd17702 100644
--- a/lib/Target/Mips/MicroMipsInstrFPU.td
+++ b/lib/Target/Mips/MicroMipsInstrFPU.td
@@ -1,9 +1,8 @@
 //==- MicroMipsInstrFPU.td - microMIPS FPU Instruction Info -*- tablegen -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -124,7 +123,7 @@ multiclass ABSS_MMM<string opstr, InstrItinClass Itin,
 defm FSQRT : ABSS_MMM<"sqrt.d", II_SQRT_D, fsqrt>, ROUND_W_FM_MM<1, 0x28>;
 defm FABS : ABSS_MMM<"abs.d", II_SQRT_D, fabs>, ABS_FM_MM<1, 0xd>;
 
-let DecoderNamespace = "MicroMips" in {
+let DecoderNamespace = "MicroMips", AdditionalPredicates = [UseAbs] in {
   def FABS_S_MM : MMRel, ABSS_FT<"abs.s", FGR32Opnd, FGR32Opnd, II_ABS, fabs>,
                   ABS_FM_MM<0, 0xd>, ISA_MICROMIPS;
 }
diff --git a/lib/Target/Mips/MicroMipsInstrFormats.td b/lib/Target/Mips/MicroMipsInstrFormats.td
index 2a4cc279ef0d730eb6b25a34c5b90009595c7de3..e9fb9b310e3b1cc1cea723b471b993392e0ed68e 100644
--- a/lib/Target/Mips/MicroMipsInstrFormats.td
+++ b/lib/Target/Mips/MicroMipsInstrFormats.td
@@ -1,9 +1,8 @@
 //===-- MicroMipsInstrFormats.td - microMIPS Inst Formats -*- tablegen -*--===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Mips/MicroMipsInstrInfo.td b/lib/Target/Mips/MicroMipsInstrInfo.td
index af380a0ec71e0a0b52d16cacb4a609e232247d79..bf13ed44a36a69f8ccfa487ed4b3e4d2055a40e7 100644
--- a/lib/Target/Mips/MicroMipsInstrInfo.td
+++ b/lib/Target/Mips/MicroMipsInstrInfo.td
@@ -1,9 +1,8 @@
 //===--- MicroMipsInstrFormats.td - microMIPS Inst Defs -*- tablegen -*----===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -426,6 +425,7 @@ class JumpLinkRegMM16<string opstr, RegisterOperand RO> :
   let isCall = 1;
   let hasDelaySlot = 1;
   let Defs = [RA];
+  let hasPostISelHook = 1;
 }
 
 // 16-bit Jump Reg
diff --git a/lib/Target/Mips/MicroMipsSizeReduction.cpp b/lib/Target/Mips/MicroMipsSizeReduction.cpp
index f9062cc23da20482f61d5377e070178741206329..70af95592aa562684f8aa09e857778d3849e119a 100644
--- a/lib/Target/Mips/MicroMipsSizeReduction.cpp
+++ b/lib/Target/Mips/MicroMipsSizeReduction.cpp
@@ -1,9 +1,8 @@
 //=== MicroMipsSizeReduction.cpp - MicroMips size reduction pass --------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///\file
diff --git a/lib/Target/Mips/Mips.h b/lib/Target/Mips/Mips.h
index ef3a807c76489c9ce0b1d30e101fa0a8c78f3f99..b3faaab436f018f5e132f33e93a153650015b694 100644
--- a/lib/Target/Mips/Mips.h
+++ b/lib/Target/Mips/Mips.h
@@ -1,9 +1,8 @@
 //===-- Mips.h - Top-level interface for Mips representation ----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -38,6 +37,7 @@ namespace llvm {
   FunctionPass *createMipsConstantIslandPass();
   FunctionPass *createMicroMipsSizeReducePass();
   FunctionPass *createMipsExpandPseudoPass();
+  FunctionPass *createMipsPreLegalizeCombiner();
 
   InstructionSelector *createMipsInstructionSelector(const MipsTargetMachine &,
                                                      MipsSubtarget &,
@@ -46,6 +46,7 @@ namespace llvm {
   void initializeMipsDelaySlotFillerPass(PassRegistry &);
   void initializeMipsBranchExpansionPass(PassRegistry &);
   void initializeMicroMipsSizeReducePass(PassRegistry &);
+  void initializeMipsPreLegalizerCombinerPass(PassRegistry&);
 } // end namespace llvm;
 
 #endif
diff --git a/lib/Target/Mips/Mips.td b/lib/Target/Mips/Mips.td
index 2f3a1c399d3ed90d89f717ad81e590b609a6e421..7b83ea8535ae6afbd8f904f00e98b4dde9a62240 100644
--- a/lib/Target/Mips/Mips.td
+++ b/lib/Target/Mips/Mips.td
@@ -1,9 +1,8 @@
 //===-- Mips.td - Describe the Mips Target Machine ---------*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 // This is the top level entry point for the Mips target.
@@ -83,6 +82,8 @@ def FeatureFPXX        : SubtargetFeature<"fpxx", "IsFPXX", "true",
                                 "Support for FPXX">;
 def FeatureNaN2008     : SubtargetFeature<"nan2008", "IsNaN2008bit", "true",
                                 "IEEE 754-2008 NaN encoding">;
+def FeatureAbs2008     : SubtargetFeature<"abs2008", "Abs2008", "true",
+                                          "Disable IEEE 754-2008 abs.fmt mode">;
 def FeatureSingleFloat : SubtargetFeature<"single-float", "IsSingleFloat",
                                 "true", "Only supports single precision float">;
 def FeatureSoftFloat   : SubtargetFeature<"soft-float", "IsSoftFloat", "true",
@@ -142,7 +143,7 @@ def FeatureMips32r6    : SubtargetFeature<"mips32r6", "MipsArchVersion",
                                 "Mips32r6",
                                 "Mips32r6 ISA Support [experimental]",
                                 [FeatureMips32r5, FeatureFP64Bit,
-                                 FeatureNaN2008]>;
+                                 FeatureNaN2008, FeatureAbs2008]>;
 def FeatureMips64      : SubtargetFeature<"mips64", "MipsArchVersion",
                                 "Mips64", "Mips64 ISA Support",
                                 [FeatureMips5, FeatureMips32]>;
@@ -159,7 +160,7 @@ def FeatureMips64r6    : SubtargetFeature<"mips64r6", "MipsArchVersion",
                                 "Mips64r6",
                                 "Mips64r6 ISA Support [experimental]",
                                 [FeatureMips32r6, FeatureMips64r5,
-                                 FeatureNaN2008]>;
+                                 FeatureNaN2008, FeatureAbs2008]>;
 def FeatureSym32       : SubtargetFeature<"sym32", "HasSym32", "true",
                                           "Symbols are 32 bit on Mips64">;
 
diff --git a/lib/Target/Mips/Mips16FrameLowering.cpp b/lib/Target/Mips/Mips16FrameLowering.cpp
index 122c1f5377b6862be70f7ada40ef510b1561cf3b..5a2a916a6b7a113fb16f761ce71928fe753ffbf5 100644
--- a/lib/Target/Mips/Mips16FrameLowering.cpp
+++ b/lib/Target/Mips/Mips16FrameLowering.cpp
@@ -1,9 +1,8 @@
 //===- Mips16FrameLowering.cpp - Mips16 Frame Information -----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Mips/Mips16FrameLowering.h b/lib/Target/Mips/Mips16FrameLowering.h
index f7fa4dc3d86d9ac28678b734e5111fd3f66eb144..6b62453f8dfe9b4229dde0a1b38dd912382df1a6 100644
--- a/lib/Target/Mips/Mips16FrameLowering.h
+++ b/lib/Target/Mips/Mips16FrameLowering.h
@@ -1,9 +1,8 @@
 //===-- Mips16FrameLowering.h - Mips16 frame lowering  ----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Mips/Mips16HardFloat.cpp b/lib/Target/Mips/Mips16HardFloat.cpp
index c310d9491af818aa3813cab86fd8f95532e6da98..e9a3c7ec4b190b8fb80dd0b28a0f58727762dba4 100644
--- a/lib/Target/Mips/Mips16HardFloat.cpp
+++ b/lib/Target/Mips/Mips16HardFloat.cpp
@@ -1,9 +1,8 @@
 //===- Mips16HardFloat.cpp for Mips16 Hard Float --------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -74,16 +73,18 @@ static FPReturnVariant whichFPReturnVariant(Type *T) {
     return FRet;
   case Type::DoubleTyID:
     return DRet;
-  case Type::StructTyID:
-    if (T->getStructNumElements() != 2)
+  case Type::StructTyID: {
+    StructType *ST = cast<StructType>(T);
+    if (ST->getNumElements() != 2)
       break;
-    if ((T->getContainedType(0)->isFloatTy()) &&
-        (T->getContainedType(1)->isFloatTy()))
+    if ((ST->getElementType(0)->isFloatTy()) &&
+        (ST->getElementType(1)->isFloatTy()))
       return CFRet;
-    if ((T->getContainedType(0)->isDoubleTy()) &&
-        (T->getContainedType(1)->isDoubleTy()))
+    if ((ST->getElementType(0)->isDoubleTy()) &&
+        (ST->getElementType(1)->isDoubleTy()))
       return CDRet;
     break;
+  }
   default:
     break;
   }
@@ -413,7 +414,7 @@ static bool fixupFPReturnAndCall(Function &F, Module *M,
                            Attribute::ReadNone);
         A = A.addAttribute(C, AttributeList::FunctionIndex,
                            Attribute::NoInline);
-        Value *F = (M->getOrInsertFunction(Name, A, MyVoid, T));
+        FunctionCallee F = (M->getOrInsertFunction(Name, A, MyVoid, T));
         CallInst::Create(F, Params, "", &I);
       } else if (const CallInst *CI = dyn_cast<CallInst>(&I)) {
         FunctionType *FT = CI->getFunctionType();
diff --git a/lib/Target/Mips/Mips16HardFloatInfo.cpp b/lib/Target/Mips/Mips16HardFloatInfo.cpp
index 2eb6e5ddd2d96ba3aa3954476cf9b338c5d2c5e3..8a02e8156175643f3395f55473b354f57a4bb606 100644
--- a/lib/Target/Mips/Mips16HardFloatInfo.cpp
+++ b/lib/Target/Mips/Mips16HardFloatInfo.cpp
@@ -1,9 +1,8 @@
 //===---- Mips16HardFloatInfo.cpp for Mips16 Hard Float              -----===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Mips/Mips16HardFloatInfo.h b/lib/Target/Mips/Mips16HardFloatInfo.h
index 7295c287576dac02ebea04403ebf7382d222c3d2..b8c485b7e2e347bcef06ed001a379bec88217330 100644
--- a/lib/Target/Mips/Mips16HardFloatInfo.h
+++ b/lib/Target/Mips/Mips16HardFloatInfo.h
@@ -1,9 +1,8 @@
 //===---- Mips16HardFloatInfo.h for Mips16 Hard Float              --------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Mips/Mips16ISelDAGToDAG.cpp b/lib/Target/Mips/Mips16ISelDAGToDAG.cpp
index a0d5bd9ef30560663eca6a59080a911a83cf11d8..3ab4f1e064da949a725c63e025b37b9f6c5120ea 100644
--- a/lib/Target/Mips/Mips16ISelDAGToDAG.cpp
+++ b/lib/Target/Mips/Mips16ISelDAGToDAG.cpp
@@ -1,9 +1,8 @@
 //===-- Mips16ISelDAGToDAG.cpp - A Dag to Dag Inst Selector for Mips16 ----===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Mips/Mips16ISelDAGToDAG.h b/lib/Target/Mips/Mips16ISelDAGToDAG.h
index bbf8cc36f241a2740c4a1ff1976204e0a91472a2..1ef194029f50990c339db77220892b75a0001fb6 100644
--- a/lib/Target/Mips/Mips16ISelDAGToDAG.h
+++ b/lib/Target/Mips/Mips16ISelDAGToDAG.h
@@ -1,9 +1,8 @@
 //===---- Mips16ISelDAGToDAG.h - A Dag to Dag Inst Selector for Mips ------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Mips/Mips16ISelLowering.cpp b/lib/Target/Mips/Mips16ISelLowering.cpp
index 8ce47e3f669d2d0bf93b4a23b08214b33b8daa32..27adae9bb65a1c5fe0deb66762705751ad24d548 100644
--- a/lib/Target/Mips/Mips16ISelLowering.cpp
+++ b/lib/Target/Mips/Mips16ISelLowering.cpp
@@ -1,9 +1,8 @@
 //===-- Mips16ISelLowering.h - Mips16 DAG Lowering Interface ----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -386,27 +385,22 @@ const char* Mips16TargetLowering::
   }
   else if (RetTy ->isDoubleTy()) {
     result = dfMips16Helper[stubNum];
-  }
-  else if (RetTy->isStructTy()) {
+  } else if (StructType *SRetTy = dyn_cast<StructType>(RetTy)) {
     // check if it's complex
-    if (RetTy->getNumContainedTypes() == 2) {
-      if ((RetTy->getContainedType(0)->isFloatTy()) &&
-          (RetTy->getContainedType(1)->isFloatTy())) {
+    if (SRetTy->getNumElements() == 2) {
+      if ((SRetTy->getElementType(0)->isFloatTy()) &&
+          (SRetTy->getElementType(1)->isFloatTy())) {
         result = scMips16Helper[stubNum];
-      }
-      else if ((RetTy->getContainedType(0)->isDoubleTy()) &&
-               (RetTy->getContainedType(1)->isDoubleTy())) {
+      } else if ((SRetTy->getElementType(0)->isDoubleTy()) &&
+                 (SRetTy->getElementType(1)->isDoubleTy())) {
         result = dcMips16Helper[stubNum];
-      }
-      else {
+      } else {
         llvm_unreachable("Uncovered condition");
       }
-    }
-    else {
+    } else {
       llvm_unreachable("Uncovered condition");
     }
-  }
-  else {
+  } else {
     if (stubNum == 0) {
       needHelper = false;
       return "";
diff --git a/lib/Target/Mips/Mips16ISelLowering.h b/lib/Target/Mips/Mips16ISelLowering.h
index 0ee0b816ef70f2d8c9ecf3ddbf1fba0aaf9c2b21..15be50959b7305564b1b1379c1da9281818890d5 100644
--- a/lib/Target/Mips/Mips16ISelLowering.h
+++ b/lib/Target/Mips/Mips16ISelLowering.h
@@ -1,9 +1,8 @@
 //===-- Mips16ISelLowering.h - Mips16 DAG Lowering Interface ----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Mips/Mips16InstrFormats.td b/lib/Target/Mips/Mips16InstrFormats.td
index 4ff68bef957efbdaacbd9aed5fc7085c3459cf8c..f4ac160c2ba57e5253cdec46c24fb6a2cd3217be 100644
--- a/lib/Target/Mips/Mips16InstrFormats.td
+++ b/lib/Target/Mips/Mips16InstrFormats.td
@@ -1,9 +1,8 @@
 //===- Mips16InstrFormats.td - Mips Instruction Formats ----*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/Mips/Mips16InstrInfo.cpp b/lib/Target/Mips/Mips16InstrInfo.cpp
index efebc99b5daebbab0af03f2f0d4cc92d825fd006..c234c309d76081d79471fbf07c6dce9b3854490f 100644
--- a/lib/Target/Mips/Mips16InstrInfo.cpp
+++ b/lib/Target/Mips/Mips16InstrInfo.cpp
@@ -1,9 +1,8 @@
 //===- Mips16InstrInfo.cpp - Mips16 Instruction Information ---------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Mips/Mips16InstrInfo.h b/lib/Target/Mips/Mips16InstrInfo.h
index 6a802e4cce5db48ccd0e509a03b7205276c6eb8b..dadcaa3055b38441af11821c0783d16a432a24a4 100644
--- a/lib/Target/Mips/Mips16InstrInfo.h
+++ b/lib/Target/Mips/Mips16InstrInfo.h
@@ -1,9 +1,8 @@
 //===- Mips16InstrInfo.h - Mips16 Instruction Information -------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Mips/Mips16InstrInfo.td b/lib/Target/Mips/Mips16InstrInfo.td
index b7a1b9ce41bf6032cdb781ec4c95c1c34a214b5d..fff9f6ad02fbaf8d8e94f8d4470ab6f91b45022b 100644
--- a/lib/Target/Mips/Mips16InstrInfo.td
+++ b/lib/Target/Mips/Mips16InstrInfo.td
@@ -1,9 +1,8 @@
 //===- Mips16InstrInfo.td - Target Description for Mips16  -*- tablegen -*-=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Mips/Mips16RegisterInfo.cpp b/lib/Target/Mips/Mips16RegisterInfo.cpp
index 751afd5ed3693c099745bfb4507e1039f998acda..5703f585a6a24826ae696ed7217535b32b3913cf 100644
--- a/lib/Target/Mips/Mips16RegisterInfo.cpp
+++ b/lib/Target/Mips/Mips16RegisterInfo.cpp
@@ -1,9 +1,8 @@
 //===-- Mips16RegisterInfo.cpp - MIPS16 Register Information --------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Mips/Mips16RegisterInfo.h b/lib/Target/Mips/Mips16RegisterInfo.h
index d67a79b640334bf70b7c3efa5d9362d85736c2b1..fca78b43f96bdd346b9868cb5bb2a87c129d35d6 100644
--- a/lib/Target/Mips/Mips16RegisterInfo.h
+++ b/lib/Target/Mips/Mips16RegisterInfo.h
@@ -1,9 +1,8 @@
 //===-- Mips16RegisterInfo.h - Mips16 Register Information ------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Mips/Mips32r6InstrFormats.td b/lib/Target/Mips/Mips32r6InstrFormats.td
index 623af570a5e6cc12ab60dbca190bf7311398eb3e..ccb6d1df777aec06162025b192ee92d04389cc62 100644
--- a/lib/Target/Mips/Mips32r6InstrFormats.td
+++ b/lib/Target/Mips/Mips32r6InstrFormats.td
@@ -1,9 +1,8 @@
 //=- Mips32r6InstrFormats.td - Mips32r6 Instruction Formats -*- tablegen -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Mips/Mips32r6InstrInfo.td b/lib/Target/Mips/Mips32r6InstrInfo.td
index 2bd0cf2d59a64d8e91635dd77aaa1bcb8698db64..9c62d5e004935822f779ebb1d44e05c618ade78f 100644
--- a/lib/Target/Mips/Mips32r6InstrInfo.td
+++ b/lib/Target/Mips/Mips32r6InstrInfo.td
@@ -1,9 +1,8 @@
 //=- Mips32r6InstrInfo.td - Mips32r6 Instruction Information -*- tablegen -*-=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -1105,7 +1104,7 @@ def : MipsPat<(select i32:$cond, immz, i32:$f),
 
 // Pseudo instructions
 let isCall = 1, isTerminator = 1, isReturn = 1, isBarrier = 1, hasDelaySlot = 1,
-    hasExtraSrcRegAllocReq = 1, isCTI = 1, Defs = [AT] in {
+    hasExtraSrcRegAllocReq = 1, isCTI = 1, Defs = [AT], hasPostISelHook = 1 in {
   class TailCallRegR6<Instruction JumpInst, Register RT, RegisterOperand RO> :
     PseudoSE<(outs), (ins RO:$rs), [(MipsTailCall RO:$rs)], II_JR>,
     PseudoInstExpansion<(JumpInst RT:$rt, RO:$rs)>;
diff --git a/lib/Target/Mips/Mips64InstrInfo.td b/lib/Target/Mips/Mips64InstrInfo.td
index 5729182deafb438772c9d1f3e9e7ccef053ceeaf..ad7ce084edb01a623bee32e2b6715a3ccd81701c 100644
--- a/lib/Target/Mips/Mips64InstrInfo.td
+++ b/lib/Target/Mips/Mips64InstrInfo.td
@@ -1,9 +1,8 @@
 //===- Mips64InstrInfo.td - Mips64 Instruction Information -*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Mips/Mips64r6InstrInfo.td b/lib/Target/Mips/Mips64r6InstrInfo.td
index ac223bc77256131d3df34d553561aa6cd0b1727a..0877bae179642742136b6f74956224fdc2f0b611 100644
--- a/lib/Target/Mips/Mips64r6InstrInfo.td
+++ b/lib/Target/Mips/Mips64r6InstrInfo.td
@@ -1,9 +1,8 @@
 //=- Mips64r6InstrInfo.td - Mips64r6 Instruction Information -*- tablegen -*-=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Mips/MipsAnalyzeImmediate.cpp b/lib/Target/Mips/MipsAnalyzeImmediate.cpp
index 4e17ee327ab6d95d45ef6e5dbafcc82fcd53c490..ae2b83c414db58ca1a261c0effcf91a40fdefcc0 100644
--- a/lib/Target/Mips/MipsAnalyzeImmediate.cpp
+++ b/lib/Target/Mips/MipsAnalyzeImmediate.cpp
@@ -1,9 +1,8 @@
 //===- MipsAnalyzeImmediate.cpp - Analyze Immediates ----------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/Mips/MipsAnalyzeImmediate.h b/lib/Target/Mips/MipsAnalyzeImmediate.h
index 1c520242fb8d54eeb7787e7e901816841a30e4bb..018b9d824526a90dcfaad3daced3d5c5cd3a5fc2 100644
--- a/lib/Target/Mips/MipsAnalyzeImmediate.h
+++ b/lib/Target/Mips/MipsAnalyzeImmediate.h
@@ -1,9 +1,8 @@
 //===- MipsAnalyzeImmediate.h - Analyze Immediates -------------*- C++ -*--===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/Mips/MipsAsmPrinter.cpp b/lib/Target/Mips/MipsAsmPrinter.cpp
index 362431fd42a6c0a1910b07c0876290e1688c2577..d4a8d41c460aeaeec88a528c1ff1f73996ea1fa6 100644
--- a/lib/Target/Mips/MipsAsmPrinter.cpp
+++ b/lib/Target/Mips/MipsAsmPrinter.cpp
@@ -1,9 +1,8 @@
 //===- MipsAsmPrinter.cpp - Mips LLVM Assembly Printer --------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -68,6 +67,8 @@ using namespace llvm;
 
 #define DEBUG_TYPE "mips-asm-printer"
 
+extern cl::opt<bool> EmitJalrReloc;
+
 MipsTargetStreamer &MipsAsmPrinter::getTargetStreamer() const {
   return static_cast<MipsTargetStreamer &>(*OutStreamer->getTargetStreamer());
 }
@@ -148,6 +149,40 @@ void MipsAsmPrinter::emitPseudoIndirectBranch(MCStreamer &OutStreamer,
   EmitToStreamer(OutStreamer, TmpInst0);
 }
 
+// If there is an MO_JALR operand, insert:
+//
+// .reloc tmplabel, R_{MICRO}MIPS_JALR, symbol
+// tmplabel:
+//
+// This is an optimization hint for the linker which may then replace
+// an indirect call with a direct branch.
+static void emitDirectiveRelocJalr(const MachineInstr &MI,
+                                   MCContext &OutContext,
+                                   TargetMachine &TM,
+                                   MCStreamer &OutStreamer,
+                                   const MipsSubtarget &Subtarget) {
+  for (unsigned int I = MI.getDesc().getNumOperands(), E = MI.getNumOperands();
+       I < E; ++I) {
+    MachineOperand MO = MI.getOperand(I);
+    if (MO.isMCSymbol() && (MO.getTargetFlags() & MipsII::MO_JALR)) {
+      MCSymbol *Callee = MO.getMCSymbol();
+      if (Callee && !Callee->getName().empty()) {
+        MCSymbol *OffsetLabel = OutContext.createTempSymbol();
+        const MCExpr *OffsetExpr =
+            MCSymbolRefExpr::create(OffsetLabel, OutContext);
+        const MCExpr *CaleeExpr =
+            MCSymbolRefExpr::create(Callee, OutContext);
+        OutStreamer.EmitRelocDirective
+            (*OffsetExpr,
+             Subtarget.inMicroMipsMode() ? "R_MICROMIPS_JALR" : "R_MIPS_JALR",
+             CaleeExpr, SMLoc(), *TM.getMCSubtargetInfo());
+        OutStreamer.EmitLabel(OffsetLabel);
+        return;
+      }
+    }
+  }
+}
+
 void MipsAsmPrinter::EmitInstruction(const MachineInstr *MI) {
   MipsTargetStreamer &TS = getTargetStreamer();
   unsigned Opc = MI->getOpcode();
@@ -207,6 +242,11 @@ void MipsAsmPrinter::EmitInstruction(const MachineInstr *MI) {
     return;
   }
 
+  if (EmitJalrReloc &&
+      (MI->isReturn() || MI->isCall() || MI->isIndirectBranch())) {
+    emitDirectiveRelocJalr(*MI, OutContext, TM, *OutStreamer, *Subtarget);
+  }
+
   MachineBasicBlock::const_instr_iterator I = MI->getIterator();
   MachineBasicBlock::const_instr_iterator E = MI->getParent()->instr_end();
 
diff --git a/lib/Target/Mips/MipsAsmPrinter.h b/lib/Target/Mips/MipsAsmPrinter.h
index eb58234e3e77b6b15cbf7df22feac82ca6b4f4bc..89d123920a02220896093e905e67318e1b286aa2 100644
--- a/lib/Target/Mips/MipsAsmPrinter.h
+++ b/lib/Target/Mips/MipsAsmPrinter.h
@@ -1,9 +1,8 @@
 //===- MipsAsmPrinter.h - Mips LLVM Assembly Printer -----------*- C++ -*--===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Mips/MipsBranchExpansion.cpp b/lib/Target/Mips/MipsBranchExpansion.cpp
index e59267c4fd9b61357d2516a21b10d0034c42386a..1523a6c020aad45252e27ab25392156429a5f901 100644
--- a/lib/Target/Mips/MipsBranchExpansion.cpp
+++ b/lib/Target/Mips/MipsBranchExpansion.cpp
@@ -1,9 +1,8 @@
 //===----------------------- MipsBranchExpansion.cpp ----------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/lib/Target/Mips/MipsCCState.cpp b/lib/Target/Mips/MipsCCState.cpp
index 90cb3f437bd5b7119aa150aaca82d62af91a25c6..ef48c850a1b866ec7c4e3023bec296984b4882a6 100644
--- a/lib/Target/Mips/MipsCCState.cpp
+++ b/lib/Target/Mips/MipsCCState.cpp
@@ -1,9 +1,8 @@
 //===---- MipsCCState.cpp - CCState with Mips specific extensions ---------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/Mips/MipsCCState.h b/lib/Target/Mips/MipsCCState.h
index 27901699480bfe194b930b03f4b9978f2b09354c..fd2fd97c8f13302e8fa7a28d0525988f75a15711 100644
--- a/lib/Target/Mips/MipsCCState.h
+++ b/lib/Target/Mips/MipsCCState.h
@@ -1,9 +1,8 @@
 //===---- MipsCCState.h - CCState with Mips specific extensions -----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/Mips/MipsCallLowering.cpp b/lib/Target/Mips/MipsCallLowering.cpp
index c550fadf66320036af906112bd6ca63e863c36ac..fd0f7921ba3a19cb152bceaccb484a90a11f84b5 100644
--- a/lib/Target/Mips/MipsCallLowering.cpp
+++ b/lib/Target/Mips/MipsCallLowering.cpp
@@ -1,9 +1,8 @@
 //===- MipsCallLowering.cpp -------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -147,15 +146,18 @@ void IncomingValueHandler::assignValueToReg(unsigned ValVReg,
 
 unsigned IncomingValueHandler::getStackAddress(const CCValAssign &VA,
                                                MachineMemOperand *&MMO) {
+  MachineFunction &MF = MIRBuilder.getMF();
   unsigned Size = alignTo(VA.getValVT().getSizeInBits(), 8) / 8;
   unsigned Offset = VA.getLocMemOffset();
-  MachineFrameInfo &MFI = MIRBuilder.getMF().getFrameInfo();
+  MachineFrameInfo &MFI = MF.getFrameInfo();
 
   int FI = MFI.CreateFixedObject(Size, Offset, true);
   MachinePointerInfo MPO =
       MachinePointerInfo::getFixedStack(MIRBuilder.getMF(), FI);
-  MMO = MIRBuilder.getMF().getMachineMemOperand(MPO, MachineMemOperand::MOLoad,
-                                                Size, /* Alignment */ 0);
+
+  const TargetFrameLowering *TFL = MF.getSubtarget().getFrameLowering();
+  unsigned Align = MinAlign(TFL->getStackAlignment(), Offset);
+  MMO = MF.getMachineMemOperand(MPO, MachineMemOperand::MOLoad, Size, Align);
 
   unsigned AddrReg = MRI.createGenericVirtualRegister(LLT::pointer(0, 32));
   MIRBuilder.buildFrameIndex(AddrReg, FI);
@@ -221,6 +223,9 @@ void OutgoingValueHandler::assignValueToReg(unsigned ValVReg,
 
 unsigned OutgoingValueHandler::getStackAddress(const CCValAssign &VA,
                                                MachineMemOperand *&MMO) {
+  MachineFunction &MF = MIRBuilder.getMF();
+  const TargetFrameLowering *TFL = MF.getSubtarget().getFrameLowering();
+
   LLT p0 = LLT::pointer(0, 32);
   LLT s32 = LLT::scalar(32);
   unsigned SPReg = MRI.createGenericVirtualRegister(p0);
@@ -236,8 +241,8 @@ unsigned OutgoingValueHandler::getStackAddress(const CCValAssign &VA,
   MachinePointerInfo MPO =
       MachinePointerInfo::getStack(MIRBuilder.getMF(), Offset);
   unsigned Size = alignTo(VA.getValVT().getSizeInBits(), 8) / 8;
-  MMO = MIRBuilder.getMF().getMachineMemOperand(MPO, MachineMemOperand::MOStore,
-                                                Size, /* Alignment */ 0);
+  unsigned Align = MinAlign(TFL->getStackAlignment(), Offset);
+  MMO = MF.getMachineMemOperand(MPO, MachineMemOperand::MOStore, Size, Align);
 
   return AddrReg;
 }
diff --git a/lib/Target/Mips/MipsCallLowering.h b/lib/Target/Mips/MipsCallLowering.h
index 9916b04ef50ca25d3ed66b1d2d2f11e6e9621e62..dc04d83733e7c400c7c24a6820de402c6705ed93 100644
--- a/lib/Target/Mips/MipsCallLowering.h
+++ b/lib/Target/Mips/MipsCallLowering.h
@@ -1,9 +1,8 @@
 //===- MipsCallLowering.h ---------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Mips/MipsCallingConv.td b/lib/Target/Mips/MipsCallingConv.td
index b5df78f89a6b90ba52e1556ee65fcc98156b11e6..88236d8e9abd84a6439c2e255342291a320a08ed 100644
--- a/lib/Target/Mips/MipsCallingConv.td
+++ b/lib/Target/Mips/MipsCallingConv.td
@@ -1,9 +1,8 @@
 //===-- MipsCallingConv.td - Calling Conventions for Mips --*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 // This describes the calling conventions for Mips architecture.
diff --git a/lib/Target/Mips/MipsCondMov.td b/lib/Target/Mips/MipsCondMov.td
index 39dc2654aa6a4569272699eb2300b6e27fec49b8..142473121d910e71ffae2c995ecc85d0f8933d1f 100644
--- a/lib/Target/Mips/MipsCondMov.td
+++ b/lib/Target/Mips/MipsCondMov.td
@@ -1,9 +1,8 @@
 //===-- MipsCondMov.td - Describe Mips Conditional Moves --*- tablegen -*--===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -296,3 +295,13 @@ def PseudoSELECTFP_F_I64 : SelectFP_Pseudo_F<GPR64Opnd>;
 def PseudoSELECTFP_F_S : SelectFP_Pseudo_F<FGR32Opnd>;
 def PseudoSELECTFP_F_D32 : SelectFP_Pseudo_F<AFGR64Opnd>, FGR_32;
 def PseudoSELECTFP_F_D64 : SelectFP_Pseudo_F<FGR64Opnd>, FGR_64;
+
+let usesCustomInserter = 1 in {
+class D_SELECT_CLASS<RegisterOperand RC> :
+  PseudoSE<(outs RC:$dst1, RC:$dst2),
+           (ins GPR32Opnd:$cond, RC:$a1, RC:$a2, RC:$b1, RC:$b2), []>,
+  ISA_MIPS1_NOT_4_32;
+}
+
+def PseudoD_SELECT_I   : D_SELECT_CLASS<GPR32Opnd>;
+def PseudoD_SELECT_I64 : D_SELECT_CLASS<GPR64Opnd>;
diff --git a/lib/Target/Mips/MipsConstantIslandPass.cpp b/lib/Target/Mips/MipsConstantIslandPass.cpp
index 744523cc6cb91f4ac80b5b1f8cbce8879c6edc49..c10d9c38f509ad8d868f75db58767b2a80302b2b 100644
--- a/lib/Target/Mips/MipsConstantIslandPass.cpp
+++ b/lib/Target/Mips/MipsConstantIslandPass.cpp
@@ -1,9 +1,8 @@
 //===- MipsConstantIslandPass.cpp - Emit Pc Relative loads ----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Mips/MipsDSPInstrFormats.td b/lib/Target/Mips/MipsDSPInstrFormats.td
index 5f0763f5ea46664f4a0eb24c30b1d0f7b158b96f..6f062d0f3c25b08130a750884ec3a498385e3c3b 100644
--- a/lib/Target/Mips/MipsDSPInstrFormats.td
+++ b/lib/Target/Mips/MipsDSPInstrFormats.td
@@ -1,9 +1,8 @@
 //===- MipsDSPInstrFormats.td - Mips Instruction Formats ---*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/Mips/MipsDSPInstrInfo.td b/lib/Target/Mips/MipsDSPInstrInfo.td
index b9824220b558086dffce9ee986b7f50244a7be80..3bc1fcfcb3751b7716d6494ba778c39584b1947a 100644
--- a/lib/Target/Mips/MipsDSPInstrInfo.td
+++ b/lib/Target/Mips/MipsDSPInstrInfo.td
@@ -1,9 +1,8 @@
 //===- MipsDSPInstrInfo.td - DSP ASE instructions -*- tablegen ------------*-=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Mips/MipsDelaySlotFiller.cpp b/lib/Target/Mips/MipsDelaySlotFiller.cpp
index e3823e0dfdb8aa851b15770eb839c9f89f160014..7f619569ec2b8626e2b21949170c0ac4add38657 100644
--- a/lib/Target/Mips/MipsDelaySlotFiller.cpp
+++ b/lib/Target/Mips/MipsDelaySlotFiller.cpp
@@ -1,9 +1,8 @@
 //===- MipsDelaySlotFiller.cpp - Mips Delay Slot Filler -------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Mips/MipsEVAInstrFormats.td b/lib/Target/Mips/MipsEVAInstrFormats.td
index 61785d0e891ab5e86dcf80673bf18404e614465e..9820e4dcfc883b3311dc6cc11492b8ba6679be30 100644
--- a/lib/Target/Mips/MipsEVAInstrFormats.td
+++ b/lib/Target/Mips/MipsEVAInstrFormats.td
@@ -1,9 +1,8 @@
 //===- MipsEVAInstrFormats.td - Mips Instruction Formats ---*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Mips/MipsEVAInstrInfo.td b/lib/Target/Mips/MipsEVAInstrInfo.td
index ff54b1f17877d9a5a9da552100e120da81e3df08..73cca8cfa5d97c4c70135f1ce71a5e3048f37e3a 100644
--- a/lib/Target/Mips/MipsEVAInstrInfo.td
+++ b/lib/Target/Mips/MipsEVAInstrInfo.td
@@ -1,9 +1,8 @@
 //===- MipsEVAInstrInfo.td - EVA ASE instructions -*- tablegen ------------*-=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Mips/MipsExpandPseudo.cpp b/lib/Target/Mips/MipsExpandPseudo.cpp
index acf66d1fb1b212f63e2c13fecde4ee04bfa82027..65d84a6c44a0bba034ec3ef4a7379b56bd829bfd 100644
--- a/lib/Target/Mips/MipsExpandPseudo.cpp
+++ b/lib/Target/Mips/MipsExpandPseudo.cpp
@@ -1,9 +1,8 @@
 //===-- MipsExpandPseudoInsts.cpp - Expand pseudo instructions ------------===//
 //
-// The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Mips/MipsFastISel.cpp b/lib/Target/Mips/MipsFastISel.cpp
index 22ade31a72cdbb3572856a33f748930ba4514021..ef4e0c5178507b234ed2bfb76414c31e72dd9b00 100644
--- a/lib/Target/Mips/MipsFastISel.cpp
+++ b/lib/Target/Mips/MipsFastISel.cpp
@@ -1,9 +1,8 @@
 //===- MipsFastISel.cpp - Mips FastISel implementation --------------------===//
 //
-// The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
@@ -56,6 +55,7 @@
 #include "llvm/IR/Type.h"
 #include "llvm/IR/User.h"
 #include "llvm/IR/Value.h"
+#include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCInstrDesc.h"
 #include "llvm/MC/MCRegisterInfo.h"
 #include "llvm/MC/MCSymbol.h"
@@ -75,6 +75,8 @@
 
 using namespace llvm;
 
+extern cl::opt<bool> EmitJalrReloc;
+
 namespace {
 
 class MipsFastISel final : public FastISel {
@@ -1551,6 +1553,16 @@ bool MipsFastISel::fastLowerCall(CallLoweringInfo &CLI) {
 
   CLI.Call = MIB;
 
+  if (EmitJalrReloc && !Subtarget->inMips16Mode()) {
+    // Attach callee address to the instruction, let asm printer emit
+    // .reloc R_MIPS_JALR.
+    if (Symbol)
+      MIB.addSym(Symbol, MipsII::MO_JALR);
+    else
+      MIB.addSym(FuncInfo.MF->getContext().getOrCreateSymbol(
+	                   Addr.getGlobalValue()->getName()), MipsII::MO_JALR);
+  }
+
   // Finish off the call including any return values.
   return finishCall(CLI, RetVT, NumBytes);
 }
diff --git a/lib/Target/Mips/MipsFrameLowering.cpp b/lib/Target/Mips/MipsFrameLowering.cpp
index 27a85970da6f80d519305c7bda0b219b0d16ab5e..8d5eabf59b71324981bb2ebd066d9aef1d25a8f0 100644
--- a/lib/Target/Mips/MipsFrameLowering.cpp
+++ b/lib/Target/Mips/MipsFrameLowering.cpp
@@ -1,9 +1,8 @@
 //===-- MipsFrameLowering.cpp - Mips Frame Information --------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Mips/MipsFrameLowering.h b/lib/Target/Mips/MipsFrameLowering.h
index 0ead56eddd2fc712edd996058a8173803f11393a..0537cfd1cb307ea2e0bc6a0b8a0293e6d3a19bf8 100644
--- a/lib/Target/Mips/MipsFrameLowering.h
+++ b/lib/Target/Mips/MipsFrameLowering.h
@@ -1,9 +1,8 @@
 //===-- MipsFrameLowering.h - Define frame lowering for Mips ----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Mips/MipsISelDAGToDAG.cpp b/lib/Target/Mips/MipsISelDAGToDAG.cpp
index f99f3a1b3e0affe44c6d296419b025e04f244bd4..9ba54d6bb73ca708f23b2bdf04a9d23f5d6e91b5 100644
--- a/lib/Target/Mips/MipsISelDAGToDAG.cpp
+++ b/lib/Target/Mips/MipsISelDAGToDAG.cpp
@@ -1,9 +1,8 @@
 //===-- MipsISelDAGToDAG.cpp - A Dag to Dag Inst Selector for Mips --------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Mips/MipsISelDAGToDAG.h b/lib/Target/Mips/MipsISelDAGToDAG.h
index 09003459d180eb707d68afca95b7e0e1b863e4df..bae3bbf71f3b22f5e8303042a2064d49623f3afc 100644
--- a/lib/Target/Mips/MipsISelDAGToDAG.h
+++ b/lib/Target/Mips/MipsISelDAGToDAG.h
@@ -1,9 +1,8 @@
 //===---- MipsISelDAGToDAG.h - A Dag to Dag Inst Selector for Mips --------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Mips/MipsISelLowering.cpp b/lib/Target/Mips/MipsISelLowering.cpp
index 104fa4090ec07c587bd873c6e9ceabdac5a90722..103d1fcb44bbd2e542d07ce299008de524e4a63a 100644
--- a/lib/Target/Mips/MipsISelLowering.cpp
+++ b/lib/Target/Mips/MipsISelLowering.cpp
@@ -1,9 +1,8 @@
 //===- MipsISelLowering.cpp - Mips DAG Lowering Implementation ------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -57,6 +56,7 @@
 #include "llvm/IR/GlobalValue.h"
 #include "llvm/IR/Type.h"
 #include "llvm/IR/Value.h"
+#include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCRegisterInfo.h"
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/CodeGen.h"
@@ -91,6 +91,8 @@ NoZeroDivCheck("mno-check-zero-division", cl::Hidden,
                cl::desc("MIPS: Don't trap on integer division by zero."),
                cl::init(false));
 
+extern cl::opt<bool> EmitJalrReloc;
+
 static const MCPhysReg Mips64DPRegs[8] = {
   Mips::D12_64, Mips::D13_64, Mips::D14_64, Mips::D15_64,
   Mips::D16_64, Mips::D17_64, Mips::D18_64, Mips::D19_64
@@ -362,6 +364,11 @@ MipsTargetLowering::MipsTargetLowering(const MipsTargetMachine &TM,
   setOperationAction(ISD::FCOPYSIGN,          MVT::f64,   Custom);
   setOperationAction(ISD::FP_TO_SINT,         MVT::i32,   Custom);
 
+  if (!(TM.Options.NoNaNsFPMath || Subtarget.inAbs2008Mode())) {
+    setOperationAction(ISD::FABS, MVT::f32, Custom);
+    setOperationAction(ISD::FABS, MVT::f64, Custom);
+  }
+
   if (Subtarget.isGP64bit()) {
     setOperationAction(ISD::GlobalAddress,      MVT::i64,   Custom);
     setOperationAction(ISD::BlockAddress,       MVT::i64,   Custom);
@@ -1216,6 +1223,7 @@ LowerOperation(SDValue Op, SelectionDAG &DAG) const
   case ISD::VASTART:            return lowerVASTART(Op, DAG);
   case ISD::VAARG:              return lowerVAARG(Op, DAG);
   case ISD::FCOPYSIGN:          return lowerFCOPYSIGN(Op, DAG);
+  case ISD::FABS:               return lowerFABS(Op, DAG);
   case ISD::FRAMEADDR:          return lowerFRAMEADDR(Op, DAG);
   case ISD::RETURNADDR:         return lowerRETURNADDR(Op, DAG);
   case ISD::EH_RETURN:          return lowerEH_RETURN(Op, DAG);
@@ -1396,6 +1404,9 @@ MipsTargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
   case Mips::PseudoSELECTFP_T_D32:
   case Mips::PseudoSELECTFP_T_D64:
     return emitPseudoSELECT(MI, BB, true, Mips::BC1T);
+  case Mips::PseudoD_SELECT_I:
+  case Mips::PseudoD_SELECT_I64:
+    return emitPseudoD_SELECT(MI, BB);
   }
 }
 
@@ -2290,6 +2301,71 @@ MipsTargetLowering::lowerFCOPYSIGN(SDValue Op, SelectionDAG &DAG) const {
   return lowerFCOPYSIGN32(Op, DAG, Subtarget.hasExtractInsert());
 }
 
+static SDValue lowerFABS32(SDValue Op, SelectionDAG &DAG,
+                           bool HasExtractInsert) {
+  SDLoc DL(Op);
+  SDValue Res, Const1 = DAG.getConstant(1, DL, MVT::i32);
+
+  // If operand is of type f64, extract the upper 32-bit. Otherwise, bitcast it
+  // to i32.
+  SDValue X = (Op.getValueType() == MVT::f32)
+                  ? DAG.getNode(ISD::BITCAST, DL, MVT::i32, Op.getOperand(0))
+                  : DAG.getNode(MipsISD::ExtractElementF64, DL, MVT::i32,
+                                Op.getOperand(0), Const1);
+
+  // Clear MSB.
+  if (HasExtractInsert)
+    Res = DAG.getNode(MipsISD::Ins, DL, MVT::i32,
+                      DAG.getRegister(Mips::ZERO, MVT::i32),
+                      DAG.getConstant(31, DL, MVT::i32), Const1, X);
+  else {
+    // TODO: Provide DAG patterns which transform (and x, cst)
+    // back to a (shl (srl x (clz cst)) (clz cst)) sequence.
+    SDValue SllX = DAG.getNode(ISD::SHL, DL, MVT::i32, X, Const1);
+    Res = DAG.getNode(ISD::SRL, DL, MVT::i32, SllX, Const1);
+  }
+
+  if (Op.getValueType() == MVT::f32)
+    return DAG.getNode(ISD::BITCAST, DL, MVT::f32, Res);
+
+  // FIXME: For mips32r2, the sequence of (BuildPairF64 (ins (ExtractElementF64
+  // Op 1), $zero, 31 1) (ExtractElementF64 Op 0)) and the Op has one use, we
+  // should be able to drop the usage of mfc1/mtc1 and rewrite the register in
+  // place.
+  SDValue LowX =
+      DAG.getNode(MipsISD::ExtractElementF64, DL, MVT::i32, Op.getOperand(0),
+                  DAG.getConstant(0, DL, MVT::i32));
+  return DAG.getNode(MipsISD::BuildPairF64, DL, MVT::f64, LowX, Res);
+}
+
+static SDValue lowerFABS64(SDValue Op, SelectionDAG &DAG,
+                           bool HasExtractInsert) {
+  SDLoc DL(Op);
+  SDValue Res, Const1 = DAG.getConstant(1, DL, MVT::i32);
+
+  // Bitcast to integer node.
+  SDValue X = DAG.getNode(ISD::BITCAST, DL, MVT::i64, Op.getOperand(0));
+
+  // Clear MSB.
+  if (HasExtractInsert)
+    Res = DAG.getNode(MipsISD::Ins, DL, MVT::i64,
+                      DAG.getRegister(Mips::ZERO_64, MVT::i64),
+                      DAG.getConstant(63, DL, MVT::i32), Const1, X);
+  else {
+    SDValue SllX = DAG.getNode(ISD::SHL, DL, MVT::i64, X, Const1);
+    Res = DAG.getNode(ISD::SRL, DL, MVT::i64, SllX, Const1);
+  }
+
+  return DAG.getNode(ISD::BITCAST, DL, MVT::f64, Res);
+}
+
+SDValue MipsTargetLowering::lowerFABS(SDValue Op, SelectionDAG &DAG) const {
+  if ((ABI.IsN32() || ABI.IsN64()) && (Op.getValueType() == MVT::f64))
+    return lowerFABS64(Op, DAG, Subtarget.hasExtractInsert());
+
+  return lowerFABS32(Op, DAG, Subtarget.hasExtractInsert());
+}
+
 SDValue MipsTargetLowering::
 lowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) const {
   // check the depth
@@ -2427,6 +2503,16 @@ SDValue MipsTargetLowering::lowerShiftRightParts(SDValue Op, SelectionDAG &DAG,
                              DAG.getConstant(VT.getSizeInBits(), DL, MVT::i32));
   SDValue Ext = DAG.getNode(ISD::SRA, DL, VT, Hi,
                             DAG.getConstant(VT.getSizeInBits() - 1, DL, VT));
+
+  if (!(Subtarget.hasMips4() || Subtarget.hasMips32())) {
+    SDVTList VTList = DAG.getVTList(VT, VT);
+    return DAG.getNode(Subtarget.isGP64bit() ? Mips::PseudoD_SELECT_I64
+                                             : Mips::PseudoD_SELECT_I,
+                       DL, VTList, Cond, ShiftRightHi,
+                       IsSRA ? Ext : DAG.getConstant(0, DL, VT), Or,
+                       ShiftRightHi);
+  }
+
   Lo = DAG.getNode(ISD::SELECT, DL, VT, Cond, ShiftRightHi, Or);
   Hi = DAG.getNode(ISD::SELECT, DL, VT, Cond,
                    IsSRA ? Ext : DAG.getConstant(0, DL, VT), ShiftRightHi);
@@ -2866,6 +2952,54 @@ getOpndList(SmallVectorImpl<SDValue> &Ops,
     Ops.push_back(InFlag);
 }
 
+void MipsTargetLowering::AdjustInstrPostInstrSelection(MachineInstr &MI,
+                                                       SDNode *Node) const {
+  switch (MI.getOpcode()) {
+    default:
+      return;
+    case Mips::JALR:
+    case Mips::JALRPseudo:
+    case Mips::JALR64:
+    case Mips::JALR64Pseudo:
+    case Mips::JALR16_MM:
+    case Mips::JALRC16_MMR6:
+    case Mips::TAILCALLREG:
+    case Mips::TAILCALLREG64:
+    case Mips::TAILCALLR6REG:
+    case Mips::TAILCALL64R6REG:
+    case Mips::TAILCALLREG_MM:
+    case Mips::TAILCALLREG_MMR6: {
+      if (!EmitJalrReloc ||
+          Subtarget.inMips16Mode() ||
+          !isPositionIndependent() ||
+          Node->getNumOperands() < 1 ||
+          Node->getOperand(0).getNumOperands() < 2) {
+        return;
+      }
+      // We are after the callee address, set by LowerCall().
+      // If added to MI, asm printer will emit .reloc R_MIPS_JALR for the
+      // symbol.
+      const SDValue TargetAddr = Node->getOperand(0).getOperand(1);
+      StringRef Sym;
+      if (const GlobalAddressSDNode *G =
+              dyn_cast_or_null<const GlobalAddressSDNode>(TargetAddr)) {
+        Sym = G->getGlobal()->getName();
+      }
+      else if (const ExternalSymbolSDNode *ES =
+                   dyn_cast_or_null<const ExternalSymbolSDNode>(TargetAddr)) {
+        Sym = ES->getSymbol();
+      }
+
+      if (Sym.empty())
+        return;
+
+      MachineFunction *MF = MI.getParent()->getParent();
+      MCSymbol *S = MF->getContext().getOrCreateSymbol(Sym);
+      MI.addOperand(MachineOperand::CreateMCSymbol(S, MipsII::MO_JALR));
+    }
+  }
+}
+
 /// LowerCall - functions arguments are copied from virtual regs to
 /// (physical regs)/(stack frame), CALLSEQ_START and CALLSEQ_END are emitted.
 SDValue
@@ -2917,7 +3051,7 @@ MipsTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
   // the maximum out going argument area (including the reserved area), and
   // preallocates the stack space on entrance to the caller.
   //
-  // FIXME: We should do the same for efficency and space.
+  // FIXME: We should do the same for efficiency and space.
 
   // Note: The check on the calling convention below must match
   //       MipsABIInfo::GetCalleeAllocdArgSizeInBytes().
@@ -4345,6 +4479,81 @@ MachineBasicBlock *MipsTargetLowering::emitPseudoSELECT(MachineInstr &MI,
   return BB;
 }
 
+MachineBasicBlock *MipsTargetLowering::emitPseudoD_SELECT(MachineInstr &MI,
+                                                          MachineBasicBlock *BB) const {
+  assert(!(Subtarget.hasMips4() || Subtarget.hasMips32()) &&
+         "Subtarget already supports SELECT nodes with the use of"
+         "conditional-move instructions.");
+
+  const TargetInstrInfo *TII = Subtarget.getInstrInfo();
+  DebugLoc DL = MI.getDebugLoc();
+
+  // D_SELECT substitutes two SELECT nodes that goes one after another and
+  // have the same condition operand. On machines which don't have
+  // conditional-move instruction, it reduces unnecessary branch instructions
+  // which are result of using two diamond patterns that are result of two
+  // SELECT pseudo instructions.
+  const BasicBlock *LLVM_BB = BB->getBasicBlock();
+  MachineFunction::iterator It = ++BB->getIterator();
+
+  //  thisMBB:
+  //  ...
+  //   TrueVal = ...
+  //   setcc r1, r2, r3
+  //   bNE   r1, r0, copy1MBB
+  //   fallthrough --> copy0MBB
+  MachineBasicBlock *thisMBB = BB;
+  MachineFunction *F = BB->getParent();
+  MachineBasicBlock *copy0MBB = F->CreateMachineBasicBlock(LLVM_BB);
+  MachineBasicBlock *sinkMBB = F->CreateMachineBasicBlock(LLVM_BB);
+  F->insert(It, copy0MBB);
+  F->insert(It, sinkMBB);
+
+  // Transfer the remainder of BB and its successor edges to sinkMBB.
+  sinkMBB->splice(sinkMBB->begin(), BB,
+                  std::next(MachineBasicBlock::iterator(MI)), BB->end());
+  sinkMBB->transferSuccessorsAndUpdatePHIs(BB);
+
+  // Next, add the true and fallthrough blocks as its successors.
+  BB->addSuccessor(copy0MBB);
+  BB->addSuccessor(sinkMBB);
+
+  // bne rs, $0, sinkMBB
+  BuildMI(BB, DL, TII->get(Mips::BNE))
+      .addReg(MI.getOperand(2).getReg())
+      .addReg(Mips::ZERO)
+      .addMBB(sinkMBB);
+
+  //  copy0MBB:
+  //   %FalseValue = ...
+  //   # fallthrough to sinkMBB
+  BB = copy0MBB;
+
+  // Update machine-CFG edges
+  BB->addSuccessor(sinkMBB);
+
+  //  sinkMBB:
+  //   %Result = phi [ %TrueValue, thisMBB ], [ %FalseValue, copy0MBB ]
+  //  ...
+  BB = sinkMBB;
+
+  // Use two PHI nodes to select two reults
+  BuildMI(*BB, BB->begin(), DL, TII->get(Mips::PHI), MI.getOperand(0).getReg())
+      .addReg(MI.getOperand(3).getReg())
+      .addMBB(thisMBB)
+      .addReg(MI.getOperand(5).getReg())
+      .addMBB(copy0MBB);
+  BuildMI(*BB, BB->begin(), DL, TII->get(Mips::PHI), MI.getOperand(1).getReg())
+      .addReg(MI.getOperand(4).getReg())
+      .addMBB(thisMBB)
+      .addReg(MI.getOperand(6).getReg())
+      .addMBB(copy0MBB);
+
+  MI.eraseFromParent(); // The pseudo instruction is gone now.
+
+  return BB;
+}
+
 // FIXME? Maybe this could be a TableGen attribute on some registers and
 // this table could be generated automatically from RegInfo.
 unsigned MipsTargetLowering::getRegisterByName(const char* RegName, EVT VT,
diff --git a/lib/Target/Mips/MipsISelLowering.h b/lib/Target/Mips/MipsISelLowering.h
index 5a0de45c44f3cf1cf5b8c40242d6771b35dca6fe..e25810c87416cf0a7cd2709cf8f9486306571585 100644
--- a/lib/Target/Mips/MipsISelLowering.h
+++ b/lib/Target/Mips/MipsISelLowering.h
@@ -1,9 +1,8 @@
 //===- MipsISelLowering.h - Mips DAG Lowering Interface ---------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -341,6 +340,9 @@ class TargetRegisterClass;
     EmitInstrWithCustomInserter(MachineInstr &MI,
                                 MachineBasicBlock *MBB) const override;
 
+    void AdjustInstrPostInstrSelection(MachineInstr &MI,
+                                       SDNode *Node) const override;
+
     void HandleByVal(CCState *, unsigned &, unsigned) const override;
 
     unsigned getRegisterByName(const char* RegName, EVT VT,
@@ -699,6 +701,8 @@ class TargetRegisterClass;
     MachineBasicBlock *emitSEL_D(MachineInstr &MI, MachineBasicBlock *BB) const;
     MachineBasicBlock *emitPseudoSELECT(MachineInstr &MI, MachineBasicBlock *BB,
                                         bool isFPCmp, unsigned Opc) const;
+    MachineBasicBlock *emitPseudoD_SELECT(MachineInstr &MI,
+                                          MachineBasicBlock *BB) const;
   };
 
   /// Create MipsTargetLowering objects.
diff --git a/lib/Target/Mips/MipsInstrFPU.td b/lib/Target/Mips/MipsInstrFPU.td
index 4cb8574e08f6ba2840995e3621655289cd73df4f..f2648a7e8e48a0fd6993b0608268404a392962a3 100644
--- a/lib/Target/Mips/MipsInstrFPU.td
+++ b/lib/Target/Mips/MipsInstrFPU.td
@@ -1,9 +1,8 @@
 //===-- MipsInstrFPU.td - Mips FPU Instruction Information -*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -487,7 +486,7 @@ let isPseudo = 1, isCodeGenOnly = 1 in {
   def PseudoCVT_D64_L : ABSS_FT<"", FGR64Opnd, GPR64Opnd, II_CVT>;
 }
 
-let AdditionalPredicates = [NotInMicroMips] in {
+let AdditionalPredicates = [NotInMicroMips, UseAbs] in {
   def FABS_S : MMRel, ABSS_FT<"abs.s", FGR32Opnd, FGR32Opnd, II_ABS, fabs>,
                ABSS_FM<0x5, 16>, ISA_MIPS1;
   defm FABS : ABSS_M<"abs.d", II_ABS, fabs>, ABSS_FM<0x5, 17>, ISA_MIPS1;
diff --git a/lib/Target/Mips/MipsInstrFormats.td b/lib/Target/Mips/MipsInstrFormats.td
index ebbdcdf0df89e7757e62064f01b94fa588dc94af..39e7e5b4eeb140ac6dfffa0ade9784163891db8c 100644
--- a/lib/Target/Mips/MipsInstrFormats.td
+++ b/lib/Target/Mips/MipsInstrFormats.td
@@ -1,9 +1,8 @@
 //===-- MipsInstrFormats.td - Mips Instruction Formats -----*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/Mips/MipsInstrInfo.cpp b/lib/Target/Mips/MipsInstrInfo.cpp
index bfb4c775205de67e16f0704a00bcadc77c62c7c6..fbd56206b249021c773674835f4e979ebf20a781 100644
--- a/lib/Target/Mips/MipsInstrInfo.cpp
+++ b/lib/Target/Mips/MipsInstrInfo.cpp
@@ -1,9 +1,8 @@
 //===- MipsInstrInfo.cpp - Mips Instruction Information -------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -578,7 +577,8 @@ unsigned MipsInstrInfo::getInstSizeInBytes(const MachineInstr &MI) const {
   switch (MI.getOpcode()) {
   default:
     return MI.getDesc().getSize();
-  case  TargetOpcode::INLINEASM: {       // Inline Asm: Variable size.
+  case  TargetOpcode::INLINEASM:
+  case  TargetOpcode::INLINEASM_BR: {       // Inline Asm: Variable size.
     const MachineFunction *MF = MI.getParent()->getParent();
     const char *AsmStr = MI.getOperand(0).getSymbolName();
     return getInlineAsmLength(AsmStr, *MF->getTarget().getMCAsmInfo());
@@ -653,6 +653,16 @@ MipsInstrInfo::genInstrWithNewOpc(unsigned NewOpc,
 
     MIB.addImm(0);
 
+    // If I has an MCSymbol operand (used by asm printer, to emit R_MIPS_JALR),
+    // add it to the new instruction.
+    for (unsigned J = I->getDesc().getNumOperands(), E = I->getNumOperands();
+         J < E; ++J) {
+      const MachineOperand &MO = I->getOperand(J);
+      if (MO.isMCSymbol() && (MO.getTargetFlags() & MipsII::MO_JALR))
+        MIB.addSym(MO.getMCSymbol(), MipsII::MO_JALR);
+    }
+
+
   } else {
     for (unsigned J = 0, E = I->getDesc().getNumOperands(); J < E; ++J) {
       if (BranchWithZeroOperand && (unsigned)ZeroOperandPosition == J)
@@ -825,7 +835,8 @@ MipsInstrInfo::getSerializableDirectMachineOperandTargetFlags() const {
     {MO_GOT_HI16,     "mips-got-hi16"},
     {MO_GOT_LO16,     "mips-got-lo16"},
     {MO_CALL_HI16,    "mips-call-hi16"},
-    {MO_CALL_LO16,    "mips-call-lo16"}
+    {MO_CALL_LO16,    "mips-call-lo16"},
+    {MO_JALR,         "mips-jalr"}
   };
   return makeArrayRef(Flags);
 }
diff --git a/lib/Target/Mips/MipsInstrInfo.h b/lib/Target/Mips/MipsInstrInfo.h
index 9d27b8f66211863a3796206d2a468ad22f70a320..a626c0c3fdb8df8a81df0c8b9cb91a58c93627a3 100644
--- a/lib/Target/Mips/MipsInstrInfo.h
+++ b/lib/Target/Mips/MipsInstrInfo.h
@@ -1,9 +1,8 @@
 //===- MipsInstrInfo.h - Mips Instruction Information -----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Mips/MipsInstrInfo.td b/lib/Target/Mips/MipsInstrInfo.td
index d9398b7d6024a58361d40583d4a307495ba58f29..f18877f9a7af6a39455614f5862b2ccad3476d73 100644
--- a/lib/Target/Mips/MipsInstrInfo.td
+++ b/lib/Target/Mips/MipsInstrInfo.td
@@ -1,9 +1,8 @@
 //===- MipsInstrInfo.td - Target Description for Mips Target -*- tablegen -*-=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -221,6 +220,8 @@ def IsNotN64    :     Predicate<"!Subtarget->isABI_N64()">;
 def RelocNotPIC :     Predicate<"!TM.isPositionIndependent()">;
 def RelocPIC    :     Predicate<"TM.isPositionIndependent()">;
 def NoNaNsFPMath :    Predicate<"TM.Options.NoNaNsFPMath">;
+def UseAbs :          Predicate<"Subtarget->inAbs2008Mode() ||"
+                                "TM.Options.NoNaNsFPMath">;
 def HasStdEnc :       Predicate<"Subtarget->hasStandardEncoding()">,
                       AssemblerPredicate<"!FeatureMips16">;
 def NotDSP :          Predicate<"!Subtarget->hasDSP()">;
@@ -1623,11 +1624,15 @@ let isCall=1, hasDelaySlot=1, isCTI=1, Defs = [RA] in {
   class JumpLinkRegPseudo<RegisterOperand RO, Instruction JALRInst,
                           Register RetReg, RegisterOperand ResRO = RO>:
     PseudoSE<(outs), (ins RO:$rs), [(MipsJmpLink RO:$rs)], II_JALR>,
-    PseudoInstExpansion<(JALRInst RetReg, ResRO:$rs)>;
+    PseudoInstExpansion<(JALRInst RetReg, ResRO:$rs)> {
+    let hasPostISelHook = 1;
+  }
 
   class JumpLinkReg<string opstr, RegisterOperand RO>:
     InstSE<(outs RO:$rd), (ins RO:$rs), !strconcat(opstr, "\t$rd, $rs"),
-           [], II_JALR, FrmR, opstr>;
+           [], II_JALR, FrmR, opstr> {
+    let hasPostISelHook = 1;
+  }
 
   class BGEZAL_FT<string opstr, DAGOperand opnd,
                   RegisterOperand RO> :
@@ -1646,7 +1651,9 @@ let isCall = 1, isTerminator = 1, isReturn = 1, isBarrier = 1, hasDelaySlot = 1,
 
   class TailCallReg<Instruction JumpInst, RegisterOperand RO> :
     PseudoSE<(outs), (ins RO:$rs), [(MipsTailCall RO:$rs)], II_JR>,
-    PseudoInstExpansion<(JumpInst RO:$rs)>;
+    PseudoInstExpansion<(JumpInst RO:$rs)> {
+    let hasPostISelHook = 1;
+  }
 }
 
 class BAL_BR_Pseudo<Instruction RealInst, DAGOperand opnd> :
diff --git a/lib/Target/Mips/MipsInstructionSelector.cpp b/lib/Target/Mips/MipsInstructionSelector.cpp
index 4964ca9cbe15048f4553029f149461a0dae96e0b..2ea76f518749b723f7a59871baa8f4f083a5159f 100644
--- a/lib/Target/Mips/MipsInstructionSelector.cpp
+++ b/lib/Target/Mips/MipsInstructionSelector.cpp
@@ -1,9 +1,8 @@
 //===- MipsInstructionSelector.cpp ------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
@@ -91,6 +90,33 @@ static bool selectCopy(MachineInstr &I, const TargetInstrInfo &TII,
   return true;
 }
 
+/// Returning Opc indicates that we failed to select MIPS instruction opcode.
+static unsigned selectLoadStoreOpCode(unsigned Opc, unsigned MemSizeInBytes) {
+  if (Opc == TargetOpcode::G_STORE)
+    switch (MemSizeInBytes) {
+    case 4:
+      return Mips::SW;
+    case 2:
+      return Mips::SH;
+    case 1:
+      return Mips::SB;
+    default:
+      return Opc;
+    }
+  else
+    // Unspecified extending load is selected into zeroExtending load.
+    switch (MemSizeInBytes) {
+    case 4:
+      return Mips::LW;
+    case 2:
+      return Opc == TargetOpcode::G_SEXTLOAD ? Mips::LH : Mips::LHu;
+    case 1:
+      return Opc == TargetOpcode::G_SEXTLOAD ? Mips::LB : Mips::LBu;
+    default:
+      return Opc;
+    }
+}
+
 bool MipsInstructionSelector::select(MachineInstr &I,
                                      CodeGenCoverage &CoverageInfo) const {
 
@@ -128,15 +154,21 @@ bool MipsInstructionSelector::select(MachineInstr &I,
     break;
   }
   case G_STORE:
-  case G_LOAD: {
+  case G_LOAD:
+  case G_ZEXTLOAD:
+  case G_SEXTLOAD: {
     const unsigned DestReg = I.getOperand(0).getReg();
     const unsigned DestRegBank = RBI.getRegBank(DestReg, MRI, TRI)->getID();
     const unsigned OpSize = MRI.getType(DestReg).getSizeInBits();
+    const unsigned OpMemSizeInBytes = (*I.memoperands_begin())->getSize();
 
     if (DestRegBank != Mips::GPRBRegBankID || OpSize != 32)
       return false;
 
-    const unsigned NewOpc = I.getOpcode() == G_STORE ? Mips::SW : Mips::LW;
+    const unsigned NewOpc =
+        selectLoadStoreOpCode(I.getOpcode(), OpMemSizeInBytes);
+    if (NewOpc == I.getOpcode())
+      return false;
 
     MI = BuildMI(MBB, I, I.getDebugLoc(), TII.get(NewOpc))
              .add(I.getOperand(0))
@@ -172,6 +204,15 @@ bool MipsInstructionSelector::select(MachineInstr &I,
     I.eraseFromParent();
     return true;
   }
+  case G_SELECT: {
+    // Handle operands with pointer type.
+    MI = BuildMI(MBB, I, I.getDebugLoc(), TII.get(Mips::MOVN_I_I))
+             .add(I.getOperand(0))
+             .add(I.getOperand(2))
+             .add(I.getOperand(1))
+             .add(I.getOperand(3));
+    break;
+  }
   case G_CONSTANT: {
     int Imm = I.getOperand(1).getCImm()->getValue().getLimitedValue();
     unsigned LUiReg = MRI.createVirtualRegister(&Mips::GPR32RegClass);
diff --git a/lib/Target/Mips/MipsLegalizerInfo.cpp b/lib/Target/Mips/MipsLegalizerInfo.cpp
index 0d80bd479d5eb7397e3fe592a45acc3863c5a9da..2fe6c48109155fefde6be06f3cc10f29a994fd4c 100644
--- a/lib/Target/Mips/MipsLegalizerInfo.cpp
+++ b/lib/Target/Mips/MipsLegalizerInfo.cpp
@@ -1,9 +1,8 @@
 //===- MipsLegalizerInfo.cpp ------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
@@ -25,28 +24,47 @@ MipsLegalizerInfo::MipsLegalizerInfo(const MipsSubtarget &ST) {
   const LLT s64 = LLT::scalar(64);
   const LLT p0 = LLT::pointer(0, 32);
 
-  getActionDefinitionsBuilder(G_ADD)
+  getActionDefinitionsBuilder({G_ADD, G_SUB})
       .legalFor({s32})
       .clampScalar(0, s32, s32);
 
-  getActionDefinitionsBuilder(G_UADDE)
+  getActionDefinitionsBuilder(G_MUL)
+      .legalFor({s32})
+      .minScalar(0, s32);
+
+  getActionDefinitionsBuilder({G_UADDE, G_USUBO, G_USUBE})
       .lowerFor({{s32, s1}});
 
   getActionDefinitionsBuilder({G_LOAD, G_STORE})
-      .legalForCartesianProduct({p0, s32}, {p0});
+      .legalForTypesWithMemSize({{s32, p0, 8},
+                                 {s32, p0, 16},
+                                 {s32, p0, 32},
+                                 {p0, p0, 32}})
+      .minScalar(0, s32);
+
+  getActionDefinitionsBuilder({G_ZEXTLOAD, G_SEXTLOAD})
+      .legalForTypesWithMemSize({{s32, p0, 8},
+                                 {s32, p0, 16}})
+      .minScalar(0, s32);
+
+  getActionDefinitionsBuilder(G_SELECT)
+      .legalForCartesianProduct({p0, s32}, {s32})
+      .minScalar(0, s32)
+      .minScalar(1, s32);
 
   getActionDefinitionsBuilder({G_AND, G_OR, G_XOR})
       .legalFor({s32})
       .clampScalar(0, s32, s32);
 
-  getActionDefinitionsBuilder({G_SHL, G_ASHR, G_LSHR})
-      .legalFor({s32});
-
   getActionDefinitionsBuilder({G_SDIV, G_SREM, G_UREM, G_UDIV})
       .legalFor({s32})
       .minScalar(0, s32)
       .libcallFor({s64});
 
+  getActionDefinitionsBuilder({G_SHL, G_ASHR, G_LSHR})
+    .legalFor({s32, s32})
+    .minScalar(1, s32);
+
   getActionDefinitionsBuilder(G_ICMP)
       .legalFor({{s32, s32}})
       .minScalar(0, s32);
diff --git a/lib/Target/Mips/MipsLegalizerInfo.h b/lib/Target/Mips/MipsLegalizerInfo.h
index 75fadd6cf6139e179ce43163d46e2aeba0db390c..e5021e081890bb69d07c2569453be600c6a8ceb5 100644
--- a/lib/Target/Mips/MipsLegalizerInfo.h
+++ b/lib/Target/Mips/MipsLegalizerInfo.h
@@ -1,9 +1,8 @@
 //===- MipsLegalizerInfo ----------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/lib/Target/Mips/MipsMCInstLower.cpp b/lib/Target/Mips/MipsMCInstLower.cpp
index 46b37ceae391816882e00dcb945b3c4aa2763691..fd984058a2bf5d879260a6a568aaefa17a927e86 100644
--- a/lib/Target/Mips/MipsMCInstLower.cpp
+++ b/lib/Target/Mips/MipsMCInstLower.cpp
@@ -1,9 +1,8 @@
 //===- MipsMCInstLower.cpp - Convert Mips MachineInstr to MCInst ----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -117,6 +116,8 @@ MCOperand MipsMCInstLower::LowerSymbolOperand(const MachineOperand &MO,
   case MipsII::MO_CALL_LO16:
     TargetKind = MipsMCExpr::MEK_CALL_LO16;
     break;
+  case MipsII::MO_JALR:
+    return MCOperand();
   }
 
   switch (MOTy) {
diff --git a/lib/Target/Mips/MipsMCInstLower.h b/lib/Target/Mips/MipsMCInstLower.h
index e19f21c98839be856965cc49be095149adfe9c09..29af6f21de82604c3b716401b71be2ac6b4b78b7 100644
--- a/lib/Target/Mips/MipsMCInstLower.h
+++ b/lib/Target/Mips/MipsMCInstLower.h
@@ -1,9 +1,8 @@
 //===- MipsMCInstLower.h - Lower MachineInstr to MCInst --------*- C++ -*--===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/Mips/MipsMSAInstrFormats.td b/lib/Target/Mips/MipsMSAInstrFormats.td
index d4e2256781844aacba0390005f7f59692d36eb12..2bfc92c85e9651f25fb7be18c3ed289bf1491e8f 100644
--- a/lib/Target/Mips/MipsMSAInstrFormats.td
+++ b/lib/Target/Mips/MipsMSAInstrFormats.td
@@ -1,9 +1,8 @@
 //===- MipsMSAInstrFormats.td - Mips Instruction Formats ---*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/Mips/MipsMSAInstrInfo.td b/lib/Target/Mips/MipsMSAInstrInfo.td
index eecc7c573df1f2439f25e60b5c9d793d37034ac6..a1ae0171502645f5f915fb08e45df6abebdc4a8c 100644
--- a/lib/Target/Mips/MipsMSAInstrInfo.td
+++ b/lib/Target/Mips/MipsMSAInstrInfo.td
@@ -1,9 +1,8 @@
 //===- MipsMSAInstrInfo.td - MSA ASE instructions -*- tablegen ------------*-=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Mips/MipsMTInstrFormats.td b/lib/Target/Mips/MipsMTInstrFormats.td
index c2c22e2ad61cbb65a65bba3d3b96b6ead44abee9..22c290b1c114bfcc4e8688eccd7c53cbffd12cad 100644
--- a/lib/Target/Mips/MipsMTInstrFormats.td
+++ b/lib/Target/Mips/MipsMTInstrFormats.td
@@ -1,9 +1,8 @@
 //===-- MipsMTInstrFormats.td - Mips Instruction Formats ---*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/Mips/MipsMTInstrInfo.td b/lib/Target/Mips/MipsMTInstrInfo.td
index 72e626cbec40a3f590ab812f52e181266a2d5fe4..3edeb57b1876a1fe6fd96d0c5f71334104b12104 100644
--- a/lib/Target/Mips/MipsMTInstrInfo.td
+++ b/lib/Target/Mips/MipsMTInstrInfo.td
@@ -1,9 +1,8 @@
 //===-- MipsMTInstrInfo.td - Mips MT Instruction Infos -----*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Mips/MipsMachineFunction.cpp b/lib/Target/Mips/MipsMachineFunction.cpp
index 81b4352670c0d767b63a0e88aacdb856ae796a12..86c90d65381157c3ea0a801f966daaf789627cd1 100644
--- a/lib/Target/Mips/MipsMachineFunction.cpp
+++ b/lib/Target/Mips/MipsMachineFunction.cpp
@@ -1,9 +1,8 @@
 //===-- MipsMachineFunctionInfo.cpp - Private data used for Mips ----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/Mips/MipsMachineFunction.h b/lib/Target/Mips/MipsMachineFunction.h
index 553a66703b268eab3462473ad1752e13a62f48cc..9e7b9b704eac5a188845ac0df7ccedd2b6f6a449 100644
--- a/lib/Target/Mips/MipsMachineFunction.h
+++ b/lib/Target/Mips/MipsMachineFunction.h
@@ -1,9 +1,8 @@
 //===- MipsMachineFunctionInfo.h - Private data used for Mips ---*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Mips/MipsOptimizePICCall.cpp b/lib/Target/Mips/MipsOptimizePICCall.cpp
index 27bc4843f410e0af9e5028d87baa4f03d17a1704..5ef07a2d283e63b669a71136f579184445c70436 100644
--- a/lib/Target/Mips/MipsOptimizePICCall.cpp
+++ b/lib/Target/Mips/MipsOptimizePICCall.cpp
@@ -1,9 +1,8 @@
 //===- MipsOptimizePICCall.cpp - Optimize PIC Calls -----------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Mips/MipsOptionRecord.h b/lib/Target/Mips/MipsOptionRecord.h
index 4708784063d3dfcc854db16e9448d81be1b1d496..7897095ef8941ee5bbef0580cf36c70b82751a0c 100644
--- a/lib/Target/Mips/MipsOptionRecord.h
+++ b/lib/Target/Mips/MipsOptionRecord.h
@@ -1,9 +1,8 @@
 //===- MipsOptionRecord.h - Abstraction for storing information -*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Mips/MipsOs16.cpp b/lib/Target/Mips/MipsOs16.cpp
index 4edcb3132ada5aa5c7419fce13cf6d25f9da5555..ac4e55f8a1f58cb2adf6da680dedb58dba08b89b 100644
--- a/lib/Target/Mips/MipsOs16.cpp
+++ b/lib/Target/Mips/MipsOs16.cpp
@@ -1,9 +1,8 @@
 //===---- MipsOs16.cpp for Mips Option -Os16                       --------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Mips/MipsPreLegalizerCombiner.cpp b/lib/Target/Mips/MipsPreLegalizerCombiner.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..85076590d407daefbc7caad5ce71e4339d92bb57
--- /dev/null
+++ b/lib/Target/Mips/MipsPreLegalizerCombiner.cpp
@@ -0,0 +1,102 @@
+//=== lib/CodeGen/GlobalISel/MipsPreLegalizerCombiner.cpp --------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This pass does combining of machine instructions at the generic MI level,
+// before the legalizer.
+//
+//===----------------------------------------------------------------------===//
+
+#include "MipsTargetMachine.h"
+#include "llvm/CodeGen/GlobalISel/Combiner.h"
+#include "llvm/CodeGen/GlobalISel/CombinerHelper.h"
+#include "llvm/CodeGen/GlobalISel/CombinerInfo.h"
+#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
+#include "llvm/CodeGen/TargetPassConfig.h"
+
+#define DEBUG_TYPE "mips-prelegalizer-combiner"
+
+using namespace llvm;
+
+namespace {
+class MipsPreLegalizerCombinerInfo : public CombinerInfo {
+public:
+  MipsPreLegalizerCombinerInfo()
+      : CombinerInfo(/*AllowIllegalOps*/ true, /*ShouldLegalizeIllegal*/ false,
+                     /*LegalizerInfo*/ nullptr) {}
+  virtual bool combine(GISelChangeObserver &Observer, MachineInstr &MI,
+                       MachineIRBuilder &B) const override;
+};
+
+bool MipsPreLegalizerCombinerInfo::combine(GISelChangeObserver &Observer,
+                                           MachineInstr &MI,
+                                           MachineIRBuilder &B) const {
+  CombinerHelper Helper(Observer, B);
+
+  switch (MI.getOpcode()) {
+  default:
+    return false;
+  case TargetOpcode::G_LOAD:
+  case TargetOpcode::G_SEXTLOAD:
+  case TargetOpcode::G_ZEXTLOAD:
+    return Helper.tryCombineExtendingLoads(MI);
+  }
+  return false;
+}
+
+// Pass boilerplate
+// ================
+
+class MipsPreLegalizerCombiner : public MachineFunctionPass {
+public:
+  static char ID;
+
+  MipsPreLegalizerCombiner();
+
+  StringRef getPassName() const override { return "MipsPreLegalizerCombiner"; }
+
+  bool runOnMachineFunction(MachineFunction &MF) override;
+
+  void getAnalysisUsage(AnalysisUsage &AU) const override;
+};
+} // end anonymous namespace
+
+void MipsPreLegalizerCombiner::getAnalysisUsage(AnalysisUsage &AU) const {
+  AU.addRequired<TargetPassConfig>();
+  AU.setPreservesCFG();
+  getSelectionDAGFallbackAnalysisUsage(AU);
+  MachineFunctionPass::getAnalysisUsage(AU);
+}
+
+MipsPreLegalizerCombiner::MipsPreLegalizerCombiner() : MachineFunctionPass(ID) {
+  initializeMipsPreLegalizerCombinerPass(*PassRegistry::getPassRegistry());
+}
+
+bool MipsPreLegalizerCombiner::runOnMachineFunction(MachineFunction &MF) {
+  if (MF.getProperties().hasProperty(
+          MachineFunctionProperties::Property::FailedISel))
+    return false;
+  auto *TPC = &getAnalysis<TargetPassConfig>();
+  MipsPreLegalizerCombinerInfo PCInfo;
+  Combiner C(PCInfo, TPC);
+  return C.combineMachineInstrs(MF, nullptr);
+}
+
+char MipsPreLegalizerCombiner::ID = 0;
+INITIALIZE_PASS_BEGIN(MipsPreLegalizerCombiner, DEBUG_TYPE,
+                      "Combine Mips machine instrs before legalization", false,
+                      false)
+INITIALIZE_PASS_DEPENDENCY(TargetPassConfig)
+INITIALIZE_PASS_END(MipsPreLegalizerCombiner, DEBUG_TYPE,
+                    "Combine Mips machine instrs before legalization", false,
+                    false)
+
+namespace llvm {
+FunctionPass *createMipsPreLegalizeCombiner() {
+  return new MipsPreLegalizerCombiner();
+}
+} // end namespace llvm
diff --git a/lib/Target/Mips/MipsRegisterBankInfo.cpp b/lib/Target/Mips/MipsRegisterBankInfo.cpp
index 53b9e42dc63a80070bc4047a6b575c301747ba88..086a30c2e6fb23a81c02c995e2a459808a969d27 100644
--- a/lib/Target/Mips/MipsRegisterBankInfo.cpp
+++ b/lib/Target/Mips/MipsRegisterBankInfo.cpp
@@ -1,9 +1,8 @@
 //===- MipsRegisterBankInfo.cpp ---------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
@@ -83,9 +82,14 @@ MipsRegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
   const ValueMapping *OperandsMapping = &Mips::ValueMappings[Mips::GPRIdx];
 
   switch (Opc) {
+  case G_TRUNC:
   case G_ADD:
+  case G_SUB:
+  case G_MUL:
   case G_LOAD:
   case G_STORE:
+  case G_ZEXTLOAD:
+  case G_SEXTLOAD:
   case G_GEP:
   case G_AND:
   case G_OR:
@@ -111,6 +115,13 @@ MipsRegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
                             &Mips::ValueMappings[Mips::GPRIdx],
                             &Mips::ValueMappings[Mips::GPRIdx]});
     break;
+  case G_SELECT:
+    OperandsMapping =
+        getOperandsMapping({&Mips::ValueMappings[Mips::GPRIdx],
+                            &Mips::ValueMappings[Mips::GPRIdx],
+                            &Mips::ValueMappings[Mips::GPRIdx],
+                            &Mips::ValueMappings[Mips::GPRIdx]});
+    break;
   default:
     return getInvalidInstructionMapping();
   }
diff --git a/lib/Target/Mips/MipsRegisterBankInfo.h b/lib/Target/Mips/MipsRegisterBankInfo.h
index 64a79abaa74d1f412b3ee754917c78e0b45f8aa4..bc8a78d1ef582c22ca3be2c2d1eb821af510a123 100644
--- a/lib/Target/Mips/MipsRegisterBankInfo.h
+++ b/lib/Target/Mips/MipsRegisterBankInfo.h
@@ -1,9 +1,8 @@
 //===- MipsRegisterBankInfo.h -----------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/lib/Target/Mips/MipsRegisterBanks.td b/lib/Target/Mips/MipsRegisterBanks.td
index 5f1687048fac6a357c6173008f2a3407adaaadb3..b591841dfef28124910fae206afb5190a9dd70f7 100644
--- a/lib/Target/Mips/MipsRegisterBanks.td
+++ b/lib/Target/Mips/MipsRegisterBanks.td
@@ -1,9 +1,8 @@
 //===- MipsRegisterBank.td ---------------------------------*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Mips/MipsRegisterInfo.cpp b/lib/Target/Mips/MipsRegisterInfo.cpp
index 3c108c2ba9b7fe545afc69367652969b5c8b589b..112163712892e9e595b7800e33e104399e0d5625 100644
--- a/lib/Target/Mips/MipsRegisterInfo.cpp
+++ b/lib/Target/Mips/MipsRegisterInfo.cpp
@@ -1,9 +1,8 @@
 //===- MipsRegisterInfo.cpp - MIPS Register Information -------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Mips/MipsRegisterInfo.h b/lib/Target/Mips/MipsRegisterInfo.h
index b84aaad05eb5a32591b00eced55279001d590710..0a407b02df8a228d9b892fffcdf4e22793f95319 100644
--- a/lib/Target/Mips/MipsRegisterInfo.h
+++ b/lib/Target/Mips/MipsRegisterInfo.h
@@ -1,9 +1,8 @@
 //===- MipsRegisterInfo.h - Mips Register Information Impl ------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Mips/MipsRegisterInfo.td b/lib/Target/Mips/MipsRegisterInfo.td
index a943a0ad4094338c73496bec6e8085fc23750b95..ba66ad7eedeb0607377c0006739f6d72209048e9 100644
--- a/lib/Target/Mips/MipsRegisterInfo.td
+++ b/lib/Target/Mips/MipsRegisterInfo.td
@@ -1,9 +1,8 @@
 //===-- MipsRegisterInfo.td - Mips Register defs -----------*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/Mips/MipsSEFrameLowering.cpp b/lib/Target/Mips/MipsSEFrameLowering.cpp
index ef1b3c09bdc4c661b9c30c97ebb173b688c059c4..4c6cc1ef771c0087596ed1776c2a2d63535fc733 100644
--- a/lib/Target/Mips/MipsSEFrameLowering.cpp
+++ b/lib/Target/Mips/MipsSEFrameLowering.cpp
@@ -1,9 +1,8 @@
 //===- MipsSEFrameLowering.cpp - Mips32/64 Frame Information --------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Mips/MipsSEFrameLowering.h b/lib/Target/Mips/MipsSEFrameLowering.h
index cb2119d6880bd1cfab2da6cd060d7c36031832fd..78ffe161d9c6b2335e98cd81ac9541419ec64060 100644
--- a/lib/Target/Mips/MipsSEFrameLowering.h
+++ b/lib/Target/Mips/MipsSEFrameLowering.h
@@ -1,9 +1,8 @@
 //===- MipsSEFrameLowering.h - Mips32/64 frame lowering ---------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/Mips/MipsSEISelDAGToDAG.cpp b/lib/Target/Mips/MipsSEISelDAGToDAG.cpp
index f030f83295dd15554bb42c6cd4298a43977e2872..5308ab8c4b3a3cdfbc86c2d94441cb92b9398fdf 100644
--- a/lib/Target/Mips/MipsSEISelDAGToDAG.cpp
+++ b/lib/Target/Mips/MipsSEISelDAGToDAG.cpp
@@ -1,9 +1,8 @@
 //===-- MipsSEISelDAGToDAG.cpp - A Dag to Dag Inst Selector for MipsSE ----===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -795,6 +794,24 @@ bool MipsSEDAGToDAGISel::trySelect(SDNode *Node) {
   switch(Opcode) {
   default: break;
 
+  case Mips::PseudoD_SELECT_I:
+  case Mips::PseudoD_SELECT_I64: {
+    MVT VT = Subtarget->isGP64bit() ? MVT::i64 : MVT::i32;
+    SDValue cond = Node->getOperand(0);
+    SDValue Hi1 = Node->getOperand(1);
+    SDValue Lo1 = Node->getOperand(2);
+    SDValue Hi2 = Node->getOperand(3);
+    SDValue Lo2 = Node->getOperand(4);
+
+    SDValue ops[] = {cond, Hi1, Lo1, Hi2, Lo2};
+    EVT NodeTys[] = {VT, VT};
+    ReplaceNode(Node, CurDAG->getMachineNode(Subtarget->isGP64bit()
+                                                 ? Mips::PseudoD_SELECT_I64
+                                                 : Mips::PseudoD_SELECT_I,
+                                             DL, NodeTys, ops));
+    return true;
+  }
+
   case ISD::ADDE: {
     selectAddE(Node, DL);
     return true;
diff --git a/lib/Target/Mips/MipsSEISelDAGToDAG.h b/lib/Target/Mips/MipsSEISelDAGToDAG.h
index eb3657aae05002c3d821f4b527c4588df9a41a72..72cbde83addb57418c108e83877490a194fc4a48 100644
--- a/lib/Target/Mips/MipsSEISelDAGToDAG.h
+++ b/lib/Target/Mips/MipsSEISelDAGToDAG.h
@@ -1,9 +1,8 @@
 //===-- MipsSEISelDAGToDAG.h - A Dag to Dag Inst Selector for MipsSE -----===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Mips/MipsSEISelLowering.cpp b/lib/Target/Mips/MipsSEISelLowering.cpp
index a78e544c35f01b8a5a116f18b0e7967e09b084c3..ab6ff197ce88276d982344e16f80f2fab7dbeb10 100644
--- a/lib/Target/Mips/MipsSEISelLowering.cpp
+++ b/lib/Target/Mips/MipsSEISelLowering.cpp
@@ -1,9 +1,8 @@
 //===- MipsSEISelLowering.cpp - MipsSE DAG Lowering Interface -------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Mips/MipsSEISelLowering.h b/lib/Target/Mips/MipsSEISelLowering.h
index 761ff3b1fa4d53f428a1ed128d95c112a1d83001..3da78921f84f5785c36932e550e0d003c19c83ba 100644
--- a/lib/Target/Mips/MipsSEISelLowering.h
+++ b/lib/Target/Mips/MipsSEISelLowering.h
@@ -1,9 +1,8 @@
 //===- MipsSEISelLowering.h - MipsSE DAG Lowering Interface -----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Mips/MipsSEInstrInfo.cpp b/lib/Target/Mips/MipsSEInstrInfo.cpp
index c7ab90ed2a3baca590dba069beb275af5f6aaaf7..1dde13f5416ff8dc9040aef559386cdb9d236d8a 100644
--- a/lib/Target/Mips/MipsSEInstrInfo.cpp
+++ b/lib/Target/Mips/MipsSEInstrInfo.cpp
@@ -1,9 +1,8 @@
 //===-- MipsSEInstrInfo.cpp - Mips32/64 Instruction Information -----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Mips/MipsSEInstrInfo.h b/lib/Target/Mips/MipsSEInstrInfo.h
index fce0fe5f58ada1658078026a559a1be3a01c423a..3111d1c21a0a421ef4954b88e69352b76178a0f9 100644
--- a/lib/Target/Mips/MipsSEInstrInfo.h
+++ b/lib/Target/Mips/MipsSEInstrInfo.h
@@ -1,9 +1,8 @@
 //===-- MipsSEInstrInfo.h - Mips32/64 Instruction Information ---*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Mips/MipsSERegisterInfo.cpp b/lib/Target/Mips/MipsSERegisterInfo.cpp
index e7d720a4b76962f46b91828e852e1432492674b5..f4b164d5c0ab8666f7026485358e3a12a4bcaf48 100644
--- a/lib/Target/Mips/MipsSERegisterInfo.cpp
+++ b/lib/Target/Mips/MipsSERegisterInfo.cpp
@@ -1,9 +1,8 @@
 //===-- MipsSERegisterInfo.cpp - MIPS32/64 Register Information -== -------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Mips/MipsSERegisterInfo.h b/lib/Target/Mips/MipsSERegisterInfo.h
index ebae1909d2331f96514e13b10f72fa9228271dc6..82ddf40f56a7f21d9790ed3c0a145c29f36fc7af 100644
--- a/lib/Target/Mips/MipsSERegisterInfo.h
+++ b/lib/Target/Mips/MipsSERegisterInfo.h
@@ -1,9 +1,8 @@
 //===-- MipsSERegisterInfo.h - Mips32/64 Register Information ---*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Mips/MipsSchedule.td b/lib/Target/Mips/MipsSchedule.td
index 410fa655a225a26e3008a70b55a416ac1e06faae..0c0ddeab22c49bfe45836130bce6efc1777d8320 100644
--- a/lib/Target/Mips/MipsSchedule.td
+++ b/lib/Target/Mips/MipsSchedule.td
@@ -1,9 +1,8 @@
 //===-- MipsSchedule.td - Mips Scheduling Definitions ------*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/Mips/MipsScheduleGeneric.td b/lib/Target/Mips/MipsScheduleGeneric.td
index 80ffe7ada7c815395d5175043fe422ca8e58460a..7d5eda4504bc20bd6187ba6d896d091556b089f2 100644
--- a/lib/Target/Mips/MipsScheduleGeneric.td
+++ b/lib/Target/Mips/MipsScheduleGeneric.td
@@ -1,9 +1,8 @@
 //=- MipsScheduleGeneric.td - Generic Scheduling Definitions -*- tablegen -*-=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Mips/MipsScheduleP5600.td b/lib/Target/Mips/MipsScheduleP5600.td
index 846fa11494c7228277ebdfbe762d0b261cfcb531..7d285b7e0c6e70713841db0f67310b6ff153d889 100644
--- a/lib/Target/Mips/MipsScheduleP5600.td
+++ b/lib/Target/Mips/MipsScheduleP5600.td
@@ -1,9 +1,8 @@
 //==- MipsScheduleP5600.td - P5600 Scheduling Definitions --*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/Mips/MipsSubtarget.cpp b/lib/Target/Mips/MipsSubtarget.cpp
index 0c39a45467c495f27c157b8ccc9db47ef56c281c..472dd72de94b93fcf549285914e7b9912fd15d95 100644
--- a/lib/Target/Mips/MipsSubtarget.cpp
+++ b/lib/Target/Mips/MipsSubtarget.cpp
@@ -1,9 +1,8 @@
 //===-- MipsSubtarget.cpp - Mips Subtarget Information --------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -73,7 +72,7 @@ MipsSubtarget::MipsSubtarget(const Triple &TT, StringRef CPU, StringRef FS,
                              unsigned StackAlignOverride)
     : MipsGenSubtargetInfo(TT, CPU, FS), MipsArchVersion(MipsDefault),
       IsLittle(little), IsSoftFloat(false), IsSingleFloat(false), IsFPXX(false),
-      NoABICalls(false), IsFP64bit(false), UseOddSPReg(true),
+      NoABICalls(false), Abs2008(false), IsFP64bit(false), UseOddSPReg(true),
       IsNaN2008bit(false), IsGP64bit(false), HasVFPU(false), HasCnMips(false),
       HasMips3_32(false), HasMips3_32r2(false), HasMips4_32(false),
       HasMips4_32r2(false), HasMips5_32r2(false), InMips16Mode(false),
@@ -129,11 +128,18 @@ MipsSubtarget::MipsSubtarget(const Triple &TT, StringRef CPU, StringRef FS,
       report_fatal_error(
           "indirect jumps with hazard barriers requires MIPS32R2 or later");
   }
+  if (inAbs2008Mode() && hasMips32() && !hasMips32r2()) {
+    report_fatal_error("IEEE 754-2008 abs.fmt is not supported for the given "
+                       "architecture.",
+                       false);
+  }
+
   if (hasMips32r6()) {
     StringRef ISA = hasMips64r6() ? "MIPS64r6" : "MIPS32r6";
 
     assert(isFP64bit());
     assert(isNaN2008());
+    assert(inAbs2008Mode());
     if (hasDSP())
       report_fatal_error(ISA + " is not compatible with the DSP ASE", false);
   }
diff --git a/lib/Target/Mips/MipsSubtarget.h b/lib/Target/Mips/MipsSubtarget.h
index ad8f4848b87090ead650d5dd7bae47ec5296dd6b..aa1200579fc8585bb7839dc74c3419b8208c347b 100644
--- a/lib/Target/Mips/MipsSubtarget.h
+++ b/lib/Target/Mips/MipsSubtarget.h
@@ -1,9 +1,8 @@
 //===-- MipsSubtarget.h - Define Subtarget for the Mips ---------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -87,6 +86,9 @@ class MipsSubtarget : public MipsGenSubtargetInfo {
   // NoABICalls - Disable SVR4-style position-independent code.
   bool NoABICalls;
 
+  // Abs2008 - Use IEEE 754-2008 abs.fmt instruction.
+  bool Abs2008;
+
   // IsFP64bit - The target processor has 64-bit floating point registers.
   bool IsFP64bit;
 
@@ -273,6 +275,7 @@ public:
   bool useOddSPReg() const { return UseOddSPReg; }
   bool noOddSPReg() const { return !UseOddSPReg; }
   bool isNaN2008() const { return IsNaN2008bit; }
+  bool inAbs2008Mode() const { return Abs2008; }
   bool isGP64bit() const { return IsGP64bit; }
   bool isGP32bit() const { return !IsGP64bit; }
   unsigned getGPRSizeInBytes() const { return isGP64bit() ? 8 : 4; }
diff --git a/lib/Target/Mips/MipsTargetMachine.cpp b/lib/Target/Mips/MipsTargetMachine.cpp
index 9cc91d302893d94994e62e1f0d4be1bc1037b5bc..92ba00db1c31d93fd015b7f449f4349f1d0caf82 100644
--- a/lib/Target/Mips/MipsTargetMachine.cpp
+++ b/lib/Target/Mips/MipsTargetMachine.cpp
@@ -1,9 +1,8 @@
 //===-- MipsTargetMachine.cpp - Define TargetMachine for Mips -------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -56,6 +55,7 @@ extern "C" void LLVMInitializeMipsTarget() {
   initializeMipsDelaySlotFillerPass(*PR);
   initializeMipsBranchExpansionPass(*PR);
   initializeMicroMipsSizeReducePass(*PR);
+  initializeMipsPreLegalizerCombinerPass(*PR);
 }
 
 static std::string computeDataLayout(const Triple &TT, StringRef CPU,
@@ -235,6 +235,7 @@ public:
   void addPreEmitPass() override;
   void addPreRegAlloc() override;
   bool addIRTranslator() override;
+  void addPreLegalizeMachineIR() override;
   bool addLegalizeMachineIR() override;
   bool addRegBankSelect() override;
   bool addGlobalInstructionSelect() override;
@@ -312,6 +313,10 @@ bool MipsPassConfig::addIRTranslator() {
   return false;
 }
 
+void MipsPassConfig::addPreLegalizeMachineIR() {
+  addPass(createMipsPreLegalizeCombiner());
+}
+
 bool MipsPassConfig::addLegalizeMachineIR() {
   addPass(new Legalizer());
   return false;
diff --git a/lib/Target/Mips/MipsTargetMachine.h b/lib/Target/Mips/MipsTargetMachine.h
index d9b73d15111943270ff605ce989eef16e8360d53..280c50e77b8090946cb5781daa3b16bf8375077b 100644
--- a/lib/Target/Mips/MipsTargetMachine.h
+++ b/lib/Target/Mips/MipsTargetMachine.h
@@ -1,9 +1,8 @@
 //===- MipsTargetMachine.h - Define TargetMachine for Mips ------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Mips/MipsTargetObjectFile.cpp b/lib/Target/Mips/MipsTargetObjectFile.cpp
index f53ee0631b5e659448d994b3ee53b41846748984..0852b5a18c68f534921905b178239eec1a41b15b 100644
--- a/lib/Target/Mips/MipsTargetObjectFile.cpp
+++ b/lib/Target/Mips/MipsTargetObjectFile.cpp
@@ -1,9 +1,8 @@
 //===-- MipsTargetObjectFile.cpp - Mips Object Files ----------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/Mips/MipsTargetObjectFile.h b/lib/Target/Mips/MipsTargetObjectFile.h
index a37ec154ff79cd83351883ed6978fde2e2b05503..bdf485f83260b9edc7766291858b65cb52608fd4 100644
--- a/lib/Target/Mips/MipsTargetObjectFile.h
+++ b/lib/Target/Mips/MipsTargetObjectFile.h
@@ -1,9 +1,8 @@
 //===-- llvm/Target/MipsTargetObjectFile.h - Mips Object Info ---*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/Mips/MipsTargetStreamer.h b/lib/Target/Mips/MipsTargetStreamer.h
index a282366f6d40d948d1d9690e0bcddf77e9f499c8..697a5c4193ea2454a5dc4a3c5dd5f9046d7715fd 100644
--- a/lib/Target/Mips/MipsTargetStreamer.h
+++ b/lib/Target/Mips/MipsTargetStreamer.h
@@ -1,9 +1,8 @@
 //===-- MipsTargetStreamer.h - Mips Target Streamer ------------*- C++ -*--===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/Mips/TargetInfo/LLVMBuild.txt b/lib/Target/Mips/TargetInfo/LLVMBuild.txt
index 6235bfc3fd18a267e455155c6257178d4e1b576f..38ab068612b4ce2768d02064a27768c85bacca78 100644
--- a/lib/Target/Mips/TargetInfo/LLVMBuild.txt
+++ b/lib/Target/Mips/TargetInfo/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/Target/Mips/TargetInfo/LLVMBuild.txt ---------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Target/Mips/TargetInfo/MipsTargetInfo.cpp b/lib/Target/Mips/TargetInfo/MipsTargetInfo.cpp
index 22be564b650233b95933a007abe017f6783b3d78..e7940c5252c27443d3e2d06c6bba8af1ab5532c8 100644
--- a/lib/Target/Mips/TargetInfo/MipsTargetInfo.cpp
+++ b/lib/Target/Mips/TargetInfo/MipsTargetInfo.cpp
@@ -1,9 +1,8 @@
 //===-- MipsTargetInfo.cpp - Mips Target Implementation -------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/NVPTX/CMakeLists.txt b/lib/Target/NVPTX/CMakeLists.txt
index 4a64fe0961e58e827124a63e96047f198266b704..d094620f1bf1975ef59733ec6c8f7f578f3f19bd 100644
--- a/lib/Target/NVPTX/CMakeLists.txt
+++ b/lib/Target/NVPTX/CMakeLists.txt
@@ -32,6 +32,7 @@ set(NVPTXCodeGen_sources
   NVPTXUtilities.cpp
   NVVMIntrRange.cpp
   NVVMReflect.cpp
+  NVPTXProxyRegErasure.cpp
   )
 
 add_llvm_target(NVPTXCodeGen ${NVPTXCodeGen_sources})
diff --git a/lib/Target/NVPTX/InstPrinter/LLVMBuild.txt b/lib/Target/NVPTX/InstPrinter/LLVMBuild.txt
index 032b57347eb024fd4d5f1acaedae723b0308ce3e..0773d2d1fadf1b83365cbb4bad173185c2874fba 100644
--- a/lib/Target/NVPTX/InstPrinter/LLVMBuild.txt
+++ b/lib/Target/NVPTX/InstPrinter/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/Target/NVPTX/InstPrinter/LLVMBuild.txt -------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Target/NVPTX/InstPrinter/NVPTXInstPrinter.cpp b/lib/Target/NVPTX/InstPrinter/NVPTXInstPrinter.cpp
index b774fe169d7109ecff915b91eb83a71fcc18c777..eafa80570dc1ce0a7ea01c1e373a72daa048bec7 100644
--- a/lib/Target/NVPTX/InstPrinter/NVPTXInstPrinter.cpp
+++ b/lib/Target/NVPTX/InstPrinter/NVPTXInstPrinter.cpp
@@ -1,9 +1,8 @@
 //===-- NVPTXInstPrinter.cpp - PTX assembly instruction printing ----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/NVPTX/InstPrinter/NVPTXInstPrinter.h b/lib/Target/NVPTX/InstPrinter/NVPTXInstPrinter.h
index f0f223aa057b0ae87c54b7fe8ea8d216f3be8793..c9279523afb1b91e1a7c45c09f0a525ec39be510 100644
--- a/lib/Target/NVPTX/InstPrinter/NVPTXInstPrinter.h
+++ b/lib/Target/NVPTX/InstPrinter/NVPTXInstPrinter.h
@@ -1,9 +1,8 @@
 //= NVPTXInstPrinter.h - Convert NVPTX MCInst to assembly syntax --*- C++ -*-=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/NVPTX/LLVMBuild.txt b/lib/Target/NVPTX/LLVMBuild.txt
index ee8aaa998bb660a605caa4c0cdf5717657cf0e2c..56fbbf74fad37d6e59b14d0b63acc03e0913c37a 100644
--- a/lib/Target/NVPTX/LLVMBuild.txt
+++ b/lib/Target/NVPTX/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/Target/NVPTX/LLVMBuild.txt -------------------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Target/NVPTX/MCTargetDesc/LLVMBuild.txt b/lib/Target/NVPTX/MCTargetDesc/LLVMBuild.txt
index 01a051a11aa69a044f2350d648d961fb5fa02764..3d1bd7797c32b43446264ae5dbe8a8c03e84c58d 100644
--- a/lib/Target/NVPTX/MCTargetDesc/LLVMBuild.txt
+++ b/lib/Target/NVPTX/MCTargetDesc/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/Target/NVPTX/MCTargetDesc/LLVMBuild.txt ------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Target/NVPTX/MCTargetDesc/NVPTXBaseInfo.h b/lib/Target/NVPTX/MCTargetDesc/NVPTXBaseInfo.h
index 1cb92005979d5e463d9fe486b212a149695a47de..815b600fe93a9fdc0e64b398bd160bc5a32bc3fe 100644
--- a/lib/Target/NVPTX/MCTargetDesc/NVPTXBaseInfo.h
+++ b/lib/Target/NVPTX/MCTargetDesc/NVPTXBaseInfo.h
@@ -1,9 +1,8 @@
 //===-- NVPTXBaseInfo.h - Top-level definitions for NVPTX -------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/NVPTX/MCTargetDesc/NVPTXMCAsmInfo.cpp b/lib/Target/NVPTX/MCTargetDesc/NVPTXMCAsmInfo.cpp
index f6cbd23f01c400b91b88132e744e74b7277a48b9..556745825a1543953123d3e5a3606e90ea774de9 100644
--- a/lib/Target/NVPTX/MCTargetDesc/NVPTXMCAsmInfo.cpp
+++ b/lib/Target/NVPTX/MCTargetDesc/NVPTXMCAsmInfo.cpp
@@ -1,9 +1,8 @@
 //===-- NVPTXMCAsmInfo.cpp - NVPTX asm properties -------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -38,12 +37,11 @@ NVPTXMCAsmInfo::NVPTXMCAsmInfo(const Triple &TheTriple) {
   HiddenDeclarationVisibilityAttr = HiddenVisibilityAttr = MCSA_Invalid;
   ProtectedVisibilityAttr = MCSA_Invalid;
 
-  // FIXME: remove comment once debug info is properly supported.
-  Data8bitsDirective = "// .b8 ";
+  Data8bitsDirective = ".b8 ";
   Data16bitsDirective = nullptr; // not supported
-  Data32bitsDirective = "// .b32 ";
-  Data64bitsDirective = "// .b64 ";
-  ZeroDirective = "// .b8";
+  Data32bitsDirective = ".b32 ";
+  Data64bitsDirective = ".b64 ";
+  ZeroDirective = ".b8";
   AsciiDirective = nullptr; // not supported
   AscizDirective = nullptr; // not supported
   SupportsQuotedNames = false;
diff --git a/lib/Target/NVPTX/MCTargetDesc/NVPTXMCAsmInfo.h b/lib/Target/NVPTX/MCTargetDesc/NVPTXMCAsmInfo.h
index 9fd7600cf67f43610928ace4a68c0362ea5b37f5..e888526da8987e6085967558582ce5f485e9d5e7 100644
--- a/lib/Target/NVPTX/MCTargetDesc/NVPTXMCAsmInfo.h
+++ b/lib/Target/NVPTX/MCTargetDesc/NVPTXMCAsmInfo.h
@@ -1,9 +1,8 @@
 //===-- NVPTXMCAsmInfo.h - NVPTX asm properties ----------------*- C++ -*--===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/NVPTX/MCTargetDesc/NVPTXMCTargetDesc.cpp b/lib/Target/NVPTX/MCTargetDesc/NVPTXMCTargetDesc.cpp
index b1a77a17ec1515b1757ea66b3cf43e35afae3af4..e92c1911a4afd88925987af0cf099b6e7038c77c 100644
--- a/lib/Target/NVPTX/MCTargetDesc/NVPTXMCTargetDesc.cpp
+++ b/lib/Target/NVPTX/MCTargetDesc/NVPTXMCTargetDesc.cpp
@@ -1,9 +1,8 @@
 //===-- NVPTXMCTargetDesc.cpp - NVPTX Target Descriptions -------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/NVPTX/MCTargetDesc/NVPTXMCTargetDesc.h b/lib/Target/NVPTX/MCTargetDesc/NVPTXMCTargetDesc.h
index 0c9ad977e7ec3152c6798301c246d028a4d40058..dbb3ce36d2652cf1aba47bf7dd809e9148e8b382 100644
--- a/lib/Target/NVPTX/MCTargetDesc/NVPTXMCTargetDesc.h
+++ b/lib/Target/NVPTX/MCTargetDesc/NVPTXMCTargetDesc.h
@@ -1,9 +1,8 @@
 //===-- NVPTXMCTargetDesc.h - NVPTX Target Descriptions ---------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/NVPTX/MCTargetDesc/NVPTXTargetStreamer.cpp b/lib/Target/NVPTX/MCTargetDesc/NVPTXTargetStreamer.cpp
index f7b4cf3a0f7243c15aebedbc8e9b893127089c8a..94959a5078cfbd6d9edba64c6084f57d04af5d02 100644
--- a/lib/Target/NVPTX/MCTargetDesc/NVPTXTargetStreamer.cpp
+++ b/lib/Target/NVPTX/MCTargetDesc/NVPTXTargetStreamer.cpp
@@ -1,9 +1,8 @@
 //=====- NVPTXTargetStreamer.cpp - NVPTXTargetStreamer class ------------=====//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -31,6 +30,11 @@ void NVPTXTargetStreamer::outputDwarfFileDirectives() {
   DwarfFiles.clear();
 }
 
+void NVPTXTargetStreamer::closeLastSection() {
+  if (HasSections)
+    getStreamer().EmitRawText("\t}");
+}
+
 void NVPTXTargetStreamer::emitDwarfFileDirective(StringRef Directive) {
   DwarfFiles.emplace_back(Directive);
 }
@@ -82,18 +86,18 @@ void NVPTXTargetStreamer::changeSection(const MCSection *CurSection,
                                         raw_ostream &OS) {
   assert(!SubSection && "SubSection is not null!");
   const MCObjectFileInfo *FI = getStreamer().getContext().getObjectFileInfo();
-  // FIXME: remove comment once debug info is properly supported.
   // Emit closing brace for DWARF sections only.
   if (isDwarfSection(FI, CurSection))
-    OS << "//\t}\n";
+    OS << "\t}\n";
   if (isDwarfSection(FI, Section)) {
     // Emit DWARF .file directives in the outermost scope.
     outputDwarfFileDirectives();
-    OS << "//\t.section";
+    OS << "\t.section";
     Section->PrintSwitchToSection(*getStreamer().getContext().getAsmInfo(),
                                   FI->getTargetTriple(), OS, SubSection);
     // DWARF sections are enclosed into braces - emit the open one.
-    OS << "//\t{\n";
+    OS << "\t{\n";
+    HasSections = true;
   }
 }
 
diff --git a/lib/Target/NVPTX/MCTargetDesc/NVPTXTargetStreamer.h b/lib/Target/NVPTX/MCTargetDesc/NVPTXTargetStreamer.h
index f18e61cdca574c9dc0f4612e9d379e433e57c93a..8185efadefdb3da7f6a6a71dab6ed6c92362b7c9 100644
--- a/lib/Target/NVPTX/MCTargetDesc/NVPTXTargetStreamer.h
+++ b/lib/Target/NVPTX/MCTargetDesc/NVPTXTargetStreamer.h
@@ -1,9 +1,8 @@
 //=====-- NVPTXTargetStreamer.h - NVPTX Target Streamer ------*- C++ -*--=====//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -19,6 +18,7 @@ class MCSection;
 class NVPTXTargetStreamer : public MCTargetStreamer {
 private:
   SmallVector<std::string, 4> DwarfFiles;
+  bool HasSections = false;
 
 public:
   NVPTXTargetStreamer(MCStreamer &S);
@@ -26,6 +26,8 @@ public:
 
   /// Outputs the list of the DWARF '.file' directives to the streamer.
   void outputDwarfFileDirectives();
+  /// Close last section.
+  void closeLastSection();
 
   /// Record DWARF file directives for later output.
   /// According to PTX ISA, CUDA Toolkit documentation, 11.5.3. Debugging
diff --git a/lib/Target/NVPTX/ManagedStringPool.h b/lib/Target/NVPTX/ManagedStringPool.h
index 7fc0156216f5b43526bd047895a0b157dcbe804c..bbcbb459804047ddd25bf6a11da60a1ccc4a5a24 100644
--- a/lib/Target/NVPTX/ManagedStringPool.h
+++ b/lib/Target/NVPTX/ManagedStringPool.h
@@ -1,9 +1,8 @@
 //===-- ManagedStringPool.h - Managed String Pool ---------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/NVPTX/NVPTX.h b/lib/Target/NVPTX/NVPTX.h
index 02b8d8fff64702ca668dd4107504c40d11a6ec68..bf62697a3e011f7d8d47a31a004dd7efdcf9fcd6 100644
--- a/lib/Target/NVPTX/NVPTX.h
+++ b/lib/Target/NVPTX/NVPTX.h
@@ -1,9 +1,8 @@
 //===-- NVPTX.h - Top-level interface for NVPTX representation --*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -53,6 +52,7 @@ FunctionPass *createNVPTXImageOptimizerPass();
 FunctionPass *createNVPTXLowerArgsPass(const NVPTXTargetMachine *TM);
 BasicBlockPass *createNVPTXLowerAllocaPass();
 MachineFunctionPass *createNVPTXPeephole();
+MachineFunctionPass *createNVPTXProxyRegErasurePass();
 
 Target &getTheNVPTXTarget32();
 Target &getTheNVPTXTarget64();
diff --git a/lib/Target/NVPTX/NVPTX.td b/lib/Target/NVPTX/NVPTX.td
index 3731b2f37f6cc2ca08e2cff52cc2233641ff081b..9f5048b3182ce3c6ef9e34429b8c22bb6f8715b6 100644
--- a/lib/Target/NVPTX/NVPTX.td
+++ b/lib/Target/NVPTX/NVPTX.td
@@ -1,9 +1,8 @@
 //===- NVPTX.td - Describe the NVPTX Target Machine -----------*- tblgen -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 // This is the top level entry point for the NVPTX target.
diff --git a/lib/Target/NVPTX/NVPTXAllocaHoisting.cpp b/lib/Target/NVPTX/NVPTXAllocaHoisting.cpp
index bf922eb8a19554fb7f086f26e6a7f9facf723165..f2c7751df1dff6c50cedc89107c86646e01b8378 100644
--- a/lib/Target/NVPTX/NVPTXAllocaHoisting.cpp
+++ b/lib/Target/NVPTX/NVPTXAllocaHoisting.cpp
@@ -1,9 +1,8 @@
 //===-- AllocaHoisting.cpp - Hoist allocas to the entry block --*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/NVPTX/NVPTXAllocaHoisting.h b/lib/Target/NVPTX/NVPTXAllocaHoisting.h
index 7a6fc7d9b14d8f6b9f9c5dcf51f7611bf8a1fefa..d7de8e3a2f46aee13f70ff830cc0fc9c86211d14 100644
--- a/lib/Target/NVPTX/NVPTXAllocaHoisting.h
+++ b/lib/Target/NVPTX/NVPTXAllocaHoisting.h
@@ -1,9 +1,8 @@
 //===-- AllocaHoisting.h - Hosist allocas to the entry block ----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/NVPTX/NVPTXAsmPrinter.cpp b/lib/Target/NVPTX/NVPTXAsmPrinter.cpp
index 63e227a12edc06b479d324e95c8cc1ace62d66e6..274f1c59b6cc1a71fa2240557c2448ad3209e0a7 100644
--- a/lib/Target/NVPTX/NVPTXAsmPrinter.cpp
+++ b/lib/Target/NVPTX/NVPTXAsmPrinter.cpp
@@ -1,9 +1,8 @@
 //===-- NVPTXAsmPrinter.cpp - NVPTX LLVM assembly writer ------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -473,6 +472,9 @@ void NVPTXAsmPrinter::EmitFunctionEntryLabel() {
   // Emit open brace for function body.
   OutStreamer->EmitRawText(StringRef("{\n"));
   setAndEmitFunctionVirtualRegisters(*MF);
+  // Emit initial .loc debug directive for correct relocation symbol data.
+  if (MMI && MMI->hasDebugInfo())
+    emitInitialRawDwarfLocDirective(*MF);
 }
 
 bool NVPTXAsmPrinter::runOnMachineFunction(MachineFunction &F) {
@@ -730,6 +732,11 @@ void NVPTXAsmPrinter::emitDeclarations(const Module &M, raw_ostream &O) {
   for (Module::const_iterator FI = M.begin(), FE = M.end(); FI != FE; ++FI) {
     const Function *F = &*FI;
 
+    if (F->getAttributes().hasFnAttribute("nvptx-libcall-callee")) {
+      emitDeclaration(F, O);
+      continue;
+    }
+
     if (F->isDeclaration()) {
       if (F->use_empty())
         continue;
@@ -788,11 +795,8 @@ bool NVPTXAsmPrinter::doInitialization(Module &M) {
   // Construct a default subtarget off of the TargetMachine defaults. The
   // rest of NVPTX isn't friendly to change subtargets per function and
   // so the default TargetMachine will have all of the options.
-  const Triple &TT = TM.getTargetTriple();
-  StringRef CPU = TM.getTargetCPU();
-  StringRef FS = TM.getTargetFeatureString();
   const NVPTXTargetMachine &NTM = static_cast<const NVPTXTargetMachine &>(TM);
-  const NVPTXSubtarget STI(TT, CPU, FS, NTM);
+  const auto* STI = static_cast<const NVPTXSubtarget*>(NTM.getSubtargetImpl());
 
   if (M.alias_size()) {
     report_fatal_error("Module has aliases, which NVPTX does not support.");
@@ -816,7 +820,7 @@ bool NVPTXAsmPrinter::doInitialization(Module &M) {
   bool Result = AsmPrinter::doInitialization(M);
 
   // Emit header before any dwarf directives are emitted below.
-  emitHeader(M, OS1, STI);
+  emitHeader(M, OS1, *STI);
   OutStreamer->EmitRawText(OS1.str());
 
   // Emit module-level inline asm if it exists.
@@ -897,9 +901,8 @@ void NVPTXAsmPrinter::emitHeader(Module &M, raw_ostream &O,
     if (HasFullDebugInfo)
       break;
   }
-  // FIXME: remove comment once debug info is properly supported.
   if (MMI && MMI->hasDebugInfo() && HasFullDebugInfo)
-    O << "//, debug";
+    O << ", debug";
 
   O << "\n";
 
@@ -950,10 +953,10 @@ bool NVPTXAsmPrinter::doFinalization(Module &M) {
   clearAnnotationCache(&M);
 
   delete[] gv_array;
-  // FIXME: remove comment once debug info is properly supported.
   // Close the last emitted section
   if (HasDebugInfo)
-    OutStreamer->EmitRawText("//\t}");
+    static_cast<NVPTXTargetStreamer *>(OutStreamer->getTargetStreamer())
+        ->closeLastSection();
 
   // Output last DWARF .file directives, if any.
   static_cast<NVPTXTargetStreamer *>(OutStreamer->getTargetStreamer())
diff --git a/lib/Target/NVPTX/NVPTXAsmPrinter.h b/lib/Target/NVPTX/NVPTXAsmPrinter.h
index 44a09f5fe513039be490bbcd3f2b914b184d3a6b..e1a876d3c6ec59d4a3c85607a2ca1e92333f1f53 100644
--- a/lib/Target/NVPTX/NVPTXAsmPrinter.h
+++ b/lib/Target/NVPTX/NVPTXAsmPrinter.h
@@ -1,9 +1,8 @@
 //===-- NVPTXAsmPrinter.h - NVPTX LLVM assembly writer ----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/NVPTX/NVPTXAssignValidGlobalNames.cpp b/lib/Target/NVPTX/NVPTXAssignValidGlobalNames.cpp
index 41e9ae827180f25b352c79f7143c29b318ec5e56..a8a43cee9ab7648e25c6a0c44b111860af37271d 100644
--- a/lib/Target/NVPTX/NVPTXAssignValidGlobalNames.cpp
+++ b/lib/Target/NVPTX/NVPTXAssignValidGlobalNames.cpp
@@ -1,9 +1,8 @@
 //===-- NVPTXAssignValidGlobalNames.cpp - Assign valid names to globals ---===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/NVPTX/NVPTXFrameLowering.cpp b/lib/Target/NVPTX/NVPTXFrameLowering.cpp
index e5e6637967b2054cde64f5503b1fe96645d400b8..46f08b23d31a63fdba9d45c01d2cf82082ed0580 100644
--- a/lib/Target/NVPTX/NVPTXFrameLowering.cpp
+++ b/lib/Target/NVPTX/NVPTXFrameLowering.cpp
@@ -1,9 +1,8 @@
 //=======- NVPTXFrameLowering.cpp - NVPTX Frame Information ---*- C++ -*-=====//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/NVPTX/NVPTXFrameLowering.h b/lib/Target/NVPTX/NVPTXFrameLowering.h
index 0a7856b9d5de1138ae98380e115652af6c43396e..40269f58f06e7750946fe643e00bc1d4d707ce0d 100644
--- a/lib/Target/NVPTX/NVPTXFrameLowering.h
+++ b/lib/Target/NVPTX/NVPTXFrameLowering.h
@@ -1,9 +1,8 @@
 //===--- NVPTXFrameLowering.h - Define frame lowering for NVPTX -*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/NVPTX/NVPTXGenericToNVVM.cpp b/lib/Target/NVPTX/NVPTXGenericToNVVM.cpp
index fd63fdbaced6662a9c48b303cc723191889e7209..b36d9b2e240a34727980bd17a64b21f5e73d9a25 100644
--- a/lib/Target/NVPTX/NVPTXGenericToNVVM.cpp
+++ b/lib/Target/NVPTX/NVPTXGenericToNVVM.cpp
@@ -1,9 +1,8 @@
 //===-- GenericToNVVM.cpp - Convert generic module to NVVM module - C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp b/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp
index ffc6a59cd6c8772d41484819eb2e46666040aead..d6e44b785ea647a175c30069b781b3c1b75919fe 100644
--- a/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp
+++ b/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp
@@ -1,9 +1,8 @@
 //===-- NVPTXISelDAGToDAG.cpp - A dag to dag inst selector for NVPTX ------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/NVPTX/NVPTXISelDAGToDAG.h b/lib/Target/NVPTX/NVPTXISelDAGToDAG.h
index e911ba0c167d3200681b8d0e61c6fd1e85de7c68..15f230a246a931da5786ed1fef43bf237138c5a1 100644
--- a/lib/Target/NVPTX/NVPTXISelDAGToDAG.h
+++ b/lib/Target/NVPTX/NVPTXISelDAGToDAG.h
@@ -1,9 +1,8 @@
 //===-- NVPTXISelDAGToDAG.h - A dag to dag inst selector for NVPTX --------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/NVPTX/NVPTXISelLowering.cpp b/lib/Target/NVPTX/NVPTXISelLowering.cpp
index 5c16c34e21d8551f1de742ebe6724260e9d0c190..21c0c8c46cb1c4cea04032afa6244b955eba5c63 100644
--- a/lib/Target/NVPTX/NVPTXISelLowering.cpp
+++ b/lib/Target/NVPTX/NVPTXISelLowering.cpp
@@ -1,9 +1,8 @@
 //===-- NVPTXISelLowering.cpp - NVPTX DAG Lowering Implementation ---------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -663,6 +662,8 @@ const char *NVPTXTargetLowering::getTargetNodeName(unsigned Opcode) const {
     return "NVPTXISD::CallSeqEnd";
   case NVPTXISD::CallPrototype:
     return "NVPTXISD::CallPrototype";
+  case NVPTXISD::ProxyReg:
+    return "NVPTXISD::ProxyReg";
   case NVPTXISD::LoadV2:
     return "NVPTXISD::LoadV2";
   case NVPTXISD::LoadV4:
@@ -1666,6 +1667,18 @@ SDValue NVPTXTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
   // indirect calls but is always null for libcalls.
   bool isIndirectCall = !Func && CS;
 
+  if (isa<ExternalSymbolSDNode>(Callee)) {
+    Function* CalleeFunc = nullptr;
+
+    // Try to find the callee in the current module.
+    Callee = DAG.getSymbolFunctionGlobalAddress(Callee, &CalleeFunc);
+    assert(CalleeFunc != nullptr && "Libcall callee must be set.");
+
+    // Set the "libcall callee" attribute to indicate that the function
+    // must always have a declaration.
+    CalleeFunc->addFnAttr("nvptx-libcall-callee", "true");
+  }
+
   if (isIndirectCall) {
     // This is indirect function call case : PTX requires a prototype of the
     // form
@@ -1738,6 +1751,9 @@ SDValue NVPTXTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
     InFlag = Chain.getValue(1);
   }
 
+  SmallVector<SDValue, 16> ProxyRegOps;
+  SmallVector<Optional<MVT>, 16> ProxyRegTruncates;
+
   // Generate loads from param memory/moves from registers for result
   if (Ins.size() > 0) {
     SmallVector<EVT, 16> VTs;
@@ -1808,11 +1824,14 @@ SDValue NVPTXTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
             MachineMemOperand::MOLoad);
 
         for (unsigned j = 0; j < NumElts; ++j) {
-          SDValue Ret = RetVal.getValue(j);
+          ProxyRegOps.push_back(RetVal.getValue(j));
+
           if (needTruncate)
-            Ret = DAG.getNode(ISD::TRUNCATE, dl, Ins[VecIdx + j].VT, Ret);
-          InVals.push_back(Ret);
+            ProxyRegTruncates.push_back(Optional<MVT>(Ins[VecIdx + j].VT));
+          else
+            ProxyRegTruncates.push_back(Optional<MVT>());
         }
+
         Chain = RetVal.getValue(NumElts);
         InFlag = RetVal.getValue(NumElts + 1);
 
@@ -1828,8 +1847,29 @@ SDValue NVPTXTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
                              DAG.getIntPtrConstant(uniqueCallSite + 1, dl,
                                                    true),
                              InFlag, dl);
+  InFlag = Chain.getValue(1);
   uniqueCallSite++;
 
+  // Append ProxyReg instructions to the chain to make sure that `callseq_end`
+  // will not get lost. Otherwise, during libcalls expansion, the nodes can become
+  // dangling.
+  for (unsigned i = 0; i < ProxyRegOps.size(); ++i) {
+    SDValue Ret = DAG.getNode(
+      NVPTXISD::ProxyReg, dl,
+      DAG.getVTList(ProxyRegOps[i].getSimpleValueType(), MVT::Other, MVT::Glue),
+      { Chain, ProxyRegOps[i], InFlag }
+    );
+
+    Chain = Ret.getValue(1);
+    InFlag = Ret.getValue(2);
+
+    if (ProxyRegTruncates[i].hasValue()) {
+      Ret = DAG.getNode(ISD::TRUNCATE, dl, ProxyRegTruncates[i].getValue(), Ret);
+    }
+
+    InVals.push_back(Ret);
+  }
+
   // set isTailCall to false for now, until we figure out how to express
   // tail call optimization in PTX
   isTailCall = false;
diff --git a/lib/Target/NVPTX/NVPTXISelLowering.h b/lib/Target/NVPTX/NVPTXISelLowering.h
index 3e109f75b66815ade98cd746469fa9b883f42c99..bbcc35f49d99c72a1546b28d70f3443645de62e6 100644
--- a/lib/Target/NVPTX/NVPTXISelLowering.h
+++ b/lib/Target/NVPTX/NVPTXISelLowering.h
@@ -1,9 +1,8 @@
 //===-- NVPTXISelLowering.h - NVPTX DAG Lowering Interface ------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -51,6 +50,7 @@ enum NodeType : unsigned {
   CallSeqBegin,
   CallSeqEnd,
   CallPrototype,
+  ProxyReg,
   FUN_SHFL_CLAMP,
   FUN_SHFR_CLAMP,
   MUL_WIDE_SIGNED,
diff --git a/lib/Target/NVPTX/NVPTXImageOptimizer.cpp b/lib/Target/NVPTX/NVPTXImageOptimizer.cpp
index ad1d7cbb52fc2048b6ca411bc1102c5c1859adea..74ab2f7b8453d3799422374dbd632621817ed1bf 100644
--- a/lib/Target/NVPTX/NVPTXImageOptimizer.cpp
+++ b/lib/Target/NVPTX/NVPTXImageOptimizer.cpp
@@ -1,9 +1,8 @@
 //===-- NVPTXImageOptimizer.cpp - Image optimization pass -----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/NVPTX/NVPTXInstrFormats.td b/lib/Target/NVPTX/NVPTXInstrFormats.td
index ffcb5d5273a2a96c4a5f93cda17a1b36674abd9d..77961c3868274161866219149677669eacd8ddf5 100644
--- a/lib/Target/NVPTX/NVPTXInstrFormats.td
+++ b/lib/Target/NVPTX/NVPTXInstrFormats.td
@@ -1,9 +1,8 @@
 //===- NVPTXInstrFormats.td - NVPTX Instruction Formats-------*- tblgen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/NVPTX/NVPTXInstrInfo.cpp b/lib/Target/NVPTX/NVPTXInstrInfo.cpp
index 50815bff6c67bff832961a7a5b7f43e58ece9ccd..f928b44c91e0ba297b80267109a25b1965b00ce5 100644
--- a/lib/Target/NVPTX/NVPTXInstrInfo.cpp
+++ b/lib/Target/NVPTX/NVPTXInstrInfo.cpp
@@ -1,9 +1,8 @@
 //===- NVPTXInstrInfo.cpp - NVPTX Instruction Information -----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/NVPTX/NVPTXInstrInfo.h b/lib/Target/NVPTX/NVPTXInstrInfo.h
index 4ab1bb481958ad27561cde6760d8822a4af976db..7c0912808f7bd68c0515de41612619ea6e13cc79 100644
--- a/lib/Target/NVPTX/NVPTXInstrInfo.h
+++ b/lib/Target/NVPTX/NVPTXInstrInfo.h
@@ -1,9 +1,8 @@
 //===- NVPTXInstrInfo.h - NVPTX Instruction Information----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the niversity of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/NVPTX/NVPTXInstrInfo.td b/lib/Target/NVPTX/NVPTXInstrInfo.td
index 48db941db9b80a7489d833e670a9bf0f891e7b89..1bf556c9287a7c7bb62a8849174b87e6fdcd8943 100644
--- a/lib/Target/NVPTX/NVPTXInstrInfo.td
+++ b/lib/Target/NVPTX/NVPTXInstrInfo.td
@@ -1,9 +1,8 @@
 //===- NVPTXInstrInfo.td - NVPTX Instruction defs -------------*- tblgen-*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -1885,6 +1884,7 @@ def SDTStoreRetvalProfile : SDTypeProfile<0, 2, [SDTCisInt<0>]>;
 def SDTStoreRetvalV2Profile : SDTypeProfile<0, 3, [SDTCisInt<0>]>;
 def SDTStoreRetvalV4Profile : SDTypeProfile<0, 5, [SDTCisInt<0>]>;
 def SDTPseudoUseParamProfile : SDTypeProfile<0, 1, []>;
+def SDTProxyRegProfile : SDTypeProfile<1, 1, []>;
 
 def DeclareParam :
   SDNode<"NVPTXISD::DeclareParam", SDTDeclareParamProfile,
@@ -1972,6 +1972,9 @@ def PseudoUseParam :
 def RETURNNode :
   SDNode<"NVPTXISD::RETURN", SDTCallArgMarkProfile,
          [SDNPHasChain, SDNPSideEffect]>;
+def ProxyReg :
+  SDNode<"NVPTXISD::ProxyReg", SDTProxyRegProfile,
+         [SDNPHasChain, SDNPOutGlue, SDNPInGlue, SDNPSideEffect]>;
 
 let mayLoad = 1 in {
   class LoadParamMemInst<NVPTXRegClass regclass, string opstr> :
@@ -2249,6 +2252,21 @@ def PseudoUseParamI16 : PseudoUseParamInst<Int16Regs>;
 def PseudoUseParamF64 : PseudoUseParamInst<Float64Regs>;
 def PseudoUseParamF32 : PseudoUseParamInst<Float32Regs>;
 
+class ProxyRegInst<string SzStr, NVPTXRegClass regclass> :
+  NVPTXInst<(outs regclass:$dst), (ins regclass:$src),
+            !strconcat("mov.", SzStr, " \t$dst, $src;"),
+            [(set regclass:$dst, (ProxyReg regclass:$src))]>;
+
+let isCodeGenOnly=1, isPseudo=1 in {
+  def ProxyRegI1    : ProxyRegInst<"pred", Int1Regs>;
+  def ProxyRegI16   : ProxyRegInst<"b16",  Int16Regs>;
+  def ProxyRegI32   : ProxyRegInst<"b32",  Int32Regs>;
+  def ProxyRegI64   : ProxyRegInst<"b64",  Int64Regs>;
+  def ProxyRegF16   : ProxyRegInst<"b16",  Float16Regs>;
+  def ProxyRegF32   : ProxyRegInst<"f32",  Float32Regs>;
+  def ProxyRegF64   : ProxyRegInst<"f64",  Float64Regs>;
+  def ProxyRegF16x2 : ProxyRegInst<"b32",  Float16x2Regs>;
+}
 
 //
 // Load / Store Handling
@@ -2541,7 +2559,7 @@ let mayStore=1, hasSideEffects=0 in {
 class F_BITCONVERT<string SzStr, NVPTXRegClass regclassIn,
   NVPTXRegClass regclassOut> :
            NVPTXInst<(outs regclassOut:$d), (ins regclassIn:$a),
-           !strconcat("mov.b", !strconcat(SzStr, " \t$d, $a;")),
+           !strconcat("mov.b", SzStr, " \t$d, $a;"),
      [(set regclassOut:$d, (bitconvert regclassIn:$a))]>;
 
 def BITCONVERT_16_I2F : F_BITCONVERT<"16", Int16Regs, Float16Regs>;
diff --git a/lib/Target/NVPTX/NVPTXIntrinsics.td b/lib/Target/NVPTX/NVPTXIntrinsics.td
index 47dcdcf6e0bdca42eaa1e7ab4753479b5633ccd1..8f9251bb0dfed69a7b785336e58841fa03c8d07f 100644
--- a/lib/Target/NVPTX/NVPTXIntrinsics.td
+++ b/lib/Target/NVPTX/NVPTXIntrinsics.td
@@ -1,9 +1,8 @@
 //===- NVPTXIntrinsics.td - PTX Intrinsics Instructions -------*- tblgen -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/NVPTX/NVPTXLowerAggrCopies.cpp b/lib/Target/NVPTX/NVPTXLowerAggrCopies.cpp
index 52ced266b91c4421ae167dbdc78d11a648729b22..0743a2986718adfd7120585aaab6c071f66caea5 100644
--- a/lib/Target/NVPTX/NVPTXLowerAggrCopies.cpp
+++ b/lib/Target/NVPTX/NVPTXLowerAggrCopies.cpp
@@ -1,9 +1,8 @@
 //===- NVPTXLowerAggrCopies.cpp - ------------------------------*- C++ -*--===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/NVPTX/NVPTXLowerAggrCopies.h b/lib/Target/NVPTX/NVPTXLowerAggrCopies.h
index 3c39f53eb30a7cb75a51c34295d11497878314b8..59d5ef40e9ac96e2564a8fcd4a4388b60b2886ed 100644
--- a/lib/Target/NVPTX/NVPTXLowerAggrCopies.h
+++ b/lib/Target/NVPTX/NVPTXLowerAggrCopies.h
@@ -1,9 +1,8 @@
 //===-- llvm/lib/Target/NVPTX/NVPTXLowerAggrCopies.h ------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/NVPTX/NVPTXLowerAlloca.cpp b/lib/Target/NVPTX/NVPTXLowerAlloca.cpp
index e94c1914029dfffeab28f35afa8e8a749d4b5290..5720ea2afdd171b0cee20c314c6da3f838563686 100644
--- a/lib/Target/NVPTX/NVPTXLowerAlloca.cpp
+++ b/lib/Target/NVPTX/NVPTXLowerAlloca.cpp
@@ -1,9 +1,8 @@
 //===-- NVPTXLowerAlloca.cpp - Make alloca to use local memory =====--===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/NVPTX/NVPTXLowerArgs.cpp b/lib/Target/NVPTX/NVPTXLowerArgs.cpp
index 139dc7fbeeda15f8aa30f7858d39b4f289771ecf..8c7f6e63e1f438d086c3735ab6fc2ca43c63975e 100644
--- a/lib/Target/NVPTX/NVPTXLowerArgs.cpp
+++ b/lib/Target/NVPTX/NVPTXLowerArgs.cpp
@@ -1,9 +1,8 @@
 //===-- NVPTXLowerArgs.cpp - Lower arguments ------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -170,7 +169,8 @@ void NVPTXLowerArgs::handleByValParam(Argument *Arg) {
   Value *ArgInParam = new AddrSpaceCastInst(
       Arg, PointerType::get(StructType, ADDRESS_SPACE_PARAM), Arg->getName(),
       FirstInst);
-  LoadInst *LI = new LoadInst(ArgInParam, Arg->getName(), FirstInst);
+  LoadInst *LI =
+      new LoadInst(StructType, ArgInParam, Arg->getName(), FirstInst);
   new StoreInst(LI, AllocA, FirstInst);
 }
 
diff --git a/lib/Target/NVPTX/NVPTXMCExpr.cpp b/lib/Target/NVPTX/NVPTXMCExpr.cpp
index a754a6a36dab77356768bcc43afef955322d61c8..5ec1b2425e68fc6e98b66a62b3747de96e5846cc 100644
--- a/lib/Target/NVPTX/NVPTXMCExpr.cpp
+++ b/lib/Target/NVPTX/NVPTXMCExpr.cpp
@@ -1,9 +1,8 @@
 //===-- NVPTXMCExpr.cpp - NVPTX specific MC expression classes ------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/NVPTX/NVPTXMCExpr.h b/lib/Target/NVPTX/NVPTXMCExpr.h
index 95741d9b0451827ad1f92dd13bbacfd00ede297c..440fa1310003e0b479d6eed3bab6250dda6100e7 100644
--- a/lib/Target/NVPTX/NVPTXMCExpr.h
+++ b/lib/Target/NVPTX/NVPTXMCExpr.h
@@ -1,9 +1,8 @@
 //===-- NVPTXMCExpr.h - NVPTX specific MC expression classes ----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/NVPTX/NVPTXMachineFunctionInfo.h b/lib/Target/NVPTX/NVPTXMachineFunctionInfo.h
index 5a9115f6f7f1d543e5b254bd880fbbf9104cb0c1..cf63fc33e62129bb6038acbcade560a48bc500f7 100644
--- a/lib/Target/NVPTX/NVPTXMachineFunctionInfo.h
+++ b/lib/Target/NVPTX/NVPTXMachineFunctionInfo.h
@@ -1,9 +1,8 @@
 //===-- NVPTXMachineFunctionInfo.h - NVPTX-specific Function Info  --------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/NVPTX/NVPTXPeephole.cpp b/lib/Target/NVPTX/NVPTXPeephole.cpp
index 02c32c68ee2ccd35b379ebe20874730124ddde57..629757db87076bec2e9ecbad57ae656da590ea5b 100644
--- a/lib/Target/NVPTX/NVPTXPeephole.cpp
+++ b/lib/Target/NVPTX/NVPTXPeephole.cpp
@@ -1,9 +1,8 @@
 //===-- NVPTXPeephole.cpp - NVPTX Peephole Optimiztions -------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/NVPTX/NVPTXPrologEpilogPass.cpp b/lib/Target/NVPTX/NVPTXPrologEpilogPass.cpp
index 2ca0ccf2dfa7c43f78a006cbe66614df788c1002..9f315c8c1af407c2cd6c26ccdbf9333ac2bb8686 100644
--- a/lib/Target/NVPTX/NVPTXPrologEpilogPass.cpp
+++ b/lib/Target/NVPTX/NVPTXPrologEpilogPass.cpp
@@ -1,9 +1,8 @@
 //===-- NVPTXPrologEpilogPass.cpp - NVPTX prolog/epilog inserter ----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/NVPTX/NVPTXProxyRegErasure.cpp b/lib/Target/NVPTX/NVPTXProxyRegErasure.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..af50a7465d1a7893e3d6881ef1b28767aa4c7f99
--- /dev/null
+++ b/lib/Target/NVPTX/NVPTXProxyRegErasure.cpp
@@ -0,0 +1,121 @@
+//===- NVPTXProxyRegErasure.cpp - NVPTX Proxy Register Instruction Erasure -==//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// The pass is needed to remove ProxyReg instructions and restore related
+// registers. The instructions were needed at instruction selection stage to
+// make sure that callseq_end nodes won't be removed as "dead nodes". This can
+// happen when we expand instructions into libcalls and the call site doesn't
+// care about the libcall chain. Call site cares about data flow only, and the
+// latest data flow node happens to be before callseq_end. Therefore the node
+// becomes dangling and "dead". The ProxyReg acts like an additional data flow
+// node *after* the callseq_end in the chain and ensures that everything will be
+// preserved.
+//
+//===----------------------------------------------------------------------===//
+
+#include "NVPTX.h"
+#include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/CodeGen/MachineInstrBuilder.h"
+#include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/CodeGen/TargetInstrInfo.h"
+#include "llvm/CodeGen/TargetRegisterInfo.h"
+
+using namespace llvm;
+
+namespace llvm {
+void initializeNVPTXProxyRegErasurePass(PassRegistry &);
+}
+
+namespace {
+
+struct NVPTXProxyRegErasure : public MachineFunctionPass {
+public:
+  static char ID;
+  NVPTXProxyRegErasure() : MachineFunctionPass(ID) {
+    initializeNVPTXProxyRegErasurePass(*PassRegistry::getPassRegistry());
+  }
+
+  bool runOnMachineFunction(MachineFunction &MF) override;
+
+  StringRef getPassName() const override {
+    return "NVPTX Proxy Register Instruction Erasure";
+  }
+
+  void getAnalysisUsage(AnalysisUsage &AU) const override {
+    MachineFunctionPass::getAnalysisUsage(AU);
+  }
+
+private:
+  void replaceMachineInstructionUsage(MachineFunction &MF, MachineInstr &MI);
+
+  void replaceRegisterUsage(MachineInstr &Instr, MachineOperand &From,
+                            MachineOperand &To);
+};
+
+} // namespace
+
+char NVPTXProxyRegErasure::ID = 0;
+
+INITIALIZE_PASS(NVPTXProxyRegErasure, "nvptx-proxyreg-erasure", "NVPTX ProxyReg Erasure", false, false)
+
+bool NVPTXProxyRegErasure::runOnMachineFunction(MachineFunction &MF) {
+  SmallVector<MachineInstr *, 16> RemoveList;
+
+  for (auto &BB : MF) {
+    for (auto &MI : BB) {
+      switch (MI.getOpcode()) {
+      case NVPTX::ProxyRegI1:
+      case NVPTX::ProxyRegI16:
+      case NVPTX::ProxyRegI32:
+      case NVPTX::ProxyRegI64:
+      case NVPTX::ProxyRegF16:
+      case NVPTX::ProxyRegF16x2:
+      case NVPTX::ProxyRegF32:
+      case NVPTX::ProxyRegF64:
+        replaceMachineInstructionUsage(MF, MI);
+        RemoveList.push_back(&MI);
+        break;
+      }
+    }
+  }
+
+  for (auto *MI : RemoveList) {
+    MI->eraseFromParent();
+  }
+
+  return !RemoveList.empty();
+}
+
+void NVPTXProxyRegErasure::replaceMachineInstructionUsage(MachineFunction &MF,
+                                                          MachineInstr &MI) {
+  auto &InOp = *MI.uses().begin();
+  auto &OutOp = *MI.defs().begin();
+
+  assert(InOp.isReg() && "ProxyReg input operand should be a register.");
+  assert(OutOp.isReg() && "ProxyReg output operand should be a register.");
+
+  for (auto &BB : MF) {
+    for (auto &I : BB) {
+      replaceRegisterUsage(I, OutOp, InOp);
+    }
+  }
+}
+
+void NVPTXProxyRegErasure::replaceRegisterUsage(MachineInstr &Instr,
+                                                MachineOperand &From,
+                                                MachineOperand &To) {
+  for (auto &Op : Instr.uses()) {
+    if (Op.isReg() && Op.getReg() == From.getReg()) {
+      Op.setReg(To.getReg());
+    }
+  }
+}
+
+MachineFunctionPass *llvm::createNVPTXProxyRegErasurePass() {
+  return new NVPTXProxyRegErasure();
+}
diff --git a/lib/Target/NVPTX/NVPTXRegisterInfo.cpp b/lib/Target/NVPTX/NVPTXRegisterInfo.cpp
index 75573832988131d8d8e0bb73893913a7d7658265..a124ac14d25b3b6b11d6e58024605f2eb13ca10b 100644
--- a/lib/Target/NVPTX/NVPTXRegisterInfo.cpp
+++ b/lib/Target/NVPTX/NVPTXRegisterInfo.cpp
@@ -1,9 +1,8 @@
 //===- NVPTXRegisterInfo.cpp - NVPTX Register Information -----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/NVPTX/NVPTXRegisterInfo.h b/lib/Target/NVPTX/NVPTXRegisterInfo.h
index 6185a0b54cacad6877cd734e6ee3da9776e010be..8dcf47c159bd3972b8105688ba8ebaafc49a569b 100644
--- a/lib/Target/NVPTX/NVPTXRegisterInfo.h
+++ b/lib/Target/NVPTX/NVPTXRegisterInfo.h
@@ -1,9 +1,8 @@
 //===- NVPTXRegisterInfo.h - NVPTX Register Information Impl ----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/NVPTX/NVPTXRegisterInfo.td b/lib/Target/NVPTX/NVPTXRegisterInfo.td
index f04764a9e9a39a36356623e881b859ac7edf541c..4b755dcb55ffc7769603dc5e0355db0e4a8a122d 100644
--- a/lib/Target/NVPTX/NVPTXRegisterInfo.td
+++ b/lib/Target/NVPTX/NVPTXRegisterInfo.td
@@ -1,9 +1,8 @@
 //===-- NVPTXRegisterInfo.td - NVPTX Register defs ---------*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/NVPTX/NVPTXReplaceImageHandles.cpp b/lib/Target/NVPTX/NVPTXReplaceImageHandles.cpp
index 82befe4b101b2ab105255c398005f9add4423caf..d08f8cc16c96cfd877611879d498181d3e7ebb95 100644
--- a/lib/Target/NVPTX/NVPTXReplaceImageHandles.cpp
+++ b/lib/Target/NVPTX/NVPTXReplaceImageHandles.cpp
@@ -1,9 +1,8 @@
 //===-- NVPTXReplaceImageHandles.cpp - Replace image handles for Fermi ----===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/NVPTX/NVPTXSubtarget.cpp b/lib/Target/NVPTX/NVPTXSubtarget.cpp
index acbee86ae386b29a20f646e5e76a5e0e93826df0..357826c2d19ca9edfbb8cc815a216002a10f36be 100644
--- a/lib/Target/NVPTX/NVPTXSubtarget.cpp
+++ b/lib/Target/NVPTX/NVPTXSubtarget.cpp
@@ -1,9 +1,8 @@
 //===- NVPTXSubtarget.cpp - NVPTX Subtarget Information -------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/NVPTX/NVPTXSubtarget.h b/lib/Target/NVPTX/NVPTXSubtarget.h
index b02822a099d97e0fbad333e89af57d4f6630e1ab..0e9fa1fd3e56eb5293785d8357aaa3d02dadfa20 100644
--- a/lib/Target/NVPTX/NVPTXSubtarget.h
+++ b/lib/Target/NVPTX/NVPTXSubtarget.h
@@ -1,9 +1,8 @@
 //=====-- NVPTXSubtarget.h - Define Subtarget for the NVPTX ---*- C++ -*--====//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/NVPTX/NVPTXTargetMachine.cpp b/lib/Target/NVPTX/NVPTXTargetMachine.cpp
index 8c009aed887706c1238c328c604d7a6cc09a9824..eb8e40a083b54afdd00e9d7b377d2021083c6821 100644
--- a/lib/Target/NVPTX/NVPTXTargetMachine.cpp
+++ b/lib/Target/NVPTX/NVPTXTargetMachine.cpp
@@ -1,9 +1,8 @@
 //===-- NVPTXTargetMachine.cpp - Define TargetMachine for NVPTX -----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -68,6 +67,7 @@ void initializeNVPTXAssignValidGlobalNamesPass(PassRegistry&);
 void initializeNVPTXLowerAggrCopiesPass(PassRegistry &);
 void initializeNVPTXLowerArgsPass(PassRegistry &);
 void initializeNVPTXLowerAllocaPass(PassRegistry &);
+void initializeNVPTXProxyRegErasurePass(PassRegistry &);
 
 } // end namespace llvm
 
@@ -87,6 +87,7 @@ extern "C" void LLVMInitializeNVPTXTarget() {
   initializeNVPTXLowerArgsPass(PR);
   initializeNVPTXLowerAllocaPass(PR);
   initializeNVPTXLowerAggrCopiesPass(PR);
+  initializeNVPTXProxyRegErasurePass(PR);
 }
 
 static std::string computeDataLayout(bool is64Bit, bool UseShortPointers) {
@@ -160,6 +161,7 @@ public:
 
   void addIRPasses() override;
   bool addInstSelector() override;
+  void addPreRegAlloc() override;
   void addPostRegAlloc() override;
   void addMachineSSAOptimization() override;
 
@@ -301,6 +303,11 @@ bool NVPTXPassConfig::addInstSelector() {
   return false;
 }
 
+void NVPTXPassConfig::addPreRegAlloc() {
+  // Remove Proxy Register pseudo instructions used to keep `callseq_end` alive.
+  addPass(createNVPTXProxyRegErasurePass());
+}
+
 void NVPTXPassConfig::addPostRegAlloc() {
   addPass(createNVPTXPrologEpilogPass(), false);
   if (getOptLevel() != CodeGenOpt::None) {
diff --git a/lib/Target/NVPTX/NVPTXTargetMachine.h b/lib/Target/NVPTX/NVPTXTargetMachine.h
index ca540b8e0389c5f386db69d33ca0bf6d31517f73..d84600c74e292bccf411260671e87448d6a65bdb 100644
--- a/lib/Target/NVPTX/NVPTXTargetMachine.h
+++ b/lib/Target/NVPTX/NVPTXTargetMachine.h
@@ -1,9 +1,8 @@
 //===-- NVPTXTargetMachine.h - Define TargetMachine for NVPTX ---*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/NVPTX/NVPTXTargetObjectFile.h b/lib/Target/NVPTX/NVPTXTargetObjectFile.h
index c706b053ab8fa203fabe36eeb87ef97d5dc4ec32..ab2a93b759227d34282cc73c4c2151322864bf28 100644
--- a/lib/Target/NVPTX/NVPTXTargetObjectFile.h
+++ b/lib/Target/NVPTX/NVPTXTargetObjectFile.h
@@ -1,9 +1,8 @@
 //===-- NVPTXTargetObjectFile.h - NVPTX Object Info -------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/NVPTX/NVPTXTargetTransformInfo.cpp b/lib/Target/NVPTX/NVPTXTargetTransformInfo.cpp
index 307654aed37f4d2843e70ec06cb3d1fcf666ccd8..6bee8fdbf5b3b2a9d85da47f2e99c9072bc13bfe 100644
--- a/lib/Target/NVPTX/NVPTXTargetTransformInfo.cpp
+++ b/lib/Target/NVPTX/NVPTXTargetTransformInfo.cpp
@@ -1,9 +1,8 @@
 //===-- NVPTXTargetTransformInfo.cpp - NVPTX specific TTI -----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/NVPTX/NVPTXTargetTransformInfo.h b/lib/Target/NVPTX/NVPTXTargetTransformInfo.h
index 14e93f7447dda517bce81bd360f8c64f59bc4c58..b4a06034a9d6dd5a9001e5d5eeaebe8d6cd37b4c 100644
--- a/lib/Target/NVPTX/NVPTXTargetTransformInfo.h
+++ b/lib/Target/NVPTX/NVPTXTargetTransformInfo.h
@@ -1,9 +1,8 @@
 //===-- NVPTXTargetTransformInfo.h - NVPTX specific TTI ---------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/lib/Target/NVPTX/NVPTXUtilities.cpp b/lib/Target/NVPTX/NVPTXUtilities.cpp
index e464f474b1d5a05f7a20c38a0b20cc56e0ea9bb3..22648589733b8fd1ec0ec217100e298594515eef 100644
--- a/lib/Target/NVPTX/NVPTXUtilities.cpp
+++ b/lib/Target/NVPTX/NVPTXUtilities.cpp
@@ -1,9 +1,8 @@
 //===- NVPTXUtilities.cpp - Utility Functions -----------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/NVPTX/NVPTXUtilities.h b/lib/Target/NVPTX/NVPTXUtilities.h
index a0cc4e78ac21dc927ddf7903beea2b33fb8fb9c1..bf1524194cfb992ec9649c12f542f2ef990938a7 100644
--- a/lib/Target/NVPTX/NVPTXUtilities.h
+++ b/lib/Target/NVPTX/NVPTXUtilities.h
@@ -1,9 +1,8 @@
 //===-- NVPTXUtilities - Utilities -----------------------------*- C++ -*-====//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/NVPTX/NVVMIntrRange.cpp b/lib/Target/NVPTX/NVVMIntrRange.cpp
index 11277f5ba5966114e114e22eecdc6cdaf5922ec8..5cf7b6691e6381aa1fdb406040fe6d3435e669cd 100644
--- a/lib/Target/NVPTX/NVVMIntrRange.cpp
+++ b/lib/Target/NVPTX/NVVMIntrRange.cpp
@@ -1,9 +1,8 @@
 //===- NVVMIntrRange.cpp - Set !range metadata for NVVM intrinsics --------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/NVPTX/NVVMReflect.cpp b/lib/Target/NVPTX/NVVMReflect.cpp
index 64c262664fda87f901705042df6e62d74916bdd4..634a052e2ee728d20342db7be0a5cd60e38c9d80 100644
--- a/lib/Target/NVPTX/NVVMReflect.cpp
+++ b/lib/Target/NVPTX/NVVMReflect.cpp
@@ -1,9 +1,8 @@
 //===- NVVMReflect.cpp - NVVM Emulate conditional compilation -------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/NVPTX/TargetInfo/LLVMBuild.txt b/lib/Target/NVPTX/TargetInfo/LLVMBuild.txt
index af7ec27278bc430773d5951dc117dce46b2f5ea5..fa5ac6901390c08e526a5d4aef0d6a1770943fb9 100644
--- a/lib/Target/NVPTX/TargetInfo/LLVMBuild.txt
+++ b/lib/Target/NVPTX/TargetInfo/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/Target/NVPTX/TargetInfo/LLVMBuild.txt --------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Target/NVPTX/TargetInfo/NVPTXTargetInfo.cpp b/lib/Target/NVPTX/TargetInfo/NVPTXTargetInfo.cpp
index 803d643844f89a3106d9f1e21ebd7bdffcca59b5..24ee1e943bba7a482f7c371e4c595de72b936188 100644
--- a/lib/Target/NVPTX/TargetInfo/NVPTXTargetInfo.cpp
+++ b/lib/Target/NVPTX/TargetInfo/NVPTXTargetInfo.cpp
@@ -1,9 +1,8 @@
 //===-- NVPTXTargetInfo.cpp - NVPTX Target Implementation -----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/Nios2/CMakeLists.txt b/lib/Target/Nios2/CMakeLists.txt
deleted file mode 100644
index 6393cc5fcb92a5f8cac8d98fd9f0326d7f71b754..0000000000000000000000000000000000000000
--- a/lib/Target/Nios2/CMakeLists.txt
+++ /dev/null
@@ -1,30 +0,0 @@
-set(LLVM_TARGET_DEFINITIONS Nios2.td)
-
-tablegen(LLVM Nios2GenAsmWriter.inc -gen-asm-writer)
-tablegen(LLVM Nios2GenCallingConv.inc -gen-callingconv)
-tablegen(LLVM Nios2GenDAGISel.inc -gen-dag-isel)
-tablegen(LLVM Nios2GenInstrInfo.inc -gen-instr-info)
-tablegen(LLVM Nios2GenRegisterInfo.inc -gen-register-info)
-tablegen(LLVM Nios2GenSubtargetInfo.inc -gen-subtarget)
-
-add_public_tablegen_target(Nios2CommonTableGen)
-
-#Nios2CodeGen should match with LLVMBuild.txt Nios2CodeGen
-add_llvm_target(Nios2CodeGen
- Nios2AsmPrinter.cpp
- Nios2FrameLowering.cpp
- Nios2InstrInfo.cpp
- Nios2ISelDAGToDAG.cpp
- Nios2ISelLowering.cpp
- Nios2MachineFunction.cpp
- Nios2MCInstLower.cpp
- Nios2RegisterInfo.cpp
- Nios2Subtarget.cpp
- Nios2TargetMachine.cpp
- Nios2TargetObjectFile.cpp
- )
-
-#Should match with "subdirectories =  InstPrinter MCTargetDesc TargetInfo" in LLVMBuild.txt
-add_subdirectory(InstPrinter)
-add_subdirectory(MCTargetDesc)
-add_subdirectory(TargetInfo)
diff --git a/lib/Target/Nios2/InstPrinter/CMakeLists.txt b/lib/Target/Nios2/InstPrinter/CMakeLists.txt
deleted file mode 100644
index dc50be755754a647b835d4fe6b6870ceeb0ee3b3..0000000000000000000000000000000000000000
--- a/lib/Target/Nios2/InstPrinter/CMakeLists.txt
+++ /dev/null
@@ -1 +0,0 @@
-add_llvm_library(LLVMNios2AsmPrinter Nios2InstPrinter.cpp)
diff --git a/lib/Target/Nios2/InstPrinter/Nios2InstPrinter.cpp b/lib/Target/Nios2/InstPrinter/Nios2InstPrinter.cpp
deleted file mode 100644
index de0a5f9e84ea7866709a10bc4ce4098a764387cd..0000000000000000000000000000000000000000
--- a/lib/Target/Nios2/InstPrinter/Nios2InstPrinter.cpp
+++ /dev/null
@@ -1,66 +0,0 @@
-//===-- Nios2InstPrinter.cpp - Convert Nios2 MCInst to assembly syntax-----===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This class prints an Nios2 MCInst to a .s file.
-//
-//===----------------------------------------------------------------------===//
-
-#include "Nios2InstPrinter.h"
-
-#include "Nios2InstrInfo.h"
-#include "llvm/MC/MCExpr.h"
-#include "llvm/MC/MCInst.h"
-#include "llvm/MC/MCInstrInfo.h"
-#include "llvm/Support/raw_ostream.h"
-using namespace llvm;
-
-#define DEBUG_TYPE "asm-printer"
-
-#define PRINT_ALIAS_INSTR
-#include "Nios2GenAsmWriter.inc"
-
-void Nios2InstPrinter::printRegName(raw_ostream &OS, unsigned RegNo) const {
-  OS << getRegisterName(RegNo);
-}
-
-void Nios2InstPrinter::printInst(const MCInst *MI, raw_ostream &O,
-                                 StringRef Annot, const MCSubtargetInfo &STI) {
-  // Try to print any aliases first.
-  if (!printAliasInstr(MI, STI, O))
-    printInstruction(MI, STI, O);
-  printAnnotation(O, Annot);
-}
-
-void Nios2InstPrinter::printOperand(const MCInst *MI, int OpNo,
-                                    const MCSubtargetInfo &STI,
-                                    raw_ostream &O) {
-  const MCOperand &Op = MI->getOperand(OpNo);
-  if (Op.isReg()) {
-    printRegName(O, Op.getReg());
-    return;
-  }
-
-  if (Op.isImm()) {
-    O << Op.getImm();
-    return;
-  }
-
-  assert(Op.isExpr() && "unknown operand kind in printOperand");
-  Op.getExpr()->print(O, &MAI, true);
-}
-
-void Nios2InstPrinter::printMemOperand(const MCInst *MI, int opNum,
-                                       const MCSubtargetInfo &STI,
-                                       raw_ostream &O, const char *Modifier) {
-  // Load/Store memory operands -- imm($reg)
-  printOperand(MI, opNum + 1, STI, O);
-  O << "(";
-  printOperand(MI, opNum, STI, O);
-  O << ")";
-}
diff --git a/lib/Target/Nios2/InstPrinter/Nios2InstPrinter.h b/lib/Target/Nios2/InstPrinter/Nios2InstPrinter.h
deleted file mode 100644
index 43a12951baea9f1e3a5f55e334d69d91d6ecf659..0000000000000000000000000000000000000000
--- a/lib/Target/Nios2/InstPrinter/Nios2InstPrinter.h
+++ /dev/null
@@ -1,49 +0,0 @@
-//= Nios2InstPrinter.h - Convert Nios2 MCInst to assembly syntax -*- C++ -*-==//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This class prints a Nios2 MCInst to a .s file.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_LIB_TARGET_NIOS2_INSTPRINTER_NIOS2INSTPRINTER_H
-#define LLVM_LIB_TARGET_NIOS2_INSTPRINTER_NIOS2INSTPRINTER_H
-
-#include "llvm/MC/MCInstPrinter.h"
-
-namespace llvm {
-
-class Nios2InstPrinter : public MCInstPrinter {
-public:
-  Nios2InstPrinter(const MCAsmInfo &MAI, const MCInstrInfo &MII,
-                   const MCRegisterInfo &MRI)
-      : MCInstPrinter(MAI, MII, MRI) {}
-
-  void printRegName(raw_ostream &OS, unsigned RegNo) const override;
-  void printInst(const MCInst *MI, raw_ostream &O, StringRef Annot,
-                 const MCSubtargetInfo &STI) override;
-
-  // Autogenerated by tblgen.
-  void printInstruction(const MCInst *MI, const MCSubtargetInfo &STI,
-                        raw_ostream &O);
-  static const char *getRegisterName(unsigned RegNo);
-
-  bool printAliasInstr(const MCInst *MI, const MCSubtargetInfo &STI,
-                       raw_ostream &O);
-
-  void printCustomAliasOperand(const MCInst *MI, unsigned OpIdx,
-                               unsigned PrintMethodIdx,
-                               const MCSubtargetInfo &STI, raw_ostream &O);
-  void printOperand(const MCInst *MI, int opNum, const MCSubtargetInfo &STI,
-                    raw_ostream &OS);
-  void printMemOperand(const MCInst *MI, int opNum, const MCSubtargetInfo &STI,
-                       raw_ostream &OS, const char *Modifier = nullptr);
-};
-} // end namespace llvm
-
-#endif
diff --git a/lib/Target/Nios2/LLVMBuild.txt b/lib/Target/Nios2/LLVMBuild.txt
deleted file mode 100644
index 46dd93308c0412da965ee365a6198500124af7df..0000000000000000000000000000000000000000
--- a/lib/Target/Nios2/LLVMBuild.txt
+++ /dev/null
@@ -1,66 +0,0 @@
-;===- ./lib/Target/Nios2/LLVMBuild.txt -------------------------*- Conf -*--===;
-;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
-;
-;===------------------------------------------------------------------------===;
-;
-; This is an LLVMBuild description file for the components in this subdirectory.
-;
-; For more information on the LLVMBuild system, please see:
-;
-;   http://llvm.org/docs/LLVMBuild.html
-;
-;===------------------------------------------------------------------------===;
-
-#Following comments extracted from http: // llvm.org/docs/LLVMBuild.html
-
-[common]
-subdirectories =
-    InstPrinter 
-    MCTargetDesc
-    TargetInfo
-
-[component_0]
-#TargetGroup components are an extension of LibraryGroups, specifically for
-#defining LLVM targets(which are handled specially in a few places).
-type = TargetGroup
-#The name of the component should always be the name of the target.(should
-#match "def Nios2 : Target" in Nios2.td)
-name = Nios2
-#Nios2 component is located in directory Target /
-parent = Target
-#Whether this target defines an assembly parser, assembly printer, disassembler
-#, and supports JIT compilation.They are optional.
-has_asmprinter = 1
-
-[component_1]
-#component_1 is a Library type and name is Nios2CodeGen.After build it will
-#in lib / libLLVMNios2CodeGen.a of your build command directory.
-type = Library
-name = Nios2CodeGen
-#Nios2CodeGen component(Library) is located in directory Nios2 /
-parent = Nios2
-#If given, a list of the names of Library or LibraryGroup components which
-#must also be linked in whenever this library is used.That is, the link time
-#dependencies for this component.When tools are built, the build system will
-#include the transitive closure of all required_libraries for the components
-#the tool needs.
-required_libraries = AsmPrinter
-                     CodeGen
-                     Core
-                     GlobalISel
-                     MC
-                     Nios2AsmPrinter
-                     Nios2Desc
-                     Nios2Info
-                     SelectionDAG
-                     Support
-                     Target
-#end of required_libraries
-
-#All LLVMBuild.txt in Target / Nios2 and subdirectory use 'add_to_library_groups
-#= Nios2'
-add_to_library_groups = Nios2
diff --git a/lib/Target/Nios2/MCTargetDesc/CMakeLists.txt b/lib/Target/Nios2/MCTargetDesc/CMakeLists.txt
deleted file mode 100644
index 138832d33abf33e7bd164f9e71a34e0c38a7aa3e..0000000000000000000000000000000000000000
--- a/lib/Target/Nios2/MCTargetDesc/CMakeLists.txt
+++ /dev/null
@@ -1,9 +0,0 @@
-#MCTargetDesc / CMakeLists.txt
-add_llvm_library(LLVMNios2Desc
-                 Nios2AsmBackend.cpp
-                 Nios2ELFObjectWriter.cpp
-                 Nios2MCAsmInfo.cpp
-                 Nios2MCExpr.cpp
-                 Nios2MCTargetDesc.cpp
-                 Nios2TargetStreamer.cpp)
-
diff --git a/lib/Target/Nios2/MCTargetDesc/LLVMBuild.txt b/lib/Target/Nios2/MCTargetDesc/LLVMBuild.txt
deleted file mode 100644
index 3794c83e504d4371f2556bae5002b05e4a747b9d..0000000000000000000000000000000000000000
--- a/lib/Target/Nios2/MCTargetDesc/LLVMBuild.txt
+++ /dev/null
@@ -1,26 +0,0 @@
-;===- ./lib/Target/Nios2/MCTargetDesc/LLVMBuild.txt ------------*- Conf -*--===;
-;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
-;
-;===------------------------------------------------------------------------===;
-;
-; This is an LLVMBuild description file for the components in this subdirectory.
-;
-; For more information on the LLVMBuild system, please see:
-;
-;   http://llvm.org/docs/LLVMBuild.html
-;
-;===------------------------------------------------------------------------===;
-
-[component_0]
-type = Library
-name = Nios2Desc
-parent = Nios2
-required_libraries = MC
-                     Nios2AsmPrinter
-                     Nios2Info
-                     Support
-add_to_library_groups = Nios2
diff --git a/lib/Target/Nios2/MCTargetDesc/Nios2AsmBackend.cpp b/lib/Target/Nios2/MCTargetDesc/Nios2AsmBackend.cpp
deleted file mode 100644
index 8ac08c6837d9c9227e11c70f09c1b670d355603f..0000000000000000000000000000000000000000
--- a/lib/Target/Nios2/MCTargetDesc/Nios2AsmBackend.cpp
+++ /dev/null
@@ -1,130 +0,0 @@
-//===-- Nios2AsmBackend.cpp - Nios2 Asm Backend  --------------------------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This file implements the Nios2AsmBackend class.
-//
-//===----------------------------------------------------------------------===//
-//
-
-#include "MCTargetDesc/Nios2AsmBackend.h"
-#include "MCTargetDesc/Nios2FixupKinds.h"
-#include "MCTargetDesc/Nios2MCTargetDesc.h"
-#include "llvm/MC/MCAssembler.h"
-#include "llvm/MC/MCELFObjectWriter.h"
-#include "llvm/MC/MCFixupKindInfo.h"
-#include "llvm/MC/MCObjectWriter.h"
-#include "llvm/MC/MCSubtargetInfo.h"
-
-using namespace llvm;
-
-// Prepare value for the target space for it
-static unsigned adjustFixupValue(const MCFixup &Fixup, uint64_t Value) {
-
-  unsigned Kind = Fixup.getKind();
-
-  // Add/subtract and shift
-  switch (Kind) {
-  default:
-    return 0;
-  case Nios2::fixup_Nios2_LO16:
-    break;
-  case Nios2::fixup_Nios2_HI16:
-    // Get the higher 16-bits. Also add 1 if bit 15 is 1.
-    Value = ((Value + 0x8000) >> 16) & 0xffff;
-    break;
-  }
-
-  return Value;
-}
-
-// Calculate index for Nios2 specific little endian byte order
-static unsigned calculateLEIndex(unsigned i) {
-  assert(i <= 3 && "Index out of range!");
-
-  return (1 - i / 2) * 2 + i % 2;
-}
-
-/// ApplyFixup - Apply the \p Value for given \p Fixup into the provided
-/// data fragment, at the offset specified by the fixup and following the
-/// fixup kind as appropriate.
-void Nios2AsmBackend::applyFixup(const MCAssembler &Asm, const MCFixup &Fixup,
-                                 const MCValue &Target,
-                                 MutableArrayRef<char> Data, uint64_t Value,
-                                 bool IsResolved) const {
-  MCFixupKind Kind = Fixup.getKind();
-  Value = adjustFixupValue(Fixup, Value);
-
-  if (!Value)
-    return; // Doesn't change encoding.
-
-  // Where do we start in the object
-  unsigned Offset = Fixup.getOffset();
-  // Number of bytes we need to fixup
-  unsigned NumBytes = (getFixupKindInfo(Kind).TargetSize + 7) / 8;
-  // Grab current value, if any, from bits.
-  uint64_t CurVal = 0;
-
-  for (unsigned i = 0; i != NumBytes; ++i) {
-    unsigned Idx = calculateLEIndex(i);
-    CurVal |= (uint64_t)((uint8_t)Data[Offset + Idx]) << (i * 8);
-  }
-
-  uint64_t Mask = ((uint64_t)(-1) >> (64 - getFixupKindInfo(Kind).TargetSize));
-  CurVal |= Value & Mask;
-
-  // Write out the fixed up bytes back to the code/data bits.
-  for (unsigned i = 0; i != NumBytes; ++i) {
-    unsigned Idx = calculateLEIndex(i);
-    Data[Offset + Idx] = (uint8_t)((CurVal >> (i * 8)) & 0xff);
-  }
-}
-
-Optional<MCFixupKind> Nios2AsmBackend::getFixupKind(StringRef Name) const {
-  return StringSwitch<Optional<MCFixupKind>>(Name)
-      .Case("R_NIOS2_NONE", (MCFixupKind)Nios2::fixup_Nios2_32)
-      .Case("R_NIOS2_32", FK_Data_4)
-      .Default(MCAsmBackend::getFixupKind(Name));
-}
-
-//@getFixupKindInfo {
-const MCFixupKindInfo &
-Nios2AsmBackend::getFixupKindInfo(MCFixupKind Kind) const {
-  const static MCFixupKindInfo Infos[Nios2::NumTargetFixupKinds] = {
-      // This table *must* be in same the order of fixup_* kinds in
-      // Nios2FixupKinds.h.
-      //
-      // name                        offset  bits  flags
-      {"fixup_Nios2_32", 0, 32, 0},
-      {"fixup_Nios2_HI16", 0, 16, 0},
-      {"fixup_Nios2_LO16", 0, 16, 0}};
-
-  if (Kind < FirstTargetFixupKind)
-    return MCAsmBackend::getFixupKindInfo(Kind);
-
-  assert(unsigned(Kind - FirstTargetFixupKind) < getNumFixupKinds() &&
-         "Invalid kind!");
-  return Infos[Kind - FirstTargetFixupKind];
-}
-
-std::unique_ptr<MCObjectTargetWriter>
-Nios2AsmBackend::createObjectTargetWriter() const {
-  return createNios2ELFObjectWriter(MCELFObjectTargetWriter::getOSABI(OSType));
-}
-
-bool Nios2AsmBackend::writeNopData(raw_ostream &OS, uint64_t Count) const {
-  return true;
-}
-
-// MCAsmBackend
-MCAsmBackend *llvm::createNios2AsmBackend(const Target &T,
-                                          const MCSubtargetInfo &STI,
-                                          const MCRegisterInfo &MRI,
-                                          const MCTargetOptions &Options) {
-  return new Nios2AsmBackend(T, STI.getTargetTriple().getOS());
-}
diff --git a/lib/Target/Nios2/MCTargetDesc/Nios2AsmBackend.h b/lib/Target/Nios2/MCTargetDesc/Nios2AsmBackend.h
deleted file mode 100644
index 1f114bd869b1154e54d49833c38f90662408f70c..0000000000000000000000000000000000000000
--- a/lib/Target/Nios2/MCTargetDesc/Nios2AsmBackend.h
+++ /dev/null
@@ -1,81 +0,0 @@
-//===-- Nios2AsmBackend.h - Nios2 Asm Backend  ----------------------------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This file defines the Nios2AsmBackend class.
-//
-//===----------------------------------------------------------------------===//
-//
-
-#ifndef LLVM_LIB_TARGET_NIOS2_MCTARGETDESC_NIOS2ASMBACKEND_H
-#define LLVM_LIB_TARGET_NIOS2_MCTARGETDESC_NIOS2ASMBACKEND_H
-
-#include "MCTargetDesc/Nios2FixupKinds.h"
-#include "llvm/ADT/Triple.h"
-#include "llvm/MC/MCAsmBackend.h"
-
-namespace llvm {
-
-class MCAssembler;
-struct MCFixupKindInfo;
-class Target;
-class MCObjectWriter;
-
-class Nios2AsmBackend : public MCAsmBackend {
-  Triple::OSType OSType;
-
-public:
-  Nios2AsmBackend(const Target &T, Triple::OSType OSType)
-      : MCAsmBackend(support::little), OSType(OSType) {}
-
-  std::unique_ptr<MCObjectTargetWriter>
-  createObjectTargetWriter() const override;
-
-  bool writeNopData(raw_ostream &OS, uint64_t Count) const override;
-
-  void applyFixup(const MCAssembler &Asm, const MCFixup &Fixup,
-                  const MCValue &Target, MutableArrayRef<char> Data,
-                  uint64_t Value, bool IsResolved) const override;
-
-  Optional<MCFixupKind> getFixupKind(StringRef Name) const override;
-  const MCFixupKindInfo &getFixupKindInfo(MCFixupKind Kind) const override;
-
-  unsigned getNumFixupKinds() const override {
-    return Nios2::NumTargetFixupKinds;
-  }
-
-  /// MayNeedRelaxation - Check whether the given instruction may need
-  /// relaxation.
-  ///
-  /// \param Inst - The instruction to test.
-  bool mayNeedRelaxation(const MCInst &Inst) const override { return false; }
-
-  /// fixupNeedsRelaxation - Target specific predicate for whether a given
-  /// fixup requires the associated instruction to be relaxed.
-  bool fixupNeedsRelaxation(const MCFixup &Fixup, uint64_t Value,
-                            const MCRelaxableFragment *DF,
-                            const MCAsmLayout &Layout) const override {
-    // FIXME.
-    llvm_unreachable("RelaxInstruction() unimplemented");
-    return false;
-  }
-
-  /// RelaxInstruction - Relax the instruction in the given fragment
-  /// to the next wider instruction.
-  ///
-  /// \param Inst - The instruction to relax, which may be the same
-  /// as the output.
-  /// \param [out] Res On return, the relaxed instruction.
-  void relaxInstruction(const MCInst &Inst, const MCSubtargetInfo &STI,
-                        MCInst &Res) const override {}
-
-}; // class Nios2AsmBackend
-
-} // namespace llvm
-
-#endif
diff --git a/lib/Target/Nios2/MCTargetDesc/Nios2BaseInfo.h b/lib/Target/Nios2/MCTargetDesc/Nios2BaseInfo.h
deleted file mode 100644
index 225671ebc8d8274b0d7f8e79d4a02ecf93d3ad81..0000000000000000000000000000000000000000
--- a/lib/Target/Nios2/MCTargetDesc/Nios2BaseInfo.h
+++ /dev/null
@@ -1,38 +0,0 @@
-//===-- Nios2BaseInfo.h - Top level definitions for NIOS2 MC ----*- C++ -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This file contains small standalone helper functions and enum definitions for
-// the Nios2 target useful for the compiler back-end and the MC libraries.
-//
-//===----------------------------------------------------------------------===//
-#ifndef LLVM_LIB_TARGET_NIOS2_MCTARGETDESC_NIOS2BASEINFO_H
-#define LLVM_LIB_TARGET_NIOS2_MCTARGETDESC_NIOS2BASEINFO_H
-
-namespace llvm {
-
-/// Nios2FG - This namespace holds all of the target specific flags that
-/// instruction info tracks.
-namespace Nios2FG {
-/// Target Operand Flag enum.
-enum TOF {
-  //===------------------------------------------------------------------===//
-  // Nios2 Specific MachineOperand flags.
-
-  MO_NO_FLAG,
-
-  /// MO_ABS_HI/LO - Represents the hi or low part of an absolute symbol
-  /// address.
-  MO_ABS_HI,
-  MO_ABS_LO,
-
-};
-} // namespace Nios2FG
-} // namespace llvm
-
-#endif
diff --git a/lib/Target/Nios2/MCTargetDesc/Nios2ELFObjectWriter.cpp b/lib/Target/Nios2/MCTargetDesc/Nios2ELFObjectWriter.cpp
deleted file mode 100644
index db432d15120de7b1090f28355b281a5a3506dbbc..0000000000000000000000000000000000000000
--- a/lib/Target/Nios2/MCTargetDesc/Nios2ELFObjectWriter.cpp
+++ /dev/null
@@ -1,43 +0,0 @@
-//===-- Nios2ELFObjectWriter.cpp - Nios2 ELF Writer -----------------------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-
-#include "MCTargetDesc/Nios2FixupKinds.h"
-#include "MCTargetDesc/Nios2MCExpr.h"
-#include "MCTargetDesc/Nios2MCTargetDesc.h"
-#include "llvm/MC/MCELFObjectWriter.h"
-#include "llvm/MC/MCObjectWriter.h"
-
-using namespace llvm;
-
-namespace {
-class Nios2ELFObjectWriter : public MCELFObjectTargetWriter {
-public:
-  Nios2ELFObjectWriter(uint8_t OSABI)
-      : MCELFObjectTargetWriter(false, OSABI, ELF::EM_ALTERA_NIOS2, false) {}
-
-  ~Nios2ELFObjectWriter() override;
-
-  unsigned getRelocType(MCContext &Ctx, const MCValue &Target,
-                        const MCFixup &Fixup, bool IsPCRel) const override;
-};
-} // namespace
-
-Nios2ELFObjectWriter::~Nios2ELFObjectWriter() {}
-
-unsigned Nios2ELFObjectWriter::getRelocType(MCContext &Ctx,
-                                            const MCValue &Target,
-                                            const MCFixup &Fixup,
-                                            bool IsPCRel) const {
-  return 0;
-}
-
-std::unique_ptr<MCObjectTargetWriter>
-llvm::createNios2ELFObjectWriter(uint8_t OSABI) {
-  return llvm::make_unique<Nios2ELFObjectWriter>(OSABI);
-}
diff --git a/lib/Target/Nios2/MCTargetDesc/Nios2FixupKinds.h b/lib/Target/Nios2/MCTargetDesc/Nios2FixupKinds.h
deleted file mode 100644
index c169a1b193712d5435ebdd748aa977271e46c857..0000000000000000000000000000000000000000
--- a/lib/Target/Nios2/MCTargetDesc/Nios2FixupKinds.h
+++ /dev/null
@@ -1,41 +0,0 @@
-//===-- Nios2FixupKinds.h - Nios2 Specific Fixup Entries --------*- C++ -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_LIB_TARGET_NIOS2_MCTARGETDESC_NIOS2FIXUPKINDS_H
-#define LLVM_LIB_TARGET_NIOS2_MCTARGETDESC_NIOS2FIXUPKINDS_H
-
-#include "llvm/MC/MCFixup.h"
-
-namespace llvm {
-namespace Nios2 {
-// Although most of the current fixup types reflect a unique relocation
-// one can have multiple fixup types for a given relocation and thus need
-// to be uniquely named.
-//
-// This table *must* be in the save order of
-// MCFixupKindInfo Infos[Nios2::NumTargetFixupKinds]
-// in Nios2AsmBackend.cpp.
-enum Fixups {
-  // Pure upper 32 bit fixup resulting in - R_NIOS2_32.
-  fixup_Nios2_32 = FirstTargetFixupKind,
-
-  // Pure upper 16 bit fixup resulting in - R_NIOS2_HI16.
-  fixup_Nios2_HI16,
-
-  // Pure lower 16 bit fixup resulting in - R_NIOS2_LO16.
-  fixup_Nios2_LO16,
-
-  // Marker
-  LastTargetFixupKind,
-  NumTargetFixupKinds = LastTargetFixupKind - FirstTargetFixupKind
-};
-} // namespace Nios2
-} // namespace llvm
-
-#endif // LLVM_NIOS2_NIOS2FIXUPKINDS_H
diff --git a/lib/Target/Nios2/MCTargetDesc/Nios2MCAsmInfo.cpp b/lib/Target/Nios2/MCTargetDesc/Nios2MCAsmInfo.cpp
deleted file mode 100644
index e3c66e6776c263daf2cf0e25fc5fbba6910d7de3..0000000000000000000000000000000000000000
--- a/lib/Target/Nios2/MCTargetDesc/Nios2MCAsmInfo.cpp
+++ /dev/null
@@ -1,44 +0,0 @@
-//===-- Nios2MCAsmInfo.cpp - Nios2 Asm Properties -------------------------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This file contains the declarations of the Nios2MCAsmInfo properties.
-//
-//===----------------------------------------------------------------------===//
-
-#include "Nios2MCAsmInfo.h"
-
-#include "llvm/ADT/Triple.h"
-
-using namespace llvm;
-
-void Nios2MCAsmInfo::anchor() {}
-
-Nios2MCAsmInfo::Nios2MCAsmInfo(const Triple &TheTriple) {
-  if ((TheTriple.getArch() == Triple::nios2))
-    IsLittleEndian = true; // the default of IsLittleEndian is true
-
-  AlignmentIsInBytes = false;
-  Data16bitsDirective = "\t.2byte\t";
-  Data32bitsDirective = "\t.4byte\t";
-  Data64bitsDirective = "\t.8byte\t";
-  PrivateLabelPrefix = ".LC";
-  CommentString = "#";
-  ZeroDirective = "\t.space\t";
-  GPRel32Directive = "\t.gpword\t";
-  GPRel64Directive = "\t.gpdword\t";
-  WeakRefDirective = "\t.weak\t";
-  GlobalDirective = "\t.global\t";
-  AscizDirective = "\t.string\t";
-  UseAssignmentForEHBegin = true;
-
-  SupportsDebugInformation = true;
-  ExceptionsType = ExceptionHandling::DwarfCFI;
-  DwarfRegNumForCFI = true;
-  UsesELFSectionDirectiveForBSS = true;
-}
diff --git a/lib/Target/Nios2/MCTargetDesc/Nios2MCAsmInfo.h b/lib/Target/Nios2/MCTargetDesc/Nios2MCAsmInfo.h
deleted file mode 100644
index 0c81276f84d80f4ada1a2272bdea18cc39363d9e..0000000000000000000000000000000000000000
--- a/lib/Target/Nios2/MCTargetDesc/Nios2MCAsmInfo.h
+++ /dev/null
@@ -1,31 +0,0 @@
-//===-- Nios2MCAsmInfo.h - Nios2 Asm Info ----------------------*- C++ -*--===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This file contains the declaration of the Nios2MCAsmInfo class.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_LIB_TARGET_NIOS2_MCTARGETDESC_NIOS2MCASMINFO_H
-#define LLVM_LIB_TARGET_NIOS2_MCTARGETDESC_NIOS2MCASMINFO_H
-
-#include "llvm/MC/MCAsmInfoELF.h"
-
-namespace llvm {
-class Triple;
-
-class Nios2MCAsmInfo : public MCAsmInfoELF {
-  void anchor() override;
-
-public:
-  explicit Nios2MCAsmInfo(const Triple &TheTriple);
-};
-
-} // namespace llvm
-
-#endif
diff --git a/lib/Target/Nios2/MCTargetDesc/Nios2MCExpr.cpp b/lib/Target/Nios2/MCTargetDesc/Nios2MCExpr.cpp
deleted file mode 100644
index 0f12c9e933787c00ba6f67f803210a3bfd94056c..0000000000000000000000000000000000000000
--- a/lib/Target/Nios2/MCTargetDesc/Nios2MCExpr.cpp
+++ /dev/null
@@ -1,76 +0,0 @@
-//===-- Nios2MCExpr.cpp - Nios2 specific MC expression classes ------------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-
-#include "Nios2.h"
-
-#include "Nios2MCExpr.h"
-#include "llvm/MC/MCAsmInfo.h"
-#include "llvm/MC/MCAssembler.h"
-#include "llvm/MC/MCContext.h"
-#include "llvm/MC/MCObjectStreamer.h"
-#include "llvm/MC/MCSymbolELF.h"
-
-using namespace llvm;
-
-#define DEBUG_TYPE "nios2mcexpr"
-
-const Nios2MCExpr *Nios2MCExpr::create(Nios2MCExpr::Nios2ExprKind Kind,
-                                       const MCExpr *Expr, MCContext &Ctx) {
-  return new (Ctx) Nios2MCExpr(Kind, Expr);
-}
-
-const Nios2MCExpr *Nios2MCExpr::create(const MCSymbol *Symbol,
-                                       Nios2MCExpr::Nios2ExprKind Kind,
-                                       MCContext &Ctx) {
-  const MCSymbolRefExpr *MCSym =
-      MCSymbolRefExpr::create(Symbol, MCSymbolRefExpr::VK_None, Ctx);
-  return new (Ctx) Nios2MCExpr(Kind, MCSym);
-}
-
-void Nios2MCExpr::printImpl(raw_ostream &OS, const MCAsmInfo *MAI) const {
-
-  switch (Kind) {
-  case CEK_None:
-  case CEK_Special:
-    llvm_unreachable("CEK_None and CEK_Special are invalid");
-    break;
-  case CEK_ABS_HI:
-    OS << "%hiadj";
-    break;
-  case CEK_ABS_LO:
-    OS << "%lo";
-    break;
-  }
-
-  OS << '(';
-  Expr->print(OS, MAI, true);
-  OS << ')';
-}
-
-bool Nios2MCExpr::evaluateAsRelocatableImpl(MCValue &Res,
-                                            const MCAsmLayout *Layout,
-                                            const MCFixup *Fixup) const {
-  return getSubExpr()->evaluateAsRelocatable(Res, Layout, Fixup);
-}
-
-void Nios2MCExpr::visitUsedExpr(MCStreamer &Streamer) const {
-  Streamer.visitUsedExpr(*getSubExpr());
-}
-
-void Nios2MCExpr::fixELFSymbolsInTLSFixups(MCAssembler &Asm) const {
-  switch (getKind()) {
-  case CEK_None:
-  case CEK_Special:
-    llvm_unreachable("CEK_None and CEK_Special are invalid");
-    break;
-  case CEK_ABS_HI:
-  case CEK_ABS_LO:
-    break;
-  }
-}
diff --git a/lib/Target/Nios2/MCTargetDesc/Nios2MCExpr.h b/lib/Target/Nios2/MCTargetDesc/Nios2MCExpr.h
deleted file mode 100644
index 5b49005eb648fd4ca4201a427cb544da3b5591dd..0000000000000000000000000000000000000000
--- a/lib/Target/Nios2/MCTargetDesc/Nios2MCExpr.h
+++ /dev/null
@@ -1,60 +0,0 @@
-//===-- Nios2MCExpr.h - Nios2 specific MC expression classes ----*- C++ -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_LIB_TARGET_NIOS2_MCTARGETDESC_NIOS2MCEXPR_H
-#define LLVM_LIB_TARGET_NIOS2_MCTARGETDESC_NIOS2MCEXPR_H
-
-#include "llvm/MC/MCAsmLayout.h"
-#include "llvm/MC/MCExpr.h"
-#include "llvm/MC/MCValue.h"
-
-namespace llvm {
-
-class Nios2MCExpr : public MCTargetExpr {
-public:
-  enum Nios2ExprKind {
-    CEK_None,
-    CEK_ABS_HI,
-    CEK_ABS_LO,
-    CEK_Special,
-  };
-
-private:
-  const Nios2ExprKind Kind;
-  const MCExpr *Expr;
-
-  explicit Nios2MCExpr(Nios2ExprKind Kind, const MCExpr *Expr)
-      : Kind(Kind), Expr(Expr) {}
-
-public:
-  static const Nios2MCExpr *create(Nios2ExprKind Kind, const MCExpr *Expr,
-                                   MCContext &Ctx);
-  static const Nios2MCExpr *create(const MCSymbol *Symbol,
-                                   Nios2MCExpr::Nios2ExprKind Kind,
-                                   MCContext &Ctx);
-
-  /// Get the kind of this expression.
-  Nios2ExprKind getKind() const { return Kind; }
-
-  /// Get the child of this expression.
-  const MCExpr *getSubExpr() const { return Expr; }
-
-  void printImpl(raw_ostream &OS, const MCAsmInfo *MAI) const override;
-  bool evaluateAsRelocatableImpl(MCValue &Res, const MCAsmLayout *Layout,
-                                 const MCFixup *Fixup) const override;
-  void visitUsedExpr(MCStreamer &Streamer) const override;
-  MCFragment *findAssociatedFragment() const override {
-    return getSubExpr()->findAssociatedFragment();
-  }
-
-  void fixELFSymbolsInTLSFixups(MCAssembler &Asm) const override;
-};
-} // end namespace llvm
-
-#endif
diff --git a/lib/Target/Nios2/MCTargetDesc/Nios2MCTargetDesc.cpp b/lib/Target/Nios2/MCTargetDesc/Nios2MCTargetDesc.cpp
deleted file mode 100644
index e57b44d3cfdc8c324551c0f1415a2e56b18a5cd7..0000000000000000000000000000000000000000
--- a/lib/Target/Nios2/MCTargetDesc/Nios2MCTargetDesc.cpp
+++ /dev/null
@@ -1,102 +0,0 @@
-//===-- Nios2MCTargetDesc.cpp - Nios2 Target Descriptions -----------------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This file provides Nios2 specific target descriptions.
-//
-//===----------------------------------------------------------------------===//
-
-#include "Nios2MCTargetDesc.h"
-#include "InstPrinter/Nios2InstPrinter.h"
-#include "Nios2MCAsmInfo.h"
-#include "Nios2TargetStreamer.h"
-#include "llvm/MC/MCInstrInfo.h"
-#include "llvm/MC/MCRegisterInfo.h"
-#include "llvm/MC/MCSubtargetInfo.h"
-#include "llvm/Support/TargetRegistry.h"
-
-using namespace llvm;
-
-#define GET_INSTRINFO_MC_DESC
-#include "Nios2GenInstrInfo.inc"
-
-#define GET_SUBTARGETINFO_MC_DESC
-#include "Nios2GenSubtargetInfo.inc"
-
-#define GET_REGINFO_MC_DESC
-#include "Nios2GenRegisterInfo.inc"
-
-static MCInstrInfo *createNios2MCInstrInfo() {
-  MCInstrInfo *X = new MCInstrInfo();
-  InitNios2MCInstrInfo(X); // defined in Nios2GenInstrInfo.inc
-  return X;
-}
-
-static MCRegisterInfo *createNios2MCRegisterInfo(const Triple &TT) {
-  MCRegisterInfo *X = new MCRegisterInfo();
-  InitNios2MCRegisterInfo(X, Nios2::R15); // defined in Nios2GenRegisterInfo.inc
-  return X;
-}
-
-static MCSubtargetInfo *
-createNios2MCSubtargetInfo(const Triple &TT, StringRef CPU, StringRef FS) {
-  if (CPU.empty() || CPU == "generic")
-    CPU = "nios2r1";
-  return createNios2MCSubtargetInfoImpl(TT, CPU, FS);
-  // createNios2MCSubtargetInfoImpl defined in Nios2GenSubtargetInfo.inc
-}
-
-static MCAsmInfo *createNios2MCAsmInfo(const MCRegisterInfo &MRI,
-                                       const Triple &TT) {
-  MCAsmInfo *MAI = new Nios2MCAsmInfo(TT);
-
-  unsigned SP = MRI.getDwarfRegNum(Nios2::SP, true);
-  MCCFIInstruction Inst = MCCFIInstruction::createDefCfa(nullptr, SP, 0);
-  MAI->addInitialFrameState(Inst);
-
-  return MAI;
-}
-
-static MCInstPrinter *createNios2MCInstPrinter(const Triple &T,
-                                               unsigned SyntaxVariant,
-                                               const MCAsmInfo &MAI,
-                                               const MCInstrInfo &MII,
-                                               const MCRegisterInfo &MRI) {
-  return new Nios2InstPrinter(MAI, MII, MRI);
-}
-
-static MCTargetStreamer *createNios2AsmTargetStreamer(MCStreamer &S,
-                                                      formatted_raw_ostream &OS,
-                                                      MCInstPrinter *InstPrint,
-                                                      bool isVerboseAsm) {
-  return new Nios2TargetAsmStreamer(S, OS);
-}
-
-extern "C" void LLVMInitializeNios2TargetMC() {
-  Target *T = &getTheNios2Target();
-
-  // Register the MC asm info.
-  RegisterMCAsmInfoFn X(*T, createNios2MCAsmInfo);
-
-  // Register the MC instruction info.
-  TargetRegistry::RegisterMCInstrInfo(*T, createNios2MCInstrInfo);
-
-  // Register the MC register info.
-  TargetRegistry::RegisterMCRegInfo(*T, createNios2MCRegisterInfo);
-
-  // Register the asm target streamer.
-  TargetRegistry::RegisterAsmTargetStreamer(*T, createNios2AsmTargetStreamer);
-
-  // Register the MC subtarget info.
-  TargetRegistry::RegisterMCSubtargetInfo(*T, createNios2MCSubtargetInfo);
-  // Register the MCInstPrinter.
-  TargetRegistry::RegisterMCInstPrinter(*T, createNios2MCInstPrinter);
-
-  // Register the asm backend.
-  TargetRegistry::RegisterMCAsmBackend(*T, createNios2AsmBackend);
-}
diff --git a/lib/Target/Nios2/MCTargetDesc/Nios2MCTargetDesc.h b/lib/Target/Nios2/MCTargetDesc/Nios2MCTargetDesc.h
deleted file mode 100644
index a7c4b16c6a3b702ceb39d6bc299d80600771be16..0000000000000000000000000000000000000000
--- a/lib/Target/Nios2/MCTargetDesc/Nios2MCTargetDesc.h
+++ /dev/null
@@ -1,52 +0,0 @@
-//===-- Nios2MCTargetDesc.h - Nios2 Target Descriptions ---------*- C++ -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This file provides Nios2 specific target descriptions.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_LIB_TARGET_NIOS2_MCTARGETDESC_NIOS2MCTARGETDESC_H
-#define LLVM_LIB_TARGET_NIOS2_MCTARGETDESC_NIOS2MCTARGETDESC_H
-
-#include <memory>
-
-namespace llvm {
-class MCAsmBackend;
-class MCObjectTargetWriter;
-class MCRegisterInfo;
-class MCSubtargetInfo;
-class MCTargetOptions;
-class Target;
-class Triple;
-class StringRef;
-class raw_pwrite_stream;
-
-Target &getTheNios2Target();
-
-MCAsmBackend *createNios2AsmBackend(const Target &T, const MCSubtargetInfo &STI,
-                                    const MCRegisterInfo &MRI,
-                                    const MCTargetOptions &Options);
-
-std::unique_ptr<MCObjectTargetWriter> createNios2ELFObjectWriter(uint8_t OSABI);
-
-} // namespace llvm
-
-// Defines symbolic names for Nios2 registers.  This defines a mapping from
-// register name to register number.
-#define GET_REGINFO_ENUM
-#include "Nios2GenRegisterInfo.inc"
-
-// Defines symbolic names for the Nios2 instructions.
-#define GET_INSTRINFO_ENUM
-#include "Nios2GenInstrInfo.inc"
-
-#define GET_SUBTARGETINFO_ENUM
-#include "Nios2GenSubtargetInfo.inc"
-
-#endif
diff --git a/lib/Target/Nios2/MCTargetDesc/Nios2TargetStreamer.cpp b/lib/Target/Nios2/MCTargetDesc/Nios2TargetStreamer.cpp
deleted file mode 100644
index 795fd0084aa32b330c09e2c27f209a50cf3e5368..0000000000000000000000000000000000000000
--- a/lib/Target/Nios2/MCTargetDesc/Nios2TargetStreamer.cpp
+++ /dev/null
@@ -1,22 +0,0 @@
-//===-- Nios2TargetStreamer.cpp - Nios2 Target Streamer Methods -----------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This file provides Nios2 specific target streamer methods.
-//
-//===----------------------------------------------------------------------===//
-
-#include "Nios2TargetStreamer.h"
-
-using namespace llvm;
-
-Nios2TargetStreamer::Nios2TargetStreamer(MCStreamer &S) : MCTargetStreamer(S) {}
-
-Nios2TargetAsmStreamer::Nios2TargetAsmStreamer(MCStreamer &S,
-                                               formatted_raw_ostream &OS)
-    : Nios2TargetStreamer(S) {}
diff --git a/lib/Target/Nios2/Nios2.h b/lib/Target/Nios2/Nios2.h
deleted file mode 100644
index d6c5c1e496621a25152c55b2f703c9f22486b528..0000000000000000000000000000000000000000
--- a/lib/Target/Nios2/Nios2.h
+++ /dev/null
@@ -1,35 +0,0 @@
-//===-- Nios2.h - Top-level interface for Nios2 representation --*- C++ -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This file contains the entry points for global functions defined in
-// the LLVM Nios2 back-end.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_LIB_TARGET_NIOS2_NIOS2_H
-#define LLVM_LIB_TARGET_NIOS2_NIOS2_H
-
-#include "MCTargetDesc/Nios2MCTargetDesc.h"
-#include "llvm/Target/TargetMachine.h"
-
-namespace llvm {
-class FunctionPass;
-class formatted_raw_ostream;
-class Nios2TargetMachine;
-class AsmPrinter;
-class MachineInstr;
-class MCInst;
-
-FunctionPass *createNios2ISelDag(Nios2TargetMachine &TM,
-                                 CodeGenOpt::Level OptLevel);
-void LowerNios2MachineInstToMCInst(const MachineInstr *MI, MCInst &OutMI,
-                                   AsmPrinter &AP);
-} // namespace llvm
-
-#endif
diff --git a/lib/Target/Nios2/Nios2.td b/lib/Target/Nios2/Nios2.td
deleted file mode 100644
index 1acf4c70c42ce6346415484171970affe5986dd3..0000000000000000000000000000000000000000
--- a/lib/Target/Nios2/Nios2.td
+++ /dev/null
@@ -1,59 +0,0 @@
-//===-- Nios2.td - Describe the Nios2 Target Machine -------*- tablegen -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-
-//===----------------------------------------------------------------------===//
-// Calling Conv, Instruction Descriptions
-//===----------------------------------------------------------------------===//
-
-include "llvm/Target/Target.td"
-include "Nios2RegisterInfo.td"
-include "Nios2Schedule.td"
-include "Nios2InstrInfo.td"
-include "Nios2CallingConv.td"
-
-//===----------------------------------------------------------------------===//
-// Nios2 Subtarget features
-//===----------------------------------------------------------------------===//
-def FeatureNios2r1     : SubtargetFeature<"nios2r1", "Nios2ArchVersion", 
-                                "Nios2r1", "Nios2 R1 ISA Support">;
-def FeatureNios2r2     : SubtargetFeature<"nios2r2", "Nios2ArchVersion",                      
-                               "Nios2r2", "Nios2 R2 ISA Support">;
-
-//===----------------------------------------------------------------------===//
-// Nios2 processors supported.
-//===----------------------------------------------------------------------===//
-
-class Proc<string Name, list<SubtargetFeature> Features>
- : Processor<Name, Nios2GenericItineraries, Features>;
-
-def : Proc<"nios2r1", [FeatureNios2r1]>;
-def : Proc<"nios2r2", [FeatureNios2r2]>;
-
-def Nios2InstrInfo : InstrInfo;
-
-def Nios2AsmParser : AsmParser {
-  let ShouldEmitMatchRegisterName = 0;
-}
-
-//===----------------------------------------------------------------------===//
-// Declare the target which we are implementing
-//===----------------------------------------------------------------------===//
-
-def Nios2AsmWriter : AsmWriter {
-  string AsmWriterClassName  = "InstPrinter";
-  int PassSubtarget = 1;
-  int Variant = 0;
-}
-
-def Nios2 : Target {
-// def Nios2InstrInfo : InstrInfo as before.
-  let InstructionSet = Nios2InstrInfo;
-  let AssemblyParsers = [Nios2AsmParser];
-  let AssemblyWriters = [Nios2AsmWriter];
-}
diff --git a/lib/Target/Nios2/Nios2AsmPrinter.cpp b/lib/Target/Nios2/Nios2AsmPrinter.cpp
deleted file mode 100644
index 1abf19591774a37381ce5aceca9d335be243d046..0000000000000000000000000000000000000000
--- a/lib/Target/Nios2/Nios2AsmPrinter.cpp
+++ /dev/null
@@ -1,153 +0,0 @@
-//===-- Nios2AsmPrinter.cpp - Nios2 LLVM Assembly Printer -----------------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This file contains a printer that converts from our internal representation
-// of machine-dependent LLVM code to GAS-format NIOS2 assembly language.
-//
-//===----------------------------------------------------------------------===//
-
-#include "InstPrinter/Nios2InstPrinter.h"
-#include "MCTargetDesc/Nios2BaseInfo.h"
-#include "Nios2.h"
-#include "Nios2TargetMachine.h"
-#include "llvm/CodeGen/AsmPrinter.h"
-#include "llvm/MC/MCStreamer.h"
-#include "llvm/Support/TargetRegistry.h"
-
-using namespace llvm;
-
-#define DEBUG_TYPE "nios2-asm-printer"
-
-namespace {
-
-class Nios2AsmPrinter : public AsmPrinter {
-
-public:
-  explicit Nios2AsmPrinter(TargetMachine &TM,
-                           std::unique_ptr<MCStreamer> Streamer)
-      : AsmPrinter(TM, std::move(Streamer)) {}
-
-  StringRef getPassName() const override { return "Nios2 Assembly Printer"; }
-
-  //- EmitInstruction() must exists or will have run time error.
-  void EmitInstruction(const MachineInstr *MI) override;
-  bool PrintAsmOperand(const MachineInstr *MI, unsigned OpNo,
-                       unsigned AsmVariant, const char *ExtraCode,
-                       raw_ostream &O) override;
-  bool PrintAsmMemoryOperand(const MachineInstr *MI, unsigned OpNum,
-                             unsigned AsmVariant, const char *ExtraCode,
-                             raw_ostream &O) override;
-  void printOperand(const MachineInstr *MI, int opNum, raw_ostream &O);
-  void EmitFunctionEntryLabel() override;
-};
-} // namespace
-
-//- EmitInstruction() must exists or will have run time error.
-void Nios2AsmPrinter::EmitInstruction(const MachineInstr *MI) {
-
-  //  Print out both ordinary instruction and boudle instruction
-  MachineBasicBlock::const_instr_iterator I = MI->getIterator();
-  MachineBasicBlock::const_instr_iterator E = MI->getParent()->instr_end();
-
-  do {
-
-    if (I->isPseudo()) {
-      llvm_unreachable("Pseudo opcode found in EmitInstruction()");
-    }
-
-    MCInst TmpInst0;
-    LowerNios2MachineInstToMCInst(&*I, TmpInst0, *this);
-    EmitToStreamer(*OutStreamer, TmpInst0);
-  } while ((++I != E) && I->isInsideBundle()); // Delay slot check
-}
-
-//		.type	main,@function
-//->		.ent	main                    # @main
-//	main:
-void Nios2AsmPrinter::EmitFunctionEntryLabel() {
-  OutStreamer->EmitLabel(CurrentFnSym);
-}
-
-// Print out an operand for an inline asm expression.
-bool Nios2AsmPrinter::PrintAsmOperand(const MachineInstr *MI, unsigned OpNum,
-                                      unsigned AsmVariant,
-                                      const char *ExtraCode, raw_ostream &O) {
-  printOperand(MI, OpNum, O);
-  return false;
-}
-
-bool Nios2AsmPrinter::PrintAsmMemoryOperand(const MachineInstr *MI,
-                                            unsigned OpNum, unsigned AsmVariant,
-                                            const char *ExtraCode,
-                                            raw_ostream &O) {
-  if (ExtraCode && ExtraCode[0])
-    return true; // Unknown modifier
-
-  const MachineOperand &MO = MI->getOperand(OpNum);
-  assert(MO.isReg() && "unexpected inline asm memory operand");
-  O << "($" << Nios2InstPrinter::getRegisterName(MO.getReg()) << ")";
-
-  return false;
-}
-
-void Nios2AsmPrinter::printOperand(const MachineInstr *MI, int opNum,
-                                   raw_ostream &O) {
-  const MachineOperand &MO = MI->getOperand(opNum);
-  bool closeP = false;
-
-  if (MO.getTargetFlags())
-    closeP = true;
-
-  switch (MO.getTargetFlags()) {
-  case Nios2FG::MO_ABS_HI:
-    O << "%hiadj(";
-    break;
-  case Nios2FG::MO_ABS_LO:
-    O << "%lo(";
-    break;
-  }
-
-  switch (MO.getType()) {
-  case MachineOperand::MO_Register:
-    O << '$'
-      << StringRef(Nios2InstPrinter::getRegisterName(MO.getReg())).lower();
-    break;
-
-  case MachineOperand::MO_Immediate:
-    O << MO.getImm();
-    break;
-
-  case MachineOperand::MO_MachineBasicBlock:
-    MO.getMBB()->getSymbol()->print(O, MAI);
-    return;
-
-  case MachineOperand::MO_GlobalAddress:
-    getSymbol(MO.getGlobal())->print(O, MAI);
-    break;
-
-  case MachineOperand::MO_BlockAddress:
-    O << GetBlockAddressSymbol(MO.getBlockAddress())->getName();
-    break;
-
-  case MachineOperand::MO_ExternalSymbol:
-    O << MO.getSymbolName();
-    break;
-
-  default:
-    llvm_unreachable("<unknown operand type>");
-  }
-
-  if (closeP)
-    O << ")";
-}
-
-// Force static initialization.
-extern "C" void LLVMInitializeNios2AsmPrinter() {
-  RegisterAsmPrinter<Nios2AsmPrinter> X(getTheNios2Target());
-}
diff --git a/lib/Target/Nios2/Nios2CallingConv.td b/lib/Target/Nios2/Nios2CallingConv.td
deleted file mode 100644
index f0b172f8422d6a171407bd3926f1faf45b1de8c4..0000000000000000000000000000000000000000
--- a/lib/Target/Nios2/Nios2CallingConv.td
+++ /dev/null
@@ -1,34 +0,0 @@
-//===- Nios2CallingConv.td - Calling Conventions for Nios2 -*- tablegen -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-// This describes the calling conventions for Nios2 architecture.
-//===----------------------------------------------------------------------===//
-
-/// CCIfSubtarget - Match if the current subtarget has a feature F.
-class CCIfSubtarget<string F, CCAction A>:
-  CCIf<!strconcat("State.getTarget().getSubtarget<Nios2Subtarget>().", F), A>;
-
-def CC_Nios2 : CallingConv<[
-  // i32 f32 arguments get passed in integer registers if there is space.
-  CCIfType<[i32, f32], CCAssignToReg<[R4, R5, R6, R7]>>,
-
-  // Alternatively, they are assigned to the stack in 4-byte aligned units.
-  CCAssignToStack<4, 4>
-]>;
-
-def RetCC_Nios2EABI : CallingConv<[
-  // i32 are returned in registers R2, R3
-  CCIfType<[i32], CCAssignToReg<[R2, R3]>>,
-  // In case of floating point (FPH2 instr.) also use the same register set
-  CCIfType<[f32], CCAssignToReg<[R2, R3]>>,
-  CCIfByVal<CCPassByVal<4, 4>>,
-  // Stack parameter slots for i32 is 32-bit words and 4-byte aligned.
-  CCIfType<[i32], CCAssignToStack<4, 4>>
-]>;
-
-def CSR : CalleeSavedRegs<(add RA, FP, (sequence "R%u", 16, 23))>;
diff --git a/lib/Target/Nios2/Nios2FrameLowering.cpp b/lib/Target/Nios2/Nios2FrameLowering.cpp
deleted file mode 100644
index 6fb28a6fd638a1e915a599ff971b7f2f4705d362..0000000000000000000000000000000000000000
--- a/lib/Target/Nios2/Nios2FrameLowering.cpp
+++ /dev/null
@@ -1,27 +0,0 @@
-//===-- Nios2FrameLowering.cpp - Nios2 Frame Information ------------------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This file contains the Nios2 implementation of TargetFrameLowering class.
-//
-//===----------------------------------------------------------------------===//
-
-#include "Nios2FrameLowering.h"
-
-#include "Nios2Subtarget.h"
-#include "llvm/CodeGen/MachineFunction.h"
-
-using namespace llvm;
-
-bool Nios2FrameLowering::hasFP(const MachineFunction &MF) const { return true; }
-
-void Nios2FrameLowering::emitPrologue(MachineFunction &MF,
-                                      MachineBasicBlock &MBB) const {}
-
-void Nios2FrameLowering::emitEpilogue(MachineFunction &MF,
-                                      MachineBasicBlock &MBB) const {}
diff --git a/lib/Target/Nios2/Nios2FrameLowering.h b/lib/Target/Nios2/Nios2FrameLowering.h
deleted file mode 100644
index 4ffb01dda36a272f855b0ea925ca3b8042847ef1..0000000000000000000000000000000000000000
--- a/lib/Target/Nios2/Nios2FrameLowering.h
+++ /dev/null
@@ -1,39 +0,0 @@
-//===-- Nios2FrameLowering.h - Define frame lowering for Nios2 --*- C++ -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-//
-//
-//===----------------------------------------------------------------------===//
-#ifndef LLVM_LIB_TARGET_NIOS2_NIOS2FRAMELOWERING_H
-#define LLVM_LIB_TARGET_NIOS2_NIOS2FRAMELOWERING_H
-
-#include "Nios2.h"
-#include "llvm/CodeGen/TargetFrameLowering.h"
-
-namespace llvm {
-class Nios2Subtarget;
-
-class Nios2FrameLowering : public TargetFrameLowering {
-protected:
-  const Nios2Subtarget &STI;
-
-public:
-  explicit Nios2FrameLowering(const Nios2Subtarget &sti)
-      : TargetFrameLowering(TargetFrameLowering::StackGrowsDown, 4, 0, 4),
-        STI(sti) {}
-
-  bool hasFP(const MachineFunction &MF) const override;
-  /// emitProlog/emitEpilog - These methods insert prolog and epilog code into
-  /// the function.
-  void emitPrologue(MachineFunction &MF, MachineBasicBlock &MBB) const override;
-  void emitEpilogue(MachineFunction &MF, MachineBasicBlock &MBB) const override;
-};
-} // namespace llvm
-
-#endif
diff --git a/lib/Target/Nios2/Nios2ISelDAGToDAG.cpp b/lib/Target/Nios2/Nios2ISelDAGToDAG.cpp
deleted file mode 100644
index 5f96794661159edef57529fe8061e2cee05e8bf1..0000000000000000000000000000000000000000
--- a/lib/Target/Nios2/Nios2ISelDAGToDAG.cpp
+++ /dev/null
@@ -1,76 +0,0 @@
-//===-- Nios2ISelDAGToDAG.cpp - A Dag to Dag Inst Selector for Nios2 ------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This file defines an instruction selector for the NIOS2 target.
-//
-//===----------------------------------------------------------------------===//
-
-#include "Nios2.h"
-#include "Nios2TargetMachine.h"
-#include "llvm/CodeGen/SelectionDAGISel.h"
-#include "llvm/Support/Debug.h"
-using namespace llvm;
-
-#define DEBUG_TYPE "nios2-isel"
-
-//===----------------------------------------------------------------------===//
-// Instruction Selector Implementation
-//===----------------------------------------------------------------------===//
-
-//===----------------------------------------------------------------------===//
-// Nios2DAGToDAGISel - NIOS2 specific code to select NIOS2 machine
-// instructions for SelectionDAG operations.
-//===----------------------------------------------------------------------===//
-
-namespace {
-
-class Nios2DAGToDAGISel : public SelectionDAGISel {
-  /// Subtarget - Keep a pointer to the Nios2 Subtarget around so that we can
-  /// make the right decision when generating code for different targets.
-  const Nios2Subtarget *Subtarget;
-
-public:
-  explicit Nios2DAGToDAGISel(Nios2TargetMachine &TM, CodeGenOpt::Level OL)
-      : SelectionDAGISel(TM, OL) {}
-
-  bool runOnMachineFunction(MachineFunction &MF) override {
-    Subtarget = &MF.getSubtarget<Nios2Subtarget>();
-    return SelectionDAGISel::runOnMachineFunction(MF);
-  }
-
-  void Select(SDNode *N) override;
-
-  // Pass Name
-  StringRef getPassName() const override {
-    return "NIOS2 DAG->DAG Pattern Instruction Selection";
-  }
-
-#include "Nios2GenDAGISel.inc"
-};
-} // namespace
-
-// Select instructions not customized! Used for
-// expanded, promoted and normal instructions
-void Nios2DAGToDAGISel::Select(SDNode *Node) {
-
-  // If we have a custom node, we already have selected!
-  if (Node->isMachineOpcode()) {
-    LLVM_DEBUG(errs() << "== "; Node->dump(CurDAG); errs() << "\n");
-    Node->setNodeId(-1);
-    return;
-  }
-
-  // Select the default instruction
-  SelectCode(Node);
-}
-
-FunctionPass *llvm::createNios2ISelDag(Nios2TargetMachine &TM,
-                                       CodeGenOpt::Level OptLevel) {
-  return new Nios2DAGToDAGISel(TM, OptLevel);
-}
diff --git a/lib/Target/Nios2/Nios2ISelLowering.cpp b/lib/Target/Nios2/Nios2ISelLowering.cpp
deleted file mode 100644
index 008ce1570722b99787f10ac8b9408453bbe48b8c..0000000000000000000000000000000000000000
--- a/lib/Target/Nios2/Nios2ISelLowering.cpp
+++ /dev/null
@@ -1,188 +0,0 @@
-//===-- Nios2ISelLowering.cpp - Nios2 DAG Lowering Implementation ---------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This file implements the interfaces that Nios2 uses to lower LLVM code into a
-// selection DAG.
-//
-//===----------------------------------------------------------------------===//
-
-#include "Nios2ISelLowering.h"
-#include "Nios2MachineFunction.h"
-#include "Nios2TargetMachine.h"
-#include "llvm/CodeGen/CallingConvLower.h"
-#include "llvm/CodeGen/MachineRegisterInfo.h"
-
-using namespace llvm;
-
-//===----------------------------------------------------------------------===//
-// Calling Convention Implementation
-//===----------------------------------------------------------------------===//
-
-#include "Nios2GenCallingConv.inc"
-
-SDValue
-Nios2TargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
-                                 bool IsVarArg,
-                                 const SmallVectorImpl<ISD::OutputArg> &Outs,
-                                 const SmallVectorImpl<SDValue> &OutVals,
-                                 const SDLoc &DL, SelectionDAG &DAG) const {
-  // CCValAssign - represent the assignment of
-  // the return value to a location
-  SmallVector<CCValAssign, 16> RVLocs;
-  MachineFunction &MF = DAG.getMachineFunction();
-
-  // CCState - Info about the registers and stack slot.
-  CCState CCInfo(CallConv, IsVarArg, MF, RVLocs, *DAG.getContext());
-  // Analyze return values.
-  CCInfo.CheckReturn(Outs, RetCC_Nios2EABI);
-
-  SDValue Flag;
-  SmallVector<SDValue, 4> RetOps(1, Chain);
-
-  // Copy the result values into the output registers.
-  for (unsigned i = 0; i != RVLocs.size(); ++i) {
-    SDValue Val = OutVals[i];
-    CCValAssign &VA = RVLocs[i];
-    assert(VA.isRegLoc() && "Can only return in registers!");
-
-    if (RVLocs[i].getValVT() != RVLocs[i].getLocVT())
-      Val = DAG.getNode(ISD::BITCAST, DL, RVLocs[i].getLocVT(), Val);
-
-    Chain = DAG.getCopyToReg(Chain, DL, VA.getLocReg(), Val, Flag);
-
-    // Guarantee that all emitted copies are stuck together with flags.
-    Flag = Chain.getValue(1);
-    RetOps.push_back(DAG.getRegister(VA.getLocReg(), VA.getLocVT()));
-  }
-
-  if (Flag.getNode())
-    RetOps.push_back(Flag);
-
-  return DAG.getNode(Nios2ISD::Ret, DL, MVT::Other, RetOps);
-}
-
-// addLiveIn - This helper function adds the specified physical register to the
-// MachineFunction as a live in value.  It also creates a corresponding
-// virtual register for it.
-static unsigned addLiveIn(MachineFunction &MF, unsigned PReg,
-                          const TargetRegisterClass *RC) {
-  unsigned VReg = MF.getRegInfo().createVirtualRegister(RC);
-  MF.getRegInfo().addLiveIn(PReg, VReg);
-  return VReg;
-}
-
-//===----------------------------------------------------------------------===//
-//            Formal Arguments Calling Convention Implementation
-//===----------------------------------------------------------------------===//
-
-// LowerFormalArguments - transform physical registers into virtual registers
-// and generate load operations for arguments places on the stack.
-SDValue Nios2TargetLowering::LowerFormalArguments(
-    SDValue Chain, CallingConv::ID CallConv, bool IsVarArg,
-    const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL,
-    SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
-  MachineFunction &MF = DAG.getMachineFunction();
-  MachineFrameInfo &MFI = MF.getFrameInfo();
-
-  // Assign locations to all of the incoming arguments.
-  SmallVector<CCValAssign, 16> ArgLocs;
-  CCState CCInfo(CallConv, IsVarArg, DAG.getMachineFunction(), ArgLocs,
-                 *DAG.getContext());
-
-  CCInfo.AnalyzeFormalArguments(Ins, CC_Nios2);
-
-  // Used with vargs to acumulate store chains.
-  std::vector<SDValue> OutChains;
-
-  for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
-    CCValAssign &VA = ArgLocs[i];
-
-    EVT ValVT = VA.getValVT();
-
-    // Arguments stored on registers
-    if (VA.isRegLoc()) {
-      MVT RegVT = VA.getLocVT();
-      unsigned ArgReg = VA.getLocReg();
-      const TargetRegisterClass *RC = getRegClassFor(RegVT);
-
-      // Transform the arguments stored on
-      // physical registers into virtual ones
-      unsigned Reg = addLiveIn(MF, ArgReg, RC);
-      SDValue ArgValue = DAG.getCopyFromReg(Chain, DL, Reg, RegVT);
-
-      // If this is an 8 or 16-bit value, it has been passed promoted
-      // to 32 bits.  Insert an assert[sz]ext to capture this, then
-      // truncate to the right size.
-      if (VA.getLocInfo() != CCValAssign::Full) {
-        unsigned Opcode = 0;
-        if (VA.getLocInfo() == CCValAssign::SExt)
-          Opcode = ISD::AssertSext;
-        else if (VA.getLocInfo() == CCValAssign::ZExt)
-          Opcode = ISD::AssertZext;
-        if (Opcode)
-          ArgValue =
-              DAG.getNode(Opcode, DL, RegVT, ArgValue, DAG.getValueType(ValVT));
-        ArgValue = DAG.getNode(ISD::TRUNCATE, DL, ValVT, ArgValue);
-      }
-
-      // Handle floating point arguments passed in integer registers.
-      if ((RegVT == MVT::i32 && ValVT == MVT::f32) ||
-          (RegVT == MVT::i64 && ValVT == MVT::f64))
-        ArgValue = DAG.getNode(ISD::BITCAST, DL, ValVT, ArgValue);
-      InVals.push_back(ArgValue);
-    } else { // VA.isRegLoc()
-      MVT LocVT = VA.getLocVT();
-
-      // sanity check
-      assert(VA.isMemLoc());
-
-      // The stack pointer offset is relative to the caller stack frame.
-      int FI = MFI.CreateFixedObject(ValVT.getSizeInBits() / 8,
-                                     VA.getLocMemOffset(), true);
-
-      // Create load nodes to retrieve arguments from the stack
-      SDValue FIN = DAG.getFrameIndex(FI, getPointerTy(DAG.getDataLayout()));
-      SDValue Load = DAG.getLoad(
-          LocVT, DL, Chain, FIN,
-          MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI));
-      InVals.push_back(Load);
-      OutChains.push_back(Load.getValue(1));
-    }
-  }
-  if (!OutChains.empty()) {
-    OutChains.push_back(Chain);
-    Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, OutChains);
-  }
-
-  return Chain;
-}
-
-//===----------------------------------------------------------------------===//
-// TargetLowering Implementation
-//===----------------------------------------------------------------------===//
-
-Nios2TargetLowering::Nios2TargetLowering(const TargetMachine &TM,
-                                         const Nios2Subtarget &STI)
-    : TargetLowering(TM), Subtarget(&STI) {
-
-  addRegisterClass(MVT::i32, &Nios2::CPURegsRegClass);
-  computeRegisterProperties(Subtarget->getRegisterInfo());
-}
-
-const char *Nios2TargetLowering::getTargetNodeName(unsigned Opcode) const {
-  switch (Opcode) {
-  case Nios2ISD::Hi:
-    return "Nios2ISD::Hi";
-  case Nios2ISD::Lo:
-    return "Nios2ISD::Lo";
-  case Nios2ISD::Ret:
-    return "Nios2ISD::Ret";
-  }
-  return nullptr;
-}
diff --git a/lib/Target/Nios2/Nios2ISelLowering.h b/lib/Target/Nios2/Nios2ISelLowering.h
deleted file mode 100644
index c3c8179054bb5478c8ed47c43b2c4538596341e8..0000000000000000000000000000000000000000
--- a/lib/Target/Nios2/Nios2ISelLowering.h
+++ /dev/null
@@ -1,63 +0,0 @@
-//===-- Nios2ISelLowering.h - Nios2 DAG Lowering Interface ------*- C++ -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This file defines the interfaces that Nios2 uses to lower LLVM code into a
-// selection DAG.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_LIB_TARGET_NIOS2_NIOS2ISELLOWERING_H
-#define LLVM_LIB_TARGET_NIOS2_NIOS2ISELLOWERING_H
-
-#include "Nios2.h"
-#include "llvm/CodeGen/TargetLowering.h"
-
-namespace llvm {
-class Nios2Subtarget;
-
-namespace Nios2ISD {
-enum NodeType {
-  // Start the numbering from where ISD NodeType finishes.
-  FIRST_NUMBER = ISD::BUILTIN_OP_END,
-
-  // Get the Higher 16 bits from a 32-bit immediate
-  // No relation with Nios2 Hi register
-  Hi,
-  // Get the Lower 16 bits from a 32-bit immediate
-  // No relation with Nios2 Lo register
-  Lo,
-  // Return
-  Ret
-};
-}
-
-class Nios2TargetLowering : public TargetLowering {
-  const Nios2Subtarget *Subtarget;
-
-public:
-  Nios2TargetLowering(const TargetMachine &TM, const Nios2Subtarget &STI);
-
-  /// getTargetNodeName - This method returns the name of a target specific
-  //  DAG node.
-  const char *getTargetNodeName(unsigned Opcode) const override;
-
-  SDValue LowerFormalArguments(SDValue Chain, CallingConv::ID CallConv,
-                               bool IsVarArg,
-                               const SmallVectorImpl<ISD::InputArg> &Ins,
-                               const SDLoc &dl, SelectionDAG &DAG,
-                               SmallVectorImpl<SDValue> &InVals) const override;
-
-  SDValue LowerReturn(SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
-                      const SmallVectorImpl<ISD::OutputArg> &Outs,
-                      const SmallVectorImpl<SDValue> &OutVals, const SDLoc &dl,
-                      SelectionDAG &DAG) const override;
-};
-} // end namespace llvm
-
-#endif // NIOS2_ISELLOWERING_H
diff --git a/lib/Target/Nios2/Nios2InstrFormats.td b/lib/Target/Nios2/Nios2InstrFormats.td
deleted file mode 100644
index f57bf03bba3cf5a82b6ad3bbb358bcd9ce99be1e..0000000000000000000000000000000000000000
--- a/lib/Target/Nios2/Nios2InstrFormats.td
+++ /dev/null
@@ -1,235 +0,0 @@
-//===-- Nios2InstrFormats.td - Nios2 Instruction Formats ---*- tablegen -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-
-//===----------------------------------------------------------------------===//
-//  Describe NIOS2 instructions format
-//
-//
-//===----------------------------------------------------------------------===//
-
-// Format specifies the encoding used by the instruction.  This is part of the
-// ad-hoc solution used to emit machine instruction encodings by our machine
-// code emitter.
-class Format<bits<6> val> {
-  bits<6> Value = val;
-}
-
-def Pseudo     : Format<0>;
-// Nios2 R1 instr formats:
-def FrmI       : Format<1>;
-def FrmR       : Format<2>;
-def FrmJ       : Format<3>;
-def FrmOther   : Format<4>;  // Instruction w/ a custom format
-// Nios2 R2 instr 32-bit formats:
-def FrmL26     : Format<5>;  // corresponds to J format in R1
-def FrmF2I16   : Format<6>;  // corresponds to I format in R1
-def FrmF2X4I12 : Format<7>;
-def FrmF1X4I12 : Format<8>;
-def FrmF1X4L17 : Format<9>;
-def FrmF3X6L5  : Format<10>; // corresponds to R format in R1
-def FrmF2X6L10 : Format<11>;
-def FrmF3X6    : Format<12>; // corresponds to R format in R1
-def FrmF3X8    : Format<13>; // corresponds to custom format in R1
-// Nios2 R2 instr 16-bit formats:
-def FrmI10     : Format<14>;
-def FrmT1I7    : Format<15>; 
-def FrmT2I4    : Format<16>;
-def FrmT1X1I6  : Format<17>;
-def FrmX1I7    : Format<18>;
-def FrmL5I4X1  : Format<19>;
-def FrmT2X1L3  : Format<20>;
-def FrmT2X1I3  : Format<21>;
-def FrmT3X1    : Format<22>;
-def FrmT2X3    : Format<23>;
-def FrmF1X1    : Format<24>;
-def FrmX2L5    : Format<25>;
-def FrmF1I5    : Format<26>;
-def FrmF2      : Format<27>;
-
-//===----------------------------------------------------------------------===//
-// Instruction Predicates:
-//===----------------------------------------------------------------------===//
-
-def isNios2r1 : Predicate<"Subtarget->isNios2r1()">;
-def isNios2r2 : Predicate<"Subtarget->isNios2r2()">;
-
-class PredicateControl {
-  // Predicates related to specific target CPU features
-  list<Predicate> FeaturePredicates = [];
-  // Predicates for the instruction group membership in given ISA
-  list<Predicate> InstrPredicates = [];
-  
-  list<Predicate> Predicates = !listconcat(FeaturePredicates, InstrPredicates);
-}
-
-//===----------------------------------------------------------------------===//
-// Base classes for 32-bit, 16-bit and pseudo instructions
-//===----------------------------------------------------------------------===//
-
-class Nios2Inst32<dag outs, dag ins, string asmstr, list<dag> pattern,
-                  InstrItinClass itin, Format f>: Instruction,
-		                                  PredicateControl {
-  field bits<32> Inst;
-  Format Form = f;
-
-  let Namespace = "Nios2";
-  let Size = 4;
-
-  bits<6> Opcode = 0;
-
-  // Bottom 6 bits are the 'opcode' field
-  let Inst{5-0} = Opcode;
-
-  let OutOperandList = outs;
-  let InOperandList  = ins;
-
-  let AsmString = asmstr;
-  let Pattern   = pattern;
-  let Itinerary = itin;
-
-  // Attributes specific to Nios2 instructions:
-
-  // TSFlags layout should be kept in sync with Nios2InstrInfo.h.
-  let TSFlags{5-0} = Form.Value;
-  let DecoderNamespace = "Nios2";
-  field bits<32> SoftFail = 0;
-}
-
-class Nios2Pseudo<dag outs, dag ins, string asmstr, list<dag> pattern,
-      InstrItinClass Itin = IIPseudo>:
-  Nios2Inst32<outs, ins, asmstr, pattern, Itin, Pseudo> {
-
-  let isCodeGenOnly = 1;
-  let isPseudo = 1;
-}
-
-//===----------------------------------------------------------------------===//
-// Base classes for R1 and R2 instructions
-//===----------------------------------------------------------------------===//
-
-class Nios2R1Inst32<dag outs, dag ins, string asmstr, list<dag> pattern,
-                    InstrItinClass itin, Format f>: 
-      Nios2Inst32<outs, ins, asmstr, pattern, itin, f> {
-  let DecoderNamespace = "Nios2";
-  let InstrPredicates = [isNios2r1];
-}
-
-class Nios2R2Inst32<dag outs, dag ins, string asmstr, list<dag> pattern,
-                    InstrItinClass itin, Format f>: 
-      Nios2Inst32<outs, ins, asmstr, pattern, itin, f> {
-  let DecoderNamespace = "Nios2r2";
-  let InstrPredicates = [isNios2r2];
-}
-
-//===----------------------------------------------------------------------===//
-// Format I instruction class in Nios2 : <|A|B|immediate|opcode|>
-//===----------------------------------------------------------------------===//
-
-class FI<bits<6> op, dag outs, dag ins, string asmstr, list<dag> pattern,
-         InstrItinClass itin>: Nios2R1Inst32<outs, ins, asmstr,
-	                                     pattern, itin, FrmI> {
-
-  bits<5>  rA;
-  bits<5>  rB;
-  bits<16> imm;
-
-  let Opcode = op;
-
-  let Inst{31-27} = rA;
-  let Inst{26-22} = rB;
-  let Inst{21-6} = imm;
-}
-
-
-//===----------------------------------------------------------------------===//
-// Format R instruction : <|A|B|C|opx|imm|opcode|>
-//===----------------------------------------------------------------------===//
-
-class FR<bits<6> opx, dag outs, dag ins, string asmstr, list<dag> pattern,
-         InstrItinClass itin>: Nios2R1Inst32<outs, ins, asmstr,
-	                                     pattern, itin, FrmR> {
-  bits<5> rA;
-  bits<5> rB;
-  bits<5> rC;
-  bits<5> imm = 0;
-
-  let Opcode = 0x3a; /* opcode is always 0x3a for R instr. */
-
-  let Inst{31-27} = rA;
-  let Inst{26-22} = rB;
-  let Inst{21-17} = rC;
-  let Inst{16-11} = opx; /* opx stands for opcode extension */
-  let Inst{10-6}  = imm; /* optional 5-bit immediate value */
-}
-
-//===----------------------------------------------------------------------===//
-// Format J instruction class in Nios2 : <|address|opcode|>
-//===----------------------------------------------------------------------===//
-
-class FJ<bits<6> op, dag outs, dag ins, string asmstr, list<dag> pattern,
-         InstrItinClass itin>:
-      Nios2R1Inst32<outs, ins, asmstr, pattern, itin, FrmJ> {
-  bits<26> addr;
-  let Opcode = op;
-  let Inst{31-6} = addr;
-}
-
-//===----------------------------------------------------------------------===//
-// Format F3X6 (R2) instruction : <|opx|RSV|C|B|A|opcode|>
-//===----------------------------------------------------------------------===//
-
-class F3X6<bits<6> opx, dag outs, dag ins, string asmstr, list<dag> pattern,
-           InstrItinClass itin>:
-      Nios2R2Inst32<outs, ins, asmstr, pattern, itin, FrmF3X6> {
-  bits<5> rC;
-  bits<5> rB;
-  bits<5> rA;
-  bits<5> rsv = 0;
-
-  let Opcode = 0x20; /* opcode is always 0x20 (OPX group) for F3X6 instr. */
-
-  let Inst{31-26} = opx; /* opx stands for opcode extension */
-  let Inst{25-21} = rsv;
-  let Inst{20-16} = rC;
-  let Inst{15-11} = rB;
-  let Inst{10-6}  = rA;
-}
-
-//===----------------------------------------------------------------------===//
-// Multiclasses for common instructions of both R1 and R2:
-//===----------------------------------------------------------------------===//
-
-// Multiclass for instructions that have R format in R1 and F3X6 format in R2
-// and their opx values differ between R1 and R2
-multiclass CommonInstr_R_F3X6_opx<bits<6> opxR1, bits<6> opxR2, dag outs,
-                                  dag ins, string asmstr, list<dag> pattern,
-                                  InstrItinClass itin> {
-  def NAME#_R1 : FR<opxR1, outs, ins, asmstr, pattern, itin>;
-  def NAME#_R2 : F3X6<opxR2, outs, ins, asmstr, pattern, itin>;
-}
-
-// Multiclass for instructions that have R format in R1 and F3X6 format in R2
-// and their opx values are the same in R1 and R2
-multiclass CommonInstr_R_F3X6<bits<6> opx, dag outs, dag ins, string asmstr,
-                              list<dag> pattern, InstrItinClass itin> :
-  CommonInstr_R_F3X6_opx<opx, opx, outs, ins, asmstr, pattern, itin>;
-
-// Multiclass for instructions that have I format in R1 and F2I16 format in R2
-// and their op code values differ between R1 and R2
-multiclass CommonInstr_I_F2I16_op<bits<6> opR1, bits<6> opR2, dag outs, dag ins,
-                                  string asmstr, list<dag> pattern,
-                                  InstrItinClass itin> {
-  def NAME#_R1 : FI<opR1, outs, ins, asmstr, pattern, itin>;
-}
-
-// Multiclass for instructions that have I format in R1 and F2I16 format in R2
-// and their op code values are the same in R1 and R2
-multiclass CommonInstr_I_F2I16<bits<6> op, dag outs, dag ins, string asmstr,
-                               list<dag> pattern, InstrItinClass itin> :
-  CommonInstr_I_F2I16_op<op, op, outs, ins, asmstr, pattern, itin>;
diff --git a/lib/Target/Nios2/Nios2InstrInfo.cpp b/lib/Target/Nios2/Nios2InstrInfo.cpp
deleted file mode 100644
index 9700cba3595b3521bc004c40a483381c018ee298..0000000000000000000000000000000000000000
--- a/lib/Target/Nios2/Nios2InstrInfo.cpp
+++ /dev/null
@@ -1,54 +0,0 @@
-//===-- Nios2InstrInfo.cpp - Nios2 Instruction Information ----------------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This file contains the Nios2 implementation of the TargetInstrInfo class.
-//
-//===----------------------------------------------------------------------===//
-
-#include "Nios2InstrInfo.h"
-#include "Nios2TargetMachine.h"
-#include "llvm/CodeGen/MachineInstrBuilder.h"
-
-using namespace llvm;
-
-#define GET_INSTRINFO_CTOR_DTOR
-#include "Nios2GenInstrInfo.inc"
-
-// Pin the vtable to this file.
-void Nios2InstrInfo::anchor() {}
-
-Nios2InstrInfo::Nios2InstrInfo(Nios2Subtarget &ST)
-    : Nios2GenInstrInfo(), RI(ST), Subtarget(ST) {}
-
-/// Expand Pseudo instructions into real backend instructions
-bool Nios2InstrInfo::expandPostRAPseudo(MachineInstr &MI) const {
-  MachineBasicBlock &MBB = *MI.getParent();
-
-  switch (MI.getDesc().getOpcode()) {
-  default:
-    return false;
-  case Nios2::RetRA:
-    BuildMI(MBB, MI, MI.getDebugLoc(), get(Nios2::RET_R1)).addReg(Nios2::RA);
-    break;
-  }
-
-  MBB.erase(MI);
-  return true;
-}
-
-void Nios2InstrInfo::copyPhysReg(MachineBasicBlock &MBB,
-                                 MachineBasicBlock::iterator I,
-                                 const DebugLoc &DL, unsigned DestReg,
-                                 unsigned SrcReg, bool KillSrc) const {
-  unsigned opc = Subtarget.hasNios2r2() ? Nios2::ADD_R2 : Nios2::ADD_R1;
-  BuildMI(MBB, I, DL, get(opc))
-    .addReg(DestReg, RegState::Define)
-    .addReg(Nios2::ZERO)
-    .addReg(SrcReg, getKillRegState(KillSrc));
-}
diff --git a/lib/Target/Nios2/Nios2InstrInfo.h b/lib/Target/Nios2/Nios2InstrInfo.h
deleted file mode 100644
index 52f6e7e9c7c82f0ea9b2ac4f17b4380e9b567f2d..0000000000000000000000000000000000000000
--- a/lib/Target/Nios2/Nios2InstrInfo.h
+++ /dev/null
@@ -1,49 +0,0 @@
-//===-- Nios2InstrInfo.h - Nios2 Instruction Information --------*- C++ -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This file contains the Nios2 implementation of the TargetInstrInfo class.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_LIB_TARGET_NIOS2_NIOS2INSTRINFO_H
-#define LLVM_LIB_TARGET_NIOS2_NIOS2INSTRINFO_H
-
-#include "Nios2RegisterInfo.h"
-#include "llvm/CodeGen/TargetInstrInfo.h"
-
-#define GET_INSTRINFO_HEADER
-#include "Nios2GenInstrInfo.inc"
-
-namespace llvm {
-
-class Nios2Subtarget;
-
-class Nios2InstrInfo : public Nios2GenInstrInfo {
-  const Nios2RegisterInfo RI;
-  const Nios2Subtarget &Subtarget;
-  virtual void anchor();
-
-public:
-  explicit Nios2InstrInfo(Nios2Subtarget &ST);
-
-  /// getRegisterInfo - TargetInstrInfo is a superset of MRegister info.  As
-  /// such, whenever a client has an instance of instruction info, it should
-  /// always be able to get register info as well (through this method).
-  ///
-  const Nios2RegisterInfo &getRegisterInfo() const { return RI; };
-
-  bool expandPostRAPseudo(MachineInstr &MI) const override;
-
-  void copyPhysReg(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
-                   const DebugLoc &DL, unsigned DestReg, unsigned SrcReg,
-                   bool KillSrc) const override;
-};
-} // namespace llvm
-
-#endif
diff --git a/lib/Target/Nios2/Nios2InstrInfo.td b/lib/Target/Nios2/Nios2InstrInfo.td
deleted file mode 100644
index dee84f74bcbe03c24557cfe0c35293a529151a81..0000000000000000000000000000000000000000
--- a/lib/Target/Nios2/Nios2InstrInfo.td
+++ /dev/null
@@ -1,109 +0,0 @@
-//===- Nios2InstrInfo.td - Target Description for Nios2 ------*- tablegen -*-=//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This file contains the Nios2 implementation of the TargetInstrInfo class.
-//
-//===----------------------------------------------------------------------===//
-
-//===----------------------------------------------------------------------===//
-// Instruction format superclass
-//===----------------------------------------------------------------------===//
-
-include "Nios2InstrFormats.td"
-
-
-//===----------------------------------------------------------------------===//
-// Nios2 Operand, Complex Patterns and Transformations Definitions.
-//===----------------------------------------------------------------------===//
-
-def simm16     : Operand<i32> {
-  let DecoderMethod= "DecodeSimm16";
-}
-
-// Node immediate fits as 16-bit sign extended on target immediate.
-// e.g. addi, andi
-def immSExt16  : PatLeaf<(imm), [{ return isInt<16>(N->getSExtValue()); }]>;
-
-// Custom return SDNode
-def Nios2Ret : SDNode<"Nios2ISD::Ret", SDTNone,
-    [SDNPHasChain, SDNPOptInGlue, SDNPVariadic]>;
-
-//===----------------------------------------------------------------------===//
-// Instructions specific format
-//===----------------------------------------------------------------------===//
-
-// Arithmetic and logical instructions with 2 registers and 16-bit immediate
-// value.
-multiclass ArithLogicRegImm16<bits<6> op, string mnemonic, SDNode opNode,
-                              Operand immOp, PatLeaf immType>:
-           CommonInstr_I_F2I16<op, (outs CPURegs:$rB),
-	                       (ins CPURegs:$rA, immOp:$imm),
-                               !strconcat(mnemonic, "\t$rB, $rA, $imm"),
-                               [(set CPURegs:$rB,
-			         (opNode CPURegs:$rA, immType:$imm))],
-                               IIAlu>;
-
-// Arithmetic and logical instructions with 3 register operands.
-// Defines R1 and R2 instruction at the same time.
-multiclass ArithLogicReg<bits<6> opx, string mnemonic,
-                         SDNode opNode>:
-  CommonInstr_R_F3X6<opx, (outs CPURegs:$rC),
-                     (ins CPURegs:$rA, CPURegs:$rB),
-                     !strconcat(mnemonic, "\t$rC, $rA, $rB"),
-                     [(set CPURegs:$rC, (opNode CPURegs:$rA, CPURegs:$rB))],
-                     IIAlu>;
-
-multiclass Return<bits<6> opx, dag outs, dag ins, string mnemonic> {
-  let rB = 0, rC = 0,
-      isReturn = 1,
-      isCodeGenOnly = 1,
-      hasCtrlDep = 1,
-      hasExtraSrcRegAllocReq = 1 in {
-    defm NAME# : CommonInstr_R_F3X6<opx, outs, ins, mnemonic, [], IIBranch>;
-  }
-}
-
-//===----------------------------------------------------------------------===//
-// Nios2 Instructions
-//===----------------------------------------------------------------------===//
-
-/// Arithmetic instructions operating on registers.
-let isCommutable = 1 ,
-    isReMaterializable = 1 in {
-  defm ADD    : ArithLogicReg<0x31, "add",    add>;
-  defm AND    : ArithLogicReg<0x0e, "and",    and>;
-  defm OR     : ArithLogicReg<0x16, "or",     or>;
-  defm XOR    : ArithLogicReg<0x1e, "xor",    xor>;
-  defm MUL    : ArithLogicReg<0x27, "mul",    mul>;
-}
-
-let isReMaterializable = 1 in {
-  defm SUB    : ArithLogicReg<0x39, "sub",    sub>;
-}
-
-defm DIVU : ArithLogicReg<0x24, "divu",   udiv>;
-defm DIV  : ArithLogicReg<0x25, "div",    sdiv>;
-
-defm SLL : ArithLogicReg<0x13, "sll",  shl>;
-defm SRL : ArithLogicReg<0x1b, "srl",  srl>;
-defm SRA : ArithLogicReg<0x3b, "sra",  sra>;
-
-/// Arithmetic Instructions (ALU Immediate)
-defm ADDI  : ArithLogicRegImm16<0x04, "addi",  add, simm16, immSExt16>;
-
-// Returns:
-defm RET  : Return<0x05, (outs), (ins CPURegs:$rA), "ret">;
-
-//===----------------------------------------------------------------------===//
-// Pseudo instructions
-//===----------------------------------------------------------------------===//
-
-// Return RA.
-let isReturn=1, isTerminator=1, hasDelaySlot=1, isBarrier=1, hasCtrlDep=1 in
-def RetRA : Nios2Pseudo<(outs), (ins), "", [(Nios2Ret)]>;
diff --git a/lib/Target/Nios2/Nios2MCInstLower.cpp b/lib/Target/Nios2/Nios2MCInstLower.cpp
deleted file mode 100644
index c43af879b8a669c2f884a1246e21c95eafe3c003..0000000000000000000000000000000000000000
--- a/lib/Target/Nios2/Nios2MCInstLower.cpp
+++ /dev/null
@@ -1,117 +0,0 @@
-//===-- Nios2MCInstLower.cpp - Convert Nios2 MachineInstr to MCInst -------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This file contains code to lower Nios2 MachineInstrs to their corresponding
-// MCInst records.
-//
-//===----------------------------------------------------------------------===//
-
-#include "MCTargetDesc/Nios2BaseInfo.h"
-#include "MCTargetDesc/Nios2MCExpr.h"
-#include "Nios2.h"
-#include "llvm/CodeGen/AsmPrinter.h"
-#include "llvm/CodeGen/MachineInstr.h"
-#include "llvm/CodeGen/MachineOperand.h"
-
-using namespace llvm;
-
-static MCOperand LowerSymbolOperand(const MachineOperand &MO, AsmPrinter &AP) {
-  MCSymbolRefExpr::VariantKind Kind = MCSymbolRefExpr::VK_None;
-  Nios2MCExpr::Nios2ExprKind TargetKind = Nios2MCExpr::CEK_None;
-  const MCSymbol *Symbol;
-
-  switch (MO.getTargetFlags()) {
-  default:
-    llvm_unreachable("Invalid target flag!");
-  case Nios2FG::MO_NO_FLAG:
-    break;
-  case Nios2FG::MO_ABS_HI:
-    TargetKind = Nios2MCExpr::CEK_ABS_HI;
-    break;
-  case Nios2FG::MO_ABS_LO:
-    TargetKind = Nios2MCExpr::CEK_ABS_LO;
-    break;
-  }
-
-  switch (MO.getType()) {
-  case MachineOperand::MO_GlobalAddress:
-    Symbol = AP.getSymbol(MO.getGlobal());
-    break;
-
-  case MachineOperand::MO_MachineBasicBlock:
-    Symbol = MO.getMBB()->getSymbol();
-    break;
-
-  case MachineOperand::MO_BlockAddress:
-    Symbol = AP.GetBlockAddressSymbol(MO.getBlockAddress());
-    break;
-
-  case MachineOperand::MO_ExternalSymbol:
-    Symbol = AP.GetExternalSymbolSymbol(MO.getSymbolName());
-    break;
-
-  case MachineOperand::MO_JumpTableIndex:
-    Symbol = AP.GetJTISymbol(MO.getIndex());
-    break;
-
-  case MachineOperand::MO_ConstantPoolIndex:
-    Symbol = AP.GetCPISymbol(MO.getIndex());
-    break;
-
-  default:
-    llvm_unreachable("<unknown operand type>");
-  }
-
-  const MCExpr *Expr = MCSymbolRefExpr::create(Symbol, Kind, AP.OutContext);
-
-  if (TargetKind != Nios2MCExpr::CEK_None)
-    Expr = Nios2MCExpr::create(TargetKind, Expr, AP.OutContext);
-
-  return MCOperand::createExpr(Expr);
-}
-
-static MCOperand LowerOperand(const MachineOperand &MO, AsmPrinter &AP) {
-
-  switch (MO.getType()) {
-  default:
-    llvm_unreachable("unknown operand type");
-  case MachineOperand::MO_Register:
-    // Ignore all implicit register operands.
-    if (MO.isImplicit())
-      break;
-    return MCOperand::createReg(MO.getReg());
-  case MachineOperand::MO_Immediate:
-    return MCOperand::createImm(MO.getImm());
-  case MachineOperand::MO_MachineBasicBlock:
-  case MachineOperand::MO_ExternalSymbol:
-  case MachineOperand::MO_JumpTableIndex:
-  case MachineOperand::MO_BlockAddress:
-  case MachineOperand::MO_GlobalAddress:
-  case MachineOperand::MO_ConstantPoolIndex:
-    return LowerSymbolOperand(MO, AP);
-  case MachineOperand::MO_RegisterMask:
-    break;
-  }
-
-  return MCOperand();
-}
-
-void llvm::LowerNios2MachineInstToMCInst(const MachineInstr *MI, MCInst &OutMI,
-                                         AsmPrinter &AP) {
-
-  OutMI.setOpcode(MI->getOpcode());
-
-  for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) {
-    const MachineOperand &MO = MI->getOperand(i);
-    MCOperand MCOp = LowerOperand(MO, AP);
-
-    if (MCOp.isValid())
-      OutMI.addOperand(MCOp);
-  }
-}
diff --git a/lib/Target/Nios2/Nios2MachineFunction.cpp b/lib/Target/Nios2/Nios2MachineFunction.cpp
deleted file mode 100644
index be5b8829fe36d923f2969819edd93ada83b520f0..0000000000000000000000000000000000000000
--- a/lib/Target/Nios2/Nios2MachineFunction.cpp
+++ /dev/null
@@ -1,14 +0,0 @@
-//===-- Nios2MachineFunctionInfo.cpp - Private data used for Nios2 --------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-
-#include "Nios2MachineFunction.h"
-
-using namespace llvm;
-
-void Nios2FunctionInfo::anchor() {}
diff --git a/lib/Target/Nios2/Nios2MachineFunction.h b/lib/Target/Nios2/Nios2MachineFunction.h
deleted file mode 100644
index 73baf9694790c9343e3345f93029a70dac3d5f4b..0000000000000000000000000000000000000000
--- a/lib/Target/Nios2/Nios2MachineFunction.h
+++ /dev/null
@@ -1,62 +0,0 @@
-//===-- Nios2MachineFunctionInfo.h - Private data used for Nios2 --*- C++ -*-=//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This file declares the Nios2 specific subclass of MachineFunctionInfo.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_LIB_TARGET_NIOS2_NIOS2MACHINEFUNCTION_H
-#define LLVM_LIB_TARGET_NIOS2_NIOS2MACHINEFUNCTION_H
-
-#include "llvm/CodeGen/MachineFunction.h"
-
-namespace llvm {
-
-/// Nios2FunctionInfo - This class is derived from MachineFunction private
-/// Nios2 target-specific information for each MachineFunction.
-class Nios2FunctionInfo : public MachineFunctionInfo {
-  virtual void anchor();
-
-private:
-  unsigned GlobalBaseReg;
-
-  /// VarArgsFrameOffset - Frame offset to start of varargs area.
-  int VarArgsFrameOffset;
-
-  /// SRetReturnReg - Holds the virtual register into which the sret
-  /// argument is passed.
-  unsigned SRetReturnReg;
-
-  /// IsLeafProc - True if the function is a leaf procedure.
-  bool IsLeafProc;
-
-public:
-  Nios2FunctionInfo()
-      : GlobalBaseReg(0), VarArgsFrameOffset(0), SRetReturnReg(0),
-        IsLeafProc(false) {}
-  explicit Nios2FunctionInfo(MachineFunction &MF)
-      : GlobalBaseReg(0), VarArgsFrameOffset(0), SRetReturnReg(0),
-        IsLeafProc(false) {}
-
-  unsigned getGlobalBaseReg() const { return GlobalBaseReg; }
-  void setGlobalBaseReg(unsigned Reg) { GlobalBaseReg = Reg; }
-
-  int getVarArgsFrameOffset() const { return VarArgsFrameOffset; }
-  void setVarArgsFrameOffset(int Offset) { VarArgsFrameOffset = Offset; }
-
-  unsigned getSRetReturnReg() const { return SRetReturnReg; }
-  void setSRetReturnReg(unsigned Reg) { SRetReturnReg = Reg; }
-
-  void setLeafProc(bool rhs) { IsLeafProc = rhs; }
-  bool isLeafProc() const { return IsLeafProc; }
-};
-
-} // end of namespace llvm
-
-#endif // NIOS2_MACHINE_FUNCTION_INFO_H
diff --git a/lib/Target/Nios2/Nios2RegisterInfo.cpp b/lib/Target/Nios2/Nios2RegisterInfo.cpp
deleted file mode 100644
index 9b892f9175353b7f8cf8afa9c9902f58e3e32c2c..0000000000000000000000000000000000000000
--- a/lib/Target/Nios2/Nios2RegisterInfo.cpp
+++ /dev/null
@@ -1,55 +0,0 @@
-//===-- Nios2RegisterInfo.cpp - Nios2 Register Information -== ------------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This file contains the Nios2 implementation of the TargetRegisterInfo class.
-//
-//===----------------------------------------------------------------------===//
-
-#define DEBUG_TYPE "nios2-reg-info"
-
-#include "Nios2RegisterInfo.h"
-
-#include "Nios2.h"
-#include "Nios2Subtarget.h"
-
-#define GET_REGINFO_TARGET_DESC
-#include "Nios2GenRegisterInfo.inc"
-
-using namespace llvm;
-
-Nios2RegisterInfo::Nios2RegisterInfo(const Nios2Subtarget &ST)
-    : Nios2GenRegisterInfo(Nios2::RA), Subtarget(ST) {}
-
-const TargetRegisterClass *Nios2RegisterInfo::intRegClass(unsigned Size) const {
-  return &Nios2::CPURegsRegClass;
-}
-
-const MCPhysReg *
-Nios2RegisterInfo::getCalleeSavedRegs(const MachineFunction *MF) const {
-  return CSR_SaveList;
-}
-
-BitVector Nios2RegisterInfo::getReservedRegs(const MachineFunction &MF) const {
-  static const MCPhysReg ReservedCPURegs[] = {Nios2::ZERO, Nios2::AT, Nios2::SP,
-                                             Nios2::RA,   Nios2::PC, Nios2::GP};
-  BitVector Reserved(getNumRegs());
-
-  for (unsigned I = 0; I < array_lengthof(ReservedCPURegs); ++I)
-    Reserved.set(ReservedCPURegs[I]);
-
-  return Reserved;
-}
-
-void Nios2RegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator II,
-                                            int SPAdj, unsigned FIOperandNum,
-                                            RegScavenger *RS) const {}
-
-unsigned Nios2RegisterInfo::getFrameRegister(const MachineFunction &MF) const {
-  return Nios2::SP;
-}
diff --git a/lib/Target/Nios2/Nios2RegisterInfo.h b/lib/Target/Nios2/Nios2RegisterInfo.h
deleted file mode 100644
index 3658343b1d2e8928e48b3aec2d3f0504c791bd17..0000000000000000000000000000000000000000
--- a/lib/Target/Nios2/Nios2RegisterInfo.h
+++ /dev/null
@@ -1,52 +0,0 @@
-//===-- Nios2RegisterInfo.h - Nios2 Register Information Impl ---*- C++ -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This file contains the Nios2 implementation of the TargetRegisterInfo class.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_LIB_TARGET_NIOS2_NIOS2REGISTERINFO_H
-#define LLVM_LIB_TARGET_NIOS2_NIOS2REGISTERINFO_H
-
-#include "Nios2.h"
-#include "llvm/CodeGen/TargetRegisterInfo.h"
-
-#define GET_REGINFO_HEADER
-#include "Nios2GenRegisterInfo.inc"
-
-namespace llvm {
-class Nios2Subtarget;
-class TargetInstrInfo;
-class Type;
-
-class Nios2RegisterInfo : public Nios2GenRegisterInfo {
-protected:
-  const Nios2Subtarget &Subtarget;
-
-public:
-  Nios2RegisterInfo(const Nios2Subtarget &Subtarget);
-
-  const MCPhysReg *getCalleeSavedRegs(const MachineFunction *MF) const override;
-
-  BitVector getReservedRegs(const MachineFunction &MF) const override;
-
-  /// Stack Frame Processing Methods
-  void eliminateFrameIndex(MachineBasicBlock::iterator II, int SPAdj,
-                           unsigned FIOperandNum,
-                           RegScavenger *RS = nullptr) const override;
-
-  /// Debug information queries.
-  unsigned getFrameRegister(const MachineFunction &MF) const override;
-
-  /// Return GPR register class.
-  const TargetRegisterClass *intRegClass(unsigned Size) const;
-};
-
-} // end namespace llvm
-#endif
diff --git a/lib/Target/Nios2/Nios2RegisterInfo.td b/lib/Target/Nios2/Nios2RegisterInfo.td
deleted file mode 100644
index 1808815816f3b3292f0f734dc980a7c1e74320e7..0000000000000000000000000000000000000000
--- a/lib/Target/Nios2/Nios2RegisterInfo.td
+++ /dev/null
@@ -1,60 +0,0 @@
-//===-- Nios2RegisterInfo.td - Nios2 Register defs ---------*- tablegen -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-
-// We have bank of 32 registers.
-class Nios2Reg<string n> : Register<n> {
-  field bits<5> Num;
-  let Namespace = "Nios2";
-}
-
-// Nios2 CPU Registers
-class Nios2GPRReg<bits<5> num, string n> : Nios2Reg<n> {
-  let Num = num;
-}
-
-//===----------------------------------------------------------------------===//
-//  Registers
-//===----------------------------------------------------------------------===//
-
-let Namespace = "Nios2" in {
-  // General Purpose Registers
-  def ZERO : Nios2GPRReg<0, "zero">, DwarfRegNum<[ 0 ]>;
-  def AT : Nios2GPRReg<1, "at">, DwarfRegNum<[ 1 ]>;
-  foreach RegNum = 2 - 23 in {
-    def R #RegNum : Nios2GPRReg<RegNum, "r" #RegNum>, DwarfRegNum<[ RegNum ]>;
-  }
-  def ET : Nios2GPRReg<24, "et">, DwarfRegNum<[ 24 ]>;
-  def BT : Nios2GPRReg<25, "bt">, DwarfRegNum<[ 25 ]>;
-  def GP : Nios2GPRReg<26, "gp">, DwarfRegNum<[ 26 ]>;
-  def SP : Nios2GPRReg<27, "sp">, DwarfRegNum<[ 27 ]>;
-  def FP : Nios2GPRReg<28, "fp">, DwarfRegNum<[ 28 ]>;
-  def EA : Nios2GPRReg<29, "ea">, DwarfRegNum<[ 29 ]>;
-  def BA : Nios2GPRReg<30, "ba">, DwarfRegNum<[ 30 ]>;
-  def RA : Nios2GPRReg<31, "ra">, DwarfRegNum<[ 31 ]>;
-  def PC : Nios2Reg<"pc">, DwarfRegNum<[ 32 ]>;
-}
-
-//===----------------------------------------------------------------------===//
-// Register Classes
-//===----------------------------------------------------------------------===//
-
-def CPURegs : RegisterClass<"Nios2", [ i32 ], 32,
-                            (add
-                            // Reserved
-                            ZERO,
-                            AT,
-                            // Return Values and Arguments
-                            (sequence "R%u", 2, 7),
-                            // Not preserved across procedure calls
-                            // Caller saved
-                            (sequence "R%u", 8, 15),
-                            // Callee saved
-                            (sequence "R%u", 16, 23),
-                            // Reserved
-                            ET, BT, GP, SP, FP, EA, BA, RA, PC)>;
diff --git a/lib/Target/Nios2/Nios2Schedule.td b/lib/Target/Nios2/Nios2Schedule.td
deleted file mode 100644
index 2d1d9d5e5f3fa105be5602fff844218468701273..0000000000000000000000000000000000000000
--- a/lib/Target/Nios2/Nios2Schedule.td
+++ /dev/null
@@ -1,39 +0,0 @@
-//===-- Nios2Schedule.td - Nios2 Scheduling Definitions ----*- tablegen -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-
-//===----------------------------------------------------------------------===//
-// Functional units across Nios2 chips sets. Based on GCC/Nios2 backend files.
-//===----------------------------------------------------------------------===//
-def ALU     : FuncUnit;
-def IMULDIV : FuncUnit;
-
-//===----------------------------------------------------------------------===//
-// Instruction Itinerary classes used for Nios2
-//===----------------------------------------------------------------------===//
-def IIAlu              : InstrItinClass;
-def IILoad             : InstrItinClass;
-def IIStore            : InstrItinClass;
-def IIFlush            : InstrItinClass;
-def IIIdiv             : InstrItinClass;
-def IIBranch           : InstrItinClass;
-
-def IIPseudo           : InstrItinClass;
-
-//===----------------------------------------------------------------------===//
-// Nios2 Generic instruction itineraries.
-//===----------------------------------------------------------------------===//
-//@ http://llvm.org/docs/doxygen/html/structllvm_1_1InstrStage.html
-def Nios2GenericItineraries : ProcessorItineraries<[ALU, IMULDIV], [], [
-  InstrItinData<IIAlu              , [InstrStage<1,  [ALU]>]>,
-  InstrItinData<IILoad             , [InstrStage<3,  [ALU]>]>,
-  InstrItinData<IIStore            , [InstrStage<1,  [ALU]>]>,
-  InstrItinData<IIFlush            , [InstrStage<1,  [ALU]>]>,
-  InstrItinData<IIIdiv             , [InstrStage<38, [IMULDIV]>]>,
-  InstrItinData<IIBranch           , [InstrStage<1,  [ALU]>]>
-]>;
diff --git a/lib/Target/Nios2/Nios2Subtarget.cpp b/lib/Target/Nios2/Nios2Subtarget.cpp
deleted file mode 100644
index 196bed20cdcc70685099cb141822ff78abd3868a..0000000000000000000000000000000000000000
--- a/lib/Target/Nios2/Nios2Subtarget.cpp
+++ /dev/null
@@ -1,56 +0,0 @@
-//===-- Nios2Subtarget.cpp - Nios2 Subtarget Information ------------------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This file implements the Nios2 specific subclass of TargetSubtargetInfo.
-//
-//===----------------------------------------------------------------------===//
-
-#include "Nios2Subtarget.h"
-#include "Nios2.h"
-
-using namespace llvm;
-
-#define DEBUG_TYPE "nios2-subtarget"
-
-#define GET_SUBTARGETINFO_TARGET_DESC
-#define GET_SUBTARGETINFO_CTOR
-#include "Nios2GenSubtargetInfo.inc"
-
-void Nios2Subtarget::anchor() {}
-
-Nios2Subtarget::Nios2Subtarget(const Triple &TT, const std::string &CPU,
-                               const std::string &FS, const TargetMachine &TM)
-    :
-
-      // Nios2GenSubtargetInfo will display features by llc -march=nios2
-      // -mcpu=help
-      Nios2GenSubtargetInfo(TT, CPU, FS), TargetTriple(TT),
-      InstrInfo(initializeSubtargetDependencies(CPU, FS)), TLInfo(TM, *this),
-      FrameLowering(*this) {}
-
-Nios2Subtarget &Nios2Subtarget::initializeSubtargetDependencies(StringRef CPU,
-                                                                StringRef FS) {
-  if (TargetTriple.getArch() == Triple::nios2) {
-    if (CPU != "nios2r2") {
-      CPU = "nios2r1";
-      Nios2ArchVersion = Nios2r1;
-    } else {
-      Nios2ArchVersion = Nios2r2;
-    }
-  } else {
-    errs() << "!!!Error, TargetTriple.getArch() = " << TargetTriple.getArch()
-           << "CPU = " << CPU << "\n";
-    exit(0);
-  }
-
-  // Parse features string.
-  ParseSubtargetFeatures(CPU, FS);
-
-  return *this;
-}
diff --git a/lib/Target/Nios2/Nios2Subtarget.h b/lib/Target/Nios2/Nios2Subtarget.h
deleted file mode 100644
index a822dff33b5b1a1f1d50371864da02da6d07701e..0000000000000000000000000000000000000000
--- a/lib/Target/Nios2/Nios2Subtarget.h
+++ /dev/null
@@ -1,97 +0,0 @@
-//===-- Nios2Subtarget.h - Define Subtarget for the Nios2 -------*- C++ -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This file declares the Nios2 specific subclass of TargetSubtargetInfo.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_LIB_TARGET_NIOS2_NIOS2SUBTARGET_H
-#define LLVM_LIB_TARGET_NIOS2_NIOS2SUBTARGET_H
-
-#include "Nios2FrameLowering.h"
-#include "Nios2ISelLowering.h"
-#include "Nios2InstrInfo.h"
-#include "llvm/CodeGen/SelectionDAGTargetInfo.h"
-#include "llvm/CodeGen/TargetFrameLowering.h"
-#include "llvm/CodeGen/TargetSubtargetInfo.h"
-
-#define GET_SUBTARGETINFO_HEADER
-#include "Nios2GenSubtargetInfo.inc"
-
-namespace llvm {
-class StringRef;
-
-class Nios2TargetMachine;
-
-class Nios2Subtarget : public Nios2GenSubtargetInfo {
-  virtual void anchor();
-
-public:
-  // Nios2 R2 features
-  // Bit manipulation instructions extension
-  bool HasBMX;
-  // Code Density instructions extension
-  bool HasCDX;
-  // Multi-Processor instructions extension
-  bool HasMPX;
-  // New mandatory instructions
-  bool HasR2Mandatory;
-
-protected:
-  enum Nios2ArchEnum {
-    // Nios2 R1 ISA
-    Nios2r1,
-    // Nios2 R2 ISA
-    Nios2r2
-  };
-
-  // Nios2 architecture version
-  Nios2ArchEnum Nios2ArchVersion;
-
-  Triple TargetTriple;
-
-  Nios2InstrInfo InstrInfo;
-  Nios2TargetLowering TLInfo;
-  SelectionDAGTargetInfo TSInfo;
-  Nios2FrameLowering FrameLowering;
-
-public:
-  /// This constructor initializes the data members to match that
-  /// of the specified triple.
-  Nios2Subtarget(const Triple &TT, const std::string &CPU,
-                 const std::string &FS, const TargetMachine &TM);
-
-  /// ParseSubtargetFeatures - Parses features string setting specified
-  /// subtarget options.  Definition of function is auto generated by tblgen.
-  void ParseSubtargetFeatures(StringRef CPU, StringRef FS);
-
-  bool hasNios2r1() const { return Nios2ArchVersion >= Nios2r1; }
-  bool isNios2r1() const { return Nios2ArchVersion == Nios2r1; }
-  bool hasNios2r2() const { return Nios2ArchVersion >= Nios2r2; }
-  bool isNios2r2() const { return Nios2ArchVersion == Nios2r2; }
-
-  Nios2Subtarget &initializeSubtargetDependencies(StringRef CPU, StringRef FS);
-
-  const Nios2InstrInfo *getInstrInfo() const override { return &InstrInfo; }
-  const TargetFrameLowering *getFrameLowering() const override {
-    return &FrameLowering;
-  }
-  const Nios2RegisterInfo *getRegisterInfo() const override {
-    return &InstrInfo.getRegisterInfo();
-  }
-  const Nios2TargetLowering *getTargetLowering() const override {
-    return &TLInfo;
-  }
-  const SelectionDAGTargetInfo *getSelectionDAGInfo() const override {
-    return &TSInfo;
-  }
-};
-} // namespace llvm
-
-#endif
diff --git a/lib/Target/Nios2/Nios2TargetMachine.cpp b/lib/Target/Nios2/Nios2TargetMachine.cpp
deleted file mode 100644
index 4f90db9e6f6a50e3d4bf6ed8482183765f16a2a1..0000000000000000000000000000000000000000
--- a/lib/Target/Nios2/Nios2TargetMachine.cpp
+++ /dev/null
@@ -1,111 +0,0 @@
-//===-- Nios2TargetMachine.cpp - Define TargetMachine for Nios2 -----------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// Implements the info about Nios2 target spec.
-//
-//===----------------------------------------------------------------------===//
-
-#include "Nios2TargetMachine.h"
-#include "Nios2.h"
-#include "Nios2TargetObjectFile.h"
-
-#include "llvm/CodeGen/TargetPassConfig.h"
-#include "llvm/Support/TargetRegistry.h"
-
-using namespace llvm;
-
-#define DEBUG_TYPE "nios2"
-
-extern "C" void LLVMInitializeNios2Target() {
-  // Register the target.
-  RegisterTargetMachine<Nios2TargetMachine> X(getTheNios2Target());
-}
-
-static std::string computeDataLayout() {
-  return "e-p:32:32:32-i8:8:32-i16:16:32-n32";
-}
-
-static Reloc::Model getEffectiveRelocModel(Optional<Reloc::Model> RM) {
-  if (!RM.hasValue())
-    return Reloc::Static;
-  return *RM;
-}
-
-Nios2TargetMachine::Nios2TargetMachine(const Target &T, const Triple &TT,
-                                       StringRef CPU, StringRef FS,
-                                       const TargetOptions &Options,
-                                       Optional<Reloc::Model> RM,
-                                       Optional<CodeModel::Model> CM,
-                                       CodeGenOpt::Level OL, bool JIT)
-    : LLVMTargetMachine(T, computeDataLayout(), TT, CPU, FS, Options,
-                        getEffectiveRelocModel(RM),
-                        getEffectiveCodeModel(CM, CodeModel::Small), OL),
-      TLOF(make_unique<Nios2TargetObjectFile>()),
-      Subtarget(TT, CPU, FS, *this) {
-  initAsmInfo();
-}
-
-Nios2TargetMachine::~Nios2TargetMachine() {}
-
-const Nios2Subtarget *
-Nios2TargetMachine::getSubtargetImpl(const Function &F) const {
-  Attribute CPUAttr = F.getFnAttribute("target-cpu");
-  Attribute FSAttr = F.getFnAttribute("target-features");
-
-  std::string CPU = !CPUAttr.hasAttribute(Attribute::None)
-                        ? CPUAttr.getValueAsString().str()
-                        : TargetCPU;
-  std::string FS = !FSAttr.hasAttribute(Attribute::None)
-                       ? FSAttr.getValueAsString().str()
-                       : TargetFS;
-
-  auto &I = SubtargetMap[CPU + FS];
-  if (!I) {
-    // This needs to be done before we create a new subtarget since any
-    // creation will depend on the TM and the code generation flags on the
-    // function that reside in TargetOptions.
-    resetTargetOptions(F);
-    I = llvm::make_unique<Nios2Subtarget>(TargetTriple, CPU, FS, *this);
-  }
-  return I.get();
-}
-
-namespace {
-/// Nios2 Code Generator Pass Configuration Options.
-class Nios2PassConfig : public TargetPassConfig {
-public:
-  Nios2PassConfig(Nios2TargetMachine &TM, PassManagerBase *PM)
-      : TargetPassConfig(TM, *PM) {}
-
-  Nios2TargetMachine &getNios2TargetMachine() const {
-    return getTM<Nios2TargetMachine>();
-  }
-
-  void addCodeGenPrepare() override;
-  bool addInstSelector() override;
-  void addIRPasses() override;
-};
-} // namespace
-
-TargetPassConfig *Nios2TargetMachine::createPassConfig(PassManagerBase &PM) {
-  return new Nios2PassConfig(*this, &PM);
-}
-
-void Nios2PassConfig::addCodeGenPrepare() {
-  TargetPassConfig::addCodeGenPrepare();
-}
-
-void Nios2PassConfig::addIRPasses() { TargetPassConfig::addIRPasses(); }
-
-// Install an instruction selector pass using
-// the ISelDag to gen Nios2 code.
-bool Nios2PassConfig::addInstSelector() {
-  addPass(createNios2ISelDag(getNios2TargetMachine(), getOptLevel()));
-  return false;
-}
diff --git a/lib/Target/Nios2/Nios2TargetMachine.h b/lib/Target/Nios2/Nios2TargetMachine.h
deleted file mode 100644
index 1ebfb397383e7b80cb010a7092e8a81c2f7821d6..0000000000000000000000000000000000000000
--- a/lib/Target/Nios2/Nios2TargetMachine.h
+++ /dev/null
@@ -1,45 +0,0 @@
-//===-- Nios2TargetMachine.h - Define TargetMachine for Nios2 ---*- C++ -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This file declares the Nios2 specific subclass of TargetMachine.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_LIB_TARGET_NIOS2_NIOS2TARGETMACHINE_H
-#define LLVM_LIB_TARGET_NIOS2_NIOS2TARGETMACHINE_H
-
-#include "Nios2Subtarget.h"
-#include "llvm/Target/TargetMachine.h"
-
-namespace llvm {
-class Nios2TargetMachine : public LLVMTargetMachine {
-  mutable StringMap<std::unique_ptr<Nios2Subtarget>> SubtargetMap;
-  std::unique_ptr<TargetLoweringObjectFile> TLOF;
-  Nios2Subtarget Subtarget;
-
-public:
-  Nios2TargetMachine(const Target &T, const Triple &TT, StringRef CPU,
-                     StringRef FS, const TargetOptions &Options,
-                     Optional<Reloc::Model> RM, Optional<CodeModel::Model> CM,
-                     CodeGenOpt::Level OL, bool JIT);
-  ~Nios2TargetMachine() override;
-
-  const Nios2Subtarget *getSubtargetImpl() const { return &Subtarget; }
-  const Nios2Subtarget *getSubtargetImpl(const Function &F) const override;
-
-  TargetLoweringObjectFile *getObjFileLowering() const override {
-    return TLOF.get();
-  }
-
-  // Pass Pipeline Configuration
-  TargetPassConfig *createPassConfig(PassManagerBase &PM) override;
-};
-} // namespace llvm
-
-#endif
diff --git a/lib/Target/Nios2/Nios2TargetObjectFile.cpp b/lib/Target/Nios2/Nios2TargetObjectFile.cpp
deleted file mode 100644
index 5fc85ef487e69088bd912473464d96e89e189feb..0000000000000000000000000000000000000000
--- a/lib/Target/Nios2/Nios2TargetObjectFile.cpp
+++ /dev/null
@@ -1,18 +0,0 @@
-//===-- Nios2TargetObjectFile.cpp - Nios2 Object Files --------------------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-
-#include "Nios2TargetObjectFile.h"
-
-using namespace llvm;
-
-void Nios2TargetObjectFile::Initialize(MCContext &Ctx,
-                                       const TargetMachine &TM) {
-  TargetLoweringObjectFileELF::Initialize(Ctx, TM);
-  InitializeELF(TM.Options.UseInitArray);
-}
diff --git a/lib/Target/Nios2/Nios2TargetObjectFile.h b/lib/Target/Nios2/Nios2TargetObjectFile.h
deleted file mode 100644
index e9ed6e31d9374462ae330c02e2a50f12ec8af456..0000000000000000000000000000000000000000
--- a/lib/Target/Nios2/Nios2TargetObjectFile.h
+++ /dev/null
@@ -1,26 +0,0 @@
-//===-- llvm/Target/Nios2TargetObjectFile.h - Nios2 Object Info -*- C++ -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_LIB_TARGET_NIOS2_NIOS2TARGETOBJECTFILE_H
-#define LLVM_LIB_TARGET_NIOS2_NIOS2TARGETOBJECTFILE_H
-
-#include "Nios2TargetMachine.h"
-#include "llvm/CodeGen/TargetLoweringObjectFileImpl.h"
-
-namespace llvm {
-
-class Nios2TargetObjectFile : public TargetLoweringObjectFileELF {
-public:
-  Nios2TargetObjectFile() : TargetLoweringObjectFileELF() {}
-
-  void Initialize(MCContext &Ctx, const TargetMachine &TM) override;
-};
-} // end namespace llvm
-
-#endif
diff --git a/lib/Target/Nios2/Nios2TargetStreamer.h b/lib/Target/Nios2/Nios2TargetStreamer.h
deleted file mode 100644
index 1520ac27e94fb8576392a4fd12faee71c5d95c6c..0000000000000000000000000000000000000000
--- a/lib/Target/Nios2/Nios2TargetStreamer.h
+++ /dev/null
@@ -1,30 +0,0 @@
-//===-- Nios2TargetStreamer.h - Nios2 Target Streamer ----------*- C++ -*--===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_LIB_TARGET_NIOS2_NIOS2TARGETSTREAMER_H
-#define LLVM_LIB_TARGET_NIOS2_NIOS2TARGETSTREAMER_H
-
-#include "llvm/MC/MCRegisterInfo.h"
-#include "llvm/MC/MCStreamer.h"
-
-namespace llvm {
-
-class Nios2TargetStreamer : public MCTargetStreamer {
-public:
-  Nios2TargetStreamer(MCStreamer &S);
-};
-
-// This part is for ascii assembly output
-class Nios2TargetAsmStreamer : public Nios2TargetStreamer {
-public:
-  Nios2TargetAsmStreamer(MCStreamer &S, formatted_raw_ostream &OS);
-};
-
-} // namespace llvm
-#endif
diff --git a/lib/Target/Nios2/TargetInfo/CMakeLists.txt b/lib/Target/Nios2/TargetInfo/CMakeLists.txt
deleted file mode 100644
index 394d2c2680b76a0403ac6741774fe1f49fc04fe6..0000000000000000000000000000000000000000
--- a/lib/Target/Nios2/TargetInfo/CMakeLists.txt
+++ /dev/null
@@ -1 +0,0 @@
-add_llvm_library(LLVMNios2Info Nios2TargetInfo.cpp)
diff --git a/lib/Target/Nios2/TargetInfo/LLVMBuild.txt b/lib/Target/Nios2/TargetInfo/LLVMBuild.txt
deleted file mode 100644
index 558f7501ea6b6e7195152b63d958259c8e8957a0..0000000000000000000000000000000000000000
--- a/lib/Target/Nios2/TargetInfo/LLVMBuild.txt
+++ /dev/null
@@ -1,23 +0,0 @@
-;===- ./lib/Target/Nios2/TargetInfo/LLVMBuild.txt --------------*- Conf -*--===;
-;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
-;
-;===------------------------------------------------------------------------===;
-;
-; This is an LLVMBuild description file for the components in this subdirectory.
-;
-; For more information on the LLVMBuild system, please see:
-;
-;   http://llvm.org/docs/LLVMBuild.html
-;
-;===------------------------------------------------------------------------===;
-
-[component_0]
-type = Library
-name = Nios2Info
-parent = Nios2
-required_libraries = Support
-add_to_library_groups = Nios2
diff --git a/lib/Target/Nios2/TargetInfo/Nios2TargetInfo.cpp b/lib/Target/Nios2/TargetInfo/Nios2TargetInfo.cpp
deleted file mode 100644
index d808a96db772bf991fdc492d8edf94b2b4da9647..0000000000000000000000000000000000000000
--- a/lib/Target/Nios2/TargetInfo/Nios2TargetInfo.cpp
+++ /dev/null
@@ -1,24 +0,0 @@
-//===-- Nios2TargetInfo.cpp - Nios2 Target Implementation -----------------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-
-#include "Nios2.h"
-#include "llvm/Support/TargetRegistry.h"
-
-using namespace llvm;
-
-Target &llvm::getTheNios2Target() {
-  static Target TheNios2Target;
-  return TheNios2Target;
-}
-
-extern "C" void LLVMInitializeNios2TargetInfo() {
-  RegisterTarget<Triple::nios2,
-                 /*HasJIT=*/true>
-      X(getTheNios2Target(), "nios2", "Nios2", "Nios2");
-}
diff --git a/lib/Target/PowerPC/AsmParser/LLVMBuild.txt b/lib/Target/PowerPC/AsmParser/LLVMBuild.txt
index 801f27bb7bc34b17bd5076198a10493ef2914ec0..20ea0ef2ae4858af94162c8c571ae6aef18b4152 100644
--- a/lib/Target/PowerPC/AsmParser/LLVMBuild.txt
+++ b/lib/Target/PowerPC/AsmParser/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/Target/PowerPC/AsmParser/LLVMBuild.txt -------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Target/PowerPC/AsmParser/PPCAsmParser.cpp b/lib/Target/PowerPC/AsmParser/PPCAsmParser.cpp
index 56307a84f2e502bc23dfe26e257010f9114c4d54..cd589013dba840040190983b26dd9fe97beb4cd5 100644
--- a/lib/Target/PowerPC/AsmParser/PPCAsmParser.cpp
+++ b/lib/Target/PowerPC/AsmParser/PPCAsmParser.cpp
@@ -1,9 +1,8 @@
 //===-- PPCAsmParser.cpp - Parse PowerPC asm to MCInst instructions -------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -21,7 +20,6 @@
 #include "llvm/MC/MCParser/MCAsmParser.h"
 #include "llvm/MC/MCParser/MCParsedAsmOperand.h"
 #include "llvm/MC/MCParser/MCTargetAsmParser.h"
-#include "llvm/MC/MCRegisterInfo.h"
 #include "llvm/MC/MCStreamer.h"
 #include "llvm/MC/MCSubtargetInfo.h"
 #include "llvm/MC/MCSymbolELF.h"
@@ -31,169 +29,7 @@
 
 using namespace llvm;
 
-static const MCPhysReg RRegs[32] = {
-  PPC::R0,  PPC::R1,  PPC::R2,  PPC::R3,
-  PPC::R4,  PPC::R5,  PPC::R6,  PPC::R7,
-  PPC::R8,  PPC::R9,  PPC::R10, PPC::R11,
-  PPC::R12, PPC::R13, PPC::R14, PPC::R15,
-  PPC::R16, PPC::R17, PPC::R18, PPC::R19,
-  PPC::R20, PPC::R21, PPC::R22, PPC::R23,
-  PPC::R24, PPC::R25, PPC::R26, PPC::R27,
-  PPC::R28, PPC::R29, PPC::R30, PPC::R31
-};
-static const MCPhysReg RRegsNoR0[32] = {
-  PPC::ZERO,
-            PPC::R1,  PPC::R2,  PPC::R3,
-  PPC::R4,  PPC::R5,  PPC::R6,  PPC::R7,
-  PPC::R8,  PPC::R9,  PPC::R10, PPC::R11,
-  PPC::R12, PPC::R13, PPC::R14, PPC::R15,
-  PPC::R16, PPC::R17, PPC::R18, PPC::R19,
-  PPC::R20, PPC::R21, PPC::R22, PPC::R23,
-  PPC::R24, PPC::R25, PPC::R26, PPC::R27,
-  PPC::R28, PPC::R29, PPC::R30, PPC::R31
-};
-static const MCPhysReg XRegs[32] = {
-  PPC::X0,  PPC::X1,  PPC::X2,  PPC::X3,
-  PPC::X4,  PPC::X5,  PPC::X6,  PPC::X7,
-  PPC::X8,  PPC::X9,  PPC::X10, PPC::X11,
-  PPC::X12, PPC::X13, PPC::X14, PPC::X15,
-  PPC::X16, PPC::X17, PPC::X18, PPC::X19,
-  PPC::X20, PPC::X21, PPC::X22, PPC::X23,
-  PPC::X24, PPC::X25, PPC::X26, PPC::X27,
-  PPC::X28, PPC::X29, PPC::X30, PPC::X31
-};
-static const MCPhysReg XRegsNoX0[32] = {
-  PPC::ZERO8,
-            PPC::X1,  PPC::X2,  PPC::X3,
-  PPC::X4,  PPC::X5,  PPC::X6,  PPC::X7,
-  PPC::X8,  PPC::X9,  PPC::X10, PPC::X11,
-  PPC::X12, PPC::X13, PPC::X14, PPC::X15,
-  PPC::X16, PPC::X17, PPC::X18, PPC::X19,
-  PPC::X20, PPC::X21, PPC::X22, PPC::X23,
-  PPC::X24, PPC::X25, PPC::X26, PPC::X27,
-  PPC::X28, PPC::X29, PPC::X30, PPC::X31
-};
-static const MCPhysReg FRegs[32] = {
-  PPC::F0,  PPC::F1,  PPC::F2,  PPC::F3,
-  PPC::F4,  PPC::F5,  PPC::F6,  PPC::F7,
-  PPC::F8,  PPC::F9,  PPC::F10, PPC::F11,
-  PPC::F12, PPC::F13, PPC::F14, PPC::F15,
-  PPC::F16, PPC::F17, PPC::F18, PPC::F19,
-  PPC::F20, PPC::F21, PPC::F22, PPC::F23,
-  PPC::F24, PPC::F25, PPC::F26, PPC::F27,
-  PPC::F28, PPC::F29, PPC::F30, PPC::F31
-};
-static const MCPhysReg SPERegs[32] = {
-  PPC::S0,  PPC::S1,  PPC::S2,  PPC::S3,
-  PPC::S4,  PPC::S5,  PPC::S6,  PPC::S7,
-  PPC::S8,  PPC::S9,  PPC::S10, PPC::S11,
-  PPC::S12, PPC::S13, PPC::S14, PPC::S15,
-  PPC::S16, PPC::S17, PPC::S18, PPC::S19,
-  PPC::S20, PPC::S21, PPC::S22, PPC::S23,
-  PPC::S24, PPC::S25, PPC::S26, PPC::S27,
-  PPC::S28, PPC::S29, PPC::S30, PPC::S31
-};
-static const MCPhysReg VFRegs[32] = {
-  PPC::VF0,  PPC::VF1,  PPC::VF2,  PPC::VF3,
-  PPC::VF4,  PPC::VF5,  PPC::VF6,  PPC::VF7,
-  PPC::VF8,  PPC::VF9,  PPC::VF10, PPC::VF11,
-  PPC::VF12, PPC::VF13, PPC::VF14, PPC::VF15,
-  PPC::VF16, PPC::VF17, PPC::VF18, PPC::VF19,
-  PPC::VF20, PPC::VF21, PPC::VF22, PPC::VF23,
-  PPC::VF24, PPC::VF25, PPC::VF26, PPC::VF27,
-  PPC::VF28, PPC::VF29, PPC::VF30, PPC::VF31
-};
-static const MCPhysReg VRegs[32] = {
-  PPC::V0,  PPC::V1,  PPC::V2,  PPC::V3,
-  PPC::V4,  PPC::V5,  PPC::V6,  PPC::V7,
-  PPC::V8,  PPC::V9,  PPC::V10, PPC::V11,
-  PPC::V12, PPC::V13, PPC::V14, PPC::V15,
-  PPC::V16, PPC::V17, PPC::V18, PPC::V19,
-  PPC::V20, PPC::V21, PPC::V22, PPC::V23,
-  PPC::V24, PPC::V25, PPC::V26, PPC::V27,
-  PPC::V28, PPC::V29, PPC::V30, PPC::V31
-};
-static const MCPhysReg VSRegs[64] = {
-  PPC::VSL0,  PPC::VSL1,  PPC::VSL2,  PPC::VSL3,
-  PPC::VSL4,  PPC::VSL5,  PPC::VSL6,  PPC::VSL7,
-  PPC::VSL8,  PPC::VSL9,  PPC::VSL10, PPC::VSL11,
-  PPC::VSL12, PPC::VSL13, PPC::VSL14, PPC::VSL15,
-  PPC::VSL16, PPC::VSL17, PPC::VSL18, PPC::VSL19,
-  PPC::VSL20, PPC::VSL21, PPC::VSL22, PPC::VSL23,
-  PPC::VSL24, PPC::VSL25, PPC::VSL26, PPC::VSL27,
-  PPC::VSL28, PPC::VSL29, PPC::VSL30, PPC::VSL31,
-
-  PPC::V0,  PPC::V1,  PPC::V2,  PPC::V3,
-  PPC::V4,  PPC::V5,  PPC::V6,  PPC::V7,
-  PPC::V8,  PPC::V9,  PPC::V10, PPC::V11,
-  PPC::V12, PPC::V13, PPC::V14, PPC::V15,
-  PPC::V16, PPC::V17, PPC::V18, PPC::V19,
-  PPC::V20, PPC::V21, PPC::V22, PPC::V23,
-  PPC::V24, PPC::V25, PPC::V26, PPC::V27,
-  PPC::V28, PPC::V29, PPC::V30, PPC::V31
-};
-static const MCPhysReg VSFRegs[64] = {
-  PPC::F0,  PPC::F1,  PPC::F2,  PPC::F3,
-  PPC::F4,  PPC::F5,  PPC::F6,  PPC::F7,
-  PPC::F8,  PPC::F9,  PPC::F10, PPC::F11,
-  PPC::F12, PPC::F13, PPC::F14, PPC::F15,
-  PPC::F16, PPC::F17, PPC::F18, PPC::F19,
-  PPC::F20, PPC::F21, PPC::F22, PPC::F23,
-  PPC::F24, PPC::F25, PPC::F26, PPC::F27,
-  PPC::F28, PPC::F29, PPC::F30, PPC::F31,
-
-  PPC::VF0,  PPC::VF1,  PPC::VF2,  PPC::VF3,
-  PPC::VF4,  PPC::VF5,  PPC::VF6,  PPC::VF7,
-  PPC::VF8,  PPC::VF9,  PPC::VF10, PPC::VF11,
-  PPC::VF12, PPC::VF13, PPC::VF14, PPC::VF15,
-  PPC::VF16, PPC::VF17, PPC::VF18, PPC::VF19,
-  PPC::VF20, PPC::VF21, PPC::VF22, PPC::VF23,
-  PPC::VF24, PPC::VF25, PPC::VF26, PPC::VF27,
-  PPC::VF28, PPC::VF29, PPC::VF30, PPC::VF31
-};
-static const MCPhysReg VSSRegs[64] = {
-  PPC::F0,  PPC::F1,  PPC::F2,  PPC::F3,
-  PPC::F4,  PPC::F5,  PPC::F6,  PPC::F7,
-  PPC::F8,  PPC::F9,  PPC::F10, PPC::F11,
-  PPC::F12, PPC::F13, PPC::F14, PPC::F15,
-  PPC::F16, PPC::F17, PPC::F18, PPC::F19,
-  PPC::F20, PPC::F21, PPC::F22, PPC::F23,
-  PPC::F24, PPC::F25, PPC::F26, PPC::F27,
-  PPC::F28, PPC::F29, PPC::F30, PPC::F31,
-
-  PPC::VF0,  PPC::VF1,  PPC::VF2,  PPC::VF3,
-  PPC::VF4,  PPC::VF5,  PPC::VF6,  PPC::VF7,
-  PPC::VF8,  PPC::VF9,  PPC::VF10, PPC::VF11,
-  PPC::VF12, PPC::VF13, PPC::VF14, PPC::VF15,
-  PPC::VF16, PPC::VF17, PPC::VF18, PPC::VF19,
-  PPC::VF20, PPC::VF21, PPC::VF22, PPC::VF23,
-  PPC::VF24, PPC::VF25, PPC::VF26, PPC::VF27,
-  PPC::VF28, PPC::VF29, PPC::VF30, PPC::VF31
-};
-static unsigned QFRegs[32] = {
-  PPC::QF0,  PPC::QF1,  PPC::QF2,  PPC::QF3,
-  PPC::QF4,  PPC::QF5,  PPC::QF6,  PPC::QF7,
-  PPC::QF8,  PPC::QF9,  PPC::QF10, PPC::QF11,
-  PPC::QF12, PPC::QF13, PPC::QF14, PPC::QF15,
-  PPC::QF16, PPC::QF17, PPC::QF18, PPC::QF19,
-  PPC::QF20, PPC::QF21, PPC::QF22, PPC::QF23,
-  PPC::QF24, PPC::QF25, PPC::QF26, PPC::QF27,
-  PPC::QF28, PPC::QF29, PPC::QF30, PPC::QF31
-};
-static const MCPhysReg CRBITRegs[32] = {
-  PPC::CR0LT, PPC::CR0GT, PPC::CR0EQ, PPC::CR0UN,
-  PPC::CR1LT, PPC::CR1GT, PPC::CR1EQ, PPC::CR1UN,
-  PPC::CR2LT, PPC::CR2GT, PPC::CR2EQ, PPC::CR2UN,
-  PPC::CR3LT, PPC::CR3GT, PPC::CR3EQ, PPC::CR3UN,
-  PPC::CR4LT, PPC::CR4GT, PPC::CR4EQ, PPC::CR4UN,
-  PPC::CR5LT, PPC::CR5GT, PPC::CR5EQ, PPC::CR5UN,
-  PPC::CR6LT, PPC::CR6GT, PPC::CR6EQ, PPC::CR6UN,
-  PPC::CR7LT, PPC::CR7GT, PPC::CR7EQ, PPC::CR7UN
-};
-static const MCPhysReg CRRegs[8] = {
-  PPC::CR0, PPC::CR1, PPC::CR2, PPC::CR3,
-  PPC::CR4, PPC::CR5, PPC::CR6, PPC::CR7
-};
+DEFINE_PPC_REGCLASSES;
 
 // Evaluate an expression containing condition register
 // or condition register field symbols.  Returns positive
diff --git a/lib/Target/PowerPC/CMakeLists.txt b/lib/Target/PowerPC/CMakeLists.txt
index 3130d10fa5ed6eb279841576c533feb74929668b..c6909202d1b234b1426050513b7adc66f3a83634 100644
--- a/lib/Target/PowerPC/CMakeLists.txt
+++ b/lib/Target/PowerPC/CMakeLists.txt
@@ -19,6 +19,7 @@ add_llvm_target(PowerPCCodeGen
   PPCAsmPrinter.cpp
   PPCBranchSelector.cpp
   PPCBranchCoalescing.cpp
+  PPCCallingConv.cpp
   PPCCCState.cpp
   PPCCTRLoops.cpp
   PPCHazardRecognizers.cpp
diff --git a/lib/Target/PowerPC/Disassembler/LLVMBuild.txt b/lib/Target/PowerPC/Disassembler/LLVMBuild.txt
index ea3e7eaf839ddd518fcc25f61fd4985bb75223cc..5d9de6d16c206f833d140ca2b3d5c566aabffd59 100644
--- a/lib/Target/PowerPC/Disassembler/LLVMBuild.txt
+++ b/lib/Target/PowerPC/Disassembler/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===-- ./lib/Target/PowerPC/Disassembler/LLVMBuild.txt ---------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Target/PowerPC/Disassembler/PPCDisassembler.cpp b/lib/Target/PowerPC/Disassembler/PPCDisassembler.cpp
index af0fbae5e291f1248be362b47a1103c8258f6537..9fb37c58e85f1236d219738a1d56207fd3a1040e 100644
--- a/lib/Target/PowerPC/Disassembler/PPCDisassembler.cpp
+++ b/lib/Target/PowerPC/Disassembler/PPCDisassembler.cpp
@@ -1,9 +1,8 @@
 //===------ PPCDisassembler.cpp - Disassembler for PowerPC ------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -17,6 +16,8 @@
 
 using namespace llvm;
 
+DEFINE_PPC_REGCLASSES;
+
 #define DEBUG_TYPE "ppc-disassembler"
 
 typedef MCDisassembler::DecodeStatus DecodeStatus;
@@ -59,187 +60,20 @@ extern "C" void LLVMInitializePowerPCDisassembler() {
                                          createPPCLEDisassembler);
 }
 
+static DecodeStatus DecodePCRel24BranchTarget(MCInst &Inst, unsigned Imm,
+                                              uint64_t Addr,
+                                              const void *Decoder) {
+  int32_t Offset = SignExtend32<24>(Imm);
+  Inst.addOperand(MCOperand::createImm(Offset));
+  return MCDisassembler::Success;
+}
+
 // FIXME: These can be generated by TableGen from the existing register
 // encoding values!
 
-static const unsigned CRRegs[] = {
-  PPC::CR0, PPC::CR1, PPC::CR2, PPC::CR3,
-  PPC::CR4, PPC::CR5, PPC::CR6, PPC::CR7
-};
-
-static const unsigned CRBITRegs[] = {
-  PPC::CR0LT, PPC::CR0GT, PPC::CR0EQ, PPC::CR0UN,
-  PPC::CR1LT, PPC::CR1GT, PPC::CR1EQ, PPC::CR1UN,
-  PPC::CR2LT, PPC::CR2GT, PPC::CR2EQ, PPC::CR2UN,
-  PPC::CR3LT, PPC::CR3GT, PPC::CR3EQ, PPC::CR3UN,
-  PPC::CR4LT, PPC::CR4GT, PPC::CR4EQ, PPC::CR4UN,
-  PPC::CR5LT, PPC::CR5GT, PPC::CR5EQ, PPC::CR5UN,
-  PPC::CR6LT, PPC::CR6GT, PPC::CR6EQ, PPC::CR6UN,
-  PPC::CR7LT, PPC::CR7GT, PPC::CR7EQ, PPC::CR7UN
-};
-
-static const unsigned FRegs[] = {
-  PPC::F0, PPC::F1, PPC::F2, PPC::F3,
-  PPC::F4, PPC::F5, PPC::F6, PPC::F7,
-  PPC::F8, PPC::F9, PPC::F10, PPC::F11,
-  PPC::F12, PPC::F13, PPC::F14, PPC::F15,
-  PPC::F16, PPC::F17, PPC::F18, PPC::F19,
-  PPC::F20, PPC::F21, PPC::F22, PPC::F23,
-  PPC::F24, PPC::F25, PPC::F26, PPC::F27,
-  PPC::F28, PPC::F29, PPC::F30, PPC::F31
-};
-
-static const unsigned VFRegs[] = {
-  PPC::VF0, PPC::VF1, PPC::VF2, PPC::VF3,
-  PPC::VF4, PPC::VF5, PPC::VF6, PPC::VF7,
-  PPC::VF8, PPC::VF9, PPC::VF10, PPC::VF11,
-  PPC::VF12, PPC::VF13, PPC::VF14, PPC::VF15,
-  PPC::VF16, PPC::VF17, PPC::VF18, PPC::VF19,
-  PPC::VF20, PPC::VF21, PPC::VF22, PPC::VF23,
-  PPC::VF24, PPC::VF25, PPC::VF26, PPC::VF27,
-  PPC::VF28, PPC::VF29, PPC::VF30, PPC::VF31
-};
-
-static const unsigned VRegs[] = {
-  PPC::V0, PPC::V1, PPC::V2, PPC::V3,
-  PPC::V4, PPC::V5, PPC::V6, PPC::V7,
-  PPC::V8, PPC::V9, PPC::V10, PPC::V11,
-  PPC::V12, PPC::V13, PPC::V14, PPC::V15,
-  PPC::V16, PPC::V17, PPC::V18, PPC::V19,
-  PPC::V20, PPC::V21, PPC::V22, PPC::V23,
-  PPC::V24, PPC::V25, PPC::V26, PPC::V27,
-  PPC::V28, PPC::V29, PPC::V30, PPC::V31
-};
-
-static const unsigned VSRegs[] = {
-  PPC::VSL0, PPC::VSL1, PPC::VSL2, PPC::VSL3,
-  PPC::VSL4, PPC::VSL5, PPC::VSL6, PPC::VSL7,
-  PPC::VSL8, PPC::VSL9, PPC::VSL10, PPC::VSL11,
-  PPC::VSL12, PPC::VSL13, PPC::VSL14, PPC::VSL15,
-  PPC::VSL16, PPC::VSL17, PPC::VSL18, PPC::VSL19,
-  PPC::VSL20, PPC::VSL21, PPC::VSL22, PPC::VSL23,
-  PPC::VSL24, PPC::VSL25, PPC::VSL26, PPC::VSL27,
-  PPC::VSL28, PPC::VSL29, PPC::VSL30, PPC::VSL31,
-
-  PPC::V0, PPC::V1, PPC::V2, PPC::V3,
-  PPC::V4, PPC::V5, PPC::V6, PPC::V7,
-  PPC::V8, PPC::V9, PPC::V10, PPC::V11,
-  PPC::V12, PPC::V13, PPC::V14, PPC::V15,
-  PPC::V16, PPC::V17, PPC::V18, PPC::V19,
-  PPC::V20, PPC::V21, PPC::V22, PPC::V23,
-  PPC::V24, PPC::V25, PPC::V26, PPC::V27,
-  PPC::V28, PPC::V29, PPC::V30, PPC::V31
-};
-
-static const unsigned VSFRegs[] = {
-  PPC::F0, PPC::F1, PPC::F2, PPC::F3,
-  PPC::F4, PPC::F5, PPC::F6, PPC::F7,
-  PPC::F8, PPC::F9, PPC::F10, PPC::F11,
-  PPC::F12, PPC::F13, PPC::F14, PPC::F15,
-  PPC::F16, PPC::F17, PPC::F18, PPC::F19,
-  PPC::F20, PPC::F21, PPC::F22, PPC::F23,
-  PPC::F24, PPC::F25, PPC::F26, PPC::F27,
-  PPC::F28, PPC::F29, PPC::F30, PPC::F31,
-
-  PPC::VF0, PPC::VF1, PPC::VF2, PPC::VF3,
-  PPC::VF4, PPC::VF5, PPC::VF6, PPC::VF7,
-  PPC::VF8, PPC::VF9, PPC::VF10, PPC::VF11,
-  PPC::VF12, PPC::VF13, PPC::VF14, PPC::VF15,
-  PPC::VF16, PPC::VF17, PPC::VF18, PPC::VF19,
-  PPC::VF20, PPC::VF21, PPC::VF22, PPC::VF23,
-  PPC::VF24, PPC::VF25, PPC::VF26, PPC::VF27,
-  PPC::VF28, PPC::VF29, PPC::VF30, PPC::VF31
-};
-
-static const unsigned VSSRegs[] = {
-  PPC::F0, PPC::F1, PPC::F2, PPC::F3,
-  PPC::F4, PPC::F5, PPC::F6, PPC::F7,
-  PPC::F8, PPC::F9, PPC::F10, PPC::F11,
-  PPC::F12, PPC::F13, PPC::F14, PPC::F15,
-  PPC::F16, PPC::F17, PPC::F18, PPC::F19,
-  PPC::F20, PPC::F21, PPC::F22, PPC::F23,
-  PPC::F24, PPC::F25, PPC::F26, PPC::F27,
-  PPC::F28, PPC::F29, PPC::F30, PPC::F31,
-
-  PPC::VF0, PPC::VF1, PPC::VF2, PPC::VF3,
-  PPC::VF4, PPC::VF5, PPC::VF6, PPC::VF7,
-  PPC::VF8, PPC::VF9, PPC::VF10, PPC::VF11,
-  PPC::VF12, PPC::VF13, PPC::VF14, PPC::VF15,
-  PPC::VF16, PPC::VF17, PPC::VF18, PPC::VF19,
-  PPC::VF20, PPC::VF21, PPC::VF22, PPC::VF23,
-  PPC::VF24, PPC::VF25, PPC::VF26, PPC::VF27,
-  PPC::VF28, PPC::VF29, PPC::VF30, PPC::VF31
-};
-
-static const unsigned GPRegs[] = {
-  PPC::R0, PPC::R1, PPC::R2, PPC::R3,
-  PPC::R4, PPC::R5, PPC::R6, PPC::R7,
-  PPC::R8, PPC::R9, PPC::R10, PPC::R11,
-  PPC::R12, PPC::R13, PPC::R14, PPC::R15,
-  PPC::R16, PPC::R17, PPC::R18, PPC::R19,
-  PPC::R20, PPC::R21, PPC::R22, PPC::R23,
-  PPC::R24, PPC::R25, PPC::R26, PPC::R27,
-  PPC::R28, PPC::R29, PPC::R30, PPC::R31
-};
-
-static const unsigned GP0Regs[] = {
-  PPC::ZERO, PPC::R1, PPC::R2, PPC::R3,
-  PPC::R4, PPC::R5, PPC::R6, PPC::R7,
-  PPC::R8, PPC::R9, PPC::R10, PPC::R11,
-  PPC::R12, PPC::R13, PPC::R14, PPC::R15,
-  PPC::R16, PPC::R17, PPC::R18, PPC::R19,
-  PPC::R20, PPC::R21, PPC::R22, PPC::R23,
-  PPC::R24, PPC::R25, PPC::R26, PPC::R27,
-  PPC::R28, PPC::R29, PPC::R30, PPC::R31
-};
-
-static const unsigned G8Regs[] = {
-  PPC::X0, PPC::X1, PPC::X2, PPC::X3,
-  PPC::X4, PPC::X5, PPC::X6, PPC::X7,
-  PPC::X8, PPC::X9, PPC::X10, PPC::X11,
-  PPC::X12, PPC::X13, PPC::X14, PPC::X15,
-  PPC::X16, PPC::X17, PPC::X18, PPC::X19,
-  PPC::X20, PPC::X21, PPC::X22, PPC::X23,
-  PPC::X24, PPC::X25, PPC::X26, PPC::X27,
-  PPC::X28, PPC::X29, PPC::X30, PPC::X31
-};
-
-static const unsigned G80Regs[] = {
-  PPC::ZERO8, PPC::X1, PPC::X2, PPC::X3,
-  PPC::X4, PPC::X5, PPC::X6, PPC::X7,
-  PPC::X8, PPC::X9, PPC::X10, PPC::X11,
-  PPC::X12, PPC::X13, PPC::X14, PPC::X15,
-  PPC::X16, PPC::X17, PPC::X18, PPC::X19,
-  PPC::X20, PPC::X21, PPC::X22, PPC::X23,
-  PPC::X24, PPC::X25, PPC::X26, PPC::X27,
-  PPC::X28, PPC::X29, PPC::X30, PPC::X31
-};
-
-static const unsigned QFRegs[] = {
-  PPC::QF0, PPC::QF1, PPC::QF2, PPC::QF3,
-  PPC::QF4, PPC::QF5, PPC::QF6, PPC::QF7,
-  PPC::QF8, PPC::QF9, PPC::QF10, PPC::QF11,
-  PPC::QF12, PPC::QF13, PPC::QF14, PPC::QF15,
-  PPC::QF16, PPC::QF17, PPC::QF18, PPC::QF19,
-  PPC::QF20, PPC::QF21, PPC::QF22, PPC::QF23,
-  PPC::QF24, PPC::QF25, PPC::QF26, PPC::QF27,
-  PPC::QF28, PPC::QF29, PPC::QF30, PPC::QF31
-};
-
-static const unsigned SPERegs[] = {
-  PPC::S0, PPC::S1, PPC::S2, PPC::S3,
-  PPC::S4, PPC::S5, PPC::S6, PPC::S7,
-  PPC::S8, PPC::S9, PPC::S10, PPC::S11,
-  PPC::S12, PPC::S13, PPC::S14, PPC::S15,
-  PPC::S16, PPC::S17, PPC::S18, PPC::S19,
-  PPC::S20, PPC::S21, PPC::S22, PPC::S23,
-  PPC::S24, PPC::S25, PPC::S26, PPC::S27,
-  PPC::S28, PPC::S29, PPC::S30, PPC::S31
-};
-
 template <std::size_t N>
 static DecodeStatus decodeRegisterClass(MCInst &Inst, uint64_t RegNo,
-                                        const unsigned (&Regs)[N]) {
+                                        const MCPhysReg (&Regs)[N]) {
   assert(RegNo < N && "Invalid register number");
   Inst.addOperand(MCOperand::createReg(Regs[RegNo]));
   return MCDisassembler::Success;
@@ -308,25 +142,25 @@ static DecodeStatus DecodeVSSRCRegisterClass(MCInst &Inst, uint64_t RegNo,
 static DecodeStatus DecodeGPRCRegisterClass(MCInst &Inst, uint64_t RegNo,
                                             uint64_t Address,
                                             const void *Decoder) {
-  return decodeRegisterClass(Inst, RegNo, GPRegs);
+  return decodeRegisterClass(Inst, RegNo, RRegs);
 }
 
 static DecodeStatus DecodeGPRC_NOR0RegisterClass(MCInst &Inst, uint64_t RegNo,
                                             uint64_t Address,
                                             const void *Decoder) {
-  return decodeRegisterClass(Inst, RegNo, GP0Regs);
+  return decodeRegisterClass(Inst, RegNo, RRegsNoR0);
 }
 
 static DecodeStatus DecodeG8RCRegisterClass(MCInst &Inst, uint64_t RegNo,
                                             uint64_t Address,
                                             const void *Decoder) {
-  return decodeRegisterClass(Inst, RegNo, G8Regs);
+  return decodeRegisterClass(Inst, RegNo, XRegs);
 }
 
 static DecodeStatus DecodeG8RC_NOX0RegisterClass(MCInst &Inst, uint64_t RegNo,
                                             uint64_t Address,
                                             const void *Decoder) {
-  return decodeRegisterClass(Inst, RegNo, G80Regs);
+  return decodeRegisterClass(Inst, RegNo, XRegsNoX0);
 }
 
 #define DecodePointerLikeRegClass0 DecodeGPRCRegisterClass
@@ -341,7 +175,7 @@ static DecodeStatus DecodeQFRCRegisterClass(MCInst &Inst, uint64_t RegNo,
 static DecodeStatus DecodeSPE4RCRegisterClass(MCInst &Inst, uint64_t RegNo,
                                             uint64_t Address,
                                             const void *Decoder) {
-  return decodeRegisterClass(Inst, RegNo, GPRegs);
+  return decodeRegisterClass(Inst, RegNo, RRegs);
 }
 
 static DecodeStatus DecodeSPERCRegisterClass(MCInst &Inst, uint64_t RegNo,
@@ -388,19 +222,19 @@ static DecodeStatus decodeMemRIOperands(MCInst &Inst, uint64_t Imm,
   case PPC::LFSU:
   case PPC::LFDU:
     // Add the tied output operand.
-    Inst.addOperand(MCOperand::createReg(GP0Regs[Base]));
+    Inst.addOperand(MCOperand::createReg(RRegsNoR0[Base]));
     break;
   case PPC::STBU:
   case PPC::STHU:
   case PPC::STWU:
   case PPC::STFSU:
   case PPC::STFDU:
-    Inst.insert(Inst.begin(), MCOperand::createReg(GP0Regs[Base]));
+    Inst.insert(Inst.begin(), MCOperand::createReg(RRegsNoR0[Base]));
     break;
   }
 
   Inst.addOperand(MCOperand::createImm(SignExtend64<16>(Disp)));
-  Inst.addOperand(MCOperand::createReg(GP0Regs[Base]));
+  Inst.addOperand(MCOperand::createReg(RRegsNoR0[Base]));
   return MCDisassembler::Success;
 }
 
@@ -416,12 +250,12 @@ static DecodeStatus decodeMemRIXOperands(MCInst &Inst, uint64_t Imm,
 
   if (Inst.getOpcode() == PPC::LDU)
     // Add the tied output operand.
-    Inst.addOperand(MCOperand::createReg(GP0Regs[Base]));
+    Inst.addOperand(MCOperand::createReg(RRegsNoR0[Base]));
   else if (Inst.getOpcode() == PPC::STDU)
-    Inst.insert(Inst.begin(), MCOperand::createReg(GP0Regs[Base]));
+    Inst.insert(Inst.begin(), MCOperand::createReg(RRegsNoR0[Base]));
 
   Inst.addOperand(MCOperand::createImm(SignExtend64<16>(Disp << 2)));
-  Inst.addOperand(MCOperand::createReg(GP0Regs[Base]));
+  Inst.addOperand(MCOperand::createReg(RRegsNoR0[Base]));
   return MCDisassembler::Success;
 }
 
@@ -436,7 +270,7 @@ static DecodeStatus decodeMemRIX16Operands(MCInst &Inst, uint64_t Imm,
   assert(Base < 32 && "Invalid base register");
 
   Inst.addOperand(MCOperand::createImm(SignExtend64<16>(Disp << 4)));
-  Inst.addOperand(MCOperand::createReg(GP0Regs[Base]));
+  Inst.addOperand(MCOperand::createReg(RRegsNoR0[Base]));
   return MCDisassembler::Success;
 }
 
@@ -451,7 +285,7 @@ static DecodeStatus decodeSPE8Operands(MCInst &Inst, uint64_t Imm,
   assert(Base < 32 && "Invalid base register");
 
   Inst.addOperand(MCOperand::createImm(Disp << 3));
-  Inst.addOperand(MCOperand::createReg(GP0Regs[Base]));
+  Inst.addOperand(MCOperand::createReg(RRegsNoR0[Base]));
   return MCDisassembler::Success;
 }
 
@@ -466,7 +300,7 @@ static DecodeStatus decodeSPE4Operands(MCInst &Inst, uint64_t Imm,
   assert(Base < 32 && "Invalid base register");
 
   Inst.addOperand(MCOperand::createImm(Disp << 2));
-  Inst.addOperand(MCOperand::createReg(GP0Regs[Base]));
+  Inst.addOperand(MCOperand::createReg(RRegsNoR0[Base]));
   return MCDisassembler::Success;
 }
 
@@ -481,7 +315,7 @@ static DecodeStatus decodeSPE2Operands(MCInst &Inst, uint64_t Imm,
   assert(Base < 32 && "Invalid base register");
 
   Inst.addOperand(MCOperand::createImm(Disp << 1));
-  Inst.addOperand(MCOperand::createReg(GP0Regs[Base]));
+  Inst.addOperand(MCOperand::createReg(RRegsNoR0[Base]));
   return MCDisassembler::Success;
 }
 
diff --git a/lib/Target/PowerPC/InstPrinter/LLVMBuild.txt b/lib/Target/PowerPC/InstPrinter/LLVMBuild.txt
index 7c691deafccfcfcff94efab9bc14449de436d2d0..83f04cb07a812bc7d5ac99ceb59883fa3082c63a 100644
--- a/lib/Target/PowerPC/InstPrinter/LLVMBuild.txt
+++ b/lib/Target/PowerPC/InstPrinter/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/Target/PowerPC/InstPrinter/LLVMBuild.txt -----------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Target/PowerPC/InstPrinter/PPCInstPrinter.cpp b/lib/Target/PowerPC/InstPrinter/PPCInstPrinter.cpp
index fc29e4effbb1281c3bb3750335889089dd86deaf..77206494990bbbc17c45c8b7e181136a3ddc0098 100644
--- a/lib/Target/PowerPC/InstPrinter/PPCInstPrinter.cpp
+++ b/lib/Target/PowerPC/InstPrinter/PPCInstPrinter.cpp
@@ -1,9 +1,8 @@
 //===-- PPCInstPrinter.cpp - Convert PPC MCInst to assembly syntax --------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -382,8 +381,11 @@ void PPCInstPrinter::printBranchOperand(const MCInst *MI, unsigned OpNo,
 
   // Branches can take an immediate operand.  This is used by the branch
   // selection pass to print .+8, an eight byte displacement from the PC.
-  O << ".+";
-  printAbsBranchOperand(MI, OpNo, O);
+  O << ".";
+  int32_t Imm = SignExtend32<32>((unsigned)MI->getOperand(OpNo).getImm() << 2);
+  if (Imm >= 0)
+    O << "+";
+  O << Imm;
 }
 
 void PPCInstPrinter::printAbsBranchOperand(const MCInst *MI, unsigned OpNo,
diff --git a/lib/Target/PowerPC/InstPrinter/PPCInstPrinter.h b/lib/Target/PowerPC/InstPrinter/PPCInstPrinter.h
index 351ccefa2da2efafc5149cca897eb9359a9df84a..71853373a48a693094710cd4c85bd0e2515ecb94 100644
--- a/lib/Target/PowerPC/InstPrinter/PPCInstPrinter.h
+++ b/lib/Target/PowerPC/InstPrinter/PPCInstPrinter.h
@@ -1,9 +1,8 @@
 //===- PPCInstPrinter.h - Convert PPC MCInst to assembly syntax -*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/PowerPC/LLVMBuild.txt b/lib/Target/PowerPC/LLVMBuild.txt
index fd5fa560912f134f979336c24674988b1ba409b1..164b0201a63a4ffd69f22e3e9c547a1298fe563d 100644
--- a/lib/Target/PowerPC/LLVMBuild.txt
+++ b/lib/Target/PowerPC/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/Target/PowerPC/LLVMBuild.txt -----------------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Target/PowerPC/MCTargetDesc/LLVMBuild.txt b/lib/Target/PowerPC/MCTargetDesc/LLVMBuild.txt
index d3a567d1581d9d2f74a92e41c7293abe9cd84135..ef5ff39b11ab34d8baef33f93c51893d36d1c83a 100644
--- a/lib/Target/PowerPC/MCTargetDesc/LLVMBuild.txt
+++ b/lib/Target/PowerPC/MCTargetDesc/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/Target/PowerPC/MCTargetDesc/LLVMBuild.txt ----------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Target/PowerPC/MCTargetDesc/PPCAsmBackend.cpp b/lib/Target/PowerPC/MCTargetDesc/PPCAsmBackend.cpp
index a405dd70c307a1e83698c08856b7bf66cd54c81d..f10faacfb95ac7bddf28c4ff07bfd63884d0a101 100644
--- a/lib/Target/PowerPC/MCTargetDesc/PPCAsmBackend.cpp
+++ b/lib/Target/PowerPC/MCTargetDesc/PPCAsmBackend.cpp
@@ -1,9 +1,8 @@
 //===-- PPCAsmBackend.cpp - PPC Assembler Backend -------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/PowerPC/MCTargetDesc/PPCELFObjectWriter.cpp b/lib/Target/PowerPC/MCTargetDesc/PPCELFObjectWriter.cpp
index a3caf9a7a5eed3b5e07862b797049b5112b3b947..aa7b05b0198598647b837be7ecdf9746661bc136 100644
--- a/lib/Target/PowerPC/MCTargetDesc/PPCELFObjectWriter.cpp
+++ b/lib/Target/PowerPC/MCTargetDesc/PPCELFObjectWriter.cpp
@@ -1,9 +1,8 @@
 //===-- PPCELFObjectWriter.cpp - PPC ELF Writer ---------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/PowerPC/MCTargetDesc/PPCFixupKinds.h b/lib/Target/PowerPC/MCTargetDesc/PPCFixupKinds.h
index dce443997ea5971de718adfb1c19254fd84c85e7..845489788c86f60132a2a331e2ec92e411062b55 100644
--- a/lib/Target/PowerPC/MCTargetDesc/PPCFixupKinds.h
+++ b/lib/Target/PowerPC/MCTargetDesc/PPCFixupKinds.h
@@ -1,9 +1,8 @@
 //===-- PPCFixupKinds.h - PPC Specific Fixup Entries ------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/PowerPC/MCTargetDesc/PPCMCAsmInfo.cpp b/lib/Target/PowerPC/MCTargetDesc/PPCMCAsmInfo.cpp
index fb7bf23509c72bf3f9cd98b36808d59f0f4cf1f5..b9e35124e794f34f5f6c8be681f6c6febda44414 100644
--- a/lib/Target/PowerPC/MCTargetDesc/PPCMCAsmInfo.cpp
+++ b/lib/Target/PowerPC/MCTargetDesc/PPCMCAsmInfo.cpp
@@ -1,9 +1,8 @@
 //===-- PPCMCAsmInfo.cpp - PPC asm properties -----------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/PowerPC/MCTargetDesc/PPCMCAsmInfo.h b/lib/Target/PowerPC/MCTargetDesc/PPCMCAsmInfo.h
index e252ac944d40be1536257a6bf7506fee4a854fc8..6a10b6eef44b58ab633b7738921eed0e9b901623 100644
--- a/lib/Target/PowerPC/MCTargetDesc/PPCMCAsmInfo.h
+++ b/lib/Target/PowerPC/MCTargetDesc/PPCMCAsmInfo.h
@@ -1,9 +1,8 @@
 //===-- PPCMCAsmInfo.h - PPC asm properties --------------------*- C++ -*--===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/PowerPC/MCTargetDesc/PPCMCCodeEmitter.cpp b/lib/Target/PowerPC/MCTargetDesc/PPCMCCodeEmitter.cpp
index 8c15ade6f9c4fd33405c92465475bc0937d74e9c..b7d465f48d764d3124362de420d45967dd5d41db 100644
--- a/lib/Target/PowerPC/MCTargetDesc/PPCMCCodeEmitter.cpp
+++ b/lib/Target/PowerPC/MCTargetDesc/PPCMCCodeEmitter.cpp
@@ -1,9 +1,8 @@
 //===-- PPCMCCodeEmitter.cpp - Convert PPC code to machine code -----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/PowerPC/MCTargetDesc/PPCMCCodeEmitter.h b/lib/Target/PowerPC/MCTargetDesc/PPCMCCodeEmitter.h
index a4bcff4b9450c31e6e9882bf6baa1fdcee3b1037..64122a7e8d5cc9abc87d6ffc0b1f1edcff07eeeb 100644
--- a/lib/Target/PowerPC/MCTargetDesc/PPCMCCodeEmitter.h
+++ b/lib/Target/PowerPC/MCTargetDesc/PPCMCCodeEmitter.h
@@ -1,9 +1,8 @@
 //===-- PPCMCCodeEmitter.h - Convert PPC code to machine code -------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/PowerPC/MCTargetDesc/PPCMCExpr.cpp b/lib/Target/PowerPC/MCTargetDesc/PPCMCExpr.cpp
index 32e6a0bdd65fe9b9e3c0bb29f5c4c4be05881b5c..d467f5c4a43921a1eb7ad5e60c8e4f6d01d7f43b 100644
--- a/lib/Target/PowerPC/MCTargetDesc/PPCMCExpr.cpp
+++ b/lib/Target/PowerPC/MCTargetDesc/PPCMCExpr.cpp
@@ -1,9 +1,8 @@
 //===-- PPCMCExpr.cpp - PPC specific MC expression classes ----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/PowerPC/MCTargetDesc/PPCMCExpr.h b/lib/Target/PowerPC/MCTargetDesc/PPCMCExpr.h
index 8bb4791d13dd103a60624b49158c0c8348e36352..449e2c34f74df0e2524654c7dde2e08c93ab0a44 100644
--- a/lib/Target/PowerPC/MCTargetDesc/PPCMCExpr.h
+++ b/lib/Target/PowerPC/MCTargetDesc/PPCMCExpr.h
@@ -1,9 +1,8 @@
 //===-- PPCMCExpr.h - PPC specific MC expression classes --------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/PowerPC/MCTargetDesc/PPCMCTargetDesc.cpp b/lib/Target/PowerPC/MCTargetDesc/PPCMCTargetDesc.cpp
index a1e4e07b25af4b527da7004d660c143209acc664..8fcf2fab364a235abc346434e55cbb01e0604e27 100644
--- a/lib/Target/PowerPC/MCTargetDesc/PPCMCTargetDesc.cpp
+++ b/lib/Target/PowerPC/MCTargetDesc/PPCMCTargetDesc.cpp
@@ -1,9 +1,8 @@
 //===-- PPCMCTargetDesc.cpp - PowerPC Target Descriptions -----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/PowerPC/MCTargetDesc/PPCMCTargetDesc.h b/lib/Target/PowerPC/MCTargetDesc/PPCMCTargetDesc.h
index 316fd2ccf35873c6cef9e32c8e2ac727c2030ec5..d7c52b0b1d2ac80aa081e8fc1d97084188c63bda 100644
--- a/lib/Target/PowerPC/MCTargetDesc/PPCMCTargetDesc.h
+++ b/lib/Target/PowerPC/MCTargetDesc/PPCMCTargetDesc.h
@@ -1,9 +1,8 @@
 //===-- PPCMCTargetDesc.h - PowerPC Target Descriptions ---------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -17,6 +16,7 @@
 // GCC #defines PPC on Linux but we use it as our namespace name
 #undef PPC
 
+#include "llvm/MC/MCRegisterInfo.h"
 #include "llvm/Support/MathExtras.h"
 #include <cstdint>
 #include <memory>
@@ -104,4 +104,63 @@ static inline bool isRunOfOnes(unsigned Val, unsigned &MB, unsigned &ME) {
 #define GET_SUBTARGETINFO_ENUM
 #include "PPCGenSubtargetInfo.inc"
 
+#define PPC_REGS0_31(X)                                                        \
+  {                                                                            \
+    X##0, X##1, X##2, X##3, X##4, X##5, X##6, X##7, X##8, X##9, X##10, X##11,  \
+        X##12, X##13, X##14, X##15, X##16, X##17, X##18, X##19, X##20, X##21,  \
+        X##22, X##23, X##24, X##25, X##26, X##27, X##28, X##29, X##30, X##31   \
+  }
+
+#define PPC_REGS_NO0_31(Z, X)                                                  \
+  {                                                                            \
+    Z, X##1, X##2, X##3, X##4, X##5, X##6, X##7, X##8, X##9, X##10, X##11,     \
+        X##12, X##13, X##14, X##15, X##16, X##17, X##18, X##19, X##20, X##21,  \
+        X##22, X##23, X##24, X##25, X##26, X##27, X##28, X##29, X##30, X##31   \
+  }
+
+#define PPC_REGS_LO_HI(LO, HI)                                                 \
+  {                                                                            \
+    LO##0, LO##1, LO##2, LO##3, LO##4, LO##5, LO##6, LO##7, LO##8, LO##9,      \
+        LO##10, LO##11, LO##12, LO##13, LO##14, LO##15, LO##16, LO##17,        \
+        LO##18, LO##19, LO##20, LO##21, LO##22, LO##23, LO##24, LO##25,        \
+        LO##26, LO##27, LO##28, LO##29, LO##30, LO##31, HI##0, HI##1, HI##2,   \
+        HI##3, HI##4, HI##5, HI##6, HI##7, HI##8, HI##9, HI##10, HI##11,       \
+        HI##12, HI##13, HI##14, HI##15, HI##16, HI##17, HI##18, HI##19,        \
+        HI##20, HI##21, HI##22, HI##23, HI##24, HI##25, HI##26, HI##27,        \
+        HI##28, HI##29, HI##30, HI##31                                         \
+  }
+
+using llvm::MCPhysReg;
+
+#define DEFINE_PPC_REGCLASSES \
+  static const MCPhysReg RRegs[32] = PPC_REGS0_31(PPC::R); \
+  static const MCPhysReg XRegs[32] = PPC_REGS0_31(PPC::X); \
+  static const MCPhysReg FRegs[32] = PPC_REGS0_31(PPC::F); \
+  static const MCPhysReg SPERegs[32] = PPC_REGS0_31(PPC::S); \
+  static const MCPhysReg VFRegs[32] = PPC_REGS0_31(PPC::VF); \
+  static const MCPhysReg VRegs[32] = PPC_REGS0_31(PPC::V); \
+  static const MCPhysReg QFRegs[32] = PPC_REGS0_31(PPC::QF); \
+  static const MCPhysReg RRegsNoR0[32] = \
+    PPC_REGS_NO0_31(PPC::ZERO, PPC::R); \
+  static const MCPhysReg XRegsNoX0[32] = \
+    PPC_REGS_NO0_31(PPC::ZERO8, PPC::X); \
+  static const MCPhysReg VSRegs[64] = \
+    PPC_REGS_LO_HI(PPC::VSL, PPC::V); \
+  static const MCPhysReg VSFRegs[64] = \
+    PPC_REGS_LO_HI(PPC::F, PPC::VF); \
+  static const MCPhysReg VSSRegs[64] = \
+    PPC_REGS_LO_HI(PPC::F, PPC::VF); \
+  static const MCPhysReg CRBITRegs[32] = { \
+    PPC::CR0LT, PPC::CR0GT, PPC::CR0EQ, PPC::CR0UN, \
+    PPC::CR1LT, PPC::CR1GT, PPC::CR1EQ, PPC::CR1UN, \
+    PPC::CR2LT, PPC::CR2GT, PPC::CR2EQ, PPC::CR2UN, \
+    PPC::CR3LT, PPC::CR3GT, PPC::CR3EQ, PPC::CR3UN, \
+    PPC::CR4LT, PPC::CR4GT, PPC::CR4EQ, PPC::CR4UN, \
+    PPC::CR5LT, PPC::CR5GT, PPC::CR5EQ, PPC::CR5UN, \
+    PPC::CR6LT, PPC::CR6GT, PPC::CR6EQ, PPC::CR6UN, \
+    PPC::CR7LT, PPC::CR7GT, PPC::CR7EQ, PPC::CR7UN}; \
+  static const MCPhysReg CRRegs[8] = { \
+    PPC::CR0, PPC::CR1, PPC::CR2, PPC::CR3, \
+    PPC::CR4, PPC::CR5, PPC::CR6, PPC::CR7}
+
 #endif // LLVM_LIB_TARGET_POWERPC_MCTARGETDESC_PPCMCTARGETDESC_H
diff --git a/lib/Target/PowerPC/MCTargetDesc/PPCMachObjectWriter.cpp b/lib/Target/PowerPC/MCTargetDesc/PPCMachObjectWriter.cpp
index ff6cf584da2305b65e8ad0556346629968798977..4cf7fd15fa75c0a145f0f17b02de19a3b6488eb1 100644
--- a/lib/Target/PowerPC/MCTargetDesc/PPCMachObjectWriter.cpp
+++ b/lib/Target/PowerPC/MCTargetDesc/PPCMachObjectWriter.cpp
@@ -1,9 +1,8 @@
 //===-- PPCMachObjectWriter.cpp - PPC Mach-O Writer -----------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/PowerPC/MCTargetDesc/PPCPredicates.cpp b/lib/Target/PowerPC/MCTargetDesc/PPCPredicates.cpp
index c2987b641c04a398fa9007146b372f51b420609f..284e52c298a2d0840cf8b215ff64af5125401aab 100644
--- a/lib/Target/PowerPC/MCTargetDesc/PPCPredicates.cpp
+++ b/lib/Target/PowerPC/MCTargetDesc/PPCPredicates.cpp
@@ -1,9 +1,8 @@
 //===-- PPCPredicates.cpp - PPC Branch Predicate Information --------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/PowerPC/MCTargetDesc/PPCPredicates.h b/lib/Target/PowerPC/MCTargetDesc/PPCPredicates.h
index 481ba3f09cc74c500fe1904b17a19bee361485f0..d686a8ea2a2281abc22851fe0175f6c3aad9252b 100644
--- a/lib/Target/PowerPC/MCTargetDesc/PPCPredicates.h
+++ b/lib/Target/PowerPC/MCTargetDesc/PPCPredicates.h
@@ -1,9 +1,8 @@
 //===-- PPCPredicates.h - PPC Branch Predicate Information ------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/PowerPC/P9InstrResources.td b/lib/Target/PowerPC/P9InstrResources.td
index 17c37964c5622ecebe3e546fbb8354c3ea5a43e6..c4c1098c757bf02731c151b1a54c326e9623b383 100644
--- a/lib/Target/PowerPC/P9InstrResources.td
+++ b/lib/Target/PowerPC/P9InstrResources.td
@@ -1,9 +1,8 @@
 //===- P9InstrResources.td - P9 Instruction Resource Defs  -*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/PowerPC/PPC.h b/lib/Target/PowerPC/PPC.h
index bfc613af3dc07eeaa6ca75e70bfc31fc7aefd30f..85df85e0c49e0de9a2b6824ae47ae5e584ef491d 100644
--- a/lib/Target/PowerPC/PPC.h
+++ b/lib/Target/PowerPC/PPC.h
@@ -1,9 +1,8 @@
 //===-- PPC.h - Top-level interface for PowerPC Target ----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/PowerPC/PPC.td b/lib/Target/PowerPC/PPC.td
index 98e6e98e69744a9e04f564b4b4e7ed3b440cf443..fbca91437ec7b731916daa8f9735fd478392c0eb 100644
--- a/lib/Target/PowerPC/PPC.td
+++ b/lib/Target/PowerPC/PPC.td
@@ -1,9 +1,8 @@
 //===-- PPC.td - Describe the PowerPC Target Machine -------*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -191,6 +190,13 @@ def FeatureP9Vector  : SubtargetFeature<"power9-vector", "HasP9Vector", "true",
                                         "Enable POWER9 vector instructions",
                                         [FeatureISA3_0, FeatureP8Vector,
                                          FeatureP9Altivec]>;
+// A separate feature for this even though it is equivalent to P9Vector
+// because this is a feature of the implementation rather than the architecture
+// and may go away with future CPU's.
+def FeatureVectorsUseTwoUnits : SubtargetFeature<"vectors-use-two-units",
+                                                 "VectorsUseTwoUnits",
+                                                 "true",
+                                                 "Vectors use two units">;
 
 // Since new processors generally contain a superset of features of those that
 // came before them, the idea is to make implementations of new processors
@@ -223,7 +229,8 @@ def ProcessorFeatures {
   list<SubtargetFeature> Power8FeatureList =
       !listconcat(Power7FeatureList, Power8SpecificFeatures);
   list<SubtargetFeature> Power9SpecificFeatures =
-      [DirectivePwr9, FeatureP9Altivec, FeatureP9Vector, FeatureISA3_0];
+      [DirectivePwr9, FeatureP9Altivec, FeatureP9Vector, FeatureISA3_0,
+       FeatureVectorsUseTwoUnits];
   list<SubtargetFeature> Power9FeatureList =
       !listconcat(Power8FeatureList, Power9SpecificFeatures);
 }
diff --git a/lib/Target/PowerPC/PPCAsmPrinter.cpp b/lib/Target/PowerPC/PPCAsmPrinter.cpp
index 04aa3c9b1e22cd2d32cc8d4ac7f4d331bb2d2e15..f4ef95caebd9bace3f41ef7e86daabaf5571240c 100644
--- a/lib/Target/PowerPC/PPCAsmPrinter.cpp
+++ b/lib/Target/PowerPC/PPCAsmPrinter.cpp
@@ -1,9 +1,8 @@
 //===-- PPCAsmPrinter.cpp - Print machine instrs to PowerPC assembly ------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/PowerPC/PPCBoolRetToInt.cpp b/lib/Target/PowerPC/PPCBoolRetToInt.cpp
index 55e105dad0e5b4736a1c7d894708d285c78ca66d..104cf2ba3c004c2c33d805758471955c49b25b50 100644
--- a/lib/Target/PowerPC/PPCBoolRetToInt.cpp
+++ b/lib/Target/PowerPC/PPCBoolRetToInt.cpp
@@ -1,9 +1,8 @@
 //===- PPCBoolRetToInt.cpp ------------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/PowerPC/PPCBranchCoalescing.cpp b/lib/Target/PowerPC/PPCBranchCoalescing.cpp
index bbb977f090c52698ce035cc5aca686cf229b2437..58f80bd46dc3c76e81164d229e703ce284064323 100644
--- a/lib/Target/PowerPC/PPCBranchCoalescing.cpp
+++ b/lib/Target/PowerPC/PPCBranchCoalescing.cpp
@@ -1,9 +1,8 @@
 //===-- CoalesceBranches.cpp - Coalesce blocks with the same condition ---===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
diff --git a/lib/Target/PowerPC/PPCBranchSelector.cpp b/lib/Target/PowerPC/PPCBranchSelector.cpp
index 0d1bb9297bcb353fc5b30ff7cd84c19cbb3ef617..c6eeaeb6c758f622dcdfcf489a51431b58c8c200 100644
--- a/lib/Target/PowerPC/PPCBranchSelector.cpp
+++ b/lib/Target/PowerPC/PPCBranchSelector.cpp
@@ -1,9 +1,8 @@
 //===-- PPCBranchSelector.cpp - Emit long conditional branches ------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/PowerPC/PPCCCState.cpp b/lib/Target/PowerPC/PPCCCState.cpp
index 5510a95430f5137f3ab1b399a79038e9e7da3298..5116f0d121f42ca1d778a45d8a5ae7e897933fa3 100644
--- a/lib/Target/PowerPC/PPCCCState.cpp
+++ b/lib/Target/PowerPC/PPCCCState.cpp
@@ -1,9 +1,8 @@
 //===---- PPCCCState.cpp - CCState with PowerPC specific extensions ---------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/PowerPC/PPCCCState.h b/lib/Target/PowerPC/PPCCCState.h
index 9be9f11dbea38a47619e102e68f91be6ec33f1e2..e3499597474c7739261d446ab2ebf0b0e28591d1 100644
--- a/lib/Target/PowerPC/PPCCCState.h
+++ b/lib/Target/PowerPC/PPCCCState.h
@@ -1,9 +1,8 @@
 //===---- PPCCCState.h - CCState with PowerPC specific extensions -----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/PowerPC/PPCCTRLoops.cpp b/lib/Target/PowerPC/PPCCTRLoops.cpp
index 6b9e2383e36f15897e4705eb4e43576a35ddf115..c52d7d83dfc5a653f62fdc9dc0b19bede33fe68c 100644
--- a/lib/Target/PowerPC/PPCCTRLoops.cpp
+++ b/lib/Target/PowerPC/PPCCTRLoops.cpp
@@ -1,9 +1,8 @@
 //===-- PPCCTRLoops.cpp - Identify and generate CTR loops -----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -661,13 +660,13 @@ bool PPCCTRLoops::convertToCTRLoop(Loop *L) {
 
   IRBuilder<> CountBuilder(Preheader->getTerminator());
   Module *M = Preheader->getParent()->getParent();
-  Value *MTCTRFunc = Intrinsic::getDeclaration(M, Intrinsic::ppc_mtctr,
-                                               CountType);
+  Function *MTCTRFunc =
+      Intrinsic::getDeclaration(M, Intrinsic::ppc_mtctr, CountType);
   CountBuilder.CreateCall(MTCTRFunc, ECValue);
 
   IRBuilder<> CondBuilder(CountedExitBranch);
-  Value *DecFunc =
-    Intrinsic::getDeclaration(M, Intrinsic::ppc_is_decremented_ctr_nonzero);
+  Function *DecFunc =
+      Intrinsic::getDeclaration(M, Intrinsic::ppc_is_decremented_ctr_nonzero);
   Value *NewCond = CondBuilder.CreateCall(DecFunc, {});
   Value *OldCond = CountedExitBranch->getCondition();
   CountedExitBranch->setCondition(NewCond);
diff --git a/lib/Target/PowerPC/PPCCallingConv.cpp b/lib/Target/PowerPC/PPCCallingConv.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..ecf1872e8fcb0e58e776aa30d31765da67de8cf6
--- /dev/null
+++ b/lib/Target/PowerPC/PPCCallingConv.cpp
@@ -0,0 +1,108 @@
+//===-- PPCCallingConv.h - --------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "PPCRegisterInfo.h"
+#include "PPCCallingConv.h"
+#include "PPCSubtarget.h"
+#include "PPCCCState.h"
+using namespace llvm;
+
+inline bool CC_PPC_AnyReg_Error(unsigned &, MVT &, MVT &,
+                                CCValAssign::LocInfo &, ISD::ArgFlagsTy &,
+                                CCState &) {
+  llvm_unreachable("The AnyReg calling convention is only supported by the " \
+                   "stackmap and patchpoint intrinsics.");
+  // gracefully fallback to PPC C calling convention on Release builds.
+  return false;
+}
+
+static bool CC_PPC32_SVR4_Custom_Dummy(unsigned &ValNo, MVT &ValVT, MVT &LocVT,
+                                       CCValAssign::LocInfo &LocInfo,
+                                       ISD::ArgFlagsTy &ArgFlags,
+                                       CCState &State) {
+  return true;
+}
+
+static bool CC_PPC32_SVR4_Custom_AlignArgRegs(unsigned &ValNo, MVT &ValVT,
+                                              MVT &LocVT,
+                                              CCValAssign::LocInfo &LocInfo,
+                                              ISD::ArgFlagsTy &ArgFlags,
+                                              CCState &State) {
+  static const MCPhysReg ArgRegs[] = {
+    PPC::R3, PPC::R4, PPC::R5, PPC::R6,
+    PPC::R7, PPC::R8, PPC::R9, PPC::R10,
+  };
+  const unsigned NumArgRegs = array_lengthof(ArgRegs);
+
+  unsigned RegNum = State.getFirstUnallocated(ArgRegs);
+
+  // Skip one register if the first unallocated register has an even register
+  // number and there are still argument registers available which have not been
+  // allocated yet. RegNum is actually an index into ArgRegs, which means we
+  // need to skip a register if RegNum is odd.
+  if (RegNum != NumArgRegs && RegNum % 2 == 1) {
+    State.AllocateReg(ArgRegs[RegNum]);
+  }
+
+  // Always return false here, as this function only makes sure that the first
+  // unallocated register has an odd register number and does not actually
+  // allocate a register for the current argument.
+  return false;
+}
+
+static bool CC_PPC32_SVR4_Custom_SkipLastArgRegsPPCF128(
+    unsigned &ValNo, MVT &ValVT, MVT &LocVT, CCValAssign::LocInfo &LocInfo,
+    ISD::ArgFlagsTy &ArgFlags, CCState &State) {
+  static const MCPhysReg ArgRegs[] = {
+    PPC::R3, PPC::R4, PPC::R5, PPC::R6,
+    PPC::R7, PPC::R8, PPC::R9, PPC::R10,
+  };
+  const unsigned NumArgRegs = array_lengthof(ArgRegs);
+
+  unsigned RegNum = State.getFirstUnallocated(ArgRegs);
+  int RegsLeft = NumArgRegs - RegNum;
+
+  // Skip if there is not enough registers left for long double type (4 gpr regs
+  // in soft float mode) and put long double argument on the stack.
+  if (RegNum != NumArgRegs && RegsLeft < 4) {
+    for (int i = 0; i < RegsLeft; i++) {
+      State.AllocateReg(ArgRegs[RegNum + i]);
+    }
+  }
+
+  return false;
+}
+
+static bool CC_PPC32_SVR4_Custom_AlignFPArgRegs(unsigned &ValNo, MVT &ValVT,
+                                                MVT &LocVT,
+                                                CCValAssign::LocInfo &LocInfo,
+                                                ISD::ArgFlagsTy &ArgFlags,
+                                                CCState &State) {
+  static const MCPhysReg ArgRegs[] = {
+    PPC::F1, PPC::F2, PPC::F3, PPC::F4, PPC::F5, PPC::F6, PPC::F7,
+    PPC::F8
+  };
+
+  const unsigned NumArgRegs = array_lengthof(ArgRegs);
+
+  unsigned RegNum = State.getFirstUnallocated(ArgRegs);
+
+  // If there is only one Floating-point register left we need to put both f64
+  // values of a split ppc_fp128 value on the stack.
+  if (RegNum != NumArgRegs && ArgRegs[RegNum] == PPC::F8) {
+    State.AllocateReg(ArgRegs[RegNum]);
+  }
+
+  // Always return false here, as this function only makes sure that the two f64
+  // values a ppc_fp128 value is split into are both passed in registers or both
+  // passed on the stack and does not actually allocate a register for the
+  // current argument.
+  return false;
+}
+
+#include "PPCGenCallingConv.inc"
diff --git a/lib/Target/PowerPC/PPCCallingConv.h b/lib/Target/PowerPC/PPCCallingConv.h
index eb904a85859242420ef8a65a98c2ae4c0e6af986..03d9be0a73d9ec9d6b563c286f66f729464bfdb7 100644
--- a/lib/Target/PowerPC/PPCCallingConv.h
+++ b/lib/Target/PowerPC/PPCCallingConv.h
@@ -1,9 +1,8 @@
 //=== PPCCallingConv.h - PPC Custom Calling Convention Routines -*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -20,14 +19,27 @@
 
 namespace llvm {
 
-inline bool CC_PPC_AnyReg_Error(unsigned &, MVT &, MVT &,
-                                CCValAssign::LocInfo &, ISD::ArgFlagsTy &,
-                                CCState &) {
-  llvm_unreachable("The AnyReg calling convention is only supported by the " \
-                   "stackmap and patchpoint intrinsics.");
-  // gracefully fallback to PPC C calling convention on Release builds.
-  return false;
-}
+bool RetCC_PPC(unsigned ValNo, MVT ValVT, MVT LocVT,
+               CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags,
+               CCState &State);
+bool RetCC_PPC64_ELF_FIS(unsigned ValNo, MVT ValVT, MVT LocVT,
+                         CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags,
+                         CCState &State);
+bool RetCC_PPC_Cold(unsigned ValNo, MVT ValVT, MVT LocVT,
+                    CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags,
+                    CCState &State);
+bool CC_PPC32_SVR4(unsigned ValNo, MVT ValVT, MVT LocVT,
+                   CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags,
+                   CCState &State);
+bool CC_PPC64_ELF_FIS(unsigned ValNo, MVT ValVT, MVT LocVT,
+                      CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags,
+                      CCState &State);
+bool CC_PPC32_SVR4_ByVal(unsigned ValNo, MVT ValVT, MVT LocVT,
+                         CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags,
+                         CCState &State);
+bool CC_PPC32_SVR4_VarArg(unsigned ValNo, MVT ValVT, MVT LocVT,
+                          CCValAssign::LocInfo LocInfo,
+                          ISD::ArgFlagsTy ArgFlags, CCState &State);
 
 } // End llvm namespace
 
diff --git a/lib/Target/PowerPC/PPCCallingConv.td b/lib/Target/PowerPC/PPCCallingConv.td
index 22842d516e7d4ad4a21855970811b8af3e4c170f..806c923897735db43923836cc75dd9ebcaa7fcbb 100644
--- a/lib/Target/PowerPC/PPCCallingConv.td
+++ b/lib/Target/PowerPC/PPCCallingConv.td
@@ -1,9 +1,8 @@
 //===- PPCCallingConv.td - Calling Conventions for PowerPC -*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -46,6 +45,7 @@ def RetCC_PPC64_AnyReg : CallingConv<[
 ]>;
 
 // Return-value convention for PowerPC coldcc.
+let Entry = 1 in
 def RetCC_PPC_Cold : CallingConv<[
   // Use the same return registers as RetCC_PPC, but limited to only
   // one return value. The remaining return values will be saved to
@@ -70,6 +70,7 @@ def RetCC_PPC_Cold : CallingConv<[
 ]>;
 
 // Return-value convention for PowerPC
+let Entry = 1 in
 def RetCC_PPC : CallingConv<[
   CCIfCC<"CallingConv::AnyReg", CCDelegateTo<RetCC_PPC64_AnyReg>>,
 
@@ -126,6 +127,7 @@ def CC_PPC64_AnyReg : CallingConv<[
 // Simple calling convention for 64-bit ELF PowerPC fast isel.
 // Only handle ints and floats.  All ints are promoted to i64.
 // Vector types and quadword ints are not handled.
+let Entry = 1 in
 def CC_PPC64_ELF_FIS : CallingConv<[
   CCIfCC<"CallingConv::AnyReg", CCDelegateTo<CC_PPC64_AnyReg>>,
 
@@ -141,6 +143,7 @@ def CC_PPC64_ELF_FIS : CallingConv<[
 // All small ints are promoted to i64.  Vector types, quadword ints,
 // and multiple register returns are "supported" to avoid compile
 // errors, but none are handled by the fast selector.
+let Entry = 1 in
 def RetCC_PPC64_ELF_FIS : CallingConv<[
   CCIfCC<"CallingConv::AnyReg", CCDelegateTo<RetCC_PPC64_AnyReg>>,
 
@@ -228,12 +231,14 @@ def CC_PPC32_SVR4_Common : CallingConv<[
 // This calling convention puts vector arguments always on the stack. It is used
 // to assign vector arguments which belong to the variable portion of the
 // parameter list of a variable argument function.
+let Entry = 1 in
 def CC_PPC32_SVR4_VarArg : CallingConv<[
   CCDelegateTo<CC_PPC32_SVR4_Common>
 ]>;
 
 // In contrast to CC_PPC32_SVR4_VarArg, this calling convention first tries to
 // put vector arguments in vector registers before putting them on the stack.
+let Entry = 1 in
 def CC_PPC32_SVR4 : CallingConv<[
   // QPX vectors mirror the scalar FP convention.
   CCIfType<[v4f64, v4f32, v4i1], CCIfSubtarget<"hasQPX()",
@@ -265,6 +270,7 @@ def CC_PPC32_SVR4 : CallingConv<[
 // The only purpose of CC_PPC32_SVR4_Custom_Dummy is to skip arguments which are
 // not passed by value.
  
+let Entry = 1 in
 def CC_PPC32_SVR4_ByVal : CallingConv<[
   CCIfByVal<CCPassByVal<4, 4>>,
   
diff --git a/lib/Target/PowerPC/PPCEarlyReturn.cpp b/lib/Target/PowerPC/PPCEarlyReturn.cpp
index ac931f7d0ec01602c894d6b836ed0ca8515a3018..8cc973a9ff4b950c23db5afdc67b0fa0fb92b944 100644
--- a/lib/Target/PowerPC/PPCEarlyReturn.cpp
+++ b/lib/Target/PowerPC/PPCEarlyReturn.cpp
@@ -1,9 +1,8 @@
 //===------------- PPCEarlyReturn.cpp - Form Early Returns ----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/PowerPC/PPCExpandISEL.cpp b/lib/Target/PowerPC/PPCExpandISEL.cpp
index a03e691ef5bb3ecf26178870e0c6bacba6c892f6..e8ef451c7ec9608a09dfd6251f05c9572b5cb587 100644
--- a/lib/Target/PowerPC/PPCExpandISEL.cpp
+++ b/lib/Target/PowerPC/PPCExpandISEL.cpp
@@ -1,9 +1,8 @@
 //===------------- PPCExpandISEL.cpp - Expand ISEL instruction ------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/PowerPC/PPCFastISel.cpp b/lib/Target/PowerPC/PPCFastISel.cpp
index aa55ac1f7acc428f63827295a46fa67978f85f2e..14a72bd9980ead4cbcaeb4c2f90a886a86703323 100644
--- a/lib/Target/PowerPC/PPCFastISel.cpp
+++ b/lib/Target/PowerPC/PPCFastISel.cpp
@@ -1,9 +1,8 @@
 //===-- PPCFastISel.cpp - PowerPC FastISel implementation -----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -152,6 +151,14 @@ class PPCFastISel final : public FastISel {
     bool isVSSRCRegClass(const TargetRegisterClass *RC) const {
       return RC->getID() == PPC::VSSRCRegClassID;
     }
+    unsigned copyRegToRegClass(const TargetRegisterClass *ToRC,
+                               unsigned SrcReg, unsigned Flag = 0,
+                               unsigned SubReg = 0) {
+      unsigned TmpReg = createResultReg(ToRC);
+      BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
+              TII.get(TargetOpcode::COPY), TmpReg).addReg(SrcReg, Flag, SubReg);
+      return TmpReg;
+    }
     bool PPCEmitCmp(const Value *Src1Value, const Value *Src2Value,
                     bool isZExt, unsigned DestReg,
                     const PPC::Predicate Pred);
@@ -187,7 +194,6 @@ class PPCFastISel final : public FastISel {
                          unsigned &NumBytes,
                          bool IsVarArg);
     bool finishCall(MVT RetVT, CallLoweringInfo &CLI, unsigned &NumBytes);
-    LLVM_ATTRIBUTE_UNUSED CCAssignFn *usePPC32CCs(unsigned Flag);
 
   private:
   #include "PPCGenFastISel.inc"
@@ -196,23 +202,6 @@ class PPCFastISel final : public FastISel {
 
 } // end anonymous namespace
 
-#include "PPCGenCallingConv.inc"
-
-// Function whose sole purpose is to kill compiler warnings
-// stemming from unused functions included from PPCGenCallingConv.inc.
-CCAssignFn *PPCFastISel::usePPC32CCs(unsigned Flag) {
-  if (Flag == 1)
-    return CC_PPC32_SVR4;
-  else if (Flag == 2)
-    return CC_PPC32_SVR4_ByVal;
-  else if (Flag == 3)
-    return CC_PPC32_SVR4_VarArg;
-  else if (Flag == 4)
-    return RetCC_PPC_Cold;
-  else
-    return RetCC_PPC;
-}
-
 static Optional<PPC::Predicate> getComparePred(CmpInst::Predicate Pred) {
   switch (Pred) {
     // These are not representable with any single compare.
@@ -861,8 +850,23 @@ bool PPCFastISel::PPCEmitCmp(const Value *SrcValue1, const Value *SrcValue2,
     }
   }
 
+  unsigned SrcReg1 = getRegForValue(SrcValue1);
+  if (SrcReg1 == 0)
+    return false;
+
+  unsigned SrcReg2 = 0;
+  if (!UseImm) {
+    SrcReg2 = getRegForValue(SrcValue2);
+    if (SrcReg2 == 0)
+      return false;
+  }
+
   unsigned CmpOpc;
   bool NeedsExt = false;
+
+  auto RC1 = MRI.getRegClass(SrcReg1);
+  auto RC2 = SrcReg2 != 0 ? MRI.getRegClass(SrcReg2) : nullptr;
+
   switch (SrcVT.SimpleTy) {
     default: return false;
     case MVT::f32:
@@ -879,8 +883,13 @@ bool PPCFastISel::PPCEmitCmp(const Value *SrcValue1, const Value *SrcValue2,
             CmpOpc = PPC::EFSCMPGT;
             break;
         }
-      } else
+      } else {
         CmpOpc = PPC::FCMPUS;
+        if (isVSSRCRegClass(RC1))
+          SrcReg1 = copyRegToRegClass(&PPC::F4RCRegClass, SrcReg1);
+        if (RC2 && isVSSRCRegClass(RC2))
+          SrcReg2 = copyRegToRegClass(&PPC::F4RCRegClass, SrcReg2);
+      }
       break;
     case MVT::f64:
       if (HasSPE) {
@@ -896,8 +905,11 @@ bool PPCFastISel::PPCEmitCmp(const Value *SrcValue1, const Value *SrcValue2,
             CmpOpc = PPC::EFDCMPGT;
             break;
         }
-      } else
+      } else if (isVSFRCRegClass(RC1) || (RC2 && isVSFRCRegClass(RC2))) {
+        CmpOpc = PPC::XSCMPUDP;
+      } else {
         CmpOpc = PPC::FCMPUD;
+      }
       break;
     case MVT::i1:
     case MVT::i8:
@@ -918,17 +930,6 @@ bool PPCFastISel::PPCEmitCmp(const Value *SrcValue1, const Value *SrcValue2,
       break;
   }
 
-  unsigned SrcReg1 = getRegForValue(SrcValue1);
-  if (SrcReg1 == 0)
-    return false;
-
-  unsigned SrcReg2 = 0;
-  if (!UseImm) {
-    SrcReg2 = getRegForValue(SrcValue2);
-    if (SrcReg2 == 0)
-      return false;
-  }
-
   if (NeedsExt) {
     unsigned ExtReg = createResultReg(&PPC::GPRCRegClass);
     if (!PPCEmitIntExt(SrcVT, SrcReg1, MVT::i32, ExtReg, IsZExt))
@@ -1209,13 +1210,8 @@ bool PPCFastISel::SelectFPToI(const Instruction *I, bool IsSigned) {
   // Convert f32 to f64 if necessary.  This is just a meaningless copy
   // to get the register class right.
   const TargetRegisterClass *InRC = MRI.getRegClass(SrcReg);
-  if (InRC == &PPC::F4RCRegClass) {
-    unsigned TmpReg = createResultReg(&PPC::F8RCRegClass);
-    BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
-            TII.get(TargetOpcode::COPY), TmpReg)
-      .addReg(SrcReg);
-    SrcReg = TmpReg;
-  }
+  if (InRC == &PPC::F4RCRegClass)
+    SrcReg = copyRegToRegClass(&PPC::F8RCRegClass, SrcReg);
 
   // Determine the opcode for the conversion, which takes place
   // entirely within FPRs.
@@ -1509,11 +1505,7 @@ bool PPCFastISel::finishCall(MVT RetVT, CallLoweringInfo &CLI, unsigned &NumByte
 
     if (RetVT == CopyVT) {
       const TargetRegisterClass *CpyRC = TLI.getRegClassFor(CopyVT);
-      ResultReg = createResultReg(CpyRC);
-
-      BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
-              TII.get(TargetOpcode::COPY), ResultReg)
-        .addReg(SourcePhysReg);
+      ResultReg = copyRegToRegClass(CpyRC, SourcePhysReg);
 
     // If necessary, round the floating result to single precision.
     } else if (CopyVT == MVT::f64) {
@@ -1526,12 +1518,9 @@ bool PPCFastISel::finishCall(MVT RetVT, CallLoweringInfo &CLI, unsigned &NumByte
     // used along the fast-isel path (not lowered), and downstream logic
     // also doesn't like a direct subreg copy on a physical reg.)
     } else if (RetVT == MVT::i8 || RetVT == MVT::i16 || RetVT == MVT::i32) {
-      ResultReg = createResultReg(&PPC::GPRCRegClass);
       // Convert physical register from G8RC to GPRC.
       SourcePhysReg -= PPC::X0 - PPC::R0;
-      BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
-              TII.get(TargetOpcode::COPY), ResultReg)
-        .addReg(SourcePhysReg);
+      ResultReg = copyRegToRegClass(&PPC::GPRCRegClass, SourcePhysReg);
     }
 
     assert(ResultReg && "ResultReg unset!");
@@ -1883,13 +1872,8 @@ bool PPCFastISel::SelectTrunc(const Instruction *I) {
     return false;
 
   // The only interesting case is when we need to switch register classes.
-  if (SrcVT == MVT::i64) {
-    unsigned ResultReg = createResultReg(&PPC::GPRCRegClass);
-    BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
-            TII.get(TargetOpcode::COPY),
-            ResultReg).addReg(SrcReg, 0, PPC::sub_32);
-    SrcReg = ResultReg;
-  }
+  if (SrcVT == MVT::i64)
+    SrcReg = copyRegToRegClass(&PPC::GPRCRegClass, SrcReg, 0, PPC::sub_32);
 
   updateValueMap(I, SrcReg);
   return true;
diff --git a/lib/Target/PowerPC/PPCFrameLowering.cpp b/lib/Target/PowerPC/PPCFrameLowering.cpp
index 8263954994d278862855d54e620aeb36c4116e66..efcdc5cc7b4887c0225823613ffac70c5bce0faa 100644
--- a/lib/Target/PowerPC/PPCFrameLowering.cpp
+++ b/lib/Target/PowerPC/PPCFrameLowering.cpp
@@ -1,9 +1,8 @@
 //===-- PPCFrameLowering.cpp - PPC Frame Information ----------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/PowerPC/PPCFrameLowering.h b/lib/Target/PowerPC/PPCFrameLowering.h
index 69bd1484d6e59f6d9609bf9a30495a77a5095d51..aa7499d49ad3eee61bfa55f38e7e953b49dd0f48 100644
--- a/lib/Target/PowerPC/PPCFrameLowering.h
+++ b/lib/Target/PowerPC/PPCFrameLowering.h
@@ -1,9 +1,8 @@
 //===-- PPCFrameLowering.h - Define frame lowering for PowerPC --*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/PowerPC/PPCHazardRecognizers.cpp b/lib/Target/PowerPC/PPCHazardRecognizers.cpp
index 5f6966cecd61a310dfb5cafc9c33c172467fb5bc..472019c8d160e9ff29305284ef9c2d8067136811 100644
--- a/lib/Target/PowerPC/PPCHazardRecognizers.cpp
+++ b/lib/Target/PowerPC/PPCHazardRecognizers.cpp
@@ -1,9 +1,8 @@
 //===-- PPCHazardRecognizers.cpp - PowerPC Hazard Recognizer Impls --------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/PowerPC/PPCHazardRecognizers.h b/lib/Target/PowerPC/PPCHazardRecognizers.h
index 4b502147ca63851a19a5dbe448f8116c2794370d..5b32147ca88d373bc060aa407550277d29ef46d8 100644
--- a/lib/Target/PowerPC/PPCHazardRecognizers.h
+++ b/lib/Target/PowerPC/PPCHazardRecognizers.h
@@ -1,9 +1,8 @@
 //===-- PPCHazardRecognizers.h - PowerPC Hazard Recognizers -----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/PowerPC/PPCISelDAGToDAG.cpp b/lib/Target/PowerPC/PPCISelDAGToDAG.cpp
index 4f05f50f6994fa06bb7781498a88abb6d677d6d3..e28f4a7690e0b65246e564f4ec6c94bb76456fc0 100644
--- a/lib/Target/PowerPC/PPCISelDAGToDAG.cpp
+++ b/lib/Target/PowerPC/PPCISelDAGToDAG.cpp
@@ -1,9 +1,8 @@
 //===-- PPCISelDAGToDAG.cpp - PPC --pattern matching inst selector --------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -688,9 +687,8 @@ bool PPCDAGToDAGISel::tryBitfieldInsert(SDNode *N) {
   SDValue Op1 = N->getOperand(1);
   SDLoc dl(N);
 
-  KnownBits LKnown, RKnown;
-  CurDAG->computeKnownBits(Op0, LKnown);
-  CurDAG->computeKnownBits(Op1, RKnown);
+  KnownBits LKnown = CurDAG->computeKnownBits(Op0);
+  KnownBits RKnown = CurDAG->computeKnownBits(Op1);
 
   unsigned TargetMask = LKnown.Zero.getZExtValue();
   unsigned InsertMask = RKnown.Zero.getZExtValue();
@@ -734,8 +732,7 @@ bool PPCDAGToDAGISel::tryBitfieldInsert(SDNode *N) {
        // The AND mask might not be a constant, and we need to make sure that
        // if we're going to fold the masking with the insert, all bits not
        // know to be zero in the mask are known to be one.
-        KnownBits MKnown;
-        CurDAG->computeKnownBits(Op1.getOperand(1), MKnown);
+        KnownBits MKnown = CurDAG->computeKnownBits(Op1.getOperand(1));
         bool CanFoldMask = InsertMask == MKnown.One.getZExtValue();
 
         unsigned SHOpc = Op1.getOperand(0).getOpcode();
@@ -1333,6 +1330,34 @@ class BitPermutationSelector {
 
       return std::make_pair(Interesting, &Bits);
     }
+    case ISD::TRUNCATE: {
+      EVT FromType = V.getOperand(0).getValueType();
+      EVT ToType = V.getValueType();
+      // We support only the case with truncate from i64 to i32.
+      if (FromType != MVT::i64 || ToType != MVT::i32)
+        break;
+      const unsigned NumAllBits = FromType.getSizeInBits();
+      SmallVector<ValueBit, 64> *InBits;
+      std::tie(Interesting, InBits) = getValueBits(V.getOperand(0),
+                                                    NumAllBits);
+      const unsigned NumValidBits = ToType.getSizeInBits();
+
+      // A 32-bit instruction cannot touch upper 32-bit part of 64-bit value.
+      // So, we cannot include this truncate.
+      bool UseUpper32bit = false;
+      for (unsigned i = 0; i < NumValidBits; ++i)
+        if ((*InBits)[i].hasValue() && (*InBits)[i].getValueBitIndex() >= 32) {
+          UseUpper32bit = true;
+          break;
+        }
+      if (UseUpper32bit)
+        break;
+
+      for (unsigned i = 0; i < NumValidBits; ++i)
+        Bits[i] = (*InBits)[i];
+
+      return std::make_pair(Interesting, &Bits);
+    }
     case ISD::AssertZext: {
       // For AssertZext, we look through the operand and
       // mark the bits known to be zero.
@@ -1678,6 +1703,17 @@ class BitPermutationSelector {
     return ExtVal;
   }
 
+  SDValue TruncateToInt32(SDValue V, const SDLoc &dl) {
+    if (V.getValueSizeInBits() == 32)
+      return V;
+
+    assert(V.getValueSizeInBits() == 64);
+    SDValue SubRegIdx = CurDAG->getTargetConstant(PPC::sub_32, dl, MVT::i32);
+    SDValue SubVal = SDValue(CurDAG->getMachineNode(PPC::EXTRACT_SUBREG, dl,
+                                                    MVT::i32, V, SubRegIdx), 0);
+    return SubVal;
+  }
+
   // Depending on the number of groups for a particular value, it might be
   // better to rotate, mask explicitly (using andi/andis), and then or the
   // result. Select this part of the result first.
@@ -1736,12 +1772,12 @@ class BitPermutationSelector {
       SDValue VRot;
       if (VRI.RLAmt) {
         SDValue Ops[] =
-          { VRI.V, getI32Imm(VRI.RLAmt, dl), getI32Imm(0, dl),
-            getI32Imm(31, dl) };
+          { TruncateToInt32(VRI.V, dl), getI32Imm(VRI.RLAmt, dl),
+            getI32Imm(0, dl), getI32Imm(31, dl) };
         VRot = SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32,
                                               Ops), 0);
       } else {
-        VRot = VRI.V;
+        VRot = TruncateToInt32(VRI.V, dl);
       }
 
       SDValue ANDIVal, ANDISVal;
@@ -1793,12 +1829,12 @@ class BitPermutationSelector {
       if (VRI.RLAmt) {
         if (InstCnt) *InstCnt += 1;
         SDValue Ops[] =
-          { VRI.V, getI32Imm(VRI.RLAmt, dl), getI32Imm(0, dl),
-            getI32Imm(31, dl) };
+          { TruncateToInt32(VRI.V, dl), getI32Imm(VRI.RLAmt, dl),
+            getI32Imm(0, dl), getI32Imm(31, dl) };
         Res = SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32, Ops),
                       0);
       } else {
-        Res = VRI.V;
+        Res = TruncateToInt32(VRI.V, dl);
       }
 
       // Now, remove all groups with this underlying value and rotation factor.
@@ -1813,13 +1849,13 @@ class BitPermutationSelector {
     for (auto &BG : BitGroups) {
       if (!Res) {
         SDValue Ops[] =
-          { BG.V, getI32Imm(BG.RLAmt, dl),
+          { TruncateToInt32(BG.V, dl), getI32Imm(BG.RLAmt, dl),
             getI32Imm(Bits.size() - BG.EndIdx - 1, dl),
             getI32Imm(Bits.size() - BG.StartIdx - 1, dl) };
         Res = SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32, Ops), 0);
       } else {
         SDValue Ops[] =
-          { Res, BG.V, getI32Imm(BG.RLAmt, dl),
+          { Res, TruncateToInt32(BG.V, dl), getI32Imm(BG.RLAmt, dl),
               getI32Imm(Bits.size() - BG.EndIdx - 1, dl),
             getI32Imm(Bits.size() - BG.StartIdx - 1, dl) };
         Res = SDValue(CurDAG->getMachineNode(PPC::RLWIMI, dl, MVT::i32, Ops), 0);
@@ -4613,8 +4649,7 @@ void PPCDAGToDAGISel::Select(SDNode *N) {
     int16_t Imm;
     if (N->getOperand(0)->getOpcode() == ISD::FrameIndex &&
         isIntS16Immediate(N->getOperand(1), Imm)) {
-      KnownBits LHSKnown;
-      CurDAG->computeKnownBits(N->getOperand(0), LHSKnown);
+      KnownBits LHSKnown = CurDAG->computeKnownBits(N->getOperand(0));
 
       // If this is equivalent to an add, then we can fold it with the
       // FrameIndex calculation.
diff --git a/lib/Target/PowerPC/PPCISelLowering.cpp b/lib/Target/PowerPC/PPCISelLowering.cpp
index 92af82dc4b9f52cb1af665dfdbd7b532ab07a939..c271ed95dd6a9d6f51de26981523e5e21410207e 100644
--- a/lib/Target/PowerPC/PPCISelLowering.cpp
+++ b/lib/Target/PowerPC/PPCISelLowering.cpp
@@ -1,9 +1,8 @@
 //===-- PPCISelLowering.cpp - PPC DAG Lowering Implementation -------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -119,6 +118,8 @@ STATISTIC(NumSiblingCalls, "Number of sibling calls");
 
 static bool isNByteElemShuffleMask(ShuffleVectorSDNode *, unsigned, int);
 
+static SDValue widenVec(SelectionDAG &DAG, SDValue Vec, const SDLoc &dl);
+
 // FIXME: Remove this once the bug has been fixed!
 extern cl::opt<bool> ANDIGlueBug;
 
@@ -640,6 +641,14 @@ PPCTargetLowering::PPCTargetLowering(const PPCTargetMachine &TM,
     // with merges, splats, etc.
     setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v16i8, Custom);
 
+    // Vector truncates to sub-word integer that fit in an Altivec/VSX register
+    // are cheap, so handle them before they get expanded to scalar.
+    setOperationAction(ISD::TRUNCATE, MVT::v8i8, Custom);
+    setOperationAction(ISD::TRUNCATE, MVT::v4i8, Custom);
+    setOperationAction(ISD::TRUNCATE, MVT::v2i8, Custom);
+    setOperationAction(ISD::TRUNCATE, MVT::v4i16, Custom);
+    setOperationAction(ISD::TRUNCATE, MVT::v2i16, Custom);
+
     setOperationAction(ISD::AND   , MVT::v4i32, Legal);
     setOperationAction(ISD::OR    , MVT::v4i32, Legal);
     setOperationAction(ISD::XOR   , MVT::v4i32, Legal);
@@ -788,8 +797,17 @@ PPCTargetLowering::PPCTargetLowering(const PPCTargetMachine &TM,
       setOperationAction(ISD::FP_TO_SINT, MVT::v2i64, Legal);
       setOperationAction(ISD::FP_TO_UINT, MVT::v2i64, Legal);
 
+      // Custom handling for partial vectors of integers converted to
+      // floating point. We already have optimal handling for v2i32 through
+      // the DAG combine, so those aren't necessary.
+      setOperationAction(ISD::UINT_TO_FP, MVT::v2i8, Custom);
+      setOperationAction(ISD::UINT_TO_FP, MVT::v4i8, Custom);
       setOperationAction(ISD::UINT_TO_FP, MVT::v2i16, Custom);
+      setOperationAction(ISD::UINT_TO_FP, MVT::v4i16, Custom);
+      setOperationAction(ISD::SINT_TO_FP, MVT::v2i8, Custom);
+      setOperationAction(ISD::SINT_TO_FP, MVT::v4i8, Custom);
       setOperationAction(ISD::SINT_TO_FP, MVT::v2i16, Custom);
+      setOperationAction(ISD::SINT_TO_FP, MVT::v4i16, Custom);
 
       setOperationAction(ISD::FNEG, MVT::v4f32, Legal);
       setOperationAction(ISD::FNEG, MVT::v2f64, Legal);
@@ -2216,11 +2234,10 @@ bool PPCTargetLowering::SelectAddressRegReg(SDValue N, SDValue &Base,
     // If this is an or of disjoint bitfields, we can codegen this as an add
     // (for better address arithmetic) if the LHS and RHS of the OR are provably
     // disjoint.
-    KnownBits LHSKnown, RHSKnown;
-    DAG.computeKnownBits(N.getOperand(0), LHSKnown);
+    KnownBits LHSKnown = DAG.computeKnownBits(N.getOperand(0));
 
     if (LHSKnown.Zero.getBoolValue()) {
-      DAG.computeKnownBits(N.getOperand(1), RHSKnown);
+      KnownBits RHSKnown = DAG.computeKnownBits(N.getOperand(1));
       // If all of the bits are known zero on the LHS or RHS, the add won't
       // carry.
       if (~(LHSKnown.Zero | RHSKnown.Zero) == 0) {
@@ -2319,8 +2336,7 @@ bool PPCTargetLowering::SelectAddressRegImm(SDValue N, SDValue &Disp,
       // If this is an or of disjoint bitfields, we can codegen this as an add
       // (for better address arithmetic) if the LHS and RHS of the OR are
       // provably disjoint.
-      KnownBits LHSKnown;
-      DAG.computeKnownBits(N.getOperand(0), LHSKnown);
+      KnownBits LHSKnown = DAG.computeKnownBits(N.getOperand(0));
 
       if ((LHSKnown.Zero.getZExtValue()|~(uint64_t)imm) == ~0ULL) {
         // If all of the bits are known zero on the LHS or RHS, the add won't
@@ -3140,101 +3156,6 @@ SDValue PPCTargetLowering::LowerVASTART(SDValue Op, SelectionDAG &DAG) const {
                       MachinePointerInfo(SV, nextOffset));
 }
 
-#include "PPCGenCallingConv.inc"
-
-// Function whose sole purpose is to kill compiler warnings
-// stemming from unused functions included from PPCGenCallingConv.inc.
-CCAssignFn *PPCTargetLowering::useFastISelCCs(unsigned Flag) const {
-  return Flag ? CC_PPC64_ELF_FIS : RetCC_PPC64_ELF_FIS;
-}
-
-bool llvm::CC_PPC32_SVR4_Custom_Dummy(unsigned &ValNo, MVT &ValVT, MVT &LocVT,
-                                      CCValAssign::LocInfo &LocInfo,
-                                      ISD::ArgFlagsTy &ArgFlags,
-                                      CCState &State) {
-  return true;
-}
-
-bool llvm::CC_PPC32_SVR4_Custom_AlignArgRegs(unsigned &ValNo, MVT &ValVT,
-                                             MVT &LocVT,
-                                             CCValAssign::LocInfo &LocInfo,
-                                             ISD::ArgFlagsTy &ArgFlags,
-                                             CCState &State) {
-  static const MCPhysReg ArgRegs[] = {
-    PPC::R3, PPC::R4, PPC::R5, PPC::R6,
-    PPC::R7, PPC::R8, PPC::R9, PPC::R10,
-  };
-  const unsigned NumArgRegs = array_lengthof(ArgRegs);
-
-  unsigned RegNum = State.getFirstUnallocated(ArgRegs);
-
-  // Skip one register if the first unallocated register has an even register
-  // number and there are still argument registers available which have not been
-  // allocated yet. RegNum is actually an index into ArgRegs, which means we
-  // need to skip a register if RegNum is odd.
-  if (RegNum != NumArgRegs && RegNum % 2 == 1) {
-    State.AllocateReg(ArgRegs[RegNum]);
-  }
-
-  // Always return false here, as this function only makes sure that the first
-  // unallocated register has an odd register number and does not actually
-  // allocate a register for the current argument.
-  return false;
-}
-
-bool
-llvm::CC_PPC32_SVR4_Custom_SkipLastArgRegsPPCF128(unsigned &ValNo, MVT &ValVT,
-                                                  MVT &LocVT,
-                                                  CCValAssign::LocInfo &LocInfo,
-                                                  ISD::ArgFlagsTy &ArgFlags,
-                                                  CCState &State) {
-  static const MCPhysReg ArgRegs[] = {
-    PPC::R3, PPC::R4, PPC::R5, PPC::R6,
-    PPC::R7, PPC::R8, PPC::R9, PPC::R10,
-  };
-  const unsigned NumArgRegs = array_lengthof(ArgRegs);
-
-  unsigned RegNum = State.getFirstUnallocated(ArgRegs);
-  int RegsLeft = NumArgRegs - RegNum;
-
-  // Skip if there is not enough registers left for long double type (4 gpr regs
-  // in soft float mode) and put long double argument on the stack.
-  if (RegNum != NumArgRegs && RegsLeft < 4) {
-    for (int i = 0; i < RegsLeft; i++) {
-      State.AllocateReg(ArgRegs[RegNum + i]);
-    }
-  }
-
-  return false;
-}
-
-bool llvm::CC_PPC32_SVR4_Custom_AlignFPArgRegs(unsigned &ValNo, MVT &ValVT,
-                                               MVT &LocVT,
-                                               CCValAssign::LocInfo &LocInfo,
-                                               ISD::ArgFlagsTy &ArgFlags,
-                                               CCState &State) {
-  static const MCPhysReg ArgRegs[] = {
-    PPC::F1, PPC::F2, PPC::F3, PPC::F4, PPC::F5, PPC::F6, PPC::F7,
-    PPC::F8
-  };
-
-  const unsigned NumArgRegs = array_lengthof(ArgRegs);
-
-  unsigned RegNum = State.getFirstUnallocated(ArgRegs);
-
-  // If there is only one Floating-point register left we need to put both f64
-  // values of a split ppc_fp128 value on the stack.
-  if (RegNum != NumArgRegs && ArgRegs[RegNum] == PPC::F8) {
-    State.AllocateReg(ArgRegs[RegNum]);
-  }
-
-  // Always return false here, as this function only makes sure that the two f64
-  // values a ppc_fp128 value is split into are both passed in registers or both
-  // passed on the stack and does not actually allocate a register for the
-  // current argument.
-  return false;
-}
-
 /// FPR - The set of FP registers that should be allocated for arguments,
 /// on Darwin.
 static const MCPhysReg FPR[] = {PPC::F1,  PPC::F2,  PPC::F3, PPC::F4, PPC::F5,
@@ -5090,9 +5011,15 @@ PrepareCall(SelectionDAG &DAG, SDValue &Callee, SDValue &InFlag, SDValue &Chain,
 
   // All calls, in both the ELF V1 and V2 ABIs, need the TOC register live
   // into the call.
-  if (isSVR4ABI && isPPC64 && !isPatchPoint) {
+  // We do need to reserve X2 to appease the verifier for the PATCHPOINT.
+  if (isSVR4ABI && isPPC64) {
     setUsesTOCBasePtr(DAG);
-    Ops.push_back(DAG.getRegister(PPC::X2, PtrVT));
+
+    // We cannot add X2 as an operand here for PATCHPOINT, because there is no
+    // way to mark dependencies as implicit here. We will add the X2 dependency
+    // in EmitInstrWithCustomInserter.
+    if (!isPatchPoint) 
+      Ops.push_back(DAG.getRegister(PPC::X2, PtrVT));
   }
 
   return CallOpc;
@@ -6877,6 +6804,61 @@ SDValue PPCTargetLowering::LowerTRUNCATE(SDValue Op, SelectionDAG &DAG) const {
                      Op.getOperand(0));
 }
 
+SDValue PPCTargetLowering::LowerTRUNCATEVector(SDValue Op,
+                                               SelectionDAG &DAG) const {
+
+  // Implements a vector truncate that fits in a vector register as a shuffle.
+  // We want to legalize vector truncates down to where the source fits in
+  // a vector register (and target is therefore smaller than vector register
+  // size).  At that point legalization will try to custom lower the sub-legal
+  // result and get here - where we can contain the truncate as a single target
+  // operation.
+
+  // For example a trunc <2 x i16> to <2 x i8> could be visualized as follows:
+  //   <MSB1|LSB1, MSB2|LSB2> to <LSB1, LSB2>
+  //
+  // We will implement it for big-endian ordering as this (where x denotes
+  // undefined):
+  //   < MSB1|LSB1, MSB2|LSB2, uu, uu, uu, uu, uu, uu> to
+  //   < LSB1, LSB2, u, u, u, u, u, u, u, u, u, u, u, u, u, u>
+  // 
+  // The same operation in little-endian ordering will be:
+  //   <uu, uu, uu, uu, uu, uu, LSB2|MSB2, LSB1|MSB1> to
+  //   <u, u, u, u, u, u, u, u, u, u, u, u, u, u, LSB2, LSB1>
+
+  assert(Op.getValueType().isVector() && "Vector type expected.");
+
+  SDLoc DL(Op);
+  SDValue N1 = Op.getOperand(0);
+  unsigned SrcSize = N1.getValueType().getSizeInBits();
+  assert(SrcSize <= 128 && "Source must fit in an Altivec/VSX vector");
+  SDValue WideSrc = SrcSize == 128 ? N1 : widenVec(DAG, N1, DL);
+
+  EVT TrgVT = Op.getValueType();
+  unsigned TrgNumElts = TrgVT.getVectorNumElements();
+  EVT EltVT = TrgVT.getVectorElementType();
+  unsigned WideNumElts = 128 / EltVT.getSizeInBits();
+  EVT WideVT = EVT::getVectorVT(*DAG.getContext(), EltVT, WideNumElts);
+
+  // First list the elements we want to keep.
+  unsigned SizeMult = SrcSize / TrgVT.getSizeInBits();
+  SmallVector<int, 16> ShuffV;
+  if (Subtarget.isLittleEndian())
+    for (unsigned i = 0; i < TrgNumElts; ++i)
+      ShuffV.push_back(i * SizeMult);
+  else
+    for (unsigned i = 1; i <= TrgNumElts; ++i)
+      ShuffV.push_back(i * SizeMult - 1);
+
+  // Populate the remaining elements with undefs.
+  for (unsigned i = TrgNumElts; i < WideNumElts; ++i)
+    // ShuffV.push_back(i + WideNumElts);
+    ShuffV.push_back(WideNumElts + 1);
+
+  SDValue Conv = DAG.getNode(ISD::BITCAST, DL, WideVT, WideSrc);
+  return DAG.getVectorShuffle(WideVT, DL, Conv, DAG.getUNDEF(WideVT), ShuffV);
+}
+
 /// LowerSELECT_CC - Lower floating point select_cc's into fsel instruction when
 /// possible.
 SDValue PPCTargetLowering::LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const {
@@ -7290,43 +7272,49 @@ static SDValue widenVec(SelectionDAG &DAG, SDValue Vec, const SDLoc &dl) {
   return DAG.getNode(ISD::CONCAT_VECTORS, dl, WideVT, Ops);
 }
 
-SDValue PPCTargetLowering::LowerINT_TO_FPVector(SDValue Op,
-                                                SelectionDAG &DAG,
+SDValue PPCTargetLowering::LowerINT_TO_FPVector(SDValue Op, SelectionDAG &DAG,
                                                 const SDLoc &dl) const {
 
   unsigned Opc = Op.getOpcode();
   assert((Opc == ISD::UINT_TO_FP || Opc == ISD::SINT_TO_FP) &&
          "Unexpected conversion type");
-  assert(Op.getValueType() == MVT::v2f64 && "Supports v2f64 only.");
+  assert((Op.getValueType() == MVT::v2f64 || Op.getValueType() == MVT::v4f32) &&
+         "Supports conversions to v2f64/v4f32 only.");
 
-  // CPU's prior to P9 don't have a way to sign-extend in vectors.
   bool SignedConv = Opc == ISD::SINT_TO_FP;
-  if (SignedConv && !Subtarget.hasP9Altivec())
-    return SDValue();
+  bool FourEltRes = Op.getValueType() == MVT::v4f32;
 
   SDValue Wide = widenVec(DAG, Op.getOperand(0), dl);
   EVT WideVT = Wide.getValueType();
   unsigned WideNumElts = WideVT.getVectorNumElements();
+  MVT IntermediateVT = FourEltRes ? MVT::v4i32 : MVT::v2i64;
 
   SmallVector<int, 16> ShuffV;
   for (unsigned i = 0; i < WideNumElts; ++i)
     ShuffV.push_back(i + WideNumElts);
 
-  if (Subtarget.isLittleEndian()) {
-    ShuffV[0] = 0;
-    ShuffV[WideNumElts / 2] = 1;
-  }
-  else {
-    ShuffV[WideNumElts / 2 - 1] = 0;
-    ShuffV[WideNumElts - 1] = 1;
-  }
+  int Stride = FourEltRes ? WideNumElts / 4 : WideNumElts / 2;
+  int SaveElts = FourEltRes ? 4 : 2;
+  if (Subtarget.isLittleEndian())
+    for (int i = 0; i < SaveElts; i++)
+      ShuffV[i * Stride] = i;
+  else
+    for (int i = 1; i <= SaveElts; i++)
+      ShuffV[i * Stride - 1] = i - 1;
 
-  SDValue ShuffleSrc2 = SignedConv ? DAG.getUNDEF(WideVT) :
-                                     DAG.getConstant(0, dl, WideVT);
+  SDValue ShuffleSrc2 =
+      SignedConv ? DAG.getUNDEF(WideVT) : DAG.getConstant(0, dl, WideVT);
   SDValue Arrange = DAG.getVectorShuffle(WideVT, dl, Wide, ShuffleSrc2, ShuffV);
-  unsigned ExtendOp = SignedConv ? (unsigned) PPCISD::SExtVElems :
-                                   (unsigned) ISD::BITCAST;
-  SDValue Extend = DAG.getNode(ExtendOp, dl, MVT::v2i64, Arrange);
+  unsigned ExtendOp =
+      SignedConv ? (unsigned)PPCISD::SExtVElems : (unsigned)ISD::BITCAST;
+
+  SDValue Extend;
+  if (!Subtarget.hasP9Altivec() && SignedConv) {
+    Arrange = DAG.getBitcast(IntermediateVT, Arrange);
+    Extend = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, IntermediateVT, Arrange,
+                         DAG.getValueType(Op.getOperand(0).getValueType()));
+  } else
+    Extend = DAG.getNode(ExtendOp, dl, IntermediateVT, Arrange);
 
   return DAG.getNode(Opc, dl, Op.getValueType(), Extend);
 }
@@ -7335,8 +7323,10 @@ SDValue PPCTargetLowering::LowerINT_TO_FP(SDValue Op,
                                           SelectionDAG &DAG) const {
   SDLoc dl(Op);
 
-  if (Op.getValueType() == MVT::v2f64 &&
-      Op.getOperand(0).getValueType() == MVT::v2i16)
+  EVT InVT = Op.getOperand(0).getValueType();
+  EVT OutVT = Op.getValueType();
+  if (OutVT.isVector() && OutVT.isFloatingPoint() &&
+      isOperationCustom(Op.getOpcode(), InVT))
     return LowerINT_TO_FPVector(Op, DAG, dl);
 
   // Conversions to f128 are legal.
@@ -9716,6 +9706,14 @@ void PPCTargetLowering::ReplaceNodeResults(SDNode *N,
       return;
     Results.push_back(LowerFP_TO_INT(SDValue(N, 0), DAG, dl));
     return;
+  case ISD::TRUNCATE: {
+    EVT TrgVT = N->getValueType(0);
+    if (TrgVT.isVector() &&
+        isOperationCustom(N->getOpcode(), TrgVT) &&
+        N->getOperand(0).getValueType().getSizeInBits() <= 128)
+      Results.push_back(LowerTRUNCATEVector(SDValue(N, 0), DAG));
+    return;
+  }
   case ISD::BITCAST:
     // Don't handle bitcast here.
     return;
@@ -9882,17 +9880,14 @@ PPCTargetLowering::EmitAtomicBinary(MachineInstr &MI, MachineBasicBlock *BB,
   return BB;
 }
 
-MachineBasicBlock *
-PPCTargetLowering::EmitPartwordAtomicBinary(MachineInstr &MI,
-                                            MachineBasicBlock *BB,
-                                            bool is8bit, // operation
-                                            unsigned BinOpcode,
-                                            unsigned CmpOpcode,
-                                            unsigned CmpPred) const {
+MachineBasicBlock *PPCTargetLowering::EmitPartwordAtomicBinary(
+    MachineInstr &MI, MachineBasicBlock *BB,
+    bool is8bit, // operation
+    unsigned BinOpcode, unsigned CmpOpcode, unsigned CmpPred) const {
   // If we support part-word atomic mnemonics, just use them
   if (Subtarget.hasPartwordAtomics())
-    return EmitAtomicBinary(MI, BB, is8bit ? 1 : 2, BinOpcode,
-                            CmpOpcode, CmpPred);
+    return EmitAtomicBinary(MI, BB, is8bit ? 1 : 2, BinOpcode, CmpOpcode,
+                            CmpPred);
 
   // This also handles ATOMIC_SWAP, indicated by BinOpcode==0.
   const TargetInstrInfo *TII = Subtarget.getInstrInfo();
@@ -9916,7 +9911,7 @@ PPCTargetLowering::EmitPartwordAtomicBinary(MachineInstr &MI,
 
   MachineBasicBlock *loopMBB = F->CreateMachineBasicBlock(LLVM_BB);
   MachineBasicBlock *loop2MBB =
-    CmpOpcode ? F->CreateMachineBasicBlock(LLVM_BB) : nullptr;
+      CmpOpcode ? F->CreateMachineBasicBlock(LLVM_BB) : nullptr;
   MachineBasicBlock *exitMBB = F->CreateMachineBasicBlock(LLVM_BB);
   F->insert(It, loopMBB);
   if (CmpOpcode)
@@ -9927,22 +9922,25 @@ PPCTargetLowering::EmitPartwordAtomicBinary(MachineInstr &MI,
   exitMBB->transferSuccessorsAndUpdatePHIs(BB);
 
   MachineRegisterInfo &RegInfo = F->getRegInfo();
-  const TargetRegisterClass *RC = is64bit ? &PPC::G8RCRegClass
-                                          : &PPC::GPRCRegClass;
+  const TargetRegisterClass *RC =
+      is64bit ? &PPC::G8RCRegClass : &PPC::GPRCRegClass;
+  const TargetRegisterClass *GPRC = &PPC::GPRCRegClass;
+
   unsigned PtrReg = RegInfo.createVirtualRegister(RC);
-  unsigned Shift1Reg = RegInfo.createVirtualRegister(RC);
+  unsigned Shift1Reg = RegInfo.createVirtualRegister(GPRC);
   unsigned ShiftReg =
-    isLittleEndian ? Shift1Reg : RegInfo.createVirtualRegister(RC);
-  unsigned Incr2Reg = RegInfo.createVirtualRegister(RC);
-  unsigned MaskReg = RegInfo.createVirtualRegister(RC);
-  unsigned Mask2Reg = RegInfo.createVirtualRegister(RC);
-  unsigned Mask3Reg = RegInfo.createVirtualRegister(RC);
-  unsigned Tmp2Reg = RegInfo.createVirtualRegister(RC);
-  unsigned Tmp3Reg = RegInfo.createVirtualRegister(RC);
-  unsigned Tmp4Reg = RegInfo.createVirtualRegister(RC);
-  unsigned TmpDestReg = RegInfo.createVirtualRegister(RC);
+      isLittleEndian ? Shift1Reg : RegInfo.createVirtualRegister(GPRC);
+  unsigned Incr2Reg = RegInfo.createVirtualRegister(GPRC);
+  unsigned MaskReg = RegInfo.createVirtualRegister(GPRC);
+  unsigned Mask2Reg = RegInfo.createVirtualRegister(GPRC);
+  unsigned Mask3Reg = RegInfo.createVirtualRegister(GPRC);
+  unsigned Tmp2Reg = RegInfo.createVirtualRegister(GPRC);
+  unsigned Tmp3Reg = RegInfo.createVirtualRegister(GPRC);
+  unsigned Tmp4Reg = RegInfo.createVirtualRegister(GPRC);
+  unsigned TmpDestReg = RegInfo.createVirtualRegister(GPRC);
   unsigned Ptr1Reg;
-  unsigned TmpReg = (!BinOpcode) ? Incr2Reg : RegInfo.createVirtualRegister(RC);
+  unsigned TmpReg =
+      (!BinOpcode) ? Incr2Reg : RegInfo.createVirtualRegister(GPRC);
 
   //  thisMBB:
   //   ...
@@ -9971,82 +9969,107 @@ PPCTargetLowering::EmitPartwordAtomicBinary(MachineInstr &MI,
   if (ptrA != ZeroReg) {
     Ptr1Reg = RegInfo.createVirtualRegister(RC);
     BuildMI(BB, dl, TII->get(is64bit ? PPC::ADD8 : PPC::ADD4), Ptr1Reg)
-      .addReg(ptrA).addReg(ptrB);
+        .addReg(ptrA)
+        .addReg(ptrB);
   } else {
     Ptr1Reg = ptrB;
   }
-  BuildMI(BB, dl, TII->get(PPC::RLWINM), Shift1Reg).addReg(Ptr1Reg)
-      .addImm(3).addImm(27).addImm(is8bit ? 28 : 27);
+  // We need use 32-bit subregister to avoid mismatch register class in 64-bit
+  // mode.
+  BuildMI(BB, dl, TII->get(PPC::RLWINM), Shift1Reg)
+      .addReg(Ptr1Reg, 0, is64bit ? PPC::sub_32 : 0)
+      .addImm(3)
+      .addImm(27)
+      .addImm(is8bit ? 28 : 27);
   if (!isLittleEndian)
-    BuildMI(BB, dl, TII->get(is64bit ? PPC::XORI8 : PPC::XORI), ShiftReg)
-        .addReg(Shift1Reg).addImm(is8bit ? 24 : 16);
+    BuildMI(BB, dl, TII->get(PPC::XORI), ShiftReg)
+        .addReg(Shift1Reg)
+        .addImm(is8bit ? 24 : 16);
   if (is64bit)
     BuildMI(BB, dl, TII->get(PPC::RLDICR), PtrReg)
-      .addReg(Ptr1Reg).addImm(0).addImm(61);
+        .addReg(Ptr1Reg)
+        .addImm(0)
+        .addImm(61);
   else
     BuildMI(BB, dl, TII->get(PPC::RLWINM), PtrReg)
-      .addReg(Ptr1Reg).addImm(0).addImm(0).addImm(29);
-  BuildMI(BB, dl, TII->get(PPC::SLW), Incr2Reg)
-      .addReg(incr).addReg(ShiftReg);
+        .addReg(Ptr1Reg)
+        .addImm(0)
+        .addImm(0)
+        .addImm(29);
+  BuildMI(BB, dl, TII->get(PPC::SLW), Incr2Reg).addReg(incr).addReg(ShiftReg);
   if (is8bit)
     BuildMI(BB, dl, TII->get(PPC::LI), Mask2Reg).addImm(255);
   else {
     BuildMI(BB, dl, TII->get(PPC::LI), Mask3Reg).addImm(0);
-    BuildMI(BB, dl, TII->get(PPC::ORI),Mask2Reg).addReg(Mask3Reg).addImm(65535);
+    BuildMI(BB, dl, TII->get(PPC::ORI), Mask2Reg)
+        .addReg(Mask3Reg)
+        .addImm(65535);
   }
   BuildMI(BB, dl, TII->get(PPC::SLW), MaskReg)
-      .addReg(Mask2Reg).addReg(ShiftReg);
+      .addReg(Mask2Reg)
+      .addReg(ShiftReg);
 
   BB = loopMBB;
   BuildMI(BB, dl, TII->get(PPC::LWARX), TmpDestReg)
-    .addReg(ZeroReg).addReg(PtrReg);
+      .addReg(ZeroReg)
+      .addReg(PtrReg);
   if (BinOpcode)
     BuildMI(BB, dl, TII->get(BinOpcode), TmpReg)
-      .addReg(Incr2Reg).addReg(TmpDestReg);
-  BuildMI(BB, dl, TII->get(is64bit ? PPC::ANDC8 : PPC::ANDC), Tmp2Reg)
-    .addReg(TmpDestReg).addReg(MaskReg);
-  BuildMI(BB, dl, TII->get(is64bit ? PPC::AND8 : PPC::AND), Tmp3Reg)
-    .addReg(TmpReg).addReg(MaskReg);
+        .addReg(Incr2Reg)
+        .addReg(TmpDestReg);
+  BuildMI(BB, dl, TII->get(PPC::ANDC), Tmp2Reg)
+      .addReg(TmpDestReg)
+      .addReg(MaskReg);
+  BuildMI(BB, dl, TII->get(PPC::AND), Tmp3Reg).addReg(TmpReg).addReg(MaskReg);
   if (CmpOpcode) {
     // For unsigned comparisons, we can directly compare the shifted values.
     // For signed comparisons we shift and sign extend.
-    unsigned SReg = RegInfo.createVirtualRegister(RC);
-    BuildMI(BB, dl, TII->get(is64bit ? PPC::AND8 : PPC::AND), SReg)
-      .addReg(TmpDestReg).addReg(MaskReg);
+    unsigned SReg = RegInfo.createVirtualRegister(GPRC);
+    BuildMI(BB, dl, TII->get(PPC::AND), SReg)
+        .addReg(TmpDestReg)
+        .addReg(MaskReg);
     unsigned ValueReg = SReg;
     unsigned CmpReg = Incr2Reg;
     if (CmpOpcode == PPC::CMPW) {
-      ValueReg = RegInfo.createVirtualRegister(RC);
+      ValueReg = RegInfo.createVirtualRegister(GPRC);
       BuildMI(BB, dl, TII->get(PPC::SRW), ValueReg)
-        .addReg(SReg).addReg(ShiftReg);
-      unsigned ValueSReg = RegInfo.createVirtualRegister(RC);
+          .addReg(SReg)
+          .addReg(ShiftReg);
+      unsigned ValueSReg = RegInfo.createVirtualRegister(GPRC);
       BuildMI(BB, dl, TII->get(is8bit ? PPC::EXTSB : PPC::EXTSH), ValueSReg)
-        .addReg(ValueReg);
+          .addReg(ValueReg);
       ValueReg = ValueSReg;
       CmpReg = incr;
     }
     BuildMI(BB, dl, TII->get(CmpOpcode), PPC::CR0)
-      .addReg(CmpReg).addReg(ValueReg);
+        .addReg(CmpReg)
+        .addReg(ValueReg);
     BuildMI(BB, dl, TII->get(PPC::BCC))
-      .addImm(CmpPred).addReg(PPC::CR0).addMBB(exitMBB);
+        .addImm(CmpPred)
+        .addReg(PPC::CR0)
+        .addMBB(exitMBB);
     BB->addSuccessor(loop2MBB);
     BB->addSuccessor(exitMBB);
     BB = loop2MBB;
   }
-  BuildMI(BB, dl, TII->get(is64bit ? PPC::OR8 : PPC::OR), Tmp4Reg)
-    .addReg(Tmp3Reg).addReg(Tmp2Reg);
+  BuildMI(BB, dl, TII->get(PPC::OR), Tmp4Reg).addReg(Tmp3Reg).addReg(Tmp2Reg);
   BuildMI(BB, dl, TII->get(PPC::STWCX))
-    .addReg(Tmp4Reg).addReg(ZeroReg).addReg(PtrReg);
+      .addReg(Tmp4Reg)
+      .addReg(ZeroReg)
+      .addReg(PtrReg);
   BuildMI(BB, dl, TII->get(PPC::BCC))
-    .addImm(PPC::PRED_NE).addReg(PPC::CR0).addMBB(loopMBB);
+      .addImm(PPC::PRED_NE)
+      .addReg(PPC::CR0)
+      .addMBB(loopMBB);
   BB->addSuccessor(loopMBB);
   BB->addSuccessor(exitMBB);
 
   //  exitMBB:
   //   ...
   BB = exitMBB;
-  BuildMI(*BB, BB->begin(), dl, TII->get(PPC::SRW), dest).addReg(TmpDestReg)
-    .addReg(ShiftReg);
+  BuildMI(*BB, BB->begin(), dl, TII->get(PPC::SRW), dest)
+      .addReg(TmpDestReg)
+      .addReg(ShiftReg);
   return BB;
 }
 
@@ -10306,7 +10329,6 @@ PPCTargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
       // way to mark the dependence as implicit there, and so the stackmap code
       // will confuse it with a regular operand. Instead, add the dependence
       // here.
-      setUsesTOCBasePtr(*BB->getParent());
       MI.addOperand(MachineOperand::CreateReg(PPC::X2, false, true));
     }
 
@@ -10331,8 +10353,8 @@ PPCTargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
   MachineFunction *F = BB->getParent();
 
   if (MI.getOpcode() == PPC::SELECT_CC_I4 ||
-       MI.getOpcode() == PPC::SELECT_CC_I8 ||
-       MI.getOpcode() == PPC::SELECT_I4 || MI.getOpcode() == PPC::SELECT_I8) {
+      MI.getOpcode() == PPC::SELECT_CC_I8 || MI.getOpcode() == PPC::SELECT_I4 ||
+      MI.getOpcode() == PPC::SELECT_I8) {
     SmallVector<MachineOperand, 2> Cond;
     if (MI.getOpcode() == PPC::SELECT_CC_I4 ||
         MI.getOpcode() == PPC::SELECT_CC_I8)
@@ -10477,9 +10499,12 @@ PPCTargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
     unsigned CmpReg = RegInfo.createVirtualRegister(&PPC::CRRCRegClass);
 
     BuildMI(BB, dl, TII->get(PPC::CMPW), CmpReg)
-      .addReg(HiReg).addReg(ReadAgainReg);
+        .addReg(HiReg)
+        .addReg(ReadAgainReg);
     BuildMI(BB, dl, TII->get(PPC::BCC))
-      .addImm(PPC::PRED_NE).addReg(CmpReg).addMBB(readMBB);
+        .addImm(PPC::PRED_NE)
+        .addReg(CmpReg)
+        .addMBB(readMBB);
 
     BB->addSuccessor(readMBB);
     BB->addSuccessor(sinkMBB);
@@ -10649,27 +10674,35 @@ PPCTargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
     //   st[bhwd]cx. dest, ptr
     // exitBB:
     BB = loop1MBB;
-    BuildMI(BB, dl, TII->get(LoadMnemonic), dest)
-      .addReg(ptrA).addReg(ptrB);
+    BuildMI(BB, dl, TII->get(LoadMnemonic), dest).addReg(ptrA).addReg(ptrB);
     BuildMI(BB, dl, TII->get(is64bit ? PPC::CMPD : PPC::CMPW), PPC::CR0)
-      .addReg(oldval).addReg(dest);
+        .addReg(oldval)
+        .addReg(dest);
     BuildMI(BB, dl, TII->get(PPC::BCC))
-      .addImm(PPC::PRED_NE).addReg(PPC::CR0).addMBB(midMBB);
+        .addImm(PPC::PRED_NE)
+        .addReg(PPC::CR0)
+        .addMBB(midMBB);
     BB->addSuccessor(loop2MBB);
     BB->addSuccessor(midMBB);
 
     BB = loop2MBB;
     BuildMI(BB, dl, TII->get(StoreMnemonic))
-      .addReg(newval).addReg(ptrA).addReg(ptrB);
+        .addReg(newval)
+        .addReg(ptrA)
+        .addReg(ptrB);
     BuildMI(BB, dl, TII->get(PPC::BCC))
-      .addImm(PPC::PRED_NE).addReg(PPC::CR0).addMBB(loop1MBB);
+        .addImm(PPC::PRED_NE)
+        .addReg(PPC::CR0)
+        .addMBB(loop1MBB);
     BuildMI(BB, dl, TII->get(PPC::B)).addMBB(exitMBB);
     BB->addSuccessor(loop1MBB);
     BB->addSuccessor(exitMBB);
 
     BB = midMBB;
     BuildMI(BB, dl, TII->get(StoreMnemonic))
-      .addReg(dest).addReg(ptrA).addReg(ptrB);
+        .addReg(dest)
+        .addReg(ptrA)
+        .addReg(ptrB);
     BB->addSuccessor(exitMBB);
 
     //  exitMBB:
@@ -10704,24 +10737,26 @@ PPCTargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
     exitMBB->transferSuccessorsAndUpdatePHIs(BB);
 
     MachineRegisterInfo &RegInfo = F->getRegInfo();
-    const TargetRegisterClass *RC = is64bit ? &PPC::G8RCRegClass
-                                            : &PPC::GPRCRegClass;
+    const TargetRegisterClass *RC =
+        is64bit ? &PPC::G8RCRegClass : &PPC::GPRCRegClass;
+    const TargetRegisterClass *GPRC = &PPC::GPRCRegClass;
+
     unsigned PtrReg = RegInfo.createVirtualRegister(RC);
-    unsigned Shift1Reg = RegInfo.createVirtualRegister(RC);
+    unsigned Shift1Reg = RegInfo.createVirtualRegister(GPRC);
     unsigned ShiftReg =
-      isLittleEndian ? Shift1Reg : RegInfo.createVirtualRegister(RC);
-    unsigned NewVal2Reg = RegInfo.createVirtualRegister(RC);
-    unsigned NewVal3Reg = RegInfo.createVirtualRegister(RC);
-    unsigned OldVal2Reg = RegInfo.createVirtualRegister(RC);
-    unsigned OldVal3Reg = RegInfo.createVirtualRegister(RC);
-    unsigned MaskReg = RegInfo.createVirtualRegister(RC);
-    unsigned Mask2Reg = RegInfo.createVirtualRegister(RC);
-    unsigned Mask3Reg = RegInfo.createVirtualRegister(RC);
-    unsigned Tmp2Reg = RegInfo.createVirtualRegister(RC);
-    unsigned Tmp4Reg = RegInfo.createVirtualRegister(RC);
-    unsigned TmpDestReg = RegInfo.createVirtualRegister(RC);
+        isLittleEndian ? Shift1Reg : RegInfo.createVirtualRegister(GPRC);
+    unsigned NewVal2Reg = RegInfo.createVirtualRegister(GPRC);
+    unsigned NewVal3Reg = RegInfo.createVirtualRegister(GPRC);
+    unsigned OldVal2Reg = RegInfo.createVirtualRegister(GPRC);
+    unsigned OldVal3Reg = RegInfo.createVirtualRegister(GPRC);
+    unsigned MaskReg = RegInfo.createVirtualRegister(GPRC);
+    unsigned Mask2Reg = RegInfo.createVirtualRegister(GPRC);
+    unsigned Mask3Reg = RegInfo.createVirtualRegister(GPRC);
+    unsigned Tmp2Reg = RegInfo.createVirtualRegister(GPRC);
+    unsigned Tmp4Reg = RegInfo.createVirtualRegister(GPRC);
+    unsigned TmpDestReg = RegInfo.createVirtualRegister(GPRC);
     unsigned Ptr1Reg;
-    unsigned TmpReg = RegInfo.createVirtualRegister(RC);
+    unsigned TmpReg = RegInfo.createVirtualRegister(GPRC);
     unsigned ZeroReg = is64bit ? PPC::ZERO8 : PPC::ZERO;
     //  thisMBB:
     //   ...
@@ -10758,74 +10793,107 @@ PPCTargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
     if (ptrA != ZeroReg) {
       Ptr1Reg = RegInfo.createVirtualRegister(RC);
       BuildMI(BB, dl, TII->get(is64bit ? PPC::ADD8 : PPC::ADD4), Ptr1Reg)
-        .addReg(ptrA).addReg(ptrB);
+          .addReg(ptrA)
+          .addReg(ptrB);
     } else {
       Ptr1Reg = ptrB;
     }
-    BuildMI(BB, dl, TII->get(PPC::RLWINM), Shift1Reg).addReg(Ptr1Reg)
-        .addImm(3).addImm(27).addImm(is8bit ? 28 : 27);
+
+    // We need use 32-bit subregister to avoid mismatch register class in 64-bit
+    // mode.
+    BuildMI(BB, dl, TII->get(PPC::RLWINM), Shift1Reg)
+        .addReg(Ptr1Reg, 0, is64bit ? PPC::sub_32 : 0)
+        .addImm(3)
+        .addImm(27)
+        .addImm(is8bit ? 28 : 27);
     if (!isLittleEndian)
-      BuildMI(BB, dl, TII->get(is64bit ? PPC::XORI8 : PPC::XORI), ShiftReg)
-          .addReg(Shift1Reg).addImm(is8bit ? 24 : 16);
+      BuildMI(BB, dl, TII->get(PPC::XORI), ShiftReg)
+          .addReg(Shift1Reg)
+          .addImm(is8bit ? 24 : 16);
     if (is64bit)
       BuildMI(BB, dl, TII->get(PPC::RLDICR), PtrReg)
-        .addReg(Ptr1Reg).addImm(0).addImm(61);
+          .addReg(Ptr1Reg)
+          .addImm(0)
+          .addImm(61);
     else
       BuildMI(BB, dl, TII->get(PPC::RLWINM), PtrReg)
-        .addReg(Ptr1Reg).addImm(0).addImm(0).addImm(29);
+          .addReg(Ptr1Reg)
+          .addImm(0)
+          .addImm(0)
+          .addImm(29);
     BuildMI(BB, dl, TII->get(PPC::SLW), NewVal2Reg)
-        .addReg(newval).addReg(ShiftReg);
+        .addReg(newval)
+        .addReg(ShiftReg);
     BuildMI(BB, dl, TII->get(PPC::SLW), OldVal2Reg)
-        .addReg(oldval).addReg(ShiftReg);
+        .addReg(oldval)
+        .addReg(ShiftReg);
     if (is8bit)
       BuildMI(BB, dl, TII->get(PPC::LI), Mask2Reg).addImm(255);
     else {
       BuildMI(BB, dl, TII->get(PPC::LI), Mask3Reg).addImm(0);
       BuildMI(BB, dl, TII->get(PPC::ORI), Mask2Reg)
-        .addReg(Mask3Reg).addImm(65535);
+          .addReg(Mask3Reg)
+          .addImm(65535);
     }
     BuildMI(BB, dl, TII->get(PPC::SLW), MaskReg)
-        .addReg(Mask2Reg).addReg(ShiftReg);
+        .addReg(Mask2Reg)
+        .addReg(ShiftReg);
     BuildMI(BB, dl, TII->get(PPC::AND), NewVal3Reg)
-        .addReg(NewVal2Reg).addReg(MaskReg);
+        .addReg(NewVal2Reg)
+        .addReg(MaskReg);
     BuildMI(BB, dl, TII->get(PPC::AND), OldVal3Reg)
-        .addReg(OldVal2Reg).addReg(MaskReg);
+        .addReg(OldVal2Reg)
+        .addReg(MaskReg);
 
     BB = loop1MBB;
     BuildMI(BB, dl, TII->get(PPC::LWARX), TmpDestReg)
-        .addReg(ZeroReg).addReg(PtrReg);
-    BuildMI(BB, dl, TII->get(PPC::AND),TmpReg)
-        .addReg(TmpDestReg).addReg(MaskReg);
+        .addReg(ZeroReg)
+        .addReg(PtrReg);
+    BuildMI(BB, dl, TII->get(PPC::AND), TmpReg)
+        .addReg(TmpDestReg)
+        .addReg(MaskReg);
     BuildMI(BB, dl, TII->get(PPC::CMPW), PPC::CR0)
-        .addReg(TmpReg).addReg(OldVal3Reg);
+        .addReg(TmpReg)
+        .addReg(OldVal3Reg);
     BuildMI(BB, dl, TII->get(PPC::BCC))
-        .addImm(PPC::PRED_NE).addReg(PPC::CR0).addMBB(midMBB);
+        .addImm(PPC::PRED_NE)
+        .addReg(PPC::CR0)
+        .addMBB(midMBB);
     BB->addSuccessor(loop2MBB);
     BB->addSuccessor(midMBB);
 
     BB = loop2MBB;
-    BuildMI(BB, dl, TII->get(PPC::ANDC),Tmp2Reg)
-        .addReg(TmpDestReg).addReg(MaskReg);
-    BuildMI(BB, dl, TII->get(PPC::OR),Tmp4Reg)
-        .addReg(Tmp2Reg).addReg(NewVal3Reg);
-    BuildMI(BB, dl, TII->get(PPC::STWCX)).addReg(Tmp4Reg)
-        .addReg(ZeroReg).addReg(PtrReg);
+    BuildMI(BB, dl, TII->get(PPC::ANDC), Tmp2Reg)
+        .addReg(TmpDestReg)
+        .addReg(MaskReg);
+    BuildMI(BB, dl, TII->get(PPC::OR), Tmp4Reg)
+        .addReg(Tmp2Reg)
+        .addReg(NewVal3Reg);
+    BuildMI(BB, dl, TII->get(PPC::STWCX))
+        .addReg(Tmp4Reg)
+        .addReg(ZeroReg)
+        .addReg(PtrReg);
     BuildMI(BB, dl, TII->get(PPC::BCC))
-      .addImm(PPC::PRED_NE).addReg(PPC::CR0).addMBB(loop1MBB);
+        .addImm(PPC::PRED_NE)
+        .addReg(PPC::CR0)
+        .addMBB(loop1MBB);
     BuildMI(BB, dl, TII->get(PPC::B)).addMBB(exitMBB);
     BB->addSuccessor(loop1MBB);
     BB->addSuccessor(exitMBB);
 
     BB = midMBB;
-    BuildMI(BB, dl, TII->get(PPC::STWCX)).addReg(TmpDestReg)
-      .addReg(ZeroReg).addReg(PtrReg);
+    BuildMI(BB, dl, TII->get(PPC::STWCX))
+        .addReg(TmpDestReg)
+        .addReg(ZeroReg)
+        .addReg(PtrReg);
     BB->addSuccessor(exitMBB);
 
     //  exitMBB:
     //   ...
     BB = exitMBB;
-    BuildMI(*BB, BB->begin(), dl, TII->get(PPC::SRW),dest).addReg(TmpReg)
-      .addReg(ShiftReg);
+    BuildMI(*BB, BB->begin(), dl, TII->get(PPC::SRW), dest)
+        .addReg(TmpReg)
+        .addReg(ShiftReg);
   } else if (MI.getOpcode() == PPC::FADDrtz) {
     // This pseudo performs an FADD with rounding mode temporarily forced
     // to round-to-zero.  We emit this via custom inserter since the FPSCR
@@ -10862,9 +10930,8 @@ PPCTargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
                  MI.getOpcode() == PPC::ANDIo_1_EQ_BIT8);
 
     MachineRegisterInfo &RegInfo = F->getRegInfo();
-    unsigned Dest = RegInfo.createVirtualRegister(Opcode == PPC::ANDIo ?
-                                                  &PPC::GPRCRegClass :
-                                                  &PPC::G8RCRegClass);
+    unsigned Dest = RegInfo.createVirtualRegister(
+        Opcode == PPC::ANDIo ? &PPC::GPRCRegClass : &PPC::G8RCRegClass);
 
     DebugLoc dl = MI.getDebugLoc();
     BuildMI(*BB, MI, dl, TII->get(Opcode), Dest)
@@ -11316,9 +11383,8 @@ SDValue PPCTargetLowering::DAGCombineTruncBoolExt(SDNode *N,
     } else {
       // This is neither a signed nor an unsigned comparison, just make sure
       // that the high bits are equal.
-      KnownBits Op1Known, Op2Known;
-      DAG.computeKnownBits(N->getOperand(0), Op1Known);
-      DAG.computeKnownBits(N->getOperand(1), Op2Known);
+      KnownBits Op1Known = DAG.computeKnownBits(N->getOperand(0));
+      KnownBits Op2Known = DAG.computeKnownBits(N->getOperand(1));
 
       // We don't really care about what is known about the first bit (if
       // anything), so clear it in all masks prior to comparing them.
@@ -14392,7 +14458,7 @@ static SDValue combineADDToADDZE(SDNode *N, SelectionDAG &DAG,
     return SDValue();
 
   SDLoc DL(N);
-  SDVTList VTs = DAG.getVTList(MVT::i64, MVT::i64);
+  SDVTList VTs = DAG.getVTList(MVT::i64, MVT::Glue);
   SDValue Cmp = RHS.getOperand(0);
   SDValue Z = Cmp.getOperand(0);
   auto *Constant = dyn_cast<ConstantSDNode>(Cmp.getOperand(1));
diff --git a/lib/Target/PowerPC/PPCISelLowering.h b/lib/Target/PowerPC/PPCISelLowering.h
index 09039cb77363d909561ccea7d3c6098d5c5e7d94..456e61345c212eeb9cf00763c8ccd1faf4c7ada1 100644
--- a/lib/Target/PowerPC/PPCISelLowering.h
+++ b/lib/Target/PowerPC/PPCISelLowering.h
@@ -1,9 +1,8 @@
 //===-- PPCISelLowering.h - PPC32 DAG Lowering Interface --------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -576,6 +575,11 @@ namespace llvm {
     /// DAG node.
     const char *getTargetNodeName(unsigned Opcode) const override;
 
+    bool isSelectSupported(SelectSupportKind Kind) const override {
+      // PowerPC does not support scalar condition selects on vectors.
+      return (Kind != SelectSupportKind::ScalarCondVectorVal);
+    }
+
     /// getPreferredVectorAction - The code we generate when vector types are
     /// legalized by promoting the integer element type is often much worse
     /// than code we generate if we widen the type for applicable vector types.
@@ -948,6 +952,8 @@ namespace llvm {
     SDValue LowerINT_TO_FPVector(SDValue Op, SelectionDAG &DAG,
                                  const SDLoc &dl) const;
 
+    SDValue LowerTRUNCATEVector(SDValue Op, SelectionDAG &DAG) const;
+
     SDValue getFramePointerFrameIndex(SelectionDAG & DAG) const;
     SDValue getReturnAddrFrameIndex(SelectionDAG & DAG) const;
 
@@ -1132,8 +1138,6 @@ namespace llvm {
                              int &RefinementSteps) const override;
     unsigned combineRepeatedFPDivisors() const override;
 
-    CCAssignFn *useFastISelCCs(unsigned Flag) const;
-
     SDValue
     combineElementTruncationToVectorTruncation(SDNode *N,
                                                DAGCombinerInfo &DCI) const;
@@ -1164,30 +1168,6 @@ namespace llvm {
 
   } // end namespace PPC
 
-  bool CC_PPC32_SVR4_Custom_Dummy(unsigned &ValNo, MVT &ValVT, MVT &LocVT,
-                                  CCValAssign::LocInfo &LocInfo,
-                                  ISD::ArgFlagsTy &ArgFlags,
-                                  CCState &State);
-
-  bool CC_PPC32_SVR4_Custom_AlignArgRegs(unsigned &ValNo, MVT &ValVT,
-                                         MVT &LocVT,
-                                         CCValAssign::LocInfo &LocInfo,
-                                         ISD::ArgFlagsTy &ArgFlags,
-                                         CCState &State);
-
-  bool
-  CC_PPC32_SVR4_Custom_SkipLastArgRegsPPCF128(unsigned &ValNo, MVT &ValVT,
-                                                 MVT &LocVT,
-                                                 CCValAssign::LocInfo &LocInfo,
-                                                 ISD::ArgFlagsTy &ArgFlags,
-                                                 CCState &State);
-
-  bool CC_PPC32_SVR4_Custom_AlignFPArgRegs(unsigned &ValNo, MVT &ValVT,
-                                           MVT &LocVT,
-                                           CCValAssign::LocInfo &LocInfo,
-                                           ISD::ArgFlagsTy &ArgFlags,
-                                           CCState &State);
-
   bool isIntS16Immediate(SDNode *N, int16_t &Imm);
   bool isIntS16Immediate(SDValue Op, int16_t &Imm);
 
diff --git a/lib/Target/PowerPC/PPCInstr64Bit.td b/lib/Target/PowerPC/PPCInstr64Bit.td
index 2ce6ad3293eb1bc0644ecbef8b9176d7f01c5655..1b362f435da960358e4802dab0d8b5af1e34a5aa 100644
--- a/lib/Target/PowerPC/PPCInstr64Bit.td
+++ b/lib/Target/PowerPC/PPCInstr64Bit.td
@@ -1,9 +1,8 @@
 //===-- PPCInstr64Bit.td - The PowerPC 64-bit Support ------*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/PowerPC/PPCInstrAltivec.td b/lib/Target/PowerPC/PPCInstrAltivec.td
index 69b19e45c3e9ccb9c118d210d1557ae1f93435ce..f34cea0babe44593ba783e7db0a10fd70e3911e0 100644
--- a/lib/Target/PowerPC/PPCInstrAltivec.td
+++ b/lib/Target/PowerPC/PPCInstrAltivec.td
@@ -1,9 +1,8 @@
 //===-- PPCInstrAltivec.td - The PowerPC Altivec Extension -*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/PowerPC/PPCInstrBuilder.h b/lib/Target/PowerPC/PPCInstrBuilder.h
index cf71b1c5986958b46c5a1bbfa8fcf2a0123aac9f..323f7e39adf7e766e66503aeceed42846f9e975f 100644
--- a/lib/Target/PowerPC/PPCInstrBuilder.h
+++ b/lib/Target/PowerPC/PPCInstrBuilder.h
@@ -1,9 +1,8 @@
 //===-- PPCInstrBuilder.h - Aides for building PPC insts --------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/PowerPC/PPCInstrFormats.td b/lib/Target/PowerPC/PPCInstrFormats.td
index 2fe765dd99e1e90fdd104f6fcdbf737486cddfe1..7efe9160faceeb2f63088d17e8e6d3cfbc8b3e6f 100644
--- a/lib/Target/PowerPC/PPCInstrFormats.td
+++ b/lib/Target/PowerPC/PPCInstrFormats.td
@@ -1,9 +1,8 @@
 //===- PowerPCInstrFormats.td - PowerPC Instruction Formats --*- tablegen -*-=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/PowerPC/PPCInstrHTM.td b/lib/Target/PowerPC/PPCInstrHTM.td
index 0efe797c765daa7107c0fdf5e1a2a80aa925c705..7a07b4259150b1e464e6cafe97883889f12d72ac 100644
--- a/lib/Target/PowerPC/PPCInstrHTM.td
+++ b/lib/Target/PowerPC/PPCInstrHTM.td
@@ -1,9 +1,8 @@
 //===-- PPCInstrHTM.td - The PowerPC Hardware Transactional Memory  -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/PowerPC/PPCInstrInfo.cpp b/lib/Target/PowerPC/PPCInstrInfo.cpp
index d26af451303b6368e0e513033860d47ffa31b104..5952d73d205bc203ef983d019e312a6aafa61047 100644
--- a/lib/Target/PowerPC/PPCInstrInfo.cpp
+++ b/lib/Target/PowerPC/PPCInstrInfo.cpp
@@ -1,9 +1,8 @@
 //===-- PPCInstrInfo.cpp - PowerPC Instruction Information ----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -2248,6 +2247,35 @@ static unsigned selectReg(int64_t Imm1, int64_t Imm2, unsigned CompareOpc,
   return PPC::NoRegister;
 }
 
+void PPCInstrInfo::replaceInstrOperandWithImm(MachineInstr &MI,
+                                              unsigned OpNo,
+                                              int64_t Imm) const {
+  assert(MI.getOperand(OpNo).isReg() && "Operand must be a REG");
+  // Replace the REG with the Immediate.
+  unsigned InUseReg = MI.getOperand(OpNo).getReg();
+  MI.getOperand(OpNo).ChangeToImmediate(Imm);
+
+  if (empty(MI.implicit_operands()))
+    return;
+
+  // We need to make sure that the MI didn't have any implicit use
+  // of this REG any more.
+  const TargetRegisterInfo *TRI = &getRegisterInfo();
+  int UseOpIdx = MI.findRegisterUseOperandIdx(InUseReg, false, TRI);
+  if (UseOpIdx >= 0) {
+    MachineOperand &MO = MI.getOperand(UseOpIdx);
+    if (MO.isImplicit())
+      // The operands must always be in the following order:
+      // - explicit reg defs,
+      // - other explicit operands (reg uses, immediates, etc.),
+      // - implicit reg defs
+      // - implicit reg uses
+      // Therefore, removing the implicit operand won't change the explicit
+      // operands layout.
+      MI.RemoveOperand(UseOpIdx);
+  }
+}
+
 // Replace an instruction with one that materializes a constant (and sets
 // CR0 if the original instruction was a record-form instruction).
 void PPCInstrInfo::replaceInstrWithLI(MachineInstr &MI,
@@ -2329,13 +2357,6 @@ MachineInstr *PPCInstrInfo::getForwardingDefMI(
         MachineBasicBlock::reverse_iterator E = MI.getParent()->rend(), It = MI;
         It++;
         unsigned Reg = MI.getOperand(i).getReg();
-        // MachineInstr::readsRegister only returns true if the machine
-        // instruction reads the exact register or its super-register. It
-        // does not consider uses of sub-registers which seems like strange
-        // behaviour. Nonetheless, if we end up with a 64-bit register here,
-        // get the corresponding 32-bit register to check.
-        if (PPC::G8RCRegClass.contains(Reg))
-          Reg = Reg - PPC::X0 + PPC::R0;
 
         // Is this register defined by some form of add-immediate (including
         // load-immediate) within this basic block?
@@ -2481,7 +2502,7 @@ bool PPCInstrInfo::convertToImmediateForm(MachineInstr &MI,
       // Can't use PPC::COPY to copy PPC::ZERO[8]. Convert it to LI[8] 0.
       if (RegToCopy == PPC::ZERO || RegToCopy == PPC::ZERO8) {
         CompareUseMI.setDesc(get(UseOpc == PPC::ISEL8 ? PPC::LI8 : PPC::LI));
-        CompareUseMI.getOperand(1).ChangeToImmediate(0);
+        replaceInstrOperandWithImm(CompareUseMI, 1, 0);
         CompareUseMI.RemoveOperand(3);
         CompareUseMI.RemoveOperand(2);
         continue;
@@ -3155,14 +3176,7 @@ bool PPCInstrInfo::isRegElgibleForForwarding(const MachineOperand &RegMO,
   if (MRI.isSSA())
     return false;
 
-  // MachineInstr::readsRegister only returns true if the machine
-  // instruction reads the exact register or its super-register. It
-  // does not consider uses of sub-registers which seems like strange
-  // behaviour. Nonetheless, if we end up with a 64-bit register here,
-  // get the corresponding 32-bit register to check.
   unsigned Reg = RegMO.getReg();
-  if (PPC::G8RCRegClass.contains(Reg))
-    Reg = Reg - PPC::X0 + PPC::R0;
 
   // Walking the inst in reverse(MI-->DefMI) to get the last DEF of the Reg.
   MachineBasicBlock::const_reverse_iterator It = MI;
@@ -3177,9 +3191,9 @@ bool PPCInstrInfo::isRegElgibleForForwarding(const MachineOperand &RegMO,
   }
   assert((&*It) == &DefMI && "DefMI is missing");
 
-  // If DefMI also uses the register to be forwarded, we can only forward it
+  // If DefMI also defines the register to be forwarded, we can only forward it
   // if DefMI is being erased.
-  if (DefMI.readsRegister(Reg, &getRegisterInfo()))
+  if (DefMI.modifiesRegister(Reg, &getRegisterInfo()))
     return KillDefMI;
 
   return true;
@@ -3292,7 +3306,7 @@ bool PPCInstrInfo::transformToImmFormFedByAdd(MachineInstr &MI,
   if (ImmMO->isImm()) {
     // If the ImmMO is Imm, change the operand that has ZERO to that Imm
     // directly.
-    MI.getOperand(III.ZeroIsSpecialOrig).ChangeToImmediate(Imm);
+    replaceInstrOperandWithImm(MI, III.ZeroIsSpecialOrig, Imm);
   }
   else {
     // Otherwise, it is Constant Pool Index(CPI) or Global,
@@ -3411,24 +3425,24 @@ bool PPCInstrInfo::transformToImmFormFedByLI(MachineInstr &MI,
           uint64_t SH = RightShift ? 32 - ShAmt : ShAmt;
           uint64_t MB = RightShift ? ShAmt : 0;
           uint64_t ME = RightShift ? 31 : 31 - ShAmt;
-          MI.getOperand(III.OpNoForForwarding).ChangeToImmediate(SH);
+          replaceInstrOperandWithImm(MI, III.OpNoForForwarding, SH);
           MachineInstrBuilder(*MI.getParent()->getParent(), MI).addImm(MB)
             .addImm(ME);
         } else {
           // Left shifts use (N, 63-N), right shifts use (64-N, N).
           uint64_t SH = RightShift ? 64 - ShAmt : ShAmt;
           uint64_t ME = RightShift ? ShAmt : 63 - ShAmt;
-          MI.getOperand(III.OpNoForForwarding).ChangeToImmediate(SH);
+          replaceInstrOperandWithImm(MI, III.OpNoForForwarding, SH);
           MachineInstrBuilder(*MI.getParent()->getParent(), MI).addImm(ME);
         }
       }
     } else
-      MI.getOperand(ConstantOpNo).ChangeToImmediate(Imm);
+      replaceInstrOperandWithImm(MI, ConstantOpNo, Imm);
   }
   // Convert commutative instructions (switch the operands and convert the
   // desired one to an immediate.
   else if (III.IsCommutative) {
-    MI.getOperand(ConstantOpNo).ChangeToImmediate(Imm);
+    replaceInstrOperandWithImm(MI, ConstantOpNo, Imm);
     swapMIOperands(MI, ConstantOpNo, III.OpNoForForwarding);
   } else
     llvm_unreachable("Should have exited early!");
@@ -3438,15 +3452,20 @@ bool PPCInstrInfo::transformToImmFormFedByLI(MachineInstr &MI,
   if (III.OpNoForForwarding != III.ImmOpNo)
     swapMIOperands(MI, III.OpNoForForwarding, III.ImmOpNo);
 
-  // If the R0/X0 register is special for the original instruction and not for
-  // the new instruction (or vice versa), we need to fix up the register class.
+  // If the special R0/X0 register index are different for original instruction
+  // and new instruction, we need to fix up the register class in new
+  // instruction.
   if (!PostRA && III.ZeroIsSpecialOrig != III.ZeroIsSpecialNew) {
-    if (!III.ZeroIsSpecialOrig) {
+    if (III.ZeroIsSpecialNew) {
+      // If operand at III.ZeroIsSpecialNew is physical reg(eg: ZERO/ZERO8), no
+      // need to fix up register class.
       unsigned RegToModify = MI.getOperand(III.ZeroIsSpecialNew).getReg();
-      const TargetRegisterClass *NewRC =
-        MRI.getRegClass(RegToModify)->hasSuperClassEq(&PPC::GPRCRegClass) ?
-        &PPC::GPRC_and_GPRC_NOR0RegClass : &PPC::G8RC_and_G8RC_NOX0RegClass;
-      MRI.setRegClass(RegToModify, NewRC);
+      if (TargetRegisterInfo::isVirtualRegister(RegToModify)) {
+        const TargetRegisterClass *NewRC =
+          MRI.getRegClass(RegToModify)->hasSuperClassEq(&PPC::GPRCRegClass) ?
+          &PPC::GPRC_and_GPRC_NOR0RegClass : &PPC::G8RC_and_G8RC_NOX0RegClass;
+        MRI.setRegClass(RegToModify, NewRC);
+      }
     }
   }
   return true;
diff --git a/lib/Target/PowerPC/PPCInstrInfo.h b/lib/Target/PowerPC/PPCInstrInfo.h
index 9c556e32496a4a74d88c4d06292efadd9964963c..ee16d439d9e4d8deda176c9b14a1e0df63d1b585 100644
--- a/lib/Target/PowerPC/PPCInstrInfo.h
+++ b/lib/Target/PowerPC/PPCInstrInfo.h
@@ -1,9 +1,8 @@
 //===-- PPCInstrInfo.h - PowerPC Instruction Information --------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -413,6 +412,8 @@ public:
   bool convertToImmediateForm(MachineInstr &MI,
                               MachineInstr **KilledDef = nullptr) const;
   void replaceInstrWithLI(MachineInstr &MI, const LoadImmediateInfo &LII) const;
+  void replaceInstrOperandWithImm(MachineInstr &MI, unsigned OpNo,
+                                  int64_t Imm) const;
 
   bool instrHasImmForm(const MachineInstr &MI, ImmInstrInfo &III,
                        bool PostRA) const;
diff --git a/lib/Target/PowerPC/PPCInstrInfo.td b/lib/Target/PowerPC/PPCInstrInfo.td
index dd3f1ac790894961d9550ffd2380c8cbd4f52424..e54292aee286a2b08a649522188c8833e76d78ea 100644
--- a/lib/Target/PowerPC/PPCInstrInfo.td
+++ b/lib/Target/PowerPC/PPCInstrInfo.td
@@ -1,9 +1,8 @@
 //===-- PPCInstrInfo.td - The PowerPC Instruction Set ------*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -737,7 +736,9 @@ def abscondbrtarget : Operand<OtherVT> {
 def calltarget : Operand<iPTR> {
   let PrintMethod = "printBranchOperand";
   let EncoderMethod = "getDirectBrEncoding";
+  let DecoderMethod = "DecodePCRel24BranchTarget";
   let ParserMatchClass = PPCDirectBrAsmOperand;
+  let OperandType = "OPERAND_PCREL";
 }
 def abscalltarget : Operand<iPTR> {
   let PrintMethod = "printAbsBranchOperand";
diff --git a/lib/Target/PowerPC/PPCInstrQPX.td b/lib/Target/PowerPC/PPCInstrQPX.td
index ef589ad01fd7fa9e496178e7fc83be6bd887ff6f..d67041d46d9f04fd32be07d7014182d05915466b 100644
--- a/lib/Target/PowerPC/PPCInstrQPX.td
+++ b/lib/Target/PowerPC/PPCInstrQPX.td
@@ -1,9 +1,8 @@
 //===- PPCInstrQPX.td - The PowerPC QPX Extension --*- tablegen -*-===//
 // 
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 // 
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/PowerPC/PPCInstrSPE.td b/lib/Target/PowerPC/PPCInstrSPE.td
index 9f5891a45f22346e6b4e63e8d91538c131bd4cc8..31093de223fa2bf9c5a1abcd0f6e57f88547f483 100644
--- a/lib/Target/PowerPC/PPCInstrSPE.td
+++ b/lib/Target/PowerPC/PPCInstrSPE.td
@@ -1,9 +1,8 @@
 //=======-- PPCInstrSPE.td - The PowerPC SPE Extension -*- tablegen -*-=======//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/PowerPC/PPCInstrVSX.td b/lib/Target/PowerPC/PPCInstrVSX.td
index 8321d7f2ecf06d0640d7147316f56cfb3cbc3941..8ca2e04835a589acce72208047f00003f180595b 100644
--- a/lib/Target/PowerPC/PPCInstrVSX.td
+++ b/lib/Target/PowerPC/PPCInstrVSX.td
@@ -1,9 +1,8 @@
 //===- PPCInstrVSX.td - The PowerPC VSX Extension --*- tablegen -*-===//
 // 
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 // 
 //===----------------------------------------------------------------------===//
 //
@@ -1217,6 +1216,27 @@ def ScalarLoads {
   dag Li32 = (i32 (load xoaddr:$src));
 }
 
+def DWToSPExtractConv {
+  dag El0US1 = (f32 (PPCfcfidus
+                    (f64 (PPCmtvsra (i64 (vector_extract v2i64:$S1, 0))))));
+  dag El1US1 = (f32 (PPCfcfidus
+                    (f64 (PPCmtvsra (i64 (vector_extract v2i64:$S1, 1))))));
+  dag El0US2 = (f32 (PPCfcfidus
+                    (f64 (PPCmtvsra (i64 (vector_extract v2i64:$S2, 0))))));
+  dag El1US2 = (f32 (PPCfcfidus
+                    (f64 (PPCmtvsra (i64 (vector_extract v2i64:$S2, 1))))));
+  dag El0SS1 = (f32 (PPCfcfids
+                    (f64 (PPCmtvsra (i64 (vector_extract v2i64:$S1, 0))))));
+  dag El1SS1 = (f32 (PPCfcfids
+                    (f64 (PPCmtvsra (i64 (vector_extract v2i64:$S1, 1))))));
+  dag El0SS2 = (f32 (PPCfcfids
+                    (f64 (PPCmtvsra (i64 (vector_extract v2i64:$S2, 0))))));
+  dag El1SS2 = (f32 (PPCfcfids
+                    (f64 (PPCmtvsra (i64 (vector_extract v2i64:$S2, 1))))));
+  dag BVU = (v4f32 (build_vector El0US1, El1US1, El0US2, El1US2));
+  dag BVS = (v4f32 (build_vector El0SS1, El1SS1, El0SS2, El1SS2));
+}
+
 // The following VSX instructions were introduced in Power ISA 2.07
 /* FIXME: if the operands are v2i64, these patterns will not match.
    we should define new patterns or otherwise match the same patterns
@@ -1452,35 +1472,27 @@ let AddedComplexity = 400 in { // Prefer VSX patterns over non-VSX patterns.
   } // UseVSXReg = 1
 
   let Predicates = [IsLittleEndian] in {
-  def : Pat<(f32 (PPCfcfids
-                   (f64 (PPCmtvsra (i64 (vector_extract v2i64:$S, 0)))))),
-            (f32 (XSCVSXDSP (COPY_TO_REGCLASS (XXPERMDI $S, $S, 2), VSFRC)))>;
-  def : Pat<(f32 (PPCfcfids
-                   (f64 (PPCmtvsra (i64 (vector_extract v2i64:$S, 1)))))),
+  def : Pat<DWToSPExtractConv.El0SS1,
+            (f32 (XSCVSXDSP (COPY_TO_REGCLASS (XXPERMDI $S1, $S1, 2), VSFRC)))>;
+  def : Pat<DWToSPExtractConv.El1SS1,
             (f32 (XSCVSXDSP (COPY_TO_REGCLASS
-                              (f64 (COPY_TO_REGCLASS $S, VSRC)), VSFRC)))>;
-  def : Pat<(f32 (PPCfcfidus
-                   (f64 (PPCmtvsra (i64 (vector_extract v2i64:$S, 0)))))),
-            (f32 (XSCVUXDSP (COPY_TO_REGCLASS (XXPERMDI $S, $S, 2), VSFRC)))>;
-  def : Pat<(f32 (PPCfcfidus
-                   (f64 (PPCmtvsra (i64 (vector_extract v2i64:$S, 1)))))),
+                              (f64 (COPY_TO_REGCLASS $S1, VSRC)), VSFRC)))>;
+  def : Pat<DWToSPExtractConv.El0US1,
+            (f32 (XSCVUXDSP (COPY_TO_REGCLASS (XXPERMDI $S1, $S1, 2), VSFRC)))>;
+  def : Pat<DWToSPExtractConv.El1US1,
             (f32 (XSCVUXDSP (COPY_TO_REGCLASS
-                              (f64 (COPY_TO_REGCLASS $S, VSRC)), VSFRC)))>;
+                              (f64 (COPY_TO_REGCLASS $S1, VSRC)), VSFRC)))>;
   }
 
   let Predicates = [IsBigEndian] in {
-  def : Pat<(f32 (PPCfcfids
-                   (f64 (PPCmtvsra (i64 (vector_extract v2i64:$S, 0)))))),
-            (f32 (XSCVSXDSP (COPY_TO_REGCLASS $S, VSFRC)))>;
-  def : Pat<(f32 (PPCfcfids
-                   (f64 (PPCmtvsra (i64 (vector_extract v2i64:$S, 1)))))),
-            (f32 (XSCVSXDSP (COPY_TO_REGCLASS (XXPERMDI $S, $S, 2), VSFRC)))>;
-  def : Pat<(f32 (PPCfcfidus
-                   (f64 (PPCmtvsra (i64 (vector_extract v2i64:$S, 0)))))),
-            (f32 (XSCVUXDSP (COPY_TO_REGCLASS $S, VSFRC)))>;
-  def : Pat<(f32 (PPCfcfidus
-                   (f64 (PPCmtvsra (i64 (vector_extract v2i64:$S, 1)))))),
-            (f32 (XSCVUXDSP (COPY_TO_REGCLASS (XXPERMDI $S, $S, 2), VSFRC)))>;
+  def : Pat<DWToSPExtractConv.El0SS1,
+            (f32 (XSCVSXDSP (COPY_TO_REGCLASS $S1, VSFRC)))>;
+  def : Pat<DWToSPExtractConv.El1SS1,
+            (f32 (XSCVSXDSP (COPY_TO_REGCLASS (XXPERMDI $S1, $S1, 2), VSFRC)))>;
+  def : Pat<DWToSPExtractConv.El0US1,
+            (f32 (XSCVUXDSP (COPY_TO_REGCLASS $S1, VSFRC)))>;
+  def : Pat<DWToSPExtractConv.El1US1,
+            (f32 (XSCVUXDSP (COPY_TO_REGCLASS (XXPERMDI $S1, $S1, 2), VSFRC)))>;
   }
 
   // Instructions for converting float to i64 feeding a store.
@@ -3294,7 +3306,27 @@ let AddedComplexity = 400, Predicates = [HasP9Vector] in {
       def : Pat<(v2f64 (scalar_to_vector (f64 (load xaddr:$src)))),
                 (v2f64 (XXPERMDIs
                 (COPY_TO_REGCLASS (XFLOADf64 xaddr:$src), VSRC), 2))>;
-    }
+      def : Pat<(store (i64 (extractelt v2i64:$A, 0)), xaddr:$src),
+                (XFSTOREf64 (EXTRACT_SUBREG (XXPERMDI $A, $A, 2),
+                             sub_64), xaddr:$src)>;
+      def : Pat<(store (f64 (extractelt v2f64:$A, 0)), xaddr:$src),
+                (XFSTOREf64 (EXTRACT_SUBREG (XXPERMDI $A, $A, 2),
+                             sub_64), xaddr:$src)>;
+      def : Pat<(store (i64 (extractelt v2i64:$A, 1)), xaddr:$src),
+                (XFSTOREf64 (EXTRACT_SUBREG $A, sub_64), xaddr:$src)>;
+      def : Pat<(store (f64 (extractelt v2f64:$A, 1)), xaddr:$src),
+                (XFSTOREf64 (EXTRACT_SUBREG $A, sub_64), xaddr:$src)>;
+      def : Pat<(store (i64 (extractelt v2i64:$A, 0)), ixaddr:$src),
+                (DFSTOREf64 (EXTRACT_SUBREG (XXPERMDI $A, $A, 2),
+                             sub_64), ixaddr:$src)>;
+      def : Pat<(store (f64 (extractelt v2f64:$A, 0)), ixaddr:$src),
+                (DFSTOREf64 (EXTRACT_SUBREG (XXPERMDI $A, $A, 2), sub_64),
+                            ixaddr:$src)>;
+      def : Pat<(store (i64 (extractelt v2i64:$A, 1)), ixaddr:$src),
+                (DFSTOREf64 (EXTRACT_SUBREG $A, sub_64), ixaddr:$src)>;
+      def : Pat<(store (f64 (extractelt v2f64:$A, 1)), ixaddr:$src),
+                (DFSTOREf64 (EXTRACT_SUBREG $A, sub_64), ixaddr:$src)>;
+    } // IsLittleEndian, HasP9Vector
 
     let Predicates = [IsBigEndian, HasP9Vector] in {
       def : Pat<(v2i64 (scalar_to_vector (i64 (load ixaddr:$src)))),
@@ -3306,7 +3338,27 @@ let AddedComplexity = 400, Predicates = [HasP9Vector] in {
                 (v2f64 (COPY_TO_REGCLASS (DFLOADf64 ixaddr:$src), VSRC))>;
       def : Pat<(v2f64 (scalar_to_vector (f64 (load xaddr:$src)))),
                 (v2f64 (COPY_TO_REGCLASS (XFLOADf64 xaddr:$src), VSRC))>;
-    }
+      def : Pat<(store (i64 (extractelt v2i64:$A, 1)), xaddr:$src),
+                (XFSTOREf64 (EXTRACT_SUBREG (XXPERMDI $A, $A, 2),
+                             sub_64), xaddr:$src)>;
+      def : Pat<(store (f64 (extractelt v2f64:$A, 1)), xaddr:$src),
+                (XFSTOREf64 (EXTRACT_SUBREG (XXPERMDI $A, $A, 2),
+                             sub_64), xaddr:$src)>;
+      def : Pat<(store (i64 (extractelt v2i64:$A, 0)), xaddr:$src),
+                (XFSTOREf64 (EXTRACT_SUBREG $A, sub_64), xaddr:$src)>;
+      def : Pat<(store (f64 (extractelt v2f64:$A, 0)), xaddr:$src),
+                (XFSTOREf64 (EXTRACT_SUBREG $A, sub_64), xaddr:$src)>;
+      def : Pat<(store (i64 (extractelt v2i64:$A, 1)), ixaddr:$src),
+                (DFSTOREf64 (EXTRACT_SUBREG (XXPERMDI $A, $A, 2),
+                             sub_64), ixaddr:$src)>;
+      def : Pat<(store (f64 (extractelt v2f64:$A, 1)), ixaddr:$src),
+                (DFSTOREf64 (EXTRACT_SUBREG (XXPERMDI $A, $A, 2),
+                             sub_64), ixaddr:$src)>;
+      def : Pat<(store (i64 (extractelt v2i64:$A, 0)), ixaddr:$src),
+                (DFSTOREf64 (EXTRACT_SUBREG $A, sub_64), ixaddr:$src)>;
+      def : Pat<(store (f64 (extractelt v2f64:$A, 0)), ixaddr:$src),
+                (DFSTOREf64 (EXTRACT_SUBREG $A, sub_64), ixaddr:$src)>;
+    } // IsBigEndian, HasP9Vector
   }
 
   let Predicates = [IsBigEndian, HasP9Vector] in {
@@ -3814,6 +3866,45 @@ let AddedComplexity = 400 in {
                                               (XFLOADf32 xoaddr:$A), VSFRC)), 0))>;
   }
 
+  let Predicates = [IsBigEndian, HasP8Vector] in {
+    def : Pat<DWToSPExtractConv.BVU,
+              (v4f32 (VPKUDUM (XXSLDWI (XVCVUXDSP $S1), (XVCVUXDSP $S1), 3),
+                              (XXSLDWI (XVCVUXDSP $S2), (XVCVUXDSP $S2), 3)))>;
+    def : Pat<DWToSPExtractConv.BVS,
+              (v4f32 (VPKUDUM (XXSLDWI (XVCVSXDSP $S1), (XVCVSXDSP $S1), 3),
+                              (XXSLDWI (XVCVSXDSP $S2), (XVCVSXDSP $S2), 3)))>;
+    def : Pat<(store (i32 (extractelt v4i32:$A, 1)), xoaddr:$src),
+              (STIWX (EXTRACT_SUBREG $A, sub_64), xoaddr:$src)>;
+    def : Pat<(store (f32 (extractelt v4f32:$A, 1)), xoaddr:$src),
+              (STIWX (EXTRACT_SUBREG $A, sub_64), xoaddr:$src)>;
+
+    // Elements in a register on a BE system are in order <0, 1, 2, 3>.
+    // The store instructions store the second word from the left.
+    // So to align element zero, we need to modulo-left-shift by 3 words.
+    // Similar logic applies for elements 2 and 3.
+    foreach Idx = [ [0,3], [2,1], [3,2] ] in {
+      def : Pat<(store (i32 (extractelt v4i32:$A, !head(Idx))), xoaddr:$src),
+                (STIWX (EXTRACT_SUBREG (XXSLDWI $A, $A, !head(!tail(Idx))),
+                                       sub_64), xoaddr:$src)>;
+      def : Pat<(store (f32 (extractelt v4f32:$A, !head(Idx))), xoaddr:$src),
+                (STIWX (EXTRACT_SUBREG (XXSLDWI $A, $A, !head(!tail(Idx))),
+                                       sub_64), xoaddr:$src)>;
+    }
+  }
+
+  let Predicates = [HasP8Vector, IsBigEndian, NoP9Vector] in {
+    def : Pat<(store (i64 (extractelt v2i64:$A, 0)), xoaddr:$src),
+              (XFSTOREf64 (EXTRACT_SUBREG $A, sub_64), xoaddr:$src)>;
+    def : Pat<(store (f64 (extractelt v2f64:$A, 0)), xoaddr:$src),
+              (XFSTOREf64 (EXTRACT_SUBREG $A, sub_64), xoaddr:$src)>;
+    def : Pat<(store (i64 (extractelt v2i64:$A, 1)), xoaddr:$src),
+              (XFSTOREf64 (EXTRACT_SUBREG (XXPERMDI $A, $A, 2), sub_64),
+                          xoaddr:$src)>;
+    def : Pat<(store (f64 (extractelt v2f64:$A, 1)), xoaddr:$src),
+              (XFSTOREf64 (EXTRACT_SUBREG (XXPERMDI $A, $A, 2), sub_64),
+                          xoaddr:$src)>;
+   }
+
   // Big endian, available on all targets with VSX
   let Predicates = [IsBigEndian, HasVSX] in {
     def : Pat<(v2f64 (build_vector f64:$A, f64:$B)),
@@ -3842,6 +3933,45 @@ let AddedComplexity = 400 in {
               (v4i32 (VMRGEW MrgWords.CVA0B0U, MrgWords.CVA1B1U))>;
   }
 
+  let Predicates = [IsLittleEndian, HasP8Vector] in {
+    def : Pat<DWToSPExtractConv.BVU,
+              (v4f32 (VPKUDUM (XXSLDWI (XVCVUXDSP $S2), (XVCVUXDSP $S2), 3),
+                              (XXSLDWI (XVCVUXDSP $S1), (XVCVUXDSP $S1), 3)))>;
+    def : Pat<DWToSPExtractConv.BVS,
+              (v4f32 (VPKUDUM (XXSLDWI (XVCVSXDSP $S2), (XVCVSXDSP $S2), 3),
+                              (XXSLDWI (XVCVSXDSP $S1), (XVCVSXDSP $S1), 3)))>;
+    def : Pat<(store (i32 (extractelt v4i32:$A, 2)), xoaddr:$src),
+              (STIWX (EXTRACT_SUBREG $A, sub_64), xoaddr:$src)>;
+    def : Pat<(store (f32 (extractelt v4f32:$A, 2)), xoaddr:$src),
+              (STIWX (EXTRACT_SUBREG $A, sub_64), xoaddr:$src)>;
+
+    // Elements in a register on a LE system are in order <3, 2, 1, 0>.
+    // The store instructions store the second word from the left.
+    // So to align element 3, we need to modulo-left-shift by 3 words.
+    // Similar logic applies for elements 0 and 1.
+    foreach Idx = [ [0,2], [1,1], [3,3] ] in {
+      def : Pat<(store (i32 (extractelt v4i32:$A, !head(Idx))), xoaddr:$src),
+                (STIWX (EXTRACT_SUBREG (XXSLDWI $A, $A, !head(!tail(Idx))),
+                                       sub_64), xoaddr:$src)>;
+      def : Pat<(store (f32 (extractelt v4f32:$A, !head(Idx))), xoaddr:$src),
+                (STIWX (EXTRACT_SUBREG (XXSLDWI $A, $A, !head(!tail(Idx))),
+                                       sub_64), xoaddr:$src)>;
+    }
+  }
+
+  let Predicates = [HasP8Vector, IsLittleEndian, NoP9Vector] in {
+    def : Pat<(store (i64 (extractelt v2i64:$A, 0)), xoaddr:$src),
+              (XFSTOREf64 (EXTRACT_SUBREG (XXPERMDI $A, $A, 2), sub_64),
+                          xoaddr:$src)>;
+    def : Pat<(store (f64 (extractelt v2f64:$A, 0)), xoaddr:$src),
+              (XFSTOREf64 (EXTRACT_SUBREG (XXPERMDI $A, $A, 2), sub_64),
+                          xoaddr:$src)>;
+    def : Pat<(store (i64 (extractelt v2i64:$A, 1)), xoaddr:$src),
+              (XFSTOREf64 (EXTRACT_SUBREG $A, sub_64), xoaddr:$src)>;
+    def : Pat<(store (f64 (extractelt v2f64:$A, 1)), xoaddr:$src),
+              (XFSTOREf64 (EXTRACT_SUBREG $A, sub_64), xoaddr:$src)>;
+   }
+
   let Predicates = [IsLittleEndian, HasVSX] in {
   // Little endian, available on all targets with VSX
     def : Pat<(v2f64 (build_vector f64:$A, f64:$B)),
diff --git a/lib/Target/PowerPC/PPCLoopPreIncPrep.cpp b/lib/Target/PowerPC/PPCLoopPreIncPrep.cpp
index 0b57dd9b618db8d908ec8cce2ccd0c98f3a76b0d..264dbbdc84e1e5540d088acac5a544381944f1a9 100644
--- a/lib/Target/PowerPC/PPCLoopPreIncPrep.cpp
+++ b/lib/Target/PowerPC/PPCLoopPreIncPrep.cpp
@@ -1,9 +1,8 @@
 //===------ PPCLoopPreIncPrep.cpp - Loop Pre-Inc. AM Prep. Pass -----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/PowerPC/PPCMCInstLower.cpp b/lib/Target/PowerPC/PPCMCInstLower.cpp
index e731c0bc0c239069ee0ade4962240941f4c9d7d7..858dc0b2940e53bd311777c9b15410abdab295d5 100644
--- a/lib/Target/PowerPC/PPCMCInstLower.cpp
+++ b/lib/Target/PowerPC/PPCMCInstLower.cpp
@@ -1,9 +1,8 @@
 //===-- PPCMCInstLower.cpp - Convert PPC MachineInstr to an MCInst --------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/PowerPC/PPCMIPeephole.cpp b/lib/Target/PowerPC/PPCMIPeephole.cpp
index 0068df19f0c8721f91f63d586c6680cbf2a96e60..255ba2d868158816e35c2f0ce2352a9a014a54d7 100644
--- a/lib/Target/PowerPC/PPCMIPeephole.cpp
+++ b/lib/Target/PowerPC/PPCMIPeephole.cpp
@@ -1,9 +1,8 @@
 //===-------------- PPCMIPeephole.cpp - MI Peephole Cleanups -------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===---------------------------------------------------------------------===//
 //
diff --git a/lib/Target/PowerPC/PPCMachineFunctionInfo.cpp b/lib/Target/PowerPC/PPCMachineFunctionInfo.cpp
index 3923417257e8ca85ca0dfc4e6ca55fad3c67f10d..2f65d6a2855be5b54ab095c1823249e2518b144c 100644
--- a/lib/Target/PowerPC/PPCMachineFunctionInfo.cpp
+++ b/lib/Target/PowerPC/PPCMachineFunctionInfo.cpp
@@ -1,9 +1,8 @@
 //===-- PPCMachineFunctionInfo.cpp - Private data used for PowerPC --------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/PowerPC/PPCMachineFunctionInfo.h b/lib/Target/PowerPC/PPCMachineFunctionInfo.h
index 8a3f50aa9565c646f3d2d069e2797ac5b6a81328..6c40e1ac6e5b2eefde590ef10e6aaf66e4a7ae29 100644
--- a/lib/Target/PowerPC/PPCMachineFunctionInfo.h
+++ b/lib/Target/PowerPC/PPCMachineFunctionInfo.h
@@ -1,9 +1,8 @@
 //===-- PPCMachineFunctionInfo.h - Private data used for PowerPC --*- C++ -*-=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/PowerPC/PPCPerfectShuffle.h b/lib/Target/PowerPC/PPCPerfectShuffle.h
index 8a1d68011c5f2c92798f0d8d6381d68a29e6967f..d0d84efdbd2033b2f36f9b0d0e7c72c6601ae011 100644
--- a/lib/Target/PowerPC/PPCPerfectShuffle.h
+++ b/lib/Target/PowerPC/PPCPerfectShuffle.h
@@ -1,9 +1,8 @@
 //===-- PPCPerfectShuffle.h - Altivec Perfect Shuffle Table -----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/PowerPC/PPCPfmCounters.td b/lib/Target/PowerPC/PPCPfmCounters.td
index d2a09f30c0f3e58a826ee8645e7617137d30e3e1..20b9efdc9df9b0ea467a4ddf83d1a0a3f4e43c00 100644
--- a/lib/Target/PowerPC/PPCPfmCounters.td
+++ b/lib/Target/PowerPC/PPCPfmCounters.td
@@ -1,9 +1,8 @@
 //===-- PPCPfmCounters.td - PPC Hardware Counters ----------*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/PowerPC/PPCPreEmitPeephole.cpp b/lib/Target/PowerPC/PPCPreEmitPeephole.cpp
index 3078a6610feae5281ae7b17fa8632305bbf78aec..d83c922768007770b46a080f25e2edbd629e8e48 100644
--- a/lib/Target/PowerPC/PPCPreEmitPeephole.cpp
+++ b/lib/Target/PowerPC/PPCPreEmitPeephole.cpp
@@ -1,9 +1,8 @@
 //===--------- PPCPreEmitPeephole.cpp - Late peephole optimizations -------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -140,7 +139,7 @@ namespace {
           // This conditional branch is always taken. So, remove all branches
           // and insert an unconditional branch to the destination of this.
           MachineBasicBlock::iterator It = Br, Er = MBB.end();
-          for (; It != Er && !SeenUse; It++) {
+          for (; It != Er; It++) {
             if (It->isDebugInstr()) continue;
             assert(It->isTerminator() && "Non-terminator after a terminator");
             InstrsToErase.push_back(&*It);
diff --git a/lib/Target/PowerPC/PPCQPXLoadSplat.cpp b/lib/Target/PowerPC/PPCQPXLoadSplat.cpp
index 25b2b54cbe98a4b36c11beb686496858578437fa..e0faae057c2225be67d078810ac21b0569e653b0 100644
--- a/lib/Target/PowerPC/PPCQPXLoadSplat.cpp
+++ b/lib/Target/PowerPC/PPCQPXLoadSplat.cpp
@@ -1,9 +1,8 @@
 //===----- PPCQPXLoadSplat.cpp - QPX Load Splat Simplification ------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/PowerPC/PPCReduceCRLogicals.cpp b/lib/Target/PowerPC/PPCReduceCRLogicals.cpp
index 173fc18b9ebffbcb98ec31abb4245f477900f13b..e4f6579f9bd7b64b9a811c70a2f60424c341d5a5 100644
--- a/lib/Target/PowerPC/PPCReduceCRLogicals.cpp
+++ b/lib/Target/PowerPC/PPCReduceCRLogicals.cpp
@@ -1,9 +1,8 @@
 //===---- PPCReduceCRLogicals.cpp - Reduce CR Bit Logical operations ------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===---------------------------------------------------------------------===//
 //
diff --git a/lib/Target/PowerPC/PPCRegisterInfo.cpp b/lib/Target/PowerPC/PPCRegisterInfo.cpp
index 96923a97a82c20bc012cbe3487251b473faa3dc9..86062dde9fa0b4c033b9e9cbc973ecc735e6c7e9 100644
--- a/lib/Target/PowerPC/PPCRegisterInfo.cpp
+++ b/lib/Target/PowerPC/PPCRegisterInfo.cpp
@@ -1,9 +1,8 @@
 //===-- PPCRegisterInfo.cpp - PowerPC Register Information ----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -673,12 +672,15 @@ void PPCRegisterInfo::lowerCRBitSpilling(MachineBasicBlock::iterator II,
   unsigned Reg = MF.getRegInfo().createVirtualRegister(LP64 ? G8RC : GPRC);
   unsigned SrcReg = MI.getOperand(0).getReg();
 
-  BuildMI(MBB, II, dl, TII.get(TargetOpcode::KILL),
-          getCRFromCRBit(SrcReg))
-          .addReg(SrcReg, getKillRegState(MI.getOperand(0).isKill()));
-
+  // We need to move the CR field that contains the CR bit we are spilling.
+  // The super register may not be explicitly defined (i.e. it can be defined
+  // by a CR-logical that only defines the subreg) so we state that the CR
+  // field is undef. Also, in order to preserve the kill flag on the CR bit,
+  // we add it as an implicit use.
   BuildMI(MBB, II, dl, TII.get(LP64 ? PPC::MFOCRF8 : PPC::MFOCRF), Reg)
-      .addReg(getCRFromCRBit(SrcReg));
+      .addReg(getCRFromCRBit(SrcReg), RegState::Undef)
+      .addReg(SrcReg,
+              RegState::Implicit | getKillRegState(MI.getOperand(0).isKill()));
 
   // If the saved register wasn't CR0LT, shift the bits left so that the bit to
   // store is the first one. Mask all but that bit.
@@ -998,7 +1000,8 @@ PPCRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator II,
 
   if (noImmForm)
     OperandBase = 1;
-  else if (OpC != TargetOpcode::INLINEASM) {
+  else if (OpC != TargetOpcode::INLINEASM &&
+           OpC != TargetOpcode::INLINEASM_BR) {
     assert(ImmToIdxMap.count(OpC) &&
            "No indexed form of load or store available!");
     unsigned NewOpcode = ImmToIdxMap.find(OpC)->second;
diff --git a/lib/Target/PowerPC/PPCRegisterInfo.h b/lib/Target/PowerPC/PPCRegisterInfo.h
index e93fe4ce345355553388e0a8bedfc5836e771b73..7aae153facf3bd4e987e02ef8311809bdec9451e 100644
--- a/lib/Target/PowerPC/PPCRegisterInfo.h
+++ b/lib/Target/PowerPC/PPCRegisterInfo.h
@@ -1,9 +1,8 @@
 //===-- PPCRegisterInfo.h - PowerPC Register Information Impl ---*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/PowerPC/PPCRegisterInfo.td b/lib/Target/PowerPC/PPCRegisterInfo.td
index d0d29b6d2c7d4d484866470f35a375fcb359dd72..3798eb4e32c8c4a8754e92778df0108fab187702 100644
--- a/lib/Target/PowerPC/PPCRegisterInfo.td
+++ b/lib/Target/PowerPC/PPCRegisterInfo.td
@@ -1,9 +1,8 @@
 //===-- PPCRegisterInfo.td - The PowerPC Register File -----*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/PowerPC/PPCSchedule.td b/lib/Target/PowerPC/PPCSchedule.td
index c8fe7d7eea78382c61949789d28cedd11862742f..1a54975cb8f3ff7a024407158a184814854d2858 100644
--- a/lib/Target/PowerPC/PPCSchedule.td
+++ b/lib/Target/PowerPC/PPCSchedule.td
@@ -1,9 +1,8 @@
 //===-- PPCSchedule.td - PowerPC Scheduling Definitions ----*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/PowerPC/PPCSchedule440.td b/lib/Target/PowerPC/PPCSchedule440.td
index 646822eedbe078560181d907f0516c5bb35a3fba..708261fc7cc8adf5e612a0c1f58ccbd09a6c4f90 100644
--- a/lib/Target/PowerPC/PPCSchedule440.td
+++ b/lib/Target/PowerPC/PPCSchedule440.td
@@ -1,9 +1,8 @@
 //===-- PPCSchedule440.td - PPC 440 Scheduling Definitions -*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/PowerPC/PPCScheduleA2.td b/lib/Target/PowerPC/PPCScheduleA2.td
index f34c1accc0fd9d2446ab1861a795a359029a9e09..c2b298524e00a1b0468a4f0717d44905e00a2d3e 100644
--- a/lib/Target/PowerPC/PPCScheduleA2.td
+++ b/lib/Target/PowerPC/PPCScheduleA2.td
@@ -1,9 +1,8 @@
 //===- PPCScheduleA2.td - PPC A2 Scheduling Definitions --*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/PowerPC/PPCScheduleE500.td b/lib/Target/PowerPC/PPCScheduleE500.td
index 479a970b2537d2bfc3576ffa6d0295d97e1b769d..74744dda54f763d08c744c060a217401e9ab5b61 100644
--- a/lib/Target/PowerPC/PPCScheduleE500.td
+++ b/lib/Target/PowerPC/PPCScheduleE500.td
@@ -1,9 +1,8 @@
 //===-- PPCScheduleE500.td - e500 Scheduling Defs ------*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/PowerPC/PPCScheduleE500mc.td b/lib/Target/PowerPC/PPCScheduleE500mc.td
index d8bda073833f8840d81a9293074c74ea3d94845a..1a1c041565b679ed8939630cec64677c0f594d5d 100644
--- a/lib/Target/PowerPC/PPCScheduleE500mc.td
+++ b/lib/Target/PowerPC/PPCScheduleE500mc.td
@@ -1,9 +1,8 @@
 //===-- PPCScheduleE500mc.td - e500mc Scheduling Defs ------*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/PowerPC/PPCScheduleE5500.td b/lib/Target/PowerPC/PPCScheduleE5500.td
index 3e50803955c45b42be16ad2f78648d5cb07a60f8..4480d7fba4fb890838672da3d7bbdd458c44896c 100644
--- a/lib/Target/PowerPC/PPCScheduleE5500.td
+++ b/lib/Target/PowerPC/PPCScheduleE5500.td
@@ -1,9 +1,8 @@
 //===-- PPCScheduleE500mc.td - e5500 Scheduling Defs -------*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/PowerPC/PPCScheduleG3.td b/lib/Target/PowerPC/PPCScheduleG3.td
index 0995b7200d93d08e0e6cd0ffa052724d0d22b7d7..8f1907f2c0163bec15b400e06f0de3decf0bcfc3 100644
--- a/lib/Target/PowerPC/PPCScheduleG3.td
+++ b/lib/Target/PowerPC/PPCScheduleG3.td
@@ -1,9 +1,8 @@
 //===-- PPCScheduleG3.td - PPC G3 Scheduling Definitions ---*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/PowerPC/PPCScheduleG4.td b/lib/Target/PowerPC/PPCScheduleG4.td
index 1b15c7b3c7ad487666e2b1999aeb20e03fb77d7b..0eabc49d7841dd378828f79640d7b8205e22d1ed 100644
--- a/lib/Target/PowerPC/PPCScheduleG4.td
+++ b/lib/Target/PowerPC/PPCScheduleG4.td
@@ -1,9 +1,8 @@
 //===-- PPCScheduleG4.td - PPC G4 Scheduling Definitions ---*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/PowerPC/PPCScheduleG4Plus.td b/lib/Target/PowerPC/PPCScheduleG4Plus.td
index 0044c3c6a4490800301a7abf21de3297f7a29b2e..9c84aec638d775f900bb6ff5b0b8b4f61090a035 100644
--- a/lib/Target/PowerPC/PPCScheduleG4Plus.td
+++ b/lib/Target/PowerPC/PPCScheduleG4Plus.td
@@ -1,9 +1,8 @@
 //===-- PPCScheduleG4Plus.td - PPC G4+ Scheduling Defs. ----*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/PowerPC/PPCScheduleG5.td b/lib/Target/PowerPC/PPCScheduleG5.td
index c802b80170fb770c352d44cae50a27b18be17629..0870735377968e95fdb805e854ad7796828abba2 100644
--- a/lib/Target/PowerPC/PPCScheduleG5.td
+++ b/lib/Target/PowerPC/PPCScheduleG5.td
@@ -1,9 +1,8 @@
 //===-- PPCScheduleG5.td - PPC G5 Scheduling Definitions ---*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/PowerPC/PPCScheduleP7.td b/lib/Target/PowerPC/PPCScheduleP7.td
index 1d6e509819da5337ca9c8cfe172b1db9c9a7e225..5a8c1eb2b837866eda96f1f92e4f7232364b0da4 100644
--- a/lib/Target/PowerPC/PPCScheduleP7.td
+++ b/lib/Target/PowerPC/PPCScheduleP7.td
@@ -1,9 +1,8 @@
 //===-- PPCScheduleP7.td - PPC P7 Scheduling Definitions ---*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/PowerPC/PPCScheduleP8.td b/lib/Target/PowerPC/PPCScheduleP8.td
index ff39dfda7016d3e8a6d977f4d88d111a919c8f47..70a58f42a98ab9f8ece5d7bc15db6ba3e2c9a722 100644
--- a/lib/Target/PowerPC/PPCScheduleP8.td
+++ b/lib/Target/PowerPC/PPCScheduleP8.td
@@ -1,9 +1,8 @@
 //===-- PPCScheduleP8.td - PPC P8 Scheduling Definitions ---*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/PowerPC/PPCScheduleP9.td b/lib/Target/PowerPC/PPCScheduleP9.td
index e1a48011731520d9ab870c1b177c9a46ca60612f..679cf3962ec9fea26164d64c702a8ebfa2c9a906 100644
--- a/lib/Target/PowerPC/PPCScheduleP9.td
+++ b/lib/Target/PowerPC/PPCScheduleP9.td
@@ -1,9 +1,8 @@
 //===-- PPCScheduleP9.td - PPC P9 Scheduling Definitions ---*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -33,6 +32,12 @@ def P9Model : SchedMachineModel {
   // A dispatch group is 6 instructions.
   let LoopMicroOpBufferSize = 60;
 
+  // As iops are dispatched to a slice, they are held in an independent slice
+  // issue queue until all register sources and other dependencies have been
+  // resolved and they can be issued. Each of four execution slices has an
+  // 11-entry iop issue queue.
+  let MicroOpBufferSize = 44;
+
   let CompleteModel = 1;
 
   // Do not support QPX (Quad Processing eXtension) or SPE (Signal Procesing
diff --git a/lib/Target/PowerPC/PPCSubtarget.cpp b/lib/Target/PowerPC/PPCSubtarget.cpp
index c0cbfd779cb908137374fa8682acbd5bdca92677..94b80dd5836b4a8b98547fc361955da623eb9ff1 100644
--- a/lib/Target/PowerPC/PPCSubtarget.cpp
+++ b/lib/Target/PowerPC/PPCSubtarget.cpp
@@ -1,9 +1,8 @@
 //===-- PowerPCSubtarget.cpp - PPC Subtarget Information ------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -108,6 +107,7 @@ void PPCSubtarget::initializeEnvironment() {
   IsISA3_0 = false;
   UseLongCalls = false;
   SecurePlt = false;
+  VectorsUseTwoUnits = false;
 
   HasPOPCNTD = POPCNTD_Unavailable;
 }
diff --git a/lib/Target/PowerPC/PPCSubtarget.h b/lib/Target/PowerPC/PPCSubtarget.h
index c56f254d6beccdb8eafa6f11a903e17f0fe1fa02..ed9d5f1ae56d2dece6dbb1bab381a6d0bfde36df 100644
--- a/lib/Target/PowerPC/PPCSubtarget.h
+++ b/lib/Target/PowerPC/PPCSubtarget.h
@@ -1,9 +1,8 @@
 //===-- PPCSubtarget.h - Define Subtarget for the PPC ----------*- C++ -*--===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -136,6 +135,7 @@ protected:
   bool IsISA3_0;
   bool UseLongCalls;
   bool SecurePlt;
+  bool VectorsUseTwoUnits;
 
   POPCNTDKind HasPOPCNTD;
 
@@ -260,6 +260,7 @@ public:
   bool isPPC4xx() const { return IsPPC4xx; }
   bool isPPC6xx() const { return IsPPC6xx; }
   bool isSecurePlt() const {return SecurePlt; }
+  bool vectorsUseTwoUnits() const {return VectorsUseTwoUnits; }
   bool isE500() const { return IsE500; }
   bool isFeatureMFTB() const { return FeatureMFTB; }
   bool isDeprecatedDST() const { return DeprecatedDST; }
diff --git a/lib/Target/PowerPC/PPCTLSDynamicCall.cpp b/lib/Target/PowerPC/PPCTLSDynamicCall.cpp
index ac36abbe84395677d6b597836315eb108699e3bc..657ec4af4be97017055e24a569845c84805f46be 100644
--- a/lib/Target/PowerPC/PPCTLSDynamicCall.cpp
+++ b/lib/Target/PowerPC/PPCTLSDynamicCall.cpp
@@ -1,9 +1,8 @@
 //===---------- PPCTLSDynamicCall.cpp - TLS Dynamic Call Fixup ------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/PowerPC/PPCTOCRegDeps.cpp b/lib/Target/PowerPC/PPCTOCRegDeps.cpp
index 17345b6ca8d30ce58c1116b9dfc92aaf73abfc86..5e45d643d69fca2d5840ebaf563ea7639947f612 100644
--- a/lib/Target/PowerPC/PPCTOCRegDeps.cpp
+++ b/lib/Target/PowerPC/PPCTOCRegDeps.cpp
@@ -1,9 +1,8 @@
 //===-- PPCTOCRegDeps.cpp - Add Extra TOC Register Dependencies -----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/PowerPC/PPCTargetMachine.cpp b/lib/Target/PowerPC/PPCTargetMachine.cpp
index 580d057602f5f711e2491a24ad32b4df7f07279d..58057d37b5555d2e6254653e67197fd6ebab991b 100644
--- a/lib/Target/PowerPC/PPCTargetMachine.cpp
+++ b/lib/Target/PowerPC/PPCTargetMachine.cpp
@@ -1,9 +1,8 @@
 //===-- PPCTargetMachine.cpp - Define TargetMachine for PowerPC -----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/PowerPC/PPCTargetMachine.h b/lib/Target/PowerPC/PPCTargetMachine.h
index 75b98a815ab42cbc5d1e64bca4b95b40859ebed4..20ab66c2c2eb9548737ecd1c8e526ba47e8004af 100644
--- a/lib/Target/PowerPC/PPCTargetMachine.h
+++ b/lib/Target/PowerPC/PPCTargetMachine.h
@@ -1,9 +1,8 @@
 //===-- PPCTargetMachine.h - Define TargetMachine for PowerPC ---*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/PowerPC/PPCTargetObjectFile.cpp b/lib/Target/PowerPC/PPCTargetObjectFile.cpp
index a049dc3fda93b42b8ce0c1e416b82e04e9e9a737..e237fab1b2679a533a4e9c2768033332ad0f329c 100644
--- a/lib/Target/PowerPC/PPCTargetObjectFile.cpp
+++ b/lib/Target/PowerPC/PPCTargetObjectFile.cpp
@@ -1,9 +1,8 @@
 //===-- PPCTargetObjectFile.cpp - PPC Object Info -------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/PowerPC/PPCTargetObjectFile.h b/lib/Target/PowerPC/PPCTargetObjectFile.h
index 417b8ed0d612a3aa03a11aa36d7b8a0e9752c951..78a5840c87c721dbd4723829db247a6cd1f00f13 100644
--- a/lib/Target/PowerPC/PPCTargetObjectFile.h
+++ b/lib/Target/PowerPC/PPCTargetObjectFile.h
@@ -1,9 +1,8 @@
 //===-- PPCTargetObjectFile.h - PPC Object Info -----------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/PowerPC/PPCTargetStreamer.h b/lib/Target/PowerPC/PPCTargetStreamer.h
index 310fea9ef09ff637dc3feb1937aaea32c29982d6..e17361d997fde0911aa7a19cd90c3e744b9bfdce 100644
--- a/lib/Target/PowerPC/PPCTargetStreamer.h
+++ b/lib/Target/PowerPC/PPCTargetStreamer.h
@@ -1,9 +1,8 @@
 //===- PPCTargetStreamer.h - PPC Target Streamer ----------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/PowerPC/PPCTargetTransformInfo.cpp b/lib/Target/PowerPC/PPCTargetTransformInfo.cpp
index bc9bcab83a0a890f76c235520e2224aba28cac5f..6185355f419f4250aa8dbf7f570bd81c6e413826 100644
--- a/lib/Target/PowerPC/PPCTargetTransformInfo.cpp
+++ b/lib/Target/PowerPC/PPCTargetTransformInfo.cpp
@@ -1,9 +1,8 @@
 //===-- PPCTargetTransformInfo.cpp - PPC specific TTI ---------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -324,6 +323,33 @@ unsigned PPCTTIImpl::getMaxInterleaveFactor(unsigned VF) {
   return 2;
 }
 
+// Adjust the cost of vector instructions on targets which there is overlap
+// between the vector and scalar units, thereby reducing the overall throughput
+// of vector code wrt. scalar code.
+int PPCTTIImpl::vectorCostAdjustment(int Cost, unsigned Opcode, Type *Ty1,
+                                     Type *Ty2) {
+  if (!ST->vectorsUseTwoUnits() || !Ty1->isVectorTy())
+    return Cost;
+
+  std::pair<int, MVT> LT1 = TLI->getTypeLegalizationCost(DL, Ty1);
+  // If type legalization involves splitting the vector, we don't want to
+  // double the cost at every step - only the last step.
+  if (LT1.first != 1 || !LT1.second.isVector())
+    return Cost;
+
+  int ISD = TLI->InstructionOpcodeToISD(Opcode);
+  if (TLI->isOperationExpand(ISD, LT1.second))
+    return Cost;
+
+  if (Ty2) {
+    std::pair<int, MVT> LT2 = TLI->getTypeLegalizationCost(DL, Ty2);
+    if (LT2.first != 1 || !LT2.second.isVector())
+      return Cost;
+  }
+
+  return Cost * 2;
+}
+
 int PPCTTIImpl::getArithmeticInstrCost(
     unsigned Opcode, Type *Ty, TTI::OperandValueKind Op1Info,
     TTI::OperandValueKind Op2Info, TTI::OperandValueProperties Opd1PropInfo,
@@ -331,8 +357,9 @@ int PPCTTIImpl::getArithmeticInstrCost(
   assert(TLI->InstructionOpcodeToISD(Opcode) && "Invalid opcode");
 
   // Fallback to the default implementation.
-  return BaseT::getArithmeticInstrCost(Opcode, Ty, Op1Info, Op2Info,
-                                       Opd1PropInfo, Opd2PropInfo);
+  int Cost = BaseT::getArithmeticInstrCost(Opcode, Ty, Op1Info, Op2Info,
+                                           Opd1PropInfo, Opd2PropInfo);
+  return vectorCostAdjustment(Cost, Opcode, Ty, nullptr);
 }
 
 int PPCTTIImpl::getShuffleCost(TTI::ShuffleKind Kind, Type *Tp, int Index,
@@ -345,19 +372,22 @@ int PPCTTIImpl::getShuffleCost(TTI::ShuffleKind Kind, Type *Tp, int Index,
   // instruction). We need one such shuffle instruction for each actual
   // register (this is not true for arbitrary shuffles, but is true for the
   // structured types of shuffles covered by TTI::ShuffleKind).
-  return LT.first;
+  return vectorCostAdjustment(LT.first, Instruction::ShuffleVector, Tp,
+                              nullptr);
 }
 
 int PPCTTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src,
                                  const Instruction *I) {
   assert(TLI->InstructionOpcodeToISD(Opcode) && "Invalid opcode");
 
-  return BaseT::getCastInstrCost(Opcode, Dst, Src);
+  int Cost = BaseT::getCastInstrCost(Opcode, Dst, Src);
+  return vectorCostAdjustment(Cost, Opcode, Dst, Src);
 }
 
 int PPCTTIImpl::getCmpSelInstrCost(unsigned Opcode, Type *ValTy, Type *CondTy,
                                    const Instruction *I) {
-  return BaseT::getCmpSelInstrCost(Opcode, ValTy, CondTy, I);
+  int Cost = BaseT::getCmpSelInstrCost(Opcode, ValTy, CondTy, I);
+  return vectorCostAdjustment(Cost, Opcode, ValTy, nullptr);
 }
 
 int PPCTTIImpl::getVectorInstrCost(unsigned Opcode, Type *Val, unsigned Index) {
@@ -366,18 +396,22 @@ int PPCTTIImpl::getVectorInstrCost(unsigned Opcode, Type *Val, unsigned Index) {
   int ISD = TLI->InstructionOpcodeToISD(Opcode);
   assert(ISD && "Invalid opcode");
 
+  int Cost = BaseT::getVectorInstrCost(Opcode, Val, Index);
+  Cost = vectorCostAdjustment(Cost, Opcode, Val, nullptr);
+
   if (ST->hasVSX() && Val->getScalarType()->isDoubleTy()) {
-    // Double-precision scalars are already located in index #0.
-    if (Index == 0)
+    // Double-precision scalars are already located in index #0 (or #1 if LE).
+    if (ISD == ISD::EXTRACT_VECTOR_ELT && Index == ST->isLittleEndian() ? 1 : 0)
       return 0;
 
-    return BaseT::getVectorInstrCost(Opcode, Val, Index);
+    return Cost;
+
   } else if (ST->hasQPX() && Val->getScalarType()->isFloatingPointTy()) {
     // Floating point scalars are already located in index #0.
     if (Index == 0)
       return 0;
 
-    return BaseT::getVectorInstrCost(Opcode, Val, Index);
+    return Cost;
   }
 
   // Estimated cost of a load-hit-store delay.  This was obtained
@@ -394,9 +428,9 @@ int PPCTTIImpl::getVectorInstrCost(unsigned Opcode, Type *Val, unsigned Index) {
   // these need to be estimated as very costly.
   if (ISD == ISD::EXTRACT_VECTOR_ELT ||
       ISD == ISD::INSERT_VECTOR_ELT)
-    return LHSPenalty + BaseT::getVectorInstrCost(Opcode, Val, Index);
+    return LHSPenalty + Cost;
 
-  return BaseT::getVectorInstrCost(Opcode, Val, Index);
+  return Cost;
 }
 
 int PPCTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src, unsigned Alignment,
@@ -407,6 +441,7 @@ int PPCTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src, unsigned Alignment,
          "Invalid Opcode");
 
   int Cost = BaseT::getMemoryOpCost(Opcode, Src, Alignment, AddressSpace);
+  Cost = vectorCostAdjustment(Cost, Opcode, Src, nullptr);
 
   bool IsAltivecType = ST->hasAltivec() &&
                        (LT.second == MVT::v16i8 || LT.second == MVT::v8i16 ||
diff --git a/lib/Target/PowerPC/PPCTargetTransformInfo.h b/lib/Target/PowerPC/PPCTargetTransformInfo.h
index 9221a910288adedf442d69f9ca4dc68de96059ae..a8d9d092854facec5ed3f8543a355a41e3ce7210 100644
--- a/lib/Target/PowerPC/PPCTargetTransformInfo.h
+++ b/lib/Target/PowerPC/PPCTargetTransformInfo.h
@@ -1,9 +1,8 @@
 //===-- PPCTargetTransformInfo.h - PPC specific TTI -------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
@@ -71,6 +70,7 @@ public:
   unsigned getCacheLineSize();
   unsigned getPrefetchDistance();
   unsigned getMaxInterleaveFactor(unsigned VF);
+  int vectorCostAdjustment(int Cost, unsigned Opcode, Type *Ty1, Type *Ty2);
   int getArithmeticInstrCost(
       unsigned Opcode, Type *Ty,
       TTI::OperandValueKind Opd1Info = TTI::OK_AnyValue,
diff --git a/lib/Target/PowerPC/PPCVSXCopy.cpp b/lib/Target/PowerPC/PPCVSXCopy.cpp
index 93fe3230ab815a81720892767e69b86ed8a1dd0d..f7c3387cc8bc00b6829943ecf6ff0bd547de0a32 100644
--- a/lib/Target/PowerPC/PPCVSXCopy.cpp
+++ b/lib/Target/PowerPC/PPCVSXCopy.cpp
@@ -1,9 +1,8 @@
 //===-------------- PPCVSXCopy.cpp - VSX Copy Legalization ----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/PowerPC/PPCVSXFMAMutate.cpp b/lib/Target/PowerPC/PPCVSXFMAMutate.cpp
index 6586f503a7b8fbe9b2be3e71ae100e1284b46b2e..ce78239df0a855647cfed0a389abe14d93c238ee 100644
--- a/lib/Target/PowerPC/PPCVSXFMAMutate.cpp
+++ b/lib/Target/PowerPC/PPCVSXFMAMutate.cpp
@@ -1,9 +1,8 @@
 //===--------------- PPCVSXFMAMutate.cpp - VSX FMA Mutation ---------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/PowerPC/PPCVSXSwapRemoval.cpp b/lib/Target/PowerPC/PPCVSXSwapRemoval.cpp
index 1be193e08c014945d3c4ea91cd1c371fed749a50..4e01e77c624fedc5c75eb7c7b29c9dd2efb55763 100644
--- a/lib/Target/PowerPC/PPCVSXSwapRemoval.cpp
+++ b/lib/Target/PowerPC/PPCVSXSwapRemoval.cpp
@@ -1,9 +1,8 @@
 //===----------- PPCVSXSwapRemoval.cpp - Remove VSX LE Swaps -------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===---------------------------------------------------------------------===//
 //
diff --git a/lib/Target/PowerPC/README.txt b/lib/Target/PowerPC/README.txt
index 7f5beca90455ac0508e11b551d29341356868cb8..b4bf635dc2c75c4d6a77969b44f25dd51ecf7cec 100644
--- a/lib/Target/PowerPC/README.txt
+++ b/lib/Target/PowerPC/README.txt
@@ -663,4 +663,3 @@ and force instruction pairs to be scheduled together.
 More general handling of any_extend and zero_extend:
 
 See https://reviews.llvm.org/D24924#555306
-
diff --git a/lib/Target/PowerPC/TargetInfo/LLVMBuild.txt b/lib/Target/PowerPC/TargetInfo/LLVMBuild.txt
index 4102346864009480e96f4055b70605fdf6bb6c26..dc7439901321b4a478c0d6c7b385e6ba2a6b2f76 100644
--- a/lib/Target/PowerPC/TargetInfo/LLVMBuild.txt
+++ b/lib/Target/PowerPC/TargetInfo/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/Target/PowerPC/TargetInfo/LLVMBuild.txt ------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Target/PowerPC/TargetInfo/PowerPCTargetInfo.cpp b/lib/Target/PowerPC/TargetInfo/PowerPCTargetInfo.cpp
index 979595264472f3c9d9d5ceb4c655f5d448b8769f..7bf854233bd8d41f566e33e9aba26163bbade651 100644
--- a/lib/Target/PowerPC/TargetInfo/PowerPCTargetInfo.cpp
+++ b/lib/Target/PowerPC/TargetInfo/PowerPCTargetInfo.cpp
@@ -1,9 +1,8 @@
 //===-- PowerPCTargetInfo.cpp - PowerPC Target Implementation -------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/RISCV/AsmParser/LLVMBuild.txt b/lib/Target/RISCV/AsmParser/LLVMBuild.txt
index 205d1248f9234ddd521388fd617e92440f4f83d1..64f2a20cb1ccead84d9a8bfe456935287316f6d7 100644
--- a/lib/Target/RISCV/AsmParser/LLVMBuild.txt
+++ b/lib/Target/RISCV/AsmParser/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/Target/RISCV/AsmParser/LLVMBuild.txt ---------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp b/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp
index 4e70ea402d8c8db1e53c015b5920a881c2859f7c..36a3e595b55d5e9e54c9ae4502a967d42f87dd87 100644
--- a/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp
+++ b/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp
@@ -1,9 +1,8 @@
 //===-- RISCVAsmParser.cpp - Parse RISCV assembly to MCInst instructions --===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -310,12 +309,14 @@ public:
     return RISCVFPRndMode::stringToRoundingMode(Str) != RISCVFPRndMode::Invalid;
   }
 
-  bool isImmXLen() const {
+  bool isImmXLenLI() const {
     int64_t Imm;
     RISCVMCExpr::VariantKind VK;
     if (!isImm())
       return false;
     bool IsConstantImm = evaluateConstantImm(getImm(), Imm, VK);
+    if (VK == RISCVMCExpr::VK_RISCV_LO || VK == RISCVMCExpr::VK_RISCV_PCREL_LO)
+      return true;
     // Given only Imm, ensuring that the actually specified constant is either
     // a signed or unsigned 64-bit number is unfortunately impossible.
     bool IsInRange = isRV64() ? true : isInt<32>(Imm) || isUInt<32>(Imm);
@@ -782,7 +783,7 @@ bool RISCVAsmParser::MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
   switch(Result) {
   default:
     break;
-  case Match_InvalidImmXLen:
+  case Match_InvalidImmXLenLI:
     if (isRV64()) {
       SMLoc ErrorLoc = ((RISCVOperand &)*Operands[ErrorInfo]).getStartLoc();
       return Error(ErrorLoc, "operand must be a constant 64-bit integer");
@@ -1031,17 +1032,10 @@ OperandMatchResultTy RISCVAsmParser::parseImmediate(OperandVector &Operands) {
   case AsmToken::Plus:
   case AsmToken::Integer:
   case AsmToken::String:
+  case AsmToken::Identifier:
     if (getParser().parseExpression(Res))
       return MatchOperand_ParseFail;
     break;
-  case AsmToken::Identifier: {
-    StringRef Identifier;
-    if (getParser().parseIdentifier(Identifier))
-      return MatchOperand_ParseFail;
-    MCSymbol *Sym = getContext().getOrCreateSymbol(Identifier);
-    Res = MCSymbolRefExpr::create(Sym, MCSymbolRefExpr::VK_None, getContext());
-    break;
-  }
   case AsmToken::Percent:
     return parseOperandWithModifier(Operands);
   }
@@ -1449,7 +1443,17 @@ bool RISCVAsmParser::processInstruction(MCInst &Inst, SMLoc IDLoc,
   Inst.setLoc(IDLoc);
 
   if (Inst.getOpcode() == RISCV::PseudoLI) {
-    auto Reg = Inst.getOperand(0).getReg();
+    unsigned Reg = Inst.getOperand(0).getReg();
+    const MCOperand &Op1 = Inst.getOperand(1);
+    if (Op1.isExpr()) {
+      // We must have li reg, %lo(sym) or li reg, %pcrel_lo(sym) or similar.
+      // Just convert to an addi. This allows compatibility with gas.
+      emitToStreamer(Out, MCInstBuilder(RISCV::ADDI)
+                              .addReg(Reg)
+                              .addReg(RISCV::X0)
+                              .addExpr(Op1.getExpr()));
+      return false;
+    }
     int64_t Imm = Inst.getOperand(1).getImm();
     // On RV32 the immediate here can either be a signed or an unsigned
     // 32-bit number. Sign extension has to be performed to ensure that Imm
diff --git a/lib/Target/RISCV/Disassembler/LLVMBuild.txt b/lib/Target/RISCV/Disassembler/LLVMBuild.txt
index 340e89d4cf357c34ae5aec85ba5ecad983bd9540..e69a640cf7e70194a66416b5f621c57e4d47b329 100644
--- a/lib/Target/RISCV/Disassembler/LLVMBuild.txt
+++ b/lib/Target/RISCV/Disassembler/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/Target/RISCV/Disassembler/LLVMBuild.txt ------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Target/RISCV/Disassembler/RISCVDisassembler.cpp b/lib/Target/RISCV/Disassembler/RISCVDisassembler.cpp
index eafa09d56315d853cdcab9f1715e7e4b0aaa6aa0..a300de6ca04c4c5b70695a97cc513c743f6990e7 100644
--- a/lib/Target/RISCV/Disassembler/RISCVDisassembler.cpp
+++ b/lib/Target/RISCV/Disassembler/RISCVDisassembler.cpp
@@ -1,9 +1,8 @@
 //===-- RISCVDisassembler.cpp - Disassembler for RISCV --------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/RISCV/InstPrinter/LLVMBuild.txt b/lib/Target/RISCV/InstPrinter/LLVMBuild.txt
index 5a28372a302e86909fb95272b13b689fc8b58929..874a657ac04cbbc4f0ed1ad7a064a2f314c92cfa 100644
--- a/lib/Target/RISCV/InstPrinter/LLVMBuild.txt
+++ b/lib/Target/RISCV/InstPrinter/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/Target/RISCV/InstPrinter/LLVMBuild.txt -------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Target/RISCV/InstPrinter/RISCVInstPrinter.cpp b/lib/Target/RISCV/InstPrinter/RISCVInstPrinter.cpp
index 979c8f4e2fa7a9c6ef2ccc5739aa8102e4afb91d..fe37b70811d845d8e06dc3bfeae7b0ce8636d27f 100644
--- a/lib/Target/RISCV/InstPrinter/RISCVInstPrinter.cpp
+++ b/lib/Target/RISCV/InstPrinter/RISCVInstPrinter.cpp
@@ -1,9 +1,8 @@
 //===-- RISCVInstPrinter.cpp - Convert RISCV MCInst to asm syntax ---------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/RISCV/InstPrinter/RISCVInstPrinter.h b/lib/Target/RISCV/InstPrinter/RISCVInstPrinter.h
index 0f9bed184996ae65453e1d428daf186f29fe985b..bd8d78a62081697f033be0dd18b60ad72f34ec2b 100644
--- a/lib/Target/RISCV/InstPrinter/RISCVInstPrinter.h
+++ b/lib/Target/RISCV/InstPrinter/RISCVInstPrinter.h
@@ -1,9 +1,8 @@
 //===-- RISCVInstPrinter.h - Convert RISCV MCInst to asm syntax ---*- C++ -*--//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/RISCV/LLVMBuild.txt b/lib/Target/RISCV/LLVMBuild.txt
index 7af605e6954d200bf175fe9602796bca9f92753a..a4b92d22cacdbb351aeb795f8ddeb90d44886b8a 100644
--- a/lib/Target/RISCV/LLVMBuild.txt
+++ b/lib/Target/RISCV/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/Target/RISCV/LLVMBuild.txt -------------------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Target/RISCV/MCTargetDesc/LLVMBuild.txt b/lib/Target/RISCV/MCTargetDesc/LLVMBuild.txt
index 92daae30e3ff78c3914ac0a4618658f04fc6d6aa..b8952c79f85bd05f7fd610a787631a539a907276 100644
--- a/lib/Target/RISCV/MCTargetDesc/LLVMBuild.txt
+++ b/lib/Target/RISCV/MCTargetDesc/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/Target/RISCV/MCTargetDesc/LLVMBuild.txt ------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.cpp b/lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.cpp
index 49239ac9099ee2d8c9c060b92e0fe1c8ae1055f4..7ce058da6175006d30df98252130d67e21eebb44 100644
--- a/lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.cpp
+++ b/lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.cpp
@@ -1,13 +1,13 @@
 //===-- RISCVAsmBackend.cpp - RISCV Assembler Backend ---------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #include "RISCVAsmBackend.h"
+#include "RISCVMCExpr.h"
 #include "llvm/ADT/APInt.h"
 #include "llvm/MC/MCAssembler.h"
 #include "llvm/MC/MCContext.h"
@@ -16,11 +16,50 @@
 #include "llvm/MC/MCExpr.h"
 #include "llvm/MC/MCObjectWriter.h"
 #include "llvm/MC/MCSymbol.h"
+#include "llvm/MC/MCValue.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/raw_ostream.h"
 
 using namespace llvm;
 
+// If linker relaxation is enabled, or the relax option had previously been
+// enabled, always emit relocations even if the fixup can be resolved. This is
+// necessary for correctness as offsets may change during relaxation.
+bool RISCVAsmBackend::shouldForceRelocation(const MCAssembler &Asm,
+                                            const MCFixup &Fixup,
+                                            const MCValue &Target) {
+  bool ShouldForce = false;
+
+  switch ((unsigned)Fixup.getKind()) {
+  default:
+    break;
+  case RISCV::fixup_riscv_pcrel_lo12_i:
+  case RISCV::fixup_riscv_pcrel_lo12_s:
+    // For pcrel_lo12, force a relocation if the target of the corresponding
+    // pcrel_hi20 is not in the same fragment.
+    const MCFixup *T = cast<RISCVMCExpr>(Fixup.getValue())->getPCRelHiFixup();
+    if (!T) {
+      Asm.getContext().reportError(Fixup.getLoc(),
+                                   "could not find corresponding %pcrel_hi");
+      return false;
+    }
+
+    switch ((unsigned)T->getKind()) {
+    default:
+      llvm_unreachable("Unexpected fixup kind for pcrel_lo12");
+      break;
+    case RISCV::fixup_riscv_pcrel_hi20:
+      ShouldForce = T->getValue()->findAssociatedFragment() !=
+                    Fixup.getValue()->findAssociatedFragment();
+      break;
+    }
+    break;
+  }
+
+  return ShouldForce || STI.getFeatureBits()[RISCV::FeatureRelax] ||
+         ForceRelocs;
+}
+
 bool RISCVAsmBackend::fixupNeedsRelaxationAdvanced(const MCFixup &Fixup,
                                                    bool Resolved,
                                                    uint64_t Value,
@@ -248,6 +287,57 @@ void RISCVAsmBackend::applyFixup(const MCAssembler &Asm, const MCFixup &Fixup,
   }
 }
 
+// Linker relaxation may change code size. We have to insert Nops
+// for .align directive when linker relaxation enabled. So then Linker
+// could satisfy alignment by removing Nops.
+// The function return the total Nops Size we need to insert.
+bool RISCVAsmBackend::shouldInsertExtraNopBytesForCodeAlign(
+    const MCAlignFragment &AF, unsigned &Size) {
+  // Calculate Nops Size only when linker relaxation enabled.
+  if (!STI.getFeatureBits()[RISCV::FeatureRelax])
+    return false;
+
+  bool HasStdExtC = STI.getFeatureBits()[RISCV::FeatureStdExtC];
+  unsigned MinNopLen = HasStdExtC ? 2 : 4;
+
+  Size = AF.getAlignment() - MinNopLen;
+  return true;
+}
+
+// We need to insert R_RISCV_ALIGN relocation type to indicate the
+// position of Nops and the total bytes of the Nops have been inserted
+// when linker relaxation enabled.
+// The function insert fixup_riscv_align fixup which eventually will
+// transfer to R_RISCV_ALIGN relocation type.
+bool RISCVAsmBackend::shouldInsertFixupForCodeAlign(MCAssembler &Asm,
+                                                    const MCAsmLayout &Layout,
+                                                    MCAlignFragment &AF) {
+  // Insert the fixup only when linker relaxation enabled.
+  if (!STI.getFeatureBits()[RISCV::FeatureRelax])
+    return false;
+
+  // Calculate total Nops we need to insert.
+  unsigned Count;
+  shouldInsertExtraNopBytesForCodeAlign(AF, Count);
+  // No Nop need to insert, simply return.
+  if (Count == 0)
+    return false;
+
+  MCContext &Ctx = Asm.getContext();
+  const MCExpr *Dummy = MCConstantExpr::create(0, Ctx);
+  // Create fixup_riscv_align fixup.
+  MCFixup Fixup =
+      MCFixup::create(0, Dummy, MCFixupKind(RISCV::fixup_riscv_align), SMLoc());
+
+  uint64_t FixedValue = 0;
+  MCValue NopBytes = MCValue::get(Count);
+
+  Asm.getWriter().recordRelocation(Asm, Layout, &AF, Fixup, NopBytes,
+                                   FixedValue);
+
+  return true;
+}
+
 std::unique_ptr<MCObjectTargetWriter>
 RISCVAsmBackend::createObjectTargetWriter() const {
   return createRISCVELFObjectWriter(OSABI, Is64Bit);
diff --git a/lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.h b/lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.h
index 5601f07f9267828c92f0d4e5a6651f840c4815ac..6fbc02ff3c00583551131e7972c854c813cedc9c 100644
--- a/lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.h
+++ b/lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.h
@@ -1,9 +1,8 @@
 //===-- RISCVAsmBackend.h - RISCV Assembler Backend -----------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -41,6 +40,16 @@ public:
   bool requiresDiffExpressionRelocations() const override {
     return STI.getFeatureBits()[RISCV::FeatureRelax] || ForceRelocs;
   }
+
+  // Return Size with extra Nop Bytes for alignment directive in code section.
+  bool shouldInsertExtraNopBytesForCodeAlign(const MCAlignFragment &AF,
+                                             unsigned &Size) override;
+
+  // Insert target specific fixup type for alignment directive in code section.
+  bool shouldInsertFixupForCodeAlign(MCAssembler &Asm,
+                                     const MCAsmLayout &Layout,
+                                     MCAlignFragment &AF) override;
+
   void applyFixup(const MCAssembler &Asm, const MCFixup &Fixup,
                   const MCValue &Target, MutableArrayRef<char> Data,
                   uint64_t Value, bool IsResolved,
@@ -49,13 +58,8 @@ public:
   std::unique_ptr<MCObjectTargetWriter>
   createObjectTargetWriter() const override;
 
-  // If linker relaxation is enabled, or the relax option had previously been
-  // enabled, always emit relocations even if the fixup can be resolved. This is
-  // necessary for correctness as offsets may change during relaxation.
   bool shouldForceRelocation(const MCAssembler &Asm, const MCFixup &Fixup,
-                             const MCValue &Target) override {
-    return STI.getFeatureBits()[RISCV::FeatureRelax] || ForceRelocs;
-  }
+                             const MCValue &Target) override;
 
   bool fixupNeedsRelaxation(const MCFixup &Fixup, uint64_t Value,
                             const MCRelaxableFragment *DF,
@@ -90,7 +94,8 @@ public:
       { "fixup_riscv_rvc_jump",      2,     11,  MCFixupKindInfo::FKF_IsPCRel },
       { "fixup_riscv_rvc_branch",    0,     16,  MCFixupKindInfo::FKF_IsPCRel },
       { "fixup_riscv_call",          0,     64,  MCFixupKindInfo::FKF_IsPCRel },
-      { "fixup_riscv_relax",         0,      0,  0 }
+      { "fixup_riscv_relax",         0,      0,  0 },
+      { "fixup_riscv_align",         0,      0,  0 }
     };
     static_assert((array_lengthof(Infos)) == RISCV::NumTargetFixupKinds,
                   "Not all fixup kinds added to Infos array");
diff --git a/lib/Target/RISCV/MCTargetDesc/RISCVELFObjectWriter.cpp b/lib/Target/RISCV/MCTargetDesc/RISCVELFObjectWriter.cpp
index 9b88614aa69384255f48a9c905c095c37acd1931..4cbbbcbc1b1d90f563dd25f95eb25db0a4ac3207 100644
--- a/lib/Target/RISCV/MCTargetDesc/RISCVELFObjectWriter.cpp
+++ b/lib/Target/RISCV/MCTargetDesc/RISCVELFObjectWriter.cpp
@@ -1,9 +1,8 @@
 //===-- RISCVELFObjectWriter.cpp - RISCV ELF Writer -----------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -96,6 +95,8 @@ unsigned RISCVELFObjectWriter::getRelocType(MCContext &Ctx,
     return ELF::R_RISCV_CALL;
   case RISCV::fixup_riscv_relax:
     return ELF::R_RISCV_RELAX;
+  case RISCV::fixup_riscv_align:
+    return ELF::R_RISCV_ALIGN;
   }
 }
 
diff --git a/lib/Target/RISCV/MCTargetDesc/RISCVELFStreamer.cpp b/lib/Target/RISCV/MCTargetDesc/RISCVELFStreamer.cpp
index a6ba1e41e9644cb01919091c2cb2c35a46a4f0ee..f7eae0b82ba91a475ae4ac86fcb2823a454a50d3 100644
--- a/lib/Target/RISCV/MCTargetDesc/RISCVELFStreamer.cpp
+++ b/lib/Target/RISCV/MCTargetDesc/RISCVELFStreamer.cpp
@@ -1,9 +1,8 @@
 //===-- RISCVELFStreamer.cpp - RISCV ELF Target Streamer Methods ----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/RISCV/MCTargetDesc/RISCVELFStreamer.h b/lib/Target/RISCV/MCTargetDesc/RISCVELFStreamer.h
index 1f36bbc4388242e06c3adb04666f20d79804c3b0..138df786eaf3f6380b87027b4f85fafbe33dc904 100644
--- a/lib/Target/RISCV/MCTargetDesc/RISCVELFStreamer.h
+++ b/lib/Target/RISCV/MCTargetDesc/RISCVELFStreamer.h
@@ -1,9 +1,8 @@
 //===-- RISCVELFStreamer.h - RISCV ELF Target Streamer ---------*- C++ -*--===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/RISCV/MCTargetDesc/RISCVFixupKinds.h b/lib/Target/RISCV/MCTargetDesc/RISCVFixupKinds.h
index 6a1224be774e46b14a390946da3c024055a435a2..3748a1fee7aea721533fc705cffcb16c413a3641 100644
--- a/lib/Target/RISCV/MCTargetDesc/RISCVFixupKinds.h
+++ b/lib/Target/RISCV/MCTargetDesc/RISCVFixupKinds.h
@@ -1,9 +1,8 @@
 //===-- RISCVFixupKinds.h - RISCV Specific Fixup Entries --------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -53,6 +52,10 @@ enum Fixups {
   // fixup_riscv_relax - Used to generate an R_RISCV_RELAX relocation type,
   // which indicates the linker may relax the instruction pair.
   fixup_riscv_relax,
+  // fixup_riscv_align - Used to generate an R_RISCV_ALIGN relocation type,
+  // which indicates the linker should fixup the alignment after linker
+  // relaxation.
+  fixup_riscv_align,
 
   // fixup_riscv_invalid - used as a sentinel and a marker, must be last fixup
   fixup_riscv_invalid,
diff --git a/lib/Target/RISCV/MCTargetDesc/RISCVMCAsmInfo.cpp b/lib/Target/RISCV/MCTargetDesc/RISCVMCAsmInfo.cpp
index 780dae410cd0d82a17583a87e830d86d6e02fa67..51106cd4e64be5a777093cdbc51119c79f36c590 100644
--- a/lib/Target/RISCV/MCTargetDesc/RISCVMCAsmInfo.cpp
+++ b/lib/Target/RISCV/MCTargetDesc/RISCVMCAsmInfo.cpp
@@ -1,9 +1,8 @@
 //===-- RISCVMCAsmInfo.cpp - RISCV Asm properties -------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/RISCV/MCTargetDesc/RISCVMCAsmInfo.h b/lib/Target/RISCV/MCTargetDesc/RISCVMCAsmInfo.h
index 901a1eba8af25f8e601180e0198381d0780f7ceb..043fdb7c08c0af1123dd7bb62c4cdfae86cdfccb 100644
--- a/lib/Target/RISCV/MCTargetDesc/RISCVMCAsmInfo.h
+++ b/lib/Target/RISCV/MCTargetDesc/RISCVMCAsmInfo.h
@@ -1,9 +1,8 @@
 //===-- RISCVMCAsmInfo.h - RISCV Asm Info ----------------------*- C++ -*--===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/RISCV/MCTargetDesc/RISCVMCCodeEmitter.cpp b/lib/Target/RISCV/MCTargetDesc/RISCVMCCodeEmitter.cpp
index c5a4ffc0e360c04050e2098006c4bb6ae1972271..4c4f9cf927af807657fbef6ab63579829cdf7ccc 100644
--- a/lib/Target/RISCV/MCTargetDesc/RISCVMCCodeEmitter.cpp
+++ b/lib/Target/RISCV/MCTargetDesc/RISCVMCCodeEmitter.cpp
@@ -1,9 +1,8 @@
 //===-- RISCVMCCodeEmitter.cpp - Convert RISCV code to machine code -------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -205,6 +204,7 @@ unsigned RISCVMCCodeEmitter::getImmOpValue(const MCInst &MI, unsigned OpNo,
   const MCExpr *Expr = MO.getExpr();
   MCExpr::ExprKind Kind = Expr->getKind();
   RISCV::Fixups FixupKind = RISCV::fixup_riscv_invalid;
+  bool RelaxCandidate = false;
   if (Kind == MCExpr::Target) {
     const RISCVMCExpr *RVExpr = cast<RISCVMCExpr>(Expr);
 
@@ -219,9 +219,11 @@ unsigned RISCVMCCodeEmitter::getImmOpValue(const MCInst &MI, unsigned OpNo,
         FixupKind = RISCV::fixup_riscv_lo12_s;
       else
         llvm_unreachable("VK_RISCV_LO used with unexpected instruction format");
+      RelaxCandidate = true;
       break;
     case RISCVMCExpr::VK_RISCV_HI:
       FixupKind = RISCV::fixup_riscv_hi20;
+      RelaxCandidate = true;
       break;
     case RISCVMCExpr::VK_RISCV_PCREL_LO:
       if (MIFrm == RISCVII::InstFormatI)
@@ -231,12 +233,15 @@ unsigned RISCVMCCodeEmitter::getImmOpValue(const MCInst &MI, unsigned OpNo,
       else
         llvm_unreachable(
             "VK_RISCV_PCREL_LO used with unexpected instruction format");
+      RelaxCandidate = true;
       break;
     case RISCVMCExpr::VK_RISCV_PCREL_HI:
       FixupKind = RISCV::fixup_riscv_pcrel_hi20;
+      RelaxCandidate = true;
       break;
     case RISCVMCExpr::VK_RISCV_CALL:
       FixupKind = RISCV::fixup_riscv_call;
+      RelaxCandidate = true;
       break;
     }
   } else if (Kind == MCExpr::SymbolRef &&
@@ -258,13 +263,15 @@ unsigned RISCVMCCodeEmitter::getImmOpValue(const MCInst &MI, unsigned OpNo,
       MCFixup::create(0, Expr, MCFixupKind(FixupKind), MI.getLoc()));
   ++MCNumFixups;
 
-  if (EnableRelax) {
-    if (FixupKind == RISCV::fixup_riscv_call) {
-      Fixups.push_back(
-      MCFixup::create(0, Expr, MCFixupKind(RISCV::fixup_riscv_relax),
-                      MI.getLoc()));
-      ++MCNumFixups;
-    }
+  // Ensure an R_RISCV_RELAX relocation will be emitted if linker relaxation is
+  // enabled and the current fixup will result in a relocation that may be
+  // relaxed.
+  if (EnableRelax && RelaxCandidate) {
+    const MCConstantExpr *Dummy = MCConstantExpr::create(0, Ctx);
+    Fixups.push_back(
+    MCFixup::create(0, Dummy, MCFixupKind(RISCV::fixup_riscv_relax),
+                    MI.getLoc()));
+    ++MCNumFixups;
   }
 
   return 0;
diff --git a/lib/Target/RISCV/MCTargetDesc/RISCVMCExpr.cpp b/lib/Target/RISCV/MCTargetDesc/RISCVMCExpr.cpp
index ad8357f9cfc96a9ec442052c3a44b0dca0fa5ea8..436b6fa540ab0ab92fbcc00c41d9916bc14428aa 100644
--- a/lib/Target/RISCV/MCTargetDesc/RISCVMCExpr.cpp
+++ b/lib/Target/RISCV/MCTargetDesc/RISCVMCExpr.cpp
@@ -1,9 +1,8 @@
 //===-- RISCVMCExpr.cpp - RISCV specific MC expression classes ------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -14,6 +13,7 @@
 
 #include "RISCV.h"
 #include "RISCVMCExpr.h"
+#include "RISCVFixupKinds.h"
 #include "llvm/MC/MCAssembler.h"
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCStreamer.h"
@@ -40,9 +40,90 @@ void RISCVMCExpr::printImpl(raw_ostream &OS, const MCAsmInfo *MAI) const {
     OS << ')';
 }
 
+const MCFixup *RISCVMCExpr::getPCRelHiFixup() const {
+  MCValue AUIPCLoc;
+  if (!getSubExpr()->evaluateAsRelocatable(AUIPCLoc, nullptr, nullptr))
+    return nullptr;
+
+  const MCSymbolRefExpr *AUIPCSRE = AUIPCLoc.getSymA();
+  if (!AUIPCSRE)
+    return nullptr;
+
+  const auto *DF =
+      dyn_cast_or_null<MCDataFragment>(AUIPCSRE->findAssociatedFragment());
+  if (!DF)
+    return nullptr;
+
+  const MCSymbol *AUIPCSymbol = &AUIPCSRE->getSymbol();
+  for (const MCFixup &F : DF->getFixups()) {
+    if (F.getOffset() != AUIPCSymbol->getOffset())
+      continue;
+
+    switch ((unsigned)F.getKind()) {
+    default:
+      continue;
+    case RISCV::fixup_riscv_pcrel_hi20:
+      return &F;
+    }
+  }
+
+  return nullptr;
+}
+
+bool RISCVMCExpr::evaluatePCRelLo(MCValue &Res, const MCAsmLayout *Layout,
+                                  const MCFixup *Fixup) const {
+  // VK_RISCV_PCREL_LO has to be handled specially.  The MCExpr inside is
+  // actually the location of a auipc instruction with a VK_RISCV_PCREL_HI fixup
+  // pointing to the real target.  We need to generate an MCValue in the form of
+  // (<real target> + <offset from this fixup to the auipc fixup>).  The Fixup
+  // is pcrel relative to the VK_RISCV_PCREL_LO fixup, so we need to add the
+  // offset to the VK_RISCV_PCREL_HI Fixup from VK_RISCV_PCREL_LO to correct.
+  MCValue AUIPCLoc;
+  if (!getSubExpr()->evaluateAsValue(AUIPCLoc, *Layout))
+    return false;
+
+  const MCSymbolRefExpr *AUIPCSRE = AUIPCLoc.getSymA();
+  // Don't try to evaluate %pcrel_hi/%pcrel_lo pairs that cross fragment
+  // boundries.
+  if (!AUIPCSRE ||
+      findAssociatedFragment() != AUIPCSRE->findAssociatedFragment())
+    return false;
+
+  const MCSymbol *AUIPCSymbol = &AUIPCSRE->getSymbol();
+  if (!AUIPCSymbol)
+    return false;
+
+  const MCFixup *TargetFixup = getPCRelHiFixup();
+  if (!TargetFixup)
+    return false;
+
+  if ((unsigned)TargetFixup->getKind() != RISCV::fixup_riscv_pcrel_hi20)
+    return false;
+
+  MCValue Target;
+  if (!TargetFixup->getValue()->evaluateAsValue(Target, *Layout))
+    return false;
+
+  if (!Target.getSymA() || !Target.getSymA()->getSymbol().isInSection())
+    return false;
+
+  if (&Target.getSymA()->getSymbol().getSection() !=
+      findAssociatedFragment()->getParent())
+    return false;
+
+  uint64_t AUIPCOffset = AUIPCSymbol->getOffset();
+
+  Res = MCValue::get(Target.getSymA(), nullptr,
+                     Target.getConstant() + (Fixup->getOffset() - AUIPCOffset));
+  return true;
+}
+
 bool RISCVMCExpr::evaluateAsRelocatableImpl(MCValue &Res,
                                             const MCAsmLayout *Layout,
                                             const MCFixup *Fixup) const {
+  if (Kind == VK_RISCV_PCREL_LO && evaluatePCRelLo(Res, Layout, Fixup))
+    return true;
+
   if (!getSubExpr()->evaluateAsRelocatable(Res, Layout, Fixup))
     return false;
 
diff --git a/lib/Target/RISCV/MCTargetDesc/RISCVMCExpr.h b/lib/Target/RISCV/MCTargetDesc/RISCVMCExpr.h
index d2e0f6b6cdae2a26562885995ee23f1d959091c0..861acc79f27192cdff1fb9ea8cd569488bcde21d 100644
--- a/lib/Target/RISCV/MCTargetDesc/RISCVMCExpr.h
+++ b/lib/Target/RISCV/MCTargetDesc/RISCVMCExpr.h
@@ -1,9 +1,8 @@
 //===-- RISCVMCExpr.h - RISCV specific MC expression classes ----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -39,6 +38,9 @@ private:
 
   int64_t evaluateAsInt64(int64_t Value) const;
 
+  bool evaluatePCRelLo(MCValue &Res, const MCAsmLayout *Layout,
+                       const MCFixup *Fixup) const;
+
   explicit RISCVMCExpr(const MCExpr *Expr, VariantKind Kind)
       : Expr(Expr), Kind(Kind) {}
 
@@ -50,6 +52,13 @@ public:
 
   const MCExpr *getSubExpr() const { return Expr; }
 
+  /// Get the MCExpr of the VK_RISCV_PCREL_HI Fixup that the
+  /// VK_RISCV_PCREL_LO points to.
+  ///
+  /// \returns nullptr if this isn't a VK_RISCV_PCREL_LO pointing to a
+  /// VK_RISCV_PCREL_HI.
+  const MCFixup *getPCRelHiFixup() const;
+
   void printImpl(raw_ostream &OS, const MCAsmInfo *MAI) const override;
   bool evaluateAsRelocatableImpl(MCValue &Res, const MCAsmLayout *Layout,
                                  const MCFixup *Fixup) const override;
diff --git a/lib/Target/RISCV/MCTargetDesc/RISCVMCTargetDesc.cpp b/lib/Target/RISCV/MCTargetDesc/RISCVMCTargetDesc.cpp
index 133f3cd3d39ae128d004963108b7a8ac17c1f380..9209aeadd7423e01a8b13fdc0afd8d7a4bd37337 100644
--- a/lib/Target/RISCV/MCTargetDesc/RISCVMCTargetDesc.cpp
+++ b/lib/Target/RISCV/MCTargetDesc/RISCVMCTargetDesc.cpp
@@ -1,9 +1,8 @@
 //===-- RISCVMCTargetDesc.cpp - RISCV Target Descriptions -----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
diff --git a/lib/Target/RISCV/MCTargetDesc/RISCVMCTargetDesc.h b/lib/Target/RISCV/MCTargetDesc/RISCVMCTargetDesc.h
index 0228253c08cb279899e06a338bb6a6dee9342013..7672863128bb6f1b693519376182bde3d3b67688 100644
--- a/lib/Target/RISCV/MCTargetDesc/RISCVMCTargetDesc.h
+++ b/lib/Target/RISCV/MCTargetDesc/RISCVMCTargetDesc.h
@@ -1,9 +1,8 @@
 //===-- RISCVMCTargetDesc.h - RISCV Target Descriptions ---------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/RISCV/MCTargetDesc/RISCVTargetStreamer.cpp b/lib/Target/RISCV/MCTargetDesc/RISCVTargetStreamer.cpp
index 8d5ef3dbd17f99e502b07ef2e8324495136faaf1..913e1f744192cb791208c2d1a2d0d8bc28ef21dc 100644
--- a/lib/Target/RISCV/MCTargetDesc/RISCVTargetStreamer.cpp
+++ b/lib/Target/RISCV/MCTargetDesc/RISCVTargetStreamer.cpp
@@ -1,9 +1,8 @@
 //===-- RISCVTargetStreamer.cpp - RISCV Target Streamer Methods -----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/RISCV/MCTargetDesc/RISCVTargetStreamer.h b/lib/Target/RISCV/MCTargetDesc/RISCVTargetStreamer.h
index 74ec9e303933d1260fa6947a9e325e149d718f30..1becc134b2a23abea4146b343565bc1b4ef3c6a5 100644
--- a/lib/Target/RISCV/MCTargetDesc/RISCVTargetStreamer.h
+++ b/lib/Target/RISCV/MCTargetDesc/RISCVTargetStreamer.h
@@ -1,9 +1,8 @@
 //===-- RISCVTargetStreamer.h - RISCV Target Streamer ----------*- C++ -*--===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/RISCV/RISCV.h b/lib/Target/RISCV/RISCV.h
index b25aee46200da6ec330b3c67a6d4dd93cdfe684f..834a1d17114359726babb684fc6b17404a5eb288 100644
--- a/lib/Target/RISCV/RISCV.h
+++ b/lib/Target/RISCV/RISCV.h
@@ -1,9 +1,8 @@
 //===-- RISCV.h - Top-level interface for RISCV -----------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/RISCV/RISCV.td b/lib/Target/RISCV/RISCV.td
index 0e86e2bc5e98c78c20425a7e51f4fb58376de8c8..3936da677a5f18532fbf77cd1abf1ce35eaebd02 100644
--- a/lib/Target/RISCV/RISCV.td
+++ b/lib/Target/RISCV/RISCV.td
@@ -1,9 +1,8 @@
 //===-- RISCV.td - Describe the RISCV Target Machine -------*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/RISCV/RISCVAsmPrinter.cpp b/lib/Target/RISCV/RISCVAsmPrinter.cpp
index bdf8e5d840b308d26c5b95aaa93233a0fb50f433..30cbb1e9b6a73d45a653899813e4c6d126b2d293 100644
--- a/lib/Target/RISCV/RISCVAsmPrinter.cpp
+++ b/lib/Target/RISCV/RISCVAsmPrinter.cpp
@@ -1,9 +1,8 @@
 //===-- RISCVAsmPrinter.cpp - RISCV LLVM assembly writer ------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/RISCV/RISCVCallingConv.td b/lib/Target/RISCV/RISCVCallingConv.td
index ef146258c3834424f882f97cdaffe251b64f50c3..65106029881425f1b7a4b70c9b15097da6ed06be 100644
--- a/lib/Target/RISCV/RISCVCallingConv.td
+++ b/lib/Target/RISCV/RISCVCallingConv.td
@@ -1,9 +1,8 @@
 //===-- RISCVCallingConv.td - Calling Conventions RISCV ----*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/RISCV/RISCVExpandPseudoInsts.cpp b/lib/Target/RISCV/RISCVExpandPseudoInsts.cpp
index 35c185aa5edd3bae2e7dcc3b889be06d0b48bdb0..d4190cbd0b9c9ee12afed82161bd7fafbe3150b0 100644
--- a/lib/Target/RISCV/RISCVExpandPseudoInsts.cpp
+++ b/lib/Target/RISCV/RISCVExpandPseudoInsts.cpp
@@ -1,9 +1,8 @@
 //===-- RISCVExpandPseudoInsts.cpp - Expand pseudo instructions -----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -87,6 +86,9 @@ bool RISCVExpandPseudo::expandMI(MachineBasicBlock &MBB,
   case RISCV::PseudoAtomicLoadNand32:
     return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Nand, false, 32,
                              NextMBBI);
+  case RISCV::PseudoAtomicLoadNand64:
+    return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Nand, false, 64,
+                             NextMBBI);
   case RISCV::PseudoMaskedAtomicSwap32:
     return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Xchg, true, 32,
                              NextMBBI);
@@ -111,6 +113,8 @@ bool RISCVExpandPseudo::expandMI(MachineBasicBlock &MBB,
                                 NextMBBI);
   case RISCV::PseudoCmpXchg32:
     return expandAtomicCmpXchg(MBB, MBBI, false, 32, NextMBBI);
+  case RISCV::PseudoCmpXchg64:
+    return expandAtomicCmpXchg(MBB, MBBI, false, 64, NextMBBI);
   case RISCV::PseudoMaskedCmpXchg32:
     return expandAtomicCmpXchg(MBB, MBBI, true, 32, NextMBBI);
   }
@@ -152,12 +156,61 @@ static unsigned getSCForRMW32(AtomicOrdering Ordering) {
   }
 }
 
+static unsigned getLRForRMW64(AtomicOrdering Ordering) {
+  switch (Ordering) {
+  default:
+    llvm_unreachable("Unexpected AtomicOrdering");
+  case AtomicOrdering::Monotonic:
+    return RISCV::LR_D;
+  case AtomicOrdering::Acquire:
+    return RISCV::LR_D_AQ;
+  case AtomicOrdering::Release:
+    return RISCV::LR_D;
+  case AtomicOrdering::AcquireRelease:
+    return RISCV::LR_D_AQ;
+  case AtomicOrdering::SequentiallyConsistent:
+    return RISCV::LR_D_AQ_RL;
+  }
+}
+
+static unsigned getSCForRMW64(AtomicOrdering Ordering) {
+  switch (Ordering) {
+  default:
+    llvm_unreachable("Unexpected AtomicOrdering");
+  case AtomicOrdering::Monotonic:
+    return RISCV::SC_D;
+  case AtomicOrdering::Acquire:
+    return RISCV::SC_D;
+  case AtomicOrdering::Release:
+    return RISCV::SC_D_RL;
+  case AtomicOrdering::AcquireRelease:
+    return RISCV::SC_D_RL;
+  case AtomicOrdering::SequentiallyConsistent:
+    return RISCV::SC_D_AQ_RL;
+  }
+}
+
+static unsigned getLRForRMW(AtomicOrdering Ordering, int Width) {
+  if (Width == 32)
+    return getLRForRMW32(Ordering);
+  if (Width == 64)
+    return getLRForRMW64(Ordering);
+  llvm_unreachable("Unexpected LR width\n");
+}
+
+static unsigned getSCForRMW(AtomicOrdering Ordering, int Width) {
+  if (Width == 32)
+    return getSCForRMW32(Ordering);
+  if (Width == 64)
+    return getSCForRMW64(Ordering);
+  llvm_unreachable("Unexpected SC width\n");
+}
+
 static void doAtomicBinOpExpansion(const RISCVInstrInfo *TII, MachineInstr &MI,
                                    DebugLoc DL, MachineBasicBlock *ThisMBB,
                                    MachineBasicBlock *LoopMBB,
                                    MachineBasicBlock *DoneMBB,
                                    AtomicRMWInst::BinOp BinOp, int Width) {
-  assert(Width == 32 && "RV64 atomic expansion currently unsupported");
   unsigned DestReg = MI.getOperand(0).getReg();
   unsigned ScratchReg = MI.getOperand(1).getReg();
   unsigned AddrReg = MI.getOperand(2).getReg();
@@ -166,11 +219,11 @@ static void doAtomicBinOpExpansion(const RISCVInstrInfo *TII, MachineInstr &MI,
       static_cast<AtomicOrdering>(MI.getOperand(4).getImm());
 
   // .loop:
-  //   lr.w dest, (addr)
+  //   lr.[w|d] dest, (addr)
   //   binop scratch, dest, val
-  //   sc.w scratch, scratch, (addr)
+  //   sc.[w|d] scratch, scratch, (addr)
   //   bnez scratch, loop
-  BuildMI(LoopMBB, DL, TII->get(getLRForRMW32(Ordering)), DestReg)
+  BuildMI(LoopMBB, DL, TII->get(getLRForRMW(Ordering, Width)), DestReg)
       .addReg(AddrReg);
   switch (BinOp) {
   default:
@@ -184,7 +237,7 @@ static void doAtomicBinOpExpansion(const RISCVInstrInfo *TII, MachineInstr &MI,
         .addImm(-1);
     break;
   }
-  BuildMI(LoopMBB, DL, TII->get(getSCForRMW32(Ordering)), ScratchReg)
+  BuildMI(LoopMBB, DL, TII->get(getSCForRMW(Ordering, Width)), ScratchReg)
       .addReg(AddrReg)
       .addReg(ScratchReg);
   BuildMI(LoopMBB, DL, TII->get(RISCV::BNE))
@@ -219,7 +272,7 @@ static void doMaskedAtomicBinOpExpansion(
     const RISCVInstrInfo *TII, MachineInstr &MI, DebugLoc DL,
     MachineBasicBlock *ThisMBB, MachineBasicBlock *LoopMBB,
     MachineBasicBlock *DoneMBB, AtomicRMWInst::BinOp BinOp, int Width) {
-  assert(Width == 32 && "RV64 atomic expansion currently unsupported");
+  assert(Width == 32 && "Should never need to expand masked 64-bit operations");
   unsigned DestReg = MI.getOperand(0).getReg();
   unsigned ScratchReg = MI.getOperand(1).getReg();
   unsigned AddrReg = MI.getOperand(2).getReg();
@@ -333,7 +386,7 @@ bool RISCVExpandPseudo::expandAtomicMinMaxOp(
     MachineBasicBlock::iterator &NextMBBI) {
   assert(IsMasked == true &&
          "Should only need to expand masked atomic max/min");
-  assert(Width == 32 && "RV64 atomic expansion currently unsupported");
+  assert(Width == 32 && "Should never need to expand masked 64-bit operations");
 
   MachineInstr &MI = *MBBI;
   DebugLoc DL = MI.getDebugLoc();
@@ -451,7 +504,6 @@ bool RISCVExpandPseudo::expandAtomicMinMaxOp(
 bool RISCVExpandPseudo::expandAtomicCmpXchg(
     MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, bool IsMasked,
     int Width, MachineBasicBlock::iterator &NextMBBI) {
-  assert(Width == 32 && "RV64 atomic expansion currently unsupported");
   MachineInstr &MI = *MBBI;
   DebugLoc DL = MI.getDebugLoc();
   MachineFunction *MF = MBB.getParent();
@@ -483,18 +535,18 @@ bool RISCVExpandPseudo::expandAtomicCmpXchg(
 
   if (!IsMasked) {
     // .loophead:
-    //   lr.w dest, (addr)
+    //   lr.[w|d] dest, (addr)
     //   bne dest, cmpval, done
-    BuildMI(LoopHeadMBB, DL, TII->get(getLRForRMW32(Ordering)), DestReg)
+    BuildMI(LoopHeadMBB, DL, TII->get(getLRForRMW(Ordering, Width)), DestReg)
         .addReg(AddrReg);
     BuildMI(LoopHeadMBB, DL, TII->get(RISCV::BNE))
         .addReg(DestReg)
         .addReg(CmpValReg)
         .addMBB(DoneMBB);
     // .looptail:
-    //   sc.w scratch, newval, (addr)
+    //   sc.[w|d] scratch, newval, (addr)
     //   bnez scratch, loophead
-    BuildMI(LoopTailMBB, DL, TII->get(getSCForRMW32(Ordering)), ScratchReg)
+    BuildMI(LoopTailMBB, DL, TII->get(getSCForRMW(Ordering, Width)), ScratchReg)
         .addReg(AddrReg)
         .addReg(NewValReg);
     BuildMI(LoopTailMBB, DL, TII->get(RISCV::BNE))
@@ -507,7 +559,7 @@ bool RISCVExpandPseudo::expandAtomicCmpXchg(
     //   and scratch, dest, mask
     //   bne scratch, cmpval, done
     unsigned MaskReg = MI.getOperand(5).getReg();
-    BuildMI(LoopHeadMBB, DL, TII->get(getLRForRMW32(Ordering)), DestReg)
+    BuildMI(LoopHeadMBB, DL, TII->get(getLRForRMW(Ordering, Width)), DestReg)
         .addReg(AddrReg);
     BuildMI(LoopHeadMBB, DL, TII->get(RISCV::AND), ScratchReg)
         .addReg(DestReg)
@@ -525,7 +577,7 @@ bool RISCVExpandPseudo::expandAtomicCmpXchg(
     //   bnez scratch, loophead
     insertMaskedMerge(TII, DL, LoopTailMBB, ScratchReg, DestReg, NewValReg,
                       MaskReg, ScratchReg);
-    BuildMI(LoopTailMBB, DL, TII->get(getSCForRMW32(Ordering)), ScratchReg)
+    BuildMI(LoopTailMBB, DL, TII->get(getSCForRMW(Ordering, Width)), ScratchReg)
         .addReg(AddrReg)
         .addReg(ScratchReg);
     BuildMI(LoopTailMBB, DL, TII->get(RISCV::BNE))
diff --git a/lib/Target/RISCV/RISCVFrameLowering.cpp b/lib/Target/RISCV/RISCVFrameLowering.cpp
index 74417899c8da80dbf38e4460877b4ccf3584be72..6e0703c7b02139df4433669c7ac368be164a4c3e 100644
--- a/lib/Target/RISCV/RISCVFrameLowering.cpp
+++ b/lib/Target/RISCV/RISCVFrameLowering.cpp
@@ -1,9 +1,8 @@
 //===-- RISCVFrameLowering.cpp - RISCV Frame Information ------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/RISCV/RISCVFrameLowering.h b/lib/Target/RISCV/RISCVFrameLowering.h
index ca653c2b9f17ccdd5480b63e82993804d1148da8..0e045c3ff853857caceadb79c2b2e68e112132c0 100644
--- a/lib/Target/RISCV/RISCVFrameLowering.h
+++ b/lib/Target/RISCV/RISCVFrameLowering.h
@@ -1,9 +1,8 @@
 //===-- RISCVFrameLowering.h - Define frame lowering for RISCV -*- C++ -*--===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/RISCV/RISCVISelDAGToDAG.cpp b/lib/Target/RISCV/RISCVISelDAGToDAG.cpp
index aa80365feb830f9e23a6f35d1cd784e1b6deab96..4b455448b380b39bc35fe5d87493c2d96ee789fb 100644
--- a/lib/Target/RISCV/RISCVISelDAGToDAG.cpp
+++ b/lib/Target/RISCV/RISCVISelDAGToDAG.cpp
@@ -1,9 +1,8 @@
 //===-- RISCVISelDAGToDAG.cpp - A dag to dag inst selector for RISCV ------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -156,6 +155,7 @@ void RISCVDAGToDAGISel::Select(SDNode *Node) {
         return;
       }
     }
+    break;
   }
   }
 
diff --git a/lib/Target/RISCV/RISCVISelLowering.cpp b/lib/Target/RISCV/RISCVISelLowering.cpp
index e78085edac38e39073636f627adf8709056bb54d..5f9d5ef609e22cf7d6a31df0862c49aa48901707 100644
--- a/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -1,9 +1,8 @@
 //===-- RISCVISelLowering.cpp - RISCV DAG Lowering Implementation  --------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -80,6 +79,12 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
   for (auto VT : {MVT::i1, MVT::i8, MVT::i16})
     setOperationAction(ISD::SIGN_EXTEND_INREG, VT, Expand);
 
+  if (Subtarget.is64Bit()) {
+    setOperationAction(ISD::SHL, MVT::i32, Custom);
+    setOperationAction(ISD::SRA, MVT::i32, Custom);
+    setOperationAction(ISD::SRL, MVT::i32, Custom);
+  }
+
   if (!Subtarget.hasStdExtM()) {
     setOperationAction(ISD::MUL, XLenVT, Expand);
     setOperationAction(ISD::MULHS, XLenVT, Expand);
@@ -90,6 +95,12 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
     setOperationAction(ISD::UREM, XLenVT, Expand);
   }
 
+  if (Subtarget.is64Bit() && Subtarget.hasStdExtM()) {
+    setOperationAction(ISD::SDIV, MVT::i32, Custom);
+    setOperationAction(ISD::UDIV, MVT::i32, Custom);
+    setOperationAction(ISD::UREM, MVT::i32, Custom);
+  }
+
   setOperationAction(ISD::SDIVREM, XLenVT, Expand);
   setOperationAction(ISD::UDIVREM, XLenVT, Expand);
   setOperationAction(ISD::SMUL_LOHI, XLenVT, Expand);
@@ -126,6 +137,9 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
       setOperationAction(Op, MVT::f32, Expand);
   }
 
+  if (Subtarget.hasStdExtF() && Subtarget.is64Bit())
+    setOperationAction(ISD::BITCAST, MVT::i32, Custom);
+
   if (Subtarget.hasStdExtD()) {
     setOperationAction(ISD::FMINNUM, MVT::f64, Legal);
     setOperationAction(ISD::FMAXNUM, MVT::f64, Legal);
@@ -327,6 +341,17 @@ SDValue RISCVTargetLowering::LowerOperation(SDValue Op,
     return lowerFRAMEADDR(Op, DAG);
   case ISD::RETURNADDR:
     return lowerRETURNADDR(Op, DAG);
+  case ISD::BITCAST: {
+    assert(Subtarget.is64Bit() && Subtarget.hasStdExtF() &&
+           "Unexpected custom legalisation");
+    SDLoc DL(Op);
+    SDValue Op0 = Op.getOperand(0);
+    if (Op.getValueType() != MVT::f32 || Op0.getValueType() != MVT::i32)
+      return SDValue();
+    SDValue NewOp0 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op0);
+    SDValue FPConv = DAG.getNode(RISCVISD::FMV_W_X_RV64, DL, MVT::f32, NewOp0);
+    return FPConv;
+  }
   }
 }
 
@@ -506,24 +531,192 @@ SDValue RISCVTargetLowering::lowerRETURNADDR(SDValue Op,
   return DAG.getCopyFromReg(DAG.getEntryNode(), DL, Reg, XLenVT);
 }
 
+// Returns the opcode of the target-specific SDNode that implements the 32-bit
+// form of the given Opcode.
+static RISCVISD::NodeType getRISCVWOpcode(unsigned Opcode) {
+  switch (Opcode) {
+  default:
+    llvm_unreachable("Unexpected opcode");
+  case ISD::SHL:
+    return RISCVISD::SLLW;
+  case ISD::SRA:
+    return RISCVISD::SRAW;
+  case ISD::SRL:
+    return RISCVISD::SRLW;
+  case ISD::SDIV:
+    return RISCVISD::DIVW;
+  case ISD::UDIV:
+    return RISCVISD::DIVUW;
+  case ISD::UREM:
+    return RISCVISD::REMUW;
+  }
+}
+
+// Converts the given 32-bit operation to a target-specific SelectionDAG node.
+// Because i32 isn't a legal type for RV64, these operations would otherwise
+// be promoted to i64, making it difficult to select the SLLW/DIVUW/.../*W
+// later one because the fact the operation was originally of type i32 is
+// lost.
+static SDValue customLegalizeToWOp(SDNode *N, SelectionDAG &DAG) {
+  SDLoc DL(N);
+  RISCVISD::NodeType WOpcode = getRISCVWOpcode(N->getOpcode());
+  SDValue NewOp0 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(0));
+  SDValue NewOp1 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
+  SDValue NewRes = DAG.getNode(WOpcode, DL, MVT::i64, NewOp0, NewOp1);
+  // ReplaceNodeResults requires we maintain the same type for the return value.
+  return DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, NewRes);
+}
+
+void RISCVTargetLowering::ReplaceNodeResults(SDNode *N,
+                                             SmallVectorImpl<SDValue> &Results,
+                                             SelectionDAG &DAG) const {
+  SDLoc DL(N);
+  switch (N->getOpcode()) {
+  default:
+    llvm_unreachable("Don't know how to custom type legalize this operation!");
+  case ISD::SHL:
+  case ISD::SRA:
+  case ISD::SRL:
+    assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
+           "Unexpected custom legalisation");
+    if (N->getOperand(1).getOpcode() == ISD::Constant)
+      return;
+    Results.push_back(customLegalizeToWOp(N, DAG));
+    break;
+  case ISD::SDIV:
+  case ISD::UDIV:
+  case ISD::UREM:
+    assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
+           Subtarget.hasStdExtM() && "Unexpected custom legalisation");
+    if (N->getOperand(0).getOpcode() == ISD::Constant ||
+        N->getOperand(1).getOpcode() == ISD::Constant)
+      return;
+    Results.push_back(customLegalizeToWOp(N, DAG));
+    break;
+  case ISD::BITCAST: {
+    assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
+           Subtarget.hasStdExtF() && "Unexpected custom legalisation");
+    SDLoc DL(N);
+    SDValue Op0 = N->getOperand(0);
+    if (Op0.getValueType() != MVT::f32)
+      return;
+    SDValue FPConv =
+        DAG.getNode(RISCVISD::FMV_X_ANYEXTW_RV64, DL, MVT::i64, Op0);
+    Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, FPConv));
+    break;
+  }
+  }
+}
+
 SDValue RISCVTargetLowering::PerformDAGCombine(SDNode *N,
                                                DAGCombinerInfo &DCI) const {
+  SelectionDAG &DAG = DCI.DAG;
+
   switch (N->getOpcode()) {
   default:
     break;
   case RISCVISD::SplitF64: {
+    SDValue Op0 = N->getOperand(0);
     // If the input to SplitF64 is just BuildPairF64 then the operation is
     // redundant. Instead, use BuildPairF64's operands directly.
+    if (Op0->getOpcode() == RISCVISD::BuildPairF64)
+      return DCI.CombineTo(N, Op0.getOperand(0), Op0.getOperand(1));
+
+    SDLoc DL(N);
+    // This is a target-specific version of a DAGCombine performed in
+    // DAGCombiner::visitBITCAST. It performs the equivalent of:
+    // fold (bitconvert (fneg x)) -> (xor (bitconvert x), signbit)
+    // fold (bitconvert (fabs x)) -> (and (bitconvert x), (not signbit))
+    if (!(Op0.getOpcode() == ISD::FNEG || Op0.getOpcode() == ISD::FABS) ||
+        !Op0.getNode()->hasOneUse())
+      break;
+    SDValue NewSplitF64 =
+        DAG.getNode(RISCVISD::SplitF64, DL, DAG.getVTList(MVT::i32, MVT::i32),
+                    Op0.getOperand(0));
+    SDValue Lo = NewSplitF64.getValue(0);
+    SDValue Hi = NewSplitF64.getValue(1);
+    APInt SignBit = APInt::getSignMask(32);
+    if (Op0.getOpcode() == ISD::FNEG) {
+      SDValue NewHi = DAG.getNode(ISD::XOR, DL, MVT::i32, Hi,
+                                  DAG.getConstant(SignBit, DL, MVT::i32));
+      return DCI.CombineTo(N, Lo, NewHi);
+    }
+    assert(Op0.getOpcode() == ISD::FABS);
+    SDValue NewHi = DAG.getNode(ISD::AND, DL, MVT::i32, Hi,
+                                DAG.getConstant(~SignBit, DL, MVT::i32));
+    return DCI.CombineTo(N, Lo, NewHi);
+  }
+  case RISCVISD::SLLW:
+  case RISCVISD::SRAW:
+  case RISCVISD::SRLW: {
+    // Only the lower 32 bits of LHS and lower 5 bits of RHS are read.
+    SDValue LHS = N->getOperand(0);
+    SDValue RHS = N->getOperand(1);
+    APInt LHSMask = APInt::getLowBitsSet(LHS.getValueSizeInBits(), 32);
+    APInt RHSMask = APInt::getLowBitsSet(RHS.getValueSizeInBits(), 5);
+    if ((SimplifyDemandedBits(N->getOperand(0), LHSMask, DCI)) ||
+        (SimplifyDemandedBits(N->getOperand(1), RHSMask, DCI)))
+      return SDValue();
+    break;
+  }
+  case RISCVISD::FMV_X_ANYEXTW_RV64: {
+    SDLoc DL(N);
     SDValue Op0 = N->getOperand(0);
-    if (Op0->getOpcode() != RISCVISD::BuildPairF64)
+    // If the input to FMV_X_ANYEXTW_RV64 is just FMV_W_X_RV64 then the
+    // conversion is unnecessary and can be replaced with an ANY_EXTEND
+    // of the FMV_W_X_RV64 operand.
+    if (Op0->getOpcode() == RISCVISD::FMV_W_X_RV64) {
+      SDValue AExtOp =
+          DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op0.getOperand(0));
+      return DCI.CombineTo(N, AExtOp);
+    }
+
+    // This is a target-specific version of a DAGCombine performed in
+    // DAGCombiner::visitBITCAST. It performs the equivalent of:
+    // fold (bitconvert (fneg x)) -> (xor (bitconvert x), signbit)
+    // fold (bitconvert (fabs x)) -> (and (bitconvert x), (not signbit))
+    if (!(Op0.getOpcode() == ISD::FNEG || Op0.getOpcode() == ISD::FABS) ||
+        !Op0.getNode()->hasOneUse())
       break;
-    return DCI.CombineTo(N, Op0.getOperand(0), Op0.getOperand(1));
+    SDValue NewFMV = DAG.getNode(RISCVISD::FMV_X_ANYEXTW_RV64, DL, MVT::i64,
+                                 Op0.getOperand(0));
+    APInt SignBit = APInt::getSignMask(32).sext(64);
+    if (Op0.getOpcode() == ISD::FNEG) {
+      return DCI.CombineTo(N,
+                           DAG.getNode(ISD::XOR, DL, MVT::i64, NewFMV,
+                                       DAG.getConstant(SignBit, DL, MVT::i64)));
+    }
+    assert(Op0.getOpcode() == ISD::FABS);
+    return DCI.CombineTo(N,
+                         DAG.getNode(ISD::AND, DL, MVT::i64, NewFMV,
+                                     DAG.getConstant(~SignBit, DL, MVT::i64)));
   }
   }
 
   return SDValue();
 }
 
+unsigned RISCVTargetLowering::ComputeNumSignBitsForTargetNode(
+    SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
+    unsigned Depth) const {
+  switch (Op.getOpcode()) {
+  default:
+    break;
+  case RISCVISD::SLLW:
+  case RISCVISD::SRAW:
+  case RISCVISD::SRLW:
+  case RISCVISD::DIVW:
+  case RISCVISD::DIVUW:
+  case RISCVISD::REMUW:
+    // TODO: As the result is sign-extended, this is conservatively correct. A
+    // more precise answer could be calculated for SRAW depending on known
+    // bits in the shift amount.
+    return 33;
+  }
+
+  return 1;
+}
+
 static MachineBasicBlock *emitSplitF64Pseudo(MachineInstr &MI,
                                              MachineBasicBlock *BB) {
   assert(MI.getOpcode() == RISCV::SplitF64Pseudo && "Unexpected instruction");
@@ -739,7 +932,11 @@ static bool CC_RISCV(const DataLayout &DL, unsigned ValNo, MVT ValVT, MVT LocVT,
   assert(XLen == 32 || XLen == 64);
   MVT XLenVT = XLen == 32 ? MVT::i32 : MVT::i64;
   if (ValVT == MVT::f32) {
-    LocVT = MVT::i32;
+    LocVT = XLenVT;
+    LocInfo = CCValAssign::BCvt;
+  }
+  if (XLen == 64 && ValVT == MVT::f64) {
+    LocVT = MVT::i64;
     LocInfo = CCValAssign::BCvt;
   }
 
@@ -849,8 +1046,9 @@ static bool CC_RISCV(const DataLayout &DL, unsigned ValNo, MVT ValVT, MVT LocVT,
     return false;
   }
 
-  if (ValVT == MVT::f32) {
-    LocVT = MVT::f32;
+  // When an f32 or f64 is passed on the stack, no bit-conversion is needed.
+  if (ValVT == MVT::f32 || ValVT == MVT::f64) {
+    LocVT = ValVT;
     LocInfo = CCValAssign::Full;
   }
   State.addLoc(CCValAssign::getMem(ValNo, ValVT, StackOffset, LocVT, LocInfo));
@@ -912,6 +1110,10 @@ static SDValue convertLocVTToValVT(SelectionDAG &DAG, SDValue Val,
   case CCValAssign::Full:
     break;
   case CCValAssign::BCvt:
+    if (VA.getLocVT() == MVT::i64 && VA.getValVT() == MVT::f32) {
+      Val = DAG.getNode(RISCVISD::FMV_W_X_RV64, DL, MVT::f32, Val);
+      break;
+    }
     Val = DAG.getNode(ISD::BITCAST, DL, VA.getValVT(), Val);
     break;
   }
@@ -947,6 +1149,10 @@ static SDValue convertValVTToLocVT(SelectionDAG &DAG, SDValue Val,
   case CCValAssign::Full:
     break;
   case CCValAssign::BCvt:
+    if (VA.getLocVT() == MVT::i64 && VA.getValVT() == MVT::f32) {
+      Val = DAG.getNode(RISCVISD::FMV_X_ANYEXTW_RV64, DL, MVT::i64, Val);
+      break;
+    }
     Val = DAG.getNode(ISD::BITCAST, DL, LocVT, Val);
     break;
   }
@@ -973,6 +1179,7 @@ static SDValue unpackFromMemLoc(SelectionDAG &DAG, SDValue Chain,
     llvm_unreachable("Unexpected CCValAssign::LocInfo");
   case CCValAssign::Full:
   case CCValAssign::Indirect:
+  case CCValAssign::BCvt:
     ExtType = ISD::NON_EXTLOAD;
     break;
   }
@@ -1612,6 +1819,22 @@ const char *RISCVTargetLowering::getTargetNodeName(unsigned Opcode) const {
     return "RISCVISD::SplitF64";
   case RISCVISD::TAIL:
     return "RISCVISD::TAIL";
+  case RISCVISD::SLLW:
+    return "RISCVISD::SLLW";
+  case RISCVISD::SRAW:
+    return "RISCVISD::SRAW";
+  case RISCVISD::SRLW:
+    return "RISCVISD::SRLW";
+  case RISCVISD::DIVW:
+    return "RISCVISD::DIVW";
+  case RISCVISD::DIVUW:
+    return "RISCVISD::DIVUW";
+  case RISCVISD::REMUW:
+    return "RISCVISD::REMUW";
+  case RISCVISD::FMV_W_X_RV64:
+    return "RISCVISD::FMV_W_X_RV64";
+  case RISCVISD::FMV_X_ANYEXTW_RV64:
+    return "RISCVISD::FMV_X_ANYEXTW_RV64";
   }
   return nullptr;
 }
@@ -1654,6 +1877,12 @@ Instruction *RISCVTargetLowering::emitTrailingFence(IRBuilder<> &Builder,
 
 TargetLowering::AtomicExpansionKind
 RISCVTargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const {
+  // atomicrmw {fadd,fsub} must be expanded to use compare-exchange, as floating
+  // point operations can't be used in an lr/sc sequence without breaking the
+  // forward-progress guarantee.
+  if (AI->isFloatingPointOperation())
+    return AtomicExpansionKind::CmpXChg;
+
   unsigned Size = AI->getType()->getPrimitiveSizeInBits();
   if (Size == 8 || Size == 16)
     return AtomicExpansionKind::MaskedIntrinsic;
@@ -1661,37 +1890,74 @@ RISCVTargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const {
 }
 
 static Intrinsic::ID
-getIntrinsicForMaskedAtomicRMWBinOp32(AtomicRMWInst::BinOp BinOp) {
-  switch (BinOp) {
-  default:
-    llvm_unreachable("Unexpected AtomicRMW BinOp");
-  case AtomicRMWInst::Xchg:
-    return Intrinsic::riscv_masked_atomicrmw_xchg_i32;
-  case AtomicRMWInst::Add:
-    return Intrinsic::riscv_masked_atomicrmw_add_i32;
-  case AtomicRMWInst::Sub:
-    return Intrinsic::riscv_masked_atomicrmw_sub_i32;
-  case AtomicRMWInst::Nand:
-    return Intrinsic::riscv_masked_atomicrmw_nand_i32;
-  case AtomicRMWInst::Max:
-    return Intrinsic::riscv_masked_atomicrmw_max_i32;
-  case AtomicRMWInst::Min:
-    return Intrinsic::riscv_masked_atomicrmw_min_i32;
-  case AtomicRMWInst::UMax:
-    return Intrinsic::riscv_masked_atomicrmw_umax_i32;
-  case AtomicRMWInst::UMin:
-    return Intrinsic::riscv_masked_atomicrmw_umin_i32;
+getIntrinsicForMaskedAtomicRMWBinOp(unsigned XLen, AtomicRMWInst::BinOp BinOp) {
+  if (XLen == 32) {
+    switch (BinOp) {
+    default:
+      llvm_unreachable("Unexpected AtomicRMW BinOp");
+    case AtomicRMWInst::Xchg:
+      return Intrinsic::riscv_masked_atomicrmw_xchg_i32;
+    case AtomicRMWInst::Add:
+      return Intrinsic::riscv_masked_atomicrmw_add_i32;
+    case AtomicRMWInst::Sub:
+      return Intrinsic::riscv_masked_atomicrmw_sub_i32;
+    case AtomicRMWInst::Nand:
+      return Intrinsic::riscv_masked_atomicrmw_nand_i32;
+    case AtomicRMWInst::Max:
+      return Intrinsic::riscv_masked_atomicrmw_max_i32;
+    case AtomicRMWInst::Min:
+      return Intrinsic::riscv_masked_atomicrmw_min_i32;
+    case AtomicRMWInst::UMax:
+      return Intrinsic::riscv_masked_atomicrmw_umax_i32;
+    case AtomicRMWInst::UMin:
+      return Intrinsic::riscv_masked_atomicrmw_umin_i32;
+    }
+  }
+
+  if (XLen == 64) {
+    switch (BinOp) {
+    default:
+      llvm_unreachable("Unexpected AtomicRMW BinOp");
+    case AtomicRMWInst::Xchg:
+      return Intrinsic::riscv_masked_atomicrmw_xchg_i64;
+    case AtomicRMWInst::Add:
+      return Intrinsic::riscv_masked_atomicrmw_add_i64;
+    case AtomicRMWInst::Sub:
+      return Intrinsic::riscv_masked_atomicrmw_sub_i64;
+    case AtomicRMWInst::Nand:
+      return Intrinsic::riscv_masked_atomicrmw_nand_i64;
+    case AtomicRMWInst::Max:
+      return Intrinsic::riscv_masked_atomicrmw_max_i64;
+    case AtomicRMWInst::Min:
+      return Intrinsic::riscv_masked_atomicrmw_min_i64;
+    case AtomicRMWInst::UMax:
+      return Intrinsic::riscv_masked_atomicrmw_umax_i64;
+    case AtomicRMWInst::UMin:
+      return Intrinsic::riscv_masked_atomicrmw_umin_i64;
+    }
   }
+
+  llvm_unreachable("Unexpected XLen\n");
 }
 
 Value *RISCVTargetLowering::emitMaskedAtomicRMWIntrinsic(
     IRBuilder<> &Builder, AtomicRMWInst *AI, Value *AlignedAddr, Value *Incr,
     Value *Mask, Value *ShiftAmt, AtomicOrdering Ord) const {
-  Value *Ordering = Builder.getInt32(static_cast<uint32_t>(AI->getOrdering()));
+  unsigned XLen = Subtarget.getXLen();
+  Value *Ordering =
+      Builder.getIntN(XLen, static_cast<uint64_t>(AI->getOrdering()));
   Type *Tys[] = {AlignedAddr->getType()};
   Function *LrwOpScwLoop = Intrinsic::getDeclaration(
       AI->getModule(),
-      getIntrinsicForMaskedAtomicRMWBinOp32(AI->getOperation()), Tys);
+      getIntrinsicForMaskedAtomicRMWBinOp(XLen, AI->getOperation()), Tys);
+
+  if (XLen == 64) {
+    Incr = Builder.CreateSExt(Incr, Builder.getInt64Ty());
+    Mask = Builder.CreateSExt(Mask, Builder.getInt64Ty());
+    ShiftAmt = Builder.CreateSExt(ShiftAmt, Builder.getInt64Ty());
+  }
+
+  Value *Result;
 
   // Must pass the shift amount needed to sign extend the loaded value prior
   // to performing a signed comparison for min/max. ShiftAmt is the number of
@@ -1703,13 +1969,18 @@ Value *RISCVTargetLowering::emitMaskedAtomicRMWIntrinsic(
     const DataLayout &DL = AI->getModule()->getDataLayout();
     unsigned ValWidth =
         DL.getTypeStoreSizeInBits(AI->getValOperand()->getType());
-    Value *SextShamt = Builder.CreateSub(
-        Builder.getInt32(Subtarget.getXLen() - ValWidth), ShiftAmt);
-    return Builder.CreateCall(LrwOpScwLoop,
-                              {AlignedAddr, Incr, Mask, SextShamt, Ordering});
+    Value *SextShamt =
+        Builder.CreateSub(Builder.getIntN(XLen, XLen - ValWidth), ShiftAmt);
+    Result = Builder.CreateCall(LrwOpScwLoop,
+                                {AlignedAddr, Incr, Mask, SextShamt, Ordering});
+  } else {
+    Result =
+        Builder.CreateCall(LrwOpScwLoop, {AlignedAddr, Incr, Mask, Ordering});
   }
 
-  return Builder.CreateCall(LrwOpScwLoop, {AlignedAddr, Incr, Mask, Ordering});
+  if (XLen == 64)
+    Result = Builder.CreateTrunc(Result, Builder.getInt32Ty());
+  return Result;
 }
 
 TargetLowering::AtomicExpansionKind
@@ -1724,10 +1995,21 @@ RISCVTargetLowering::shouldExpandAtomicCmpXchgInIR(
 Value *RISCVTargetLowering::emitMaskedAtomicCmpXchgIntrinsic(
     IRBuilder<> &Builder, AtomicCmpXchgInst *CI, Value *AlignedAddr,
     Value *CmpVal, Value *NewVal, Value *Mask, AtomicOrdering Ord) const {
-  Value *Ordering = Builder.getInt32(static_cast<uint32_t>(Ord));
+  unsigned XLen = Subtarget.getXLen();
+  Value *Ordering = Builder.getIntN(XLen, static_cast<uint64_t>(Ord));
+  Intrinsic::ID CmpXchgIntrID = Intrinsic::riscv_masked_cmpxchg_i32;
+  if (XLen == 64) {
+    CmpVal = Builder.CreateSExt(CmpVal, Builder.getInt64Ty());
+    NewVal = Builder.CreateSExt(NewVal, Builder.getInt64Ty());
+    Mask = Builder.CreateSExt(Mask, Builder.getInt64Ty());
+    CmpXchgIntrID = Intrinsic::riscv_masked_cmpxchg_i64;
+  }
   Type *Tys[] = {AlignedAddr->getType()};
-  Function *MaskedCmpXchg = Intrinsic::getDeclaration(
-      CI->getModule(), Intrinsic::riscv_masked_cmpxchg_i32, Tys);
-  return Builder.CreateCall(MaskedCmpXchg,
-                            {AlignedAddr, CmpVal, NewVal, Mask, Ordering});
+  Function *MaskedCmpXchg =
+      Intrinsic::getDeclaration(CI->getModule(), CmpXchgIntrID, Tys);
+  Value *Result = Builder.CreateCall(
+      MaskedCmpXchg, {AlignedAddr, CmpVal, NewVal, Mask, Ordering});
+  if (XLen == 64)
+    Result = Builder.CreateTrunc(Result, Builder.getInt32Ty());
+  return Result;
 }
diff --git a/lib/Target/RISCV/RISCVISelLowering.h b/lib/Target/RISCV/RISCVISelLowering.h
index 6970900bb0622af57b46eb655ada4c6c4051cc07..12b63208543e18bdfbe6c62ca607fa88440f92ef 100644
--- a/lib/Target/RISCV/RISCVISelLowering.h
+++ b/lib/Target/RISCV/RISCVISelLowering.h
@@ -1,9 +1,8 @@
 //===-- RISCVISelLowering.h - RISCV DAG Lowering Interface ------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -32,7 +31,24 @@ enum NodeType : unsigned {
   SELECT_CC,
   BuildPairF64,
   SplitF64,
-  TAIL
+  TAIL,
+  // RV64I shifts, directly matching the semantics of the named RISC-V
+  // instructions.
+  SLLW,
+  SRAW,
+  SRLW,
+  // 32-bit operations from RV64M that can't be simply matched with a pattern
+  // at instruction selection time.
+  DIVW,
+  DIVUW,
+  REMUW,
+  // FPR32<->GPR transfer operations for RV64. Needed as an i32<->f32 bitcast
+  // is not legal on RV64. FMV_W_X_RV64 matches the semantics of the FMV.W.X.
+  // FMV_X_ANYEXTW_RV64 is similar to FMV.X.W but has an any-extended result.
+  // This is a more convenient semantic for producing dagcombines that remove
+  // unnecessary GPR->FPR->GPR moves.
+  FMV_W_X_RV64,
+  FMV_X_ANYEXTW_RV64
 };
 }
 
@@ -58,9 +74,16 @@ public:
 
   // Provide custom lowering hooks for some operations.
   SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override;
+  void ReplaceNodeResults(SDNode *N, SmallVectorImpl<SDValue> &Results,
+                          SelectionDAG &DAG) const override;
 
   SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const override;
 
+  unsigned ComputeNumSignBitsForTargetNode(SDValue Op,
+                                           const APInt &DemandedElts,
+                                           const SelectionDAG &DAG,
+                                           unsigned Depth) const override;
+
   // This method returns the name of a target specific DAG node.
   const char *getTargetNodeName(unsigned Opcode) const override;
 
diff --git a/lib/Target/RISCV/RISCVInstrFormats.td b/lib/Target/RISCV/RISCVInstrFormats.td
index ebd676a6056ef366917c7e8a01548d07d498f20f..f2c6230938ed55a4f3c1f3c223237b2419a15416 100644
--- a/lib/Target/RISCV/RISCVInstrFormats.td
+++ b/lib/Target/RISCV/RISCVInstrFormats.td
@@ -1,9 +1,8 @@
 //===-- RISCVInstrFormats.td - RISCV Instruction Formats ---*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/RISCV/RISCVInstrFormatsC.td b/lib/Target/RISCV/RISCVInstrFormatsC.td
index bda8bbb558eb6ce32c01605ea16e0a3504e18c3b..690bec5181e2aabdde9a959eaf85cdf7bf6d2ae5 100644
--- a/lib/Target/RISCV/RISCVInstrFormatsC.td
+++ b/lib/Target/RISCV/RISCVInstrFormatsC.td
@@ -1,9 +1,8 @@
 //===-- RISCVInstrFormatsC.td - RISCV C Instruction Formats --*- tablegen -*-=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/RISCV/RISCVInstrInfo.cpp b/lib/Target/RISCV/RISCVInstrInfo.cpp
index 76c74368ca111a0285831569a77b03a5f0217489..ddb976b47fbc4dcd0e3ef3a3164e8c8837c953a6 100644
--- a/lib/Target/RISCV/RISCVInstrInfo.cpp
+++ b/lib/Target/RISCV/RISCVInstrInfo.cpp
@@ -1,9 +1,8 @@
 //===-- RISCVInstrInfo.cpp - RISCV Instruction Information ------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -440,7 +439,8 @@ unsigned RISCVInstrInfo::getInstSizeInBytes(const MachineInstr &MI) const {
   case RISCV::PseudoCALL:
   case RISCV::PseudoTAIL:
     return 8;
-  case TargetOpcode::INLINEASM: {
+  case TargetOpcode::INLINEASM:
+  case TargetOpcode::INLINEASM_BR: {
     const MachineFunction &MF = *MI.getParent()->getParent();
     const auto &TM = static_cast<const RISCVTargetMachine &>(MF.getTarget());
     return getInlineAsmLength(MI.getOperand(0).getSymbolName(),
@@ -448,3 +448,16 @@ unsigned RISCVInstrInfo::getInstSizeInBytes(const MachineInstr &MI) const {
   }
   }
 }
+
+bool RISCVInstrInfo::isAsCheapAsAMove(const MachineInstr &MI) const {
+  const unsigned Opcode = MI.getOpcode();
+  switch(Opcode) {
+    default:
+      break;
+    case RISCV::ADDI:
+    case RISCV::ORI:
+    case RISCV::XORI:
+      return (MI.getOperand(1).isReg() && MI.getOperand(1).getReg() == RISCV::X0);
+  }
+  return MI.isAsCheapAsAMove();
+}
diff --git a/lib/Target/RISCV/RISCVInstrInfo.h b/lib/Target/RISCV/RISCVInstrInfo.h
index 1d3279c3d31e7dee80d45904a4a4a57d333ac2de..ff098e660d1915c52ddacd326460bb36f31b8a54 100644
--- a/lib/Target/RISCV/RISCVInstrInfo.h
+++ b/lib/Target/RISCV/RISCVInstrInfo.h
@@ -1,9 +1,8 @@
 //===-- RISCVInstrInfo.h - RISCV Instruction Information --------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -79,6 +78,8 @@ public:
 
   bool isBranchOffsetInRange(unsigned BranchOpc,
                              int64_t BrOffset) const override;
+
+  bool isAsCheapAsAMove(const MachineInstr &MI) const override;
 };
 }
 #endif
diff --git a/lib/Target/RISCV/RISCVInstrInfo.td b/lib/Target/RISCV/RISCVInstrInfo.td
index 60c0f0b96a62b4140ff9e98c3ead532116ffe42e..ab55ad2b1fae275a057aaf518a0b0ff275a597d9 100644
--- a/lib/Target/RISCV/RISCVInstrInfo.td
+++ b/lib/Target/RISCV/RISCVInstrInfo.td
@@ -1,9 +1,8 @@
 //===-- RISCVInstrInfo.td - Target Description for RISCV ---*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -17,36 +16,44 @@ include "RISCVInstrFormats.td"
 // RISC-V specific DAG Nodes.
 //===----------------------------------------------------------------------===//
 
-def SDT_RISCVCall         : SDTypeProfile<0, -1, [SDTCisVT<0, XLenVT>]>;
-def SDT_RISCVCallSeqStart : SDCallSeqStart<[SDTCisVT<0, i32>,
-                                            SDTCisVT<1, i32>]>;
-def SDT_RISCVCallSeqEnd   : SDCallSeqEnd<[SDTCisVT<0, i32>,
-                                          SDTCisVT<1, i32>]>;
-def SDT_RISCVSelectCC     : SDTypeProfile<1, 5, [SDTCisSameAs<1, 2>,
-                                                 SDTCisSameAs<0, 4>,
-                                                 SDTCisSameAs<4, 5>]>;
-
-
-def Call         : SDNode<"RISCVISD::CALL", SDT_RISCVCall,
-                          [SDNPHasChain, SDNPOptInGlue, SDNPOutGlue,
-                           SDNPVariadic]>;
-def CallSeqStart : SDNode<"ISD::CALLSEQ_START", SDT_RISCVCallSeqStart,
-                          [SDNPHasChain, SDNPOutGlue]>;
-def CallSeqEnd   : SDNode<"ISD::CALLSEQ_END", SDT_RISCVCallSeqEnd,
-                          [SDNPHasChain, SDNPOptInGlue, SDNPOutGlue]>;
-def RetFlag      : SDNode<"RISCVISD::RET_FLAG", SDTNone,
-                          [SDNPHasChain, SDNPOptInGlue, SDNPVariadic]>;
-def URetFlag     : SDNode<"RISCVISD::URET_FLAG", SDTNone,
-                          [SDNPHasChain, SDNPOptInGlue]>;
-def SRetFlag     : SDNode<"RISCVISD::SRET_FLAG", SDTNone,
-                          [SDNPHasChain, SDNPOptInGlue]>;
-def MRetFlag     : SDNode<"RISCVISD::MRET_FLAG", SDTNone,
-                          [SDNPHasChain, SDNPOptInGlue]>;
-def SelectCC     : SDNode<"RISCVISD::SELECT_CC", SDT_RISCVSelectCC,
-                          [SDNPInGlue]>;
-def Tail         : SDNode<"RISCVISD::TAIL", SDT_RISCVCall,
-                          [SDNPHasChain, SDNPOptInGlue, SDNPOutGlue,
-                           SDNPVariadic]>;
+// Target-independent type requirements, but with target-specific formats.
+def SDT_CallSeqStart : SDCallSeqStart<[SDTCisVT<0, i32>,
+                                       SDTCisVT<1, i32>]>;
+def SDT_CallSeqEnd   : SDCallSeqEnd<[SDTCisVT<0, i32>,
+                                     SDTCisVT<1, i32>]>;
+
+// Target-dependent type requirements.
+def SDT_RISCVCall     : SDTypeProfile<0, -1, [SDTCisVT<0, XLenVT>]>;
+def SDT_RISCVSelectCC : SDTypeProfile<1, 5, [SDTCisSameAs<1, 2>,
+                                             SDTCisSameAs<0, 4>,
+                                             SDTCisSameAs<4, 5>]>;
+
+// Target-independent nodes, but with target-specific formats.
+def callseq_start : SDNode<"ISD::CALLSEQ_START", SDT_CallSeqStart,
+                           [SDNPHasChain, SDNPOutGlue]>;
+def callseq_end   : SDNode<"ISD::CALLSEQ_END", SDT_CallSeqEnd,
+                           [SDNPHasChain, SDNPOptInGlue, SDNPOutGlue]>;
+
+// Target-dependent nodes.
+def riscv_call      : SDNode<"RISCVISD::CALL", SDT_RISCVCall,
+                             [SDNPHasChain, SDNPOptInGlue, SDNPOutGlue,
+                              SDNPVariadic]>;
+def riscv_ret_flag  : SDNode<"RISCVISD::RET_FLAG", SDTNone,
+                             [SDNPHasChain, SDNPOptInGlue, SDNPVariadic]>;
+def riscv_uret_flag : SDNode<"RISCVISD::URET_FLAG", SDTNone,
+                             [SDNPHasChain, SDNPOptInGlue]>;
+def riscv_sret_flag : SDNode<"RISCVISD::SRET_FLAG", SDTNone,
+                             [SDNPHasChain, SDNPOptInGlue]>;
+def riscv_mret_flag : SDNode<"RISCVISD::MRET_FLAG", SDTNone,
+                             [SDNPHasChain, SDNPOptInGlue]>;
+def riscv_selectcc  : SDNode<"RISCVISD::SELECT_CC", SDT_RISCVSelectCC,
+                             [SDNPInGlue]>;
+def riscv_tail      : SDNode<"RISCVISD::TAIL", SDT_RISCVCall,
+                             [SDNPHasChain, SDNPOptInGlue, SDNPOutGlue,
+                              SDNPVariadic]>;
+def riscv_sllw      : SDNode<"RISCVISD::SLLW", SDTIntShiftOp>;
+def riscv_sraw      : SDNode<"RISCVISD::SRAW", SDTIntShiftOp>;
+def riscv_srlw      : SDNode<"RISCVISD::SRLW", SDTIntShiftOp>;
 
 //===----------------------------------------------------------------------===//
 // Operand and SDNode transformation definitions.
@@ -198,8 +205,10 @@ def csr_sysreg : Operand<XLenVT> {
 }
 
 // A parameterized register class alternative to i32imm/i64imm from Target.td.
-def ixlenimm : Operand<XLenVT> {
-  let ParserMatchClass = ImmXLenAsmOperand<"">;
+def ixlenimm : Operand<XLenVT>;
+
+def ixlenimm_li : Operand<XLenVT> {
+  let ParserMatchClass = ImmXLenAsmOperand<"", "LI">;
 }
 
 // Standalone (codegen-only) immleaf patterns.
@@ -305,7 +314,8 @@ class Priv<string opcodestr, bits<7> funct7>
 // Instructions
 //===----------------------------------------------------------------------===//
 
-let hasSideEffects = 0, isReMaterializable = 1, mayLoad = 0, mayStore = 0 in {
+let hasSideEffects = 0, mayLoad = 0, mayStore = 0 in {
+let isReMaterializable = 1, isAsCheapAsAMove = 1 in
 def LUI : RVInstU<OPC_LUI, (outs GPR:$rd), (ins uimm20_lui:$imm20),
                   "lui", "$rd, $imm20">;
 
@@ -341,13 +351,17 @@ def SW : Store_rri<0b010, "sw">;
 
 // ADDI isn't always rematerializable, but isReMaterializable will be used as
 // a hint which is verified in isReallyTriviallyReMaterializable.
-let isReMaterializable = 1 in
+let isReMaterializable = 1, isAsCheapAsAMove = 1 in
 def ADDI  : ALU_ri<0b000, "addi">;
 
 def SLTI  : ALU_ri<0b010, "slti">;
 def SLTIU : ALU_ri<0b011, "sltiu">;
+
+let isReMaterializable = 1, isAsCheapAsAMove = 1 in {
 def XORI  : ALU_ri<0b100, "xori">;
 def ORI   : ALU_ri<0b110, "ori">;
+}
+
 def ANDI  : ALU_ri<0b111, "andi">;
 
 def SLLI : Shift_ri<0, 0b001, "slli">;
@@ -497,7 +511,7 @@ def : InstAlias<"nop",           (ADDI      X0,      X0,       0)>;
 // expanded to real instructions immediately.
 let hasSideEffects = 0, mayLoad = 0, mayStore = 0, Size = 32,
     isCodeGenOnly = 0, isAsmParserOnly = 1 in
-def PseudoLI : Pseudo<(outs GPR:$rd), (ins ixlenimm:$imm), [],
+def PseudoLI : Pseudo<(outs GPR:$rd), (ins ixlenimm_li:$imm), [],
                       "li", "$rd, $imm">;
 
 def : InstAlias<"mv $rd, $rs",   (ADDI GPR:$rd, GPR:$rs,       0)>;
@@ -652,6 +666,18 @@ class PatGprUimmLog2XLen<SDPatternOperator OpNode, RVInstIShift Inst>
 def IsOrAdd: PatFrag<(ops node:$A, node:$B), (or node:$A, node:$B), [{
   return isOrEquivalentToAdd(N);
 }]>;
+def assertsexti32 : PatFrag<(ops node:$src), (assertsext node:$src), [{
+  return cast<VTSDNode>(N->getOperand(1))->getVT() == MVT::i32;
+}]>;
+def sexti32 : PatFrags<(ops node:$src),
+                       [(sext_inreg node:$src, i32),
+                        (assertsexti32 node:$src)]>;
+def assertzexti32 : PatFrag<(ops node:$src), (assertzext node:$src), [{
+  return cast<VTSDNode>(N->getOperand(1))->getVT() == MVT::i32;
+}]>;
+def zexti32 : PatFrags<(ops node:$src),
+                       [(and node:$src, 0xffffffff),
+                        (assertzexti32 node:$src)]>;
 
 /// Immediates
 
@@ -720,7 +746,7 @@ class SelectCC_rrirr<RegisterClass valty, RegisterClass cmpty>
     : Pseudo<(outs valty:$dst),
              (ins cmpty:$lhs, cmpty:$rhs, ixlenimm:$imm,
               valty:$truev, valty:$falsev),
-             [(set valty:$dst, (SelectCC cmpty:$lhs, cmpty:$rhs,
+             [(set valty:$dst, (riscv_selectcc cmpty:$lhs, cmpty:$rhs,
               (XLenVT imm:$imm), valty:$truev, valty:$falsev))]>;
 
 def Select_GPR_Using_CC_GPR : SelectCC_rrirr<GPR, GPR>;
@@ -776,22 +802,23 @@ def : Pat<(brind (add GPR:$rs1, simm12:$imm12)),
 // Define isCodeGenOnly = 0 to support parsing assembly "call" instruction.
 let isCall = 1, Defs = [X1], isCodeGenOnly = 0 in
 def PseudoCALL : Pseudo<(outs), (ins bare_symbol:$func),
-                        [(Call tglobaladdr:$func)]> {
+                        [(riscv_call tglobaladdr:$func)]> {
   let AsmString = "call\t$func";
 }
 
-def : Pat<(Call texternalsym:$func), (PseudoCALL texternalsym:$func)>;
+def : Pat<(riscv_call texternalsym:$func), (PseudoCALL texternalsym:$func)>;
 
-def : Pat<(URetFlag), (URET X0, X0)>;
-def : Pat<(SRetFlag), (SRET X0, X0)>;
-def : Pat<(MRetFlag), (MRET X0, X0)>;
+def : Pat<(riscv_uret_flag), (URET X0, X0)>;
+def : Pat<(riscv_sret_flag), (SRET X0, X0)>;
+def : Pat<(riscv_mret_flag), (MRET X0, X0)>;
 
 let isCall = 1, Defs = [X1] in
-def PseudoCALLIndirect : Pseudo<(outs), (ins GPR:$rs1), [(Call GPR:$rs1)]>,
+def PseudoCALLIndirect : Pseudo<(outs), (ins GPR:$rs1),
+                                [(riscv_call GPR:$rs1)]>,
                          PseudoInstExpansion<(JALR X1, GPR:$rs1, 0)>;
 
 let isBarrier = 1, isReturn = 1, isTerminator = 1 in
-def PseudoRET : Pseudo<(outs), (ins), [(RetFlag)]>,
+def PseudoRET : Pseudo<(outs), (ins), [(riscv_ret_flag)]>,
                 PseudoInstExpansion<(JALR X0, X1, 0)>;
 
 // PseudoTAIL is a pseudo instruction similar to PseudoCALL and will eventually
@@ -804,12 +831,13 @@ def PseudoTAIL : Pseudo<(outs), (ins bare_symbol:$dst), []> {
 }
 
 let isCall = 1, isTerminator = 1, isReturn = 1, isBarrier = 1, Uses = [X2] in
-def PseudoTAILIndirect : Pseudo<(outs), (ins GPRTC:$rs1), [(Tail GPRTC:$rs1)]>,
+def PseudoTAILIndirect : Pseudo<(outs), (ins GPRTC:$rs1),
+                                [(riscv_tail GPRTC:$rs1)]>,
                          PseudoInstExpansion<(JALR X0, GPR:$rs1, 0)>;
 
-def : Pat<(Tail (iPTR tglobaladdr:$dst)),
+def : Pat<(riscv_tail (iPTR tglobaladdr:$dst)),
           (PseudoTAIL texternalsym:$dst)>;
-def : Pat<(Tail (iPTR texternalsym:$dst)),
+def : Pat<(riscv_tail (iPTR texternalsym:$dst)),
           (PseudoTAIL texternalsym:$dst)>;
 
 let hasSideEffects = 0, mayLoad = 0, mayStore = 0, isCodeGenOnly = 0,
@@ -880,9 +908,9 @@ def : Pat<(atomic_fence (XLenVT 7), (imm)), (FENCE 0b11, 0b11)>;
 // Pessimistically assume the stack pointer will be clobbered
 let Defs = [X2], Uses = [X2] in {
 def ADJCALLSTACKDOWN : Pseudo<(outs), (ins i32imm:$amt1, i32imm:$amt2),
-                              [(CallSeqStart timm:$amt1, timm:$amt2)]>;
+                              [(callseq_start timm:$amt1, timm:$amt2)]>;
 def ADJCALLSTACKUP   : Pseudo<(outs), (ins i32imm:$amt1, i32imm:$amt2),
-                              [(CallSeqEnd timm:$amt1, timm:$amt2)]>;
+                              [(callseq_end timm:$amt1, timm:$amt2)]>;
 } // Defs = [X2], Uses = [X2]
 
 /// RV64 patterns
@@ -909,7 +937,9 @@ def : Pat<(sext_inreg (shl GPR:$rs1, uimm5:$shamt), i32),
 def : Pat<(sra (sext_inreg GPR:$rs1, i32), uimm5:$shamt),
           (SRAIW GPR:$rs1, uimm5:$shamt)>;
 
-// TODO: patterns for SLLW/SRLW/SRAW.
+def : PatGprGpr<riscv_sllw, SLLW>;
+def : PatGprGpr<riscv_srlw, SRLW>;
+def : PatGprGpr<riscv_sraw, SRAW>;
 
 /// Loads
 
diff --git a/lib/Target/RISCV/RISCVInstrInfoA.td b/lib/Target/RISCV/RISCVInstrInfoA.td
index 9cb1d2f0b627d54987c34663aa73473e23549094..b35c2f128bae03ccaea8afc5122720a5509ac90a 100644
--- a/lib/Target/RISCV/RISCVInstrInfoA.td
+++ b/lib/Target/RISCV/RISCVInstrInfoA.td
@@ -1,9 +1,8 @@
 //===-- RISCVInstrInfoA.td - RISC-V 'A' instructions -------*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -235,7 +234,7 @@ def : PseudoMaskedAMOPat<int_riscv_masked_atomicrmw_umin_i32,
 
 class PseudoCmpXchg
     : Pseudo<(outs GPR:$res, GPR:$scratch),
-             (ins GPR:$addr, GPR:$cmpval, GPR:$newval, i32imm:$ordering), []> {
+             (ins GPR:$addr, GPR:$cmpval, GPR:$newval, ixlenimm:$ordering), []> {
   let Constraints = "@earlyclobber $res,@earlyclobber $scratch";
   let mayLoad = 1;
   let mayStore = 1;
@@ -263,7 +262,7 @@ defm : PseudoCmpXchgPat<"atomic_cmp_swap_32", PseudoCmpXchg32>;
 def PseudoMaskedCmpXchg32
     : Pseudo<(outs GPR:$res, GPR:$scratch),
              (ins GPR:$addr, GPR:$cmpval, GPR:$newval, GPR:$mask,
-              i32imm:$ordering), []> {
+              ixlenimm:$ordering), []> {
   let Constraints = "@earlyclobber $res,@earlyclobber $scratch";
   let mayLoad = 1;
   let mayStore = 1;
@@ -276,3 +275,79 @@ def : Pat<(int_riscv_masked_cmpxchg_i32
             GPR:$addr, GPR:$cmpval, GPR:$newval, GPR:$mask, imm:$ordering)>;
 
 } // Predicates = [HasStdExtA]
+
+let Predicates = [HasStdExtA, IsRV64] in {
+
+/// 64-bit atomic loads and stores
+
+// Fences will be inserted for atomic load/stores according to the logic in
+// RISCVTargetLowering::{emitLeadingFence,emitTrailingFence}.
+defm : LdPat<atomic_load_64, LD>;
+defm : AtomicStPat<atomic_store_64, SD, GPR>;
+
+defm : AMOPat<"atomic_swap_64", "AMOSWAP_D">;
+defm : AMOPat<"atomic_load_add_64", "AMOADD_D">;
+defm : AMOPat<"atomic_load_and_64", "AMOAND_D">;
+defm : AMOPat<"atomic_load_or_64", "AMOOR_D">;
+defm : AMOPat<"atomic_load_xor_64", "AMOXOR_D">;
+defm : AMOPat<"atomic_load_max_64", "AMOMAX_D">;
+defm : AMOPat<"atomic_load_min_64", "AMOMIN_D">;
+defm : AMOPat<"atomic_load_umax_64", "AMOMAXU_D">;
+defm : AMOPat<"atomic_load_umin_64", "AMOMINU_D">;
+
+/// 64-bit AMOs
+
+def : Pat<(atomic_load_sub_64_monotonic GPR:$addr, GPR:$incr),
+          (AMOADD_D GPR:$addr, (SUB X0, GPR:$incr))>;
+def : Pat<(atomic_load_sub_64_acquire GPR:$addr, GPR:$incr),
+          (AMOADD_D_AQ GPR:$addr, (SUB X0, GPR:$incr))>;
+def : Pat<(atomic_load_sub_64_release GPR:$addr, GPR:$incr),
+          (AMOADD_D_RL GPR:$addr, (SUB X0, GPR:$incr))>;
+def : Pat<(atomic_load_sub_64_acq_rel GPR:$addr, GPR:$incr),
+          (AMOADD_D_AQ_RL GPR:$addr, (SUB X0, GPR:$incr))>;
+def : Pat<(atomic_load_sub_64_seq_cst GPR:$addr, GPR:$incr),
+          (AMOADD_D_AQ_RL GPR:$addr, (SUB X0, GPR:$incr))>;
+
+/// 64-bit pseudo AMOs
+
+def PseudoAtomicLoadNand64 : PseudoAMO;
+// Ordering constants must be kept in sync with the AtomicOrdering enum in
+// AtomicOrdering.h.
+def : Pat<(atomic_load_nand_64_monotonic GPR:$addr, GPR:$incr),
+          (PseudoAtomicLoadNand64 GPR:$addr, GPR:$incr, 2)>;
+def : Pat<(atomic_load_nand_64_acquire GPR:$addr, GPR:$incr),
+          (PseudoAtomicLoadNand64 GPR:$addr, GPR:$incr, 4)>;
+def : Pat<(atomic_load_nand_64_release GPR:$addr, GPR:$incr),
+          (PseudoAtomicLoadNand64 GPR:$addr, GPR:$incr, 5)>;
+def : Pat<(atomic_load_nand_64_acq_rel GPR:$addr, GPR:$incr),
+          (PseudoAtomicLoadNand64 GPR:$addr, GPR:$incr, 6)>;
+def : Pat<(atomic_load_nand_64_seq_cst GPR:$addr, GPR:$incr),
+          (PseudoAtomicLoadNand64 GPR:$addr, GPR:$incr, 7)>;
+
+def : PseudoMaskedAMOPat<int_riscv_masked_atomicrmw_xchg_i64,
+                         PseudoMaskedAtomicSwap32>;
+def : PseudoMaskedAMOPat<int_riscv_masked_atomicrmw_add_i64,
+                         PseudoMaskedAtomicLoadAdd32>;
+def : PseudoMaskedAMOPat<int_riscv_masked_atomicrmw_sub_i64,
+                         PseudoMaskedAtomicLoadSub32>;
+def : PseudoMaskedAMOPat<int_riscv_masked_atomicrmw_nand_i64,
+                         PseudoMaskedAtomicLoadNand32>;
+def : PseudoMaskedAMOMinMaxPat<int_riscv_masked_atomicrmw_max_i64,
+                               PseudoMaskedAtomicLoadMax32>;
+def : PseudoMaskedAMOMinMaxPat<int_riscv_masked_atomicrmw_min_i64,
+                               PseudoMaskedAtomicLoadMin32>;
+def : PseudoMaskedAMOPat<int_riscv_masked_atomicrmw_umax_i64,
+                         PseudoMaskedAtomicLoadUMax32>;
+def : PseudoMaskedAMOPat<int_riscv_masked_atomicrmw_umin_i64,
+                         PseudoMaskedAtomicLoadUMin32>;
+
+/// 64-bit compare and exchange
+
+def PseudoCmpXchg64 : PseudoCmpXchg;
+defm : PseudoCmpXchgPat<"atomic_cmp_swap_64", PseudoCmpXchg64>;
+
+def : Pat<(int_riscv_masked_cmpxchg_i64
+            GPR:$addr, GPR:$cmpval, GPR:$newval, GPR:$mask, imm:$ordering),
+          (PseudoMaskedCmpXchg32
+            GPR:$addr, GPR:$cmpval, GPR:$newval, GPR:$mask, imm:$ordering)>;
+} // Predicates = [HasStdExtA, IsRV64]
diff --git a/lib/Target/RISCV/RISCVInstrInfoC.td b/lib/Target/RISCV/RISCVInstrInfoC.td
index ad68b5a7dc976314e27aefdd445cb90917e6b2ed..efb1587849fbd8ac5f5dad1d2342e50eabde9b3e 100644
--- a/lib/Target/RISCV/RISCVInstrInfoC.td
+++ b/lib/Target/RISCV/RISCVInstrInfoC.td
@@ -1,9 +1,8 @@
 //===- RISCVInstrInfoC.td - Compressed RISCV instructions -*- tblgen-*-----===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/RISCV/RISCVInstrInfoD.td b/lib/Target/RISCV/RISCVInstrInfoD.td
index 9f1cd50de595d4592b529d6750ed6a61889d586f..87efd36e9d2eb0c6f4e1e3624fca1f03a948fe78 100644
--- a/lib/Target/RISCV/RISCVInstrInfoD.td
+++ b/lib/Target/RISCV/RISCVInstrInfoD.td
@@ -1,9 +1,8 @@
 //===-- RISCVInstrInfoD.td - RISC-V 'D' instructions -------*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -308,3 +307,26 @@ def : Pat<(fp_to_uint FPR64:$rs1), (FCVT_WU_D FPR64:$rs1, 0b001)>;
 def : Pat<(sint_to_fp GPR:$rs1), (FCVT_D_W GPR:$rs1)>;
 def : Pat<(uint_to_fp GPR:$rs1), (FCVT_D_WU GPR:$rs1)>;
 } // Predicates = [HasStdExtD, IsRV32]
+
+let Predicates = [HasStdExtD, IsRV64] in {
+def : Pat<(bitconvert GPR:$rs1), (FMV_D_X GPR:$rs1)>;
+def : Pat<(bitconvert FPR64:$rs1), (FMV_X_D FPR64:$rs1)>;
+
+// FP->[u]int32 is mostly handled by the FP->[u]int64 patterns. This is safe
+// because fpto[u|s]i produce poison if the value can't fit into the target.
+// We match the single case below because fcvt.wu.d sign-extends its result so
+// is cheaper than fcvt.lu.d+sext.w.
+def : Pat<(sext_inreg (zexti32 (fp_to_uint FPR64:$rs1)), i32),
+          (FCVT_WU_D $rs1, 0b001)>;
+
+// [u]int32->fp
+def : Pat<(sint_to_fp (sext_inreg GPR:$rs1, i32)), (FCVT_D_W $rs1)>;
+def : Pat<(uint_to_fp (zexti32 GPR:$rs1)), (FCVT_D_WU $rs1)>;
+
+def : Pat<(fp_to_sint FPR64:$rs1), (FCVT_L_D FPR64:$rs1, 0b001)>;
+def : Pat<(fp_to_uint FPR64:$rs1), (FCVT_LU_D FPR64:$rs1, 0b001)>;
+
+// [u]int64->fp. Match GCC and default to using dynamic rounding mode.
+def : Pat<(sint_to_fp GPR:$rs1), (FCVT_D_L GPR:$rs1, 0b111)>;
+def : Pat<(uint_to_fp GPR:$rs1), (FCVT_D_LU GPR:$rs1, 0b111)>;
+} // Predicates = [HasStdExtD, IsRV64]
diff --git a/lib/Target/RISCV/RISCVInstrInfoF.td b/lib/Target/RISCV/RISCVInstrInfoF.td
index 03bdac45873d0b90adf10d5fe084de97f6463f72..0fb205202ff5c63610b12be6642c4089e7a299c7 100644
--- a/lib/Target/RISCV/RISCVInstrInfoF.td
+++ b/lib/Target/RISCV/RISCVInstrInfoF.td
@@ -1,9 +1,8 @@
 //===-- RISCVInstrInfoF.td - RISC-V 'F' instructions -------*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -12,6 +11,20 @@
 //
 //===----------------------------------------------------------------------===//
 
+//===----------------------------------------------------------------------===//
+// RISC-V specific DAG Nodes.
+//===----------------------------------------------------------------------===//
+
+def SDT_RISCVFMV_W_X_RV64
+    : SDTypeProfile<1, 1, [SDTCisVT<0, f32>, SDTCisVT<1, i64>]>;
+def SDT_RISCVFMV_X_ANYEXTW_RV64
+    : SDTypeProfile<1, 1, [SDTCisVT<0, i64>, SDTCisVT<1, f32>]>;
+
+def riscv_fmv_w_x_rv64
+    : SDNode<"RISCVISD::FMV_W_X_RV64", SDT_RISCVFMV_W_X_RV64>;
+def riscv_fmv_x_anyextw_rv64
+    : SDNode<"RISCVISD::FMV_X_ANYEXTW_RV64", SDT_RISCVFMV_X_ANYEXTW_RV64>;
+
 //===----------------------------------------------------------------------===//
 // Operand and SDNode transformation definitions.
 //===----------------------------------------------------------------------===//
@@ -334,3 +347,37 @@ def : Pat<(fp_to_uint FPR32:$rs1), (FCVT_WU_S $rs1, 0b001)>;
 def : Pat<(sint_to_fp GPR:$rs1), (FCVT_S_W $rs1, 0b111)>;
 def : Pat<(uint_to_fp GPR:$rs1), (FCVT_S_WU $rs1, 0b111)>;
 } // Predicates = [HasStdExtF, IsRV32]
+
+let Predicates = [HasStdExtF, IsRV32] in {
+// FP->[u]int. Round-to-zero must be used
+def : Pat<(fp_to_sint FPR32:$rs1), (FCVT_W_S $rs1, 0b001)>;
+def : Pat<(fp_to_uint FPR32:$rs1), (FCVT_WU_S $rs1, 0b001)>;
+
+// [u]int->fp. Match GCC and default to using dynamic rounding mode.
+def : Pat<(sint_to_fp GPR:$rs1), (FCVT_S_W $rs1, 0b111)>;
+def : Pat<(uint_to_fp GPR:$rs1), (FCVT_S_WU $rs1, 0b111)>;
+} // Predicates = [HasStdExtF, IsRV32]
+
+let Predicates = [HasStdExtF, IsRV64] in {
+def : Pat<(riscv_fmv_w_x_rv64 GPR:$src), (FMV_W_X GPR:$src)>;
+def : Pat<(riscv_fmv_x_anyextw_rv64 FPR32:$src), (FMV_X_W FPR32:$src)>;
+def : Pat<(sexti32 (riscv_fmv_x_anyextw_rv64 FPR32:$src)),
+          (FMV_X_W FPR32:$src)>;
+
+// FP->[u]int32 is mostly handled by the FP->[u]int64 patterns. This is safe
+// because fpto[u|s]i produces poison if the value can't fit into the target.
+// We match the single case below because fcvt.wu.s sign-extends its result so
+// is cheaper than fcvt.lu.s+sext.w.
+def : Pat<(sext_inreg (assertzexti32 (fp_to_uint FPR32:$rs1)), i32),
+          (FCVT_WU_S $rs1, 0b001)>;
+
+// FP->[u]int64
+def : Pat<(fp_to_sint FPR32:$rs1), (FCVT_L_S $rs1, 0b001)>;
+def : Pat<(fp_to_uint FPR32:$rs1), (FCVT_LU_S $rs1, 0b001)>;
+
+// [u]int->fp. Match GCC and default to using dynamic rounding mode.
+def : Pat<(sint_to_fp (sext_inreg GPR:$rs1, i32)), (FCVT_S_W $rs1, 0b111)>;
+def : Pat<(uint_to_fp (zexti32 GPR:$rs1)), (FCVT_S_WU $rs1, 0b111)>;
+def : Pat<(sint_to_fp GPR:$rs1), (FCVT_S_L $rs1, 0b111)>;
+def : Pat<(uint_to_fp GPR:$rs1), (FCVT_S_LU $rs1, 0b111)>;
+} // Predicates = [HasStdExtF, IsRV64]
diff --git a/lib/Target/RISCV/RISCVInstrInfoM.td b/lib/Target/RISCV/RISCVInstrInfoM.td
index 2dd10ada4003eb80923d0b20e30c986c698fbe69..e75151ba99c77e3a5c4917c64678a8806bcc9d52 100644
--- a/lib/Target/RISCV/RISCVInstrInfoM.td
+++ b/lib/Target/RISCV/RISCVInstrInfoM.td
@@ -1,9 +1,8 @@
 //===-- RISCVInstrInfoM.td - RISC-V 'M' instructions -------*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -12,6 +11,14 @@
 //
 //===----------------------------------------------------------------------===//
 
+//===----------------------------------------------------------------------===//
+// RISC-V specific DAG Nodes.
+//===----------------------------------------------------------------------===//
+
+def riscv_divw  : SDNode<"RISCVISD::DIVW",  SDTIntBinOp>;
+def riscv_divuw : SDNode<"RISCVISD::DIVUW", SDTIntBinOp>;
+def riscv_remuw : SDNode<"RISCVISD::REMUW", SDTIntBinOp>;
+
 //===----------------------------------------------------------------------===//
 // Instructions
 //===----------------------------------------------------------------------===//
@@ -49,3 +56,29 @@ def : PatGprGpr<udiv, DIVU>;
 def : PatGprGpr<srem, REM>;
 def : PatGprGpr<urem, REMU>;
 } // Predicates = [HasStdExtM]
+
+let Predicates = [HasStdExtM, IsRV64] in {
+def : Pat<(sext_inreg (mul GPR:$rs1, GPR:$rs2), i32),
+          (MULW GPR:$rs1, GPR:$rs2)>;
+
+def : PatGprGpr<riscv_divw, DIVW>;
+def : PatGprGpr<riscv_divuw, DIVUW>;
+def : PatGprGpr<riscv_remuw, REMUW>;
+
+// Handle the specific cases where using DIVU/REMU would be correct and result
+// in fewer instructions than emitting DIVUW/REMUW then zero-extending the
+// result.
+def : Pat<(zexti32 (riscv_divuw (zexti32 GPR:$rs1), (zexti32 GPR:$rs2))),
+          (DIVU GPR:$rs1, GPR:$rs2)>;
+def : Pat<(zexti32 (riscv_remuw (zexti32 GPR:$rs1), (zexti32 GPR:$rs2))),
+          (REMU GPR:$rs1, GPR:$rs2)>;
+
+// Although the sexti32 operands may not have originated from an i32 srem,
+// this pattern is safe as it is impossible for two sign extended inputs to
+// produce a result where res[63:32]=0 and res[31]=1.
+def : Pat<(srem (sexti32 GPR:$rs1), (sexti32 GPR:$rs2)),
+          (REMW GPR:$rs1, GPR:$rs2)>;
+def : Pat<(sext_inreg (srem (sexti32 GPR:$rs1),
+                            (sexti32 GPR:$rs2)), i32),
+          (REMW GPR:$rs1, GPR:$rs2)>;
+} // Predicates = [HasStdExtM, IsRV64]
diff --git a/lib/Target/RISCV/RISCVMCInstLower.cpp b/lib/Target/RISCV/RISCVMCInstLower.cpp
index e0100b1679be61d2400a0ad1af1f5e1037c3955b..3e50be867ced95b63a139e404d6a41d49973d2af 100644
--- a/lib/Target/RISCV/RISCVMCInstLower.cpp
+++ b/lib/Target/RISCV/RISCVMCInstLower.cpp
@@ -1,9 +1,8 @@
 //===-- RISCVMCInstLower.cpp - Convert RISCV MachineInstr to an MCInst ------=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/RISCV/RISCVMachineFunctionInfo.h b/lib/Target/RISCV/RISCVMachineFunctionInfo.h
index 2fea3a1bdd2f43ad8d354f2d58e8a18013c8cf1e..66766e040342dcfdfaed60f04d94d9612ff48212 100644
--- a/lib/Target/RISCV/RISCVMachineFunctionInfo.h
+++ b/lib/Target/RISCV/RISCVMachineFunctionInfo.h
@@ -1,9 +1,8 @@
 //=- RISCVMachineFunctionInfo.h - RISCV machine function info -----*- C++ -*-=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/RISCV/RISCVMergeBaseOffset.cpp b/lib/Target/RISCV/RISCVMergeBaseOffset.cpp
index cea009c5447db4740b50844bd657eafdf63d866f..82b1209cb8e776874986277dc98d8ed8e3eb8a64 100644
--- a/lib/Target/RISCV/RISCVMergeBaseOffset.cpp
+++ b/lib/Target/RISCV/RISCVMergeBaseOffset.cpp
@@ -1,9 +1,8 @@
 //===----- RISCVMergeBaseOffset.cpp - Optimise address calculations  ------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/RISCV/RISCVRegisterInfo.cpp b/lib/Target/RISCV/RISCVRegisterInfo.cpp
index 3ed1dec434ceef7b036cf1e47df292f0e6671a65..75ab0a4d4447bda4f57e40643f221bf2239338f4 100644
--- a/lib/Target/RISCV/RISCVRegisterInfo.cpp
+++ b/lib/Target/RISCV/RISCVRegisterInfo.cpp
@@ -1,9 +1,8 @@
 //===-- RISCVRegisterInfo.cpp - RISCV Register Information ------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/RISCV/RISCVRegisterInfo.h b/lib/Target/RISCV/RISCVRegisterInfo.h
index cbbb70079dd174e012dd6dc73ab72489d64e2d63..6552856c5a842f4d4dd63e98d0c4499f10ee0c0a 100644
--- a/lib/Target/RISCV/RISCVRegisterInfo.h
+++ b/lib/Target/RISCV/RISCVRegisterInfo.h
@@ -1,9 +1,8 @@
 //===-- RISCVRegisterInfo.h - RISCV Register Information Impl ---*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/RISCV/RISCVRegisterInfo.td b/lib/Target/RISCV/RISCVRegisterInfo.td
index 4be8ff9200e92f0e0e4a728d57668a5bf92bf9b6..9f6c0908138c5305a6b7fb7e839af0a8648bc4f3 100644
--- a/lib/Target/RISCV/RISCVRegisterInfo.td
+++ b/lib/Target/RISCV/RISCVRegisterInfo.td
@@ -1,9 +1,8 @@
 //===-- RISCVRegisterInfo.td - RISC-V Register defs --------*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/RISCV/RISCVSubtarget.cpp b/lib/Target/RISCV/RISCVSubtarget.cpp
index b221ea84a33c2f7e2c252c63e5e3518e4427a4c9..41b85c46587f1e7b8aeb121eb55ba79df67e8160 100644
--- a/lib/Target/RISCV/RISCVSubtarget.cpp
+++ b/lib/Target/RISCV/RISCVSubtarget.cpp
@@ -1,9 +1,8 @@
 //===-- RISCVSubtarget.cpp - RISCV Subtarget Information ------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/RISCV/RISCVSubtarget.h b/lib/Target/RISCV/RISCVSubtarget.h
index 0e09391e7829b9ec4a7ab4a2dd2efcdeea7a2333..66c77301037a2afcbe218ee5e58fe43976176aed 100644
--- a/lib/Target/RISCV/RISCVSubtarget.h
+++ b/lib/Target/RISCV/RISCVSubtarget.h
@@ -1,9 +1,8 @@
 //===-- RISCVSubtarget.h - Define Subtarget for the RISCV -------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/RISCV/RISCVSystemOperands.td b/lib/Target/RISCV/RISCVSystemOperands.td
index f1b7984ffe6bc293a65b5238921dc3056b963c1d..c36633d5c6339141d47fe35a6d5c57218b7a1736 100644
--- a/lib/Target/RISCV/RISCVSystemOperands.td
+++ b/lib/Target/RISCV/RISCVSystemOperands.td
@@ -1,9 +1,8 @@
 //===- RISCVSystemOperands.td ----------------------------*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/RISCV/RISCVTargetMachine.cpp b/lib/Target/RISCV/RISCVTargetMachine.cpp
index 8937ec200bd74cf8b59aed8ac1327eeddfede59d..f341d9c18265b19cbe737e9b42ca64d4359ba05c 100644
--- a/lib/Target/RISCV/RISCVTargetMachine.cpp
+++ b/lib/Target/RISCV/RISCVTargetMachine.cpp
@@ -1,9 +1,8 @@
 //===-- RISCVTargetMachine.cpp - Define TargetMachine for RISCV -----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/RISCV/RISCVTargetMachine.h b/lib/Target/RISCV/RISCVTargetMachine.h
index 02361dddebf7b462fd685f141836dae74ac2bb62..dd0110c6b69efc5fee826efbbd0ac9165850efa3 100644
--- a/lib/Target/RISCV/RISCVTargetMachine.h
+++ b/lib/Target/RISCV/RISCVTargetMachine.h
@@ -1,9 +1,8 @@
 //===-- RISCVTargetMachine.h - Define TargetMachine for RISCV ---*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/RISCV/RISCVTargetObjectFile.cpp b/lib/Target/RISCV/RISCVTargetObjectFile.cpp
index 46e81b628b65b1f466cca68418fa9db2de1e39ab..ec61ab76096b98b10215dd2a8786879f169a7818 100644
--- a/lib/Target/RISCV/RISCVTargetObjectFile.cpp
+++ b/lib/Target/RISCV/RISCVTargetObjectFile.cpp
@@ -1,9 +1,8 @@
 //===-- RISCVTargetObjectFile.cpp - RISCV Object Info -----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/RISCV/RISCVTargetObjectFile.h b/lib/Target/RISCV/RISCVTargetObjectFile.h
index 5467220301c1f63f46917674acab57ecd8c96bf7..e622c5a6c1a3be790afab3b887438f0a3b35b16a 100644
--- a/lib/Target/RISCV/RISCVTargetObjectFile.h
+++ b/lib/Target/RISCV/RISCVTargetObjectFile.h
@@ -1,9 +1,8 @@
 //===-- RISCVTargetObjectFile.h - RISCV Object Info -*- C++ ---------*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/RISCV/TargetInfo/LLVMBuild.txt b/lib/Target/RISCV/TargetInfo/LLVMBuild.txt
index db7f66f94bf42ac5a7abcd2f3114175e34beac54..632cfcf24332769fdc1c959f4d5c27624b157294 100644
--- a/lib/Target/RISCV/TargetInfo/LLVMBuild.txt
+++ b/lib/Target/RISCV/TargetInfo/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/Target/RISCV/TargetInfo/LLVMBuild.txt --------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Target/RISCV/TargetInfo/RISCVTargetInfo.cpp b/lib/Target/RISCV/TargetInfo/RISCVTargetInfo.cpp
index 0f369d960fe1495b53443a4b287254b5419e37f8..60dd4f251ec450ee88b5377f74bfc820fab23c16 100644
--- a/lib/Target/RISCV/TargetInfo/RISCVTargetInfo.cpp
+++ b/lib/Target/RISCV/TargetInfo/RISCVTargetInfo.cpp
@@ -1,9 +1,8 @@
 //===-- RISCVTargetInfo.cpp - RISCV Target Implementation -----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/RISCV/Utils/LLVMBuild.txt b/lib/Target/RISCV/Utils/LLVMBuild.txt
index ec75b93036693499f1f8e28c1f266607f5255f35..4803c939fa7ad56fa8e5ae4baccdba543403b649 100644
--- a/lib/Target/RISCV/Utils/LLVMBuild.txt
+++ b/lib/Target/RISCV/Utils/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/Target/RISCV/Utils/LLVMBuild.txt ----------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Target/RISCV/Utils/RISCVBaseInfo.h b/lib/Target/RISCV/Utils/RISCVBaseInfo.h
index 372e0e80bbaf8c40cf9364744ccc1f91c192c1bf..49cae3a9df5610b588e71bb55c07c6b8a63d7f94 100644
--- a/lib/Target/RISCV/Utils/RISCVBaseInfo.h
+++ b/lib/Target/RISCV/Utils/RISCVBaseInfo.h
@@ -1,9 +1,8 @@
 //===-- RISCVBaseInfo.h - Top level definitions for RISCV MC ----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/RISCV/Utils/RISCVMatInt.cpp b/lib/Target/RISCV/Utils/RISCVMatInt.cpp
index 3dc298246bc562ef8df573d877cb3519ca2d7c0b..bb9ca66a4555729c5a59d9bce1246c3f69aaacd7 100644
--- a/lib/Target/RISCV/Utils/RISCVMatInt.cpp
+++ b/lib/Target/RISCV/Utils/RISCVMatInt.cpp
@@ -1,9 +1,8 @@
 //===- RISCVMatInt.cpp - Immediate materialisation -------------*- C++ -*--===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/RISCV/Utils/RISCVMatInt.h b/lib/Target/RISCV/Utils/RISCVMatInt.h
index 49d1d89adc7a1af5532098b50ed3e45c3edda9e1..bb23d389bdf6100a4825635352508bc8d51f79cb 100644
--- a/lib/Target/RISCV/Utils/RISCVMatInt.h
+++ b/lib/Target/RISCV/Utils/RISCVMatInt.h
@@ -1,9 +1,8 @@
 //===- RISCVMatInt.h - Immediate materialisation ---------------*- C++ -*--===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/Sparc/AsmParser/LLVMBuild.txt b/lib/Target/Sparc/AsmParser/LLVMBuild.txt
index 08fdc9d49b22fc54c5f4a8f6f228dfdc62231605..43efc237192200295fcff6eaa955b2f4f1d20303 100644
--- a/lib/Target/Sparc/AsmParser/LLVMBuild.txt
+++ b/lib/Target/Sparc/AsmParser/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/Target/Sparc/AsmParser/LLVMBuild.txt ---------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Target/Sparc/AsmParser/SparcAsmParser.cpp b/lib/Target/Sparc/AsmParser/SparcAsmParser.cpp
index 691421e533eae975603700cb3bef5c88c673312f..a1bff894aedfe08a0b6adcbea42624dceff2f3ec 100644
--- a/lib/Target/Sparc/AsmParser/SparcAsmParser.cpp
+++ b/lib/Target/Sparc/AsmParser/SparcAsmParser.cpp
@@ -1,9 +1,8 @@
 //===-- SparcAsmParser.cpp - Parse Sparc assembly to MCInst instructions --===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/Sparc/DelaySlotFiller.cpp b/lib/Target/Sparc/DelaySlotFiller.cpp
index 6290e5a15a8b4fbbd106131c61993885f66ea0d4..f1ca8e18c22813dc8bf544e6cb5f121c18a60d3b 100644
--- a/lib/Target/Sparc/DelaySlotFiller.cpp
+++ b/lib/Target/Sparc/DelaySlotFiller.cpp
@@ -1,9 +1,8 @@
 //===-- DelaySlotFiller.cpp - SPARC delay slot filler ---------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Sparc/Disassembler/LLVMBuild.txt b/lib/Target/Sparc/Disassembler/LLVMBuild.txt
index bd5397dac314fe0390e877e6fe48b0e3c7a16886..8a9a205dfb25e7cc721c60f0b03d308d20b918bd 100644
--- a/lib/Target/Sparc/Disassembler/LLVMBuild.txt
+++ b/lib/Target/Sparc/Disassembler/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/Target/Sparc/Disassembler/LLVMBuild.txt ------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Target/Sparc/Disassembler/SparcDisassembler.cpp b/lib/Target/Sparc/Disassembler/SparcDisassembler.cpp
index 0045e63a824e1dc439100908422f1e46dfb8d073..bfbd1e1769c449cf65dfec31ab33f21b84cc947b 100644
--- a/lib/Target/Sparc/Disassembler/SparcDisassembler.cpp
+++ b/lib/Target/Sparc/Disassembler/SparcDisassembler.cpp
@@ -1,9 +1,8 @@
 //===- SparcDisassembler.cpp - Disassembler for Sparc -----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Sparc/InstPrinter/LLVMBuild.txt b/lib/Target/Sparc/InstPrinter/LLVMBuild.txt
index b4c8802b6a3814472929b1e22fa4fbc0359cbb12..cb40190ad85b5203f8a1bf38f5cf1cbc68bfb061 100644
--- a/lib/Target/Sparc/InstPrinter/LLVMBuild.txt
+++ b/lib/Target/Sparc/InstPrinter/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/Target/Sparc/InstPrinter/LLVMBuild.txt -------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Target/Sparc/InstPrinter/SparcInstPrinter.cpp b/lib/Target/Sparc/InstPrinter/SparcInstPrinter.cpp
index d152efae6d1f6f2cac5c90fbfcd77b9a760c464e..c479459786d7be3f199e7b4fc2830fda2bf878f3 100644
--- a/lib/Target/Sparc/InstPrinter/SparcInstPrinter.cpp
+++ b/lib/Target/Sparc/InstPrinter/SparcInstPrinter.cpp
@@ -1,9 +1,8 @@
 //===-- SparcInstPrinter.cpp - Convert Sparc MCInst to assembly syntax -----==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Sparc/InstPrinter/SparcInstPrinter.h b/lib/Target/Sparc/InstPrinter/SparcInstPrinter.h
index 89015eb137c20167a3310d39fa964c16b9b99282..c8c52b59ca4ae9a9a5385a0ec11eecf2240c3c6b 100644
--- a/lib/Target/Sparc/InstPrinter/SparcInstPrinter.h
+++ b/lib/Target/Sparc/InstPrinter/SparcInstPrinter.h
@@ -1,9 +1,8 @@
 //===-- SparcInstPrinter.h - Convert Sparc MCInst to assembly syntax ------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Sparc/LLVMBuild.txt b/lib/Target/Sparc/LLVMBuild.txt
index 765912b82b2b14822b13c26d1ef703fbc298ac9b..b13d8445db33c99221344fedbc781d7ef9307099 100644
--- a/lib/Target/Sparc/LLVMBuild.txt
+++ b/lib/Target/Sparc/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/Target/Sparc/LLVMBuild.txt -------------------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Target/Sparc/LeonFeatures.td b/lib/Target/Sparc/LeonFeatures.td
index 61e5f16e0a1e8498be5175e53c9f063dcfd0d9c2..e0ea4e9c7645f30fc3394aeabbb9fad30cdbe4fe 100644
--- a/lib/Target/Sparc/LeonFeatures.td
+++ b/lib/Target/Sparc/LeonFeatures.td
@@ -1,9 +1,8 @@
 //===-- LeonFeatures.td - Describe the Leon Features -------*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Sparc/LeonPasses.cpp b/lib/Target/Sparc/LeonPasses.cpp
index 5ce00db365ab2ce60c76455387ea2a3237266b28..e9d3aaeb9cfe23bcfed8e83eee53788540cbaa9e 100644
--- a/lib/Target/Sparc/LeonPasses.cpp
+++ b/lib/Target/Sparc/LeonPasses.cpp
@@ -1,9 +1,8 @@
 //===------ LeonPasses.cpp - Define passes specific to LEON ---------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Sparc/LeonPasses.h b/lib/Target/Sparc/LeonPasses.h
index 1b3d9a7a32f907155f72b5b3040154a5a36d3df8..154a2b467e16cf937c5afa8ca6da9fb4b68060b4 100644
--- a/lib/Target/Sparc/LeonPasses.h
+++ b/lib/Target/Sparc/LeonPasses.h
@@ -1,9 +1,8 @@
 //===------- LeonPasses.h - Define passes specific to LEON ----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Sparc/MCTargetDesc/LLVMBuild.txt b/lib/Target/Sparc/MCTargetDesc/LLVMBuild.txt
index 22515e6d6dc13aa4a7401fe34df0a8ad8639148c..249a195996dc83205730b71650712216d4cc7964 100644
--- a/lib/Target/Sparc/MCTargetDesc/LLVMBuild.txt
+++ b/lib/Target/Sparc/MCTargetDesc/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/Target/Sparc/MCTargetDesc/LLVMBuild.txt ------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Target/Sparc/MCTargetDesc/SparcAsmBackend.cpp b/lib/Target/Sparc/MCTargetDesc/SparcAsmBackend.cpp
index d7f1e3a1ab1d8251dced795f1ba8a5be861c5ef0..2e8fa0dbaf4c4235fe82b64dc81f8ca733a807ce 100644
--- a/lib/Target/Sparc/MCTargetDesc/SparcAsmBackend.cpp
+++ b/lib/Target/Sparc/MCTargetDesc/SparcAsmBackend.cpp
@@ -1,9 +1,8 @@
 //===-- SparcAsmBackend.cpp - Sparc Assembler Backend ---------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/Sparc/MCTargetDesc/SparcELFObjectWriter.cpp b/lib/Target/Sparc/MCTargetDesc/SparcELFObjectWriter.cpp
index 5a730947796e1b5e249dc7435583ee0feb1c3722..88547075c5ae1b45ef5c8ad5debde46055d6bfdd 100644
--- a/lib/Target/Sparc/MCTargetDesc/SparcELFObjectWriter.cpp
+++ b/lib/Target/Sparc/MCTargetDesc/SparcELFObjectWriter.cpp
@@ -1,9 +1,8 @@
 //===-- SparcELFObjectWriter.cpp - Sparc ELF Writer -----------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/Sparc/MCTargetDesc/SparcFixupKinds.h b/lib/Target/Sparc/MCTargetDesc/SparcFixupKinds.h
index 99aa63fe22904ab75efe419459b8eb65ba3260c0..b5fac02640191dba90d801fb4525bd81130a974e 100644
--- a/lib/Target/Sparc/MCTargetDesc/SparcFixupKinds.h
+++ b/lib/Target/Sparc/MCTargetDesc/SparcFixupKinds.h
@@ -1,9 +1,8 @@
 //===-- SparcFixupKinds.h - Sparc Specific Fixup Entries --------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/Sparc/MCTargetDesc/SparcMCAsmInfo.cpp b/lib/Target/Sparc/MCTargetDesc/SparcMCAsmInfo.cpp
index 50e8825b15e8cadf9701854e3da8a0806570064c..1a2a040990ae90d3c195e914509b795f65cb76e3 100644
--- a/lib/Target/Sparc/MCTargetDesc/SparcMCAsmInfo.cpp
+++ b/lib/Target/Sparc/MCTargetDesc/SparcMCAsmInfo.cpp
@@ -1,9 +1,8 @@
 //===- SparcMCAsmInfo.cpp - Sparc asm properties --------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Sparc/MCTargetDesc/SparcMCAsmInfo.h b/lib/Target/Sparc/MCTargetDesc/SparcMCAsmInfo.h
index 5e8d0cb5031297dfe97a77caa47d1b6dcb090e1a..c9162f2dc8a5324bf740d7291a2fea6c0fee8f20 100644
--- a/lib/Target/Sparc/MCTargetDesc/SparcMCAsmInfo.h
+++ b/lib/Target/Sparc/MCTargetDesc/SparcMCAsmInfo.h
@@ -1,9 +1,8 @@
 //===- SparcMCAsmInfo.h - Sparc asm properties -----------------*- C++ -*--===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Sparc/MCTargetDesc/SparcMCCodeEmitter.cpp b/lib/Target/Sparc/MCTargetDesc/SparcMCCodeEmitter.cpp
index 647be159a1516ccf83f5d403a82c567d52415afd..8cb9108b49f9d8eb91aa0717c1b90c8df0cdb82f 100644
--- a/lib/Target/Sparc/MCTargetDesc/SparcMCCodeEmitter.cpp
+++ b/lib/Target/Sparc/MCTargetDesc/SparcMCCodeEmitter.cpp
@@ -1,9 +1,8 @@
 //===-- SparcMCCodeEmitter.cpp - Convert Sparc code to machine code -------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Sparc/MCTargetDesc/SparcMCExpr.cpp b/lib/Target/Sparc/MCTargetDesc/SparcMCExpr.cpp
index 4ddb72643a91ec1ca8041eccaec717524d4a534e..00f319fc37e15795057aa4e67311696840412a32 100644
--- a/lib/Target/Sparc/MCTargetDesc/SparcMCExpr.cpp
+++ b/lib/Target/Sparc/MCTargetDesc/SparcMCExpr.cpp
@@ -1,9 +1,8 @@
 //===-- SparcMCExpr.cpp - Sparc specific MC expression classes --------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Sparc/MCTargetDesc/SparcMCExpr.h b/lib/Target/Sparc/MCTargetDesc/SparcMCExpr.h
index cf2db067749c847491c9a7db07aebbae6390ece9..c2467faca257d1361a4ebdce5c0ad8f4c66e18de 100644
--- a/lib/Target/Sparc/MCTargetDesc/SparcMCExpr.h
+++ b/lib/Target/Sparc/MCTargetDesc/SparcMCExpr.h
@@ -1,9 +1,8 @@
 //====- SparcMCExpr.h - Sparc specific MC expression classes --*- C++ -*-=====//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Sparc/MCTargetDesc/SparcMCTargetDesc.cpp b/lib/Target/Sparc/MCTargetDesc/SparcMCTargetDesc.cpp
index bd6596faee5d1399d992c80c06327bc049900663..18d5108bb93e07c835d24d3f89ab9605d935e998 100644
--- a/lib/Target/Sparc/MCTargetDesc/SparcMCTargetDesc.cpp
+++ b/lib/Target/Sparc/MCTargetDesc/SparcMCTargetDesc.cpp
@@ -1,9 +1,8 @@
 //===-- SparcMCTargetDesc.cpp - Sparc Target Descriptions -----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Sparc/MCTargetDesc/SparcMCTargetDesc.h b/lib/Target/Sparc/MCTargetDesc/SparcMCTargetDesc.h
index 3cd24104c443d6207828bdbc494c9450ee95cd67..c2a70689a693cc5a4a1cca14698ac2a6a8ffa5d0 100644
--- a/lib/Target/Sparc/MCTargetDesc/SparcMCTargetDesc.h
+++ b/lib/Target/Sparc/MCTargetDesc/SparcMCTargetDesc.h
@@ -1,9 +1,8 @@
 //===-- SparcMCTargetDesc.h - Sparc Target Descriptions ---------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Sparc/MCTargetDesc/SparcTargetStreamer.cpp b/lib/Target/Sparc/MCTargetDesc/SparcTargetStreamer.cpp
index 94af791e0e75d4c6082d3d3bf643bc43ab113886..c5ed7f2351f0da94f70d52d01013de0556854509 100644
--- a/lib/Target/Sparc/MCTargetDesc/SparcTargetStreamer.cpp
+++ b/lib/Target/Sparc/MCTargetDesc/SparcTargetStreamer.cpp
@@ -1,9 +1,8 @@
 //===-- SparcTargetStreamer.cpp - Sparc Target Streamer Methods -----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Sparc/MCTargetDesc/SparcTargetStreamer.h b/lib/Target/Sparc/MCTargetDesc/SparcTargetStreamer.h
index 8bb418e39ab433eec20e54db4fe8b0a265af2652..9f729a6c2cf4304617391f5f2788384c655a119c 100644
--- a/lib/Target/Sparc/MCTargetDesc/SparcTargetStreamer.h
+++ b/lib/Target/Sparc/MCTargetDesc/SparcTargetStreamer.h
@@ -1,9 +1,8 @@
 //===-- SparcTargetStreamer.h - Sparc Target Streamer ----------*- C++ -*--===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/Sparc/Sparc.h b/lib/Target/Sparc/Sparc.h
index 0cea53b359ebb6a814ba43b4e0cc837271c3e703..967c463f5281f9ff3a82d16f2d78ec39d96c61c8 100644
--- a/lib/Target/Sparc/Sparc.h
+++ b/lib/Target/Sparc/Sparc.h
@@ -1,9 +1,8 @@
 //===-- Sparc.h - Top-level interface for Sparc representation --*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Sparc/Sparc.td b/lib/Target/Sparc/Sparc.td
index 0412215be8abc4a2f506672232553ddaa2f7bd62..ca6147edc46bafec021dec0807d1d27556057c15 100644
--- a/lib/Target/Sparc/Sparc.td
+++ b/lib/Target/Sparc/Sparc.td
@@ -1,9 +1,8 @@
 //===-- Sparc.td - Describe the Sparc Target Machine -------*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Sparc/SparcAsmPrinter.cpp b/lib/Target/Sparc/SparcAsmPrinter.cpp
index 5f0e359a3b006e46a21432142dde1261d7060e3e..2254e7bc4778c9398afd7af5758375f75e0537e4 100644
--- a/lib/Target/Sparc/SparcAsmPrinter.cpp
+++ b/lib/Target/Sparc/SparcAsmPrinter.cpp
@@ -1,9 +1,8 @@
 //===-- SparcAsmPrinter.cpp - Sparc LLVM assembly writer ------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Sparc/SparcCallingConv.td b/lib/Target/Sparc/SparcCallingConv.td
index 0aa29d186dc1829e13ef665f985b8fbb163c6d52..4be432211f1dfdf217542f4e1ed6383992b041f0 100644
--- a/lib/Target/Sparc/SparcCallingConv.td
+++ b/lib/Target/Sparc/SparcCallingConv.td
@@ -1,9 +1,8 @@
 //===-- SparcCallingConv.td - Calling Conventions Sparc ----*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Sparc/SparcFrameLowering.cpp b/lib/Target/Sparc/SparcFrameLowering.cpp
index 9f6c7d65592d4031fb6200a1c435ba370c63850e..1834a6fd861da182d8a1b1282e1ca762452047a3 100644
--- a/lib/Target/Sparc/SparcFrameLowering.cpp
+++ b/lib/Target/Sparc/SparcFrameLowering.cpp
@@ -1,9 +1,8 @@
 //===-- SparcFrameLowering.cpp - Sparc Frame Information ------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Sparc/SparcFrameLowering.h b/lib/Target/Sparc/SparcFrameLowering.h
index 6098afa6898528e7f4cbf0d761469147693aef55..8e6001da05db21224a763228886a9445d88f751f 100644
--- a/lib/Target/Sparc/SparcFrameLowering.h
+++ b/lib/Target/Sparc/SparcFrameLowering.h
@@ -1,9 +1,8 @@
 //===-- SparcFrameLowering.h - Define frame lowering for Sparc --*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Sparc/SparcISelDAGToDAG.cpp b/lib/Target/Sparc/SparcISelDAGToDAG.cpp
index f845c41ede45adf13b2191545a0a9cae0a53c3ca..8cff50d19ed46a0680c76f436dcfe63a135e4003 100644
--- a/lib/Target/Sparc/SparcISelDAGToDAG.cpp
+++ b/lib/Target/Sparc/SparcISelDAGToDAG.cpp
@@ -1,9 +1,8 @@
 //===-- SparcISelDAGToDAG.cpp - A dag to dag inst selector for Sparc ------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -313,7 +312,7 @@ bool SparcDAGToDAGISel::tryInlineAsm(SDNode *N){
 
   SelectInlineAsmMemoryOperands(AsmNodeOperands, SDLoc(N));
 
-  SDValue New = CurDAG->getNode(ISD::INLINEASM, SDLoc(N),
+  SDValue New = CurDAG->getNode(N->getOpcode(), SDLoc(N),
       CurDAG->getVTList(MVT::Other, MVT::Glue), AsmNodeOperands);
   New->setNodeId(-1);
   ReplaceNode(N, New.getNode());
@@ -329,7 +328,8 @@ void SparcDAGToDAGISel::Select(SDNode *N) {
 
   switch (N->getOpcode()) {
   default: break;
-  case ISD::INLINEASM: {
+  case ISD::INLINEASM:
+  case ISD::INLINEASM_BR: {
     if (tryInlineAsm(N))
       return;
     break;
diff --git a/lib/Target/Sparc/SparcISelLowering.cpp b/lib/Target/Sparc/SparcISelLowering.cpp
index 7d908bb6b3207b24c467aa1e21e05a4d30ccef3d..0592dc8570d4c50eddff87c6a53a3e0fb76ab2d4 100644
--- a/lib/Target/Sparc/SparcISelLowering.cpp
+++ b/lib/Target/Sparc/SparcISelLowering.cpp
@@ -1,9 +1,8 @@
 //===-- SparcISelLowering.cpp - Sparc DAG Lowering Implementation ---------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -1866,8 +1865,8 @@ void SparcTargetLowering::computeKnownBitsForTargetNode
   case SPISD::SELECT_ICC:
   case SPISD::SELECT_XCC:
   case SPISD::SELECT_FCC:
-    DAG.computeKnownBits(Op.getOperand(1), Known, Depth+1);
-    DAG.computeKnownBits(Op.getOperand(0), Known2, Depth+1);
+    Known = DAG.computeKnownBits(Op.getOperand(1), Depth + 1);
+    Known2 = DAG.computeKnownBits(Op.getOperand(0), Depth + 1);
 
     // Only known if known in both the LHS and RHS.
     Known.One &= Known2.One;
diff --git a/lib/Target/Sparc/SparcISelLowering.h b/lib/Target/Sparc/SparcISelLowering.h
index 718851db25bff7aaa5248b72f92489fc0de09793..8d557a4225e54f4e290f1dd918c689f0d6f13eab 100644
--- a/lib/Target/Sparc/SparcISelLowering.h
+++ b/lib/Target/Sparc/SparcISelLowering.h
@@ -1,9 +1,8 @@
 //===-- SparcISelLowering.h - Sparc DAG Lowering Interface ------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Sparc/SparcInstr64Bit.td b/lib/Target/Sparc/SparcInstr64Bit.td
index 0b94c6b614ebbaf004e5fb3dcb6d1edb71a5655a..2d4f687f72d2ac7e185b34ee182b7ba0f5ff21da 100644
--- a/lib/Target/Sparc/SparcInstr64Bit.td
+++ b/lib/Target/Sparc/SparcInstr64Bit.td
@@ -1,9 +1,8 @@
 //===-- SparcInstr64Bit.td - 64-bit instructions for Sparc Target ---------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Sparc/SparcInstrAliases.td b/lib/Target/Sparc/SparcInstrAliases.td
index 35987390d7ba2f67c3dacd36ea0a2461e21656fe..d4d056ea0af68137bc4c2785b4d49b262468ac9d 100644
--- a/lib/Target/Sparc/SparcInstrAliases.td
+++ b/lib/Target/Sparc/SparcInstrAliases.td
@@ -1,9 +1,8 @@
 //===-- SparcInstrAliases.td - Instruction Aliases for Sparc Target -------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Sparc/SparcInstrFormats.td b/lib/Target/Sparc/SparcInstrFormats.td
index 76366c6695f448092f9bf4b6b6bc9b13b93b0dac..fbf08b49d60ceffb2090d4691be129f81c2be60a 100644
--- a/lib/Target/Sparc/SparcInstrFormats.td
+++ b/lib/Target/Sparc/SparcInstrFormats.td
@@ -1,9 +1,8 @@
 //===-- SparcInstrFormats.td - Sparc Instruction Formats ---*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/Sparc/SparcInstrInfo.cpp b/lib/Target/Sparc/SparcInstrInfo.cpp
index 47b42444b94da791a4d41245d7fc36a33d6a6d17..ad343fe6f80a4896ede772831233cebe8aecaa50 100644
--- a/lib/Target/Sparc/SparcInstrInfo.cpp
+++ b/lib/Target/Sparc/SparcInstrInfo.cpp
@@ -1,9 +1,8 @@
 //===-- SparcInstrInfo.cpp - Sparc Instruction Information ----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Sparc/SparcInstrInfo.h b/lib/Target/Sparc/SparcInstrInfo.h
index 524b5d054163243651671e9acf84ce715b829080..b587b28c25fc0a76c129389af095f143ea4b7498 100644
--- a/lib/Target/Sparc/SparcInstrInfo.h
+++ b/lib/Target/Sparc/SparcInstrInfo.h
@@ -1,9 +1,8 @@
 //===-- SparcInstrInfo.h - Sparc Instruction Information --------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Sparc/SparcInstrInfo.td b/lib/Target/Sparc/SparcInstrInfo.td
index 558b37aeebcb895267679184c29a8d474277d1bb..8474c7abffb3568cf851ba121f8437bdc1caf608 100644
--- a/lib/Target/Sparc/SparcInstrInfo.td
+++ b/lib/Target/Sparc/SparcInstrInfo.td
@@ -1,9 +1,8 @@
 //===-- SparcInstrInfo.td - Target Description for Sparc Target -----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Sparc/SparcInstrVIS.td b/lib/Target/Sparc/SparcInstrVIS.td
index d9adf3e8b0f52cbf34c104ba41e10a6d5bc85a52..bdefc70869d744b895df40e01e109efbde448708 100644
--- a/lib/Target/Sparc/SparcInstrVIS.td
+++ b/lib/Target/Sparc/SparcInstrVIS.td
@@ -1,9 +1,8 @@
 //===---- SparcInstrVIS.td - Visual Instruction Set extensions (VIS) -----===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Sparc/SparcMCInstLower.cpp b/lib/Target/Sparc/SparcMCInstLower.cpp
index a784124ff68839e87e460b1a992bd9e776e645c9..8ea317fdd45339b086005747da3245bb80eed321 100644
--- a/lib/Target/Sparc/SparcMCInstLower.cpp
+++ b/lib/Target/Sparc/SparcMCInstLower.cpp
@@ -1,9 +1,8 @@
 //===-- SparcMCInstLower.cpp - Convert Sparc MachineInstr to MCInst -------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Sparc/SparcMachineFunctionInfo.cpp b/lib/Target/Sparc/SparcMachineFunctionInfo.cpp
index e7442826e78b472f6714ff1e1798f1471fb2955f..7c36c4ab865f0df979fc86a61bacb3c2ed6ad8d4 100644
--- a/lib/Target/Sparc/SparcMachineFunctionInfo.cpp
+++ b/lib/Target/Sparc/SparcMachineFunctionInfo.cpp
@@ -1,9 +1,8 @@
 //===-- SparcMachineFunctionInfo.cpp - Sparc Machine Function Info --------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/Sparc/SparcMachineFunctionInfo.h b/lib/Target/Sparc/SparcMachineFunctionInfo.h
index 104744279d9d1e01cf887e36ae976ba955f268b8..fe5705878693316f39b6212556c3e1c45a25bfd0 100644
--- a/lib/Target/Sparc/SparcMachineFunctionInfo.h
+++ b/lib/Target/Sparc/SparcMachineFunctionInfo.h
@@ -1,9 +1,8 @@
 //===- SparcMachineFunctionInfo.h - Sparc Machine Function Info -*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Sparc/SparcRegisterInfo.cpp b/lib/Target/Sparc/SparcRegisterInfo.cpp
index 33caa66154ff2b72ebe563e1324d269a70cdc788..9a3010639f25569b20df0b83fd8b754b25e28b3f 100644
--- a/lib/Target/Sparc/SparcRegisterInfo.cpp
+++ b/lib/Target/Sparc/SparcRegisterInfo.cpp
@@ -1,9 +1,8 @@
 //===-- SparcRegisterInfo.cpp - SPARC Register Information ----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Sparc/SparcRegisterInfo.h b/lib/Target/Sparc/SparcRegisterInfo.h
index 8dd2569d10defa280f5ffaaa46a863a0c79120fb..a6b5b88f43ccb65b29bff9732e4f243510a58ded 100644
--- a/lib/Target/Sparc/SparcRegisterInfo.h
+++ b/lib/Target/Sparc/SparcRegisterInfo.h
@@ -1,9 +1,8 @@
 //===-- SparcRegisterInfo.h - Sparc Register Information Impl ---*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Sparc/SparcRegisterInfo.td b/lib/Target/Sparc/SparcRegisterInfo.td
index 6625eaafd992634435692b0063e38d998a871e9e..98959d512955151584a9947425361c43be194373 100644
--- a/lib/Target/Sparc/SparcRegisterInfo.td
+++ b/lib/Target/Sparc/SparcRegisterInfo.td
@@ -1,9 +1,8 @@
 //===-- SparcRegisterInfo.td - Sparc Register defs ---------*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/Sparc/SparcSchedule.td b/lib/Target/Sparc/SparcSchedule.td
index f243546b029b1eab9404038ca136369499265405..31e43c9bd95dc70a67aadf88761716e805dc5e2b 100644
--- a/lib/Target/Sparc/SparcSchedule.td
+++ b/lib/Target/Sparc/SparcSchedule.td
@@ -1,9 +1,8 @@
 //===-- SparcSchedule.td - Describe the Sparc Itineries ----*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Sparc/SparcSubtarget.cpp b/lib/Target/Sparc/SparcSubtarget.cpp
index 5301fc30a006f5c4555f38c2dc70f23decc1a508..075a002a358d76d506789012c72904d43058f0f5 100644
--- a/lib/Target/Sparc/SparcSubtarget.cpp
+++ b/lib/Target/Sparc/SparcSubtarget.cpp
@@ -1,9 +1,8 @@
 //===-- SparcSubtarget.cpp - SPARC Subtarget Information ------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Sparc/SparcSubtarget.h b/lib/Target/Sparc/SparcSubtarget.h
index 24ea41a266e7d1f354f2b27a189c780fab7c7da7..db19f99e3c9c7f151b6fdc4882c8bc13fa87043c 100644
--- a/lib/Target/Sparc/SparcSubtarget.h
+++ b/lib/Target/Sparc/SparcSubtarget.h
@@ -1,9 +1,8 @@
 //===-- SparcSubtarget.h - Define Subtarget for the SPARC -------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Sparc/SparcTargetMachine.cpp b/lib/Target/Sparc/SparcTargetMachine.cpp
index 5b467235f809d20743692fc1d81b577d9e3e70d5..cb9df3ba4929aa48809e48cafd747fa0e4ebdd74 100644
--- a/lib/Target/Sparc/SparcTargetMachine.cpp
+++ b/lib/Target/Sparc/SparcTargetMachine.cpp
@@ -1,9 +1,8 @@
 //===-- SparcTargetMachine.cpp - Define TargetMachine for Sparc -----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Sparc/SparcTargetMachine.h b/lib/Target/Sparc/SparcTargetMachine.h
index d1eb1d329a4cc3e02e5db5c9aa2cf0b23e21fab4..4083f61433b1678d0ebb2e823869ef146a157854 100644
--- a/lib/Target/Sparc/SparcTargetMachine.h
+++ b/lib/Target/Sparc/SparcTargetMachine.h
@@ -1,9 +1,8 @@
 //===-- SparcTargetMachine.h - Define TargetMachine for Sparc ---*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/Sparc/SparcTargetObjectFile.cpp b/lib/Target/Sparc/SparcTargetObjectFile.cpp
index d0db854f784966889f6b042501ef953152abf423..e6ad4d2d67aa7517d5b1a3386e0b4df9a12bd71c 100644
--- a/lib/Target/Sparc/SparcTargetObjectFile.cpp
+++ b/lib/Target/Sparc/SparcTargetObjectFile.cpp
@@ -1,9 +1,8 @@
 //===------- SparcTargetObjectFile.cpp - Sparc Object Info Impl -----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/Sparc/SparcTargetObjectFile.h b/lib/Target/Sparc/SparcTargetObjectFile.h
index 3b1b345c3b193020e370c5a8546c616575571a59..9bbe602b32b30d16c8ce293f76fe9ddc114aa141 100644
--- a/lib/Target/Sparc/SparcTargetObjectFile.h
+++ b/lib/Target/Sparc/SparcTargetObjectFile.h
@@ -1,9 +1,8 @@
 //===-- SparcTargetObjectFile.h - Sparc Object Info -------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/Sparc/TargetInfo/LLVMBuild.txt b/lib/Target/Sparc/TargetInfo/LLVMBuild.txt
index e992d3e81f180f78d5a379bdaac803387a7037d8..e355080d3780f0500e8b678fbda9b7c64e516869 100644
--- a/lib/Target/Sparc/TargetInfo/LLVMBuild.txt
+++ b/lib/Target/Sparc/TargetInfo/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/Target/Sparc/TargetInfo/LLVMBuild.txt --------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Target/Sparc/TargetInfo/SparcTargetInfo.cpp b/lib/Target/Sparc/TargetInfo/SparcTargetInfo.cpp
index d030bd9f232db85dfc1468f6129d42718bdfe9cf..08003dda4add02bba41dccbc9ec91150ae887ad9 100644
--- a/lib/Target/Sparc/TargetInfo/SparcTargetInfo.cpp
+++ b/lib/Target/Sparc/TargetInfo/SparcTargetInfo.cpp
@@ -1,9 +1,8 @@
 //===-- SparcTargetInfo.cpp - Sparc Target Implementation -----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/SystemZ/AsmParser/LLVMBuild.txt b/lib/Target/SystemZ/AsmParser/LLVMBuild.txt
index 76aa5a4aa9dcacfe10c56c7e85cbff1f3190f659..cf093dae18faa086a216f74e0f1a4e2759d41df1 100644
--- a/lib/Target/SystemZ/AsmParser/LLVMBuild.txt
+++ b/lib/Target/SystemZ/AsmParser/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/Target/SystemZ/AsmParser/LLVMBuild.txt -------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Target/SystemZ/AsmParser/SystemZAsmParser.cpp b/lib/Target/SystemZ/AsmParser/SystemZAsmParser.cpp
index 91959b4151b32334300f93f7cc5da0cbcd32ab8f..a9741bbda4dfcaa63dfbb5242e3d74fcfef018e6 100644
--- a/lib/Target/SystemZ/AsmParser/SystemZAsmParser.cpp
+++ b/lib/Target/SystemZ/AsmParser/SystemZAsmParser.cpp
@@ -1,9 +1,8 @@
 //===-- SystemZAsmParser.cpp - Parse SystemZ assembly instructions --------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/SystemZ/Disassembler/LLVMBuild.txt b/lib/Target/SystemZ/Disassembler/LLVMBuild.txt
index fd7826999bc53a27a6b8688beafe1f8cb1fbab77..b326216f370de82283e48dfa1e336a8ded7c17b4 100644
--- a/lib/Target/SystemZ/Disassembler/LLVMBuild.txt
+++ b/lib/Target/SystemZ/Disassembler/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===-- ./lib/Target/SystemZ/Disassembler/LLVMBuild.txt ---------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Target/SystemZ/Disassembler/SystemZDisassembler.cpp b/lib/Target/SystemZ/Disassembler/SystemZDisassembler.cpp
index 8903b57ffd0b456cc5ddab4fd95a79c44b5831d0..7b50b9ce807207e59bdc00421e857825fc54927e 100644
--- a/lib/Target/SystemZ/Disassembler/SystemZDisassembler.cpp
+++ b/lib/Target/SystemZ/Disassembler/SystemZDisassembler.cpp
@@ -1,9 +1,8 @@
 //===-- SystemZDisassembler.cpp - Disassembler for SystemZ ------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/SystemZ/InstPrinter/LLVMBuild.txt b/lib/Target/SystemZ/InstPrinter/LLVMBuild.txt
index fdfd738683949ee4d34ef9880b46112d76b7276f..b1b8717dbb5fbeb4469eec7d2f314816c0a5158a 100644
--- a/lib/Target/SystemZ/InstPrinter/LLVMBuild.txt
+++ b/lib/Target/SystemZ/InstPrinter/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/Target/SystemZ/InstPrinter/LLVMBuild.txt -----------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Target/SystemZ/InstPrinter/SystemZInstPrinter.cpp b/lib/Target/SystemZ/InstPrinter/SystemZInstPrinter.cpp
index 6cd12e13e220c02c51d85c5c5e3a17a77fc2faa1..91cb35dd72f26ff452f3a86760ade8e1d00a25fb 100644
--- a/lib/Target/SystemZ/InstPrinter/SystemZInstPrinter.cpp
+++ b/lib/Target/SystemZ/InstPrinter/SystemZInstPrinter.cpp
@@ -1,9 +1,8 @@
 //===- SystemZInstPrinter.cpp - Convert SystemZ MCInst to assembly syntax -===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/SystemZ/InstPrinter/SystemZInstPrinter.h b/lib/Target/SystemZ/InstPrinter/SystemZInstPrinter.h
index d65c661545eb5749e16e0f9da4a2c17ccc182926..454b2c095e2f6721e6fd22e07fbf088aca657fdb 100644
--- a/lib/Target/SystemZ/InstPrinter/SystemZInstPrinter.h
+++ b/lib/Target/SystemZ/InstPrinter/SystemZInstPrinter.h
@@ -1,9 +1,8 @@
 //==- SystemZInstPrinter.h - Convert SystemZ MCInst to assembly --*- C++ -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/SystemZ/LLVMBuild.txt b/lib/Target/SystemZ/LLVMBuild.txt
index 9b8b141fd52ab74c95faf84abeceb7ad0dd6be3b..35919fea7288e4ae2557fd31c12b2877098f0720 100644
--- a/lib/Target/SystemZ/LLVMBuild.txt
+++ b/lib/Target/SystemZ/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/Target/SystemZ/LLVMBuild.txt -----------------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Target/SystemZ/MCTargetDesc/LLVMBuild.txt b/lib/Target/SystemZ/MCTargetDesc/LLVMBuild.txt
index dabd214ba3f581ff1028f56c6452f6d9afee5805..a159aa43e79016212d5bc65758de2fb326d56563 100644
--- a/lib/Target/SystemZ/MCTargetDesc/LLVMBuild.txt
+++ b/lib/Target/SystemZ/MCTargetDesc/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/Target/SystemZ/MCTargetDesc/LLVMBuild.txt ----------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Target/SystemZ/MCTargetDesc/SystemZMCAsmBackend.cpp b/lib/Target/SystemZ/MCTargetDesc/SystemZMCAsmBackend.cpp
index 2146832f77944c14b25db235e2d95821f8d06466..23d8585095ccad10b10dd4fcfbf09c12247447a2 100644
--- a/lib/Target/SystemZ/MCTargetDesc/SystemZMCAsmBackend.cpp
+++ b/lib/Target/SystemZ/MCTargetDesc/SystemZMCAsmBackend.cpp
@@ -1,9 +1,8 @@
 //===-- SystemZMCAsmBackend.cpp - SystemZ assembler backend ---------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/SystemZ/MCTargetDesc/SystemZMCAsmInfo.cpp b/lib/Target/SystemZ/MCTargetDesc/SystemZMCAsmInfo.cpp
index 6e00981939b638b788ddc38e459b4673c9743eb1..d6cdacfcab92401177ce04244e3ed8d620104558 100644
--- a/lib/Target/SystemZ/MCTargetDesc/SystemZMCAsmInfo.cpp
+++ b/lib/Target/SystemZ/MCTargetDesc/SystemZMCAsmInfo.cpp
@@ -1,9 +1,8 @@
 //===-- SystemZMCAsmInfo.cpp - SystemZ asm properties ---------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/SystemZ/MCTargetDesc/SystemZMCAsmInfo.h b/lib/Target/SystemZ/MCTargetDesc/SystemZMCAsmInfo.h
index 800f89232063221a3fcc9e8977c6c9b1fada53fd..b8818a65f9e3989a5f8aa50fc06d0cb0c5f8c966 100644
--- a/lib/Target/SystemZ/MCTargetDesc/SystemZMCAsmInfo.h
+++ b/lib/Target/SystemZ/MCTargetDesc/SystemZMCAsmInfo.h
@@ -1,9 +1,8 @@
 //====-- SystemZMCAsmInfo.h - SystemZ asm properties -----------*- C++ -*--===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/SystemZ/MCTargetDesc/SystemZMCCodeEmitter.cpp b/lib/Target/SystemZ/MCTargetDesc/SystemZMCCodeEmitter.cpp
index d188f56512ab7aff24c050a7ad81dadab2d67ef7..6503bb8be68fe6a9bbe9abf3a5b2b45b5e3d7b3a 100644
--- a/lib/Target/SystemZ/MCTargetDesc/SystemZMCCodeEmitter.cpp
+++ b/lib/Target/SystemZ/MCTargetDesc/SystemZMCCodeEmitter.cpp
@@ -1,9 +1,8 @@
 //===-- SystemZMCCodeEmitter.cpp - Convert SystemZ code to machine code ---===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/SystemZ/MCTargetDesc/SystemZMCFixups.h b/lib/Target/SystemZ/MCTargetDesc/SystemZMCFixups.h
index c012accc14dddc17deb184564c0f3febdfc287c3..14f6198183b975fbe7c063058ac2a6904f17956b 100644
--- a/lib/Target/SystemZ/MCTargetDesc/SystemZMCFixups.h
+++ b/lib/Target/SystemZ/MCTargetDesc/SystemZMCFixups.h
@@ -1,9 +1,8 @@
 //===-- SystemZMCFixups.h - SystemZ-specific fixup entries ------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/SystemZ/MCTargetDesc/SystemZMCObjectWriter.cpp b/lib/Target/SystemZ/MCTargetDesc/SystemZMCObjectWriter.cpp
index 888be519fb1637f8b0ea2d5114ba272d315abf34..13bd7ee4be956d2ba7bd6169754c280a19507959 100644
--- a/lib/Target/SystemZ/MCTargetDesc/SystemZMCObjectWriter.cpp
+++ b/lib/Target/SystemZ/MCTargetDesc/SystemZMCObjectWriter.cpp
@@ -1,9 +1,8 @@
 //===-- SystemZMCObjectWriter.cpp - SystemZ ELF writer --------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/SystemZ/MCTargetDesc/SystemZMCTargetDesc.cpp b/lib/Target/SystemZ/MCTargetDesc/SystemZMCTargetDesc.cpp
index 05688ed8efbb6841b9019a460033d8cbcd14dd33..84e4a2eb455c6a35448adcc79c94c7245cc100de 100644
--- a/lib/Target/SystemZ/MCTargetDesc/SystemZMCTargetDesc.cpp
+++ b/lib/Target/SystemZ/MCTargetDesc/SystemZMCTargetDesc.cpp
@@ -1,9 +1,8 @@
 //===-- SystemZMCTargetDesc.cpp - SystemZ target descriptions -------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/SystemZ/MCTargetDesc/SystemZMCTargetDesc.h b/lib/Target/SystemZ/MCTargetDesc/SystemZMCTargetDesc.h
index 1617a807e65a8c45ac16afd467ba2bcb41a1fd24..53fa358293355447fbcb250e69705440d5e0875a 100644
--- a/lib/Target/SystemZ/MCTargetDesc/SystemZMCTargetDesc.h
+++ b/lib/Target/SystemZ/MCTargetDesc/SystemZMCTargetDesc.h
@@ -1,9 +1,8 @@
 //===-- SystemZMCTargetDesc.h - SystemZ target descriptions -----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/SystemZ/SystemZ.h b/lib/Target/SystemZ/SystemZ.h
index fdbde3d8dbc3a381ada43f08286ff481e1718e0e..fbffea183bc39777d57044f0c9f0dbf585af2cd0 100644
--- a/lib/Target/SystemZ/SystemZ.h
+++ b/lib/Target/SystemZ/SystemZ.h
@@ -1,9 +1,8 @@
 //==- SystemZ.h - Top-Level Interface for SystemZ representation -*- C++ -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/SystemZ/SystemZ.td b/lib/Target/SystemZ/SystemZ.td
index 3800f7a26b790a24dc301d32e52aaa37c6c4766d..ebbc6ffd2f1e0e7c6c47bcec9035ef407469e31c 100644
--- a/lib/Target/SystemZ/SystemZ.td
+++ b/lib/Target/SystemZ/SystemZ.td
@@ -1,9 +1,8 @@
 //===-- SystemZ.td - Describe the SystemZ target machine -----*- tblgen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/SystemZ/SystemZAsmPrinter.cpp b/lib/Target/SystemZ/SystemZAsmPrinter.cpp
index e2de721be56869249ec7cca62c546554618da092..162d30f819816dd53522777847a0f6f248826a27 100644
--- a/lib/Target/SystemZ/SystemZAsmPrinter.cpp
+++ b/lib/Target/SystemZ/SystemZAsmPrinter.cpp
@@ -1,9 +1,8 @@
 //===-- SystemZAsmPrinter.cpp - SystemZ LLVM assembly printer -------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/SystemZ/SystemZAsmPrinter.h b/lib/Target/SystemZ/SystemZAsmPrinter.h
index cb88ec32f83af1b97dc71600d4230b9ce2e8d9fd..434df99153c4ebba3b055b46e641158e07ac6f9c 100644
--- a/lib/Target/SystemZ/SystemZAsmPrinter.h
+++ b/lib/Target/SystemZ/SystemZAsmPrinter.h
@@ -1,9 +1,8 @@
 //===-- SystemZAsmPrinter.h - SystemZ LLVM assembly printer ----*- C++ -*--===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/SystemZ/SystemZCallingConv.cpp b/lib/Target/SystemZ/SystemZCallingConv.cpp
index 72da51f74b104d6ddedd8352d7afe462497bb405..91c7fae17a75a617ce8430cfc09e8eef9da470d5 100644
--- a/lib/Target/SystemZ/SystemZCallingConv.cpp
+++ b/lib/Target/SystemZ/SystemZCallingConv.cpp
@@ -1,9 +1,8 @@
 //===-- SystemZCallingConv.cpp - Calling conventions for SystemZ ----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/SystemZ/SystemZCallingConv.h b/lib/Target/SystemZ/SystemZCallingConv.h
index b5523e586f4cecaa4a26f687c8999775532168cb..82f29b6361f1ba234d630dfe738f7908395a19aa 100644
--- a/lib/Target/SystemZ/SystemZCallingConv.h
+++ b/lib/Target/SystemZ/SystemZCallingConv.h
@@ -1,9 +1,8 @@
 //===-- SystemZCallingConv.h - Calling conventions for SystemZ --*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/SystemZ/SystemZCallingConv.td b/lib/Target/SystemZ/SystemZCallingConv.td
index deba27fee7fe502954e0b4a3c0698b0944e4f78b..bbd51546ac9f2d0bd3408ff46abbe5f621cee612 100644
--- a/lib/Target/SystemZ/SystemZCallingConv.td
+++ b/lib/Target/SystemZ/SystemZCallingConv.td
@@ -1,9 +1,8 @@
 //=- SystemZCallingConv.td - Calling conventions for SystemZ -*- tablegen -*-=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 // This describes the calling conventions for the SystemZ ABI.
diff --git a/lib/Target/SystemZ/SystemZConstantPoolValue.cpp b/lib/Target/SystemZ/SystemZConstantPoolValue.cpp
index 4a6beb67f1820f561fca14b11b1ca8251993b5f9..ffeee4da95ccd782b9280552de189c761bbfdabf 100644
--- a/lib/Target/SystemZ/SystemZConstantPoolValue.cpp
+++ b/lib/Target/SystemZ/SystemZConstantPoolValue.cpp
@@ -1,9 +1,8 @@
 //===-- SystemZConstantPoolValue.cpp - SystemZ constant-pool value --------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/SystemZ/SystemZConstantPoolValue.h b/lib/Target/SystemZ/SystemZConstantPoolValue.h
index a71b595560d2f168d0cbffc651048dbe4ec04d85..6cb7710abdfe31df0a3f6a6c5b3929548856976a 100644
--- a/lib/Target/SystemZ/SystemZConstantPoolValue.h
+++ b/lib/Target/SystemZ/SystemZConstantPoolValue.h
@@ -1,9 +1,8 @@
 //===- SystemZConstantPoolValue.h - SystemZ constant-pool value -*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/SystemZ/SystemZElimCompare.cpp b/lib/Target/SystemZ/SystemZElimCompare.cpp
index 668a77ac014ff63b5a370a9e2da4bec41b749816..4c13cd6bfca7cbec0b4023b41c3ae6c5139bbb28 100644
--- a/lib/Target/SystemZ/SystemZElimCompare.cpp
+++ b/lib/Target/SystemZ/SystemZElimCompare.cpp
@@ -1,9 +1,8 @@
 //===-- SystemZElimCompare.cpp - Eliminate comparison instructions --------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -147,6 +146,9 @@ static bool resultTests(MachineInstr &MI, unsigned Reg) {
 // Describe the references to Reg or any of its aliases in MI.
 Reference SystemZElimCompare::getRegReferences(MachineInstr &MI, unsigned Reg) {
   Reference Ref;
+  if (MI.isDebugInstr())
+    return Ref;
+
   for (unsigned I = 0, E = MI.getNumOperands(); I != E; ++I) {
     const MachineOperand &MO = MI.getOperand(I);
     if (MO.isReg()) {
diff --git a/lib/Target/SystemZ/SystemZExpandPseudo.cpp b/lib/Target/SystemZ/SystemZExpandPseudo.cpp
index 67c80899d4916436b3f7d40de7667d87ade96762..09708fb4241c9083fd5702139563338249413a1d 100644
--- a/lib/Target/SystemZ/SystemZExpandPseudo.cpp
+++ b/lib/Target/SystemZ/SystemZExpandPseudo.cpp
@@ -1,9 +1,8 @@
 //==-- SystemZExpandPseudo.cpp - Expand pseudo instructions -------*- C++ -*-=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/SystemZ/SystemZFeatures.td b/lib/Target/SystemZ/SystemZFeatures.td
index beff45dba81dd1426b7ec1370fa92b54ef96722d..2dc44edcc05deb05d862692fb42ffc3a8bf62e99 100644
--- a/lib/Target/SystemZ/SystemZFeatures.td
+++ b/lib/Target/SystemZ/SystemZFeatures.td
@@ -1,9 +1,8 @@
 //===-- SystemZ.td - SystemZ processors and features ---------*- tblgen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/SystemZ/SystemZFrameLowering.cpp b/lib/Target/SystemZ/SystemZFrameLowering.cpp
index 565299c90139f123b65cc3ad27ea2ddd67fabb00..da28faebb3264be8d3f463a12a5d2ae1c93b2283 100644
--- a/lib/Target/SystemZ/SystemZFrameLowering.cpp
+++ b/lib/Target/SystemZ/SystemZFrameLowering.cpp
@@ -1,9 +1,8 @@
 //===-- SystemZFrameLowering.cpp - Frame lowering for SystemZ -------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/SystemZ/SystemZFrameLowering.h b/lib/Target/SystemZ/SystemZFrameLowering.h
index 08c84c785cc0c5f1855e478df68de38660a4eaf7..71ef3e4dc2408c07d7c8bff930bc8a20ba7a43b9 100644
--- a/lib/Target/SystemZ/SystemZFrameLowering.h
+++ b/lib/Target/SystemZ/SystemZFrameLowering.h
@@ -1,9 +1,8 @@
 //===-- SystemZFrameLowering.h - Frame lowering for SystemZ -----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/SystemZ/SystemZHazardRecognizer.cpp b/lib/Target/SystemZ/SystemZHazardRecognizer.cpp
index 8726b56bc94f9e896451a2433919506fa526a69a..e2af0222799919257a93606bf54ed87bd88f3cd2 100644
--- a/lib/Target/SystemZ/SystemZHazardRecognizer.cpp
+++ b/lib/Target/SystemZ/SystemZHazardRecognizer.cpp
@@ -1,9 +1,8 @@
 //=-- SystemZHazardRecognizer.h - SystemZ Hazard Recognizer -----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/SystemZ/SystemZHazardRecognizer.h b/lib/Target/SystemZ/SystemZHazardRecognizer.h
index 6292feefbfea2b58bb0a85ce4b551605cc7ee2d2..38bf41ebe96a13e004f853fa3f32e8aa3f0adda3 100644
--- a/lib/Target/SystemZ/SystemZHazardRecognizer.h
+++ b/lib/Target/SystemZ/SystemZHazardRecognizer.h
@@ -1,9 +1,8 @@
 //=-- SystemZHazardRecognizer.h - SystemZ Hazard Recognizer -----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/SystemZ/SystemZISelDAGToDAG.cpp b/lib/Target/SystemZ/SystemZISelDAGToDAG.cpp
index 0d2c2389847f6b643f1cfa5ee22a8ab89020e7fa..a963638be9828b7c16eb619267106ac97cd702f0 100644
--- a/lib/Target/SystemZ/SystemZISelDAGToDAG.cpp
+++ b/lib/Target/SystemZ/SystemZISelDAGToDAG.cpp
@@ -1,9 +1,8 @@
 //===-- SystemZISelDAGToDAG.cpp - A dag to dag inst selector for SystemZ --===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -12,6 +11,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "SystemZTargetMachine.h"
+#include "SystemZISelLowering.h"
 #include "llvm/Analysis/AliasAnalysis.h"
 #include "llvm/CodeGen/SelectionDAGISel.h"
 #include "llvm/Support/Debug.h"
@@ -728,8 +728,7 @@ bool SystemZDAGToDAGISel::detectOrAndInsertion(SDValue &Op,
   // The inner check covers all cases but is more expensive.
   uint64_t Used = allOnes(Op.getValueSizeInBits());
   if (Used != (AndMask | InsertMask)) {
-    KnownBits Known;
-    CurDAG->computeKnownBits(Op.getOperand(0), Known);
+    KnownBits Known = CurDAG->computeKnownBits(Op.getOperand(0));
     if (Used != (AndMask | InsertMask | Known.Zero.getZExtValue()))
       return false;
   }
@@ -787,8 +786,7 @@ bool SystemZDAGToDAGISel::expandRxSBG(RxSBGOperands &RxSBG) const {
       // If some bits of Input are already known zeros, those bits will have
       // been removed from the mask.  See if adding them back in makes the
       // mask suitable.
-      KnownBits Known;
-      CurDAG->computeKnownBits(Input, Known);
+      KnownBits Known = CurDAG->computeKnownBits(Input);
       Mask |= Known.Zero.getZExtValue();
       if (!refineRxSBGMask(RxSBG, Mask))
         return false;
@@ -811,8 +809,7 @@ bool SystemZDAGToDAGISel::expandRxSBG(RxSBGOperands &RxSBG) const {
       // If some bits of Input are already known ones, those bits will have
       // been removed from the mask.  See if adding them back in makes the
       // mask suitable.
-      KnownBits Known;
-      CurDAG->computeKnownBits(Input, Known);
+      KnownBits Known = CurDAG->computeKnownBits(Input);
       Mask &= ~Known.One.getZExtValue();
       if (!refineRxSBGMask(RxSBG, Mask))
         return false;
@@ -1147,7 +1144,7 @@ bool SystemZDAGToDAGISel::tryGather(SDNode *N, unsigned Opcode) {
     return false;
 
   auto *Load = dyn_cast<LoadSDNode>(N->getOperand(1));
-  if (!Load || !Load->hasOneUse())
+  if (!Load || !Load->hasNUsesOfValue(1, 0))
     return false;
   if (Load->getMemoryVT().getSizeInBits() !=
       Load->getValueType(0).getSizeInBits())
@@ -1530,6 +1527,44 @@ void SystemZDAGToDAGISel::Select(SDNode *Node) {
     break;
   }
 
+  case ISD::BUILD_VECTOR: {
+    auto *BVN = cast<BuildVectorSDNode>(Node);
+    SDLoc DL(Node);
+    EVT VT = Node->getValueType(0);
+    uint64_t Mask = 0;
+    if (SystemZTargetLowering::tryBuildVectorByteMask(BVN, Mask)) {
+      SDNode *Res = CurDAG->getMachineNode(SystemZ::VGBM, DL, VT,
+                                CurDAG->getTargetConstant(Mask, DL, MVT::i32));
+      ReplaceNode(Node, Res);
+      return;
+    }
+    break;
+  }
+
+  case ISD::ConstantFP: {
+    APFloat Imm = cast<ConstantFPSDNode>(Node)->getValueAPF();
+    if (Imm.isZero() || Imm.isNegZero())
+      break;
+    const SystemZInstrInfo *TII = getInstrInfo();
+    EVT VT = Node->getValueType(0);
+    unsigned Start, End;
+    unsigned BitWidth = VT.getSizeInBits();
+    bool Success = SystemZTargetLowering::analyzeFPImm(Imm, BitWidth, Start,
+              End, static_cast<const SystemZInstrInfo *>(TII)); (void)Success;
+    assert(Success && "Expected legal FP immediate");
+    SDLoc DL(Node);
+    unsigned Opcode = (BitWidth == 32 ? SystemZ::VGMF : SystemZ::VGMG);
+    SDNode *Res = CurDAG->getMachineNode(Opcode, DL, VT,
+                            CurDAG->getTargetConstant(Start, DL, MVT::i32),
+                            CurDAG->getTargetConstant(End, DL, MVT::i32));
+    unsigned SubRegIdx = (BitWidth == 32 ? SystemZ::subreg_h32
+                                         : SystemZ::subreg_h64);
+    Res = CurDAG->getTargetExtractSubreg(SubRegIdx, DL, VT, SDValue(Res, 0))
+            .getNode();
+    ReplaceNode(Node, Res);
+    return;
+  }
+
   case ISD::STORE: {
     if (tryFoldLoadStoreIntoMemOperand(Node))
       return;
diff --git a/lib/Target/SystemZ/SystemZISelLowering.cpp b/lib/Target/SystemZ/SystemZISelLowering.cpp
index d7951ca810eecc0a1173c05c2d1ffe02643c6998..6bbc4e9fd10e72ce0dc4c9ddbf5fbbfc9d9eacc3 100644
--- a/lib/Target/SystemZ/SystemZISelLowering.cpp
+++ b/lib/Target/SystemZ/SystemZISelLowering.cpp
@@ -1,9 +1,8 @@
 //===-- SystemZISelLowering.cpp - SystemZ DAG lowering implementation -----===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -250,6 +249,7 @@ SystemZTargetLowering::SystemZTargetLowering(const TargetMachine &TM,
 
   // We have native support for a 64-bit CTLZ, via FLOGR.
   setOperationAction(ISD::CTLZ, MVT::i32, Promote);
+  setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i32, Promote);
   setOperationAction(ISD::CTLZ, MVT::i64, Legal);
 
   // Give LowerOperation the chance to replace 64-bit ORs with subregs.
@@ -527,6 +527,7 @@ SystemZTargetLowering::SystemZTargetLowering(const TargetMachine &TM,
   setTargetDAGCombine(ISD::STORE);
   setTargetDAGCombine(ISD::EXTRACT_VECTOR_ELT);
   setTargetDAGCombine(ISD::FP_ROUND);
+  setTargetDAGCombine(ISD::FP_EXTEND);
   setTargetDAGCombine(ISD::BSWAP);
   setTargetDAGCombine(ISD::SDIV);
   setTargetDAGCombine(ISD::UDIV);
@@ -576,9 +577,39 @@ bool SystemZTargetLowering::isFMAFasterThanFMulAndFAdd(EVT VT) const {
   return false;
 }
 
+
+// Return true if Imm can be generated with a vector instruction, such as VGM.
+bool SystemZTargetLowering::
+analyzeFPImm(const APFloat &Imm, unsigned BitWidth, unsigned &Start,
+             unsigned &End, const SystemZInstrInfo *TII) {
+  APInt IntImm = Imm.bitcastToAPInt();
+  if (IntImm.getActiveBits() > 64)
+    return false;
+
+  // See if this immediate could be generated with VGM.
+  bool Success = TII->isRxSBGMask(IntImm.getZExtValue(), BitWidth, Start, End);
+  if (!Success)
+    return false;
+  // isRxSBGMask returns the bit numbers for a full 64-bit value,
+  // with 0 denoting 1 << 63 and 63 denoting 1.  Convert them to
+  // bit numbers for an BitsPerElement value, so that 0 denotes
+  // 1 << (BitsPerElement-1).
+  Start -= 64 - BitWidth;
+  End -= 64 - BitWidth;
+  return true;
+}
+
 bool SystemZTargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT) const {
   // We can load zero using LZ?R and negative zero using LZ?R;LC?BR.
-  return Imm.isZero() || Imm.isNegZero();
+  if (Imm.isZero() || Imm.isNegZero())
+    return true;
+
+  if (!Subtarget.hasVector())
+    return false;
+  const SystemZInstrInfo *TII =
+      static_cast<const SystemZInstrInfo *>(Subtarget.getInstrInfo());
+  unsigned Start, End;
+  return analyzeFPImm(Imm, VT.getSizeInBits(), Start, End, TII);
 }
 
 bool SystemZTargetLowering::isLegalICmpImmediate(int64_t Imm) const {
@@ -2218,8 +2249,7 @@ static void adjustForRedundantAnd(SelectionDAG &DAG, const SDLoc &DL,
   auto *Mask = dyn_cast<ConstantSDNode>(C.Op0.getOperand(1));
   if (!Mask)
     return;
-  KnownBits Known;
-  DAG.computeKnownBits(C.Op0.getOperand(0), Known);
+  KnownBits Known = DAG.computeKnownBits(C.Op0.getOperand(0));
   if ((~Known.Zero).getZExtValue() & ~Mask->getZExtValue())
     return;
 
@@ -2511,9 +2541,8 @@ SDValue SystemZTargetLowering::lowerVectorSETCC(SelectionDAG &DAG,
     break;
   }
   if (Invert) {
-    SDValue Mask = DAG.getNode(SystemZISD::BYTE_MASK, DL, MVT::v16i8,
-                               DAG.getConstant(65535, DL, MVT::i32));
-    Mask = DAG.getNode(ISD::BITCAST, DL, VT, Mask);
+    SDValue Mask =
+      DAG.getSplatBuildVector(VT, DL, DAG.getConstant(-1, DL, MVT::i64));
     Cmp = DAG.getNode(ISD::XOR, DL, VT, Cmp, Mask);
   }
   return Cmp;
@@ -3165,10 +3194,9 @@ SDValue SystemZTargetLowering::lowerOR(SDValue Op, SelectionDAG &DAG) const {
   assert(Op.getValueType() == MVT::i64 && "Should be 64-bit operation");
 
   // Get the known-zero masks for each operand.
-  SDValue Ops[] = { Op.getOperand(0), Op.getOperand(1) };
-  KnownBits Known[2];
-  DAG.computeKnownBits(Ops[0], Known[0]);
-  DAG.computeKnownBits(Ops[1], Known[1]);
+  SDValue Ops[] = {Op.getOperand(0), Op.getOperand(1)};
+  KnownBits Known[2] = {DAG.computeKnownBits(Ops[0]),
+                        DAG.computeKnownBits(Ops[1])};
 
   // See if the upper 32 bits of one operand and the lower 32 bits of the
   // other are known zero.  They are the low and high operands respectively.
@@ -3332,14 +3360,14 @@ SDValue SystemZTargetLowering::lowerCTPOP(SDValue Op,
       break;
     }
     case 32: {
-      SDValue Tmp = DAG.getNode(SystemZISD::BYTE_MASK, DL, MVT::v16i8,
-                                DAG.getConstant(0, DL, MVT::i32));
+      SDValue Tmp = DAG.getSplatBuildVector(MVT::v16i8, DL,
+                                            DAG.getConstant(0, DL, MVT::i32));
       Op = DAG.getNode(SystemZISD::VSUM, DL, VT, Op, Tmp);
       break;
     }
     case 64: {
-      SDValue Tmp = DAG.getNode(SystemZISD::BYTE_MASK, DL, MVT::v16i8,
-                                DAG.getConstant(0, DL, MVT::i32));
+      SDValue Tmp = DAG.getSplatBuildVector(MVT::v16i8, DL,
+                                            DAG.getConstant(0, DL, MVT::i32));
       Op = DAG.getNode(SystemZISD::VSUM, DL, MVT::v4i32, Op, Tmp);
       Op = DAG.getNode(SystemZISD::VSUM, DL, VT, Op, Tmp);
       break;
@@ -3351,8 +3379,7 @@ SDValue SystemZTargetLowering::lowerCTPOP(SDValue Op,
   }
 
   // Get the known-zero mask for the operand.
-  KnownBits Known;
-  DAG.computeKnownBits(Op, Known);
+  KnownBits Known = DAG.computeKnownBits(Op);
   unsigned NumSignificantBits = (~Known.Zero).getActiveBits();
   if (NumSignificantBits == 0)
     return DAG.getConstant(0, DL, VT);
@@ -4262,10 +4289,10 @@ static SDValue joinDwords(SelectionDAG &DAG, const SDLoc &DL, SDValue Op0,
   return DAG.getNode(SystemZISD::JOIN_DWORDS, DL, MVT::v2i64, Op0, Op1);
 }
 
-// Try to represent constant BUILD_VECTOR node BVN using a
-// SystemZISD::BYTE_MASK-style mask.  Store the mask value in Mask
-// on success.
-static bool tryBuildVectorByteMask(BuildVectorSDNode *BVN, uint64_t &Mask) {
+// Try to represent constant BUILD_VECTOR node BVN using a BYTE MASK style
+// mask.  Store the mask value in Mask on success.
+bool SystemZTargetLowering::
+tryBuildVectorByteMask(BuildVectorSDNode *BVN, uint64_t &Mask) {
   EVT ElemVT = BVN->getValueType(0).getVectorElementType();
   unsigned BytesPerElement = ElemVT.getStoreSize();
   for (unsigned I = 0, E = BVN->getNumOperands(); I != E; ++I) {
@@ -4544,13 +4571,11 @@ SDValue SystemZTargetLowering::lowerBUILD_VECTOR(SDValue Op,
     // Try using VECTOR GENERATE BYTE MASK.  This is the architecturally-
     // preferred way of creating all-zero and all-one vectors so give it
     // priority over other methods below.
-    uint64_t Mask = 0;
-    if (tryBuildVectorByteMask(BVN, Mask)) {
-      SDValue Op = DAG.getNode(
-          SystemZISD::BYTE_MASK, DL, MVT::v16i8,
-          DAG.getConstant(Mask, DL, MVT::i32, false, true /*isOpaque*/));
-      return DAG.getNode(ISD::BITCAST, DL, VT, Op);
-    }
+    uint64_t Mask;
+    if (ISD::isBuildVectorAllZeros(Op.getNode()) ||
+        ISD::isBuildVectorAllOnes(Op.getNode()) ||
+        (VT.isInteger() && tryBuildVectorByteMask(BVN, Mask)))
+      return Op;
 
     // Try using some form of replication.
     APInt SplatBits, SplatUndef;
@@ -5030,7 +5055,6 @@ const char *SystemZTargetLowering::getTargetNodeName(unsigned Opcode) const {
     OPCODE(TBEGIN);
     OPCODE(TBEGIN_NOFLOAT);
     OPCODE(TEND);
-    OPCODE(BYTE_MASK);
     OPCODE(ROTATE_MASK);
     OPCODE(REPLICATE);
     OPCODE(JOIN_DWORDS);
@@ -5342,8 +5366,7 @@ SDValue SystemZTargetLowering::combineMERGE(
   SDValue Op1 = N->getOperand(1);
   if (Op0.getOpcode() == ISD::BITCAST)
     Op0 = Op0.getOperand(0);
-  if (Op0.getOpcode() == SystemZISD::BYTE_MASK &&
-      cast<ConstantSDNode>(Op0.getOperand(0))->getZExtValue() == 0) {
+  if (ISD::isBuildVectorAllZeros(Op0.getNode())) {
     // (z_merge_* 0, 0) -> 0.  This is mostly useful for using VLLEZF
     // for v4f32.
     if (Op1 == N->getOperand(0))
@@ -5485,7 +5508,7 @@ SDValue SystemZTargetLowering::combineFP_ROUND(
   // (fpround (extract_vector_elt X 0))
   // (fpround (extract_vector_elt X 1)) ->
   // (extract_vector_elt (VROUND X) 0)
-  // (extract_vector_elt (VROUND X) 1)
+  // (extract_vector_elt (VROUND X) 2)
   //
   // This is a special case since the target doesn't really support v2f32s.
   SelectionDAG &DAG = DCI.DAG;
@@ -5527,6 +5550,53 @@ SDValue SystemZTargetLowering::combineFP_ROUND(
   return SDValue();
 }
 
+SDValue SystemZTargetLowering::combineFP_EXTEND(
+    SDNode *N, DAGCombinerInfo &DCI) const {
+  // (fpextend (extract_vector_elt X 0))
+  // (fpextend (extract_vector_elt X 2)) ->
+  // (extract_vector_elt (VEXTEND X) 0)
+  // (extract_vector_elt (VEXTEND X) 1)
+  //
+  // This is a special case since the target doesn't really support v2f32s.
+  SelectionDAG &DAG = DCI.DAG;
+  SDValue Op0 = N->getOperand(0);
+  if (N->getValueType(0) == MVT::f64 &&
+      Op0.hasOneUse() &&
+      Op0.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
+      Op0.getOperand(0).getValueType() == MVT::v4f32 &&
+      Op0.getOperand(1).getOpcode() == ISD::Constant &&
+      cast<ConstantSDNode>(Op0.getOperand(1))->getZExtValue() == 0) {
+    SDValue Vec = Op0.getOperand(0);
+    for (auto *U : Vec->uses()) {
+      if (U != Op0.getNode() &&
+          U->hasOneUse() &&
+          U->getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
+          U->getOperand(0) == Vec &&
+          U->getOperand(1).getOpcode() == ISD::Constant &&
+          cast<ConstantSDNode>(U->getOperand(1))->getZExtValue() == 2) {
+        SDValue OtherExtend = SDValue(*U->use_begin(), 0);
+        if (OtherExtend.getOpcode() == ISD::FP_EXTEND &&
+            OtherExtend.getOperand(0) == SDValue(U, 0) &&
+            OtherExtend.getValueType() == MVT::f64) {
+          SDValue VExtend = DAG.getNode(SystemZISD::VEXTEND, SDLoc(N),
+                                        MVT::v2f64, Vec);
+          DCI.AddToWorklist(VExtend.getNode());
+          SDValue Extract1 =
+            DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(U), MVT::f64,
+                        VExtend, DAG.getConstant(1, SDLoc(U), MVT::i32));
+          DCI.AddToWorklist(Extract1.getNode());
+          DAG.ReplaceAllUsesOfValueWith(OtherExtend, Extract1);
+          SDValue Extract0 =
+            DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(Op0), MVT::f64,
+                        VExtend, DAG.getConstant(0, SDLoc(Op0), MVT::i32));
+          return Extract0;
+        }
+      }
+    }
+  }
+  return SDValue();
+}
+
 SDValue SystemZTargetLowering::combineBSWAP(
     SDNode *N, DAGCombinerInfo &DCI) const {
   SelectionDAG &DAG = DCI.DAG;
@@ -5573,55 +5643,96 @@ SDValue SystemZTargetLowering::combineBSWAP(
 static bool combineCCMask(SDValue &CCReg, int &CCValid, int &CCMask) {
   // We have a SELECT_CCMASK or BR_CCMASK comparing the condition code
   // set by the CCReg instruction using the CCValid / CCMask masks,
-  // If the CCReg instruction is itself a (ICMP (SELECT_CCMASK)) testing
-  // the condition code set by some other instruction, see whether we
-  // can directly use that condition code.
-  bool Invert = false;
+  // If the CCReg instruction is itself a ICMP testing the condition
+  // code set by some other instruction, see whether we can directly
+  // use that condition code.
 
-  // Verify that we have an appropriate mask for a EQ or NE comparison.
+  // Verify that we have an ICMP against some constant.
   if (CCValid != SystemZ::CCMASK_ICMP)
     return false;
-  if (CCMask == SystemZ::CCMASK_CMP_NE)
-    Invert = !Invert;
-  else if (CCMask != SystemZ::CCMASK_CMP_EQ)
-    return false;
-
-  // Verify that we have an ICMP that is the user of a SELECT_CCMASK.
-  SDNode *ICmp = CCReg.getNode();
+  auto *ICmp = CCReg.getNode();
   if (ICmp->getOpcode() != SystemZISD::ICMP)
     return false;
-  SDNode *Select = ICmp->getOperand(0).getNode();
-  if (Select->getOpcode() != SystemZISD::SELECT_CCMASK)
+  auto *CompareLHS = ICmp->getOperand(0).getNode();
+  auto *CompareRHS = dyn_cast<ConstantSDNode>(ICmp->getOperand(1));
+  if (!CompareRHS)
     return false;
 
-  // Verify that the ICMP compares against one of select values.
-  auto *CompareVal = dyn_cast<ConstantSDNode>(ICmp->getOperand(1));
-  if (!CompareVal)
-    return false;
-  auto *TrueVal = dyn_cast<ConstantSDNode>(Select->getOperand(0));
-  if (!TrueVal)
-    return false;
-  auto *FalseVal = dyn_cast<ConstantSDNode>(Select->getOperand(1));
-  if (!FalseVal)
-    return false;
-  if (CompareVal->getZExtValue() == FalseVal->getZExtValue())
-    Invert = !Invert;
-  else if (CompareVal->getZExtValue() != TrueVal->getZExtValue())
-    return false;
+  // Optimize the case where CompareLHS is a SELECT_CCMASK.
+  if (CompareLHS->getOpcode() == SystemZISD::SELECT_CCMASK) {
+    // Verify that we have an appropriate mask for a EQ or NE comparison.
+    bool Invert = false;
+    if (CCMask == SystemZ::CCMASK_CMP_NE)
+      Invert = !Invert;
+    else if (CCMask != SystemZ::CCMASK_CMP_EQ)
+      return false;
 
-  // Compute the effective CC mask for the new branch or select.
-  auto *NewCCValid = dyn_cast<ConstantSDNode>(Select->getOperand(2));
-  auto *NewCCMask = dyn_cast<ConstantSDNode>(Select->getOperand(3));
-  if (!NewCCValid || !NewCCMask)
-    return false;
-  CCValid = NewCCValid->getZExtValue();
-  CCMask = NewCCMask->getZExtValue();
-  if (Invert)
-    CCMask ^= CCValid;
+    // Verify that the ICMP compares against one of select values.
+    auto *TrueVal = dyn_cast<ConstantSDNode>(CompareLHS->getOperand(0));
+    if (!TrueVal)
+      return false;
+    auto *FalseVal = dyn_cast<ConstantSDNode>(CompareLHS->getOperand(1));
+    if (!FalseVal)
+      return false;
+    if (CompareRHS->getZExtValue() == FalseVal->getZExtValue())
+      Invert = !Invert;
+    else if (CompareRHS->getZExtValue() != TrueVal->getZExtValue())
+      return false;
 
-  // Return the updated CCReg link.
-  CCReg = Select->getOperand(4);
-  return true;
+    // Compute the effective CC mask for the new branch or select.
+    auto *NewCCValid = dyn_cast<ConstantSDNode>(CompareLHS->getOperand(2));
+    auto *NewCCMask = dyn_cast<ConstantSDNode>(CompareLHS->getOperand(3));
+    if (!NewCCValid || !NewCCMask)
+      return false;
+    CCValid = NewCCValid->getZExtValue();
+    CCMask = NewCCMask->getZExtValue();
+    if (Invert)
+      CCMask ^= CCValid;
+
+    // Return the updated CCReg link.
+    CCReg = CompareLHS->getOperand(4);
+    return true;
+  }
+
+  // Optimize the case where CompareRHS is (SRA (SHL (IPM))).
+  if (CompareLHS->getOpcode() == ISD::SRA) {
+    auto *SRACount = dyn_cast<ConstantSDNode>(CompareLHS->getOperand(1));
+    if (!SRACount || SRACount->getZExtValue() != 30)
+      return false;
+    auto *SHL = CompareLHS->getOperand(0).getNode();
+    if (SHL->getOpcode() != ISD::SHL)
+      return false;
+    auto *SHLCount = dyn_cast<ConstantSDNode>(SHL->getOperand(1));
+    if (!SHLCount || SHLCount->getZExtValue() != 30 - SystemZ::IPM_CC)
+      return false;
+    auto *IPM = SHL->getOperand(0).getNode();
+    if (IPM->getOpcode() != SystemZISD::IPM)
+      return false;
+
+    // Avoid introducing CC spills (because SRA would clobber CC).
+    if (!CompareLHS->hasOneUse())
+      return false;
+    // Verify that the ICMP compares against zero.
+    if (CompareRHS->getZExtValue() != 0)
+      return false;
+
+    // Compute the effective CC mask for the new branch or select.
+    switch (CCMask) {
+    case SystemZ::CCMASK_CMP_EQ: break;
+    case SystemZ::CCMASK_CMP_NE: break;
+    case SystemZ::CCMASK_CMP_LT: CCMask = SystemZ::CCMASK_CMP_GT; break;
+    case SystemZ::CCMASK_CMP_GT: CCMask = SystemZ::CCMASK_CMP_LT; break;
+    case SystemZ::CCMASK_CMP_LE: CCMask = SystemZ::CCMASK_CMP_GE; break;
+    case SystemZ::CCMASK_CMP_GE: CCMask = SystemZ::CCMASK_CMP_LE; break;
+    default: return false;
+    }
+
+    // Return the updated CCReg link.
+    CCReg = IPM->getOperand(0);
+    return true;
+  }
+
+  return false;
 }
 
 SDValue SystemZTargetLowering::combineBR_CCMASK(
@@ -5731,6 +5842,12 @@ SDValue SystemZTargetLowering::combineIntDIVREM(
   return SDValue();
 }
 
+SDValue SystemZTargetLowering::unwrapAddress(SDValue N) const {
+  if (N->getOpcode() == SystemZISD::PCREL_WRAPPER)
+    return N->getOperand(0);
+  return N;
+}
+
 SDValue SystemZTargetLowering::PerformDAGCombine(SDNode *N,
                                                  DAGCombinerInfo &DCI) const {
   switch(N->getOpcode()) {
@@ -5745,6 +5862,7 @@ SDValue SystemZTargetLowering::PerformDAGCombine(SDNode *N,
   case ISD::EXTRACT_VECTOR_ELT: return combineEXTRACT_VECTOR_ELT(N, DCI);
   case SystemZISD::JOIN_DWORDS: return combineJOIN_DWORDS(N, DCI);
   case ISD::FP_ROUND:           return combineFP_ROUND(N, DCI);
+  case ISD::FP_EXTEND:          return combineFP_EXTEND(N, DCI);
   case ISD::BSWAP:              return combineBSWAP(N, DCI);
   case SystemZISD::BR_CCMASK:   return combineBR_CCMASK(N, DCI);
   case SystemZISD::SELECT_CCMASK: return combineSELECT_CCMASK(N, DCI);
@@ -5863,10 +5981,10 @@ static void computeKnownBitsBinOp(const SDValue Op, KnownBits &Known,
                                   unsigned OpNo) {
   APInt Src0DemE = getDemandedSrcElements(Op, DemandedElts, OpNo);
   APInt Src1DemE = getDemandedSrcElements(Op, DemandedElts, OpNo + 1);
-  unsigned SrcBitWidth = Op.getOperand(OpNo).getScalarValueSizeInBits();
-  KnownBits LHSKnown(SrcBitWidth), RHSKnown(SrcBitWidth);
-  DAG.computeKnownBits(Op.getOperand(OpNo), LHSKnown, Src0DemE, Depth + 1);
-  DAG.computeKnownBits(Op.getOperand(OpNo + 1), RHSKnown, Src1DemE, Depth + 1);
+  KnownBits LHSKnown =
+      DAG.computeKnownBits(Op.getOperand(OpNo), Src0DemE, Depth + 1);
+  KnownBits RHSKnown =
+      DAG.computeKnownBits(Op.getOperand(OpNo + 1), Src1DemE, Depth + 1);
   Known.Zero = LHSKnown.Zero & RHSKnown.Zero;
   Known.One = LHSKnown.One & RHSKnown.One;
 }
@@ -5932,9 +6050,8 @@ SystemZTargetLowering::computeKnownBitsForTargetNode(const SDValue Op,
     case Intrinsic::s390_vuplf: {
       SDValue SrcOp = Op.getOperand(1);
       unsigned SrcBitWidth = SrcOp.getScalarValueSizeInBits();
-      Known = KnownBits(SrcBitWidth);
       APInt SrcDemE = getDemandedSrcElements(Op, DemandedElts, 0);
-      DAG.computeKnownBits(SrcOp, Known, SrcDemE, Depth + 1);
+      Known = DAG.computeKnownBits(SrcOp, SrcDemE, Depth + 1);
       if (IsLogical) {
         Known = Known.zext(BitWidth);
         Known.Zero.setBitsFrom(SrcBitWidth);
@@ -5953,7 +6070,7 @@ SystemZTargetLowering::computeKnownBitsForTargetNode(const SDValue Op,
       break;
     case SystemZISD::REPLICATE: {
       SDValue SrcOp = Op.getOperand(0);
-      DAG.computeKnownBits(SrcOp, Known, Depth + 1);
+      Known = DAG.computeKnownBits(SrcOp, Depth + 1);
       if (Known.getBitWidth() < BitWidth && isa<ConstantSDNode>(SrcOp))
         Known = Known.sext(BitWidth); // VREPI sign extends the immedate.
       break;
@@ -6168,7 +6285,8 @@ static void createPHIsForSelects(MachineBasicBlock::iterator MIItBegin,
   // destination registers, and the registers that went into the PHI.
   DenseMap<unsigned, std::pair<unsigned, unsigned>> RegRewriteTable;
 
-  for (MachineBasicBlock::iterator MIIt = MIItBegin; MIIt != MIItEnd; ++MIIt) {
+  for (MachineBasicBlock::iterator MIIt = MIItBegin; MIIt != MIItEnd;
+       MIIt = skipDebugInstructionsForward(++MIIt, MIItEnd)) {
     unsigned DestReg = MIIt->getOperand(0).getReg();
     unsigned TrueReg = MIIt->getOperand(1).getReg();
     unsigned FalseReg = MIIt->getOperand(2).getReg();
@@ -6192,6 +6310,8 @@ static void createPHIsForSelects(MachineBasicBlock::iterator MIItBegin,
     // Add this PHI to the rewrite table.
     RegRewriteTable[DestReg] = std::make_pair(TrueReg, FalseReg);
   }
+
+  MF->getProperties().reset(MachineFunctionProperties::Property::NoPHIs);
 }
 
 // Implement EmitInstrWithCustomInserter for pseudo Select* instruction MI.
@@ -6209,8 +6329,8 @@ SystemZTargetLowering::emitSelect(MachineInstr &MI,
   // same condition code value, we want to expand all of them into
   // a single pair of basic blocks using the same condition.
   MachineInstr *LastMI = &MI;
-  MachineBasicBlock::iterator NextMIIt =
-      std::next(MachineBasicBlock::iterator(MI));
+  MachineBasicBlock::iterator NextMIIt = skipDebugInstructionsForward(
+      std::next(MachineBasicBlock::iterator(MI)), MBB->end());
 
   if (isSelectPseudo(MI))
     while (NextMIIt != MBB->end() && isSelectPseudo(*NextMIIt) &&
@@ -6218,7 +6338,7 @@ SystemZTargetLowering::emitSelect(MachineInstr &MI,
            (NextMIIt->getOperand(4).getImm() == CCMask ||
             NextMIIt->getOperand(4).getImm() == (CCValid ^ CCMask))) {
       LastMI = &*NextMIIt;
-      ++NextMIIt;
+      NextMIIt = skipDebugInstructionsForward(++NextMIIt, MBB->end());
     }
 
   MachineBasicBlock *StartMBB = MBB;
@@ -6251,8 +6371,8 @@ SystemZTargetLowering::emitSelect(MachineInstr &MI,
   //  ...
   MBB = JoinMBB;
   MachineBasicBlock::iterator MIItBegin = MachineBasicBlock::iterator(MI);
-  MachineBasicBlock::iterator MIItEnd =
-      std::next(MachineBasicBlock::iterator(LastMI));
+  MachineBasicBlock::iterator MIItEnd = skipDebugInstructionsForward(
+      std::next(MachineBasicBlock::iterator(LastMI)), MBB->end());
   createPHIsForSelects(MIItBegin, MIItEnd, StartMBB, FalseMBB, MBB);
 
   StartMBB->erase(MIItBegin, MIItEnd);
diff --git a/lib/Target/SystemZ/SystemZISelLowering.h b/lib/Target/SystemZ/SystemZISelLowering.h
index 172dbeec26d0fab594205743f82d22144940f1c5..8efe2204581575cc2abc5d238ceed2b7ceba9c92 100644
--- a/lib/Target/SystemZ/SystemZISelLowering.h
+++ b/lib/Target/SystemZ/SystemZISelLowering.h
@@ -1,9 +1,8 @@
 //===-- SystemZISelLowering.h - SystemZ DAG lowering interface --*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -16,6 +15,7 @@
 #define LLVM_LIB_TARGET_SYSTEMZ_SYSTEMZISELLOWERING_H
 
 #include "SystemZ.h"
+#include "SystemZInstrInfo.h"
 #include "llvm/CodeGen/MachineBasicBlock.h"
 #include "llvm/CodeGen/SelectionDAG.h"
 #include "llvm/CodeGen/TargetLowering.h"
@@ -162,10 +162,6 @@ enum NodeType : unsigned {
   // Transaction end.  Just the chain operand.  Returns CC value and chain.
   TEND,
 
-  // Create a vector constant by filling byte N of the result with bit
-  // 15-N of the single operand.
-  BYTE_MASK,
-
   // Create a vector constant by replicating an element-sized RISBG-style mask.
   // The first operand specifies the starting set bit and the second operand
   // specifies the ending set bit.  Both operands count from the MSB of the
@@ -396,6 +392,7 @@ public:
       return TypeWidenVector;
     return TargetLoweringBase::getPreferredVectorAction(VT);
   }
+  bool isCheapToSpeculateCtlz() const override { return true; }
   EVT getSetCCResultType(const DataLayout &DL, LLVMContext &,
                          EVT) const override;
   bool isFMAFasterThanFMulAndFAdd(EVT VT) const override;
@@ -516,6 +513,9 @@ public:
     return true;
   }
 
+  static bool tryBuildVectorByteMask(BuildVectorSDNode *BVN, uint64_t &Mask);
+  static bool analyzeFPImm(const APFloat &Imm, unsigned BitWidth,
+                 unsigned &Start, unsigned &End, const SystemZInstrInfo *TII);
 private:
   const SystemZSubtarget &Subtarget;
 
@@ -592,12 +592,15 @@ private:
   SDValue combineEXTRACT_VECTOR_ELT(SDNode *N, DAGCombinerInfo &DCI) const;
   SDValue combineJOIN_DWORDS(SDNode *N, DAGCombinerInfo &DCI) const;
   SDValue combineFP_ROUND(SDNode *N, DAGCombinerInfo &DCI) const;
+  SDValue combineFP_EXTEND(SDNode *N, DAGCombinerInfo &DCI) const;
   SDValue combineBSWAP(SDNode *N, DAGCombinerInfo &DCI) const;
   SDValue combineBR_CCMASK(SDNode *N, DAGCombinerInfo &DCI) const;
   SDValue combineSELECT_CCMASK(SDNode *N, DAGCombinerInfo &DCI) const;
   SDValue combineGET_CCMASK(SDNode *N, DAGCombinerInfo &DCI) const;
   SDValue combineIntDIVREM(SDNode *N, DAGCombinerInfo &DCI) const;
 
+  SDValue unwrapAddress(SDValue N) const override;
+
   // If the last instruction before MBBI in MBB was some form of COMPARE,
   // try to replace it with a COMPARE AND BRANCH just before MBBI.
   // CCMask and Target are the BRC-like operands for the branch.
diff --git a/lib/Target/SystemZ/SystemZInstrBuilder.h b/lib/Target/SystemZ/SystemZInstrBuilder.h
index 896b665d25eb751069589ec3d9c39432693b4695..ec7639e71f81918204c34e902966da4ab2feebc3 100644
--- a/lib/Target/SystemZ/SystemZInstrBuilder.h
+++ b/lib/Target/SystemZ/SystemZInstrBuilder.h
@@ -1,9 +1,8 @@
 //===-- SystemZInstrBuilder.h - Functions to aid building insts -*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/SystemZ/SystemZInstrDFP.td b/lib/Target/SystemZ/SystemZInstrDFP.td
index 08ab2d7bbc523dc32d08d6f83c59298e2923ce82..15a499cd11bf8ad383ce75559b7169607340f521 100644
--- a/lib/Target/SystemZ/SystemZInstrDFP.td
+++ b/lib/Target/SystemZ/SystemZInstrDFP.td
@@ -1,9 +1,8 @@
 //==- SystemZInstrDFP.td - Floating-point SystemZ instructions -*- tblgen-*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/SystemZ/SystemZInstrFP.td b/lib/Target/SystemZ/SystemZInstrFP.td
index 1374ee91fa2977cc5752c301101574dd61c60838..72fa7f9a217efa80e18794ab5029a2be1fffcfd6 100644
--- a/lib/Target/SystemZ/SystemZInstrFP.td
+++ b/lib/Target/SystemZ/SystemZInstrFP.td
@@ -1,9 +1,8 @@
 //==- SystemZInstrFP.td - Floating-point SystemZ instructions --*- tblgen-*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/SystemZ/SystemZInstrFormats.td b/lib/Target/SystemZ/SystemZInstrFormats.td
index e3f9a9645d13c22bb386f589be7b0176d0c2716a..8848b259678e6dfbdaa8f9350f083bf2d6bb3338 100644
--- a/lib/Target/SystemZ/SystemZInstrFormats.td
+++ b/lib/Target/SystemZ/SystemZInstrFormats.td
@@ -1,9 +1,8 @@
 //==- SystemZInstrFormats.td - SystemZ Instruction Formats --*- tablegen -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -2900,7 +2899,7 @@ multiclass UnaryExtraVRRaSPair<string mnemonic, bits<16> opcode,
 }
 
 multiclass UnaryExtraVRRaSPairGeneric<string mnemonic, bits<16> opcode> {
-  let M4 = 0 in
+  let M4 = 0, Defs = [CC] in
     def "" : InstVRRa<opcode, (outs VR128:$V1),
                      (ins VR128:$V2, imm32zx4:$M3, imm32zx4:$M5),
                      mnemonic#"\t$V1, $V2, $M3, $M5", []>;
@@ -3472,7 +3471,9 @@ multiclass BinaryVRRbSPair<string mnemonic, bits<16> opcode,
 class BinaryVRRbSPairGeneric<string mnemonic, bits<16> opcode>
   : InstVRRb<opcode, (outs VR128:$V1),
              (ins VR128:$V2, VR128:$V3, imm32zx4:$M4, imm32zx4:$M5),
-             mnemonic#"\t$V1, $V2, $V3, $M4, $M5", []>;
+             mnemonic#"\t$V1, $V2, $V3, $M4, $M5", []> {
+  let Defs = [CC];
+}
 
 // Declare a pair of instructions, one which sets CC and one which doesn't.
 // The CC-setting form ends with "S" and sets the low bit of M5.
@@ -3496,9 +3497,10 @@ multiclass BinaryExtraVRRbSPair<string mnemonic, bits<16> opcode,
 }
 
 multiclass BinaryExtraVRRbSPairGeneric<string mnemonic, bits<16> opcode> {
-  def "" : InstVRRb<opcode, (outs VR128:$V1),
-                   (ins VR128:$V2, VR128:$V3, imm32zx4:$M4, imm32zx4:$M5),
-                   mnemonic#"\t$V1, $V2, $V3, $M4, $M5", []>;
+  let Defs = [CC] in
+    def "" : InstVRRb<opcode, (outs VR128:$V1),
+                     (ins VR128:$V2, VR128:$V3, imm32zx4:$M4, imm32zx4:$M5),
+                     mnemonic#"\t$V1, $V2, $V3, $M4, $M5", []>;
   def : InstAlias<mnemonic#"\t$V1, $V2, $V3, $M4",
                   (!cast<Instruction>(NAME) VR128:$V1, VR128:$V2, VR128:$V3,
                                             imm32zx4:$M4, 0)>;
@@ -4185,9 +4187,10 @@ multiclass TernaryOptVRRbSPair<string mnemonic, bits<16> opcode,
 }
 
 multiclass TernaryOptVRRbSPairGeneric<string mnemonic, bits<16> opcode> {
-  def "" : InstVRRb<opcode, (outs VR128:$V1),
-                   (ins VR128:$V2, VR128:$V3, imm32zx4:$M4, imm32zx4:$M5),
-                   mnemonic#"\t$V1, $V2, $V3, $M4, $M5", []>;
+  let Defs = [CC] in
+    def "" : InstVRRb<opcode, (outs VR128:$V1),
+                     (ins VR128:$V2, VR128:$V3, imm32zx4:$M4, imm32zx4:$M5),
+                     mnemonic#"\t$V1, $V2, $V3, $M4, $M5", []>;
   def : InstAlias<mnemonic#"\t$V1, $V2, $V3, $M4",
                   (!cast<Instruction>(NAME) VR128:$V1, VR128:$V2, VR128:$V3,
                                             imm32zx4:$M4, 0)>;
@@ -4385,7 +4388,8 @@ multiclass QuaternaryOptVRRdSPair<string mnemonic, bits<16> opcode,
 }
 
 multiclass QuaternaryOptVRRdSPairGeneric<string mnemonic, bits<16> opcode> {
-  def "" : QuaternaryVRRdGeneric<mnemonic, opcode>;
+  let Defs = [CC] in
+    def "" : QuaternaryVRRdGeneric<mnemonic, opcode>;
   def : InstAlias<mnemonic#"\t$V1, $V2, $V3, $V4, $M5",
                   (!cast<Instruction>(NAME) VR128:$V1, VR128:$V2, VR128:$V3,
                                             VR128:$V4, imm32zx4:$M5, 0)>;
diff --git a/lib/Target/SystemZ/SystemZInstrHFP.td b/lib/Target/SystemZ/SystemZInstrHFP.td
index 6d5b4b92f6508630c09e45b9adb4f9c5b6c006ef..2e3c9932d621426d53e32fb2a38aebd8affaf785 100644
--- a/lib/Target/SystemZ/SystemZInstrHFP.td
+++ b/lib/Target/SystemZ/SystemZInstrHFP.td
@@ -1,9 +1,8 @@
 //==- SystemZInstrHFP.td - Floating-point SystemZ instructions -*- tblgen-*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/SystemZ/SystemZInstrInfo.cpp b/lib/Target/SystemZ/SystemZInstrInfo.cpp
index b03b4edaa4abc82bc734599440e6989b593e4209..3a1747ca1e5cb3c8e3cdd232432e6934fcc664e3 100644
--- a/lib/Target/SystemZ/SystemZInstrInfo.cpp
+++ b/lib/Target/SystemZ/SystemZInstrInfo.cpp
@@ -1,9 +1,8 @@
 //===-- SystemZInstrInfo.cpp - SystemZ instruction information ------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -557,80 +556,6 @@ bool SystemZInstrInfo::analyzeCompare(const MachineInstr &MI, unsigned &SrcReg,
   return false;
 }
 
-// If Reg is a virtual register, return its definition, otherwise return null.
-static MachineInstr *getDef(unsigned Reg,
-                            const MachineRegisterInfo *MRI) {
-  if (TargetRegisterInfo::isPhysicalRegister(Reg))
-    return nullptr;
-  return MRI->getUniqueVRegDef(Reg);
-}
-
-// Return true if MI is a shift of type Opcode by Imm bits.
-static bool isShift(MachineInstr *MI, unsigned Opcode, int64_t Imm) {
-  return (MI->getOpcode() == Opcode &&
-          !MI->getOperand(2).getReg() &&
-          MI->getOperand(3).getImm() == Imm);
-}
-
-// If the destination of MI has no uses, delete it as dead.
-static void eraseIfDead(MachineInstr *MI, const MachineRegisterInfo *MRI) {
-  if (MRI->use_nodbg_empty(MI->getOperand(0).getReg()))
-    MI->eraseFromParent();
-}
-
-// Compare compares SrcReg against zero.  Check whether SrcReg contains
-// the result of an IPM sequence whose input CC survives until Compare,
-// and whether Compare is therefore redundant.  Delete it and return
-// true if so.
-static bool removeIPMBasedCompare(MachineInstr &Compare, unsigned SrcReg,
-                                  const MachineRegisterInfo *MRI,
-                                  const TargetRegisterInfo *TRI) {
-  MachineInstr *LGFR = nullptr;
-  MachineInstr *RLL = getDef(SrcReg, MRI);
-  if (RLL && RLL->getOpcode() == SystemZ::LGFR) {
-    LGFR = RLL;
-    RLL = getDef(LGFR->getOperand(1).getReg(), MRI);
-  }
-  if (!RLL || !isShift(RLL, SystemZ::RLL, 31))
-    return false;
-
-  MachineInstr *SRL = getDef(RLL->getOperand(1).getReg(), MRI);
-  if (!SRL || !isShift(SRL, SystemZ::SRL, SystemZ::IPM_CC))
-    return false;
-
-  MachineInstr *IPM = getDef(SRL->getOperand(1).getReg(), MRI);
-  if (!IPM || IPM->getOpcode() != SystemZ::IPM)
-    return false;
-
-  // Check that there are no assignments to CC between the IPM and Compare,
-  if (IPM->getParent() != Compare.getParent())
-    return false;
-  MachineBasicBlock::iterator MBBI = IPM, MBBE = Compare.getIterator();
-  for (++MBBI; MBBI != MBBE; ++MBBI) {
-    MachineInstr &MI = *MBBI;
-    if (MI.modifiesRegister(SystemZ::CC, TRI))
-      return false;
-  }
-
-  Compare.eraseFromParent();
-  if (LGFR)
-    eraseIfDead(LGFR, MRI);
-  eraseIfDead(RLL, MRI);
-  eraseIfDead(SRL, MRI);
-  eraseIfDead(IPM, MRI);
-
-  return true;
-}
-
-bool SystemZInstrInfo::optimizeCompareInstr(
-    MachineInstr &Compare, unsigned SrcReg, unsigned SrcReg2, int Mask,
-    int Value, const MachineRegisterInfo *MRI) const {
-  assert(!SrcReg2 && "Only optimizing constant comparisons so far");
-  bool IsLogical = (Compare.getDesc().TSFlags & SystemZII::IsLogical) != 0;
-  return Value == 0 && !IsLogical &&
-         removeIPMBasedCompare(Compare, SrcReg, MRI, &RI);
-}
-
 bool SystemZInstrInfo::canInsertSelect(const MachineBasicBlock &MBB,
                                        ArrayRef<MachineOperand> Pred,
                                        unsigned TrueReg, unsigned FalseReg,
@@ -1506,7 +1431,7 @@ bool SystemZInstrInfo::expandPostRAPseudo(MachineInstr &MI) const {
 }
 
 unsigned SystemZInstrInfo::getInstSizeInBytes(const MachineInstr &MI) const {
-  if (MI.getOpcode() == TargetOpcode::INLINEASM) {
+  if (MI.isInlineAsm()) {
     const MachineFunction *MF = MI.getParent()->getParent();
     const char *AsmStr = MI.getOperand(0).getSymbolName();
     return getInlineAsmLength(AsmStr, *MF->getTarget().getMCAsmInfo());
diff --git a/lib/Target/SystemZ/SystemZInstrInfo.h b/lib/Target/SystemZ/SystemZInstrInfo.h
index 216139eb7c7969855cdbfbfbb798bd99835e567a..c5d1917b3292db42af966d71314e453f977e5119 100644
--- a/lib/Target/SystemZ/SystemZInstrInfo.h
+++ b/lib/Target/SystemZ/SystemZInstrInfo.h
@@ -1,9 +1,8 @@
 //===-- SystemZInstrInfo.h - SystemZ instruction information ----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -208,9 +207,6 @@ public:
                         int *BytesAdded = nullptr) const override;
   bool analyzeCompare(const MachineInstr &MI, unsigned &SrcReg,
                       unsigned &SrcReg2, int &Mask, int &Value) const override;
-  bool optimizeCompareInstr(MachineInstr &CmpInstr, unsigned SrcReg,
-                            unsigned SrcReg2, int Mask, int Value,
-                            const MachineRegisterInfo *MRI) const override;
   bool canInsertSelect(const MachineBasicBlock&, ArrayRef<MachineOperand> Cond,
                        unsigned, unsigned, int&, int&, int&) const override;
   void insertSelect(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
diff --git a/lib/Target/SystemZ/SystemZInstrInfo.td b/lib/Target/SystemZ/SystemZInstrInfo.td
index 8d3b1011d0a7d67b670382f0a383faedfa3e1169..57c0696dee5589ad251f3002365d922506dcd6c9 100644
--- a/lib/Target/SystemZ/SystemZInstrInfo.td
+++ b/lib/Target/SystemZ/SystemZInstrInfo.td
@@ -1,9 +1,8 @@
 //===-- SystemZInstrInfo.td - General SystemZ instructions ----*- tblgen-*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/SystemZ/SystemZInstrSystem.td b/lib/Target/SystemZ/SystemZInstrSystem.td
index c351577fa5bd1187fcdef5afc0d28e3b7c9882db..ecce16c9cd7389ac3afcc3e7e3bc62d17543f6c2 100644
--- a/lib/Target/SystemZ/SystemZInstrSystem.td
+++ b/lib/Target/SystemZ/SystemZInstrSystem.td
@@ -1,9 +1,8 @@
 //==- SystemZInstrSystem.td - SystemZ system instructions -*- tblgen-*-----==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/SystemZ/SystemZInstrVector.td b/lib/Target/SystemZ/SystemZInstrVector.td
index 8523af7e57386519cf1f188e64091687d8801ae2..dd2a0d58cdfcf1028e52082da9cde348306e7b43 100644
--- a/lib/Target/SystemZ/SystemZInstrVector.td
+++ b/lib/Target/SystemZ/SystemZInstrVector.td
@@ -1,9 +1,8 @@
 //==- SystemZInstrVector.td - SystemZ Vector instructions ------*- tblgen-*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -61,7 +60,7 @@ let Predicates = [FeatureVector] in {
     // Generate byte mask.
     def VZERO : InherentVRIa<"vzero", 0xE744, 0>;
     def VONE  : InherentVRIa<"vone", 0xE744, 0xffff>;
-    def VGBM  : UnaryVRIa<"vgbm", 0xE744, z_byte_mask, v128b, imm32zx16>;
+    def VGBM  : UnaryVRIa<"vgbm", 0xE744, null_frag, v128b, imm32zx16>;
 
     // Generate mask.
     def VGM  : BinaryVRIbGeneric<"vgm", 0xE746>;
@@ -151,13 +150,13 @@ let Predicates = [FeatureVector] in {
   def VLLEZH : UnaryVRX<"vllezh", 0xE704, z_vllezi16, v128h, 2, 1>;
   def VLLEZF : UnaryVRX<"vllezf", 0xE704, z_vllezi32, v128f, 4, 2>;
   def VLLEZG : UnaryVRX<"vllezg", 0xE704, z_vllezi64, v128g, 8, 3>;
-  def : Pat<(v4f32 (z_vllezf32 bdxaddr12only:$addr)),
+  def : Pat<(z_vllezf32 bdxaddr12only:$addr),
             (VLLEZF bdxaddr12only:$addr)>;
-  def : Pat<(v2f64 (z_vllezf64 bdxaddr12only:$addr)),
+  def : Pat<(z_vllezf64 bdxaddr12only:$addr),
             (VLLEZG bdxaddr12only:$addr)>;
   let Predicates = [FeatureVectorEnhancements1] in {
     def VLLEZLF : UnaryVRX<"vllezlf", 0xE704, z_vllezli32, v128f, 4, 6>;
-    def : Pat<(v4f32 (z_vllezlf32 bdxaddr12only:$addr)),
+    def : Pat<(z_vllezlf32 bdxaddr12only:$addr),
               (VLLEZLF bdxaddr12only:$addr)>;
   }
 
diff --git a/lib/Target/SystemZ/SystemZLDCleanup.cpp b/lib/Target/SystemZ/SystemZLDCleanup.cpp
index f532e9e23b1f1267934ca8820e887cf098abdf88..06d893d043e9dc9f98d80fd171d9f6b50e977f35 100644
--- a/lib/Target/SystemZ/SystemZLDCleanup.cpp
+++ b/lib/Target/SystemZ/SystemZLDCleanup.cpp
@@ -1,9 +1,8 @@
 //===-- SystemZLDCleanup.cpp - Clean up local-dynamic TLS accesses --------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/SystemZ/SystemZLongBranch.cpp b/lib/Target/SystemZ/SystemZLongBranch.cpp
index 802962bd4db037e630b455a5dda0ab6810922d4d..95d7e22dec328d3ff0807271feec883aae5e6b61 100644
--- a/lib/Target/SystemZ/SystemZLongBranch.cpp
+++ b/lib/Target/SystemZ/SystemZLongBranch.cpp
@@ -1,9 +1,8 @@
 //===-- SystemZLongBranch.cpp - Branch lengthening for SystemZ ------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/SystemZ/SystemZMCInstLower.cpp b/lib/Target/SystemZ/SystemZMCInstLower.cpp
index 2655e4866b20d93ea511ee16abf7302d68ce6587..ef39f80a94ef2444231865df0b79548bd0cbac41 100644
--- a/lib/Target/SystemZ/SystemZMCInstLower.cpp
+++ b/lib/Target/SystemZ/SystemZMCInstLower.cpp
@@ -1,9 +1,8 @@
 //===-- SystemZMCInstLower.cpp - Lower MachineInstr to MCInst -------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/SystemZ/SystemZMCInstLower.h b/lib/Target/SystemZ/SystemZMCInstLower.h
index 7173cfa42959ba0bd381ebf6f4a5a14af388fce0..14ad06488312b24c4edd4b1b923914db6871d58e 100644
--- a/lib/Target/SystemZ/SystemZMCInstLower.h
+++ b/lib/Target/SystemZ/SystemZMCInstLower.h
@@ -1,9 +1,8 @@
 //===-- SystemZMCInstLower.h - Lower MachineInstr to MCInst ----*- C++ -*--===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/SystemZ/SystemZMachineFunctionInfo.cpp b/lib/Target/SystemZ/SystemZMachineFunctionInfo.cpp
index 1a7c0d7f687a7cc349b7a763c07d1b9effa94401..9b6aa3593ce0181805d00f3012cc17577fa84ca5 100644
--- a/lib/Target/SystemZ/SystemZMachineFunctionInfo.cpp
+++ b/lib/Target/SystemZ/SystemZMachineFunctionInfo.cpp
@@ -1,9 +1,8 @@
 //=== SystemZMachineFunctionInfo.cpp - SystemZ machine function info ------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/SystemZ/SystemZMachineFunctionInfo.h b/lib/Target/SystemZ/SystemZMachineFunctionInfo.h
index 4f64f4c65f1d0d8dfea006c5daed183a8504490f..9eec3f37bc28cf13591d06e6eba22450d067d71b 100644
--- a/lib/Target/SystemZ/SystemZMachineFunctionInfo.h
+++ b/lib/Target/SystemZ/SystemZMachineFunctionInfo.h
@@ -1,9 +1,8 @@
 //=== SystemZMachineFunctionInfo.h - SystemZ machine function info -*- C++ -*-//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/SystemZ/SystemZMachineScheduler.cpp b/lib/Target/SystemZ/SystemZMachineScheduler.cpp
index 98e761ef87feaec994817ffb4db08ffdebdd738c..0becfaa1d49c74f7fafa8af7169becc51138ff4a 100644
--- a/lib/Target/SystemZ/SystemZMachineScheduler.cpp
+++ b/lib/Target/SystemZ/SystemZMachineScheduler.cpp
@@ -1,9 +1,8 @@
 //-- SystemZMachineScheduler.cpp - SystemZ Scheduler Interface -*- C++ -*---==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/SystemZ/SystemZMachineScheduler.h b/lib/Target/SystemZ/SystemZMachineScheduler.h
index ab820e5d3e63d6b70f43034c5d9141382be79e7b..0d5cc2e03e8db9db2dd4c0f9c972b6e0c8271dd2 100644
--- a/lib/Target/SystemZ/SystemZMachineScheduler.h
+++ b/lib/Target/SystemZ/SystemZMachineScheduler.h
@@ -1,9 +1,8 @@
 //==- SystemZMachineScheduler.h - SystemZ Scheduler Interface ----*- C++ -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/SystemZ/SystemZOperands.td b/lib/Target/SystemZ/SystemZOperands.td
index 7bf32bf19a4ac412b3f212e9d453d94d245b568b..4d0c5d02ffa27275cb969b609a522b88aabd6755 100644
--- a/lib/Target/SystemZ/SystemZOperands.td
+++ b/lib/Target/SystemZ/SystemZOperands.td
@@ -1,9 +1,8 @@
 //===-- SystemZOperands.td - SystemZ instruction operands ----*- tblgen-*--===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/SystemZ/SystemZOperators.td b/lib/Target/SystemZ/SystemZOperators.td
index c55a6273f5e447036540ca72c034fdc60cc0e659..9914db8651c0714bfa1d5a3c35280fab693fefc9 100644
--- a/lib/Target/SystemZ/SystemZOperators.td
+++ b/lib/Target/SystemZ/SystemZOperators.td
@@ -1,9 +1,8 @@
 //===-- SystemZOperators.td - SystemZ-specific operators ------*- tblgen-*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -287,7 +286,6 @@ def z_vector_insert     : SDNode<"ISD::INSERT_VECTOR_ELT",
                                  SDT_ZInsertVectorElt>;
 def z_vector_extract    : SDNode<"ISD::EXTRACT_VECTOR_ELT",
                                  SDT_ZExtractVectorElt>;
-def z_byte_mask         : SDNode<"SystemZISD::BYTE_MASK", SDT_ZReplicate>;
 def z_rotate_mask       : SDNode<"SystemZISD::ROTATE_MASK", SDT_ZRotateMask>;
 def z_replicate         : SDNode<"SystemZISD::REPLICATE", SDT_ZReplicate>;
 def z_join_dwords       : SDNode<"SystemZISD::JOIN_DWORDS", SDT_ZJoinDwords>;
@@ -709,10 +707,6 @@ class shiftop<SDPatternOperator operator>
              [(operator node:$val, node:$count),
               (operator node:$val, (and node:$count, imm32bottom6set))]>;
 
-// Vector representation of all-zeros and all-ones.
-def z_vzero : PatFrag<(ops), (bitconvert (v16i8 (z_byte_mask (i32 0))))>;
-def z_vones : PatFrag<(ops), (bitconvert (v16i8 (z_byte_mask (i32 65535))))>;
-
 // Load a scalar and replicate it in all elements of a vector.
 class z_replicate_load<ValueType scalartype, SDPatternOperator load>
   : PatFrag<(ops node:$addr),
@@ -740,42 +734,44 @@ def z_vlef64 : z_vle<f64, load>;
 // zeroed vector.
 class z_vllez<ValueType scalartype, SDPatternOperator load, int index>
   : PatFrag<(ops node:$addr),
-            (z_vector_insert (z_vzero),
+            (z_vector_insert (immAllZerosV),
                              (scalartype (load node:$addr)), (i32 index))>;
 def z_vllezi8  : z_vllez<i32, anyextloadi8, 7>;
 def z_vllezi16 : z_vllez<i32, anyextloadi16, 3>;
 def z_vllezi32 : z_vllez<i32, load, 1>;
-def z_vllezi64 : PatFrag<(ops node:$addr),
-                         (z_join_dwords (i64 (load node:$addr)), (i64 0))>;
+def z_vllezi64 : PatFrags<(ops node:$addr),
+                          [(z_vector_insert (immAllZerosV),
+                                            (i64 (load node:$addr)), (i32 0)),
+                           (z_join_dwords (i64 (load node:$addr)), (i64 0))]>;
 // We use high merges to form a v4f32 from four f32s.  Propagating zero
 // into all elements but index 1 gives this expression.
 def z_vllezf32 : PatFrag<(ops node:$addr),
-                         (bitconvert
-                          (z_merge_high
-                           (v2i64
-                            (z_unpackl_high
-                             (v4i32
-                              (bitconvert
-                               (v4f32 (scalar_to_vector
-                                       (f32 (load node:$addr)))))))),
-                           (v2i64 (z_vzero))))>;
+                         (z_merge_high
+                          (v2i64
+                           (z_unpackl_high
+                            (v4i32
+                             (bitconvert
+                              (v4f32 (scalar_to_vector
+                                      (f32 (load node:$addr)))))))),
+                          (v2i64
+                           (bitconvert (v4f32 (immAllZerosV)))))>;
 def z_vllezf64 : PatFrag<(ops node:$addr),
                          (z_merge_high
-                          (scalar_to_vector (f64 (load node:$addr))),
-                          (z_vzero))>;
+                          (v2f64 (scalar_to_vector (f64 (load node:$addr)))),
+                          (immAllZerosV))>;
 
 // Similarly for the high element of a zeroed vector.
 def z_vllezli32 : z_vllez<i32, load, 0>;
 def z_vllezlf32 : PatFrag<(ops node:$addr),
-                          (bitconvert
-                           (z_merge_high
-                            (v2i64
-                             (bitconvert
-                              (z_merge_high
-                               (v4f32 (scalar_to_vector
-                                       (f32 (load node:$addr)))),
-                               (v4f32 (z_vzero))))),
-                            (v2i64 (z_vzero))))>;
+                          (z_merge_high
+                           (v2i64
+                            (bitconvert
+                             (z_merge_high
+                              (v4f32 (scalar_to_vector
+                                      (f32 (load node:$addr)))),
+                              (v4f32 (immAllZerosV))))),
+                           (v2i64
+                            (bitconvert (v4f32 (immAllZerosV)))))>;
 
 // Store one element of a vector.
 class z_vste<ValueType scalartype, SDPatternOperator store>
@@ -790,16 +786,16 @@ def z_vstef32 : z_vste<f32, store>;
 def z_vstef64 : z_vste<f64, store>;
 
 // Arithmetic negation on vectors.
-def z_vneg : PatFrag<(ops node:$x), (sub (z_vzero), node:$x)>;
+def z_vneg : PatFrag<(ops node:$x), (sub (immAllZerosV), node:$x)>;
 
 // Bitwise negation on vectors.
-def z_vnot : PatFrag<(ops node:$x), (xor node:$x, (z_vones))>;
+def z_vnot : PatFrag<(ops node:$x), (xor node:$x, (immAllOnesV))>;
 
 // Signed "integer greater than zero" on vectors.
-def z_vicmph_zero : PatFrag<(ops node:$x), (z_vicmph node:$x, (z_vzero))>;
+def z_vicmph_zero : PatFrag<(ops node:$x), (z_vicmph node:$x, (immAllZerosV))>;
 
 // Signed "integer less than zero" on vectors.
-def z_vicmpl_zero : PatFrag<(ops node:$x), (z_vicmph (z_vzero), node:$x)>;
+def z_vicmpl_zero : PatFrag<(ops node:$x), (z_vicmph (immAllZerosV), node:$x)>;
 
 // Integer absolute on vectors.
 class z_viabs<int shift>
diff --git a/lib/Target/SystemZ/SystemZPatterns.td b/lib/Target/SystemZ/SystemZPatterns.td
index 152521fb66a8df7b524b5cdf5077377a403d8929..beaf4de285a3ac8c4e61ddeb0089b79f4920d4a6 100644
--- a/lib/Target/SystemZ/SystemZPatterns.td
+++ b/lib/Target/SystemZ/SystemZPatterns.td
@@ -1,9 +1,8 @@
 //===-- SystemZPatterns.td - SystemZ-specific pattern rules ---*- tblgen-*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/SystemZ/SystemZProcessors.td b/lib/Target/SystemZ/SystemZProcessors.td
index 0dca4582dc0d6d7d9ad4b3638b7e25a591652b4d..3dd0ea52b7122d82ac25c68773e7de7d922ab834 100644
--- a/lib/Target/SystemZ/SystemZProcessors.td
+++ b/lib/Target/SystemZ/SystemZProcessors.td
@@ -1,9 +1,8 @@
 //===-- SystemZ.td - SystemZ processors and features ---------*- tblgen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/SystemZ/SystemZRegisterInfo.cpp b/lib/Target/SystemZ/SystemZRegisterInfo.cpp
index e9f9188048da8a288ab7362ede6814838b8373e2..0e9e7e33420b1783e6371624cb90792da72000a8 100644
--- a/lib/Target/SystemZ/SystemZRegisterInfo.cpp
+++ b/lib/Target/SystemZ/SystemZRegisterInfo.cpp
@@ -1,9 +1,8 @@
 //===-- SystemZRegisterInfo.cpp - SystemZ register information ------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/SystemZ/SystemZRegisterInfo.h b/lib/Target/SystemZ/SystemZRegisterInfo.h
index 9fd2e4ae4f00a76e174ef02e8679b32140c221fd..61b8e013f319f620ec97b5c072d22e692bd58d46 100644
--- a/lib/Target/SystemZ/SystemZRegisterInfo.h
+++ b/lib/Target/SystemZ/SystemZRegisterInfo.h
@@ -1,9 +1,8 @@
 //===-- SystemZRegisterInfo.h - SystemZ register information ----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/SystemZ/SystemZRegisterInfo.td b/lib/Target/SystemZ/SystemZRegisterInfo.td
index cea88c088b86f40e9d30bf891179d6468d60129c..ed3f965a2558a4372b49424e2d577139723b31c1 100644
--- a/lib/Target/SystemZ/SystemZRegisterInfo.td
+++ b/lib/Target/SystemZ/SystemZRegisterInfo.td
@@ -1,9 +1,8 @@
 //==- SystemZRegisterInfo.td - SystemZ register definitions -*- tablegen -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/SystemZ/SystemZSchedule.td b/lib/Target/SystemZ/SystemZSchedule.td
index 83bf97e6841a3835744b67189869b0ca4ca18cae..c56bb17e8f113e9ca664b4bd7cf8473eed8e1001 100644
--- a/lib/Target/SystemZ/SystemZSchedule.td
+++ b/lib/Target/SystemZ/SystemZSchedule.td
@@ -1,9 +1,8 @@
 //==-- SystemZSchedule.td - SystemZ Scheduling Definitions ----*- tblgen -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/SystemZ/SystemZScheduleZ13.td b/lib/Target/SystemZ/SystemZScheduleZ13.td
index 74e1dad87908bc82fdf5e13b023745f7d652bc9a..fb6a1578dd122b20da5c4ebf326d85357cf500ee 100644
--- a/lib/Target/SystemZ/SystemZScheduleZ13.td
+++ b/lib/Target/SystemZ/SystemZScheduleZ13.td
@@ -1,9 +1,8 @@
 //-- SystemZScheduleZ13.td - SystemZ Scheduling Definitions ----*- tblgen -*-=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/SystemZ/SystemZScheduleZ14.td b/lib/Target/SystemZ/SystemZScheduleZ14.td
index 1962fdf3a1d1fabe3cfea69ef6f16439e3b6a8a7..33305c750e5ec443f656f1aee48d1b92b9104c32 100644
--- a/lib/Target/SystemZ/SystemZScheduleZ14.td
+++ b/lib/Target/SystemZ/SystemZScheduleZ14.td
@@ -1,9 +1,8 @@
 //-- SystemZScheduleZ14.td - SystemZ Scheduling Definitions ----*- tblgen -*-=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/SystemZ/SystemZScheduleZ196.td b/lib/Target/SystemZ/SystemZScheduleZ196.td
index 7535739f813acb9b99a601fa7647ec8aec21d4a2..ca714ef1a702b6f0a4846c3ba6e6a4f616afa4ca 100644
--- a/lib/Target/SystemZ/SystemZScheduleZ196.td
+++ b/lib/Target/SystemZ/SystemZScheduleZ196.td
@@ -1,9 +1,8 @@
 //=- SystemZScheduleZ196.td - SystemZ Scheduling Definitions ---*- tblgen -*-=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/SystemZ/SystemZScheduleZEC12.td b/lib/Target/SystemZ/SystemZScheduleZEC12.td
index a21d2c4cef70f7e591a508dc884fed3b4e2e9bc5..fb226be678dad8db85057c713380ee1c3ec9fc53 100644
--- a/lib/Target/SystemZ/SystemZScheduleZEC12.td
+++ b/lib/Target/SystemZ/SystemZScheduleZEC12.td
@@ -1,9 +1,8 @@
 //=- SystemZScheduleZEC12.td - SystemZ Scheduling Definitions --*- tblgen -*-=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/SystemZ/SystemZSelectionDAGInfo.cpp b/lib/Target/SystemZ/SystemZSelectionDAGInfo.cpp
index e0d7bca9a94b9e0f4adb008868e086b365f4849d..a50e6aa5971187c6129ca9dd0ac697ec7c5f960f 100644
--- a/lib/Target/SystemZ/SystemZSelectionDAGInfo.cpp
+++ b/lib/Target/SystemZ/SystemZSelectionDAGInfo.cpp
@@ -1,9 +1,8 @@
 //===-- SystemZSelectionDAGInfo.cpp - SystemZ SelectionDAG Info -----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -164,17 +163,17 @@ static SDValue emitCLC(SelectionDAG &DAG, const SDLoc &DL, SDValue Chain,
 }
 
 // Convert the current CC value into an integer that is 0 if CC == 0,
-// less than zero if CC == 1 and greater than zero if CC >= 2.
+// greater than zero if CC == 1 and less than zero if CC >= 2.
 // The sequence starts with IPM, which puts CC into bits 29 and 28
 // of an integer and clears bits 30 and 31.
 static SDValue addIPMSequence(const SDLoc &DL, SDValue CCReg,
                               SelectionDAG &DAG) {
   SDValue IPM = DAG.getNode(SystemZISD::IPM, DL, MVT::i32, CCReg);
-  SDValue SRL = DAG.getNode(ISD::SRL, DL, MVT::i32, IPM,
-                            DAG.getConstant(SystemZ::IPM_CC, DL, MVT::i32));
-  SDValue ROTL = DAG.getNode(ISD::ROTL, DL, MVT::i32, SRL,
-                             DAG.getConstant(31, DL, MVT::i32));
-  return ROTL;
+  SDValue SHL = DAG.getNode(ISD::SHL, DL, MVT::i32, IPM,
+                            DAG.getConstant(30 - SystemZ::IPM_CC, DL, MVT::i32));
+  SDValue SRA = DAG.getNode(ISD::SRA, DL, MVT::i32, SHL,
+                            DAG.getConstant(30, DL, MVT::i32));
+  return SRA;
 }
 
 std::pair<SDValue, SDValue> SystemZSelectionDAGInfo::EmitTargetCodeForMemcmp(
@@ -184,7 +183,8 @@ std::pair<SDValue, SDValue> SystemZSelectionDAGInfo::EmitTargetCodeForMemcmp(
   if (auto *CSize = dyn_cast<ConstantSDNode>(Size)) {
     uint64_t Bytes = CSize->getZExtValue();
     assert(Bytes > 0 && "Caller should have handled 0-size case");
-    SDValue CCReg = emitCLC(DAG, DL, Chain, Src1, Src2, Bytes);
+    // Swap operands to invert CC == 1 vs. CC == 2 cases.
+    SDValue CCReg = emitCLC(DAG, DL, Chain, Src2, Src1, Bytes);
     Chain = CCReg.getValue(1);
     return std::make_pair(addIPMSequence(DL, CCReg, DAG), Chain);
   }
@@ -232,7 +232,8 @@ std::pair<SDValue, SDValue> SystemZSelectionDAGInfo::EmitTargetCodeForStrcmp(
     SDValue Src2, MachinePointerInfo Op1PtrInfo,
     MachinePointerInfo Op2PtrInfo) const {
   SDVTList VTs = DAG.getVTList(Src1.getValueType(), MVT::i32, MVT::Other);
-  SDValue Unused = DAG.getNode(SystemZISD::STRCMP, DL, VTs, Chain, Src1, Src2,
+  // Swap operands to invert CC == 1 vs. CC == 2 cases.
+  SDValue Unused = DAG.getNode(SystemZISD::STRCMP, DL, VTs, Chain, Src2, Src1,
                                DAG.getConstant(0, DL, MVT::i32));
   SDValue CCReg = Unused.getValue(1);
   Chain = Unused.getValue(2);
diff --git a/lib/Target/SystemZ/SystemZSelectionDAGInfo.h b/lib/Target/SystemZ/SystemZSelectionDAGInfo.h
index 93cd970c30c600ed07366e602a169ce8b87e402f..7d63bae83cf33ad23bb7503d3d332d54be9b7f3b 100644
--- a/lib/Target/SystemZ/SystemZSelectionDAGInfo.h
+++ b/lib/Target/SystemZ/SystemZSelectionDAGInfo.h
@@ -1,9 +1,8 @@
 //===-- SystemZSelectionDAGInfo.h - SystemZ SelectionDAG Info ---*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/SystemZ/SystemZShortenInst.cpp b/lib/Target/SystemZ/SystemZShortenInst.cpp
index 195fa20a2c9038076a8f50c6f5c6420f2e728797..7a80a615619843b70ddaaabcd5979b0aea48e8db 100644
--- a/lib/Target/SystemZ/SystemZShortenInst.cpp
+++ b/lib/Target/SystemZ/SystemZShortenInst.cpp
@@ -1,9 +1,8 @@
 //===-- SystemZShortenInst.cpp - Instruction-shortening pass --------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/SystemZ/SystemZSubtarget.cpp b/lib/Target/SystemZ/SystemZSubtarget.cpp
index fb030a207bc755db4eab5a3b45cb65859dfa3388..127d9fb99cd96cab3912e620c393c7245c56888e 100644
--- a/lib/Target/SystemZ/SystemZSubtarget.cpp
+++ b/lib/Target/SystemZ/SystemZSubtarget.cpp
@@ -1,9 +1,8 @@
 //===-- SystemZSubtarget.cpp - SystemZ subtarget information --------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/SystemZ/SystemZSubtarget.h b/lib/Target/SystemZ/SystemZSubtarget.h
index cb6b21a1d465cbfaf439c70b320825d0ac15271c..cb9425aea87da564822f665c9236a309db45d0c5 100644
--- a/lib/Target/SystemZ/SystemZSubtarget.h
+++ b/lib/Target/SystemZ/SystemZSubtarget.h
@@ -1,9 +1,8 @@
 //===-- SystemZSubtarget.h - SystemZ subtarget information -----*- C++ -*--===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/SystemZ/SystemZTDC.cpp b/lib/Target/SystemZ/SystemZTDC.cpp
index 5dbd23d420a309c39f5903f38e15467f608e8882..478848c30701f7a33b6b671fb19106568e63a572 100644
--- a/lib/Target/SystemZ/SystemZTDC.cpp
+++ b/lib/Target/SystemZ/SystemZTDC.cpp
@@ -1,9 +1,8 @@
 //===-- SystemZTDC.cpp - Utilize Test Data Class instruction --------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -356,8 +355,8 @@ bool SystemZTDCPass::runOnFunction(Function &F) {
       if (!Worthy)
         continue;
       // Call the intrinsic, compare result with 0.
-      Value *TDCFunc = Intrinsic::getDeclaration(&M, Intrinsic::s390_tdc,
-                                                 V->getType());
+      Function *TDCFunc =
+          Intrinsic::getDeclaration(&M, Intrinsic::s390_tdc, V->getType());
       IRBuilder<> IRB(I);
       Value *MaskVal = ConstantInt::get(Type::getInt64Ty(Ctx), Mask);
       Instruction *TDC = IRB.CreateCall(TDCFunc, {V, MaskVal});
diff --git a/lib/Target/SystemZ/SystemZTargetMachine.cpp b/lib/Target/SystemZ/SystemZTargetMachine.cpp
index 9596a2b6388d62a756ee918ccc1657ff539ee430..10ca9b649ecc2d50d0e2f3a4d27a34c4de58b833 100644
--- a/lib/Target/SystemZ/SystemZTargetMachine.cpp
+++ b/lib/Target/SystemZ/SystemZTargetMachine.cpp
@@ -1,9 +1,8 @@
 //===-- SystemZTargetMachine.cpp - Define TargetMachine for SystemZ -------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/SystemZ/SystemZTargetMachine.h b/lib/Target/SystemZ/SystemZTargetMachine.h
index 52bf8bba55deec07fdd5057cf6554bd1b017e1a2..ac04a080f5808bb7142668d0ca3edb57b7507759 100644
--- a/lib/Target/SystemZ/SystemZTargetMachine.h
+++ b/lib/Target/SystemZ/SystemZTargetMachine.h
@@ -1,9 +1,8 @@
 //=- SystemZTargetMachine.h - Define TargetMachine for SystemZ ----*- C++ -*-=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp b/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp
index 129610fe095b03a45f41f1c5f8d675241d8ab59c..304ee79d2b2eae0a2949e54f720d1bb1e4cdb186 100644
--- a/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp
+++ b/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp
@@ -1,9 +1,8 @@
 //===-- SystemZTargetTransformInfo.cpp - SystemZ-specific TTI -------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/SystemZ/SystemZTargetTransformInfo.h b/lib/Target/SystemZ/SystemZTargetTransformInfo.h
index e79bee1ea3a806dff55092e1a48650d5573278cf..16ce2ef1d7a00589fbc610b41bd2974700e39789 100644
--- a/lib/Target/SystemZ/SystemZTargetTransformInfo.h
+++ b/lib/Target/SystemZ/SystemZTargetTransformInfo.h
@@ -1,9 +1,8 @@
 //===-- SystemZTargetTransformInfo.h - SystemZ-specific TTI ---------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/SystemZ/TargetInfo/LLVMBuild.txt b/lib/Target/SystemZ/TargetInfo/LLVMBuild.txt
index a5547e68a4ae5d702862f174625b74798513e6ec..f4bed28b4fad4585aaf93db74db4067ab7c9b89b 100644
--- a/lib/Target/SystemZ/TargetInfo/LLVMBuild.txt
+++ b/lib/Target/SystemZ/TargetInfo/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/Target/SystemZ/TargetInfo/LLVMBuild.txt ------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Target/SystemZ/TargetInfo/SystemZTargetInfo.cpp b/lib/Target/SystemZ/TargetInfo/SystemZTargetInfo.cpp
index e2b9efd35d3e5ae142f75e530467d788aabd98f0..07093dcf9f30148f9e0400a98f3d077cc695d3f4 100644
--- a/lib/Target/SystemZ/TargetInfo/SystemZTargetInfo.cpp
+++ b/lib/Target/SystemZ/TargetInfo/SystemZTargetInfo.cpp
@@ -1,9 +1,8 @@
 //===-- SystemZTargetInfo.cpp - SystemZ target implementation -------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/Target.cpp b/lib/Target/Target.cpp
index f23ea72eb5131771979ced9751547ce4e3fdb2b9..8a46c77492c5894976c7e311c6d2058de209cd6e 100644
--- a/lib/Target/Target.cpp
+++ b/lib/Target/Target.cpp
@@ -1,9 +1,8 @@
 //===-- Target.cpp --------------------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/TargetIntrinsicInfo.cpp b/lib/Target/TargetIntrinsicInfo.cpp
index e8b71924e0d955f983ad0be7bbaac9a2f7c3f266..256514c8c22d6ec9df23127be6d70eba24b766e7 100644
--- a/lib/Target/TargetIntrinsicInfo.cpp
+++ b/lib/Target/TargetIntrinsicInfo.cpp
@@ -1,9 +1,8 @@
 //===-- TargetIntrinsicInfo.cpp - Target Instruction Information ----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/TargetLoweringObjectFile.cpp b/lib/Target/TargetLoweringObjectFile.cpp
index bb937923b47ead28e467ed013529688233ec3ab6..ee32d01572461edcf32587845e90a43dae9cfd58 100644
--- a/lib/Target/TargetLoweringObjectFile.cpp
+++ b/lib/Target/TargetLoweringObjectFile.cpp
@@ -1,9 +1,8 @@
 //===-- llvm/Target/TargetLoweringObjectFile.cpp - Object File Info -------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/TargetMachine.cpp b/lib/Target/TargetMachine.cpp
index 39d5705b2a53543823c84848b6eded48b77bd123..c08ed43fe6b063be176cb817df9b891a1d7bc94a 100644
--- a/lib/Target/TargetMachine.cpp
+++ b/lib/Target/TargetMachine.cpp
@@ -1,9 +1,8 @@
 //===-- TargetMachine.cpp - General Target Information ---------------------==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/TargetMachineC.cpp b/lib/Target/TargetMachineC.cpp
index 37d398d580f8d5067380509eee9446585911fa9d..5d9029682fdd9c0006adf0d9f87202b1fdd85231 100644
--- a/lib/Target/TargetMachineC.cpp
+++ b/lib/Target/TargetMachineC.cpp
@@ -1,9 +1,8 @@
 //===-- TargetMachine.cpp -------------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -115,6 +114,15 @@ LLVMTargetMachineRef LLVMCreateTargetMachine(LLVMTargetRef T,
     case LLVMRelocDynamicNoPic:
       RM = Reloc::DynamicNoPIC;
       break;
+    case LLVMRelocROPI:
+      RM = Reloc::ROPI;
+      break;
+    case LLVMRelocRWPI:
+      RM = Reloc::RWPI;
+      break;
+    case LLVMRelocROPI_RWPI:
+      RM = Reloc::ROPI_RWPI;
+      break;
     default:
       break;
   }
diff --git a/lib/Target/WebAssembly/AsmParser/LLVMBuild.txt b/lib/Target/WebAssembly/AsmParser/LLVMBuild.txt
index 4c0652617bb847d9ca2e6073eecce9e880e07c09..5521f8237c34ed1cbc6018e2465cfeb76ce5b781 100644
--- a/lib/Target/WebAssembly/AsmParser/LLVMBuild.txt
+++ b/lib/Target/WebAssembly/AsmParser/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===-- ./lib/Target/WebAssembly/Disassembler/LLVMBuild.txt -----*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmParser.cpp b/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmParser.cpp
index f463aeaf8275084008533ea9618a2b91c388ad14..d596a5168387817e2f2d7e9debb285db1ba67655 100644
--- a/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmParser.cpp
+++ b/lib/Target/WebAssembly/AsmParser/WebAssemblyAsmParser.cpp
@@ -1,9 +1,8 @@
 //==- WebAssemblyAsmParser.cpp - Assembler for WebAssembly -*- C++ -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
@@ -18,10 +17,12 @@
 #include "MCTargetDesc/WebAssemblyTargetStreamer.h"
 #include "WebAssembly.h"
 #include "llvm/MC/MCContext.h"
+#include "llvm/MC/MCExpr.h"
 #include "llvm/MC/MCInst.h"
 #include "llvm/MC/MCInstrInfo.h"
 #include "llvm/MC/MCParser/MCParsedAsmOperand.h"
 #include "llvm/MC/MCParser/MCTargetAsmParser.h"
+#include "llvm/MC/MCSectionWasm.h"
 #include "llvm/MC/MCStreamer.h"
 #include "llvm/MC/MCSubtargetInfo.h"
 #include "llvm/MC/MCSymbol.h"
@@ -170,11 +171,25 @@ class WebAssemblyAsmParser final : public MCTargetAsmParser {
     FunctionStart,
     FunctionLocals,
     Instructions,
+    EndFunction,
   } CurrentState = FileStart;
 
+  // For ensuring blocks are properly nested.
+  enum NestingType {
+    Function,
+    Block,
+    Loop,
+    Try,
+    If,
+    Else,
+    Undefined,
+  };
+  std::vector<NestingType> NestingStack;
+
   // We track this to see if a .functype following a label is the same,
   // as this is how we recognize the start of a function.
   MCSymbol *LastLabel = nullptr;
+  MCSymbol *LastFunctionLabel = nullptr;
 
 public:
   WebAssemblyAsmParser(const MCSubtargetInfo &STI, MCAsmParser &Parser,
@@ -184,10 +199,6 @@ public:
     setAvailableFeatures(ComputeAvailableFeatures(STI.getFeatureBits()));
   }
 
-  void addSignature(std::unique_ptr<wasm::WasmSignature> &&Sig) {
-    Signatures.push_back(std::move(Sig));
-  }
-
 #define GET_ASSEMBLER_HEADER
 #include "WebAssemblyGenAsmMatcher.inc"
 
@@ -197,10 +208,60 @@ public:
     llvm_unreachable("ParseRegister is not implemented.");
   }
 
-  bool error(const StringRef &Msg, const AsmToken &Tok) {
+  bool error(const Twine &Msg, const AsmToken &Tok) {
     return Parser.Error(Tok.getLoc(), Msg + Tok.getString());
   }
 
+  bool error(const Twine &Msg) {
+    return Parser.Error(Lexer.getTok().getLoc(), Msg);
+  }
+
+  void addSignature(std::unique_ptr<wasm::WasmSignature> &&Sig) {
+    Signatures.push_back(std::move(Sig));
+  }
+
+  std::pair<StringRef, StringRef> nestingString(NestingType NT) {
+    switch (NT) {
+    case Function:
+      return {"function", "end_function"};
+    case Block:
+      return {"block", "end_block"};
+    case Loop:
+      return {"loop", "end_loop"};
+    case Try:
+      return {"try", "end_try"};
+    case If:
+      return {"if", "end_if"};
+    case Else:
+      return {"else", "end_if"};
+    default:
+      llvm_unreachable("unknown NestingType");
+    }
+  }
+
+  void push(NestingType NT) { NestingStack.push_back(NT); }
+
+  bool pop(StringRef Ins, NestingType NT1, NestingType NT2 = Undefined) {
+    if (NestingStack.empty())
+      return error(Twine("End of block construct with no start: ") + Ins);
+    auto Top = NestingStack.back();
+    if (Top != NT1 && Top != NT2)
+      return error(Twine("Block construct type mismatch, expected: ") +
+                   nestingString(Top).second + ", instead got: " + Ins);
+    NestingStack.pop_back();
+    return false;
+  }
+
+  bool ensureEmptyNestingStack() {
+    auto Err = !NestingStack.empty();
+    while (!NestingStack.empty()) {
+      error(Twine("Unmatched block construct(s) at function end: ") +
+            nestingString(NestingStack.back()).first);
+      NestingStack.pop_back();
+    }
+    return Err;
+  }
+
   bool isNext(AsmToken::TokenKind Kind) {
     auto Ok = Lexer.is(Kind);
     if (Ok)
@@ -240,14 +301,28 @@ public:
         Type == "i32x4" || Type == "i64x2" || Type == "f32x4" ||
         Type == "f64x2")
       return wasm::ValType::V128;
+    if (Type == "except_ref")
+      return wasm::ValType::EXCEPT_REF;
     return Optional<wasm::ValType>();
   }
 
+  WebAssembly::ExprType parseBlockType(StringRef ID) {
+    return StringSwitch<WebAssembly::ExprType>(ID)
+        .Case("i32", WebAssembly::ExprType::I32)
+        .Case("i64", WebAssembly::ExprType::I64)
+        .Case("f32", WebAssembly::ExprType::F32)
+        .Case("f64", WebAssembly::ExprType::F64)
+        .Case("v128", WebAssembly::ExprType::V128)
+        .Case("except_ref", WebAssembly::ExprType::ExceptRef)
+        .Case("void", WebAssembly::ExprType::Void)
+        .Default(WebAssembly::ExprType::Invalid);
+  }
+
   bool parseRegTypeList(SmallVectorImpl<wasm::ValType> &Types) {
     while (Lexer.is(AsmToken::Identifier)) {
       auto Type = parseType(Lexer.getTok().getString());
       if (!Type)
-        return true;
+        return error("unknown type: ", Lexer.getTok());
       Types.push_back(Type.getValue());
       Parser.Lex();
       if (!isNext(AsmToken::Comma))
@@ -293,6 +368,13 @@ public:
     return false;
   }
 
+  void addBlockTypeOperand(OperandVector &Operands, SMLoc NameLoc,
+                           WebAssembly::ExprType BT) {
+    Operands.push_back(make_unique<WebAssemblyOperand>(
+        WebAssemblyOperand::Integer, NameLoc, NameLoc,
+        WebAssemblyOperand::IntOp{static_cast<int64_t>(BT)}));
+  }
+
   bool ParseInstruction(ParseInstructionInfo & /*Info*/, StringRef Name,
                         SMLoc NameLoc, OperandVector &Operands) override {
     // Note: Name does NOT point into the sourcecode, but to a local, so
@@ -327,18 +409,73 @@ public:
     // If no '.', there is no type prefix.
     auto BaseName = NamePair.second.empty() ? NamePair.first : NamePair.second;
 
+    // If this instruction is part of a control flow structure, ensure
+    // proper nesting.
+    bool ExpectBlockType = false;
+    if (BaseName == "block") {
+      push(Block);
+      ExpectBlockType = true;
+    } else if (BaseName == "loop") {
+      push(Loop);
+      ExpectBlockType = true;
+    } else if (BaseName == "try") {
+      push(Try);
+      ExpectBlockType = true;
+    } else if (BaseName == "if") {
+      push(If);
+      ExpectBlockType = true;
+    } else if (BaseName == "else") {
+      if (pop(BaseName, If))
+        return true;
+      push(Else);
+    } else if (BaseName == "catch") {
+      if (pop(BaseName, Try))
+        return true;
+      push(Try);
+    } else if (BaseName == "catch_all") {
+      if (pop(BaseName, Try))
+        return true;
+      push(Try);
+    } else if (BaseName == "end_if") {
+      if (pop(BaseName, If, Else))
+        return true;
+    } else if (BaseName == "end_try") {
+      if (pop(BaseName, Try))
+        return true;
+    } else if (BaseName == "end_loop") {
+      if (pop(BaseName, Loop))
+        return true;
+    } else if (BaseName == "end_block") {
+      if (pop(BaseName, Block))
+        return true;
+    } else if (BaseName == "end_function") {
+      CurrentState = EndFunction;
+      if (pop(BaseName, Function) || ensureEmptyNestingStack())
+        return true;
+    }
+
     while (Lexer.isNot(AsmToken::EndOfStatement)) {
       auto &Tok = Lexer.getTok();
       switch (Tok.getKind()) {
       case AsmToken::Identifier: {
         auto &Id = Lexer.getTok();
-        const MCExpr *Val;
-        SMLoc End;
-        if (Parser.parsePrimaryExpr(Val, End))
-          return error("Cannot parse symbol: ", Lexer.getTok());
-        Operands.push_back(make_unique<WebAssemblyOperand>(
-            WebAssemblyOperand::Symbol, Id.getLoc(), Id.getEndLoc(),
-            WebAssemblyOperand::SymOp{Val}));
+        if (ExpectBlockType) {
+          // Assume this identifier is a block_type.
+          auto BT = parseBlockType(Id.getString());
+          if (BT == WebAssembly::ExprType::Invalid)
+            return error("Unknown block type: ", Id);
+          addBlockTypeOperand(Operands, NameLoc, BT);
+          Parser.Lex();
+        } else {
+          // Assume this identifier is a label.
+          const MCExpr *Val;
+          SMLoc End;
+          if (Parser.parsePrimaryExpr(Val, End))
+            return error("Cannot parse symbol: ", Lexer.getTok());
+          Operands.push_back(make_unique<WebAssemblyOperand>(
+              WebAssemblyOperand::Symbol, Id.getLoc(), Id.getEndLoc(),
+              WebAssemblyOperand::SymOp{Val}));
+        }
         break;
       }
       case AsmToken::Minus:
@@ -385,17 +522,11 @@ public:
           return true;
       }
     }
-    Parser.Lex();
-
-    // Block instructions require a signature index, but these are missing in
-    // assembly, so we add a dummy one explicitly (since we have no control
-    // over signature tables here, we assume these will be regenerated when
-    // the wasm module is generated).
-    if (BaseName == "block" || BaseName == "loop" || BaseName == "try") {
-      Operands.push_back(make_unique<WebAssemblyOperand>(
-          WebAssemblyOperand::Integer, NameLoc, NameLoc,
-          WebAssemblyOperand::IntOp{-1}));
+    if (ExpectBlockType && Operands.size() == 1) {
+      // Support blocks with no operands as default to void.
+      addBlockTypeOperand(Operands, NameLoc, WebAssembly::ExprType::Void);
     }
+    Parser.Lex();
     return false;
   }
 
@@ -436,6 +567,7 @@ public:
     auto &Out = getStreamer();
     auto &TOut =
         reinterpret_cast<WebAssemblyTargetStreamer &>(*Out.getTargetStreamer());
+    auto &Ctx = Out.getContext();
 
     // TODO: any time we return an error, at least one token must have been
     // consumed, otherwise this will not signal an error to the caller.
@@ -453,8 +585,7 @@ public:
       if (!Type)
         return error("Unknown type in .globaltype directive: ", TypeTok);
       // Now set this symbol with the correct type.
-      auto WasmSym = cast<MCSymbolWasm>(
-          TOut.getStreamer().getContext().getOrCreateSymbol(SymName));
+      auto WasmSym = cast<MCSymbolWasm>(Ctx.getOrCreateSymbol(SymName));
       WasmSym->setType(wasm::WASM_SYMBOL_TYPE_GLOBAL);
       WasmSym->setGlobalType(
           wasm::WasmGlobalType{uint8_t(Type.getValue()), true});
@@ -472,11 +603,14 @@ public:
       auto SymName = expectIdent();
       if (SymName.empty())
         return true;
-      auto WasmSym = cast<MCSymbolWasm>(
-          TOut.getStreamer().getContext().getOrCreateSymbol(SymName));
+      auto WasmSym = cast<MCSymbolWasm>(Ctx.getOrCreateSymbol(SymName));
       if (CurrentState == Label && WasmSym == LastLabel) {
         // This .functype indicates a start of a function.
+        if (ensureEmptyNestingStack())
+          return true;
         CurrentState = FunctionStart;
+        LastFunctionLabel = LastLabel;
+        push(Function);
       }
       auto Signature = make_unique<wasm::WasmSignature>();
       if (parseSignature(Signature.get()))
@@ -493,8 +627,7 @@ public:
       auto SymName = expectIdent();
       if (SymName.empty())
         return true;
-      auto WasmSym = cast<MCSymbolWasm>(
-          TOut.getStreamer().getContext().getOrCreateSymbol(SymName));
+      auto WasmSym = cast<MCSymbolWasm>(Ctx.getOrCreateSymbol(SymName));
       auto Signature = make_unique<wasm::WasmSignature>();
       if (parseRegTypeList(Signature->Params))
         return true;
@@ -539,8 +672,12 @@ public:
             *Out.getTargetStreamer());
         TOut.emitLocal(SmallVector<wasm::ValType, 0>());
       }
-      CurrentState = Instructions;
       Out.EmitInstruction(Inst, getSTI());
+      if (CurrentState == EndFunction) {
+        onEndOfFunction();
+      } else {
+        CurrentState = Instructions;
+      }
       return false;
     }
     case Match_MissingFeature:
@@ -565,6 +702,32 @@ public:
     }
     llvm_unreachable("Implement any new match types added!");
   }
+
+  void doBeforeLabelEmit(MCSymbol *Symbol) override {
+    // Start a new section for the next function automatically, since our
+    // object writer expects each function to have its own section. This way
+    // The user can't forget this "convention".
+    auto SymName = Symbol->getName();
+    if (SymName.startswith(".L"))
+      return; // Local Symbol.
+    auto SecName = ".text." + SymName;
+    auto WS = getContext().getWasmSection(SecName, SectionKind::getText());
+    getStreamer().SwitchSection(WS);
+  }
+
+  void onEndOfFunction() {
+    // Automatically output a .size directive, so it becomes optional for the
+    // user.
+    auto TempSym = getContext().createLinkerPrivateTempSymbol();
+    getStreamer().EmitLabel(TempSym);
+    auto Start = MCSymbolRefExpr::create(LastLabel, getContext());
+    auto End = MCSymbolRefExpr::create(TempSym, getContext());
+    auto Expr =
+        MCBinaryExpr::create(MCBinaryExpr::Sub, End, Start, getContext());
+    getStreamer().emitELFSize(LastFunctionLabel, Expr);
+  }
+
+  void onEndOfFile() override { ensureEmptyNestingStack(); }
 };
 } // end anonymous namespace
 
diff --git a/lib/Target/WebAssembly/CMakeLists.txt b/lib/Target/WebAssembly/CMakeLists.txt
index 549229ad572b25e45a0c1880b573d3b3064a1acb..4fc2e26ccf359ee74ae98b23271bb955abe0d1a7 100644
--- a/lib/Target/WebAssembly/CMakeLists.txt
+++ b/lib/Target/WebAssembly/CMakeLists.txt
@@ -19,8 +19,8 @@ add_llvm_target(WebAssemblyCodeGen
   WebAssemblyCallIndirectFixup.cpp
   WebAssemblyCFGStackify.cpp
   WebAssemblyCFGSort.cpp
+  WebAssemblyDebugValueManager.cpp
   WebAssemblyLateEHPrepare.cpp
-  WebAssemblyEHRestoreStackPointer.cpp
   WebAssemblyExceptionInfo.cpp
   WebAssemblyExplicitLocals.cpp
   WebAssemblyFastISel.cpp
@@ -47,7 +47,7 @@ add_llvm_target(WebAssemblyCodeGen
   WebAssemblyRuntimeLibcallSignatures.cpp
   WebAssemblySelectionDAGInfo.cpp
   WebAssemblySetP2AlignOperands.cpp
-  WebAssemblyStoreResults.cpp
+  WebAssemblyMemIntrinsicResults.cpp
   WebAssemblySubtarget.cpp
   WebAssemblyTargetMachine.cpp
   WebAssemblyTargetObjectFile.cpp
diff --git a/lib/Target/WebAssembly/Disassembler/LLVMBuild.txt b/lib/Target/WebAssembly/Disassembler/LLVMBuild.txt
index a452ca1acd04b17f04ec07ae5a13b849deec1b11..8fab4aaca7b0e32e447ba8089ffaf790c343c776 100644
--- a/lib/Target/WebAssembly/Disassembler/LLVMBuild.txt
+++ b/lib/Target/WebAssembly/Disassembler/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===-- ./lib/Target/WebAssembly/Disassembler/LLVMBuild.txt -----*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
@@ -19,5 +18,5 @@
 type = Library
 name = WebAssemblyDisassembler
 parent = WebAssembly
-required_libraries = MCDisassembler WebAssemblyInfo Support
+required_libraries = MCDisassembler WebAssemblyInfo WebAssemblyAsmPrinter Support
 add_to_library_groups = WebAssembly
diff --git a/lib/Target/WebAssembly/Disassembler/WebAssemblyDisassembler.cpp b/lib/Target/WebAssembly/Disassembler/WebAssemblyDisassembler.cpp
index c068d6b43d492fbaffa043dcf447862ad0441b05..23cf60c1877c9b4d84e1dcc4b1c745ecf415ee5e 100644
--- a/lib/Target/WebAssembly/Disassembler/WebAssemblyDisassembler.cpp
+++ b/lib/Target/WebAssembly/Disassembler/WebAssemblyDisassembler.cpp
@@ -1,9 +1,8 @@
 //==- WebAssemblyDisassembler.cpp - Disassembler for WebAssembly -*- C++ -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
@@ -15,6 +14,7 @@
 ///
 //===----------------------------------------------------------------------===//
 
+#include "InstPrinter/WebAssemblyInstPrinter.h"
 #include "MCTargetDesc/WebAssemblyMCTargetDesc.h"
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCDisassembler/MCDisassembler.h"
@@ -45,6 +45,10 @@ class WebAssemblyDisassembler final : public MCDisassembler {
                               ArrayRef<uint8_t> Bytes, uint64_t Address,
                               raw_ostream &VStream,
                               raw_ostream &CStream) const override;
+  DecodeStatus onSymbolStart(StringRef Name, uint64_t &Size,
+                             ArrayRef<uint8_t> Bytes, uint64_t Address,
+                             raw_ostream &VStream,
+                             raw_ostream &CStream) const override;
 
 public:
   WebAssemblyDisassembler(const MCSubtargetInfo &STI, MCContext &Ctx,
@@ -77,7 +81,7 @@ static int nextByte(ArrayRef<uint8_t> Bytes, uint64_t &Size) {
 }
 
 static bool nextLEB(int64_t &Val, ArrayRef<uint8_t> Bytes, uint64_t &Size,
-                    bool Signed = false) {
+                    bool Signed) {
   unsigned N = 0;
   const char *Error = nullptr;
   Val = Signed ? decodeSLEB128(Bytes.data() + Size, &N,
@@ -104,9 +108,8 @@ template <typename T>
 bool parseImmediate(MCInst &MI, uint64_t &Size, ArrayRef<uint8_t> Bytes) {
   if (Size + sizeof(T) > Bytes.size())
     return false;
-  T Val;
-  memcpy(&Val, Bytes.data() + Size, sizeof(T));
-  support::endian::byte_swap<T, support::endianness::little>(Val);
+  T Val = support::endian::read<T, support::endianness::little, 1>(
+      Bytes.data() + Size);
   Size += sizeof(T);
   if (std::is_floating_point<T>::value) {
     MI.addOperand(MCOperand::createFPImm(static_cast<double>(Val)));
@@ -116,6 +119,41 @@ bool parseImmediate(MCInst &MI, uint64_t &Size, ArrayRef<uint8_t> Bytes) {
   return true;
 }
 
+MCDisassembler::DecodeStatus WebAssemblyDisassembler::onSymbolStart(
+    StringRef Name, uint64_t &Size, ArrayRef<uint8_t> Bytes, uint64_t Address,
+    raw_ostream &VStream, raw_ostream &CStream) const {
+  Size = 0;
+  if (Address == 0) {
+    // Start of a code section: we're parsing only the function count.
+    int64_t FunctionCount;
+    if (!nextLEB(FunctionCount, Bytes, Size, false))
+      return MCDisassembler::Fail;
+    outs() << "        # " << FunctionCount << " functions in section.";
+  } else {
+    // Parse the start of a single function.
+    int64_t BodySize, LocalEntryCount;
+    if (!nextLEB(BodySize, Bytes, Size, false) ||
+        !nextLEB(LocalEntryCount, Bytes, Size, false))
+      return MCDisassembler::Fail;
+    if (LocalEntryCount) {
+      outs() << "        .local ";
+      for (int64_t I = 0; I < LocalEntryCount; I++) {
+        int64_t Count, Type;
+        if (!nextLEB(Count, Bytes, Size, false) ||
+            !nextLEB(Type, Bytes, Size, false))
+          return MCDisassembler::Fail;
+        for (int64_t J = 0; J < Count; J++) {
+          if (I || J)
+            outs() << ", ";
+          outs() << WebAssembly::anyTypeToString(Type);
+        }
+      }
+    }
+  }
+  outs() << "\n";
+  return MCDisassembler::Success;
+}
+
 MCDisassembler::DecodeStatus WebAssemblyDisassembler::getInstruction(
     MCInst &MI, uint64_t &Size, ArrayRef<uint8_t> Bytes, uint64_t /*Address*/,
     raw_ostream & /*OS*/, raw_ostream &CS) const {
@@ -138,7 +176,7 @@ MCDisassembler::DecodeStatus WebAssemblyDisassembler::getInstruction(
     if (!WasmInst)
       return MCDisassembler::Fail;
     int64_t PrefixedOpc;
-    if (!nextLEB(PrefixedOpc, Bytes, Size))
+    if (!nextLEB(PrefixedOpc, Bytes, Size, false))
       return MCDisassembler::Fail;
     if (PrefixedOpc < 0 || PrefixedOpc >= WebAssemblyInstructionTableSize)
       return MCDisassembler::Fail;
@@ -151,7 +189,8 @@ MCDisassembler::DecodeStatus WebAssemblyDisassembler::getInstruction(
   MI.setOpcode(WasmInst->Opcode);
   // Parse any operands.
   for (uint8_t OPI = 0; OPI < WasmInst->NumOperands; OPI++) {
-    switch (OperandTable[WasmInst->OperandStart + OPI]) {
+    auto OT = OperandTable[WasmInst->OperandStart + OPI];
+    switch (OT) {
     // ULEB operands:
     case WebAssembly::OPERAND_BASIC_BLOCK:
     case WebAssembly::OPERAND_LOCAL:
@@ -167,12 +206,17 @@ MCDisassembler::DecodeStatus WebAssemblyDisassembler::getInstruction(
     }
     // SLEB operands:
     case WebAssembly::OPERAND_I32IMM:
-    case WebAssembly::OPERAND_I64IMM:
-    case WebAssembly::OPERAND_SIGNATURE: {
+    case WebAssembly::OPERAND_I64IMM: {
       if (!parseLEBImmediate(MI, Size, Bytes, true))
         return MCDisassembler::Fail;
       break;
     }
+    // block_type operands (uint8_t).
+    case WebAssembly::OPERAND_SIGNATURE: {
+      if (!parseImmediate<uint8_t>(MI, Size, Bytes))
+        return MCDisassembler::Fail;
+      break;
+    }
     // FP operands.
     case WebAssembly::OPERAND_F32IMM: {
       if (!parseImmediate<float>(MI, Size, Bytes))
@@ -205,6 +249,19 @@ MCDisassembler::DecodeStatus WebAssemblyDisassembler::getInstruction(
         return MCDisassembler::Fail;
       break;
     }
+    case WebAssembly::OPERAND_BRLIST: {
+      int64_t TargetTableLen;
+      if (!nextLEB(TargetTableLen, Bytes, Size, false))
+        return MCDisassembler::Fail;
+      for (int64_t I = 0; I < TargetTableLen; I++) {
+        if (!parseLEBImmediate(MI, Size, Bytes, false))
+          return MCDisassembler::Fail;
+      }
+      // Default case.
+      if (!parseLEBImmediate(MI, Size, Bytes, false))
+        return MCDisassembler::Fail;
+      break;
+    }
     case MCOI::OPERAND_REGISTER:
       // The tablegen header currently does not have any register operands since
       // we use only the stack (_S) instructions.
diff --git a/lib/Target/WebAssembly/InstPrinter/LLVMBuild.txt b/lib/Target/WebAssembly/InstPrinter/LLVMBuild.txt
index 54df6d65570a017204647661cecedf2f3c1cdccf..76fcec192ff3606bf50a137983a9ee5657518edd 100644
--- a/lib/Target/WebAssembly/InstPrinter/LLVMBuild.txt
+++ b/lib/Target/WebAssembly/InstPrinter/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/Target/WebAssembly/InstPrinter/LLVMBuild.txt -------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Target/WebAssembly/InstPrinter/WebAssemblyInstPrinter.cpp b/lib/Target/WebAssembly/InstPrinter/WebAssemblyInstPrinter.cpp
index eaf9750792bba97c01f93247ab324882d3afd0c8..deced385154c5bfb658bf1d52dc9c5e9391c571c 100644
--- a/lib/Target/WebAssembly/InstPrinter/WebAssemblyInstPrinter.cpp
+++ b/lib/Target/WebAssembly/InstPrinter/WebAssemblyInstPrinter.cpp
@@ -1,9 +1,8 @@
 //=- WebAssemblyInstPrinter.cpp - WebAssembly assembly instruction printing -=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
@@ -40,7 +39,7 @@ WebAssemblyInstPrinter::WebAssemblyInstPrinter(const MCAsmInfo &MAI,
 void WebAssemblyInstPrinter::printRegName(raw_ostream &OS,
                                           unsigned RegNo) const {
   assert(RegNo != WebAssemblyFunctionInfo::UnusedReg);
-  // Note that there's an implicit get_local/set_local here!
+  // Note that there's an implicit local.get/local.set here!
   OS << "$" << RegNo;
 }
 
@@ -53,15 +52,15 @@ void WebAssemblyInstPrinter::printInst(const MCInst *MI, raw_ostream &OS,
   // Print any additional variadic operands.
   const MCInstrDesc &Desc = MII.get(MI->getOpcode());
   if (Desc.isVariadic())
-    for (auto i = Desc.getNumOperands(), e = MI->getNumOperands(); i < e; ++i) {
+    for (auto I = Desc.getNumOperands(), E = MI->getNumOperands(); I < E; ++I) {
       // FIXME: For CALL_INDIRECT_VOID, don't print a leading comma, because
       // we have an extra flags operand which is not currently printed, for
       // compatiblity reasons.
-      if (i != 0 && ((MI->getOpcode() != WebAssembly::CALL_INDIRECT_VOID &&
+      if (I != 0 && ((MI->getOpcode() != WebAssembly::CALL_INDIRECT_VOID &&
                       MI->getOpcode() != WebAssembly::CALL_INDIRECT_VOID_S) ||
-                     i != Desc.getNumOperands()))
+                     I != Desc.getNumOperands()))
         OS << ", ";
-      printOperand(MI, i, OS);
+      printOperand(MI, I, OS);
     }
 
   // Print any added annotation.
@@ -95,80 +94,84 @@ void WebAssemblyInstPrinter::printInst(const MCInst *MI, raw_ostream &OS,
 
     case WebAssembly::END_LOOP:
     case WebAssembly::END_LOOP_S:
-      assert(!ControlFlowStack.empty() && "End marker mismatch!");
-      ControlFlowStack.pop_back();
+      if (ControlFlowStack.empty()) {
+        printAnnotation(OS, "End marker mismatch!");
+      } else {
+        ControlFlowStack.pop_back();
+      }
       break;
 
     case WebAssembly::END_BLOCK:
     case WebAssembly::END_BLOCK_S:
-      assert(!ControlFlowStack.empty() && "End marker mismatch!");
-      printAnnotation(
-          OS, "label" + utostr(ControlFlowStack.pop_back_val().first) + ':');
+      if (ControlFlowStack.empty()) {
+        printAnnotation(OS, "End marker mismatch!");
+      } else {
+        printAnnotation(
+            OS, "label" + utostr(ControlFlowStack.pop_back_val().first) + ':');
+      }
       break;
 
     case WebAssembly::END_TRY:
     case WebAssembly::END_TRY_S:
-      assert(!ControlFlowStack.empty() && "End marker mismatch!");
-      printAnnotation(
-          OS, "label" + utostr(ControlFlowStack.pop_back_val().first) + ':');
-      LastSeenEHInst = END_TRY;
+      if (ControlFlowStack.empty()) {
+        printAnnotation(OS, "End marker mismatch!");
+      } else {
+        printAnnotation(
+            OS, "label" + utostr(ControlFlowStack.pop_back_val().first) + ':');
+        LastSeenEHInst = END_TRY;
+      }
       break;
 
-    case WebAssembly::CATCH_I32:
-    case WebAssembly::CATCH_I32_S:
-    case WebAssembly::CATCH_I64:
-    case WebAssembly::CATCH_I64_S:
-    case WebAssembly::CATCH_ALL:
-    case WebAssembly::CATCH_ALL_S:
-      // There can be multiple catch instructions for one try instruction, so we
-      // print a label only for the first 'catch' label.
-      if (LastSeenEHInst != CATCH) {
-        assert(!EHPadStack.empty() && "try-catch mismatch!");
+    case WebAssembly::CATCH:
+    case WebAssembly::CATCH_S:
+      if (EHPadStack.empty()) {
+        printAnnotation(OS, "try-catch mismatch!");
+      } else {
         printAnnotation(OS, "catch" + utostr(EHPadStack.pop_back_val()) + ':');
       }
-      LastSeenEHInst = CATCH;
       break;
     }
 
     // Annotate any control flow label references.
-    unsigned NumFixedOperands = Desc.NumOperands;
-    SmallSet<uint64_t, 8> Printed;
-    for (unsigned i = 0, e = MI->getNumOperands(); i < e; ++i) {
-      // See if this operand denotes a basic block target.
-      if (i < NumFixedOperands) {
-        // A non-variable_ops operand, check its type.
-        if (Desc.OpInfo[i].OperandType != WebAssembly::OPERAND_BASIC_BLOCK)
-          continue;
+
+    // rethrow instruction does not take any depth argument and rethrows to the
+    // nearest enclosing catch scope, if any. If there's no enclosing catch
+    // scope, it throws up to the caller.
+    if (Opc == WebAssembly::RETHROW || Opc == WebAssembly::RETHROW_S) {
+      if (EHPadStack.empty()) {
+        printAnnotation(OS, "to caller");
       } else {
-        // A variable_ops operand, which currently can be immediates (used in
-        // br_table) which are basic block targets, or for call instructions
-        // when using -wasm-keep-registers (in which case they are registers,
-        // and should not be processed).
-        if (!MI->getOperand(i).isImm())
-          continue;
+        printAnnotation(OS, "down to catch" + utostr(EHPadStack.back()));
       }
-      uint64_t Depth = MI->getOperand(i).getImm();
-      if (!Printed.insert(Depth).second)
-        continue;
 
-      if (Opc == WebAssembly::RETHROW || Opc == WebAssembly::RETHROW_S) {
-        assert(Depth <= EHPadStack.size() && "Invalid depth argument!");
-        if (Depth == EHPadStack.size()) {
-          // This can happen when rethrow instruction breaks out of all nests
-          // and throws up to the current function's caller.
-          printAnnotation(OS, utostr(Depth) + ": " + "to caller");
+    } else {
+      unsigned NumFixedOperands = Desc.NumOperands;
+      SmallSet<uint64_t, 8> Printed;
+      for (unsigned I = 0, E = MI->getNumOperands(); I < E; ++I) {
+        // See if this operand denotes a basic block target.
+        if (I < NumFixedOperands) {
+          // A non-variable_ops operand, check its type.
+          if (Desc.OpInfo[I].OperandType != WebAssembly::OPERAND_BASIC_BLOCK)
+            continue;
         } else {
-          uint64_t CatchNo = EHPadStack.rbegin()[Depth];
-          printAnnotation(OS, utostr(Depth) + ": " + "down to catch" +
-                                  utostr(CatchNo));
+          // A variable_ops operand, which currently can be immediates (used in
+          // br_table) which are basic block targets, or for call instructions
+          // when using -wasm-keep-registers (in which case they are registers,
+          // and should not be processed).
+          if (!MI->getOperand(I).isImm())
+            continue;
+        }
+        uint64_t Depth = MI->getOperand(I).getImm();
+        if (!Printed.insert(Depth).second)
+          continue;
+        if (Depth >= ControlFlowStack.size()) {
+          printAnnotation(OS, "Invalid depth argument!");
+        } else {
+          const auto &Pair = ControlFlowStack.rbegin()[Depth];
+          printAnnotation(OS, utostr(Depth) + ": " +
+                                  (Pair.second ? "up" : "down") + " to label" +
+                                  utostr(Pair.first));
         }
-
-      } else {
-        assert(Depth < ControlFlowStack.size() && "Invalid depth argument!");
-        const auto &Pair = ControlFlowStack.rbegin()[Depth];
-        printAnnotation(OS, utostr(Depth) + ": " +
-                                (Pair.second ? "up" : "down") + " to label" +
-                                utostr(Pair.first));
       }
     }
   }
@@ -189,13 +192,13 @@ static std::string toString(const APFloat &FP) {
 
   // Use C99's hexadecimal floating-point representation.
   static const size_t BufBytes = 128;
-  char buf[BufBytes];
+  char Buf[BufBytes];
   auto Written = FP.convertToHexString(
-      buf, /*hexDigits=*/0, /*upperCase=*/false, APFloat::rmNearestTiesToEven);
+      Buf, /*hexDigits=*/0, /*upperCase=*/false, APFloat::rmNearestTiesToEven);
   (void)Written;
   assert(Written != 0);
   assert(Written < BufBytes);
-  return buf;
+  return Buf;
 }
 
 void WebAssemblyInstPrinter::printOperand(const MCInst *MI, unsigned OpNo,
@@ -256,45 +259,38 @@ void WebAssemblyInstPrinter::printWebAssemblyP2AlignOperand(const MCInst *MI,
 void WebAssemblyInstPrinter::printWebAssemblySignatureOperand(const MCInst *MI,
                                                               unsigned OpNo,
                                                               raw_ostream &O) {
-  int64_t Imm = MI->getOperand(OpNo).getImm();
-  switch (WebAssembly::ExprType(Imm)) {
-  case WebAssembly::ExprType::Void:
-    break;
-  case WebAssembly::ExprType::I32:
-    O << "i32";
-    break;
-  case WebAssembly::ExprType::I64:
-    O << "i64";
-    break;
-  case WebAssembly::ExprType::F32:
-    O << "f32";
-    break;
-  case WebAssembly::ExprType::F64:
-    O << "f64";
-    break;
-  case WebAssembly::ExprType::V128:
-    O << "v128";
-    break;
-  case WebAssembly::ExprType::ExceptRef:
-    O << "except_ref";
-    break;
-  }
+  auto Imm = static_cast<unsigned>(MI->getOperand(OpNo).getImm());
+  if (Imm != wasm::WASM_TYPE_NORESULT)
+    O << WebAssembly::anyTypeToString(Imm);
 }
 
-const char *llvm::WebAssembly::TypeToString(wasm::ValType Ty) {
+// We have various enums representing a subset of these types, use this
+// function to convert any of them to text.
+const char *llvm::WebAssembly::anyTypeToString(unsigned Ty) {
   switch (Ty) {
-  case wasm::ValType::I32:
+  case wasm::WASM_TYPE_I32:
     return "i32";
-  case wasm::ValType::I64:
+  case wasm::WASM_TYPE_I64:
     return "i64";
-  case wasm::ValType::F32:
+  case wasm::WASM_TYPE_F32:
     return "f32";
-  case wasm::ValType::F64:
+  case wasm::WASM_TYPE_F64:
     return "f64";
-  case wasm::ValType::V128:
+  case wasm::WASM_TYPE_V128:
     return "v128";
-  case wasm::ValType::EXCEPT_REF:
+  case wasm::WASM_TYPE_FUNCREF:
+    return "funcref";
+  case wasm::WASM_TYPE_FUNC:
+    return "func";
+  case wasm::WASM_TYPE_EXCEPT_REF:
     return "except_ref";
+  case wasm::WASM_TYPE_NORESULT:
+    return "void";
+  default:
+    return "invalid_type";
   }
-  llvm_unreachable("Unknown wasm::ValType");
+}
+
+const char *llvm::WebAssembly::typeToString(wasm::ValType Ty) {
+  return anyTypeToString(static_cast<unsigned>(Ty));
 }
diff --git a/lib/Target/WebAssembly/InstPrinter/WebAssemblyInstPrinter.h b/lib/Target/WebAssembly/InstPrinter/WebAssemblyInstPrinter.h
index fd968bc9ea304be46d670023d6365143f5273a64..b979de5028bff0cbe4b2b56a46f7cc3f40e147a3 100644
--- a/lib/Target/WebAssembly/InstPrinter/WebAssemblyInstPrinter.h
+++ b/lib/Target/WebAssembly/InstPrinter/WebAssemblyInstPrinter.h
@@ -1,9 +1,8 @@
 // WebAssemblyInstPrinter.h - Print wasm MCInst to assembly syntax -*- C++ -*-//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
@@ -56,7 +55,8 @@ public:
 
 namespace WebAssembly {
 
-const char *TypeToString(wasm::ValType Ty);
+const char *typeToString(wasm::ValType Ty);
+const char *anyTypeToString(unsigned Ty);
 
 } // end namespace WebAssembly
 
diff --git a/lib/Target/WebAssembly/LLVMBuild.txt b/lib/Target/WebAssembly/LLVMBuild.txt
index 055c32bf0cbf3a834043a78325912cbc8c56f29d..08a6e6a7ec747bb10c9f6e398200bede0dfa8327 100644
--- a/lib/Target/WebAssembly/LLVMBuild.txt
+++ b/lib/Target/WebAssembly/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/Target/WebAssembly/LLVMBuild.txt -------------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
@@ -30,5 +29,5 @@ has_disassembler = 1
 type = Library
 name = WebAssemblyCodeGen
 parent = WebAssembly
-required_libraries = Analysis AsmPrinter CodeGen Core MC Scalar SelectionDAG Support Target TransformUtils WebAssemblyAsmPrinter WebAssemblyDesc WebAssemblyInfo
+required_libraries = Analysis AsmPrinter BinaryFormat CodeGen Core MC Scalar SelectionDAG Support Target TransformUtils WebAssemblyAsmPrinter WebAssemblyDesc WebAssemblyInfo
 add_to_library_groups = WebAssembly
diff --git a/lib/Target/WebAssembly/MCTargetDesc/LLVMBuild.txt b/lib/Target/WebAssembly/MCTargetDesc/LLVMBuild.txt
index ce7cb5dd4daf5ac984ca14705b86e73e449ec836..06b85c5afbac2b037aa4ecee81298796551f2570 100644
--- a/lib/Target/WebAssembly/MCTargetDesc/LLVMBuild.txt
+++ b/lib/Target/WebAssembly/MCTargetDesc/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/Target/WebAssembly/MCTargetDesc/LLVMBuild.txt ------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyAsmBackend.cpp b/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyAsmBackend.cpp
index 0726dd4811745e1fca98d0bb7a67c4074b8547ce..80c8db46915166a8f67f7ce88daf75d73e081d0b 100644
--- a/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyAsmBackend.cpp
+++ b/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyAsmBackend.cpp
@@ -1,9 +1,8 @@
 //===-- WebAssemblyAsmBackend.cpp - WebAssembly Assembler Backend ---------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
@@ -36,7 +35,6 @@ class WebAssemblyAsmBackend final : public MCAsmBackend {
 public:
   explicit WebAssemblyAsmBackend(bool Is64Bit)
       : MCAsmBackend(support::little), Is64Bit(Is64Bit) {}
-  ~WebAssemblyAsmBackend() override {}
 
   unsigned getNumFixupKinds() const override {
     return WebAssembly::NumTargetFixupKinds;
@@ -92,7 +90,7 @@ WebAssemblyAsmBackend::getFixupKindInfo(MCFixupKind Kind) const {
 
 bool WebAssemblyAsmBackend::writeNopData(raw_ostream &OS,
                                          uint64_t Count) const {
-  for (uint64_t i = 0; i < Count; ++i)
+  for (uint64_t I = 0; I < Count; ++I)
     OS << char(WebAssembly::Nop);
 
   return true;
@@ -119,8 +117,8 @@ void WebAssemblyAsmBackend::applyFixup(const MCAssembler &Asm,
 
   // For each byte of the fragment that the fixup touches, mask in the
   // bits from the fixup value.
-  for (unsigned i = 0; i != NumBytes; ++i)
-    Data[Offset + i] |= uint8_t((Value >> (i * 8)) & 0xff);
+  for (unsigned I = 0; I != NumBytes; ++I)
+    Data[Offset + I] |= uint8_t((Value >> (I * 8)) & 0xff);
 }
 
 std::unique_ptr<MCObjectTargetWriter>
diff --git a/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyFixupKinds.h b/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyFixupKinds.h
index c2fac5f93a2f3d904f1e8cd9acd376ee7edc32d8..da6d0063cfbdc4e208b3926b58c17dfec1d6b5fc 100644
--- a/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyFixupKinds.h
+++ b/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyFixupKinds.h
@@ -1,9 +1,8 @@
 //=- WebAssemblyFixupKinds.h - WebAssembly Specific Fixup Entries -*- C++ -*-=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCAsmInfo.cpp b/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCAsmInfo.cpp
index 44fcc129c39e853ae0c7aa8e1d38629405bf0e3d..8f6531563e1baaa2d2f90c605e7c36b4fc4aef15 100644
--- a/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCAsmInfo.cpp
+++ b/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCAsmInfo.cpp
@@ -1,9 +1,8 @@
 //===-- WebAssemblyMCAsmInfo.cpp - WebAssembly asm properties -------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
@@ -20,7 +19,7 @@ using namespace llvm;
 
 #define DEBUG_TYPE "wasm-mc-asm-info"
 
-WebAssemblyMCAsmInfo::~WebAssemblyMCAsmInfo() {}
+WebAssemblyMCAsmInfo::~WebAssemblyMCAsmInfo() = default; // anchor.
 
 WebAssemblyMCAsmInfo::WebAssemblyMCAsmInfo(const Triple &T) {
   CodePointerSize = CalleeSaveStackSlotSize = T.isArch64Bit() ? 8 : 4;
diff --git a/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCAsmInfo.h b/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCAsmInfo.h
index 8627a6e40c6adee65d3b22376862ed813221749a..9efbbf881f5904be56189e1ba35cd396547fc613 100644
--- a/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCAsmInfo.h
+++ b/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCAsmInfo.h
@@ -1,9 +1,8 @@
 //===-- WebAssemblyMCAsmInfo.h - WebAssembly asm properties -----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
diff --git a/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCCodeEmitter.cpp b/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCCodeEmitter.cpp
index 065a4dc94ca6a0bf09d0a7458cc19c63625444f2..8d89c568cf9af27777eab9989225b744f85d7083 100644
--- a/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCCodeEmitter.cpp
+++ b/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCCodeEmitter.cpp
@@ -1,9 +1,8 @@
 //=- WebAssemblyMCCodeEmitter.cpp - Convert WebAssembly code to machine code -//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
@@ -49,7 +48,7 @@ class WebAssemblyMCCodeEmitter final : public MCCodeEmitter {
                          const MCSubtargetInfo &STI) const override;
 
 public:
-  WebAssemblyMCCodeEmitter(const MCInstrInfo &mcii) : MCII(mcii) {}
+  WebAssemblyMCCodeEmitter(const MCInstrInfo &MCII) : MCII(MCII) {}
 };
 } // end anonymous namespace
 
@@ -82,14 +81,14 @@ void WebAssemblyMCCodeEmitter::encodeInstruction(
     encodeULEB128(MI.getNumOperands() - 2, OS);
 
   const MCInstrDesc &Desc = MCII.get(MI.getOpcode());
-  for (unsigned i = 0, e = MI.getNumOperands(); i < e; ++i) {
-    const MCOperand &MO = MI.getOperand(i);
+  for (unsigned I = 0, E = MI.getNumOperands(); I < E; ++I) {
+    const MCOperand &MO = MI.getOperand(I);
     if (MO.isReg()) {
       /* nothing to encode */
 
     } else if (MO.isImm()) {
-      if (i < Desc.getNumOperands()) {
-        const MCOperandInfo &Info = Desc.OpInfo[i];
+      if (I < Desc.getNumOperands()) {
+        const MCOperandInfo &Info = Desc.OpInfo[I];
         LLVM_DEBUG(dbgs() << "Encoding immediate: type="
                           << int(Info.OperandType) << "\n");
         switch (Info.OperandType) {
@@ -127,20 +126,20 @@ void WebAssemblyMCCodeEmitter::encodeInstruction(
       }
 
     } else if (MO.isFPImm()) {
-      const MCOperandInfo &Info = Desc.OpInfo[i];
+      const MCOperandInfo &Info = Desc.OpInfo[I];
       if (Info.OperandType == WebAssembly::OPERAND_F32IMM) {
         // TODO: MC converts all floating point immediate operands to double.
         // This is fine for numeric values, but may cause NaNs to change bits.
-        float f = float(MO.getFPImm());
-        support::endian::write<float>(OS, f, support::little);
+        auto F = float(MO.getFPImm());
+        support::endian::write<float>(OS, F, support::little);
       } else {
         assert(Info.OperandType == WebAssembly::OPERAND_F64IMM);
-        double d = MO.getFPImm();
-        support::endian::write<double>(OS, d, support::little);
+        double D = MO.getFPImm();
+        support::endian::write<double>(OS, D, support::little);
       }
 
     } else if (MO.isExpr()) {
-      const MCOperandInfo &Info = Desc.OpInfo[i];
+      const MCOperandInfo &Info = Desc.OpInfo[I];
       llvm::MCFixupKind FixupKind;
       size_t PaddedSize = 5;
       switch (Info.OperandType) {
diff --git a/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCTargetDesc.cpp b/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCTargetDesc.cpp
index 390f367c29785231867e031f95c1173397026694..0667816c0265307246501731cc5067ee58ffeb0d 100644
--- a/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCTargetDesc.cpp
+++ b/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCTargetDesc.cpp
@@ -1,9 +1,8 @@
 //===-- WebAssemblyMCTargetDesc.cpp - WebAssembly Target Descriptions -----===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
@@ -40,13 +39,13 @@ static MCAsmInfo *createMCAsmInfo(const MCRegisterInfo & /*MRI*/,
 }
 
 static MCInstrInfo *createMCInstrInfo() {
-  MCInstrInfo *X = new MCInstrInfo();
+  auto *X = new MCInstrInfo();
   InitWebAssemblyMCInstrInfo(X);
   return X;
 }
 
 static MCRegisterInfo *createMCRegisterInfo(const Triple & /*T*/) {
-  MCRegisterInfo *X = new MCRegisterInfo();
+  auto *X = new MCRegisterInfo();
   InitWebAssemblyMCRegisterInfo(X, 0);
   return X;
 }
diff --git a/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCTargetDesc.h b/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCTargetDesc.h
index 4ca20f5936831b3133da63444b713052b0f14981..faa8c69baa4b92215ecc09ceca6978e7518f9b21 100644
--- a/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCTargetDesc.h
+++ b/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCTargetDesc.h
@@ -1,9 +1,8 @@
 //==- WebAssemblyMCTargetDesc.h - WebAssembly Target Descriptions -*- C++ -*-=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
@@ -79,6 +78,8 @@ enum OperandType {
   OPERAND_TYPEINDEX,
   /// Event index.
   OPERAND_EVENT,
+  /// A list of branch targets for br_list.
+  OPERAND_BRLIST,
 };
 } // end namespace WebAssembly
 
@@ -346,7 +347,8 @@ enum class ExprType : unsigned {
   F32 = 0x7D,
   F64 = 0x7C,
   V128 = 0x7B,
-  ExceptRef = 0x68
+  ExceptRef = 0x68,
+  Invalid = 0x00
 };
 
 /// Instruction opcodes emitted via means other than CodeGen.
diff --git a/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyTargetStreamer.cpp b/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyTargetStreamer.cpp
index 70ac50272545b9cde60550bc5d3f6168e81e69d0..4bc731a7ef45a5727044f69bcfcaaec77ddf942f 100644
--- a/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyTargetStreamer.cpp
+++ b/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyTargetStreamer.cpp
@@ -1,9 +1,8 @@
 //==-- WebAssemblyTargetStreamer.cpp - WebAssembly Target Streamer Methods --=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
@@ -47,7 +46,7 @@ static void printTypes(formatted_raw_ostream &OS,
       First = false;
     else
       OS << ", ";
-    OS << WebAssembly::TypeToString(Type);
+    OS << WebAssembly::typeToString(Type);
   }
   OS << '\n';
 }
@@ -76,7 +75,7 @@ void WebAssemblyTargetAsmStreamer::emitParamList(
   for (auto &Ty : Params) {
     if (&Ty != &Params[0])
       OS << ", ";
-    OS << WebAssembly::TypeToString(Ty);
+    OS << WebAssembly::typeToString(Ty);
   }
 }
 
@@ -86,7 +85,7 @@ void WebAssemblyTargetAsmStreamer::emitReturnList(
   for (auto &Ty : Returns) {
     if (&Ty != &Returns[0])
       OS << ", ";
-    OS << WebAssembly::TypeToString(Ty);
+    OS << WebAssembly::typeToString(Ty);
   }
 }
 
@@ -99,10 +98,10 @@ void WebAssemblyTargetAsmStreamer::emitFunctionType(const MCSymbolWasm *Sym) {
 
 void WebAssemblyTargetAsmStreamer::emitGlobalType(const MCSymbolWasm *Sym) {
   assert(Sym->isGlobal());
-  OS << "\t.globaltype\t" << Sym->getName() << ", " <<
-        WebAssembly::TypeToString(
-          static_cast<wasm::ValType>(Sym->getGlobalType().Type)) <<
-        '\n';
+  OS << "\t.globaltype\t" << Sym->getName() << ", "
+     << WebAssembly::typeToString(
+            static_cast<wasm::ValType>(Sym->getGlobalType().Type))
+     << '\n';
 }
 
 void WebAssemblyTargetAsmStreamer::emitEventType(const MCSymbolWasm *Sym) {
@@ -113,8 +112,15 @@ void WebAssemblyTargetAsmStreamer::emitEventType(const MCSymbolWasm *Sym) {
 }
 
 void WebAssemblyTargetAsmStreamer::emitImportModule(const MCSymbolWasm *Sym,
-                                                    StringRef ModuleName) {
-  OS << "\t.import_module\t" << Sym->getName() << ", " << ModuleName << '\n';
+                                                    StringRef ImportModule) {
+  OS << "\t.import_module\t" << Sym->getName() << ", "
+                             << ImportModule << '\n';
+}
+
+void WebAssemblyTargetAsmStreamer::emitImportName(const MCSymbolWasm *Sym,
+                                                  StringRef ImportName) {
+  OS << "\t.import_name\t" << Sym->getName() << ", "
+                           << ImportName << '\n';
 }
 
 void WebAssemblyTargetAsmStreamer::emitIndIdx(const MCExpr *Value) {
diff --git a/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyTargetStreamer.h b/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyTargetStreamer.h
index 3073938118b458be1e3e0c9e9ecffd6b91ed47ce..5ea62b179d22f6ac9312e25ec951afeccf3c2f9d 100644
--- a/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyTargetStreamer.h
+++ b/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyTargetStreamer.h
@@ -1,9 +1,8 @@
 //==-- WebAssemblyTargetStreamer.h - WebAssembly Target Streamer -*- C++ -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
@@ -45,7 +44,10 @@ public:
   virtual void emitEventType(const MCSymbolWasm *Sym) = 0;
   /// .import_module
   virtual void emitImportModule(const MCSymbolWasm *Sym,
-                                StringRef ModuleName) = 0;
+                                StringRef ImportModule) = 0;
+  /// .import_name
+  virtual void emitImportName(const MCSymbolWasm *Sym,
+                              StringRef ImportName) = 0;
 
 protected:
   void emitValueType(wasm::ValType Type);
@@ -67,7 +69,8 @@ public:
   void emitIndIdx(const MCExpr *Value) override;
   void emitGlobalType(const MCSymbolWasm *Sym) override;
   void emitEventType(const MCSymbolWasm *Sym) override;
-  void emitImportModule(const MCSymbolWasm *Sym, StringRef ModuleName) override;
+  void emitImportModule(const MCSymbolWasm *Sym, StringRef ImportModule) override;
+  void emitImportName(const MCSymbolWasm *Sym, StringRef ImportName) override;
 };
 
 /// This part is for Wasm object output
@@ -82,7 +85,9 @@ public:
   void emitGlobalType(const MCSymbolWasm *Sym) override {}
   void emitEventType(const MCSymbolWasm *Sym) override {}
   void emitImportModule(const MCSymbolWasm *Sym,
-                        StringRef ModuleName) override {}
+                        StringRef ImportModule) override {}
+  void emitImportName(const MCSymbolWasm *Sym,
+                      StringRef ImportName) override {}
 };
 
 /// This part is for null output
@@ -98,6 +103,7 @@ public:
   void emitGlobalType(const MCSymbolWasm *) override {}
   void emitEventType(const MCSymbolWasm *) override {}
   void emitImportModule(const MCSymbolWasm *, StringRef) override {}
+  void emitImportName(const MCSymbolWasm *, StringRef) override {}
 };
 
 } // end namespace llvm
diff --git a/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyWasmObjectWriter.cpp b/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyWasmObjectWriter.cpp
index 763e30be8e02002f241450e1ccb8e967a37d9cb1..2c97aadd7f9edb499fb7345887d29505395a6d30 100644
--- a/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyWasmObjectWriter.cpp
+++ b/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyWasmObjectWriter.cpp
@@ -1,9 +1,8 @@
 //===-- WebAssemblyWasmObjectWriter.cpp - WebAssembly Wasm Writer ---------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
@@ -44,25 +43,25 @@ WebAssemblyWasmObjectWriter::WebAssemblyWasmObjectWriter(bool Is64Bit)
     : MCWasmObjectTargetWriter(Is64Bit) {}
 
 // Test whether the given expression computes a function address.
-static bool IsFunctionExpr(const MCExpr *Expr) {
+static bool isFunctionExpr(const MCExpr *Expr) {
   if (auto SyExp = dyn_cast<MCSymbolRefExpr>(Expr))
     return cast<MCSymbolWasm>(SyExp->getSymbol()).isFunction();
 
   if (auto BinOp = dyn_cast<MCBinaryExpr>(Expr))
-    return IsFunctionExpr(BinOp->getLHS()) != IsFunctionExpr(BinOp->getRHS());
+    return isFunctionExpr(BinOp->getLHS()) != isFunctionExpr(BinOp->getRHS());
 
   if (auto UnOp = dyn_cast<MCUnaryExpr>(Expr))
-    return IsFunctionExpr(UnOp->getSubExpr());
+    return isFunctionExpr(UnOp->getSubExpr());
 
   return false;
 }
 
-static bool IsFunctionType(const MCValue &Target) {
+static bool isFunctionType(const MCValue &Target) {
   const MCSymbolRefExpr *RefA = Target.getSymA();
   return RefA && RefA->getKind() == MCSymbolRefExpr::VK_WebAssembly_TYPEINDEX;
 }
 
-static const MCSection *GetFixupSection(const MCExpr *Expr) {
+static const MCSection *getFixupSection(const MCExpr *Expr) {
   if (auto SyExp = dyn_cast<MCSymbolRefExpr>(Expr)) {
     if (SyExp->getSymbol().isInSection())
       return &SyExp->getSymbol().getSection();
@@ -70,23 +69,23 @@ static const MCSection *GetFixupSection(const MCExpr *Expr) {
   }
 
   if (auto BinOp = dyn_cast<MCBinaryExpr>(Expr)) {
-    auto SectionLHS = GetFixupSection(BinOp->getLHS());
-    auto SectionRHS = GetFixupSection(BinOp->getRHS());
+    auto SectionLHS = getFixupSection(BinOp->getLHS());
+    auto SectionRHS = getFixupSection(BinOp->getRHS());
     return SectionLHS == SectionRHS ? nullptr : SectionLHS;
   }
 
   if (auto UnOp = dyn_cast<MCUnaryExpr>(Expr))
-    return GetFixupSection(UnOp->getSubExpr());
+    return getFixupSection(UnOp->getSubExpr());
 
   return nullptr;
 }
 
-static bool IsGlobalType(const MCValue &Target) {
+static bool isGlobalType(const MCValue &Target) {
   const MCSymbolRefExpr *RefA = Target.getSymA();
   return RefA && RefA->getKind() == MCSymbolRefExpr::VK_WebAssembly_GLOBAL;
 }
 
-static bool IsEventType(const MCValue &Target) {
+static bool isEventType(const MCValue &Target) {
   const MCSymbolRefExpr *RefA = Target.getSymA();
   return RefA && RefA->getKind() == MCSymbolRefExpr::VK_WebAssembly_EVENT;
 }
@@ -95,36 +94,36 @@ unsigned WebAssemblyWasmObjectWriter::getRelocType(const MCValue &Target,
                                                    const MCFixup &Fixup) const {
   // WebAssembly functions are not allocated in the data address space. To
   // resolve a pointer to a function, we must use a special relocation type.
-  bool IsFunction = IsFunctionExpr(Fixup.getValue());
+  bool IsFunction = isFunctionExpr(Fixup.getValue());
 
   switch (unsigned(Fixup.getKind())) {
   case WebAssembly::fixup_code_sleb128_i32:
     if (IsFunction)
-      return wasm::R_WEBASSEMBLY_TABLE_INDEX_SLEB;
-    return wasm::R_WEBASSEMBLY_MEMORY_ADDR_SLEB;
+      return wasm::R_WASM_TABLE_INDEX_SLEB;
+    return wasm::R_WASM_MEMORY_ADDR_SLEB;
   case WebAssembly::fixup_code_sleb128_i64:
     llvm_unreachable("fixup_sleb128_i64 not implemented yet");
   case WebAssembly::fixup_code_uleb128_i32:
-    if (IsGlobalType(Target))
-      return wasm::R_WEBASSEMBLY_GLOBAL_INDEX_LEB;
-    if (IsFunctionType(Target))
-      return wasm::R_WEBASSEMBLY_TYPE_INDEX_LEB;
+    if (isGlobalType(Target))
+      return wasm::R_WASM_GLOBAL_INDEX_LEB;
+    if (isFunctionType(Target))
+      return wasm::R_WASM_TYPE_INDEX_LEB;
     if (IsFunction)
-      return wasm::R_WEBASSEMBLY_FUNCTION_INDEX_LEB;
-    if (IsEventType(Target))
-      return wasm::R_WEBASSEMBLY_EVENT_INDEX_LEB;
-    return wasm::R_WEBASSEMBLY_MEMORY_ADDR_LEB;
+      return wasm::R_WASM_FUNCTION_INDEX_LEB;
+    if (isEventType(Target))
+      return wasm::R_WASM_EVENT_INDEX_LEB;
+    return wasm::R_WASM_MEMORY_ADDR_LEB;
   case FK_Data_4:
     if (IsFunction)
-      return wasm::R_WEBASSEMBLY_TABLE_INDEX_I32;
+      return wasm::R_WASM_TABLE_INDEX_I32;
     if (auto Section = static_cast<const MCSectionWasm *>(
-            GetFixupSection(Fixup.getValue()))) {
+            getFixupSection(Fixup.getValue()))) {
       if (Section->getKind().isText())
-        return wasm::R_WEBASSEMBLY_FUNCTION_OFFSET_I32;
+        return wasm::R_WASM_FUNCTION_OFFSET_I32;
       else if (!Section->isWasmData())
-        return wasm::R_WEBASSEMBLY_SECTION_OFFSET_I32;
+        return wasm::R_WASM_SECTION_OFFSET_I32;
     }
-    return wasm::R_WEBASSEMBLY_MEMORY_ADDR_I32;
+    return wasm::R_WASM_MEMORY_ADDR_I32;
   case FK_Data_8:
     llvm_unreachable("FK_Data_8 not implemented yet");
   default:
diff --git a/lib/Target/WebAssembly/README.txt b/lib/Target/WebAssembly/README.txt
index ef0099f07efb71d0be18c5c06c3e6f760e2598a1..a154b4bf7ea85891164063101063d2af89123fa5 100644
--- a/lib/Target/WebAssembly/README.txt
+++ b/lib/Target/WebAssembly/README.txt
@@ -94,10 +94,10 @@ WebAssemblyTargetLowering.
 //===---------------------------------------------------------------------===//
 
 Instead of the OptimizeReturned pass, which should consider preserving the
-"returned" attribute through to MachineInstrs and extending the StoreResults
-pass to do this optimization on calls too. That would also let the
-WebAssemblyPeephole pass clean up dead defs for such calls, as it does for
-stores.
+"returned" attribute through to MachineInstrs and extending the
+MemIntrinsicResults pass to do this optimization on calls too. That would also
+let the WebAssemblyPeephole pass clean up dead defs for such calls, as it does
+for stores.
 
 //===---------------------------------------------------------------------===//
 
@@ -120,8 +120,8 @@ code like this:
 It could be done with a smaller encoding like this:
 
     i32.const   $push5=, 0
-    tee_local   $push6=, $4=, $pop5
-    copy_local  $3=, $pop6
+    local.tee   $push6=, $4=, $pop5
+    local.copy  $3=, $pop6
 
 //===---------------------------------------------------------------------===//
 
@@ -180,11 +180,11 @@ floating-point constants.
 //===---------------------------------------------------------------------===//
 
 The function @dynamic_alloca_redzone in test/CodeGen/WebAssembly/userstack.ll
-ends up with a tee_local in its prolog which has an unused result, requiring
+ends up with a local.tee in its prolog which has an unused result, requiring
 an extra drop:
 
-    get_global  $push8=, 0
-    tee_local   $push9=, 1, $pop8
+    global.get  $push8=, 0
+    local.tee   $push9=, 1, $pop8
     drop        $pop9
     [...]
 
diff --git a/lib/Target/WebAssembly/TargetInfo/LLVMBuild.txt b/lib/Target/WebAssembly/TargetInfo/LLVMBuild.txt
index f4da9239bbe8a0f8e852af62fac0f01ca6e51145..ff520740ce26e1a033f91e26a01aded1a76bbdd2 100644
--- a/lib/Target/WebAssembly/TargetInfo/LLVMBuild.txt
+++ b/lib/Target/WebAssembly/TargetInfo/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/Target/WebAssembly/TargetInfo/LLVMBuild.txt --------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Target/WebAssembly/TargetInfo/WebAssemblyTargetInfo.cpp b/lib/Target/WebAssembly/TargetInfo/WebAssemblyTargetInfo.cpp
index f7a417c0ed493e5827aa04ae21d2c2242e796e24..416e61d7989ddd4b11489581adcf3c408265c49c 100644
--- a/lib/Target/WebAssembly/TargetInfo/WebAssemblyTargetInfo.cpp
+++ b/lib/Target/WebAssembly/TargetInfo/WebAssemblyTargetInfo.cpp
@@ -1,9 +1,8 @@
 //===-- WebAssemblyTargetInfo.cpp - WebAssembly Target Implementation -----===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
diff --git a/lib/Target/WebAssembly/WebAssembly.h b/lib/Target/WebAssembly/WebAssembly.h
index 87975cad02a856ed7ee142233257986d402ed3d8..2a7efe1176f3fd8bd157deaba3df54f1cc514612 100644
--- a/lib/Target/WebAssembly/WebAssembly.h
+++ b/lib/Target/WebAssembly/WebAssembly.h
@@ -1,9 +1,8 @@
 //===-- WebAssembly.h - Top-level interface for WebAssembly  ----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
@@ -39,11 +38,10 @@ FunctionPass *createWebAssemblyArgumentMove();
 FunctionPass *createWebAssemblySetP2AlignOperands();
 
 // Late passes.
-FunctionPass *createWebAssemblyEHRestoreStackPointer();
 FunctionPass *createWebAssemblyReplacePhysRegs();
 FunctionPass *createWebAssemblyPrepareForLiveIntervals();
 FunctionPass *createWebAssemblyOptimizeLiveIntervals();
-FunctionPass *createWebAssemblyStoreResults();
+FunctionPass *createWebAssemblyMemIntrinsicResults();
 FunctionPass *createWebAssemblyRegStackify();
 FunctionPass *createWebAssemblyRegColoring();
 FunctionPass *createWebAssemblyExplicitLocals();
@@ -64,11 +62,10 @@ void initializeFixFunctionBitcastsPass(PassRegistry &);
 void initializeOptimizeReturnedPass(PassRegistry &);
 void initializeWebAssemblyArgumentMovePass(PassRegistry &);
 void initializeWebAssemblySetP2AlignOperandsPass(PassRegistry &);
-void initializeWebAssemblyEHRestoreStackPointerPass(PassRegistry &);
 void initializeWebAssemblyReplacePhysRegsPass(PassRegistry &);
 void initializeWebAssemblyPrepareForLiveIntervalsPass(PassRegistry &);
 void initializeWebAssemblyOptimizeLiveIntervalsPass(PassRegistry &);
-void initializeWebAssemblyStoreResultsPass(PassRegistry &);
+void initializeWebAssemblyMemIntrinsicResultsPass(PassRegistry &);
 void initializeWebAssemblyRegStackifyPass(PassRegistry &);
 void initializeWebAssemblyRegColoringPass(PassRegistry &);
 void initializeWebAssemblyExplicitLocalsPass(PassRegistry &);
diff --git a/lib/Target/WebAssembly/WebAssembly.td b/lib/Target/WebAssembly/WebAssembly.td
index ec9dbffde7f2f6f4ce0b08c900c791a15ec6156f..230c1208e54d703ad4428b80ccd9e16efe84b2eb 100644
--- a/lib/Target/WebAssembly/WebAssembly.td
+++ b/lib/Target/WebAssembly/WebAssembly.td
@@ -1,9 +1,8 @@
 //- WebAssembly.td - Describe the WebAssembly Target Machine --*- tablegen -*-//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
@@ -23,8 +22,15 @@ include "llvm/Target/Target.td"
 // WebAssembly Subtarget features.
 //===----------------------------------------------------------------------===//
 
-def FeatureSIMD128 : SubtargetFeature<"simd128", "HasSIMD128", "true",
+def FeatureSIMD128 : SubtargetFeature<"simd128", "SIMDLevel", "SIMD128",
                                       "Enable 128-bit SIMD">;
+
+def FeatureUnimplementedSIMD128 :
+      SubtargetFeature<"unimplemented-simd128",
+                       "SIMDLevel", "UnimplementedSIMD128",
+                       "Enable 128-bit SIMD not yet implemented in engines",
+                       [FeatureSIMD128]>;
+
 def FeatureAtomics : SubtargetFeature<"atomics", "HasAtomics", "true",
                                       "Enable Atomics">;
 def FeatureNontrappingFPToInt :
@@ -41,6 +47,10 @@ def FeatureExceptionHandling :
       SubtargetFeature<"exception-handling", "HasExceptionHandling", "true",
                        "Enable Wasm exception handling">;
 
+def FeatureBulkMemory :
+      SubtargetFeature<"bulk-memory", "HasBulkMemory", "true",
+                       "Enable bulk memory operations">;
+
 //===----------------------------------------------------------------------===//
 // Architectures.
 //===----------------------------------------------------------------------===//
diff --git a/lib/Target/WebAssembly/WebAssemblyAddMissingPrototypes.cpp b/lib/Target/WebAssembly/WebAssemblyAddMissingPrototypes.cpp
index 3f17160964b6e537f96c664ede240be6802940d4..97ca0615d77d33d5b7613d7eca65e0f5415c427a 100644
--- a/lib/Target/WebAssembly/WebAssemblyAddMissingPrototypes.cpp
+++ b/lib/Target/WebAssembly/WebAssemblyAddMissingPrototypes.cpp
@@ -1,9 +1,8 @@
 //===-- WebAssemblyAddMissingPrototypes.cpp - Fix prototypeless functions -===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
@@ -60,7 +59,7 @@ ModulePass *llvm::createWebAssemblyAddMissingPrototypes() {
 }
 
 bool WebAssemblyAddMissingPrototypes::runOnModule(Module &M) {
-  LLVM_DEBUG(dbgs() << "runnning AddMissingPrototypes\n");
+  LLVM_DEBUG(dbgs() << "********** Add Missing Prototypes **********\n");
 
   std::vector<std::pair<Function *, Function *>> Replacements;
 
@@ -96,9 +95,10 @@ bool WebAssemblyAddMissingPrototypes::runOnModule(Module &M) {
           if (!NewType) {
             // Create a new function with the correct type
             NewType = DestType;
-            NewF = Function::Create(NewType, F.getLinkage(), F.getName());
+            NewF = Function::Create(NewType, F.getLinkage(), F.getName() + ".fixed_sig");
             NewF->setAttributes(F.getAttributes());
             NewF->removeFnAttr("no-prototype");
+            Replacements.emplace_back(&F, NewF);
           } else {
             if (NewType != DestType) {
               report_fatal_error("Prototypeless function used with "
@@ -116,45 +116,17 @@ bool WebAssemblyAddMissingPrototypes::runOnModule(Module &M) {
                         F.getName() + "\n");
       continue;
     }
-
-    SmallVector<Instruction *, 4> DeadInsts;
-
-    for (Use &US : F.uses()) {
-      User *U = US.getUser();
-      if (auto *BC = dyn_cast<BitCastOperator>(U)) {
-        if (auto *Inst = dyn_cast<BitCastInst>(U)) {
-          // Replace with a new bitcast
-          IRBuilder<> Builder(Inst);
-          Value *NewCast = Builder.CreatePointerCast(NewF, BC->getDestTy());
-          Inst->replaceAllUsesWith(NewCast);
-          DeadInsts.push_back(Inst);
-        } else if (auto *Const = dyn_cast<ConstantExpr>(U)) {
-          Constant *NewConst =
-              ConstantExpr::getPointerCast(NewF, BC->getDestTy());
-          Const->replaceAllUsesWith(NewConst);
-        } else {
-          dbgs() << *U->getType() << "\n";
-#ifndef NDEBUG
-          U->dump();
-#endif
-          report_fatal_error("unexpected use of prototypeless function: " +
-                             F.getName() + "\n");
-        }
-      }
-    }
-
-    for (auto I : DeadInsts)
-      I->eraseFromParent();
-    Replacements.emplace_back(&F, NewF);
   }
 
-
-  // Finally replace the old function declarations with the new ones
   for (auto &Pair : Replacements) {
-    Function *Old = Pair.first;
-    Function *New = Pair.second;
-    Old->eraseFromParent();
-    M.getFunctionList().push_back(New);
+    Function *OldF = Pair.first;
+    Function *NewF = Pair.second;
+    std::string Name = OldF->getName();
+    M.getFunctionList().push_back(NewF);
+    OldF->replaceAllUsesWith(
+      ConstantExpr::getPointerBitCastOrAddrSpaceCast(NewF, OldF->getType()));
+    OldF->eraseFromParent();
+    NewF->setName(Name);
   }
 
   return !Replacements.empty();
diff --git a/lib/Target/WebAssembly/WebAssemblyArgumentMove.cpp b/lib/Target/WebAssembly/WebAssemblyArgumentMove.cpp
index 7c8a631cde8a8323c3b5729d6e14da7af9087bb8..3dcf34aeb280ffe4355d0240c8a2431dd0f64344 100644
--- a/lib/Target/WebAssembly/WebAssemblyArgumentMove.cpp
+++ b/lib/Target/WebAssembly/WebAssemblyArgumentMove.cpp
@@ -1,9 +1,8 @@
 //===-- WebAssemblyArgumentMove.cpp - Argument instruction moving ---------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
diff --git a/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp b/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp
index c4f03dfa7f9e451ed81859023d2c64cde7bb15f7..563c7cc203d49c68cecfde1c9c216bdcbf0d9955 100644
--- a/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp
+++ b/lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp
@@ -1,9 +1,8 @@
 //===-- WebAssemblyAsmPrinter.cpp - WebAssembly LLVM assembly writer ------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
@@ -22,6 +21,7 @@
 #include "WebAssemblyMCInstLower.h"
 #include "WebAssemblyMachineFunctionInfo.h"
 #include "WebAssemblyRegisterInfo.h"
+#include "llvm/ADT/SmallSet.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/CodeGen/Analysis.h"
 #include "llvm/CodeGen/AsmPrinter.h"
@@ -29,6 +29,7 @@
 #include "llvm/CodeGen/MachineInstr.h"
 #include "llvm/CodeGen/MachineModuleInfoImpls.h"
 #include "llvm/IR/DataLayout.h"
+#include "llvm/IR/DebugInfoMetadata.h"
 #include "llvm/IR/GlobalVariable.h"
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCSectionWasm.h"
@@ -42,6 +43,8 @@ using namespace llvm;
 
 #define DEBUG_TYPE "asm-printer"
 
+extern cl::opt<bool> WasmKeepRegisters;
+
 //===----------------------------------------------------------------------===//
 // Helpers.
 //===----------------------------------------------------------------------===//
@@ -92,11 +95,11 @@ void WebAssemblyAsmPrinter::EmitEndOfAsmFile(Module &M) {
     if (F.isDeclarationForLinker() && !F.isIntrinsic()) {
       SmallVector<MVT, 4> Results;
       SmallVector<MVT, 4> Params;
-      ComputeSignatureVTs(F.getFunctionType(), F, TM, Params, Results);
+      computeSignatureVTs(F.getFunctionType(), F, TM, Params, Results);
       auto *Sym = cast<MCSymbolWasm>(getSymbol(&F));
       Sym->setType(wasm::WASM_SYMBOL_TYPE_FUNCTION);
       if (!Sym->getSignature()) {
-        auto Signature = SignatureFromMVTs(Results, Params);
+        auto Signature = signatureFromMVTs(Results, Params);
         Sym->setSignature(Signature.get());
         addSignature(std::move(Signature));
       }
@@ -111,9 +114,16 @@ void WebAssemblyAsmPrinter::EmitEndOfAsmFile(Module &M) {
           F.hasFnAttribute("wasm-import-module")) {
         StringRef Name =
             F.getFnAttribute("wasm-import-module").getValueAsString();
-        Sym->setModuleName(Name);
+        Sym->setImportModule(Name);
         getTargetStreamer()->emitImportModule(Sym, Name);
       }
+      if (TM.getTargetTriple().isOSBinFormatWasm() &&
+          F.hasFnAttribute("wasm-import-name")) {
+        StringRef Name =
+            F.getFnAttribute("wasm-import-name").getValueAsString();
+        Sym->setImportName(Name);
+        getTargetStreamer()->emitImportName(Sym, Name);
+      }
     }
   }
 
@@ -129,7 +139,7 @@ void WebAssemblyAsmPrinter::EmitEndOfAsmFile(Module &M) {
 
   if (const NamedMDNode *Named = M.getNamedMetadata("wasm.custom_sections")) {
     for (const Metadata *MD : Named->operands()) {
-      const MDTuple *Tuple = dyn_cast<MDTuple>(MD);
+      const auto *Tuple = dyn_cast<MDTuple>(MD);
       if (!Tuple || Tuple->getNumOperands() != 2)
         continue;
       const MDString *Name = dyn_cast<MDString>(Tuple->getOperand(0));
@@ -139,13 +149,66 @@ void WebAssemblyAsmPrinter::EmitEndOfAsmFile(Module &M) {
 
       OutStreamer->PushSection();
       std::string SectionName = (".custom_section." + Name->getString()).str();
-      MCSectionWasm *mySection =
+      MCSectionWasm *MySection =
           OutContext.getWasmSection(SectionName, SectionKind::getMetadata());
-      OutStreamer->SwitchSection(mySection);
+      OutStreamer->SwitchSection(MySection);
       OutStreamer->EmitBytes(Contents->getString());
       OutStreamer->PopSection();
     }
   }
+
+  EmitProducerInfo(M);
+}
+
+void WebAssemblyAsmPrinter::EmitProducerInfo(Module &M) {
+  llvm::SmallVector<std::pair<std::string, std::string>, 4> Languages;
+  if (const NamedMDNode *Debug = M.getNamedMetadata("llvm.dbg.cu")) {
+    llvm::SmallSet<StringRef, 4> SeenLanguages;
+    for (size_t I = 0, E = Debug->getNumOperands(); I < E; ++I) {
+      const auto *CU = cast<DICompileUnit>(Debug->getOperand(I));
+      StringRef Language = dwarf::LanguageString(CU->getSourceLanguage());
+      Language.consume_front("DW_LANG_");
+      if (SeenLanguages.insert(Language).second)
+        Languages.emplace_back(Language.str(), "");
+    }
+  }
+
+  llvm::SmallVector<std::pair<std::string, std::string>, 4> Tools;
+  if (const NamedMDNode *Ident = M.getNamedMetadata("llvm.ident")) {
+    llvm::SmallSet<StringRef, 4> SeenTools;
+    for (size_t I = 0, E = Ident->getNumOperands(); I < E; ++I) {
+      const auto *S = cast<MDString>(Ident->getOperand(I)->getOperand(0));
+      std::pair<StringRef, StringRef> Field = S->getString().split("version");
+      StringRef Name = Field.first.trim();
+      StringRef Version = Field.second.trim();
+      if (SeenTools.insert(Name).second)
+        Tools.emplace_back(Name.str(), Version.str());
+    }
+  }
+
+  int FieldCount = int(!Languages.empty()) + int(!Tools.empty());
+  if (FieldCount != 0) {
+    MCSectionWasm *Producers = OutContext.getWasmSection(
+        ".custom_section.producers", SectionKind::getMetadata());
+    OutStreamer->PushSection();
+    OutStreamer->SwitchSection(Producers);
+    OutStreamer->EmitULEB128IntValue(FieldCount);
+    for (auto &Producers : {std::make_pair("language", &Languages),
+            std::make_pair("processed-by", &Tools)}) {
+      if (Producers.second->empty())
+        continue;
+      OutStreamer->EmitULEB128IntValue(strlen(Producers.first));
+      OutStreamer->EmitBytes(Producers.first);
+      OutStreamer->EmitULEB128IntValue(Producers.second->size());
+      for (auto &Producer : *Producers.second) {
+        OutStreamer->EmitULEB128IntValue(Producer.first.size());
+        OutStreamer->EmitBytes(Producer.first);
+        OutStreamer->EmitULEB128IntValue(Producer.second.size());
+        OutStreamer->EmitBytes(Producer.second);
+      }
+    }
+    OutStreamer->PopSection();
+  }
 }
 
 void WebAssemblyAsmPrinter::EmitConstantPool() {
@@ -161,8 +224,8 @@ void WebAssemblyAsmPrinter::EmitFunctionBodyStart() {
   const Function &F = MF->getFunction();
   SmallVector<MVT, 1> ResultVTs;
   SmallVector<MVT, 4> ParamVTs;
-  ComputeSignatureVTs(F.getFunctionType(), F, TM, ParamVTs, ResultVTs);
-  auto Signature = SignatureFromMVTs(ResultVTs, ParamVTs);
+  computeSignatureVTs(F.getFunctionType(), F, TM, ParamVTs, ResultVTs);
+  auto Signature = signatureFromMVTs(ResultVTs, ParamVTs);
   auto *WasmSym = cast<MCSymbolWasm>(CurrentFnSym);
   WasmSym->setSignature(Signature.get());
   addSignature(std::move(Signature));
@@ -180,7 +243,7 @@ void WebAssemblyAsmPrinter::EmitFunctionBodyStart() {
   }
 
   SmallVector<wasm::ValType, 16> Locals;
-  ValTypesFromMVTs(MFI->getLocals(), Locals);
+  valTypesFromMVTs(MFI->getLocals(), Locals);
   getTargetStreamer()->emitLocal(Locals);
 
   AsmPrinter::EmitFunctionBodyStart();
@@ -250,10 +313,18 @@ void WebAssemblyAsmPrinter::EmitInstruction(const MachineInstr *MI) {
       OutStreamer->AddBlankLine();
     }
     break;
+  case WebAssembly::EXTRACT_EXCEPTION_I32:
+  case WebAssembly::EXTRACT_EXCEPTION_I32_S:
+    // These are pseudo instructions that simulates popping values from stack.
+    // We print these only when we have -wasm-keep-registers on for assembly
+    // readability.
+    if (!WasmKeepRegisters)
+      break;
+    LLVM_FALLTHROUGH;
   default: {
     WebAssemblyMCInstLower MCInstLowering(OutContext, *this);
     MCInst TmpInst;
-    MCInstLowering.Lower(MI, TmpInst);
+    MCInstLowering.lower(MI, TmpInst);
     EmitToStreamer(*OutStreamer, TmpInst);
     break;
   }
@@ -261,7 +332,7 @@ void WebAssemblyAsmPrinter::EmitInstruction(const MachineInstr *MI) {
 }
 
 const MCExpr *WebAssemblyAsmPrinter::lowerConstant(const Constant *CV) {
-  if (const GlobalValue *GV = dyn_cast<GlobalValue>(CV))
+  if (const auto *GV = dyn_cast<GlobalValue>(CV))
     if (GV->getValueType()->isFunctionTy()) {
       return MCSymbolRefExpr::create(
           getSymbol(GV), MCSymbolRefExpr::VK_WebAssembly_FUNCTION, OutContext);
diff --git a/lib/Target/WebAssembly/WebAssemblyAsmPrinter.h b/lib/Target/WebAssembly/WebAssemblyAsmPrinter.h
index f6cb5610bad3c205186fae8417fc2a8b14da6c1e..c4300cf7916f67318d64ce166583016e68b25dc9 100644
--- a/lib/Target/WebAssembly/WebAssemblyAsmPrinter.h
+++ b/lib/Target/WebAssembly/WebAssemblyAsmPrinter.h
@@ -1,9 +1,8 @@
 // WebAssemblyAsmPrinter.h - WebAssembly implementation of AsmPrinter-*- C++ -*-
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -59,6 +58,7 @@ public:
   //===------------------------------------------------------------------===//
 
   void EmitEndOfAsmFile(Module &M) override;
+  void EmitProducerInfo(Module &M);
   void EmitJumpTableInfo() override;
   void EmitConstantPool() override;
   void EmitFunctionBodyStart() override;
diff --git a/lib/Target/WebAssembly/WebAssemblyCFGSort.cpp b/lib/Target/WebAssembly/WebAssemblyCFGSort.cpp
index fc827e9d57806d36cffea1c6ad042d48624eda81..a67a3af16415bc89b0fcc0bc3994ea977086d653 100644
--- a/lib/Target/WebAssembly/WebAssemblyCFGSort.cpp
+++ b/lib/Target/WebAssembly/WebAssemblyCFGSort.cpp
@@ -1,9 +1,8 @@
 //===-- WebAssemblyCFGSort.cpp - CFG Sorting ------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
@@ -133,7 +132,7 @@ FunctionPass *llvm::createWebAssemblyCFGSort() {
   return new WebAssemblyCFGSort();
 }
 
-static void MaybeUpdateTerminator(MachineBasicBlock *MBB) {
+static void maybeUpdateTerminator(MachineBasicBlock *MBB) {
 #ifndef NDEBUG
   bool AnyBarrier = false;
 #endif
@@ -228,7 +227,7 @@ struct Entry {
 /// interrupted by blocks not dominated by their header.
 /// TODO: There are many opportunities for improving the heuristics here.
 /// Explore them.
-static void SortBlocks(MachineFunction &MF, const MachineLoopInfo &MLI,
+static void sortBlocks(MachineFunction &MF, const MachineLoopInfo &MLI,
                        const WebAssemblyExceptionInfo &WEI,
                        const MachineDominatorTree &MDT) {
   // Prepare for a topological sort: Record the number of predecessors each
@@ -320,7 +319,7 @@ static void SortBlocks(MachineFunction &MF, const MachineLoopInfo &MLI,
     if (!Next) {
       // If there are no more blocks to process, we're done.
       if (Ready.empty()) {
-        MaybeUpdateTerminator(MBB);
+        maybeUpdateTerminator(MBB);
         break;
       }
       for (;;) {
@@ -338,7 +337,7 @@ static void SortBlocks(MachineFunction &MF, const MachineLoopInfo &MLI,
     }
     // Move the next block into place and iterate.
     Next->moveAfter(MBB);
-    MaybeUpdateTerminator(MBB);
+    maybeUpdateTerminator(MBB);
     MBB = Next;
   }
   assert(Entries.empty() && "Active sort region list not finished");
@@ -404,7 +403,7 @@ bool WebAssemblyCFGSort::runOnMachineFunction(MachineFunction &MF) {
   MF.getRegInfo().invalidateLiveness();
 
   // Sort the blocks, with contiguous sort regions.
-  SortBlocks(MF, MLI, WEI, MDT);
+  sortBlocks(MF, MLI, WEI, MDT);
 
   return true;
 }
diff --git a/lib/Target/WebAssembly/WebAssemblyCFGStackify.cpp b/lib/Target/WebAssembly/WebAssemblyCFGStackify.cpp
index f8f5f4040c86a3e9427c9ff052e672049fadae07..49057ded0a684595a611ce2b4a8f27554fbe9372 100644
--- a/lib/Target/WebAssembly/WebAssemblyCFGStackify.cpp
+++ b/lib/Target/WebAssembly/WebAssemblyCFGStackify.cpp
@@ -1,9 +1,8 @@
 //===-- WebAssemblyCFGStackify.cpp - CFG Stackification -------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
@@ -38,6 +37,7 @@
 #include "llvm/MC/MCAsmInfo.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/raw_ostream.h"
+#include <cstring>
 using namespace llvm;
 
 #define DEBUG_TYPE "wasm-cfg-stackify"
@@ -108,14 +108,12 @@ FunctionPass *llvm::createWebAssemblyCFGStackify() {
 /// code) for a branch instruction to both branch to a block and fallthrough
 /// to it, so we check the actual branch operands to see if there are any
 /// explicit mentions.
-static bool ExplicitlyBranchesTo(MachineBasicBlock *Pred,
+static bool explicitlyBranchesTo(MachineBasicBlock *Pred,
                                  MachineBasicBlock *MBB) {
   for (MachineInstr &MI : Pred->terminators())
-    // Even if a rethrow takes a BB argument, it is not a branch
-    if (!WebAssembly::isRethrow(MI))
-      for (MachineOperand &MO : MI.explicit_operands())
-        if (MO.isMBB() && MO.getMBB() == MBB)
-          return true;
+    for (MachineOperand &MO : MI.explicit_operands())
+      if (MO.isMBB() && MO.getMBB() == MBB)
+        return true;
   return false;
 }
 
@@ -125,7 +123,7 @@ static bool ExplicitlyBranchesTo(MachineBasicBlock *Pred,
 // ones that should go after the marker. In this function, AfterSet is only
 // used for sanity checking.
 static MachineBasicBlock::iterator
-GetEarliestInsertPos(MachineBasicBlock *MBB,
+getEarliestInsertPos(MachineBasicBlock *MBB,
                      const SmallPtrSet<const MachineInstr *, 4> &BeforeSet,
                      const SmallPtrSet<const MachineInstr *, 4> &AfterSet) {
   auto InsertPos = MBB->end();
@@ -149,7 +147,7 @@ GetEarliestInsertPos(MachineBasicBlock *MBB,
 // ones that should go after the marker. In this function, BeforeSet is only
 // used for sanity checking.
 static MachineBasicBlock::iterator
-GetLatestInsertPos(MachineBasicBlock *MBB,
+getLatestInsertPos(MachineBasicBlock *MBB,
                    const SmallPtrSet<const MachineInstr *, 4> &BeforeSet,
                    const SmallPtrSet<const MachineInstr *, 4> &AfterSet) {
   auto InsertPos = MBB->begin();
@@ -218,12 +216,20 @@ void WebAssemblyCFGStackify::placeBlockMarker(MachineBasicBlock &MBB) {
   // which reduces overall stack height.
   MachineBasicBlock *Header = nullptr;
   bool IsBranchedTo = false;
+  bool IsBrOnExn = false;
+  MachineInstr *BrOnExn = nullptr;
   int MBBNumber = MBB.getNumber();
   for (MachineBasicBlock *Pred : MBB.predecessors()) {
     if (Pred->getNumber() < MBBNumber) {
       Header = Header ? MDT.findNearestCommonDominator(Header, Pred) : Pred;
-      if (ExplicitlyBranchesTo(Pred, &MBB))
+      if (explicitlyBranchesTo(Pred, &MBB)) {
         IsBranchedTo = true;
+        if (Pred->getFirstTerminator()->getOpcode() == WebAssembly::BR_ON_EXN) {
+          IsBrOnExn = true;
+          assert(!BrOnExn && "There should be only one br_on_exn per block");
+          BrOnExn = &*Pred->getFirstTerminator();
+        }
+      }
     }
   }
   if (!Header)
@@ -300,11 +306,27 @@ void WebAssemblyCFGStackify::placeBlockMarker(MachineBasicBlock &MBB) {
   }
 
   // Add the BLOCK.
-  auto InsertPos = GetLatestInsertPos(Header, BeforeSet, AfterSet);
+
+  // 'br_on_exn' extracts except_ref object and pushes variable number of values
+  // depending on its tag. For C++ exception, its a single i32 value, and the
+  // generated code will be in the form of:
+  // block i32
+  //   br_on_exn 0, $__cpp_exception
+  //   rethrow
+  // end_block
+  WebAssembly::ExprType ReturnType = WebAssembly::ExprType::Void;
+  if (IsBrOnExn) {
+    const char *TagName = BrOnExn->getOperand(1).getSymbolName();
+    if (std::strcmp(TagName, "__cpp_exception") != 0)
+      llvm_unreachable("Only C++ exception is supported");
+    ReturnType = WebAssembly::ExprType::I32;
+  }
+
+  auto InsertPos = getLatestInsertPos(Header, BeforeSet, AfterSet);
   MachineInstr *Begin =
       BuildMI(*Header, InsertPos, Header->findDebugLoc(InsertPos),
               TII.get(WebAssembly::BLOCK))
-          .addImm(int64_t(WebAssembly::ExprType::Void));
+          .addImm(int64_t(ReturnType));
 
   // Decide where in Header to put the END_BLOCK.
   BeforeSet.clear();
@@ -333,7 +355,7 @@ void WebAssemblyCFGStackify::placeBlockMarker(MachineBasicBlock &MBB) {
   }
 
   // Mark the end of the block.
-  InsertPos = GetEarliestInsertPos(&MBB, BeforeSet, AfterSet);
+  InsertPos = getEarliestInsertPos(&MBB, BeforeSet, AfterSet);
   MachineInstr *End = BuildMI(MBB, InsertPos, MBB.findPrevDebugLoc(InsertPos),
                               TII.get(WebAssembly::END_BLOCK));
   registerScope(Begin, End);
@@ -383,7 +405,7 @@ void WebAssemblyCFGStackify::placeLoopMarker(MachineBasicBlock &MBB) {
   }
 
   // Mark the beginning of the loop.
-  auto InsertPos = GetEarliestInsertPos(&MBB, BeforeSet, AfterSet);
+  auto InsertPos = getEarliestInsertPos(&MBB, BeforeSet, AfterSet);
   MachineInstr *Begin = BuildMI(MBB, InsertPos, MBB.findDebugLoc(InsertPos),
                                 TII.get(WebAssembly::LOOP))
                             .addImm(int64_t(WebAssembly::ExprType::Void));
@@ -400,7 +422,7 @@ void WebAssemblyCFGStackify::placeLoopMarker(MachineBasicBlock &MBB) {
 
   // Mark the end of the loop (using arbitrary debug location that branched to
   // the loop end as its location).
-  InsertPos = GetEarliestInsertPos(AfterLoop, BeforeSet, AfterSet);
+  InsertPos = getEarliestInsertPos(AfterLoop, BeforeSet, AfterSet);
   DebugLoc EndDL = (*AfterLoop->pred_rbegin())->findBranchDebugLoc();
   MachineInstr *End =
       BuildMI(*AfterLoop, InsertPos, EndDL, TII.get(WebAssembly::END_LOOP));
@@ -417,11 +439,6 @@ void WebAssemblyCFGStackify::placeTryMarker(MachineBasicBlock &MBB) {
   if (!MBB.isEHPad())
     return;
 
-  // catch_all terminate pad is grouped together with catch terminate pad and
-  // does not need a separate TRY and END_TRY marker.
-  if (WebAssembly::isCatchAllTerminatePad(MBB))
-    return;
-
   MachineFunction &MF = *MBB.getParent();
   auto &MDT = getAnalysis<MachineDominatorTree>();
   const auto &TII = *MF.getSubtarget<WebAssemblySubtarget>().getInstrInfo();
@@ -434,7 +451,7 @@ void WebAssemblyCFGStackify::placeTryMarker(MachineBasicBlock &MBB) {
   for (auto *Pred : MBB.predecessors()) {
     if (Pred->getNumber() < MBBNumber) {
       Header = Header ? MDT.findNearestCommonDominator(Header, Pred) : Pred;
-      assert(!ExplicitlyBranchesTo(Pred, &MBB) &&
+      assert(!explicitlyBranchesTo(Pred, &MBB) &&
              "Explicit branch to an EH pad!");
     }
   }
@@ -530,7 +547,8 @@ void WebAssemblyCFGStackify::placeTryMarker(MachineBasicBlock &MBB) {
   // throw.
   if (MBB.isPredecessor(Header)) {
     auto TermPos = Header->getFirstTerminator();
-    if (TermPos == Header->end() || !WebAssembly::isRethrow(*TermPos)) {
+    if (TermPos == Header->end() ||
+        TermPos->getOpcode() != WebAssembly::RETHROW) {
       for (const auto &MI : reverse(*Header)) {
         if (MI.isCall()) {
           AfterSet.insert(&MI);
@@ -541,7 +559,7 @@ void WebAssemblyCFGStackify::placeTryMarker(MachineBasicBlock &MBB) {
   }
 
   // Add the TRY.
-  auto InsertPos = GetLatestInsertPos(Header, BeforeSet, AfterSet);
+  auto InsertPos = getLatestInsertPos(Header, BeforeSet, AfterSet);
   MachineInstr *Begin =
       BuildMI(*Header, InsertPos, Header->findDebugLoc(InsertPos),
               TII.get(WebAssembly::TRY))
@@ -577,7 +595,7 @@ void WebAssemblyCFGStackify::placeTryMarker(MachineBasicBlock &MBB) {
   }
 
   // Mark the end of the TRY.
-  InsertPos = GetEarliestInsertPos(AfterTry, BeforeSet, AfterSet);
+  InsertPos = getEarliestInsertPos(AfterTry, BeforeSet, AfterSet);
   MachineInstr *End =
       BuildMI(*AfterTry, InsertPos, Bottom->findBranchDebugLoc(),
               TII.get(WebAssembly::END_TRY));
@@ -591,7 +609,7 @@ void WebAssemblyCFGStackify::placeTryMarker(MachineBasicBlock &MBB) {
 }
 
 static unsigned
-GetDepth(const SmallVectorImpl<const MachineBasicBlock *> &Stack,
+getDepth(const SmallVectorImpl<const MachineBasicBlock *> &Stack,
          const MachineBasicBlock *MBB) {
   unsigned Depth = 0;
   for (auto X : reverse(Stack)) {
@@ -617,19 +635,19 @@ void WebAssemblyCFGStackify::fixEndsAtEndOfFunction(MachineFunction &MF) {
   if (MFI.getResults().empty())
     return;
 
-  WebAssembly::ExprType retType;
+  WebAssembly::ExprType RetType;
   switch (MFI.getResults().front().SimpleTy) {
   case MVT::i32:
-    retType = WebAssembly::ExprType::I32;
+    RetType = WebAssembly::ExprType::I32;
     break;
   case MVT::i64:
-    retType = WebAssembly::ExprType::I64;
+    RetType = WebAssembly::ExprType::I64;
     break;
   case MVT::f32:
-    retType = WebAssembly::ExprType::F32;
+    RetType = WebAssembly::ExprType::F32;
     break;
   case MVT::f64:
-    retType = WebAssembly::ExprType::F64;
+    RetType = WebAssembly::ExprType::F64;
     break;
   case MVT::v16i8:
   case MVT::v8i16:
@@ -637,10 +655,10 @@ void WebAssemblyCFGStackify::fixEndsAtEndOfFunction(MachineFunction &MF) {
   case MVT::v2i64:
   case MVT::v4f32:
   case MVT::v2f64:
-    retType = WebAssembly::ExprType::V128;
+    RetType = WebAssembly::ExprType::V128;
     break;
   case MVT::ExceptRef:
-    retType = WebAssembly::ExprType::ExceptRef;
+    RetType = WebAssembly::ExprType::ExceptRef;
     break;
   default:
     llvm_unreachable("unexpected return type");
@@ -651,11 +669,11 @@ void WebAssemblyCFGStackify::fixEndsAtEndOfFunction(MachineFunction &MF) {
       if (MI.isPosition() || MI.isDebugInstr())
         continue;
       if (MI.getOpcode() == WebAssembly::END_BLOCK) {
-        EndToBegin[&MI]->getOperand(0).setImm(int32_t(retType));
+        EndToBegin[&MI]->getOperand(0).setImm(int32_t(RetType));
         continue;
       }
       if (MI.getOpcode() == WebAssembly::END_LOOP) {
-        EndToBegin[&MI]->getOperand(0).setImm(int32_t(retType));
+        EndToBegin[&MI]->getOperand(0).setImm(int32_t(RetType));
         continue;
       }
       // Something other than an `end`. We're done.
@@ -666,7 +684,7 @@ void WebAssemblyCFGStackify::fixEndsAtEndOfFunction(MachineFunction &MF) {
 
 // WebAssembly functions end with an end instruction, as if the function body
 // were a block.
-static void AppendEndToFunction(MachineFunction &MF,
+static void appendEndToFunction(MachineFunction &MF,
                                 const WebAssemblyInstrInfo &TII) {
   BuildMI(MF.back(), MF.back().end(),
           MF.back().findPrevDebugLoc(MF.back().end()),
@@ -675,7 +693,6 @@ static void AppendEndToFunction(MachineFunction &MF,
 
 /// Insert LOOP/TRY/BLOCK markers at appropriate places.
 void WebAssemblyCFGStackify::placeMarkers(MachineFunction &MF) {
-  const MCAsmInfo *MCAI = MF.getTarget().getMCAsmInfo();
   // We allocate one more than the number of blocks in the function to
   // accommodate for the possible fake block we may insert at the end.
   ScopeTops.resize(MF.getNumBlockIDs() + 1);
@@ -683,6 +700,7 @@ void WebAssemblyCFGStackify::placeMarkers(MachineFunction &MF) {
   for (auto &MBB : MF)
     placeLoopMarker(MBB);
   // Place the TRY for MBB if MBB is the EH pad of an exception.
+  const MCAsmInfo *MCAI = MF.getTarget().getMCAsmInfo();
   if (MCAI->getExceptionHandlingType() == ExceptionHandling::Wasm &&
       MF.getFunction().hasPersonalityFn())
     for (auto &MBB : MF)
@@ -693,12 +711,8 @@ void WebAssemblyCFGStackify::placeMarkers(MachineFunction &MF) {
 }
 
 void WebAssemblyCFGStackify::rewriteDepthImmediates(MachineFunction &MF) {
-  const auto &TII = *MF.getSubtarget<WebAssemblySubtarget>().getInstrInfo();
   // Now rewrite references to basic blocks to be depth immediates.
-  // We need two stacks: one for normal scopes and the other for EH pad scopes.
-  // EH pad stack is used to rewrite depths in rethrow instructions.
   SmallVector<const MachineBasicBlock *, 8> Stack;
-  SmallVector<const MachineBasicBlock *, 8> EHPadStack;
   for (auto &MBB : reverse(MF)) {
     for (auto I = MBB.rbegin(), E = MBB.rend(); I != E; ++I) {
       MachineInstr &MI = *I;
@@ -715,26 +729,6 @@ void WebAssemblyCFGStackify::rewriteDepthImmediates(MachineFunction &MF) {
                    MBB.getNumber() &&
                "Block/try marker should be balanced");
         Stack.pop_back();
-        EHPadStack.pop_back();
-        break;
-
-      case WebAssembly::CATCH_I32:
-      case WebAssembly::CATCH_I64:
-      case WebAssembly::CATCH_ALL:
-        // Currently the only case there are more than one catch for a try is
-        // for catch terminate pad, in the form of
-        //   try
-        //   catch
-        //     call @__clang_call_terminate
-        //     unreachable
-        //   catch_all
-        //     call @std::terminate
-        //     unreachable
-        //   end
-        // So we shouldn't push the current BB for the second catch_all block
-        // here.
-        if (!WebAssembly::isCatchAllTerminatePad(MBB))
-          EHPadStack.push_back(&MBB);
         break;
 
       case WebAssembly::LOOP:
@@ -751,23 +745,6 @@ void WebAssemblyCFGStackify::rewriteDepthImmediates(MachineFunction &MF) {
         Stack.push_back(EndToBegin[&MI]->getParent());
         break;
 
-      case WebAssembly::RETHROW: {
-        // Rewrite MBB operands to be depth immediates.
-        unsigned EHPadDepth = GetDepth(EHPadStack, MI.getOperand(0).getMBB());
-        MI.RemoveOperand(0);
-        MI.addOperand(MF, MachineOperand::CreateImm(EHPadDepth));
-        break;
-      }
-
-      case WebAssembly::RETHROW_TO_CALLER: {
-        MachineInstr *Rethrow =
-            BuildMI(MBB, MI, MI.getDebugLoc(), TII.get(WebAssembly::RETHROW))
-                .addImm(EHPadStack.size());
-        MI.eraseFromParent();
-        I = MachineBasicBlock::reverse_iterator(Rethrow);
-        break;
-      }
-
       default:
         if (MI.isTerminator()) {
           // Rewrite MBB operands to be depth immediates.
@@ -776,7 +753,7 @@ void WebAssemblyCFGStackify::rewriteDepthImmediates(MachineFunction &MF) {
             MI.RemoveOperand(MI.getNumOperands() - 1);
           for (auto MO : Ops) {
             if (MO.isMBB())
-              MO = MachineOperand::CreateImm(GetDepth(Stack, MO.getMBB()));
+              MO = MachineOperand::CreateImm(getDepth(Stack, MO.getMBB()));
             MI.addOperand(MF, MO);
           }
         }
@@ -821,7 +798,7 @@ bool WebAssemblyCFGStackify::runOnMachineFunction(MachineFunction &MF) {
   if (!MF.getSubtarget<WebAssemblySubtarget>()
            .getTargetTriple()
            .isOSBinFormatELF())
-    AppendEndToFunction(MF, TII);
+    appendEndToFunction(MF, TII);
 
   return true;
 }
diff --git a/lib/Target/WebAssembly/WebAssemblyCallIndirectFixup.cpp b/lib/Target/WebAssembly/WebAssemblyCallIndirectFixup.cpp
index 326e838052d8203e70dbed5765ae8a15e0308a79..f1196925f6ceb77f236ae1b41ee97d90d1ab9cac 100644
--- a/lib/Target/WebAssembly/WebAssemblyCallIndirectFixup.cpp
+++ b/lib/Target/WebAssembly/WebAssemblyCallIndirectFixup.cpp
@@ -1,9 +1,8 @@
 //===-- WebAssemblyCallIndirectFixup.cpp - Fix call_indirects -------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
@@ -61,7 +60,7 @@ FunctionPass *llvm::createWebAssemblyCallIndirectFixup() {
   return new WebAssemblyCallIndirectFixup();
 }
 
-static unsigned GetNonPseudoCallIndirectOpcode(const MachineInstr &MI) {
+static unsigned getNonPseudoCallIndirectOpcode(const MachineInstr &MI) {
   switch (MI.getOpcode()) {
     using namespace WebAssembly;
   case PCALL_INDIRECT_VOID:
@@ -91,14 +90,14 @@ static unsigned GetNonPseudoCallIndirectOpcode(const MachineInstr &MI) {
   }
 }
 
-static bool IsPseudoCallIndirect(const MachineInstr &MI) {
-  return GetNonPseudoCallIndirectOpcode(MI) !=
+static bool isPseudoCallIndirect(const MachineInstr &MI) {
+  return getNonPseudoCallIndirectOpcode(MI) !=
          WebAssembly::INSTRUCTION_LIST_END;
 }
 
 bool WebAssemblyCallIndirectFixup::runOnMachineFunction(MachineFunction &MF) {
   LLVM_DEBUG(dbgs() << "********** Fixing up CALL_INDIRECTs **********\n"
-                    << MF.getName() << '\n');
+                    << "********** Function: " << MF.getName() << '\n');
 
   bool Changed = false;
   const WebAssemblyInstrInfo *TII =
@@ -106,11 +105,11 @@ bool WebAssemblyCallIndirectFixup::runOnMachineFunction(MachineFunction &MF) {
 
   for (MachineBasicBlock &MBB : MF) {
     for (MachineInstr &MI : MBB) {
-      if (IsPseudoCallIndirect(MI)) {
+      if (isPseudoCallIndirect(MI)) {
         LLVM_DEBUG(dbgs() << "Found call_indirect: " << MI << '\n');
 
         // Rewrite pseudo to non-pseudo
-        const MCInstrDesc &Desc = TII->get(GetNonPseudoCallIndirectOpcode(MI));
+        const MCInstrDesc &Desc = TII->get(getNonPseudoCallIndirectOpcode(MI));
         MI.setDesc(Desc);
 
         // Rewrite argument order
diff --git a/lib/Target/WebAssembly/WebAssemblyDebugValueManager.cpp b/lib/Target/WebAssembly/WebAssemblyDebugValueManager.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..579377c9a5d755d8c8cb29ece7af07bfe990f646
--- /dev/null
+++ b/lib/Target/WebAssembly/WebAssemblyDebugValueManager.cpp
@@ -0,0 +1,45 @@
+//===-- WebAssemblyDebugValueManager.cpp - WebAssembly DebugValue Manager -===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file implements the manager for MachineInstr DebugValues.
+///
+//===----------------------------------------------------------------------===//
+
+#include "WebAssemblyDebugValueManager.h"
+#include "WebAssemblyMachineFunctionInfo.h"
+#include "llvm/CodeGen/MachineInstr.h"
+
+using namespace llvm;
+
+WebAssemblyDebugValueManager::WebAssemblyDebugValueManager(
+    MachineInstr *Instr) {
+  Instr->collectDebugValues(DbgValues);
+}
+
+void WebAssemblyDebugValueManager::move(MachineInstr *Insert) {
+  MachineBasicBlock *MBB = Insert->getParent();
+  for (MachineInstr *DBI : reverse(DbgValues))
+    MBB->splice(Insert, DBI->getParent(), DBI);
+}
+
+void WebAssemblyDebugValueManager::updateReg(unsigned Reg) {
+  for (auto *DBI : DbgValues)
+    DBI->getOperand(0).setReg(Reg);
+}
+
+void WebAssemblyDebugValueManager::clone(MachineInstr *Insert,
+                                         unsigned NewReg) {
+  MachineBasicBlock *MBB = Insert->getParent();
+  MachineFunction *MF = MBB->getParent();
+  for (MachineInstr *DBI : reverse(DbgValues)) {
+    MachineInstr *Clone = MF->CloneMachineInstr(DBI);
+    Clone->getOperand(0).setReg(NewReg);
+    MBB->insert(Insert, Clone);
+  }
+}
diff --git a/lib/Target/WebAssembly/WebAssemblyDebugValueManager.h b/lib/Target/WebAssembly/WebAssemblyDebugValueManager.h
new file mode 100644
index 0000000000000000000000000000000000000000..06e8805b5ad08199a792b83d485cd902c9e77e71
--- /dev/null
+++ b/lib/Target/WebAssembly/WebAssemblyDebugValueManager.h
@@ -0,0 +1,37 @@
+// WebAssemblyDebugValueManager.h - WebAssembly DebugValue Manager -*- C++ -*-//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file contains the declaration of the WebAssembly-specific
+/// manager for DebugValues associated with the specific MachineInstr.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_TARGET_WEBASSEMBLY_WEBASSEMBLYDEBUGVALUEMANAGER_H
+#define LLVM_LIB_TARGET_WEBASSEMBLY_WEBASSEMBLYDEBUGVALUEMANAGER_H
+
+#include "llvm/ADT/SmallVector.h"
+
+namespace llvm {
+
+class MachineInstr;
+
+class WebAssemblyDebugValueManager {
+  SmallVector<MachineInstr *, 2> DbgValues;
+
+public:
+  WebAssemblyDebugValueManager(MachineInstr *Instr);
+
+  void move(MachineInstr *Insert);
+  void updateReg(unsigned Reg);
+  void clone(MachineInstr *Insert, unsigned NewReg);
+};
+
+} // end namespace llvm
+
+#endif
diff --git a/lib/Target/WebAssembly/WebAssemblyEHRestoreStackPointer.cpp b/lib/Target/WebAssembly/WebAssemblyEHRestoreStackPointer.cpp
deleted file mode 100644
index bd1ab472c1e29ff6a5d0acbe26d5056f1f473bce..0000000000000000000000000000000000000000
--- a/lib/Target/WebAssembly/WebAssemblyEHRestoreStackPointer.cpp
+++ /dev/null
@@ -1,83 +0,0 @@
-//===-- WebAssemblyEHRestoreStackPointer.cpp - __stack_pointer restoration ===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-///
-/// \file
-/// After the stack is unwound due to a thrown exception, the __stack_pointer
-/// global can point to an invalid address. This inserts instructions that
-/// restore __stack_pointer global.
-///
-//===----------------------------------------------------------------------===//
-
-#include "MCTargetDesc/WebAssemblyMCTargetDesc.h"
-#include "WebAssembly.h"
-#include "WebAssemblySubtarget.h"
-#include "WebAssemblyUtilities.h"
-#include "llvm/CodeGen/MachineFrameInfo.h"
-#include "llvm/MC/MCAsmInfo.h"
-using namespace llvm;
-
-#define DEBUG_TYPE "wasm-eh-restore-stack-pointer"
-
-namespace {
-class WebAssemblyEHRestoreStackPointer final : public MachineFunctionPass {
-public:
-  static char ID; // Pass identification, replacement for typeid
-  WebAssemblyEHRestoreStackPointer() : MachineFunctionPass(ID) {}
-
-  StringRef getPassName() const override {
-    return "WebAssembly Restore Stack Pointer for Exception Handling";
-  }
-
-  void getAnalysisUsage(AnalysisUsage &AU) const override {
-    AU.setPreservesCFG();
-    MachineFunctionPass::getAnalysisUsage(AU);
-  }
-
-  bool runOnMachineFunction(MachineFunction &MF) override;
-};
-} // end anonymous namespace
-
-char WebAssemblyEHRestoreStackPointer::ID = 0;
-INITIALIZE_PASS(WebAssemblyEHRestoreStackPointer, DEBUG_TYPE,
-                "Restore Stack Pointer for Exception Handling", true, false)
-
-FunctionPass *llvm::createWebAssemblyEHRestoreStackPointer() {
-  return new WebAssemblyEHRestoreStackPointer();
-}
-
-bool WebAssemblyEHRestoreStackPointer::runOnMachineFunction(
-    MachineFunction &MF) {
-  const auto *FrameLowering = static_cast<const WebAssemblyFrameLowering *>(
-      MF.getSubtarget().getFrameLowering());
-  if (!FrameLowering->needsPrologForEH(MF))
-    return false;
-  bool Changed = false;
-
-  for (auto &MBB : MF) {
-    if (!MBB.isEHPad())
-      continue;
-    Changed = true;
-
-    // Insert __stack_pointer restoring instructions at the beginning of each EH
-    // pad, after the catch instruction. (Catch instructions may have been
-    // reordered, and catch_all instructions have not been inserted yet, but
-    // those cases are handled in LateEHPrepare).
-    //
-    // Here it is safe to assume that SP32 holds the latest value of
-    // __stack_pointer, because the only exception for this case is when a
-    // function uses the red zone, but that only happens with leaf functions,
-    // and we don't restore __stack_pointer in leaf functions anyway.
-    auto InsertPos = MBB.begin();
-    if (WebAssembly::isCatch(*MBB.begin()))
-      InsertPos++;
-    FrameLowering->writeSPToGlobal(WebAssembly::SP32, MF, MBB, InsertPos,
-                                   MBB.begin()->getDebugLoc());
-  }
-  return Changed;
-}
diff --git a/lib/Target/WebAssembly/WebAssemblyExceptionInfo.cpp b/lib/Target/WebAssembly/WebAssemblyExceptionInfo.cpp
index dca7902b8fb920830f43958a552fdfd3a3361b3f..0387957b14c2954bcc361ba62d548e1e2a267098 100644
--- a/lib/Target/WebAssembly/WebAssemblyExceptionInfo.cpp
+++ b/lib/Target/WebAssembly/WebAssemblyExceptionInfo.cpp
@@ -1,9 +1,8 @@
 //===--- WebAssemblyExceptionInfo.cpp - Exception Infomation --------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
@@ -32,7 +31,10 @@ INITIALIZE_PASS_DEPENDENCY(MachineDominanceFrontier)
 INITIALIZE_PASS_END(WebAssemblyExceptionInfo, DEBUG_TYPE,
                     "WebAssembly Exception Information", true, true)
 
-bool WebAssemblyExceptionInfo::runOnMachineFunction(MachineFunction &F) {
+bool WebAssemblyExceptionInfo::runOnMachineFunction(MachineFunction &MF) {
+  LLVM_DEBUG(dbgs() << "********** Exception Info Calculation **********\n"
+                       "********** Function: "
+                    << MF.getName() << '\n');
   releaseMemory();
   auto &MDT = getAnalysis<MachineDominatorTree>();
   auto &MDF = getAnalysis<MachineDominanceFrontier>();
@@ -48,10 +50,6 @@ void WebAssemblyExceptionInfo::recalculate(
     MachineBasicBlock *EHPad = DomNode->getBlock();
     if (!EHPad->isEHPad())
       continue;
-    // We group catch & catch-all terminate pads together, so skip the second
-    // one
-    if (WebAssembly::isCatchAllTerminatePad(*EHPad))
-      continue;
     auto *WE = new WebAssemblyException(EHPad);
     discoverAndMapException(WE, MDT, MDF);
     Exceptions.push_back(WE);
@@ -102,16 +100,6 @@ void WebAssemblyExceptionInfo::discoverAndMapException(
 
   // Map blocks that belong to a catchpad / cleanuppad
   MachineBasicBlock *EHPad = WE->getEHPad();
-
-  // We group catch & catch-all terminate pads together within an exception
-  if (WebAssembly::isCatchTerminatePad(*EHPad)) {
-    assert(EHPad->succ_size() == 1 &&
-           "Catch terminate pad has more than one successors");
-    changeExceptionFor(EHPad, WE);
-    changeExceptionFor(*(EHPad->succ_begin()), WE);
-    return;
-  }
-
   SmallVector<MachineBasicBlock *, 8> WL;
   WL.push_back(EHPad);
   while (!WL.empty()) {
diff --git a/lib/Target/WebAssembly/WebAssemblyExceptionInfo.h b/lib/Target/WebAssembly/WebAssemblyExceptionInfo.h
index fcd7e2366e03b656953e28ae1cacb6f14b05f41c..9a90d7df7d47d54fd4de55351ae43e32d49d5c43 100644
--- a/lib/Target/WebAssembly/WebAssemblyExceptionInfo.h
+++ b/lib/Target/WebAssembly/WebAssemblyExceptionInfo.h
@@ -1,9 +1,8 @@
 //===-- WebAssemblyExceptionInfo.h - WebAssembly Exception Info -*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
diff --git a/lib/Target/WebAssembly/WebAssemblyExplicitLocals.cpp b/lib/Target/WebAssembly/WebAssemblyExplicitLocals.cpp
index 384c0d60db535b4db22d3aaba96c993e77ead238..8494737d43d73e9f21df3172f347e94dd1f84adf 100644
--- a/lib/Target/WebAssembly/WebAssemblyExplicitLocals.cpp
+++ b/lib/Target/WebAssembly/WebAssemblyExplicitLocals.cpp
@@ -1,9 +1,8 @@
 //===-- WebAssemblyExplicitLocals.cpp - Make Locals Explicit --------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
@@ -11,7 +10,7 @@
 /// This file converts any remaining registers into WebAssembly locals.
 ///
 /// After register stackification and register coloring, convert non-stackified
-/// registers into locals, inserting explicit get_local and set_local
+/// registers into locals, inserting explicit local.get and local.set
 /// instructions.
 ///
 //===----------------------------------------------------------------------===//
@@ -96,54 +95,54 @@ static unsigned getDropOpcode(const TargetRegisterClass *RC) {
   llvm_unreachable("Unexpected register class");
 }
 
-/// Get the appropriate get_local opcode for the given register class.
+/// Get the appropriate local.get opcode for the given register class.
 static unsigned getGetLocalOpcode(const TargetRegisterClass *RC) {
   if (RC == &WebAssembly::I32RegClass)
-    return WebAssembly::GET_LOCAL_I32;
+    return WebAssembly::LOCAL_GET_I32;
   if (RC == &WebAssembly::I64RegClass)
-    return WebAssembly::GET_LOCAL_I64;
+    return WebAssembly::LOCAL_GET_I64;
   if (RC == &WebAssembly::F32RegClass)
-    return WebAssembly::GET_LOCAL_F32;
+    return WebAssembly::LOCAL_GET_F32;
   if (RC == &WebAssembly::F64RegClass)
-    return WebAssembly::GET_LOCAL_F64;
+    return WebAssembly::LOCAL_GET_F64;
   if (RC == &WebAssembly::V128RegClass)
-    return WebAssembly::GET_LOCAL_V128;
+    return WebAssembly::LOCAL_GET_V128;
   if (RC == &WebAssembly::EXCEPT_REFRegClass)
-    return WebAssembly::GET_LOCAL_EXCEPT_REF;
+    return WebAssembly::LOCAL_GET_EXCEPT_REF;
   llvm_unreachable("Unexpected register class");
 }
 
-/// Get the appropriate set_local opcode for the given register class.
+/// Get the appropriate local.set opcode for the given register class.
 static unsigned getSetLocalOpcode(const TargetRegisterClass *RC) {
   if (RC == &WebAssembly::I32RegClass)
-    return WebAssembly::SET_LOCAL_I32;
+    return WebAssembly::LOCAL_SET_I32;
   if (RC == &WebAssembly::I64RegClass)
-    return WebAssembly::SET_LOCAL_I64;
+    return WebAssembly::LOCAL_SET_I64;
   if (RC == &WebAssembly::F32RegClass)
-    return WebAssembly::SET_LOCAL_F32;
+    return WebAssembly::LOCAL_SET_F32;
   if (RC == &WebAssembly::F64RegClass)
-    return WebAssembly::SET_LOCAL_F64;
+    return WebAssembly::LOCAL_SET_F64;
   if (RC == &WebAssembly::V128RegClass)
-    return WebAssembly::SET_LOCAL_V128;
+    return WebAssembly::LOCAL_SET_V128;
   if (RC == &WebAssembly::EXCEPT_REFRegClass)
-    return WebAssembly::SET_LOCAL_EXCEPT_REF;
+    return WebAssembly::LOCAL_SET_EXCEPT_REF;
   llvm_unreachable("Unexpected register class");
 }
 
-/// Get the appropriate tee_local opcode for the given register class.
+/// Get the appropriate local.tee opcode for the given register class.
 static unsigned getTeeLocalOpcode(const TargetRegisterClass *RC) {
   if (RC == &WebAssembly::I32RegClass)
-    return WebAssembly::TEE_LOCAL_I32;
+    return WebAssembly::LOCAL_TEE_I32;
   if (RC == &WebAssembly::I64RegClass)
-    return WebAssembly::TEE_LOCAL_I64;
+    return WebAssembly::LOCAL_TEE_I64;
   if (RC == &WebAssembly::F32RegClass)
-    return WebAssembly::TEE_LOCAL_F32;
+    return WebAssembly::LOCAL_TEE_F32;
   if (RC == &WebAssembly::F64RegClass)
-    return WebAssembly::TEE_LOCAL_F64;
+    return WebAssembly::LOCAL_TEE_F64;
   if (RC == &WebAssembly::V128RegClass)
-    return WebAssembly::TEE_LOCAL_V128;
+    return WebAssembly::LOCAL_TEE_V128;
   if (RC == &WebAssembly::EXCEPT_REFRegClass)
-    return WebAssembly::TEE_LOCAL_EXCEPT_REF;
+    return WebAssembly::LOCAL_TEE_EXCEPT_REF;
   llvm_unreachable("Unexpected register class");
 }
 
@@ -233,8 +232,8 @@ bool WebAssemblyExplicitLocals::runOnMachineFunction(MachineFunction &MF) {
       if (MI.isDebugInstr() || MI.isLabel())
         continue;
 
-      // Replace tee instructions with tee_local. The difference is that tee
-      // instructins have two defs, while tee_local instructions have one def
+      // Replace tee instructions with local.tee. The difference is that tee
+      // instructions have two defs, while local.tee instructions have one def
       // and an index of a local to write to.
       if (WebAssembly::isTee(MI)) {
         assert(MFI.isVRegStackified(MI.getOperand(0).getReg()));
@@ -253,7 +252,7 @@ bool WebAssemblyExplicitLocals::runOnMachineFunction(MachineFunction &MF) {
           MFI.stackifyVReg(NewReg);
         }
 
-        // Replace the TEE with a TEE_LOCAL.
+        // Replace the TEE with a LOCAL_TEE.
         unsigned LocalId =
             getLocalId(Reg2Local, CurLocal, MI.getOperand(1).getReg());
         unsigned Opc = getTeeLocalOpcode(RC);
@@ -267,7 +266,7 @@ bool WebAssemblyExplicitLocals::runOnMachineFunction(MachineFunction &MF) {
         continue;
       }
 
-      // Insert set_locals for any defs that aren't stackified yet. Currently
+      // Insert local.sets for any defs that aren't stackified yet. Currently
       // we handle at most one def.
       assert(MI.getDesc().getNumDefs() <= 1);
       if (MI.getDesc().getNumDefs() == 1) {
@@ -296,15 +295,16 @@ bool WebAssemblyExplicitLocals::runOnMachineFunction(MachineFunction &MF) {
                 .addReg(NewReg);
           }
           MI.getOperand(0).setReg(NewReg);
-          // This register operand is now being used by the inserted drop
-          // instruction, so make it undead.
+          // This register operand of the original instruction is now being used
+          // by the inserted drop or local.set instruction, so make it not dead
+          // yet.
           MI.getOperand(0).setIsDead(false);
           MFI.stackifyVReg(NewReg);
           Changed = true;
         }
       }
 
-      // Insert get_locals for any uses that aren't stackified yet.
+      // Insert local.gets for any uses that aren't stackified yet.
       MachineInstr *InsertPt = &MI;
       for (MachineOperand &MO : reverse(MI.explicit_uses())) {
         if (!MO.isReg())
@@ -316,7 +316,7 @@ bool WebAssemblyExplicitLocals::runOnMachineFunction(MachineFunction &MF) {
         // with inline asm register operands is to provide local indices as
         // immediates.
         if (MO.isDef()) {
-          assert(MI.getOpcode() == TargetOpcode::INLINEASM);
+          assert(MI.isInlineAsm());
           unsigned LocalId = getLocalId(Reg2Local, CurLocal, OldReg);
           // If this register operand is tied to another operand, we can't
           // change it to an immediate. Untie it first.
@@ -326,7 +326,7 @@ bool WebAssemblyExplicitLocals::runOnMachineFunction(MachineFunction &MF) {
         }
 
         // If we see a stackified register, prepare to insert subsequent
-        // get_locals before the start of its tree.
+        // local.gets before the start of its tree.
         if (MFI.isVRegStackified(OldReg)) {
           InsertPt = findStartOfTree(MO, MRI, MFI);
           continue;
@@ -334,7 +334,7 @@ bool WebAssemblyExplicitLocals::runOnMachineFunction(MachineFunction &MF) {
 
         // Our contract with inline asm register operands is to provide local
         // indices as immediates.
-        if (MI.getOpcode() == TargetOpcode::INLINEASM) {
+        if (MI.isInlineAsm()) {
           unsigned LocalId = getLocalId(Reg2Local, CurLocal, OldReg);
           // Untie it first if this reg operand is tied to another operand.
           MI.untieRegOperand(MI.getOperandNo(&MO));
@@ -342,7 +342,7 @@ bool WebAssemblyExplicitLocals::runOnMachineFunction(MachineFunction &MF) {
           continue;
         }
 
-        // Insert a get_local.
+        // Insert a local.get.
         unsigned LocalId = getLocalId(Reg2Local, CurLocal, OldReg);
         const TargetRegisterClass *RC = MRI.getRegClass(OldReg);
         unsigned NewReg = MRI.createVirtualRegister(RC);
diff --git a/lib/Target/WebAssembly/WebAssemblyFastISel.cpp b/lib/Target/WebAssembly/WebAssemblyFastISel.cpp
index 00e37a4af2963829dbdb7c6471792b635a985386..15df3d4ba0653d225e5a77f44cc2f442bbf2bcd4 100644
--- a/lib/Target/WebAssembly/WebAssemblyFastISel.cpp
+++ b/lib/Target/WebAssembly/WebAssemblyFastISel.cpp
@@ -1,9 +1,8 @@
 //===-- WebAssemblyFastISel.cpp - WebAssembly FastISel implementation -----===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
@@ -50,22 +49,22 @@ class WebAssemblyFastISel final : public FastISel {
   // All possible address modes.
   class Address {
   public:
-    typedef enum { RegBase, FrameIndexBase } BaseKind;
+    using BaseKind = enum { RegBase, FrameIndexBase };
 
   private:
-    BaseKind Kind;
+    BaseKind Kind = RegBase;
     union {
       unsigned Reg;
       int FI;
     } Base;
 
-    int64_t Offset;
+    int64_t Offset = 0;
 
-    const GlobalValue *GV;
+    const GlobalValue *GV = nullptr;
 
   public:
     // Innocuous defaults for our address.
-    Address() : Kind(RegBase), Offset(0), GV(0) { Base.Reg = 0; }
+    Address() { Base.Reg = 0; }
     void setKind(BaseKind K) {
       assert(!isSet() && "Can't change kind with non-zero base");
       Kind = K;
@@ -92,9 +91,9 @@ class WebAssemblyFastISel final : public FastISel {
       return Base.FI;
     }
 
-    void setOffset(int64_t Offset_) {
-      assert(Offset_ >= 0 && "Offsets must be non-negative");
-      Offset = Offset_;
+    void setOffset(int64_t NewOffset) {
+      assert(NewOffset >= 0 && "Offsets must be non-negative");
+      Offset = NewOffset;
     }
     int64_t getOffset() const { return Offset; }
     void setGlobalValue(const GlobalValue *G) { GV = G; }
@@ -137,12 +136,15 @@ private:
     case MVT::v16i8:
     case MVT::v8i16:
     case MVT::v4i32:
-    case MVT::v2i64:
     case MVT::v4f32:
-    case MVT::v2f64:
       if (Subtarget->hasSIMD128())
         return VT;
       break;
+    case MVT::v2i64:
+    case MVT::v2f64:
+      if (Subtarget->hasUnimplementedSIMD128())
+        return VT;
+      break;
     default:
       break;
     }
@@ -208,7 +210,7 @@ bool WebAssemblyFastISel::computeAddress(const Value *Obj, Address &Addr) {
 
   const User *U = nullptr;
   unsigned Opcode = Instruction::UserOp1;
-  if (const Instruction *I = dyn_cast<Instruction>(Obj)) {
+  if (const auto *I = dyn_cast<Instruction>(Obj)) {
     // Don't walk into other basic blocks unless the object is an alloca from
     // another block, otherwise it may not have a virtual register assigned.
     if (FuncInfo.StaticAllocaMap.count(static_cast<const AllocaInst *>(Obj)) ||
@@ -216,7 +218,7 @@ bool WebAssemblyFastISel::computeAddress(const Value *Obj, Address &Addr) {
       Opcode = I->getOpcode();
       U = I;
     }
-  } else if (const ConstantExpr *C = dyn_cast<ConstantExpr>(Obj)) {
+  } else if (const auto *C = dyn_cast<ConstantExpr>(Obj)) {
     Opcode = C->getOpcode();
     U = C;
   }
@@ -227,7 +229,7 @@ bool WebAssemblyFastISel::computeAddress(const Value *Obj, Address &Addr) {
       // address spaces.
       return false;
 
-  if (const GlobalValue *GV = dyn_cast<GlobalValue>(Obj)) {
+  if (const auto *GV = dyn_cast<GlobalValue>(Obj)) {
     if (Addr.getGlobalValue())
       return false;
     Addr.setGlobalValue(GV);
@@ -272,7 +274,7 @@ bool WebAssemblyFastISel::computeAddress(const Value *Obj, Address &Addr) {
       } else {
         uint64_t S = DL.getTypeAllocSize(GTI.getIndexedType());
         for (;;) {
-          if (const ConstantInt *CI = dyn_cast<ConstantInt>(Op)) {
+          if (const auto *CI = dyn_cast<ConstantInt>(Op)) {
             // Constant-offset addressing.
             TmpOffset += CI->getSExtValue() * S;
             break;
@@ -287,8 +289,7 @@ bool WebAssemblyFastISel::computeAddress(const Value *Obj, Address &Addr) {
           }
           if (canFoldAddIntoGEP(U, Op)) {
             // A compatible add with a constant operand. Fold the constant.
-            ConstantInt *CI =
-                cast<ConstantInt>(cast<AddOperator>(Op)->getOperand(1));
+            auto *CI = cast<ConstantInt>(cast<AddOperator>(Op)->getOperand(1));
             TmpOffset += CI->getSExtValue() * S;
             // Iterate on the other operand.
             Op = cast<AddOperator>(Op)->getOperand(0);
@@ -312,7 +313,7 @@ bool WebAssemblyFastISel::computeAddress(const Value *Obj, Address &Addr) {
     break;
   }
   case Instruction::Alloca: {
-    const AllocaInst *AI = cast<AllocaInst>(Obj);
+    const auto *AI = cast<AllocaInst>(Obj);
     DenseMap<const AllocaInst *, int>::iterator SI =
         FuncInfo.StaticAllocaMap.find(AI);
     if (SI != FuncInfo.StaticAllocaMap.end()) {
@@ -333,7 +334,7 @@ bool WebAssemblyFastISel::computeAddress(const Value *Obj, Address &Addr) {
     if (isa<ConstantInt>(LHS))
       std::swap(LHS, RHS);
 
-    if (const ConstantInt *CI = dyn_cast<ConstantInt>(RHS)) {
+    if (const auto *CI = dyn_cast<ConstantInt>(RHS)) {
       uint64_t TmpOffset = Addr.getOffset() + CI->getSExtValue();
       if (int64_t(TmpOffset) >= 0) {
         Addr.setOffset(TmpOffset);
@@ -353,7 +354,7 @@ bool WebAssemblyFastISel::computeAddress(const Value *Obj, Address &Addr) {
     const Value *LHS = U->getOperand(0);
     const Value *RHS = U->getOperand(1);
 
-    if (const ConstantInt *CI = dyn_cast<ConstantInt>(RHS)) {
+    if (const auto *CI = dyn_cast<ConstantInt>(RHS)) {
       int64_t TmpOffset = Addr.getOffset() - CI->getSExtValue();
       if (TmpOffset >= 0) {
         Addr.setOffset(TmpOffset);
@@ -413,7 +414,7 @@ unsigned WebAssemblyFastISel::maskI1Value(unsigned Reg, const Value *V) {
 }
 
 unsigned WebAssemblyFastISel::getRegForI1Value(const Value *V, bool &Not) {
-  if (const ICmpInst *ICmp = dyn_cast<ICmpInst>(V))
+  if (const auto *ICmp = dyn_cast<ICmpInst>(V))
     if (const ConstantInt *C = dyn_cast<ConstantInt>(ICmp->getOperand(1)))
       if (ICmp->isEquality() && C->isZero() && C->getType()->isIntegerTy(32)) {
         Not = ICmp->isTrueWhenEqual();
@@ -440,13 +441,11 @@ unsigned WebAssemblyFastISel::zeroExtendToI32(unsigned Reg, const Value *V,
 
   switch (From) {
   case MVT::i1:
-    // If the value is naturally an i1, we don't need to mask it.
-    // TODO: Recursively examine selects, phis, and, or, xor, constants.
-    if (From == MVT::i1 && V != nullptr) {
-      if (isa<CmpInst>(V) ||
-          (isa<Argument>(V) && cast<Argument>(V)->hasZExtAttr()))
-        return copyValue(Reg);
-    }
+    // If the value is naturally an i1, we don't need to mask it. We only know
+    // if a value is naturally an i1 if it is definitely lowered by FastISel,
+    // not a DAG ISel fallback.
+    if (V != nullptr && isa<Argument>(V) && cast<Argument>(V)->hasZExtAttr())
+      return copyValue(Reg);
     break;
   case MVT::i8:
   case MVT::i16:
@@ -605,7 +604,7 @@ unsigned WebAssemblyFastISel::fastMaterializeAlloca(const AllocaInst *AI) {
 }
 
 unsigned WebAssemblyFastISel::fastMaterializeConstant(const Constant *C) {
-  if (const GlobalValue *GV = dyn_cast<GlobalValue>(C)) {
+  if (const auto *GV = dyn_cast<GlobalValue>(C)) {
     unsigned ResultReg =
         createResultReg(Subtarget->hasAddr64() ? &WebAssembly::I64RegClass
                                                : &WebAssembly::I32RegClass);
@@ -628,14 +627,14 @@ bool WebAssemblyFastISel::fastLowerArguments() {
   if (F->isVarArg())
     return false;
 
-  unsigned i = 0;
+  unsigned I = 0;
   for (auto const &Arg : F->args()) {
     const AttributeList &Attrs = F->getAttributes();
-    if (Attrs.hasParamAttribute(i, Attribute::ByVal) ||
-        Attrs.hasParamAttribute(i, Attribute::SwiftSelf) ||
-        Attrs.hasParamAttribute(i, Attribute::SwiftError) ||
-        Attrs.hasParamAttribute(i, Attribute::InAlloca) ||
-        Attrs.hasParamAttribute(i, Attribute::Nest))
+    if (Attrs.hasParamAttribute(I, Attribute::ByVal) ||
+        Attrs.hasParamAttribute(I, Attribute::SwiftSelf) ||
+        Attrs.hasParamAttribute(I, Attribute::SwiftError) ||
+        Attrs.hasParamAttribute(I, Attribute::InAlloca) ||
+        Attrs.hasParamAttribute(I, Attribute::Nest))
       return false;
 
     Type *ArgTy = Arg.getType();
@@ -699,10 +698,10 @@ bool WebAssemblyFastISel::fastLowerArguments() {
     }
     unsigned ResultReg = createResultReg(RC);
     BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(Opc), ResultReg)
-        .addImm(i);
+        .addImm(I);
     updateValueMap(&Arg, ResultReg);
 
-    ++i;
+    ++I;
   }
 
   MRI.addLiveIn(WebAssembly::ARGUMENTS);
@@ -731,7 +730,7 @@ bool WebAssemblyFastISel::fastLowerArguments() {
 }
 
 bool WebAssemblyFastISel::selectCall(const Instruction *I) {
-  const CallInst *Call = cast<CallInst>(I);
+  const auto *Call = cast<CallInst>(I);
 
   if (Call->isMustTailCall() || Call->isInlineAsm() ||
       Call->getFunctionType()->isVarArg())
@@ -817,25 +816,25 @@ bool WebAssemblyFastISel::selectCall(const Instruction *I) {
   }
 
   SmallVector<unsigned, 8> Args;
-  for (unsigned i = 0, e = Call->getNumArgOperands(); i < e; ++i) {
-    Value *V = Call->getArgOperand(i);
+  for (unsigned I = 0, E = Call->getNumArgOperands(); I < E; ++I) {
+    Value *V = Call->getArgOperand(I);
     MVT::SimpleValueType ArgTy = getSimpleType(V->getType());
     if (ArgTy == MVT::INVALID_SIMPLE_VALUE_TYPE)
       return false;
 
     const AttributeList &Attrs = Call->getAttributes();
-    if (Attrs.hasParamAttribute(i, Attribute::ByVal) ||
-        Attrs.hasParamAttribute(i, Attribute::SwiftSelf) ||
-        Attrs.hasParamAttribute(i, Attribute::SwiftError) ||
-        Attrs.hasParamAttribute(i, Attribute::InAlloca) ||
-        Attrs.hasParamAttribute(i, Attribute::Nest))
+    if (Attrs.hasParamAttribute(I, Attribute::ByVal) ||
+        Attrs.hasParamAttribute(I, Attribute::SwiftSelf) ||
+        Attrs.hasParamAttribute(I, Attribute::SwiftError) ||
+        Attrs.hasParamAttribute(I, Attribute::InAlloca) ||
+        Attrs.hasParamAttribute(I, Attribute::Nest))
       return false;
 
     unsigned Reg;
 
-    if (Attrs.hasParamAttribute(i, Attribute::SExt))
+    if (Attrs.hasParamAttribute(I, Attribute::SExt))
       Reg = getRegForSignedValue(V);
-    else if (Attrs.hasParamAttribute(i, Attribute::ZExt))
+    else if (Attrs.hasParamAttribute(I, Attribute::ZExt))
       Reg = getRegForUnsignedValue(V);
     else
       Reg = getRegForValue(V);
@@ -869,7 +868,7 @@ bool WebAssemblyFastISel::selectCall(const Instruction *I) {
 }
 
 bool WebAssemblyFastISel::selectSelect(const Instruction *I) {
-  const SelectInst *Select = cast<SelectInst>(I);
+  const auto *Select = cast<SelectInst>(I);
 
   bool Not;
   unsigned CondReg = getRegForI1Value(Select->getCondition(), Not);
@@ -928,7 +927,7 @@ bool WebAssemblyFastISel::selectSelect(const Instruction *I) {
 }
 
 bool WebAssemblyFastISel::selectTrunc(const Instruction *I) {
-  const TruncInst *Trunc = cast<TruncInst>(I);
+  const auto *Trunc = cast<TruncInst>(I);
 
   unsigned Reg = getRegForValue(Trunc->getOperand(0));
   if (Reg == 0)
@@ -947,7 +946,7 @@ bool WebAssemblyFastISel::selectTrunc(const Instruction *I) {
 }
 
 bool WebAssemblyFastISel::selectZExt(const Instruction *I) {
-  const ZExtInst *ZExt = cast<ZExtInst>(I);
+  const auto *ZExt = cast<ZExtInst>(I);
 
   const Value *Op = ZExt->getOperand(0);
   MVT::SimpleValueType From = getSimpleType(Op->getType());
@@ -964,7 +963,7 @@ bool WebAssemblyFastISel::selectZExt(const Instruction *I) {
 }
 
 bool WebAssemblyFastISel::selectSExt(const Instruction *I) {
-  const SExtInst *SExt = cast<SExtInst>(I);
+  const auto *SExt = cast<SExtInst>(I);
 
   const Value *Op = SExt->getOperand(0);
   MVT::SimpleValueType From = getSimpleType(Op->getType());
@@ -981,11 +980,11 @@ bool WebAssemblyFastISel::selectSExt(const Instruction *I) {
 }
 
 bool WebAssemblyFastISel::selectICmp(const Instruction *I) {
-  const ICmpInst *ICmp = cast<ICmpInst>(I);
+  const auto *ICmp = cast<ICmpInst>(I);
 
   bool I32 = getSimpleType(ICmp->getOperand(0)->getType()) != MVT::i64;
   unsigned Opc;
-  bool isSigned = false;
+  bool IsSigned = false;
   switch (ICmp->getPredicate()) {
   case ICmpInst::ICMP_EQ:
     Opc = I32 ? WebAssembly::EQ_I32 : WebAssembly::EQ_I64;
@@ -1007,29 +1006,29 @@ bool WebAssemblyFastISel::selectICmp(const Instruction *I) {
     break;
   case ICmpInst::ICMP_SGT:
     Opc = I32 ? WebAssembly::GT_S_I32 : WebAssembly::GT_S_I64;
-    isSigned = true;
+    IsSigned = true;
     break;
   case ICmpInst::ICMP_SGE:
     Opc = I32 ? WebAssembly::GE_S_I32 : WebAssembly::GE_S_I64;
-    isSigned = true;
+    IsSigned = true;
     break;
   case ICmpInst::ICMP_SLT:
     Opc = I32 ? WebAssembly::LT_S_I32 : WebAssembly::LT_S_I64;
-    isSigned = true;
+    IsSigned = true;
     break;
   case ICmpInst::ICMP_SLE:
     Opc = I32 ? WebAssembly::LE_S_I32 : WebAssembly::LE_S_I64;
-    isSigned = true;
+    IsSigned = true;
     break;
   default:
     return false;
   }
 
-  unsigned LHS = getRegForPromotedValue(ICmp->getOperand(0), isSigned);
+  unsigned LHS = getRegForPromotedValue(ICmp->getOperand(0), IsSigned);
   if (LHS == 0)
     return false;
 
-  unsigned RHS = getRegForPromotedValue(ICmp->getOperand(1), isSigned);
+  unsigned RHS = getRegForPromotedValue(ICmp->getOperand(1), IsSigned);
   if (RHS == 0)
     return false;
 
@@ -1042,7 +1041,7 @@ bool WebAssemblyFastISel::selectICmp(const Instruction *I) {
 }
 
 bool WebAssemblyFastISel::selectFCmp(const Instruction *I) {
-  const FCmpInst *FCmp = cast<FCmpInst>(I);
+  const auto *FCmp = cast<FCmpInst>(I);
 
   unsigned LHS = getRegForValue(FCmp->getOperand(0));
   if (LHS == 0)
@@ -1138,7 +1137,7 @@ bool WebAssemblyFastISel::selectBitCast(const Instruction *I) {
 }
 
 bool WebAssemblyFastISel::selectLoad(const Instruction *I) {
-  const LoadInst *Load = cast<LoadInst>(I);
+  const auto *Load = cast<LoadInst>(I);
   if (Load->isAtomic())
     return false;
   if (!Subtarget->hasSIMD128() && Load->getType()->isVectorTy())
@@ -1195,7 +1194,7 @@ bool WebAssemblyFastISel::selectLoad(const Instruction *I) {
 }
 
 bool WebAssemblyFastISel::selectStore(const Instruction *I) {
-  const StoreInst *Store = cast<StoreInst>(I);
+  const auto *Store = cast<StoreInst>(I);
   if (Store->isAtomic())
     return false;
   if (!Subtarget->hasSIMD128() &&
@@ -1251,7 +1250,7 @@ bool WebAssemblyFastISel::selectStore(const Instruction *I) {
 }
 
 bool WebAssemblyFastISel::selectBr(const Instruction *I) {
-  const BranchInst *Br = cast<BranchInst>(I);
+  const auto *Br = cast<BranchInst>(I);
   if (Br->isUnconditional()) {
     MachineBasicBlock *MSucc = FuncInfo.MBBMap[Br->getSuccessor(0)];
     fastEmitBranch(MSucc, Br->getDebugLoc());
@@ -1282,7 +1281,7 @@ bool WebAssemblyFastISel::selectRet(const Instruction *I) {
   if (!FuncInfo.CanLowerReturn)
     return false;
 
-  const ReturnInst *Ret = cast<ReturnInst>(I);
+  const auto *Ret = cast<ReturnInst>(I);
 
   if (Ret->getNumOperands() == 0) {
     BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
diff --git a/lib/Target/WebAssembly/WebAssemblyFixFunctionBitcasts.cpp b/lib/Target/WebAssembly/WebAssemblyFixFunctionBitcasts.cpp
index 0644f1232f6c3ded6719ab21e561eeda4ed02fbb..b7fc65401fc48594c3830bb863d7d790b0094a3a 100644
--- a/lib/Target/WebAssembly/WebAssemblyFixFunctionBitcasts.cpp
+++ b/lib/Target/WebAssembly/WebAssemblyFixFunctionBitcasts.cpp
@@ -1,9 +1,8 @@
 //===-- WebAssemblyFixFunctionBitcasts.cpp - Fix function bitcasts --------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
@@ -36,11 +35,6 @@ using namespace llvm;
 
 #define DEBUG_TYPE "wasm-fix-function-bitcasts"
 
-static cl::opt<bool>
-    TemporaryWorkarounds("wasm-temporary-workarounds",
-                         cl::desc("Apply certain temporary workarounds"),
-                         cl::init(true), cl::Hidden);
-
 namespace {
 class FixFunctionBitcasts final : public ModulePass {
   StringRef getPassName() const override {
@@ -70,12 +64,12 @@ ModulePass *llvm::createWebAssemblyFixFunctionBitcasts() {
 
 // Recursively descend the def-use lists from V to find non-bitcast users of
 // bitcasts of V.
-static void FindUses(Value *V, Function &F,
+static void findUses(Value *V, Function &F,
                      SmallVectorImpl<std::pair<Use *, Function *>> &Uses,
                      SmallPtrSetImpl<Constant *> &ConstantBCs) {
   for (Use &U : V->uses()) {
-    if (BitCastOperator *BC = dyn_cast<BitCastOperator>(U.getUser()))
-      FindUses(BC, F, Uses, ConstantBCs);
+    if (auto *BC = dyn_cast<BitCastOperator>(U.getUser()))
+      findUses(BC, F, Uses, ConstantBCs);
     else if (U.get()->getType() != F.getType()) {
       CallSite CS(U.getUser());
       if (!CS)
@@ -87,8 +81,8 @@ static void FindUses(Value *V, Function &F,
         continue;
       if (isa<Constant>(U.get())) {
         // Only add constant bitcasts to the list once; they get RAUW'd
-        auto c = ConstantBCs.insert(cast<Constant>(U.get()));
-        if (!c.second)
+        auto C = ConstantBCs.insert(cast<Constant>(U.get()));
+        if (!C.second)
           continue;
       }
       Uses.push_back(std::make_pair(&U, &F));
@@ -119,7 +113,7 @@ static void FindUses(Value *V, Function &F,
 // For bitcasts that involve struct types we don't know at this stage if they
 // would be equivalent at the wasm level and so we can't know if we need to
 // generate a wrapper.
-static Function *CreateWrapper(Function *F, FunctionType *Ty) {
+static Function *createWrapper(Function *F, FunctionType *Ty) {
   Module *M = F->getParent();
 
   Function *Wrapper = Function::Create(Ty, Function::PrivateLinkage,
@@ -157,11 +151,11 @@ static Function *CreateWrapper(Function *F, FunctionType *Ty) {
         BB->getInstList().push_back(PtrCast);
         Args.push_back(PtrCast);
       } else if (ArgType->isStructTy() || ParamType->isStructTy()) {
-        LLVM_DEBUG(dbgs() << "CreateWrapper: struct param type in bitcast: "
+        LLVM_DEBUG(dbgs() << "createWrapper: struct param type in bitcast: "
                           << F->getName() << "\n");
         WrapperNeeded = false;
       } else {
-        LLVM_DEBUG(dbgs() << "CreateWrapper: arg type mismatch calling: "
+        LLVM_DEBUG(dbgs() << "createWrapper: arg type mismatch calling: "
                           << F->getName() << "\n");
         LLVM_DEBUG(dbgs() << "Arg[" << Args.size() << "] Expected: "
                           << *ParamType << " Got: " << *ArgType << "\n");
@@ -197,11 +191,11 @@ static Function *CreateWrapper(Function *F, FunctionType *Ty) {
       BB->getInstList().push_back(Cast);
       ReturnInst::Create(M->getContext(), Cast, BB);
     } else if (RtnType->isStructTy() || ExpectedRtnType->isStructTy()) {
-      LLVM_DEBUG(dbgs() << "CreateWrapper: struct return type in bitcast: "
+      LLVM_DEBUG(dbgs() << "createWrapper: struct return type in bitcast: "
                         << F->getName() << "\n");
       WrapperNeeded = false;
     } else {
-      LLVM_DEBUG(dbgs() << "CreateWrapper: return type mismatch calling: "
+      LLVM_DEBUG(dbgs() << "createWrapper: return type mismatch calling: "
                         << F->getName() << "\n");
       LLVM_DEBUG(dbgs() << "Expected: " << *ExpectedRtnType
                         << " Got: " << *RtnType << "\n");
@@ -218,16 +212,29 @@ static Function *CreateWrapper(Function *F, FunctionType *Ty) {
     new UnreachableInst(M->getContext(), BB);
     Wrapper->setName(F->getName() + "_bitcast_invalid");
   } else if (!WrapperNeeded) {
-    LLVM_DEBUG(dbgs() << "CreateWrapper: no wrapper needed: " << F->getName()
+    LLVM_DEBUG(dbgs() << "createWrapper: no wrapper needed: " << F->getName()
                       << "\n");
     Wrapper->eraseFromParent();
     return nullptr;
   }
-  LLVM_DEBUG(dbgs() << "CreateWrapper: " << F->getName() << "\n");
+  LLVM_DEBUG(dbgs() << "createWrapper: " << F->getName() << "\n");
   return Wrapper;
 }
 
+// Test whether a main function with type FuncTy should be rewritten to have
+// type MainTy.
+static bool shouldFixMainFunction(FunctionType *FuncTy, FunctionType *MainTy) {
+  // Only fix the main function if it's the standard zero-arg form. That way,
+  // the standard cases will work as expected, and users will see signature
+  // mismatches from the linker for non-standard cases.
+  return FuncTy->getReturnType() == MainTy->getReturnType() &&
+         FuncTy->getNumParams() == 0 &&
+         !FuncTy->isVarArg();
+}
+
 bool FixFunctionBitcasts::runOnModule(Module &M) {
+  LLVM_DEBUG(dbgs() << "********** Fix Function Bitcasts **********\n");
+
   Function *Main = nullptr;
   CallInst *CallMain = nullptr;
   SmallVector<std::pair<Use *, Function *>, 0> Uses;
@@ -235,27 +242,27 @@ bool FixFunctionBitcasts::runOnModule(Module &M) {
 
   // Collect all the places that need wrappers.
   for (Function &F : M) {
-    FindUses(&F, F, Uses, ConstantBCs);
+    findUses(&F, F, Uses, ConstantBCs);
 
     // If we have a "main" function, and its type isn't
     // "int main(int argc, char *argv[])", create an artificial call with it
     // bitcasted to that type so that we generate a wrapper for it, so that
     // the C runtime can call it.
-    if (!TemporaryWorkarounds && !F.isDeclaration() && F.getName() == "main") {
+    if (F.getName() == "main") {
       Main = &F;
       LLVMContext &C = M.getContext();
       Type *MainArgTys[] = {Type::getInt32Ty(C),
                             PointerType::get(Type::getInt8PtrTy(C), 0)};
       FunctionType *MainTy = FunctionType::get(Type::getInt32Ty(C), MainArgTys,
                                                /*isVarArg=*/false);
-      if (F.getFunctionType() != MainTy) {
+      if (shouldFixMainFunction(F.getFunctionType(), MainTy)) {
         LLVM_DEBUG(dbgs() << "Found `main` function with incorrect type: "
                           << *F.getFunctionType() << "\n");
         Value *Args[] = {UndefValue::get(MainArgTys[0]),
                          UndefValue::get(MainArgTys[1])};
         Value *Casted =
             ConstantExpr::getBitCast(Main, PointerType::get(MainTy, 0));
-        CallMain = CallInst::Create(Casted, Args, "call_main");
+        CallMain = CallInst::Create(MainTy, Casted, Args, "call_main");
         Use *UseMain = &CallMain->getOperandUse(2);
         Uses.push_back(std::make_pair(UseMain, &F));
       }
@@ -267,8 +274,8 @@ bool FixFunctionBitcasts::runOnModule(Module &M) {
   for (auto &UseFunc : Uses) {
     Use *U = UseFunc.first;
     Function *F = UseFunc.second;
-    PointerType *PTy = cast<PointerType>(U->get()->getType());
-    FunctionType *Ty = dyn_cast<FunctionType>(PTy->getElementType());
+    auto *PTy = cast<PointerType>(U->get()->getType());
+    auto *Ty = dyn_cast<FunctionType>(PTy->getElementType());
 
     // If the function is casted to something like i8* as a "generic pointer"
     // to be later casted to something else, we can't generate a wrapper for it.
@@ -278,7 +285,7 @@ bool FixFunctionBitcasts::runOnModule(Module &M) {
 
     auto Pair = Wrappers.insert(std::make_pair(std::make_pair(F, Ty), nullptr));
     if (Pair.second)
-      Pair.first->second = CreateWrapper(F, Ty);
+      Pair.first->second = createWrapper(F, Ty);
 
     Function *Wrapper = Pair.first->second;
     if (!Wrapper)
@@ -294,14 +301,20 @@ bool FixFunctionBitcasts::runOnModule(Module &M) {
   // one that gets called from startup.
   if (CallMain) {
     Main->setName("__original_main");
-    Function *MainWrapper =
+    auto *MainWrapper =
         cast<Function>(CallMain->getCalledValue()->stripPointerCasts());
-    MainWrapper->setName("main");
-    MainWrapper->setLinkage(Main->getLinkage());
-    MainWrapper->setVisibility(Main->getVisibility());
-    Main->setLinkage(Function::PrivateLinkage);
-    Main->setVisibility(Function::DefaultVisibility);
     delete CallMain;
+    if (Main->isDeclaration()) {
+      // The wrapper is not needed in this case as we don't need to export
+      // it to anyone else.
+      MainWrapper->eraseFromParent();
+    } else {
+      // Otherwise give the wrapper the same linkage as the original main
+      // function, so that it can be called from the same places.
+      MainWrapper->setName("main");
+      MainWrapper->setLinkage(Main->getLinkage());
+      MainWrapper->setVisibility(Main->getVisibility());
+    }
   }
 
   return true;
diff --git a/lib/Target/WebAssembly/WebAssemblyFixIrreducibleControlFlow.cpp b/lib/Target/WebAssembly/WebAssemblyFixIrreducibleControlFlow.cpp
index bea027be7711bf09e42acd85c439257aee81c02e..17fb8e19091a0b8aab36f1665a388faabfbaff0f 100644
--- a/lib/Target/WebAssembly/WebAssemblyFixIrreducibleControlFlow.cpp
+++ b/lib/Target/WebAssembly/WebAssemblyFixIrreducibleControlFlow.cpp
@@ -1,15 +1,14 @@
 //=- WebAssemblyFixIrreducibleControlFlow.cpp - Fix irreducible control flow -//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
 /// \file
-/// This file implements a pass that transforms irreducible control flow
-/// into reducible control flow. Irreducible control flow means multiple-entry
+/// This file implements a pass that transforms irreducible control flow into
+/// reducible control flow. Irreducible control flow means multiple-entry
 /// loops; they appear as CFG cycles that are not recorded in MachineLoopInfo
 /// due to being unnatural.
 ///
@@ -17,12 +16,36 @@
 /// it linearizes control flow, turning diamonds into two triangles, which is
 /// both unnecessary and undesirable for WebAssembly.
 ///
-/// TODO: The transformation implemented here handles all irreducible control
-/// flow, without exponential code-size expansion, though it does so by creating
-/// inefficient code in many cases. Ideally, we should add other
-/// transformations, including code-duplicating cases, which can be more
-/// efficient in common cases, and they can fall back to this conservative
-/// implementation as needed.
+/// The big picture: Ignoring natural loops (seeing them monolithically), we
+/// find all the blocks which can return to themselves ("loopers"). Loopers
+/// reachable from the non-loopers are loop entries: if there are 2 or more,
+/// then we have irreducible control flow. We fix that as follows: a new block
+/// is created that can dispatch to each of the loop entries, based on the
+/// value of a label "helper" variable, and we replace direct branches to the
+/// entries with assignments to the label variable and a branch to the dispatch
+/// block. Then the dispatch block is the single entry in a new natural loop.
+///
+/// This is similar to what the Relooper [1] does, both identify looping code
+/// that requires multiple entries, and resolve it in a similar way. In
+/// Relooper terminology, we implement a Multiple shape in a Loop shape. Note
+/// also that like the Relooper, we implement a "minimal" intervention: we only
+/// use the "label" helper for the blocks we absolutely must and no others. We
+/// also prioritize code size and do not perform node splitting (i.e. we don't
+/// duplicate code in order to resolve irreducibility).
+///
+/// The difference between this code and the Relooper is that the Relooper also
+/// generates ifs and loops and works in a recursive manner, knowing at each
+/// point what the entries are, and recursively breaks down the problem. Here
+/// we just want to resolve irreducible control flow, and we also want to use
+/// as much LLVM infrastructure as possible. So we use the MachineLoopInfo to
+/// identify natural loops, etc., and we start with the whole CFG and must
+/// identify both the looping code and its entries.
+///
+/// [1] Alon Zakai. 2011. Emscripten: an LLVM-to-JavaScript compiler. In
+/// Proceedings of the ACM international conference companion on Object oriented
+/// programming systems languages and applications companion (SPLASH '11). ACM,
+/// New York, NY, USA, 301-312. DOI=10.1145/2048147.2048224
+/// http://doi.acm.org/10.1145/2048147.2048224
 ///
 //===----------------------------------------------------------------------===//
 
@@ -46,141 +69,203 @@ using namespace llvm;
 #define DEBUG_TYPE "wasm-fix-irreducible-control-flow"
 
 namespace {
-class WebAssemblyFixIrreducibleControlFlow final : public MachineFunctionPass {
-  StringRef getPassName() const override {
-    return "WebAssembly Fix Irreducible Control Flow";
-  }
-
-  void getAnalysisUsage(AnalysisUsage &AU) const override {
-    AU.setPreservesCFG();
-    AU.addRequired<MachineDominatorTree>();
-    AU.addPreserved<MachineDominatorTree>();
-    AU.addRequired<MachineLoopInfo>();
-    AU.addPreserved<MachineLoopInfo>();
-    MachineFunctionPass::getAnalysisUsage(AU);
-  }
-
-  bool runOnMachineFunction(MachineFunction &MF) override;
-
-  bool VisitLoop(MachineFunction &MF, MachineLoopInfo &MLI, MachineLoop *Loop);
 
+class LoopFixer {
 public:
-  static char ID; // Pass identification, replacement for typeid
-  WebAssemblyFixIrreducibleControlFlow() : MachineFunctionPass(ID) {}
-};
-} // end anonymous namespace
-
-char WebAssemblyFixIrreducibleControlFlow::ID = 0;
-INITIALIZE_PASS(WebAssemblyFixIrreducibleControlFlow, DEBUG_TYPE,
-                "Removes irreducible control flow", false, false)
-
-FunctionPass *llvm::createWebAssemblyFixIrreducibleControlFlow() {
-  return new WebAssemblyFixIrreducibleControlFlow();
-}
-
-namespace {
-
-/// A utility for walking the blocks of a loop, handling a nested inner
-/// loop as a monolithic conceptual block.
-class MetaBlock {
-  MachineBasicBlock *Block;
-  SmallVector<MachineBasicBlock *, 2> Preds;
-  SmallVector<MachineBasicBlock *, 2> Succs;
-
-public:
-  explicit MetaBlock(MachineBasicBlock *MBB)
-      : Block(MBB), Preds(MBB->pred_begin(), MBB->pred_end()),
-        Succs(MBB->succ_begin(), MBB->succ_end()) {}
-
-  explicit MetaBlock(MachineLoop *Loop) : Block(Loop->getHeader()) {
-    Loop->getExitBlocks(Succs);
-    for (MachineBasicBlock *Pred : Block->predecessors())
-      if (!Loop->contains(Pred))
-        Preds.push_back(Pred);
+  LoopFixer(MachineFunction &MF, MachineLoopInfo &MLI, MachineLoop *Loop)
+      : MF(MF), MLI(MLI), Loop(Loop) {}
+
+  // Run the fixer on the given inputs. Returns whether changes were made.
+  bool run();
+
+private:
+  MachineFunction &MF;
+  MachineLoopInfo &MLI;
+  MachineLoop *Loop;
+
+  MachineBasicBlock *Header;
+  SmallPtrSet<MachineBasicBlock *, 4> LoopBlocks;
+
+  using BlockSet = SmallPtrSet<MachineBasicBlock *, 4>;
+  DenseMap<MachineBasicBlock *, BlockSet> Reachable;
+
+  // The worklist contains pairs of recent additions, (a, b), where we just
+  // added a link a => b.
+  using BlockPair = std::pair<MachineBasicBlock *, MachineBasicBlock *>;
+  SmallVector<BlockPair, 4> WorkList;
+
+  // Get a canonical block to represent a block or a loop: the block, or if in
+  // an inner loop, the loop header, of it in an outer loop scope, we can
+  // ignore it. We need to call this on all blocks we work on.
+  MachineBasicBlock *canonicalize(MachineBasicBlock *MBB) {
+    MachineLoop *InnerLoop = MLI.getLoopFor(MBB);
+    if (InnerLoop == Loop) {
+      return MBB;
+    } else {
+      // This is either in an outer or an inner loop, and not in ours.
+      if (!LoopBlocks.count(MBB)) {
+        // It's in outer code, ignore it.
+        return nullptr;
+      }
+      assert(InnerLoop);
+      // It's in an inner loop, canonicalize it to the header of that loop.
+      return InnerLoop->getHeader();
+    }
   }
 
-  MachineBasicBlock *getBlock() const { return Block; }
-
-  const SmallVectorImpl<MachineBasicBlock *> &predecessors() const {
-    return Preds;
-  }
-  const SmallVectorImpl<MachineBasicBlock *> &successors() const {
-    return Succs;
+  // For a successor we can additionally ignore it if it's a branch back to a
+  // natural loop top, as when we are in the scope of a loop, we just care
+  // about internal irreducibility, and can ignore the loop we are in. We need
+  // to call this on all blocks in a context where they are a successor.
+  MachineBasicBlock *canonicalizeSuccessor(MachineBasicBlock *MBB) {
+    if (Loop && MBB == Loop->getHeader()) {
+      // Ignore branches going to the loop's natural header.
+      return nullptr;
+    }
+    return canonicalize(MBB);
   }
 
-  bool operator==(const MetaBlock &MBB) { return Block == MBB.Block; }
-  bool operator!=(const MetaBlock &MBB) { return Block != MBB.Block; }
+  // Potentially insert a new reachable edge, and if so, note it as further
+  // work.
+  void maybeInsert(MachineBasicBlock *MBB, MachineBasicBlock *Succ) {
+    assert(MBB == canonicalize(MBB));
+    assert(Succ);
+    // Succ may not be interesting as a sucessor.
+    Succ = canonicalizeSuccessor(Succ);
+    if (!Succ)
+      return;
+    if (Reachable[MBB].insert(Succ).second) {
+      // For there to be further work, it means that we have
+      //   X => MBB => Succ
+      // for some other X, and in that case X => Succ would be a new edge for
+      // us to discover later. However, if we don't care about MBB as a
+      // successor, then we don't care about that anyhow.
+      if (canonicalizeSuccessor(MBB)) {
+        WorkList.emplace_back(MBB, Succ);
+      }
+    }
+  }
 };
 
-class SuccessorList final : public MetaBlock {
-  size_t Index;
-  size_t Num;
+bool LoopFixer::run() {
+  Header = Loop ? Loop->getHeader() : &*MF.begin();
 
-public:
-  explicit SuccessorList(MachineBasicBlock *MBB)
-      : MetaBlock(MBB), Index(0), Num(successors().size()) {}
+  // Identify all the blocks in this loop scope.
+  if (Loop) {
+    for (auto *MBB : Loop->getBlocks()) {
+      LoopBlocks.insert(MBB);
+    }
+  } else {
+    for (auto &MBB : MF) {
+      LoopBlocks.insert(&MBB);
+    }
+  }
 
-  explicit SuccessorList(MachineLoop *Loop)
-      : MetaBlock(Loop), Index(0), Num(successors().size()) {}
+  // Compute which (canonicalized) blocks each block can reach.
 
-  bool HasNext() const { return Index != Num; }
+  // Add all the initial work.
+  for (auto *MBB : LoopBlocks) {
+    MachineLoop *InnerLoop = MLI.getLoopFor(MBB);
 
-  MachineBasicBlock *Next() {
-    assert(HasNext());
-    return successors()[Index++];
+    if (InnerLoop == Loop) {
+      for (auto *Succ : MBB->successors()) {
+        maybeInsert(MBB, Succ);
+      }
+    } else {
+      // It can't be in an outer loop - we loop on LoopBlocks - and so it must
+      // be an inner loop.
+      assert(InnerLoop);
+      // Check if we are the canonical block for this loop.
+      if (canonicalize(MBB) != MBB) {
+        continue;
+      }
+      // The successors are those of the loop.
+      SmallVector<MachineBasicBlock *, 2> ExitBlocks;
+      InnerLoop->getExitBlocks(ExitBlocks);
+      for (auto *Succ : ExitBlocks) {
+        maybeInsert(MBB, Succ);
+      }
+    }
   }
-};
 
-} // end anonymous namespace
+  // Do work until we are all done.
+  while (!WorkList.empty()) {
+    MachineBasicBlock *MBB;
+    MachineBasicBlock *Succ;
+    std::tie(MBB, Succ) = WorkList.pop_back_val();
+    // The worklist item is an edge we just added, so it must have valid blocks
+    // (and not something canonicalized to nullptr).
+    assert(MBB);
+    assert(Succ);
+    // The successor in that pair must also be a valid successor.
+    assert(MBB == canonicalizeSuccessor(MBB));
+    // We recently added MBB => Succ, and that means we may have enabled
+    // Pred => MBB => Succ. Check all the predecessors. Note that our loop here
+    // is correct for both a block and a block representing a loop, as the loop
+    // is natural and so the predecessors are all predecessors of the loop
+    // header, which is the block we have here.
+    for (auto *Pred : MBB->predecessors()) {
+      // Canonicalize, make sure it's relevant, and check it's not the same
+      // block (an update to the block itself doesn't help compute that same
+      // block).
+      Pred = canonicalize(Pred);
+      if (Pred && Pred != MBB) {
+        maybeInsert(Pred, Succ);
+      }
+    }
+  }
 
-bool WebAssemblyFixIrreducibleControlFlow::VisitLoop(MachineFunction &MF,
-                                                     MachineLoopInfo &MLI,
-                                                     MachineLoop *Loop) {
-  MachineBasicBlock *Header = Loop ? Loop->getHeader() : &*MF.begin();
-  SetVector<MachineBasicBlock *> RewriteSuccs;
-
-  // DFS through Loop's body, looking for irreducible control flow. Loop is
-  // natural, and we stay in its body, and we treat any nested loops
-  // monolithically, so any cycles we encounter indicate irreducibility.
-  SmallPtrSet<MachineBasicBlock *, 8> OnStack;
-  SmallPtrSet<MachineBasicBlock *, 8> Visited;
-  SmallVector<SuccessorList, 4> LoopWorklist;
-  LoopWorklist.push_back(SuccessorList(Header));
-  OnStack.insert(Header);
-  Visited.insert(Header);
-  while (!LoopWorklist.empty()) {
-    SuccessorList &Top = LoopWorklist.back();
-    if (Top.HasNext()) {
-      MachineBasicBlock *Next = Top.Next();
-      if (Next == Header || (Loop && !Loop->contains(Next)))
-        continue;
-      if (LLVM_LIKELY(OnStack.insert(Next).second)) {
-        if (!Visited.insert(Next).second) {
-          OnStack.erase(Next);
-          continue;
-        }
-        MachineLoop *InnerLoop = MLI.getLoopFor(Next);
-        if (InnerLoop != Loop)
-          LoopWorklist.push_back(SuccessorList(InnerLoop));
-        else
-          LoopWorklist.push_back(SuccessorList(Next));
-      } else {
-        RewriteSuccs.insert(Top.getBlock());
+  // It's now trivial to identify the loopers.
+  SmallPtrSet<MachineBasicBlock *, 4> Loopers;
+  for (auto MBB : LoopBlocks) {
+    if (Reachable[MBB].count(MBB)) {
+      Loopers.insert(MBB);
+    }
+  }
+  // The header cannot be a looper. At the toplevel, LLVM does not allow the
+  // entry to be in a loop, and in a natural loop we should ignore the header.
+  assert(Loopers.count(Header) == 0);
+
+  // Find the entries, loopers reachable from non-loopers.
+  SmallPtrSet<MachineBasicBlock *, 4> Entries;
+  SmallVector<MachineBasicBlock *, 4> SortedEntries;
+  for (auto *Looper : Loopers) {
+    for (auto *Pred : Looper->predecessors()) {
+      Pred = canonicalize(Pred);
+      if (Pred && !Loopers.count(Pred)) {
+        Entries.insert(Looper);
+        SortedEntries.push_back(Looper);
+        break;
       }
-      continue;
     }
-    OnStack.erase(Top.getBlock());
-    LoopWorklist.pop_back();
   }
 
-  // Most likely, we didn't find any irreducible control flow.
-  if (LLVM_LIKELY(RewriteSuccs.empty()))
+  // Check if we found irreducible control flow.
+  if (LLVM_LIKELY(Entries.size() <= 1))
     return false;
 
-  LLVM_DEBUG(dbgs() << "Irreducible control flow detected!\n");
+  // Sort the entries to ensure a deterministic build.
+  llvm::sort(SortedEntries,
+             [&](const MachineBasicBlock *A, const MachineBasicBlock *B) {
+               auto ANum = A->getNumber();
+               auto BNum = B->getNumber();
+               return ANum < BNum;
+             });
+
+#ifndef NDEBUG
+  for (auto Block : SortedEntries)
+    assert(Block->getNumber() != -1);
+  if (SortedEntries.size() > 1) {
+    for (auto I = SortedEntries.begin(), E = SortedEntries.end() - 1;
+         I != E; ++I) {
+      auto ANum = (*I)->getNumber();
+      auto BNum = (*(std::next(I)))->getNumber();
+      assert(ANum != BNum);
+    }
+  }
+#endif
 
-  // Ok. We have irreducible control flow! Create a dispatch block which will
-  // contains a jump table to any block in the problematic set of blocks.
+  // Create a dispatch block which will contain a jump table to the entries.
   MachineBasicBlock *Dispatch = MF.CreateMachineBasicBlock();
   MF.insert(MF.end(), Dispatch);
   MLI.changeLoopFor(Dispatch, Loop);
@@ -196,43 +281,43 @@ bool WebAssemblyFixIrreducibleControlFlow::VisitLoop(MachineFunction &MF,
   unsigned Reg = MRI.createVirtualRegister(&WebAssembly::I32RegClass);
   MIB.addReg(Reg);
 
-  // Collect all the blocks which need to have their successors rewritten,
-  // add the successors to the jump table, and remember their index.
+  // Compute the indices in the superheader, one for each bad block, and
+  // add them as successors.
   DenseMap<MachineBasicBlock *, unsigned> Indices;
-  SmallVector<MachineBasicBlock *, 4> SuccWorklist(RewriteSuccs.begin(),
-                                                   RewriteSuccs.end());
-  while (!SuccWorklist.empty()) {
-    MachineBasicBlock *MBB = SuccWorklist.pop_back_val();
+  for (auto *MBB : SortedEntries) {
     auto Pair = Indices.insert(std::make_pair(MBB, 0));
-    if (!Pair.second)
+    if (!Pair.second) {
       continue;
+    }
 
     unsigned Index = MIB.getInstr()->getNumExplicitOperands() - 1;
-    LLVM_DEBUG(dbgs() << printMBBReference(*MBB) << " has index " << Index
-                      << "\n");
-
     Pair.first->second = Index;
-    for (auto Pred : MBB->predecessors())
-      RewriteSuccs.insert(Pred);
 
     MIB.addMBB(MBB);
     Dispatch->addSuccessor(MBB);
+  }
 
-    MetaBlock Meta(MBB);
-    for (auto *Succ : Meta.successors())
-      if (Succ != Header && (!Loop || Loop->contains(Succ)))
-        SuccWorklist.push_back(Succ);
+  // Rewrite the problematic successors for every block that wants to reach the
+  // bad blocks. For simplicity, we just introduce a new block for every edge
+  // we need to rewrite. (Fancier things are possible.)
+
+  SmallVector<MachineBasicBlock *, 4> AllPreds;
+  for (auto *MBB : SortedEntries) {
+    for (auto *Pred : MBB->predecessors()) {
+      if (Pred != Dispatch) {
+        AllPreds.push_back(Pred);
+      }
+    }
   }
 
-  // Rewrite the problematic successors for every block in RewriteSuccs.
-  // For simplicity, we just introduce a new block for every edge we need to
-  // rewrite. Fancier things are possible.
-  for (MachineBasicBlock *MBB : RewriteSuccs) {
+  for (MachineBasicBlock *MBB : AllPreds) {
     DenseMap<MachineBasicBlock *, MachineBasicBlock *> Map;
     for (auto *Succ : MBB->successors()) {
-      if (!Indices.count(Succ))
+      if (!Entries.count(Succ)) {
         continue;
+      }
 
+      // This is a successor we need to rewrite.
       MachineBasicBlock *Split = MF.CreateMachineBasicBlock();
       MF.insert(MBB->isLayoutSuccessor(Succ) ? MachineFunction::iterator(Succ)
                                              : MF.end(),
@@ -266,6 +351,55 @@ bool WebAssemblyFixIrreducibleControlFlow::VisitLoop(MachineFunction &MF,
   return true;
 }
 
+class WebAssemblyFixIrreducibleControlFlow final : public MachineFunctionPass {
+  StringRef getPassName() const override {
+    return "WebAssembly Fix Irreducible Control Flow";
+  }
+
+  void getAnalysisUsage(AnalysisUsage &AU) const override {
+    AU.setPreservesCFG();
+    AU.addRequired<MachineDominatorTree>();
+    AU.addPreserved<MachineDominatorTree>();
+    AU.addRequired<MachineLoopInfo>();
+    AU.addPreserved<MachineLoopInfo>();
+    MachineFunctionPass::getAnalysisUsage(AU);
+  }
+
+  bool runOnMachineFunction(MachineFunction &MF) override;
+
+  bool runIteration(MachineFunction &MF, MachineLoopInfo &MLI) {
+    // Visit the function body, which is identified as a null loop.
+    if (LoopFixer(MF, MLI, nullptr).run()) {
+      return true;
+    }
+
+    // Visit all the loops.
+    SmallVector<MachineLoop *, 8> Worklist(MLI.begin(), MLI.end());
+    while (!Worklist.empty()) {
+      MachineLoop *Loop = Worklist.pop_back_val();
+      Worklist.append(Loop->begin(), Loop->end());
+      if (LoopFixer(MF, MLI, Loop).run()) {
+        return true;
+      }
+    }
+
+    return false;
+  }
+
+public:
+  static char ID; // Pass identification, replacement for typeid
+  WebAssemblyFixIrreducibleControlFlow() : MachineFunctionPass(ID) {}
+};
+} // end anonymous namespace
+
+char WebAssemblyFixIrreducibleControlFlow::ID = 0;
+INITIALIZE_PASS(WebAssemblyFixIrreducibleControlFlow, DEBUG_TYPE,
+                "Removes irreducible control flow", false, false)
+
+FunctionPass *llvm::createWebAssemblyFixIrreducibleControlFlow() {
+  return new WebAssemblyFixIrreducibleControlFlow();
+}
+
 bool WebAssemblyFixIrreducibleControlFlow::runOnMachineFunction(
     MachineFunction &MF) {
   LLVM_DEBUG(dbgs() << "********** Fixing Irreducible Control Flow **********\n"
@@ -275,24 +409,19 @@ bool WebAssemblyFixIrreducibleControlFlow::runOnMachineFunction(
   bool Changed = false;
   auto &MLI = getAnalysis<MachineLoopInfo>();
 
-  // Visit the function body, which is identified as a null loop.
-  Changed |= VisitLoop(MF, MLI, nullptr);
-
-  // Visit all the loops.
-  SmallVector<MachineLoop *, 8> Worklist(MLI.begin(), MLI.end());
-  while (!Worklist.empty()) {
-    MachineLoop *CurLoop = Worklist.pop_back_val();
-    Worklist.append(CurLoop->begin(), CurLoop->end());
-    Changed |= VisitLoop(MF, MLI, CurLoop);
-  }
-
-  // If we made any changes, completely recompute everything.
-  if (LLVM_UNLIKELY(Changed)) {
-    LLVM_DEBUG(dbgs() << "Recomputing dominators and loops.\n");
+  // When we modify something, bail out and recompute MLI, then start again, as
+  // we create a new natural loop when we resolve irreducible control flow, and
+  // other loops may become nested in it, etc. In practice this is not an issue
+  // because irreducible control flow is rare, only very few cycles are needed
+  // here.
+  while (LLVM_UNLIKELY(runIteration(MF, MLI))) {
+    // We rewrote part of the function; recompute MLI and start again.
+    LLVM_DEBUG(dbgs() << "Recomputing loops.\n");
     MF.getRegInfo().invalidateLiveness();
     MF.RenumberBlocks();
     getAnalysis<MachineDominatorTree>().runOnMachineFunction(MF);
     MLI.runOnMachineFunction(MF);
+    Changed = true;
   }
 
   return Changed;
diff --git a/lib/Target/WebAssembly/WebAssemblyFrameLowering.cpp b/lib/Target/WebAssembly/WebAssemblyFrameLowering.cpp
index 9db343042139efe7230adc49b7343f85d578c33d..0f4e1abc6d50db0780e586112c6ec508a656b65c 100644
--- a/lib/Target/WebAssembly/WebAssemblyFrameLowering.cpp
+++ b/lib/Target/WebAssembly/WebAssemblyFrameLowering.cpp
@@ -1,9 +1,8 @@
 //===-- WebAssemblyFrameLowering.cpp - WebAssembly Frame Lowering ----------==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
@@ -130,7 +129,7 @@ void WebAssemblyFrameLowering::writeSPToGlobal(
 
   const char *ES = "__stack_pointer";
   auto *SPSymbol = MF.createExternalSymbolName(ES);
-  BuildMI(MBB, InsertStore, DL, TII->get(WebAssembly::SET_GLOBAL_I32))
+  BuildMI(MBB, InsertStore, DL, TII->get(WebAssembly::GLOBAL_SET_I32))
       .addExternalSymbol(SPSymbol, WebAssemblyII::MO_SYMBOL_GLOBAL)
       .addReg(SrcReg);
 }
@@ -177,7 +176,7 @@ void WebAssemblyFrameLowering::emitPrologue(MachineFunction &MF,
 
   const char *ES = "__stack_pointer";
   auto *SPSymbol = MF.createExternalSymbolName(ES);
-  BuildMI(MBB, InsertPt, DL, TII->get(WebAssembly::GET_GLOBAL_I32), SPReg)
+  BuildMI(MBB, InsertPt, DL, TII->get(WebAssembly::GLOBAL_GET_I32), SPReg)
       .addExternalSymbol(SPSymbol, WebAssemblyII::MO_SYMBOL_GLOBAL);
 
   bool HasBP = hasBP(MF);
diff --git a/lib/Target/WebAssembly/WebAssemblyFrameLowering.h b/lib/Target/WebAssembly/WebAssemblyFrameLowering.h
index c6fa8261b03faf028089a1195f0e694c8a4b5381..daddd4ca16ff89e8356ac3219653e89ff231f172 100644
--- a/lib/Target/WebAssembly/WebAssemblyFrameLowering.h
+++ b/lib/Target/WebAssembly/WebAssemblyFrameLowering.h
@@ -1,9 +1,8 @@
 // WebAssemblyFrameLowering.h - TargetFrameLowering for WebAssembly -*- C++ -*-/
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
diff --git a/lib/Target/WebAssembly/WebAssemblyISD.def b/lib/Target/WebAssembly/WebAssemblyISD.def
index e987d7f7f43a12946c8c4a3b4400df45de9a03e4..5f1715508846b33f260fa9f9a473ce3315e2b306 100644
--- a/lib/Target/WebAssembly/WebAssemblyISD.def
+++ b/lib/Target/WebAssembly/WebAssemblyISD.def
@@ -1,9 +1,8 @@
 //- WebAssemblyISD.def - WebAssembly ISD ---------------------------*- C++ -*-//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
@@ -26,5 +25,6 @@ HANDLE_NODETYPE(VEC_SHL)
 HANDLE_NODETYPE(VEC_SHR_S)
 HANDLE_NODETYPE(VEC_SHR_U)
 HANDLE_NODETYPE(THROW)
+HANDLE_NODETYPE(MEMORY_COPY)
 
 // add memory opcodes starting at ISD::FIRST_TARGET_MEMORY_OPCODE here...
diff --git a/lib/Target/WebAssembly/WebAssemblyISelDAGToDAG.cpp b/lib/Target/WebAssembly/WebAssemblyISelDAGToDAG.cpp
index fdf3a30a5c0e2d9c9c7b84ccb9428d0899ecf81c..40ec91555e6e295febf4e8f30d3ba68bfd74cf91 100644
--- a/lib/Target/WebAssembly/WebAssemblyISelDAGToDAG.cpp
+++ b/lib/Target/WebAssembly/WebAssemblyISelDAGToDAG.cpp
@@ -1,9 +1,8 @@
 //- WebAssemblyISelDAGToDAG.cpp - A dag to dag inst selector for WebAssembly -//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
@@ -38,9 +37,9 @@ class WebAssemblyDAGToDAGISel final : public SelectionDAGISel {
   bool ForCodeSize;
 
 public:
-  WebAssemblyDAGToDAGISel(WebAssemblyTargetMachine &tm,
+  WebAssemblyDAGToDAGISel(WebAssemblyTargetMachine &TM,
                           CodeGenOpt::Level OptLevel)
-      : SelectionDAGISel(tm, OptLevel), Subtarget(nullptr), ForCodeSize(false) {
+      : SelectionDAGISel(TM, OptLevel), Subtarget(nullptr), ForCodeSize(false) {
   }
 
   StringRef getPassName() const override {
@@ -48,6 +47,10 @@ public:
   }
 
   bool runOnMachineFunction(MachineFunction &MF) override {
+    LLVM_DEBUG(dbgs() << "********** ISelDAGToDAG **********\n"
+                         "********** Function: "
+                      << MF.getName() << '\n');
+
     ForCodeSize = MF.getFunction().hasFnAttribute(Attribute::OptimizeForSize) ||
                   MF.getFunction().hasFnAttribute(Attribute::MinSize);
     Subtarget = &MF.getSubtarget<WebAssemblySubtarget>();
diff --git a/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp b/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp
index a0382157f76918dfe6ed4cf59d997b9b8a889e88..ca13161afb55b26666e99099955ba2b77d608eed 100644
--- a/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp
+++ b/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp
@@ -1,9 +1,8 @@
 //=- WebAssemblyISelLowering.cpp - WebAssembly DAG Lowering Implementation -==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
@@ -37,12 +36,6 @@ using namespace llvm;
 
 #define DEBUG_TYPE "wasm-lower"
 
-// Emit proposed instructions that may not have been implemented in engines
-cl::opt<bool> EnableUnimplementedWasmSIMDInstrs(
-    "wasm-enable-unimplemented-simd",
-    cl::desc("Emit potentially-unimplemented WebAssembly SIMD instructions"),
-    cl::init(false));
-
 WebAssemblyTargetLowering::WebAssemblyTargetLowering(
     const TargetMachine &TM, const WebAssemblySubtarget &STI)
     : TargetLowering(TM), Subtarget(&STI) {
@@ -70,10 +63,10 @@ WebAssemblyTargetLowering::WebAssemblyTargetLowering(
     addRegisterClass(MVT::v8i16, &WebAssembly::V128RegClass);
     addRegisterClass(MVT::v4i32, &WebAssembly::V128RegClass);
     addRegisterClass(MVT::v4f32, &WebAssembly::V128RegClass);
-    if (EnableUnimplementedWasmSIMDInstrs) {
-      addRegisterClass(MVT::v2i64, &WebAssembly::V128RegClass);
-      addRegisterClass(MVT::v2f64, &WebAssembly::V128RegClass);
-    }
+  }
+  if (Subtarget->hasUnimplementedSIMD128()) {
+    addRegisterClass(MVT::v2i64, &WebAssembly::V128RegClass);
+    addRegisterClass(MVT::v2f64, &WebAssembly::V128RegClass);
   }
   // Compute derived properties from the register classes.
   computeRegisterProperties(Subtarget->getRegisterInfo());
@@ -117,70 +110,85 @@ WebAssemblyTargetLowering::WebAssemblyTargetLowering(
     setTruncStoreAction(T, MVT::f16, Expand);
   }
 
-  // Support saturating add for i8x16 and i16x8
-  if (Subtarget->hasSIMD128())
-    for (auto T : {MVT::v16i8, MVT::v8i16})
-      for (auto Op : {ISD::SADDSAT, ISD::UADDSAT})
-        setOperationAction(Op, T, Legal);
-
   // Expand unavailable integer operations.
   for (auto Op :
        {ISD::BSWAP, ISD::SMUL_LOHI, ISD::UMUL_LOHI, ISD::MULHS, ISD::MULHU,
         ISD::SDIVREM, ISD::UDIVREM, ISD::SHL_PARTS, ISD::SRA_PARTS,
         ISD::SRL_PARTS, ISD::ADDC, ISD::ADDE, ISD::SUBC, ISD::SUBE}) {
-    for (auto T : {MVT::i32, MVT::i64}) {
+    for (auto T : {MVT::i32, MVT::i64})
       setOperationAction(Op, T, Expand);
-    }
-    if (Subtarget->hasSIMD128()) {
-      for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32}) {
+    if (Subtarget->hasSIMD128())
+      for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32})
         setOperationAction(Op, T, Expand);
-      }
-      if (EnableUnimplementedWasmSIMDInstrs) {
-        setOperationAction(Op, MVT::v2i64, Expand);
-      }
-    }
+    if (Subtarget->hasUnimplementedSIMD128())
+      setOperationAction(Op, MVT::v2i64, Expand);
   }
 
-  // There is no i64x2.mul instruction
-  setOperationAction(ISD::MUL, MVT::v2i64, Expand);
-
-  // We have custom shuffle lowering to expose the shuffle mask
+  // SIMD-specific configuration
   if (Subtarget->hasSIMD128()) {
-    for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v4f32}) {
+    // Support saturating add for i8x16 and i16x8
+    for (auto Op : {ISD::SADDSAT, ISD::UADDSAT})
+      for (auto T : {MVT::v16i8, MVT::v8i16})
+        setOperationAction(Op, T, Legal);
+
+    // Custom lower BUILD_VECTORs to minimize number of replace_lanes
+    for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v4f32})
+      setOperationAction(ISD::BUILD_VECTOR, T, Custom);
+    if (Subtarget->hasUnimplementedSIMD128())
+      for (auto T : {MVT::v2i64, MVT::v2f64})
+        setOperationAction(ISD::BUILD_VECTOR, T, Custom);
+
+    // We have custom shuffle lowering to expose the shuffle mask
+    for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v4f32})
       setOperationAction(ISD::VECTOR_SHUFFLE, T, Custom);
-    }
-    if (EnableUnimplementedWasmSIMDInstrs) {
-      setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v2i64, Custom);
-      setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v2f64, Custom);
-    }
-  }
+    if (Subtarget->hasUnimplementedSIMD128())
+      for (auto T: {MVT::v2i64, MVT::v2f64})
+        setOperationAction(ISD::VECTOR_SHUFFLE, T, Custom);
 
-  // Custom lowering since wasm shifts must have a scalar shift amount
-  if (Subtarget->hasSIMD128()) {
-    for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32})
-      for (auto Op : {ISD::SHL, ISD::SRA, ISD::SRL})
+    // Custom lowering since wasm shifts must have a scalar shift amount
+    for (auto Op : {ISD::SHL, ISD::SRA, ISD::SRL}) {
+      for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32})
         setOperationAction(Op, T, Custom);
-    if (EnableUnimplementedWasmSIMDInstrs)
-      for (auto Op : {ISD::SHL, ISD::SRA, ISD::SRL})
+      if (Subtarget->hasUnimplementedSIMD128())
         setOperationAction(Op, MVT::v2i64, Custom);
-  }
+    }
 
-  // There are no select instructions for vectors
-  if (Subtarget->hasSIMD128())
+    // Custom lower lane accesses to expand out variable indices
+    for (auto Op : {ISD::EXTRACT_VECTOR_ELT, ISD::INSERT_VECTOR_ELT}) {
+      for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v4f32})
+        setOperationAction(Op, T, Custom);
+      if (Subtarget->hasUnimplementedSIMD128())
+        for (auto T : {MVT::v2i64, MVT::v2f64})
+          setOperationAction(Op, T, Custom);
+    }
+
+    // There is no i64x2.mul instruction
+    setOperationAction(ISD::MUL, MVT::v2i64, Expand);
+
+    // There are no vector select instructions
     for (auto Op : {ISD::VSELECT, ISD::SELECT_CC, ISD::SELECT}) {
       for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v4f32})
         setOperationAction(Op, T, Expand);
-      if (EnableUnimplementedWasmSIMDInstrs)
+      if (Subtarget->hasUnimplementedSIMD128())
         for (auto T : {MVT::v2i64, MVT::v2f64})
           setOperationAction(Op, T, Expand);
     }
 
+    // Expand additional SIMD ops that V8 hasn't implemented yet
+    if (!Subtarget->hasUnimplementedSIMD128()) {
+      setOperationAction(ISD::FSQRT, MVT::v4f32, Expand);
+      setOperationAction(ISD::FDIV, MVT::v4f32, Expand);
+    }
+  }
+
   // As a special case, these operators use the type to mean the type to
   // sign-extend from.
   setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1, Expand);
   if (!Subtarget->hasSignExt()) {
+    // Sign extends are legal only when extending a vector extract
+    auto Action = Subtarget->hasSIMD128() ? Custom : Expand;
     for (auto T : {MVT::i8, MVT::i16, MVT::i32})
-      setOperationAction(ISD::SIGN_EXTEND_INREG, T, Expand);
+      setOperationAction(ISD::SIGN_EXTEND_INREG, T, Action);
   }
   for (auto T : MVT::integer_vector_valuetypes())
     setOperationAction(ISD::SIGN_EXTEND_INREG, T, Expand);
@@ -224,19 +232,8 @@ WebAssemblyTargetLowering::WebAssemblyTargetLowering(
     }
   }
 
-  // Custom lower lane accesses to expand out variable indices
-  if (Subtarget->hasSIMD128()) {
-    for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v4f32}) {
-      setOperationAction(ISD::EXTRACT_VECTOR_ELT, T, Custom);
-      setOperationAction(ISD::INSERT_VECTOR_ELT, T, Custom);
-    }
-    if (EnableUnimplementedWasmSIMDInstrs) {
-      for (auto T : {MVT::v2i64, MVT::v2f64}) {
-        setOperationAction(ISD::EXTRACT_VECTOR_ELT, T, Custom);
-        setOperationAction(ISD::INSERT_VECTOR_ELT, T, Custom);
-      }
-    }
-  }
+  // Don't do anything clever with build_pairs
+  setOperationAction(ISD::BUILD_PAIR, MVT::i64, Expand);
 
   // Trap lowers to wasm unreachable
   setOperationAction(ISD::TRAP, MVT::Other, Legal);
@@ -246,6 +243,14 @@ WebAssemblyTargetLowering::WebAssemblyTargetLowering(
   setOperationAction(ISD::INTRINSIC_VOID, MVT::Other, Custom);
 
   setMaxAtomicSizeInBitsSupported(64);
+
+  if (Subtarget->hasBulkMemory()) {
+    // Using memory.copy is always better than using multiple loads and stores
+    MaxStoresPerMemcpy = 1;
+    MaxStoresPerMemcpyOptSize = 1;
+    MaxStoresPerMemmove = 1;
+    MaxStoresPerMemmoveOptSize = 1;
+  }
 }
 
 TargetLowering::AtomicExpansionKind
@@ -322,11 +327,11 @@ static MachineBasicBlock *LowerFPToInt(MachineInstr &MI, DebugLoc DL,
   auto &Context = BB->getParent()->getFunction().getContext();
   Type *Ty = Float64 ? Type::getDoubleTy(Context) : Type::getFloatTy(Context);
 
-  const BasicBlock *LLVM_BB = BB->getBasicBlock();
+  const BasicBlock *LLVMBB = BB->getBasicBlock();
   MachineFunction *F = BB->getParent();
-  MachineBasicBlock *TrueMBB = F->CreateMachineBasicBlock(LLVM_BB);
-  MachineBasicBlock *FalseMBB = F->CreateMachineBasicBlock(LLVM_BB);
-  MachineBasicBlock *DoneMBB = F->CreateMachineBasicBlock(LLVM_BB);
+  MachineBasicBlock *TrueMBB = F->CreateMachineBasicBlock(LLVMBB);
+  MachineBasicBlock *FalseMBB = F->CreateMachineBasicBlock(LLVMBB);
+  MachineBasicBlock *DoneMBB = F->CreateMachineBasicBlock(LLVMBB);
 
   MachineFunction::iterator It = ++BB->getIterator();
   F->insert(It, FalseMBB);
@@ -576,14 +581,14 @@ bool WebAssemblyTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
 // Lowering Code
 //===----------------------------------------------------------------------===//
 
-static void fail(const SDLoc &DL, SelectionDAG &DAG, const char *msg) {
+static void fail(const SDLoc &DL, SelectionDAG &DAG, const char *Msg) {
   MachineFunction &MF = DAG.getMachineFunction();
   DAG.getContext()->diagnose(
-      DiagnosticInfoUnsupported(MF.getFunction(), msg, DL.getDebugLoc()));
+      DiagnosticInfoUnsupported(MF.getFunction(), Msg, DL.getDebugLoc()));
 }
 
 // Test whether the given calling convention is supported.
-static bool CallingConvSupported(CallingConv::ID CallConv) {
+static bool callingConvSupported(CallingConv::ID CallConv) {
   // We currently support the language-independent target-independent
   // conventions. We don't yet have a way to annotate calls with properties like
   // "cold", and we don't have any call-clobbered registers, so these are mostly
@@ -606,7 +611,7 @@ WebAssemblyTargetLowering::LowerCall(CallLoweringInfo &CLI,
   auto Layout = MF.getDataLayout();
 
   CallingConv::ID CallConv = CLI.CallConv;
-  if (!CallingConvSupported(CallConv))
+  if (!callingConvSupported(CallConv))
     fail(DL, DAG,
          "WebAssembly doesn't support language-specific or target-specific "
          "calling conventions yet");
@@ -628,9 +633,9 @@ WebAssemblyTargetLowering::LowerCall(CallLoweringInfo &CLI,
   SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs;
   SmallVectorImpl<SDValue> &OutVals = CLI.OutVals;
   unsigned NumFixedArgs = 0;
-  for (unsigned i = 0; i < Outs.size(); ++i) {
-    const ISD::OutputArg &Out = Outs[i];
-    SDValue &OutVal = OutVals[i];
+  for (unsigned I = 0; I < Outs.size(); ++I) {
+    const ISD::OutputArg &Out = Outs[I];
+    SDValue &OutVal = OutVals[I];
     if (Out.Flags.isNest())
       fail(DL, DAG, "WebAssembly hasn't implemented nest arguments");
     if (Out.Flags.isInAlloca())
@@ -766,7 +771,7 @@ SDValue WebAssemblyTargetLowering::LowerReturn(
     const SmallVectorImpl<SDValue> &OutVals, const SDLoc &DL,
     SelectionDAG &DAG) const {
   assert(Outs.size() <= 1 && "WebAssembly can only return up to one value");
-  if (!CallingConvSupported(CallConv))
+  if (!callingConvSupported(CallConv))
     fail(DL, DAG, "WebAssembly doesn't support non-C calling conventions");
 
   SmallVector<SDValue, 4> RetOps(1, Chain);
@@ -793,7 +798,7 @@ SDValue WebAssemblyTargetLowering::LowerFormalArguments(
     SDValue Chain, CallingConv::ID CallConv, bool IsVarArg,
     const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL,
     SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
-  if (!CallingConvSupported(CallConv))
+  if (!callingConvSupported(CallConv))
     fail(DL, DAG, "WebAssembly doesn't support non-C calling conventions");
 
   MachineFunction &MF = DAG.getMachineFunction();
@@ -840,7 +845,7 @@ SDValue WebAssemblyTargetLowering::LowerFormalArguments(
   // Record the number and types of arguments and results.
   SmallVector<MVT, 4> Params;
   SmallVector<MVT, 4> Results;
-  ComputeSignatureVTs(MF.getFunction().getFunctionType(), MF.getFunction(),
+  computeSignatureVTs(MF.getFunction().getFunctionType(), MF.getFunction(),
                       DAG.getTarget(), Params, Results);
   for (MVT VT : Results)
     MFI->addResult(VT);
@@ -887,13 +892,17 @@ SDValue WebAssemblyTargetLowering::LowerOperation(SDValue Op,
     return LowerFRAMEADDR(Op, DAG);
   case ISD::CopyToReg:
     return LowerCopyToReg(Op, DAG);
-  case ISD::INTRINSIC_WO_CHAIN:
-    return LowerINTRINSIC_WO_CHAIN(Op, DAG);
   case ISD::EXTRACT_VECTOR_ELT:
   case ISD::INSERT_VECTOR_ELT:
     return LowerAccessVectorElement(Op, DAG);
   case ISD::INTRINSIC_VOID:
-    return LowerINTRINSIC_VOID(Op, DAG);
+  case ISD::INTRINSIC_WO_CHAIN:
+  case ISD::INTRINSIC_W_CHAIN:
+    return LowerIntrinsic(Op, DAG);
+  case ISD::SIGN_EXTEND_INREG:
+    return LowerSIGN_EXTEND_INREG(Op, DAG);
+  case ISD::BUILD_VECTOR:
+    return LowerBUILD_VECTOR(Op, DAG);
   case ISD::VECTOR_SHUFFLE:
     return LowerVECTOR_SHUFFLE(Op, DAG);
   case ISD::SHL:
@@ -911,7 +920,7 @@ SDValue WebAssemblyTargetLowering::LowerCopyToReg(SDValue Op,
     // the FI to some LEA-like instruction, but since we don't have that, we
     // need to insert some kind of instruction that can take an FI operand and
     // produces a value usable by CopyToReg (i.e. in a vreg). So insert a dummy
-    // copy_local between Op and its FI operand.
+    // local.copy between Op and its FI operand.
     SDValue Chain = Op.getOperand(0);
     SDLoc DL(Op);
     unsigned Reg = cast<RegisterSDNode>(Op.getOperand(1))->getReg();
@@ -1034,17 +1043,28 @@ SDValue WebAssemblyTargetLowering::LowerVASTART(SDValue Op,
                       MachinePointerInfo(SV), 0);
 }
 
-SDValue
-WebAssemblyTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
-                                                   SelectionDAG &DAG) const {
-  unsigned IntNo = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
+SDValue WebAssemblyTargetLowering::LowerIntrinsic(SDValue Op,
+                                                  SelectionDAG &DAG) const {
+  MachineFunction &MF = DAG.getMachineFunction();
+  unsigned IntNo;
+  switch (Op.getOpcode()) {
+  case ISD::INTRINSIC_VOID:
+  case ISD::INTRINSIC_W_CHAIN:
+    IntNo = cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue();
+    break;
+  case ISD::INTRINSIC_WO_CHAIN:
+    IntNo = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
+    break;
+  default:
+    llvm_unreachable("Invalid intrinsic");
+  }
   SDLoc DL(Op);
+
   switch (IntNo) {
   default:
-    return {}; // Don't custom lower most intrinsics.
+    return SDValue(); // Don't custom lower most intrinsics.
 
   case Intrinsic::wasm_lsda: {
-    MachineFunction &MF = DAG.getMachineFunction();
     EVT VT = Op.getValueType();
     const TargetLowering &TLI = DAG.getTargetLoweringInfo();
     MVT PtrVT = TLI.getPointerTy(DAG.getDataLayout());
@@ -1054,45 +1074,145 @@ WebAssemblyTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
     return DAG.getNode(WebAssemblyISD::Wrapper, DL, VT,
                        DAG.getMCSymbol(S, PtrVT));
   }
+
+  case Intrinsic::wasm_throw: {
+    // We only support C++ exceptions for now
+    int Tag = cast<ConstantSDNode>(Op.getOperand(2).getNode())->getZExtValue();
+    if (Tag != CPP_EXCEPTION)
+      llvm_unreachable("Invalid tag!");
+    const TargetLowering &TLI = DAG.getTargetLoweringInfo();
+    MVT PtrVT = TLI.getPointerTy(DAG.getDataLayout());
+    const char *SymName = MF.createExternalSymbolName("__cpp_exception");
+    SDValue SymNode =
+        DAG.getNode(WebAssemblyISD::Wrapper, DL, PtrVT,
+                    DAG.getTargetExternalSymbol(
+                        SymName, PtrVT, WebAssemblyII::MO_SYMBOL_EVENT));
+    return DAG.getNode(WebAssemblyISD::THROW, DL,
+                       MVT::Other, // outchain type
+                       {
+                           Op.getOperand(0), // inchain
+                           SymNode,          // exception symbol
+                           Op.getOperand(3)  // thrown value
+                       });
+  }
   }
 }
 
 SDValue
-WebAssemblyTargetLowering::LowerINTRINSIC_VOID(SDValue Op,
-                                               SelectionDAG &DAG) const {
-  MachineFunction &MF = DAG.getMachineFunction();
-  unsigned IntNo = cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue();
-  SDLoc DL(Op);
-
-  switch (IntNo) {
-  default:
-    return {}; // Don't custom lower most intrinsics.
+WebAssemblyTargetLowering::LowerSIGN_EXTEND_INREG(SDValue Op,
+                                                  SelectionDAG &DAG) const {
+  // If sign extension operations are disabled, allow sext_inreg only if operand
+  // is a vector extract. SIMD does not depend on sign extension operations, but
+  // allowing sext_inreg in this context lets us have simple patterns to select
+  // extract_lane_s instructions. Expanding sext_inreg everywhere would be
+  // simpler in this file, but would necessitate large and brittle patterns to
+  // undo the expansion and select extract_lane_s instructions.
+  assert(!Subtarget->hasSignExt() && Subtarget->hasSIMD128());
+  if (Op.getOperand(0).getOpcode() == ISD::EXTRACT_VECTOR_ELT)
+    return Op;
+  // Otherwise expand
+  return SDValue();
+}
 
-  case Intrinsic::wasm_throw: {
-    int Tag = cast<ConstantSDNode>(Op.getOperand(2).getNode())->getZExtValue();
-    switch (Tag) {
-    case CPP_EXCEPTION: {
-      const TargetLowering &TLI = DAG.getTargetLoweringInfo();
-      MVT PtrVT = TLI.getPointerTy(DAG.getDataLayout());
-      const char *SymName = MF.createExternalSymbolName("__cpp_exception");
-      SDValue SymNode =
-          DAG.getNode(WebAssemblyISD::Wrapper, DL, PtrVT,
-                      DAG.getTargetExternalSymbol(
-                          SymName, PtrVT, WebAssemblyII::MO_SYMBOL_EVENT));
-      return DAG.getNode(WebAssemblyISD::THROW, DL,
-                         MVT::Other, // outchain type
-                         {
-                             Op.getOperand(0), // inchain
-                             SymNode,          // exception symbol
-                             Op.getOperand(3)  // thrown value
-                         });
+SDValue WebAssemblyTargetLowering::LowerBUILD_VECTOR(SDValue Op,
+                                                     SelectionDAG &DAG) const {
+  SDLoc DL(Op);
+  const EVT VecT = Op.getValueType();
+  const EVT LaneT = Op.getOperand(0).getValueType();
+  const size_t Lanes = Op.getNumOperands();
+  auto IsConstant = [](const SDValue &V) {
+    return V.getOpcode() == ISD::Constant || V.getOpcode() == ISD::ConstantFP;
+  };
+
+  // Find the most common operand, which is approximately the best to splat
+  using Entry = std::pair<SDValue, size_t>;
+  SmallVector<Entry, 16> ValueCounts;
+  size_t NumConst = 0, NumDynamic = 0;
+  for (const SDValue &Lane : Op->op_values()) {
+    if (Lane.isUndef()) {
+      continue;
+    } else if (IsConstant(Lane)) {
+      NumConst++;
+    } else {
+      NumDynamic++;
     }
-    default:
-      llvm_unreachable("Invalid tag!");
+    auto CountIt = std::find_if(ValueCounts.begin(), ValueCounts.end(),
+                                [&Lane](Entry A) { return A.first == Lane; });
+    if (CountIt == ValueCounts.end()) {
+      ValueCounts.emplace_back(Lane, 1);
+    } else {
+      CountIt->second++;
     }
-    break;
   }
+  auto CommonIt =
+      std::max_element(ValueCounts.begin(), ValueCounts.end(),
+                       [](Entry A, Entry B) { return A.second < B.second; });
+  assert(CommonIt != ValueCounts.end() && "Unexpected all-undef build_vector");
+  SDValue SplatValue = CommonIt->first;
+  size_t NumCommon = CommonIt->second;
+
+  // If v128.const is available, consider using it instead of a splat
+  if (Subtarget->hasUnimplementedSIMD128()) {
+    // {i32,i64,f32,f64}.const opcode, and value
+    const size_t ConstBytes = 1 + std::max(size_t(4), 16 / Lanes);
+    // SIMD prefix and opcode
+    const size_t SplatBytes = 2;
+    const size_t SplatConstBytes = SplatBytes + ConstBytes;
+    // SIMD prefix, opcode, and lane index
+    const size_t ReplaceBytes = 3;
+    const size_t ReplaceConstBytes = ReplaceBytes + ConstBytes;
+    // SIMD prefix, v128.const opcode, and 128-bit value
+    const size_t VecConstBytes = 18;
+    // Initial v128.const and a replace_lane for each non-const operand
+    const size_t ConstInitBytes = VecConstBytes + NumDynamic * ReplaceBytes;
+    // Initial splat and all necessary replace_lanes
+    const size_t SplatInitBytes =
+        IsConstant(SplatValue)
+            // Initial constant splat
+            ? (SplatConstBytes +
+               // Constant replace_lanes
+               (NumConst - NumCommon) * ReplaceConstBytes +
+               // Dynamic replace_lanes
+               (NumDynamic * ReplaceBytes))
+            // Initial dynamic splat
+            : (SplatBytes +
+               // Constant replace_lanes
+               (NumConst * ReplaceConstBytes) +
+               // Dynamic replace_lanes
+               (NumDynamic - NumCommon) * ReplaceBytes);
+    if (ConstInitBytes < SplatInitBytes) {
+      // Create build_vector that will lower to initial v128.const
+      SmallVector<SDValue, 16> ConstLanes;
+      for (const SDValue &Lane : Op->op_values()) {
+        if (IsConstant(Lane)) {
+          ConstLanes.push_back(Lane);
+        } else if (LaneT.isFloatingPoint()) {
+          ConstLanes.push_back(DAG.getConstantFP(0, DL, LaneT));
+        } else {
+          ConstLanes.push_back(DAG.getConstant(0, DL, LaneT));
+        }
+      }
+      SDValue Result = DAG.getBuildVector(VecT, DL, ConstLanes);
+      // Add replace_lane instructions for non-const lanes
+      for (size_t I = 0; I < Lanes; ++I) {
+        const SDValue &Lane = Op->getOperand(I);
+        if (!Lane.isUndef() && !IsConstant(Lane))
+          Result = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, VecT, Result, Lane,
+                               DAG.getConstant(I, DL, MVT::i32));
+      }
+      return Result;
+    }
+  }
+  // Use a splat for the initial vector
+  SDValue Result = DAG.getSplatBuildVector(VecT, DL, SplatValue);
+  // Add replace_lane instructions for other values
+  for (size_t I = 0; I < Lanes; ++I) {
+    const SDValue &Lane = Op->getOperand(I);
+    if (Lane != SplatValue)
+      Result = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, VecT, Result, Lane,
+                           DAG.getConstant(I, DL, MVT::i32));
   }
+  return Result;
 }
 
 SDValue
@@ -1111,11 +1231,10 @@ WebAssemblyTargetLowering::LowerVECTOR_SHUFFLE(SDValue Op,
   Ops[OpIdx++] = Op.getOperand(1);
 
   // Expand mask indices to byte indices and materialize them as operands
-  for (size_t I = 0, Lanes = Mask.size(); I < Lanes; ++I) {
+  for (int M : Mask) {
     for (size_t J = 0; J < LaneBytes; ++J) {
       // Lower undefs (represented by -1 in mask) to zero
-      uint64_t ByteIndex =
-          Mask[I] == -1 ? 0 : (uint64_t)Mask[I] * LaneBytes + J;
+      uint64_t ByteIndex = M == -1 ? 0 : (uint64_t)M * LaneBytes + J;
       Ops[OpIdx++] = DAG.getConstant(ByteIndex, DL, MVT::i32);
     }
   }
@@ -1135,6 +1254,31 @@ WebAssemblyTargetLowering::LowerAccessVectorElement(SDValue Op,
     return SDValue();
 }
 
+static SDValue unrollVectorShift(SDValue Op, SelectionDAG &DAG) {
+  EVT LaneT = Op.getSimpleValueType().getVectorElementType();
+  // 32-bit and 64-bit unrolled shifts will have proper semantics
+  if (LaneT.bitsGE(MVT::i32))
+    return DAG.UnrollVectorOp(Op.getNode());
+  // Otherwise mask the shift value to get proper semantics from 32-bit shift
+  SDLoc DL(Op);
+  SDValue ShiftVal = Op.getOperand(1);
+  uint64_t MaskVal = LaneT.getSizeInBits() - 1;
+  SDValue MaskedShiftVal = DAG.getNode(
+      ISD::AND,                    // mask opcode
+      DL, ShiftVal.getValueType(), // masked value type
+      ShiftVal,                    // original shift value operand
+      DAG.getConstant(MaskVal, DL, ShiftVal.getValueType()) // mask operand
+  );
+
+  return DAG.UnrollVectorOp(
+      DAG.getNode(Op.getOpcode(),        // original shift opcode
+                  DL, Op.getValueType(), // original return type
+                  Op.getOperand(0),      // original vector operand,
+                  MaskedShiftVal         // new masked shift value operand
+                  )
+          .getNode());
+}
+
 SDValue WebAssemblyTargetLowering::LowerShift(SDValue Op,
                                               SelectionDAG &DAG) const {
   SDLoc DL(Op);
@@ -1142,15 +1286,20 @@ SDValue WebAssemblyTargetLowering::LowerShift(SDValue Op,
   // Only manually lower vector shifts
   assert(Op.getSimpleValueType().isVector());
 
+  // Expand all vector shifts until V8 fixes its implementation
+  // TODO: remove this once V8 is fixed
+  if (!Subtarget->hasUnimplementedSIMD128())
+    return unrollVectorShift(Op, DAG);
+
   // Unroll non-splat vector shifts
   BuildVectorSDNode *ShiftVec;
   SDValue SplatVal;
   if (!(ShiftVec = dyn_cast<BuildVectorSDNode>(Op.getOperand(1).getNode())) ||
       !(SplatVal = ShiftVec->getSplatValue()))
-    return DAG.UnrollVectorOp(Op.getNode());
+    return unrollVectorShift(Op, DAG);
 
   // All splats except i64x2 const splats are handled by patterns
-  ConstantSDNode *SplatConst = dyn_cast<ConstantSDNode>(SplatVal);
+  auto *SplatConst = dyn_cast<ConstantSDNode>(SplatVal);
   if (!SplatConst || Op.getSimpleValueType() != MVT::v2i64)
     return Op;
 
diff --git a/lib/Target/WebAssembly/WebAssemblyISelLowering.h b/lib/Target/WebAssembly/WebAssemblyISelLowering.h
index 80076818de7335293226909c1a7047b6a2cb2805..ee2e863685a8d5f57369b3859ef7304a4068330a 100644
--- a/lib/Target/WebAssembly/WebAssemblyISelLowering.h
+++ b/lib/Target/WebAssembly/WebAssemblyISelLowering.h
@@ -1,9 +1,8 @@
 //- WebAssemblyISelLowering.h - WebAssembly DAG Lowering Interface -*- C++ -*-//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
@@ -97,8 +96,9 @@ private:
   SDValue LowerJumpTable(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerVASTART(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerCopyToReg(SDValue Op, SelectionDAG &DAG) const;
-  SDValue LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG) const;
-  SDValue LowerINTRINSIC_VOID(SDValue Op, SelectionDAG &DAG) const;
+  SDValue LowerIntrinsic(SDValue Op, SelectionDAG &DAG) const;
+  SDValue LowerSIGN_EXTEND_INREG(SDValue Op, SelectionDAG &DAG) const;
+  SDValue LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerAccessVectorElement(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerShift(SDValue Op, SelectionDAG &DAG) const;
diff --git a/lib/Target/WebAssembly/WebAssemblyInstrAtomics.td b/lib/Target/WebAssembly/WebAssemblyInstrAtomics.td
index f9d092e4b8a695cfac6ea5b015140d7f2cacd5ff..6314b4d14b5561cb8e774f9f9006b79c470a26c3 100644
--- a/lib/Target/WebAssembly/WebAssemblyInstrAtomics.td
+++ b/lib/Target/WebAssembly/WebAssemblyInstrAtomics.td
@@ -1,9 +1,8 @@
 // WebAssemblyInstrAtomics.td-WebAssembly Atomic codegen support-*- tablegen -*-
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
@@ -114,7 +113,7 @@ def : LoadPatNoOffset<i32, atomic_load_8, ATOMIC_LOAD8_U_I32>;
 def : LoadPatNoOffset<i32, atomic_load_16, ATOMIC_LOAD16_U_I32>;
 def : LoadPatNoOffset<i64, sext_aload_8_64, ATOMIC_LOAD8_U_I64>;
 def : LoadPatNoOffset<i64, sext_aload_16_64, ATOMIC_LOAD16_U_I64>;
-// 32->64 sext load gets selected as i32.atomic.load, i64.extend_s/i32
+// 32->64 sext load gets selected as i32.atomic.load, i64.extend_i32_s
 
 // Zero-extending loads with constant offset
 def : LoadPatImmOff<i32, zext_aload_8_32, regPlusImm, ATOMIC_LOAD8_U_I32>;
@@ -344,82 +343,82 @@ multiclass WebAssemblyBinRMW<WebAssemblyRegClass rc, string Name, int Opcode> {
 defm ATOMIC_RMW_ADD_I32 : WebAssemblyBinRMW<I32, "i32.atomic.rmw.add", 0xfe1e>;
 defm ATOMIC_RMW_ADD_I64 : WebAssemblyBinRMW<I64, "i64.atomic.rmw.add", 0xfe1f>;
 defm ATOMIC_RMW8_U_ADD_I32 :
-  WebAssemblyBinRMW<I32, "i32.atomic.rmw8_u.add", 0xfe20>;
+  WebAssemblyBinRMW<I32, "i32.atomic.rmw8.add_u", 0xfe20>;
 defm ATOMIC_RMW16_U_ADD_I32 :
-  WebAssemblyBinRMW<I32, "i32.atomic.rmw16_u.add", 0xfe21>;
+  WebAssemblyBinRMW<I32, "i32.atomic.rmw16.add_u", 0xfe21>;
 defm ATOMIC_RMW8_U_ADD_I64 :
-  WebAssemblyBinRMW<I64, "i64.atomic.rmw8_u.add", 0xfe22>;
+  WebAssemblyBinRMW<I64, "i64.atomic.rmw8.add_u", 0xfe22>;
 defm ATOMIC_RMW16_U_ADD_I64 :
-  WebAssemblyBinRMW<I64, "i64.atomic.rmw16_u.add", 0xfe23>;
+  WebAssemblyBinRMW<I64, "i64.atomic.rmw16.add_u", 0xfe23>;
 defm ATOMIC_RMW32_U_ADD_I64 :
-  WebAssemblyBinRMW<I64, "i64.atomic.rmw32_u.add", 0xfe24>;
+  WebAssemblyBinRMW<I64, "i64.atomic.rmw32.add_u", 0xfe24>;
 
 defm ATOMIC_RMW_SUB_I32 : WebAssemblyBinRMW<I32, "i32.atomic.rmw.sub", 0xfe25>;
 defm ATOMIC_RMW_SUB_I64 : WebAssemblyBinRMW<I64, "i64.atomic.rmw.sub", 0xfe26>;
 defm ATOMIC_RMW8_U_SUB_I32 :
-  WebAssemblyBinRMW<I32, "i32.atomic.rmw8_u.sub", 0xfe27>;
+  WebAssemblyBinRMW<I32, "i32.atomic.rmw8.sub_u", 0xfe27>;
 defm ATOMIC_RMW16_U_SUB_I32 :
-  WebAssemblyBinRMW<I32, "i32.atomic.rmw16_u.sub", 0xfe28>;
+  WebAssemblyBinRMW<I32, "i32.atomic.rmw16.sub_u", 0xfe28>;
 defm ATOMIC_RMW8_U_SUB_I64 :
-  WebAssemblyBinRMW<I64, "i64.atomic.rmw8_u.sub", 0xfe29>;
+  WebAssemblyBinRMW<I64, "i64.atomic.rmw8.sub_u", 0xfe29>;
 defm ATOMIC_RMW16_U_SUB_I64 :
-  WebAssemblyBinRMW<I64, "i64.atomic.rmw16_u.sub", 0xfe2a>;
+  WebAssemblyBinRMW<I64, "i64.atomic.rmw16.sub_u", 0xfe2a>;
 defm ATOMIC_RMW32_U_SUB_I64 :
-  WebAssemblyBinRMW<I64, "i64.atomic.rmw32_u.sub", 0xfe2b>;
+  WebAssemblyBinRMW<I64, "i64.atomic.rmw32.sub_u", 0xfe2b>;
 
 defm ATOMIC_RMW_AND_I32 : WebAssemblyBinRMW<I32, "i32.atomic.rmw.and", 0xfe2c>;
 defm ATOMIC_RMW_AND_I64 : WebAssemblyBinRMW<I64, "i64.atomic.rmw.and", 0xfe2d>;
 defm ATOMIC_RMW8_U_AND_I32 :
-  WebAssemblyBinRMW<I32, "i32.atomic.rmw8_u.and", 0xfe2e>;
+  WebAssemblyBinRMW<I32, "i32.atomic.rmw8.and_u", 0xfe2e>;
 defm ATOMIC_RMW16_U_AND_I32 :
-  WebAssemblyBinRMW<I32, "i32.atomic.rmw16_u.and", 0xfe2f>;
+  WebAssemblyBinRMW<I32, "i32.atomic.rmw16.and_u", 0xfe2f>;
 defm ATOMIC_RMW8_U_AND_I64 :
-  WebAssemblyBinRMW<I64, "i64.atomic.rmw8_u.and", 0xfe30>;
+  WebAssemblyBinRMW<I64, "i64.atomic.rmw8.and_u", 0xfe30>;
 defm ATOMIC_RMW16_U_AND_I64 :
-  WebAssemblyBinRMW<I64, "i64.atomic.rmw16_u.and", 0xfe31>;
+  WebAssemblyBinRMW<I64, "i64.atomic.rmw16.and_u", 0xfe31>;
 defm ATOMIC_RMW32_U_AND_I64 :
-  WebAssemblyBinRMW<I64, "i64.atomic.rmw32_u.and", 0xfe32>;
+  WebAssemblyBinRMW<I64, "i64.atomic.rmw32.and_u", 0xfe32>;
 
 defm ATOMIC_RMW_OR_I32 : WebAssemblyBinRMW<I32, "i32.atomic.rmw.or", 0xfe33>;
 defm ATOMIC_RMW_OR_I64 : WebAssemblyBinRMW<I64, "i64.atomic.rmw.or", 0xfe34>;
 defm ATOMIC_RMW8_U_OR_I32 :
-  WebAssemblyBinRMW<I32, "i32.atomic.rmw8_u.or", 0xfe35>;
+  WebAssemblyBinRMW<I32, "i32.atomic.rmw8.or_u", 0xfe35>;
 defm ATOMIC_RMW16_U_OR_I32 :
-  WebAssemblyBinRMW<I32, "i32.atomic.rmw16_u.or", 0xfe36>;
+  WebAssemblyBinRMW<I32, "i32.atomic.rmw16.or_u", 0xfe36>;
 defm ATOMIC_RMW8_U_OR_I64 :
-  WebAssemblyBinRMW<I64, "i64.atomic.rmw8_u.or", 0xfe37>;
+  WebAssemblyBinRMW<I64, "i64.atomic.rmw8.or_u", 0xfe37>;
 defm ATOMIC_RMW16_U_OR_I64 :
-  WebAssemblyBinRMW<I64, "i64.atomic.rmw16_u.or", 0xfe38>;
+  WebAssemblyBinRMW<I64, "i64.atomic.rmw16.or_u", 0xfe38>;
 defm ATOMIC_RMW32_U_OR_I64 :
-  WebAssemblyBinRMW<I64, "i64.atomic.rmw32_u.or", 0xfe39>;
+  WebAssemblyBinRMW<I64, "i64.atomic.rmw32.or_u", 0xfe39>;
 
 defm ATOMIC_RMW_XOR_I32 : WebAssemblyBinRMW<I32, "i32.atomic.rmw.xor", 0xfe3a>;
 defm ATOMIC_RMW_XOR_I64 : WebAssemblyBinRMW<I64, "i64.atomic.rmw.xor", 0xfe3b>;
 defm ATOMIC_RMW8_U_XOR_I32 :
-  WebAssemblyBinRMW<I32, "i32.atomic.rmw8_u.xor", 0xfe3c>;
+  WebAssemblyBinRMW<I32, "i32.atomic.rmw8.xor_u", 0xfe3c>;
 defm ATOMIC_RMW16_U_XOR_I32 :
-  WebAssemblyBinRMW<I32, "i32.atomic.rmw16_u.xor", 0xfe3d>;
+  WebAssemblyBinRMW<I32, "i32.atomic.rmw16.xor_u", 0xfe3d>;
 defm ATOMIC_RMW8_U_XOR_I64 :
-  WebAssemblyBinRMW<I64, "i64.atomic.rmw8_u.xor", 0xfe3e>;
+  WebAssemblyBinRMW<I64, "i64.atomic.rmw8.xor_u", 0xfe3e>;
 defm ATOMIC_RMW16_U_XOR_I64 :
-  WebAssemblyBinRMW<I64, "i64.atomic.rmw16_u.xor", 0xfe3f>;
+  WebAssemblyBinRMW<I64, "i64.atomic.rmw16.xor_u", 0xfe3f>;
 defm ATOMIC_RMW32_U_XOR_I64 :
-  WebAssemblyBinRMW<I64, "i64.atomic.rmw32_u.xor", 0xfe40>;
+  WebAssemblyBinRMW<I64, "i64.atomic.rmw32.xor_u", 0xfe40>;
 
 defm ATOMIC_RMW_XCHG_I32 :
   WebAssemblyBinRMW<I32, "i32.atomic.rmw.xchg", 0xfe41>;
 defm ATOMIC_RMW_XCHG_I64 :
   WebAssemblyBinRMW<I64, "i64.atomic.rmw.xchg", 0xfe42>;
 defm ATOMIC_RMW8_U_XCHG_I32 :
-  WebAssemblyBinRMW<I32, "i32.atomic.rmw8_u.xchg", 0xfe43>;
+  WebAssemblyBinRMW<I32, "i32.atomic.rmw8.xchg_u", 0xfe43>;
 defm ATOMIC_RMW16_U_XCHG_I32 :
-  WebAssemblyBinRMW<I32, "i32.atomic.rmw16_u.xchg", 0xfe44>;
+  WebAssemblyBinRMW<I32, "i32.atomic.rmw16.xchg_u", 0xfe44>;
 defm ATOMIC_RMW8_U_XCHG_I64 :
-  WebAssemblyBinRMW<I64, "i64.atomic.rmw8_u.xchg", 0xfe45>;
+  WebAssemblyBinRMW<I64, "i64.atomic.rmw8.xchg_u", 0xfe45>;
 defm ATOMIC_RMW16_U_XCHG_I64 :
-  WebAssemblyBinRMW<I64, "i64.atomic.rmw16_u.xchg", 0xfe46>;
+  WebAssemblyBinRMW<I64, "i64.atomic.rmw16.xchg_u", 0xfe46>;
 defm ATOMIC_RMW32_U_XCHG_I64 :
-  WebAssemblyBinRMW<I64, "i64.atomic.rmw32_u.xchg", 0xfe47>;
+  WebAssemblyBinRMW<I64, "i64.atomic.rmw32.xchg_u", 0xfe47>;
 
 // Select binary RMWs with no constant offset.
 class BinRMWPatNoOffset<ValueType ty, PatFrag kind, NI inst> :
@@ -530,7 +529,7 @@ class sext_bin_rmw_8_64<PatFrag kind> :
   PatFrag<(ops node:$addr, node:$val),
           (anyext (i32 (kind node:$addr, (i32 (trunc (i64 node:$val))))))>;
 class sext_bin_rmw_16_64<PatFrag kind> : sext_bin_rmw_8_64<kind>;
-// 32->64 sext RMW gets selected as i32.atomic.rmw.***, i64.extend_s/i32
+// 32->64 sext RMW gets selected as i32.atomic.rmw.***, i64.extend_i32_s
 
 // Patterns for various addressing modes for truncating-extending binary RMWs.
 multiclass BinRMWTruncExtPattern<
@@ -677,15 +676,15 @@ defm ATOMIC_RMW_CMPXCHG_I32 :
 defm ATOMIC_RMW_CMPXCHG_I64 :
   WebAssemblyTerRMW<I64, "i64.atomic.rmw.cmpxchg", 0xfe49>;
 defm ATOMIC_RMW8_U_CMPXCHG_I32 :
-  WebAssemblyTerRMW<I32, "i32.atomic.rmw8_u.cmpxchg", 0xfe4a>;
+  WebAssemblyTerRMW<I32, "i32.atomic.rmw8.cmpxchg_u", 0xfe4a>;
 defm ATOMIC_RMW16_U_CMPXCHG_I32 :
-  WebAssemblyTerRMW<I32, "i32.atomic.rmw16_u.cmpxchg", 0xfe4b>;
+  WebAssemblyTerRMW<I32, "i32.atomic.rmw16.cmpxchg_u", 0xfe4b>;
 defm ATOMIC_RMW8_U_CMPXCHG_I64 :
-  WebAssemblyTerRMW<I64, "i64.atomic.rmw8_u.cmpxchg", 0xfe4c>;
+  WebAssemblyTerRMW<I64, "i64.atomic.rmw8.cmpxchg_u", 0xfe4c>;
 defm ATOMIC_RMW16_U_CMPXCHG_I64 :
-  WebAssemblyTerRMW<I64, "i64.atomic.rmw16_u.cmpxchg", 0xfe4d>;
+  WebAssemblyTerRMW<I64, "i64.atomic.rmw16.cmpxchg_u", 0xfe4d>;
 defm ATOMIC_RMW32_U_CMPXCHG_I64 :
-  WebAssemblyTerRMW<I64, "i64.atomic.rmw32_u.cmpxchg", 0xfe4e>;
+  WebAssemblyTerRMW<I64, "i64.atomic.rmw32.cmpxchg_u", 0xfe4e>;
 
 // Select ternary RMWs with no constant offset.
 class TerRMWPatNoOffset<ValueType ty, PatFrag kind, NI inst> :
@@ -749,10 +748,9 @@ multiclass TerRMWPattern<PatFrag rmw_32, PatFrag rmw_64, NI inst_32,
   def : TerRMWPatExternSymOffOnly<i64, rmw_64, inst_64>;
 }
 
-let Predicates = [HasAtomics] in {
+let Predicates = [HasAtomics] in
 defm : TerRMWPattern<atomic_cmp_swap_32, atomic_cmp_swap_64,
                      ATOMIC_RMW_CMPXCHG_I32, ATOMIC_RMW_CMPXCHG_I64>;
-} // Predicates = [HasAtomics]
 
 // Truncating & zero-extending ternary RMW patterns.
 // DAG legalization & optimization before instruction selection may introduce
@@ -790,7 +788,7 @@ class sext_ter_rmw_8_64<PatFrag kind> :
                   (i32 (trunc (i64 node:$exp))),
                   (i32 (trunc (i64 node:$new))))))))>;
 class sext_ter_rmw_16_64<PatFrag kind> : sext_ter_rmw_8_64<kind>;
-// 32->64 sext RMW gets selected as i32.atomic.rmw.***, i64.extend_s/i32
+// 32->64 sext RMW gets selected as i32.atomic.rmw.***, i64.extend_i32_s
 
 // Patterns for various addressing modes for truncating-extending ternary RMWs.
 multiclass TerRMWTruncExtPattern<
@@ -886,13 +884,12 @@ multiclass TerRMWTruncExtPattern<
   def : TerRMWPatExternSymOffOnly<i64, sext_ter_rmw_16_64<rmw_16>, inst16_64>;
 }
 
-let Predicates = [HasAtomics] in {
+let Predicates = [HasAtomics] in
 defm : TerRMWTruncExtPattern<
   atomic_cmp_swap_8, atomic_cmp_swap_16, atomic_cmp_swap_32, atomic_cmp_swap_64,
   ATOMIC_RMW8_U_CMPXCHG_I32, ATOMIC_RMW16_U_CMPXCHG_I32,
   ATOMIC_RMW8_U_CMPXCHG_I64, ATOMIC_RMW16_U_CMPXCHG_I64,
   ATOMIC_RMW32_U_CMPXCHG_I64>;
-}
 
 //===----------------------------------------------------------------------===//
 // Atomic wait / notify
diff --git a/lib/Target/WebAssembly/WebAssemblyInstrBulkMemory.td b/lib/Target/WebAssembly/WebAssemblyInstrBulkMemory.td
new file mode 100644
index 0000000000000000000000000000000000000000..646b16cf7bf536902a1e6f908f903f6a0fcd27d0
--- /dev/null
+++ b/lib/Target/WebAssembly/WebAssemblyInstrBulkMemory.td
@@ -0,0 +1,68 @@
+// WebAssemblyInstrBulkMemory.td - bulk memory codegen support --*- tablegen -*-
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// WebAssembly bulk memory codegen constructs.
+///
+//===----------------------------------------------------------------------===//
+
+// Instruction requiring HasBulkMemory and the bulk memory prefix byte
+multiclass BULK_I<dag oops_r, dag iops_r, dag oops_s, dag iops_s,
+                  list<dag> pattern_r, string asmstr_r = "",
+                  string asmstr_s = "", bits<32> simdop = -1> {
+  defm "" : I<oops_r, iops_r, oops_s, iops_s, pattern_r, asmstr_r, asmstr_s,
+              !or(0xfc00, !and(0xff, simdop))>,
+            Requires<[HasBulkMemory]>;
+}
+
+// Bespoke types and nodes for bulk memory ops
+def wasm_memcpy_t : SDTypeProfile<0, 5,
+  [SDTCisInt<0>, SDTCisInt<1>, SDTCisPtrTy<2>, SDTCisPtrTy<3>, SDTCisInt<4>]
+>;
+def wasm_memcpy : SDNode<"WebAssemblyISD::MEMORY_COPY", wasm_memcpy_t,
+                         [SDNPHasChain, SDNPMayLoad, SDNPMayStore]>;
+
+//===----------------------------------------------------------------------===//
+// memory.init
+//===----------------------------------------------------------------------===//
+
+let mayStore = 1 in
+defm MEMORY_INIT :
+  BULK_I<(outs),
+         (ins i32imm_op:$seg, i32imm_op:$idx, I32:$dest,
+              I32:$offset, I32:$size),
+         (outs), (ins i32imm_op:$seg, i32imm_op:$idx),
+         [(int_wasm_memory_init (i32 imm:$seg), (i32 imm:$idx), I32:$dest,
+            I32:$offset, I32:$size
+          )],
+         "memory.init\t$seg, $idx, $dest, $offset, $size",
+         "memory.init\t$seg, $idx", 0x08>;
+
+//===----------------------------------------------------------------------===//
+// data.drop
+//===----------------------------------------------------------------------===//
+
+defm DATA_DROP :
+  BULK_I<(outs), (ins i32imm_op:$seg), (outs), (ins i32imm_op:$seg),
+         [(int_wasm_data_drop (i32 imm:$seg))],
+         "data.drop\t$seg", "data.drop\t$seg", 0x09>;
+
+//===----------------------------------------------------------------------===//
+// memory.copy
+//===----------------------------------------------------------------------===//
+
+let mayLoad = 1, mayStore = 1 in
+defm MEMORY_COPY :
+  BULK_I<(outs), (ins i32imm_op:$src_idx, i32imm_op:$dst_idx,
+                      I32:$dst, I32:$src, I32:$len),
+         (outs), (ins i32imm_op:$src_idx, i32imm_op:$dst_idx),
+         [(wasm_memcpy (i32 imm:$src_idx), (i32 imm:$dst_idx),
+           I32:$dst, I32:$src, I32:$len
+         )],
+         "memory.copy\t$src_idx, $dst_idx, $dst, $src, $len",
+         "memory.copy\t$src_idx, $dst_idx", 0x0a>;
diff --git a/lib/Target/WebAssembly/WebAssemblyInstrCall.td b/lib/Target/WebAssembly/WebAssemblyInstrCall.td
index 07839b7901142e27df529bc6cc3242800d8c5b73..190328c82e524cf6d438a6d272bb46451e0059bb 100644
--- a/lib/Target/WebAssembly/WebAssemblyInstrCall.td
+++ b/lib/Target/WebAssembly/WebAssemblyInstrCall.td
@@ -1,9 +1,8 @@
 //===- WebAssemblyInstrCall.td-WebAssembly Call codegen support -*- tablegen -*-
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
@@ -22,7 +21,7 @@ defm ADJCALLSTACKDOWN : NRI<(outs), (ins i32imm:$amt, i32imm:$amt2),
                             [(WebAssemblycallseq_start timm:$amt, timm:$amt2)]>;
 defm ADJCALLSTACKUP : NRI<(outs), (ins i32imm:$amt, i32imm:$amt2),
                           [(WebAssemblycallseq_end timm:$amt, timm:$amt2)]>;
-} // isCodeGenOnly = 1
+} // Uses = [SP32, SP64], Defs = [SP32, SP64], isCodeGenOnly = 1
 
 multiclass CALL<WebAssemblyRegClass vt, string prefix> {
   defm CALL_#vt : I<(outs vt:$dst), (ins function32_op:$callee, variable_ops),
@@ -32,13 +31,12 @@ multiclass CALL<WebAssemblyRegClass vt, string prefix> {
                     !strconcat(prefix, "call\t$callee"),
                     0x10>;
 
-  let isCodeGenOnly = 1 in {
-    defm PCALL_INDIRECT_#vt : I<(outs vt:$dst), (ins I32:$callee, variable_ops),
-                                (outs), (ins I32:$callee),
-                               [(set vt:$dst, (WebAssemblycall1 I32:$callee))],
-                               "PSEUDO CALL INDIRECT\t$callee",
-                               "PSEUDO CALL INDIRECT\t$callee">;
-  } // isCodeGenOnly = 1
+  let isCodeGenOnly = 1 in
+  defm PCALL_INDIRECT_#vt : I<(outs vt:$dst), (ins I32:$callee, variable_ops),
+                              (outs), (ins I32:$callee),
+                              [(set vt:$dst, (WebAssemblycall1 I32:$callee))],
+                              "PSEUDO CALL INDIRECT\t$callee",
+                              "PSEUDO CALL INDIRECT\t$callee">;
 
   defm CALL_INDIRECT_#vt : I<(outs vt:$dst),
                              (ins TypeIndex:$type, i32imm:$flags, variable_ops),
@@ -60,16 +58,15 @@ multiclass SIMD_CALL<ValueType vt, string prefix> {
                     0x10>,
                   Requires<[HasSIMD128]>;
 
-  let isCodeGenOnly = 1 in {
-    defm PCALL_INDIRECT_#vt : I<(outs V128:$dst),
-                                (ins I32:$callee, variable_ops),
-                                (outs), (ins I32:$callee),
-                                [(set (vt V128:$dst),
-                                      (WebAssemblycall1 I32:$callee))],
-                                "PSEUDO CALL INDIRECT\t$callee",
-                                "PSEUDO CALL INDIRECT\t$callee">,
+  let isCodeGenOnly = 1 in
+  defm PCALL_INDIRECT_#vt : I<(outs V128:$dst),
+                              (ins I32:$callee, variable_ops),
+                              (outs), (ins I32:$callee),
+                              [(set (vt V128:$dst),
+                                    (WebAssemblycall1 I32:$callee))],
+                              "PSEUDO CALL INDIRECT\t$callee",
+                              "PSEUDO CALL INDIRECT\t$callee">,
                               Requires<[HasSIMD128]>;
-  } // isCodeGenOnly = 1
 
   defm CALL_INDIRECT_#vt : I<(outs V128:$dst),
                              (ins TypeIndex:$type, i32imm:$flags, variable_ops),
@@ -78,42 +75,43 @@ multiclass SIMD_CALL<ValueType vt, string prefix> {
                              !strconcat(prefix, "call_indirect\t$dst"),
                              !strconcat(prefix, "call_indirect\t$type"),
                              0x11>,
-                           Requires<[HasSIMD128]>;
+                             Requires<[HasSIMD128]>;
 }
 
 let Uses = [SP32, SP64], isCall = 1 in {
-  defm "" : CALL<I32, "i32.">;
-  defm "" : CALL<I64, "i64.">;
-  defm "" : CALL<F32, "f32.">;
-  defm "" : CALL<F64, "f64.">;
-  defm "" : CALL<EXCEPT_REF, "except_ref.">;
-  defm "" : SIMD_CALL<v16i8, "v128.">;
-  defm "" : SIMD_CALL<v8i16, "v128.">;
-  defm "" : SIMD_CALL<v4i32, "v128.">;
-  defm "" : SIMD_CALL<v2i64, "v128.">;
-  defm "" : SIMD_CALL<v4f32, "v128.">;
-  defm "" : SIMD_CALL<v2f64, "v128.">;
+defm "" : CALL<I32, "i32.">;
+defm "" : CALL<I64, "i64.">;
+defm "" : CALL<F32, "f32.">;
+defm "" : CALL<F64, "f64.">;
+defm "" : CALL<EXCEPT_REF, "except_ref.">;
+defm "" : SIMD_CALL<v16i8, "v128.">;
+defm "" : SIMD_CALL<v8i16, "v128.">;
+defm "" : SIMD_CALL<v4i32, "v128.">;
+defm "" : SIMD_CALL<v2i64, "v128.">;
+defm "" : SIMD_CALL<v4f32, "v128.">;
+defm "" : SIMD_CALL<v2f64, "v128.">;
 
-  defm CALL_VOID : I<(outs), (ins function32_op:$callee, variable_ops),
-                     (outs), (ins function32_op:$callee),
-                     [(WebAssemblycall0 (i32 imm:$callee))],
-                     "call    \t$callee", "call\t$callee", 0x10>;
+let IsCanonical = 1 in {
+defm CALL_VOID : I<(outs), (ins function32_op:$callee, variable_ops),
+                   (outs), (ins function32_op:$callee),
+                   [(WebAssemblycall0 (i32 imm:$callee))],
+                   "call    \t$callee", "call\t$callee", 0x10>;
 
-  let isCodeGenOnly = 1 in {
-    defm PCALL_INDIRECT_VOID : I<(outs), (ins I32:$callee, variable_ops),
-                                 (outs), (ins I32:$callee),
-                                 [(WebAssemblycall0 I32:$callee)],
-                                 "PSEUDO CALL INDIRECT\t$callee",
-                                 "PSEUDO CALL INDIRECT\t$callee">;
-  } // isCodeGenOnly = 1
+let isCodeGenOnly = 1 in
+defm PCALL_INDIRECT_VOID : I<(outs), (ins I32:$callee, variable_ops),
+                             (outs), (ins I32:$callee),
+                             [(WebAssemblycall0 I32:$callee)],
+                             "PSEUDO CALL INDIRECT\t$callee",
+                             "PSEUDO CALL INDIRECT\t$callee">;
 
-  defm CALL_INDIRECT_VOID : I<(outs),
-                              (ins TypeIndex:$type, i32imm:$flags,
-                                variable_ops),
-                              (outs), (ins TypeIndex:$type, i32imm:$flags),
-                              [],
-                              "call_indirect\t", "call_indirect\t$type",
-                              0x11>;
+defm CALL_INDIRECT_VOID : I<(outs),
+                            (ins TypeIndex:$type, i32imm:$flags,
+                              variable_ops),
+                            (outs), (ins TypeIndex:$type, i32imm:$flags),
+                            [],
+                            "call_indirect\t", "call_indirect\t$type",
+                            0x11>;
+} // IsCanonical = 1
 } // Uses = [SP32,SP64], isCall = 1
 
 // Patterns for matching a direct call to a global address.
diff --git a/lib/Target/WebAssembly/WebAssemblyInstrControl.td b/lib/Target/WebAssembly/WebAssemblyInstrControl.td
index a7fc7564009007474825f53e500007dc8bf5deb7..d44458f790a4c014a51374681f2d5c648e48cd80 100644
--- a/lib/Target/WebAssembly/WebAssemblyInstrControl.td
+++ b/lib/Target/WebAssembly/WebAssemblyInstrControl.td
@@ -1,9 +1,8 @@
 //===- WebAssemblyInstrControl.td-WebAssembly control-flow ------*- tablegen -*-
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
@@ -21,11 +20,10 @@ defm BR_IF : I<(outs), (ins bb_op:$dst, I32:$cond),
 let isCodeGenOnly = 1 in
 defm BR_UNLESS : I<(outs), (ins bb_op:$dst, I32:$cond),
                    (outs), (ins bb_op:$dst), []>;
-let isBarrier = 1 in {
+let isBarrier = 1 in
 defm BR   : NRI<(outs), (ins bb_op:$dst),
                 [(br bb:$dst)],
                 "br      \t$dst", 0x0c>;
-} // isBarrier = 1
 } // isBranch = 1, isTerminator = 1, hasCtrlDep = 1
 
 def : Pat<(brcond (i32 (setne I32:$cond, 0)), bb:$dst),
@@ -36,6 +34,7 @@ def : Pat<(brcond (i32 (seteq I32:$cond, 0)), bb:$dst),
 // A list of branch targets enclosed in {} and separated by comma.
 // Used by br_table only.
 def BrListAsmOperand : AsmOperandClass { let Name = "BrList"; }
+let OperandNamespace = "WebAssembly", OperandType = "OPERAND_BRLIST" in
 def brlist : Operand<i32> {
   let ParserMatchClass = BrListAsmOperand;
   let PrintMethod = "printBrList";
@@ -44,27 +43,17 @@ def brlist : Operand<i32> {
 // TODO: SelectionDAG's lowering insists on using a pointer as the index for
 // jump tables, so in practice we don't ever use BR_TABLE_I64 in wasm32 mode
 // currently.
-// FIXME: this can't inherit from I<> since there is no way to inherit from a
-// multiclass and still have the let statements.
 let isTerminator = 1, hasCtrlDep = 1, isBarrier = 1 in {
-let isCodeGenOnly = 1 in
-def BR_TABLE_I32 : NI<(outs), (ins I32:$index, variable_ops),
-                      [(WebAssemblybr_table I32:$index)], "false",
-                      "br_table \t$index", 0x0e> {
-}
-let BaseName = "BR_TABLE_I32" in
-def BR_TABLE_I32_S : NI<(outs), (ins brlist:$brl), [], "true",
-                        "br_table \t$brl", 0x0e> {
-}
-let isCodeGenOnly = 1 in
-def BR_TABLE_I64 : NI<(outs), (ins I64:$index, variable_ops),
-                      [(WebAssemblybr_table I64:$index)], "false",
-                      "br_table \t$index"> {
-}
-let BaseName = "BR_TABLE_I64" in
-def BR_TABLE_I64_S : NI<(outs), (ins brlist:$brl), [], "true",
-                        "br_table \t$brl"> {
-}
+defm BR_TABLE_I32 : I<(outs), (ins I32:$index, variable_ops),
+                      (outs), (ins brlist:$brl),
+                      [(WebAssemblybr_table I32:$index)],
+                      "br_table \t$index", "br_table \t$brl",
+                      0x0e>;
+defm BR_TABLE_I64 : I<(outs), (ins I64:$index, variable_ops),
+                      (outs), (ins brlist:$brl),
+                      [(WebAssemblybr_table I64:$index)],
+                      "br_table \t$index", "br_table \t$brl",
+                      0x0e>;
 } // isTerminator = 1, hasCtrlDep = 1, isBarrier = 1
 
 // This is technically a control-flow instruction, since all it affects is the
@@ -75,13 +64,22 @@ defm NOP : NRI<(outs), (ins), [], "nop", 0x01>;
 // These use/clobber VALUE_STACK to prevent them from being moved into the
 // middle of an expression tree.
 let Uses = [VALUE_STACK], Defs = [VALUE_STACK] in {
-defm BLOCK     : NRI<(outs), (ins Signature:$sig), [], "block   \t$sig", 0x02>;
-defm LOOP      : NRI<(outs), (ins Signature:$sig), [], "loop    \t$sig", 0x03>;
+defm BLOCK : NRI<(outs), (ins Signature:$sig), [], "block   \t$sig", 0x02>;
+defm LOOP  : NRI<(outs), (ins Signature:$sig), [], "loop    \t$sig", 0x03>;
+
+defm IF : I<(outs), (ins Signature:$sig, I32:$cond),
+            (outs), (ins Signature:$sig),
+            [], "if    \t$sig, $cond", "if    \t$sig", 0x04>;
+defm ELSE : NRI<(outs), (ins), [], "else", 0x05>;
 
-// END_BLOCK, END_LOOP, and END_FUNCTION are represented with the same opcode in
-// wasm.
+// END_BLOCK, END_LOOP, END_IF and END_FUNCTION are represented with the same
+// opcode in wasm.
 defm END_BLOCK : NRI<(outs), (ins), [], "end_block", 0x0b>;
 defm END_LOOP  : NRI<(outs), (ins), [], "end_loop", 0x0b>;
+defm END_IF    : NRI<(outs), (ins), [], "end_if", 0x0b>;
+// Generic instruction, for disassembler.
+let IsCanonical = 1 in
+defm END       : NRI<(outs), (ins), [], "end", 0x0b>;
 let isTerminator = 1, isBarrier = 1 in
 defm END_FUNCTION : NRI<(outs), (ins), [], "end_function", 0x0b>;
 } // Uses = [VALUE_STACK], Defs = [VALUE_STACK]
@@ -106,7 +104,7 @@ multiclass SIMD_RETURN<ValueType vt> {
   let isCodeGenOnly = 1 in
   defm FALLTHROUGH_RETURN_#vt : I<(outs), (ins V128:$val), (outs), (ins),
                                   []>,
-                                Requires<[HasSIMD128]>;
+                                  Requires<[HasSIMD128]>;
 }
 
 let isTerminator = 1, hasCtrlDep = 1, isBarrier = 1 in {
@@ -142,23 +140,11 @@ let Predicates = [HasExceptionHandling] in {
 
 // Throwing an exception: throw / rethrow
 let isTerminator = 1, hasCtrlDep = 1, isBarrier = 1 in {
-defm THROW_I32 : I<(outs), (ins event_op:$tag, I32:$val),
-                   (outs), (ins event_op:$tag),
-                   [(WebAssemblythrow (WebAssemblywrapper texternalsym:$tag),
-                                      I32:$val)],
-                   "throw   \t$tag, $val", "throw   \t$tag",
-                   0x08>;
-defm THROW_I64 : I<(outs), (ins event_op:$tag, I64:$val),
-                   (outs), (ins event_op:$tag),
-                   [(WebAssemblythrow (WebAssemblywrapper texternalsym:$tag),
-                                      I64:$val)],
-                   "throw   \t$tag, $val", "throw   \t$tag",
-                   0x08>;
-defm RETHROW : NRI<(outs), (ins bb_op:$dst), [], "rethrow \t$dst", 0x09>;
-let isCodeGenOnly = 1 in
-// This is used when the destination for rethrow is the caller function. This
-// will be converted to a rethrow in CFGStackify.
-defm RETHROW_TO_CALLER : NRI<(outs), (ins), [], "rethrow">;
+defm THROW : I<(outs), (ins event_op:$tag, variable_ops),
+               (outs), (ins event_op:$tag),
+               [(WebAssemblythrow (WebAssemblywrapper texternalsym:$tag))],
+               "throw   \t$tag", "throw   \t$tag", 0x08>;
+defm RETHROW : NRI<(outs), (ins), [(int_wasm_rethrow)], "rethrow", 0x09>;
 } // isTerminator = 1, hasCtrlDep = 1, isBarrier = 1
 
 // Region within which an exception is caught: try / end_try
@@ -167,24 +153,33 @@ defm TRY     : NRI<(outs), (ins Signature:$sig), [], "try     \t$sig", 0x06>;
 defm END_TRY : NRI<(outs), (ins), [], "end_try", 0x0b>;
 } // Uses = [VALUE_STACK], Defs = [VALUE_STACK]
 
-// Catching an exception: catch / catch_all
-let hasCtrlDep = 1, hasSideEffects = 1 in {
-defm CATCH_I32 : I<(outs I32:$dst), (ins i32imm:$tag),
-                   (outs), (ins i32imm:$tag),
-                   [(set I32:$dst, (int_wasm_catch imm:$tag))],
-                   "i32.catch   \t$dst, $tag", "i32.catch   \t$tag", 0x07>;
-defm CATCH_I64 : I<(outs I64:$dst), (ins i32imm:$tag),
-                   (outs), (ins i32imm:$tag),
-                   [(set I64:$dst, (int_wasm_catch imm:$tag))],
-                   "i64.catch   \t$dst, $tag", "i64.catch   \t$tag", 0x07>;
-defm CATCH_ALL : NRI<(outs), (ins), [], "catch_all", 0x05>;
-}
+// Catching an exception: catch / extract_exception
+let hasCtrlDep = 1, hasSideEffects = 1 in
+defm CATCH : I<(outs EXCEPT_REF:$dst), (ins), (outs), (ins), [],
+               "catch   \t$dst", "catch", 0x07>;
+
+// Querying / extracing exception: br_on_exn
+// br_on_exn queries an except_ref to see if it matches the corresponding
+// exception tag index. If true it branches to the given label and pushes the
+// corresponding argument values of the exception onto the stack.
+let isBranch = 1, isTerminator = 1, hasCtrlDep = 1 in
+defm BR_ON_EXN : I<(outs), (ins bb_op:$dst, event_op:$tag, EXCEPT_REF:$exn),
+                   (outs), (ins bb_op:$dst, event_op:$tag), [],
+                   "br_on_exn \t$dst, $tag, $exn", "br_on_exn \t$dst, $tag",
+                   0x0a>;
+// This is a pseudo instruction that simulates popping a value from stack, which
+// has been pushed by br_on_exn
+let isCodeGenOnly = 1, hasSideEffects = 1 in
+defm EXTRACT_EXCEPTION_I32 : NRI<(outs I32:$dst), (ins),
+                                 [(set I32:$dst, (int_wasm_extract_exception))],
+                                 "extract_exception\t$dst">;
 
 // Pseudo instructions: cleanupret / catchret
 let isTerminator = 1, hasSideEffects = 1, isBarrier = 1, hasCtrlDep = 1,
-    isCodeGenOnly = 1, isEHScopeReturn = 1 in {
-  defm CLEANUPRET : NRI<(outs), (ins), [(cleanupret)], "", 0>;
+    isPseudo = 1, isEHScopeReturn = 1 in {
+  defm CLEANUPRET : NRI<(outs), (ins), [(cleanupret)], "cleanupret", 0>;
   defm CATCHRET : NRI<(outs), (ins bb_op:$dst, bb_op:$from),
-                   [(catchret bb:$dst, bb:$from)], "", 0>;
-}
-}
+                      [(catchret bb:$dst, bb:$from)], "catchret", 0>;
+} // isTerminator = 1, hasSideEffects = 1, isBarrier = 1, hasCtrlDep = 1,
+  // isPseudo = 1, isEHScopeReturn = 1
+} // Predicates = [HasExceptionHandling]
diff --git a/lib/Target/WebAssembly/WebAssemblyInstrConv.td b/lib/Target/WebAssembly/WebAssemblyInstrConv.td
index 0d772c743a759a7cea874027990fdd49394661e5..661fee2715ba1a07e09755858532f33d1b62064f 100644
--- a/lib/Target/WebAssembly/WebAssemblyInstrConv.td
+++ b/lib/Target/WebAssembly/WebAssemblyInstrConv.td
@@ -1,9 +1,8 @@
 //===-- WebAssemblyInstrConv.td-WebAssembly Conversion support -*- tablegen -*-=
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
@@ -15,15 +14,15 @@
 
 defm I32_WRAP_I64 : I<(outs I32:$dst), (ins I64:$src), (outs), (ins),
                       [(set I32:$dst, (trunc I64:$src))],
-                      "i32.wrap/i64\t$dst, $src", "i32.wrap/i64", 0xa7>;
+                      "i32.wrap_i64\t$dst, $src", "i32.wrap_i64", 0xa7>;
 
 defm I64_EXTEND_S_I32 : I<(outs I64:$dst), (ins I32:$src), (outs), (ins),
                           [(set I64:$dst, (sext I32:$src))],
-                          "i64.extend_s/i32\t$dst, $src", "i64.extend_s/i32",
+                          "i64.extend_i32_s\t$dst, $src", "i64.extend_i32_s",
                           0xac>;
 defm I64_EXTEND_U_I32 : I<(outs I64:$dst), (ins I32:$src), (outs), (ins),
                           [(set I64:$dst, (zext I32:$src))],
-                          "i64.extend_u/i32\t$dst, $src", "i64.extend_u/i32",
+                          "i64.extend_i32_u\t$dst, $src", "i64.extend_i32_u",
                           0xad>;
 
 let Predicates = [HasSignExt] in {
@@ -58,43 +57,43 @@ def : Pat<(i64 (anyext I32:$src)), (I64_EXTEND_U_I32 I32:$src)>;
 // overflow or invalid.
 defm I32_TRUNC_S_SAT_F32 : I<(outs I32:$dst), (ins F32:$src), (outs), (ins),
                              [(set I32:$dst, (fp_to_sint F32:$src))],
-                             "i32.trunc_s:sat/f32\t$dst, $src",
-                             "i32.trunc_s:sat/f32", 0xfc00>,
+                             "i32.trunc_sat_f32_s\t$dst, $src",
+                             "i32.trunc_sat_f32_s", 0xfc00>,
                              Requires<[HasNontrappingFPToInt]>;
 defm I32_TRUNC_U_SAT_F32 : I<(outs I32:$dst), (ins F32:$src), (outs), (ins),
                              [(set I32:$dst, (fp_to_uint F32:$src))],
-                             "i32.trunc_u:sat/f32\t$dst, $src",
-                             "i32.trunc_u:sat/f32", 0xfc01>,
+                             "i32.trunc_sat_f32_u\t$dst, $src",
+                             "i32.trunc_sat_f32_u", 0xfc01>,
                              Requires<[HasNontrappingFPToInt]>;
 defm I64_TRUNC_S_SAT_F32 : I<(outs I64:$dst), (ins F32:$src), (outs), (ins),
                              [(set I64:$dst, (fp_to_sint F32:$src))],
-                             "i64.trunc_s:sat/f32\t$dst, $src",
-                             "i64.trunc_s:sat/f32", 0xfc04>,
+                             "i64.trunc_sat_f32_s\t$dst, $src",
+                             "i64.trunc_sat_f32_s", 0xfc04>,
                              Requires<[HasNontrappingFPToInt]>;
 defm I64_TRUNC_U_SAT_F32 : I<(outs I64:$dst), (ins F32:$src), (outs), (ins),
                              [(set I64:$dst, (fp_to_uint F32:$src))],
-                             "i64.trunc_u:sat/f32\t$dst, $src",
-                             "i64.trunc_u:sat/f32", 0xfc05>,
+                             "i64.trunc_sat_f32_u\t$dst, $src",
+                             "i64.trunc_sat_f32_u", 0xfc05>,
                              Requires<[HasNontrappingFPToInt]>;
 defm I32_TRUNC_S_SAT_F64 : I<(outs I32:$dst), (ins F64:$src), (outs), (ins),
                              [(set I32:$dst, (fp_to_sint F64:$src))],
-                             "i32.trunc_s:sat/f64\t$dst, $src",
-                             "i32.trunc_s:sat/f64", 0xfc02>,
+                             "i32.trunc_sat_f64_s\t$dst, $src",
+                             "i32.trunc_sat_f64_s", 0xfc02>,
                              Requires<[HasNontrappingFPToInt]>;
 defm I32_TRUNC_U_SAT_F64 : I<(outs I32:$dst), (ins F64:$src), (outs), (ins),
                              [(set I32:$dst, (fp_to_uint F64:$src))],
-                             "i32.trunc_u:sat/f64\t$dst, $src",
-                             "i32.trunc_u:sat/f64", 0xfc03>,
+                             "i32.trunc_sat_f64_u\t$dst, $src",
+                             "i32.trunc_sat_f64_u", 0xfc03>,
                              Requires<[HasNontrappingFPToInt]>;
 defm I64_TRUNC_S_SAT_F64 : I<(outs I64:$dst), (ins F64:$src), (outs), (ins),
                              [(set I64:$dst, (fp_to_sint F64:$src))],
-                             "i64.trunc_s:sat/f64\t$dst, $src",
-                             "i64.trunc_s:sat/f64", 0xfc06>,
+                             "i64.trunc_sat_f64_s\t$dst, $src",
+                             "i64.trunc_sat_f64_s", 0xfc06>,
                              Requires<[HasNontrappingFPToInt]>;
 defm I64_TRUNC_U_SAT_F64 : I<(outs I64:$dst), (ins F64:$src), (outs), (ins),
                              [(set I64:$dst, (fp_to_uint F64:$src))],
-                             "i64.trunc_u:sat/f64\t$dst, $src",
-                             "i64.trunc_u:sat/f64", 0xfc07>,
+                             "i64.trunc_sat_f64_u\t$dst, $src",
+                             "i64.trunc_sat_f64_u", 0xfc07>,
                              Requires<[HasNontrappingFPToInt]>;
 
 // Lower llvm.wasm.trunc.saturate.* to saturating instructions
@@ -147,86 +146,86 @@ defm FP_TO_UINT_I64_F64 : I<(outs I64:$dst), (ins F64:$src), (outs), (ins),
 // Conversion from floating point to integer traps on overflow and invalid.
 let hasSideEffects = 1 in {
 defm I32_TRUNC_S_F32 : I<(outs I32:$dst), (ins F32:$src), (outs), (ins),
-                         [], "i32.trunc_s/f32\t$dst, $src", "i32.trunc_s/f32",
+                         [], "i32.trunc_f32_s\t$dst, $src", "i32.trunc_f32_s",
                          0xa8>;
 defm I32_TRUNC_U_F32 : I<(outs I32:$dst), (ins F32:$src), (outs), (ins),
-                         [], "i32.trunc_u/f32\t$dst, $src", "i32.trunc_u/f32",
+                         [], "i32.trunc_f32_u\t$dst, $src", "i32.trunc_f32_u",
                          0xa9>;
 defm I64_TRUNC_S_F32 : I<(outs I64:$dst), (ins F32:$src), (outs), (ins),
-                         [], "i64.trunc_s/f32\t$dst, $src", "i64.trunc_s/f32",
+                         [], "i64.trunc_f32_s\t$dst, $src", "i64.trunc_f32_s",
                          0xae>;
 defm I64_TRUNC_U_F32 : I<(outs I64:$dst), (ins F32:$src), (outs), (ins),
-                         [], "i64.trunc_u/f32\t$dst, $src", "i64.trunc_u/f32",
+                         [], "i64.trunc_f32_u\t$dst, $src", "i64.trunc_f32_u",
                          0xaf>;
 defm I32_TRUNC_S_F64 : I<(outs I32:$dst), (ins F64:$src), (outs), (ins),
-                         [], "i32.trunc_s/f64\t$dst, $src", "i32.trunc_s/f64",
+                         [], "i32.trunc_f64_s\t$dst, $src", "i32.trunc_f64_s",
                          0xaa>;
 defm I32_TRUNC_U_F64 : I<(outs I32:$dst), (ins F64:$src), (outs), (ins),
-                         [], "i32.trunc_u/f64\t$dst, $src", "i32.trunc_u/f64",
+                         [], "i32.trunc_f64_u\t$dst, $src", "i32.trunc_f64_u",
                          0xab>;
 defm I64_TRUNC_S_F64 : I<(outs I64:$dst), (ins F64:$src), (outs), (ins),
-                         [], "i64.trunc_s/f64\t$dst, $src", "i64.trunc_s/f64",
+                         [], "i64.trunc_f64_s\t$dst, $src", "i64.trunc_f64_s",
                          0xb0>;
 defm I64_TRUNC_U_F64 : I<(outs I64:$dst), (ins F64:$src), (outs), (ins),
-                         [], "i64.trunc_u/f64\t$dst, $src", "i64.trunc_u/f64",
+                         [], "i64.trunc_f64_u\t$dst, $src", "i64.trunc_f64_u",
                          0xb1>;
 } // hasSideEffects = 1
 
 defm F32_CONVERT_S_I32 : I<(outs F32:$dst), (ins I32:$src), (outs), (ins),
                            [(set F32:$dst, (sint_to_fp I32:$src))],
-                           "f32.convert_s/i32\t$dst, $src", "f32.convert_s/i32",
+                           "f32.convert_i32_s\t$dst, $src", "f32.convert_i32_s",
                            0xb2>;
 defm F32_CONVERT_U_I32 : I<(outs F32:$dst), (ins I32:$src), (outs), (ins),
                            [(set F32:$dst, (uint_to_fp I32:$src))],
-                           "f32.convert_u/i32\t$dst, $src", "f32.convert_u/i32",
+                           "f32.convert_i32_u\t$dst, $src", "f32.convert_i32_u",
                            0xb3>;
 defm F64_CONVERT_S_I32 : I<(outs F64:$dst), (ins I32:$src), (outs), (ins),
                            [(set F64:$dst, (sint_to_fp I32:$src))],
-                           "f64.convert_s/i32\t$dst, $src", "f64.convert_s/i32",
+                           "f64.convert_i32_s\t$dst, $src", "f64.convert_i32_s",
                            0xb7>;
 defm F64_CONVERT_U_I32 : I<(outs F64:$dst), (ins I32:$src), (outs), (ins),
                            [(set F64:$dst, (uint_to_fp I32:$src))],
-                           "f64.convert_u/i32\t$dst, $src", "f64.convert_u/i32",
+                           "f64.convert_i32_u\t$dst, $src", "f64.convert_i32_u",
                            0xb8>;
 defm F32_CONVERT_S_I64 : I<(outs F32:$dst), (ins I64:$src), (outs), (ins),
                            [(set F32:$dst, (sint_to_fp I64:$src))],
-                           "f32.convert_s/i64\t$dst, $src", "f32.convert_s/i64",
+                           "f32.convert_i64_s\t$dst, $src", "f32.convert_i64_s",
                            0xb4>;
 defm F32_CONVERT_U_I64 : I<(outs F32:$dst), (ins I64:$src), (outs), (ins),
                            [(set F32:$dst, (uint_to_fp I64:$src))],
-                           "f32.convert_u/i64\t$dst, $src", "f32.convert_u/i64",
+                           "f32.convert_i64_u\t$dst, $src", "f32.convert_i64_u",
                            0xb5>;
 defm F64_CONVERT_S_I64 : I<(outs F64:$dst), (ins I64:$src), (outs), (ins),
                            [(set F64:$dst, (sint_to_fp I64:$src))],
-                           "f64.convert_s/i64\t$dst, $src", "f64.convert_s/i64",
+                           "f64.convert_i64_s\t$dst, $src", "f64.convert_i64_s",
                            0xb9>;
 defm F64_CONVERT_U_I64 : I<(outs F64:$dst), (ins I64:$src), (outs), (ins),
                            [(set F64:$dst, (uint_to_fp I64:$src))],
-                           "f64.convert_u/i64\t$dst, $src", "f64.convert_u/i64",
+                           "f64.convert_i64_u\t$dst, $src", "f64.convert_i64_u",
                            0xba>;
 
 defm F64_PROMOTE_F32 : I<(outs F64:$dst), (ins F32:$src), (outs), (ins),
                          [(set F64:$dst, (fpextend F32:$src))],
-                         "f64.promote/f32\t$dst, $src", "f64.promote/f32",
+                         "f64.promote_f32\t$dst, $src", "f64.promote_f32",
                          0xbb>;
 defm F32_DEMOTE_F64 : I<(outs F32:$dst), (ins F64:$src), (outs), (ins),
                         [(set F32:$dst, (fpround F64:$src))],
-                        "f32.demote/f64\t$dst, $src", "f32.demote/f64",
+                        "f32.demote_f64\t$dst, $src", "f32.demote_f64",
                         0xb6>;
 
 defm I32_REINTERPRET_F32 : I<(outs I32:$dst), (ins F32:$src), (outs), (ins),
                              [(set I32:$dst, (bitconvert F32:$src))],
-                             "i32.reinterpret/f32\t$dst, $src",
-                             "i32.reinterpret/f32", 0xbc>;
+                             "i32.reinterpret_f32\t$dst, $src",
+                             "i32.reinterpret_f32", 0xbc>;
 defm F32_REINTERPRET_I32 : I<(outs F32:$dst), (ins I32:$src), (outs), (ins),
                              [(set F32:$dst, (bitconvert I32:$src))],
-                             "f32.reinterpret/i32\t$dst, $src",
-                             "f32.reinterpret/i32", 0xbe>;
+                             "f32.reinterpret_i32\t$dst, $src",
+                             "f32.reinterpret_i32", 0xbe>;
 defm I64_REINTERPRET_F64 : I<(outs I64:$dst), (ins F64:$src), (outs), (ins),
                              [(set I64:$dst, (bitconvert F64:$src))],
-                             "i64.reinterpret/f64\t$dst, $src",
-                             "i64.reinterpret/f64", 0xbd>;
+                             "i64.reinterpret_f64\t$dst, $src",
+                             "i64.reinterpret_f64", 0xbd>;
 defm F64_REINTERPRET_I64 : I<(outs F64:$dst), (ins I64:$src), (outs), (ins),
                              [(set F64:$dst, (bitconvert I64:$src))],
-                             "f64.reinterpret/i64\t$dst, $src",
-                             "f64.reinterpret/i64", 0xbf>;
+                             "f64.reinterpret_i64\t$dst, $src",
+                             "f64.reinterpret_i64", 0xbf>;
diff --git a/lib/Target/WebAssembly/WebAssemblyInstrExceptRef.td b/lib/Target/WebAssembly/WebAssemblyInstrExceptRef.td
index a251d60b89ee283cd541b2f96f41aca16c69666e..33a4f2519545d8081d68a81b3c6d6e180ad1a9f4 100644
--- a/lib/Target/WebAssembly/WebAssemblyInstrExceptRef.td
+++ b/lib/Target/WebAssembly/WebAssemblyInstrExceptRef.td
@@ -1,9 +1,8 @@
 // WebAssemblyInstrExceptRef.td-WebAssembly except_ref codegen --*- tablegen -*-
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
diff --git a/lib/Target/WebAssembly/WebAssemblyInstrFloat.td b/lib/Target/WebAssembly/WebAssemblyInstrFloat.td
index c5290f00b431b520818df570ed2bc5414b6d4d33..5c9b34f447346104adab24be5c417837896e3c13 100644
--- a/lib/Target/WebAssembly/WebAssemblyInstrFloat.td
+++ b/lib/Target/WebAssembly/WebAssemblyInstrFloat.td
@@ -1,9 +1,8 @@
 // WebAssemblyInstrFloat.td-WebAssembly Float codegen support ---*- tablegen -*-
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
diff --git a/lib/Target/WebAssembly/WebAssemblyInstrFormats.td b/lib/Target/WebAssembly/WebAssemblyInstrFormats.td
index 97583ea0e6ac05cf08b94e83c6ace23a1ef81b38..aff4d20d8d823c997f14505710dac2ad38ebb071 100644
--- a/lib/Target/WebAssembly/WebAssemblyInstrFormats.td
+++ b/lib/Target/WebAssembly/WebAssemblyInstrFormats.td
@@ -1,9 +1,8 @@
 //=- WebAssemblyInstrFormats.td - WebAssembly Instr. Formats -*- tablegen -*-=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
@@ -23,6 +22,9 @@ class WebAssemblyInst<bits<32> inst, string asmstr, string stack> : StackRel,
   let Namespace   = "WebAssembly";
   let Pattern     = [];
   let AsmString   = asmstr;
+  // When there are multiple instructions that map to the same encoding (in
+  // e.g. the disassembler use case) prefer the one where IsCanonical == 1.
+  bit IsCanonical = 0;
 }
 
 // Normal instructions. Default instantiation of a WebAssemblyInst.
@@ -40,10 +42,10 @@ class NI<dag oops, dag iops, list<dag> pattern, string stack,
 // based version of this instruction, as well as the corresponding asmstr.
 // The register versions have virtual-register operands which correspond to wasm
 // locals or stack locations. Each use and def of the register corresponds to an
-// implicit get_local / set_local or access of stack operands in wasm. These
+// implicit local.get / local.set or access of stack operands in wasm. These
 // instructions are used for ISel and all MI passes. The stack versions of the
 // instructions do not have register operands (they implicitly operate on the
-// stack), and get_locals and set_locals are explicit. The register instructions
+// stack), and local.gets and local.sets are explicit. The register instructions
 // are converted to their corresponding stack instructions before lowering to
 // MC.
 // Every instruction should want to be based on this multi-class to guarantee
diff --git a/lib/Target/WebAssembly/WebAssemblyInstrInfo.cpp b/lib/Target/WebAssembly/WebAssemblyInstrInfo.cpp
index 5efff32d616768d1abfc4c2cfdcaadc0d58a89db..995dc88bb333f4b98ba0b989abe01b346f9c1083 100644
--- a/lib/Target/WebAssembly/WebAssemblyInstrInfo.cpp
+++ b/lib/Target/WebAssembly/WebAssemblyInstrInfo.cpp
@@ -1,9 +1,8 @@
 //===-- WebAssemblyInstrInfo.cpp - WebAssembly Instruction Information ----===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
@@ -135,6 +134,17 @@ bool WebAssemblyInstrInfo::analyzeBranch(MachineBasicBlock &MBB,
       else
         FBB = MI.getOperand(0).getMBB();
       break;
+    case WebAssembly::BR_ON_EXN:
+      if (HaveCond)
+        return true;
+      // If we're running after CFGStackify, we can't optimize further.
+      if (!MI.getOperand(0).isMBB())
+        return true;
+      Cond.push_back(MachineOperand::CreateImm(true));
+      Cond.push_back(MI.getOperand(2));
+      TBB = MI.getOperand(0).getMBB();
+      HaveCond = true;
+      break;
     }
     if (MI.isBarrier())
       break;
@@ -180,9 +190,22 @@ unsigned WebAssemblyInstrInfo::insertBranch(
 
   assert(Cond.size() == 2 && "Expected a flag and a successor block");
 
+  MachineFunction &MF = *MBB.getParent();
+  auto &MRI = MF.getRegInfo();
+  bool IsBrOnExn = Cond[1].isReg() && MRI.getRegClass(Cond[1].getReg()) ==
+                                          &WebAssembly::EXCEPT_REFRegClass;
+
   if (Cond[0].getImm()) {
-    BuildMI(&MBB, DL, get(WebAssembly::BR_IF)).addMBB(TBB).add(Cond[1]);
+    if (IsBrOnExn) {
+      const char *CPPExnSymbol = MF.createExternalSymbolName("__cpp_exception");
+      BuildMI(&MBB, DL, get(WebAssembly::BR_ON_EXN))
+          .addMBB(TBB)
+          .addExternalSymbol(CPPExnSymbol, WebAssemblyII::MO_SYMBOL_EVENT)
+          .add(Cond[1]);
+    } else
+      BuildMI(&MBB, DL, get(WebAssembly::BR_IF)).addMBB(TBB).add(Cond[1]);
   } else {
+    assert(!IsBrOnExn && "br_on_exn does not have a reversed condition");
     BuildMI(&MBB, DL, get(WebAssembly::BR_UNLESS)).addMBB(TBB).add(Cond[1]);
   }
   if (!FBB)
@@ -194,7 +217,15 @@ unsigned WebAssemblyInstrInfo::insertBranch(
 
 bool WebAssemblyInstrInfo::reverseBranchCondition(
     SmallVectorImpl<MachineOperand> &Cond) const {
-  assert(Cond.size() == 2 && "Expected a flag and a successor block");
+  assert(Cond.size() == 2 && "Expected a flag and a condition expression");
+
+  // br_on_exn's condition cannot be reversed
+  MachineFunction &MF = *Cond[1].getParent()->getParent()->getParent();
+  auto &MRI = MF.getRegInfo();
+  if (Cond[1].isReg() &&
+      MRI.getRegClass(Cond[1].getReg()) == &WebAssembly::EXCEPT_REFRegClass)
+    return true;
+
   Cond.front() = MachineOperand::CreateImm(!Cond.front().getImm());
   return false;
 }
diff --git a/lib/Target/WebAssembly/WebAssemblyInstrInfo.h b/lib/Target/WebAssembly/WebAssemblyInstrInfo.h
index 4a3763c345b0efdd5472a87d6bd8955fdf411e8c..20eec684b7705fec648f0493819101dde063ab72 100644
--- a/lib/Target/WebAssembly/WebAssemblyInstrInfo.h
+++ b/lib/Target/WebAssembly/WebAssemblyInstrInfo.h
@@ -1,9 +1,8 @@
 //=- WebAssemblyInstrInfo.h - WebAssembly Instruction Information -*- C++ -*-=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
diff --git a/lib/Target/WebAssembly/WebAssemblyInstrInfo.td b/lib/Target/WebAssembly/WebAssemblyInstrInfo.td
index 085c5a77e68af79de223bdd232c0341b3608b0e6..4a2bf2a991441219b026eccfe0ef9af2d91f63b2 100644
--- a/lib/Target/WebAssembly/WebAssemblyInstrInfo.td
+++ b/lib/Target/WebAssembly/WebAssemblyInstrInfo.td
@@ -1,9 +1,8 @@
 // WebAssemblyInstrInfo.td-Describe the WebAssembly Instructions-*- tablegen -*-
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
@@ -17,37 +16,48 @@
 //===----------------------------------------------------------------------===//
 
 def HasAddr32 : Predicate<"!Subtarget->hasAddr64()">;
+
 def HasAddr64 : Predicate<"Subtarget->hasAddr64()">;
-def HasSIMD128 : Predicate<"Subtarget->hasSIMD128()">,
-                           AssemblerPredicate<"FeatureSIMD128", "simd128">;
-def HasAtomics : Predicate<"Subtarget->hasAtomics()">,
-                           AssemblerPredicate<"FeatureAtomics", "atomics">;
+
+def HasSIMD128 :
+    Predicate<"Subtarget->hasSIMD128()">,
+    AssemblerPredicate<"FeatureSIMD128", "simd128">;
+
+def HasUnimplementedSIMD128 :
+    Predicate<"Subtarget->hasUnimplementedSIMD128()">,
+    AssemblerPredicate<"FeatureUnimplementedSIMD128", "unimplemented-simd128">;
+
+def HasAtomics :
+    Predicate<"Subtarget->hasAtomics()">,
+    AssemblerPredicate<"FeatureAtomics", "atomics">;
+
 def HasNontrappingFPToInt :
     Predicate<"Subtarget->hasNontrappingFPToInt()">,
-              AssemblerPredicate<"FeatureNontrappingFPToInt",
-                                 "nontrapping-fptoint">;
+    AssemblerPredicate<"FeatureNontrappingFPToInt", "nontrapping-fptoint">;
+
 def NotHasNontrappingFPToInt :
     Predicate<"!Subtarget->hasNontrappingFPToInt()">,
-              AssemblerPredicate<"!FeatureNontrappingFPToInt",
-                                 "nontrapping-fptoint">;
+    AssemblerPredicate<"!FeatureNontrappingFPToInt", "nontrapping-fptoint">;
+
 def HasSignExt :
     Predicate<"Subtarget->hasSignExt()">,
-              AssemblerPredicate<"FeatureSignExt",
-                                 "sign-ext">;
+    AssemblerPredicate<"FeatureSignExt", "sign-ext">;
+
 def NotHasSignExt :
     Predicate<"!Subtarget->hasSignExt()">,
-              AssemblerPredicate<"!FeatureSignExt",
-                                 "sign-ext">;
+    AssemblerPredicate<"!FeatureSignExt", "sign-ext">;
 
 def HasExceptionHandling :
     Predicate<"Subtarget->hasExceptionHandling()">,
-              AssemblerPredicate<"FeatureExceptionHandling",
-                                 "exception-handling">;
+    AssemblerPredicate<"FeatureExceptionHandling", "exception-handling">;
 
 def NotHasExceptionHandling :
     Predicate<"!Subtarget->hasExceptionHandling()">,
-              AssemblerPredicate<"!FeatureExceptionHandling",
-                                 "exception-handling">;
+    AssemblerPredicate<"!FeatureExceptionHandling", "exception-handling">;
+
+def HasBulkMemory :
+    Predicate<"Subtarget->hasBulkMemory()">,
+    AssemblerPredicate<"FeatureBulkMemory", "bulk-memory">;
 
 //===----------------------------------------------------------------------===//
 // WebAssembly-specific DAG Node Types.
@@ -64,7 +74,7 @@ def SDT_WebAssemblyArgument : SDTypeProfile<1, 1, [SDTCisVT<1, i32>]>;
 def SDT_WebAssemblyReturn   : SDTypeProfile<0, -1, []>;
 def SDT_WebAssemblyWrapper  : SDTypeProfile<1, 1, [SDTCisSameAs<0, 1>,
                                                    SDTCisPtrTy<0>]>;
-def SDT_WebAssemblyThrow    : SDTypeProfile<0, 2, [SDTCisPtrTy<0>]>;
+def SDT_WebAssemblyThrow    : SDTypeProfile<0, -1, [SDTCisPtrTy<0>]>;
 
 //===----------------------------------------------------------------------===//
 // WebAssembly-specific DAG Nodes.
@@ -92,7 +102,7 @@ def WebAssemblyreturn   : SDNode<"WebAssemblyISD::RETURN",
 def WebAssemblywrapper  : SDNode<"WebAssemblyISD::Wrapper",
                                  SDT_WebAssemblyWrapper>;
 def WebAssemblythrow : SDNode<"WebAssemblyISD::THROW", SDT_WebAssemblyThrow,
-                              [SDNPHasChain]>;
+                              [SDNPHasChain, SDNPVariadic]>;
 
 //===----------------------------------------------------------------------===//
 // WebAssembly-specific Operands.
@@ -149,11 +159,10 @@ def event_op : Operand<i32>;
 
 } // OperandType = "OPERAND_P2ALIGN"
 
-let OperandType = "OPERAND_SIGNATURE" in {
+let OperandType = "OPERAND_SIGNATURE" in
 def Signature : Operand<i32> {
   let PrintMethod = "printWebAssemblySignatureOperand";
 }
-} // OperandType = "OPERAND_SIGNATURE"
 
 let OperandType = "OPERAND_TYPEINDEX" in
 def TypeIndex : Operand<i32>;
@@ -184,8 +193,8 @@ include "WebAssemblyInstrFormats.td"
 //===----------------------------------------------------------------------===//
 
 multiclass ARGUMENT<WebAssemblyRegClass reg, ValueType vt> {
-  let hasSideEffects = 1, isCodeGenOnly = 1,
-      Defs = []<Register>, Uses = [ARGUMENTS] in
+  let hasSideEffects = 1, isCodeGenOnly = 1, Defs = []<Register>,
+      Uses = [ARGUMENTS] in
   defm ARGUMENT_#vt :
     I<(outs reg:$res), (ins i32imm:$argno), (outs), (ins i32imm:$argno),
       [(set (vt reg:$res), (WebAssemblyargument timm:$argno))]>;
@@ -196,49 +205,49 @@ defm "": ARGUMENT<F32, f32>;
 defm "": ARGUMENT<F64, f64>;
 defm "": ARGUMENT<EXCEPT_REF, ExceptRef>;
 
-// get_local and set_local are not generated by instruction selection; they
+// local.get and local.set are not generated by instruction selection; they
 // are implied by virtual register uses and defs.
 multiclass LOCAL<WebAssemblyRegClass vt> {
-let hasSideEffects = 0 in {
-  // COPY is not an actual instruction in wasm, but since we allow get_local and
-  // set_local to be implicit during most of codegen, we can have a COPY which
-  // is actually a no-op because all the work is done in the implied get_local
-  // and set_local. COPYs are eliminated (and replaced with
-  // get_local/set_local) in the ExplicitLocals pass.
+  let hasSideEffects = 0 in {
+  // COPY is not an actual instruction in wasm, but since we allow local.get and
+  // local.set to be implicit during most of codegen, we can have a COPY which
+  // is actually a no-op because all the work is done in the implied local.get
+  // and local.set. COPYs are eliminated (and replaced with
+  // local.get/local.set) in the ExplicitLocals pass.
   let isAsCheapAsAMove = 1, isCodeGenOnly = 1 in
   defm COPY_#vt : I<(outs vt:$res), (ins vt:$src), (outs), (ins), [],
-                    "copy_local\t$res, $src", "copy_local">;
+                    "local.copy\t$res, $src", "local.copy">;
 
   // TEE is similar to COPY, but writes two copies of its result. Typically
   // this would be used to stackify one result and write the other result to a
   // local.
   let isAsCheapAsAMove = 1, isCodeGenOnly = 1 in
   defm TEE_#vt : I<(outs vt:$res, vt:$also), (ins vt:$src), (outs), (ins), [],
-                   "tee_local\t$res, $also, $src", "tee_local">;
+                   "local.tee\t$res, $also, $src", "local.tee">;
 
-  // This is the actual get_local instruction in wasm. These are made explicit
+  // This is the actual local.get instruction in wasm. These are made explicit
   // by the ExplicitLocals pass. It has mayLoad because it reads from a wasm
   // local, which is a side effect not otherwise modeled in LLVM.
   let mayLoad = 1, isAsCheapAsAMove = 1 in
-  defm GET_LOCAL_#vt : I<(outs vt:$res), (ins local_op:$local),
+  defm LOCAL_GET_#vt : I<(outs vt:$res), (ins local_op:$local),
                          (outs), (ins local_op:$local), [],
-                         "get_local\t$res, $local", "get_local\t$local", 0x20>;
+                         "local.get\t$res, $local", "local.get\t$local", 0x20>;
 
-  // This is the actual set_local instruction in wasm. These are made explicit
+  // This is the actual local.set instruction in wasm. These are made explicit
   // by the ExplicitLocals pass. It has mayStore because it writes to a wasm
   // local, which is a side effect not otherwise modeled in LLVM.
   let mayStore = 1, isAsCheapAsAMove = 1 in
-  defm SET_LOCAL_#vt : I<(outs), (ins local_op:$local, vt:$src),
+  defm LOCAL_SET_#vt : I<(outs), (ins local_op:$local, vt:$src),
                          (outs), (ins local_op:$local), [],
-                         "set_local\t$local, $src", "set_local\t$local", 0x21>;
+                         "local.set\t$local, $src", "local.set\t$local", 0x21>;
 
-  // This is the actual tee_local instruction in wasm. TEEs are turned into
-  // TEE_LOCALs by the ExplicitLocals pass. It has mayStore for the same reason
-  // as SET_LOCAL.
+  // This is the actual local.tee instruction in wasm. TEEs are turned into
+  // LOCAL_TEEs by the ExplicitLocals pass. It has mayStore for the same reason
+  // as LOCAL_SET.
   let mayStore = 1, isAsCheapAsAMove = 1 in
-  defm TEE_LOCAL_#vt : I<(outs vt:$res), (ins local_op:$local, vt:$src),
+  defm LOCAL_TEE_#vt : I<(outs vt:$res), (ins local_op:$local, vt:$src),
                          (outs), (ins local_op:$local), [],
-                         "tee_local\t$res, $local, $src", "tee_local\t$local",
+                         "local.tee\t$res, $local, $src", "local.tee\t$local",
                          0x22>;
 
   // Unused values must be dropped in some contexts.
@@ -246,15 +255,15 @@ let hasSideEffects = 0 in {
                     "drop\t$src", "drop", 0x1a>;
 
   let mayLoad = 1 in
-  defm GET_GLOBAL_#vt : I<(outs vt:$res), (ins global_op:$local),
+  defm GLOBAL_GET_#vt : I<(outs vt:$res), (ins global_op:$local),
                           (outs), (ins global_op:$local), [],
-                          "get_global\t$res, $local", "get_global\t$local",
+                          "global.get\t$res, $local", "global.get\t$local",
                           0x23>;
 
   let mayStore = 1 in
-  defm SET_GLOBAL_#vt : I<(outs), (ins global_op:$local, vt:$src),
+  defm GLOBAL_SET_#vt : I<(outs), (ins global_op:$local, vt:$src),
                           (outs), (ins global_op:$local), [],
-                          "set_global\t$local, $src", "set_global\t$local",
+                          "global.set\t$local, $src", "global.set\t$local",
                           0x24>;
 
 } // hasSideEffects = 0
@@ -305,3 +314,4 @@ include "WebAssemblyInstrFloat.td"
 include "WebAssemblyInstrAtomics.td"
 include "WebAssemblyInstrSIMD.td"
 include "WebAssemblyInstrExceptRef.td"
+include "WebAssemblyInstrBulkMemory.td"
diff --git a/lib/Target/WebAssembly/WebAssemblyInstrInteger.td b/lib/Target/WebAssembly/WebAssemblyInstrInteger.td
index d5b63d64369735be6cc4459e983500bc95acd4d8..837fc556497b9896b8517fe4a547f9c65889594c 100644
--- a/lib/Target/WebAssembly/WebAssemblyInstrInteger.td
+++ b/lib/Target/WebAssembly/WebAssemblyInstrInteger.td
@@ -1,9 +1,8 @@
 // WebAssemblyInstrInteger.td-WebAssembly Integer codegen -------*- tablegen -*-
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
diff --git a/lib/Target/WebAssembly/WebAssemblyInstrMemory.td b/lib/Target/WebAssembly/WebAssemblyInstrMemory.td
index ccc331d1bf03203f5cbde16a1a9042dc52123a9b..2070f8ad82287f79f6a25dcaba9797e1efe095b3 100644
--- a/lib/Target/WebAssembly/WebAssemblyInstrMemory.td
+++ b/lib/Target/WebAssembly/WebAssemblyInstrMemory.td
@@ -1,9 +1,8 @@
 // WebAssemblyInstrMemory.td-WebAssembly Memory codegen support -*- tablegen -*-
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
@@ -33,10 +32,8 @@ def or_is_add : PatFrag<(ops node:$lhs, node:$rhs), (or node:$lhs, node:$rhs),[{
   if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N->getOperand(1)))
     return CurDAG->MaskedValueIsZero(N->getOperand(0), CN->getAPIntValue());
 
-  KnownBits Known0;
-  CurDAG->computeKnownBits(N->getOperand(0), Known0, 0);
-  KnownBits Known1;
-  CurDAG->computeKnownBits(N->getOperand(1), Known1, 0);
+  KnownBits Known0 = CurDAG->computeKnownBits(N->getOperand(0), 0);
+  KnownBits Known1 = CurDAG->computeKnownBits(N->getOperand(1), 0);
   return (~Known0.Zero & ~Known1.Zero) == 0;
 }]>;
 
@@ -438,17 +435,6 @@ defm MEMORY_SIZE_I32 : I<(outs I32:$dst), (ins i32imm:$flags),
                          "memory.size\t$dst, $flags", "memory.size\t$flags",
                          0x3f>,
                        Requires<[HasAddr32]>;
-defm MEM_SIZE_I32 : I<(outs I32:$dst), (ins i32imm:$flags),
-                      (outs), (ins i32imm:$flags),
-                      [(set I32:$dst, (int_wasm_mem_size (i32 imm:$flags)))],
-                      "mem.size\t$dst, $flags", "mem.size\t$flags", 0x3f>,
-                    Requires<[HasAddr32]>;
-defm CURRENT_MEMORY_I32 : I<(outs I32:$dst), (ins i32imm:$flags),
-                            (outs), (ins i32imm:$flags),
-                            [],
-                            "current_memory\t$dst",
-                            "current_memory\t$flags", 0x3f>,
-                          Requires<[HasAddr32]>;
 
 // Grow memory.
 defm MEMORY_GROW_I32 : I<(outs I32:$dst), (ins i32imm:$flags, I32:$delta),
@@ -459,21 +445,3 @@ defm MEMORY_GROW_I32 : I<(outs I32:$dst), (ins i32imm:$flags, I32:$delta),
                          "memory.grow\t$dst, $flags, $delta",
                          "memory.grow\t$flags", 0x40>,
                        Requires<[HasAddr32]>;
-defm MEM_GROW_I32 : I<(outs I32:$dst), (ins i32imm:$flags, I32:$delta),
-                      (outs), (ins i32imm:$flags),
-                      [(set I32:$dst,
-                            (int_wasm_mem_grow (i32 imm:$flags), I32:$delta))],
-                      "mem.grow\t$dst, $flags, $delta", "mem.grow\t$flags",
-                      0x40>,
-                    Requires<[HasAddr32]>;
-defm GROW_MEMORY_I32 : I<(outs I32:$dst), (ins i32imm:$flags, I32:$delta),
-                         (outs), (ins i32imm:$flags),
-                         [],
-                         "grow_memory\t$dst, $delta", "grow_memory\t$flags",
-                         0x40>,
-                       Requires<[HasAddr32]>;
-
-def : Pat<(int_wasm_current_memory),
-          (CURRENT_MEMORY_I32 0)>;
-def : Pat<(int_wasm_grow_memory I32:$delta),
-          (GROW_MEMORY_I32 0, $delta)>;
diff --git a/lib/Target/WebAssembly/WebAssemblyInstrSIMD.td b/lib/Target/WebAssembly/WebAssemblyInstrSIMD.td
index 7ac2d15c295188fbbbe28265d4a0273442a7023f..8169e6a6233f05528e2f0cf145ef00811f4eff62 100644
--- a/lib/Target/WebAssembly/WebAssemblyInstrSIMD.td
+++ b/lib/Target/WebAssembly/WebAssemblyInstrSIMD.td
@@ -1,9 +1,8 @@
 // WebAssemblyInstrSIMD.td - WebAssembly SIMD codegen support -*- tablegen -*-//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
@@ -31,7 +30,7 @@ defm "" : ARGUMENT<V128, v2f64>;
 // Constrained immediate argument types
 foreach SIZE = [8, 16] in
 def ImmI#SIZE : ImmLeaf<i32,
-  "return ((uint64_t)Imm & ((1UL << "#SIZE#") - 1)) == (uint64_t)Imm;"
+  "return -(1 << ("#SIZE#" - 1)) <= Imm && Imm < (1 << ("#SIZE#" - 1));"
 >;
 foreach SIZE = [2, 4, 8, 16, 32] in
 def LaneIdx#SIZE : ImmLeaf<i32, "return 0 <= Imm && Imm < "#SIZE#";">;
@@ -94,7 +93,8 @@ def : StorePatExternSymOffOnly<vec_t, store, !cast<NI>("STORE_"#vec_t)>;
 
 // Constant: v128.const
 multiclass ConstVec<ValueType vec_t, dag ops, dag pat, string args> {
-  let isMoveImm = 1, isReMaterializable = 1 in
+  let isMoveImm = 1, isReMaterializable = 1,
+      Predicates = [HasSIMD128, HasUnimplementedSIMD128] in
   defm CONST_V128_#vec_t : SIMD_I<(outs V128:$dst), ops, (outs), ops,
                                   [(set V128:$dst, (vec_t pat))],
                                   "v128.const\t$dst, "#args,
@@ -125,6 +125,7 @@ defm "" : ConstVec<v8i16,
                      ImmI16:$i0, ImmI16:$i1, ImmI16:$i2, ImmI16:$i3,
                      ImmI16:$i4, ImmI16:$i5, ImmI16:$i6, ImmI16:$i7),
                    "$i0, $i1, $i2, $i3, $i4, $i5, $i6, $i7">;
+let IsCanonical = 1 in
 defm "" : ConstVec<v4i32,
                    (ins vec_i32imm_op:$i0, vec_i32imm_op:$i1,
                         vec_i32imm_op:$i2, vec_i32imm_op:$i3),
@@ -230,6 +231,19 @@ defm "" : Splat<v2i64, "i64x2", I64, splat2, 15>;
 defm "" : Splat<v4f32, "f32x4", F32, splat4, 18>;
 defm "" : Splat<v2f64, "f64x2", F64, splat2, 21>;
 
+// scalar_to_vector leaves high lanes undefined, so can be a splat
+class ScalarSplatPat<ValueType vec_t, ValueType lane_t,
+                     WebAssemblyRegClass reg_t> :
+  Pat<(vec_t (scalar_to_vector (lane_t reg_t:$x))),
+      (!cast<Instruction>("SPLAT_"#vec_t) reg_t:$x)>;
+
+def : ScalarSplatPat<v16i8, i32, I32>;
+def : ScalarSplatPat<v8i16, i32, I32>;
+def : ScalarSplatPat<v4i32, i32, I32>;
+def : ScalarSplatPat<v2i64, i64, I64>;
+def : ScalarSplatPat<v4f32, f32, F32>;
+def : ScalarSplatPat<v2f64, f64, F64>;
+
 //===----------------------------------------------------------------------===//
 // Accessing lanes
 //===----------------------------------------------------------------------===//
@@ -276,17 +290,19 @@ multiclass ExtractLaneExtended<string sign, bits<32> baseInst> {
 }
 
 defm "" : ExtractLaneExtended<"_s", 5>;
+let Predicates = [HasSIMD128, HasUnimplementedSIMD128] in
 defm "" : ExtractLaneExtended<"_u", 6>;
 defm "" : ExtractLane<v4i32, "i32x4", LaneIdx4, I32, 13>;
 defm "" : ExtractLane<v2i64, "i64x2", LaneIdx2, I64, 16>;
 defm "" : ExtractLane<v4f32, "f32x4", LaneIdx4, F32, 19>;
 defm "" : ExtractLane<v2f64, "f64x2", LaneIdx2, F64, 22>;
 
-// Follow convention of making implicit expansions unsigned
+// It would be more conventional to use unsigned extracts, but v8
+// doesn't implement them yet
 def : Pat<(i32 (vector_extract (v16i8 V128:$vec), (i32 LaneIdx16:$idx))),
-          (EXTRACT_LANE_v16i8_u V128:$vec, (i32 LaneIdx16:$idx))>;
+          (EXTRACT_LANE_v16i8_s V128:$vec, (i32 LaneIdx16:$idx))>;
 def : Pat<(i32 (vector_extract (v8i16 V128:$vec), (i32 LaneIdx8:$idx))),
-          (EXTRACT_LANE_v8i16_u V128:$vec, (i32 LaneIdx8:$idx))>;
+          (EXTRACT_LANE_v8i16_s V128:$vec, (i32 LaneIdx8:$idx))>;
 
 // Lower undef lane indices to zero
 def : Pat<(and (i32 (vector_extract (v16i8 V128:$vec), undef)), (i32 0xff)),
@@ -344,118 +360,6 @@ def : Pat<(vector_insert (v4f32 V128:$vec), F32:$x, undef),
 def : Pat<(vector_insert (v2f64 V128:$vec), F64:$x, undef),
           (REPLACE_LANE_v2f64 V128:$vec, 0, F64:$x)>;
 
-// Arbitrary other BUILD_VECTOR patterns
-def : Pat<(v16i8 (build_vector
-            (i32 I32:$x0), (i32 I32:$x1), (i32 I32:$x2), (i32 I32:$x3),
-            (i32 I32:$x4), (i32 I32:$x5), (i32 I32:$x6), (i32 I32:$x7),
-            (i32 I32:$x8), (i32 I32:$x9), (i32 I32:$x10), (i32 I32:$x11),
-            (i32 I32:$x12), (i32 I32:$x13), (i32 I32:$x14), (i32 I32:$x15)
-          )),
-          (v16i8 (REPLACE_LANE_v16i8
-            (v16i8 (REPLACE_LANE_v16i8
-              (v16i8 (REPLACE_LANE_v16i8
-                (v16i8 (REPLACE_LANE_v16i8
-                  (v16i8 (REPLACE_LANE_v16i8
-                    (v16i8 (REPLACE_LANE_v16i8
-                      (v16i8 (REPLACE_LANE_v16i8
-                        (v16i8 (REPLACE_LANE_v16i8
-                          (v16i8 (REPLACE_LANE_v16i8
-                            (v16i8 (REPLACE_LANE_v16i8
-                              (v16i8 (REPLACE_LANE_v16i8
-                                (v16i8 (REPLACE_LANE_v16i8
-                                  (v16i8 (REPLACE_LANE_v16i8
-                                    (v16i8 (REPLACE_LANE_v16i8
-                                      (v16i8 (REPLACE_LANE_v16i8
-                                        (v16i8 (SPLAT_v16i8 (i32 I32:$x0))),
-                                        1, I32:$x1
-                                      )),
-                                      2, I32:$x2
-                                    )),
-                                    3, I32:$x3
-                                  )),
-                                  4, I32:$x4
-                                )),
-                                5, I32:$x5
-                              )),
-                              6, I32:$x6
-                            )),
-                            7, I32:$x7
-                          )),
-                          8, I32:$x8
-                        )),
-                        9, I32:$x9
-                      )),
-                      10, I32:$x10
-                    )),
-                    11, I32:$x11
-                  )),
-                  12, I32:$x12
-                )),
-                13, I32:$x13
-              )),
-              14, I32:$x14
-            )),
-            15, I32:$x15
-          ))>;
-def : Pat<(v8i16 (build_vector
-            (i32 I32:$x0), (i32 I32:$x1), (i32 I32:$x2), (i32 I32:$x3),
-            (i32 I32:$x4), (i32 I32:$x5), (i32 I32:$x6), (i32 I32:$x7)
-          )),
-          (v8i16 (REPLACE_LANE_v8i16
-            (v8i16 (REPLACE_LANE_v8i16
-              (v8i16 (REPLACE_LANE_v8i16
-                (v8i16 (REPLACE_LANE_v8i16
-                  (v8i16 (REPLACE_LANE_v8i16
-                    (v8i16 (REPLACE_LANE_v8i16
-                      (v8i16 (REPLACE_LANE_v8i16
-                        (v8i16 (SPLAT_v8i16 (i32 I32:$x0))),
-                        1, I32:$x1
-                      )),
-                      2, I32:$x2
-                    )),
-                    3, I32:$x3
-                  )),
-                  4, I32:$x4
-                )),
-                5, I32:$x5
-              )),
-              6, I32:$x6
-            )),
-            7, I32:$x7
-          ))>;
-def : Pat<(v4i32 (build_vector
-            (i32 I32:$x0), (i32 I32:$x1), (i32 I32:$x2), (i32 I32:$x3)
-          )),
-          (v4i32 (REPLACE_LANE_v4i32
-            (v4i32 (REPLACE_LANE_v4i32
-              (v4i32 (REPLACE_LANE_v4i32
-                (v4i32 (SPLAT_v4i32 (i32 I32:$x0))),
-                1, I32:$x1
-              )),
-              2, I32:$x2
-            )),
-            3, I32:$x3
-          ))>;
-def : Pat<(v2i64 (build_vector (i64 I64:$x0), (i64 I64:$x1))),
-          (v2i64 (REPLACE_LANE_v2i64
-            (v2i64 (SPLAT_v2i64 (i64 I64:$x0))), 1, I64:$x1))>;
-def : Pat<(v4f32 (build_vector
-            (f32 F32:$x0), (f32 F32:$x1), (f32 F32:$x2), (f32 F32:$x3)
-          )),
-          (v4f32 (REPLACE_LANE_v4f32
-            (v4f32 (REPLACE_LANE_v4f32
-              (v4f32 (REPLACE_LANE_v4f32
-                (v4f32 (SPLAT_v4f32 (f32 F32:$x0))),
-                1, F32:$x1
-              )),
-              2, F32:$x2
-            )),
-            3, F32:$x3
-          ))>;
-def : Pat<(v2f64 (build_vector (f64 F64:$x0), (f64 F64:$x1))),
-          (v2f64 (REPLACE_LANE_v2f64
-            (v2f64 (SPLAT_v2f64 (f64 F64:$x0))), 1, F64:$x1))>;
-
 //===----------------------------------------------------------------------===//
 // Comparisons
 //===----------------------------------------------------------------------===//
@@ -576,7 +480,7 @@ foreach vec_t = [v16i8, v8i16, v4i32, v2i64, v4f32, v2f64] in
     SIMD_I<(outs V128:$dst), (ins V128:$v1, V128:$v2, V128:$c), (outs), (ins),
            [(set (vec_t V128:$dst),
              (vec_t (int_wasm_bitselect
-               (vec_t V128:$c), (vec_t V128:$v1), (vec_t V128:$v2)
+               (vec_t V128:$v1), (vec_t V128:$v2), (vec_t V128:$c)
              ))
            )],
            "v128.bitselect\t$dst, $v1, $v2, $c", "v128.bitselect", 80>;
@@ -725,6 +629,7 @@ defm ABS : SIMDUnaryFP<fabs, "abs", 149>;
 defm NEG : SIMDUnaryFP<fneg, "neg", 150>;
 
 // Square root: sqrt
+let Predicates = [HasSIMD128, HasUnimplementedSIMD128] in
 defm SQRT : SIMDUnaryFP<fsqrt, "sqrt", 151>;
 
 //===----------------------------------------------------------------------===//
@@ -748,6 +653,7 @@ let isCommutable = 1 in
 defm MUL : SIMDBinaryFP<fmul, "mul", 156>;
 
 // Division: div
+let Predicates = [HasSIMD128, HasUnimplementedSIMD128] in
 defm DIV : SIMDBinaryFP<fdiv, "div", 157>;
 
 // NaN-propagating minimum: min
@@ -768,17 +674,17 @@ multiclass SIMDConvert<ValueType vec_t, ValueType arg_t, SDNode op,
            name#"\t$dst, $vec", name, simdop>;
 }
 
-// Integer to floating point: convert_s / convert_u
-defm "" : SIMDConvert<v4f32, v4i32, sint_to_fp, "f32x4.convert_s/i32x4", 175>;
-defm "" : SIMDConvert<v4f32, v4i32, uint_to_fp, "f32x4.convert_u/i32x4", 176>;
-defm "" : SIMDConvert<v2f64, v2i64, sint_to_fp, "f64x2.convert_s/i64x2", 177>;
-defm "" : SIMDConvert<v2f64, v2i64, uint_to_fp, "f64x2.convert_u/i64x2", 178>;
-
-// Floating point to integer with saturation: trunc_sat_s / trunc_sat_u
-defm "" : SIMDConvert<v4i32, v4f32, fp_to_sint, "i32x4.trunc_sat_s/f32x4", 171>;
-defm "" : SIMDConvert<v4i32, v4f32, fp_to_uint, "i32x4.trunc_sat_u/f32x4", 172>;
-defm "" : SIMDConvert<v2i64, v2f64, fp_to_sint, "i64x2.trunc_sat_s/f64x2", 173>;
-defm "" : SIMDConvert<v2i64, v2f64, fp_to_uint, "i64x2.trunc_sat_u/f64x2", 174>;
+// Integer to floating point: convert
+defm "" : SIMDConvert<v4f32, v4i32, sint_to_fp, "f32x4.convert_i32x4_s", 175>;
+defm "" : SIMDConvert<v4f32, v4i32, uint_to_fp, "f32x4.convert_i32x4_u", 176>;
+defm "" : SIMDConvert<v2f64, v2i64, sint_to_fp, "f64x2.convert_i64x2_s", 177>;
+defm "" : SIMDConvert<v2f64, v2i64, uint_to_fp, "f64x2.convert_i64x2_u", 178>;
+
+// Floating point to integer with saturation: trunc_sat
+defm "" : SIMDConvert<v4i32, v4f32, fp_to_sint, "i32x4.trunc_sat_f32x4_s", 171>;
+defm "" : SIMDConvert<v4i32, v4f32, fp_to_uint, "i32x4.trunc_sat_f32x4_u", 172>;
+defm "" : SIMDConvert<v2i64, v2f64, fp_to_sint, "i64x2.trunc_sat_f64x2_s", 173>;
+defm "" : SIMDConvert<v2i64, v2f64, fp_to_uint, "i64x2.trunc_sat_f64x2_u", 174>;
 
 // Lower llvm.wasm.trunc.saturate.* to saturating instructions
 def : Pat<(v4i32 (int_wasm_trunc_saturate_signed (v4f32 V128:$src))),
diff --git a/lib/Target/WebAssembly/WebAssemblyLateEHPrepare.cpp b/lib/Target/WebAssembly/WebAssemblyLateEHPrepare.cpp
index 871e92082a488286ab0ef55390253830ac16f4d4..2962de476537d4e7b1215033f1792ab0e2a646f7 100644
--- a/lib/Target/WebAssembly/WebAssemblyLateEHPrepare.cpp
+++ b/lib/Target/WebAssembly/WebAssemblyLateEHPrepare.cpp
@@ -1,9 +1,8 @@
 //=== WebAssemblyLateEHPrepare.cpp - WebAssembly Exception Preparation -===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
@@ -16,6 +15,7 @@
 #include "WebAssembly.h"
 #include "WebAssemblySubtarget.h"
 #include "WebAssemblyUtilities.h"
+#include "llvm/ADT/SmallSet.h"
 #include "llvm/CodeGen/MachineInstrBuilder.h"
 #include "llvm/CodeGen/WasmEHFuncInfo.h"
 #include "llvm/MC/MCAsmInfo.h"
@@ -26,19 +26,15 @@ using namespace llvm;
 namespace {
 class WebAssemblyLateEHPrepare final : public MachineFunctionPass {
   StringRef getPassName() const override {
-    return "WebAssembly Prepare Exception";
+    return "WebAssembly Late Prepare Exception";
   }
 
   bool runOnMachineFunction(MachineFunction &MF) override;
-
   bool removeUnnecessaryUnreachables(MachineFunction &MF);
   bool replaceFuncletReturns(MachineFunction &MF);
-  bool hoistCatches(MachineFunction &MF);
-  bool addCatchAlls(MachineFunction &MF);
-  bool addRethrows(MachineFunction &MF);
-  bool ensureSingleBBTermPads(MachineFunction &MF);
-  bool mergeTerminatePads(MachineFunction &MF);
-  bool addCatchAllTerminatePads(MachineFunction &MF);
+  bool addCatches(MachineFunction &MF);
+  bool addExceptionExtraction(MachineFunction &MF);
+  bool restoreStackPointer(MachineFunction &MF);
 
 public:
   static char ID; // Pass identification, replacement for typeid
@@ -103,21 +99,22 @@ static void eraseDeadBBsAndChildren(const Container &MBBs) {
 }
 
 bool WebAssemblyLateEHPrepare::runOnMachineFunction(MachineFunction &MF) {
+  LLVM_DEBUG(dbgs() << "********** Late EH Prepare **********\n"
+                       "********** Function: "
+                    << MF.getName() << '\n');
+
   if (MF.getTarget().getMCAsmInfo()->getExceptionHandlingType() !=
       ExceptionHandling::Wasm)
     return false;
 
   bool Changed = false;
   Changed |= removeUnnecessaryUnreachables(MF);
-  Changed |= addRethrows(MF);
   if (!MF.getFunction().hasPersonalityFn())
     return Changed;
   Changed |= replaceFuncletReturns(MF);
-  Changed |= hoistCatches(MF);
-  Changed |= addCatchAlls(MF);
-  Changed |= ensureSingleBBTermPads(MF);
-  Changed |= mergeTerminatePads(MF);
-  Changed |= addCatchAllTerminatePads(MF);
+  Changed |= addCatches(MF);
+  Changed |= addExceptionExtraction(MF);
+  Changed |= restoreStackPointer(MF);
   return Changed;
 }
 
@@ -126,7 +123,8 @@ bool WebAssemblyLateEHPrepare::removeUnnecessaryUnreachables(
   bool Changed = false;
   for (auto &MBB : MF) {
     for (auto &MI : MBB) {
-      if (!WebAssembly::isThrow(MI))
+      if (MI.getOpcode() != WebAssembly::THROW &&
+          MI.getOpcode() != WebAssembly::RETHROW)
         continue;
       Changed = true;
 
@@ -149,7 +147,6 @@ bool WebAssemblyLateEHPrepare::removeUnnecessaryUnreachables(
 bool WebAssemblyLateEHPrepare::replaceFuncletReturns(MachineFunction &MF) {
   bool Changed = false;
   const auto &TII = *MF.getSubtarget<WebAssemblySubtarget>().getInstrInfo();
-  auto *EHInfo = MF.getWasmEHFuncInfo();
 
   for (auto &MBB : MF) {
     auto Pos = MBB.getFirstTerminator();
@@ -170,13 +167,7 @@ bool WebAssemblyLateEHPrepare::replaceFuncletReturns(MachineFunction &MF) {
     }
     case WebAssembly::CLEANUPRET: {
       // Replace a cleanupret with a rethrow
-      if (EHInfo->hasThrowUnwindDest(&MBB))
-        BuildMI(MBB, TI, TI->getDebugLoc(), TII.get(WebAssembly::RETHROW))
-            .addMBB(EHInfo->getThrowUnwindDest(&MBB));
-      else
-        BuildMI(MBB, TI, TI->getDebugLoc(),
-                TII.get(WebAssembly::RETHROW_TO_CALLER));
-
+      BuildMI(MBB, TI, TI->getDebugLoc(), TII.get(WebAssembly::RETHROW));
       TI->eraseFromParent();
       Changed = true;
       break;
@@ -186,233 +177,188 @@ bool WebAssemblyLateEHPrepare::replaceFuncletReturns(MachineFunction &MF) {
   return Changed;
 }
 
-// Hoist catch instructions to the beginning of their matching EH pad BBs in
-// case,
-// (1) catch instruction is not the first instruction in EH pad.
-// ehpad:
-//   some_other_instruction
-//   ...
-//   %exn = catch 0
-// (2) catch instruction is in a non-EH pad BB. For example,
-// ehpad:
-//   br bb0
-// bb0:
-//   %exn = catch 0
-bool WebAssemblyLateEHPrepare::hoistCatches(MachineFunction &MF) {
-  bool Changed = false;
-  SmallVector<MachineInstr *, 16> Catches;
-  for (auto &MBB : MF)
-    for (auto &MI : MBB)
-      if (WebAssembly::isCatch(MI))
-        Catches.push_back(&MI);
-
-  for (auto *Catch : Catches) {
-    MachineBasicBlock *EHPad = getMatchingEHPad(Catch);
-    assert(EHPad && "No matching EH pad for catch");
-    if (EHPad->begin() == Catch)
-      continue;
-    Changed = true;
-    EHPad->insert(EHPad->begin(), Catch->removeFromParent());
-  }
-  return Changed;
-}
-
-// Add catch_all to beginning of cleanup pads.
-bool WebAssemblyLateEHPrepare::addCatchAlls(MachineFunction &MF) {
+// Add catch instruction to beginning of catchpads and cleanuppads.
+bool WebAssemblyLateEHPrepare::addCatches(MachineFunction &MF) {
   bool Changed = false;
   const auto &TII = *MF.getSubtarget<WebAssemblySubtarget>().getInstrInfo();
-
+  MachineRegisterInfo &MRI = MF.getRegInfo();
   for (auto &MBB : MF) {
-    if (!MBB.isEHPad())
-      continue;
-    // This runs after hoistCatches(), so we assume that if there is a catch,
-    // that should be the first instruction in an EH pad.
-    if (!WebAssembly::isCatch(*MBB.begin())) {
+    if (MBB.isEHPad()) {
       Changed = true;
+      unsigned DstReg =
+          MRI.createVirtualRegister(&WebAssembly::EXCEPT_REFRegClass);
       BuildMI(MBB, MBB.begin(), MBB.begin()->getDebugLoc(),
-              TII.get(WebAssembly::CATCH_ALL));
+              TII.get(WebAssembly::CATCH), DstReg);
     }
   }
   return Changed;
 }
 
-// Add a 'rethrow' instruction after __cxa_rethrow() call
-bool WebAssemblyLateEHPrepare::addRethrows(MachineFunction &MF) {
-  bool Changed = false;
+// Wasm uses 'br_on_exn' instruction to check the tag of an exception. It takes
+// except_ref type object returned by 'catch', and branches to the destination
+// if it matches a given tag. We currently use __cpp_exception symbol to
+// represent the tag for all C++ exceptions.
+//
+// block $l (result i32)
+//   ...
+//   ;; except_ref $e is on the stack at this point
+//   br_on_exn $l $e ;; branch to $l with $e's arguments
+//   ...
+// end
+// ;; Here we expect the extracted values are on top of the wasm value stack
+// ... Handle exception using values ...
+//
+// br_on_exn takes an except_ref object and branches if it matches the given
+// tag. There can be multiple br_on_exn instructions if we want to match for
+// another tag, but for now we only test for __cpp_exception tag, and if it does
+// not match, i.e., it is a foreign exception, we rethrow it.
+//
+// In the destination BB that's the target of br_on_exn, extracted exception
+// values (in C++'s case a single i32, which represents an exception pointer)
+// are placed on top of the wasm stack. Because we can't model wasm stack in
+// LLVM instruction, we use 'extract_exception' pseudo instruction to retrieve
+// it. The pseudo instruction will be deleted later.
+bool WebAssemblyLateEHPrepare::addExceptionExtraction(MachineFunction &MF) {
   const auto &TII = *MF.getSubtarget<WebAssemblySubtarget>().getInstrInfo();
   auto *EHInfo = MF.getWasmEHFuncInfo();
-
-  for (auto &MBB : MF)
+  SmallVector<MachineInstr *, 16> ExtractInstrs;
+  for (auto &MBB : MF) {
     for (auto &MI : MBB) {
-      // Check if it is a call to __cxa_rethrow()
-      if (!MI.isCall())
-        continue;
-      MachineOperand &CalleeOp = MI.getOperand(0);
-      if (!CalleeOp.isGlobal() ||
-          CalleeOp.getGlobal()->getName() != WebAssembly::CxaRethrowFn)
-        continue;
-
-      // Now we have __cxa_rethrow() call
-      Changed = true;
-      auto InsertPt = std::next(MachineBasicBlock::iterator(MI));
-      while (InsertPt != MBB.end() && InsertPt->isLabel()) // Skip EH_LABELs
-        ++InsertPt;
-      MachineInstr *Rethrow = nullptr;
-      if (EHInfo->hasThrowUnwindDest(&MBB))
-        Rethrow = BuildMI(MBB, InsertPt, MI.getDebugLoc(),
-                          TII.get(WebAssembly::RETHROW))
-                      .addMBB(EHInfo->getThrowUnwindDest(&MBB));
-      else
-        Rethrow = BuildMI(MBB, InsertPt, MI.getDebugLoc(),
-                          TII.get(WebAssembly::RETHROW_TO_CALLER));
-
-      // Because __cxa_rethrow does not return, the instruction after the
-      // rethrow should be an unreachable or a branch to another BB that should
-      // eventually lead to an unreachable. Delete it because rethrow itself is
-      // a terminator, and also delete non-EH pad successors if any.
-      MBB.erase(std::next(MachineBasicBlock::iterator(Rethrow)), MBB.end());
-      SmallVector<MachineBasicBlock *, 8> NonPadSuccessors;
-      for (auto *Succ : MBB.successors())
-        if (!Succ->isEHPad())
-          NonPadSuccessors.push_back(Succ);
-      for (auto *Succ : NonPadSuccessors)
-        MBB.removeSuccessor(Succ);
-      eraseDeadBBsAndChildren(NonPadSuccessors);
+      if (MI.getOpcode() == WebAssembly::EXTRACT_EXCEPTION_I32) {
+        if (MI.getOperand(0).isDead())
+          MI.eraseFromParent();
+        else
+          ExtractInstrs.push_back(&MI);
+      }
     }
-  return Changed;
-}
-
-// Terminate pads are an single-BB EH pad in the form of
-// termpad:
-//   %exn = catch 0
-//   call @__clang_call_terminate(%exn)
-//   unreachable
-// (There can be set_local and get_locals before the call if we didn't run
-// RegStackify)
-// But code transformations can change or add more control flow, so the call to
-// __clang_call_terminate() function may not be in the original EH pad anymore.
-// This ensures every terminate pad is a single BB in the form illustrated
-// above.
-bool WebAssemblyLateEHPrepare::ensureSingleBBTermPads(MachineFunction &MF) {
-  const auto &TII = *MF.getSubtarget<WebAssemblySubtarget>().getInstrInfo();
+  }
+  if (ExtractInstrs.empty())
+    return false;
 
-  // Find calls to __clang_call_terminate()
-  SmallVector<MachineInstr *, 8> ClangCallTerminateCalls;
-  for (auto &MBB : MF)
-    for (auto &MI : MBB)
+  // Find terminate pads.
+  SmallSet<MachineBasicBlock *, 8> TerminatePads;
+  for (auto &MBB : MF) {
+    for (auto &MI : MBB) {
       if (MI.isCall()) {
         const MachineOperand &CalleeOp = MI.getOperand(0);
         if (CalleeOp.isGlobal() && CalleeOp.getGlobal()->getName() ==
                                        WebAssembly::ClangCallTerminateFn)
-          ClangCallTerminateCalls.push_back(&MI);
+          TerminatePads.insert(getMatchingEHPad(&MI));
       }
+    }
+  }
 
-  bool Changed = false;
-  for (auto *Call : ClangCallTerminateCalls) {
-    MachineBasicBlock *EHPad = getMatchingEHPad(Call);
-    assert(EHPad && "No matching EH pad for catch");
-
-    // If it is already the form we want, skip it
-    if (Call->getParent() == EHPad &&
-        Call->getNextNode()->getOpcode() == WebAssembly::UNREACHABLE)
-      continue;
-
-    // In case the __clang_call_terminate() call is not in its matching EH pad,
-    // move the call to the end of EH pad and add an unreachable instruction
-    // after that. Delete all successors and their children if any, because here
-    // the program terminates.
-    Changed = true;
+  for (auto *Extract : ExtractInstrs) {
+    MachineBasicBlock *EHPad = getMatchingEHPad(Extract);
+    assert(EHPad && "No matching EH pad for extract_exception");
     MachineInstr *Catch = &*EHPad->begin();
-    // This runs after hoistCatches(), so catch instruction should be at the top
-    assert(WebAssembly::isCatch(*Catch));
-    // Takes the result register of the catch instruction as argument. There may
-    // have been some other set_local/get_locals in between, but at this point
-    // we don't care.
-    Call->getOperand(1).setReg(Catch->getOperand(0).getReg());
-    auto InsertPos = std::next(MachineBasicBlock::iterator(Catch));
-    EHPad->insert(InsertPos, Call->removeFromParent());
-    BuildMI(*EHPad, InsertPos, Call->getDebugLoc(),
-            TII.get(WebAssembly::UNREACHABLE));
-    EHPad->erase(InsertPos, EHPad->end());
-    SmallVector<MachineBasicBlock *, 8> Succs(EHPad->succ_begin(),
-                                              EHPad->succ_end());
-    for (auto *Succ : Succs)
-      EHPad->removeSuccessor(Succ);
-    eraseDeadBBsAndChildren(Succs);
+    if (Catch->getNextNode() != Extract)
+      EHPad->insert(Catch->getNextNode(), Extract->removeFromParent());
+
+    // - Before:
+    // ehpad:
+    //   %exnref:except_ref = catch
+    //   %exn:i32 = extract_exception
+    //   ... use exn ...
+    //
+    // - After:
+    // ehpad:
+    //   %exnref:except_ref = catch
+    //   br_on_exn %thenbb, $__cpp_exception, %exnref
+    //   br %elsebb
+    // elsebb:
+    //   rethrow
+    // thenbb:
+    //   %exn:i32 = extract_exception
+    //   ... use exn ...
+    unsigned ExnRefReg = Catch->getOperand(0).getReg();
+    auto *ThenMBB = MF.CreateMachineBasicBlock();
+    auto *ElseMBB = MF.CreateMachineBasicBlock();
+    MF.insert(std::next(MachineFunction::iterator(EHPad)), ElseMBB);
+    MF.insert(std::next(MachineFunction::iterator(ElseMBB)), ThenMBB);
+    ThenMBB->splice(ThenMBB->end(), EHPad, Extract, EHPad->end());
+    ThenMBB->transferSuccessors(EHPad);
+    EHPad->addSuccessor(ThenMBB);
+    EHPad->addSuccessor(ElseMBB);
+
+    DebugLoc DL = Extract->getDebugLoc();
+    const char *CPPExnSymbol = MF.createExternalSymbolName("__cpp_exception");
+    BuildMI(EHPad, DL, TII.get(WebAssembly::BR_ON_EXN))
+        .addMBB(ThenMBB)
+        .addExternalSymbol(CPPExnSymbol, WebAssemblyII::MO_SYMBOL_EVENT)
+        .addReg(ExnRefReg);
+    BuildMI(EHPad, DL, TII.get(WebAssembly::BR)).addMBB(ElseMBB);
+
+    // When this is a terminate pad with __clang_call_terminate() call, we don't
+    // rethrow it anymore and call __clang_call_terminate() with a nullptr
+    // argument, which will call std::terminate().
+    //
+    // - Before:
+    // ehpad:
+    //   %exnref:except_ref = catch
+    //   %exn:i32 = extract_exception
+    //   call @__clang_call_terminate(%exn)
+    //   unreachable
+    //
+    // - After:
+    // ehpad:
+    //   %exnref:except_ref = catch
+    //   br_on_exn %thenbb, $__cpp_exception, %exnref
+    //   br %elsebb
+    // elsebb:
+    //   call @__clang_call_terminate(0)
+    //   unreachable
+    // thenbb:
+    //   %exn:i32 = extract_exception
+    //   call @__clang_call_terminate(%exn)
+    //   unreachable
+    if (TerminatePads.count(EHPad)) {
+      Function *ClangCallTerminateFn =
+          MF.getFunction().getParent()->getFunction(
+              WebAssembly::ClangCallTerminateFn);
+      assert(ClangCallTerminateFn &&
+             "There is no __clang_call_terminate() function");
+      BuildMI(ElseMBB, DL, TII.get(WebAssembly::CALL_VOID))
+          .addGlobalAddress(ClangCallTerminateFn)
+          .addImm(0);
+      BuildMI(ElseMBB, DL, TII.get(WebAssembly::UNREACHABLE));
+
+    } else {
+      BuildMI(ElseMBB, DL, TII.get(WebAssembly::RETHROW));
+      if (EHInfo->hasEHPadUnwindDest(EHPad))
+        EHInfo->setThrowUnwindDest(ElseMBB, EHInfo->getEHPadUnwindDest(EHPad));
+    }
   }
-  return Changed;
-}
 
-// In case there are multiple terminate pads, merge them into one for code size.
-// This runs after ensureSingleBBTermPads() and assumes every terminate pad is a
-// single BB.
-// In principle this violates EH scope relationship because it can merge
-// multiple inner EH scopes, each of which is in different outer EH scope. But
-// getEHScopeMembership() function will not be called after this, so it is fine.
-bool WebAssemblyLateEHPrepare::mergeTerminatePads(MachineFunction &MF) {
-  SmallVector<MachineBasicBlock *, 8> TermPads;
-  for (auto &MBB : MF)
-    if (WebAssembly::isCatchTerminatePad(MBB))
-      TermPads.push_back(&MBB);
-  if (TermPads.empty())
-    return false;
-
-  MachineBasicBlock *UniqueTermPad = TermPads.front();
-  for (auto *TermPad :
-       llvm::make_range(std::next(TermPads.begin()), TermPads.end())) {
-    SmallVector<MachineBasicBlock *, 2> Preds(TermPad->pred_begin(),
-                                              TermPad->pred_end());
-    for (auto *Pred : Preds)
-      Pred->replaceSuccessor(TermPad, UniqueTermPad);
-    TermPad->eraseFromParent();
-  }
   return true;
 }
 
-// Terminate pads are cleanup pads, so they should start with a 'catch_all'
-// instruction. But in the Itanium model, when we have a C++ exception object,
-// we pass them to __clang_call_terminate function, which calls __cxa_end_catch
-// with the passed exception pointer and then std::terminate. This is the reason
-// that terminate pads are generated with not a catch_all but a catch
-// instruction in clang and earlier llvm passes. Here we append a terminate pad
-// with a catch_all after each existing terminate pad so we can also catch
-// foreign exceptions. For every terminate pad:
-//   %exn = catch 0
-//   call @__clang_call_terminate(%exn)
-//   unreachable
-// We append this BB right after that:
-//   catch_all
-//   call @std::terminate()
-//   unreachable
-bool WebAssemblyLateEHPrepare::addCatchAllTerminatePads(MachineFunction &MF) {
-  const auto &TII = *MF.getSubtarget<WebAssemblySubtarget>().getInstrInfo();
-  SmallVector<MachineBasicBlock *, 8> TermPads;
-  for (auto &MBB : MF)
-    if (WebAssembly::isCatchTerminatePad(MBB))
-      TermPads.push_back(&MBB);
-  if (TermPads.empty())
+// After the stack is unwound due to a thrown exception, the __stack_pointer
+// global can point to an invalid address. This inserts instructions that
+// restore __stack_pointer global.
+bool WebAssemblyLateEHPrepare::restoreStackPointer(MachineFunction &MF) {
+  const auto *FrameLowering = static_cast<const WebAssemblyFrameLowering *>(
+      MF.getSubtarget().getFrameLowering());
+  if (!FrameLowering->needsPrologForEH(MF))
     return false;
+  bool Changed = false;
 
-  Function *StdTerminateFn =
-      MF.getFunction().getParent()->getFunction(WebAssembly::StdTerminateFn);
-  assert(StdTerminateFn && "There is no std::terminate() function");
-  for (auto *CatchTermPad : TermPads) {
-    DebugLoc DL = CatchTermPad->findDebugLoc(CatchTermPad->begin());
-    auto *CatchAllTermPad = MF.CreateMachineBasicBlock();
-    MF.insert(std::next(MachineFunction::iterator(CatchTermPad)),
-              CatchAllTermPad);
-    CatchAllTermPad->setIsEHPad();
-    BuildMI(CatchAllTermPad, DL, TII.get(WebAssembly::CATCH_ALL));
-    BuildMI(CatchAllTermPad, DL, TII.get(WebAssembly::CALL_VOID))
-        .addGlobalAddress(StdTerminateFn);
-    BuildMI(CatchAllTermPad, DL, TII.get(WebAssembly::UNREACHABLE));
+  for (auto &MBB : MF) {
+    if (!MBB.isEHPad())
+      continue;
+    Changed = true;
 
-    // Actually this CatchAllTermPad (new terminate pad with a catch_all) is not
-    // a successor of an existing terminate pad. CatchAllTermPad should have all
-    // predecessors CatchTermPad has instead. This is a hack to force
-    // CatchAllTermPad be always sorted right after CatchTermPad; the correct
-    // predecessor-successor relationships will be restored in CFGStackify pass.
-    CatchTermPad->addSuccessor(CatchAllTermPad);
+    // Insert __stack_pointer restoring instructions at the beginning of each EH
+    // pad, after the catch instruction. Here it is safe to assume that SP32
+    // holds the latest value of __stack_pointer, because the only exception for
+    // this case is when a function uses the red zone, but that only happens
+    // with leaf functions, and we don't restore __stack_pointer in leaf
+    // functions anyway.
+    auto InsertPos = MBB.begin();
+    if (MBB.begin()->getOpcode() == WebAssembly::CATCH)
+      InsertPos++;
+    FrameLowering->writeSPToGlobal(WebAssembly::SP32, MF, MBB, InsertPos,
+                                   MBB.begin()->getDebugLoc());
   }
-  return true;
+  return Changed;
 }
diff --git a/lib/Target/WebAssembly/WebAssemblyLowerBrUnless.cpp b/lib/Target/WebAssembly/WebAssemblyLowerBrUnless.cpp
index c9a3527d3fbd69fcb268e6980a5cdae1eea515d3..34a8195ac4b4cf8ef4cb39a8ebbff90f839c737c 100644
--- a/lib/Target/WebAssembly/WebAssemblyLowerBrUnless.cpp
+++ b/lib/Target/WebAssembly/WebAssemblyLowerBrUnless.cpp
@@ -1,9 +1,8 @@
 //===-- WebAssemblyLowerBrUnless.cpp - Lower br_unless --------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
diff --git a/lib/Target/WebAssembly/WebAssemblyLowerEmscriptenEHSjLj.cpp b/lib/Target/WebAssembly/WebAssemblyLowerEmscriptenEHSjLj.cpp
index 8755198f45b52481ae73854e18d8d7248e93a380..9c33508a3df18fe0de1cb685f7c818bbf649b136 100644
--- a/lib/Target/WebAssembly/WebAssemblyLowerEmscriptenEHSjLj.cpp
+++ b/lib/Target/WebAssembly/WebAssemblyLowerEmscriptenEHSjLj.cpp
@@ -1,9 +1,8 @@
 //=== WebAssemblyLowerEmscriptenEHSjLj.cpp - Lower exceptions for Emscripten =//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
@@ -240,16 +239,16 @@ class WebAssemblyLowerEmscriptenEHSjLj final : public ModulePass {
   bool EnableEH;   // Enable exception handling
   bool EnableSjLj; // Enable setjmp/longjmp handling
 
-  GlobalVariable *ThrewGV;
-  GlobalVariable *ThrewValueGV;
-  Function *GetTempRet0Func;
-  Function *SetTempRet0Func;
-  Function *ResumeF;
-  Function *EHTypeIDF;
-  Function *EmLongjmpF;
-  Function *EmLongjmpJmpbufF;
-  Function *SaveSetjmpF;
-  Function *TestSetjmpF;
+  GlobalVariable *ThrewGV = nullptr;
+  GlobalVariable *ThrewValueGV = nullptr;
+  Function *GetTempRet0Func = nullptr;
+  Function *SetTempRet0Func = nullptr;
+  Function *ResumeF = nullptr;
+  Function *EHTypeIDF = nullptr;
+  Function *EmLongjmpF = nullptr;
+  Function *EmLongjmpJmpbufF = nullptr;
+  Function *SaveSetjmpF = nullptr;
+  Function *TestSetjmpF = nullptr;
 
   // __cxa_find_matching_catch_N functions.
   // Indexed by the number of clauses in an original landingpad instruction.
@@ -282,11 +281,7 @@ public:
   static char ID;
 
   WebAssemblyLowerEmscriptenEHSjLj(bool EnableEH = true, bool EnableSjLj = true)
-      : ModulePass(ID), EnableEH(EnableEH), EnableSjLj(EnableSjLj),
-        ThrewGV(nullptr), ThrewValueGV(nullptr), GetTempRet0Func(nullptr),
-        SetTempRet0Func(nullptr), ResumeF(nullptr), EHTypeIDF(nullptr),
-        EmLongjmpF(nullptr), EmLongjmpJmpbufF(nullptr), SaveSetjmpF(nullptr),
-        TestSetjmpF(nullptr) {
+      : ModulePass(ID), EnableEH(EnableEH), EnableSjLj(EnableSjLj) {
     EHWhitelistSet.insert(EHWhitelist.begin(), EHWhitelist.end());
   }
   bool runOnModule(Module &M) override;
@@ -433,8 +428,8 @@ Value *WebAssemblyLowerEmscriptenEHSjLj::wrapInvoke(CallOrInvoke *CI) {
   // No attributes for the callee pointer.
   ArgAttributes.push_back(AttributeSet());
   // Copy the argument attributes from the original
-  for (unsigned i = 0, e = CI->getNumArgOperands(); i < e; ++i)
-    ArgAttributes.push_back(InvokeAL.getParamAttributes(i));
+  for (unsigned I = 0, E = CI->getNumArgOperands(); I < E; ++I)
+    ArgAttributes.push_back(InvokeAL.getParamAttributes(I));
 
   // Reconstruct the AttributesList based on the vector we constructed.
   AttributeList NewCallAL =
@@ -446,7 +441,8 @@ Value *WebAssemblyLowerEmscriptenEHSjLj::wrapInvoke(CallOrInvoke *CI) {
 
   // Post-invoke
   // %__THREW__.val = __THREW__; __THREW__ = 0;
-  Value *Threw = IRB.CreateLoad(ThrewGV, ThrewGV->getName() + ".val");
+  Value *Threw =
+      IRB.CreateLoad(IRB.getInt32Ty(), ThrewGV, ThrewGV->getName() + ".val");
   IRB.CreateStore(IRB.getInt32(0), ThrewGV);
   return Threw;
 }
@@ -549,8 +545,8 @@ void WebAssemblyLowerEmscriptenEHSjLj::wrapTestSetjmp(
   BasicBlock *ElseBB1 = BasicBlock::Create(C, "if.else1", F);
   BasicBlock *EndBB1 = BasicBlock::Create(C, "if.end", F);
   Value *ThrewCmp = IRB.CreateICmpNE(Threw, IRB.getInt32(0));
-  Value *ThrewValue =
-      IRB.CreateLoad(ThrewValueGV, ThrewValueGV->getName() + ".val");
+  Value *ThrewValue = IRB.CreateLoad(IRB.getInt32Ty(), ThrewValueGV,
+                                     ThrewValueGV->getName() + ".val");
   Value *ThrewValueCmp = IRB.CreateICmpNE(ThrewValue, IRB.getInt32(0));
   Value *Cmp1 = IRB.CreateAnd(ThrewCmp, ThrewValueCmp, "cmp1");
   IRB.CreateCondBr(Cmp1, ThenBB1, ElseBB1);
@@ -562,8 +558,8 @@ void WebAssemblyLowerEmscriptenEHSjLj::wrapTestSetjmp(
   BasicBlock *EndBB2 = BasicBlock::Create(C, "if.end2", F);
   Value *ThrewInt = IRB.CreateIntToPtr(Threw, Type::getInt32PtrTy(C),
                                        Threw->getName() + ".i32p");
-  Value *LoadedThrew =
-      IRB.CreateLoad(ThrewInt, ThrewInt->getName() + ".loaded");
+  Value *LoadedThrew = IRB.CreateLoad(IRB.getInt32Ty(), ThrewInt,
+                                      ThrewInt->getName() + ".loaded");
   Value *ThenLabel = IRB.CreateCall(
       TestSetjmpF, {LoadedThrew, SetjmpTable, SetjmpTableSize}, "label");
   Value *Cmp2 = IRB.CreateICmpEQ(ThenLabel, IRB.getInt32(0));
@@ -606,11 +602,11 @@ void WebAssemblyLowerEmscriptenEHSjLj::rebuildSSA(Function &F) {
         ++UI;
         SSA.Initialize(I.getType(), I.getName());
         SSA.AddAvailableValue(&BB, &I);
-        Instruction *User = cast<Instruction>(U.getUser());
+        auto *User = cast<Instruction>(U.getUser());
         if (User->getParent() == &BB)
           continue;
 
-        if (PHINode *UserPN = dyn_cast<PHINode>(User))
+        if (auto *UserPN = dyn_cast<PHINode>(User))
           if (UserPN->getIncomingBlock(U) == &BB)
             continue;
 
@@ -623,6 +619,8 @@ void WebAssemblyLowerEmscriptenEHSjLj::rebuildSSA(Function &F) {
 }
 
 bool WebAssemblyLowerEmscriptenEHSjLj::runOnModule(Module &M) {
+  LLVM_DEBUG(dbgs() << "********** Lower Emscripten EH & SjLj **********\n");
+
   LLVMContext &C = M.getContext();
   IRBuilder<> IRB(C);
 
@@ -767,7 +765,8 @@ bool WebAssemblyLowerEmscriptenEHSjLj::runEHOnFunction(Function &F) {
       // This can't throw, and we don't need this invoke, just replace it with a
       // call+branch
       SmallVector<Value *, 16> Args(II->arg_begin(), II->arg_end());
-      CallInst *NewCall = IRB.CreateCall(II->getCalledValue(), Args);
+      CallInst *NewCall =
+          IRB.CreateCall(II->getFunctionType(), II->getCalledValue(), Args);
       NewCall->takeName(II);
       NewCall->setCallingConv(II->getCallingConv());
       NewCall->setDebugLoc(II->getDebugLoc());
@@ -834,15 +833,15 @@ bool WebAssemblyLowerEmscriptenEHSjLj::runEHOnFunction(Function &F) {
   for (LandingPadInst *LPI : LandingPads) {
     IRB.SetInsertPoint(LPI);
     SmallVector<Value *, 16> FMCArgs;
-    for (unsigned i = 0, e = LPI->getNumClauses(); i < e; ++i) {
-      Constant *Clause = LPI->getClause(i);
+    for (unsigned I = 0, E = LPI->getNumClauses(); I < E; ++I) {
+      Constant *Clause = LPI->getClause(I);
       // As a temporary workaround for the lack of aggregate varargs support
       // in the interface between JS and wasm, break out filter operands into
       // their component elements.
-      if (LPI->isFilter(i)) {
+      if (LPI->isFilter(I)) {
         auto *ATy = cast<ArrayType>(Clause->getType());
-        for (unsigned j = 0, e = ATy->getNumElements(); j < e; ++j) {
-          Value *EV = IRB.CreateExtractValue(Clause, makeArrayRef(j), "filter");
+        for (unsigned J = 0, E = ATy->getNumElements(); J < E; ++J) {
+          Value *EV = IRB.CreateExtractValue(Clause, makeArrayRef(J), "filter");
           FMCArgs.push_back(EV);
         }
       } else
@@ -952,8 +951,8 @@ bool WebAssemblyLowerEmscriptenEHSjLj::runSjLjOnFunction(Function &F) {
     BBs.push_back(&BB);
 
   // BBs.size() will change within the loop, so we query it every time
-  for (unsigned i = 0; i < BBs.size(); i++) {
-    BasicBlock *BB = BBs[i];
+  for (unsigned I = 0; I < BBs.size(); I++) {
+    BasicBlock *BB = BBs[I];
     for (Instruction &I : *BB) {
       assert(!isa<InvokeInst>(&I));
       auto *CI = dyn_cast<CallInst>(&I);
@@ -1026,9 +1025,9 @@ bool WebAssemblyLowerEmscriptenEHSjLj::runSjLjOnFunction(Function &F) {
       // switch case). 0 means a longjmp that is not ours to handle, needs a
       // rethrow. Otherwise the index is the same as the index in P+1 (to avoid
       // 0).
-      for (unsigned i = 0; i < SetjmpRetPHIs.size(); i++) {
-        SI->addCase(IRB.getInt32(i + 1), SetjmpRetPHIs[i]->getParent());
-        SetjmpRetPHIs[i]->addIncoming(LongjmpResult, EndBB);
+      for (unsigned I = 0; I < SetjmpRetPHIs.size(); I++) {
+        SI->addCase(IRB.getInt32(I + 1), SetjmpRetPHIs[I]->getParent());
+        SetjmpRetPHIs[I]->addIncoming(LongjmpResult, EndBB);
       }
 
       // We are splitting the block here, and must continue to find other calls
@@ -1075,7 +1074,7 @@ bool WebAssemblyLowerEmscriptenEHSjLj::runSjLjOnFunction(Function &F) {
     Use &U = *UI;
     // Increment the iterator before removing the use from the list.
     ++UI;
-    if (Instruction *I = dyn_cast<Instruction>(U.getUser()))
+    if (auto *I = dyn_cast<Instruction>(U.getUser()))
       if (I->getParent() != &EntryBB)
         SetjmpTableSSA.RewriteUse(U);
   }
@@ -1083,7 +1082,7 @@ bool WebAssemblyLowerEmscriptenEHSjLj::runSjLjOnFunction(Function &F) {
        UI != UE;) {
     Use &U = *UI;
     ++UI;
-    if (Instruction *I = dyn_cast<Instruction>(U.getUser()))
+    if (auto *I = dyn_cast<Instruction>(U.getUser()))
       if (I->getParent() != &EntryBB)
         SetjmpTableSizeSSA.RewriteUse(U);
   }
diff --git a/lib/Target/WebAssembly/WebAssemblyLowerGlobalDtors.cpp b/lib/Target/WebAssembly/WebAssemblyLowerGlobalDtors.cpp
index 3988189ba7ce73eaad2a591208da78c0ec20dab7..72e0c33e1e0267e09009f11e5464a02d63605959 100644
--- a/lib/Target/WebAssembly/WebAssemblyLowerGlobalDtors.cpp
+++ b/lib/Target/WebAssembly/WebAssemblyLowerGlobalDtors.cpp
@@ -1,9 +1,8 @@
 //===-- WebAssemblyLowerGlobalDtors.cpp - Lower @llvm.global_dtors --------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
@@ -59,6 +58,8 @@ ModulePass *llvm::createWebAssemblyLowerGlobalDtors() {
 }
 
 bool LowerGlobalDtors::runOnModule(Module &M) {
+  LLVM_DEBUG(dbgs() << "********** Lower Global Destructors **********\n");
+
   GlobalVariable *GV = M.getGlobalVariable("llvm.global_dtors");
   if (!GV)
     return false;
@@ -68,7 +69,7 @@ bool LowerGlobalDtors::runOnModule(Module &M) {
     return false;
 
   // Sanity-check @llvm.global_dtor's type.
-  StructType *ETy = dyn_cast<StructType>(InitList->getType()->getElementType());
+  auto *ETy = dyn_cast<StructType>(InitList->getType()->getElementType());
   if (!ETy || ETy->getNumElements() != 3 ||
       !ETy->getTypeAtIndex(0U)->isIntegerTy() ||
       !ETy->getTypeAtIndex(1U)->isPointerTy() ||
@@ -79,11 +80,11 @@ bool LowerGlobalDtors::runOnModule(Module &M) {
   // associated symbol.
   std::map<uint16_t, MapVector<Constant *, std::vector<Constant *>>> DtorFuncs;
   for (Value *O : InitList->operands()) {
-    ConstantStruct *CS = dyn_cast<ConstantStruct>(O);
+    auto *CS = dyn_cast<ConstantStruct>(O);
     if (!CS)
       continue; // Malformed.
 
-    ConstantInt *Priority = dyn_cast<ConstantInt>(CS->getOperand(0));
+    auto *Priority = dyn_cast<ConstantInt>(CS->getOperand(0));
     if (!Priority)
       continue; // Malformed.
     uint16_t PriorityValue = Priority->getLimitedValue(UINT16_MAX);
@@ -108,10 +109,11 @@ bool LowerGlobalDtors::runOnModule(Module &M) {
       FunctionType::get(Type::getVoidTy(C), AtExitFuncArgs,
                         /*isVarArg=*/false);
 
-  Type *AtExitArgs[] = {PointerType::get(AtExitFuncTy, 0), VoidStar, VoidStar};
-  FunctionType *AtExitTy = FunctionType::get(Type::getInt32Ty(C), AtExitArgs,
-                                             /*isVarArg=*/false);
-  Constant *AtExit = M.getOrInsertFunction("__cxa_atexit", AtExitTy);
+  FunctionCallee AtExit = M.getOrInsertFunction(
+      "__cxa_atexit",
+      FunctionType::get(Type::getInt32Ty(C),
+                        {PointerType::get(AtExitFuncTy, 0), VoidStar, VoidStar},
+                        /*isVarArg=*/false));
 
   // Declare __dso_local.
   Constant *DsoHandle = M.getNamedValue("__dso_handle");
@@ -141,13 +143,13 @@ bool LowerGlobalDtors::runOnModule(Module &M) {
                                           : Twine()),
           &M);
       BasicBlock *BB = BasicBlock::Create(C, "body", CallDtors);
+      FunctionType *VoidVoid = FunctionType::get(Type::getVoidTy(C),
+                                                 /*isVarArg=*/false);
 
       for (auto Dtor : AssociatedAndMore.second)
-        CallInst::Create(Dtor, "", BB);
+        CallInst::Create(VoidVoid, Dtor, "", BB);
       ReturnInst::Create(C, BB);
 
-      FunctionType *VoidVoid = FunctionType::get(Type::getVoidTy(C),
-                                                 /*isVarArg=*/false);
       Function *RegisterCallDtors = Function::Create(
           VoidVoid, Function::PrivateLinkage,
           "register_call_dtors" +
diff --git a/lib/Target/WebAssembly/WebAssemblyMCInstLower.cpp b/lib/Target/WebAssembly/WebAssemblyMCInstLower.cpp
index fa862fbaa634c2b3772f9958af28049ed00d2983..054fe5496d593386cedf7f29a70f0ee56e9165f0 100644
--- a/lib/Target/WebAssembly/WebAssemblyMCInstLower.cpp
+++ b/lib/Target/WebAssembly/WebAssemblyMCInstLower.cpp
@@ -1,9 +1,8 @@
 // WebAssemblyMCInstLower.cpp - Convert WebAssembly MachineInstr to an MCInst //
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
@@ -37,7 +36,7 @@ using namespace llvm;
 
 // This disables the removal of registers when lowering into MC, as required
 // by some current tests.
-static cl::opt<bool>
+cl::opt<bool>
     WasmKeepRegisters("wasm-keep-registers", cl::Hidden,
                       cl::desc("WebAssembly: output stack registers in"
                                " instruction output for test purposes only."),
@@ -48,7 +47,7 @@ static void removeRegisterOperands(const MachineInstr *MI, MCInst &OutMI);
 MCSymbol *
 WebAssemblyMCInstLower::GetGlobalAddressSymbol(const MachineOperand &MO) const {
   const GlobalValue *Global = MO.getGlobal();
-  MCSymbolWasm *WasmSym = cast<MCSymbolWasm>(Printer.getSymbol(Global));
+  auto *WasmSym = cast<MCSymbolWasm>(Printer.getSymbol(Global));
 
   if (const auto *FuncTy = dyn_cast<FunctionType>(Global->getValueType())) {
     const MachineFunction &MF = *MO.getParent()->getParent()->getParent();
@@ -57,9 +56,9 @@ WebAssemblyMCInstLower::GetGlobalAddressSymbol(const MachineOperand &MO) const {
 
     SmallVector<MVT, 1> ResultMVTs;
     SmallVector<MVT, 4> ParamMVTs;
-    ComputeSignatureVTs(FuncTy, CurrentFunc, TM, ParamMVTs, ResultMVTs);
+    computeSignatureVTs(FuncTy, CurrentFunc, TM, ParamMVTs, ResultMVTs);
 
-    auto Signature = SignatureFromMVTs(ResultMVTs, ParamMVTs);
+    auto Signature = signatureFromMVTs(ResultMVTs, ParamMVTs);
     WasmSym->setSignature(Signature.get());
     Printer.addSignature(std::move(Signature));
     WasmSym->setType(wasm::WASM_SYMBOL_TYPE_FUNCTION);
@@ -71,8 +70,7 @@ WebAssemblyMCInstLower::GetGlobalAddressSymbol(const MachineOperand &MO) const {
 MCSymbol *WebAssemblyMCInstLower::GetExternalSymbolSymbol(
     const MachineOperand &MO) const {
   const char *Name = MO.getSymbolName();
-  MCSymbolWasm *WasmSym =
-      cast<MCSymbolWasm>(Printer.GetExternalSymbolSymbol(Name));
+  auto *WasmSym = cast<MCSymbolWasm>(Printer.GetExternalSymbolSymbol(Name));
   const WebAssemblySubtarget &Subtarget = Printer.getSubtarget();
 
   // Except for the two exceptions (__stack_pointer and __cpp_exception), all
@@ -110,7 +108,7 @@ MCSymbol *WebAssemblyMCInstLower::GetExternalSymbolSymbol(
                                            : wasm::ValType::I32);
   } else { // Function symbols
     WasmSym->setType(wasm::WASM_SYMBOL_TYPE_FUNCTION);
-    GetLibcallSignature(Subtarget, Name, Returns, Params);
+    getLibcallSignature(Subtarget, Name, Returns, Params);
   }
   auto Signature =
       make_unique<wasm::WasmSignature>(std::move(Returns), std::move(Params));
@@ -120,7 +118,7 @@ MCSymbol *WebAssemblyMCInstLower::GetExternalSymbolSymbol(
   return WasmSym;
 }
 
-MCOperand WebAssemblyMCInstLower::LowerSymbolOperand(MCSymbol *Sym,
+MCOperand WebAssemblyMCInstLower::lowerSymbolOperand(MCSymbol *Sym,
                                                      int64_t Offset,
                                                      bool IsFunc, bool IsGlob,
                                                      bool IsEvent) const {
@@ -161,13 +159,13 @@ static wasm::ValType getType(const TargetRegisterClass *RC) {
   llvm_unreachable("Unexpected register class");
 }
 
-void WebAssemblyMCInstLower::Lower(const MachineInstr *MI,
+void WebAssemblyMCInstLower::lower(const MachineInstr *MI,
                                    MCInst &OutMI) const {
   OutMI.setOpcode(MI->getOpcode());
 
   const MCInstrDesc &Desc = MI->getDesc();
-  for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) {
-    const MachineOperand &MO = MI->getOperand(i);
+  for (unsigned I = 0, E = MI->getNumOperands(); I != E; ++I) {
+    const MachineOperand &MO = MI->getOperand(I);
 
     MCOperand MCOp;
     switch (MO.getType()) {
@@ -188,8 +186,8 @@ void WebAssemblyMCInstLower::Lower(const MachineInstr *MI,
       break;
     }
     case MachineOperand::MO_Immediate:
-      if (i < Desc.NumOperands) {
-        const MCOperandInfo &Info = Desc.OpInfo[i];
+      if (I < Desc.NumOperands) {
+        const MCOperandInfo &Info = Desc.OpInfo[I];
         if (Info.OperandType == WebAssembly::OPERAND_TYPEINDEX) {
           MCSymbol *Sym = Printer.createTempSymbol("typeindex");
 
@@ -209,7 +207,7 @@ void WebAssemblyMCInstLower::Lower(const MachineInstr *MI,
           if (WebAssembly::isCallIndirect(*MI))
             Params.pop_back();
 
-          MCSymbolWasm *WasmSym = cast<MCSymbolWasm>(Sym);
+          auto *WasmSym = cast<MCSymbolWasm>(Sym);
           auto Signature = make_unique<wasm::WasmSignature>(std::move(Returns),
                                                             std::move(Params));
           WasmSym->setSignature(Signature.get());
@@ -239,7 +237,7 @@ void WebAssemblyMCInstLower::Lower(const MachineInstr *MI,
     case MachineOperand::MO_GlobalAddress:
       assert(MO.getTargetFlags() == WebAssemblyII::MO_NO_FLAG &&
              "WebAssembly does not use target flags on GlobalAddresses");
-      MCOp = LowerSymbolOperand(GetGlobalAddressSymbol(MO), MO.getOffset(),
+      MCOp = lowerSymbolOperand(GetGlobalAddressSymbol(MO), MO.getOffset(),
                                 MO.getGlobal()->getValueType()->isFunctionTy(),
                                 false, false);
       break;
@@ -248,7 +246,7 @@ void WebAssemblyMCInstLower::Lower(const MachineInstr *MI,
       // variable or a function.
       assert((MO.getTargetFlags() & ~WebAssemblyII::MO_SYMBOL_MASK) == 0 &&
              "WebAssembly uses only symbol flags on ExternalSymbols");
-      MCOp = LowerSymbolOperand(
+      MCOp = lowerSymbolOperand(
           GetExternalSymbolSymbol(MO), /*Offset=*/0,
           (MO.getTargetFlags() & WebAssemblyII::MO_SYMBOL_FUNCTION) != 0,
           (MO.getTargetFlags() & WebAssemblyII::MO_SYMBOL_GLOBAL) != 0,
@@ -259,7 +257,7 @@ void WebAssemblyMCInstLower::Lower(const MachineInstr *MI,
       // because global addresses or other external symbols are handled above.
       assert(MO.getTargetFlags() == 0 &&
              "WebAssembly does not use target flags on MCSymbol");
-      MCOp = LowerSymbolOperand(MO.getMCSymbol(), /*Offset=*/0, false, false,
+      MCOp = lowerSymbolOperand(MO.getMCSymbol(), /*Offset=*/0, false, false,
                                 false);
       break;
     }
diff --git a/lib/Target/WebAssembly/WebAssemblyMCInstLower.h b/lib/Target/WebAssembly/WebAssemblyMCInstLower.h
index fa7a0ea61b3b386a95a455a2174227823899f2ed..5753fb760a1a00fab1a68bda478287d3e1b36bd1 100644
--- a/lib/Target/WebAssembly/WebAssemblyMCInstLower.h
+++ b/lib/Target/WebAssembly/WebAssemblyMCInstLower.h
@@ -1,9 +1,8 @@
 //===-- WebAssemblyMCInstLower.h - Lower MachineInstr to MCInst -*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
@@ -33,13 +32,13 @@ class LLVM_LIBRARY_VISIBILITY WebAssemblyMCInstLower {
 
   MCSymbol *GetGlobalAddressSymbol(const MachineOperand &MO) const;
   MCSymbol *GetExternalSymbolSymbol(const MachineOperand &MO) const;
-  MCOperand LowerSymbolOperand(MCSymbol *Sym, int64_t Offset, bool IsFunc,
+  MCOperand lowerSymbolOperand(MCSymbol *Sym, int64_t Offset, bool IsFunc,
                                bool IsGlob, bool IsEvent) const;
 
 public:
   WebAssemblyMCInstLower(MCContext &ctx, WebAssemblyAsmPrinter &printer)
       : Ctx(ctx), Printer(printer) {}
-  void Lower(const MachineInstr *MI, MCInst &OutMI) const;
+  void lower(const MachineInstr *MI, MCInst &OutMI) const;
 };
 } // end namespace llvm
 
diff --git a/lib/Target/WebAssembly/WebAssemblyMachineFunctionInfo.cpp b/lib/Target/WebAssembly/WebAssemblyMachineFunctionInfo.cpp
index 0157af0f85109dbaa85d30d8d041f1876f1fe881..b1923dba8d6419329f276c51d3dcd2398d771223 100644
--- a/lib/Target/WebAssembly/WebAssemblyMachineFunctionInfo.cpp
+++ b/lib/Target/WebAssembly/WebAssemblyMachineFunctionInfo.cpp
@@ -1,9 +1,8 @@
 //=- WebAssemblyMachineFunctionInfo.cpp - WebAssembly Machine Function Info -=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
@@ -19,7 +18,7 @@
 #include "llvm/CodeGen/Analysis.h"
 using namespace llvm;
 
-WebAssemblyFunctionInfo::~WebAssemblyFunctionInfo() {}
+WebAssemblyFunctionInfo::~WebAssemblyFunctionInfo() = default; // anchor.
 
 void WebAssemblyFunctionInfo::initWARegs() {
   assert(WARegs.empty());
@@ -27,7 +26,7 @@ void WebAssemblyFunctionInfo::initWARegs() {
   WARegs.resize(MF.getRegInfo().getNumVirtRegs(), Reg);
 }
 
-void llvm::ComputeLegalValueVTs(const Function &F, const TargetMachine &TM,
+void llvm::computeLegalValueVTs(const Function &F, const TargetMachine &TM,
                                 Type *Ty, SmallVectorImpl<MVT> &ValueVTs) {
   const DataLayout &DL(F.getParent()->getDataLayout());
   const WebAssemblyTargetLowering &TLI =
@@ -38,16 +37,16 @@ void llvm::ComputeLegalValueVTs(const Function &F, const TargetMachine &TM,
   for (EVT VT : VTs) {
     unsigned NumRegs = TLI.getNumRegisters(F.getContext(), VT);
     MVT RegisterVT = TLI.getRegisterType(F.getContext(), VT);
-    for (unsigned i = 0; i != NumRegs; ++i)
+    for (unsigned I = 0; I != NumRegs; ++I)
       ValueVTs.push_back(RegisterVT);
   }
 }
 
-void llvm::ComputeSignatureVTs(const FunctionType *Ty, const Function &F,
+void llvm::computeSignatureVTs(const FunctionType *Ty, const Function &F,
                                const TargetMachine &TM,
                                SmallVectorImpl<MVT> &Params,
                                SmallVectorImpl<MVT> &Results) {
-  ComputeLegalValueVTs(F, TM, Ty->getReturnType(), Results);
+  computeLegalValueVTs(F, TM, Ty->getReturnType(), Results);
 
   MVT PtrVT = MVT::getIntegerVT(TM.createDataLayout().getPointerSizeInBits());
   if (Results.size() > 1) {
@@ -59,22 +58,22 @@ void llvm::ComputeSignatureVTs(const FunctionType *Ty, const Function &F,
   }
 
   for (auto *Param : Ty->params())
-    ComputeLegalValueVTs(F, TM, Param, Params);
+    computeLegalValueVTs(F, TM, Param, Params);
   if (Ty->isVarArg())
     Params.push_back(PtrVT);
 }
 
-void llvm::ValTypesFromMVTs(const ArrayRef<MVT> &In,
+void llvm::valTypesFromMVTs(const ArrayRef<MVT> &In,
                             SmallVectorImpl<wasm::ValType> &Out) {
   for (MVT Ty : In)
     Out.push_back(WebAssembly::toValType(Ty));
 }
 
 std::unique_ptr<wasm::WasmSignature>
-llvm::SignatureFromMVTs(const SmallVectorImpl<MVT> &Results,
+llvm::signatureFromMVTs(const SmallVectorImpl<MVT> &Results,
                         const SmallVectorImpl<MVT> &Params) {
   auto Sig = make_unique<wasm::WasmSignature>();
-  ValTypesFromMVTs(Results, Sig->Returns);
-  ValTypesFromMVTs(Params, Sig->Params);
+  valTypesFromMVTs(Results, Sig->Returns);
+  valTypesFromMVTs(Params, Sig->Params);
   return Sig;
 }
diff --git a/lib/Target/WebAssembly/WebAssemblyMachineFunctionInfo.h b/lib/Target/WebAssembly/WebAssemblyMachineFunctionInfo.h
index 4be4beb85d048b560d9809b7ca021c82faa9eecd..3962e8db946e98720fa598c1d529d4d5ae11d9bb 100644
--- a/lib/Target/WebAssembly/WebAssemblyMachineFunctionInfo.h
+++ b/lib/Target/WebAssembly/WebAssemblyMachineFunctionInfo.h
@@ -1,9 +1,8 @@
 // WebAssemblyMachineFunctionInfo.h-WebAssembly machine function info-*- C++ -*-
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
@@ -120,20 +119,20 @@ public:
   }
 };
 
-void ComputeLegalValueVTs(const Function &F, const TargetMachine &TM, Type *Ty,
+void computeLegalValueVTs(const Function &F, const TargetMachine &TM, Type *Ty,
                           SmallVectorImpl<MVT> &ValueVTs);
 
 // Compute the signature for a given FunctionType (Ty). Note that it's not the
 // signature for F (F is just used to get varous context)
-void ComputeSignatureVTs(const FunctionType *Ty, const Function &F,
+void computeSignatureVTs(const FunctionType *Ty, const Function &F,
                          const TargetMachine &TM, SmallVectorImpl<MVT> &Params,
                          SmallVectorImpl<MVT> &Results);
 
-void ValTypesFromMVTs(const ArrayRef<MVT> &In,
+void valTypesFromMVTs(const ArrayRef<MVT> &In,
                       SmallVectorImpl<wasm::ValType> &Out);
 
 std::unique_ptr<wasm::WasmSignature>
-SignatureFromMVTs(const SmallVectorImpl<MVT> &Results,
+signatureFromMVTs(const SmallVectorImpl<MVT> &Results,
                   const SmallVectorImpl<MVT> &Params);
 
 } // end namespace llvm
diff --git a/lib/Target/WebAssembly/WebAssemblyStoreResults.cpp b/lib/Target/WebAssembly/WebAssemblyMemIntrinsicResults.cpp
similarity index 67%
rename from lib/Target/WebAssembly/WebAssemblyStoreResults.cpp
rename to lib/Target/WebAssembly/WebAssemblyMemIntrinsicResults.cpp
index de104d42be49fbc0767b89a4a75bef73177f94d1..a26873cd452cd54cb3c24d7c7c7bcc8b7c484581 100644
--- a/lib/Target/WebAssembly/WebAssemblyStoreResults.cpp
+++ b/lib/Target/WebAssembly/WebAssemblyMemIntrinsicResults.cpp
@@ -1,26 +1,28 @@
-//===-- WebAssemblyStoreResults.cpp - Optimize using store result values --===//
+//== WebAssemblyMemIntrinsicResults.cpp - Optimize memory intrinsic results ==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
 /// \file
-/// This file implements an optimization pass using store result values.
+/// This file implements an optimization pass using memory intrinsic results.
+///
+/// Calls to memory intrinsics (memcpy, memmove, memset) return the destination
+/// address. They are in the form of
+///   %dst_new = call @memcpy %dst, %src, %len
+/// where %dst and %dst_new registers contain the same value.
 ///
-/// WebAssembly's store instructions return the stored value. This is to enable
-/// an optimization wherein uses of the stored value can be replaced by uses of
-/// the store's result value, making the stored value register more likely to
-/// be single-use, thus more likely to be useful to register stackifying, and
-/// potentially also exposing the store to register stackifying. These both can
-/// reduce get_local/set_local traffic.
+/// This is to enable an optimization wherein uses of the %dst register used in
+/// the parameter can be replaced by uses of the %dst_new register used in the
+/// result, making the %dst register more likely to be single-use, thus more
+/// likely to be useful to register stackifying, and potentially also exposing
+/// the call instruction itself to register stackifying. These both can reduce
+/// local.get/local.set traffic.
 ///
-/// This pass also performs this optimization for memcpy, memmove, and memset
-/// calls, since the LLVM intrinsics for these return void so they can't use the
-/// returned attribute and consequently aren't handled by the OptimizeReturned
-/// pass.
+/// The LLVM intrinsics for these return void so they can't use the returned
+/// attribute and consequently aren't handled by the OptimizeReturned pass.
 ///
 //===----------------------------------------------------------------------===//
 
@@ -38,15 +40,17 @@
 #include "llvm/Support/raw_ostream.h"
 using namespace llvm;
 
-#define DEBUG_TYPE "wasm-store-results"
+#define DEBUG_TYPE "wasm-mem-intrinsic-results"
 
 namespace {
-class WebAssemblyStoreResults final : public MachineFunctionPass {
+class WebAssemblyMemIntrinsicResults final : public MachineFunctionPass {
 public:
   static char ID; // Pass identification, replacement for typeid
-  WebAssemblyStoreResults() : MachineFunctionPass(ID) {}
+  WebAssemblyMemIntrinsicResults() : MachineFunctionPass(ID) {}
 
-  StringRef getPassName() const override { return "WebAssembly Store Results"; }
+  StringRef getPassName() const override {
+    return "WebAssembly Memory Intrinsic Results";
+  }
 
   void getAnalysisUsage(AnalysisUsage &AU) const override {
     AU.setPreservesCFG();
@@ -67,16 +71,17 @@ private:
 };
 } // end anonymous namespace
 
-char WebAssemblyStoreResults::ID = 0;
-INITIALIZE_PASS(WebAssemblyStoreResults, DEBUG_TYPE,
-                "Optimize store result values for WebAssembly", false, false)
+char WebAssemblyMemIntrinsicResults::ID = 0;
+INITIALIZE_PASS(WebAssemblyMemIntrinsicResults, DEBUG_TYPE,
+                "Optimize memory intrinsic result values for WebAssembly",
+                false, false)
 
-FunctionPass *llvm::createWebAssemblyStoreResults() {
-  return new WebAssemblyStoreResults();
+FunctionPass *llvm::createWebAssemblyMemIntrinsicResults() {
+  return new WebAssemblyMemIntrinsicResults();
 }
 
 // Replace uses of FromReg with ToReg if they are dominated by MI.
-static bool ReplaceDominatedUses(MachineBasicBlock &MBB, MachineInstr &MI,
+static bool replaceDominatedUses(MachineBasicBlock &MBB, MachineInstr &MI,
                                  unsigned FromReg, unsigned ToReg,
                                  const MachineRegisterInfo &MRI,
                                  MachineDominatorTree &MDT,
@@ -151,10 +156,10 @@ static bool optimizeCall(MachineBasicBlock &MBB, MachineInstr &MI,
     return false;
 
   StringRef Name(Op1.getSymbolName());
-  bool callReturnsInput = Name == TLI.getLibcallName(RTLIB::MEMCPY) ||
+  bool CallReturnsInput = Name == TLI.getLibcallName(RTLIB::MEMCPY) ||
                           Name == TLI.getLibcallName(RTLIB::MEMMOVE) ||
                           Name == TLI.getLibcallName(RTLIB::MEMSET);
-  if (!callReturnsInput)
+  if (!CallReturnsInput)
     return false;
 
   LibFunc Func;
@@ -164,29 +169,30 @@ static bool optimizeCall(MachineBasicBlock &MBB, MachineInstr &MI,
   unsigned FromReg = MI.getOperand(2).getReg();
   unsigned ToReg = MI.getOperand(0).getReg();
   if (MRI.getRegClass(FromReg) != MRI.getRegClass(ToReg))
-    report_fatal_error("Store results: call to builtin function with wrong "
-                       "signature, from/to mismatch");
-  return ReplaceDominatedUses(MBB, MI, FromReg, ToReg, MRI, MDT, LIS);
+    report_fatal_error("Memory Intrinsic results: call to builtin function "
+                       "with wrong signature, from/to mismatch");
+  return replaceDominatedUses(MBB, MI, FromReg, ToReg, MRI, MDT, LIS);
 }
 
-bool WebAssemblyStoreResults::runOnMachineFunction(MachineFunction &MF) {
+bool WebAssemblyMemIntrinsicResults::runOnMachineFunction(MachineFunction &MF) {
   LLVM_DEBUG({
-    dbgs() << "********** Store Results **********\n"
+    dbgs() << "********** Memory Intrinsic Results **********\n"
            << "********** Function: " << MF.getName() << '\n';
   });
 
   MachineRegisterInfo &MRI = MF.getRegInfo();
-  MachineDominatorTree &MDT = getAnalysis<MachineDominatorTree>();
+  auto &MDT = getAnalysis<MachineDominatorTree>();
   const WebAssemblyTargetLowering &TLI =
       *MF.getSubtarget<WebAssemblySubtarget>().getTargetLowering();
   const auto &LibInfo = getAnalysis<TargetLibraryInfoWrapperPass>().getTLI();
-  LiveIntervals &LIS = getAnalysis<LiveIntervals>();
+  auto &LIS = getAnalysis<LiveIntervals>();
   bool Changed = false;
 
   // We don't preserve SSA form.
   MRI.leaveSSA();
 
-  assert(MRI.tracksLiveness() && "StoreResults expects liveness tracking");
+  assert(MRI.tracksLiveness() &&
+         "MemIntrinsicResults expects liveness tracking");
 
   for (auto &MBB : MF) {
     LLVM_DEBUG(dbgs() << "Basic Block: " << MBB.getName() << '\n');
diff --git a/lib/Target/WebAssembly/WebAssemblyOptimizeLiveIntervals.cpp b/lib/Target/WebAssembly/WebAssemblyOptimizeLiveIntervals.cpp
index 3d0a15244ee0075abb204d0606b1bfeb61064948..8c7c3305c201e082e37e14a93c0ccddb25b7cefc 100644
--- a/lib/Target/WebAssembly/WebAssemblyOptimizeLiveIntervals.cpp
+++ b/lib/Target/WebAssembly/WebAssemblyOptimizeLiveIntervals.cpp
@@ -1,9 +1,8 @@
 //===--- WebAssemblyOptimizeLiveIntervals.cpp - LiveInterval processing ---===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
@@ -72,7 +71,7 @@ bool WebAssemblyOptimizeLiveIntervals::runOnMachineFunction(
                     << MF.getName() << '\n');
 
   MachineRegisterInfo &MRI = MF.getRegInfo();
-  LiveIntervals &LIS = getAnalysis<LiveIntervals>();
+  auto &LIS = getAnalysis<LiveIntervals>();
 
   // We don't preserve SSA form.
   MRI.leaveSSA();
@@ -81,8 +80,8 @@ bool WebAssemblyOptimizeLiveIntervals::runOnMachineFunction(
 
   // Split multiple-VN LiveIntervals into multiple LiveIntervals.
   SmallVector<LiveInterval *, 4> SplitLIs;
-  for (unsigned i = 0, e = MRI.getNumVirtRegs(); i < e; ++i) {
-    unsigned Reg = TargetRegisterInfo::index2VirtReg(i);
+  for (unsigned I = 0, E = MRI.getNumVirtRegs(); I < E; ++I) {
+    unsigned Reg = TargetRegisterInfo::index2VirtReg(I);
     if (MRI.reg_nodbg_empty(Reg))
       continue;
 
diff --git a/lib/Target/WebAssembly/WebAssemblyOptimizeReturned.cpp b/lib/Target/WebAssembly/WebAssemblyOptimizeReturned.cpp
index 113ee2532bce2ff2e1ee78829a262d04e10e3a3d..d20352259e07369788a3ae111d185f2c3f714c34 100644
--- a/lib/Target/WebAssembly/WebAssemblyOptimizeReturned.cpp
+++ b/lib/Target/WebAssembly/WebAssemblyOptimizeReturned.cpp
@@ -1,9 +1,8 @@
 //===-- WebAssemblyOptimizeReturned.cpp - Optimize "returned" attributes --===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
@@ -37,11 +36,11 @@ class OptimizeReturned final : public FunctionPass,
 
   bool runOnFunction(Function &F) override;
 
-  DominatorTree *DT;
+  DominatorTree *DT = nullptr;
 
 public:
   static char ID;
-  OptimizeReturned() : FunctionPass(ID), DT(nullptr) {}
+  OptimizeReturned() : FunctionPass(ID) {}
 
   void visitCallSite(CallSite CS);
 };
@@ -57,10 +56,10 @@ FunctionPass *llvm::createWebAssemblyOptimizeReturned() {
 }
 
 void OptimizeReturned::visitCallSite(CallSite CS) {
-  for (unsigned i = 0, e = CS.getNumArgOperands(); i < e; ++i)
-    if (CS.paramHasAttr(i, Attribute::Returned)) {
+  for (unsigned I = 0, E = CS.getNumArgOperands(); I < E; ++I)
+    if (CS.paramHasAttr(I, Attribute::Returned)) {
       Instruction *Inst = CS.getInstruction();
-      Value *Arg = CS.getArgOperand(i);
+      Value *Arg = CS.getArgOperand(I);
       // Ignore constants, globals, undef, etc.
       if (isa<Constant>(Arg))
         continue;
@@ -74,6 +73,10 @@ void OptimizeReturned::visitCallSite(CallSite CS) {
 }
 
 bool OptimizeReturned::runOnFunction(Function &F) {
+  LLVM_DEBUG(dbgs() << "********** Optimize returned Attributes **********\n"
+                       "********** Function: "
+                    << F.getName() << '\n');
+
   DT = &getAnalysis<DominatorTreeWrapperPass>().getDomTree();
   visit(F);
   return true;
diff --git a/lib/Target/WebAssembly/WebAssemblyPeephole.cpp b/lib/Target/WebAssembly/WebAssemblyPeephole.cpp
index 2dfd85953f14c07278f35c46440fd2471fdd49f6..dc8f3c3efbe4661bab608a5f33ab93e9c2579061 100644
--- a/lib/Target/WebAssembly/WebAssemblyPeephole.cpp
+++ b/lib/Target/WebAssembly/WebAssemblyPeephole.cpp
@@ -1,9 +1,8 @@
 //===-- WebAssemblyPeephole.cpp - WebAssembly Peephole Optimiztions -------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
@@ -58,7 +57,7 @@ FunctionPass *llvm::createWebAssemblyPeephole() {
 }
 
 /// If desirable, rewrite NewReg to a drop register.
-static bool MaybeRewriteToDrop(unsigned OldReg, unsigned NewReg,
+static bool maybeRewriteToDrop(unsigned OldReg, unsigned NewReg,
                                MachineOperand &MO, WebAssemblyFunctionInfo &MFI,
                                MachineRegisterInfo &MRI) {
   bool Changed = false;
@@ -72,7 +71,7 @@ static bool MaybeRewriteToDrop(unsigned OldReg, unsigned NewReg,
   return Changed;
 }
 
-static bool MaybeRewriteToFallthrough(MachineInstr &MI, MachineBasicBlock &MBB,
+static bool maybeRewriteToFallthrough(MachineInstr &MI, MachineBasicBlock &MBB,
                                       const MachineFunction &MF,
                                       WebAssemblyFunctionInfo &MFI,
                                       MachineRegisterInfo &MRI,
@@ -150,7 +149,7 @@ bool WebAssemblyPeephole::runOnMachineFunction(MachineFunction &MF) {
               if (MRI.getRegClass(NewReg) != MRI.getRegClass(OldReg))
                 report_fatal_error("Peephole: call to builtin function with "
                                    "wrong signature, from/to mismatch");
-              Changed |= MaybeRewriteToDrop(OldReg, NewReg, MO, MFI, MRI);
+              Changed |= maybeRewriteToDrop(OldReg, NewReg, MO, MFI, MRI);
             }
           }
         }
@@ -158,57 +157,57 @@ bool WebAssemblyPeephole::runOnMachineFunction(MachineFunction &MF) {
       }
       // Optimize away an explicit void return at the end of the function.
       case WebAssembly::RETURN_I32:
-        Changed |= MaybeRewriteToFallthrough(
+        Changed |= maybeRewriteToFallthrough(
             MI, MBB, MF, MFI, MRI, TII, WebAssembly::FALLTHROUGH_RETURN_I32,
             WebAssembly::COPY_I32);
         break;
       case WebAssembly::RETURN_I64:
-        Changed |= MaybeRewriteToFallthrough(
+        Changed |= maybeRewriteToFallthrough(
             MI, MBB, MF, MFI, MRI, TII, WebAssembly::FALLTHROUGH_RETURN_I64,
             WebAssembly::COPY_I64);
         break;
       case WebAssembly::RETURN_F32:
-        Changed |= MaybeRewriteToFallthrough(
+        Changed |= maybeRewriteToFallthrough(
             MI, MBB, MF, MFI, MRI, TII, WebAssembly::FALLTHROUGH_RETURN_F32,
             WebAssembly::COPY_F32);
         break;
       case WebAssembly::RETURN_F64:
-        Changed |= MaybeRewriteToFallthrough(
+        Changed |= maybeRewriteToFallthrough(
             MI, MBB, MF, MFI, MRI, TII, WebAssembly::FALLTHROUGH_RETURN_F64,
             WebAssembly::COPY_F64);
         break;
       case WebAssembly::RETURN_v16i8:
-        Changed |= MaybeRewriteToFallthrough(
+        Changed |= maybeRewriteToFallthrough(
             MI, MBB, MF, MFI, MRI, TII, WebAssembly::FALLTHROUGH_RETURN_v16i8,
             WebAssembly::COPY_V128);
         break;
       case WebAssembly::RETURN_v8i16:
-        Changed |= MaybeRewriteToFallthrough(
+        Changed |= maybeRewriteToFallthrough(
             MI, MBB, MF, MFI, MRI, TII, WebAssembly::FALLTHROUGH_RETURN_v8i16,
             WebAssembly::COPY_V128);
         break;
       case WebAssembly::RETURN_v4i32:
-        Changed |= MaybeRewriteToFallthrough(
+        Changed |= maybeRewriteToFallthrough(
             MI, MBB, MF, MFI, MRI, TII, WebAssembly::FALLTHROUGH_RETURN_v4i32,
             WebAssembly::COPY_V128);
         break;
       case WebAssembly::RETURN_v2i64:
-        Changed |= MaybeRewriteToFallthrough(
+        Changed |= maybeRewriteToFallthrough(
             MI, MBB, MF, MFI, MRI, TII, WebAssembly::FALLTHROUGH_RETURN_v2i64,
             WebAssembly::COPY_V128);
         break;
       case WebAssembly::RETURN_v4f32:
-        Changed |= MaybeRewriteToFallthrough(
+        Changed |= maybeRewriteToFallthrough(
             MI, MBB, MF, MFI, MRI, TII, WebAssembly::FALLTHROUGH_RETURN_v4f32,
             WebAssembly::COPY_V128);
         break;
       case WebAssembly::RETURN_v2f64:
-        Changed |= MaybeRewriteToFallthrough(
+        Changed |= maybeRewriteToFallthrough(
             MI, MBB, MF, MFI, MRI, TII, WebAssembly::FALLTHROUGH_RETURN_v2f64,
             WebAssembly::COPY_V128);
         break;
       case WebAssembly::RETURN_VOID:
-        Changed |= MaybeRewriteToFallthrough(
+        Changed |= maybeRewriteToFallthrough(
             MI, MBB, MF, MFI, MRI, TII, WebAssembly::FALLTHROUGH_RETURN_VOID,
             WebAssembly::INSTRUCTION_LIST_END);
         break;
diff --git a/lib/Target/WebAssembly/WebAssemblyPrepareForLiveIntervals.cpp b/lib/Target/WebAssembly/WebAssemblyPrepareForLiveIntervals.cpp
index 0be0ba657830452ae23ce7d0bb12a09e086f3159..12b70f7ce4f130a997f6e1a2d24789406ab49421 100644
--- a/lib/Target/WebAssembly/WebAssemblyPrepareForLiveIntervals.cpp
+++ b/lib/Target/WebAssembly/WebAssemblyPrepareForLiveIntervals.cpp
@@ -1,9 +1,8 @@
 //===- WebAssemblyPrepareForLiveIntervals.cpp - Prepare for LiveIntervals -===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
@@ -63,7 +62,7 @@ FunctionPass *llvm::createWebAssemblyPrepareForLiveIntervals() {
 }
 
 // Test whether the given register has an ARGUMENT def.
-static bool HasArgumentDef(unsigned Reg, const MachineRegisterInfo &MRI) {
+static bool hasArgumentDef(unsigned Reg, const MachineRegisterInfo &MRI) {
   for (const auto &Def : MRI.def_instructions(Reg))
     if (WebAssembly::isArgument(Def))
       return true;
@@ -95,15 +94,15 @@ bool WebAssemblyPrepareForLiveIntervals::runOnMachineFunction(
   //
   // TODO: This is fairly heavy-handed; find a better approach.
   //
-  for (unsigned i = 0, e = MRI.getNumVirtRegs(); i < e; ++i) {
-    unsigned Reg = TargetRegisterInfo::index2VirtReg(i);
+  for (unsigned I = 0, E = MRI.getNumVirtRegs(); I < E; ++I) {
+    unsigned Reg = TargetRegisterInfo::index2VirtReg(I);
 
     // Skip unused registers.
     if (MRI.use_nodbg_empty(Reg))
       continue;
 
     // Skip registers that have an ARGUMENT definition.
-    if (HasArgumentDef(Reg, MRI))
+    if (hasArgumentDef(Reg, MRI))
       continue;
 
     BuildMI(Entry, Entry.begin(), DebugLoc(),
diff --git a/lib/Target/WebAssembly/WebAssemblyRegColoring.cpp b/lib/Target/WebAssembly/WebAssemblyRegColoring.cpp
index d97b13a8d699a65c1064c8f64fb633eb004b9493..6f09c45b6642e28715f24445364b6cc119f329e3 100644
--- a/lib/Target/WebAssembly/WebAssemblyRegColoring.cpp
+++ b/lib/Target/WebAssembly/WebAssemblyRegColoring.cpp
@@ -1,9 +1,8 @@
 //===-- WebAssemblyRegColoring.cpp - Register coloring --------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
@@ -66,11 +65,11 @@ FunctionPass *llvm::createWebAssemblyRegColoring() {
 static float computeWeight(const MachineRegisterInfo *MRI,
                            const MachineBlockFrequencyInfo *MBFI,
                            unsigned VReg) {
-  float weight = 0.0f;
+  float Weight = 0.0f;
   for (MachineOperand &MO : MRI->reg_nodbg_operands(VReg))
-    weight += LiveIntervals::getSpillWeight(MO.isDef(), MO.isUse(), MBFI,
+    Weight += LiveIntervals::getSpillWeight(MO.isDef(), MO.isUse(), MBFI,
                                             *MO.getParent());
-  return weight;
+  return Weight;
 }
 
 bool WebAssemblyRegColoring::runOnMachineFunction(MachineFunction &MF) {
@@ -98,8 +97,8 @@ bool WebAssemblyRegColoring::runOnMachineFunction(MachineFunction &MF) {
   SortedIntervals.reserve(NumVRegs);
 
   LLVM_DEBUG(dbgs() << "Interesting register intervals:\n");
-  for (unsigned i = 0; i < NumVRegs; ++i) {
-    unsigned VReg = TargetRegisterInfo::index2VirtReg(i);
+  for (unsigned I = 0; I < NumVRegs; ++I) {
+    unsigned VReg = TargetRegisterInfo::index2VirtReg(I);
     if (MFI.isVRegStackified(VReg))
       continue;
     // Skip unused registers, which can use $drop.
@@ -134,10 +133,10 @@ bool WebAssemblyRegColoring::runOnMachineFunction(MachineFunction &MF) {
       SortedIntervals.size());
   BitVector UsedColors(SortedIntervals.size());
   bool Changed = false;
-  for (size_t i = 0, e = SortedIntervals.size(); i < e; ++i) {
-    LiveInterval *LI = SortedIntervals[i];
+  for (size_t I = 0, E = SortedIntervals.size(); I < E; ++I) {
+    LiveInterval *LI = SortedIntervals[I];
     unsigned Old = LI->reg;
-    size_t Color = i;
+    size_t Color = I;
     const TargetRegisterClass *RC = MRI->getRegClass(Old);
 
     // Check if it's possible to reuse any of the used colors.
@@ -154,7 +153,7 @@ bool WebAssemblyRegColoring::runOnMachineFunction(MachineFunction &MF) {
       }
 
     unsigned New = SortedIntervals[Color]->reg;
-    SlotMapping[i] = New;
+    SlotMapping[I] = New;
     Changed |= Old != New;
     UsedColors.set(Color);
     Assignments[Color].push_back(LI);
@@ -166,9 +165,9 @@ bool WebAssemblyRegColoring::runOnMachineFunction(MachineFunction &MF) {
     return false;
 
   // Rewrite register operands.
-  for (size_t i = 0, e = SortedIntervals.size(); i < e; ++i) {
-    unsigned Old = SortedIntervals[i]->reg;
-    unsigned New = SlotMapping[i];
+  for (size_t I = 0, E = SortedIntervals.size(); I < E; ++I) {
+    unsigned Old = SortedIntervals[I]->reg;
+    unsigned New = SlotMapping[I];
     if (Old != New)
       MRI->replaceRegWith(Old, New);
   }
diff --git a/lib/Target/WebAssembly/WebAssemblyRegNumbering.cpp b/lib/Target/WebAssembly/WebAssemblyRegNumbering.cpp
index 1e2a248f097e9f83d9aa8f1fdcd25f0489ace15e..424b2ca2f841c68c7ce5b51e61b0e6c45756cea1 100644
--- a/lib/Target/WebAssembly/WebAssemblyRegNumbering.cpp
+++ b/lib/Target/WebAssembly/WebAssemblyRegNumbering.cpp
@@ -1,9 +1,8 @@
 //===-- WebAssemblyRegNumbering.cpp - Register Numbering ------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
diff --git a/lib/Target/WebAssembly/WebAssemblyRegStackify.cpp b/lib/Target/WebAssembly/WebAssemblyRegStackify.cpp
index dc2aab875932926e45dab49124fa2aa6022ed3c8..6463e268c905916dc83a5667aae0f9424909b226 100644
--- a/lib/Target/WebAssembly/WebAssemblyRegStackify.cpp
+++ b/lib/Target/WebAssembly/WebAssemblyRegStackify.cpp
@@ -1,9 +1,8 @@
 //===-- WebAssemblyRegStackify.cpp - Register Stackification --------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
@@ -22,6 +21,7 @@
 
 #include "MCTargetDesc/WebAssemblyMCTargetDesc.h" // for WebAssembly::ARGUMENT_*
 #include "WebAssembly.h"
+#include "WebAssemblyDebugValueManager.h"
 #include "WebAssemblyMachineFunctionInfo.h"
 #include "WebAssemblySubtarget.h"
 #include "WebAssemblyUtilities.h"
@@ -79,7 +79,7 @@ FunctionPass *llvm::createWebAssemblyRegStackify() {
 // Decorate the given instruction with implicit operands that enforce the
 // expression stack ordering constraints for an instruction which is on
 // the expression stack.
-static void ImposeStackOrdering(MachineInstr *MI) {
+static void imposeStackOrdering(MachineInstr *MI) {
   // Write the opaque VALUE_STACK register.
   if (!MI->definesRegister(WebAssembly::VALUE_STACK))
     MI->addOperand(MachineOperand::CreateReg(WebAssembly::VALUE_STACK,
@@ -95,10 +95,11 @@ static void ImposeStackOrdering(MachineInstr *MI) {
 
 // Convert an IMPLICIT_DEF instruction into an instruction which defines
 // a constant zero value.
-static void ConvertImplicitDefToConstZero(MachineInstr *MI,
+static void convertImplicitDefToConstZero(MachineInstr *MI,
                                           MachineRegisterInfo &MRI,
                                           const TargetInstrInfo *TII,
-                                          MachineFunction &MF) {
+                                          MachineFunction &MF,
+                                          LiveIntervals &LIS) {
   assert(MI->getOpcode() == TargetOpcode::IMPLICIT_DEF);
 
   const auto *RegClass = MRI.getRegClass(MI->getOperand(0).getReg());
@@ -110,19 +111,22 @@ static void ConvertImplicitDefToConstZero(MachineInstr *MI,
     MI->addOperand(MachineOperand::CreateImm(0));
   } else if (RegClass == &WebAssembly::F32RegClass) {
     MI->setDesc(TII->get(WebAssembly::CONST_F32));
-    ConstantFP *Val = cast<ConstantFP>(Constant::getNullValue(
+    auto *Val = cast<ConstantFP>(Constant::getNullValue(
         Type::getFloatTy(MF.getFunction().getContext())));
     MI->addOperand(MachineOperand::CreateFPImm(Val));
   } else if (RegClass == &WebAssembly::F64RegClass) {
     MI->setDesc(TII->get(WebAssembly::CONST_F64));
-    ConstantFP *Val = cast<ConstantFP>(Constant::getNullValue(
+    auto *Val = cast<ConstantFP>(Constant::getNullValue(
         Type::getDoubleTy(MF.getFunction().getContext())));
     MI->addOperand(MachineOperand::CreateFPImm(Val));
   } else if (RegClass == &WebAssembly::V128RegClass) {
-    // TODO: make splat instead of constant
-    MI->setDesc(TII->get(WebAssembly::CONST_V128_v16i8));
-    for (int I = 0; I < 16; ++I)
-      MI->addOperand(MachineOperand::CreateImm(0));
+    unsigned TempReg = MRI.createVirtualRegister(&WebAssembly::I32RegClass);
+    MI->setDesc(TII->get(WebAssembly::SPLAT_v4i32));
+    MI->addOperand(MachineOperand::CreateReg(TempReg, false));
+    MachineInstr *Const = BuildMI(*MI->getParent(), MI, MI->getDebugLoc(),
+                                  TII->get(WebAssembly::CONST_I32), TempReg)
+                              .addImm(0);
+    LIS.InsertMachineInstrInMaps(*Const);
   } else {
     llvm_unreachable("Unexpected reg class");
   }
@@ -131,7 +135,7 @@ static void ConvertImplicitDefToConstZero(MachineInstr *MI,
 // Determine whether a call to the callee referenced by
 // MI->getOperand(CalleeOpNo) reads memory, writes memory, and/or has side
 // effects.
-static void QueryCallee(const MachineInstr &MI, unsigned CalleeOpNo, bool &Read,
+static void queryCallee(const MachineInstr &MI, unsigned CalleeOpNo, bool &Read,
                         bool &Write, bool &Effects, bool &StackPointer) {
   // All calls can use the stack pointer.
   StackPointer = true;
@@ -139,11 +143,11 @@ static void QueryCallee(const MachineInstr &MI, unsigned CalleeOpNo, bool &Read,
   const MachineOperand &MO = MI.getOperand(CalleeOpNo);
   if (MO.isGlobal()) {
     const Constant *GV = MO.getGlobal();
-    if (const GlobalAlias *GA = dyn_cast<GlobalAlias>(GV))
+    if (const auto *GA = dyn_cast<GlobalAlias>(GV))
       if (!GA->isInterposable())
         GV = GA->getAliasee();
 
-    if (const Function *F = dyn_cast<Function>(GV)) {
+    if (const auto *F = dyn_cast<Function>(GV)) {
       if (!F->doesNotThrow())
         Effects = true;
       if (F->doesNotAccessMemory())
@@ -163,7 +167,7 @@ static void QueryCallee(const MachineInstr &MI, unsigned CalleeOpNo, bool &Read,
 
 // Determine whether MI reads memory, writes memory, has side effects,
 // and/or uses the stack pointer value.
-static void Query(const MachineInstr &MI, AliasAnalysis &AA, bool &Read,
+static void query(const MachineInstr &MI, AliasAnalysis &AA, bool &Read,
                   bool &Write, bool &Effects, bool &StackPointer) {
   assert(!MI.isTerminator());
 
@@ -177,19 +181,6 @@ static void Query(const MachineInstr &MI, AliasAnalysis &AA, bool &Read,
   // Check for stores.
   if (MI.mayStore()) {
     Write = true;
-
-    // Check for stores to __stack_pointer.
-    for (auto MMO : MI.memoperands()) {
-      const MachinePointerInfo &MPI = MMO->getPointerInfo();
-      if (MPI.V.is<const PseudoSourceValue *>()) {
-        auto PSV = MPI.V.get<const PseudoSourceValue *>();
-        if (const ExternalSymbolPseudoSourceValue *EPSV =
-                dyn_cast<ExternalSymbolPseudoSourceValue>(PSV))
-          if (StringRef(EPSV->getSymbol()) == "__stack_pointer") {
-            StackPointer = true;
-          }
-      }
-    }
   } else if (MI.hasOrderedMemoryRef()) {
     switch (MI.getOpcode()) {
     case WebAssembly::DIV_S_I32:
@@ -254,15 +245,20 @@ static void Query(const MachineInstr &MI, AliasAnalysis &AA, bool &Read,
     }
   }
 
+  // Check for writes to __stack_pointer global.
+  if (MI.getOpcode() == WebAssembly::GLOBAL_SET_I32 &&
+      strcmp(MI.getOperand(0).getSymbolName(), "__stack_pointer") == 0)
+    StackPointer = true;
+
   // Analyze calls.
   if (MI.isCall()) {
     unsigned CalleeOpNo = WebAssembly::getCalleeOpNo(MI);
-    QueryCallee(MI, CalleeOpNo, Read, Write, Effects, StackPointer);
+    queryCallee(MI, CalleeOpNo, Read, Write, Effects, StackPointer);
   }
 }
 
 // Test whether Def is safe and profitable to rematerialize.
-static bool ShouldRematerialize(const MachineInstr &Def, AliasAnalysis &AA,
+static bool shouldRematerialize(const MachineInstr &Def, AliasAnalysis &AA,
                                 const WebAssemblyInstrInfo *TII) {
   return Def.isAsCheapAsAMove() && TII->isTriviallyReMaterializable(Def, &AA);
 }
@@ -270,7 +266,7 @@ static bool ShouldRematerialize(const MachineInstr &Def, AliasAnalysis &AA,
 // Identify the definition for this register at this point. This is a
 // generalization of MachineRegisterInfo::getUniqueVRegDef that uses
 // LiveIntervals to handle complex cases.
-static MachineInstr *GetVRegDef(unsigned Reg, const MachineInstr *Insert,
+static MachineInstr *getVRegDef(unsigned Reg, const MachineInstr *Insert,
                                 const MachineRegisterInfo &MRI,
                                 const LiveIntervals &LIS) {
   // Most registers are in SSA form here so we try a quick MRI query first.
@@ -288,7 +284,7 @@ static MachineInstr *GetVRegDef(unsigned Reg, const MachineInstr *Insert,
 // Test whether Reg, as defined at Def, has exactly one use. This is a
 // generalization of MachineRegisterInfo::hasOneUse that uses LiveIntervals
 // to handle complex cases.
-static bool HasOneUse(unsigned Reg, MachineInstr *Def, MachineRegisterInfo &MRI,
+static bool hasOneUse(unsigned Reg, MachineInstr *Def, MachineRegisterInfo &MRI,
                       MachineDominatorTree &MDT, LiveIntervals &LIS) {
   // Most registers are in SSA form here so we try a quick MRI query first.
   if (MRI.hasOneUse(Reg))
@@ -317,10 +313,22 @@ static bool HasOneUse(unsigned Reg, MachineInstr *Def, MachineRegisterInfo &MRI,
 // walking the block.
 // TODO: Compute memory dependencies in a way that uses AliasAnalysis to be
 // more precise.
-static bool IsSafeToMove(const MachineInstr *Def, const MachineInstr *Insert,
+static bool isSafeToMove(const MachineInstr *Def, const MachineInstr *Insert,
                          AliasAnalysis &AA, const MachineRegisterInfo &MRI) {
   assert(Def->getParent() == Insert->getParent());
 
+  // 'catch' and 'extract_exception' should be the first instruction of a BB and
+  // cannot move.
+  if (Def->getOpcode() == WebAssembly::CATCH ||
+      Def->getOpcode() == WebAssembly::EXTRACT_EXCEPTION_I32) {
+    const MachineBasicBlock *MBB = Def->getParent();
+    auto NextI = std::next(MachineBasicBlock::const_iterator(Def));
+    for (auto E = MBB->end(); NextI != E && NextI->isDebugInstr(); ++NextI)
+      ;
+    if (NextI != Insert)
+      return false;
+  }
+
   // Check for register dependencies.
   SmallVector<unsigned, 4> MutableRegisters;
   for (const MachineOperand &MO : Def->operands()) {
@@ -353,7 +361,7 @@ static bool IsSafeToMove(const MachineInstr *Def, const MachineInstr *Insert,
   }
 
   bool Read = false, Write = false, Effects = false, StackPointer = false;
-  Query(*Def, AA, Read, Write, Effects, StackPointer);
+  query(*Def, AA, Read, Write, Effects, StackPointer);
 
   // If the instruction does not access memory and has no side effects, it has
   // no additional dependencies.
@@ -368,7 +376,7 @@ static bool IsSafeToMove(const MachineInstr *Def, const MachineInstr *Insert,
     bool InterveningWrite = false;
     bool InterveningEffects = false;
     bool InterveningStackPointer = false;
-    Query(*I, AA, InterveningRead, InterveningWrite, InterveningEffects,
+    query(*I, AA, InterveningRead, InterveningWrite, InterveningEffects,
           InterveningStackPointer);
     if (Effects && InterveningEffects)
       return false;
@@ -389,7 +397,7 @@ static bool IsSafeToMove(const MachineInstr *Def, const MachineInstr *Insert,
 }
 
 /// Test whether OneUse, a use of Reg, dominates all of Reg's other uses.
-static bool OneUseDominatesOtherUses(unsigned Reg, const MachineOperand &OneUse,
+static bool oneUseDominatesOtherUses(unsigned Reg, const MachineOperand &OneUse,
                                      const MachineBasicBlock &MBB,
                                      const MachineRegisterInfo &MRI,
                                      const MachineDominatorTree &MDT,
@@ -410,7 +418,6 @@ static bool OneUseDominatesOtherUses(unsigned Reg, const MachineOperand &OneUse,
     if (UseVNI != OneUseVNI)
       continue;
 
-    const MachineInstr *OneUseInst = OneUse.getParent();
     if (UseInst == OneUseInst) {
       // Another use in the same instruction. We need to ensure that the one
       // selected use happens "before" it.
@@ -422,8 +429,8 @@ static bool OneUseDominatesOtherUses(unsigned Reg, const MachineOperand &OneUse,
         // Actually, dominating is over-conservative. Test that the use would
         // happen after the one selected use in the stack evaluation order.
         //
-        // This is needed as a consequence of using implicit get_locals for
-        // uses and implicit set_locals for defs.
+        // This is needed as a consequence of using implicit local.gets for
+        // uses and implicit local.sets for defs.
         if (UseInst->getDesc().getNumDefs() == 0)
           return false;
         const MachineOperand &MO = UseInst->getOperand(0);
@@ -449,7 +456,7 @@ static bool OneUseDominatesOtherUses(unsigned Reg, const MachineOperand &OneUse,
 }
 
 /// Get the appropriate tee opcode for the given register class.
-static unsigned GetTeeOpcode(const TargetRegisterClass *RC) {
+static unsigned getTeeOpcode(const TargetRegisterClass *RC) {
   if (RC == &WebAssembly::I32RegClass)
     return WebAssembly::TEE_I32;
   if (RC == &WebAssembly::I64RegClass)
@@ -464,45 +471,25 @@ static unsigned GetTeeOpcode(const TargetRegisterClass *RC) {
 }
 
 // Shrink LI to its uses, cleaning up LI.
-static void ShrinkToUses(LiveInterval &LI, LiveIntervals &LIS) {
+static void shrinkToUses(LiveInterval &LI, LiveIntervals &LIS) {
   if (LIS.shrinkToUses(&LI)) {
     SmallVector<LiveInterval *, 4> SplitLIs;
     LIS.splitSeparateComponents(LI, SplitLIs);
   }
 }
 
-static void MoveDebugValues(unsigned Reg, MachineInstr *Insert,
-                            MachineBasicBlock &MBB, MachineRegisterInfo &MRI) {
-  for (auto &Op : MRI.reg_operands(Reg)) {
-    MachineInstr *MI = Op.getParent();
-    assert(MI != nullptr);
-    if (MI->isDebugValue() && MI->getParent() == &MBB)
-      MBB.splice(Insert, &MBB, MI);
-  }
-}
-
-static void UpdateDebugValuesReg(unsigned Reg, unsigned NewReg,
-                                 MachineBasicBlock &MBB,
-                                 MachineRegisterInfo &MRI) {
-  for (auto &Op : MRI.reg_operands(Reg)) {
-    MachineInstr *MI = Op.getParent();
-    assert(MI != nullptr);
-    if (MI->isDebugValue() && MI->getParent() == &MBB)
-      Op.setReg(NewReg);
-  }
-}
-
 /// A single-use def in the same block with no intervening memory or register
 /// dependencies; move the def down and nest it with the current instruction.
-static MachineInstr *MoveForSingleUse(unsigned Reg, MachineOperand &Op,
+static MachineInstr *moveForSingleUse(unsigned Reg, MachineOperand &Op,
                                       MachineInstr *Def, MachineBasicBlock &MBB,
                                       MachineInstr *Insert, LiveIntervals &LIS,
                                       WebAssemblyFunctionInfo &MFI,
                                       MachineRegisterInfo &MRI) {
   LLVM_DEBUG(dbgs() << "Move for single use: "; Def->dump());
 
+  WebAssemblyDebugValueManager DefDIs(Def);
   MBB.splice(Insert, &MBB, Def);
-  MoveDebugValues(Reg, Insert, MBB, MRI);
+  DefDIs.move(Insert);
   LIS.handleMove(*Def);
 
   if (MRI.hasOneDef(Reg) && MRI.hasOneUse(Reg)) {
@@ -527,41 +514,18 @@ static MachineInstr *MoveForSingleUse(unsigned Reg, MachineOperand &Op,
 
     MFI.stackifyVReg(NewReg);
 
-    UpdateDebugValuesReg(Reg, NewReg, MBB, MRI);
+    DefDIs.updateReg(NewReg);
 
     LLVM_DEBUG(dbgs() << " - Replaced register: "; Def->dump());
   }
 
-  ImposeStackOrdering(Def);
+  imposeStackOrdering(Def);
   return Def;
 }
 
-static void CloneDebugValues(unsigned Reg, MachineInstr *Insert,
-                             unsigned TargetReg, MachineBasicBlock &MBB,
-                             MachineRegisterInfo &MRI,
-                             const WebAssemblyInstrInfo *TII) {
-  SmallPtrSet<MachineInstr *, 4> Instrs;
-  for (auto &Op : MRI.reg_operands(Reg)) {
-    MachineInstr *MI = Op.getParent();
-    assert(MI != nullptr);
-    if (MI->isDebugValue() && MI->getParent() == &MBB &&
-        Instrs.find(MI) == Instrs.end())
-      Instrs.insert(MI);
-  }
-  for (const auto &MI : Instrs) {
-    MachineInstr &Clone = TII->duplicate(MBB, Insert, *MI);
-    for (unsigned i = 0, e = Clone.getNumOperands(); i != e; ++i) {
-      MachineOperand &MO = Clone.getOperand(i);
-      if (MO.isReg() && MO.getReg() == Reg)
-        MO.setReg(TargetReg);
-    }
-    LLVM_DEBUG(dbgs() << " - - Cloned DBG_VALUE: "; Clone.dump());
-  }
-}
-
 /// A trivially cloneable instruction; clone it and nest the new copy with the
 /// current instruction.
-static MachineInstr *RematerializeCheapDef(
+static MachineInstr *rematerializeCheapDef(
     unsigned Reg, MachineOperand &Op, MachineInstr &Def, MachineBasicBlock &MBB,
     MachineBasicBlock::instr_iterator Insert, LiveIntervals &LIS,
     WebAssemblyFunctionInfo &MFI, MachineRegisterInfo &MRI,
@@ -569,6 +533,8 @@ static MachineInstr *RematerializeCheapDef(
   LLVM_DEBUG(dbgs() << "Rematerializing cheap def: "; Def.dump());
   LLVM_DEBUG(dbgs() << " - for use in "; Op.getParent()->dump());
 
+  WebAssemblyDebugValueManager DefDIs(&Def);
+
   unsigned NewReg = MRI.createVirtualRegister(MRI.getRegClass(Reg));
   TII->reMaterialize(MBB, Insert, NewReg, 0, Def, *TRI);
   Op.setReg(NewReg);
@@ -576,7 +542,7 @@ static MachineInstr *RematerializeCheapDef(
   LIS.InsertMachineInstrInMaps(*Clone);
   LIS.createAndComputeVirtRegInterval(NewReg);
   MFI.stackifyVReg(NewReg);
-  ImposeStackOrdering(Clone);
+  imposeStackOrdering(Clone);
 
   LLVM_DEBUG(dbgs() << " - Cloned to "; Clone->dump());
 
@@ -584,7 +550,7 @@ static MachineInstr *RematerializeCheapDef(
   bool IsDead = MRI.use_empty(Reg);
   if (!IsDead) {
     LiveInterval &LI = LIS.getInterval(Reg);
-    ShrinkToUses(LI, LIS);
+    shrinkToUses(LI, LIS);
     IsDead = !LI.liveAt(LIS.getInstructionIndex(Def).getDeadSlot());
   }
 
@@ -598,10 +564,10 @@ static MachineInstr *RematerializeCheapDef(
     LIS.RemoveMachineInstrFromMaps(Def);
     Def.eraseFromParent();
 
-    MoveDebugValues(Reg, &*Insert, MBB, MRI);
-    UpdateDebugValuesReg(Reg, NewReg, MBB, MRI);
+    DefDIs.move(&*Insert);
+    DefDIs.updateReg(NewReg);
   } else {
-    CloneDebugValues(Reg, &*Insert, NewReg, MBB, MRI, TII);
+    DefDIs.clone(&*Insert, NewReg);
   }
 
   return Clone;
@@ -625,14 +591,16 @@ static MachineInstr *RematerializeCheapDef(
 ///    INST ..., Reg, ...
 ///    INST ..., Reg, ...
 ///
-/// with DefReg and TeeReg stackified. This eliminates a get_local from the
+/// with DefReg and TeeReg stackified. This eliminates a local.get from the
 /// resulting code.
-static MachineInstr *MoveAndTeeForMultiUse(
+static MachineInstr *moveAndTeeForMultiUse(
     unsigned Reg, MachineOperand &Op, MachineInstr *Def, MachineBasicBlock &MBB,
     MachineInstr *Insert, LiveIntervals &LIS, WebAssemblyFunctionInfo &MFI,
     MachineRegisterInfo &MRI, const WebAssemblyInstrInfo *TII) {
   LLVM_DEBUG(dbgs() << "Move and tee for multi-use:"; Def->dump());
 
+  WebAssemblyDebugValueManager DefDIs(Def);
+
   // Move Def into place.
   MBB.splice(Insert, &MBB, Def);
   LIS.handleMove(*Def);
@@ -643,7 +611,7 @@ static MachineInstr *MoveAndTeeForMultiUse(
   unsigned DefReg = MRI.createVirtualRegister(RegClass);
   MachineOperand &DefMO = Def->getOperand(0);
   MachineInstr *Tee = BuildMI(MBB, Insert, Insert->getDebugLoc(),
-                              TII->get(GetTeeOpcode(RegClass)), TeeReg)
+                              TII->get(getTeeOpcode(RegClass)), TeeReg)
                           .addReg(Reg, RegState::Define)
                           .addReg(DefReg, getUndefRegState(DefMO.isDead()));
   Op.setReg(TeeReg);
@@ -651,7 +619,7 @@ static MachineInstr *MoveAndTeeForMultiUse(
   SlotIndex TeeIdx = LIS.InsertMachineInstrInMaps(*Tee).getRegSlot();
   SlotIndex DefIdx = LIS.getInstructionIndex(*Def).getRegSlot();
 
-  MoveDebugValues(Reg, Insert, MBB, MRI);
+  DefDIs.move(Insert);
 
   // Tell LiveIntervals we moved the original vreg def from Def to Tee.
   LiveInterval &LI = LIS.getInterval(Reg);
@@ -659,18 +627,18 @@ static MachineInstr *MoveAndTeeForMultiUse(
   VNInfo *ValNo = LI.getVNInfoAt(DefIdx);
   I->start = TeeIdx;
   ValNo->def = TeeIdx;
-  ShrinkToUses(LI, LIS);
+  shrinkToUses(LI, LIS);
 
   // Finish stackifying the new regs.
   LIS.createAndComputeVirtRegInterval(TeeReg);
   LIS.createAndComputeVirtRegInterval(DefReg);
   MFI.stackifyVReg(DefReg);
   MFI.stackifyVReg(TeeReg);
-  ImposeStackOrdering(Def);
-  ImposeStackOrdering(Tee);
+  imposeStackOrdering(Def);
+  imposeStackOrdering(Tee);
 
-  CloneDebugValues(Reg, Tee, DefReg, MBB, MRI, TII);
-  CloneDebugValues(Reg, Insert, TeeReg, MBB, MRI, TII);
+  DefDIs.clone(Tee, DefReg);
+  DefDIs.clone(Insert, TeeReg);
 
   LLVM_DEBUG(dbgs() << " - Replaced register: "; Def->dump());
   LLVM_DEBUG(dbgs() << " - Tee instruction: "; Tee->dump());
@@ -681,9 +649,9 @@ namespace {
 /// A stack for walking the tree of instructions being built, visiting the
 /// MachineOperands in DFS order.
 class TreeWalkerState {
-  typedef MachineInstr::mop_iterator mop_iterator;
-  typedef std::reverse_iterator<mop_iterator> mop_reverse_iterator;
-  typedef iterator_range<mop_reverse_iterator> RangeTy;
+  using mop_iterator = MachineInstr::mop_iterator;
+  using mop_reverse_iterator = std::reverse_iterator<mop_iterator>;
+  using RangeTy = iterator_range<mop_reverse_iterator>;
   SmallVector<RangeTy, 4> Worklist;
 
 public:
@@ -693,9 +661,9 @@ public:
       Worklist.push_back(reverse(Range));
   }
 
-  bool Done() const { return Worklist.empty(); }
+  bool done() const { return Worklist.empty(); }
 
-  MachineOperand &Pop() {
+  MachineOperand &pop() {
     RangeTy &Range = Worklist.back();
     MachineOperand &Op = *Range.begin();
     Range = drop_begin(Range, 1);
@@ -708,7 +676,7 @@ public:
   }
 
   /// Push Instr's operands onto the stack to be visited.
-  void PushOperands(MachineInstr *Instr) {
+  void pushOperands(MachineInstr *Instr) {
     const iterator_range<mop_iterator> &Range(Instr->explicit_uses());
     if (Range.begin() != Range.end())
       Worklist.push_back(reverse(Range));
@@ -716,8 +684,8 @@ public:
 
   /// Some of Instr's operands are on the top of the stack; remove them and
   /// re-insert them starting from the beginning (because we've commuted them).
-  void ResetTopOperands(MachineInstr *Instr) {
-    assert(HasRemainingOperands(Instr) &&
+  void resetTopOperands(MachineInstr *Instr) {
+    assert(hasRemainingOperands(Instr) &&
            "Reseting operands should only be done when the instruction has "
            "an operand still on the stack");
     Worklist.back() = reverse(Instr->explicit_uses());
@@ -725,7 +693,7 @@ public:
 
   /// Test whether Instr has operands remaining to be visited at the top of
   /// the stack.
-  bool HasRemainingOperands(const MachineInstr *Instr) const {
+  bool hasRemainingOperands(const MachineInstr *Instr) const {
     if (Worklist.empty())
       return false;
     const RangeTy &Range = Worklist.back();
@@ -736,9 +704,9 @@ public:
   /// operand in the tree that we haven't visited yet. Moving a definition of
   /// Reg to a point in the tree after that would change its value.
   ///
-  /// This is needed as a consequence of using implicit get_locals for
-  /// uses and implicit set_locals for defs.
-  bool IsOnStack(unsigned Reg) const {
+  /// This is needed as a consequence of using implicit local.gets for
+  /// uses and implicit local.sets for defs.
+  bool isOnStack(unsigned Reg) const {
     for (const RangeTy &Range : Worklist)
       for (const MachineOperand &MO : Range)
         if (MO.isReg() && MO.getReg() == Reg)
@@ -751,24 +719,22 @@ public:
 /// tried for the current instruction and didn't work.
 class CommutingState {
   /// There are effectively three states: the initial state where we haven't
-  /// started commuting anything and we don't know anything yet, the tenative
+  /// started commuting anything and we don't know anything yet, the tentative
   /// state where we've commuted the operands of the current instruction and are
-  /// revisting it, and the declined state where we've reverted the operands
+  /// revisiting it, and the declined state where we've reverted the operands
   /// back to their original order and will no longer commute it further.
-  bool TentativelyCommuting;
-  bool Declined;
+  bool TentativelyCommuting = false;
+  bool Declined = false;
 
   /// During the tentative state, these hold the operand indices of the commuted
   /// operands.
   unsigned Operand0, Operand1;
 
 public:
-  CommutingState() : TentativelyCommuting(false), Declined(false) {}
-
   /// Stackification for an operand was not successful due to ordering
   /// constraints. If possible, and if we haven't already tried it and declined
   /// it, commute Insert's operands and prepare to revisit it.
-  void MaybeCommute(MachineInstr *Insert, TreeWalkerState &TreeWalker,
+  void maybeCommute(MachineInstr *Insert, TreeWalkerState &TreeWalker,
                     const WebAssemblyInstrInfo *TII) {
     if (TentativelyCommuting) {
       assert(!Declined &&
@@ -777,13 +743,13 @@ public:
       TII->commuteInstruction(*Insert, /*NewMI=*/false, Operand0, Operand1);
       TentativelyCommuting = false;
       Declined = true;
-    } else if (!Declined && TreeWalker.HasRemainingOperands(Insert)) {
+    } else if (!Declined && TreeWalker.hasRemainingOperands(Insert)) {
       Operand0 = TargetInstrInfo::CommuteAnyOperandIndex;
       Operand1 = TargetInstrInfo::CommuteAnyOperandIndex;
       if (TII->findCommutedOpIndices(*Insert, Operand0, Operand1)) {
         // Tentatively commute the operands and try again.
         TII->commuteInstruction(*Insert, /*NewMI=*/false, Operand0, Operand1);
-        TreeWalker.ResetTopOperands(Insert);
+        TreeWalker.resetTopOperands(Insert);
         TentativelyCommuting = true;
         Declined = false;
       }
@@ -792,7 +758,7 @@ public:
 
   /// Stackification for some operand was successful. Reset to the default
   /// state.
-  void Reset() {
+  void reset() {
     TentativelyCommuting = false;
     Declined = false;
   }
@@ -810,8 +776,8 @@ bool WebAssemblyRegStackify::runOnMachineFunction(MachineFunction &MF) {
   const auto *TII = MF.getSubtarget<WebAssemblySubtarget>().getInstrInfo();
   const auto *TRI = MF.getSubtarget<WebAssemblySubtarget>().getRegisterInfo();
   AliasAnalysis &AA = getAnalysis<AAResultsWrapperPass>().getAAResults();
-  MachineDominatorTree &MDT = getAnalysis<MachineDominatorTree>();
-  LiveIntervals &LIS = getAnalysis<LiveIntervals>();
+  auto &MDT = getAnalysis<MachineDominatorTree>();
+  auto &LIS = getAnalysis<LiveIntervals>();
 
   // Walk the instructions from the bottom up. Currently we don't look past
   // block boundaries, and the blocks aren't ordered so the block visitation
@@ -823,19 +789,19 @@ bool WebAssemblyRegStackify::runOnMachineFunction(MachineFunction &MF) {
       MachineInstr *Insert = &*MII;
       // Don't nest anything inside an inline asm, because we don't have
       // constraints for $push inputs.
-      if (Insert->getOpcode() == TargetOpcode::INLINEASM)
+      if (Insert->isInlineAsm())
         continue;
 
       // Ignore debugging intrinsics.
-      if (Insert->getOpcode() == TargetOpcode::DBG_VALUE)
+      if (Insert->isDebugValue())
         continue;
 
       // Iterate through the inputs in reverse order, since we'll be pulling
       // operands off the stack in LIFO order.
       CommutingState Commuting;
       TreeWalkerState TreeWalker(Insert);
-      while (!TreeWalker.Done()) {
-        MachineOperand &Op = TreeWalker.Pop();
+      while (!TreeWalker.done()) {
+        MachineOperand &Op = TreeWalker.pop();
 
         // We're only interested in explicit virtual register operands.
         if (!Op.isReg())
@@ -849,13 +815,13 @@ bool WebAssemblyRegStackify::runOnMachineFunction(MachineFunction &MF) {
           continue;
 
         // Identify the definition for this register at this point.
-        MachineInstr *Def = GetVRegDef(Reg, Insert, MRI, LIS);
+        MachineInstr *Def = getVRegDef(Reg, Insert, MRI, LIS);
         if (!Def)
           continue;
 
         // Don't nest an INLINE_ASM def into anything, because we don't have
         // constraints for $pop outputs.
-        if (Def->getOpcode() == TargetOpcode::INLINEASM)
+        if (Def->isInlineAsm())
           continue;
 
         // Argument instructions represent live-in registers and not real
@@ -863,6 +829,24 @@ bool WebAssemblyRegStackify::runOnMachineFunction(MachineFunction &MF) {
         if (WebAssembly::isArgument(*Def))
           continue;
 
+        // Currently catch's return value register cannot be stackified, because
+        // the wasm LLVM backend currently does not support live-in values
+        // entering blocks, which is a part of multi-value proposal.
+        //
+        // Once we support live-in values of wasm blocks, this can be:
+        // catch                           ; push except_ref value onto stack
+        // block except_ref -> i32
+        // br_on_exn $__cpp_exception      ; pop the except_ref value
+        // end_block
+        //
+        // But because we don't support it yet, the catch instruction's dst
+        // register should be assigned to a local to be propagated across
+        // 'block' boundary now.
+        //
+        // TODO Fix this once we support the multi-value proposal.
+        if (Def->getOpcode() == WebAssembly::CATCH)
+          continue;
+
         // Decide which strategy to take. Prefer to move a single-use value
         // over cloning it, and prefer cloning over introducing a tee.
         // For moving, we require the def to be in the same block as the use;
@@ -870,23 +854,23 @@ bool WebAssemblyRegStackify::runOnMachineFunction(MachineFunction &MF) {
         // supports intra-block moves) and it's MachineSink's job to catch all
         // the sinking opportunities anyway.
         bool SameBlock = Def->getParent() == &MBB;
-        bool CanMove = SameBlock && IsSafeToMove(Def, Insert, AA, MRI) &&
-                       !TreeWalker.IsOnStack(Reg);
-        if (CanMove && HasOneUse(Reg, Def, MRI, MDT, LIS)) {
-          Insert = MoveForSingleUse(Reg, Op, Def, MBB, Insert, LIS, MFI, MRI);
-        } else if (ShouldRematerialize(*Def, AA, TII)) {
+        bool CanMove = SameBlock && isSafeToMove(Def, Insert, AA, MRI) &&
+                       !TreeWalker.isOnStack(Reg);
+        if (CanMove && hasOneUse(Reg, Def, MRI, MDT, LIS)) {
+          Insert = moveForSingleUse(Reg, Op, Def, MBB, Insert, LIS, MFI, MRI);
+        } else if (shouldRematerialize(*Def, AA, TII)) {
           Insert =
-              RematerializeCheapDef(Reg, Op, *Def, MBB, Insert->getIterator(),
+              rematerializeCheapDef(Reg, Op, *Def, MBB, Insert->getIterator(),
                                     LIS, MFI, MRI, TII, TRI);
         } else if (CanMove &&
-                   OneUseDominatesOtherUses(Reg, Op, MBB, MRI, MDT, LIS, MFI)) {
-          Insert = MoveAndTeeForMultiUse(Reg, Op, Def, MBB, Insert, LIS, MFI,
+                   oneUseDominatesOtherUses(Reg, Op, MBB, MRI, MDT, LIS, MFI)) {
+          Insert = moveAndTeeForMultiUse(Reg, Op, Def, MBB, Insert, LIS, MFI,
                                          MRI, TII);
         } else {
           // We failed to stackify the operand. If the problem was ordering
           // constraints, Commuting may be able to help.
           if (!CanMove && SameBlock)
-            Commuting.MaybeCommute(Insert, TreeWalker, TII);
+            Commuting.maybeCommute(Insert, TreeWalker, TII);
           // Proceed to the next operand.
           continue;
         }
@@ -895,18 +879,18 @@ bool WebAssemblyRegStackify::runOnMachineFunction(MachineFunction &MF) {
         // to a constant 0 so that the def is explicit, and the push/pop
         // correspondence is maintained.
         if (Insert->getOpcode() == TargetOpcode::IMPLICIT_DEF)
-          ConvertImplicitDefToConstZero(Insert, MRI, TII, MF);
+          convertImplicitDefToConstZero(Insert, MRI, TII, MF, LIS);
 
         // We stackified an operand. Add the defining instruction's operands to
         // the worklist stack now to continue to build an ever deeper tree.
-        Commuting.Reset();
-        TreeWalker.PushOperands(Insert);
+        Commuting.reset();
+        TreeWalker.pushOperands(Insert);
       }
 
       // If we stackified any operands, skip over the tree to start looking for
       // the next instruction we can build a tree on.
       if (Insert != &*MII) {
-        ImposeStackOrdering(&*MII);
+        imposeStackOrdering(&*MII);
         MII = MachineBasicBlock::iterator(Insert).getReverse();
         Changed = true;
       }
diff --git a/lib/Target/WebAssembly/WebAssemblyRegisterInfo.cpp b/lib/Target/WebAssembly/WebAssemblyRegisterInfo.cpp
index 1f0870865b06c003ecdb7be628c8e16001ac22b6..ae808dd6597de91fe32b092013da95d0d208441a 100644
--- a/lib/Target/WebAssembly/WebAssemblyRegisterInfo.cpp
+++ b/lib/Target/WebAssembly/WebAssemblyRegisterInfo.cpp
@@ -1,9 +1,8 @@
 //===-- WebAssemblyRegisterInfo.cpp - WebAssembly Register Information ----===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
diff --git a/lib/Target/WebAssembly/WebAssemblyRegisterInfo.h b/lib/Target/WebAssembly/WebAssemblyRegisterInfo.h
index 2a73dfd4b065dd4fce4b40fd889e8edcbca825cf..b8d8ac923049b7bbe2fed2527b517432f944c275 100644
--- a/lib/Target/WebAssembly/WebAssemblyRegisterInfo.h
+++ b/lib/Target/WebAssembly/WebAssemblyRegisterInfo.h
@@ -1,9 +1,8 @@
 // WebAssemblyRegisterInfo.h - WebAssembly Register Information Impl -*- C++ -*-
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
diff --git a/lib/Target/WebAssembly/WebAssemblyRegisterInfo.td b/lib/Target/WebAssembly/WebAssemblyRegisterInfo.td
index a7c3d177724d3b16499051c13fb82aa070f9ce96..4d202f70caad7f55d499b3058cdb23d1da69b08a 100644
--- a/lib/Target/WebAssembly/WebAssemblyRegisterInfo.td
+++ b/lib/Target/WebAssembly/WebAssemblyRegisterInfo.td
@@ -1,9 +1,8 @@
 //WebAssemblyRegisterInfo.td-Describe the WebAssembly Registers -*- tablegen -*-
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
diff --git a/lib/Target/WebAssembly/WebAssemblyReplacePhysRegs.cpp b/lib/Target/WebAssembly/WebAssemblyReplacePhysRegs.cpp
index e5a3e47a3bcd0859b860bb808cc8a1634b92d79b..5eafd6c54e78248fc58c748f3081835e229cb1a0 100644
--- a/lib/Target/WebAssembly/WebAssemblyReplacePhysRegs.cpp
+++ b/lib/Target/WebAssembly/WebAssemblyReplacePhysRegs.cpp
@@ -1,9 +1,8 @@
 //===-- WebAssemblyReplacePhysRegs.cpp - Replace phys regs with virt regs -===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
diff --git a/lib/Target/WebAssembly/WebAssemblyRuntimeLibcallSignatures.cpp b/lib/Target/WebAssembly/WebAssemblyRuntimeLibcallSignatures.cpp
index 6cf81a9d77b31909c4ae39feb25dd9fa3b720ad4..fbdf9c576a9d3d081cac107504cc94bfe8276c49 100644
--- a/lib/Target/WebAssembly/WebAssemblyRuntimeLibcallSignatures.cpp
+++ b/lib/Target/WebAssembly/WebAssemblyRuntimeLibcallSignatures.cpp
@@ -1,9 +1,8 @@
 // CodeGen/RuntimeLibcallSignatures.cpp - R.T. Lib. Call Signatures -*- C++ -*--
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
@@ -485,14 +484,14 @@ struct StaticLibcallNameMap {
 
 } // end anonymous namespace
 
-void llvm::GetLibcallSignature(const WebAssemblySubtarget &Subtarget,
+void llvm::getLibcallSignature(const WebAssemblySubtarget &Subtarget,
                                RTLIB::Libcall LC,
                                SmallVectorImpl<wasm::ValType> &Rets,
                                SmallVectorImpl<wasm::ValType> &Params) {
   assert(Rets.empty());
   assert(Params.empty());
 
-  wasm::ValType iPTR =
+  wasm::ValType PtrTy =
       Subtarget.hasAddr64() ? wasm::ValType::I64 : wasm::ValType::I32;
 
   auto &Table = RuntimeLibcallSignatures->Table;
@@ -600,13 +599,13 @@ void llvm::GetLibcallSignature(const WebAssemblySubtarget &Subtarget,
     break;
   case func_f32_iPTR_iPTR:
     Params.push_back(wasm::ValType::F32);
-    Params.push_back(iPTR);
-    Params.push_back(iPTR);
+    Params.push_back(PtrTy);
+    Params.push_back(PtrTy);
     break;
   case func_f64_iPTR_iPTR:
     Params.push_back(wasm::ValType::F64);
-    Params.push_back(iPTR);
-    Params.push_back(iPTR);
+    Params.push_back(PtrTy);
+    Params.push_back(PtrTy);
     break;
   case i16_func_i16_i16:
     Rets.push_back(wasm::ValType::I32);
@@ -632,7 +631,7 @@ void llvm::GetLibcallSignature(const WebAssemblySubtarget &Subtarget,
     Rets.push_back(wasm::ValType::I32);
     Params.push_back(wasm::ValType::I32);
     Params.push_back(wasm::ValType::I32);
-    Params.push_back(iPTR);
+    Params.push_back(PtrTy);
     break;
   case i64_func_i64_i64:
     Rets.push_back(wasm::ValType::I64);
@@ -643,14 +642,14 @@ void llvm::GetLibcallSignature(const WebAssemblySubtarget &Subtarget,
     Rets.push_back(wasm::ValType::I64);
     Params.push_back(wasm::ValType::I64);
     Params.push_back(wasm::ValType::I64);
-    Params.push_back(iPTR);
+    Params.push_back(PtrTy);
     break;
   case i64_i64_func_f32:
 #if 0 // TODO: Enable this when wasm gets multiple-return-value support.
     Rets.push_back(wasm::ValType::I64);
     Rets.push_back(wasm::ValType::I64);
 #else
-    Params.push_back(iPTR);
+    Params.push_back(PtrTy);
 #endif
     Params.push_back(wasm::ValType::F32);
     break;
@@ -659,7 +658,7 @@ void llvm::GetLibcallSignature(const WebAssemblySubtarget &Subtarget,
     Rets.push_back(wasm::ValType::I64);
     Rets.push_back(wasm::ValType::I64);
 #else
-    Params.push_back(iPTR);
+    Params.push_back(PtrTy);
 #endif
     Params.push_back(wasm::ValType::F64);
     break;
@@ -668,7 +667,7 @@ void llvm::GetLibcallSignature(const WebAssemblySubtarget &Subtarget,
     Rets.push_back(wasm::ValType::I32);
     Rets.push_back(wasm::ValType::I32);
 #else
-    Params.push_back(iPTR);
+    Params.push_back(PtrTy);
 #endif
     Params.push_back(wasm::ValType::I32);
     Params.push_back(wasm::ValType::I32);
@@ -678,7 +677,7 @@ void llvm::GetLibcallSignature(const WebAssemblySubtarget &Subtarget,
     Rets.push_back(wasm::ValType::I32);
     Rets.push_back(wasm::ValType::I32);
 #else
-    Params.push_back(iPTR);
+    Params.push_back(PtrTy);
 #endif
     Params.push_back(wasm::ValType::I32);
     Params.push_back(wasm::ValType::I32);
@@ -688,7 +687,7 @@ void llvm::GetLibcallSignature(const WebAssemblySubtarget &Subtarget,
     Rets.push_back(wasm::ValType::I64);
     Rets.push_back(wasm::ValType::I64);
 #else
-    Params.push_back(iPTR);
+    Params.push_back(PtrTy);
 #endif
     Params.push_back(wasm::ValType::I64);
     Params.push_back(wasm::ValType::I64);
@@ -698,7 +697,7 @@ void llvm::GetLibcallSignature(const WebAssemblySubtarget &Subtarget,
     Rets.push_back(wasm::ValType::I64);
     Rets.push_back(wasm::ValType::I64);
 #else
-    Params.push_back(iPTR);
+    Params.push_back(PtrTy);
 #endif
     Params.push_back(wasm::ValType::I64);
     Params.push_back(wasm::ValType::I64);
@@ -710,13 +709,13 @@ void llvm::GetLibcallSignature(const WebAssemblySubtarget &Subtarget,
     Rets.push_back(wasm::ValType::I64);
     Rets.push_back(wasm::ValType::I64);
 #else
-    Params.push_back(iPTR);
+    Params.push_back(PtrTy);
 #endif
     Params.push_back(wasm::ValType::I64);
     Params.push_back(wasm::ValType::I64);
     Params.push_back(wasm::ValType::I64);
     Params.push_back(wasm::ValType::I64);
-    Params.push_back(iPTR);
+    Params.push_back(PtrTy);
     break;
   case i64_i64_i64_i64_func_i64_i64_i64_i64:
 #if 0 // TODO: Enable this when wasm gets multiple-return-value support.
@@ -725,7 +724,7 @@ void llvm::GetLibcallSignature(const WebAssemblySubtarget &Subtarget,
     Rets.push_back(wasm::ValType::I64);
     Rets.push_back(wasm::ValType::I64);
 #else
-    Params.push_back(iPTR);
+    Params.push_back(PtrTy);
 #endif
     Params.push_back(wasm::ValType::I64);
     Params.push_back(wasm::ValType::I64);
@@ -739,23 +738,23 @@ void llvm::GetLibcallSignature(const WebAssemblySubtarget &Subtarget,
     Rets.push_back(wasm::ValType::I64);
     Rets.push_back(wasm::ValType::I64);
 #else
-    Params.push_back(iPTR);
+    Params.push_back(PtrTy);
 #endif
     Params.push_back(wasm::ValType::I64);
     Params.push_back(wasm::ValType::I64);
     Params.push_back(wasm::ValType::I32);
     break;
   case iPTR_func_iPTR_i32_iPTR:
-    Rets.push_back(iPTR);
-    Params.push_back(iPTR);
+    Rets.push_back(PtrTy);
+    Params.push_back(PtrTy);
     Params.push_back(wasm::ValType::I32);
-    Params.push_back(iPTR);
+    Params.push_back(PtrTy);
     break;
   case iPTR_func_iPTR_iPTR_iPTR:
-    Rets.push_back(iPTR);
-    Params.push_back(iPTR);
-    Params.push_back(iPTR);
-    Params.push_back(iPTR);
+    Rets.push_back(PtrTy);
+    Params.push_back(PtrTy);
+    Params.push_back(PtrTy);
+    Params.push_back(PtrTy);
     break;
   case f32_func_f32_f32_f32:
     Rets.push_back(wasm::ValType::F32);
@@ -772,39 +771,39 @@ void llvm::GetLibcallSignature(const WebAssemblySubtarget &Subtarget,
   case func_i64_i64_iPTR_iPTR:
     Params.push_back(wasm::ValType::I64);
     Params.push_back(wasm::ValType::I64);
-    Params.push_back(iPTR);
-    Params.push_back(iPTR);
+    Params.push_back(PtrTy);
+    Params.push_back(PtrTy);
     break;
   case func_iPTR_f32:
-    Params.push_back(iPTR);
+    Params.push_back(PtrTy);
     Params.push_back(wasm::ValType::F32);
     break;
   case func_iPTR_f64:
-    Params.push_back(iPTR);
+    Params.push_back(PtrTy);
     Params.push_back(wasm::ValType::F64);
     break;
   case func_iPTR_i32:
-    Params.push_back(iPTR);
+    Params.push_back(PtrTy);
     Params.push_back(wasm::ValType::I32);
     break;
   case func_iPTR_i64:
-    Params.push_back(iPTR);
+    Params.push_back(PtrTy);
     Params.push_back(wasm::ValType::I64);
     break;
   case func_iPTR_i64_i64:
-    Params.push_back(iPTR);
+    Params.push_back(PtrTy);
     Params.push_back(wasm::ValType::I64);
     Params.push_back(wasm::ValType::I64);
     break;
   case func_iPTR_i64_i64_i64_i64:
-    Params.push_back(iPTR);
+    Params.push_back(PtrTy);
     Params.push_back(wasm::ValType::I64);
     Params.push_back(wasm::ValType::I64);
     Params.push_back(wasm::ValType::I64);
     Params.push_back(wasm::ValType::I64);
     break;
   case func_iPTR_i64_i64_i64_i64_i64_i64:
-    Params.push_back(iPTR);
+    Params.push_back(PtrTy);
     Params.push_back(wasm::ValType::I64);
     Params.push_back(wasm::ValType::I64);
     Params.push_back(wasm::ValType::I64);
@@ -832,12 +831,12 @@ void llvm::GetLibcallSignature(const WebAssemblySubtarget &Subtarget,
 static ManagedStatic<StaticLibcallNameMap> LibcallNameMap;
 // TODO: If the RTLIB::Libcall-taking flavor of GetSignature remains unsed
 // other than here, just roll its logic into this version.
-void llvm::GetLibcallSignature(const WebAssemblySubtarget &Subtarget,
+void llvm::getLibcallSignature(const WebAssemblySubtarget &Subtarget,
                                const char *Name,
                                SmallVectorImpl<wasm::ValType> &Rets,
                                SmallVectorImpl<wasm::ValType> &Params) {
   auto &Map = LibcallNameMap->Map;
-  auto val = Map.find(Name);
-  assert(val != Map.end() && "unexpected runtime library name");
-  return GetLibcallSignature(Subtarget, val->second, Rets, Params);
+  auto Val = Map.find(Name);
+  assert(Val != Map.end() && "unexpected runtime library name");
+  return getLibcallSignature(Subtarget, Val->second, Rets, Params);
 }
diff --git a/lib/Target/WebAssembly/WebAssemblyRuntimeLibcallSignatures.h b/lib/Target/WebAssembly/WebAssemblyRuntimeLibcallSignatures.h
index 7fa70bea96dec6d5cbc03c8e45237c898b2f00d2..6ae8aaaba59cd30a20c7c7a752d51ccc024f5227 100644
--- a/lib/Target/WebAssembly/WebAssemblyRuntimeLibcallSignatures.h
+++ b/lib/Target/WebAssembly/WebAssemblyRuntimeLibcallSignatures.h
@@ -1,9 +1,8 @@
 // CodeGen/RuntimeLibcallSignatures.h - R.T. Lib. Call Signatures -*- C++ -*--//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
@@ -23,12 +22,12 @@ namespace llvm {
 
 class WebAssemblySubtarget;
 
-extern void GetLibcallSignature(const WebAssemblySubtarget &Subtarget,
+extern void getLibcallSignature(const WebAssemblySubtarget &Subtarget,
                                 RTLIB::Libcall LC,
                                 SmallVectorImpl<wasm::ValType> &Rets,
                                 SmallVectorImpl<wasm::ValType> &Params);
 
-extern void GetLibcallSignature(const WebAssemblySubtarget &Subtarget,
+extern void getLibcallSignature(const WebAssemblySubtarget &Subtarget,
                                 const char *Name,
                                 SmallVectorImpl<wasm::ValType> &Rets,
                                 SmallVectorImpl<wasm::ValType> &Params);
diff --git a/lib/Target/WebAssembly/WebAssemblySelectionDAGInfo.cpp b/lib/Target/WebAssembly/WebAssemblySelectionDAGInfo.cpp
index bec72049258a84cf60a97e57124e13f85b4a59ab..34d93ecafce81d34a714ce57716b8a4ae072d0ca 100644
--- a/lib/Target/WebAssembly/WebAssemblySelectionDAGInfo.cpp
+++ b/lib/Target/WebAssembly/WebAssemblySelectionDAGInfo.cpp
@@ -1,9 +1,8 @@
 //===-- WebAssemblySelectionDAGInfo.cpp - WebAssembly SelectionDAG Info ---===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
@@ -17,4 +16,28 @@ using namespace llvm;
 
 #define DEBUG_TYPE "wasm-selectiondag-info"
 
-WebAssemblySelectionDAGInfo::~WebAssemblySelectionDAGInfo() {}
+WebAssemblySelectionDAGInfo::~WebAssemblySelectionDAGInfo() = default; // anchor
+
+SDValue WebAssemblySelectionDAGInfo::EmitTargetCodeForMemcpy(
+    SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Dst, SDValue Src,
+    SDValue Size, unsigned Align, bool IsVolatile, bool AlwaysInline,
+    MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo) const {
+  if (!DAG.getMachineFunction()
+           .getSubtarget<WebAssemblySubtarget>()
+           .hasBulkMemory())
+    return SDValue();
+
+  SDValue MemIdx = DAG.getConstant(0, DL, MVT::i32);
+  return DAG.getNode(WebAssemblyISD::MEMORY_COPY, DL, MVT::Other,
+                     {Chain, MemIdx, MemIdx, Dst, Src,
+                      DAG.getZExtOrTrunc(Size, DL, MVT::i32)});
+}
+
+SDValue WebAssemblySelectionDAGInfo::EmitTargetCodeForMemmove(
+    SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Op1, SDValue Op2,
+    SDValue Op3, unsigned Align, bool IsVolatile,
+    MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo) const {
+  return EmitTargetCodeForMemcpy(DAG, DL, Chain, Op1, Op2, Op3, Align,
+                                 IsVolatile, false, DstPtrInfo,
+                                 SrcPtrInfo);
+}
diff --git a/lib/Target/WebAssembly/WebAssemblySelectionDAGInfo.h b/lib/Target/WebAssembly/WebAssemblySelectionDAGInfo.h
index 31d150eded674cd21f58449056ebf8cd357dcfed..29e23e96aeb5b8de951e0d3f4eadc49b2d85bcb0 100644
--- a/lib/Target/WebAssembly/WebAssemblySelectionDAGInfo.h
+++ b/lib/Target/WebAssembly/WebAssemblySelectionDAGInfo.h
@@ -1,9 +1,8 @@
 //=- WebAssemblySelectionDAGInfo.h - WebAssembly SelectionDAG Info -*- C++ -*-//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
@@ -23,6 +22,17 @@ namespace llvm {
 class WebAssemblySelectionDAGInfo final : public SelectionDAGTargetInfo {
 public:
   ~WebAssemblySelectionDAGInfo() override;
+  SDValue EmitTargetCodeForMemcpy(SelectionDAG &DAG, const SDLoc &dl,
+                                  SDValue Chain, SDValue Op1, SDValue Op2,
+                                  SDValue Op3, unsigned Align, bool isVolatile,
+                                  bool AlwaysInline,
+                                  MachinePointerInfo DstPtrInfo,
+                                  MachinePointerInfo SrcPtrInfo) const override;
+  SDValue EmitTargetCodeForMemmove(SelectionDAG &DAG, const SDLoc &dl,
+                                   SDValue Chain, SDValue Op1, SDValue Op2,
+                                   SDValue Op3, unsigned Align, bool isVolatile,
+                                   MachinePointerInfo DstPtrInfo,
+                                   MachinePointerInfo SrcPtrInfo) const override;
 };
 
 } // end namespace llvm
diff --git a/lib/Target/WebAssembly/WebAssemblySetP2AlignOperands.cpp b/lib/Target/WebAssembly/WebAssemblySetP2AlignOperands.cpp
index c95af88c6f4356a124a16d7eb16d0fb0c418d39b..68ab925e433080a4d252262e5d2213167d700158 100644
--- a/lib/Target/WebAssembly/WebAssemblySetP2AlignOperands.cpp
+++ b/lib/Target/WebAssembly/WebAssemblySetP2AlignOperands.cpp
@@ -1,9 +1,8 @@
 //=- WebAssemblySetP2AlignOperands.cpp - Set alignments on loads and stores -=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
@@ -54,7 +53,7 @@ FunctionPass *llvm::createWebAssemblySetP2AlignOperands() {
   return new WebAssemblySetP2AlignOperands();
 }
 
-static void RewriteP2Align(MachineInstr &MI, unsigned OperandNo) {
+static void rewriteP2Align(MachineInstr &MI, unsigned OperandNo) {
   assert(MI.getOperand(OperandNo).getImm() == 0 &&
          "ISel should set p2align operands to 0");
   assert(MI.hasOneMemOperand() &&
@@ -164,7 +163,7 @@ bool WebAssemblySetP2AlignOperands::runOnMachineFunction(MachineFunction &MF) {
       case WebAssembly::ATOMIC_NOTIFY:
       case WebAssembly::ATOMIC_WAIT_I32:
       case WebAssembly::ATOMIC_WAIT_I64:
-        RewriteP2Align(MI, WebAssembly::LoadP2AlignOperandNo);
+        rewriteP2Align(MI, WebAssembly::LoadP2AlignOperandNo);
         break;
       case WebAssembly::STORE_I32:
       case WebAssembly::STORE_I64:
@@ -188,7 +187,7 @@ bool WebAssemblySetP2AlignOperands::runOnMachineFunction(MachineFunction &MF) {
       case WebAssembly::ATOMIC_STORE8_I64:
       case WebAssembly::ATOMIC_STORE16_I64:
       case WebAssembly::ATOMIC_STORE32_I64:
-        RewriteP2Align(MI, WebAssembly::StoreP2AlignOperandNo);
+        rewriteP2Align(MI, WebAssembly::StoreP2AlignOperandNo);
         break;
       default:
         break;
diff --git a/lib/Target/WebAssembly/WebAssemblySubtarget.cpp b/lib/Target/WebAssembly/WebAssemblySubtarget.cpp
index d6af0fb219d7a7a8e0166cf0b96da2186cb5254f..bf2f70ca2000258704d1310dae7546227e92486c 100644
--- a/lib/Target/WebAssembly/WebAssemblySubtarget.cpp
+++ b/lib/Target/WebAssembly/WebAssemblySubtarget.cpp
@@ -1,9 +1,8 @@
 //===-- WebAssemblySubtarget.cpp - WebAssembly Subtarget Information ------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
@@ -40,10 +39,9 @@ WebAssemblySubtarget::WebAssemblySubtarget(const Triple &TT,
                                            const std::string &CPU,
                                            const std::string &FS,
                                            const TargetMachine &TM)
-    : WebAssemblyGenSubtargetInfo(TT, CPU, FS), HasSIMD128(false),
-      HasAtomics(false), HasNontrappingFPToInt(false), HasSignExt(false),
-      HasExceptionHandling(false), CPUString(CPU), TargetTriple(TT),
-      FrameLowering(), InstrInfo(initializeSubtargetDependencies(FS)), TSInfo(),
+    : WebAssemblyGenSubtargetInfo(TT, CPU, FS), CPUString(CPU),
+      TargetTriple(TT), FrameLowering(),
+      InstrInfo(initializeSubtargetDependencies(FS)), TSInfo(),
       TLInfo(TM, *this) {}
 
 bool WebAssemblySubtarget::enableMachineScheduler() const {
diff --git a/lib/Target/WebAssembly/WebAssemblySubtarget.h b/lib/Target/WebAssembly/WebAssemblySubtarget.h
index b170dbff3b32af59cfc106cdb288e908ac57ab94..b6f3e6b4a108483cb4af736293b710dfb1e8ab5a 100644
--- a/lib/Target/WebAssembly/WebAssemblySubtarget.h
+++ b/lib/Target/WebAssembly/WebAssemblySubtarget.h
@@ -1,9 +1,8 @@
 //=- WebAssemblySubtarget.h - Define Subtarget for the WebAssembly -*- C++ -*-//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
@@ -29,11 +28,17 @@
 namespace llvm {
 
 class WebAssemblySubtarget final : public WebAssemblyGenSubtargetInfo {
-  bool HasSIMD128;
-  bool HasAtomics;
-  bool HasNontrappingFPToInt;
-  bool HasSignExt;
-  bool HasExceptionHandling;
+  enum SIMDEnum {
+    NoSIMD,
+    SIMD128,
+    UnimplementedSIMD128,
+  } SIMDLevel = NoSIMD;
+
+  bool HasAtomics = false;
+  bool HasNontrappingFPToInt = false;
+  bool HasSignExt = false;
+  bool HasExceptionHandling = false;
+  bool HasBulkMemory = false;
 
   /// String name of used CPU.
   std::string CPUString;
@@ -77,11 +82,15 @@ public:
 
   // Predicates used by WebAssemblyInstrInfo.td.
   bool hasAddr64() const { return TargetTriple.isArch64Bit(); }
-  bool hasSIMD128() const { return HasSIMD128; }
+  bool hasSIMD128() const { return SIMDLevel >= SIMD128; }
+  bool hasUnimplementedSIMD128() const {
+    return SIMDLevel >= UnimplementedSIMD128;
+  }
   bool hasAtomics() const { return HasAtomics; }
   bool hasNontrappingFPToInt() const { return HasNontrappingFPToInt; }
   bool hasSignExt() const { return HasSignExt; }
   bool hasExceptionHandling() const { return HasExceptionHandling; }
+  bool hasBulkMemory() const { return HasBulkMemory; }
 
   /// Parses features string setting specified subtarget options. Definition of
   /// function is auto generated by tblgen.
diff --git a/lib/Target/WebAssembly/WebAssemblyTargetMachine.cpp b/lib/Target/WebAssembly/WebAssemblyTargetMachine.cpp
index a6fb78280dad82cd22c99d5dd1a5e3df7906a3d3..8e8b4878c52c1cb36ff22566b8bbdce8ce9b3374 100644
--- a/lib/Target/WebAssembly/WebAssemblyTargetMachine.cpp
+++ b/lib/Target/WebAssembly/WebAssemblyTargetMachine.cpp
@@ -1,9 +1,8 @@
 //===- WebAssemblyTargetMachine.cpp - Define TargetMachine for WebAssembly -==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
@@ -58,11 +57,10 @@ extern "C" void LLVMInitializeWebAssemblyTarget() {
   initializeOptimizeReturnedPass(PR);
   initializeWebAssemblyArgumentMovePass(PR);
   initializeWebAssemblySetP2AlignOperandsPass(PR);
-  initializeWebAssemblyEHRestoreStackPointerPass(PR);
   initializeWebAssemblyReplacePhysRegsPass(PR);
   initializeWebAssemblyPrepareForLiveIntervalsPass(PR);
   initializeWebAssemblyOptimizeLiveIntervalsPass(PR);
-  initializeWebAssemblyStoreResultsPass(PR);
+  initializeWebAssemblyMemIntrinsicResultsPass(PR);
   initializeWebAssemblyRegStackifyPass(PR);
   initializeWebAssemblyRegColoringPass(PR);
   initializeWebAssemblyExplicitLocalsPass(PR);
@@ -122,7 +120,7 @@ WebAssemblyTargetMachine::WebAssemblyTargetMachine(
   // splitting and tail merging.
 }
 
-WebAssemblyTargetMachine::~WebAssemblyTargetMachine() {}
+WebAssemblyTargetMachine::~WebAssemblyTargetMachine() = default; // anchor.
 
 const WebAssemblySubtarget *
 WebAssemblyTargetMachine::getSubtargetImpl(const Function &F) const {
@@ -285,19 +283,24 @@ void WebAssemblyPassConfig::addPostRegAlloc() {
 void WebAssemblyPassConfig::addPreEmitPass() {
   TargetPassConfig::addPreEmitPass();
 
-  // Restore __stack_pointer global after an exception is thrown.
-  addPass(createWebAssemblyEHRestoreStackPointer());
+  // Rewrite pseudo call_indirect instructions as real instructions.
+  // This needs to run before register stackification, because we change the
+  // order of the arguments.
+  addPass(createWebAssemblyCallIndirectFixup());
+
+  // Eliminate multiple-entry loops.
+  addPass(createWebAssemblyFixIrreducibleControlFlow());
+
+  // Do various transformations for exception handling.
+  // Every CFG-changing optimizations should come before this.
+  addPass(createWebAssemblyLateEHPrepare());
 
   // Now that we have a prologue and epilogue and all frame indices are
   // rewritten, eliminate SP and FP. This allows them to be stackified,
   // colored, and numbered with the rest of the registers.
   addPass(createWebAssemblyReplacePhysRegs());
 
-  // Rewrite pseudo call_indirect instructions as real instructions.
-  // This needs to run before register stackification, because we change the
-  // order of the arguments.
-  addPass(createWebAssemblyCallIndirectFixup());
-
+  // Preparations and optimizations related to register stackification.
   if (getOptLevel() != CodeGenOpt::None) {
     // LiveIntervals isn't commonly run this late. Re-establish preconditions.
     addPass(createWebAssemblyPrepareForLiveIntervals());
@@ -305,13 +308,14 @@ void WebAssemblyPassConfig::addPreEmitPass() {
     // Depend on LiveIntervals and perform some optimizations on it.
     addPass(createWebAssemblyOptimizeLiveIntervals());
 
-    // Prepare store instructions for register stackifying.
-    addPass(createWebAssemblyStoreResults());
+    // Prepare memory intrinsic calls for register stackifying.
+    addPass(createWebAssemblyMemIntrinsicResults());
 
     // Mark registers as representing wasm's value stack. This is a key
     // code-compression technique in WebAssembly. We run this pass (and
-    // StoreResults above) very late, so that it sees as much code as possible,
-    // including code emitted by PEI and expanded by late tail duplication.
+    // MemIntrinsicResults above) very late, so that it sees as much code as
+    // possible, including code emitted by PEI and expanded by late tail
+    // duplication.
     addPass(createWebAssemblyRegStackify());
 
     // Run the register coloring pass to reduce the total number of registers.
@@ -320,17 +324,9 @@ void WebAssemblyPassConfig::addPreEmitPass() {
     addPass(createWebAssemblyRegColoring());
   }
 
-  // Eliminate multiple-entry loops. Do this before inserting explicit get_local
-  // and set_local operators because we create a new variable that we want
-  // converted into a local.
-  addPass(createWebAssemblyFixIrreducibleControlFlow());
-
-  // Insert explicit get_local and set_local operators.
+  // Insert explicit local.get and local.set operators.
   addPass(createWebAssemblyExplicitLocals());
 
-  // Do various transformations for exception handling
-  addPass(createWebAssemblyLateEHPrepare());
-
   // Sort the blocks of the CFG into topological order, a prerequisite for
   // BLOCK and LOOP markers.
   addPass(createWebAssemblyCFGSort());
diff --git a/lib/Target/WebAssembly/WebAssemblyTargetMachine.h b/lib/Target/WebAssembly/WebAssemblyTargetMachine.h
index 41001e7a0cc7b00c36a41a1e3a694bc737ff930d..e13ad7d4eacbfec89d01062eacb497877c102d74 100644
--- a/lib/Target/WebAssembly/WebAssemblyTargetMachine.h
+++ b/lib/Target/WebAssembly/WebAssemblyTargetMachine.h
@@ -1,9 +1,8 @@
 // WebAssemblyTargetMachine.h - Define TargetMachine for WebAssembly -*- C++ -*-
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
diff --git a/lib/Target/WebAssembly/WebAssemblyTargetObjectFile.cpp b/lib/Target/WebAssembly/WebAssemblyTargetObjectFile.cpp
index 0459bfca418de71dd77a564de28e59c223db5743..ad57c600db101038981064e603f6aabbedafb007 100644
--- a/lib/Target/WebAssembly/WebAssemblyTargetObjectFile.cpp
+++ b/lib/Target/WebAssembly/WebAssemblyTargetObjectFile.cpp
@@ -1,9 +1,8 @@
 //===-- WebAssemblyTargetObjectFile.cpp - WebAssembly Object Info ---------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
diff --git a/lib/Target/WebAssembly/WebAssemblyTargetObjectFile.h b/lib/Target/WebAssembly/WebAssemblyTargetObjectFile.h
index ce744ba8b8e8d93b308edab3b4c2f8968584bc99..f46bb2040a7d4b1355a0190157643c83bf00d7f2 100644
--- a/lib/Target/WebAssembly/WebAssemblyTargetObjectFile.h
+++ b/lib/Target/WebAssembly/WebAssemblyTargetObjectFile.h
@@ -1,9 +1,8 @@
 //===-- WebAssemblyTargetObjectFile.h - WebAssembly Object Info -*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
diff --git a/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.cpp b/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.cpp
index 4a2777cc3a9f677602f130470fc5789668e23a27..46ef765ce0f4b382d91b3d3237a372daad4c5741 100644
--- a/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.cpp
+++ b/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.cpp
@@ -1,9 +1,8 @@
 //===-- WebAssemblyTargetTransformInfo.cpp - WebAssembly-specific TTI -----===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
@@ -51,7 +50,7 @@ unsigned WebAssemblyTTIImpl::getArithmeticInstrCost(
   unsigned Cost = BasicTTIImplBase<WebAssemblyTTIImpl>::getArithmeticInstrCost(
       Opcode, Ty, Opd1Info, Opd2Info, Opd1PropInfo, Opd2PropInfo);
 
-  if (VectorType *VTy = dyn_cast<VectorType>(Ty)) {
+  if (auto *VTy = dyn_cast<VectorType>(Ty)) {
     switch (Opcode) {
     case Instruction::LShr:
     case Instruction::AShr:
diff --git a/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.h b/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.h
index 4300ca3defbf0a133fab4ce284a60de33faa693d..1b11b4b631eb930e051c87026b7063a1b2a0b4c1 100644
--- a/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.h
+++ b/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.h
@@ -1,9 +1,8 @@
 //==- WebAssemblyTargetTransformInfo.h - WebAssembly-specific TTI -*- C++ -*-=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
diff --git a/lib/Target/WebAssembly/WebAssemblyUtilities.cpp b/lib/Target/WebAssembly/WebAssemblyUtilities.cpp
index ada6fb9a96d79d43256c7ac989f43b9bf18e1189..4edd54602fb2e94dd0dd7c3e4ed6a210abdd641b 100644
--- a/lib/Target/WebAssembly/WebAssemblyUtilities.cpp
+++ b/lib/Target/WebAssembly/WebAssemblyUtilities.cpp
@@ -1,9 +1,8 @@
 //===-- WebAssemblyUtilities.cpp - WebAssembly Utility Functions ----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
@@ -243,50 +242,10 @@ bool WebAssembly::isMarker(const MachineInstr &MI) {
   }
 }
 
-bool WebAssembly::isThrow(const MachineInstr &MI) {
-  switch (MI.getOpcode()) {
-  case WebAssembly::THROW_I32:
-  case WebAssembly::THROW_I32_S:
-  case WebAssembly::THROW_I64:
-  case WebAssembly::THROW_I64_S:
-    return true;
-  default:
-    return false;
-  }
-}
-
-bool WebAssembly::isRethrow(const MachineInstr &MI) {
-  switch (MI.getOpcode()) {
-  case WebAssembly::RETHROW:
-  case WebAssembly::RETHROW_S:
-  case WebAssembly::RETHROW_TO_CALLER:
-  case WebAssembly::RETHROW_TO_CALLER_S:
-    return true;
-  default:
-    return false;
-  }
-}
-
-bool WebAssembly::isCatch(const MachineInstr &MI) {
-  switch (MI.getOpcode()) {
-  case WebAssembly::CATCH_I32:
-  case WebAssembly::CATCH_I32_S:
-  case WebAssembly::CATCH_I64:
-  case WebAssembly::CATCH_I64_S:
-  case WebAssembly::CATCH_ALL:
-  case WebAssembly::CATCH_ALL_S:
-    return true;
-  default:
-    return false;
-  }
-}
-
 bool WebAssembly::mayThrow(const MachineInstr &MI) {
   switch (MI.getOpcode()) {
-  case WebAssembly::THROW_I32:
-  case WebAssembly::THROW_I32_S:
-  case WebAssembly::THROW_I64:
-  case WebAssembly::THROW_I64_S:
+  case WebAssembly::THROW:
+  case WebAssembly::THROW_S:
   case WebAssembly::RETHROW:
   case WebAssembly::RETHROW_S:
     return true;
@@ -309,41 +268,3 @@ bool WebAssembly::mayThrow(const MachineInstr &MI) {
     return false;
   return true;
 }
-
-bool WebAssembly::isCatchTerminatePad(const MachineBasicBlock &MBB) {
-  if (!MBB.isEHPad())
-    return false;
-  bool SeenCatch = false;
-  for (auto &MI : MBB) {
-    if (MI.getOpcode() == WebAssembly::CATCH_I32 ||
-        MI.getOpcode() == WebAssembly::CATCH_I64 ||
-        MI.getOpcode() == WebAssembly::CATCH_I32_S ||
-        MI.getOpcode() == WebAssembly::CATCH_I64_S)
-      SeenCatch = true;
-    if (SeenCatch && MI.isCall()) {
-      const MachineOperand &CalleeOp = MI.getOperand(getCalleeOpNo(MI));
-      if (CalleeOp.isGlobal() &&
-          CalleeOp.getGlobal()->getName() == ClangCallTerminateFn)
-        return true;
-    }
-  }
-  return false;
-}
-
-bool WebAssembly::isCatchAllTerminatePad(const MachineBasicBlock &MBB) {
-  if (!MBB.isEHPad())
-    return false;
-  bool SeenCatchAll = false;
-  for (auto &MI : MBB) {
-    if (MI.getOpcode() == WebAssembly::CATCH_ALL ||
-        MI.getOpcode() == WebAssembly::CATCH_ALL_S)
-      SeenCatchAll = true;
-    if (SeenCatchAll && MI.isCall()) {
-      const MachineOperand &CalleeOp = MI.getOperand(getCalleeOpNo(MI));
-      if (CalleeOp.isGlobal() &&
-          CalleeOp.getGlobal()->getName() == StdTerminateFn)
-        return true;
-    }
-  }
-  return false;
-}
diff --git a/lib/Target/WebAssembly/WebAssemblyUtilities.h b/lib/Target/WebAssembly/WebAssemblyUtilities.h
index cdb7873e9013bd900d70a26813d1865416311400..f80b49662ea64215293ab472815d11f8e1ded71d 100644
--- a/lib/Target/WebAssembly/WebAssemblyUtilities.h
+++ b/lib/Target/WebAssembly/WebAssemblyUtilities.h
@@ -1,9 +1,8 @@
 //===-- WebAssemblyUtilities - WebAssembly Utility Functions ---*- C++ -*-====//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
@@ -31,22 +30,12 @@ bool isChild(const MachineInstr &MI, const WebAssemblyFunctionInfo &MFI);
 bool isCallDirect(const MachineInstr &MI);
 bool isCallIndirect(const MachineInstr &MI);
 bool isMarker(const MachineInstr &MI);
-bool isThrow(const MachineInstr &MI);
-bool isRethrow(const MachineInstr &MI);
-bool isCatch(const MachineInstr &MI);
 bool mayThrow(const MachineInstr &MI);
 
 /// Returns the operand number of a callee, assuming the argument is a call
 /// instruction.
 unsigned getCalleeOpNo(const MachineInstr &MI);
 
-/// Returns if the given BB is a single BB terminate pad which starts with a
-/// 'catch' instruction.
-bool isCatchTerminatePad(const MachineBasicBlock &MBB);
-/// Returns if the given BB is a single BB terminate pad which starts with a
-/// 'catch_all' insrtruction.
-bool isCatchAllTerminatePad(const MachineBasicBlock &MBB);
-
 // Exception-related function names
 extern const char *const ClangCallTerminateFn;
 extern const char *const CxaBeginCatchFn;
diff --git a/lib/Target/WebAssembly/known_gcc_test_failures.txt b/lib/Target/WebAssembly/known_gcc_test_failures.txt
index 364c871f61b0d1626fe076027c44f9d20c489e39..a379567953147f50ad25b0de2397d377c985b486 100644
--- a/lib/Target/WebAssembly/known_gcc_test_failures.txt
+++ b/lib/Target/WebAssembly/known_gcc_test_failures.txt
@@ -75,7 +75,6 @@ pr41935.c
 920501-3.c
 920728-1.c
 pr28865.c
-widechar-2.c
 attr-alias-1.C
 attr-alias-2.C
 attr-ifunc-1.C
diff --git a/lib/Target/X86/AsmParser/LLVMBuild.txt b/lib/Target/X86/AsmParser/LLVMBuild.txt
index 67c0d1358d80cddc3261b682abe7e77d19c5cfc2..e80d48cb54a0cdd8503349811e819d90391c79e1 100644
--- a/lib/Target/X86/AsmParser/LLVMBuild.txt
+++ b/lib/Target/X86/AsmParser/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/Target/X86/AsmParser/LLVMBuild.txt -----------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Target/X86/AsmParser/X86AsmInstrumentation.cpp b/lib/Target/X86/AsmParser/X86AsmInstrumentation.cpp
index 2c376fd062ca6d3ff0f9e15c7c069ea0c742b155..f65f6ee5596b6401bf701a1df8c87f19d394c046 100644
--- a/lib/Target/X86/AsmParser/X86AsmInstrumentation.cpp
+++ b/lib/Target/X86/AsmParser/X86AsmInstrumentation.cpp
@@ -1,9 +1,8 @@
 //===-- X86AsmInstrumentation.cpp - Instrument X86 inline assembly --------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -195,8 +194,7 @@ public:
   // X86AsmInstrumentation implementation:
   void InstrumentAndEmitInstruction(const MCInst &Inst, OperandVector &Operands,
                                     MCContext &Ctx, const MCInstrInfo &MII,
-                                    MCStreamer &Out,
-                                    /* unused */ bool) override {
+                                    MCStreamer &Out) override {
     InstrumentMOVS(Inst, Operands, Ctx, MII, Out);
     if (RepPrefix)
       EmitInstruction(Out, MCInstBuilder(X86::REP_PREFIX));
@@ -1044,13 +1042,13 @@ X86AsmInstrumentation::~X86AsmInstrumentation() = default;
 
 void X86AsmInstrumentation::InstrumentAndEmitInstruction(
     const MCInst &Inst, OperandVector &Operands, MCContext &Ctx,
-    const MCInstrInfo &MII, MCStreamer &Out, bool PrintSchedInfoEnabled) {
-  EmitInstruction(Out, Inst, PrintSchedInfoEnabled);
+    const MCInstrInfo &MII, MCStreamer &Out) {
+  EmitInstruction(Out, Inst);
 }
 
-void X86AsmInstrumentation::EmitInstruction(MCStreamer &Out, const MCInst &Inst,
-                                            bool PrintSchedInfoEnabled) {
-  Out.EmitInstruction(Inst, *STI, PrintSchedInfoEnabled);
+void X86AsmInstrumentation::EmitInstruction(MCStreamer &Out,
+                                            const MCInst &Inst) {
+  Out.EmitInstruction(Inst, *STI);
 }
 
 unsigned X86AsmInstrumentation::GetFrameRegGeneric(const MCContext &Ctx,
diff --git a/lib/Target/X86/AsmParser/X86AsmInstrumentation.h b/lib/Target/X86/AsmParser/X86AsmInstrumentation.h
index 42a9dc3ba26aac344176a675843c8c6611023226..58ecd7d967569386eaef314475bd057299487d0b 100644
--- a/lib/Target/X86/AsmParser/X86AsmInstrumentation.h
+++ b/lib/Target/X86/AsmParser/X86AsmInstrumentation.h
@@ -1,9 +1,8 @@
 //===- X86AsmInstrumentation.h - Instrument X86 inline assembly -*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -42,8 +41,7 @@ public:
   virtual void InstrumentAndEmitInstruction(
       const MCInst &Inst,
       SmallVectorImpl<std::unique_ptr<MCParsedAsmOperand>> &Operands,
-      MCContext &Ctx, const MCInstrInfo &MII, MCStreamer &Out,
-      bool PrintSchedInfoEnabled);
+      MCContext &Ctx, const MCInstrInfo &MII, MCStreamer &Out);
 
 protected:
   friend X86AsmInstrumentation *
@@ -55,8 +53,7 @@ protected:
 
   unsigned GetFrameRegGeneric(const MCContext &Ctx, MCStreamer &Out);
 
-  void EmitInstruction(MCStreamer &Out, const MCInst &Inst,
-                       bool PrintSchedInfoEnabled = false);
+  void EmitInstruction(MCStreamer &Out, const MCInst &Inst);
 
   const MCSubtargetInfo *&STI;
 
diff --git a/lib/Target/X86/AsmParser/X86AsmParser.cpp b/lib/Target/X86/AsmParser/X86AsmParser.cpp
index 4801078925ccfadec855772a673d26eb61adb2c3..44173d9b2b3eda9db70b66390e8ab81533349e59 100644
--- a/lib/Target/X86/AsmParser/X86AsmParser.cpp
+++ b/lib/Target/X86/AsmParser/X86AsmParser.cpp
@@ -1,9 +1,8 @@
 //===-- X86AsmParser.cpp - Parse X86 assembly to MCInst instructions ------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -804,8 +803,8 @@ private:
     return Parser.Error(L, Msg, Range);
   }
 
-  std::nullptr_t ErrorOperand(SMLoc Loc, StringRef Msg) {
-    Error(Loc, Msg);
+  std::nullptr_t ErrorOperand(SMLoc Loc, StringRef Msg, SMRange R = SMRange()) {
+    Error(Loc, Msg, R);
     return nullptr;
   }
 
@@ -835,7 +834,10 @@ private:
                                      InlineAsmIdentifierInfo &Info,
                                      bool IsUnevaluatedOperand, SMLoc &End);
 
-  std::unique_ptr<X86Operand> ParseMemOperand(unsigned SegReg, SMLoc MemStart);
+  std::unique_ptr<X86Operand> ParseMemOperand(unsigned SegReg,
+                                              const MCExpr *&Disp,
+                                              const SMLoc &StartLoc,
+                                              SMLoc &EndLoc);
 
   bool ParseIntelMemoryOperandSize(unsigned &Size);
   std::unique_ptr<X86Operand>
@@ -1011,8 +1013,7 @@ static bool CheckBaseRegAndIndexRegAndScale(unsigned BaseReg, unsigned IndexReg,
   // and then only in non-64-bit modes.
   if (X86MCRegisterClasses[X86::GR16RegClassID].contains(BaseReg) &&
       (Is64BitMode || (BaseReg != X86::BX && BaseReg != X86::BP &&
-                       BaseReg != X86::SI && BaseReg != X86::DI)) &&
-      BaseReg != X86::DX) {
+                       BaseReg != X86::SI && BaseReg != X86::DI))) {
     ErrMsg = "invalid 16-bit base register";
     return true;
   }
@@ -1103,15 +1104,17 @@ bool X86AsmParser::ParseRegister(unsigned &RegNo,
     if (RegNo == X86::RIZ || RegNo == X86::RIP ||
         X86MCRegisterClasses[X86::GR64RegClassID].contains(RegNo) ||
         X86II::isX86_64NonExtLowByteReg(RegNo) ||
-        X86II::isX86_64ExtendedReg(RegNo))
-      return Error(StartLoc, "register %"
-                   + Tok.getString() + " is only available in 64-bit mode",
+        X86II::isX86_64ExtendedReg(RegNo)) {
+      StringRef RegName = Tok.getString();
+      Parser.Lex(); // Eat register name.
+      return Error(StartLoc,
+                   "register %" + RegName + " is only available in 64-bit mode",
                    SMRange(StartLoc, EndLoc));
+    }
   }
 
   // Parse "%st" as "%st(0)" and "%st(1)", which is multiple tokens.
-  if (RegNo == 0 && (Tok.getString() == "st" || Tok.getString() == "ST")) {
-    RegNo = X86::ST0;
+  if (RegNo == X86::ST0) {
     Parser.Lex(); // Eat 'st'
 
     // Check to see if we have '(4)' after %st.
@@ -1936,49 +1939,61 @@ std::unique_ptr<X86Operand> X86AsmParser::ParseIntelOperand() {
 std::unique_ptr<X86Operand> X86AsmParser::ParseATTOperand() {
   MCAsmParser &Parser = getParser();
   switch (getLexer().getKind()) {
-  default:
-    // Parse a memory operand with no segment register.
-    return ParseMemOperand(0, Parser.getTok().getLoc());
-  case AsmToken::Percent: {
-    // Read the register.
-    unsigned RegNo;
-    SMLoc Start, End;
-    if (ParseRegister(RegNo, Start, End)) return nullptr;
-    if (RegNo == X86::EIZ || RegNo == X86::RIZ) {
-      Error(Start, "%eiz and %riz can only be used as index registers",
-            SMRange(Start, End));
-      return nullptr;
-    }
-    if (RegNo == X86::RIP) {
-      Error(Start, "%rip can only be used as a base register",
-            SMRange(Start, End));
-      return nullptr;
-    }
-
-    // If this is a segment register followed by a ':', then this is the start
-    // of a memory reference, otherwise this is a normal register reference.
-    if (getLexer().isNot(AsmToken::Colon))
-      return X86Operand::CreateReg(RegNo, Start, End);
-
-    if (!X86MCRegisterClasses[X86::SEGMENT_REGRegClassID].contains(RegNo))
-      return ErrorOperand(Start, "invalid segment register");
-
-    getParser().Lex(); // Eat the colon.
-    return ParseMemOperand(RegNo, Start);
-  }
   case AsmToken::Dollar: {
-    // $42 -> immediate.
+    // $42 or $ID -> immediate.
     SMLoc Start = Parser.getTok().getLoc(), End;
     Parser.Lex();
     const MCExpr *Val;
-    if (getParser().parseExpression(Val, End))
+    // This is an immediate, so we should not parse a register. Do a precheck
+    // for '%' to supercede intra-register parse errors.
+    SMLoc L = Parser.getTok().getLoc();
+    if (check(getLexer().is(AsmToken::Percent), L,
+              "expected immediate expression") ||
+        getParser().parseExpression(Val, End) ||
+        check(isa<X86MCExpr>(Val), L, "expected immediate expression"))
       return nullptr;
     return X86Operand::CreateImm(Val, Start, End);
   }
-  case AsmToken::LCurly:{
+  case AsmToken::LCurly: {
     SMLoc Start = Parser.getTok().getLoc();
     return ParseRoundingModeOp(Start);
   }
+  default: {
+    // This a memory operand or a register. We have some parsing complications
+    // as a '(' may be part of an immediate expression or the addressing mode
+    // block. This is complicated by the fact that an assembler-level variable
+    // may refer either to a register or an immediate expression.
+
+    SMLoc Loc = Parser.getTok().getLoc(), EndLoc;
+    const MCExpr *Expr = nullptr;
+    unsigned Reg = 0;
+    if (getLexer().isNot(AsmToken::LParen)) {
+      // No '(' so this is either a displacement expression or a register.
+      if (Parser.parseExpression(Expr, EndLoc))
+        return nullptr;
+      if (auto *RE = dyn_cast<X86MCExpr>(Expr)) {
+        // Segment Register. Reset Expr and copy value to register.
+        Expr = nullptr;
+        Reg = RE->getRegNo();
+
+        // Sanity check register.
+        if (Reg == X86::EIZ || Reg == X86::RIZ)
+          return ErrorOperand(
+              Loc, "%eiz and %riz can only be used as index registers",
+              SMRange(Loc, EndLoc));
+        if (Reg == X86::RIP)
+          return ErrorOperand(Loc, "%rip can only be used as a base register",
+                              SMRange(Loc, EndLoc));
+        // Return register that are not segment prefixes immediately.
+        if (!Parser.parseOptionalToken(AsmToken::Colon))
+          return X86Operand::CreateReg(Reg, Loc, EndLoc);
+        if (!X86MCRegisterClasses[X86::SEGMENT_REGRegClassID].contains(Reg))
+          return ErrorOperand(Loc, "invalid segment register");
+      }
+    }
+    // This is a Memory operand.
+    return ParseMemOperand(Reg, Expr, Loc, EndLoc);
+  }
   }
 }
 
@@ -2087,199 +2102,201 @@ bool X86AsmParser::HandleAVX512Operand(OperandVector &Operands,
   return false;
 }
 
-/// ParseMemOperand: segment: disp(basereg, indexreg, scale).  The '%ds:' prefix
-/// has already been parsed if present.
+/// ParseMemOperand: 'seg : disp(basereg, indexreg, scale)'.  The '%ds:' prefix
+/// has already been parsed if present. disp may be provided as well.
 std::unique_ptr<X86Operand> X86AsmParser::ParseMemOperand(unsigned SegReg,
-                                                          SMLoc MemStart) {
-
+                                                          const MCExpr *&Disp,
+                                                          const SMLoc &StartLoc,
+                                                          SMLoc &EndLoc) {
   MCAsmParser &Parser = getParser();
-  // We have to disambiguate a parenthesized expression "(4+5)" from the start
-  // of a memory operand with a missing displacement "(%ebx)" or "(,%eax)".  The
-  // only way to do this without lookahead is to eat the '(' and see what is
-  // after it.
-  const MCExpr *Disp = MCConstantExpr::create(0, getParser().getContext());
-  if (getLexer().isNot(AsmToken::LParen)) {
-    SMLoc ExprEnd;
-    if (getParser().parseExpression(Disp, ExprEnd)) return nullptr;
-    // Disp may be a variable, handle register values.
-    if (auto *RE = dyn_cast<X86MCExpr>(Disp))
-      return X86Operand::CreateReg(RE->getRegNo(), MemStart, ExprEnd);
-
-    // After parsing the base expression we could either have a parenthesized
-    // memory address or not.  If not, return now.  If so, eat the (.
-    if (getLexer().isNot(AsmToken::LParen)) {
-      // Unless we have a segment register, treat this as an immediate.
-      if (SegReg == 0)
-        return X86Operand::CreateMem(getPointerWidth(), Disp, MemStart, ExprEnd);
-      return X86Operand::CreateMem(getPointerWidth(), SegReg, Disp, 0, 0, 1,
-                                   MemStart, ExprEnd);
+  SMLoc Loc;
+  // Based on the initial passed values, we may be in any of these cases, we are
+  // in one of these cases (with current position (*)):
+
+  //   1. seg : * disp  (base-index-scale-expr)
+  //   2. seg : *(disp) (base-index-scale-expr)
+  //   3. seg :       *(base-index-scale-expr)
+  //   4.        disp  *(base-index-scale-expr)
+  //   5.      *(disp)  (base-index-scale-expr)
+  //   6.             *(base-index-scale-expr)
+  //   7.  disp *
+  //   8. *(disp)
+
+  // If we do not have an displacement yet, check if we're in cases 4 or 6 by
+  // checking if the first object after the parenthesis is a register (or an
+  // identifier referring to a register) and parse the displacement or default
+  // to 0 as appropriate.
+  auto isAtMemOperand = [this]() {
+    if (this->getLexer().isNot(AsmToken::LParen))
+      return false;
+    AsmToken Buf[2];
+    StringRef Id;
+    auto TokCount = this->getLexer().peekTokens(Buf, true);
+    if (TokCount == 0)
+      return false;
+    switch (Buf[0].getKind()) {
+    case AsmToken::Percent:
+    case AsmToken::Comma:
+      return true;
+    // These lower cases are doing a peekIdentifier.
+    case AsmToken::At:
+    case AsmToken::Dollar:
+      if ((TokCount > 1) &&
+          (Buf[1].is(AsmToken::Identifier) || Buf[1].is(AsmToken::String)) &&
+          (Buf[0].getLoc().getPointer() + 1 == Buf[1].getLoc().getPointer()))
+        Id = StringRef(Buf[0].getLoc().getPointer(),
+                       Buf[1].getIdentifier().size() + 1);
+      break;
+    case AsmToken::Identifier:
+    case AsmToken::String:
+      Id = Buf[0].getIdentifier();
+      break;
+    default:
+      return false;
     }
+    // We have an ID. Check if it is bound to a register.
+    if (!Id.empty()) {
+      MCSymbol *Sym = this->getContext().getOrCreateSymbol(Id);
+      if (Sym->isVariable()) {
+        auto V = Sym->getVariableValue(/*SetUsed*/ false);
+        return isa<X86MCExpr>(V);
+      }
+    }
+    return false;
+  };
 
-    // Eat the '('.
-    Parser.Lex();
-  } else {
-    // Okay, we have a '('.  We don't know if this is an expression or not, but
-    // so we have to eat the ( to see beyond it.
-    SMLoc LParenLoc = Parser.getTok().getLoc();
-    Parser.Lex(); // Eat the '('.
-
-    if (getLexer().is(AsmToken::Percent) || getLexer().is(AsmToken::Comma)) {
-      // Nothing to do here, fall into the code below with the '(' part of the
-      // memory operand consumed.
-    } else {
-      SMLoc ExprEnd;
-      getLexer().UnLex(AsmToken(AsmToken::LParen, "("));
-
-      // It must be either an parenthesized expression, or an expression that
-      // begins from a parenthesized expression, parse it now. Example: (1+2) or
-      // (1+2)+3
-      if (getParser().parseExpression(Disp, ExprEnd))
+  if (!Disp) {
+    // Parse immediate if we're not at a mem operand yet.
+    if (!isAtMemOperand()) {
+      if (Parser.parseTokenLoc(Loc) || Parser.parseExpression(Disp, EndLoc))
         return nullptr;
+      assert(!isa<X86MCExpr>(Disp) && "Expected non-register here.");
+    } else {
+      // Disp is implicitly zero if we haven't parsed it yet.
+      Disp = MCConstantExpr::create(0, Parser.getContext());
+    }
+  }
 
-      // After parsing the base expression we could either have a parenthesized
-      // memory address or not.  If not, return now.  If so, eat the (.
-      if (getLexer().isNot(AsmToken::LParen)) {
-        // Unless we have a segment register, treat this as an immediate.
-        if (SegReg == 0)
-          return X86Operand::CreateMem(getPointerWidth(), Disp, LParenLoc,
-                                       ExprEnd);
-        return X86Operand::CreateMem(getPointerWidth(), SegReg, Disp, 0, 0, 1,
-                                     MemStart, ExprEnd);
-      }
+  // We are now either at the end of the operand or at the '(' at the start of a
+  // base-index-scale-expr.
 
-      // Eat the '('.
-      Parser.Lex();
-    }
+  if (!parseOptionalToken(AsmToken::LParen)) {
+    if (SegReg == 0)
+      return X86Operand::CreateMem(getPointerWidth(), Disp, StartLoc, EndLoc);
+    return X86Operand::CreateMem(getPointerWidth(), SegReg, Disp, 0, 0, 1,
+                                 StartLoc, EndLoc);
   }
 
-  // If we reached here, then we just ate the ( of the memory operand.  Process
+  // If we reached here, then eat the '(' and Process
   // the rest of the memory operand.
   unsigned BaseReg = 0, IndexReg = 0, Scale = 1;
-  SMLoc IndexLoc, BaseLoc;
-
-  if (getLexer().is(AsmToken::Percent)) {
-    SMLoc StartLoc, EndLoc;
-    BaseLoc = Parser.getTok().getLoc();
-    if (ParseRegister(BaseReg, StartLoc, EndLoc)) return nullptr;
-    if (BaseReg == X86::EIZ || BaseReg == X86::RIZ) {
-      Error(StartLoc, "eiz and riz can only be used as index registers",
-            SMRange(StartLoc, EndLoc));
+  SMLoc BaseLoc = getLexer().getLoc();
+  const MCExpr *E;
+  StringRef ErrMsg;
+
+  // Parse BaseReg if one is provided.
+  if (getLexer().isNot(AsmToken::Comma) && getLexer().isNot(AsmToken::RParen)) {
+    if (Parser.parseExpression(E, EndLoc) ||
+        check(!isa<X86MCExpr>(E), BaseLoc, "expected register here"))
       return nullptr;
-    }
-  }
 
-  if (getLexer().is(AsmToken::Comma)) {
-    Parser.Lex(); // Eat the comma.
-    IndexLoc = Parser.getTok().getLoc();
+    // Sanity check register.
+    BaseReg = cast<X86MCExpr>(E)->getRegNo();
+    if (BaseReg == X86::EIZ || BaseReg == X86::RIZ)
+      return ErrorOperand(BaseLoc,
+                          "eiz and riz can only be used as index registers",
+                          SMRange(BaseLoc, EndLoc));
+  }
 
+  if (parseOptionalToken(AsmToken::Comma)) {
     // Following the comma we should have either an index register, or a scale
     // value. We don't support the later form, but we want to parse it
     // correctly.
     //
-    // Not that even though it would be completely consistent to support syntax
-    // like "1(%eax,,1)", the assembler doesn't. Use "eiz" or "riz" for this.
-    if (getLexer().is(AsmToken::Percent)) {
-      SMLoc L;
-      if (ParseRegister(IndexReg, L, L))
+    // Even though it would be completely consistent to support syntax like
+    // "1(%eax,,1)", the assembler doesn't. Use "eiz" or "riz" for this.
+    if (getLexer().isNot(AsmToken::RParen)) {
+      if (Parser.parseTokenLoc(Loc) || Parser.parseExpression(E, EndLoc))
         return nullptr;
-      if (BaseReg == X86::RIP) {
-        Error(IndexLoc, "%rip as base register can not have an index register");
-        return nullptr;
-      }
-      if (IndexReg == X86::RIP) {
-        Error(IndexLoc, "%rip is not allowed as an index register");
-        return nullptr;
-      }
 
-      if (getLexer().isNot(AsmToken::RParen)) {
-        // Parse the scale amount:
-        //  ::= ',' [scale-expression]
-        if (parseToken(AsmToken::Comma, "expected comma in scale expression"))
-          return nullptr;
-
-        if (getLexer().isNot(AsmToken::RParen)) {
-          SMLoc Loc = Parser.getTok().getLoc();
-
-          int64_t ScaleVal;
-          if (getParser().parseAbsoluteExpression(ScaleVal)){
-            Error(Loc, "expected scale expression");
-            return nullptr;
-          }
-
-          // Validate the scale amount.
-          if (X86MCRegisterClasses[X86::GR16RegClassID].contains(BaseReg) &&
-              ScaleVal != 1) {
-            Error(Loc, "scale factor in 16-bit address must be 1");
-            return nullptr;
-          }
-          if (ScaleVal != 1 && ScaleVal != 2 && ScaleVal != 4 &&
-              ScaleVal != 8) {
-            Error(Loc, "scale factor in address must be 1, 2, 4 or 8");
-            return nullptr;
+      if (!isa<X86MCExpr>(E)) {
+        // We've parsed an unexpected Scale Value instead of an index
+        // register. Interpret it as an absolute.
+        int64_t ScaleVal;
+        if (!E->evaluateAsAbsolute(ScaleVal, getStreamer().getAssemblerPtr()))
+          return ErrorOperand(Loc, "expected absolute expression");
+        if (ScaleVal != 1)
+          Warning(Loc, "scale factor without index register is ignored");
+        Scale = 1;
+      } else { // IndexReg Found.
+        IndexReg = cast<X86MCExpr>(E)->getRegNo();
+
+        if (BaseReg == X86::RIP)
+          return ErrorOperand(
+              Loc, "%rip as base register can not have an index register");
+        if (IndexReg == X86::RIP)
+          return ErrorOperand(Loc, "%rip is not allowed as an index register");
+
+        if (parseOptionalToken(AsmToken::Comma)) {
+          // Parse the scale amount:
+          //  ::= ',' [scale-expression]
+
+          // A scale amount without an index is ignored.
+          if (getLexer().isNot(AsmToken::RParen)) {
+            int64_t ScaleVal;
+            if (Parser.parseTokenLoc(Loc) ||
+                Parser.parseAbsoluteExpression(ScaleVal))
+              return ErrorOperand(Loc, "expected scale expression");
+            Scale = (unsigned)ScaleVal;
+            // Validate the scale amount.
+            if (X86MCRegisterClasses[X86::GR16RegClassID].contains(BaseReg) &&
+                Scale != 1)
+              return ErrorOperand(Loc,
+                                  "scale factor in 16-bit address must be 1");
+            if (checkScale(Scale, ErrMsg))
+              return ErrorOperand(Loc, ErrMsg);
           }
-          Scale = (unsigned)ScaleVal;
         }
       }
-    } else if (getLexer().isNot(AsmToken::RParen)) {
-      // A scale amount without an index is ignored.
-      // index.
-      SMLoc Loc = Parser.getTok().getLoc();
-
-      int64_t Value;
-      if (getParser().parseAbsoluteExpression(Value))
-        return nullptr;
-
-      if (Value != 1)
-        Warning(Loc, "scale factor without index register is ignored");
-      Scale = 1;
     }
   }
 
   // Ok, we've eaten the memory operand, verify we have a ')' and eat it too.
-  SMLoc MemEnd = Parser.getTok().getEndLoc();
   if (parseToken(AsmToken::RParen, "unexpected token in memory operand"))
     return nullptr;
 
-  // This is a terrible hack to handle "out[s]?[bwl]? %al, (%dx)" ->
-  // "outb %al, %dx".  Out doesn't take a memory form, but this is a widely
-  // documented form in various unofficial manuals, so a lot of code uses it.
-  if (BaseReg == X86::DX && IndexReg == 0 && Scale == 1 &&
-      SegReg == 0 && isa<MCConstantExpr>(Disp) &&
-      cast<MCConstantExpr>(Disp)->getValue() == 0)
+  // This is to support otherwise illegal operand (%dx) found in various
+  // unofficial manuals examples (e.g. "out[s]?[bwl]? %al, (%dx)") and must now
+  // be supported. Mark such DX variants separately fix only in special cases.
+  if (BaseReg == X86::DX && IndexReg == 0 && Scale == 1 && SegReg == 0 &&
+      isa<MCConstantExpr>(Disp) && cast<MCConstantExpr>(Disp)->getValue() == 0)
     return X86Operand::CreateDXReg(BaseLoc, BaseLoc);
 
-  StringRef ErrMsg;
   if (CheckBaseRegAndIndexRegAndScale(BaseReg, IndexReg, Scale, is64BitMode(),
-                                      ErrMsg)) {
-    Error(BaseLoc, ErrMsg);
-    return nullptr;
-  }
+                                      ErrMsg))
+    return ErrorOperand(BaseLoc, ErrMsg);
 
   if (SegReg || BaseReg || IndexReg)
     return X86Operand::CreateMem(getPointerWidth(), SegReg, Disp, BaseReg,
-                                 IndexReg, Scale, MemStart, MemEnd);
-  return X86Operand::CreateMem(getPointerWidth(), Disp, MemStart, MemEnd);
+                                 IndexReg, Scale, StartLoc, EndLoc);
+  return X86Operand::CreateMem(getPointerWidth(), Disp, StartLoc, EndLoc);
 }
 
 // Parse either a standard primary expression or a register.
 bool X86AsmParser::parsePrimaryExpr(const MCExpr *&Res, SMLoc &EndLoc) {
   MCAsmParser &Parser = getParser();
-  if (Parser.parsePrimaryExpr(Res, EndLoc)) {
+  // See if this is a register first.
+  if (getTok().is(AsmToken::Percent) ||
+      (isParsingIntelSyntax() && getTok().is(AsmToken::Identifier) &&
+       MatchRegisterName(Parser.getTok().getString()))) {
     SMLoc StartLoc = Parser.getTok().getLoc();
-    // Normal Expression parse fails, check if it could be a register.
     unsigned RegNo;
-    bool TryRegParse =
-        getTok().is(AsmToken::Percent) ||
-        (isParsingIntelSyntax() && getTok().is(AsmToken::Identifier));
-    if (!TryRegParse || ParseRegister(RegNo, StartLoc, EndLoc))
+    if (ParseRegister(RegNo, StartLoc, EndLoc))
       return true;
-    // Clear previous parse error and return correct expression.
-    Parser.clearPendingErrors();
     Res = X86MCExpr::create(RegNo, Parser.getContext());
     return false;
   }
-
-  return false;
+  return Parser.parsePrimaryExpr(Res, EndLoc);
 }
 
 bool X86AsmParser::ParseInstruction(ParseInstructionInfo &Info, StringRef Name,
@@ -2847,8 +2864,7 @@ static const char *getSubtargetFeatureName(uint64_t Val);
 void X86AsmParser::EmitInstruction(MCInst &Inst, OperandVector &Operands,
                                    MCStreamer &Out) {
   Instrumentation->InstrumentAndEmitInstruction(
-      Inst, Operands, getContext(), MII, Out,
-      getParser().shouldPrintSchedInfo());
+      Inst, Operands, getContext(), MII, Out);
 }
 
 bool X86AsmParser::MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
diff --git a/lib/Target/X86/AsmParser/X86AsmParserCommon.h b/lib/Target/X86/AsmParser/X86AsmParserCommon.h
index c45a3f14ef116c9618388d602efbff01fd4351ad..5bc979d1f18c1641621b17d06b2fb0df655da5fd 100644
--- a/lib/Target/X86/AsmParser/X86AsmParserCommon.h
+++ b/lib/Target/X86/AsmParser/X86AsmParserCommon.h
@@ -1,9 +1,8 @@
 //===-- X86AsmParserCommon.h - Common functions for X86AsmParser ---------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/X86/AsmParser/X86Operand.h b/lib/Target/X86/AsmParser/X86Operand.h
index 4d4aae0a1c6ab1bb6606cf5597b365a0bd92107f..0f489e4449749a3f5f0240b90a049955fca4d9aa 100644
--- a/lib/Target/X86/AsmParser/X86Operand.h
+++ b/lib/Target/X86/AsmParser/X86Operand.h
@@ -1,9 +1,8 @@
 //===- X86Operand.h - Parsed X86 machine instruction ------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/X86/CMakeLists.txt b/lib/Target/X86/CMakeLists.txt
index 524b4ae53be12f8df06ba4718ee3929ca108c560..645ca49f2a1983e7ff0f1ec913928d7331f67786 100644
--- a/lib/Target/X86/CMakeLists.txt
+++ b/lib/Target/X86/CMakeLists.txt
@@ -7,13 +7,13 @@ tablegen(LLVM X86GenCallingConv.inc -gen-callingconv)
 tablegen(LLVM X86GenDAGISel.inc -gen-dag-isel)
 tablegen(LLVM X86GenDisassemblerTables.inc -gen-disassembler)
 tablegen(LLVM X86GenEVEX2VEXTables.inc -gen-x86-EVEX2VEX-tables)
+tablegen(LLVM X86GenExegesis.inc -gen-exegesis)
 tablegen(LLVM X86GenFastISel.inc -gen-fast-isel)
 tablegen(LLVM X86GenGlobalISel.inc -gen-global-isel)
 tablegen(LLVM X86GenInstrInfo.inc -gen-instr-info)
 tablegen(LLVM X86GenRegisterBank.inc -gen-register-bank)
 tablegen(LLVM X86GenRegisterInfo.inc -gen-register-info)
 tablegen(LLVM X86GenSubtargetInfo.inc -gen-subtarget)
-tablegen(LLVM X86GenExegesis.inc -gen-exegesis)
 
 if (X86_GEN_FOLD_TABLES)
   tablegen(LLVM X86GenFoldTables.inc -gen-x86-fold-tables)
diff --git a/lib/Target/X86/Disassembler/LLVMBuild.txt b/lib/Target/X86/Disassembler/LLVMBuild.txt
index e003fc9f996e5175acd51bac9a75cfbdba3887b9..c250c5ec4755d7de244b1a3b25617b18e4d83822 100644
--- a/lib/Target/X86/Disassembler/LLVMBuild.txt
+++ b/lib/Target/X86/Disassembler/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/Target/X86/Disassembler/LLVMBuild.txt --------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Target/X86/Disassembler/X86Disassembler.cpp b/lib/Target/X86/Disassembler/X86Disassembler.cpp
index 62312777318e93a7cd692d65370ff1a6d3427b63..8337995622223a11ebf6cfc90176b2e8d279a608 100644
--- a/lib/Target/X86/Disassembler/X86Disassembler.cpp
+++ b/lib/Target/X86/Disassembler/X86Disassembler.cpp
@@ -1,9 +1,8 @@
 //===-- X86Disassembler.cpp - Disassembler for x86 and x86_64 -------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/X86/Disassembler/X86DisassemblerDecoder.cpp b/lib/Target/X86/Disassembler/X86DisassemblerDecoder.cpp
index 54d550b606520f2cf4dabc0fd6c40cee6db85fc9..9a7bcc8bd5d1020c75c8feae806c9135b18041bc 100644
--- a/lib/Target/X86/Disassembler/X86DisassemblerDecoder.cpp
+++ b/lib/Target/X86/Disassembler/X86DisassemblerDecoder.cpp
@@ -1,9 +1,8 @@
 //===-- X86DisassemblerDecoder.cpp - Disassembler decoder -----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/X86/Disassembler/X86DisassemblerDecoder.h b/lib/Target/X86/Disassembler/X86DisassemblerDecoder.h
index 3b8a4f732eedca583ebd87a9bbba45ec40cc8066..8e0749d10cffe23cae3fd2207078decb2824fe2a 100644
--- a/lib/Target/X86/Disassembler/X86DisassemblerDecoder.h
+++ b/lib/Target/X86/Disassembler/X86DisassemblerDecoder.h
@@ -1,9 +1,8 @@
 //===-- X86DisassemblerDecoderInternal.h - Disassembler decoder -*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/X86/InstPrinter/LLVMBuild.txt b/lib/Target/X86/InstPrinter/LLVMBuild.txt
index 6868ddefa51fd302efd6ec6c33ec72f8f2c35ede..48676d4893a7570778129b491a781373f410273e 100644
--- a/lib/Target/X86/InstPrinter/LLVMBuild.txt
+++ b/lib/Target/X86/InstPrinter/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/Target/X86/InstPrinter/LLVMBuild.txt ---------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Target/X86/InstPrinter/X86ATTInstPrinter.cpp b/lib/Target/X86/InstPrinter/X86ATTInstPrinter.cpp
index 0e861d5ddbc9d99c2e4e9f12825e56023c39e910..b70ce4bab6fff45e59412220f5462cd73134e0f1 100644
--- a/lib/Target/X86/InstPrinter/X86ATTInstPrinter.cpp
+++ b/lib/Target/X86/InstPrinter/X86ATTInstPrinter.cpp
@@ -1,9 +1,8 @@
 //===-- X86ATTInstPrinter.cpp - AT&T assembly instruction printing --------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -200,3 +199,14 @@ void X86ATTInstPrinter::printU8Imm(const MCInst *MI, unsigned Op,
   O << markup("<imm:") << '$' << formatImm(MI->getOperand(Op).getImm() & 0xff)
     << markup(">");
 }
+
+void X86ATTInstPrinter::printSTiRegOperand(const MCInst *MI, unsigned OpNo,
+                                           raw_ostream &OS) {
+  const MCOperand &Op = MI->getOperand(OpNo);
+  unsigned Reg = Op.getReg();
+  // Override the default printing to print st(0) instead st.
+  if (Reg == X86::ST0)
+    OS << markup("<reg:") << "%st(0)" << markup(">");
+  else
+    printRegName(OS, Reg);
+}
diff --git a/lib/Target/X86/InstPrinter/X86ATTInstPrinter.h b/lib/Target/X86/InstPrinter/X86ATTInstPrinter.h
index 57422bc9a0b2a3b03166b1f7c21e19eae57f24ea..977b4445589267694ded77492c20266cf42b621b 100644
--- a/lib/Target/X86/InstPrinter/X86ATTInstPrinter.h
+++ b/lib/Target/X86/InstPrinter/X86ATTInstPrinter.h
@@ -1,9 +1,8 @@
 //=- X86ATTInstPrinter.h - Convert X86 MCInst to assembly syntax --*- C++ -*-=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -44,6 +43,7 @@ public:
   void printSrcIdx(const MCInst *MI, unsigned Op, raw_ostream &O);
   void printDstIdx(const MCInst *MI, unsigned Op, raw_ostream &O);
   void printU8Imm(const MCInst *MI, unsigned Op, raw_ostream &OS);
+  void printSTiRegOperand(const MCInst *MI, unsigned OpNo, raw_ostream &OS);
 
   void printanymem(const MCInst *MI, unsigned OpNo, raw_ostream &O) {
     printMemReference(MI, OpNo, O);
diff --git a/lib/Target/X86/InstPrinter/X86InstComments.cpp b/lib/Target/X86/InstPrinter/X86InstComments.cpp
index 37bed37b09946bfa807d434af920b0f55dc20356..a6d07c1c5be835fabbc515aae2d1a86102b3000f 100644
--- a/lib/Target/X86/InstPrinter/X86InstComments.cpp
+++ b/lib/Target/X86/InstPrinter/X86InstComments.cpp
@@ -1,9 +1,8 @@
 //===-- X86InstComments.cpp - Generate verbose-asm comments for instrs ----===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -1304,6 +1303,7 @@ bool llvm::EmitAnyX86InstComments(const MCInst *MI, raw_ostream &OS,
     OS << ']';
     --i; // For loop increments element #.
   }
+  OS << '\n';
 
   // We successfully added a comment to this instruction.
   return true;
diff --git a/lib/Target/X86/InstPrinter/X86InstComments.h b/lib/Target/X86/InstPrinter/X86InstComments.h
index 40dffa5fbb8a8bd36bcd0b3bef731992423e637f..05f1683848a89230c6d216067cc2feaac779687c 100644
--- a/lib/Target/X86/InstPrinter/X86InstComments.h
+++ b/lib/Target/X86/InstPrinter/X86InstComments.h
@@ -1,9 +1,8 @@
 //=- X86InstComments.h - Generate verbose-asm comments for instrs -*- C++ -*-=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/X86/InstPrinter/X86InstPrinterCommon.cpp b/lib/Target/X86/InstPrinter/X86InstPrinterCommon.cpp
index 432cd47ae4994a6020dbe99ad27af0e4d8e03d4b..f31641fd733e66760ad296d4bc4200356d094d26 100644
--- a/lib/Target/X86/InstPrinter/X86InstPrinterCommon.cpp
+++ b/lib/Target/X86/InstPrinter/X86InstPrinterCommon.cpp
@@ -1,9 +1,8 @@
 //===--- X86InstPrinterCommon.cpp - X86 assembly instruction printing -----===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/X86/InstPrinter/X86InstPrinterCommon.h b/lib/Target/X86/InstPrinter/X86InstPrinterCommon.h
index f2875e71f22cdd0bf80d41306a441d703721a113..917f7121807445aa69a021512a6f3d476fb617c8 100644
--- a/lib/Target/X86/InstPrinter/X86InstPrinterCommon.h
+++ b/lib/Target/X86/InstPrinter/X86InstPrinterCommon.h
@@ -1,9 +1,8 @@
 //===-- X86InstPrinterCommon.cpp - X86 assembly instruction printing ------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/X86/InstPrinter/X86IntelInstPrinter.cpp b/lib/Target/X86/InstPrinter/X86IntelInstPrinter.cpp
index 044b715641520edb0dfc51c78d95952eca56d8d7..9015aa14768c6ff07a2a1a7399b675610f53d444 100644
--- a/lib/Target/X86/InstPrinter/X86IntelInstPrinter.cpp
+++ b/lib/Target/X86/InstPrinter/X86IntelInstPrinter.cpp
@@ -1,9 +1,8 @@
 //===-- X86IntelInstPrinter.cpp - Intel assembly instruction printing -----===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -160,3 +159,14 @@ void X86IntelInstPrinter::printU8Imm(const MCInst *MI, unsigned Op,
 
   O << formatImm(MI->getOperand(Op).getImm() & 0xff);
 }
+
+void X86IntelInstPrinter::printSTiRegOperand(const MCInst *MI, unsigned OpNo,
+                                            raw_ostream &OS) {
+  const MCOperand &Op = MI->getOperand(OpNo);
+  unsigned Reg = Op.getReg();
+  // Override the default printing to print st(0) instead st.
+  if (Reg == X86::ST0)
+    OS << "st(0)";
+  else
+    printRegName(OS, Reg);
+}
diff --git a/lib/Target/X86/InstPrinter/X86IntelInstPrinter.h b/lib/Target/X86/InstPrinter/X86IntelInstPrinter.h
index 3b34a8052becba52ae602c0675d00629e20fd0c0..5155dee178a7dec08eef49662d36335ffe58f885 100644
--- a/lib/Target/X86/InstPrinter/X86IntelInstPrinter.h
+++ b/lib/Target/X86/InstPrinter/X86IntelInstPrinter.h
@@ -1,9 +1,8 @@
 //= X86IntelInstPrinter.h - Convert X86 MCInst to assembly syntax -*- C++ -*-=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -39,6 +38,7 @@ public:
   void printSrcIdx(const MCInst *MI, unsigned OpNo, raw_ostream &O);
   void printDstIdx(const MCInst *MI, unsigned OpNo, raw_ostream &O);
   void printU8Imm(const MCInst *MI, unsigned Op, raw_ostream &O);
+  void printSTiRegOperand(const MCInst *MI, unsigned OpNo, raw_ostream &OS);
 
   void printanymem(const MCInst *MI, unsigned OpNo, raw_ostream &O) {
     printMemReference(MI, OpNo, O);
diff --git a/lib/Target/X86/LLVMBuild.txt b/lib/Target/X86/LLVMBuild.txt
index 055336baac195e30b1e50813f507e7e7eefb1234..f65dde68db0ef780669d7b7af68719774d9c6bb6 100644
--- a/lib/Target/X86/LLVMBuild.txt
+++ b/lib/Target/X86/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/Target/X86/LLVMBuild.txt ---------------------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Target/X86/MCTargetDesc/LLVMBuild.txt b/lib/Target/X86/MCTargetDesc/LLVMBuild.txt
index b9fdc9c483fd17171e54914cd5588f2e4010f20c..7bcf75e89d7a02e60e6170a23893c33082071c79 100644
--- a/lib/Target/X86/MCTargetDesc/LLVMBuild.txt
+++ b/lib/Target/X86/MCTargetDesc/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/Target/X86/MCTargetDesc/LLVMBuild.txt --------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp b/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp
index 0e4c4398e49d323df4c7e71fe5b71e996ea7b8b2..1e32d002afd8ef33f797b762fdc6b4416df6214f 100644
--- a/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp
+++ b/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp
@@ -1,9 +1,8 @@
 //===-- X86AsmBackend.cpp - X86 Assembler Backend -------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -540,7 +539,6 @@ protected:
     unsigned InstrOffset = 0;
     unsigned StackAdjust = 0;
     unsigned StackSize = 0;
-    unsigned PrevStackSize = 0;
     unsigned NumDefCFAOffsets = 0;
 
     for (unsigned i = 0, e = Instrs.size(); i != e; ++i) {
@@ -588,7 +586,6 @@ protected:
         //  L0:
         //     .cfi_def_cfa_offset 80
         //
-        PrevStackSize = StackSize;
         StackSize = std::abs(Inst.getOffset()) / StackDivide;
         ++NumDefCFAOffsets;
         break;
@@ -635,16 +632,6 @@ protected:
       CompactUnwindEncoding |= (StackAdjust & 0xFF) << 16;
       CompactUnwindEncoding |= RegEnc & CU::UNWIND_BP_FRAME_REGISTERS;
     } else {
-      // If the amount of the stack allocation is the size of a register, then
-      // we "push" the RAX/EAX register onto the stack instead of adjusting the
-      // stack pointer with a SUB instruction. We don't support the push of the
-      // RAX/EAX register with compact unwind. So we check for that situation
-      // here.
-      if ((NumDefCFAOffsets == SavedRegIdx + 1 &&
-           StackSize - PrevStackSize == 1) ||
-          (Instrs.size() == 1 && NumDefCFAOffsets == 1 && StackSize == 2))
-        return CU::UNWIND_MODE_DWARF;
-
       SubtractInstrIdx += InstrOffset;
       ++StackAdjust;
 
diff --git a/lib/Target/X86/MCTargetDesc/X86BaseInfo.h b/lib/Target/X86/MCTargetDesc/X86BaseInfo.h
index c85ce9bbd5a45c9c4fd7cfc730aadcc0c7374e79..62dd685b36078aaa539526b149a52e81c1f1cdc9 100644
--- a/lib/Target/X86/MCTargetDesc/X86BaseInfo.h
+++ b/lib/Target/X86/MCTargetDesc/X86BaseInfo.h
@@ -1,9 +1,8 @@
 //===-- X86BaseInfo.h - Top level definitions for X86 -------- --*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -60,9 +59,7 @@ namespace X86 {
     IP_HAS_REPEAT_NE = 4,
     IP_HAS_REPEAT = 8,
     IP_HAS_LOCK = 16,
-    NO_SCHED_INFO = 32, // Don't add sched comment to the current instr because
-                        // it was already added
-    IP_HAS_NOTRACK = 64
+    IP_HAS_NOTRACK = 32
   };
 } // end namespace X86;
 
diff --git a/lib/Target/X86/MCTargetDesc/X86ELFObjectWriter.cpp b/lib/Target/X86/MCTargetDesc/X86ELFObjectWriter.cpp
index b724a89f81d2e5c447a76d6d2e7e567a18c327c8..910a9fcc3c351219fb2ea52e95bd348e26321b26 100644
--- a/lib/Target/X86/MCTargetDesc/X86ELFObjectWriter.cpp
+++ b/lib/Target/X86/MCTargetDesc/X86ELFObjectWriter.cpp
@@ -1,9 +1,8 @@
 //===-- X86ELFObjectWriter.cpp - X86 ELF Writer ---------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/X86/MCTargetDesc/X86FixupKinds.h b/lib/Target/X86/MCTargetDesc/X86FixupKinds.h
index 3c04b13e002e05bc52453a17bfa1424e94441ece..2d5217115d07cb0bf72c1d4900959401a6b80ca1 100644
--- a/lib/Target/X86/MCTargetDesc/X86FixupKinds.h
+++ b/lib/Target/X86/MCTargetDesc/X86FixupKinds.h
@@ -1,9 +1,8 @@
 //===-- X86FixupKinds.h - X86 Specific Fixup Entries ------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/X86/MCTargetDesc/X86MCAsmInfo.cpp b/lib/Target/X86/MCTargetDesc/X86MCAsmInfo.cpp
index fa7c352a1b63f409ae71a0d0e3ed6eda4d5eb39d..e1125c176b253488d162ec0475cef48e078be90d 100644
--- a/lib/Target/X86/MCTargetDesc/X86MCAsmInfo.cpp
+++ b/lib/Target/X86/MCTargetDesc/X86MCAsmInfo.cpp
@@ -1,9 +1,8 @@
 //===-- X86MCAsmInfo.cpp - X86 asm properties -----------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/X86/MCTargetDesc/X86MCAsmInfo.h b/lib/Target/X86/MCTargetDesc/X86MCAsmInfo.h
index 30d5c802d1ed521b768c3f3c466f17a853477673..b2369647a40f21bf96f1db70526a95e042832f81 100644
--- a/lib/Target/X86/MCTargetDesc/X86MCAsmInfo.h
+++ b/lib/Target/X86/MCTargetDesc/X86MCAsmInfo.h
@@ -1,9 +1,8 @@
 //===-- X86MCAsmInfo.h - X86 asm properties --------------------*- C++ -*--===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/X86/MCTargetDesc/X86MCCodeEmitter.cpp b/lib/Target/X86/MCTargetDesc/X86MCCodeEmitter.cpp
index f5371db9e77a3a8e5f5b6e1ba4b17da0ee41147c..fd95387b0766331aef772c9d9d9b4e5c648900a8 100644
--- a/lib/Target/X86/MCTargetDesc/X86MCCodeEmitter.cpp
+++ b/lib/Target/X86/MCTargetDesc/X86MCCodeEmitter.cpp
@@ -1,9 +1,8 @@
 //===-- X86MCCodeEmitter.cpp - Convert X86 code to machine code -----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/X86/MCTargetDesc/X86MCExpr.h b/lib/Target/X86/MCTargetDesc/X86MCExpr.h
index 1070f70468fa2870c426913625ab5591f1a589eb..cbda873e92ad2560bc960896630254111f031f34 100644
--- a/lib/Target/X86/MCTargetDesc/X86MCExpr.h
+++ b/lib/Target/X86/MCTargetDesc/X86MCExpr.h
@@ -1,9 +1,8 @@
 //=--- X86MCExpr.h - X86 specific MC expression classes ---*- C++ -*-=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/X86/MCTargetDesc/X86MCTargetDesc.cpp b/lib/Target/X86/MCTargetDesc/X86MCTargetDesc.cpp
index ea4aaf14223d1ff8545218a79e9966d8bca104df..4a1f481f4a92628236b7057854a05afa7850d549 100644
--- a/lib/Target/X86/MCTargetDesc/X86MCTargetDesc.cpp
+++ b/lib/Target/X86/MCTargetDesc/X86MCTargetDesc.cpp
@@ -1,9 +1,8 @@
 //===-- X86MCTargetDesc.cpp - X86 Target Descriptions ---------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/X86/MCTargetDesc/X86MCTargetDesc.h b/lib/Target/X86/MCTargetDesc/X86MCTargetDesc.h
index 4e9f5ba60d2e4ebc16837d4a90467c7c4451222c..ff8487312e35ecfb2eea2ff8fd16215959217698 100644
--- a/lib/Target/X86/MCTargetDesc/X86MCTargetDesc.h
+++ b/lib/Target/X86/MCTargetDesc/X86MCTargetDesc.h
@@ -1,9 +1,8 @@
 //===-- X86MCTargetDesc.h - X86 Target Descriptions -------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/X86/MCTargetDesc/X86MachObjectWriter.cpp b/lib/Target/X86/MCTargetDesc/X86MachObjectWriter.cpp
index 883278b7bc1f11ee70d49b94ca97f3566f6999c1..fc7e99f61e5e67fb333d6a1ec2f13511dd4d35de 100644
--- a/lib/Target/X86/MCTargetDesc/X86MachObjectWriter.cpp
+++ b/lib/Target/X86/MCTargetDesc/X86MachObjectWriter.cpp
@@ -1,9 +1,8 @@
 //===-- X86MachObjectWriter.cpp - X86 Mach-O Writer -----------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/X86/MCTargetDesc/X86TargetStreamer.h b/lib/Target/X86/MCTargetDesc/X86TargetStreamer.h
index 10a282dd2962e026844684c029f71df498ba7b4b..3b1e9e7c34fb04b8ae26b152c724a8f53152a331 100644
--- a/lib/Target/X86/MCTargetDesc/X86TargetStreamer.h
+++ b/lib/Target/X86/MCTargetDesc/X86TargetStreamer.h
@@ -1,9 +1,8 @@
 //===- X86TargetStreamer.h ------------------------------*- C++ -*---------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/X86/MCTargetDesc/X86WinCOFFObjectWriter.cpp b/lib/Target/X86/MCTargetDesc/X86WinCOFFObjectWriter.cpp
index 2aec695b2dbf1877c2bb9b70e8d865ddf151d0db..3baab9da1c41e7bb7b78d4515e022aeb8f924136 100644
--- a/lib/Target/X86/MCTargetDesc/X86WinCOFFObjectWriter.cpp
+++ b/lib/Target/X86/MCTargetDesc/X86WinCOFFObjectWriter.cpp
@@ -1,9 +1,8 @@
 //===-- X86WinCOFFObjectWriter.cpp - X86 Win COFF Writer ------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/X86/MCTargetDesc/X86WinCOFFStreamer.cpp b/lib/Target/X86/MCTargetDesc/X86WinCOFFStreamer.cpp
index 0085787e576a8bcaa11f369ccf1ab47d5d9fb66a..796a27a1725515b602ad524aa04458af29b29c32 100644
--- a/lib/Target/X86/MCTargetDesc/X86WinCOFFStreamer.cpp
+++ b/lib/Target/X86/MCTargetDesc/X86WinCOFFStreamer.cpp
@@ -1,9 +1,8 @@
 //===-- X86WinCOFFStreamer.cpp - X86 Target WinCOFF Streamer ----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/X86/MCTargetDesc/X86WinCOFFTargetStreamer.cpp b/lib/Target/X86/MCTargetDesc/X86WinCOFFTargetStreamer.cpp
index bee9b7046338f7a9cd3181ae2595ac550d0d0308..e9987d1f62bd0632cadc38d4c228948fc12efbe0 100644
--- a/lib/Target/X86/MCTargetDesc/X86WinCOFFTargetStreamer.cpp
+++ b/lib/Target/X86/MCTargetDesc/X86WinCOFFTargetStreamer.cpp
@@ -1,9 +1,8 @@
 //===-- X86WinCOFFTargetStreamer.cpp ----------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/X86/README-SSE.txt b/lib/Target/X86/README-SSE.txt
index 190ca861967a8737617d7ba9298a77d89a3f554c..d52840e5c489677f0868823951b2dc66beaf8172 100644
--- a/lib/Target/X86/README-SSE.txt
+++ b/lib/Target/X86/README-SSE.txt
@@ -827,15 +827,3 @@ and inversion with an rsqrtss instruction, which computes 1/sqrt faster at the
 cost of reduced accuracy.
 
 //===---------------------------------------------------------------------===//
-
-This function should be matched to haddpd when the appropriate CPU is enabled:
-
-#include <x86intrin.h>
-double f (__m128d p) {
-  return p[0] + p[1];
-}
-
-similarly, v[0]-v[1] should match to hsubpd, and {v[0]-v[1], w[0]-w[1]} should
-turn into hsubpd also.
-
-//===---------------------------------------------------------------------===//
diff --git a/lib/Target/X86/ShadowCallStack.cpp b/lib/Target/X86/ShadowCallStack.cpp
index ab2cebcb58ee827acad25dc9e80cafc7bb1a0bdc..6b7f33a2feaca17109022962aae5d7e8f784241b 100644
--- a/lib/Target/X86/ShadowCallStack.cpp
+++ b/lib/Target/X86/ShadowCallStack.cpp
@@ -1,9 +1,8 @@
 //===------- ShadowCallStack.cpp - Shadow Call Stack pass -----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/X86/TargetInfo/LLVMBuild.txt b/lib/Target/X86/TargetInfo/LLVMBuild.txt
index 6a52ea61d87e813c049298830a277c1b6ef5e747..89e6dd33887443fef6d47a0c9693f900e700411b 100644
--- a/lib/Target/X86/TargetInfo/LLVMBuild.txt
+++ b/lib/Target/X86/TargetInfo/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/Target/X86/TargetInfo/LLVMBuild.txt ----------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Target/X86/TargetInfo/X86TargetInfo.cpp b/lib/Target/X86/TargetInfo/X86TargetInfo.cpp
index 16c2b56c48b57c88872690aae3159734782b5f93..a1710980b8919c8909633aeedf473ee9944f9d05 100644
--- a/lib/Target/X86/TargetInfo/X86TargetInfo.cpp
+++ b/lib/Target/X86/TargetInfo/X86TargetInfo.cpp
@@ -1,9 +1,8 @@
 //===-- X86TargetInfo.cpp - X86 Target Implementation ---------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/X86/Utils/LLVMBuild.txt b/lib/Target/X86/Utils/LLVMBuild.txt
index fdb886f53a082030076fb6ed61b96ba57c43718c..6460c48884bd00cf656db22137b73bcb46ead3f4 100644
--- a/lib/Target/X86/Utils/LLVMBuild.txt
+++ b/lib/Target/X86/Utils/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/Target/X86/Utils/LLVMBuild.txt ---------------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Target/X86/Utils/X86ShuffleDecode.cpp b/lib/Target/X86/Utils/X86ShuffleDecode.cpp
index bed940d0d0e9ab759c5ae2109ad3a3dd2b7f5b0e..f78e21b684c19075e8fba4efafcb31fcf089d29f 100644
--- a/lib/Target/X86/Utils/X86ShuffleDecode.cpp
+++ b/lib/Target/X86/Utils/X86ShuffleDecode.cpp
@@ -1,9 +1,8 @@
 //===-- X86ShuffleDecode.cpp - X86 shuffle decode logic -------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/X86/Utils/X86ShuffleDecode.h b/lib/Target/X86/Utils/X86ShuffleDecode.h
index 85cde14a32410a84e663f76e556cf582a7dadcc5..d1678d57a8f914fc5d7b5fee2b58e547370332ff 100644
--- a/lib/Target/X86/Utils/X86ShuffleDecode.h
+++ b/lib/Target/X86/Utils/X86ShuffleDecode.h
@@ -1,9 +1,8 @@
 //===-- X86ShuffleDecode.h - X86 shuffle decode logic -----------*-C++-*---===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/X86/X86.h b/lib/Target/X86/X86.h
index 1c8813815b863a339732d69feb4e163a5de9ed05..60e9cb9c33da79a73471e4e1d0d1c53ee205248f 100644
--- a/lib/Target/X86/X86.h
+++ b/lib/Target/X86/X86.h
@@ -1,9 +1,8 @@
 //===-- X86.h - Top-level interface for X86 representation ------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/X86/X86.td b/lib/Target/X86/X86.td
index 6b1749fc75005965e9e6908cc869ce965070031b..4c5b98c49149d1a72efcbc458a1f6bc9266529b3 100644
--- a/lib/Target/X86/X86.td
+++ b/lib/Target/X86/X86.td
@@ -1,9 +1,8 @@
 //===-- X86.td - Target definition file for the Intel X86 --*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/X86/X86AsmPrinter.cpp b/lib/Target/X86/X86AsmPrinter.cpp
index b4be9edf1666b7f7ef0bf701812c1b923b84da5a..cd2ae8249822ceaad0d2e398fdb30101c276e674 100644
--- a/lib/Target/X86/X86AsmPrinter.cpp
+++ b/lib/Target/X86/X86AsmPrinter.cpp
@@ -1,9 +1,8 @@
 //===-- X86AsmPrinter.cpp - Convert X86 LLVM code to AT&T assembly --------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -254,6 +253,11 @@ static void printOperand(X86AsmPrinter &P, const MachineInstr *MI,
     printSymbolOperand(P, MO, O);
     break;
   }
+  case MachineOperand::MO_BlockAddress: {
+    MCSymbol *Sym = P.GetBlockAddressSymbol(MO.getBlockAddress());
+    Sym->print(O, P.MAI);
+    break;
+  }
   }
 }
 
@@ -572,9 +576,9 @@ void X86AsmPrinter::EmitStartOfAsmFile(Module &M) {
 
       // Emitting an Elf_Prop for the CET properties.
       OutStreamer->EmitIntValue(ELF::GNU_PROPERTY_X86_FEATURE_1_AND, 4);
-      OutStreamer->EmitIntValue(WordSize, 4);               // data size
-      OutStreamer->EmitIntValue(FeatureFlagsAnd, WordSize); // data
-      EmitAlignment(WordSize == 4 ? 2 : 3);                 // padding
+      OutStreamer->EmitIntValue(4, 4);               // data size
+      OutStreamer->EmitIntValue(FeatureFlagsAnd, 4); // data
+      EmitAlignment(WordSize == 4 ? 2 : 3);          // padding
 
       OutStreamer->endSection(Nt);
       OutStreamer->SwitchSection(Cur);
@@ -683,26 +687,31 @@ void X86AsmPrinter::EmitEndOfAsmFile(Module &M) {
     // stripping. Since LLVM never generates code that does this, it is always
     // safe to set.
     OutStreamer->EmitAssemblerFlag(MCAF_SubsectionsViaSymbols);
-    return;
-  }
-
-  if (TT.isKnownWindowsMSVCEnvironment() && MMI->usesVAFloatArgument()) {
-    StringRef SymbolName =
-        (TT.getArch() == Triple::x86_64) ? "_fltused" : "__fltused";
-    MCSymbol *S = MMI->getContext().getOrCreateSymbol(SymbolName);
-    OutStreamer->EmitSymbolAttribute(S, MCSA_Global);
-    return;
-  }
-
-  if (TT.isOSBinFormatCOFF()) {
+  } else if (TT.isOSBinFormatCOFF()) {
+    if (MMI->usesMSVCFloatingPoint()) {
+      // In Windows' libcmt.lib, there is a file which is linked in only if the
+      // symbol _fltused is referenced. Linking this in causes some
+      // side-effects:
+      //
+      // 1. For x86-32, it will set the x87 rounding mode to 53-bit instead of
+      // 64-bit mantissas at program start.
+      //
+      // 2. It links in support routines for floating-point in scanf and printf.
+      //
+      // MSVC emits an undefined reference to _fltused when there are any
+      // floating point operations in the program (including calls). A program
+      // that only has: `scanf("%f", &global_float);` may fail to trigger this,
+      // but oh well...that's a documented issue.
+      StringRef SymbolName =
+          (TT.getArch() == Triple::x86) ? "__fltused" : "_fltused";
+      MCSymbol *S = MMI->getContext().getOrCreateSymbol(SymbolName);
+      OutStreamer->EmitSymbolAttribute(S, MCSA_Global);
+      return;
+    }
     emitStackMaps(SM);
-    return;
-  }
-
-  if (TT.isOSBinFormatELF()) {
+  } else if (TT.isOSBinFormatELF()) {
     emitStackMaps(SM);
     FM.serializeToFaultMapSection();
-    return;
   }
 }
 
diff --git a/lib/Target/X86/X86AsmPrinter.h b/lib/Target/X86/X86AsmPrinter.h
index 55abdf2ba6018a0e4766595d8b6604d4288f3b22..d7c8b8d3930cb82f4913c2c22f7b168803f0cbeb 100644
--- a/lib/Target/X86/X86AsmPrinter.h
+++ b/lib/Target/X86/X86AsmPrinter.h
@@ -1,9 +1,8 @@
 //===-- X86AsmPrinter.h - X86 implementation of AsmPrinter ------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/X86/X86AvoidStoreForwardingBlocks.cpp b/lib/Target/X86/X86AvoidStoreForwardingBlocks.cpp
index 2850baf7a65e7a898190f1d5d7943e4085fd43cc..3ac0b1ae5143d7aaaec5b7275698cd113b53ab4e 100644
--- a/lib/Target/X86/X86AvoidStoreForwardingBlocks.cpp
+++ b/lib/Target/X86/X86AvoidStoreForwardingBlocks.cpp
@@ -1,9 +1,8 @@
 //===- X86AvoidStoreForwardingBlockis.cpp - Avoid HW Store Forward Block --===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -641,21 +640,22 @@ removeRedundantBlockingStores(DisplacementSizeMap &BlockingStoresDispSizeMap) {
   if (BlockingStoresDispSizeMap.size() <= 1)
     return;
 
-  int64_t PrevDisp = BlockingStoresDispSizeMap.begin()->first;
-  unsigned PrevSize = BlockingStoresDispSizeMap.begin()->second;
-  SmallVector<int64_t, 2> ForRemoval;
-  for (auto DispSizePair = std::next(BlockingStoresDispSizeMap.begin());
-       DispSizePair != BlockingStoresDispSizeMap.end(); ++DispSizePair) {
-    int64_t CurrDisp = DispSizePair->first;
-    unsigned CurrSize = DispSizePair->second;
-    if (CurrDisp + CurrSize <= PrevDisp + PrevSize) {
-      ForRemoval.push_back(PrevDisp);
+  SmallVector<std::pair<int64_t, unsigned>, 0> DispSizeStack;
+  for (auto DispSizePair : BlockingStoresDispSizeMap) {
+    int64_t CurrDisp = DispSizePair.first;
+    unsigned CurrSize = DispSizePair.second;
+    while (DispSizeStack.size()) {
+      int64_t PrevDisp = DispSizeStack.back().first;
+      unsigned PrevSize = DispSizeStack.back().second;
+      if (CurrDisp + CurrSize > PrevDisp + PrevSize)
+        break;
+      DispSizeStack.pop_back();
     }
-    PrevDisp = CurrDisp;
-    PrevSize = CurrSize;
+    DispSizeStack.push_back(DispSizePair);
   }
-  for (auto Disp : ForRemoval)
-    BlockingStoresDispSizeMap.erase(Disp);
+  BlockingStoresDispSizeMap.clear();
+  for (auto Disp : DispSizeStack)
+    BlockingStoresDispSizeMap.insert(Disp);
 }
 
 bool X86AvoidSFBPass::runOnMachineFunction(MachineFunction &MF) {
diff --git a/lib/Target/X86/X86CallFrameOptimization.cpp b/lib/Target/X86/X86CallFrameOptimization.cpp
index 24d7a219e7510132b10d6d3ed6c23709711c25f1..b38e2bbe492c2a730a941821aed90129d8426087 100644
--- a/lib/Target/X86/X86CallFrameOptimization.cpp
+++ b/lib/Target/X86/X86CallFrameOptimization.cpp
@@ -1,9 +1,8 @@
 //===----- X86CallFrameOptimization.cpp - Optimize x86 call sequences -----===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/X86/X86CallLowering.cpp b/lib/Target/X86/X86CallLowering.cpp
index 1dc83b76595d510db7b80c98dc5d4989ecf6a964..048e4ca79330a1faa14b8a2e60c16d8bb483936e 100644
--- a/lib/Target/X86/X86CallLowering.cpp
+++ b/lib/Target/X86/X86CallLowering.cpp
@@ -1,9 +1,8 @@
 //===- llvm/lib/Target/X86/X86CallLowering.cpp - Call lowering ------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -48,8 +47,6 @@
 
 using namespace llvm;
 
-#include "X86GenCallingConv.inc"
-
 X86CallLowering::X86CallLowering(const X86TargetLowering &TLI)
     : CallLowering(&TLI) {}
 
@@ -151,7 +148,7 @@ struct OutgoingValueHandler : public CallLowering::ValueHandler {
     unsigned ExtReg = extendRegister(ValVReg, VA);
     auto MMO = MIRBuilder.getMF().getMachineMemOperand(
         MPO, MachineMemOperand::MOStore, VA.getLocVT().getStoreSize(),
-        /* Alignment */ 0);
+        /* Alignment */ 1);
     MIRBuilder.buildStore(ExtReg, Addr, *MMO);
   }
 
@@ -247,7 +244,7 @@ struct IncomingValueHandler : public CallLowering::ValueHandler {
                             MachinePointerInfo &MPO, CCValAssign &VA) override {
     auto MMO = MIRBuilder.getMF().getMachineMemOperand(
         MPO, MachineMemOperand::MOLoad | MachineMemOperand::MOInvariant, Size,
-        0);
+        1);
     MIRBuilder.buildLoad(ValVReg, Addr, *MMO);
   }
 
diff --git a/lib/Target/X86/X86CallLowering.h b/lib/Target/X86/X86CallLowering.h
index f5f8f9a3ef6d02b4cb41d54389d842850bb66001..90e99ca00fb3e1f6d3ffbf986bc15f888431c7e9 100644
--- a/lib/Target/X86/X86CallLowering.h
+++ b/lib/Target/X86/X86CallLowering.h
@@ -1,9 +1,8 @@
 //===- llvm/lib/Target/X86/X86CallLowering.h - Call lowering ----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/X86/X86CallingConv.cpp b/lib/Target/X86/X86CallingConv.cpp
index 59dde982f512d6d9ac70e49a1348435465b38f1e..7a58e9ae0692a93cc882374c4ab0cec4f2d1e98d 100644
--- a/lib/Target/X86/X86CallingConv.cpp
+++ b/lib/Target/X86/X86CallingConv.cpp
@@ -1,9 +1,8 @@
 //=== X86CallingConv.cpp - X86 Custom Calling Convention Impl   -*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -12,16 +11,23 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "MCTargetDesc/X86MCTargetDesc.h"
+#include "X86CallingConv.h"
 #include "X86Subtarget.h"
+#include "llvm/ADT/SmallVector.h"
 #include "llvm/CodeGen/CallingConvLower.h"
 #include "llvm/IR/CallingConv.h"
 
-namespace llvm {
-
-bool CC_X86_32_RegCall_Assign2Regs(unsigned &ValNo, MVT &ValVT, MVT &LocVT,
-                                   CCValAssign::LocInfo &LocInfo,
-                                   ISD::ArgFlagsTy &ArgFlags, CCState &State) {
+using namespace llvm;
+
+/// When regcall calling convention compiled to 32 bit arch, special treatment
+/// is required for 64 bit masks.
+/// The value should be assigned to two GPRs.
+/// \return true if registers were allocated and false otherwise.
+static bool CC_X86_32_RegCall_Assign2Regs(unsigned &ValNo, MVT &ValVT,
+                                          MVT &LocVT,
+                                          CCValAssign::LocInfo &LocInfo,
+                                          ISD::ArgFlagsTy &ArgFlags,
+                                          CCState &State) {
   // List of GPR registers that are available to store values in regcall
   // calling convention.
   static const MCPhysReg RegList[] = {X86::EAX, X86::ECX, X86::EDX, X86::EDI,
@@ -113,9 +119,15 @@ static bool CC_X86_VectorCallAssignRegister(unsigned &ValNo, MVT &ValVT,
   return false;
 }
 
-bool CC_X86_64_VectorCall(unsigned &ValNo, MVT &ValVT, MVT &LocVT,
-                          CCValAssign::LocInfo &LocInfo,
-                          ISD::ArgFlagsTy &ArgFlags, CCState &State) {
+/// Vectorcall calling convention has special handling for vector types or
+/// HVA for 64 bit arch.
+/// For HVAs shadow registers might be allocated on the first pass
+/// and actual XMM registers are allocated on the second pass.
+/// For vector types, actual XMM registers are allocated on the first pass.
+/// \return true if registers were allocated and false otherwise.
+static bool CC_X86_64_VectorCall(unsigned &ValNo, MVT &ValVT, MVT &LocVT,
+                                 CCValAssign::LocInfo &LocInfo,
+                                 ISD::ArgFlagsTy &ArgFlags, CCState &State) {
   // On the second pass, go through the HVAs only.
   if (ArgFlags.isSecArgPass()) {
     if (ArgFlags.isHva())
@@ -165,9 +177,14 @@ bool CC_X86_64_VectorCall(unsigned &ValNo, MVT &ValVT, MVT &LocVT,
   return ArgFlags.isHva();
 }
 
-bool CC_X86_32_VectorCall(unsigned &ValNo, MVT &ValVT, MVT &LocVT,
-                          CCValAssign::LocInfo &LocInfo,
-                          ISD::ArgFlagsTy &ArgFlags, CCState &State) {
+/// Vectorcall calling convention has special handling for vector types or
+/// HVA for 32 bit arch.
+/// For HVAs actual XMM registers are allocated on the second pass.
+/// For vector types, actual XMM registers are allocated on the first pass.
+/// \return true if registers were allocated and false otherwise.
+static bool CC_X86_32_VectorCall(unsigned &ValNo, MVT &ValVT, MVT &LocVT,
+                                 CCValAssign::LocInfo &LocInfo,
+                                 ISD::ArgFlagsTy &ArgFlags, CCState &State) {
   // On the second pass, go through the HVAs only.
   if (ArgFlags.isSecArgPass()) {
     if (ArgFlags.isHva())
@@ -205,4 +222,70 @@ bool CC_X86_32_VectorCall(unsigned &ValNo, MVT &ValVT, MVT &LocVT,
   return false; // No register was assigned - Continue the search.
 }
 
-} // End llvm namespace
+static bool CC_X86_AnyReg_Error(unsigned &, MVT &, MVT &,
+                                CCValAssign::LocInfo &, ISD::ArgFlagsTy &,
+                                CCState &) {
+  llvm_unreachable("The AnyReg calling convention is only supported by the "
+                   "stackmap and patchpoint intrinsics.");
+  // gracefully fallback to X86 C calling convention on Release builds.
+  return false;
+}
+
+static bool CC_X86_32_MCUInReg(unsigned &ValNo, MVT &ValVT, MVT &LocVT,
+                               CCValAssign::LocInfo &LocInfo,
+                               ISD::ArgFlagsTy &ArgFlags, CCState &State) {
+  // This is similar to CCAssignToReg<[EAX, EDX, ECX]>, but makes sure
+  // not to split i64 and double between a register and stack
+  static const MCPhysReg RegList[] = {X86::EAX, X86::EDX, X86::ECX};
+  static const unsigned NumRegs = sizeof(RegList) / sizeof(RegList[0]);
+
+  SmallVectorImpl<CCValAssign> &PendingMembers = State.getPendingLocs();
+
+  // If this is the first part of an double/i64/i128, or if we're already
+  // in the middle of a split, add to the pending list. If this is not
+  // the end of the split, return, otherwise go on to process the pending
+  // list
+  if (ArgFlags.isSplit() || !PendingMembers.empty()) {
+    PendingMembers.push_back(
+        CCValAssign::getPending(ValNo, ValVT, LocVT, LocInfo));
+    if (!ArgFlags.isSplitEnd())
+      return true;
+  }
+
+  // If there are no pending members, we are not in the middle of a split,
+  // so do the usual inreg stuff.
+  if (PendingMembers.empty()) {
+    if (unsigned Reg = State.AllocateReg(RegList)) {
+      State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
+      return true;
+    }
+    return false;
+  }
+
+  assert(ArgFlags.isSplitEnd());
+
+  // We now have the entire original argument in PendingMembers, so decide
+  // whether to use registers or the stack.
+  // Per the MCU ABI:
+  // a) To use registers, we need to have enough of them free to contain
+  // the entire argument.
+  // b) We never want to use more than 2 registers for a single argument.
+
+  unsigned FirstFree = State.getFirstUnallocated(RegList);
+  bool UseRegs = PendingMembers.size() <= std::min(2U, NumRegs - FirstFree);
+
+  for (auto &It : PendingMembers) {
+    if (UseRegs)
+      It.convertToReg(State.AllocateReg(RegList[FirstFree++]));
+    else
+      It.convertToMem(State.AllocateStack(4, 4));
+    State.addLoc(It);
+  }
+
+  PendingMembers.clear();
+
+  return true;
+}
+
+// Provides entry points of CC_X86 and RetCC_X86.
+#include "X86GenCallingConv.inc"
diff --git a/lib/Target/X86/X86CallingConv.h b/lib/Target/X86/X86CallingConv.h
index d0fcbd31331252f48f8eaf56026b80edc29bad26..191e0fa619b24f4258eb49062806c7b63750b569 100644
--- a/lib/Target/X86/X86CallingConv.h
+++ b/lib/Target/X86/X86CallingConv.h
@@ -1,9 +1,8 @@
 //=== X86CallingConv.h - X86 Custom Calling Convention Routines -*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -21,99 +20,12 @@
 
 namespace llvm {
 
-/// When regcall calling convention compiled to 32 bit arch, special treatment
-/// is required for 64 bit masks.
-/// The value should be assigned to two GPRs.
-/// \return true if registers were allocated and false otherwise.
-bool CC_X86_32_RegCall_Assign2Regs(unsigned &ValNo, MVT &ValVT, MVT &LocVT,
-                                   CCValAssign::LocInfo &LocInfo,
-                                   ISD::ArgFlagsTy &ArgFlags, CCState &State);
-
-/// Vectorcall calling convention has special handling for vector types or
-/// HVA for 64 bit arch.
-/// For HVAs shadow registers might be allocated on the first pass
-/// and actual XMM registers are allocated on the second pass.
-/// For vector types, actual XMM registers are allocated on the first pass.
-/// \return true if registers were allocated and false otherwise.
-bool CC_X86_64_VectorCall(unsigned &ValNo, MVT &ValVT, MVT &LocVT,
-                          CCValAssign::LocInfo &LocInfo,
-                          ISD::ArgFlagsTy &ArgFlags, CCState &State);
-
-/// Vectorcall calling convention has special handling for vector types or
-/// HVA for 32 bit arch.
-/// For HVAs actual XMM registers are allocated on the second pass.
-/// For vector types, actual XMM registers are allocated on the first pass.
-/// \return true if registers were allocated and false otherwise.
-bool CC_X86_32_VectorCall(unsigned &ValNo, MVT &ValVT, MVT &LocVT,
-                          CCValAssign::LocInfo &LocInfo,
-                          ISD::ArgFlagsTy &ArgFlags, CCState &State);
-
-inline bool CC_X86_AnyReg_Error(unsigned &, MVT &, MVT &,
-                                CCValAssign::LocInfo &, ISD::ArgFlagsTy &,
-                                CCState &) {
-  llvm_unreachable("The AnyReg calling convention is only supported by the " \
-                   "stackmap and patchpoint intrinsics.");
-  // gracefully fallback to X86 C calling convention on Release builds.
-  return false;
-}
-
-inline bool CC_X86_32_MCUInReg(unsigned &ValNo, MVT &ValVT,
-                                         MVT &LocVT,
-                                         CCValAssign::LocInfo &LocInfo,
-                                         ISD::ArgFlagsTy &ArgFlags,
-                                         CCState &State) {
-  // This is similar to CCAssignToReg<[EAX, EDX, ECX]>, but makes sure
-  // not to split i64 and double between a register and stack
-  static const MCPhysReg RegList[] = {X86::EAX, X86::EDX, X86::ECX};
-  static const unsigned NumRegs = sizeof(RegList)/sizeof(RegList[0]);
-
-  SmallVectorImpl<CCValAssign> &PendingMembers = State.getPendingLocs();
-
-  // If this is the first part of an double/i64/i128, or if we're already
-  // in the middle of a split, add to the pending list. If this is not
-  // the end of the split, return, otherwise go on to process the pending
-  // list
-  if (ArgFlags.isSplit() || !PendingMembers.empty()) {
-    PendingMembers.push_back(
-        CCValAssign::getPending(ValNo, ValVT, LocVT, LocInfo));
-    if (!ArgFlags.isSplitEnd())
-      return true;
-  }
-
-  // If there are no pending members, we are not in the middle of a split,
-  // so do the usual inreg stuff.
-  if (PendingMembers.empty()) {
-    if (unsigned Reg = State.AllocateReg(RegList)) {
-      State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
-      return true;
-    }
-    return false;
-  }
-
-  assert(ArgFlags.isSplitEnd());
-
-  // We now have the entire original argument in PendingMembers, so decide
-  // whether to use registers or the stack.
-  // Per the MCU ABI:
-  // a) To use registers, we need to have enough of them free to contain
-  // the entire argument.
-  // b) We never want to use more than 2 registers for a single argument.
-
-  unsigned FirstFree = State.getFirstUnallocated(RegList);
-  bool UseRegs = PendingMembers.size() <= std::min(2U, NumRegs - FirstFree);
-
-  for (auto &It : PendingMembers) {
-    if (UseRegs)
-      It.convertToReg(State.AllocateReg(RegList[FirstFree++]));
-    else
-      It.convertToMem(State.AllocateStack(4, 4));
-    State.addLoc(It);
-  }
-
-  PendingMembers.clear();
+bool RetCC_X86(unsigned ValNo, MVT ValVT, MVT LocVT,
+               CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags,
+               CCState &State);
 
-  return true;
-}
+bool CC_X86(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo,
+            ISD::ArgFlagsTy ArgFlags, CCState &State);
 
 } // End llvm namespace
 
diff --git a/lib/Target/X86/X86CallingConv.td b/lib/Target/X86/X86CallingConv.td
index fe49c9ffbd958719659a395447475ba6098d3fb9..d12e8af1cac3e77dcbc0f9b4efd1eeefeb11a42b 100644
--- a/lib/Target/X86/X86CallingConv.td
+++ b/lib/Target/X86/X86CallingConv.td
@@ -1,9 +1,8 @@
 //===-- X86CallingConv.td - Calling Conventions X86 32/64 --*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -477,6 +476,7 @@ def RetCC_X86_64 : CallingConv<[
 ]>;
 
 // This is the return-value convention used for the entire X86 backend.
+let Entry = 1 in
 def RetCC_X86 : CallingConv<[
 
   // Check if this is the Intel OpenCL built-ins calling convention
@@ -1039,6 +1039,7 @@ def CC_X86_64 : CallingConv<[
 ]>;
 
 // This is the argument convention used for the entire X86 backend.
+let Entry = 1 in
 def CC_X86 : CallingConv<[
   CCIfCC<"CallingConv::Intel_OCL_BI", CCDelegateTo<CC_Intel_OCL_BI>>,
   CCIfSubtarget<"is64Bit()", CCDelegateTo<CC_X86_64>>,
diff --git a/lib/Target/X86/X86CmovConversion.cpp b/lib/Target/X86/X86CmovConversion.cpp
index c3e76fd2a856ce52e1b61a4c9d104eed22bf127b..e8f6e8f61fcd29e90c26dd94ecb31666d75b6f38 100644
--- a/lib/Target/X86/X86CmovConversion.cpp
+++ b/lib/Target/X86/X86CmovConversion.cpp
@@ -1,9 +1,8 @@
 //====- X86CmovConversion.cpp - Convert Cmov to Branch --------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/X86/X86CondBrFolding.cpp b/lib/Target/X86/X86CondBrFolding.cpp
index 7ce443c4656af40827ff32455a51b0eb558705cc..c14b3a0884ef1d364e2055802f7b162784e2871a 100644
--- a/lib/Target/X86/X86CondBrFolding.cpp
+++ b/lib/Target/X86/X86CondBrFolding.cpp
@@ -1,9 +1,8 @@
 //===---- X86CondBrFolding.cpp - optimize conditional branches ------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 // This file defines a pass that optimizes condition branches on x86 by taking
diff --git a/lib/Target/X86/X86DiscriminateMemOps.cpp b/lib/Target/X86/X86DiscriminateMemOps.cpp
index 5653babedf8eccd67fcb41be29fd61c377c6000a..f6bd5804261947f6f7c4e1cecb48872832677be2 100644
--- a/lib/Target/X86/X86DiscriminateMemOps.cpp
+++ b/lib/Target/X86/X86DiscriminateMemOps.cpp
@@ -1,9 +1,8 @@
 //===- X86DiscriminateMemOps.cpp - Unique IDs for Mem Ops -----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
@@ -21,11 +20,20 @@
 #include "llvm/IR/DebugInfoMetadata.h"
 #include "llvm/ProfileData/SampleProf.h"
 #include "llvm/ProfileData/SampleProfReader.h"
+#include "llvm/Support/Debug.h"
 #include "llvm/Transforms/IPO/SampleProfile.h"
 using namespace llvm;
 
 #define DEBUG_TYPE "x86-discriminate-memops"
 
+static cl::opt<bool> EnableDiscriminateMemops(
+    DEBUG_TYPE, cl::init(false),
+    cl::desc("Generate unique debug info for each instruction with a memory "
+             "operand. Should be enabled for profile-drived cache prefetching, "
+             "both in the build of the binary being profiled, as well as in "
+             "the build of the binary consuming the profile."),
+    cl::Hidden);
+
 namespace {
 
 using Location = std::pair<StringRef, unsigned>;
@@ -66,6 +74,9 @@ char X86DiscriminateMemOps::ID = 0;
 X86DiscriminateMemOps::X86DiscriminateMemOps() : MachineFunctionPass(ID) {}
 
 bool X86DiscriminateMemOps::runOnMachineFunction(MachineFunction &MF) {
+  if (!EnableDiscriminateMemops)
+    return false;
+
   DISubprogram *FDI = MF.getFunction().getSubprogram();
   if (!FDI || !FDI->getUnit()->getDebugInfoForProfiling())
     return false;
@@ -107,27 +118,37 @@ bool X86DiscriminateMemOps::runOnMachineFunction(MachineFunction &MF) {
       if (!DI) {
         DI = ReferenceDI;
       }
-      DenseSet<unsigned> &Set = Seen[diToLocation(DI)];
+      Location L = diToLocation(DI);
+      DenseSet<unsigned> &Set = Seen[L];
       const std::pair<DenseSet<unsigned>::iterator, bool> TryInsert =
           Set.insert(DI->getBaseDiscriminator());
       if (!TryInsert.second) {
-        DI = DI->setBaseDiscriminator(++MemOpDiscriminators[diToLocation(DI)]);
-        updateDebugInfo(&MI, DI);
-        Changed = true;
-        const std::pair<DenseSet<unsigned>::iterator, bool> MustInsert =
-            Set.insert(DI->getBaseDiscriminator());
-        // FIXME (mtrofin): check if the to-be inserted base discriminator can
-        // be added. This requires a new API on DILocation.
-        // The assumption is that this scenario is infrequent/OK not to support.
-        // If evidence points otherwise, we can explore synthesize unique DIs by
-        // adding fake line numbers.
-        if (!MustInsert.second) {
-          LLVM_DEBUG(dbgs()
-                     << "Unable to create a unique discriminator in "
+        unsigned BF, DF, CI = 0;
+        DILocation::decodeDiscriminator(DI->getDiscriminator(), BF, DF, CI);
+        Optional<unsigned> EncodedDiscriminator = DILocation::encodeDiscriminator(
+            MemOpDiscriminators[L] + 1, DF, CI);
+
+        if (!EncodedDiscriminator) {
+          // FIXME(mtrofin): The assumption is that this scenario is infrequent/OK
+          // not to support. If evidence points otherwise, we can explore synthesizeing
+          // unique DIs by adding fake line numbers, or by constructing 64 bit
+          // discriminators.
+          LLVM_DEBUG(dbgs() << "Unable to create a unique discriminator "
+                     "for instruction with memory operand in: "
                      << DI->getFilename() << " Line: " << DI->getLine()
                      << " Column: " << DI->getColumn()
-                     << ". This is likely due to a large macro expansion.\n");
+                     << ". This is likely due to a large macro expansion. \n");
+          continue;
         }
+        // Since we were able to encode, bump the MemOpDiscriminators.
+        ++MemOpDiscriminators[L];
+        DI = DI->cloneWithDiscriminator(EncodedDiscriminator.getValue());
+        updateDebugInfo(&MI, DI);
+        Changed = true;
+        std::pair<DenseSet<unsigned>::iterator, bool> MustInsert =
+            Set.insert(DI->getBaseDiscriminator());
+        (void)MustInsert; // Silence warning in release build.
+        assert(MustInsert.second && "New discriminator shouldn't be present in set");
       }
 
       // Bump the reference DI to avoid cramming discriminators on line 0.
diff --git a/lib/Target/X86/X86DomainReassignment.cpp b/lib/Target/X86/X86DomainReassignment.cpp
index 7e1f1e7876c037ee03ce8c560ccdf0e66b3b8ec0..86d9de22775d40838aea64274e9cc8926b584360 100644
--- a/lib/Target/X86/X86DomainReassignment.cpp
+++ b/lib/Target/X86/X86DomainReassignment.cpp
@@ -1,9 +1,8 @@
 //===--- X86DomainReassignment.cpp - Selectively switch register classes---===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -732,7 +731,10 @@ bool X86DomainReassignment::runOnMachineFunction(MachineFunction &MF) {
   STI = &MF.getSubtarget<X86Subtarget>();
   // GPR->K is the only transformation currently supported, bail out early if no
   // AVX512.
-  if (!STI->hasAVX512())
+  // TODO: We're also bailing of AVX512BW isn't supported since we use VK32 and
+  // VK64 for GR32/GR64, but those aren't legal classes on KNL. If the register
+  // coalescer doesn't clean it up and we generate a spill we will crash.
+  if (!STI->hasAVX512() || !STI->hasBWI())
     return false;
 
   MRI = &MF.getRegInfo();
diff --git a/lib/Target/X86/X86EvexToVex.cpp b/lib/Target/X86/X86EvexToVex.cpp
index 80674c7251fe5a3c6372fe1ae6679076006afdfc..a7c5bab67f9437a855be73cf959f9e455c917535 100644
--- a/lib/Target/X86/X86EvexToVex.cpp
+++ b/lib/Target/X86/X86EvexToVex.cpp
@@ -1,10 +1,9 @@
 //===- X86EvexToVex.cpp ---------------------------------------------------===//
 // Compress EVEX instructions to VEX encoding when possible to reduce code size
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/X86/X86ExpandPseudo.cpp b/lib/Target/X86/X86ExpandPseudo.cpp
index 1dd73163080ba66c5a97b4e8c00894b37827dfe0..963c5181e3d7c88905fc2f3046651543ab868b68 100644
--- a/lib/Target/X86/X86ExpandPseudo.cpp
+++ b/lib/Target/X86/X86ExpandPseudo.cpp
@@ -1,9 +1,8 @@
 //===------- X86ExpandPseudo.cpp - Expand pseudo instructions -------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/X86/X86FastISel.cpp b/lib/Target/X86/X86FastISel.cpp
index cbfdc4b3b93b3cb7b478ac58896ec484f4388b20..8f34f49444f9331510a34b5b90b9793bef873427 100644
--- a/lib/Target/X86/X86FastISel.cpp
+++ b/lib/Target/X86/X86FastISel.cpp
@@ -1,9 +1,8 @@
 //===-- X86FastISel.cpp - X86 FastISel implementation ---------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -312,8 +311,6 @@ bool X86FastISel::isTypeLegal(Type *Ty, MVT &VT, bool AllowI1) {
   return (AllowI1 && VT == MVT::i1) || TLI.isTypeLegal(VT);
 }
 
-#include "X86GenCallingConv.inc"
-
 /// X86FastEmitLoad - Emit a machine instruction to load a value of type VT.
 /// The address is either pre-computed, i.e. Ptr, or a GlobalAddress, i.e. GV.
 /// Return true and the result register by reference if it is possible.
@@ -2900,23 +2897,15 @@ bool X86FastISel::fastLowerIntrinsicCall(const IntrinsicInst *II) {
         isCommutativeIntrinsic(II))
       std::swap(LHS, RHS);
 
-    bool UseIncDec = false;
-    if (isa<ConstantInt>(RHS) && cast<ConstantInt>(RHS)->isOne())
-      UseIncDec = true;
-
     unsigned BaseOpc, CondOpc;
     switch (II->getIntrinsicID()) {
     default: llvm_unreachable("Unexpected intrinsic!");
     case Intrinsic::sadd_with_overflow:
-      BaseOpc = UseIncDec ? unsigned(X86ISD::INC) : unsigned(ISD::ADD);
-      CondOpc = X86::SETOr;
-      break;
+      BaseOpc = ISD::ADD; CondOpc = X86::SETOr; break;
     case Intrinsic::uadd_with_overflow:
       BaseOpc = ISD::ADD; CondOpc = X86::SETBr; break;
     case Intrinsic::ssub_with_overflow:
-      BaseOpc = UseIncDec ? unsigned(X86ISD::DEC) : unsigned(ISD::SUB);
-      CondOpc = X86::SETOr;
-      break;
+      BaseOpc = ISD::SUB; CondOpc = X86::SETOr; break;
     case Intrinsic::usub_with_overflow:
       BaseOpc = ISD::SUB; CondOpc = X86::SETBr; break;
     case Intrinsic::smul_with_overflow:
@@ -2938,9 +2927,11 @@ bool X86FastISel::fastLowerIntrinsicCall(const IntrinsicInst *II) {
         { X86::DEC8r, X86::DEC16r, X86::DEC32r, X86::DEC64r }
       };
 
-      if (BaseOpc == X86ISD::INC || BaseOpc == X86ISD::DEC) {
+      if (CI->isOne() && (BaseOpc == ISD::ADD || BaseOpc == ISD::SUB) &&
+          CondOpc == X86::SETOr) {
+        // We can use INC/DEC.
         ResultReg = createResultReg(TLI.getRegClassFor(VT));
-        bool IsDec = BaseOpc == X86ISD::DEC;
+        bool IsDec = BaseOpc == ISD::SUB;
         BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc,
                 TII.get(Opc[IsDec][VT.SimpleTy-MVT::i8]), ResultReg)
           .addReg(LHSReg, getKillRegState(LHSIsKill));
diff --git a/lib/Target/X86/X86FixupBWInsts.cpp b/lib/Target/X86/X86FixupBWInsts.cpp
index ed297e6782030cb32cb50788f25eda22ae9ce53c..be4479f871ec03b1cb7ce093a159cb2c56e7e5d0 100644
--- a/lib/Target/X86/X86FixupBWInsts.cpp
+++ b/lib/Target/X86/X86FixupBWInsts.cpp
@@ -1,9 +1,8 @@
 //===-- X86FixupBWInsts.cpp - Fixup Byte or Word instructions -----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/lib/Target/X86/X86FixupLEAs.cpp b/lib/Target/X86/X86FixupLEAs.cpp
index ad42cb878046c45420e2723d162f0c0afba0613b..273173efd623f8a3c2535d946b3a071403a6a1f7 100644
--- a/lib/Target/X86/X86FixupLEAs.cpp
+++ b/lib/Target/X86/X86FixupLEAs.cpp
@@ -1,9 +1,8 @@
 //===-- X86FixupLEAs.cpp - use or replace LEA instructions -----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -333,13 +332,12 @@ static inline int getADDriFromLEA(int LEAOpcode, const MachineOperand &Offset) {
 static inline bool isLEASimpleIncOrDec(MachineInstr &LEA) {
   unsigned SrcReg = LEA.getOperand(1 + X86::AddrBaseReg).getReg();
   unsigned DstReg = LEA.getOperand(0).getReg();
-  unsigned AddrDispOp = 1 + X86::AddrDisp;
+  const MachineOperand &AddrDisp = LEA.getOperand(1 + X86::AddrDisp);
   return SrcReg == DstReg &&
          LEA.getOperand(1 + X86::AddrIndexReg).getReg() == 0 &&
          LEA.getOperand(1 + X86::AddrSegmentReg).getReg() == 0 &&
-         LEA.getOperand(AddrDispOp).isImm() &&
-         (LEA.getOperand(AddrDispOp).getImm() == 1 ||
-          LEA.getOperand(AddrDispOp).getImm() == -1);
+         AddrDisp.isImm() &&
+         (AddrDisp.getImm() == 1 || AddrDisp.getImm() == -1);
 }
 
 bool FixupLEAPass::fixupIncDec(MachineBasicBlock::iterator &I,
@@ -351,7 +349,7 @@ bool FixupLEAPass::fixupIncDec(MachineBasicBlock::iterator &I,
 
   if (isLEASimpleIncOrDec(MI) && TII->isSafeToClobberEFLAGS(*MFI, I)) {
     int NewOpcode;
-    bool isINC = MI.getOperand(4).getImm() == 1;
+    bool isINC = MI.getOperand(1 + X86::AddrDisp).getImm() == 1;
     switch (Opcode) {
     case X86::LEA16r:
       NewOpcode = isINC ? X86::INC16r : X86::DEC16r;
@@ -368,7 +366,7 @@ bool FixupLEAPass::fixupIncDec(MachineBasicBlock::iterator &I,
     MachineInstr *NewMI =
         BuildMI(*MFI, I, MI.getDebugLoc(), TII->get(NewOpcode))
             .add(MI.getOperand(0))
-            .add(MI.getOperand(1));
+            .add(MI.getOperand(1 + X86::AddrBaseReg));
     MFI->erase(I);
     I = static_cast<MachineBasicBlock::iterator>(NewMI);
     return true;
@@ -420,15 +418,23 @@ void FixupLEAPass::processInstructionForSlowLEA(MachineBasicBlock::iterator &I,
   const int Opcode = MI.getOpcode();
   if (!isLEA(Opcode))
     return;
-  if (MI.getOperand(5).getReg() != 0 || !MI.getOperand(4).isImm() ||
+
+  const MachineOperand &Dst =     MI.getOperand(0);
+  const MachineOperand &Base =    MI.getOperand(1 + X86::AddrBaseReg);
+  const MachineOperand &Scale =   MI.getOperand(1 + X86::AddrScaleAmt);
+  const MachineOperand &Index =   MI.getOperand(1 + X86::AddrIndexReg);
+  const MachineOperand &Offset =  MI.getOperand(1 + X86::AddrDisp);
+  const MachineOperand &Segment = MI.getOperand(1 + X86::AddrSegmentReg);
+
+  if (Segment.getReg() != 0 || !Offset.isImm() ||
       !TII->isSafeToClobberEFLAGS(*MFI, I))
     return;
-  const unsigned DstR = MI.getOperand(0).getReg();
-  const unsigned SrcR1 = MI.getOperand(1).getReg();
-  const unsigned SrcR2 = MI.getOperand(3).getReg();
+  const unsigned DstR = Dst.getReg();
+  const unsigned SrcR1 = Base.getReg();
+  const unsigned SrcR2 = Index.getReg();
   if ((SrcR1 == 0 || SrcR1 != DstR) && (SrcR2 == 0 || SrcR2 != DstR))
     return;
-  if (MI.getOperand(2).getImm() > 1)
+  if (Scale.getImm() > 1)
     return;
   LLVM_DEBUG(dbgs() << "FixLEA: Candidate to replace:"; I->dump(););
   LLVM_DEBUG(dbgs() << "FixLEA: Replaced by: ";);
@@ -436,19 +442,19 @@ void FixupLEAPass::processInstructionForSlowLEA(MachineBasicBlock::iterator &I,
   // Make ADD instruction for two registers writing to LEA's destination
   if (SrcR1 != 0 && SrcR2 != 0) {
     const MCInstrDesc &ADDrr = TII->get(getADDrrFromLEA(Opcode));
-    const MachineOperand &Src = MI.getOperand(SrcR1 == DstR ? 3 : 1);
+    const MachineOperand &Src = SrcR1 == DstR ? Index : Base;
     NewMI =
         BuildMI(*MFI, I, MI.getDebugLoc(), ADDrr, DstR).addReg(DstR).add(Src);
     LLVM_DEBUG(NewMI->dump(););
   }
   // Make ADD instruction for immediate
-  if (MI.getOperand(4).getImm() != 0) {
+  if (Offset.getImm() != 0) {
     const MCInstrDesc &ADDri =
-        TII->get(getADDriFromLEA(Opcode, MI.getOperand(4)));
-    const MachineOperand &SrcR = MI.getOperand(SrcR1 == DstR ? 1 : 3);
+        TII->get(getADDriFromLEA(Opcode, Offset));
+    const MachineOperand &SrcR = SrcR1 == DstR ? Base : Index;
     NewMI = BuildMI(*MFI, I, MI.getDebugLoc(), ADDri, DstR)
                 .add(SrcR)
-                .addImm(MI.getOperand(4).getImm());
+                .addImm(Offset.getImm());
     LLVM_DEBUG(NewMI->dump(););
   }
   if (NewMI) {
@@ -465,12 +471,12 @@ FixupLEAPass::processInstrForSlow3OpLEA(MachineInstr &MI,
   if (!isLEA(LEAOpcode))
     return nullptr;
 
-  const MachineOperand &Dst = MI.getOperand(0);
-  const MachineOperand &Base = MI.getOperand(1);
-  const MachineOperand &Scale = MI.getOperand(2);
-  const MachineOperand &Index = MI.getOperand(3);
-  const MachineOperand &Offset = MI.getOperand(4);
-  const MachineOperand &Segment = MI.getOperand(5);
+  const MachineOperand &Dst =     MI.getOperand(0);
+  const MachineOperand &Base =    MI.getOperand(1 + X86::AddrBaseReg);
+  const MachineOperand &Scale =   MI.getOperand(1 + X86::AddrScaleAmt);
+  const MachineOperand &Index =   MI.getOperand(1 + X86::AddrIndexReg);
+  const MachineOperand &Offset =  MI.getOperand(1 + X86::AddrDisp);
+  const MachineOperand &Segment = MI.getOperand(1 + X86::AddrSegmentReg);
 
   if (!(TII->isThreeOperandsLEA(MI) ||
         hasInefficientLEABaseReg(Base, Index)) ||
diff --git a/lib/Target/X86/X86FixupSetCC.cpp b/lib/Target/X86/X86FixupSetCC.cpp
index a86eb997635e569adf0e7c3026e3aae3d6e2bb4e..5bfad71ec05bdf5efa95312296dda9afb13952b6 100644
--- a/lib/Target/X86/X86FixupSetCC.cpp
+++ b/lib/Target/X86/X86FixupSetCC.cpp
@@ -1,9 +1,8 @@
 //===---- X86FixupSetCC.cpp - optimize usage of LEA instructions ----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -103,7 +102,7 @@ X86FixupSetCCPass::findFlagsImpDef(MachineBasicBlock *MBB,
   auto MBBStart = MBB->rend();
   for (int i = 0; (i < SearchBound) && (MI != MBBStart); ++i, ++MI)
     for (auto &Op : MI->implicit_operands())
-      if ((Op.getReg() == X86::EFLAGS) && (Op.isDef()))
+      if (Op.isReg() && (Op.getReg() == X86::EFLAGS) && Op.isDef())
         return &*MI;
 
   return nullptr;
@@ -111,7 +110,7 @@ X86FixupSetCCPass::findFlagsImpDef(MachineBasicBlock *MBB,
 
 bool X86FixupSetCCPass::impUsesFlags(MachineInstr *MI) {
   for (auto &Op : MI->implicit_operands())
-    if ((Op.getReg() == X86::EFLAGS) && (Op.isUse()))
+    if (Op.isReg() && (Op.getReg() == X86::EFLAGS) && Op.isUse())
       return true;
 
   return false;
diff --git a/lib/Target/X86/X86FlagsCopyLowering.cpp b/lib/Target/X86/X86FlagsCopyLowering.cpp
index 778aa505b2d9e4d59d372bb54be9f3094c22089f..6d77170f7099d741da7c49240a7550fc399e9aef 100644
--- a/lib/Target/X86/X86FlagsCopyLowering.cpp
+++ b/lib/Target/X86/X86FlagsCopyLowering.cpp
@@ -1,9 +1,8 @@
 //====- X86FlagsCopyLowering.cpp - Lowers COPY nodes of EFLAGS ------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/lib/Target/X86/X86FloatingPoint.cpp b/lib/Target/X86/X86FloatingPoint.cpp
index f330acff61a19394e8fbd8e69d926b94c98dd5b5..3ef2c1b51714fe578c1c9a23245b61ee2a3863fa 100644
--- a/lib/Target/X86/X86FloatingPoint.cpp
+++ b/lib/Target/X86/X86FloatingPoint.cpp
@@ -1,9 +1,8 @@
 //===-- X86FloatingPoint.cpp - Floating point Reg -> Stack converter ------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -1096,6 +1095,8 @@ void FPS::handleZeroArgFP(MachineBasicBlock::iterator &I) {
   // Change from the pseudo instruction to the concrete instruction.
   MI.RemoveOperand(0); // Remove the explicit ST(0) operand
   MI.setDesc(TII->get(getConcreteOpcode(MI.getOpcode())));
+  MI.addOperand(
+      MachineOperand::CreateReg(X86::ST0, /*isDef*/ true, /*isImp*/ true));
 
   // Result gets pushed on the stack.
   pushReg(DestReg);
@@ -1140,6 +1141,8 @@ void FPS::handleOneArgFP(MachineBasicBlock::iterator &I) {
   // Convert from the pseudo instruction to the concrete instruction.
   MI.RemoveOperand(NumOps - 1); // Remove explicit ST(0) operand
   MI.setDesc(TII->get(getConcreteOpcode(MI.getOpcode())));
+  MI.addOperand(
+      MachineOperand::CreateReg(X86::ST0, /*isDef*/ false, /*isImp*/ true));
 
   if (MI.getOpcode() == X86::IST_FP64m || MI.getOpcode() == X86::ISTT_FP16m ||
       MI.getOpcode() == X86::ISTT_FP32m || MI.getOpcode() == X86::ISTT_FP64m ||
@@ -1369,8 +1372,6 @@ void FPS::handleTwoArgFP(MachineBasicBlock::iterator &I) {
 /// register arguments and no explicit destinations.
 ///
 void FPS::handleCompareFP(MachineBasicBlock::iterator &I) {
-  ASSERT_SORTED(ForwardST0Table); ASSERT_SORTED(ReverseST0Table);
-  ASSERT_SORTED(ForwardSTiTable); ASSERT_SORTED(ReverseSTiTable);
   MachineInstr &MI = *I;
 
   unsigned NumOperands = MI.getDesc().getNumOperands();
@@ -1475,7 +1476,8 @@ void FPS::handleSpecialFP(MachineBasicBlock::iterator &Inst) {
     break;
   }
 
-  case TargetOpcode::INLINEASM: {
+  case TargetOpcode::INLINEASM:
+  case TargetOpcode::INLINEASM_BR: {
     // The inline asm MachineInstr currently only *uses* FP registers for the
     // 'f' constraint.  These should be turned into the current ST(x) register
     // in the machine instr.
diff --git a/lib/Target/X86/X86FrameLowering.cpp b/lib/Target/X86/X86FrameLowering.cpp
index d722c75c4f0a516a924bca273c89483190c02caf..fb8a5396d3d91e9af29941756e5cd3c49d49cbb7 100644
--- a/lib/Target/X86/X86FrameLowering.cpp
+++ b/lib/Target/X86/X86FrameLowering.cpp
@@ -1,9 +1,8 @@
 //===-- X86FrameLowering.cpp - X86 Frame Information ----------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -185,7 +184,8 @@ static unsigned findDeadCallerSavedReg(MachineBasicBlock &MBB,
     }
 
     for (auto CS : AvailableRegs)
-      if (!Uses.count(CS) && CS != X86::RIP)
+      if (!Uses.count(CS) && CS != X86::RIP && CS != X86::RSP &&
+          CS != X86::ESP)
         return CS;
   }
   }
diff --git a/lib/Target/X86/X86FrameLowering.h b/lib/Target/X86/X86FrameLowering.h
index 3bd805aae123a3fcffda59609df023c8fec53a28..148ee33550d68da6bbc1e6812b821a753462abe5 100644
--- a/lib/Target/X86/X86FrameLowering.h
+++ b/lib/Target/X86/X86FrameLowering.h
@@ -1,9 +1,8 @@
 //===-- X86TargetFrameLowering.h - Define frame lowering for X86 -*- C++ -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/X86/X86GenRegisterBankInfo.def b/lib/Target/X86/X86GenRegisterBankInfo.def
index 9cd3f96f83accdc658a2b8c48055922033c21778..0fdea9071c290dd90eede8b2fd4c30633e5615d5 100644
--- a/lib/Target/X86/X86GenRegisterBankInfo.def
+++ b/lib/Target/X86/X86GenRegisterBankInfo.def
@@ -1,9 +1,8 @@
 //===- X86GenRegisterBankInfo.def ----------------------------*- C++ -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/lib/Target/X86/X86ISelDAGToDAG.cpp b/lib/Target/X86/X86ISelDAGToDAG.cpp
index 43d0538c4cda660d9646c6335e802ac22890029b..25fb5d7469848d3225db9c660bad75ed7e63df4b 100644
--- a/lib/Target/X86/X86ISelDAGToDAG.cpp
+++ b/lib/Target/X86/X86ISelDAGToDAG.cpp
@@ -1,9 +1,8 @@
 //===- X86ISelDAGToDAG.cpp - A DAG pattern matching inst selector for X86 -===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -472,6 +471,10 @@ namespace {
                                 SDValue &InFlag);
 
     bool tryOptimizeRem8Extend(SDNode *N);
+
+    bool onlyUsesZeroFlag(SDValue Flags) const;
+    bool hasNoSignFlagUses(SDValue Flags) const;
+    bool hasNoCarryFlagUses(SDValue Flags) const;
   };
 }
 
@@ -898,9 +901,89 @@ void X86DAGToDAGISel::PostprocessISelDAG() {
       continue;
     }
 
-    // Attempt to remove vectors moves that were inserted to zero upper bits.
+    // Look for a TESTrr+ANDrr pattern where both operands of the test are
+    // the same. Rewrite to remove the AND.
+    unsigned Opc = N->getMachineOpcode();
+    if ((Opc == X86::TEST8rr || Opc == X86::TEST16rr ||
+         Opc == X86::TEST32rr || Opc == X86::TEST64rr) &&
+        N->getOperand(0) == N->getOperand(1) &&
+        N->isOnlyUserOf(N->getOperand(0).getNode()) &&
+        N->getOperand(0).isMachineOpcode()) {
+      SDValue And = N->getOperand(0);
+      unsigned N0Opc = And.getMachineOpcode();
+      if (N0Opc == X86::AND8rr || N0Opc == X86::AND16rr ||
+          N0Opc == X86::AND32rr || N0Opc == X86::AND64rr) {
+        MachineSDNode *Test = CurDAG->getMachineNode(Opc, SDLoc(N),
+                                                     MVT::i32,
+                                                     And.getOperand(0),
+                                                     And.getOperand(1));
+        ReplaceUses(N, Test);
+        MadeChange = true;
+        continue;
+      }
+      if (N0Opc == X86::AND8rm || N0Opc == X86::AND16rm ||
+          N0Opc == X86::AND32rm || N0Opc == X86::AND64rm) {
+        unsigned NewOpc;
+        switch (N0Opc) {
+        case X86::AND8rm:  NewOpc = X86::TEST8mr; break;
+        case X86::AND16rm: NewOpc = X86::TEST16mr; break;
+        case X86::AND32rm: NewOpc = X86::TEST32mr; break;
+        case X86::AND64rm: NewOpc = X86::TEST64mr; break;
+        }
+
+        // Need to swap the memory and register operand.
+        SDValue Ops[] = { And.getOperand(1),
+                          And.getOperand(2),
+                          And.getOperand(3),
+                          And.getOperand(4),
+                          And.getOperand(5),
+                          And.getOperand(0),
+                          And.getOperand(6)  /* Chain */ };
+        MachineSDNode *Test = CurDAG->getMachineNode(NewOpc, SDLoc(N),
+                                                     MVT::i32, MVT::Other, Ops);
+        ReplaceUses(N, Test);
+        MadeChange = true;
+        continue;
+      }
+    }
 
-    if (N->getMachineOpcode() != TargetOpcode::SUBREG_TO_REG)
+    // Look for a KAND+KORTEST and turn it into KTEST if only the zero flag is
+    // used. We're doing this late so we can prefer to fold the AND into masked
+    // comparisons. Doing that can be better for the live range of the mask
+    // register.
+    if ((Opc == X86::KORTESTBrr || Opc == X86::KORTESTWrr ||
+         Opc == X86::KORTESTDrr || Opc == X86::KORTESTQrr) &&
+        N->getOperand(0) == N->getOperand(1) &&
+        N->isOnlyUserOf(N->getOperand(0).getNode()) &&
+        N->getOperand(0).isMachineOpcode() &&
+        onlyUsesZeroFlag(SDValue(N, 0))) {
+      SDValue And = N->getOperand(0);
+      unsigned N0Opc = And.getMachineOpcode();
+      // KANDW is legal with AVX512F, but KTESTW requires AVX512DQ. The other
+      // KAND instructions and KTEST use the same ISA feature.
+      if (N0Opc == X86::KANDBrr ||
+          (N0Opc == X86::KANDWrr && Subtarget->hasDQI()) ||
+          N0Opc == X86::KANDDrr || N0Opc == X86::KANDQrr) {
+        unsigned NewOpc;
+        switch (Opc) {
+        default: llvm_unreachable("Unexpected opcode!");
+        case X86::KORTESTBrr: NewOpc = X86::KTESTBrr; break;
+        case X86::KORTESTWrr: NewOpc = X86::KTESTWrr; break;
+        case X86::KORTESTDrr: NewOpc = X86::KTESTDrr; break;
+        case X86::KORTESTQrr: NewOpc = X86::KTESTQrr; break;
+        }
+        MachineSDNode *KTest = CurDAG->getMachineNode(NewOpc, SDLoc(N),
+                                                      MVT::i32,
+                                                      And.getOperand(0),
+                                                      And.getOperand(1));
+        ReplaceUses(N, KTest);
+        MadeChange = true;
+        continue;
+      }
+    }
+
+    // Attempt to remove vectors moves that were inserted to zero upper bits.
+    if (Opc != TargetOpcode::SUBREG_TO_REG)
       continue;
 
     unsigned SubRegIdx = N->getConstantOperandVal(2);
@@ -1356,8 +1439,7 @@ static bool foldMaskAndShiftToScale(SelectionDAG &DAG, SDValue N,
   }
   APInt MaskedHighBits =
     APInt::getHighBitsSet(X.getSimpleValueType().getSizeInBits(), MaskLZ);
-  KnownBits Known;
-  DAG.computeKnownBits(X, Known);
+  KnownBits Known = DAG.computeKnownBits(X);
   if (MaskedHighBits != Known.Zero) return true;
 
   // We've identified a pattern that can be transformed into a single shift
@@ -2191,9 +2273,21 @@ bool X86DAGToDAGISel::isSExtAbsoluteSymbolRef(unsigned Width, SDNode *N) const {
          CR->getSignedMax().slt(1ull << Width);
 }
 
-/// Test whether the given X86ISD::CMP node has any uses which require the SF
-/// flag to be accurate.
-static bool hasNoSignFlagUses(SDValue Flags) {
+static X86::CondCode getCondFromOpc(unsigned Opc) {
+  X86::CondCode CC = X86::COND_INVALID;
+  if (CC == X86::COND_INVALID)
+    CC = X86::getCondFromBranchOpc(Opc);
+  if (CC == X86::COND_INVALID)
+    CC = X86::getCondFromSETOpc(Opc);
+  if (CC == X86::COND_INVALID)
+    CC = X86::getCondFromCMovOpc(Opc);
+
+  return CC;
+}
+
+/// Test whether the given X86ISD::CMP node has any users that use a flag
+/// other than ZF.
+bool X86DAGToDAGISel::onlyUsesZeroFlag(SDValue Flags) const {
   // Examine each user of the node.
   for (SDNode::use_iterator UI = Flags->use_begin(), UE = Flags->use_end();
          UI != UE; ++UI) {
@@ -2211,57 +2305,25 @@ static bool hasNoSignFlagUses(SDValue Flags) {
       if (FlagUI.getUse().getResNo() != 1) continue;
       // Anything unusual: assume conservatively.
       if (!FlagUI->isMachineOpcode()) return false;
-      // Examine the opcode of the user.
-      switch (FlagUI->getMachineOpcode()) {
-      // These comparisons don't treat the most significant bit specially.
-      case X86::SETAr: case X86::SETAEr: case X86::SETBr: case X86::SETBEr:
-      case X86::SETEr: case X86::SETNEr: case X86::SETOr: case X86::SETNOr:
-      case X86::SETPr: case X86::SETNPr:
-      case X86::SETAm: case X86::SETAEm: case X86::SETBm: case X86::SETBEm:
-      case X86::SETEm: case X86::SETNEm: case X86::SETOm: case X86::SETNOm:
-      case X86::SETPm: case X86::SETNPm:
-      case X86::JA_1: case X86::JAE_1: case X86::JB_1: case X86::JBE_1:
-      case X86::JE_1: case X86::JNE_1: case X86::JO_1: case X86::JNO_1:
-      case X86::JP_1: case X86::JNP_1:
-      case X86::CMOVA16rr: case X86::CMOVA16rm:
-      case X86::CMOVA32rr: case X86::CMOVA32rm:
-      case X86::CMOVA64rr: case X86::CMOVA64rm:
-      case X86::CMOVAE16rr: case X86::CMOVAE16rm:
-      case X86::CMOVAE32rr: case X86::CMOVAE32rm:
-      case X86::CMOVAE64rr: case X86::CMOVAE64rm:
-      case X86::CMOVB16rr: case X86::CMOVB16rm:
-      case X86::CMOVB32rr: case X86::CMOVB32rm:
-      case X86::CMOVB64rr: case X86::CMOVB64rm:
-      case X86::CMOVBE16rr: case X86::CMOVBE16rm:
-      case X86::CMOVBE32rr: case X86::CMOVBE32rm:
-      case X86::CMOVBE64rr: case X86::CMOVBE64rm:
-      case X86::CMOVE16rr: case X86::CMOVE16rm:
-      case X86::CMOVE32rr: case X86::CMOVE32rm:
-      case X86::CMOVE64rr: case X86::CMOVE64rm:
-      case X86::CMOVNE16rr: case X86::CMOVNE16rm:
-      case X86::CMOVNE32rr: case X86::CMOVNE32rm:
-      case X86::CMOVNE64rr: case X86::CMOVNE64rm:
-      case X86::CMOVNP16rr: case X86::CMOVNP16rm:
-      case X86::CMOVNP32rr: case X86::CMOVNP32rm:
-      case X86::CMOVNP64rr: case X86::CMOVNP64rm:
-      case X86::CMOVO16rr: case X86::CMOVO16rm:
-      case X86::CMOVO32rr: case X86::CMOVO32rm:
-      case X86::CMOVO64rr: case X86::CMOVO64rm:
-      case X86::CMOVP16rr: case X86::CMOVP16rm:
-      case X86::CMOVP32rr: case X86::CMOVP32rm:
-      case X86::CMOVP64rr: case X86::CMOVP64rm:
+      // Examine the condition code of the user.
+      X86::CondCode CC = getCondFromOpc(FlagUI->getMachineOpcode());
+
+      switch (CC) {
+      // Comparisons which only use the zero flag.
+      case X86::COND_E: case X86::COND_NE:
         continue;
       // Anything else: assume conservatively.
-      default: return false;
+      default:
+        return false;
       }
     }
   }
   return true;
 }
 
-/// Test whether the given node which sets flags has any uses which require the
-/// CF flag to be accurate.
-static bool hasNoCarryFlagUses(SDValue Flags) {
+/// Test whether the given X86ISD::CMP node has any uses which require the SF
+/// flag to be accurate.
+bool X86DAGToDAGISel::hasNoSignFlagUses(SDValue Flags) const {
   // Examine each user of the node.
   for (SDNode::use_iterator UI = Flags->use_begin(), UE = Flags->use_end();
          UI != UE; ++UI) {
@@ -2273,47 +2335,22 @@ static bool hasNoCarryFlagUses(SDValue Flags) {
         cast<RegisterSDNode>(UI->getOperand(1))->getReg() != X86::EFLAGS)
       return false;
     // Examine each user of the CopyToReg use.
-    for (SDNode::use_iterator FlagUI = UI->use_begin(), FlagUE = UI->use_end();
-         FlagUI != FlagUE; ++FlagUI) {
+    for (SDNode::use_iterator FlagUI = UI->use_begin(),
+           FlagUE = UI->use_end(); FlagUI != FlagUE; ++FlagUI) {
       // Only examine the Flag result.
-      if (FlagUI.getUse().getResNo() != 1)
-        continue;
+      if (FlagUI.getUse().getResNo() != 1) continue;
       // Anything unusual: assume conservatively.
-      if (!FlagUI->isMachineOpcode())
-        return false;
-      // Examine the opcode of the user.
-      switch (FlagUI->getMachineOpcode()) {
-      // Comparisons which don't examine the CF flag.
-      case X86::SETOr: case X86::SETNOr: case X86::SETEr: case X86::SETNEr:
-      case X86::SETSr: case X86::SETNSr: case X86::SETPr: case X86::SETNPr:
-      case X86::SETLr: case X86::SETGEr: case X86::SETLEr: case X86::SETGr:
-      case X86::JO_1: case X86::JNO_1: case X86::JE_1: case X86::JNE_1:
-      case X86::JS_1: case X86::JNS_1: case X86::JP_1: case X86::JNP_1:
-      case X86::JL_1: case X86::JGE_1: case X86::JLE_1: case X86::JG_1:
-      case X86::CMOVO16rr: case X86::CMOVO32rr: case X86::CMOVO64rr:
-      case X86::CMOVO16rm: case X86::CMOVO32rm: case X86::CMOVO64rm:
-      case X86::CMOVNO16rr: case X86::CMOVNO32rr: case X86::CMOVNO64rr:
-      case X86::CMOVNO16rm: case X86::CMOVNO32rm: case X86::CMOVNO64rm:
-      case X86::CMOVE16rr: case X86::CMOVE32rr: case X86::CMOVE64rr:
-      case X86::CMOVE16rm: case X86::CMOVE32rm: case X86::CMOVE64rm:
-      case X86::CMOVNE16rr: case X86::CMOVNE32rr: case X86::CMOVNE64rr:
-      case X86::CMOVNE16rm: case X86::CMOVNE32rm: case X86::CMOVNE64rm:
-      case X86::CMOVS16rr: case X86::CMOVS32rr: case X86::CMOVS64rr:
-      case X86::CMOVS16rm: case X86::CMOVS32rm: case X86::CMOVS64rm:
-      case X86::CMOVNS16rr: case X86::CMOVNS32rr: case X86::CMOVNS64rr:
-      case X86::CMOVNS16rm: case X86::CMOVNS32rm: case X86::CMOVNS64rm:
-      case X86::CMOVP16rr: case X86::CMOVP32rr: case X86::CMOVP64rr:
-      case X86::CMOVP16rm: case X86::CMOVP32rm: case X86::CMOVP64rm:
-      case X86::CMOVNP16rr: case X86::CMOVNP32rr: case X86::CMOVNP64rr:
-      case X86::CMOVNP16rm: case X86::CMOVNP32rm: case X86::CMOVNP64rm:
-      case X86::CMOVL16rr: case X86::CMOVL32rr: case X86::CMOVL64rr:
-      case X86::CMOVL16rm: case X86::CMOVL32rm: case X86::CMOVL64rm:
-      case X86::CMOVGE16rr: case X86::CMOVGE32rr: case X86::CMOVGE64rr:
-      case X86::CMOVGE16rm: case X86::CMOVGE32rm: case X86::CMOVGE64rm:
-      case X86::CMOVLE16rr: case X86::CMOVLE32rr: case X86::CMOVLE64rr:
-      case X86::CMOVLE16rm: case X86::CMOVLE32rm: case X86::CMOVLE64rm:
-      case X86::CMOVG16rr: case X86::CMOVG32rr: case X86::CMOVG64rr:
-      case X86::CMOVG16rm: case X86::CMOVG32rm: case X86::CMOVG64rm:
+      if (!FlagUI->isMachineOpcode()) return false;
+      // Examine the condition code of the user.
+      X86::CondCode CC = getCondFromOpc(FlagUI->getMachineOpcode());
+
+      switch (CC) {
+      // Comparisons which don't examine the SF flag.
+      case X86::COND_A: case X86::COND_AE:
+      case X86::COND_B: case X86::COND_BE:
+      case X86::COND_E: case X86::COND_NE:
+      case X86::COND_O: case X86::COND_NO:
+      case X86::COND_P: case X86::COND_NP:
         continue;
       // Anything else: assume conservatively.
       default:
@@ -2324,6 +2361,78 @@ static bool hasNoCarryFlagUses(SDValue Flags) {
   return true;
 }
 
+static bool mayUseCarryFlag(X86::CondCode CC) {
+  switch (CC) {
+  // Comparisons which don't examine the CF flag.
+  case X86::COND_O: case X86::COND_NO:
+  case X86::COND_E: case X86::COND_NE:
+  case X86::COND_S: case X86::COND_NS:
+  case X86::COND_P: case X86::COND_NP:
+  case X86::COND_L: case X86::COND_GE:
+  case X86::COND_G: case X86::COND_LE:
+    return false;
+  // Anything else: assume conservatively.
+  default:
+    return true;
+  }
+}
+
+/// Test whether the given node which sets flags has any uses which require the
+/// CF flag to be accurate.
+ bool X86DAGToDAGISel::hasNoCarryFlagUses(SDValue Flags) const {
+  // Examine each user of the node.
+  for (SDNode::use_iterator UI = Flags->use_begin(), UE = Flags->use_end();
+         UI != UE; ++UI) {
+    // Only check things that use the flags.
+    if (UI.getUse().getResNo() != Flags.getResNo())
+      continue;
+
+    unsigned UIOpc = UI->getOpcode();
+
+    if (UIOpc == ISD::CopyToReg) {
+      // Only examine CopyToReg uses that copy to EFLAGS.
+      if (cast<RegisterSDNode>(UI->getOperand(1))->getReg() != X86::EFLAGS)
+        return false;
+      // Examine each user of the CopyToReg use.
+      for (SDNode::use_iterator FlagUI = UI->use_begin(), FlagUE = UI->use_end();
+           FlagUI != FlagUE; ++FlagUI) {
+        // Only examine the Flag result.
+        if (FlagUI.getUse().getResNo() != 1)
+          continue;
+        // Anything unusual: assume conservatively.
+        if (!FlagUI->isMachineOpcode())
+          return false;
+        // Examine the condition code of the user.
+        X86::CondCode CC = getCondFromOpc(FlagUI->getMachineOpcode());
+
+        if (mayUseCarryFlag(CC))
+          return false;
+      }
+
+      // This CopyToReg is ok. Move on to the next user.
+      continue;
+    }
+
+    // This might be an unselected node. So look for the pre-isel opcodes that
+    // use flags.
+    unsigned CCOpNo;
+    switch (UIOpc) {
+    default:
+      // Something unusual. Be conservative.
+      return false;
+    case X86ISD::SETCC:       CCOpNo = 0; break;
+    case X86ISD::SETCC_CARRY: CCOpNo = 0; break;
+    case X86ISD::CMOV:        CCOpNo = 2; break;
+    case X86ISD::BRCOND:      CCOpNo = 2; break;
+    }
+
+    X86::CondCode CC = (X86::CondCode)UI->getConstantOperandVal(CCOpNo);
+    if (mayUseCarryFlag(CC))
+      return false;
+  }
+  return true;
+}
+
 /// Check whether or not the chain ending in StoreNode is suitable for doing
 /// the {load; op; store} to modify transformation.
 static bool isFusableLoadOpStorePattern(StoreSDNode *StoreNode,
@@ -2475,8 +2584,6 @@ bool X86DAGToDAGISel::foldLoadStoreIntoMemOperand(SDNode *Node) {
   switch (Opc) {
   default:
     return false;
-  case X86ISD::INC:
-  case X86ISD::DEC:
   case X86ISD::SUB:
   case X86ISD::SBB:
     break;
@@ -2527,20 +2634,27 @@ bool X86DAGToDAGISel::foldLoadStoreIntoMemOperand(SDNode *Node) {
 
   MachineSDNode *Result;
   switch (Opc) {
-  case X86ISD::INC:
-  case X86ISD::DEC: {
-    unsigned NewOpc =
-        Opc == X86ISD::INC
-            ? SelectOpcode(X86::INC64m, X86::INC32m, X86::INC16m, X86::INC8m)
-            : SelectOpcode(X86::DEC64m, X86::DEC32m, X86::DEC16m, X86::DEC8m);
-    const SDValue Ops[] = {Base, Scale, Index, Disp, Segment, InputChain};
-    Result =
-        CurDAG->getMachineNode(NewOpc, SDLoc(Node), MVT::i32, MVT::Other, Ops);
-    break;
-  }
   case X86ISD::ADD:
-  case X86ISD::ADC:
   case X86ISD::SUB:
+    // Try to match inc/dec.
+    if (!Subtarget->slowIncDec() ||
+        CurDAG->getMachineFunction().getFunction().optForSize()) {
+      bool IsOne = isOneConstant(StoredVal.getOperand(1));
+      bool IsNegOne = isAllOnesConstant(StoredVal.getOperand(1));
+      // ADD/SUB with 1/-1 and carry flag isn't used can use inc/dec.
+      if ((IsOne || IsNegOne) && hasNoCarryFlagUses(StoredVal.getValue(1))) {
+        unsigned NewOpc = 
+          ((Opc == X86ISD::ADD) == IsOne)
+              ? SelectOpcode(X86::INC64m, X86::INC32m, X86::INC16m, X86::INC8m)
+              : SelectOpcode(X86::DEC64m, X86::DEC32m, X86::DEC16m, X86::DEC8m);
+        const SDValue Ops[] = {Base, Scale, Index, Disp, Segment, InputChain};
+        Result = CurDAG->getMachineNode(NewOpc, SDLoc(Node), MVT::i32,
+                                        MVT::Other, Ops);
+        break;
+      }
+    }
+    LLVM_FALLTHROUGH;
+  case X86ISD::ADC:
   case X86ISD::SBB:
   case X86ISD::AND:
   case X86ISD::OR:
@@ -2831,25 +2945,45 @@ bool X86DAGToDAGISel::matchBitExtract(SDNode *Node) {
 
   SDLoc DL(Node);
 
-  SDValue OrigNBits = NBits;
-  if (NBits.getValueType() != NVT) {
-    // Truncate the shift amount.
-    NBits = CurDAG->getNode(ISD::TRUNCATE, DL, MVT::i8, NBits);
-    insertDAGNode(*CurDAG, OrigNBits, NBits);
-
-    // Insert 8-bit NBits into lowest 8 bits of NVT-sized (32 or 64-bit)
-    // register. All the other bits are undefined, we do not care about them.
-    SDValue ImplDef =
-        SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, DL, NVT), 0);
-    insertDAGNode(*CurDAG, OrigNBits, ImplDef);
-    NBits =
-        CurDAG->getTargetInsertSubreg(X86::sub_8bit, DL, NVT, ImplDef, NBits);
-    insertDAGNode(*CurDAG, OrigNBits, NBits);
+  // If we do *NOT* have BMI2, let's find out if the if the 'X' is *logically*
+  // shifted (potentially with one-use trunc inbetween),
+  // and if so look past one-use truncation.
+  MVT XVT = NVT;
+  if (!Subtarget->hasBMI2() && X.getOpcode() == ISD::TRUNCATE &&
+      X.hasOneUse() && X.getOperand(0).getOpcode() == ISD::SRL) {
+    assert(NVT == MVT::i32 && "Expected target valuetype to be i32");
+    X = X.getOperand(0);
+    XVT = X.getSimpleValueType();
+    assert(XVT == MVT::i64 && "Expected truncation from i64");
   }
 
+  SDValue OrigNBits = NBits;
+  // Truncate the shift amount.
+  NBits = CurDAG->getNode(ISD::TRUNCATE, DL, MVT::i8, NBits);
+  insertDAGNode(*CurDAG, OrigNBits, NBits);
+
+  // Insert 8-bit NBits into lowest 8 bits of 32-bit register.
+  // All the other bits are undefined, we do not care about them.
+  SDValue ImplDef = SDValue(
+      CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, DL, MVT::i32), 0);
+  insertDAGNode(*CurDAG, OrigNBits, ImplDef);
+  NBits = CurDAG->getTargetInsertSubreg(X86::sub_8bit, DL, MVT::i32, ImplDef,
+                                        NBits);
+  insertDAGNode(*CurDAG, OrigNBits, NBits);
+
   if (Subtarget->hasBMI2()) {
     // Great, just emit the the BZHI..
-    SDValue Extract = CurDAG->getNode(X86ISD::BZHI, DL, NVT, X, NBits);
+    if (XVT != MVT::i32) {
+      // But have to place the bit count into the wide-enough register first.
+      SDValue ImplDef = SDValue(
+          CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, DL, XVT), 0);
+      insertDAGNode(*CurDAG, OrigNBits, ImplDef);
+      NBits = CurDAG->getTargetInsertSubreg(X86::sub_32bit, DL, XVT, ImplDef,
+                                            NBits);
+      insertDAGNode(*CurDAG, OrigNBits, NBits);
+    }
+
+    SDValue Extract = CurDAG->getNode(X86ISD::BZHI, DL, XVT, X, NBits);
     ReplaceNode(Node, Extract.getNode());
     SelectCode(Extract.getNode());
     return true;
@@ -2864,7 +2998,7 @@ bool X86DAGToDAGISel::matchBitExtract(SDNode *Node) {
   // Shift NBits left by 8 bits, thus producing 'control'.
   // This makes the low 8 bits to be zero.
   SDValue C8 = CurDAG->getConstant(8, DL, MVT::i8);
-  SDValue Control = CurDAG->getNode(ISD::SHL, DL, NVT, NBits, C8);
+  SDValue Control = CurDAG->getNode(ISD::SHL, DL, MVT::i32, NBits, C8);
   insertDAGNode(*CurDAG, OrigNBits, Control);
 
   // If the 'X' is *logically* shifted, we can fold that shift into 'control'.
@@ -2876,17 +3010,35 @@ bool X86DAGToDAGISel::matchBitExtract(SDNode *Node) {
            "Expected shift amount to be i8");
 
     // Now, *zero*-extend the shift amount. The bits 8...15 *must* be zero!
+    // We could zext to i16 in some form, but we intentionally don't do that.
     SDValue OrigShiftAmt = ShiftAmt;
-    ShiftAmt = CurDAG->getNode(ISD::ZERO_EXTEND, DL, NVT, ShiftAmt);
+    ShiftAmt = CurDAG->getNode(ISD::ZERO_EXTEND, DL, MVT::i32, ShiftAmt);
     insertDAGNode(*CurDAG, OrigShiftAmt, ShiftAmt);
 
     // And now 'or' these low 8 bits of shift amount into the 'control'.
-    Control = CurDAG->getNode(ISD::OR, DL, NVT, Control, ShiftAmt);
+    Control = CurDAG->getNode(ISD::OR, DL, MVT::i32, Control, ShiftAmt);
+    insertDAGNode(*CurDAG, OrigNBits, Control);
+  }
+
+  // But have to place the 'control' into the wide-enough register first.
+  if (XVT != MVT::i32) {
+    SDValue ImplDef =
+        SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, DL, XVT), 0);
+    insertDAGNode(*CurDAG, OrigNBits, ImplDef);
+    Control = CurDAG->getTargetInsertSubreg(X86::sub_32bit, DL, XVT, ImplDef,
+                                            Control);
     insertDAGNode(*CurDAG, OrigNBits, Control);
   }
 
   // And finally, form the BEXTR itself.
-  SDValue Extract = CurDAG->getNode(X86ISD::BEXTR, DL, NVT, X, Control);
+  SDValue Extract = CurDAG->getNode(X86ISD::BEXTR, DL, XVT, X, Control);
+
+  // The 'X' was originally truncated. Do that now.
+  if (XVT != NVT) {
+    insertDAGNode(*CurDAG, OrigNBits, Extract);
+    Extract = CurDAG->getNode(ISD::TRUNCATE, DL, NVT, Extract);
+  }
+
   ReplaceNode(Node, Extract.getNode());
   SelectCode(Extract.getNode());
 
@@ -3247,14 +3399,17 @@ void X86DAGToDAGISel::Select(SDNode *Node) {
     }
     break;
 
-  case X86ISD::SELECT:
-  case X86ISD::SHRUNKBLEND: {
-    // SHRUNKBLEND selects like a regular VSELECT. Same with X86ISD::SELECT.
-    SDValue VSelect = CurDAG->getNode(
-        ISD::VSELECT, SDLoc(Node), Node->getValueType(0), Node->getOperand(0),
+  case ISD::VSELECT: {
+    // Replace VSELECT with non-mask conditions with with BLENDV.
+    if (Node->getOperand(0).getValueType().getVectorElementType() == MVT::i1)
+      break;
+
+    assert(Subtarget->hasSSE41() && "Expected SSE4.1 support!");
+    SDValue Blendv = CurDAG->getNode(
+        X86ISD::BLENDV, SDLoc(Node), Node->getValueType(0), Node->getOperand(0),
         Node->getOperand(1), Node->getOperand(2));
-    ReplaceNode(Node, VSelect.getNode());
-    SelectCode(VSelect.getNode());
+    ReplaceNode(Node, Blendv.getNode());
+    SelectCode(Blendv.getNode());
     // We already called ReplaceUses.
     return;
   }
@@ -3365,45 +3520,85 @@ void X86DAGToDAGISel::Select(SDNode *Node) {
                            getI8Imm(ShlVal, dl));
     return;
   }
-  case X86ISD::UMUL8:
-  case X86ISD::SMUL8: {
-    SDValue N0 = Node->getOperand(0);
-    SDValue N1 = Node->getOperand(1);
-
-    unsigned Opc = (Opcode == X86ISD::SMUL8 ? X86::IMUL8r : X86::MUL8r);
-
-    SDValue InFlag = CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, X86::AL,
-                                          N0, SDValue()).getValue(1);
-
-    SDVTList VTs = CurDAG->getVTList(NVT, MVT::i32);
-    SDValue Ops[] = {N1, InFlag};
-    SDNode *CNode = CurDAG->getMachineNode(Opc, dl, VTs, Ops);
-
-    ReplaceNode(Node, CNode);
-    return;
-  }
-
+  case X86ISD::SMUL:
+    // i16/i32/i64 are handled with isel patterns.
+    if (NVT != MVT::i8)
+      break;
+    LLVM_FALLTHROUGH;
   case X86ISD::UMUL: {
     SDValue N0 = Node->getOperand(0);
     SDValue N1 = Node->getOperand(1);
 
-    unsigned LoReg, Opc;
+    unsigned LoReg, ROpc, MOpc;
     switch (NVT.SimpleTy) {
     default: llvm_unreachable("Unsupported VT!");
-    // MVT::i8 is handled by X86ISD::UMUL8.
-    case MVT::i16: LoReg = X86::AX;  Opc = X86::MUL16r; break;
-    case MVT::i32: LoReg = X86::EAX; Opc = X86::MUL32r; break;
-    case MVT::i64: LoReg = X86::RAX; Opc = X86::MUL64r; break;
+    case MVT::i8:
+      LoReg = X86::AL;
+      ROpc = Opcode == X86ISD::SMUL ? X86::IMUL8r : X86::MUL8r;
+      MOpc = Opcode == X86ISD::SMUL ? X86::IMUL8m : X86::MUL8m;
+      break;
+    case MVT::i16:
+      LoReg = X86::AX;
+      ROpc = X86::MUL16r;
+      MOpc = X86::MUL16m;
+      break;
+    case MVT::i32:
+      LoReg = X86::EAX;
+      ROpc = X86::MUL32r;
+      MOpc = X86::MUL32m;
+      break;
+    case MVT::i64:
+      LoReg = X86::RAX;
+      ROpc = X86::MUL64r;
+      MOpc = X86::MUL64m;
+      break;
+    }
+
+    SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
+    bool FoldedLoad = tryFoldLoad(Node, N1, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4);
+    // Multiply is commmutative.
+    if (!FoldedLoad) {
+      FoldedLoad = tryFoldLoad(Node, N0, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4);
+      if (FoldedLoad)
+        std::swap(N0, N1);
     }
 
     SDValue InFlag = CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, LoReg,
                                           N0, SDValue()).getValue(1);
 
-    SDVTList VTs = CurDAG->getVTList(NVT, NVT, MVT::i32);
-    SDValue Ops[] = {N1, InFlag};
-    SDNode *CNode = CurDAG->getMachineNode(Opc, dl, VTs, Ops);
+    MachineSDNode *CNode;
+    if (FoldedLoad) {
+      // i16/i32/i64 use an instruction that produces a low and high result even
+      // though only the low result is used.
+      SDVTList VTs;
+      if (NVT == MVT::i8)
+        VTs = CurDAG->getVTList(NVT, MVT::i32, MVT::Other);
+      else
+        VTs = CurDAG->getVTList(NVT, NVT, MVT::i32, MVT::Other);
 
-    ReplaceNode(Node, CNode);
+      SDValue Ops[] = { Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, N1.getOperand(0),
+                        InFlag };
+      CNode = CurDAG->getMachineNode(MOpc, dl, VTs, Ops);
+
+      // Update the chain.
+      ReplaceUses(N1.getValue(1), SDValue(CNode, NVT == MVT::i8 ? 2 : 3));
+      // Record the mem-refs
+      CurDAG->setNodeMemRefs(CNode, {cast<LoadSDNode>(N1)->getMemOperand()});
+    } else {
+      // i16/i32/i64 use an instruction that produces a low and high result even
+      // though only the low result is used.
+      SDVTList VTs;
+      if (NVT == MVT::i8)
+        VTs = CurDAG->getVTList(NVT, MVT::i32);
+      else
+        VTs = CurDAG->getVTList(NVT, NVT, MVT::i32);
+
+      CNode = CurDAG->getMachineNode(ROpc, dl, VTs, {N1, InFlag});
+    }
+
+    ReplaceUses(SDValue(Node, 0), SDValue(CNode, 0));
+    ReplaceUses(SDValue(Node, 1), SDValue(CNode, NVT == MVT::i8 ? 1 : 2));
+    CurDAG->RemoveDeadNode(Node);
     return;
   }
 
@@ -3676,6 +3871,10 @@ void X86DAGToDAGISel::Select(SDNode *Node) {
     SDValue N0 = Node->getOperand(0);
     SDValue N1 = Node->getOperand(1);
 
+    // Optimizations for TEST compares.
+    if (!isNullConstant(N1))
+      break;
+
     // Save the original VT of the compare.
     MVT CmpVT = N0.getSimpleValueType();
 
@@ -3683,7 +3882,7 @@ void X86DAGToDAGISel::Select(SDNode *Node) {
     // by a test instruction. The test should be removed later by
     // analyzeCompare if we are using only the zero flag.
     // TODO: Should we check the users and use the BEXTR flags directly?
-    if (isNullConstant(N1) && N0.getOpcode() == ISD::AND && N0.hasOneUse()) {
+    if (N0.getOpcode() == ISD::AND && N0.hasOneUse()) {
       if (MachineSDNode *NewNode = matchBEXTRFromAndImm(N0.getNode())) {
         unsigned TestOpc = CmpVT == MVT::i64 ? X86::TEST64rr
                                              : X86::TEST32rr;
@@ -3704,12 +3903,40 @@ void X86DAGToDAGISel::Select(SDNode *Node) {
     // Look past the truncate if CMP is the only use of it.
     if (N0.getOpcode() == ISD::AND &&
         N0.getNode()->hasOneUse() &&
-        N0.getValueType() != MVT::i8 &&
-        isNullConstant(N1)) {
+        N0.getValueType() != MVT::i8) {
       ConstantSDNode *C = dyn_cast<ConstantSDNode>(N0.getOperand(1));
       if (!C) break;
       uint64_t Mask = C->getZExtValue();
 
+      // Check if we can replace AND+IMM64 with a shift. This is possible for
+      // masks/ like 0xFF000000 or 0x00FFFFFF and if we care only about the zero
+      // flag.
+      if (CmpVT == MVT::i64 && !isInt<32>(Mask) &&
+          onlyUsesZeroFlag(SDValue(Node, 0))) {
+        if (isMask_64(~Mask)) {
+          unsigned TrailingZeros = countTrailingZeros(Mask);
+          SDValue Imm = CurDAG->getTargetConstant(TrailingZeros, dl, MVT::i64);
+          SDValue Shift =
+            SDValue(CurDAG->getMachineNode(X86::SHR64ri, dl, MVT::i64,
+                                           N0.getOperand(0), Imm), 0);
+          MachineSDNode *Test = CurDAG->getMachineNode(X86::TEST64rr, dl,
+                                                       MVT::i32, Shift, Shift);
+          ReplaceNode(Node, Test);
+          return;
+        }
+        if (isMask_64(Mask)) {
+          unsigned LeadingZeros = countLeadingZeros(Mask);
+          SDValue Imm = CurDAG->getTargetConstant(LeadingZeros, dl, MVT::i64);
+          SDValue Shift =
+            SDValue(CurDAG->getMachineNode(X86::SHL64ri, dl, MVT::i64,
+                                           N0.getOperand(0), Imm), 0);
+          MachineSDNode *Test = CurDAG->getMachineNode(X86::TEST64rr, dl,
+                                                       MVT::i32, Shift, Shift);
+          ReplaceNode(Node, Test);
+          return;
+        }
+      }
+
       MVT VT;
       int SubRegOp;
       unsigned ROpc, MOpc;
diff --git a/lib/Target/X86/X86ISelLowering.cpp b/lib/Target/X86/X86ISelLowering.cpp
index 6eb21ab7087d9c001aac3bf9e227fe76894d262e..ac4bce0e45409bee8989015f0b5990e07ed961af 100644
--- a/lib/Target/X86/X86ISelLowering.cpp
+++ b/lib/Target/X86/X86ISelLowering.cpp
@@ -1,9 +1,8 @@
 //===-- X86ISelLowering.cpp - X86 DAG Lowering Implementation -------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -540,6 +539,10 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
       // Use ANDPD and ORPD to simulate FCOPYSIGN.
       setOperationAction(ISD::FCOPYSIGN, VT, Custom);
 
+      // These might be better off as horizontal vector ops.
+      setOperationAction(ISD::FADD, VT, Custom);
+      setOperationAction(ISD::FSUB, VT, Custom);
+
       // We don't support sin/cos/fmod
       setOperationAction(ISD::FSIN   , VT, Expand);
       setOperationAction(ISD::FCOS   , VT, Expand);
@@ -837,13 +840,16 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
     setOperationAction(ISD::SADDSAT,            MVT::v8i16, Legal);
     setOperationAction(ISD::USUBSAT,            MVT::v8i16, Legal);
     setOperationAction(ISD::SSUBSAT,            MVT::v8i16, Legal);
-    // Use widening instead of promotion.
-    for (auto VT : { MVT::v8i8, MVT::v4i8, MVT::v2i8,
-                     MVT::v4i16, MVT::v2i16 }) {
-      setOperationAction(ISD::UADDSAT, VT, Custom);
-      setOperationAction(ISD::SADDSAT, VT, Custom);
-      setOperationAction(ISD::USUBSAT, VT, Custom);
-      setOperationAction(ISD::SSUBSAT, VT, Custom);
+
+    if (!ExperimentalVectorWideningLegalization) {
+      // Use widening instead of promotion.
+      for (auto VT : { MVT::v8i8, MVT::v4i8, MVT::v2i8,
+                       MVT::v4i16, MVT::v2i16 }) {
+        setOperationAction(ISD::UADDSAT, VT, Custom);
+        setOperationAction(ISD::SADDSAT, VT, Custom);
+        setOperationAction(ISD::USUBSAT, VT, Custom);
+        setOperationAction(ISD::SSUBSAT, VT, Custom);
+      }
     }
 
     setOperationAction(ISD::INSERT_VECTOR_ELT,  MVT::v8i16, Custom);
@@ -860,6 +866,7 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
     for (auto VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64 }) {
       setOperationAction(ISD::SETCC,              VT, Custom);
       setOperationAction(ISD::CTPOP,              VT, Custom);
+      setOperationAction(ISD::ABS,                VT, Custom);
 
       // The condition codes aren't legal in SSE/AVX and under AVX512 we use
       // setcc all the way to isel and prefer SETGT in some isel patterns.
@@ -880,12 +887,6 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
     // scalars) and extend in-register to a legal 128-bit vector type. For sext
     // loads these must work with a single scalar load.
     for (MVT VT : MVT::integer_vector_valuetypes()) {
-      if (!ExperimentalVectorWideningLegalization) {
-        // We don't want narrow result types here when widening.
-        setLoadExtAction(ISD::SEXTLOAD, VT, MVT::v4i8, Custom);
-        setLoadExtAction(ISD::SEXTLOAD, VT, MVT::v4i16, Custom);
-        setLoadExtAction(ISD::SEXTLOAD, VT, MVT::v8i8, Custom);
-      }
       setLoadExtAction(ISD::EXTLOAD, VT, MVT::v2i8, Custom);
       setLoadExtAction(ISD::EXTLOAD, VT, MVT::v2i16, Custom);
       setLoadExtAction(ISD::EXTLOAD, VT, MVT::v2i32, Custom);
@@ -894,14 +895,6 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
       setLoadExtAction(ISD::EXTLOAD, VT, MVT::v8i8, Custom);
     }
 
-    if (ExperimentalVectorWideningLegalization &&
-        !Subtarget.hasSSE41() && Subtarget.is64Bit()) {
-      // This lets DAG combine create sextloads that get split and scalarized.
-      // TODO: Does this make sense? What about v2i8->v2i64?
-      setLoadExtAction(ISD::SEXTLOAD, MVT::v4i64, MVT::v4i8,  Custom);
-      setLoadExtAction(ISD::SEXTLOAD, MVT::v8i64, MVT::v8i8,  Custom);
-    }
-
     for (auto VT : { MVT::v2f64, MVT::v2i64 }) {
       setOperationAction(ISD::BUILD_VECTOR,       VT, Custom);
       setOperationAction(ISD::VECTOR_SHUFFLE,     VT, Custom);
@@ -990,6 +983,8 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
       setOperationAction(ISD::TRUNCATE,    MVT::v4i8,  Custom);
       setOperationAction(ISD::TRUNCATE,    MVT::v4i16, Custom);
       setOperationAction(ISD::TRUNCATE,    MVT::v8i8,  Custom);
+    } else {
+      setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, MVT::v4i64, Custom);
     }
 
     // In the customized shift lowering, the legal v4i32/v2i64 cases
@@ -1003,8 +998,8 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
     setOperationAction(ISD::ROTL,               MVT::v4i32, Custom);
     setOperationAction(ISD::ROTL,               MVT::v8i16, Custom);
 
-    // With BWI, expanding (and promoting the shifts) is the better.
-    if (!Subtarget.hasBWI())
+    // With AVX512, expanding (and promoting the shifts) is better.
+    if (!Subtarget.hasAVX512())
       setOperationAction(ISD::ROTL,             MVT::v16i8, Custom);
   }
 
@@ -1017,6 +1012,12 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
     setOperationAction(ISD::CTLZ,               MVT::v8i16, Custom);
     setOperationAction(ISD::CTLZ,               MVT::v4i32, Custom);
     setOperationAction(ISD::CTLZ,               MVT::v2i64, Custom);
+
+    // These might be better off as horizontal vector ops.
+    setOperationAction(ISD::ADD,                MVT::i16, Custom);
+    setOperationAction(ISD::ADD,                MVT::i32, Custom);
+    setOperationAction(ISD::SUB,                MVT::i16, Custom);
+    setOperationAction(ISD::SUB,                MVT::i32, Custom);
   }
 
   if (!Subtarget.useSoftFloat() && Subtarget.hasSSE41()) {
@@ -1212,6 +1213,7 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
     setOperationAction(ISD::MULHU,     MVT::v32i8,  Custom);
     setOperationAction(ISD::MULHS,     MVT::v32i8,  Custom);
 
+    setOperationAction(ISD::ABS,       MVT::v4i64,  Custom);
     setOperationAction(ISD::SMAX,      MVT::v4i64,  Custom);
     setOperationAction(ISD::UMAX,      MVT::v4i64,  Custom);
     setOperationAction(ISD::SMIN,      MVT::v4i64,  Custom);
@@ -1535,6 +1537,13 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
       setOperationAction(ISD::BITCAST, MVT::v32i16, Custom);
       setOperationAction(ISD::BITCAST, MVT::v64i8,  Custom);
     }
+
+    if (Subtarget.hasVBMI2()) {
+      for (auto VT : { MVT::v16i32, MVT::v8i64 }) {
+        setOperationAction(ISD::FSHL, VT, Custom);
+        setOperationAction(ISD::FSHR, VT, Custom);
+      }
+    }
   }// has  AVX-512
 
   // This block controls legalization for operations that don't have
@@ -1710,6 +1719,11 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
       for (auto VT : { MVT::v64i8, MVT::v32i16 })
         setOperationAction(ISD::CTPOP, VT, Legal);
     }
+
+    if (Subtarget.hasVBMI2()) {
+      setOperationAction(ISD::FSHL, MVT::v32i16, Custom);
+      setOperationAction(ISD::FSHR, MVT::v32i16, Custom);
+    }
   }
 
   if (!Subtarget.useSoftFloat() && Subtarget.hasBWI()) {
@@ -1756,6 +1770,15 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
       setTruncStoreAction(MVT::v16i16,  MVT::v16i8, Legal);
       setTruncStoreAction(MVT::v8i16,   MVT::v8i8,  Legal);
     }
+
+    if (Subtarget.hasVBMI2()) {
+      // TODO: Make these legal even without VLX?
+      for (auto VT : { MVT::v8i16,  MVT::v4i32, MVT::v2i64,
+                       MVT::v16i16, MVT::v8i32, MVT::v4i64 }) {
+        setOperationAction(ISD::FSHL, VT, Custom);
+        setOperationAction(ISD::FSHR, VT, Custom);
+      }
+    }
   }
 
   // We want to custom lower some of our intrinsics.
@@ -1764,7 +1787,6 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
   setOperationAction(ISD::INTRINSIC_VOID, MVT::Other, Custom);
   if (!Subtarget.is64Bit()) {
     setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::i64, Custom);
-    setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::i64, Custom);
   }
 
   // Only custom-lower 64-bit SADDO and friends on 64-bit because we don't
@@ -2257,12 +2279,13 @@ void X86TargetLowering::insertSSPDeclarations(Module &M) const {
                         Type::getInt8PtrTy(M.getContext()));
 
     // MSVC CRT has a function to validate security cookie.
-    auto *SecurityCheckCookie = cast<Function>(
-        M.getOrInsertFunction("__security_check_cookie",
-                              Type::getVoidTy(M.getContext()),
-                              Type::getInt8PtrTy(M.getContext())));
-    SecurityCheckCookie->setCallingConv(CallingConv::X86_FastCall);
-    SecurityCheckCookie->addAttribute(1, Attribute::AttrKind::InReg);
+    FunctionCallee SecurityCheckCookie = M.getOrInsertFunction(
+        "__security_check_cookie", Type::getVoidTy(M.getContext()),
+        Type::getInt8PtrTy(M.getContext()));
+    if (Function *F = dyn_cast<Function>(SecurityCheckCookie.getCallee())) {
+      F->setCallingConv(CallingConv::X86_FastCall);
+      F->addAttribute(1, Attribute::AttrKind::InReg);
+    }
     return;
   }
   // glibc, bionic, and Fuchsia have a special slot for the stack guard.
@@ -2280,7 +2303,7 @@ Value *X86TargetLowering::getSDagStackGuard(const Module &M) const {
   return TargetLowering::getSDagStackGuard(M);
 }
 
-Value *X86TargetLowering::getSSPStackGuardCheck(const Module &M) const {
+Function *X86TargetLowering::getSSPStackGuardCheck(const Module &M) const {
   // MSVC CRT has a function to validate security cookie.
   if (Subtarget.getTargetTriple().isWindowsMSVCEnvironment() ||
       Subtarget.getTargetTriple().isWindowsItaniumEnvironment()) {
@@ -2323,8 +2346,6 @@ bool X86TargetLowering::isNoopAddrSpaceCast(unsigned SrcAS,
 //               Return Value Calling Convention Implementation
 //===----------------------------------------------------------------------===//
 
-#include "X86GenCallingConv.inc"
-
 bool X86TargetLowering::CanLowerReturn(
     CallingConv::ID CallConv, MachineFunction &MF, bool isVarArg,
     const SmallVectorImpl<ISD::OutputArg> &Outs, LLVMContext &Context) const {
@@ -4786,6 +4807,29 @@ bool X86TargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
     Info.flags |= MachineMemOperand::MOStore;
     break;
   }
+  case GATHER:
+  case GATHER_AVX2: {
+    Info.ptrVal = nullptr;
+    MVT DataVT = MVT::getVT(I.getType());
+    MVT IndexVT = MVT::getVT(I.getArgOperand(2)->getType());
+    unsigned NumElts = std::min(DataVT.getVectorNumElements(),
+                                IndexVT.getVectorNumElements());
+    Info.memVT = MVT::getVectorVT(DataVT.getVectorElementType(), NumElts);
+    Info.align = 1;
+    Info.flags |= MachineMemOperand::MOLoad;
+    break;
+  }
+  case SCATTER: {
+    Info.ptrVal = nullptr;
+    MVT DataVT = MVT::getVT(I.getArgOperand(3)->getType());
+    MVT IndexVT = MVT::getVT(I.getArgOperand(2)->getType());
+    unsigned NumElts = std::min(DataVT.getVectorNumElements(),
+                                IndexVT.getVectorNumElements());
+    Info.memVT = MVT::getVectorVT(DataVT.getVectorElementType(), NumElts);
+    Info.align = 1;
+    Info.flags |= MachineMemOperand::MOStore;
+    break;
+  }
   default:
     return false;
   }
@@ -4884,6 +4928,18 @@ bool X86TargetLowering::isExtractSubvectorCheap(EVT ResVT, EVT SrcVT,
   return (Index % ResVT.getVectorNumElements()) == 0;
 }
 
+bool X86TargetLowering::shouldScalarizeBinop(SDValue VecOp) const {
+  // If the vector op is not supported, try to convert to scalar.
+  EVT VecVT = VecOp.getValueType();
+  if (!isOperationLegalOrCustomOrPromote(VecOp.getOpcode(), VecVT))
+    return true;
+
+  // If the vector op is supported, but the scalar op is not, the transform may
+  // not be worthwhile.
+  EVT ScalarVT = VecVT.getScalarType();
+  return isOperationLegalOrCustomOrPromote(VecOp.getOpcode(), ScalarVT);
+}
+
 bool X86TargetLowering::isCheapToSpeculateCttz() const {
   // Speculate cttz only if we can directly use TZCNT.
   return Subtarget.hasBMI();
@@ -4942,17 +4998,14 @@ bool X86TargetLowering::hasAndNotCompare(SDValue Y) const {
   if (VT != MVT::i32 && VT != MVT::i64)
     return false;
 
-  // A mask and compare against constant is ok for an 'andn' too
-  // even though the BMI instruction doesn't have an immediate form.
-
-  return true;
+  return !isa<ConstantSDNode>(Y);
 }
 
 bool X86TargetLowering::hasAndNot(SDValue Y) const {
   EVT VT = Y.getValueType();
 
-  if (!VT.isVector()) // x86 can't form 'andn' with an immediate.
-    return !isa<ConstantSDNode>(Y) && hasAndNotCompare(Y);
+  if (!VT.isVector())
+    return hasAndNotCompare(Y);
 
   // Vector.
 
@@ -5015,8 +5068,8 @@ static bool isUndefOrZero(int Val) {
   return ((Val == SM_SentinelUndef) || (Val == SM_SentinelZero));
 }
 
-/// Return true if every element in Mask, beginning
-/// from position Pos and ending in Pos+Size is the undef sentinel value.
+/// Return true if every element in Mask, beginning from position Pos and ending
+/// in Pos+Size is the undef sentinel value.
 static bool isUndefInRange(ArrayRef<int> Mask, unsigned Pos, unsigned Size) {
   for (unsigned i = Pos, e = Pos + Size; i != e; ++i)
     if (Mask[i] != SM_SentinelUndef)
@@ -5024,6 +5077,18 @@ static bool isUndefInRange(ArrayRef<int> Mask, unsigned Pos, unsigned Size) {
   return true;
 }
 
+/// Return true if the mask creates a vector whose lower half is undefined.
+static bool isUndefLowerHalf(ArrayRef<int> Mask) {
+  unsigned NumElts = Mask.size();
+  return isUndefInRange(Mask, 0, NumElts / 2);
+}
+
+/// Return true if the mask creates a vector whose upper half is undefined.
+static bool isUndefUpperHalf(ArrayRef<int> Mask) {
+  unsigned NumElts = Mask.size();
+  return isUndefInRange(Mask, NumElts / 2, NumElts / 2);
+}
+
 /// Return true if Val falls within the specified range (L, H].
 static bool isInRange(int Val, int Low, int Hi) {
   return (Val >= Low && Val < Hi);
@@ -6435,14 +6500,14 @@ static bool setTargetShuffleZeroElements(SDValue N,
 static bool resolveTargetShuffleInputs(SDValue Op,
                                        SmallVectorImpl<SDValue> &Inputs,
                                        SmallVectorImpl<int> &Mask,
-                                       const SelectionDAG &DAG);
+                                       SelectionDAG &DAG);
 
 // Attempt to decode ops that could be represented as a shuffle mask.
 // The decoded shuffle mask may contain a different number of elements to the
 // destination value type.
 static bool getFauxShuffleMask(SDValue N, SmallVectorImpl<int> &Mask,
                                SmallVectorImpl<SDValue> &Ops,
-                               const SelectionDAG &DAG) {
+                               SelectionDAG &DAG) {
   Mask.clear();
   Ops.clear();
 
@@ -6450,8 +6515,8 @@ static bool getFauxShuffleMask(SDValue N, SmallVectorImpl<int> &Mask,
   unsigned NumElts = VT.getVectorNumElements();
   unsigned NumSizeInBits = VT.getSizeInBits();
   unsigned NumBitsPerElt = VT.getScalarSizeInBits();
-  assert((NumBitsPerElt % 8) == 0 && (NumSizeInBits % 8) == 0 &&
-         "Expected byte aligned value types");
+  if ((NumBitsPerElt % 8) != 0 || (NumSizeInBits % 8) != 0)
+    return false;
 
   unsigned Opcode = N.getOpcode();
   switch (Opcode) {
@@ -6525,9 +6590,7 @@ static bool getFauxShuffleMask(SDValue N, SmallVectorImpl<int> &Mask,
     return true;
   }
   case ISD::INSERT_SUBVECTOR: {
-    // Handle INSERT_SUBVECTOR(SRC0, SHUFFLE(EXTRACT_SUBVECTOR(SRC1)) where
-    // SRC0/SRC1 are both of the same valuetype VT.
-    // TODO - add peekThroughOneUseBitcasts support.
+    // Handle INSERT_SUBVECTOR(SRC0, SHUFFLE(SRC1)).
     SDValue Src = N.getOperand(0);
     SDValue Sub = N.getOperand(1);
     EVT SubVT = Sub.getValueType();
@@ -6537,26 +6600,41 @@ static bool getFauxShuffleMask(SDValue N, SmallVectorImpl<int> &Mask,
       return false;
     SmallVector<int, 64> SubMask;
     SmallVector<SDValue, 2> SubInputs;
-    if (!resolveTargetShuffleInputs(Sub, SubInputs, SubMask, DAG) ||
-        SubMask.size() != NumSubElts)
+    if (!resolveTargetShuffleInputs(peekThroughOneUseBitcasts(Sub), SubInputs,
+                                    SubMask, DAG))
       return false;
+    int InsertIdx = N.getConstantOperandVal(2);
+    if (SubMask.size() != NumSubElts) {
+      assert(((SubMask.size() % NumSubElts) == 0 ||
+              (NumSubElts % SubMask.size()) == 0) && "Illegal submask scale");
+      if ((NumSubElts % SubMask.size()) == 0) {
+        int Scale = NumSubElts / SubMask.size();
+        SmallVector<int,64> ScaledSubMask;
+        scaleShuffleMask<int>(Scale, SubMask, ScaledSubMask);
+        SubMask = ScaledSubMask;
+      } else {
+        int Scale = SubMask.size() / NumSubElts;
+        NumSubElts = SubMask.size();
+        NumElts *= Scale;
+        InsertIdx *= Scale;
+      }
+    }
     Ops.push_back(Src);
     for (SDValue &SubInput : SubInputs) {
-      if (SubInput.getOpcode() != ISD::EXTRACT_SUBVECTOR ||
-          SubInput.getOperand(0).getValueType() != VT ||
-          !isa<ConstantSDNode>(SubInput.getOperand(1)))
-        return false;
-      Ops.push_back(SubInput.getOperand(0));
+      EVT SubSVT = SubInput.getValueType().getScalarType();
+      EVT AltVT = EVT::getVectorVT(*DAG.getContext(), SubSVT,
+                                   NumSizeInBits / SubSVT.getSizeInBits());
+      Ops.push_back(DAG.getNode(ISD::INSERT_SUBVECTOR, SDLoc(N), AltVT,
+                                DAG.getUNDEF(AltVT), SubInput,
+                                DAG.getIntPtrConstant(0, SDLoc(N))));
     }
-    int InsertIdx = N.getConstantOperandVal(2);
     for (int i = 0; i != (int)NumElts; ++i)
       Mask.push_back(i);
     for (int i = 0; i != (int)NumSubElts; ++i) {
       int M = SubMask[i];
       if (0 <= M) {
         int InputIdx = M / NumSubElts;
-        int ExtractIdx = SubInputs[InputIdx].getConstantOperandVal(1);
-        M = (NumElts * (1 + InputIdx)) + ExtractIdx + (M % NumSubElts);
+        M = (NumElts * (1 + InputIdx)) + (M % NumSubElts);
       }
       Mask[i + InsertIdx] = M;
     }
@@ -6695,15 +6773,49 @@ static bool getFauxShuffleMask(SDValue N, SmallVectorImpl<int> &Mask,
     }
     return true;
   }
-  case ISD::ZERO_EXTEND_VECTOR_INREG:
-  case ISD::ZERO_EXTEND: {
-    // TODO - add support for VPMOVZX with smaller input vector types.
+  case X86ISD::VBROADCAST: {
     SDValue Src = N.getOperand(0);
     MVT SrcVT = Src.getSimpleValueType();
-    if (NumSizeInBits != SrcVT.getSizeInBits())
-      break;
-    DecodeZeroExtendMask(SrcVT.getScalarSizeInBits(), NumBitsPerElt, NumElts,
-                         Mask);
+    if (!SrcVT.isVector())
+      return false;
+
+    if (NumSizeInBits != SrcVT.getSizeInBits()) {
+      assert((NumSizeInBits % SrcVT.getSizeInBits()) == 0 &&
+             "Illegal broadcast type");
+      SrcVT = MVT::getVectorVT(SrcVT.getScalarType(),
+                               NumSizeInBits / SrcVT.getScalarSizeInBits());
+      Src = DAG.getNode(ISD::INSERT_SUBVECTOR, SDLoc(N), SrcVT,
+                        DAG.getUNDEF(SrcVT), Src,
+                        DAG.getIntPtrConstant(0, SDLoc(N)));
+    }
+
+    Ops.push_back(Src);
+    Mask.append(NumElts, 0);
+    return true;
+  }
+  case ISD::ZERO_EXTEND:
+  case ISD::ZERO_EXTEND_VECTOR_INREG: {
+    SDValue Src = N.getOperand(0);
+    EVT SrcVT = Src.getValueType();
+
+    // Zero-extended source must be a simple vector.
+    if (!SrcVT.isSimple() || (SrcVT.getSizeInBits() % 128) != 0 ||
+        (SrcVT.getScalarSizeInBits() % 8) != 0)
+      return false;
+
+    unsigned NumSrcBitsPerElt = SrcVT.getScalarSizeInBits();
+    DecodeZeroExtendMask(NumSrcBitsPerElt, NumBitsPerElt, NumElts, Mask);
+
+    if (NumSizeInBits != SrcVT.getSizeInBits()) {
+      assert((NumSizeInBits % SrcVT.getSizeInBits()) == 0 &&
+             "Illegal zero-extension type");
+      SrcVT = MVT::getVectorVT(SrcVT.getSimpleVT().getScalarType(),
+                               NumSizeInBits / NumSrcBitsPerElt);
+      Src = DAG.getNode(ISD::INSERT_SUBVECTOR, SDLoc(N), SrcVT,
+                        DAG.getUNDEF(SrcVT), Src,
+                        DAG.getIntPtrConstant(0, SDLoc(N)));
+    }
+
     Ops.push_back(Src);
     return true;
   }
@@ -6747,7 +6859,7 @@ static void resolveTargetShuffleInputsAndMask(SmallVectorImpl<SDValue> &Inputs,
 static bool resolveTargetShuffleInputs(SDValue Op,
                                        SmallVectorImpl<SDValue> &Inputs,
                                        SmallVectorImpl<int> &Mask,
-                                       const SelectionDAG &DAG) {
+                                       SelectionDAG &DAG) {
   if (!setTargetShuffleZeroElements(Op, Mask, Inputs))
     if (!getFauxShuffleMask(Op, Mask, Inputs, DAG))
       return false;
@@ -6948,6 +7060,26 @@ static SDValue LowerBuildVectorv8i16(SDValue Op, unsigned NonZeros,
 /// Custom lower build_vector of v4i32 or v4f32.
 static SDValue LowerBuildVectorv4x32(SDValue Op, SelectionDAG &DAG,
                                      const X86Subtarget &Subtarget) {
+  // If this is a splat of a pair of elements, use MOVDDUP (unless the target
+  // has XOP; in that case defer lowering to potentially use VPERMIL2PS).
+  // Because we're creating a less complicated build vector here, we may enable
+  // further folding of the MOVDDUP via shuffle transforms.
+  if (Subtarget.hasSSE3() && !Subtarget.hasXOP() &&
+      Op.getOperand(0) == Op.getOperand(2) &&
+      Op.getOperand(1) == Op.getOperand(3) &&
+      Op.getOperand(0) != Op.getOperand(1)) {
+    SDLoc DL(Op);
+    MVT VT = Op.getSimpleValueType();
+    MVT EltVT = VT.getVectorElementType();
+    // Create a new build vector with the first 2 elements followed by undef
+    // padding, bitcast to v2f64, duplicate, and bitcast back.
+    SDValue Ops[4] = { Op.getOperand(0), Op.getOperand(1),
+                       DAG.getUNDEF(EltVT), DAG.getUNDEF(EltVT) };
+    SDValue NewBV = DAG.getBitcast(MVT::v2f64, DAG.getBuildVector(VT, DL, Ops));
+    SDValue Dup = DAG.getNode(X86ISD::MOVDDUP, DL, MVT::v2f64, NewBV);
+    return DAG.getBitcast(VT, Dup);
+  }
+
   // Find all zeroable elements.
   std::bitset<4> Zeroable;
   for (int i=0; i < 4; ++i) {
@@ -7147,9 +7279,9 @@ static SDValue EltsFromConsecutiveLoads(EVT VT, ArrayRef<SDValue> Elts,
   unsigned NumElems = Elts.size();
 
   int LastLoadedElt = -1;
-  SmallBitVector LoadMask(NumElems, false);
-  SmallBitVector ZeroMask(NumElems, false);
-  SmallBitVector UndefMask(NumElems, false);
+  APInt LoadMask = APInt::getNullValue(NumElems);
+  APInt ZeroMask = APInt::getNullValue(NumElems);
+  APInt UndefMask = APInt::getNullValue(NumElems);
 
   // For each element in the initializer, see if we've found a load, zero or an
   // undef.
@@ -7159,11 +7291,11 @@ static SDValue EltsFromConsecutiveLoads(EVT VT, ArrayRef<SDValue> Elts,
       return SDValue();
 
     if (Elt.isUndef())
-      UndefMask[i] = true;
+      UndefMask.setBit(i);
     else if (X86::isZeroNode(Elt) || ISD::isBuildVectorAllZeros(Elt.getNode()))
-      ZeroMask[i] = true;
+      ZeroMask.setBit(i);
     else if (ISD::isNON_EXTLoad(Elt.getNode())) {
-      LoadMask[i] = true;
+      LoadMask.setBit(i);
       LastLoadedElt = i;
       // Each loaded element must be the correct fractional portion of the
       // requested vector load.
@@ -7172,20 +7304,21 @@ static SDValue EltsFromConsecutiveLoads(EVT VT, ArrayRef<SDValue> Elts,
     } else
       return SDValue();
   }
-  assert((ZeroMask | UndefMask | LoadMask).count() == NumElems &&
+  assert((ZeroMask.countPopulation() + UndefMask.countPopulation() +
+          LoadMask.countPopulation()) == NumElems &&
          "Incomplete element masks");
 
   // Handle Special Cases - all undef or undef/zero.
-  if (UndefMask.count() == NumElems)
+  if (UndefMask.countPopulation() == NumElems)
     return DAG.getUNDEF(VT);
 
   // FIXME: Should we return this as a BUILD_VECTOR instead?
-  if ((ZeroMask | UndefMask).count() == NumElems)
+  if ((ZeroMask.countPopulation() + UndefMask.countPopulation()) == NumElems)
     return VT.isInteger() ? DAG.getConstant(0, DL, VT)
                           : DAG.getConstantFP(0.0, DL, VT);
 
   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
-  int FirstLoadedElt = LoadMask.find_first();
+  int FirstLoadedElt = LoadMask.countTrailingZeros();
   SDValue EltBase = peekThroughBitcasts(Elts[FirstLoadedElt]);
   LoadSDNode *LDBase = cast<LoadSDNode>(EltBase);
   EVT LDBaseVT = EltBase.getValueType();
@@ -7806,13 +7939,14 @@ static SDValue LowerBUILD_VECTORvXi1(SDValue Op, SelectionDAG &DAG,
   return DstVec;
 }
 
-/// Return true if \p N implements a horizontal binop and return the
-/// operands for the horizontal binop into V0 and V1.
-///
 /// This is a helper function of LowerToHorizontalOp().
 /// This function checks that the build_vector \p N in input implements a
-/// horizontal operation. Parameter \p Opcode defines the kind of horizontal
-/// operation to match.
+/// 128-bit partial horizontal operation on a 256-bit vector, but that operation
+/// may not match the layout of an x86 256-bit horizontal instruction.
+/// In other words, if this returns true, then some extraction/insertion will
+/// be required to produce a valid horizontal instruction.
+///
+/// Parameter \p Opcode defines the kind of horizontal operation to match.
 /// For example, if \p Opcode is equal to ISD::ADD, then this function
 /// checks if \p N implements a horizontal arithmetic add; if instead \p Opcode
 /// is equal to ISD::SUB, then this function checks if this is a horizontal
@@ -7820,12 +7954,17 @@ static SDValue LowerBUILD_VECTORvXi1(SDValue Op, SelectionDAG &DAG,
 ///
 /// This function only analyzes elements of \p N whose indices are
 /// in range [BaseIdx, LastIdx).
-static bool isHorizontalBinOp(const BuildVectorSDNode *N, unsigned Opcode,
-                              SelectionDAG &DAG,
-                              unsigned BaseIdx, unsigned LastIdx,
-                              SDValue &V0, SDValue &V1) {
+///
+/// TODO: This function was originally used to match both real and fake partial
+/// horizontal operations, but the index-matching logic is incorrect for that.
+/// See the corrected implementation in isHopBuildVector(). Can we reduce this
+/// code because it is only used for partial h-op matching now?
+static bool isHorizontalBinOpPart(const BuildVectorSDNode *N, unsigned Opcode,
+                                  SelectionDAG &DAG,
+                                  unsigned BaseIdx, unsigned LastIdx,
+                                  SDValue &V0, SDValue &V1) {
   EVT VT = N->getValueType(0);
-
+  assert(VT.is256BitVector() && "Only use for matching partial 256-bit h-ops");
   assert(BaseIdx * 2 <= LastIdx && "Invalid Indices in input!");
   assert(VT.isVector() && VT.getVectorNumElements() >= LastIdx &&
          "Invalid Vector in input!");
@@ -8166,17 +8305,158 @@ static SDValue lowerToAddSubOrFMAddSub(const BuildVectorSDNode *BV,
   return DAG.getNode(X86ISD::ADDSUB, DL, VT, Opnd0, Opnd1);
 }
 
+static bool isHopBuildVector(const BuildVectorSDNode *BV, SelectionDAG &DAG,
+                             unsigned &HOpcode, SDValue &V0, SDValue &V1) {
+  // Initialize outputs to known values.
+  MVT VT = BV->getSimpleValueType(0);
+  HOpcode = ISD::DELETED_NODE;
+  V0 = DAG.getUNDEF(VT);
+  V1 = DAG.getUNDEF(VT);
+
+  // x86 256-bit horizontal ops are defined in a non-obvious way. Each 128-bit
+  // half of the result is calculated independently from the 128-bit halves of
+  // the inputs, so that makes the index-checking logic below more complicated.
+  unsigned NumElts = VT.getVectorNumElements();
+  unsigned GenericOpcode = ISD::DELETED_NODE;
+  unsigned Num128BitChunks = VT.is256BitVector() ? 2 : 1;
+  unsigned NumEltsIn128Bits = NumElts / Num128BitChunks;
+  unsigned NumEltsIn64Bits = NumEltsIn128Bits / 2;
+  for (unsigned i = 0; i != Num128BitChunks; ++i) {
+    for (unsigned j = 0; j != NumEltsIn128Bits; ++j) {
+      // Ignore undef elements.
+      SDValue Op = BV->getOperand(i * NumEltsIn128Bits + j);
+      if (Op.isUndef())
+        continue;
+
+      // If there's an opcode mismatch, we're done.
+      if (HOpcode != ISD::DELETED_NODE && Op.getOpcode() != GenericOpcode)
+        return false;
+
+      // Initialize horizontal opcode.
+      if (HOpcode == ISD::DELETED_NODE) {
+        GenericOpcode = Op.getOpcode();
+        switch (GenericOpcode) {
+        case ISD::ADD: HOpcode = X86ISD::HADD; break;
+        case ISD::SUB: HOpcode = X86ISD::HSUB; break;
+        case ISD::FADD: HOpcode = X86ISD::FHADD; break;
+        case ISD::FSUB: HOpcode = X86ISD::FHSUB; break;
+        default: return false;
+        }
+      }
+
+      SDValue Op0 = Op.getOperand(0);
+      SDValue Op1 = Op.getOperand(1);
+      if (Op0.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
+          Op1.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
+          Op0.getOperand(0) != Op1.getOperand(0) ||
+          !isa<ConstantSDNode>(Op0.getOperand(1)) ||
+          !isa<ConstantSDNode>(Op1.getOperand(1)) || !Op.hasOneUse())
+        return false;
+
+      // The source vector is chosen based on which 64-bit half of the
+      // destination vector is being calculated.
+      if (j < NumEltsIn64Bits) {
+        if (V0.isUndef())
+          V0 = Op0.getOperand(0);
+      } else {
+        if (V1.isUndef())
+          V1 = Op0.getOperand(0);
+      }
+
+      SDValue SourceVec = (j < NumEltsIn64Bits) ? V0 : V1;
+      if (SourceVec != Op0.getOperand(0))
+        return false;
+
+      // op (extract_vector_elt A, I), (extract_vector_elt A, I+1)
+      unsigned ExtIndex0 = Op0.getConstantOperandVal(1);
+      unsigned ExtIndex1 = Op1.getConstantOperandVal(1);
+      unsigned ExpectedIndex = i * NumEltsIn128Bits +
+                               (j % NumEltsIn64Bits) * 2;
+      if (ExpectedIndex == ExtIndex0 && ExtIndex1 == ExtIndex0 + 1)
+        continue;
+
+      // If this is not a commutative op, this does not match.
+      if (GenericOpcode != ISD::ADD && GenericOpcode != ISD::FADD)
+        return false;
+
+      // Addition is commutative, so try swapping the extract indexes.
+      // op (extract_vector_elt A, I+1), (extract_vector_elt A, I)
+      if (ExpectedIndex == ExtIndex1 && ExtIndex0 == ExtIndex1 + 1)
+        continue;
+
+      // Extract indexes do not match horizontal requirement.
+      return false;
+    }
+  }
+  // We matched. Opcode and operands are returned by reference as arguments.
+  return true;
+}
+
+static SDValue getHopForBuildVector(const BuildVectorSDNode *BV,
+                                    SelectionDAG &DAG, unsigned HOpcode,
+                                    SDValue V0, SDValue V1) {
+  // If either input vector is not the same size as the build vector,
+  // extract/insert the low bits to the correct size.
+  // This is free (examples: zmm --> xmm, xmm --> ymm).
+  MVT VT = BV->getSimpleValueType(0);
+  unsigned Width = VT.getSizeInBits();
+  if (V0.getValueSizeInBits() > Width)
+    V0 = extractSubVector(V0, 0, DAG, SDLoc(BV), Width);
+  else if (V0.getValueSizeInBits() < Width)
+    V0 = insertSubVector(DAG.getUNDEF(VT), V0, 0, DAG, SDLoc(BV), Width);
+
+  if (V1.getValueSizeInBits() > Width)
+    V1 = extractSubVector(V1, 0, DAG, SDLoc(BV), Width);
+  else if (V1.getValueSizeInBits() < Width)
+    V1 = insertSubVector(DAG.getUNDEF(VT), V1, 0, DAG, SDLoc(BV), Width);
+
+  return DAG.getNode(HOpcode, SDLoc(BV), VT, V0, V1);
+}
+
 /// Lower BUILD_VECTOR to a horizontal add/sub operation if possible.
 static SDValue LowerToHorizontalOp(const BuildVectorSDNode *BV,
                                    const X86Subtarget &Subtarget,
                                    SelectionDAG &DAG) {
+  // We need at least 2 non-undef elements to make this worthwhile by default.
+  unsigned NumNonUndefs = 0;
+  for (const SDValue &V : BV->op_values())
+    if (!V.isUndef())
+      ++NumNonUndefs;
+
+  if (NumNonUndefs < 2)
+    return SDValue();
+
+  // There are 4 sets of horizontal math operations distinguished by type:
+  // int/FP at 128-bit/256-bit. Each type was introduced with a different
+  // subtarget feature. Try to match those "native" patterns first.
   MVT VT = BV->getSimpleValueType(0);
+  unsigned HOpcode;
+  SDValue V0, V1;
+  if ((VT == MVT::v4f32 || VT == MVT::v2f64) && Subtarget.hasSSE3())
+    if (isHopBuildVector(BV, DAG, HOpcode, V0, V1))
+      return getHopForBuildVector(BV, DAG, HOpcode, V0, V1);
+
+  if ((VT == MVT::v8i16 || VT == MVT::v4i32) && Subtarget.hasSSSE3())
+    if (isHopBuildVector(BV, DAG, HOpcode, V0, V1))
+      return getHopForBuildVector(BV, DAG, HOpcode, V0, V1);
+
+  if ((VT == MVT::v8f32 || VT == MVT::v4f64) && Subtarget.hasAVX())
+    if (isHopBuildVector(BV, DAG, HOpcode, V0, V1))
+      return getHopForBuildVector(BV, DAG, HOpcode, V0, V1);
+
+  if ((VT == MVT::v16i16 || VT == MVT::v8i32) && Subtarget.hasAVX2())
+    if (isHopBuildVector(BV, DAG, HOpcode, V0, V1))
+      return getHopForBuildVector(BV, DAG, HOpcode, V0, V1);
+
+  // Try harder to match 256-bit ops by using extract/concat.
+  if (!Subtarget.hasAVX() || !VT.is256BitVector())
+    return SDValue();
+
+  // Count the number of UNDEF operands in the build_vector in input.
   unsigned NumElts = VT.getVectorNumElements();
+  unsigned Half = NumElts / 2;
   unsigned NumUndefsLO = 0;
   unsigned NumUndefsHI = 0;
-  unsigned Half = NumElts/2;
-
-  // Count the number of UNDEF operands in the build_vector in input.
   for (unsigned i = 0, e = Half; i != e; ++i)
     if (BV->getOperand(i)->isUndef())
       NumUndefsLO++;
@@ -8185,96 +8465,61 @@ static SDValue LowerToHorizontalOp(const BuildVectorSDNode *BV,
     if (BV->getOperand(i)->isUndef())
       NumUndefsHI++;
 
-  // Early exit if this is either a build_vector of all UNDEFs or all the
-  // operands but one are UNDEF.
-  if (NumUndefsLO + NumUndefsHI + 1 >= NumElts)
-    return SDValue();
-
   SDLoc DL(BV);
   SDValue InVec0, InVec1;
-  if ((VT == MVT::v4f32 || VT == MVT::v2f64) && Subtarget.hasSSE3()) {
-    // Try to match an SSE3 float HADD/HSUB.
-    if (isHorizontalBinOp(BV, ISD::FADD, DAG, 0, NumElts, InVec0, InVec1))
-      return DAG.getNode(X86ISD::FHADD, DL, VT, InVec0, InVec1);
-
-    if (isHorizontalBinOp(BV, ISD::FSUB, DAG, 0, NumElts, InVec0, InVec1))
-      return DAG.getNode(X86ISD::FHSUB, DL, VT, InVec0, InVec1);
-  } else if ((VT == MVT::v4i32 || VT == MVT::v8i16) && Subtarget.hasSSSE3()) {
-    // Try to match an SSSE3 integer HADD/HSUB.
-    if (isHorizontalBinOp(BV, ISD::ADD, DAG, 0, NumElts, InVec0, InVec1))
-      return DAG.getNode(X86ISD::HADD, DL, VT, InVec0, InVec1);
-
-    if (isHorizontalBinOp(BV, ISD::SUB, DAG, 0, NumElts, InVec0, InVec1))
-      return DAG.getNode(X86ISD::HSUB, DL, VT, InVec0, InVec1);
-  }
-
-  if (!Subtarget.hasAVX())
-    return SDValue();
-
-  if ((VT == MVT::v8f32 || VT == MVT::v4f64)) {
-    // Try to match an AVX horizontal add/sub of packed single/double
-    // precision floating point values from 256-bit vectors.
-    SDValue InVec2, InVec3;
-    if (isHorizontalBinOp(BV, ISD::FADD, DAG, 0, Half, InVec0, InVec1) &&
-        isHorizontalBinOp(BV, ISD::FADD, DAG, Half, NumElts, InVec2, InVec3) &&
-        ((InVec0.isUndef() || InVec2.isUndef()) || InVec0 == InVec2) &&
-        ((InVec1.isUndef() || InVec3.isUndef()) || InVec1 == InVec3))
-      return DAG.getNode(X86ISD::FHADD, DL, VT, InVec0, InVec1);
-
-    if (isHorizontalBinOp(BV, ISD::FSUB, DAG, 0, Half, InVec0, InVec1) &&
-        isHorizontalBinOp(BV, ISD::FSUB, DAG, Half, NumElts, InVec2, InVec3) &&
-        ((InVec0.isUndef() || InVec2.isUndef()) || InVec0 == InVec2) &&
-        ((InVec1.isUndef() || InVec3.isUndef()) || InVec1 == InVec3))
-      return DAG.getNode(X86ISD::FHSUB, DL, VT, InVec0, InVec1);
-  } else if (VT == MVT::v8i32 || VT == MVT::v16i16) {
-    // Try to match an AVX2 horizontal add/sub of signed integers.
+  if (VT == MVT::v8i32 || VT == MVT::v16i16) {
     SDValue InVec2, InVec3;
     unsigned X86Opcode;
     bool CanFold = true;
 
-    if (isHorizontalBinOp(BV, ISD::ADD, DAG, 0, Half, InVec0, InVec1) &&
-        isHorizontalBinOp(BV, ISD::ADD, DAG, Half, NumElts, InVec2, InVec3) &&
+    if (isHorizontalBinOpPart(BV, ISD::ADD, DAG, 0, Half, InVec0, InVec1) &&
+        isHorizontalBinOpPart(BV, ISD::ADD, DAG, Half, NumElts, InVec2,
+                              InVec3) &&
         ((InVec0.isUndef() || InVec2.isUndef()) || InVec0 == InVec2) &&
         ((InVec1.isUndef() || InVec3.isUndef()) || InVec1 == InVec3))
       X86Opcode = X86ISD::HADD;
-    else if (isHorizontalBinOp(BV, ISD::SUB, DAG, 0, Half, InVec0, InVec1) &&
-        isHorizontalBinOp(BV, ISD::SUB, DAG, Half, NumElts, InVec2, InVec3) &&
-        ((InVec0.isUndef() || InVec2.isUndef()) || InVec0 == InVec2) &&
-        ((InVec1.isUndef() || InVec3.isUndef()) || InVec1 == InVec3))
+    else if (isHorizontalBinOpPart(BV, ISD::SUB, DAG, 0, Half, InVec0,
+                                   InVec1) &&
+             isHorizontalBinOpPart(BV, ISD::SUB, DAG, Half, NumElts, InVec2,
+                                   InVec3) &&
+             ((InVec0.isUndef() || InVec2.isUndef()) || InVec0 == InVec2) &&
+             ((InVec1.isUndef() || InVec3.isUndef()) || InVec1 == InVec3))
       X86Opcode = X86ISD::HSUB;
     else
       CanFold = false;
 
     if (CanFold) {
-      // Fold this build_vector into a single horizontal add/sub.
-      // Do this only if the target has AVX2.
-      if (Subtarget.hasAVX2())
-        return DAG.getNode(X86Opcode, DL, VT, InVec0, InVec1);
-
       // Do not try to expand this build_vector into a pair of horizontal
       // add/sub if we can emit a pair of scalar add/sub.
       if (NumUndefsLO + 1 == Half || NumUndefsHI + 1 == Half)
         return SDValue();
 
-      // Convert this build_vector into a pair of horizontal binop followed by
-      // a concat vector.
+      // Convert this build_vector into a pair of horizontal binops followed by
+      // a concat vector. We must adjust the outputs from the partial horizontal
+      // matching calls above to account for undefined vector halves.
+      SDValue V0 = InVec0.isUndef() ? InVec2 : InVec0;
+      SDValue V1 = InVec1.isUndef() ? InVec3 : InVec1;
+      assert((!V0.isUndef() || !V1.isUndef()) && "Horizontal-op of undefs?");
       bool isUndefLO = NumUndefsLO == Half;
       bool isUndefHI = NumUndefsHI == Half;
-      return ExpandHorizontalBinOp(InVec0, InVec1, DL, DAG, X86Opcode, false,
-                                   isUndefLO, isUndefHI);
+      return ExpandHorizontalBinOp(V0, V1, DL, DAG, X86Opcode, false, isUndefLO,
+                                   isUndefHI);
     }
   }
 
-  if ((VT == MVT::v8f32 || VT == MVT::v4f64 || VT == MVT::v8i32 ||
-       VT == MVT::v16i16) && Subtarget.hasAVX()) {
+  if (VT == MVT::v8f32 || VT == MVT::v4f64 || VT == MVT::v8i32 ||
+      VT == MVT::v16i16) {
     unsigned X86Opcode;
-    if (isHorizontalBinOp(BV, ISD::ADD, DAG, 0, NumElts, InVec0, InVec1))
+    if (isHorizontalBinOpPart(BV, ISD::ADD, DAG, 0, NumElts, InVec0, InVec1))
       X86Opcode = X86ISD::HADD;
-    else if (isHorizontalBinOp(BV, ISD::SUB, DAG, 0, NumElts, InVec0, InVec1))
+    else if (isHorizontalBinOpPart(BV, ISD::SUB, DAG, 0, NumElts, InVec0,
+                                   InVec1))
       X86Opcode = X86ISD::HSUB;
-    else if (isHorizontalBinOp(BV, ISD::FADD, DAG, 0, NumElts, InVec0, InVec1))
+    else if (isHorizontalBinOpPart(BV, ISD::FADD, DAG, 0, NumElts, InVec0,
+                                   InVec1))
       X86Opcode = X86ISD::FHADD;
-    else if (isHorizontalBinOp(BV, ISD::FSUB, DAG, 0, NumElts, InVec0, InVec1))
+    else if (isHorizontalBinOpPart(BV, ISD::FSUB, DAG, 0, NumElts, InVec0,
+                                   InVec1))
       X86Opcode = X86ISD::FHSUB;
     else
       return SDValue();
@@ -9522,6 +9767,21 @@ static bool isUnpackWdShuffleMask(ArrayRef<int> Mask, MVT VT) {
   return IsUnpackwdMask;
 }
 
+static bool is128BitUnpackShuffleMask(ArrayRef<int> Mask) {
+  // Create 128-bit vector type based on mask size.
+  MVT EltVT = MVT::getIntegerVT(128 / Mask.size());
+  MVT VT = MVT::getVectorVT(EltVT, Mask.size());
+
+  // Match any of unary/binary or low/high.
+  for (unsigned i = 0; i != 4; ++i) {
+    SmallVector<int, 16> UnpackMask;
+    createUnpackShuffleMask(VT, UnpackMask, (i >> 1) % 2, i % 2);
+    if (isTargetShuffleEquivalent(Mask, UnpackMask))
+      return true;
+  }
+  return false;
+}
+
 /// Get a 4-lane 8-bit shuffle immediate for a mask.
 ///
 /// This helper function produces an 8-bit shuffle immediate corresponding to
@@ -9661,12 +9921,11 @@ static bool isNonZeroElementsInOrder(const APInt &Zeroable,
 }
 
 /// Try to lower a shuffle with a single PSHUFB of V1 or V2.
-static SDValue lowerVectorShuffleWithPSHUFB(const SDLoc &DL, MVT VT,
-                                            ArrayRef<int> Mask, SDValue V1,
-                                            SDValue V2,
-                                            const APInt &Zeroable,
-                                            const X86Subtarget &Subtarget,
-                                            SelectionDAG &DAG) {
+static SDValue lowerShuffleWithPSHUFB(const SDLoc &DL, MVT VT,
+                                      ArrayRef<int> Mask, SDValue V1,
+                                      SDValue V2, const APInt &Zeroable,
+                                      const X86Subtarget &Subtarget,
+                                      SelectionDAG &DAG) {
   int Size = Mask.size();
   int LaneSize = 128 / VT.getScalarSizeInBits();
   const int NumBytes = VT.getSizeInBits() / 8;
@@ -9720,11 +9979,11 @@ static SDValue getMaskNode(SDValue Mask, MVT MaskVT,
                            const SDLoc &dl);
 
 // X86 has dedicated shuffle that can be lowered to VEXPAND
-static SDValue lowerVectorShuffleToEXPAND(const SDLoc &DL, MVT VT,
-                                          const APInt &Zeroable,
-                                          ArrayRef<int> Mask, SDValue &V1,
-                                          SDValue &V2, SelectionDAG &DAG,
-                                          const X86Subtarget &Subtarget) {
+static SDValue lowerShuffleToEXPAND(const SDLoc &DL, MVT VT,
+                                    const APInt &Zeroable,
+                                    ArrayRef<int> Mask, SDValue &V1,
+                                    SDValue &V2, SelectionDAG &DAG,
+                                    const X86Subtarget &Subtarget) {
   bool IsLeftZeroSide = true;
   if (!isNonZeroElementsInOrder(Zeroable, Mask, V1.getValueType(),
                                 IsLeftZeroSide))
@@ -9740,9 +9999,7 @@ static SDValue lowerVectorShuffleToEXPAND(const SDLoc &DL, MVT VT,
                               Subtarget, DAG, DL);
   SDValue ZeroVector = getZeroVector(VT, Subtarget, DAG, DL);
   SDValue ExpandedVector = IsLeftZeroSide ? V2 : V1;
-  return DAG.getSelect(DL, VT, VMask,
-                       DAG.getNode(X86ISD::EXPAND, DL, VT, ExpandedVector),
-                       ZeroVector);
+  return DAG.getNode(X86ISD::EXPAND, DL, VT, ExpandedVector, ZeroVector, VMask);
 }
 
 static bool matchVectorShuffleWithUNPCK(MVT VT, SDValue &V1, SDValue &V2,
@@ -9832,9 +10089,9 @@ static bool matchVectorShuffleWithUNPCK(MVT VT, SDValue &V1, SDValue &V2,
 
 // X86 has dedicated unpack instructions that can handle specific blend
 // operations: UNPCKH and UNPCKL.
-static SDValue lowerVectorShuffleWithUNPCK(const SDLoc &DL, MVT VT,
-                                           ArrayRef<int> Mask, SDValue V1,
-                                           SDValue V2, SelectionDAG &DAG) {
+static SDValue lowerShuffleWithUNPCK(const SDLoc &DL, MVT VT,
+                                     ArrayRef<int> Mask, SDValue V1, SDValue V2,
+                                     SelectionDAG &DAG) {
   SmallVector<int, 8> Unpckl;
   createUnpackShuffleMask(VT, Unpckl, /* Lo = */ true, /* Unary = */ false);
   if (isShuffleEquivalent(V1, V2, Mask, Unpckl))
@@ -9896,10 +10153,10 @@ static bool matchVectorShuffleAsVPMOV(ArrayRef<int> Mask, bool SwappedOps,
 //
 // But when avx512vl is available, one can just use a single vpmovdw
 // instruction.
-static SDValue lowerVectorShuffleWithVPMOV(const SDLoc &DL, ArrayRef<int> Mask,
-                                           MVT VT, SDValue V1, SDValue V2,
-                                           SelectionDAG &DAG,
-                                           const X86Subtarget &Subtarget) {
+static SDValue lowerShuffleWithVPMOV(const SDLoc &DL, ArrayRef<int> Mask,
+                                     MVT VT, SDValue V1, SDValue V2,
+                                     SelectionDAG &DAG,
+                                     const X86Subtarget &Subtarget) {
   if (VT != MVT::v16i8 && VT != MVT::v8i16)
     return SDValue();
 
@@ -10004,10 +10261,9 @@ static bool matchVectorShuffleWithPACK(MVT VT, MVT &SrcVT, SDValue &V1,
   return false;
 }
 
-static SDValue lowerVectorShuffleWithPACK(const SDLoc &DL, MVT VT,
-                                          ArrayRef<int> Mask, SDValue V1,
-                                          SDValue V2, SelectionDAG &DAG,
-                                          const X86Subtarget &Subtarget) {
+static SDValue lowerShuffleWithPACK(const SDLoc &DL, MVT VT, ArrayRef<int> Mask,
+                                    SDValue V1, SDValue V2, SelectionDAG &DAG,
+                                    const X86Subtarget &Subtarget) {
   MVT PackVT;
   unsigned PackOpcode;
   if (matchVectorShuffleWithPACK(VT, PackVT, V1, V2, PackOpcode, Mask, DAG,
@@ -10022,10 +10278,9 @@ static SDValue lowerVectorShuffleWithPACK(const SDLoc &DL, MVT VT,
 ///
 /// This handles cases where we can model a blend exactly as a bitmask due to
 /// one of the inputs being zeroable.
-static SDValue lowerVectorShuffleAsBitMask(const SDLoc &DL, MVT VT, SDValue V1,
-                                           SDValue V2, ArrayRef<int> Mask,
-                                           const APInt &Zeroable,
-                                           SelectionDAG &DAG) {
+static SDValue lowerShuffleAsBitMask(const SDLoc &DL, MVT VT, SDValue V1,
+                                     SDValue V2, ArrayRef<int> Mask,
+                                     const APInt &Zeroable, SelectionDAG &DAG) {
   assert(!VT.isFloatingPoint() && "Floating point types are not supported");
   MVT EltVT = VT.getVectorElementType();
   SDValue Zero = DAG.getConstant(0, DL, EltVT);
@@ -10056,9 +10311,9 @@ static SDValue lowerVectorShuffleAsBitMask(const SDLoc &DL, MVT VT, SDValue V1,
 /// This is used as a fallback approach when first class blend instructions are
 /// unavailable. Currently it is only suitable for integer vectors, but could
 /// be generalized for floating point vectors if desirable.
-static SDValue lowerVectorShuffleAsBitBlend(const SDLoc &DL, MVT VT, SDValue V1,
-                                            SDValue V2, ArrayRef<int> Mask,
-                                            SelectionDAG &DAG) {
+static SDValue lowerShuffleAsBitBlend(const SDLoc &DL, MVT VT, SDValue V1,
+                                      SDValue V2, ArrayRef<int> Mask,
+                                      SelectionDAG &DAG) {
   assert(VT.isInteger() && "Only supports integer vector types!");
   MVT EltVT = VT.getVectorElementType();
   SDValue Zero = DAG.getConstant(0, DL, EltVT);
@@ -10140,11 +10395,11 @@ static uint64_t scaleVectorShuffleBlendMask(uint64_t BlendMask, int Size,
 /// these values. It relies on the availability of the X86ISD::BLENDI pattern to
 /// be matched in the backend with the type given. What it does check for is
 /// that the shuffle mask is a blend, or convertible into a blend with zero.
-static SDValue lowerVectorShuffleAsBlend(const SDLoc &DL, MVT VT, SDValue V1,
-                                         SDValue V2, ArrayRef<int> Original,
-                                         const APInt &Zeroable,
-                                         const X86Subtarget &Subtarget,
-                                         SelectionDAG &DAG) {
+static SDValue lowerShuffleAsBlend(const SDLoc &DL, MVT VT, SDValue V1,
+                                   SDValue V2, ArrayRef<int> Original,
+                                   const APInt &Zeroable,
+                                   const X86Subtarget &Subtarget,
+                                   SelectionDAG &DAG) {
   SmallVector<int, 64> Mask = createTargetShuffleMask(Original, Zeroable);
 
   uint64_t BlendMask = 0;
@@ -10232,8 +10487,8 @@ static SDValue lowerVectorShuffleAsBlend(const SDLoc &DL, MVT VT, SDValue V1,
            "256-bit byte-blends require AVX2 support!");
 
     // Attempt to lower to a bitmask if we can. VPAND is faster than VPBLENDVB.
-    if (SDValue Masked =
-            lowerVectorShuffleAsBitMask(DL, VT, V1, V2, Mask, Zeroable, DAG))
+    if (SDValue Masked = lowerShuffleAsBitMask(DL, VT, V1, V2, Mask, Zeroable,
+                                               DAG))
       return Masked;
 
     if (Subtarget.hasBWI() && Subtarget.hasVLX()) {
@@ -10306,11 +10561,11 @@ static SDValue lowerVectorShuffleAsBlend(const SDLoc &DL, MVT VT, SDValue V1,
 ///
 /// This matches the pattern where we can blend elements from two inputs and
 /// then reduce the shuffle to a single-input permutation.
-static SDValue lowerVectorShuffleAsBlendAndPermute(const SDLoc &DL, MVT VT,
-                                                   SDValue V1, SDValue V2,
-                                                   ArrayRef<int> Mask,
-                                                   SelectionDAG &DAG,
-                                                   bool ImmBlends = false) {
+static SDValue lowerShuffleAsBlendAndPermute(const SDLoc &DL, MVT VT,
+                                             SDValue V1, SDValue V2,
+                                             ArrayRef<int> Mask,
+                                             SelectionDAG &DAG,
+                                             bool ImmBlends = false) {
   // We build up the blend mask while checking whether a blend is a viable way
   // to reduce the shuffle.
   SmallVector<int, 32> BlendMask(Mask.size(), -1);
@@ -10345,10 +10600,10 @@ static SDValue lowerVectorShuffleAsBlendAndPermute(const SDLoc &DL, MVT VT,
 ///
 /// This matches the pattern where we can unpack elements from two inputs and
 /// then reduce the shuffle to a single-input (wider) permutation.
-static SDValue lowerVectorShuffleAsUNPCKAndPermute(const SDLoc &DL, MVT VT,
-                                                   SDValue V1, SDValue V2,
-                                                   ArrayRef<int> Mask,
-                                                   SelectionDAG &DAG) {
+static SDValue lowerShuffleAsUNPCKAndPermute(const SDLoc &DL, MVT VT,
+                                             SDValue V1, SDValue V2,
+                                             ArrayRef<int> Mask,
+                                             SelectionDAG &DAG) {
   int NumElts = Mask.size();
   int NumLanes = VT.getSizeInBits() / 128;
   int NumLaneElts = NumElts / NumLanes;
@@ -10408,7 +10663,7 @@ static SDValue lowerVectorShuffleAsUNPCKAndPermute(const SDLoc &DL, MVT VT,
 
 /// Helper to form a PALIGNR-based rotate+permute, merging 2 inputs and then
 /// permuting the elements of the result in place.
-static SDValue lowerVectorShuffleAsByteRotateAndPermute(
+static SDValue lowerShuffleAsByteRotateAndPermute(
     const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,
     const X86Subtarget &Subtarget, SelectionDAG &DAG) {
   if ((VT.is128BitVector() && !Subtarget.hasSSSE3()) ||
@@ -10499,7 +10754,7 @@ static SDValue lowerVectorShuffleAsByteRotateAndPermute(
 /// shuffle+blend operations on newer X86 ISAs where we have very fast blend
 /// operations. It will try to pick the best arrangement of shuffles and
 /// blends.
-static SDValue lowerVectorShuffleAsDecomposedShuffleBlend(
+static SDValue lowerShuffleAsDecomposedShuffleBlend(
     const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,
     const X86Subtarget &Subtarget, SelectionDAG &DAG) {
   // Shuffle the input elements into the desired positions in V1 and V2 and
@@ -10523,18 +10778,18 @@ static SDValue lowerVectorShuffleAsDecomposedShuffleBlend(
   // pre-shuffle first is a better strategy.
   if (!isNoopShuffleMask(V1Mask) && !isNoopShuffleMask(V2Mask)) {
     // Only prefer immediate blends to unpack/rotate.
-    if (SDValue BlendPerm = lowerVectorShuffleAsBlendAndPermute(
-            DL, VT, V1, V2, Mask, DAG, true))
+    if (SDValue BlendPerm = lowerShuffleAsBlendAndPermute(DL, VT, V1, V2, Mask,
+                                                          DAG, true))
       return BlendPerm;
-    if (SDValue UnpackPerm =
-            lowerVectorShuffleAsUNPCKAndPermute(DL, VT, V1, V2, Mask, DAG))
+    if (SDValue UnpackPerm = lowerShuffleAsUNPCKAndPermute(DL, VT, V1, V2, Mask,
+                                                           DAG))
       return UnpackPerm;
-    if (SDValue RotatePerm = lowerVectorShuffleAsByteRotateAndPermute(
+    if (SDValue RotatePerm = lowerShuffleAsByteRotateAndPermute(
             DL, VT, V1, V2, Mask, Subtarget, DAG))
       return RotatePerm;
     // Unpack/rotate failed - try again with variable blends.
-    if (SDValue BlendPerm =
-            lowerVectorShuffleAsBlendAndPermute(DL, VT, V1, V2, Mask, DAG))
+    if (SDValue BlendPerm = lowerShuffleAsBlendAndPermute(DL, VT, V1, V2, Mask,
+                                                          DAG))
       return BlendPerm;
   }
 
@@ -10546,8 +10801,7 @@ static SDValue lowerVectorShuffleAsDecomposedShuffleBlend(
 /// Try to lower a vector shuffle as a rotation.
 ///
 /// This is used for support PALIGNR for SSSE3 or VALIGND/Q for AVX512.
-static int matchVectorShuffleAsRotate(SDValue &V1, SDValue &V2,
-                                      ArrayRef<int> Mask) {
+static int matchShuffleAsRotate(SDValue &V1, SDValue &V2, ArrayRef<int> Mask) {
   int NumElts = Mask.size();
 
   // We need to detect various ways of spelling a rotation:
@@ -10631,8 +10885,8 @@ static int matchVectorShuffleAsRotate(SDValue &V1, SDValue &V2,
 /// elements, and takes the low elements as the result. Note that while this is
 /// specified as a *right shift* because x86 is little-endian, it is a *left
 /// rotate* of the vector lanes.
-static int matchVectorShuffleAsByteRotate(MVT VT, SDValue &V1, SDValue &V2,
-                                          ArrayRef<int> Mask) {
+static int matchShuffleAsByteRotate(MVT VT, SDValue &V1, SDValue &V2,
+                                    ArrayRef<int> Mask) {
   // Don't accept any shuffles with zero elements.
   if (any_of(Mask, [](int M) { return M == SM_SentinelZero; }))
     return -1;
@@ -10642,7 +10896,7 @@ static int matchVectorShuffleAsByteRotate(MVT VT, SDValue &V1, SDValue &V2,
   if (!is128BitLaneRepeatedShuffleMask(VT, Mask, RepeatedMask))
     return -1;
 
-  int Rotation = matchVectorShuffleAsRotate(V1, V2, RepeatedMask);
+  int Rotation = matchShuffleAsRotate(V1, V2, RepeatedMask);
   if (Rotation <= 0)
     return -1;
 
@@ -10653,15 +10907,14 @@ static int matchVectorShuffleAsByteRotate(MVT VT, SDValue &V1, SDValue &V2,
   return Rotation * Scale;
 }
 
-static SDValue lowerVectorShuffleAsByteRotate(const SDLoc &DL, MVT VT,
-                                              SDValue V1, SDValue V2,
-                                              ArrayRef<int> Mask,
-                                              const X86Subtarget &Subtarget,
-                                              SelectionDAG &DAG) {
+static SDValue lowerShuffleAsByteRotate(const SDLoc &DL, MVT VT, SDValue V1,
+                                        SDValue V2, ArrayRef<int> Mask,
+                                        const X86Subtarget &Subtarget,
+                                        SelectionDAG &DAG) {
   assert(!isNoopShuffleMask(Mask) && "We shouldn't lower no-op shuffles!");
 
   SDValue Lo = V1, Hi = V2;
-  int ByteRotation = matchVectorShuffleAsByteRotate(VT, Lo, Hi, Mask);
+  int ByteRotation = matchShuffleAsByteRotate(VT, Lo, Hi, Mask);
   if (ByteRotation <= 0)
     return SDValue();
 
@@ -10709,11 +10962,10 @@ static SDValue lowerVectorShuffleAsByteRotate(const SDLoc &DL, MVT VT,
 /// elements, and takes the low elements as the result. Note that while this is
 /// specified as a *right shift* because x86 is little-endian, it is a *left
 /// rotate* of the vector lanes.
-static SDValue lowerVectorShuffleAsRotate(const SDLoc &DL, MVT VT,
-                                          SDValue V1, SDValue V2,
-                                          ArrayRef<int> Mask,
-                                          const X86Subtarget &Subtarget,
-                                          SelectionDAG &DAG) {
+static SDValue lowerShuffleAsRotate(const SDLoc &DL, MVT VT, SDValue V1,
+                                    SDValue V2, ArrayRef<int> Mask,
+                                    const X86Subtarget &Subtarget,
+                                    SelectionDAG &DAG) {
   assert((VT.getScalarType() == MVT::i32 || VT.getScalarType() == MVT::i64) &&
          "Only 32-bit and 64-bit elements are supported!");
 
@@ -10722,7 +10974,7 @@ static SDValue lowerVectorShuffleAsRotate(const SDLoc &DL, MVT VT,
          && "VLX required for 128/256-bit vectors");
 
   SDValue Lo = V1, Hi = V2;
-  int Rotation = matchVectorShuffleAsRotate(Lo, Hi, Mask);
+  int Rotation = matchShuffleAsRotate(Lo, Hi, Mask);
   if (Rotation <= 0)
     return SDValue();
 
@@ -10730,6 +10982,69 @@ static SDValue lowerVectorShuffleAsRotate(const SDLoc &DL, MVT VT,
                      DAG.getConstant(Rotation, DL, MVT::i8));
 }
 
+/// Try to lower a vector shuffle as a byte shift sequence.
+static SDValue lowerVectorShuffleAsByteShiftMask(
+    const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,
+    const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG) {
+  assert(!isNoopShuffleMask(Mask) && "We shouldn't lower no-op shuffles!");
+  assert(VT.is128BitVector() && "Only 128-bit vectors supported");
+
+  // We need a shuffle that has zeros at one/both ends and a sequential
+  // shuffle from one source within.
+  unsigned ZeroLo = Zeroable.countTrailingOnes();
+  unsigned ZeroHi = Zeroable.countLeadingOnes();
+  if (!ZeroLo && !ZeroHi)
+    return SDValue();
+
+  unsigned NumElts = Mask.size();
+  unsigned Len = NumElts - (ZeroLo + ZeroHi);
+  if (!isSequentialOrUndefInRange(Mask, ZeroLo, Len, Mask[ZeroLo]))
+    return SDValue();
+
+  unsigned Scale = VT.getScalarSizeInBits() / 8;
+  ArrayRef<int> StubMask = Mask.slice(ZeroLo, Len);
+  if (!isUndefOrInRange(StubMask, 0, NumElts) &&
+      !isUndefOrInRange(StubMask, NumElts, 2 * NumElts))
+    return SDValue();
+
+  SDValue Res = Mask[ZeroLo] < (int)NumElts ? V1 : V2;
+  Res = DAG.getBitcast(MVT::v16i8, Res);
+
+  // Use VSHLDQ/VSRLDQ ops to zero the ends of a vector and leave an
+  // inner sequential set of elements, possibly offset:
+  // 01234567 --> zzzzzz01 --> 1zzzzzzz
+  // 01234567 --> 4567zzzz --> zzzzz456
+  // 01234567 --> z0123456 --> 3456zzzz --> zz3456zz
+  if (ZeroLo == 0) {
+    unsigned Shift = (NumElts - 1) - (Mask[ZeroLo + Len - 1] % NumElts);
+    Res = DAG.getNode(X86ISD::VSHLDQ, DL, MVT::v16i8, Res,
+                      DAG.getConstant(Scale * Shift, DL, MVT::i8));
+    Res = DAG.getNode(X86ISD::VSRLDQ, DL, MVT::v16i8, Res,
+                      DAG.getConstant(Scale * ZeroHi, DL, MVT::i8));
+  } else if (ZeroHi == 0) {
+    unsigned Shift = Mask[ZeroLo] % NumElts;
+    Res = DAG.getNode(X86ISD::VSRLDQ, DL, MVT::v16i8, Res,
+                      DAG.getConstant(Scale * Shift, DL, MVT::i8));
+    Res = DAG.getNode(X86ISD::VSHLDQ, DL, MVT::v16i8, Res,
+                      DAG.getConstant(Scale * ZeroLo, DL, MVT::i8));
+  } else if (!Subtarget.hasSSSE3()) {
+    // If we don't have PSHUFB then its worth avoiding an AND constant mask
+    // by performing 3 byte shifts. Shuffle combining can kick in above that.
+    // TODO: There may be some cases where VSH{LR}DQ+PAND is still better.
+    unsigned Shift = (NumElts - 1) - (Mask[ZeroLo + Len - 1] % NumElts);
+    Res = DAG.getNode(X86ISD::VSHLDQ, DL, MVT::v16i8, Res,
+                      DAG.getConstant(Scale * Shift, DL, MVT::i8));
+    Shift += Mask[ZeroLo] % NumElts;
+    Res = DAG.getNode(X86ISD::VSRLDQ, DL, MVT::v16i8, Res,
+                      DAG.getConstant(Scale * Shift, DL, MVT::i8));
+    Res = DAG.getNode(X86ISD::VSHLDQ, DL, MVT::v16i8, Res,
+                      DAG.getConstant(Scale * ZeroLo, DL, MVT::i8));
+  } else
+    return SDValue();
+
+  return DAG.getBitcast(VT, Res);
+}
+
 /// Try to lower a vector shuffle as a bit shift (shifts in zeros).
 ///
 /// Attempts to match a shuffle mask against the PSLL(W/D/Q/DQ) and
@@ -10753,11 +11068,10 @@ static SDValue lowerVectorShuffleAsRotate(const SDLoc &DL, MVT VT,
 /// [  5, 6,  7, zz, zz, zz, zz, zz]
 /// [ -1, 5,  6,  7, zz, zz, zz, zz]
 /// [  1, 2, -1, -1, -1, -1, zz, zz]
-static int matchVectorShuffleAsShift(MVT &ShiftVT, unsigned &Opcode,
-                                     unsigned ScalarSizeInBits,
-                                     ArrayRef<int> Mask, int MaskOffset,
-                                     const APInt &Zeroable,
-                                     const X86Subtarget &Subtarget) {
+static int matchShuffleAsShift(MVT &ShiftVT, unsigned &Opcode,
+                               unsigned ScalarSizeInBits, ArrayRef<int> Mask,
+                               int MaskOffset, const APInt &Zeroable,
+                               const X86Subtarget &Subtarget) {
   int Size = Mask.size();
   unsigned SizeInBits = Size * ScalarSizeInBits;
 
@@ -10816,11 +11130,11 @@ static int matchVectorShuffleAsShift(MVT &ShiftVT, unsigned &Opcode,
   return -1;
 }
 
-static SDValue lowerVectorShuffleAsShift(const SDLoc &DL, MVT VT, SDValue V1,
-                                         SDValue V2, ArrayRef<int> Mask,
-                                         const APInt &Zeroable,
-                                         const X86Subtarget &Subtarget,
-                                         SelectionDAG &DAG) {
+static SDValue lowerShuffleAsShift(const SDLoc &DL, MVT VT, SDValue V1,
+                                   SDValue V2, ArrayRef<int> Mask,
+                                   const APInt &Zeroable,
+                                   const X86Subtarget &Subtarget,
+                                   SelectionDAG &DAG) {
   int Size = Mask.size();
   assert(Size == (int)VT.getVectorNumElements() && "Unexpected mask size");
 
@@ -10829,14 +11143,13 @@ static SDValue lowerVectorShuffleAsShift(const SDLoc &DL, MVT VT, SDValue V1,
   unsigned Opcode;
 
   // Try to match shuffle against V1 shift.
-  int ShiftAmt = matchVectorShuffleAsShift(
-      ShiftVT, Opcode, VT.getScalarSizeInBits(), Mask, 0, Zeroable, Subtarget);
+  int ShiftAmt = matchShuffleAsShift(ShiftVT, Opcode, VT.getScalarSizeInBits(),
+                                     Mask, 0, Zeroable, Subtarget);
 
   // If V1 failed, try to match shuffle against V2 shift.
   if (ShiftAmt < 0) {
-    ShiftAmt =
-        matchVectorShuffleAsShift(ShiftVT, Opcode, VT.getScalarSizeInBits(),
-                                  Mask, Size, Zeroable, Subtarget);
+    ShiftAmt = matchShuffleAsShift(ShiftVT, Opcode, VT.getScalarSizeInBits(),
+                                   Mask, Size, Zeroable, Subtarget);
     V = V2;
   }
 
@@ -10853,16 +11166,16 @@ static SDValue lowerVectorShuffleAsShift(const SDLoc &DL, MVT VT, SDValue V1,
 
 // EXTRQ: Extract Len elements from lower half of source, starting at Idx.
 // Remainder of lower half result is zero and upper half is all undef.
-static bool matchVectorShuffleAsEXTRQ(MVT VT, SDValue &V1, SDValue &V2,
-                                      ArrayRef<int> Mask, uint64_t &BitLen,
-                                      uint64_t &BitIdx, const APInt &Zeroable) {
+static bool matchShuffleAsEXTRQ(MVT VT, SDValue &V1, SDValue &V2,
+                                ArrayRef<int> Mask, uint64_t &BitLen,
+                                uint64_t &BitIdx, const APInt &Zeroable) {
   int Size = Mask.size();
   int HalfSize = Size / 2;
   assert(Size == (int)VT.getVectorNumElements() && "Unexpected mask size");
   assert(!Zeroable.isAllOnesValue() && "Fully zeroable shuffle mask");
 
   // Upper half must be undefined.
-  if (!isUndefInRange(Mask, HalfSize, HalfSize))
+  if (!isUndefUpperHalf(Mask))
     return false;
 
   // Determine the extraction length from the part of the
@@ -10909,15 +11222,15 @@ static bool matchVectorShuffleAsEXTRQ(MVT VT, SDValue &V1, SDValue &V2,
 // INSERTQ: Extract lowest Len elements from lower half of second source and
 // insert over first source, starting at Idx.
 // { A[0], .., A[Idx-1], B[0], .., B[Len-1], A[Idx+Len], .., UNDEF, ... }
-static bool matchVectorShuffleAsINSERTQ(MVT VT, SDValue &V1, SDValue &V2,
-                                        ArrayRef<int> Mask, uint64_t &BitLen,
-                                        uint64_t &BitIdx) {
+static bool matchShuffleAsINSERTQ(MVT VT, SDValue &V1, SDValue &V2,
+                                  ArrayRef<int> Mask, uint64_t &BitLen,
+                                  uint64_t &BitIdx) {
   int Size = Mask.size();
   int HalfSize = Size / 2;
   assert(Size == (int)VT.getVectorNumElements() && "Unexpected mask size");
 
   // Upper half must be undefined.
-  if (!isUndefInRange(Mask, HalfSize, HalfSize))
+  if (!isUndefUpperHalf(Mask))
     return false;
 
   for (int Idx = 0; Idx != HalfSize; ++Idx) {
@@ -10975,17 +11288,16 @@ static bool matchVectorShuffleAsINSERTQ(MVT VT, SDValue &V1, SDValue &V2,
 }
 
 /// Try to lower a vector shuffle using SSE4a EXTRQ/INSERTQ.
-static SDValue lowerVectorShuffleWithSSE4A(const SDLoc &DL, MVT VT, SDValue V1,
-                                           SDValue V2, ArrayRef<int> Mask,
-                                           const APInt &Zeroable,
-                                           SelectionDAG &DAG) {
+static SDValue lowerShuffleWithSSE4A(const SDLoc &DL, MVT VT, SDValue V1,
+                                     SDValue V2, ArrayRef<int> Mask,
+                                     const APInt &Zeroable, SelectionDAG &DAG) {
   uint64_t BitLen, BitIdx;
-  if (matchVectorShuffleAsEXTRQ(VT, V1, V2, Mask, BitLen, BitIdx, Zeroable))
+  if (matchShuffleAsEXTRQ(VT, V1, V2, Mask, BitLen, BitIdx, Zeroable))
     return DAG.getNode(X86ISD::EXTRQI, DL, VT, V1,
                        DAG.getConstant(BitLen, DL, MVT::i8),
                        DAG.getConstant(BitIdx, DL, MVT::i8));
 
-  if (matchVectorShuffleAsINSERTQ(VT, V1, V2, Mask, BitLen, BitIdx))
+  if (matchShuffleAsINSERTQ(VT, V1, V2, Mask, BitLen, BitIdx))
     return DAG.getNode(X86ISD::INSERTQI, DL, VT, V1 ? V1 : DAG.getUNDEF(VT),
                        V2 ? V2 : DAG.getUNDEF(VT),
                        DAG.getConstant(BitLen, DL, MVT::i8),
@@ -11003,7 +11315,7 @@ static SDValue lowerVectorShuffleWithSSE4A(const SDLoc &DL, MVT VT, SDValue V1,
 /// avoid excess shuffling the offset must either being in the bottom lane
 /// or at the start of a higher lane. All extended elements must be from
 /// the same lane.
-static SDValue lowerVectorShuffleAsSpecificZeroOrAnyExtend(
+static SDValue lowerShuffleAsSpecificZeroOrAnyExtend(
     const SDLoc &DL, MVT VT, int Scale, int Offset, bool AnyExt, SDValue InputV,
     ArrayRef<int> Mask, const X86Subtarget &Subtarget, SelectionDAG &DAG) {
   assert(Scale > 1 && "Need a scale to extend.");
@@ -11088,8 +11400,7 @@ static SDValue lowerVectorShuffleAsSpecificZeroOrAnyExtend(
                                 DAG.getConstant(EltBits, DL, MVT::i8),
                                 DAG.getConstant(LoIdx, DL, MVT::i8)));
 
-    if (isUndefInRange(Mask, NumElements / 2, NumElements / 2) ||
-        !SafeOffset(Offset + 1))
+    if (isUndefUpperHalf(Mask) || !SafeOffset(Offset + 1))
       return DAG.getBitcast(VT, Lo);
 
     int HiIdx = (Offset + 1) * EltBits;
@@ -11161,7 +11472,7 @@ static SDValue lowerVectorShuffleAsSpecificZeroOrAnyExtend(
 ///
 /// The reason we have dedicated lowering for zext-style shuffles is that they
 /// are both incredibly common and often quite performance sensitive.
-static SDValue lowerVectorShuffleAsZeroOrAnyExtend(
+static SDValue lowerShuffleAsZeroOrAnyExtend(
     const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,
     const APInt &Zeroable, const X86Subtarget &Subtarget,
     SelectionDAG &DAG) {
@@ -11232,8 +11543,8 @@ static SDValue lowerVectorShuffleAsZeroOrAnyExtend(
     if (Offset != 0 && Matches < 2)
       return SDValue();
 
-    return lowerVectorShuffleAsSpecificZeroOrAnyExtend(
-        DL, VT, Scale, Offset, AnyExt, InputV, Mask, Subtarget, DAG);
+    return lowerShuffleAsSpecificZeroOrAnyExtend(DL, VT, Scale, Offset, AnyExt,
+                                                 InputV, Mask, Subtarget, DAG);
   };
 
   // The widest scale possible for extending is to a 64-bit integer.
@@ -11317,7 +11628,7 @@ static bool isShuffleFoldableLoad(SDValue V) {
 ///
 /// This is a common pattern that we have especially efficient patterns to lower
 /// across all subtarget feature sets.
-static SDValue lowerVectorShuffleAsElementInsertion(
+static SDValue lowerShuffleAsElementInsertion(
     const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,
     const APInt &Zeroable, const X86Subtarget &Subtarget,
     SelectionDAG &DAG) {
@@ -11415,10 +11726,10 @@ static SDValue lowerVectorShuffleAsElementInsertion(
 /// coming from a scalar_to_vector/build_vector node \p V0 with larger elements.
 ///
 /// This assumes we have AVX2.
-static SDValue lowerVectorShuffleAsTruncBroadcast(const SDLoc &DL, MVT VT,
-                                                  SDValue V0, int BroadcastIdx,
-                                                  const X86Subtarget &Subtarget,
-                                                  SelectionDAG &DAG) {
+static SDValue lowerShuffleAsTruncBroadcast(const SDLoc &DL, MVT VT, SDValue V0,
+                                            int BroadcastIdx,
+                                            const X86Subtarget &Subtarget,
+                                            SelectionDAG &DAG) {
   assert(Subtarget.hasAVX2() &&
          "We can only lower integer broadcasts with AVX2!");
 
@@ -11464,16 +11775,90 @@ static SDValue lowerVectorShuffleAsTruncBroadcast(const SDLoc &DL, MVT VT,
                      DAG.getNode(ISD::TRUNCATE, DL, EltVT, Scalar));
 }
 
+/// Test whether this can be lowered with a single SHUFPS instruction.
+///
+/// This is used to disable more specialized lowerings when the shufps lowering
+/// will happen to be efficient.
+static bool isSingleSHUFPSMask(ArrayRef<int> Mask) {
+  // This routine only handles 128-bit shufps.
+  assert(Mask.size() == 4 && "Unsupported mask size!");
+  assert(Mask[0] >= -1 && Mask[0] < 8 && "Out of bound mask element!");
+  assert(Mask[1] >= -1 && Mask[1] < 8 && "Out of bound mask element!");
+  assert(Mask[2] >= -1 && Mask[2] < 8 && "Out of bound mask element!");
+  assert(Mask[3] >= -1 && Mask[3] < 8 && "Out of bound mask element!");
+
+  // To lower with a single SHUFPS we need to have the low half and high half
+  // each requiring a single input.
+  if (Mask[0] >= 0 && Mask[1] >= 0 && (Mask[0] < 4) != (Mask[1] < 4))
+    return false;
+  if (Mask[2] >= 0 && Mask[3] >= 0 && (Mask[2] < 4) != (Mask[3] < 4))
+    return false;
+
+  return true;
+}
+
+/// If we are extracting two 128-bit halves of a vector and shuffling the
+/// result, match that to a 256-bit AVX2 vperm* instruction to avoid a
+/// multi-shuffle lowering.
+static SDValue lowerShuffleOfExtractsAsVperm(const SDLoc &DL, SDValue N0,
+                                             SDValue N1, ArrayRef<int> Mask,
+                                             SelectionDAG &DAG) {
+  EVT VT = N0.getValueType();
+  assert((VT.is128BitVector() &&
+          (VT.getScalarSizeInBits() == 32 || VT.getScalarSizeInBits() == 64)) &&
+         "VPERM* family of shuffles requires 32-bit or 64-bit elements");
+
+  // Check that both sources are extracts of the same source vector.
+  if (!N0.hasOneUse() || !N1.hasOneUse() ||
+      N0.getOpcode() != ISD::EXTRACT_SUBVECTOR ||
+      N1.getOpcode() != ISD::EXTRACT_SUBVECTOR ||
+      N0.getOperand(0) != N1.getOperand(0))
+    return SDValue();
+
+  SDValue WideVec = N0.getOperand(0);
+  EVT WideVT = WideVec.getValueType();
+  if (!WideVT.is256BitVector() || !isa<ConstantSDNode>(N0.getOperand(1)) ||
+      !isa<ConstantSDNode>(N1.getOperand(1)))
+    return SDValue();
+
+  // Match extracts of each half of the wide source vector. Commute the shuffle
+  // if the extract of the low half is N1.
+  unsigned NumElts = VT.getVectorNumElements();
+  SmallVector<int, 4> NewMask(Mask.begin(), Mask.end());
+  const APInt &ExtIndex0 = N0.getConstantOperandAPInt(1);
+  const APInt &ExtIndex1 = N1.getConstantOperandAPInt(1);
+  if (ExtIndex1 == 0 && ExtIndex0 == NumElts)
+    ShuffleVectorSDNode::commuteMask(NewMask);
+  else if (ExtIndex0 != 0 || ExtIndex1 != NumElts)
+    return SDValue();
+
+  // Final bailout: if the mask is simple, we are better off using an extract
+  // and a simple narrow shuffle. Prefer extract+unpack(h/l)ps to vpermps
+  // because that avoids a constant load from memory.
+  if (NumElts == 4 &&
+      (isSingleSHUFPSMask(NewMask) || is128BitUnpackShuffleMask(NewMask)))
+    return SDValue();
+
+  // Extend the shuffle mask with undef elements.
+  NewMask.append(NumElts, -1);
+
+  // shuf (extract X, 0), (extract X, 4), M --> extract (shuf X, undef, M'), 0
+  SDValue Shuf = DAG.getVectorShuffle(WideVT, DL, WideVec, DAG.getUNDEF(WideVT),
+                                      NewMask);
+  // This is free: ymm -> xmm.
+  return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Shuf,
+                     DAG.getIntPtrConstant(0, DL));
+}
+
 /// Try to lower broadcast of a single element.
 ///
 /// For convenience, this code also bundles all of the subtarget feature set
 /// filtering. While a little annoying to re-dispatch on type here, there isn't
 /// a convenient way to factor it out.
-static SDValue lowerVectorShuffleAsBroadcast(const SDLoc &DL, MVT VT,
-                                             SDValue V1, SDValue V2,
-                                             ArrayRef<int> Mask,
-                                             const X86Subtarget &Subtarget,
-                                             SelectionDAG &DAG) {
+static SDValue lowerShuffleAsBroadcast(const SDLoc &DL, MVT VT, SDValue V1,
+                                       SDValue V2, ArrayRef<int> Mask,
+                                       const X86Subtarget &Subtarget,
+                                       SelectionDAG &DAG) {
   if (!((Subtarget.hasSSE3() && VT == MVT::v2f64) ||
         (Subtarget.hasAVX() && VT.isFloatingPoint()) ||
         (Subtarget.hasAVX2() && VT.isInteger())))
@@ -11574,7 +11959,7 @@ static SDValue lowerVectorShuffleAsBroadcast(const SDLoc &DL, MVT VT,
   // type than the shuffle, the broadcast element is in essence truncated.
   // Make that explicit to ease folding.
   if (V.getOpcode() == ISD::BITCAST && VT.isInteger())
-    if (SDValue TruncBroadcast = lowerVectorShuffleAsTruncBroadcast(
+    if (SDValue TruncBroadcast = lowerShuffleAsTruncBroadcast(
             DL, VT, V.getOperand(0), BroadcastIdx, Subtarget, DAG))
       return TruncBroadcast;
 
@@ -11684,11 +12069,10 @@ static SDValue lowerVectorShuffleAsBroadcast(const SDLoc &DL, MVT VT,
 // are much smaller to encode than a SHUFPS and an INSERTPS. We can also
 // perform INSERTPS if a single V1 element is out of place and all V2
 // elements are zeroable.
-static bool matchVectorShuffleAsInsertPS(SDValue &V1, SDValue &V2,
-                                         unsigned &InsertPSMask,
-                                         const APInt &Zeroable,
-                                         ArrayRef<int> Mask,
-                                         SelectionDAG &DAG) {
+static bool matchShuffleAsInsertPS(SDValue &V1, SDValue &V2,
+                                   unsigned &InsertPSMask,
+                                   const APInt &Zeroable,
+                                   ArrayRef<int> Mask, SelectionDAG &DAG) {
   assert(V1.getSimpleValueType().is128BitVector() && "Bad operand type!");
   assert(V2.getSimpleValueType().is128BitVector() && "Bad operand type!");
   assert(Mask.size() == 4 && "Unexpected mask size for v4 shuffle!");
@@ -11773,16 +12157,15 @@ static bool matchVectorShuffleAsInsertPS(SDValue &V1, SDValue &V2,
   return false;
 }
 
-static SDValue lowerVectorShuffleAsInsertPS(const SDLoc &DL, SDValue V1,
-                                            SDValue V2, ArrayRef<int> Mask,
-                                            const APInt &Zeroable,
-                                            SelectionDAG &DAG) {
+static SDValue lowerShuffleAsInsertPS(const SDLoc &DL, SDValue V1, SDValue V2,
+                                      ArrayRef<int> Mask, const APInt &Zeroable,
+                                      SelectionDAG &DAG) {
   assert(V1.getSimpleValueType() == MVT::v4f32 && "Bad operand type!");
   assert(V2.getSimpleValueType() == MVT::v4f32 && "Bad operand type!");
 
   // Attempt to match the insertps pattern.
   unsigned InsertPSMask;
-  if (!matchVectorShuffleAsInsertPS(V1, V2, InsertPSMask, Zeroable, Mask, DAG))
+  if (!matchShuffleAsInsertPS(V1, V2, InsertPSMask, Zeroable, Mask, DAG))
     return SDValue();
 
   // Insert the V2 element into the desired position.
@@ -11799,10 +12182,9 @@ static SDValue lowerVectorShuffleAsInsertPS(const SDLoc &DL, SDValue V1,
 /// because for floating point vectors we have a generalized SHUFPS lowering
 /// strategy that handles everything that doesn't *exactly* match an unpack,
 /// making this clever lowering unnecessary.
-static SDValue lowerVectorShuffleAsPermuteAndUnpack(const SDLoc &DL, MVT VT,
-                                                    SDValue V1, SDValue V2,
-                                                    ArrayRef<int> Mask,
-                                                    SelectionDAG &DAG) {
+static SDValue lowerShuffleAsPermuteAndUnpack(
+    const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,
+    const X86Subtarget &Subtarget, SelectionDAG &DAG) {
   assert(!VT.isFloatingPoint() &&
          "This routine only supports integer vectors.");
   assert(VT.is128BitVector() &&
@@ -11871,6 +12253,12 @@ static SDValue lowerVectorShuffleAsPermuteAndUnpack(const SDLoc &DL, MVT VT,
     if (SDValue Unpack = TryUnpack(ScalarSize, ScalarSize / OrigScalarSize))
       return Unpack;
 
+  // If we're shuffling with a zero vector then we're better off not doing
+  // VECTOR_SHUFFLE(UNPCK()) as we lose track of those zero elements.
+  if (ISD::isBuildVectorAllZeros(V1.getNode()) ||
+      ISD::isBuildVectorAllZeros(V2.getNode()))
+    return SDValue();
+
   // If none of the unpack-rooted lowerings worked (or were profitable) try an
   // initial unpack.
   if (NumLoInputs == 0 || NumHiInputs == 0) {
@@ -11909,19 +12297,18 @@ static SDValue lowerVectorShuffleAsPermuteAndUnpack(const SDLoc &DL, MVT VT,
 /// instructions will incur a domain crossing penalty on some chips though so
 /// it is better to avoid lowering through this for integer vectors where
 /// possible.
-static SDValue lowerV2F64VectorShuffle(const SDLoc &DL, ArrayRef<int> Mask,
-                                       const APInt &Zeroable,
-                                       SDValue V1, SDValue V2,
-                                       const X86Subtarget &Subtarget,
-                                       SelectionDAG &DAG) {
+static SDValue lowerV2F64Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
+                                 const APInt &Zeroable, SDValue V1, SDValue V2,
+                                 const X86Subtarget &Subtarget,
+                                 SelectionDAG &DAG) {
   assert(V1.getSimpleValueType() == MVT::v2f64 && "Bad operand type!");
   assert(V2.getSimpleValueType() == MVT::v2f64 && "Bad operand type!");
   assert(Mask.size() == 2 && "Unexpected mask size for v2 shuffle!");
 
   if (V2.isUndef()) {
     // Check for being able to broadcast a single element.
-    if (SDValue Broadcast = lowerVectorShuffleAsBroadcast(
-            DL, MVT::v2f64, V1, V2, Mask, Subtarget, DAG))
+    if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v2f64, V1, V2,
+                                                    Mask, Subtarget, DAG))
       return Broadcast;
 
     // Straight shuffle of a single input vector. Simulate this by using the
@@ -11946,16 +12333,20 @@ static SDValue lowerV2F64VectorShuffle(const SDLoc &DL, ArrayRef<int> Mask,
   assert(Mask[0] < 2 && "We sort V1 to be the first input.");
   assert(Mask[1] >= 2 && "We sort V2 to be the second input.");
 
+  if (Subtarget.hasAVX2())
+    if (SDValue Extract = lowerShuffleOfExtractsAsVperm(DL, V1, V2, Mask, DAG))
+      return Extract;
+
   // When loading a scalar and then shuffling it into a vector we can often do
   // the insertion cheaply.
-  if (SDValue Insertion = lowerVectorShuffleAsElementInsertion(
+  if (SDValue Insertion = lowerShuffleAsElementInsertion(
           DL, MVT::v2f64, V1, V2, Mask, Zeroable, Subtarget, DAG))
     return Insertion;
   // Try inverting the insertion since for v2 masks it is easy to do and we
   // can't reliably sort the mask one way or the other.
   int InverseMask[2] = {Mask[0] < 0 ? -1 : (Mask[0] ^ 2),
                         Mask[1] < 0 ? -1 : (Mask[1] ^ 2)};
-  if (SDValue Insertion = lowerVectorShuffleAsElementInsertion(
+  if (SDValue Insertion = lowerShuffleAsElementInsertion(
           DL, MVT::v2f64, V2, V1, InverseMask, Zeroable, Subtarget, DAG))
     return Insertion;
 
@@ -11971,13 +12362,12 @@ static SDValue lowerV2F64VectorShuffle(const SDLoc &DL, ArrayRef<int> Mask,
           DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v2f64, V1S));
 
   if (Subtarget.hasSSE41())
-    if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v2f64, V1, V2, Mask,
-                                                  Zeroable, Subtarget, DAG))
+    if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v2f64, V1, V2, Mask,
+                                            Zeroable, Subtarget, DAG))
       return Blend;
 
   // Use dedicated unpack instructions for masks that match their pattern.
-  if (SDValue V =
-          lowerVectorShuffleWithUNPCK(DL, MVT::v2f64, Mask, V1, V2, DAG))
+  if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v2f64, Mask, V1, V2, DAG))
     return V;
 
   unsigned SHUFPDMask = (Mask[0] == 1) | (((Mask[1] - 2) == 1) << 1);
@@ -11991,19 +12381,18 @@ static SDValue lowerV2F64VectorShuffle(const SDLoc &DL, ArrayRef<int> Mask,
 /// the integer unit to minimize domain crossing penalties. However, for blends
 /// it falls back to the floating point shuffle operation with appropriate bit
 /// casting.
-static SDValue lowerV2I64VectorShuffle(const SDLoc &DL, ArrayRef<int> Mask,
-                                       const APInt &Zeroable,
-                                       SDValue V1, SDValue V2,
-                                       const X86Subtarget &Subtarget,
-                                       SelectionDAG &DAG) {
+static SDValue lowerV2I64Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
+                                 const APInt &Zeroable, SDValue V1, SDValue V2,
+                                 const X86Subtarget &Subtarget,
+                                 SelectionDAG &DAG) {
   assert(V1.getSimpleValueType() == MVT::v2i64 && "Bad operand type!");
   assert(V2.getSimpleValueType() == MVT::v2i64 && "Bad operand type!");
   assert(Mask.size() == 2 && "Unexpected mask size for v2 shuffle!");
 
   if (V2.isUndef()) {
     // Check for being able to broadcast a single element.
-    if (SDValue Broadcast = lowerVectorShuffleAsBroadcast(
-            DL, MVT::v2i64, V1, V2, Mask, Subtarget, DAG))
+    if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v2i64, V1, V2,
+                                                    Mask, Subtarget, DAG))
       return Broadcast;
 
     // Straight shuffle of a single input vector. For everything from SSE2
@@ -12023,20 +12412,24 @@ static SDValue lowerV2I64VectorShuffle(const SDLoc &DL, ArrayRef<int> Mask,
   assert(Mask[0] < 2 && "We sort V1 to be the first input.");
   assert(Mask[1] >= 2 && "We sort V2 to be the second input.");
 
+  if (Subtarget.hasAVX2())
+    if (SDValue Extract = lowerShuffleOfExtractsAsVperm(DL, V1, V2, Mask, DAG))
+      return Extract;
+
   // Try to use shift instructions.
-  if (SDValue Shift = lowerVectorShuffleAsShift(DL, MVT::v2i64, V1, V2, Mask,
-                                                Zeroable, Subtarget, DAG))
+  if (SDValue Shift = lowerShuffleAsShift(DL, MVT::v2i64, V1, V2, Mask,
+                                          Zeroable, Subtarget, DAG))
     return Shift;
 
   // When loading a scalar and then shuffling it into a vector we can often do
   // the insertion cheaply.
-  if (SDValue Insertion = lowerVectorShuffleAsElementInsertion(
+  if (SDValue Insertion = lowerShuffleAsElementInsertion(
           DL, MVT::v2i64, V1, V2, Mask, Zeroable, Subtarget, DAG))
     return Insertion;
   // Try inverting the insertion since for v2 masks it is easy to do and we
   // can't reliably sort the mask one way or the other.
   int InverseMask[2] = {Mask[0] ^ 2, Mask[1] ^ 2};
-  if (SDValue Insertion = lowerVectorShuffleAsElementInsertion(
+  if (SDValue Insertion = lowerShuffleAsElementInsertion(
           DL, MVT::v2i64, V2, V1, InverseMask, Zeroable, Subtarget, DAG))
     return Insertion;
 
@@ -12044,33 +12437,32 @@ static SDValue lowerV2I64VectorShuffle(const SDLoc &DL, ArrayRef<int> Mask,
   // *exact* same predicate.
   bool IsBlendSupported = Subtarget.hasSSE41();
   if (IsBlendSupported)
-    if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v2i64, V1, V2, Mask,
-                                                  Zeroable, Subtarget, DAG))
+    if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v2i64, V1, V2, Mask,
+                                            Zeroable, Subtarget, DAG))
       return Blend;
 
   // Use dedicated unpack instructions for masks that match their pattern.
-  if (SDValue V =
-          lowerVectorShuffleWithUNPCK(DL, MVT::v2i64, Mask, V1, V2, DAG))
+  if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v2i64, Mask, V1, V2, DAG))
     return V;
 
   // Try to use byte rotation instructions.
   // Its more profitable for pre-SSSE3 to use shuffles/unpacks.
   if (Subtarget.hasSSSE3()) {
     if (Subtarget.hasVLX())
-      if (SDValue Rotate = lowerVectorShuffleAsRotate(DL, MVT::v2i64, V1, V2,
-                                                      Mask, Subtarget, DAG))
+      if (SDValue Rotate = lowerShuffleAsRotate(DL, MVT::v2i64, V1, V2, Mask,
+                                                Subtarget, DAG))
         return Rotate;
 
-    if (SDValue Rotate = lowerVectorShuffleAsByteRotate(
-            DL, MVT::v2i64, V1, V2, Mask, Subtarget, DAG))
+    if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v2i64, V1, V2, Mask,
+                                                  Subtarget, DAG))
       return Rotate;
   }
 
   // If we have direct support for blends, we should lower by decomposing into
   // a permute. That will be faster than the domain cross.
   if (IsBlendSupported)
-    return lowerVectorShuffleAsDecomposedShuffleBlend(DL, MVT::v2i64, V1, V2,
-                                                      Mask, Subtarget, DAG);
+    return lowerShuffleAsDecomposedShuffleBlend(DL, MVT::v2i64, V1, V2, Mask,
+                                                Subtarget, DAG);
 
   // We implement this with SHUFPD which is pretty lame because it will likely
   // incur 2 cycles of stall for integer vectors on Nehalem and older chips.
@@ -12082,36 +12474,14 @@ static SDValue lowerV2I64VectorShuffle(const SDLoc &DL, ArrayRef<int> Mask,
                         DAG.getVectorShuffle(MVT::v2f64, DL, V1, V2, Mask));
 }
 
-/// Test whether this can be lowered with a single SHUFPS instruction.
-///
-/// This is used to disable more specialized lowerings when the shufps lowering
-/// will happen to be efficient.
-static bool isSingleSHUFPSMask(ArrayRef<int> Mask) {
-  // This routine only handles 128-bit shufps.
-  assert(Mask.size() == 4 && "Unsupported mask size!");
-  assert(Mask[0] >= -1 && Mask[0] < 8 && "Out of bound mask element!");
-  assert(Mask[1] >= -1 && Mask[1] < 8 && "Out of bound mask element!");
-  assert(Mask[2] >= -1 && Mask[2] < 8 && "Out of bound mask element!");
-  assert(Mask[3] >= -1 && Mask[3] < 8 && "Out of bound mask element!");
-
-  // To lower with a single SHUFPS we need to have the low half and high half
-  // each requiring a single input.
-  if (Mask[0] >= 0 && Mask[1] >= 0 && (Mask[0] < 4) != (Mask[1] < 4))
-    return false;
-  if (Mask[2] >= 0 && Mask[3] >= 0 && (Mask[2] < 4) != (Mask[3] < 4))
-    return false;
-
-  return true;
-}
-
 /// Lower a vector shuffle using the SHUFPS instruction.
 ///
 /// This is a helper routine dedicated to lowering vector shuffles using SHUFPS.
 /// It makes no assumptions about whether this is the *best* lowering, it simply
 /// uses it.
-static SDValue lowerVectorShuffleWithSHUFPS(const SDLoc &DL, MVT VT,
-                                            ArrayRef<int> Mask, SDValue V1,
-                                            SDValue V2, SelectionDAG &DAG) {
+static SDValue lowerShuffleWithSHUFPS(const SDLoc &DL, MVT VT,
+                                      ArrayRef<int> Mask, SDValue V1,
+                                      SDValue V2, SelectionDAG &DAG) {
   SDValue LowV = V1, HighV = V2;
   int NewMask[4] = {Mask[0], Mask[1], Mask[2], Mask[3]};
 
@@ -12196,11 +12566,10 @@ static SDValue lowerVectorShuffleWithSHUFPS(const SDLoc &DL, MVT VT,
 /// Uses instructions exclusively from the floating point unit to minimize
 /// domain crossing penalties, as these are sufficient to implement all v4f32
 /// shuffles.
-static SDValue lowerV4F32VectorShuffle(const SDLoc &DL, ArrayRef<int> Mask,
-                                       const APInt &Zeroable,
-                                       SDValue V1, SDValue V2,
-                                       const X86Subtarget &Subtarget,
-                                       SelectionDAG &DAG) {
+static SDValue lowerV4F32Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
+                                 const APInt &Zeroable, SDValue V1, SDValue V2,
+                                 const X86Subtarget &Subtarget,
+                                 SelectionDAG &DAG) {
   assert(V1.getSimpleValueType() == MVT::v4f32 && "Bad operand type!");
   assert(V2.getSimpleValueType() == MVT::v4f32 && "Bad operand type!");
   assert(Mask.size() == 4 && "Unexpected mask size for v4 shuffle!");
@@ -12209,8 +12578,8 @@ static SDValue lowerV4F32VectorShuffle(const SDLoc &DL, ArrayRef<int> Mask,
 
   if (NumV2Elements == 0) {
     // Check for being able to broadcast a single element.
-    if (SDValue Broadcast = lowerVectorShuffleAsBroadcast(
-            DL, MVT::v4f32, V1, V2, Mask, Subtarget, DAG))
+    if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v4f32, V1, V2,
+                                                    Mask, Subtarget, DAG))
       return Broadcast;
 
     // Use even/odd duplicate instructions for masks that match their pattern.
@@ -12243,29 +12612,32 @@ static SDValue lowerV4F32VectorShuffle(const SDLoc &DL, ArrayRef<int> Mask,
                        getV4X86ShuffleImm8ForMask(Mask, DL, DAG));
   }
 
+  if (Subtarget.hasAVX2())
+    if (SDValue Extract = lowerShuffleOfExtractsAsVperm(DL, V1, V2, Mask, DAG))
+      return Extract;
+
   // There are special ways we can lower some single-element blends. However, we
   // have custom ways we can lower more complex single-element blends below that
   // we defer to if both this and BLENDPS fail to match, so restrict this to
   // when the V2 input is targeting element 0 of the mask -- that is the fast
   // case here.
   if (NumV2Elements == 1 && Mask[0] >= 4)
-    if (SDValue V = lowerVectorShuffleAsElementInsertion(
+    if (SDValue V = lowerShuffleAsElementInsertion(
             DL, MVT::v4f32, V1, V2, Mask, Zeroable, Subtarget, DAG))
       return V;
 
   if (Subtarget.hasSSE41()) {
-    if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v4f32, V1, V2, Mask,
-                                                  Zeroable, Subtarget, DAG))
+    if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v4f32, V1, V2, Mask,
+                                            Zeroable, Subtarget, DAG))
       return Blend;
 
     // Use INSERTPS if we can complete the shuffle efficiently.
-    if (SDValue V =
-            lowerVectorShuffleAsInsertPS(DL, V1, V2, Mask, Zeroable, DAG))
+    if (SDValue V = lowerShuffleAsInsertPS(DL, V1, V2, Mask, Zeroable, DAG))
       return V;
 
     if (!isSingleSHUFPSMask(Mask))
-      if (SDValue BlendPerm = lowerVectorShuffleAsBlendAndPermute(
-              DL, MVT::v4f32, V1, V2, Mask, DAG))
+      if (SDValue BlendPerm = lowerShuffleAsBlendAndPermute(DL, MVT::v4f32, V1,
+                                                            V2, Mask, DAG))
         return BlendPerm;
   }
 
@@ -12279,23 +12651,21 @@ static SDValue lowerV4F32VectorShuffle(const SDLoc &DL, ArrayRef<int> Mask,
   }
 
   // Use dedicated unpack instructions for masks that match their pattern.
-  if (SDValue V =
-          lowerVectorShuffleWithUNPCK(DL, MVT::v4f32, Mask, V1, V2, DAG))
+  if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v4f32, Mask, V1, V2, DAG))
     return V;
 
   // Otherwise fall back to a SHUFPS lowering strategy.
-  return lowerVectorShuffleWithSHUFPS(DL, MVT::v4f32, Mask, V1, V2, DAG);
+  return lowerShuffleWithSHUFPS(DL, MVT::v4f32, Mask, V1, V2, DAG);
 }
 
 /// Lower 4-lane i32 vector shuffles.
 ///
 /// We try to handle these with integer-domain shuffles where we can, but for
 /// blends we use the floating point domain blend instructions.
-static SDValue lowerV4I32VectorShuffle(const SDLoc &DL, ArrayRef<int> Mask,
-                                       const APInt &Zeroable,
-                                       SDValue V1, SDValue V2,
-                                       const X86Subtarget &Subtarget,
-                                       SelectionDAG &DAG) {
+static SDValue lowerV4I32Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
+                                 const APInt &Zeroable, SDValue V1, SDValue V2,
+                                 const X86Subtarget &Subtarget,
+                                 SelectionDAG &DAG) {
   assert(V1.getSimpleValueType() == MVT::v4i32 && "Bad operand type!");
   assert(V2.getSimpleValueType() == MVT::v4i32 && "Bad operand type!");
   assert(Mask.size() == 4 && "Unexpected mask size for v4 shuffle!");
@@ -12303,16 +12673,16 @@ static SDValue lowerV4I32VectorShuffle(const SDLoc &DL, ArrayRef<int> Mask,
   // Whenever we can lower this as a zext, that instruction is strictly faster
   // than any alternative. It also allows us to fold memory operands into the
   // shuffle in many cases.
-  if (SDValue ZExt = lowerVectorShuffleAsZeroOrAnyExtend(
-          DL, MVT::v4i32, V1, V2, Mask, Zeroable, Subtarget, DAG))
+  if (SDValue ZExt = lowerShuffleAsZeroOrAnyExtend(DL, MVT::v4i32, V1, V2, Mask,
+                                                   Zeroable, Subtarget, DAG))
     return ZExt;
 
   int NumV2Elements = count_if(Mask, [](int M) { return M >= 4; });
 
   if (NumV2Elements == 0) {
     // Check for being able to broadcast a single element.
-    if (SDValue Broadcast = lowerVectorShuffleAsBroadcast(
-            DL, MVT::v4i32, V1, V2, Mask, Subtarget, DAG))
+    if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v4i32, V1, V2,
+                                                    Mask, Subtarget, DAG))
       return Broadcast;
 
     // Straight shuffle of a single input vector. For everything from SSE2
@@ -12331,14 +12701,18 @@ static SDValue lowerV4I32VectorShuffle(const SDLoc &DL, ArrayRef<int> Mask,
                        getV4X86ShuffleImm8ForMask(Mask, DL, DAG));
   }
 
+  if (Subtarget.hasAVX2())
+    if (SDValue Extract = lowerShuffleOfExtractsAsVperm(DL, V1, V2, Mask, DAG))
+      return Extract;
+
   // Try to use shift instructions.
-  if (SDValue Shift = lowerVectorShuffleAsShift(DL, MVT::v4i32, V1, V2, Mask,
-                                                Zeroable, Subtarget, DAG))
+  if (SDValue Shift = lowerShuffleAsShift(DL, MVT::v4i32, V1, V2, Mask,
+                                          Zeroable, Subtarget, DAG))
     return Shift;
 
   // There are special ways we can lower some single-element blends.
   if (NumV2Elements == 1)
-    if (SDValue V = lowerVectorShuffleAsElementInsertion(
+    if (SDValue V = lowerShuffleAsElementInsertion(
             DL, MVT::v4i32, V1, V2, Mask, Zeroable, Subtarget, DAG))
       return V;
 
@@ -12346,29 +12720,28 @@ static SDValue lowerV4I32VectorShuffle(const SDLoc &DL, ArrayRef<int> Mask,
   // *exact* same predicate.
   bool IsBlendSupported = Subtarget.hasSSE41();
   if (IsBlendSupported)
-    if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v4i32, V1, V2, Mask,
-                                                  Zeroable, Subtarget, DAG))
+    if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v4i32, V1, V2, Mask,
+                                            Zeroable, Subtarget, DAG))
       return Blend;
 
-  if (SDValue Masked = lowerVectorShuffleAsBitMask(DL, MVT::v4i32, V1, V2, Mask,
-                                                   Zeroable, DAG))
+  if (SDValue Masked = lowerShuffleAsBitMask(DL, MVT::v4i32, V1, V2, Mask,
+                                             Zeroable, DAG))
     return Masked;
 
   // Use dedicated unpack instructions for masks that match their pattern.
-  if (SDValue V =
-          lowerVectorShuffleWithUNPCK(DL, MVT::v4i32, Mask, V1, V2, DAG))
+  if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v4i32, Mask, V1, V2, DAG))
     return V;
 
   // Try to use byte rotation instructions.
   // Its more profitable for pre-SSSE3 to use shuffles/unpacks.
   if (Subtarget.hasSSSE3()) {
     if (Subtarget.hasVLX())
-      if (SDValue Rotate = lowerVectorShuffleAsRotate(DL, MVT::v4i32, V1, V2,
-                                                      Mask, Subtarget, DAG))
+      if (SDValue Rotate = lowerShuffleAsRotate(DL, MVT::v4i32, V1, V2, Mask,
+                                                Subtarget, DAG))
         return Rotate;
 
-    if (SDValue Rotate = lowerVectorShuffleAsByteRotate(
-            DL, MVT::v4i32, V1, V2, Mask, Subtarget, DAG))
+    if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v4i32, V1, V2, Mask,
+                                                  Subtarget, DAG))
       return Rotate;
   }
 
@@ -12379,12 +12752,12 @@ static SDValue lowerV4I32VectorShuffle(const SDLoc &DL, ArrayRef<int> Mask,
     // If we have direct support for blends, we should lower by decomposing into
     // a permute. That will be faster than the domain cross.
     if (IsBlendSupported)
-      return lowerVectorShuffleAsDecomposedShuffleBlend(DL, MVT::v4i32, V1, V2,
-                                                        Mask, Subtarget, DAG);
+      return lowerShuffleAsDecomposedShuffleBlend(DL, MVT::v4i32, V1, V2, Mask,
+                                                  Subtarget, DAG);
 
     // Try to lower by permuting the inputs into an unpack instruction.
-    if (SDValue Unpack = lowerVectorShuffleAsPermuteAndUnpack(
-            DL, MVT::v4i32, V1, V2, Mask, DAG))
+    if (SDValue Unpack = lowerShuffleAsPermuteAndUnpack(DL, MVT::v4i32, V1, V2,
+                                                        Mask, Subtarget, DAG))
       return Unpack;
   }
 
@@ -12415,7 +12788,7 @@ static SDValue lowerV4I32VectorShuffle(const SDLoc &DL, ArrayRef<int> Mask,
 /// lane must shuffle the *exact* same way. In fact, you must pass a v8 Mask to
 /// this routine for it to work correctly. To shuffle a 256-bit or 512-bit i16
 /// vector, form the analogous 128-bit 8-element Mask.
-static SDValue lowerV8I16GeneralSingleInputVectorShuffle(
+static SDValue lowerV8I16GeneralSingleInputShuffle(
     const SDLoc &DL, MVT VT, SDValue V, MutableArrayRef<int> Mask,
     const X86Subtarget &Subtarget, SelectionDAG &DAG) {
   assert(VT.getVectorElementType() == MVT::i16 && "Bad input type!");
@@ -12655,8 +13028,7 @@ static SDValue lowerV8I16GeneralSingleInputVectorShuffle(
 
     // Recurse back into this routine to re-compute state now that this isn't
     // a 3 and 1 problem.
-    return lowerV8I16GeneralSingleInputVectorShuffle(DL, VT, V, Mask, Subtarget,
-                                                     DAG);
+    return lowerV8I16GeneralSingleInputShuffle(DL, VT, V, Mask, Subtarget, DAG);
   };
   if ((NumLToL == 3 && NumHToL == 1) || (NumLToL == 1 && NumHToL == 3))
     return balanceSides(LToLInputs, HToLInputs, HToHInputs, LToHInputs, 0, 4);
@@ -12914,7 +13286,7 @@ static SDValue lowerV8I16GeneralSingleInputVectorShuffle(
 
 /// Helper to form a PSHUFB-based shuffle+blend, opportunistically avoiding the
 /// blend if only one input is used.
-static SDValue lowerVectorShuffleAsBlendOfPSHUFBs(
+static SDValue lowerShuffleAsBlendOfPSHUFBs(
     const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,
     const APInt &Zeroable, SelectionDAG &DAG, bool &V1InUse, bool &V2InUse) {
   assert(!is128BitLaneCrossingShuffleMask(VT, Mask) &&
@@ -12977,54 +13349,51 @@ static SDValue lowerVectorShuffleAsBlendOfPSHUFBs(
 /// the two inputs, try to interleave them. Otherwise, blend the low and high
 /// halves of the inputs separately (making them have relatively few inputs)
 /// and then concatenate them.
-static SDValue lowerV8I16VectorShuffle(const SDLoc &DL, ArrayRef<int> Mask,
-                                       const APInt &Zeroable,
-                                       SDValue V1, SDValue V2,
-                                       const X86Subtarget &Subtarget,
-                                       SelectionDAG &DAG) {
+static SDValue lowerV8I16Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
+                                 const APInt &Zeroable, SDValue V1, SDValue V2,
+                                 const X86Subtarget &Subtarget,
+                                 SelectionDAG &DAG) {
   assert(V1.getSimpleValueType() == MVT::v8i16 && "Bad operand type!");
   assert(V2.getSimpleValueType() == MVT::v8i16 && "Bad operand type!");
   assert(Mask.size() == 8 && "Unexpected mask size for v8 shuffle!");
 
   // Whenever we can lower this as a zext, that instruction is strictly faster
   // than any alternative.
-  if (SDValue ZExt = lowerVectorShuffleAsZeroOrAnyExtend(
-          DL, MVT::v8i16, V1, V2, Mask, Zeroable, Subtarget, DAG))
+  if (SDValue ZExt = lowerShuffleAsZeroOrAnyExtend(DL, MVT::v8i16, V1, V2, Mask,
+                                                   Zeroable, Subtarget, DAG))
     return ZExt;
 
   int NumV2Inputs = count_if(Mask, [](int M) { return M >= 8; });
 
   if (NumV2Inputs == 0) {
     // Check for being able to broadcast a single element.
-    if (SDValue Broadcast = lowerVectorShuffleAsBroadcast(
-            DL, MVT::v8i16, V1, V2, Mask, Subtarget, DAG))
+    if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v8i16, V1, V2,
+                                                    Mask, Subtarget, DAG))
       return Broadcast;
 
     // Try to use shift instructions.
-    if (SDValue Shift = lowerVectorShuffleAsShift(DL, MVT::v8i16, V1, V1, Mask,
-                                                  Zeroable, Subtarget, DAG))
+    if (SDValue Shift = lowerShuffleAsShift(DL, MVT::v8i16, V1, V1, Mask,
+                                            Zeroable, Subtarget, DAG))
       return Shift;
 
     // Use dedicated unpack instructions for masks that match their pattern.
-    if (SDValue V =
-            lowerVectorShuffleWithUNPCK(DL, MVT::v8i16, Mask, V1, V2, DAG))
+    if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v8i16, Mask, V1, V2, DAG))
       return V;
 
     // Use dedicated pack instructions for masks that match their pattern.
-    if (SDValue V = lowerVectorShuffleWithPACK(DL, MVT::v8i16, Mask, V1, V2,
-                                               DAG, Subtarget))
+    if (SDValue V = lowerShuffleWithPACK(DL, MVT::v8i16, Mask, V1, V2, DAG,
+                                         Subtarget))
       return V;
 
     // Try to use byte rotation instructions.
-    if (SDValue Rotate = lowerVectorShuffleAsByteRotate(DL, MVT::v8i16, V1, V1,
-                                                        Mask, Subtarget, DAG))
+    if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v8i16, V1, V1, Mask,
+                                                  Subtarget, DAG))
       return Rotate;
 
     // Make a copy of the mask so it can be modified.
     SmallVector<int, 8> MutableMask(Mask.begin(), Mask.end());
-    return lowerV8I16GeneralSingleInputVectorShuffle(DL, MVT::v8i16, V1,
-                                                     MutableMask, Subtarget,
-                                                     DAG);
+    return lowerV8I16GeneralSingleInputShuffle(DL, MVT::v8i16, V1, MutableMask,
+                                               Subtarget, DAG);
   }
 
   assert(llvm::any_of(Mask, [](int M) { return M >= 0 && M < 8; }) &&
@@ -13032,19 +13401,19 @@ static SDValue lowerV8I16VectorShuffle(const SDLoc &DL, ArrayRef<int> Mask,
          "shuffles.");
 
   // Try to use shift instructions.
-  if (SDValue Shift = lowerVectorShuffleAsShift(DL, MVT::v8i16, V1, V2, Mask,
-                                                Zeroable, Subtarget, DAG))
+  if (SDValue Shift = lowerShuffleAsShift(DL, MVT::v8i16, V1, V2, Mask,
+                                          Zeroable, Subtarget, DAG))
     return Shift;
 
   // See if we can use SSE4A Extraction / Insertion.
   if (Subtarget.hasSSE4A())
-    if (SDValue V = lowerVectorShuffleWithSSE4A(DL, MVT::v8i16, V1, V2, Mask,
-                                                Zeroable, DAG))
+    if (SDValue V = lowerShuffleWithSSE4A(DL, MVT::v8i16, V1, V2, Mask,
+                                          Zeroable, DAG))
       return V;
 
   // There are special ways we can lower some single-element blends.
   if (NumV2Inputs == 1)
-    if (SDValue V = lowerVectorShuffleAsElementInsertion(
+    if (SDValue V = lowerShuffleAsElementInsertion(
             DL, MVT::v8i16, V1, V2, Mask, Zeroable, Subtarget, DAG))
       return V;
 
@@ -13052,50 +13421,54 @@ static SDValue lowerV8I16VectorShuffle(const SDLoc &DL, ArrayRef<int> Mask,
   // *exact* same predicate.
   bool IsBlendSupported = Subtarget.hasSSE41();
   if (IsBlendSupported)
-    if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v8i16, V1, V2, Mask,
-                                                  Zeroable, Subtarget, DAG))
+    if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v8i16, V1, V2, Mask,
+                                            Zeroable, Subtarget, DAG))
       return Blend;
 
-  if (SDValue Masked = lowerVectorShuffleAsBitMask(DL, MVT::v8i16, V1, V2, Mask,
-                                                   Zeroable, DAG))
+  if (SDValue Masked = lowerShuffleAsBitMask(DL, MVT::v8i16, V1, V2, Mask,
+                                             Zeroable, DAG))
     return Masked;
 
   // Use dedicated unpack instructions for masks that match their pattern.
-  if (SDValue V =
-          lowerVectorShuffleWithUNPCK(DL, MVT::v8i16, Mask, V1, V2, DAG))
+  if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v8i16, Mask, V1, V2, DAG))
     return V;
 
   // Use dedicated pack instructions for masks that match their pattern.
-  if (SDValue V = lowerVectorShuffleWithPACK(DL, MVT::v8i16, Mask, V1, V2, DAG,
-                                             Subtarget))
+  if (SDValue V = lowerShuffleWithPACK(DL, MVT::v8i16, Mask, V1, V2, DAG,
+                                       Subtarget))
     return V;
 
   // Try to use byte rotation instructions.
-  if (SDValue Rotate = lowerVectorShuffleAsByteRotate(
-          DL, MVT::v8i16, V1, V2, Mask, Subtarget, DAG))
+  if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v8i16, V1, V2, Mask,
+                                                Subtarget, DAG))
     return Rotate;
 
   if (SDValue BitBlend =
-          lowerVectorShuffleAsBitBlend(DL, MVT::v8i16, V1, V2, Mask, DAG))
+          lowerShuffleAsBitBlend(DL, MVT::v8i16, V1, V2, Mask, DAG))
     return BitBlend;
 
+  // Try to use byte shift instructions to mask.
+  if (SDValue V = lowerVectorShuffleAsByteShiftMask(
+          DL, MVT::v8i16, V1, V2, Mask, Zeroable, Subtarget, DAG))
+    return V;
+
   // Try to lower by permuting the inputs into an unpack instruction.
-  if (SDValue Unpack = lowerVectorShuffleAsPermuteAndUnpack(DL, MVT::v8i16, V1,
-                                                            V2, Mask, DAG))
+  if (SDValue Unpack = lowerShuffleAsPermuteAndUnpack(DL, MVT::v8i16, V1, V2,
+                                                      Mask, Subtarget, DAG))
     return Unpack;
 
   // If we can't directly blend but can use PSHUFB, that will be better as it
   // can both shuffle and set up the inefficient blend.
   if (!IsBlendSupported && Subtarget.hasSSSE3()) {
     bool V1InUse, V2InUse;
-    return lowerVectorShuffleAsBlendOfPSHUFBs(DL, MVT::v8i16, V1, V2, Mask,
-                                              Zeroable, DAG, V1InUse, V2InUse);
+    return lowerShuffleAsBlendOfPSHUFBs(DL, MVT::v8i16, V1, V2, Mask,
+                                        Zeroable, DAG, V1InUse, V2InUse);
   }
 
   // We can always bit-blend if we have to so the fallback strategy is to
   // decompose into single-input permutes and blends.
-  return lowerVectorShuffleAsDecomposedShuffleBlend(DL, MVT::v8i16, V1, V2,
-                                                    Mask, Subtarget, DAG);
+  return lowerShuffleAsDecomposedShuffleBlend(DL, MVT::v8i16, V1, V2,
+                                              Mask, Subtarget, DAG);
 }
 
 /// Check whether a compaction lowering can be done by dropping even
@@ -13164,9 +13537,9 @@ static int canLowerByDroppingEvenElements(ArrayRef<int> Mask,
   return 0;
 }
 
-static SDValue lowerVectorShuffleWithPERMV(const SDLoc &DL, MVT VT,
-                                           ArrayRef<int> Mask, SDValue V1,
-                                           SDValue V2, SelectionDAG &DAG) {
+static SDValue lowerShuffleWithPERMV(const SDLoc &DL, MVT VT,
+                                     ArrayRef<int> Mask, SDValue V1,
+                                     SDValue V2, SelectionDAG &DAG) {
   MVT MaskEltVT = MVT::getIntegerVT(VT.getScalarSizeInBits());
   MVT MaskVecVT = MVT::getVectorVT(MaskEltVT, VT.getVectorNumElements());
 
@@ -13184,39 +13557,38 @@ static SDValue lowerVectorShuffleWithPERMV(const SDLoc &DL, MVT VT,
 /// UNPCK to spread the i8 elements across two i16-element vectors, and uses
 /// the existing lowering for v8i16 blends on each half, finally PACK-ing them
 /// back together.
-static SDValue lowerV16I8VectorShuffle(const SDLoc &DL, ArrayRef<int> Mask,
-                                       const APInt &Zeroable,
-                                       SDValue V1, SDValue V2,
-                                       const X86Subtarget &Subtarget,
-                                       SelectionDAG &DAG) {
+static SDValue lowerV16I8Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
+                                 const APInt &Zeroable, SDValue V1, SDValue V2,
+                                 const X86Subtarget &Subtarget,
+                                 SelectionDAG &DAG) {
   assert(V1.getSimpleValueType() == MVT::v16i8 && "Bad operand type!");
   assert(V2.getSimpleValueType() == MVT::v16i8 && "Bad operand type!");
   assert(Mask.size() == 16 && "Unexpected mask size for v16 shuffle!");
 
   // Try to use shift instructions.
-  if (SDValue Shift = lowerVectorShuffleAsShift(DL, MVT::v16i8, V1, V2, Mask,
-                                                Zeroable, Subtarget, DAG))
+  if (SDValue Shift = lowerShuffleAsShift(DL, MVT::v16i8, V1, V2, Mask,
+                                          Zeroable, Subtarget, DAG))
     return Shift;
 
   // Try to use byte rotation instructions.
-  if (SDValue Rotate = lowerVectorShuffleAsByteRotate(
-          DL, MVT::v16i8, V1, V2, Mask, Subtarget, DAG))
+  if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v16i8, V1, V2, Mask,
+                                                Subtarget, DAG))
     return Rotate;
 
   // Use dedicated pack instructions for masks that match their pattern.
-  if (SDValue V = lowerVectorShuffleWithPACK(DL, MVT::v16i8, Mask, V1, V2, DAG,
-                                             Subtarget))
+  if (SDValue V = lowerShuffleWithPACK(DL, MVT::v16i8, Mask, V1, V2, DAG,
+                                       Subtarget))
     return V;
 
   // Try to use a zext lowering.
-  if (SDValue ZExt = lowerVectorShuffleAsZeroOrAnyExtend(
-          DL, MVT::v16i8, V1, V2, Mask, Zeroable, Subtarget, DAG))
+  if (SDValue ZExt = lowerShuffleAsZeroOrAnyExtend(DL, MVT::v16i8, V1, V2, Mask,
+                                                   Zeroable, Subtarget, DAG))
     return ZExt;
 
   // See if we can use SSE4A Extraction / Insertion.
   if (Subtarget.hasSSE4A())
-    if (SDValue V = lowerVectorShuffleWithSSE4A(DL, MVT::v16i8, V1, V2, Mask,
-                                                Zeroable, DAG))
+    if (SDValue V = lowerShuffleWithSSE4A(DL, MVT::v16i8, V1, V2, Mask,
+                                          Zeroable, DAG))
       return V;
 
   int NumV2Elements = count_if(Mask, [](int M) { return M >= 16; });
@@ -13224,12 +13596,11 @@ static SDValue lowerV16I8VectorShuffle(const SDLoc &DL, ArrayRef<int> Mask,
   // For single-input shuffles, there are some nicer lowering tricks we can use.
   if (NumV2Elements == 0) {
     // Check for being able to broadcast a single element.
-    if (SDValue Broadcast = lowerVectorShuffleAsBroadcast(
-            DL, MVT::v16i8, V1, V2, Mask, Subtarget, DAG))
+    if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v16i8, V1, V2,
+                                                    Mask, Subtarget, DAG))
       return Broadcast;
 
-    if (SDValue V =
-            lowerVectorShuffleWithUNPCK(DL, MVT::v16i8, Mask, V1, V2, DAG))
+    if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v16i8, Mask, V1, V2, DAG))
       return V;
 
     // Check whether we can widen this to an i16 shuffle by duplicating bytes.
@@ -13322,13 +13693,17 @@ static SDValue lowerV16I8VectorShuffle(const SDLoc &DL, ArrayRef<int> Mask,
       return V;
   }
 
-  if (SDValue Masked = lowerVectorShuffleAsBitMask(DL, MVT::v16i8, V1, V2, Mask,
-                                                   Zeroable, DAG))
+  if (SDValue Masked = lowerShuffleAsBitMask(DL, MVT::v16i8, V1, V2, Mask,
+                                             Zeroable, DAG))
     return Masked;
 
   // Use dedicated unpack instructions for masks that match their pattern.
-  if (SDValue V =
-          lowerVectorShuffleWithUNPCK(DL, MVT::v16i8, Mask, V1, V2, DAG))
+  if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v16i8, Mask, V1, V2, DAG))
+    return V;
+
+  // Try to use byte shift instructions to mask.
+  if (SDValue V = lowerVectorShuffleAsByteShiftMask(
+          DL, MVT::v16i8, V1, V2, Mask, Zeroable, Subtarget, DAG))
     return V;
 
   // Check for SSSE3 which lets us lower all v16i8 shuffles much more directly
@@ -13348,7 +13723,7 @@ static SDValue lowerV16I8VectorShuffle(const SDLoc &DL, ArrayRef<int> Mask,
     bool V1InUse = false;
     bool V2InUse = false;
 
-    SDValue PSHUFB = lowerVectorShuffleAsBlendOfPSHUFBs(
+    SDValue PSHUFB = lowerShuffleAsBlendOfPSHUFBs(
         DL, MVT::v16i8, V1, V2, Mask, Zeroable, DAG, V1InUse, V2InUse);
 
     // If both V1 and V2 are in use and we can use a direct blend or an unpack,
@@ -13356,8 +13731,8 @@ static SDValue lowerV16I8VectorShuffle(const SDLoc &DL, ArrayRef<int> Mask,
     // important as a single pshufb is significantly faster for that.
     if (V1InUse && V2InUse) {
       if (Subtarget.hasSSE41())
-        if (SDValue Blend = lowerVectorShuffleAsBlend(
-                DL, MVT::v16i8, V1, V2, Mask, Zeroable, Subtarget, DAG))
+        if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v16i8, V1, V2, Mask,
+                                                Zeroable, Subtarget, DAG))
           return Blend;
 
       // We can use an unpack to do the blending rather than an or in some
@@ -13368,17 +13743,17 @@ static SDValue lowerV16I8VectorShuffle(const SDLoc &DL, ArrayRef<int> Mask,
       // FIXME: It might be worth trying to detect if the unpack-feeding
       // shuffles will both be pshufb, in which case we shouldn't bother with
       // this.
-      if (SDValue Unpack = lowerVectorShuffleAsPermuteAndUnpack(
-              DL, MVT::v16i8, V1, V2, Mask, DAG))
+      if (SDValue Unpack = lowerShuffleAsPermuteAndUnpack(
+              DL, MVT::v16i8, V1, V2, Mask, Subtarget, DAG))
         return Unpack;
 
       // If we have VBMI we can use one VPERM instead of multiple PSHUFBs.
       if (Subtarget.hasVBMI() && Subtarget.hasVLX())
-        return lowerVectorShuffleWithPERMV(DL, MVT::v16i8, Mask, V1, V2, DAG);
+        return lowerShuffleWithPERMV(DL, MVT::v16i8, Mask, V1, V2, DAG);
 
       // Use PALIGNR+Permute if possible - permute might become PSHUFB but the
       // PALIGNR will be cheaper than the second PSHUFB+OR.
-      if (SDValue V = lowerVectorShuffleAsByteRotateAndPermute(
+      if (SDValue V = lowerShuffleAsByteRotateAndPermute(
               DL, MVT::v16i8, V1, V2, Mask, Subtarget, DAG))
         return V;
     }
@@ -13388,13 +13763,12 @@ static SDValue lowerV16I8VectorShuffle(const SDLoc &DL, ArrayRef<int> Mask,
 
   // There are special ways we can lower some single-element blends.
   if (NumV2Elements == 1)
-    if (SDValue V = lowerVectorShuffleAsElementInsertion(
+    if (SDValue V = lowerShuffleAsElementInsertion(
             DL, MVT::v16i8, V1, V2, Mask, Zeroable, Subtarget, DAG))
       return V;
 
-  if (SDValue BitBlend =
-          lowerVectorShuffleAsBitBlend(DL, MVT::v16i8, V1, V2, Mask, DAG))
-    return BitBlend;
+  if (SDValue Blend = lowerShuffleAsBitBlend(DL, MVT::v16i8, V1, V2, Mask, DAG))
+    return Blend;
 
   // Check whether a compaction lowering can be done. This handles shuffles
   // which take every Nth element for some even N. See the helper function for
@@ -13435,8 +13809,8 @@ static SDValue lowerV16I8VectorShuffle(const SDLoc &DL, ArrayRef<int> Mask,
 
   // Handle multi-input cases by blending single-input shuffles.
   if (NumV2Elements > 0)
-    return lowerVectorShuffleAsDecomposedShuffleBlend(DL, MVT::v16i8, V1, V2,
-                                                      Mask, Subtarget, DAG);
+    return lowerShuffleAsDecomposedShuffleBlend(DL, MVT::v16i8, V1, V2, Mask,
+                                                Subtarget, DAG);
 
   // The fallback path for single-input shuffles widens this into two v8i16
   // vectors with unpacks, shuffles those, and then pulls them back together
@@ -13491,24 +13865,24 @@ static SDValue lowerV16I8VectorShuffle(const SDLoc &DL, ArrayRef<int> Mask,
 ///
 /// This routine breaks down the specific type of 128-bit shuffle and
 /// dispatches to the lowering routines accordingly.
-static SDValue lower128BitVectorShuffle(const SDLoc &DL, ArrayRef<int> Mask,
-                                        MVT VT, SDValue V1, SDValue V2,
-                                        const APInt &Zeroable,
-                                        const X86Subtarget &Subtarget,
-                                        SelectionDAG &DAG) {
+static SDValue lower128BitShuffle(const SDLoc &DL, ArrayRef<int> Mask,
+                                  MVT VT, SDValue V1, SDValue V2,
+                                  const APInt &Zeroable,
+                                  const X86Subtarget &Subtarget,
+                                  SelectionDAG &DAG) {
   switch (VT.SimpleTy) {
   case MVT::v2i64:
-    return lowerV2I64VectorShuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
+    return lowerV2I64Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
   case MVT::v2f64:
-    return lowerV2F64VectorShuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
+    return lowerV2F64Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
   case MVT::v4i32:
-    return lowerV4I32VectorShuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
+    return lowerV4I32Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
   case MVT::v4f32:
-    return lowerV4F32VectorShuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
+    return lowerV4F32Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
   case MVT::v8i16:
-    return lowerV8I16VectorShuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
+    return lowerV8I16Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
   case MVT::v16i8:
-    return lowerV16I8VectorShuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
+    return lowerV16I8Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
 
   default:
     llvm_unreachable("Unimplemented!");
@@ -13520,9 +13894,9 @@ static SDValue lower128BitVectorShuffle(const SDLoc &DL, ArrayRef<int> Mask,
 /// This routine just extracts two subvectors, shuffles them independently, and
 /// then concatenates them back together. This should work effectively with all
 /// AVX vector shuffle types.
-static SDValue splitAndLowerVectorShuffle(const SDLoc &DL, MVT VT, SDValue V1,
-                                          SDValue V2, ArrayRef<int> Mask,
-                                          SelectionDAG &DAG) {
+static SDValue splitAndLowerShuffle(const SDLoc &DL, MVT VT, SDValue V1,
+                                    SDValue V2, ArrayRef<int> Mask,
+                                    SelectionDAG &DAG) {
   assert(VT.getSizeInBits() >= 256 &&
          "Only for 256-bit or wider vector shuffles!");
   assert(V1.getSimpleValueType() == VT && "Bad operand type!");
@@ -13646,11 +14020,10 @@ static SDValue splitAndLowerVectorShuffle(const SDLoc &DL, MVT VT, SDValue V1,
 /// between splitting the shuffle into 128-bit components and stitching those
 /// back together vs. extracting the single-input shuffles and blending those
 /// results.
-static SDValue lowerVectorShuffleAsSplitOrBlend(const SDLoc &DL, MVT VT,
-                                                SDValue V1, SDValue V2,
-                                                ArrayRef<int> Mask,
-                                                const X86Subtarget &Subtarget,
-                                                SelectionDAG &DAG) {
+static SDValue lowerShuffleAsSplitOrBlend(const SDLoc &DL, MVT VT, SDValue V1,
+                                          SDValue V2, ArrayRef<int> Mask,
+                                          const X86Subtarget &Subtarget,
+                                          SelectionDAG &DAG) {
   assert(!V2.isUndef() && "This routine must not be used to lower single-input "
          "shuffles as it could then recurse on itself.");
   int Size = Mask.size();
@@ -13675,8 +14048,8 @@ static SDValue lowerVectorShuffleAsSplitOrBlend(const SDLoc &DL, MVT VT,
     return true;
   };
   if (DoBothBroadcast())
-    return lowerVectorShuffleAsDecomposedShuffleBlend(DL, VT, V1, V2, Mask,
-                                                      Subtarget, DAG);
+    return lowerShuffleAsDecomposedShuffleBlend(DL, VT, V1, V2, Mask,
+                                                Subtarget, DAG);
 
   // If the inputs all stem from a single 128-bit lane of each input, then we
   // split them rather than blending because the split will decompose to
@@ -13690,12 +14063,12 @@ static SDValue lowerVectorShuffleAsSplitOrBlend(const SDLoc &DL, MVT VT,
     if (Mask[i] >= 0)
       LaneInputs[Mask[i] / Size][(Mask[i] % Size) / LaneSize] = true;
   if (LaneInputs[0].count() <= 1 && LaneInputs[1].count() <= 1)
-    return splitAndLowerVectorShuffle(DL, VT, V1, V2, Mask, DAG);
+    return splitAndLowerShuffle(DL, VT, V1, V2, Mask, DAG);
 
   // Otherwise, just fall back to decomposed shuffles and a blend. This requires
   // that the decomposed single-input shuffles don't end up here.
-  return lowerVectorShuffleAsDecomposedShuffleBlend(DL, VT, V1, V2, Mask,
-                                                    Subtarget, DAG);
+  return lowerShuffleAsDecomposedShuffleBlend(DL, VT, V1, V2, Mask, Subtarget,
+                                              DAG);
 }
 
 /// Lower a vector shuffle crossing multiple 128-bit lanes as
@@ -13704,9 +14077,9 @@ static SDValue lowerVectorShuffleAsSplitOrBlend(const SDLoc &DL, MVT VT,
 /// This is mainly for cases where we can have non-repeating permutes
 /// in each lane.
 ///
-/// TODO: This is very similar to lowerVectorShuffleByMerging128BitLanes,
+/// TODO: This is very similar to lowerShuffleByMerging128BitLanes,
 /// we should investigate merging them.
-static SDValue lowerVectorShuffleAsLanePermuteAndPermute(
+static SDValue lowerShuffleAsLanePermuteAndPermute(
     const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,
     SelectionDAG &DAG, const X86Subtarget &Subtarget) {
   int NumElts = VT.getVectorNumElements();
@@ -13761,11 +14134,9 @@ static SDValue lowerVectorShuffleAsLanePermuteAndPermute(
 /// is lower than any other fully general cross-lane shuffle strategy I'm aware
 /// of. Special cases for each particular shuffle pattern should be handled
 /// prior to trying this lowering.
-static SDValue lowerVectorShuffleAsLanePermuteAndBlend(const SDLoc &DL, MVT VT,
-                                                       SDValue V1, SDValue V2,
-                                                       ArrayRef<int> Mask,
-                                                       SelectionDAG &DAG,
-                                                       const X86Subtarget &Subtarget) {
+static SDValue lowerShuffleAsLanePermuteAndBlend(
+    const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,
+    SelectionDAG &DAG, const X86Subtarget &Subtarget) {
   // FIXME: This should probably be generalized for 512-bit vectors as well.
   assert(VT.is256BitVector() && "Only for 256-bit vector shuffles!");
   int Size = Mask.size();
@@ -13780,14 +14151,14 @@ static SDValue lowerVectorShuffleAsLanePermuteAndBlend(const SDLoc &DL, MVT VT,
       if (Mask[i] >= 0 && (Mask[i] % Size) / LaneSize != i / LaneSize)
         LaneCrossing[(Mask[i] % Size) / LaneSize] = true;
     if (!LaneCrossing[0] || !LaneCrossing[1])
-      return splitAndLowerVectorShuffle(DL, VT, V1, V2, Mask, DAG);
+      return splitAndLowerShuffle(DL, VT, V1, V2, Mask, DAG);
   } else {
     bool LaneUsed[2] = {false, false};
     for (int i = 0; i < Size; ++i)
       if (Mask[i] >= 0)
         LaneUsed[(Mask[i] / LaneSize)] = true;
     if (!LaneUsed[0] || !LaneUsed[1])
-      return splitAndLowerVectorShuffle(DL, VT, V1, V2, Mask, DAG);
+      return splitAndLowerShuffle(DL, VT, V1, V2, Mask, DAG);
   }
 
   assert(V2.isUndef() &&
@@ -13811,11 +14182,11 @@ static SDValue lowerVectorShuffleAsLanePermuteAndBlend(const SDLoc &DL, MVT VT,
 }
 
 /// Handle lowering 2-lane 128-bit shuffles.
-static SDValue lowerV2X128VectorShuffle(const SDLoc &DL, MVT VT, SDValue V1,
-                                        SDValue V2, ArrayRef<int> Mask,
-                                        const APInt &Zeroable,
-                                        const X86Subtarget &Subtarget,
-                                        SelectionDAG &DAG) {
+static SDValue lowerV2X128Shuffle(const SDLoc &DL, MVT VT, SDValue V1,
+                                  SDValue V2, ArrayRef<int> Mask,
+                                  const APInt &Zeroable,
+                                  const X86Subtarget &Subtarget,
+                                  SelectionDAG &DAG) {
   // With AVX2, use VPERMQ/VPERMPD for unary shuffles to allow memory folding.
   if (Subtarget.hasAVX2() && V2.isUndef())
     return SDValue();
@@ -13842,8 +14213,8 @@ static SDValue lowerV2X128VectorShuffle(const SDLoc &DL, MVT VT, SDValue V1,
   // instruction bytes needed to explicitly generate the zero vector.
 
   // Blends are faster and handle all the non-lane-crossing cases.
-  if (SDValue Blend = lowerVectorShuffleAsBlend(DL, VT, V1, V2, Mask,
-                                                Zeroable, Subtarget, DAG))
+  if (SDValue Blend = lowerShuffleAsBlend(DL, VT, V1, V2, Mask, Zeroable,
+                                          Subtarget, DAG))
     return Blend;
 
   // If either input operand is a zero vector, use VPERM2X128 because its mask
@@ -13914,9 +14285,7 @@ static SDValue lowerV2X128VectorShuffle(const SDLoc &DL, MVT VT, SDValue V1,
 /// or two of the lanes of the inputs. The lanes of the input vectors are
 /// shuffled in one or two independent shuffles to get the lanes into the
 /// position needed by the final shuffle.
-///
-/// FIXME: This should be generalized to 512-bit shuffles.
-static SDValue lowerVectorShuffleByMerging128BitLanes(
+static SDValue lowerShuffleByMerging128BitLanes(
     const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,
     const X86Subtarget &Subtarget, SelectionDAG &DAG) {
   assert(!V2.isUndef() && "This is only useful with multiple inputs.");
@@ -13925,12 +14294,10 @@ static SDValue lowerVectorShuffleByMerging128BitLanes(
     return SDValue();
 
   int Size = Mask.size();
+  int NumLanes = VT.getSizeInBits() / 128;
   int LaneSize = 128 / VT.getScalarSizeInBits();
-  int NumLanes = Size / LaneSize;
-  assert(NumLanes == 2 && "Only handles 256-bit shuffles.");
-
   SmallVector<int, 16> RepeatMask(LaneSize, -1);
-  int LaneSrcs[2][2] = { { -1, -1 }, { -1 , -1 } };
+  SmallVector<std::array<int, 2>, 2> LaneSrcs(NumLanes, {{-1, -1}});
 
   // First pass will try to fill in the RepeatMask from lanes that need two
   // sources.
@@ -13941,7 +14308,7 @@ static SDValue lowerVectorShuffleByMerging128BitLanes(
       int M = Mask[(Lane * LaneSize) + i];
       if (M < 0)
         continue;
-      // Determine which of the 4 possible input lanes (2 from each source)
+      // Determine which of the possible input lanes (NumLanes from each source)
       // this element comes from. Assign that as one of the sources for this
       // lane. We can assign up to 2 sources for this lane. If we run out
       // sources we can't do anything.
@@ -14080,54 +14447,30 @@ static SDValue lowerVectorShuffleByMerging128BitLanes(
   return DAG.getVectorShuffle(VT, DL, NewV1, NewV2, NewMask);
 }
 
-/// Lower shuffles where an entire half of a 256 or 512-bit vector is UNDEF.
-/// This allows for fast cases such as subvector extraction/insertion
-/// or shuffling smaller vector types which can lower more efficiently.
-static SDValue lowerVectorShuffleWithUndefHalf(const SDLoc &DL, MVT VT,
-                                               SDValue V1, SDValue V2,
-                                               ArrayRef<int> Mask,
-                                               const X86Subtarget &Subtarget,
-                                               SelectionDAG &DAG) {
-  assert((VT.is256BitVector() || VT.is512BitVector()) &&
-         "Expected 256-bit or 512-bit vector");
-
-  unsigned NumElts = VT.getVectorNumElements();
-  unsigned HalfNumElts = NumElts / 2;
-  MVT HalfVT = MVT::getVectorVT(VT.getVectorElementType(), HalfNumElts);
-
-  bool UndefLower = isUndefInRange(Mask, 0, HalfNumElts);
-  bool UndefUpper = isUndefInRange(Mask, HalfNumElts, HalfNumElts);
-  if (!UndefLower && !UndefUpper)
-    return SDValue();
-
-  // Upper half is undef and lower half is whole upper subvector.
-  // e.g. vector_shuffle <4, 5, 6, 7, u, u, u, u> or <2, 3, u, u>
-  if (UndefUpper &&
-      isSequentialOrUndefInRange(Mask, 0, HalfNumElts, HalfNumElts)) {
-    SDValue Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HalfVT, V1,
-                             DAG.getIntPtrConstant(HalfNumElts, DL));
-    return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, DAG.getUNDEF(VT), Hi,
-                       DAG.getIntPtrConstant(0, DL));
-  }
-
-  // Lower half is undef and upper half is whole lower subvector.
-  // e.g. vector_shuffle <u, u, u, u, 0, 1, 2, 3> or <u, u, 0, 1>
-  if (UndefLower &&
-      isSequentialOrUndefInRange(Mask, HalfNumElts, HalfNumElts, 0)) {
-    SDValue Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HalfVT, V1,
-                             DAG.getIntPtrConstant(0, DL));
-    return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, DAG.getUNDEF(VT), Hi,
-                       DAG.getIntPtrConstant(HalfNumElts, DL));
-  }
+/// If the input shuffle mask results in a vector that is undefined in all upper
+/// or lower half elements and that mask accesses only 2 halves of the
+/// shuffle's operands, return true. A mask of half the width with mask indexes
+/// adjusted to access the extracted halves of the original shuffle operands is
+/// returned in HalfMask. HalfIdx1 and HalfIdx2 return whether the upper or
+/// lower half of each input operand is accessed.
+static bool
+getHalfShuffleMask(ArrayRef<int> Mask, MutableArrayRef<int> HalfMask,
+                   int &HalfIdx1, int &HalfIdx2) {
+  assert((Mask.size() == HalfMask.size() * 2) &&
+         "Expected input mask to be twice as long as output");
+
+  // Exactly one half of the result must be undef to allow narrowing.
+  bool UndefLower = isUndefLowerHalf(Mask);
+  bool UndefUpper = isUndefUpperHalf(Mask);
+  if (UndefLower == UndefUpper)
+    return false;
 
-  // If the shuffle only uses two of the four halves of the input operands,
-  // then extract them and perform the 'half' shuffle at half width.
-  // e.g. vector_shuffle <X, X, X, X, u, u, u, u> or <X, X, u, u>
-  int HalfIdx1 = -1, HalfIdx2 = -1;
-  SmallVector<int, 8> HalfMask(HalfNumElts);
-  unsigned Offset = UndefLower ? HalfNumElts : 0;
+  unsigned HalfNumElts = HalfMask.size();
+  unsigned MaskIndexOffset = UndefLower ? HalfNumElts : 0;
+  HalfIdx1 = -1;
+  HalfIdx2 = -1;
   for (unsigned i = 0; i != HalfNumElts; ++i) {
-    int M = Mask[i + Offset];
+    int M = Mask[i + MaskIndexOffset];
     if (M < 0) {
       HalfMask[i] = M;
       continue;
@@ -14154,42 +14497,27 @@ static SDValue lowerVectorShuffleWithUndefHalf(const SDLoc &DL, MVT VT,
     }
 
     // Too many half vectors referenced.
-    return SDValue();
+    return false;
   }
-  assert(HalfMask.size() == HalfNumElts && "Unexpected shuffle mask length");
-
-  // Only shuffle the halves of the inputs when useful.
-  int NumLowerHalves =
-      (HalfIdx1 == 0 || HalfIdx1 == 2) + (HalfIdx2 == 0 || HalfIdx2 == 2);
-  int NumUpperHalves =
-      (HalfIdx1 == 1 || HalfIdx1 == 3) + (HalfIdx2 == 1 || HalfIdx2 == 3);
-
-  // uuuuXXXX - don't extract uppers just to insert again.
-  if (UndefLower && NumUpperHalves != 0)
-    return SDValue();
 
-  // XXXXuuuu - don't extract both uppers, instead shuffle and then extract.
-  if (UndefUpper && NumUpperHalves == 2)
-    return SDValue();
+  return true;
+}
 
-  // AVX2 - XXXXuuuu - always extract lowers.
-  if (Subtarget.hasAVX2() && !(UndefUpper && NumUpperHalves == 0)) {
-    // AVX2 supports efficient immediate 64-bit element cross-lane shuffles.
-    if (VT == MVT::v4f64 || VT == MVT::v4i64)
-      return SDValue();
-    // AVX2 supports variable 32-bit element cross-lane shuffles.
-    if (VT == MVT::v8f32 || VT == MVT::v8i32) {
-      // XXXXuuuu - don't extract lowers and uppers.
-      if (UndefUpper && NumLowerHalves != 0 && NumUpperHalves != 0)
-        return SDValue();
-    }
-  }
+/// Given the output values from getHalfShuffleMask(), create a half width
+/// shuffle of extracted vectors followed by an insert back to full width.
+static SDValue getShuffleHalfVectors(const SDLoc &DL, SDValue V1, SDValue V2,
+                                     ArrayRef<int> HalfMask, int HalfIdx1,
+                                     int HalfIdx2, bool UndefLower,
+                                     SelectionDAG &DAG) {
+  assert(V1.getValueType() == V2.getValueType() && "Different sized vectors?");
+  assert(V1.getValueType().isSimple() && "Expecting only simple types");
 
-  // AVX512 - XXXXuuuu - always extract lowers.
-  if (VT.is512BitVector() && !(UndefUpper && NumUpperHalves == 0))
-    return SDValue();
+  MVT VT = V1.getSimpleValueType();
+  unsigned NumElts = VT.getVectorNumElements();
+  unsigned HalfNumElts = NumElts / 2;
+  MVT HalfVT = MVT::getVectorVT(VT.getVectorElementType(), HalfNumElts);
 
-  auto GetHalfVector = [&](int HalfIdx) {
+  auto getHalfVector = [&](int HalfIdx) {
     if (HalfIdx < 0)
       return DAG.getUNDEF(HalfVT);
     SDValue V = (HalfIdx < 2 ? V1 : V2);
@@ -14198,13 +14526,125 @@ static SDValue lowerVectorShuffleWithUndefHalf(const SDLoc &DL, MVT VT,
                        DAG.getIntPtrConstant(HalfIdx, DL));
   };
 
-  SDValue Half1 = GetHalfVector(HalfIdx1);
-  SDValue Half2 = GetHalfVector(HalfIdx2);
+  // ins undef, (shuf (ext V1, HalfIdx1), (ext V2, HalfIdx2), HalfMask), Offset
+  SDValue Half1 = getHalfVector(HalfIdx1);
+  SDValue Half2 = getHalfVector(HalfIdx2);
   SDValue V = DAG.getVectorShuffle(HalfVT, DL, Half1, Half2, HalfMask);
+  unsigned Offset = UndefLower ? HalfNumElts : 0;
   return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, DAG.getUNDEF(VT), V,
                      DAG.getIntPtrConstant(Offset, DL));
 }
 
+/// Lower shuffles where an entire half of a 256 or 512-bit vector is UNDEF.
+/// This allows for fast cases such as subvector extraction/insertion
+/// or shuffling smaller vector types which can lower more efficiently.
+static SDValue lowerShuffleWithUndefHalf(const SDLoc &DL, MVT VT, SDValue V1,
+                                         SDValue V2, ArrayRef<int> Mask,
+                                         const X86Subtarget &Subtarget,
+                                         SelectionDAG &DAG) {
+  assert((VT.is256BitVector() || VT.is512BitVector()) &&
+         "Expected 256-bit or 512-bit vector");
+
+  bool UndefLower = isUndefLowerHalf(Mask);
+  if (!UndefLower && !isUndefUpperHalf(Mask))
+    return SDValue();
+
+  assert((!UndefLower || !isUndefUpperHalf(Mask)) &&
+         "Completely undef shuffle mask should have been simplified already");
+
+  // Upper half is undef and lower half is whole upper subvector.
+  // e.g. vector_shuffle <4, 5, 6, 7, u, u, u, u> or <2, 3, u, u>
+  unsigned NumElts = VT.getVectorNumElements();
+  unsigned HalfNumElts = NumElts / 2;
+  MVT HalfVT = MVT::getVectorVT(VT.getVectorElementType(), HalfNumElts);
+  if (!UndefLower &&
+      isSequentialOrUndefInRange(Mask, 0, HalfNumElts, HalfNumElts)) {
+    SDValue Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HalfVT, V1,
+                             DAG.getIntPtrConstant(HalfNumElts, DL));
+    return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, DAG.getUNDEF(VT), Hi,
+                       DAG.getIntPtrConstant(0, DL));
+  }
+
+  // Lower half is undef and upper half is whole lower subvector.
+  // e.g. vector_shuffle <u, u, u, u, 0, 1, 2, 3> or <u, u, 0, 1>
+  if (UndefLower &&
+      isSequentialOrUndefInRange(Mask, HalfNumElts, HalfNumElts, 0)) {
+    SDValue Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HalfVT, V1,
+                             DAG.getIntPtrConstant(0, DL));
+    return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, DAG.getUNDEF(VT), Hi,
+                       DAG.getIntPtrConstant(HalfNumElts, DL));
+  }
+
+  int HalfIdx1, HalfIdx2;
+  SmallVector<int, 8> HalfMask(HalfNumElts);
+  if (!getHalfShuffleMask(Mask, HalfMask, HalfIdx1, HalfIdx2))
+    return SDValue();
+
+  assert(HalfMask.size() == HalfNumElts && "Unexpected shuffle mask length");
+
+  // Only shuffle the halves of the inputs when useful.
+  unsigned NumLowerHalves =
+      (HalfIdx1 == 0 || HalfIdx1 == 2) + (HalfIdx2 == 0 || HalfIdx2 == 2);
+  unsigned NumUpperHalves =
+      (HalfIdx1 == 1 || HalfIdx1 == 3) + (HalfIdx2 == 1 || HalfIdx2 == 3);
+  assert(NumLowerHalves + NumUpperHalves <= 2 && "Only 1 or 2 halves allowed");
+
+  // Determine the larger pattern of undef/halves, then decide if it's worth
+  // splitting the shuffle based on subtarget capabilities and types.
+  unsigned EltWidth = VT.getVectorElementType().getSizeInBits();
+  if (!UndefLower) {
+    // XXXXuuuu: no insert is needed.
+    // Always extract lowers when setting lower - these are all free subreg ops.
+    if (NumUpperHalves == 0)
+      return getShuffleHalfVectors(DL, V1, V2, HalfMask, HalfIdx1, HalfIdx2,
+                                   UndefLower, DAG);
+
+    if (NumUpperHalves == 1) {
+      // AVX2 has efficient 32/64-bit element cross-lane shuffles.
+      if (Subtarget.hasAVX2()) {
+        // extract128 + vunpckhps, is better than vblend + vpermps.
+        // TODO: Refine to account for unary shuffle, splat, and other masks?
+        if (EltWidth == 32 && NumLowerHalves &&
+            HalfVT.is128BitVector() && !is128BitUnpackShuffleMask(HalfMask))
+          return SDValue();
+        // If this is a unary shuffle (assume that the 2nd operand is
+        // canonicalized to undef), then we can use vpermpd. Otherwise, we
+        // are better off extracting the upper half of 1 operand and using a
+        // narrow shuffle.
+        if (EltWidth == 64 && V2.isUndef())
+          return SDValue();
+      }
+      // AVX512 has efficient cross-lane shuffles for all legal 512-bit types.
+      if (Subtarget.hasAVX512() && VT.is512BitVector())
+        return SDValue();
+      // Extract + narrow shuffle is better than the wide alternative.
+      return getShuffleHalfVectors(DL, V1, V2, HalfMask, HalfIdx1, HalfIdx2,
+                                   UndefLower, DAG);
+    }
+
+    // Don't extract both uppers, instead shuffle and then extract.
+    assert(NumUpperHalves == 2 && "Half vector count went wrong");
+    return SDValue();
+  }
+
+  // UndefLower - uuuuXXXX: an insert to high half is required if we split this.
+  if (NumUpperHalves == 0) {
+    // AVX2 has efficient 64-bit element cross-lane shuffles.
+    // TODO: Refine to account for unary shuffle, splat, and other masks?
+    if (Subtarget.hasAVX2() && EltWidth == 64)
+      return SDValue();
+    // AVX512 has efficient cross-lane shuffles for all legal 512-bit types.
+    if (Subtarget.hasAVX512() && VT.is512BitVector())
+      return SDValue();
+    // Narrow shuffle + insert is better than the wide alternative.
+    return getShuffleHalfVectors(DL, V1, V2, HalfMask, HalfIdx1, HalfIdx2,
+                                 UndefLower, DAG);
+  }
+
+  // NumUpperHalves != 0: don't bother with extract, shuffle, and then insert.
+  return SDValue();
+}
+
 /// Test whether the specified input (0 or 1) is in-place blended by the
 /// given mask.
 ///
@@ -14390,9 +14830,8 @@ static SDValue lowerShuffleAsRepeatedMaskAndLanePermute(
                               SubLaneMask);
 }
 
-static bool matchVectorShuffleWithSHUFPD(MVT VT, SDValue &V1, SDValue &V2,
-                                         unsigned &ShuffleImm,
-                                         ArrayRef<int> Mask) {
+static bool matchShuffleWithSHUFPD(MVT VT, SDValue &V1, SDValue &V2,
+                                   unsigned &ShuffleImm, ArrayRef<int> Mask) {
   int NumElts = VT.getVectorNumElements();
   assert(VT.getScalarSizeInBits() == 64 &&
          (NumElts == 2 || NumElts == 4 || NumElts == 8) &&
@@ -14427,14 +14866,14 @@ static bool matchVectorShuffleWithSHUFPD(MVT VT, SDValue &V1, SDValue &V2,
   return false;
 }
 
-static SDValue lowerVectorShuffleWithSHUFPD(const SDLoc &DL, MVT VT,
-                                            ArrayRef<int> Mask, SDValue V1,
-                                            SDValue V2, SelectionDAG &DAG) {
+static SDValue lowerShuffleWithSHUFPD(const SDLoc &DL, MVT VT,
+                                      ArrayRef<int> Mask, SDValue V1,
+                                      SDValue V2, SelectionDAG &DAG) {
   assert((VT == MVT::v2f64 || VT == MVT::v4f64 || VT == MVT::v8f64)&&
          "Unexpected data type for VSHUFPD");
 
   unsigned Immediate = 0;
-  if (!matchVectorShuffleWithSHUFPD(VT, V1, V2, Immediate, Mask))
+  if (!matchShuffleWithSHUFPD(VT, V1, V2, Immediate, Mask))
     return SDValue();
 
   return DAG.getNode(X86ISD::SHUFP, DL, VT, V1, V2,
@@ -14445,23 +14884,22 @@ static SDValue lowerVectorShuffleWithSHUFPD(const SDLoc &DL, MVT VT,
 ///
 /// Also ends up handling lowering of 4-lane 64-bit integer shuffles when AVX2
 /// isn't available.
-static SDValue lowerV4F64VectorShuffle(const SDLoc &DL, ArrayRef<int> Mask,
-                                       const APInt &Zeroable,
-                                       SDValue V1, SDValue V2,
-                                       const X86Subtarget &Subtarget,
-                                       SelectionDAG &DAG) {
+static SDValue lowerV4F64Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
+                                 const APInt &Zeroable, SDValue V1, SDValue V2,
+                                 const X86Subtarget &Subtarget,
+                                 SelectionDAG &DAG) {
   assert(V1.getSimpleValueType() == MVT::v4f64 && "Bad operand type!");
   assert(V2.getSimpleValueType() == MVT::v4f64 && "Bad operand type!");
   assert(Mask.size() == 4 && "Unexpected mask size for v4 shuffle!");
 
-  if (SDValue V = lowerV2X128VectorShuffle(DL, MVT::v4f64, V1, V2, Mask,
-                                           Zeroable, Subtarget, DAG))
+  if (SDValue V = lowerV2X128Shuffle(DL, MVT::v4f64, V1, V2, Mask, Zeroable,
+                                     Subtarget, DAG))
     return V;
 
   if (V2.isUndef()) {
     // Check for being able to broadcast a single element.
-    if (SDValue Broadcast = lowerVectorShuffleAsBroadcast(
-            DL, MVT::v4f64, V1, V2, Mask, Subtarget, DAG))
+    if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v4f64, V1, V2,
+                                                    Mask, Subtarget, DAG))
       return Broadcast;
 
     // Use low duplicate instructions for masks that match their pattern.
@@ -14489,27 +14927,25 @@ static SDValue lowerV4F64VectorShuffle(const SDLoc &DL, ArrayRef<int> Mask,
       return V;
 
     // Try to permute the lanes and then use a per-lane permute.
-    if (SDValue V = lowerVectorShuffleAsLanePermuteAndPermute(
-            DL, MVT::v4f64, V1, V2, Mask, DAG, Subtarget))
+    if (SDValue V = lowerShuffleAsLanePermuteAndPermute(DL, MVT::v4f64, V1, V2,
+                                                        Mask, DAG, Subtarget))
       return V;
 
     // Otherwise, fall back.
-    return lowerVectorShuffleAsLanePermuteAndBlend(DL, MVT::v4f64, V1, V2, Mask,
-                                                   DAG, Subtarget);
+    return lowerShuffleAsLanePermuteAndBlend(DL, MVT::v4f64, V1, V2, Mask, DAG,
+                                             Subtarget);
   }
 
   // Use dedicated unpack instructions for masks that match their pattern.
-  if (SDValue V =
-          lowerVectorShuffleWithUNPCK(DL, MVT::v4f64, Mask, V1, V2, DAG))
+  if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v4f64, Mask, V1, V2, DAG))
     return V;
 
-  if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v4f64, V1, V2, Mask,
-                                                Zeroable, Subtarget, DAG))
+  if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v4f64, V1, V2, Mask,
+                                          Zeroable, Subtarget, DAG))
     return Blend;
 
   // Check if the blend happens to exactly fit that of SHUFPD.
-  if (SDValue Op =
-      lowerVectorShuffleWithSHUFPD(DL, MVT::v4f64, Mask, V1, V2, DAG))
+  if (SDValue Op = lowerShuffleWithSHUFPD(DL, MVT::v4f64, Mask, V1, V2, DAG))
     return Op;
 
   // Try to create an in-lane repeating shuffle mask and then shuffle the
@@ -14524,52 +14960,51 @@ static SDValue lowerV4F64VectorShuffle(const SDLoc &DL, ArrayRef<int> Mask,
   // instruction so skip this pattern.
   if (!(Subtarget.hasAVX2() && (isShuffleMaskInputInPlace(0, Mask) ||
                                 isShuffleMaskInputInPlace(1, Mask))))
-    if (SDValue Result = lowerVectorShuffleByMerging128BitLanes(
-            DL, MVT::v4f64, V1, V2, Mask, Subtarget, DAG))
-      return Result;
+    if (SDValue V = lowerShuffleByMerging128BitLanes(DL, MVT::v4f64, V1, V2,
+                                                     Mask, Subtarget, DAG))
+      return V;
 
   // If we have VLX support, we can use VEXPAND.
   if (Subtarget.hasVLX())
-    if (SDValue V = lowerVectorShuffleToEXPAND(DL, MVT::v4f64, Zeroable, Mask,
-                                               V1, V2, DAG, Subtarget))
+    if (SDValue V = lowerShuffleToEXPAND(DL, MVT::v4f64, Zeroable, Mask, V1, V2,
+                                         DAG, Subtarget))
       return V;
 
   // If we have AVX2 then we always want to lower with a blend because an v4 we
   // can fully permute the elements.
   if (Subtarget.hasAVX2())
-    return lowerVectorShuffleAsDecomposedShuffleBlend(DL, MVT::v4f64, V1, V2,
-                                                      Mask, Subtarget, DAG);
+    return lowerShuffleAsDecomposedShuffleBlend(DL, MVT::v4f64, V1, V2, Mask,
+                                                Subtarget, DAG);
 
   // Otherwise fall back on generic lowering.
-  return lowerVectorShuffleAsSplitOrBlend(DL, MVT::v4f64, V1, V2, Mask,
-                                          Subtarget, DAG);
+  return lowerShuffleAsSplitOrBlend(DL, MVT::v4f64, V1, V2, Mask,
+                                    Subtarget, DAG);
 }
 
 /// Handle lowering of 4-lane 64-bit integer shuffles.
 ///
 /// This routine is only called when we have AVX2 and thus a reasonable
 /// instruction set for v4i64 shuffling..
-static SDValue lowerV4I64VectorShuffle(const SDLoc &DL, ArrayRef<int> Mask,
-                                       const APInt &Zeroable,
-                                       SDValue V1, SDValue V2,
-                                       const X86Subtarget &Subtarget,
-                                       SelectionDAG &DAG) {
+static SDValue lowerV4I64Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
+                                 const APInt &Zeroable, SDValue V1, SDValue V2,
+                                 const X86Subtarget &Subtarget,
+                                 SelectionDAG &DAG) {
   assert(V1.getSimpleValueType() == MVT::v4i64 && "Bad operand type!");
   assert(V2.getSimpleValueType() == MVT::v4i64 && "Bad operand type!");
   assert(Mask.size() == 4 && "Unexpected mask size for v4 shuffle!");
   assert(Subtarget.hasAVX2() && "We can only lower v4i64 with AVX2!");
 
-  if (SDValue V = lowerV2X128VectorShuffle(DL, MVT::v4i64, V1, V2, Mask,
-                                           Zeroable, Subtarget, DAG))
+  if (SDValue V = lowerV2X128Shuffle(DL, MVT::v4i64, V1, V2, Mask, Zeroable,
+                                     Subtarget, DAG))
     return V;
 
-  if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v4i64, V1, V2, Mask,
-                                                Zeroable, Subtarget, DAG))
+  if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v4i64, V1, V2, Mask,
+                                          Zeroable, Subtarget, DAG))
     return Blend;
 
   // Check for being able to broadcast a single element.
-  if (SDValue Broadcast = lowerVectorShuffleAsBroadcast(DL, MVT::v4i64, V1, V2,
-                                                        Mask, Subtarget, DAG))
+  if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v4i64, V1, V2, Mask,
+                                                  Subtarget, DAG))
     return Broadcast;
 
   if (V2.isUndef()) {
@@ -14593,29 +15028,28 @@ static SDValue lowerV4I64VectorShuffle(const SDLoc &DL, ArrayRef<int> Mask,
   }
 
   // Try to use shift instructions.
-  if (SDValue Shift = lowerVectorShuffleAsShift(DL, MVT::v4i64, V1, V2, Mask,
-                                                Zeroable, Subtarget, DAG))
+  if (SDValue Shift = lowerShuffleAsShift(DL, MVT::v4i64, V1, V2, Mask,
+                                          Zeroable, Subtarget, DAG))
     return Shift;
 
   // If we have VLX support, we can use VALIGN or VEXPAND.
   if (Subtarget.hasVLX()) {
-    if (SDValue Rotate = lowerVectorShuffleAsRotate(DL, MVT::v4i64, V1, V2,
-                                                    Mask, Subtarget, DAG))
+    if (SDValue Rotate = lowerShuffleAsRotate(DL, MVT::v4i64, V1, V2, Mask,
+                                              Subtarget, DAG))
       return Rotate;
 
-    if (SDValue V = lowerVectorShuffleToEXPAND(DL, MVT::v4i64, Zeroable, Mask,
-                                               V1, V2, DAG, Subtarget))
+    if (SDValue V = lowerShuffleToEXPAND(DL, MVT::v4i64, Zeroable, Mask, V1, V2,
+                                         DAG, Subtarget))
       return V;
   }
 
   // Try to use PALIGNR.
-  if (SDValue Rotate = lowerVectorShuffleAsByteRotate(DL, MVT::v4i64, V1, V2,
-                                                      Mask, Subtarget, DAG))
+  if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v4i64, V1, V2, Mask,
+                                                Subtarget, DAG))
     return Rotate;
 
   // Use dedicated unpack instructions for masks that match their pattern.
-  if (SDValue V =
-          lowerVectorShuffleWithUNPCK(DL, MVT::v4i64, Mask, V1, V2, DAG))
+  if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v4i64, Mask, V1, V2, DAG))
     return V;
 
   // Try to create an in-lane repeating shuffle mask and then shuffle the
@@ -14630,35 +15064,34 @@ static SDValue lowerV4I64VectorShuffle(const SDLoc &DL, ArrayRef<int> Mask,
   // instruction so skip this pattern.
   if (!isShuffleMaskInputInPlace(0, Mask) &&
       !isShuffleMaskInputInPlace(1, Mask))
-    if (SDValue Result = lowerVectorShuffleByMerging128BitLanes(
+    if (SDValue Result = lowerShuffleByMerging128BitLanes(
             DL, MVT::v4i64, V1, V2, Mask, Subtarget, DAG))
       return Result;
 
   // Otherwise fall back on generic blend lowering.
-  return lowerVectorShuffleAsDecomposedShuffleBlend(DL, MVT::v4i64, V1, V2,
-                                                    Mask, Subtarget, DAG);
+  return lowerShuffleAsDecomposedShuffleBlend(DL, MVT::v4i64, V1, V2, Mask,
+                                              Subtarget, DAG);
 }
 
 /// Handle lowering of 8-lane 32-bit floating point shuffles.
 ///
 /// Also ends up handling lowering of 8-lane 32-bit integer shuffles when AVX2
 /// isn't available.
-static SDValue lowerV8F32VectorShuffle(const SDLoc &DL, ArrayRef<int> Mask,
-                                       const APInt &Zeroable,
-                                       SDValue V1, SDValue V2,
-                                       const X86Subtarget &Subtarget,
-                                       SelectionDAG &DAG) {
+static SDValue lowerV8F32Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
+                                 const APInt &Zeroable, SDValue V1, SDValue V2,
+                                 const X86Subtarget &Subtarget,
+                                 SelectionDAG &DAG) {
   assert(V1.getSimpleValueType() == MVT::v8f32 && "Bad operand type!");
   assert(V2.getSimpleValueType() == MVT::v8f32 && "Bad operand type!");
   assert(Mask.size() == 8 && "Unexpected mask size for v8 shuffle!");
 
-  if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v8f32, V1, V2, Mask,
-                                                Zeroable, Subtarget, DAG))
+  if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v8f32, V1, V2, Mask,
+                                          Zeroable, Subtarget, DAG))
     return Blend;
 
   // Check for being able to broadcast a single element.
-  if (SDValue Broadcast = lowerVectorShuffleAsBroadcast(DL, MVT::v8f32, V1, V2,
-                                                        Mask, Subtarget, DAG))
+  if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v8f32, V1, V2, Mask,
+                                                  Subtarget, DAG))
     return Broadcast;
 
   // If the shuffle mask is repeated in each 128-bit lane, we have many more
@@ -14679,13 +15112,12 @@ static SDValue lowerV8F32VectorShuffle(const SDLoc &DL, ArrayRef<int> Mask,
                          getV4X86ShuffleImm8ForMask(RepeatedMask, DL, DAG));
 
     // Use dedicated unpack instructions for masks that match their pattern.
-    if (SDValue V =
-            lowerVectorShuffleWithUNPCK(DL, MVT::v8f32, Mask, V1, V2, DAG))
+    if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v8f32, Mask, V1, V2, DAG))
       return V;
 
     // Otherwise, fall back to a SHUFPS sequence. Here it is important that we
     // have already handled any direct blends.
-    return lowerVectorShuffleWithSHUFPS(DL, MVT::v8f32, RepeatedMask, V1, V2, DAG);
+    return lowerShuffleWithSHUFPS(DL, MVT::v8f32, RepeatedMask, V1, V2, DAG);
   }
 
   // Try to create an in-lane repeating shuffle mask and then shuffle the
@@ -14705,49 +15137,48 @@ static SDValue lowerV8F32VectorShuffle(const SDLoc &DL, ArrayRef<int> Mask,
       return DAG.getNode(X86ISD::VPERMV, DL, MVT::v8f32, VPermMask, V1);
 
     // Otherwise, fall back.
-    return lowerVectorShuffleAsLanePermuteAndBlend(DL, MVT::v8f32, V1, V2, Mask,
-                                                   DAG, Subtarget);
+    return lowerShuffleAsLanePermuteAndBlend(DL, MVT::v8f32, V1, V2, Mask,
+                                             DAG, Subtarget);
   }
 
   // Try to simplify this by merging 128-bit lanes to enable a lane-based
   // shuffle.
-  if (SDValue Result = lowerVectorShuffleByMerging128BitLanes(
+  if (SDValue Result = lowerShuffleByMerging128BitLanes(
           DL, MVT::v8f32, V1, V2, Mask, Subtarget, DAG))
     return Result;
   // If we have VLX support, we can use VEXPAND.
   if (Subtarget.hasVLX())
-    if (SDValue V = lowerVectorShuffleToEXPAND(DL, MVT::v8f32, Zeroable, Mask,
-                                               V1, V2, DAG, Subtarget))
+    if (SDValue V = lowerShuffleToEXPAND(DL, MVT::v8f32, Zeroable, Mask, V1, V2,
+                                         DAG, Subtarget))
       return V;
 
   // For non-AVX512 if the Mask is of 16bit elements in lane then try to split
   // since after split we get a more efficient code using vpunpcklwd and
   // vpunpckhwd instrs than vblend.
   if (!Subtarget.hasAVX512() && isUnpackWdShuffleMask(Mask, MVT::v8f32))
-    if (SDValue V = lowerVectorShuffleAsSplitOrBlend(DL, MVT::v8f32, V1, V2,
-                                                     Mask, Subtarget, DAG))
+    if (SDValue V = lowerShuffleAsSplitOrBlend(DL, MVT::v8f32, V1, V2, Mask,
+                                               Subtarget, DAG))
       return V;
 
   // If we have AVX2 then we always want to lower with a blend because at v8 we
   // can fully permute the elements.
   if (Subtarget.hasAVX2())
-    return lowerVectorShuffleAsDecomposedShuffleBlend(DL, MVT::v8f32, V1, V2,
-                                                      Mask, Subtarget, DAG);
+    return lowerShuffleAsDecomposedShuffleBlend(DL, MVT::v8f32, V1, V2, Mask,
+                                                Subtarget, DAG);
 
   // Otherwise fall back on generic lowering.
-  return lowerVectorShuffleAsSplitOrBlend(DL, MVT::v8f32, V1, V2, Mask,
-                                          Subtarget, DAG);
+  return lowerShuffleAsSplitOrBlend(DL, MVT::v8f32, V1, V2, Mask,
+                                    Subtarget, DAG);
 }
 
 /// Handle lowering of 8-lane 32-bit integer shuffles.
 ///
 /// This routine is only called when we have AVX2 and thus a reasonable
 /// instruction set for v8i32 shuffling..
-static SDValue lowerV8I32VectorShuffle(const SDLoc &DL, ArrayRef<int> Mask,
-                                       const APInt &Zeroable,
-                                       SDValue V1, SDValue V2,
-                                       const X86Subtarget &Subtarget,
-                                       SelectionDAG &DAG) {
+static SDValue lowerV8I32Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
+                                 const APInt &Zeroable, SDValue V1, SDValue V2,
+                                 const X86Subtarget &Subtarget,
+                                 SelectionDAG &DAG) {
   assert(V1.getSimpleValueType() == MVT::v8i32 && "Bad operand type!");
   assert(V2.getSimpleValueType() == MVT::v8i32 && "Bad operand type!");
   assert(Mask.size() == 8 && "Unexpected mask size for v8 shuffle!");
@@ -14756,8 +15187,8 @@ static SDValue lowerV8I32VectorShuffle(const SDLoc &DL, ArrayRef<int> Mask,
   // Whenever we can lower this as a zext, that instruction is strictly faster
   // than any alternative. It also allows us to fold memory operands into the
   // shuffle in many cases.
-  if (SDValue ZExt = lowerVectorShuffleAsZeroOrAnyExtend(
-          DL, MVT::v8i32, V1, V2, Mask, Zeroable, Subtarget, DAG))
+  if (SDValue ZExt = lowerShuffleAsZeroOrAnyExtend(DL, MVT::v8i32, V1, V2, Mask,
+                                                   Zeroable, Subtarget, DAG))
     return ZExt;
 
   // For non-AVX512 if the Mask is of 16bit elements in lane then try to split
@@ -14765,17 +15196,17 @@ static SDValue lowerV8I32VectorShuffle(const SDLoc &DL, ArrayRef<int> Mask,
   // vpunpcklwd and vpunpckhwd instrs.
   if (isUnpackWdShuffleMask(Mask, MVT::v8i32) && !V2.isUndef() &&
       !Subtarget.hasAVX512())
-    if (SDValue V = lowerVectorShuffleAsSplitOrBlend(DL, MVT::v8i32, V1, V2,
-                                                     Mask, Subtarget, DAG))
+    if (SDValue V = lowerShuffleAsSplitOrBlend(DL, MVT::v8i32, V1, V2, Mask,
+                                               Subtarget, DAG))
       return V;
 
-  if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v8i32, V1, V2, Mask,
-                                                Zeroable, Subtarget, DAG))
+  if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v8i32, V1, V2, Mask,
+                                          Zeroable, Subtarget, DAG))
     return Blend;
 
   // Check for being able to broadcast a single element.
-  if (SDValue Broadcast = lowerVectorShuffleAsBroadcast(DL, MVT::v8i32, V1, V2,
-                                                        Mask, Subtarget, DAG))
+  if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v8i32, V1, V2, Mask,
+                                                  Subtarget, DAG))
     return Broadcast;
 
   // If the shuffle mask is repeated in each 128-bit lane we can use more
@@ -14791,30 +15222,29 @@ static SDValue lowerV8I32VectorShuffle(const SDLoc &DL, ArrayRef<int> Mask,
                          getV4X86ShuffleImm8ForMask(RepeatedMask, DL, DAG));
 
     // Use dedicated unpack instructions for masks that match their pattern.
-    if (SDValue V =
-            lowerVectorShuffleWithUNPCK(DL, MVT::v8i32, Mask, V1, V2, DAG))
+    if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v8i32, Mask, V1, V2, DAG))
       return V;
   }
 
   // Try to use shift instructions.
-  if (SDValue Shift = lowerVectorShuffleAsShift(DL, MVT::v8i32, V1, V2, Mask,
-                                                Zeroable, Subtarget, DAG))
+  if (SDValue Shift = lowerShuffleAsShift(DL, MVT::v8i32, V1, V2, Mask,
+                                          Zeroable, Subtarget, DAG))
     return Shift;
 
   // If we have VLX support, we can use VALIGN or EXPAND.
   if (Subtarget.hasVLX()) {
-    if (SDValue Rotate = lowerVectorShuffleAsRotate(DL, MVT::v8i32, V1, V2,
-                                                    Mask, Subtarget, DAG))
+    if (SDValue Rotate = lowerShuffleAsRotate(DL, MVT::v8i32, V1, V2, Mask,
+                                              Subtarget, DAG))
       return Rotate;
 
-    if (SDValue V = lowerVectorShuffleToEXPAND(DL, MVT::v8i32, Zeroable, Mask,
-                                               V1, V2, DAG, Subtarget))
+    if (SDValue V = lowerShuffleToEXPAND(DL, MVT::v8i32, Zeroable, Mask, V1, V2,
+                                         DAG, Subtarget))
       return V;
   }
 
   // Try to use byte rotation instructions.
-  if (SDValue Rotate = lowerVectorShuffleAsByteRotate(
-          DL, MVT::v8i32, V1, V2, Mask, Subtarget, DAG))
+  if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v8i32, V1, V2, Mask,
+                                                Subtarget, DAG))
     return Rotate;
 
   // Try to create an in-lane repeating shuffle mask and then shuffle the
@@ -14836,31 +15266,30 @@ static SDValue lowerV8I32VectorShuffle(const SDLoc &DL, ArrayRef<int> Mask,
   if (Is128BitLaneRepeatedShuffle && isSingleSHUFPSMask(RepeatedMask)) {
     SDValue CastV1 = DAG.getBitcast(MVT::v8f32, V1);
     SDValue CastV2 = DAG.getBitcast(MVT::v8f32, V2);
-    SDValue ShufPS = lowerVectorShuffleWithSHUFPS(DL, MVT::v8f32, RepeatedMask,
-                                                  CastV1, CastV2, DAG);
+    SDValue ShufPS = lowerShuffleWithSHUFPS(DL, MVT::v8f32, RepeatedMask,
+                                            CastV1, CastV2, DAG);
     return DAG.getBitcast(MVT::v8i32, ShufPS);
   }
 
   // Try to simplify this by merging 128-bit lanes to enable a lane-based
   // shuffle.
-  if (SDValue Result = lowerVectorShuffleByMerging128BitLanes(
+  if (SDValue Result = lowerShuffleByMerging128BitLanes(
           DL, MVT::v8i32, V1, V2, Mask, Subtarget, DAG))
     return Result;
 
   // Otherwise fall back on generic blend lowering.
-  return lowerVectorShuffleAsDecomposedShuffleBlend(DL, MVT::v8i32, V1, V2,
-                                                    Mask, Subtarget, DAG);
+  return lowerShuffleAsDecomposedShuffleBlend(DL, MVT::v8i32, V1, V2, Mask,
+                                              Subtarget, DAG);
 }
 
 /// Handle lowering of 16-lane 16-bit integer shuffles.
 ///
 /// This routine is only called when we have AVX2 and thus a reasonable
 /// instruction set for v16i16 shuffling..
-static SDValue lowerV16I16VectorShuffle(const SDLoc &DL, ArrayRef<int> Mask,
-                                        const APInt &Zeroable,
-                                        SDValue V1, SDValue V2,
-                                        const X86Subtarget &Subtarget,
-                                        SelectionDAG &DAG) {
+static SDValue lowerV16I16Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
+                                  const APInt &Zeroable, SDValue V1, SDValue V2,
+                                  const X86Subtarget &Subtarget,
+                                  SelectionDAG &DAG) {
   assert(V1.getSimpleValueType() == MVT::v16i16 && "Bad operand type!");
   assert(V2.getSimpleValueType() == MVT::v16i16 && "Bad operand type!");
   assert(Mask.size() == 16 && "Unexpected mask size for v16 shuffle!");
@@ -14869,37 +15298,36 @@ static SDValue lowerV16I16VectorShuffle(const SDLoc &DL, ArrayRef<int> Mask,
   // Whenever we can lower this as a zext, that instruction is strictly faster
   // than any alternative. It also allows us to fold memory operands into the
   // shuffle in many cases.
-  if (SDValue ZExt = lowerVectorShuffleAsZeroOrAnyExtend(
+  if (SDValue ZExt = lowerShuffleAsZeroOrAnyExtend(
           DL, MVT::v16i16, V1, V2, Mask, Zeroable, Subtarget, DAG))
     return ZExt;
 
   // Check for being able to broadcast a single element.
-  if (SDValue Broadcast = lowerVectorShuffleAsBroadcast(DL, MVT::v16i16, V1, V2,
-                                                        Mask, Subtarget, DAG))
+  if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v16i16, V1, V2, Mask,
+                                                  Subtarget, DAG))
     return Broadcast;
 
-  if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v16i16, V1, V2, Mask,
-                                                Zeroable, Subtarget, DAG))
+  if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v16i16, V1, V2, Mask,
+                                          Zeroable, Subtarget, DAG))
     return Blend;
 
   // Use dedicated unpack instructions for masks that match their pattern.
-  if (SDValue V =
-          lowerVectorShuffleWithUNPCK(DL, MVT::v16i16, Mask, V1, V2, DAG))
+  if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v16i16, Mask, V1, V2, DAG))
     return V;
 
   // Use dedicated pack instructions for masks that match their pattern.
-  if (SDValue V = lowerVectorShuffleWithPACK(DL, MVT::v16i16, Mask, V1, V2, DAG,
-                                             Subtarget))
+  if (SDValue V = lowerShuffleWithPACK(DL, MVT::v16i16, Mask, V1, V2, DAG,
+                                       Subtarget))
     return V;
 
   // Try to use shift instructions.
-  if (SDValue Shift = lowerVectorShuffleAsShift(DL, MVT::v16i16, V1, V2, Mask,
-                                                Zeroable, Subtarget, DAG))
+  if (SDValue Shift = lowerShuffleAsShift(DL, MVT::v16i16, V1, V2, Mask,
+                                          Zeroable, Subtarget, DAG))
     return Shift;
 
   // Try to use byte rotation instructions.
-  if (SDValue Rotate = lowerVectorShuffleAsByteRotate(
-          DL, MVT::v16i16, V1, V2, Mask, Subtarget, DAG))
+  if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v16i16, V1, V2, Mask,
+                                                Subtarget, DAG))
     return Rotate;
 
   // Try to create an in-lane repeating shuffle mask and then shuffle the
@@ -14912,12 +15340,12 @@ static SDValue lowerV16I16VectorShuffle(const SDLoc &DL, ArrayRef<int> Mask,
     // There are no generalized cross-lane shuffle operations available on i16
     // element types.
     if (is128BitLaneCrossingShuffleMask(MVT::v16i16, Mask)) {
-      if (SDValue V = lowerVectorShuffleAsLanePermuteAndPermute(
+      if (SDValue V = lowerShuffleAsLanePermuteAndPermute(
               DL, MVT::v16i16, V1, V2, Mask, DAG, Subtarget))
         return V;
 
-      return lowerVectorShuffleAsLanePermuteAndBlend(DL, MVT::v16i16, V1, V2,
-                                                     Mask, DAG, Subtarget);
+      return lowerShuffleAsLanePermuteAndBlend(DL, MVT::v16i16, V1, V2, Mask,
+                                               DAG, Subtarget);
     }
 
     SmallVector<int, 8> RepeatedMask;
@@ -14925,44 +15353,43 @@ static SDValue lowerV16I16VectorShuffle(const SDLoc &DL, ArrayRef<int> Mask,
       // As this is a single-input shuffle, the repeated mask should be
       // a strictly valid v8i16 mask that we can pass through to the v8i16
       // lowering to handle even the v16 case.
-      return lowerV8I16GeneralSingleInputVectorShuffle(
+      return lowerV8I16GeneralSingleInputShuffle(
           DL, MVT::v16i16, V1, RepeatedMask, Subtarget, DAG);
     }
   }
 
-  if (SDValue PSHUFB = lowerVectorShuffleWithPSHUFB(
-          DL, MVT::v16i16, Mask, V1, V2, Zeroable, Subtarget, DAG))
+  if (SDValue PSHUFB = lowerShuffleWithPSHUFB(DL, MVT::v16i16, Mask, V1, V2,
+                                              Zeroable, Subtarget, DAG))
     return PSHUFB;
 
   // AVX512BWVL can lower to VPERMW.
   if (Subtarget.hasBWI() && Subtarget.hasVLX())
-    return lowerVectorShuffleWithPERMV(DL, MVT::v16i16, Mask, V1, V2, DAG);
+    return lowerShuffleWithPERMV(DL, MVT::v16i16, Mask, V1, V2, DAG);
 
   // Try to simplify this by merging 128-bit lanes to enable a lane-based
   // shuffle.
-  if (SDValue Result = lowerVectorShuffleByMerging128BitLanes(
+  if (SDValue Result = lowerShuffleByMerging128BitLanes(
           DL, MVT::v16i16, V1, V2, Mask, Subtarget, DAG))
     return Result;
 
   // Try to permute the lanes and then use a per-lane permute.
-  if (SDValue V = lowerVectorShuffleAsLanePermuteAndPermute(
+  if (SDValue V = lowerShuffleAsLanePermuteAndPermute(
           DL, MVT::v16i16, V1, V2, Mask, DAG, Subtarget))
     return V;
 
   // Otherwise fall back on generic lowering.
-  return lowerVectorShuffleAsSplitOrBlend(DL, MVT::v16i16, V1, V2, Mask,
-                                          Subtarget, DAG);
+  return lowerShuffleAsSplitOrBlend(DL, MVT::v16i16, V1, V2, Mask,
+                                    Subtarget, DAG);
 }
 
 /// Handle lowering of 32-lane 8-bit integer shuffles.
 ///
 /// This routine is only called when we have AVX2 and thus a reasonable
 /// instruction set for v32i8 shuffling..
-static SDValue lowerV32I8VectorShuffle(const SDLoc &DL, ArrayRef<int> Mask,
-                                       const APInt &Zeroable,
-                                       SDValue V1, SDValue V2,
-                                       const X86Subtarget &Subtarget,
-                                       SelectionDAG &DAG) {
+static SDValue lowerV32I8Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
+                                 const APInt &Zeroable, SDValue V1, SDValue V2,
+                                 const X86Subtarget &Subtarget,
+                                 SelectionDAG &DAG) {
   assert(V1.getSimpleValueType() == MVT::v32i8 && "Bad operand type!");
   assert(V2.getSimpleValueType() == MVT::v32i8 && "Bad operand type!");
   assert(Mask.size() == 32 && "Unexpected mask size for v32 shuffle!");
@@ -14971,37 +15398,36 @@ static SDValue lowerV32I8VectorShuffle(const SDLoc &DL, ArrayRef<int> Mask,
   // Whenever we can lower this as a zext, that instruction is strictly faster
   // than any alternative. It also allows us to fold memory operands into the
   // shuffle in many cases.
-  if (SDValue ZExt = lowerVectorShuffleAsZeroOrAnyExtend(
-          DL, MVT::v32i8, V1, V2, Mask, Zeroable, Subtarget, DAG))
+  if (SDValue ZExt = lowerShuffleAsZeroOrAnyExtend(DL, MVT::v32i8, V1, V2, Mask,
+                                                   Zeroable, Subtarget, DAG))
     return ZExt;
 
   // Check for being able to broadcast a single element.
-  if (SDValue Broadcast = lowerVectorShuffleAsBroadcast(DL, MVT::v32i8, V1, V2,
-                                                        Mask, Subtarget, DAG))
+  if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v32i8, V1, V2, Mask,
+                                                  Subtarget, DAG))
     return Broadcast;
 
-  if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v32i8, V1, V2, Mask,
-                                                Zeroable, Subtarget, DAG))
+  if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v32i8, V1, V2, Mask,
+                                          Zeroable, Subtarget, DAG))
     return Blend;
 
   // Use dedicated unpack instructions for masks that match their pattern.
-  if (SDValue V =
-          lowerVectorShuffleWithUNPCK(DL, MVT::v32i8, Mask, V1, V2, DAG))
+  if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v32i8, Mask, V1, V2, DAG))
     return V;
 
   // Use dedicated pack instructions for masks that match their pattern.
-  if (SDValue V = lowerVectorShuffleWithPACK(DL, MVT::v32i8, Mask, V1, V2, DAG,
-                                             Subtarget))
+  if (SDValue V = lowerShuffleWithPACK(DL, MVT::v32i8, Mask, V1, V2, DAG,
+                                       Subtarget))
     return V;
 
   // Try to use shift instructions.
-  if (SDValue Shift = lowerVectorShuffleAsShift(DL, MVT::v32i8, V1, V2, Mask,
+  if (SDValue Shift = lowerShuffleAsShift(DL, MVT::v32i8, V1, V2, Mask,
                                                 Zeroable, Subtarget, DAG))
     return Shift;
 
   // Try to use byte rotation instructions.
-  if (SDValue Rotate = lowerVectorShuffleAsByteRotate(
-          DL, MVT::v32i8, V1, V2, Mask, Subtarget, DAG))
+  if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v32i8, V1, V2, Mask,
+                                                Subtarget, DAG))
     return Rotate;
 
   // Try to create an in-lane repeating shuffle mask and then shuffle the
@@ -15013,36 +15439,36 @@ static SDValue lowerV32I8VectorShuffle(const SDLoc &DL, ArrayRef<int> Mask,
   // There are no generalized cross-lane shuffle operations available on i8
   // element types.
   if (V2.isUndef() && is128BitLaneCrossingShuffleMask(MVT::v32i8, Mask)) {
-    if (SDValue V = lowerVectorShuffleAsLanePermuteAndPermute(
+    if (SDValue V = lowerShuffleAsLanePermuteAndPermute(
             DL, MVT::v32i8, V1, V2, Mask, DAG, Subtarget))
       return V;
 
-    return lowerVectorShuffleAsLanePermuteAndBlend(DL, MVT::v32i8, V1, V2, Mask,
-                                                   DAG, Subtarget);
+    return lowerShuffleAsLanePermuteAndBlend(DL, MVT::v32i8, V1, V2, Mask, DAG,
+                                             Subtarget);
   }
 
-  if (SDValue PSHUFB = lowerVectorShuffleWithPSHUFB(
-          DL, MVT::v32i8, Mask, V1, V2, Zeroable, Subtarget, DAG))
+  if (SDValue PSHUFB = lowerShuffleWithPSHUFB(DL, MVT::v32i8, Mask, V1, V2,
+                                              Zeroable, Subtarget, DAG))
     return PSHUFB;
 
   // AVX512VBMIVL can lower to VPERMB.
   if (Subtarget.hasVBMI() && Subtarget.hasVLX())
-    return lowerVectorShuffleWithPERMV(DL, MVT::v32i8, Mask, V1, V2, DAG);
+    return lowerShuffleWithPERMV(DL, MVT::v32i8, Mask, V1, V2, DAG);
 
   // Try to simplify this by merging 128-bit lanes to enable a lane-based
   // shuffle.
-  if (SDValue Result = lowerVectorShuffleByMerging128BitLanes(
+  if (SDValue Result = lowerShuffleByMerging128BitLanes(
           DL, MVT::v32i8, V1, V2, Mask, Subtarget, DAG))
     return Result;
 
   // Try to permute the lanes and then use a per-lane permute.
-  if (SDValue V = lowerVectorShuffleAsLanePermuteAndPermute(
+  if (SDValue V = lowerShuffleAsLanePermuteAndPermute(
           DL, MVT::v32i8, V1, V2, Mask, DAG, Subtarget))
     return V;
 
   // Otherwise fall back on generic lowering.
-  return lowerVectorShuffleAsSplitOrBlend(DL, MVT::v32i8, V1, V2, Mask,
-                                          Subtarget, DAG);
+  return lowerShuffleAsSplitOrBlend(DL, MVT::v32i8, V1, V2, Mask,
+                                    Subtarget, DAG);
 }
 
 /// High-level routine to lower various 256-bit x86 vector shuffles.
@@ -15050,24 +15476,23 @@ static SDValue lowerV32I8VectorShuffle(const SDLoc &DL, ArrayRef<int> Mask,
 /// This routine either breaks down the specific type of a 256-bit x86 vector
 /// shuffle or splits it into two 128-bit shuffles and fuses the results back
 /// together based on the available instructions.
-static SDValue lower256BitVectorShuffle(const SDLoc &DL, ArrayRef<int> Mask,
-                                        MVT VT, SDValue V1, SDValue V2,
-                                        const APInt &Zeroable,
-                                        const X86Subtarget &Subtarget,
-                                        SelectionDAG &DAG) {
+static SDValue lower256BitShuffle(const SDLoc &DL, ArrayRef<int> Mask, MVT VT,
+                                  SDValue V1, SDValue V2, const APInt &Zeroable,
+                                  const X86Subtarget &Subtarget,
+                                  SelectionDAG &DAG) {
   // If we have a single input to the zero element, insert that into V1 if we
   // can do so cheaply.
   int NumElts = VT.getVectorNumElements();
   int NumV2Elements = count_if(Mask, [NumElts](int M) { return M >= NumElts; });
 
   if (NumV2Elements == 1 && Mask[0] >= NumElts)
-    if (SDValue Insertion = lowerVectorShuffleAsElementInsertion(
+    if (SDValue Insertion = lowerShuffleAsElementInsertion(
             DL, VT, V1, V2, Mask, Zeroable, Subtarget, DAG))
       return Insertion;
 
   // Handle special cases where the lower or upper half is UNDEF.
   if (SDValue V =
-          lowerVectorShuffleWithUndefHalf(DL, VT, V1, V2, Mask, Subtarget, DAG))
+          lowerShuffleWithUndefHalf(DL, VT, V1, V2, Mask, Subtarget, DAG))
     return V;
 
   // There is a really nice hard cut-over between AVX1 and AVX2 that means we
@@ -15081,12 +15506,12 @@ static SDValue lower256BitVectorShuffle(const SDLoc &DL, ArrayRef<int> Mask,
     if (ElementBits < 32) {
       // No floating point type available, if we can't use the bit operations
       // for masking/blending then decompose into 128-bit vectors.
-      if (SDValue V =
-              lowerVectorShuffleAsBitMask(DL, VT, V1, V2, Mask, Zeroable, DAG))
+      if (SDValue V = lowerShuffleAsBitMask(DL, VT, V1, V2, Mask, Zeroable,
+                                            DAG))
         return V;
-      if (SDValue V = lowerVectorShuffleAsBitBlend(DL, VT, V1, V2, Mask, DAG))
+      if (SDValue V = lowerShuffleAsBitBlend(DL, VT, V1, V2, Mask, DAG))
         return V;
-      return splitAndLowerVectorShuffle(DL, VT, V1, V2, Mask, DAG);
+      return splitAndLowerShuffle(DL, VT, V1, V2, Mask, DAG);
     }
 
     MVT FpVT = MVT::getVectorVT(MVT::getFloatingPointVT(ElementBits),
@@ -15098,17 +15523,17 @@ static SDValue lower256BitVectorShuffle(const SDLoc &DL, ArrayRef<int> Mask,
 
   switch (VT.SimpleTy) {
   case MVT::v4f64:
-    return lowerV4F64VectorShuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
+    return lowerV4F64Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
   case MVT::v4i64:
-    return lowerV4I64VectorShuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
+    return lowerV4I64Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
   case MVT::v8f32:
-    return lowerV8F32VectorShuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
+    return lowerV8F32Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
   case MVT::v8i32:
-    return lowerV8I32VectorShuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
+    return lowerV8I32Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
   case MVT::v16i16:
-    return lowerV16I16VectorShuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
+    return lowerV16I16Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
   case MVT::v32i8:
-    return lowerV32I8VectorShuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
+    return lowerV32I8Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
 
   default:
     llvm_unreachable("Not a valid 256-bit x86 vector type!");
@@ -15116,12 +15541,10 @@ static SDValue lower256BitVectorShuffle(const SDLoc &DL, ArrayRef<int> Mask,
 }
 
 /// Try to lower a vector shuffle as a 128-bit shuffles.
-static SDValue lowerV4X128VectorShuffle(const SDLoc &DL, MVT VT,
-                                        ArrayRef<int> Mask,
-                                        const APInt &Zeroable,
-                                        SDValue V1, SDValue V2,
-                                        const X86Subtarget &Subtarget,
-                                        SelectionDAG &DAG) {
+static SDValue lowerV4X128Shuffle(const SDLoc &DL, MVT VT, ArrayRef<int> Mask,
+                                  const APInt &Zeroable, SDValue V1, SDValue V2,
+                                  const X86Subtarget &Subtarget,
+                                  SelectionDAG &DAG) {
   assert(VT.getScalarSizeInBits() == 64 &&
          "Unexpected element type size for 128bit shuffle.");
 
@@ -15218,11 +15641,10 @@ static SDValue lowerV4X128VectorShuffle(const SDLoc &DL, MVT VT,
 }
 
 /// Handle lowering of 8-lane 64-bit floating point shuffles.
-static SDValue lowerV8F64VectorShuffle(const SDLoc &DL, ArrayRef<int> Mask,
-                                       const APInt &Zeroable,
-                                       SDValue V1, SDValue V2,
-                                       const X86Subtarget &Subtarget,
-                                       SelectionDAG &DAG) {
+static SDValue lowerV8F64Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
+                                 const APInt &Zeroable, SDValue V1, SDValue V2,
+                                 const X86Subtarget &Subtarget,
+                                 SelectionDAG &DAG) {
   assert(V1.getSimpleValueType() == MVT::v8f64 && "Bad operand type!");
   assert(V2.getSimpleValueType() == MVT::v8f64 && "Bad operand type!");
   assert(Mask.size() == 8 && "Unexpected mask size for v8 shuffle!");
@@ -15249,37 +15671,33 @@ static SDValue lowerV8F64VectorShuffle(const SDLoc &DL, ArrayRef<int> Mask,
                          getV4X86ShuffleImm8ForMask(RepeatedMask, DL, DAG));
   }
 
-  if (SDValue Shuf128 =
-          lowerV4X128VectorShuffle(DL, MVT::v8f64, Mask, Zeroable, V1, V2,
-                                   Subtarget, DAG))
+  if (SDValue Shuf128 = lowerV4X128Shuffle(DL, MVT::v8f64, Mask, Zeroable, V1,
+                                           V2, Subtarget, DAG))
     return Shuf128;
 
-  if (SDValue Unpck =
-          lowerVectorShuffleWithUNPCK(DL, MVT::v8f64, Mask, V1, V2, DAG))
+  if (SDValue Unpck = lowerShuffleWithUNPCK(DL, MVT::v8f64, Mask, V1, V2, DAG))
     return Unpck;
 
   // Check if the blend happens to exactly fit that of SHUFPD.
-  if (SDValue Op =
-      lowerVectorShuffleWithSHUFPD(DL, MVT::v8f64, Mask, V1, V2, DAG))
+  if (SDValue Op = lowerShuffleWithSHUFPD(DL, MVT::v8f64, Mask, V1, V2, DAG))
     return Op;
 
-  if (SDValue V = lowerVectorShuffleToEXPAND(DL, MVT::v8f64, Zeroable, Mask, V1,
-                                             V2, DAG, Subtarget))
+  if (SDValue V = lowerShuffleToEXPAND(DL, MVT::v8f64, Zeroable, Mask, V1, V2,
+                                       DAG, Subtarget))
     return V;
 
-  if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v8f64, V1, V2, Mask,
-                                                Zeroable, Subtarget, DAG))
+  if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v8f64, V1, V2, Mask,
+                                          Zeroable, Subtarget, DAG))
     return Blend;
 
-  return lowerVectorShuffleWithPERMV(DL, MVT::v8f64, Mask, V1, V2, DAG);
+  return lowerShuffleWithPERMV(DL, MVT::v8f64, Mask, V1, V2, DAG);
 }
 
 /// Handle lowering of 16-lane 32-bit floating point shuffles.
-static SDValue lowerV16F32VectorShuffle(const SDLoc &DL, ArrayRef<int> Mask,
-                                        const APInt &Zeroable,
-                                        SDValue V1, SDValue V2,
-                                        const X86Subtarget &Subtarget,
-                                        SelectionDAG &DAG) {
+static SDValue lowerV16F32Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
+                                  const APInt &Zeroable, SDValue V1, SDValue V2,
+                                  const X86Subtarget &Subtarget,
+                                  SelectionDAG &DAG) {
   assert(V1.getSimpleValueType() == MVT::v16f32 && "Bad operand type!");
   assert(V2.getSimpleValueType() == MVT::v16f32 && "Bad operand type!");
   assert(Mask.size() == 16 && "Unexpected mask size for v16 shuffle!");
@@ -15301,16 +15719,15 @@ static SDValue lowerV16F32VectorShuffle(const SDLoc &DL, ArrayRef<int> Mask,
                          getV4X86ShuffleImm8ForMask(RepeatedMask, DL, DAG));
 
     // Use dedicated unpack instructions for masks that match their pattern.
-    if (SDValue Unpck =
-            lowerVectorShuffleWithUNPCK(DL, MVT::v16f32, Mask, V1, V2, DAG))
-      return Unpck;
+    if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v16f32, Mask, V1, V2, DAG))
+      return V;
 
-    if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v16f32, V1, V2, Mask,
-                                                  Zeroable, Subtarget, DAG))
+    if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v16f32, V1, V2, Mask,
+                                            Zeroable, Subtarget, DAG))
       return Blend;
 
     // Otherwise, fall back to a SHUFPS sequence.
-    return lowerVectorShuffleWithSHUFPS(DL, MVT::v16f32, RepeatedMask, V1, V2, DAG);
+    return lowerShuffleWithSHUFPS(DL, MVT::v16f32, RepeatedMask, V1, V2, DAG);
   }
 
   // If we have a single input shuffle with different shuffle patterns in the
@@ -15322,19 +15739,18 @@ static SDValue lowerV16F32VectorShuffle(const SDLoc &DL, ArrayRef<int> Mask,
   }
 
   // If we have AVX512F support, we can use VEXPAND.
-  if (SDValue V = lowerVectorShuffleToEXPAND(DL, MVT::v16f32, Zeroable, Mask,
+  if (SDValue V = lowerShuffleToEXPAND(DL, MVT::v16f32, Zeroable, Mask,
                                              V1, V2, DAG, Subtarget))
     return V;
 
-  return lowerVectorShuffleWithPERMV(DL, MVT::v16f32, Mask, V1, V2, DAG);
+  return lowerShuffleWithPERMV(DL, MVT::v16f32, Mask, V1, V2, DAG);
 }
 
 /// Handle lowering of 8-lane 64-bit integer shuffles.
-static SDValue lowerV8I64VectorShuffle(const SDLoc &DL, ArrayRef<int> Mask,
-                                       const APInt &Zeroable,
-                                       SDValue V1, SDValue V2,
-                                       const X86Subtarget &Subtarget,
-                                       SelectionDAG &DAG) {
+static SDValue lowerV8I64Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
+                                 const APInt &Zeroable, SDValue V1, SDValue V2,
+                                 const X86Subtarget &Subtarget,
+                                 SelectionDAG &DAG) {
   assert(V1.getSimpleValueType() == MVT::v8i64 && "Bad operand type!");
   assert(V2.getSimpleValueType() == MVT::v8i64 && "Bad operand type!");
   assert(Mask.size() == 8 && "Unexpected mask size for v8 shuffle!");
@@ -15360,47 +15776,44 @@ static SDValue lowerV8I64VectorShuffle(const SDLoc &DL, ArrayRef<int> Mask,
                          getV4X86ShuffleImm8ForMask(Repeated256Mask, DL, DAG));
   }
 
-  if (SDValue Shuf128 =
-          lowerV4X128VectorShuffle(DL, MVT::v8i64, Mask, Zeroable,
-                                   V1, V2, Subtarget, DAG))
+  if (SDValue Shuf128 = lowerV4X128Shuffle(DL, MVT::v8i64, Mask, Zeroable, V1,
+                                           V2, Subtarget, DAG))
     return Shuf128;
 
   // Try to use shift instructions.
-  if (SDValue Shift = lowerVectorShuffleAsShift(DL, MVT::v8i64, V1, V2, Mask,
-                                                Zeroable, Subtarget, DAG))
+  if (SDValue Shift = lowerShuffleAsShift(DL, MVT::v8i64, V1, V2, Mask,
+                                          Zeroable, Subtarget, DAG))
     return Shift;
 
   // Try to use VALIGN.
-  if (SDValue Rotate = lowerVectorShuffleAsRotate(DL, MVT::v8i64, V1, V2,
-                                                  Mask, Subtarget, DAG))
+  if (SDValue Rotate = lowerShuffleAsRotate(DL, MVT::v8i64, V1, V2, Mask,
+                                            Subtarget, DAG))
     return Rotate;
 
   // Try to use PALIGNR.
-  if (SDValue Rotate = lowerVectorShuffleAsByteRotate(DL, MVT::v8i64, V1, V2,
-                                                      Mask, Subtarget, DAG))
+  if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v8i64, V1, V2, Mask,
+                                                Subtarget, DAG))
     return Rotate;
 
-  if (SDValue Unpck =
-          lowerVectorShuffleWithUNPCK(DL, MVT::v8i64, Mask, V1, V2, DAG))
+  if (SDValue Unpck = lowerShuffleWithUNPCK(DL, MVT::v8i64, Mask, V1, V2, DAG))
     return Unpck;
   // If we have AVX512F support, we can use VEXPAND.
-  if (SDValue V = lowerVectorShuffleToEXPAND(DL, MVT::v8i64, Zeroable, Mask, V1,
-                                             V2, DAG, Subtarget))
+  if (SDValue V = lowerShuffleToEXPAND(DL, MVT::v8i64, Zeroable, Mask, V1, V2,
+                                       DAG, Subtarget))
     return V;
 
-  if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v8i64, V1, V2, Mask,
-                                                Zeroable, Subtarget, DAG))
+  if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v8i64, V1, V2, Mask,
+                                          Zeroable, Subtarget, DAG))
     return Blend;
 
-  return lowerVectorShuffleWithPERMV(DL, MVT::v8i64, Mask, V1, V2, DAG);
+  return lowerShuffleWithPERMV(DL, MVT::v8i64, Mask, V1, V2, DAG);
 }
 
 /// Handle lowering of 16-lane 32-bit integer shuffles.
-static SDValue lowerV16I32VectorShuffle(const SDLoc &DL, ArrayRef<int> Mask,
-                                        const APInt &Zeroable,
-                                        SDValue V1, SDValue V2,
-                                        const X86Subtarget &Subtarget,
-                                        SelectionDAG &DAG) {
+static SDValue lowerV16I32Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
+                                  const APInt &Zeroable, SDValue V1, SDValue V2,
+                                  const X86Subtarget &Subtarget,
+                                  SelectionDAG &DAG) {
   assert(V1.getSimpleValueType() == MVT::v16i32 && "Bad operand type!");
   assert(V2.getSimpleValueType() == MVT::v16i32 && "Bad operand type!");
   assert(Mask.size() == 16 && "Unexpected mask size for v16 shuffle!");
@@ -15408,7 +15821,7 @@ static SDValue lowerV16I32VectorShuffle(const SDLoc &DL, ArrayRef<int> Mask,
   // Whenever we can lower this as a zext, that instruction is strictly faster
   // than any alternative. It also allows us to fold memory operands into the
   // shuffle in many cases.
-  if (SDValue ZExt = lowerVectorShuffleAsZeroOrAnyExtend(
+  if (SDValue ZExt = lowerShuffleAsZeroOrAnyExtend(
           DL, MVT::v16i32, V1, V2, Mask, Zeroable, Subtarget, DAG))
     return ZExt;
 
@@ -15425,25 +15838,24 @@ static SDValue lowerV16I32VectorShuffle(const SDLoc &DL, ArrayRef<int> Mask,
                          getV4X86ShuffleImm8ForMask(RepeatedMask, DL, DAG));
 
     // Use dedicated unpack instructions for masks that match their pattern.
-    if (SDValue V =
-            lowerVectorShuffleWithUNPCK(DL, MVT::v16i32, Mask, V1, V2, DAG))
+    if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v16i32, Mask, V1, V2, DAG))
       return V;
   }
 
   // Try to use shift instructions.
-  if (SDValue Shift = lowerVectorShuffleAsShift(DL, MVT::v16i32, V1, V2, Mask,
-                                                Zeroable, Subtarget, DAG))
+  if (SDValue Shift = lowerShuffleAsShift(DL, MVT::v16i32, V1, V2, Mask,
+                                          Zeroable, Subtarget, DAG))
     return Shift;
 
   // Try to use VALIGN.
-  if (SDValue Rotate = lowerVectorShuffleAsRotate(DL, MVT::v16i32, V1, V2,
-                                                  Mask, Subtarget, DAG))
+  if (SDValue Rotate = lowerShuffleAsRotate(DL, MVT::v16i32, V1, V2, Mask,
+                                            Subtarget, DAG))
     return Rotate;
 
   // Try to use byte rotation instructions.
   if (Subtarget.hasBWI())
-    if (SDValue Rotate = lowerVectorShuffleAsByteRotate(
-            DL, MVT::v16i32, V1, V2, Mask, Subtarget, DAG))
+    if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v16i32, V1, V2, Mask,
+                                                  Subtarget, DAG))
       return Rotate;
 
   // Assume that a single SHUFPS is faster than using a permv shuffle.
@@ -15451,27 +15863,26 @@ static SDValue lowerV16I32VectorShuffle(const SDLoc &DL, ArrayRef<int> Mask,
   if (Is128BitLaneRepeatedShuffle && isSingleSHUFPSMask(RepeatedMask)) {
     SDValue CastV1 = DAG.getBitcast(MVT::v16f32, V1);
     SDValue CastV2 = DAG.getBitcast(MVT::v16f32, V2);
-    SDValue ShufPS = lowerVectorShuffleWithSHUFPS(DL, MVT::v16f32, RepeatedMask,
-                                                  CastV1, CastV2, DAG);
+    SDValue ShufPS = lowerShuffleWithSHUFPS(DL, MVT::v16f32, RepeatedMask,
+                                            CastV1, CastV2, DAG);
     return DAG.getBitcast(MVT::v16i32, ShufPS);
   }
   // If we have AVX512F support, we can use VEXPAND.
-  if (SDValue V = lowerVectorShuffleToEXPAND(DL, MVT::v16i32, Zeroable, Mask,
-                                             V1, V2, DAG, Subtarget))
+  if (SDValue V = lowerShuffleToEXPAND(DL, MVT::v16i32, Zeroable, Mask, V1, V2,
+                                       DAG, Subtarget))
     return V;
 
-  if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v16i32, V1, V2, Mask,
-                                                Zeroable, Subtarget, DAG))
+  if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v16i32, V1, V2, Mask,
+                                          Zeroable, Subtarget, DAG))
     return Blend;
-  return lowerVectorShuffleWithPERMV(DL, MVT::v16i32, Mask, V1, V2, DAG);
+  return lowerShuffleWithPERMV(DL, MVT::v16i32, Mask, V1, V2, DAG);
 }
 
 /// Handle lowering of 32-lane 16-bit integer shuffles.
-static SDValue lowerV32I16VectorShuffle(const SDLoc &DL, ArrayRef<int> Mask,
-                                        const APInt &Zeroable,
-                                        SDValue V1, SDValue V2,
-                                        const X86Subtarget &Subtarget,
-                                        SelectionDAG &DAG) {
+static SDValue lowerV32I16Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
+                                  const APInt &Zeroable, SDValue V1, SDValue V2,
+                                  const X86Subtarget &Subtarget,
+                                  SelectionDAG &DAG) {
   assert(V1.getSimpleValueType() == MVT::v32i16 && "Bad operand type!");
   assert(V2.getSimpleValueType() == MVT::v32i16 && "Bad operand type!");
   assert(Mask.size() == 32 && "Unexpected mask size for v32 shuffle!");
@@ -15480,23 +15891,22 @@ static SDValue lowerV32I16VectorShuffle(const SDLoc &DL, ArrayRef<int> Mask,
   // Whenever we can lower this as a zext, that instruction is strictly faster
   // than any alternative. It also allows us to fold memory operands into the
   // shuffle in many cases.
-  if (SDValue ZExt = lowerVectorShuffleAsZeroOrAnyExtend(
+  if (SDValue ZExt = lowerShuffleAsZeroOrAnyExtend(
           DL, MVT::v32i16, V1, V2, Mask, Zeroable, Subtarget, DAG))
     return ZExt;
 
   // Use dedicated unpack instructions for masks that match their pattern.
-  if (SDValue V =
-          lowerVectorShuffleWithUNPCK(DL, MVT::v32i16, Mask, V1, V2, DAG))
+  if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v32i16, Mask, V1, V2, DAG))
     return V;
 
   // Try to use shift instructions.
-  if (SDValue Shift = lowerVectorShuffleAsShift(DL, MVT::v32i16, V1, V2, Mask,
-                                                Zeroable, Subtarget, DAG))
+  if (SDValue Shift = lowerShuffleAsShift(DL, MVT::v32i16, V1, V2, Mask,
+                                          Zeroable, Subtarget, DAG))
     return Shift;
 
   // Try to use byte rotation instructions.
-  if (SDValue Rotate = lowerVectorShuffleAsByteRotate(
-          DL, MVT::v32i16, V1, V2, Mask, Subtarget, DAG))
+  if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v32i16, V1, V2, Mask,
+                                                Subtarget, DAG))
     return Rotate;
 
   if (V2.isUndef()) {
@@ -15505,28 +15915,27 @@ static SDValue lowerV32I16VectorShuffle(const SDLoc &DL, ArrayRef<int> Mask,
       // As this is a single-input shuffle, the repeated mask should be
       // a strictly valid v8i16 mask that we can pass through to the v8i16
       // lowering to handle even the v32 case.
-      return lowerV8I16GeneralSingleInputVectorShuffle(
+      return lowerV8I16GeneralSingleInputShuffle(
           DL, MVT::v32i16, V1, RepeatedMask, Subtarget, DAG);
     }
   }
 
-  if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v32i16, V1, V2, Mask,
+  if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v32i16, V1, V2, Mask,
                                                 Zeroable, Subtarget, DAG))
     return Blend;
 
-  if (SDValue PSHUFB = lowerVectorShuffleWithPSHUFB(
-          DL, MVT::v32i16, Mask, V1, V2, Zeroable, Subtarget, DAG))
+  if (SDValue PSHUFB = lowerShuffleWithPSHUFB(DL, MVT::v32i16, Mask, V1, V2,
+                                              Zeroable, Subtarget, DAG))
     return PSHUFB;
 
-  return lowerVectorShuffleWithPERMV(DL, MVT::v32i16, Mask, V1, V2, DAG);
+  return lowerShuffleWithPERMV(DL, MVT::v32i16, Mask, V1, V2, DAG);
 }
 
 /// Handle lowering of 64-lane 8-bit integer shuffles.
-static SDValue lowerV64I8VectorShuffle(const SDLoc &DL, ArrayRef<int> Mask,
-                                       const APInt &Zeroable,
-                                       SDValue V1, SDValue V2,
-                                       const X86Subtarget &Subtarget,
-                                       SelectionDAG &DAG) {
+static SDValue lowerV64I8Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
+                                 const APInt &Zeroable, SDValue V1, SDValue V2,
+                                 const X86Subtarget &Subtarget,
+                                 SelectionDAG &DAG) {
   assert(V1.getSimpleValueType() == MVT::v64i8 && "Bad operand type!");
   assert(V2.getSimpleValueType() == MVT::v64i8 && "Bad operand type!");
   assert(Mask.size() == 64 && "Unexpected mask size for v64 shuffle!");
@@ -15535,37 +15944,36 @@ static SDValue lowerV64I8VectorShuffle(const SDLoc &DL, ArrayRef<int> Mask,
   // Whenever we can lower this as a zext, that instruction is strictly faster
   // than any alternative. It also allows us to fold memory operands into the
   // shuffle in many cases.
-  if (SDValue ZExt = lowerVectorShuffleAsZeroOrAnyExtend(
+  if (SDValue ZExt = lowerShuffleAsZeroOrAnyExtend(
           DL, MVT::v64i8, V1, V2, Mask, Zeroable, Subtarget, DAG))
     return ZExt;
 
   // Use dedicated unpack instructions for masks that match their pattern.
-  if (SDValue V =
-          lowerVectorShuffleWithUNPCK(DL, MVT::v64i8, Mask, V1, V2, DAG))
+  if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v64i8, Mask, V1, V2, DAG))
     return V;
 
   // Use dedicated pack instructions for masks that match their pattern.
-  if (SDValue V = lowerVectorShuffleWithPACK(DL, MVT::v64i8, Mask, V1, V2, DAG,
-                                             Subtarget))
+  if (SDValue V = lowerShuffleWithPACK(DL, MVT::v64i8, Mask, V1, V2, DAG,
+                                       Subtarget))
     return V;
 
   // Try to use shift instructions.
-  if (SDValue Shift = lowerVectorShuffleAsShift(DL, MVT::v64i8, V1, V2, Mask,
-                                                Zeroable, Subtarget, DAG))
+  if (SDValue Shift = lowerShuffleAsShift(DL, MVT::v64i8, V1, V2, Mask,
+                                          Zeroable, Subtarget, DAG))
     return Shift;
 
   // Try to use byte rotation instructions.
-  if (SDValue Rotate = lowerVectorShuffleAsByteRotate(
-          DL, MVT::v64i8, V1, V2, Mask, Subtarget, DAG))
+  if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v64i8, V1, V2, Mask,
+                                                Subtarget, DAG))
     return Rotate;
 
-  if (SDValue PSHUFB = lowerVectorShuffleWithPSHUFB(
-          DL, MVT::v64i8, Mask, V1, V2, Zeroable, Subtarget, DAG))
+  if (SDValue PSHUFB = lowerShuffleWithPSHUFB(DL, MVT::v64i8, Mask, V1, V2,
+                                              Zeroable, Subtarget, DAG))
     return PSHUFB;
 
   // VBMI can use VPERMV/VPERMV3 byte shuffles.
   if (Subtarget.hasVBMI())
-    return lowerVectorShuffleWithPERMV(DL, MVT::v64i8, Mask, V1, V2, DAG);
+    return lowerShuffleWithPERMV(DL, MVT::v64i8, Mask, V1, V2, DAG);
 
   // Try to create an in-lane repeating shuffle mask and then shuffle the
   // results into the target lanes.
@@ -15573,12 +15981,19 @@ static SDValue lowerV64I8VectorShuffle(const SDLoc &DL, ArrayRef<int> Mask,
           DL, MVT::v64i8, V1, V2, Mask, Subtarget, DAG))
     return V;
 
-  if (SDValue Blend = lowerVectorShuffleAsBlend(DL, MVT::v64i8, V1, V2, Mask,
-                                                Zeroable, Subtarget, DAG))
+  if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v64i8, V1, V2, Mask,
+                                          Zeroable, Subtarget, DAG))
     return Blend;
 
+  // Try to simplify this by merging 128-bit lanes to enable a lane-based
+  // shuffle.
+  if (!V2.isUndef())
+    if (SDValue Result = lowerShuffleByMerging128BitLanes(
+            DL, MVT::v64i8, V1, V2, Mask, Subtarget, DAG))
+      return Result;
+
   // FIXME: Implement direct support for this type!
-  return splitAndLowerVectorShuffle(DL, MVT::v64i8, V1, V2, Mask, DAG);
+  return splitAndLowerShuffle(DL, MVT::v64i8, V1, V2, Mask, DAG);
 }
 
 /// High-level routine to lower various 512-bit x86 vector shuffles.
@@ -15586,11 +16001,11 @@ static SDValue lowerV64I8VectorShuffle(const SDLoc &DL, ArrayRef<int> Mask,
 /// This routine either breaks down the specific type of a 512-bit x86 vector
 /// shuffle or splits it into two 256-bit shuffles and fuses the results back
 /// together based on the available instructions.
-static SDValue lower512BitVectorShuffle(const SDLoc &DL, ArrayRef<int> Mask,
-                                        MVT VT, SDValue V1, SDValue V2,
-                                        const APInt &Zeroable,
-                                        const X86Subtarget &Subtarget,
-                                        SelectionDAG &DAG) {
+static SDValue lower512BitShuffle(const SDLoc &DL, ArrayRef<int> Mask,
+                                  MVT VT, SDValue V1, SDValue V2,
+                                  const APInt &Zeroable,
+                                  const X86Subtarget &Subtarget,
+                                  SelectionDAG &DAG) {
   assert(Subtarget.hasAVX512() &&
          "Cannot lower 512-bit vectors w/ basic ISA!");
 
@@ -15600,18 +16015,18 @@ static SDValue lower512BitVectorShuffle(const SDLoc &DL, ArrayRef<int> Mask,
   int NumV2Elements = count_if(Mask, [NumElts](int M) { return M >= NumElts; });
 
   if (NumV2Elements == 1 && Mask[0] >= NumElts)
-    if (SDValue Insertion = lowerVectorShuffleAsElementInsertion(
+    if (SDValue Insertion = lowerShuffleAsElementInsertion(
             DL, VT, V1, V2, Mask, Zeroable, Subtarget, DAG))
       return Insertion;
 
   // Handle special cases where the lower or upper half is UNDEF.
   if (SDValue V =
-        lowerVectorShuffleWithUndefHalf(DL, VT, V1, V2, Mask, Subtarget, DAG))
+          lowerShuffleWithUndefHalf(DL, VT, V1, V2, Mask, Subtarget, DAG))
     return V;
 
   // Check for being able to broadcast a single element.
-  if (SDValue Broadcast =
-          lowerVectorShuffleAsBroadcast(DL, VT, V1, V2, Mask, Subtarget, DAG))
+  if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, VT, V1, V2, Mask,
+                                                  Subtarget, DAG))
     return Broadcast;
 
   // Dispatch to each element type for lowering. If we don't have support for
@@ -15620,17 +16035,17 @@ static SDValue lower512BitVectorShuffle(const SDLoc &DL, ArrayRef<int> Mask,
   // the requisite ISA extensions for that element type are available.
   switch (VT.SimpleTy) {
   case MVT::v8f64:
-    return lowerV8F64VectorShuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
+    return lowerV8F64Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
   case MVT::v16f32:
-    return lowerV16F32VectorShuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
+    return lowerV16F32Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
   case MVT::v8i64:
-    return lowerV8I64VectorShuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
+    return lowerV8I64Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
   case MVT::v16i32:
-    return lowerV16I32VectorShuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
+    return lowerV16I32Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
   case MVT::v32i16:
-    return lowerV32I16VectorShuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
+    return lowerV32I16Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
   case MVT::v64i8:
-    return lowerV64I8VectorShuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
+    return lowerV64I8Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
 
   default:
     llvm_unreachable("Not a valid 512-bit x86 vector type!");
@@ -15639,7 +16054,7 @@ static SDValue lower512BitVectorShuffle(const SDLoc &DL, ArrayRef<int> Mask,
 
 // Determine if this shuffle can be implemented with a KSHIFT instruction.
 // Returns the shift amount if possible or -1 if not. This is a simplified
-// version of matchVectorShuffleAsShift.
+// version of matchShuffleAsShift.
 static int match1BitShuffleAsKSHIFT(unsigned &Opcode, ArrayRef<int> Mask,
                                     int MaskOffset, const APInt &Zeroable) {
   int Size = Mask.size();
@@ -15674,11 +16089,11 @@ static int match1BitShuffleAsKSHIFT(unsigned &Opcode, ArrayRef<int> Mask,
 // There is no a dedicated instruction on AVX-512 that shuffles the masks.
 // The only way to shuffle bits is to sign-extend the mask vector to SIMD
 // vector, shuffle and then truncate it back.
-static SDValue lower1BitVectorShuffle(const SDLoc &DL, ArrayRef<int> Mask,
-                                      MVT VT, SDValue V1, SDValue V2,
-                                      const APInt &Zeroable,
-                                      const X86Subtarget &Subtarget,
-                                      SelectionDAG &DAG) {
+static SDValue lower1BitShuffle(const SDLoc &DL, ArrayRef<int> Mask,
+                                MVT VT, SDValue V1, SDValue V2,
+                                const APInt &Zeroable,
+                                const X86Subtarget &Subtarget,
+                                SelectionDAG &DAG) {
   assert(Subtarget.hasAVX512() &&
          "Cannot lower 512-bit vectors w/o basic ISA!");
 
@@ -15913,8 +16328,8 @@ static SDValue lowerVectorShuffle(SDValue Op, const X86Subtarget &Subtarget,
     // by obfuscating the operands with bitcasts.
     // TODO: Avoid lowering directly from this top-level function: make this
     // a query (canLowerAsBroadcast) and defer lowering to the type-based calls.
-    if (SDValue Broadcast =
-            lowerVectorShuffleAsBroadcast(DL, VT, V1, V2, Mask, Subtarget, DAG))
+    if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, VT, V1, V2, Mask,
+                                                    Subtarget, DAG))
       return Broadcast;
 
     MVT NewEltVT = VT.isFloatingPoint()
@@ -15952,26 +16367,21 @@ static SDValue lowerVectorShuffle(SDValue Op, const X86Subtarget &Subtarget,
   if (canonicalizeShuffleMaskWithCommute(Mask))
     return DAG.getCommutedVectorShuffle(*SVOp);
 
-  if (SDValue V =
-          lowerVectorShuffleWithVPMOV(DL, Mask, VT, V1, V2, DAG, Subtarget))
+  if (SDValue V = lowerShuffleWithVPMOV(DL, Mask, VT, V1, V2, DAG, Subtarget))
     return V;
 
   // For each vector width, delegate to a specialized lowering routine.
   if (VT.is128BitVector())
-    return lower128BitVectorShuffle(DL, Mask, VT, V1, V2, Zeroable, Subtarget,
-                                    DAG);
+    return lower128BitShuffle(DL, Mask, VT, V1, V2, Zeroable, Subtarget, DAG);
 
   if (VT.is256BitVector())
-    return lower256BitVectorShuffle(DL, Mask, VT, V1, V2, Zeroable, Subtarget,
-                                    DAG);
+    return lower256BitShuffle(DL, Mask, VT, V1, V2, Zeroable, Subtarget, DAG);
 
   if (VT.is512BitVector())
-    return lower512BitVectorShuffle(DL, Mask, VT, V1, V2, Zeroable, Subtarget,
-                                    DAG);
+    return lower512BitShuffle(DL, Mask, VT, V1, V2, Zeroable, Subtarget, DAG);
 
   if (Is1BitVector)
-    return lower1BitVectorShuffle(DL, Mask, VT, V1, V2, Zeroable, Subtarget,
-                                  DAG);
+    return lower1BitShuffle(DL, Mask, VT, V1, V2, Zeroable, Subtarget, DAG);
 
   llvm_unreachable("Unimplemented!");
 }
@@ -16149,34 +16559,25 @@ static SDValue ExtractBitFromMaskVector(SDValue Op, SelectionDAG &DAG,
   }
 
   unsigned IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
+  if (IdxVal == 0) // the operation is legal
+    return Op;
 
-  // If the kshift instructions of the correct width aren't natively supported
-  // then we need to promote the vector to the native size to get the correct
-  // zeroing behavior.
-  if (VecVT.getVectorNumElements() < 16) {
-    VecVT = MVT::v16i1;
-    Vec = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, MVT::v16i1,
-                      DAG.getUNDEF(VecVT), Vec,
+  // Extend to natively supported kshift.
+  unsigned NumElems = VecVT.getVectorNumElements();
+  MVT WideVecVT = VecVT;
+  if ((!Subtarget.hasDQI() && NumElems == 8) || NumElems < 8) {
+    WideVecVT = Subtarget.hasDQI() ? MVT::v8i1 : MVT::v16i1;
+    Vec = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, WideVecVT,
+                      DAG.getUNDEF(WideVecVT), Vec,
                       DAG.getIntPtrConstant(0, dl));
   }
 
-  // Extracts from element 0 are always allowed.
-  if (IdxVal != 0) {
-    // Use kshiftr instruction to move to the lower element.
-    Vec = DAG.getNode(X86ISD::KSHIFTR, dl, VecVT, Vec,
-                      DAG.getConstant(IdxVal, dl, MVT::i8));
-  }
-
-  // Shrink to v16i1 since that's always legal.
-  if (VecVT.getVectorNumElements() > 16) {
-    VecVT = MVT::v16i1;
-    Vec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VecVT, Vec,
-                      DAG.getIntPtrConstant(0, dl));
-  }
+  // Use kshiftr instruction to move to the lower element.
+  Vec = DAG.getNode(X86ISD::KSHIFTR, dl, WideVecVT, Vec,
+                    DAG.getConstant(IdxVal, dl, MVT::i8));
 
-  // Convert to a bitcast+aext/trunc.
-  MVT CastVT = MVT::getIntegerVT(VecVT.getVectorNumElements());
-  return DAG.getAnyExtOrTrunc(DAG.getBitcast(CastVT, Vec), dl, EltVT);
+  return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, Op.getValueType(), Vec,
+                     DAG.getIntPtrConstant(0, dl));
 }
 
 SDValue
@@ -17077,20 +17478,39 @@ static SDValue LowerFunnelShift(SDValue Op, const X86Subtarget &Subtarget,
   MVT VT = Op.getSimpleValueType();
   assert((Op.getOpcode() == ISD::FSHL || Op.getOpcode() == ISD::FSHR) &&
          "Unexpected funnel shift opcode!");
-  assert((VT == MVT::i16 || VT == MVT::i32 || VT == MVT::i64) &&
-         "Unexpected funnel shift type!");
 
   SDLoc DL(Op);
   SDValue Op0 = Op.getOperand(0);
   SDValue Op1 = Op.getOperand(1);
   SDValue Amt = Op.getOperand(2);
 
+  bool IsFSHR = Op.getOpcode() == ISD::FSHR;
+
+  if (VT.isVector()) {
+    assert(Subtarget.hasVBMI2() && "Expected VBMI2");
+
+    if (IsFSHR)
+      std::swap(Op0, Op1);
+
+    APInt APIntShiftAmt;
+    if (isConstantSplat(Amt, APIntShiftAmt)) {
+      uint64_t ShiftAmt = APIntShiftAmt.getZExtValue();
+      return DAG.getNode(IsFSHR ? X86ISD::VSHRD : X86ISD::VSHLD, DL, VT,
+                         Op0, Op1, DAG.getConstant(ShiftAmt, DL, MVT::i8));
+    }
+
+    return DAG.getNode(IsFSHR ? X86ISD::VSHRDV : X86ISD::VSHLDV, DL, VT,
+                       Op0, Op1, Amt);
+  }
+
+  assert((VT == MVT::i16 || VT == MVT::i32 || VT == MVT::i64) &&
+         "Unexpected funnel shift type!");
+
   // Expand slow SHLD/SHRD cases if we are not optimizing for size.
   bool OptForSize = DAG.getMachineFunction().getFunction().optForSize();
   if (!OptForSize && Subtarget.isSHLDSlow())
     return SDValue();
 
-  bool IsFSHR = Op.getOpcode() == ISD::FSHR;
   if (IsFSHR)
     std::swap(Op0, Op1);
 
@@ -17131,6 +17551,57 @@ static SDValue LowerI64IntToFP_AVX512DQ(SDValue Op, SelectionDAG &DAG,
                      DAG.getIntPtrConstant(0, dl));
 }
 
+static bool useVectorCast(unsigned Opcode, MVT FromVT, MVT ToVT,
+                          const X86Subtarget &Subtarget) {
+  switch (Opcode) {
+    case ISD::SINT_TO_FP:
+      // TODO: Handle wider types with AVX/AVX512.
+      if (!Subtarget.hasSSE2() || FromVT != MVT::v4i32)
+        return false;
+      // CVTDQ2PS or (V)CVTDQ2PD
+      return ToVT == MVT::v4f32 || (Subtarget.hasAVX() && ToVT == MVT::v4f64);
+
+    case ISD::UINT_TO_FP:
+      // TODO: Handle wider types and i64 elements.
+      if (!Subtarget.hasAVX512() || FromVT != MVT::v4i32)
+        return false;
+      // VCVTUDQ2PS or VCVTUDQ2PD
+      return ToVT == MVT::v4f32 || ToVT == MVT::v4f64;
+
+    default:
+      return false;
+  }
+}
+
+/// Given a scalar cast operation that is extracted from a vector, try to
+/// vectorize the cast op followed by extraction. This will avoid an expensive
+/// round-trip between XMM and GPR.
+static SDValue vectorizeExtractedCast(SDValue Cast, SelectionDAG &DAG,
+                                      const X86Subtarget &Subtarget) {
+  // TODO: The limitation for extracting from the 0-element is not required,
+  // but if we extract from some other element, it will require shuffling to
+  // get the result into the right place.
+  // TODO: This could be enhanced to handle smaller integer types by peeking
+  // through an extend.
+  SDValue Extract = Cast.getOperand(0);
+  MVT DestVT = Cast.getSimpleValueType();
+  if (Extract.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
+      !isNullConstant(Extract.getOperand(1)))
+    return SDValue();
+
+  SDValue VecOp = Extract.getOperand(0);
+  MVT FromVT = VecOp.getSimpleValueType();
+  MVT ToVT = MVT::getVectorVT(DestVT, FromVT.getVectorNumElements());
+  if (!useVectorCast(Cast.getOpcode(), FromVT, ToVT, Subtarget))
+    return SDValue();
+
+  // cast (extract V, Y) --> extract (cast V), Y
+  SDLoc DL(Cast);
+  SDValue VCast = DAG.getNode(Cast.getOpcode(), DL, ToVT, VecOp);
+  return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, DestVT, VCast,
+                     Extract.getOperand(1));
+}
+
 SDValue X86TargetLowering::LowerSINT_TO_FP(SDValue Op,
                                            SelectionDAG &DAG) const {
   SDValue Src = Op.getOperand(0);
@@ -17138,6 +17609,9 @@ SDValue X86TargetLowering::LowerSINT_TO_FP(SDValue Op,
   MVT VT = Op.getSimpleValueType();
   SDLoc dl(Op);
 
+  if (SDValue Extract = vectorizeExtractedCast(Op, DAG, Subtarget))
+    return Extract;
+
   if (SrcVT.isVector()) {
     if (SrcVT == MVT::v2i32 && VT == MVT::v2f64) {
       return DAG.getNode(X86ISD::CVTSI2P, dl, VT,
@@ -17204,7 +17678,7 @@ SDValue X86TargetLowering::BuildFILD(SDValue Op, EVT SrcVT, SDValue Chain,
     MMO = cast<LoadSDNode>(StackSlot)->getMemOperand();
     StackSlot = StackSlot.getOperand(1);
   }
-  SDValue Ops[] = { Chain, StackSlot, DAG.getValueType(SrcVT) };
+  SDValue Ops[] = { Chain, StackSlot };
   SDValue Result = DAG.getMemIntrinsicNode(useSSE ? X86ISD::FILD_FLAG :
                                            X86ISD::FILD, DL,
                                            Tys, Ops, SrcVT, MMO);
@@ -17222,9 +17696,7 @@ SDValue X86TargetLowering::BuildFILD(SDValue Op, EVT SrcVT, SDValue Chain,
     auto PtrVT = getPointerTy(MF.getDataLayout());
     SDValue StackSlot = DAG.getFrameIndex(SSFI, PtrVT);
     Tys = DAG.getVTList(MVT::Other);
-    SDValue Ops[] = {
-      Chain, Result, StackSlot, DAG.getValueType(Op.getValueType()), InFlag
-    };
+    SDValue Ops[] = { Chain, Result, StackSlot, InFlag };
     MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
         MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SSFI),
         MachineMemOperand::MOStore, SSFISize, SSFISize);
@@ -17500,6 +17972,9 @@ SDValue X86TargetLowering::LowerUINT_TO_FP(SDValue Op,
   if (Op.getSimpleValueType().isVector())
     return lowerUINT_TO_FP_vec(Op, DAG, Subtarget);
 
+  if (SDValue Extract = vectorizeExtractedCast(Op, DAG, Subtarget))
+    return Extract;
+
   MVT SrcVT = N0.getSimpleValueType();
   MVT DstVT = Op.getSimpleValueType();
 
@@ -17552,7 +18027,7 @@ SDValue X86TargetLowering::LowerUINT_TO_FP(SDValue Op,
       MachineMemOperand::MOLoad, 8, 8);
 
   SDVTList Tys = DAG.getVTList(MVT::f80, MVT::Other);
-  SDValue Ops[] = { Store, StackSlot, DAG.getValueType(MVT::i64) };
+  SDValue Ops[] = { Store, StackSlot };
   SDValue Fild = DAG.getMemIntrinsicNode(X86ISD::FILD, dl, Tys, Ops,
                                          MVT::i64, MMO);
 
@@ -17595,9 +18070,9 @@ SDValue X86TargetLowering::LowerUINT_TO_FP(SDValue Op,
 // Otherwise we lower it to a sequence ending with a FIST, return a
 // <FIST, StackSlot> pair, and the caller is responsible for loading
 // the final integer result from StackSlot.
-std::pair<SDValue,SDValue>
+SDValue
 X86TargetLowering::FP_TO_INTHelper(SDValue Op, SelectionDAG &DAG,
-                                   bool IsSigned, bool IsReplace) const {
+                                   bool IsSigned) const {
   SDLoc DL(Op);
 
   EVT DstTy = Op.getValueType();
@@ -17607,7 +18082,7 @@ X86TargetLowering::FP_TO_INTHelper(SDValue Op, SelectionDAG &DAG,
   if (TheVT != MVT::f32 && TheVT != MVT::f64 && TheVT != MVT::f80) {
     // f16 must be promoted before using the lowering in this routine.
     // fp128 does not use this lowering.
-    return std::make_pair(SDValue(), SDValue());
+    return SDValue();
   }
 
   // If using FIST to compute an unsigned i64, we'll need some fixup
@@ -17618,7 +18093,9 @@ X86TargetLowering::FP_TO_INTHelper(SDValue Op, SelectionDAG &DAG,
                        (!Subtarget.is64Bit() ||
                         !isScalarFPTypeInSSEReg(TheVT));
 
-  if (!IsSigned && DstTy != MVT::i64 && !Subtarget.hasAVX512()) {
+  if (!IsSigned && DstTy != MVT::i64) {
+    assert(!Subtarget.hasAVX512() &&
+           "AVX512 should have already been handled!");
     // Replace the fp-to-uint32 operation with an fp-to-sint64 FIST.
     // The low 32 bits of the fist result will have the correct uint32 result.
     assert(DstTy == MVT::i32 && "Unexpected FP_TO_UINT");
@@ -17630,13 +18107,10 @@ X86TargetLowering::FP_TO_INTHelper(SDValue Op, SelectionDAG &DAG,
          "Unknown FP_TO_INT to lower!");
 
   // These are really Legal.
-  if (DstTy == MVT::i32 &&
-      isScalarFPTypeInSSEReg(Op.getOperand(0).getValueType()))
-    return std::make_pair(SDValue(), SDValue());
-  if (Subtarget.is64Bit() &&
-      DstTy == MVT::i64 &&
-      isScalarFPTypeInSSEReg(Op.getOperand(0).getValueType()))
-    return std::make_pair(SDValue(), SDValue());
+  if (DstTy == MVT::i32 && isScalarFPTypeInSSEReg(TheVT))
+    return SDValue();
+  if (Subtarget.is64Bit() && DstTy == MVT::i64 && isScalarFPTypeInSSEReg(TheVT))
+    return SDValue();
 
   // We lower FP->int64 into FISTP64 followed by a load from a temporary
   // stack slot.
@@ -17645,14 +18119,6 @@ X86TargetLowering::FP_TO_INTHelper(SDValue Op, SelectionDAG &DAG,
   int SSFI = MF.getFrameInfo().CreateStackObject(MemSize, MemSize, false);
   SDValue StackSlot = DAG.getFrameIndex(SSFI, PtrVT);
 
-  unsigned Opc;
-  switch (DstTy.getSimpleVT().SimpleTy) {
-  default: llvm_unreachable("Invalid FP_TO_SINT to lower!");
-  case MVT::i16: Opc = X86ISD::FP_TO_INT16_IN_MEM; break;
-  case MVT::i32: Opc = X86ISD::FP_TO_INT32_IN_MEM; break;
-  case MVT::i64: Opc = X86ISD::FP_TO_INT64_IN_MEM; break;
-  }
-
   SDValue Chain = DAG.getEntryNode();
   SDValue Value = Op.getOperand(0);
   SDValue Adjust; // 0x0 or 0x80000000, for result sign bit adjustment.
@@ -17710,65 +18176,55 @@ X86TargetLowering::FP_TO_INTHelper(SDValue Op, SelectionDAG &DAG,
     assert(DstTy == MVT::i64 && "Invalid FP_TO_SINT to lower!");
     Chain = DAG.getStore(Chain, DL, Value, StackSlot,
                          MachinePointerInfo::getFixedStack(MF, SSFI));
-    SDVTList Tys = DAG.getVTList(Op.getOperand(0).getValueType(), MVT::Other);
-    SDValue Ops[] = {
-      Chain, StackSlot, DAG.getValueType(TheVT)
-    };
+    SDVTList Tys = DAG.getVTList(TheVT, MVT::Other);
+    SDValue Ops[] = { Chain, StackSlot };
 
+    unsigned FLDSize = TheVT.getStoreSize();
+    assert(FLDSize <= MemSize && "Stack slot not big enough");
     MachineMemOperand *MMO =
         MF.getMachineMemOperand(MachinePointerInfo::getFixedStack(MF, SSFI),
-                                MachineMemOperand::MOLoad, MemSize, MemSize);
-    Value = DAG.getMemIntrinsicNode(X86ISD::FLD, DL, Tys, Ops, DstTy, MMO);
+                                MachineMemOperand::MOLoad, FLDSize, FLDSize);
+    Value = DAG.getMemIntrinsicNode(X86ISD::FLD, DL, Tys, Ops, TheVT, MMO);
     Chain = Value.getValue(1);
     SSFI = MF.getFrameInfo().CreateStackObject(MemSize, MemSize, false);
     StackSlot = DAG.getFrameIndex(SSFI, PtrVT);
   }
 
+  // Build the FP_TO_INT*_IN_MEM
   MachineMemOperand *MMO =
       MF.getMachineMemOperand(MachinePointerInfo::getFixedStack(MF, SSFI),
                               MachineMemOperand::MOStore, MemSize, MemSize);
+  SDValue Ops[] = { Chain, Value, StackSlot };
+  SDValue FIST = DAG.getMemIntrinsicNode(X86ISD::FP_TO_INT_IN_MEM, DL,
+                                         DAG.getVTList(MVT::Other),
+                                         Ops, DstTy, MMO);
 
-  if (UnsignedFixup) {
-
-    // Insert the FIST, load its result as two i32's,
-    // and XOR the high i32 with Adjust.
+  if (!UnsignedFixup)
+    return DAG.getLoad(Op.getValueType(), SDLoc(Op), FIST, StackSlot,
+                       MachinePointerInfo());
 
-    SDValue FistOps[] = { Chain, Value, StackSlot };
-    SDValue FIST = DAG.getMemIntrinsicNode(Opc, DL, DAG.getVTList(MVT::Other),
-                                           FistOps, DstTy, MMO);
+  // Insert the FIST, load its result as two i32's,
+  // and XOR the high i32 with Adjust.
 
-    SDValue Low32 =
-        DAG.getLoad(MVT::i32, DL, FIST, StackSlot, MachinePointerInfo());
-    SDValue HighAddr = DAG.getMemBasePlusOffset(StackSlot, 4, DL);
+  SDValue Low32 =
+      DAG.getLoad(MVT::i32, DL, FIST, StackSlot, MachinePointerInfo());
+  SDValue HighAddr = DAG.getMemBasePlusOffset(StackSlot, 4, DL);
 
-    SDValue High32 =
-        DAG.getLoad(MVT::i32, DL, FIST, HighAddr, MachinePointerInfo());
-    High32 = DAG.getNode(ISD::XOR, DL, MVT::i32, High32, Adjust);
+  SDValue High32 =
+      DAG.getLoad(MVT::i32, DL, FIST, HighAddr, MachinePointerInfo());
+  High32 = DAG.getNode(ISD::XOR, DL, MVT::i32, High32, Adjust);
 
-    if (Subtarget.is64Bit()) {
-      // Join High32 and Low32 into a 64-bit result.
-      // (High32 << 32) | Low32
-      Low32 = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, Low32);
-      High32 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, High32);
-      High32 = DAG.getNode(ISD::SHL, DL, MVT::i64, High32,
-                           DAG.getConstant(32, DL, MVT::i8));
-      SDValue Result = DAG.getNode(ISD::OR, DL, MVT::i64, High32, Low32);
-      return std::make_pair(Result, SDValue());
-    }
-
-    SDValue ResultOps[] = { Low32, High32 };
-
-    SDValue pair = IsReplace
-      ? DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, ResultOps)
-      : DAG.getMergeValues(ResultOps, DL);
-    return std::make_pair(pair, SDValue());
-  } else {
-    // Build the FP_TO_INT*_IN_MEM
-    SDValue Ops[] = { Chain, Value, StackSlot };
-    SDValue FIST = DAG.getMemIntrinsicNode(Opc, DL, DAG.getVTList(MVT::Other),
-                                           Ops, DstTy, MMO);
-    return std::make_pair(FIST, StackSlot);
+  if (Subtarget.is64Bit()) {
+    // Join High32 and Low32 into a 64-bit result.
+    // (High32 << 32) | Low32
+    Low32 = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, Low32);
+    High32 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, High32);
+    High32 = DAG.getNode(ISD::SHL, DL, MVT::i64, High32,
+                         DAG.getConstant(32, DL, MVT::i8));
+    return DAG.getNode(ISD::OR, DL, MVT::i64, High32, Low32);
   }
+
+  return DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, { Low32, High32 });
 }
 
 static SDValue LowerAVXExtend(SDValue Op, SelectionDAG &DAG,
@@ -17925,9 +18381,10 @@ static SDValue truncateVectorWithPACK(unsigned Opcode, EVT DstVT, SDValue In,
                                       const X86Subtarget &Subtarget) {
   assert((Opcode == X86ISD::PACKSS || Opcode == X86ISD::PACKUS) &&
          "Unexpected PACK opcode");
+  assert(DstVT.isVector() && "VT not a vector?");
 
   // Requires SSE2 but AVX512 has fast vector truncate.
-  if (!Subtarget.hasSSE2() || Subtarget.hasAVX512() || !DstVT.isVector())
+  if (!Subtarget.hasSSE2())
     return SDValue();
 
   EVT SrcVT = In.getValueType();
@@ -18131,8 +18588,7 @@ SDValue X86TargetLowering::LowerTRUNCATE(SDValue Op, SelectionDAG &DAG) const {
   // Truncate with PACKUS if we are truncating a vector with leading zero bits
   // that extend all the way to the packed/truncated value.
   // Pre-SSE41 we can only use PACKUSWB.
-  KnownBits Known;
-  DAG.computeKnownBits(In, Known);
+  KnownBits Known = DAG.computeKnownBits(In);
   if ((InNumEltBits - NumPackedZeroBits) <= Known.countMinLeadingZeros())
     if (SDValue V =
             truncateVectorWithPACK(X86ISD::PACKUS, VT, In, DL, DAG, Subtarget))
@@ -18278,19 +18734,22 @@ SDValue X86TargetLowering::LowerFP_TO_INT(SDValue Op, SelectionDAG &DAG) const {
 
   assert(!VT.isVector());
 
-  std::pair<SDValue,SDValue> Vals = FP_TO_INTHelper(Op, DAG,
-    IsSigned, /*IsReplace=*/ false);
-  SDValue FIST = Vals.first, StackSlot = Vals.second;
-  // If FP_TO_INTHelper failed, the node is actually supposed to be Legal.
-  if (!FIST.getNode())
-    return Op;
+  if (!IsSigned && Subtarget.hasAVX512()) {
+    SDValue Src = Op.getOperand(0);
+    // Conversions from f32/f64 should be legal.
+    if (Src.getValueType() != MVT::f80)
+      return Op;
+
+    // Use default expansion.
+    if (VT == MVT::i64)
+      return SDValue();
+  }
 
-  if (StackSlot.getNode())
-    // Load the result.
-    return DAG.getLoad(VT, SDLoc(Op), FIST, StackSlot, MachinePointerInfo());
+  if (SDValue V = FP_TO_INTHelper(Op, DAG, IsSigned))
+    return V;
 
-  // The node is the result.
-  return FIST;
+  // If FP_TO_INTHelper failed, the node is actually supposed to be Legal.
+  return Op;
 }
 
 static SDValue LowerFP_EXTEND(SDValue Op, SelectionDAG &DAG) {
@@ -18306,6 +18765,98 @@ static SDValue LowerFP_EXTEND(SDValue Op, SelectionDAG &DAG) {
                                  In, DAG.getUNDEF(SVT)));
 }
 
+/// Horizontal vector math instructions may be slower than normal math with
+/// shuffles. Limit horizontal op codegen based on size/speed trade-offs, uarch
+/// implementation, and likely shuffle complexity of the alternate sequence.
+static bool shouldUseHorizontalOp(bool IsSingleSource, SelectionDAG &DAG,
+                                  const X86Subtarget &Subtarget) {
+  bool IsOptimizingSize = DAG.getMachineFunction().getFunction().optForSize();
+  bool HasFastHOps = Subtarget.hasFastHorizontalOps();
+  return !IsSingleSource || IsOptimizingSize || HasFastHOps;
+}
+
+/// Depending on uarch and/or optimizing for size, we might prefer to use a
+/// vector operation in place of the typical scalar operation.
+static SDValue lowerAddSubToHorizontalOp(SDValue Op, SelectionDAG &DAG,
+                                         const X86Subtarget &Subtarget) {
+  // If both operands have other uses, this is probably not profitable.
+  SDValue LHS = Op.getOperand(0);
+  SDValue RHS = Op.getOperand(1);
+  if (!LHS.hasOneUse() && !RHS.hasOneUse())
+    return Op;
+
+  // FP horizontal add/sub were added with SSE3. Integer with SSSE3.
+  bool IsFP = Op.getSimpleValueType().isFloatingPoint();
+  if (IsFP && !Subtarget.hasSSE3())
+    return Op;
+  if (!IsFP && !Subtarget.hasSSSE3())
+    return Op;
+
+  // Defer forming the minimal horizontal op if the vector source has more than
+  // the 2 extract element uses that we're matching here. In that case, we might
+  // form a horizontal op that includes more than 1 add/sub op.
+  if (LHS.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
+      RHS.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
+      LHS.getOperand(0) != RHS.getOperand(0) ||
+      !LHS.getOperand(0)->hasNUsesOfValue(2, 0))
+    return Op;
+
+  if (!isa<ConstantSDNode>(LHS.getOperand(1)) ||
+      !isa<ConstantSDNode>(RHS.getOperand(1)) ||
+      !shouldUseHorizontalOp(true, DAG, Subtarget))
+    return Op;
+
+  // Allow commuted 'hadd' ops.
+  // TODO: Allow commuted (f)sub by negating the result of (F)HSUB?
+  unsigned HOpcode;
+  switch (Op.getOpcode()) {
+    case ISD::ADD: HOpcode = X86ISD::HADD; break;
+    case ISD::SUB: HOpcode = X86ISD::HSUB; break;
+    case ISD::FADD: HOpcode = X86ISD::FHADD; break;
+    case ISD::FSUB: HOpcode = X86ISD::FHSUB; break;
+    default:
+      llvm_unreachable("Trying to lower unsupported opcode to horizontal op");
+  }
+  unsigned LExtIndex = LHS.getConstantOperandVal(1);
+  unsigned RExtIndex = RHS.getConstantOperandVal(1);
+  if (LExtIndex == 1 && RExtIndex == 0 &&
+      (HOpcode == X86ISD::HADD || HOpcode == X86ISD::FHADD))
+    std::swap(LExtIndex, RExtIndex);
+
+  // TODO: This can be extended to handle other adjacent extract pairs.
+  if (LExtIndex != 0 || RExtIndex != 1)
+    return Op;
+
+  SDValue X = LHS.getOperand(0);
+  EVT VecVT = X.getValueType();
+  unsigned BitWidth = VecVT.getSizeInBits();
+  assert((BitWidth == 128 || BitWidth == 256 || BitWidth == 512) &&
+         "Not expecting illegal vector widths here");
+
+  // Creating a 256-bit horizontal op would be wasteful, and there is no 512-bit
+  // equivalent, so extract the 256/512-bit source op to 128-bit.
+  // This is free: ymm/zmm -> xmm.
+  SDLoc DL(Op);
+  if (BitWidth == 256 || BitWidth == 512)
+    X = extract128BitVector(X, 0, DAG, DL);
+
+  // add (extractelt (X, 0), extractelt (X, 1)) --> extractelt (hadd X, X), 0
+  // add (extractelt (X, 1), extractelt (X, 0)) --> extractelt (hadd X, X), 0
+  // sub (extractelt (X, 0), extractelt (X, 1)) --> extractelt (hsub X, X), 0
+  SDValue HOp = DAG.getNode(HOpcode, DL, X.getValueType(), X, X);
+  return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, Op.getSimpleValueType(), HOp,
+                     DAG.getIntPtrConstant(0, DL));
+}
+
+/// Depending on uarch and/or optimizing for size, we might prefer to use a
+/// vector operation in place of the typical scalar operation.
+static SDValue lowerFaddFsub(SDValue Op, SelectionDAG &DAG,
+                             const X86Subtarget &Subtarget) {
+  assert((Op.getValueType() == MVT::f32 || Op.getValueType() == MVT::f64) &&
+         "Only expecting float/double");
+  return lowerAddSubToHorizontalOp(Op, DAG, Subtarget);
+}
+
 /// The only differences between FABS and FNEG are the mask and the logic op.
 /// FNEG also has a folding opportunity for FNEG(FABS(x)).
 static SDValue LowerFABSorFNEG(SDValue Op, SelectionDAG &DAG) {
@@ -18463,7 +19014,8 @@ static SDValue getSETCC(X86::CondCode Cond, SDValue EFLAGS, const SDLoc &dl,
 // Check whether an OR'd tree is PTEST-able.
 static SDValue LowerVectorAllZeroTest(SDValue Op, ISD::CondCode CC,
                                       const X86Subtarget &Subtarget,
-                                      SelectionDAG &DAG) {
+                                      SelectionDAG &DAG,
+                                      SDValue &X86CC) {
   assert(Op.getOpcode() == ISD::OR && "Only check OR'd tree.");
 
   if (!Subtarget.hasSSE41())
@@ -18549,9 +19101,10 @@ static SDValue LowerVectorAllZeroTest(SDValue Op, ISD::CondCode CC,
     VecIns.push_back(DAG.getNode(ISD::OR, DL, TestVT, LHS, RHS));
   }
 
-  SDValue Res = DAG.getNode(X86ISD::PTEST, DL, MVT::i32,
-                            VecIns.back(), VecIns.back());
-  return getSETCC(CC == ISD::SETEQ ? X86::COND_E : X86::COND_NE, Res, DL, DAG);
+  X86CC = DAG.getConstant(CC == ISD::SETEQ ? X86::COND_E : X86::COND_NE,
+                          DL, MVT::i8);
+  return DAG.getNode(X86ISD::PTEST, DL, MVT::i32,
+                     VecIns.back(), VecIns.back());
 }
 
 /// return true if \c Op has a use that doesn't just read flags.
@@ -18621,120 +19174,24 @@ static SDValue EmitTest(SDValue Op, unsigned X86CC, const SDLoc &dl,
 
   SDValue ArithOp = Op;
 
-  // Sometimes flags can be set either with an AND or with an SRL/SHL
-  // instruction. SRL/SHL variant should be preferred for masks longer than this
-  // number of bits.
-  const int ShiftToAndMaxMaskWidth = 32;
-  const bool ZeroCheck = (X86CC == X86::COND_E || X86CC == X86::COND_NE);
-
   // NOTICE: In the code below we use ArithOp to hold the arithmetic operation
   // which may be the result of a CAST.  We use the variable 'Op', which is the
   // non-casted variable when we check for possible users.
   switch (ArithOp.getOpcode()) {
-  case ISD::ADD:
-    // We only want to rewrite this as a target-specific node with attached
-    // flags if there is a reasonable chance of either using that to do custom
-    // instructions selection that can fold some of the memory operands, or if
-    // only the flags are used. If there are other uses, leave the node alone
-    // and emit a test instruction.
-    for (SDNode::use_iterator UI = Op.getNode()->use_begin(),
-         UE = Op.getNode()->use_end(); UI != UE; ++UI)
-      if (UI->getOpcode() != ISD::CopyToReg &&
-          UI->getOpcode() != ISD::SETCC &&
-          UI->getOpcode() != ISD::STORE)
-        goto default_case;
-
-    if (auto *C = dyn_cast<ConstantSDNode>(ArithOp.getOperand(1))) {
-      // An add of one will be selected as an INC.
-      if (C->isOne() &&
-          (!Subtarget.slowIncDec() ||
-           DAG.getMachineFunction().getFunction().optForSize())) {
-        Opcode = X86ISD::INC;
-        NumOperands = 1;
-        break;
-      }
-
-      // An add of negative one (subtract of one) will be selected as a DEC.
-      if (C->isAllOnesValue() &&
-          (!Subtarget.slowIncDec() ||
-           DAG.getMachineFunction().getFunction().optForSize())) {
-        Opcode = X86ISD::DEC;
-        NumOperands = 1;
-        break;
-      }
-    }
-
-    // Otherwise use a regular EFLAGS-setting add.
-    Opcode = X86ISD::ADD;
-    NumOperands = 2;
-    break;
-
   case ISD::AND:
     // If the primary 'and' result isn't used, don't bother using X86ISD::AND,
-    // because a TEST instruction will be better. However, AND should be
-    // preferred if the instruction can be combined into ANDN.
-    if (!hasNonFlagsUse(Op)) {
-      SDValue Op0 = ArithOp->getOperand(0);
-      SDValue Op1 = ArithOp->getOperand(1);
-      EVT VT = ArithOp.getValueType();
-      bool isAndn = isBitwiseNot(Op0) || isBitwiseNot(Op1);
-      bool isLegalAndnType = VT == MVT::i32 || VT == MVT::i64;
-      bool isProperAndn = isAndn && isLegalAndnType && Subtarget.hasBMI();
-
-      // If we cannot select an ANDN instruction, check if we can replace
-      // AND+IMM64 with a shift before giving up. This is possible for masks
-      // like 0xFF000000 or 0x00FFFFFF and if we care only about the zero flag.
-      if (!isProperAndn) {
-        if (!ZeroCheck)
-          break;
-
-        // And with cosntant should be canonicalized unless we're dealing
-        // with opaque constants.
-        assert((!isa<ConstantSDNode>(Op0) ||
-                (isa<ConstantSDNode>(Op1) &&
-                 (cast<ConstantSDNode>(Op0)->isOpaque() ||
-                  cast<ConstantSDNode>(Op1)->isOpaque()))) &&
-               "AND node isn't canonicalized");
-        auto *CN = dyn_cast<ConstantSDNode>(Op1);
-        if (!CN)
-          break;
-
-        const APInt &Mask = CN->getAPIntValue();
-        if (Mask.isSignedIntN(ShiftToAndMaxMaskWidth))
-          break; // Prefer TEST instruction.
-
-        unsigned BitWidth = Mask.getBitWidth();
-        unsigned LeadingOnes = Mask.countLeadingOnes();
-        unsigned TrailingZeros = Mask.countTrailingZeros();
-
-        if (LeadingOnes + TrailingZeros == BitWidth) {
-          assert(TrailingZeros < VT.getSizeInBits() &&
-                 "Shift amount should be less than the type width");
-          SDValue ShAmt = DAG.getConstant(TrailingZeros, dl, MVT::i8);
-          Op = DAG.getNode(ISD::SRL, dl, VT, Op0, ShAmt);
-          break;
-        }
-
-        unsigned LeadingZeros = Mask.countLeadingZeros();
-        unsigned TrailingOnes = Mask.countTrailingOnes();
-
-        if (LeadingZeros + TrailingOnes == BitWidth) {
-          assert(LeadingZeros < VT.getSizeInBits() &&
-                 "Shift amount should be less than the type width");
-          SDValue ShAmt = DAG.getConstant(LeadingZeros, dl, MVT::i8);
-          Op = DAG.getNode(ISD::SHL, dl, VT, Op0, ShAmt);
-          break;
-        }
+    // because a TEST instruction will be better.
+    if (!hasNonFlagsUse(Op))
+      break;
 
-        break;
-      }
-    }
     LLVM_FALLTHROUGH;
+  case ISD::ADD:
   case ISD::SUB:
   case ISD::OR:
   case ISD::XOR:
-    // Similar to ISD::ADD above, check if the uses will preclude useful
-    // lowering of the target-specific node.
+    // Transform to an x86-specific ALU node with flags if there is a chance of
+    // using an RMW op or only the flags are used. Otherwise, leave
+    // the node alone and emit a 'test' instruction.
     for (SDNode::use_iterator UI = Op.getNode()->use_begin(),
            UE = Op.getNode()->use_end(); UI != UE; ++UI)
       if (UI->getOpcode() != ISD::CopyToReg &&
@@ -18745,6 +19202,7 @@ static SDValue EmitTest(SDValue Op, unsigned X86CC, const SDLoc &dl,
     // Otherwise use a regular EFLAGS-setting instruction.
     switch (ArithOp.getOpcode()) {
     default: llvm_unreachable("unexpected operator!");
+    case ISD::ADD: Opcode = X86ISD::ADD; break;
     case ISD::SUB: Opcode = X86ISD::SUB; break;
     case ISD::XOR: Opcode = X86ISD::XOR; break;
     case ISD::AND: Opcode = X86ISD::AND; break;
@@ -18755,8 +19213,6 @@ static SDValue EmitTest(SDValue Op, unsigned X86CC, const SDLoc &dl,
     break;
   case X86ISD::ADD:
   case X86ISD::SUB:
-  case X86ISD::INC:
-  case X86ISD::DEC:
   case X86ISD::OR:
   case X86ISD::XOR:
   case X86ISD::AND:
@@ -18928,39 +19384,11 @@ unsigned X86TargetLowering::combineRepeatedFPDivisors() const {
   return 2;
 }
 
-/// Create a BT (Bit Test) node - Test bit \p BitNo in \p Src and set condition
-/// according to equal/not-equal condition code \p CC.
-static SDValue getBitTestCondition(SDValue Src, SDValue BitNo, ISD::CondCode CC,
-                                   const SDLoc &dl, SelectionDAG &DAG) {
-  // If Src is i8, promote it to i32 with any_extend.  There is no i8 BT
-  // instruction.  Since the shift amount is in-range-or-undefined, we know
-  // that doing a bittest on the i32 value is ok.  We extend to i32 because
-  // the encoding for the i16 version is larger than the i32 version.
-  // Also promote i16 to i32 for performance / code size reason.
-  if (Src.getValueType() == MVT::i8 || Src.getValueType() == MVT::i16)
-    Src = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, Src);
-
-  // See if we can use the 32-bit instruction instead of the 64-bit one for a
-  // shorter encoding. Since the former takes the modulo 32 of BitNo and the
-  // latter takes the modulo 64, this is only valid if the 5th bit of BitNo is
-  // known to be zero.
-  if (Src.getValueType() == MVT::i64 &&
-      DAG.MaskedValueIsZero(BitNo, APInt(BitNo.getValueSizeInBits(), 32)))
-    Src = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Src);
-
-  // If the operand types disagree, extend the shift amount to match.  Since
-  // BT ignores high bits (like shifts) we can use anyextend.
-  if (Src.getValueType() != BitNo.getValueType())
-    BitNo = DAG.getNode(ISD::ANY_EXTEND, dl, Src.getValueType(), BitNo);
-
-  SDValue BT = DAG.getNode(X86ISD::BT, dl, MVT::i32, Src, BitNo);
-  X86::CondCode Cond = CC == ISD::SETEQ ? X86::COND_AE : X86::COND_B;
-  return getSETCC(Cond, BT, dl , DAG);
-}
-
 /// Result of 'and' is compared against zero. Change to a BT node if possible.
+/// Returns the BT node and the condition code needed to use it.
 static SDValue LowerAndToBT(SDValue And, ISD::CondCode CC,
-                            const SDLoc &dl, SelectionDAG &DAG) {
+                            const SDLoc &dl, SelectionDAG &DAG,
+                            SDValue &X86CC) {
   assert(And.getOpcode() == ISD::AND && "Expected AND node!");
   SDValue Op0 = And.getOperand(0);
   SDValue Op1 = And.getOperand(1);
@@ -18969,7 +19397,7 @@ static SDValue LowerAndToBT(SDValue And, ISD::CondCode CC,
   if (Op1.getOpcode() == ISD::TRUNCATE)
     Op1 = Op1.getOperand(0);
 
-  SDValue LHS, RHS;
+  SDValue Src, BitNo;
   if (Op1.getOpcode() == ISD::SHL)
     std::swap(Op0, Op1);
   if (Op0.getOpcode() == ISD::SHL) {
@@ -18979,13 +19407,12 @@ static SDValue LowerAndToBT(SDValue And, ISD::CondCode CC,
       unsigned BitWidth = Op0.getValueSizeInBits();
       unsigned AndBitWidth = And.getValueSizeInBits();
       if (BitWidth > AndBitWidth) {
-        KnownBits Known;
-        DAG.computeKnownBits(Op0, Known);
+        KnownBits Known = DAG.computeKnownBits(Op0);
         if (Known.countMinLeadingZeros() < BitWidth - AndBitWidth)
           return SDValue();
       }
-      LHS = Op1;
-      RHS = Op0.getOperand(1);
+      Src = Op1;
+      BitNo = Op0.getOperand(1);
     }
   } else if (Op1.getOpcode() == ISD::Constant) {
     ConstantSDNode *AndRHS = cast<ConstantSDNode>(Op1);
@@ -18993,24 +19420,49 @@ static SDValue LowerAndToBT(SDValue And, ISD::CondCode CC,
     SDValue AndLHS = Op0;
 
     if (AndRHSVal == 1 && AndLHS.getOpcode() == ISD::SRL) {
-      LHS = AndLHS.getOperand(0);
-      RHS = AndLHS.getOperand(1);
+      Src = AndLHS.getOperand(0);
+      BitNo = AndLHS.getOperand(1);
     } else {
       // Use BT if the immediate can't be encoded in a TEST instruction or we
       // are optimizing for size and the immedaite won't fit in a byte.
       bool OptForSize = DAG.getMachineFunction().getFunction().optForSize();
       if ((!isUInt<32>(AndRHSVal) || (OptForSize && !isUInt<8>(AndRHSVal))) &&
           isPowerOf2_64(AndRHSVal)) {
-        LHS = AndLHS;
-        RHS = DAG.getConstant(Log2_64_Ceil(AndRHSVal), dl, LHS.getValueType());
+        Src = AndLHS;
+        BitNo = DAG.getConstant(Log2_64_Ceil(AndRHSVal), dl,
+                                Src.getValueType());
       }
     }
   }
 
-  if (LHS.getNode())
-    return getBitTestCondition(LHS, RHS, CC, dl, DAG);
+  // No patterns found, give up.
+  if (!Src.getNode())
+    return SDValue();
 
-  return SDValue();
+  // If Src is i8, promote it to i32 with any_extend.  There is no i8 BT
+  // instruction.  Since the shift amount is in-range-or-undefined, we know
+  // that doing a bittest on the i32 value is ok.  We extend to i32 because
+  // the encoding for the i16 version is larger than the i32 version.
+  // Also promote i16 to i32 for performance / code size reason.
+  if (Src.getValueType() == MVT::i8 || Src.getValueType() == MVT::i16)
+    Src = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, Src);
+
+  // See if we can use the 32-bit instruction instead of the 64-bit one for a
+  // shorter encoding. Since the former takes the modulo 32 of BitNo and the
+  // latter takes the modulo 64, this is only valid if the 5th bit of BitNo is
+  // known to be zero.
+  if (Src.getValueType() == MVT::i64 &&
+      DAG.MaskedValueIsZero(BitNo, APInt(BitNo.getValueSizeInBits(), 32)))
+    Src = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Src);
+
+  // If the operand types disagree, extend the shift amount to match.  Since
+  // BT ignores high bits (like shifts) we can use anyextend.
+  if (Src.getValueType() != BitNo.getValueType())
+    BitNo = DAG.getNode(ISD::ANY_EXTEND, dl, Src.getValueType(), BitNo);
+
+  X86CC = DAG.getConstant(CC == ISD::SETEQ ? X86::COND_AE : X86::COND_B,
+                          dl, MVT::i8);
+  return DAG.getNode(X86ISD::BT, dl, MVT::i32, Src, BitNo);
 }
 
 /// Turns an ISD::CondCode into a value suitable for SSE floating-point mask
@@ -19357,17 +19809,20 @@ static SDValue LowerVSETCC(SDValue Op, const X86Subtarget &Subtarget,
       TLI.isOperationLegal(ISD::UMIN, VT)) {
     // If we have a constant operand, increment/decrement it and change the
     // condition to avoid an invert.
-    // TODO: This could be extended to handle a non-splat constant by checking
-    // that each element of the constant is not the max/null value.
-    APInt C;
-    if (Cond == ISD::SETUGT && isConstantSplat(Op1, C) && !C.isMaxValue()) {
+    if (Cond == ISD::SETUGT &&
+        ISD::matchUnaryPredicate(Op1, [](ConstantSDNode *C) {
+          return !C->getAPIntValue().isMaxValue();
+        })) {
       // X > C --> X >= (C+1) --> X == umax(X, C+1)
-      Op1 = DAG.getConstant(C + 1, dl, VT);
+      Op1 = DAG.getNode(ISD::ADD, dl, VT, Op1, DAG.getConstant(1, dl, VT));
       Cond = ISD::SETUGE;
     }
-    if (Cond == ISD::SETULT && isConstantSplat(Op1, C) && !C.isNullValue()) {
+    if (Cond == ISD::SETULT &&
+        ISD::matchUnaryPredicate(Op1, [](ConstantSDNode *C) {
+          return !C->getAPIntValue().isNullValue();
+        })) {
       // X < C --> X <= (C-1) --> X == umin(X, C-1)
-      Op1 = DAG.getConstant(C - 1, dl, VT);
+      Op1 = DAG.getNode(ISD::SUB, dl, VT, Op1, DAG.getConstant(1, dl, VT));
       Cond = ISD::SETULE;
     }
     bool Invert = false;
@@ -19495,7 +19950,8 @@ static SDValue LowerVSETCC(SDValue Op, const X86Subtarget &Subtarget,
 // Try to select this as a KORTEST+SETCC if possible.
 static SDValue EmitKORTEST(SDValue Op0, SDValue Op1, ISD::CondCode CC,
                            const SDLoc &dl, SelectionDAG &DAG,
-                           const X86Subtarget &Subtarget) {
+                           const X86Subtarget &Subtarget,
+                           SDValue &X86CC) {
   // Only support equality comparisons.
   if (CC != ISD::SETEQ && CC != ISD::SETNE)
     return SDValue();
@@ -19511,12 +19967,12 @@ static SDValue EmitKORTEST(SDValue Op0, SDValue Op1, ISD::CondCode CC,
       !(Subtarget.hasBWI() && (VT == MVT::v32i1 || VT == MVT::v64i1)))
     return SDValue();
 
-  X86::CondCode X86CC;
+  X86::CondCode X86Cond;
   if (isNullConstant(Op1)) {
-    X86CC = CC == ISD::SETEQ ? X86::COND_E : X86::COND_NE;
+    X86Cond = CC == ISD::SETEQ ? X86::COND_E : X86::COND_NE;
   } else if (isAllOnesConstant(Op1)) {
     // C flag is set for all ones.
-    X86CC = CC == ISD::SETEQ ? X86::COND_B : X86::COND_AE;
+    X86Cond = CC == ISD::SETEQ ? X86::COND_B : X86::COND_AE;
   } else
     return SDValue();
 
@@ -19528,70 +19984,87 @@ static SDValue EmitKORTEST(SDValue Op0, SDValue Op1, ISD::CondCode CC,
     RHS = Op0.getOperand(1);
   }
 
-  SDValue KORTEST = DAG.getNode(X86ISD::KORTEST, dl, MVT::i32, LHS, RHS);
-  return getSETCC(X86CC, KORTEST, dl, DAG);
+  X86CC = DAG.getConstant(X86Cond, dl, MVT::i8);
+  return DAG.getNode(X86ISD::KORTEST, dl, MVT::i32, LHS, RHS);
 }
 
-SDValue X86TargetLowering::LowerSETCC(SDValue Op, SelectionDAG &DAG) const {
-
-  MVT VT = Op.getSimpleValueType();
-
-  if (VT.isVector()) return LowerVSETCC(Op, Subtarget, DAG);
-
-  assert(VT == MVT::i8 && "SetCC type must be 8-bit integer");
-  SDValue Op0 = Op.getOperand(0);
-  SDValue Op1 = Op.getOperand(1);
-  SDLoc dl(Op);
-  ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get();
-
+/// Emit flags for the given setcc condition and operands. Also returns the
+/// corresponding X86 condition code constant in X86CC.
+SDValue X86TargetLowering::emitFlagsForSetcc(SDValue Op0, SDValue Op1,
+                                             ISD::CondCode CC, const SDLoc &dl,
+                                             SelectionDAG &DAG,
+                                             SDValue &X86CC) const {
   // Optimize to BT if possible.
   // Lower (X & (1 << N)) == 0 to BT(X, N).
   // Lower ((X >>u N) & 1) != 0 to BT(X, N).
   // Lower ((X >>s N) & 1) != 0 to BT(X, N).
   if (Op0.getOpcode() == ISD::AND && Op0.hasOneUse() && isNullConstant(Op1) &&
       (CC == ISD::SETEQ || CC == ISD::SETNE)) {
-    if (SDValue NewSetCC = LowerAndToBT(Op0, CC, dl, DAG))
-      return NewSetCC;
+    if (SDValue BT = LowerAndToBT(Op0, CC, dl, DAG, X86CC))
+      return BT;
   }
 
   // Try to use PTEST for a tree ORs equality compared with 0.
   // TODO: We could do AND tree with all 1s as well by using the C flag.
   if (Op0.getOpcode() == ISD::OR && isNullConstant(Op1) &&
       (CC == ISD::SETEQ || CC == ISD::SETNE)) {
-    if (SDValue NewSetCC = LowerVectorAllZeroTest(Op0, CC, Subtarget, DAG))
-      return NewSetCC;
+    if (SDValue PTEST = LowerVectorAllZeroTest(Op0, CC, Subtarget, DAG, X86CC))
+      return PTEST;
   }
 
   // Try to lower using KORTEST.
-  if (SDValue NewSetCC = EmitKORTEST(Op0, Op1, CC, dl, DAG, Subtarget))
-    return NewSetCC;
+  if (SDValue KORTEST = EmitKORTEST(Op0, Op1, CC, dl, DAG, Subtarget, X86CC))
+    return KORTEST;
 
   // Look for X == 0, X == 1, X != 0, or X != 1.  We can simplify some forms of
   // these.
   if ((isOneConstant(Op1) || isNullConstant(Op1)) &&
       (CC == ISD::SETEQ || CC == ISD::SETNE)) {
-
     // If the input is a setcc, then reuse the input setcc or use a new one with
     // the inverted condition.
     if (Op0.getOpcode() == X86ISD::SETCC) {
-      X86::CondCode CCode = (X86::CondCode)Op0.getConstantOperandVal(0);
       bool Invert = (CC == ISD::SETNE) ^ isNullConstant(Op1);
-      if (!Invert)
-        return Op0;
 
-      CCode = X86::GetOppositeBranchCondition(CCode);
-      return getSETCC(CCode, Op0.getOperand(1), dl, DAG);
+      X86CC = Op0.getOperand(0);
+      if (Invert) {
+        X86::CondCode CCode = (X86::CondCode)Op0.getConstantOperandVal(0);
+        CCode = X86::GetOppositeBranchCondition(CCode);
+        X86CC = DAG.getConstant(CCode, dl, MVT::i8);
+      }
+
+      return Op0.getOperand(1);
     }
   }
 
   bool IsFP = Op1.getSimpleValueType().isFloatingPoint();
-  X86::CondCode X86CC = TranslateX86CC(CC, dl, IsFP, Op0, Op1, DAG);
-  if (X86CC == X86::COND_INVALID)
+  X86::CondCode CondCode = TranslateX86CC(CC, dl, IsFP, Op0, Op1, DAG);
+  if (CondCode == X86::COND_INVALID)
     return SDValue();
 
-  SDValue EFLAGS = EmitCmp(Op0, Op1, X86CC, dl, DAG);
+  SDValue EFLAGS = EmitCmp(Op0, Op1, CondCode, dl, DAG);
   EFLAGS = ConvertCmpIfNecessary(EFLAGS, DAG);
-  return getSETCC(X86CC, EFLAGS, dl, DAG);
+  X86CC = DAG.getConstant(CondCode, dl, MVT::i8);
+  return EFLAGS;
+}
+
+SDValue X86TargetLowering::LowerSETCC(SDValue Op, SelectionDAG &DAG) const {
+
+  MVT VT = Op.getSimpleValueType();
+
+  if (VT.isVector()) return LowerVSETCC(Op, Subtarget, DAG);
+
+  assert(VT == MVT::i8 && "SetCC type must be 8-bit integer");
+  SDValue Op0 = Op.getOperand(0);
+  SDValue Op1 = Op.getOperand(1);
+  SDLoc dl(Op);
+  ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get();
+
+  SDValue X86CC;
+  SDValue EFLAGS = emitFlagsForSetcc(Op0, Op1, CC, dl, DAG, X86CC);
+  if (!EFLAGS)
+    return SDValue();
+
+  return DAG.getNode(X86ISD::SETCC, dl, MVT::i8, X86CC, EFLAGS);
 }
 
 SDValue X86TargetLowering::LowerSETCCCARRY(SDValue Op, SelectionDAG &DAG) const {
@@ -19615,6 +20088,70 @@ SDValue X86TargetLowering::LowerSETCCCARRY(SDValue Op, SelectionDAG &DAG) const
   return getSETCC(CC, Cmp.getValue(1), DL, DAG);
 }
 
+// This function returns three things: the arithmetic computation itself
+// (Value), an EFLAGS result (Overflow), and a condition code (Cond).  The
+// flag and the condition code define the case in which the arithmetic
+// computation overflows.
+static std::pair<SDValue, SDValue>
+getX86XALUOOp(X86::CondCode &Cond, SDValue Op, SelectionDAG &DAG) {
+  assert(Op.getResNo() == 0 && "Unexpected result number!");
+  SDValue Value, Overflow;
+  SDValue LHS = Op.getOperand(0);
+  SDValue RHS = Op.getOperand(1);
+  unsigned BaseOp = 0;
+  SDLoc DL(Op);
+  switch (Op.getOpcode()) {
+  default: llvm_unreachable("Unknown ovf instruction!");
+  case ISD::SADDO:
+    BaseOp = X86ISD::ADD;
+    Cond = X86::COND_O;
+    break;
+  case ISD::UADDO:
+    BaseOp = X86ISD::ADD;
+    Cond = isOneConstant(RHS) ? X86::COND_E : X86::COND_B;
+    break;
+  case ISD::SSUBO:
+    BaseOp = X86ISD::SUB;
+    Cond = X86::COND_O;
+    break;
+  case ISD::USUBO:
+    BaseOp = X86ISD::SUB;
+    Cond = X86::COND_B;
+    break;
+  case ISD::SMULO:
+    BaseOp = X86ISD::SMUL;
+    Cond = X86::COND_O;
+    break;
+  case ISD::UMULO:
+    BaseOp = X86ISD::UMUL;
+    Cond = X86::COND_O;
+    break;
+  }
+
+  if (BaseOp) {
+    // Also sets EFLAGS.
+    SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::i32);
+    Value = DAG.getNode(BaseOp, DL, VTs, LHS, RHS);
+    Overflow = Value.getValue(1);
+  }
+
+  return std::make_pair(Value, Overflow);
+}
+
+static SDValue LowerXALUO(SDValue Op, SelectionDAG &DAG) {
+  // Lower the "add/sub/mul with overflow" instruction into a regular ins plus
+  // a "setcc" instruction that checks the overflow flag. The "brcond" lowering
+  // looks for this combo and may remove the "setcc" instruction if the "setcc"
+  // has only one use.
+  SDLoc DL(Op);
+  X86::CondCode Cond;
+  SDValue Value, Overflow;
+  std::tie(Value, Overflow) = getX86XALUOOp(Cond, Op, DAG);
+
+  SDValue SetCC = getSETCC(Cond, Overflow, DL, DAG);
+  return DAG.getNode(ISD::MERGE_VALUES, DL, Op->getVTList(), Value, SetCC);
+}
+
 /// Return true if opcode is a X86 logical comparison.
 static bool isX86LogicalCmp(SDValue Op) {
   unsigned Opc = Op.getOpcode();
@@ -19623,12 +20160,8 @@ static bool isX86LogicalCmp(SDValue Op) {
     return true;
   if (Op.getResNo() == 1 &&
       (Opc == X86ISD::ADD || Opc == X86ISD::SUB || Opc == X86ISD::ADC ||
-       Opc == X86ISD::SBB || Opc == X86ISD::SMUL ||
-       Opc == X86ISD::INC || Opc == X86ISD::DEC || Opc == X86ISD::OR ||
-       Opc == X86ISD::XOR || Opc == X86ISD::AND))
-    return true;
-
-  if (Op.getResNo() == 2 && Opc == X86ISD::UMUL)
+       Opc == X86ISD::SBB || Opc == X86ISD::SMUL || Opc == X86ISD::UMUL ||
+       Opc == X86ISD::OR || Opc == X86ISD::XOR || Opc == X86ISD::AND))
     return true;
 
   return false;
@@ -19875,34 +20408,10 @@ SDValue X86TargetLowering::LowerSELECT(SDValue Op, SelectionDAG &DAG) const {
     }
   } else if (CondOpcode == ISD::USUBO || CondOpcode == ISD::SSUBO ||
              CondOpcode == ISD::UADDO || CondOpcode == ISD::SADDO ||
-             ((CondOpcode == ISD::UMULO || CondOpcode == ISD::SMULO) &&
-              Cond.getOperand(0).getValueType() != MVT::i8)) {
-    SDValue LHS = Cond.getOperand(0);
-    SDValue RHS = Cond.getOperand(1);
-    unsigned X86Opcode;
-    unsigned X86Cond;
-    SDVTList VTs;
-    switch (CondOpcode) {
-    case ISD::UADDO: X86Opcode = X86ISD::ADD; X86Cond = X86::COND_B; break;
-    case ISD::SADDO: X86Opcode = X86ISD::ADD; X86Cond = X86::COND_O; break;
-    case ISD::USUBO: X86Opcode = X86ISD::SUB; X86Cond = X86::COND_B; break;
-    case ISD::SSUBO: X86Opcode = X86ISD::SUB; X86Cond = X86::COND_O; break;
-    case ISD::UMULO: X86Opcode = X86ISD::UMUL; X86Cond = X86::COND_O; break;
-    case ISD::SMULO: X86Opcode = X86ISD::SMUL; X86Cond = X86::COND_O; break;
-    default: llvm_unreachable("unexpected overflowing operator");
-    }
-    if (CondOpcode == ISD::UMULO)
-      VTs = DAG.getVTList(LHS.getValueType(), LHS.getValueType(),
-                          MVT::i32);
-    else
-      VTs = DAG.getVTList(LHS.getValueType(), MVT::i32);
-
-    SDValue X86Op = DAG.getNode(X86Opcode, DL, VTs, LHS, RHS);
-
-    if (CondOpcode == ISD::UMULO)
-      Cond = X86Op.getValue(2);
-    else
-      Cond = X86Op.getValue(1);
+             CondOpcode == ISD::UMULO || CondOpcode == ISD::SMULO) {
+    SDValue Value;
+    X86::CondCode X86Cond;
+    std::tie(Value, Cond) = getX86XALUOOp(X86Cond, Cond.getValue(0), DAG);
 
     CC = DAG.getConstant(X86Cond, DL, MVT::i8);
     AddTest = false;
@@ -19916,9 +20425,10 @@ SDValue X86TargetLowering::LowerSELECT(SDValue Op, SelectionDAG &DAG) const {
     // We know the result of AND is compared against zero. Try to match
     // it to BT.
     if (Cond.getOpcode() == ISD::AND && Cond.hasOneUse()) {
-      if (SDValue NewSetCC = LowerAndToBT(Cond, ISD::SETNE, DL, DAG)) {
-        CC = NewSetCC.getOperand(0);
-        Cond = NewSetCC.getOperand(1);
+      SDValue BTCC;
+      if (SDValue BT = LowerAndToBT(Cond, ISD::SETNE, DL, DAG, BTCC)) {
+        CC = BTCC;
+        Cond = BT;
         AddTest = false;
       }
     }
@@ -20240,10 +20750,11 @@ static SDValue LowerStore(SDValue Op, const X86Subtarget &Subtarget,
     assert(Subtarget.hasAVX512() && !Subtarget.hasDQI() &&
            "Expected AVX512F without AVX512DQI");
 
-    StoredVal = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, MVT::v8i1,
-                            DAG.getUNDEF(MVT::v8i1), StoredVal,
+    StoredVal = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, MVT::v16i1,
+                            DAG.getUNDEF(MVT::v16i1), StoredVal,
                             DAG.getIntPtrConstant(0, dl));
-    StoredVal = DAG.getBitcast(MVT::i8, StoredVal);
+    StoredVal = DAG.getBitcast(MVT::i16, StoredVal);
+    StoredVal = DAG.getNode(ISD::TRUNCATE, dl, MVT::i8, StoredVal);
 
     return DAG.getStore(St->getChain(), dl, StoredVal, St->getBasePtr(),
                         St->getPointerInfo(), St->getAlignment(),
@@ -20309,10 +20820,11 @@ static SDValue LowerLoad(SDValue Op, const X86Subtarget &Subtarget,
     // Replace chain users with the new chain.
     assert(NewLd->getNumValues() == 2 && "Loads must carry a chain!");
 
-    SDValue Extract = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, RegVT,
-                                  DAG.getBitcast(MVT::v8i1, NewLd),
-                                  DAG.getIntPtrConstant(0, dl));
-    return DAG.getMergeValues({Extract, NewLd.getValue(1)}, dl);
+    SDValue Val = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i16, NewLd);
+    Val = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, RegVT,
+                      DAG.getBitcast(MVT::v16i1, Val),
+                      DAG.getIntPtrConstant(0, dl));
+    return DAG.getMergeValues({Val, NewLd.getValue(1)}, dl);
   }
 
   // Nothing useful we can do without SSE2 shuffles.
@@ -20581,49 +21093,13 @@ SDValue X86TargetLowering::LowerBRCOND(SDValue Op, SelectionDAG &DAG) const {
   CondOpcode = Cond.getOpcode();
   if (CondOpcode == ISD::UADDO || CondOpcode == ISD::SADDO ||
       CondOpcode == ISD::USUBO || CondOpcode == ISD::SSUBO ||
-      ((CondOpcode == ISD::UMULO || CondOpcode == ISD::SMULO) &&
-       Cond.getOperand(0).getValueType() != MVT::i8)) {
-    SDValue LHS = Cond.getOperand(0);
-    SDValue RHS = Cond.getOperand(1);
-    unsigned X86Opcode;
-    unsigned X86Cond;
-    SDVTList VTs;
-    // Keep this in sync with LowerXALUO, otherwise we might create redundant
-    // instructions that can't be removed afterwards (i.e. X86ISD::ADD and
-    // X86ISD::INC).
-    switch (CondOpcode) {
-    case ISD::UADDO: X86Opcode = X86ISD::ADD; X86Cond = X86::COND_B; break;
-    case ISD::SADDO:
-      if (isOneConstant(RHS)) {
-          X86Opcode = X86ISD::INC; X86Cond = X86::COND_O;
-          break;
-        }
-      X86Opcode = X86ISD::ADD; X86Cond = X86::COND_O; break;
-    case ISD::USUBO: X86Opcode = X86ISD::SUB; X86Cond = X86::COND_B; break;
-    case ISD::SSUBO:
-      if (isOneConstant(RHS)) {
-          X86Opcode = X86ISD::DEC; X86Cond = X86::COND_O;
-          break;
-        }
-      X86Opcode = X86ISD::SUB; X86Cond = X86::COND_O; break;
-    case ISD::UMULO: X86Opcode = X86ISD::UMUL; X86Cond = X86::COND_O; break;
-    case ISD::SMULO: X86Opcode = X86ISD::SMUL; X86Cond = X86::COND_O; break;
-    default: llvm_unreachable("unexpected overflowing operator");
-    }
-    if (Inverted)
-      X86Cond = X86::GetOppositeBranchCondition((X86::CondCode)X86Cond);
-    if (CondOpcode == ISD::UMULO)
-      VTs = DAG.getVTList(LHS.getValueType(), LHS.getValueType(),
-                          MVT::i32);
-    else
-      VTs = DAG.getVTList(LHS.getValueType(), MVT::i32);
-
-    SDValue X86Op = DAG.getNode(X86Opcode, dl, VTs, LHS, RHS);
+      CondOpcode == ISD::UMULO || CondOpcode == ISD::SMULO) {
+    SDValue Value;
+    X86::CondCode X86Cond;
+    std::tie(Value, Cond) = getX86XALUOOp(X86Cond, Cond.getValue(0), DAG);
 
-    if (CondOpcode == ISD::UMULO)
-      Cond = X86Op.getValue(2);
-    else
-      Cond = X86Op.getValue(1);
+    if (Inverted)
+      X86Cond = X86::GetOppositeBranchCondition(X86Cond);
 
     CC = DAG.getConstant(X86Cond, dl, MVT::i8);
     addTest = false;
@@ -20724,34 +21200,17 @@ SDValue X86TargetLowering::LowerBRCOND(SDValue Op, SelectionDAG &DAG) const {
     } else if (Cond.getOpcode() == ISD::SETCC &&
                cast<CondCodeSDNode>(Cond.getOperand(2))->get() == ISD::SETUNE) {
       // For FCMP_UNE, we can emit
-      // two branches instead of an explicit AND instruction with a
-      // separate test. However, we only do this if this block doesn't
-      // have a fall-through edge, because this requires an explicit
-      // jmp when the condition is false.
-      if (Op.getNode()->hasOneUse()) {
-        SDNode *User = *Op.getNode()->use_begin();
-        // Look for an unconditional branch following this conditional branch.
-        // We need this because we need to reverse the successors in order
-        // to implement FCMP_UNE.
-        if (User->getOpcode() == ISD::BR) {
-          SDValue FalseBB = User->getOperand(1);
-          SDNode *NewBR =
-            DAG.UpdateNodeOperands(User, User->getOperand(0), Dest);
-          assert(NewBR == User);
-          (void)NewBR;
-
-          SDValue Cmp = DAG.getNode(X86ISD::CMP, dl, MVT::i32,
-                                    Cond.getOperand(0), Cond.getOperand(1));
-          Cmp = ConvertCmpIfNecessary(Cmp, DAG);
-          CC = DAG.getConstant(X86::COND_NE, dl, MVT::i8);
-          Chain = DAG.getNode(X86ISD::BRCOND, dl, Op.getValueType(),
-                              Chain, Dest, CC, Cmp);
-          CC = DAG.getConstant(X86::COND_NP, dl, MVT::i8);
-          Cond = Cmp;
-          addTest = false;
-          Dest = FalseBB;
-        }
-      }
+      // two branches instead of an explicit OR instruction with a
+      // separate test.
+      SDValue Cmp = DAG.getNode(X86ISD::CMP, dl, MVT::i32,
+                                Cond.getOperand(0), Cond.getOperand(1));
+      Cmp = ConvertCmpIfNecessary(Cmp, DAG);
+      CC = DAG.getConstant(X86::COND_NE, dl, MVT::i8);
+      Chain = DAG.getNode(X86ISD::BRCOND, dl, Op.getValueType(),
+                          Chain, Dest, CC, Cmp);
+      CC = DAG.getConstant(X86::COND_P, dl, MVT::i8);
+      Cond = Cmp;
+      addTest = false;
     }
   }
 
@@ -20763,9 +21222,10 @@ SDValue X86TargetLowering::LowerBRCOND(SDValue Op, SelectionDAG &DAG) const {
     // We know the result of AND is compared against zero. Try to match
     // it to BT.
     if (Cond.getOpcode() == ISD::AND && Cond.hasOneUse()) {
-      if (SDValue NewSetCC = LowerAndToBT(Cond, ISD::SETNE, dl, DAG)) {
-        CC = NewSetCC.getOperand(0);
-        Cond = NewSetCC.getOperand(1);
+      SDValue BTCC;
+      if (SDValue BT = LowerAndToBT(Cond, ISD::SETNE, dl, DAG, BTCC)) {
+        CC = BTCC;
+        Cond = BT;
         addTest = false;
       }
     }
@@ -21192,11 +21652,7 @@ static SDValue getMaskNode(SDValue Mask, MVT MaskVT,
   if (X86::isZeroNode(Mask))
     return DAG.getConstant(0, dl, MaskVT);
 
-  if (MaskVT.bitsGT(Mask.getSimpleValueType())) {
-    // Mask should be extended
-    Mask = DAG.getNode(ISD::ANY_EXTEND, dl,
-                       MVT::getIntegerVT(MaskVT.getSizeInBits()), Mask);
-  }
+  assert(MaskVT.bitsLE(Mask.getSimpleValueType()) && "Unexpected mask size!");
 
   if (Mask.getSimpleValueType() == MVT::i64 && Subtarget.is32Bit()) {
     assert(MaskVT == MVT::v64i1 && "Expected v64i1 mask!");
@@ -21240,24 +21696,6 @@ static SDValue getVectorMaskingNode(SDValue Op, SDValue Mask,
 
   SDValue VMask = getMaskNode(Mask, MaskVT, Subtarget, DAG, dl);
 
-  switch (Op.getOpcode()) {
-  default: break;
-  case X86ISD::CMPM:
-  case X86ISD::CMPM_RND:
-  case X86ISD::VPSHUFBITQMB:
-  case X86ISD::VFPCLASS:
-    return DAG.getNode(ISD::AND, dl, VT, Op, VMask);
-  case ISD::TRUNCATE:
-  case X86ISD::VTRUNC:
-  case X86ISD::VTRUNCS:
-  case X86ISD::VTRUNCUS:
-  case X86ISD::CVTPS2PH:
-    // We can't use ISD::VSELECT here because it is not always "Legal"
-    // for the destination type. For example vpmovqb require only AVX512
-    // and vselect that can operate on byte element type require BWI
-    OpcodeSelect = X86ISD::SELECT;
-    break;
-  }
   if (PreservedSrc.isUndef())
     PreservedSrc = getZeroVector(VT, Subtarget, DAG, dl);
   return DAG.getNode(OpcodeSelect, dl, VT, VMask, Op, PreservedSrc);
@@ -21388,13 +21826,9 @@ SDValue X86TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
       }
       return DAG.getNode(IntrData->Opc0, dl, Op.getValueType(), Op.getOperand(1));
     }
-    case INTR_TYPE_2OP:
-    case INTR_TYPE_2OP_IMM8: {
+    case INTR_TYPE_2OP: {
       SDValue Src2 = Op.getOperand(2);
 
-      if (IntrData->Type == INTR_TYPE_2OP_IMM8)
-        Src2 = DAG.getNode(ISD::TRUNCATE, dl, MVT::i8, Src2);
-
       // We specify 2 possible opcodes for intrinsics with rounding modes.
       // First, we check if the intrinsic may have non-default rounding mode,
       // (IntrData->Opc1 != 0), then we check the rounding mode operand.
@@ -21619,86 +22053,29 @@ SDValue X86TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
                                               Src1, Src2, Src3),
                                   Mask, PassThru, Subtarget, DAG);
     }
-    case VPERM_2OP : {
+    case BLENDV: {
       SDValue Src1 = Op.getOperand(1);
       SDValue Src2 = Op.getOperand(2);
+      SDValue Src3 = Op.getOperand(3);
 
-      // Swap Src1 and Src2 in the node creation
-      return DAG.getNode(IntrData->Opc0, dl, VT,Src2, Src1);
+      EVT MaskVT = Src3.getValueType().changeVectorElementTypeToInteger();
+      Src3 = DAG.getBitcast(MaskVT, Src3);
+
+      // Reverse the operands to match VSELECT order.
+      return DAG.getNode(IntrData->Opc0, dl, VT, Src3, Src2, Src1);
     }
-    case FMA_OP_MASKZ:
-    case FMA_OP_MASK: {
+    case VPERM_2OP : {
       SDValue Src1 = Op.getOperand(1);
       SDValue Src2 = Op.getOperand(2);
-      SDValue Src3 = Op.getOperand(3);
-      SDValue Mask = Op.getOperand(4);
-      MVT VT = Op.getSimpleValueType();
-      SDValue PassThru = SDValue();
-
-      // set PassThru element
-      if (IntrData->Type == FMA_OP_MASKZ)
-        PassThru = getZeroVector(VT, Subtarget, DAG, dl);
-      else
-        PassThru = Src1;
 
-      // We specify 2 possible opcodes for intrinsics with rounding modes.
-      // First, we check if the intrinsic may have non-default rounding mode,
-      // (IntrData->Opc1 != 0), then we check the rounding mode operand.
-      unsigned IntrWithRoundingModeOpcode = IntrData->Opc1;
-      if (IntrWithRoundingModeOpcode != 0) {
-        SDValue Rnd = Op.getOperand(5);
-        if (!isRoundModeCurDirection(Rnd))
-          return getVectorMaskingNode(DAG.getNode(IntrWithRoundingModeOpcode,
-                                                  dl, Op.getValueType(),
-                                                  Src1, Src2, Src3, Rnd),
-                                      Mask, PassThru, Subtarget, DAG);
-      }
-      return getVectorMaskingNode(DAG.getNode(IntrData->Opc0,
-                                              dl, Op.getValueType(),
-                                              Src1, Src2, Src3),
-                                  Mask, PassThru, Subtarget, DAG);
+      // Swap Src1 and Src2 in the node creation
+      return DAG.getNode(IntrData->Opc0, dl, VT,Src2, Src1);
     }
     case IFMA_OP:
       // NOTE: We need to swizzle the operands to pass the multiply operands
       // first.
       return DAG.getNode(IntrData->Opc0, dl, Op.getValueType(),
                          Op.getOperand(2), Op.getOperand(3), Op.getOperand(1));
-    case CVTPD2PS:
-      // ISD::FP_ROUND has a second argument that indicates if the truncation
-      // does not change the value. Set it to 0 since it can change.
-      return DAG.getNode(IntrData->Opc0, dl, VT, Op.getOperand(1),
-                         DAG.getIntPtrConstant(0, dl));
-    case CVTPD2PS_MASK: {
-      SDValue Src = Op.getOperand(1);
-      SDValue PassThru = Op.getOperand(2);
-      SDValue Mask = Op.getOperand(3);
-      // We add rounding mode to the Node when
-      //   - RM Opcode is specified and
-      //   - RM is not "current direction".
-      unsigned IntrWithRoundingModeOpcode = IntrData->Opc1;
-      if (IntrWithRoundingModeOpcode != 0) {
-        SDValue Rnd = Op.getOperand(4);
-        if (!isRoundModeCurDirection(Rnd)) {
-          return getVectorMaskingNode(DAG.getNode(IntrWithRoundingModeOpcode,
-                                      dl, Op.getValueType(),
-                                      Src, Rnd),
-                                      Mask, PassThru, Subtarget, DAG);
-        }
-      }
-      assert(IntrData->Opc0 == ISD::FP_ROUND && "Unexpected opcode!");
-      // ISD::FP_ROUND has a second argument that indicates if the truncation
-      // does not change the value. Set it to 0 since it can change.
-      return getVectorMaskingNode(DAG.getNode(IntrData->Opc0, dl, VT, Src,
-                                              DAG.getIntPtrConstant(0, dl)),
-                                  Mask, PassThru, Subtarget, DAG);
-    }
-    case FPCLASS: {
-      // FPclass intrinsics
-      SDValue Src1 = Op.getOperand(1);
-      MVT MaskVT = Op.getSimpleValueType();
-      SDValue Imm = Op.getOperand(2);
-      return DAG.getNode(IntrData->Opc0, dl, MaskVT, Src1, Imm);
-    }
     case FPCLASSS: {
       SDValue Src1 = Op.getOperand(1);
       SDValue Imm = Op.getOperand(2);
@@ -21713,32 +22090,6 @@ SDValue X86TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
                                 FPclassMask, DAG.getIntPtrConstant(0, dl));
       return DAG.getBitcast(MVT::i8, Ins);
     }
-    case CMP_MASK: {
-      // Comparison intrinsics with masks.
-      // Example of transformation:
-      // (i8 (int_x86_avx512_mask_pcmpeq_q_128
-      //             (v2i64 %a), (v2i64 %b), (i8 %mask))) ->
-      // (i8 (bitcast
-      //   (v8i1 (insert_subvector zero,
-      //           (v2i1 (and (PCMPEQM %a, %b),
-      //                      (extract_subvector
-      //                         (v8i1 (bitcast %mask)), 0))), 0))))
-      MVT VT = Op.getOperand(1).getSimpleValueType();
-      MVT MaskVT = MVT::getVectorVT(MVT::i1, VT.getVectorNumElements());
-      SDValue Mask = Op.getOperand((IntrData->Type == CMP_MASK_CC) ? 4 : 3);
-      MVT BitcastVT = MVT::getVectorVT(MVT::i1,
-                                       Mask.getSimpleValueType().getSizeInBits());
-      SDValue Cmp = DAG.getNode(IntrData->Opc0, dl, MaskVT, Op.getOperand(1),
-                                Op.getOperand(2));
-      SDValue CmpMask = getVectorMaskingNode(Cmp, Mask, SDValue(),
-                                             Subtarget, DAG);
-      // Need to fill with zeros to ensure the bitcast will produce zeroes
-      // for the upper bits in the v2i1/v4i1 case.
-      SDValue Res = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, BitcastVT,
-                                DAG.getConstant(0, dl, BitcastVT),
-                                CmpMask, DAG.getIntPtrConstant(0, dl));
-      return DAG.getBitcast(Op.getValueType(), Res);
-    }
 
     case CMP_MASK_CC: {
       MVT MaskVT = Op.getSimpleValueType();
@@ -21774,7 +22125,7 @@ SDValue X86TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
           Cmp = DAG.getNode(IntrData->Opc1, dl, MVT::v1i1, Src1, Src2, CC, Rnd);
       }
       //default rounding mode
-      if(!Cmp.getNode())
+      if (!Cmp.getNode())
         Cmp = DAG.getNode(IntrData->Opc0, dl, MVT::v1i1, Src1, Src2, CC);
 
       SDValue CmpMask = getScalarMaskingNode(Cmp, Mask, SDValue(),
@@ -21853,12 +22204,15 @@ SDValue X86TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
       SDValue Mask = Op.getOperand(3);
       SDValue DataToCompress = Op.getOperand(1);
       SDValue PassThru = Op.getOperand(2);
-      if (isAllOnesConstant(Mask)) // return data as is
+      if (ISD::isBuildVectorAllOnes(Mask.getNode())) // return data as is
         return Op.getOperand(1);
 
-      return getVectorMaskingNode(DAG.getNode(IntrData->Opc0, dl, VT,
-                                              DataToCompress),
-                                  Mask, PassThru, Subtarget, DAG);
+      // Avoid false dependency.
+      if (PassThru.isUndef())
+        PassThru = DAG.getConstant(0, dl, VT);
+
+      return DAG.getNode(IntrData->Opc0, dl, VT, DataToCompress, PassThru,
+                         Mask);
     }
     case FIXUPIMMS:
     case FIXUPIMMS_MASKZ:
@@ -21930,6 +22284,39 @@ SDValue X86TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
       SDValue Results[] = { SetCC, Res };
       return DAG.getMergeValues(Results, dl);
     }
+    case CVTPD2PS_MASK:
+    case CVTPD2DQ_MASK:
+    case CVTQQ2PS_MASK:
+    case TRUNCATE_TO_REG: {
+      SDValue Src = Op.getOperand(1);
+      SDValue PassThru = Op.getOperand(2);
+      SDValue Mask = Op.getOperand(3);
+
+      if (isAllOnesConstant(Mask))
+        return DAG.getNode(IntrData->Opc0, dl, Op.getValueType(), Src);
+
+      MVT SrcVT = Src.getSimpleValueType();
+      MVT MaskVT = MVT::getVectorVT(MVT::i1, SrcVT.getVectorNumElements());
+      Mask = getMaskNode(Mask, MaskVT, Subtarget, DAG, dl);
+      return DAG.getNode(IntrData->Opc1, dl, Op.getValueType(), Src, PassThru,
+                         Mask);
+    }
+    case CVTPS2PH_MASK: {
+      SDValue Src = Op.getOperand(1);
+      SDValue Rnd = Op.getOperand(2);
+      SDValue PassThru = Op.getOperand(3);
+      SDValue Mask = Op.getOperand(4);
+
+      if (isAllOnesConstant(Mask))
+        return DAG.getNode(IntrData->Opc0, dl, Op.getValueType(), Src, Rnd);
+
+      MVT SrcVT = Src.getSimpleValueType();
+      MVT MaskVT = MVT::getVectorVT(MVT::i1, SrcVT.getVectorNumElements());
+      Mask = getMaskNode(Mask, MaskVT, Subtarget, DAG, dl);
+      return DAG.getNode(IntrData->Opc1, dl, Op.getValueType(), Src, Rnd,
+                         PassThru, Mask);
+
+    }
     default:
       break;
     }
@@ -22141,14 +22528,14 @@ SDValue X86TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
     return DAG.getNode(X86ISD::Wrapper, dl, VT, Result);
   }
 
-  case Intrinsic::x86_seh_recoverfp: {
+  case Intrinsic::eh_recoverfp: {
     SDValue FnOp = Op.getOperand(1);
     SDValue IncomingFPOp = Op.getOperand(2);
     GlobalAddressSDNode *GSD = dyn_cast<GlobalAddressSDNode>(FnOp);
     auto *Fn = dyn_cast_or_null<Function>(GSD ? GSD->getGlobal() : nullptr);
     if (!Fn)
       report_fatal_error(
-          "llvm.x86.seh.recoverfp must take a function as the first argument");
+          "llvm.eh.recoverfp must take a function as the first argument");
     return recoverFramePointer(DAG, Fn, IncomingFPOp);
   }
 
@@ -22160,8 +22547,13 @@ SDValue X86TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
     unsigned Reg;
     if (RegInfo->hasBasePointer(MF))
       Reg = RegInfo->getBaseRegister();
-    else // This function handles the SP or FP case.
-      Reg = RegInfo->getPtrSizedFrameRegister(MF);
+    else { // Handles the SP or FP case.
+      bool CantUseFP = RegInfo->needsStackRealignment(MF);
+      if (CantUseFP)
+        Reg = RegInfo->getPtrSizedStackRegister(MF);
+      else
+        Reg = RegInfo->getPtrSizedFrameRegister(MF);
+    }
     return DAG.getCopyFromReg(DAG.getEntryNode(), dl, Reg, VT);
   }
   }
@@ -22177,47 +22569,55 @@ static SDValue getAVX2GatherNode(unsigned Opc, SDValue Op, SelectionDAG &DAG,
   if (!C)
     return SDValue();
   SDValue Scale = DAG.getTargetConstant(C->getZExtValue(), dl, MVT::i8);
-  EVT MaskVT = Mask.getValueType();
+  EVT MaskVT = Mask.getValueType().changeVectorElementTypeToInteger();
   SDVTList VTs = DAG.getVTList(Op.getValueType(), MaskVT, MVT::Other);
-  SDValue Disp = DAG.getTargetConstant(0, dl, MVT::i32);
-  SDValue Segment = DAG.getRegister(0, MVT::i32);
   // If source is undef or we know it won't be used, use a zero vector
   // to break register dependency.
   // TODO: use undef instead and let BreakFalseDeps deal with it?
   if (Src.isUndef() || ISD::isBuildVectorAllOnes(Mask.getNode()))
     Src = getZeroVector(Op.getSimpleValueType(), Subtarget, DAG, dl);
-  SDValue Ops[] = {Src, Base, Scale, Index, Disp, Segment, Mask, Chain};
-  SDNode *Res = DAG.getMachineNode(Opc, dl, VTs, Ops);
-  SDValue RetOps[] = { SDValue(Res, 0), SDValue(Res, 2) };
-  return DAG.getMergeValues(RetOps, dl);
+
+  MemIntrinsicSDNode *MemIntr = cast<MemIntrinsicSDNode>(Op);
+
+  SDValue Ops[] = {Chain, Src, Mask, Base, Index, Scale };
+  SDValue Res = DAG.getTargetMemSDNode<X86MaskedGatherSDNode>(
+    VTs, Ops, dl, MemIntr->getMemoryVT(), MemIntr->getMemOperand());
+  return DAG.getMergeValues({ Res, Res.getValue(2) }, dl);
 }
 
-static SDValue getGatherNode(unsigned Opc, SDValue Op, SelectionDAG &DAG,
-                              SDValue Src, SDValue Mask, SDValue Base,
-                              SDValue Index, SDValue ScaleOp, SDValue Chain,
-                              const X86Subtarget &Subtarget) {
+static SDValue getGatherNode(SDValue Op, SelectionDAG &DAG,
+                             SDValue Src, SDValue Mask, SDValue Base,
+                             SDValue Index, SDValue ScaleOp, SDValue Chain,
+                             const X86Subtarget &Subtarget) {
+  MVT VT = Op.getSimpleValueType();
   SDLoc dl(Op);
   auto *C = dyn_cast<ConstantSDNode>(ScaleOp);
   // Scale must be constant.
   if (!C)
     return SDValue();
   SDValue Scale = DAG.getTargetConstant(C->getZExtValue(), dl, MVT::i8);
-  MVT MaskVT = MVT::getVectorVT(MVT::i1,
-                             Index.getSimpleValueType().getVectorNumElements());
+  unsigned MinElts = std::min(Index.getSimpleValueType().getVectorNumElements(),
+                              VT.getVectorNumElements());
+  MVT MaskVT = MVT::getVectorVT(MVT::i1, MinElts);
+
+  // We support two versions of the gather intrinsics. One with scalar mask and
+  // one with vXi1 mask. Convert scalar to vXi1 if necessary.
+  if (Mask.getValueType() != MaskVT)
+    Mask = getMaskNode(Mask, MaskVT, Subtarget, DAG, dl);
 
-  SDValue VMask = getMaskNode(Mask, MaskVT, Subtarget, DAG, dl);
   SDVTList VTs = DAG.getVTList(Op.getValueType(), MaskVT, MVT::Other);
-  SDValue Disp = DAG.getTargetConstant(0, dl, MVT::i32);
-  SDValue Segment = DAG.getRegister(0, MVT::i32);
   // If source is undef or we know it won't be used, use a zero vector
   // to break register dependency.
   // TODO: use undef instead and let BreakFalseDeps deal with it?
-  if (Src.isUndef() || ISD::isBuildVectorAllOnes(VMask.getNode()))
+  if (Src.isUndef() || ISD::isBuildVectorAllOnes(Mask.getNode()))
     Src = getZeroVector(Op.getSimpleValueType(), Subtarget, DAG, dl);
-  SDValue Ops[] = {Src, VMask, Base, Scale, Index, Disp, Segment, Chain};
-  SDNode *Res = DAG.getMachineNode(Opc, dl, VTs, Ops);
-  SDValue RetOps[] = { SDValue(Res, 0), SDValue(Res, 2) };
-  return DAG.getMergeValues(RetOps, dl);
+
+  MemIntrinsicSDNode *MemIntr = cast<MemIntrinsicSDNode>(Op);
+
+  SDValue Ops[] = {Chain, Src, Mask, Base, Index, Scale };
+  SDValue Res = DAG.getTargetMemSDNode<X86MaskedGatherSDNode>(
+    VTs, Ops, dl, MemIntr->getMemoryVT(), MemIntr->getMemOperand());
+  return DAG.getMergeValues({ Res, Res.getValue(2) }, dl);
 }
 
 static SDValue getScatterNode(unsigned Opc, SDValue Op, SelectionDAG &DAG,
@@ -22230,16 +22630,22 @@ static SDValue getScatterNode(unsigned Opc, SDValue Op, SelectionDAG &DAG,
   if (!C)
     return SDValue();
   SDValue Scale = DAG.getTargetConstant(C->getZExtValue(), dl, MVT::i8);
-  SDValue Disp = DAG.getTargetConstant(0, dl, MVT::i32);
-  SDValue Segment = DAG.getRegister(0, MVT::i32);
-  MVT MaskVT = MVT::getVectorVT(MVT::i1,
-                             Index.getSimpleValueType().getVectorNumElements());
+  unsigned MinElts = std::min(Index.getSimpleValueType().getVectorNumElements(),
+                              Src.getSimpleValueType().getVectorNumElements());
+  MVT MaskVT = MVT::getVectorVT(MVT::i1, MinElts);
+
+  // We support two versions of the scatter intrinsics. One with scalar mask and
+  // one with vXi1 mask. Convert scalar to vXi1 if necessary.
+  if (Mask.getValueType() != MaskVT)
+    Mask = getMaskNode(Mask, MaskVT, Subtarget, DAG, dl);
+
+  MemIntrinsicSDNode *MemIntr = cast<MemIntrinsicSDNode>(Op);
 
-  SDValue VMask = getMaskNode(Mask, MaskVT, Subtarget, DAG, dl);
   SDVTList VTs = DAG.getVTList(MaskVT, MVT::Other);
-  SDValue Ops[] = {Base, Scale, Index, Disp, Segment, VMask, Src, Chain};
-  SDNode *Res = DAG.getMachineNode(Opc, dl, VTs, Ops);
-  return SDValue(Res, 1);
+  SDValue Ops[] = {Chain, Src, Mask, Base, Index, Scale};
+  SDValue Res = DAG.getTargetMemSDNode<X86MaskedScatterSDNode>(
+      VTs, Ops, dl, MemIntr->getMemoryVT(), MemIntr->getMemOperand());
+  return Res.getValue(1);
 }
 
 static SDValue getPrefetchNode(unsigned Opc, SDValue Op, SelectionDAG &DAG,
@@ -22577,7 +22983,7 @@ static SDValue LowerINTRINSIC_W_CHAIN(SDValue Op, const X86Subtarget &Subtarget,
     SDValue Index = Op.getOperand(4);
     SDValue Mask  = Op.getOperand(5);
     SDValue Scale = Op.getOperand(6);
-    return getGatherNode(IntrData->Opc0, Op, DAG, Src, Mask, Base, Index, Scale,
+    return getGatherNode(Op, DAG, Src, Mask, Base, Index, Scale,
                          Chain, Subtarget);
   }
   case SCATTER: {
@@ -23393,11 +23799,16 @@ static SDValue split512IntArith(SDValue Op, SelectionDAG &DAG) {
                      DAG.getNode(Op.getOpcode(), dl, NewVT, LHS2, RHS2));
 }
 
-static SDValue LowerADD_SUB(SDValue Op, SelectionDAG &DAG) {
+static SDValue lowerAddSub(SDValue Op, SelectionDAG &DAG,
+                           const X86Subtarget &Subtarget) {
   MVT VT = Op.getSimpleValueType();
+  if (VT == MVT::i16 || VT == MVT::i32)
+    return lowerAddSubToHorizontalOp(Op, DAG, Subtarget);
+
   if (VT.getScalarType() == MVT::i1)
     return DAG.getNode(ISD::XOR, SDLoc(Op), VT,
                        Op.getOperand(0), Op.getOperand(1));
+
   assert(Op.getSimpleValueType().is256BitVector() &&
          Op.getSimpleValueType().isInteger() &&
          "Only handle AVX 256-bit vector integer operation");
@@ -23426,7 +23837,8 @@ static SDValue LowerADDSAT_SUBSAT(SDValue Op, SelectionDAG &DAG) {
   return split256IntArith(Op, DAG);
 }
 
-static SDValue LowerABS(SDValue Op, SelectionDAG &DAG) {
+static SDValue LowerABS(SDValue Op, const X86Subtarget &Subtarget,
+                        SelectionDAG &DAG) {
   MVT VT = Op.getSimpleValueType();
   if (VT == MVT::i16 || VT == MVT::i32 || VT == MVT::i64) {
     // Since X86 does not have CMOV for 8-bit integer, we don't convert
@@ -23440,10 +23852,23 @@ static SDValue LowerABS(SDValue Op, SelectionDAG &DAG) {
     return DAG.getNode(X86ISD::CMOV, DL, VT, Ops);
   }
 
-  assert(Op.getSimpleValueType().is256BitVector() &&
-         Op.getSimpleValueType().isInteger() &&
-         "Only handle AVX 256-bit vector integer operation");
-  return Lower256IntUnary(Op, DAG);
+  // ABS(vXi64 X) --> VPBLENDVPD(X, 0-X, X).
+  if ((VT == MVT::v2i64 || VT == MVT::v4i64) && Subtarget.hasSSE41()) {
+    SDLoc DL(Op);
+    SDValue Src = Op.getOperand(0);
+    SDValue Sub =
+        DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), Src);
+    return DAG.getNode(X86ISD::BLENDV, DL, VT, Src, Sub, Src);
+  }
+
+  if (VT.is256BitVector() && !Subtarget.hasInt256()) {
+    assert(VT.isInteger() &&
+           "Only handle AVX 256-bit vector integer operation");
+    return Lower256IntUnary(Op, DAG);
+  }
+
+  // Default to expand.
+  return SDValue();
 }
 
 static SDValue LowerMINMAX(SDValue Op, SelectionDAG &DAG) {
@@ -23594,9 +24019,8 @@ static SDValue LowerMUL(SDValue Op, const X86Subtarget &Subtarget,
   //
   //  Hi = psllqi(AloBhi + AhiBlo, 32);
   //  return AloBlo + Hi;
-  KnownBits AKnown, BKnown;
-  DAG.computeKnownBits(A, AKnown);
-  DAG.computeKnownBits(B, BKnown);
+  KnownBits AKnown = DAG.computeKnownBits(A);
+  KnownBits BKnown = DAG.computeKnownBits(B);
 
   APInt LowerBitsMask = APInt::getLowBitsSet(64, 32);
   bool ALoIsZero = LowerBitsMask.isSubsetOf(AKnown.Zero);
@@ -24946,78 +25370,6 @@ static SDValue LowerRotate(SDValue Op, const X86Subtarget &Subtarget,
                      DAG.getVectorShuffle(VT, DL, Res02, Res13, {1, 5, 3, 7}));
 }
 
-static SDValue LowerXALUO(SDValue Op, SelectionDAG &DAG) {
-  // Lower the "add/sub/mul with overflow" instruction into a regular ins plus
-  // a "setcc" instruction that checks the overflow flag. The "brcond" lowering
-  // looks for this combo and may remove the "setcc" instruction if the "setcc"
-  // has only one use.
-  SDNode *N = Op.getNode();
-  SDValue LHS = N->getOperand(0);
-  SDValue RHS = N->getOperand(1);
-  unsigned BaseOp = 0;
-  X86::CondCode Cond;
-  SDLoc DL(Op);
-  switch (Op.getOpcode()) {
-  default: llvm_unreachable("Unknown ovf instruction!");
-  case ISD::SADDO:
-    // A subtract of one will be selected as a INC. Note that INC doesn't
-    // set CF, so we can't do this for UADDO.
-    if (isOneConstant(RHS)) {
-      BaseOp = X86ISD::INC;
-      Cond = X86::COND_O;
-      break;
-    }
-    BaseOp = X86ISD::ADD;
-    Cond = X86::COND_O;
-    break;
-  case ISD::UADDO:
-    BaseOp = X86ISD::ADD;
-    Cond = X86::COND_B;
-    break;
-  case ISD::SSUBO:
-    // A subtract of one will be selected as a DEC. Note that DEC doesn't
-    // set CF, so we can't do this for USUBO.
-    if (isOneConstant(RHS)) {
-      BaseOp = X86ISD::DEC;
-      Cond = X86::COND_O;
-      break;
-    }
-    BaseOp = X86ISD::SUB;
-    Cond = X86::COND_O;
-    break;
-  case ISD::USUBO:
-    BaseOp = X86ISD::SUB;
-    Cond = X86::COND_B;
-    break;
-  case ISD::SMULO:
-    BaseOp = N->getValueType(0) == MVT::i8 ? X86ISD::SMUL8 : X86ISD::SMUL;
-    Cond = X86::COND_O;
-    break;
-  case ISD::UMULO: { // i64, i8 = umulo lhs, rhs --> i64, i64, i32 umul lhs,rhs
-    if (N->getValueType(0) == MVT::i8) {
-      BaseOp = X86ISD::UMUL8;
-      Cond = X86::COND_O;
-      break;
-    }
-    SDVTList VTs = DAG.getVTList(N->getValueType(0), N->getValueType(0),
-                                 MVT::i32);
-    SDValue Sum = DAG.getNode(X86ISD::UMUL, DL, VTs, LHS, RHS);
-
-    SDValue SetCC = getSETCC(X86::COND_O, SDValue(Sum.getNode(), 2), DL, DAG);
-
-    return DAG.getNode(ISD::MERGE_VALUES, DL, N->getVTList(), Sum, SetCC);
-  }
-  }
-
-  // Also sets EFLAGS.
-  SDVTList VTs = DAG.getVTList(N->getValueType(0), MVT::i32);
-  SDValue Sum = DAG.getNode(BaseOp, DL, VTs, LHS, RHS);
-
-  SDValue SetCC = getSETCC(Cond, SDValue(Sum.getNode(), 1), DL, DAG);
-
-  return DAG.getNode(ISD::MERGE_VALUES, DL, N->getVTList(), Sum, SetCC);
-}
-
 /// Returns true if the operand type is exactly twice the native width, and
 /// the corresponding cmpxchg8b or cmpxchg16b instruction is available.
 /// Used to know whether to use cmpxchg8/16b when expanding atomic operations
@@ -25079,6 +25431,8 @@ X86TargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const {
   case AtomicRMWInst::Min:
   case AtomicRMWInst::UMax:
   case AtomicRMWInst::UMin:
+  case AtomicRMWInst::FAdd:
+  case AtomicRMWInst::FSub:
     // These always require a non-trivial set of data operations on x86. We must
     // use a cmpxchg loop.
     return AtomicExpansionKind::CmpXChg;
@@ -25101,7 +25455,6 @@ X86TargetLowering::lowerIdempotentRMWIntoFencedLoad(AtomicRMWInst *AI) const {
   // We must restrict the ordering to avoid generating loads with Release or
   // ReleaseAcquire orderings.
   auto Order = AtomicCmpXchgInst::getStrongestFailureOrdering(AI->getOrdering());
-  auto Ptr = AI->getPointerOperand();
 
   // Before the load we need a fence. Here is an example lifted from
   // http://www.hpl.hp.com/techreports/2012/HPL-2012-68.pdf showing why a fence
@@ -25136,8 +25489,9 @@ X86TargetLowering::lowerIdempotentRMWIntoFencedLoad(AtomicRMWInst *AI) const {
   Builder.CreateCall(MFence, {});
 
   // Finally we can emit the atomic load.
-  LoadInst *Loaded = Builder.CreateAlignedLoad(Ptr,
-          AI->getType()->getPrimitiveSizeInBits());
+  LoadInst *Loaded =
+      Builder.CreateAlignedLoad(AI->getType(), AI->getPointerOperand(),
+                                AI->getType()->getPrimitiveSizeInBits());
   Loaded->setAtomic(Order, SSID);
   AI->replaceAllUsesWith(Loaded);
   AI->eraseFromParent();
@@ -25160,7 +25514,7 @@ static SDValue LowerATOMIC_FENCE(SDValue Op, const X86Subtarget &Subtarget,
       return DAG.getNode(X86ISD::MFENCE, dl, MVT::Other, Op.getOperand(0));
 
     SDValue Chain = Op.getOperand(0);
-    SDValue Zero = DAG.getConstant(0, dl, MVT::i32);
+    SDValue Zero = DAG.getTargetConstant(0, dl, MVT::i32);
     SDValue Ops[] = {
       DAG.getRegister(X86::ESP, MVT::i32),     // Base
       DAG.getTargetConstant(1, dl, MVT::i8),   // Scale
@@ -25170,7 +25524,7 @@ static SDValue LowerATOMIC_FENCE(SDValue Op, const X86Subtarget &Subtarget,
       Zero,
       Chain
     };
-    SDNode *Res = DAG.getMachineNode(X86::OR32mrLocked, dl, MVT::Other, Ops);
+    SDNode *Res = DAG.getMachineNode(X86::OR32mi8Locked, dl, MVT::Other, Ops);
     return SDValue(Res, 0);
   }
 
@@ -25594,8 +25948,7 @@ static SDValue LowerBITREVERSE(SDValue Op, const X86Subtarget &Subtarget,
 }
 
 static SDValue lowerAtomicArithWithLOCK(SDValue N, SelectionDAG &DAG,
-                                        const X86Subtarget &Subtarget,
-                                        bool AllowIncDec = true) {
+                                        const X86Subtarget &Subtarget) {
   unsigned NewOpc = 0;
   switch (N->getOpcode()) {
   case ISD::ATOMIC_LOAD_ADD:
@@ -25619,25 +25972,6 @@ static SDValue lowerAtomicArithWithLOCK(SDValue N, SelectionDAG &DAG,
 
   MachineMemOperand *MMO = cast<MemSDNode>(N)->getMemOperand();
 
-  if (auto *C = dyn_cast<ConstantSDNode>(N->getOperand(2))) {
-    // Convert to inc/dec if they aren't slow or we are optimizing for size.
-    if (AllowIncDec && (!Subtarget.slowIncDec() ||
-                        DAG.getMachineFunction().getFunction().optForSize())) {
-      if ((NewOpc == X86ISD::LADD && C->isOne()) ||
-          (NewOpc == X86ISD::LSUB && C->isAllOnesValue()))
-        return DAG.getMemIntrinsicNode(X86ISD::LINC, SDLoc(N),
-                                       DAG.getVTList(MVT::i32, MVT::Other),
-                                       {N->getOperand(0), N->getOperand(1)},
-                                       /*MemVT=*/N->getSimpleValueType(0), MMO);
-      if ((NewOpc == X86ISD::LSUB && C->isOne()) ||
-          (NewOpc == X86ISD::LADD && C->isAllOnesValue()))
-        return DAG.getMemIntrinsicNode(X86ISD::LDEC, SDLoc(N),
-                                       DAG.getVTList(MVT::i32, MVT::Other),
-                                       {N->getOperand(0), N->getOperand(1)},
-                                       /*MemVT=*/N->getSimpleValueType(0), MMO);
-    }
-  }
-
   return DAG.getMemIntrinsicNode(
       NewOpc, SDLoc(N), DAG.getVTList(MVT::i32, MVT::Other),
       {N->getOperand(0), N->getOperand(1), N->getOperand(2)},
@@ -26158,6 +26492,8 @@ SDValue X86TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
   case ISD::FP_EXTEND:          return LowerFP_EXTEND(Op, DAG);
   case ISD::LOAD:               return LowerLoad(Op, Subtarget, DAG);
   case ISD::STORE:              return LowerStore(Op, Subtarget, DAG);
+  case ISD::FADD:
+  case ISD::FSUB:               return lowerFaddFsub(Op, DAG, Subtarget);
   case ISD::FABS:
   case ISD::FNEG:               return LowerFABSorFNEG(Op, DAG);
   case ISD::FCOPYSIGN:          return LowerFCOPYSIGN(Op, DAG);
@@ -26210,7 +26546,7 @@ SDValue X86TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
   case ISD::ADDCARRY:
   case ISD::SUBCARRY:           return LowerADDSUBCARRY(Op, DAG);
   case ISD::ADD:
-  case ISD::SUB:                return LowerADD_SUB(Op, DAG);
+  case ISD::SUB:                return lowerAddSub(Op, DAG, Subtarget);
   case ISD::UADDSAT:
   case ISD::SADDSAT:
   case ISD::USUBSAT:
@@ -26219,7 +26555,7 @@ SDValue X86TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
   case ISD::SMIN:
   case ISD::UMAX:
   case ISD::UMIN:               return LowerMINMAX(Op, DAG);
-  case ISD::ABS:                return LowerABS(Op, DAG);
+  case ISD::ABS:                return LowerABS(Op, Subtarget, DAG);
   case ISD::FSINCOS:            return LowerFSINCOS(Op, Subtarget, DAG);
   case ISD::MLOAD:              return LowerMLOAD(Op, Subtarget, DAG);
   case ISD::MSTORE:             return LowerMSTORE(Op, Subtarget, DAG);
@@ -26259,7 +26595,6 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N,
                                            SmallVectorImpl<SDValue>&Results,
                                            SelectionDAG &DAG) const {
   SDLoc dl(N);
-  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
   switch (N->getOpcode()) {
   default:
     llvm_unreachable("Do not know how to custom type legalize this operation!");
@@ -26271,13 +26606,9 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N,
       // Promote to a pattern that will be turned into PMULUDQ.
       SDValue N0 = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::v2i64,
                                N->getOperand(0));
-      N0 = DAG.getNode(ISD::AND, dl, MVT::v2i64, N0,
-                       DAG.getConstant(0xffffffff, dl, MVT::v2i64));
       SDValue N1 = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::v2i64,
                                N->getOperand(1));
-      N1 = DAG.getNode(ISD::AND, dl, MVT::v2i64, N1,
-                       DAG.getConstant(0xffffffff, dl, MVT::v2i64));
-      SDValue Mul = DAG.getNode(ISD::MUL, dl, MVT::v2i64, N0, N1);
+      SDValue Mul = DAG.getNode(X86ISD::PMULUDQ, dl, MVT::v2i64, N0, N1);
       Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, VT, Mul));
     } else if (getTypeAction(*DAG.getContext(), VT) == TypeWidenVector &&
                VT.getVectorElementType() == MVT::i8) {
@@ -26381,7 +26712,7 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N,
         unsigned NumConcats = 128 / VT.getSizeInBits();
         SmallVector<SDValue, 8> Ops0(NumConcats, DAG.getUNDEF(VT));
         Ops0[0] = N->getOperand(0);
-        EVT ResVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
+        EVT ResVT = getTypeToTransformTo(*DAG.getContext(), VT);
         SDValue N0 = DAG.getNode(ISD::CONCAT_VECTORS, dl, ResVT, Ops0);
         SDValue N1 = DAG.getConstant(SplatVal, dl, ResVT);
         SDValue Res = DAG.getNode(N->getOpcode(), dl, ResVT, N0, N1);
@@ -26460,6 +26791,48 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N,
     }
     return;
   }
+  case ISD::SIGN_EXTEND_VECTOR_INREG: {
+    if (ExperimentalVectorWideningLegalization)
+      return;
+
+    EVT VT = N->getValueType(0);
+    SDValue In = N->getOperand(0);
+    EVT InVT = In.getValueType();
+    if (!Subtarget.hasSSE41() && VT == MVT::v4i64 &&
+        (InVT == MVT::v16i16 || InVT == MVT::v32i8)) {
+      // Custom split this so we can extend i8/i16->i32 invec. This is better
+      // since sign_extend_inreg i8/i16->i64 requires an extend to i32 using
+      // sra. Then extending from i32 to i64 using pcmpgt. By custom splitting
+      // we allow the sra from the extend to i32 to be shared by the split.
+      EVT ExtractVT = EVT::getVectorVT(*DAG.getContext(),
+                                       InVT.getVectorElementType(),
+                                       InVT.getVectorNumElements() / 2);
+      MVT ExtendVT = MVT::getVectorVT(MVT::i32,
+                                      VT.getVectorNumElements());
+      In = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, ExtractVT,
+                       In, DAG.getIntPtrConstant(0, dl));
+      In = DAG.getNode(ISD::SIGN_EXTEND_VECTOR_INREG, dl, MVT::v4i32, In);
+
+      // Fill a vector with sign bits for each element.
+      SDValue Zero = DAG.getConstant(0, dl, ExtendVT);
+      SDValue SignBits = DAG.getSetCC(dl, ExtendVT, Zero, In, ISD::SETGT);
+
+      EVT LoVT, HiVT;
+      std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
+
+      // Create an unpackl and unpackh to interleave the sign bits then bitcast
+      // to vXi64.
+      SDValue Lo = getUnpackl(DAG, dl, ExtendVT, In, SignBits);
+      Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo);
+      SDValue Hi = getUnpackh(DAG, dl, ExtendVT, In, SignBits);
+      Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi);
+
+      SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, Lo, Hi);
+      Results.push_back(Res);
+      return;
+    }
+    return;
+  }
   case ISD::SIGN_EXTEND:
   case ISD::ZERO_EXTEND: {
     if (!ExperimentalVectorWideningLegalization)
@@ -26471,8 +26844,9 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N,
     if (!Subtarget.hasSSE41() && VT == MVT::v4i64 &&
         (InVT == MVT::v4i16 || InVT == MVT::v4i8)) {
       // Custom split this so we can extend i8/i16->i32 invec. This is better
-      // since sign_extend_inreg i8/i16->i64 requires two sra operations. So
-      // this allows the first to be shared.
+      // since sign_extend_inreg i8/i16->i64 requires an extend to i32 using
+      // sra. Then extending from i32 to i64 using pcmpgt. By custom splitting
+      // we allow the sra from the extend to i32 to be shared by the split.
       In = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::v4i32, In);
 
       // Fill a vector with sign bits for each element.
@@ -26639,17 +27013,8 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N,
       return;
     }
 
-    std::pair<SDValue,SDValue> Vals =
-        FP_TO_INTHelper(SDValue(N, 0), DAG, IsSigned, /*IsReplace=*/ true);
-    SDValue FIST = Vals.first, StackSlot = Vals.second;
-    if (FIST.getNode()) {
-      // Return a load from the stack slot.
-      if (StackSlot.getNode())
-        Results.push_back(
-            DAG.getLoad(VT, dl, FIST, StackSlot, MachinePointerInfo()));
-      else
-        Results.push_back(FIST);
-    }
+    if (SDValue V = FP_TO_INTHelper(SDValue(N, 0), DAG, IsSigned))
+      Results.push_back(V);
     return;
   }
   case ISD::SINT_TO_FP: {
@@ -26685,7 +27050,7 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N,
     return;
   }
   case ISD::FP_ROUND: {
-    if (!TLI.isTypeLegal(N->getOperand(0).getValueType()))
+    if (!isTypeLegal(N->getOperand(0).getValueType()))
         return;
     SDValue V = DAG.getNode(X86ISD::VFPROUND, dl, MVT::v4f32, N->getOperand(0));
     Results.push_back(V);
@@ -26716,11 +27081,6 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N,
       return getExtendedControlRegister(N, dl, DAG, Subtarget, Results);
     }
   }
-  case ISD::INTRINSIC_WO_CHAIN: {
-    if (SDValue V = LowerINTRINSIC_WO_CHAIN(SDValue(N, 0), DAG))
-      Results.push_back(V);
-    return;
-  }
   case ISD::READCYCLECOUNTER: {
     return getReadTimeStampCounter(N, dl, X86ISD::RDTSC_DAG, DAG, Subtarget,
                                    Results);
@@ -26996,9 +27356,7 @@ const char *X86TargetLowering::getTargetNodeName(unsigned Opcode) const {
   case X86ISD::FXOR:               return "X86ISD::FXOR";
   case X86ISD::FILD:               return "X86ISD::FILD";
   case X86ISD::FILD_FLAG:          return "X86ISD::FILD_FLAG";
-  case X86ISD::FP_TO_INT16_IN_MEM: return "X86ISD::FP_TO_INT16_IN_MEM";
-  case X86ISD::FP_TO_INT32_IN_MEM: return "X86ISD::FP_TO_INT32_IN_MEM";
-  case X86ISD::FP_TO_INT64_IN_MEM: return "X86ISD::FP_TO_INT64_IN_MEM";
+  case X86ISD::FP_TO_INT_IN_MEM:   return "X86ISD::FP_TO_INT_IN_MEM";
   case X86ISD::FLD:                return "X86ISD::FLD";
   case X86ISD::FST:                return "X86ISD::FST";
   case X86ISD::CALL:               return "X86ISD::CALL";
@@ -27036,7 +27394,7 @@ const char *X86TargetLowering::getTargetNodeName(unsigned Opcode) const {
   case X86ISD::PSHUFB:             return "X86ISD::PSHUFB";
   case X86ISD::ANDNP:              return "X86ISD::ANDNP";
   case X86ISD::BLENDI:             return "X86ISD::BLENDI";
-  case X86ISD::SHRUNKBLEND:        return "X86ISD::SHRUNKBLEND";
+  case X86ISD::BLENDV:             return "X86ISD::BLENDV";
   case X86ISD::HADD:               return "X86ISD::HADD";
   case X86ISD::HSUB:               return "X86ISD::HSUB";
   case X86ISD::FHADD:              return "X86ISD::FHADD";
@@ -27079,13 +27437,14 @@ const char *X86TargetLowering::getTargetNodeName(unsigned Opcode) const {
   case X86ISD::LOR:                return "X86ISD::LOR";
   case X86ISD::LXOR:               return "X86ISD::LXOR";
   case X86ISD::LAND:               return "X86ISD::LAND";
-  case X86ISD::LINC:               return "X86ISD::LINC";
-  case X86ISD::LDEC:               return "X86ISD::LDEC";
   case X86ISD::VZEXT_MOVL:         return "X86ISD::VZEXT_MOVL";
   case X86ISD::VZEXT_LOAD:         return "X86ISD::VZEXT_LOAD";
   case X86ISD::VTRUNC:             return "X86ISD::VTRUNC";
   case X86ISD::VTRUNCS:            return "X86ISD::VTRUNCS";
   case X86ISD::VTRUNCUS:           return "X86ISD::VTRUNCUS";
+  case X86ISD::VMTRUNC:            return "X86ISD::VMTRUNC";
+  case X86ISD::VMTRUNCS:           return "X86ISD::VMTRUNCS";
+  case X86ISD::VMTRUNCUS:          return "X86ISD::VMTRUNCUS";
   case X86ISD::VTRUNCSTORES:       return "X86ISD::VTRUNCSTORES";
   case X86ISD::VTRUNCSTOREUS:      return "X86ISD::VTRUNCSTOREUS";
   case X86ISD::VMTRUNCSTORES:      return "X86ISD::VMTRUNCSTORES";
@@ -27094,6 +27453,7 @@ const char *X86TargetLowering::getTargetNodeName(unsigned Opcode) const {
   case X86ISD::VFPEXT_RND:         return "X86ISD::VFPEXT_RND";
   case X86ISD::VFPEXTS_RND:        return "X86ISD::VFPEXTS_RND";
   case X86ISD::VFPROUND:           return "X86ISD::VFPROUND";
+  case X86ISD::VMFPROUND:          return "X86ISD::VMFPROUND";
   case X86ISD::VFPROUND_RND:       return "X86ISD::VFPROUND_RND";
   case X86ISD::VFPROUNDS_RND:      return "X86ISD::VFPROUNDS_RND";
   case X86ISD::VSHLDQ:             return "X86ISD::VSHLDQ";
@@ -27104,6 +27464,8 @@ const char *X86TargetLowering::getTargetNodeName(unsigned Opcode) const {
   case X86ISD::VSHLI:              return "X86ISD::VSHLI";
   case X86ISD::VSRLI:              return "X86ISD::VSRLI";
   case X86ISD::VSRAI:              return "X86ISD::VSRAI";
+  case X86ISD::VSHLV:              return "X86ISD::VSHLV";
+  case X86ISD::VSRLV:              return "X86ISD::VSRLV";
   case X86ISD::VSRAV:              return "X86ISD::VSRAV";
   case X86ISD::VROTLI:             return "X86ISD::VROTLI";
   case X86ISD::VROTRI:             return "X86ISD::VROTRI";
@@ -27118,10 +27480,6 @@ const char *X86TargetLowering::getTargetNodeName(unsigned Opcode) const {
   case X86ISD::SBB:                return "X86ISD::SBB";
   case X86ISD::SMUL:               return "X86ISD::SMUL";
   case X86ISD::UMUL:               return "X86ISD::UMUL";
-  case X86ISD::SMUL8:              return "X86ISD::SMUL8";
-  case X86ISD::UMUL8:              return "X86ISD::UMUL8";
-  case X86ISD::INC:                return "X86ISD::INC";
-  case X86ISD::DEC:                return "X86ISD::DEC";
   case X86ISD::OR:                 return "X86ISD::OR";
   case X86ISD::XOR:                return "X86ISD::XOR";
   case X86ISD::AND:                return "X86ISD::AND";
@@ -27224,7 +27582,6 @@ const char *X86TargetLowering::getTargetNodeName(unsigned Opcode) const {
   case X86ISD::XTEST:              return "X86ISD::XTEST";
   case X86ISD::COMPRESS:           return "X86ISD::COMPRESS";
   case X86ISD::EXPAND:             return "X86ISD::EXPAND";
-  case X86ISD::SELECT:             return "X86ISD::SELECT";
   case X86ISD::SELECTS:            return "X86ISD::SELECTS";
   case X86ISD::ADDSUB:             return "X86ISD::ADDSUB";
   case X86ISD::RCP14:              return "X86ISD::RCP14";
@@ -27256,6 +27613,8 @@ const char *X86TargetLowering::getTargetNodeName(unsigned Opcode) const {
   case X86ISD::UINT_TO_FP_RND:     return "X86ISD::UINT_TO_FP_RND";
   case X86ISD::CVTTP2SI:           return "X86ISD::CVTTP2SI";
   case X86ISD::CVTTP2UI:           return "X86ISD::CVTTP2UI";
+  case X86ISD::MCVTTP2SI:          return "X86ISD::MCVTTP2SI";
+  case X86ISD::MCVTTP2UI:          return "X86ISD::MCVTTP2UI";
   case X86ISD::CVTTP2SI_RND:       return "X86ISD::CVTTP2SI_RND";
   case X86ISD::CVTTP2UI_RND:       return "X86ISD::CVTTP2UI_RND";
   case X86ISD::CVTTS2SI:           return "X86ISD::CVTTS2SI";
@@ -27264,16 +27623,21 @@ const char *X86TargetLowering::getTargetNodeName(unsigned Opcode) const {
   case X86ISD::CVTTS2UI_RND:       return "X86ISD::CVTTS2UI_RND";
   case X86ISD::CVTSI2P:            return "X86ISD::CVTSI2P";
   case X86ISD::CVTUI2P:            return "X86ISD::CVTUI2P";
+  case X86ISD::MCVTSI2P:           return "X86ISD::MCVTSI2P";
+  case X86ISD::MCVTUI2P:           return "X86ISD::MCVTUI2P";
   case X86ISD::VFPCLASS:           return "X86ISD::VFPCLASS";
   case X86ISD::VFPCLASSS:          return "X86ISD::VFPCLASSS";
   case X86ISD::MULTISHIFT:         return "X86ISD::MULTISHIFT";
   case X86ISD::SCALAR_SINT_TO_FP_RND: return "X86ISD::SCALAR_SINT_TO_FP_RND";
   case X86ISD::SCALAR_UINT_TO_FP_RND: return "X86ISD::SCALAR_UINT_TO_FP_RND";
   case X86ISD::CVTPS2PH:           return "X86ISD::CVTPS2PH";
+  case X86ISD::MCVTPS2PH:          return "X86ISD::MCVTPS2PH";
   case X86ISD::CVTPH2PS:           return "X86ISD::CVTPH2PS";
   case X86ISD::CVTPH2PS_RND:       return "X86ISD::CVTPH2PS_RND";
   case X86ISD::CVTP2SI:            return "X86ISD::CVTP2SI";
   case X86ISD::CVTP2UI:            return "X86ISD::CVTP2UI";
+  case X86ISD::MCVTP2SI:           return "X86ISD::MCVTP2SI";
+  case X86ISD::MCVTP2UI:           return "X86ISD::MCVTP2UI";
   case X86ISD::CVTP2SI_RND:        return "X86ISD::CVTP2SI_RND";
   case X86ISD::CVTP2UI_RND:        return "X86ISD::CVTP2UI_RND";
   case X86ISD::CVTS2SI:            return "X86ISD::CVTS2SI";
@@ -29994,11 +30358,12 @@ void X86TargetLowering::computeKnownBitsForTargetNode(const SDValue Op,
     EVT SrcVT = Src.getValueType();
     APInt DemandedElt = APInt::getOneBitSet(SrcVT.getVectorNumElements(),
                                             Op.getConstantOperandVal(1));
-    DAG.computeKnownBits(Src, Known, DemandedElt, Depth + 1);
+    Known = DAG.computeKnownBits(Src, DemandedElt, Depth + 1);
     Known = Known.zextOrTrunc(BitWidth);
     Known.Zero.setBitsFrom(SrcVT.getScalarSizeInBits());
     break;
   }
+  case X86ISD::VSRAI:
   case X86ISD::VSHLI:
   case X86ISD::VSRLI: {
     if (auto *ShiftImm = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
@@ -30007,18 +30372,21 @@ void X86TargetLowering::computeKnownBitsForTargetNode(const SDValue Op,
         break;
       }
 
-      DAG.computeKnownBits(Op.getOperand(0), Known, DemandedElts, Depth + 1);
+      Known = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
       unsigned ShAmt = ShiftImm->getZExtValue();
       if (Opc == X86ISD::VSHLI) {
         Known.Zero <<= ShAmt;
         Known.One <<= ShAmt;
         // Low bits are known zero.
         Known.Zero.setLowBits(ShAmt);
-      } else {
+      } else if (Opc == X86ISD::VSRLI) {
         Known.Zero.lshrInPlace(ShAmt);
         Known.One.lshrInPlace(ShAmt);
         // High bits are known zero.
         Known.Zero.setHighBits(ShAmt);
+      } else {
+        Known.Zero.ashrInPlace(ShAmt);
+        Known.One.ashrInPlace(ShAmt);
       }
     }
     break;
@@ -30033,12 +30401,12 @@ void X86TargetLowering::computeKnownBitsForTargetNode(const SDValue Op,
 
     KnownBits Known2;
     if (!!DemandedLHS) {
-      DAG.computeKnownBits(Op.getOperand(0), Known2, DemandedLHS, Depth + 1);
+      Known2 = DAG.computeKnownBits(Op.getOperand(0), DemandedLHS, Depth + 1);
       Known.One &= Known2.One;
       Known.Zero &= Known2.Zero;
     }
     if (!!DemandedRHS) {
-      DAG.computeKnownBits(Op.getOperand(1), Known2, DemandedRHS, Depth + 1);
+      Known2 = DAG.computeKnownBits(Op.getOperand(1), DemandedRHS, Depth + 1);
       Known.One &= Known2.One;
       Known.Zero &= Known2.Zero;
     }
@@ -30048,13 +30416,33 @@ void X86TargetLowering::computeKnownBitsForTargetNode(const SDValue Op,
     Known = Known.trunc(BitWidth);
     break;
   }
+  case X86ISD::ANDNP: {
+    KnownBits Known2;
+    Known = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
+    Known2 = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
+
+    // ANDNP = (~X & Y);
+    Known.One &= Known2.Zero;
+    Known.Zero |= Known2.One;
+    break;
+  }
+  case X86ISD::FOR: {
+    KnownBits Known2;
+    Known = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
+    Known2 = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
+
+    // Output known-0 bits are only known if clear in both the LHS & RHS.
+    Known.Zero &= Known2.Zero;
+    // Output known-1 are known to be set if set in either the LHS | RHS.
+    Known.One |= Known2.One;
+    break;
+  }
   case X86ISD::CMOV: {
-    DAG.computeKnownBits(Op.getOperand(1), Known, Depth+1);
+    Known = DAG.computeKnownBits(Op.getOperand(1), Depth+1);
     // If we don't know any bits, early out.
     if (Known.isUnknown())
       break;
-    KnownBits Known2;
-    DAG.computeKnownBits(Op.getOperand(0), Known2, Depth+1);
+    KnownBits Known2 = DAG.computeKnownBits(Op.getOperand(0), Depth+1);
 
     // Only known if known in both the LHS and RHS.
     Known.One &= Known2.One;
@@ -30105,8 +30493,8 @@ void X86TargetLowering::computeKnownBitsForTargetNode(const SDValue Op,
         for (unsigned i = 0; i != NumOps && !Known.isUnknown(); ++i) {
           if (!DemandedOps[i])
             continue;
-          KnownBits Known2;
-          DAG.computeKnownBits(Ops[i], Known2, DemandedOps[i], Depth + 1);
+          KnownBits Known2 =
+              DAG.computeKnownBits(Ops[i], DemandedOps[i], Depth + 1);
           Known.One &= Known2.One;
           Known.Zero &= Known2.Zero;
         }
@@ -30156,7 +30544,7 @@ unsigned X86TargetLowering::ComputeNumSignBitsForTargetNode(
 
   case X86ISD::VSHLI: {
     SDValue Src = Op.getOperand(0);
-    APInt ShiftVal = cast<ConstantSDNode>(Op.getOperand(1))->getAPIntValue();
+    const APInt &ShiftVal = Op.getConstantOperandAPInt(1);
     if (ShiftVal.uge(VTBits))
       return VTBits; // Shifted all bits out --> zero.
     unsigned Tmp = DAG.ComputeNumSignBits(Src, DemandedElts, Depth + 1);
@@ -30167,7 +30555,7 @@ unsigned X86TargetLowering::ComputeNumSignBitsForTargetNode(
 
   case X86ISD::VSRAI: {
     SDValue Src = Op.getOperand(0);
-    APInt ShiftVal = cast<ConstantSDNode>(Op.getOperand(1))->getAPIntValue();
+    APInt ShiftVal = Op.getConstantOperandAPInt(1);
     if (ShiftVal.uge(VTBits - 1))
       return VTBits; // Sign splat.
     unsigned Tmp = DAG.ComputeNumSignBits(Src, DemandedElts, Depth + 1);
@@ -30204,12 +30592,11 @@ SDValue X86TargetLowering::unwrapAddress(SDValue N) const {
 // Attempt to match a combined shuffle mask against supported unary shuffle
 // instructions.
 // TODO: Investigate sharing more of this with shuffle lowering.
-static bool matchUnaryVectorShuffle(MVT MaskVT, ArrayRef<int> Mask,
-                                    bool AllowFloatDomain, bool AllowIntDomain,
-                                    SDValue &V1, const SDLoc &DL,
-                                    SelectionDAG &DAG,
-                                    const X86Subtarget &Subtarget,
-                                    unsigned &Shuffle, MVT &SrcVT, MVT &DstVT) {
+static bool matchUnaryShuffle(MVT MaskVT, ArrayRef<int> Mask,
+                              bool AllowFloatDomain, bool AllowIntDomain,
+                              SDValue &V1, const SDLoc &DL, SelectionDAG &DAG,
+                              const X86Subtarget &Subtarget, unsigned &Shuffle,
+                              MVT &SrcVT, MVT &DstVT) {
   unsigned NumMaskElts = Mask.size();
   unsigned MaskEltSize = MaskVT.getScalarSizeInBits();
 
@@ -30267,7 +30654,7 @@ static bool matchUnaryVectorShuffle(MVT MaskVT, ArrayRef<int> Mask,
   // instructions are no slower than UNPCKLPD but has the option to
   // fold the input operand into even an unaligned memory load.
   if (MaskVT.is128BitVector() && Subtarget.hasSSE3() && AllowFloatDomain) {
-    if (!Subtarget.hasAVX2() && isTargetShuffleEquivalent(Mask, {0, 0})) {
+    if (isTargetShuffleEquivalent(Mask, {0, 0})) {
       Shuffle = X86ISD::MOVDDUP;
       SrcVT = DstVT = MVT::v2f64;
       return true;
@@ -30325,29 +30712,18 @@ static bool matchUnaryVectorShuffle(MVT MaskVT, ArrayRef<int> Mask,
     }
   }
 
-  // Attempt to match against broadcast-from-vector.
-  if (Subtarget.hasAVX2()) {
-    SmallVector<int, 64> BroadcastMask(NumMaskElts, 0);
-    if (isTargetShuffleEquivalent(Mask, BroadcastMask)) {
-      SrcVT = DstVT = MaskVT;
-      Shuffle = X86ISD::VBROADCAST;
-      return true;
-    }
-  }
-
   return false;
 }
 
 // Attempt to match a combined shuffle mask against supported unary immediate
 // permute instructions.
 // TODO: Investigate sharing more of this with shuffle lowering.
-static bool matchUnaryPermuteVectorShuffle(MVT MaskVT, ArrayRef<int> Mask,
-                                           const APInt &Zeroable,
-                                           bool AllowFloatDomain,
-                                           bool AllowIntDomain,
-                                           const X86Subtarget &Subtarget,
-                                           unsigned &Shuffle, MVT &ShuffleVT,
-                                           unsigned &PermuteImm) {
+static bool matchUnaryPermuteShuffle(MVT MaskVT, ArrayRef<int> Mask,
+                                     const APInt &Zeroable,
+                                     bool AllowFloatDomain, bool AllowIntDomain,
+                                     const X86Subtarget &Subtarget,
+                                     unsigned &Shuffle, MVT &ShuffleVT,
+                                     unsigned &PermuteImm) {
   unsigned NumMaskElts = Mask.size();
   unsigned InputSizeInBits = MaskVT.getSizeInBits();
   unsigned MaskScalarSizeInBits = InputSizeInBits / NumMaskElts;
@@ -30448,9 +30824,8 @@ static bool matchUnaryPermuteVectorShuffle(MVT MaskVT, ArrayRef<int> Mask,
   // FIXME: Add 512-bit support.
   if (AllowIntDomain && ((MaskVT.is128BitVector() && Subtarget.hasSSE2()) ||
                          (MaskVT.is256BitVector() && Subtarget.hasAVX2()))) {
-    int ShiftAmt = matchVectorShuffleAsShift(ShuffleVT, Shuffle,
-                                             MaskScalarSizeInBits, Mask,
-                                             0, Zeroable, Subtarget);
+    int ShiftAmt = matchShuffleAsShift(ShuffleVT, Shuffle, MaskScalarSizeInBits,
+                                       Mask, 0, Zeroable, Subtarget);
     if (0 < ShiftAmt) {
       PermuteImm = (unsigned)ShiftAmt;
       return true;
@@ -30463,13 +30838,12 @@ static bool matchUnaryPermuteVectorShuffle(MVT MaskVT, ArrayRef<int> Mask,
 // Attempt to match a combined unary shuffle mask against supported binary
 // shuffle instructions.
 // TODO: Investigate sharing more of this with shuffle lowering.
-static bool matchBinaryVectorShuffle(MVT MaskVT, ArrayRef<int> Mask,
-                                     bool AllowFloatDomain, bool AllowIntDomain,
-                                     SDValue &V1, SDValue &V2, const SDLoc &DL,
-                                     SelectionDAG &DAG,
-                                     const X86Subtarget &Subtarget,
-                                     unsigned &Shuffle, MVT &SrcVT, MVT &DstVT,
-                                     bool IsUnary) {
+static bool matchBinaryShuffle(MVT MaskVT, ArrayRef<int> Mask,
+                               bool AllowFloatDomain, bool AllowIntDomain,
+                               SDValue &V1, SDValue &V2, const SDLoc &DL,
+                               SelectionDAG &DAG, const X86Subtarget &Subtarget,
+                               unsigned &Shuffle, MVT &SrcVT, MVT &DstVT,
+                               bool IsUnary) {
   unsigned EltSizeInBits = MaskVT.getScalarSizeInBits();
 
   if (MaskVT.is128BitVector()) {
@@ -30501,9 +30875,10 @@ static bool matchBinaryVectorShuffle(MVT MaskVT, ArrayRef<int> Mask,
     }
   }
 
-  // Attempt to match against either a unary or binary PACKSS/PACKUS shuffle.
-  // TODO add support for 256/512-bit types.
-  if ((MaskVT == MVT::v8i16 || MaskVT == MVT::v16i8) && Subtarget.hasSSE2()) {
+  // Attempt to match against either an unary or binary PACKSS/PACKUS shuffle.
+  if (((MaskVT == MVT::v8i16 || MaskVT == MVT::v16i8) && Subtarget.hasSSE2()) ||
+      ((MaskVT == MVT::v16i16 || MaskVT == MVT::v32i8) && Subtarget.hasInt256()) ||
+      ((MaskVT == MVT::v32i16 || MaskVT == MVT::v64i8) && Subtarget.hasBWI())) {
     if (matchVectorShuffleWithPACK(MaskVT, SrcVT, V1, V2, Shuffle, Mask, DAG,
                                    Subtarget)) {
       DstVT = MaskVT;
@@ -30529,7 +30904,7 @@ static bool matchBinaryVectorShuffle(MVT MaskVT, ArrayRef<int> Mask,
   return false;
 }
 
-static bool matchBinaryPermuteVectorShuffle(
+static bool matchBinaryPermuteShuffle(
     MVT MaskVT, ArrayRef<int> Mask, const APInt &Zeroable,
     bool AllowFloatDomain, bool AllowIntDomain, SDValue &V1, SDValue &V2,
     const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget,
@@ -30540,7 +30915,7 @@ static bool matchBinaryPermuteVectorShuffle(
   // Attempt to match against PALIGNR byte rotate.
   if (AllowIntDomain && ((MaskVT.is128BitVector() && Subtarget.hasSSSE3()) ||
                          (MaskVT.is256BitVector() && Subtarget.hasAVX2()))) {
-    int ByteRotation = matchVectorShuffleAsByteRotate(MaskVT, V1, V2, Mask);
+    int ByteRotation = matchShuffleAsByteRotate(MaskVT, V1, V2, Mask);
     if (0 < ByteRotation) {
       Shuffle = X86ISD::PALIGNR;
       ShuffleVT = MVT::getVectorVT(MVT::i8, MaskVT.getSizeInBits() / 8);
@@ -30613,7 +30988,7 @@ static bool matchBinaryPermuteVectorShuffle(
   if (AllowFloatDomain && EltSizeInBits == 32 && Subtarget.hasSSE41() &&
       MaskVT.is128BitVector()) {
     if (Zeroable.getBoolValue() &&
-        matchVectorShuffleAsInsertPS(V1, V2, PermuteImm, Zeroable, Mask, DAG)) {
+        matchShuffleAsInsertPS(V1, V2, PermuteImm, Zeroable, Mask, DAG)) {
       Shuffle = X86ISD::INSERTPS;
       ShuffleVT = MVT::v4f32;
       return true;
@@ -30625,7 +31000,7 @@ static bool matchBinaryPermuteVectorShuffle(
       ((MaskVT.is128BitVector() && Subtarget.hasSSE2()) ||
        (MaskVT.is256BitVector() && Subtarget.hasAVX()) ||
        (MaskVT.is512BitVector() && Subtarget.hasAVX512()))) {
-    if (matchVectorShuffleWithSHUFPD(MaskVT, V1, V2, PermuteImm, Mask)) {
+    if (matchShuffleWithSHUFPD(MaskVT, V1, V2, PermuteImm, Mask)) {
       Shuffle = X86ISD::SHUFP;
       ShuffleVT = MVT::getVectorVT(MVT::f64, MaskVT.getSizeInBits() / 64);
       return true;
@@ -30739,6 +31114,24 @@ static SDValue combineX86ShuffleChain(ArrayRef<SDValue> Inputs, SDValue Root,
   bool IsEVEXShuffle =
       RootSizeInBits == 512 || (Subtarget.hasVLX() && RootSizeInBits >= 128);
 
+  // Attempt to match a subvector broadcast.
+  // shuffle(insert_subvector(undef, sub, 0), undef, 0, 0, 0, 0)
+  if (UnaryShuffle &&
+      (BaseMaskEltSizeInBits == 128 || BaseMaskEltSizeInBits == 256)) {
+    SmallVector<int, 64> BroadcastMask(NumBaseMaskElts, 0);
+    if (isTargetShuffleEquivalent(BaseMask, BroadcastMask)) {
+      SDValue Src = Inputs[0];
+      if (Src.getOpcode() == ISD::INSERT_SUBVECTOR &&
+          Src.getOperand(0).isUndef() &&
+          Src.getOperand(1).getValueSizeInBits() == BaseMaskEltSizeInBits &&
+          MayFoldLoad(Src.getOperand(1)) && isNullConstant(Src.getOperand(2))) {
+        return DAG.getBitcast(RootVT, DAG.getNode(X86ISD::SUBV_BROADCAST, DL,
+                                                  Src.getValueType(),
+                                                  Src.getOperand(1)));
+      }
+    }
+  }
+
   // TODO - handle 128/256-bit lane shuffles of 512-bit vectors.
 
   // Handle 128-bit lane shuffles of 256-bit vectors.
@@ -30816,10 +31209,35 @@ static SDValue combineX86ShuffleChain(ArrayRef<SDValue> Inputs, SDValue Root,
       }
     }
 
+    // Attempt to match against broadcast-from-vector.
+    // Limit AVX1 to cases where we're loading+broadcasting a scalar element.
+    if ((Subtarget.hasAVX2() || (Subtarget.hasAVX() && 32 <= MaskEltSizeInBits))
+        && (!IsEVEXShuffle || NumRootElts == NumMaskElts)) {
+      SmallVector<int, 64> BroadcastMask(NumMaskElts, 0);
+      if (isTargetShuffleEquivalent(Mask, BroadcastMask)) {
+        if (V1.getValueType() == MaskVT &&
+            V1.getOpcode() == ISD::SCALAR_TO_VECTOR &&
+            MayFoldLoad(V1.getOperand(0))) {
+          if (Depth == 1 && Root.getOpcode() == X86ISD::VBROADCAST)
+            return SDValue(); // Nothing to do!
+          Res = V1.getOperand(0);
+          Res = DAG.getNode(X86ISD::VBROADCAST, DL, MaskVT, Res);
+          return DAG.getBitcast(RootVT, Res);
+        }
+        if (Subtarget.hasAVX2()) {
+          if (Depth == 1 && Root.getOpcode() == X86ISD::VBROADCAST)
+            return SDValue(); // Nothing to do!
+          Res = DAG.getBitcast(MaskVT, V1);
+          Res = DAG.getNode(X86ISD::VBROADCAST, DL, MaskVT, Res);
+          return DAG.getBitcast(RootVT, Res);
+        }
+      }
+    }
+
     SDValue NewV1 = V1; // Save operand in case early exit happens.
-    if (matchUnaryVectorShuffle(MaskVT, Mask, AllowFloatDomain, AllowIntDomain,
-                                NewV1, DL, DAG, Subtarget, Shuffle,
-                                ShuffleSrcVT, ShuffleVT) &&
+    if (matchUnaryShuffle(MaskVT, Mask, AllowFloatDomain, AllowIntDomain, NewV1,
+                          DL, DAG, Subtarget, Shuffle, ShuffleSrcVT,
+                          ShuffleVT) &&
         (!IsEVEXShuffle || (NumRootElts == ShuffleVT.getVectorNumElements()))) {
       if (Depth == 1 && Root.getOpcode() == Shuffle)
         return SDValue(); // Nothing to do!
@@ -30828,9 +31246,9 @@ static SDValue combineX86ShuffleChain(ArrayRef<SDValue> Inputs, SDValue Root,
       return DAG.getBitcast(RootVT, Res);
     }
 
-    if (matchUnaryPermuteVectorShuffle(MaskVT, Mask, Zeroable, AllowFloatDomain,
-                                       AllowIntDomain, Subtarget, Shuffle,
-                                       ShuffleVT, PermuteImm) &&
+    if (matchUnaryPermuteShuffle(MaskVT, Mask, Zeroable, AllowFloatDomain,
+                                 AllowIntDomain, Subtarget, Shuffle, ShuffleVT,
+                                 PermuteImm) &&
         (!IsEVEXShuffle || (NumRootElts == ShuffleVT.getVectorNumElements()))) {
       if (Depth == 1 && Root.getOpcode() == Shuffle)
         return SDValue(); // Nothing to do!
@@ -30843,9 +31261,9 @@ static SDValue combineX86ShuffleChain(ArrayRef<SDValue> Inputs, SDValue Root,
 
   SDValue NewV1 = V1; // Save operands in case early exit happens.
   SDValue NewV2 = V2;
-  if (matchBinaryVectorShuffle(MaskVT, Mask, AllowFloatDomain, AllowIntDomain,
-                               NewV1, NewV2, DL, DAG, Subtarget, Shuffle,
-                               ShuffleSrcVT, ShuffleVT, UnaryShuffle) &&
+  if (matchBinaryShuffle(MaskVT, Mask, AllowFloatDomain, AllowIntDomain, NewV1,
+                         NewV2, DL, DAG, Subtarget, Shuffle, ShuffleSrcVT,
+                         ShuffleVT, UnaryShuffle) &&
       (!IsEVEXShuffle || (NumRootElts == ShuffleVT.getVectorNumElements()))) {
     if (Depth == 1 && Root.getOpcode() == Shuffle)
       return SDValue(); // Nothing to do!
@@ -30857,7 +31275,7 @@ static SDValue combineX86ShuffleChain(ArrayRef<SDValue> Inputs, SDValue Root,
 
   NewV1 = V1; // Save operands in case early exit happens.
   NewV2 = V2;
-  if (matchBinaryPermuteVectorShuffle(
+  if (matchBinaryPermuteShuffle(
           MaskVT, Mask, Zeroable, AllowFloatDomain, AllowIntDomain, NewV1,
           NewV2, DL, DAG, Subtarget, Shuffle, ShuffleVT, PermuteImm) &&
       (!IsEVEXShuffle || (NumRootElts == ShuffleVT.getVectorNumElements()))) {
@@ -30877,8 +31295,8 @@ static SDValue combineX86ShuffleChain(ArrayRef<SDValue> Inputs, SDValue Root,
   // Annoyingly, SSE4A instructions don't map into the above match helpers.
   if (Subtarget.hasSSE4A() && AllowIntDomain && RootSizeInBits == 128) {
     uint64_t BitLen, BitIdx;
-    if (matchVectorShuffleAsEXTRQ(IntMaskVT, V1, V2, Mask, BitLen, BitIdx,
-                                  Zeroable)) {
+    if (matchShuffleAsEXTRQ(IntMaskVT, V1, V2, Mask, BitLen, BitIdx,
+                            Zeroable)) {
       if (Depth == 1 && Root.getOpcode() == X86ISD::EXTRQI)
         return SDValue(); // Nothing to do!
       V1 = DAG.getBitcast(IntMaskVT, V1);
@@ -30888,7 +31306,7 @@ static SDValue combineX86ShuffleChain(ArrayRef<SDValue> Inputs, SDValue Root,
       return DAG.getBitcast(RootVT, Res);
     }
 
-    if (matchVectorShuffleAsINSERTQ(IntMaskVT, V1, V2, Mask, BitLen, BitIdx)) {
+    if (matchShuffleAsINSERTQ(IntMaskVT, V1, V2, Mask, BitLen, BitIdx)) {
       if (Depth == 1 && Root.getOpcode() == X86ISD::INSERTQI)
         return SDValue(); // Nothing to do!
       V1 = DAG.getBitcast(IntMaskVT, V1);
@@ -31453,6 +31871,14 @@ static SDValue combineX86ShufflesRecursively(
                                 AllowVariableMask, DAG, Subtarget);
 }
 
+/// Helper entry wrapper to combineX86ShufflesRecursively.
+static SDValue combineX86ShufflesRecursively(SDValue Op, SelectionDAG &DAG,
+                                             const X86Subtarget &Subtarget) {
+  return combineX86ShufflesRecursively({Op}, 0, Op, {0}, {}, /*Depth*/ 1,
+                                       /*HasVarMask*/ false,
+                                       /*AllowVarMask*/ true, DAG, Subtarget);
+}
+
 /// Get the PSHUF-style mask from PSHUF node.
 ///
 /// This is a very minor wrapper around getTargetShuffleMask to easy forming v4
@@ -31668,12 +32094,13 @@ static SDValue combineTargetShuffle(SDValue N, SelectionDAG &DAG,
 
   switch (Opcode) {
   case X86ISD::VBROADCAST: {
-    // If broadcasting from another shuffle, attempt to simplify it.
-    // TODO - we really need a general SimplifyDemandedVectorElts mechanism.
     SDValue Src = N.getOperand(0);
     SDValue BC = peekThroughBitcasts(Src);
     EVT SrcVT = Src.getValueType();
     EVT BCVT = BC.getValueType();
+
+    // If broadcasting from another shuffle, attempt to simplify it.
+    // TODO - we really need a general SimplifyDemandedVectorElts mechanism.
     if (isTargetShuffle(BC.getOpcode()) &&
         VT.getScalarSizeInBits() % BCVT.getScalarSizeInBits() == 0) {
       unsigned Scale = VT.getScalarSizeInBits() / BCVT.getScalarSizeInBits();
@@ -31687,6 +32114,33 @@ static SDValue combineTargetShuffle(SDValue N, SelectionDAG &DAG,
         return DAG.getNode(X86ISD::VBROADCAST, DL, VT,
                            DAG.getBitcast(SrcVT, Res));
     }
+
+    // broadcast(bitcast(src)) -> bitcast(broadcast(src))
+    // 32-bit targets have to bitcast i64 to f64, so better to bitcast upward.
+    if (Src.getOpcode() == ISD::BITCAST &&
+        SrcVT.getScalarSizeInBits() == BCVT.getScalarSizeInBits()) {
+      EVT NewVT = EVT::getVectorVT(*DAG.getContext(), BCVT.getScalarType(),
+                                   VT.getVectorNumElements());
+      return DAG.getBitcast(VT, DAG.getNode(X86ISD::VBROADCAST, DL, NewVT, BC));
+    }
+
+    // Reduce broadcast source vector to lowest 128-bits.
+    if (SrcVT.getSizeInBits() > 128)
+      return DAG.getNode(X86ISD::VBROADCAST, DL, VT,
+                         extract128BitVector(Src, 0, DAG, DL));
+
+    // broadcast(scalar_to_vector(x)) -> broadcast(x).
+    if (Src.getOpcode() == ISD::SCALAR_TO_VECTOR)
+      return DAG.getNode(X86ISD::VBROADCAST, DL, VT, Src.getOperand(0));
+
+    // Share broadcast with the longest vector and extract low subvector (free).
+    for (SDNode *User : Src->uses())
+      if (User != N.getNode() && User->getOpcode() == X86ISD::VBROADCAST &&
+          User->getValueSizeInBits(0) > VT.getSizeInBits()) {
+        return extractSubVector(SDValue(User, 0), 0, DAG, DL,
+                                VT.getSizeInBits());
+      }
+
     return SDValue();
   }
   case X86ISD::PSHUFD:
@@ -32150,14 +32604,51 @@ static SDValue foldShuffleOfHorizOp(SDNode *N) {
   return SDValue();
 }
 
+/// If we have a shuffle of AVX/AVX512 (256/512 bit) vectors that only uses the
+/// low half of each source vector and does not set any high half elements in
+/// the destination vector, narrow the shuffle to half its original size.
+static SDValue narrowShuffle(ShuffleVectorSDNode *Shuf, SelectionDAG &DAG) {
+  if (!Shuf->getValueType(0).isSimple())
+    return SDValue();
+  MVT VT = Shuf->getSimpleValueType(0);
+  if (!VT.is256BitVector() && !VT.is512BitVector())
+    return SDValue();
+
+  // See if we can ignore all of the high elements of the shuffle.
+  ArrayRef<int> Mask = Shuf->getMask();
+  if (!isUndefUpperHalf(Mask))
+    return SDValue();
+
+  // Check if the shuffle mask accesses only the low half of each input vector
+  // (half-index output is 0 or 2).
+  int HalfIdx1, HalfIdx2;
+  SmallVector<int, 8> HalfMask(Mask.size() / 2);
+  if (!getHalfShuffleMask(Mask, HalfMask, HalfIdx1, HalfIdx2) ||
+      (HalfIdx1 % 2 == 1) || (HalfIdx2 % 2 == 1))
+    return SDValue();
+
+  // Create a half-width shuffle to replace the unnecessarily wide shuffle.
+  // The trick is knowing that all of the insert/extract are actually free
+  // subregister (zmm<->ymm or ymm<->xmm) ops. That leaves us with a shuffle
+  // of narrow inputs into a narrow output, and that is always cheaper than
+  // the wide shuffle that we started with.
+  return getShuffleHalfVectors(SDLoc(Shuf), Shuf->getOperand(0),
+                               Shuf->getOperand(1), HalfMask, HalfIdx1,
+                               HalfIdx2, false, DAG);
+}
+
 static SDValue combineShuffle(SDNode *N, SelectionDAG &DAG,
                               TargetLowering::DAGCombinerInfo &DCI,
                               const X86Subtarget &Subtarget) {
+  if (auto *Shuf = dyn_cast<ShuffleVectorSDNode>(N))
+    if (SDValue V = narrowShuffle(Shuf, DAG))
+      return V;
+
+  // If we have legalized the vector types, look for blends of FADD and FSUB
+  // nodes that we can fuse into an ADDSUB, FMADDSUB, or FMSUBADD node.
   SDLoc dl(N);
   EVT VT = N->getValueType(0);
   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
-  // If we have legalized the vector types, look for blends of FADD and FSUB
-  // nodes that we can fuse into an ADDSUB, FMADDSUB, or FMSUBADD node.
   if (TLI.isTypeLegal(VT)) {
     if (SDValue AddSub = combineShuffleToAddSubOrFMAddSub(N, Subtarget, DAG))
       return AddSub;
@@ -32263,9 +32754,7 @@ static SDValue combineShuffle(SDNode *N, SelectionDAG &DAG,
     // specific PSHUF instruction sequences into their minimal form so that we
     // can evaluate how many specialized shuffle instructions are involved in
     // a particular chain.
-    if (SDValue Res = combineX86ShufflesRecursively(
-            {Op}, 0, Op, {0}, {}, /*Depth*/ 1,
-            /*HasVarMask*/ false, /*AllowVarMask*/ true, DAG, Subtarget))
+    if (SDValue Res = combineX86ShufflesRecursively(Op, DAG, Subtarget))
       return Res;
 
     // Simplify source operands based on shuffle mask.
@@ -32276,6 +32765,78 @@ static SDValue combineShuffle(SDNode *N, SelectionDAG &DAG,
       return SDValue(N, 0);
   }
 
+  // Look for a v2i64/v2f64 VZEXT_MOVL of a node that already produces zeros
+  // in the upper 64 bits.
+  // TODO: Can we generalize this using computeKnownBits.
+  if (N->getOpcode() == X86ISD::VZEXT_MOVL &&
+      (VT == MVT::v2f64 || VT == MVT::v2i64) &&
+      N->getOperand(0).getOpcode() == ISD::BITCAST &&
+      (N->getOperand(0).getOperand(0).getValueType() == MVT::v4f32 ||
+       N->getOperand(0).getOperand(0).getValueType() == MVT::v4i32)) {
+    SDValue In = N->getOperand(0).getOperand(0);
+    switch (In.getOpcode()) {
+    default:
+      break;
+    case X86ISD::CVTP2SI:   case X86ISD::CVTP2UI:
+    case X86ISD::MCVTP2SI:  case X86ISD::MCVTP2UI:
+    case X86ISD::CVTTP2SI:  case X86ISD::CVTTP2UI:
+    case X86ISD::MCVTTP2SI: case X86ISD::MCVTTP2UI:
+    case X86ISD::CVTSI2P:   case X86ISD::CVTUI2P:
+    case X86ISD::MCVTSI2P:  case X86ISD::MCVTUI2P:
+    case X86ISD::VFPROUND:  case X86ISD::VMFPROUND:
+      if (In.getOperand(0).getValueType() == MVT::v2f64 ||
+          In.getOperand(0).getValueType() == MVT::v2i64)
+        return N->getOperand(0); // return the bitcast
+      break;
+    }
+  }
+
+  // Look for a truncating shuffle to v2i32 of a PMULUDQ where one of the
+  // operands is an extend from v2i32 to v2i64. Turn it into a pmulld.
+  // FIXME: This can probably go away once we default to widening legalization.
+  if (Subtarget.hasSSE41() && VT == MVT::v4i32 &&
+      N->getOpcode() == ISD::VECTOR_SHUFFLE &&
+      N->getOperand(0).getOpcode() == ISD::BITCAST &&
+      N->getOperand(0).getOperand(0).getOpcode() == X86ISD::PMULUDQ) {
+    SDValue BC = N->getOperand(0);
+    SDValue MULUDQ = BC.getOperand(0);
+    ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(N);
+    ArrayRef<int> Mask = SVOp->getMask();
+    if (BC.hasOneUse() && MULUDQ.hasOneUse() &&
+        Mask[0] == 0 && Mask[1] == 2 && Mask[2] == -1 && Mask[3] == -1) {
+      SDValue Op0 = MULUDQ.getOperand(0);
+      SDValue Op1 = MULUDQ.getOperand(1);
+      if (Op0.getOpcode() == ISD::BITCAST &&
+          Op0.getOperand(0).getOpcode() == ISD::VECTOR_SHUFFLE &&
+          Op0.getOperand(0).getValueType() == MVT::v4i32) {
+        ShuffleVectorSDNode *SVOp0 =
+          cast<ShuffleVectorSDNode>(Op0.getOperand(0));
+        ArrayRef<int> Mask2 = SVOp0->getMask();
+        if (Mask2[0] == 0 && Mask2[1] == -1 &&
+            Mask2[2] == 1 && Mask2[3] == -1) {
+          Op0 = SVOp0->getOperand(0);
+          Op1 = DAG.getBitcast(MVT::v4i32, Op1);
+          Op1 = DAG.getVectorShuffle(MVT::v4i32, dl, Op1, Op1, Mask);
+          return DAG.getNode(ISD::MUL, dl, MVT::v4i32, Op0, Op1);
+        }
+      }
+      if (Op1.getOpcode() == ISD::BITCAST &&
+          Op1.getOperand(0).getOpcode() == ISD::VECTOR_SHUFFLE &&
+          Op1.getOperand(0).getValueType() == MVT::v4i32) {
+        ShuffleVectorSDNode *SVOp1 =
+          cast<ShuffleVectorSDNode>(Op1.getOperand(0));
+        ArrayRef<int> Mask2 = SVOp1->getMask();
+        if (Mask2[0] == 0 && Mask2[1] == -1 &&
+            Mask2[2] == 1 && Mask2[3] == -1) {
+          Op0 = DAG.getBitcast(MVT::v4i32, Op0);
+          Op0 = DAG.getVectorShuffle(MVT::v4i32, dl, Op0, Op0, Mask);
+          Op1 = SVOp1->getOperand(0);
+          return DAG.getNode(ISD::MUL, dl, MVT::v4i32, Op0, Op1);
+        }
+      }
+    }
+  }
+
   return SDValue();
 }
 
@@ -32339,6 +32900,27 @@ bool X86TargetLowering::SimplifyDemandedVectorEltsForTargetNode(
       return true;
     break;
   }
+  case X86ISD::BLENDV: {
+    APInt SelUndef, SelZero;
+    if (SimplifyDemandedVectorElts(Op.getOperand(0), DemandedElts, SelUndef,
+                                   SelZero, TLO, Depth + 1))
+      return true;
+
+    // TODO: Use SelZero to adjust LHS/RHS DemandedElts.
+    APInt LHSUndef, LHSZero;
+    if (SimplifyDemandedVectorElts(Op.getOperand(1), DemandedElts, LHSUndef,
+                                   LHSZero, TLO, Depth + 1))
+      return true;
+
+    APInt RHSUndef, RHSZero;
+    if (SimplifyDemandedVectorElts(Op.getOperand(2), DemandedElts, RHSUndef,
+                                   RHSZero, TLO, Depth + 1))
+      return true;
+
+    KnownZero = LHSZero & RHSZero;
+    KnownUndef = LHSUndef & RHSUndef;
+    break;
+  }
   case X86ISD::VBROADCAST: {
     SDValue Src = Op.getOperand(0);
     MVT SrcVT = Src.getSimpleValueType();
@@ -32346,7 +32928,7 @@ bool X86TargetLowering::SimplifyDemandedVectorEltsForTargetNode(
       return false;
     // Don't bother broadcasting if we just need the 0'th element.
     if (DemandedElts == 1) {
-      if(Src.getValueType() != VT)
+      if (Src.getValueType() != VT)
         Src = widenSubVector(VT.getSimpleVT(), Src, false, Subtarget, TLO.DAG,
                              SDLoc(Op));
       return TLO.CombineTo(Op, Src);
@@ -32367,6 +32949,24 @@ bool X86TargetLowering::SimplifyDemandedVectorEltsForTargetNode(
       return true;
     break;
   }
+  case X86ISD::HADD:
+  case X86ISD::HSUB:
+  case X86ISD::FHADD:
+  case X86ISD::FHSUB: {
+    // 256-bit horizontal ops are two 128-bit ops glued together. If we do not
+    // demand any of the high elements, then narrow the h-op to 128-bits:
+    // (hop ymm0, ymm1) --> insert undef, (hop xmm0, xmm1), 0
+    if (VT.is256BitVector() && DemandedElts.lshr(NumElts / 2) == 0) {
+      SDLoc DL(Op);
+      SDValue Ext0 = extract128BitVector(Op.getOperand(0), 0, TLO.DAG, DL);
+      SDValue Ext1 = extract128BitVector(Op.getOperand(1), 0, TLO.DAG, DL);
+      SDValue Hop = TLO.DAG.getNode(Opc, DL, Ext0.getValueType(), Ext0, Ext1);
+      SDValue UndefVec = TLO.DAG.getUNDEF(VT);
+      SDValue Insert = insert128BitVector(UndefVec, Hop, 0, TLO.DAG, DL);
+      return TLO.CombineTo(Op, Insert);
+    }
+    break;
+  }
   }
 
   // Simplify target shuffles.
@@ -32465,15 +33065,38 @@ bool X86TargetLowering::SimplifyDemandedBitsForTargetNode(
     break;
   }
   case X86ISD::VSHLI: {
-    if (auto *ShiftImm = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
+    SDValue Op0 = Op.getOperand(0);
+    SDValue Op1 = Op.getOperand(1);
+
+    if (auto *ShiftImm = dyn_cast<ConstantSDNode>(Op1)) {
       if (ShiftImm->getAPIntValue().uge(BitWidth))
         break;
 
       unsigned ShAmt = ShiftImm->getZExtValue();
       APInt DemandedMask = OriginalDemandedBits.lshr(ShAmt);
 
-      if (SimplifyDemandedBits(Op.getOperand(0), DemandedMask,
-                               OriginalDemandedElts, Known, TLO, Depth + 1))
+      // If this is ((X >>u C1) << ShAmt), see if we can simplify this into a
+      // single shift.  We can do this if the bottom bits (which are shifted
+      // out) are never demanded.
+      if (Op0.getOpcode() == X86ISD::VSRLI &&
+          OriginalDemandedBits.countTrailingZeros() >= ShAmt) {
+        if (auto *Shift2Imm = dyn_cast<ConstantSDNode>(Op0.getOperand(1))) {
+          if (Shift2Imm->getAPIntValue().ult(BitWidth)) {
+            int Diff = ShAmt - Shift2Imm->getZExtValue();
+            if (Diff == 0)
+              return TLO.CombineTo(Op, Op0.getOperand(0));
+
+            unsigned NewOpc = Diff < 0 ? X86ISD::VSRLI : X86ISD::VSHLI;
+            SDValue NewShift = TLO.DAG.getNode(
+                NewOpc, SDLoc(Op), VT, Op0.getOperand(0),
+                TLO.DAG.getConstant(std::abs(Diff), SDLoc(Op), MVT::i8));
+            return TLO.CombineTo(Op, NewShift);
+          }
+        }
+      }
+
+      if (SimplifyDemandedBits(Op0, DemandedMask, OriginalDemandedElts, Known,
+                               TLO, Depth + 1))
         return true;
 
       assert(!Known.hasConflict() && "Bits known to be one AND zero?");
@@ -32556,6 +33179,13 @@ bool X86TargetLowering::SimplifyDemandedBitsForTargetNode(
     }
     break;
   }
+  case X86ISD::PCMPGT:
+    // icmp sgt(0, R) == ashr(R, BitWidth-1).
+    // iff we only need the sign bit then we can use R directly.
+    if (OriginalDemandedBits.isSignMask() &&
+        ISD::isBuildVectorAllZeros(Op.getOperand(0).getNode()))
+      return TLO.CombineTo(Op, Op.getOperand(1));
+    break;
   case X86ISD::MOVMSK: {
     SDValue Src = Op.getOperand(0);
     MVT SrcVT = Src.getSimpleValueType();
@@ -32707,14 +33337,23 @@ static SDValue combineBitcastvxi1(SelectionDAG &DAG, SDValue BitCast,
                                   const X86Subtarget &Subtarget) {
   EVT VT = BitCast.getValueType();
   SDValue N0 = BitCast.getOperand(0);
-  EVT VecVT = N0->getValueType(0);
+  EVT VecVT = N0.getValueType();
 
   if (!VT.isScalarInteger() || !VecVT.isSimple())
     return SDValue();
 
+  // If the input is a truncate from v16i8 or v32i8 go ahead and use a
+  // movmskb even with avx512. This will be better than truncating to vXi1 and
+  // using a kmov. This can especially help KNL if the input is a v16i8/v32i8
+  // vpcmpeqb/vpcmpgtb.
+  bool IsTruncated = N0.getOpcode() == ISD::TRUNCATE && N0.hasOneUse() &&
+                     (N0.getOperand(0).getValueType() == MVT::v16i8 ||
+                      N0.getOperand(0).getValueType() == MVT::v32i8 ||
+                      N0.getOperand(0).getValueType() == MVT::v64i8);
+
   // With AVX512 vxi1 types are legal and we prefer using k-regs.
   // MOVMSK is supported in SSE2 or later.
-  if (Subtarget.hasAVX512() || !Subtarget.hasSSE2())
+  if (!Subtarget.hasSSE2() || (Subtarget.hasAVX512() && !IsTruncated))
     return SDValue();
 
   // There are MOVMSK flavors for types v16i8, v32i8, v4f32, v8f32, v4f64 and
@@ -32738,8 +33377,8 @@ static SDValue combineBitcastvxi1(SelectionDAG &DAG, SDValue BitCast,
     SExtVT = MVT::v4i32;
     // For cases such as (i4 bitcast (v4i1 setcc v4i64 v1, v2))
     // sign-extend to a 256-bit operation to avoid truncation.
-    if (N0->getOpcode() == ISD::SETCC && Subtarget.hasAVX() &&
-        N0->getOperand(0).getValueType().is256BitVector()) {
+    if (N0.getOpcode() == ISD::SETCC && Subtarget.hasAVX() &&
+        N0.getOperand(0).getValueType().is256BitVector()) {
       SExtVT = MVT::v4i64;
     }
     break;
@@ -32750,9 +33389,9 @@ static SDValue combineBitcastvxi1(SelectionDAG &DAG, SDValue BitCast,
     // If the setcc operand is 128-bit, prefer sign-extending to 128-bit over
     // 256-bit because the shuffle is cheaper than sign extending the result of
     // the compare.
-    if (N0->getOpcode() == ISD::SETCC && Subtarget.hasAVX() &&
-        (N0->getOperand(0).getValueType().is256BitVector() ||
-         N0->getOperand(0).getValueType().is512BitVector())) {
+    if (N0.getOpcode() == ISD::SETCC && Subtarget.hasAVX() &&
+        (N0.getOperand(0).getValueType().is256BitVector() ||
+         N0.getOperand(0).getValueType().is512BitVector())) {
       SExtVT = MVT::v8i32;
     }
     break;
@@ -32766,12 +33405,30 @@ static SDValue combineBitcastvxi1(SelectionDAG &DAG, SDValue BitCast,
   case MVT::v32i1:
     SExtVT = MVT::v32i8;
     break;
+  case MVT::v64i1:
+    // If we have AVX512F, but not AVX512BW and the input is truncated from
+    // v64i8 checked earlier. Then split the input and make two pmovmskbs.
+    if (Subtarget.hasAVX512() && !Subtarget.hasBWI()) {
+      SExtVT = MVT::v64i8;
+      break;
+    }
+    return SDValue();
   };
 
   SDLoc DL(BitCast);
   SDValue V = DAG.getNode(ISD::SIGN_EXTEND, DL, SExtVT, N0);
 
-  if (SExtVT == MVT::v16i8 || SExtVT == MVT::v32i8) {
+  if (SExtVT == MVT::v64i8) {
+    SDValue Lo, Hi;
+    std::tie(Lo, Hi) = DAG.SplitVector(V, DL);
+    Lo = DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, Lo);
+    Lo = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, Lo);
+    Hi = DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, Hi);
+    Hi = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Hi);
+    Hi = DAG.getNode(ISD::SHL, DL, MVT::i64, Hi,
+                     DAG.getConstant(32, DL, MVT::i8));
+    V = DAG.getNode(ISD::OR, DL, MVT::i64, Lo, Hi);
+  } else if (SExtVT == MVT::v16i8 || SExtVT == MVT::v32i8) {
     V = getPMOVMSKB(DL, V, DAG, Subtarget);
   } else {
     if (SExtVT == MVT::v8i16)
@@ -33453,15 +34110,19 @@ static SDValue combineExtractWithShuffle(SDNode *N, SelectionDAG &DAG,
   if (SrcSVT == MVT::i1 || !isa<ConstantSDNode>(Idx))
     return SDValue();
 
+  SDValue SrcBC = peekThroughBitcasts(Src);
+
   // Handle extract(broadcast(scalar_value)), it doesn't matter what index is.
-  if (X86ISD::VBROADCAST == Src.getOpcode() &&
-      Src.getOperand(0).getValueType() == VT)
-    return Src.getOperand(0);
+  if (X86ISD::VBROADCAST == SrcBC.getOpcode()) {
+    SDValue SrcOp = SrcBC.getOperand(0);
+    if (SrcOp.getValueSizeInBits() == VT.getSizeInBits())
+      return DAG.getBitcast(VT, SrcOp);
+  }
 
   // Resolve the target shuffle inputs and mask.
   SmallVector<int, 16> Mask;
   SmallVector<SDValue, 2> Ops;
-  if (!resolveTargetShuffleInputs(peekThroughBitcasts(Src), Ops, Mask, DAG))
+  if (!resolveTargetShuffleInputs(SrcBC, Ops, Mask, DAG))
     return SDValue();
 
   // Attempt to narrow/widen the shuffle mask to the correct size.
@@ -33472,6 +34133,15 @@ static SDValue combineExtractWithShuffle(SDNode *N, SelectionDAG &DAG,
       scaleShuffleMask<int>(Scale, Mask, ScaledMask);
       Mask = std::move(ScaledMask);
     } else if ((Mask.size() % NumSrcElts) == 0) {
+      // Simplify Mask based on demanded element.
+      int ExtractIdx = (int)N->getConstantOperandVal(1);
+      int Scale = Mask.size() / NumSrcElts;
+      int Lo = Scale * ExtractIdx;
+      int Hi = Scale * (ExtractIdx + 1);
+      for (int i = 0, e = (int)Mask.size(); i != e; ++i)
+        if (i < Lo || Hi <= i)
+          Mask[i] = SM_SentinelUndef;
+
       SmallVector<int, 16> WidenedMask;
       while (Mask.size() > NumSrcElts &&
              canWidenShuffleElements(Mask, WidenedMask))
@@ -33497,7 +34167,6 @@ static SDValue combineExtractWithShuffle(SDNode *N, SelectionDAG &DAG,
                                 : DAG.getConstant(0, dl, VT);
 
   SDValue SrcOp = Ops[SrcIdx / Mask.size()];
-  SrcOp = DAG.getBitcast(SrcVT, SrcOp);
   SrcIdx = SrcIdx % Mask.size();
 
   // We can only extract other elements from 128-bit vectors and in certain
@@ -33507,6 +34176,7 @@ static SDValue combineExtractWithShuffle(SDNode *N, SelectionDAG &DAG,
   if ((SrcVT == MVT::v4i32 || SrcVT == MVT::v2i64) &&
       ((SrcIdx == 0 && Subtarget.hasSSE2()) || Subtarget.hasSSE41())) {
     assert(SrcSVT == VT && "Unexpected extraction type");
+    SrcOp = DAG.getBitcast(SrcVT, SrcOp);
     return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, SrcSVT, SrcOp,
                        DAG.getIntPtrConstant(SrcIdx, dl));
   }
@@ -33516,6 +34186,7 @@ static SDValue combineExtractWithShuffle(SDNode *N, SelectionDAG &DAG,
     assert(VT.getSizeInBits() >= SrcSVT.getSizeInBits() &&
            "Unexpected extraction type");
     unsigned OpCode = (SrcVT == MVT::v8i16 ? X86ISD::PEXTRW : X86ISD::PEXTRB);
+    SrcOp = DAG.getBitcast(SrcVT, SrcOp);
     SDValue ExtOp = DAG.getNode(OpCode, dl, MVT::i32, SrcOp,
                                 DAG.getIntPtrConstant(SrcIdx, dl));
     return DAG.getZExtOrTrunc(ExtOp, dl, VT);
@@ -33575,8 +34246,7 @@ static SDValue combineExtractVectorElt(SDNode *N, SelectionDAG &DAG,
       isa<ConstantSDNode>(EltIdx) &&
       isa<ConstantSDNode>(InputVector.getOperand(0))) {
     uint64_t ExtractedElt = N->getConstantOperandVal(1);
-    auto *InputC = cast<ConstantSDNode>(InputVector.getOperand(0));
-    const APInt &InputValue = InputC->getAPIntValue();
+    const APInt &InputValue = InputVector.getConstantOperandAPInt(0);
     uint64_t Res = InputValue[ExtractedElt];
     return DAG.getConstant(Res, dl, MVT::i1);
   }
@@ -33641,8 +34311,8 @@ combineVSelectWithAllOnesOrZeros(SDNode *N, SelectionDAG &DAG,
   bool FValIsAllZeros = ISD::isBuildVectorAllZeros(RHS.getNode());
 
   // Try to invert the condition if true value is not all 1s and false value is
-  // not all 0s.
-  if (!TValIsAllOnes && !FValIsAllZeros &&
+  // not all 0s. Only do this if the condition has one use.
+  if (!TValIsAllOnes && !FValIsAllZeros && Cond.hasOneUse() &&
       // Check if the selector will be produced by CMPP*/PCMP*.
       Cond.getOpcode() == ISD::SETCC &&
       // Check if SETCC has already been promoted.
@@ -33766,11 +34436,14 @@ static SDValue combineSelectOfTwoConstants(SDNode *N, SelectionDAG &DAG) {
 /// If this is a *dynamic* select (non-constant condition) and we can match
 /// this node with one of the variable blend instructions, restructure the
 /// condition so that blends can use the high (sign) bit of each element.
-static SDValue combineVSelectToShrunkBlend(SDNode *N, SelectionDAG &DAG,
+/// This function will also call SimplfiyDemandedBits on already created
+/// BLENDV to perform additional simplifications.
+static SDValue combineVSelectToBLENDV(SDNode *N, SelectionDAG &DAG,
                                            TargetLowering::DAGCombinerInfo &DCI,
                                            const X86Subtarget &Subtarget) {
   SDValue Cond = N->getOperand(0);
-  if (N->getOpcode() != ISD::VSELECT ||
+  if ((N->getOpcode() != ISD::VSELECT &&
+       N->getOpcode() != X86ISD::BLENDV) ||
       ISD::isBuildVectorOfConstantSDNodes(Cond.getNode()))
     return SDValue();
 
@@ -33812,7 +34485,9 @@ static SDValue combineVSelectToShrunkBlend(SDNode *N, SelectionDAG &DAG,
   // TODO: Add other opcodes eventually lowered into BLEND.
   for (SDNode::use_iterator UI = Cond->use_begin(), UE = Cond->use_end();
        UI != UE; ++UI)
-    if (UI->getOpcode() != ISD::VSELECT || UI.getOperandNo() != 0)
+    if ((UI->getOpcode() != ISD::VSELECT &&
+         UI->getOpcode() != X86ISD::BLENDV) ||
+        UI.getOperandNo() != 0)
       return SDValue();
 
   APInt DemandedMask(APInt::getSignMask(BitWidth));
@@ -33828,9 +34503,13 @@ static SDValue combineVSelectToShrunkBlend(SDNode *N, SelectionDAG &DAG,
   // optimizations as we messed with the actual expectation for the vector
   // boolean values.
   for (SDNode *U : Cond->uses()) {
-    SDValue SB = DAG.getNode(X86ISD::SHRUNKBLEND, SDLoc(U), U->getValueType(0),
+    if (U->getOpcode() == X86ISD::BLENDV)
+      continue;
+
+    SDValue SB = DAG.getNode(X86ISD::BLENDV, SDLoc(U), U->getValueType(0),
                              Cond, U->getOperand(1), U->getOperand(2));
     DAG.ReplaceAllUsesOfValueWith(SDValue(U, 0), SB);
+    DCI.AddToWorklist(U);
   }
   DCI.CommitTargetLoweringOpt(TLO);
   return SDValue(N, 0);
@@ -33846,7 +34525,7 @@ static SDValue combineSelect(SDNode *N, SelectionDAG &DAG,
   SDValue RHS = N->getOperand(2);
 
   // Try simplification again because we use this function to optimize
-  // SHRUNKBLEND nodes that are not handled by the generic combiner.
+  // BLENDV nodes that are not handled by the generic combiner.
   if (SDValue V = DAG.simplifySelect(Cond, LHS, RHS))
     return V;
 
@@ -34122,14 +34801,16 @@ static SDValue combineSelect(SDNode *N, SelectionDAG &DAG,
           // If the RHS is a constant we have to reverse the const
           // canonicalization.
           // x > C-1 ? x+-C : 0 --> subus x, C
-          // TODO: Handle build_vectors with undef elements.
           auto MatchUSUBSAT = [](ConstantSDNode *Op, ConstantSDNode *Cond) {
-            return Cond->getAPIntValue() == (-Op->getAPIntValue() - 1);
+            return (!Op && !Cond) ||
+                   (Op && Cond &&
+                    Cond->getAPIntValue() == (-Op->getAPIntValue() - 1));
           };
           if (CC == ISD::SETUGT && Other->getOpcode() == ISD::ADD &&
-              ISD::matchBinaryPredicate(OpRHS, CondRHS, MatchUSUBSAT)) {
-            OpRHS = DAG.getNode(ISD::SUB, DL, VT,
-                                DAG.getConstant(0, DL, VT), OpRHS);
+              ISD::matchBinaryPredicate(OpRHS, CondRHS, MatchUSUBSAT,
+                                        /*AllowUndefs*/ true)) {
+            OpRHS = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT),
+                                OpRHS);
             return DAG.getNode(ISD::USUBSAT, DL, VT, OpLHS, OpRHS);
           }
 
@@ -34213,7 +34894,7 @@ static SDValue combineSelect(SDNode *N, SelectionDAG &DAG,
   if (SDValue V = combineVSelectWithAllOnesOrZeros(N, DAG, DCI, Subtarget))
     return V;
 
-  if (SDValue V = combineVSelectToShrunkBlend(N, DAG, DCI, Subtarget))
+  if (SDValue V = combineVSelectToBLENDV(N, DAG, DCI, Subtarget))
     return V;
 
   // Custom action for SELECT MMX
@@ -34295,16 +34976,7 @@ static SDValue combineSetCCAtomicArith(SDValue Cmp, X86::CondCode &CC,
         /*Chain*/ CmpLHS.getOperand(0), /*LHS*/ CmpLHS.getOperand(1),
         /*RHS*/ DAG.getConstant(-Addend, SDLoc(CmpRHS), CmpRHS.getValueType()),
         AN->getMemOperand());
-    // If the comparision uses the CF flag we can't use INC/DEC instructions.
-    bool NeedCF = false;
-    switch (CC) {
-    default: break;
-    case X86::COND_A: case X86::COND_AE:
-    case X86::COND_B: case X86::COND_BE:
-      NeedCF = true;
-      break;
-    }
-    auto LockOp = lowerAtomicArithWithLOCK(AtomicSub, DAG, Subtarget, !NeedCF);
+    auto LockOp = lowerAtomicArithWithLOCK(AtomicSub, DAG, Subtarget);
     DAG.ReplaceAllUsesOfValueWith(CmpLHS.getValue(0),
                                   DAG.getUNDEF(CmpLHS.getValueType()));
     DAG.ReplaceAllUsesOfValueWith(CmpLHS.getValue(1), LockOp.getValue(1));
@@ -34508,7 +35180,7 @@ static bool checkBoolTestAndOrSetCCCombine(SDValue Cond, X86::CondCode &CC0,
 // When legalizing carry, we create carries via add X, -1
 // If that comes from an actual carry, via setcc, we use the
 // carry directly.
-static SDValue combineCarryThroughADD(SDValue EFLAGS) {
+static SDValue combineCarryThroughADD(SDValue EFLAGS, SelectionDAG &DAG) {
   if (EFLAGS.getOpcode() == X86ISD::ADD) {
     if (isAllOnesConstant(EFLAGS.getOperand(1))) {
       SDValue Carry = EFLAGS.getOperand(0);
@@ -34521,8 +35193,34 @@ static SDValue combineCarryThroughADD(SDValue EFLAGS) {
         Carry = Carry.getOperand(0);
       if (Carry.getOpcode() == X86ISD::SETCC ||
           Carry.getOpcode() == X86ISD::SETCC_CARRY) {
-        if (Carry.getConstantOperandVal(0) == X86::COND_B)
-          return Carry.getOperand(1);
+        // TODO: Merge this code with equivalent in combineAddOrSubToADCOrSBB?
+        uint64_t CarryCC = Carry.getConstantOperandVal(0);
+        SDValue CarryOp1 = Carry.getOperand(1);
+        if (CarryCC == X86::COND_B)
+          return CarryOp1;
+        if (CarryCC == X86::COND_A) {
+          // Try to convert COND_A into COND_B in an attempt to facilitate
+          // materializing "setb reg".
+          //
+          // Do not flip "e > c", where "c" is a constant, because Cmp
+          // instruction cannot take an immediate as its first operand.
+          //
+          if (CarryOp1.getOpcode() == X86ISD::SUB &&
+              CarryOp1.getNode()->hasOneUse() &&
+              CarryOp1.getValueType().isInteger() &&
+              !isa<ConstantSDNode>(CarryOp1.getOperand(1))) {
+            SDValue SubCommute =
+                DAG.getNode(X86ISD::SUB, SDLoc(CarryOp1), CarryOp1->getVTList(),
+                            CarryOp1.getOperand(1), CarryOp1.getOperand(0));
+            return SDValue(SubCommute.getNode(), CarryOp1.getResNo());
+          }
+        }
+        // If this is a check of the z flag of an add with 1, switch to the
+        // C flag.
+        if (CarryCC == X86::COND_E &&
+            CarryOp1.getOpcode() == X86ISD::ADD &&
+            isOneConstant(CarryOp1.getOperand(1)))
+          return CarryOp1;
       }
     }
   }
@@ -34537,7 +35235,7 @@ static SDValue combineSetCCEFLAGS(SDValue EFLAGS, X86::CondCode &CC,
                                   SelectionDAG &DAG,
                                   const X86Subtarget &Subtarget) {
   if (CC == X86::COND_B)
-    if (SDValue Flags = combineCarryThroughADD(EFLAGS))
+    if (SDValue Flags = combineCarryThroughADD(EFLAGS, DAG))
       return Flags;
 
   if (SDValue R = checkBoolTestSetCCCombine(EFLAGS, CC))
@@ -34866,86 +35564,85 @@ static SDValue reduceVMULWidth(SDNode *N, SelectionDAG &DAG,
     // Generate the lower part of mul: pmullw. For MULU8/MULS8, only the
     // lower part is needed.
     SDValue MulLo = DAG.getNode(ISD::MUL, DL, ReducedVT, NewN0, NewN1);
-    if (Mode == MULU8 || Mode == MULS8) {
+    if (Mode == MULU8 || Mode == MULS8)
       return DAG.getNode((Mode == MULU8) ? ISD::ZERO_EXTEND : ISD::SIGN_EXTEND,
                          DL, VT, MulLo);
-    } else {
-      MVT ResVT = MVT::getVectorVT(MVT::i32, NumElts / 2);
-      // Generate the higher part of mul: pmulhw/pmulhuw. For MULU16/MULS16,
-      // the higher part is also needed.
-      SDValue MulHi = DAG.getNode(Mode == MULS16 ? ISD::MULHS : ISD::MULHU, DL,
-                                  ReducedVT, NewN0, NewN1);
-
-      // Repack the lower part and higher part result of mul into a wider
-      // result.
-      // Generate shuffle functioning as punpcklwd.
-      SmallVector<int, 16> ShuffleMask(NumElts);
-      for (unsigned i = 0, e = NumElts / 2; i < e; i++) {
-        ShuffleMask[2 * i] = i;
-        ShuffleMask[2 * i + 1] = i + NumElts;
-      }
-      SDValue ResLo =
-          DAG.getVectorShuffle(ReducedVT, DL, MulLo, MulHi, ShuffleMask);
-      ResLo = DAG.getBitcast(ResVT, ResLo);
-      // Generate shuffle functioning as punpckhwd.
-      for (unsigned i = 0, e = NumElts / 2; i < e; i++) {
-        ShuffleMask[2 * i] = i + NumElts / 2;
-        ShuffleMask[2 * i + 1] = i + NumElts * 3 / 2;
-      }
-      SDValue ResHi =
-          DAG.getVectorShuffle(ReducedVT, DL, MulLo, MulHi, ShuffleMask);
-      ResHi = DAG.getBitcast(ResVT, ResHi);
-      return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, ResLo, ResHi);
-    }
-  } else {
-    // When VT.getVectorNumElements() < OpsVT.getVectorNumElements(), we want
-    // to legalize the mul explicitly because implicit legalization for type
-    // <4 x i16> to <4 x i32> sometimes involves unnecessary unpack
-    // instructions which will not exist when we explicitly legalize it by
-    // extending <4 x i16> to <8 x i16> (concatenating the <4 x i16> val with
-    // <4 x i16> undef).
-    //
-    // Legalize the operands of mul.
-    // FIXME: We may be able to handle non-concatenated vectors by insertion.
-    unsigned ReducedSizeInBits = ReducedVT.getSizeInBits();
-    if ((RegSize % ReducedSizeInBits) != 0)
-      return SDValue();
 
-    SmallVector<SDValue, 16> Ops(RegSize / ReducedSizeInBits,
-                                 DAG.getUNDEF(ReducedVT));
-    Ops[0] = NewN0;
-    NewN0 = DAG.getNode(ISD::CONCAT_VECTORS, DL, OpsVT, Ops);
-    Ops[0] = NewN1;
-    NewN1 = DAG.getNode(ISD::CONCAT_VECTORS, DL, OpsVT, Ops);
-
-    if (Mode == MULU8 || Mode == MULS8) {
-      // Generate lower part of mul: pmullw. For MULU8/MULS8, only the lower
-      // part is needed.
-      SDValue Mul = DAG.getNode(ISD::MUL, DL, OpsVT, NewN0, NewN1);
-
-      // convert the type of mul result to VT.
-      MVT ResVT = MVT::getVectorVT(MVT::i32, RegSize / 32);
-      SDValue Res = DAG.getNode(Mode == MULU8 ? ISD::ZERO_EXTEND_VECTOR_INREG
-                                              : ISD::SIGN_EXTEND_VECTOR_INREG,
-                                DL, ResVT, Mul);
-      return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Res,
-                         DAG.getIntPtrConstant(0, DL));
-    } else {
-      // Generate the lower and higher part of mul: pmulhw/pmulhuw. For
-      // MULU16/MULS16, both parts are needed.
-      SDValue MulLo = DAG.getNode(ISD::MUL, DL, OpsVT, NewN0, NewN1);
-      SDValue MulHi = DAG.getNode(Mode == MULS16 ? ISD::MULHS : ISD::MULHU, DL,
-                                  OpsVT, NewN0, NewN1);
-
-      // Repack the lower part and higher part result of mul into a wider
-      // result. Make sure the type of mul result is VT.
-      MVT ResVT = MVT::getVectorVT(MVT::i32, RegSize / 32);
-      SDValue Res = getUnpackl(DAG, DL, OpsVT, MulLo, MulHi);
-      Res = DAG.getBitcast(ResVT, Res);
-      return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Res,
-                         DAG.getIntPtrConstant(0, DL));
-    }
+    MVT ResVT = MVT::getVectorVT(MVT::i32, NumElts / 2);
+    // Generate the higher part of mul: pmulhw/pmulhuw. For MULU16/MULS16,
+    // the higher part is also needed.
+    SDValue MulHi = DAG.getNode(Mode == MULS16 ? ISD::MULHS : ISD::MULHU, DL,
+                                ReducedVT, NewN0, NewN1);
+
+    // Repack the lower part and higher part result of mul into a wider
+    // result.
+    // Generate shuffle functioning as punpcklwd.
+    SmallVector<int, 16> ShuffleMask(NumElts);
+    for (unsigned i = 0, e = NumElts / 2; i < e; i++) {
+      ShuffleMask[2 * i] = i;
+      ShuffleMask[2 * i + 1] = i + NumElts;
+    }
+    SDValue ResLo =
+        DAG.getVectorShuffle(ReducedVT, DL, MulLo, MulHi, ShuffleMask);
+    ResLo = DAG.getBitcast(ResVT, ResLo);
+    // Generate shuffle functioning as punpckhwd.
+    for (unsigned i = 0, e = NumElts / 2; i < e; i++) {
+      ShuffleMask[2 * i] = i + NumElts / 2;
+      ShuffleMask[2 * i + 1] = i + NumElts * 3 / 2;
+    }
+    SDValue ResHi =
+        DAG.getVectorShuffle(ReducedVT, DL, MulLo, MulHi, ShuffleMask);
+    ResHi = DAG.getBitcast(ResVT, ResHi);
+    return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, ResLo, ResHi);
+  }
+
+  // When VT.getVectorNumElements() < OpsVT.getVectorNumElements(), we want
+  // to legalize the mul explicitly because implicit legalization for type
+  // <4 x i16> to <4 x i32> sometimes involves unnecessary unpack
+  // instructions which will not exist when we explicitly legalize it by
+  // extending <4 x i16> to <8 x i16> (concatenating the <4 x i16> val with
+  // <4 x i16> undef).
+  //
+  // Legalize the operands of mul.
+  // FIXME: We may be able to handle non-concatenated vectors by insertion.
+  unsigned ReducedSizeInBits = ReducedVT.getSizeInBits();
+  if ((RegSize % ReducedSizeInBits) != 0)
+    return SDValue();
+
+  SmallVector<SDValue, 16> Ops(RegSize / ReducedSizeInBits,
+                               DAG.getUNDEF(ReducedVT));
+  Ops[0] = NewN0;
+  NewN0 = DAG.getNode(ISD::CONCAT_VECTORS, DL, OpsVT, Ops);
+  Ops[0] = NewN1;
+  NewN1 = DAG.getNode(ISD::CONCAT_VECTORS, DL, OpsVT, Ops);
+
+  if (Mode == MULU8 || Mode == MULS8) {
+    // Generate lower part of mul: pmullw. For MULU8/MULS8, only the lower
+    // part is needed.
+    SDValue Mul = DAG.getNode(ISD::MUL, DL, OpsVT, NewN0, NewN1);
+
+    // convert the type of mul result to VT.
+    MVT ResVT = MVT::getVectorVT(MVT::i32, RegSize / 32);
+    SDValue Res = DAG.getNode(Mode == MULU8 ? ISD::ZERO_EXTEND_VECTOR_INREG
+                                            : ISD::SIGN_EXTEND_VECTOR_INREG,
+                              DL, ResVT, Mul);
+    return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Res,
+                       DAG.getIntPtrConstant(0, DL));
   }
+
+  // Generate the lower and higher part of mul: pmulhw/pmulhuw. For
+  // MULU16/MULS16, both parts are needed.
+  SDValue MulLo = DAG.getNode(ISD::MUL, DL, OpsVT, NewN0, NewN1);
+  SDValue MulHi = DAG.getNode(Mode == MULS16 ? ISD::MULHS : ISD::MULHU, DL,
+                              OpsVT, NewN0, NewN1);
+
+  // Repack the lower part and higher part result of mul into a wider
+  // result. Make sure the type of mul result is VT.
+  MVT ResVT = MVT::getVectorVT(MVT::i32, RegSize / 32);
+  SDValue Res = getUnpackl(DAG, DL, OpsVT, MulLo, MulHi);
+  Res = DAG.getBitcast(ResVT, Res);
+  return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Res,
+                     DAG.getIntPtrConstant(0, DL));
 }
 
 static SDValue combineMulSpecial(uint64_t MulAmt, SDNode *N, SelectionDAG &DAG,
@@ -35094,7 +35791,8 @@ static SDValue combineMulToPMULDQ(SDNode *N, SelectionDAG &DAG,
 
   // Only support vXi64 vectors.
   if (!VT.isVector() || VT.getVectorElementType() != MVT::i64 ||
-      !DAG.getTargetLoweringInfo().isTypeLegal(VT))
+      VT.getVectorNumElements() < 2 ||
+      !isPowerOf2_32(VT.getVectorNumElements()))
     return SDValue();
 
   SDValue N0 = N->getOperand(0);
@@ -35133,26 +35831,6 @@ static SDValue combineMul(SDNode *N, SelectionDAG &DAG,
                           const X86Subtarget &Subtarget) {
   EVT VT = N->getValueType(0);
 
-  // Look for multiply of 2 identical shuffles with a zero vector. Shuffle the
-  // result and insert the zero there instead. This can occur due to
-  // type legalization of v2i32 multiply to a PMULUDQ pattern.
-  SDValue LHS = N->getOperand(0);
-  SDValue RHS = N->getOperand(1);
-  if (!DCI.isBeforeLegalize() && isa<ShuffleVectorSDNode>(LHS) &&
-      isa<ShuffleVectorSDNode>(RHS) && LHS.hasOneUse() && RHS.hasOneUse() &&
-      LHS.getOperand(1) == RHS.getOperand(1) &&
-      ISD::isBuildVectorAllZeros(LHS.getOperand(1).getNode())) {
-    ShuffleVectorSDNode *SVN0 = cast<ShuffleVectorSDNode>(LHS);
-    ShuffleVectorSDNode *SVN1 = cast<ShuffleVectorSDNode>(RHS);
-    if (SVN0->getMask().equals(SVN1->getMask())) {
-      SDLoc dl(N);
-      SDValue Mul = DAG.getNode(ISD::MUL, dl, VT, LHS.getOperand(0),
-                                RHS.getOperand(0));
-      return DAG.getVectorShuffle(VT, dl, Mul, DAG.getConstant(0, dl, VT),
-                                  SVN0->getMask());
-    }
-  }
-
   if (SDValue V = combineMulToPMADDWD(N, DAG, Subtarget))
     return V;
 
@@ -35302,7 +35980,7 @@ static SDValue combineShiftLeft(SDNode *N, SelectionDAG &DAG) {
       N1C && N0.getOpcode() == ISD::AND &&
       N0.getOperand(1).getOpcode() == ISD::Constant) {
     SDValue N00 = N0.getOperand(0);
-    APInt Mask = cast<ConstantSDNode>(N0.getOperand(1))->getAPIntValue();
+    APInt Mask = N0.getConstantOperandAPInt(1);
     Mask <<= N1C->getAPIntValue();
     bool MaskOK = false;
     // We can handle cases concerning bit-widening nodes containing setcc_c if
@@ -35490,8 +36168,8 @@ static SDValue combineVectorPack(SDNode *N, SelectionDAG &DAG,
   // Constant Folding.
   APInt UndefElts0, UndefElts1;
   SmallVector<APInt, 32> EltBits0, EltBits1;
-  if ((N0->isUndef() || N->isOnlyUserOf(N0.getNode())) &&
-      (N1->isUndef() || N->isOnlyUserOf(N1.getNode())) &&
+  if ((N0.isUndef() || N->isOnlyUserOf(N0.getNode())) &&
+      (N1.isUndef() || N->isOnlyUserOf(N1.getNode())) &&
       getTargetConstantBitsFromNode(N0, SrcBitsPerElt, UndefElts0, EltBits0) &&
       getTargetConstantBitsFromNode(N1, SrcBitsPerElt, UndefElts1, EltBits1)) {
     unsigned NumLanes = VT.getSizeInBits() / 128;
@@ -35563,10 +36241,7 @@ static SDValue combineVectorPack(SDNode *N, SelectionDAG &DAG,
 
   // Attempt to combine as shuffle.
   SDValue Op(N, 0);
-  if (SDValue Res =
-          combineX86ShufflesRecursively({Op}, 0, Op, {0}, {}, /*Depth*/ 1,
-                                        /*HasVarMask*/ false,
-                                        /*AllowVarMask*/ true, DAG, Subtarget))
+  if (SDValue Res = combineX86ShufflesRecursively(Op, DAG, Subtarget))
     return Res;
 
   return SDValue();
@@ -35642,9 +36317,7 @@ static SDValue combineVectorShiftImm(SDNode *N, SelectionDAG &DAG,
   // We can decode 'whole byte' logical bit shifts as shuffles.
   if (LogicalShift && (ShiftVal % 8) == 0) {
     SDValue Op(N, 0);
-    if (SDValue Res = combineX86ShufflesRecursively(
-            {Op}, 0, Op, {0}, {}, /*Depth*/ 1,
-            /*HasVarMask*/ false, /*AllowVarMask*/ true, DAG, Subtarget))
+    if (SDValue Res = combineX86ShufflesRecursively(Op, DAG, Subtarget))
       return Res;
   }
 
@@ -35685,10 +36358,7 @@ static SDValue combineVectorInsert(SDNode *N, SelectionDAG &DAG,
 
   // Attempt to combine PINSRB/PINSRW patterns to a shuffle.
   SDValue Op(N, 0);
-  if (SDValue Res =
-          combineX86ShufflesRecursively({Op}, 0, Op, {0}, {}, /*Depth*/ 1,
-                                        /*HasVarMask*/ false,
-                                        /*AllowVarMask*/ true, DAG, Subtarget))
+  if (SDValue Res = combineX86ShufflesRecursively(Op, DAG, Subtarget))
     return Res;
 
   return SDValue();
@@ -35707,8 +36377,8 @@ static SDValue combineCompareEqual(SDNode *N, SelectionDAG &DAG,
   if (Subtarget.hasSSE2() && isAndOrOfSetCCs(SDValue(N, 0U), opcode)) {
     SDValue N0 = N->getOperand(0);
     SDValue N1 = N->getOperand(1);
-    SDValue CMP0 = N0->getOperand(1);
-    SDValue CMP1 = N1->getOperand(1);
+    SDValue CMP0 = N0.getOperand(1);
+    SDValue CMP1 = N1.getOperand(1);
     SDLoc DL(N);
 
     // The SETCCs should both refer to the same CMP.
@@ -35859,7 +36529,7 @@ static SDValue PromoteMaskArithmetic(SDNode *N, SelectionDAG &DAG,
     return SDValue();
 
   // The type of the truncated inputs.
-  if (N0->getOperand(0).getValueType() != VT)
+  if (N0.getOperand(0).getValueType() != VT)
     return SDValue();
 
   // The right side has to be a 'trunc' or a constant vector.
@@ -35875,9 +36545,9 @@ static SDValue PromoteMaskArithmetic(SDNode *N, SelectionDAG &DAG,
     return SDValue();
 
   // Set N0 and N1 to hold the inputs to the new wide operation.
-  N0 = N0->getOperand(0);
+  N0 = N0.getOperand(0);
   if (RHSTrunc)
-    N1 = N1->getOperand(0);
+    N1 = N1.getOperand(0);
   else
     N1 = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, N1);
 
@@ -36205,9 +36875,7 @@ static SDValue combineAnd(SDNode *N, SelectionDAG &DAG,
   // Attempt to recursively combine a bitmask AND with shuffles.
   if (VT.isVector() && (VT.getScalarSizeInBits() % 8) == 0) {
     SDValue Op(N, 0);
-    if (SDValue Res = combineX86ShufflesRecursively(
-            {Op}, 0, Op, {0}, {}, /*Depth*/ 1,
-            /*HasVarMask*/ false, /*AllowVarMask*/ true, DAG, Subtarget))
+    if (SDValue Res = combineX86ShufflesRecursively(Op, DAG, Subtarget))
       return Res;
   }
 
@@ -36253,6 +36921,52 @@ static SDValue combineAnd(SDNode *N, SelectionDAG &DAG,
   return SDValue();
 }
 
+// Canonicalize OR(AND(X,C),AND(Y,~C)) -> OR(AND(X,C),ANDNP(C,Y))
+static SDValue canonicalizeBitSelect(SDNode *N, SelectionDAG &DAG,
+                                     const X86Subtarget &Subtarget) {
+  assert(N->getOpcode() == ISD::OR && "Unexpected Opcode");
+
+  EVT VT = N->getValueType(0);
+  if (!VT.isVector())
+    return SDValue();
+
+  SDValue N0 = peekThroughBitcasts(N->getOperand(0));
+  SDValue N1 = peekThroughBitcasts(N->getOperand(1));
+  if (N0.getOpcode() != ISD::AND || N1.getOpcode() != ISD::AND)
+    return SDValue();
+
+  // On XOP we'll lower to PCMOV so accept one use, otherwise only
+  // do this if either mask has multiple uses already.
+  if (!(Subtarget.hasXOP() || !N0.getOperand(1).hasOneUse() ||
+        !N1.getOperand(1).hasOneUse()))
+    return SDValue();
+
+  // Attempt to extract constant byte masks.
+  APInt UndefElts0, UndefElts1;
+  SmallVector<APInt, 32> EltBits0, EltBits1;
+  if (!getTargetConstantBitsFromNode(N0.getOperand(1), 8, UndefElts0, EltBits0,
+                                     false, false))
+    return SDValue();
+  if (!getTargetConstantBitsFromNode(N1.getOperand(1), 8, UndefElts1, EltBits1,
+                                     false, false))
+    return SDValue();
+
+  for (unsigned i = 0, e = EltBits0.size(); i != e; ++i) {
+    // TODO - add UNDEF elts support.
+    if (UndefElts0[i] || UndefElts1[i])
+      return SDValue();
+    if (EltBits0[i] != ~EltBits1[i])
+      return SDValue();
+  }
+
+  SDLoc DL(N);
+  SDValue X = N->getOperand(0);
+  SDValue Y =
+      DAG.getNode(X86ISD::ANDNP, DL, VT, DAG.getBitcast(VT, N0.getOperand(1)),
+                  DAG.getBitcast(VT, N1.getOperand(0)));
+  return DAG.getNode(ISD::OR, DL, VT, X, Y);
+}
+
 // Try to match OR(AND(~MASK,X),AND(MASK,Y)) logic pattern.
 static bool matchLogicBlend(SDNode *N, SDValue &X, SDValue &Y, SDValue &Mask) {
   if (N->getOpcode() != ISD::OR)
@@ -36515,15 +37229,16 @@ static SDValue combineOr(SDNode *N, SelectionDAG &DAG,
   if (SDValue FPLogic = convertIntLogicToFPLogic(N, DAG, Subtarget))
     return FPLogic;
 
+  if (SDValue R = canonicalizeBitSelect(N, DAG, Subtarget))
+    return R;
+
   if (SDValue R = combineLogicBlendIntoPBLENDV(N, DAG, Subtarget))
     return R;
 
   // Attempt to recursively combine an OR of shuffles.
   if (VT.isVector() && (VT.getScalarSizeInBits() % 8) == 0) {
     SDValue Op(N, 0);
-    if (SDValue Res = combineX86ShufflesRecursively(
-            {Op}, 0, Op, {0}, {}, /*Depth*/ 1,
-            /*HasVarMask*/ false, /*AllowVarMask*/ true, DAG, Subtarget))
+    if (SDValue Res = combineX86ShufflesRecursively(Op, DAG, Subtarget))
       return Res;
   }
 
@@ -36532,6 +37247,7 @@ static SDValue combineOr(SDNode *N, SelectionDAG &DAG,
 
   // fold (or (x << c) | (y >> (64 - c))) ==> (shld64 x, y, c)
   bool OptForSize = DAG.getMachineFunction().getFunction().optForSize();
+  unsigned Bits = VT.getScalarSizeInBits();
 
   // SHLD/SHRD instructions have lower register pressure, but on some
   // platforms they have higher latency than the equivalent
@@ -36554,65 +37270,86 @@ static SDValue combineOr(SDNode *N, SelectionDAG &DAG,
   SDValue ShAmt1 = N1.getOperand(1);
   if (ShAmt1.getValueType() != MVT::i8)
     return SDValue();
+
+  // Peek through any modulo shift masks.
+  SDValue ShMsk0;
+  if (ShAmt0.getOpcode() == ISD::AND &&
+      isa<ConstantSDNode>(ShAmt0.getOperand(1)) &&
+      ShAmt0.getConstantOperandVal(1) == (Bits - 1)) {
+    ShMsk0 = ShAmt0;
+    ShAmt0 = ShAmt0.getOperand(0);
+  }
+  SDValue ShMsk1;
+  if (ShAmt1.getOpcode() == ISD::AND &&
+      isa<ConstantSDNode>(ShAmt1.getOperand(1)) &&
+      ShAmt1.getConstantOperandVal(1) == (Bits - 1)) {
+    ShMsk1 = ShAmt1;
+    ShAmt1 = ShAmt1.getOperand(0);
+  }
+
   if (ShAmt0.getOpcode() == ISD::TRUNCATE)
     ShAmt0 = ShAmt0.getOperand(0);
   if (ShAmt1.getOpcode() == ISD::TRUNCATE)
     ShAmt1 = ShAmt1.getOperand(0);
 
   SDLoc DL(N);
-  unsigned Opc = X86ISD::SHLD;
+  unsigned Opc = ISD::FSHL;
   SDValue Op0 = N0.getOperand(0);
   SDValue Op1 = N1.getOperand(0);
-  if (ShAmt0.getOpcode() == ISD::SUB ||
-      ShAmt0.getOpcode() == ISD::XOR) {
-    Opc = X86ISD::SHRD;
+  if (ShAmt0.getOpcode() == ISD::SUB || ShAmt0.getOpcode() == ISD::XOR) {
+    Opc = ISD::FSHR;
     std::swap(Op0, Op1);
     std::swap(ShAmt0, ShAmt1);
+    std::swap(ShMsk0, ShMsk1);
   }
 
-  // OR( SHL( X, C ), SRL( Y, 32 - C ) ) -> SHLD( X, Y, C )
-  // OR( SRL( X, C ), SHL( Y, 32 - C ) ) -> SHRD( X, Y, C )
-  // OR( SHL( X, C ), SRL( SRL( Y, 1 ), XOR( C, 31 ) ) ) -> SHLD( X, Y, C )
-  // OR( SRL( X, C ), SHL( SHL( Y, 1 ), XOR( C, 31 ) ) ) -> SHRD( X, Y, C )
-  unsigned Bits = VT.getScalarSizeInBits();
+  auto GetFunnelShift = [&DAG, &DL, VT, Opc](SDValue Op0, SDValue Op1,
+                                             SDValue Amt) {
+    if (Opc == ISD::FSHR)
+      std::swap(Op0, Op1);
+    return DAG.getNode(Opc, DL, VT, Op0, Op1,
+                       DAG.getNode(ISD::TRUNCATE, DL, MVT::i8, Amt));
+  };
+
+  // OR( SHL( X, C ), SRL( Y, 32 - C ) ) -> FSHL( X, Y, C )
+  // OR( SRL( X, C ), SHL( Y, 32 - C ) ) -> FSHR( Y, X, C )
+  // OR( SHL( X, C ), SRL( SRL( Y, 1 ), XOR( C, 31 ) ) ) -> FSHL( X, Y, C )
+  // OR( SRL( X, C ), SHL( SHL( Y, 1 ), XOR( C, 31 ) ) ) -> FSHR( Y, X, C )
+  // OR( SHL( X, AND( C, 31 ) ), SRL( Y, AND( 0 - C, 31 ) ) ) -> FSHL( X, Y, C )
+  // OR( SRL( X, AND( C, 31 ) ), SHL( Y, AND( 0 - C, 31 ) ) ) -> FSHR( Y, X, C )
   if (ShAmt1.getOpcode() == ISD::SUB) {
     SDValue Sum = ShAmt1.getOperand(0);
     if (auto *SumC = dyn_cast<ConstantSDNode>(Sum)) {
       SDValue ShAmt1Op1 = ShAmt1.getOperand(1);
       if (ShAmt1Op1.getOpcode() == ISD::TRUNCATE)
         ShAmt1Op1 = ShAmt1Op1.getOperand(0);
-      if (SumC->getSExtValue() == Bits && ShAmt1Op1 == ShAmt0)
-        return DAG.getNode(Opc, DL, VT,
-                           Op0, Op1,
-                           DAG.getNode(ISD::TRUNCATE, DL,
-                                       MVT::i8, ShAmt0));
+      if ((SumC->getAPIntValue() == Bits ||
+           (SumC->getAPIntValue() == 0 && ShMsk1)) &&
+          ShAmt1Op1 == ShAmt0)
+        return GetFunnelShift(Op0, Op1, ShAmt0);
     }
   } else if (auto *ShAmt1C = dyn_cast<ConstantSDNode>(ShAmt1)) {
     auto *ShAmt0C = dyn_cast<ConstantSDNode>(ShAmt0);
     if (ShAmt0C && (ShAmt0C->getSExtValue() + ShAmt1C->getSExtValue()) == Bits)
-      return DAG.getNode(Opc, DL, VT,
-                         N0.getOperand(0), N1.getOperand(0),
-                         DAG.getNode(ISD::TRUNCATE, DL,
-                                       MVT::i8, ShAmt0));
+      return GetFunnelShift(Op0, Op1, ShAmt0);
   } else if (ShAmt1.getOpcode() == ISD::XOR) {
     SDValue Mask = ShAmt1.getOperand(1);
     if (auto *MaskC = dyn_cast<ConstantSDNode>(Mask)) {
-      unsigned InnerShift = (X86ISD::SHLD == Opc ? ISD::SRL : ISD::SHL);
+      unsigned InnerShift = (ISD::FSHL == Opc ? ISD::SRL : ISD::SHL);
       SDValue ShAmt1Op0 = ShAmt1.getOperand(0);
       if (ShAmt1Op0.getOpcode() == ISD::TRUNCATE)
         ShAmt1Op0 = ShAmt1Op0.getOperand(0);
-      if (MaskC->getSExtValue() == (Bits - 1) && ShAmt1Op0 == ShAmt0) {
+      if (MaskC->getSExtValue() == (Bits - 1) &&
+          (ShAmt1Op0 == ShAmt0 || ShAmt1Op0 == ShMsk0)) {
         if (Op1.getOpcode() == InnerShift &&
             isa<ConstantSDNode>(Op1.getOperand(1)) &&
             Op1.getConstantOperandVal(1) == 1) {
-          return DAG.getNode(Opc, DL, VT, Op0, Op1.getOperand(0),
-                             DAG.getNode(ISD::TRUNCATE, DL, MVT::i8, ShAmt0));
+          return GetFunnelShift(Op0, Op1.getOperand(0), ShAmt0);
         }
         // Test for ADD( Y, Y ) as an equivalent to SHL( Y, 1 ).
         if (InnerShift == ISD::SHL && Op1.getOpcode() == ISD::ADD &&
             Op1.getOperand(0) == Op1.getOperand(1)) {
-          return DAG.getNode(Opc, DL, VT, Op0, Op1.getOperand(0),
-                     DAG.getNode(ISD::TRUNCATE, DL, MVT::i8, ShAmt0));
+          return GetFunnelShift(Op0, Op1.getOperand(0), ShAmt0);
         }
       }
     }
@@ -36883,6 +37620,7 @@ static SDValue combineTruncateWithSat(SDValue In, EVT VT, const SDLoc &DL,
       return DAG.getNode(X86ISD::VTRUNCUS, DL, VT, USatVal);
   }
   if (VT.isVector() && isPowerOf2_32(VT.getVectorNumElements()) &&
+      !Subtarget.hasAVX512() &&
       (SVT == MVT::i8 || SVT == MVT::i16) &&
       (InSVT == MVT::i16 || InSVT == MVT::i32)) {
     if (auto USatVal = detectSSatPattern(In, VT, true)) {
@@ -37225,7 +37963,7 @@ static SDValue combineMaskedLoad(SDNode *N, SelectionDAG &DAG,
         return Blend;
   }
 
-  if (Mld->getExtensionType() != ISD::SEXTLOAD)
+  if (Mld->getExtensionType() != ISD::EXTLOAD)
     return SDValue();
 
   // Resolve extending loads.
@@ -37295,8 +38033,20 @@ static SDValue combineMaskedLoad(SDNode *N, SelectionDAG &DAG,
                                      Mld->getBasePtr(), NewMask, WidePassThru,
                                      Mld->getMemoryVT(), Mld->getMemOperand(),
                                      ISD::NON_EXTLOAD);
-  SDValue NewVec = getExtendInVec(/*Signed*/true, dl, VT, WideLd, DAG);
-  return DCI.CombineTo(N, NewVec, WideLd.getValue(1), true);
+
+  SDValue SlicedVec = DAG.getBitcast(WideVecVT, WideLd);
+  SmallVector<int, 16> ShuffleVec(NumElems * SizeRatio, -1);
+  for (unsigned i = 0; i != NumElems; ++i)
+    ShuffleVec[i * SizeRatio] = i;
+
+  // Can't shuffle using an illegal type.
+  assert(DAG.getTargetLoweringInfo().isTypeLegal(WideVecVT) &&
+         "WideVecVT should be legal");
+  SlicedVec = DAG.getVectorShuffle(WideVecVT, dl, SlicedVec,
+                                   DAG.getUNDEF(WideVecVT), ShuffleVec);
+  SlicedVec = DAG.getBitcast(VT, SlicedVec);
+
+  return DCI.CombineTo(N, SlicedVec, WideLd.getValue(1), true);
 }
 
 /// If exactly one element of the mask is set for a non-truncating masked store,
@@ -37334,6 +38084,10 @@ static SDValue combineMaskedStore(SDNode *N, SelectionDAG &DAG,
     return SDValue();
 
   EVT VT = Mst->getValue().getValueType();
+  EVT StVT = Mst->getMemoryVT();
+  SDLoc dl(Mst);
+  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
+
   if (!Mst->isTruncatingStore()) {
     if (SDValue ScalarStore = reduceMaskedStoreToScalarStore(Mst, DAG))
       return ScalarStore;
@@ -37342,7 +38096,6 @@ static SDValue combineMaskedStore(SDNode *N, SelectionDAG &DAG,
     // simplify ops leading up to it. We only demand the MSB of each lane.
     SDValue Mask = Mst->getMask();
     if (Mask.getScalarValueSizeInBits() != 1) {
-      const TargetLowering &TLI = DAG.getTargetLoweringInfo();
       APInt DemandedMask(APInt::getSignMask(VT.getScalarSizeInBits()));
       if (TLI.SimplifyDemandedBits(Mask, DemandedMask, DCI))
         return SDValue(N, 0);
@@ -37352,20 +38105,25 @@ static SDValue combineMaskedStore(SDNode *N, SelectionDAG &DAG,
     // pattern above, but that pattern will be different. It will either need to
     // match setcc more generally or match PCMPGTM later (in tablegen?).
 
+    SDValue Value = Mst->getValue();
+    if (Value.getOpcode() == ISD::TRUNCATE && Value.getNode()->hasOneUse() &&
+        TLI.isTruncStoreLegal(Value.getOperand(0).getValueType(),
+                              Mst->getMemoryVT())) {
+      return DAG.getMaskedStore(Mst->getChain(), SDLoc(N), Value.getOperand(0),
+                                Mst->getBasePtr(), Mask,
+                                Mst->getMemoryVT(), Mst->getMemOperand(), true);
+    }
+
     return SDValue();
   }
 
   // Resolve truncating stores.
   unsigned NumElems = VT.getVectorNumElements();
-  EVT StVT = Mst->getMemoryVT();
-  SDLoc dl(Mst);
 
   assert(StVT != VT && "Cannot truncate to the same type");
   unsigned FromSz = VT.getScalarSizeInBits();
   unsigned ToSz = StVT.getScalarSizeInBits();
 
-  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
-
   // The truncating store is legal in some cases. For example
   // vpmovqb, vpmovqw, vpmovqd, vpmovdb, vpmovdw
   // are designated for truncate store.
@@ -37866,16 +38624,6 @@ static bool isHorizontalBinOp(SDValue &LHS, SDValue &RHS, bool IsCommutative) {
   return true;
 }
 
-/// Horizontal vector math instructions may be slower than normal math with
-/// shuffles. Limit horizontal op codegen based on size/speed trade-offs, uarch
-/// implementation, and likely shuffle complexity of the alternate sequence.
-static bool shouldCombineToHorizontalOp(bool IsSingleSource, SelectionDAG &DAG,
-                                        const X86Subtarget &Subtarget) {
-  bool IsOptimizingSize = DAG.getMachineFunction().getFunction().optForSize();
-  bool HasFastHOps = Subtarget.hasFastHorizontalOps();
-  return !IsSingleSource || IsOptimizingSize || HasFastHOps;
-}
-
 /// Do target-specific dag combines on floating-point adds/subs.
 static SDValue combineFaddFsub(SDNode *N, SelectionDAG &DAG,
                                const X86Subtarget &Subtarget) {
@@ -37883,16 +38631,16 @@ static SDValue combineFaddFsub(SDNode *N, SelectionDAG &DAG,
   SDValue LHS = N->getOperand(0);
   SDValue RHS = N->getOperand(1);
   bool IsFadd = N->getOpcode() == ISD::FADD;
+  auto HorizOpcode = IsFadd ? X86ISD::FHADD : X86ISD::FHSUB;
   assert((IsFadd || N->getOpcode() == ISD::FSUB) && "Wrong opcode");
 
   // Try to synthesize horizontal add/sub from adds/subs of shuffles.
   if (((Subtarget.hasSSE3() && (VT == MVT::v4f32 || VT == MVT::v2f64)) ||
        (Subtarget.hasAVX() && (VT == MVT::v8f32 || VT == MVT::v4f64))) &&
       isHorizontalBinOp(LHS, RHS, IsFadd) &&
-      shouldCombineToHorizontalOp(LHS == RHS, DAG, Subtarget)) {
-    auto NewOpcode = IsFadd ? X86ISD::FHADD : X86ISD::FHSUB;
-    return DAG.getNode(NewOpcode, SDLoc(N), VT, LHS, RHS);
-  }
+      shouldUseHorizontalOp(LHS == RHS, DAG, Subtarget))
+    return DAG.getNode(HorizOpcode, SDLoc(N), VT, LHS, RHS);
+
   return SDValue();
 }
 
@@ -38119,8 +38867,7 @@ static SDValue combineVectorSignBitsTruncation(SDNode *N, const SDLoc &DL,
 
   // Use PACKUS if the input has zero-bits that extend all the way to the
   // packed/truncated value. e.g. masks, zext_in_reg, etc.
-  KnownBits Known;
-  DAG.computeKnownBits(In, Known);
+  KnownBits Known = DAG.computeKnownBits(In);
   unsigned NumLeadingZeroBits = Known.countMinLeadingZeros();
   if (NumLeadingZeroBits >= (InSVT.getSizeInBits() - NumPackedZeroBits))
     return truncateVectorWithPACK(X86ISD::PACKUS, VT, In, DL, DAG, Subtarget);
@@ -38816,9 +39563,7 @@ static SDValue combineAndnp(SDNode *N, SelectionDAG &DAG,
   // Attempt to recursively combine a bitmask ANDNP with shuffles.
   if (VT.isVector() && (VT.getScalarSizeInBits() % 8) == 0) {
     SDValue Op(N, 0);
-    if (SDValue Res = combineX86ShufflesRecursively(
-            {Op}, 0, Op, {0}, {}, /*Depth*/ 1,
-            /*HasVarMask*/ false, /*AllowVarMask*/ true, DAG, Subtarget))
+    if (SDValue Res = combineX86ShufflesRecursively(Op, DAG, Subtarget))
       return Res;
   }
 
@@ -39259,7 +40004,7 @@ static SDValue combineExtSetcc(SDNode *N, SelectionDAG &DAG,
   SDLoc dl(N);
 
   // Only do this combine with AVX512 for vector extends.
-  if (!Subtarget.hasAVX512() || !VT.isVector() || N0->getOpcode() != ISD::SETCC)
+  if (!Subtarget.hasAVX512() || !VT.isVector() || N0.getOpcode() != ISD::SETCC)
     return SDValue();
 
   // Only combine legal element types.
@@ -39275,7 +40020,7 @@ static SDValue combineExtSetcc(SDNode *N, SelectionDAG &DAG,
 
   // Don't fold if the condition code can't be handled by PCMPEQ/PCMPGT since
   // that's the only integer compares with we have.
-  ISD::CondCode CC = cast<CondCodeSDNode>(N0->getOperand(2))->get();
+  ISD::CondCode CC = cast<CondCodeSDNode>(N0.getOperand(2))->get();
   if (ISD::isUnsignedIntSetCC(CC))
     return SDValue();
 
@@ -39681,9 +40426,8 @@ static SDValue combineMOVMSK(SDNode *N, SelectionDAG &DAG,
     assert(VT== MVT::i32 && "Unexpected result type");
     APInt Imm(32, 0);
     for (unsigned Idx = 0, e = Src.getNumOperands(); Idx < e; ++Idx) {
-      SDValue In = Src.getOperand(Idx);
-      if (!In.isUndef() &&
-          cast<ConstantSDNode>(In)->getAPIntValue().isNegative())
+      if (!Src.getOperand(Idx).isUndef() &&
+          Src.getConstantOperandAPInt(Idx).isNegative())
         Imm.setBit(Idx);
     }
     return DAG.getConstant(Imm, SDLoc(N), VT);
@@ -39867,8 +40611,7 @@ static SDValue combineVectorCompareAndMaskUnaryOp(SDNode *N,
   // make the transformation for non-constant splats as well, but it's unclear
   // that would be a benefit as it would not eliminate any operations, just
   // perform one more step in scalar code before moving to the vector unit.
-  if (BuildVectorSDNode *BV =
-          dyn_cast<BuildVectorSDNode>(N->getOperand(0)->getOperand(1))) {
+  if (auto *BV = dyn_cast<BuildVectorSDNode>(N->getOperand(0).getOperand(1))) {
     // Bail out if the vector isn't a constant.
     if (!BV->isConstant())
       return SDValue();
@@ -40139,8 +40882,24 @@ static SDValue combineCMP(SDNode *N, SelectionDAG &DAG) {
   return Op.getValue(1);
 }
 
+static SDValue combineX86AddSub(SDNode *N, SelectionDAG &DAG) {
+  assert((X86ISD::ADD == N->getOpcode() || X86ISD::SUB == N->getOpcode()) &&
+         "Expected X86ISD::ADD or X86ISD::SUB");
+
+  // If we don't use the flag result, simplify back to a simple ADD/SUB.
+  if (N->hasAnyUseOfValue(1))
+    return SDValue();
+
+  SDLoc DL(N);
+  SDValue LHS = N->getOperand(0);
+  SDValue RHS = N->getOperand(1);
+  SDValue Res = DAG.getNode(X86ISD::ADD == N->getOpcode() ? ISD::ADD : ISD::SUB,
+                            DL, LHS.getSimpleValueType(), LHS, RHS);
+  return DAG.getMergeValues({Res, DAG.getConstant(0, DL, MVT::i32)}, DL);
+}
+
 static SDValue combineSBB(SDNode *N, SelectionDAG &DAG) {
-  if (SDValue Flags = combineCarryThroughADD(N->getOperand(2))) {
+  if (SDValue Flags = combineCarryThroughADD(N->getOperand(2), DAG)) {
     MVT VT = N->getSimpleValueType(0);
     SDVTList VTs = DAG.getVTList(VT, MVT::i32);
     return DAG.getNode(X86ISD::SBB, SDLoc(N), VTs,
@@ -40148,6 +40907,15 @@ static SDValue combineSBB(SDNode *N, SelectionDAG &DAG) {
                        Flags);
   }
 
+  // Fold SBB(SUB(X,Y),0,Carry) -> SBB(X,Y,Carry)
+  // iff the flag result is dead.
+  SDValue Op0 = N->getOperand(0);
+  SDValue Op1 = N->getOperand(1);
+  if (Op0.getOpcode() == ISD::SUB && isNullConstant(Op1) &&
+      !N->hasAnyUseOfValue(1))
+    return DAG.getNode(X86ISD::SBB, SDLoc(N), N->getVTList(), Op0.getOperand(0),
+                       Op0.getOperand(1), N->getOperand(2));
+
   return SDValue();
 }
 
@@ -40174,7 +40942,7 @@ static SDValue combineADC(SDNode *N, SelectionDAG &DAG,
     return DCI.CombineTo(N, Res1, CarryOut);
   }
 
-  if (SDValue Flags = combineCarryThroughADD(N->getOperand(2))) {
+  if (SDValue Flags = combineCarryThroughADD(N->getOperand(2), DAG)) {
     MVT VT = N->getSimpleValueType(0);
     SDVTList VTs = DAG.getVTList(VT, MVT::i32);
     return DAG.getNode(X86ISD::ADC, SDLoc(N), VTs,
@@ -40270,7 +41038,7 @@ static SDValue combineAddOrSubToADCOrSBB(SDNode *N, SelectionDAG &DAG) {
     // Do not flip "e > c", where "c" is a constant, because Cmp instruction
     // cannot take an immediate as its first operand.
     //
-    if (EFLAGS.getOpcode() == X86ISD::SUB && EFLAGS.hasOneUse() &&
+    if (EFLAGS.getOpcode() == X86ISD::SUB && EFLAGS.getNode()->hasOneUse() &&
         EFLAGS.getValueType().isInteger() &&
         !isa<ConstantSDNode>(EFLAGS.getOperand(1))) {
       SDValue NewSub = DAG.getNode(X86ISD::SUB, SDLoc(EFLAGS),
@@ -40617,39 +41385,6 @@ static SDValue matchPMADDWD(SelectionDAG &DAG, SDValue Op0, SDValue Op1,
                           PMADDBuilder);
 }
 
-// Try to turn (add (umax X, C), -C) into (psubus X, C)
-static SDValue combineAddToSUBUS(SDNode *N, SelectionDAG &DAG,
-                                 const X86Subtarget &Subtarget) {
-  if (!Subtarget.hasSSE2())
-    return SDValue();
-
-  EVT VT = N->getValueType(0);
-
-  // psubus is available in SSE2 for i8 and i16 vectors.
-  if (!VT.isVector() || VT.getVectorNumElements() < 2 ||
-      !isPowerOf2_32(VT.getVectorNumElements()) ||
-      !(VT.getVectorElementType() == MVT::i8 ||
-        VT.getVectorElementType() == MVT::i16))
-    return SDValue();
-
-  SDValue Op0 = N->getOperand(0);
-  SDValue Op1 = N->getOperand(1);
-  if (Op0.getOpcode() != ISD::UMAX)
-    return SDValue();
-
-  // The add should have a constant that is the negative of the max.
-  // TODO: Handle build_vectors with undef elements.
-  auto MatchUSUBSAT = [](ConstantSDNode *Max, ConstantSDNode *Op) {
-    return Max->getAPIntValue() == (-Op->getAPIntValue());
-  };
-  if (!ISD::matchBinaryPredicate(Op0.getOperand(1), Op1, MatchUSUBSAT))
-    return SDValue();
-
-  SDLoc DL(N);
-  return DAG.getNode(ISD::USUBSAT, DL, VT, Op0.getOperand(0),
-                     Op0.getOperand(1));
-}
-
 // Attempt to turn this pattern into PMADDWD.
 // (mul (add (zext (build_vector)), (zext (build_vector))),
 //      (add (zext (build_vector)), (zext (build_vector)))
@@ -40793,7 +41528,7 @@ static SDValue combineAdd(SDNode *N, SelectionDAG &DAG,
   if ((VT == MVT::v8i16 || VT == MVT::v4i32 || VT == MVT::v16i16 ||
        VT == MVT::v8i32) &&
       Subtarget.hasSSSE3() && isHorizontalBinOp(Op0, Op1, true) &&
-      shouldCombineToHorizontalOp(Op0 == Op1, DAG, Subtarget)) {
+      shouldUseHorizontalOp(Op0 == Op1, DAG, Subtarget)) {
     auto HADDBuilder = [](SelectionDAG &DAG, const SDLoc &DL,
                           ArrayRef<SDValue> Ops) {
       return DAG.getNode(X86ISD::HADD, DL, Ops[0].getValueType(), Ops);
@@ -40805,9 +41540,6 @@ static SDValue combineAdd(SDNode *N, SelectionDAG &DAG,
   if (SDValue V = combineIncDecVector(N, DAG))
     return V;
 
-  if (SDValue V = combineAddToSUBUS(N, DAG, Subtarget))
-    return V;
-
   return combineAddOrSubToADCOrSBB(N, DAG);
 }
 
@@ -40868,8 +41600,7 @@ static SDValue combineSubToSubus(SDNode *N, SelectionDAG &DAG,
   // if the value was zero extended from 16 bit,
   // so we require first 16 bits to be zeros for 32 bit
   // values, or first 48 bits for 64 bit values.
-  KnownBits Known;
-  DAG.computeKnownBits(SubusLHS, Known);
+  KnownBits Known = DAG.computeKnownBits(SubusLHS);
   unsigned NumZeros = Known.countMinLeadingZeros();
   if ((VT == MVT::v8i64 && NumZeros < 48) || NumZeros < 16)
     return SDValue();
@@ -40913,7 +41644,7 @@ static SDValue combineSub(SDNode *N, SelectionDAG &DAG,
     // X-Y -> X+~Y+1, saving one register.
     if (Op1->hasOneUse() && Op1.getOpcode() == ISD::XOR &&
         isa<ConstantSDNode>(Op1.getOperand(1))) {
-      APInt XorC = cast<ConstantSDNode>(Op1.getOperand(1))->getAPIntValue();
+      const APInt &XorC = Op1.getConstantOperandAPInt(1);
       EVT VT = Op0.getValueType();
       SDValue NewXor = DAG.getNode(ISD::XOR, SDLoc(Op1), VT,
                                    Op1.getOperand(0),
@@ -40928,7 +41659,7 @@ static SDValue combineSub(SDNode *N, SelectionDAG &DAG,
   if ((VT == MVT::v8i16 || VT == MVT::v4i32 || VT == MVT::v16i16 ||
        VT == MVT::v8i32) &&
       Subtarget.hasSSSE3() && isHorizontalBinOp(Op0, Op1, false) &&
-      shouldCombineToHorizontalOp(Op0 == Op1, DAG, Subtarget)) {
+      shouldUseHorizontalOp(Op0 == Op1, DAG, Subtarget)) {
     auto HSUBBuilder = [](SelectionDAG &DAG, const SDLoc &DL,
                           ArrayRef<SDValue> Ops) {
       return DAG.getNode(X86ISD::HSUB, DL, Ops[0].getValueType(), Ops);
@@ -41051,6 +41782,42 @@ static SDValue combineInsertSubvector(SDNode *N, SelectionDAG &DAG,
     }
   }
 
+  // Push subvector bitcasts to the output, adjusting the index as we go.
+  // insert_subvector(bitcast(v), bitcast(s), c1) ->
+  // bitcast(insert_subvector(v,s,c2))
+  // TODO: Move this to generic - which only supports same scalar sizes.
+  if ((Vec.isUndef() || Vec.getOpcode() == ISD::BITCAST) &&
+      SubVec.getOpcode() == ISD::BITCAST) {
+    SDValue VecSrc = peekThroughBitcasts(Vec);
+    SDValue SubVecSrc = peekThroughBitcasts(SubVec);
+    MVT VecSrcSVT = VecSrc.getSimpleValueType().getScalarType();
+    MVT SubVecSrcSVT = SubVecSrc.getSimpleValueType().getScalarType();
+    if (Vec.isUndef() || VecSrcSVT == SubVecSrcSVT) {
+      MVT NewOpVT;
+      SDValue NewIdx;
+      unsigned NumElts = OpVT.getVectorNumElements();
+      unsigned EltSizeInBits = OpVT.getScalarSizeInBits();
+      if ((EltSizeInBits % SubVecSrcSVT.getSizeInBits()) == 0) {
+        unsigned Scale = EltSizeInBits / SubVecSrcSVT.getSizeInBits();
+        NewOpVT = MVT::getVectorVT(SubVecSrcSVT, NumElts * Scale);
+        NewIdx = DAG.getIntPtrConstant(IdxVal * Scale, dl);
+      } else if ((SubVecSrcSVT.getSizeInBits() % EltSizeInBits) == 0) {
+        unsigned Scale = SubVecSrcSVT.getSizeInBits() / EltSizeInBits;
+        if ((IdxVal % Scale) == 0) {
+          NewOpVT = MVT::getVectorVT(SubVecSrcSVT, NumElts / Scale);
+          NewIdx = DAG.getIntPtrConstant(IdxVal / Scale, dl);
+        }
+      }
+      if (NewIdx && DAG.getTargetLoweringInfo().isOperationLegal(
+                        ISD::INSERT_SUBVECTOR, NewOpVT)) {
+        SDValue Res = DAG.getBitcast(NewOpVT, VecSrc);
+        Res = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, NewOpVT, Res, SubVecSrc,
+                          NewIdx);
+        return DAG.getBitcast(OpVT, Res);
+      }
+    }
+  }
+
   // Fold two 16-byte or 32-byte subvector loads into one 32-byte or 64-byte
   // load:
   // (insert_subvector (insert_subvector undef, (load16 addr), 0),
@@ -41093,11 +41860,11 @@ static SDValue combineInsertSubvector(SDNode *N, SelectionDAG &DAG,
         if (SubVec2 == SubVec && ISD::isNormalLoad(Ld) && Vec.hasOneUse())
           return DAG.getNode(X86ISD::SUBV_BROADCAST, dl, OpVT, SubVec);
 
-      // If this is subv_broadcast insert into both halves, use a larger
-      // subv_broadcast.
-      if (SubVec.getOpcode() == X86ISD::SUBV_BROADCAST && SubVec == SubVec2)
-        return DAG.getNode(X86ISD::SUBV_BROADCAST, dl, OpVT,
-                           SubVec.getOperand(0));
+      // If this broadcast/subv_broadcast is inserted into both halves, use a
+      // larger broadcast/subv_broadcast.
+      if (SubVec == SubVec2 && (SubVec.getOpcode() == X86ISD::VBROADCAST ||
+                                SubVec.getOpcode() == X86ISD::SUBV_BROADCAST))
+        return DAG.getNode(SubVec.getOpcode(), dl, OpVT, SubVec.getOperand(0));
 
       // If we're inserting all zeros into the upper half, change this to
       // an insert into an all zeros vector. We will match this to a move
@@ -41116,11 +41883,14 @@ static SDValue combineInsertSubvector(SDNode *N, SelectionDAG &DAG,
                           SubVec2, Vec.getOperand(2));
         return DAG.getNode(ISD::INSERT_SUBVECTOR, dl, OpVT, Vec, SubVec,
                            N->getOperand(2));
-
       }
     }
   }
 
+  // If this is a broadcast insert into an upper undef, use a larger broadcast.
+  if (Vec.isUndef() && IdxVal != 0 && SubVec.getOpcode() == X86ISD::VBROADCAST)
+    return DAG.getNode(X86ISD::VBROADCAST, dl, OpVT, SubVec.getOperand(0));
+
   return SDValue();
 }
 
@@ -41242,6 +42012,15 @@ static SDValue combineScalarToVector(SDNode *N, SelectionDAG &DAG) {
         return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), MVT::v1i1,
                            Src.getOperand(0));
 
+  // Combine scalar_to_vector of an extract_vector_elt into an extract_subvec.
+  if (VT == MVT::v1i1 && Src.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
+      Src.hasOneUse() && Src.getOperand(0).getValueType().isVector() &&
+      Src.getOperand(0).getValueType().getVectorElementType() == MVT::i1)
+    if (auto *C = dyn_cast<ConstantSDNode>(Src.getOperand(1)))
+      if (C->isNullValue())
+        return DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N), VT,
+                           Src.getOperand(0), Src.getOperand(1));
+
   return SDValue();
 }
 
@@ -41260,6 +42039,15 @@ static SDValue combinePMULDQ(SDNode *N, SelectionDAG &DAG,
   if (ISD::isBuildVectorAllZeros(RHS.getNode()))
     return RHS;
 
+  // Aggressively peek through ops to get at the demanded low bits.
+  APInt DemandedMask = APInt::getLowBitsSet(64, 32);
+  SDValue DemandedLHS = DAG.GetDemandedBits(LHS, DemandedMask);
+  SDValue DemandedRHS = DAG.GetDemandedBits(RHS, DemandedMask);
+  if (DemandedLHS || DemandedRHS)
+    return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0),
+                       DemandedLHS ? DemandedLHS : LHS,
+                       DemandedRHS ? DemandedRHS : RHS);
+
   // PMULDQ/PMULUDQ only uses lower 32 bits from each vector element.
   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
   if (TLI.SimplifyDemandedBits(SDValue(N, 0), APInt::getAllOnesValue(64), DCI))
@@ -41285,12 +42073,14 @@ SDValue X86TargetLowering::PerformDAGCombine(SDNode *N,
     return combineExtractSubvector(N, DAG, DCI, Subtarget);
   case ISD::VSELECT:
   case ISD::SELECT:
-  case X86ISD::SHRUNKBLEND: return combineSelect(N, DAG, DCI, Subtarget);
+  case X86ISD::BLENDV:      return combineSelect(N, DAG, DCI, Subtarget);
   case ISD::BITCAST:        return combineBitcast(N, DAG, DCI, Subtarget);
   case X86ISD::CMOV:        return combineCMov(N, DAG, DCI, Subtarget);
   case X86ISD::CMP:         return combineCMP(N, DAG);
   case ISD::ADD:            return combineAdd(N, DAG, Subtarget);
   case ISD::SUB:            return combineSub(N, DAG, Subtarget);
+  case X86ISD::ADD:
+  case X86ISD::SUB:         return combineX86AddSub(N, DAG);
   case X86ISD::SBB:         return combineSBB(N, DAG);
   case X86ISD::ADC:         return combineADC(N, DAG, DCI);
   case ISD::MUL:            return combineMul(N, DAG, DCI, Subtarget);
@@ -41320,7 +42110,7 @@ SDValue X86TargetLowering::PerformDAGCombine(SDNode *N,
   case X86ISD::FMAX:        return combineFMinFMax(N, DAG);
   case ISD::FMINNUM:
   case ISD::FMAXNUM:        return combineFMinNumFMaxNum(N, DAG, Subtarget);
-  case X86ISD::CVTSI2P:  
+  case X86ISD::CVTSI2P:
   case X86ISD::CVTUI2P:     return combineX86INT_TO_FP(N, DAG, DCI);
   case X86ISD::BT:          return combineBT(N, DAG, DCI);
   case ISD::ANY_EXTEND:
@@ -41674,6 +42464,40 @@ bool X86TargetLowering::ExpandInlineAsm(CallInst *CI) const {
   return false;
 }
 
+static X86::CondCode parseConstraintCode(llvm::StringRef Constraint) {
+  X86::CondCode Cond = StringSwitch<X86::CondCode>(Constraint)
+                           .Case("{@cca}", X86::COND_A)
+                           .Case("{@ccae}", X86::COND_AE)
+                           .Case("{@ccb}", X86::COND_B)
+                           .Case("{@ccbe}", X86::COND_BE)
+                           .Case("{@ccc}", X86::COND_B)
+                           .Case("{@cce}", X86::COND_E)
+                           .Case("{@ccz}", X86::COND_NE)
+                           .Case("{@ccg}", X86::COND_G)
+                           .Case("{@ccge}", X86::COND_GE)
+                           .Case("{@ccl}", X86::COND_L)
+                           .Case("{@ccle}", X86::COND_LE)
+                           .Case("{@ccna}", X86::COND_BE)
+                           .Case("{@ccnae}", X86::COND_B)
+                           .Case("{@ccnb}", X86::COND_AE)
+                           .Case("{@ccnbe}", X86::COND_A)
+                           .Case("{@ccnc}", X86::COND_AE)
+                           .Case("{@ccne}", X86::COND_NE)
+                           .Case("{@ccnz}", X86::COND_NE)
+                           .Case("{@ccng}", X86::COND_LE)
+                           .Case("{@ccnge}", X86::COND_L)
+                           .Case("{@ccnl}", X86::COND_GE)
+                           .Case("{@ccnle}", X86::COND_G)
+                           .Case("{@ccno}", X86::COND_NO)
+                           .Case("{@ccnp}", X86::COND_P)
+                           .Case("{@ccns}", X86::COND_NS)
+                           .Case("{@cco}", X86::COND_O)
+                           .Case("{@ccp}", X86::COND_P)
+                           .Case("{@ccs}", X86::COND_S)
+                           .Default(X86::COND_INVALID);
+  return Cond;
+}
+
 /// Given a constraint letter, return the type of constraint for this target.
 X86TargetLowering::ConstraintType
 X86TargetLowering::getConstraintType(StringRef Constraint) const {
@@ -41734,7 +42558,8 @@ X86TargetLowering::getConstraintType(StringRef Constraint) const {
         return C_RegisterClass;
       }
     }
-  }
+  } else if (parseConstraintCode(Constraint) != X86::COND_INVALID)
+    return C_Other;
   return TargetLowering::getConstraintType(Constraint);
 }
 
@@ -41905,6 +42730,33 @@ LowerXConstraint(EVT ConstraintVT) const {
   return TargetLowering::LowerXConstraint(ConstraintVT);
 }
 
+// Lower @cc targets via setcc.
+SDValue X86TargetLowering::LowerAsmOutputForConstraint(
+    SDValue &Chain, SDValue *Flag, SDLoc DL, const AsmOperandInfo &OpInfo,
+    SelectionDAG &DAG) const {
+  X86::CondCode Cond = parseConstraintCode(OpInfo.ConstraintCode);
+  if (Cond == X86::COND_INVALID)
+    return SDValue();
+  // Check that return type is valid.
+  if (OpInfo.ConstraintVT.isVector() || !OpInfo.ConstraintVT.isInteger() ||
+      OpInfo.ConstraintVT.getSizeInBits() < 8)
+    report_fatal_error("Flag output operand is of invalid type");
+
+  // Get EFLAGS register. Only update chain when copyfrom is glued.
+  SDValue EFlags;
+  if (Flag) {
+    EFlags = DAG.getCopyFromReg(Chain, DL, X86::EFLAGS, MVT::i32, *Flag);
+    Chain = EFlags.getValue(1);
+  } else
+    EFlags = DAG.getCopyFromReg(Chain, DL, X86::EFLAGS, MVT::i32);
+  // Extract CC code.
+  SDValue CC = getSETCC(Cond, EFlags, DL, DAG);
+  // Extend to 32-bits
+  SDValue Result = DAG.getNode(ISD::ZERO_EXTEND, DL, OpInfo.ConstraintVT, CC);
+
+  return Result;
+}
+
 /// Lower the specified operand into the Ops vector.
 /// If it is invalid, don't add anything to Ops.
 void X86TargetLowering::LowerAsmOperandForConstraint(SDValue Op,
@@ -42102,6 +42954,14 @@ X86TargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
     // GCC Constraint Letters
     switch (Constraint[0]) {
     default: break;
+    // 'A' means [ER]AX + [ER]DX.
+    case 'A':
+      if (Subtarget.is64Bit())
+        return std::make_pair(X86::RAX, &X86::GR64_ADRegClass);
+      assert((Subtarget.is32Bit() || Subtarget.is16Bit()) &&
+             "Expecting 64, 32 or 16 bit subtarget");
+      return std::make_pair(X86::EAX, &X86::GR32_ADRegClass);
+
       // TODO: Slight differences here in allocation order and leaving
       // RIP in the class. Do they matter any more here than they do
       // in the normal allocation?
@@ -42261,6 +43121,9 @@ X86TargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
     }
   }
 
+  if (parseConstraintCode(Constraint) != X86::COND_INVALID)
+    return std::make_pair(0U, &X86::GR32RegClass);
+
   // Use the default implementation in TargetLowering to convert the register
   // constraint into a member of a register class.
   std::pair<unsigned, const TargetRegisterClass*> Res;
@@ -42270,14 +43133,17 @@ X86TargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
   if (!Res.second) {
     // Map st(0) -> st(7) -> ST0
     if (Constraint.size() == 7 && Constraint[0] == '{' &&
-        tolower(Constraint[1]) == 's' &&
-        tolower(Constraint[2]) == 't' &&
+        tolower(Constraint[1]) == 's' && tolower(Constraint[2]) == 't' &&
         Constraint[3] == '(' &&
         (Constraint[4] >= '0' && Constraint[4] <= '7') &&
-        Constraint[5] == ')' &&
-        Constraint[6] == '}')
+        Constraint[5] == ')' && Constraint[6] == '}') {
+      // st(7) is not allocatable and thus not a member of RFP80. Return
+      // singleton class in cases where we have a reference to it.
+      if (Constraint[4] == '7')
+        return std::make_pair(X86::FP7, &X86::RFP80_7RegClass);
       return std::make_pair(X86::FP0 + Constraint[4] - '0',
                             &X86::RFP80RegClass);
+    }
 
     // GCC allows "st(0)" to be called just plain "st".
     if (StringRef("{st}").equals_lower(Constraint))
@@ -42287,14 +43153,14 @@ X86TargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
     if (StringRef("{flags}").equals_lower(Constraint))
       return std::make_pair(X86::EFLAGS, &X86::CCRRegClass);
 
-    // 'A' means [ER]AX + [ER]DX.
-    if (Constraint == "A") {
-      if (Subtarget.is64Bit())
-        return std::make_pair(X86::RAX, &X86::GR64_ADRegClass);
-      assert((Subtarget.is32Bit() || Subtarget.is16Bit()) &&
-             "Expecting 64, 32 or 16 bit subtarget");
-      return std::make_pair(X86::EAX, &X86::GR32_ADRegClass);
-    }
+    // dirflag -> DF
+    if (StringRef("{dirflag}").equals_lower(Constraint))
+      return std::make_pair(X86::DF, &X86::DFCCRRegClass);
+
+    // fpsr -> FPSW
+    if (StringRef("{fpsr}").equals_lower(Constraint))
+      return std::make_pair(X86::FPSW, &X86::FPCCRRegClass);
+
     return Res;
   }
 
diff --git a/lib/Target/X86/X86ISelLowering.h b/lib/Target/X86/X86ISelLowering.h
index 17fd315a2b4c5f5e50cb9dd083dda5c0068ff14f..45d73085259bac161961901f4dba29b57255ad27 100644
--- a/lib/Target/X86/X86ISelLowering.h
+++ b/lib/Target/X86/X86ISelLowering.h
@@ -1,9 +1,8 @@
 //===-- X86ISelLowering.h - X86 DAG Lowering Interface ----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -98,7 +97,7 @@ namespace llvm {
       SETCC,
 
       /// X86 Select
-      SELECT, SELECTS,
+      SELECTS,
 
       // Same as SETCC except it's materialized with a sbb and the value is all
       // one's or all zero's.
@@ -203,8 +202,10 @@ namespace llvm {
 
       /// Dynamic (non-constant condition) vector blend where only the sign bits
       /// of the condition elements are used. This is used to enforce that the
-      /// condition mask is not valid for generic VSELECT optimizations.
-      SHRUNKBLEND,
+      /// condition mask is not valid for generic VSELECT optimizations. This
+      /// is also used to implement the intrinsics.
+      /// Operands are in VSELECT order: MASK, TRUE, FALSE
+      BLENDV,
 
       /// Combined add and sub on an FP vector.
       ADDSUB,
@@ -292,22 +293,30 @@ namespace llvm {
       // Vector integer truncate with unsigned/signed saturation.
       VTRUNCUS, VTRUNCS,
 
+      // Masked version of the above. Used when less than a 128-bit result is
+      // produced since the mask only applies to the lower elements and can't
+      // be represented by a select.
+      // SRC, PASSTHRU, MASK
+      VMTRUNC, VMTRUNCUS, VMTRUNCS,
+
       // Vector FP extend.
       VFPEXT, VFPEXT_RND, VFPEXTS_RND,
 
       // Vector FP round.
       VFPROUND, VFPROUND_RND, VFPROUNDS_RND,
 
+      // Masked version of above. Used for v2f64->v4f32.
+      // SRC, PASSTHRU, MASK
+      VMFPROUND,
+
       // 128-bit vector logical left / right shift
       VSHLDQ, VSRLDQ,
 
       // Vector shift elements
       VSHL, VSRL, VSRA,
 
-      // Vector variable shift right arithmetic.
-      // Unlike ISD::SRA, in case shift count greater then element size
-      // use sign bit to fill destination data element.
-      VSRAV,
+      // Vector variable shift
+      VSHLV, VSRLV, VSRAV,
 
       // Vector shift elements by immediate
       VSHLI, VSRLI, VSRAI,
@@ -336,8 +345,8 @@ namespace llvm {
       CMPM_RND,
 
       // Arithmetic operations with FLAGS results.
-      ADD, SUB, ADC, SBB, SMUL,
-      INC, DEC, OR, XOR, AND,
+      ADD, SUB, ADC, SBB, SMUL, UMUL,
+      OR, XOR, AND,
 
       // Bit field extract.
       BEXTR,
@@ -345,12 +354,6 @@ namespace llvm {
       // Zero High Bits Starting with Specified Bit Position.
       BZHI,
 
-      // LOW, HI, FLAGS = umul LHS, RHS.
-      UMUL,
-
-      // 8-bit SMUL/UMUL - AX, FLAGS = smul8/umul8 AL, RHS.
-      SMUL8, UMUL8,
-
       // X86-specific multiply by immediate.
       MUL_IMM,
 
@@ -509,6 +512,11 @@ namespace llvm {
       // Vector signed/unsigned integer to float/double.
       CVTSI2P, CVTUI2P,
 
+      // Masked versions of above. Used for v2f64->v4f32.
+      // SRC, PASSTHRU, MASK
+      MCVTP2SI, MCVTP2UI, MCVTTP2SI, MCVTTP2UI,
+      MCVTSI2P, MCVTUI2P,
+
       // Save xmm argument registers to the stack, according to %al. An operator
       // is needed so that this can be expanded with control flow.
       VASTART_SAVE_XMM_REGS,
@@ -556,6 +564,10 @@ namespace llvm {
       // Conversions between float and half-float.
       CVTPS2PH, CVTPH2PS, CVTPH2PS_RND,
 
+      // Masked version of above.
+      // SRC, RND, PASSTHRU, MASK
+      MCVTPS2PH,
+
       // Galois Field Arithmetic Instructions
       GF2P8AFFINEINVQB, GF2P8AFFINEQB, GF2P8MULB,
 
@@ -574,7 +586,7 @@ namespace llvm {
 
       /// LOCK-prefixed arithmetic read-modify-write instructions.
       /// EFLAGS, OUTCHAIN = LADD(INCHAIN, PTR, RHS)
-      LADD, LSUB, LOR, LXOR, LAND, LINC, LDEC,
+      LADD, LSUB, LOR, LXOR, LAND,
 
       // Load, scalar_to_vector, and zero extend.
       VZEXT_LOAD,
@@ -586,29 +598,27 @@ namespace llvm {
       /// integer destination in memory and a FP reg source.  This corresponds
       /// to the X86::FIST*m instructions and the rounding mode change stuff. It
       /// has two inputs (token chain and address) and two outputs (int value
-      /// and token chain).
-      FP_TO_INT16_IN_MEM,
-      FP_TO_INT32_IN_MEM,
-      FP_TO_INT64_IN_MEM,
+      /// and token chain). Memory VT specifies the type to store to.
+      FP_TO_INT_IN_MEM,
 
       /// This instruction implements SINT_TO_FP with the
       /// integer source in memory and FP reg result.  This corresponds to the
-      /// X86::FILD*m instructions. It has three inputs (token chain, address,
-      /// and source type) and two outputs (FP value and token chain). FILD_FLAG
-      /// also produces a flag).
+      /// X86::FILD*m instructions. It has two inputs (token chain and address)
+      /// and two outputs (FP value and token chain). FILD_FLAG also produces a
+      /// flag). The integer source type is specified by the memory VT.
       FILD,
       FILD_FLAG,
 
       /// This instruction implements an extending load to FP stack slots.
       /// This corresponds to the X86::FLD32m / X86::FLD64m. It takes a chain
-      /// operand, ptr to load from, and a ValueType node indicating the type
-      /// to load to.
+      /// operand, and ptr to load from. The memory VT specifies the type to
+      /// load from.
       FLD,
 
       /// This instruction implements a truncating store to FP stack
       /// slots. This corresponds to the X86::FST32m / X86::FST64m. It takes a
-      /// chain operand, value to store, address, and a ValueType to store it
-      /// as.
+      /// chain operand, value to store, and address. The memory VT specifies
+      /// the type to store as.
       FST,
 
       /// This instruction grabs the address of the next argument
@@ -819,6 +829,12 @@ namespace llvm {
       return VTIsOk(XVT) && VTIsOk(KeptBitsVT);
     }
 
+    bool shouldExpandShift(SelectionDAG &DAG, SDNode *N) const override {
+      if (DAG.getMachineFunction().getFunction().optForMinSize())
+        return false;
+      return true;
+    }
+
     bool shouldSplatInsEltVarIndex(EVT VT) const override;
 
     bool convertSetCCLogicToBitwiseLogic(EVT VT) const override {
@@ -905,6 +921,11 @@ namespace llvm {
       return TargetLowering::getInlineAsmMemConstraint(ConstraintCode);
     }
 
+    /// Handle Lowering flag assembly outputs.
+    SDValue LowerAsmOutputForConstraint(SDValue &Chain, SDValue *Flag, SDLoc DL,
+                                        const AsmOperandInfo &Constraint,
+                                        SelectionDAG &DAG) const override;
+
     /// Given a physical register constraint
     /// (e.g. {edx}), return the register number and the register class for the
     /// register.  This should only be used for C_Register constraints.  On
@@ -1045,6 +1066,11 @@ namespace llvm {
     bool isExtractSubvectorCheap(EVT ResVT, EVT SrcVT,
                                  unsigned Index) const override;
 
+    /// Scalar ops always have equal or better analysis/performance/power than
+    /// the vector equivalent, so this always makes sense if the scalar op is
+    /// supported.
+    bool shouldScalarizeBinop(SDValue) const override;
+
     bool storeOfVectorConstantIsCheap(EVT MemVT, unsigned NumElem,
                                       unsigned AddrSpace) const override {
       // If we can replace more than 2 scalar stores, there will be a reduction
@@ -1087,7 +1113,7 @@ namespace llvm {
     bool useStackGuardXorFP() const override;
     void insertSSPDeclarations(Module &M) const override;
     Value *getSDagStackGuard(const Module &M) const override;
-    Value *getSSPStackGuardCheck(const Module &M) const override;
+    Function *getSSPStackGuardCheck(const Module &M) const override;
     SDValue emitStackGuardXorFP(SelectionDAG &DAG, SDValue Val,
                                 const SDLoc &DL) const override;
 
@@ -1203,9 +1229,7 @@ namespace llvm {
 
     unsigned getAddressSpace(void) const;
 
-    std::pair<SDValue,SDValue> FP_TO_INTHelper(SDValue Op, SelectionDAG &DAG,
-                                               bool isSigned,
-                                               bool isReplace) const;
+    SDValue FP_TO_INTHelper(SDValue Op, SelectionDAG &DAG, bool isSigned) const;
 
     SDValue LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const;
     SDValue LowerVSELECT(SDValue Op, SelectionDAG &DAG) const;
@@ -1363,6 +1387,13 @@ namespace llvm {
     /// Convert a comparison if required by the subtarget.
     SDValue ConvertCmpIfNecessary(SDValue Cmp, SelectionDAG &DAG) const;
 
+    /// Emit flags for the given setcc condition and operands. Also returns the
+    /// corresponding X86 condition code constant in X86CC.
+    SDValue emitFlagsForSetcc(SDValue Op0, SDValue Op1,
+                              ISD::CondCode CC, const SDLoc &dl,
+                              SelectionDAG &DAG,
+                              SDValue &X86CC) const;
+
     /// Check if replacement of SQRT with RSQRT should be disabled.
     bool isFsqrtCheap(SDValue Operand, SelectionDAG &DAG) const override;
 
diff --git a/lib/Target/X86/X86IndirectBranchTracking.cpp b/lib/Target/X86/X86IndirectBranchTracking.cpp
index 7c00c9260d15b2f0e9ff70213f9b1dc5f6164dd9..35866d74edb005b0bca4443726f243b256ee5998 100644
--- a/lib/Target/X86/X86IndirectBranchTracking.cpp
+++ b/lib/Target/X86/X86IndirectBranchTracking.cpp
@@ -1,9 +1,8 @@
 //===---- X86IndirectBranchTracking.cpp - Enables CET IBT mechanism -------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/X86/X86InsertPrefetch.cpp b/lib/Target/X86/X86InsertPrefetch.cpp
index 30b46a09ef0f90131127c47f366eb09ce8ce4530..02ae73706a3491b90e5512ae5e34497434059794 100644
--- a/lib/Target/X86/X86InsertPrefetch.cpp
+++ b/lib/Target/X86/X86InsertPrefetch.cpp
@@ -1,9 +1,8 @@
 //===------- X86InsertPrefetch.cpp - Insert cache prefetch hints ----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -34,7 +33,8 @@ using namespace sampleprof;
 
 static cl::opt<std::string>
     PrefetchHintsFile("prefetch-hints-file",
-                      cl::desc("Path to the prefetch hints profile."),
+                      cl::desc("Path to the prefetch hints profile. See also "
+                               "-x86-discriminate-memops"),
                       cl::Hidden);
 namespace {
 
diff --git a/lib/Target/X86/X86Instr3DNow.td b/lib/Target/X86/X86Instr3DNow.td
index 49e9e924887a7acf9028718aac59456cd866e8ae..cd1b0636597126a0923c741c5c5407b6e78ce3d9 100644
--- a/lib/Target/X86/X86Instr3DNow.td
+++ b/lib/Target/X86/X86Instr3DNow.td
@@ -1,9 +1,8 @@
 //===-- X86Instr3DNow.td - The 3DNow! Instruction Set ------*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -74,7 +73,9 @@ defm PFSUBR   : I3DNow_binop_rm_int<0xAA, "pfsubr", WriteFAdd, 1>;
 defm PI2FD    : I3DNow_conv_rm_int<0x0D, "pi2fd", WriteCvtI2PS>;
 defm PMULHRW  : I3DNow_binop_rm_int<0xB7, "pmulhrw", SchedWriteVecIMul.MMX, 1>;
 
-let SchedRW = [WriteEMMS] in
+let SchedRW = [WriteEMMS],
+    Defs = [MM0, MM1, MM2, MM3, MM4, MM5, MM6, MM7,
+            ST0, ST1, ST2, ST3, ST4, ST5, ST6, ST7] in
 def FEMMS : I3DNow<0x0E, RawFrm, (outs), (ins), "femms",
                    [(int_x86_mmx_femms)]>, TB;
 
diff --git a/lib/Target/X86/X86InstrAVX512.td b/lib/Target/X86/X86InstrAVX512.td
index 7e60b9caf05b8d83d7c79853e4983b5997649cca..0fba8cb4d09f2e1155659ac687bad47d8f446239 100644
--- a/lib/Target/X86/X86InstrAVX512.td
+++ b/lib/Target/X86/X86InstrAVX512.td
@@ -1,9 +1,8 @@
 //===-- X86InstrAVX512.td - AVX512 Instruction Set ---------*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -753,6 +752,7 @@ defm : vinsert_for_mask_cast<"VINSERTI64x4Z", v32i8x_info, v64i8_info,
 
 // vinsertps - insert f32 to XMM
 let ExeDomain = SSEPackedSingle in {
+let isCommutable = 1 in
 def VINSERTPSZrr : AVX512AIi8<0x21, MRMSrcReg, (outs VR128X:$dst),
       (ins VR128X:$src1, VR128X:$src2, u8imm:$src3),
       "vinsertps\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}",
@@ -2887,8 +2887,15 @@ let Predicates = [HasDQI] in {
 let Predicates = [HasAVX512] in {
   def : Pat<(v8i1 (bitconvert (i8 (load addr:$src)))),
             (COPY_TO_REGCLASS (MOVZX32rm8 addr:$src), VK8)>;
+  def : Pat<(v16i1 (bitconvert (loadi16 addr:$src))),
+            (KMOVWkm addr:$src)>;
 }
 
+def X86kextract : SDNode<"ISD::EXTRACT_VECTOR_ELT",
+                         SDTypeProfile<1, 2, [SDTCisVT<0, i8>,
+                                              SDTCVecEltisVT<1, i1>,
+                                              SDTCisPtrTy<2>]>>;
+
 let Predicates = [HasAVX512] in {
   multiclass operation_gpr_mask_copy_lowering<RegisterClass maskRC, ValueType maskVT> {
     def : Pat<(maskVT (scalar_to_vector GR32:$src)),
@@ -2896,6 +2903,12 @@ let Predicates = [HasAVX512] in {
 
     def : Pat<(maskVT (scalar_to_vector GR8:$src)),
               (COPY_TO_REGCLASS (INSERT_SUBREG (i32 (IMPLICIT_DEF)), GR8:$src, sub_8bit), maskRC)>;
+
+    def : Pat<(i8 (X86kextract maskRC:$src, (iPTR 0))),
+              (EXTRACT_SUBREG (i32 (COPY_TO_REGCLASS maskRC:$src, GR32)), sub_8bit)>;
+
+    def : Pat<(i32 (anyext (i8 (X86kextract maskRC:$src, (iPTR 0))))),
+              (i32 (COPY_TO_REGCLASS maskRC:$src, GR32))>;
   }
 
   defm : operation_gpr_mask_copy_lowering<VK1,  v1i1>;
@@ -6432,52 +6445,53 @@ defm : avx512_var_shift_lowering<avx512vl_i16_info, "VPSRAVW", sra, [HasBWI, NoV
 defm : avx512_var_shift_lowering<avx512vl_i16_info, "VPSRLVW", srl, [HasBWI, NoVLX]>;
 
 // Special handing for handling VPSRAV intrinsics.
-multiclass avx512_var_shift_int_lowering<string InstrStr, X86VectorVTInfo _,
-                                         list<Predicate> p> {
+multiclass avx512_var_shift_int_lowering<string InstrStr, SDNode OpNode,
+                                         X86VectorVTInfo _, list<Predicate> p> {
   let Predicates = p in {
-    def : Pat<(_.VT (X86vsrav _.RC:$src1, _.RC:$src2)),
+    def : Pat<(_.VT (OpNode _.RC:$src1, _.RC:$src2)),
               (!cast<Instruction>(InstrStr#_.ZSuffix#rr) _.RC:$src1,
                _.RC:$src2)>;
-    def : Pat<(_.VT (X86vsrav _.RC:$src1, (_.LdFrag addr:$src2))),
+    def : Pat<(_.VT (OpNode _.RC:$src1, (_.LdFrag addr:$src2))),
               (!cast<Instruction>(InstrStr#_.ZSuffix##rm)
                _.RC:$src1, addr:$src2)>;
     def : Pat<(_.VT (vselect _.KRCWM:$mask,
-                     (X86vsrav _.RC:$src1, _.RC:$src2), _.RC:$src0)),
+                     (OpNode _.RC:$src1, _.RC:$src2), _.RC:$src0)),
               (!cast<Instruction>(InstrStr#_.ZSuffix#rrk) _.RC:$src0,
                _.KRC:$mask, _.RC:$src1, _.RC:$src2)>;
     def : Pat<(_.VT (vselect _.KRCWM:$mask,
-                     (X86vsrav _.RC:$src1, (_.LdFrag addr:$src2)),
+                     (OpNode _.RC:$src1, (_.LdFrag addr:$src2)),
                      _.RC:$src0)),
               (!cast<Instruction>(InstrStr#_.ZSuffix##rmk) _.RC:$src0,
                _.KRC:$mask, _.RC:$src1, addr:$src2)>;
     def : Pat<(_.VT (vselect _.KRCWM:$mask,
-                     (X86vsrav _.RC:$src1, _.RC:$src2), _.ImmAllZerosV)),
+                     (OpNode _.RC:$src1, _.RC:$src2), _.ImmAllZerosV)),
               (!cast<Instruction>(InstrStr#_.ZSuffix#rrkz) _.KRC:$mask,
                _.RC:$src1, _.RC:$src2)>;
     def : Pat<(_.VT (vselect _.KRCWM:$mask,
-                     (X86vsrav _.RC:$src1, (_.LdFrag addr:$src2)),
+                     (OpNode _.RC:$src1, (_.LdFrag addr:$src2)),
                      _.ImmAllZerosV)),
               (!cast<Instruction>(InstrStr#_.ZSuffix##rmkz) _.KRC:$mask,
                _.RC:$src1, addr:$src2)>;
   }
 }
 
-multiclass avx512_var_shift_int_lowering_mb<string InstrStr, X86VectorVTInfo _,
-                                         list<Predicate> p> :
-           avx512_var_shift_int_lowering<InstrStr, _, p> {
+multiclass avx512_var_shift_int_lowering_mb<string InstrStr, SDNode OpNode,
+                                            X86VectorVTInfo _,
+                                            list<Predicate> p> :
+           avx512_var_shift_int_lowering<InstrStr, OpNode, _, p> {
   let Predicates = p in {
-    def : Pat<(_.VT (X86vsrav _.RC:$src1,
+    def : Pat<(_.VT (OpNode _.RC:$src1,
                      (X86VBroadcast (_.ScalarLdFrag addr:$src2)))),
               (!cast<Instruction>(InstrStr#_.ZSuffix##rmb)
                _.RC:$src1, addr:$src2)>;
     def : Pat<(_.VT (vselect _.KRCWM:$mask,
-                     (X86vsrav _.RC:$src1,
+                     (OpNode _.RC:$src1,
                       (X86VBroadcast (_.ScalarLdFrag addr:$src2))),
                      _.RC:$src0)),
               (!cast<Instruction>(InstrStr#_.ZSuffix##rmbk) _.RC:$src0,
                _.KRC:$mask, _.RC:$src1, addr:$src2)>;
     def : Pat<(_.VT (vselect _.KRCWM:$mask,
-                     (X86vsrav _.RC:$src1,
+                     (OpNode _.RC:$src1,
                       (X86VBroadcast (_.ScalarLdFrag addr:$src2))),
                      _.ImmAllZerosV)),
               (!cast<Instruction>(InstrStr#_.ZSuffix##rmbkz) _.KRC:$mask,
@@ -6485,15 +6499,47 @@ multiclass avx512_var_shift_int_lowering_mb<string InstrStr, X86VectorVTInfo _,
   }
 }
 
-defm : avx512_var_shift_int_lowering<"VPSRAVW", v8i16x_info, [HasVLX, HasBWI]>;
-defm : avx512_var_shift_int_lowering<"VPSRAVW", v16i16x_info, [HasVLX, HasBWI]>;
-defm : avx512_var_shift_int_lowering<"VPSRAVW", v32i16_info, [HasBWI]>;
-defm : avx512_var_shift_int_lowering_mb<"VPSRAVD", v4i32x_info, [HasVLX]>;
-defm : avx512_var_shift_int_lowering_mb<"VPSRAVD", v8i32x_info, [HasVLX]>;
-defm : avx512_var_shift_int_lowering_mb<"VPSRAVD", v16i32_info, [HasAVX512]>;
-defm : avx512_var_shift_int_lowering_mb<"VPSRAVQ", v2i64x_info, [HasVLX]>;
-defm : avx512_var_shift_int_lowering_mb<"VPSRAVQ", v4i64x_info, [HasVLX]>;
-defm : avx512_var_shift_int_lowering_mb<"VPSRAVQ", v8i64_info, [HasAVX512]>;
+multiclass avx512_var_shift_int_lowering_vl<string InstrStr, SDNode OpNode,
+                                            AVX512VLVectorVTInfo VTInfo,
+                                            Predicate p> {
+  defm : avx512_var_shift_int_lowering<InstrStr, OpNode, VTInfo.info512, [p]>;
+  defm : avx512_var_shift_int_lowering<InstrStr, OpNode, VTInfo.info256,
+                                       [HasVLX, p]>;
+  defm : avx512_var_shift_int_lowering<InstrStr, OpNode, VTInfo.info128,
+                                       [HasVLX, p]>;
+}
+
+multiclass avx512_var_shift_int_lowering_mb_vl<string InstrStr, SDNode OpNode,
+                                               AVX512VLVectorVTInfo VTInfo,
+                                               Predicate p> {
+  defm : avx512_var_shift_int_lowering_mb<InstrStr, OpNode, VTInfo.info512, [p]>;
+  defm : avx512_var_shift_int_lowering_mb<InstrStr, OpNode, VTInfo.info256,
+                                          [HasVLX, p]>;
+  defm : avx512_var_shift_int_lowering_mb<InstrStr, OpNode, VTInfo.info128,
+                                          [HasVLX, p]>;
+}
+
+defm : avx512_var_shift_int_lowering_vl<"VPSRAVW", X86vsrav, avx512vl_i16_info,
+                                        HasBWI>;
+defm : avx512_var_shift_int_lowering_mb_vl<"VPSRAVD", X86vsrav,
+                                           avx512vl_i32_info, HasAVX512>;
+defm : avx512_var_shift_int_lowering_mb_vl<"VPSRAVQ", X86vsrav,
+                                           avx512vl_i64_info, HasAVX512>;
+
+defm : avx512_var_shift_int_lowering_vl<"VPSRLVW", X86vsrlv, avx512vl_i16_info,
+                                        HasBWI>;
+defm : avx512_var_shift_int_lowering_mb_vl<"VPSRLVD", X86vsrlv,
+                                           avx512vl_i32_info, HasAVX512>;
+defm : avx512_var_shift_int_lowering_mb_vl<"VPSRLVQ", X86vsrlv,
+                                           avx512vl_i64_info, HasAVX512>;
+
+defm : avx512_var_shift_int_lowering_vl<"VPSLLVW", X86vshlv, avx512vl_i16_info,
+                                        HasBWI>;
+defm : avx512_var_shift_int_lowering_mb_vl<"VPSLLVD", X86vshlv,
+                                           avx512vl_i32_info, HasAVX512>;
+defm : avx512_var_shift_int_lowering_mb_vl<"VPSLLVQ", X86vshlv,
+                                           avx512vl_i64_info, HasAVX512>;
+
 
 // Use 512bit VPROL/VPROLI version to implement v2i64/v4i64 + v4i32/v8i32 in case NoVLX.
 let Predicates = [HasAVX512, NoVLX] in {
@@ -7462,7 +7508,7 @@ multiclass avx512_vcvtsi<bits<8> opc, SDNode OpNode, X86FoldableSchedWrite sched
     def rr : SI<opc, MRMSrcReg, (outs DstVT.FRC:$dst),
               (ins DstVT.FRC:$src1, SrcRC:$src),
               !strconcat(asm,"\t{$src, $src1, $dst|$dst, $src1, $src}"), []>,
-              EVEX_4V, Sched<[sched]>;
+              EVEX_4V, Sched<[sched, ReadDefault, ReadInt2Fpu]>;
     let mayLoad = 1 in
       def rm : SI<opc, MRMSrcMem, (outs DstVT.FRC:$dst),
               (ins DstVT.FRC:$src1, x86memop:$src),
@@ -7477,7 +7523,7 @@ multiclass avx512_vcvtsi<bits<8> opc, SDNode OpNode, X86FoldableSchedWrite sched
                         (OpNode (DstVT.VT DstVT.RC:$src1),
                                  SrcRC:$src2,
                                  (i32 FROUND_CURRENT)))]>,
-                 EVEX_4V, Sched<[sched]>;
+                 EVEX_4V, Sched<[sched, ReadDefault, ReadInt2Fpu]>;
 
     def rm_Int : SI<opc, MRMSrcMem, (outs DstVT.RC:$dst),
                   (ins DstVT.RC:$src1, x86memop:$src2),
@@ -7501,7 +7547,7 @@ multiclass avx512_vcvtsi_round<bits<8> opc, SDNode OpNode,
                     (OpNode (DstVT.VT DstVT.RC:$src1),
                              SrcRC:$src2,
                              (i32 imm:$rc)))]>,
-              EVEX_4V, EVEX_B, EVEX_RC, Sched<[sched]>;
+              EVEX_4V, EVEX_B, EVEX_RC, Sched<[sched, ReadDefault, ReadInt2Fpu]>;
 }
 
 multiclass avx512_vcvtsi_common<bits<8> opc, SDNode OpNode,
@@ -7613,12 +7659,12 @@ multiclass avx512_cvt_s_int_round<bits<8> opc, X86VectorVTInfo SrcVT,
                 [(set DstVT.RC:$dst, (OpNode
                       (SrcVT.VT SrcVT.ScalarIntMemCPat:$src)))]>,
                 EVEX, VEX_LIG, Sched<[sched.Folded, sched.ReadAfterFold]>;
-
-    def : InstAlias<"v" # asm # aliasStr # "\t{$src, $dst|$dst, $src}",
-            (!cast<Instruction>(NAME # "rr_Int") DstVT.RC:$dst, SrcVT.RC:$src), 0, "att">;
-    def : InstAlias<"v" # asm # aliasStr # "\t{$rc, $src, $dst|$dst, $src, $rc}",
-            (!cast<Instruction>(NAME # "rrb_Int") DstVT.RC:$dst, SrcVT.RC:$src, AVX512RC:$rc), 0, "att">;
   } // Predicates = [HasAVX512]
+
+  def : InstAlias<"v" # asm # aliasStr # "\t{$src, $dst|$dst, $src}",
+          (!cast<Instruction>(NAME # "rr_Int") DstVT.RC:$dst, SrcVT.RC:$src), 0, "att">;
+  def : InstAlias<"v" # asm # aliasStr # "\t{$rc, $src, $dst|$dst, $src, $rc}",
+          (!cast<Instruction>(NAME # "rrb_Int") DstVT.RC:$dst, SrcVT.RC:$src, AVX512RC:$rc), 0, "att">;
 }
 
 multiclass avx512_cvt_s_int_round_aliases<bits<8> opc, X86VectorVTInfo SrcVT,
@@ -7627,11 +7673,9 @@ multiclass avx512_cvt_s_int_round_aliases<bits<8> opc, X86VectorVTInfo SrcVT,
                                           X86FoldableSchedWrite sched, string asm,
                                           string aliasStr> :
   avx512_cvt_s_int_round<opc, SrcVT, DstVT, OpNode, OpNodeRnd, sched, asm, aliasStr, 0> {
-  let Predicates = [HasAVX512] in {
-    def : InstAlias<"v" # asm # aliasStr # "\t{$src, $dst|$dst, $src}",
-            (!cast<Instruction>(NAME # "rm_Int") DstVT.RC:$dst,
-                                            SrcVT.IntScalarMemOp:$src), 0, "att">;
-  } // Predicates = [HasAVX512]
+  def : InstAlias<"v" # asm # aliasStr # "\t{$src, $dst|$dst, $src}",
+          (!cast<Instruction>(NAME # "rm_Int") DstVT.RC:$dst,
+                                          SrcVT.IntScalarMemOp:$src), 0, "att">;
 }
 
 // Convert float/double to signed/unsigned int 32/64
@@ -7778,12 +7822,12 @@ let Predicates = [HasAVX512] in {
               [(set _DstRC.RC:$dst,
                 (OpNodeInt (_SrcRC.VT _SrcRC.ScalarIntMemCPat:$src)))]>,
               EVEX, VEX_LIG, Sched<[sched.Folded, sched.ReadAfterFold]>;
+} //HasAVX512
 
   def : InstAlias<asm # aliasStr # "\t{$src, $dst|$dst, $src}",
           (!cast<Instruction>(NAME # "rr_Int") _DstRC.RC:$dst, _SrcRC.RC:$src), 0, "att">;
   def : InstAlias<asm # aliasStr # "\t{{sae}, $src, $dst|$dst, $src, {sae}}",
           (!cast<Instruction>(NAME # "rrb_Int") _DstRC.RC:$dst, _SrcRC.RC:$src), 0, "att">;
-} //HasAVX512
 }
 
 multiclass avx512_cvt_s_all_unsigned<bits<8> opc, string asm,
@@ -7794,12 +7838,10 @@ multiclass avx512_cvt_s_all_unsigned<bits<8> opc, string asm,
                                      string aliasStr> :
   avx512_cvt_s_all<opc, asm, _SrcRC, _DstRC, OpNode, OpNodeInt, OpNodeRnd, sched,
                    aliasStr, 0> {
-let Predicates = [HasAVX512] in {
   def : InstAlias<asm # aliasStr # "\t{$src, $dst|$dst, $src}",
           (!cast<Instruction>(NAME # "rm_Int") _DstRC.RC:$dst,
                                           _SrcRC.IntScalarMemOp:$src), 0, "att">;
 }
-}
 
 defm VCVTTSS2SIZ: avx512_cvt_s_all<0x2C, "vcvttss2si", f32x_info, i32x_info,
                         fp_to_sint, X86cvtts2Int, X86cvtts2IntRnd, WriteCvtSS2I,
@@ -7956,26 +7998,53 @@ multiclass avx512_vcvt_fp<bits<8> opc, string OpcodeStr, X86VectorVTInfo _,
                           X86VectorVTInfo _Src, SDNode OpNode,
                           X86FoldableSchedWrite sched,
                           string Broadcast = _.BroadcastStr,
-                          string Alias = "", X86MemOperand MemOp = _Src.MemOp> {
+                          string Alias = "", X86MemOperand MemOp = _Src.MemOp,
+                          RegisterClass MaskRC = _.KRCWM> {
 
-  defm rr : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),
-                         (ins _Src.RC:$src), OpcodeStr, "$src", "$src",
-                         (_.VT (OpNode (_Src.VT _Src.RC:$src)))>,
+  defm rr : AVX512_maskable_common<opc, MRMSrcReg, _, (outs _.RC:$dst),
+                         (ins _Src.RC:$src),
+                         (ins _.RC:$src0, MaskRC:$mask, _Src.RC:$src),
+                         (ins MaskRC:$mask, _Src.RC:$src),
+                          OpcodeStr, "$src", "$src",
+                         (_.VT (OpNode (_Src.VT _Src.RC:$src))),
+                         (vselect MaskRC:$mask,
+                                  (_.VT (OpNode (_Src.VT _Src.RC:$src))),
+                                  _.RC:$src0),
+                         vselect, "$src0 = $dst">,
                          EVEX, Sched<[sched]>;
 
-  defm rm : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
-                         (ins MemOp:$src), OpcodeStr#Alias, "$src", "$src",
+  defm rm : AVX512_maskable_common<opc, MRMSrcMem, _, (outs _.RC:$dst),
+                         (ins MemOp:$src),
+                         (ins _.RC:$src0, MaskRC:$mask, MemOp:$src),
+                         (ins MaskRC:$mask, MemOp:$src),
+                         OpcodeStr#Alias, "$src", "$src",
                          (_.VT (OpNode (_Src.VT
-                             (_Src.LdFrag addr:$src))))>,
+                             (_Src.LdFrag addr:$src)))),
+                         (vselect MaskRC:$mask,
+                                  (_.VT (OpNode (_Src.VT
+                                                 (_Src.LdFrag addr:$src)))),
+                                  _.RC:$src0),
+                         vselect, "$src0 = $dst">,
                          EVEX, Sched<[sched.Folded]>;
 
-  defm rmb : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
-                         (ins _Src.ScalarMemOp:$src), OpcodeStr,
+  defm rmb : AVX512_maskable_common<opc, MRMSrcMem, _, (outs _.RC:$dst),
+                         (ins _Src.ScalarMemOp:$src),
+                         (ins _.RC:$src0, MaskRC:$mask, _Src.ScalarMemOp:$src),
+                         (ins MaskRC:$mask, _Src.ScalarMemOp:$src),
+                         OpcodeStr,
                          "${src}"##Broadcast, "${src}"##Broadcast,
                          (_.VT (OpNode (_Src.VT
                                   (X86VBroadcast (_Src.ScalarLdFrag addr:$src)))
-                            ))>, EVEX, EVEX_B,
-                         Sched<[sched.Folded]>;
+                            )),
+                         (vselect MaskRC:$mask,
+                                  (_.VT
+                                   (OpNode
+                                    (_Src.VT
+                                     (X86VBroadcast
+                                      (_Src.ScalarLdFrag addr:$src))))),
+                                  _.RC:$src0),
+                         vselect, "$src0 = $dst">,
+                         EVEX, EVEX_B, Sched<[sched.Folded]>;
 }
 // Coversion with SAE - suppress all exceptions
 multiclass avx512_vcvt_fp_sae<bits<8> opc, string OpcodeStr, X86VectorVTInfo _,
@@ -8020,25 +8089,26 @@ multiclass avx512_cvtps2pd<bits<8> opc, string OpcodeStr,
 // Truncate Double to Float
 multiclass avx512_cvtpd2ps<bits<8> opc, string OpcodeStr, X86SchedWriteWidths sched> {
   let Predicates = [HasAVX512] in {
-    defm Z : avx512_vcvt_fp<opc, OpcodeStr, v8f32x_info, v8f64_info, fpround, sched.ZMM>,
+    defm Z : avx512_vcvt_fp<opc, OpcodeStr, v8f32x_info, v8f64_info, X86vfpround, sched.ZMM>,
              avx512_vcvt_fp_rc<opc, OpcodeStr, v8f32x_info, v8f64_info,
                                X86vfproundRnd, sched.ZMM>, EVEX_V512;
   }
   let Predicates = [HasVLX] in {
     defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v4f32x_info, v2f64x_info,
-                               X86vfpround, sched.XMM, "{1to2}", "{x}">, EVEX_V128;
-    defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v4f32x_info, v4f64x_info, fpround,
+                               null_frag, sched.XMM, "{1to2}", "{x}", f128mem, VK2WM>,
+                               EVEX_V128;
+    defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v4f32x_info, v4f64x_info, X86vfpround,
                                sched.YMM, "{1to4}", "{y}">, EVEX_V256;
-
-    def : InstAlias<OpcodeStr##"x\t{$src, $dst|$dst, $src}",
-                    (!cast<Instruction>(NAME # "Z128rr") VR128X:$dst, VR128X:$src), 0>;
-    def : InstAlias<OpcodeStr##"x\t{$src, $dst|$dst, $src}",
-                    (!cast<Instruction>(NAME # "Z128rm") VR128X:$dst, f128mem:$src), 0, "intel">;
-    def : InstAlias<OpcodeStr##"y\t{$src, $dst|$dst, $src}",
-                    (!cast<Instruction>(NAME # "Z256rr") VR128X:$dst, VR256X:$src), 0>;
-    def : InstAlias<OpcodeStr##"y\t{$src, $dst|$dst, $src}",
-                    (!cast<Instruction>(NAME # "Z256rm") VR128X:$dst, f256mem:$src), 0, "intel">;
   }
+
+  def : InstAlias<OpcodeStr##"x\t{$src, $dst|$dst, $src}",
+                  (!cast<Instruction>(NAME # "Z128rr") VR128X:$dst, VR128X:$src), 0>;
+  def : InstAlias<OpcodeStr##"x\t{$src, $dst|$dst, $src}",
+                  (!cast<Instruction>(NAME # "Z128rm") VR128X:$dst, f128mem:$src), 0, "intel">;
+  def : InstAlias<OpcodeStr##"y\t{$src, $dst|$dst, $src}",
+                  (!cast<Instruction>(NAME # "Z256rr") VR128X:$dst, VR256X:$src), 0>;
+  def : InstAlias<OpcodeStr##"y\t{$src, $dst|$dst, $src}",
+                  (!cast<Instruction>(NAME # "Z256rm") VR128X:$dst, f256mem:$src), 0, "intel">;
 }
 
 defm VCVTPD2PS : avx512_cvtpd2ps<0x5A, "vcvtpd2ps", SchedWriteCvtPD2PS>,
@@ -8046,20 +8116,103 @@ defm VCVTPD2PS : avx512_cvtpd2ps<0x5A, "vcvtpd2ps", SchedWriteCvtPD2PS>,
 defm VCVTPS2PD : avx512_cvtps2pd<0x5A, "vcvtps2pd", SchedWriteCvtPS2PD>,
                                   PS, EVEX_CD8<32, CD8VH>;
 
-def : Pat<(v8f64 (extloadv8f32 addr:$src)),
-            (VCVTPS2PDZrm addr:$src)>;
+let Predicates = [HasAVX512] in {
+  def : Pat<(v8f32 (fpround (v8f64 VR512:$src))),
+            (VCVTPD2PSZrr VR512:$src)>;
+  def : Pat<(vselect VK8WM:$mask, (v8f32 (fpround (v8f64 VR512:$src))),
+                     VR256X:$src0),
+            (VCVTPD2PSZrrk VR256X:$src0, VK8WM:$mask, VR512:$src)>;
+  def : Pat<(vselect VK8WM:$mask, (v8f32 (fpround (v8f64 VR512:$src))),
+                     v8f32x_info.ImmAllZerosV),
+            (VCVTPD2PSZrrkz VK8WM:$mask, VR512:$src)>;
+
+  def : Pat<(v8f32 (fpround (loadv8f64 addr:$src))),
+            (VCVTPD2PSZrm addr:$src)>;
+  def : Pat<(vselect VK8WM:$mask, (v8f32 (fpround (loadv8f64 addr:$src))),
+                     VR256X:$src0),
+            (VCVTPD2PSZrmk VR256X:$src0, VK8WM:$mask, addr:$src)>;
+  def : Pat<(vselect VK8WM:$mask, (v8f32 (fpround (loadv8f64 addr:$src))),
+                     v8f32x_info.ImmAllZerosV),
+            (VCVTPD2PSZrmkz VK8WM:$mask, addr:$src)>;
+
+  def : Pat<(v8f32 (fpround (v8f64 (X86VBroadcast (loadf64 addr:$src))))),
+            (VCVTPD2PSZrmb addr:$src)>;
+  def : Pat<(vselect VK8WM:$mask,
+                     (fpround (v8f64 (X86VBroadcast (loadf64 addr:$src)))),
+                     (v8f32 VR256X:$src0)),
+            (VCVTPD2PSZrmbk VR256X:$src0, VK8WM:$mask, addr:$src)>;
+  def : Pat<(vselect VK8WM:$mask,
+                     (fpround (v8f64 (X86VBroadcast (loadf64 addr:$src)))),
+                     v8f32x_info.ImmAllZerosV),
+            (VCVTPD2PSZrmbkz VK8WM:$mask, addr:$src)>;
+
+  def : Pat<(v8f64 (extloadv8f32 addr:$src)),
+              (VCVTPS2PDZrm addr:$src)>;
+}
 
 let Predicates = [HasVLX] in {
-  def : Pat<(X86vzmovl (v2f64 (bitconvert
-                               (v4f32 (X86vfpround (v2f64 VR128X:$src)))))),
-            (VCVTPD2PSZ128rr VR128X:$src)>;
-  def : Pat<(X86vzmovl (v2f64 (bitconvert
-                               (v4f32 (X86vfpround (loadv2f64 addr:$src)))))),
-            (VCVTPD2PSZ128rm addr:$src)>;
+  def : Pat<(v4f32 (fpround (v4f64 VR256X:$src))),
+            (VCVTPD2PSZ256rr VR256X:$src)>;
+  def : Pat<(vselect VK4WM:$mask, (v4f32 (fpround (v4f64 VR256X:$src))),
+                     VR128X:$src0),
+            (VCVTPD2PSZ256rrk VR128X:$src0, VK4WM:$mask, VR256X:$src)>;
+  def : Pat<(vselect VK4WM:$mask, (v4f32 (fpround (v4f64 VR256X:$src))),
+                     v4f32x_info.ImmAllZerosV),
+            (VCVTPD2PSZ256rrkz VK4WM:$mask, VR256X:$src)>;
+
+  def : Pat<(v4f32 (fpround (loadv4f64 addr:$src))),
+            (VCVTPD2PSZ256rm addr:$src)>;
+  def : Pat<(vselect VK4WM:$mask, (v4f32 (fpround (loadv4f64 addr:$src))),
+                     VR128X:$src0),
+            (VCVTPD2PSZ256rmk VR128X:$src0, VK4WM:$mask, addr:$src)>;
+  def : Pat<(vselect VK4WM:$mask, (v4f32 (fpround (loadv4f64 addr:$src))),
+                     v4f32x_info.ImmAllZerosV),
+            (VCVTPD2PSZ256rmkz VK4WM:$mask, addr:$src)>;
+
+  def : Pat<(v4f32 (fpround (v4f64 (X86VBroadcast (loadf64 addr:$src))))),
+            (VCVTPD2PSZ256rmb addr:$src)>;
+  def : Pat<(vselect VK4WM:$mask,
+                     (v4f32 (fpround (v4f64 (X86VBroadcast (loadf64 addr:$src))))),
+                     VR128X:$src0),
+            (VCVTPD2PSZ256rmbk VR128X:$src0, VK4WM:$mask, addr:$src)>;
+  def : Pat<(vselect VK4WM:$mask,
+                     (v4f32 (fpround (v4f64 (X86VBroadcast (loadf64 addr:$src))))),
+                     v4f32x_info.ImmAllZerosV),
+            (VCVTPD2PSZ256rmbkz VK4WM:$mask, addr:$src)>;
+
   def : Pat<(v2f64 (extloadv2f32 addr:$src)),
               (VCVTPS2PDZ128rm addr:$src)>;
   def : Pat<(v4f64 (extloadv4f32 addr:$src)),
               (VCVTPS2PDZ256rm addr:$src)>;
+
+  // Special patterns to allow use of X86vmfpround for masking. Instruction
+  // patterns have been disabled with null_frag.
+  def : Pat<(X86vfpround (v2f64 VR128X:$src)),
+            (VCVTPD2PSZ128rr VR128X:$src)>;
+  def : Pat<(X86vmfpround (v2f64 VR128X:$src), (v4f32 VR128X:$src0),
+                          VK2WM:$mask),
+            (VCVTPD2PSZ128rrk VR128X:$src0, VK2WM:$mask, VR128X:$src)>;
+  def : Pat<(X86vmfpround (v2f64 VR128X:$src), v4f32x_info.ImmAllZerosV,
+                          VK2WM:$mask),
+            (VCVTPD2PSZ128rrkz VK2WM:$mask, VR128X:$src)>;
+
+  def : Pat<(X86vfpround (loadv2f64 addr:$src)),
+            (VCVTPD2PSZ128rm addr:$src)>;
+  def : Pat<(X86vmfpround (loadv2f64 addr:$src), (v4f32 VR128X:$src0),
+                          VK2WM:$mask),
+            (VCVTPD2PSZ128rmk VR128X:$src0, VK2WM:$mask, addr:$src)>;
+  def : Pat<(X86vmfpround (loadv2f64 addr:$src), v4f32x_info.ImmAllZerosV,
+                          VK2WM:$mask),
+            (VCVTPD2PSZ128rmkz VK2WM:$mask, addr:$src)>;
+
+  def : Pat<(X86vfpround (v2f64 (X86VBroadcast (loadf64 addr:$src)))),
+            (VCVTPD2PSZ128rmb addr:$src)>;
+  def : Pat<(X86vmfpround (v2f64 (X86VBroadcast (loadf64 addr:$src))),
+                          (v4f32 VR128X:$src0), VK2WM:$mask),
+            (VCVTPD2PSZ128rmbk VR128X:$src0, VK2WM:$mask, addr:$src)>;
+  def : Pat<(X86vmfpround (v2f64 (X86VBroadcast (loadf64 addr:$src))),
+                          v4f32x_info.ImmAllZerosV, VK2WM:$mask),
+            (VCVTPD2PSZ128rmbkz VK2WM:$mask, addr:$src)>;
 }
 
 // Convert Signed/Unsigned Doubleword to Double
@@ -8144,19 +8297,20 @@ multiclass avx512_cvttpd2dq<bits<8> opc, string OpcodeStr, SDNode OpNode,
     // dest type - 'v4i32x_info'. We also specify the broadcast string explicitly
     // due to the same reason.
     defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v4i32x_info, v2f64x_info,
-                               OpNode, sched.XMM, "{1to2}", "{x}">, EVEX_V128;
+                               null_frag, sched.XMM, "{1to2}", "{x}", f128mem,
+                               VK2WM>, EVEX_V128;
     defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v4i32x_info, v4f64x_info, OpNode,
                                sched.YMM, "{1to4}", "{y}">, EVEX_V256;
-
-    def : InstAlias<OpcodeStr##"x\t{$src, $dst|$dst, $src}",
-                    (!cast<Instruction>(NAME # "Z128rr") VR128X:$dst, VR128X:$src), 0>;
-    def : InstAlias<OpcodeStr##"x\t{$src, $dst|$dst, $src}",
-                    (!cast<Instruction>(NAME # "Z128rm") VR128X:$dst, i128mem:$src), 0, "intel">;
-    def : InstAlias<OpcodeStr##"y\t{$src, $dst|$dst, $src}",
-                    (!cast<Instruction>(NAME # "Z256rr") VR128X:$dst, VR256X:$src), 0>;
-    def : InstAlias<OpcodeStr##"y\t{$src, $dst|$dst, $src}",
-                    (!cast<Instruction>(NAME # "Z256rm") VR128X:$dst, i256mem:$src), 0, "intel">;
   }
+
+  def : InstAlias<OpcodeStr##"x\t{$src, $dst|$dst, $src}",
+                  (!cast<Instruction>(NAME # "Z128rr") VR128X:$dst, VR128X:$src), 0>;
+  def : InstAlias<OpcodeStr##"x\t{$src, $dst|$dst, $src}",
+                  (!cast<Instruction>(NAME # "Z128rm") VR128X:$dst, i128mem:$src), 0, "intel">;
+  def : InstAlias<OpcodeStr##"y\t{$src, $dst|$dst, $src}",
+                  (!cast<Instruction>(NAME # "Z256rr") VR128X:$dst, VR256X:$src), 0>;
+  def : InstAlias<OpcodeStr##"y\t{$src, $dst|$dst, $src}",
+                  (!cast<Instruction>(NAME # "Z256rm") VR128X:$dst, i256mem:$src), 0, "intel">;
 }
 
 // Convert Double to Signed/Unsigned Doubleword
@@ -8173,20 +8327,21 @@ multiclass avx512_cvtpd2dq<bits<8> opc, string OpcodeStr, SDNode OpNode,
     // memory forms of these instructions in Asm Parcer. They have the same
     // dest type - 'v4i32x_info'. We also specify the broadcast string explicitly
     // due to the same reason.
-    defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v4i32x_info, v2f64x_info, OpNode,
-                               sched.XMM, "{1to2}", "{x}">, EVEX_V128;
+    defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v4i32x_info, v2f64x_info,
+                               null_frag, sched.XMM, "{1to2}", "{x}", f128mem,
+                               VK2WM>, EVEX_V128;
     defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v4i32x_info, v4f64x_info, OpNode,
                                sched.YMM, "{1to4}", "{y}">, EVEX_V256;
-
-    def : InstAlias<OpcodeStr##"x\t{$src, $dst|$dst, $src}",
-                    (!cast<Instruction>(NAME # "Z128rr") VR128X:$dst, VR128X:$src), 0>;
-    def : InstAlias<OpcodeStr##"x\t{$src, $dst|$dst, $src}",
-                    (!cast<Instruction>(NAME # "Z128rm") VR128X:$dst, f128mem:$src), 0, "intel">;
-    def : InstAlias<OpcodeStr##"y\t{$src, $dst|$dst, $src}",
-                    (!cast<Instruction>(NAME # "Z256rr") VR128X:$dst, VR256X:$src), 0>;
-    def : InstAlias<OpcodeStr##"y\t{$src, $dst|$dst, $src}",
-                    (!cast<Instruction>(NAME # "Z256rm") VR128X:$dst, f256mem:$src), 0, "intel">;
   }
+
+  def : InstAlias<OpcodeStr##"x\t{$src, $dst|$dst, $src}",
+                  (!cast<Instruction>(NAME # "Z128rr") VR128X:$dst, VR128X:$src), 0>;
+  def : InstAlias<OpcodeStr##"x\t{$src, $dst|$dst, $src}",
+                  (!cast<Instruction>(NAME # "Z128rm") VR128X:$dst, f128mem:$src), 0, "intel">;
+  def : InstAlias<OpcodeStr##"y\t{$src, $dst|$dst, $src}",
+                  (!cast<Instruction>(NAME # "Z256rr") VR128X:$dst, VR256X:$src), 0>;
+  def : InstAlias<OpcodeStr##"y\t{$src, $dst|$dst, $src}",
+                  (!cast<Instruction>(NAME # "Z256rm") VR128X:$dst, f256mem:$src), 0, "intel">;
 }
 
 // Convert Double to Signed/Unsigned Quardword
@@ -8279,8 +8434,7 @@ multiclass avx512_cvttps2qq<bits<8> opc, string OpcodeStr, SDNode OpNode,
 
 // Convert Signed/Unsigned Quardword to Float
 multiclass avx512_cvtqq2ps<bits<8> opc, string OpcodeStr, SDNode OpNode,
-                           SDNode OpNode128, SDNode OpNodeRnd,
-                           X86SchedWriteWidths sched> {
+                           SDNode OpNodeRnd, X86SchedWriteWidths sched> {
   let Predicates = [HasDQI] in {
     defm Z : avx512_vcvt_fp<opc, OpcodeStr, v8f32x_info, v8i64_info, OpNode,
                             sched.ZMM>,
@@ -8292,22 +8446,22 @@ multiclass avx512_cvtqq2ps<bits<8> opc, string OpcodeStr, SDNode OpNode,
     // memory forms of these instructions in Asm Parcer. They have the same
     // dest type - 'v4i32x_info'. We also specify the broadcast string explicitly
     // due to the same reason.
-    defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v4f32x_info, v2i64x_info, OpNode128,
-                               sched.XMM, "{1to2}", "{x}">, EVEX_V128,
-                               NotEVEX2VEXConvertible;
+    defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v4f32x_info, v2i64x_info, null_frag,
+                               sched.XMM, "{1to2}", "{x}", i128mem, VK2WM>,
+                               EVEX_V128, NotEVEX2VEXConvertible;
     defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v4f32x_info, v4i64x_info, OpNode,
                                sched.YMM, "{1to4}", "{y}">, EVEX_V256,
                                NotEVEX2VEXConvertible;
-
-    def : InstAlias<OpcodeStr##"x\t{$src, $dst|$dst, $src}",
-                    (!cast<Instruction>(NAME # "Z128rr") VR128X:$dst, VR128X:$src), 0>;
-    def : InstAlias<OpcodeStr##"x\t{$src, $dst|$dst, $src}",
-                    (!cast<Instruction>(NAME # "Z128rm") VR128X:$dst, i128mem:$src), 0, "intel">;
-    def : InstAlias<OpcodeStr##"y\t{$src, $dst|$dst, $src}",
-                    (!cast<Instruction>(NAME # "Z256rr") VR128X:$dst, VR256X:$src), 0>;
-    def : InstAlias<OpcodeStr##"y\t{$src, $dst|$dst, $src}",
-                    (!cast<Instruction>(NAME # "Z256rm") VR128X:$dst, i256mem:$src), 0, "intel">;
   }
+
+  def : InstAlias<OpcodeStr##"x\t{$src, $dst|$dst, $src}",
+                  (!cast<Instruction>(NAME # "Z128rr") VR128X:$dst, VR128X:$src), 0>;
+  def : InstAlias<OpcodeStr##"x\t{$src, $dst|$dst, $src}",
+                  (!cast<Instruction>(NAME # "Z128rm") VR128X:$dst, i128mem:$src), 0, "intel">;
+  def : InstAlias<OpcodeStr##"y\t{$src, $dst|$dst, $src}",
+                  (!cast<Instruction>(NAME # "Z256rr") VR128X:$dst, VR256X:$src), 0>;
+  def : InstAlias<OpcodeStr##"y\t{$src, $dst|$dst, $src}",
+                  (!cast<Instruction>(NAME # "Z256rm") VR128X:$dst, i256mem:$src), 0, "intel">;
 }
 
 defm VCVTDQ2PD : avx512_cvtdq2pd<0xE6, "vcvtdq2pd", sint_to_fp, X86VSintToFP,
@@ -8397,11 +8551,11 @@ defm VCVTUQQ2PD : avx512_cvtqq2pd<0x7A, "vcvtuqq2pd", uint_to_fp,
                             X86VUintToFpRnd, SchedWriteCvtDQ2PD>, VEX_W, XS,
                             EVEX_CD8<64, CD8VF>;
 
-defm VCVTQQ2PS : avx512_cvtqq2ps<0x5B, "vcvtqq2ps", sint_to_fp, X86VSintToFP,
+defm VCVTQQ2PS : avx512_cvtqq2ps<0x5B, "vcvtqq2ps", sint_to_fp,
                             X86VSintToFpRnd, SchedWriteCvtDQ2PS>, VEX_W, PS,
                             EVEX_CD8<64, CD8VF>;
 
-defm VCVTUQQ2PS : avx512_cvtqq2ps<0x7A, "vcvtuqq2ps", uint_to_fp, X86VUintToFP,
+defm VCVTUQQ2PS : avx512_cvtqq2ps<0x7A, "vcvtuqq2ps", uint_to_fp,
                             X86VUintToFpRnd, SchedWriteCvtDQ2PS>, VEX_W, XD,
                             EVEX_CD8<64, CD8VF>;
 
@@ -8457,6 +8611,122 @@ let Predicates = [HasVLX] in {
             (VCVTTPD2UDQZ256rr VR256X:$src)>;
   def : Pat<(v4i32 (fp_to_uint (loadv4f64 addr:$src))),
             (VCVTTPD2UDQZ256rm addr:$src)>;
+
+  // Special patterns to allow use of X86mcvtp2Int for masking. Instruction
+  // patterns have been disabled with null_frag.
+  def : Pat<(v4i32 (X86cvtp2Int (v2f64 VR128X:$src))),
+            (VCVTPD2DQZ128rr VR128X:$src)>;
+  def : Pat<(X86mcvtp2Int (v2f64 VR128X:$src), (v4i32 VR128X:$src0),
+                          VK2WM:$mask),
+            (VCVTPD2DQZ128rrk VR128X:$src0, VK2WM:$mask, VR128X:$src)>;
+  def : Pat<(X86mcvtp2Int (v2f64 VR128X:$src), v4i32x_info.ImmAllZerosV,
+                          VK2WM:$mask),
+            (VCVTPD2DQZ128rrkz VK2WM:$mask, VR128X:$src)>;
+
+  def : Pat<(v4i32 (X86cvtp2Int (loadv2f64 addr:$src))),
+            (VCVTPD2DQZ128rm addr:$src)>;
+  def : Pat<(X86mcvtp2Int (loadv2f64 addr:$src), (v4i32 VR128X:$src0),
+                          VK2WM:$mask),
+            (VCVTPD2DQZ128rmk VR128X:$src0, VK2WM:$mask, addr:$src)>;
+  def : Pat<(X86mcvtp2Int (loadv2f64 addr:$src), v4i32x_info.ImmAllZerosV,
+                          VK2WM:$mask),
+            (VCVTPD2DQZ128rmkz VK2WM:$mask, addr:$src)>;
+
+  def : Pat<(v4i32 (X86cvtp2Int (v2f64 (X86VBroadcast (loadf64 addr:$src))))),
+            (VCVTPD2DQZ128rmb addr:$src)>;
+  def : Pat<(X86mcvtp2Int (v2f64 (X86VBroadcast (loadf64 addr:$src))),
+                          (v4i32 VR128X:$src0), VK2WM:$mask),
+            (VCVTPD2DQZ128rmbk VR128X:$src0, VK2WM:$mask, addr:$src)>;
+  def : Pat<(X86mcvtp2Int (v2f64 (X86VBroadcast (loadf64 addr:$src))),
+                          v4i32x_info.ImmAllZerosV, VK2WM:$mask),
+            (VCVTPD2DQZ128rmbkz VK2WM:$mask, addr:$src)>;
+
+  // Special patterns to allow use of X86mcvttp2si for masking. Instruction
+  // patterns have been disabled with null_frag.
+  def : Pat<(v4i32 (X86cvttp2si (v2f64 VR128X:$src))),
+            (VCVTTPD2DQZ128rr VR128X:$src)>;
+  def : Pat<(X86mcvttp2si (v2f64 VR128X:$src), (v4i32 VR128X:$src0),
+                          VK2WM:$mask),
+            (VCVTTPD2DQZ128rrk VR128X:$src0, VK2WM:$mask, VR128X:$src)>;
+  def : Pat<(X86mcvttp2si (v2f64 VR128X:$src), v4i32x_info.ImmAllZerosV,
+                          VK2WM:$mask),
+            (VCVTTPD2DQZ128rrkz VK2WM:$mask, VR128X:$src)>;
+
+  def : Pat<(v4i32 (X86cvttp2si (loadv2f64 addr:$src))),
+            (VCVTTPD2DQZ128rm addr:$src)>;
+  def : Pat<(X86mcvttp2si (loadv2f64 addr:$src), (v4i32 VR128X:$src0),
+                          VK2WM:$mask),
+            (VCVTTPD2DQZ128rmk VR128X:$src0, VK2WM:$mask, addr:$src)>;
+  def : Pat<(X86mcvttp2si (loadv2f64 addr:$src), v4i32x_info.ImmAllZerosV,
+                          VK2WM:$mask),
+            (VCVTTPD2DQZ128rmkz VK2WM:$mask, addr:$src)>;
+
+  def : Pat<(v4i32 (X86cvttp2si (v2f64 (X86VBroadcast (loadf64 addr:$src))))),
+            (VCVTTPD2DQZ128rmb addr:$src)>;
+  def : Pat<(X86mcvttp2si (v2f64 (X86VBroadcast (loadf64 addr:$src))),
+                          (v4i32 VR128X:$src0), VK2WM:$mask),
+            (VCVTTPD2DQZ128rmbk VR128X:$src0, VK2WM:$mask, addr:$src)>;
+  def : Pat<(X86mcvttp2si (v2f64 (X86VBroadcast (loadf64 addr:$src))),
+                          v4i32x_info.ImmAllZerosV, VK2WM:$mask),
+            (VCVTTPD2DQZ128rmbkz VK2WM:$mask, addr:$src)>;
+
+  // Special patterns to allow use of X86mcvtp2UInt for masking. Instruction
+  // patterns have been disabled with null_frag.
+  def : Pat<(v4i32 (X86cvtp2UInt (v2f64 VR128X:$src))),
+            (VCVTPD2UDQZ128rr VR128X:$src)>;
+  def : Pat<(X86mcvtp2UInt (v2f64 VR128X:$src), (v4i32 VR128X:$src0),
+                           VK2WM:$mask),
+            (VCVTPD2UDQZ128rrk VR128X:$src0, VK2WM:$mask, VR128X:$src)>;
+  def : Pat<(X86mcvtp2UInt (v2f64 VR128X:$src), v4i32x_info.ImmAllZerosV,
+                           VK2WM:$mask),
+            (VCVTPD2UDQZ128rrkz VK2WM:$mask, VR128X:$src)>;
+
+  def : Pat<(v4i32 (X86cvtp2UInt (loadv2f64 addr:$src))),
+            (VCVTPD2UDQZ128rm addr:$src)>;
+  def : Pat<(X86mcvtp2UInt (loadv2f64 addr:$src), (v4i32 VR128X:$src0),
+                           VK2WM:$mask),
+            (VCVTPD2UDQZ128rmk VR128X:$src0, VK2WM:$mask, addr:$src)>;
+  def : Pat<(X86mcvtp2UInt (loadv2f64 addr:$src), v4i32x_info.ImmAllZerosV,
+                           VK2WM:$mask),
+            (VCVTPD2UDQZ128rmkz VK2WM:$mask, addr:$src)>;
+
+  def : Pat<(v4i32 (X86cvtp2UInt (v2f64 (X86VBroadcast (loadf64 addr:$src))))),
+            (VCVTPD2UDQZ128rmb addr:$src)>;
+  def : Pat<(X86mcvtp2UInt (v2f64 (X86VBroadcast (loadf64 addr:$src))),
+                           (v4i32 VR128X:$src0), VK2WM:$mask),
+            (VCVTPD2UDQZ128rmbk VR128X:$src0, VK2WM:$mask, addr:$src)>;
+  def : Pat<(X86mcvtp2UInt (v2f64 (X86VBroadcast (loadf64 addr:$src))),
+                           v4i32x_info.ImmAllZerosV, VK2WM:$mask),
+            (VCVTPD2UDQZ128rmbkz VK2WM:$mask, addr:$src)>;
+
+  // Special patterns to allow use of X86mcvtp2UInt for masking. Instruction
+  // patterns have been disabled with null_frag.
+  def : Pat<(v4i32 (X86cvttp2ui (v2f64 VR128X:$src))),
+            (VCVTTPD2UDQZ128rr VR128X:$src)>;
+  def : Pat<(X86mcvttp2ui (v2f64 VR128X:$src), (v4i32 VR128X:$src0),
+                          VK2WM:$mask),
+            (VCVTTPD2UDQZ128rrk VR128X:$src0, VK2WM:$mask, VR128X:$src)>;
+  def : Pat<(X86mcvttp2ui (v2f64 VR128X:$src), v4i32x_info.ImmAllZerosV,
+                          VK2WM:$mask),
+            (VCVTTPD2UDQZ128rrkz VK2WM:$mask, VR128X:$src)>;
+
+  def : Pat<(v4i32 (X86cvttp2ui (loadv2f64 addr:$src))),
+            (VCVTTPD2UDQZ128rm addr:$src)>;
+  def : Pat<(X86mcvttp2ui (loadv2f64 addr:$src), (v4i32 VR128X:$src0),
+                          VK2WM:$mask),
+            (VCVTTPD2UDQZ128rmk VR128X:$src0, VK2WM:$mask, addr:$src)>;
+  def : Pat<(X86mcvttp2ui (loadv2f64 addr:$src), v4i32x_info.ImmAllZerosV,
+                          VK2WM:$mask),
+            (VCVTTPD2UDQZ128rmkz VK2WM:$mask, addr:$src)>;
+
+  def : Pat<(v4i32 (X86cvttp2ui (v2f64 (X86VBroadcast (loadf64 addr:$src))))),
+            (VCVTTPD2UDQZ128rmb addr:$src)>;
+  def : Pat<(X86mcvttp2ui (v2f64 (X86VBroadcast (loadf64 addr:$src))),
+                          (v4i32 VR128X:$src0), VK2WM:$mask),
+            (VCVTTPD2UDQZ128rmbk VR128X:$src0, VK2WM:$mask, addr:$src)>;
+  def : Pat<(X86mcvttp2ui (v2f64 (X86VBroadcast (loadf64 addr:$src))),
+                          v4i32x_info.ImmAllZerosV, VK2WM:$mask),
+            (VCVTTPD2UDQZ128rmbkz VK2WM:$mask, addr:$src)>;
 }
 
 let Predicates = [HasDQI] in {
@@ -8551,25 +8821,6 @@ def : Pat<(v2f64 (X86VUintToFP (v4i32 VR128X:$src1))),
 }
 
 let Predicates = [HasAVX512, HasVLX] in {
-  def : Pat<(X86vzmovl (v2i64 (bitconvert
-                              (v4i32 (X86cvtp2Int (v2f64 VR128X:$src)))))),
-            (VCVTPD2DQZ128rr VR128X:$src)>;
-  def : Pat<(X86vzmovl (v2i64 (bitconvert
-                              (v4i32 (X86cvtp2Int (loadv2f64 addr:$src)))))),
-            (VCVTPD2DQZ128rm addr:$src)>;
-  def : Pat<(X86vzmovl (v2i64 (bitconvert
-                               (v4i32 (X86cvtp2UInt (v2f64 VR128X:$src)))))),
-            (VCVTPD2UDQZ128rr VR128X:$src)>;
-  def : Pat<(X86vzmovl (v2i64 (bitconvert
-                              (v4i32 (X86cvttp2si (v2f64 VR128X:$src)))))),
-            (VCVTTPD2DQZ128rr VR128X:$src)>;
-  def : Pat<(X86vzmovl (v2i64 (bitconvert
-                              (v4i32 (X86cvttp2si (loadv2f64 addr:$src)))))),
-            (VCVTTPD2DQZ128rm addr:$src)>;
-  def : Pat<(X86vzmovl (v2i64 (bitconvert
-                               (v4i32 (X86cvttp2ui (v2f64 VR128X:$src)))))),
-            (VCVTTPD2UDQZ128rr VR128X:$src)>;
-
   def : Pat<(v2f64 (X86VSintToFP (bc_v4i32 (v2i64 (scalar_to_vector (loadi64 addr:$src)))))),
             (VCVTDQ2PDZ128rm addr:$src)>;
   def : Pat<(v2f64 (X86VSintToFP (bc_v4i32 (v2i64 (X86vzload addr:$src))))),
@@ -8581,20 +8832,64 @@ let Predicates = [HasAVX512, HasVLX] in {
             (VCVTUDQ2PDZ128rm addr:$src)>;
 }
 
-let Predicates = [HasAVX512] in {
-  def : Pat<(v8f32 (fpround (loadv8f64 addr:$src))),
-            (VCVTPD2PSZrm addr:$src)>;
-  def : Pat<(v8f64 (extloadv8f32 addr:$src)),
-            (VCVTPS2PDZrm addr:$src)>;
-}
-
 let Predicates = [HasDQI, HasVLX] in {
-  def : Pat<(X86vzmovl (v2f64 (bitconvert
-                              (v4f32 (X86VSintToFP (v2i64 VR128X:$src)))))),
+  // Special patterns to allow use of X86VMSintToFP for masking. Instruction
+  // patterns have been disabled with null_frag.
+  def : Pat<(v4f32 (X86VSintToFP (v2i64 VR128X:$src))),
             (VCVTQQ2PSZ128rr VR128X:$src)>;
-  def : Pat<(X86vzmovl (v2f64 (bitconvert
-                              (v4f32 (X86VUintToFP (v2i64 VR128X:$src)))))),
+  def : Pat<(X86VMSintToFP (v2i64 VR128X:$src), (v4f32 VR128X:$src0),
+                           VK2WM:$mask),
+            (VCVTQQ2PSZ128rrk VR128X:$src0, VK2WM:$mask, VR128X:$src)>;
+  def : Pat<(X86VMSintToFP (v2i64 VR128X:$src), v4f32x_info.ImmAllZerosV,
+                           VK2WM:$mask),
+            (VCVTQQ2PSZ128rrkz VK2WM:$mask, VR128X:$src)>;
+
+  def : Pat<(v4f32 (X86VSintToFP (loadv2i64 addr:$src))),
+            (VCVTQQ2PSZ128rm addr:$src)>;
+  def : Pat<(X86VMSintToFP (loadv2i64 addr:$src), (v4f32 VR128X:$src0),
+                           VK2WM:$mask),
+            (VCVTQQ2PSZ128rmk VR128X:$src0, VK2WM:$mask, addr:$src)>;
+  def : Pat<(X86VMSintToFP (loadv2i64 addr:$src), v4f32x_info.ImmAllZerosV,
+                           VK2WM:$mask),
+            (VCVTQQ2PSZ128rmkz VK2WM:$mask, addr:$src)>;
+
+  def : Pat<(v4f32 (X86VSintToFP (v2i64 (X86VBroadcast (loadi64 addr:$src))))),
+            (VCVTQQ2PSZ128rmb addr:$src)>;
+  def : Pat<(X86VMSintToFP (v2i64 (X86VBroadcast (loadi64 addr:$src))),
+                           (v4f32 VR128X:$src0), VK2WM:$mask),
+            (VCVTQQ2PSZ128rmbk VR128X:$src0, VK2WM:$mask, addr:$src)>;
+  def : Pat<(X86VMSintToFP (v2i64 (X86VBroadcast (loadi64 addr:$src))),
+                           v4f32x_info.ImmAllZerosV, VK2WM:$mask),
+            (VCVTQQ2PSZ128rmbkz VK2WM:$mask, addr:$src)>;
+
+  // Special patterns to allow use of X86VMUintToFP for masking. Instruction
+  // patterns have been disabled with null_frag.
+  def : Pat<(v4f32 (X86VUintToFP (v2i64 VR128X:$src))),
             (VCVTUQQ2PSZ128rr VR128X:$src)>;
+  def : Pat<(X86VMUintToFP (v2i64 VR128X:$src), (v4f32 VR128X:$src0),
+                           VK2WM:$mask),
+            (VCVTUQQ2PSZ128rrk VR128X:$src0, VK2WM:$mask, VR128X:$src)>;
+  def : Pat<(X86VMUintToFP (v2i64 VR128X:$src), v4f32x_info.ImmAllZerosV,
+                           VK2WM:$mask),
+            (VCVTUQQ2PSZ128rrkz VK2WM:$mask, VR128X:$src)>;
+
+  def : Pat<(v4f32 (X86VUintToFP (loadv2i64 addr:$src))),
+            (VCVTUQQ2PSZ128rm addr:$src)>;
+  def : Pat<(X86VMUintToFP (loadv2i64 addr:$src), (v4f32 VR128X:$src0),
+                           VK2WM:$mask),
+            (VCVTUQQ2PSZ128rmk VR128X:$src0, VK2WM:$mask, addr:$src)>;
+  def : Pat<(X86VMUintToFP (loadv2i64 addr:$src), v4f32x_info.ImmAllZerosV,
+                           VK2WM:$mask),
+            (VCVTUQQ2PSZ128rmkz VK2WM:$mask, addr:$src)>;
+
+  def : Pat<(v4f32 (X86VUintToFP (v2i64 (X86VBroadcast (loadi64 addr:$src))))),
+            (VCVTUQQ2PSZ128rmb addr:$src)>;
+  def : Pat<(X86VMUintToFP (v2i64 (X86VBroadcast (loadi64 addr:$src))),
+                           (v4f32 VR128X:$src0), VK2WM:$mask),
+            (VCVTUQQ2PSZ128rmbk VR128X:$src0, VK2WM:$mask, addr:$src)>;
+  def : Pat<(X86VMUintToFP (v2i64 (X86VBroadcast (loadi64 addr:$src))),
+                           v4f32x_info.ImmAllZerosV, VK2WM:$mask),
+            (VCVTUQQ2PSZ128rmbkz VK2WM:$mask, addr:$src)>;
 }
 
 let Predicates = [HasDQI, NoVLX] in {
@@ -8713,12 +9008,28 @@ let Predicates = [HasVLX] in {
 
 multiclass avx512_cvtps2ph<X86VectorVTInfo _dest, X86VectorVTInfo _src,
                            X86MemOperand x86memop, SchedWrite RR, SchedWrite MR> {
-  defm rr : AVX512_maskable<0x1D, MRMDestReg, _dest ,(outs _dest.RC:$dst),
-                   (ins _src.RC:$src1, i32u8imm:$src2),
-                   "vcvtps2ph", "$src2, $src1", "$src1, $src2",
-                   (X86cvtps2ph (_src.VT _src.RC:$src1),
-                                (i32 imm:$src2)), 0, 0>,
-                   AVX512AIi8Base, Sched<[RR]>;
+let ExeDomain = GenericDomain in {
+  def rr : AVX512AIi8<0x1D, MRMDestReg, (outs _dest.RC:$dst),
+             (ins _src.RC:$src1, i32u8imm:$src2),
+             "vcvtps2ph\t{$src2, $src1, $dst|$dst, $src1, $src2}",
+             [(set _dest.RC:$dst,
+                   (X86cvtps2ph (_src.VT _src.RC:$src1), (i32 imm:$src2)))]>,
+             Sched<[RR]>;
+  let Constraints = "$src0 = $dst" in
+  def rrk : AVX512AIi8<0x1D, MRMDestReg, (outs _dest.RC:$dst),
+             (ins _dest.RC:$src0, _src.KRCWM:$mask, _src.RC:$src1, i32u8imm:$src2),
+             "vcvtps2ph\t{$src2, $src1, $dst {${mask}}|$dst {${mask}}, $src1, $src2}",
+             [(set _dest.RC:$dst,
+                   (X86mcvtps2ph (_src.VT _src.RC:$src1), (i32 imm:$src2),
+                                 _dest.RC:$src0, _src.KRCWM:$mask))]>,
+             Sched<[RR]>, EVEX_K;
+  def rrkz : AVX512AIi8<0x1D, MRMDestReg, (outs _dest.RC:$dst),
+             (ins _src.KRCWM:$mask, _src.RC:$src1, i32u8imm:$src2),
+             "vcvtps2ph\t{$src2, $src1, $dst {${mask}} {z}|$dst {${mask}} {z}, $src1, $src2}",
+             [(set _dest.RC:$dst,
+                   (X86mcvtps2ph (_src.VT _src.RC:$src1), (i32 imm:$src2),
+                                 _dest.ImmAllZerosV, _src.KRCWM:$mask))]>,
+             Sched<[RR]>, EVEX_KZ;
   let hasSideEffects = 0, mayStore = 1 in {
     def mr : AVX512AIi8<0x1D, MRMDestMem, (outs),
                (ins x86memop:$dst, _src.RC:$src1, i32u8imm:$src2),
@@ -8730,6 +9041,7 @@ multiclass avx512_cvtps2ph<X86VectorVTInfo _dest, X86VectorVTInfo _src,
                 EVEX_K, Sched<[MR]>, NotMemoryFoldable;
   }
 }
+}
 
 multiclass avx512_cvtps2ph_sae<X86VectorVTInfo _dest, X86VectorVTInfo _src,
                                SchedWrite Sched> {
@@ -9307,14 +9619,47 @@ defm : avx512_masked_scalar_imm<fceil, "RNDSCALESD", X86Movsd,
 // Integer truncate and extend operations
 //-------------------------------------------------
 
+// PatFrags that contain a select and a truncate op. The take operands in the
+// same order as X86vmtrunc, X86vmtruncs, X86vmtruncus. This allows us to pass
+// either to the multiclasses.
+def select_trunc : PatFrag<(ops node:$src, node:$src0, node:$mask),
+                           (vselect node:$mask,
+                                    (trunc node:$src), node:$src0)>;
+def select_truncs : PatFrag<(ops node:$src, node:$src0, node:$mask),
+                            (vselect node:$mask,
+                                     (X86vtruncs node:$src), node:$src0)>;
+def select_truncus : PatFrag<(ops node:$src, node:$src0, node:$mask),
+                             (vselect node:$mask,
+                                      (X86vtruncus node:$src), node:$src0)>;
+
 multiclass avx512_trunc_common<bits<8> opc, string OpcodeStr, SDNode OpNode,
+                              SDPatternOperator MaskNode,
                               X86FoldableSchedWrite sched, X86VectorVTInfo SrcInfo,
                               X86VectorVTInfo DestInfo, X86MemOperand x86memop> {
-  let ExeDomain = DestInfo.ExeDomain in
-  defm rr  : AVX512_maskable<opc, MRMDestReg, DestInfo, (outs DestInfo.RC:$dst),
-                      (ins SrcInfo.RC:$src1), OpcodeStr ,"$src1", "$src1",
-                      (DestInfo.VT (OpNode (SrcInfo.VT SrcInfo.RC:$src1)))>,
-                      EVEX, T8XS, Sched<[sched]>;
+  let ExeDomain = DestInfo.ExeDomain in {
+  def rr : AVX512XS8I<opc, MRMDestReg, (outs DestInfo.RC:$dst),
+             (ins SrcInfo.RC:$src),
+             OpcodeStr # "\t{$src, $dst|$dst, $src}",
+             [(set DestInfo.RC:$dst,
+                   (DestInfo.VT (OpNode (SrcInfo.VT SrcInfo.RC:$src))))]>,
+             EVEX, Sched<[sched]>;
+  let Constraints = "$src0 = $dst" in
+  def rrk : AVX512XS8I<opc, MRMDestReg, (outs DestInfo.RC:$dst),
+             (ins DestInfo.RC:$src0, SrcInfo.KRCWM:$mask, SrcInfo.RC:$src),
+             OpcodeStr # "\t{$src, $dst {${mask}}|$dst {${mask}}, $src}",
+             [(set DestInfo.RC:$dst,
+                   (MaskNode (SrcInfo.VT SrcInfo.RC:$src),
+                             (DestInfo.VT DestInfo.RC:$src0),
+                             SrcInfo.KRCWM:$mask))]>,
+             EVEX, EVEX_K, Sched<[sched]>;
+  def rrkz : AVX512XS8I<opc, MRMDestReg, (outs DestInfo.RC:$dst),
+             (ins SrcInfo.KRCWM:$mask, SrcInfo.RC:$src),
+             OpcodeStr # "\t{$src, $dst {${mask}} {z}|$dst {${mask}} {z}, $src}",
+             [(set DestInfo.RC:$dst,
+                   (DestInfo.VT (MaskNode (SrcInfo.VT SrcInfo.RC:$src),
+                             DestInfo.ImmAllZerosV, SrcInfo.KRCWM:$mask)))]>,
+             EVEX, EVEX_KZ, Sched<[sched]>;
+  }
 
   let mayStore = 1, hasSideEffects = 0, ExeDomain = DestInfo.ExeDomain in {
     def mr : AVX512XS8I<opc, MRMDestMem, (outs),
@@ -9345,7 +9690,11 @@ multiclass avx512_trunc_mr_lowering<X86VectorVTInfo SrcInfo,
 }
 
 multiclass avx512_trunc<bits<8> opc, string OpcodeStr, SDNode OpNode128,
-                        SDNode OpNode256, SDNode OpNode512, X86FoldableSchedWrite sched,
+                        SDNode OpNode256, SDNode OpNode512,
+                        SDPatternOperator MaskNode128,
+                        SDPatternOperator MaskNode256,
+                        SDPatternOperator MaskNode512,
+                        X86FoldableSchedWrite sched,
                         AVX512VLVectorVTInfo VTSrcInfo,
                         X86VectorVTInfo DestInfoZ128,
                         X86VectorVTInfo DestInfoZ256, X86VectorVTInfo DestInfoZ,
@@ -9354,118 +9703,167 @@ multiclass avx512_trunc<bits<8> opc, string OpcodeStr, SDNode OpNode128,
                         PatFrag mtruncFrag, Predicate prd = HasAVX512>{
 
   let Predicates = [HasVLX, prd] in {
-    defm Z128:  avx512_trunc_common<opc, OpcodeStr, OpNode128, sched,
+    defm Z128:  avx512_trunc_common<opc, OpcodeStr, OpNode128, MaskNode128, sched,
                              VTSrcInfo.info128, DestInfoZ128, x86memopZ128>,
                 avx512_trunc_mr_lowering<VTSrcInfo.info128, DestInfoZ128,
                              truncFrag, mtruncFrag, NAME>, EVEX_V128;
 
-    defm Z256:  avx512_trunc_common<opc, OpcodeStr, OpNode256, sched,
+    defm Z256:  avx512_trunc_common<opc, OpcodeStr, OpNode256, MaskNode256, sched,
                              VTSrcInfo.info256, DestInfoZ256, x86memopZ256>,
                 avx512_trunc_mr_lowering<VTSrcInfo.info256, DestInfoZ256,
                              truncFrag, mtruncFrag, NAME>, EVEX_V256;
   }
   let Predicates = [prd] in
-    defm Z:     avx512_trunc_common<opc, OpcodeStr, OpNode512, sched,
+    defm Z:     avx512_trunc_common<opc, OpcodeStr, OpNode512, MaskNode512, sched,
                              VTSrcInfo.info512, DestInfoZ, x86memopZ>,
                 avx512_trunc_mr_lowering<VTSrcInfo.info512, DestInfoZ,
                              truncFrag, mtruncFrag, NAME>, EVEX_V512;
 }
 
 multiclass avx512_trunc_qb<bits<8> opc, string OpcodeStr, SDNode OpNode,
+                           SDPatternOperator MaskNode,
                            X86FoldableSchedWrite sched, PatFrag StoreNode,
-                           PatFrag MaskedStoreNode, SDNode InVecNode = OpNode> {
-  defm NAME: avx512_trunc<opc, OpcodeStr, InVecNode, InVecNode, InVecNode, sched,
+                           PatFrag MaskedStoreNode, SDNode InVecNode,
+                           SDPatternOperator InVecMaskNode> {
+  defm NAME: avx512_trunc<opc, OpcodeStr, InVecNode, InVecNode, InVecNode,
+                          InVecMaskNode, InVecMaskNode, InVecMaskNode, sched,
                           avx512vl_i64_info, v16i8x_info, v16i8x_info,
                           v16i8x_info, i16mem, i32mem, i64mem, StoreNode,
                           MaskedStoreNode>, EVEX_CD8<8, CD8VO>;
 }
 
 multiclass avx512_trunc_qw<bits<8> opc, string OpcodeStr, SDNode OpNode,
+                           SDPatternOperator MaskNode,
                            X86FoldableSchedWrite sched, PatFrag StoreNode,
-                           PatFrag MaskedStoreNode, SDNode InVecNode = OpNode> {
-  defm NAME: avx512_trunc<opc, OpcodeStr, InVecNode, InVecNode, OpNode, sched,
+                           PatFrag MaskedStoreNode, SDNode InVecNode,
+                           SDPatternOperator InVecMaskNode> {
+  defm NAME: avx512_trunc<opc, OpcodeStr, InVecNode, InVecNode, OpNode,
+                          InVecMaskNode, InVecMaskNode, MaskNode, sched,
                           avx512vl_i64_info, v8i16x_info, v8i16x_info,
                           v8i16x_info, i32mem, i64mem, i128mem, StoreNode,
                           MaskedStoreNode>, EVEX_CD8<16, CD8VQ>;
 }
 
 multiclass avx512_trunc_qd<bits<8> opc, string OpcodeStr, SDNode OpNode,
+                           SDPatternOperator MaskNode,
                            X86FoldableSchedWrite sched, PatFrag StoreNode,
-                           PatFrag MaskedStoreNode, SDNode InVecNode = OpNode> {
-  defm NAME: avx512_trunc<opc, OpcodeStr, InVecNode, OpNode, OpNode, sched,
+                           PatFrag MaskedStoreNode, SDNode InVecNode,
+                           SDPatternOperator InVecMaskNode> {
+  defm NAME: avx512_trunc<opc, OpcodeStr, InVecNode, OpNode, OpNode,
+                          InVecMaskNode, MaskNode, MaskNode, sched,
                           avx512vl_i64_info, v4i32x_info, v4i32x_info,
                           v8i32x_info, i64mem, i128mem, i256mem, StoreNode,
                           MaskedStoreNode>, EVEX_CD8<32, CD8VH>;
 }
 
 multiclass avx512_trunc_db<bits<8> opc, string OpcodeStr, SDNode OpNode,
+                           SDPatternOperator MaskNode,
                            X86FoldableSchedWrite sched, PatFrag StoreNode,
-                           PatFrag MaskedStoreNode, SDNode InVecNode = OpNode> {
-  defm NAME: avx512_trunc<opc, OpcodeStr, InVecNode, InVecNode, OpNode, sched,
+                           PatFrag MaskedStoreNode, SDNode InVecNode,
+                           SDPatternOperator InVecMaskNode> {
+  defm NAME: avx512_trunc<opc, OpcodeStr, InVecNode, InVecNode, OpNode,
+                          InVecMaskNode, InVecMaskNode, MaskNode, sched,
                           avx512vl_i32_info, v16i8x_info, v16i8x_info,
                           v16i8x_info, i32mem, i64mem, i128mem, StoreNode,
                           MaskedStoreNode>, EVEX_CD8<8, CD8VQ>;
 }
 
 multiclass avx512_trunc_dw<bits<8> opc, string OpcodeStr, SDNode OpNode,
+                           SDPatternOperator MaskNode,
                            X86FoldableSchedWrite sched, PatFrag StoreNode,
-                           PatFrag MaskedStoreNode, SDNode InVecNode = OpNode> {
-  defm NAME: avx512_trunc<opc, OpcodeStr, InVecNode, OpNode, OpNode, sched,
+                           PatFrag MaskedStoreNode, SDNode InVecNode,
+                           SDPatternOperator InVecMaskNode> {
+  defm NAME: avx512_trunc<opc, OpcodeStr, InVecNode, OpNode, OpNode,
+                          InVecMaskNode, MaskNode, MaskNode, sched,
                           avx512vl_i32_info, v8i16x_info, v8i16x_info,
                           v16i16x_info, i64mem, i128mem, i256mem, StoreNode,
                           MaskedStoreNode>, EVEX_CD8<16, CD8VH>;
 }
 
 multiclass avx512_trunc_wb<bits<8> opc, string OpcodeStr, SDNode OpNode,
+                           SDPatternOperator MaskNode,
                            X86FoldableSchedWrite sched, PatFrag StoreNode,
-                           PatFrag MaskedStoreNode, SDNode InVecNode = OpNode> {
+                           PatFrag MaskedStoreNode, SDNode InVecNode,
+                           SDPatternOperator InVecMaskNode> {
   defm NAME: avx512_trunc<opc, OpcodeStr, InVecNode, OpNode, OpNode,
-                          sched, avx512vl_i16_info, v16i8x_info, v16i8x_info,
+                          InVecMaskNode, MaskNode, MaskNode, sched,
+                          avx512vl_i16_info, v16i8x_info, v16i8x_info,
                           v32i8x_info, i64mem, i128mem, i256mem, StoreNode,
                           MaskedStoreNode, HasBWI>, EVEX_CD8<16, CD8VH>;
 }
 
-defm VPMOVQB    : avx512_trunc_qb<0x32, "vpmovqb",   trunc, WriteShuffle256,
-                                  truncstorevi8, masked_truncstorevi8, X86vtrunc>;
-defm VPMOVSQB   : avx512_trunc_qb<0x22, "vpmovsqb",  X86vtruncs, WriteShuffle256,
-                                  truncstore_s_vi8, masked_truncstore_s_vi8>;
-defm VPMOVUSQB  : avx512_trunc_qb<0x12, "vpmovusqb", X86vtruncus, WriteShuffle256,
-                                  truncstore_us_vi8, masked_truncstore_us_vi8>;
-
-defm VPMOVQW    : avx512_trunc_qw<0x34, "vpmovqw",   trunc, WriteShuffle256,
-                                  truncstorevi16, masked_truncstorevi16, X86vtrunc>;
-defm VPMOVSQW   : avx512_trunc_qw<0x24, "vpmovsqw",  X86vtruncs, WriteShuffle256,
-                                  truncstore_s_vi16, masked_truncstore_s_vi16>;
-defm VPMOVUSQW  : avx512_trunc_qw<0x14, "vpmovusqw", X86vtruncus, WriteShuffle256,
-                                  truncstore_us_vi16, masked_truncstore_us_vi16>;
-
-defm VPMOVQD    : avx512_trunc_qd<0x35, "vpmovqd",   trunc, WriteShuffle256,
-                                  truncstorevi32, masked_truncstorevi32, X86vtrunc>;
-defm VPMOVSQD   : avx512_trunc_qd<0x25, "vpmovsqd",  X86vtruncs, WriteShuffle256,
-                                  truncstore_s_vi32, masked_truncstore_s_vi32>;
-defm VPMOVUSQD  : avx512_trunc_qd<0x15, "vpmovusqd", X86vtruncus, WriteShuffle256,
-                                  truncstore_us_vi32, masked_truncstore_us_vi32>;
-
-defm VPMOVDB    : avx512_trunc_db<0x31, "vpmovdb", trunc, WriteShuffle256,
-                                  truncstorevi8, masked_truncstorevi8, X86vtrunc>;
-defm VPMOVSDB   : avx512_trunc_db<0x21, "vpmovsdb",   X86vtruncs, WriteShuffle256,
-                                  truncstore_s_vi8, masked_truncstore_s_vi8>;
-defm VPMOVUSDB  : avx512_trunc_db<0x11, "vpmovusdb",  X86vtruncus, WriteShuffle256,
-                                  truncstore_us_vi8, masked_truncstore_us_vi8>;
-
-defm VPMOVDW    : avx512_trunc_dw<0x33, "vpmovdw", trunc, WriteShuffle256,
-                                  truncstorevi16, masked_truncstorevi16, X86vtrunc>;
-defm VPMOVSDW   : avx512_trunc_dw<0x23, "vpmovsdw",   X86vtruncs, WriteShuffle256,
-                                  truncstore_s_vi16, masked_truncstore_s_vi16>;
-defm VPMOVUSDW  : avx512_trunc_dw<0x13, "vpmovusdw",  X86vtruncus, WriteShuffle256,
-                                  truncstore_us_vi16, masked_truncstore_us_vi16>;
-
-defm VPMOVWB    : avx512_trunc_wb<0x30, "vpmovwb", trunc, WriteShuffle256,
-                                  truncstorevi8, masked_truncstorevi8, X86vtrunc>;
-defm VPMOVSWB   : avx512_trunc_wb<0x20, "vpmovswb",   X86vtruncs, WriteShuffle256,
-                                  truncstore_s_vi8, masked_truncstore_s_vi8>;
-defm VPMOVUSWB  : avx512_trunc_wb<0x10, "vpmovuswb",  X86vtruncus, WriteShuffle256,
-                                  truncstore_us_vi8, masked_truncstore_us_vi8>;
+defm VPMOVQB    : avx512_trunc_qb<0x32, "vpmovqb",   trunc, select_trunc,
+                                  WriteShuffle256, truncstorevi8,
+                                  masked_truncstorevi8, X86vtrunc, X86vmtrunc>;
+defm VPMOVSQB   : avx512_trunc_qb<0x22, "vpmovsqb",  X86vtruncs, select_truncs,
+                                  WriteShuffle256, truncstore_s_vi8,
+                                  masked_truncstore_s_vi8, X86vtruncs,
+                                  X86vmtruncs>;
+defm VPMOVUSQB  : avx512_trunc_qb<0x12, "vpmovusqb", X86vtruncus,
+                                  select_truncus, WriteShuffle256,
+                                  truncstore_us_vi8, masked_truncstore_us_vi8,
+                                  X86vtruncus, X86vmtruncus>;
+
+defm VPMOVQW    : avx512_trunc_qw<0x34, "vpmovqw", trunc, select_trunc,
+                                  WriteShuffle256, truncstorevi16,
+                                  masked_truncstorevi16, X86vtrunc, X86vmtrunc>;
+defm VPMOVSQW   : avx512_trunc_qw<0x24, "vpmovsqw",  X86vtruncs, select_truncs,
+                                  WriteShuffle256, truncstore_s_vi16,
+                                  masked_truncstore_s_vi16, X86vtruncs,
+                                  X86vmtruncs>;
+defm VPMOVUSQW  : avx512_trunc_qw<0x14, "vpmovusqw", X86vtruncus,
+                                  select_truncus, WriteShuffle256,
+                                  truncstore_us_vi16, masked_truncstore_us_vi16,
+                                  X86vtruncus, X86vmtruncus>;
+
+defm VPMOVQD    : avx512_trunc_qd<0x35, "vpmovqd", trunc, select_trunc,
+                                  WriteShuffle256, truncstorevi32,
+                                  masked_truncstorevi32, X86vtrunc, X86vmtrunc>;
+defm VPMOVSQD   : avx512_trunc_qd<0x25, "vpmovsqd",  X86vtruncs, select_truncs,
+                                  WriteShuffle256, truncstore_s_vi32,
+                                  masked_truncstore_s_vi32, X86vtruncs,
+                                  X86vmtruncs>;
+defm VPMOVUSQD  : avx512_trunc_qd<0x15, "vpmovusqd", X86vtruncus,
+                                  select_truncus, WriteShuffle256,
+                                  truncstore_us_vi32, masked_truncstore_us_vi32,
+                                  X86vtruncus, X86vmtruncus>;
+
+defm VPMOVDB    : avx512_trunc_db<0x31, "vpmovdb", trunc, select_trunc,
+                                  WriteShuffle256, truncstorevi8,
+                                  masked_truncstorevi8, X86vtrunc, X86vmtrunc>;
+defm VPMOVSDB   : avx512_trunc_db<0x21, "vpmovsdb", X86vtruncs, select_truncs,
+                                  WriteShuffle256, truncstore_s_vi8,
+                                  masked_truncstore_s_vi8, X86vtruncs,
+                                  X86vmtruncs>;
+defm VPMOVUSDB  : avx512_trunc_db<0x11, "vpmovusdb",  X86vtruncus,
+                                  select_truncus, WriteShuffle256,
+                                  truncstore_us_vi8, masked_truncstore_us_vi8,
+                                  X86vtruncus, X86vmtruncus>;
+
+defm VPMOVDW    : avx512_trunc_dw<0x33, "vpmovdw", trunc, select_trunc,
+                                  WriteShuffle256, truncstorevi16,
+                                  masked_truncstorevi16, X86vtrunc, X86vmtrunc>;
+defm VPMOVSDW   : avx512_trunc_dw<0x23, "vpmovsdw", X86vtruncs, select_truncs,
+                                  WriteShuffle256, truncstore_s_vi16,
+                                  masked_truncstore_s_vi16, X86vtruncs,
+                                  X86vmtruncs>;
+defm VPMOVUSDW  : avx512_trunc_dw<0x13, "vpmovusdw", X86vtruncus,
+                                  select_truncus, WriteShuffle256,
+                                  truncstore_us_vi16, masked_truncstore_us_vi16,
+                                  X86vtruncus, X86vmtruncus>;
+
+defm VPMOVWB    : avx512_trunc_wb<0x30, "vpmovwb", trunc, select_trunc,
+                                  WriteShuffle256, truncstorevi8,
+                                  masked_truncstorevi8, X86vtrunc,
+                                  X86vmtrunc>;
+defm VPMOVSWB   : avx512_trunc_wb<0x20, "vpmovswb", X86vtruncs, select_truncs,
+                                  WriteShuffle256, truncstore_s_vi8,
+                                  masked_truncstore_s_vi8, X86vtruncs,
+                                  X86vmtruncs>;
+defm VPMOVUSWB  : avx512_trunc_wb<0x10, "vpmovuswb", X86vtruncus,
+                                  select_truncus, WriteShuffle256,
+                                  truncstore_us_vi8, masked_truncstore_us_vi8,
+                                  X86vtruncus, X86vmtruncus>;
 
 let Predicates = [HasAVX512, NoVLX] in {
 def: Pat<(v8i16 (trunc (v8i32 VR256X:$src))),
@@ -9484,6 +9882,44 @@ def: Pat<(v16i8 (trunc (v16i16 VR256X:$src))),
                                             VR256X:$src, sub_ymm))), sub_xmm))>;
 }
 
+// Without BWI we can't use vXi16/vXi8 vselect so we have to use vmtrunc nodes.
+multiclass mtrunc_lowering<string InstrName, SDNode OpNode,
+                           X86VectorVTInfo DestInfo,
+                           X86VectorVTInfo SrcInfo> {
+  def : Pat<(DestInfo.VT (OpNode (SrcInfo.VT SrcInfo.RC:$src),
+                                 DestInfo.RC:$src0,
+                                 SrcInfo.KRCWM:$mask)),
+            (!cast<Instruction>(InstrName#"rrk") DestInfo.RC:$src0,
+                                                 SrcInfo.KRCWM:$mask,
+                                                 SrcInfo.RC:$src)>;
+
+  def : Pat<(DestInfo.VT (OpNode (SrcInfo.VT SrcInfo.RC:$src),
+                                 DestInfo.ImmAllZerosV,
+                                 SrcInfo.KRCWM:$mask)),
+            (!cast<Instruction>(InstrName#"rrkz") SrcInfo.KRCWM:$mask,
+                                                  SrcInfo.RC:$src)>;
+}
+
+let Predicates = [HasVLX] in {
+defm : mtrunc_lowering<"VPMOVDWZ256", X86vmtrunc, v8i16x_info, v8i32x_info>;
+defm : mtrunc_lowering<"VPMOVSDWZ256", X86vmtruncs, v8i16x_info, v8i32x_info>;
+defm : mtrunc_lowering<"VPMOVUSDWZ256", X86vmtruncus, v8i16x_info, v8i32x_info>;
+}
+
+let Predicates = [HasAVX512] in {
+defm : mtrunc_lowering<"VPMOVDWZ", X86vmtrunc, v16i16x_info, v16i32_info>;
+defm : mtrunc_lowering<"VPMOVSDWZ", X86vmtruncs, v16i16x_info, v16i32_info>;
+defm : mtrunc_lowering<"VPMOVUSDWZ", X86vmtruncus, v16i16x_info, v16i32_info>;
+
+defm : mtrunc_lowering<"VPMOVDBZ", X86vmtrunc, v16i8x_info, v16i32_info>;
+defm : mtrunc_lowering<"VPMOVSDBZ", X86vmtruncs, v16i8x_info, v16i32_info>;
+defm : mtrunc_lowering<"VPMOVUSDBZ", X86vmtruncus, v16i8x_info, v16i32_info>;
+
+defm : mtrunc_lowering<"VPMOVQWZ", X86vmtrunc, v8i16x_info, v8i64_info>;
+defm : mtrunc_lowering<"VPMOVSQWZ", X86vmtruncs, v8i16x_info, v8i64_info>;
+defm : mtrunc_lowering<"VPMOVUSQWZ", X86vmtruncus, v8i16x_info, v8i64_info>;
+}
+
 multiclass WriteShuffle256_common<bits<8> opc, string OpcodeStr, X86FoldableSchedWrite sched,
               X86VectorVTInfo DestInfo, X86VectorVTInfo SrcInfo,
               X86MemOperand x86memop, PatFrag LdFrag, SDNode OpNode>{
@@ -10128,7 +10564,7 @@ multiclass compress_by_vec_width_common<bits<8> opc, X86VectorVTInfo _,
                                  string OpcodeStr, X86FoldableSchedWrite sched> {
   defm rr : AVX512_maskable<opc, MRMDestReg, _, (outs _.RC:$dst),
               (ins _.RC:$src1), OpcodeStr, "$src1", "$src1",
-              (_.VT (X86compress _.RC:$src1))>, AVX5128IBase,
+              (null_frag)>, AVX5128IBase,
               Sched<[sched]>;
 
   let mayStore = 1, hasSideEffects = 0 in
@@ -10150,6 +10586,13 @@ multiclass compress_by_vec_width_lowering<X86VectorVTInfo _, string Name> {
   def : Pat<(X86mCompressingStore (_.VT _.RC:$src), addr:$dst, _.KRCWM:$mask),
             (!cast<Instruction>(Name#_.ZSuffix##mrk)
                             addr:$dst, _.KRCWM:$mask, _.RC:$src)>;
+
+  def : Pat<(X86compress (_.VT _.RC:$src), _.RC:$src0, _.KRCWM:$mask),
+            (!cast<Instruction>(Name#_.ZSuffix##rrk)
+                            _.RC:$src0, _.KRCWM:$mask, _.RC:$src)>;
+  def : Pat<(X86compress (_.VT _.RC:$src), _.ImmAllZerosV, _.KRCWM:$mask),
+            (!cast<Instruction>(Name#_.ZSuffix##rrkz)
+                            _.KRCWM:$mask, _.RC:$src)>;
 }
 
 multiclass compress_by_elt_width<bits<8> opc, string OpcodeStr,
@@ -10183,13 +10626,12 @@ multiclass expand_by_vec_width<bits<8> opc, X86VectorVTInfo _,
                                  string OpcodeStr, X86FoldableSchedWrite sched> {
   defm rr : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),
               (ins _.RC:$src1), OpcodeStr, "$src1", "$src1",
-              (_.VT (X86expand _.RC:$src1))>, AVX5128IBase,
+              (null_frag)>, AVX5128IBase,
               Sched<[sched]>;
 
   defm rm : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
               (ins _.MemOp:$src1), OpcodeStr, "$src1", "$src1",
-              (_.VT (X86expand (_.VT (bitconvert
-                                      (_.LdFrag addr:$src1)))))>,
+              (null_frag)>,
             AVX5128IBase, EVEX_CD8<_.EltSize, CD8VT1>,
             Sched<[sched.Folded, sched.ReadAfterFold]>;
 }
@@ -10208,6 +10650,13 @@ multiclass expand_by_vec_width_lowering<X86VectorVTInfo _, string Name> {
                                                (_.VT _.RC:$src0))),
             (!cast<Instruction>(Name#_.ZSuffix##rmk)
                             _.RC:$src0, _.KRCWM:$mask, addr:$src)>;
+
+  def : Pat<(X86expand (_.VT _.RC:$src), _.RC:$src0, _.KRCWM:$mask),
+            (!cast<Instruction>(Name#_.ZSuffix##rrk)
+                            _.RC:$src0, _.KRCWM:$mask, _.RC:$src)>;
+  def : Pat<(X86expand (_.VT _.RC:$src), _.ImmAllZerosV, _.KRCWM:$mask),
+            (!cast<Instruction>(Name#_.ZSuffix##rrkz)
+                            _.KRCWM:$mask, _.RC:$src)>;
 }
 
 multiclass expand_by_elt_width<bits<8> opc, string OpcodeStr,
@@ -11217,6 +11666,8 @@ def : Pat<(v2f64 (X86VBroadcast f64:$src)),
           (VMOVDDUPZ128rr (v2f64 (COPY_TO_REGCLASS FR64X:$src, VR128X)))>;
 def : Pat<(v2f64 (X86VBroadcast (loadv2f64 addr:$src))),
           (VMOVDDUPZ128rm addr:$src)>;
+def : Pat<(v2f64 (X86VBroadcast (v2f64 (X86vzload addr:$src)))),
+          (VMOVDDUPZ128rm addr:$src)>;
 
 def : Pat<(vselect (v2i1 VK2WM:$mask), (v2f64 (X86VBroadcast f64:$src)),
                    (v2f64 VR128X:$src0)),
diff --git a/lib/Target/X86/X86InstrArithmetic.td b/lib/Target/X86/X86InstrArithmetic.td
index 3288fe22f65ac003fa501d6d9712a7f9d3ae90aa..f0b42a29591985d78dc96a3657c867b7d0d63c9b 100644
--- a/lib/Target/X86/X86InstrArithmetic.td
+++ b/lib/Target/X86/X86InstrArithmetic.td
@@ -1,9 +1,8 @@
 //===-- X86InstrArithmetic.td - Integer Arithmetic Instrs --*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -422,22 +421,35 @@ def NOT64m : RI<0xF7, MRM2m, (outs), (ins i64mem:$dst), "not{q}\t$dst",
 } // SchedRW
 } // CodeSize
 
+def X86add_flag_nocf : PatFrag<(ops node:$lhs, node:$rhs),
+                               (X86add_flag node:$lhs, node:$rhs), [{
+  return hasNoCarryFlagUses(SDValue(N, 1));
+}]>;
+
+def X86sub_flag_nocf : PatFrag<(ops node:$lhs, node:$rhs),
+                               (X86sub_flag node:$lhs, node:$rhs), [{
+  // Only use DEC if the result is used.
+  return !SDValue(N, 0).use_empty() && hasNoCarryFlagUses(SDValue(N, 1));
+}]>;
+
 // TODO: inc/dec is slow for P4, but fast for Pentium-M.
 let Defs = [EFLAGS] in {
 let Constraints = "$src1 = $dst", SchedRW = [WriteALU] in {
 let CodeSize = 2 in
 def INC8r  : I<0xFE, MRM0r, (outs GR8 :$dst), (ins GR8 :$src1),
                "inc{b}\t$dst",
-               [(set GR8:$dst, EFLAGS, (X86inc_flag GR8:$src1))]>;
+               [(set GR8:$dst, EFLAGS, (X86add_flag_nocf GR8:$src1, 1))]>;
 let isConvertibleToThreeAddress = 1, CodeSize = 2 in { // Can xform into LEA.
 def INC16r : I<0xFF, MRM0r, (outs GR16:$dst), (ins GR16:$src1),
                "inc{w}\t$dst",
-               [(set GR16:$dst, EFLAGS, (X86inc_flag GR16:$src1))]>, OpSize16;
+               [(set GR16:$dst, EFLAGS, (X86add_flag_nocf GR16:$src1, 1))]>,
+               OpSize16;
 def INC32r : I<0xFF, MRM0r, (outs GR32:$dst), (ins GR32:$src1),
                "inc{l}\t$dst",
-               [(set GR32:$dst, EFLAGS, (X86inc_flag GR32:$src1))]>, OpSize32;
+               [(set GR32:$dst, EFLAGS, (X86add_flag_nocf GR32:$src1, 1))]>,
+               OpSize32;
 def INC64r : RI<0xFF, MRM0r, (outs GR64:$dst), (ins GR64:$src1), "inc{q}\t$dst",
-                [(set GR64:$dst, EFLAGS, (X86inc_flag GR64:$src1))]>;
+                [(set GR64:$dst, EFLAGS, (X86add_flag_nocf GR64:$src1, 1))]>;
 } // isConvertibleToThreeAddress = 1, CodeSize = 2
 
 // Short forms only valid in 32-bit mode. Selected during MCInst lowering.
@@ -474,16 +486,18 @@ let Constraints = "$src1 = $dst", SchedRW = [WriteALU] in {
 let CodeSize = 2 in
 def DEC8r  : I<0xFE, MRM1r, (outs GR8 :$dst), (ins GR8 :$src1),
                "dec{b}\t$dst",
-               [(set GR8:$dst, EFLAGS, (X86dec_flag GR8:$src1))]>;
+               [(set GR8:$dst, EFLAGS, (X86sub_flag_nocf GR8:$src1, 1))]>;
 let isConvertibleToThreeAddress = 1, CodeSize = 2 in { // Can xform into LEA.
 def DEC16r : I<0xFF, MRM1r, (outs GR16:$dst), (ins GR16:$src1),
                "dec{w}\t$dst",
-               [(set GR16:$dst, EFLAGS, (X86dec_flag GR16:$src1))]>, OpSize16;
+               [(set GR16:$dst, EFLAGS, (X86sub_flag_nocf GR16:$src1, 1))]>,
+               OpSize16;
 def DEC32r : I<0xFF, MRM1r, (outs GR32:$dst), (ins GR32:$src1),
                "dec{l}\t$dst",
-               [(set GR32:$dst, EFLAGS, (X86dec_flag GR32:$src1))]>, OpSize32;
+               [(set GR32:$dst, EFLAGS, (X86sub_flag_nocf GR32:$src1, 1))]>,
+               OpSize32;
 def DEC64r : RI<0xFF, MRM1r, (outs GR64:$dst), (ins GR64:$src1), "dec{q}\t$dst",
-                [(set GR64:$dst, EFLAGS, (X86dec_flag GR64:$src1))]>;
+                [(set GR64:$dst, EFLAGS, (X86sub_flag_nocf GR64:$src1, 1))]>;
 } // isConvertibleToThreeAddress = 1, CodeSize = 2
 
 // Short forms only valid in 32-bit mode. Selected during MCInst lowering.
@@ -1212,16 +1226,21 @@ def X86testpat : PatFrag<(ops node:$lhs, node:$rhs),
 let isCompare = 1 in {
   let Defs = [EFLAGS] in {
     let isCommutable = 1 in {
-      def TEST8rr  : BinOpRR_F<0x84, "test", Xi8 , X86testpat>;
-      def TEST16rr : BinOpRR_F<0x84, "test", Xi16, X86testpat>;
-      def TEST32rr : BinOpRR_F<0x84, "test", Xi32, X86testpat>;
-      def TEST64rr : BinOpRR_F<0x84, "test", Xi64, X86testpat>;
+      // Avoid selecting these and instead use a test+and. Post processing will
+      // combine them. This gives bunch of other patterns that start with
+      // and a chance to match.
+      def TEST8rr  : BinOpRR_F<0x84, "test", Xi8 , null_frag>;
+      def TEST16rr : BinOpRR_F<0x84, "test", Xi16, null_frag>;
+      def TEST32rr : BinOpRR_F<0x84, "test", Xi32, null_frag>;
+      def TEST64rr : BinOpRR_F<0x84, "test", Xi64, null_frag>;
     } // isCommutable
 
-    def TEST8mr    : BinOpMR_F<0x84, "test", Xi8 , X86testpat>;
-    def TEST16mr   : BinOpMR_F<0x84, "test", Xi16, X86testpat>;
-    def TEST32mr   : BinOpMR_F<0x84, "test", Xi32, X86testpat>;
-    def TEST64mr   : BinOpMR_F<0x84, "test", Xi64, X86testpat>;
+    let hasSideEffects = 0, mayLoad = 1 in {
+    def TEST8mr    : BinOpMR_F<0x84, "test", Xi8 , null_frag>;
+    def TEST16mr   : BinOpMR_F<0x84, "test", Xi16, null_frag>;
+    def TEST32mr   : BinOpMR_F<0x84, "test", Xi32, null_frag>;
+    def TEST64mr   : BinOpMR_F<0x84, "test", Xi64, null_frag>;
+    }
 
     def TEST8ri    : BinOpRI_F<0xF6, "test", Xi8 , X86testpat, MRM0r>;
     def TEST16ri   : BinOpRI_F<0xF6, "test", Xi16, X86testpat, MRM0r>;
diff --git a/lib/Target/X86/X86InstrBuilder.h b/lib/Target/X86/X86InstrBuilder.h
index dcce7b9951f266d3d7443bb92b353bb67f809b05..50aed98112c309ab3c29344144d2e4e74512e9d6 100644
--- a/lib/Target/X86/X86InstrBuilder.h
+++ b/lib/Target/X86/X86InstrBuilder.h
@@ -1,9 +1,8 @@
 //===-- X86InstrBuilder.h - Functions to aid building x86 insts -*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/X86/X86InstrCMovSetCC.td b/lib/Target/X86/X86InstrCMovSetCC.td
index f5494fc0b13fe89e887a2a7db29e1f505e07db63..2edb25abc113c763766069efadc6afe1397230b5 100644
--- a/lib/Target/X86/X86InstrCMovSetCC.td
+++ b/lib/Target/X86/X86InstrCMovSetCC.td
@@ -1,9 +1,8 @@
 //===-- X86InstrCMovSetCC.td - Conditional Move and SetCC --*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/X86/X86InstrCompiler.td b/lib/Target/X86/X86InstrCompiler.td
index 529d054d081a82c9bacd2cbd335126fdb0f56a12..55e87aabb78524eb4979640a9d706427c1a4a4b2 100644
--- a/lib/Target/X86/X86InstrCompiler.td
+++ b/lib/Target/X86/X86InstrCompiler.td
@@ -1,9 +1,8 @@
 //===- X86InstrCompiler.td - Compiler Pseudos and Patterns -*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -662,12 +661,11 @@ def : Pat<(v8f64 (X86cmov VR512:$t, VR512:$f, imm:$cond, EFLAGS)),
 
 // Memory barriers
 
-// TODO: Get this to fold the constant into the instruction.
 let isCodeGenOnly = 1, Defs = [EFLAGS] in
-def OR32mrLocked  : I<0x09, MRMDestMem, (outs), (ins i32mem:$dst, GR32:$zero),
-                      "or{l}\t{$zero, $dst|$dst, $zero}", []>,
-                      Requires<[Not64BitMode]>, OpSize32, LOCK,
-                      Sched<[WriteALURMW]>;
+def OR32mi8Locked  : Ii8<0x83, MRM1m, (outs), (ins i32mem:$dst, i32i8imm:$zero),
+                         "or{l}\t{$zero, $dst|$dst, $zero}", []>,
+                         Requires<[Not64BitMode]>, OpSize32, LOCK,
+                         Sched<[WriteALURMW]>;
 
 let hasSideEffects = 1 in
 def Int_MemBarrier : I<0, Pseudo, (outs), (ins),
@@ -777,53 +775,64 @@ defm LOCK_OR  : LOCK_ArithBinOp<0x08, 0x80, 0x83, MRM1m, X86lock_or , "or">;
 defm LOCK_AND : LOCK_ArithBinOp<0x20, 0x80, 0x83, MRM4m, X86lock_and, "and">;
 defm LOCK_XOR : LOCK_ArithBinOp<0x30, 0x80, 0x83, MRM6m, X86lock_xor, "xor">;
 
-multiclass LOCK_ArithUnOp<bits<8> Opc8, bits<8> Opc, Format Form,
-                          string frag, string mnemonic> {
-let Defs = [EFLAGS], mayLoad = 1, mayStore = 1, isCodeGenOnly = 1,
-    SchedRW = [WriteALURMW] in {
-def NAME#8m  : I<Opc8, Form, (outs), (ins i8mem :$dst),
-                 !strconcat(mnemonic, "{b}\t$dst"),
-                 [(set EFLAGS, (!cast<PatFrag>(frag # "_8") addr:$dst))]>,
-                 LOCK;
-def NAME#16m : I<Opc, Form, (outs), (ins i16mem:$dst),
-                 !strconcat(mnemonic, "{w}\t$dst"),
-                 [(set EFLAGS, (!cast<PatFrag>(frag # "_16") addr:$dst))]>,
-                 OpSize16, LOCK;
-def NAME#32m : I<Opc, Form, (outs), (ins i32mem:$dst),
-                 !strconcat(mnemonic, "{l}\t$dst"),
-                 [(set EFLAGS, (!cast<PatFrag>(frag # "_32") addr:$dst))]>,
-                 OpSize32, LOCK;
-def NAME#64m : RI<Opc, Form, (outs), (ins i64mem:$dst),
-                  !strconcat(mnemonic, "{q}\t$dst"),
-                  [(set EFLAGS, (!cast<PatFrag>(frag # "_64") addr:$dst))]>,
-                  LOCK;
-}
-}
+def X86lock_add_nocf : PatFrag<(ops node:$lhs, node:$rhs),
+                               (X86lock_add node:$lhs, node:$rhs), [{
+  return hasNoCarryFlagUses(SDValue(N, 0));
+}]>;
 
-multiclass unary_atomic_intrin<SDNode atomic_op> {
-  def _8 : PatFrag<(ops node:$ptr),
-                   (atomic_op  node:$ptr), [{
-    return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i8;
-  }]>;
-  def _16 : PatFrag<(ops node:$ptr),
-                    (atomic_op node:$ptr), [{
-    return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i16;
-  }]>;
-  def _32 : PatFrag<(ops node:$ptr),
-                    (atomic_op node:$ptr), [{
-    return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i32;
-  }]>;
-  def _64 : PatFrag<(ops node:$ptr),
-                    (atomic_op node:$ptr), [{
-    return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i64;
-  }]>;
-}
+def X86lock_sub_nocf : PatFrag<(ops node:$lhs, node:$rhs),
+                               (X86lock_sub node:$lhs, node:$rhs), [{
+  return hasNoCarryFlagUses(SDValue(N, 0));
+}]>;
 
-defm X86lock_inc : unary_atomic_intrin<X86lock_inc>;
-defm X86lock_dec : unary_atomic_intrin<X86lock_dec>;
+let Predicates = [UseIncDec] in {
+  let Defs = [EFLAGS], mayLoad = 1, mayStore = 1, isCodeGenOnly = 1,
+      SchedRW = [WriteALURMW]  in {
+    def LOCK_INC8m  : I<0xFE, MRM0m, (outs), (ins i8mem :$dst),
+                        "inc{b}\t$dst",
+                        [(set EFLAGS, (X86lock_add_nocf addr:$dst, (i8 1)))]>,
+                        LOCK;
+    def LOCK_INC16m : I<0xFF, MRM0m, (outs), (ins i16mem:$dst),
+                        "inc{w}\t$dst",
+                        [(set EFLAGS, (X86lock_add_nocf addr:$dst, (i16 1)))]>,
+                        OpSize16, LOCK;
+    def LOCK_INC32m : I<0xFF, MRM0m, (outs), (ins i32mem:$dst),
+                        "inc{l}\t$dst",
+                        [(set EFLAGS, (X86lock_add_nocf addr:$dst, (i32 1)))]>,
+                        OpSize32, LOCK;
+    def LOCK_INC64m : RI<0xFF, MRM0m, (outs), (ins i64mem:$dst),
+                         "inc{q}\t$dst",
+                         [(set EFLAGS, (X86lock_add_nocf addr:$dst, (i64 1)))]>,
+                         LOCK;
+
+    def LOCK_DEC8m  : I<0xFE, MRM1m, (outs), (ins i8mem :$dst),
+                        "dec{b}\t$dst",
+                        [(set EFLAGS, (X86lock_sub_nocf addr:$dst, (i8 1)))]>,
+                        LOCK;
+    def LOCK_DEC16m : I<0xFF, MRM1m, (outs), (ins i16mem:$dst),
+                        "dec{w}\t$dst",
+                        [(set EFLAGS, (X86lock_sub_nocf addr:$dst, (i16 1)))]>,
+                        OpSize16, LOCK;
+    def LOCK_DEC32m : I<0xFF, MRM1m, (outs), (ins i32mem:$dst),
+                        "dec{l}\t$dst",
+                        [(set EFLAGS, (X86lock_sub_nocf addr:$dst, (i32 1)))]>,
+                        OpSize32, LOCK;
+    def LOCK_DEC64m : RI<0xFF, MRM1m, (outs), (ins i64mem:$dst),
+                         "dec{q}\t$dst",
+                         [(set EFLAGS, (X86lock_sub_nocf addr:$dst, (i64 1)))]>,
+                         LOCK;
+  }
 
-defm LOCK_INC    : LOCK_ArithUnOp<0xFE, 0xFF, MRM0m, "X86lock_inc", "inc">;
-defm LOCK_DEC    : LOCK_ArithUnOp<0xFE, 0xFF, MRM1m, "X86lock_dec", "dec">;
+  // Additional patterns for -1 constant.
+  def : Pat<(X86lock_add addr:$dst, (i8  -1)), (LOCK_DEC8m  addr:$dst)>;
+  def : Pat<(X86lock_add addr:$dst, (i16 -1)), (LOCK_DEC16m addr:$dst)>;
+  def : Pat<(X86lock_add addr:$dst, (i32 -1)), (LOCK_DEC32m addr:$dst)>;
+  def : Pat<(X86lock_add addr:$dst, (i64 -1)), (LOCK_DEC64m addr:$dst)>;
+  def : Pat<(X86lock_sub addr:$dst, (i8  -1)), (LOCK_INC8m  addr:$dst)>;
+  def : Pat<(X86lock_sub addr:$dst, (i16 -1)), (LOCK_INC16m addr:$dst)>;
+  def : Pat<(X86lock_sub addr:$dst, (i32 -1)), (LOCK_INC32m addr:$dst)>;
+  def : Pat<(X86lock_sub addr:$dst, (i64 -1)), (LOCK_INC64m addr:$dst)>;
+}
 
 // Atomic compare and swap.
 multiclass LCMPXCHG_UnOp<bits<8> Opc, Format Form, string mnemonic,
@@ -1266,7 +1275,8 @@ defm : CMOVmr<X86_COND_NO, CMOVO16rm , CMOVO32rm , CMOVO64rm>;
 // i1 stored in one byte in zero-extended form.
 // Upper bits cleanup should be executed before Store.
 def : Pat<(zextloadi8i1  addr:$src), (MOV8rm addr:$src)>;
-def : Pat<(zextloadi16i1 addr:$src), (MOVZX16rm8 addr:$src)>;
+def : Pat<(zextloadi16i1 addr:$src),
+          (EXTRACT_SUBREG (MOVZX32rm8 addr:$src), sub_16bit)>;
 def : Pat<(zextloadi32i1 addr:$src), (MOVZX32rm8 addr:$src)>;
 def : Pat<(zextloadi64i1 addr:$src),
           (SUBREG_TO_REG (i64 0), (MOVZX32rm8 addr:$src), sub_32bit)>;
@@ -1277,9 +1287,11 @@ def : Pat<(zextloadi64i1 addr:$src),
 // defined, avoiding partial-register updates.
 
 def : Pat<(extloadi8i1 addr:$src),   (MOV8rm      addr:$src)>;
-def : Pat<(extloadi16i1 addr:$src),  (MOVZX16rm8  addr:$src)>;
+def : Pat<(extloadi16i1 addr:$src),
+          (EXTRACT_SUBREG (MOVZX32rm8 addr:$src), sub_16bit)>;
 def : Pat<(extloadi32i1 addr:$src),  (MOVZX32rm8  addr:$src)>;
-def : Pat<(extloadi16i8 addr:$src),  (MOVZX16rm8  addr:$src)>;
+def : Pat<(extloadi16i8 addr:$src),
+          (EXTRACT_SUBREG (MOVZX32rm8 addr:$src), sub_16bit)>;
 def : Pat<(extloadi32i8 addr:$src),  (MOVZX32rm8  addr:$src)>;
 def : Pat<(extloadi32i16 addr:$src), (MOVZX32rm16 addr:$src)>;
 
@@ -1338,6 +1350,8 @@ def def32 : PatLeaf<(i32 GR32:$src), [{
 // we can use a SUBREG_TO_REG.
 def : Pat<(i64 (zext def32:$src)),
           (SUBREG_TO_REG (i64 0), GR32:$src, sub_32bit)>;
+def : Pat<(i64 (and (anyext def32:$src), 0x00000000FFFFFFFF)),
+          (SUBREG_TO_REG (i64 0), GR32:$src, sub_32bit)>;
 
 //===----------------------------------------------------------------------===//
 // Pattern match OR as ADD
@@ -1354,10 +1368,8 @@ def or_is_add : PatFrag<(ops node:$lhs, node:$rhs), (or node:$lhs, node:$rhs),[{
   if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N->getOperand(1)))
     return CurDAG->MaskedValueIsZero(N->getOperand(0), CN->getAPIntValue());
 
-  KnownBits Known0;
-  CurDAG->computeKnownBits(N->getOperand(0), Known0, 0);
-  KnownBits Known1;
-  CurDAG->computeKnownBits(N->getOperand(1), Known1, 0);
+  KnownBits Known0 = CurDAG->computeKnownBits(N->getOperand(0), 0);
+  KnownBits Known1 = CurDAG->computeKnownBits(N->getOperand(1), 0);
   return (~Known0.Zero & ~Known1.Zero) == 0;
 }]>;
 
@@ -1424,8 +1436,7 @@ def ADD64ri32_DB : I<0, Pseudo,
 
 def sub_is_xor : PatFrag<(ops node:$lhs, node:$rhs), (sub node:$lhs, node:$rhs),[{
   if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N->getOperand(0))) {
-    KnownBits Known;
-    CurDAG->computeKnownBits(N->getOperand(1), Known);
+    KnownBits Known = CurDAG->computeKnownBits(N->getOperand(1));
 
     // If all possible ones in the RHS are set in the LHS then there can't be
     // a borrow and we can use xor.
@@ -2022,6 +2033,15 @@ let Predicates = [UseIncDec] in {
   def : Pat<(add GR16:$src, -1), (DEC16r GR16:$src)>;
   def : Pat<(add GR32:$src, -1), (DEC32r GR32:$src)>;
   def : Pat<(add GR64:$src, -1), (DEC64r GR64:$src)>;
+
+  def : Pat<(X86add_flag_nocf GR8:$src, -1),  (DEC8r GR8:$src)>;
+  def : Pat<(X86add_flag_nocf GR16:$src, -1), (DEC16r GR16:$src)>;
+  def : Pat<(X86add_flag_nocf GR32:$src, -1), (DEC32r GR32:$src)>;
+  def : Pat<(X86add_flag_nocf GR64:$src, -1), (DEC64r GR64:$src)>;
+  def : Pat<(X86sub_flag_nocf GR8:$src, -1),  (INC8r GR8:$src)>;
+  def : Pat<(X86sub_flag_nocf GR16:$src, -1), (INC16r GR16:$src)>;
+  def : Pat<(X86sub_flag_nocf GR32:$src, -1), (INC32r GR32:$src)>;
+  def : Pat<(X86sub_flag_nocf GR64:$src, -1), (INC64r GR64:$src)>;
 }
 
 // or reg/reg.
diff --git a/lib/Target/X86/X86InstrControl.td b/lib/Target/X86/X86InstrControl.td
index a7c7aaab228579fa552ae12f0e69b2736a7efb0d..09911fcde473a84111cb465cbbac665d30dbba2d 100644
--- a/lib/Target/X86/X86InstrControl.td
+++ b/lib/Target/X86/X86InstrControl.td
@@ -1,9 +1,8 @@
 //===-- X86InstrControl.td - Control Flow Instructions -----*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/X86/X86InstrExtension.td b/lib/Target/X86/X86InstrExtension.td
index c24d6d5b8df1d10db102865c1d6d7d1d06b00021..0b06b3e58fd0fc6069fe66bc6099e3651fa925d0 100644
--- a/lib/Target/X86/X86InstrExtension.td
+++ b/lib/Target/X86/X86InstrExtension.td
@@ -1,9 +1,8 @@
 //===-- X86InstrExtension.td - Sign and Zero Extensions ----*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/X86/X86InstrFMA.td b/lib/Target/X86/X86InstrFMA.td
index 1a8e529431afc4ebafa82106e4304ceb9aaf52a5..313a95995fd3856fc93a0e90143838edefc3c97b 100644
--- a/lib/Target/X86/X86InstrFMA.td
+++ b/lib/Target/X86/X86InstrFMA.td
@@ -1,9 +1,8 @@
 //===-- X86InstrFMA.td - FMA Instruction Set ---------------*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/X86/X86InstrFMA3Info.cpp b/lib/Target/X86/X86InstrFMA3Info.cpp
index def732a2dd00e7a4053f644b518131a315b84622..d1cbd828303bbe4ec1133cdc36f285c5238d8441 100644
--- a/lib/Target/X86/X86InstrFMA3Info.cpp
+++ b/lib/Target/X86/X86InstrFMA3Info.cpp
@@ -1,9 +1,8 @@
 //===-- X86InstrFMA3Info.cpp - X86 FMA3 Instruction Information -----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/X86/X86InstrFMA3Info.h b/lib/Target/X86/X86InstrFMA3Info.h
index 6eec1db98bf89c9ce574dd98a6cc0c7b06b8a818..7fa6f5917862d5d18bd3f0a11c6793cc7b47a8fa 100644
--- a/lib/Target/X86/X86InstrFMA3Info.h
+++ b/lib/Target/X86/X86InstrFMA3Info.h
@@ -1,9 +1,8 @@
 //===- X86InstrFMA3Info.h - X86 FMA3 Instruction Information ----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/X86/X86InstrFPStack.td b/lib/Target/X86/X86InstrFPStack.td
index 5912a31996131be5c4c581fcd8ae8835f8b38043..753e33b5f99d5dea323b7eb36d08a3fd5bd3df22 100644
--- a/lib/Target/X86/X86InstrFPStack.td
+++ b/lib/Target/X86/X86InstrFPStack.td
@@ -1,9 +1,8 @@
 //===- X86InstrFPStack.td - FPU Instruction Set ------------*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -17,18 +16,12 @@
 // FPStack specific DAG Nodes.
 //===----------------------------------------------------------------------===//
 
-def SDTX86FpGet2    : SDTypeProfile<2, 0, [SDTCisVT<0, f80>,
-                                           SDTCisVT<1, f80>]>;
-def SDTX86Fld       : SDTypeProfile<1, 2, [SDTCisFP<0>,
-                                           SDTCisPtrTy<1>,
-                                           SDTCisVT<2, OtherVT>]>;
-def SDTX86Fst       : SDTypeProfile<0, 3, [SDTCisFP<0>,
-                                           SDTCisPtrTy<1>,
-                                           SDTCisVT<2, OtherVT>]>;
-def SDTX86Fild      : SDTypeProfile<1, 2, [SDTCisFP<0>, SDTCisPtrTy<1>,
-                                           SDTCisVT<2, OtherVT>]>;
+def SDTX86Fld       : SDTypeProfile<1, 1, [SDTCisFP<0>,
+                                           SDTCisPtrTy<1>]>;
+def SDTX86Fst       : SDTypeProfile<0, 2, [SDTCisFP<0>,
+                                           SDTCisPtrTy<1>]>;
+def SDTX86Fild      : SDTypeProfile<1, 1, [SDTCisFP<0>, SDTCisPtrTy<1>]>;
 def SDTX86Fnstsw    : SDTypeProfile<1, 1, [SDTCisVT<0, i16>, SDTCisVT<1, i16>]>;
-def SDTX86FpToIMem  : SDTypeProfile<0, 2, [SDTCisFP<0>, SDTCisPtrTy<1>]>;
 
 def SDTX86CwdStore  : SDTypeProfile<0, 1, [SDTCisPtrTy<0>]>;
 
@@ -43,16 +36,62 @@ def X86fildflag     : SDNode<"X86ISD::FILD_FLAG", SDTX86Fild,
                              [SDNPHasChain, SDNPOutGlue, SDNPMayLoad,
                               SDNPMemOperand]>;
 def X86fp_stsw      : SDNode<"X86ISD::FNSTSW16r", SDTX86Fnstsw>;
-def X86fp_to_i16mem : SDNode<"X86ISD::FP_TO_INT16_IN_MEM", SDTX86FpToIMem,
-                             [SDNPHasChain, SDNPMayStore, SDNPMemOperand]>;
-def X86fp_to_i32mem : SDNode<"X86ISD::FP_TO_INT32_IN_MEM", SDTX86FpToIMem,
-                             [SDNPHasChain, SDNPMayStore, SDNPMemOperand]>;
-def X86fp_to_i64mem : SDNode<"X86ISD::FP_TO_INT64_IN_MEM", SDTX86FpToIMem,
-                             [SDNPHasChain, SDNPMayStore, SDNPMemOperand]>;
+def X86fp_to_mem : SDNode<"X86ISD::FP_TO_INT_IN_MEM", SDTX86Fst,
+                          [SDNPHasChain, SDNPMayStore, SDNPMemOperand]>;
 def X86fp_cwd_get16 : SDNode<"X86ISD::FNSTCW16m",          SDTX86CwdStore,
                              [SDNPHasChain, SDNPMayStore, SDNPSideEffect,
                               SDNPMemOperand]>;
 
+def X86fstf32 : PatFrag<(ops node:$val, node:$ptr),
+                        (X86fst node:$val, node:$ptr), [{
+  return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::f32;
+}]>;
+def X86fstf64 : PatFrag<(ops node:$val, node:$ptr),
+                        (X86fst node:$val, node:$ptr), [{
+  return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::f64;
+}]>;
+def X86fstf80 : PatFrag<(ops node:$val, node:$ptr),
+                        (X86fst node:$val, node:$ptr), [{
+  return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::f80;
+}]>;
+
+def X86fldf32 : PatFrag<(ops node:$ptr), (X86fld node:$ptr), [{
+  return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::f32;
+}]>;
+def X86fldf64 : PatFrag<(ops node:$ptr), (X86fld node:$ptr), [{
+  return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::f64;
+}]>;
+def X86fldf80 : PatFrag<(ops node:$ptr), (X86fld node:$ptr), [{
+  return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::f80;
+}]>;
+
+def X86fild16 : PatFrag<(ops node:$ptr), (X86fild node:$ptr), [{
+  return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i16;
+}]>;
+def X86fild32 : PatFrag<(ops node:$ptr), (X86fild node:$ptr), [{
+  return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i32;
+}]>;
+def X86fild64 : PatFrag<(ops node:$ptr), (X86fild node:$ptr), [{
+  return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i64;
+}]>;
+
+def X86fildflag64 : PatFrag<(ops node:$ptr), (X86fildflag node:$ptr), [{
+  return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i64;
+}]>;
+
+def X86fp_to_i16mem : PatFrag<(ops node:$val, node:$ptr),
+                              (X86fp_to_mem node:$val, node:$ptr), [{
+  return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i16;
+}]>;
+def X86fp_to_i32mem : PatFrag<(ops node:$val, node:$ptr),
+                              (X86fp_to_mem node:$val, node:$ptr), [{
+  return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i32;
+}]>;
+def X86fp_to_i64mem : PatFrag<(ops node:$val, node:$ptr),
+                              (X86fp_to_mem node:$val, node:$ptr), [{
+  return cast<MemIntrinsicSDNode>(N)->getMemoryVT() == MVT::i64;
+}]>;
+
 //===----------------------------------------------------------------------===//
 // FPStack pattern fragments
 //===----------------------------------------------------------------------===//
@@ -185,44 +224,44 @@ def _FpI16m32 : FpIf32<(outs RFP32:$dst), (ins RFP32:$src1, i16mem:$src2),
                        OneArgFPRW,
                        [!if(Forward,
                             (set RFP32:$dst,
-                             (OpNode RFP32:$src1, (X86fild addr:$src2, i16))),
+                             (OpNode RFP32:$src1, (X86fild16 addr:$src2))),
                             (set RFP32:$dst,
-                             (OpNode (X86fild addr:$src2, i16), RFP32:$src1)))]>;
+                             (OpNode (X86fild16 addr:$src2), RFP32:$src1)))]>;
 def _FpI32m32 : FpIf32<(outs RFP32:$dst), (ins RFP32:$src1, i32mem:$src2),
                        OneArgFPRW,
                        [!if(Forward,
                             (set RFP32:$dst,
-                             (OpNode RFP32:$src1, (X86fild addr:$src2, i32))),
+                             (OpNode RFP32:$src1, (X86fild32 addr:$src2))),
                             (set RFP32:$dst,
-                             (OpNode (X86fild addr:$src2, i32), RFP32:$src1)))]>;
+                             (OpNode (X86fild32 addr:$src2), RFP32:$src1)))]>;
 def _FpI16m64 : FpIf64<(outs RFP64:$dst), (ins RFP64:$src1, i16mem:$src2),
                        OneArgFPRW,
                        [!if(Forward,
                             (set RFP64:$dst,
-                             (OpNode RFP64:$src1, (X86fild addr:$src2, i16))),
+                             (OpNode RFP64:$src1, (X86fild16 addr:$src2))),
                             (set RFP64:$dst,
-                             (OpNode (X86fild addr:$src2, i16), RFP64:$src1)))]>;
+                             (OpNode (X86fild16 addr:$src2), RFP64:$src1)))]>;
 def _FpI32m64 : FpIf64<(outs RFP64:$dst), (ins RFP64:$src1, i32mem:$src2),
                        OneArgFPRW,
                        [!if(Forward,
                             (set RFP64:$dst,
-                             (OpNode RFP64:$src1, (X86fild addr:$src2, i32))),
+                             (OpNode RFP64:$src1, (X86fild32 addr:$src2))),
                             (set RFP64:$dst,
-                             (OpNode (X86fild addr:$src2, i32), RFP64:$src1)))]>;
+                             (OpNode (X86fild32 addr:$src2), RFP64:$src1)))]>;
 def _FpI16m80 : FpI_<(outs RFP80:$dst), (ins RFP80:$src1, i16mem:$src2),
                      OneArgFPRW,
                      [!if(Forward,
                           (set RFP80:$dst,
-                           (OpNode RFP80:$src1, (X86fild addr:$src2, i16))),
+                           (OpNode RFP80:$src1, (X86fild16 addr:$src2))),
                           (set RFP80:$dst,
-                           (OpNode (X86fild addr:$src2, i16), RFP80:$src1)))]>;
+                           (OpNode (X86fild16 addr:$src2), RFP80:$src1)))]>;
 def _FpI32m80 : FpI_<(outs RFP80:$dst), (ins RFP80:$src1, i32mem:$src2),
                      OneArgFPRW,
                      [!if(Forward,
                           (set RFP80:$dst,
-                           (OpNode RFP80:$src1, (X86fild addr:$src2, i32))),
+                           (OpNode RFP80:$src1, (X86fild32 addr:$src2))),
                           (set RFP80:$dst,
-                           (OpNode (X86fild addr:$src2, i32), RFP80:$src1)))]>;
+                           (OpNode (X86fild32 addr:$src2), RFP80:$src1)))]>;
 def _FI16m  : FPI<0xDE, fp, (outs), (ins i16mem:$src),
                   !strconcat("fi", asmstring, "{s}\t$src")>;
 def _FI32m  : FPI<0xDA, fp, (outs), (ins i32mem:$src),
@@ -230,7 +269,7 @@ def _FI32m  : FPI<0xDA, fp, (outs), (ins i32mem:$src),
 } // mayLoad = 1, hasSideEffects = 1
 }
 
-let Defs = [FPSW] in {
+let Defs = [FPSW], Uses = [FPCW] in {
 // FPBinary_rr just defines pseudo-instructions, no need to set a scheduling
 // resources.
 let hasNoSchedulingInfo = 1 in {
@@ -258,42 +297,42 @@ defm DIVR: FPBinary<fdiv, MRM7m, "divr", 0>;
 } // Defs = [FPSW]
 
 class FPST0rInst<Format fp, string asm>
-  : FPI<0xD8, fp, (outs), (ins RST:$op), asm>;
+  : FPI<0xD8, fp, (outs), (ins RSTi:$op), asm>;
 class FPrST0Inst<Format fp, string asm>
-  : FPI<0xDC, fp, (outs), (ins RST:$op), asm>;
+  : FPI<0xDC, fp, (outs), (ins RSTi:$op), asm>;
 class FPrST0PInst<Format fp, string asm>
-  : FPI<0xDE, fp, (outs), (ins RST:$op), asm>;
+  : FPI<0xDE, fp, (outs), (ins RSTi:$op), asm>;
 
 // NOTE: GAS and apparently all other AT&T style assemblers have a broken notion
 // of some of the 'reverse' forms of the fsub and fdiv instructions.  As such,
 // we have to put some 'r's in and take them out of weird places.
-let SchedRW = [WriteFAdd] in {
-def ADD_FST0r   : FPST0rInst <MRM0r, "fadd\t$op">;
-def ADD_FrST0   : FPrST0Inst <MRM0r, "fadd\t{%st(0), $op|$op, st(0)}">;
-def ADD_FPrST0  : FPrST0PInst<MRM0r, "faddp\t$op">;
-def SUBR_FST0r  : FPST0rInst <MRM5r, "fsubr\t$op">;
-def SUB_FrST0   : FPrST0Inst <MRM5r, "fsub{r}\t{%st(0), $op|$op, st(0)}">;
-def SUB_FPrST0  : FPrST0PInst<MRM5r, "fsub{r}p\t$op">;
-def SUB_FST0r   : FPST0rInst <MRM4r, "fsub\t$op">;
-def SUBR_FrST0  : FPrST0Inst <MRM4r, "fsub{|r}\t{%st(0), $op|$op, st(0)}">;
-def SUBR_FPrST0 : FPrST0PInst<MRM4r, "fsub{|r}p\t$op">;
+let SchedRW = [WriteFAdd], Defs = [FPSW], Uses = [FPCW] in {
+def ADD_FST0r   : FPST0rInst <MRM0r, "fadd\t{$op, %st|st, $op}">;
+def ADD_FrST0   : FPrST0Inst <MRM0r, "fadd\t{%st, $op|$op, st}">;
+def ADD_FPrST0  : FPrST0PInst<MRM0r, "faddp\t{%st, $op|$op, st}">;
+def SUBR_FST0r  : FPST0rInst <MRM5r, "fsubr\t{$op, %st|st, $op}">;
+def SUB_FrST0   : FPrST0Inst <MRM5r, "fsub{r}\t{%st, $op|$op, st}">;
+def SUB_FPrST0  : FPrST0PInst<MRM5r, "fsub{r}p\t{%st, $op|$op, st}">;
+def SUB_FST0r   : FPST0rInst <MRM4r, "fsub\t{$op, %st|st, $op}">;
+def SUBR_FrST0  : FPrST0Inst <MRM4r, "fsub{|r}\t{%st, $op|$op, st}">;
+def SUBR_FPrST0 : FPrST0PInst<MRM4r, "fsub{|r}p\t{%st, $op|$op, st}">;
 } // SchedRW
-let SchedRW = [WriteFCom] in {
+let SchedRW = [WriteFCom], Defs = [FPSW], Uses = [FPCW] in {
 def COM_FST0r   : FPST0rInst <MRM2r, "fcom\t$op">;
 def COMP_FST0r  : FPST0rInst <MRM3r, "fcomp\t$op">;
 } // SchedRW
-let SchedRW = [WriteFMul] in {
-def MUL_FST0r   : FPST0rInst <MRM1r, "fmul\t$op">;
-def MUL_FrST0   : FPrST0Inst <MRM1r, "fmul\t{%st(0), $op|$op, st(0)}">;
-def MUL_FPrST0  : FPrST0PInst<MRM1r, "fmulp\t$op">;
+let SchedRW = [WriteFMul], Defs = [FPSW], Uses = [FPCW] in {
+def MUL_FST0r   : FPST0rInst <MRM1r, "fmul\t{$op, %st|st, $op}">;
+def MUL_FrST0   : FPrST0Inst <MRM1r, "fmul\t{%st, $op|$op, st}">;
+def MUL_FPrST0  : FPrST0PInst<MRM1r, "fmulp\t{%st, $op|$op, st}">;
 } // SchedRW
-let SchedRW = [WriteFDiv] in {
-def DIVR_FST0r  : FPST0rInst <MRM7r, "fdivr\t$op">;
-def DIV_FrST0   : FPrST0Inst <MRM7r, "fdiv{r}\t{%st(0), $op|$op, st(0)}">;
-def DIV_FPrST0  : FPrST0PInst<MRM7r, "fdiv{r}p\t$op">;
-def DIV_FST0r   : FPST0rInst <MRM6r, "fdiv\t$op">;
-def DIVR_FrST0  : FPrST0Inst <MRM6r, "fdiv{|r}\t{%st(0), $op|$op, st(0)}">;
-def DIVR_FPrST0 : FPrST0PInst<MRM6r, "fdiv{|r}p\t$op">;
+let SchedRW = [WriteFDiv], Defs = [FPSW], Uses = [FPCW] in {
+def DIVR_FST0r  : FPST0rInst <MRM7r, "fdivr\t{$op, %st|st, $op}">;
+def DIV_FrST0   : FPrST0Inst <MRM7r, "fdiv{r}\t{%st, $op|$op, st}">;
+def DIV_FPrST0  : FPrST0PInst<MRM7r, "fdiv{r}p\t{%st, $op|$op, st}">;
+def DIV_FST0r   : FPST0rInst <MRM6r, "fdiv\t{$op, %st|st, $op}">;
+def DIVR_FrST0  : FPrST0Inst <MRM6r, "fdiv{|r}\t{%st, $op|$op, st}">;
+def DIVR_FPrST0 : FPrST0PInst<MRM6r, "fdiv{|r}p\t{%st, $op|$op, st}">;
 } // SchedRW
 
 // Unary operations.
@@ -307,7 +346,7 @@ def _Fp80  : FpI_<(outs RFP80:$dst), (ins RFP80:$src), OneArgFPRW,
 def _F     : FPI<0xD9, fp, (outs), (ins), asmstring>;
 }
 
-let Defs = [FPSW] in {
+let Defs = [FPSW], Uses = [FPCW] in {
 
 let SchedRW = [WriteFSign] in {
 defm CHS : FPUnary<fneg, MRM_E0, "fchs">;
@@ -335,7 +374,7 @@ def TST_F  : FPI<0xD9, MRM_E4, (outs), (ins), "ftst">;
 
 // Versions of FP instructions that take a single memory operand.  Added for the
 //   disassembler; remove as they are included with patterns elsewhere.
-let SchedRW = [WriteFComLd] in {
+let SchedRW = [WriteFComLd], Defs = [FPSW], Uses = [FPCW] in {
 def FCOM32m  : FPI<0xD8, MRM2m, (outs), (ins f32mem:$src), "fcom{s}\t$src">;
 def FCOMP32m : FPI<0xD8, MRM3m, (outs), (ins f32mem:$src), "fcomp{s}\t$src">;
 
@@ -398,32 +437,31 @@ defm CMOVNP : FPCMov<X86_COND_NP>;
 
 let Predicates = [HasCMov] in {
 // These are not factored because there's no clean way to pass DA/DB.
-def CMOVB_F  : FPI<0xDA, MRM0r, (outs), (ins RST:$op),
-                  "fcmovb\t{$op, %st(0)|st(0), $op}">;
-def CMOVBE_F : FPI<0xDA, MRM2r, (outs), (ins RST:$op),
-                  "fcmovbe\t{$op, %st(0)|st(0), $op}">;
-def CMOVE_F  : FPI<0xDA, MRM1r, (outs), (ins RST:$op),
-                  "fcmove\t{$op, %st(0)|st(0), $op}">;
-def CMOVP_F  : FPI<0xDA, MRM3r, (outs), (ins RST:$op),
-                  "fcmovu\t{$op, %st(0)|st(0), $op}">;
-def CMOVNB_F : FPI<0xDB, MRM0r, (outs), (ins RST:$op),
-                  "fcmovnb\t{$op, %st(0)|st(0), $op}">;
-def CMOVNBE_F: FPI<0xDB, MRM2r, (outs), (ins RST:$op),
-                  "fcmovnbe\t{$op, %st(0)|st(0), $op}">;
-def CMOVNE_F : FPI<0xDB, MRM1r, (outs), (ins RST:$op),
-                  "fcmovne\t{$op, %st(0)|st(0), $op}">;
-def CMOVNP_F : FPI<0xDB, MRM3r, (outs), (ins RST:$op),
-                  "fcmovnu\t{$op, %st(0)|st(0), $op}">;
+def CMOVB_F  : FPI<0xDA, MRM0r, (outs), (ins RSTi:$op),
+                  "fcmovb\t{$op, %st|st, $op}">;
+def CMOVBE_F : FPI<0xDA, MRM2r, (outs), (ins RSTi:$op),
+                  "fcmovbe\t{$op, %st|st, $op}">;
+def CMOVE_F  : FPI<0xDA, MRM1r, (outs), (ins RSTi:$op),
+                  "fcmove\t{$op, %st|st, $op}">;
+def CMOVP_F  : FPI<0xDA, MRM3r, (outs), (ins RSTi:$op),
+                  "fcmovu\t{$op, %st|st, $op}">;
+def CMOVNB_F : FPI<0xDB, MRM0r, (outs), (ins RSTi:$op),
+                  "fcmovnb\t{$op, %st|st, $op}">;
+def CMOVNBE_F: FPI<0xDB, MRM2r, (outs), (ins RSTi:$op),
+                  "fcmovnbe\t{$op, %st|st, $op}">;
+def CMOVNE_F : FPI<0xDB, MRM1r, (outs), (ins RSTi:$op),
+                  "fcmovne\t{$op, %st|st, $op}">;
+def CMOVNP_F : FPI<0xDB, MRM3r, (outs), (ins RSTi:$op),
+                  "fcmovnu\t{$op, %st|st, $op}">;
 } // Predicates = [HasCMov]
 } // SchedRW
 
 // Floating point loads & stores.
-let SchedRW = [WriteLoad] in {
+let SchedRW = [WriteLoad], Uses = [FPCW] in {
 let canFoldAsLoad = 1 in {
 def LD_Fp32m   : FpIf32<(outs RFP32:$dst), (ins f32mem:$src), ZeroArgFP,
                   [(set RFP32:$dst, (loadf32 addr:$src))]>;
-let isReMaterializable = 1 in
-  def LD_Fp64m : FpIf64<(outs RFP64:$dst), (ins f64mem:$src), ZeroArgFP,
+def LD_Fp64m : FpIf64<(outs RFP64:$dst), (ins f64mem:$src), ZeroArgFP,
                   [(set RFP64:$dst, (loadf64 addr:$src))]>;
 def LD_Fp80m   : FpI_<(outs RFP80:$dst), (ins f80mem:$src), ZeroArgFP,
                   [(set RFP80:$dst, (loadf80 addr:$src))]>;
@@ -435,26 +473,26 @@ def LD_Fp64m80 : FpI_<(outs RFP80:$dst), (ins f64mem:$src), ZeroArgFP,
 def LD_Fp32m80 : FpI_<(outs RFP80:$dst), (ins f32mem:$src), ZeroArgFP,
                   [(set RFP80:$dst, (f80 (extloadf32 addr:$src)))]>;
 def ILD_Fp16m32: FpIf32<(outs RFP32:$dst), (ins i16mem:$src), ZeroArgFP,
-                  [(set RFP32:$dst, (X86fild addr:$src, i16))]>;
+                  [(set RFP32:$dst, (X86fild16 addr:$src))]>;
 def ILD_Fp32m32: FpIf32<(outs RFP32:$dst), (ins i32mem:$src), ZeroArgFP,
-                  [(set RFP32:$dst, (X86fild addr:$src, i32))]>;
+                  [(set RFP32:$dst, (X86fild32 addr:$src))]>;
 def ILD_Fp64m32: FpIf32<(outs RFP32:$dst), (ins i64mem:$src), ZeroArgFP,
-                  [(set RFP32:$dst, (X86fild addr:$src, i64))]>;
+                  [(set RFP32:$dst, (X86fild64 addr:$src))]>;
 def ILD_Fp16m64: FpIf64<(outs RFP64:$dst), (ins i16mem:$src), ZeroArgFP,
-                  [(set RFP64:$dst, (X86fild addr:$src, i16))]>;
+                  [(set RFP64:$dst, (X86fild16 addr:$src))]>;
 def ILD_Fp32m64: FpIf64<(outs RFP64:$dst), (ins i32mem:$src), ZeroArgFP,
-                  [(set RFP64:$dst, (X86fild addr:$src, i32))]>;
+                  [(set RFP64:$dst, (X86fild32 addr:$src))]>;
 def ILD_Fp64m64: FpIf64<(outs RFP64:$dst), (ins i64mem:$src), ZeroArgFP,
-                  [(set RFP64:$dst, (X86fild addr:$src, i64))]>;
+                  [(set RFP64:$dst, (X86fild64 addr:$src))]>;
 def ILD_Fp16m80: FpI_<(outs RFP80:$dst), (ins i16mem:$src), ZeroArgFP,
-                  [(set RFP80:$dst, (X86fild addr:$src, i16))]>;
+                  [(set RFP80:$dst, (X86fild16 addr:$src))]>;
 def ILD_Fp32m80: FpI_<(outs RFP80:$dst), (ins i32mem:$src), ZeroArgFP,
-                  [(set RFP80:$dst, (X86fild addr:$src, i32))]>;
+                  [(set RFP80:$dst, (X86fild32 addr:$src))]>;
 def ILD_Fp64m80: FpI_<(outs RFP80:$dst), (ins i64mem:$src), ZeroArgFP,
-                  [(set RFP80:$dst, (X86fild addr:$src, i64))]>;
+                  [(set RFP80:$dst, (X86fild64 addr:$src))]>;
 } // SchedRW
 
-let SchedRW = [WriteStore] in {
+let SchedRW = [WriteStore], Uses = [FPCW] in {
 def ST_Fp32m   : FpIf32<(outs), (ins f32mem:$op, RFP32:$src), OneArgFP,
                   [(store RFP32:$src, addr:$op)]>;
 def ST_Fp64m32 : FpIf64<(outs), (ins f32mem:$op, RFP64:$src), OneArgFP,
@@ -489,9 +527,9 @@ def IST_Fp16m80  : FpI_<(outs), (ins i16mem:$op, RFP80:$src), OneArgFP, []>;
 def IST_Fp32m80  : FpI_<(outs), (ins i32mem:$op, RFP80:$src), OneArgFP, []>;
 def IST_Fp64m80  : FpI_<(outs), (ins i64mem:$op, RFP80:$src), OneArgFP, []>;
 } // mayStore
-} // SchedRW
+} // SchedRW, Uses = [FPCW]
 
-let mayLoad = 1, SchedRW = [WriteLoad] in {
+let mayLoad = 1, SchedRW = [WriteLoad], Uses = [FPCW] in {
 def LD_F32m   : FPI<0xD9, MRM0m, (outs), (ins f32mem:$src), "fld{s}\t$src">;
 def LD_F64m   : FPI<0xDD, MRM0m, (outs), (ins f64mem:$src), "fld{l}\t$src">;
 def LD_F80m   : FPI<0xDB, MRM5m, (outs), (ins f80mem:$src), "fld{t}\t$src">;
@@ -499,7 +537,7 @@ def ILD_F16m  : FPI<0xDF, MRM0m, (outs), (ins i16mem:$src), "fild{s}\t$src">;
 def ILD_F32m  : FPI<0xDB, MRM0m, (outs), (ins i32mem:$src), "fild{l}\t$src">;
 def ILD_F64m  : FPI<0xDF, MRM5m, (outs), (ins i64mem:$src), "fild{ll}\t$src">;
 }
-let mayStore = 1, SchedRW = [WriteStore] in {
+let mayStore = 1, SchedRW = [WriteStore], Uses = [FPCW] in {
 def ST_F32m   : FPI<0xD9, MRM2m, (outs), (ins f32mem:$dst), "fst{s}\t$dst">;
 def ST_F64m   : FPI<0xDD, MRM2m, (outs), (ins f64mem:$dst), "fst{l}\t$dst">;
 def ST_FP32m  : FPI<0xD9, MRM3m, (outs), (ins f32mem:$dst), "fstp{s}\t$dst">;
@@ -513,7 +551,7 @@ def IST_FP64m : FPI<0xDF, MRM7m, (outs), (ins i64mem:$dst), "fistp{ll}\t$dst">;
 }
 
 // FISTTP requires SSE3 even though it's a FPStack op.
-let Predicates = [HasSSE3], SchedRW = [WriteStore] in {
+let Predicates = [HasSSE3], SchedRW = [WriteStore], Uses = [FPCW] in {
 def ISTT_Fp16m32 : FpI_<(outs), (ins i16mem:$op, RFP32:$src), OneArgFP,
                     [(X86fp_to_i16mem RFP32:$src, addr:$op)]>;
 def ISTT_Fp32m32 : FpI_<(outs), (ins i32mem:$op, RFP32:$src), OneArgFP,
@@ -534,22 +572,22 @@ def ISTT_Fp64m80 : FpI_<(outs), (ins i64mem:$op, RFP80:$src), OneArgFP,
                     [(X86fp_to_i64mem RFP80:$src, addr:$op)]>;
 } // Predicates = [HasSSE3]
 
-let mayStore = 1, SchedRW = [WriteStore] in {
+let mayStore = 1, SchedRW = [WriteStore], Uses = [FPCW] in {
 def ISTT_FP16m : FPI<0xDF, MRM1m, (outs), (ins i16mem:$dst), "fisttp{s}\t$dst">;
 def ISTT_FP32m : FPI<0xDB, MRM1m, (outs), (ins i32mem:$dst), "fisttp{l}\t$dst">;
 def ISTT_FP64m : FPI<0xDD, MRM1m, (outs), (ins i64mem:$dst), "fisttp{ll}\t$dst">;
 }
 
 // FP Stack manipulation instructions.
-let SchedRW = [WriteMove] in {
-def LD_Frr   : FPI<0xD9, MRM0r, (outs), (ins RST:$op), "fld\t$op">;
-def ST_Frr   : FPI<0xDD, MRM2r, (outs), (ins RST:$op), "fst\t$op">;
-def ST_FPrr  : FPI<0xDD, MRM3r, (outs), (ins RST:$op), "fstp\t$op">;
-def XCH_F    : FPI<0xD9, MRM1r, (outs), (ins RST:$op), "fxch\t$op">;
+let SchedRW = [WriteMove], Uses = [FPCW] in {
+def LD_Frr   : FPI<0xD9, MRM0r, (outs), (ins RSTi:$op), "fld\t$op">;
+def ST_Frr   : FPI<0xDD, MRM2r, (outs), (ins RSTi:$op), "fst\t$op">;
+def ST_FPrr  : FPI<0xDD, MRM3r, (outs), (ins RSTi:$op), "fstp\t$op">;
+def XCH_F    : FPI<0xD9, MRM1r, (outs), (ins RSTi:$op), "fxch\t$op">;
 }
 
 // Floating point constant loads.
-let isReMaterializable = 1, SchedRW = [WriteZero] in {
+let SchedRW = [WriteZero], Uses = [FPCW] in {
 def LD_Fp032 : FpIf32<(outs RFP32:$dst), (ins), ZeroArgFP,
                 [(set RFP32:$dst, fpimm0)]>;
 def LD_Fp132 : FpIf32<(outs RFP32:$dst), (ins), ZeroArgFP,
@@ -564,13 +602,13 @@ def LD_Fp180 : FpI_<(outs RFP80:$dst), (ins), ZeroArgFP,
                 [(set RFP80:$dst, fpimm1)]>;
 }
 
-let SchedRW = [WriteFLD0] in
+let SchedRW = [WriteFLD0], Uses = [FPCW] in
 def LD_F0 : FPI<0xD9, MRM_EE, (outs), (ins), "fldz">;
 
-let SchedRW = [WriteFLD1] in
+let SchedRW = [WriteFLD1], Uses = [FPCW] in
 def LD_F1 : FPI<0xD9, MRM_E8, (outs), (ins), "fld1">;
 
-let SchedRW = [WriteFLDC], Defs = [FPSW] in {
+let SchedRW = [WriteFLDC], Uses = [FPCW] in {
 def FLDL2T : I<0xD9, MRM_E9, (outs), (ins), "fldl2t", []>;
 def FLDL2E : I<0xD9, MRM_EA, (outs), (ins), "fldl2e", []>;
 def FLDPI : I<0xD9, MRM_EB, (outs), (ins), "fldpi", []>;
@@ -579,7 +617,7 @@ def FLDLN2 : I<0xD9, MRM_ED, (outs), (ins), "fldln2", []>;
 } // SchedRW
 
 // Floating point compares.
-let SchedRW = [WriteFCom] in {
+let SchedRW = [WriteFCom], Uses = [FPCW] in {
 def UCOM_Fpr32 : FpIf32<(outs), (ins RFP32:$lhs, RFP32:$rhs), CompareFP,
                         [(set FPSW, (trunc (X86cmp RFP32:$lhs, RFP32:$rhs)))]>;
 def UCOM_Fpr64 : FpIf64<(outs), (ins RFP64:$lhs, RFP64:$rhs), CompareFP,
@@ -591,37 +629,37 @@ def UCOM_Fpr80 : FpI_  <(outs), (ins RFP80:$lhs, RFP80:$rhs), CompareFP,
 
 let SchedRW = [WriteFCom] in {
 // CC = ST(0) cmp ST(i)
-let Defs = [EFLAGS, FPSW] in {
-let Predicates = [FPStackf32, HasCMov] in
-def UCOM_FpIr32: FpIf32<(outs), (ins RFP32:$lhs, RFP32:$rhs), CompareFP,
-                  [(set EFLAGS, (X86cmp RFP32:$lhs, RFP32:$rhs))]>;
-let Predicates = [FPStackf64, HasCMov] in
-def UCOM_FpIr64: FpIf64<(outs), (ins RFP64:$lhs, RFP64:$rhs), CompareFP,
-                  [(set EFLAGS, (X86cmp RFP64:$lhs, RFP64:$rhs))]>;
-let Predicates = [HasCMov] in
+let Defs = [EFLAGS, FPSW], Uses = [FPCW] in {
+def UCOM_FpIr32: FpI_<(outs), (ins RFP32:$lhs, RFP32:$rhs), CompareFP,
+                  [(set EFLAGS, (X86cmp RFP32:$lhs, RFP32:$rhs))]>,
+                  Requires<[FPStackf32, HasCMov]>;
+def UCOM_FpIr64: FpI_<(outs), (ins RFP64:$lhs, RFP64:$rhs), CompareFP,
+                  [(set EFLAGS, (X86cmp RFP64:$lhs, RFP64:$rhs))]>,
+                  Requires<[FPStackf64, HasCMov]>;
 def UCOM_FpIr80: FpI_<(outs), (ins RFP80:$lhs, RFP80:$rhs), CompareFP,
-                  [(set EFLAGS, (X86cmp RFP80:$lhs, RFP80:$rhs))]>;
+                  [(set EFLAGS, (X86cmp RFP80:$lhs, RFP80:$rhs))]>,
+                  Requires<[HasCMov]>;
 }
 
-let Defs = [FPSW], Uses = [ST0] in {
+let Defs = [FPSW], Uses = [ST0, FPCW] in {
 def UCOM_Fr    : FPI<0xDD, MRM4r,    // FPSW = cmp ST(0) with ST(i)
-                    (outs), (ins RST:$reg), "fucom\t$reg">;
+                    (outs), (ins RSTi:$reg), "fucom\t$reg">;
 def UCOM_FPr   : FPI<0xDD, MRM5r,    // FPSW = cmp ST(0) with ST(i), pop
-                    (outs), (ins RST:$reg), "fucomp\t$reg">;
+                    (outs), (ins RSTi:$reg), "fucomp\t$reg">;
 def UCOM_FPPr  : FPI<0xDA, MRM_E9,       // cmp ST(0) with ST(1), pop, pop
                     (outs), (ins), "fucompp">;
 }
 
-let Defs = [EFLAGS, FPSW], Uses = [ST0] in {
+let Defs = [EFLAGS, FPSW], Uses = [ST0, FPCW] in {
 def UCOM_FIr   : FPI<0xDB, MRM5r,     // CC = cmp ST(0) with ST(i)
-                    (outs), (ins RST:$reg), "fucomi\t$reg">;
+                    (outs), (ins RSTi:$reg), "fucomi\t{$reg, %st|st, $reg}">;
 def UCOM_FIPr  : FPI<0xDF, MRM5r,     // CC = cmp ST(0) with ST(i), pop
-                    (outs), (ins RST:$reg), "fucompi\t$reg">;
-}
+                    (outs), (ins RSTi:$reg), "fucompi\t{$reg, %st|st, $reg}">;
 
-let Defs = [EFLAGS, FPSW] in {
-def COM_FIr : FPI<0xDB, MRM6r, (outs), (ins RST:$reg), "fcomi\t$reg">;
-def COM_FIPr : FPI<0xDF, MRM6r, (outs), (ins RST:$reg), "fcompi\t$reg">;
+def COM_FIr : FPI<0xDB, MRM6r, (outs), (ins RSTi:$reg),
+                  "fcomi\t{$reg, %st|st, $reg}">;
+def COM_FIPr : FPI<0xDF, MRM6r, (outs), (ins RSTi:$reg),
+                   "fcompi\t{$reg, %st|st, $reg}">;
 }
 } // SchedRW
 
@@ -631,12 +669,12 @@ let Defs = [AX], Uses = [FPSW] in
 def FNSTSW16r : I<0xDF, MRM_E0,                  // AX = fp flags
                   (outs), (ins), "fnstsw\t{%ax|ax}",
                   [(set AX, (X86fp_stsw FPSW))]>;
-let Defs = [FPSW] in
+let Defs = [FPSW], Uses = [FPCW] in
 def FNSTCW16m : I<0xD9, MRM7m,                   // [mem16] = X87 control world
                   (outs), (ins i16mem:$dst), "fnstcw\t$dst",
                   [(X86fp_cwd_get16 addr:$dst)]>;
 } // SchedRW
-let Defs = [FPSW], mayLoad = 1 in
+let Defs = [FPSW,FPCW], mayLoad = 1 in
 def FLDCW16m  : I<0xD9, MRM5m,                   // X87 control world = [mem16]
                   (outs), (ins i16mem:$dst), "fldcw\t$dst", []>,
                 Sched<[WriteLoad]>;
@@ -645,8 +683,8 @@ def FLDCW16m  : I<0xD9, MRM5m,                   // X87 control world = [mem16]
 let SchedRW = [WriteMicrocoded] in {
 let Defs = [FPSW] in {
 def FNINIT : I<0xDB, MRM_E3, (outs), (ins), "fninit", []>;
-def FFREE : FPI<0xDD, MRM0r, (outs), (ins RST:$reg), "ffree\t$reg">;
-def FFREEP : FPI<0xDF, MRM0r, (outs), (ins RST:$reg), "ffreep\t$reg">;
+def FFREE : FPI<0xDD, MRM0r, (outs), (ins RSTi:$reg), "ffree\t$reg">;
+def FFREEP : FPI<0xDF, MRM0r, (outs), (ins RSTi:$reg), "ffreep\t$reg">;
 
 // Clear exceptions
 def FNCLEX : I<0xDB, MRM_E2, (outs), (ins), "fnclex", []>;
@@ -695,21 +733,17 @@ def FXRSTOR64 : RI<0xAE, MRM1m, (outs), (ins opaquemem:$src),
 //===----------------------------------------------------------------------===//
 
 // Required for RET of f32 / f64 / f80 values.
-def : Pat<(X86fld addr:$src, f32), (LD_Fp32m addr:$src)>;
-def : Pat<(X86fld addr:$src, f64), (LD_Fp64m addr:$src)>;
-def : Pat<(X86fld addr:$src, f80), (LD_Fp80m addr:$src)>;
+def : Pat<(X86fldf32 addr:$src), (LD_Fp32m addr:$src)>;
+def : Pat<(X86fldf64 addr:$src), (LD_Fp64m addr:$src)>;
+def : Pat<(X86fldf80 addr:$src), (LD_Fp80m addr:$src)>;
 
 // Required for CALL which return f32 / f64 / f80 values.
-def : Pat<(X86fst RFP32:$src, addr:$op, f32), (ST_Fp32m addr:$op, RFP32:$src)>;
-def : Pat<(X86fst RFP64:$src, addr:$op, f32), (ST_Fp64m32 addr:$op,
-                                                          RFP64:$src)>;
-def : Pat<(X86fst RFP64:$src, addr:$op, f64), (ST_Fp64m addr:$op, RFP64:$src)>;
-def : Pat<(X86fst RFP80:$src, addr:$op, f32), (ST_Fp80m32 addr:$op,
-                                                          RFP80:$src)>;
-def : Pat<(X86fst RFP80:$src, addr:$op, f64), (ST_Fp80m64 addr:$op,
-                                                          RFP80:$src)>;
-def : Pat<(X86fst RFP80:$src, addr:$op, f80), (ST_FpP80m addr:$op,
-                                                         RFP80:$src)>;
+def : Pat<(X86fstf32 RFP32:$src, addr:$op), (ST_Fp32m addr:$op, RFP32:$src)>;
+def : Pat<(X86fstf32 RFP64:$src, addr:$op), (ST_Fp64m32 addr:$op, RFP64:$src)>;
+def : Pat<(X86fstf64 RFP64:$src, addr:$op), (ST_Fp64m addr:$op, RFP64:$src)>;
+def : Pat<(X86fstf32 RFP80:$src, addr:$op), (ST_Fp80m32 addr:$op, RFP80:$src)>;
+def : Pat<(X86fstf64 RFP80:$src, addr:$op), (ST_Fp80m64 addr:$op, RFP80:$src)>;
+def : Pat<(X86fstf80 RFP80:$src, addr:$op), (ST_FpP80m addr:$op, RFP80:$src)>;
 
 // Floating point constant -0.0 and -1.0
 def : Pat<(f32 fpimmneg0), (CHS_Fp32 (LD_Fp032))>, Requires<[FPStackf32]>;
@@ -720,7 +754,7 @@ def : Pat<(f80 fpimmneg0), (CHS_Fp80 (LD_Fp080))>;
 def : Pat<(f80 fpimmneg1), (CHS_Fp80 (LD_Fp180))>;
 
 // Used to conv. i64 to f64 since there isn't a SSE version.
-def : Pat<(X86fildflag addr:$src, i64), (ILD_Fp64m64 addr:$src)>;
+def : Pat<(X86fildflag64 addr:$src), (ILD_Fp64m64 addr:$src)>;
 
 // FP extensions map onto simple pseudo-value conversions if they are to/from
 // the FP stack.
diff --git a/lib/Target/X86/X86InstrFoldTables.cpp b/lib/Target/X86/X86InstrFoldTables.cpp
index 7d31cfab4137e1781404e0ce3584020552ea8a0f..17ef1d67af916d97d800aea7d696dfadb1059f24 100644
--- a/lib/Target/X86/X86InstrFoldTables.cpp
+++ b/lib/Target/X86/X86InstrFoldTables.cpp
@@ -1,9 +1,8 @@
 //===-- X86InstrFoldTables.cpp - X86 Instruction Folding Tables -----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/X86/X86InstrFoldTables.h b/lib/Target/X86/X86InstrFoldTables.h
index 90016baead96d7b81cb13f33b376613160670fde..419baf98f61df5dd6c7e4aca030e560669cb322b 100644
--- a/lib/Target/X86/X86InstrFoldTables.h
+++ b/lib/Target/X86/X86InstrFoldTables.h
@@ -1,9 +1,8 @@
 //===-- X86InstrFoldTables.h - X86 Instruction Folding Tables ---*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/X86/X86InstrFormats.td b/lib/Target/X86/X86InstrFormats.td
index 47d4719d306058a87bdf15067fb3ae3877ee4f87..d8a442dc9d6a52b5c87f9e1ae3806e45ccfee7b1 100644
--- a/lib/Target/X86/X86InstrFormats.td
+++ b/lib/Target/X86/X86InstrFormats.td
@@ -1,9 +1,8 @@
 //===-- X86InstrFormats.td - X86 Instruction Formats -------*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/X86/X86InstrFragmentsSIMD.td b/lib/Target/X86/X86InstrFragmentsSIMD.td
index 0b98abaa7a267b4592765fdbd9f9e8099594cfeb..cde6337d5761e43c01d502dad2afd8e4f85e6a9e 100644
--- a/lib/Target/X86/X86InstrFragmentsSIMD.td
+++ b/lib/Target/X86/X86InstrFragmentsSIMD.td
@@ -1,9 +1,8 @@
 //===-- X86InstrFragmentsSIMD.td - x86 SIMD ISA ------------*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -75,7 +74,7 @@ def X86dbpsadbw : SDNode<"X86ISD::DBPSADBW",
                   SDTypeProfile<1, 3, [SDTCVecEltisVT<0, i16>,
                                        SDTCVecEltisVT<1, i8>,
                                        SDTCisSameSizeAs<0,1>,
-                                       SDTCisSameAs<1,2>, SDTCisInt<3>]>>;
+                                       SDTCisSameAs<1,2>, SDTCisVT<3, i8>]>>;
 def X86andnp   : SDNode<"X86ISD::ANDNP",
                  SDTypeProfile<1, 2, [SDTCisVec<0>, SDTCisSameAs<0,1>,
                                       SDTCisSameAs<0,2>]>>;
@@ -104,12 +103,21 @@ def X86vzload  : SDNode<"X86ISD::VZEXT_LOAD", SDTLoad,
                         [SDNPHasChain, SDNPMayLoad, SDNPMemOperand]>;
 
 def SDTVtrunc    : SDTypeProfile<1, 1, [SDTCisVec<0>, SDTCisVec<1>,
-                                       SDTCisInt<0>, SDTCisInt<1>,
-                                       SDTCisOpSmallerThanOp<0, 1>]>;
+                                        SDTCisInt<0>, SDTCisInt<1>,
+                                        SDTCisOpSmallerThanOp<0, 1>]>;
+def SDTVmtrunc   : SDTypeProfile<1, 3, [SDTCisVec<0>, SDTCisVec<1>,
+                                        SDTCisInt<0>, SDTCisInt<1>,
+                                        SDTCisOpSmallerThanOp<0, 1>,
+                                        SDTCisSameAs<0, 2>,
+                                        SDTCVecEltisVT<3, i1>,
+                                        SDTCisSameNumEltsAs<1, 3>]>;
 
 def X86vtrunc    : SDNode<"X86ISD::VTRUNC",   SDTVtrunc>;
 def X86vtruncs   : SDNode<"X86ISD::VTRUNCS",  SDTVtrunc>;
 def X86vtruncus  : SDNode<"X86ISD::VTRUNCUS", SDTVtrunc>;
+def X86vmtrunc   : SDNode<"X86ISD::VMTRUNC",   SDTVmtrunc>;
+def X86vmtruncs  : SDNode<"X86ISD::VMTRUNCS",  SDTVmtrunc>;
+def X86vmtruncus : SDNode<"X86ISD::VMTRUNCUS", SDTVmtrunc>;
 
 def X86vfpext  : SDNode<"X86ISD::VFPEXT",
                         SDTypeProfile<1, 1, [SDTCVecEltisVT<0, f64>,
@@ -118,7 +126,7 @@ def X86vfpext  : SDNode<"X86ISD::VFPEXT",
 def X86vfpround: SDNode<"X86ISD::VFPROUND",
                         SDTypeProfile<1, 1, [SDTCVecEltisVT<0, f32>,
                                              SDTCVecEltisVT<1, f64>,
-                                             SDTCisSameSizeAs<0, 1>]>>;
+                                             SDTCisOpSmallerThanOp<0, 1>]>>;
 
 def X86froundRnd: SDNode<"X86ISD::VFPROUNDS_RND",
                         SDTypeProfile<1, 3, [SDTCVecEltisVT<0, f32>,
@@ -134,6 +142,14 @@ def X86fpextRnd  : SDNode<"X86ISD::VFPEXTS_RND",
                                              SDTCisSameSizeAs<0, 2>,
                                              SDTCisVT<3, i32>]>>;
 
+def X86vmfpround: SDNode<"X86ISD::VMFPROUND",
+                         SDTypeProfile<1, 3, [SDTCVecEltisVT<0, f32>,
+                                              SDTCVecEltisVT<1, f64>,
+                                              SDTCisSameSizeAs<0, 1>,
+                                              SDTCisSameAs<0, 2>,
+                                              SDTCVecEltisVT<3, i1>,
+                                              SDTCisSameNumEltsAs<1, 3>]>>;
+
 def X86vshiftimm : SDTypeProfile<1, 2, [SDTCisVec<0>, SDTCisSameAs<0,1>,
                                         SDTCisVT<2, i8>, SDTCisInt<0>]>;
 
@@ -172,7 +188,7 @@ def X86phminpos: SDNode<"X86ISD::PHMINPOS",
 
 def X86vshiftuniform : SDTypeProfile<1, 2, [SDTCisVec<0>, SDTCisSameAs<0,1>,
                                             SDTCisVec<2>, SDTCisInt<0>,
-                                            SDTCisInt<1>]>;
+                                            SDTCisInt<2>]>;
 
 def X86vshl    : SDNode<"X86ISD::VSHL", X86vshiftuniform>;
 def X86vsrl    : SDNode<"X86ISD::VSRL", X86vshiftuniform>;
@@ -181,6 +197,8 @@ def X86vsra    : SDNode<"X86ISD::VSRA", X86vshiftuniform>;
 def X86vshiftvariable : SDTypeProfile<1, 2, [SDTCisVec<0>, SDTCisSameAs<0,1>,
                                              SDTCisSameAs<0,2>, SDTCisInt<0>]>;
 
+def X86vshlv   : SDNode<"X86ISD::VSHLV", X86vshiftvariable>;
+def X86vsrlv   : SDNode<"X86ISD::VSRLV", X86vshiftvariable>;
 def X86vsrav   : SDNode<"X86ISD::VSRAV", X86vshiftvariable>;
 
 def X86vshli   : SDNode<"X86ISD::VSHLI", X86vshiftimm>;
@@ -431,6 +449,12 @@ def X86VBroadcast : SDNode<"X86ISD::VBROADCAST", SDTVBroadcast>;
 def X86VBroadcastm : SDNode<"X86ISD::VBROADCASTM", SDTVBroadcastm>;
 
 def X86Blendi    : SDNode<"X86ISD::BLENDI",   SDTBlend>;
+def X86Blendv    : SDNode<"X86ISD::BLENDV",
+                     SDTypeProfile<1, 3, [SDTCisVec<0>, SDTCisInt<1>,
+                                          SDTCisSameAs<0, 2>,
+                                          SDTCisSameAs<2, 3>,
+                                          SDTCisSameNumEltsAs<0, 1>,
+                                          SDTCisSameSizeAs<0, 1>]>>;
 
 def X86Addsub    : SDNode<"X86ISD::ADDSUB", SDTFPBinOp>;
 
@@ -500,10 +524,14 @@ def X86RndScalesRnd : SDNode<"X86ISD::VRNDSCALES_RND", SDTFPBinOpImmRound>;
 def X86ReducesRnd   : SDNode<"X86ISD::VREDUCES_RND",   SDTFPBinOpImmRound>;
 def X86GetMantsRnd  : SDNode<"X86ISD::VGETMANTS_RND",  SDTFPBinOpImmRound>;
 
-def X86compress: SDNode<"X86ISD::COMPRESS", SDTypeProfile<1, 1,
-                              [SDTCisSameAs<0, 1>, SDTCisVec<1>]>, []>;
-def X86expand  : SDNode<"X86ISD::EXPAND", SDTypeProfile<1, 1,
-                              [SDTCisSameAs<0, 1>, SDTCisVec<1>]>, []>;
+def X86compress: SDNode<"X86ISD::COMPRESS", SDTypeProfile<1, 3,
+                              [SDTCisSameAs<0, 1>, SDTCisVec<1>,
+                               SDTCisSameAs<0, 2>, SDTCVecEltisVT<3, i1>,
+                               SDTCisSameNumEltsAs<0, 3>]>, []>;
+def X86expand  : SDNode<"X86ISD::EXPAND", SDTypeProfile<1, 3,
+                              [SDTCisSameAs<0, 1>, SDTCisVec<1>,
+                               SDTCisSameAs<0, 2>, SDTCVecEltisVT<3, i1>,
+                               SDTCisSameNumEltsAs<0, 3>]>, []>;
 
 // vpshufbitqmb
 def X86Vpshufbitqmb : SDNode<"X86ISD::VPSHUFBITQMB",
@@ -573,6 +601,29 @@ def X86cvtp2Int      : SDNode<"X86ISD::CVTP2SI",  SDTFloatToInt>;
 def X86cvtp2UInt     : SDNode<"X86ISD::CVTP2UI",  SDTFloatToInt>;
 
 
+// Masked versions of above
+def SDTMVintToFP: SDTypeProfile<1, 3, [SDTCisVec<0>, SDTCisVec<1>,
+                                       SDTCisFP<0>, SDTCisInt<1>,
+                                       SDTCisSameSizeAs<0, 1>,
+                                       SDTCisSameAs<0, 2>,
+                                       SDTCVecEltisVT<3, i1>,
+                                       SDTCisSameNumEltsAs<1, 3>]>;
+def SDTMFloatToInt: SDTypeProfile<1, 3, [SDTCisVec<0>, SDTCisVec<1>,
+                                         SDTCisInt<0>, SDTCisFP<1>,
+                                         SDTCisSameSizeAs<0, 1>,
+                                         SDTCisSameAs<0, 2>,
+                                         SDTCVecEltisVT<3, i1>,
+                                         SDTCisSameNumEltsAs<1, 3>]>;
+
+def X86VMSintToFP    : SDNode<"X86ISD::MCVTSI2P",  SDTMVintToFP>;
+def X86VMUintToFP    : SDNode<"X86ISD::MCVTUI2P",  SDTMVintToFP>;
+
+def X86mcvtp2Int     : SDNode<"X86ISD::MCVTP2SI",  SDTMFloatToInt>;
+def X86mcvtp2UInt    : SDNode<"X86ISD::MCVTP2UI",  SDTMFloatToInt>;
+def X86mcvttp2si     : SDNode<"X86ISD::MCVTTP2SI", SDTMFloatToInt>;
+def X86mcvttp2ui     : SDNode<"X86ISD::MCVTTP2UI", SDTMFloatToInt>;
+
+
 def X86cvtph2ps     : SDNode<"X86ISD::CVTPH2PS",
                               SDTypeProfile<1, 1, [SDTCVecEltisVT<0, f32>,
                                                    SDTCVecEltisVT<1, i16>]> >;
@@ -586,6 +637,13 @@ def X86cvtps2ph   : SDNode<"X86ISD::CVTPS2PH",
                         SDTypeProfile<1, 2, [SDTCVecEltisVT<0, i16>,
                                              SDTCVecEltisVT<1, f32>,
                                              SDTCisVT<2, i32>]> >;
+def X86mcvtps2ph   : SDNode<"X86ISD::MCVTPS2PH",
+                        SDTypeProfile<1, 4, [SDTCVecEltisVT<0, i16>,
+                                             SDTCVecEltisVT<1, f32>,
+                                             SDTCisVT<2, i32>,
+                                             SDTCisSameAs<0, 3>,
+                                             SDTCVecEltisVT<4, i1>,
+                                             SDTCisSameNumEltsAs<1, 4>]> >;
 def X86vfpextRnd  : SDNode<"X86ISD::VFPEXT_RND",
                         SDTypeProfile<1, 2, [SDTCVecEltisVT<0, f64>,
                                              SDTCVecEltisVT<1, f32>,
diff --git a/lib/Target/X86/X86InstrInfo.cpp b/lib/Target/X86/X86InstrInfo.cpp
index 40de049bfe8bab671a64ff346ff0f8e358ab7c0d..91939398d959c04340df0b9446bcbce0d02f71ef 100644
--- a/lib/Target/X86/X86InstrInfo.cpp
+++ b/lib/Target/X86/X86InstrInfo.cpp
@@ -1,9 +1,8 @@
 //===-- X86InstrInfo.cpp - X86 Instruction Information --------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -483,7 +482,6 @@ bool X86InstrInfo::isReallyTriviallyReMaterializable(const MachineInstr &MI,
   case X86::MOV16rm:
   case X86::MOV32rm:
   case X86::MOV64rm:
-  case X86::LD_Fp64m:
   case X86::MOVSSrm:
   case X86::MOVSDrm:
   case X86::MOVAPSrm:
@@ -1570,6 +1568,28 @@ MachineInstr *X86InstrInfo::commuteInstructionImpl(MachineInstr &MI, bool NewMI,
     return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false,
                                                    OpIdx1, OpIdx2);
   }
+  case X86::INSERTPSrr:
+  case X86::VINSERTPSrr:
+  case X86::VINSERTPSZrr: {
+    unsigned Imm = MI.getOperand(MI.getNumOperands() - 1).getImm();
+    unsigned ZMask = Imm & 15;
+    unsigned DstIdx = (Imm >> 4) & 3;
+    unsigned SrcIdx = (Imm >> 6) & 3;
+
+    // We can commute insertps if we zero 2 of the elements, the insertion is
+    // "inline" and we don't override the insertion with a zero.
+    if (DstIdx == SrcIdx && (ZMask & (1 << DstIdx)) == 0 &&
+        countPopulation(ZMask) == 2) {
+      unsigned AltIdx = findFirstSet((ZMask | (1 << DstIdx)) ^ 15);
+      assert(AltIdx < 4 && "Illegal insertion index");
+      unsigned AltImm = (AltIdx << 6) | (AltIdx << 4) | ZMask;
+      auto &WorkingMI = cloneIfNew(MI);
+      WorkingMI.getOperand(MI.getNumOperands() - 1).setImm(AltImm);
+      return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false,
+                                                     OpIdx1, OpIdx2);
+    }
+    return nullptr;
+  }
   case X86::MOVSDrr:
   case X86::MOVSSrr:
   case X86::VMOVSDrr:
@@ -3455,7 +3475,9 @@ inline static bool isRedundantFlagInstr(const MachineInstr &FlagI,
 
 /// Check whether the definition can be converted
 /// to remove a comparison against zero.
-inline static bool isDefConvertible(const MachineInstr &MI) {
+inline static bool isDefConvertible(const MachineInstr &MI, bool &NoSignFlag) {
+  NoSignFlag = false;
+
   switch (MI.getOpcode()) {
   default: return false;
 
@@ -3520,8 +3542,6 @@ inline static bool isDefConvertible(const MachineInstr &MI) {
   case X86::SHL8r1:    case X86::SHL16r1:  case X86::SHL32r1:case X86::SHL64r1:
   case X86::ANDN32rr:  case X86::ANDN32rm:
   case X86::ANDN64rr:  case X86::ANDN64rm:
-  case X86::BEXTR32rr: case X86::BEXTR64rr:
-  case X86::BEXTR32rm: case X86::BEXTR64rm:
   case X86::BLSI32rr:  case X86::BLSI32rm:
   case X86::BLSI64rr:  case X86::BLSI64rm:
   case X86::BLSMSK32rr:case X86::BLSMSK32rm:
@@ -3539,8 +3559,6 @@ inline static bool isDefConvertible(const MachineInstr &MI) {
   case X86::TZCNT16rr: case X86::TZCNT16rm:
   case X86::TZCNT32rr: case X86::TZCNT32rm:
   case X86::TZCNT64rr: case X86::TZCNT64rm:
-  case X86::BEXTRI32ri:  case X86::BEXTRI32mi:
-  case X86::BEXTRI64ri:  case X86::BEXTRI64mi:
   case X86::BLCFILL32rr: case X86::BLCFILL32rm:
   case X86::BLCFILL64rr: case X86::BLCFILL64rm:
   case X86::BLCI32rr:    case X86::BLCI32rm:
@@ -3560,6 +3578,13 @@ inline static bool isDefConvertible(const MachineInstr &MI) {
   case X86::TZMSK32rr:   case X86::TZMSK32rm:
   case X86::TZMSK64rr:   case X86::TZMSK64rm:
     return true;
+  case X86::BEXTR32rr:   case X86::BEXTR64rr:
+  case X86::BEXTR32rm:   case X86::BEXTR64rm:
+  case X86::BEXTRI32ri:  case X86::BEXTRI32mi:
+  case X86::BEXTRI64ri:  case X86::BEXTRI64mi:
+    // BEXTR doesn't update the sign flag so we can't use it.
+    NoSignFlag = true;
+    return true;
   }
 }
 
@@ -3662,8 +3687,9 @@ bool X86InstrInfo::optimizeCompareInstr(MachineInstr &CmpInstr, unsigned SrcReg,
   // instruction we can eliminate the compare iff the use sets EFLAGS in the
   // right way.
   bool ShouldUpdateCC = false;
+  bool NoSignFlag = false;
   X86::CondCode NewCC = X86::COND_INVALID;
-  if (IsCmpZero && !isDefConvertible(*MI)) {
+  if (IsCmpZero && !isDefConvertible(*MI, NoSignFlag)) {
     // Scan forward from the use until we hit the use we're looking for or the
     // compare instruction.
     for (MachineBasicBlock::iterator J = MI;; ++J) {
@@ -3782,6 +3808,12 @@ bool X86InstrInfo::optimizeCompareInstr(MachineInstr &CmpInstr, unsigned SrcReg,
       case X86::COND_O: case X86::COND_NO:
         // CF and OF are used, we can't perform this optimization.
         return false;
+      case X86::COND_S: case X86::COND_NS:
+        // If SF is used, but the instruction doesn't update the SF, then we
+        // can't do the optimization.
+        if (NoSignFlag)
+          return false;
+        break;
       }
 
       // If we're updating the condition code check if we have to reverse the
@@ -6007,6 +6039,8 @@ static const uint16_t ReplaceableInstrs[][3] = {
   { X86::VBROADCASTSSZ256m, X86::VBROADCASTSSZ256m, X86::VPBROADCASTDZ256m },
   { X86::VBROADCASTSSZr,    X86::VBROADCASTSSZr,    X86::VPBROADCASTDZr },
   { X86::VBROADCASTSSZm,    X86::VBROADCASTSSZm,    X86::VPBROADCASTDZm },
+  { X86::VMOVDDUPZ128rr,    X86::VMOVDDUPZ128rr,    X86::VPBROADCASTQZ128r },
+  { X86::VMOVDDUPZ128rm,    X86::VMOVDDUPZ128rm,    X86::VPBROADCASTQZ128m },
   { X86::VBROADCASTSDZ256r, X86::VBROADCASTSDZ256r, X86::VPBROADCASTQZ256r },
   { X86::VBROADCASTSDZ256m, X86::VBROADCASTSDZ256m, X86::VPBROADCASTQZ256m },
   { X86::VBROADCASTSDZr,    X86::VBROADCASTSDZr,    X86::VPBROADCASTQZr },
@@ -6097,6 +6131,8 @@ static const uint16_t ReplaceableInstrsAVX2[][3] = {
   { X86::VPERM2F128rr,   X86::VPERM2F128rr,   X86::VPERM2I128rr },
   { X86::VBROADCASTSSrm, X86::VBROADCASTSSrm, X86::VPBROADCASTDrm},
   { X86::VBROADCASTSSrr, X86::VBROADCASTSSrr, X86::VPBROADCASTDrr},
+  { X86::VMOVDDUPrm,     X86::VMOVDDUPrm,     X86::VPBROADCASTQrm},
+  { X86::VMOVDDUPrr,     X86::VMOVDDUPrr,     X86::VPBROADCASTQrr},
   { X86::VBROADCASTSSYrr, X86::VBROADCASTSSYrr, X86::VPBROADCASTDYrr},
   { X86::VBROADCASTSSYrm, X86::VBROADCASTSSYrm, X86::VPBROADCASTDYrm},
   { X86::VBROADCASTSDYrr, X86::VBROADCASTSDYrr, X86::VPBROADCASTQYrr},
diff --git a/lib/Target/X86/X86InstrInfo.h b/lib/Target/X86/X86InstrInfo.h
index 159cb50afc5c017fcb10ffb52f85f0528d88a1a1..bfaa76d49f985dc83745a025f4d6a0f7a261a6f5 100644
--- a/lib/Target/X86/X86InstrInfo.h
+++ b/lib/Target/X86/X86InstrInfo.h
@@ -1,9 +1,8 @@
 //===-- X86InstrInfo.h - X86 Instruction Information ------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/X86/X86InstrInfo.td b/lib/Target/X86/X86InstrInfo.td
index fe9b530e8ea1a33e17dc054c5ec8bc6a3c0d74ec..5cedac36c8d20f6cc5477716f60ec332f252cc68 100644
--- a/lib/Target/X86/X86InstrInfo.td
+++ b/lib/Target/X86/X86InstrInfo.td
@@ -1,9 +1,8 @@
 //===-- X86InstrInfo.td - Main X86 Instruction Definition --*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -253,8 +252,6 @@ def X86umul_flag : SDNode<"X86ISD::UMUL", SDT2ResultBinaryArithWithFlags,
 def X86adc_flag  : SDNode<"X86ISD::ADC",  SDTBinaryArithWithFlagsInOut>;
 def X86sbb_flag  : SDNode<"X86ISD::SBB",  SDTBinaryArithWithFlagsInOut>;
 
-def X86inc_flag  : SDNode<"X86ISD::INC",  SDTUnaryArithWithFlags>;
-def X86dec_flag  : SDNode<"X86ISD::DEC",  SDTUnaryArithWithFlags>;
 def X86or_flag   : SDNode<"X86ISD::OR",   SDTBinaryArithWithFlags,
                           [SDNPCommutative]>;
 def X86xor_flag  : SDNode<"X86ISD::XOR",  SDTBinaryArithWithFlags,
@@ -278,13 +275,6 @@ def X86lock_and  : SDNode<"X86ISD::LAND",  SDTLockBinaryArithWithFlags,
                           [SDNPHasChain, SDNPMayStore, SDNPMayLoad,
                            SDNPMemOperand]>;
 
-def X86lock_inc  : SDNode<"X86ISD::LINC",  SDTLockUnaryArithWithFlags,
-                          [SDNPHasChain, SDNPMayStore, SDNPMayLoad,
-                           SDNPMemOperand]>;
-def X86lock_dec  : SDNode<"X86ISD::LDEC",  SDTLockUnaryArithWithFlags,
-                          [SDNPHasChain, SDNPMayStore, SDNPMayLoad,
-                           SDNPMemOperand]>;
-
 def X86bextr  : SDNode<"X86ISD::BEXTR",  SDTIntBinOp>;
 
 def X86bzhi   : SDNode<"X86ISD::BZHI",   SDTIntBinOp>;
@@ -2387,6 +2377,21 @@ let Predicates = [HasBMI], Defs = [EFLAGS] in {
 // Pattern fragments to auto generate BMI instructions.
 //===----------------------------------------------------------------------===//
 
+def or_flag_nocf : PatFrag<(ops node:$lhs, node:$rhs),
+                           (X86or_flag node:$lhs, node:$rhs), [{
+  return hasNoCarryFlagUses(SDValue(N, 1));
+}]>;
+
+def xor_flag_nocf : PatFrag<(ops node:$lhs, node:$rhs),
+                            (X86xor_flag node:$lhs, node:$rhs), [{
+  return hasNoCarryFlagUses(SDValue(N, 1));
+}]>;
+
+def and_flag_nocf : PatFrag<(ops node:$lhs, node:$rhs),
+                            (X86and_flag node:$lhs, node:$rhs), [{
+  return hasNoCarryFlagUses(SDValue(N, 1));
+}]>;
+
 let Predicates = [HasBMI] in {
   // FIXME: patterns for the load versions are not implemented
   def : Pat<(and GR32:$src, (add GR32:$src, -1)),
@@ -2403,6 +2408,17 @@ let Predicates = [HasBMI] in {
             (BLSI32rr GR32:$src)>;
   def : Pat<(and GR64:$src, (ineg GR64:$src)),
             (BLSI64rr GR64:$src)>;
+
+  // Versions to match flag producing ops.
+  def : Pat<(xor_flag_nocf GR32:$src, (add GR32:$src, -1)),
+            (BLSMSK32rr GR32:$src)>;
+  def : Pat<(xor_flag_nocf GR64:$src, (add GR64:$src, -1)),
+            (BLSMSK64rr GR64:$src)>;
+
+  def : Pat<(and_flag_nocf GR32:$src, (add GR32:$src, -1)),
+            (BLSR32rr GR32:$src)>;
+  def : Pat<(and_flag_nocf GR64:$src, (add GR64:$src, -1)),
+            (BLSR64rr GR64:$src)>;
 }
 
 multiclass bmi_bextr<bits<8> opc, string mnemonic, RegisterClass RC,
@@ -2801,6 +2817,45 @@ let Predicates = [HasTBM] in {
             (TZMSK32rr GR32:$src)>;
   def : Pat<(and (not GR64:$src), (add GR64:$src, -1)),
             (TZMSK64rr GR64:$src)>;
+
+  // Patterns to match flag producing ops.
+  // X86and_flag nodes are rarely created. Those should use CMP+AND. We do
+  // TESTrr matching in PostProcessISelDAG to allow BLSR/BLSI to be formed.
+  def : Pat<(or_flag_nocf GR32:$src, (not (add GR32:$src, 1))),
+            (BLCI32rr GR32:$src)>;
+  def : Pat<(or_flag_nocf GR64:$src, (not (add GR64:$src, 1))),
+            (BLCI64rr GR64:$src)>;
+
+  // Extra patterns because opt can optimize the above patterns to this.
+  def : Pat<(or_flag_nocf GR32:$src, (sub -2, GR32:$src)),
+            (BLCI32rr GR32:$src)>;
+  def : Pat<(or_flag_nocf GR64:$src, (sub -2, GR64:$src)),
+            (BLCI64rr GR64:$src)>;
+
+  def : Pat<(xor_flag_nocf GR32:$src, (add GR32:$src, 1)),
+            (BLCMSK32rr GR32:$src)>;
+  def : Pat<(xor_flag_nocf GR64:$src, (add GR64:$src, 1)),
+            (BLCMSK64rr GR64:$src)>;
+
+  def : Pat<(or_flag_nocf GR32:$src, (add GR32:$src, 1)),
+            (BLCS32rr GR32:$src)>;
+  def : Pat<(or_flag_nocf GR64:$src, (add GR64:$src, 1)),
+            (BLCS64rr GR64:$src)>;
+
+  def : Pat<(or_flag_nocf GR32:$src, (add GR32:$src, -1)),
+            (BLSFILL32rr GR32:$src)>;
+  def : Pat<(or_flag_nocf GR64:$src, (add GR64:$src, -1)),
+            (BLSFILL64rr GR64:$src)>;
+
+  def : Pat<(or_flag_nocf (not GR32:$src), (add GR32:$src, -1)),
+            (BLSIC32rr GR32:$src)>;
+  def : Pat<(or_flag_nocf (not GR64:$src), (add GR64:$src, -1)),
+            (BLSIC64rr GR64:$src)>;
+
+  def : Pat<(or_flag_nocf (not GR32:$src), (add GR32:$src, 1)),
+            (T1MSKC32rr GR32:$src)>;
+  def : Pat<(or_flag_nocf (not GR64:$src), (add GR64:$src, 1)),
+            (T1MSKC64rr GR64:$src)>;
 } // HasTBM
 
 //===----------------------------------------------------------------------===//
@@ -3183,39 +3238,39 @@ def : InstAlias<"fucompi",      (UCOM_FIPr   ST1), 0>;
 // instructions like "fadd %st(0), %st(0)" as "fadd %st(0)" for consistency with
 // gas.
 multiclass FpUnaryAlias<string Mnemonic, Instruction Inst, bit EmitAlias = 1> {
- def : InstAlias<!strconcat(Mnemonic, "\t{$op, %st(0)|st(0), $op}"),
-                 (Inst RST:$op), EmitAlias>;
- def : InstAlias<!strconcat(Mnemonic, "\t{%st(0), %st(0)|st(0), st(0)}"),
+ def : InstAlias<!strconcat(Mnemonic, "\t$op"),
+                 (Inst RSTi:$op), EmitAlias>;
+ def : InstAlias<!strconcat(Mnemonic, "\t{%st, %st|st, st}"),
                  (Inst ST0), EmitAlias>;
 }
 
-defm : FpUnaryAlias<"fadd",   ADD_FST0r>;
+defm : FpUnaryAlias<"fadd",   ADD_FST0r, 0>;
 defm : FpUnaryAlias<"faddp",  ADD_FPrST0, 0>;
-defm : FpUnaryAlias<"fsub",   SUB_FST0r>;
-defm : FpUnaryAlias<"fsub{|r}p",  SUBR_FPrST0>;
-defm : FpUnaryAlias<"fsubr",  SUBR_FST0r>;
-defm : FpUnaryAlias<"fsub{r|}p", SUB_FPrST0>;
-defm : FpUnaryAlias<"fmul",   MUL_FST0r>;
-defm : FpUnaryAlias<"fmulp",  MUL_FPrST0>;
-defm : FpUnaryAlias<"fdiv",   DIV_FST0r>;
-defm : FpUnaryAlias<"fdiv{|r}p",  DIVR_FPrST0>;
-defm : FpUnaryAlias<"fdivr",  DIVR_FST0r>;
-defm : FpUnaryAlias<"fdiv{r|}p", DIV_FPrST0>;
+defm : FpUnaryAlias<"fsub",   SUB_FST0r, 0>;
+defm : FpUnaryAlias<"fsub{|r}p",  SUBR_FPrST0, 0>;
+defm : FpUnaryAlias<"fsubr",  SUBR_FST0r, 0>;
+defm : FpUnaryAlias<"fsub{r|}p", SUB_FPrST0, 0>;
+defm : FpUnaryAlias<"fmul",   MUL_FST0r, 0>;
+defm : FpUnaryAlias<"fmulp",  MUL_FPrST0, 0>;
+defm : FpUnaryAlias<"fdiv",   DIV_FST0r, 0>;
+defm : FpUnaryAlias<"fdiv{|r}p",  DIVR_FPrST0, 0>;
+defm : FpUnaryAlias<"fdivr",  DIVR_FST0r, 0>;
+defm : FpUnaryAlias<"fdiv{r|}p", DIV_FPrST0, 0>;
 defm : FpUnaryAlias<"fcomi",   COM_FIr, 0>;
 defm : FpUnaryAlias<"fucomi",  UCOM_FIr, 0>;
-defm : FpUnaryAlias<"fcompi",   COM_FIPr>;
-defm : FpUnaryAlias<"fucompi",  UCOM_FIPr>;
+defm : FpUnaryAlias<"fcompi",   COM_FIPr, 0>;
+defm : FpUnaryAlias<"fucompi",  UCOM_FIPr, 0>;
 
 
-// Handle "f{mulp,addp} st(0), $op" the same as "f{mulp,addp} $op", since they
+// Handle "f{mulp,addp} $op, %st(0)" the same as "f{mulp,addp} $op", since they
 // commute.  We also allow fdiv[r]p/fsubrp even though they don't commute,
 // solely because gas supports it.
-def : InstAlias<"faddp\t{%st(0), $op|$op, st(0)}", (ADD_FPrST0 RST:$op), 0>;
-def : InstAlias<"fmulp\t{%st(0), $op|$op, st(0)}", (MUL_FPrST0 RST:$op)>;
-def : InstAlias<"fsub{|r}p\t{%st(0), $op|$op, st(0)}", (SUBR_FPrST0 RST:$op)>;
-def : InstAlias<"fsub{r|}p\t{%st(0), $op|$op, st(0)}", (SUB_FPrST0 RST:$op)>;
-def : InstAlias<"fdiv{|r}p\t{%st(0), $op|$op, st(0)}", (DIVR_FPrST0 RST:$op)>;
-def : InstAlias<"fdiv{r|}p\t{%st(0), $op|$op, st(0)}", (DIV_FPrST0 RST:$op)>;
+def : InstAlias<"faddp\t{$op, %st|st, $op}", (ADD_FPrST0 RSTi:$op), 0>;
+def : InstAlias<"fmulp\t{$op, %st|st, $op}", (MUL_FPrST0 RSTi:$op), 0>;
+def : InstAlias<"fsub{|r}p\t{$op, %st|st, $op}", (SUBR_FPrST0 RSTi:$op), 0>;
+def : InstAlias<"fsub{r|}p\t{$op, %st|st, $op}", (SUB_FPrST0 RSTi:$op), 0>;
+def : InstAlias<"fdiv{|r}p\t{$op, %st|st, $op}", (DIVR_FPrST0 RSTi:$op), 0>;
+def : InstAlias<"fdiv{r|}p\t{$op, %st|st, $op}", (DIV_FPrST0 RSTi:$op), 0>;
 
 def : InstAlias<"fnstsw"     , (FNSTSW16r), 0>;
 
diff --git a/lib/Target/X86/X86InstrMMX.td b/lib/Target/X86/X86InstrMMX.td
index 8f33571705763539c476beb3f4d4f07fe49b5878..57835b1a256aa86b015c6abd6a4b4725fa58c80f 100644
--- a/lib/Target/X86/X86InstrMMX.td
+++ b/lib/Target/X86/X86InstrMMX.td
@@ -1,9 +1,8 @@
 //===-- X86InstrMMX.td - Describe the MMX Instruction Set --*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -153,7 +152,9 @@ multiclass sse12_cvt_pint_3addr<bits<8> opc, RegisterClass SrcRC,
 // MMX EMMS Instruction
 //===----------------------------------------------------------------------===//
 
-let SchedRW = [WriteEMMS] in
+let SchedRW = [WriteEMMS],
+    Defs = [MM0, MM1, MM2, MM3, MM4, MM5, MM6, MM7,
+            ST0, ST1, ST2, ST3, ST4, ST5, ST6, ST7] in
 def MMX_EMMS  : MMXI<0x77, RawFrm, (outs), (ins), "emms", [(int_x86_mmx_emms)]>;
 
 //===----------------------------------------------------------------------===//
@@ -544,7 +545,7 @@ let Predicates = [HasMMX, HasSSE1] in {
                     "pinsrw\t{$src3, $src2, $dst|$dst, $src2, $src3}",
                     [(set VR64:$dst, (int_x86_mmx_pinsr_w VR64:$src1,
                                       GR32orGR64:$src2, imm:$src3))]>,
-                    Sched<[WriteVecInsert]>;
+                    Sched<[WriteVecInsert, ReadDefault, ReadInt2Fpu]>;
 
   def MMX_PINSRWrm : MMXIi8<0xC4, MRMSrcMem,
                    (outs VR64:$dst),
diff --git a/lib/Target/X86/X86InstrMPX.td b/lib/Target/X86/X86InstrMPX.td
index c1a8cc7c5fbf169e4030bece42a6bc61377dc37c..f7d931510fe20bc65a951fd8cf73830c29ea075a 100644
--- a/lib/Target/X86/X86InstrMPX.td
+++ b/lib/Target/X86/X86InstrMPX.td
@@ -1,9 +1,8 @@
 //===-- X86InstrMPX.td - MPX Instruction Set ---------*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/X86/X86InstrSGX.td b/lib/Target/X86/X86InstrSGX.td
index 488cc4438076a697a95d8a10e952199f8bf0e163..747f5aa86653d6def74f3c2cf77e7cd63522b072 100644
--- a/lib/Target/X86/X86InstrSGX.td
+++ b/lib/Target/X86/X86InstrSGX.td
@@ -1,9 +1,8 @@
 //===-- X86InstrSGX.td - SGX Instruction Set Extension -----*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/X86/X86InstrSSE.td b/lib/Target/X86/X86InstrSSE.td
index 94cd5a611f2ca78cb3c8da3c2c682bd1365d714e..bc2622446bc8769f8769f3078173f5873656bcb5 100644
--- a/lib/Target/X86/X86InstrSSE.td
+++ b/lib/Target/X86/X86InstrSSE.td
@@ -1,9 +1,8 @@
 //===-- X86InstrSSE.td - SSE Instruction Set ---------------*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -847,10 +846,11 @@ let Constraints = "$src1 = $dst" in {
 
 multiclass sse12_cvt_s<bits<8> opc, RegisterClass SrcRC, RegisterClass DstRC,
                      SDNode OpNode, X86MemOperand x86memop, PatFrag ld_frag,
-                     string asm, X86FoldableSchedWrite sched> {
+                     string asm, X86FoldableSchedWrite sched,
+                     SchedRead Int2Fpu = ReadDefault> {
   def rr : SI<opc, MRMSrcReg, (outs DstRC:$dst), (ins SrcRC:$src), asm,
                         [(set DstRC:$dst, (OpNode SrcRC:$src))]>,
-                        Sched<[sched]>;
+                        Sched<[sched, Int2Fpu]>;
   def rm : SI<opc, MRMSrcMem, (outs DstRC:$dst), (ins x86memop:$src), asm,
                         [(set DstRC:$dst, (OpNode (ld_frag addr:$src)))]>,
                         Sched<[sched.Folded]>;
@@ -877,7 +877,7 @@ multiclass sse12_vcvt_avx<bits<8> opc, RegisterClass SrcRC, RegisterClass DstRC,
 let hasSideEffects = 0, Predicates = [UseAVX] in {
   def rr : SI<opc, MRMSrcReg, (outs DstRC:$dst), (ins DstRC:$src1, SrcRC:$src),
               !strconcat(asm,"\t{$src, $src1, $dst|$dst, $src1, $src}"), []>,
-              Sched<[sched]>;
+              Sched<[sched, ReadDefault, ReadInt2Fpu]>;
   let mayLoad = 1 in
   def rm : SI<opc, MRMSrcMem, (outs DstRC:$dst),
               (ins DstRC:$src1, x86memop:$src),
@@ -934,12 +934,12 @@ defm VCVTSI2SD   : sse12_vcvt_avx<0x2A, GR32, FR64, i32mem, "cvtsi2sd{l}",
 defm VCVTSI642SD : sse12_vcvt_avx<0x2A, GR64, FR64, i64mem, "cvtsi2sd{q}",
                                   WriteCvtI2SD>, XD, VEX_4V, VEX_W, VEX_LIG;
 
-let Predicates = [UseAVX] in {
-  def : InstAlias<"vcvtsi2ss\t{$src, $src1, $dst|$dst, $src1, $src}",
-                (VCVTSI2SSrm FR64:$dst, FR64:$src1, i32mem:$src), 0, "att">;
-  def : InstAlias<"vcvtsi2sd\t{$src, $src1, $dst|$dst, $src1, $src}",
-                (VCVTSI2SDrm FR64:$dst, FR64:$src1, i32mem:$src), 0, "att">;
+def : InstAlias<"vcvtsi2ss\t{$src, $src1, $dst|$dst, $src1, $src}",
+              (VCVTSI2SSrm FR64:$dst, FR64:$src1, i32mem:$src), 0, "att">;
+def : InstAlias<"vcvtsi2sd\t{$src, $src1, $dst|$dst, $src1, $src}",
+              (VCVTSI2SDrm FR64:$dst, FR64:$src1, i32mem:$src), 0, "att">;
 
+let Predicates = [UseAVX] in {
   def : Pat<(f32 (sint_to_fp (loadi32 addr:$src))),
             (VCVTSI2SSrm (f32 (IMPLICIT_DEF)), addr:$src)>;
   def : Pat<(f32 (sint_to_fp (loadi64 addr:$src))),
@@ -973,16 +973,16 @@ defm CVTTSD2SI64 : sse12_cvt_s<0x2C, FR64, GR64, fp_to_sint, f64mem, loadf64,
                       WriteCvtSD2I>, XD, REX_W;
 defm CVTSI2SS  : sse12_cvt_s<0x2A, GR32, FR32, sint_to_fp, i32mem, loadi32,
                       "cvtsi2ss{l}\t{$src, $dst|$dst, $src}",
-                      WriteCvtI2SS>, XS;
+                      WriteCvtI2SS, ReadInt2Fpu>, XS;
 defm CVTSI642SS : sse12_cvt_s<0x2A, GR64, FR32, sint_to_fp, i64mem, loadi64,
                       "cvtsi2ss{q}\t{$src, $dst|$dst, $src}",
-                      WriteCvtI2SS>, XS, REX_W;
+                      WriteCvtI2SS, ReadInt2Fpu>, XS, REX_W;
 defm CVTSI2SD  : sse12_cvt_s<0x2A, GR32, FR64, sint_to_fp, i32mem, loadi32,
                       "cvtsi2sd{l}\t{$src, $dst|$dst, $src}",
-                      WriteCvtI2SD>, XD;
+                      WriteCvtI2SD, ReadInt2Fpu>, XD;
 defm CVTSI642SD : sse12_cvt_s<0x2A, GR64, FR64, sint_to_fp, i64mem, loadi64,
                       "cvtsi2sd{q}\t{$src, $dst|$dst, $src}",
-                      WriteCvtI2SD>, XD, REX_W;
+                      WriteCvtI2SD, ReadInt2Fpu>, XD, REX_W;
 
 def : InstAlias<"cvttss2si{l}\t{$src, $dst|$dst, $src}",
                 (CVTTSS2SIrr GR32:$dst, FR32:$src), 0, "att">;
@@ -1032,7 +1032,7 @@ let hasSideEffects = 0 in {
                   !if(Is2Addr,
                       !strconcat(asm, "\t{$src2, $dst|$dst, $src2}"),
                       !strconcat(asm, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")),
-                  []>, Sched<[sched]>;
+                  []>, Sched<[sched, ReadDefault, ReadInt2Fpu]>;
   let mayLoad = 1 in
   def rm_Int : SI<opc, MRMSrcMem, (outs DstRC:$dst),
                   (ins DstRC:$src1, x86memop:$src2),
@@ -1143,7 +1143,7 @@ defm CVTDQ2PS : sse12_cvt_p<0x5B, VR128, i128mem, v4f32, v4i32, memop,
                             SSEPackedSingle, WriteCvtI2PS>,
                             PS, Requires<[UseSSE2]>;
 
-let Predicates = [UseAVX] in {
+// AVX aliases
 def : InstAlias<"vcvtss2si{l}\t{$src, $dst|$dst, $src}",
                 (VCVTSS2SIrr_Int GR32:$dst, VR128:$src), 0, "att">;
 def : InstAlias<"vcvtss2si{l}\t{$src, $dst|$dst, $src}",
@@ -1160,8 +1160,8 @@ def : InstAlias<"vcvtsd2si{q}\t{$src, $dst|$dst, $src}",
                 (VCVTSD2SI64rr_Int GR64:$dst, VR128:$src), 0, "att">;
 def : InstAlias<"vcvtsd2si{q}\t{$src, $dst|$dst, $src}",
                 (VCVTSD2SI64rm_Int GR64:$dst, sdmem:$src), 0, "att">;
-}
 
+// SSE aliases
 def : InstAlias<"cvtss2si{l}\t{$src, $dst|$dst, $src}",
                 (CVTSS2SIrr_Int GR32:$dst, VR128:$src), 0, "att">;
 def : InstAlias<"cvtss2si{l}\t{$src, $dst|$dst, $src}",
@@ -1476,15 +1476,11 @@ def VCVTPD2DQrr  : SDI<0xE6, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src),
                        VEX, Sched<[WriteCvtPD2I]>, VEX_WIG;
 
 // XMM only
-def : InstAlias<"vcvtpd2dqx\t{$src, $dst|$dst, $src}",
-                (VCVTPD2DQrr VR128:$dst, VR128:$src), 0>;
 def VCVTPD2DQrm : SDI<0xE6, MRMSrcMem, (outs VR128:$dst), (ins f128mem:$src),
                       "vcvtpd2dq{x}\t{$src, $dst|$dst, $src}",
                       [(set VR128:$dst,
                         (v4i32 (X86cvtp2Int (loadv2f64 addr:$src))))]>, VEX,
                       Sched<[WriteCvtPD2ILd]>, VEX_WIG;
-def : InstAlias<"vcvtpd2dqx\t{$src, $dst|$dst, $src}",
-                (VCVTPD2DQrm VR128:$dst, f128mem:$src), 0, "intel">;
 
 // YMM only
 def VCVTPD2DQYrr : SDI<0xE6, MRMSrcReg, (outs VR128:$dst), (ins VR256:$src),
@@ -1497,11 +1493,16 @@ def VCVTPD2DQYrm : SDI<0xE6, MRMSrcMem, (outs VR128:$dst), (ins f256mem:$src),
                        [(set VR128:$dst,
                          (v4i32 (X86cvtp2Int (loadv4f64 addr:$src))))]>,
                        VEX, VEX_L, Sched<[WriteCvtPD2IYLd]>, VEX_WIG;
+}
+
+def : InstAlias<"vcvtpd2dqx\t{$src, $dst|$dst, $src}",
+                (VCVTPD2DQrr VR128:$dst, VR128:$src), 0>;
+def : InstAlias<"vcvtpd2dqx\t{$src, $dst|$dst, $src}",
+                (VCVTPD2DQrm VR128:$dst, f128mem:$src), 0, "intel">;
 def : InstAlias<"vcvtpd2dqy\t{$src, $dst|$dst, $src}",
                 (VCVTPD2DQYrr VR128:$dst, VR256:$src), 0>;
 def : InstAlias<"vcvtpd2dqy\t{$src, $dst|$dst, $src}",
                 (VCVTPD2DQYrm VR128:$dst, f256mem:$src), 0, "intel">;
-}
 
 def CVTPD2DQrm  : SDI<0xE6, MRMSrcMem, (outs VR128:$dst), (ins f128mem:$src),
                       "cvtpd2dq\t{$src, $dst|$dst, $src}",
@@ -1569,32 +1570,23 @@ let Predicates = [UseSSE2] in {
             (CVTTPS2DQrm addr:$src)>;
 }
 
-let Predicates = [HasAVX, NoVLX] in
+// The assembler can recognize rr 256-bit instructions by seeing a ymm
+// register, but the same isn't true when using memory operands instead.
+// Provide other assembly rr and rm forms to address this explicitly.
+let Predicates = [HasAVX, NoVLX] in {
+// XMM only
 def VCVTTPD2DQrr : VPDI<0xE6, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src),
                         "cvttpd2dq\t{$src, $dst|$dst, $src}",
                         [(set VR128:$dst,
                           (v4i32 (X86cvttp2si (v2f64 VR128:$src))))]>,
                         VEX, Sched<[WriteCvtPD2I]>, VEX_WIG;
-
-// The assembler can recognize rr 256-bit instructions by seeing a ymm
-// register, but the same isn't true when using memory operands instead.
-// Provide other assembly rr and rm forms to address this explicitly.
-
-// XMM only
-def : InstAlias<"vcvttpd2dqx\t{$src, $dst|$dst, $src}",
-                (VCVTTPD2DQrr VR128:$dst, VR128:$src), 0>;
-
-let Predicates = [HasAVX, NoVLX] in
 def VCVTTPD2DQrm : VPDI<0xE6, MRMSrcMem, (outs VR128:$dst), (ins f128mem:$src),
                         "cvttpd2dq{x}\t{$src, $dst|$dst, $src}",
                         [(set VR128:$dst,
                           (v4i32 (X86cvttp2si (loadv2f64 addr:$src))))]>,
                         VEX, Sched<[WriteCvtPD2ILd]>, VEX_WIG;
-def : InstAlias<"vcvttpd2dqx\t{$src, $dst|$dst, $src}",
-                (VCVTTPD2DQrm VR128:$dst, f128mem:$src), 0, "intel">;
 
 // YMM only
-let Predicates = [HasAVX, NoVLX] in {
 def VCVTTPD2DQYrr : VPDI<0xE6, MRMSrcReg, (outs VR128:$dst), (ins VR256:$src),
                          "cvttpd2dq\t{$src, $dst|$dst, $src}",
                          [(set VR128:$dst,
@@ -1605,7 +1597,12 @@ def VCVTTPD2DQYrm : VPDI<0xE6, MRMSrcMem, (outs VR128:$dst), (ins f256mem:$src),
                          [(set VR128:$dst,
                            (v4i32 (X86cvttp2si (loadv4f64 addr:$src))))]>,
                          VEX, VEX_L, Sched<[WriteCvtPD2IYLd]>, VEX_WIG;
-}
+} // Predicates = [HasAVX, NoVLX]
+
+def : InstAlias<"vcvttpd2dqx\t{$src, $dst|$dst, $src}",
+                (VCVTTPD2DQrr VR128:$dst, VR128:$src), 0>;
+def : InstAlias<"vcvttpd2dqx\t{$src, $dst|$dst, $src}",
+                (VCVTTPD2DQrm VR128:$dst, f128mem:$src), 0, "intel">;
 def : InstAlias<"vcvttpd2dqy\t{$src, $dst|$dst, $src}",
                 (VCVTTPD2DQYrr VR128:$dst, VR256:$src), 0>;
 def : InstAlias<"vcvttpd2dqy\t{$src, $dst|$dst, $src}",
@@ -1618,21 +1615,6 @@ let Predicates = [HasAVX, NoVLX] in {
             (VCVTTPD2DQYrm addr:$src)>;
 }
 
-let Predicates = [HasAVX, NoVLX] in {
-  def : Pat<(X86vzmovl (v2i64 (bitconvert
-                               (v4i32 (X86cvtp2Int (v2f64 VR128:$src)))))),
-            (VCVTPD2DQrr VR128:$src)>;
-  def : Pat<(X86vzmovl (v2i64 (bitconvert
-                               (v4i32 (X86cvtp2Int (loadv2f64 addr:$src)))))),
-            (VCVTPD2DQrm addr:$src)>;
-  def : Pat<(X86vzmovl (v2i64 (bitconvert
-                               (v4i32 (X86cvttp2si (v2f64 VR128:$src)))))),
-            (VCVTTPD2DQrr VR128:$src)>;
-  def : Pat<(X86vzmovl (v2i64 (bitconvert
-                               (v4i32 (X86cvttp2si (loadv2f64 addr:$src)))))),
-            (VCVTTPD2DQrm addr:$src)>;
-} // Predicates = [HasAVX, NoVLX]
-
 def CVTTPD2DQrr : PDI<0xE6, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src),
                       "cvttpd2dq\t{$src, $dst|$dst, $src}",
                       [(set VR128:$dst,
@@ -1644,21 +1626,6 @@ def CVTTPD2DQrm : PDI<0xE6, MRMSrcMem, (outs VR128:$dst),(ins f128mem:$src),
                         (v4i32 (X86cvttp2si (memopv2f64 addr:$src))))]>,
                       Sched<[WriteCvtPD2ILd]>;
 
-let Predicates = [UseSSE2] in {
-  def : Pat<(X86vzmovl (v2i64 (bitconvert
-                               (v4i32 (X86cvtp2Int (v2f64 VR128:$src)))))),
-            (CVTPD2DQrr VR128:$src)>;
-  def : Pat<(X86vzmovl (v2i64 (bitconvert
-                               (v4i32 (X86cvtp2Int (memopv2f64 addr:$src)))))),
-            (CVTPD2DQrm addr:$src)>;
-  def : Pat<(X86vzmovl (v2i64 (bitconvert
-                               (v4i32 (X86cvttp2si (v2f64 VR128:$src)))))),
-            (CVTTPD2DQrr VR128:$src)>;
-  def : Pat<(X86vzmovl (v2i64 (bitconvert
-                               (v4i32 (X86cvttp2si (memopv2f64 addr:$src)))))),
-            (CVTTPD2DQrm addr:$src)>;
-} // Predicates = [UseSSE2]
-
 // Convert packed single to packed double
 let Predicates = [HasAVX, NoVLX] in {
                   // SSE2 instructions without OpSize prefix
@@ -1749,34 +1716,31 @@ let Predicates = [UseSSE2] in {
 // The assembler can recognize rr 256-bit instructions by seeing a ymm
 // register, but the same isn't true when using memory operands instead.
 // Provide other assembly rr and rm forms to address this explicitly.
-let Predicates = [HasAVX, NoVLX] in
+let Predicates = [HasAVX, NoVLX] in {
+// XMM only
 def VCVTPD2PSrr : VPDI<0x5A, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src),
                        "cvtpd2ps\t{$src, $dst|$dst, $src}",
                        [(set VR128:$dst, (X86vfpround (v2f64 VR128:$src)))]>,
                        VEX, Sched<[WriteCvtPD2PS]>, VEX_WIG;
-
-// XMM only
-def : InstAlias<"vcvtpd2psx\t{$src, $dst|$dst, $src}",
-                (VCVTPD2PSrr VR128:$dst, VR128:$src), 0>;
-let Predicates = [HasAVX, NoVLX] in
 def VCVTPD2PSrm : VPDI<0x5A, MRMSrcMem, (outs VR128:$dst), (ins f128mem:$src),
                        "cvtpd2ps{x}\t{$src, $dst|$dst, $src}",
                        [(set VR128:$dst, (X86vfpround (loadv2f64 addr:$src)))]>,
                        VEX, Sched<[WriteCvtPD2PS.Folded]>, VEX_WIG;
-def : InstAlias<"vcvtpd2psx\t{$src, $dst|$dst, $src}",
-                (VCVTPD2PSrm VR128:$dst, f128mem:$src), 0, "intel">;
 
-// YMM only
-let Predicates = [HasAVX, NoVLX] in {
 def VCVTPD2PSYrr : VPDI<0x5A, MRMSrcReg, (outs VR128:$dst), (ins VR256:$src),
                         "cvtpd2ps\t{$src, $dst|$dst, $src}",
-                        [(set VR128:$dst, (fpround VR256:$src))]>,
+                        [(set VR128:$dst, (X86vfpround VR256:$src))]>,
                         VEX, VEX_L, Sched<[WriteCvtPD2PSY]>, VEX_WIG;
 def VCVTPD2PSYrm : VPDI<0x5A, MRMSrcMem, (outs VR128:$dst), (ins f256mem:$src),
                         "cvtpd2ps{y}\t{$src, $dst|$dst, $src}",
-                        [(set VR128:$dst, (fpround (loadv4f64 addr:$src)))]>,
+                        [(set VR128:$dst, (X86vfpround (loadv4f64 addr:$src)))]>,
                         VEX, VEX_L, Sched<[WriteCvtPD2PSY.Folded]>, VEX_WIG;
-}
+} // Predicates = [HasAVX, NoVLX]
+
+def : InstAlias<"vcvtpd2psx\t{$src, $dst|$dst, $src}",
+                (VCVTPD2PSrr VR128:$dst, VR128:$src), 0>;
+def : InstAlias<"vcvtpd2psx\t{$src, $dst|$dst, $src}",
+                (VCVTPD2PSrm VR128:$dst, f128mem:$src), 0, "intel">;
 def : InstAlias<"vcvtpd2psy\t{$src, $dst|$dst, $src}",
                 (VCVTPD2PSYrr VR128:$dst, VR256:$src), 0>;
 def : InstAlias<"vcvtpd2psy\t{$src, $dst|$dst, $src}",
@@ -1791,28 +1755,11 @@ def CVTPD2PSrm : PDI<0x5A, MRMSrcMem, (outs VR128:$dst), (ins f128mem:$src),
                      [(set VR128:$dst, (X86vfpround (memopv2f64 addr:$src)))]>,
                      Sched<[WriteCvtPD2PS.Folded]>;
 
-// AVX 256-bit register conversion intrinsics
-// FIXME: Migrate SSE conversion intrinsics matching to use patterns as below
-// whenever possible to avoid declaring two versions of each one.
-
 let Predicates = [HasAVX, NoVLX] in {
-  // Match fpround and fpextend for 128/256-bit conversions
-  def : Pat<(X86vzmovl (v2f64 (bitconvert
-                               (v4f32 (X86vfpround (v2f64 VR128:$src)))))),
-            (VCVTPD2PSrr VR128:$src)>;
-  def : Pat<(X86vzmovl (v2f64 (bitconvert
-                               (v4f32 (X86vfpround (loadv2f64 addr:$src)))))),
-            (VCVTPD2PSrm addr:$src)>;
-}
-
-let Predicates = [UseSSE2] in {
-  // Match fpround and fpextend for 128 conversions
-  def : Pat<(X86vzmovl (v2f64 (bitconvert
-                               (v4f32 (X86vfpround (v2f64 VR128:$src)))))),
-            (CVTPD2PSrr VR128:$src)>;
-  def : Pat<(X86vzmovl (v2f64 (bitconvert
-                               (v4f32 (X86vfpround (memopv2f64 addr:$src)))))),
-            (CVTPD2PSrm addr:$src)>;
+  def : Pat<(v4f32 (fpround (v4f64 VR256:$src))),
+            (VCVTPD2PSYrr VR256:$src)>;
+  def : Pat<(v4f32 (fpround (loadv4f64 addr:$src))),
+            (VCVTPD2PSYrm addr:$src)>;
 }
 
 //===----------------------------------------------------------------------===//
@@ -4118,7 +4065,7 @@ multiclass sse2_pinsrw<bit Is2Addr = 1> {
            "vpinsrw\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"),
        [(set VR128:$dst,
          (X86pinsrw VR128:$src1, GR32orGR64:$src2, imm:$src3))]>,
-       Sched<[WriteVecInsert]>;
+       Sched<[WriteVecInsert, ReadDefault, ReadInt2Fpu]>;
   def rm : Ii8<0xC4, MRMSrcMem,
                       (outs VR128:$dst), (ins VR128:$src1,
                        i16mem:$src2, u8imm:$src3),
@@ -4669,12 +4616,16 @@ defm MOVDDUP : sse3_replicate_dfp<"movddup", SchedWriteFShuffle>;
 let Predicates = [HasAVX, NoVLX] in {
   def : Pat<(X86Movddup (loadv2f64 addr:$src)),
             (VMOVDDUPrm addr:$src)>, Requires<[HasAVX]>;
+  def : Pat<(X86Movddup (v2f64 (X86vzload addr:$src))),
+            (VMOVDDUPrm addr:$src)>, Requires<[HasAVX]>;
 }
 
 let Predicates = [UseSSE3] in {
   // No need for aligned memory as this only loads 64-bits.
   def : Pat<(X86Movddup (loadv2f64 addr:$src)),
             (MOVDDUPrm addr:$src)>;
+  def : Pat<(X86Movddup (v2f64 (X86vzload addr:$src))),
+            (MOVDDUPrm addr:$src)>;
 }
 
 //===---------------------------------------------------------------------===//
@@ -5569,7 +5520,7 @@ multiclass SS41I_insert8<bits<8> opc, string asm, bit Is2Addr = 1> {
                    "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}")),
       [(set VR128:$dst,
         (X86pinsrb VR128:$src1, GR32orGR64:$src2, imm:$src3))]>,
-      Sched<[WriteVecInsert]>;
+      Sched<[WriteVecInsert, ReadDefault, ReadInt2Fpu]>;
   def rm : SS4AIi8<opc, MRMSrcMem, (outs VR128:$dst),
       (ins VR128:$src1, i8mem:$src2, u8imm:$src3),
       !if(Is2Addr,
@@ -5595,7 +5546,7 @@ multiclass SS41I_insert32<bits<8> opc, string asm, bit Is2Addr = 1> {
                    "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}")),
       [(set VR128:$dst,
         (v4i32 (insertelt VR128:$src1, GR32:$src2, imm:$src3)))]>,
-      Sched<[WriteVecInsert]>;
+      Sched<[WriteVecInsert, ReadDefault, ReadInt2Fpu]>;
   def rm : SS4AIi8<opc, MRMSrcMem, (outs VR128:$dst),
       (ins VR128:$src1, i32mem:$src2, u8imm:$src3),
       !if(Is2Addr,
@@ -5621,7 +5572,7 @@ multiclass SS41I_insert64<bits<8> opc, string asm, bit Is2Addr = 1> {
                    "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}")),
       [(set VR128:$dst,
         (v2i64 (insertelt VR128:$src1, GR64:$src2, imm:$src3)))]>,
-      Sched<[WriteVecInsert]>;
+      Sched<[WriteVecInsert, ReadDefault, ReadInt2Fpu]>;
   def rm : SS4AIi8<opc, MRMSrcMem, (outs VR128:$dst),
       (ins VR128:$src1, i64mem:$src2, u8imm:$src3),
       !if(Is2Addr,
@@ -5643,6 +5594,7 @@ let Constraints = "$src1 = $dst" in
 // vector. The next one matches the intrinsic and could zero arbitrary elements
 // in the target vector.
 multiclass SS41I_insertf32<bits<8> opc, string asm, bit Is2Addr = 1> {
+  let isCommutable = 1 in
   def rr : SS4AIi8<opc, MRMSrcReg, (outs VR128:$dst),
       (ins VR128:$src1, VR128:$src2, u8imm:$src3),
       !if(Is2Addr,
@@ -6578,16 +6530,16 @@ def : Pat<(insert_subvector (v8f32 VR256:$src1), (v4f32 VR128:$src2), (iPTR 0)),
                                        VR128:$src2, sub_xmm), 0xf)>;
 }
 
-/// SS41I_quaternary_int_avx - AVX SSE 4.1 with 4 operators
-multiclass SS41I_quaternary_int_avx<bits<8> opc, string OpcodeStr,
-                                    RegisterClass RC, X86MemOperand x86memop,
-                                    PatFrag mem_frag, Intrinsic IntId,
-                                    X86FoldableSchedWrite sched> {
+/// SS41I_quaternary_vx - AVX SSE 4.1 with 4 operators
+multiclass SS41I_quaternary_avx<bits<8> opc, string OpcodeStr, RegisterClass RC,
+                                X86MemOperand x86memop, ValueType VT,
+                                PatFrag mem_frag, SDNode OpNode,
+                                X86FoldableSchedWrite sched> {
   def rr : Ii8Reg<opc, MRMSrcReg, (outs RC:$dst),
                   (ins RC:$src1, RC:$src2, RC:$src3),
                   !strconcat(OpcodeStr,
                     "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"),
-                  [(set RC:$dst, (IntId RC:$src1, RC:$src2, RC:$src3))],
+                  [(set RC:$dst, (VT (OpNode RC:$src3, RC:$src2, RC:$src1)))],
                   SSEPackedInt>, TAPD, VEX_4V,
                 Sched<[sched]>;
 
@@ -6596,8 +6548,8 @@ multiclass SS41I_quaternary_int_avx<bits<8> opc, string OpcodeStr,
                   !strconcat(OpcodeStr,
                     "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"),
                   [(set RC:$dst,
-                        (IntId RC:$src1, (mem_frag addr:$src2),
-                               RC:$src3))], SSEPackedInt>, TAPD, VEX_4V,
+                        (OpNode RC:$src3, (mem_frag addr:$src2),
+                                RC:$src1))], SSEPackedInt>, TAPD, VEX_4V,
                 Sched<[sched.Folded, sched.ReadAfterFold,
                        // x86memop:$src2
                        ReadDefault, ReadDefault, ReadDefault, ReadDefault,
@@ -6608,66 +6560,45 @@ multiclass SS41I_quaternary_int_avx<bits<8> opc, string OpcodeStr,
 
 let Predicates = [HasAVX] in {
 let ExeDomain = SSEPackedDouble in {
-defm VBLENDVPD  : SS41I_quaternary_int_avx<0x4B, "vblendvpd", VR128, f128mem,
-                                           load, int_x86_sse41_blendvpd,
-                                           SchedWriteFVarBlend.XMM>;
-defm VBLENDVPDY : SS41I_quaternary_int_avx<0x4B, "vblendvpd", VR256, f256mem,
-                                  loadv4f64, int_x86_avx_blendv_pd_256,
-                                  SchedWriteFVarBlend.YMM>, VEX_L;
+defm VBLENDVPD  : SS41I_quaternary_avx<0x4B, "vblendvpd", VR128, f128mem,
+                                       v2f64, loadv2f64, X86Blendv,
+                                       SchedWriteFVarBlend.XMM>;
+defm VBLENDVPDY : SS41I_quaternary_avx<0x4B, "vblendvpd", VR256, f256mem,
+                                       v4f64, loadv4f64, X86Blendv,
+                                       SchedWriteFVarBlend.YMM>, VEX_L;
 } // ExeDomain = SSEPackedDouble
 let ExeDomain = SSEPackedSingle in {
-defm VBLENDVPS  : SS41I_quaternary_int_avx<0x4A, "vblendvps", VR128, f128mem,
-                                           load, int_x86_sse41_blendvps,
-                                           SchedWriteFVarBlend.XMM>;
-defm VBLENDVPSY : SS41I_quaternary_int_avx<0x4A, "vblendvps", VR256, f256mem,
-                                  loadv8f32, int_x86_avx_blendv_ps_256,
-                                  SchedWriteFVarBlend.YMM>, VEX_L;
+defm VBLENDVPS  : SS41I_quaternary_avx<0x4A, "vblendvps", VR128, f128mem,
+                                       v4f32, loadv4f32, X86Blendv,
+                                       SchedWriteFVarBlend.XMM>;
+defm VBLENDVPSY : SS41I_quaternary_avx<0x4A, "vblendvps", VR256, f256mem,
+                                       v8f32, loadv8f32, X86Blendv,
+                                       SchedWriteFVarBlend.YMM>, VEX_L;
 } // ExeDomain = SSEPackedSingle
-defm VPBLENDVB  : SS41I_quaternary_int_avx<0x4C, "vpblendvb", VR128, i128mem,
-                                           load, int_x86_sse41_pblendvb,
-                                           SchedWriteVarBlend.XMM>;
+defm VPBLENDVB  : SS41I_quaternary_avx<0x4C, "vpblendvb", VR128, i128mem,
+                                       v16i8, loadv16i8, X86Blendv,
+                                       SchedWriteVarBlend.XMM>;
 }
 
 let Predicates = [HasAVX2] in {
-defm VPBLENDVBY : SS41I_quaternary_int_avx<0x4C, "vpblendvb", VR256, i256mem,
-                                      load, int_x86_avx2_pblendvb,
-                                      SchedWriteVarBlend.YMM>, VEX_L;
+defm VPBLENDVBY : SS41I_quaternary_avx<0x4C, "vpblendvb", VR256, i256mem,
+                                       v32i8, loadv32i8, X86Blendv,
+                                       SchedWriteVarBlend.YMM>, VEX_L;
 }
 
 let Predicates = [HasAVX] in {
-  def : Pat<(v16i8 (vselect (v16i8 VR128:$mask), (v16i8 VR128:$src1),
-                            (v16i8 VR128:$src2))),
-            (VPBLENDVBrr VR128:$src2, VR128:$src1, VR128:$mask)>;
-  def : Pat<(v4i32 (vselect (v4i32 VR128:$mask), (v4i32 VR128:$src1),
-                            (v4i32 VR128:$src2))),
-            (VBLENDVPSrr VR128:$src2, VR128:$src1, VR128:$mask)>;
-  def : Pat<(v4f32 (vselect (v4i32 VR128:$mask), (v4f32 VR128:$src1),
-                            (v4f32 VR128:$src2))),
+  def : Pat<(v4i32 (X86Blendv (v4i32 VR128:$mask), (v4i32 VR128:$src1),
+                              (v4i32 VR128:$src2))),
             (VBLENDVPSrr VR128:$src2, VR128:$src1, VR128:$mask)>;
-  def : Pat<(v2i64 (vselect (v2i64 VR128:$mask), (v2i64 VR128:$src1),
-                            (v2i64 VR128:$src2))),
+  def : Pat<(v2i64 (X86Blendv (v2i64 VR128:$mask), (v2i64 VR128:$src1),
+                              (v2i64 VR128:$src2))),
             (VBLENDVPDrr VR128:$src2, VR128:$src1, VR128:$mask)>;
-  def : Pat<(v2f64 (vselect (v2i64 VR128:$mask), (v2f64 VR128:$src1),
-                            (v2f64 VR128:$src2))),
-            (VBLENDVPDrr VR128:$src2, VR128:$src1, VR128:$mask)>;
-  def : Pat<(v8i32 (vselect (v8i32 VR256:$mask), (v8i32 VR256:$src1),
-                            (v8i32 VR256:$src2))),
-            (VBLENDVPSYrr VR256:$src2, VR256:$src1, VR256:$mask)>;
-  def : Pat<(v8f32 (vselect (v8i32 VR256:$mask), (v8f32 VR256:$src1),
-                            (v8f32 VR256:$src2))),
+  def : Pat<(v8i32 (X86Blendv (v8i32 VR256:$mask), (v8i32 VR256:$src1),
+                              (v8i32 VR256:$src2))),
             (VBLENDVPSYrr VR256:$src2, VR256:$src1, VR256:$mask)>;
-  def : Pat<(v4i64 (vselect (v4i64 VR256:$mask), (v4i64 VR256:$src1),
-                            (v4i64 VR256:$src2))),
+  def : Pat<(v4i64 (X86Blendv (v4i64 VR256:$mask), (v4i64 VR256:$src1),
+                              (v4i64 VR256:$src2))),
             (VBLENDVPDYrr VR256:$src2, VR256:$src1, VR256:$mask)>;
-  def : Pat<(v4f64 (vselect (v4i64 VR256:$mask), (v4f64 VR256:$src1),
-                            (v4f64 VR256:$src2))),
-            (VBLENDVPDYrr VR256:$src2, VR256:$src1, VR256:$mask)>;
-}
-
-let Predicates = [HasAVX2] in {
-  def : Pat<(v32i8 (vselect (v32i8 VR256:$mask), (v32i8 VR256:$src1),
-                            (v32i8 VR256:$src2))),
-            (VPBLENDVBYrr VR256:$src2, VR256:$src1, VR256:$mask)>;
 }
 
 // Prefer a movss or movsd over a blendps when optimizing for size. these were
@@ -6743,16 +6674,17 @@ let Predicates = [UseSSE41, OptForSpeed] in {
 }
 
 
-/// SS41I_ternary_int - SSE 4.1 ternary operator
+/// SS41I_ternary - SSE 4.1 ternary operator
 let Uses = [XMM0], Constraints = "$src1 = $dst" in {
-  multiclass SS41I_ternary_int<bits<8> opc, string OpcodeStr, PatFrag mem_frag,
-                               X86MemOperand x86memop, Intrinsic IntId,
-                               X86FoldableSchedWrite sched> {
+  multiclass SS41I_ternary<bits<8> opc, string OpcodeStr, ValueType VT,
+                           PatFrag mem_frag, X86MemOperand x86memop,
+                           SDNode OpNode, X86FoldableSchedWrite sched> {
     def rr0 : SS48I<opc, MRMSrcReg, (outs VR128:$dst),
                     (ins VR128:$src1, VR128:$src2),
                     !strconcat(OpcodeStr,
                      "\t{%xmm0, $src2, $dst|$dst, $src2, xmm0}"),
-                    [(set VR128:$dst, (IntId VR128:$src1, VR128:$src2, XMM0))]>,
+                    [(set VR128:$dst,
+                      (VT (OpNode XMM0, VR128:$src2, VR128:$src1)))]>,
                     Sched<[sched]>;
 
     def rm0 : SS48I<opc, MRMSrcMem, (outs VR128:$dst),
@@ -6760,20 +6692,19 @@ let Uses = [XMM0], Constraints = "$src1 = $dst" in {
                     !strconcat(OpcodeStr,
                      "\t{%xmm0, $src2, $dst|$dst, $src2, xmm0}"),
                     [(set VR128:$dst,
-                      (IntId VR128:$src1,
-                       (mem_frag addr:$src2), XMM0))]>,
+                      (OpNode XMM0, (mem_frag addr:$src2), VR128:$src1))]>,
                     Sched<[sched.Folded, sched.ReadAfterFold]>;
   }
 }
 
 let ExeDomain = SSEPackedDouble in
-defm BLENDVPD : SS41I_ternary_int<0x15, "blendvpd", memop, f128mem,
-                                  int_x86_sse41_blendvpd, SchedWriteFVarBlend.XMM>;
+defm BLENDVPD : SS41I_ternary<0x15, "blendvpd", v2f64, memopv2f64, f128mem,
+                              X86Blendv, SchedWriteFVarBlend.XMM>;
 let ExeDomain = SSEPackedSingle in
-defm BLENDVPS : SS41I_ternary_int<0x14, "blendvps", memop, f128mem,
-                                  int_x86_sse41_blendvps, SchedWriteFVarBlend.XMM>;
-defm PBLENDVB : SS41I_ternary_int<0x10, "pblendvb", memop, i128mem,
-                                  int_x86_sse41_pblendvb, SchedWriteVarBlend.XMM>;
+defm BLENDVPS : SS41I_ternary<0x14, "blendvps", v4f32, memopv4f32, f128mem,
+                              X86Blendv, SchedWriteFVarBlend.XMM>;
+defm PBLENDVB : SS41I_ternary<0x10, "pblendvb", v16i8, memopv16i8, i128mem,
+                              X86Blendv, SchedWriteVarBlend.XMM>;
 
 // Aliases with the implicit xmm0 argument
 def : InstAlias<"blendvpd\t{$src2, $dst|$dst, $src2}",
@@ -6790,20 +6721,11 @@ def : InstAlias<"pblendvb\t{$src2, $dst|$dst, $src2}",
                 (PBLENDVBrm0 VR128:$dst, i128mem:$src2), 0>;
 
 let Predicates = [UseSSE41] in {
-  def : Pat<(v16i8 (vselect (v16i8 XMM0), (v16i8 VR128:$src1),
-                            (v16i8 VR128:$src2))),
-            (PBLENDVBrr0 VR128:$src2, VR128:$src1)>;
-  def : Pat<(v4i32 (vselect (v4i32 XMM0), (v4i32 VR128:$src1),
-                            (v4i32 VR128:$src2))),
+  def : Pat<(v4i32 (X86Blendv (v4i32 XMM0), (v4i32 VR128:$src1),
+                              (v4i32 VR128:$src2))),
             (BLENDVPSrr0 VR128:$src2, VR128:$src1)>;
-  def : Pat<(v4f32 (vselect (v4i32 XMM0), (v4f32 VR128:$src1),
-                            (v4f32 VR128:$src2))),
-            (BLENDVPSrr0 VR128:$src2, VR128:$src1)>;
-  def : Pat<(v2i64 (vselect (v2i64 XMM0), (v2i64 VR128:$src1),
-                            (v2i64 VR128:$src2))),
-            (BLENDVPDrr0 VR128:$src2, VR128:$src1)>;
-  def : Pat<(v2f64 (vselect (v2i64 XMM0), (v2f64 VR128:$src1),
-                            (v2f64 VR128:$src2))),
+  def : Pat<(v2i64 (X86Blendv (v2i64 XMM0), (v2i64 VR128:$src1),
+                              (v2i64 VR128:$src2))),
             (BLENDVPDrr0 VR128:$src2, VR128:$src1)>;
 }
 
@@ -8034,6 +7956,8 @@ let Predicates = [HasAVX, NoVLX] in {
             (VMOVDDUPrr VR128:$src)>;
   def : Pat<(v2f64 (X86VBroadcast (loadv2f64 addr:$src))),
             (VMOVDDUPrm addr:$src)>;
+  def : Pat<(v2f64 (X86VBroadcast (v2f64 (X86vzload addr:$src)))),
+            (VMOVDDUPrm addr:$src)>;
 }
 
 let Predicates = [HasAVX1Only] in {
@@ -8312,7 +8236,7 @@ def : Pat<(v32i8 (X86SubVBroadcast (v16i8 VR128:$src))),
 // Variable Bit Shifts
 //
 multiclass avx2_var_shift<bits<8> opc, string OpcodeStr, SDNode OpNode,
-                          ValueType vt128, ValueType vt256> {
+                          SDNode IntrinNode, ValueType vt128, ValueType vt256> {
   def rr  : AVX28I<opc, MRMSrcReg, (outs VR128:$dst),
              (ins VR128:$src1, VR128:$src2),
              !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
@@ -8341,23 +8265,23 @@ multiclass avx2_var_shift<bits<8> opc, string OpcodeStr, SDNode OpNode,
                        (vt256 (load addr:$src2)))))]>,
              VEX_4V, VEX_L, Sched<[SchedWriteVarVecShift.YMM.Folded,
                                    SchedWriteVarVecShift.YMM.ReadAfterFold]>;
+
+  def : Pat<(vt128 (IntrinNode VR128:$src1, VR128:$src2)),
+            (!cast<Instruction>(NAME#"rr") VR128:$src1, VR128:$src2)>;
+  def : Pat<(vt128 (IntrinNode VR128:$src1, (load addr:$src2))),
+            (!cast<Instruction>(NAME#"rm") VR128:$src1, addr:$src2)>;
+  def : Pat<(vt256 (IntrinNode VR256:$src1, VR256:$src2)),
+            (!cast<Instruction>(NAME#"Yrr") VR256:$src1, VR256:$src2)>;
+  def : Pat<(vt256 (IntrinNode VR256:$src1, (load addr:$src2))),
+            (!cast<Instruction>(NAME#"Yrm") VR256:$src1, addr:$src2)>;
 }
 
 let Predicates = [HasAVX2, NoVLX] in {
-  defm VPSLLVD : avx2_var_shift<0x47, "vpsllvd", shl, v4i32, v8i32>;
-  defm VPSLLVQ : avx2_var_shift<0x47, "vpsllvq", shl, v2i64, v4i64>, VEX_W;
-  defm VPSRLVD : avx2_var_shift<0x45, "vpsrlvd", srl, v4i32, v8i32>;
-  defm VPSRLVQ : avx2_var_shift<0x45, "vpsrlvq", srl, v2i64, v4i64>, VEX_W;
-  defm VPSRAVD : avx2_var_shift<0x46, "vpsravd", sra, v4i32, v8i32>;
-
-  def : Pat<(v4i32 (X86vsrav VR128:$src1, VR128:$src2)),
-            (VPSRAVDrr VR128:$src1, VR128:$src2)>;
-  def : Pat<(v4i32 (X86vsrav VR128:$src1, (load addr:$src2))),
-            (VPSRAVDrm VR128:$src1, addr:$src2)>;
-  def : Pat<(v8i32 (X86vsrav VR256:$src1, VR256:$src2)),
-            (VPSRAVDYrr VR256:$src1, VR256:$src2)>;
-  def : Pat<(v8i32 (X86vsrav VR256:$src1, (load addr:$src2))),
-            (VPSRAVDYrm VR256:$src1, addr:$src2)>;
+  defm VPSLLVD : avx2_var_shift<0x47, "vpsllvd", shl, X86vshlv, v4i32, v8i32>;
+  defm VPSLLVQ : avx2_var_shift<0x47, "vpsllvq", shl, X86vshlv, v2i64, v4i64>, VEX_W;
+  defm VPSRLVD : avx2_var_shift<0x45, "vpsrlvd", srl, X86vsrlv, v4i32, v8i32>;
+  defm VPSRLVQ : avx2_var_shift<0x45, "vpsrlvq", srl, X86vsrlv, v2i64, v4i64>, VEX_W;
+  defm VPSRAVD : avx2_var_shift<0x46, "vpsravd", sra, X86vsrav, v4i32, v8i32>;
 }
 
 //===----------------------------------------------------------------------===//
@@ -8387,7 +8311,7 @@ multiclass avx2_gather<bits<8> opc, string OpcodeStr, ValueType VTx,
             VEX, VEX_L, Sched<[WriteLoad]>;
 }
 
-let Predicates = [UseAVX2] in {
+let Predicates = [HasAVX2] in {
   let mayLoad = 1, hasSideEffects = 0, Constraints
     = "@earlyclobber $dst,@earlyclobber $mask_wb, $src1 = $dst, $mask = $mask_wb"
     in {
diff --git a/lib/Target/X86/X86InstrSVM.td b/lib/Target/X86/X86InstrSVM.td
index 2dc6e8b43667a913aba2d2c2d27bb1212dc5720d..82c8e74156b25e937e2da476917acf2a1db492df 100644
--- a/lib/Target/X86/X86InstrSVM.td
+++ b/lib/Target/X86/X86InstrSVM.td
@@ -1,9 +1,8 @@
 //===-- X86InstrSVM.td - SVM Instruction Set Extension -----*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/X86/X86InstrShiftRotate.td b/lib/Target/X86/X86InstrShiftRotate.td
index 7cd63a6dd820624020304a33265096fc7ed9ea4c..21a20c1c62e3c8718635ec2c52cb29876d7e1d9b 100644
--- a/lib/Target/X86/X86InstrShiftRotate.td
+++ b/lib/Target/X86/X86InstrShiftRotate.td
@@ -1,9 +1,8 @@
 //===-- X86InstrShiftRotate.td - Shift and Rotate Instrs ---*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/X86/X86InstrSystem.td b/lib/Target/X86/X86InstrSystem.td
index 35ee00b9e016b0bd95970fbec870d8ae591b8386..013a2c01d2c660445cd0ee61aaa8ac80d0e92c2d 100644
--- a/lib/Target/X86/X86InstrSystem.td
+++ b/lib/Target/X86/X86InstrSystem.td
@@ -1,9 +1,8 @@
 //===-- X86InstrSystem.td - System Instructions ------------*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/X86/X86InstrTSX.td b/lib/Target/X86/X86InstrTSX.td
index 10c6eef786391742d9f4e52df0bad09f0bc96531..fc0da845299f56fab79d7869b681b9b7415471c1 100644
--- a/lib/Target/X86/X86InstrTSX.td
+++ b/lib/Target/X86/X86InstrTSX.td
@@ -1,9 +1,8 @@
 //===-- X86InstrVMX.td - TSX Instruction Set Extension -----*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/X86/X86InstrVMX.td b/lib/Target/X86/X86InstrVMX.td
index 06a438ebfcade5807032a9417049f00023c7debe..37bc4ce2e053701e99d93af3471dde65ae2a2324 100644
--- a/lib/Target/X86/X86InstrVMX.td
+++ b/lib/Target/X86/X86InstrVMX.td
@@ -1,9 +1,8 @@
 //===-- X86InstrVMX.td - VMX Instruction Set Extension -----*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/X86/X86InstrVecCompiler.td b/lib/Target/X86/X86InstrVecCompiler.td
index c417dc99b84dd2bfbdb4414b7d838556e0026fc6..e820072d5e410f58a965b7fd8b17e3d2a53d95ee 100644
--- a/lib/Target/X86/X86InstrVecCompiler.td
+++ b/lib/Target/X86/X86InstrVecCompiler.td
@@ -1,9 +1,8 @@
 //===- X86InstrVecCompiler.td - Vector Compiler Patterns ---*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -398,7 +397,7 @@ let Predicates = [HasBWI, HasDQI] in {
             (COPY_TO_REGCLASS (KMOVBkk VK8:$mask), VK64)>;
 }
 
-let Predicates = [HasBWI, HasVLX] in {
+let Predicates = [HasBWI] in {
   def : Pat<(v32i1 (insert_subvector (v32i1 immAllZerosV),
                                      (v1i1 VK1:$mask), (iPTR 0))),
             (KSHIFTRDri (KSHIFTLDri (COPY_TO_REGCLASS VK1:$mask, VK32),
diff --git a/lib/Target/X86/X86InstrXOP.td b/lib/Target/X86/X86InstrXOP.td
index 9d810a675e3b3eb76873821fc683702eabad5933..bc2e3c6e66a6efe973f422b4676ca4e768d9660c 100644
--- a/lib/Target/X86/X86InstrXOP.td
+++ b/lib/Target/X86/X86InstrXOP.td
@@ -1,9 +1,8 @@
 //===-- X86InstrXOP.td - XOP Instruction Set ---------------*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/X86/X86InstructionSelector.cpp b/lib/Target/X86/X86InstructionSelector.cpp
index c20336387b2db2ac990b4ddc4020b07b9fd697c6..0fdd6e379b4ff461a811e441e645bf7ca78d5a62 100644
--- a/lib/Target/X86/X86InstructionSelector.cpp
+++ b/lib/Target/X86/X86InstructionSelector.cpp
@@ -1,9 +1,8 @@
 //===- X86InstructionSelector.cpp -----------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
@@ -1530,15 +1529,14 @@ bool X86InstructionSelector::selectShift(MachineInstr &I,
 
   const static struct ShiftEntry {
     unsigned SizeInBits;
-    unsigned CReg;
     unsigned OpLSHR;
     unsigned OpASHR;
     unsigned OpSHL;
   } OpTable[] = {
-      {8, X86::CL, X86::SHR8rCL, X86::SAR8rCL, X86::SHL8rCL},      // i8
-      {16, X86::CX, X86::SHR16rCL, X86::SAR16rCL, X86::SHL16rCL},  // i16
-      {32, X86::ECX, X86::SHR32rCL, X86::SAR32rCL, X86::SHL32rCL}, // i32
-      {64, X86::RCX, X86::SHR64rCL, X86::SAR64rCL, X86::SHL64rCL}  // i64
+      {8, X86::SHR8rCL, X86::SAR8rCL, X86::SHL8rCL},     // i8
+      {16, X86::SHR16rCL, X86::SAR16rCL, X86::SHL16rCL}, // i16
+      {32, X86::SHR32rCL, X86::SAR32rCL, X86::SHL32rCL}, // i32
+      {64, X86::SHR64rCL, X86::SAR64rCL, X86::SHL64rCL}  // i64
   };
 
   if (DstRB.getID() != X86::GPRRegBankID)
@@ -1551,7 +1549,6 @@ bool X86InstructionSelector::selectShift(MachineInstr &I,
   if (ShiftEntryIt == std::end(OpTable))
     return false;
 
-  unsigned CReg = ShiftEntryIt->CReg;
   unsigned Opcode = 0;
   switch (I.getOpcode()) {
   case TargetOpcode::G_SHL:
@@ -1570,16 +1567,11 @@ bool X86InstructionSelector::selectShift(MachineInstr &I,
   unsigned Op0Reg = I.getOperand(1).getReg();
   unsigned Op1Reg = I.getOperand(2).getReg();
 
-  BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(TargetOpcode::COPY),
-          ShiftEntryIt->CReg)
-      .addReg(Op1Reg);
+  assert(MRI.getType(Op1Reg).getSizeInBits() == 8);
 
-  // The shift instruction uses X86::CL. If we defined a super-register
-  // of X86::CL, emit a subreg KILL to precisely describe what we're doing here.
-  if (CReg != X86::CL)
-    BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(TargetOpcode::KILL),
-            X86::CL)
-        .addReg(CReg, RegState::Kill);
+  BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(TargetOpcode::COPY),
+          X86::CL)
+    .addReg(Op1Reg);
 
   MachineInstr &ShiftInst =
       *BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode), DstReg)
diff --git a/lib/Target/X86/X86InterleavedAccess.cpp b/lib/Target/X86/X86InterleavedAccess.cpp
index 28940754a2032bcdd28b0ebb618711c24ba0303f..a95eeffd94dd94a733d924a2b50a0531adaf33e8 100644
--- a/lib/Target/X86/X86InterleavedAccess.cpp
+++ b/lib/Target/X86/X86InterleavedAccess.cpp
@@ -1,9 +1,8 @@
 //===- X86InterleavedAccess.cpp -------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -194,7 +193,7 @@ void X86InterleavedAccessGroup::decompose(
 
   // Decompose the load instruction.
   LoadInst *LI = cast<LoadInst>(VecInst);
-  Type *VecBasePtrTy = SubVecTy->getPointerTo(LI->getPointerAddressSpace());
+  Type *VecBaseTy, *VecBasePtrTy;
   Value *VecBasePtr;
   unsigned int NumLoads = NumSubVectors;
   // In the case of stride 3 with a vector of 32 elements load the information
@@ -202,18 +201,22 @@ void X86InterleavedAccessGroup::decompose(
   // [0,1...,VF/2-1,VF/2+VF,VF/2+VF+1,...,2VF-1]
   unsigned VecLength = DL.getTypeSizeInBits(VecWidth);
   if (VecLength == 768 || VecLength == 1536) {
-    Type *VecTran =
-        VectorType::get(Type::getInt8Ty(LI->getContext()), 16)->getPointerTo();
-    VecBasePtr = Builder.CreateBitCast(LI->getPointerOperand(), VecTran);
+    VecBaseTy = VectorType::get(Type::getInt8Ty(LI->getContext()), 16);
+    VecBasePtrTy = VecBaseTy->getPointerTo(LI->getPointerAddressSpace());
+    VecBasePtr = Builder.CreateBitCast(LI->getPointerOperand(), VecBasePtrTy);
     NumLoads = NumSubVectors * (VecLength / 384);
-  } else
+  } else {
+    VecBaseTy = SubVecTy;
+    VecBasePtrTy = VecBaseTy->getPointerTo(LI->getPointerAddressSpace());
     VecBasePtr = Builder.CreateBitCast(LI->getPointerOperand(), VecBasePtrTy);
+  }
   // Generate N loads of T type.
   for (unsigned i = 0; i < NumLoads; i++) {
     // TODO: Support inbounds GEP.
-    Value *NewBasePtr = Builder.CreateGEP(VecBasePtr, Builder.getInt32(i));
+    Value *NewBasePtr =
+        Builder.CreateGEP(VecBaseTy, VecBasePtr, Builder.getInt32(i));
     Instruction *NewLoad =
-        Builder.CreateAlignedLoad(NewBasePtr, LI->getAlignment());
+        Builder.CreateAlignedLoad(VecBaseTy, NewBasePtr, LI->getAlignment());
     DecomposedVectors.push_back(NewLoad);
   }
 }
diff --git a/lib/Target/X86/X86IntrinsicsInfo.h b/lib/Target/X86/X86IntrinsicsInfo.h
index fdcd31c3f9dd16187b38df1200468ee517113da8..9e161b3266cf02d9a3f7eeea343d050307927a6b 100644
--- a/lib/Target/X86/X86IntrinsicsInfo.h
+++ b/lib/Target/X86/X86IntrinsicsInfo.h
@@ -1,9 +1,8 @@
 //===-- X86IntrinsicsInfo.h - X86 Intrinsics ------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -20,18 +19,18 @@
 namespace llvm {
 
 enum IntrinsicType : uint16_t {
-  GATHER, SCATTER, PREFETCH, RDSEED, RDRAND, RDPMC, RDTSC, XTEST, XGETBV, ADX, FPCLASS, FPCLASSS,
+  GATHER, SCATTER, PREFETCH, RDSEED, RDRAND, RDPMC, RDTSC, XTEST, XGETBV, ADX, FPCLASSS,
   INTR_TYPE_1OP, INTR_TYPE_2OP, INTR_TYPE_3OP, INTR_TYPE_4OP,
-  INTR_TYPE_2OP_IMM8, INTR_TYPE_3OP_IMM8,
-  CMP_MASK, CMP_MASK_CC,CMP_MASK_SCALAR_CC, VSHIFT, COMI, COMI_RM,
-  CVTPD2PS, CVTPD2PS_MASK,
+  INTR_TYPE_3OP_IMM8,
+  CMP_MASK_CC,CMP_MASK_SCALAR_CC, VSHIFT, COMI, COMI_RM, BLENDV,
+  CVTPD2PS_MASK,
   INTR_TYPE_1OP_MASK, INTR_TYPE_1OP_MASK_RM,
   INTR_TYPE_2OP_MASK, INTR_TYPE_2OP_MASK_RM,
   INTR_TYPE_3OP_MASK,
-  FMA_OP_MASK, FMA_OP_MASKZ, FMA_OP_SCALAR,
   IFMA_OP, VPERM_2OP, INTR_TYPE_SCALAR_MASK,
   INTR_TYPE_SCALAR_MASK_RM, INTR_TYPE_3OP_SCALAR_MASK,
   COMPRESS_EXPAND_IN_REG,
+  TRUNCATE_TO_REG, CVTPS2PH_MASK, CVTPD2DQ_MASK, CVTQQ2PS_MASK,
   TRUNCATE_TO_MEM_VI8, TRUNCATE_TO_MEM_VI16, TRUNCATE_TO_MEM_VI32,
   FIXUPIMM, FIXUPIMM_MASKZ, FIXUPIMMS,
   FIXUPIMMS_MASKZ, GATHER_AVX2,
@@ -64,47 +63,47 @@ struct IntrinsicData {
  * the alphabetical order.
  */
 static const IntrinsicData IntrinsicsWithChain[] = {
-  X86_INTRINSIC_DATA(avx2_gather_d_d,      GATHER_AVX2, X86::VPGATHERDDrm, 0),
-  X86_INTRINSIC_DATA(avx2_gather_d_d_256,  GATHER_AVX2, X86::VPGATHERDDYrm, 0),
-  X86_INTRINSIC_DATA(avx2_gather_d_pd,     GATHER_AVX2, X86::VGATHERDPDrm, 0),
-  X86_INTRINSIC_DATA(avx2_gather_d_pd_256, GATHER_AVX2, X86::VGATHERDPDYrm, 0),
-  X86_INTRINSIC_DATA(avx2_gather_d_ps,     GATHER_AVX2, X86::VGATHERDPSrm, 0),
-  X86_INTRINSIC_DATA(avx2_gather_d_ps_256, GATHER_AVX2, X86::VGATHERDPSYrm, 0),
-  X86_INTRINSIC_DATA(avx2_gather_d_q,      GATHER_AVX2, X86::VPGATHERDQrm, 0),
-  X86_INTRINSIC_DATA(avx2_gather_d_q_256,  GATHER_AVX2, X86::VPGATHERDQYrm, 0),
-  X86_INTRINSIC_DATA(avx2_gather_q_d,      GATHER_AVX2, X86::VPGATHERQDrm, 0),
-  X86_INTRINSIC_DATA(avx2_gather_q_d_256,  GATHER_AVX2, X86::VPGATHERQDYrm, 0),
-  X86_INTRINSIC_DATA(avx2_gather_q_pd,     GATHER_AVX2, X86::VGATHERQPDrm, 0),
-  X86_INTRINSIC_DATA(avx2_gather_q_pd_256, GATHER_AVX2, X86::VGATHERQPDYrm, 0),
-  X86_INTRINSIC_DATA(avx2_gather_q_ps,     GATHER_AVX2, X86::VGATHERQPSrm, 0),
-  X86_INTRINSIC_DATA(avx2_gather_q_ps_256, GATHER_AVX2, X86::VGATHERQPSYrm, 0),
-  X86_INTRINSIC_DATA(avx2_gather_q_q,      GATHER_AVX2, X86::VPGATHERQQrm, 0),
-  X86_INTRINSIC_DATA(avx2_gather_q_q_256,  GATHER_AVX2, X86::VPGATHERQQYrm, 0),
+  X86_INTRINSIC_DATA(avx2_gather_d_d,      GATHER_AVX2, 0, 0),
+  X86_INTRINSIC_DATA(avx2_gather_d_d_256,  GATHER_AVX2, 0, 0),
+  X86_INTRINSIC_DATA(avx2_gather_d_pd,     GATHER_AVX2, 0, 0),
+  X86_INTRINSIC_DATA(avx2_gather_d_pd_256, GATHER_AVX2, 0, 0),
+  X86_INTRINSIC_DATA(avx2_gather_d_ps,     GATHER_AVX2, 0, 0),
+  X86_INTRINSIC_DATA(avx2_gather_d_ps_256, GATHER_AVX2, 0, 0),
+  X86_INTRINSIC_DATA(avx2_gather_d_q,      GATHER_AVX2, 0, 0),
+  X86_INTRINSIC_DATA(avx2_gather_d_q_256,  GATHER_AVX2, 0, 0),
+  X86_INTRINSIC_DATA(avx2_gather_q_d,      GATHER_AVX2, 0, 0),
+  X86_INTRINSIC_DATA(avx2_gather_q_d_256,  GATHER_AVX2, 0, 0),
+  X86_INTRINSIC_DATA(avx2_gather_q_pd,     GATHER_AVX2, 0, 0),
+  X86_INTRINSIC_DATA(avx2_gather_q_pd_256, GATHER_AVX2, 0, 0),
+  X86_INTRINSIC_DATA(avx2_gather_q_ps,     GATHER_AVX2, 0, 0),
+  X86_INTRINSIC_DATA(avx2_gather_q_ps_256, GATHER_AVX2, 0, 0),
+  X86_INTRINSIC_DATA(avx2_gather_q_q,      GATHER_AVX2, 0, 0),
+  X86_INTRINSIC_DATA(avx2_gather_q_q_256,  GATHER_AVX2, 0, 0),
 
-  X86_INTRINSIC_DATA(avx512_gather_dpd_512, GATHER, X86::VGATHERDPDZrm, 0),
-  X86_INTRINSIC_DATA(avx512_gather_dpi_512, GATHER, X86::VPGATHERDDZrm, 0),
-  X86_INTRINSIC_DATA(avx512_gather_dpq_512, GATHER, X86::VPGATHERDQZrm, 0),
-  X86_INTRINSIC_DATA(avx512_gather_dps_512, GATHER, X86::VGATHERDPSZrm, 0),
-  X86_INTRINSIC_DATA(avx512_gather_qpd_512, GATHER, X86::VGATHERQPDZrm, 0),
-  X86_INTRINSIC_DATA(avx512_gather_qpi_512, GATHER, X86::VPGATHERQDZrm, 0),
-  X86_INTRINSIC_DATA(avx512_gather_qpq_512, GATHER, X86::VPGATHERQQZrm, 0),
-  X86_INTRINSIC_DATA(avx512_gather_qps_512, GATHER, X86::VGATHERQPSZrm, 0),
-  X86_INTRINSIC_DATA(avx512_gather3div2_df, GATHER, X86::VGATHERQPDZ128rm, 0),
-  X86_INTRINSIC_DATA(avx512_gather3div2_di, GATHER, X86::VPGATHERQQZ128rm, 0),
-  X86_INTRINSIC_DATA(avx512_gather3div4_df, GATHER, X86::VGATHERQPDZ256rm, 0),
-  X86_INTRINSIC_DATA(avx512_gather3div4_di, GATHER, X86::VPGATHERQQZ256rm, 0),
-  X86_INTRINSIC_DATA(avx512_gather3div4_sf, GATHER, X86::VGATHERQPSZ128rm, 0),
-  X86_INTRINSIC_DATA(avx512_gather3div4_si, GATHER, X86::VPGATHERQDZ128rm, 0),
-  X86_INTRINSIC_DATA(avx512_gather3div8_sf, GATHER, X86::VGATHERQPSZ256rm, 0),
-  X86_INTRINSIC_DATA(avx512_gather3div8_si, GATHER, X86::VPGATHERQDZ256rm, 0),
-  X86_INTRINSIC_DATA(avx512_gather3siv2_df, GATHER, X86::VGATHERDPDZ128rm, 0),
-  X86_INTRINSIC_DATA(avx512_gather3siv2_di, GATHER, X86::VPGATHERDQZ128rm, 0),
-  X86_INTRINSIC_DATA(avx512_gather3siv4_df, GATHER, X86::VGATHERDPDZ256rm, 0),
-  X86_INTRINSIC_DATA(avx512_gather3siv4_di, GATHER, X86::VPGATHERDQZ256rm, 0),
-  X86_INTRINSIC_DATA(avx512_gather3siv4_sf, GATHER, X86::VGATHERDPSZ128rm, 0),
-  X86_INTRINSIC_DATA(avx512_gather3siv4_si, GATHER, X86::VPGATHERDDZ128rm, 0),
-  X86_INTRINSIC_DATA(avx512_gather3siv8_sf, GATHER, X86::VGATHERDPSZ256rm, 0),
-  X86_INTRINSIC_DATA(avx512_gather3siv8_si, GATHER, X86::VPGATHERDDZ256rm, 0),
+  X86_INTRINSIC_DATA(avx512_gather_dpd_512, GATHER, 0, 0),
+  X86_INTRINSIC_DATA(avx512_gather_dpi_512, GATHER, 0, 0),
+  X86_INTRINSIC_DATA(avx512_gather_dpq_512, GATHER, 0, 0),
+  X86_INTRINSIC_DATA(avx512_gather_dps_512, GATHER, 0, 0),
+  X86_INTRINSIC_DATA(avx512_gather_qpd_512, GATHER, 0, 0),
+  X86_INTRINSIC_DATA(avx512_gather_qpi_512, GATHER, 0, 0),
+  X86_INTRINSIC_DATA(avx512_gather_qpq_512, GATHER, 0, 0),
+  X86_INTRINSIC_DATA(avx512_gather_qps_512, GATHER, 0, 0),
+  X86_INTRINSIC_DATA(avx512_gather3div2_df, GATHER, 0, 0),
+  X86_INTRINSIC_DATA(avx512_gather3div2_di, GATHER, 0, 0),
+  X86_INTRINSIC_DATA(avx512_gather3div4_df, GATHER, 0, 0),
+  X86_INTRINSIC_DATA(avx512_gather3div4_di, GATHER, 0, 0),
+  X86_INTRINSIC_DATA(avx512_gather3div4_sf, GATHER, 0, 0),
+  X86_INTRINSIC_DATA(avx512_gather3div4_si, GATHER, 0, 0),
+  X86_INTRINSIC_DATA(avx512_gather3div8_sf, GATHER, 0, 0),
+  X86_INTRINSIC_DATA(avx512_gather3div8_si, GATHER, 0, 0),
+  X86_INTRINSIC_DATA(avx512_gather3siv2_df, GATHER, 0, 0),
+  X86_INTRINSIC_DATA(avx512_gather3siv2_di, GATHER, 0, 0),
+  X86_INTRINSIC_DATA(avx512_gather3siv4_df, GATHER, 0, 0),
+  X86_INTRINSIC_DATA(avx512_gather3siv4_di, GATHER, 0, 0),
+  X86_INTRINSIC_DATA(avx512_gather3siv4_sf, GATHER, 0, 0),
+  X86_INTRINSIC_DATA(avx512_gather3siv4_si, GATHER, 0, 0),
+  X86_INTRINSIC_DATA(avx512_gather3siv8_sf, GATHER, 0, 0),
+  X86_INTRINSIC_DATA(avx512_gather3siv8_si, GATHER, 0, 0),
 
   X86_INTRINSIC_DATA(avx512_gatherpf_dpd_512, PREFETCH,
                      X86::VGATHERPF0DPDm, X86::VGATHERPF1DPDm),
@@ -115,6 +114,31 @@ static const IntrinsicData IntrinsicsWithChain[] = {
   X86_INTRINSIC_DATA(avx512_gatherpf_qps_512, PREFETCH,
                      X86::VGATHERPF0QPSm, X86::VGATHERPF1QPSm),
 
+  X86_INTRINSIC_DATA(avx512_mask_gather_dpd_512, GATHER, 0, 0),
+  X86_INTRINSIC_DATA(avx512_mask_gather_dpi_512, GATHER, 0, 0),
+  X86_INTRINSIC_DATA(avx512_mask_gather_dpq_512, GATHER, 0, 0),
+  X86_INTRINSIC_DATA(avx512_mask_gather_dps_512, GATHER, 0, 0),
+  X86_INTRINSIC_DATA(avx512_mask_gather_qpd_512, GATHER, 0, 0),
+  X86_INTRINSIC_DATA(avx512_mask_gather_qpi_512, GATHER, 0, 0),
+  X86_INTRINSIC_DATA(avx512_mask_gather_qpq_512, GATHER, 0, 0),
+  X86_INTRINSIC_DATA(avx512_mask_gather_qps_512, GATHER, 0, 0),
+  X86_INTRINSIC_DATA(avx512_mask_gather3div2_df, GATHER, 0, 0),
+  X86_INTRINSIC_DATA(avx512_mask_gather3div2_di, GATHER, 0, 0),
+  X86_INTRINSIC_DATA(avx512_mask_gather3div4_df, GATHER, 0, 0),
+  X86_INTRINSIC_DATA(avx512_mask_gather3div4_di, GATHER, 0, 0),
+  X86_INTRINSIC_DATA(avx512_mask_gather3div4_sf, GATHER, 0, 0),
+  X86_INTRINSIC_DATA(avx512_mask_gather3div4_si, GATHER, 0, 0),
+  X86_INTRINSIC_DATA(avx512_mask_gather3div8_sf, GATHER, 0, 0),
+  X86_INTRINSIC_DATA(avx512_mask_gather3div8_si, GATHER, 0, 0),
+  X86_INTRINSIC_DATA(avx512_mask_gather3siv2_df, GATHER, 0, 0),
+  X86_INTRINSIC_DATA(avx512_mask_gather3siv2_di, GATHER, 0, 0),
+  X86_INTRINSIC_DATA(avx512_mask_gather3siv4_df, GATHER, 0, 0),
+  X86_INTRINSIC_DATA(avx512_mask_gather3siv4_di, GATHER, 0, 0),
+  X86_INTRINSIC_DATA(avx512_mask_gather3siv4_sf, GATHER, 0, 0),
+  X86_INTRINSIC_DATA(avx512_mask_gather3siv4_si, GATHER, 0, 0),
+  X86_INTRINSIC_DATA(avx512_mask_gather3siv8_sf, GATHER, 0, 0),
+  X86_INTRINSIC_DATA(avx512_mask_gather3siv8_si, GATHER, 0, 0),
+
   X86_INTRINSIC_DATA(avx512_mask_pmov_db_mem_128, TRUNCATE_TO_MEM_VI8,
                      X86ISD::VTRUNC, 0),
   X86_INTRINSIC_DATA(avx512_mask_pmov_db_mem_256, TRUNCATE_TO_MEM_VI8,
@@ -224,22 +248,47 @@ static const IntrinsicData IntrinsicsWithChain[] = {
   X86_INTRINSIC_DATA(avx512_mask_pmovus_wb_mem_512, TRUNCATE_TO_MEM_VI8,
                      X86ISD::VTRUNCUS, 0),
 
-  X86_INTRINSIC_DATA(avx512_scatter_dpd_512, SCATTER, X86::VSCATTERDPDZmr, 0),
-  X86_INTRINSIC_DATA(avx512_scatter_dpi_512, SCATTER, X86::VPSCATTERDDZmr, 0),
-  X86_INTRINSIC_DATA(avx512_scatter_dpq_512, SCATTER, X86::VPSCATTERDQZmr, 0),
-  X86_INTRINSIC_DATA(avx512_scatter_dps_512, SCATTER, X86::VSCATTERDPSZmr, 0),
-  X86_INTRINSIC_DATA(avx512_scatter_qpd_512, SCATTER, X86::VSCATTERQPDZmr, 0),
-  X86_INTRINSIC_DATA(avx512_scatter_qpi_512, SCATTER, X86::VPSCATTERQDZmr, 0),
-  X86_INTRINSIC_DATA(avx512_scatter_qpq_512, SCATTER, X86::VPSCATTERQQZmr, 0),
-  X86_INTRINSIC_DATA(avx512_scatter_qps_512, SCATTER, X86::VSCATTERQPSZmr, 0),
-  X86_INTRINSIC_DATA(avx512_scatterdiv2_df, SCATTER, X86::VSCATTERQPDZ128mr, 0),
-  X86_INTRINSIC_DATA(avx512_scatterdiv2_di, SCATTER, X86::VPSCATTERQQZ128mr, 0),
-  X86_INTRINSIC_DATA(avx512_scatterdiv4_df, SCATTER, X86::VSCATTERQPDZ256mr, 0),
-  X86_INTRINSIC_DATA(avx512_scatterdiv4_di, SCATTER, X86::VPSCATTERQQZ256mr, 0),
-  X86_INTRINSIC_DATA(avx512_scatterdiv4_sf, SCATTER, X86::VSCATTERQPSZ128mr, 0),
-  X86_INTRINSIC_DATA(avx512_scatterdiv4_si, SCATTER, X86::VPSCATTERQDZ128mr, 0),
-  X86_INTRINSIC_DATA(avx512_scatterdiv8_sf, SCATTER, X86::VSCATTERQPSZ256mr, 0),
-  X86_INTRINSIC_DATA(avx512_scatterdiv8_si, SCATTER, X86::VPSCATTERQDZ256mr, 0),
+  X86_INTRINSIC_DATA(avx512_mask_scatter_dpd_512, SCATTER, 0, 0),
+  X86_INTRINSIC_DATA(avx512_mask_scatter_dpi_512, SCATTER, 0, 0),
+  X86_INTRINSIC_DATA(avx512_mask_scatter_dpq_512, SCATTER, 0, 0),
+  X86_INTRINSIC_DATA(avx512_mask_scatter_dps_512, SCATTER, 0, 0),
+  X86_INTRINSIC_DATA(avx512_mask_scatter_qpd_512, SCATTER, 0, 0),
+  X86_INTRINSIC_DATA(avx512_mask_scatter_qpi_512, SCATTER, 0, 0),
+  X86_INTRINSIC_DATA(avx512_mask_scatter_qpq_512, SCATTER, 0, 0),
+  X86_INTRINSIC_DATA(avx512_mask_scatter_qps_512, SCATTER, 0, 0),
+  X86_INTRINSIC_DATA(avx512_mask_scatterdiv2_df, SCATTER, 0, 0),
+  X86_INTRINSIC_DATA(avx512_mask_scatterdiv2_di, SCATTER, 0, 0),
+  X86_INTRINSIC_DATA(avx512_mask_scatterdiv4_df, SCATTER, 0, 0),
+  X86_INTRINSIC_DATA(avx512_mask_scatterdiv4_di, SCATTER, 0, 0),
+  X86_INTRINSIC_DATA(avx512_mask_scatterdiv4_sf, SCATTER, 0, 0),
+  X86_INTRINSIC_DATA(avx512_mask_scatterdiv4_si, SCATTER, 0, 0),
+  X86_INTRINSIC_DATA(avx512_mask_scatterdiv8_sf, SCATTER, 0, 0),
+  X86_INTRINSIC_DATA(avx512_mask_scatterdiv8_si, SCATTER, 0, 0),
+  X86_INTRINSIC_DATA(avx512_mask_scattersiv2_df, SCATTER, 0, 0),
+  X86_INTRINSIC_DATA(avx512_mask_scattersiv2_di, SCATTER, 0, 0),
+  X86_INTRINSIC_DATA(avx512_mask_scattersiv4_df, SCATTER, 0, 0),
+  X86_INTRINSIC_DATA(avx512_mask_scattersiv4_di, SCATTER, 0, 0),
+  X86_INTRINSIC_DATA(avx512_mask_scattersiv4_sf, SCATTER, 0, 0),
+  X86_INTRINSIC_DATA(avx512_mask_scattersiv4_si, SCATTER, 0, 0),
+  X86_INTRINSIC_DATA(avx512_mask_scattersiv8_sf, SCATTER, 0, 0),
+  X86_INTRINSIC_DATA(avx512_mask_scattersiv8_si, SCATTER, 0, 0),
+
+  X86_INTRINSIC_DATA(avx512_scatter_dpd_512, SCATTER, 0, 0),
+  X86_INTRINSIC_DATA(avx512_scatter_dpi_512, SCATTER, 0, 0),
+  X86_INTRINSIC_DATA(avx512_scatter_dpq_512, SCATTER, 0, 0),
+  X86_INTRINSIC_DATA(avx512_scatter_dps_512, SCATTER, 0, 0),
+  X86_INTRINSIC_DATA(avx512_scatter_qpd_512, SCATTER, 0, 0),
+  X86_INTRINSIC_DATA(avx512_scatter_qpi_512, SCATTER, 0, 0),
+  X86_INTRINSIC_DATA(avx512_scatter_qpq_512, SCATTER, 0, 0),
+  X86_INTRINSIC_DATA(avx512_scatter_qps_512, SCATTER, 0, 0),
+  X86_INTRINSIC_DATA(avx512_scatterdiv2_df, SCATTER, 0, 0),
+  X86_INTRINSIC_DATA(avx512_scatterdiv2_di, SCATTER, 0, 0),
+  X86_INTRINSIC_DATA(avx512_scatterdiv4_df, SCATTER, 0, 0),
+  X86_INTRINSIC_DATA(avx512_scatterdiv4_di, SCATTER, 0, 0),
+  X86_INTRINSIC_DATA(avx512_scatterdiv4_sf, SCATTER, 0, 0),
+  X86_INTRINSIC_DATA(avx512_scatterdiv4_si, SCATTER, 0, 0),
+  X86_INTRINSIC_DATA(avx512_scatterdiv8_sf, SCATTER, 0, 0),
+  X86_INTRINSIC_DATA(avx512_scatterdiv8_si, SCATTER, 0, 0),
   X86_INTRINSIC_DATA(avx512_scatterpf_dpd_512, PREFETCH, X86::VSCATTERPF0DPDm,
                      X86::VSCATTERPF1DPDm),
   X86_INTRINSIC_DATA(avx512_scatterpf_dps_512, PREFETCH, X86::VSCATTERPF0DPSm,
@@ -248,14 +297,14 @@ static const IntrinsicData IntrinsicsWithChain[] = {
                      X86::VSCATTERPF1QPDm),
   X86_INTRINSIC_DATA(avx512_scatterpf_qps_512, PREFETCH, X86::VSCATTERPF0QPSm,
                      X86::VSCATTERPF1QPSm),
-  X86_INTRINSIC_DATA(avx512_scattersiv2_df, SCATTER, X86::VSCATTERDPDZ128mr, 0),
-  X86_INTRINSIC_DATA(avx512_scattersiv2_di, SCATTER, X86::VPSCATTERDQZ128mr, 0),
-  X86_INTRINSIC_DATA(avx512_scattersiv4_df, SCATTER, X86::VSCATTERDPDZ256mr, 0),
-  X86_INTRINSIC_DATA(avx512_scattersiv4_di, SCATTER, X86::VPSCATTERDQZ256mr, 0),
-  X86_INTRINSIC_DATA(avx512_scattersiv4_sf, SCATTER, X86::VSCATTERDPSZ128mr, 0),
-  X86_INTRINSIC_DATA(avx512_scattersiv4_si, SCATTER, X86::VPSCATTERDDZ128mr, 0),
-  X86_INTRINSIC_DATA(avx512_scattersiv8_sf, SCATTER, X86::VSCATTERDPSZ256mr, 0),
-  X86_INTRINSIC_DATA(avx512_scattersiv8_si, SCATTER, X86::VPSCATTERDDZ256mr, 0),
+  X86_INTRINSIC_DATA(avx512_scattersiv2_df, SCATTER, 0, 0),
+  X86_INTRINSIC_DATA(avx512_scattersiv2_di, SCATTER, 0, 0),
+  X86_INTRINSIC_DATA(avx512_scattersiv4_df, SCATTER, 0, 0),
+  X86_INTRINSIC_DATA(avx512_scattersiv4_di, SCATTER, 0, 0),
+  X86_INTRINSIC_DATA(avx512_scattersiv4_sf, SCATTER, 0, 0),
+  X86_INTRINSIC_DATA(avx512_scattersiv4_si, SCATTER, 0, 0),
+  X86_INTRINSIC_DATA(avx512_scattersiv8_sf, SCATTER, 0, 0),
+  X86_INTRINSIC_DATA(avx512_scattersiv8_si, SCATTER, 0, 0),
   X86_INTRINSIC_DATA(rdpmc,     RDPMC,  X86ISD::RDPMC_DAG, 0),
   X86_INTRINSIC_DATA(rdrand_16, RDRAND, X86ISD::RDRAND, 0),
   X86_INTRINSIC_DATA(rdrand_32, RDRAND, X86ISD::RDRAND, 0),
@@ -290,9 +339,11 @@ static const IntrinsicData  IntrinsicsWithoutChain[] = {
   X86_INTRINSIC_DATA(addcarry_64,       ADX, X86ISD::ADC, X86ISD::ADD),
   X86_INTRINSIC_DATA(avx_addsub_pd_256, INTR_TYPE_2OP, X86ISD::ADDSUB, 0),
   X86_INTRINSIC_DATA(avx_addsub_ps_256, INTR_TYPE_2OP, X86ISD::ADDSUB, 0),
+  X86_INTRINSIC_DATA(avx_blendv_pd_256, BLENDV, X86ISD::BLENDV, 0),
+  X86_INTRINSIC_DATA(avx_blendv_ps_256, BLENDV, X86ISD::BLENDV, 0),
   X86_INTRINSIC_DATA(avx_cmp_pd_256,    INTR_TYPE_3OP, X86ISD::CMPP, 0),
   X86_INTRINSIC_DATA(avx_cmp_ps_256,    INTR_TYPE_3OP, X86ISD::CMPP, 0),
-  X86_INTRINSIC_DATA(avx_cvt_pd2_ps_256,CVTPD2PS, ISD::FP_ROUND, 0),
+  X86_INTRINSIC_DATA(avx_cvt_pd2_ps_256,INTR_TYPE_1OP, X86ISD::VFPROUND, 0),
   X86_INTRINSIC_DATA(avx_cvt_pd2dq_256, INTR_TYPE_1OP, X86ISD::CVTP2SI, 0),
   X86_INTRINSIC_DATA(avx_cvt_ps2dq_256, INTR_TYPE_1OP, X86ISD::CVTP2SI, 0),
   X86_INTRINSIC_DATA(avx_cvtt_pd2dq_256,INTR_TYPE_1OP, X86ISD::CVTTP2SI, 0),
@@ -319,8 +370,7 @@ static const IntrinsicData  IntrinsicsWithoutChain[] = {
   X86_INTRINSIC_DATA(avx2_packsswb, INTR_TYPE_2OP, X86ISD::PACKSS, 0),
   X86_INTRINSIC_DATA(avx2_packusdw, INTR_TYPE_2OP, X86ISD::PACKUS, 0),
   X86_INTRINSIC_DATA(avx2_packuswb, INTR_TYPE_2OP, X86ISD::PACKUS, 0),
-  X86_INTRINSIC_DATA(avx2_padds_b, INTR_TYPE_2OP, ISD::SADDSAT, 0),
-  X86_INTRINSIC_DATA(avx2_padds_w, INTR_TYPE_2OP, ISD::SADDSAT, 0),
+  X86_INTRINSIC_DATA(avx2_pblendvb, BLENDV, X86ISD::BLENDV, 0),
   X86_INTRINSIC_DATA(avx2_permd, VPERM_2OP, X86ISD::VPERMV, 0),
   X86_INTRINSIC_DATA(avx2_permps, VPERM_2OP, X86ISD::VPERMV, 0),
   X86_INTRINSIC_DATA(avx2_phadd_d, INTR_TYPE_2OP, X86ISD::HADD, 0),
@@ -341,10 +391,10 @@ static const IntrinsicData  IntrinsicsWithoutChain[] = {
   X86_INTRINSIC_DATA(avx2_pslli_d, VSHIFT, X86ISD::VSHLI, 0),
   X86_INTRINSIC_DATA(avx2_pslli_q, VSHIFT, X86ISD::VSHLI, 0),
   X86_INTRINSIC_DATA(avx2_pslli_w, VSHIFT, X86ISD::VSHLI, 0),
-  X86_INTRINSIC_DATA(avx2_psllv_d, INTR_TYPE_2OP, ISD::SHL, 0),
-  X86_INTRINSIC_DATA(avx2_psllv_d_256, INTR_TYPE_2OP, ISD::SHL, 0),
-  X86_INTRINSIC_DATA(avx2_psllv_q, INTR_TYPE_2OP, ISD::SHL, 0),
-  X86_INTRINSIC_DATA(avx2_psllv_q_256, INTR_TYPE_2OP, ISD::SHL, 0),
+  X86_INTRINSIC_DATA(avx2_psllv_d, INTR_TYPE_2OP, X86ISD::VSHLV, 0),
+  X86_INTRINSIC_DATA(avx2_psllv_d_256, INTR_TYPE_2OP, X86ISD::VSHLV, 0),
+  X86_INTRINSIC_DATA(avx2_psllv_q, INTR_TYPE_2OP, X86ISD::VSHLV, 0),
+  X86_INTRINSIC_DATA(avx2_psllv_q_256, INTR_TYPE_2OP, X86ISD::VSHLV, 0),
   X86_INTRINSIC_DATA(avx2_psra_d, INTR_TYPE_2OP, X86ISD::VSRA, 0),
   X86_INTRINSIC_DATA(avx2_psra_w, INTR_TYPE_2OP, X86ISD::VSRA, 0),
   X86_INTRINSIC_DATA(avx2_psrai_d, VSHIFT, X86ISD::VSRAI, 0),
@@ -357,12 +407,10 @@ static const IntrinsicData  IntrinsicsWithoutChain[] = {
   X86_INTRINSIC_DATA(avx2_psrli_d, VSHIFT, X86ISD::VSRLI, 0),
   X86_INTRINSIC_DATA(avx2_psrli_q, VSHIFT, X86ISD::VSRLI, 0),
   X86_INTRINSIC_DATA(avx2_psrli_w, VSHIFT, X86ISD::VSRLI, 0),
-  X86_INTRINSIC_DATA(avx2_psrlv_d, INTR_TYPE_2OP, ISD::SRL, 0),
-  X86_INTRINSIC_DATA(avx2_psrlv_d_256, INTR_TYPE_2OP, ISD::SRL, 0),
-  X86_INTRINSIC_DATA(avx2_psrlv_q, INTR_TYPE_2OP, ISD::SRL, 0),
-  X86_INTRINSIC_DATA(avx2_psrlv_q_256, INTR_TYPE_2OP, ISD::SRL, 0),
-  X86_INTRINSIC_DATA(avx2_psubs_b, INTR_TYPE_2OP, ISD::SSUBSAT, 0),
-  X86_INTRINSIC_DATA(avx2_psubs_w, INTR_TYPE_2OP, ISD::SSUBSAT, 0),
+  X86_INTRINSIC_DATA(avx2_psrlv_d, INTR_TYPE_2OP, X86ISD::VSRLV, 0),
+  X86_INTRINSIC_DATA(avx2_psrlv_d_256, INTR_TYPE_2OP, X86ISD::VSRLV, 0),
+  X86_INTRINSIC_DATA(avx2_psrlv_q, INTR_TYPE_2OP, X86ISD::VSRLV, 0),
+  X86_INTRINSIC_DATA(avx2_psrlv_q_256, INTR_TYPE_2OP, X86ISD::VSRLV, 0),
   X86_INTRINSIC_DATA(avx512_add_pd_512, INTR_TYPE_2OP, ISD::FADD, X86ISD::FADD_RND),
   X86_INTRINSIC_DATA(avx512_add_ps_512, INTR_TYPE_2OP, ISD::FADD, X86ISD::FADD_RND),
   X86_INTRINSIC_DATA(avx512_cmp_pd_128, CMP_MASK_CC, X86ISD::CMPM, 0),
@@ -371,6 +419,12 @@ static const IntrinsicData  IntrinsicsWithoutChain[] = {
   X86_INTRINSIC_DATA(avx512_cmp_ps_128, CMP_MASK_CC, X86ISD::CMPM, 0),
   X86_INTRINSIC_DATA(avx512_cmp_ps_256, CMP_MASK_CC, X86ISD::CMPM, 0),
   X86_INTRINSIC_DATA(avx512_cmp_ps_512, CMP_MASK_CC, X86ISD::CMPM, X86ISD::CMPM_RND),
+  X86_INTRINSIC_DATA(avx512_conflict_d_128, INTR_TYPE_1OP, X86ISD::CONFLICT, 0),
+  X86_INTRINSIC_DATA(avx512_conflict_d_256, INTR_TYPE_1OP, X86ISD::CONFLICT, 0),
+  X86_INTRINSIC_DATA(avx512_conflict_d_512, INTR_TYPE_1OP, X86ISD::CONFLICT, 0),
+  X86_INTRINSIC_DATA(avx512_conflict_q_128, INTR_TYPE_1OP, X86ISD::CONFLICT, 0),
+  X86_INTRINSIC_DATA(avx512_conflict_q_256, INTR_TYPE_1OP, X86ISD::CONFLICT, 0),
+  X86_INTRINSIC_DATA(avx512_conflict_q_512, INTR_TYPE_1OP, X86ISD::CONFLICT, 0),
   X86_INTRINSIC_DATA(avx512_cvtsi2sd64,  INTR_TYPE_3OP, X86ISD::SCALAR_SINT_TO_FP_RND, 0),
   X86_INTRINSIC_DATA(avx512_cvtsi2ss32,  INTR_TYPE_3OP, X86ISD::SCALAR_SINT_TO_FP_RND, 0),
   X86_INTRINSIC_DATA(avx512_cvtsi2ss64,  INTR_TYPE_3OP, X86ISD::SCALAR_SINT_TO_FP_RND, 0),
@@ -392,12 +446,12 @@ static const IntrinsicData  IntrinsicsWithoutChain[] = {
   X86_INTRINSIC_DATA(avx512_div_ps_512, INTR_TYPE_2OP, ISD::FDIV, X86ISD::FDIV_RND),
   X86_INTRINSIC_DATA(avx512_exp2_pd, INTR_TYPE_1OP_MASK_RM, X86ISD::EXP2, 0),
   X86_INTRINSIC_DATA(avx512_exp2_ps, INTR_TYPE_1OP_MASK_RM, X86ISD::EXP2, 0),
-  X86_INTRINSIC_DATA(avx512_fpclass_pd_128, FPCLASS, X86ISD::VFPCLASS, 0),
-  X86_INTRINSIC_DATA(avx512_fpclass_pd_256, FPCLASS, X86ISD::VFPCLASS, 0),
-  X86_INTRINSIC_DATA(avx512_fpclass_pd_512, FPCLASS, X86ISD::VFPCLASS, 0),
-  X86_INTRINSIC_DATA(avx512_fpclass_ps_128, FPCLASS, X86ISD::VFPCLASS, 0),
-  X86_INTRINSIC_DATA(avx512_fpclass_ps_256, FPCLASS, X86ISD::VFPCLASS, 0),
-  X86_INTRINSIC_DATA(avx512_fpclass_ps_512, FPCLASS, X86ISD::VFPCLASS, 0),
+  X86_INTRINSIC_DATA(avx512_fpclass_pd_128, INTR_TYPE_2OP, X86ISD::VFPCLASS, 0),
+  X86_INTRINSIC_DATA(avx512_fpclass_pd_256, INTR_TYPE_2OP, X86ISD::VFPCLASS, 0),
+  X86_INTRINSIC_DATA(avx512_fpclass_pd_512, INTR_TYPE_2OP, X86ISD::VFPCLASS, 0),
+  X86_INTRINSIC_DATA(avx512_fpclass_ps_128, INTR_TYPE_2OP, X86ISD::VFPCLASS, 0),
+  X86_INTRINSIC_DATA(avx512_fpclass_ps_256, INTR_TYPE_2OP, X86ISD::VFPCLASS, 0),
+  X86_INTRINSIC_DATA(avx512_fpclass_ps_512, INTR_TYPE_2OP, X86ISD::VFPCLASS, 0),
   X86_INTRINSIC_DATA(avx512_kadd_b, INTR_TYPE_2OP, X86ISD::KADD, 0),
   X86_INTRINSIC_DATA(avx512_kadd_d, INTR_TYPE_2OP, X86ISD::KADD, 0),
   X86_INTRINSIC_DATA(avx512_kadd_q, INTR_TYPE_2OP, X86ISD::KADD, 0),
@@ -411,72 +465,24 @@ static const IntrinsicData  IntrinsicsWithoutChain[] = {
   X86_INTRINSIC_DATA(avx512_mask_cmp_ss,     CMP_MASK_SCALAR_CC,
                      X86ISD::FSETCCM, X86ISD::FSETCCM_RND),
 
-  X86_INTRINSIC_DATA(avx512_mask_compress_b_128,  COMPRESS_EXPAND_IN_REG,
-                     X86ISD::COMPRESS, 0),
-  X86_INTRINSIC_DATA(avx512_mask_compress_b_256,  COMPRESS_EXPAND_IN_REG,
-                     X86ISD::COMPRESS, 0),
-  X86_INTRINSIC_DATA(avx512_mask_compress_b_512,  COMPRESS_EXPAND_IN_REG,
-                     X86ISD::COMPRESS, 0),
-  X86_INTRINSIC_DATA(avx512_mask_compress_d_128,  COMPRESS_EXPAND_IN_REG,
-                     X86ISD::COMPRESS, 0),
-  X86_INTRINSIC_DATA(avx512_mask_compress_d_256,  COMPRESS_EXPAND_IN_REG,
-                     X86ISD::COMPRESS, 0),
-  X86_INTRINSIC_DATA(avx512_mask_compress_d_512,  COMPRESS_EXPAND_IN_REG,
-                     X86ISD::COMPRESS, 0),
-  X86_INTRINSIC_DATA(avx512_mask_compress_pd_128, COMPRESS_EXPAND_IN_REG,
-                     X86ISD::COMPRESS, 0),
-  X86_INTRINSIC_DATA(avx512_mask_compress_pd_256, COMPRESS_EXPAND_IN_REG,
-                     X86ISD::COMPRESS, 0),
-  X86_INTRINSIC_DATA(avx512_mask_compress_pd_512, COMPRESS_EXPAND_IN_REG,
+  X86_INTRINSIC_DATA(avx512_mask_compress,        COMPRESS_EXPAND_IN_REG,
                      X86ISD::COMPRESS, 0),
-  X86_INTRINSIC_DATA(avx512_mask_compress_ps_128, COMPRESS_EXPAND_IN_REG,
-                     X86ISD::COMPRESS, 0),
-  X86_INTRINSIC_DATA(avx512_mask_compress_ps_256, COMPRESS_EXPAND_IN_REG,
-                     X86ISD::COMPRESS, 0),
-  X86_INTRINSIC_DATA(avx512_mask_compress_ps_512, COMPRESS_EXPAND_IN_REG,
-                     X86ISD::COMPRESS, 0),
-  X86_INTRINSIC_DATA(avx512_mask_compress_q_128,  COMPRESS_EXPAND_IN_REG,
-                     X86ISD::COMPRESS, 0),
-  X86_INTRINSIC_DATA(avx512_mask_compress_q_256,  COMPRESS_EXPAND_IN_REG,
-                     X86ISD::COMPRESS, 0),
-  X86_INTRINSIC_DATA(avx512_mask_compress_q_512,  COMPRESS_EXPAND_IN_REG,
-                     X86ISD::COMPRESS, 0),
-  X86_INTRINSIC_DATA(avx512_mask_compress_w_128,  COMPRESS_EXPAND_IN_REG,
-                     X86ISD::COMPRESS, 0),
-  X86_INTRINSIC_DATA(avx512_mask_compress_w_256,  COMPRESS_EXPAND_IN_REG,
-                     X86ISD::COMPRESS, 0),
-  X86_INTRINSIC_DATA(avx512_mask_compress_w_512,  COMPRESS_EXPAND_IN_REG,
-                     X86ISD::COMPRESS, 0),
-  X86_INTRINSIC_DATA(avx512_mask_conflict_d_128, INTR_TYPE_1OP_MASK,
-                     X86ISD::CONFLICT, 0),
-  X86_INTRINSIC_DATA(avx512_mask_conflict_d_256, INTR_TYPE_1OP_MASK,
-                     X86ISD::CONFLICT, 0),
-  X86_INTRINSIC_DATA(avx512_mask_conflict_d_512, INTR_TYPE_1OP_MASK,
-                     X86ISD::CONFLICT, 0),
-  X86_INTRINSIC_DATA(avx512_mask_conflict_q_128, INTR_TYPE_1OP_MASK,
-                     X86ISD::CONFLICT, 0),
-  X86_INTRINSIC_DATA(avx512_mask_conflict_q_256, INTR_TYPE_1OP_MASK,
-                     X86ISD::CONFLICT, 0),
-  X86_INTRINSIC_DATA(avx512_mask_conflict_q_512, INTR_TYPE_1OP_MASK,
-                     X86ISD::CONFLICT, 0),
-  X86_INTRINSIC_DATA(avx512_mask_cvtdq2ps_512, INTR_TYPE_1OP_MASK,
-                     ISD::SINT_TO_FP, X86ISD::SINT_TO_FP_RND), //er
-  X86_INTRINSIC_DATA(avx512_mask_cvtpd2dq_128, INTR_TYPE_1OP_MASK,
-                     X86ISD::CVTP2SI, 0),
+  X86_INTRINSIC_DATA(avx512_mask_cvtpd2dq_128, CVTPD2DQ_MASK,
+                     X86ISD::CVTP2SI, X86ISD::MCVTP2SI),
   X86_INTRINSIC_DATA(avx512_mask_cvtpd2dq_512, INTR_TYPE_1OP_MASK,
-                    X86ISD::CVTP2SI, X86ISD::CVTP2SI_RND),
-  X86_INTRINSIC_DATA(avx512_mask_cvtpd2ps,     INTR_TYPE_1OP_MASK,
-                    X86ISD::VFPROUND, 0),
-  X86_INTRINSIC_DATA(avx512_mask_cvtpd2ps_512, CVTPD2PS_MASK,
-                     ISD::FP_ROUND, X86ISD::VFPROUND_RND),
+                     X86ISD::CVTP2SI, X86ISD::CVTP2SI_RND),
+  X86_INTRINSIC_DATA(avx512_mask_cvtpd2ps,     CVTPD2PS_MASK,
+                     X86ISD::VFPROUND, X86ISD::VMFPROUND),
+  X86_INTRINSIC_DATA(avx512_mask_cvtpd2ps_512, INTR_TYPE_1OP_MASK,
+                     X86ISD::VFPROUND, X86ISD::VFPROUND_RND),
   X86_INTRINSIC_DATA(avx512_mask_cvtpd2qq_128, INTR_TYPE_1OP_MASK,
                      X86ISD::CVTP2SI, 0),
   X86_INTRINSIC_DATA(avx512_mask_cvtpd2qq_256, INTR_TYPE_1OP_MASK,
                      X86ISD::CVTP2SI, 0),
   X86_INTRINSIC_DATA(avx512_mask_cvtpd2qq_512, INTR_TYPE_1OP_MASK,
                      X86ISD::CVTP2SI, X86ISD::CVTP2SI_RND),
-  X86_INTRINSIC_DATA(avx512_mask_cvtpd2udq_128, INTR_TYPE_1OP_MASK,
-                     X86ISD::CVTP2UI, 0),
+  X86_INTRINSIC_DATA(avx512_mask_cvtpd2udq_128, CVTPD2DQ_MASK,
+                     X86ISD::CVTP2UI, X86ISD::MCVTP2UI),
   X86_INTRINSIC_DATA(avx512_mask_cvtpd2udq_256, INTR_TYPE_1OP_MASK,
                      X86ISD::CVTP2UI, 0),
   X86_INTRINSIC_DATA(avx512_mask_cvtpd2udq_512, INTR_TYPE_1OP_MASK,
@@ -513,20 +519,14 @@ static const IntrinsicData  IntrinsicsWithoutChain[] = {
                      X86ISD::CVTP2UI, 0),
   X86_INTRINSIC_DATA(avx512_mask_cvtps2uqq_512, INTR_TYPE_1OP_MASK,
                      X86ISD::CVTP2UI, X86ISD::CVTP2UI_RND),
-  X86_INTRINSIC_DATA(avx512_mask_cvtqq2pd_512, INTR_TYPE_1OP_MASK,
-                     ISD::SINT_TO_FP, X86ISD::SINT_TO_FP_RND),
-  X86_INTRINSIC_DATA(avx512_mask_cvtqq2ps_128, INTR_TYPE_1OP_MASK,
-                     X86ISD::CVTSI2P, 0),
-  X86_INTRINSIC_DATA(avx512_mask_cvtqq2ps_256, INTR_TYPE_1OP_MASK,
-                     ISD::SINT_TO_FP, 0),
-  X86_INTRINSIC_DATA(avx512_mask_cvtqq2ps_512, INTR_TYPE_1OP_MASK,
-                     ISD::SINT_TO_FP, X86ISD::SINT_TO_FP_RND),
+  X86_INTRINSIC_DATA(avx512_mask_cvtqq2ps_128, CVTQQ2PS_MASK,
+                     X86ISD::CVTSI2P, X86ISD::MCVTSI2P),
   X86_INTRINSIC_DATA(avx512_mask_cvtsd2ss_round, INTR_TYPE_SCALAR_MASK_RM,
                      X86ISD::VFPROUNDS_RND, 0),
   X86_INTRINSIC_DATA(avx512_mask_cvtss2sd_round, INTR_TYPE_SCALAR_MASK_RM,
                      X86ISD::VFPEXTS_RND, 0),
-  X86_INTRINSIC_DATA(avx512_mask_cvttpd2dq_128, INTR_TYPE_1OP_MASK,
-                     X86ISD::CVTTP2SI, 0),
+  X86_INTRINSIC_DATA(avx512_mask_cvttpd2dq_128, CVTPD2DQ_MASK,
+                     X86ISD::CVTTP2SI, X86ISD::MCVTTP2SI),
   X86_INTRINSIC_DATA(avx512_mask_cvttpd2dq_512, INTR_TYPE_1OP_MASK,
                      X86ISD::CVTTP2SI, X86ISD::CVTTP2SI_RND),
   X86_INTRINSIC_DATA(avx512_mask_cvttpd2qq_128, INTR_TYPE_1OP_MASK,
@@ -535,8 +535,8 @@ static const IntrinsicData  IntrinsicsWithoutChain[] = {
                      X86ISD::CVTTP2SI, 0),
   X86_INTRINSIC_DATA(avx512_mask_cvttpd2qq_512, INTR_TYPE_1OP_MASK,
                      X86ISD::CVTTP2SI, X86ISD::CVTTP2SI_RND),
-  X86_INTRINSIC_DATA(avx512_mask_cvttpd2udq_128, INTR_TYPE_1OP_MASK,
-                     X86ISD::CVTTP2UI, 0),
+  X86_INTRINSIC_DATA(avx512_mask_cvttpd2udq_128, CVTPD2DQ_MASK,
+                     X86ISD::CVTTP2UI, X86ISD::MCVTTP2UI),
   X86_INTRINSIC_DATA(avx512_mask_cvttpd2udq_256, INTR_TYPE_1OP_MASK,
                      X86ISD::CVTTP2UI, 0),
   X86_INTRINSIC_DATA(avx512_mask_cvttpd2udq_512, INTR_TYPE_1OP_MASK,
@@ -567,55 +567,13 @@ static const IntrinsicData  IntrinsicsWithoutChain[] = {
                      X86ISD::CVTTP2UI, 0),
   X86_INTRINSIC_DATA(avx512_mask_cvttps2uqq_512, INTR_TYPE_1OP_MASK,
                      X86ISD::CVTTP2UI, X86ISD::CVTTP2UI_RND),
-  X86_INTRINSIC_DATA(avx512_mask_cvtudq2ps_512, INTR_TYPE_1OP_MASK,
-                     ISD::UINT_TO_FP, X86ISD::UINT_TO_FP_RND),
-  X86_INTRINSIC_DATA(avx512_mask_cvtuqq2pd_512, INTR_TYPE_1OP_MASK,
-                     ISD::UINT_TO_FP, X86ISD::UINT_TO_FP_RND),
-  X86_INTRINSIC_DATA(avx512_mask_cvtuqq2ps_128, INTR_TYPE_1OP_MASK,
-                     X86ISD::CVTUI2P, 0),
-  X86_INTRINSIC_DATA(avx512_mask_cvtuqq2ps_256, INTR_TYPE_1OP_MASK,
-                     ISD::UINT_TO_FP, 0),
-  X86_INTRINSIC_DATA(avx512_mask_cvtuqq2ps_512, INTR_TYPE_1OP_MASK,
-                     ISD::UINT_TO_FP, X86ISD::UINT_TO_FP_RND),
+  X86_INTRINSIC_DATA(avx512_mask_cvtuqq2ps_128, CVTQQ2PS_MASK,
+                     X86ISD::CVTUI2P, X86ISD::MCVTUI2P),
   X86_INTRINSIC_DATA(avx512_mask_div_sd_round, INTR_TYPE_SCALAR_MASK_RM,
                      X86ISD::FDIVS_RND, 0),
   X86_INTRINSIC_DATA(avx512_mask_div_ss_round, INTR_TYPE_SCALAR_MASK_RM,
                      X86ISD::FDIVS_RND, 0),
-  X86_INTRINSIC_DATA(avx512_mask_expand_b_128,  COMPRESS_EXPAND_IN_REG,
-                     X86ISD::EXPAND, 0),
-  X86_INTRINSIC_DATA(avx512_mask_expand_b_256,  COMPRESS_EXPAND_IN_REG,
-                     X86ISD::EXPAND, 0),
-  X86_INTRINSIC_DATA(avx512_mask_expand_b_512,  COMPRESS_EXPAND_IN_REG,
-                     X86ISD::EXPAND, 0),
-  X86_INTRINSIC_DATA(avx512_mask_expand_d_128,  COMPRESS_EXPAND_IN_REG,
-                     X86ISD::EXPAND, 0),
-  X86_INTRINSIC_DATA(avx512_mask_expand_d_256,  COMPRESS_EXPAND_IN_REG,
-                     X86ISD::EXPAND, 0),
-  X86_INTRINSIC_DATA(avx512_mask_expand_d_512,  COMPRESS_EXPAND_IN_REG,
-                     X86ISD::EXPAND, 0),
-  X86_INTRINSIC_DATA(avx512_mask_expand_pd_128, COMPRESS_EXPAND_IN_REG,
-                     X86ISD::EXPAND, 0),
-  X86_INTRINSIC_DATA(avx512_mask_expand_pd_256, COMPRESS_EXPAND_IN_REG,
-                     X86ISD::EXPAND, 0),
-  X86_INTRINSIC_DATA(avx512_mask_expand_pd_512, COMPRESS_EXPAND_IN_REG,
-                     X86ISD::EXPAND, 0),
-  X86_INTRINSIC_DATA(avx512_mask_expand_ps_128, COMPRESS_EXPAND_IN_REG,
-                     X86ISD::EXPAND, 0),
-  X86_INTRINSIC_DATA(avx512_mask_expand_ps_256, COMPRESS_EXPAND_IN_REG,
-                     X86ISD::EXPAND, 0),
-  X86_INTRINSIC_DATA(avx512_mask_expand_ps_512, COMPRESS_EXPAND_IN_REG,
-                     X86ISD::EXPAND, 0),
-  X86_INTRINSIC_DATA(avx512_mask_expand_q_128,  COMPRESS_EXPAND_IN_REG,
-                     X86ISD::EXPAND, 0),
-  X86_INTRINSIC_DATA(avx512_mask_expand_q_256,  COMPRESS_EXPAND_IN_REG,
-                     X86ISD::EXPAND, 0),
-  X86_INTRINSIC_DATA(avx512_mask_expand_q_512,  COMPRESS_EXPAND_IN_REG,
-                     X86ISD::EXPAND, 0),
-  X86_INTRINSIC_DATA(avx512_mask_expand_w_128,  COMPRESS_EXPAND_IN_REG,
-                     X86ISD::EXPAND, 0),
-  X86_INTRINSIC_DATA(avx512_mask_expand_w_256,  COMPRESS_EXPAND_IN_REG,
-                     X86ISD::EXPAND, 0),
-  X86_INTRINSIC_DATA(avx512_mask_expand_w_512,  COMPRESS_EXPAND_IN_REG,
+  X86_INTRINSIC_DATA(avx512_mask_expand,        COMPRESS_EXPAND_IN_REG,
                      X86ISD::EXPAND, 0),
   X86_INTRINSIC_DATA(avx512_mask_fixupimm_pd_128, FIXUPIMM, X86ISD::VFIXUPIMM, 0),
   X86_INTRINSIC_DATA(avx512_mask_fixupimm_pd_256, FIXUPIMM, X86ISD::VFIXUPIMM, 0),
@@ -671,120 +629,106 @@ static const IntrinsicData  IntrinsicsWithoutChain[] = {
                      X86ISD::FMULS_RND, 0),
   X86_INTRINSIC_DATA(avx512_mask_mul_ss_round, INTR_TYPE_SCALAR_MASK_RM,
                      X86ISD::FMULS_RND, 0),
-  X86_INTRINSIC_DATA(avx512_mask_pmov_db_128, INTR_TYPE_1OP_MASK,
-                     X86ISD::VTRUNC, 0),
-  X86_INTRINSIC_DATA(avx512_mask_pmov_db_256, INTR_TYPE_1OP_MASK,
-                     X86ISD::VTRUNC, 0),
-  X86_INTRINSIC_DATA(avx512_mask_pmov_db_512, INTR_TYPE_1OP_MASK,
-                     ISD::TRUNCATE, 0),
-  X86_INTRINSIC_DATA(avx512_mask_pmov_dw_128, INTR_TYPE_1OP_MASK,
-                     X86ISD::VTRUNC, 0),
-  X86_INTRINSIC_DATA(avx512_mask_pmov_dw_256, INTR_TYPE_1OP_MASK,
-                     ISD::TRUNCATE, 0),
-  X86_INTRINSIC_DATA(avx512_mask_pmov_dw_512, INTR_TYPE_1OP_MASK,
-                     ISD::TRUNCATE, 0),
-  X86_INTRINSIC_DATA(avx512_mask_pmov_qb_128, INTR_TYPE_1OP_MASK,
-                     X86ISD::VTRUNC, 0),
-  X86_INTRINSIC_DATA(avx512_mask_pmov_qb_256, INTR_TYPE_1OP_MASK,
-                     X86ISD::VTRUNC, 0),
-  X86_INTRINSIC_DATA(avx512_mask_pmov_qb_512, INTR_TYPE_1OP_MASK,
-                     X86ISD::VTRUNC, 0),
-  X86_INTRINSIC_DATA(avx512_mask_pmov_qd_128, INTR_TYPE_1OP_MASK,
-                     X86ISD::VTRUNC, 0),
-  X86_INTRINSIC_DATA(avx512_mask_pmov_qd_256, INTR_TYPE_1OP_MASK,
-                     ISD::TRUNCATE, 0),
-  X86_INTRINSIC_DATA(avx512_mask_pmov_qd_512, INTR_TYPE_1OP_MASK,
-                     ISD::TRUNCATE, 0),
-  X86_INTRINSIC_DATA(avx512_mask_pmov_qw_128, INTR_TYPE_1OP_MASK,
-                     X86ISD::VTRUNC, 0),
-  X86_INTRINSIC_DATA(avx512_mask_pmov_qw_256, INTR_TYPE_1OP_MASK,
-                     X86ISD::VTRUNC, 0),
-  X86_INTRINSIC_DATA(avx512_mask_pmov_qw_512, INTR_TYPE_1OP_MASK,
-                     ISD::TRUNCATE, 0),
-  X86_INTRINSIC_DATA(avx512_mask_pmov_wb_128, INTR_TYPE_1OP_MASK,
-                     X86ISD::VTRUNC, 0),
-  X86_INTRINSIC_DATA(avx512_mask_pmov_wb_256, INTR_TYPE_1OP_MASK,
-                     ISD::TRUNCATE, 0),
-  X86_INTRINSIC_DATA(avx512_mask_pmov_wb_512, INTR_TYPE_1OP_MASK,
-                     ISD::TRUNCATE, 0),
-  X86_INTRINSIC_DATA(avx512_mask_pmovs_db_128, INTR_TYPE_1OP_MASK,
-                     X86ISD::VTRUNCS, 0),
-  X86_INTRINSIC_DATA(avx512_mask_pmovs_db_256, INTR_TYPE_1OP_MASK,
-                     X86ISD::VTRUNCS, 0),
-  X86_INTRINSIC_DATA(avx512_mask_pmovs_db_512, INTR_TYPE_1OP_MASK,
-                     X86ISD::VTRUNCS, 0),
-  X86_INTRINSIC_DATA(avx512_mask_pmovs_dw_128, INTR_TYPE_1OP_MASK,
-                     X86ISD::VTRUNCS, 0),
-  X86_INTRINSIC_DATA(avx512_mask_pmovs_dw_256, INTR_TYPE_1OP_MASK,
-                     X86ISD::VTRUNCS, 0),
-  X86_INTRINSIC_DATA(avx512_mask_pmovs_dw_512, INTR_TYPE_1OP_MASK,
-                     X86ISD::VTRUNCS, 0),
-  X86_INTRINSIC_DATA(avx512_mask_pmovs_qb_128, INTR_TYPE_1OP_MASK,
-                     X86ISD::VTRUNCS, 0),
-  X86_INTRINSIC_DATA(avx512_mask_pmovs_qb_256, INTR_TYPE_1OP_MASK,
-                     X86ISD::VTRUNCS, 0),
-  X86_INTRINSIC_DATA(avx512_mask_pmovs_qb_512, INTR_TYPE_1OP_MASK,
-                     X86ISD::VTRUNCS, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmov_db_128, TRUNCATE_TO_REG,
+                     X86ISD::VTRUNC, X86ISD::VMTRUNC),
+  X86_INTRINSIC_DATA(avx512_mask_pmov_db_256, TRUNCATE_TO_REG,
+                     X86ISD::VTRUNC, X86ISD::VMTRUNC),
+  X86_INTRINSIC_DATA(avx512_mask_pmov_db_512, TRUNCATE_TO_REG,
+                     ISD::TRUNCATE, X86ISD::VMTRUNC),
+  X86_INTRINSIC_DATA(avx512_mask_pmov_dw_128, TRUNCATE_TO_REG,
+                     X86ISD::VTRUNC, X86ISD::VMTRUNC),
+  X86_INTRINSIC_DATA(avx512_mask_pmov_dw_256, TRUNCATE_TO_REG,
+                     ISD::TRUNCATE, X86ISD::VMTRUNC),
+  X86_INTRINSIC_DATA(avx512_mask_pmov_dw_512, TRUNCATE_TO_REG,
+                     ISD::TRUNCATE, X86ISD::VMTRUNC),
+  X86_INTRINSIC_DATA(avx512_mask_pmov_qb_128, TRUNCATE_TO_REG,
+                     X86ISD::VTRUNC, X86ISD::VMTRUNC),
+  X86_INTRINSIC_DATA(avx512_mask_pmov_qb_256, TRUNCATE_TO_REG,
+                     X86ISD::VTRUNC, X86ISD::VMTRUNC),
+  X86_INTRINSIC_DATA(avx512_mask_pmov_qb_512, TRUNCATE_TO_REG,
+                     X86ISD::VTRUNC, X86ISD::VMTRUNC),
+  X86_INTRINSIC_DATA(avx512_mask_pmov_qd_128, TRUNCATE_TO_REG,
+                     X86ISD::VTRUNC, X86ISD::VMTRUNC),
+  X86_INTRINSIC_DATA(avx512_mask_pmov_qw_128, TRUNCATE_TO_REG,
+                     X86ISD::VTRUNC, X86ISD::VMTRUNC),
+  X86_INTRINSIC_DATA(avx512_mask_pmov_qw_256, TRUNCATE_TO_REG,
+                     X86ISD::VTRUNC, X86ISD::VMTRUNC),
+  X86_INTRINSIC_DATA(avx512_mask_pmov_qw_512, TRUNCATE_TO_REG,
+                     ISD::TRUNCATE, X86ISD::VMTRUNC),
+  X86_INTRINSIC_DATA(avx512_mask_pmov_wb_128, TRUNCATE_TO_REG,
+                     X86ISD::VTRUNC, X86ISD::VMTRUNC),
+  X86_INTRINSIC_DATA(avx512_mask_pmovs_db_128, TRUNCATE_TO_REG,
+                     X86ISD::VTRUNCS, X86ISD::VMTRUNCS),
+  X86_INTRINSIC_DATA(avx512_mask_pmovs_db_256, TRUNCATE_TO_REG,
+                     X86ISD::VTRUNCS, X86ISD::VMTRUNCS),
+  X86_INTRINSIC_DATA(avx512_mask_pmovs_db_512, TRUNCATE_TO_REG,
+                     X86ISD::VTRUNCS, X86ISD::VMTRUNCS),
+  X86_INTRINSIC_DATA(avx512_mask_pmovs_dw_128, TRUNCATE_TO_REG,
+                     X86ISD::VTRUNCS, X86ISD::VMTRUNCS),
+  X86_INTRINSIC_DATA(avx512_mask_pmovs_dw_256, TRUNCATE_TO_REG,
+                     X86ISD::VTRUNCS, X86ISD::VMTRUNCS),
+  X86_INTRINSIC_DATA(avx512_mask_pmovs_dw_512, TRUNCATE_TO_REG,
+                     X86ISD::VTRUNCS, X86ISD::VMTRUNCS),
+  X86_INTRINSIC_DATA(avx512_mask_pmovs_qb_128, TRUNCATE_TO_REG,
+                     X86ISD::VTRUNCS, X86ISD::VMTRUNCS),
+  X86_INTRINSIC_DATA(avx512_mask_pmovs_qb_256, TRUNCATE_TO_REG,
+                     X86ISD::VTRUNCS, X86ISD::VMTRUNCS),
+  X86_INTRINSIC_DATA(avx512_mask_pmovs_qb_512, TRUNCATE_TO_REG,
+                     X86ISD::VTRUNCS, X86ISD::VMTRUNCS),
   X86_INTRINSIC_DATA(avx512_mask_pmovs_qd_128, INTR_TYPE_1OP_MASK,
                      X86ISD::VTRUNCS, 0),
   X86_INTRINSIC_DATA(avx512_mask_pmovs_qd_256, INTR_TYPE_1OP_MASK,
                      X86ISD::VTRUNCS, 0),
   X86_INTRINSIC_DATA(avx512_mask_pmovs_qd_512, INTR_TYPE_1OP_MASK,
                      X86ISD::VTRUNCS, 0),
-  X86_INTRINSIC_DATA(avx512_mask_pmovs_qw_128, INTR_TYPE_1OP_MASK,
-                     X86ISD::VTRUNCS, 0),
-  X86_INTRINSIC_DATA(avx512_mask_pmovs_qw_256, INTR_TYPE_1OP_MASK,
-                     X86ISD::VTRUNCS, 0),
-  X86_INTRINSIC_DATA(avx512_mask_pmovs_qw_512, INTR_TYPE_1OP_MASK,
-                     X86ISD::VTRUNCS, 0),
-  X86_INTRINSIC_DATA(avx512_mask_pmovs_wb_128, INTR_TYPE_1OP_MASK,
-                     X86ISD::VTRUNCS, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmovs_qw_128, TRUNCATE_TO_REG,
+                     X86ISD::VTRUNCS, X86ISD::VMTRUNCS),
+  X86_INTRINSIC_DATA(avx512_mask_pmovs_qw_256, TRUNCATE_TO_REG,
+                     X86ISD::VTRUNCS, X86ISD::VMTRUNCS),
+  X86_INTRINSIC_DATA(avx512_mask_pmovs_qw_512, TRUNCATE_TO_REG,
+                     X86ISD::VTRUNCS, X86ISD::VMTRUNCS),
+  X86_INTRINSIC_DATA(avx512_mask_pmovs_wb_128, TRUNCATE_TO_REG,
+                     X86ISD::VTRUNCS, X86ISD::VMTRUNCS),
   X86_INTRINSIC_DATA(avx512_mask_pmovs_wb_256, INTR_TYPE_1OP_MASK,
                      X86ISD::VTRUNCS, 0),
   X86_INTRINSIC_DATA(avx512_mask_pmovs_wb_512, INTR_TYPE_1OP_MASK,
                      X86ISD::VTRUNCS, 0),
-  X86_INTRINSIC_DATA(avx512_mask_pmovus_db_128, INTR_TYPE_1OP_MASK,
-                     X86ISD::VTRUNCUS, 0),
-  X86_INTRINSIC_DATA(avx512_mask_pmovus_db_256, INTR_TYPE_1OP_MASK,
-                     X86ISD::VTRUNCUS, 0),
-  X86_INTRINSIC_DATA(avx512_mask_pmovus_db_512, INTR_TYPE_1OP_MASK,
-                     X86ISD::VTRUNCUS, 0),
-  X86_INTRINSIC_DATA(avx512_mask_pmovus_dw_128, INTR_TYPE_1OP_MASK,
-                     X86ISD::VTRUNCUS, 0),
-  X86_INTRINSIC_DATA(avx512_mask_pmovus_dw_256, INTR_TYPE_1OP_MASK,
-                     X86ISD::VTRUNCUS, 0),
-  X86_INTRINSIC_DATA(avx512_mask_pmovus_dw_512, INTR_TYPE_1OP_MASK,
-                     X86ISD::VTRUNCUS, 0),
-  X86_INTRINSIC_DATA(avx512_mask_pmovus_qb_128, INTR_TYPE_1OP_MASK,
-                     X86ISD::VTRUNCUS, 0),
-  X86_INTRINSIC_DATA(avx512_mask_pmovus_qb_256, INTR_TYPE_1OP_MASK,
-                     X86ISD::VTRUNCUS, 0),
-  X86_INTRINSIC_DATA(avx512_mask_pmovus_qb_512, INTR_TYPE_1OP_MASK,
-                     X86ISD::VTRUNCUS, 0),
-  X86_INTRINSIC_DATA(avx512_mask_pmovus_qd_128, INTR_TYPE_1OP_MASK,
-                     X86ISD::VTRUNCUS, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmovus_db_128, TRUNCATE_TO_REG,
+                     X86ISD::VTRUNCUS, X86ISD::VMTRUNCUS),
+  X86_INTRINSIC_DATA(avx512_mask_pmovus_db_256, TRUNCATE_TO_REG,
+                     X86ISD::VTRUNCUS, X86ISD::VMTRUNCUS),
+  X86_INTRINSIC_DATA(avx512_mask_pmovus_db_512, TRUNCATE_TO_REG,
+                     X86ISD::VTRUNCUS, X86ISD::VMTRUNCUS),
+  X86_INTRINSIC_DATA(avx512_mask_pmovus_dw_128, TRUNCATE_TO_REG,
+                     X86ISD::VTRUNCUS, X86ISD::VMTRUNCUS),
+  X86_INTRINSIC_DATA(avx512_mask_pmovus_dw_256, TRUNCATE_TO_REG,
+                     X86ISD::VTRUNCUS, X86ISD::VMTRUNCUS),
+  X86_INTRINSIC_DATA(avx512_mask_pmovus_dw_512, TRUNCATE_TO_REG,
+                     X86ISD::VTRUNCUS, X86ISD::VMTRUNCUS),
+  X86_INTRINSIC_DATA(avx512_mask_pmovus_qb_128, TRUNCATE_TO_REG,
+                     X86ISD::VTRUNCUS, X86ISD::VMTRUNCUS),
+  X86_INTRINSIC_DATA(avx512_mask_pmovus_qb_256, TRUNCATE_TO_REG,
+                     X86ISD::VTRUNCUS, X86ISD::VMTRUNCUS),
+  X86_INTRINSIC_DATA(avx512_mask_pmovus_qb_512, TRUNCATE_TO_REG,
+                     X86ISD::VTRUNCUS, X86ISD::VMTRUNCUS),
+  X86_INTRINSIC_DATA(avx512_mask_pmovus_qd_128, TRUNCATE_TO_REG,
+                     X86ISD::VTRUNCUS, X86ISD::VMTRUNCUS),
   X86_INTRINSIC_DATA(avx512_mask_pmovus_qd_256, INTR_TYPE_1OP_MASK,
                      X86ISD::VTRUNCUS, 0),
   X86_INTRINSIC_DATA(avx512_mask_pmovus_qd_512, INTR_TYPE_1OP_MASK,
                      X86ISD::VTRUNCUS, 0),
-  X86_INTRINSIC_DATA(avx512_mask_pmovus_qw_128, INTR_TYPE_1OP_MASK,
-                     X86ISD::VTRUNCUS, 0),
-  X86_INTRINSIC_DATA(avx512_mask_pmovus_qw_256, INTR_TYPE_1OP_MASK,
-                     X86ISD::VTRUNCUS, 0),
-  X86_INTRINSIC_DATA(avx512_mask_pmovus_qw_512, INTR_TYPE_1OP_MASK,
-                     X86ISD::VTRUNCUS, 0),
-  X86_INTRINSIC_DATA(avx512_mask_pmovus_wb_128, INTR_TYPE_1OP_MASK,
-                     X86ISD::VTRUNCUS, 0),
+  X86_INTRINSIC_DATA(avx512_mask_pmovus_qw_128, TRUNCATE_TO_REG,
+                     X86ISD::VTRUNCUS, X86ISD::VMTRUNCUS),
+  X86_INTRINSIC_DATA(avx512_mask_pmovus_qw_256, TRUNCATE_TO_REG,
+                     X86ISD::VTRUNCUS, X86ISD::VMTRUNCUS),
+  X86_INTRINSIC_DATA(avx512_mask_pmovus_qw_512, TRUNCATE_TO_REG,
+                     X86ISD::VTRUNCUS, X86ISD::VMTRUNCUS),
+  X86_INTRINSIC_DATA(avx512_mask_pmovus_wb_128, TRUNCATE_TO_REG,
+                     X86ISD::VTRUNCUS, X86ISD::VMTRUNCUS),
   X86_INTRINSIC_DATA(avx512_mask_pmovus_wb_256, INTR_TYPE_1OP_MASK,
                      X86ISD::VTRUNCUS, 0),
   X86_INTRINSIC_DATA(avx512_mask_pmovus_wb_512, INTR_TYPE_1OP_MASK,
                      X86ISD::VTRUNCUS, 0),
-  X86_INTRINSIC_DATA(avx512_mask_pmultishift_qb_128, INTR_TYPE_2OP_MASK,
-                     X86ISD::MULTISHIFT, 0),
-  X86_INTRINSIC_DATA(avx512_mask_pmultishift_qb_256, INTR_TYPE_2OP_MASK,
-                     X86ISD::MULTISHIFT, 0),
-  X86_INTRINSIC_DATA(avx512_mask_pmultishift_qb_512, INTR_TYPE_2OP_MASK,
-                     X86ISD::MULTISHIFT, 0),
   X86_INTRINSIC_DATA(avx512_mask_range_pd_128, INTR_TYPE_3OP_MASK, X86ISD::VRANGE, 0),
   X86_INTRINSIC_DATA(avx512_mask_range_pd_256, INTR_TYPE_3OP_MASK, X86ISD::VRANGE, 0),
   X86_INTRINSIC_DATA(avx512_mask_range_pd_512, INTR_TYPE_3OP_MASK, X86ISD::VRANGE, X86ISD::VRANGE_RND),
@@ -841,38 +785,12 @@ static const IntrinsicData  IntrinsicsWithoutChain[] = {
                      X86ISD::CVTPH2PS, 0),
   X86_INTRINSIC_DATA(avx512_mask_vcvtph2ps_512, INTR_TYPE_1OP_MASK,
                      X86ISD::CVTPH2PS, X86ISD::CVTPH2PS_RND),
-  X86_INTRINSIC_DATA(avx512_mask_vcvtps2ph_128, INTR_TYPE_2OP_MASK,
-                     X86ISD::CVTPS2PH, 0),
-  X86_INTRINSIC_DATA(avx512_mask_vcvtps2ph_256, INTR_TYPE_2OP_MASK,
-                     X86ISD::CVTPS2PH, 0),
-  X86_INTRINSIC_DATA(avx512_mask_vcvtps2ph_512, INTR_TYPE_2OP_MASK,
-                     X86ISD::CVTPS2PH, 0),
-
-  X86_INTRINSIC_DATA(avx512_mask_vpshldv_d_128, FMA_OP_MASK, X86ISD::VSHLDV, 0),
-  X86_INTRINSIC_DATA(avx512_mask_vpshldv_d_256, FMA_OP_MASK, X86ISD::VSHLDV, 0),
-  X86_INTRINSIC_DATA(avx512_mask_vpshldv_d_512, FMA_OP_MASK, X86ISD::VSHLDV, 0),
-  X86_INTRINSIC_DATA(avx512_mask_vpshldv_q_128, FMA_OP_MASK, X86ISD::VSHLDV, 0),
-  X86_INTRINSIC_DATA(avx512_mask_vpshldv_q_256, FMA_OP_MASK, X86ISD::VSHLDV, 0),
-  X86_INTRINSIC_DATA(avx512_mask_vpshldv_q_512, FMA_OP_MASK, X86ISD::VSHLDV, 0),
-  X86_INTRINSIC_DATA(avx512_mask_vpshldv_w_128, FMA_OP_MASK, X86ISD::VSHLDV, 0),
-  X86_INTRINSIC_DATA(avx512_mask_vpshldv_w_256, FMA_OP_MASK, X86ISD::VSHLDV, 0),
-  X86_INTRINSIC_DATA(avx512_mask_vpshldv_w_512, FMA_OP_MASK, X86ISD::VSHLDV, 0),
-  X86_INTRINSIC_DATA(avx512_mask_vpshrdv_d_128, FMA_OP_MASK, X86ISD::VSHRDV, 0),
-  X86_INTRINSIC_DATA(avx512_mask_vpshrdv_d_256, FMA_OP_MASK, X86ISD::VSHRDV, 0),
-  X86_INTRINSIC_DATA(avx512_mask_vpshrdv_d_512, FMA_OP_MASK, X86ISD::VSHRDV, 0),
-  X86_INTRINSIC_DATA(avx512_mask_vpshrdv_q_128, FMA_OP_MASK, X86ISD::VSHRDV, 0),
-  X86_INTRINSIC_DATA(avx512_mask_vpshrdv_q_256, FMA_OP_MASK, X86ISD::VSHRDV, 0),
-  X86_INTRINSIC_DATA(avx512_mask_vpshrdv_q_512, FMA_OP_MASK, X86ISD::VSHRDV, 0),
-  X86_INTRINSIC_DATA(avx512_mask_vpshrdv_w_128, FMA_OP_MASK, X86ISD::VSHRDV, 0),
-  X86_INTRINSIC_DATA(avx512_mask_vpshrdv_w_256, FMA_OP_MASK, X86ISD::VSHRDV, 0),
-  X86_INTRINSIC_DATA(avx512_mask_vpshrdv_w_512, FMA_OP_MASK, X86ISD::VSHRDV, 0),
-
-  X86_INTRINSIC_DATA(avx512_mask_vpshufbitqmb_128, CMP_MASK,
-                     X86ISD::VPSHUFBITQMB, 0),
-  X86_INTRINSIC_DATA(avx512_mask_vpshufbitqmb_256, CMP_MASK,
-                     X86ISD::VPSHUFBITQMB, 0),
-  X86_INTRINSIC_DATA(avx512_mask_vpshufbitqmb_512, CMP_MASK,
-                     X86ISD::VPSHUFBITQMB, 0),
+  X86_INTRINSIC_DATA(avx512_mask_vcvtps2ph_128, CVTPS2PH_MASK,
+                     X86ISD::CVTPS2PH, X86ISD::MCVTPS2PH),
+  X86_INTRINSIC_DATA(avx512_mask_vcvtps2ph_256, CVTPS2PH_MASK,
+                     X86ISD::CVTPS2PH, X86ISD::MCVTPS2PH),
+  X86_INTRINSIC_DATA(avx512_mask_vcvtps2ph_512, CVTPS2PH_MASK,
+                     X86ISD::CVTPS2PH, X86ISD::MCVTPS2PH),
 
   X86_INTRINSIC_DATA(avx512_maskz_fixupimm_pd_128, FIXUPIMM_MASKZ,
                      X86ISD::VFIXUPIMM, 0),
@@ -891,25 +809,6 @@ static const IntrinsicData  IntrinsicsWithoutChain[] = {
   X86_INTRINSIC_DATA(avx512_maskz_fixupimm_ss, FIXUPIMMS_MASKZ,
                      X86ISD::VFIXUPIMMS, 0),
 
-  X86_INTRINSIC_DATA(avx512_maskz_vpshldv_d_128, FMA_OP_MASKZ, X86ISD::VSHLDV, 0),
-  X86_INTRINSIC_DATA(avx512_maskz_vpshldv_d_256, FMA_OP_MASKZ, X86ISD::VSHLDV, 0),
-  X86_INTRINSIC_DATA(avx512_maskz_vpshldv_d_512, FMA_OP_MASKZ, X86ISD::VSHLDV, 0),
-  X86_INTRINSIC_DATA(avx512_maskz_vpshldv_q_128, FMA_OP_MASKZ, X86ISD::VSHLDV, 0),
-  X86_INTRINSIC_DATA(avx512_maskz_vpshldv_q_256, FMA_OP_MASKZ, X86ISD::VSHLDV, 0),
-  X86_INTRINSIC_DATA(avx512_maskz_vpshldv_q_512, FMA_OP_MASKZ, X86ISD::VSHLDV, 0),
-  X86_INTRINSIC_DATA(avx512_maskz_vpshldv_w_128, FMA_OP_MASKZ, X86ISD::VSHLDV, 0),
-  X86_INTRINSIC_DATA(avx512_maskz_vpshldv_w_256, FMA_OP_MASKZ, X86ISD::VSHLDV, 0),
-  X86_INTRINSIC_DATA(avx512_maskz_vpshldv_w_512, FMA_OP_MASKZ, X86ISD::VSHLDV, 0),
-  X86_INTRINSIC_DATA(avx512_maskz_vpshrdv_d_128, FMA_OP_MASKZ, X86ISD::VSHRDV, 0),
-  X86_INTRINSIC_DATA(avx512_maskz_vpshrdv_d_256, FMA_OP_MASKZ, X86ISD::VSHRDV, 0),
-  X86_INTRINSIC_DATA(avx512_maskz_vpshrdv_d_512, FMA_OP_MASKZ, X86ISD::VSHRDV, 0),
-  X86_INTRINSIC_DATA(avx512_maskz_vpshrdv_q_128, FMA_OP_MASKZ, X86ISD::VSHRDV, 0),
-  X86_INTRINSIC_DATA(avx512_maskz_vpshrdv_q_256, FMA_OP_MASKZ, X86ISD::VSHRDV, 0),
-  X86_INTRINSIC_DATA(avx512_maskz_vpshrdv_q_512, FMA_OP_MASKZ, X86ISD::VSHRDV, 0),
-  X86_INTRINSIC_DATA(avx512_maskz_vpshrdv_w_128, FMA_OP_MASKZ, X86ISD::VSHRDV, 0),
-  X86_INTRINSIC_DATA(avx512_maskz_vpshrdv_w_256, FMA_OP_MASKZ, X86ISD::VSHRDV, 0),
-  X86_INTRINSIC_DATA(avx512_maskz_vpshrdv_w_512, FMA_OP_MASKZ, X86ISD::VSHRDV, 0),
-
   X86_INTRINSIC_DATA(avx512_max_pd_512, INTR_TYPE_2OP, X86ISD::FMAX, X86ISD::FMAX_RND),
   X86_INTRINSIC_DATA(avx512_max_ps_512, INTR_TYPE_2OP, X86ISD::FMAX, X86ISD::FMAX_RND),
   X86_INTRINSIC_DATA(avx512_min_pd_512, INTR_TYPE_2OP, X86ISD::FMIN, X86ISD::FMIN_RND),
@@ -920,8 +819,6 @@ static const IntrinsicData  IntrinsicsWithoutChain[] = {
   X86_INTRINSIC_DATA(avx512_packsswb_512, INTR_TYPE_2OP, X86ISD::PACKSS, 0),
   X86_INTRINSIC_DATA(avx512_packusdw_512, INTR_TYPE_2OP, X86ISD::PACKUS, 0),
   X86_INTRINSIC_DATA(avx512_packuswb_512, INTR_TYPE_2OP, X86ISD::PACKUS, 0),
-  X86_INTRINSIC_DATA(avx512_padds_b_512, INTR_TYPE_2OP, ISD::SADDSAT, 0),
-  X86_INTRINSIC_DATA(avx512_padds_w_512, INTR_TYPE_2OP, ISD::SADDSAT, 0),
   X86_INTRINSIC_DATA(avx512_permvar_df_256, VPERM_2OP, X86ISD::VPERMV, 0),
   X86_INTRINSIC_DATA(avx512_permvar_df_512, VPERM_2OP, X86ISD::VPERMV, 0),
   X86_INTRINSIC_DATA(avx512_permvar_di_256, VPERM_2OP, X86ISD::VPERMV, 0),
@@ -939,30 +836,9 @@ static const IntrinsicData  IntrinsicsWithoutChain[] = {
   X86_INTRINSIC_DATA(avx512_pmul_hr_sw_512, INTR_TYPE_2OP, X86ISD::MULHRS, 0),
   X86_INTRINSIC_DATA(avx512_pmulh_w_512, INTR_TYPE_2OP, ISD::MULHS, 0),
   X86_INTRINSIC_DATA(avx512_pmulhu_w_512, INTR_TYPE_2OP, ISD::MULHU, 0),
-  X86_INTRINSIC_DATA(avx512_prol_d_128,  INTR_TYPE_2OP_IMM8, X86ISD::VROTLI, 0),
-  X86_INTRINSIC_DATA(avx512_prol_d_256,  INTR_TYPE_2OP_IMM8, X86ISD::VROTLI, 0),
-  X86_INTRINSIC_DATA(avx512_prol_d_512,  INTR_TYPE_2OP_IMM8, X86ISD::VROTLI, 0),
-  X86_INTRINSIC_DATA(avx512_prol_q_128,  INTR_TYPE_2OP_IMM8, X86ISD::VROTLI, 0),
-  X86_INTRINSIC_DATA(avx512_prol_q_256,  INTR_TYPE_2OP_IMM8, X86ISD::VROTLI, 0),
-  X86_INTRINSIC_DATA(avx512_prol_q_512,  INTR_TYPE_2OP_IMM8, X86ISD::VROTLI, 0),
-  X86_INTRINSIC_DATA(avx512_prolv_d_128, INTR_TYPE_2OP, ISD::ROTL, 0),
-  X86_INTRINSIC_DATA(avx512_prolv_d_256, INTR_TYPE_2OP, ISD::ROTL, 0),
-  X86_INTRINSIC_DATA(avx512_prolv_d_512, INTR_TYPE_2OP, ISD::ROTL, 0),
-  X86_INTRINSIC_DATA(avx512_prolv_q_128, INTR_TYPE_2OP, ISD::ROTL, 0),
-  X86_INTRINSIC_DATA(avx512_prolv_q_256, INTR_TYPE_2OP, ISD::ROTL, 0),
-  X86_INTRINSIC_DATA(avx512_prolv_q_512, INTR_TYPE_2OP, ISD::ROTL, 0),
-  X86_INTRINSIC_DATA(avx512_pror_d_128,  INTR_TYPE_2OP_IMM8, X86ISD::VROTRI, 0),
-  X86_INTRINSIC_DATA(avx512_pror_d_256,  INTR_TYPE_2OP_IMM8, X86ISD::VROTRI, 0),
-  X86_INTRINSIC_DATA(avx512_pror_d_512,  INTR_TYPE_2OP_IMM8, X86ISD::VROTRI, 0),
-  X86_INTRINSIC_DATA(avx512_pror_q_128,  INTR_TYPE_2OP_IMM8, X86ISD::VROTRI, 0),
-  X86_INTRINSIC_DATA(avx512_pror_q_256,  INTR_TYPE_2OP_IMM8, X86ISD::VROTRI, 0),
-  X86_INTRINSIC_DATA(avx512_pror_q_512,  INTR_TYPE_2OP_IMM8, X86ISD::VROTRI, 0),
-  X86_INTRINSIC_DATA(avx512_prorv_d_128, INTR_TYPE_2OP, ISD::ROTR, 0),
-  X86_INTRINSIC_DATA(avx512_prorv_d_256, INTR_TYPE_2OP, ISD::ROTR, 0),
-  X86_INTRINSIC_DATA(avx512_prorv_d_512, INTR_TYPE_2OP, ISD::ROTR, 0),
-  X86_INTRINSIC_DATA(avx512_prorv_q_128, INTR_TYPE_2OP, ISD::ROTR, 0),
-  X86_INTRINSIC_DATA(avx512_prorv_q_256, INTR_TYPE_2OP, ISD::ROTR, 0),
-  X86_INTRINSIC_DATA(avx512_prorv_q_512, INTR_TYPE_2OP, ISD::ROTR, 0),
+  X86_INTRINSIC_DATA(avx512_pmultishift_qb_128, INTR_TYPE_2OP, X86ISD::MULTISHIFT, 0),
+  X86_INTRINSIC_DATA(avx512_pmultishift_qb_256, INTR_TYPE_2OP, X86ISD::MULTISHIFT, 0),
+  X86_INTRINSIC_DATA(avx512_pmultishift_qb_512, INTR_TYPE_2OP, X86ISD::MULTISHIFT, 0),
   X86_INTRINSIC_DATA(avx512_psad_bw_512, INTR_TYPE_2OP, X86ISD::PSADBW, 0),
   X86_INTRINSIC_DATA(avx512_pshuf_b_512, INTR_TYPE_2OP, X86ISD::PSHUFB, 0),
   X86_INTRINSIC_DATA(avx512_psll_d_512, INTR_TYPE_2OP, X86ISD::VSHL, 0),
@@ -971,11 +847,11 @@ static const IntrinsicData  IntrinsicsWithoutChain[] = {
   X86_INTRINSIC_DATA(avx512_pslli_d_512, VSHIFT, X86ISD::VSHLI, 0),
   X86_INTRINSIC_DATA(avx512_pslli_q_512, VSHIFT, X86ISD::VSHLI, 0),
   X86_INTRINSIC_DATA(avx512_pslli_w_512, VSHIFT, X86ISD::VSHLI, 0),
-  X86_INTRINSIC_DATA(avx512_psllv_d_512, INTR_TYPE_2OP, ISD::SHL, 0),
-  X86_INTRINSIC_DATA(avx512_psllv_q_512, INTR_TYPE_2OP, ISD::SHL, 0),
-  X86_INTRINSIC_DATA(avx512_psllv_w_128, INTR_TYPE_2OP, ISD::SHL, 0),
-  X86_INTRINSIC_DATA(avx512_psllv_w_256, INTR_TYPE_2OP, ISD::SHL, 0),
-  X86_INTRINSIC_DATA(avx512_psllv_w_512, INTR_TYPE_2OP, ISD::SHL, 0),
+  X86_INTRINSIC_DATA(avx512_psllv_d_512, INTR_TYPE_2OP, X86ISD::VSHLV, 0),
+  X86_INTRINSIC_DATA(avx512_psllv_q_512, INTR_TYPE_2OP, X86ISD::VSHLV, 0),
+  X86_INTRINSIC_DATA(avx512_psllv_w_128, INTR_TYPE_2OP, X86ISD::VSHLV, 0),
+  X86_INTRINSIC_DATA(avx512_psllv_w_256, INTR_TYPE_2OP, X86ISD::VSHLV, 0),
+  X86_INTRINSIC_DATA(avx512_psllv_w_512, INTR_TYPE_2OP, X86ISD::VSHLV, 0),
   X86_INTRINSIC_DATA(avx512_psra_d_512, INTR_TYPE_2OP, X86ISD::VSRA, 0),
   X86_INTRINSIC_DATA(avx512_psra_q_128, INTR_TYPE_2OP, X86ISD::VSRA, 0),
   X86_INTRINSIC_DATA(avx512_psra_q_256, INTR_TYPE_2OP, X86ISD::VSRA, 0),
@@ -999,13 +875,11 @@ static const IntrinsicData  IntrinsicsWithoutChain[] = {
   X86_INTRINSIC_DATA(avx512_psrli_d_512, VSHIFT, X86ISD::VSRLI, 0),
   X86_INTRINSIC_DATA(avx512_psrli_q_512, VSHIFT, X86ISD::VSRLI, 0),
   X86_INTRINSIC_DATA(avx512_psrli_w_512, VSHIFT, X86ISD::VSRLI, 0),
-  X86_INTRINSIC_DATA(avx512_psrlv_d_512, INTR_TYPE_2OP, ISD::SRL, 0),
-  X86_INTRINSIC_DATA(avx512_psrlv_q_512, INTR_TYPE_2OP, ISD::SRL, 0),
-  X86_INTRINSIC_DATA(avx512_psrlv_w_128, INTR_TYPE_2OP, ISD::SRL, 0),
-  X86_INTRINSIC_DATA(avx512_psrlv_w_256, INTR_TYPE_2OP, ISD::SRL, 0),
-  X86_INTRINSIC_DATA(avx512_psrlv_w_512, INTR_TYPE_2OP, ISD::SRL, 0),
-  X86_INTRINSIC_DATA(avx512_psubs_b_512, INTR_TYPE_2OP, ISD::SSUBSAT, 0),
-  X86_INTRINSIC_DATA(avx512_psubs_w_512, INTR_TYPE_2OP, ISD::SSUBSAT, 0),
+  X86_INTRINSIC_DATA(avx512_psrlv_d_512, INTR_TYPE_2OP, X86ISD::VSRLV, 0),
+  X86_INTRINSIC_DATA(avx512_psrlv_q_512, INTR_TYPE_2OP, X86ISD::VSRLV, 0),
+  X86_INTRINSIC_DATA(avx512_psrlv_w_128, INTR_TYPE_2OP, X86ISD::VSRLV, 0),
+  X86_INTRINSIC_DATA(avx512_psrlv_w_256, INTR_TYPE_2OP, X86ISD::VSRLV, 0),
+  X86_INTRINSIC_DATA(avx512_psrlv_w_512, INTR_TYPE_2OP, X86ISD::VSRLV, 0),
   X86_INTRINSIC_DATA(avx512_pternlog_d_128, INTR_TYPE_4OP, X86ISD::VPTERNLOG, 0),
   X86_INTRINSIC_DATA(avx512_pternlog_d_256, INTR_TYPE_4OP, X86ISD::VPTERNLOG, 0),
   X86_INTRINSIC_DATA(avx512_pternlog_d_512, INTR_TYPE_4OP, X86ISD::VPTERNLOG, 0),
@@ -1036,10 +910,12 @@ static const IntrinsicData  IntrinsicsWithoutChain[] = {
   X86_INTRINSIC_DATA(avx512_rsqrt28_ps, INTR_TYPE_1OP_MASK_RM,X86ISD::RSQRT28, 0),
   X86_INTRINSIC_DATA(avx512_rsqrt28_sd, INTR_TYPE_SCALAR_MASK_RM,X86ISD::RSQRT28S, 0),
   X86_INTRINSIC_DATA(avx512_rsqrt28_ss, INTR_TYPE_SCALAR_MASK_RM,X86ISD::RSQRT28S, 0),
+  X86_INTRINSIC_DATA(avx512_sitofp_round, INTR_TYPE_1OP, ISD::SINT_TO_FP, X86ISD::SINT_TO_FP_RND),
   X86_INTRINSIC_DATA(avx512_sqrt_pd_512, INTR_TYPE_1OP, ISD::FSQRT, X86ISD::FSQRT_RND),
   X86_INTRINSIC_DATA(avx512_sqrt_ps_512, INTR_TYPE_1OP, ISD::FSQRT, X86ISD::FSQRT_RND),
   X86_INTRINSIC_DATA(avx512_sub_pd_512, INTR_TYPE_2OP, ISD::FSUB, X86ISD::FSUB_RND),
   X86_INTRINSIC_DATA(avx512_sub_ps_512, INTR_TYPE_2OP, ISD::FSUB, X86ISD::FSUB_RND),
+  X86_INTRINSIC_DATA(avx512_uitofp_round, INTR_TYPE_1OP, ISD::UINT_TO_FP, X86ISD::UINT_TO_FP_RND),
   X86_INTRINSIC_DATA(avx512_vcomi_sd, COMI_RM, X86ISD::COMI, X86ISD::UCOMI),
   X86_INTRINSIC_DATA(avx512_vcomi_ss, COMI_RM, X86ISD::COMI, X86ISD::UCOMI),
   X86_INTRINSIC_DATA(avx512_vcvtsd2si32, INTR_TYPE_1OP, X86ISD::CVTS2SI, X86ISD::CVTS2SI_RND),
@@ -1098,24 +974,9 @@ static const IntrinsicData  IntrinsicsWithoutChain[] = {
   X86_INTRINSIC_DATA(avx512_vpmadd52l_uq_128 , IFMA_OP, X86ISD::VPMADD52L, 0),
   X86_INTRINSIC_DATA(avx512_vpmadd52l_uq_256 , IFMA_OP, X86ISD::VPMADD52L, 0),
   X86_INTRINSIC_DATA(avx512_vpmadd52l_uq_512 , IFMA_OP, X86ISD::VPMADD52L, 0),
-  X86_INTRINSIC_DATA(avx512_vpshld_d_128, INTR_TYPE_3OP_IMM8, X86ISD::VSHLD, 0),
-  X86_INTRINSIC_DATA(avx512_vpshld_d_256, INTR_TYPE_3OP_IMM8, X86ISD::VSHLD, 0),
-  X86_INTRINSIC_DATA(avx512_vpshld_d_512, INTR_TYPE_3OP_IMM8, X86ISD::VSHLD, 0),
-  X86_INTRINSIC_DATA(avx512_vpshld_q_128, INTR_TYPE_3OP_IMM8, X86ISD::VSHLD, 0),
-  X86_INTRINSIC_DATA(avx512_vpshld_q_256, INTR_TYPE_3OP_IMM8, X86ISD::VSHLD, 0),
-  X86_INTRINSIC_DATA(avx512_vpshld_q_512, INTR_TYPE_3OP_IMM8, X86ISD::VSHLD, 0),
-  X86_INTRINSIC_DATA(avx512_vpshld_w_128, INTR_TYPE_3OP_IMM8, X86ISD::VSHLD, 0),
-  X86_INTRINSIC_DATA(avx512_vpshld_w_256, INTR_TYPE_3OP_IMM8, X86ISD::VSHLD, 0),
-  X86_INTRINSIC_DATA(avx512_vpshld_w_512, INTR_TYPE_3OP_IMM8, X86ISD::VSHLD, 0),
-  X86_INTRINSIC_DATA(avx512_vpshrd_d_128, INTR_TYPE_3OP_IMM8, X86ISD::VSHRD, 0),
-  X86_INTRINSIC_DATA(avx512_vpshrd_d_256, INTR_TYPE_3OP_IMM8, X86ISD::VSHRD, 0),
-  X86_INTRINSIC_DATA(avx512_vpshrd_d_512, INTR_TYPE_3OP_IMM8, X86ISD::VSHRD, 0),
-  X86_INTRINSIC_DATA(avx512_vpshrd_q_128, INTR_TYPE_3OP_IMM8, X86ISD::VSHRD, 0),
-  X86_INTRINSIC_DATA(avx512_vpshrd_q_256, INTR_TYPE_3OP_IMM8, X86ISD::VSHRD, 0),
-  X86_INTRINSIC_DATA(avx512_vpshrd_q_512, INTR_TYPE_3OP_IMM8, X86ISD::VSHRD, 0),
-  X86_INTRINSIC_DATA(avx512_vpshrd_w_128, INTR_TYPE_3OP_IMM8, X86ISD::VSHRD, 0),
-  X86_INTRINSIC_DATA(avx512_vpshrd_w_256, INTR_TYPE_3OP_IMM8, X86ISD::VSHRD, 0),
-  X86_INTRINSIC_DATA(avx512_vpshrd_w_512, INTR_TYPE_3OP_IMM8, X86ISD::VSHRD, 0),
+  X86_INTRINSIC_DATA(avx512_vpshufbitqmb_128, INTR_TYPE_2OP, X86ISD::VPSHUFBITQMB, 0),
+  X86_INTRINSIC_DATA(avx512_vpshufbitqmb_256, INTR_TYPE_2OP, X86ISD::VPSHUFBITQMB, 0),
+  X86_INTRINSIC_DATA(avx512_vpshufbitqmb_512, INTR_TYPE_2OP, X86ISD::VPSHUFBITQMB, 0),
   X86_INTRINSIC_DATA(bmi_bextr_32,         INTR_TYPE_2OP, X86ISD::BEXTR, 0),
   X86_INTRINSIC_DATA(bmi_bextr_64,         INTR_TYPE_2OP, X86ISD::BEXTR, 0),
   X86_INTRINSIC_DATA(bmi_bzhi_32,          INTR_TYPE_2OP, X86ISD::BZHI, 0),
@@ -1168,8 +1029,6 @@ static const IntrinsicData  IntrinsicsWithoutChain[] = {
   X86_INTRINSIC_DATA(sse2_packssdw_128, INTR_TYPE_2OP, X86ISD::PACKSS, 0),
   X86_INTRINSIC_DATA(sse2_packsswb_128, INTR_TYPE_2OP, X86ISD::PACKSS, 0),
   X86_INTRINSIC_DATA(sse2_packuswb_128, INTR_TYPE_2OP, X86ISD::PACKUS, 0),
-  X86_INTRINSIC_DATA(sse2_padds_b,      INTR_TYPE_2OP, ISD::SADDSAT, 0),
-  X86_INTRINSIC_DATA(sse2_padds_w,      INTR_TYPE_2OP, ISD::SADDSAT, 0),
   X86_INTRINSIC_DATA(sse2_pmadd_wd,     INTR_TYPE_2OP, X86ISD::VPMADDWD, 0),
   X86_INTRINSIC_DATA(sse2_pmovmskb_128, INTR_TYPE_1OP, X86ISD::MOVMSK, 0),
   X86_INTRINSIC_DATA(sse2_pmulh_w,      INTR_TYPE_2OP, ISD::MULHS, 0),
@@ -1191,8 +1050,6 @@ static const IntrinsicData  IntrinsicsWithoutChain[] = {
   X86_INTRINSIC_DATA(sse2_psrli_d,      VSHIFT, X86ISD::VSRLI, 0),
   X86_INTRINSIC_DATA(sse2_psrli_q,      VSHIFT, X86ISD::VSRLI, 0),
   X86_INTRINSIC_DATA(sse2_psrli_w,      VSHIFT, X86ISD::VSRLI, 0),
-  X86_INTRINSIC_DATA(sse2_psubs_b,      INTR_TYPE_2OP, ISD::SSUBSAT, 0),
-  X86_INTRINSIC_DATA(sse2_psubs_w,      INTR_TYPE_2OP, ISD::SSUBSAT, 0),
   X86_INTRINSIC_DATA(sse2_ucomieq_sd,   COMI, X86ISD::UCOMI, ISD::SETEQ),
   X86_INTRINSIC_DATA(sse2_ucomige_sd,   COMI, X86ISD::UCOMI, ISD::SETGE),
   X86_INTRINSIC_DATA(sse2_ucomigt_sd,   COMI, X86ISD::UCOMI, ISD::SETGT),
@@ -1205,8 +1062,11 @@ static const IntrinsicData  IntrinsicsWithoutChain[] = {
   X86_INTRINSIC_DATA(sse3_hadd_ps,      INTR_TYPE_2OP, X86ISD::FHADD, 0),
   X86_INTRINSIC_DATA(sse3_hsub_pd,      INTR_TYPE_2OP, X86ISD::FHSUB, 0),
   X86_INTRINSIC_DATA(sse3_hsub_ps,      INTR_TYPE_2OP, X86ISD::FHSUB, 0),
+  X86_INTRINSIC_DATA(sse41_blendvpd,    BLENDV, X86ISD::BLENDV, 0),
+  X86_INTRINSIC_DATA(sse41_blendvps,    BLENDV, X86ISD::BLENDV, 0),
   X86_INTRINSIC_DATA(sse41_insertps,    INTR_TYPE_3OP, X86ISD::INSERTPS, 0),
   X86_INTRINSIC_DATA(sse41_packusdw,    INTR_TYPE_2OP, X86ISD::PACKUS, 0),
+  X86_INTRINSIC_DATA(sse41_pblendvb,    BLENDV, X86ISD::BLENDV, 0),
   X86_INTRINSIC_DATA(sse41_phminposuw,  INTR_TYPE_1OP, X86ISD::PHMINPOS, 0),
   X86_INTRINSIC_DATA(sse41_round_pd,    ROUNDP, X86ISD::VRNDSCALE, 0),
   X86_INTRINSIC_DATA(sse41_round_ps,    ROUNDP, X86ISD::VRNDSCALE, 0),
@@ -1249,27 +1109,11 @@ static const IntrinsicData  IntrinsicsWithoutChain[] = {
   X86_INTRINSIC_DATA(vgf2p8mulb_512, INTR_TYPE_2OP,
                      X86ISD::GF2P8MULB, 0),
 
-  X86_INTRINSIC_DATA(xop_vpcomb,        INTR_TYPE_3OP, X86ISD::VPCOM, 0),
-  X86_INTRINSIC_DATA(xop_vpcomd,        INTR_TYPE_3OP, X86ISD::VPCOM, 0),
-  X86_INTRINSIC_DATA(xop_vpcomq,        INTR_TYPE_3OP, X86ISD::VPCOM, 0),
-  X86_INTRINSIC_DATA(xop_vpcomub,       INTR_TYPE_3OP, X86ISD::VPCOMU, 0),
-  X86_INTRINSIC_DATA(xop_vpcomud,       INTR_TYPE_3OP, X86ISD::VPCOMU, 0),
-  X86_INTRINSIC_DATA(xop_vpcomuq,       INTR_TYPE_3OP, X86ISD::VPCOMU, 0),
-  X86_INTRINSIC_DATA(xop_vpcomuw,       INTR_TYPE_3OP, X86ISD::VPCOMU, 0),
-  X86_INTRINSIC_DATA(xop_vpcomw,        INTR_TYPE_3OP, X86ISD::VPCOM, 0),
   X86_INTRINSIC_DATA(xop_vpermil2pd,     INTR_TYPE_4OP, X86ISD::VPERMIL2, 0),
   X86_INTRINSIC_DATA(xop_vpermil2pd_256, INTR_TYPE_4OP, X86ISD::VPERMIL2, 0),
   X86_INTRINSIC_DATA(xop_vpermil2ps,     INTR_TYPE_4OP, X86ISD::VPERMIL2, 0),
   X86_INTRINSIC_DATA(xop_vpermil2ps_256, INTR_TYPE_4OP, X86ISD::VPERMIL2, 0),
   X86_INTRINSIC_DATA(xop_vpperm,        INTR_TYPE_3OP, X86ISD::VPPERM, 0),
-  X86_INTRINSIC_DATA(xop_vprotb,        INTR_TYPE_2OP, ISD::ROTL, 0),
-  X86_INTRINSIC_DATA(xop_vprotbi,       INTR_TYPE_2OP, X86ISD::VROTLI, 0),
-  X86_INTRINSIC_DATA(xop_vprotd,        INTR_TYPE_2OP, ISD::ROTL, 0),
-  X86_INTRINSIC_DATA(xop_vprotdi,       INTR_TYPE_2OP, X86ISD::VROTLI, 0),
-  X86_INTRINSIC_DATA(xop_vprotq,        INTR_TYPE_2OP, ISD::ROTL, 0),
-  X86_INTRINSIC_DATA(xop_vprotqi,       INTR_TYPE_2OP, X86ISD::VROTLI, 0),
-  X86_INTRINSIC_DATA(xop_vprotw,        INTR_TYPE_2OP, ISD::ROTL, 0),
-  X86_INTRINSIC_DATA(xop_vprotwi,       INTR_TYPE_2OP, X86ISD::VROTLI, 0),
   X86_INTRINSIC_DATA(xop_vpshab,        INTR_TYPE_2OP, X86ISD::VPSHA, 0),
   X86_INTRINSIC_DATA(xop_vpshad,        INTR_TYPE_2OP, X86ISD::VPSHA, 0),
   X86_INTRINSIC_DATA(xop_vpshaq,        INTR_TYPE_2OP, X86ISD::VPSHA, 0),
diff --git a/lib/Target/X86/X86LegalizerInfo.cpp b/lib/Target/X86/X86LegalizerInfo.cpp
index 4a49fa68dd0606721b1b8eb0f7507a42e84d3b10..00fb1b573858037393c38e5d09514b73fe3ab16f 100644
--- a/lib/Target/X86/X86LegalizerInfo.cpp
+++ b/lib/Target/X86/X86LegalizerInfo.cpp
@@ -1,9 +1,8 @@
 //===- X86LegalizerInfo.cpp --------------------------------------*- C++ -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
@@ -134,9 +133,15 @@ void X86LegalizerInfo::setLegalizerInfo32bit() {
 
     // Shifts and SDIV
     getActionDefinitionsBuilder(
-        {G_SHL, G_LSHR, G_ASHR, G_SDIV, G_SREM, G_UDIV, G_UREM})
-        .legalFor({s8, s16, s32})
-        .clampScalar(0, s8, s32);
+        {G_SDIV, G_SREM, G_UDIV, G_UREM})
+      .legalFor({s8, s16, s32})
+      .clampScalar(0, s8, s32);
+
+    getActionDefinitionsBuilder(
+        {G_SHL, G_LSHR, G_ASHR})
+      .legalFor({{s8, s8}, {s16, s8}, {s32, s8}})
+      .clampScalar(0, s8, s32)
+      .clampScalar(1, s8, s8);
   }
 
   // Control-flow
@@ -236,12 +241,19 @@ void X86LegalizerInfo::setLegalizerInfo64bit() {
       .clampScalar(1, s32, s64)
       .widenScalarToNextPow2(1);
 
-  // Shifts and SDIV
+  // Divisions
   getActionDefinitionsBuilder(
-      {G_SHL, G_LSHR, G_ASHR, G_SDIV, G_SREM, G_UDIV, G_UREM})
+      {G_SDIV, G_SREM, G_UDIV, G_UREM})
       .legalFor({s8, s16, s32, s64})
       .clampScalar(0, s8, s64);
 
+  // Shifts
+  getActionDefinitionsBuilder(
+    {G_SHL, G_LSHR, G_ASHR})
+    .legalFor({{s8, s8}, {s16, s8}, {s32, s8}, {s64, s8}})
+    .clampScalar(0, s8, s64)
+    .clampScalar(1, s8, s8);
+
   // Merge/Unmerge
   setAction({G_MERGE_VALUES, s128}, Legal);
   setAction({G_UNMERGE_VALUES, 1, s128}, Legal);
diff --git a/lib/Target/X86/X86LegalizerInfo.h b/lib/Target/X86/X86LegalizerInfo.h
index 135950a95f84454dc06ba5fc930c22b0f318dbe5..d21707b9ab9b395b8ff15d2d4e56c2b33977fb11 100644
--- a/lib/Target/X86/X86LegalizerInfo.h
+++ b/lib/Target/X86/X86LegalizerInfo.h
@@ -1,10 +1,9 @@
 //===- X86LegalizerInfo.h ------------------------------------------*- C++
 //-*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/lib/Target/X86/X86MCInstLower.cpp b/lib/Target/X86/X86MCInstLower.cpp
index 2816f8c62bfb1a4ef5fd58af276da1cb87b82393..cc2a1a43b6757b304683b87b87a6fd01689d046c 100644
--- a/lib/Target/X86/X86MCInstLower.cpp
+++ b/lib/Target/X86/X86MCInstLower.cpp
@@ -1,9 +1,8 @@
 //===-- X86MCInstLower.cpp - Convert X86 MachineInstr to an MCInst --------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -101,9 +100,7 @@ void X86AsmPrinter::StackMapShadowTracker::emitShadowPadding(
 }
 
 void X86AsmPrinter::EmitAndCountInstruction(MCInst &Inst) {
-  OutStreamer->EmitInstruction(Inst, getSubtargetInfo(),
-                               EnablePrintSchedInfo &&
-                                   !(Inst.getFlags() & X86::NO_SCHED_INFO));
+  OutStreamer->EmitInstruction(Inst, getSubtargetInfo());
   SMShadowTracker.count(Inst, getSubtargetInfo(), CodeEmitter.get());
 }
 
@@ -1861,8 +1858,7 @@ void X86AsmPrinter::EmitInstruction(const MachineInstr *MI) {
       SmallVector<int, 64> Mask;
       DecodePSHUFBMask(C, Width, Mask);
       if (!Mask.empty())
-        OutStreamer->AddComment(getShuffleComment(MI, SrcIdx, SrcIdx, Mask),
-                                !EnablePrintSchedInfo);
+        OutStreamer->AddComment(getShuffleComment(MI, SrcIdx, SrcIdx, Mask));
     }
     break;
   }
@@ -1934,8 +1930,7 @@ void X86AsmPrinter::EmitInstruction(const MachineInstr *MI) {
       SmallVector<int, 16> Mask;
       DecodeVPERMILPMask(C, ElSize, Width, Mask);
       if (!Mask.empty())
-        OutStreamer->AddComment(getShuffleComment(MI, SrcIdx, SrcIdx, Mask),
-                                !EnablePrintSchedInfo);
+        OutStreamer->AddComment(getShuffleComment(MI, SrcIdx, SrcIdx, Mask));
     }
     break;
   }
@@ -1966,8 +1961,7 @@ void X86AsmPrinter::EmitInstruction(const MachineInstr *MI) {
       SmallVector<int, 16> Mask;
       DecodeVPERMIL2PMask(C, (unsigned)CtrlOp.getImm(), ElSize, Width, Mask);
       if (!Mask.empty())
-        OutStreamer->AddComment(getShuffleComment(MI, 1, 2, Mask),
-                                !EnablePrintSchedInfo);
+        OutStreamer->AddComment(getShuffleComment(MI, 1, 2, Mask));
     }
     break;
   }
@@ -1984,8 +1978,7 @@ void X86AsmPrinter::EmitInstruction(const MachineInstr *MI) {
       SmallVector<int, 16> Mask;
       DecodeVPPERMMask(C, Width, Mask);
       if (!Mask.empty())
-        OutStreamer->AddComment(getShuffleComment(MI, 1, 2, Mask),
-                                !EnablePrintSchedInfo);
+        OutStreamer->AddComment(getShuffleComment(MI, 1, 2, Mask));
     }
     break;
   }
@@ -2002,7 +1995,7 @@ void X86AsmPrinter::EmitInstruction(const MachineInstr *MI) {
       CS << X86ATTInstPrinter::getRegisterName(DstOp.getReg()) << " = ";
       if (auto *CF = dyn_cast<ConstantFP>(C)) {
         CS << "0x" << CF->getValueAPF().bitcastToAPInt().toString(16, false);
-        OutStreamer->AddComment(CS.str(), !EnablePrintSchedInfo);
+        OutStreamer->AddComment(CS.str());
       }
     }
     break;
@@ -2099,7 +2092,7 @@ void X86AsmPrinter::EmitInstruction(const MachineInstr *MI) {
           }
         }
         CS << "]";
-        OutStreamer->AddComment(CS.str(), !EnablePrintSchedInfo);
+        OutStreamer->AddComment(CS.str());
       } else if (auto *CV = dyn_cast<ConstantVector>(C)) {
         CS << "<";
         for (int l = 0; l != NumLanes; ++l) {
@@ -2111,7 +2104,7 @@ void X86AsmPrinter::EmitInstruction(const MachineInstr *MI) {
           }
         }
         CS << ">";
-        OutStreamer->AddComment(CS.str(), !EnablePrintSchedInfo);
+        OutStreamer->AddComment(CS.str());
       }
     }
     break;
@@ -2198,14 +2191,12 @@ void X86AsmPrinter::EmitInstruction(const MachineInstr *MI) {
         printConstant(C, CS);
       }
       CS << "]";
-      OutStreamer->AddComment(CS.str(), !EnablePrintSchedInfo);
+      OutStreamer->AddComment(CS.str());
     }
   }
 
   MCInst TmpInst;
   MCInstLowering.Lower(MI, TmpInst);
-  if (MI->getAsmPrinterFlag(MachineInstr::NoSchedComment))
-    TmpInst.setFlags(TmpInst.getFlags() | X86::NO_SCHED_INFO);
 
   // Stackmap shadows cannot include branch targets, so we can count the bytes
   // in a call towards the shadow, but must ensure that the no thread returns
diff --git a/lib/Target/X86/X86MachineFunctionInfo.cpp b/lib/Target/X86/X86MachineFunctionInfo.cpp
index 5433033671f3d29518e94caafb48ab3ed23f7391..05f846bfb219e3bd4fa6a7d73c4e020ce28a00b9 100644
--- a/lib/Target/X86/X86MachineFunctionInfo.cpp
+++ b/lib/Target/X86/X86MachineFunctionInfo.cpp
@@ -1,9 +1,8 @@
 //===-- X86MachineFunctionInfo.cpp - X86 machine function info ------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/X86/X86MachineFunctionInfo.h b/lib/Target/X86/X86MachineFunctionInfo.h
index e1183bd14796c75a244edca8ae59ea77b1941a92..d7e535598d81f1dcfdb1f7f6a1ceedd8c04735dc 100644
--- a/lib/Target/X86/X86MachineFunctionInfo.h
+++ b/lib/Target/X86/X86MachineFunctionInfo.h
@@ -1,9 +1,8 @@
 //===-- X86MachineFunctionInfo.h - X86 machine function info ----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/X86/X86MacroFusion.cpp b/lib/Target/X86/X86MacroFusion.cpp
index 5c09597d04423a4f12fec66489a42e2e2412bfad..8c2d0fe6690de067b1009d631de4310815f1b5a5 100644
--- a/lib/Target/X86/X86MacroFusion.cpp
+++ b/lib/Target/X86/X86MacroFusion.cpp
@@ -1,9 +1,8 @@
 //===- X86MacroFusion.cpp - X86 Macro Fusion ------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/X86/X86MacroFusion.h b/lib/Target/X86/X86MacroFusion.h
index 97ef1d6d3b61e9feb011f4bcf708500a656713c0..d4ae54f657a5bf56dfd2d87de625e79f4e59d902 100644
--- a/lib/Target/X86/X86MacroFusion.h
+++ b/lib/Target/X86/X86MacroFusion.h
@@ -1,9 +1,8 @@
 //===- X86MacroFusion.h - X86 Macro Fusion --------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/X86/X86OptimizeLEAs.cpp b/lib/Target/X86/X86OptimizeLEAs.cpp
index b56d02b6bfb6fafde538896d28bfb96d2f3847eb..6cec0b806045bc5222fe7d2621b9ea93a34dcba0 100644
--- a/lib/Target/X86/X86OptimizeLEAs.cpp
+++ b/lib/Target/X86/X86OptimizeLEAs.cpp
@@ -1,9 +1,8 @@
 //===- X86OptimizeLEAs.cpp - optimize usage of LEA instructions -----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/X86/X86PadShortFunction.cpp b/lib/Target/X86/X86PadShortFunction.cpp
index 85b9aecc21068cad5636e7e9f02c7abf0892b225..99df3d60d98c5e6fc8b805795b8c3bb99761ec70 100644
--- a/lib/Target/X86/X86PadShortFunction.cpp
+++ b/lib/Target/X86/X86PadShortFunction.cpp
@@ -1,9 +1,8 @@
 //===-------- X86PadShortFunction.cpp - pad short functions -----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/X86/X86PfmCounters.td b/lib/Target/X86/X86PfmCounters.td
index a1a4210b5ebf8e4ef226218f19b58e4568ab8f31..5610f4bc887321718b39eecb417bea8746b72d89 100644
--- a/lib/Target/X86/X86PfmCounters.td
+++ b/lib/Target/X86/X86PfmCounters.td
@@ -1,9 +1,8 @@
 //===-- X86PfmCounters.td - X86 Hardware Counters ----------*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/X86/X86RegisterBankInfo.cpp b/lib/Target/X86/X86RegisterBankInfo.cpp
index 355291916ee8b41d97af9d58ffccff7e8676d26f..3ea4c59aa497a0a184deec44c6fac1c50d17a7fc 100644
--- a/lib/Target/X86/X86RegisterBankInfo.cpp
+++ b/lib/Target/X86/X86RegisterBankInfo.cpp
@@ -1,9 +1,8 @@
 //===- X86RegisterBankInfo.cpp -----------------------------------*- C++ -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
@@ -174,17 +173,25 @@ X86RegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
   case TargetOpcode::G_ADD:
   case TargetOpcode::G_SUB:
   case TargetOpcode::G_MUL:
-  case TargetOpcode::G_SHL:
-  case TargetOpcode::G_LSHR:
-  case TargetOpcode::G_ASHR:
     return getSameOperandsMapping(MI, false);
-    break;
   case TargetOpcode::G_FADD:
   case TargetOpcode::G_FSUB:
   case TargetOpcode::G_FMUL:
   case TargetOpcode::G_FDIV:
     return getSameOperandsMapping(MI, true);
-    break;
+  case TargetOpcode::G_SHL:
+  case TargetOpcode::G_LSHR:
+  case TargetOpcode::G_ASHR: {
+    const MachineFunction &MF = *MI.getParent()->getParent();
+    const MachineRegisterInfo &MRI = MF.getRegInfo();
+
+    unsigned NumOperands = MI.getNumOperands();
+    LLT Ty = MRI.getType(MI.getOperand(0).getReg());
+
+    auto Mapping = getValueMapping(getPartialMappingIdx(Ty, false), 3);
+    return getInstructionMapping(DefaultMappingID, 1, Mapping, NumOperands);
+
+  }
   default:
     break;
   }
diff --git a/lib/Target/X86/X86RegisterBankInfo.h b/lib/Target/X86/X86RegisterBankInfo.h
index e227880427f3c4a33a89a1a5437ba740a0833288..c1f3001c618064259c1fc21b162d4eb6c81c21af 100644
--- a/lib/Target/X86/X86RegisterBankInfo.h
+++ b/lib/Target/X86/X86RegisterBankInfo.h
@@ -1,9 +1,8 @@
 //===- X86RegisterBankInfo ---------------------------------------*- C++ -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/lib/Target/X86/X86RegisterBanks.td b/lib/Target/X86/X86RegisterBanks.td
index 6d17cd53a0c14373e79b358a7f5f167ff52ca8ba..74c515850ab15231894609648fd7eda85716fb7e 100644
--- a/lib/Target/X86/X86RegisterBanks.td
+++ b/lib/Target/X86/X86RegisterBanks.td
@@ -1,9 +1,8 @@
 //=- X86RegisterBank.td - Describe the AArch64 Banks -----*- tablegen -*-=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/X86/X86RegisterInfo.cpp b/lib/Target/X86/X86RegisterInfo.cpp
index 55842a4a209144140c8cc0a1d031e57ed0d1ffa4..9f5f22b56101d14f8de1f263f630f8304b9541e3 100644
--- a/lib/Target/X86/X86RegisterInfo.cpp
+++ b/lib/Target/X86/X86RegisterInfo.cpp
@@ -1,9 +1,8 @@
 //===-- X86RegisterInfo.cpp - X86 Register Information --------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -497,6 +496,9 @@ BitVector X86RegisterInfo::getReservedRegs(const MachineFunction &MF) const {
   BitVector Reserved(getNumRegs());
   const X86FrameLowering *TFI = getFrameLowering(MF);
 
+  // Set the floating point control register as reserved.
+  Reserved.set(X86::FPCW);
+
   // Set the stack-pointer register and its aliases as reserved.
   for (MCSubRegIterator I(X86::RSP, this, /*IncludeSelf=*/true); I.isValid();
        ++I)
@@ -760,3 +762,12 @@ X86RegisterInfo::getPtrSizedFrameRegister(const MachineFunction &MF) const {
     FrameReg = getX86SubSuperRegister(FrameReg, 32);
   return FrameReg;
 }
+
+unsigned
+X86RegisterInfo::getPtrSizedStackRegister(const MachineFunction &MF) const {
+  const X86Subtarget &Subtarget = MF.getSubtarget<X86Subtarget>();
+  unsigned StackReg = getStackRegister();
+  if (Subtarget.isTarget64BitILP32())
+    StackReg = getX86SubSuperRegister(StackReg, 32);
+  return StackReg;
+}
diff --git a/lib/Target/X86/X86RegisterInfo.h b/lib/Target/X86/X86RegisterInfo.h
index 29401dadead04edefc09ee1b86529b1f3d901da9..0f940e3f21bb05e3329ded6ab6b032d28181f999 100644
--- a/lib/Target/X86/X86RegisterInfo.h
+++ b/lib/Target/X86/X86RegisterInfo.h
@@ -1,9 +1,8 @@
 //===-- X86RegisterInfo.h - X86 Register Information Impl -------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -131,6 +130,7 @@ public:
   // Debug information queries.
   unsigned getFrameRegister(const MachineFunction &MF) const override;
   unsigned getPtrSizedFrameRegister(const MachineFunction &MF) const;
+  unsigned getPtrSizedStackRegister(const MachineFunction &MF) const;
   unsigned getStackRegister() const { return StackPtr; }
   unsigned getBaseRegister() const { return BasePtr; }
   /// Returns physical register used as frame pointer.
diff --git a/lib/Target/X86/X86RegisterInfo.td b/lib/Target/X86/X86RegisterInfo.td
index 0c1b05fd3abe7015d7fe23af93be7cbc486de7d5..d81f08b0f0610f8fdc774394fa4f18cbf32e521a 100644
--- a/lib/Target/X86/X86RegisterInfo.td
+++ b/lib/Target/X86/X86RegisterInfo.td
@@ -1,9 +1,8 @@
 //===- X86RegisterInfo.td - Describe the X86 Register File --*- tablegen -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -278,7 +277,7 @@ def K7 : X86Reg<"k7", 7>, DwarfRegNum<[125, 100, 100]>;
 // pseudo registers, but we still mark them as aliasing FP registers. That
 // way both kinds can be live without exceeding the stack depth. ST registers
 // are only live around inline assembly.
-def ST0 : X86Reg<"st(0)", 0>, DwarfRegNum<[33, 12, 11]>;
+def ST0 : X86Reg<"st", 0>, DwarfRegNum<[33, 12, 11]>;
 def ST1 : X86Reg<"st(1)", 1>, DwarfRegNum<[34, 13, 12]>;
 def ST2 : X86Reg<"st(2)", 2>, DwarfRegNum<[35, 14, 13]>;
 def ST3 : X86Reg<"st(3)", 3>, DwarfRegNum<[36, 15, 14]>;
@@ -288,7 +287,10 @@ def ST6 : X86Reg<"st(6)", 6>, DwarfRegNum<[39, 18, 17]>;
 def ST7 : X86Reg<"st(7)", 7>, DwarfRegNum<[40, 19, 18]>;
 
 // Floating-point status word
-def FPSW : X86Reg<"fpsw", 0>;
+def FPSW : X86Reg<"fpsr", 0>;
+
+// Floating-point control word
+def FPCW : X86Reg<"fpcr", 0>;
 
 // Status flags register.
 //
@@ -522,10 +524,16 @@ def FR64 : RegisterClass<"X86", [f64], 64, (add FR32)>;
 // faster on common hardware.  In reality, this should be controlled by a
 // command line option or something.
 
+
 def RFP32 : RegisterClass<"X86",[f32], 32, (sequence "FP%u", 0, 6)>;
 def RFP64 : RegisterClass<"X86",[f64], 32, (add RFP32)>;
 def RFP80 : RegisterClass<"X86",[f80], 32, (add RFP32)>;
 
+// st(7) may be is not allocatable.
+def RFP80_7 : RegisterClass<"X86",[f80], 32, (add FP7)> {
+  let isAllocatable = 0;
+}
+
 // Floating point stack registers (these are not allocatable by the
 // register allocator - the floating point stackifier is responsible
 // for transforming FPn allocations to STn registers)
@@ -533,6 +541,9 @@ def RST : RegisterClass<"X86", [f80, f64, f32], 32, (sequence "ST%u", 0, 7)> {
   let isAllocatable = 0;
 }
 
+// Helper to allow %st to print as %st(0) when its encoded in the instruction.
+def RSTi : RegisterOperand<RST, "printSTiRegOperand">;
+
 // Generic vector registers: VR64 and VR128.
 // Ensure that float types are declared first - only float is legal on SSE1.
 def VR64: RegisterClass<"X86", [x86mmx], 64, (sequence "MM%u", 0, 7)>;
diff --git a/lib/Target/X86/X86RetpolineThunks.cpp b/lib/Target/X86/X86RetpolineThunks.cpp
index 08994cccb21e69f78cd62c3a3f0490ad4bc51958..b435b22e8ac79dbeadc0d74433facd8d1f653a28 100644
--- a/lib/Target/X86/X86RetpolineThunks.cpp
+++ b/lib/Target/X86/X86RetpolineThunks.cpp
@@ -1,9 +1,8 @@
 //======- X86RetpolineThunks.cpp - Construct retpoline thunks for x86  --=====//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/lib/Target/X86/X86SchedBroadwell.td b/lib/Target/X86/X86SchedBroadwell.td
index 971a50196e45a477d2b30d9fd45daf4e1a09be3c..821948d2d85937b9b48c24d7d9cc8aa4332b11ae 100644
--- a/lib/Target/X86/X86SchedBroadwell.td
+++ b/lib/Target/X86/X86SchedBroadwell.td
@@ -1,9 +1,8 @@
 //=- X86SchedBroadwell.td - X86 Broadwell Scheduling ---------*- tablegen -*-=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -82,6 +81,8 @@ def : ReadAdvance<ReadAfterVecLd, 5>;
 def : ReadAdvance<ReadAfterVecXLd, 5>;
 def : ReadAdvance<ReadAfterVecYLd, 6>;
 
+def : ReadAdvance<ReadInt2Fpu, 0>;
+
 // Many SchedWrites are defined in pairs with and without a folded load.
 // Instructions with folded loads are usually micro-fused, so they only appear
 // as two micro-ops when queued in the reservation station.
diff --git a/lib/Target/X86/X86SchedHaswell.td b/lib/Target/X86/X86SchedHaswell.td
index 06a32fb0b1cdfb490ee7d8cce6872465bfaf739d..dc040e0e7d321e9f6ad2e1b4baa37a58c7904fdf 100644
--- a/lib/Target/X86/X86SchedHaswell.td
+++ b/lib/Target/X86/X86SchedHaswell.td
@@ -1,9 +1,8 @@
 //=- X86SchedHaswell.td - X86 Haswell Scheduling -------------*- tablegen -*-=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -87,6 +86,8 @@ def : ReadAdvance<ReadAfterVecLd, 5>;
 def : ReadAdvance<ReadAfterVecXLd, 6>;
 def : ReadAdvance<ReadAfterVecYLd, 7>;
 
+def : ReadAdvance<ReadInt2Fpu, 0>;
+
 // Many SchedWrites are defined in pairs with and without a folded load.
 // Instructions with folded loads are usually micro-fused, so they only appear
 // as two micro-ops when queued in the reservation station.
diff --git a/lib/Target/X86/X86SchedPredicates.td b/lib/Target/X86/X86SchedPredicates.td
index 1c7f24375f612b64aaef3a7eec493e93dc02be88..62ed351b194b2d0865434e12e7a25443a38a4e3e 100644
--- a/lib/Target/X86/X86SchedPredicates.td
+++ b/lib/Target/X86/X86SchedPredicates.td
@@ -1,9 +1,8 @@
 //===-- X86SchedPredicates.td - X86 Scheduling Predicates --*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/X86/X86SchedSandyBridge.td b/lib/Target/X86/X86SchedSandyBridge.td
index 9dbf0976989f83595ca0d715bd12bd3bd95b544c..503b905842b3646505e718e9b1aa67447f40f170 100644
--- a/lib/Target/X86/X86SchedSandyBridge.td
+++ b/lib/Target/X86/X86SchedSandyBridge.td
@@ -1,9 +1,8 @@
 //=- X86SchedSandyBridge.td - X86 Sandy Bridge Scheduling ----*- tablegen -*-=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -77,6 +76,8 @@ def : ReadAdvance<ReadAfterVecLd, 5>;
 def : ReadAdvance<ReadAfterVecXLd, 6>;
 def : ReadAdvance<ReadAfterVecYLd, 7>;
 
+def : ReadAdvance<ReadInt2Fpu, 0>;
+
 // Many SchedWrites are defined in pairs with and without a folded load.
 // Instructions with folded loads are usually micro-fused, so they only appear
 // as two micro-ops when queued in the reservation station.
diff --git a/lib/Target/X86/X86SchedSkylakeClient.td b/lib/Target/X86/X86SchedSkylakeClient.td
index 2c9eb7516085c1d6e63da3a91a142e8c2fc314be..71045897376fab24da094baac62f041742b62cbc 100644
--- a/lib/Target/X86/X86SchedSkylakeClient.td
+++ b/lib/Target/X86/X86SchedSkylakeClient.td
@@ -1,9 +1,8 @@
 //=- X86SchedSkylake.td - X86 Skylake Client Scheduling ------*- tablegen -*-=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -81,6 +80,8 @@ def : ReadAdvance<ReadAfterVecLd, 5>;
 def : ReadAdvance<ReadAfterVecXLd, 6>;
 def : ReadAdvance<ReadAfterVecYLd, 7>;
 
+def : ReadAdvance<ReadInt2Fpu, 0>;
+
 // Many SchedWrites are defined in pairs with and without a folded load.
 // Instructions with folded loads are usually micro-fused, so they only appear
 // as two micro-ops when queued in the reservation station.
diff --git a/lib/Target/X86/X86SchedSkylakeServer.td b/lib/Target/X86/X86SchedSkylakeServer.td
index ec8e4db02d8aa59474082ea9cc3a6e7d99cd49f2..8bd80078891dd4dad582cefc06144086908e37ae 100644
--- a/lib/Target/X86/X86SchedSkylakeServer.td
+++ b/lib/Target/X86/X86SchedSkylakeServer.td
@@ -1,9 +1,8 @@
 //=- X86SchedSkylake.td - X86 Skylake Server Scheduling ------*- tablegen -*-=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -81,6 +80,8 @@ def : ReadAdvance<ReadAfterVecLd, 5>;
 def : ReadAdvance<ReadAfterVecXLd, 6>;
 def : ReadAdvance<ReadAfterVecYLd, 7>;
 
+def : ReadAdvance<ReadInt2Fpu, 0>;
+
 // Many SchedWrites are defined in pairs with and without a folded load.
 // Instructions with folded loads are usually micro-fused, so they only appear
 // as two micro-ops when queued in the reservation station.
diff --git a/lib/Target/X86/X86Schedule.td b/lib/Target/X86/X86Schedule.td
index 25aa83f96d3a113ec9c2a1bae4dbeb7b7fc3fe20..f50cb6210466aab06a80fb23560962cc9a4c5872 100644
--- a/lib/Target/X86/X86Schedule.td
+++ b/lib/Target/X86/X86Schedule.td
@@ -1,9 +1,8 @@
 //===-- X86Schedule.td - X86 Scheduling Definitions --------*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -18,6 +17,12 @@ def ReadAfterVecLd : SchedRead;
 def ReadAfterVecXLd : SchedRead;
 def ReadAfterVecYLd : SchedRead;
 
+// Instructions that move data between general purpose registers and vector
+// registers may be subject to extra latency due to data bypass delays.
+// This SchedRead describes a bypass delay caused by data being moved from the
+// integer unit to the floating point unit.
+def ReadInt2Fpu : SchedRead;
+
 // Instructions with both a load and a store folded are modeled as a folded
 // load + WriteRMW.
 def WriteRMW : SchedWrite;
diff --git a/lib/Target/X86/X86ScheduleAtom.td b/lib/Target/X86/X86ScheduleAtom.td
index 1589ff2ef402a7fc3029d3a9f66714a51f38bb6c..bf50aeee1dffb29876d33ad5bf9df73112ad20b0 100644
--- a/lib/Target/X86/X86ScheduleAtom.td
+++ b/lib/Target/X86/X86ScheduleAtom.td
@@ -1,9 +1,8 @@
 //===- X86ScheduleAtom.td - X86 Atom Scheduling Definitions -*- tablegen -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -47,6 +46,8 @@ def : ReadAdvance<ReadAfterVecLd, 3>;
 def : ReadAdvance<ReadAfterVecXLd, 3>;
 def : ReadAdvance<ReadAfterVecYLd, 3>;
 
+def : ReadAdvance<ReadInt2Fpu, 0>;
+
 // Many SchedWrites are defined in pairs with and without a folded load.
 // Instructions with folded loads are usually micro-fused, so they only appear
 // as two micro-ops when dispatched by the schedulers.
diff --git a/lib/Target/X86/X86ScheduleBdVer2.td b/lib/Target/X86/X86ScheduleBdVer2.td
index 5798e1b2671b9581558c5e7e764485466821246c..8e8fc6fd1ff1f583a80d62b2e6d8e94d086b0fd5 100644
--- a/lib/Target/X86/X86ScheduleBdVer2.td
+++ b/lib/Target/X86/X86ScheduleBdVer2.td
@@ -1,9 +1,8 @@
 //=- X86ScheduleBdVer2.td - X86 BdVer2 (Piledriver) Scheduling * tablegen -*-=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -251,6 +250,11 @@ def : ReadAdvance<ReadAfterVecLd, 5>;
 def : ReadAdvance<ReadAfterVecXLd, 5>;
 def : ReadAdvance<ReadAfterVecYLd, 5>;
 
+// Transfer from int domain to ivec domain incurs additional latency of 8..10cy
+// Reference: Agner, Microarchitecture, "AMD Bulldozer, Piledriver, Steamroller
+// and Excavator pipeline", "Data delay between different execution domains"
+def : ReadAdvance<ReadInt2Fpu, -10>;
+
 // A folded store needs a cycle on the PdStore for the store data.
 def : WriteRes<WriteRMW, [PdStore]>;
 
diff --git a/lib/Target/X86/X86ScheduleBtVer2.td b/lib/Target/X86/X86ScheduleBtVer2.td
index 33a6b01546d7656597ca4e5e158f4a5f146fc0d5..7931a956b52516a58c72a1d6d1c30d709f8f3420 100644
--- a/lib/Target/X86/X86ScheduleBtVer2.td
+++ b/lib/Target/X86/X86ScheduleBtVer2.td
@@ -1,9 +1,8 @@
 //=- X86ScheduleBtVer2.td - X86 BtVer2 (Jaguar) Scheduling ---*- tablegen -*-=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -109,6 +108,11 @@ def : ReadAdvance<ReadAfterVecLd, 5>;
 def : ReadAdvance<ReadAfterVecXLd, 5>;
 def : ReadAdvance<ReadAfterVecYLd, 5>;
 
+/// "Additional 6 cycle transfer operation which moves a floating point
+/// operation input value from the integer unit to the floating point unit.
+/// Reference: AMDfam16h SOG (Appendix A "Instruction Latencies", Section A.2).
+def : ReadAdvance<ReadInt2Fpu, -6>;
+
 // Many SchedWrites are defined in pairs with and without a folded load.
 // Instructions with folded loads are usually micro-fused, so they only appear
 // as two micro-ops when dispatched by the schedulers.
@@ -174,6 +178,8 @@ multiclass JWriteResYMMPair<X86FoldableSchedWrite SchedRW,
   }
 }
 
+// Instructions that have local forwarding disabled have an extra +1cy latency.
+
 // A folded store needs a cycle on the SAGU for the store data,
 // most RMW instructions don't need an extra uop.
 defm : X86WriteRes<WriteRMW, [JSAGU], 1, [1], 0>;
@@ -262,14 +268,13 @@ defm : X86WriteRes<WriteSHDmrcl,[JLAGU, JALU01], 9, [1, 22], 8>;
 // Loads, stores, and moves, not folded with other operations.
 ////////////////////////////////////////////////////////////////////////////////
 
-def : WriteRes<WriteLoad,    [JLAGU]> { let Latency = 5; }
+def : WriteRes<WriteLoad,    [JLAGU]> { let Latency = 3; }
 def : WriteRes<WriteStore,   [JSAGU]>;
 def : WriteRes<WriteStoreNT, [JSAGU]>;
 def : WriteRes<WriteMove,    [JALU01]>;
 
 // Load/store MXCSR.
-// FIXME: These are copy and pasted from WriteLoad/Store.
-def : WriteRes<WriteLDMXCSR, [JLAGU]> { let Latency = 5; }
+def : WriteRes<WriteLDMXCSR, [JLAGU]> { let Latency = 3; }
 def : WriteRes<WriteSTMXCSR, [JSAGU]>;
 
 // Treat misc copies as a move.
@@ -400,8 +405,8 @@ defm : X86WriteResPairUnsupported<WriteFTestZ>;
 defm : JWriteResFpuPair<WriteFShuffle,    [JFPU01, JFPX],  1>;
 defm : JWriteResYMMPair<WriteFShuffleY,   [JFPU01, JFPX],  1, [2, 2], 2>;
 defm : X86WriteResPairUnsupported<WriteFShuffleZ>;
-defm : JWriteResFpuPair<WriteFVarShuffle, [JFPU01, JFPX],  2, [1, 4], 3>;
-defm : JWriteResYMMPair<WriteFVarShuffleY,[JFPU01, JFPX],  3, [2, 6], 6>;
+defm : JWriteResFpuPair<WriteFVarShuffle, [JFPU01, JFPX],  3, [1, 4], 3>; // +1cy latency.
+defm : JWriteResYMMPair<WriteFVarShuffleY,[JFPU01, JFPX],  4, [2, 6], 6>; // +1cy latency.
 defm : X86WriteResPairUnsupported<WriteFVarShuffleZ>;
 defm : JWriteResFpuPair<WriteFBlend,      [JFPU01, JFPX],  1>;
 defm : JWriteResYMMPair<WriteFBlendY,     [JFPU01, JFPX],  1, [2, 2], 2>;
@@ -425,12 +430,13 @@ defm : JWriteResFpuPair<WriteCvtPD2I,      [JFPU1, JSTC], 3, [1,1], 1>;
 defm : JWriteResYMMPair<WriteCvtPD2IY,     [JFPU1, JSTC, JFPX], 6, [2,2,4], 3>;
 defm : X86WriteResPairUnsupported<WriteCvtPD2IZ>;
 
-// FIXME: f+3 ST, LD+STC latency
-defm : JWriteResFpuPair<WriteCvtI2SS,      [JFPU1, JSTC], 9, [1,1], 2>;
+defm : X86WriteRes<WriteCvtI2SS,           [JFPU1, JSTC], 4, [1,1], 2>;
+defm : X86WriteRes<WriteCvtI2SSLd,         [JLAGU, JFPU1, JSTC], 9, [1,1,1], 1>;
 defm : JWriteResFpuPair<WriteCvtI2PS,      [JFPU1, JSTC], 3, [1,1], 1>;
 defm : JWriteResYMMPair<WriteCvtI2PSY,     [JFPU1, JSTC], 3, [2,2], 2>;
 defm : X86WriteResPairUnsupported<WriteCvtI2PSZ>;
-defm : JWriteResFpuPair<WriteCvtI2SD,      [JFPU1, JSTC], 9, [1,1], 2>;
+defm : X86WriteRes<WriteCvtI2SD,           [JFPU1, JSTC], 4, [1,1], 2>;
+defm : X86WriteRes<WriteCvtI2SDLd,         [JLAGU, JFPU1, JSTC], 9, [1,1,1], 1>;
 defm : JWriteResFpuPair<WriteCvtI2PD,      [JFPU1, JSTC], 3, [1,1], 1>;
 defm : JWriteResYMMPair<WriteCvtI2PDY,     [JFPU1, JSTC], 3, [2,2], 2>;
 defm : X86WriteResPairUnsupported<WriteCvtI2PDZ>;
@@ -487,11 +493,11 @@ defm : JWriteResFpuPair<WriteVecALUX,     [JFPU01, JVALU], 1>;
 defm : X86WriteResPairUnsupported<WriteVecALUY>;
 defm : X86WriteResPairUnsupported<WriteVecALUZ>;
 defm : JWriteResFpuPair<WriteVecShift,    [JFPU01, JVALU], 1>;
-defm : JWriteResFpuPair<WriteVecShiftX,   [JFPU01, JVALU], 1>;
+defm : JWriteResFpuPair<WriteVecShiftX,   [JFPU01, JVALU], 2>; // +1cy latency.
 defm : X86WriteResPairUnsupported<WriteVecShiftY>;
 defm : X86WriteResPairUnsupported<WriteVecShiftZ>;
 defm : JWriteResFpuPair<WriteVecShiftImm, [JFPU01, JVALU], 1>;
-defm : JWriteResFpuPair<WriteVecShiftImmX,[JFPU01, JVALU], 1>;
+defm : JWriteResFpuPair<WriteVecShiftImmX,[JFPU01, JVALU], 2>; // +1cy latency.
 defm : X86WriteResPairUnsupported<WriteVecShiftImmY>;
 defm : X86WriteResPairUnsupported<WriteVecShiftImmZ>;
 defm : X86WriteResPairUnsupported<WriteVarVecShift>;
@@ -540,7 +546,7 @@ defm : X86WriteResPairUnsupported<WriteVarShuffle256>;
 // Vector insert/extract operations.
 ////////////////////////////////////////////////////////////////////////////////
 
-defm : X86WriteRes<WriteVecInsert,      [JFPU01, JVALU], 7, [1,1], 2>;
+defm : X86WriteRes<WriteVecInsert,      [JFPU01, JVALU], 1, [1,1], 2>;
 defm : X86WriteRes<WriteVecInsertLd,    [JFPU01, JVALU, JLAGU], 4, [1,1,1], 1>;
 defm : X86WriteRes<WriteVecExtract,     [JFPU0, JFPA, JALU0], 3, [1,1,1], 1>;
 defm : X86WriteRes<WriteVecExtractSt,   [JFPU1, JSTC, JSAGU], 3, [1,1,1], 1>;
@@ -575,10 +581,10 @@ defm : JWriteResFpuPair<WriteAESDecEnc,   [JFPU01, JVALU, JFPU0, JVIMUL], 3, [1,
 // Horizontal add/sub  instructions.
 ////////////////////////////////////////////////////////////////////////////////
 
-defm : JWriteResFpuPair<WriteFHAdd,         [JFPU0, JFPA], 3>;
-defm : JWriteResYMMPair<WriteFHAddY,        [JFPU0, JFPA], 3, [2,2], 2>;
-defm : JWriteResFpuPair<WritePHAdd,       [JFPU01, JVALU], 1>;
-defm : JWriteResFpuPair<WritePHAddX,      [JFPU01, JVALU], 1>;
+defm : JWriteResFpuPair<WriteFHAdd,         [JFPU0, JFPA], 4>;            // +1cy latency.
+defm : JWriteResYMMPair<WriteFHAddY,        [JFPU0, JFPA], 4, [2,2], 2>;  // +1cy latency.
+defm : JWriteResFpuPair<WritePHAdd,         [JFPU01, JVALU], 1>;
+defm : JWriteResFpuPair<WritePHAddX,        [JFPU01, JVALU], 2>;          // +1cy latency.
 defm : X86WriteResPairUnsupported<WritePHAddY>;
 
 ////////////////////////////////////////////////////////////////////////////////
diff --git a/lib/Target/X86/X86ScheduleSLM.td b/lib/Target/X86/X86ScheduleSLM.td
index fcaff7cf810ff0a64db32fb44a5f1967a12f67e7..fc150fca5458ac8c2bc3d572da0eefb14d7fa0dc 100644
--- a/lib/Target/X86/X86ScheduleSLM.td
+++ b/lib/Target/X86/X86ScheduleSLM.td
@@ -1,9 +1,8 @@
 //=- X86ScheduleSLM.td - X86 Silvermont Scheduling -----------*- tablegen -*-=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -53,6 +52,8 @@ def : ReadAdvance<ReadAfterVecLd, 3>;
 def : ReadAdvance<ReadAfterVecXLd, 3>;
 def : ReadAdvance<ReadAfterVecYLd, 3>;
 
+def : ReadAdvance<ReadInt2Fpu, 0>;
+
 // Many SchedWrites are defined in pairs with and without a folded load.
 // Instructions with folded loads are usually micro-fused, so they only appear
 // as two micro-ops when queued in the reservation station.
diff --git a/lib/Target/X86/X86ScheduleZnver1.td b/lib/Target/X86/X86ScheduleZnver1.td
index a866f843106b641439136fbf30324a3d3a2b3fd9..1a75281cf0cd0c2d0943555509167c3f4dff789c 100644
--- a/lib/Target/X86/X86ScheduleZnver1.td
+++ b/lib/Target/X86/X86ScheduleZnver1.td
@@ -1,9 +1,8 @@
 //=- X86ScheduleZnver1.td - X86 Znver1 Scheduling -------------*- tablegen -*-=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -95,6 +94,8 @@ def : ReadAdvance<ReadAfterVecLd, 8>;
 def : ReadAdvance<ReadAfterVecXLd, 8>;
 def : ReadAdvance<ReadAfterVecYLd, 8>;
 
+def : ReadAdvance<ReadInt2Fpu, 0>;
+
 // The Integer PRF for Zen is 168 entries, and it holds the architectural and
 // speculative version of the 64-bit integer registers.
 // Reference: "Software Optimization Guide for AMD Family 17h Processors"
diff --git a/lib/Target/X86/X86SelectionDAGInfo.cpp b/lib/Target/X86/X86SelectionDAGInfo.cpp
index 008a9ec2ba3ce35ae6184fe73541fc2b5aaeadf2..fea4d84e98c5963bfe860f3dd268b3cda9bbe0e5 100644
--- a/lib/Target/X86/X86SelectionDAGInfo.cpp
+++ b/lib/Target/X86/X86SelectionDAGInfo.cpp
@@ -1,9 +1,8 @@
 //===-- X86SelectionDAGInfo.cpp - X86 SelectionDAG Info -------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/X86/X86SelectionDAGInfo.h b/lib/Target/X86/X86SelectionDAGInfo.h
index f4a285a5f916f593ccfa7b8ba196be22db483c02..0f2d979f91e383958fa33800331be9311eb1cb6f 100644
--- a/lib/Target/X86/X86SelectionDAGInfo.h
+++ b/lib/Target/X86/X86SelectionDAGInfo.h
@@ -1,9 +1,8 @@
 //===-- X86SelectionDAGInfo.h - X86 SelectionDAG Info -----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/X86/X86ShuffleDecodeConstantPool.cpp b/lib/Target/X86/X86ShuffleDecodeConstantPool.cpp
index 720be8afa62c2c44b7e8e3e5d6985a95e083163c..a202fc63637b9576bffc2d9320b7559f0771a1a5 100644
--- a/lib/Target/X86/X86ShuffleDecodeConstantPool.cpp
+++ b/lib/Target/X86/X86ShuffleDecodeConstantPool.cpp
@@ -1,9 +1,8 @@
 //===-- X86ShuffleDecodeConstantPool.cpp - X86 shuffle decode -------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/X86/X86ShuffleDecodeConstantPool.h b/lib/Target/X86/X86ShuffleDecodeConstantPool.h
index b08c31935d281011e2fb0a5eb9fe886b0c0a96e8..29634151757957f8e006f140d94eb4133ecebc50 100644
--- a/lib/Target/X86/X86ShuffleDecodeConstantPool.h
+++ b/lib/Target/X86/X86ShuffleDecodeConstantPool.h
@@ -1,9 +1,8 @@
 //===-- X86ShuffleDecodeConstantPool.h - X86 shuffle decode -----*-C++-*---===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/X86/X86SpeculativeLoadHardening.cpp b/lib/Target/X86/X86SpeculativeLoadHardening.cpp
index a729161a1beb5c0fe90210ba3a368fe9f74cb157..1d94f2a80a0b3126227837673a60acb15a015162 100644
--- a/lib/Target/X86/X86SpeculativeLoadHardening.cpp
+++ b/lib/Target/X86/X86SpeculativeLoadHardening.cpp
@@ -1,9 +1,8 @@
 //====- X86SpeculativeLoadHardening.cpp - A Spectre v1 mitigation ---------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/lib/Target/X86/X86Subtarget.cpp b/lib/Target/X86/X86Subtarget.cpp
index 0c9ce8802e1bcfa0ecb0d1e57fc36e0c7b7ac6ed..d6ffad0137f0e3088e806ca6c38dd81e3c418b04 100644
--- a/lib/Target/X86/X86Subtarget.cpp
+++ b/lib/Target/X86/X86Subtarget.cpp
@@ -1,9 +1,8 @@
 //===-- X86Subtarget.cpp - X86 Subtarget Information ----------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/X86/X86Subtarget.h b/lib/Target/X86/X86Subtarget.h
index b1103f823e7f97fa19b64accb3d8497deda8be57..6e2e4708005c1131d5bec46842d624ed56b79af8 100644
--- a/lib/Target/X86/X86Subtarget.h
+++ b/lib/Target/X86/X86Subtarget.h
@@ -1,9 +1,8 @@
 //===-- X86Subtarget.h - Define Subtarget for the X86 ----------*- C++ -*--===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -834,9 +833,6 @@ public:
   /// Enable the MachineScheduler pass for all X86 subtargets.
   bool enableMachineScheduler() const override { return true; }
 
-  // TODO: Update the regression tests and return true.
-  bool supportPrintSchedInfo() const override { return false; }
-
   bool enableEarlyIfConversion() const override;
 
   AntiDepBreakMode getAntiDepBreakMode() const override {
diff --git a/lib/Target/X86/X86TargetMachine.cpp b/lib/Target/X86/X86TargetMachine.cpp
index afcb49dc22632f8c475f1dac861c2822c16601c1..e49615f2c1a5250878498b7708173d85cf7214d7 100644
--- a/lib/Target/X86/X86TargetMachine.cpp
+++ b/lib/Target/X86/X86TargetMachine.cpp
@@ -1,9 +1,8 @@
 //===-- X86TargetMachine.cpp - Define TargetMachine for the X86 -----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/X86/X86TargetMachine.h b/lib/Target/X86/X86TargetMachine.h
index f5b45da0c3dc2225c58cb014faf23892f319fc07..b999e2e86af68b96cfd361af5229ac521b3cc3ba 100644
--- a/lib/Target/X86/X86TargetMachine.h
+++ b/lib/Target/X86/X86TargetMachine.h
@@ -1,9 +1,8 @@
 //===-- X86TargetMachine.h - Define TargetMachine for the X86 ---*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/X86/X86TargetObjectFile.cpp b/lib/Target/X86/X86TargetObjectFile.cpp
index 505c4fa07b774355def69965c64e91fe7dd9a3f4..92e0779c2e74ad7bb049e7b336830473ecef9d6b 100644
--- a/lib/Target/X86/X86TargetObjectFile.cpp
+++ b/lib/Target/X86/X86TargetObjectFile.cpp
@@ -1,9 +1,8 @@
 //===-- X86TargetObjectFile.cpp - X86 Object Info -------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/X86/X86TargetObjectFile.h b/lib/Target/X86/X86TargetObjectFile.h
index d045094edb1eb0aa9835a872a723602f20abb972..13d7b4ad70d64baf408637a9d892db1f75ac3207 100644
--- a/lib/Target/X86/X86TargetObjectFile.h
+++ b/lib/Target/X86/X86TargetObjectFile.h
@@ -1,9 +1,8 @@
 //===-- X86TargetObjectFile.h - X86 Object Info -----------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/X86/X86TargetTransformInfo.cpp b/lib/Target/X86/X86TargetTransformInfo.cpp
index 7889322159262d3962050614431e93a9798f4473..9ae4a928fb5c547df9727acda9a06d4c1753721d 100644
--- a/lib/Target/X86/X86TargetTransformInfo.cpp
+++ b/lib/Target/X86/X86TargetTransformInfo.cpp
@@ -1,9 +1,8 @@
 //===-- X86TargetTransformInfo.cpp - X86 specific TTI pass ----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
@@ -147,6 +146,13 @@ unsigned X86TTIImpl::getRegisterBitWidth(bool Vector) const {
   return 32;
 }
 
+// Use horizontal 128-bit operations, which use low and high
+// 64-bit parts of vector register. This also allows vectorizer
+// to use partial vector operations.
+unsigned X86TTIImpl::getMinVectorRegisterBitWidth() const {
+  return 64;
+}
+
 unsigned X86TTIImpl::getLoadStoreVecRegBitWidth(unsigned) const {
   return getRegisterBitWidth(true);
 }
@@ -832,6 +838,12 @@ int X86TTIImpl::getArithmeticInstrCost(
     { ISD::FDIV, MVT::v4f32,      39 }, // Pentium IV from http://www.agner.org/
     { ISD::FDIV, MVT::f64,        38 }, // Pentium IV from http://www.agner.org/
     { ISD::FDIV, MVT::v2f64,      69 }, // Pentium IV from http://www.agner.org/
+
+    { ISD::FADD, MVT::f32,         2 }, // Pentium IV from http://www.agner.org/
+    { ISD::FADD, MVT::f64,         2 }, // Pentium IV from http://www.agner.org/
+
+    { ISD::FSUB, MVT::f32,         2 }, // Pentium IV from http://www.agner.org/
+    { ISD::FSUB, MVT::f64,         2 }, // Pentium IV from http://www.agner.org/
   };
 
   if (ST->hasSSE2())
@@ -841,6 +853,20 @@ int X86TTIImpl::getArithmeticInstrCost(
   static const CostTblEntry SSE1CostTable[] = {
     { ISD::FDIV, MVT::f32,   17 }, // Pentium III from http://www.agner.org/
     { ISD::FDIV, MVT::v4f32, 34 }, // Pentium III from http://www.agner.org/
+
+    { ISD::FADD, MVT::f32,    1 }, // Pentium III from http://www.agner.org/
+    { ISD::FADD, MVT::v4f32,  2 }, // Pentium III from http://www.agner.org/
+
+    { ISD::FSUB, MVT::f32,    1 }, // Pentium III from http://www.agner.org/
+    { ISD::FSUB, MVT::v4f32,  2 }, // Pentium III from http://www.agner.org/
+
+    { ISD::ADD, MVT::i8,      1 }, // Pentium III from http://www.agner.org/
+    { ISD::ADD, MVT::i16,     1 }, // Pentium III from http://www.agner.org/
+    { ISD::ADD, MVT::i32,     1 }, // Pentium III from http://www.agner.org/
+
+    { ISD::SUB, MVT::i8,      1 }, // Pentium III from http://www.agner.org/
+    { ISD::SUB, MVT::i16,     1 }, // Pentium III from http://www.agner.org/
+    { ISD::SUB, MVT::i32,     1 }, // Pentium III from http://www.agner.org/
   };
 
   if (ST->hasSSE1())
@@ -1631,17 +1657,77 @@ int X86TTIImpl::getCmpSelInstrCost(unsigned Opcode, Type *ValTy, Type *CondTy,
   int ISD = TLI->InstructionOpcodeToISD(Opcode);
   assert(ISD && "Invalid opcode");
 
-  static const CostTblEntry SSE2CostTbl[] = {
-    { ISD::SETCC,   MVT::v2i64,   8 },
-    { ISD::SETCC,   MVT::v4i32,   1 },
-    { ISD::SETCC,   MVT::v8i16,   1 },
-    { ISD::SETCC,   MVT::v16i8,   1 },
+  unsigned ExtraCost = 0;
+  if (I && (Opcode == Instruction::ICmp || Opcode == Instruction::FCmp)) {
+    // Some vector comparison predicates cost extra instructions.
+    if (MTy.isVector() &&
+        !((ST->hasXOP() && (!ST->hasAVX2() || MTy.is128BitVector())) ||
+          (ST->hasAVX512() && 32 <= MTy.getScalarSizeInBits()) ||
+          ST->hasBWI())) {
+      switch (cast<CmpInst>(I)->getPredicate()) {
+      case CmpInst::Predicate::ICMP_NE:
+        // xor(cmpeq(x,y),-1)
+        ExtraCost = 1;
+        break;
+      case CmpInst::Predicate::ICMP_SGE:
+      case CmpInst::Predicate::ICMP_SLE:
+        // xor(cmpgt(x,y),-1)
+        ExtraCost = 1;
+        break;
+      case CmpInst::Predicate::ICMP_ULT:
+      case CmpInst::Predicate::ICMP_UGT:
+        // cmpgt(xor(x,signbit),xor(y,signbit))
+        // xor(cmpeq(pmaxu(x,y),x),-1)
+        ExtraCost = 2;
+        break;
+      case CmpInst::Predicate::ICMP_ULE:
+      case CmpInst::Predicate::ICMP_UGE:
+        if ((ST->hasSSE41() && MTy.getScalarSizeInBits() == 32) ||
+            (ST->hasSSE2() && MTy.getScalarSizeInBits() < 32)) {
+          // cmpeq(psubus(x,y),0)
+          // cmpeq(pminu(x,y),x)
+          ExtraCost = 1;
+        } else {
+          // xor(cmpgt(xor(x,signbit),xor(y,signbit)),-1)
+          ExtraCost = 3;
+        }
+        break;
+      default:
+        break;
+      }
+    }
+  }
+
+  static const CostTblEntry AVX512BWCostTbl[] = {
+    { ISD::SETCC,   MVT::v32i16,  1 },
+    { ISD::SETCC,   MVT::v64i8,   1 },
+
+    { ISD::SELECT,  MVT::v32i16,  1 },
+    { ISD::SELECT,  MVT::v64i8,   1 },
   };
 
-  static const CostTblEntry SSE42CostTbl[] = {
-    { ISD::SETCC,   MVT::v2f64,   1 },
-    { ISD::SETCC,   MVT::v4f32,   1 },
-    { ISD::SETCC,   MVT::v2i64,   1 },
+  static const CostTblEntry AVX512CostTbl[] = {
+    { ISD::SETCC,   MVT::v8i64,   1 },
+    { ISD::SETCC,   MVT::v16i32,  1 },
+    { ISD::SETCC,   MVT::v8f64,   1 },
+    { ISD::SETCC,   MVT::v16f32,  1 },
+
+    { ISD::SELECT,  MVT::v8i64,   1 },
+    { ISD::SELECT,  MVT::v16i32,  1 },
+    { ISD::SELECT,  MVT::v8f64,   1 },
+    { ISD::SELECT,  MVT::v16f32,  1 },
+  };
+
+  static const CostTblEntry AVX2CostTbl[] = {
+    { ISD::SETCC,   MVT::v4i64,   1 },
+    { ISD::SETCC,   MVT::v8i32,   1 },
+    { ISD::SETCC,   MVT::v16i16,  1 },
+    { ISD::SETCC,   MVT::v32i8,   1 },
+
+    { ISD::SELECT,  MVT::v4i64,   1 }, // pblendvb
+    { ISD::SELECT,  MVT::v8i32,   1 }, // pblendvb
+    { ISD::SELECT,  MVT::v16i16,  1 }, // pblendvb
+    { ISD::SELECT,  MVT::v32i8,   1 }, // pblendvb
   };
 
   static const CostTblEntry AVX1CostTbl[] = {
@@ -1652,50 +1738,83 @@ int X86TTIImpl::getCmpSelInstrCost(unsigned Opcode, Type *ValTy, Type *CondTy,
     { ISD::SETCC,   MVT::v8i32,   4 },
     { ISD::SETCC,   MVT::v16i16,  4 },
     { ISD::SETCC,   MVT::v32i8,   4 },
+
+    { ISD::SELECT,  MVT::v4f64,   1 }, // vblendvpd
+    { ISD::SELECT,  MVT::v8f32,   1 }, // vblendvps
+    { ISD::SELECT,  MVT::v4i64,   1 }, // vblendvpd
+    { ISD::SELECT,  MVT::v8i32,   1 }, // vblendvps
+    { ISD::SELECT,  MVT::v16i16,  3 }, // vandps + vandnps + vorps
+    { ISD::SELECT,  MVT::v32i8,   3 }, // vandps + vandnps + vorps
   };
 
-  static const CostTblEntry AVX2CostTbl[] = {
-    { ISD::SETCC,   MVT::v4i64,   1 },
-    { ISD::SETCC,   MVT::v8i32,   1 },
-    { ISD::SETCC,   MVT::v16i16,  1 },
-    { ISD::SETCC,   MVT::v32i8,   1 },
+  static const CostTblEntry SSE42CostTbl[] = {
+    { ISD::SETCC,   MVT::v2f64,   1 },
+    { ISD::SETCC,   MVT::v4f32,   1 },
+    { ISD::SETCC,   MVT::v2i64,   1 },
   };
 
-  static const CostTblEntry AVX512CostTbl[] = {
-    { ISD::SETCC,   MVT::v8i64,   1 },
-    { ISD::SETCC,   MVT::v16i32,  1 },
-    { ISD::SETCC,   MVT::v8f64,   1 },
-    { ISD::SETCC,   MVT::v16f32,  1 },
+  static const CostTblEntry SSE41CostTbl[] = {
+    { ISD::SELECT,  MVT::v2f64,   1 }, // blendvpd
+    { ISD::SELECT,  MVT::v4f32,   1 }, // blendvps
+    { ISD::SELECT,  MVT::v2i64,   1 }, // pblendvb
+    { ISD::SELECT,  MVT::v4i32,   1 }, // pblendvb
+    { ISD::SELECT,  MVT::v8i16,   1 }, // pblendvb
+    { ISD::SELECT,  MVT::v16i8,   1 }, // pblendvb
   };
 
-  static const CostTblEntry AVX512BWCostTbl[] = {
-    { ISD::SETCC,   MVT::v32i16,  1 },
-    { ISD::SETCC,   MVT::v64i8,   1 },
+  static const CostTblEntry SSE2CostTbl[] = {
+    { ISD::SETCC,   MVT::v2f64,   2 },
+    { ISD::SETCC,   MVT::f64,     1 },
+    { ISD::SETCC,   MVT::v2i64,   8 },
+    { ISD::SETCC,   MVT::v4i32,   1 },
+    { ISD::SETCC,   MVT::v8i16,   1 },
+    { ISD::SETCC,   MVT::v16i8,   1 },
+
+    { ISD::SELECT,  MVT::v2f64,   3 }, // andpd + andnpd + orpd
+    { ISD::SELECT,  MVT::v2i64,   3 }, // pand + pandn + por
+    { ISD::SELECT,  MVT::v4i32,   3 }, // pand + pandn + por
+    { ISD::SELECT,  MVT::v8i16,   3 }, // pand + pandn + por
+    { ISD::SELECT,  MVT::v16i8,   3 }, // pand + pandn + por
+  };
+
+  static const CostTblEntry SSE1CostTbl[] = {
+    { ISD::SETCC,   MVT::v4f32,   2 },
+    { ISD::SETCC,   MVT::f32,     1 },
+
+    { ISD::SELECT,  MVT::v4f32,   3 }, // andps + andnps + orps
   };
 
   if (ST->hasBWI())
     if (const auto *Entry = CostTableLookup(AVX512BWCostTbl, ISD, MTy))
-      return LT.first * Entry->Cost;
+      return LT.first * (ExtraCost + Entry->Cost);
 
   if (ST->hasAVX512())
     if (const auto *Entry = CostTableLookup(AVX512CostTbl, ISD, MTy))
-      return LT.first * Entry->Cost;
+      return LT.first * (ExtraCost + Entry->Cost);
 
   if (ST->hasAVX2())
     if (const auto *Entry = CostTableLookup(AVX2CostTbl, ISD, MTy))
-      return LT.first * Entry->Cost;
+      return LT.first * (ExtraCost + Entry->Cost);
 
   if (ST->hasAVX())
     if (const auto *Entry = CostTableLookup(AVX1CostTbl, ISD, MTy))
-      return LT.first * Entry->Cost;
+      return LT.first * (ExtraCost + Entry->Cost);
 
   if (ST->hasSSE42())
     if (const auto *Entry = CostTableLookup(SSE42CostTbl, ISD, MTy))
-      return LT.first * Entry->Cost;
+      return LT.first * (ExtraCost + Entry->Cost);
+
+  if (ST->hasSSE41())
+    if (const auto *Entry = CostTableLookup(SSE41CostTbl, ISD, MTy))
+      return LT.first * (ExtraCost + Entry->Cost);
 
   if (ST->hasSSE2())
     if (const auto *Entry = CostTableLookup(SSE2CostTbl, ISD, MTy))
-      return LT.first * Entry->Cost;
+      return LT.first * (ExtraCost + Entry->Cost);
+
+  if (ST->hasSSE1())
+    if (const auto *Entry = CostTableLookup(SSE1CostTbl, ISD, MTy))
+      return LT.first * (ExtraCost + Entry->Cost);
 
   return BaseT::getCmpSelInstrCost(Opcode, ValTy, CondTy, I);
 }
@@ -1742,6 +1861,14 @@ int X86TTIImpl::getIntrinsicInstrCost(Intrinsic::ID IID, Type *RetTy,
     { ISD::CTTZ,       MVT::v16i32, 14 },
     { ISD::CTTZ,       MVT::v32i16, 12 },
     { ISD::CTTZ,       MVT::v64i8,   9 },
+    { ISD::SADDSAT,    MVT::v32i16,  1 },
+    { ISD::SADDSAT,    MVT::v64i8,   1 },
+    { ISD::SSUBSAT,    MVT::v32i16,  1 },
+    { ISD::SSUBSAT,    MVT::v64i8,   1 },
+    { ISD::UADDSAT,    MVT::v32i16,  1 },
+    { ISD::UADDSAT,    MVT::v64i8,   1 },
+    { ISD::USUBSAT,    MVT::v32i16,  1 },
+    { ISD::USUBSAT,    MVT::v64i8,   1 },
   };
   static const CostTblEntry AVX512CostTbl[] = {
     { ISD::BITREVERSE, MVT::v8i64,  36 },
@@ -1752,6 +1879,14 @@ int X86TTIImpl::getIntrinsicInstrCost(Intrinsic::ID IID, Type *RetTy,
     { ISD::CTPOP,      MVT::v16i32, 24 },
     { ISD::CTTZ,       MVT::v8i64,  20 },
     { ISD::CTTZ,       MVT::v16i32, 28 },
+    { ISD::USUBSAT,    MVT::v16i32,  2 }, // pmaxud + psubd
+    { ISD::USUBSAT,    MVT::v2i64,   2 }, // pmaxuq + psubq
+    { ISD::USUBSAT,    MVT::v4i64,   2 }, // pmaxuq + psubq
+    { ISD::USUBSAT,    MVT::v8i64,   2 }, // pmaxuq + psubq
+    { ISD::UADDSAT,    MVT::v16i32,  3 }, // not + pminud + paddd
+    { ISD::UADDSAT,    MVT::v2i64,   3 }, // not + pminuq + paddq
+    { ISD::UADDSAT,    MVT::v4i64,   3 }, // not + pminuq + paddq
+    { ISD::UADDSAT,    MVT::v8i64,   3 }, // not + pminuq + paddq
   };
   static const CostTblEntry XOPCostTbl[] = {
     { ISD::BITREVERSE, MVT::v4i64,   4 },
@@ -1787,6 +1922,16 @@ int X86TTIImpl::getIntrinsicInstrCost(Intrinsic::ID IID, Type *RetTy,
     { ISD::CTTZ,       MVT::v8i32,  14 },
     { ISD::CTTZ,       MVT::v16i16, 12 },
     { ISD::CTTZ,       MVT::v32i8,   9 },
+    { ISD::SADDSAT,    MVT::v16i16,  1 },
+    { ISD::SADDSAT,    MVT::v32i8,   1 },
+    { ISD::SSUBSAT,    MVT::v16i16,  1 },
+    { ISD::SSUBSAT,    MVT::v32i8,   1 },
+    { ISD::UADDSAT,    MVT::v16i16,  1 },
+    { ISD::UADDSAT,    MVT::v32i8,   1 },
+    { ISD::UADDSAT,    MVT::v8i32,   3 }, // not + pminud + paddd
+    { ISD::USUBSAT,    MVT::v16i16,  1 },
+    { ISD::USUBSAT,    MVT::v32i8,   1 },
+    { ISD::USUBSAT,    MVT::v8i32,   2 }, // pmaxud + psubd
     { ISD::FSQRT,      MVT::f32,     7 }, // Haswell from http://www.agner.org/
     { ISD::FSQRT,      MVT::v4f32,   7 }, // Haswell from http://www.agner.org/
     { ISD::FSQRT,      MVT::v8f32,  14 }, // Haswell from http://www.agner.org/
@@ -1814,6 +1959,16 @@ int X86TTIImpl::getIntrinsicInstrCost(Intrinsic::ID IID, Type *RetTy,
     { ISD::CTTZ,       MVT::v8i32,  30 }, // 2 x 128-bit Op + extract/insert
     { ISD::CTTZ,       MVT::v16i16, 26 }, // 2 x 128-bit Op + extract/insert
     { ISD::CTTZ,       MVT::v32i8,  20 }, // 2 x 128-bit Op + extract/insert
+    { ISD::SADDSAT,    MVT::v16i16,  4 }, // 2 x 128-bit Op + extract/insert
+    { ISD::SADDSAT,    MVT::v32i8,   4 }, // 2 x 128-bit Op + extract/insert
+    { ISD::SSUBSAT,    MVT::v16i16,  4 }, // 2 x 128-bit Op + extract/insert
+    { ISD::SSUBSAT,    MVT::v32i8,   4 }, // 2 x 128-bit Op + extract/insert
+    { ISD::UADDSAT,    MVT::v16i16,  4 }, // 2 x 128-bit Op + extract/insert
+    { ISD::UADDSAT,    MVT::v32i8,   4 }, // 2 x 128-bit Op + extract/insert
+    { ISD::UADDSAT,    MVT::v8i32,   8 }, // 2 x 128-bit Op + extract/insert
+    { ISD::USUBSAT,    MVT::v16i16,  4 }, // 2 x 128-bit Op + extract/insert
+    { ISD::USUBSAT,    MVT::v32i8,   4 }, // 2 x 128-bit Op + extract/insert
+    { ISD::USUBSAT,    MVT::v8i32,   6 }, // 2 x 128-bit Op + extract/insert
     { ISD::FSQRT,      MVT::f32,    14 }, // SNB from http://www.agner.org/
     { ISD::FSQRT,      MVT::v4f32,  14 }, // SNB from http://www.agner.org/
     { ISD::FSQRT,      MVT::v8f32,  28 }, // SNB from http://www.agner.org/
@@ -1834,6 +1989,8 @@ int X86TTIImpl::getIntrinsicInstrCost(Intrinsic::ID IID, Type *RetTy,
     { ISD::FSQRT, MVT::v2f64, 70 }, // sqrtpd
   };
   static const CostTblEntry SSE42CostTbl[] = {
+    { ISD::USUBSAT,    MVT::v4i32,   2 }, // pmaxud + psubd
+    { ISD::UADDSAT,    MVT::v4i32,   3 }, // not + pminud + paddd
     { ISD::FSQRT,      MVT::f32,    18 }, // Nehalem from http://www.agner.org/
     { ISD::FSQRT,      MVT::v4f32,  18 }, // Nehalem from http://www.agner.org/
   };
@@ -1878,6 +2035,14 @@ int X86TTIImpl::getIntrinsicInstrCost(Intrinsic::ID IID, Type *RetTy,
     { ISD::CTTZ,       MVT::v4i32,  18 },
     { ISD::CTTZ,       MVT::v8i16,  16 },
     { ISD::CTTZ,       MVT::v16i8,  13 },
+    { ISD::SADDSAT,    MVT::v8i16,   1 },
+    { ISD::SADDSAT,    MVT::v16i8,   1 },
+    { ISD::SSUBSAT,    MVT::v8i16,   1 },
+    { ISD::SSUBSAT,    MVT::v16i8,   1 },
+    { ISD::UADDSAT,    MVT::v8i16,   1 },
+    { ISD::UADDSAT,    MVT::v16i8,   1 },
+    { ISD::USUBSAT,    MVT::v8i16,   1 },
+    { ISD::USUBSAT,    MVT::v16i8,   1 },
     { ISD::FSQRT,      MVT::f64,    32 }, // Nehalem from http://www.agner.org/
     { ISD::FSQRT,      MVT::v2f64,  32 }, // Nehalem from http://www.agner.org/
   };
@@ -1886,14 +2051,23 @@ int X86TTIImpl::getIntrinsicInstrCost(Intrinsic::ID IID, Type *RetTy,
     { ISD::FSQRT,      MVT::v4f32,  56 }, // Pentium III from http://www.agner.org/
   };
   static const CostTblEntry X64CostTbl[] = { // 64-bit targets
-    { ISD::BITREVERSE, MVT::i64,    14 } 
+    { ISD::BITREVERSE, MVT::i64,    14 },
+    { ISD::SADDO,      MVT::i64,     1 },
+    { ISD::UADDO,      MVT::i64,     1 },
   };
   static const CostTblEntry X86CostTbl[] = { // 32 or 64-bit targets
     { ISD::BITREVERSE, MVT::i32,    14 },
     { ISD::BITREVERSE, MVT::i16,    14 },
-    { ISD::BITREVERSE, MVT::i8,     11 }
+    { ISD::BITREVERSE, MVT::i8,     11 },
+    { ISD::SADDO,      MVT::i32,     1 },
+    { ISD::SADDO,      MVT::i16,     1 },
+    { ISD::SADDO,      MVT::i8,      1 },
+    { ISD::UADDO,      MVT::i32,     1 },
+    { ISD::UADDO,      MVT::i16,     1 },
+    { ISD::UADDO,      MVT::i8,      1 },
   };
 
+  Type *OpTy = RetTy;
   unsigned ISD = ISD::DELETED_NODE;
   switch (IID) {
   default:
@@ -1913,14 +2087,38 @@ int X86TTIImpl::getIntrinsicInstrCost(Intrinsic::ID IID, Type *RetTy,
   case Intrinsic::cttz:
     ISD = ISD::CTTZ;
     break;
+  case Intrinsic::sadd_sat:
+    ISD = ISD::SADDSAT;
+    break;
+  case Intrinsic::ssub_sat:
+    ISD = ISD::SSUBSAT;
+    break;
+  case Intrinsic::uadd_sat:
+    ISD = ISD::UADDSAT;
+    break;
+  case Intrinsic::usub_sat:
+    ISD = ISD::USUBSAT;
+    break;
   case Intrinsic::sqrt:
     ISD = ISD::FSQRT;
     break;
+  case Intrinsic::sadd_with_overflow:
+  case Intrinsic::ssub_with_overflow:
+    // SSUBO has same costs so don't duplicate.
+    ISD = ISD::SADDO;
+    OpTy = RetTy->getContainedType(0);
+    break;
+  case Intrinsic::uadd_with_overflow:
+  case Intrinsic::usub_with_overflow:
+    // USUBO has same costs so don't duplicate.
+    ISD = ISD::UADDO;
+    OpTy = RetTy->getContainedType(0);
+    break;
   }
 
   if (ISD != ISD::DELETED_NODE) {
     // Legalize the type.
-    std::pair<int, MVT> LT = TLI->getTypeLegalizationCost(DL, RetTy);
+    std::pair<int, MVT> LT = TLI->getTypeLegalizationCost(DL, OpTy);
     MVT MTy = LT.second;
 
     // Attempt to lookup cost.
@@ -2899,6 +3097,9 @@ X86TTIImpl::enableMemCmpExpansion(bool IsZeroCmp) const {
     Options.LoadSizes.push_back(4);
     Options.LoadSizes.push_back(2);
     Options.LoadSizes.push_back(1);
+    // All GPR and vector loads can be unaligned. SIMD compare requires integer
+    // vectors (SSE2/AVX2).
+    Options.AllowOverlappingLoads = true;
     return Options;
   }();
   return IsZeroCmp ? &EqZeroOptions : &ThreeWayOptions;
diff --git a/lib/Target/X86/X86TargetTransformInfo.h b/lib/Target/X86/X86TargetTransformInfo.h
index 1637592c81f8a413be473f595cb2310991ee024a..b2e3e05cbe32dc95def2e368f9d412e529e3d21a 100644
--- a/lib/Target/X86/X86TargetTransformInfo.h
+++ b/lib/Target/X86/X86TargetTransformInfo.h
@@ -1,9 +1,8 @@
 //===-- X86TargetTransformInfo.h - X86 specific TTI -------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
@@ -60,6 +59,7 @@ public:
 
   unsigned getNumberOfRegisters(bool Vector);
   unsigned getRegisterBitWidth(bool Vector) const;
+  unsigned getMinVectorRegisterBitWidth() const;
   unsigned getLoadStoreVecRegBitWidth(unsigned AS) const;
   unsigned getMaxInterleaveFactor(unsigned VF);
   int getArithmeticInstrCost(
diff --git a/lib/Target/X86/X86VZeroUpper.cpp b/lib/Target/X86/X86VZeroUpper.cpp
index f882b760927c0f772409987549599d465f0b968c..a07d2f20acab817816187deb2af9f185a3d7fb90 100644
--- a/lib/Target/X86/X86VZeroUpper.cpp
+++ b/lib/Target/X86/X86VZeroUpper.cpp
@@ -1,9 +1,8 @@
 //===- X86VZeroUpper.cpp - AVX vzeroupper instruction inserter ------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/X86/X86WinAllocaExpander.cpp b/lib/Target/X86/X86WinAllocaExpander.cpp
index d298aaa97ecd11cd89dc66687739663825fee194..438c7a5ed696972c77688da477989916f76df6fd 100644
--- a/lib/Target/X86/X86WinAllocaExpander.cpp
+++ b/lib/Target/X86/X86WinAllocaExpander.cpp
@@ -1,9 +1,8 @@
 //===----- X86WinAllocaExpander.cpp - Expand WinAlloca pseudo instruction -===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/X86/X86WinEHState.cpp b/lib/Target/X86/X86WinEHState.cpp
index 185deda97c1fc7978a059fc43ca07ae70ab804f8..df9a171f582fef16b2aaa9ccf930b1f9996c3a0f 100644
--- a/lib/Target/X86/X86WinEHState.cpp
+++ b/lib/Target/X86/X86WinEHState.cpp
@@ -1,9 +1,8 @@
 //===-- X86WinEHState - Insert EH state updates for win32 exceptions ------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -87,15 +86,15 @@ private:
   StructType *EHLinkRegistrationTy = nullptr;
   StructType *CXXEHRegistrationTy = nullptr;
   StructType *SEHRegistrationTy = nullptr;
-  Constant *SetJmp3 = nullptr;
-  Constant *CxxLongjmpUnwind = nullptr;
+  FunctionCallee SetJmp3 = nullptr;
+  FunctionCallee CxxLongjmpUnwind = nullptr;
 
   // Per-function state
   EHPersonality Personality = EHPersonality::Unknown;
   Function *PersonalityFn = nullptr;
   bool UseStackGuard = false;
   int ParentBaseState;
-  Constant *SehLongjmpUnwind = nullptr;
+  FunctionCallee SehLongjmpUnwind = nullptr;
   Constant *Cookie = nullptr;
 
   /// The stack allocation containing all EH data, including the link in the
@@ -304,7 +303,7 @@ void WinEHStatePass::emitExceptionRegistrationRecord(Function *F) {
     CxxLongjmpUnwind = TheModule->getOrInsertFunction(
         "__CxxLongjmpUnwind",
         FunctionType::get(VoidTy, Int8PtrType, /*isVarArg=*/false));
-    cast<Function>(CxxLongjmpUnwind->stripPointerCasts())
+    cast<Function>(CxxLongjmpUnwind.getCallee()->stripPointerCasts())
         ->setCallingConv(CallingConv::X86_StdCall);
   } else if (Personality == EHPersonality::MSVC_X86SEH) {
     // If _except_handler4 is in use, some additional guard checks and prologue
@@ -357,7 +356,7 @@ void WinEHStatePass::emitExceptionRegistrationRecord(Function *F) {
         UseStackGuard ? "_seh_longjmp_unwind4" : "_seh_longjmp_unwind",
         FunctionType::get(Type::getVoidTy(TheModule->getContext()), Int8PtrType,
                           /*isVarArg=*/false));
-    cast<Function>(SehLongjmpUnwind->stripPointerCasts())
+    cast<Function>(SehLongjmpUnwind.getCallee()->stripPointerCasts())
         ->setCallingConv(CallingConv::X86_StdCall);
   } else {
     llvm_unreachable("unexpected personality function");
@@ -412,7 +411,7 @@ Function *WinEHStatePass::generateLSDAInEAXThunk(Function *ParentFunc) {
       Builder.CreateBitCast(PersonalityFn, TargetFuncTy->getPointerTo());
   auto AI = Trampoline->arg_begin();
   Value *Args[5] = {LSDA, &*AI++, &*AI++, &*AI++, &*AI++};
-  CallInst *Call = Builder.CreateCall(CastPersonality, Args);
+  CallInst *Call = Builder.CreateCall(TargetFuncTy, CastPersonality, Args);
   // Can't use musttail due to prototype mismatch, but we can use tail.
   Call->setTailCall(true);
   // Set inreg so we pass it in EAX.
@@ -433,7 +432,7 @@ void WinEHStatePass::linkExceptionRegistration(IRBuilder<> &Builder,
   // Next = [fs:00]
   Constant *FSZero =
       Constant::getNullValue(LinkTy->getPointerTo()->getPointerTo(257));
-  Value *Next = Builder.CreateLoad(FSZero);
+  Value *Next = Builder.CreateLoad(LinkTy->getPointerTo(), FSZero);
   Builder.CreateStore(Next, Builder.CreateStructGEP(LinkTy, Link, 0));
   // [fs:00] = Link
   Builder.CreateStore(Link, FSZero);
@@ -448,8 +447,8 @@ void WinEHStatePass::unlinkExceptionRegistration(IRBuilder<> &Builder) {
   }
   Type *LinkTy = getEHLinkRegistrationType();
   // [fs:00] = Link->Next
-  Value *Next =
-      Builder.CreateLoad(Builder.CreateStructGEP(LinkTy, Link, 0));
+  Value *Next = Builder.CreateLoad(LinkTy->getPointerTo(),
+                                   Builder.CreateStructGEP(LinkTy, Link, 0));
   Constant *FSZero =
       Constant::getNullValue(LinkTy->getPointerTo()->getPointerTo(257));
   Builder.CreateStore(Next, FSZero);
@@ -472,11 +471,11 @@ void WinEHStatePass::rewriteSetJmpCallSite(IRBuilder<> &Builder, Function &F,
 
   SmallVector<Value *, 3> OptionalArgs;
   if (Personality == EHPersonality::MSVC_CXX) {
-    OptionalArgs.push_back(CxxLongjmpUnwind);
+    OptionalArgs.push_back(CxxLongjmpUnwind.getCallee());
     OptionalArgs.push_back(State);
     OptionalArgs.push_back(emitEHLSDA(Builder, &F));
   } else if (Personality == EHPersonality::MSVC_X86SEH) {
-    OptionalArgs.push_back(SehLongjmpUnwind);
+    OptionalArgs.push_back(SehLongjmpUnwind.getCallee());
     OptionalArgs.push_back(State);
     if (UseStackGuard)
       OptionalArgs.push_back(Cookie);
@@ -767,7 +766,7 @@ void WinEHStatePass::addStateStores(Function &F, WinEHFuncInfo &FuncInfo) {
       if (!CS)
         continue;
       if (CS.getCalledValue()->stripPointerCasts() !=
-          SetJmp3->stripPointerCasts())
+          SetJmp3.getCallee()->stripPointerCasts())
         continue;
 
       SetJmp3CallSites.push_back(CS);
@@ -782,9 +781,9 @@ void WinEHStatePass::addStateStores(Function &F, WinEHFuncInfo &FuncInfo) {
     IRBuilder<> Builder(CS.getInstruction());
     Value *State;
     if (InCleanup) {
-      Value *StateField =
-          Builder.CreateStructGEP(nullptr, RegNode, StateFieldIndex);
-      State = Builder.CreateLoad(StateField);
+      Value *StateField = Builder.CreateStructGEP(RegNode->getAllocatedType(),
+                                                  RegNode, StateFieldIndex);
+      State = Builder.CreateLoad(Builder.getInt32Ty(), StateField);
     } else {
       State = Builder.getInt32(getStateForCallSite(BlockColors, FuncInfo, CS));
     }
@@ -794,7 +793,7 @@ void WinEHStatePass::addStateStores(Function &F, WinEHFuncInfo &FuncInfo) {
 
 void WinEHStatePass::insertStateNumberStore(Instruction *IP, int State) {
   IRBuilder<> Builder(IP);
-  Value *StateField =
-      Builder.CreateStructGEP(nullptr, RegNode, StateFieldIndex);
+  Value *StateField = Builder.CreateStructGEP(RegNode->getAllocatedType(),
+                                              RegNode, StateFieldIndex);
   Builder.CreateStore(Builder.getInt32(State), StateField);
 }
diff --git a/lib/Target/XCore/Disassembler/LLVMBuild.txt b/lib/Target/XCore/Disassembler/LLVMBuild.txt
index cc30d0400c4bb907833aa148d1f38494ba947a97..f27972cc3f7c8e5e23152d933e4e116286f738dc 100644
--- a/lib/Target/XCore/Disassembler/LLVMBuild.txt
+++ b/lib/Target/XCore/Disassembler/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/Target/XCore/Disassembler/LLVMBuild.txt ------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Target/XCore/Disassembler/XCoreDisassembler.cpp b/lib/Target/XCore/Disassembler/XCoreDisassembler.cpp
index faf66e5944ab8b352ddbed4dc7a58252763277e7..657a201d9a6cc255d00613963b899436ef564b33 100644
--- a/lib/Target/XCore/Disassembler/XCoreDisassembler.cpp
+++ b/lib/Target/XCore/Disassembler/XCoreDisassembler.cpp
@@ -1,9 +1,8 @@
 //===- XCoreDisassembler.cpp - Disassembler for XCore -----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
diff --git a/lib/Target/XCore/InstPrinter/LLVMBuild.txt b/lib/Target/XCore/InstPrinter/LLVMBuild.txt
index 8750bc7acedc27903db1de334b8da77b585077e8..493b07c7631e3b22fa6fb4e08aa890effa6819fe 100644
--- a/lib/Target/XCore/InstPrinter/LLVMBuild.txt
+++ b/lib/Target/XCore/InstPrinter/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/Target/XCore/InstPrinter/LLVMBuild.txt -------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Target/XCore/InstPrinter/XCoreInstPrinter.cpp b/lib/Target/XCore/InstPrinter/XCoreInstPrinter.cpp
index b03c1852281d149628e71227a2f50a340e85902f..d231e0981324a6343c581b13489b78dac36e2018 100644
--- a/lib/Target/XCore/InstPrinter/XCoreInstPrinter.cpp
+++ b/lib/Target/XCore/InstPrinter/XCoreInstPrinter.cpp
@@ -1,9 +1,8 @@
 //===-- XCoreInstPrinter.cpp - Convert XCore MCInst to assembly syntax ----===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/XCore/InstPrinter/XCoreInstPrinter.h b/lib/Target/XCore/InstPrinter/XCoreInstPrinter.h
index a0b48002646911780b9111d8142d31d1cb1c67a3..4554ebcf2f96fe6ab67fe3486ffd97ef3fb57ed0 100644
--- a/lib/Target/XCore/InstPrinter/XCoreInstPrinter.h
+++ b/lib/Target/XCore/InstPrinter/XCoreInstPrinter.h
@@ -1,9 +1,8 @@
 //== XCoreInstPrinter.h - Convert XCore MCInst to assembly syntax -*- C++ -*-=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
diff --git a/lib/Target/XCore/LLVMBuild.txt b/lib/Target/XCore/LLVMBuild.txt
index 401e0526f5806d6933568cfd87acedd293255235..b876772aae92194fb11f9db470c563daff644a89 100644
--- a/lib/Target/XCore/LLVMBuild.txt
+++ b/lib/Target/XCore/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/Target/XCore/LLVMBuild.txt -------------------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Target/XCore/MCTargetDesc/LLVMBuild.txt b/lib/Target/XCore/MCTargetDesc/LLVMBuild.txt
index 6d390d272748787ac1fe8186b7a4c6c8ac9597ba..5e3e1c62eb1778369d518e0268793b2cc482ff53 100644
--- a/lib/Target/XCore/MCTargetDesc/LLVMBuild.txt
+++ b/lib/Target/XCore/MCTargetDesc/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/Target/XCore/MCTargetDesc/LLVMBuild.txt ------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Target/XCore/MCTargetDesc/XCoreMCAsmInfo.cpp b/lib/Target/XCore/MCTargetDesc/XCoreMCAsmInfo.cpp
index 3178a4edbb3b0e42da4940a4a22c6df5df13a727..ae19e2a78eec91cd2cd444889c46baceffaf67fd 100644
--- a/lib/Target/XCore/MCTargetDesc/XCoreMCAsmInfo.cpp
+++ b/lib/Target/XCore/MCTargetDesc/XCoreMCAsmInfo.cpp
@@ -1,9 +1,8 @@
 //===-- XCoreMCAsmInfo.cpp - XCore asm properties -------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/XCore/MCTargetDesc/XCoreMCAsmInfo.h b/lib/Target/XCore/MCTargetDesc/XCoreMCAsmInfo.h
index 39581e424e8c07b2e33a8e3df78020210d46f639..b1dd247f84683d14db61b355bcdc71c11d670ef6 100644
--- a/lib/Target/XCore/MCTargetDesc/XCoreMCAsmInfo.h
+++ b/lib/Target/XCore/MCTargetDesc/XCoreMCAsmInfo.h
@@ -1,9 +1,8 @@
 //===-- XCoreMCAsmInfo.h - XCore asm properties ----------------*- C++ -*--===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/XCore/MCTargetDesc/XCoreMCTargetDesc.cpp b/lib/Target/XCore/MCTargetDesc/XCoreMCTargetDesc.cpp
index 805f1c18b609a6ddf2f47cd9e0c677760f32bc36..e2ced98027c840eebd0e6faa0b00301ac7cc6df1 100644
--- a/lib/Target/XCore/MCTargetDesc/XCoreMCTargetDesc.cpp
+++ b/lib/Target/XCore/MCTargetDesc/XCoreMCTargetDesc.cpp
@@ -1,9 +1,8 @@
 //===-- XCoreMCTargetDesc.cpp - XCore Target Descriptions -----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/XCore/MCTargetDesc/XCoreMCTargetDesc.h b/lib/Target/XCore/MCTargetDesc/XCoreMCTargetDesc.h
index 1dc384fadf69bd84942cffab846e57968a88335b..042a40fd2cf234f0fc6d821eee2efaefa5a85044 100644
--- a/lib/Target/XCore/MCTargetDesc/XCoreMCTargetDesc.h
+++ b/lib/Target/XCore/MCTargetDesc/XCoreMCTargetDesc.h
@@ -1,9 +1,8 @@
 //===-- XCoreMCTargetDesc.h - XCore Target Descriptions ---------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/XCore/TargetInfo/LLVMBuild.txt b/lib/Target/XCore/TargetInfo/LLVMBuild.txt
index 45ff75f14359e42809c75507f34e410cb9ff7d33..21e2ee3402747cfdd853c81ef9dfec76a7a4c35a 100644
--- a/lib/Target/XCore/TargetInfo/LLVMBuild.txt
+++ b/lib/Target/XCore/TargetInfo/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/Target/XCore/TargetInfo/LLVMBuild.txt --------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Target/XCore/TargetInfo/XCoreTargetInfo.cpp b/lib/Target/XCore/TargetInfo/XCoreTargetInfo.cpp
index 41f4078cc3287ed93dc01ec85d5de1e4b82c70dc..54c2df80023949ecde0ce680b93f840a6415d605 100644
--- a/lib/Target/XCore/TargetInfo/XCoreTargetInfo.cpp
+++ b/lib/Target/XCore/TargetInfo/XCoreTargetInfo.cpp
@@ -1,9 +1,8 @@
 //===-- XCoreTargetInfo.cpp - XCore Target Implementation -----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/XCore/XCore.h b/lib/Target/XCore/XCore.h
index ba6ca843671e7587d9aecf5dbe5202da769fd272..b7b86be9ab511ed505c5cf002995a2c2cd525291 100644
--- a/lib/Target/XCore/XCore.h
+++ b/lib/Target/XCore/XCore.h
@@ -1,9 +1,8 @@
 //===-- XCore.h - Top-level interface for XCore representation --*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/XCore/XCore.td b/lib/Target/XCore/XCore.td
index 04a1dd5e95bec9a8193ac6a7431e04306b3ed755..a97b3dd1d0a2bcdc7636ef70e82271cf816099a2 100644
--- a/lib/Target/XCore/XCore.td
+++ b/lib/Target/XCore/XCore.td
@@ -1,9 +1,8 @@
 //===-- XCore.td - Describe the XCore Target Machine -------*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/XCore/XCoreAsmPrinter.cpp b/lib/Target/XCore/XCoreAsmPrinter.cpp
index 916bca6392de81584b8725c7602d662945f6ee97..f13f058536419647e4780c02b3891eb0890b3dbf 100644
--- a/lib/Target/XCore/XCoreAsmPrinter.cpp
+++ b/lib/Target/XCore/XCoreAsmPrinter.cpp
@@ -1,9 +1,8 @@
 //===-- XCoreAsmPrinter.cpp - XCore LLVM assembly writer ------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/XCore/XCoreCallingConv.td b/lib/Target/XCore/XCoreCallingConv.td
index e149e6d9ec20bde6734c737f9ebc8d9f5c42fbc4..aec109b83fa2fa1ada0cf8ecc7dfe8f1d28c995f 100644
--- a/lib/Target/XCore/XCoreCallingConv.td
+++ b/lib/Target/XCore/XCoreCallingConv.td
@@ -1,9 +1,8 @@
 //===- XCoreCallingConv.td - Calling Conventions for XCore -*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 // This describes the calling conventions for XCore architecture.
diff --git a/lib/Target/XCore/XCoreFrameLowering.cpp b/lib/Target/XCore/XCoreFrameLowering.cpp
index fff8a66d0e755d5b2eb0e9664134557c1eb8a28e..5066407c74aabed102903f39e122f3c3384d6bc9 100644
--- a/lib/Target/XCore/XCoreFrameLowering.cpp
+++ b/lib/Target/XCore/XCoreFrameLowering.cpp
@@ -1,9 +1,8 @@
 //===-- XCoreFrameLowering.cpp - Frame info for XCore Target --------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/XCore/XCoreFrameLowering.h b/lib/Target/XCore/XCoreFrameLowering.h
index e98e9cda11db67aba80e0433ac1db955fcf26517..95c3a2973033d98efb9681100af7b9533fd08373 100644
--- a/lib/Target/XCore/XCoreFrameLowering.h
+++ b/lib/Target/XCore/XCoreFrameLowering.h
@@ -1,9 +1,8 @@
 //===-- XCoreFrameLowering.h - Frame info for XCore Target ------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/XCore/XCoreFrameToArgsOffsetElim.cpp b/lib/Target/XCore/XCoreFrameToArgsOffsetElim.cpp
index 4b10e71be03d4bbce0b2f749253e761548e8340f..e433d21c59b7aec4e03575f3a77460bf75bc7055 100644
--- a/lib/Target/XCore/XCoreFrameToArgsOffsetElim.cpp
+++ b/lib/Target/XCore/XCoreFrameToArgsOffsetElim.cpp
@@ -1,9 +1,8 @@
 //===-- XCoreFrameToArgsOffsetElim.cpp ----------------------------*- C++ -*-=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/XCore/XCoreISelDAGToDAG.cpp b/lib/Target/XCore/XCoreISelDAGToDAG.cpp
index 1688c38efc1d5ea8e21fd91b732e7607f9b11a05..5fd9e23258b0ea6448c63f5677d8f9b577aa3cd1 100644
--- a/lib/Target/XCore/XCoreISelDAGToDAG.cpp
+++ b/lib/Target/XCore/XCoreISelDAGToDAG.cpp
@@ -1,9 +1,8 @@
 //===-- XCoreISelDAGToDAG.cpp - A dag to dag inst selector for XCore ------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/XCore/XCoreISelLowering.cpp b/lib/Target/XCore/XCoreISelLowering.cpp
index 99e76144cba398b812463cef402e75274f6b7a02..8979ee727d0bb9e123e99ba6fd36ca1c51c911c5 100644
--- a/lib/Target/XCore/XCoreISelLowering.cpp
+++ b/lib/Target/XCore/XCoreISelLowering.cpp
@@ -1,9 +1,8 @@
 //===-- XCoreISelLowering.cpp - XCore DAG Lowering Implementation ---------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -403,8 +402,7 @@ SDValue XCoreTargetLowering::lowerLoadWordFromAlignedBasePlusOffset(
 
 static bool isWordAligned(SDValue Value, SelectionDAG &DAG)
 {
-  KnownBits Known;
-  DAG.computeKnownBits(Value, Known);
+  KnownBits Known = DAG.computeKnownBits(Value);
   return Known.countMinTrailingZeros() >= 2;
 }
 
@@ -1649,10 +1647,9 @@ SDValue XCoreTargetLowering::PerformDAGCombine(SDNode *N,
     // fold (ladd x, 0, y) -> 0, add x, y iff carry is unused and y has only the
     // low bit set
     if (N1C && N1C->isNullValue() && N->hasNUsesOfValue(0, 1)) {
-      KnownBits Known;
       APInt Mask = APInt::getHighBitsSet(VT.getSizeInBits(),
                                          VT.getSizeInBits() - 1);
-      DAG.computeKnownBits(N2, Known);
+      KnownBits Known = DAG.computeKnownBits(N2);
       if ((Known.Zero & Mask) == Mask) {
         SDValue Carry = DAG.getConstant(0, dl, VT);
         SDValue Result = DAG.getNode(ISD::ADD, dl, VT, N0, N2);
@@ -1672,10 +1669,9 @@ SDValue XCoreTargetLowering::PerformDAGCombine(SDNode *N,
 
     // fold (lsub 0, 0, x) -> x, -x iff x has only the low bit set
     if (N0C && N0C->isNullValue() && N1C && N1C->isNullValue()) {
-      KnownBits Known;
       APInt Mask = APInt::getHighBitsSet(VT.getSizeInBits(),
                                          VT.getSizeInBits() - 1);
-      DAG.computeKnownBits(N2, Known);
+      KnownBits Known = DAG.computeKnownBits(N2);
       if ((Known.Zero & Mask) == Mask) {
         SDValue Borrow = N2;
         SDValue Result = DAG.getNode(ISD::SUB, dl, VT,
@@ -1688,10 +1684,9 @@ SDValue XCoreTargetLowering::PerformDAGCombine(SDNode *N,
     // fold (lsub x, 0, y) -> 0, sub x, y iff borrow is unused and y has only the
     // low bit set
     if (N1C && N1C->isNullValue() && N->hasNUsesOfValue(0, 1)) {
-      KnownBits Known;
       APInt Mask = APInt::getHighBitsSet(VT.getSizeInBits(),
                                          VT.getSizeInBits() - 1);
-      DAG.computeKnownBits(N2, Known);
+      KnownBits Known = DAG.computeKnownBits(N2);
       if ((Known.Zero & Mask) == Mask) {
         SDValue Borrow = DAG.getConstant(0, dl, VT);
         SDValue Result = DAG.getNode(ISD::SUB, dl, VT, N0, N2);
diff --git a/lib/Target/XCore/XCoreISelLowering.h b/lib/Target/XCore/XCoreISelLowering.h
index 7a99389e54a72b178b0dd2dcf6b3e3df4ee3958d..eb5b17e0530cafd1a1a69661be32aebdd78977df 100644
--- a/lib/Target/XCore/XCoreISelLowering.h
+++ b/lib/Target/XCore/XCoreISelLowering.h
@@ -1,9 +1,8 @@
 //===-- XCoreISelLowering.h - XCore DAG Lowering Interface ------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/XCore/XCoreInstrFormats.td b/lib/Target/XCore/XCoreInstrFormats.td
index 379cc39aa6170590710a1bcf0a2ec2e4ad9e1cf3..deb899ddb1afa6775294c84aac6b51dfb3318596 100644
--- a/lib/Target/XCore/XCoreInstrFormats.td
+++ b/lib/Target/XCore/XCoreInstrFormats.td
@@ -1,9 +1,8 @@
 //===-- XCoreInstrFormats.td - XCore Instruction Formats ---*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/XCore/XCoreInstrInfo.cpp b/lib/Target/XCore/XCoreInstrInfo.cpp
index b0de048672df273a466f51053a4ac434c7bc98dd..bbad8e3545864c0f6099a402e395aa807b06e603 100644
--- a/lib/Target/XCore/XCoreInstrInfo.cpp
+++ b/lib/Target/XCore/XCoreInstrInfo.cpp
@@ -1,9 +1,8 @@
 //===-- XCoreInstrInfo.cpp - XCore Instruction Information ----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/XCore/XCoreInstrInfo.h b/lib/Target/XCore/XCoreInstrInfo.h
index 9d9ee33ce222f88727f3456ed1046d6a4f7fdaa8..b9621f136589c5b27ee6d517a9d3b99ceb891dbf 100644
--- a/lib/Target/XCore/XCoreInstrInfo.h
+++ b/lib/Target/XCore/XCoreInstrInfo.h
@@ -1,9 +1,8 @@
 //===-- XCoreInstrInfo.h - XCore Instruction Information --------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/XCore/XCoreInstrInfo.td b/lib/Target/XCore/XCoreInstrInfo.td
index b87ba6548962207c4cf6f72a856504a3635e866a..18f02e1d80f0fdd0f70d5e8ab3973ecc2c16c14e 100644
--- a/lib/Target/XCore/XCoreInstrInfo.td
+++ b/lib/Target/XCore/XCoreInstrInfo.td
@@ -1,9 +1,8 @@
 //===-- XCoreInstrInfo.td - Target Description for XCore ---*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/XCore/XCoreLowerThreadLocal.cpp b/lib/Target/XCore/XCoreLowerThreadLocal.cpp
index 7455cd997ad6729e85c0d8d099ce2bea25687bd1..a18fb28f2fe9b6f5f8c7025cfa3490be3f729128 100644
--- a/lib/Target/XCore/XCoreLowerThreadLocal.cpp
+++ b/lib/Target/XCore/XCoreLowerThreadLocal.cpp
@@ -1,9 +1,8 @@
 //===-- XCoreLowerThreadLocal - Lower thread local variables --------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
diff --git a/lib/Target/XCore/XCoreMCInstLower.cpp b/lib/Target/XCore/XCoreMCInstLower.cpp
index 21270192b2344c492f90799d77a8a7b7b4400247..cd28fa5cd1441f409c6104a93de23a86162acbe8 100644
--- a/lib/Target/XCore/XCoreMCInstLower.cpp
+++ b/lib/Target/XCore/XCoreMCInstLower.cpp
@@ -1,9 +1,8 @@
 //===-- XCoreMCInstLower.cpp - Convert XCore MachineInstr to MCInst -------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
diff --git a/lib/Target/XCore/XCoreMCInstLower.h b/lib/Target/XCore/XCoreMCInstLower.h
index abcb80fcf766a83c1fee735fb389abc53f44e901..0eaa84ef736b1b8dcfe343329781d41c0e59f71d 100644
--- a/lib/Target/XCore/XCoreMCInstLower.h
+++ b/lib/Target/XCore/XCoreMCInstLower.h
@@ -1,9 +1,8 @@
 //===-- XCoreMCInstLower.h - Lower MachineInstr to MCInst ------*- C++ -*--===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/XCore/XCoreMachineFunctionInfo.cpp b/lib/Target/XCore/XCoreMachineFunctionInfo.cpp
index b7b0daab98063514189f155d65c3d4c3d7a8cc81..0b4fcffbc655948dae6354a6649087733b0c4c80 100644
--- a/lib/Target/XCore/XCoreMachineFunctionInfo.cpp
+++ b/lib/Target/XCore/XCoreMachineFunctionInfo.cpp
@@ -1,9 +1,8 @@
 //===-- XCoreMachineFunctionInfo.cpp - XCore machine function info --------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/XCore/XCoreMachineFunctionInfo.h b/lib/Target/XCore/XCoreMachineFunctionInfo.h
index 6c05ab3f10df7b7a4ed0f05f8afc711eaae9b561..aebe11b15b5490565d1dd4ba347610d093f8c684 100644
--- a/lib/Target/XCore/XCoreMachineFunctionInfo.h
+++ b/lib/Target/XCore/XCoreMachineFunctionInfo.h
@@ -1,9 +1,8 @@
 //===- XCoreMachineFunctionInfo.h - XCore machine function info -*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/XCore/XCoreRegisterInfo.cpp b/lib/Target/XCore/XCoreRegisterInfo.cpp
index e119d9555f9d69d1f6f5825223845094e7799b68..77417d042ff18b74260354005d65f93539ac941d 100644
--- a/lib/Target/XCore/XCoreRegisterInfo.cpp
+++ b/lib/Target/XCore/XCoreRegisterInfo.cpp
@@ -1,9 +1,8 @@
 //===-- XCoreRegisterInfo.cpp - XCore Register Information ----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/XCore/XCoreRegisterInfo.h b/lib/Target/XCore/XCoreRegisterInfo.h
index 2e9fd98ed34fc6c9f657dba9996de0f329187e5b..2881cb5911e3e38f62330a558185940ea14ba617 100644
--- a/lib/Target/XCore/XCoreRegisterInfo.h
+++ b/lib/Target/XCore/XCoreRegisterInfo.h
@@ -1,9 +1,8 @@
 //===-- XCoreRegisterInfo.h - XCore Register Information Impl ---*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/XCore/XCoreRegisterInfo.td b/lib/Target/XCore/XCoreRegisterInfo.td
index 6694b2882acab9548ca3f3da056b8b7cd8f43542..d9502939bae37316da563b3fd7039166f71d193e 100644
--- a/lib/Target/XCore/XCoreRegisterInfo.td
+++ b/lib/Target/XCore/XCoreRegisterInfo.td
@@ -1,9 +1,8 @@
 //===-- XCoreRegisterInfo.td - XCore Register defs ---------*- tablegen -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/XCore/XCoreSelectionDAGInfo.cpp b/lib/Target/XCore/XCoreSelectionDAGInfo.cpp
index 646309e02de8252745aa562144646a63a1afa14d..c86756e345a94a18eb2d120056fdff5afcdf0bf6 100644
--- a/lib/Target/XCore/XCoreSelectionDAGInfo.cpp
+++ b/lib/Target/XCore/XCoreSelectionDAGInfo.cpp
@@ -1,9 +1,8 @@
 //===-- XCoreSelectionDAGInfo.cpp - XCore SelectionDAG Info ---------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/XCore/XCoreSelectionDAGInfo.h b/lib/Target/XCore/XCoreSelectionDAGInfo.h
index 7cd0d8216e910a891987160c68c53242d70c4501..5dcef08391c90360faeae1910813907db643af40 100644
--- a/lib/Target/XCore/XCoreSelectionDAGInfo.h
+++ b/lib/Target/XCore/XCoreSelectionDAGInfo.h
@@ -1,9 +1,8 @@
 //===-- XCoreSelectionDAGInfo.h - XCore SelectionDAG Info -------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/XCore/XCoreSubtarget.cpp b/lib/Target/XCore/XCoreSubtarget.cpp
index 99ad2c88504f17c7921f7ab42ac46836c38624fb..ffeb0862c94545c5c83fe5fd9ec1f7ad577708db 100644
--- a/lib/Target/XCore/XCoreSubtarget.cpp
+++ b/lib/Target/XCore/XCoreSubtarget.cpp
@@ -1,9 +1,8 @@
 //===-- XCoreSubtarget.cpp - XCore Subtarget Information ------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/XCore/XCoreSubtarget.h b/lib/Target/XCore/XCoreSubtarget.h
index ed9936ebf2b840f19ca81c7dac4b92b72ef08601..68139da9d1d0bc2964d31f88d864015706010a5d 100644
--- a/lib/Target/XCore/XCoreSubtarget.h
+++ b/lib/Target/XCore/XCoreSubtarget.h
@@ -1,9 +1,8 @@
 //===-- XCoreSubtarget.h - Define Subtarget for the XCore -------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/XCore/XCoreTargetMachine.cpp b/lib/Target/XCore/XCoreTargetMachine.cpp
index 2aa9932e2465a51e1a15eed7ea68c55404b02088..2d66a41de28307d01858c04abf096e8ce594da1d 100644
--- a/lib/Target/XCore/XCoreTargetMachine.cpp
+++ b/lib/Target/XCore/XCoreTargetMachine.cpp
@@ -1,9 +1,8 @@
 //===-- XCoreTargetMachine.cpp - Define TargetMachine for XCore -----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/XCore/XCoreTargetMachine.h b/lib/Target/XCore/XCoreTargetMachine.h
index 965b9b2c4d65737776087a4b58423117dd1d9f29..9c3bdcf78f9c437e8c54076a572ca33c74cde173 100644
--- a/lib/Target/XCore/XCoreTargetMachine.h
+++ b/lib/Target/XCore/XCoreTargetMachine.h
@@ -1,9 +1,8 @@
 //===-- XCoreTargetMachine.h - Define TargetMachine for XCore ---*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Target/XCore/XCoreTargetObjectFile.cpp b/lib/Target/XCore/XCoreTargetObjectFile.cpp
index c60a262e719cb8bcf5bd9584f2b5114efba7329e..fe743b28b4b497de467a0962f9a5a564cae648d7 100644
--- a/lib/Target/XCore/XCoreTargetObjectFile.cpp
+++ b/lib/Target/XCore/XCoreTargetObjectFile.cpp
@@ -1,9 +1,8 @@
 //===-- XCoreTargetObjectFile.cpp - XCore object files --------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/XCore/XCoreTargetObjectFile.h b/lib/Target/XCore/XCoreTargetObjectFile.h
index 5eb423a7435e44fa68352495dc02234067a8eae6..fd172c55919fa33426a550084163df8bc37ff9d3 100644
--- a/lib/Target/XCore/XCoreTargetObjectFile.h
+++ b/lib/Target/XCore/XCoreTargetObjectFile.h
@@ -1,9 +1,8 @@
 //===-- XCoreTargetObjectFile.h - XCore Object Info -------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/XCore/XCoreTargetStreamer.h b/lib/Target/XCore/XCoreTargetStreamer.h
index 3563dbc5cb7b1f66b7c3fae12af87cc8d100edf4..3543fc52ea7fc01a3b568b9ecc7386aa2544b89e 100644
--- a/lib/Target/XCore/XCoreTargetStreamer.h
+++ b/lib/Target/XCore/XCoreTargetStreamer.h
@@ -1,9 +1,8 @@
 //===-- XCoreTargetStreamer.h - XCore Target Streamer ----------*- C++ -*--===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Target/XCore/XCoreTargetTransformInfo.h b/lib/Target/XCore/XCoreTargetTransformInfo.h
index aa068b333425404a0370fa1c53d7422d84ca6fdc..3fecaaa597224c3d10c25ff5bcccefbfca44bbee 100644
--- a/lib/Target/XCore/XCoreTargetTransformInfo.h
+++ b/lib/Target/XCore/XCoreTargetTransformInfo.h
@@ -1,9 +1,8 @@
 //===-- XCoreTargetTransformInfo.h - XCore specific TTI ---------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/lib/Testing/LLVMBuild.txt b/lib/Testing/LLVMBuild.txt
index cdf83736298e4698bd1d209588712656e0aab416..4b331468a03ec113a767921e6f3d6fd3c8fb46d8 100644
--- a/lib/Testing/LLVMBuild.txt
+++ b/lib/Testing/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/Testing/LLVMBuild.txt ------------------------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Testing/Support/Error.cpp b/lib/Testing/Support/Error.cpp
index 5692cdfcdf7b704500d8be1276d9c34211296022..a5f8f9b47b3f62ddf1f27193cc9ef6fb26fa8377 100644
--- a/lib/Testing/Support/Error.cpp
+++ b/lib/Testing/Support/Error.cpp
@@ -1,9 +1,8 @@
 //===- llvm/Testing/Support/Error.cpp -------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Testing/Support/LLVMBuild.txt b/lib/Testing/Support/LLVMBuild.txt
index 173cfb4a5587e54f05cc6a5f2b6f01f6890d4893..98278f9c5b0b2f509ddbfa783648019dbeacef60 100644
--- a/lib/Testing/Support/LLVMBuild.txt
+++ b/lib/Testing/Support/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./Testing/Support/LLVMBuild.txt --------------------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/TextAPI/ELF/ELFStub.cpp b/lib/TextAPI/ELF/ELFStub.cpp
index 248a078a2404dbb1b0082421f315d1456f708ccc..f8463497093b14ab38f387d43c14491b6ff2fc5b 100644
--- a/lib/TextAPI/ELF/ELFStub.cpp
+++ b/lib/TextAPI/ELF/ELFStub.cpp
@@ -1,9 +1,8 @@
 //===- ELFStub.cpp --------------------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===-----------------------------------------------------------------------===/
 
diff --git a/lib/TextAPI/ELF/TBEHandler.cpp b/lib/TextAPI/ELF/TBEHandler.cpp
index dde980292a167623ee52fd15fbee0370b74648cf..cb597d8896e811f350c1931fbb1e60fc4549abf7 100644
--- a/lib/TextAPI/ELF/TBEHandler.cpp
+++ b/lib/TextAPI/ELF/TBEHandler.cpp
@@ -1,9 +1,8 @@
 //===- TBEHandler.cpp -----------------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===-----------------------------------------------------------------------===/
 
@@ -105,6 +104,7 @@ template <> struct MappingTraits<ELFSymbol> {
       IO.mapRequired("Size", Symbol.Size);
     }
     IO.mapOptional("Undefined", Symbol.Undefined, false);
+    IO.mapOptional("Weak", Symbol.Weak, false);
     IO.mapOptional("Warning", Symbol.Warning);
   }
 
diff --git a/lib/TextAPI/LLVMBuild.txt b/lib/TextAPI/LLVMBuild.txt
index 83733a72f506b93729748ddca4696be291deb881..fafab9f079812f09e9b57da077617efa1c890912 100644
--- a/lib/TextAPI/LLVMBuild.txt
+++ b/lib/TextAPI/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/TextAPI/LLVMBuild.txt ------------------------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/ToolDrivers/LLVMBuild.txt b/lib/ToolDrivers/LLVMBuild.txt
index a49e04bdf3c1f07072024c3864f9c5d0f46341a8..cd7ba29998b8d07ee03c8a84a73028173eb05b91 100644
--- a/lib/ToolDrivers/LLVMBuild.txt
+++ b/lib/ToolDrivers/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/ToolDrivers/LLVMBuild.txt --------------------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/ToolDrivers/llvm-dlltool/DlltoolDriver.cpp b/lib/ToolDrivers/llvm-dlltool/DlltoolDriver.cpp
index c5a28d4f1c080e56703deb09db27a4dee1958e50..f894211b6447f41e4c7541d5ee1ed2075a5c4f9f 100644
--- a/lib/ToolDrivers/llvm-dlltool/DlltoolDriver.cpp
+++ b/lib/ToolDrivers/llvm-dlltool/DlltoolDriver.cpp
@@ -1,9 +1,8 @@
 //===- DlltoolDriver.cpp - dlltool.exe-compatible driver ------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/ToolDrivers/llvm-dlltool/LLVMBuild.txt b/lib/ToolDrivers/llvm-dlltool/LLVMBuild.txt
index 11736eb47bcb82d363e2faf43b25fe49909a5e4c..b1f4ad90a4437ba31c86aedd596ba8eac107a7ae 100644
--- a/lib/ToolDrivers/llvm-dlltool/LLVMBuild.txt
+++ b/lib/ToolDrivers/llvm-dlltool/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/ToolDrivers/llvm-dlltool/LLVMBuild.txt -------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/ToolDrivers/llvm-lib/LLVMBuild.txt b/lib/ToolDrivers/llvm-lib/LLVMBuild.txt
index e4b32ec4af90d86d5377472100873c5c29eaa286..ba981317ac789fb7a642d4fb62b453d4aa846dcb 100644
--- a/lib/ToolDrivers/llvm-lib/LLVMBuild.txt
+++ b/lib/ToolDrivers/llvm-lib/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/LibDriver/LLVMBuild.txt ----------------------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/ToolDrivers/llvm-lib/LibDriver.cpp b/lib/ToolDrivers/llvm-lib/LibDriver.cpp
index 64f4fe423f25e4d217eea8d78551596a7f8d52be..3a5b9726295cece91b5ac04e74b667cb9fa2cc81 100644
--- a/lib/ToolDrivers/llvm-lib/LibDriver.cpp
+++ b/lib/ToolDrivers/llvm-lib/LibDriver.cpp
@@ -1,9 +1,8 @@
 //===- LibDriver.cpp - lib.exe-compatible driver --------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -97,6 +96,47 @@ static std::string findInputFile(StringRef File, ArrayRef<StringRef> Paths) {
   return "";
 }
 
+static void fatalOpenError(llvm::Error E, Twine File) {
+  if (!E)
+    return;
+  handleAllErrors(std::move(E), [&](const llvm::ErrorInfoBase &EIB) {
+    llvm::errs() << "error opening '" << File << "': " << EIB.message() << '\n';
+    exit(1);
+  });
+}
+
+static void doList(opt::InputArgList& Args) {
+  // lib.exe prints the contents of the first archive file.
+  std::unique_ptr<MemoryBuffer> B;
+  for (auto *Arg : Args.filtered(OPT_INPUT)) {
+    // Create or open the archive object.
+    ErrorOr<std::unique_ptr<MemoryBuffer>> MaybeBuf =
+        MemoryBuffer::getFile(Arg->getValue(), -1, false);
+    fatalOpenError(errorCodeToError(MaybeBuf.getError()), Arg->getValue());
+
+    if (identify_magic(MaybeBuf.get()->getBuffer()) == file_magic::archive) {
+      B = std::move(MaybeBuf.get());
+      break;
+    }
+  }
+
+  // lib.exe doesn't print an error if no .lib files are passed.
+  if (!B)
+    return;
+
+  Error Err = Error::success();
+  object::Archive Archive(B.get()->getMemBufferRef(), Err);
+  fatalOpenError(std::move(Err), B->getBufferIdentifier());
+
+  for (auto &C : Archive.children(Err)) {
+    Expected<StringRef> NameOrErr = C.getName();
+    fatalOpenError(NameOrErr.takeError(), B->getBufferIdentifier());
+    StringRef Name = NameOrErr.get();
+    llvm::outs() << Name << '\n';
+  }
+  fatalOpenError(std::move(Err), B->getBufferIdentifier());
+}
+
 int llvm::libDriverMain(ArrayRef<const char *> ArgsArr) {
   BumpPtrAllocator Alloc;
   StringSaver Saver(Alloc);
@@ -131,6 +171,11 @@ int llvm::libDriverMain(ArrayRef<const char *> ArgsArr) {
   if (!Args.hasArgNoClaim(OPT_INPUT))
     return 0;
 
+  if (Args.hasArg(OPT_lst)) {
+    doList(Args);
+    return 0;
+  }
+
   std::vector<StringRef> SearchPaths = getSearchPaths(&Args, Saver);
 
   // Create a NewArchiveMember for each input file.
diff --git a/lib/ToolDrivers/llvm-lib/Options.td b/lib/ToolDrivers/llvm-lib/Options.td
index dd41952b78786bf14063c7f30b13222e68af5a15..4c297f749ed95f36c5d585f9d8f07ceb62a41a55 100644
--- a/lib/ToolDrivers/llvm-lib/Options.td
+++ b/lib/ToolDrivers/llvm-lib/Options.td
@@ -10,6 +10,9 @@ class P<string name, string help> :
       Joined<["/", "-", "-?"], name#":">, HelpText<help>;
 
 def libpath: P<"libpath", "Object file search path">;
+
+// Can't be called "list" since that's a keyword.
+def lst    : F<"list">, HelpText<"List contents of .lib file on stdout">;
 def out    : P<"out", "Path to file to write output">;
 
 def llvmlibthin : F<"llvmlibthin">,
diff --git a/lib/Transforms/AggressiveInstCombine/AggressiveInstCombine.cpp b/lib/Transforms/AggressiveInstCombine/AggressiveInstCombine.cpp
index a9442255e34c1432818bbbc2b16bd932431f8faa..06222d7e7e44f8da21f8f81d9331bccd1dc33664 100644
--- a/lib/Transforms/AggressiveInstCombine/AggressiveInstCombine.cpp
+++ b/lib/Transforms/AggressiveInstCombine/AggressiveInstCombine.cpp
@@ -1,9 +1,8 @@
 //===- AggressiveInstCombine.cpp ------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -81,8 +80,8 @@ static bool foldGuardedRotateToFunnelShift(Instruction &I) {
     auto Sub = m_Sub(m_SpecificInt(Width), m_Value(R1));
 
     // rotate_left(X, Y) == (X << Y) | (X >> (Width - Y))
-    auto RotL = m_OneUse(m_c_Or(m_Shl(m_Value(L0), m_Value(L1)),
-                                m_LShr(m_Value(R0), Sub)));
+    auto RotL = m_OneUse(
+        m_c_Or(m_Shl(m_Value(L0), m_Value(L1)), m_LShr(m_Value(R0), Sub)));
     if (RotL.match(V) && L0 == R0 && L1 == R1) {
       X = L0;
       Y = L1;
@@ -90,8 +89,8 @@ static bool foldGuardedRotateToFunnelShift(Instruction &I) {
     }
 
     // rotate_right(X, Y) == (X >> Y) | (X << (Width - Y))
-    auto RotR = m_OneUse(m_c_Or(m_LShr(m_Value(L0), m_Value(L1)),
-                                m_Shl(m_Value(R0), Sub)));
+    auto RotR = m_OneUse(
+        m_c_Or(m_LShr(m_Value(L0), m_Value(L1)), m_Shl(m_Value(R0), Sub)));
     if (RotR.match(V) && L0 == R0 && L1 == R1) {
       X = L0;
       Y = L1;
@@ -124,8 +123,8 @@ static bool foldGuardedRotateToFunnelShift(Instruction &I) {
   Instruction *TermI = GuardBB->getTerminator();
   BasicBlock *TrueBB, *FalseBB;
   ICmpInst::Predicate Pred;
-  if (!match(TermI, m_Br(m_ICmp(Pred, m_Specific(RotAmt), m_ZeroInt()),
-                         TrueBB, FalseBB)))
+  if (!match(TermI, m_Br(m_ICmp(Pred, m_Specific(RotAmt), m_ZeroInt()), TrueBB,
+                         FalseBB)))
     return false;
 
   BasicBlock *PhiBB = Phi.getParent();
@@ -162,9 +161,9 @@ struct MaskOps {
   bool MatchAndChain;
   bool FoundAnd1;
 
-  MaskOps(unsigned BitWidth, bool MatchAnds) :
-      Root(nullptr), Mask(APInt::getNullValue(BitWidth)),
-      MatchAndChain(MatchAnds), FoundAnd1(false) {}
+  MaskOps(unsigned BitWidth, bool MatchAnds)
+      : Root(nullptr), Mask(APInt::getNullValue(BitWidth)),
+        MatchAndChain(MatchAnds), FoundAnd1(false) {}
 };
 
 /// This is a recursive helper for foldAnyOrAllBitsSet() that walks through a
@@ -245,8 +244,8 @@ static bool foldAnyOrAllBitsSet(Instruction &I) {
   IRBuilder<> Builder(&I);
   Constant *Mask = ConstantInt::get(I.getType(), MOps.Mask);
   Value *And = Builder.CreateAnd(MOps.Root, Mask);
-  Value *Cmp = MatchAllBitsSet ? Builder.CreateICmpEQ(And, Mask) :
-                                 Builder.CreateIsNotNull(And);
+  Value *Cmp = MatchAllBitsSet ? Builder.CreateICmpEQ(And, Mask)
+                               : Builder.CreateIsNotNull(And);
   Value *Zext = Builder.CreateZExt(Cmp, I.getType());
   I.replaceAllUsesWith(Zext);
   return true;
diff --git a/lib/Transforms/AggressiveInstCombine/AggressiveInstCombineInternal.h b/lib/Transforms/AggressiveInstCombine/AggressiveInstCombineInternal.h
index f3c8bde9f8ffb3d5a4da6f5e33bd09eb6c60655b..44e1c45664e7eafd8f1f0c903bb24043a9eaf9b7 100644
--- a/lib/Transforms/AggressiveInstCombine/AggressiveInstCombineInternal.h
+++ b/lib/Transforms/AggressiveInstCombine/AggressiveInstCombineInternal.h
@@ -1,9 +1,8 @@
 //===- AggressiveInstCombineInternal.h --------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Transforms/AggressiveInstCombine/LLVMBuild.txt b/lib/Transforms/AggressiveInstCombine/LLVMBuild.txt
index c05844f33de9c51512fe6f5cbfc274713515fffa..13728abb0d306fa4dc88ce7566f80ef29bb2fbc4 100644
--- a/lib/Transforms/AggressiveInstCombine/LLVMBuild.txt
+++ b/lib/Transforms/AggressiveInstCombine/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/Transforms/AggressiveInstCombine/LLVMBuild.txt -----*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Transforms/AggressiveInstCombine/TruncInstCombine.cpp b/lib/Transforms/AggressiveInstCombine/TruncInstCombine.cpp
index 8289b2d68f8a84334baa550c6837c66dd187eafa..7c5767912fd37fb8a7bb52ca131b73d5172bdcf7 100644
--- a/lib/Transforms/AggressiveInstCombine/TruncInstCombine.cpp
+++ b/lib/Transforms/AggressiveInstCombine/TruncInstCombine.cpp
@@ -1,9 +1,8 @@
 //===- TruncInstCombine.cpp -----------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Transforms/Coroutines/CoroCleanup.cpp b/lib/Transforms/Coroutines/CoroCleanup.cpp
index 359876627fce13735d98ceb6f2002c2319c8337e..1fb0a114d0c7ed7f14c1219b4531a0c4a5fbeb47 100644
--- a/lib/Transforms/Coroutines/CoroCleanup.cpp
+++ b/lib/Transforms/Coroutines/CoroCleanup.cpp
@@ -1,9 +1,8 @@
 //===- CoroCleanup.cpp - Coroutine Cleanup Pass ---------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 // This pass lowers all remaining coroutine intrinsics.
@@ -50,7 +49,7 @@ static void lowerSubFn(IRBuilder<> &Builder, CoroSubFnInst *SubFn) {
   Builder.SetInsertPoint(SubFn);
   auto *FramePtr = Builder.CreateBitCast(FrameRaw, FramePtrTy);
   auto *Gep = Builder.CreateConstInBoundsGEP2_32(FrameTy, FramePtr, 0, Index);
-  auto *Load = Builder.CreateLoad(Gep);
+  auto *Load = Builder.CreateLoad(FrameTy->getElementType(Index), Gep);
 
   SubFn->replaceAllUsesWith(Load);
 }
diff --git a/lib/Transforms/Coroutines/CoroEarly.cpp b/lib/Transforms/Coroutines/CoroEarly.cpp
index ac47a06281a5d63d8f6d131e166fb665322dedda..e6f501e5d196c47b343a699b432fbc7bbc2e3549 100644
--- a/lib/Transforms/Coroutines/CoroEarly.cpp
+++ b/lib/Transforms/Coroutines/CoroEarly.cpp
@@ -1,9 +1,8 @@
 //===- CoroEarly.cpp - Coroutine Early Function Pass ----------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 // This pass lowers coroutine intrinsics that hide the details of the exact
@@ -98,7 +97,7 @@ void Lowerer::lowerCoroDone(IntrinsicInst *II) {
   Builder.SetInsertPoint(II);
   auto *BCI = Builder.CreateBitCast(Operand, FramePtrTy);
   auto *Gep = Builder.CreateConstInBoundsGEP1_32(FrameTy, BCI, 0);
-  auto *Load = Builder.CreateLoad(Gep);
+  auto *Load = Builder.CreateLoad(FrameTy, Gep);
   auto *Cond = Builder.CreateICmpEQ(Load, NullPtr);
 
   II->replaceAllUsesWith(Cond);
diff --git a/lib/Transforms/Coroutines/CoroElide.cpp b/lib/Transforms/Coroutines/CoroElide.cpp
index 58f952b54f3a7d4451f63e6dab3ef2dd93d6af75..6707aa1c827d618c782bda26d66cf834712cc181 100644
--- a/lib/Transforms/Coroutines/CoroElide.cpp
+++ b/lib/Transforms/Coroutines/CoroElide.cpp
@@ -1,9 +1,8 @@
 //===- CoroElide.cpp - Coroutine Frame Allocation Elision Pass ------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 // This pass replaces dynamic allocation of coroutine frame with alloca and
diff --git a/lib/Transforms/Coroutines/CoroFrame.cpp b/lib/Transforms/Coroutines/CoroFrame.cpp
index 4cb0a52961cca8522a679a816315ce83f0f599aa..24d685b7113a58b88556a73ff0595e39b09def52 100644
--- a/lib/Transforms/Coroutines/CoroFrame.cpp
+++ b/lib/Transforms/Coroutines/CoroFrame.cpp
@@ -1,9 +1,8 @@
 //===- CoroFrame.cpp - Builds and manipulates coroutine frame -------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 // This file contains classes used to discover if for a particular value
@@ -472,10 +471,10 @@ static Instruction *splitBeforeCatchSwitch(CatchSwitchInst *CatchSwitch) {
 static Instruction *insertSpills(SpillInfo &Spills, coro::Shape &Shape) {
   auto *CB = Shape.CoroBegin;
   IRBuilder<> Builder(CB->getNextNode());
-  PointerType *FramePtrTy = Shape.FrameTy->getPointerTo();
+  StructType *FrameTy = Shape.FrameTy;
+  PointerType *FramePtrTy = FrameTy->getPointerTo();
   auto *FramePtr =
       cast<Instruction>(Builder.CreateBitCast(CB, FramePtrTy, "FramePtr"));
-  Type *FrameTy = FramePtrTy->getElementType();
 
   Value *CurrentValue = nullptr;
   BasicBlock *CurrentBlock = nullptr;
@@ -502,7 +501,7 @@ static Instruction *insertSpills(SpillInfo &Spills, coro::Shape &Shape) {
                                                      Twine(".reload.addr"));
     return isa<AllocaInst>(CurrentValue)
                ? G
-               : Builder.CreateLoad(G,
+               : Builder.CreateLoad(FrameTy->getElementType(Index), G,
                                     CurrentValue->getName() + Twine(".reload"));
   };
 
diff --git a/lib/Transforms/Coroutines/CoroInstr.h b/lib/Transforms/Coroutines/CoroInstr.h
index 9a8cc5a2591c9d1b2e95f02b5155774fbcf968fb..5e19d7642e38e8595c94f8ca011feece4fa41213 100644
--- a/lib/Transforms/Coroutines/CoroInstr.h
+++ b/lib/Transforms/Coroutines/CoroInstr.h
@@ -1,9 +1,8 @@
 //===-- CoroInstr.h - Coroutine Intrinsics Instruction Wrappers -*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 // This file defines classes that make it really easy to deal with intrinsic
diff --git a/lib/Transforms/Coroutines/CoroInternal.h b/lib/Transforms/Coroutines/CoroInternal.h
index 8e690d649cf5e464c5e7d25522fc50f1e3ee0aed..441c8a20f1f3a5c60141307d80d517780164c1ef 100644
--- a/lib/Transforms/Coroutines/CoroInternal.h
+++ b/lib/Transforms/Coroutines/CoroInternal.h
@@ -1,9 +1,8 @@
 //===- CoroInternal.h - Internal Coroutine interfaces ---------*- C++ -*---===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 // Common definitions/declarations used internally by coroutine lowering passes.
diff --git a/lib/Transforms/Coroutines/CoroSplit.cpp b/lib/Transforms/Coroutines/CoroSplit.cpp
index 9eeceb217ba8bc03e353b8f9971eb24e9681e41d..f7d3072ccbcb496ddc39735bc7e125732a20ba5a 100644
--- a/lib/Transforms/Coroutines/CoroSplit.cpp
+++ b/lib/Transforms/Coroutines/CoroSplit.cpp
@@ -1,9 +1,8 @@
 //===- CoroSplit.cpp - Converts a coroutine into a state machine ----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 // This pass builds the coroutine frame and outlines resume and destroy parts
@@ -94,7 +93,7 @@ static BasicBlock *createResumeEntryBlock(Function &F, coro::Shape &Shape) {
   auto *FrameTy = Shape.FrameTy;
   auto *GepIndex = Builder.CreateConstInBoundsGEP2_32(
       FrameTy, FramePtr, 0, coro::Shape::IndexField, "index.addr");
-  auto *Index = Builder.CreateLoad(GepIndex, "index");
+  auto *Index = Builder.CreateLoad(Shape.getIndexType(), GepIndex, "index");
   auto *Switch =
       Builder.CreateSwitch(Index, UnreachBB, Shape.CoroSuspends.size());
   Shape.ResumeSwitch = Switch;
@@ -230,7 +229,8 @@ static void handleFinalSuspend(IRBuilder<> &Builder, Value *FramePtr,
     Builder.SetInsertPoint(OldSwitchBB->getTerminator());
     auto *GepIndex = Builder.CreateConstInBoundsGEP2_32(Shape.FrameTy, FramePtr,
                                                         0, 0, "ResumeFn.addr");
-    auto *Load = Builder.CreateLoad(GepIndex);
+    auto *Load = Builder.CreateLoad(
+        Shape.FrameTy->getElementType(coro::Shape::ResumeField), GepIndex);
     auto *NullPtr =
         ConstantPointerNull::get(cast<PointerType>(Load->getType()));
     auto *Cond = Builder.CreateICmpEQ(Load, NullPtr);
@@ -828,6 +828,7 @@ static void splitCoroutine(Function &F, CallGraph &CG, CallGraphSCC &SCC) {
 // split.
 static void prepareForSplit(Function &F, CallGraph &CG) {
   Module &M = *F.getParent();
+  LLVMContext &Context = F.getContext();
 #ifndef NDEBUG
   Function *DevirtFn = M.getFunction(CORO_DEVIRT_TRIGGER_FN);
   assert(DevirtFn && "coro.devirt.trigger function not found");
@@ -842,10 +843,12 @@ static void prepareForSplit(Function &F, CallGraph &CG) {
   //    call void %1(i8* null)
   coro::LowererBase Lowerer(M);
   Instruction *InsertPt = F.getEntryBlock().getTerminator();
-  auto *Null = ConstantPointerNull::get(Type::getInt8PtrTy(F.getContext()));
+  auto *Null = ConstantPointerNull::get(Type::getInt8PtrTy(Context));
   auto *DevirtFnAddr =
       Lowerer.makeSubFnCall(Null, CoroSubFnInst::RestartTrigger, InsertPt);
-  auto *IndirectCall = CallInst::Create(DevirtFnAddr, Null, "", InsertPt);
+  FunctionType *FnTy = FunctionType::get(Type::getVoidTy(Context),
+                                         {Type::getInt8PtrTy(Context)}, false);
+  auto *IndirectCall = CallInst::Create(FnTy, DevirtFnAddr, Null, "", InsertPt);
 
   // Update CG graph with an indirect call we just added.
   CG[&F]->addCalledFunction(IndirectCall, CG.getCallsExternalNode());
diff --git a/lib/Transforms/Coroutines/Coroutines.cpp b/lib/Transforms/Coroutines/Coroutines.cpp
index cf84f916e24b7934446d4ef36e7a2c9f1771dc02..7bd87fd29a630c51a913a10c2eaef2db9892b92f 100644
--- a/lib/Transforms/Coroutines/Coroutines.cpp
+++ b/lib/Transforms/Coroutines/Coroutines.cpp
@@ -1,9 +1,8 @@
 //===- Coroutines.cpp -----------------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Transforms/Coroutines/LLVMBuild.txt b/lib/Transforms/Coroutines/LLVMBuild.txt
index 41dc4b031a49ad63d7b07a15a600decd59601d8b..63c7fd04493c5b3440daf7cc9dca0fd72746e3ac 100644
--- a/lib/Transforms/Coroutines/LLVMBuild.txt
+++ b/lib/Transforms/Coroutines/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/Transforms/Coroutines/LLVMBuild.txt ----------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Transforms/Hello/CMakeLists.txt b/lib/Transforms/Hello/CMakeLists.txt
index 4a55dd9c04b8019b37efe6b2d55513d23aea70ae..c4f10247c1a6ccab7d9d872761702c834282fcc8 100644
--- a/lib/Transforms/Hello/CMakeLists.txt
+++ b/lib/Transforms/Hello/CMakeLists.txt
@@ -10,7 +10,7 @@ if(WIN32 OR CYGWIN)
   set(LLVM_LINK_COMPONENTS Core Support)
 endif()
 
-add_llvm_loadable_module( LLVMHello
+add_llvm_library( LLVMHello MODULE BUILDTREE_ONLY
   Hello.cpp
 
   DEPENDS
diff --git a/lib/Transforms/Hello/Hello.cpp b/lib/Transforms/Hello/Hello.cpp
index 29b9bb8a94eaf86fc3f87008b3bb33de3f0d6482..b0adb5401f8912427c0c69ee1ccaed299e521ab2 100644
--- a/lib/Transforms/Hello/Hello.cpp
+++ b/lib/Transforms/Hello/Hello.cpp
@@ -1,9 +1,8 @@
 //===- Hello.cpp - Example code from "Writing an LLVM Pass" ---------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Transforms/IPO/AlwaysInliner.cpp b/lib/Transforms/IPO/AlwaysInliner.cpp
index 07138718ce2cdcebcc23dc907da1dd98325f9ed6..0058bb9832afb6ff946432886074a914d078f04f 100644
--- a/lib/Transforms/IPO/AlwaysInliner.cpp
+++ b/lib/Transforms/IPO/AlwaysInliner.cpp
@@ -1,9 +1,8 @@
 //===- InlineAlways.cpp - Code to inline always_inline functions ----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -146,11 +145,20 @@ InlineCost AlwaysInlinerLegacyPass::getInlineCost(CallSite CS) {
   Function *Callee = CS.getCalledFunction();
 
   // Only inline direct calls to functions with always-inline attributes
-  // that are viable for inlining. FIXME: We shouldn't even get here for
-  // declarations.
-  if (Callee && !Callee->isDeclaration() &&
-      CS.hasFnAttr(Attribute::AlwaysInline) && isInlineViable(*Callee))
-    return InlineCost::getAlways("always inliner");
+  // that are viable for inlining.
+  if (!Callee)
+    return InlineCost::getNever("indirect call");
+
+  // FIXME: We shouldn't even get here for declarations.
+  if (Callee->isDeclaration())
+    return InlineCost::getNever("no definition");
+
+  if (!CS.hasFnAttr(Attribute::AlwaysInline))
+    return InlineCost::getNever("no alwaysinline attribute");
+
+  auto IsViable = isInlineViable(*Callee);
+  if (!IsViable)
+    return InlineCost::getNever(IsViable.message);
 
-  return InlineCost::getNever("always inliner");
+  return InlineCost::getAlways("always inliner");
 }
diff --git a/lib/Transforms/IPO/ArgumentPromotion.cpp b/lib/Transforms/IPO/ArgumentPromotion.cpp
index 517a9c082a4cf6f4fe46c79a7be73ca77d4a0a75..30763f26971f677bb129572a717bca0669f52477 100644
--- a/lib/Transforms/IPO/ArgumentPromotion.cpp
+++ b/lib/Transforms/IPO/ArgumentPromotion.cpp
@@ -1,9 +1,8 @@
 //===- ArgumentPromotion.cpp - Promote by-reference arguments -------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -49,6 +48,7 @@
 #include "llvm/Analysis/Loads.h"
 #include "llvm/Analysis/MemoryLocation.h"
 #include "llvm/Analysis/TargetLibraryInfo.h"
+#include "llvm/Analysis/TargetTransformInfo.h"
 #include "llvm/IR/Argument.h"
 #include "llvm/IR/Attributes.h"
 #include "llvm/IR/BasicBlock.h"
@@ -216,10 +216,7 @@ doPromotion(Function *F, SmallPtrSetImpl<Argument *> &ArgsToPromote,
   Function *NF = Function::Create(NFTy, F->getLinkage(), F->getAddressSpace(),
                                   F->getName());
   NF->copyAttributesFrom(F);
-
-  // Patch the pointer to LLVM function in debug info descriptor.
-  NF->setSubprogram(F->getSubprogram());
-  F->setSubprogram(nullptr);
+  NF->copyMetadata(F, 0);
 
   LLVM_DEBUG(dbgs() << "ARG PROMOTION:  Promoting to:" << *NF << "\n"
                     << "From: " << *F);
@@ -263,7 +260,8 @@ doPromotion(Function *F, SmallPtrSetImpl<Argument *> &ArgsToPromote,
           Value *Idx = GetElementPtrInst::Create(
               STy, *AI, Idxs, (*AI)->getName() + "." + Twine(i), Call);
           // TODO: Tell AA about the new values?
-          Args.push_back(new LoadInst(Idx, Idx->getName() + ".val", Call));
+          Args.push_back(new LoadInst(STy->getElementType(i), Idx,
+                                      Idx->getName() + ".val", Call));
           ArgAttrVec.push_back(AttributeSet());
         }
       } else if (!I->use_empty()) {
@@ -299,7 +297,8 @@ doPromotion(Function *F, SmallPtrSetImpl<Argument *> &ArgsToPromote,
           }
           // Since we're replacing a load make sure we take the alignment
           // of the previous load.
-          LoadInst *newLoad = new LoadInst(V, V->getName() + ".val", Call);
+          LoadInst *newLoad =
+              new LoadInst(OrigLoad->getType(), V, V->getName() + ".val", Call);
           newLoad->setAlignment(OrigLoad->getAlignment());
           // Transfer the AA info too.
           AAMDNodes AAInfo;
@@ -809,6 +808,21 @@ static bool canPaddingBeAccessed(Argument *arg) {
   return false;
 }
 
+static bool areFunctionArgsABICompatible(
+    const Function &F, const TargetTransformInfo &TTI,
+    SmallPtrSetImpl<Argument *> &ArgsToPromote,
+    SmallPtrSetImpl<Argument *> &ByValArgsToTransform) {
+  for (const Use &U : F.uses()) {
+    CallSite CS(U.getUser());
+    const Function *Caller = CS.getCaller();
+    const Function *Callee = CS.getCalledFunction();
+    if (!TTI.areFunctionArgsABICompatible(Caller, Callee, ArgsToPromote) ||
+        !TTI.areFunctionArgsABICompatible(Caller, Callee, ByValArgsToTransform))
+      return false;
+  }
+  return true;
+}
+
 /// PromoteArguments - This method checks the specified function to see if there
 /// are any promotable arguments and if it is safe to promote the function (for
 /// example, all callers are direct).  If safe to promote some arguments, it
@@ -817,7 +831,8 @@ static Function *
 promoteArguments(Function *F, function_ref<AAResults &(Function &F)> AARGetter,
                  unsigned MaxElements,
                  Optional<function_ref<void(CallSite OldCS, CallSite NewCS)>>
-                     ReplaceCallSite) {
+                     ReplaceCallSite,
+                 const TargetTransformInfo &TTI) {
   // Don't perform argument promotion for naked functions; otherwise we can end
   // up removing parameters that are seemingly 'not used' as they are referred
   // to in the assembly.
@@ -846,7 +861,7 @@ promoteArguments(Function *F, function_ref<AAResults &(Function &F)> AARGetter,
 
   // Second check: make sure that all callers are direct callers.  We can't
   // transform functions that have indirect callers.  Also see if the function
-  // is self-recursive.
+  // is self-recursive and check that target features are compatible.
   bool isSelfRecursive = false;
   for (Use &U : F->uses()) {
     CallSite CS(U.getUser());
@@ -955,6 +970,10 @@ promoteArguments(Function *F, function_ref<AAResults &(Function &F)> AARGetter,
   if (ArgsToPromote.empty() && ByValArgsToTransform.empty())
     return nullptr;
 
+  if (!areFunctionArgsABICompatible(*F, TTI, ArgsToPromote,
+                                    ByValArgsToTransform))
+    return nullptr;
+
   return doPromotion(F, ArgsToPromote, ByValArgsToTransform, ReplaceCallSite);
 }
 
@@ -980,7 +999,9 @@ PreservedAnalyses ArgumentPromotionPass::run(LazyCallGraph::SCC &C,
         return FAM.getResult<AAManager>(F);
       };
 
-      Function *NewF = promoteArguments(&OldF, AARGetter, MaxElements, None);
+      const TargetTransformInfo &TTI = FAM.getResult<TargetIRAnalysis>(OldF);
+      Function *NewF =
+          promoteArguments(&OldF, AARGetter, MaxElements, None, TTI);
       if (!NewF)
         continue;
       LocalChange = true;
@@ -1018,6 +1039,7 @@ struct ArgPromotion : public CallGraphSCCPass {
   void getAnalysisUsage(AnalysisUsage &AU) const override {
     AU.addRequired<AssumptionCacheTracker>();
     AU.addRequired<TargetLibraryInfoWrapperPass>();
+    AU.addRequired<TargetTransformInfoWrapperPass>();
     getAAResultsAnalysisUsage(AU);
     CallGraphSCCPass::getAnalysisUsage(AU);
   }
@@ -1043,6 +1065,7 @@ INITIALIZE_PASS_BEGIN(ArgPromotion, "argpromotion",
 INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker)
 INITIALIZE_PASS_DEPENDENCY(CallGraphWrapperPass)
 INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)
+INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass)
 INITIALIZE_PASS_END(ArgPromotion, "argpromotion",
                     "Promote 'by reference' arguments to scalars", false, false)
 
@@ -1079,8 +1102,10 @@ bool ArgPromotion::runOnSCC(CallGraphSCC &SCC) {
         CallerNode->replaceCallEdge(OldCS, NewCS, NewCalleeNode);
       };
 
+      const TargetTransformInfo &TTI =
+          getAnalysis<TargetTransformInfoWrapperPass>().getTTI(*OldF);
       if (Function *NewF = promoteArguments(OldF, AARGetter, MaxElements,
-                                            {ReplaceCallSite})) {
+                                            {ReplaceCallSite}, TTI)) {
         LocalChange = true;
 
         // Update the call graph for the newly promoted function.
diff --git a/lib/Transforms/IPO/BarrierNoopPass.cpp b/lib/Transforms/IPO/BarrierNoopPass.cpp
index 05fc3dd6950c1ecbe4ae6cd4c394aa59029bd1ed..6b68aa90c56764e13fb48ae2569c54bdc9c71289 100644
--- a/lib/Transforms/IPO/BarrierNoopPass.cpp
+++ b/lib/Transforms/IPO/BarrierNoopPass.cpp
@@ -1,9 +1,8 @@
 //===- BarrierNoopPass.cpp - A barrier pass for the pass manager ----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Transforms/IPO/BlockExtractor.cpp b/lib/Transforms/IPO/BlockExtractor.cpp
index ff5ee817da49bdf95e6a2d3fc09959051a555449..eb12d032c529f651d9f0b6c6f3c18c832bfb9d72 100644
--- a/lib/Transforms/IPO/BlockExtractor.cpp
+++ b/lib/Transforms/IPO/BlockExtractor.cpp
@@ -1,9 +1,8 @@
 //===- BlockExtractor.cpp - Extracts blocks into their own functions ------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Transforms/IPO/CalledValuePropagation.cpp b/lib/Transforms/IPO/CalledValuePropagation.cpp
index de62cfc0c1db18429bafb674829277814181b1b4..20cb3213628ec558ac606e5688e2f2767cb06ad0 100644
--- a/lib/Transforms/IPO/CalledValuePropagation.cpp
+++ b/lib/Transforms/IPO/CalledValuePropagation.cpp
@@ -1,9 +1,8 @@
 //===- CalledValuePropagation.cpp - Propagate called values -----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Transforms/IPO/ConstantMerge.cpp b/lib/Transforms/IPO/ConstantMerge.cpp
index 81f3634eaf28315169a964bf78387c0caf07784b..ad877ae1786c9cf890d21da444848ce62fbac791 100644
--- a/lib/Transforms/IPO/ConstantMerge.cpp
+++ b/lib/Transforms/IPO/ConstantMerge.cpp
@@ -1,9 +1,8 @@
 //===- ConstantMerge.cpp - Merge duplicate global constants ---------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -91,6 +90,16 @@ static unsigned getAlignment(GlobalVariable *GV) {
   return GV->getParent()->getDataLayout().getPreferredAlignment(GV);
 }
 
+static bool
+isUnmergeableGlobal(GlobalVariable *GV,
+                    const SmallPtrSetImpl<const GlobalValue *> &UsedGlobals) {
+  // Only process constants with initializers in the default address space.
+  return !GV->isConstant() || !GV->hasDefinitiveInitializer() ||
+         GV->getType()->getAddressSpace() != 0 || GV->hasSection() ||
+         // Don't touch values marked with attribute(used).
+         UsedGlobals.count(GV);
+}
+
 enum class CanMerge { No, Yes };
 static CanMerge makeMergeable(GlobalVariable *Old, GlobalVariable *New) {
   if (!Old->hasGlobalUnnamedAddr() && !New->hasGlobalUnnamedAddr())
@@ -155,11 +164,7 @@ static bool mergeConstants(Module &M) {
         continue;
       }
 
-      // Only process constants with initializers in the default address space.
-      if (!GV->isConstant() || !GV->hasDefinitiveInitializer() ||
-          GV->getType()->getAddressSpace() != 0 || GV->hasSection() ||
-          // Don't touch values marked with attribute(used).
-          UsedGlobals.count(GV))
+      if (isUnmergeableGlobal(GV, UsedGlobals))
         continue;
 
       // This transformation is legal for weak ODR globals in the sense it
@@ -197,11 +202,7 @@ static bool mergeConstants(Module &M) {
          GVI != E; ) {
       GlobalVariable *GV = &*GVI++;
 
-      // Only process constants with initializers in the default address space.
-      if (!GV->isConstant() || !GV->hasDefinitiveInitializer() ||
-          GV->getType()->getAddressSpace() != 0 || GV->hasSection() ||
-          // Don't touch values marked with attribute(used).
-          UsedGlobals.count(GV))
+      if (isUnmergeableGlobal(GV, UsedGlobals))
         continue;
 
       // We can only replace constant with local linkage.
diff --git a/lib/Transforms/IPO/CrossDSOCFI.cpp b/lib/Transforms/IPO/CrossDSOCFI.cpp
index 666f6cc37bfd55d5ebbf2caad840b94e2280555f..e30b33aa48724e204d24254df2de85aab0ee3d2f 100644
--- a/lib/Transforms/IPO/CrossDSOCFI.cpp
+++ b/lib/Transforms/IPO/CrossDSOCFI.cpp
@@ -1,9 +1,8 @@
 //===-- CrossDSOCFI.cpp - Externalize this module's CFI checks ------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -106,10 +105,10 @@ void CrossDSOCFI::buildCFICheck(Module &M) {
   }
 
   LLVMContext &Ctx = M.getContext();
-  Constant *C = M.getOrInsertFunction(
+  FunctionCallee C = M.getOrInsertFunction(
       "__cfi_check", Type::getVoidTy(Ctx), Type::getInt64Ty(Ctx),
       Type::getInt8PtrTy(Ctx), Type::getInt8PtrTy(Ctx));
-  Function *F = dyn_cast<Function>(C);
+  Function *F = dyn_cast<Function>(C.getCallee());
   // Take over the existing function. The frontend emits a weak stub so that the
   // linker knows about the symbol; this pass replaces the function body.
   F->deleteBody();
@@ -133,9 +132,9 @@ void CrossDSOCFI::buildCFICheck(Module &M) {
 
   BasicBlock *TrapBB = BasicBlock::Create(Ctx, "fail", F);
   IRBuilder<> IRBFail(TrapBB);
-  Constant *CFICheckFailFn = M.getOrInsertFunction(
-      "__cfi_check_fail", Type::getVoidTy(Ctx), Type::getInt8PtrTy(Ctx),
-      Type::getInt8PtrTy(Ctx));
+  FunctionCallee CFICheckFailFn =
+      M.getOrInsertFunction("__cfi_check_fail", Type::getVoidTy(Ctx),
+                            Type::getInt8PtrTy(Ctx), Type::getInt8PtrTy(Ctx));
   IRBFail.CreateCall(CFICheckFailFn, {&CFICheckFailData, &Addr});
   IRBFail.CreateBr(ExitBB);
 
diff --git a/lib/Transforms/IPO/DeadArgumentElimination.cpp b/lib/Transforms/IPO/DeadArgumentElimination.cpp
index cb30e8f46a54dfc92f38f955caccaabf288c8abf..968a13110b16d00605e4748bfdd4ff109401a240 100644
--- a/lib/Transforms/IPO/DeadArgumentElimination.cpp
+++ b/lib/Transforms/IPO/DeadArgumentElimination.cpp
@@ -1,9 +1,8 @@
 //===- DeadArgumentElimination.cpp - Eliminate dead arguments -------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -939,7 +938,7 @@ bool DeadArgumentEliminationPass::RemoveDeadStuffFromFunction(Function *F) {
       NewCS = InvokeInst::Create(NF, II->getNormalDest(), II->getUnwindDest(),
                                  Args, OpBundles, "", Call->getParent());
     } else {
-      NewCS = CallInst::Create(NF, Args, OpBundles, "", Call);
+      NewCS = CallInst::Create(NFTy, NF, Args, OpBundles, "", Call);
       cast<CallInst>(NewCS.getInstruction())
           ->setTailCallKind(cast<CallInst>(Call)->getTailCallKind());
     }
diff --git a/lib/Transforms/IPO/ElimAvailExtern.cpp b/lib/Transforms/IPO/ElimAvailExtern.cpp
index d5fef59286dd566b849c1816a89947295255d903..fc52db562c623341516b248ad8f8342704b6532a 100644
--- a/lib/Transforms/IPO/ElimAvailExtern.cpp
+++ b/lib/Transforms/IPO/ElimAvailExtern.cpp
@@ -1,9 +1,8 @@
 //===- ElimAvailExtern.cpp - DCE unreachable internal functions -----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Transforms/IPO/ExtractGV.cpp b/lib/Transforms/IPO/ExtractGV.cpp
index a744d7f2d2d9597202c32c98183d8b80ed845827..f77b528fc42da3f79786cd2d43a53634e381e809 100644
--- a/lib/Transforms/IPO/ExtractGV.cpp
+++ b/lib/Transforms/IPO/ExtractGV.cpp
@@ -1,9 +1,8 @@
 //===-- ExtractGV.cpp - Global Value extraction pass ----------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Transforms/IPO/ForceFunctionAttrs.cpp b/lib/Transforms/IPO/ForceFunctionAttrs.cpp
index 4dc1529ddbf5cfa085dfe3911073f479ffed9d40..cd1fc3798201fb465df2e7a0eb13e35115409c8d 100644
--- a/lib/Transforms/IPO/ForceFunctionAttrs.cpp
+++ b/lib/Transforms/IPO/ForceFunctionAttrs.cpp
@@ -1,9 +1,8 @@
 //===- ForceFunctionAttrs.cpp - Force function attrs for debugging --------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Transforms/IPO/FunctionAttrs.cpp b/lib/Transforms/IPO/FunctionAttrs.cpp
index 3a04f7a00d413b65b8ca8af08a6e246cdbb719da..77e98ecd9103cfa7d4bd10294fe808985c53bb91 100644
--- a/lib/Transforms/IPO/FunctionAttrs.cpp
+++ b/lib/Transforms/IPO/FunctionAttrs.cpp
@@ -1,9 +1,8 @@
 //===- FunctionAttrs.cpp - Pass which marks functions attributes ----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -130,16 +129,15 @@ static MemoryAccessKind checkFunctionMemoryAccess(Function &F, bool ThisBody,
 
     // Some instructions can be ignored even if they read or write memory.
     // Detect these now, skipping to the next instruction if one is found.
-    CallSite CS(cast<Value>(I));
-    if (CS) {
+    if (auto *Call = dyn_cast<CallBase>(I)) {
       // Ignore calls to functions in the same SCC, as long as the call sites
       // don't have operand bundles.  Calls with operand bundles are allowed to
       // have memory effects not described by the memory effects of the call
       // target.
-      if (!CS.hasOperandBundles() && CS.getCalledFunction() &&
-          SCCNodes.count(CS.getCalledFunction()))
+      if (!Call->hasOperandBundles() && Call->getCalledFunction() &&
+          SCCNodes.count(Call->getCalledFunction()))
         continue;
-      FunctionModRefBehavior MRB = AAR.getModRefBehavior(CS);
+      FunctionModRefBehavior MRB = AAR.getModRefBehavior(Call);
       ModRefInfo MRI = createModRefInfo(MRB);
 
       // If the call doesn't access memory, we're done.
@@ -158,7 +156,7 @@ static MemoryAccessKind checkFunctionMemoryAccess(Function &F, bool ThisBody,
 
       // Check whether all pointer arguments point to local memory, and
       // ignore calls that only access local memory.
-      for (CallSite::arg_iterator CI = CS.arg_begin(), CE = CS.arg_end();
+      for (CallSite::arg_iterator CI = Call->arg_begin(), CE = Call->arg_end();
            CI != CE; ++CI) {
         Value *Arg = *CI;
         if (!Arg->getType()->isPtrOrPtrVectorTy())
diff --git a/lib/Transforms/IPO/FunctionImport.cpp b/lib/Transforms/IPO/FunctionImport.cpp
index 531a7c19e360075466d83a3dbd7d030ff432cb46..dcd246d0d7eb42d3890887176f16e65049fd8333 100644
--- a/lib/Transforms/IPO/FunctionImport.cpp
+++ b/lib/Transforms/IPO/FunctionImport.cpp
@@ -1,9 +1,8 @@
 //===- FunctionImport.cpp - ThinLTO Summary-based Function Import ---------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -777,9 +776,12 @@ void llvm::computeDeadSymbols(
     VI = updateValueInfoForIndirectCalls(Index, VI);
     if (!VI)
       return;
-    for (auto &S : VI.getSummaryList())
-      if (S->isLive())
-        return;
+
+    if (llvm::any_of(VI.getSummaryList(),
+                     [](const std::unique_ptr<llvm::GlobalValueSummary> &S) {
+                       return S->isLive();
+                     }))
+      return;
 
     // We only keep live symbols that are known to be non-prevailing if any are
     // available_externally, linkonceodr, weakodr. Those symbols are discarded
@@ -815,12 +817,23 @@ void llvm::computeDeadSymbols(
   while (!Worklist.empty()) {
     auto VI = Worklist.pop_back_val();
     for (auto &Summary : VI.getSummaryList()) {
-      GlobalValueSummary *Base = Summary->getBaseObject();
-      // Set base value live in case it is an alias.
-      Base->setLive(true);
-      for (auto Ref : Base->refs())
+      if (auto *AS = dyn_cast<AliasSummary>(Summary.get())) {
+        // If this is an alias, visit the aliasee VI to ensure that all copies
+        // are marked live and it is added to the worklist for further
+        // processing of its references.
+        // FIXME: The aliasee GUID is only populated in the summary when we
+        // read them from bitcode, which is currently the only way we can
+        // get here (we don't yet support reading the summary index directly
+        // from LLVM assembly code in tools that can perform a thin link).
+        // If that ever changes, the below call to getAliaseGUID will assert.
+        visit(Index.getValueInfo(AS->getAliaseeGUID()));
+        continue;
+      }
+
+      Summary->setLive(true);
+      for (auto Ref : Summary->refs())
         visit(Ref);
-      if (auto *FS = dyn_cast<FunctionSummary>(Base))
+      if (auto *FS = dyn_cast<FunctionSummary>(Summary.get()))
         for (auto Call : FS->calls())
           visit(Call.first);
     }
diff --git a/lib/Transforms/IPO/GlobalDCE.cpp b/lib/Transforms/IPO/GlobalDCE.cpp
index 34de87433367c8c576dde9e853efebc4ca1b1b77..86b7f3e49ee6f83cbaa7beec1e87290bc122693c 100644
--- a/lib/Transforms/IPO/GlobalDCE.cpp
+++ b/lib/Transforms/IPO/GlobalDCE.cpp
@@ -1,9 +1,8 @@
 //===-- GlobalDCE.cpp - DCE unreachable internal functions ----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Transforms/IPO/GlobalOpt.cpp b/lib/Transforms/IPO/GlobalOpt.cpp
index 3005aafd06b13b43861e366be5b5043744625238..c89541dc73295c3294e6f8f2d29549f51cfdb777 100644
--- a/lib/Transforms/IPO/GlobalOpt.cpp
+++ b/lib/Transforms/IPO/GlobalOpt.cpp
@@ -1,9 +1,8 @@
 //===- GlobalOpt.cpp - Optimize Global Variables --------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -730,7 +729,8 @@ static bool OptimizeAwayTrappingUsesOfValue(Value *V, Constant *NewV) {
           break;
       if (Idxs.size() == GEPI->getNumOperands()-1)
         Changed |= OptimizeAwayTrappingUsesOfValue(
-            GEPI, ConstantExpr::getGetElementPtr(nullptr, NewV, Idxs));
+            GEPI, ConstantExpr::getGetElementPtr(GEPI->getSourceElementType(),
+                                                 NewV, Idxs));
       if (GEPI->use_empty()) {
         Changed = true;
         GEPI->eraseFromParent();
@@ -906,9 +906,10 @@ OptimizeGlobalAddressOfMalloc(GlobalVariable *GV, CallInst *CI, Type *AllocTy,
 
       // Replace the cmp X, 0 with a use of the bool value.
       // Sink the load to where the compare was, if atomic rules allow us to.
-      Value *LV = new LoadInst(InitBool, InitBool->getName()+".val", false, 0,
+      Value *LV = new LoadInst(InitBool->getValueType(), InitBool,
+                               InitBool->getName() + ".val", false, 0,
                                LI->getOrdering(), LI->getSyncScopeID(),
-                               LI->isUnordered() ? (Instruction*)ICI : LI);
+                               LI->isUnordered() ? (Instruction *)ICI : LI);
       InitBoolUsed = true;
       switch (ICI->getPredicate()) {
       default: llvm_unreachable("Unknown ICmp Predicate!");
@@ -1041,7 +1042,8 @@ static void ReplaceUsesOfMallocWithGlobal(Instruction *Alloc,
     }
 
     // Insert a load from the global, and use it instead of the malloc.
-    Value *NL = new LoadInst(GV, GV->getName()+".val", InsertPt);
+    Value *NL =
+        new LoadInst(GV->getValueType(), GV, GV->getName() + ".val", InsertPt);
     U->replaceUsesOfWith(Alloc, NL);
   }
 }
@@ -1164,10 +1166,10 @@ static Value *GetHeapSROAValue(Value *V, unsigned FieldNo,
   if (LoadInst *LI = dyn_cast<LoadInst>(V)) {
     // This is a scalarized version of the load from the global.  Just create
     // a new Load of the scalarized global.
-    Result = new LoadInst(GetHeapSROAValue(LI->getOperand(0), FieldNo,
-                                           InsertedScalarizedValues,
-                                           PHIsToRewrite),
-                          LI->getName()+".f"+Twine(FieldNo), LI);
+    Value *V = GetHeapSROAValue(LI->getOperand(0), FieldNo,
+                                InsertedScalarizedValues, PHIsToRewrite);
+    Result = new LoadInst(V->getType()->getPointerElementType(), V,
+                          LI->getName() + ".f" + Twine(FieldNo), LI);
   } else {
     PHINode *PN = cast<PHINode>(V);
     // PN's type is pointer to struct.  Make a new PHI of pointer to struct
@@ -1357,7 +1359,9 @@ static GlobalVariable *PerformHeapAllocSRoA(GlobalVariable *GV, CallInst *CI,
   // Within the NullPtrBlock, we need to emit a comparison and branch for each
   // pointer, because some may be null while others are not.
   for (unsigned i = 0, e = FieldGlobals.size(); i != e; ++i) {
-    Value *GVVal = new LoadInst(FieldGlobals[i], "tmp", NullPtrBlock);
+    Value *GVVal =
+        new LoadInst(cast<GlobalVariable>(FieldGlobals[i])->getValueType(),
+                     FieldGlobals[i], "tmp", NullPtrBlock);
     Value *Cmp = new ICmpInst(*NullPtrBlock, ICmpInst::ICMP_NE, GVVal,
                               Constant::getNullValue(GVVal->getType()));
     BasicBlock *FreeBlock = BasicBlock::Create(Cmp->getContext(), "free_it",
@@ -1701,7 +1705,8 @@ static bool TryToShrinkGlobalToBoolean(GlobalVariable *GV, Constant *OtherVal) {
         if (LoadInst *LI = dyn_cast<LoadInst>(StoredVal)) {
           assert(LI->getOperand(0) == GV && "Not a copy!");
           // Insert a new load, to preserve the saved value.
-          StoreVal = new LoadInst(NewGV, LI->getName()+".b", false, 0,
+          StoreVal = new LoadInst(NewGV->getValueType(), NewGV,
+                                  LI->getName() + ".b", false, 0,
                                   LI->getOrdering(), LI->getSyncScopeID(), LI);
         } else {
           assert((isa<CastInst>(StoredVal) || isa<SelectInst>(StoredVal)) &&
@@ -1717,8 +1722,9 @@ static bool TryToShrinkGlobalToBoolean(GlobalVariable *GV, Constant *OtherVal) {
     } else {
       // Change the load into a load of bool then a select.
       LoadInst *LI = cast<LoadInst>(UI);
-      LoadInst *NLI = new LoadInst(NewGV, LI->getName()+".b", false, 0,
-                                   LI->getOrdering(), LI->getSyncScopeID(), LI);
+      LoadInst *NLI =
+          new LoadInst(NewGV->getValueType(), NewGV, LI->getName() + ".b",
+                       false, 0, LI->getOrdering(), LI->getSyncScopeID(), LI);
       Instruction *NSI;
       if (IsOneZero)
         NSI = new ZExtInst(NLI, LI->getType(), "", LI);
@@ -2808,46 +2814,20 @@ static Function *FindCXAAtExit(Module &M, TargetLibraryInfo *TLI) {
 /// Returns whether the given function is an empty C++ destructor and can
 /// therefore be eliminated.
 /// Note that we assume that other optimization passes have already simplified
-/// the code so we only look for a function with a single basic block, where
-/// the only allowed instructions are 'ret', 'call' to an empty C++ dtor and
-/// other side-effect free instructions.
-static bool cxxDtorIsEmpty(const Function &Fn,
-                           SmallPtrSet<const Function *, 8> &CalledFunctions) {
+/// the code so we simply check for 'ret'.
+static bool cxxDtorIsEmpty(const Function &Fn) {
   // FIXME: We could eliminate C++ destructors if they're readonly/readnone and
   // nounwind, but that doesn't seem worth doing.
   if (Fn.isDeclaration())
     return false;
 
-  if (++Fn.begin() != Fn.end())
-    return false;
-
-  const BasicBlock &EntryBlock = Fn.getEntryBlock();
-  for (BasicBlock::const_iterator I = EntryBlock.begin(), E = EntryBlock.end();
-       I != E; ++I) {
-    if (const CallInst *CI = dyn_cast<CallInst>(I)) {
-      // Ignore debug intrinsics.
-      if (isa<DbgInfoIntrinsic>(CI))
-        continue;
-
-      const Function *CalledFn = CI->getCalledFunction();
-
-      if (!CalledFn)
-        return false;
-
-      SmallPtrSet<const Function *, 8> NewCalledFunctions(CalledFunctions);
-
-      // Don't treat recursive functions as empty.
-      if (!NewCalledFunctions.insert(CalledFn).second)
-        return false;
-
-      if (!cxxDtorIsEmpty(*CalledFn, NewCalledFunctions))
-        return false;
-    } else if (isa<ReturnInst>(*I))
-      return true; // We're done.
-    else if (I->mayHaveSideEffects())
-      return false; // Destructor with side effects, bail.
+  for (auto &I : Fn.getEntryBlock()) {
+    if (isa<DbgInfoIntrinsic>(I))
+      continue;
+    if (isa<ReturnInst>(I))
+      return true;
+    break;
   }
-
   return false;
 }
 
@@ -2879,11 +2859,7 @@ static bool OptimizeEmptyGlobalCXXDtors(Function *CXAAtExitFn) {
 
     Function *DtorFn =
       dyn_cast<Function>(CI->getArgOperand(0)->stripPointerCasts());
-    if (!DtorFn)
-      continue;
-
-    SmallPtrSet<const Function *, 8> CalledFunctions;
-    if (!cxxDtorIsEmpty(*DtorFn, CalledFunctions))
+    if (!DtorFn || !cxxDtorIsEmpty(*DtorFn))
       continue;
 
     // Just remove the call.
diff --git a/lib/Transforms/IPO/GlobalSplit.cpp b/lib/Transforms/IPO/GlobalSplit.cpp
index 792f4b3052a3fd07dee1a22b637e981e632143a3..060043a40b89a2023cc3800aad80ce2da14be141 100644
--- a/lib/Transforms/IPO/GlobalSplit.cpp
+++ b/lib/Transforms/IPO/GlobalSplit.cpp
@@ -1,9 +1,8 @@
 //===- GlobalSplit.cpp - global variable splitter -------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Transforms/IPO/HotColdSplitting.cpp b/lib/Transforms/IPO/HotColdSplitting.cpp
index 5d989a40f65f2058485aac8d98b10ed3ab34cc8f..b8def7ad3ea986b7bd22faf66c45035d5af5f301 100644
--- a/lib/Transforms/IPO/HotColdSplitting.cpp
+++ b/lib/Transforms/IPO/HotColdSplitting.cpp
@@ -1,16 +1,28 @@
 //===- HotColdSplitting.cpp -- Outline Cold Regions -------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
-//
-// Outline cold regions to a separate function.
-// TODO: Update BFI and BPI
-// TODO: Add all the outlined functions to a separate section.
-//
+///
+/// \file
+/// The goal of hot/cold splitting is to improve the memory locality of code.
+/// The splitting pass does this by identifying cold blocks and moving them into
+/// separate functions.
+///
+/// When the splitting pass finds a cold block (referred to as "the sink"), it
+/// grows a maximal cold region around that block. The maximal region contains
+/// all blocks (post-)dominated by the sink [*]. In theory, these blocks are as
+/// cold as the sink. Once a region is found, it's split out of the original
+/// function provided it's profitable to do so.
+///
+/// [*] In practice, there is some added complexity because some blocks are not
+/// safe to extract.
+///
+/// TODO: Use the PM to get domtrees, and preserve BFI/BPI.
+/// TODO: Reorder outlined functions.
+///
 //===----------------------------------------------------------------------===//
 
 #include "llvm/ADT/PostOrderIterator.h"
@@ -53,7 +65,6 @@
 #include "llvm/Transforms/Utils/Cloning.h"
 #include "llvm/Transforms/Utils/CodeExtractor.h"
 #include "llvm/Transforms/Utils/Local.h"
-#include "llvm/Transforms/Utils/SSAUpdater.h"
 #include "llvm/Transforms/Utils/ValueMapper.h"
 #include <algorithm>
 #include <cassert>
@@ -69,16 +80,12 @@ static cl::opt<bool> EnableStaticAnalyis("hot-cold-static-analysis",
                               cl::init(true), cl::Hidden);
 
 static cl::opt<int>
-    MinOutliningThreshold("min-outlining-thresh", cl::init(3), cl::Hidden,
-                          cl::desc("Code size threshold for outlining within a "
-                                   "single BB (as a multiple of TCC_Basic)"));
+    SplittingThreshold("hotcoldsplit-threshold", cl::init(2), cl::Hidden,
+                       cl::desc("Base penalty for splitting cold code (as a "
+                                "multiple of TCC_Basic)"));
 
 namespace {
 
-struct PostDomTree : PostDomTreeBase<BasicBlock> {
-  PostDomTree(Function &F) { recalculate(F); }
-};
-
 /// A sequence of basic blocks.
 ///
 /// A 0-sized SmallVector is slightly cheaper to move than a std::vector.
@@ -101,7 +108,7 @@ bool blockEndsInUnreachable(const BasicBlock &BB) {
 
 bool unlikelyExecuted(BasicBlock &BB) {
   // Exception handling blocks are unlikely executed.
-  if (BB.isEHPad())
+  if (BB.isEHPad() || isa<ResumeInst>(BB.getTerminator()))
     return true;
 
   // The block is cold if it calls/invokes a cold function.
@@ -125,38 +132,39 @@ bool unlikelyExecuted(BasicBlock &BB) {
 
 /// Check whether it's safe to outline \p BB.
 static bool mayExtractBlock(const BasicBlock &BB) {
-  return !BB.hasAddressTaken() && !BB.isEHPad();
-}
-
-/// Check whether \p Region is profitable to outline.
-static bool isProfitableToOutline(const BlockSequence &Region,
-                                  TargetTransformInfo &TTI) {
-  if (Region.size() > 1)
-    return true;
-
-  int Cost = 0;
-  const BasicBlock &BB = *Region[0];
-  for (const Instruction &I : BB) {
-    if (isa<DbgInfoIntrinsic>(&I) || &I == BB.getTerminator())
-      continue;
-
-    Cost += TTI.getInstructionCost(&I, TargetTransformInfo::TCK_CodeSize);
-
-    if (Cost >= (MinOutliningThreshold * TargetTransformInfo::TCC_Basic))
-      return true;
-  }
-  return false;
+  // EH pads are unsafe to outline because doing so breaks EH type tables. It
+  // follows that invoke instructions cannot be extracted, because CodeExtractor
+  // requires unwind destinations to be within the extraction region.
+  //
+  // Resumes that are not reachable from a cleanup landing pad are considered to
+  // be unreachable. It’s not safe to split them out either.
+  auto Term = BB.getTerminator();
+  return !BB.hasAddressTaken() && !BB.isEHPad() && !isa<InvokeInst>(Term) &&
+         !isa<ResumeInst>(Term);
 }
 
-/// Mark \p F cold. Return true if it's changed.
-static bool markEntireFunctionCold(Function &F) {
+/// Mark \p F cold. Based on this assumption, also optimize it for minimum size.
+/// If \p UpdateEntryCount is true (set when this is a new split function and
+/// module has profile data), set entry count to 0 to ensure treated as cold.
+/// Return true if the function is changed.
+static bool markFunctionCold(Function &F, bool UpdateEntryCount = false) {
   assert(!F.hasFnAttribute(Attribute::OptimizeNone) && "Can't mark this cold");
   bool Changed = false;
+  if (!F.hasFnAttribute(Attribute::Cold)) {
+    F.addFnAttr(Attribute::Cold);
+    Changed = true;
+  }
   if (!F.hasFnAttribute(Attribute::MinSize)) {
     F.addFnAttr(Attribute::MinSize);
     Changed = true;
   }
-  // TODO: Move this function into a cold section.
+  if (UpdateEntryCount) {
+    // Set the entry count to 0 to ensure it is placed in the unlikely text
+    // section when function sections are enabled.
+    F.setEntryCount(0);
+    Changed = true;
+  }
+
   return Changed;
 }
 
@@ -165,24 +173,24 @@ public:
   HotColdSplitting(ProfileSummaryInfo *ProfSI,
                    function_ref<BlockFrequencyInfo *(Function &)> GBFI,
                    function_ref<TargetTransformInfo &(Function &)> GTTI,
-                   std::function<OptimizationRemarkEmitter &(Function &)> *GORE)
-      : PSI(ProfSI), GetBFI(GBFI), GetTTI(GTTI), GetORE(GORE) {}
+                   std::function<OptimizationRemarkEmitter &(Function &)> *GORE,
+                   function_ref<AssumptionCache *(Function &)> LAC)
+      : PSI(ProfSI), GetBFI(GBFI), GetTTI(GTTI), GetORE(GORE), LookupAC(LAC) {}
   bool run(Module &M);
 
 private:
+  bool isFunctionCold(const Function &F) const;
   bool shouldOutlineFrom(const Function &F) const;
-  bool outlineColdRegions(Function &F, ProfileSummaryInfo &PSI,
-                          BlockFrequencyInfo *BFI, TargetTransformInfo &TTI,
-                          DominatorTree &DT, PostDomTree &PDT,
-                          OptimizationRemarkEmitter &ORE);
+  bool outlineColdRegions(Function &F, bool HasProfileSummary);
   Function *extractColdRegion(const BlockSequence &Region, DominatorTree &DT,
                               BlockFrequencyInfo *BFI, TargetTransformInfo &TTI,
-                              OptimizationRemarkEmitter &ORE, unsigned Count);
-  SmallPtrSet<const Function *, 2> OutlinedFunctions;
+                              OptimizationRemarkEmitter &ORE,
+                              AssumptionCache *AC, unsigned Count);
   ProfileSummaryInfo *PSI;
   function_ref<BlockFrequencyInfo *(Function &)> GetBFI;
   function_ref<TargetTransformInfo &(Function &)> GetTTI;
   std::function<OptimizationRemarkEmitter &(Function &)> *GetORE;
+  function_ref<AssumptionCache *(Function &)> LookupAC;
 };
 
 class HotColdSplittingLegacyPass : public ModulePass {
@@ -193,10 +201,10 @@ public:
   }
 
   void getAnalysisUsage(AnalysisUsage &AU) const override {
-    AU.addRequired<AssumptionCacheTracker>();
     AU.addRequired<BlockFrequencyInfoWrapperPass>();
     AU.addRequired<ProfileSummaryInfoWrapperPass>();
     AU.addRequired<TargetTransformInfoWrapperPass>();
+    AU.addUsedIfAvailable<AssumptionCacheTracker>();
   }
 
   bool runOnModule(Module &M) override;
@@ -204,33 +212,106 @@ public:
 
 } // end anonymous namespace
 
+/// Check whether \p F is inherently cold.
+bool HotColdSplitting::isFunctionCold(const Function &F) const {
+  if (F.hasFnAttribute(Attribute::Cold))
+    return true;
+
+  if (F.getCallingConv() == CallingConv::Cold)
+    return true;
+
+  if (PSI->isFunctionEntryCold(&F))
+    return true;
+
+  return false;
+}
+
 // Returns false if the function should not be considered for hot-cold split
 // optimization.
 bool HotColdSplitting::shouldOutlineFrom(const Function &F) const {
-  // Do not try to outline again from an already outlined cold function.
-  if (OutlinedFunctions.count(&F))
+  if (F.hasFnAttribute(Attribute::AlwaysInline))
     return false;
 
-  if (F.size() <= 2)
+  if (F.hasFnAttribute(Attribute::NoInline))
     return false;
 
-  // TODO: Consider only skipping functions marked `optnone` or `cold`.
+  return true;
+}
 
-  if (F.hasAddressTaken())
-    return false;
+/// Get the benefit score of outlining \p Region.
+static int getOutliningBenefit(ArrayRef<BasicBlock *> Region,
+                               TargetTransformInfo &TTI) {
+  // Sum up the code size costs of non-terminator instructions. Tight coupling
+  // with \ref getOutliningPenalty is needed to model the costs of terminators.
+  int Benefit = 0;
+  for (BasicBlock *BB : Region)
+    for (Instruction &I : BB->instructionsWithoutDebug())
+      if (&I != BB->getTerminator())
+        Benefit +=
+            TTI.getInstructionCost(&I, TargetTransformInfo::TCK_CodeSize);
+
+  return Benefit;
+}
 
-  if (F.hasFnAttribute(Attribute::AlwaysInline))
-    return false;
+/// Get the penalty score for outlining \p Region.
+static int getOutliningPenalty(ArrayRef<BasicBlock *> Region,
+                               unsigned NumInputs, unsigned NumOutputs) {
+  int Penalty = SplittingThreshold;
+  LLVM_DEBUG(dbgs() << "Applying penalty for splitting: " << Penalty << "\n");
+
+  // If the splitting threshold is set at or below zero, skip the usual
+  // profitability check.
+  if (SplittingThreshold <= 0)
+    return Penalty;
+
+  // The typical code size cost for materializing an argument for the outlined
+  // call.
+  LLVM_DEBUG(dbgs() << "Applying penalty for: " << NumInputs << " inputs\n");
+  const int CostForArgMaterialization = TargetTransformInfo::TCC_Basic;
+  Penalty += CostForArgMaterialization * NumInputs;
+
+  // The typical code size cost for an output alloca, its associated store, and
+  // its associated reload.
+  LLVM_DEBUG(dbgs() << "Applying penalty for: " << NumOutputs << " outputs\n");
+  const int CostForRegionOutput = 3 * TargetTransformInfo::TCC_Basic;
+  Penalty += CostForRegionOutput * NumOutputs;
+
+  // Find the number of distinct exit blocks for the region. Use a conservative
+  // check to determine whether control returns from the region.
+  bool NoBlocksReturn = true;
+  SmallPtrSet<BasicBlock *, 2> SuccsOutsideRegion;
+  for (BasicBlock *BB : Region) {
+    // If a block has no successors, only assume it does not return if it's
+    // unreachable.
+    if (succ_empty(BB)) {
+      NoBlocksReturn &= isa<UnreachableInst>(BB->getTerminator());
+      continue;
+    }
 
-  if (F.hasFnAttribute(Attribute::NoInline))
-    return false;
+    for (BasicBlock *SuccBB : successors(BB)) {
+      if (find(Region, SuccBB) == Region.end()) {
+        NoBlocksReturn = false;
+        SuccsOutsideRegion.insert(SuccBB);
+      }
+    }
+  }
 
-  if (F.getCallingConv() == CallingConv::Cold)
-    return false;
+  // Apply a `noreturn` bonus.
+  if (NoBlocksReturn) {
+    LLVM_DEBUG(dbgs() << "Applying bonus for: " << Region.size()
+                      << " non-returning terminators\n");
+    Penalty -= Region.size();
+  }
 
-  if (PSI->isFunctionEntryCold(&F))
-    return false;
-  return true;
+  // Apply a penalty for having more than one successor outside of the region.
+  // This penalty accounts for the switch needed in the caller.
+  if (!SuccsOutsideRegion.empty()) {
+    LLVM_DEBUG(dbgs() << "Applying penalty for: " << SuccsOutsideRegion.size()
+                      << " non-region successors\n");
+    Penalty += (SuccsOutsideRegion.size() - 1) * TargetTransformInfo::TCC_Basic;
+  }
+
+  return Penalty;
 }
 
 Function *HotColdSplitting::extractColdRegion(const BlockSequence &Region,
@@ -238,25 +319,28 @@ Function *HotColdSplitting::extractColdRegion(const BlockSequence &Region,
                                               BlockFrequencyInfo *BFI,
                                               TargetTransformInfo &TTI,
                                               OptimizationRemarkEmitter &ORE,
+                                              AssumptionCache *AC,
                                               unsigned Count) {
   assert(!Region.empty());
 
   // TODO: Pass BFI and BPI to update profile information.
   CodeExtractor CE(Region, &DT, /* AggregateArgs */ false, /* BFI */ nullptr,
-                   /* BPI */ nullptr, /* AllowVarArgs */ false,
+                   /* BPI */ nullptr, AC, /* AllowVarArgs */ false,
                    /* AllowAlloca */ false,
                    /* Suffix */ "cold." + std::to_string(Count));
 
+  // Perform a simple cost/benefit analysis to decide whether or not to permit
+  // splitting.
   SetVector<Value *> Inputs, Outputs, Sinks;
   CE.findInputsOutputs(Inputs, Outputs, Sinks);
-
-  // Do not extract regions that have live exit variables.
-  if (Outputs.size() > 0) {
-    LLVM_DEBUG(llvm::dbgs() << "Not outlining; live outputs\n");
+  int OutliningBenefit = getOutliningBenefit(Region, TTI);
+  int OutliningPenalty =
+      getOutliningPenalty(Region, Inputs.size(), Outputs.size());
+  LLVM_DEBUG(dbgs() << "Split profitability: benefit = " << OutliningBenefit
+                    << ", penalty = " << OutliningPenalty << "\n");
+  if (OutliningBenefit <= OutliningPenalty)
     return nullptr;
-  }
 
-  // TODO: Run MergeBasicBlockIntoOnlyPred on the outlined function.
   Function *OrigF = Region[0]->getParent();
   if (Function *OutF = CE.extractCodeRegion()) {
     User *U = *OutF->user_begin();
@@ -269,9 +353,7 @@ Function *HotColdSplitting::extractColdRegion(const BlockSequence &Region,
     }
     CI->setIsNoInline();
 
-    // Try to make the outlined code as small as possible on the assumption
-    // that it's cold.
-    markEntireFunctionCold(*OutF);
+    markFunctionCold(*OutF, BFI != nullptr);
 
     LLVM_DEBUG(llvm::dbgs() << "Outlined Region: " << *OutF);
     ORE.emit([&]() {
@@ -295,8 +377,11 @@ Function *HotColdSplitting::extractColdRegion(const BlockSequence &Region,
 /// A pair of (basic block, score).
 using BlockTy = std::pair<BasicBlock *, unsigned>;
 
+namespace {
 /// A maximal outlining region. This contains all blocks post-dominated by a
 /// sink block, the sink block itself, and all blocks dominated by the sink.
+/// If sink-predecessors and sink-successors cannot be extracted in one region,
+/// the static constructor returns a list of suitable extraction regions.
 class OutliningRegion {
   /// A list of (block, score) pairs. A block's score is non-zero iff it's a
   /// viable sub-region entry point. Blocks with higher scores are better entry
@@ -311,12 +396,9 @@ class OutliningRegion {
   /// Whether the entire function is cold.
   bool EntireFunctionCold = false;
 
-  /// Whether or not \p BB could be the entry point of an extracted region.
-  static bool isViableEntryPoint(BasicBlock &BB) { return !BB.isEHPad(); }
-
   /// If \p BB is a viable entry point, return \p Score. Return 0 otherwise.
   static unsigned getEntryPointScore(BasicBlock &BB, unsigned Score) {
-    return isViableEntryPoint(BB) ? Score : 0;
+    return mayExtractBlock(BB) ? Score : 0;
   }
 
   /// These scores should be lower than the score for predecessor blocks,
@@ -332,21 +414,23 @@ public:
   OutliningRegion(OutliningRegion &&) = default;
   OutliningRegion &operator=(OutliningRegion &&) = default;
 
-  static OutliningRegion create(BasicBlock &SinkBB, const DominatorTree &DT,
-                                const PostDomTree &PDT) {
-    OutliningRegion ColdRegion;
-
+  static std::vector<OutliningRegion> create(BasicBlock &SinkBB,
+                                             const DominatorTree &DT,
+                                             const PostDominatorTree &PDT) {
+    std::vector<OutliningRegion> Regions;
     SmallPtrSet<BasicBlock *, 4> RegionBlocks;
 
+    Regions.emplace_back();
+    OutliningRegion *ColdRegion = &Regions.back();
+
     auto addBlockToRegion = [&](BasicBlock *BB, unsigned Score) {
       RegionBlocks.insert(BB);
-      ColdRegion.Blocks.emplace_back(BB, Score);
-      assert(RegionBlocks.size() == ColdRegion.Blocks.size() && "Duplicate BB");
+      ColdRegion->Blocks.emplace_back(BB, Score);
     };
 
     // The ancestor farthest-away from SinkBB, and also post-dominated by it.
     unsigned SinkScore = getEntryPointScore(SinkBB, ScoreForSinkBlock);
-    ColdRegion.SuggestedEntryPoint = (SinkScore > 0) ? &SinkBB : nullptr;
+    ColdRegion->SuggestedEntryPoint = (SinkScore > 0) ? &SinkBB : nullptr;
     unsigned BestScore = SinkScore;
 
     // Visit SinkBB's ancestors using inverse DFS.
@@ -359,8 +443,8 @@ public:
       // If the predecessor is cold and has no predecessors, the entire
       // function must be cold.
       if (SinkPostDom && pred_empty(&PredBB)) {
-        ColdRegion.EntireFunctionCold = true;
-        return ColdRegion;
+        ColdRegion->EntireFunctionCold = true;
+        return Regions;
       }
 
       // If SinkBB does not post-dominate a predecessor, do not mark the
@@ -375,7 +459,7 @@ public:
       // considered as entry points before the sink block.
       unsigned PredScore = getEntryPointScore(PredBB, PredIt.getPathLength());
       if (PredScore > BestScore) {
-        ColdRegion.SuggestedEntryPoint = &PredBB;
+        ColdRegion->SuggestedEntryPoint = &PredBB;
         BestScore = PredScore;
       }
 
@@ -383,9 +467,19 @@ public:
       ++PredIt;
     }
 
-    // Add SinkBB to the cold region. It's considered as an entry point before
-    // any sink-successor blocks.
-    addBlockToRegion(&SinkBB, SinkScore);
+    // If the sink can be added to the cold region, do so. It's considered as
+    // an entry point before any sink-successor blocks.
+    //
+    // Otherwise, split cold sink-successor blocks using a separate region.
+    // This satisfies the requirement that all extraction blocks other than the
+    // first have predecessors within the extraction region.
+    if (mayExtractBlock(SinkBB)) {
+      addBlockToRegion(&SinkBB, SinkScore);
+    } else {
+      Regions.emplace_back();
+      ColdRegion = &Regions.back();
+      BestScore = 0;
+    }
 
     // Find all successors of SinkBB dominated by SinkBB using DFS.
     auto SuccIt = ++df_begin(&SinkBB);
@@ -406,7 +500,7 @@ public:
 
       unsigned SuccScore = getEntryPointScore(SuccBB, ScoreForSuccBlock);
       if (SuccScore > BestScore) {
-        ColdRegion.SuggestedEntryPoint = &SuccBB;
+        ColdRegion->SuggestedEntryPoint = &SuccBB;
         BestScore = SuccScore;
       }
 
@@ -414,7 +508,7 @@ public:
       ++SuccIt;
     }
 
-    return ColdRegion;
+    return Regions;
   }
 
   /// Whether this region has nothing to extract.
@@ -458,12 +552,9 @@ public:
     return SubRegion;
   }
 };
+} // namespace
 
-bool HotColdSplitting::outlineColdRegions(Function &F, ProfileSummaryInfo &PSI,
-                                          BlockFrequencyInfo *BFI,
-                                          TargetTransformInfo &TTI,
-                                          DominatorTree &DT, PostDomTree &PDT,
-                                          OptimizationRemarkEmitter &ORE) {
+bool HotColdSplitting::outlineColdRegions(Function &F, bool HasProfileSummary) {
   bool Changed = false;
 
   // The set of cold blocks.
@@ -477,17 +568,28 @@ bool HotColdSplitting::outlineColdRegions(Function &F, ProfileSummaryInfo &PSI,
   // the first region to contain a block.
   ReversePostOrderTraversal<Function *> RPOT(&F);
 
+  // Calculate domtrees lazily. This reduces compile-time significantly.
+  std::unique_ptr<DominatorTree> DT;
+  std::unique_ptr<PostDominatorTree> PDT;
+
+  // Calculate BFI lazily (it's only used to query ProfileSummaryInfo). This
+  // reduces compile-time significantly. TODO: When we *do* use BFI, we should
+  // be able to salvage its domtrees instead of recomputing them.
+  BlockFrequencyInfo *BFI = nullptr;
+  if (HasProfileSummary)
+    BFI = GetBFI(F);
+
+  TargetTransformInfo &TTI = GetTTI(F);
+  OptimizationRemarkEmitter &ORE = (*GetORE)(F);
+  AssumptionCache *AC = LookupAC(F);
+
   // Find all cold regions.
   for (BasicBlock *BB : RPOT) {
-    // Skip blocks which can't be outlined.
-    if (!mayExtractBlock(*BB))
-      continue;
-
     // This block is already part of some outlining region.
     if (ColdBlocks.count(BB))
       continue;
 
-    bool Cold = PSI.isColdBlock(BB, BFI) ||
+    bool Cold = (BFI && PSI->isColdBlock(BB, BFI)) ||
                 (EnableStaticAnalyis && unlikelyExecuted(*BB));
     if (!Cold)
       continue;
@@ -497,28 +599,35 @@ bool HotColdSplitting::outlineColdRegions(Function &F, ProfileSummaryInfo &PSI,
       BB->dump();
     });
 
-    auto Region = OutliningRegion::create(*BB, DT, PDT);
-    if (Region.empty())
-      continue;
+    if (!DT)
+      DT = make_unique<DominatorTree>(F);
+    if (!PDT)
+      PDT = make_unique<PostDominatorTree>(F);
 
-    if (Region.isEntireFunctionCold()) {
-      LLVM_DEBUG(dbgs() << "Entire function is cold\n");
-      return markEntireFunctionCold(F);
-    }
+    auto Regions = OutliningRegion::create(*BB, *DT, *PDT);
+    for (OutliningRegion &Region : Regions) {
+      if (Region.empty())
+        continue;
 
-    // If this outlining region intersects with another, drop the new region.
-    //
-    // TODO: It's theoretically possible to outline more by only keeping the
-    // largest region which contains a block, but the extra bookkeeping to do
-    // this is tricky/expensive.
-    bool RegionsOverlap = any_of(Region.blocks(), [&](const BlockTy &Block) {
-      return !ColdBlocks.insert(Block.first).second;
-    });
-    if (RegionsOverlap)
-      continue;
+      if (Region.isEntireFunctionCold()) {
+        LLVM_DEBUG(dbgs() << "Entire function is cold\n");
+        return markFunctionCold(F);
+      }
 
-    OutliningWorklist.emplace_back(std::move(Region));
-    ++NumColdRegionsFound;
+      // If this outlining region intersects with another, drop the new region.
+      //
+      // TODO: It's theoretically possible to outline more by only keeping the
+      // largest region which contains a block, but the extra bookkeeping to do
+      // this is tricky/expensive.
+      bool RegionsOverlap = any_of(Region.blocks(), [&](const BlockTy &Block) {
+        return !ColdBlocks.insert(Block.first).second;
+      });
+      if (RegionsOverlap)
+        continue;
+
+      OutliningWorklist.emplace_back(std::move(Region));
+      ++NumColdRegionsFound;
+    }
   }
 
   // Outline single-entry cold regions, splitting up larger regions as needed.
@@ -527,26 +636,17 @@ bool HotColdSplitting::outlineColdRegions(Function &F, ProfileSummaryInfo &PSI,
     OutliningRegion Region = OutliningWorklist.pop_back_val();
     assert(!Region.empty() && "Empty outlining region in worklist");
     do {
-      BlockSequence SubRegion = Region.takeSingleEntrySubRegion(DT);
-      if (!isProfitableToOutline(SubRegion, TTI)) {
-        LLVM_DEBUG({
-          dbgs() << "Skipping outlining; not profitable to outline\n";
-          SubRegion[0]->dump();
-        });
-        continue;
-      }
-
+      BlockSequence SubRegion = Region.takeSingleEntrySubRegion(*DT);
       LLVM_DEBUG({
         dbgs() << "Hot/cold splitting attempting to outline these blocks:\n";
         for (BasicBlock *BB : SubRegion)
           BB->dump();
       });
 
-      Function *Outlined =
-          extractColdRegion(SubRegion, DT, BFI, TTI, ORE, OutlinedFunctionID);
+      Function *Outlined = extractColdRegion(SubRegion, *DT, BFI, TTI, ORE, AC,
+                                             OutlinedFunctionID);
       if (Outlined) {
         ++OutlinedFunctionID;
-        OutlinedFunctions.insert(Outlined);
         Changed = true;
       }
     } while (!Region.empty());
@@ -557,20 +657,31 @@ bool HotColdSplitting::outlineColdRegions(Function &F, ProfileSummaryInfo &PSI,
 
 bool HotColdSplitting::run(Module &M) {
   bool Changed = false;
-  OutlinedFunctions.clear();
-  for (auto &F : M) {
+  bool HasProfileSummary = M.getProfileSummary();
+  for (auto It = M.begin(), End = M.end(); It != End; ++It) {
+    Function &F = *It;
+
+    // Do not touch declarations.
+    if (F.isDeclaration())
+      continue;
+
+    // Do not modify `optnone` functions.
+    if (F.hasFnAttribute(Attribute::OptimizeNone))
+      continue;
+
+    // Detect inherently cold functions and mark them as such.
+    if (isFunctionCold(F)) {
+      Changed |= markFunctionCold(F);
+      continue;
+    }
+
     if (!shouldOutlineFrom(F)) {
       LLVM_DEBUG(llvm::dbgs() << "Skipping " << F.getName() << "\n");
       continue;
     }
+
     LLVM_DEBUG(llvm::dbgs() << "Outlining in " << F.getName() << "\n");
-    DominatorTree DT(F);
-    PostDomTree PDT(F);
-    PDT.recalculate(F);
-    BlockFrequencyInfo *BFI = GetBFI(F);
-    TargetTransformInfo &TTI = GetTTI(F);
-    OptimizationRemarkEmitter &ORE = (*GetORE)(F);
-    Changed |= outlineColdRegions(F, *PSI, BFI, TTI, DT, PDT, ORE);
+    Changed |= outlineColdRegions(F, HasProfileSummary);
   }
   return Changed;
 }
@@ -592,17 +703,21 @@ bool HotColdSplittingLegacyPass::runOnModule(Module &M) {
     ORE.reset(new OptimizationRemarkEmitter(&F));
     return *ORE.get();
   };
+  auto LookupAC = [this](Function &F) -> AssumptionCache * {
+    if (auto *ACT = getAnalysisIfAvailable<AssumptionCacheTracker>())
+      return ACT->lookupAssumptionCache(F);
+    return nullptr;
+  };
 
-  return HotColdSplitting(PSI, GBFI, GTTI, &GetORE).run(M);
+  return HotColdSplitting(PSI, GBFI, GTTI, &GetORE, LookupAC).run(M);
 }
 
 PreservedAnalyses
 HotColdSplittingPass::run(Module &M, ModuleAnalysisManager &AM) {
   auto &FAM = AM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager();
 
-  std::function<AssumptionCache &(Function &)> GetAssumptionCache =
-      [&FAM](Function &F) -> AssumptionCache & {
-    return FAM.getResult<AssumptionAnalysis>(F);
+  auto LookupAC = [&FAM](Function &F) -> AssumptionCache * {
+    return FAM.getCachedResult<AssumptionAnalysis>(F);
   };
 
   auto GBFI = [&FAM](Function &F) {
@@ -623,7 +738,7 @@ HotColdSplittingPass::run(Module &M, ModuleAnalysisManager &AM) {
 
   ProfileSummaryInfo *PSI = &AM.getResult<ProfileSummaryAnalysis>(M);
 
-  if (HotColdSplitting(PSI, GBFI, GTTI, &GetORE).run(M))
+  if (HotColdSplitting(PSI, GBFI, GTTI, &GetORE, LookupAC).run(M))
     return PreservedAnalyses::none();
   return PreservedAnalyses::all();
 }
diff --git a/lib/Transforms/IPO/IPConstantPropagation.cpp b/lib/Transforms/IPO/IPConstantPropagation.cpp
index 7d55ebecbf9239c4ce7ceccf8980229963889fab..7dc4d9ee9e341a970f9bc551c0f0da6642045385 100644
--- a/lib/Transforms/IPO/IPConstantPropagation.cpp
+++ b/lib/Transforms/IPO/IPConstantPropagation.cpp
@@ -1,9 +1,8 @@
 //===-- IPConstantPropagation.cpp - Propagate constants through calls -----===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -62,32 +61,55 @@ static bool PropagateConstantsIntoArguments(Function &F) {
     // Ignore blockaddress uses.
     if (isa<BlockAddress>(UR)) continue;
 
-    // Used by a non-instruction, or not the callee of a function, do not
-    // transform.
-    if (!isa<CallInst>(UR) && !isa<InvokeInst>(UR))
+    // If no abstract call site was created we did not understand the use, bail.
+    AbstractCallSite ACS(&U);
+    if (!ACS)
       return false;
 
-    CallSite CS(cast<Instruction>(UR));
-    if (!CS.isCallee(&U))
+    // Mismatched argument count is undefined behavior. Simply bail out to avoid
+    // handling of such situations below (avoiding asserts/crashes).
+    unsigned NumActualArgs = ACS.getNumArgOperands();
+    if (F.isVarArg() ? ArgumentConstants.size() > NumActualArgs
+                     : ArgumentConstants.size() != NumActualArgs)
       return false;
 
     // Check out all of the potentially constant arguments.  Note that we don't
     // inspect varargs here.
-    CallSite::arg_iterator AI = CS.arg_begin();
     Function::arg_iterator Arg = F.arg_begin();
-    for (unsigned i = 0, e = ArgumentConstants.size(); i != e;
-         ++i, ++AI, ++Arg) {
+    for (unsigned i = 0, e = ArgumentConstants.size(); i != e; ++i, ++Arg) {
 
       // If this argument is known non-constant, ignore it.
       if (ArgumentConstants[i].second)
         continue;
 
-      Constant *C = dyn_cast<Constant>(*AI);
+      Value *V = ACS.getCallArgOperand(i);
+      Constant *C = dyn_cast_or_null<Constant>(V);
+
+      // Mismatched argument type is undefined behavior. Simply bail out to avoid
+      // handling of such situations below (avoiding asserts/crashes).
+      if (C && Arg->getType() != C->getType())
+        return false;
+
+      // We can only propagate thread independent values through callbacks.
+      // This is different to direct/indirect call sites because for them we
+      // know the thread executing the caller and callee is the same. For
+      // callbacks this is not guaranteed, thus a thread dependent value could
+      // be different for the caller and callee, making it invalid to propagate.
+      if (C && ACS.isCallbackCall() && C->isThreadDependent()) {
+        // Argument became non-constant. If all arguments are non-constant now,
+        // give up on this function.
+        if (++NumNonconstant == ArgumentConstants.size())
+          return false;
+
+        ArgumentConstants[i].second = true;
+        continue;
+      }
+
       if (C && ArgumentConstants[i].first == nullptr) {
         ArgumentConstants[i].first = C;   // First constant seen.
       } else if (C && ArgumentConstants[i].first == C) {
         // Still the constant value we think it is.
-      } else if (*AI == &*Arg) {
+      } else if (V == &*Arg) {
         // Ignore recursive calls passing argument down.
       } else {
         // Argument became non-constant.  If all arguments are non-constant now,
diff --git a/lib/Transforms/IPO/IPO.cpp b/lib/Transforms/IPO/IPO.cpp
index 973382e2b097d2ad52c06c1ee9eedaa85b26af7c..75219e9cac4226eea91404d053cb29e8f2b3f662 100644
--- a/lib/Transforms/IPO/IPO.cpp
+++ b/lib/Transforms/IPO/IPO.cpp
@@ -1,9 +1,8 @@
 //===-- IPO.cpp -----------------------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Transforms/IPO/InferFunctionAttrs.cpp b/lib/Transforms/IPO/InferFunctionAttrs.cpp
index 470f97b8ba610d56a2629e95d79316b5558551f4..cac83f5aecbed1caad8c1d8cd1bef5aa0c76c51c 100644
--- a/lib/Transforms/IPO/InferFunctionAttrs.cpp
+++ b/lib/Transforms/IPO/InferFunctionAttrs.cpp
@@ -1,9 +1,8 @@
 //===- InferFunctionAttrs.cpp - Infer implicit function attributes --------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Transforms/IPO/InlineSimple.cpp b/lib/Transforms/IPO/InlineSimple.cpp
index 82bba1e5c93bdb32fa8b8a12024aa6c48337b055..c9ce14f28922550f89c68753b58f364e571055d5 100644
--- a/lib/Transforms/IPO/InlineSimple.cpp
+++ b/lib/Transforms/IPO/InlineSimple.cpp
@@ -1,9 +1,8 @@
 //===- InlineSimple.cpp - Code to perform simple function inlining --------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Transforms/IPO/Inliner.cpp b/lib/Transforms/IPO/Inliner.cpp
index 44001b5779cc354dbc89bfc4eea60d9bc5c666a5..d50592d829c30e9472097c6793687c2374b6a274 100644
--- a/lib/Transforms/IPO/Inliner.cpp
+++ b/lib/Transforms/IPO/Inliner.cpp
@@ -1,9 +1,8 @@
 //===- Inliner.cpp - Code common to all inliners --------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -311,6 +310,11 @@ shouldBeDeferred(Function *Caller, CallSite CS, InlineCost IC,
   // For now we only handle local or inline functions.
   if (!Caller->hasLocalLinkage() && !Caller->hasLinkOnceODRLinkage())
     return false;
+  // If the cost of inlining CS is non-positive, it is not going to prevent the
+  // caller from being inlined into its callers and hence we don't need to
+  // defer.
+  if (IC.getCost() <= 0)
+    return false;
   // Try to detect the case where the current inlining candidate caller (call
   // it B) is a static or linkonce-ODR function and is an inlining candidate
   // elsewhere, and the current candidate callee (call it C) is large enough
@@ -330,25 +334,31 @@ shouldBeDeferred(Function *Caller, CallSite CS, InlineCost IC,
   TotalSecondaryCost = 0;
   // The candidate cost to be imposed upon the current function.
   int CandidateCost = IC.getCost() - 1;
-  // This bool tracks what happens if we do NOT inline C into B.
-  bool callerWillBeRemoved = Caller->hasLocalLinkage();
+  // If the caller has local linkage and can be inlined to all its callers, we
+  // can apply a huge negative bonus to TotalSecondaryCost.
+  bool ApplyLastCallBonus = Caller->hasLocalLinkage() && !Caller->hasOneUse();
   // This bool tracks what happens if we DO inline C into B.
   bool inliningPreventsSomeOuterInline = false;
   for (User *U : Caller->users()) {
+    // If the caller will not be removed (either because it does not have a
+    // local linkage or because the LastCallToStaticBonus has been already
+    // applied), then we can exit the loop early.
+    if (!ApplyLastCallBonus && TotalSecondaryCost >= IC.getCost())
+      return false;
     CallSite CS2(U);
 
     // If this isn't a call to Caller (it could be some other sort
     // of reference) skip it.  Such references will prevent the caller
     // from being removed.
     if (!CS2 || CS2.getCalledFunction() != Caller) {
-      callerWillBeRemoved = false;
+      ApplyLastCallBonus = false;
       continue;
     }
 
     InlineCost IC2 = GetInlineCost(CS2);
     ++NumCallerCallersAnalyzed;
     if (!IC2) {
-      callerWillBeRemoved = false;
+      ApplyLastCallBonus = false;
       continue;
     }
     if (IC2.isAlways())
@@ -366,7 +376,7 @@ shouldBeDeferred(Function *Caller, CallSite CS, InlineCost IC,
   // one is set very low by getInlineCost, in anticipation that Caller will
   // be removed entirely.  We did not account for this above unless there
   // is only one caller of Caller.
-  if (callerWillBeRemoved && !Caller->hasOneUse())
+  if (ApplyLastCallBonus)
     TotalSecondaryCost -= InlineConstants::LastCallToStaticBonus;
 
   if (inliningPreventsSomeOuterInline && TotalSecondaryCost < IC.getCost())
diff --git a/lib/Transforms/IPO/Internalize.cpp b/lib/Transforms/IPO/Internalize.cpp
index a6542d28dfd8c5e969800d21079c440ad8dfb90e..008817c3c594e4cd3baa51339f7ebb703b45416e 100644
--- a/lib/Transforms/IPO/Internalize.cpp
+++ b/lib/Transforms/IPO/Internalize.cpp
@@ -1,9 +1,8 @@
 //===-- Internalize.cpp - Mark functions internal -------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Transforms/IPO/LLVMBuild.txt b/lib/Transforms/IPO/LLVMBuild.txt
index 54ce23876e66b4dc966333ea4545ec1f68c2a103..14aa4e4ba3725161a179396de9c0ecdd22ece94e 100644
--- a/lib/Transforms/IPO/LLVMBuild.txt
+++ b/lib/Transforms/IPO/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/Transforms/IPO/LLVMBuild.txt -----------------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Transforms/IPO/LoopExtractor.cpp b/lib/Transforms/IPO/LoopExtractor.cpp
index 733235d45a09623424d5f58d3668d6a983f2afff..91c7b5f5f135db6d674933023ad236a31099db15 100644
--- a/lib/Transforms/IPO/LoopExtractor.cpp
+++ b/lib/Transforms/IPO/LoopExtractor.cpp
@@ -1,9 +1,8 @@
 //===- LoopExtractor.cpp - Extract each loop into a new function ----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -15,6 +14,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/ADT/Statistic.h"
+#include "llvm/Analysis/AssumptionCache.h"
 #include "llvm/Analysis/LoopPass.h"
 #include "llvm/IR/Dominators.h"
 #include "llvm/IR/Instructions.h"
@@ -51,6 +51,7 @@ namespace {
       AU.addRequiredID(LoopSimplifyID);
       AU.addRequired<DominatorTreeWrapperPass>();
       AU.addRequired<LoopInfoWrapperPass>();
+      AU.addUsedIfAvailable<AssumptionCacheTracker>();
     }
   };
 }
@@ -139,7 +140,10 @@ bool LoopExtractor::runOnLoop(Loop *L, LPPassManager &LPM) {
   if (ShouldExtractLoop) {
     if (NumLoops == 0) return Changed;
     --NumLoops;
-    CodeExtractor Extractor(DT, *L);
+    AssumptionCache *AC = nullptr;
+    if (auto *ACT = getAnalysisIfAvailable<AssumptionCacheTracker>())
+      AC = ACT->lookupAssumptionCache(*L->getHeader()->getParent());
+    CodeExtractor Extractor(DT, *L, false, nullptr, nullptr, AC);
     if (Extractor.extractCodeRegion() != nullptr) {
       Changed = true;
       // After extraction, the loop is replaced by a function call, so
diff --git a/lib/Transforms/IPO/LowerTypeTests.cpp b/lib/Transforms/IPO/LowerTypeTests.cpp
index e4dcd4d4dd7758cf690d3ca182a214db483f8b6a..2f8a96be875631d92a7d71fb104597d02fb9d5f5 100644
--- a/lib/Transforms/IPO/LowerTypeTests.cpp
+++ b/lib/Transforms/IPO/LowerTypeTests.cpp
@@ -1,9 +1,8 @@
 //===- LowerTypeTests.cpp - type metadata lowering pass -------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -619,7 +618,7 @@ Value *LowerTypeTestsModule::createBitSetTest(IRBuilder<> &B,
     }
 
     Value *ByteAddr = B.CreateGEP(Int8Ty, ByteArray, BitOffset);
-    Value *Byte = B.CreateLoad(ByteAddr);
+    Value *Byte = B.CreateLoad(Int8Ty, ByteAddr);
 
     Value *ByteAndMask =
         B.CreateAnd(Byte, ConstantExpr::getPtrToInt(TIL.BitMask, Int8Ty));
@@ -1702,6 +1701,13 @@ bool LowerTypeTestsModule::lower() {
       !ExportSummary && !ImportSummary)
     return false;
 
+  // If only some of the modules were split, we cannot correctly handle
+  // code that contains type tests.
+  if (TypeTestFunc && !TypeTestFunc->use_empty() &&
+      ((ExportSummary && ExportSummary->partiallySplitLTOUnits()) ||
+       (ImportSummary && ImportSummary->partiallySplitLTOUnits())))
+    report_fatal_error("inconsistent LTO Unit splitting with llvm.type.test");
+
   if (ImportSummary) {
     if (TypeTestFunc) {
       for (auto UI = TypeTestFunc->use_begin(), UE = TypeTestFunc->use_end();
diff --git a/lib/Transforms/IPO/MergeFunctions.cpp b/lib/Transforms/IPO/MergeFunctions.cpp
index 550750d17438cfb408f0c5d1ccb7b123ab9bfb89..1ca4ebecd080b0ed97468af8dd9e332a95e2152c 100644
--- a/lib/Transforms/IPO/MergeFunctions.cpp
+++ b/lib/Transforms/IPO/MergeFunctions.cpp
@@ -1,9 +1,8 @@
 //===- MergeFunctions.cpp - Merge identical functions ---------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -281,8 +280,8 @@ private:
   // Replace G with an alias to F (deleting function G)
   void writeAlias(Function *F, Function *G);
 
-  // Replace G with an alias to F if possible, or a thunk to F if
-  // profitable. Returns false if neither is the case.
+  // Replace G with an alias to F if possible, or a thunk to F if possible.
+  // Returns false if neither is the case.
   bool writeThunkOrAlias(Function *F, Function *G);
 
   /// Replace function F with function G in the function tree.
@@ -383,6 +382,11 @@ bool MergeFunctions::doSanityCheck(std::vector<WeakTrackingVH> &Worklist) {
 }
 #endif
 
+/// Check whether \p F is eligible for function merging.
+static bool isEligibleForMerging(Function &F) {
+  return !F.isDeclaration() && !F.hasAvailableExternallyLinkage();
+}
+
 bool MergeFunctions::runOnModule(Module &M) {
   if (skipModule(M))
     return false;
@@ -394,7 +398,7 @@ bool MergeFunctions::runOnModule(Module &M) {
   std::vector<std::pair<FunctionComparator::FunctionHash, Function *>>
     HashedFuncs;
   for (Function &Func : M) {
-    if (!Func.isDeclaration() && !Func.hasAvailableExternallyLinkage()) {
+    if (isEligibleForMerging(Func)) {
       HashedFuncs.push_back({FunctionComparator::functionHash(Func), &Func});
     }
   }
@@ -474,7 +478,7 @@ void MergeFunctions::replaceDirectCallers(Function *Old, Function *New) {
                                           NewPAL.getRetAttributes(),
                                           NewArgAttrs));
 
-      remove(CS.getInstruction()->getParent()->getParent());
+      remove(CS.getInstruction()->getFunction());
       U->set(BitcastNew);
     }
   }
@@ -654,12 +658,16 @@ void MergeFunctions::filterInstsUnrelatedToPDI(
   LLVM_DEBUG(dbgs() << " }\n");
 }
 
-// Don't merge tiny functions using a thunk, since it can just end up
-// making the function larger.
-static bool isThunkProfitable(Function * F) {
+/// Whether this function may be replaced by a forwarding thunk.
+static bool canCreateThunkFor(Function *F) {
+  if (F->isVarArg())
+    return false;
+
+  // Don't merge tiny functions using a thunk, since it can just end up
+  // making the function larger.
   if (F->size() == 1) {
     if (F->front().size() <= 2) {
-      LLVM_DEBUG(dbgs() << "isThunkProfitable: " << F->getName()
+      LLVM_DEBUG(dbgs() << "canCreateThunkFor: " << F->getName()
                         << " is too small to bother creating a thunk for\n");
       return false;
     }
@@ -787,7 +795,7 @@ bool MergeFunctions::writeThunkOrAlias(Function *F, Function *G) {
     writeAlias(F, G);
     return true;
   }
-  if (isThunkProfitable(F)) {
+  if (canCreateThunkFor(F)) {
     writeThunk(F, G);
     return true;
   }
@@ -802,9 +810,9 @@ void MergeFunctions::mergeTwoFunctions(Function *F, Function *G) {
     // Both writeThunkOrAlias() calls below must succeed, either because we can
     // create aliases for G and NewF, or because a thunk for F is profitable.
     // F here has the same signature as NewF below, so that's what we check.
-    if (!isThunkProfitable(F) && (!canCreateAliasFor(F) || !canCreateAliasFor(G))) {
+    if (!canCreateThunkFor(F) &&
+        (!canCreateAliasFor(F) || !canCreateAliasFor(G)))
       return;
-    }
 
     // Make them both thunks to the same internal function.
     Function *NewF = Function::Create(F->getFunctionType(), F->getLinkage(),
@@ -845,7 +853,7 @@ void MergeFunctions::mergeTwoFunctions(Function *F, Function *G) {
     // If G was internal then we may have replaced all uses of G with F. If so,
     // stop here and delete G. There's no need for a thunk. (See note on
     // MergeFunctionsPDI above).
-    if (G->hasLocalLinkage() && G->use_empty() && !MergeFunctionsPDI) {
+    if (G->isDiscardableIfUnused() && G->use_empty() && !MergeFunctionsPDI) {
       G->eraseFromParent();
       ++NumFunctionsMerged;
       return;
@@ -954,7 +962,7 @@ void MergeFunctions::removeUsers(Value *V) {
 
     for (User *U : V->users()) {
       if (Instruction *I = dyn_cast<Instruction>(U)) {
-        remove(I->getParent()->getParent());
+        remove(I->getFunction());
       } else if (isa<GlobalValue>(U)) {
         // do nothing
       } else if (Constant *C = dyn_cast<Constant>(U)) {
diff --git a/lib/Transforms/IPO/PartialInlining.cpp b/lib/Transforms/IPO/PartialInlining.cpp
index 917582a706b36c5bc294a7e7951dbd490b34e615..2839d144f04ec5ebefde615497c64a46f01f2a7a 100644
--- a/lib/Transforms/IPO/PartialInlining.cpp
+++ b/lib/Transforms/IPO/PartialInlining.cpp
@@ -1,9 +1,8 @@
 //===- PartialInlining.cpp - Inline parts of functions --------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -182,11 +181,11 @@ struct FunctionOutliningMultiRegionInfo {
 
   // Container for outline regions
   struct OutlineRegionInfo {
-    OutlineRegionInfo(SmallVector<BasicBlock *, 8> Region,
+    OutlineRegionInfo(ArrayRef<BasicBlock *> Region,
                       BasicBlock *EntryBlock, BasicBlock *ExitBlock,
                       BasicBlock *ReturnBlock)
-        : Region(Region), EntryBlock(EntryBlock), ExitBlock(ExitBlock),
-          ReturnBlock(ReturnBlock) {}
+        : Region(Region.begin(), Region.end()), EntryBlock(EntryBlock),
+          ExitBlock(ExitBlock), ReturnBlock(ReturnBlock) {}
     SmallVector<BasicBlock *, 8> Region;
     BasicBlock *EntryBlock;
     BasicBlock *ExitBlock;
@@ -200,10 +199,12 @@ struct PartialInlinerImpl {
 
   PartialInlinerImpl(
       std::function<AssumptionCache &(Function &)> *GetAC,
+      function_ref<AssumptionCache *(Function &)> LookupAC,
       std::function<TargetTransformInfo &(Function &)> *GTTI,
       Optional<function_ref<BlockFrequencyInfo &(Function &)>> GBFI,
       ProfileSummaryInfo *ProfSI)
-      : GetAssumptionCache(GetAC), GetTTI(GTTI), GetBFI(GBFI), PSI(ProfSI) {}
+      : GetAssumptionCache(GetAC), LookupAssumptionCache(LookupAC),
+        GetTTI(GTTI), GetBFI(GBFI), PSI(ProfSI) {}
 
   bool run(Module &M);
   // Main part of the transformation that calls helper functions to find
@@ -223,9 +224,11 @@ struct PartialInlinerImpl {
     // Two constructors, one for single region outlining, the other for
     // multi-region outlining.
     FunctionCloner(Function *F, FunctionOutliningInfo *OI,
-                   OptimizationRemarkEmitter &ORE);
+                   OptimizationRemarkEmitter &ORE,
+                   function_ref<AssumptionCache *(Function &)> LookupAC);
     FunctionCloner(Function *F, FunctionOutliningMultiRegionInfo *OMRI,
-                   OptimizationRemarkEmitter &ORE);
+                   OptimizationRemarkEmitter &ORE,
+                   function_ref<AssumptionCache *(Function &)> LookupAC);
     ~FunctionCloner();
 
     // Prepare for function outlining: making sure there is only
@@ -261,11 +264,13 @@ struct PartialInlinerImpl {
     std::unique_ptr<FunctionOutliningMultiRegionInfo> ClonedOMRI = nullptr;
     std::unique_ptr<BlockFrequencyInfo> ClonedFuncBFI = nullptr;
     OptimizationRemarkEmitter &ORE;
+    function_ref<AssumptionCache *(Function &)> LookupAC;
   };
 
 private:
   int NumPartialInlining = 0;
   std::function<AssumptionCache &(Function &)> *GetAssumptionCache;
+  function_ref<AssumptionCache *(Function &)> LookupAssumptionCache;
   std::function<TargetTransformInfo &(Function &)> *GetTTI;
   Optional<function_ref<BlockFrequencyInfo &(Function &)>> GetBFI;
   ProfileSummaryInfo *PSI;
@@ -366,12 +371,17 @@ struct PartialInlinerLegacyPass : public ModulePass {
       return ACT->getAssumptionCache(F);
     };
 
+    auto LookupAssumptionCache = [ACT](Function &F) -> AssumptionCache * {
+      return ACT->lookupAssumptionCache(F);
+    };
+
     std::function<TargetTransformInfo &(Function &)> GetTTI =
         [&TTIWP](Function &F) -> TargetTransformInfo & {
       return TTIWP->getTTI(F);
     };
 
-    return PartialInlinerImpl(&GetAssumptionCache, &GetTTI, NoneType::None, PSI)
+    return PartialInlinerImpl(&GetAssumptionCache, LookupAssumptionCache,
+                              &GetTTI, NoneType::None, PSI)
         .run(M);
   }
 };
@@ -525,7 +535,6 @@ PartialInlinerImpl::computeOutliningColdRegionsInfo(Function *F,
       // assert(ReturnBlock && "ReturnBlock is NULL somehow!");
       FunctionOutliningMultiRegionInfo::OutlineRegionInfo RegInfo(
           DominateVector, DominateVector.front(), ExitBlock, ReturnBlock);
-      RegInfo.Region = DominateVector;
       OutliningInfo->ORI.push_back(RegInfo);
 #ifndef NDEBUG
       if (TracePartialInlining) {
@@ -851,12 +860,8 @@ int PartialInlinerImpl::computeBBInlineCost(BasicBlock *BB) {
       break;
     }
 
-    IntrinsicInst *IntrInst = dyn_cast<IntrinsicInst>(&I);
-    if (IntrInst) {
-      if (IntrInst->getIntrinsicID() == Intrinsic::lifetime_start ||
-          IntrInst->getIntrinsicID() == Intrinsic::lifetime_end)
-        continue;
-    }
+    if (I.isLifetimeStartOrEnd())
+      continue;
 
     if (CallInst *CI = dyn_cast<CallInst>(&I)) {
       InlineCost += getCallsiteCost(CallSite(CI), DL);
@@ -953,8 +958,9 @@ void PartialInlinerImpl::computeCallsiteToProfCountMap(
 }
 
 PartialInlinerImpl::FunctionCloner::FunctionCloner(
-    Function *F, FunctionOutliningInfo *OI, OptimizationRemarkEmitter &ORE)
-    : OrigFunc(F), ORE(ORE) {
+    Function *F, FunctionOutliningInfo *OI, OptimizationRemarkEmitter &ORE,
+    function_ref<AssumptionCache *(Function &)> LookupAC)
+    : OrigFunc(F), ORE(ORE), LookupAC(LookupAC) {
   ClonedOI = llvm::make_unique<FunctionOutliningInfo>();
 
   // Clone the function, so that we can hack away on it.
@@ -977,8 +983,9 @@ PartialInlinerImpl::FunctionCloner::FunctionCloner(
 
 PartialInlinerImpl::FunctionCloner::FunctionCloner(
     Function *F, FunctionOutliningMultiRegionInfo *OI,
-    OptimizationRemarkEmitter &ORE)
-    : OrigFunc(F), ORE(ORE) {
+    OptimizationRemarkEmitter &ORE,
+    function_ref<AssumptionCache *(Function &)> LookupAC)
+    : OrigFunc(F), ORE(ORE), LookupAC(LookupAC) {
   ClonedOMRI = llvm::make_unique<FunctionOutliningMultiRegionInfo>();
 
   // Clone the function, so that we can hack away on it.
@@ -1116,7 +1123,9 @@ bool PartialInlinerImpl::FunctionCloner::doMultiRegionFunctionOutlining() {
     int CurrentOutlinedRegionCost = ComputeRegionCost(RegionInfo.Region);
 
     CodeExtractor CE(RegionInfo.Region, &DT, /*AggregateArgs*/ false,
-                     ClonedFuncBFI.get(), &BPI, /* AllowVarargs */ false);
+                     ClonedFuncBFI.get(), &BPI,
+                     LookupAC(*RegionInfo.EntryBlock->getParent()),
+                     /* AllowVarargs */ false);
 
     CE.findInputsOutputs(Inputs, Outputs, Sinks);
 
@@ -1198,7 +1207,7 @@ PartialInlinerImpl::FunctionCloner::doSingleRegionFunctionOutlining() {
   // Extract the body of the if.
   Function *OutlinedFunc =
       CodeExtractor(ToExtract, &DT, /*AggregateArgs*/ false,
-                    ClonedFuncBFI.get(), &BPI,
+                    ClonedFuncBFI.get(), &BPI, LookupAC(*ClonedFunc),
                     /* AllowVarargs */ true)
           .extractCodeRegion();
 
@@ -1262,7 +1271,7 @@ std::pair<bool, Function *> PartialInlinerImpl::unswitchFunction(Function *F) {
     std::unique_ptr<FunctionOutliningMultiRegionInfo> OMRI =
         computeOutliningColdRegionsInfo(F, ORE);
     if (OMRI) {
-      FunctionCloner Cloner(F, OMRI.get(), ORE);
+      FunctionCloner Cloner(F, OMRI.get(), ORE, LookupAssumptionCache);
 
 #ifndef NDEBUG
       if (TracePartialInlining) {
@@ -1295,7 +1304,7 @@ std::pair<bool, Function *> PartialInlinerImpl::unswitchFunction(Function *F) {
   if (!OI)
     return {false, nullptr};
 
-  FunctionCloner Cloner(F, OI.get(), ORE);
+  FunctionCloner Cloner(F, OI.get(), ORE, LookupAssumptionCache);
   Cloner.NormalizeReturnBlock();
 
   Function *OutlinedFunction = Cloner.doSingleRegionFunctionOutlining();
@@ -1489,6 +1498,10 @@ PreservedAnalyses PartialInlinerPass::run(Module &M,
     return FAM.getResult<AssumptionAnalysis>(F);
   };
 
+  auto LookupAssumptionCache = [&FAM](Function &F) -> AssumptionCache * {
+    return FAM.getCachedResult<AssumptionAnalysis>(F);
+  };
+
   std::function<BlockFrequencyInfo &(Function &)> GetBFI =
       [&FAM](Function &F) -> BlockFrequencyInfo & {
     return FAM.getResult<BlockFrequencyAnalysis>(F);
@@ -1501,7 +1514,8 @@ PreservedAnalyses PartialInlinerPass::run(Module &M,
 
   ProfileSummaryInfo *PSI = &AM.getResult<ProfileSummaryAnalysis>(M);
 
-  if (PartialInlinerImpl(&GetAssumptionCache, &GetTTI, {GetBFI}, PSI)
+  if (PartialInlinerImpl(&GetAssumptionCache, LookupAssumptionCache, &GetTTI,
+                         {GetBFI}, PSI)
           .run(M))
     return PreservedAnalyses::none();
   return PreservedAnalyses::all();
diff --git a/lib/Transforms/IPO/PassManagerBuilder.cpp b/lib/Transforms/IPO/PassManagerBuilder.cpp
index 60bd6def6cefea8923acc78d57988cbe5773b0f8..8f2860ba51b0ed43b527b2714802faafcfa19031 100644
--- a/lib/Transforms/IPO/PassManagerBuilder.cpp
+++ b/lib/Transforms/IPO/PassManagerBuilder.cpp
@@ -1,9 +1,8 @@
 //===- PassManagerBuilder.cpp - Build Standard Pass -----------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -104,23 +103,13 @@ static cl::opt<bool>
     EnablePrepareForThinLTO("prepare-for-thinlto", cl::init(false), cl::Hidden,
                             cl::desc("Enable preparation for ThinLTO."));
 
+static cl::opt<bool>
+    EnablePerformThinLTO("perform-thinlto", cl::init(false), cl::Hidden,
+                         cl::desc("Enable performing ThinLTO."));
+
 cl::opt<bool> EnableHotColdSplit("hot-cold-split", cl::init(false), cl::Hidden,
     cl::desc("Enable hot-cold splitting pass"));
 
-
-static cl::opt<bool> RunPGOInstrGen(
-    "profile-generate", cl::init(false), cl::Hidden,
-    cl::desc("Enable PGO instrumentation."));
-
-static cl::opt<std::string>
-    PGOOutputFile("profile-generate-file", cl::init(""), cl::Hidden,
-                      cl::desc("Specify the path of profile data file."));
-
-static cl::opt<std::string> RunPGOInstrUse(
-    "profile-use", cl::init(""), cl::Hidden, cl::value_desc("filename"),
-    cl::desc("Enable use phase of PGO instrumentation and specify the path "
-             "of profile data file"));
-
 static cl::opt<bool> UseLoopVersioningLICM(
     "enable-loop-versioning-licm", cl::init(false), cl::Hidden,
     cl::desc("Enable the experimental Loop Versioning LICM pass"));
@@ -160,6 +149,11 @@ static cl::opt<bool>
     EnableCHR("enable-chr", cl::init(true), cl::Hidden,
               cl::desc("Enable control height reduction optimization (CHR)"));
 
+cl::opt<bool> FlattenedProfileUsed(
+    "flattened-profile-used", cl::init(false), cl::Hidden,
+    cl::desc("Indicate the sample profile being used is flattened, i.e., "
+             "no inline hierachy exists in the profile. "));
+
 PassManagerBuilder::PassManagerBuilder() {
     OptLevel = 2;
     SizeLevel = 0;
@@ -175,11 +169,12 @@ PassManagerBuilder::PassManagerBuilder() {
     VerifyOutput = false;
     MergeFunctions = false;
     PrepareForLTO = false;
-    EnablePGOInstrGen = RunPGOInstrGen;
-    PGOInstrGen = PGOOutputFile;
-    PGOInstrUse = RunPGOInstrUse;
+    EnablePGOInstrGen = false;
+    PGOInstrGen = "";
+    PGOInstrUse = "";
+    PGOSampleUse = "";
     PrepareForThinLTO = EnablePrepareForThinLTO;
-    PerformThinLTO = false;
+    PerformThinLTO = EnablePerformThinLTO;
     DivergentTarget = false;
 }
 
@@ -425,9 +420,17 @@ void PassManagerBuilder::addFunctionSimplificationPasses(
 
 void PassManagerBuilder::populateModulePassManager(
     legacy::PassManagerBase &MPM) {
+  // Whether this is a default or *LTO pre-link pipeline. The FullLTO post-link
+  // is handled separately, so just check this is not the ThinLTO post-link.
+  bool DefaultOrPreLinkPipeline = !PerformThinLTO;
+
   if (!PGOSampleUse.empty()) {
     MPM.add(createPruneEHPass());
-    MPM.add(createSampleProfileLoaderPass(PGOSampleUse));
+    // In ThinLTO mode, when flattened profile is used, all the available
+    // profile information will be annotated in PreLink phase so there is
+    // no need to load the profile again in PostLink.
+    if (!(FlattenedProfileUsed && PerformThinLTO))
+      MPM.add(createSampleProfileLoaderPass(PGOSampleUse));
   }
 
   // Allow forcing function attributes as a debugging and tuning aid.
@@ -462,12 +465,14 @@ void PassManagerBuilder::populateModulePassManager(
 
     addExtensionsToPM(EP_EnabledOnOptLevel0, MPM);
 
-    // Rename anon globals to be able to export them in the summary.
-    // This has to be done after we add the extensions to the pass manager
-    // as there could be passes (e.g. Adddress sanitizer) which introduce
-    // new unnamed globals.
-    if (PrepareForLTO || PrepareForThinLTO)
+    if (PrepareForLTO || PrepareForThinLTO) {
+      MPM.add(createCanonicalizeAliasesPass());
+      // Rename anon globals to be able to export them in the summary.
+      // This has to be done after we add the extensions to the pass manager
+      // as there could be passes (e.g. Adddress sanitizer) which introduce
+      // new unnamed globals.
       MPM.add(createNameAnonGlobalPass());
+    }
     return;
   }
 
@@ -521,9 +526,14 @@ void PassManagerBuilder::populateModulePassManager(
   // profile annotation in backend more difficult.
   // PGO instrumentation is added during the compile phase for ThinLTO, do
   // not run it a second time
-  if (!PerformThinLTO && !PrepareForThinLTOUsingPGOSampleProfile)
+  if (DefaultOrPreLinkPipeline && !PrepareForThinLTOUsingPGOSampleProfile)
     addPGOInstrPasses(MPM);
 
+  // Split out cold code before inlining. See comment in the new PM
+  // (\ref buildModuleSimplificationPipeline).
+  if (EnableHotColdSplit && DefaultOrPreLinkPipeline)
+    MPM.add(createHotColdSplittingPass());
+
   // We add a module alias analysis pass here. In part due to bugs in the
   // analysis infrastructure this "works" in that the analysis stays alive
   // for the entire SCC pass run below.
@@ -585,6 +595,7 @@ void PassManagerBuilder::populateModulePassManager(
     // Ensure we perform any last passes, but do so before renaming anonymous
     // globals in case the passes add any.
     addExtensionsToPM(EP_OptimizerLast, MPM);
+    MPM.add(createCanonicalizeAliasesPass());
     // Rename anon globals to be able to export them in the summary.
     MPM.add(createNameAnonGlobalPass());
     return;
@@ -735,18 +746,17 @@ void PassManagerBuilder::populateModulePassManager(
   // flattening of blocks.
   MPM.add(createDivRemPairsPass());
 
-  if (EnableHotColdSplit)
-    MPM.add(createHotColdSplittingPass());
-
   // LoopSink (and other loop passes since the last simplifyCFG) might have
   // resulted in single-entry-single-exit or empty blocks. Clean up the CFG.
   MPM.add(createCFGSimplificationPass());
 
   addExtensionsToPM(EP_OptimizerLast, MPM);
 
-  // Rename anon globals to be able to handle them in the summary
-  if (PrepareForLTO)
+  if (PrepareForLTO) {
+    MPM.add(createCanonicalizeAliasesPass());
+    // Rename anon globals to be able to handle them in the summary
     MPM.add(createNameAnonGlobalPass());
+  }
 }
 
 void PassManagerBuilder::addLTOOptimizationPasses(legacy::PassManagerBase &PM) {
diff --git a/lib/Transforms/IPO/PruneEH.cpp b/lib/Transforms/IPO/PruneEH.cpp
index ae586c017471a7885c3aa9d365053097c71c89d6..46d42764fd245dba72181c03f9458cf10996a960 100644
--- a/lib/Transforms/IPO/PruneEH.cpp
+++ b/lib/Transforms/IPO/PruneEH.cpp
@@ -1,9 +1,8 @@
 //===- PruneEH.cpp - Pass which deletes unused exception handlers ---------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Transforms/IPO/SampleProfile.cpp b/lib/Transforms/IPO/SampleProfile.cpp
index 06a1ce89827f65265ecce14a60021fce6504415f..0b880b1e431893938e973f5d64fc1d148f4f062f 100644
--- a/lib/Transforms/IPO/SampleProfile.cpp
+++ b/lib/Transforms/IPO/SampleProfile.cpp
@@ -1,9 +1,8 @@
 //===- SampleProfile.cpp - Incorporate sample profiles into the IR --------===//
 //
-//                      The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -218,6 +217,7 @@ protected:
   const FunctionSamples *findCalleeFunctionSamples(const Instruction &I) const;
   std::vector<const FunctionSamples *>
   findIndirectCallFunctionSamples(const Instruction &I, uint64_t &Sum) const;
+  mutable DenseMap<const DILocation *, const FunctionSamples *> DILocation2SampleMap;
   const FunctionSamples *findFunctionSamples(const Instruction &I) const;
   bool inlineCallInstruction(Instruction *I);
   bool inlineHotFunctions(Function &F,
@@ -318,6 +318,14 @@ protected:
 
   /// Optimization Remark Emitter used to emit diagnostic remarks.
   OptimizationRemarkEmitter *ORE = nullptr;
+
+  // Information recorded when we declined to inline a call site
+  // because we have determined it is too cold is accumulated for
+  // each callee function. Initially this is just the entry count.
+  struct NotInlinedProfileInfo {
+    uint64_t entryCount;
+  };
+  DenseMap<Function *, NotInlinedProfileInfo> notInlinedCallInfo;
 };
 
 class SampleProfileLoaderLegacyPass : public ModulePass {
@@ -544,10 +552,10 @@ ErrorOr<uint64_t> SampleProfileLoader::getInstWeight(const Instruction &Inst) {
   if (!FS)
     return std::error_code();
 
-  // Ignore all intrinsics and branch instructions.
-  // Branch instruction usually contains debug info from sources outside of
+  // Ignore all intrinsics, phinodes and branch instructions.
+  // Branch and phinodes instruction usually contains debug info from sources outside of
   // the residing basic block, thus we ignore them during annotation.
-  if (isa<BranchInst>(Inst) || isa<IntrinsicInst>(Inst))
+  if (isa<BranchInst>(Inst) || isa<IntrinsicInst>(Inst) || isa<PHINode>(Inst))
     return std::error_code();
 
   // If a direct call/invoke instruction is inlined in profile
@@ -719,12 +727,14 @@ SampleProfileLoader::findIndirectCallFunctionSamples(
 /// \returns the FunctionSamples pointer to the inlined instance.
 const FunctionSamples *
 SampleProfileLoader::findFunctionSamples(const Instruction &Inst) const {
-  SmallVector<std::pair<LineLocation, StringRef>, 10> S;
   const DILocation *DIL = Inst.getDebugLoc();
   if (!DIL)
     return Samples;
 
-  return Samples->findFunctionSamples(DIL);
+  auto it = DILocation2SampleMap.try_emplace(DIL,nullptr);
+  if (it.second)
+    it.first->second = Samples->findFunctionSamples(DIL);
+  return it.first->second;
 }
 
 bool SampleProfileLoader::inlineCallInstruction(Instruction *I) {
@@ -776,6 +786,8 @@ bool SampleProfileLoader::inlineCallInstruction(Instruction *I) {
 bool SampleProfileLoader::inlineHotFunctions(
     Function &F, DenseSet<GlobalValue::GUID> &InlinedGUIDs) {
   DenseSet<Instruction *> PromotedInsns;
+
+  DenseMap<Instruction *, const FunctionSamples *> localNotInlinedCallSites;
   bool Changed = false;
   while (true) {
     bool LocalChanged = false;
@@ -788,6 +800,8 @@ bool SampleProfileLoader::inlineHotFunctions(
         if ((isa<CallInst>(I) || isa<InvokeInst>(I)) &&
             !isa<IntrinsicInst>(I) && (FS = findCalleeFunctionSamples(I))) {
           Candidates.push_back(&I);
+          if (FS->getEntrySamples() > 0)
+            localNotInlinedCallSites.try_emplace(&I, FS);
           if (callsiteIsHot(FS, PSI))
             Hot = true;
         }
@@ -833,8 +847,10 @@ bool SampleProfileLoader::inlineHotFunctions(
             PromotedInsns.insert(I);
             // If profile mismatches, we should not attempt to inline DI.
             if ((isa<CallInst>(DI) || isa<InvokeInst>(DI)) &&
-                inlineCallInstruction(DI))
+                inlineCallInstruction(DI)) {
+              localNotInlinedCallSites.erase(I);
               LocalChanged = true;
+            }
           } else {
             LLVM_DEBUG(dbgs()
                        << "\nFailed to promote indirect call to "
@@ -843,8 +859,10 @@ bool SampleProfileLoader::inlineHotFunctions(
         }
       } else if (CalledFunction && CalledFunction->getSubprogram() &&
                  !CalledFunction->isDeclaration()) {
-        if (inlineCallInstruction(I))
+        if (inlineCallInstruction(I)) {
+          localNotInlinedCallSites.erase(I);
           LocalChanged = true;
+        }
       } else if (IsThinLTOPreLink) {
         findCalleeFunctionSamples(*I)->findInlinedFunctions(
             InlinedGUIDs, F.getParent(), PSI->getOrCompHotCountThreshold());
@@ -856,6 +874,18 @@ bool SampleProfileLoader::inlineHotFunctions(
       break;
     }
   }
+
+  // Accumulate not inlined callsite information into notInlinedSamples
+  for (const auto &Pair : localNotInlinedCallSites) {
+    Instruction *I = Pair.getFirst();
+    Function *Callee = CallSite(I).getCalledFunction();
+    if (!Callee || Callee->isDeclaration())
+      continue;
+    const FunctionSamples *FS = Pair.getSecond();
+    auto pair =
+        notInlinedCallInfo.try_emplace(Callee, NotInlinedProfileInfo{0});
+    pair.first->second.entryCount += FS->getEntrySamples();
+  }
   return Changed;
 }
 
@@ -1297,9 +1327,9 @@ void SampleProfileLoader::propagateWeights(Function &F) {
                             SortedCallTargets, Sum, IPVK_IndirectCallTarget,
                             SortedCallTargets.size());
         } else if (!dyn_cast<IntrinsicInst>(&I)) {
-          SmallVector<uint32_t, 1> Weights;
-          Weights.push_back(BlockWeights[BB]);
-          I.setMetadata(LLVMContext::MD_prof, MDB.createBranchWeights(Weights));
+          I.setMetadata(LLVMContext::MD_prof,
+                        MDB.createBranchWeights(
+                            {static_cast<uint32_t>(BlockWeights[BB])}));
         }
       }
     }
@@ -1598,6 +1628,12 @@ bool SampleProfileLoader::runOnModule(Module &M, ModuleAnalysisManager *AM,
       clearFunctionData();
       retval |= runOnFunction(F, AM);
     }
+
+  // Account for cold calls not inlined....
+  for (const std::pair<Function *, NotInlinedProfileInfo> &pair :
+       notInlinedCallInfo)
+    updateProfileCallee(pair.first, pair.second.entryCount);
+
   return retval;
 }
 
@@ -1610,6 +1646,8 @@ bool SampleProfileLoaderLegacyPass::runOnModule(Module &M) {
 }
 
 bool SampleProfileLoader::runOnFunction(Function &F, ModuleAnalysisManager *AM) {
+  
+  DILocation2SampleMap.clear();
   // By default the entry count is initialized to -1, which will be treated
   // conservatively by getEntryCount as the same as unknown (None). This is
   // to avoid newly added code to be treated as cold. If we have samples
diff --git a/lib/Transforms/IPO/StripDeadPrototypes.cpp b/lib/Transforms/IPO/StripDeadPrototypes.cpp
index 3c3c5dd19d1f02fecd9c12b7e0e81ba37ee26d29..106db3c8bd9d912466826accdade2c715b001718 100644
--- a/lib/Transforms/IPO/StripDeadPrototypes.cpp
+++ b/lib/Transforms/IPO/StripDeadPrototypes.cpp
@@ -1,9 +1,8 @@
 //===-- StripDeadPrototypes.cpp - Remove unused function declarations ----===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Transforms/IPO/StripSymbols.cpp b/lib/Transforms/IPO/StripSymbols.cpp
index c9afb060a91a75cd7760f04095c163f0645a04ae..67a473612fc103cfe3d613a17e1f8a64d7f8e2ff 100644
--- a/lib/Transforms/IPO/StripSymbols.cpp
+++ b/lib/Transforms/IPO/StripSymbols.cpp
@@ -1,9 +1,8 @@
 //===- StripSymbols.cpp - Strip symbols and debug info from a module ------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Transforms/IPO/SyntheticCountsPropagation.cpp b/lib/Transforms/IPO/SyntheticCountsPropagation.cpp
index 64837d4f5d6196a60c2a2625f35c15e9c7c10c26..45fd432fd721e723788b67d84c0e3dd2ac5c6512 100644
--- a/lib/Transforms/IPO/SyntheticCountsPropagation.cpp
+++ b/lib/Transforms/IPO/SyntheticCountsPropagation.cpp
@@ -1,9 +1,8 @@
 //=- SyntheticCountsPropagation.cpp - Propagate function counts --*- C++ -*-=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -30,6 +29,7 @@
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/Analysis/BlockFrequencyInfo.h"
 #include "llvm/Analysis/CallGraph.h"
+#include "llvm/Analysis/ProfileSummaryInfo.h"
 #include "llvm/Analysis/SyntheticCountsUtils.h"
 #include "llvm/IR/CallSite.h"
 #include "llvm/IR/Function.h"
@@ -98,13 +98,15 @@ PreservedAnalyses SyntheticCountsPropagation::run(Module &M,
                                                   ModuleAnalysisManager &MAM) {
   FunctionAnalysisManager &FAM =
       MAM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager();
-  DenseMap<Function *, uint64_t> Counts;
+  DenseMap<Function *, Scaled64> Counts;
   // Set initial entry counts.
-  initializeCounts(M, [&](Function *F, uint64_t Count) { Counts[F] = Count; });
+  initializeCounts(
+      M, [&](Function *F, uint64_t Count) { Counts[F] = Scaled64(Count, 0); });
 
-  // Compute the relative block frequency for a call edge. Use scaled numbers
-  // and not integers since the relative block frequency could be less than 1.
-  auto GetCallSiteRelFreq = [&](const CallGraphNode::CallRecord &Edge) {
+  // Edge includes information about the source. Hence ignore the first
+  // parameter.
+  auto GetCallSiteProfCount = [&](const CallGraphNode *,
+                                  const CallGraphNode::CallRecord &Edge) {
     Optional<Scaled64> Res = None;
     if (!Edge.first)
       return Res;
@@ -112,29 +114,33 @@ PreservedAnalyses SyntheticCountsPropagation::run(Module &M,
     CallSite CS(cast<Instruction>(Edge.first));
     Function *Caller = CS.getCaller();
     auto &BFI = FAM.getResult<BlockFrequencyAnalysis>(*Caller);
+
+    // Now compute the callsite count from relative frequency and
+    // entry count:
     BasicBlock *CSBB = CS.getInstruction()->getParent();
     Scaled64 EntryFreq(BFI.getEntryFreq(), 0);
-    Scaled64 BBFreq(BFI.getBlockFreq(CSBB).getFrequency(), 0);
-    BBFreq /= EntryFreq;
-    return Optional<Scaled64>(BBFreq);
+    Scaled64 BBCount(BFI.getBlockFreq(CSBB).getFrequency(), 0);
+    BBCount /= EntryFreq;
+    BBCount *= Counts[Caller];
+    return Optional<Scaled64>(BBCount);
   };
 
   CallGraph CG(M);
   // Propgate the entry counts on the callgraph.
   SyntheticCountsUtils<const CallGraph *>::propagate(
-      &CG, GetCallSiteRelFreq,
-      [&](const CallGraphNode *N) { return Counts[N->getFunction()]; },
-      [&](const CallGraphNode *N, uint64_t New) {
+      &CG, GetCallSiteProfCount, [&](const CallGraphNode *N, Scaled64 New) {
         auto F = N->getFunction();
         if (!F || F->isDeclaration())
           return;
+
         Counts[F] += New;
       });
 
   // Set the counts as metadata.
-  for (auto Entry : Counts)
-    Entry.first->setEntryCount(
-        ProfileCount(Entry.second, Function::PCT_Synthetic));
+  for (auto Entry : Counts) {
+    Entry.first->setEntryCount(ProfileCount(
+        Entry.second.template toInt<uint64_t>(), Function::PCT_Synthetic));
+  }
 
   return PreservedAnalyses::all();
 }
diff --git a/lib/Transforms/IPO/ThinLTOBitcodeWriter.cpp b/lib/Transforms/IPO/ThinLTOBitcodeWriter.cpp
index a5382c4638d8baf7d4c71f8a8b3699d0adbc60b1..7c81bec11905a24d0d820a093561363556980f11 100644
--- a/lib/Transforms/IPO/ThinLTOBitcodeWriter.cpp
+++ b/lib/Transforms/IPO/ThinLTOBitcodeWriter.cpp
@@ -1,9 +1,8 @@
 //===- ThinLTOBitcodeWriter.cpp - Bitcode writing pass for ThinLTO --------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -418,8 +417,18 @@ void splitAndWriteThinLTOBitcode(
   }
 }
 
-// Returns whether this module needs to be split because it uses type metadata.
+// Returns whether this module needs to be split because splitting is
+// enabled and it uses type metadata.
 bool requiresSplit(Module &M) {
+  // First check if the LTO Unit splitting has been enabled.
+  bool EnableSplitLTOUnit = false;
+  if (auto *MD = mdconst::extract_or_null<ConstantInt>(
+          M.getModuleFlag("EnableSplitLTOUnit")))
+    EnableSplitLTOUnit = MD->getZExtValue();
+  if (!EnableSplitLTOUnit)
+    return false;
+
+  // Module only needs to be split if it contains type metadata.
   for (auto &GO : M.global_objects()) {
     if (GO.hasMetadata(LLVMContext::MD_type))
       return true;
@@ -431,7 +440,7 @@ bool requiresSplit(Module &M) {
 void writeThinLTOBitcode(raw_ostream &OS, raw_ostream *ThinLinkOS,
                          function_ref<AAResults &(Function &)> AARGetter,
                          Module &M, const ModuleSummaryIndex *Index) {
-  // See if this module has any type metadata. If so, we need to split it.
+  // Split module if splitting is enabled and it contains any type metadata.
   if (requiresSplit(M))
     return splitAndWriteThinLTOBitcode(OS, ThinLinkOS, AARGetter, M);
 
diff --git a/lib/Transforms/IPO/WholeProgramDevirt.cpp b/lib/Transforms/IPO/WholeProgramDevirt.cpp
index 37905daee4cbd792f83b47ff93f1fb3946b8874a..ab6f0eb660f8bc82373b4e14266cc7c78a7f9c17 100644
--- a/lib/Transforms/IPO/WholeProgramDevirt.cpp
+++ b/lib/Transforms/IPO/WholeProgramDevirt.cpp
@@ -1,9 +1,8 @@
 //===- WholeProgramDevirt.cpp - Whole program virtual call optimization ---===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -882,7 +881,7 @@ void DevirtModule::tryICallBranchFunnel(
   }
 
   BasicBlock *BB = BasicBlock::Create(M.getContext(), "", JT, nullptr);
-  Constant *Intr =
+  Function *Intr =
       Intrinsic::getDeclaration(&M, llvm::Intrinsic::icall_branch_funnel, {});
 
   auto *CI = CallInst::Create(Intr, JTArgs, "", BB);
@@ -921,9 +920,10 @@ void DevirtModule::applyICallBranchFunnel(VTableSlotInfo &SlotInfo,
       NewArgs.push_back(Int8PtrTy);
       for (Type *T : CS.getFunctionType()->params())
         NewArgs.push_back(T);
-      PointerType *NewFT = PointerType::getUnqual(
+      FunctionType *NewFT =
           FunctionType::get(CS.getFunctionType()->getReturnType(), NewArgs,
-                            CS.getFunctionType()->isVarArg()));
+                            CS.getFunctionType()->isVarArg());
+      PointerType *NewFTPtr = PointerType::getUnqual(NewFT);
 
       IRBuilder<> IRB(CS.getInstruction());
       std::vector<Value *> Args;
@@ -933,10 +933,10 @@ void DevirtModule::applyICallBranchFunnel(VTableSlotInfo &SlotInfo,
 
       CallSite NewCS;
       if (CS.isCall())
-        NewCS = IRB.CreateCall(IRB.CreateBitCast(JT, NewFT), Args);
+        NewCS = IRB.CreateCall(NewFT, IRB.CreateBitCast(JT, NewFTPtr), Args);
       else
         NewCS = IRB.CreateInvoke(
-            IRB.CreateBitCast(JT, NewFT),
+            NewFT, IRB.CreateBitCast(JT, NewFTPtr),
             cast<InvokeInst>(CS.getInstruction())->getNormalDest(),
             cast<InvokeInst>(CS.getInstruction())->getUnwindDest(), Args);
       NewCS.setCallingConv(CS.getCallingConv());
@@ -1183,7 +1183,7 @@ void DevirtModule::applyVirtualConstProp(CallSiteInfo &CSInfo, StringRef FnName,
     Value *Addr =
         B.CreateGEP(Int8Ty, B.CreateBitCast(Call.VTable, Int8PtrTy), Byte);
     if (RetType->getBitWidth() == 1) {
-      Value *Bits = B.CreateLoad(Addr);
+      Value *Bits = B.CreateLoad(Int8Ty, Addr);
       Value *BitsAndBit = B.CreateAnd(Bits, Bit);
       auto IsBitSet = B.CreateICmpNE(BitsAndBit, ConstantInt::get(Int8Ty, 0));
       Call.replaceAndErase("virtual-const-prop-1-bit", FnName, RemarksEnabled,
@@ -1495,8 +1495,10 @@ void DevirtModule::importResolution(VTableSlot Slot, VTableSlotInfo &SlotInfo) {
   if (Res.TheKind == WholeProgramDevirtResolution::SingleImpl) {
     // The type of the function in the declaration is irrelevant because every
     // call site will cast it to the correct type.
-    auto *SingleImpl = M.getOrInsertFunction(
-        Res.SingleImplName, Type::getVoidTy(M.getContext()));
+    Constant *SingleImpl =
+        cast<Constant>(M.getOrInsertFunction(Res.SingleImplName,
+                                             Type::getVoidTy(M.getContext()))
+                           .getCallee());
 
     // This is the import phase so we should not be exporting anything.
     bool IsExported = false;
@@ -1538,8 +1540,12 @@ void DevirtModule::importResolution(VTableSlot Slot, VTableSlotInfo &SlotInfo) {
   }
 
   if (Res.TheKind == WholeProgramDevirtResolution::BranchFunnel) {
-    auto *JT = M.getOrInsertFunction(getGlobalName(Slot, {}, "branch_funnel"),
-                                     Type::getVoidTy(M.getContext()));
+    // The type of the function is irrelevant, because it's bitcast at calls
+    // anyhow.
+    Constant *JT = cast<Constant>(
+        M.getOrInsertFunction(getGlobalName(Slot, {}, "branch_funnel"),
+                              Type::getVoidTy(M.getContext()))
+            .getCallee());
     bool IsExported = false;
     applyICallBranchFunnel(SlotInfo, JT, IsExported);
     assert(!IsExported);
@@ -1563,6 +1569,17 @@ bool DevirtModule::run() {
       M.getFunction(Intrinsic::getName(Intrinsic::type_checked_load));
   Function *AssumeFunc = M.getFunction(Intrinsic::getName(Intrinsic::assume));
 
+  // If only some of the modules were split, we cannot correctly handle
+  // code that contains type tests or type checked loads.
+  if ((ExportSummary && ExportSummary->partiallySplitLTOUnits()) ||
+      (ImportSummary && ImportSummary->partiallySplitLTOUnits())) {
+    if ((TypeTestFunc && !TypeTestFunc->use_empty()) ||
+        (TypeCheckedLoadFunc && !TypeCheckedLoadFunc->use_empty()))
+      report_fatal_error("inconsistent LTO Unit splitting with llvm.type.test "
+                         "or llvm.type.checked.load");
+    return false;
+  }
+
   // Normally if there are no users of the devirtualization intrinsics in the
   // module, this pass has nothing to do. But if we are exporting, we also need
   // to handle any users that appear only in the function summaries.
diff --git a/lib/Transforms/InstCombine/CMakeLists.txt b/lib/Transforms/InstCombine/CMakeLists.txt
index 8a3a58e9ecc98be71f7d181a26fb4d4b87a3b44b..bba399bba35ae970924ba0939eb8851a2744ff70 100644
--- a/lib/Transforms/InstCombine/CMakeLists.txt
+++ b/lib/Transforms/InstCombine/CMakeLists.txt
@@ -5,6 +5,7 @@ add_public_tablegen_target(InstCombineTableGen)
 add_llvm_library(LLVMInstCombine
   InstructionCombining.cpp
   InstCombineAddSub.cpp
+  InstCombineAtomicRMW.cpp
   InstCombineAndOrXor.cpp
   InstCombineCalls.cpp
   InstCombineCasts.cpp
diff --git a/lib/Transforms/InstCombine/InstCombineAddSub.cpp b/lib/Transforms/InstCombine/InstCombineAddSub.cpp
index f5db8531d3a44f01c884008e45812662f4c68f35..bf92b5dc5594b1a824c4401bb38dea5e3519e20d 100644
--- a/lib/Transforms/InstCombine/InstCombineAddSub.cpp
+++ b/lib/Transforms/InstCombine/InstCombineAddSub.cpp
@@ -1,9 +1,8 @@
 //===- InstCombineAddSub.cpp ------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -1649,15 +1648,14 @@ Instruction *InstCombiner::visitSub(BinaryOperator &I) {
     // X - A*-B -> X + A*B
     // X - -A*B -> X + A*B
     Value *A, *B;
-    Constant *CI;
     if (match(Op1, m_c_Mul(m_Value(A), m_Neg(m_Value(B)))))
       return BinaryOperator::CreateAdd(Op0, Builder.CreateMul(A, B));
 
-    // X - A*CI -> X + A*-CI
+    // X - A*C -> X + A*-C
     // No need to handle commuted multiply because multiply handling will
     // ensure constant will be move to the right hand side.
-    if (match(Op1, m_Mul(m_Value(A), m_Constant(CI)))) {
-      Value *NewMul = Builder.CreateMul(A, ConstantExpr::getNeg(CI));
+    if (match(Op1, m_Mul(m_Value(A), m_Constant(C))) && !isa<ConstantExpr>(C)) {
+      Value *NewMul = Builder.CreateMul(A, ConstantExpr::getNeg(C));
       return BinaryOperator::CreateAdd(Op0, NewMul);
     }
   }
diff --git a/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp b/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp
index 24a82ba9ae40049de5fe15b691ddaa2e9494ac7c..aaa883a70370e05f1572f2e0c1db71f6d42364d9 100644
--- a/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp
+++ b/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp
@@ -1,9 +1,8 @@
 //===- InstCombineAndOrXor.cpp --------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -1810,6 +1809,61 @@ Instruction *InstCombiner::matchBSwap(BinaryOperator &Or) {
   return LastInst;
 }
 
+/// Transform UB-safe variants of bitwise rotate to the funnel shift intrinsic.
+static Instruction *matchRotate(Instruction &Or) {
+  // TODO: Can we reduce the code duplication between this and the related
+  // rotate matching code under visitSelect and visitTrunc?
+  unsigned Width = Or.getType()->getScalarSizeInBits();
+  if (!isPowerOf2_32(Width))
+    return nullptr;
+
+  // First, find an or'd pair of opposite shifts with the same shifted operand:
+  // or (lshr ShVal, ShAmt0), (shl ShVal, ShAmt1)
+  BinaryOperator *Or0, *Or1;
+  if (!match(Or.getOperand(0), m_BinOp(Or0)) ||
+      !match(Or.getOperand(1), m_BinOp(Or1)))
+    return nullptr;
+
+  Value *ShVal, *ShAmt0, *ShAmt1;
+  if (!match(Or0, m_OneUse(m_LogicalShift(m_Value(ShVal), m_Value(ShAmt0)))) ||
+      !match(Or1, m_OneUse(m_LogicalShift(m_Specific(ShVal), m_Value(ShAmt1)))))
+    return nullptr;
+
+  BinaryOperator::BinaryOps ShiftOpcode0 = Or0->getOpcode();
+  BinaryOperator::BinaryOps ShiftOpcode1 = Or1->getOpcode();
+  if (ShiftOpcode0 == ShiftOpcode1)
+    return nullptr;
+
+  // Match the shift amount operands for a rotate pattern. This always matches
+  // a subtraction on the R operand.
+  auto matchShiftAmount = [](Value *L, Value *R, unsigned Width) -> Value * {
+    // The shift amount may be masked with negation:
+    // (shl ShVal, (X & (Width - 1))) | (lshr ShVal, ((-X) & (Width - 1)))
+    Value *X;
+    unsigned Mask = Width - 1;
+    if (match(L, m_And(m_Value(X), m_SpecificInt(Mask))) &&
+        match(R, m_And(m_Neg(m_Specific(X)), m_SpecificInt(Mask))))
+      return X;
+
+    return nullptr;
+  };
+
+  Value *ShAmt = matchShiftAmount(ShAmt0, ShAmt1, Width);
+  bool SubIsOnLHS = false;
+  if (!ShAmt) {
+    ShAmt = matchShiftAmount(ShAmt1, ShAmt0, Width);
+    SubIsOnLHS = true;
+  }
+  if (!ShAmt)
+    return nullptr;
+
+  bool IsFshl = (!SubIsOnLHS && ShiftOpcode0 == BinaryOperator::Shl) ||
+                (SubIsOnLHS && ShiftOpcode1 == BinaryOperator::Shl);
+  Intrinsic::ID IID = IsFshl ? Intrinsic::fshl : Intrinsic::fshr;
+  Function *F = Intrinsic::getDeclaration(Or.getModule(), IID, Or.getType());
+  return IntrinsicInst::Create(F, { ShVal, ShVal, ShAmt });
+}
+
 /// If all elements of two constant vectors are 0/-1 and inverses, return true.
 static bool areInverseVectorBitmasks(Constant *C1, Constant *C2) {
   unsigned NumElts = C1->getType()->getVectorNumElements();
@@ -2170,6 +2224,9 @@ Instruction *InstCombiner::visitOr(BinaryOperator &I) {
   if (Instruction *BSwap = matchBSwap(I))
     return BSwap;
 
+  if (Instruction *Rotate = matchRotate(I))
+    return Rotate;
+
   Value *X, *Y;
   const APInt *CV;
   if (match(&I, m_c_Or(m_OneUse(m_Xor(m_Value(X), m_APInt(CV))), m_Value(Y))) &&
diff --git a/lib/Transforms/InstCombine/InstCombineAtomicRMW.cpp b/lib/Transforms/InstCombine/InstCombineAtomicRMW.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..86bbfb15986cfb646fd7095b839c138347a5ed89
--- /dev/null
+++ b/lib/Transforms/InstCombine/InstCombineAtomicRMW.cpp
@@ -0,0 +1,48 @@
+//===- InstCombineAtomicRMW.cpp -------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements the visit functions for atomic rmw instructions.
+//
+//===----------------------------------------------------------------------===//
+#include "InstCombineInternal.h"
+#include "llvm/IR/Instructions.h"
+
+using namespace llvm;
+
+Instruction *InstCombiner::visitAtomicRMWInst(AtomicRMWInst &RMWI) {
+  switch (RMWI.getOperation()) {
+  default:
+    break;
+  case AtomicRMWInst::Add:
+  case AtomicRMWInst::Sub:
+  case AtomicRMWInst::Or:
+    // Replace atomicrmw <op> addr, 0 => load atomic addr.
+
+    // Volatile RMWs perform a load and a store, we cannot replace
+    // this by just a load.
+    if (RMWI.isVolatile())
+      break;
+
+    auto *CI = dyn_cast<ConstantInt>(RMWI.getValOperand());
+    if (!CI || !CI->isZero())
+      break;
+    // Check if the required ordering is compatible with an
+    // atomic load.
+    AtomicOrdering Ordering = RMWI.getOrdering();
+    assert(Ordering != AtomicOrdering::NotAtomic &&
+           Ordering != AtomicOrdering::Unordered &&
+           "AtomicRMWs don't make sense with Unordered or NotAtomic");
+    if (Ordering != AtomicOrdering::Acquire &&
+        Ordering != AtomicOrdering::Monotonic)
+      break;
+    LoadInst *Load = new LoadInst(RMWI.getType(), RMWI.getPointerOperand());
+    Load->setAtomic(Ordering, RMWI.getSyncScopeID());
+    return Load;
+  }
+  return nullptr;
+}
diff --git a/lib/Transforms/InstCombine/InstCombineCalls.cpp b/lib/Transforms/InstCombine/InstCombineCalls.cpp
index ae158aeabdff6c9a206f305e0ccad9bd2c7ff473..56d5c899a431666ff5ed6885b6b3218046f8bfb7 100644
--- a/lib/Transforms/InstCombine/InstCombineCalls.cpp
+++ b/lib/Transforms/InstCombine/InstCombineCalls.cpp
@@ -1,13 +1,12 @@
 //===- InstCombineCalls.cpp -----------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
-// This file implements the visitCall and visitInvoke functions.
+// This file implements the visitCall, visitInvoke, and visitCallBr functions.
 //
 //===----------------------------------------------------------------------===//
 
@@ -28,7 +27,6 @@
 #include "llvm/Analysis/ValueTracking.h"
 #include "llvm/IR/Attributes.h"
 #include "llvm/IR/BasicBlock.h"
-#include "llvm/IR/CallSite.h"
 #include "llvm/IR/Constant.h"
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/DataLayout.h"
@@ -136,6 +134,14 @@ Instruction *InstCombiner::SimplifyAnyMemTransfer(AnyMemTransferInst *MI) {
   if (Size > 8 || (Size&(Size-1)))
     return nullptr;  // If not 1/2/4/8 bytes, exit.
 
+  // If it is an atomic and alignment is less than the size then we will
+  // introduce the unaligned memory access which will be later transformed
+  // into libcall in CodeGen. This is not evident performance gain so disable
+  // it now.
+  if (isa<AtomicMemTransferInst>(MI))
+    if (CopyDstAlign < Size || CopySrcAlign < Size)
+      return nullptr;
+
   // Use an integer load+store unless we can find something better.
   unsigned SrcAddrSp =
     cast<PointerType>(MI->getArgOperand(1)->getType())->getAddressSpace();
@@ -165,7 +171,7 @@ Instruction *InstCombiner::SimplifyAnyMemTransfer(AnyMemTransferInst *MI) {
 
   Value *Src = Builder.CreateBitCast(MI->getArgOperand(1), NewSrcPtrTy);
   Value *Dest = Builder.CreateBitCast(MI->getArgOperand(0), NewDstPtrTy);
-  LoadInst *L = Builder.CreateLoad(Src);
+  LoadInst *L = Builder.CreateLoad(IntType, Src);
   // Alignment from the mem intrinsic will be better, so use it.
   L->setAlignment(CopySrcAlign);
   if (CopyMD)
@@ -174,6 +180,9 @@ Instruction *InstCombiner::SimplifyAnyMemTransfer(AnyMemTransferInst *MI) {
     MI->getMetadata(LLVMContext::MD_mem_parallel_loop_access);
   if (LoopMemParallelMD)
     L->setMetadata(LLVMContext::MD_mem_parallel_loop_access, LoopMemParallelMD);
+  MDNode *AccessGroupMD = MI->getMetadata(LLVMContext::MD_access_group);
+  if (AccessGroupMD)
+    L->setMetadata(LLVMContext::MD_access_group, AccessGroupMD);
 
   StoreInst *S = Builder.CreateStore(L, Dest);
   // Alignment from the mem intrinsic will be better, so use it.
@@ -182,6 +191,8 @@ Instruction *InstCombiner::SimplifyAnyMemTransfer(AnyMemTransferInst *MI) {
     S->setMetadata(LLVMContext::MD_tbaa, CopyMD);
   if (LoopMemParallelMD)
     S->setMetadata(LLVMContext::MD_mem_parallel_loop_access, LoopMemParallelMD);
+  if (AccessGroupMD)
+    S->setMetadata(LLVMContext::MD_access_group, AccessGroupMD);
 
   if (auto *MT = dyn_cast<MemTransferInst>(MI)) {
     // non-atomics can be volatile
@@ -215,6 +226,18 @@ Instruction *InstCombiner::SimplifyAnyMemSet(AnyMemSetInst *MI) {
   Alignment = MI->getDestAlignment();
   assert(Len && "0-sized memory setting should be removed already.");
 
+  // Alignment 0 is identity for alignment 1 for memset, but not store.
+  if (Alignment == 0)
+    Alignment = 1;
+
+  // If it is an atomic and alignment is less than the size then we will
+  // introduce the unaligned memory access which will be later transformed
+  // into libcall in CodeGen. This is not evident performance gain so disable
+  // it now.
+  if (isa<AtomicMemSetInst>(MI))
+    if (Alignment < Len)
+      return nullptr;
+
   // memset(s,c,n) -> store s, c (for n=1,2,4,8)
   if (Len <= 8 && isPowerOf2_32((uint32_t)Len)) {
     Type *ITy = IntegerType::get(MI->getContext(), Len*8);  // n=1 -> i8.
@@ -224,9 +247,6 @@ Instruction *InstCombiner::SimplifyAnyMemSet(AnyMemSetInst *MI) {
     Type *NewDstPtrTy = PointerType::get(ITy, DstAddrSp);
     Dest = Builder.CreateBitCast(Dest, NewDstPtrTy);
 
-    // Alignment 0 is identity for alignment 1 for memset, but not store.
-    if (Alignment == 0) Alignment = 1;
-
     // Extract the fill value and store.
     uint64_t Fill = FillC->getZExtValue()*0x0101010101010101ULL;
     StoreInst *S = Builder.CreateStore(ConstantInt::get(ITy, Fill), Dest,
@@ -243,67 +263,6 @@ Instruction *InstCombiner::SimplifyAnyMemSet(AnyMemSetInst *MI) {
   return nullptr;
 }
 
-static Value *simplifyX86AddsSubs(const IntrinsicInst &II,
-                                  InstCombiner::BuilderTy &Builder) {
-  bool IsAddition;
-
-  switch (II.getIntrinsicID()) {
-  default: llvm_unreachable("Unexpected intrinsic!");
-  case Intrinsic::x86_sse2_padds_b:
-  case Intrinsic::x86_sse2_padds_w:
-  case Intrinsic::x86_avx2_padds_b:
-  case Intrinsic::x86_avx2_padds_w:
-  case Intrinsic::x86_avx512_padds_b_512:
-  case Intrinsic::x86_avx512_padds_w_512:
-    IsAddition = true;
-    break;
-  case Intrinsic::x86_sse2_psubs_b:
-  case Intrinsic::x86_sse2_psubs_w:
-  case Intrinsic::x86_avx2_psubs_b:
-  case Intrinsic::x86_avx2_psubs_w:
-  case Intrinsic::x86_avx512_psubs_b_512:
-  case Intrinsic::x86_avx512_psubs_w_512:
-    IsAddition = false;
-    break;
-  }
-
-  auto *Arg0 = dyn_cast<Constant>(II.getOperand(0));
-  auto *Arg1 = dyn_cast<Constant>(II.getOperand(1));
-  auto VT = cast<VectorType>(II.getType());
-  auto SVT = VT->getElementType();
-  unsigned NumElems = VT->getNumElements();
-
-  if (!Arg0 || !Arg1)
-    return nullptr;
-
-  SmallVector<Constant *, 64> Result;
-
-  APInt MaxValue = APInt::getSignedMaxValue(SVT->getIntegerBitWidth());
-  APInt MinValue = APInt::getSignedMinValue(SVT->getIntegerBitWidth());
-  for (unsigned i = 0; i < NumElems; ++i) {
-    auto *Elt0 = Arg0->getAggregateElement(i);
-    auto *Elt1 = Arg1->getAggregateElement(i);
-    if (isa<UndefValue>(Elt0) || isa<UndefValue>(Elt1)) {
-      Result.push_back(UndefValue::get(SVT));
-      continue;
-    }
-
-    if (!isa<ConstantInt>(Elt0) || !isa<ConstantInt>(Elt1))
-      return nullptr;
-
-    const APInt &Val0 = cast<ConstantInt>(Elt0)->getValue();
-    const APInt &Val1 = cast<ConstantInt>(Elt1)->getValue();
-    bool Overflow = false;
-    APInt ResultElem = IsAddition ? Val0.sadd_ov(Val1, Overflow)
-                                  : Val0.ssub_ov(Val1, Overflow);
-    if (Overflow)
-      ResultElem = Val0.isNegative() ? MinValue : MaxValue;
-    Result.push_back(Constant::getIntegerValue(SVT, ResultElem));
-  }
-
-  return ConstantVector::get(Result);
-}
-
 static Value *simplifyX86immShift(const IntrinsicInst &II,
                                   InstCombiner::BuilderTy &Builder) {
   bool LogicalShift = false;
@@ -792,6 +751,33 @@ static Value *simplifyX86movmsk(const IntrinsicInst &II,
   return nullptr;
 }
 
+static Value *simplifyX86addcarry(const IntrinsicInst &II,
+                                  InstCombiner::BuilderTy &Builder) {
+  Value *CarryIn = II.getArgOperand(0);
+  Value *Op1 = II.getArgOperand(1);
+  Value *Op2 = II.getArgOperand(2);
+  Type *RetTy = II.getType();
+  Type *OpTy = Op1->getType();
+  assert(RetTy->getStructElementType(0)->isIntegerTy(8) &&
+         RetTy->getStructElementType(1) == OpTy && OpTy == Op2->getType() &&
+         "Unexpected types for x86 addcarry");
+
+  // If carry-in is zero, this is just an unsigned add with overflow.
+  if (match(CarryIn, m_ZeroInt())) {
+    Value *UAdd = Builder.CreateIntrinsic(Intrinsic::uadd_with_overflow, OpTy,
+                                          { Op1, Op2 });
+    // The types have to be adjusted to match the x86 call types.
+    Value *UAddResult = Builder.CreateExtractValue(UAdd, 0);
+    Value *UAddOV = Builder.CreateZExt(Builder.CreateExtractValue(UAdd, 1),
+                                       Builder.getInt8Ty());
+    Value *Res = UndefValue::get(RetTy);
+    Res = Builder.CreateInsertValue(Res, UAddOV, 0);
+    return Builder.CreateInsertValue(Res, UAddResult, 1);
+  }
+
+  return nullptr;
+}
+
 static Value *simplifyX86insertps(const IntrinsicInst &II,
                                   InstCombiner::BuilderTy &Builder) {
   auto *CInt = dyn_cast<ConstantInt>(II.getArgOperand(2));
@@ -931,7 +917,7 @@ static Value *simplifyX86extrq(IntrinsicInst &II, Value *Op0,
     if (II.getIntrinsicID() == Intrinsic::x86_sse4a_extrq) {
       Value *Args[] = {Op0, CILength, CIIndex};
       Module *M = II.getModule();
-      Value *F = Intrinsic::getDeclaration(M, Intrinsic::x86_sse4a_extrqi);
+      Function *F = Intrinsic::getDeclaration(M, Intrinsic::x86_sse4a_extrqi);
       return Builder.CreateCall(F, Args);
     }
   }
@@ -1032,7 +1018,7 @@ static Value *simplifyX86insertq(IntrinsicInst &II, Value *Op0, Value *Op1,
 
     Value *Args[] = {Op0, Op1, CILength, CIIndex};
     Module *M = II.getModule();
-    Value *F = Intrinsic::getDeclaration(M, Intrinsic::x86_sse4a_insertqi);
+    Function *F = Intrinsic::getDeclaration(M, Intrinsic::x86_sse4a_insertqi);
     return Builder.CreateCall(F, Args);
   }
 
@@ -1173,45 +1159,6 @@ static Value *simplifyX86vpermv(const IntrinsicInst &II,
   return Builder.CreateShuffleVector(V1, V2, ShuffleMask);
 }
 
-/// Decode XOP integer vector comparison intrinsics.
-static Value *simplifyX86vpcom(const IntrinsicInst &II,
-                               InstCombiner::BuilderTy &Builder,
-                               bool IsSigned) {
-  if (auto *CInt = dyn_cast<ConstantInt>(II.getArgOperand(2))) {
-    uint64_t Imm = CInt->getZExtValue() & 0x7;
-    VectorType *VecTy = cast<VectorType>(II.getType());
-    CmpInst::Predicate Pred = ICmpInst::BAD_ICMP_PREDICATE;
-
-    switch (Imm) {
-    case 0x0:
-      Pred = IsSigned ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT;
-      break;
-    case 0x1:
-      Pred = IsSigned ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE;
-      break;
-    case 0x2:
-      Pred = IsSigned ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT;
-      break;
-    case 0x3:
-      Pred = IsSigned ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE;
-      break;
-    case 0x4:
-      Pred = ICmpInst::ICMP_EQ; break;
-    case 0x5:
-      Pred = ICmpInst::ICMP_NE; break;
-    case 0x6:
-      return ConstantInt::getSigned(VecTy, 0); // FALSE
-    case 0x7:
-      return ConstantInt::getSigned(VecTy, -1); // TRUE
-    }
-
-    if (Value *Cmp = Builder.CreateICmp(Pred, II.getArgOperand(0),
-                                        II.getArgOperand(1)))
-      return Builder.CreateSExtOrTrunc(Cmp, VecTy);
-  }
-  return nullptr;
-}
-
 static bool maskIsAllOneOrUndef(Value *Mask) {
   auto *ConstMask = dyn_cast<Constant>(Mask);
   if (!ConstMask)
@@ -1235,7 +1182,8 @@ static Value *simplifyMaskedLoad(const IntrinsicInst &II,
   if (maskIsAllOneOrUndef(II.getArgOperand(2))) {
     Value *LoadPtr = II.getArgOperand(0);
     unsigned Alignment = cast<ConstantInt>(II.getArgOperand(1))->getZExtValue();
-    return Builder.CreateAlignedLoad(LoadPtr, Alignment, "unmaskedload");
+    return Builder.CreateAlignedLoad(II.getType(), LoadPtr, Alignment,
+                                     "unmaskedload");
   }
 
   return nullptr;
@@ -1552,7 +1500,7 @@ static Value *simplifyNeonVld1(const IntrinsicInst &II,
 
   auto *BCastInst = Builder.CreateBitCast(II.getArgOperand(0),
                                           PointerType::get(II.getType(), 0));
-  return Builder.CreateAlignedLoad(BCastInst, Alignment);
+  return Builder.CreateAlignedLoad(II.getType(), BCastInst, Alignment);
 }
 
 // Returns true iff the 2 intrinsics have the same operands, limiting the
@@ -1867,7 +1815,7 @@ static Instruction *canonicalizeConstantArg0ToArg1(CallInst &Call) {
 }
 
 /// CallInst simplification. This mostly only handles folding of intrinsic
-/// instructions. For normal calls, it allows visitCallSite to do the heavy
+/// instructions. For normal calls, it allows visitCallBase to do the heavy
 /// lifting.
 Instruction *InstCombiner::visitCallInst(CallInst &CI) {
   if (Value *V = SimplifyCall(&CI, SQ.getWithInstruction(&CI)))
@@ -1884,10 +1832,10 @@ Instruction *InstCombiner::visitCallInst(CallInst &CI) {
   }
 
   IntrinsicInst *II = dyn_cast<IntrinsicInst>(&CI);
-  if (!II) return visitCallSite(&CI);
+  if (!II) return visitCallBase(CI);
 
-  // Intrinsics cannot occur in an invoke, so handle them here instead of in
-  // visitCallSite.
+  // Intrinsics cannot occur in an invoke or a callbr, so handle them here
+  // instead of in visitCallBase.
   if (auto *MI = dyn_cast<AnyMemIntrinsic>(II)) {
     bool Changed = false;
 
@@ -1947,6 +1895,19 @@ Instruction *InstCombiner::visitCallInst(CallInst &CI) {
     if (Changed) return II;
   }
 
+  // For vector result intrinsics, use the generic demanded vector support to
+  // simplify any operands before moving on to the per-intrinsic rules.    
+  if (II->getType()->isVectorTy()) {
+    auto VWidth = II->getType()->getVectorNumElements();
+    APInt UndefElts(VWidth, 0);
+    APInt AllOnesEltMask(APInt::getAllOnesValue(VWidth));
+    if (Value *V = SimplifyDemandedVectorElts(II, AllOnesEltMask, UndefElts)) {
+      if (V != II)
+        return replaceInstUsesWith(*II, V);
+      return II;
+    }
+  }
+
   if (Instruction *I = SimplifyNVVMIntrinsic(II, *this))
     return I;
 
@@ -1960,9 +1921,8 @@ Instruction *InstCombiner::visitCallInst(CallInst &CI) {
   switch (II->getIntrinsicID()) {
   default: break;
   case Intrinsic::objectsize:
-    if (ConstantInt *N =
-            lowerObjectSizeCall(II, DL, &TLI, /*MustSucceed=*/false))
-      return replaceInstUsesWith(CI, N);
+    if (Value *V = lowerObjectSizeCall(II, DL, &TLI, /*MustSucceed=*/false))
+      return replaceInstUsesWith(CI, V);
     return nullptr;
   case Intrinsic::bswap: {
     Value *IIOperand = II->getArgOperand(0);
@@ -2341,7 +2301,7 @@ Instruction *InstCombiner::visitCallInst(CallInst &CI) {
                                    &DT) >= 16) {
       Value *Ptr = Builder.CreateBitCast(II->getArgOperand(0),
                                          PointerType::getUnqual(II->getType()));
-      return new LoadInst(Ptr);
+      return new LoadInst(II->getType(), Ptr);
     }
     break;
   case Intrinsic::ppc_vsx_lxvw4x:
@@ -2349,7 +2309,7 @@ Instruction *InstCombiner::visitCallInst(CallInst &CI) {
     // Turn PPC VSX loads into normal loads.
     Value *Ptr = Builder.CreateBitCast(II->getArgOperand(0),
                                        PointerType::getUnqual(II->getType()));
-    return new LoadInst(Ptr, Twine(""), false, 1);
+    return new LoadInst(II->getType(), Ptr, Twine(""), false, 1);
   }
   case Intrinsic::ppc_altivec_stvx:
   case Intrinsic::ppc_altivec_stvxl:
@@ -2377,7 +2337,7 @@ Instruction *InstCombiner::visitCallInst(CallInst &CI) {
                                   II->getType()->getVectorNumElements());
       Value *Ptr = Builder.CreateBitCast(II->getArgOperand(0),
                                          PointerType::getUnqual(VTy));
-      Value *Load = Builder.CreateLoad(Ptr);
+      Value *Load = Builder.CreateLoad(VTy, Ptr);
       return new FPExtInst(Load, II->getType());
     }
     break;
@@ -2387,7 +2347,7 @@ Instruction *InstCombiner::visitCallInst(CallInst &CI) {
                                    &DT) >= 32) {
       Value *Ptr = Builder.CreateBitCast(II->getArgOperand(0),
                                          PointerType::getUnqual(II->getType()));
-      return new LoadInst(Ptr);
+      return new LoadInst(II->getType(), Ptr);
     }
     break;
   case Intrinsic::ppc_qpx_qvstfs:
@@ -2745,62 +2705,15 @@ Instruction *InstCombiner::visitCallInst(CallInst &CI) {
         return replaceInstUsesWith(*II, V);
       }
     }
-    LLVM_FALLTHROUGH;
+    break;
 
-  // X86 scalar intrinsics simplified with SimplifyDemandedVectorElts.
-  case Intrinsic::x86_avx512_mask_max_ss_round:
-  case Intrinsic::x86_avx512_mask_min_ss_round:
-  case Intrinsic::x86_avx512_mask_max_sd_round:
-  case Intrinsic::x86_avx512_mask_min_sd_round:
-  case Intrinsic::x86_sse_cmp_ss:
-  case Intrinsic::x86_sse_min_ss:
-  case Intrinsic::x86_sse_max_ss:
-  case Intrinsic::x86_sse2_cmp_sd:
-  case Intrinsic::x86_sse2_min_sd:
-  case Intrinsic::x86_sse2_max_sd:
-  case Intrinsic::x86_xop_vfrcz_ss:
-  case Intrinsic::x86_xop_vfrcz_sd: {
-   unsigned VWidth = II->getType()->getVectorNumElements();
-   APInt UndefElts(VWidth, 0);
-   APInt AllOnesEltMask(APInt::getAllOnesValue(VWidth));
-   if (Value *V = SimplifyDemandedVectorElts(II, AllOnesEltMask, UndefElts)) {
-     if (V != II)
-       return replaceInstUsesWith(*II, V);
-     return II;
-   }
-   break;
-  }
   case Intrinsic::x86_sse41_round_ss:
   case Intrinsic::x86_sse41_round_sd: {
-    unsigned VWidth = II->getType()->getVectorNumElements();
-    APInt UndefElts(VWidth, 0);
-    APInt AllOnesEltMask(APInt::getAllOnesValue(VWidth));
-    if (Value *V = SimplifyDemandedVectorElts(II, AllOnesEltMask, UndefElts)) {
-      if (V != II)
-        return replaceInstUsesWith(*II, V);
-      return II;
-    } else if (Value *V = simplifyX86round(*II, Builder))
+    if (Value *V = simplifyX86round(*II, Builder))
       return replaceInstUsesWith(*II, V);
     break;
   }
 
-  // Constant fold add/sub with saturation intrinsics.
-  case Intrinsic::x86_sse2_padds_b:
-  case Intrinsic::x86_sse2_padds_w:
-  case Intrinsic::x86_sse2_psubs_b:
-  case Intrinsic::x86_sse2_psubs_w:
-  case Intrinsic::x86_avx2_padds_b:
-  case Intrinsic::x86_avx2_padds_w:
-  case Intrinsic::x86_avx2_psubs_b:
-  case Intrinsic::x86_avx2_psubs_w:
-  case Intrinsic::x86_avx512_padds_b_512:
-  case Intrinsic::x86_avx512_padds_w_512:
-  case Intrinsic::x86_avx512_psubs_b_512:
-  case Intrinsic::x86_avx512_psubs_w_512:
-    if (Value *V = simplifyX86AddsSubs(*II, Builder))
-      return replaceInstUsesWith(*II, V);
-    break;
-
   // Constant fold ashr( <A x Bi>, Ci ).
   // Constant fold lshr( <A x Bi>, Ci ).
   // Constant fold shl( <A x Bi>, Ci ).
@@ -3224,19 +3137,9 @@ Instruction *InstCombiner::visitCallInst(CallInst &CI) {
       return nullptr;
     break;
 
-  case Intrinsic::x86_xop_vpcomb:
-  case Intrinsic::x86_xop_vpcomd:
-  case Intrinsic::x86_xop_vpcomq:
-  case Intrinsic::x86_xop_vpcomw:
-    if (Value *V = simplifyX86vpcom(*II, Builder, true))
-      return replaceInstUsesWith(*II, V);
-    break;
-
-  case Intrinsic::x86_xop_vpcomub:
-  case Intrinsic::x86_xop_vpcomud:
-  case Intrinsic::x86_xop_vpcomuq:
-  case Intrinsic::x86_xop_vpcomuw:
-    if (Value *V = simplifyX86vpcom(*II, Builder, false))
+  case Intrinsic::x86_addcarry_32:
+  case Intrinsic::x86_addcarry_64:
+    if (Value *V = simplifyX86addcarry(*II, Builder))
       return replaceInstUsesWith(*II, V);
     break;
 
@@ -3765,7 +3668,7 @@ Instruction *InstCombiner::visitCallInst(CallInst &CI) {
         // register (which contains the bitmask of live threads). So a
         // comparison that always returns true is the same as a read of the
         // EXEC register.
-        Value *NewF = Intrinsic::getDeclaration(
+        Function *NewF = Intrinsic::getDeclaration(
             II->getModule(), Intrinsic::read_register, II->getType());
         Metadata *MDArgs[] = {MDString::get(II->getContext(), "exec")};
         MDNode *MD = MDNode::get(II->getContext(), MDArgs);
@@ -3833,6 +3736,11 @@ Instruction *InstCombiner::visitCallInst(CallInst &CI) {
         // Promote to next legal integer type.
         unsigned Width = CmpType->getBitWidth();
         unsigned NewWidth = Width;
+
+        // Don't do anything for i1 comparisons.
+        if (Width == 1)
+          break;
+
         if (Width <= 16)
           NewWidth = 16;
         else if (Width <= 32)
@@ -3855,8 +3763,8 @@ Instruction *InstCombiner::visitCallInst(CallInst &CI) {
       } else if (!Ty->isFloatTy() && !Ty->isDoubleTy() && !Ty->isHalfTy())
         break;
 
-      Value *NewF = Intrinsic::getDeclaration(II->getModule(), NewIID,
-                                              SrcLHS->getType());
+      Function *NewF =
+          Intrinsic::getDeclaration(II->getModule(), NewIID, SrcLHS->getType());
       Value *Args[] = { SrcLHS, SrcRHS,
                         ConstantInt::get(CC->getType(), SrcPred) };
       CallInst *NewCall = Builder.CreateCall(NewF, Args);
@@ -3971,16 +3879,20 @@ Instruction *InstCombiner::visitCallInst(CallInst &CI) {
     // Canonicalize assume(a && b) -> assume(a); assume(b);
     // Note: New assumption intrinsics created here are registered by
     // the InstCombineIRInserter object.
-    Value *AssumeIntrinsic = II->getCalledValue(), *A, *B;
+    FunctionType *AssumeIntrinsicTy = II->getFunctionType();
+    Value *AssumeIntrinsic = II->getCalledValue();
+    Value *A, *B;
     if (match(IIOperand, m_And(m_Value(A), m_Value(B)))) {
-      Builder.CreateCall(AssumeIntrinsic, A, II->getName());
-      Builder.CreateCall(AssumeIntrinsic, B, II->getName());
+      Builder.CreateCall(AssumeIntrinsicTy, AssumeIntrinsic, A, II->getName());
+      Builder.CreateCall(AssumeIntrinsicTy, AssumeIntrinsic, B, II->getName());
       return eraseInstFromFunction(*II);
     }
     // assume(!(a || b)) -> assume(!a); assume(!b);
     if (match(IIOperand, m_Not(m_Or(m_Value(A), m_Value(B))))) {
-      Builder.CreateCall(AssumeIntrinsic, Builder.CreateNot(A), II->getName());
-      Builder.CreateCall(AssumeIntrinsic, Builder.CreateNot(B), II->getName());
+      Builder.CreateCall(AssumeIntrinsicTy, AssumeIntrinsic,
+                         Builder.CreateNot(A), II->getName());
+      Builder.CreateCall(AssumeIntrinsicTy, AssumeIntrinsic,
+                         Builder.CreateNot(B), II->getName());
       return eraseInstFromFunction(*II);
     }
 
@@ -4087,7 +3999,7 @@ Instruction *InstCombiner::visitCallInst(CallInst &CI) {
     break;
   }
   }
-  return visitCallSite(II);
+  return visitCallBase(*II);
 }
 
 // Fence instruction simplification
@@ -4102,12 +4014,17 @@ Instruction *InstCombiner::visitFenceInst(FenceInst &FI) {
 
 // InvokeInst simplification
 Instruction *InstCombiner::visitInvokeInst(InvokeInst &II) {
-  return visitCallSite(&II);
+  return visitCallBase(II);
+}
+
+// CallBrInst simplification
+Instruction *InstCombiner::visitCallBrInst(CallBrInst &CBI) {
+  return visitCallBase(CBI);
 }
 
 /// If this cast does not affect the value passed through the varargs area, we
 /// can eliminate the use of the cast.
-static bool isSafeToEliminateVarargsCast(const CallSite CS,
+static bool isSafeToEliminateVarargsCast(const CallBase &Call,
                                          const DataLayout &DL,
                                          const CastInst *const CI,
                                          const int ix) {
@@ -4119,13 +4036,13 @@ static bool isSafeToEliminateVarargsCast(const CallSite CS,
   // TODO: This is probably something which should be expanded to all
   // intrinsics since the entire point of intrinsics is that
   // they are understandable by the optimizer.
-  if (isStatepoint(CS) || isGCRelocate(CS) || isGCResult(CS))
+  if (isStatepoint(&Call) || isGCRelocate(&Call) || isGCResult(&Call))
     return false;
 
   // The size of ByVal or InAlloca arguments is derived from the type, so we
   // can't change to a type with a different size.  If the size were
   // passed explicitly we could avoid this check.
-  if (!CS.isByValOrInAllocaArgument(ix))
+  if (!Call.isByValOrInAllocaArgument(ix))
     return true;
 
   Type* SrcTy =
@@ -4233,10 +4150,10 @@ static IntrinsicInst *findInitTrampoline(Value *Callee) {
   return nullptr;
 }
 
-/// Improvements for call and invoke instructions.
-Instruction *InstCombiner::visitCallSite(CallSite CS) {
-  if (isAllocLikeFn(CS.getInstruction(), &TLI))
-    return visitAllocSite(*CS.getInstruction());
+/// Improvements for call, callbr and invoke instructions.
+Instruction *InstCombiner::visitCallBase(CallBase &Call) {
+  if (isAllocLikeFn(&Call, &TLI))
+    return visitAllocSite(Call);
 
   bool Changed = false;
 
@@ -4246,49 +4163,49 @@ Instruction *InstCombiner::visitCallSite(CallSite CS) {
   SmallVector<unsigned, 4> ArgNos;
   unsigned ArgNo = 0;
 
-  for (Value *V : CS.args()) {
+  for (Value *V : Call.args()) {
     if (V->getType()->isPointerTy() &&
-        !CS.paramHasAttr(ArgNo, Attribute::NonNull) &&
-        isKnownNonZero(V, DL, 0, &AC, CS.getInstruction(), &DT))
+        !Call.paramHasAttr(ArgNo, Attribute::NonNull) &&
+        isKnownNonZero(V, DL, 0, &AC, &Call, &DT))
       ArgNos.push_back(ArgNo);
     ArgNo++;
   }
 
-  assert(ArgNo == CS.arg_size() && "sanity check");
+  assert(ArgNo == Call.arg_size() && "sanity check");
 
   if (!ArgNos.empty()) {
-    AttributeList AS = CS.getAttributes();
-    LLVMContext &Ctx = CS.getInstruction()->getContext();
+    AttributeList AS = Call.getAttributes();
+    LLVMContext &Ctx = Call.getContext();
     AS = AS.addParamAttribute(Ctx, ArgNos,
                               Attribute::get(Ctx, Attribute::NonNull));
-    CS.setAttributes(AS);
+    Call.setAttributes(AS);
     Changed = true;
   }
 
   // If the callee is a pointer to a function, attempt to move any casts to the
-  // arguments of the call/invoke.
-  Value *Callee = CS.getCalledValue();
-  if (!isa<Function>(Callee) && transformConstExprCastCall(CS))
+  // arguments of the call/callbr/invoke.
+  Value *Callee = Call.getCalledValue();
+  if (!isa<Function>(Callee) && transformConstExprCastCall(Call))
     return nullptr;
 
   if (Function *CalleeF = dyn_cast<Function>(Callee)) {
     // Remove the convergent attr on calls when the callee is not convergent.
-    if (CS.isConvergent() && !CalleeF->isConvergent() &&
+    if (Call.isConvergent() && !CalleeF->isConvergent() &&
         !CalleeF->isIntrinsic()) {
-      LLVM_DEBUG(dbgs() << "Removing convergent attr from instr "
-                        << CS.getInstruction() << "\n");
-      CS.setNotConvergent();
-      return CS.getInstruction();
+      LLVM_DEBUG(dbgs() << "Removing convergent attr from instr " << Call
+                        << "\n");
+      Call.setNotConvergent();
+      return &Call;
     }
 
     // If the call and callee calling conventions don't match, this call must
     // be unreachable, as the call is undefined.
-    if (CalleeF->getCallingConv() != CS.getCallingConv() &&
+    if (CalleeF->getCallingConv() != Call.getCallingConv() &&
         // Only do this for calls to a function with a body.  A prototype may
         // not actually end up matching the implementation's calling conv for a
         // variety of reasons (e.g. it may be written in assembly).
         !CalleeF->isDeclaration()) {
-      Instruction *OldCall = CS.getInstruction();
+      Instruction *OldCall = &Call;
       new StoreInst(ConstantInt::getTrue(Callee->getContext()),
                 UndefValue::get(Type::getInt1PtrTy(Callee->getContext())),
                                   OldCall);
@@ -4299,25 +4216,25 @@ Instruction *InstCombiner::visitCallSite(CallSite CS) {
       if (isa<CallInst>(OldCall))
         return eraseInstFromFunction(*OldCall);
 
-      // We cannot remove an invoke, because it would change the CFG, just
-      // change the callee to a null pointer.
-      cast<InvokeInst>(OldCall)->setCalledFunction(
-                                    Constant::getNullValue(CalleeF->getType()));
+      // We cannot remove an invoke or a callbr, because it would change thexi
+      // CFG, just change the callee to a null pointer.
+      cast<CallBase>(OldCall)->setCalledFunction(
+          CalleeF->getFunctionType(),
+          Constant::getNullValue(CalleeF->getType()));
       return nullptr;
     }
   }
 
   if ((isa<ConstantPointerNull>(Callee) &&
-       !NullPointerIsDefined(CS.getInstruction()->getFunction())) ||
+       !NullPointerIsDefined(Call.getFunction())) ||
       isa<UndefValue>(Callee)) {
-    // If CS does not return void then replaceAllUsesWith undef.
+    // If Call does not return void then replaceAllUsesWith undef.
     // This allows ValueHandlers and custom metadata to adjust itself.
-    if (!CS.getInstruction()->getType()->isVoidTy())
-      replaceInstUsesWith(*CS.getInstruction(),
-                          UndefValue::get(CS.getInstruction()->getType()));
+    if (!Call.getType()->isVoidTy())
+      replaceInstUsesWith(Call, UndefValue::get(Call.getType()));
 
-    if (isa<InvokeInst>(CS.getInstruction())) {
-      // Can't remove an invoke because we cannot change the CFG.
+    if (Call.isTerminator()) {
+      // Can't remove an invoke or callbr because we cannot change the CFG.
       return nullptr;
     }
 
@@ -4326,13 +4243,13 @@ Instruction *InstCombiner::visitCallSite(CallSite CS) {
     // that we can't modify the CFG here.
     new StoreInst(ConstantInt::getTrue(Callee->getContext()),
                   UndefValue::get(Type::getInt1PtrTy(Callee->getContext())),
-                  CS.getInstruction());
+                  &Call);
 
-    return eraseInstFromFunction(*CS.getInstruction());
+    return eraseInstFromFunction(Call);
   }
 
   if (IntrinsicInst *II = findInitTrampoline(Callee))
-    return transformCallThroughTrampoline(CS, II);
+    return transformCallThroughTrampoline(Call, *II);
 
   PointerType *PTy = cast<PointerType>(Callee->getType());
   FunctionType *FTy = cast<FunctionType>(PTy->getElementType());
@@ -4340,39 +4257,39 @@ Instruction *InstCombiner::visitCallSite(CallSite CS) {
     int ix = FTy->getNumParams();
     // See if we can optimize any arguments passed through the varargs area of
     // the call.
-    for (CallSite::arg_iterator I = CS.arg_begin() + FTy->getNumParams(),
-           E = CS.arg_end(); I != E; ++I, ++ix) {
+    for (auto I = Call.arg_begin() + FTy->getNumParams(), E = Call.arg_end();
+         I != E; ++I, ++ix) {
       CastInst *CI = dyn_cast<CastInst>(*I);
-      if (CI && isSafeToEliminateVarargsCast(CS, DL, CI, ix)) {
+      if (CI && isSafeToEliminateVarargsCast(Call, DL, CI, ix)) {
         *I = CI->getOperand(0);
         Changed = true;
       }
     }
   }
 
-  if (isa<InlineAsm>(Callee) && !CS.doesNotThrow()) {
+  if (isa<InlineAsm>(Callee) && !Call.doesNotThrow()) {
     // Inline asm calls cannot throw - mark them 'nounwind'.
-    CS.setDoesNotThrow();
+    Call.setDoesNotThrow();
     Changed = true;
   }
 
   // Try to optimize the call if possible, we require DataLayout for most of
   // this.  None of these calls are seen as possibly dead so go ahead and
   // delete the instruction now.
-  if (CallInst *CI = dyn_cast<CallInst>(CS.getInstruction())) {
+  if (CallInst *CI = dyn_cast<CallInst>(&Call)) {
     Instruction *I = tryOptimizeCall(CI);
     // If we changed something return the result, etc. Otherwise let
     // the fallthrough check.
     if (I) return eraseInstFromFunction(*I);
   }
 
-  return Changed ? CS.getInstruction() : nullptr;
+  return Changed ? &Call : nullptr;
 }
 
 /// If the callee is a constexpr cast of a function, attempt to move the cast to
-/// the arguments of the call/invoke.
-bool InstCombiner::transformConstExprCastCall(CallSite CS) {
-  auto *Callee = dyn_cast<Function>(CS.getCalledValue()->stripPointerCasts());
+/// the arguments of the call/callbr/invoke.
+bool InstCombiner::transformConstExprCastCall(CallBase &Call) {
+  auto *Callee = dyn_cast<Function>(Call.getCalledValue()->stripPointerCasts());
   if (!Callee)
     return false;
 
@@ -4386,11 +4303,11 @@ bool InstCombiner::transformConstExprCastCall(CallSite CS) {
   // prototype with the exception of pointee types. The code below doesn't
   // implement that, so we can't do this transform.
   // TODO: Do the transform if it only requires adding pointer casts.
-  if (CS.isMustTailCall())
+  if (Call.isMustTailCall())
     return false;
 
-  Instruction *Caller = CS.getInstruction();
-  const AttributeList &CallerPAL = CS.getAttributes();
+  Instruction *Caller = &Call;
+  const AttributeList &CallerPAL = Call.getAttributes();
 
   // Okay, this is a cast from a function to a different type.  Unless doing so
   // would cause a type conversion of one of our arguments, change this call to
@@ -4421,20 +4338,24 @@ bool InstCombiner::transformConstExprCastCall(CallSite CS) {
         return false;   // Attribute not compatible with transformed value.
     }
 
-    // If the callsite is an invoke instruction, and the return value is used by
-    // a PHI node in a successor, we cannot change the return type of the call
-    // because there is no place to put the cast instruction (without breaking
-    // the critical edge).  Bail out in this case.
-    if (!Caller->use_empty())
+    // If the callbase is an invoke/callbr instruction, and the return value is
+    // used by a PHI node in a successor, we cannot change the return type of
+    // the call because there is no place to put the cast instruction (without
+    // breaking the critical edge).  Bail out in this case.
+    if (!Caller->use_empty()) {
       if (InvokeInst *II = dyn_cast<InvokeInst>(Caller))
         for (User *U : II->users())
           if (PHINode *PN = dyn_cast<PHINode>(U))
             if (PN->getParent() == II->getNormalDest() ||
                 PN->getParent() == II->getUnwindDest())
               return false;
+      // FIXME: Be conservative for callbr to avoid a quadratic search.
+      if (isa<CallBrInst>(Caller))
+        return false;
+    }
   }
 
-  unsigned NumActualArgs = CS.arg_size();
+  unsigned NumActualArgs = Call.arg_size();
   unsigned NumCommonArgs = std::min(FT->getNumParams(), NumActualArgs);
 
   // Prevent us turning:
@@ -4449,7 +4370,7 @@ bool InstCombiner::transformConstExprCastCall(CallSite CS) {
       Callee->getAttributes().hasAttrSomewhere(Attribute::ByVal))
     return false;
 
-  CallSite::arg_iterator AI = CS.arg_begin();
+  auto AI = Call.arg_begin();
   for (unsigned i = 0, e = NumCommonArgs; i != e; ++i, ++AI) {
     Type *ParamTy = FT->getParamType(i);
     Type *ActTy = (*AI)->getType();
@@ -4461,7 +4382,7 @@ bool InstCombiner::transformConstExprCastCall(CallSite CS) {
             .overlaps(AttributeFuncs::typeIncompatible(ParamTy)))
       return false;   // Attribute not compatible with transformed value.
 
-    if (CS.isInAllocaArgument(i))
+    if (Call.isInAllocaArgument(i))
       return false;   // Cannot transform to and from inalloca.
 
     // If the parameter is passed as a byval argument, then we have to have a
@@ -4486,7 +4407,7 @@ bool InstCombiner::transformConstExprCastCall(CallSite CS) {
     // If the callee is just a declaration, don't change the varargsness of the
     // call.  We don't want to introduce a varargs call where one doesn't
     // already exist.
-    PointerType *APTy = cast<PointerType>(CS.getCalledValue()->getType());
+    PointerType *APTy = cast<PointerType>(Call.getCalledValue()->getType());
     if (FT->isVarArg()!=cast<FunctionType>(APTy->getElementType())->isVarArg())
       return false;
 
@@ -4525,7 +4446,7 @@ bool InstCombiner::transformConstExprCastCall(CallSite CS) {
   // with the existing attributes.  Wipe out any problematic attributes.
   RAttrs.remove(AttributeFuncs::typeIncompatible(NewRetTy));
 
-  AI = CS.arg_begin();
+  AI = Call.arg_begin();
   for (unsigned i = 0; i != NumCommonArgs; ++i, ++AI) {
     Type *ParamTy = FT->getParamType(i);
 
@@ -4579,40 +4500,46 @@ bool InstCombiner::transformConstExprCastCall(CallSite CS) {
       Ctx, FnAttrs, AttributeSet::get(Ctx, RAttrs), ArgAttrs);
 
   SmallVector<OperandBundleDef, 1> OpBundles;
-  CS.getOperandBundlesAsDefs(OpBundles);
+  Call.getOperandBundlesAsDefs(OpBundles);
 
-  CallSite NewCS;
+  CallBase *NewCall;
   if (InvokeInst *II = dyn_cast<InvokeInst>(Caller)) {
-    NewCS = Builder.CreateInvoke(Callee, II->getNormalDest(),
-                                 II->getUnwindDest(), Args, OpBundles);
+    NewCall = Builder.CreateInvoke(Callee, II->getNormalDest(),
+                                   II->getUnwindDest(), Args, OpBundles);
+  } else if (CallBrInst *CBI = dyn_cast<CallBrInst>(Caller)) {
+    NewCall = Builder.CreateCallBr(Callee, CBI->getDefaultDest(),
+                                   CBI->getIndirectDests(), Args, OpBundles);
   } else {
-    NewCS = Builder.CreateCall(Callee, Args, OpBundles);
-    cast<CallInst>(NewCS.getInstruction())
-        ->setTailCallKind(cast<CallInst>(Caller)->getTailCallKind());
+    NewCall = Builder.CreateCall(Callee, Args, OpBundles);
+    cast<CallInst>(NewCall)->setTailCallKind(
+        cast<CallInst>(Caller)->getTailCallKind());
   }
-  NewCS->takeName(Caller);
-  NewCS.setCallingConv(CS.getCallingConv());
-  NewCS.setAttributes(NewCallerPAL);
+  NewCall->takeName(Caller);
+  NewCall->setCallingConv(Call.getCallingConv());
+  NewCall->setAttributes(NewCallerPAL);
 
   // Preserve the weight metadata for the new call instruction. The metadata
   // is used by SamplePGO to check callsite's hotness.
   uint64_t W;
   if (Caller->extractProfTotalWeight(W))
-    NewCS->setProfWeight(W);
+    NewCall->setProfWeight(W);
 
   // Insert a cast of the return type as necessary.
-  Instruction *NC = NewCS.getInstruction();
+  Instruction *NC = NewCall;
   Value *NV = NC;
   if (OldRetTy != NV->getType() && !Caller->use_empty()) {
     if (!NV->getType()->isVoidTy()) {
       NV = NC = CastInst::CreateBitOrPointerCast(NC, OldRetTy);
       NC->setDebugLoc(Caller->getDebugLoc());
 
-      // If this is an invoke instruction, we should insert it after the first
-      // non-phi, instruction in the normal successor block.
+      // If this is an invoke/callbr instruction, we should insert it after the
+      // first non-phi instruction in the normal successor block.
       if (InvokeInst *II = dyn_cast<InvokeInst>(Caller)) {
         BasicBlock::iterator I = II->getNormalDest()->getFirstInsertionPt();
         InsertNewInstBefore(NC, *I);
+      } else if (CallBrInst *CBI = dyn_cast<CallBrInst>(Caller)) {
+        BasicBlock::iterator I = CBI->getDefaultDest()->getFirstInsertionPt();
+        InsertNewInstBefore(NC, *I);
       } else {
         // Otherwise, it's a call, just insert cast right after the call.
         InsertNewInstBefore(NC, *Caller);
@@ -4641,23 +4568,20 @@ bool InstCombiner::transformConstExprCastCall(CallSite CS) {
 /// Turn a call to a function created by init_trampoline / adjust_trampoline
 /// intrinsic pair into a direct call to the underlying function.
 Instruction *
-InstCombiner::transformCallThroughTrampoline(CallSite CS,
-                                             IntrinsicInst *Tramp) {
-  Value *Callee = CS.getCalledValue();
-  PointerType *PTy = cast<PointerType>(Callee->getType());
-  FunctionType *FTy = cast<FunctionType>(PTy->getElementType());
-  AttributeList Attrs = CS.getAttributes();
+InstCombiner::transformCallThroughTrampoline(CallBase &Call,
+                                             IntrinsicInst &Tramp) {
+  Value *Callee = Call.getCalledValue();
+  Type *CalleeTy = Callee->getType();
+  FunctionType *FTy = Call.getFunctionType();
+  AttributeList Attrs = Call.getAttributes();
 
   // If the call already has the 'nest' attribute somewhere then give up -
   // otherwise 'nest' would occur twice after splicing in the chain.
   if (Attrs.hasAttrSomewhere(Attribute::Nest))
     return nullptr;
 
-  assert(Tramp &&
-         "transformCallThroughTrampoline called with incorrect CallSite.");
-
-  Function *NestF =cast<Function>(Tramp->getArgOperand(1)->stripPointerCasts());
-  FunctionType *NestFTy = cast<FunctionType>(NestF->getValueType());
+  Function *NestF = cast<Function>(Tramp.getArgOperand(1)->stripPointerCasts());
+  FunctionType *NestFTy = NestF->getFunctionType();
 
   AttributeList NestAttrs = NestF->getAttributes();
   if (!NestAttrs.isEmpty()) {
@@ -4679,22 +4603,21 @@ InstCombiner::transformCallThroughTrampoline(CallSite CS,
     }
 
     if (NestTy) {
-      Instruction *Caller = CS.getInstruction();
       std::vector<Value*> NewArgs;
       std::vector<AttributeSet> NewArgAttrs;
-      NewArgs.reserve(CS.arg_size() + 1);
-      NewArgAttrs.reserve(CS.arg_size());
+      NewArgs.reserve(Call.arg_size() + 1);
+      NewArgAttrs.reserve(Call.arg_size());
 
       // Insert the nest argument into the call argument list, which may
       // mean appending it.  Likewise for attributes.
 
       {
         unsigned ArgNo = 0;
-        CallSite::arg_iterator I = CS.arg_begin(), E = CS.arg_end();
+        auto I = Call.arg_begin(), E = Call.arg_end();
         do {
           if (ArgNo == NestArgNo) {
             // Add the chain argument and attributes.
-            Value *NestVal = Tramp->getArgOperand(2);
+            Value *NestVal = Tramp.getArgOperand(2);
             if (NestVal->getType() != NestTy)
               NestVal = Builder.CreateBitCast(NestVal, NestTy, "nest");
             NewArgs.push_back(NestVal);
@@ -4756,24 +4679,30 @@ InstCombiner::transformCallThroughTrampoline(CallSite CS,
                              Attrs.getRetAttributes(), NewArgAttrs);
 
       SmallVector<OperandBundleDef, 1> OpBundles;
-      CS.getOperandBundlesAsDefs(OpBundles);
+      Call.getOperandBundlesAsDefs(OpBundles);
 
       Instruction *NewCaller;
-      if (InvokeInst *II = dyn_cast<InvokeInst>(Caller)) {
-        NewCaller = InvokeInst::Create(NewCallee,
+      if (InvokeInst *II = dyn_cast<InvokeInst>(&Call)) {
+        NewCaller = InvokeInst::Create(NewFTy, NewCallee,
                                        II->getNormalDest(), II->getUnwindDest(),
                                        NewArgs, OpBundles);
         cast<InvokeInst>(NewCaller)->setCallingConv(II->getCallingConv());
         cast<InvokeInst>(NewCaller)->setAttributes(NewPAL);
+      } else if (CallBrInst *CBI = dyn_cast<CallBrInst>(&Call)) {
+        NewCaller =
+            CallBrInst::Create(NewFTy, NewCallee, CBI->getDefaultDest(),
+                               CBI->getIndirectDests(), NewArgs, OpBundles);
+        cast<CallBrInst>(NewCaller)->setCallingConv(CBI->getCallingConv());
+        cast<CallBrInst>(NewCaller)->setAttributes(NewPAL);
       } else {
-        NewCaller = CallInst::Create(NewCallee, NewArgs, OpBundles);
+        NewCaller = CallInst::Create(NewFTy, NewCallee, NewArgs, OpBundles);
         cast<CallInst>(NewCaller)->setTailCallKind(
-            cast<CallInst>(Caller)->getTailCallKind());
+            cast<CallInst>(Call).getTailCallKind());
         cast<CallInst>(NewCaller)->setCallingConv(
-            cast<CallInst>(Caller)->getCallingConv());
+            cast<CallInst>(Call).getCallingConv());
         cast<CallInst>(NewCaller)->setAttributes(NewPAL);
       }
-      NewCaller->setDebugLoc(Caller->getDebugLoc());
+      NewCaller->setDebugLoc(Call.getDebugLoc());
 
       return NewCaller;
     }
@@ -4782,9 +4711,7 @@ InstCombiner::transformCallThroughTrampoline(CallSite CS,
   // Replace the trampoline call with a direct call.  Since there is no 'nest'
   // parameter, there is no need to adjust the argument list.  Let the generic
   // code sort out any function type mismatches.
-  Constant *NewCallee =
-    NestF->getType() == PTy ? NestF :
-                              ConstantExpr::getBitCast(NestF, PTy);
-  CS.setCalledFunction(NewCallee);
-  return CS.getInstruction();
+  Constant *NewCallee = ConstantExpr::getBitCast(NestF, CalleeTy);
+  Call.setCalledFunction(FTy, NewCallee);
+  return &Call;
 }
diff --git a/lib/Transforms/InstCombine/InstCombineCasts.cpp b/lib/Transforms/InstCombine/InstCombineCasts.cpp
index 7a8c762d494fb481862277ba4d68150cf62ef723..91fdda45ac75ecec055ca57ca0753bacb6ea1451 100644
--- a/lib/Transforms/InstCombine/InstCombineCasts.cpp
+++ b/lib/Transforms/InstCombine/InstCombineCasts.cpp
@@ -1,9 +1,8 @@
 //===- InstCombineCasts.cpp -----------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -492,7 +491,7 @@ static Instruction *foldVecTruncToExtElt(TruncInst &Trunc, InstCombiner &IC) {
 }
 
 /// Rotate left/right may occur in a wider type than necessary because of type
-/// promotion rules. Try to narrow all of the component instructions.
+/// promotion rules. Try to narrow the inputs and convert to funnel shift.
 Instruction *InstCombiner::narrowRotate(TruncInst &Trunc) {
   assert((isa<VectorType>(Trunc.getSrcTy()) ||
           shouldChangeType(Trunc.getSrcTy(), Trunc.getType())) &&
@@ -563,23 +562,15 @@ Instruction *InstCombiner::narrowRotate(TruncInst &Trunc) {
 
   // We have an unnecessarily wide rotate!
   // trunc (or (lshr ShVal, ShAmt), (shl ShVal, BitWidth - ShAmt))
-  // Narrow it down to eliminate the zext/trunc:
-  // or (lshr trunc(ShVal), ShAmt0'), (shl trunc(ShVal), ShAmt1')
+  // Narrow the inputs and convert to funnel shift intrinsic:
+  // llvm.fshl.i8(trunc(ShVal), trunc(ShVal), trunc(ShAmt))
   Value *NarrowShAmt = Builder.CreateTrunc(ShAmt, DestTy);
-  Value *NegShAmt = Builder.CreateNeg(NarrowShAmt);
-
-  // Mask both shift amounts to ensure there's no UB from oversized shifts.
-  Constant *MaskC = ConstantInt::get(DestTy, NarrowWidth - 1);
-  Value *MaskedShAmt = Builder.CreateAnd(NarrowShAmt, MaskC);
-  Value *MaskedNegShAmt = Builder.CreateAnd(NegShAmt, MaskC);
-
-  // Truncate the original value and use narrow ops.
   Value *X = Builder.CreateTrunc(ShVal, DestTy);
-  Value *NarrowShAmt0 = SubIsOnLHS ? MaskedNegShAmt : MaskedShAmt;
-  Value *NarrowShAmt1 = SubIsOnLHS ? MaskedShAmt : MaskedNegShAmt;
-  Value *NarrowSh0 = Builder.CreateBinOp(ShiftOpcode0, X, NarrowShAmt0);
-  Value *NarrowSh1 = Builder.CreateBinOp(ShiftOpcode1, X, NarrowShAmt1);
-  return BinaryOperator::CreateOr(NarrowSh0, NarrowSh1);
+  bool IsFshl = (!SubIsOnLHS && ShiftOpcode0 == BinaryOperator::Shl) ||
+                (SubIsOnLHS && ShiftOpcode1 == BinaryOperator::Shl);
+  Intrinsic::ID IID = IsFshl ? Intrinsic::fshl : Intrinsic::fshr;
+  Function *F = Intrinsic::getDeclaration(Trunc.getModule(), IID, DestTy);
+  return IntrinsicInst::Create(F, { X, X, NarrowShAmt });
 }
 
 /// Try to narrow the width of math or bitwise logic instructions by pulling a
@@ -1665,8 +1656,8 @@ Instruction *InstCombiner::visitFPTrunc(FPTruncInst &FPT) {
                                                      II->getIntrinsicID(), Ty);
       SmallVector<OperandBundleDef, 1> OpBundles;
       II->getOperandBundlesAsDefs(OpBundles);
-      CallInst *NewCI = CallInst::Create(Overload, { InnerTrunc }, OpBundles,
-                                         II->getName());
+      CallInst *NewCI =
+          CallInst::Create(Overload, {InnerTrunc}, OpBundles, II->getName());
       NewCI->copyFastMathFlags(II);
       return NewCI;
     }
@@ -2175,7 +2166,7 @@ Instruction *InstCombiner::optimizeBitCastFromPhi(CastInst &CI, PHINode *PN) {
   SmallSetVector<PHINode *, 4> OldPhiNodes;
 
   // Find all of the A->B casts and PHI nodes.
-  // We need to inpect all related PHI nodes, but PHIs can be cyclic, so
+  // We need to inspect all related PHI nodes, but PHIs can be cyclic, so
   // OldPhiNodes is used to track all known PHI nodes, before adding a new
   // PHI to PhiWorklist, it is checked against and added to OldPhiNodes first.
   PhiWorklist.push_back(PN);
@@ -2250,20 +2241,43 @@ Instruction *InstCombiner::optimizeBitCastFromPhi(CastInst &CI, PHINode *PN) {
     }
   }
 
+  // Traverse all accumulated PHI nodes and process its users,
+  // which are Stores and BitcCasts. Without this processing
+  // NewPHI nodes could be replicated and could lead to extra
+  // moves generated after DeSSA.
   // If there is a store with type B, change it to type A.
-  for (User *U : PN->users()) {
-    auto *SI = dyn_cast<StoreInst>(U);
-    if (SI && SI->isSimple() && SI->getOperand(0) == PN) {
-      Builder.SetInsertPoint(SI);
-      auto *NewBC =
-          cast<BitCastInst>(Builder.CreateBitCast(NewPNodes[PN], SrcTy));
-      SI->setOperand(0, NewBC);
-      Worklist.Add(SI);
-      assert(hasStoreUsersOnly(*NewBC));
+
+
+  // Replace users of BitCast B->A with NewPHI. These will help
+  // later to get rid off a closure formed by OldPHI nodes.
+  Instruction *RetVal = nullptr;
+  for (auto *OldPN : OldPhiNodes) {
+    PHINode *NewPN = NewPNodes[OldPN];
+    for (User *V : OldPN->users()) {
+      if (auto *SI = dyn_cast<StoreInst>(V)) {
+        if (SI->isSimple() && SI->getOperand(0) == OldPN) {
+          Builder.SetInsertPoint(SI);
+          auto *NewBC =
+            cast<BitCastInst>(Builder.CreateBitCast(NewPN, SrcTy));
+          SI->setOperand(0, NewBC);
+          Worklist.Add(SI);
+          assert(hasStoreUsersOnly(*NewBC));
+        }
+      }
+      else if (auto *BCI = dyn_cast<BitCastInst>(V)) {
+        // Verify it's a B->A cast.
+        Type *TyB = BCI->getOperand(0)->getType();
+        Type *TyA = BCI->getType();
+        if (TyA == DestTy && TyB == SrcTy) {
+          Instruction *I = replaceInstUsesWith(*BCI, NewPN);
+          if (BCI == &CI)
+            RetVal = I;
+        }
+      }
     }
   }
 
-  return replaceInstUsesWith(CI, NewPNodes[PN]);
+  return RetVal;
 }
 
 Instruction *InstCombiner::visitBitCast(BitCastInst &CI) {
@@ -2318,7 +2332,8 @@ Instruction *InstCombiner::visitBitCast(BitCastInst &CI) {
     // If we found a path from the src to dest, create the getelementptr now.
     if (SrcElTy == DstElTy) {
       SmallVector<Value *, 8> Idxs(NumZeros + 1, Builder.getInt32(0));
-      return GetElementPtrInst::CreateInBounds(Src, Idxs);
+      return GetElementPtrInst::CreateInBounds(SrcPTy->getElementType(), Src,
+                                               Idxs);
     }
   }
 
diff --git a/lib/Transforms/InstCombine/InstCombineCompares.cpp b/lib/Transforms/InstCombine/InstCombineCompares.cpp
index b5bbb09935e2731e378164a89565f1ea8346522e..dc2672537aaa1a5b7832def08fcec4496b109cdc 100644
--- a/lib/Transforms/InstCombine/InstCombineCompares.cpp
+++ b/lib/Transforms/InstCombine/InstCombineCompares.cpp
@@ -1,9 +1,8 @@
 //===- InstCombineCompares.cpp --------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -1292,8 +1291,8 @@ static Instruction *processUGT_ADDCST_ADD(ICmpInst &I, Value *A, Value *B,
   // use the sadd_with_overflow intrinsic to efficiently compute both the
   // result and the overflow bit.
   Type *NewType = IntegerType::get(OrigAdd->getContext(), NewWidth);
-  Value *F = Intrinsic::getDeclaration(I.getModule(),
-                                       Intrinsic::sadd_with_overflow, NewType);
+  Function *F = Intrinsic::getDeclaration(
+      I.getModule(), Intrinsic::sadd_with_overflow, NewType);
 
   InstCombiner::BuilderTy &Builder = IC.Builder;
 
@@ -1772,13 +1771,22 @@ Instruction *InstCombiner::foldICmpOrConstant(ICmpInst &Cmp, BinaryOperator *Or,
                           ConstantInt::get(V->getType(), 1));
   }
 
-  // X | C == C --> X <=u C
-  // X | C != C --> X  >u C
-  //   iff C+1 is a power of 2 (C is a bitmask of the low bits)
-  if (Cmp.isEquality() && Cmp.getOperand(1) == Or->getOperand(1) &&
-      (C + 1).isPowerOf2()) {
-    Pred = (Pred == CmpInst::ICMP_EQ) ? CmpInst::ICMP_ULE : CmpInst::ICMP_UGT;
-    return new ICmpInst(Pred, Or->getOperand(0), Or->getOperand(1));
+  Value *OrOp0 = Or->getOperand(0), *OrOp1 = Or->getOperand(1);
+  if (Cmp.isEquality() && Cmp.getOperand(1) == OrOp1) {
+    // X | C == C --> X <=u C
+    // X | C != C --> X  >u C
+    //   iff C+1 is a power of 2 (C is a bitmask of the low bits)
+    if ((C + 1).isPowerOf2()) {
+      Pred = (Pred == CmpInst::ICMP_EQ) ? CmpInst::ICMP_ULE : CmpInst::ICMP_UGT;
+      return new ICmpInst(Pred, OrOp0, OrOp1);
+    }
+    // More general: are all bits outside of a mask constant set or not set?
+    // X | C == C --> (X & ~C) == 0
+    // X | C != C --> (X & ~C) != 0
+    if (Or->hasOneUse()) {
+      Value *A = Builder.CreateAnd(OrOp0, ~C);
+      return new ICmpInst(Pred, A, ConstantInt::getNullValue(OrOp0->getType()));
+    }
   }
 
   if (!Cmp.isEquality() || !C.isNullValue() || !Or->hasOneUse())
@@ -1799,8 +1807,8 @@ Instruction *InstCombiner::foldICmpOrConstant(ICmpInst &Cmp, BinaryOperator *Or,
   // Are we using xors to bitwise check for a pair of (in)equalities? Convert to
   // a shorter form that has more potential to be folded even further.
   Value *X1, *X2, *X3, *X4;
-  if (match(Or->getOperand(0), m_OneUse(m_Xor(m_Value(X1), m_Value(X2)))) &&
-      match(Or->getOperand(1), m_OneUse(m_Xor(m_Value(X3), m_Value(X4))))) {
+  if (match(OrOp0, m_OneUse(m_Xor(m_Value(X1), m_Value(X2)))) &&
+      match(OrOp1, m_OneUse(m_Xor(m_Value(X3), m_Value(X4))))) {
     // ((X1 ^ X2) || (X3 ^ X4)) == 0 --> (X1 == X2) && (X3 == X4)
     // ((X1 ^ X2) || (X3 ^ X4)) != 0 --> (X1 != X2) || (X3 != X4)
     Value *Cmp12 = Builder.CreateICmp(Pred, X1, X2);
@@ -2482,20 +2490,76 @@ Instruction *InstCombiner::foldICmpSelectConstant(ICmpInst &Cmp,
     // the entire original Cmp can be simplified to a false.
     Value *Cond = Builder.getFalse();
     if (TrueWhenLessThan)
-      Cond = Builder.CreateOr(Cond, Builder.CreateICmp(ICmpInst::ICMP_SLT, OrigLHS, OrigRHS));
+      Cond = Builder.CreateOr(Cond, Builder.CreateICmp(ICmpInst::ICMP_SLT,
+                                                       OrigLHS, OrigRHS));
     if (TrueWhenEqual)
-      Cond = Builder.CreateOr(Cond, Builder.CreateICmp(ICmpInst::ICMP_EQ, OrigLHS, OrigRHS));
+      Cond = Builder.CreateOr(Cond, Builder.CreateICmp(ICmpInst::ICMP_EQ,
+                                                       OrigLHS, OrigRHS));
     if (TrueWhenGreaterThan)
-      Cond = Builder.CreateOr(Cond, Builder.CreateICmp(ICmpInst::ICMP_SGT, OrigLHS, OrigRHS));
+      Cond = Builder.CreateOr(Cond, Builder.CreateICmp(ICmpInst::ICMP_SGT,
+                                                       OrigLHS, OrigRHS));
 
     return replaceInstUsesWith(Cmp, Cond);
   }
   return nullptr;
 }
 
-Instruction *InstCombiner::foldICmpBitCastConstant(ICmpInst &Cmp,
-                                                   BitCastInst *Bitcast,
-                                                   const APInt &C) {
+static Instruction *foldICmpBitCast(ICmpInst &Cmp,
+                                    InstCombiner::BuilderTy &Builder) {
+  auto *Bitcast = dyn_cast<BitCastInst>(Cmp.getOperand(0));
+  if (!Bitcast)
+    return nullptr;
+
+  ICmpInst::Predicate Pred = Cmp.getPredicate();
+  Value *Op1 = Cmp.getOperand(1);
+  Value *BCSrcOp = Bitcast->getOperand(0);
+
+  // Make sure the bitcast doesn't change the number of vector elements.
+  if (Bitcast->getSrcTy()->getScalarSizeInBits() ==
+          Bitcast->getDestTy()->getScalarSizeInBits()) {
+    // Zero-equality and sign-bit checks are preserved through sitofp + bitcast.
+    Value *X;
+    if (match(BCSrcOp, m_SIToFP(m_Value(X)))) {
+      // icmp  eq (bitcast (sitofp X)), 0 --> icmp  eq X, 0
+      // icmp  ne (bitcast (sitofp X)), 0 --> icmp  ne X, 0
+      // icmp slt (bitcast (sitofp X)), 0 --> icmp slt X, 0
+      // icmp sgt (bitcast (sitofp X)), 0 --> icmp sgt X, 0
+      if ((Pred == ICmpInst::ICMP_EQ || Pred == ICmpInst::ICMP_SLT ||
+           Pred == ICmpInst::ICMP_NE || Pred == ICmpInst::ICMP_SGT) &&
+          match(Op1, m_Zero()))
+        return new ICmpInst(Pred, X, ConstantInt::getNullValue(X->getType()));
+
+      // icmp slt (bitcast (sitofp X)), 1 --> icmp slt X, 1
+      if (Pred == ICmpInst::ICMP_SLT && match(Op1, m_One()))
+        return new ICmpInst(Pred, X, ConstantInt::get(X->getType(), 1));
+
+      // icmp sgt (bitcast (sitofp X)), -1 --> icmp sgt X, -1
+      if (Pred == ICmpInst::ICMP_SGT && match(Op1, m_AllOnes()))
+        return new ICmpInst(Pred, X,
+                            ConstantInt::getAllOnesValue(X->getType()));
+    }
+
+    // Zero-equality checks are preserved through unsigned floating-point casts:
+    // icmp eq (bitcast (uitofp X)), 0 --> icmp eq X, 0
+    // icmp ne (bitcast (uitofp X)), 0 --> icmp ne X, 0
+    if (match(BCSrcOp, m_UIToFP(m_Value(X))))
+      if (Cmp.isEquality() && match(Op1, m_Zero()))
+        return new ICmpInst(Pred, X, ConstantInt::getNullValue(X->getType()));
+  }
+
+  // Test to see if the operands of the icmp are casted versions of other
+  // values. If the ptr->ptr cast can be stripped off both arguments, do so.
+  if (Bitcast->getType()->isPointerTy() &&
+      (isa<Constant>(Op1) || isa<BitCastInst>(Op1))) {
+    // If operand #1 is a bitcast instruction, it must also be a ptr->ptr cast
+    // so eliminate it as well.
+    if (auto *BC2 = dyn_cast<BitCastInst>(Op1))
+      Op1 = BC2->getOperand(0);
+
+    Op1 = Builder.CreateBitCast(Op1, BCSrcOp->getType());
+    return new ICmpInst(Pred, BCSrcOp, Op1);
+  }
+
   // Folding: icmp <pred> iN X, C
   //  where X = bitcast <M x iK> (shufflevector <M x iK> %vec, undef, SC)) to iN
   //    and C is a splat of a K-bit pattern
@@ -2503,28 +2567,28 @@ Instruction *InstCombiner::foldICmpBitCastConstant(ICmpInst &Cmp,
   // Into:
   //   %E = extractelement <M x iK> %vec, i32 C'
   //   icmp <pred> iK %E, trunc(C)
-  if (!Bitcast->getType()->isIntegerTy() ||
+  const APInt *C;
+  if (!match(Cmp.getOperand(1), m_APInt(C)) ||
+      !Bitcast->getType()->isIntegerTy() ||
       !Bitcast->getSrcTy()->isIntOrIntVectorTy())
     return nullptr;
 
-  Value *BCIOp = Bitcast->getOperand(0);
-  Value *Vec = nullptr;     // 1st vector arg of the shufflevector
-  Constant *Mask = nullptr; // Mask arg of the shufflevector
-  if (match(BCIOp,
+  Value *Vec;
+  Constant *Mask;
+  if (match(BCSrcOp,
             m_ShuffleVector(m_Value(Vec), m_Undef(), m_Constant(Mask)))) {
     // Check whether every element of Mask is the same constant
     if (auto *Elem = dyn_cast_or_null<ConstantInt>(Mask->getSplatValue())) {
-      auto *VecTy = cast<VectorType>(BCIOp->getType());
+      auto *VecTy = cast<VectorType>(BCSrcOp->getType());
       auto *EltTy = cast<IntegerType>(VecTy->getElementType());
-      auto Pred = Cmp.getPredicate();
-      if (C.isSplat(EltTy->getBitWidth())) {
+      if (C->isSplat(EltTy->getBitWidth())) {
         // Fold the icmp based on the value of C
         // If C is M copies of an iK sized bit pattern,
         // then:
         //   =>  %E = extractelement <N x iK> %vec, i32 Elem
         //       icmp <pred> iK %SplatVal, <pattern>
         Value *Extract = Builder.CreateExtractElement(Vec, Elem);
-        Value *NewC = ConstantInt::get(EltTy, C.trunc(EltTy->getBitWidth()));
+        Value *NewC = ConstantInt::get(EltTy, C->trunc(EltTy->getBitWidth()));
         return new ICmpInst(Pred, Extract, NewC);
       }
     }
@@ -2606,13 +2670,9 @@ Instruction *InstCombiner::foldICmpInstWithConstant(ICmpInst &Cmp) {
       return I;
   }
 
-  if (auto *BCI = dyn_cast<BitCastInst>(Cmp.getOperand(0))) {
-    if (Instruction *I = foldICmpBitCastConstant(Cmp, BCI, *C))
+  if (auto *II = dyn_cast<IntrinsicInst>(Cmp.getOperand(0)))
+    if (Instruction *I = foldICmpIntrinsicWithConstant(Cmp, II, *C))
       return I;
-  }
-
-  if (Instruction *I = foldICmpIntrinsicWithConstant(Cmp, *C))
-    return I;
 
   return nullptr;
 }
@@ -2756,14 +2816,10 @@ Instruction *InstCombiner::foldICmpBinOpEqualityWithConstant(ICmpInst &Cmp,
   return nullptr;
 }
 
-/// Fold an icmp with LLVM intrinsic and constant operand: icmp Pred II, C.
-Instruction *InstCombiner::foldICmpIntrinsicWithConstant(ICmpInst &Cmp,
-                                                         const APInt &C) {
-  IntrinsicInst *II = dyn_cast<IntrinsicInst>(Cmp.getOperand(0));
-  if (!II || !Cmp.isEquality())
-    return nullptr;
-
-  // Handle icmp {eq|ne} <intrinsic>, Constant.
+/// Fold an equality icmp with LLVM intrinsic and constant operand.
+Instruction *InstCombiner::foldICmpEqIntrinsicWithConstant(ICmpInst &Cmp,
+                                                           IntrinsicInst *II,
+                                                           const APInt &C) {
   Type *Ty = II->getType();
   unsigned BitWidth = C.getBitWidth();
   switch (II->getIntrinsicID()) {
@@ -2823,6 +2879,65 @@ Instruction *InstCombiner::foldICmpIntrinsicWithConstant(ICmpInst &Cmp,
   return nullptr;
 }
 
+/// Fold an icmp with LLVM intrinsic and constant operand: icmp Pred II, C.
+Instruction *InstCombiner::foldICmpIntrinsicWithConstant(ICmpInst &Cmp,
+                                                         IntrinsicInst *II,
+                                                         const APInt &C) {
+  if (Cmp.isEquality())
+    return foldICmpEqIntrinsicWithConstant(Cmp, II, C);
+
+  Type *Ty = II->getType();
+  unsigned BitWidth = C.getBitWidth();
+  switch (II->getIntrinsicID()) {
+  case Intrinsic::ctlz: {
+    // ctlz(0bXXXXXXXX) > 3 -> 0bXXXXXXXX < 0b00010000
+    if (Cmp.getPredicate() == ICmpInst::ICMP_UGT && C.ult(BitWidth)) {
+      unsigned Num = C.getLimitedValue();
+      APInt Limit = APInt::getOneBitSet(BitWidth, BitWidth - Num - 1);
+      return CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_ULT,
+                             II->getArgOperand(0), ConstantInt::get(Ty, Limit));
+    }
+
+    // ctlz(0bXXXXXXXX) < 3 -> 0bXXXXXXXX > 0b00011111
+    if (Cmp.getPredicate() == ICmpInst::ICMP_ULT &&
+        C.uge(1) && C.ule(BitWidth)) {
+      unsigned Num = C.getLimitedValue();
+      APInt Limit = APInt::getLowBitsSet(BitWidth, BitWidth - Num);
+      return CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_UGT,
+                             II->getArgOperand(0), ConstantInt::get(Ty, Limit));
+    }
+    break;
+  }
+  case Intrinsic::cttz: {
+    // Limit to one use to ensure we don't increase instruction count.
+    if (!II->hasOneUse())
+      return nullptr;
+
+    // cttz(0bXXXXXXXX) > 3 -> 0bXXXXXXXX & 0b00001111 == 0
+    if (Cmp.getPredicate() == ICmpInst::ICMP_UGT && C.ult(BitWidth)) {
+      APInt Mask = APInt::getLowBitsSet(BitWidth, C.getLimitedValue() + 1);
+      return CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_EQ,
+                             Builder.CreateAnd(II->getArgOperand(0), Mask),
+                             ConstantInt::getNullValue(Ty));
+    }
+
+    // cttz(0bXXXXXXXX) < 3 -> 0bXXXXXXXX & 0b00000111 != 0
+    if (Cmp.getPredicate() == ICmpInst::ICMP_ULT &&
+        C.uge(1) && C.ule(BitWidth)) {
+      APInt Mask = APInt::getLowBitsSet(BitWidth, C.getLimitedValue());
+      return CmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_NE,
+                             Builder.CreateAnd(II->getArgOperand(0), Mask),
+                             ConstantInt::getNullValue(Ty));
+    }
+    break;
+  }
+  default:
+    break;
+  }
+
+  return nullptr;
+}
+
 /// Handle icmp with constant (but not simple integer constant) RHS.
 Instruction *InstCombiner::foldICmpInstWithConstantNotInt(ICmpInst &I) {
   Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1);
@@ -4084,8 +4199,8 @@ static Instruction *processUMulZExtIdiom(ICmpInst &I, Value *MulVal,
     MulA = Builder.CreateZExt(A, MulType);
   if (WidthB < MulWidth)
     MulB = Builder.CreateZExt(B, MulType);
-  Value *F = Intrinsic::getDeclaration(I.getModule(),
-                                       Intrinsic::umul_with_overflow, MulType);
+  Function *F = Intrinsic::getDeclaration(
+      I.getModule(), Intrinsic::umul_with_overflow, MulType);
   CallInst *Call = Builder.CreateCall(F, {MulA, MulB}, "umul");
   IC.Worklist.Add(MulInstr);
 
@@ -4881,61 +4996,8 @@ Instruction *InstCombiner::visitICmpInst(ICmpInst &I) {
         return New;
   }
 
-  // Zero-equality and sign-bit checks are preserved through sitofp + bitcast.
-  Value *X;
-  if (match(Op0, m_BitCast(m_SIToFP(m_Value(X))))) {
-    // icmp  eq (bitcast (sitofp X)), 0 --> icmp  eq X, 0
-    // icmp  ne (bitcast (sitofp X)), 0 --> icmp  ne X, 0
-    // icmp slt (bitcast (sitofp X)), 0 --> icmp slt X, 0
-    // icmp sgt (bitcast (sitofp X)), 0 --> icmp sgt X, 0
-    if ((Pred == ICmpInst::ICMP_EQ || Pred == ICmpInst::ICMP_SLT ||
-         Pred == ICmpInst::ICMP_NE || Pred == ICmpInst::ICMP_SGT) &&
-        match(Op1, m_Zero()))
-      return new ICmpInst(Pred, X, ConstantInt::getNullValue(X->getType()));
-
-    // icmp slt (bitcast (sitofp X)), 1 --> icmp slt X, 1
-    if (Pred == ICmpInst::ICMP_SLT && match(Op1, m_One()))
-      return new ICmpInst(Pred, X, ConstantInt::get(X->getType(), 1));
-
-    // icmp sgt (bitcast (sitofp X)), -1 --> icmp sgt X, -1
-    if (Pred == ICmpInst::ICMP_SGT && match(Op1, m_AllOnes()))
-      return new ICmpInst(Pred, X, ConstantInt::getAllOnesValue(X->getType()));
-  }
-
-  // Zero-equality checks are preserved through unsigned floating-point casts:
-  // icmp eq (bitcast (uitofp X)), 0 --> icmp eq X, 0
-  // icmp ne (bitcast (uitofp X)), 0 --> icmp ne X, 0
-  if (match(Op0, m_BitCast(m_UIToFP(m_Value(X)))))
-    if (I.isEquality() && match(Op1, m_Zero()))
-      return new ICmpInst(Pred, X, ConstantInt::getNullValue(X->getType()));
-
-  // Test to see if the operands of the icmp are casted versions of other
-  // values.  If the ptr->ptr cast can be stripped off both arguments, we do so
-  // now.
-  if (BitCastInst *CI = dyn_cast<BitCastInst>(Op0)) {
-    if (Op0->getType()->isPointerTy() &&
-        (isa<Constant>(Op1) || isa<BitCastInst>(Op1))) {
-      // We keep moving the cast from the left operand over to the right
-      // operand, where it can often be eliminated completely.
-      Op0 = CI->getOperand(0);
-
-      // If operand #1 is a bitcast instruction, it must also be a ptr->ptr cast
-      // so eliminate it as well.
-      if (BitCastInst *CI2 = dyn_cast<BitCastInst>(Op1))
-        Op1 = CI2->getOperand(0);
-
-      // If Op1 is a constant, we can fold the cast into the constant.
-      if (Op0->getType() != Op1->getType()) {
-        if (Constant *Op1C = dyn_cast<Constant>(Op1)) {
-          Op1 = ConstantExpr::getBitCast(Op1C, Op0->getType());
-        } else {
-          // Otherwise, cast the RHS right before the icmp
-          Op1 = Builder.CreateBitCast(Op1, Op0->getType());
-        }
-      }
-      return new ICmpInst(I.getPredicate(), Op0, Op1);
-    }
-  }
+  if (Instruction *Res = foldICmpBitCast(I, Builder))
+    return Res;
 
   if (isa<CastInst>(Op0)) {
     // Handle the special case of: icmp (cast bool to X), <cst>
diff --git a/lib/Transforms/InstCombine/InstCombineInternal.h b/lib/Transforms/InstCombine/InstCombineInternal.h
index e507630f9ea0b2722bb356913c02c7ab868b611d..5b0c7fce0d1f8199711c5269393e99eb4b7b1075 100644
--- a/lib/Transforms/InstCombine/InstCombineInternal.h
+++ b/lib/Transforms/InstCombine/InstCombineInternal.h
@@ -1,9 +1,8 @@
 //===- InstCombineInternal.h - InstCombine pass internals -------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -53,7 +52,6 @@ namespace llvm {
 
 class APInt;
 class AssumptionCache;
-class CallSite;
 class DataLayout;
 class DominatorTree;
 class GEPOperator;
@@ -394,6 +392,7 @@ public:
   Instruction *visitSelectInst(SelectInst &SI);
   Instruction *visitCallInst(CallInst &CI);
   Instruction *visitInvokeInst(InvokeInst &II);
+  Instruction *visitCallBrInst(CallBrInst &CBI);
 
   Instruction *SliceUpIllegalIntegerPHI(PHINode &PN);
   Instruction *visitPHINode(PHINode &PN);
@@ -403,6 +402,7 @@ public:
   Instruction *visitFree(CallInst &FI);
   Instruction *visitLoadInst(LoadInst &LI);
   Instruction *visitStoreInst(StoreInst &SI);
+  Instruction *visitAtomicRMWInst(AtomicRMWInst &SI);
   Instruction *visitBranchInst(BranchInst &BI);
   Instruction *visitFenceInst(FenceInst &FI);
   Instruction *visitSwitchInst(SwitchInst &SI);
@@ -468,11 +468,11 @@ private:
                              Instruction &CtxI, Value *&OperationResult,
                              Constant *&OverflowResult);
 
-  Instruction *visitCallSite(CallSite CS);
+  Instruction *visitCallBase(CallBase &Call);
   Instruction *tryOptimizeCall(CallInst *CI);
-  bool transformConstExprCastCall(CallSite CS);
-  Instruction *transformCallThroughTrampoline(CallSite CS,
-                                              IntrinsicInst *Tramp);
+  bool transformConstExprCastCall(CallBase &Call);
+  Instruction *transformCallThroughTrampoline(CallBase &Call,
+                                              IntrinsicInst &Tramp);
 
   /// Transform (zext icmp) to bitwise / integer operations in order to
   /// eliminate it.
@@ -867,8 +867,6 @@ private:
 
   Instruction *foldICmpSelectConstant(ICmpInst &Cmp, SelectInst *Select,
                                       ConstantInt *C);
-  Instruction *foldICmpBitCastConstant(ICmpInst &Cmp, BitCastInst *Bitcast,
-                                       const APInt &C);
   Instruction *foldICmpTruncConstant(ICmpInst &Cmp, TruncInst *Trunc,
                                      const APInt &C);
   Instruction *foldICmpAndConstant(ICmpInst &Cmp, BinaryOperator *And,
@@ -903,7 +901,10 @@ private:
   Instruction *foldICmpBinOpEqualityWithConstant(ICmpInst &Cmp,
                                                  BinaryOperator *BO,
                                                  const APInt &C);
-  Instruction *foldICmpIntrinsicWithConstant(ICmpInst &ICI, const APInt &C);
+  Instruction *foldICmpIntrinsicWithConstant(ICmpInst &ICI, IntrinsicInst *II,
+                                             const APInt &C);
+  Instruction *foldICmpEqIntrinsicWithConstant(ICmpInst &ICI, IntrinsicInst *II,
+                                               const APInt &C);
 
   // Helpers of visitSelectInst().
   Instruction *foldSelectExtConst(SelectInst &Sel);
diff --git a/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp b/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp
index 02ebb5ef294681473a8a4592ccba602c4b195944..38ce7717e0501f9099660188cbd15413c736b19a 100644
--- a/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp
+++ b/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp
@@ -1,9 +1,8 @@
 //===- InstCombineLoadStoreAlloca.cpp -------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -89,40 +88,37 @@ isOnlyCopiedFromConstantGlobal(Value *V, MemTransferInst *&TheCopy,
         continue;
       }
 
-      if (auto CS = CallSite(I)) {
+      if (auto *Call = dyn_cast<CallBase>(I)) {
         // If this is the function being called then we treat it like a load and
         // ignore it.
-        if (CS.isCallee(&U))
+        if (Call->isCallee(&U))
           continue;
 
-        unsigned DataOpNo = CS.getDataOperandNo(&U);
-        bool IsArgOperand = CS.isArgOperand(&U);
+        unsigned DataOpNo = Call->getDataOperandNo(&U);
+        bool IsArgOperand = Call->isArgOperand(&U);
 
         // Inalloca arguments are clobbered by the call.
-        if (IsArgOperand && CS.isInAllocaArgument(DataOpNo))
+        if (IsArgOperand && Call->isInAllocaArgument(DataOpNo))
           return false;
 
         // If this is a readonly/readnone call site, then we know it is just a
         // load (but one that potentially returns the value itself), so we can
         // ignore it if we know that the value isn't captured.
-        if (CS.onlyReadsMemory() &&
-            (CS.getInstruction()->use_empty() || CS.doesNotCapture(DataOpNo)))
+        if (Call->onlyReadsMemory() &&
+            (Call->use_empty() || Call->doesNotCapture(DataOpNo)))
           continue;
 
         // If this is being passed as a byval argument, the caller is making a
         // copy, so it is only a read of the alloca.
-        if (IsArgOperand && CS.isByValArgument(DataOpNo))
+        if (IsArgOperand && Call->isByValArgument(DataOpNo))
           continue;
       }
 
       // Lifetime intrinsics can be handled by the caller.
-      if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(I)) {
-        if (II->getIntrinsicID() == Intrinsic::lifetime_start ||
-            II->getIntrinsicID() == Intrinsic::lifetime_end) {
-          assert(II->use_empty() && "Lifetime markers have no result to use!");
-          ToDelete.push_back(II);
-          continue;
-        }
+      if (I->isLifetimeStartOrEnd()) {
+        assert(I->use_empty() && "Lifetime markers have no result to use!");
+        ToDelete.push_back(I);
+        continue;
       }
 
       // If this is isn't our memcpy/memmove, reject it as something we can't
@@ -216,8 +212,8 @@ static Instruction *simplifyAllocaArraySize(InstCombiner &IC, AllocaInst &AI) {
       Type *IdxTy = IC.getDataLayout().getIntPtrType(AI.getType());
       Value *NullIdx = Constant::getNullValue(IdxTy);
       Value *Idx[2] = {NullIdx, NullIdx};
-      Instruction *GEP =
-          GetElementPtrInst::CreateInBounds(New, Idx, New->getName() + ".sub");
+      Instruction *GEP = GetElementPtrInst::CreateInBounds(
+          NewTy, New, Idx, New->getName() + ".sub");
       IC.InsertNewInstBefore(GEP, *It);
 
       // Now make everything use the getelementptr instead of the original
@@ -302,7 +298,7 @@ void PointerReplacer::replace(Instruction *I) {
   if (auto *LT = dyn_cast<LoadInst>(I)) {
     auto *V = getReplacement(LT->getPointerOperand());
     assert(V && "Operand not replaced");
-    auto *NewI = new LoadInst(V);
+    auto *NewI = new LoadInst(I->getType(), V);
     NewI->takeName(LT);
     IC.InsertNewInstWith(NewI, *LT);
     IC.replaceInstUsesWith(*LT, NewI);
@@ -469,7 +465,7 @@ static LoadInst *combineLoadToNewType(InstCombiner &IC, LoadInst &LI, Type *NewT
     NewPtr = IC.Builder.CreateBitCast(Ptr, NewTy->getPointerTo(AS));
 
   LoadInst *NewLoad = IC.Builder.CreateAlignedLoad(
-      NewPtr, LI.getAlignment(), LI.isVolatile(), LI.getName() + Suffix);
+      NewTy, NewPtr, LI.getAlignment(), LI.isVolatile(), LI.getName() + Suffix);
   NewLoad->setAtomic(LI.getOrdering(), LI.getSyncScopeID());
   MDBuilder MDB(NewLoad->getContext());
   for (const auto &MDPair : MD) {
@@ -493,6 +489,7 @@ static LoadInst *combineLoadToNewType(InstCombiner &IC, LoadInst &LI, Type *NewT
     case LLVMContext::MD_noalias:
     case LLVMContext::MD_nontemporal:
     case LLVMContext::MD_mem_parallel_loop_access:
+    case LLVMContext::MD_access_group:
       // All of these directly apply.
       NewLoad->setMetadata(ID, N);
       break;
@@ -552,10 +549,10 @@ static StoreInst *combineStoreToNewValue(InstCombiner &IC, StoreInst &SI, Value
     case LLVMContext::MD_noalias:
     case LLVMContext::MD_nontemporal:
     case LLVMContext::MD_mem_parallel_loop_access:
+    case LLVMContext::MD_access_group:
       // All of these directly apply.
       NewStore->setMetadata(ID, N);
       break;
-
     case LLVMContext::MD_invariant_load:
     case LLVMContext::MD_nonnull:
     case LLVMContext::MD_range:
@@ -727,7 +724,8 @@ static Instruction *unpackLoadToAggregate(InstCombiner &IC, LoadInst &LI) {
       auto *Ptr = IC.Builder.CreateInBoundsGEP(ST, Addr, makeArrayRef(Indices),
                                                Name + ".elt");
       auto EltAlign = MinAlign(Align, SL->getElementOffset(i));
-      auto *L = IC.Builder.CreateAlignedLoad(Ptr, EltAlign, Name + ".unpack");
+      auto *L = IC.Builder.CreateAlignedLoad(ST->getElementType(i), Ptr,
+                                             EltAlign, Name + ".unpack");
       // Propagate AA metadata. It'll still be valid on the narrowed load.
       AAMDNodes AAMD;
       LI.getAAMetadata(AAMD);
@@ -777,8 +775,8 @@ static Instruction *unpackLoadToAggregate(InstCombiner &IC, LoadInst &LI) {
       };
       auto *Ptr = IC.Builder.CreateInBoundsGEP(AT, Addr, makeArrayRef(Indices),
                                                Name + ".elt");
-      auto *L = IC.Builder.CreateAlignedLoad(Ptr, MinAlign(Align, Offset),
-                                             Name + ".unpack");
+      auto *L = IC.Builder.CreateAlignedLoad(
+          AT->getElementType(), Ptr, MinAlign(Align, Offset), Name + ".unpack");
       AAMDNodes AAMD;
       LI.getAAMetadata(AAMD);
       L->setAAMetadata(AAMD);
@@ -1068,10 +1066,12 @@ Instruction *InstCombiner::visitLoadInst(LoadInst &LI) {
       unsigned Align = LI.getAlignment();
       if (isSafeToLoadUnconditionally(SI->getOperand(1), Align, DL, SI) &&
           isSafeToLoadUnconditionally(SI->getOperand(2), Align, DL, SI)) {
-        LoadInst *V1 = Builder.CreateLoad(SI->getOperand(1),
-                                          SI->getOperand(1)->getName()+".val");
-        LoadInst *V2 = Builder.CreateLoad(SI->getOperand(2),
-                                          SI->getOperand(2)->getName()+".val");
+        LoadInst *V1 =
+            Builder.CreateLoad(LI.getType(), SI->getOperand(1),
+                               SI->getOperand(1)->getName() + ".val");
+        LoadInst *V2 =
+            Builder.CreateLoad(LI.getType(), SI->getOperand(2),
+                               SI->getOperand(2)->getName() + ".val");
         assert(LI.isUnordered() && "implied by above");
         V1->setAlignment(Align);
         V1->setAtomic(LI.getOrdering(), LI.getSyncScopeID());
diff --git a/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp b/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp
index c348aecb2d44a9a6054316da902ce38b9fad2185..03855f6bac41620c8b33def93c49146ca599b3fe 100644
--- a/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp
+++ b/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp
@@ -1,9 +1,8 @@
 //===- InstCombineMulDivRem.cpp -------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -244,6 +243,11 @@ Instruction *InstCombiner::visitMul(BinaryOperator &I) {
     return NewMul;
   }
 
+  // -X * Y --> -(X * Y)
+  // X * -Y --> -(X * Y)
+  if (match(&I, m_c_Mul(m_OneUse(m_Neg(m_Value(X))), m_Value(Y))))
+    return BinaryOperator::CreateNeg(Builder.CreateMul(X, Y));
+
   // (X / Y) *  Y = X - (X % Y)
   // (X / Y) * -Y = (X % Y) - X
   {
@@ -437,6 +441,26 @@ Instruction *InstCombiner::visitFMul(BinaryOperator &I) {
       return replaceInstUsesWith(I, Sqrt);
     }
 
+    // exp(X) * exp(Y) -> exp(X + Y)
+    // Match as long as at least one of exp has only one use.
+    if (match(Op0, m_Intrinsic<Intrinsic::exp>(m_Value(X))) &&
+        match(Op1, m_Intrinsic<Intrinsic::exp>(m_Value(Y))) &&
+        (Op0->hasOneUse() || Op1->hasOneUse())) {
+      Value *XY = Builder.CreateFAddFMF(X, Y, &I);
+      Value *Exp = Builder.CreateUnaryIntrinsic(Intrinsic::exp, XY, &I);
+      return replaceInstUsesWith(I, Exp);
+    }
+
+    // exp2(X) * exp2(Y) -> exp2(X + Y)
+    // Match as long as at least one of exp2 has only one use.
+    if (match(Op0, m_Intrinsic<Intrinsic::exp2>(m_Value(X))) &&
+        match(Op1, m_Intrinsic<Intrinsic::exp2>(m_Value(Y))) &&
+        (Op0->hasOneUse() || Op1->hasOneUse())) {
+      Value *XY = Builder.CreateFAddFMF(X, Y, &I);
+      Value *Exp2 = Builder.CreateUnaryIntrinsic(Intrinsic::exp2, XY, &I);
+      return replaceInstUsesWith(I, Exp2);
+    }
+
     // (X*Y) * X => (X*X) * Y where Y != X
     //  The purpose is two-fold:
     //   1) to form a power expression (of X).
@@ -1156,7 +1180,8 @@ Instruction *InstCombiner::visitFDiv(BinaryOperator &I) {
       IRBuilder<> B(&I);
       IRBuilder<>::FastMathFlagGuard FMFGuard(B);
       B.setFastMathFlags(I.getFastMathFlags());
-      AttributeList Attrs = CallSite(Op0).getCalledFunction()->getAttributes();
+      AttributeList Attrs =
+          cast<CallBase>(Op0)->getCalledFunction()->getAttributes();
       Value *Res = emitUnaryFloatFnCall(X, &TLI, LibFunc_tan, LibFunc_tanf,
                                         LibFunc_tanl, B, Attrs);
       if (IsCot)
diff --git a/lib/Transforms/InstCombine/InstCombinePHI.cpp b/lib/Transforms/InstCombine/InstCombinePHI.cpp
index a71ebdcd346a4e3888b1b59239dcd06673b522d1..e217adec7ed56e59829c56c95d3988cc25924c80 100644
--- a/lib/Transforms/InstCombine/InstCombinePHI.cpp
+++ b/lib/Transforms/InstCombine/InstCombinePHI.cpp
@@ -1,9 +1,8 @@
 //===- InstCombinePHI.cpp -------------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -596,7 +595,8 @@ Instruction *InstCombiner::FoldPHIArgLoadIntoPHI(PHINode &PN) {
 
   Value *InVal = FirstLI->getOperand(0);
   NewPN->addIncoming(InVal, PN.getIncomingBlock(0));
-  LoadInst *NewLI = new LoadInst(NewPN, "", isVolatile, LoadAlignment);
+  LoadInst *NewLI =
+      new LoadInst(FirstLI->getType(), NewPN, "", isVolatile, LoadAlignment);
 
   unsigned KnownIDs[] = {
     LLVMContext::MD_tbaa,
@@ -608,6 +608,7 @@ Instruction *InstCombiner::FoldPHIArgLoadIntoPHI(PHINode &PN) {
     LLVMContext::MD_align,
     LLVMContext::MD_dereferenceable,
     LLVMContext::MD_dereferenceable_or_null,
+    LLVMContext::MD_access_group,
   };
 
   for (unsigned ID : KnownIDs)
diff --git a/lib/Transforms/InstCombine/InstCombineSelect.cpp b/lib/Transforms/InstCombine/InstCombineSelect.cpp
index 19858ae149aba951db98330eb0823061342cc022..5922e6f7506311ee1f1b0d7abdb493125983e162 100644
--- a/lib/Transforms/InstCombine/InstCombineSelect.cpp
+++ b/lib/Transforms/InstCombine/InstCombineSelect.cpp
@@ -1,9 +1,8 @@
 //===- InstCombineSelect.cpp ----------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -676,6 +675,23 @@ static Value *canonicalizeSaturatedSubtract(const ICmpInst *ICI,
   return IsNegative ? Builder.CreateSub(B, Max) : Builder.CreateSub(Max, B);
 }
 
+static Value *canonicalizeSaturatedAdd(ICmpInst *Cmp, Value *TVal, Value *FVal,
+                                       InstCombiner::BuilderTy &Builder) {
+  // Match an unsigned saturated add with constant.
+  Value *X = Cmp->getOperand(0);
+  const APInt *CmpC, *AddC;
+  if (!Cmp->hasOneUse() || Cmp->getPredicate() != ICmpInst::ICMP_ULT ||
+      !match(Cmp->getOperand(1), m_APInt(CmpC)) || !match(FVal, m_AllOnes()) ||
+      !match(TVal, m_Add(m_Specific(X), m_APInt(AddC))) || ~(*AddC) != *CmpC)
+    return nullptr;
+
+  // Commute compare and select operands:
+  // select (icmp ult X, C), (add X, ~C), -1 -->
+  // select (icmp ugt X, C), -1, (add X, ~C)
+  Value *NewCmp = Builder.CreateICmp(ICmpInst::ICMP_UGT, X, Cmp->getOperand(1));
+  return Builder.CreateSelect(NewCmp, FVal, TVal);
+}
+
 /// Attempt to fold a cttz/ctlz followed by a icmp plus select into a single
 /// call to cttz/ctlz with flag 'is_zero_undef' cleared.
 ///
@@ -709,17 +725,18 @@ static Value *foldSelectCttzCtlz(ICmpInst *ICI, Value *TrueVal, Value *FalseVal,
       match(Count, m_Trunc(m_Value(V))))
     Count = V;
 
+  // Check that 'Count' is a call to intrinsic cttz/ctlz. Also check that the
+  // input to the cttz/ctlz is used as LHS for the compare instruction.
+  if (!match(Count, m_Intrinsic<Intrinsic::cttz>(m_Specific(CmpLHS))) &&
+      !match(Count, m_Intrinsic<Intrinsic::ctlz>(m_Specific(CmpLHS))))
+    return nullptr;
+
+  IntrinsicInst *II = cast<IntrinsicInst>(Count);
+
   // Check if the value propagated on zero is a constant number equal to the
   // sizeof in bits of 'Count'.
   unsigned SizeOfInBits = Count->getType()->getScalarSizeInBits();
-  if (!match(ValueOnZero, m_SpecificInt(SizeOfInBits)))
-    return nullptr;
-
-  // Check that 'Count' is a call to intrinsic cttz/ctlz. Also check that the
-  // input to the cttz/ctlz is used as LHS for the compare instruction.
-  if (match(Count, m_Intrinsic<Intrinsic::cttz>(m_Specific(CmpLHS))) ||
-      match(Count, m_Intrinsic<Intrinsic::ctlz>(m_Specific(CmpLHS)))) {
-    IntrinsicInst *II = cast<IntrinsicInst>(Count);
+  if (match(ValueOnZero, m_SpecificInt(SizeOfInBits))) {
     // Explicitly clear the 'undef_on_zero' flag.
     IntrinsicInst *NewI = cast<IntrinsicInst>(II->clone());
     NewI->setArgOperand(1, ConstantInt::getFalse(NewI->getContext()));
@@ -727,6 +744,12 @@ static Value *foldSelectCttzCtlz(ICmpInst *ICI, Value *TrueVal, Value *FalseVal,
     return Builder.CreateZExtOrTrunc(NewI, ValueOnZero->getType());
   }
 
+  // If the ValueOnZero is not the bitwidth, we can at least make use of the
+  // fact that the cttz/ctlz result will not be used if the input is zero, so
+  // it's okay to relax it to undef for that case.
+  if (II->hasOneUse() && !match(II->getArgOperand(1), m_One()))
+    II->setArgOperand(1, ConstantInt::getTrue(II->getContext()));
+
   return nullptr;
 }
 
@@ -1042,6 +1065,9 @@ Instruction *InstCombiner::foldSelectInstWithICmp(SelectInst &SI,
   if (Value *V = canonicalizeSaturatedSubtract(ICI, TrueVal, FalseVal, Builder))
     return replaceInstUsesWith(SI, V);
 
+  if (Value *V = canonicalizeSaturatedAdd(ICI, TrueVal, FalseVal, Builder))
+    return replaceInstUsesWith(SI, V);
+
   return Changed ? &SI : nullptr;
 }
 
@@ -1547,11 +1573,10 @@ static Instruction *factorizeMinMaxTree(SelectPatternFlavor SPF, Value *LHS,
 }
 
 /// Try to reduce a rotate pattern that includes a compare and select into a
-/// sequence of ALU ops only. Example:
+/// funnel shift intrinsic. Example:
 /// rotl32(a, b) --> (b == 0 ? a : ((a >> (32 - b)) | (a << b)))
-///              --> (a >> (-b & 31)) | (a << (b & 31))
-static Instruction *foldSelectRotate(SelectInst &Sel,
-                                     InstCombiner::BuilderTy &Builder) {
+///              --> call llvm.fshl.i32(a, a, b)
+static Instruction *foldSelectRotate(SelectInst &Sel) {
   // The false value of the select must be a rotate of the true value.
   Value *Or0, *Or1;
   if (!match(Sel.getFalseValue(), m_OneUse(m_Or(m_Value(Or0), m_Value(Or1)))))
@@ -1593,17 +1618,12 @@ static Instruction *foldSelectRotate(SelectInst &Sel,
     return nullptr;
 
   // This is a rotate that avoids shift-by-bitwidth UB in a suboptimal way.
-  // Convert to safely bitmasked shifts.
-  // TODO: When we can canonicalize to funnel shift intrinsics without risk of
-  // performance regressions, replace this sequence with that call.
-  Value *NegShAmt = Builder.CreateNeg(ShAmt);
-  Value *MaskedShAmt = Builder.CreateAnd(ShAmt, Width - 1);
-  Value *MaskedNegShAmt = Builder.CreateAnd(NegShAmt, Width - 1);
-  Value *NewSA0 = ShAmt == SA0 ? MaskedShAmt : MaskedNegShAmt;
-  Value *NewSA1 = ShAmt == SA1 ? MaskedShAmt : MaskedNegShAmt;
-  Value *NewSh0 = Builder.CreateBinOp(ShiftOpcode0, TVal, NewSA0);
-  Value *NewSh1 = Builder.CreateBinOp(ShiftOpcode1, TVal, NewSA1);
-  return BinaryOperator::CreateOr(NewSh0, NewSh1);
+  // Convert to funnel shift intrinsic.
+  bool IsFshl = (ShAmt == SA0 && ShiftOpcode0 == BinaryOperator::Shl) ||
+                (ShAmt == SA1 && ShiftOpcode1 == BinaryOperator::Shl);
+  Intrinsic::ID IID = IsFshl ? Intrinsic::fshl : Intrinsic::fshr;
+  Function *F = Intrinsic::getDeclaration(Sel.getModule(), IID, Sel.getType());
+  return IntrinsicInst::Create(F, { TVal, TVal, ShAmt });
 }
 
 Instruction *InstCombiner::visitSelectInst(SelectInst &SI) {
@@ -2045,7 +2065,7 @@ Instruction *InstCombiner::visitSelectInst(SelectInst &SI) {
   if (Instruction *Select = foldSelectBinOpIdentity(SI, TLI))
     return Select;
 
-  if (Instruction *Rot = foldSelectRotate(SI, Builder))
+  if (Instruction *Rot = foldSelectRotate(SI))
     return Rot;
 
   return nullptr;
diff --git a/lib/Transforms/InstCombine/InstCombineShifts.cpp b/lib/Transforms/InstCombine/InstCombineShifts.cpp
index c562d45a9e2b919aff71d45971eb0d0a1007fedd..fbbba4632d02c23ef8578e6749e4486befc42b77 100644
--- a/lib/Transforms/InstCombine/InstCombineShifts.cpp
+++ b/lib/Transforms/InstCombine/InstCombineShifts.cpp
@@ -1,9 +1,8 @@
 //===- InstCombineShifts.cpp ----------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp b/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp
index a193dde1c39517e4c971aca666ac9e9709a67328..19cdea6923045418f1ca028326875e93ba163c67 100644
--- a/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp
+++ b/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp
@@ -1,9 +1,8 @@
 //===- InstCombineSimplifyDemanded.cpp ------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -967,6 +966,9 @@ InstCombiner::simplifyShrShlDemandedBits(Instruction *Shr, const APInt &ShrOp1,
 }
 
 /// Implement SimplifyDemandedVectorElts for amdgcn buffer and image intrinsics.
+///
+/// Note: This only supports non-TFE/LWE image intrinsic calls; those have
+///       struct returns.
 Value *InstCombiner::simplifyAMDGCNMemoryIntrinsicDemanded(IntrinsicInst *II,
                                                            APInt DemandedElts,
                                                            int DMaskIdx) {
@@ -1171,6 +1173,18 @@ Value *InstCombiner::SimplifyDemandedVectorElts(Value *V, APInt DemandedElts,
   switch (I->getOpcode()) {
   default: break;
 
+  case Instruction::GetElementPtr: {
+    // Conservatively track the demanded elements back through any vector
+    // operands we may have.  We know there must be at least one, or we
+    // wouldn't have a vector result to get here. Note that we intentionally
+    // merge the undef bits here since gepping with either an undef base or
+    // index results in undef. 
+    for (unsigned i = 0; i < I->getNumOperands(); i++)
+      if (I->getOperand(i)->getType()->isVectorTy())
+        simplifyAndSetOp(I, i, DemandedElts, UndefElts);
+
+    break;
+  }
   case Instruction::InsertElement: {
     // If this is a variable index, we don't know which element it overwrites.
     // demand exactly the same input as we produce.
@@ -1619,10 +1633,21 @@ Value *InstCombiner::SimplifyDemandedVectorElts(Value *V, APInt DemandedElts,
       break;
     case Intrinsic::amdgcn_buffer_load:
     case Intrinsic::amdgcn_buffer_load_format:
+    case Intrinsic::amdgcn_raw_buffer_load:
+    case Intrinsic::amdgcn_raw_buffer_load_format:
+    case Intrinsic::amdgcn_struct_buffer_load:
+    case Intrinsic::amdgcn_struct_buffer_load_format:
       return simplifyAMDGCNMemoryIntrinsicDemanded(II, DemandedElts);
     default: {
-      if (getAMDGPUImageDMaskIntrinsic(II->getIntrinsicID()))
+      if (getAMDGPUImageDMaskIntrinsic(II->getIntrinsicID())) {
+        LLVM_DEBUG(
+          Value *TFC = II->getArgOperand(II->getNumOperands() - 2);
+          assert(!isa<ConstantInt>(TFC) ||
+                 dyn_cast<ConstantInt>(TFC)->getZExtValue() == 0);
+        );
+
         return simplifyAMDGCNMemoryIntrinsicDemanded(II, DemandedElts, 0);
+      }
 
       break;
     }
diff --git a/lib/Transforms/InstCombine/InstCombineVectorOps.cpp b/lib/Transforms/InstCombine/InstCombineVectorOps.cpp
index 0ad1fc0e791f140a28c910ae44bf17570143652d..6889cd9189ce13f4db56899046ac6c714387c8d6 100644
--- a/lib/Transforms/InstCombine/InstCombineVectorOps.cpp
+++ b/lib/Transforms/InstCombine/InstCombineVectorOps.cpp
@@ -1,9 +1,8 @@
 //===- InstCombineVectorOps.cpp -------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -1499,6 +1498,11 @@ static Instruction *foldIdentityExtractShuffle(ShuffleVectorInst &Shuf) {
   if (!match(Op0, m_ShuffleVector(m_Value(X), m_Value(Y), m_Constant(Mask))))
     return nullptr;
 
+  // Be conservative with shuffle transforms. If we can't kill the 1st shuffle,
+  // then combining may result in worse codegen.
+  if (!Op0->hasOneUse())
+    return nullptr;
+
   // We are extracting a subvector from a shuffle. Remove excess elements from
   // the 1st shuffle mask to eliminate the extract.
   //
diff --git a/lib/Transforms/InstCombine/InstructionCombining.cpp b/lib/Transforms/InstCombine/InstructionCombining.cpp
index be7d43bbcf2c32fbfe25920e40f20b24fdef4369..8cbf57a97ef7e8d86ac26280bf904ab718d8fbdc 100644
--- a/lib/Transforms/InstCombine/InstructionCombining.cpp
+++ b/lib/Transforms/InstCombine/InstructionCombining.cpp
@@ -1,9 +1,8 @@
 //===- InstructionCombining.cpp - Combine multiple instructions -----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -922,8 +921,8 @@ Instruction *InstCombiner::foldOpIntoPhi(Instruction &I, PHINode *PN) {
 
     // If the InVal is an invoke at the end of the pred block, then we can't
     // insert a computation after it without breaking the edge.
-    if (InvokeInst *II = dyn_cast<InvokeInst>(InVal))
-      if (II->getParent() == NonConstBB)
+    if (isa<InvokeInst>(InVal))
+      if (cast<Instruction>(InVal)->getParent() == NonConstBB)
         return nullptr;
 
     // If the incoming non-constant value is in I's block, we will remove one
@@ -1755,9 +1754,9 @@ Instruction *InstCombiner::visitGetElementPtrInst(GetElementPtrInst &GEP) {
             // put NewSrc at same location as %src
             Builder.SetInsertPoint(cast<Instruction>(PtrOp));
             auto *NewSrc = cast<GetElementPtrInst>(
-                Builder.CreateGEP(SO0, GO1, Src->getName()));
+                Builder.CreateGEP(GEPEltType, SO0, GO1, Src->getName()));
             NewSrc->setIsInBounds(Src->isInBounds());
-            auto *NewGEP = GetElementPtrInst::Create(nullptr, NewSrc, {SO1});
+            auto *NewGEP = GetElementPtrInst::Create(GEPEltType, NewSrc, {SO1});
             NewGEP->setIsInBounds(GEP.isInBounds());
             return NewGEP;
           }
@@ -1881,6 +1880,8 @@ Instruction *InstCombiner::visitGetElementPtrInst(GetElementPtrInst &GEP) {
 
   if (StrippedPtr != PtrOp) {
     bool HasZeroPointerIndex = false;
+    Type *StrippedPtrEltTy = StrippedPtrTy->getElementType();
+
     if (auto *C = dyn_cast<ConstantInt>(GEP.getOperand(1)))
       HasZeroPointerIndex = C->isZero();
 
@@ -1894,11 +1895,11 @@ Instruction *InstCombiner::visitGetElementPtrInst(GetElementPtrInst &GEP) {
     if (HasZeroPointerIndex) {
       if (auto *CATy = dyn_cast<ArrayType>(GEPEltType)) {
         // GEP (bitcast i8* X to [0 x i8]*), i32 0, ... ?
-        if (CATy->getElementType() == StrippedPtrTy->getElementType()) {
+        if (CATy->getElementType() == StrippedPtrEltTy) {
           // -> GEP i8* X, ...
           SmallVector<Value*, 8> Idx(GEP.idx_begin()+1, GEP.idx_end());
           GetElementPtrInst *Res = GetElementPtrInst::Create(
-              StrippedPtrTy->getElementType(), StrippedPtr, Idx, GEP.getName());
+              StrippedPtrEltTy, StrippedPtr, Idx, GEP.getName());
           Res->setIsInBounds(GEP.isInBounds());
           if (StrippedPtrTy->getAddressSpace() == GEP.getAddressSpace())
             return Res;
@@ -1911,7 +1912,7 @@ Instruction *InstCombiner::visitGetElementPtrInst(GetElementPtrInst &GEP) {
           return new AddrSpaceCastInst(Builder.Insert(Res), GEPType);
         }
 
-        if (auto *XATy = dyn_cast<ArrayType>(StrippedPtrTy->getElementType())) {
+        if (auto *XATy = dyn_cast<ArrayType>(StrippedPtrEltTy)) {
           // GEP (bitcast [10 x i8]* X to [0 x i8]*), i32 0, ... ?
           if (CATy->getElementType() == XATy->getElementType()) {
             // -> GEP [10 x i8]* X, i32 0, ...
@@ -1934,11 +1935,12 @@ Instruction *InstCombiner::visitGetElementPtrInst(GetElementPtrInst &GEP) {
             // %0 = GEP [10 x i8] addrspace(1)* X, ...
             // addrspacecast i8 addrspace(1)* %0 to i8*
             SmallVector<Value*, 8> Idx(GEP.idx_begin(), GEP.idx_end());
-            Value *NewGEP = GEP.isInBounds()
-                                ? Builder.CreateInBoundsGEP(
-                                      nullptr, StrippedPtr, Idx, GEP.getName())
-                                : Builder.CreateGEP(nullptr, StrippedPtr, Idx,
-                                                    GEP.getName());
+            Value *NewGEP =
+                GEP.isInBounds()
+                    ? Builder.CreateInBoundsGEP(StrippedPtrEltTy, StrippedPtr,
+                                                Idx, GEP.getName())
+                    : Builder.CreateGEP(StrippedPtrEltTy, StrippedPtr, Idx,
+                                        GEP.getName());
             return new AddrSpaceCastInst(NewGEP, GEPType);
           }
         }
@@ -1947,17 +1949,17 @@ Instruction *InstCombiner::visitGetElementPtrInst(GetElementPtrInst &GEP) {
       // Transform things like:
       // %t = getelementptr i32* bitcast ([2 x i32]* %str to i32*), i32 %V
       // into:  %t1 = getelementptr [2 x i32]* %str, i32 0, i32 %V; bitcast
-      Type *SrcEltTy = StrippedPtrTy->getElementType();
-      if (SrcEltTy->isArrayTy() &&
-          DL.getTypeAllocSize(SrcEltTy->getArrayElementType()) ==
+      if (StrippedPtrEltTy->isArrayTy() &&
+          DL.getTypeAllocSize(StrippedPtrEltTy->getArrayElementType()) ==
               DL.getTypeAllocSize(GEPEltType)) {
         Type *IdxType = DL.getIndexType(GEPType);
         Value *Idx[2] = { Constant::getNullValue(IdxType), GEP.getOperand(1) };
         Value *NewGEP =
             GEP.isInBounds()
-                ? Builder.CreateInBoundsGEP(nullptr, StrippedPtr, Idx,
+                ? Builder.CreateInBoundsGEP(StrippedPtrEltTy, StrippedPtr, Idx,
                                             GEP.getName())
-                : Builder.CreateGEP(nullptr, StrippedPtr, Idx, GEP.getName());
+                : Builder.CreateGEP(StrippedPtrEltTy, StrippedPtr, Idx,
+                                    GEP.getName());
 
         // V and GEP are both pointer types --> BitCast
         return CastInst::CreatePointerBitCastOrAddrSpaceCast(NewGEP, GEPType);
@@ -1967,11 +1969,11 @@ Instruction *InstCombiner::visitGetElementPtrInst(GetElementPtrInst &GEP) {
       // %V = mul i64 %N, 4
       // %t = getelementptr i8* bitcast (i32* %arr to i8*), i32 %V
       // into:  %t1 = getelementptr i32* %arr, i32 %N; bitcast
-      if (GEPEltType->isSized() && SrcEltTy->isSized()) {
+      if (GEPEltType->isSized() && StrippedPtrEltTy->isSized()) {
         // Check that changing the type amounts to dividing the index by a scale
         // factor.
         uint64_t ResSize = DL.getTypeAllocSize(GEPEltType);
-        uint64_t SrcSize = DL.getTypeAllocSize(SrcEltTy);
+        uint64_t SrcSize = DL.getTypeAllocSize(StrippedPtrEltTy);
         if (ResSize && SrcSize % ResSize == 0) {
           Value *Idx = GEP.getOperand(1);
           unsigned BitWidth = Idx->getType()->getPrimitiveSizeInBits();
@@ -1990,9 +1992,9 @@ Instruction *InstCombiner::visitGetElementPtrInst(GetElementPtrInst &GEP) {
             // GEP may not be "inbounds".
             Value *NewGEP =
                 GEP.isInBounds() && NSW
-                    ? Builder.CreateInBoundsGEP(nullptr, StrippedPtr, NewIdx,
-                                                GEP.getName())
-                    : Builder.CreateGEP(nullptr, StrippedPtr, NewIdx,
+                    ? Builder.CreateInBoundsGEP(StrippedPtrEltTy, StrippedPtr,
+                                                NewIdx, GEP.getName())
+                    : Builder.CreateGEP(StrippedPtrEltTy, StrippedPtr, NewIdx,
                                         GEP.getName());
 
             // The NewGEP must be pointer typed, so must the old one -> BitCast
@@ -2006,13 +2008,13 @@ Instruction *InstCombiner::visitGetElementPtrInst(GetElementPtrInst &GEP) {
       // getelementptr i8* bitcast ([100 x double]* X to i8*), i32 %tmp
       //   (where tmp = 8*tmp2) into:
       // getelementptr [100 x double]* %arr, i32 0, i32 %tmp2; bitcast
-      if (GEPEltType->isSized() && SrcEltTy->isSized() &&
-          SrcEltTy->isArrayTy()) {
+      if (GEPEltType->isSized() && StrippedPtrEltTy->isSized() &&
+          StrippedPtrEltTy->isArrayTy()) {
         // Check that changing to the array element type amounts to dividing the
         // index by a scale factor.
         uint64_t ResSize = DL.getTypeAllocSize(GEPEltType);
         uint64_t ArrayEltSize =
-            DL.getTypeAllocSize(SrcEltTy->getArrayElementType());
+            DL.getTypeAllocSize(StrippedPtrEltTy->getArrayElementType());
         if (ResSize && ArrayEltSize % ResSize == 0) {
           Value *Idx = GEP.getOperand(1);
           unsigned BitWidth = Idx->getType()->getPrimitiveSizeInBits();
@@ -2032,11 +2034,12 @@ Instruction *InstCombiner::visitGetElementPtrInst(GetElementPtrInst &GEP) {
             Type *IndTy = DL.getIndexType(GEPType);
             Value *Off[2] = {Constant::getNullValue(IndTy), NewIdx};
 
-            Value *NewGEP = GEP.isInBounds() && NSW
-                                ? Builder.CreateInBoundsGEP(
-                                      SrcEltTy, StrippedPtr, Off, GEP.getName())
-                                : Builder.CreateGEP(SrcEltTy, StrippedPtr, Off,
-                                                    GEP.getName());
+            Value *NewGEP =
+                GEP.isInBounds() && NSW
+                    ? Builder.CreateInBoundsGEP(StrippedPtrEltTy, StrippedPtr,
+                                                Off, GEP.getName())
+                    : Builder.CreateGEP(StrippedPtrEltTy, StrippedPtr, Off,
+                                        GEP.getName());
             // The NewGEP must be pointer typed, so must the old one -> BitCast
             return CastInst::CreatePointerBitCastOrAddrSpaceCast(NewGEP,
                                                                  GEPType);
@@ -2084,8 +2087,8 @@ Instruction *InstCombiner::visitGetElementPtrInst(GetElementPtrInst &GEP) {
       // constructing an AddrSpaceCastInst
       Value *NGEP =
           GEP.isInBounds()
-              ? Builder.CreateInBoundsGEP(nullptr, SrcOp, {Ops[1], Ops[2]})
-              : Builder.CreateGEP(nullptr, SrcOp, {Ops[1], Ops[2]});
+              ? Builder.CreateInBoundsGEP(SrcEltType, SrcOp, {Ops[1], Ops[2]})
+              : Builder.CreateGEP(SrcEltType, SrcOp, {Ops[1], Ops[2]});
       NGEP->takeName(&GEP);
 
       // Preserve GEP address space to satisfy users
@@ -2132,8 +2135,8 @@ Instruction *InstCombiner::visitGetElementPtrInst(GetElementPtrInst &GEP) {
       if (FindElementAtOffset(SrcType, Offset.getSExtValue(), NewIndices)) {
         Value *NGEP =
             GEP.isInBounds()
-                ? Builder.CreateInBoundsGEP(nullptr, SrcOp, NewIndices)
-                : Builder.CreateGEP(nullptr, SrcOp, NewIndices);
+                ? Builder.CreateInBoundsGEP(SrcEltType, SrcOp, NewIndices)
+                : Builder.CreateGEP(SrcEltType, SrcOp, NewIndices);
 
         if (NGEP->getType() == GEPType)
           return replaceInstUsesWith(GEP, NGEP);
@@ -2159,7 +2162,8 @@ Instruction *InstCombiner::visitGetElementPtrInst(GetElementPtrInst &GEP) {
         APInt AllocSize(IdxWidth, DL.getTypeAllocSize(AI->getAllocatedType()));
         if (BasePtrOffset.ule(AllocSize)) {
           return GetElementPtrInst::CreateInBounds(
-              PtrOp, makeArrayRef(Ops).slice(1), GEP.getName());
+              GEP.getSourceElementType(), PtrOp, makeArrayRef(Ops).slice(1),
+              GEP.getName());
         }
       }
     }
@@ -2296,8 +2300,8 @@ Instruction *InstCombiner::visitAllocSite(Instruction &MI) {
 
       if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(I)) {
         if (II->getIntrinsicID() == Intrinsic::objectsize) {
-          ConstantInt *Result = lowerObjectSizeCall(II, DL, &TLI,
-                                                    /*MustSucceed=*/true);
+          Value *Result =
+              lowerObjectSizeCall(II, DL, &TLI, /*MustSucceed=*/true);
           replaceInstUsesWith(*I, Result);
           eraseInstFromFunction(*I);
           Users[i] = nullptr; // Skip examining in the next loop.
@@ -2687,7 +2691,7 @@ Instruction *InstCombiner::visitExtractValueInst(ExtractValueInst &EV) {
       Builder.SetInsertPoint(L);
       Value *GEP = Builder.CreateInBoundsGEP(L->getType(),
                                              L->getPointerOperand(), Indices);
-      Instruction *NL = Builder.CreateLoad(GEP);
+      Instruction *NL = Builder.CreateLoad(EV.getType(), GEP);
       // Whatever aliasing information we had for the orignal load must also
       // hold for the smaller load, so propagate the annotations.
       AAMDNodes Nodes;
@@ -3065,9 +3069,11 @@ static bool TryToSinkInstruction(Instruction *I, BasicBlock *DestBlock) {
       I->isTerminator())
     return false;
 
-  // Do not sink alloca instructions out of the entry block.
-  if (isa<AllocaInst>(I) && I->getParent() ==
-        &DestBlock->getParent()->getEntryBlock())
+  // Do not sink static or dynamic alloca instructions. Static allocas must
+  // remain in the entry block, and dynamic allocas must not be sunk in between
+  // a stacksave / stackrestore pair, which would incorrectly shorten its
+  // lifetime.
+  if (isa<AllocaInst>(I))
     return false;
 
   // Do not sink into catchswitch blocks.
@@ -3093,13 +3099,35 @@ static bool TryToSinkInstruction(Instruction *I, BasicBlock *DestBlock) {
   ++NumSunkInst;
 
   // Also sink all related debug uses from the source basic block. Otherwise we
-  // get debug use before the def.
-  SmallVector<DbgVariableIntrinsic *, 1> DbgUsers;
+  // get debug use before the def. Attempt to salvage debug uses first, to
+  // maximise the range variables have location for. If we cannot salvage, then
+  // mark the location undef: we know it was supposed to receive a new location
+  // here, but that computation has been sunk.
+  SmallVector<DbgVariableIntrinsic *, 2> DbgUsers;
   findDbgUsers(DbgUsers, I);
-  for (auto *DII : DbgUsers) {
+  for (auto *DII : reverse(DbgUsers)) {
     if (DII->getParent() == SrcBlock) {
-      DII->moveBefore(&*InsertPos);
-      LLVM_DEBUG(dbgs() << "SINK: " << *DII << '\n');
+      // dbg.value is in the same basic block as the sunk inst, see if we can
+      // salvage it. Clone a new copy of the instruction: on success we need
+      // both salvaged and unsalvaged copies.
+      SmallVector<DbgVariableIntrinsic *, 1> TmpUser{
+          cast<DbgVariableIntrinsic>(DII->clone())};
+
+      if (!salvageDebugInfoForDbgValues(*I, TmpUser)) {
+        // We are unable to salvage: sink the cloned dbg.value, and mark the
+        // original as undef, terminating any earlier variable location.
+        LLVM_DEBUG(dbgs() << "SINK: " << *DII << '\n');
+        TmpUser[0]->insertBefore(&*InsertPos);
+        Value *Undef = UndefValue::get(I->getType());
+        DII->setOperand(0, MetadataAsValue::get(DII->getContext(),
+                                                ValueAsMetadata::get(Undef)));
+      } else {
+        // We successfully salvaged: place the salvaged dbg.value in the
+        // original location, and move the unmodified dbg.value to sink with
+        // the sunk inst.
+        TmpUser[0]->insertBefore(DII);
+        DII->moveBefore(&*InsertPos);
+      }
     }
   }
   return true;
diff --git a/lib/Transforms/InstCombine/LLVMBuild.txt b/lib/Transforms/InstCombine/LLVMBuild.txt
index c26e0e3ea40b76bfcc463315a0c9ca3b7aab1512..d1f91c071c06ab408c70c0c32835b0f294508ced 100644
--- a/lib/Transforms/InstCombine/LLVMBuild.txt
+++ b/lib/Transforms/InstCombine/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/Transforms/InstCombine/LLVMBuild.txt ---------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Transforms/Instrumentation/AddressSanitizer.cpp b/lib/Transforms/Instrumentation/AddressSanitizer.cpp
index 2ede46774b24ebff28080fefea5e8d0581710326..e95b88b579c596cb96c67f326ac20bdcc02f5191 100644
--- a/lib/Transforms/Instrumentation/AddressSanitizer.cpp
+++ b/lib/Transforms/Instrumentation/AddressSanitizer.cpp
@@ -1,9 +1,8 @@
 //===- AddressSanitizer.cpp - memory error detector -----------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -716,19 +715,19 @@ private:
   Type *IntptrTy;
   ShadowMapping Mapping;
   DominatorTree *DT;
-  Function *AsanHandleNoReturnFunc;
-  Function *AsanPtrCmpFunction, *AsanPtrSubFunction;
+  FunctionCallee AsanHandleNoReturnFunc;
+  FunctionCallee AsanPtrCmpFunction, AsanPtrSubFunction;
   Constant *AsanShadowGlobal;
 
   // These arrays is indexed by AccessIsWrite, Experiment and log2(AccessSize).
-  Function *AsanErrorCallback[2][2][kNumberOfAccessSizes];
-  Function *AsanMemoryAccessCallback[2][2][kNumberOfAccessSizes];
+  FunctionCallee AsanErrorCallback[2][2][kNumberOfAccessSizes];
+  FunctionCallee AsanMemoryAccessCallback[2][2][kNumberOfAccessSizes];
 
   // These arrays is indexed by AccessIsWrite and Experiment.
-  Function *AsanErrorCallbackSized[2][2];
-  Function *AsanMemoryAccessCallbackSized[2][2];
+  FunctionCallee AsanErrorCallbackSized[2][2];
+  FunctionCallee AsanMemoryAccessCallbackSized[2][2];
 
-  Function *AsanMemmove, *AsanMemcpy, *AsanMemset;
+  FunctionCallee AsanMemmove, AsanMemcpy, AsanMemset;
   InlineAsm *EmptyAsm;
   Value *LocalDynamicShadow = nullptr;
   GlobalsMetadata GlobalsMD;
@@ -810,14 +809,14 @@ private:
   LLVMContext *C;
   Triple TargetTriple;
   ShadowMapping Mapping;
-  Function *AsanPoisonGlobals;
-  Function *AsanUnpoisonGlobals;
-  Function *AsanRegisterGlobals;
-  Function *AsanUnregisterGlobals;
-  Function *AsanRegisterImageGlobals;
-  Function *AsanUnregisterImageGlobals;
-  Function *AsanRegisterElfGlobals;
-  Function *AsanUnregisterElfGlobals;
+  FunctionCallee AsanPoisonGlobals;
+  FunctionCallee AsanUnpoisonGlobals;
+  FunctionCallee AsanRegisterGlobals;
+  FunctionCallee AsanUnregisterGlobals;
+  FunctionCallee AsanRegisterImageGlobals;
+  FunctionCallee AsanUnregisterImageGlobals;
+  FunctionCallee AsanRegisterElfGlobals;
+  FunctionCallee AsanUnregisterElfGlobals;
 
   Function *AsanCtorFunction = nullptr;
   Function *AsanDtorFunction = nullptr;
@@ -846,11 +845,11 @@ struct FunctionStackPoisoner : public InstVisitor<FunctionStackPoisoner> {
   SmallVector<Instruction *, 8> RetVec;
   unsigned StackAlignment;
 
-  Function *AsanStackMallocFunc[kMaxAsanStackMallocSizeClass + 1],
-      *AsanStackFreeFunc[kMaxAsanStackMallocSizeClass + 1];
-  Function *AsanSetShadowFunc[0x100] = {};
-  Function *AsanPoisonStackMemoryFunc, *AsanUnpoisonStackMemoryFunc;
-  Function *AsanAllocaPoisonFunc, *AsanAllocasUnpoisonFunc;
+  FunctionCallee AsanStackMallocFunc[kMaxAsanStackMallocSizeClass + 1],
+      AsanStackFreeFunc[kMaxAsanStackMallocSizeClass + 1];
+  FunctionCallee AsanSetShadowFunc[0x100] = {};
+  FunctionCallee AsanPoisonStackMemoryFunc, AsanUnpoisonStackMemoryFunc;
+  FunctionCallee AsanAllocaPoisonFunc, AsanAllocasUnpoisonFunc;
 
   // Stores a place and arguments of poisoning/unpoisoning call for alloca.
   struct AllocaPoisonCall {
@@ -950,8 +949,9 @@ struct FunctionStackPoisoner : public InstVisitor<FunctionStackPoisoner> {
                                      DynamicAreaOffset);
     }
 
-    IRB.CreateCall(AsanAllocasUnpoisonFunc,
-                   {IRB.CreateLoad(DynamicAllocaLayout), DynamicAreaPtr});
+    IRB.CreateCall(
+        AsanAllocasUnpoisonFunc,
+        {IRB.CreateLoad(IntptrTy, DynamicAllocaLayout), DynamicAreaPtr});
   }
 
   // Unpoison dynamic allocas redzones.
@@ -1005,7 +1005,7 @@ struct FunctionStackPoisoner : public InstVisitor<FunctionStackPoisoner> {
     if (ID == Intrinsic::localescape) LocalEscapeCall = &II;
     if (!ASan.UseAfterScope)
       return;
-    if (ID != Intrinsic::lifetime_start && ID != Intrinsic::lifetime_end)
+    if (!II.isLifetimeStartOrEnd())
       return;
     // Found lifetime intrinsic, add ASan instrumentation if necessary.
     ConstantInt *Size = dyn_cast<ConstantInt>(II.getArgOperand(0));
@@ -1334,7 +1334,7 @@ bool AddressSanitizer::GlobalIsLinkerInitialized(GlobalVariable *G) {
 void AddressSanitizer::instrumentPointerComparisonOrSubtraction(
     Instruction *I) {
   IRBuilder<> IRB(I);
-  Function *F = isa<ICmpInst>(I) ? AsanPtrCmpFunction : AsanPtrSubFunction;
+  FunctionCallee F = isa<ICmpInst>(I) ? AsanPtrCmpFunction : AsanPtrSubFunction;
   Value *Param[2] = {I->getOperand(0), I->getOperand(1)};
   for (Value *&i : Param) {
     if (i->getType()->isPointerTy())
@@ -1392,7 +1392,7 @@ static void instrumentMaskedLoadOrStore(AddressSanitizer *Pass,
 
     IRBuilder<> IRB(InsertBefore);
     InstrumentedAddress =
-        IRB.CreateGEP(Addr, {Zero, ConstantInt::get(IntptrTy, Idx)});
+        IRB.CreateGEP(VTy, Addr, {Zero, ConstantInt::get(IntptrTy, Idx)});
     doInstrumentAddress(Pass, I, InsertBefore, InstrumentedAddress, Alignment,
                         Granularity, ElemTypeSize, IsWrite, SizeArgument,
                         UseCalls, Exp);
@@ -1553,7 +1553,7 @@ void AddressSanitizer::instrumentAddress(Instruction *OrigIns,
   Value *ShadowPtr = memToShadow(AddrLong, IRB);
   Value *CmpVal = Constant::getNullValue(ShadowTy);
   Value *ShadowValue =
-      IRB.CreateLoad(IRB.CreateIntToPtr(ShadowPtr, ShadowPtrTy));
+      IRB.CreateLoad(ShadowTy, IRB.CreateIntToPtr(ShadowPtr, ShadowPtrTy));
 
   Value *Cmp = IRB.CreateICmpNE(ShadowValue, CmpVal);
   size_t Granularity = 1ULL << Mapping.Scale;
@@ -1796,43 +1796,30 @@ void AddressSanitizerModule::initializeCallbacks(Module &M) {
   IRBuilder<> IRB(*C);
 
   // Declare our poisoning and unpoisoning functions.
-  AsanPoisonGlobals = checkSanitizerInterfaceFunction(M.getOrInsertFunction(
-      kAsanPoisonGlobalsName, IRB.getVoidTy(), IntptrTy));
-  AsanPoisonGlobals->setLinkage(Function::ExternalLinkage);
-  AsanUnpoisonGlobals = checkSanitizerInterfaceFunction(M.getOrInsertFunction(
-      kAsanUnpoisonGlobalsName, IRB.getVoidTy()));
-  AsanUnpoisonGlobals->setLinkage(Function::ExternalLinkage);
+  AsanPoisonGlobals =
+      M.getOrInsertFunction(kAsanPoisonGlobalsName, IRB.getVoidTy(), IntptrTy);
+  AsanUnpoisonGlobals =
+      M.getOrInsertFunction(kAsanUnpoisonGlobalsName, IRB.getVoidTy());
 
   // Declare functions that register/unregister globals.
-  AsanRegisterGlobals = checkSanitizerInterfaceFunction(M.getOrInsertFunction(
-      kAsanRegisterGlobalsName, IRB.getVoidTy(), IntptrTy, IntptrTy));
-  AsanRegisterGlobals->setLinkage(Function::ExternalLinkage);
-  AsanUnregisterGlobals = checkSanitizerInterfaceFunction(
-      M.getOrInsertFunction(kAsanUnregisterGlobalsName, IRB.getVoidTy(),
-                            IntptrTy, IntptrTy));
-  AsanUnregisterGlobals->setLinkage(Function::ExternalLinkage);
+  AsanRegisterGlobals = M.getOrInsertFunction(
+      kAsanRegisterGlobalsName, IRB.getVoidTy(), IntptrTy, IntptrTy);
+  AsanUnregisterGlobals = M.getOrInsertFunction(
+      kAsanUnregisterGlobalsName, IRB.getVoidTy(), IntptrTy, IntptrTy);
 
   // Declare the functions that find globals in a shared object and then invoke
   // the (un)register function on them.
-  AsanRegisterImageGlobals =
-      checkSanitizerInterfaceFunction(M.getOrInsertFunction(
-          kAsanRegisterImageGlobalsName, IRB.getVoidTy(), IntptrTy));
-  AsanRegisterImageGlobals->setLinkage(Function::ExternalLinkage);
+  AsanRegisterImageGlobals = M.getOrInsertFunction(
+      kAsanRegisterImageGlobalsName, IRB.getVoidTy(), IntptrTy);
+  AsanUnregisterImageGlobals = M.getOrInsertFunction(
+      kAsanUnregisterImageGlobalsName, IRB.getVoidTy(), IntptrTy);
 
-  AsanUnregisterImageGlobals =
-      checkSanitizerInterfaceFunction(M.getOrInsertFunction(
-          kAsanUnregisterImageGlobalsName, IRB.getVoidTy(), IntptrTy));
-  AsanUnregisterImageGlobals->setLinkage(Function::ExternalLinkage);
-
-  AsanRegisterElfGlobals = checkSanitizerInterfaceFunction(
+  AsanRegisterElfGlobals =
       M.getOrInsertFunction(kAsanRegisterElfGlobalsName, IRB.getVoidTy(),
-                            IntptrTy, IntptrTy, IntptrTy));
-  AsanRegisterElfGlobals->setLinkage(Function::ExternalLinkage);
-
-  AsanUnregisterElfGlobals = checkSanitizerInterfaceFunction(
+                            IntptrTy, IntptrTy, IntptrTy);
+  AsanUnregisterElfGlobals =
       M.getOrInsertFunction(kAsanUnregisterElfGlobalsName, IRB.getVoidTy(),
-                            IntptrTy, IntptrTy, IntptrTy));
-  AsanUnregisterElfGlobals->setLinkage(Function::ExternalLinkage);
+                            IntptrTy, IntptrTy, IntptrTy);
 }
 
 // Put the metadata and the instrumented global in the same group. This ensures
@@ -2150,6 +2137,10 @@ bool AddressSanitizerModule::InstrumentGlobals(IRBuilder<> &IRB, Module &M, bool
     NewGlobal->copyAttributesFrom(G);
     NewGlobal->setComdat(G->getComdat());
     NewGlobal->setAlignment(MinRZ);
+    // Don't fold globals with redzones. ODR violation detector and redzone
+    // poisoning implicitly creates a dependence on the global's address, so it
+    // is no longer valid for it to be marked unnamed_addr.
+    NewGlobal->setUnnamedAddr(GlobalValue::UnnamedAddr::None);
 
     // Move null-terminated C strings to "__asan_cstring" section on Darwin.
     if (TargetTriple.isOSBinFormatMachO() && !G->hasSection() &&
@@ -2197,10 +2188,8 @@ bool AddressSanitizerModule::InstrumentGlobals(IRBuilder<> &IRB, Module &M, bool
           GlobalAlias::create(GlobalValue::PrivateLinkage, "", NewGlobal);
     }
 
-    // ODR check is not useful for the following, but we see false reports
-    // caused by linker optimizations.
-    if (NewGlobal->hasLocalLinkage() || NewGlobal->hasLinkOnceODRLinkage() ||
-        NewGlobal->hasWeakODRLinkage()) {
+    // ODR should not happen for local linkage.
+    if (NewGlobal->hasLocalLinkage()) {
       ODRIndicator = ConstantExpr::getIntToPtr(ConstantInt::get(IntptrTy, -1),
                                                IRB.getInt8PtrTy());
     } else if (UseOdrIndicator) {
@@ -2344,51 +2333,49 @@ void AddressSanitizer::initializeCallbacks(Module &M) {
         Args2.push_back(ExpType);
         Args1.push_back(ExpType);
       }
-      AsanErrorCallbackSized[AccessIsWrite][Exp] =
-          checkSanitizerInterfaceFunction(M.getOrInsertFunction(
-              kAsanReportErrorTemplate + ExpStr + TypeStr + "_n" + EndingStr,
-              FunctionType::get(IRB.getVoidTy(), Args2, false)));
+      AsanErrorCallbackSized[AccessIsWrite][Exp] = M.getOrInsertFunction(
+          kAsanReportErrorTemplate + ExpStr + TypeStr + "_n" + EndingStr,
+          FunctionType::get(IRB.getVoidTy(), Args2, false));
 
-      AsanMemoryAccessCallbackSized[AccessIsWrite][Exp] =
-          checkSanitizerInterfaceFunction(M.getOrInsertFunction(
-              ClMemoryAccessCallbackPrefix + ExpStr + TypeStr + "N" + EndingStr,
-              FunctionType::get(IRB.getVoidTy(), Args2, false)));
+      AsanMemoryAccessCallbackSized[AccessIsWrite][Exp] = M.getOrInsertFunction(
+          ClMemoryAccessCallbackPrefix + ExpStr + TypeStr + "N" + EndingStr,
+          FunctionType::get(IRB.getVoidTy(), Args2, false));
 
       for (size_t AccessSizeIndex = 0; AccessSizeIndex < kNumberOfAccessSizes;
            AccessSizeIndex++) {
         const std::string Suffix = TypeStr + itostr(1ULL << AccessSizeIndex);
         AsanErrorCallback[AccessIsWrite][Exp][AccessSizeIndex] =
-            checkSanitizerInterfaceFunction(M.getOrInsertFunction(
+            M.getOrInsertFunction(
                 kAsanReportErrorTemplate + ExpStr + Suffix + EndingStr,
-                FunctionType::get(IRB.getVoidTy(), Args1, false)));
+                FunctionType::get(IRB.getVoidTy(), Args1, false));
 
         AsanMemoryAccessCallback[AccessIsWrite][Exp][AccessSizeIndex] =
-            checkSanitizerInterfaceFunction(M.getOrInsertFunction(
+            M.getOrInsertFunction(
                 ClMemoryAccessCallbackPrefix + ExpStr + Suffix + EndingStr,
-                FunctionType::get(IRB.getVoidTy(), Args1, false)));
+                FunctionType::get(IRB.getVoidTy(), Args1, false));
       }
     }
   }
 
   const std::string MemIntrinCallbackPrefix =
       CompileKernel ? std::string("") : ClMemoryAccessCallbackPrefix;
-  AsanMemmove = checkSanitizerInterfaceFunction(M.getOrInsertFunction(
-      MemIntrinCallbackPrefix + "memmove", IRB.getInt8PtrTy(),
-      IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), IntptrTy));
-  AsanMemcpy = checkSanitizerInterfaceFunction(M.getOrInsertFunction(
-      MemIntrinCallbackPrefix + "memcpy", IRB.getInt8PtrTy(),
-      IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), IntptrTy));
-  AsanMemset = checkSanitizerInterfaceFunction(M.getOrInsertFunction(
-      MemIntrinCallbackPrefix + "memset", IRB.getInt8PtrTy(),
-      IRB.getInt8PtrTy(), IRB.getInt32Ty(), IntptrTy));
-
-  AsanHandleNoReturnFunc = checkSanitizerInterfaceFunction(
-      M.getOrInsertFunction(kAsanHandleNoReturnName, IRB.getVoidTy()));
-
-  AsanPtrCmpFunction = checkSanitizerInterfaceFunction(M.getOrInsertFunction(
-      kAsanPtrCmp, IRB.getVoidTy(), IntptrTy, IntptrTy));
-  AsanPtrSubFunction = checkSanitizerInterfaceFunction(M.getOrInsertFunction(
-      kAsanPtrSub, IRB.getVoidTy(), IntptrTy, IntptrTy));
+  AsanMemmove = M.getOrInsertFunction(MemIntrinCallbackPrefix + "memmove",
+                                      IRB.getInt8PtrTy(), IRB.getInt8PtrTy(),
+                                      IRB.getInt8PtrTy(), IntptrTy);
+  AsanMemcpy = M.getOrInsertFunction(MemIntrinCallbackPrefix + "memcpy",
+                                     IRB.getInt8PtrTy(), IRB.getInt8PtrTy(),
+                                     IRB.getInt8PtrTy(), IntptrTy);
+  AsanMemset = M.getOrInsertFunction(MemIntrinCallbackPrefix + "memset",
+                                     IRB.getInt8PtrTy(), IRB.getInt8PtrTy(),
+                                     IRB.getInt32Ty(), IntptrTy);
+
+  AsanHandleNoReturnFunc =
+      M.getOrInsertFunction(kAsanHandleNoReturnName, IRB.getVoidTy());
+
+  AsanPtrCmpFunction =
+      M.getOrInsertFunction(kAsanPtrCmp, IRB.getVoidTy(), IntptrTy, IntptrTy);
+  AsanPtrSubFunction =
+      M.getOrInsertFunction(kAsanPtrSub, IRB.getVoidTy(), IntptrTy, IntptrTy);
   // We insert an empty inline asm after __asan_report* to avoid callback merge.
   EmptyAsm = InlineAsm::get(FunctionType::get(IRB.getVoidTy(), false),
                             StringRef(""), StringRef(""),
@@ -2426,7 +2413,7 @@ bool AddressSanitizer::maybeInsertAsanInitAtFunctionEntry(Function &F) {
   // We cannot just ignore these methods, because they may call other
   // instrumented functions.
   if (F.getName().find(" load]") != std::string::npos) {
-    Function *AsanInitFunction =
+    FunctionCallee AsanInitFunction =
         declareSanitizerInitFunction(*F.getParent(), kAsanInitName, {});
     IRBuilder<> IRB(&F.front(), F.front().begin());
     IRB.CreateCall(AsanInitFunction, {});
@@ -2458,7 +2445,7 @@ void AddressSanitizer::maybeInsertDynamicShadowAtFunctionEntry(Function &F) {
   } else {
     Value *GlobalDynamicAddress = F.getParent()->getOrInsertGlobal(
         kAsanShadowMemoryDynamicAddress, IntptrTy);
-    LocalDynamicShadow = IRB.CreateLoad(GlobalDynamicAddress);
+    LocalDynamicShadow = IRB.CreateLoad(IntptrTy, GlobalDynamicAddress);
   }
 }
 
@@ -2567,7 +2554,8 @@ bool AddressSanitizer::runOnFunction(Function &F) {
         if (CS) {
           // A call inside BB.
           TempsToInstrument.clear();
-          if (CS.doesNotReturn()) NoReturnCalls.push_back(CS.getInstruction());
+          if (CS.doesNotReturn() && !CS->getMetadata("nosanitize"))
+            NoReturnCalls.push_back(CS.getInstruction());
         }
         if (CallInst *CI = dyn_cast<CallInst>(&Inst))
           maybeMarkSanitizerLibraryCallNoBuiltin(CI, TLI);
@@ -2604,7 +2592,7 @@ bool AddressSanitizer::runOnFunction(Function &F) {
   FunctionStackPoisoner FSP(F, *this);
   bool ChangedStack = FSP.runOnFunction();
 
-  // We must unpoison the stack before every NoReturn call (throw, _exit, etc).
+  // We must unpoison the stack before NoReturn calls (throw, _exit, etc).
   // See e.g. https://github.com/google/sanitizers/issues/37
   for (auto CI : NoReturnCalls) {
     IRBuilder<> IRB(CI);
@@ -2641,20 +2629,17 @@ void FunctionStackPoisoner::initializeCallbacks(Module &M) {
   IRBuilder<> IRB(*C);
   for (int i = 0; i <= kMaxAsanStackMallocSizeClass; i++) {
     std::string Suffix = itostr(i);
-    AsanStackMallocFunc[i] = checkSanitizerInterfaceFunction(
-        M.getOrInsertFunction(kAsanStackMallocNameTemplate + Suffix, IntptrTy,
-                              IntptrTy));
-    AsanStackFreeFunc[i] = checkSanitizerInterfaceFunction(
+    AsanStackMallocFunc[i] = M.getOrInsertFunction(
+        kAsanStackMallocNameTemplate + Suffix, IntptrTy, IntptrTy);
+    AsanStackFreeFunc[i] =
         M.getOrInsertFunction(kAsanStackFreeNameTemplate + Suffix,
-                              IRB.getVoidTy(), IntptrTy, IntptrTy));
+                              IRB.getVoidTy(), IntptrTy, IntptrTy);
   }
   if (ASan.UseAfterScope) {
-    AsanPoisonStackMemoryFunc = checkSanitizerInterfaceFunction(
-        M.getOrInsertFunction(kAsanPoisonStackMemoryName, IRB.getVoidTy(),
-                              IntptrTy, IntptrTy));
-    AsanUnpoisonStackMemoryFunc = checkSanitizerInterfaceFunction(
-        M.getOrInsertFunction(kAsanUnpoisonStackMemoryName, IRB.getVoidTy(),
-                              IntptrTy, IntptrTy));
+    AsanPoisonStackMemoryFunc = M.getOrInsertFunction(
+        kAsanPoisonStackMemoryName, IRB.getVoidTy(), IntptrTy, IntptrTy);
+    AsanUnpoisonStackMemoryFunc = M.getOrInsertFunction(
+        kAsanUnpoisonStackMemoryName, IRB.getVoidTy(), IntptrTy, IntptrTy);
   }
 
   for (size_t Val : {0x00, 0xf1, 0xf2, 0xf3, 0xf5, 0xf8}) {
@@ -2662,15 +2647,13 @@ void FunctionStackPoisoner::initializeCallbacks(Module &M) {
     Name << kAsanSetShadowPrefix;
     Name << std::setw(2) << std::setfill('0') << std::hex << Val;
     AsanSetShadowFunc[Val] =
-        checkSanitizerInterfaceFunction(M.getOrInsertFunction(
-            Name.str(), IRB.getVoidTy(), IntptrTy, IntptrTy));
+        M.getOrInsertFunction(Name.str(), IRB.getVoidTy(), IntptrTy, IntptrTy);
   }
 
-  AsanAllocaPoisonFunc = checkSanitizerInterfaceFunction(M.getOrInsertFunction(
-      kAsanAllocaPoison, IRB.getVoidTy(), IntptrTy, IntptrTy));
-  AsanAllocasUnpoisonFunc =
-      checkSanitizerInterfaceFunction(M.getOrInsertFunction(
-          kAsanAllocasUnpoison, IRB.getVoidTy(), IntptrTy, IntptrTy));
+  AsanAllocaPoisonFunc = M.getOrInsertFunction(
+      kAsanAllocaPoison, IRB.getVoidTy(), IntptrTy, IntptrTy);
+  AsanAllocasUnpoisonFunc = M.getOrInsertFunction(
+      kAsanAllocasUnpoison, IRB.getVoidTy(), IntptrTy, IntptrTy);
 }
 
 void FunctionStackPoisoner::copyToShadowInline(ArrayRef<uint8_t> ShadowMask,
@@ -2967,9 +2950,9 @@ void FunctionStackPoisoner::processStaticAllocas() {
     // void *LocalStackBase = (FakeStack) ? FakeStack : alloca(LocalStackSize);
     Constant *OptionDetectUseAfterReturn = F.getParent()->getOrInsertGlobal(
         kAsanOptionDetectUseAfterReturn, IRB.getInt32Ty());
-    Value *UseAfterReturnIsEnabled =
-        IRB.CreateICmpNE(IRB.CreateLoad(OptionDetectUseAfterReturn),
-                         Constant::getNullValue(IRB.getInt32Ty()));
+    Value *UseAfterReturnIsEnabled = IRB.CreateICmpNE(
+        IRB.CreateLoad(IRB.getInt32Ty(), OptionDetectUseAfterReturn),
+        Constant::getNullValue(IRB.getInt32Ty()));
     Instruction *Term =
         SplitBlockAndInsertIfThen(UseAfterReturnIsEnabled, InsBefore, false);
     IRBuilder<> IRBIf(Term);
@@ -3103,7 +3086,7 @@ void FunctionStackPoisoner::processStaticAllocas() {
             FakeStack,
             ConstantInt::get(IntptrTy, ClassSize - ASan.LongSize / 8));
         Value *SavedFlagPtr = IRBPoison.CreateLoad(
-            IRBPoison.CreateIntToPtr(SavedFlagPtrPtr, IntptrPtrTy));
+            IntptrTy, IRBPoison.CreateIntToPtr(SavedFlagPtrPtr, IntptrPtrTy));
         IRBPoison.CreateStore(
             Constant::getNullValue(IRBPoison.getInt8Ty()),
             IRBPoison.CreateIntToPtr(SavedFlagPtr, IRBPoison.getInt8PtrTy()));
diff --git a/lib/Transforms/Instrumentation/BoundsChecking.cpp b/lib/Transforms/Instrumentation/BoundsChecking.cpp
index a0c78e0468c6163fe622c41d3dae2e4aff637ec8..4dc9b611c15675d56af3fd33d7aa55ed646676ea 100644
--- a/lib/Transforms/Instrumentation/BoundsChecking.cpp
+++ b/lib/Transforms/Instrumentation/BoundsChecking.cpp
@@ -1,9 +1,8 @@
 //===- BoundsChecking.cpp - Instrumentation for run-time bounds checking --===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -143,8 +142,9 @@ static void insertBoundsCheck(Value *Or, BuilderTy IRB, GetTrapBBT GetTrapBB) {
 static bool addBoundsChecking(Function &F, TargetLibraryInfo &TLI,
                               ScalarEvolution &SE) {
   const DataLayout &DL = F.getParent()->getDataLayout();
-  ObjectSizeOffsetEvaluator ObjSizeEval(DL, &TLI, F.getContext(),
-                                           /*RoundToAlign=*/true);
+  ObjectSizeOpts EvalOpts;
+  EvalOpts.RoundToAlign = true;
+  ObjectSizeOffsetEvaluator ObjSizeEval(DL, &TLI, F.getContext(), EvalOpts);
 
   // check HANDLE_MEMORY_INST in include/llvm/Instruction.def for memory
   // touching instructions
diff --git a/lib/Transforms/Instrumentation/CFGMST.h b/lib/Transforms/Instrumentation/CFGMST.h
index e178ef386e68fab63c795da286491c0a8f1c1c1c..b4164972be40dc91fac5555d94302f580473817a 100644
--- a/lib/Transforms/Instrumentation/CFGMST.h
+++ b/lib/Transforms/Instrumentation/CFGMST.h
@@ -1,9 +1,8 @@
 //===-- CFGMST.h - Minimum Spanning Tree for CFG ----------------*- C++ -*-===//
 //
-//                      The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Transforms/Instrumentation/CGProfile.cpp b/lib/Transforms/Instrumentation/CGProfile.cpp
index cdcd017269061f242a495742dec7f9aebdb1b664..358abab3cceb76fa7f75742e6ecafb07058d1a36 100644
--- a/lib/Transforms/Instrumentation/CGProfile.cpp
+++ b/lib/Transforms/Instrumentation/CGProfile.cpp
@@ -1,9 +1,8 @@
 //===-- CGProfile.cpp -----------------------------------------------------===//
 //
-//                      The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Transforms/Instrumentation/ControlHeightReduction.cpp b/lib/Transforms/Instrumentation/ControlHeightReduction.cpp
index 1ada0b7130927aeed0efd624dde2d837de362e09..c24826774708b0aeba486cfafcf53c8c617e0f4f 100644
--- a/lib/Transforms/Instrumentation/ControlHeightReduction.cpp
+++ b/lib/Transforms/Instrumentation/ControlHeightReduction.cpp
@@ -1,9 +1,8 @@
 //===-- ControlHeightReduction.cpp - Control Height Reduction -------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp b/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp
index 4c3c6c9added057dba2acd6da91d5f2c78071738..3f166b1134155b1f7cd33f4b2dc905c4fce1d287 100644
--- a/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp
+++ b/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp
@@ -1,9 +1,8 @@
 //===- DataFlowSanitizer.cpp - dynamic data flow analysis -----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -333,6 +332,8 @@ class DataFlowSanitizer : public ModulePass {
   Constant *RetvalTLS;
   void *(*GetArgTLSPtr)();
   void *(*GetRetvalTLSPtr)();
+  FunctionType *GetArgTLSTy;
+  FunctionType *GetRetvalTLSTy;
   Constant *GetArgTLS;
   Constant *GetRetvalTLS;
   Constant *ExternalShadowMask;
@@ -342,13 +343,13 @@ class DataFlowSanitizer : public ModulePass {
   FunctionType *DFSanSetLabelFnTy;
   FunctionType *DFSanNonzeroLabelFnTy;
   FunctionType *DFSanVarargWrapperFnTy;
-  Constant *DFSanUnionFn;
-  Constant *DFSanCheckedUnionFn;
-  Constant *DFSanUnionLoadFn;
-  Constant *DFSanUnimplementedFn;
-  Constant *DFSanSetLabelFn;
-  Constant *DFSanNonzeroLabelFn;
-  Constant *DFSanVarargWrapperFn;
+  FunctionCallee DFSanUnionFn;
+  FunctionCallee DFSanCheckedUnionFn;
+  FunctionCallee DFSanUnionLoadFn;
+  FunctionCallee DFSanUnimplementedFn;
+  FunctionCallee DFSanSetLabelFn;
+  FunctionCallee DFSanNonzeroLabelFn;
+  FunctionCallee DFSanVarargWrapperFn;
   MDNode *ColdCallWeights;
   DFSanABIList ABIList;
   DenseMap<Value *, Function *> UnwrappedFnMap;
@@ -581,17 +582,17 @@ bool DataFlowSanitizer::doInitialization(Module &M) {
   if (GetArgTLSPtr) {
     Type *ArgTLSTy = ArrayType::get(ShadowTy, 64);
     ArgTLS = nullptr;
+    GetArgTLSTy = FunctionType::get(PointerType::getUnqual(ArgTLSTy), false);
     GetArgTLS = ConstantExpr::getIntToPtr(
         ConstantInt::get(IntptrTy, uintptr_t(GetArgTLSPtr)),
-        PointerType::getUnqual(
-            FunctionType::get(PointerType::getUnqual(ArgTLSTy), false)));
+        PointerType::getUnqual(GetArgTLSTy));
   }
   if (GetRetvalTLSPtr) {
     RetvalTLS = nullptr;
+    GetRetvalTLSTy = FunctionType::get(PointerType::getUnqual(ShadowTy), false);
     GetRetvalTLS = ConstantExpr::getIntToPtr(
         ConstantInt::get(IntptrTy, uintptr_t(GetRetvalTLSPtr)),
-        PointerType::getUnqual(
-            FunctionType::get(PointerType::getUnqual(ShadowTy), false)));
+        PointerType::getUnqual(GetRetvalTLSTy));
   }
 
   ColdCallWeights = MDBuilder(*Ctx).createBranchWeights(1, 1000);
@@ -678,8 +679,8 @@ DataFlowSanitizer::buildWrapperFunction(Function *F, StringRef NewFName,
 Constant *DataFlowSanitizer::getOrBuildTrampolineFunction(FunctionType *FT,
                                                           StringRef FName) {
   FunctionType *FTT = getTrampolineFunctionType(FT);
-  Constant *C = Mod->getOrInsertFunction(FName, FTT);
-  Function *F = dyn_cast<Function>(C);
+  FunctionCallee C = Mod->getOrInsertFunction(FName, FTT);
+  Function *F = dyn_cast<Function>(C.getCallee());
   if (F && F->isDeclaration()) {
     F->setLinkage(GlobalValue::LinkOnceODRLinkage);
     BasicBlock *BB = BasicBlock::Create(*Ctx, "entry", F);
@@ -687,7 +688,7 @@ Constant *DataFlowSanitizer::getOrBuildTrampolineFunction(FunctionType *FT,
     Function::arg_iterator AI = F->arg_begin(); ++AI;
     for (unsigned N = FT->getNumParams(); N != 0; ++AI, --N)
       Args.push_back(&*AI);
-    CallInst *CI = CallInst::Create(&*F->arg_begin(), Args, "", BB);
+    CallInst *CI = CallInst::Create(FT, &*F->arg_begin(), Args, "", BB);
     ReturnInst *RI;
     if (FT->getReturnType()->isVoidTy())
       RI = ReturnInst::Create(*Ctx, BB);
@@ -704,7 +705,7 @@ Constant *DataFlowSanitizer::getOrBuildTrampolineFunction(FunctionType *FT,
                     &*std::prev(F->arg_end()), RI);
   }
 
-  return C;
+  return cast<Constant>(C.getCallee());
 }
 
 bool DataFlowSanitizer::runOnModule(Module &M) {
@@ -726,35 +727,51 @@ bool DataFlowSanitizer::runOnModule(Module &M) {
   ExternalShadowMask =
       Mod->getOrInsertGlobal(kDFSanExternShadowPtrMask, IntptrTy);
 
-  DFSanUnionFn = Mod->getOrInsertFunction("__dfsan_union", DFSanUnionFnTy);
-  if (Function *F = dyn_cast<Function>(DFSanUnionFn)) {
-    F->addAttribute(AttributeList::FunctionIndex, Attribute::NoUnwind);
-    F->addAttribute(AttributeList::FunctionIndex, Attribute::ReadNone);
-    F->addAttribute(AttributeList::ReturnIndex, Attribute::ZExt);
-    F->addParamAttr(0, Attribute::ZExt);
-    F->addParamAttr(1, Attribute::ZExt);
+  {
+    AttributeList AL;
+    AL = AL.addAttribute(M.getContext(), AttributeList::FunctionIndex,
+                         Attribute::NoUnwind);
+    AL = AL.addAttribute(M.getContext(), AttributeList::FunctionIndex,
+                         Attribute::ReadNone);
+    AL = AL.addAttribute(M.getContext(), AttributeList::ReturnIndex,
+                         Attribute::ZExt);
+    AL = AL.addParamAttribute(M.getContext(), 0, Attribute::ZExt);
+    AL = AL.addParamAttribute(M.getContext(), 1, Attribute::ZExt);
+    DFSanUnionFn =
+        Mod->getOrInsertFunction("__dfsan_union", DFSanUnionFnTy, AL);
   }
-  DFSanCheckedUnionFn = Mod->getOrInsertFunction("dfsan_union", DFSanUnionFnTy);
-  if (Function *F = dyn_cast<Function>(DFSanCheckedUnionFn)) {
-    F->addAttribute(AttributeList::FunctionIndex, Attribute::NoUnwind);
-    F->addAttribute(AttributeList::FunctionIndex, Attribute::ReadNone);
-    F->addAttribute(AttributeList::ReturnIndex, Attribute::ZExt);
-    F->addParamAttr(0, Attribute::ZExt);
-    F->addParamAttr(1, Attribute::ZExt);
+
+  {
+    AttributeList AL;
+    AL = AL.addAttribute(M.getContext(), AttributeList::FunctionIndex,
+                         Attribute::NoUnwind);
+    AL = AL.addAttribute(M.getContext(), AttributeList::FunctionIndex,
+                         Attribute::ReadNone);
+    AL = AL.addAttribute(M.getContext(), AttributeList::ReturnIndex,
+                         Attribute::ZExt);
+    AL = AL.addParamAttribute(M.getContext(), 0, Attribute::ZExt);
+    AL = AL.addParamAttribute(M.getContext(), 1, Attribute::ZExt);
+    DFSanCheckedUnionFn =
+        Mod->getOrInsertFunction("dfsan_union", DFSanUnionFnTy, AL);
   }
-  DFSanUnionLoadFn =
-      Mod->getOrInsertFunction("__dfsan_union_load", DFSanUnionLoadFnTy);
-  if (Function *F = dyn_cast<Function>(DFSanUnionLoadFn)) {
-    F->addAttribute(AttributeList::FunctionIndex, Attribute::NoUnwind);
-    F->addAttribute(AttributeList::FunctionIndex, Attribute::ReadOnly);
-    F->addAttribute(AttributeList::ReturnIndex, Attribute::ZExt);
+  {
+    AttributeList AL;
+    AL = AL.addAttribute(M.getContext(), AttributeList::FunctionIndex,
+                         Attribute::NoUnwind);
+    AL = AL.addAttribute(M.getContext(), AttributeList::FunctionIndex,
+                         Attribute::ReadOnly);
+    AL = AL.addAttribute(M.getContext(), AttributeList::ReturnIndex,
+                         Attribute::ZExt);
+    DFSanUnionLoadFn =
+        Mod->getOrInsertFunction("__dfsan_union_load", DFSanUnionLoadFnTy, AL);
   }
   DFSanUnimplementedFn =
       Mod->getOrInsertFunction("__dfsan_unimplemented", DFSanUnimplementedFnTy);
-  DFSanSetLabelFn =
-      Mod->getOrInsertFunction("__dfsan_set_label", DFSanSetLabelFnTy);
-  if (Function *F = dyn_cast<Function>(DFSanSetLabelFn)) {
-    F->addParamAttr(0, Attribute::ZExt);
+  {
+    AttributeList AL;
+    AL = AL.addParamAttribute(M.getContext(), 0, Attribute::ZExt);
+    DFSanSetLabelFn =
+        Mod->getOrInsertFunction("__dfsan_set_label", DFSanSetLabelFnTy, AL);
   }
   DFSanNonzeroLabelFn =
       Mod->getOrInsertFunction("__dfsan_nonzero_label", DFSanNonzeroLabelFnTy);
@@ -765,13 +782,13 @@ bool DataFlowSanitizer::runOnModule(Module &M) {
   SmallPtrSet<Function *, 2> FnsWithNativeABI;
   for (Function &i : M) {
     if (!i.isIntrinsic() &&
-        &i != DFSanUnionFn &&
-        &i != DFSanCheckedUnionFn &&
-        &i != DFSanUnionLoadFn &&
-        &i != DFSanUnimplementedFn &&
-        &i != DFSanSetLabelFn &&
-        &i != DFSanNonzeroLabelFn &&
-        &i != DFSanVarargWrapperFn)
+        &i != DFSanUnionFn.getCallee()->stripPointerCasts() &&
+        &i != DFSanCheckedUnionFn.getCallee()->stripPointerCasts() &&
+        &i != DFSanUnionLoadFn.getCallee()->stripPointerCasts() &&
+        &i != DFSanUnimplementedFn.getCallee()->stripPointerCasts() &&
+        &i != DFSanSetLabelFn.getCallee()->stripPointerCasts() &&
+        &i != DFSanNonzeroLabelFn.getCallee()->stripPointerCasts() &&
+        &i != DFSanVarargWrapperFn.getCallee()->stripPointerCasts())
       FnsToInstrument.push_back(&i);
   }
 
@@ -982,7 +999,7 @@ Value *DFSanFunction::getArgTLSPtr() {
     return ArgTLSPtr = DFS.ArgTLS;
 
   IRBuilder<> IRB(&F->getEntryBlock().front());
-  return ArgTLSPtr = IRB.CreateCall(DFS.GetArgTLS, {});
+  return ArgTLSPtr = IRB.CreateCall(DFS.GetArgTLSTy, DFS.GetArgTLS, {});
 }
 
 Value *DFSanFunction::getRetvalTLS() {
@@ -992,12 +1009,14 @@ Value *DFSanFunction::getRetvalTLS() {
     return RetvalTLSPtr = DFS.RetvalTLS;
 
   IRBuilder<> IRB(&F->getEntryBlock().front());
-  return RetvalTLSPtr = IRB.CreateCall(DFS.GetRetvalTLS, {});
+  return RetvalTLSPtr =
+             IRB.CreateCall(DFS.GetRetvalTLSTy, DFS.GetRetvalTLS, {});
 }
 
 Value *DFSanFunction::getArgTLS(unsigned Idx, Instruction *Pos) {
   IRBuilder<> IRB(Pos);
-  return IRB.CreateConstGEP2_64(getArgTLSPtr(), 0, Idx);
+  return IRB.CreateConstGEP2_64(ArrayType::get(DFS.ShadowTy, 64),
+                                getArgTLSPtr(), 0, Idx);
 }
 
 Value *DFSanFunction::getShadow(Value *V) {
@@ -1015,7 +1034,8 @@ Value *DFSanFunction::getShadow(Value *V) {
             DFS.ArgTLS ? &*F->getEntryBlock().begin()
                        : cast<Instruction>(ArgTLSPtr)->getNextNode();
         IRBuilder<> IRB(ArgTLSPos);
-        Shadow = IRB.CreateLoad(getArgTLS(A->getArgNo(), ArgTLSPos));
+        Shadow =
+            IRB.CreateLoad(DFS.ShadowTy, getArgTLS(A->getArgNo(), ArgTLSPos));
         break;
       }
       case DataFlowSanitizer::IA_Args: {
@@ -1165,7 +1185,7 @@ Value *DFSanFunction::loadShadow(Value *Addr, uint64_t Size, uint64_t Align,
     const auto i = AllocaShadowMap.find(AI);
     if (i != AllocaShadowMap.end()) {
       IRBuilder<> IRB(Pos);
-      return IRB.CreateLoad(i->second);
+      return IRB.CreateLoad(DFS.ShadowTy, i->second);
     }
   }
 
@@ -1190,7 +1210,7 @@ Value *DFSanFunction::loadShadow(Value *Addr, uint64_t Size, uint64_t Align,
   case 0:
     return DFS.ZeroShadow;
   case 1: {
-    LoadInst *LI = new LoadInst(ShadowAddr, "", Pos);
+    LoadInst *LI = new LoadInst(DFS.ShadowTy, ShadowAddr, "", Pos);
     LI->setAlignment(ShadowAlign);
     return LI;
   }
@@ -1198,8 +1218,9 @@ Value *DFSanFunction::loadShadow(Value *Addr, uint64_t Size, uint64_t Align,
     IRBuilder<> IRB(Pos);
     Value *ShadowAddr1 = IRB.CreateGEP(DFS.ShadowTy, ShadowAddr,
                                        ConstantInt::get(DFS.IntptrTy, 1));
-    return combineShadows(IRB.CreateAlignedLoad(ShadowAddr, ShadowAlign),
-                          IRB.CreateAlignedLoad(ShadowAddr1, ShadowAlign), Pos);
+    return combineShadows(
+        IRB.CreateAlignedLoad(DFS.ShadowTy, ShadowAddr, ShadowAlign),
+        IRB.CreateAlignedLoad(DFS.ShadowTy, ShadowAddr1, ShadowAlign), Pos);
   }
   }
   if (!AvoidNewBlocks && Size % (64 / DFS.ShadowWidth) == 0) {
@@ -1218,7 +1239,8 @@ Value *DFSanFunction::loadShadow(Value *Addr, uint64_t Size, uint64_t Align,
     IRBuilder<> IRB(Pos);
     Value *WideAddr =
         IRB.CreateBitCast(ShadowAddr, Type::getInt64PtrTy(*DFS.Ctx));
-    Value *WideShadow = IRB.CreateAlignedLoad(WideAddr, ShadowAlign);
+    Value *WideShadow =
+        IRB.CreateAlignedLoad(IRB.getInt64Ty(), WideAddr, ShadowAlign);
     Value *TruncShadow = IRB.CreateTrunc(WideShadow, DFS.ShadowTy);
     Value *ShlShadow = IRB.CreateShl(WideShadow, DFS.ShadowWidth);
     Value *ShrShadow = IRB.CreateLShr(WideShadow, 64 - DFS.ShadowWidth);
@@ -1251,7 +1273,8 @@ Value *DFSanFunction::loadShadow(Value *Addr, uint64_t Size, uint64_t Align,
       IRBuilder<> NextIRB(NextBB);
       WideAddr = NextIRB.CreateGEP(Type::getInt64Ty(*DFS.Ctx), WideAddr,
                                    ConstantInt::get(DFS.IntptrTy, 1));
-      Value *NextWideShadow = NextIRB.CreateAlignedLoad(WideAddr, ShadowAlign);
+      Value *NextWideShadow = NextIRB.CreateAlignedLoad(NextIRB.getInt64Ty(),
+                                                        WideAddr, ShadowAlign);
       ShadowsEq = NextIRB.CreateICmpEQ(WideShadow, NextWideShadow);
       LastBr->setSuccessor(0, NextBB);
       LastBr = NextIRB.CreateCondBr(ShadowsEq, FallbackBB, FallbackBB);
@@ -1470,7 +1493,7 @@ void DFSanVisitor::visitMemTransferInst(MemTransferInst &I) {
   DestShadow = IRB.CreateBitCast(DestShadow, Int8Ptr);
   SrcShadow = IRB.CreateBitCast(SrcShadow, Int8Ptr);
   auto *MTI = cast<MemTransferInst>(
-      IRB.CreateCall(I.getCalledValue(),
+      IRB.CreateCall(I.getFunctionType(), I.getCalledValue(),
                      {DestShadow, SrcShadow, LenShadow, I.getVolatileCst()}));
   if (ClPreserveAlignment) {
     MTI->setDestAlignment(I.getDestAlignment() * (DFSF.DFS.ShadowWidth / 8));
@@ -1513,7 +1536,7 @@ void DFSanVisitor::visitCallSite(CallSite CS) {
 
   // Calls to this function are synthesized in wrappers, and we shouldn't
   // instrument them.
-  if (F == DFSF.DFS.DFSanVarargWrapperFn)
+  if (F == DFSF.DFS.DFSanVarargWrapperFn.getCallee()->stripPointerCasts())
     return;
 
   IRBuilder<> IRB(CS.getInstruction());
@@ -1546,9 +1569,9 @@ void DFSanVisitor::visitCallSite(CallSite CS) {
         TransformedFunction CustomFn = DFSF.DFS.getCustomFunctionType(FT);
         std::string CustomFName = "__dfsw_";
         CustomFName += F->getName();
-        Constant *CustomF = DFSF.DFS.Mod->getOrInsertFunction(
+        FunctionCallee CustomF = DFSF.DFS.Mod->getOrInsertFunction(
             CustomFName, CustomFn.TransformedType);
-        if (Function *CustomFn = dyn_cast<Function>(CustomF)) {
+        if (Function *CustomFn = dyn_cast<Function>(CustomF.getCallee())) {
           CustomFn->copyAttributesFrom(F);
 
           // Custom functions returning non-void will write to the return label.
@@ -1628,7 +1651,8 @@ void DFSanVisitor::visitCallSite(CallSite CS) {
         }
 
         if (!FT->getReturnType()->isVoidTy()) {
-          LoadInst *LabelLoad = IRB.CreateLoad(DFSF.LabelReturnAlloca);
+          LoadInst *LabelLoad =
+              IRB.CreateLoad(DFSF.DFS.ShadowTy, DFSF.LabelReturnAlloca);
           DFSF.setShadow(CustomCI, LabelLoad);
         }
 
@@ -1666,7 +1690,7 @@ void DFSanVisitor::visitCallSite(CallSite CS) {
 
     if (DFSF.DFS.getInstrumentedABI() == DataFlowSanitizer::IA_TLS) {
       IRBuilder<> NextIRB(Next);
-      LoadInst *LI = NextIRB.CreateLoad(DFSF.getRetvalTLS());
+      LoadInst *LI = NextIRB.CreateLoad(DFSF.DFS.ShadowTy, DFSF.getRetvalTLS());
       DFSF.SkipInsts.insert(LI);
       DFSF.setShadow(CS.getInstruction(), LI);
       DFSF.NonZeroChecks.push_back(LI);
@@ -1706,10 +1730,10 @@ void DFSanVisitor::visitCallSite(CallSite CS) {
 
     CallSite NewCS;
     if (InvokeInst *II = dyn_cast<InvokeInst>(CS.getInstruction())) {
-      NewCS = IRB.CreateInvoke(Func, II->getNormalDest(), II->getUnwindDest(),
-                               Args);
+      NewCS = IRB.CreateInvoke(NewFT, Func, II->getNormalDest(),
+                               II->getUnwindDest(), Args);
     } else {
-      NewCS = IRB.CreateCall(Func, Args);
+      NewCS = IRB.CreateCall(NewFT, Func, Args);
     }
     NewCS.setCallingConv(CS.getCallingConv());
     NewCS.setAttributes(CS.getAttributes().removeAttributes(
diff --git a/lib/Transforms/Instrumentation/EfficiencySanitizer.cpp b/lib/Transforms/Instrumentation/EfficiencySanitizer.cpp
index db438e78ded95b6db798bb2181801dc133f6fd53..26c4cbcb63fb531d6ca85a67f69a1ac7c89af84b 100644
--- a/lib/Transforms/Instrumentation/EfficiencySanitizer.cpp
+++ b/lib/Transforms/Instrumentation/EfficiencySanitizer.cpp
@@ -1,9 +1,8 @@
 //===-- EfficiencySanitizer.cpp - performance tuner -----------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -203,13 +202,13 @@ private:
   // Our slowpath involves callouts to the runtime library.
   // Access sizes are powers of two: 1, 2, 4, 8, 16.
   static const size_t NumberOfAccessSizes = 5;
-  Function *EsanAlignedLoad[NumberOfAccessSizes];
-  Function *EsanAlignedStore[NumberOfAccessSizes];
-  Function *EsanUnalignedLoad[NumberOfAccessSizes];
-  Function *EsanUnalignedStore[NumberOfAccessSizes];
+  FunctionCallee EsanAlignedLoad[NumberOfAccessSizes];
+  FunctionCallee EsanAlignedStore[NumberOfAccessSizes];
+  FunctionCallee EsanUnalignedLoad[NumberOfAccessSizes];
+  FunctionCallee EsanUnalignedStore[NumberOfAccessSizes];
   // For irregular sizes of any alignment:
-  Function *EsanUnalignedLoadN, *EsanUnalignedStoreN;
-  Function *MemmoveFn, *MemcpyFn, *MemsetFn;
+  FunctionCallee EsanUnalignedLoadN, EsanUnalignedStoreN;
+  FunctionCallee MemmoveFn, MemcpyFn, MemsetFn;
   Function *EsanCtorFunction;
   Function *EsanDtorFunction;
   // Remember the counter variable for each struct type to avoid
@@ -250,37 +249,31 @@ void EfficiencySanitizer::initializeCallbacks(Module &M) {
     // We'll inline the most common (i.e., aligned and frequent sizes)
     // load + store instrumentation: these callouts are for the slowpath.
     SmallString<32> AlignedLoadName("__esan_aligned_load" + ByteSizeStr);
-    EsanAlignedLoad[Idx] =
-        checkSanitizerInterfaceFunction(M.getOrInsertFunction(
-            AlignedLoadName, IRB.getVoidTy(), IRB.getInt8PtrTy()));
+    EsanAlignedLoad[Idx] = M.getOrInsertFunction(
+        AlignedLoadName, IRB.getVoidTy(), IRB.getInt8PtrTy());
     SmallString<32> AlignedStoreName("__esan_aligned_store" + ByteSizeStr);
-    EsanAlignedStore[Idx] =
-        checkSanitizerInterfaceFunction(M.getOrInsertFunction(
-            AlignedStoreName, IRB.getVoidTy(), IRB.getInt8PtrTy()));
+    EsanAlignedStore[Idx] = M.getOrInsertFunction(
+        AlignedStoreName, IRB.getVoidTy(), IRB.getInt8PtrTy());
     SmallString<32> UnalignedLoadName("__esan_unaligned_load" + ByteSizeStr);
-    EsanUnalignedLoad[Idx] =
-        checkSanitizerInterfaceFunction(M.getOrInsertFunction(
-            UnalignedLoadName, IRB.getVoidTy(), IRB.getInt8PtrTy()));
+    EsanUnalignedLoad[Idx] = M.getOrInsertFunction(
+        UnalignedLoadName, IRB.getVoidTy(), IRB.getInt8PtrTy());
     SmallString<32> UnalignedStoreName("__esan_unaligned_store" + ByteSizeStr);
-    EsanUnalignedStore[Idx] =
-        checkSanitizerInterfaceFunction(M.getOrInsertFunction(
-            UnalignedStoreName, IRB.getVoidTy(), IRB.getInt8PtrTy()));
+    EsanUnalignedStore[Idx] = M.getOrInsertFunction(
+        UnalignedStoreName, IRB.getVoidTy(), IRB.getInt8PtrTy());
   }
-  EsanUnalignedLoadN = checkSanitizerInterfaceFunction(
-      M.getOrInsertFunction("__esan_unaligned_loadN", IRB.getVoidTy(),
-                            IRB.getInt8PtrTy(), IntptrTy));
-  EsanUnalignedStoreN = checkSanitizerInterfaceFunction(
-      M.getOrInsertFunction("__esan_unaligned_storeN", IRB.getVoidTy(),
-                            IRB.getInt8PtrTy(), IntptrTy));
-  MemmoveFn = checkSanitizerInterfaceFunction(
+  EsanUnalignedLoadN = M.getOrInsertFunction(
+      "__esan_unaligned_loadN", IRB.getVoidTy(), IRB.getInt8PtrTy(), IntptrTy);
+  EsanUnalignedStoreN = M.getOrInsertFunction(
+      "__esan_unaligned_storeN", IRB.getVoidTy(), IRB.getInt8PtrTy(), IntptrTy);
+  MemmoveFn =
       M.getOrInsertFunction("memmove", IRB.getInt8PtrTy(), IRB.getInt8PtrTy(),
-                            IRB.getInt8PtrTy(), IntptrTy));
-  MemcpyFn = checkSanitizerInterfaceFunction(
+                            IRB.getInt8PtrTy(), IntptrTy);
+  MemcpyFn =
       M.getOrInsertFunction("memcpy", IRB.getInt8PtrTy(), IRB.getInt8PtrTy(),
-                            IRB.getInt8PtrTy(), IntptrTy));
-  MemsetFn = checkSanitizerInterfaceFunction(
+                            IRB.getInt8PtrTy(), IntptrTy);
+  MemsetFn =
       M.getOrInsertFunction("memset", IRB.getInt8PtrTy(), IRB.getInt8PtrTy(),
-                            IRB.getInt32Ty(), IntptrTy));
+                            IRB.getInt32Ty(), IntptrTy);
 }
 
 bool EfficiencySanitizer::shouldIgnoreStructType(StructType *StructTy) {
@@ -511,10 +504,8 @@ void EfficiencySanitizer::createDestructor(Module &M, Constant *ToolInfoArg) {
                                       EsanModuleDtorName, &M);
   ReturnInst::Create(*Ctx, BasicBlock::Create(*Ctx, "", EsanDtorFunction));
   IRBuilder<> IRB_Dtor(EsanDtorFunction->getEntryBlock().getTerminator());
-  Function *EsanExit = checkSanitizerInterfaceFunction(
-      M.getOrInsertFunction(EsanExitName, IRB_Dtor.getVoidTy(),
-                            Int8PtrTy));
-  EsanExit->setLinkage(Function::ExternalLinkage);
+  FunctionCallee EsanExit =
+      M.getOrInsertFunction(EsanExitName, IRB_Dtor.getVoidTy(), Int8PtrTy);
   IRB_Dtor.CreateCall(EsanExit, {ToolInfoArg});
   appendToGlobalDtors(M, EsanDtorFunction, EsanCtorAndDtorPriority);
 }
@@ -670,7 +661,7 @@ bool EfficiencySanitizer::instrumentLoadOrStore(Instruction *I,
 
   Type *OrigTy = cast<PointerType>(Addr->getType())->getElementType();
   const uint32_t TypeSizeBytes = DL.getTypeStoreSizeInBits(OrigTy) / 8;
-  Value *OnAccessFunc = nullptr;
+  FunctionCallee OnAccessFunc = nullptr;
 
   // Convert 0 to the default alignment.
   if (Alignment == 0)
@@ -799,7 +790,7 @@ bool EfficiencySanitizer::insertCounterUpdate(Instruction *I,
     ConstantExpr::getGetElementPtr(
         ArrayType::get(IRB.getInt64Ty(), getStructCounterSize(StructTy)),
         CounterArray, Indices);
-  Value *Load = IRB.CreateLoad(Counter);
+  Value *Load = IRB.CreateLoad(IRB.getInt64Ty(), Counter);
   IRB.CreateStore(IRB.CreateAdd(Load, ConstantInt::get(IRB.getInt64Ty(), 1)),
                   Counter);
   return true;
@@ -884,7 +875,8 @@ bool EfficiencySanitizer::instrumentFastpathWorkingSet(
   // memory access, if they are not already set.
   Value *ValueMask = ConstantInt::get(ShadowTy, 0x81); // 10000001B
 
-  Value *OldValue = IRB.CreateLoad(IRB.CreateIntToPtr(ShadowPtr, ShadowPtrTy));
+  Value *OldValue =
+      IRB.CreateLoad(ShadowTy, IRB.CreateIntToPtr(ShadowPtr, ShadowPtrTy));
   // The AND and CMP will be turned into a TEST instruction by the compiler.
   Value *Cmp = IRB.CreateICmpNE(IRB.CreateAnd(OldValue, ValueMask), ValueMask);
   Instruction *CmpTerm = SplitBlockAndInsertIfThen(Cmp, I, false);
diff --git a/lib/Transforms/Instrumentation/GCOVProfiling.cpp b/lib/Transforms/Instrumentation/GCOVProfiling.cpp
index 829123911c61b4f6c54bc0a832d72b9bccf36925..59950ffc4e9a60f87e38626a149fc1db515fe5bf 100644
--- a/lib/Transforms/Instrumentation/GCOVProfiling.cpp
+++ b/lib/Transforms/Instrumentation/GCOVProfiling.cpp
@@ -1,9 +1,8 @@
 //===- GCOVProfiling.cpp - Insert edge counters for gcov profiling --------===//
 //
-//                      The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -103,11 +102,11 @@ private:
                                       std::vector<Regex> &Regexes);
 
   // Get pointers to the functions in the runtime library.
-  Constant *getStartFileFunc();
-  Constant *getEmitFunctionFunc();
-  Constant *getEmitArcsFunc();
-  Constant *getSummaryInfoFunc();
-  Constant *getEndFileFunc();
+  FunctionCallee getStartFileFunc();
+  FunctionCallee getEmitFunctionFunc();
+  FunctionCallee getEmitArcsFunc();
+  FunctionCallee getSummaryInfoFunc();
+  FunctionCallee getEndFileFunc();
 
   // Add the function to write out all our counters to the global destructor
   // list.
@@ -185,7 +184,7 @@ static StringRef getFunctionName(const DISubprogram *SP) {
 /// Prefer relative paths in the coverage notes. Clang also may split
 /// up absolute paths into a directory and filename component. When
 /// the relative path doesn't exist, reconstruct the absolute path.
-SmallString<128> getFilename(const DISubprogram *SP) {
+static SmallString<128> getFilename(const DISubprogram *SP) {
   SmallString<128> Path;
   StringRef RelPath = SP->getFilename();
   if (sys::fs::exists(RelPath))
@@ -648,7 +647,7 @@ void GCOVProfiler::AddFlushBeforeForkAndExec() {
   for (auto I : ForkAndExecs) {
     IRBuilder<> Builder(I);
     FunctionType *FTy = FunctionType::get(Builder.getVoidTy(), {}, false);
-    Constant *GCOVFlush = M->getOrInsertFunction("__gcov_flush", FTy);
+    FunctionCallee GCOVFlush = M->getOrInsertFunction("__gcov_flush", FTy);
     Builder.CreateCall(GCOVFlush);
     I->getParent()->splitBasicBlock(I);
   }
@@ -811,14 +810,14 @@ bool GCOVProfiler::emitProfileArcs() {
             auto It = EdgeToCounter.find({Pred, &BB});
             assert(It != EdgeToCounter.end());
             const unsigned Edge = It->second;
-            Value *EdgeCounter =
-                BuilderForPhi.CreateConstInBoundsGEP2_64(Counters, 0, Edge);
+            Value *EdgeCounter = BuilderForPhi.CreateConstInBoundsGEP2_64(
+                Counters->getValueType(), Counters, 0, Edge);
             Phi->addIncoming(EdgeCounter, Pred);
           }
 
           // Skip phis, landingpads.
           IRBuilder<> Builder(&*BB.getFirstInsertionPt());
-          Value *Count = Builder.CreateLoad(Phi);
+          Value *Count = Builder.CreateLoad(Builder.getInt64Ty(), Phi);
           Count = Builder.CreateAdd(Count, Builder.getInt64(1));
           Builder.CreateStore(Count, Phi);
 
@@ -827,9 +826,9 @@ bool GCOVProfiler::emitProfileArcs() {
             auto It = EdgeToCounter.find({&BB, nullptr});
             assert(It != EdgeToCounter.end());
             const unsigned Edge = It->second;
-            Value *Counter =
-                Builder.CreateConstInBoundsGEP2_64(Counters, 0, Edge);
-            Value *Count = Builder.CreateLoad(Counter);
+            Value *Counter = Builder.CreateConstInBoundsGEP2_64(
+                Counters->getValueType(), Counters, 0, Edge);
+            Value *Count = Builder.CreateLoad(Builder.getInt64Ty(), Counter);
             Count = Builder.CreateAdd(Count, Builder.getInt64(1));
             Builder.CreateStore(Count, Counter);
           }
@@ -864,7 +863,7 @@ bool GCOVProfiler::emitProfileArcs() {
 
     // Initialize the environment and register the local writeout and flush
     // functions.
-    Constant *GCOVInit = M->getOrInsertFunction("llvm_gcov_init", FTy);
+    FunctionCallee GCOVInit = M->getOrInsertFunction("llvm_gcov_init", FTy);
     Builder.CreateCall(GCOVInit, {WriteoutF, FlushF});
     Builder.CreateRetVoid();
 
@@ -874,22 +873,21 @@ bool GCOVProfiler::emitProfileArcs() {
   return Result;
 }
 
-Constant *GCOVProfiler::getStartFileFunc() {
+FunctionCallee GCOVProfiler::getStartFileFunc() {
   Type *Args[] = {
     Type::getInt8PtrTy(*Ctx),  // const char *orig_filename
     Type::getInt8PtrTy(*Ctx),  // const char version[4]
     Type::getInt32Ty(*Ctx),    // uint32_t checksum
   };
   FunctionType *FTy = FunctionType::get(Type::getVoidTy(*Ctx), Args, false);
-  auto *Res = M->getOrInsertFunction("llvm_gcda_start_file", FTy);
-  if (Function *FunRes = dyn_cast<Function>(Res))
-    if (auto AK = TLI->getExtAttrForI32Param(false))
-      FunRes->addParamAttr(2, AK);
+  AttributeList AL;
+  if (auto AK = TLI->getExtAttrForI32Param(false))
+    AL = AL.addParamAttribute(*Ctx, 2, AK);
+  FunctionCallee Res = M->getOrInsertFunction("llvm_gcda_start_file", FTy, AL);
   return Res;
-
 }
 
-Constant *GCOVProfiler::getEmitFunctionFunc() {
+FunctionCallee GCOVProfiler::getEmitFunctionFunc() {
   Type *Args[] = {
     Type::getInt32Ty(*Ctx),    // uint32_t ident
     Type::getInt8PtrTy(*Ctx),  // const char *function_name
@@ -898,36 +896,34 @@ Constant *GCOVProfiler::getEmitFunctionFunc() {
     Type::getInt32Ty(*Ctx),    // uint32_t cfg_checksum
   };
   FunctionType *FTy = FunctionType::get(Type::getVoidTy(*Ctx), Args, false);
-  auto *Res = M->getOrInsertFunction("llvm_gcda_emit_function", FTy);
-  if (Function *FunRes = dyn_cast<Function>(Res))
-    if (auto AK = TLI->getExtAttrForI32Param(false)) {
-      FunRes->addParamAttr(0, AK);
-      FunRes->addParamAttr(2, AK);
-      FunRes->addParamAttr(3, AK);
-      FunRes->addParamAttr(4, AK);
-    }
-  return Res;
+  AttributeList AL;
+  if (auto AK = TLI->getExtAttrForI32Param(false)) {
+    AL = AL.addParamAttribute(*Ctx, 0, AK);
+    AL = AL.addParamAttribute(*Ctx, 2, AK);
+    AL = AL.addParamAttribute(*Ctx, 3, AK);
+    AL = AL.addParamAttribute(*Ctx, 4, AK);
+  }
+  return M->getOrInsertFunction("llvm_gcda_emit_function", FTy);
 }
 
-Constant *GCOVProfiler::getEmitArcsFunc() {
+FunctionCallee GCOVProfiler::getEmitArcsFunc() {
   Type *Args[] = {
     Type::getInt32Ty(*Ctx),     // uint32_t num_counters
     Type::getInt64PtrTy(*Ctx),  // uint64_t *counters
   };
   FunctionType *FTy = FunctionType::get(Type::getVoidTy(*Ctx), Args, false);
-  auto *Res = M->getOrInsertFunction("llvm_gcda_emit_arcs", FTy);
-  if (Function *FunRes = dyn_cast<Function>(Res))
-    if (auto AK = TLI->getExtAttrForI32Param(false))
-      FunRes->addParamAttr(0, AK);
-  return Res;
+  AttributeList AL;
+  if (auto AK = TLI->getExtAttrForI32Param(false))
+    AL = AL.addParamAttribute(*Ctx, 0, AK);
+  return M->getOrInsertFunction("llvm_gcda_emit_arcs", FTy, AL);
 }
 
-Constant *GCOVProfiler::getSummaryInfoFunc() {
+FunctionCallee GCOVProfiler::getSummaryInfoFunc() {
   FunctionType *FTy = FunctionType::get(Type::getVoidTy(*Ctx), false);
   return M->getOrInsertFunction("llvm_gcda_summary_info", FTy);
 }
 
-Constant *GCOVProfiler::getEndFileFunc() {
+FunctionCallee GCOVProfiler::getEndFileFunc() {
   FunctionType *FTy = FunctionType::get(Type::getVoidTy(*Ctx), false);
   return M->getOrInsertFunction("llvm_gcda_end_file", FTy);
 }
@@ -947,11 +943,11 @@ Function *GCOVProfiler::insertCounterWriteout(
   BasicBlock *BB = BasicBlock::Create(*Ctx, "entry", WriteoutF);
   IRBuilder<> Builder(BB);
 
-  Constant *StartFile = getStartFileFunc();
-  Constant *EmitFunction = getEmitFunctionFunc();
-  Constant *EmitArcs = getEmitArcsFunc();
-  Constant *SummaryInfo = getSummaryInfoFunc();
-  Constant *EndFile = getEndFileFunc();
+  FunctionCallee StartFile = getStartFileFunc();
+  FunctionCallee EmitFunction = getEmitFunctionFunc();
+  FunctionCallee EmitArcs = getEmitArcsFunc();
+  FunctionCallee SummaryInfo = getSummaryInfoFunc();
+  FunctionCallee EndFile = getEndFileFunc();
 
   NamedMDNode *CUNodes = M->getNamedMetadata("llvm.dbg.cu");
   if (!CUNodes) {
@@ -1088,22 +1084,32 @@ Function *GCOVProfiler::insertCounterWriteout(
   PHINode *IV =
       Builder.CreatePHI(Builder.getInt32Ty(), /*NumReservedValues*/ 2);
   IV->addIncoming(Builder.getInt32(0), BB);
-  auto *FileInfoPtr =
-      Builder.CreateInBoundsGEP(FileInfoArrayGV, {Builder.getInt32(0), IV});
-  auto *StartFileCallArgsPtr = Builder.CreateStructGEP(FileInfoPtr, 0);
+  auto *FileInfoPtr = Builder.CreateInBoundsGEP(
+      FileInfoArrayTy, FileInfoArrayGV, {Builder.getInt32(0), IV});
+  auto *StartFileCallArgsPtr =
+      Builder.CreateStructGEP(FileInfoTy, FileInfoPtr, 0);
   auto *StartFileCall = Builder.CreateCall(
       StartFile,
-      {Builder.CreateLoad(Builder.CreateStructGEP(StartFileCallArgsPtr, 0)),
-       Builder.CreateLoad(Builder.CreateStructGEP(StartFileCallArgsPtr, 1)),
-       Builder.CreateLoad(Builder.CreateStructGEP(StartFileCallArgsPtr, 2))});
+      {Builder.CreateLoad(StartFileCallArgsTy->getElementType(0),
+                          Builder.CreateStructGEP(StartFileCallArgsTy,
+                                                  StartFileCallArgsPtr, 0)),
+       Builder.CreateLoad(StartFileCallArgsTy->getElementType(1),
+                          Builder.CreateStructGEP(StartFileCallArgsTy,
+                                                  StartFileCallArgsPtr, 1)),
+       Builder.CreateLoad(StartFileCallArgsTy->getElementType(2),
+                          Builder.CreateStructGEP(StartFileCallArgsTy,
+                                                  StartFileCallArgsPtr, 2))});
   if (auto AK = TLI->getExtAttrForI32Param(false))
     StartFileCall->addParamAttr(2, AK);
   auto *NumCounters =
-      Builder.CreateLoad(Builder.CreateStructGEP(FileInfoPtr, 1));
+      Builder.CreateLoad(FileInfoTy->getElementType(1),
+                         Builder.CreateStructGEP(FileInfoTy, FileInfoPtr, 1));
   auto *EmitFunctionCallArgsArray =
-      Builder.CreateLoad(Builder.CreateStructGEP(FileInfoPtr, 2));
+      Builder.CreateLoad(FileInfoTy->getElementType(2),
+                         Builder.CreateStructGEP(FileInfoTy, FileInfoPtr, 2));
   auto *EmitArcsCallArgsArray =
-      Builder.CreateLoad(Builder.CreateStructGEP(FileInfoPtr, 3));
+      Builder.CreateLoad(FileInfoTy->getElementType(3),
+                         Builder.CreateStructGEP(FileInfoTy, FileInfoPtr, 3));
   auto *EnterCounterLoopCond =
       Builder.CreateICmpSLT(Builder.getInt32(0), NumCounters);
   Builder.CreateCondBr(EnterCounterLoopCond, CounterLoopHeader, FileLoopLatch);
@@ -1111,16 +1117,26 @@ Function *GCOVProfiler::insertCounterWriteout(
   Builder.SetInsertPoint(CounterLoopHeader);
   auto *JV = Builder.CreatePHI(Builder.getInt32Ty(), /*NumReservedValues*/ 2);
   JV->addIncoming(Builder.getInt32(0), FileLoopHeader);
-  auto *EmitFunctionCallArgsPtr =
-      Builder.CreateInBoundsGEP(EmitFunctionCallArgsArray, {JV});
+  auto *EmitFunctionCallArgsPtr = Builder.CreateInBoundsGEP(
+      EmitFunctionCallArgsTy, EmitFunctionCallArgsArray, JV);
   auto *EmitFunctionCall = Builder.CreateCall(
       EmitFunction,
-      {Builder.CreateLoad(Builder.CreateStructGEP(EmitFunctionCallArgsPtr, 0)),
-       Builder.CreateLoad(Builder.CreateStructGEP(EmitFunctionCallArgsPtr, 1)),
-       Builder.CreateLoad(Builder.CreateStructGEP(EmitFunctionCallArgsPtr, 2)),
-       Builder.CreateLoad(Builder.CreateStructGEP(EmitFunctionCallArgsPtr, 3)),
-       Builder.CreateLoad(
-           Builder.CreateStructGEP(EmitFunctionCallArgsPtr, 4))});
+      {Builder.CreateLoad(EmitFunctionCallArgsTy->getElementType(0),
+                          Builder.CreateStructGEP(EmitFunctionCallArgsTy,
+                                                  EmitFunctionCallArgsPtr, 0)),
+       Builder.CreateLoad(EmitFunctionCallArgsTy->getElementType(1),
+                          Builder.CreateStructGEP(EmitFunctionCallArgsTy,
+                                                  EmitFunctionCallArgsPtr, 1)),
+       Builder.CreateLoad(EmitFunctionCallArgsTy->getElementType(2),
+                          Builder.CreateStructGEP(EmitFunctionCallArgsTy,
+                                                  EmitFunctionCallArgsPtr, 2)),
+       Builder.CreateLoad(EmitFunctionCallArgsTy->getElementType(3),
+                          Builder.CreateStructGEP(EmitFunctionCallArgsTy,
+                                                  EmitFunctionCallArgsPtr, 3)),
+       Builder.CreateLoad(EmitFunctionCallArgsTy->getElementType(4),
+                          Builder.CreateStructGEP(EmitFunctionCallArgsTy,
+                                                  EmitFunctionCallArgsPtr,
+                                                  4))});
   if (auto AK = TLI->getExtAttrForI32Param(false)) {
     EmitFunctionCall->addParamAttr(0, AK);
     EmitFunctionCall->addParamAttr(2, AK);
@@ -1128,11 +1144,15 @@ Function *GCOVProfiler::insertCounterWriteout(
     EmitFunctionCall->addParamAttr(4, AK);
   }
   auto *EmitArcsCallArgsPtr =
-      Builder.CreateInBoundsGEP(EmitArcsCallArgsArray, {JV});
+      Builder.CreateInBoundsGEP(EmitArcsCallArgsTy, EmitArcsCallArgsArray, JV);
   auto *EmitArcsCall = Builder.CreateCall(
       EmitArcs,
-      {Builder.CreateLoad(Builder.CreateStructGEP(EmitArcsCallArgsPtr, 0)),
-       Builder.CreateLoad(Builder.CreateStructGEP(EmitArcsCallArgsPtr, 1))});
+      {Builder.CreateLoad(
+           EmitArcsCallArgsTy->getElementType(0),
+           Builder.CreateStructGEP(EmitArcsCallArgsTy, EmitArcsCallArgsPtr, 0)),
+       Builder.CreateLoad(EmitArcsCallArgsTy->getElementType(1),
+                          Builder.CreateStructGEP(EmitArcsCallArgsTy,
+                                                  EmitArcsCallArgsPtr, 1))});
   if (auto AK = TLI->getExtAttrForI32Param(false))
     EmitArcsCall->addParamAttr(0, AK);
   auto *NextJV = Builder.CreateAdd(JV, Builder.getInt32(1));
@@ -1172,7 +1192,7 @@ insertFlush(ArrayRef<std::pair<GlobalVariable*, MDNode*> > CountersBySP) {
   BasicBlock *Entry = BasicBlock::Create(*Ctx, "entry", FlushF);
 
   // Write out the current counters.
-  Constant *WriteoutF = M->getFunction("__llvm_gcov_writeout");
+  Function *WriteoutF = M->getFunction("__llvm_gcov_writeout");
   assert(WriteoutF && "Need to create the writeout function first!");
 
   IRBuilder<> Builder(Entry);
diff --git a/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp b/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp
index 042e8ea0d2945fea576259ec523aa97442ef75b0..6ecbd239f5b2c621cf286e3d4abf725ef4579f07 100644
--- a/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp
+++ b/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp
@@ -1,9 +1,8 @@
 //===- HWAddressSanitizer.cpp - detector of uninitialized reads -------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -152,6 +151,19 @@ static cl::opt<bool>
                               cl::desc("create static frame descriptions"),
                               cl::Hidden, cl::init(true));
 
+static cl::opt<bool>
+    ClInstrumentMemIntrinsics("hwasan-instrument-mem-intrinsics",
+                              cl::desc("instrument memory intrinsics"),
+                              cl::Hidden, cl::init(true));
+
+static cl::opt<bool> ClInlineAllChecks("hwasan-inline-all-checks",
+                                       cl::desc("inline all checks"),
+                                       cl::Hidden, cl::init(false));
+
+static cl::opt<bool> ClAllowIfunc("hwasan-allow-ifunc",
+                                  cl::desc("allow the use of ifunc"),
+                                  cl::Hidden, cl::init(false));
+
 namespace {
 
 /// An instrumentation pass implementing detection of addressability bugs
@@ -175,13 +187,16 @@ public:
 
   void initializeCallbacks(Module &M);
 
+  Value *getDynamicShadowIfunc(IRBuilder<> &IRB);
   Value *getDynamicShadowNonTls(IRBuilder<> &IRB);
 
   void untagPointerOperand(Instruction *I, Value *Addr);
-  Value *memToShadow(Value *Shadow, Type *Ty, IRBuilder<> &IRB);
-  void instrumentMemAccessInline(Value *PtrLong, bool IsWrite,
+  Value *shadowBase();
+  Value *memToShadow(Value *Shadow, IRBuilder<> &IRB);
+  void instrumentMemAccessInline(Value *Ptr, bool IsWrite,
                                  unsigned AccessSizeIndex,
                                  Instruction *InsertBefore);
+  void instrumentMemIntrinsic(MemIntrinsic *MI);
   bool instrumentMemAccess(Instruction *I);
   Value *isInterestingMemoryAccess(Instruction *I, bool *IsWrite,
                                    uint64_t *TypeSize, unsigned *Alignment,
@@ -206,6 +221,7 @@ private:
   LLVMContext *C;
   std::string CurModuleUniqueId;
   Triple TargetTriple;
+  FunctionCallee HWAsanMemmove, HWAsanMemcpy, HWAsanMemset;
 
   // Frame description is a way to pass names/sizes of local variables
   // to the run-time w/o adding extra executable code in every function.
@@ -247,17 +263,19 @@ private:
   Type *IntptrTy;
   Type *Int8PtrTy;
   Type *Int8Ty;
+  Type *Int32Ty;
 
   bool CompileKernel;
   bool Recover;
 
   Function *HwasanCtorFunction;
 
-  Function *HwasanMemoryAccessCallback[2][kNumberOfAccessSizes];
-  Function *HwasanMemoryAccessCallbackSized[2];
+  FunctionCallee HwasanMemoryAccessCallback[2][kNumberOfAccessSizes];
+  FunctionCallee HwasanMemoryAccessCallbackSized[2];
 
-  Function *HwasanTagMemoryFunc;
-  Function *HwasanGenerateTagFunc;
+  FunctionCallee HwasanTagMemoryFunc;
+  FunctionCallee HwasanGenerateTagFunc;
+  FunctionCallee HwasanThreadEnterFunc;
 
   Constant *ShadowGlobal;
 
@@ -301,6 +319,7 @@ bool HWAddressSanitizer::doInitialization(Module &M) {
   IntptrTy = IRB.getIntPtrTy(DL);
   Int8PtrTy = IRB.getInt8PtrTy();
   Int8Ty = IRB.getInt8Ty();
+  Int32Ty = IRB.getInt32Ty();
 
   HwasanCtorFunction = nullptr;
   if (!CompileKernel) {
@@ -350,29 +369,52 @@ void HWAddressSanitizer::initializeCallbacks(Module &M) {
     const std::string TypeStr = AccessIsWrite ? "store" : "load";
     const std::string EndingStr = Recover ? "_noabort" : "";
 
-    HwasanMemoryAccessCallbackSized[AccessIsWrite] =
-        checkSanitizerInterfaceFunction(M.getOrInsertFunction(
-            ClMemoryAccessCallbackPrefix + TypeStr + "N" + EndingStr,
-            FunctionType::get(IRB.getVoidTy(), {IntptrTy, IntptrTy}, false)));
+    HwasanMemoryAccessCallbackSized[AccessIsWrite] = M.getOrInsertFunction(
+        ClMemoryAccessCallbackPrefix + TypeStr + "N" + EndingStr,
+        FunctionType::get(IRB.getVoidTy(), {IntptrTy, IntptrTy}, false));
 
     for (size_t AccessSizeIndex = 0; AccessSizeIndex < kNumberOfAccessSizes;
          AccessSizeIndex++) {
       HwasanMemoryAccessCallback[AccessIsWrite][AccessSizeIndex] =
-          checkSanitizerInterfaceFunction(M.getOrInsertFunction(
+          M.getOrInsertFunction(
               ClMemoryAccessCallbackPrefix + TypeStr +
                   itostr(1ULL << AccessSizeIndex) + EndingStr,
-              FunctionType::get(IRB.getVoidTy(), {IntptrTy}, false)));
+              FunctionType::get(IRB.getVoidTy(), {IntptrTy}, false));
     }
   }
 
-  HwasanTagMemoryFunc = checkSanitizerInterfaceFunction(M.getOrInsertFunction(
-      "__hwasan_tag_memory", IRB.getVoidTy(), Int8PtrTy, Int8Ty, IntptrTy));
-  HwasanGenerateTagFunc = checkSanitizerInterfaceFunction(
-      M.getOrInsertFunction("__hwasan_generate_tag", Int8Ty));
+  HwasanTagMemoryFunc = M.getOrInsertFunction(
+      "__hwasan_tag_memory", IRB.getVoidTy(), Int8PtrTy, Int8Ty, IntptrTy);
+  HwasanGenerateTagFunc =
+      M.getOrInsertFunction("__hwasan_generate_tag", Int8Ty);
+
+  ShadowGlobal = M.getOrInsertGlobal("__hwasan_shadow",
+                                     ArrayType::get(IRB.getInt8Ty(), 0));
+
+  const std::string MemIntrinCallbackPrefix =
+      CompileKernel ? std::string("") : ClMemoryAccessCallbackPrefix;
+  HWAsanMemmove = M.getOrInsertFunction(MemIntrinCallbackPrefix + "memmove",
+                                        IRB.getInt8PtrTy(), IRB.getInt8PtrTy(),
+                                        IRB.getInt8PtrTy(), IntptrTy);
+  HWAsanMemcpy = M.getOrInsertFunction(MemIntrinCallbackPrefix + "memcpy",
+                                       IRB.getInt8PtrTy(), IRB.getInt8PtrTy(),
+                                       IRB.getInt8PtrTy(), IntptrTy);
+  HWAsanMemset = M.getOrInsertFunction(MemIntrinCallbackPrefix + "memset",
+                                       IRB.getInt8PtrTy(), IRB.getInt8PtrTy(),
+                                       IRB.getInt32Ty(), IntptrTy);
+
+  HwasanThreadEnterFunc =
+      M.getOrInsertFunction("__hwasan_thread_enter", IRB.getVoidTy());
+}
 
-  if (Mapping.InGlobal)
-    ShadowGlobal = M.getOrInsertGlobal("__hwasan_shadow",
-                                       ArrayType::get(IRB.getInt8Ty(), 0));
+Value *HWAddressSanitizer::getDynamicShadowIfunc(IRBuilder<> &IRB) {
+  // An empty inline asm with input reg == output reg.
+  // An opaque no-op cast, basically.
+  InlineAsm *Asm = InlineAsm::get(
+      FunctionType::get(Int8PtrTy, {ShadowGlobal->getType()}, false),
+      StringRef(""), StringRef("=r,0"),
+      /*hasSideEffects=*/false);
+  return IRB.CreateCall(Asm, {ShadowGlobal}, ".hwasan.shadow");
 }
 
 Value *HWAddressSanitizer::getDynamicShadowNonTls(IRBuilder<> &IRB) {
@@ -381,18 +423,12 @@ Value *HWAddressSanitizer::getDynamicShadowNonTls(IRBuilder<> &IRB) {
     return nullptr;
 
   if (Mapping.InGlobal) {
-    // An empty inline asm with input reg == output reg.
-    // An opaque pointer-to-int cast, basically.
-    InlineAsm *Asm = InlineAsm::get(
-        FunctionType::get(IntptrTy, {ShadowGlobal->getType()}, false),
-        StringRef(""), StringRef("=r,0"),
-        /*hasSideEffects=*/false);
-    return IRB.CreateCall(Asm, {ShadowGlobal}, ".hwasan.shadow");
+    return getDynamicShadowIfunc(IRB);
   } else {
     Value *GlobalDynamicAddress =
         IRB.GetInsertBlock()->getParent()->getParent()->getOrInsertGlobal(
-            kHwasanShadowMemoryDynamicAddress, IntptrTy);
-    return IRB.CreateLoad(GlobalDynamicAddress);
+            kHwasanShadowMemoryDynamicAddress, Int8PtrTy);
+    return IRB.CreateLoad(Int8PtrTy, GlobalDynamicAddress);
   }
 }
 
@@ -484,29 +520,44 @@ void HWAddressSanitizer::untagPointerOperand(Instruction *I, Value *Addr) {
   I->setOperand(getPointerOperandIndex(I), UntaggedPtr);
 }
 
-Value *HWAddressSanitizer::memToShadow(Value *Mem, Type *Ty, IRBuilder<> &IRB) {
+Value *HWAddressSanitizer::shadowBase() {
+  if (LocalDynamicShadow)
+    return LocalDynamicShadow;
+  return ConstantExpr::getIntToPtr(ConstantInt::get(IntptrTy, Mapping.Offset),
+                                   Int8PtrTy);
+}
+
+Value *HWAddressSanitizer::memToShadow(Value *Mem, IRBuilder<> &IRB) {
   // Mem >> Scale
   Value *Shadow = IRB.CreateLShr(Mem, Mapping.Scale);
   if (Mapping.Offset == 0)
-    return Shadow;
+    return IRB.CreateIntToPtr(Shadow, Int8PtrTy);
   // (Mem >> Scale) + Offset
-  Value *ShadowBase;
-  if (LocalDynamicShadow)
-    ShadowBase = LocalDynamicShadow;
-  else
-    ShadowBase = ConstantInt::get(Ty, Mapping.Offset);
-  return IRB.CreateAdd(Shadow, ShadowBase);
+  return IRB.CreateGEP(Int8Ty, shadowBase(), Shadow);
 }
 
-void HWAddressSanitizer::instrumentMemAccessInline(Value *PtrLong, bool IsWrite,
+void HWAddressSanitizer::instrumentMemAccessInline(Value *Ptr, bool IsWrite,
                                                    unsigned AccessSizeIndex,
                                                    Instruction *InsertBefore) {
+  const int64_t AccessInfo = Recover * 0x20 + IsWrite * 0x10 + AccessSizeIndex;
   IRBuilder<> IRB(InsertBefore);
+
+  if (!ClInlineAllChecks && TargetTriple.isAArch64() &&
+      TargetTriple.isOSBinFormatELF() && !Recover) {
+    Module *M = IRB.GetInsertBlock()->getParent()->getParent();
+    Ptr = IRB.CreateBitCast(Ptr, Int8PtrTy);
+    IRB.CreateCall(
+        Intrinsic::getDeclaration(M, Intrinsic::hwasan_check_memaccess),
+        {shadowBase(), Ptr, ConstantInt::get(Int32Ty, AccessInfo)});
+    return;
+  }
+
+  Value *PtrLong = IRB.CreatePointerCast(Ptr, IntptrTy);
   Value *PtrTag = IRB.CreateTrunc(IRB.CreateLShr(PtrLong, kPointerTagShift),
                                   IRB.getInt8Ty());
   Value *AddrLong = untagPointer(IRB, PtrLong);
-  Value *ShadowLong = memToShadow(AddrLong, PtrLong->getType(), IRB);
-  Value *MemTag = IRB.CreateLoad(IRB.CreateIntToPtr(ShadowLong, Int8PtrTy));
+  Value *Shadow = memToShadow(AddrLong, IRB);
+  Value *MemTag = IRB.CreateLoad(Int8Ty, Shadow);
   Value *TagMismatch = IRB.CreateICmpNE(PtrTag, MemTag);
 
   int matchAllTag = ClMatchAllTag.getNumOccurrences() > 0 ?
@@ -522,7 +573,6 @@ void HWAddressSanitizer::instrumentMemAccessInline(Value *PtrLong, bool IsWrite,
                                 MDBuilder(*C).createBranchWeights(1, 100000));
 
   IRB.SetInsertPoint(CheckTerm);
-  const int64_t AccessInfo = Recover * 0x20 + IsWrite * 0x10 + AccessSizeIndex;
   InlineAsm *Asm;
   switch (TargetTriple.getArch()) {
     case Triple::x86_64:
@@ -548,12 +598,36 @@ void HWAddressSanitizer::instrumentMemAccessInline(Value *PtrLong, bool IsWrite,
   IRB.CreateCall(Asm, PtrLong);
 }
 
+void HWAddressSanitizer::instrumentMemIntrinsic(MemIntrinsic *MI) {
+  IRBuilder<> IRB(MI);
+  if (isa<MemTransferInst>(MI)) {
+    IRB.CreateCall(
+        isa<MemMoveInst>(MI) ? HWAsanMemmove : HWAsanMemcpy,
+        {IRB.CreatePointerCast(MI->getOperand(0), IRB.getInt8PtrTy()),
+         IRB.CreatePointerCast(MI->getOperand(1), IRB.getInt8PtrTy()),
+         IRB.CreateIntCast(MI->getOperand(2), IntptrTy, false)});
+  } else if (isa<MemSetInst>(MI)) {
+    IRB.CreateCall(
+        HWAsanMemset,
+        {IRB.CreatePointerCast(MI->getOperand(0), IRB.getInt8PtrTy()),
+         IRB.CreateIntCast(MI->getOperand(1), IRB.getInt32Ty(), false),
+         IRB.CreateIntCast(MI->getOperand(2), IntptrTy, false)});
+  }
+  MI->eraseFromParent();
+}
+
 bool HWAddressSanitizer::instrumentMemAccess(Instruction *I) {
   LLVM_DEBUG(dbgs() << "Instrumenting: " << *I << "\n");
   bool IsWrite = false;
   unsigned Alignment = 0;
   uint64_t TypeSize = 0;
   Value *MaybeMask = nullptr;
+
+  if (ClInstrumentMemIntrinsics && isa<MemIntrinsic>(I)) {
+    instrumentMemIntrinsic(cast<MemIntrinsic>(I));
+    return true;
+  }
+
   Value *Addr =
       isInterestingMemoryAccess(I, &IsWrite, &TypeSize, &Alignment, &MaybeMask);
 
@@ -564,7 +638,6 @@ bool HWAddressSanitizer::instrumentMemAccess(Instruction *I) {
     return false; //FIXME
 
   IRBuilder<> IRB(I);
-  Value *AddrLong = IRB.CreatePointerCast(Addr, IntptrTy);
   if (isPowerOf2_64(TypeSize) &&
       (TypeSize / 8 <= (1UL << (kNumberOfAccessSizes - 1))) &&
       (Alignment >= (1UL << Mapping.Scale) || Alignment == 0 ||
@@ -572,13 +645,14 @@ bool HWAddressSanitizer::instrumentMemAccess(Instruction *I) {
     size_t AccessSizeIndex = TypeSizeToSizeIndex(TypeSize);
     if (ClInstrumentWithCalls) {
       IRB.CreateCall(HwasanMemoryAccessCallback[IsWrite][AccessSizeIndex],
-                     AddrLong);
+                     IRB.CreatePointerCast(Addr, IntptrTy));
     } else {
-      instrumentMemAccessInline(AddrLong, IsWrite, AccessSizeIndex, I);
+      instrumentMemAccessInline(Addr, IsWrite, AccessSizeIndex, I);
     }
   } else {
     IRB.CreateCall(HwasanMemoryAccessCallbackSized[IsWrite],
-                   {AddrLong, ConstantInt::get(IntptrTy, TypeSize / 8)});
+                   {IRB.CreatePointerCast(Addr, IntptrTy),
+                    ConstantInt::get(IntptrTy, TypeSize / 8)});
   }
   untagPointerOperand(I, Addr);
 
@@ -609,9 +683,7 @@ bool HWAddressSanitizer::tagAlloca(IRBuilder<> &IRB, AllocaInst *AI,
                     ConstantInt::get(IntptrTy, Size)});
   } else {
     size_t ShadowSize = Size >> Mapping.Scale;
-    Value *ShadowPtr = IRB.CreateIntToPtr(
-        memToShadow(IRB.CreatePointerCast(AI, IntptrTy), AI->getType(), IRB),
-        Int8PtrTy);
+    Value *ShadowPtr = memToShadow(IRB.CreatePointerCast(AI, IntptrTy), IRB);
     // If this memset is not inlined, it will be intercepted in the hwasan
     // runtime library. That's OK, because the interceptor skips the checks if
     // the address is in the shadow region.
@@ -712,10 +784,13 @@ Value *HWAddressSanitizer::untagPointer(IRBuilder<> &IRB, Value *PtrLong) {
 Value *HWAddressSanitizer::getHwasanThreadSlotPtr(IRBuilder<> &IRB, Type *Ty) {
   Module *M = IRB.GetInsertBlock()->getParent()->getParent();
   if (TargetTriple.isAArch64() && TargetTriple.isAndroid()) {
+    // Android provides a fixed TLS slot for sanitizers. See TLS_SLOT_SANITIZER
+    // in Bionic's libc/private/bionic_tls.h.
     Function *ThreadPointerFunc =
         Intrinsic::getDeclaration(M, Intrinsic::thread_pointer);
     Value *SlotPtr = IRB.CreatePointerCast(
-        IRB.CreateConstGEP1_32(IRB.CreateCall(ThreadPointerFunc), 0x40),
+        IRB.CreateConstGEP1_32(IRB.getInt8Ty(),
+                               IRB.CreateCall(ThreadPointerFunc), 0x30),
         Ty->getPointerTo(0));
     return SlotPtr;
   }
@@ -761,17 +836,41 @@ Value *HWAddressSanitizer::emitPrologue(IRBuilder<> &IRB,
   if (!Mapping.InTls)
     return getDynamicShadowNonTls(IRB);
 
+  if (ClAllowIfunc && !WithFrameRecord && TargetTriple.isAndroid())
+    return getDynamicShadowIfunc(IRB);
+
   Value *SlotPtr = getHwasanThreadSlotPtr(IRB, IntptrTy);
   assert(SlotPtr);
 
-  Value *ThreadLong = IRB.CreateLoad(SlotPtr);
+  Instruction *ThreadLong = IRB.CreateLoad(IntptrTy, SlotPtr);
+
+  Function *F = IRB.GetInsertBlock()->getParent();
+  if (F->getFnAttribute("hwasan-abi").getValueAsString() == "interceptor") {
+    Value *ThreadLongEqZero =
+        IRB.CreateICmpEQ(ThreadLong, ConstantInt::get(IntptrTy, 0));
+    auto *Br = cast<BranchInst>(SplitBlockAndInsertIfThen(
+        ThreadLongEqZero, cast<Instruction>(ThreadLongEqZero)->getNextNode(),
+        false, MDBuilder(*C).createBranchWeights(1, 100000)));
+
+    IRB.SetInsertPoint(Br);
+    // FIXME: This should call a new runtime function with a custom calling
+    // convention to avoid needing to spill all arguments here.
+    IRB.CreateCall(HwasanThreadEnterFunc);
+    LoadInst *ReloadThreadLong = IRB.CreateLoad(IntptrTy, SlotPtr);
+
+    IRB.SetInsertPoint(&*Br->getSuccessor(0)->begin());
+    PHINode *ThreadLongPhi = IRB.CreatePHI(IntptrTy, 2);
+    ThreadLongPhi->addIncoming(ThreadLong, ThreadLong->getParent());
+    ThreadLongPhi->addIncoming(ReloadThreadLong, ReloadThreadLong->getParent());
+    ThreadLong = ThreadLongPhi;
+  }
+
   // Extract the address field from ThreadLong. Unnecessary on AArch64 with TBI.
   Value *ThreadLongMaybeUntagged =
       TargetTriple.isAArch64() ? ThreadLong : untagPointer(IRB, ThreadLong);
 
   if (WithFrameRecord) {
     // Prepare ring buffer data.
-    Function *F = IRB.GetInsertBlock()->getParent();
     auto PC = IRB.CreatePtrToInt(F, IntptrTy);
     auto GetStackPointerFn =
         Intrinsic::getDeclaration(F->getParent(), Intrinsic::frameaddress);
@@ -815,6 +914,7 @@ Value *HWAddressSanitizer::emitPrologue(IRBuilder<> &IRB,
           ThreadLongMaybeUntagged,
           ConstantInt::get(IntptrTy, (1ULL << kShadowBaseAlignment) - 1)),
       ConstantInt::get(IntptrTy, 1), "hwasan.shadow");
+  ShadowBase = IRB.CreateIntToPtr(ShadowBase, Int8PtrTy);
   return ShadowBase;
 }
 
@@ -938,6 +1038,21 @@ bool HWAddressSanitizer::runOnFunction(Function &F) {
     Changed |= instrumentStack(AllocasToInstrument, RetVec, StackTag);
   }
 
+  // If we split the entry block, move any allocas that were originally in the
+  // entry block back into the entry block so that they aren't treated as
+  // dynamic allocas.
+  if (EntryIRB.GetInsertBlock() != &F.getEntryBlock()) {
+    InsertPt = &*F.getEntryBlock().begin();
+    for (auto II = EntryIRB.GetInsertBlock()->begin(),
+              IE = EntryIRB.GetInsertBlock()->end();
+         II != IE;) {
+      Instruction *I = &*II++;
+      if (auto *AI = dyn_cast<AllocaInst>(I))
+        if (isa<ConstantInt>(AI->getArraySize()))
+          I->moveBefore(InsertPt);
+    }
+  }
+
   for (auto Inst : ToInstrument)
     Changed |= instrumentMemAccess(Inst);
 
diff --git a/lib/Transforms/Instrumentation/IndirectCallPromotion.cpp b/lib/Transforms/Instrumentation/IndirectCallPromotion.cpp
index 7edc53d07c7565fd3892d709744ad2bffef9a975..79c219aeed5cf79d84b6af12e11ed7264b4fc52f 100644
--- a/lib/Transforms/Instrumentation/IndirectCallPromotion.cpp
+++ b/lib/Transforms/Instrumentation/IndirectCallPromotion.cpp
@@ -1,9 +1,8 @@
 //===- IndirectCallPromotion.cpp - Optimizations based on value profiling -===//
 //
-//                      The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -19,7 +18,7 @@
 #include "llvm/ADT/Statistic.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Analysis/IndirectCallPromotionAnalysis.h"
-#include "llvm/Analysis/IndirectCallSiteVisitor.h"
+#include "llvm/Analysis/IndirectCallVisitor.h"
 #include "llvm/Analysis/OptimizationRemarkEmitter.h"
 #include "llvm/Analysis/ProfileSummaryInfo.h"
 #include "llvm/IR/Attributes.h"
@@ -41,8 +40,8 @@
 #include "llvm/ProfileData/InstrProf.h"
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/CommandLine.h"
-#include "llvm/Support/Error.h"
 #include "llvm/Support/Debug.h"
+#include "llvm/Support/Error.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Transforms/Instrumentation.h"
 #include "llvm/Transforms/Instrumentation/PGOInstrumentation.h"
@@ -311,10 +310,10 @@ Instruction *llvm::pgo::promoteIndirectCall(Instruction *Inst,
       promoteCallWithIfThenElse(CallSite(Inst), DirectCallee, BranchWeights);
 
   if (AttachProfToDirectCall) {
-    SmallVector<uint32_t, 1> Weights;
-    Weights.push_back(Count);
     MDBuilder MDB(NewInst->getContext());
-    NewInst->setMetadata(LLVMContext::MD_prof, MDB.createBranchWeights(Weights));
+    NewInst->setMetadata(
+        LLVMContext::MD_prof,
+        MDB.createBranchWeights({static_cast<uint32_t>(Count)}));
   }
 
   using namespace ore;
@@ -352,7 +351,7 @@ uint32_t ICallPromotionFunc::tryToPromote(
 bool ICallPromotionFunc::processFunction(ProfileSummaryInfo *PSI) {
   bool Changed = false;
   ICallPromotionAnalysis ICallAnalysis;
-  for (auto &I : findIndirectCallSites(F)) {
+  for (auto &I : findIndirectCalls(F)) {
     uint32_t NumVals, NumCandidates;
     uint64_t TotalCount;
     auto ICallProfDataRef = ICallAnalysis.getPromotionCandidatesForInstruction(
diff --git a/lib/Transforms/Instrumentation/InstrProfiling.cpp b/lib/Transforms/Instrumentation/InstrProfiling.cpp
index 15b94388cbe51429b513478eaee6f1af8e251742..e77427aa91ddc612ef560e9b24d3c4b523bdeb68 100644
--- a/lib/Transforms/Instrumentation/InstrProfiling.cpp
+++ b/lib/Transforms/Instrumentation/InstrProfiling.cpp
@@ -1,9 +1,8 @@
 //===-- InstrProfiling.cpp - Frontend instrumentation based profiling -----===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -187,7 +186,7 @@ public:
     SSA.AddAvailableValue(PH, Init);
   }
 
-  void doExtraRewritesBeforeFinalDeletion() const override {
+  void doExtraRewritesBeforeFinalDeletion() override {
     for (unsigned i = 0, e = ExitBlocks.size(); i != e; ++i) {
       BasicBlock *ExitBlock = ExitBlocks[i];
       Instruction *InsertPos = InsertPts[i];
@@ -196,6 +195,7 @@ public:
       // block.
       Value *LiveInValue = SSA.GetValueInMiddleOfBlock(ExitBlock);
       Value *Addr = cast<StoreInst>(Store)->getPointerOperand();
+      Type *Ty = LiveInValue->getType();
       IRBuilder<> Builder(InsertPos);
       if (AtomicCounterUpdatePromoted)
         // automic update currently can only be promoted across the current
@@ -203,7 +203,7 @@ public:
         Builder.CreateAtomicRMW(AtomicRMWInst::Add, Addr, LiveInValue,
                                 AtomicOrdering::SequentiallyConsistent);
       else {
-        LoadInst *OldVal = Builder.CreateLoad(Addr, "pgocount.promoted");
+        LoadInst *OldVal = Builder.CreateLoad(Ty, Addr, "pgocount.promoted");
         auto *NewVal = Builder.CreateAdd(OldVal, LiveInValue);
         auto *NewStore = Builder.CreateStore(NewVal, Addr);
 
@@ -509,13 +509,16 @@ bool InstrProfiling::run(Module &M, const TargetLibraryInfo &TLI) {
   return true;
 }
 
-static Constant *getOrInsertValueProfilingCall(Module &M,
-                                               const TargetLibraryInfo &TLI,
-                                               bool IsRange = false) {
+static FunctionCallee
+getOrInsertValueProfilingCall(Module &M, const TargetLibraryInfo &TLI,
+                              bool IsRange = false) {
   LLVMContext &Ctx = M.getContext();
   auto *ReturnTy = Type::getVoidTy(M.getContext());
 
-  Constant *Res;
+  AttributeList AL;
+  if (auto AK = TLI.getExtAttrForI32Param(false))
+    AL = AL.addParamAttribute(M.getContext(), 2, AK);
+
   if (!IsRange) {
     Type *ParamTypes[] = {
 #define VALUE_PROF_FUNC_PARAM(ParamType, ParamName, ParamLLVMType) ParamLLVMType
@@ -523,8 +526,8 @@ static Constant *getOrInsertValueProfilingCall(Module &M,
     };
     auto *ValueProfilingCallTy =
         FunctionType::get(ReturnTy, makeArrayRef(ParamTypes), false);
-    Res = M.getOrInsertFunction(getInstrProfValueProfFuncName(),
-                                ValueProfilingCallTy);
+    return M.getOrInsertFunction(getInstrProfValueProfFuncName(),
+                                 ValueProfilingCallTy, AL);
   } else {
     Type *RangeParamTypes[] = {
 #define VALUE_RANGE_PROF 1
@@ -534,15 +537,9 @@ static Constant *getOrInsertValueProfilingCall(Module &M,
     };
     auto *ValueRangeProfilingCallTy =
         FunctionType::get(ReturnTy, makeArrayRef(RangeParamTypes), false);
-    Res = M.getOrInsertFunction(getInstrProfValueRangeProfFuncName(),
-                                ValueRangeProfilingCallTy);
-  }
-
-  if (Function *FunRes = dyn_cast<Function>(Res)) {
-    if (auto AK = TLI.getExtAttrForI32Param(false))
-      FunRes->addParamAttr(2, AK);
+    return M.getOrInsertFunction(getInstrProfValueRangeProfFuncName(),
+                                 ValueRangeProfilingCallTy, AL);
   }
-  return Res;
 }
 
 void InstrProfiling::computeNumValueSiteCounts(InstrProfValueProfileInst *Ind) {
@@ -601,13 +598,15 @@ void InstrProfiling::lowerIncrement(InstrProfIncrementInst *Inc) {
 
   IRBuilder<> Builder(Inc);
   uint64_t Index = Inc->getIndex()->getZExtValue();
-  Value *Addr = Builder.CreateConstInBoundsGEP2_64(Counters, 0, Index);
+  Value *Addr = Builder.CreateConstInBoundsGEP2_64(Counters->getValueType(),
+                                                   Counters, 0, Index);
 
   if (Options.Atomic || AtomicCounterUpdateAll) {
     Builder.CreateAtomicRMW(AtomicRMWInst::Add, Addr, Inc->getStep(),
                             AtomicOrdering::Monotonic);
   } else {
-    Value *Load = Builder.CreateLoad(Addr, "pgocount");
+    Value *IncStep = Inc->getStep();
+    Value *Load = Builder.CreateLoad(IncStep->getType(), Addr, "pgocount");
     auto *Count = Builder.CreateAdd(Load, Inc->getStep());
     auto *Store = Builder.CreateStore(Count, Addr);
     if (isCounterPromotionEnabled())
@@ -678,32 +677,14 @@ static inline bool shouldRecordFunctionAddr(Function *F) {
   return F->hasAddressTaken() || F->hasLinkOnceLinkage();
 }
 
-static inline Comdat *getOrCreateProfileComdat(Module &M, Function &F,
-                                               InstrProfIncrementInst *Inc) {
-  if (!needsComdatForCounter(F, M))
-    return nullptr;
-
-  // COFF format requires a COMDAT section to have a key symbol with the same
-  // name. The linker targeting COFF also requires that the COMDAT
-  // a section is associated to must precede the associating section. For this
-  // reason, we must choose the counter var's name as the name of the comdat.
-  StringRef ComdatPrefix = (Triple(M.getTargetTriple()).isOSBinFormatCOFF()
-                                ? getInstrProfCountersVarPrefix()
-                                : getInstrProfComdatPrefix());
-  return M.getOrInsertComdat(StringRef(getVarName(Inc, ComdatPrefix)));
-}
-
-static bool needsRuntimeRegistrationOfSectionRange(const Module &M) {
+static bool needsRuntimeRegistrationOfSectionRange(const Triple &TT) {
   // Don't do this for Darwin.  compiler-rt uses linker magic.
-  if (Triple(M.getTargetTriple()).isOSDarwin())
+  if (TT.isOSDarwin())
     return false;
 
   // Use linker script magic to get data/cnts/name start/end.
-  if (Triple(M.getTargetTriple()).isOSLinux() ||
-      Triple(M.getTargetTriple()).isOSFreeBSD() ||
-      Triple(M.getTargetTriple()).isOSNetBSD() ||
-      Triple(M.getTargetTriple()).isOSFuchsia() ||
-      Triple(M.getTargetTriple()).isPS4CPU())
+  if (TT.isOSLinux() || TT.isOSFreeBSD() || TT.isOSNetBSD() ||
+      TT.isOSFuchsia() || TT.isPS4CPU() || TT.isOSWindows())
     return false;
 
   return true;
@@ -720,13 +701,44 @@ InstrProfiling::getOrCreateRegionCounters(InstrProfIncrementInst *Inc) {
     PD = It->second;
   }
 
-  // Move the name variable to the right section. Place them in a COMDAT group
-  // if the associated function is a COMDAT. This will make sure that
-  // only one copy of counters of the COMDAT function will be emitted after
-  // linking.
+  // Match the linkage and visibility of the name global, except on COFF, where
+  // the linkage must be local and consequentially the visibility must be
+  // default.
   Function *Fn = Inc->getParent()->getParent();
-  Comdat *ProfileVarsComdat = nullptr;
-  ProfileVarsComdat = getOrCreateProfileComdat(*M, *Fn, Inc);
+  GlobalValue::LinkageTypes Linkage = NamePtr->getLinkage();
+  GlobalValue::VisibilityTypes Visibility = NamePtr->getVisibility();
+  if (TT.isOSBinFormatCOFF()) {
+    Linkage = GlobalValue::InternalLinkage;
+    Visibility = GlobalValue::DefaultVisibility;
+  }
+
+  // Move the name variable to the right section. Place them in a COMDAT group
+  // if the associated function is a COMDAT. This will make sure that only one
+  // copy of counters of the COMDAT function will be emitted after linking.
+  Comdat *Cmdt = nullptr;
+  GlobalValue::LinkageTypes CounterLinkage = Linkage;
+  if (needsComdatForCounter(*Fn, *M)) {
+    if (TT.isOSBinFormatCOFF()) {
+      // There are two cases that need a comdat on COFF:
+      // 1. Functions that already have comdats (standard case)
+      // 2. available_externally functions (dllimport and C99 inline)
+      // In the first case, put all the data in the original function comdat. In
+      // the second case, create a new comdat group using the counter as the
+      // leader. It's linkage must be external, so use linkonce_odr linkage in
+      // that case.
+      if (Comdat *C = Fn->getComdat()) {
+        Cmdt = C;
+      } else {
+        Cmdt = M->getOrInsertComdat(
+            getVarName(Inc, getInstrProfCountersVarPrefix()));
+        CounterLinkage = GlobalValue::LinkOnceODRLinkage;
+      }
+    } else {
+      // For other platforms that use comdats (ELF), make a new comdat group for
+      // all the profile data. It will be deduplicated within the current DSO.
+      Cmdt = M->getOrInsertComdat(getVarName(Inc, getInstrProfComdatPrefix()));
+    }
+  }
 
   uint64_t NumCounters = Inc->getNumCounters()->getZExtValue();
   LLVMContext &Ctx = M->getContext();
@@ -734,20 +746,21 @@ InstrProfiling::getOrCreateRegionCounters(InstrProfIncrementInst *Inc) {
 
   // Create the counters variable.
   auto *CounterPtr =
-      new GlobalVariable(*M, CounterTy, false, NamePtr->getLinkage(),
+      new GlobalVariable(*M, CounterTy, false, Linkage,
                          Constant::getNullValue(CounterTy),
                          getVarName(Inc, getInstrProfCountersVarPrefix()));
-  CounterPtr->setVisibility(NamePtr->getVisibility());
+  CounterPtr->setVisibility(Visibility);
   CounterPtr->setSection(
       getInstrProfSectionName(IPSK_cnts, TT.getObjectFormat()));
   CounterPtr->setAlignment(8);
-  CounterPtr->setComdat(ProfileVarsComdat);
+  CounterPtr->setComdat(Cmdt);
+  CounterPtr->setLinkage(CounterLinkage);
 
   auto *Int8PtrTy = Type::getInt8PtrTy(Ctx);
   // Allocate statically the array of pointers to value profile nodes for
   // the current function.
   Constant *ValuesPtrExpr = ConstantPointerNull::get(Int8PtrTy);
-  if (ValueProfileStaticAlloc && !needsRuntimeRegistrationOfSectionRange(*M)) {
+  if (ValueProfileStaticAlloc && !needsRuntimeRegistrationOfSectionRange(TT)) {
     uint64_t NS = 0;
     for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
       NS += PD.NumValueSites[Kind];
@@ -755,14 +768,14 @@ InstrProfiling::getOrCreateRegionCounters(InstrProfIncrementInst *Inc) {
       ArrayType *ValuesTy = ArrayType::get(Type::getInt64Ty(Ctx), NS);
 
       auto *ValuesVar =
-          new GlobalVariable(*M, ValuesTy, false, NamePtr->getLinkage(),
+          new GlobalVariable(*M, ValuesTy, false, Linkage,
                              Constant::getNullValue(ValuesTy),
                              getVarName(Inc, getInstrProfValuesVarPrefix()));
-      ValuesVar->setVisibility(NamePtr->getVisibility());
+      ValuesVar->setVisibility(Visibility);
       ValuesVar->setSection(
           getInstrProfSectionName(IPSK_vals, TT.getObjectFormat()));
       ValuesVar->setAlignment(8);
-      ValuesVar->setComdat(ProfileVarsComdat);
+      ValuesVar->setComdat(Cmdt);
       ValuesPtrExpr =
           ConstantExpr::getBitCast(ValuesVar, Type::getInt8PtrTy(Ctx));
     }
@@ -789,13 +802,13 @@ InstrProfiling::getOrCreateRegionCounters(InstrProfIncrementInst *Inc) {
 #define INSTR_PROF_DATA(Type, LLVMType, Name, Init) Init,
 #include "llvm/ProfileData/InstrProfData.inc"
   };
-  auto *Data = new GlobalVariable(*M, DataTy, false, NamePtr->getLinkage(),
+  auto *Data = new GlobalVariable(*M, DataTy, false, Linkage,
                                   ConstantStruct::get(DataTy, DataVals),
                                   getVarName(Inc, getInstrProfDataVarPrefix()));
-  Data->setVisibility(NamePtr->getVisibility());
+  Data->setVisibility(Visibility);
   Data->setSection(getInstrProfSectionName(IPSK_data, TT.getObjectFormat()));
   Data->setAlignment(INSTR_PROF_DATA_ALIGNMENT);
-  Data->setComdat(ProfileVarsComdat);
+  Data->setComdat(Cmdt);
 
   PD.RegionCounters = CounterPtr;
   PD.DataVar = Data;
@@ -820,7 +833,7 @@ void InstrProfiling::emitVNodes() {
   // For now only support this on platforms that do
   // not require runtime registration to discover
   // named section start/end.
-  if (needsRuntimeRegistrationOfSectionRange(*M))
+  if (needsRuntimeRegistrationOfSectionRange(TT))
     return;
 
   size_t TotalNS = 0;
@@ -881,6 +894,10 @@ void InstrProfiling::emitNameData() {
   NamesSize = CompressedNameStr.size();
   NamesVar->setSection(
       getInstrProfSectionName(IPSK_name, TT.getObjectFormat()));
+  // On COFF, it's important to reduce the alignment down to 1 to prevent the
+  // linker from inserting padding before the start of the names section or
+  // between names entries.
+  NamesVar->setAlignment(1);
   UsedVars.push_back(NamesVar);
 
   for (auto *NamePtr : ReferencedNames)
@@ -888,7 +905,7 @@ void InstrProfiling::emitNameData() {
 }
 
 void InstrProfiling::emitRegistration() {
-  if (!needsRuntimeRegistrationOfSectionRange(*M))
+  if (!needsRuntimeRegistrationOfSectionRange(TT))
     return;
 
   // Construct the function.
@@ -929,7 +946,7 @@ void InstrProfiling::emitRegistration() {
 bool InstrProfiling::emitRuntimeHook() {
   // We expect the linker to be invoked with -u<hook_var> flag for linux,
   // for which case there is no need to emit the user function.
-  if (Triple(M->getTargetTriple()).isOSLinux())
+  if (TT.isOSLinux())
     return false;
 
   // If the module's provided its own runtime, we don't need to do anything.
@@ -950,11 +967,11 @@ bool InstrProfiling::emitRuntimeHook() {
   if (Options.NoRedZone)
     User->addFnAttr(Attribute::NoRedZone);
   User->setVisibility(GlobalValue::HiddenVisibility);
-  if (Triple(M->getTargetTriple()).supportsCOMDAT())
+  if (TT.supportsCOMDAT())
     User->setComdat(M->getOrInsertComdat(User->getName()));
 
   IRBuilder<> IRB(BasicBlock::Create(M->getContext(), "", User));
-  auto *Load = IRB.CreateLoad(Var);
+  auto *Load = IRB.CreateLoad(Int32Ty, Var);
   IRB.CreateRet(Load);
 
   // Mark the user variable as used so that it isn't stripped out.
@@ -968,23 +985,9 @@ void InstrProfiling::emitUses() {
 }
 
 void InstrProfiling::emitInitialization() {
-  StringRef InstrProfileOutput = Options.InstrProfileOutput;
-
-  if (!InstrProfileOutput.empty()) {
-    // Create variable for profile name.
-    Constant *ProfileNameConst =
-        ConstantDataArray::getString(M->getContext(), InstrProfileOutput, true);
-    GlobalVariable *ProfileNameVar = new GlobalVariable(
-        *M, ProfileNameConst->getType(), true, GlobalValue::WeakAnyLinkage,
-        ProfileNameConst, INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_NAME_VAR));
-    if (TT.supportsCOMDAT()) {
-      ProfileNameVar->setLinkage(GlobalValue::ExternalLinkage);
-      ProfileNameVar->setComdat(M->getOrInsertComdat(
-          StringRef(INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_NAME_VAR))));
-    }
-  }
-
-  Constant *RegisterF = M->getFunction(getInstrProfRegFuncsName());
+  // Create variable for profile name.
+  createProfileFileNameVar(*M, Options.InstrProfileOutput);
+  Function *RegisterF = M->getFunction(getInstrProfRegFuncsName());
   if (!RegisterF)
     return;
 
@@ -1000,8 +1003,7 @@ void InstrProfiling::emitInitialization() {
 
   // Add the basic block and the necessary calls.
   IRBuilder<> IRB(BasicBlock::Create(M->getContext(), "", F));
-  if (RegisterF)
-    IRB.CreateCall(RegisterF, {});
+  IRB.CreateCall(RegisterF, {});
   IRB.CreateRetVoid();
 
   appendToGlobalCtors(*M, F, 0);
diff --git a/lib/Transforms/Instrumentation/Instrumentation.cpp b/lib/Transforms/Instrumentation/Instrumentation.cpp
index eb6a3730ad9a8ec8b5134a8aaa764f922680c49c..2f73bb2b0c04fadaf5d05f712c3c275990a1678b 100644
--- a/lib/Transforms/Instrumentation/Instrumentation.cpp
+++ b/lib/Transforms/Instrumentation/Instrumentation.cpp
@@ -1,9 +1,8 @@
 //===-- Instrumentation.cpp - TransformUtils Infrastructure ---------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -111,9 +110,9 @@ void llvm::initializeInstrumentation(PassRegistry &Registry) {
   initializePGOIndirectCallPromotionLegacyPassPass(Registry);
   initializePGOMemOPSizeOptLegacyPassPass(Registry);
   initializeInstrProfilingLegacyPassPass(Registry);
-  initializeMemorySanitizerPass(Registry);
+  initializeMemorySanitizerLegacyPassPass(Registry);
   initializeHWAddressSanitizerPass(Registry);
-  initializeThreadSanitizerPass(Registry);
+  initializeThreadSanitizerLegacyPassPass(Registry);
   initializeSanitizerCoverageModulePass(Registry);
   initializeDataFlowSanitizerPass(Registry);
   initializeEfficiencySanitizerPass(Registry);
diff --git a/lib/Transforms/Instrumentation/LLVMBuild.txt b/lib/Transforms/Instrumentation/LLVMBuild.txt
index bcefe795c1930b91da2d85d5dd614644cb17156f..16776c768bfdb1e19c9b3d8cb329e71f1c5c2fee 100644
--- a/lib/Transforms/Instrumentation/LLVMBuild.txt
+++ b/lib/Transforms/Instrumentation/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/Transforms/Instrumentation/LLVMBuild.txt -----------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Transforms/Instrumentation/MaximumSpanningTree.h b/lib/Transforms/Instrumentation/MaximumSpanningTree.h
index 4eb758c69c581a255e0a3076db95ec0e62367801..19035ffb8032dc2fd7229dbf3b11240f61773051 100644
--- a/lib/Transforms/Instrumentation/MaximumSpanningTree.h
+++ b/lib/Transforms/Instrumentation/MaximumSpanningTree.h
@@ -1,9 +1,8 @@
 //===- llvm/Analysis/MaximumSpanningTree.h - Interface ----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Transforms/Instrumentation/MemorySanitizer.cpp b/lib/Transforms/Instrumentation/MemorySanitizer.cpp
index 0bbf3a90b9560c13a59af88d3f574a2168634bec..4b217e947f1f28e01f978d070a1c6b85a2b745f5 100644
--- a/lib/Transforms/Instrumentation/MemorySanitizer.cpp
+++ b/lib/Transforms/Instrumentation/MemorySanitizer.cpp
@@ -1,9 +1,8 @@
 //===- MemorySanitizer.cpp - detector of uninitialized reads --------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -99,9 +98,8 @@
 /// also possible that the arguments only indicate the offset for a base taken
 /// from a segment register, so it's dangerous to treat any asm() arguments as
 /// pointers. We take a conservative approach generating calls to
-///   __msan_instrument_asm_load(ptr, size) and
 ///   __msan_instrument_asm_store(ptr, size)
-/// , which defer the memory checking/unpoisoning to the runtime library.
+/// , which defer the memory unpoisoning to the runtime library.
 /// The latter can perform more complex address checks to figure out whether
 /// it's safe to touch the shadow memory.
 /// Like with atomic operations, we call __msan_instrument_asm_store() before
@@ -141,6 +139,7 @@
 ///
 //===----------------------------------------------------------------------===//
 
+#include "llvm/Transforms/Instrumentation/MemorySanitizer.h"
 #include "llvm/ADT/APInt.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/DepthFirstIterator.h"
@@ -150,7 +149,6 @@
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/Triple.h"
 #include "llvm/Analysis/TargetLibraryInfo.h"
-#include "llvm/Transforms/Utils/Local.h"
 #include "llvm/IR/Argument.h"
 #include "llvm/IR/Attributes.h"
 #include "llvm/IR/BasicBlock.h"
@@ -188,6 +186,7 @@
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Transforms/Instrumentation.h"
 #include "llvm/Transforms/Utils/BasicBlockUtils.h"
+#include "llvm/Transforms/Utils/Local.h"
 #include "llvm/Transforms/Utils/ModuleUtils.h"
 #include <algorithm>
 #include <cassert>
@@ -447,37 +446,34 @@ static const PlatformMemoryMapParams NetBSD_X86_MemoryMapParams = {
 
 namespace {
 
-/// An instrumentation pass implementing detection of uninitialized
-/// reads.
+/// Instrument functions of a module to detect uninitialized reads.
 ///
-/// MemorySanitizer: instrument the code in module to find
-/// uninitialized reads.
-class MemorySanitizer : public FunctionPass {
+/// Instantiating MemorySanitizer inserts the msan runtime library API function
+/// declarations into the module if they don't exist already. Instantiating
+/// ensures the __msan_init function is in the list of global constructors for
+/// the module.
+class MemorySanitizer {
 public:
-  // Pass identification, replacement for typeid.
-  static char ID;
-
-  MemorySanitizer(int TrackOrigins = 0, bool Recover = false,
-                  bool EnableKmsan = false)
-      : FunctionPass(ID) {
+  MemorySanitizer(Module &M, MemorySanitizerOptions Options) {
     this->CompileKernel =
-        ClEnableKmsan.getNumOccurrences() > 0 ? ClEnableKmsan : EnableKmsan;
+        ClEnableKmsan.getNumOccurrences() > 0 ? ClEnableKmsan : Options.Kernel;
     if (ClTrackOrigins.getNumOccurrences() > 0)
       this->TrackOrigins = ClTrackOrigins;
     else
-      this->TrackOrigins = this->CompileKernel ? 2 : TrackOrigins;
+      this->TrackOrigins = this->CompileKernel ? 2 : Options.TrackOrigins;
     this->Recover = ClKeepGoing.getNumOccurrences() > 0
                         ? ClKeepGoing
-                        : (this->CompileKernel | Recover);
+                        : (this->CompileKernel | Options.Recover);
+    initializeModule(M);
   }
-  StringRef getPassName() const override { return "MemorySanitizer"; }
 
-  void getAnalysisUsage(AnalysisUsage &AU) const override {
-    AU.addRequired<TargetLibraryInfoWrapperPass>();
-  }
+  // MSan cannot be moved or copied because of MapParams.
+  MemorySanitizer(MemorySanitizer &&) = delete;
+  MemorySanitizer &operator=(MemorySanitizer &&) = delete;
+  MemorySanitizer(const MemorySanitizer &) = delete;
+  MemorySanitizer &operator=(const MemorySanitizer &) = delete;
 
-  bool runOnFunction(Function &F) override;
-  bool doInitialization(Module &M) override;
+  bool sanitizeFunction(Function &F, TargetLibraryInfo &TLI);
 
 private:
   friend struct MemorySanitizerVisitor;
@@ -486,13 +482,13 @@ private:
   friend struct VarArgAArch64Helper;
   friend struct VarArgPowerPC64Helper;
 
+  void initializeModule(Module &M);
   void initializeCallbacks(Module &M);
   void createKernelApi(Module &M);
   void createUserspaceApi(Module &M);
 
   /// True if we're compiling the Linux kernel.
   bool CompileKernel;
-
   /// Track origins (allocation points) of uninitialized values.
   int TrackOrigins;
   bool Recover;
@@ -539,41 +535,42 @@ private:
   bool CallbacksInitialized = false;
 
   /// The run-time callback to print a warning.
-  Value *WarningFn;
+  FunctionCallee WarningFn;
 
   // These arrays are indexed by log2(AccessSize).
-  Value *MaybeWarningFn[kNumberOfAccessSizes];
-  Value *MaybeStoreOriginFn[kNumberOfAccessSizes];
+  FunctionCallee MaybeWarningFn[kNumberOfAccessSizes];
+  FunctionCallee MaybeStoreOriginFn[kNumberOfAccessSizes];
 
   /// Run-time helper that generates a new origin value for a stack
   /// allocation.
-  Value *MsanSetAllocaOrigin4Fn;
+  FunctionCallee MsanSetAllocaOrigin4Fn;
 
   /// Run-time helper that poisons stack on function entry.
-  Value *MsanPoisonStackFn;
+  FunctionCallee MsanPoisonStackFn;
 
   /// Run-time helper that records a store (or any event) of an
   /// uninitialized value and returns an updated origin id encoding this info.
-  Value *MsanChainOriginFn;
+  FunctionCallee MsanChainOriginFn;
 
   /// MSan runtime replacements for memmove, memcpy and memset.
-  Value *MemmoveFn, *MemcpyFn, *MemsetFn;
+  FunctionCallee MemmoveFn, MemcpyFn, MemsetFn;
 
   /// KMSAN callback for task-local function argument shadow.
-  Value *MsanGetContextStateFn;
+  StructType *MsanContextStateTy;
+  FunctionCallee MsanGetContextStateFn;
 
   /// Functions for poisoning/unpoisoning local variables
-  Value *MsanPoisonAllocaFn, *MsanUnpoisonAllocaFn;
+  FunctionCallee MsanPoisonAllocaFn, MsanUnpoisonAllocaFn;
 
   /// Each of the MsanMetadataPtrXxx functions returns a pair of shadow/origin
   /// pointers.
-  Value *MsanMetadataPtrForLoadN, *MsanMetadataPtrForStoreN;
-  Value *MsanMetadataPtrForLoad_1_8[4];
-  Value *MsanMetadataPtrForStore_1_8[4];
-  Value *MsanInstrumentAsmStoreFn, *MsanInstrumentAsmLoadFn;
+  FunctionCallee MsanMetadataPtrForLoadN, MsanMetadataPtrForStoreN;
+  FunctionCallee MsanMetadataPtrForLoad_1_8[4];
+  FunctionCallee MsanMetadataPtrForStore_1_8[4];
+  FunctionCallee MsanInstrumentAsmStoreFn;
 
   /// Helper to choose between different MsanMetadataPtrXxx().
-  Value *getKmsanShadowOriginAccessFn(bool isStore, int size);
+  FunctionCallee getKmsanShadowOriginAccessFn(bool isStore, int size);
 
   /// Memory map parameters used in application-to-shadow calculation.
   const MemoryMapParams *MapParams;
@@ -593,21 +590,54 @@ private:
   Function *MsanCtorFunction;
 };
 
+/// A legacy function pass for msan instrumentation.
+///
+/// Instruments functions to detect unitialized reads.
+struct MemorySanitizerLegacyPass : public FunctionPass {
+  // Pass identification, replacement for typeid.
+  static char ID;
+
+  MemorySanitizerLegacyPass(MemorySanitizerOptions Options = {})
+      : FunctionPass(ID), Options(Options) {}
+  StringRef getPassName() const override { return "MemorySanitizerLegacyPass"; }
+
+  void getAnalysisUsage(AnalysisUsage &AU) const override {
+    AU.addRequired<TargetLibraryInfoWrapperPass>();
+  }
+
+  bool runOnFunction(Function &F) override {
+    return MSan->sanitizeFunction(
+        F, getAnalysis<TargetLibraryInfoWrapperPass>().getTLI());
+  }
+  bool doInitialization(Module &M) override;
+
+  Optional<MemorySanitizer> MSan;
+  MemorySanitizerOptions Options;
+};
+
 } // end anonymous namespace
 
-char MemorySanitizer::ID = 0;
+PreservedAnalyses MemorySanitizerPass::run(Function &F,
+                                           FunctionAnalysisManager &FAM) {
+  MemorySanitizer Msan(*F.getParent(), Options);
+  if (Msan.sanitizeFunction(F, FAM.getResult<TargetLibraryAnalysis>(F)))
+    return PreservedAnalyses::none();
+  return PreservedAnalyses::all();
+}
+
+char MemorySanitizerLegacyPass::ID = 0;
 
-INITIALIZE_PASS_BEGIN(
-    MemorySanitizer, "msan",
-    "MemorySanitizer: detects uninitialized reads.", false, false)
+INITIALIZE_PASS_BEGIN(MemorySanitizerLegacyPass, "msan",
+                      "MemorySanitizer: detects uninitialized reads.", false,
+                      false)
 INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)
-INITIALIZE_PASS_END(
-    MemorySanitizer, "msan",
-    "MemorySanitizer: detects uninitialized reads.", false, false)
+INITIALIZE_PASS_END(MemorySanitizerLegacyPass, "msan",
+                    "MemorySanitizer: detects uninitialized reads.", false,
+                    false)
 
-FunctionPass *llvm::createMemorySanitizerPass(int TrackOrigins, bool Recover,
-                                              bool CompileKernel) {
-  return new MemorySanitizer(TrackOrigins, Recover, CompileKernel);
+FunctionPass *
+llvm::createMemorySanitizerLegacyPassPass(MemorySanitizerOptions Options) {
+  return new MemorySanitizerLegacyPass(Options);
 }
 
 /// Create a non-const global initialized with the given string.
@@ -642,18 +672,15 @@ void MemorySanitizer::createKernelApi(Module &M) {
                                     IRB.getInt32Ty());
   // Requests the per-task context state (kmsan_context_state*) from the
   // runtime library.
+  MsanContextStateTy = StructType::get(
+      ArrayType::get(IRB.getInt64Ty(), kParamTLSSize / 8),
+      ArrayType::get(IRB.getInt64Ty(), kRetvalTLSSize / 8),
+      ArrayType::get(IRB.getInt64Ty(), kParamTLSSize / 8),
+      ArrayType::get(IRB.getInt64Ty(), kParamTLSSize / 8), /* va_arg_origin */
+      IRB.getInt64Ty(), ArrayType::get(OriginTy, kParamTLSSize / 4), OriginTy,
+      OriginTy);
   MsanGetContextStateFn = M.getOrInsertFunction(
-      "__msan_get_context_state",
-      PointerType::get(
-          StructType::get(ArrayType::get(IRB.getInt64Ty(), kParamTLSSize / 8),
-                          ArrayType::get(IRB.getInt64Ty(), kRetvalTLSSize / 8),
-                          ArrayType::get(IRB.getInt64Ty(), kParamTLSSize / 8),
-                          ArrayType::get(IRB.getInt64Ty(),
-                                         kParamTLSSize / 8), /* va_arg_origin */
-                          IRB.getInt64Ty(),
-                          ArrayType::get(OriginTy, kParamTLSSize / 4), OriginTy,
-                          OriginTy),
-          0));
+      "__msan_get_context_state", PointerType::get(MsanContextStateTy, 0));
 
   Type *RetTy = StructType::get(PointerType::get(IRB.getInt8Ty(), 0),
                                 PointerType::get(IRB.getInt32Ty(), 0));
@@ -684,6 +711,14 @@ void MemorySanitizer::createKernelApi(Module &M) {
       "__msan_unpoison_alloca", IRB.getVoidTy(), IRB.getInt8PtrTy(), IntptrTy);
 }
 
+static Constant *getOrInsertGlobal(Module &M, StringRef Name, Type *Ty) {
+  return M.getOrInsertGlobal(Name, Ty, [&] {
+    return new GlobalVariable(M, Ty, false, GlobalVariable::ExternalLinkage,
+                              nullptr, Name, nullptr,
+                              GlobalVariable::InitialExecTLSModel);
+  });
+}
+
 /// Insert declarations for userspace-specific functions and globals.
 void MemorySanitizer::createUserspaceApi(Module &M) {
   IRBuilder<> IRB(*C);
@@ -695,42 +730,31 @@ void MemorySanitizer::createUserspaceApi(Module &M) {
   WarningFn = M.getOrInsertFunction(WarningFnName, IRB.getVoidTy());
 
   // Create the global TLS variables.
-  RetvalTLS = new GlobalVariable(
-      M, ArrayType::get(IRB.getInt64Ty(), kRetvalTLSSize / 8), false,
-      GlobalVariable::ExternalLinkage, nullptr, "__msan_retval_tls", nullptr,
-      GlobalVariable::InitialExecTLSModel);
-
-  RetvalOriginTLS = new GlobalVariable(
-      M, OriginTy, false, GlobalVariable::ExternalLinkage, nullptr,
-      "__msan_retval_origin_tls", nullptr, GlobalVariable::InitialExecTLSModel);
-
-  ParamTLS = new GlobalVariable(
-      M, ArrayType::get(IRB.getInt64Ty(), kParamTLSSize / 8), false,
-      GlobalVariable::ExternalLinkage, nullptr, "__msan_param_tls", nullptr,
-      GlobalVariable::InitialExecTLSModel);
-
-  ParamOriginTLS = new GlobalVariable(
-      M, ArrayType::get(OriginTy, kParamTLSSize / 4), false,
-      GlobalVariable::ExternalLinkage, nullptr, "__msan_param_origin_tls",
-      nullptr, GlobalVariable::InitialExecTLSModel);
-
-  VAArgTLS = new GlobalVariable(
-      M, ArrayType::get(IRB.getInt64Ty(), kParamTLSSize / 8), false,
-      GlobalVariable::ExternalLinkage, nullptr, "__msan_va_arg_tls", nullptr,
-      GlobalVariable::InitialExecTLSModel);
-
-  VAArgOriginTLS = new GlobalVariable(
-      M, ArrayType::get(OriginTy, kParamTLSSize / 4), false,
-      GlobalVariable::ExternalLinkage, nullptr, "__msan_va_arg_origin_tls",
-      nullptr, GlobalVariable::InitialExecTLSModel);
-
-  VAArgOverflowSizeTLS = new GlobalVariable(
-      M, IRB.getInt64Ty(), false, GlobalVariable::ExternalLinkage, nullptr,
-      "__msan_va_arg_overflow_size_tls", nullptr,
-      GlobalVariable::InitialExecTLSModel);
-  OriginTLS = new GlobalVariable(
-      M, IRB.getInt32Ty(), false, GlobalVariable::ExternalLinkage, nullptr,
-      "__msan_origin_tls", nullptr, GlobalVariable::InitialExecTLSModel);
+  RetvalTLS =
+      getOrInsertGlobal(M, "__msan_retval_tls",
+                        ArrayType::get(IRB.getInt64Ty(), kRetvalTLSSize / 8));
+
+  RetvalOriginTLS = getOrInsertGlobal(M, "__msan_retval_origin_tls", OriginTy);
+
+  ParamTLS =
+      getOrInsertGlobal(M, "__msan_param_tls",
+                        ArrayType::get(IRB.getInt64Ty(), kParamTLSSize / 8));
+
+  ParamOriginTLS =
+      getOrInsertGlobal(M, "__msan_param_origin_tls",
+                        ArrayType::get(OriginTy, kParamTLSSize / 4));
+
+  VAArgTLS =
+      getOrInsertGlobal(M, "__msan_va_arg_tls",
+                        ArrayType::get(IRB.getInt64Ty(), kParamTLSSize / 8));
+
+  VAArgOriginTLS =
+      getOrInsertGlobal(M, "__msan_va_arg_origin_tls",
+                        ArrayType::get(OriginTy, kParamTLSSize / 4));
+
+  VAArgOverflowSizeTLS =
+      getOrInsertGlobal(M, "__msan_va_arg_overflow_size_tls", IRB.getInt64Ty());
+  OriginTLS = getOrInsertGlobal(M, "__msan_origin_tls", IRB.getInt32Ty());
 
   for (size_t AccessSizeIndex = 0; AccessSizeIndex < kNumberOfAccessSizes;
        AccessSizeIndex++) {
@@ -779,9 +803,6 @@ void MemorySanitizer::initializeCallbacks(Module &M) {
                             StringRef(""), StringRef(""),
                             /*hasSideEffects=*/true);
 
-  MsanInstrumentAsmLoadFn =
-      M.getOrInsertFunction("__msan_instrument_asm_load", IRB.getVoidTy(),
-                            PointerType::get(IRB.getInt8Ty(), 0), IntptrTy);
   MsanInstrumentAsmStoreFn =
       M.getOrInsertFunction("__msan_instrument_asm_store", IRB.getVoidTy(),
                             PointerType::get(IRB.getInt8Ty(), 0), IntptrTy);
@@ -794,8 +815,9 @@ void MemorySanitizer::initializeCallbacks(Module &M) {
   CallbacksInitialized = true;
 }
 
-Value *MemorySanitizer::getKmsanShadowOriginAccessFn(bool isStore, int size) {
-  Value **Fns =
+FunctionCallee MemorySanitizer::getKmsanShadowOriginAccessFn(bool isStore,
+                                                             int size) {
+  FunctionCallee *Fns =
       isStore ? MsanMetadataPtrForStore_1_8 : MsanMetadataPtrForLoad_1_8;
   switch (size) {
   case 1:
@@ -814,7 +836,7 @@ Value *MemorySanitizer::getKmsanShadowOriginAccessFn(bool isStore, int size) {
 /// Module-level initialization.
 ///
 /// inserts a call to __msan_init to the module's constructor list.
-bool MemorySanitizer::doInitialization(Module &M) {
+void MemorySanitizer::initializeModule(Module &M) {
   auto &DL = M.getDataLayout();
 
   bool ShadowPassed = ClShadowBase.getNumOccurrences() > 0;
@@ -889,26 +911,40 @@ bool MemorySanitizer::doInitialization(Module &M) {
 
   if (!CompileKernel) {
     std::tie(MsanCtorFunction, std::ignore) =
-        createSanitizerCtorAndInitFunctions(M, kMsanModuleCtorName,
-                                            kMsanInitName,
-                                            /*InitArgTypes=*/{},
-                                            /*InitArgs=*/{});
-    if (ClWithComdat) {
-      Comdat *MsanCtorComdat = M.getOrInsertComdat(kMsanModuleCtorName);
-      MsanCtorFunction->setComdat(MsanCtorComdat);
-      appendToGlobalCtors(M, MsanCtorFunction, 0, MsanCtorFunction);
-    } else {
-      appendToGlobalCtors(M, MsanCtorFunction, 0);
-    }
+        getOrCreateSanitizerCtorAndInitFunctions(
+            M, kMsanModuleCtorName, kMsanInitName,
+            /*InitArgTypes=*/{},
+            /*InitArgs=*/{},
+            // This callback is invoked when the functions are created the first
+            // time. Hook them into the global ctors list in that case:
+            [&](Function *Ctor, FunctionCallee) {
+              if (!ClWithComdat) {
+                appendToGlobalCtors(M, Ctor, 0);
+                return;
+              }
+              Comdat *MsanCtorComdat = M.getOrInsertComdat(kMsanModuleCtorName);
+              Ctor->setComdat(MsanCtorComdat);
+              appendToGlobalCtors(M, Ctor, 0, Ctor);
+            });
 
     if (TrackOrigins)
-      new GlobalVariable(M, IRB.getInt32Ty(), true, GlobalValue::WeakODRLinkage,
-                         IRB.getInt32(TrackOrigins), "__msan_track_origins");
+      M.getOrInsertGlobal("__msan_track_origins", IRB.getInt32Ty(), [&] {
+        return new GlobalVariable(
+            M, IRB.getInt32Ty(), true, GlobalValue::WeakODRLinkage,
+            IRB.getInt32(TrackOrigins), "__msan_track_origins");
+      });
 
     if (Recover)
-      new GlobalVariable(M, IRB.getInt32Ty(), true, GlobalValue::WeakODRLinkage,
-                         IRB.getInt32(Recover), "__msan_keep_going");
-  }
+      M.getOrInsertGlobal("__msan_keep_going", IRB.getInt32Ty(), [&] {
+        return new GlobalVariable(M, IRB.getInt32Ty(), true,
+                                  GlobalValue::WeakODRLinkage,
+                                  IRB.getInt32(Recover), "__msan_keep_going");
+      });
+}
+}
+
+bool MemorySanitizerLegacyPass::doInitialization(Module &M) {
+  MSan.emplace(M, Options);
   return true;
 }
 
@@ -989,8 +1025,9 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
   SmallVector<ShadowOriginAndInsertPoint, 16> InstrumentationList;
   SmallVector<StoreInst *, 16> StoreList;
 
-  MemorySanitizerVisitor(Function &F, MemorySanitizer &MS)
-      : F(F), MS(MS), VAHelper(CreateVarArgHelper(F, MS, *this)) {
+  MemorySanitizerVisitor(Function &F, MemorySanitizer &MS,
+                         const TargetLibraryInfo &TLI)
+      : F(F), MS(MS), VAHelper(CreateVarArgHelper(F, MS, *this)), TLI(&TLI) {
     bool SanitizeFunction = F.hasFnAttribute(Attribute::SanitizeMemory);
     InsertChecks = SanitizeFunction;
     PropagateShadow = SanitizeFunction;
@@ -999,7 +1036,6 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
     // FIXME: Consider using SpecialCaseList to specify a list of functions that
     // must always return fully initialized values. For now, we hardcode "main".
     CheckReturnValue = SanitizeFunction && (F.getName() == "main");
-    TLI = &MS.getAnalysis<TargetLibraryInfoWrapperPass>().getTLI();
 
     MS.initializeCallbacks(*F.getParent());
     if (MS.CompileKernel)
@@ -1052,7 +1088,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
 
     for (unsigned i = Ofs; i < (Size + kOriginSize - 1) / kOriginSize; ++i) {
       Value *GEP =
-          i ? IRB.CreateConstGEP1_32(nullptr, OriginPtr, i) : OriginPtr;
+          i ? IRB.CreateConstGEP1_32(MS.OriginTy, OriginPtr, i) : OriginPtr;
       IRB.CreateAlignedStore(Origin, GEP, CurrentAlignment);
       CurrentAlignment = kMinOriginAlignment;
     }
@@ -1080,7 +1116,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
           DL.getTypeSizeInBits(ConvertedShadow->getType());
       unsigned SizeIndex = TypeSizeToSizeIndex(TypeSizeInBits);
       if (AsCall && SizeIndex < kNumberOfAccessSizes && !MS.CompileKernel) {
-        Value *Fn = MS.MaybeStoreOriginFn[SizeIndex];
+        FunctionCallee Fn = MS.MaybeStoreOriginFn[SizeIndex];
         Value *ConvertedShadow2 = IRB.CreateZExt(
             ConvertedShadow, IRB.getIntNTy(8 * (1 << SizeIndex)));
         IRB.CreateCall(Fn, {ConvertedShadow2,
@@ -1162,7 +1198,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
     unsigned TypeSizeInBits = DL.getTypeSizeInBits(ConvertedShadow->getType());
     unsigned SizeIndex = TypeSizeToSizeIndex(TypeSizeInBits);
     if (AsCall && SizeIndex < kNumberOfAccessSizes && !MS.CompileKernel) {
-      Value *Fn = MS.MaybeWarningFn[SizeIndex];
+      FunctionCallee Fn = MS.MaybeWarningFn[SizeIndex];
       Value *ConvertedShadow2 =
           IRB.CreateZExt(ConvertedShadow, IRB.getIntNTy(8 * (1 << SizeIndex)));
       IRB.CreateCall(Fn, {ConvertedShadow2, MS.TrackOrigins && Origin
@@ -1197,20 +1233,22 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
     IRBuilder<> IRB(F.getEntryBlock().getFirstNonPHI());
     Value *ContextState = IRB.CreateCall(MS.MsanGetContextStateFn, {});
     Constant *Zero = IRB.getInt32(0);
-    MS.ParamTLS =
-        IRB.CreateGEP(ContextState, {Zero, IRB.getInt32(0)}, "param_shadow");
-    MS.RetvalTLS =
-        IRB.CreateGEP(ContextState, {Zero, IRB.getInt32(1)}, "retval_shadow");
-    MS.VAArgTLS =
-        IRB.CreateGEP(ContextState, {Zero, IRB.getInt32(2)}, "va_arg_shadow");
-    MS.VAArgOriginTLS =
-        IRB.CreateGEP(ContextState, {Zero, IRB.getInt32(3)}, "va_arg_origin");
-    MS.VAArgOverflowSizeTLS = IRB.CreateGEP(
-        ContextState, {Zero, IRB.getInt32(4)}, "va_arg_overflow_size");
-    MS.ParamOriginTLS =
-        IRB.CreateGEP(ContextState, {Zero, IRB.getInt32(5)}, "param_origin");
+    MS.ParamTLS = IRB.CreateGEP(MS.MsanContextStateTy, ContextState,
+                                {Zero, IRB.getInt32(0)}, "param_shadow");
+    MS.RetvalTLS = IRB.CreateGEP(MS.MsanContextStateTy, ContextState,
+                                 {Zero, IRB.getInt32(1)}, "retval_shadow");
+    MS.VAArgTLS = IRB.CreateGEP(MS.MsanContextStateTy, ContextState,
+                                {Zero, IRB.getInt32(2)}, "va_arg_shadow");
+    MS.VAArgOriginTLS = IRB.CreateGEP(MS.MsanContextStateTy, ContextState,
+                                      {Zero, IRB.getInt32(3)}, "va_arg_origin");
+    MS.VAArgOverflowSizeTLS =
+        IRB.CreateGEP(MS.MsanContextStateTy, ContextState,
+                      {Zero, IRB.getInt32(4)}, "va_arg_overflow_size");
+    MS.ParamOriginTLS = IRB.CreateGEP(MS.MsanContextStateTy, ContextState,
+                                      {Zero, IRB.getInt32(5)}, "param_origin");
     MS.RetvalOriginTLS =
-        IRB.CreateGEP(ContextState, {Zero, IRB.getInt32(6)}, "retval_origin");
+        IRB.CreateGEP(MS.MsanContextStateTy, ContextState,
+                      {Zero, IRB.getInt32(6)}, "retval_origin");
     return ret;
   }
 
@@ -1357,7 +1395,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
             IRB.CreateAnd(OriginLong, ConstantInt::get(MS.IntptrTy, ~Mask));
       }
       OriginPtr =
-          IRB.CreateIntToPtr(OriginLong, PointerType::get(IRB.getInt32Ty(), 0));
+          IRB.CreateIntToPtr(OriginLong, PointerType::get(MS.OriginTy, 0));
     }
     return std::make_pair(ShadowPtr, OriginPtr);
   }
@@ -1369,7 +1407,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
     const DataLayout &DL = F.getParent()->getDataLayout();
     int Size = DL.getTypeStoreSize(ShadowTy);
 
-    Value *Getter = MS.getKmsanShadowOriginAccessFn(isStore, Size);
+    FunctionCallee Getter = MS.getKmsanShadowOriginAccessFn(isStore, Size);
     Value *AddrCast =
         IRB.CreatePointerCast(Addr, PointerType::get(IRB.getInt8Ty(), 0));
     if (Getter) {
@@ -1574,8 +1612,8 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
               // ParamTLS overflow.
               *ShadowPtr = getCleanShadow(V);
             } else {
-              *ShadowPtr =
-                  EntryIRB.CreateAlignedLoad(Base, kShadowTLSAlignment);
+              *ShadowPtr = EntryIRB.CreateAlignedLoad(getShadowTy(&FArg), Base,
+                                                      kShadowTLSAlignment);
             }
           }
           LLVM_DEBUG(dbgs()
@@ -1583,7 +1621,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
           if (MS.TrackOrigins && !Overflow) {
             Value *OriginPtr =
                 getOriginPtrForArgument(&FArg, EntryIRB, ArgOffset);
-            setOrigin(A, EntryIRB.CreateLoad(OriginPtr));
+            setOrigin(A, EntryIRB.CreateLoad(MS.OriginTy, OriginPtr));
           } else {
             setOrigin(A, getCleanOrigin());
           }
@@ -1714,7 +1752,8 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
     if (PropagateShadow) {
       std::tie(ShadowPtr, OriginPtr) =
           getShadowOriginPtr(Addr, IRB, ShadowTy, Alignment, /*isStore*/ false);
-      setShadow(&I, IRB.CreateAlignedLoad(ShadowPtr, Alignment, "_msld"));
+      setShadow(&I,
+                IRB.CreateAlignedLoad(ShadowTy, ShadowPtr, Alignment, "_msld"));
     } else {
       setShadow(&I, getCleanShadow(&I));
     }
@@ -1728,7 +1767,8 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
     if (MS.TrackOrigins) {
       if (PropagateShadow) {
         unsigned OriginAlignment = std::max(kMinOriginAlignment, Alignment);
-        setOrigin(&I, IRB.CreateAlignedLoad(OriginPtr, OriginAlignment));
+        setOrigin(
+            &I, IRB.CreateAlignedLoad(MS.OriginTy, OriginPtr, OriginAlignment));
       } else {
         setOrigin(&I, getCleanOrigin());
       }
@@ -2408,7 +2448,8 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
       unsigned Alignment = 1;
       std::tie(ShadowPtr, OriginPtr) =
           getShadowOriginPtr(Addr, IRB, ShadowTy, Alignment, /*isStore*/ false);
-      setShadow(&I, IRB.CreateAlignedLoad(ShadowPtr, Alignment, "_msld"));
+      setShadow(&I,
+                IRB.CreateAlignedLoad(ShadowTy, ShadowPtr, Alignment, "_msld"));
     } else {
       setShadow(&I, getCleanShadow(&I));
     }
@@ -2418,7 +2459,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
 
     if (MS.TrackOrigins) {
       if (PropagateShadow)
-        setOrigin(&I, IRB.CreateLoad(OriginPtr));
+        setOrigin(&I, IRB.CreateLoad(MS.OriginTy, OriginPtr));
       else
         setOrigin(&I, getCleanOrigin());
     }
@@ -2626,7 +2667,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
                              : Lower64ShadowExtend(IRB, S2, getShadowTy(&I));
     Value *V1 = I.getOperand(0);
     Value *V2 = I.getOperand(1);
-    Value *Shift = IRB.CreateCall(I.getCalledValue(),
+    Value *Shift = IRB.CreateCall(I.getFunctionType(), I.getCalledValue(),
                                   {IRB.CreateBitCast(S1, V1->getType()), V2});
     Shift = IRB.CreateBitCast(Shift, getShadowTy(&I));
     setShadow(&I, IRB.CreateOr(Shift, S2Conv));
@@ -2801,9 +2842,9 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
     if (ClCheckAccessAddress)
       insertShadowCheck(Addr, &I);
 
-    Value *Shadow = IRB.CreateAlignedLoad(ShadowPtr, Alignment, "_ldmxcsr");
-    Value *Origin =
-        MS.TrackOrigins ? IRB.CreateLoad(OriginPtr) : getCleanOrigin();
+    Value *Shadow = IRB.CreateAlignedLoad(Ty, ShadowPtr, Alignment, "_ldmxcsr");
+    Value *Origin = MS.TrackOrigins ? IRB.CreateLoad(MS.OriginTy, OriginPtr)
+                                    : getCleanOrigin();
     insertShadowCheck(Shadow, Origin, &I);
   }
 
@@ -2877,7 +2918,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
 
         Value *Origin = IRB.CreateSelect(
             IRB.CreateICmpNE(Acc, Constant::getNullValue(Acc->getType())),
-            getOrigin(PassThru), IRB.CreateLoad(OriginPtr));
+            getOrigin(PassThru), IRB.CreateLoad(MS.OriginTy, OriginPtr));
 
         setOrigin(&I, Origin);
       } else {
@@ -3103,6 +3144,12 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
       handleVectorComparePackedIntrinsic(I);
       break;
 
+    case Intrinsic::is_constant:
+      // The result of llvm.is.constant() is always defined.
+      setShadow(&I, getCleanShadow(&I));
+      setOrigin(&I, getCleanOrigin());
+      break;
+
     default:
       if (!handleUnknownIntrinsic(I))
         visitInstruction(I);
@@ -3200,8 +3247,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
     }
     LLVM_DEBUG(dbgs() << "  done with call args\n");
 
-    FunctionType *FT =
-      cast<FunctionType>(CS.getCalledValue()->getType()->getContainedType(0));
+    FunctionType *FT = CS.getFunctionType();
     if (FT->isVarArg()) {
       VAHelper->visitCallSite(CS, IRB);
     }
@@ -3235,12 +3281,13 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
              "Could not find insertion point for retval shadow load");
     }
     IRBuilder<> IRBAfter(&*NextInsn);
-    Value *RetvalShadow =
-      IRBAfter.CreateAlignedLoad(getShadowPtrForRetval(&I, IRBAfter),
-                                 kShadowTLSAlignment, "_msret");
+    Value *RetvalShadow = IRBAfter.CreateAlignedLoad(
+        getShadowTy(&I), getShadowPtrForRetval(&I, IRBAfter),
+        kShadowTLSAlignment, "_msret");
     setShadow(&I, RetvalShadow);
     if (MS.TrackOrigins)
-      setOrigin(&I, IRBAfter.CreateLoad(getOriginPtrForRetval(IRBAfter)));
+      setOrigin(&I, IRBAfter.CreateLoad(MS.OriginTy,
+                                        getOriginPtrForRetval(IRBAfter)));
   }
 
   bool isAMustTailRetVal(Value *RetVal) {
@@ -3482,19 +3529,17 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
     Type *OpType = Operand->getType();
     // Check the operand value itself.
     insertShadowCheck(Operand, &I);
-    if (!OpType->isPointerTy()) {
+    if (!OpType->isPointerTy() || !isOutput) {
       assert(!isOutput);
       return;
     }
-    Value *Hook =
-        isOutput ? MS.MsanInstrumentAsmStoreFn : MS.MsanInstrumentAsmLoadFn;
     Type *ElType = OpType->getPointerElementType();
     if (!ElType->isSized())
       return;
     int Size = DL.getTypeStoreSize(ElType);
     Value *Ptr = IRB.CreatePointerCast(Operand, IRB.getInt8PtrTy());
     Value *SizeVal = ConstantInt::get(MS.IntptrTy, Size);
-    IRB.CreateCall(Hook, {Ptr, SizeVal});
+    IRB.CreateCall(MS.MsanInstrumentAsmStoreFn, {Ptr, SizeVal});
   }
 
   /// Get the number of output arguments returned by pointers.
@@ -3790,7 +3835,8 @@ struct VarArgAMD64Helper : public VarArgHelper {
       // If there is a va_start in this function, make a backup copy of
       // va_arg_tls somewhere in the function entry block.
       IRBuilder<> IRB(MSV.ActualFnStart->getFirstNonPHI());
-      VAArgOverflowSize = IRB.CreateLoad(MS.VAArgOverflowSizeTLS);
+      VAArgOverflowSize =
+          IRB.CreateLoad(IRB.getInt64Ty(), MS.VAArgOverflowSizeTLS);
       Value *CopySize =
         IRB.CreateAdd(ConstantInt::get(MS.IntptrTy, AMD64FpEndOffset),
                       VAArgOverflowSize);
@@ -3809,11 +3855,13 @@ struct VarArgAMD64Helper : public VarArgHelper {
       IRBuilder<> IRB(OrigInst->getNextNode());
       Value *VAListTag = OrigInst->getArgOperand(0);
 
+      Type *RegSaveAreaPtrTy = Type::getInt64PtrTy(*MS.C);
       Value *RegSaveAreaPtrPtr = IRB.CreateIntToPtr(
           IRB.CreateAdd(IRB.CreatePtrToInt(VAListTag, MS.IntptrTy),
                         ConstantInt::get(MS.IntptrTy, 16)),
-          PointerType::get(Type::getInt64PtrTy(*MS.C), 0));
-      Value *RegSaveAreaPtr = IRB.CreateLoad(RegSaveAreaPtrPtr);
+          PointerType::get(RegSaveAreaPtrTy, 0));
+      Value *RegSaveAreaPtr =
+          IRB.CreateLoad(RegSaveAreaPtrTy, RegSaveAreaPtrPtr);
       Value *RegSaveAreaShadowPtr, *RegSaveAreaOriginPtr;
       unsigned Alignment = 16;
       std::tie(RegSaveAreaShadowPtr, RegSaveAreaOriginPtr) =
@@ -3824,11 +3872,13 @@ struct VarArgAMD64Helper : public VarArgHelper {
       if (MS.TrackOrigins)
         IRB.CreateMemCpy(RegSaveAreaOriginPtr, Alignment, VAArgTLSOriginCopy,
                          Alignment, AMD64FpEndOffset);
+      Type *OverflowArgAreaPtrTy = Type::getInt64PtrTy(*MS.C);
       Value *OverflowArgAreaPtrPtr = IRB.CreateIntToPtr(
           IRB.CreateAdd(IRB.CreatePtrToInt(VAListTag, MS.IntptrTy),
                         ConstantInt::get(MS.IntptrTy, 8)),
-          PointerType::get(Type::getInt64PtrTy(*MS.C), 0));
-      Value *OverflowArgAreaPtr = IRB.CreateLoad(OverflowArgAreaPtrPtr);
+          PointerType::get(OverflowArgAreaPtrTy, 0));
+      Value *OverflowArgAreaPtr =
+          IRB.CreateLoad(OverflowArgAreaPtrTy, OverflowArgAreaPtrPtr);
       Value *OverflowArgAreaShadowPtr, *OverflowArgAreaOriginPtr;
       std::tie(OverflowArgAreaShadowPtr, OverflowArgAreaOriginPtr) =
           MSV.getShadowOriginPtr(OverflowArgAreaPtr, IRB, IRB.getInt8Ty(),
@@ -3930,7 +3980,7 @@ struct VarArgMIPS64Helper : public VarArgHelper {
     assert(!VAArgSize && !VAArgTLSCopy &&
            "finalizeInstrumentation called twice");
     IRBuilder<> IRB(MSV.ActualFnStart->getFirstNonPHI());
-    VAArgSize = IRB.CreateLoad(MS.VAArgOverflowSizeTLS);
+    VAArgSize = IRB.CreateLoad(IRB.getInt64Ty(), MS.VAArgOverflowSizeTLS);
     Value *CopySize = IRB.CreateAdd(ConstantInt::get(MS.IntptrTy, 0),
                                     VAArgSize);
 
@@ -3947,10 +3997,12 @@ struct VarArgMIPS64Helper : public VarArgHelper {
       CallInst *OrigInst = VAStartInstrumentationList[i];
       IRBuilder<> IRB(OrigInst->getNextNode());
       Value *VAListTag = OrigInst->getArgOperand(0);
+      Type *RegSaveAreaPtrTy = Type::getInt64PtrTy(*MS.C);
       Value *RegSaveAreaPtrPtr =
           IRB.CreateIntToPtr(IRB.CreatePtrToInt(VAListTag, MS.IntptrTy),
-                             PointerType::get(Type::getInt64PtrTy(*MS.C), 0));
-      Value *RegSaveAreaPtr = IRB.CreateLoad(RegSaveAreaPtrPtr);
+                             PointerType::get(RegSaveAreaPtrTy, 0));
+      Value *RegSaveAreaPtr =
+          IRB.CreateLoad(RegSaveAreaPtrTy, RegSaveAreaPtrPtr);
       Value *RegSaveAreaShadowPtr, *RegSaveAreaOriginPtr;
       unsigned Alignment = 8;
       std::tie(RegSaveAreaShadowPtr, RegSaveAreaOriginPtr) =
@@ -4100,7 +4152,7 @@ struct VarArgAArch64Helper : public VarArgHelper {
         IRB.CreateAdd(IRB.CreatePtrToInt(VAListTag, MS.IntptrTy),
                       ConstantInt::get(MS.IntptrTy, offset)),
         Type::getInt64PtrTy(*MS.C));
-    return IRB.CreateLoad(SaveAreaPtrPtr);
+    return IRB.CreateLoad(Type::getInt64Ty(*MS.C), SaveAreaPtrPtr);
   }
 
   // Retrieve a va_list field of 'int' size.
@@ -4110,7 +4162,7 @@ struct VarArgAArch64Helper : public VarArgHelper {
         IRB.CreateAdd(IRB.CreatePtrToInt(VAListTag, MS.IntptrTy),
                       ConstantInt::get(MS.IntptrTy, offset)),
         Type::getInt32PtrTy(*MS.C));
-    Value *SaveArea32 = IRB.CreateLoad(SaveAreaPtr);
+    Value *SaveArea32 = IRB.CreateLoad(IRB.getInt32Ty(), SaveAreaPtr);
     return IRB.CreateSExt(SaveArea32, MS.IntptrTy);
   }
 
@@ -4121,7 +4173,8 @@ struct VarArgAArch64Helper : public VarArgHelper {
       // If there is a va_start in this function, make a backup copy of
       // va_arg_tls somewhere in the function entry block.
       IRBuilder<> IRB(MSV.ActualFnStart->getFirstNonPHI());
-      VAArgOverflowSize = IRB.CreateLoad(MS.VAArgOverflowSizeTLS);
+      VAArgOverflowSize =
+          IRB.CreateLoad(IRB.getInt64Ty(), MS.VAArgOverflowSizeTLS);
       Value *CopySize =
         IRB.CreateAdd(ConstantInt::get(MS.IntptrTy, AArch64VAEndOffset),
                       VAArgOverflowSize);
@@ -4364,7 +4417,7 @@ struct VarArgPowerPC64Helper : public VarArgHelper {
     assert(!VAArgSize && !VAArgTLSCopy &&
            "finalizeInstrumentation called twice");
     IRBuilder<> IRB(MSV.ActualFnStart->getFirstNonPHI());
-    VAArgSize = IRB.CreateLoad(MS.VAArgOverflowSizeTLS);
+    VAArgSize = IRB.CreateLoad(IRB.getInt64Ty(), MS.VAArgOverflowSizeTLS);
     Value *CopySize = IRB.CreateAdd(ConstantInt::get(MS.IntptrTy, 0),
                                     VAArgSize);
 
@@ -4381,10 +4434,12 @@ struct VarArgPowerPC64Helper : public VarArgHelper {
       CallInst *OrigInst = VAStartInstrumentationList[i];
       IRBuilder<> IRB(OrigInst->getNextNode());
       Value *VAListTag = OrigInst->getArgOperand(0);
+      Type *RegSaveAreaPtrTy = Type::getInt64PtrTy(*MS.C);
       Value *RegSaveAreaPtrPtr =
           IRB.CreateIntToPtr(IRB.CreatePtrToInt(VAListTag, MS.IntptrTy),
-                             PointerType::get(Type::getInt64PtrTy(*MS.C), 0));
-      Value *RegSaveAreaPtr = IRB.CreateLoad(RegSaveAreaPtrPtr);
+                             PointerType::get(RegSaveAreaPtrTy, 0));
+      Value *RegSaveAreaPtr =
+          IRB.CreateLoad(RegSaveAreaPtrTy, RegSaveAreaPtrPtr);
       Value *RegSaveAreaShadowPtr, *RegSaveAreaOriginPtr;
       unsigned Alignment = 8;
       std::tie(RegSaveAreaShadowPtr, RegSaveAreaOriginPtr) =
@@ -4430,10 +4485,10 @@ static VarArgHelper *CreateVarArgHelper(Function &Func, MemorySanitizer &Msan,
     return new VarArgNoOpHelper(Func, Msan, Visitor);
 }
 
-bool MemorySanitizer::runOnFunction(Function &F) {
+bool MemorySanitizer::sanitizeFunction(Function &F, TargetLibraryInfo &TLI) {
   if (!CompileKernel && (&F == MsanCtorFunction))
     return false;
-  MemorySanitizerVisitor Visitor(F, *this);
+  MemorySanitizerVisitor Visitor(F, *this, TLI);
 
   // Clear out readonly/readnone attributes.
   AttrBuilder B;
diff --git a/lib/Transforms/Instrumentation/PGOInstrumentation.cpp b/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
index 876ae23dfd29db28875eec38573d6d53751e0c89..bb2e3359881c077bd2c7f4cf3f2f7c6e10495cf9 100644
--- a/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
+++ b/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
@@ -1,9 +1,8 @@
 //===- PGOInstrumentation.cpp - MST-based PGO Instrumentation -------------===//
 //
-//                      The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -63,7 +62,7 @@
 #include "llvm/Analysis/BlockFrequencyInfo.h"
 #include "llvm/Analysis/BranchProbabilityInfo.h"
 #include "llvm/Analysis/CFG.h"
-#include "llvm/Analysis/IndirectCallSiteVisitor.h"
+#include "llvm/Analysis/IndirectCallVisitor.h"
 #include "llvm/Analysis/LoopInfo.h"
 #include "llvm/Analysis/OptimizationRemarkEmitter.h"
 #include "llvm/IR/Attributes.h"
@@ -544,7 +543,7 @@ public:
     MIVisitor.countMemIntrinsics(Func);
     NumOfPGOSelectInsts += SIVisitor.getNumOfSelectInsts();
     NumOfPGOMemIntrinsics += MIVisitor.getNumOfMemIntrinsics();
-    ValueSites[IPVK_IndirectCallTarget] = findIndirectCallSites(Func);
+    ValueSites[IPVK_IndirectCallTarget] = findIndirectCalls(Func);
     ValueSites[IPVK_MemOPSize] = MIVisitor.findMemIntrinsics(Func);
 
     FuncName = getPGOFuncName(F);
@@ -754,12 +753,12 @@ static void instrumentOneFunc(
   if (DisableValueProfiling)
     return;
 
-  unsigned NumIndirectCallSites = 0;
+  unsigned NumIndirectCalls = 0;
   for (auto &I : FuncInfo.ValueSites[IPVK_IndirectCallTarget]) {
     CallSite CS(I);
     Value *Callee = CS.getCalledValue();
     LLVM_DEBUG(dbgs() << "Instrument one indirect call: CallSite Index = "
-                      << NumIndirectCallSites << "\n");
+                      << NumIndirectCalls << "\n");
     IRBuilder<> Builder(I);
     assert(Builder.GetInsertPoint() != I->getParent()->end() &&
            "Cannot get the Instrumentation point");
@@ -769,9 +768,9 @@ static void instrumentOneFunc(
          Builder.getInt64(FuncInfo.FunctionHash),
          Builder.CreatePtrToInt(Callee, Builder.getInt64Ty()),
          Builder.getInt32(IPVK_IndirectCallTarget),
-         Builder.getInt32(NumIndirectCallSites++)});
+         Builder.getInt32(NumIndirectCalls++)});
   }
-  NumOfPGOICall += NumIndirectCallSites;
+  NumOfPGOICall += NumIndirectCalls;
 
   // Now instrument memop intrinsic calls.
   FuncInfo.MIVisitor.instrumentMemIntrinsics(
diff --git a/lib/Transforms/Instrumentation/PGOMemOPSizeOpt.cpp b/lib/Transforms/Instrumentation/PGOMemOPSizeOpt.cpp
index 2c71e75dadcc4008d6a86356d8ba62459e1f1dc5..188f95b4676bb738b0b15177571499f6ac4257d3 100644
--- a/lib/Transforms/Instrumentation/PGOMemOPSizeOpt.cpp
+++ b/lib/Transforms/Instrumentation/PGOMemOPSizeOpt.cpp
@@ -1,9 +1,8 @@
 //===-- PGOMemOPSizeOpt.cpp - Optimizations based on value profiling ===//
 //
-//                      The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -20,12 +19,12 @@
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/Twine.h"
 #include "llvm/Analysis/BlockFrequencyInfo.h"
+#include "llvm/Analysis/DomTreeUpdater.h"
 #include "llvm/Analysis/GlobalsModRef.h"
 #include "llvm/Analysis/OptimizationRemarkEmitter.h"
 #include "llvm/IR/BasicBlock.h"
 #include "llvm/IR/CallSite.h"
 #include "llvm/IR/DerivedTypes.h"
-#include "llvm/IR/DomTreeUpdater.h"
 #include "llvm/IR/Dominators.h"
 #include "llvm/IR/Function.h"
 #include "llvm/IR/IRBuilder.h"
diff --git a/lib/Transforms/Instrumentation/SanitizerCoverage.cpp b/lib/Transforms/Instrumentation/SanitizerCoverage.cpp
index 7f683ad089fbad766a3e694ab6ecd31e3a1b928d..21ad41c47f63049919d8115724103a841bc8ff1a 100644
--- a/lib/Transforms/Instrumentation/SanitizerCoverage.cpp
+++ b/lib/Transforms/Instrumentation/SanitizerCoverage.cpp
@@ -1,9 +1,8 @@
 //===-- SanitizerCoverage.cpp - coverage instrumentation for sanitizers ---===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -223,13 +222,13 @@ private:
   std::string getSectionName(const std::string &Section) const;
   std::string getSectionStart(const std::string &Section) const;
   std::string getSectionEnd(const std::string &Section) const;
-  Function *SanCovTracePCIndir;
-  Function *SanCovTracePC, *SanCovTracePCGuard;
-  Function *SanCovTraceCmpFunction[4];
-  Function *SanCovTraceConstCmpFunction[4];
-  Function *SanCovTraceDivFunction[2];
-  Function *SanCovTraceGepFunction;
-  Function *SanCovTraceSwitchFunction;
+  FunctionCallee SanCovTracePCIndir;
+  FunctionCallee SanCovTracePC, SanCovTracePCGuard;
+  FunctionCallee SanCovTraceCmpFunction[4];
+  FunctionCallee SanCovTraceConstCmpFunction[4];
+  FunctionCallee SanCovTraceDivFunction[2];
+  FunctionCallee SanCovTraceGepFunction;
+  FunctionCallee SanCovTraceSwitchFunction;
   GlobalVariable *SanCovLowestStack;
   InlineAsm *EmptyAsm;
   Type *IntptrTy, *IntptrPtrTy, *Int64Ty, *Int64PtrTy, *Int32Ty, *Int32PtrTy,
@@ -264,13 +263,13 @@ SanitizerCoverageModule::CreateSecStartEnd(Module &M, const char *Section,
   SecEnd->setVisibility(GlobalValue::HiddenVisibility);
   IRBuilder<> IRB(M.getContext());
   Value *SecEndPtr = IRB.CreatePointerCast(SecEnd, Ty);
-  if (TargetTriple.getObjectFormat() != Triple::COFF)
+  if (!TargetTriple.isOSBinFormatCOFF())
     return std::make_pair(IRB.CreatePointerCast(SecStart, Ty), SecEndPtr);
 
   // Account for the fact that on windows-msvc __start_* symbols actually
   // point to a uint64_t before the start of the array.
   auto SecStartI8Ptr = IRB.CreatePointerCast(SecStart, Int8PtrTy);
-  auto GEP = IRB.CreateGEP(SecStartI8Ptr,
+  auto GEP = IRB.CreateGEP(Int8Ty, SecStartI8Ptr,
                            ConstantInt::get(IntptrTy, sizeof(uint64_t)));
   return std::make_pair(IRB.CreatePointerCast(GEP, Ty), SecEndPtr);
 }
@@ -293,24 +292,15 @@ Function *SanitizerCoverageModule::CreateInitCallsForSections(
     appendToGlobalCtors(M, CtorFunc, SanCtorAndDtorPriority);
   }
 
-  if (TargetTriple.getObjectFormat() == Triple::COFF) {
+  if (TargetTriple.isOSBinFormatCOFF()) {
     // In COFF files, if the contructors are set as COMDAT (they are because
     // COFF supports COMDAT) and the linker flag /OPT:REF (strip unreferenced
     // functions and data) is used, the constructors get stripped. To prevent
-    // this, give the constructors weak ODR linkage and tell the linker to
-    // always include the sancov constructor. This way the linker can
-    // deduplicate the constructors but always leave one copy.
+    // this, give the constructors weak ODR linkage and ensure the linker knows
+    // to include the sancov constructor. This way the linker can deduplicate
+    // the constructors but always leave one copy.
     CtorFunc->setLinkage(GlobalValue::WeakODRLinkage);
-    SmallString<20> PartialIncDirective("/include:");
-    // Get constructor's mangled name in order to support i386.
-    SmallString<40> MangledName;
-    Mangler().getNameWithPrefix(MangledName, CtorFunc, true);
-    Twine IncDirective = PartialIncDirective + MangledName;
-    Metadata *Args[1] = {MDString::get(*C, IncDirective.str())};
-    MDNode *MetadataNode = MDNode::get(*C, Args);
-    NamedMDNode *NamedMetadata =
-        M.getOrInsertNamedMetadata("llvm.linker.options");
-    NamedMetadata->addOperand(MetadataNode);
+    appendToUsed(M, CtorFunc);
   }
   return CtorFunc;
 }
@@ -338,77 +328,74 @@ bool SanitizerCoverageModule::runOnModule(Module &M) {
   Int16Ty = IRB.getInt16Ty();
   Int8Ty = IRB.getInt8Ty();
 
-  SanCovTracePCIndir = checkSanitizerInterfaceFunction(
-      M.getOrInsertFunction(SanCovTracePCIndirName, VoidTy, IntptrTy));
+  SanCovTracePCIndir =
+      M.getOrInsertFunction(SanCovTracePCIndirName, VoidTy, IntptrTy);
+  // Make sure smaller parameters are zero-extended to i64 as required by the
+  // x86_64 ABI.
+  AttributeList SanCovTraceCmpZeroExtAL;
+  if (TargetTriple.getArch() == Triple::x86_64) {
+    SanCovTraceCmpZeroExtAL =
+        SanCovTraceCmpZeroExtAL.addParamAttribute(*C, 0, Attribute::ZExt);
+    SanCovTraceCmpZeroExtAL =
+        SanCovTraceCmpZeroExtAL.addParamAttribute(*C, 1, Attribute::ZExt);
+  }
+
   SanCovTraceCmpFunction[0] =
-      checkSanitizerInterfaceFunction(M.getOrInsertFunction(
-          SanCovTraceCmp1, VoidTy, IRB.getInt8Ty(), IRB.getInt8Ty()));
-  SanCovTraceCmpFunction[1] = checkSanitizerInterfaceFunction(
-      M.getOrInsertFunction(SanCovTraceCmp2, VoidTy, IRB.getInt16Ty(),
-                            IRB.getInt16Ty()));
-  SanCovTraceCmpFunction[2] = checkSanitizerInterfaceFunction(
-      M.getOrInsertFunction(SanCovTraceCmp4, VoidTy, IRB.getInt32Ty(),
-                            IRB.getInt32Ty()));
+      M.getOrInsertFunction(SanCovTraceCmp1, SanCovTraceCmpZeroExtAL, VoidTy,
+                            IRB.getInt8Ty(), IRB.getInt8Ty());
+  SanCovTraceCmpFunction[1] =
+      M.getOrInsertFunction(SanCovTraceCmp2, SanCovTraceCmpZeroExtAL, VoidTy,
+                            IRB.getInt16Ty(), IRB.getInt16Ty());
+  SanCovTraceCmpFunction[2] =
+      M.getOrInsertFunction(SanCovTraceCmp4, SanCovTraceCmpZeroExtAL, VoidTy,
+                            IRB.getInt32Ty(), IRB.getInt32Ty());
   SanCovTraceCmpFunction[3] =
-      checkSanitizerInterfaceFunction(M.getOrInsertFunction(
-          SanCovTraceCmp8, VoidTy, Int64Ty, Int64Ty));
-
-  SanCovTraceConstCmpFunction[0] =
-      checkSanitizerInterfaceFunction(M.getOrInsertFunction(
-          SanCovTraceConstCmp1, VoidTy, Int8Ty, Int8Ty));
-  SanCovTraceConstCmpFunction[1] =
-      checkSanitizerInterfaceFunction(M.getOrInsertFunction(
-          SanCovTraceConstCmp2, VoidTy, Int16Ty, Int16Ty));
-  SanCovTraceConstCmpFunction[2] =
-      checkSanitizerInterfaceFunction(M.getOrInsertFunction(
-          SanCovTraceConstCmp4, VoidTy, Int32Ty, Int32Ty));
+      M.getOrInsertFunction(SanCovTraceCmp8, VoidTy, Int64Ty, Int64Ty);
+
+  SanCovTraceConstCmpFunction[0] = M.getOrInsertFunction(
+      SanCovTraceConstCmp1, SanCovTraceCmpZeroExtAL, VoidTy, Int8Ty, Int8Ty);
+  SanCovTraceConstCmpFunction[1] = M.getOrInsertFunction(
+      SanCovTraceConstCmp2, SanCovTraceCmpZeroExtAL, VoidTy, Int16Ty, Int16Ty);
+  SanCovTraceConstCmpFunction[2] = M.getOrInsertFunction(
+      SanCovTraceConstCmp4, SanCovTraceCmpZeroExtAL, VoidTy, Int32Ty, Int32Ty);
   SanCovTraceConstCmpFunction[3] =
-      checkSanitizerInterfaceFunction(M.getOrInsertFunction(
-          SanCovTraceConstCmp8, VoidTy, Int64Ty, Int64Ty));
-
-  SanCovTraceDivFunction[0] =
-      checkSanitizerInterfaceFunction(M.getOrInsertFunction(
-          SanCovTraceDiv4, VoidTy, IRB.getInt32Ty()));
+      M.getOrInsertFunction(SanCovTraceConstCmp8, VoidTy, Int64Ty, Int64Ty);
+
+  {
+    AttributeList AL;
+    if (TargetTriple.getArch() == Triple::x86_64)
+      AL = AL.addParamAttribute(*C, 0, Attribute::ZExt);
+    SanCovTraceDivFunction[0] =
+        M.getOrInsertFunction(SanCovTraceDiv4, AL, VoidTy, IRB.getInt32Ty());
+  }
   SanCovTraceDivFunction[1] =
-      checkSanitizerInterfaceFunction(M.getOrInsertFunction(
-          SanCovTraceDiv8, VoidTy, Int64Ty));
+      M.getOrInsertFunction(SanCovTraceDiv8, VoidTy, Int64Ty);
   SanCovTraceGepFunction =
-      checkSanitizerInterfaceFunction(M.getOrInsertFunction(
-          SanCovTraceGep, VoidTy, IntptrTy));
+      M.getOrInsertFunction(SanCovTraceGep, VoidTy, IntptrTy);
   SanCovTraceSwitchFunction =
-      checkSanitizerInterfaceFunction(M.getOrInsertFunction(
-          SanCovTraceSwitchName, VoidTy, Int64Ty, Int64PtrTy));
+      M.getOrInsertFunction(SanCovTraceSwitchName, VoidTy, Int64Ty, Int64PtrTy);
 
   Constant *SanCovLowestStackConstant =
       M.getOrInsertGlobal(SanCovLowestStackName, IntptrTy);
-  SanCovLowestStack = cast<GlobalVariable>(SanCovLowestStackConstant);
+  SanCovLowestStack = dyn_cast<GlobalVariable>(SanCovLowestStackConstant);
+  if (!SanCovLowestStack) {
+    C->emitError(StringRef("'") + SanCovLowestStackName +
+                 "' should not be declared by the user");
+    return true;
+  }
   SanCovLowestStack->setThreadLocalMode(
       GlobalValue::ThreadLocalMode::InitialExecTLSModel);
   if (Options.StackDepth && !SanCovLowestStack->isDeclaration())
     SanCovLowestStack->setInitializer(Constant::getAllOnesValue(IntptrTy));
 
-  // Make sure smaller parameters are zero-extended to i64 as required by the
-  // x86_64 ABI.
-  if (TargetTriple.getArch() == Triple::x86_64) {
-    for (int i = 0; i < 3; i++) {
-      SanCovTraceCmpFunction[i]->addParamAttr(0, Attribute::ZExt);
-      SanCovTraceCmpFunction[i]->addParamAttr(1, Attribute::ZExt);
-      SanCovTraceConstCmpFunction[i]->addParamAttr(0, Attribute::ZExt);
-      SanCovTraceConstCmpFunction[i]->addParamAttr(1, Attribute::ZExt);
-    }
-    SanCovTraceDivFunction[0]->addParamAttr(0, Attribute::ZExt);
-  }
-
-
   // We insert an empty inline asm after cov callbacks to avoid callback merge.
   EmptyAsm = InlineAsm::get(FunctionType::get(IRB.getVoidTy(), false),
                             StringRef(""), StringRef(""),
                             /*hasSideEffects=*/true);
 
-  SanCovTracePC = checkSanitizerInterfaceFunction(
-      M.getOrInsertFunction(SanCovTracePCName, VoidTy));
-  SanCovTracePCGuard = checkSanitizerInterfaceFunction(M.getOrInsertFunction(
-      SanCovTracePCGuardName, VoidTy, Int32PtrTy));
+  SanCovTracePC = M.getOrInsertFunction(SanCovTracePCName, VoidTy);
+  SanCovTracePCGuard =
+      M.getOrInsertFunction(SanCovTracePCGuardName, VoidTy, Int32PtrTy);
 
   for (auto &F : M)
     runOnFunction(F);
@@ -423,7 +410,7 @@ bool SanitizerCoverageModule::runOnModule(Module &M) {
                                       SanCovCountersSectionName);
   if (Ctor && Options.PCTable) {
     auto SecStartEnd = CreateSecStartEnd(M, SanCovPCsSectionName, IntptrPtrTy);
-    Function *InitFunction = declareSanitizerInitFunction(
+    FunctionCallee InitFunction = declareSanitizerInitFunction(
         M, SanCovPCsInitName, {IntptrPtrTy, IntptrPtrTy});
     IRBuilder<> IRBCtor(Ctor->getEntryBlock().getTerminator());
     IRBCtor.CreateCall(InitFunction, {SecStartEnd.first, SecStartEnd.second});
@@ -493,6 +480,37 @@ static bool shouldInstrumentBlock(const Function &F, const BasicBlock *BB,
     && !(isFullPostDominator(BB, PDT) && !BB->getSinglePredecessor());
 }
 
+
+// Returns true iff From->To is a backedge.
+// A twist here is that we treat From->To as a backedge if
+//   * To dominates From or
+//   * To->UniqueSuccessor dominates From
+static bool IsBackEdge(BasicBlock *From, BasicBlock *To,
+                       const DominatorTree *DT) {
+  if (DT->dominates(To, From))
+    return true;
+  if (auto Next = To->getUniqueSuccessor())
+    if (DT->dominates(Next, From))
+      return true;
+  return false;
+}
+
+// Prunes uninteresting Cmp instrumentation:
+//   * CMP instructions that feed into loop backedge branch.
+//
+// Note that Cmp pruning is controlled by the same flag as the
+// BB pruning.
+static bool IsInterestingCmp(ICmpInst *CMP, const DominatorTree *DT,
+                             const SanitizerCoverageOptions &Options) {
+  if (!Options.NoPrune)
+    if (CMP->hasOneUse())
+      if (auto BR = dyn_cast<BranchInst>(CMP->user_back()))
+        for (BasicBlock *B : BR->successors())
+          if (IsBackEdge(BR->getParent(), B, DT))
+            return false;
+  return true;
+}
+
 bool SanitizerCoverageModule::runOnFunction(Function &F) {
   if (F.empty())
     return false;
@@ -541,8 +559,9 @@ bool SanitizerCoverageModule::runOnFunction(Function &F) {
           IndirCalls.push_back(&Inst);
       }
       if (Options.TraceCmp) {
-        if (isa<ICmpInst>(&Inst))
-          CmpTraceTargets.push_back(&Inst);
+        if (ICmpInst *CMP = dyn_cast<ICmpInst>(&Inst))
+          if (IsInterestingCmp(CMP, DT, Options))
+            CmpTraceTargets.push_back(&Inst);
         if (isa<SwitchInst>(&Inst))
           SwitchTraceTargets.push_back(&Inst);
       }
@@ -577,7 +596,7 @@ GlobalVariable *SanitizerCoverageModule::CreateFunctionLocalArrayInSection(
       *CurModule, ArrayTy, false, GlobalVariable::PrivateLinkage,
       Constant::getNullValue(ArrayTy), "__sancov_gen_");
 
-  if (TargetTriple.supportsCOMDAT())
+  if (TargetTriple.supportsCOMDAT() && !F.isInterposable())
     if (auto Comdat =
             GetOrCreateFunctionComdat(F, TargetTriple, CurModuleUniqueId))
       Array->setComdat(Comdat);
@@ -806,9 +825,9 @@ void SanitizerCoverageModule::InjectCoverageAtBlock(Function &F, BasicBlock &BB,
   }
   if (Options.Inline8bitCounters) {
     auto CounterPtr = IRB.CreateGEP(
-        Function8bitCounterArray,
+        Function8bitCounterArray->getValueType(), Function8bitCounterArray,
         {ConstantInt::get(IntptrTy, 0), ConstantInt::get(IntptrTy, Idx)});
-    auto Load = IRB.CreateLoad(CounterPtr);
+    auto Load = IRB.CreateLoad(Int8Ty, CounterPtr);
     auto Inc = IRB.CreateAdd(Load, ConstantInt::get(Int8Ty, 1));
     auto Store = IRB.CreateStore(Inc, CounterPtr);
     SetNoSanitizeMetadata(Load);
@@ -821,7 +840,7 @@ void SanitizerCoverageModule::InjectCoverageAtBlock(Function &F, BasicBlock &BB,
     auto FrameAddrPtr =
         IRB.CreateCall(GetFrameAddr, {Constant::getNullValue(Int32Ty)});
     auto FrameAddrInt = IRB.CreatePtrToInt(FrameAddrPtr, IntptrTy);
-    auto LowestStack = IRB.CreateLoad(SanCovLowestStack);
+    auto LowestStack = IRB.CreateLoad(IntptrTy, SanCovLowestStack);
     auto IsStackLower = IRB.CreateICmpULT(FrameAddrInt, LowestStack);
     auto ThenTerm = SplitBlockAndInsertIfThen(IsStackLower, &*IP, false);
     IRBuilder<> ThenIRB(ThenTerm);
@@ -833,7 +852,7 @@ void SanitizerCoverageModule::InjectCoverageAtBlock(Function &F, BasicBlock &BB,
 
 std::string
 SanitizerCoverageModule::getSectionName(const std::string &Section) const {
-  if (TargetTriple.getObjectFormat() == Triple::COFF) {
+  if (TargetTriple.isOSBinFormatCOFF()) {
     if (Section == SanCovCountersSectionName)
       return ".SCOV$CM";
     if (Section == SanCovPCsSectionName)
diff --git a/lib/Transforms/Instrumentation/ThreadSanitizer.cpp b/lib/Transforms/Instrumentation/ThreadSanitizer.cpp
index fa1e5a157a0fc39777b72a39042ea091f6ec230c..5be13fa745cbdf37e5bf39db60abcfeda44de8a5 100644
--- a/lib/Transforms/Instrumentation/ThreadSanitizer.cpp
+++ b/lib/Transforms/Instrumentation/ThreadSanitizer.cpp
@@ -1,9 +1,8 @@
 //===-- ThreadSanitizer.cpp - race detector -------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -19,6 +18,7 @@
 // The rest is handled by the run-time library.
 //===----------------------------------------------------------------------===//
 
+#include "llvm/Transforms/Instrumentation/ThreadSanitizer.h"
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/SmallVector.h"
@@ -86,15 +86,16 @@ static const char *const kTsanInitName = "__tsan_init";
 namespace {
 
 /// ThreadSanitizer: instrument the code in module to find races.
-struct ThreadSanitizer : public FunctionPass {
-  ThreadSanitizer() : FunctionPass(ID) {}
-  StringRef getPassName() const override;
-  void getAnalysisUsage(AnalysisUsage &AU) const override;
-  bool runOnFunction(Function &F) override;
-  bool doInitialization(Module &M) override;
-  static char ID;  // Pass identification, replacement for typeid.
-
- private:
+///
+/// Instantiating ThreadSanitizer inserts the tsan runtime library API function
+/// declarations into the module if they don't exist already. Instantiating
+/// ensures the __tsan_init function is in the list of global constructors for
+/// the module.
+struct ThreadSanitizer {
+  ThreadSanitizer(Module &M);
+  bool sanitizeFunction(Function &F, const TargetLibraryInfo &TLI);
+
+private:
   void initializeCallbacks(Module &M);
   bool instrumentLoadOrStore(Instruction *I, const DataLayout &DL);
   bool instrumentAtomic(Instruction *I, const DataLayout &DL);
@@ -109,48 +110,77 @@ struct ThreadSanitizer : public FunctionPass {
   Type *IntptrTy;
   IntegerType *OrdTy;
   // Callbacks to run-time library are computed in doInitialization.
-  Function *TsanFuncEntry;
-  Function *TsanFuncExit;
-  Function *TsanIgnoreBegin;
-  Function *TsanIgnoreEnd;
+  FunctionCallee TsanFuncEntry;
+  FunctionCallee TsanFuncExit;
+  FunctionCallee TsanIgnoreBegin;
+  FunctionCallee TsanIgnoreEnd;
   // Accesses sizes are powers of two: 1, 2, 4, 8, 16.
   static const size_t kNumberOfAccessSizes = 5;
-  Function *TsanRead[kNumberOfAccessSizes];
-  Function *TsanWrite[kNumberOfAccessSizes];
-  Function *TsanUnalignedRead[kNumberOfAccessSizes];
-  Function *TsanUnalignedWrite[kNumberOfAccessSizes];
-  Function *TsanAtomicLoad[kNumberOfAccessSizes];
-  Function *TsanAtomicStore[kNumberOfAccessSizes];
-  Function *TsanAtomicRMW[AtomicRMWInst::LAST_BINOP + 1][kNumberOfAccessSizes];
-  Function *TsanAtomicCAS[kNumberOfAccessSizes];
-  Function *TsanAtomicThreadFence;
-  Function *TsanAtomicSignalFence;
-  Function *TsanVptrUpdate;
-  Function *TsanVptrLoad;
-  Function *MemmoveFn, *MemcpyFn, *MemsetFn;
+  FunctionCallee TsanRead[kNumberOfAccessSizes];
+  FunctionCallee TsanWrite[kNumberOfAccessSizes];
+  FunctionCallee TsanUnalignedRead[kNumberOfAccessSizes];
+  FunctionCallee TsanUnalignedWrite[kNumberOfAccessSizes];
+  FunctionCallee TsanAtomicLoad[kNumberOfAccessSizes];
+  FunctionCallee TsanAtomicStore[kNumberOfAccessSizes];
+  FunctionCallee TsanAtomicRMW[AtomicRMWInst::LAST_BINOP + 1]
+                              [kNumberOfAccessSizes];
+  FunctionCallee TsanAtomicCAS[kNumberOfAccessSizes];
+  FunctionCallee TsanAtomicThreadFence;
+  FunctionCallee TsanAtomicSignalFence;
+  FunctionCallee TsanVptrUpdate;
+  FunctionCallee TsanVptrLoad;
+  FunctionCallee MemmoveFn, MemcpyFn, MemsetFn;
   Function *TsanCtorFunction;
 };
+
+struct ThreadSanitizerLegacyPass : FunctionPass {
+  ThreadSanitizerLegacyPass() : FunctionPass(ID) {}
+  StringRef getPassName() const override;
+  void getAnalysisUsage(AnalysisUsage &AU) const override;
+  bool runOnFunction(Function &F) override;
+  bool doInitialization(Module &M) override;
+  static char ID; // Pass identification, replacement for typeid.
+private:
+  Optional<ThreadSanitizer> TSan;
+};
 }  // namespace
 
-char ThreadSanitizer::ID = 0;
-INITIALIZE_PASS_BEGIN(
-    ThreadSanitizer, "tsan",
-    "ThreadSanitizer: detects data races.",
-    false, false)
+PreservedAnalyses ThreadSanitizerPass::run(Function &F,
+                                           FunctionAnalysisManager &FAM) {
+  ThreadSanitizer TSan(*F.getParent());
+  if (TSan.sanitizeFunction(F, FAM.getResult<TargetLibraryAnalysis>(F)))
+    return PreservedAnalyses::none();
+  return PreservedAnalyses::all();
+}
+
+char ThreadSanitizerLegacyPass::ID = 0;
+INITIALIZE_PASS_BEGIN(ThreadSanitizerLegacyPass, "tsan",
+                      "ThreadSanitizer: detects data races.", false, false)
 INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)
-INITIALIZE_PASS_END(
-    ThreadSanitizer, "tsan",
-    "ThreadSanitizer: detects data races.",
-    false, false)
+INITIALIZE_PASS_END(ThreadSanitizerLegacyPass, "tsan",
+                    "ThreadSanitizer: detects data races.", false, false)
 
-StringRef ThreadSanitizer::getPassName() const { return "ThreadSanitizer"; }
+StringRef ThreadSanitizerLegacyPass::getPassName() const {
+  return "ThreadSanitizerLegacyPass";
+}
 
-void ThreadSanitizer::getAnalysisUsage(AnalysisUsage &AU) const {
+void ThreadSanitizerLegacyPass::getAnalysisUsage(AnalysisUsage &AU) const {
   AU.addRequired<TargetLibraryInfoWrapperPass>();
 }
 
-FunctionPass *llvm::createThreadSanitizerPass() {
-  return new ThreadSanitizer();
+bool ThreadSanitizerLegacyPass::doInitialization(Module &M) {
+  TSan.emplace(M);
+  return true;
+}
+
+bool ThreadSanitizerLegacyPass::runOnFunction(Function &F) {
+  auto &TLI = getAnalysis<TargetLibraryInfoWrapperPass>().getTLI();
+  TSan->sanitizeFunction(F, TLI);
+  return true;
+}
+
+FunctionPass *llvm::createThreadSanitizerLegacyPassPass() {
+  return new ThreadSanitizerLegacyPass();
 }
 
 void ThreadSanitizer::initializeCallbacks(Module &M) {
@@ -159,14 +189,14 @@ void ThreadSanitizer::initializeCallbacks(Module &M) {
   Attr = Attr.addAttribute(M.getContext(), AttributeList::FunctionIndex,
                            Attribute::NoUnwind);
   // Initialize the callbacks.
-  TsanFuncEntry = checkSanitizerInterfaceFunction(M.getOrInsertFunction(
-      "__tsan_func_entry", Attr, IRB.getVoidTy(), IRB.getInt8PtrTy()));
-  TsanFuncExit = checkSanitizerInterfaceFunction(
-      M.getOrInsertFunction("__tsan_func_exit", Attr, IRB.getVoidTy()));
-  TsanIgnoreBegin = checkSanitizerInterfaceFunction(M.getOrInsertFunction(
-      "__tsan_ignore_thread_begin", Attr, IRB.getVoidTy()));
-  TsanIgnoreEnd = checkSanitizerInterfaceFunction(M.getOrInsertFunction(
-      "__tsan_ignore_thread_end", Attr, IRB.getVoidTy()));
+  TsanFuncEntry = M.getOrInsertFunction("__tsan_func_entry", Attr,
+                                        IRB.getVoidTy(), IRB.getInt8PtrTy());
+  TsanFuncExit =
+      M.getOrInsertFunction("__tsan_func_exit", Attr, IRB.getVoidTy());
+  TsanIgnoreBegin = M.getOrInsertFunction("__tsan_ignore_thread_begin", Attr,
+                                          IRB.getVoidTy());
+  TsanIgnoreEnd =
+      M.getOrInsertFunction("__tsan_ignore_thread_end", Attr, IRB.getVoidTy());
   OrdTy = IRB.getInt32Ty();
   for (size_t i = 0; i < kNumberOfAccessSizes; ++i) {
     const unsigned ByteSize = 1U << i;
@@ -174,32 +204,30 @@ void ThreadSanitizer::initializeCallbacks(Module &M) {
     std::string ByteSizeStr = utostr(ByteSize);
     std::string BitSizeStr = utostr(BitSize);
     SmallString<32> ReadName("__tsan_read" + ByteSizeStr);
-    TsanRead[i] = checkSanitizerInterfaceFunction(M.getOrInsertFunction(
-        ReadName, Attr, IRB.getVoidTy(), IRB.getInt8PtrTy()));
+    TsanRead[i] = M.getOrInsertFunction(ReadName, Attr, IRB.getVoidTy(),
+                                        IRB.getInt8PtrTy());
 
     SmallString<32> WriteName("__tsan_write" + ByteSizeStr);
-    TsanWrite[i] = checkSanitizerInterfaceFunction(M.getOrInsertFunction(
-        WriteName, Attr, IRB.getVoidTy(), IRB.getInt8PtrTy()));
+    TsanWrite[i] = M.getOrInsertFunction(WriteName, Attr, IRB.getVoidTy(),
+                                         IRB.getInt8PtrTy());
 
     SmallString<64> UnalignedReadName("__tsan_unaligned_read" + ByteSizeStr);
-    TsanUnalignedRead[i] =
-        checkSanitizerInterfaceFunction(M.getOrInsertFunction(
-            UnalignedReadName, Attr, IRB.getVoidTy(), IRB.getInt8PtrTy()));
+    TsanUnalignedRead[i] = M.getOrInsertFunction(
+        UnalignedReadName, Attr, IRB.getVoidTy(), IRB.getInt8PtrTy());
 
     SmallString<64> UnalignedWriteName("__tsan_unaligned_write" + ByteSizeStr);
-    TsanUnalignedWrite[i] =
-        checkSanitizerInterfaceFunction(M.getOrInsertFunction(
-            UnalignedWriteName, Attr, IRB.getVoidTy(), IRB.getInt8PtrTy()));
+    TsanUnalignedWrite[i] = M.getOrInsertFunction(
+        UnalignedWriteName, Attr, IRB.getVoidTy(), IRB.getInt8PtrTy());
 
     Type *Ty = Type::getIntNTy(M.getContext(), BitSize);
     Type *PtrTy = Ty->getPointerTo();
     SmallString<32> AtomicLoadName("__tsan_atomic" + BitSizeStr + "_load");
-    TsanAtomicLoad[i] = checkSanitizerInterfaceFunction(
-        M.getOrInsertFunction(AtomicLoadName, Attr, Ty, PtrTy, OrdTy));
+    TsanAtomicLoad[i] =
+        M.getOrInsertFunction(AtomicLoadName, Attr, Ty, PtrTy, OrdTy);
 
     SmallString<32> AtomicStoreName("__tsan_atomic" + BitSizeStr + "_store");
-    TsanAtomicStore[i] = checkSanitizerInterfaceFunction(M.getOrInsertFunction(
-        AtomicStoreName, Attr, IRB.getVoidTy(), PtrTy, Ty, OrdTy));
+    TsanAtomicStore[i] = M.getOrInsertFunction(
+        AtomicStoreName, Attr, IRB.getVoidTy(), PtrTy, Ty, OrdTy);
 
     for (int op = AtomicRMWInst::FIRST_BINOP;
         op <= AtomicRMWInst::LAST_BINOP; ++op) {
@@ -222,46 +250,48 @@ void ThreadSanitizer::initializeCallbacks(Module &M) {
       else
         continue;
       SmallString<32> RMWName("__tsan_atomic" + itostr(BitSize) + NamePart);
-      TsanAtomicRMW[op][i] = checkSanitizerInterfaceFunction(
-          M.getOrInsertFunction(RMWName, Attr, Ty, PtrTy, Ty, OrdTy));
+      TsanAtomicRMW[op][i] =
+          M.getOrInsertFunction(RMWName, Attr, Ty, PtrTy, Ty, OrdTy);
     }
 
     SmallString<32> AtomicCASName("__tsan_atomic" + BitSizeStr +
                                   "_compare_exchange_val");
-    TsanAtomicCAS[i] = checkSanitizerInterfaceFunction(M.getOrInsertFunction(
-        AtomicCASName, Attr, Ty, PtrTy, Ty, Ty, OrdTy, OrdTy));
+    TsanAtomicCAS[i] = M.getOrInsertFunction(AtomicCASName, Attr, Ty, PtrTy, Ty,
+                                             Ty, OrdTy, OrdTy);
   }
-  TsanVptrUpdate = checkSanitizerInterfaceFunction(
+  TsanVptrUpdate =
       M.getOrInsertFunction("__tsan_vptr_update", Attr, IRB.getVoidTy(),
-                            IRB.getInt8PtrTy(), IRB.getInt8PtrTy()));
-  TsanVptrLoad = checkSanitizerInterfaceFunction(M.getOrInsertFunction(
-      "__tsan_vptr_read", Attr, IRB.getVoidTy(), IRB.getInt8PtrTy()));
-  TsanAtomicThreadFence = checkSanitizerInterfaceFunction(M.getOrInsertFunction(
-      "__tsan_atomic_thread_fence", Attr, IRB.getVoidTy(), OrdTy));
-  TsanAtomicSignalFence = checkSanitizerInterfaceFunction(M.getOrInsertFunction(
-      "__tsan_atomic_signal_fence", Attr, IRB.getVoidTy(), OrdTy));
-
-  MemmoveFn = checkSanitizerInterfaceFunction(
-      M.getOrInsertFunction("memmove", Attr, IRB.getInt8PtrTy(), IRB.getInt8PtrTy(),
-                            IRB.getInt8PtrTy(), IntptrTy));
-  MemcpyFn = checkSanitizerInterfaceFunction(
-      M.getOrInsertFunction("memcpy", Attr, IRB.getInt8PtrTy(), IRB.getInt8PtrTy(),
-                            IRB.getInt8PtrTy(), IntptrTy));
-  MemsetFn = checkSanitizerInterfaceFunction(
-      M.getOrInsertFunction("memset", Attr, IRB.getInt8PtrTy(), IRB.getInt8PtrTy(),
-                            IRB.getInt32Ty(), IntptrTy));
+                            IRB.getInt8PtrTy(), IRB.getInt8PtrTy());
+  TsanVptrLoad = M.getOrInsertFunction("__tsan_vptr_read", Attr,
+                                       IRB.getVoidTy(), IRB.getInt8PtrTy());
+  TsanAtomicThreadFence = M.getOrInsertFunction("__tsan_atomic_thread_fence",
+                                                Attr, IRB.getVoidTy(), OrdTy);
+  TsanAtomicSignalFence = M.getOrInsertFunction("__tsan_atomic_signal_fence",
+                                                Attr, IRB.getVoidTy(), OrdTy);
+
+  MemmoveFn =
+      M.getOrInsertFunction("memmove", Attr, IRB.getInt8PtrTy(),
+                            IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), IntptrTy);
+  MemcpyFn =
+      M.getOrInsertFunction("memcpy", Attr, IRB.getInt8PtrTy(),
+                            IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), IntptrTy);
+  MemsetFn =
+      M.getOrInsertFunction("memset", Attr, IRB.getInt8PtrTy(),
+                            IRB.getInt8PtrTy(), IRB.getInt32Ty(), IntptrTy);
 }
 
-bool ThreadSanitizer::doInitialization(Module &M) {
+ThreadSanitizer::ThreadSanitizer(Module &M) {
   const DataLayout &DL = M.getDataLayout();
   IntptrTy = DL.getIntPtrType(M.getContext());
-  std::tie(TsanCtorFunction, std::ignore) = createSanitizerCtorAndInitFunctions(
-      M, kTsanModuleCtorName, kTsanInitName, /*InitArgTypes=*/{},
-      /*InitArgs=*/{});
-
-  appendToGlobalCtors(M, TsanCtorFunction, 0);
-
-  return true;
+  std::tie(TsanCtorFunction, std::ignore) =
+      getOrCreateSanitizerCtorAndInitFunctions(
+          M, kTsanModuleCtorName, kTsanInitName, /*InitArgTypes=*/{},
+          /*InitArgs=*/{},
+          // This callback is invoked when the functions are created the first
+          // time. Hook them into the global ctors list in that case:
+          [&](Function *Ctor, FunctionCallee) {
+            appendToGlobalCtors(M, Ctor, 0);
+          });
 }
 
 static bool isVtableAccess(Instruction *I) {
@@ -402,7 +432,8 @@ void ThreadSanitizer::InsertRuntimeIgnores(Function &F) {
   }
 }
 
-bool ThreadSanitizer::runOnFunction(Function &F) {
+bool ThreadSanitizer::sanitizeFunction(Function &F,
+                                       const TargetLibraryInfo &TLI) {
   // This is required to prevent instrumenting call to __tsan_init from within
   // the module constructor.
   if (&F == TsanCtorFunction)
@@ -416,8 +447,6 @@ bool ThreadSanitizer::runOnFunction(Function &F) {
   bool HasCalls = false;
   bool SanitizeFunction = F.hasFnAttribute(Attribute::SanitizeThread);
   const DataLayout &DL = F.getParent()->getDataLayout();
-  const TargetLibraryInfo *TLI =
-      &getAnalysis<TargetLibraryInfoWrapperPass>().getTLI();
 
   // Traverse all instructions, collect loads/stores/returns, check for calls.
   for (auto &BB : F) {
@@ -428,7 +457,7 @@ bool ThreadSanitizer::runOnFunction(Function &F) {
         LocalLoadsAndStores.push_back(&Inst);
       else if (isa<CallInst>(Inst) || isa<InvokeInst>(Inst)) {
         if (CallInst *CI = dyn_cast<CallInst>(&Inst))
-          maybeMarkSanitizerLibraryCallNoBuiltin(CI, TLI);
+          maybeMarkSanitizerLibraryCallNoBuiltin(CI, &TLI);
         if (isa<MemIntrinsic>(Inst))
           MemIntrinCalls.push_back(&Inst);
         HasCalls = true;
@@ -530,7 +559,7 @@ bool ThreadSanitizer::instrumentLoadOrStore(Instruction *I,
       : cast<LoadInst>(I)->getAlignment();
   Type *OrigTy = cast<PointerType>(Addr->getType())->getElementType();
   const uint32_t TypeSize = DL.getTypeStoreSizeInBits(OrigTy);
-  Value *OnAccessFunc = nullptr;
+  FunctionCallee OnAccessFunc = nullptr;
   if (Alignment == 0 || Alignment >= 8 || (Alignment % (TypeSize / 8)) == 0)
     OnAccessFunc = IsWrite ? TsanWrite[Idx] : TsanRead[Idx];
   else
@@ -630,7 +659,7 @@ bool ThreadSanitizer::instrumentAtomic(Instruction *I, const DataLayout &DL) {
     int Idx = getMemoryAccessFuncIndex(Addr, DL);
     if (Idx < 0)
       return false;
-    Function *F = TsanAtomicRMW[RMWI->getOperation()][Idx];
+    FunctionCallee F = TsanAtomicRMW[RMWI->getOperation()][Idx];
     if (!F)
       return false;
     const unsigned ByteSize = 1U << Idx;
@@ -677,8 +706,9 @@ bool ThreadSanitizer::instrumentAtomic(Instruction *I, const DataLayout &DL) {
     I->eraseFromParent();
   } else if (FenceInst *FI = dyn_cast<FenceInst>(I)) {
     Value *Args[] = {createOrdering(&IRB, FI->getOrdering())};
-    Function *F = FI->getSyncScopeID() == SyncScope::SingleThread ?
-        TsanAtomicSignalFence : TsanAtomicThreadFence;
+    FunctionCallee F = FI->getSyncScopeID() == SyncScope::SingleThread
+                           ? TsanAtomicSignalFence
+                           : TsanAtomicThreadFence;
     CallInst *C = CallInst::Create(F, Args);
     ReplaceInstWithInst(I, C);
   }
diff --git a/lib/Transforms/LLVMBuild.txt b/lib/Transforms/LLVMBuild.txt
index f061c6d9285e3ea94bcf42d0c6f539194c003683..75c614fd2c47db7495b37d0608d40bf79074ec21 100644
--- a/lib/Transforms/LLVMBuild.txt
+++ b/lib/Transforms/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/Transforms/LLVMBuild.txt ---------------------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Transforms/ObjCARC/ARCRuntimeEntryPoints.h b/lib/Transforms/ObjCARC/ARCRuntimeEntryPoints.h
index 7f6b157304a3808589fe4245e6a120bc9469af18..e1e95cd6a4075d652dcd73cfc5564594c04f8b63 100644
--- a/lib/Transforms/ObjCARC/ARCRuntimeEntryPoints.h
+++ b/lib/Transforms/ObjCARC/ARCRuntimeEntryPoints.h
@@ -1,9 +1,8 @@
 //===- ARCRuntimeEntryPoints.h - ObjC ARC Optimization ----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -34,7 +33,7 @@
 
 namespace llvm {
 
-class Constant;
+class Function;
 class LLVMContext;
 
 namespace objcarc {
@@ -70,7 +69,7 @@ public:
     RetainAutoreleaseRV = nullptr;
   }
 
-  Constant *get(ARCRuntimeEntryPointKind kind) {
+  Function *get(ARCRuntimeEntryPointKind kind) {
     assert(TheModule != nullptr && "Not initialized.");
 
     switch (kind) {
@@ -106,33 +105,33 @@ private:
   Module *TheModule = nullptr;
 
   /// Declaration for ObjC runtime function objc_autoreleaseReturnValue.
-  Constant *AutoreleaseRV = nullptr;
+  Function *AutoreleaseRV = nullptr;
 
   /// Declaration for ObjC runtime function objc_release.
-  Constant *Release = nullptr;
+  Function *Release = nullptr;
 
   /// Declaration for ObjC runtime function objc_retain.
-  Constant *Retain = nullptr;
+  Function *Retain = nullptr;
 
   /// Declaration for ObjC runtime function objc_retainBlock.
-  Constant *RetainBlock = nullptr;
+  Function *RetainBlock = nullptr;
 
   /// Declaration for ObjC runtime function objc_autorelease.
-  Constant *Autorelease = nullptr;
+  Function *Autorelease = nullptr;
 
   /// Declaration for objc_storeStrong().
-  Constant *StoreStrong = nullptr;
+  Function *StoreStrong = nullptr;
 
   /// Declaration for objc_retainAutoreleasedReturnValue().
-  Constant *RetainRV = nullptr;
+  Function *RetainRV = nullptr;
 
   /// Declaration for objc_retainAutorelease().
-  Constant *RetainAutorelease = nullptr;
+  Function *RetainAutorelease = nullptr;
 
   /// Declaration for objc_retainAutoreleaseReturnValue().
-  Constant *RetainAutoreleaseRV = nullptr;
+  Function *RetainAutoreleaseRV = nullptr;
 
-  Constant *getIntrinsicEntryPoint(Constant *&Decl, Intrinsic::ID IntID) {
+  Function *getIntrinsicEntryPoint(Function *&Decl, Intrinsic::ID IntID) {
     if (Decl)
       return Decl;
 
diff --git a/lib/Transforms/ObjCARC/BlotMapVector.h b/lib/Transforms/ObjCARC/BlotMapVector.h
index 9ade14c1177a528015bb84792884f2b9e533bcfa..2fa07cfb32c0c80f7533af215c067d5bca659707 100644
--- a/lib/Transforms/ObjCARC/BlotMapVector.h
+++ b/lib/Transforms/ObjCARC/BlotMapVector.h
@@ -1,9 +1,8 @@
 //===- BlotMapVector.h - A MapVector with the blot operation ----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Transforms/ObjCARC/DependencyAnalysis.cpp b/lib/Transforms/ObjCARC/DependencyAnalysis.cpp
index 52a5e8c96abe39d44f9193cb4addefefd2ed990b..e8f8fb6f3a7c009c7b6ce4b6db6e8f9435254dd9 100644
--- a/lib/Transforms/ObjCARC/DependencyAnalysis.cpp
+++ b/lib/Transforms/ObjCARC/DependencyAnalysis.cpp
@@ -1,9 +1,8 @@
 //===- DependencyAnalysis.cpp - ObjC ARC Optimization ---------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
@@ -45,18 +44,15 @@ bool llvm::objcarc::CanAlterRefCount(const Instruction *Inst, const Value *Ptr,
   default: break;
   }
 
-  ImmutableCallSite CS(Inst);
-  assert(CS && "Only calls can alter reference counts!");
+  const auto *Call = cast<CallBase>(Inst);
 
   // See if AliasAnalysis can help us with the call.
-  FunctionModRefBehavior MRB = PA.getAA()->getModRefBehavior(CS);
+  FunctionModRefBehavior MRB = PA.getAA()->getModRefBehavior(Call);
   if (AliasAnalysis::onlyReadsMemory(MRB))
     return false;
   if (AliasAnalysis::onlyAccessesArgPointees(MRB)) {
     const DataLayout &DL = Inst->getModule()->getDataLayout();
-    for (ImmutableCallSite::arg_iterator I = CS.arg_begin(), E = CS.arg_end();
-         I != E; ++I) {
-      const Value *Op = *I;
+    for (const Value *Op : Call->args()) {
       if (IsPotentialRetainableObjPtr(Op, *PA.getAA()) &&
           PA.related(Ptr, Op, DL))
         return true;
diff --git a/lib/Transforms/ObjCARC/DependencyAnalysis.h b/lib/Transforms/ObjCARC/DependencyAnalysis.h
index 0f13b02c806f2559a9eb52a444f19f5ba79a6e7c..ed89c8c8fc893a3cf19b34a27573fbec6ebe2411 100644
--- a/lib/Transforms/ObjCARC/DependencyAnalysis.h
+++ b/lib/Transforms/ObjCARC/DependencyAnalysis.h
@@ -1,9 +1,8 @@
 //===- DependencyAnalysis.h - ObjC ARC Optimization ---*- C++ -*-----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/lib/Transforms/ObjCARC/LLVMBuild.txt b/lib/Transforms/ObjCARC/LLVMBuild.txt
index 90a233851a3c1b4186fd7eeabe732e99f9b7afd7..97af03be240028cd2160e0c52d21d301014599d8 100644
--- a/lib/Transforms/ObjCARC/LLVMBuild.txt
+++ b/lib/Transforms/ObjCARC/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/Transforms/ObjCARC/LLVMBuild.txt -------------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Transforms/ObjCARC/ObjCARC.cpp b/lib/Transforms/ObjCARC/ObjCARC.cpp
index c30aaebd0f4d0d598b5dc060f90a6cede275cf85..f4da51650a7d04becfa8e0f11c2fef70de806913 100644
--- a/lib/Transforms/ObjCARC/ObjCARC.cpp
+++ b/lib/Transforms/ObjCARC/ObjCARC.cpp
@@ -1,9 +1,8 @@
 //===-- ObjCARC.cpp -------------------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Transforms/ObjCARC/ObjCARC.h b/lib/Transforms/ObjCARC/ObjCARC.h
index 1dbe72c7569f4f519e245b9b6099d2d47c8fd80e..d465630800b9a8853a67b18698c458d83ed478f7 100644
--- a/lib/Transforms/ObjCARC/ObjCARC.h
+++ b/lib/Transforms/ObjCARC/ObjCARC.h
@@ -1,9 +1,8 @@
 //===- ObjCARC.h - ObjC ARC Optimization --------------*- C++ -*-----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
@@ -58,7 +57,7 @@ static inline void EraseInstruction(Instruction *CI) {
     // Replace the return value with the argument.
     assert((IsForwarding(GetBasicARCInstKind(CI)) ||
             (IsNoopOnNull(GetBasicARCInstKind(CI)) &&
-             isa<ConstantPointerNull>(OldArg))) &&
+             IsNullOrUndef(OldArg->stripPointerCasts()))) &&
            "Can't delete non-forwarding instruction with users!");
     CI->replaceAllUsesWith(OldArg);
   }
diff --git a/lib/Transforms/ObjCARC/ObjCARCAPElim.cpp b/lib/Transforms/ObjCARC/ObjCARCAPElim.cpp
index 8d3ef8fde53472b837063242711d865d9718c122..b341dd807508c20e31c7d250876e1db86eb16f1a 100644
--- a/lib/Transforms/ObjCARC/ObjCARCAPElim.cpp
+++ b/lib/Transforms/ObjCARC/ObjCARCAPElim.cpp
@@ -1,9 +1,8 @@
 //===- ObjCARCAPElim.cpp - ObjC ARC Optimization --------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/lib/Transforms/ObjCARC/ObjCARCContract.cpp b/lib/Transforms/ObjCARC/ObjCARCContract.cpp
index abe2871c0b8f97f7f411de2a0f68a4a4a848e2fe..6613f6a76d0d45c893ad54fc5c6376ad5ae50191 100644
--- a/lib/Transforms/ObjCARC/ObjCARCContract.cpp
+++ b/lib/Transforms/ObjCARC/ObjCARCContract.cpp
@@ -1,9 +1,8 @@
 //===- ObjCARCContract.cpp - ObjC ARC Optimization ------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
@@ -140,7 +139,7 @@ bool ObjCARCContract::optimizeRetainCall(Function &F, Instruction *Retain) {
 
   // We do not have to worry about tail calls/does not throw since
   // retain/retainRV have the same properties.
-  Constant *Decl = EP.get(ARCRuntimeEntryPointKind::RetainRV);
+  Function *Decl = EP.get(ARCRuntimeEntryPointKind::RetainRV);
   cast<CallInst>(Retain)->setCalledFunction(Decl);
 
   LLVM_DEBUG(dbgs() << "New: " << *Retain << "\n");
@@ -189,7 +188,7 @@ bool ObjCARCContract::contractAutorelease(
                        "        Retain: "
                     << *Retain << "\n");
 
-  Constant *Decl = EP.get(Class == ARCInstKind::AutoreleaseRV
+  Function *Decl = EP.get(Class == ARCInstKind::AutoreleaseRV
                               ? ARCRuntimeEntryPointKind::RetainAutoreleaseRV
                               : ARCRuntimeEntryPointKind::RetainAutorelease);
   Retain->setCalledFunction(Decl);
@@ -314,8 +313,8 @@ findRetainForStoreStrongContraction(Value *New, StoreInst *Store,
 /// Create a call instruction with the correct funclet token. Should be used
 /// instead of calling CallInst::Create directly.
 static CallInst *
-createCallInst(Value *Func, ArrayRef<Value *> Args, const Twine &NameStr,
-               Instruction *InsertBefore,
+createCallInst(FunctionType *FTy, Value *Func, ArrayRef<Value *> Args,
+               const Twine &NameStr, Instruction *InsertBefore,
                const DenseMap<BasicBlock *, ColorVector> &BlockColors) {
   SmallVector<OperandBundleDef, 1> OpBundles;
   if (!BlockColors.empty()) {
@@ -326,7 +325,15 @@ createCallInst(Value *Func, ArrayRef<Value *> Args, const Twine &NameStr,
       OpBundles.emplace_back("funclet", EHPad);
   }
 
-  return CallInst::Create(Func, Args, OpBundles, NameStr, InsertBefore);
+  return CallInst::Create(FTy, Func, Args, OpBundles, NameStr, InsertBefore);
+}
+
+static CallInst *
+createCallInst(FunctionCallee Func, ArrayRef<Value *> Args, const Twine &NameStr,
+               Instruction *InsertBefore,
+               const DenseMap<BasicBlock *, ColorVector> &BlockColors) {
+  return createCallInst(Func.getFunctionType(), Func.getCallee(), Args, NameStr,
+                        InsertBefore, BlockColors);
 }
 
 /// Attempt to merge an objc_release with a store, load, and objc_retain to form
@@ -409,7 +416,7 @@ void ObjCARCContract::tryToContractReleaseIntoStoreStrong(
     Args[0] = new BitCastInst(Args[0], I8XX, "", Store);
   if (Args[1]->getType() != I8X)
     Args[1] = new BitCastInst(Args[1], I8X, "", Store);
-  Constant *Decl = EP.get(ARCRuntimeEntryPointKind::StoreStrong);
+  Function *Decl = EP.get(ARCRuntimeEntryPointKind::StoreStrong);
   CallInst *StoreStrong = createCallInst(Decl, Args, "", Store, BlockColors);
   StoreStrong->setDoesNotThrow();
   StoreStrong->setDebugLoc(Store->getDebugLoc());
diff --git a/lib/Transforms/ObjCARC/ObjCARCExpand.cpp b/lib/Transforms/ObjCARC/ObjCARCExpand.cpp
index 6a345ef56e1bb4c66580e4e7e45519e7e7542cd1..04e98d8f55776867e1d33a54e05e58ac19b7afe8 100644
--- a/lib/Transforms/ObjCARC/ObjCARCExpand.cpp
+++ b/lib/Transforms/ObjCARC/ObjCARCExpand.cpp
@@ -1,9 +1,8 @@
 //===- ObjCARCExpand.cpp - ObjC ARC Optimization --------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/lib/Transforms/ObjCARC/ObjCARCOpts.cpp b/lib/Transforms/ObjCARC/ObjCARCOpts.cpp
index 6ffaadc2b5fc52d1fd38ee09557c3342291de903..9572674f7581af8417279b8db2e7d186729cbd0c 100644
--- a/lib/Transforms/ObjCARC/ObjCARCOpts.cpp
+++ b/lib/Transforms/ObjCARC/ObjCARCOpts.cpp
@@ -1,9 +1,8 @@
 //===- ObjCARCOpts.cpp - ObjC ARC Optimization ----------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -600,6 +599,17 @@ ObjCARCOpt::OptimizeRetainRVCall(Function &F, Instruction *RetainRV) {
     }
   }
 
+  // Track PHIs which are equivalent to our Arg.
+  SmallDenseSet<const Value*, 2> EquivalentArgs;
+  EquivalentArgs.insert(Arg);
+
+  // Add PHIs that are equivalent to Arg to ArgUsers.
+  if (const PHINode *PN = dyn_cast<PHINode>(Arg)) {
+    SmallVector<const Value *, 2> ArgUsers;
+    getEquivalentPHIs(*PN, ArgUsers);
+    EquivalentArgs.insert(ArgUsers.begin(), ArgUsers.end());
+  }
+
   // Check for being preceded by an objc_autoreleaseReturnValue on the same
   // pointer. In this case, we can delete the pair.
   BasicBlock::iterator I = RetainRV->getIterator(),
@@ -609,7 +619,7 @@ ObjCARCOpt::OptimizeRetainRVCall(Function &F, Instruction *RetainRV) {
       --I;
     while (I != Begin && IsNoopInstruction(&*I));
     if (GetBasicARCInstKind(&*I) == ARCInstKind::AutoreleaseRV &&
-        GetArgRCIdentityRoot(&*I) == Arg) {
+        EquivalentArgs.count(GetArgRCIdentityRoot(&*I))) {
       Changed = true;
       ++NumPeeps;
 
@@ -631,7 +641,7 @@ ObjCARCOpt::OptimizeRetainRVCall(Function &F, Instruction *RetainRV) {
                        "Old = "
                     << *RetainRV << "\n");
 
-  Constant *NewDecl = EP.get(ARCRuntimeEntryPointKind::Retain);
+  Function *NewDecl = EP.get(ARCRuntimeEntryPointKind::Retain);
   cast<CallInst>(RetainRV)->setCalledFunction(NewDecl);
 
   LLVM_DEBUG(dbgs() << "New = " << *RetainRV << "\n");
@@ -680,7 +690,7 @@ void ObjCARCOpt::OptimizeAutoreleaseRVCall(Function &F,
              << *AutoreleaseRV << "\n");
 
   CallInst *AutoreleaseRVCI = cast<CallInst>(AutoreleaseRV);
-  Constant *NewDecl = EP.get(ARCRuntimeEntryPointKind::Autorelease);
+  Function *NewDecl = EP.get(ARCRuntimeEntryPointKind::Autorelease);
   AutoreleaseRVCI->setCalledFunction(NewDecl);
   AutoreleaseRVCI->setTailCall(false); // Never tail call objc_autorelease.
   Class = ARCInstKind::Autorelease;
@@ -819,7 +829,7 @@ void ObjCARCOpt::OptimizeIndividualCalls(Function &F) {
         // Create the declaration lazily.
         LLVMContext &C = Inst->getContext();
 
-        Constant *Decl = EP.get(ARCRuntimeEntryPointKind::Release);
+        Function *Decl = EP.get(ARCRuntimeEntryPointKind::Release);
         CallInst *NewCall = CallInst::Create(Decl, Call->getArgOperand(0), "",
                                              Call);
         NewCall->setMetadata(MDKindCache.get(ARCMDKindID::ImpreciseRelease),
@@ -1517,7 +1527,7 @@ void ObjCARCOpt::MoveCalls(Value *Arg, RRInfo &RetainsToMove,
   for (Instruction *InsertPt : ReleasesToMove.ReverseInsertPts) {
     Value *MyArg = ArgTy == ParamTy ? Arg :
                    new BitCastInst(Arg, ParamTy, "", InsertPt);
-    Constant *Decl = EP.get(ARCRuntimeEntryPointKind::Retain);
+    Function *Decl = EP.get(ARCRuntimeEntryPointKind::Retain);
     CallInst *Call = CallInst::Create(Decl, MyArg, "", InsertPt);
     Call->setDoesNotThrow();
     Call->setTailCall();
@@ -1530,7 +1540,7 @@ void ObjCARCOpt::MoveCalls(Value *Arg, RRInfo &RetainsToMove,
   for (Instruction *InsertPt : RetainsToMove.ReverseInsertPts) {
     Value *MyArg = ArgTy == ParamTy ? Arg :
                    new BitCastInst(Arg, ParamTy, "", InsertPt);
-    Constant *Decl = EP.get(ARCRuntimeEntryPointKind::Release);
+    Function *Decl = EP.get(ARCRuntimeEntryPointKind::Release);
     CallInst *Call = CallInst::Create(Decl, MyArg, "", InsertPt);
     // Attach a clang.imprecise_release metadata tag, if appropriate.
     if (MDNode *M = ReleasesToMove.ReleaseMetadata)
@@ -1866,7 +1876,7 @@ void ObjCARCOpt::OptimizeWeakCalls(Function &F) {
           Changed = true;
           // If the load has a builtin retain, insert a plain retain for it.
           if (Class == ARCInstKind::LoadWeakRetained) {
-            Constant *Decl = EP.get(ARCRuntimeEntryPointKind::Retain);
+            Function *Decl = EP.get(ARCRuntimeEntryPointKind::Retain);
             CallInst *CI = CallInst::Create(Decl, EarlierCall, "", Call);
             CI->setTailCall();
           }
@@ -1895,7 +1905,7 @@ void ObjCARCOpt::OptimizeWeakCalls(Function &F) {
           Changed = true;
           // If the load has a builtin retain, insert a plain retain for it.
           if (Class == ARCInstKind::LoadWeakRetained) {
-            Constant *Decl = EP.get(ARCRuntimeEntryPointKind::Retain);
+            Function *Decl = EP.get(ARCRuntimeEntryPointKind::Retain);
             CallInst *CI = CallInst::Create(Decl, EarlierCall, "", Call);
             CI->setTailCall();
           }
diff --git a/lib/Transforms/ObjCARC/ProvenanceAnalysis.cpp b/lib/Transforms/ObjCARC/ProvenanceAnalysis.cpp
index 3004fffb9745ad935f53f3fc879de81a4fb6e6c4..c6138edba95ad29e60fffe28e17bd43c447cb702 100644
--- a/lib/Transforms/ObjCARC/ProvenanceAnalysis.cpp
+++ b/lib/Transforms/ObjCARC/ProvenanceAnalysis.cpp
@@ -1,9 +1,8 @@
 //===- ProvenanceAnalysis.cpp - ObjC ARC Optimization ---------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Transforms/ObjCARC/ProvenanceAnalysis.h b/lib/Transforms/ObjCARC/ProvenanceAnalysis.h
index 1276f564a0228e5318588947412dd7d792d79987..8fd842fd42d641ce28218db67f12c3b3e59dc2f3 100644
--- a/lib/Transforms/ObjCARC/ProvenanceAnalysis.h
+++ b/lib/Transforms/ObjCARC/ProvenanceAnalysis.h
@@ -1,9 +1,8 @@
 //===- ProvenanceAnalysis.h - ObjC ARC Optimization -------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Transforms/ObjCARC/ProvenanceAnalysisEvaluator.cpp b/lib/Transforms/ObjCARC/ProvenanceAnalysisEvaluator.cpp
index 870a5f600fd84af1742ea50641f2c342502474c9..b768f7973b87b505b2b488d328e814ae6a68e22f 100644
--- a/lib/Transforms/ObjCARC/ProvenanceAnalysisEvaluator.cpp
+++ b/lib/Transforms/ObjCARC/ProvenanceAnalysisEvaluator.cpp
@@ -1,9 +1,8 @@
 //===- ProvenanceAnalysisEvaluator.cpp - ObjC ARC Optimization ------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Transforms/ObjCARC/PtrState.cpp b/lib/Transforms/ObjCARC/PtrState.cpp
index 8a7b6a74fae2fdf3308adf21aa271c7016169938..3243481dee0d993faa41430767fcd7c802617167 100644
--- a/lib/Transforms/ObjCARC/PtrState.cpp
+++ b/lib/Transforms/ObjCARC/PtrState.cpp
@@ -1,9 +1,8 @@
 //===- PtrState.cpp -------------------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Transforms/ObjCARC/PtrState.h b/lib/Transforms/ObjCARC/PtrState.h
index f5b9b853d8e381e0c9a554fa7438b9bd90e7a8ea..66614c06cb7994bd8169e2f8fc29595e476002a4 100644
--- a/lib/Transforms/ObjCARC/PtrState.h
+++ b/lib/Transforms/ObjCARC/PtrState.h
@@ -1,9 +1,8 @@
 //===- PtrState.h - ARC State for a Ptr -------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Transforms/Scalar/ADCE.cpp b/lib/Transforms/Scalar/ADCE.cpp
index b0602d96798c3119299abd9b57b5f65ed749306b..8dcf6393f46027df7b5c2195ebf5256707b7fce1 100644
--- a/lib/Transforms/Scalar/ADCE.cpp
+++ b/lib/Transforms/Scalar/ADCE.cpp
@@ -1,9 +1,8 @@
 //===- ADCE.cpp - Code to perform dead code elimination -------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -23,6 +22,7 @@
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/Statistic.h"
+#include "llvm/Analysis/DomTreeUpdater.h"
 #include "llvm/Analysis/GlobalsModRef.h"
 #include "llvm/Analysis/IteratedDominanceFrontier.h"
 #include "llvm/Analysis/PostDominators.h"
@@ -30,7 +30,6 @@
 #include "llvm/IR/CFG.h"
 #include "llvm/IR/DebugInfoMetadata.h"
 #include "llvm/IR/DebugLoc.h"
-#include "llvm/IR/DomTreeUpdater.h"
 #include "llvm/IR/Dominators.h"
 #include "llvm/IR/Function.h"
 #include "llvm/IR/IRBuilder.h"
diff --git a/lib/Transforms/Scalar/AlignmentFromAssumptions.cpp b/lib/Transforms/Scalar/AlignmentFromAssumptions.cpp
index 0830ff5dd0422dd4b0cf114994a1ecc3717788da..de9a62e88c27396dedc79796e61e932d04a7c6c4 100644
--- a/lib/Transforms/Scalar/AlignmentFromAssumptions.cpp
+++ b/lib/Transforms/Scalar/AlignmentFromAssumptions.cpp
@@ -1,10 +1,9 @@
 //===----------------------- AlignmentFromAssumptions.cpp -----------------===//
 //                  Set Load/Store Alignments From Assumptions
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Transforms/Scalar/BDCE.cpp b/lib/Transforms/Scalar/BDCE.cpp
index f63182e57c1a3186015848994ed9cf3a947efc5a..7ada9f5ab7f998efec683bbe56e632a0eb38fe63 100644
--- a/lib/Transforms/Scalar/BDCE.cpp
+++ b/lib/Transforms/Scalar/BDCE.cpp
@@ -1,9 +1,8 @@
 //===---- BDCE.cpp - Bit-tracking dead code elimination -------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -96,30 +95,41 @@ static bool bitTrackingDCE(Function &F, DemandedBits &DB) {
     if (I.mayHaveSideEffects() && I.use_empty())
       continue;
 
-    if (I.getType()->isIntOrIntVectorTy() &&
-        !DB.getDemandedBits(&I).getBoolValue()) {
-      // For live instructions that have all dead bits, first make them dead by
-      // replacing all uses with something else. Then, if they don't need to
-      // remain live (because they have side effects, etc.) we can remove them.
-      LLVM_DEBUG(dbgs() << "BDCE: Trivializing: " << I << " (all bits dead)\n");
+    // Remove instructions that are dead, either because they were not reached
+    // during analysis or have no demanded bits.
+    if (DB.isInstructionDead(&I) ||
+        (I.getType()->isIntOrIntVectorTy() &&
+         DB.getDemandedBits(&I).isNullValue() &&
+         wouldInstructionBeTriviallyDead(&I))) {
+      salvageDebugInfo(I);
+      Worklist.push_back(&I);
+      I.dropAllReferences();
+      Changed = true;
+      continue;
+    }
+
+    for (Use &U : I.operands()) {
+      // DemandedBits only detects dead integer uses.
+      if (!U->getType()->isIntOrIntVectorTy())
+        continue;
+
+      if (!isa<Instruction>(U) && !isa<Argument>(U))
+        continue;
+
+      if (!DB.isUseDead(&U))
+        continue;
+
+      LLVM_DEBUG(dbgs() << "BDCE: Trivializing: " << U << " (all bits dead)\n");
 
       clearAssumptionsOfUsers(&I, DB);
 
       // FIXME: In theory we could substitute undef here instead of zero.
       // This should be reconsidered once we settle on the semantics of
       // undef, poison, etc.
-      Value *Zero = ConstantInt::get(I.getType(), 0);
+      U.set(ConstantInt::get(U->getType(), 0));
       ++NumSimplified;
-      I.replaceNonMetadataUsesWith(Zero);
       Changed = true;
     }
-    if (!DB.isInstructionDead(&I))
-      continue;
-
-    salvageDebugInfo(I);
-    Worklist.push_back(&I);
-    I.dropAllReferences();
-    Changed = true;
   }
 
   for (Instruction *&I : Worklist) {
diff --git a/lib/Transforms/Scalar/CMakeLists.txt b/lib/Transforms/Scalar/CMakeLists.txt
index e3548ce5cd0afdd7472a9107d66939de49709ff6..9c33971ce0f2f723397f0b153465f3b2b70c0e69 100644
--- a/lib/Transforms/Scalar/CMakeLists.txt
+++ b/lib/Transforms/Scalar/CMakeLists.txt
@@ -45,6 +45,7 @@ add_llvm_library(LLVMScalarOpts
   LowerAtomic.cpp
   LowerExpectIntrinsic.cpp
   LowerGuardIntrinsic.cpp
+  LowerWidenableCondition.cpp
   MakeGuardsExplicit.cpp
   MemCpyOptimizer.cpp
   MergeICmps.cpp
diff --git a/lib/Transforms/Scalar/CallSiteSplitting.cpp b/lib/Transforms/Scalar/CallSiteSplitting.cpp
index a806d6faed60a6333eedf4529647366b41f9322a..92840e563133d99cf55e2a1e570ed85df79df936 100644
--- a/lib/Transforms/Scalar/CallSiteSplitting.cpp
+++ b/lib/Transforms/Scalar/CallSiteSplitting.cpp
@@ -1,9 +1,8 @@
 //===- CallSiteSplitting.cpp ----------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Transforms/Scalar/ConstantHoisting.cpp b/lib/Transforms/Scalar/ConstantHoisting.cpp
index beac0d967a980609e9d3c28bfe1f1d2dbcde1c2d..27d4f67ccc0f6ea3697dcd2d6756e0fa6bb8a5c7 100644
--- a/lib/Transforms/Scalar/ConstantHoisting.cpp
+++ b/lib/Transforms/Scalar/ConstantHoisting.cpp
@@ -1,9 +1,8 @@
 //===- ConstantHoisting.cpp - Prepare code for expensive constants --------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Transforms/Scalar/ConstantProp.cpp b/lib/Transforms/Scalar/ConstantProp.cpp
index 51032b0625f886a3c0a466fecf75c6493d0464e1..770321c740a0fc6f4309c43f2c907540358b3ed2 100644
--- a/lib/Transforms/Scalar/ConstantProp.cpp
+++ b/lib/Transforms/Scalar/ConstantProp.cpp
@@ -1,9 +1,8 @@
 //===- ConstantProp.cpp - Code to perform Simple Constant Propagation -----===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Transforms/Scalar/CorrelatedValuePropagation.cpp b/lib/Transforms/Scalar/CorrelatedValuePropagation.cpp
index d0105701c73fcc3478fe0eb6b3fd884bad83f7b7..5ae7036dc6c0701fc84c8ceb0de09031a352c3cf 100644
--- a/lib/Transforms/Scalar/CorrelatedValuePropagation.cpp
+++ b/lib/Transforms/Scalar/CorrelatedValuePropagation.cpp
@@ -1,9 +1,8 @@
 //===- CorrelatedValuePropagation.cpp - Propagate CFG-derived info --------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -16,6 +15,7 @@
 #include "llvm/ADT/Optional.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/Statistic.h"
+#include "llvm/Analysis/DomTreeUpdater.h"
 #include "llvm/Analysis/GlobalsModRef.h"
 #include "llvm/Analysis/InstructionSimplify.h"
 #include "llvm/Analysis/LazyValueInfo.h"
@@ -27,7 +27,6 @@
 #include "llvm/IR/ConstantRange.h"
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/DerivedTypes.h"
-#include "llvm/IR/DomTreeUpdater.h"
 #include "llvm/IR/Function.h"
 #include "llvm/IR/IRBuilder.h"
 #include "llvm/IR/InstrTypes.h"
@@ -463,6 +462,30 @@ static bool processCallSite(CallSite CS, LazyValueInfo *LVI) {
     }
   }
 
+  // Deopt bundle operands are intended to capture state with minimal
+  // perturbance of the code otherwise.  If we can find a constant value for
+  // any such operand and remove a use of the original value, that's
+  // desireable since it may allow further optimization of that value (e.g. via
+  // single use rules in instcombine).  Since deopt uses tend to,
+  // idiomatically, appear along rare conditional paths, it's reasonable likely
+  // we may have a conditional fact with which LVI can fold.   
+  if (auto DeoptBundle = CS.getOperandBundle(LLVMContext::OB_deopt)) {
+    bool Progress = false;
+    for (const Use &ConstU : DeoptBundle->Inputs) {
+      Use &U = const_cast<Use&>(ConstU);
+      Value *V = U.get();
+      if (V->getType()->isVectorTy()) continue;
+      if (isa<Constant>(V)) continue;
+
+      Constant *C = LVI->getConstant(V, CS.getParent(), CS.getInstruction());
+      if (!C) continue;
+      U.set(C);
+      Progress = true;
+    }
+    if (Progress)
+      return true;
+  }
+
   for (Value *V : CS.args()) {
     PointerType *Type = dyn_cast<PointerType>(V->getType());
     // Try to mark pointer typed parameters as non-null.  We skip the
diff --git a/lib/Transforms/Scalar/DCE.cpp b/lib/Transforms/Scalar/DCE.cpp
index 4c964e6e888cabbe6e8c460966a3968b38dd67a9..479e0ed7407430566d6b1f9e0c5285798666fb71 100644
--- a/lib/Transforms/Scalar/DCE.cpp
+++ b/lib/Transforms/Scalar/DCE.cpp
@@ -1,9 +1,8 @@
 //===- DCE.cpp - Code to perform dead code elimination --------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Transforms/Scalar/DeadStoreElimination.cpp b/lib/Transforms/Scalar/DeadStoreElimination.cpp
index 69112f3cee277049585fe1092de6c8bd894276a7..863ee834e57dae815a92059f6e7cdaf2f8d1851a 100644
--- a/lib/Transforms/Scalar/DeadStoreElimination.cpp
+++ b/lib/Transforms/Scalar/DeadStoreElimination.cpp
@@ -1,9 +1,8 @@
 //===- DeadStoreElimination.cpp - Fast Dead Store Elimination -------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -834,7 +833,7 @@ static bool handleEndBlock(BasicBlock &BB, AliasAnalysis *AA,
       continue;
     }
 
-    if (auto CS = CallSite(&*BBI)) {
+    if (auto *Call = dyn_cast<CallBase>(&*BBI)) {
       // Remove allocation function calls from the list of dead stack objects;
       // there can't be any references before the definition.
       if (isAllocLikeFn(&*BBI, TLI))
@@ -842,15 +841,15 @@ static bool handleEndBlock(BasicBlock &BB, AliasAnalysis *AA,
 
       // If this call does not access memory, it can't be loading any of our
       // pointers.
-      if (AA->doesNotAccessMemory(CS))
+      if (AA->doesNotAccessMemory(Call))
         continue;
 
       // If the call might load from any of our allocas, then any store above
       // the call is live.
       DeadStackObjects.remove_if([&](Value *I) {
         // See if the call site touches the value.
-        return isRefSet(AA->getModRefInfo(CS, I, getPointerSize(I, DL, *TLI,
-                                                                BB.getParent())));
+        return isRefSet(AA->getModRefInfo(
+            Call, I, getPointerSize(I, DL, *TLI, BB.getParent())));
       });
 
       // If all of the allocas were clobbered by the call then we're not going
@@ -946,6 +945,7 @@ static bool tryToShorten(Instruction *EarlierWrite, int64_t &EarlierOffset,
     Value *Indices[1] = {
         ConstantInt::get(EarlierWriteLength->getType(), OffsetMoved)};
     GetElementPtrInst *NewDestGEP = GetElementPtrInst::CreateInBounds(
+        EarlierIntrinsic->getRawDest()->getType()->getPointerElementType(),
         EarlierIntrinsic->getRawDest(), Indices, "", EarlierWrite);
     EarlierIntrinsic->setDest(NewDestGEP);
     EarlierOffset = EarlierOffset + OffsetMoved;
diff --git a/lib/Transforms/Scalar/DivRemPairs.cpp b/lib/Transforms/Scalar/DivRemPairs.cpp
index ffcf34f1cf7a237a7af0943ebb752edef9184f09..876681b4f9ded360efe3b0a7eb88e1e33be9955b 100644
--- a/lib/Transforms/Scalar/DivRemPairs.cpp
+++ b/lib/Transforms/Scalar/DivRemPairs.cpp
@@ -1,9 +1,8 @@
 //===- DivRemPairs.cpp - Hoist/decompose division and remainder -*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Transforms/Scalar/EarlyCSE.cpp b/lib/Transforms/Scalar/EarlyCSE.cpp
index 1f09979b33824da181180b37bac02b36d2ec9d11..bec84ae10577787a66b36bf595842404227b9c8f 100644
--- a/lib/Transforms/Scalar/EarlyCSE.cpp
+++ b/lib/Transforms/Scalar/EarlyCSE.cpp
@@ -1,9 +1,8 @@
 //===- EarlyCSE.cpp - Simple and fast CSE pass ----------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -608,36 +607,11 @@ private:
       MSSA->verifyMemorySSA();
     // Removing a store here can leave MemorySSA in an unoptimized state by
     // creating MemoryPhis that have identical arguments and by creating
-    // MemoryUses whose defining access is not an actual clobber.  We handle the
-    // phi case eagerly here.  The non-optimized MemoryUse case is lazily
-    // updated by MemorySSA getClobberingMemoryAccess.
-    if (MemoryAccess *MA = MSSA->getMemoryAccess(Inst)) {
-      // Optimize MemoryPhi nodes that may become redundant by having all the
-      // same input values once MA is removed.
-      SmallSetVector<MemoryPhi *, 4> PhisToCheck;
-      SmallVector<MemoryAccess *, 8> WorkQueue;
-      WorkQueue.push_back(MA);
-      // Process MemoryPhi nodes in FIFO order using a ever-growing vector since
-      // we shouldn't be processing that many phis and this will avoid an
-      // allocation in almost all cases.
-      for (unsigned I = 0; I < WorkQueue.size(); ++I) {
-        MemoryAccess *WI = WorkQueue[I];
-
-        for (auto *U : WI->users())
-          if (MemoryPhi *MP = dyn_cast<MemoryPhi>(U))
-            PhisToCheck.insert(MP);
-
-        MSSAUpdater->removeMemoryAccess(WI);
-
-        for (MemoryPhi *MP : PhisToCheck) {
-          MemoryAccess *FirstIn = MP->getIncomingValue(0);
-          if (llvm::all_of(MP->incoming_values(),
-                           [=](Use &In) { return In == FirstIn; }))
-            WorkQueue.push_back(MP);
-        }
-        PhisToCheck.clear();
-      }
-    }
+    // MemoryUses whose defining access is not an actual clobber. The phi case
+    // is handled by MemorySSA when passing OptimizePhis = true to
+    // removeMemoryAccess.  The non-optimized MemoryUse case is lazily updated
+    // by MemorySSA's getClobberingMemoryAccess.
+    MSSAUpdater->removeMemoryAccess(Inst, true);
   }
 };
 
diff --git a/lib/Transforms/Scalar/FlattenCFGPass.cpp b/lib/Transforms/Scalar/FlattenCFGPass.cpp
index 117b19fb8a42253e162fb114adb91fd1191b1bfc..31670b1464e41f45da64e22d40b859ce45d5db32 100644
--- a/lib/Transforms/Scalar/FlattenCFGPass.cpp
+++ b/lib/Transforms/Scalar/FlattenCFGPass.cpp
@@ -1,9 +1,8 @@
 //===- FlattenCFGPass.cpp - CFG Flatten Pass ----------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Transforms/Scalar/Float2Int.cpp b/lib/Transforms/Scalar/Float2Int.cpp
index f2828e80bc580825954cd324b65ea88e2df0a92a..4a1275d3661277fa663c386a0853ac61d6c24cad 100644
--- a/lib/Transforms/Scalar/Float2Int.cpp
+++ b/lib/Transforms/Scalar/Float2Int.cpp
@@ -1,9 +1,8 @@
 //===- Float2Int.cpp - Demote floating point ops to work on integers ------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Transforms/Scalar/GVN.cpp b/lib/Transforms/Scalar/GVN.cpp
index 440ea4a5bc789668d2f4cf39d14d20ef37460ce0..a02f32f5643511f004b20d77a1d7f3da2a1e3c5f 100644
--- a/lib/Transforms/Scalar/GVN.cpp
+++ b/lib/Transforms/Scalar/GVN.cpp
@@ -1,9 +1,8 @@
 //===- GVN.cpp - Eliminate redundant values and loads ---------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -30,6 +29,7 @@
 #include "llvm/Analysis/AliasAnalysis.h"
 #include "llvm/Analysis/AssumptionCache.h"
 #include "llvm/Analysis/CFG.h"
+#include "llvm/Analysis/DomTreeUpdater.h"
 #include "llvm/Analysis/GlobalsModRef.h"
 #include "llvm/Analysis/InstructionSimplify.h"
 #include "llvm/Analysis/LoopInfo.h"
@@ -47,7 +47,6 @@
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/DataLayout.h"
 #include "llvm/IR/DebugLoc.h"
-#include "llvm/IR/DomTreeUpdater.h"
 #include "llvm/IR/Dominators.h"
 #include "llvm/IR/Function.h"
 #include "llvm/IR/InstrTypes.h"
@@ -437,7 +436,7 @@ uint32_t GVN::ValueTable::lookupOrAddCall(CallInst *C) {
 
     // Non-local case.
     const MemoryDependenceResults::NonLocalDepInfo &deps =
-      MD->getNonLocalCallDependency(CallSite(C));
+        MD->getNonLocalCallDependency(C);
     // FIXME: Move the checking logic to MemDep!
     CallInst* cdep = nullptr;
 
@@ -1132,6 +1131,14 @@ bool GVN::PerformLoadPRE(LoadInst *LI, AvailValInBlkVect &ValuesPerBlock,
         return false;
       }
 
+      // FIXME: Can we support the fallthrough edge?
+      if (isa<CallBrInst>(Pred->getTerminator())) {
+        LLVM_DEBUG(
+            dbgs() << "COULD NOT PRE LOAD BECAUSE OF CALLBR CRITICAL EDGE '"
+                   << Pred->getName() << "': " << *LI << '\n');
+        return false;
+      }
+
       if (LoadBB->isEHPad()) {
         LLVM_DEBUG(
             dbgs() << "COULD NOT PRE LOAD BECAUSE OF AN EH PAD CRITICAL EDGE '"
@@ -1235,10 +1242,10 @@ bool GVN::PerformLoadPRE(LoadInst *LI, AvailValInBlkVect &ValuesPerBlock,
     BasicBlock *UnavailablePred = PredLoad.first;
     Value *LoadPtr = PredLoad.second;
 
-    auto *NewLoad = new LoadInst(LoadPtr, LI->getName()+".pre",
-                                 LI->isVolatile(), LI->getAlignment(),
-                                 LI->getOrdering(), LI->getSyncScopeID(),
-                                 UnavailablePred->getTerminator());
+    auto *NewLoad =
+        new LoadInst(LI->getType(), LoadPtr, LI->getName() + ".pre",
+                     LI->isVolatile(), LI->getAlignment(), LI->getOrdering(),
+                     LI->getSyncScopeID(), UnavailablePred->getTerminator());
     NewLoad->setDebugLoc(LI->getDebugLoc());
 
     // Transfer the old load's AA tags to the new load.
@@ -1645,10 +1652,12 @@ static bool isOnlyReachableViaThisEdge(const BasicBlockEdge &E,
 }
 
 void GVN::assignBlockRPONumber(Function &F) {
+  BlockRPONumber.clear();
   uint32_t NextBlockNumber = 1;
   ReversePostOrderTraversal<Function *> RPOT(&F);
   for (BasicBlock *BB : RPOT)
     BlockRPONumber[BB] = NextBlockNumber++;
+  InvalidBlockRPONumbers = false;
 }
 
 // Tries to replace instruction with const, using information from
@@ -1992,6 +2001,7 @@ bool GVN::runImpl(Function &F, AssumptionCache &RunAC, DominatorTree &RunDT,
   ICF = &ImplicitCFT;
   VN.setMemDep(MD);
   ORE = RunORE;
+  InvalidBlockRPONumbers = true;
 
   bool Changed = false;
   bool ShouldContinue = true;
@@ -2021,7 +2031,6 @@ bool GVN::runImpl(Function &F, AssumptionCache &RunAC, DominatorTree &RunDT,
     // Fabricate val-num for dead-code in order to suppress assertion in
     // performPRE().
     assignValNumForDeadCode();
-    assignBlockRPONumber(F);
     bool PREChanged = true;
     while (PREChanged) {
       PREChanged = performPRE(F);
@@ -2079,10 +2088,9 @@ bool GVN::processBlock(BasicBlock *BB) {
       salvageDebugInfo(*I);
       if (MD) MD->removeInstruction(I);
       LLVM_DEBUG(verifyRemoved(I));
+      ICF->removeInstruction(I);
       I->eraseFromParent();
     }
-
-    ICF->invalidateBlock(BB);
     InstrsToErase.clear();
 
     if (AtStart)
@@ -2167,8 +2175,8 @@ bool GVN::performScalarPRE(Instruction *CurInst) {
     return false;
 
   // We don't currently value number ANY inline asm calls.
-  if (CallInst *CallI = dyn_cast<CallInst>(CurInst))
-    if (CallI->isInlineAsm())
+  if (auto *CallB = dyn_cast<CallBase>(CurInst))
+    if (CallB->isInlineAsm())
       return false;
 
   uint32_t ValNo = VN.lookup(CurInst);
@@ -2184,6 +2192,10 @@ bool GVN::performScalarPRE(Instruction *CurInst) {
   BasicBlock *PREPred = nullptr;
   BasicBlock *CurrentBlock = CurInst->getParent();
 
+  // Update the RPO numbers for this function.
+  if (InvalidBlockRPONumbers)
+    assignBlockRPONumber(*CurrentBlock->getParent());
+
   SmallVector<std::pair<Value *, BasicBlock *>, 8> predMap;
   for (BasicBlock *P : predecessors(CurrentBlock)) {
     // We're not interested in PRE where blocks with predecessors that are
@@ -2195,6 +2207,8 @@ bool GVN::performScalarPRE(Instruction *CurInst) {
     // It is not safe to do PRE when P->CurrentBlock is a loop backedge, and
     // when CurInst has operand defined in CurrentBlock (so it may be defined
     // by phi in the loop header).
+    assert(BlockRPONumber.count(P) && BlockRPONumber.count(CurrentBlock) &&
+           "Invalid BlockRPONumber map.");
     if (BlockRPONumber[P] >= BlockRPONumber[CurrentBlock] &&
         llvm::any_of(CurInst->operands(), [&](const Use &U) {
           if (auto *Inst = dyn_cast<Instruction>(U.get()))
@@ -2245,6 +2259,11 @@ bool GVN::performScalarPRE(Instruction *CurInst) {
     if (isa<IndirectBrInst>(PREPred->getTerminator()))
       return false;
 
+    // Don't do PRE across callbr.
+    // FIXME: Can we do this across the fallthrough edge?
+    if (isa<CallBrInst>(PREPred->getTerminator()))
+      return false;
+
     // We can't do PRE safely on a critical edge, so instead we schedule
     // the edge to be split and perform the PRE the next time we iterate
     // on the function.
@@ -2301,7 +2320,7 @@ bool GVN::performScalarPRE(Instruction *CurInst) {
   LLVM_DEBUG(verifyRemoved(CurInst));
   // FIXME: Intended to be markInstructionForDeletion(CurInst), but it causes
   // some assertion failures.
-  ICF->invalidateBlock(CurrentBlock);
+  ICF->removeInstruction(CurInst);
   CurInst->eraseFromParent();
   ++NumGVNInstr;
 
@@ -2342,6 +2361,7 @@ BasicBlock *GVN::splitCriticalEdges(BasicBlock *Pred, BasicBlock *Succ) {
       SplitCriticalEdge(Pred, Succ, CriticalEdgeSplittingOptions(DT));
   if (MD)
     MD->invalidateCachedPredecessors();
+  InvalidBlockRPONumbers = true;
   return BB;
 }
 
@@ -2356,6 +2376,7 @@ bool GVN::splitCriticalEdges() {
                       CriticalEdgeSplittingOptions(DT));
   } while (!toSplit.empty());
   if (MD) MD->invalidateCachedPredecessors();
+  InvalidBlockRPONumbers = true;
   return true;
 }
 
@@ -2382,6 +2403,7 @@ void GVN::cleanupGlobalSets() {
   BlockRPONumber.clear();
   TableAllocator.Reset();
   ICF->clear();
+  InvalidBlockRPONumbers = true;
 }
 
 /// Verify that the specified instruction does not occur in our
diff --git a/lib/Transforms/Scalar/GVNHoist.cpp b/lib/Transforms/Scalar/GVNHoist.cpp
index 0797ce9adeaff1c72e1199afba6b0b6a97945ea9..ca8f92c388721dc18c4187d6bb7dfaa6735bf0ae 100644
--- a/lib/Transforms/Scalar/GVNHoist.cpp
+++ b/lib/Transforms/Scalar/GVNHoist.cpp
@@ -1,9 +1,8 @@
 //===- GVNHoist.cpp - Hoist scalar and load expressions -------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -246,7 +245,7 @@ static void combineKnownMetadata(Instruction *ReplInst, Instruction *I) {
       LLVMContext::MD_tbaa,           LLVMContext::MD_alias_scope,
       LLVMContext::MD_noalias,        LLVMContext::MD_range,
       LLVMContext::MD_fpmath,         LLVMContext::MD_invariant_load,
-      LLVMContext::MD_invariant_group};
+      LLVMContext::MD_invariant_group, LLVMContext::MD_access_group};
   combineMetadata(ReplInst, I, KnownIDs, true);
 }
 
diff --git a/lib/Transforms/Scalar/GVNSink.cpp b/lib/Transforms/Scalar/GVNSink.cpp
index 1df5f5400c148f508f1f965311caa6755b5abc5f..94cc219a4a7334813f14e18482f2419def15ea4c 100644
--- a/lib/Transforms/Scalar/GVNSink.cpp
+++ b/lib/Transforms/Scalar/GVNSink.cpp
@@ -1,9 +1,8 @@
 //===- GVNSink.cpp - sink expressions into successors ---------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Transforms/Scalar/GuardWidening.cpp b/lib/Transforms/Scalar/GuardWidening.cpp
index cbbd7b861d8b79617a4779e67a0fd6f7ea092f7f..ec1f9144a15073b43825b6dbd5e0834bf0801e59 100644
--- a/lib/Transforms/Scalar/GuardWidening.cpp
+++ b/lib/Transforms/Scalar/GuardWidening.cpp
@@ -1,9 +1,8 @@
 //===- GuardWidening.cpp - ---- Guard widening ----------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -83,6 +82,11 @@ static cl::opt<unsigned> FrequentBranchThreshold(
              "it is considered frequently taken"),
     cl::init(1000));
 
+static cl::opt<bool>
+    WidenBranchGuards("guard-widening-widen-branch-guards", cl::Hidden,
+                      cl::desc("Whether or not we should widen guards  "
+                               "expressed as branches by widenable conditions"),
+                      cl::init(true));
 
 namespace {
 
@@ -93,6 +97,10 @@ static Value *getCondition(Instruction *I) {
            "Bad guard intrinsic?");
     return GI->getArgOperand(0);
   }
+  if (isGuardAsWidenableBranch(I)) {
+    auto *Cond = cast<BranchInst>(I)->getCondition();
+    return cast<BinaryOperator>(Cond)->getOperand(0);
+  }
   return cast<BranchInst>(I)->getCondition();
 }
 
@@ -133,12 +141,12 @@ class GuardWideningImpl {
   /// guards.
   DenseSet<Instruction *> WidenedGuards;
 
-  /// Try to eliminate guard \p Guard by widening it into an earlier dominating
-  /// guard.  \p DFSI is the DFS iterator on the dominator tree that is
-  /// currently visiting the block containing \p Guard, and \p GuardsPerBlock
+  /// Try to eliminate instruction \p Instr by widening it into an earlier
+  /// dominating guard.  \p DFSI is the DFS iterator on the dominator tree that
+  /// is currently visiting the block containing \p Guard, and \p GuardsPerBlock
   /// maps BasicBlocks to the set of guards seen in that block.
-  bool eliminateGuardViaWidening(
-      Instruction *Guard, const df_iterator<DomTreeNode *> &DFSI,
+  bool eliminateInstrViaWidening(
+      Instruction *Instr, const df_iterator<DomTreeNode *> &DFSI,
       const DenseMap<BasicBlock *, SmallVector<Instruction *, 8>> &
           GuardsPerBlock, bool InvertCondition = false);
 
@@ -162,28 +170,25 @@ class GuardWideningImpl {
 
   static StringRef scoreTypeToString(WideningScore WS);
 
-  /// Compute the score for widening the condition in \p DominatedGuard
-  /// (contained in \p DominatedGuardLoop) into \p DominatingGuard (contained in
-  /// \p DominatingGuardLoop). If \p InvertCond is set, then we widen the
+  /// Compute the score for widening the condition in \p DominatedInstr
+  /// into \p DominatingGuard. If \p InvertCond is set, then we widen the
   /// inverted condition of the dominating guard.
-  WideningScore computeWideningScore(Instruction *DominatedGuard,
-                                     Loop *DominatedGuardLoop,
+  WideningScore computeWideningScore(Instruction *DominatedInstr,
                                      Instruction *DominatingGuard,
-                                     Loop *DominatingGuardLoop,
                                      bool InvertCond);
 
   /// Helper to check if \p V can be hoisted to \p InsertPos.
-  bool isAvailableAt(Value *V, Instruction *InsertPos) {
-    SmallPtrSet<Instruction *, 8> Visited;
+  bool isAvailableAt(const Value *V, const Instruction *InsertPos) const {
+    SmallPtrSet<const Instruction *, 8> Visited;
     return isAvailableAt(V, InsertPos, Visited);
   }
 
-  bool isAvailableAt(Value *V, Instruction *InsertPos,
-                     SmallPtrSetImpl<Instruction *> &Visited);
+  bool isAvailableAt(const Value *V, const Instruction *InsertPos,
+                     SmallPtrSetImpl<const Instruction *> &Visited) const;
 
   /// Helper to hoist \p V to \p InsertPos.  Guaranteed to succeed if \c
   /// isAvailableAt returned true.
-  void makeAvailableAt(Value *V, Instruction *InsertPos);
+  void makeAvailableAt(Value *V, Instruction *InsertPos) const;
 
   /// Common helper used by \c widenGuard and \c isWideningCondProfitable.  Try
   /// to generate an expression computing the logical AND of \p Cond0 and (\p
@@ -200,23 +205,23 @@ class GuardWideningImpl {
   /// pre-existing instruction in the IR that computes the result of this range
   /// check.
   class RangeCheck {
-    Value *Base;
-    ConstantInt *Offset;
-    Value *Length;
+    const Value *Base;
+    const ConstantInt *Offset;
+    const Value *Length;
     ICmpInst *CheckInst;
 
   public:
-    explicit RangeCheck(Value *Base, ConstantInt *Offset, Value *Length,
-                        ICmpInst *CheckInst)
+    explicit RangeCheck(const Value *Base, const ConstantInt *Offset,
+                        const Value *Length, ICmpInst *CheckInst)
         : Base(Base), Offset(Offset), Length(Length), CheckInst(CheckInst) {}
 
-    void setBase(Value *NewBase) { Base = NewBase; }
-    void setOffset(ConstantInt *NewOffset) { Offset = NewOffset; }
+    void setBase(const Value *NewBase) { Base = NewBase; }
+    void setOffset(const ConstantInt *NewOffset) { Offset = NewOffset; }
 
-    Value *getBase() const { return Base; }
-    ConstantInt *getOffset() const { return Offset; }
+    const Value *getBase() const { return Base; }
+    const ConstantInt *getOffset() const { return Offset; }
     const APInt &getOffsetValue() const { return getOffset()->getValue(); }
-    Value *getLength() const { return Length; };
+    const Value *getLength() const { return Length; };
     ICmpInst *getCheckInst() const { return CheckInst; }
 
     void print(raw_ostream &OS, bool PrintTypes = false) {
@@ -238,19 +243,19 @@ class GuardWideningImpl {
   /// append them to \p Checks.  Returns true on success, may clobber \c Checks
   /// on failure.
   bool parseRangeChecks(Value *CheckCond, SmallVectorImpl<RangeCheck> &Checks) {
-    SmallPtrSet<Value *, 8> Visited;
+    SmallPtrSet<const Value *, 8> Visited;
     return parseRangeChecks(CheckCond, Checks, Visited);
   }
 
   bool parseRangeChecks(Value *CheckCond, SmallVectorImpl<RangeCheck> &Checks,
-                        SmallPtrSetImpl<Value *> &Visited);
+                        SmallPtrSetImpl<const Value *> &Visited);
 
   /// Combine the checks in \p Checks into a smaller set of checks and append
   /// them into \p CombinedChecks.  Return true on success (i.e. all of checks
   /// in \p Checks were combined into \p CombinedChecks).  Clobbers \p Checks
   /// and \p CombinedChecks on success and on failure.
   bool combineRangeChecks(SmallVectorImpl<RangeCheck> &Checks,
-                          SmallVectorImpl<RangeCheck> &CombinedChecks);
+                          SmallVectorImpl<RangeCheck> &CombinedChecks) const;
 
   /// Can we compute the logical AND of \p Cond0 and \p Cond1 for the price of
   /// computing only one of the two expressions?
@@ -266,8 +271,16 @@ class GuardWideningImpl {
   void widenGuard(Instruction *ToWiden, Value *NewCondition,
                   bool InvertCondition) {
     Value *Result;
-    widenCondCommon(ToWiden->getOperand(0), NewCondition, ToWiden, Result,
+    widenCondCommon(getCondition(ToWiden), NewCondition, ToWiden, Result,
                     InvertCondition);
+    Value *WidenableCondition = nullptr;
+    if (isGuardAsWidenableBranch(ToWiden)) {
+      auto *Cond = cast<BranchInst>(ToWiden)->getCondition();
+      WidenableCondition = cast<BinaryOperator>(Cond)->getOperand(1);
+    }
+    if (WidenableCondition)
+      Result = BinaryOperator::CreateAnd(Result, WidenableCondition,
+                                         "guard.chk", ToWiden);
     setCondition(ToWiden, Result);
   }
 
@@ -285,6 +298,14 @@ public:
 };
 }
 
+static bool isSupportedGuardInstruction(const Instruction *Insn) {
+  if (isGuard(Insn))
+    return true;
+  if (WidenBranchGuards && isGuardAsWidenableBranch(Insn))
+    return true;
+  return false;
+}
+
 bool GuardWideningImpl::run() {
   DenseMap<BasicBlock *, SmallVector<Instruction *, 8>> GuardsInBlock;
   bool Changed = false;
@@ -304,20 +325,20 @@ bool GuardWideningImpl::run() {
     auto &CurrentList = GuardsInBlock[BB];
 
     for (auto &I : *BB)
-      if (isGuard(&I))
+      if (isSupportedGuardInstruction(&I))
         CurrentList.push_back(cast<Instruction>(&I));
 
     for (auto *II : CurrentList)
-      Changed |= eliminateGuardViaWidening(II, DFI, GuardsInBlock);
+      Changed |= eliminateInstrViaWidening(II, DFI, GuardsInBlock);
     if (WidenFrequentBranches && BPI)
       if (auto *BI = dyn_cast<BranchInst>(BB->getTerminator()))
         if (BI->isConditional()) {
           // If one of branches of a conditional is likely taken, try to
           // eliminate it.
           if (BPI->getEdgeProbability(BB, 0U) >= *LikelyTaken)
-            Changed |= eliminateGuardViaWidening(BI, DFI, GuardsInBlock);
+            Changed |= eliminateInstrViaWidening(BI, DFI, GuardsInBlock);
           else if (BPI->getEdgeProbability(BB, 1U) >= *LikelyTaken)
-            Changed |= eliminateGuardViaWidening(BI, DFI, GuardsInBlock,
+            Changed |= eliminateInstrViaWidening(BI, DFI, GuardsInBlock,
                                                  /*InvertCondition*/true);
         }
   }
@@ -326,7 +347,7 @@ bool GuardWideningImpl::run() {
   for (auto *I : EliminatedGuardsAndBranches)
     if (!WidenedGuards.count(I)) {
       assert(isa<ConstantInt>(getCondition(I)) && "Should be!");
-      if (isGuard(I))
+      if (isSupportedGuardInstruction(I))
         eliminateGuard(I);
       else {
         assert(isa<BranchInst>(I) &&
@@ -338,19 +359,18 @@ bool GuardWideningImpl::run() {
   return Changed;
 }
 
-bool GuardWideningImpl::eliminateGuardViaWidening(
-    Instruction *GuardInst, const df_iterator<DomTreeNode *> &DFSI,
+bool GuardWideningImpl::eliminateInstrViaWidening(
+    Instruction *Instr, const df_iterator<DomTreeNode *> &DFSI,
     const DenseMap<BasicBlock *, SmallVector<Instruction *, 8>> &
         GuardsInBlock, bool InvertCondition) {
   // Ignore trivial true or false conditions. These instructions will be
   // trivially eliminated by any cleanup pass. Do not erase them because other
   // guards can possibly be widened into them.
-  if (isa<ConstantInt>(getCondition(GuardInst)))
+  if (isa<ConstantInt>(getCondition(Instr)))
     return false;
 
   Instruction *BestSoFar = nullptr;
   auto BestScoreSoFar = WS_IllegalOrNegative;
-  auto *GuardInstLoop = LI.getLoopFor(GuardInst->getParent());
 
   // In the set of dominating guards, find the one we can merge GuardInst with
   // for the most profit.
@@ -358,12 +378,13 @@ bool GuardWideningImpl::eliminateGuardViaWidening(
     auto *CurBB = DFSI.getPath(i)->getBlock();
     if (!BlockFilter(CurBB))
       break;
-    auto *CurLoop = LI.getLoopFor(CurBB);
     assert(GuardsInBlock.count(CurBB) && "Must have been populated by now!");
     const auto &GuardsInCurBB = GuardsInBlock.find(CurBB)->second;
 
     auto I = GuardsInCurBB.begin();
-    auto E = GuardsInCurBB.end();
+    auto E = Instr->getParent() == CurBB
+                 ? std::find(GuardsInCurBB.begin(), GuardsInCurBB.end(), Instr)
+                 : GuardsInCurBB.end();
 
 #ifndef NDEBUG
     {
@@ -379,21 +400,11 @@ bool GuardWideningImpl::eliminateGuardViaWidening(
     }
 #endif
 
-    assert((i == (e - 1)) == (GuardInst->getParent() == CurBB) && "Bad DFS?");
-
-    if (i == (e - 1) && CurBB->getTerminator() != GuardInst) {
-      // Corner case: make sure we're only looking at guards strictly dominating
-      // GuardInst when visiting GuardInst->getParent().
-      auto NewEnd = std::find(I, E, GuardInst);
-      assert(NewEnd != E && "GuardInst not in its own block?");
-      E = NewEnd;
-    }
+    assert((i == (e - 1)) == (Instr->getParent() == CurBB) && "Bad DFS?");
 
     for (auto *Candidate : make_range(I, E)) {
-      auto Score =
-          computeWideningScore(GuardInst, GuardInstLoop, Candidate, CurLoop,
-                               InvertCondition);
-      LLVM_DEBUG(dbgs() << "Score between " << *getCondition(GuardInst)
+      auto Score = computeWideningScore(Instr, Candidate, InvertCondition);
+      LLVM_DEBUG(dbgs() << "Score between " << *getCondition(Instr)
                         << " and " << *getCondition(Candidate) << " is "
                         << scoreTypeToString(Score) << "\n");
       if (Score > BestScoreSoFar) {
@@ -404,42 +415,45 @@ bool GuardWideningImpl::eliminateGuardViaWidening(
   }
 
   if (BestScoreSoFar == WS_IllegalOrNegative) {
-    LLVM_DEBUG(dbgs() << "Did not eliminate guard " << *GuardInst << "\n");
+    LLVM_DEBUG(dbgs() << "Did not eliminate guard " << *Instr << "\n");
     return false;
   }
 
-  assert(BestSoFar != GuardInst && "Should have never visited same guard!");
-  assert(DT.dominates(BestSoFar, GuardInst) && "Should be!");
+  assert(BestSoFar != Instr && "Should have never visited same guard!");
+  assert(DT.dominates(BestSoFar, Instr) && "Should be!");
 
-  LLVM_DEBUG(dbgs() << "Widening " << *GuardInst << " into " << *BestSoFar
+  LLVM_DEBUG(dbgs() << "Widening " << *Instr << " into " << *BestSoFar
                     << " with score " << scoreTypeToString(BestScoreSoFar)
                     << "\n");
-  widenGuard(BestSoFar, getCondition(GuardInst), InvertCondition);
+  widenGuard(BestSoFar, getCondition(Instr), InvertCondition);
   auto NewGuardCondition = InvertCondition
-                               ? ConstantInt::getFalse(GuardInst->getContext())
-                               : ConstantInt::getTrue(GuardInst->getContext());
-  setCondition(GuardInst, NewGuardCondition);
-  EliminatedGuardsAndBranches.push_back(GuardInst);
+                               ? ConstantInt::getFalse(Instr->getContext())
+                               : ConstantInt::getTrue(Instr->getContext());
+  setCondition(Instr, NewGuardCondition);
+  EliminatedGuardsAndBranches.push_back(Instr);
   WidenedGuards.insert(BestSoFar);
   return true;
 }
 
-GuardWideningImpl::WideningScore GuardWideningImpl::computeWideningScore(
-    Instruction *DominatedGuard, Loop *DominatedGuardLoop,
-    Instruction *DominatingGuard, Loop *DominatingGuardLoop, bool InvertCond) {
+GuardWideningImpl::WideningScore
+GuardWideningImpl::computeWideningScore(Instruction *DominatedInstr,
+                                        Instruction *DominatingGuard,
+                                        bool InvertCond) {
+  Loop *DominatedInstrLoop = LI.getLoopFor(DominatedInstr->getParent());
+  Loop *DominatingGuardLoop = LI.getLoopFor(DominatingGuard->getParent());
   bool HoistingOutOfLoop = false;
 
-  if (DominatingGuardLoop != DominatedGuardLoop) {
+  if (DominatingGuardLoop != DominatedInstrLoop) {
     // Be conservative and don't widen into a sibling loop.  TODO: If the
     // sibling is colder, we should consider allowing this.
     if (DominatingGuardLoop &&
-        !DominatingGuardLoop->contains(DominatedGuardLoop))
+        !DominatingGuardLoop->contains(DominatedInstrLoop))
       return WS_IllegalOrNegative;
 
     HoistingOutOfLoop = true;
   }
 
-  if (!isAvailableAt(getCondition(DominatedGuard), DominatingGuard))
+  if (!isAvailableAt(getCondition(DominatedInstr), DominatingGuard))
     return WS_IllegalOrNegative;
 
   // If the guard was conditional executed, it may never be reached
@@ -450,7 +464,7 @@ GuardWideningImpl::WideningScore GuardWideningImpl::computeWideningScore(
   // here.  TODO: evaluate cost model for spurious deopt
   // NOTE: As written, this also lets us hoist right over another guard which
   // is essentially just another spelling for control flow.
-  if (isWideningCondProfitable(getCondition(DominatedGuard),
+  if (isWideningCondProfitable(getCondition(DominatedInstr),
                                getCondition(DominatingGuard), InvertCond))
     return HoistingOutOfLoop ? WS_VeryPositive : WS_Positive;
 
@@ -462,7 +476,9 @@ GuardWideningImpl::WideningScore GuardWideningImpl::computeWideningScore(
   // throw, etc...).  That choice appears arbitrary.
   auto MaybeHoistingOutOfIf = [&]() {
     auto *DominatingBlock = DominatingGuard->getParent();
-    auto *DominatedBlock = DominatedGuard->getParent();
+    auto *DominatedBlock = DominatedInstr->getParent();
+    if (isGuardAsWidenableBranch(DominatingGuard))
+      DominatingBlock = cast<BranchInst>(DominatingGuard)->getSuccessor(0);
 
     // Same Block?
     if (DominatedBlock == DominatingBlock)
@@ -472,15 +488,15 @@ GuardWideningImpl::WideningScore GuardWideningImpl::computeWideningScore(
       return false;
     // TODO: diamond, triangle cases
     if (!PDT) return true;
-    return !PDT->dominates(DominatedGuard->getParent(),
-                           DominatingGuard->getParent());
+    return !PDT->dominates(DominatedBlock, DominatingBlock);
   };
 
   return MaybeHoistingOutOfIf() ? WS_IllegalOrNegative : WS_Neutral;
 }
 
-bool GuardWideningImpl::isAvailableAt(Value *V, Instruction *Loc,
-                                      SmallPtrSetImpl<Instruction *> &Visited) {
+bool GuardWideningImpl::isAvailableAt(
+    const Value *V, const Instruction *Loc,
+    SmallPtrSetImpl<const Instruction *> &Visited) const {
   auto *Inst = dyn_cast<Instruction>(V);
   if (!Inst || DT.dominates(Inst, Loc) || Visited.count(Inst))
     return true;
@@ -500,7 +516,7 @@ bool GuardWideningImpl::isAvailableAt(Value *V, Instruction *Loc,
                 [&](Value *Op) { return isAvailableAt(Op, Loc, Visited); });
 }
 
-void GuardWideningImpl::makeAvailableAt(Value *V, Instruction *Loc) {
+void GuardWideningImpl::makeAvailableAt(Value *V, Instruction *Loc) const {
   auto *Inst = dyn_cast<Instruction>(V);
   if (!Inst || DT.dominates(Inst, Loc))
     return;
@@ -598,7 +614,7 @@ bool GuardWideningImpl::widenCondCommon(Value *Cond0, Value *Cond1,
 
 bool GuardWideningImpl::parseRangeChecks(
     Value *CheckCond, SmallVectorImpl<GuardWideningImpl::RangeCheck> &Checks,
-    SmallPtrSetImpl<Value *> &Visited) {
+    SmallPtrSetImpl<const Value *> &Visited) {
   if (!Visited.insert(CheckCond).second)
     return true;
 
@@ -617,7 +633,7 @@ bool GuardWideningImpl::parseRangeChecks(
        IC->getPredicate() != ICmpInst::ICMP_UGT))
     return false;
 
-  Value *CmpLHS = IC->getOperand(0), *CmpRHS = IC->getOperand(1);
+  const Value *CmpLHS = IC->getOperand(0), *CmpRHS = IC->getOperand(1);
   if (IC->getPredicate() == ICmpInst::ICMP_UGT)
     std::swap(CmpLHS, CmpRHS);
 
@@ -670,13 +686,13 @@ bool GuardWideningImpl::parseRangeChecks(
 
 bool GuardWideningImpl::combineRangeChecks(
     SmallVectorImpl<GuardWideningImpl::RangeCheck> &Checks,
-    SmallVectorImpl<GuardWideningImpl::RangeCheck> &RangeChecksOut) {
+    SmallVectorImpl<GuardWideningImpl::RangeCheck> &RangeChecksOut) const {
   unsigned OldCount = Checks.size();
   while (!Checks.empty()) {
     // Pick all of the range checks with a specific base and length, and try to
     // merge them.
-    Value *CurrentBase = Checks.front().getBase();
-    Value *CurrentLength = Checks.front().getLength();
+    const Value *CurrentBase = Checks.front().getBase();
+    const Value *CurrentLength = Checks.front().getLength();
 
     SmallVector<GuardWideningImpl::RangeCheck, 3> CurrentChecks;
 
@@ -705,8 +721,8 @@ bool GuardWideningImpl::combineRangeChecks(
 
     // Note: std::sort should not invalidate the ChecksStart iterator.
 
-    ConstantInt *MinOffset = CurrentChecks.front().getOffset(),
-                *MaxOffset = CurrentChecks.back().getOffset();
+    const ConstantInt *MinOffset = CurrentChecks.front().getOffset();
+    const ConstantInt *MaxOffset = CurrentChecks.back().getOffset();
 
     unsigned BitWidth = MaxOffset->getValue().getBitWidth();
     if ((MaxOffset->getValue() - MinOffset->getValue())
diff --git a/lib/Transforms/Scalar/IVUsersPrinter.cpp b/lib/Transforms/Scalar/IVUsersPrinter.cpp
index 8075933792837530d04e0dfba34ed19f97759093..e2022aba97c46f7d6e35155d6210ab0784c1add6 100644
--- a/lib/Transforms/Scalar/IVUsersPrinter.cpp
+++ b/lib/Transforms/Scalar/IVUsersPrinter.cpp
@@ -1,9 +1,8 @@
 //===- IVUsersPrinter.cpp - Induction Variable Users Printer ----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Transforms/Scalar/IndVarSimplify.cpp b/lib/Transforms/Scalar/IndVarSimplify.cpp
index 48d8e457ba7c1d561b28a6b935c5b5e51749ed64..ecf0bef13df252cf45d9a9f8c2cbf22bbd786cef 100644
--- a/lib/Transforms/Scalar/IndVarSimplify.cpp
+++ b/lib/Transforms/Scalar/IndVarSimplify.cpp
@@ -1,9 +1,8 @@
 //===- IndVarSimplify.cpp - Induction Variable Elimination ----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -218,7 +217,9 @@ bool IndVarSimplify::isValidRewrite(Value *FromVal, Value *ToVal) {
 /// Determine the insertion point for this user. By default, insert immediately
 /// before the user. SCEVExpander or LICM will hoist loop invariants out of the
 /// loop. For PHI nodes, there may be multiple uses, so compute the nearest
-/// common dominator for the incoming blocks.
+/// common dominator for the incoming blocks. A nullptr can be returned if no
+/// viable location is found: it may happen if User is a PHI and Def only comes
+/// to this PHI from unreachable blocks.
 static Instruction *getInsertPointForUses(Instruction *User, Value *Def,
                                           DominatorTree *DT, LoopInfo *LI) {
   PHINode *PHI = dyn_cast<PHINode>(User);
@@ -231,6 +232,10 @@ static Instruction *getInsertPointForUses(Instruction *User, Value *Def,
       continue;
 
     BasicBlock *InsertBB = PHI->getIncomingBlock(i);
+
+    if (!DT->isReachableFromEntry(InsertBB))
+      continue;
+
     if (!InsertPt) {
       InsertPt = InsertBB->getTerminator();
       continue;
@@ -238,7 +243,11 @@ static Instruction *getInsertPointForUses(Instruction *User, Value *Def,
     InsertBB = DT->findNearestCommonDominator(InsertPt->getParent(), InsertBB);
     InsertPt = InsertBB->getTerminator();
   }
-  assert(InsertPt && "Missing phi operand");
+
+  // If we have skipped all inputs, it means that Def only comes to Phi from
+  // unreachable blocks.
+  if (!InsertPt)
+    return nullptr;
 
   auto *DefI = dyn_cast<Instruction>(Def);
   if (!DefI)
@@ -1308,10 +1317,12 @@ WidenIV::WidenedRecTy WidenIV::getWideRecurrence(NarrowIVDefUse DU) {
 /// This IV user cannot be widen. Replace this use of the original narrow IV
 /// with a truncation of the new wide IV to isolate and eliminate the narrow IV.
 static void truncateIVUse(NarrowIVDefUse DU, DominatorTree *DT, LoopInfo *LI) {
+  auto *InsertPt = getInsertPointForUses(DU.NarrowUse, DU.NarrowDef, DT, LI);
+  if (!InsertPt)
+    return;
   LLVM_DEBUG(dbgs() << "INDVARS: Truncate IV " << *DU.WideDef << " for user "
                     << *DU.NarrowUse << "\n");
-  IRBuilder<> Builder(
-      getInsertPointForUses(DU.NarrowUse, DU.NarrowDef, DT, LI));
+  IRBuilder<> Builder(InsertPt);
   Value *Trunc = Builder.CreateTrunc(DU.WideDef, DU.NarrowDef->getType());
   DU.NarrowUse->replaceUsesOfWith(DU.NarrowDef, Trunc);
 }
@@ -1348,8 +1359,10 @@ bool WidenIV::widenLoopCompare(NarrowIVDefUse DU) {
   assert(CastWidth <= IVWidth && "Unexpected width while widening compare.");
 
   // Widen the compare instruction.
-  IRBuilder<> Builder(
-      getInsertPointForUses(DU.NarrowUse, DU.NarrowDef, DT, LI));
+  auto *InsertPt = getInsertPointForUses(DU.NarrowUse, DU.NarrowDef, DT, LI);
+  if (!InsertPt)
+    return false;
+  IRBuilder<> Builder(InsertPt);
   DU.NarrowUse->replaceUsesOfWith(DU.NarrowDef, DU.WideDef);
 
   // Widen the other operand of the compare, if necessary.
@@ -2289,7 +2302,8 @@ static Value *genLoopLimit(PHINode *IndVar, const SCEV *IVCount, Loop *L,
            "unit stride pointer IV must be i8*");
 
     IRBuilder<> Builder(L->getLoopPreheader()->getTerminator());
-    return Builder.CreateGEP(nullptr, GEPBase, GEPOffset, "lftr.limit");
+    return Builder.CreateGEP(GEPBase->getType()->getPointerElementType(),
+                             GEPBase, GEPOffset, "lftr.limit");
   } else {
     // In any other case, convert both IVInit and IVCount to integers before
     // comparing. This may result in SCEV expansion of pointers, but in practice
diff --git a/lib/Transforms/Scalar/InductiveRangeCheckElimination.cpp b/lib/Transforms/Scalar/InductiveRangeCheckElimination.cpp
index c5ed6d5c1b87f55a3175eddbe5d5790552ce91d8..f89b8e498f75e3d7e7b155419d8e11cab6be8773 100644
--- a/lib/Transforms/Scalar/InductiveRangeCheckElimination.cpp
+++ b/lib/Transforms/Scalar/InductiveRangeCheckElimination.cpp
@@ -1,9 +1,8 @@
 //===- InductiveRangeCheckElimination.cpp - -------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -116,6 +115,11 @@ static cl::opt<bool> SkipProfitabilityChecks("irce-skip-profitability-checks",
 static cl::opt<bool> AllowUnsignedLatchCondition("irce-allow-unsigned-latch",
                                                  cl::Hidden, cl::init(true));
 
+static cl::opt<bool> AllowNarrowLatchCondition(
+    "irce-allow-narrow-latch", cl::Hidden, cl::init(true),
+    cl::desc("If set to true, IRCE may eliminate wide range checks in loops "
+             "with narrow latch condition."));
+
 static const char *ClonedLoopTag = "irce.loop.clone";
 
 #define DEBUG_TYPE "irce"
@@ -133,34 +137,16 @@ namespace {
 ///     taken by the containing loop's induction variable.
 ///
 class InductiveRangeCheck {
-  // Classifies a range check
-  enum RangeCheckKind : unsigned {
-    // Range check of the form "0 <= I".
-    RANGE_CHECK_LOWER = 1,
-
-    // Range check of the form "I < L" where L is known positive.
-    RANGE_CHECK_UPPER = 2,
-
-    // The logical and of the RANGE_CHECK_LOWER and RANGE_CHECK_UPPER
-    // conditions.
-    RANGE_CHECK_BOTH = RANGE_CHECK_LOWER | RANGE_CHECK_UPPER,
-
-    // Unrecognized range check condition.
-    RANGE_CHECK_UNKNOWN = (unsigned)-1
-  };
-
-  static StringRef rangeCheckKindToStr(RangeCheckKind);
 
   const SCEV *Begin = nullptr;
   const SCEV *Step = nullptr;
   const SCEV *End = nullptr;
   Use *CheckUse = nullptr;
-  RangeCheckKind Kind = RANGE_CHECK_UNKNOWN;
   bool IsSigned = true;
 
-  static RangeCheckKind parseRangeCheckICmp(Loop *L, ICmpInst *ICI,
-                                            ScalarEvolution &SE, Value *&Index,
-                                            Value *&Length, bool &IsSigned);
+  static bool parseRangeCheckICmp(Loop *L, ICmpInst *ICI, ScalarEvolution &SE,
+                                  Value *&Index, Value *&Length,
+                                  bool &IsSigned);
 
   static void
   extractRangeChecksFromCond(Loop *L, ScalarEvolution &SE, Use &ConditionUse,
@@ -175,7 +161,6 @@ public:
 
   void print(raw_ostream &OS) const {
     OS << "InductiveRangeCheck:\n";
-    OS << "  Kind: " << rangeCheckKindToStr(Kind) << "\n";
     OS << "  Begin: ";
     Begin->print(OS);
     OS << "  Step: ";
@@ -283,32 +268,11 @@ INITIALIZE_PASS_DEPENDENCY(LoopPass)
 INITIALIZE_PASS_END(IRCELegacyPass, "irce", "Inductive range check elimination",
                     false, false)
 
-StringRef InductiveRangeCheck::rangeCheckKindToStr(
-    InductiveRangeCheck::RangeCheckKind RCK) {
-  switch (RCK) {
-  case InductiveRangeCheck::RANGE_CHECK_UNKNOWN:
-    return "RANGE_CHECK_UNKNOWN";
-
-  case InductiveRangeCheck::RANGE_CHECK_UPPER:
-    return "RANGE_CHECK_UPPER";
-
-  case InductiveRangeCheck::RANGE_CHECK_LOWER:
-    return "RANGE_CHECK_LOWER";
-
-  case InductiveRangeCheck::RANGE_CHECK_BOTH:
-    return "RANGE_CHECK_BOTH";
-  }
-
-  llvm_unreachable("unknown range check type!");
-}
-
 /// Parse a single ICmp instruction, `ICI`, into a range check.  If `ICI` cannot
-/// be interpreted as a range check, return `RANGE_CHECK_UNKNOWN` and set
-/// `Index` and `Length` to `nullptr`.  Otherwise set `Index` to the value being
-/// range checked, and set `Length` to the upper limit `Index` is being range
-/// checked with if (and only if) the range check type is stronger or equal to
-/// RANGE_CHECK_UPPER.
-InductiveRangeCheck::RangeCheckKind
+/// be interpreted as a range check, return false and set `Index` and `Length`
+/// to `nullptr`.  Otherwise set `Index` to the value being range checked, and
+/// set `Length` to the upper limit `Index` is being range checked.
+bool
 InductiveRangeCheck::parseRangeCheckICmp(Loop *L, ICmpInst *ICI,
                                          ScalarEvolution &SE, Value *&Index,
                                          Value *&Length, bool &IsSigned) {
@@ -322,7 +286,7 @@ InductiveRangeCheck::parseRangeCheckICmp(Loop *L, ICmpInst *ICI,
 
   switch (Pred) {
   default:
-    return RANGE_CHECK_UNKNOWN;
+    return false;
 
   case ICmpInst::ICMP_SLE:
     std::swap(LHS, RHS);
@@ -331,9 +295,9 @@ InductiveRangeCheck::parseRangeCheckICmp(Loop *L, ICmpInst *ICI,
     IsSigned = true;
     if (match(RHS, m_ConstantInt<0>())) {
       Index = LHS;
-      return RANGE_CHECK_LOWER;
+      return true; // Lower.
     }
-    return RANGE_CHECK_UNKNOWN;
+    return false;
 
   case ICmpInst::ICMP_SLT:
     std::swap(LHS, RHS);
@@ -342,15 +306,15 @@ InductiveRangeCheck::parseRangeCheckICmp(Loop *L, ICmpInst *ICI,
     IsSigned = true;
     if (match(RHS, m_ConstantInt<-1>())) {
       Index = LHS;
-      return RANGE_CHECK_LOWER;
+      return true; // Lower.
     }
 
     if (IsLoopInvariant(LHS)) {
       Index = RHS;
       Length = LHS;
-      return RANGE_CHECK_UPPER;
+      return true; // Upper.
     }
-    return RANGE_CHECK_UNKNOWN;
+    return false;
 
   case ICmpInst::ICMP_ULT:
     std::swap(LHS, RHS);
@@ -360,9 +324,9 @@ InductiveRangeCheck::parseRangeCheckICmp(Loop *L, ICmpInst *ICI,
     if (IsLoopInvariant(LHS)) {
       Index = RHS;
       Length = LHS;
-      return RANGE_CHECK_BOTH;
+      return true; // Both lower and upper.
     }
-    return RANGE_CHECK_UNKNOWN;
+    return false;
   }
 
   llvm_unreachable("default clause returns!");
@@ -391,8 +355,7 @@ void InductiveRangeCheck::extractRangeChecksFromCond(
 
   Value *Length = nullptr, *Index;
   bool IsSigned;
-  auto RCKind = parseRangeCheckICmp(L, ICI, SE, Index, Length, IsSigned);
-  if (RCKind == InductiveRangeCheck::RANGE_CHECK_UNKNOWN)
+  if (!parseRangeCheckICmp(L, ICI, SE, Index, Length, IsSigned))
     return;
 
   const auto *IndexAddRec = dyn_cast<SCEVAddRecExpr>(SE.getSCEV(Index));
@@ -408,7 +371,6 @@ void InductiveRangeCheck::extractRangeChecksFromCond(
   if (Length)
     End = SE.getSCEV(Length);
   else {
-    assert(RCKind == InductiveRangeCheck::RANGE_CHECK_LOWER && "invariant!");
     // So far we can only reach this point for Signed range check. This may
     // change in future. In this case we will need to pick Unsigned max for the
     // unsigned range check.
@@ -422,7 +384,6 @@ void InductiveRangeCheck::extractRangeChecksFromCond(
   IRC.Begin = IndexAddRec->getStart();
   IRC.Step = IndexAddRec->getStepRecurrence(SE);
   IRC.CheckUse = &ConditionUse;
-  IRC.Kind = RCKind;
   IRC.IsSigned = IsSigned;
   Checks.push_back(IRC);
 }
@@ -689,17 +650,6 @@ void LoopConstrainer::replacePHIBlock(PHINode *PN, BasicBlock *Block,
       PN->setIncomingBlock(i, ReplaceBy);
 }
 
-static bool CannotBeMaxInLoop(const SCEV *BoundSCEV, Loop *L,
-                              ScalarEvolution &SE, bool Signed) {
-  unsigned BitWidth = cast<IntegerType>(BoundSCEV->getType())->getBitWidth();
-  APInt Max = Signed ? APInt::getSignedMaxValue(BitWidth) :
-    APInt::getMaxValue(BitWidth);
-  auto Predicate = Signed ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT;
-  return SE.isAvailableAtLoopEntry(BoundSCEV, L) &&
-         SE.isLoopEntryGuardedByCond(L, Predicate, BoundSCEV,
-                                     SE.getConstant(Max));
-}
-
 /// Given a loop with an deccreasing induction variable, is it possible to
 /// safely calculate the bounds of a new loop using the given Predicate.
 static bool isSafeDecreasingBound(const SCEV *Start,
@@ -795,31 +745,6 @@ static bool isSafeIncreasingBound(const SCEV *Start,
           SE.isLoopEntryGuardedByCond(L, BoundPred, BoundSCEV, Limit));
 }
 
-static bool CannotBeMinInLoop(const SCEV *BoundSCEV, Loop *L,
-                              ScalarEvolution &SE, bool Signed) {
-  unsigned BitWidth = cast<IntegerType>(BoundSCEV->getType())->getBitWidth();
-  APInt Min = Signed ? APInt::getSignedMinValue(BitWidth) :
-    APInt::getMinValue(BitWidth);
-  auto Predicate = Signed ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT;
-  return SE.isAvailableAtLoopEntry(BoundSCEV, L) &&
-         SE.isLoopEntryGuardedByCond(L, Predicate, BoundSCEV,
-                                     SE.getConstant(Min));
-}
-
-static bool isKnownNonNegativeInLoop(const SCEV *BoundSCEV, const Loop *L,
-                                     ScalarEvolution &SE) {
-  const SCEV *Zero = SE.getZero(BoundSCEV->getType());
-  return SE.isAvailableAtLoopEntry(BoundSCEV, L) &&
-         SE.isLoopEntryGuardedByCond(L, ICmpInst::ICMP_SGE, BoundSCEV, Zero);
-}
-
-static bool isKnownNegativeInLoop(const SCEV *BoundSCEV, const Loop *L,
-                                  ScalarEvolution &SE) {
-  const SCEV *Zero = SE.getZero(BoundSCEV->getType());
-  return SE.isAvailableAtLoopEntry(BoundSCEV, L) &&
-         SE.isLoopEntryGuardedByCond(L, ICmpInst::ICMP_SLT, BoundSCEV, Zero);
-}
-
 Optional<LoopStructure>
 LoopStructure::parseLoopStructure(ScalarEvolution &SE,
                                   BranchProbabilityInfo *BPI, Loop &L,
@@ -977,12 +902,12 @@ LoopStructure::parseLoopStructure(ScalarEvolution &SE,
         //   ...                          ...
         // }                            }
         if (IndVarBase->getNoWrapFlags(SCEV::FlagNUW) &&
-            CannotBeMinInLoop(RightSCEV, &L, SE, /*Signed*/false)) {
+            cannotBeMinInLoop(RightSCEV, &L, SE, /*Signed*/false)) {
           Pred = ICmpInst::ICMP_UGT;
           RightSCEV = SE.getMinusSCEV(RightSCEV,
                                       SE.getOne(RightSCEV->getType()));
           DecreasedRightValueByOne = true;
-        } else if (CannotBeMinInLoop(RightSCEV, &L, SE, /*Signed*/true)) {
+        } else if (cannotBeMinInLoop(RightSCEV, &L, SE, /*Signed*/true)) {
           Pred = ICmpInst::ICMP_SGT;
           RightSCEV = SE.getMinusSCEV(RightSCEV,
                                       SE.getOne(RightSCEV->getType()));
@@ -1042,11 +967,11 @@ LoopStructure::parseLoopStructure(ScalarEvolution &SE,
         //   ...                          ...
         // }                            }
         if (IndVarBase->getNoWrapFlags(SCEV::FlagNUW) &&
-            CannotBeMaxInLoop(RightSCEV, &L, SE, /* Signed */ false)) {
+            cannotBeMaxInLoop(RightSCEV, &L, SE, /* Signed */ false)) {
           Pred = ICmpInst::ICMP_ULT;
           RightSCEV = SE.getAddExpr(RightSCEV, SE.getOne(RightSCEV->getType()));
           IncreasedRightValueByOne = true;
-        } else if (CannotBeMaxInLoop(RightSCEV, &L, SE, /* Signed */ true)) {
+        } else if (cannotBeMaxInLoop(RightSCEV, &L, SE, /* Signed */ true)) {
           Pred = ICmpInst::ICMP_SLT;
           RightSCEV = SE.getAddExpr(RightSCEV, SE.getOne(RightSCEV->getType()));
           IncreasedRightValueByOne = true;
@@ -1124,11 +1049,23 @@ LoopStructure::parseLoopStructure(ScalarEvolution &SE,
   return Result;
 }
 
+/// If the type of \p S matches with \p Ty, return \p S. Otherwise, return
+/// signed or unsigned extension of \p S to type \p Ty.
+static const SCEV *NoopOrExtend(const SCEV *S, Type *Ty, ScalarEvolution &SE,
+                                bool Signed) {
+  return Signed ? SE.getNoopOrSignExtend(S, Ty) : SE.getNoopOrZeroExtend(S, Ty);
+}
+
 Optional<LoopConstrainer::SubRanges>
 LoopConstrainer::calculateSubRanges(bool IsSignedPredicate) const {
   IntegerType *Ty = cast<IntegerType>(LatchTakenCount->getType());
 
-  if (Range.getType() != Ty)
+  auto *RTy = cast<IntegerType>(Range.getType());
+
+  // We only support wide range checks and narrow latches.
+  if (!AllowNarrowLatchCondition && RTy != Ty)
+    return None;
+  if (RTy->getBitWidth() < Ty->getBitWidth())
     return None;
 
   LoopConstrainer::SubRanges Result;
@@ -1136,8 +1073,10 @@ LoopConstrainer::calculateSubRanges(bool IsSignedPredicate) const {
   // I think we can be more aggressive here and make this nuw / nsw if the
   // addition that feeds into the icmp for the latch's terminating branch is nuw
   // / nsw.  In any case, a wrapping 2's complement addition is safe.
-  const SCEV *Start = SE.getSCEV(MainLoopStructure.IndVarStart);
-  const SCEV *End = SE.getSCEV(MainLoopStructure.LoopExitAt);
+  const SCEV *Start = NoopOrExtend(SE.getSCEV(MainLoopStructure.IndVarStart),
+                                   RTy, SE, IsSignedPredicate);
+  const SCEV *End = NoopOrExtend(SE.getSCEV(MainLoopStructure.LoopExitAt), RTy,
+                                 SE, IsSignedPredicate);
 
   bool Increasing = MainLoopStructure.IndVarIncreasing;
 
@@ -1147,7 +1086,7 @@ LoopConstrainer::calculateSubRanges(bool IsSignedPredicate) const {
 
   const SCEV *Smallest = nullptr, *Greatest = nullptr, *GreatestSeen = nullptr;
 
-  const SCEV *One = SE.getOne(Ty);
+  const SCEV *One = SE.getOne(RTy);
   if (Increasing) {
     Smallest = Start;
     Greatest = End;
@@ -1336,32 +1275,31 @@ LoopConstrainer::RewrittenRangeInfo LoopConstrainer::changeIterationSpaceEnd(
   bool IsSignedPredicate = LS.IsSignedPredicate;
 
   IRBuilder<> B(PreheaderJump);
+  auto *RangeTy = Range.getBegin()->getType();
+  auto NoopOrExt = [&](Value *V) {
+    if (V->getType() == RangeTy)
+      return V;
+    return IsSignedPredicate ? B.CreateSExt(V, RangeTy, "wide." + V->getName())
+                             : B.CreateZExt(V, RangeTy, "wide." + V->getName());
+  };
 
   // EnterLoopCond - is it okay to start executing this `LS'?
   Value *EnterLoopCond = nullptr;
-  if (Increasing)
-    EnterLoopCond = IsSignedPredicate
-                        ? B.CreateICmpSLT(LS.IndVarStart, ExitSubloopAt)
-                        : B.CreateICmpULT(LS.IndVarStart, ExitSubloopAt);
-  else
-    EnterLoopCond = IsSignedPredicate
-                        ? B.CreateICmpSGT(LS.IndVarStart, ExitSubloopAt)
-                        : B.CreateICmpUGT(LS.IndVarStart, ExitSubloopAt);
+  auto Pred =
+      Increasing
+          ? (IsSignedPredicate ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT)
+          : (IsSignedPredicate ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT);
+  Value *IndVarStart = NoopOrExt(LS.IndVarStart);
+  EnterLoopCond = B.CreateICmp(Pred, IndVarStart, ExitSubloopAt);
 
   B.CreateCondBr(EnterLoopCond, LS.Header, RRI.PseudoExit);
   PreheaderJump->eraseFromParent();
 
   LS.LatchBr->setSuccessor(LS.LatchBrExitIdx, RRI.ExitSelector);
   B.SetInsertPoint(LS.LatchBr);
-  Value *TakeBackedgeLoopCond = nullptr;
-  if (Increasing)
-    TakeBackedgeLoopCond = IsSignedPredicate
-                        ? B.CreateICmpSLT(LS.IndVarBase, ExitSubloopAt)
-                        : B.CreateICmpULT(LS.IndVarBase, ExitSubloopAt);
-  else
-    TakeBackedgeLoopCond = IsSignedPredicate
-                        ? B.CreateICmpSGT(LS.IndVarBase, ExitSubloopAt)
-                        : B.CreateICmpUGT(LS.IndVarBase, ExitSubloopAt);
+  Value *IndVarBase = NoopOrExt(LS.IndVarBase);
+  Value *TakeBackedgeLoopCond = B.CreateICmp(Pred, IndVarBase, ExitSubloopAt);
+
   Value *CondForBranch = LS.LatchBrExitIdx == 1
                              ? TakeBackedgeLoopCond
                              : B.CreateNot(TakeBackedgeLoopCond);
@@ -1373,15 +1311,8 @@ LoopConstrainer::RewrittenRangeInfo LoopConstrainer::changeIterationSpaceEnd(
   // IterationsLeft - are there any more iterations left, given the original
   // upper bound on the induction variable?  If not, we branch to the "real"
   // exit.
-  Value *IterationsLeft = nullptr;
-  if (Increasing)
-    IterationsLeft = IsSignedPredicate
-                         ? B.CreateICmpSLT(LS.IndVarBase, LS.LoopExitAt)
-                         : B.CreateICmpULT(LS.IndVarBase, LS.LoopExitAt);
-  else
-    IterationsLeft = IsSignedPredicate
-                         ? B.CreateICmpSGT(LS.IndVarBase, LS.LoopExitAt)
-                         : B.CreateICmpUGT(LS.IndVarBase, LS.LoopExitAt);
+  Value *LoopExitAt = NoopOrExt(LS.LoopExitAt);
+  Value *IterationsLeft = B.CreateICmp(Pred, IndVarBase, LoopExitAt);
   B.CreateCondBr(IterationsLeft, RRI.PseudoExit, LS.LatchExit);
 
   BranchInst *BranchToContinuation =
@@ -1400,10 +1331,10 @@ LoopConstrainer::RewrittenRangeInfo LoopConstrainer::changeIterationSpaceEnd(
     RRI.PHIValuesAtPseudoExit.push_back(NewPHI);
   }
 
-  RRI.IndVarEnd = PHINode::Create(LS.IndVarBase->getType(), 2, "indvar.end",
+  RRI.IndVarEnd = PHINode::Create(IndVarBase->getType(), 2, "indvar.end",
                                   BranchToContinuation);
-  RRI.IndVarEnd->addIncoming(LS.IndVarStart, Preheader);
-  RRI.IndVarEnd->addIncoming(LS.IndVarBase, RRI.ExitSelector);
+  RRI.IndVarEnd->addIncoming(IndVarStart, Preheader);
+  RRI.IndVarEnd->addIncoming(IndVarBase, RRI.ExitSelector);
 
   // The latch exit now has a branch from `RRI.ExitSelector' instead of
   // `LS.Latch'.  The PHI nodes need to be updated to reflect that.
@@ -1489,7 +1420,7 @@ bool LoopConstrainer::run() {
   SubRanges SR = MaybeSR.getValue();
   bool Increasing = MainLoopStructure.IndVarIncreasing;
   IntegerType *IVTy =
-      cast<IntegerType>(MainLoopStructure.IndVarBase->getType());
+      cast<IntegerType>(Range.getBegin()->getType());
 
   SCEVExpander Expander(SE, F.getParent()->getDataLayout(), "irce");
   Instruction *InsertPt = OriginalPreheader->getTerminator();
@@ -1513,16 +1444,14 @@ bool LoopConstrainer::run() {
 
     if (Increasing)
       ExitPreLoopAtSCEV = *SR.LowLimit;
+    else if (cannotBeMinInLoop(*SR.HighLimit, &OriginalLoop, SE,
+                               IsSignedPredicate))
+      ExitPreLoopAtSCEV = SE.getAddExpr(*SR.HighLimit, MinusOneS);
     else {
-      if (CannotBeMinInLoop(*SR.HighLimit, &OriginalLoop, SE,
-                            IsSignedPredicate))
-        ExitPreLoopAtSCEV = SE.getAddExpr(*SR.HighLimit, MinusOneS);
-      else {
-        LLVM_DEBUG(dbgs() << "irce: could not prove no-overflow when computing "
-                          << "preloop exit limit.  HighLimit = "
-                          << *(*SR.HighLimit) << "\n");
-        return false;
-      }
+      LLVM_DEBUG(dbgs() << "irce: could not prove no-overflow when computing "
+                        << "preloop exit limit.  HighLimit = "
+                        << *(*SR.HighLimit) << "\n");
+      return false;
     }
 
     if (!isSafeToExpandAt(ExitPreLoopAtSCEV, InsertPt, SE)) {
@@ -1542,16 +1471,14 @@ bool LoopConstrainer::run() {
 
     if (Increasing)
       ExitMainLoopAtSCEV = *SR.HighLimit;
+    else if (cannotBeMinInLoop(*SR.LowLimit, &OriginalLoop, SE,
+                               IsSignedPredicate))
+      ExitMainLoopAtSCEV = SE.getAddExpr(*SR.LowLimit, MinusOneS);
     else {
-      if (CannotBeMinInLoop(*SR.LowLimit, &OriginalLoop, SE,
-                            IsSignedPredicate))
-        ExitMainLoopAtSCEV = SE.getAddExpr(*SR.LowLimit, MinusOneS);
-      else {
-        LLVM_DEBUG(dbgs() << "irce: could not prove no-overflow when computing "
-                          << "mainloop exit limit.  LowLimit = "
-                          << *(*SR.LowLimit) << "\n");
-        return false;
-      }
+      LLVM_DEBUG(dbgs() << "irce: could not prove no-overflow when computing "
+                        << "mainloop exit limit.  LowLimit = "
+                        << *(*SR.LowLimit) << "\n");
+      return false;
     }
 
     if (!isSafeToExpandAt(ExitMainLoopAtSCEV, InsertPt, SE)) {
@@ -1656,6 +1583,12 @@ Optional<InductiveRangeCheck::Range>
 InductiveRangeCheck::computeSafeIterationSpace(
     ScalarEvolution &SE, const SCEVAddRecExpr *IndVar,
     bool IsLatchSigned) const {
+  // We can deal when types of latch check and range checks don't match in case
+  // if latch check is more narrow.
+  auto *IVType = cast<IntegerType>(IndVar->getType());
+  auto *RCType = cast<IntegerType>(getBegin()->getType());
+  if (IVType->getBitWidth() > RCType->getBitWidth())
+    return None;
   // IndVar is of the form "A + B * I" (where "I" is the canonical induction
   // variable, that may or may not exist as a real llvm::Value in the loop) and
   // this inductive range check is a range check on the "C + D * I" ("C" is
@@ -1679,8 +1612,9 @@ InductiveRangeCheck::computeSafeIterationSpace(
   if (!IndVar->isAffine())
     return None;
 
-  const SCEV *A = IndVar->getStart();
-  const SCEVConstant *B = dyn_cast<SCEVConstant>(IndVar->getStepRecurrence(SE));
+  const SCEV *A = NoopOrExtend(IndVar->getStart(), RCType, SE, IsLatchSigned);
+  const SCEVConstant *B = dyn_cast<SCEVConstant>(
+      NoopOrExtend(IndVar->getStepRecurrence(SE), RCType, SE, IsLatchSigned));
   if (!B)
     return None;
   assert(!B->isZero() && "Recurrence with zero step?");
@@ -1691,7 +1625,7 @@ InductiveRangeCheck::computeSafeIterationSpace(
     return None;
 
   assert(!D->getValue()->isZero() && "Recurrence with zero step?");
-  unsigned BitWidth = cast<IntegerType>(IndVar->getType())->getBitWidth();
+  unsigned BitWidth = RCType->getBitWidth();
   const SCEV *SIntMax = SE.getConstant(APInt::getSignedMaxValue(BitWidth));
 
   // Subtract Y from X so that it does not go through border of the IV
diff --git a/lib/Transforms/Scalar/InferAddressSpaces.cpp b/lib/Transforms/Scalar/InferAddressSpaces.cpp
index fbbc09eb487f0008aaee3ed656bc1f33387b3879..14fa24bf105f4b511ccf7c34489b9982249e9a68 100644
--- a/lib/Transforms/Scalar/InferAddressSpaces.cpp
+++ b/lib/Transforms/Scalar/InferAddressSpaces.cpp
@@ -1,9 +1,8 @@
 //===- InferAddressSpace.cpp - --------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Transforms/Scalar/InstSimplifyPass.cpp b/lib/Transforms/Scalar/InstSimplifyPass.cpp
index 05cd48d83267bad50b577fc9cf2efb6839790bcb..6616364ab2034b57fa23159bf7b4adb4f3788d23 100644
--- a/lib/Transforms/Scalar/InstSimplifyPass.cpp
+++ b/lib/Transforms/Scalar/InstSimplifyPass.cpp
@@ -1,9 +1,8 @@
 //===- InstSimplifyPass.cpp -----------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Transforms/Scalar/JumpThreading.cpp b/lib/Transforms/Scalar/JumpThreading.cpp
index 14296293ddaa67609b9565c1d513379b0d335c37..f74f7e28e527f5805e45f2d9e5ef3621b6a9880c 100644
--- a/lib/Transforms/Scalar/JumpThreading.cpp
+++ b/lib/Transforms/Scalar/JumpThreading.cpp
@@ -1,9 +1,8 @@
 //===- JumpThreading.cpp - Thread control through conditional blocks ------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -24,6 +23,7 @@
 #include "llvm/Analysis/BranchProbabilityInfo.h"
 #include "llvm/Analysis/CFG.h"
 #include "llvm/Analysis/ConstantFolding.h"
+#include "llvm/Analysis/DomTreeUpdater.h"
 #include "llvm/Analysis/GlobalsModRef.h"
 #include "llvm/Analysis/GuardUtils.h"
 #include "llvm/Analysis/InstructionSimplify.h"
@@ -38,7 +38,6 @@
 #include "llvm/IR/ConstantRange.h"
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/DataLayout.h"
-#include "llvm/IR/DomTreeUpdater.h"
 #include "llvm/IR/Dominators.h"
 #include "llvm/IR/Function.h"
 #include "llvm/IR/InstrTypes.h"
@@ -1056,7 +1055,7 @@ bool JumpThreadingPass::ProcessBlock(BasicBlock *BB) {
     Condition = IB->getAddress()->stripPointerCasts();
     Preference = WantBlockAddress;
   } else {
-    return false; // Must be an invoke.
+    return false; // Must be an invoke or callbr.
   }
 
   // Run constant folding to see if we can reduce the condition to a simple
@@ -1171,6 +1170,9 @@ bool JumpThreadingPass::ProcessBlock(BasicBlock *BB) {
     }
   }
 
+  if (SwitchInst *SI = dyn_cast<SwitchInst>(BB->getTerminator()))
+    TryToUnfoldSelect(SI, BB);
+
   // Check for some cases that are worth simplifying.  Right now we want to look
   // for loads that are used by a switch or by the condition for the branch.  If
   // we see one, check to see if it's partially redundant.  If so, insert a PHI
@@ -1426,7 +1428,9 @@ bool JumpThreadingPass::SimplifyPartiallyRedundantLoad(LoadInst *LoadI) {
     // Add all the unavailable predecessors to the PredsToSplit list.
     for (BasicBlock *P : predecessors(LoadBB)) {
       // If the predecessor is an indirect goto, we can't split the edge.
-      if (isa<IndirectBrInst>(P->getTerminator()))
+      // Same for CallBr.
+      if (isa<IndirectBrInst>(P->getTerminator()) ||
+          isa<CallBrInst>(P->getTerminator()))
         return false;
 
       if (!AvailablePredSet.count(P))
@@ -1443,11 +1447,11 @@ bool JumpThreadingPass::SimplifyPartiallyRedundantLoad(LoadInst *LoadI) {
   if (UnavailablePred) {
     assert(UnavailablePred->getTerminator()->getNumSuccessors() == 1 &&
            "Can't handle critical edge here!");
-    LoadInst *NewVal =
-        new LoadInst(LoadedPtr->DoPHITranslation(LoadBB, UnavailablePred),
-                     LoadI->getName() + ".pr", false, LoadI->getAlignment(),
-                     LoadI->getOrdering(), LoadI->getSyncScopeID(),
-                     UnavailablePred->getTerminator());
+    LoadInst *NewVal = new LoadInst(
+        LoadI->getType(), LoadedPtr->DoPHITranslation(LoadBB, UnavailablePred),
+        LoadI->getName() + ".pr", false, LoadI->getAlignment(),
+        LoadI->getOrdering(), LoadI->getSyncScopeID(),
+        UnavailablePred->getTerminator());
     NewVal->setDebugLoc(LoadI->getDebugLoc());
     if (AATags)
       NewVal->setAAMetadata(AATags);
@@ -1639,8 +1643,9 @@ bool JumpThreadingPass::ProcessThreadableEdges(Value *Cond, BasicBlock *BB,
     ++PredWithKnownDest;
 
     // If the predecessor ends with an indirect goto, we can't change its
-    // destination.
-    if (isa<IndirectBrInst>(Pred->getTerminator()))
+    // destination. Same for CallBr.
+    if (isa<IndirectBrInst>(Pred->getTerminator()) ||
+        isa<CallBrInst>(Pred->getTerminator()))
       continue;
 
     PredToDestList.push_back(std::make_pair(Pred, DestBB));
@@ -2388,6 +2393,72 @@ bool JumpThreadingPass::DuplicateCondBranchOnPHIIntoPred(
   return true;
 }
 
+// Pred is a predecessor of BB with an unconditional branch to BB. SI is
+// a Select instruction in Pred. BB has other predecessors and SI is used in
+// a PHI node in BB. SI has no other use.
+// A new basic block, NewBB, is created and SI is converted to compare and 
+// conditional branch. SI is erased from parent.
+void JumpThreadingPass::UnfoldSelectInstr(BasicBlock *Pred, BasicBlock *BB,
+                                          SelectInst *SI, PHINode *SIUse,
+                                          unsigned Idx) {
+  // Expand the select.
+  //
+  // Pred --
+  //  |    v
+  //  |  NewBB
+  //  |    |
+  //  |-----
+  //  v
+  // BB
+  BranchInst *PredTerm = dyn_cast<BranchInst>(Pred->getTerminator());
+  BasicBlock *NewBB = BasicBlock::Create(BB->getContext(), "select.unfold",
+                                         BB->getParent(), BB);
+  // Move the unconditional branch to NewBB.
+  PredTerm->removeFromParent();
+  NewBB->getInstList().insert(NewBB->end(), PredTerm);
+  // Create a conditional branch and update PHI nodes.
+  BranchInst::Create(NewBB, BB, SI->getCondition(), Pred);
+  SIUse->setIncomingValue(Idx, SI->getFalseValue());
+  SIUse->addIncoming(SI->getTrueValue(), NewBB);
+
+  // The select is now dead.
+  SI->eraseFromParent();
+  DTU->applyUpdates({{DominatorTree::Insert, NewBB, BB},
+                    {DominatorTree::Insert, Pred, NewBB}});
+
+  // Update any other PHI nodes in BB.
+  for (BasicBlock::iterator BI = BB->begin();
+       PHINode *Phi = dyn_cast<PHINode>(BI); ++BI)
+    if (Phi != SIUse)
+      Phi->addIncoming(Phi->getIncomingValueForBlock(Pred), NewBB);
+}
+
+bool JumpThreadingPass::TryToUnfoldSelect(SwitchInst *SI, BasicBlock *BB) {
+  PHINode *CondPHI = dyn_cast<PHINode>(SI->getCondition());
+
+  if (!CondPHI || CondPHI->getParent() != BB)
+    return false;
+
+  for (unsigned I = 0, E = CondPHI->getNumIncomingValues(); I != E; ++I) {
+    BasicBlock *Pred = CondPHI->getIncomingBlock(I);
+    SelectInst *PredSI = dyn_cast<SelectInst>(CondPHI->getIncomingValue(I));
+
+    // The second and third condition can be potentially relaxed. Currently
+    // the conditions help to simplify the code and allow us to reuse existing
+    // code, developed for TryToUnfoldSelect(CmpInst *, BasicBlock *)
+    if (!PredSI || PredSI->getParent() != Pred || !PredSI->hasOneUse())
+      continue;
+
+    BranchInst *PredTerm = dyn_cast<BranchInst>(Pred->getTerminator());
+    if (!PredTerm || !PredTerm->isUnconditional())
+      continue;
+
+    UnfoldSelectInstr(Pred, BB, PredSI, CondPHI, I);
+    return true;
+  }
+  return false;
+}
+
 /// TryToUnfoldSelect - Look for blocks of the form
 /// bb1:
 ///   %a = select
@@ -2438,34 +2509,7 @@ bool JumpThreadingPass::TryToUnfoldSelect(CmpInst *CondCmp, BasicBlock *BB) {
     if ((LHSFolds != LazyValueInfo::Unknown ||
          RHSFolds != LazyValueInfo::Unknown) &&
         LHSFolds != RHSFolds) {
-      // Expand the select.
-      //
-      // Pred --
-      //  |    v
-      //  |  NewBB
-      //  |    |
-      //  |-----
-      //  v
-      // BB
-      BasicBlock *NewBB = BasicBlock::Create(BB->getContext(), "select.unfold",
-                                             BB->getParent(), BB);
-      // Move the unconditional branch to NewBB.
-      PredTerm->removeFromParent();
-      NewBB->getInstList().insert(NewBB->end(), PredTerm);
-      // Create a conditional branch and update PHI nodes.
-      BranchInst::Create(NewBB, BB, SI->getCondition(), Pred);
-      CondLHS->setIncomingValue(I, SI->getFalseValue());
-      CondLHS->addIncoming(SI->getTrueValue(), NewBB);
-      // The select is now dead.
-      SI->eraseFromParent();
-
-      DTU->applyUpdates({{DominatorTree::Insert, NewBB, BB},
-                         {DominatorTree::Insert, Pred, NewBB}});
-      // Update any other PHI nodes in BB.
-      for (BasicBlock::iterator BI = BB->begin();
-           PHINode *Phi = dyn_cast<PHINode>(BI); ++BI)
-        if (Phi != CondLHS)
-          Phi->addIncoming(Phi->getIncomingValueForBlock(Pred), NewBB);
+      UnfoldSelectInstr(Pred, BB, SI, CondLHS, I);
       return true;
     }
   }
diff --git a/lib/Transforms/Scalar/LICM.cpp b/lib/Transforms/Scalar/LICM.cpp
index 695eaf6cf4a73aa42196ab555cf2574b41084469..7384ea0b0a35d7fcb6005d599d1ed9a9e5cbfeda 100644
--- a/lib/Transforms/Scalar/LICM.cpp
+++ b/lib/Transforms/Scalar/LICM.cpp
@@ -1,9 +1,8 @@
 //===-- LICM.cpp - Loop Invariant Code Motion Pass ------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -46,11 +45,11 @@
 #include "llvm/Analysis/LoopPass.h"
 #include "llvm/Analysis/MemoryBuiltins.h"
 #include "llvm/Analysis/MemorySSA.h"
+#include "llvm/Analysis/MemorySSAUpdater.h"
 #include "llvm/Analysis/OptimizationRemarkEmitter.h"
 #include "llvm/Analysis/ScalarEvolution.h"
 #include "llvm/Analysis/ScalarEvolutionAliasAnalysis.h"
 #include "llvm/Analysis/TargetLibraryInfo.h"
-#include "llvm/Transforms/Utils/Local.h"
 #include "llvm/Analysis/ValueTracking.h"
 #include "llvm/IR/CFG.h"
 #include "llvm/IR/Constants.h"
@@ -69,6 +68,7 @@
 #include "llvm/Transforms/Scalar.h"
 #include "llvm/Transforms/Scalar/LoopPassManager.h"
 #include "llvm/Transforms/Utils/BasicBlockUtils.h"
+#include "llvm/Transforms/Utils/Local.h"
 #include "llvm/Transforms/Utils/LoopUtils.h"
 #include "llvm/Transforms/Utils/SSAUpdater.h"
 #include <algorithm>
@@ -106,16 +106,41 @@ static cl::opt<int>
 LICMN2Theshold("licm-n2-threshold", cl::Hidden, cl::init(0),
                cl::desc("How many instruction to cross product using AA"));
 
+// Experimental option to allow imprecision in LICM in pathological cases, in
+// exchange for faster compile. This is to be removed if MemorySSA starts to
+// address the same issue. This flag applies only when LICM uses MemorySSA
+// instead on AliasSetTracker. LICM calls MemorySSAWalker's
+// getClobberingMemoryAccess, up to the value of the Cap, getting perfect
+// accuracy. Afterwards, LICM will call into MemorySSA's getDefiningAccess,
+// which may not be precise, since optimizeUses is capped. The result is
+// correct, but we may not get as "far up" as possible to get which access is
+// clobbering the one queried.
+static cl::opt<int> LicmMssaOptCap(
+    "licm-mssa-optimization-cap", cl::init(100), cl::Hidden,
+    cl::desc("Enable imprecision in LICM in pathological cases, in exchange "
+             "for faster compile. Caps the MemorySSA clobbering calls."));
+
+// Experimentally, memory promotion carries less importance than sinking and
+// hoisting. Limit when we do promotion when using MemorySSA, in order to save
+// compile time.
+static cl::opt<unsigned> AccessCapForMSSAPromotion(
+    "max-acc-licm-promotion", cl::init(250), cl::Hidden,
+    cl::desc("[LICM & MemorySSA] When MSSA in LICM is disabled, this has no "
+             "effect. When MSSA in LICM is enabled, then this is the maximum "
+             "number of accesses allowed to be present in a loop in order to "
+             "enable memory promotion."));
+
 static bool inSubLoop(BasicBlock *BB, Loop *CurLoop, LoopInfo *LI);
 static bool isNotUsedOrFreeInLoop(const Instruction &I, const Loop *CurLoop,
                                   const LoopSafetyInfo *SafetyInfo,
                                   TargetTransformInfo *TTI, bool &FreeInLoop);
 static void hoist(Instruction &I, const DominatorTree *DT, const Loop *CurLoop,
                   BasicBlock *Dest, ICFLoopSafetyInfo *SafetyInfo,
-                  OptimizationRemarkEmitter *ORE);
+                  MemorySSAUpdater *MSSAU, OptimizationRemarkEmitter *ORE);
 static bool sink(Instruction &I, LoopInfo *LI, DominatorTree *DT,
                  const Loop *CurLoop, ICFLoopSafetyInfo *SafetyInfo,
-                 OptimizationRemarkEmitter *ORE, bool FreeInLoop);
+                 MemorySSAUpdater *MSSAU, OptimizationRemarkEmitter *ORE,
+                 bool FreeInLoop);
 static bool isSafeToExecuteUnconditionally(Instruction &Inst,
                                            const DominatorTree *DT,
                                            const Loop *CurLoop,
@@ -125,17 +150,19 @@ static bool isSafeToExecuteUnconditionally(Instruction &Inst,
 static bool pointerInvalidatedByLoop(MemoryLocation MemLoc,
                                      AliasSetTracker *CurAST, Loop *CurLoop,
                                      AliasAnalysis *AA);
-
-static Instruction *
-CloneInstructionInExitBlock(Instruction &I, BasicBlock &ExitBlock, PHINode &PN,
-                            const LoopInfo *LI,
-                            const LoopSafetyInfo *SafetyInfo);
+static bool pointerInvalidatedByLoopWithMSSA(MemorySSA *MSSA, MemoryUse *MU,
+                                             Loop *CurLoop,
+                                             int &LicmMssaOptCounter);
+static Instruction *CloneInstructionInExitBlock(
+    Instruction &I, BasicBlock &ExitBlock, PHINode &PN, const LoopInfo *LI,
+    const LoopSafetyInfo *SafetyInfo, MemorySSAUpdater *MSSAU);
 
 static void eraseInstruction(Instruction &I, ICFLoopSafetyInfo &SafetyInfo,
-                             AliasSetTracker *AST);
+                             AliasSetTracker *AST, MemorySSAUpdater *MSSAU);
 
 static void moveInstructionBefore(Instruction &I, Instruction &Dest,
-                                  ICFLoopSafetyInfo &SafetyInfo);
+                                  ICFLoopSafetyInfo &SafetyInfo,
+                                  MemorySSAUpdater *MSSAU);
 
 namespace {
 struct LoopInvariantCodeMotion {
@@ -152,6 +179,9 @@ private:
 
   std::unique_ptr<AliasSetTracker>
   collectAliasInfoForLoop(Loop *L, LoopInfo *LI, AliasAnalysis *AA);
+  std::unique_ptr<AliasSetTracker>
+  collectAliasInfoForLoopWithMSSA(Loop *L, AliasAnalysis *AA,
+                                  MemorySSAUpdater *MSSAU);
 };
 
 struct LegacyLICMPass : public LoopPass {
@@ -194,8 +224,10 @@ struct LegacyLICMPass : public LoopPass {
     AU.addPreserved<DominatorTreeWrapperPass>();
     AU.addPreserved<LoopInfoWrapperPass>();
     AU.addRequired<TargetLibraryInfoWrapperPass>();
-    if (EnableMSSALoopDependency)
+    if (EnableMSSALoopDependency) {
       AU.addRequired<MemorySSAWrapperPass>();
+      AU.addPreserved<MemorySSAWrapperPass>();
+    }
     AU.addRequired<TargetTransformInfoWrapperPass>();
     getLoopAnalysisUsage(AU);
   }
@@ -275,7 +307,34 @@ bool LoopInvariantCodeMotion::runOnLoop(
 
   assert(L->isLCSSAForm(*DT) && "Loop is not in LCSSA form.");
 
-  std::unique_ptr<AliasSetTracker> CurAST = collectAliasInfoForLoop(L, LI, AA);
+  std::unique_ptr<AliasSetTracker> CurAST;
+  std::unique_ptr<MemorySSAUpdater> MSSAU;
+  bool NoOfMemAccTooLarge = false;
+  int LicmMssaOptCounter = 0;
+
+  if (!MSSA) {
+    LLVM_DEBUG(dbgs() << "LICM: Using Alias Set Tracker.\n");
+    CurAST = collectAliasInfoForLoop(L, LI, AA);
+  } else {
+    LLVM_DEBUG(dbgs() << "LICM: Using MemorySSA.\n");
+    MSSAU = make_unique<MemorySSAUpdater>(MSSA);
+
+    unsigned AccessCapCount = 0;
+    for (auto *BB : L->getBlocks()) {
+      if (auto *Accesses = MSSA->getBlockAccesses(BB)) {
+        for (const auto &MA : *Accesses) {
+          (void)MA;
+          AccessCapCount++;
+          if (AccessCapCount > AccessCapForMSSAPromotion) {
+            NoOfMemAccTooLarge = true;
+            break;
+          }
+        }
+      }
+      if (NoOfMemAccTooLarge)
+        break;
+    }
+  }
 
   // Get the preheader block to move instructions into...
   BasicBlock *Preheader = L->getLoopPreheader();
@@ -296,10 +355,12 @@ bool LoopInvariantCodeMotion::runOnLoop(
   //
   if (L->hasDedicatedExits())
     Changed |= sinkRegion(DT->getNode(L->getHeader()), AA, LI, DT, TLI, TTI, L,
-                          CurAST.get(), &SafetyInfo, ORE);
+                          CurAST.get(), MSSAU.get(), &SafetyInfo,
+                          NoOfMemAccTooLarge, LicmMssaOptCounter, ORE);
   if (Preheader)
     Changed |= hoistRegion(DT->getNode(L->getHeader()), AA, LI, DT, TLI, L,
-                           CurAST.get(), &SafetyInfo, ORE);
+                           CurAST.get(), MSSAU.get(), &SafetyInfo,
+                           NoOfMemAccTooLarge, LicmMssaOptCounter, ORE);
 
   // Now that all loop invariants have been removed from the loop, promote any
   // memory references to scalars that we can.
@@ -308,7 +369,8 @@ bool LoopInvariantCodeMotion::runOnLoop(
   // make sure we catch that. An additional load may be generated in the
   // preheader for SSA updater, so also avoid sinking when no preheader
   // is available.
-  if (!DisablePromotion && Preheader && L->hasDedicatedExits()) {
+  if (!DisablePromotion && Preheader && L->hasDedicatedExits() &&
+      !NoOfMemAccTooLarge) {
     // Figure out the loop exits and their insertion points
     SmallVector<BasicBlock *, 8> ExitBlocks;
     L->getUniqueExitBlocks(ExitBlocks);
@@ -320,14 +382,24 @@ bool LoopInvariantCodeMotion::runOnLoop(
 
     if (!HasCatchSwitch) {
       SmallVector<Instruction *, 8> InsertPts;
+      SmallVector<MemoryAccess *, 8> MSSAInsertPts;
       InsertPts.reserve(ExitBlocks.size());
-      for (BasicBlock *ExitBlock : ExitBlocks)
+      if (MSSAU)
+        MSSAInsertPts.reserve(ExitBlocks.size());
+      for (BasicBlock *ExitBlock : ExitBlocks) {
         InsertPts.push_back(&*ExitBlock->getFirstInsertionPt());
+        if (MSSAU)
+          MSSAInsertPts.push_back(nullptr);
+      }
 
       PredIteratorCache PIC;
 
       bool Promoted = false;
 
+      // Build an AST using MSSA.
+      if (!CurAST.get())
+        CurAST = collectAliasInfoForLoopWithMSSA(L, AA, MSSAU.get());
+
       // Loop over all of the alias sets in the tracker object.
       for (AliasSet &AS : *CurAST) {
         // We can promote this alias set if it has a store, if it is a "Must"
@@ -346,8 +418,8 @@ bool LoopInvariantCodeMotion::runOnLoop(
           PointerMustAliases.insert(ASI.getValue());
 
         Promoted |= promoteLoopAccessesToScalars(
-            PointerMustAliases, ExitBlocks, InsertPts, PIC, LI, DT, TLI, L,
-            CurAST.get(), &SafetyInfo, ORE);
+            PointerMustAliases, ExitBlocks, InsertPts, MSSAInsertPts, PIC, LI,
+            DT, TLI, L, CurAST.get(), MSSAU.get(), &SafetyInfo, ORE);
       }
 
       // Once we have promoted values across the loop body we have to
@@ -372,9 +444,12 @@ bool LoopInvariantCodeMotion::runOnLoop(
 
   // If this loop is nested inside of another one, save the alias information
   // for when we process the outer loop.
-  if (L->getParentLoop() && !DeleteAST)
+  if (!MSSAU.get() && CurAST.get() && L->getParentLoop() && !DeleteAST)
     LoopToAliasSetMap[L] = std::move(CurAST);
 
+  if (MSSAU.get() && VerifyMemorySSA)
+    MSSAU->getMemorySSA()->verifyMemorySSA();
+
   if (Changed && SE)
     SE->forgetLoopDispositions(L);
   return Changed;
@@ -388,13 +463,16 @@ bool LoopInvariantCodeMotion::runOnLoop(
 bool llvm::sinkRegion(DomTreeNode *N, AliasAnalysis *AA, LoopInfo *LI,
                       DominatorTree *DT, TargetLibraryInfo *TLI,
                       TargetTransformInfo *TTI, Loop *CurLoop,
-                      AliasSetTracker *CurAST, ICFLoopSafetyInfo *SafetyInfo,
-                      OptimizationRemarkEmitter *ORE) {
+                      AliasSetTracker *CurAST, MemorySSAUpdater *MSSAU,
+                      ICFLoopSafetyInfo *SafetyInfo, bool NoOfMemAccTooLarge,
+                      int &LicmMssaOptCounter, OptimizationRemarkEmitter *ORE) {
 
   // Verify inputs.
   assert(N != nullptr && AA != nullptr && LI != nullptr && DT != nullptr &&
-         CurLoop != nullptr && CurAST && SafetyInfo != nullptr &&
-         "Unexpected input to sinkRegion");
+         CurLoop != nullptr && SafetyInfo != nullptr &&
+         "Unexpected input to sinkRegion.");
+  assert(((CurAST != nullptr) ^ (MSSAU != nullptr)) &&
+         "Either AliasSetTracker or MemorySSA should be initialized.");
 
   // We want to visit children before parents. We will enque all the parents
   // before their children in the worklist and process the worklist in reverse
@@ -418,7 +496,7 @@ bool llvm::sinkRegion(DomTreeNode *N, AliasAnalysis *AA, LoopInfo *LI,
         LLVM_DEBUG(dbgs() << "LICM deleting dead inst: " << I << '\n');
         salvageDebugInfo(I);
         ++II;
-        eraseInstruction(I, *SafetyInfo, CurAST);
+        eraseInstruction(I, *SafetyInfo, CurAST, MSSAU);
         Changed = true;
         continue;
       }
@@ -430,21 +508,25 @@ bool llvm::sinkRegion(DomTreeNode *N, AliasAnalysis *AA, LoopInfo *LI,
       //
       bool FreeInLoop = false;
       if (isNotUsedOrFreeInLoop(I, CurLoop, SafetyInfo, TTI, FreeInLoop) &&
-          canSinkOrHoistInst(I, AA, DT, CurLoop, CurAST, true, ORE) &&
+          canSinkOrHoistInst(I, AA, DT, CurLoop, CurAST, MSSAU, true,
+                             NoOfMemAccTooLarge, &LicmMssaOptCounter, ORE) &&
           !I.mayHaveSideEffects()) {
-        if (sink(I, LI, DT, CurLoop, SafetyInfo, ORE, FreeInLoop)) {
+        if (sink(I, LI, DT, CurLoop, SafetyInfo, MSSAU, ORE, FreeInLoop)) {
           if (!FreeInLoop) {
             ++II;
-            eraseInstruction(I, *SafetyInfo, CurAST);
+            eraseInstruction(I, *SafetyInfo, CurAST, MSSAU);
           }
           Changed = true;
         }
       }
     }
   }
+  if (MSSAU && VerifyMemorySSA)
+    MSSAU->getMemorySSA()->verifyMemorySSA();
   return Changed;
 }
 
+namespace {
 // This is a helper class for hoistRegion to make it able to hoist control flow
 // in order to be able to hoist phis. The way this works is that we initially
 // start hoisting to the loop preheader, and when we see a loop invariant branch
@@ -458,6 +540,7 @@ private:
   LoopInfo *LI;
   DominatorTree *DT;
   Loop *CurLoop;
+  MemorySSAUpdater *MSSAU;
 
   // A map of blocks in the loop to the block their instructions will be hoisted
   // to.
@@ -468,8 +551,9 @@ private:
   DenseMap<BranchInst *, BasicBlock *> HoistableBranches;
 
 public:
-  ControlFlowHoister(LoopInfo *LI, DominatorTree *DT, Loop *CurLoop)
-      : LI(LI), DT(DT), CurLoop(CurLoop) {}
+  ControlFlowHoister(LoopInfo *LI, DominatorTree *DT, Loop *CurLoop,
+                     MemorySSAUpdater *MSSAU)
+      : LI(LI), DT(DT), CurLoop(CurLoop), MSSAU(MSSAU) {}
 
   void registerPossiblyHoistableBranch(BranchInst *BI) {
     // We can only hoist conditional branches with loop invariant operands.
@@ -644,6 +728,9 @@ public:
     if (HoistTarget == InitialPreheader) {
       // Phis in the loop header now need to use the new preheader.
       InitialPreheader->replaceSuccessorsPhiUsesWith(HoistCommonSucc);
+      if (MSSAU)
+        MSSAU->wireOldPredecessorsToNewImmediatePredecessor(
+            HoistTarget->getSingleSuccessor(), HoistCommonSucc, {HoistTarget});
       // The new preheader dominates the loop header.
       DomTreeNode *PreheaderNode = DT->getNode(HoistCommonSucc);
       DomTreeNode *HeaderNode = DT->getNode(CurLoop->getHeader());
@@ -666,6 +753,7 @@ public:
     return HoistDestinationMap[BB];
   }
 };
+} // namespace
 
 /// Walk the specified region of the CFG (defined by all blocks dominated by
 /// the specified block, and that are in the current loop) in depth first
@@ -674,14 +762,18 @@ public:
 ///
 bool llvm::hoistRegion(DomTreeNode *N, AliasAnalysis *AA, LoopInfo *LI,
                        DominatorTree *DT, TargetLibraryInfo *TLI, Loop *CurLoop,
-                       AliasSetTracker *CurAST, ICFLoopSafetyInfo *SafetyInfo,
+                       AliasSetTracker *CurAST, MemorySSAUpdater *MSSAU,
+                       ICFLoopSafetyInfo *SafetyInfo, bool NoOfMemAccTooLarge,
+                       int &LicmMssaOptCounter,
                        OptimizationRemarkEmitter *ORE) {
   // Verify inputs.
   assert(N != nullptr && AA != nullptr && LI != nullptr && DT != nullptr &&
-         CurLoop != nullptr && CurAST != nullptr && SafetyInfo != nullptr &&
-         "Unexpected input to hoistRegion");
+         CurLoop != nullptr && SafetyInfo != nullptr &&
+         "Unexpected input to hoistRegion.");
+  assert(((CurAST != nullptr) ^ (MSSAU != nullptr)) &&
+         "Either AliasSetTracker or MemorySSA should be initialized.");
 
-  ControlFlowHoister CFH(LI, DT, CurLoop);
+  ControlFlowHoister CFH(LI, DT, CurLoop, MSSAU);
 
   // Keep track of instructions that have been hoisted, as they may need to be
   // re-hoisted if they end up not dominating all of their uses.
@@ -708,10 +800,12 @@ bool llvm::hoistRegion(DomTreeNode *N, AliasAnalysis *AA, LoopInfo *LI,
               &I, I.getModule()->getDataLayout(), TLI)) {
         LLVM_DEBUG(dbgs() << "LICM folding inst: " << I << "  --> " << *C
                           << '\n');
-        CurAST->copyValue(&I, C);
+        if (CurAST)
+          CurAST->copyValue(&I, C);
+        // FIXME MSSA: Such replacements may make accesses unoptimized (D51960).
         I.replaceAllUsesWith(C);
         if (isInstructionTriviallyDead(&I, TLI))
-          eraseInstruction(I, *SafetyInfo, CurAST);
+          eraseInstruction(I, *SafetyInfo, CurAST, MSSAU);
         Changed = true;
         continue;
       }
@@ -723,11 +817,13 @@ bool llvm::hoistRegion(DomTreeNode *N, AliasAnalysis *AA, LoopInfo *LI,
       // and we have accurately duplicated the control flow from the loop header
       // to that block.
       if (CurLoop->hasLoopInvariantOperands(&I) &&
-          canSinkOrHoistInst(I, AA, DT, CurLoop, CurAST, true, ORE) &&
+          canSinkOrHoistInst(I, AA, DT, CurLoop, CurAST, MSSAU, true,
+                             NoOfMemAccTooLarge, &LicmMssaOptCounter, ORE) &&
           isSafeToExecuteUnconditionally(
               I, DT, CurLoop, SafetyInfo, ORE,
               CurLoop->getLoopPreheader()->getTerminator())) {
-        hoist(I, DT, CurLoop, CFH.getOrCreateHoistedBlock(BB), SafetyInfo, ORE);
+        hoist(I, DT, CurLoop, CFH.getOrCreateHoistedBlock(BB), SafetyInfo,
+              MSSAU, ORE);
         HoistedInstructions.push_back(&I);
         Changed = true;
         continue;
@@ -742,19 +838,19 @@ bool llvm::hoistRegion(DomTreeNode *N, AliasAnalysis *AA, LoopInfo *LI,
         auto One = llvm::ConstantFP::get(Divisor->getType(), 1.0);
         auto ReciprocalDivisor = BinaryOperator::CreateFDiv(One, Divisor);
         ReciprocalDivisor->setFastMathFlags(I.getFastMathFlags());
-        SafetyInfo->insertInstructionTo(I.getParent());
+        SafetyInfo->insertInstructionTo(ReciprocalDivisor, I.getParent());
         ReciprocalDivisor->insertBefore(&I);
 
         auto Product =
             BinaryOperator::CreateFMul(I.getOperand(0), ReciprocalDivisor);
         Product->setFastMathFlags(I.getFastMathFlags());
-        SafetyInfo->insertInstructionTo(I.getParent());
+        SafetyInfo->insertInstructionTo(Product, I.getParent());
         Product->insertAfter(&I);
         I.replaceAllUsesWith(Product);
-        eraseInstruction(I, *SafetyInfo, CurAST);
+        eraseInstruction(I, *SafetyInfo, CurAST, MSSAU);
 
         hoist(*ReciprocalDivisor, DT, CurLoop, CFH.getOrCreateHoistedBlock(BB),
-              SafetyInfo, ORE);
+              SafetyInfo, MSSAU, ORE);
         HoistedInstructions.push_back(ReciprocalDivisor);
         Changed = true;
         continue;
@@ -767,7 +863,8 @@ bool llvm::hoistRegion(DomTreeNode *N, AliasAnalysis *AA, LoopInfo *LI,
           CurLoop->hasLoopInvariantOperands(&I) &&
           SafetyInfo->isGuaranteedToExecute(I, DT, CurLoop) &&
           SafetyInfo->doesNotWriteMemoryBefore(I, CurLoop)) {
-        hoist(I, DT, CurLoop, CFH.getOrCreateHoistedBlock(BB), SafetyInfo, ORE);
+        hoist(I, DT, CurLoop, CFH.getOrCreateHoistedBlock(BB), SafetyInfo,
+              MSSAU, ORE);
         HoistedInstructions.push_back(&I);
         Changed = true;
         continue;
@@ -781,7 +878,7 @@ bool llvm::hoistRegion(DomTreeNode *N, AliasAnalysis *AA, LoopInfo *LI,
             PN->setIncomingBlock(
                 i, CFH.getOrCreateHoistedBlock(PN->getIncomingBlock(i)));
           hoist(*PN, DT, CurLoop, CFH.getOrCreateHoistedBlock(BB), SafetyInfo,
-                ORE);
+                MSSAU, ORE);
           assert(DT->dominates(PN, BB) && "Conditional PHIs not expected");
           Changed = true;
           continue;
@@ -809,22 +906,26 @@ bool llvm::hoistRegion(DomTreeNode *N, AliasAnalysis *AA, LoopInfo *LI,
                         [&](Use &U) { return DT->dominates(I, U); })) {
         BasicBlock *Dominator =
             DT->getNode(I->getParent())->getIDom()->getBlock();
-        LLVM_DEBUG(dbgs() << "LICM rehoisting to " << Dominator->getName()
-                          << ": " << *I << "\n");
-        if (!HoistPoint || HoistPoint->getParent() != Dominator) {
+        if (!HoistPoint || !DT->dominates(HoistPoint->getParent(), Dominator)) {
           if (HoistPoint)
             assert(DT->dominates(Dominator, HoistPoint->getParent()) &&
                    "New hoist point expected to dominate old hoist point");
           HoistPoint = Dominator->getTerminator();
         }
-        moveInstructionBefore(*I, *HoistPoint, *SafetyInfo);
+        LLVM_DEBUG(dbgs() << "LICM rehoisting to "
+                          << HoistPoint->getParent()->getName()
+                          << ": " << *I << "\n");
+        moveInstructionBefore(*I, *HoistPoint, *SafetyInfo, MSSAU);
         HoistPoint = I;
         Changed = true;
       }
     }
   }
+  if (MSSAU && VerifyMemorySSA)
+    MSSAU->getMemorySSA()->verifyMemorySSA();
 
-  // Now that we've finished hoisting make sure that LI and DT are still valid.
+    // Now that we've finished hoisting make sure that LI and DT are still
+    // valid.
 #ifndef NDEBUG
   if (Changed) {
     assert(DT->verify(DominatorTree::VerificationLevel::Fast) &&
@@ -904,25 +1005,56 @@ bool isHoistableAndSinkableInst(Instruction &I) {
           isa<ExtractValueInst>(I) || isa<InsertValueInst>(I));
 }
 /// Return true if all of the alias sets within this AST are known not to
-/// contain a Mod.
-bool isReadOnly(AliasSetTracker *CurAST) {
-  for (AliasSet &AS : *CurAST) {
-    if (!AS.isForwardingAliasSet() && AS.isMod()) {
-      return false;
+/// contain a Mod, or if MSSA knows thare are no MemoryDefs in the loop.
+bool isReadOnly(AliasSetTracker *CurAST, const MemorySSAUpdater *MSSAU,
+                const Loop *L) {
+  if (CurAST) {
+    for (AliasSet &AS : *CurAST) {
+      if (!AS.isForwardingAliasSet() && AS.isMod()) {
+        return false;
+      }
     }
+    return true;
+  } else { /*MSSAU*/
+    for (auto *BB : L->getBlocks())
+      if (MSSAU->getMemorySSA()->getBlockDefs(BB))
+        return false;
+    return true;
   }
+}
+
+/// Return true if I is the only Instruction with a MemoryAccess in L.
+bool isOnlyMemoryAccess(const Instruction *I, const Loop *L,
+                        const MemorySSAUpdater *MSSAU) {
+  for (auto *BB : L->getBlocks())
+    if (auto *Accs = MSSAU->getMemorySSA()->getBlockAccesses(BB)) {
+      int NotAPhi = 0;
+      for (const auto &Acc : *Accs) {
+        if (isa<MemoryPhi>(&Acc))
+          continue;
+        const auto *MUD = cast<MemoryUseOrDef>(&Acc);
+        if (MUD->getMemoryInst() != I || NotAPhi++ == 1)
+          return false;
+      }
+    }
   return true;
 }
 }
 
 bool llvm::canSinkOrHoistInst(Instruction &I, AAResults *AA, DominatorTree *DT,
                               Loop *CurLoop, AliasSetTracker *CurAST,
+                              MemorySSAUpdater *MSSAU,
                               bool TargetExecutesOncePerLoop,
+                              bool NoOfMemAccTooLarge, int *LicmMssaOptCounter,
                               OptimizationRemarkEmitter *ORE) {
   // If we don't understand the instruction, bail early.
   if (!isHoistableAndSinkableInst(I))
     return false;
-  
+
+  MemorySSA *MSSA = MSSAU ? MSSAU->getMemorySSA() : nullptr;
+  if (MSSA)
+    assert(LicmMssaOptCounter != nullptr && "Counter cannot be null.");
+
   // Loads have extra constraints we have to verify before we can hoist them.
   if (LoadInst *LI = dyn_cast<LoadInst>(&I)) {
     if (!LI->isUnordered())
@@ -942,8 +1074,14 @@ bool llvm::canSinkOrHoistInst(Instruction &I, AAResults *AA, DominatorTree *DT,
     if (isLoadInvariantInLoop(LI, DT, CurLoop))
       return true;
 
-    bool Invalidated = pointerInvalidatedByLoop(MemoryLocation::get(LI),
-                                                CurAST, CurLoop, AA);
+    bool Invalidated;
+    if (CurAST)
+      Invalidated = pointerInvalidatedByLoop(MemoryLocation::get(LI), CurAST,
+                                             CurLoop, AA);
+    else
+      Invalidated = pointerInvalidatedByLoopWithMSSA(
+          MSSA, cast<MemoryUse>(MSSA->getMemoryAccess(LI)), CurLoop,
+          *LicmMssaOptCounter);
     // Check loop-invariant address because this may also be a sinkable load
     // whose address is not necessarily loop-invariant.
     if (ORE && Invalidated && CurLoop->isLoopInvariant(LI->getPointerOperand()))
@@ -968,7 +1106,7 @@ bool llvm::canSinkOrHoistInst(Instruction &I, AAResults *AA, DominatorTree *DT,
     if (match(CI, m_Intrinsic<Intrinsic::assume>()))
       // Assumes don't actually alias anything or throw
       return true;
-    
+
     // Handle simple cases by querying alias analysis.
     FunctionModRefBehavior Behavior = AA->getModRefBehavior(CI);
     if (Behavior == FMRB_DoesNotAccessMemory)
@@ -980,17 +1118,25 @@ bool llvm::canSinkOrHoistInst(Instruction &I, AAResults *AA, DominatorTree *DT,
       if (AliasAnalysis::onlyAccessesArgPointees(Behavior)) {
         // TODO: expand to writeable arguments
         for (Value *Op : CI->arg_operands())
-          if (Op->getType()->isPointerTy() &&
-              pointerInvalidatedByLoop(
+          if (Op->getType()->isPointerTy()) {
+            bool Invalidated;
+            if (CurAST)
+              Invalidated = pointerInvalidatedByLoop(
                   MemoryLocation(Op, LocationSize::unknown(), AAMDNodes()),
-                  CurAST, CurLoop, AA))
-            return false;
+                  CurAST, CurLoop, AA);
+            else
+              Invalidated = pointerInvalidatedByLoopWithMSSA(
+                  MSSA, cast<MemoryUse>(MSSA->getMemoryAccess(CI)), CurLoop,
+                  *LicmMssaOptCounter);
+            if (Invalidated)
+              return false;
+          }
         return true;
       }
 
       // If this call only reads from memory and there are no writes to memory
       // in the loop, we can hoist or sink the call as appropriate.
-      if (isReadOnly(CurAST))
+      if (isReadOnly(CurAST, MSSAU, CurLoop))
         return true;
     }
 
@@ -1001,18 +1147,21 @@ bool llvm::canSinkOrHoistInst(Instruction &I, AAResults *AA, DominatorTree *DT,
   } else if (auto *FI = dyn_cast<FenceInst>(&I)) {
     // Fences alias (most) everything to provide ordering.  For the moment,
     // just give up if there are any other memory operations in the loop.
-    auto Begin = CurAST->begin();
-    assert(Begin != CurAST->end() && "must contain FI");
-    if (std::next(Begin) != CurAST->end())
-      // constant memory for instance, TODO: handle better
-      return false;
-    auto *UniqueI = Begin->getUniqueInstruction();
-    if (!UniqueI)
-      // other memory op, give up
-      return false;
-    (void)FI; //suppress unused variable warning
-    assert(UniqueI == FI && "AS must contain FI");
-    return true;
+    if (CurAST) {
+      auto Begin = CurAST->begin();
+      assert(Begin != CurAST->end() && "must contain FI");
+      if (std::next(Begin) != CurAST->end())
+        // constant memory for instance, TODO: handle better
+        return false;
+      auto *UniqueI = Begin->getUniqueInstruction();
+      if (!UniqueI)
+        // other memory op, give up
+        return false;
+      (void)FI; // suppress unused variable warning
+      assert(UniqueI == FI && "AS must contain FI");
+      return true;
+    } else // MSSAU
+      return isOnlyMemoryAccess(FI, CurLoop, MSSAU);
   } else if (auto *SI = dyn_cast<StoreInst>(&I)) {
     if (!SI->isUnordered())
       return false; // Don't sink/hoist volatile or ordered atomic store!
@@ -1022,17 +1171,49 @@ bool llvm::canSinkOrHoistInst(Instruction &I, AAResults *AA, DominatorTree *DT,
     // load store promotion instead.  TODO: We can extend this to cases where
     // there is exactly one write to the location and that write dominates an
     // arbitrary number of reads in the loop.
-    auto &AS = CurAST->getAliasSetFor(MemoryLocation::get(SI));
+    if (CurAST) {
+      auto &AS = CurAST->getAliasSetFor(MemoryLocation::get(SI));
 
-    if (AS.isRef() || !AS.isMustAlias())
-      // Quick exit test, handled by the full path below as well.
-      return false;
-    auto *UniqueI = AS.getUniqueInstruction();
-    if (!UniqueI)
-      // other memory op, give up
+      if (AS.isRef() || !AS.isMustAlias())
+        // Quick exit test, handled by the full path below as well.
+        return false;
+      auto *UniqueI = AS.getUniqueInstruction();
+      if (!UniqueI)
+        // other memory op, give up
+        return false;
+      assert(UniqueI == SI && "AS must contain SI");
+      return true;
+    } else { // MSSAU
+      if (isOnlyMemoryAccess(SI, CurLoop, MSSAU))
+        return true;
+      if (*LicmMssaOptCounter < LicmMssaOptCap) {
+        auto *Source = MSSA->getSkipSelfWalker()->getClobberingMemoryAccess(SI);
+        (*LicmMssaOptCounter)++;
+        // If there are no clobbering Defs in the loop, we still need to check
+        // for interfering Uses. If there are more accesses than the Promotion
+        // cap, give up, we're not walking a list that long. Otherwise, walk the
+        // list, check each Use if it's optimized to an access outside the loop.
+        // If yes, store is safe to hoist. This is fairly restrictive, but
+        // conservatively correct.
+        // TODO: Cache set of Uses on the first walk in runOnLoop, update when
+        // moving accesses. Can also extend to dominating uses.
+        if ((!MSSA->isLiveOnEntryDef(Source) &&
+             CurLoop->contains(Source->getBlock())) ||
+            NoOfMemAccTooLarge)
+          return false;
+        for (auto *BB : CurLoop->getBlocks())
+          if (auto *Accesses = MSSA->getBlockAccesses(BB))
+            for (const auto &MA : *Accesses)
+              if (const auto *MU = dyn_cast<MemoryUse>(&MA)) {
+                auto *MD = MU->getDefiningAccess();
+                if (!MSSA->isLiveOnEntryDef(MD) &&
+                    CurLoop->contains(MD->getBlock()))
+                  return false;
+              }
+        return true;
+      }
       return false;
-    assert(UniqueI == SI && "AS must contain SI");
-    return true;
+    }
   }
 
   assert(!I.mayReadOrWriteMemory() && "unhandled aliasing");
@@ -1116,10 +1297,9 @@ static bool isNotUsedOrFreeInLoop(const Instruction &I, const Loop *CurLoop,
   return true;
 }
 
-static Instruction *
-CloneInstructionInExitBlock(Instruction &I, BasicBlock &ExitBlock, PHINode &PN,
-                            const LoopInfo *LI,
-                            const LoopSafetyInfo *SafetyInfo) {
+static Instruction *CloneInstructionInExitBlock(
+    Instruction &I, BasicBlock &ExitBlock, PHINode &PN, const LoopInfo *LI,
+    const LoopSafetyInfo *SafetyInfo, MemorySSAUpdater *MSSAU) {
   Instruction *New;
   if (auto *CI = dyn_cast<CallInst>(&I)) {
     const auto &BlockColors = SafetyInfo->getBlockColors();
@@ -1155,6 +1335,21 @@ CloneInstructionInExitBlock(Instruction &I, BasicBlock &ExitBlock, PHINode &PN,
   if (!I.getName().empty())
     New->setName(I.getName() + ".le");
 
+  MemoryAccess *OldMemAcc;
+  if (MSSAU && (OldMemAcc = MSSAU->getMemorySSA()->getMemoryAccess(&I))) {
+    // Create a new MemoryAccess and let MemorySSA set its defining access.
+    MemoryAccess *NewMemAcc = MSSAU->createMemoryAccessInBB(
+        New, nullptr, New->getParent(), MemorySSA::Beginning);
+    if (NewMemAcc) {
+      if (auto *MemDef = dyn_cast<MemoryDef>(NewMemAcc))
+        MSSAU->insertDef(MemDef, /*RenameUses=*/true);
+      else {
+        auto *MemUse = cast<MemoryUse>(NewMemAcc);
+        MSSAU->insertUse(MemUse);
+      }
+    }
+  }
+
   // Build LCSSA PHI nodes for any in-loop operands. Note that this is
   // particularly cheap because we can rip off the PHI node that we're
   // replacing for the number and blocks of the predecessors.
@@ -1178,24 +1373,32 @@ CloneInstructionInExitBlock(Instruction &I, BasicBlock &ExitBlock, PHINode &PN,
 }
 
 static void eraseInstruction(Instruction &I, ICFLoopSafetyInfo &SafetyInfo,
-                             AliasSetTracker *AST) {
+                             AliasSetTracker *AST, MemorySSAUpdater *MSSAU) {
   if (AST)
     AST->deleteValue(&I);
+  if (MSSAU)
+    MSSAU->removeMemoryAccess(&I);
   SafetyInfo.removeInstruction(&I);
   I.eraseFromParent();
 }
 
 static void moveInstructionBefore(Instruction &I, Instruction &Dest,
-                                  ICFLoopSafetyInfo &SafetyInfo) {
+                                  ICFLoopSafetyInfo &SafetyInfo,
+                                  MemorySSAUpdater *MSSAU) {
   SafetyInfo.removeInstruction(&I);
-  SafetyInfo.insertInstructionTo(Dest.getParent());
+  SafetyInfo.insertInstructionTo(&I, Dest.getParent());
   I.moveBefore(&Dest);
+  if (MSSAU)
+    if (MemoryUseOrDef *OldMemAcc = cast_or_null<MemoryUseOrDef>(
+            MSSAU->getMemorySSA()->getMemoryAccess(&I)))
+      MSSAU->moveToPlace(OldMemAcc, Dest.getParent(), MemorySSA::End);
 }
 
 static Instruction *sinkThroughTriviallyReplaceablePHI(
     PHINode *TPN, Instruction *I, LoopInfo *LI,
     SmallDenseMap<BasicBlock *, Instruction *, 32> &SunkCopies,
-    const LoopSafetyInfo *SafetyInfo, const Loop *CurLoop) {
+    const LoopSafetyInfo *SafetyInfo, const Loop *CurLoop,
+    MemorySSAUpdater *MSSAU) {
   assert(isTriviallyReplaceablePHI(*TPN, *I) &&
          "Expect only trivially replaceable PHI");
   BasicBlock *ExitBlock = TPN->getParent();
@@ -1204,8 +1407,8 @@ static Instruction *sinkThroughTriviallyReplaceablePHI(
   if (It != SunkCopies.end())
     New = It->second;
   else
-    New = SunkCopies[ExitBlock] =
-        CloneInstructionInExitBlock(*I, *ExitBlock, *TPN, LI, SafetyInfo);
+    New = SunkCopies[ExitBlock] = CloneInstructionInExitBlock(
+        *I, *ExitBlock, *TPN, LI, SafetyInfo, MSSAU);
   return New;
 }
 
@@ -1229,7 +1432,8 @@ static bool canSplitPredecessors(PHINode *PN, LoopSafetyInfo *SafetyInfo) {
 
 static void splitPredecessorsOfLoopExit(PHINode *PN, DominatorTree *DT,
                                         LoopInfo *LI, const Loop *CurLoop,
-                                        LoopSafetyInfo *SafetyInfo) {
+                                        LoopSafetyInfo *SafetyInfo,
+                                        MemorySSAUpdater *MSSAU) {
 #ifndef NDEBUG
   SmallVector<BasicBlock *, 32> ExitBlocks;
   CurLoop->getUniqueExitBlocks(ExitBlocks);
@@ -1279,7 +1483,7 @@ static void splitPredecessorsOfLoopExit(PHINode *PN, DominatorTree *DT,
            "Expect all predecessors are in the loop");
     if (PN->getBasicBlockIndex(PredBB) >= 0) {
       BasicBlock *NewPred = SplitBlockPredecessors(
-          ExitBB, PredBB, ".split.loop.exit", DT, LI, nullptr, true);
+          ExitBB, PredBB, ".split.loop.exit", DT, LI, MSSAU, true);
       // Since we do not allow splitting EH-block with BlockColors in
       // canSplitPredecessors(), we can simply assign predecessor's color to
       // the new block.
@@ -1300,7 +1504,8 @@ static void splitPredecessorsOfLoopExit(PHINode *PN, DominatorTree *DT,
 ///
 static bool sink(Instruction &I, LoopInfo *LI, DominatorTree *DT,
                  const Loop *CurLoop, ICFLoopSafetyInfo *SafetyInfo,
-                 OptimizationRemarkEmitter *ORE, bool FreeInLoop) {
+                 MemorySSAUpdater *MSSAU, OptimizationRemarkEmitter *ORE,
+                 bool FreeInLoop) {
   LLVM_DEBUG(dbgs() << "LICM sinking instruction: " << I << "\n");
   ORE->emit([&]() {
     return OptimizationRemark(DEBUG_TYPE, "InstSunk", &I)
@@ -1352,7 +1557,7 @@ static bool sink(Instruction &I, LoopInfo *LI, DominatorTree *DT,
 
     // Split predecessors of the PHI so that we can make users trivially
     // replaceable.
-    splitPredecessorsOfLoopExit(PN, DT, LI, CurLoop, SafetyInfo);
+    splitPredecessorsOfLoopExit(PN, DT, LI, CurLoop, SafetyInfo, MSSAU);
 
     // Should rebuild the iterators, as they may be invalidated by
     // splitPredecessorsOfLoopExit().
@@ -1387,10 +1592,10 @@ static bool sink(Instruction &I, LoopInfo *LI, DominatorTree *DT,
     assert(ExitBlockSet.count(PN->getParent()) &&
            "The LCSSA PHI is not in an exit block!");
     // The PHI must be trivially replaceable.
-    Instruction *New = sinkThroughTriviallyReplaceablePHI(PN, &I, LI, SunkCopies,
-                                                          SafetyInfo, CurLoop);
+    Instruction *New = sinkThroughTriviallyReplaceablePHI(
+        PN, &I, LI, SunkCopies, SafetyInfo, CurLoop, MSSAU);
     PN->replaceAllUsesWith(New);
-    eraseInstruction(*PN, *SafetyInfo, nullptr);
+    eraseInstruction(*PN, *SafetyInfo, nullptr, nullptr);
     Changed = true;
   }
   return Changed;
@@ -1401,7 +1606,7 @@ static bool sink(Instruction &I, LoopInfo *LI, DominatorTree *DT,
 ///
 static void hoist(Instruction &I, const DominatorTree *DT, const Loop *CurLoop,
                   BasicBlock *Dest, ICFLoopSafetyInfo *SafetyInfo,
-                  OptimizationRemarkEmitter *ORE) {
+                  MemorySSAUpdater *MSSAU, OptimizationRemarkEmitter *ORE) {
   LLVM_DEBUG(dbgs() << "LICM hoisting to " << Dest->getName() << ": " << I
                     << "\n");
   ORE->emit([&]() {
@@ -1422,10 +1627,10 @@ static void hoist(Instruction &I, const DominatorTree *DT, const Loop *CurLoop,
 
   if (isa<PHINode>(I))
     // Move the new node to the end of the phi list in the destination block.
-    moveInstructionBefore(I, *Dest->getFirstNonPHI(), *SafetyInfo);
+    moveInstructionBefore(I, *Dest->getFirstNonPHI(), *SafetyInfo, MSSAU);
   else
     // Move the new node to the destination block, before its terminator.
-    moveInstructionBefore(I, *Dest->getTerminator(), *SafetyInfo);
+    moveInstructionBefore(I, *Dest->getTerminator(), *SafetyInfo, MSSAU);
 
   // Do not retain debug locations when we are moving instructions to different
   // basic blocks, because we want to avoid jumpy line tables. Calls, however,
@@ -1477,8 +1682,10 @@ class LoopPromoter : public LoadAndStorePromoter {
   const SmallSetVector<Value *, 8> &PointerMustAliases;
   SmallVectorImpl<BasicBlock *> &LoopExitBlocks;
   SmallVectorImpl<Instruction *> &LoopInsertPts;
+  SmallVectorImpl<MemoryAccess *> &MSSAInsertPts;
   PredIteratorCache &PredCache;
   AliasSetTracker &AST;
+  MemorySSAUpdater *MSSAU;
   LoopInfo &LI;
   DebugLoc DL;
   int Alignment;
@@ -1505,15 +1712,16 @@ public:
   LoopPromoter(Value *SP, ArrayRef<const Instruction *> Insts, SSAUpdater &S,
                const SmallSetVector<Value *, 8> &PMA,
                SmallVectorImpl<BasicBlock *> &LEB,
-               SmallVectorImpl<Instruction *> &LIP, PredIteratorCache &PIC,
-               AliasSetTracker &ast, LoopInfo &li, DebugLoc dl, int alignment,
-               bool UnorderedAtomic, const AAMDNodes &AATags,
-               ICFLoopSafetyInfo &SafetyInfo)
+               SmallVectorImpl<Instruction *> &LIP,
+               SmallVectorImpl<MemoryAccess *> &MSSAIP, PredIteratorCache &PIC,
+               AliasSetTracker &ast, MemorySSAUpdater *MSSAU, LoopInfo &li,
+               DebugLoc dl, int alignment, bool UnorderedAtomic,
+               const AAMDNodes &AATags, ICFLoopSafetyInfo &SafetyInfo)
       : LoadAndStorePromoter(Insts, S), SomePtr(SP), PointerMustAliases(PMA),
-        LoopExitBlocks(LEB), LoopInsertPts(LIP), PredCache(PIC), AST(ast),
-        LI(li), DL(std::move(dl)), Alignment(alignment),
-        UnorderedAtomic(UnorderedAtomic), AATags(AATags), SafetyInfo(SafetyInfo)
-      {}
+        LoopExitBlocks(LEB), LoopInsertPts(LIP), MSSAInsertPts(MSSAIP),
+        PredCache(PIC), AST(ast), MSSAU(MSSAU), LI(li), DL(std::move(dl)),
+        Alignment(alignment), UnorderedAtomic(UnorderedAtomic), AATags(AATags),
+        SafetyInfo(SafetyInfo) {}
 
   bool isInstInList(Instruction *I,
                     const SmallVectorImpl<Instruction *> &) const override {
@@ -1525,7 +1733,7 @@ public:
     return PointerMustAliases.count(Ptr);
   }
 
-  void doExtraRewritesBeforeFinalDeletion() const override {
+  void doExtraRewritesBeforeFinalDeletion() override {
     // Insert stores after in the loop exit blocks.  Each exit block gets a
     // store of the live-out values that feed them.  Since we've already told
     // the SSA updater about the defs in the loop and the preheader
@@ -1543,6 +1751,21 @@ public:
       NewSI->setDebugLoc(DL);
       if (AATags)
         NewSI->setAAMetadata(AATags);
+
+      if (MSSAU) {
+        MemoryAccess *MSSAInsertPoint = MSSAInsertPts[i];
+        MemoryAccess *NewMemAcc;
+        if (!MSSAInsertPoint) {
+          NewMemAcc = MSSAU->createMemoryAccessInBB(
+              NewSI, nullptr, NewSI->getParent(), MemorySSA::Beginning);
+        } else {
+          NewMemAcc =
+              MSSAU->createMemoryAccessAfter(NewSI, nullptr, MSSAInsertPoint);
+        }
+        MSSAInsertPts[i] = NewMemAcc;
+        MSSAU->insertDef(cast<MemoryDef>(NewMemAcc), true);
+        // FIXME: true for safety, false may still be correct.
+      }
     }
   }
 
@@ -1553,6 +1776,8 @@ public:
   void instructionDeleted(Instruction *I) const override {
     SafetyInfo.removeInstruction(I);
     AST.deleteValue(I);
+    if (MSSAU)
+      MSSAU->removeMemoryAccess(I);
   }
 };
 
@@ -1589,10 +1814,11 @@ bool isKnownNonEscaping(Value *Object, const TargetLibraryInfo *TLI) {
 bool llvm::promoteLoopAccessesToScalars(
     const SmallSetVector<Value *, 8> &PointerMustAliases,
     SmallVectorImpl<BasicBlock *> &ExitBlocks,
-    SmallVectorImpl<Instruction *> &InsertPts, PredIteratorCache &PIC,
+    SmallVectorImpl<Instruction *> &InsertPts,
+    SmallVectorImpl<MemoryAccess *> &MSSAInsertPts, PredIteratorCache &PIC,
     LoopInfo *LI, DominatorTree *DT, const TargetLibraryInfo *TLI,
-    Loop *CurLoop, AliasSetTracker *CurAST, ICFLoopSafetyInfo *SafetyInfo,
-    OptimizationRemarkEmitter *ORE) {
+    Loop *CurLoop, AliasSetTracker *CurAST, MemorySSAUpdater *MSSAU,
+    ICFLoopSafetyInfo *SafetyInfo, OptimizationRemarkEmitter *ORE) {
   // Verify inputs.
   assert(LI != nullptr && DT != nullptr && CurLoop != nullptr &&
          CurAST != nullptr && SafetyInfo != nullptr &&
@@ -1809,13 +2035,14 @@ bool llvm::promoteLoopAccessesToScalars(
   SmallVector<PHINode *, 16> NewPHIs;
   SSAUpdater SSA(&NewPHIs);
   LoopPromoter Promoter(SomePtr, LoopUses, SSA, PointerMustAliases, ExitBlocks,
-                        InsertPts, PIC, *CurAST, *LI, DL, Alignment,
-                        SawUnorderedAtomic, AATags, *SafetyInfo);
+                        InsertPts, MSSAInsertPts, PIC, *CurAST, MSSAU, *LI, DL,
+                        Alignment, SawUnorderedAtomic, AATags, *SafetyInfo);
 
   // Set up the preheader to have a definition of the value.  It is the live-out
   // value from the preheader that uses in the loop will use.
   LoadInst *PreheaderLoad = new LoadInst(
-      SomePtr, SomePtr->getName() + ".promoted", Preheader->getTerminator());
+      SomePtr->getType()->getPointerElementType(), SomePtr,
+      SomePtr->getName() + ".promoted", Preheader->getTerminator());
   if (SawUnorderedAtomic)
     PreheaderLoad->setOrdering(AtomicOrdering::Unordered);
   PreheaderLoad->setAlignment(Alignment);
@@ -1824,13 +2051,21 @@ bool llvm::promoteLoopAccessesToScalars(
     PreheaderLoad->setAAMetadata(AATags);
   SSA.AddAvailableValue(Preheader, PreheaderLoad);
 
+  MemoryAccess *PreheaderLoadMemoryAccess;
+  if (MSSAU) {
+    PreheaderLoadMemoryAccess = MSSAU->createMemoryAccessInBB(
+        PreheaderLoad, nullptr, PreheaderLoad->getParent(), MemorySSA::End);
+    MemoryUse *NewMemUse = cast<MemoryUse>(PreheaderLoadMemoryAccess);
+    MSSAU->insertUse(NewMemUse);
+  }
+
   // Rewrite all the loads in the loop and remember all the definitions from
   // stores in the loop.
   Promoter.run(LoopUses);
 
   // If the SSAUpdater didn't use the load in the preheader, just zap it now.
   if (PreheaderLoad->use_empty())
-    eraseInstruction(*PreheaderLoad, *SafetyInfo, CurAST);
+    eraseInstruction(*PreheaderLoad, *SafetyInfo, CurAST, MSSAU);
 
   return true;
 }
@@ -1883,6 +2118,15 @@ LoopInvariantCodeMotion::collectAliasInfoForLoop(Loop *L, LoopInfo *LI,
   return CurAST;
 }
 
+std::unique_ptr<AliasSetTracker>
+LoopInvariantCodeMotion::collectAliasInfoForLoopWithMSSA(
+    Loop *L, AliasAnalysis *AA, MemorySSAUpdater *MSSAU) {
+  auto *MSSA = MSSAU->getMemorySSA();
+  auto CurAST = make_unique<AliasSetTracker>(*AA, MSSA, L);
+  CurAST->addAllInstructionsInLoopUsingMSSA();
+  return CurAST;
+}
+
 /// Simple analysis hook. Clone alias set info.
 ///
 void LegacyLICMPass::cloneBasicBlockAnalysis(BasicBlock *From, BasicBlock *To,
@@ -1960,6 +2204,21 @@ static bool pointerInvalidatedByLoop(MemoryLocation MemLoc,
   return false;
 }
 
+static bool pointerInvalidatedByLoopWithMSSA(MemorySSA *MSSA, MemoryUse *MU,
+                                             Loop *CurLoop,
+                                             int &LicmMssaOptCounter) {
+  MemoryAccess *Source;
+  // See declaration of LicmMssaOptCap for usage details.
+  if (LicmMssaOptCounter >= LicmMssaOptCap)
+    Source = MU->getDefiningAccess();
+  else {
+    Source = MSSA->getSkipSelfWalker()->getClobberingMemoryAccess(MU);
+    LicmMssaOptCounter++;
+  }
+  return !MSSA->isLiveOnEntryDef(Source) &&
+         CurLoop->contains(Source->getBlock());
+}
+
 /// Little predicate that returns true if the specified basic block is in
 /// a subloop of the current one, not the current one itself.
 ///
diff --git a/lib/Transforms/Scalar/LLVMBuild.txt b/lib/Transforms/Scalar/LLVMBuild.txt
index ffe35f041b35abde90892183027548be048e0977..69f6c82f27f9871ce291923f5c0b3e35584e2eb9 100644
--- a/lib/Transforms/Scalar/LLVMBuild.txt
+++ b/lib/Transforms/Scalar/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/Transforms/Scalar/LLVMBuild.txt --------------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Transforms/Scalar/LoopAccessAnalysisPrinter.cpp b/lib/Transforms/Scalar/LoopAccessAnalysisPrinter.cpp
index a64c99117d64bdaf5594749ea0d97855c0165ab0..1c3ff1a61b7e23115850e6d4cdf96146b64ac787 100644
--- a/lib/Transforms/Scalar/LoopAccessAnalysisPrinter.cpp
+++ b/lib/Transforms/Scalar/LoopAccessAnalysisPrinter.cpp
@@ -1,9 +1,8 @@
 //===- LoopAccessAnalysisPrinter.cpp - Loop Access Analysis Printer --------==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Transforms/Scalar/LoopDataPrefetch.cpp b/lib/Transforms/Scalar/LoopDataPrefetch.cpp
index 3b41b5d96c86e40d4387e8df8bbaaee7dc3094a0..1fcf1315a1777a1c2aad345a4bd660e9f3700913 100644
--- a/lib/Transforms/Scalar/LoopDataPrefetch.cpp
+++ b/lib/Transforms/Scalar/LoopDataPrefetch.cpp
@@ -1,9 +1,8 @@
 //===-------- LoopDataPrefetch.cpp - Loop Data Prefetching Pass -----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -313,7 +312,8 @@ bool LoopDataPrefetch::runOnLoop(Loop *L) {
       IRBuilder<> Builder(MemI);
       Module *M = BB->getParent()->getParent();
       Type *I32 = Type::getInt32Ty(BB->getContext());
-      Value *PrefetchFunc = Intrinsic::getDeclaration(M, Intrinsic::prefetch);
+      Function *PrefetchFunc =
+          Intrinsic::getDeclaration(M, Intrinsic::prefetch);
       Builder.CreateCall(
           PrefetchFunc,
           {PrefPtrValue,
@@ -333,4 +333,3 @@ bool LoopDataPrefetch::runOnLoop(Loop *L) {
 
   return MadeChange;
 }
-
diff --git a/lib/Transforms/Scalar/LoopDeletion.cpp b/lib/Transforms/Scalar/LoopDeletion.cpp
index d412025d7e9458dfb0735a9cf28f39c0ca9753b2..8371367e24e7e53bba966c7e02294574cfa1b660 100644
--- a/lib/Transforms/Scalar/LoopDeletion.cpp
+++ b/lib/Transforms/Scalar/LoopDeletion.cpp
@@ -1,9 +1,8 @@
 //===- LoopDeletion.cpp - Dead Loop Deletion Pass ---------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Transforms/Scalar/LoopDistribute.cpp b/lib/Transforms/Scalar/LoopDistribute.cpp
index d797c9dc9e726e02deb175f54387475f9b3f87ed..4f5344b65b7c6a36c9ddc65bdd0194377d908543 100644
--- a/lib/Transforms/Scalar/LoopDistribute.cpp
+++ b/lib/Transforms/Scalar/LoopDistribute.cpp
@@ -1,9 +1,8 @@
 //===- LoopDistribute.cpp - Loop Distribution Pass ------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Transforms/Scalar/LoopIdiomRecognize.cpp b/lib/Transforms/Scalar/LoopIdiomRecognize.cpp
index 241dbed30e1b7af32c524571061d70def9ed8a53..5f9f54c35cbbc49d40d7aff377b2f799a4590450 100644
--- a/lib/Transforms/Scalar/LoopIdiomRecognize.cpp
+++ b/lib/Transforms/Scalar/LoopIdiomRecognize.cpp
@@ -1,9 +1,8 @@
 //===- LoopIdiomRecognize.cpp - Loop idiom recognition --------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -26,7 +25,7 @@
 // Future floating point idioms to recognize in -ffast-math mode:
 //   fpowi
 // Future integer operation idioms to recognize:
-//   ctpop, ctlz, cttz
+//   ctpop
 //
 // Beware that isel's default lowering for ctpop is highly inefficient for
 // i64 and larger types when i64 is legal and the value has few bits set.  It
@@ -187,9 +186,10 @@ private:
   bool recognizePopcount();
   void transformLoopToPopcount(BasicBlock *PreCondBB, Instruction *CntInst,
                                PHINode *CntPhi, Value *Var);
-  bool recognizeAndInsertCTLZ();
-  void transformLoopToCountable(BasicBlock *PreCondBB, Instruction *CntInst,
-                                PHINode *CntPhi, Value *Var, Instruction *DefX,
+  bool recognizeAndInsertFFS();  /// Find First Set: ctlz or cttz
+  void transformLoopToCountable(Intrinsic::ID IntrinID, BasicBlock *PreCondBB,
+                                Instruction *CntInst, PHINode *CntPhi,
+                                Value *Var, Instruction *DefX,
                                 const DebugLoc &DL, bool ZeroCheck,
                                 bool IsCntPhiUsedOutsideLoop);
 
@@ -779,12 +779,13 @@ mayLoopAccessLocation(Value *Ptr, ModRefInfo Access, Loop *L,
   // Get the location that may be stored across the loop.  Since the access is
   // strided positively through memory, we say that the modified location starts
   // at the pointer and has infinite size.
-  uint64_t AccessSize = MemoryLocation::UnknownSize;
+  LocationSize AccessSize = LocationSize::unknown();
 
   // If the loop iterates a fixed number of times, we can refine the access size
   // to be exactly the size of the memset, which is (BECount+1)*StoreSize
   if (const SCEVConstant *BECst = dyn_cast<SCEVConstant>(BECount))
-    AccessSize = (BECst->getValue()->getZExtValue() + 1) * StoreSize;
+    AccessSize = LocationSize::precise((BECst->getValue()->getZExtValue() + 1) *
+                                       StoreSize);
 
   // TODO: For this to be really effective, we have to dive into the pointer
   // operand in the store.  Store to &A[i] of 100 will always return may alias
@@ -929,9 +930,8 @@ bool LoopIdiomRecognize::processLoopStridedStore(
 
     Module *M = TheStore->getModule();
     StringRef FuncName = "memset_pattern16";
-    Value *MSP =
-        M->getOrInsertFunction(FuncName, Builder.getVoidTy(),
-                               Int8PtrTy, Int8PtrTy, IntPtr);
+    FunctionCallee MSP = M->getOrInsertFunction(FuncName, Builder.getVoidTy(),
+                                                Int8PtrTy, Int8PtrTy, IntPtr);
     inferLibFuncAttributes(M, FuncName, *TLI);
 
     // Otherwise we should form a memset_pattern16.  PatternValue is known to be
@@ -1107,15 +1107,17 @@ bool LoopIdiomRecognize::avoidLIRForMultiBlockLoop(bool IsMemset,
 }
 
 bool LoopIdiomRecognize::runOnNoncountableLoop() {
-  return recognizePopcount() || recognizeAndInsertCTLZ();
+  return recognizePopcount() || recognizeAndInsertFFS();
 }
 
 /// Check if the given conditional branch is based on the comparison between
-/// a variable and zero, and if the variable is non-zero, the control yields to
-/// the loop entry. If the branch matches the behavior, the variable involved
-/// in the comparison is returned. This function will be called to see if the
-/// precondition and postcondition of the loop are in desirable form.
-static Value *matchCondition(BranchInst *BI, BasicBlock *LoopEntry) {
+/// a variable and zero, and if the variable is non-zero or zero (JmpOnZero is
+/// true), the control yields to the loop entry. If the branch matches the
+/// behavior, the variable involved in the comparison is returned. This function
+/// will be called to see if the precondition and postcondition of the loop are
+/// in desirable form.
+static Value *matchCondition(BranchInst *BI, BasicBlock *LoopEntry,
+                             bool JmpOnZero = false) {
   if (!BI || !BI->isConditional())
     return nullptr;
 
@@ -1127,9 +1129,14 @@ static Value *matchCondition(BranchInst *BI, BasicBlock *LoopEntry) {
   if (!CmpZero || !CmpZero->isZero())
     return nullptr;
 
+  BasicBlock *TrueSucc = BI->getSuccessor(0);
+  BasicBlock *FalseSucc = BI->getSuccessor(1);
+  if (JmpOnZero)
+    std::swap(TrueSucc, FalseSucc);
+
   ICmpInst::Predicate Pred = Cond->getPredicate();
-  if ((Pred == ICmpInst::ICMP_NE && BI->getSuccessor(0) == LoopEntry) ||
-      (Pred == ICmpInst::ICMP_EQ && BI->getSuccessor(1) == LoopEntry))
+  if ((Pred == ICmpInst::ICMP_NE && TrueSucc == LoopEntry) ||
+      (Pred == ICmpInst::ICMP_EQ && FalseSucc == LoopEntry))
     return Cond->getOperand(0);
 
   return nullptr;
@@ -1305,14 +1312,14 @@ static bool detectPopcountIdiom(Loop *CurLoop, BasicBlock *PreCondBB,
 ///
 /// loop-exit:
 /// \endcode
-static bool detectCTLZIdiom(Loop *CurLoop, PHINode *&PhiX,
-                            Instruction *&CntInst, PHINode *&CntPhi,
-                            Instruction *&DefX) {
+static bool detectShiftUntilZeroIdiom(Loop *CurLoop, const DataLayout &DL,
+                                      Intrinsic::ID &IntrinID, Value *&InitX,
+                                      Instruction *&CntInst, PHINode *&CntPhi,
+                                      Instruction *&DefX) {
   BasicBlock *LoopEntry;
   Value *VarX = nullptr;
 
   DefX = nullptr;
-  PhiX = nullptr;
   CntInst = nullptr;
   CntPhi = nullptr;
   LoopEntry = *(CurLoop->block_begin());
@@ -1324,20 +1331,28 @@ static bool detectCTLZIdiom(Loop *CurLoop, PHINode *&PhiX,
   else
     return false;
 
-  // step 2: detect instructions corresponding to "x.next = x >> 1"
-  if (!DefX || (DefX->getOpcode() != Instruction::AShr &&
-                DefX->getOpcode() != Instruction::LShr))
+  // step 2: detect instructions corresponding to "x.next = x >> 1 or x << 1"
+  if (!DefX || !DefX->isShift())
     return false;
+  IntrinID = DefX->getOpcode() == Instruction::Shl ? Intrinsic::cttz :
+                                                     Intrinsic::ctlz;
   ConstantInt *Shft = dyn_cast<ConstantInt>(DefX->getOperand(1));
   if (!Shft || !Shft->isOne())
     return false;
   VarX = DefX->getOperand(0);
 
   // step 3: Check the recurrence of variable X
-  PhiX = getRecurrenceVar(VarX, DefX, LoopEntry);
+  PHINode *PhiX = getRecurrenceVar(VarX, DefX, LoopEntry);
   if (!PhiX)
     return false;
 
+  InitX = PhiX->getIncomingValueForBlock(CurLoop->getLoopPreheader());
+
+  // Make sure the initial value can't be negative otherwise the ashr in the
+  // loop might never reach zero which would make the loop infinite.
+  if (DefX->getOpcode() == Instruction::AShr && !isKnownNonNegative(InitX, DL))
+    return false;
+
   // step 4: Find the instruction which count the CTLZ: cnt.next = cnt + 1
   // TODO: We can skip the step. If loop trip count is known (CTLZ),
   //       then all uses of "cnt.next" could be optimized to the trip count
@@ -1369,17 +1384,25 @@ static bool detectCTLZIdiom(Loop *CurLoop, PHINode *&PhiX,
   return true;
 }
 
-/// Recognize CTLZ idiom in a non-countable loop and convert the loop
-/// to countable (with CTLZ trip count).
-/// If CTLZ inserted as a new trip count returns true; otherwise, returns false.
-bool LoopIdiomRecognize::recognizeAndInsertCTLZ() {
+/// Recognize CTLZ or CTTZ idiom in a non-countable loop and convert the loop
+/// to countable (with CTLZ / CTTZ trip count). If CTLZ / CTTZ inserted as a new
+/// trip count returns true; otherwise, returns false.
+bool LoopIdiomRecognize::recognizeAndInsertFFS() {
   // Give up if the loop has multiple blocks or multiple backedges.
   if (CurLoop->getNumBackEdges() != 1 || CurLoop->getNumBlocks() != 1)
     return false;
 
-  Instruction *CntInst, *DefX;
-  PHINode *CntPhi, *PhiX;
-  if (!detectCTLZIdiom(CurLoop, PhiX, CntInst, CntPhi, DefX))
+  Intrinsic::ID IntrinID;
+  Value *InitX;
+  Instruction *DefX = nullptr;
+  PHINode *CntPhi = nullptr;
+  Instruction *CntInst = nullptr;
+  // Help decide if transformation is profitable. For ShiftUntilZero idiom,
+  // this is always 6.
+  size_t IdiomCanonicalSize = 6;
+
+  if (!detectShiftUntilZeroIdiom(CurLoop, *DL, IntrinID, InitX,
+                                 CntInst, CntPhi, DefX))
     return false;
 
   bool IsCntPhiUsedOutsideLoop = false;
@@ -1406,12 +1429,6 @@ bool LoopIdiomRecognize::recognizeAndInsertCTLZ() {
   // It is safe to assume Preheader exist as it was checked in
   // parent function RunOnLoop.
   BasicBlock *PH = CurLoop->getLoopPreheader();
-  Value *InitX = PhiX->getIncomingValueForBlock(PH);
-
-  // Make sure the initial value can't be negative otherwise the ashr in the
-  // loop might never reach zero which would make the loop infinite.
-  if (DefX->getOpcode() == Instruction::AShr && !isKnownNonNegative(InitX, *DL))
-    return false;
 
   // If we are using the count instruction outside the loop, make sure we
   // have a zero check as a precondition. Without the check the loop would run
@@ -1429,8 +1446,10 @@ bool LoopIdiomRecognize::recognizeAndInsertCTLZ() {
     ZeroCheck = true;
   }
 
-  // Check if CTLZ intrinsic is profitable. Assume it is always profitable
-  // if we delete the loop (the loop has only 6 instructions):
+  // Check if CTLZ / CTTZ intrinsic is profitable. Assume it is always
+  // profitable if we delete the loop.
+
+  // the loop has only 6 instructions:
   //  %n.addr.0 = phi [ %n, %entry ], [ %shr, %while.cond ]
   //  %i.0 = phi [ %i0, %entry ], [ %inc, %while.cond ]
   //  %shr = ashr %n.addr.0, 1
@@ -1441,12 +1460,18 @@ bool LoopIdiomRecognize::recognizeAndInsertCTLZ() {
   const Value *Args[] =
       {InitX, ZeroCheck ? ConstantInt::getTrue(InitX->getContext())
                         : ConstantInt::getFalse(InitX->getContext())};
-  if (CurLoop->getHeader()->size() != 6 &&
-      TTI->getIntrinsicCost(Intrinsic::ctlz, InitX->getType(), Args) >
+
+  // @llvm.dbg doesn't count as they have no semantic effect.
+  auto InstWithoutDebugIt = CurLoop->getHeader()->instructionsWithoutDebug();
+  uint32_t HeaderSize =
+      std::distance(InstWithoutDebugIt.begin(), InstWithoutDebugIt.end());
+
+  if (HeaderSize != IdiomCanonicalSize &&
+      TTI->getIntrinsicCost(IntrinID, InitX->getType(), Args) >
           TargetTransformInfo::TCC_Basic)
     return false;
 
-  transformLoopToCountable(PH, CntInst, CntPhi, InitX, DefX,
+  transformLoopToCountable(IntrinID, PH, CntInst, CntPhi, InitX, DefX,
                            DefX->getDebugLoc(), ZeroCheck,
                            IsCntPhiUsedOutsideLoop);
   return true;
@@ -1508,27 +1533,28 @@ static CallInst *createPopcntIntrinsic(IRBuilder<> &IRBuilder, Value *Val,
   Type *Tys[] = {Val->getType()};
 
   Module *M = IRBuilder.GetInsertBlock()->getParent()->getParent();
-  Value *Func = Intrinsic::getDeclaration(M, Intrinsic::ctpop, Tys);
+  Function *Func = Intrinsic::getDeclaration(M, Intrinsic::ctpop, Tys);
   CallInst *CI = IRBuilder.CreateCall(Func, Ops);
   CI->setDebugLoc(DL);
 
   return CI;
 }
 
-static CallInst *createCTLZIntrinsic(IRBuilder<> &IRBuilder, Value *Val,
-                                     const DebugLoc &DL, bool ZeroCheck) {
+static CallInst *createFFSIntrinsic(IRBuilder<> &IRBuilder, Value *Val,
+                                    const DebugLoc &DL, bool ZeroCheck,
+                                    Intrinsic::ID IID) {
   Value *Ops[] = {Val, ZeroCheck ? IRBuilder.getTrue() : IRBuilder.getFalse()};
   Type *Tys[] = {Val->getType()};
 
   Module *M = IRBuilder.GetInsertBlock()->getParent()->getParent();
-  Value *Func = Intrinsic::getDeclaration(M, Intrinsic::ctlz, Tys);
+  Function *Func = Intrinsic::getDeclaration(M, IID, Tys);
   CallInst *CI = IRBuilder.CreateCall(Func, Ops);
   CI->setDebugLoc(DL);
 
   return CI;
 }
 
-/// Transform the following loop:
+/// Transform the following loop (Using CTLZ, CTTZ is similar):
 /// loop:
 ///   CntPhi = PHI [Cnt0, CntInst]
 ///   PhiX = PHI [InitX, DefX]
@@ -1560,19 +1586,19 @@ static CallInst *createCTLZIntrinsic(IRBuilder<> &IRBuilder, Value *Val,
 /// If LOOP_BODY is empty the loop will be deleted.
 /// If CntInst and DefX are not used in LOOP_BODY they will be removed.
 void LoopIdiomRecognize::transformLoopToCountable(
-    BasicBlock *Preheader, Instruction *CntInst, PHINode *CntPhi, Value *InitX,
-    Instruction *DefX, const DebugLoc &DL, bool ZeroCheck,
-    bool IsCntPhiUsedOutsideLoop) {
+    Intrinsic::ID IntrinID, BasicBlock *Preheader, Instruction *CntInst,
+    PHINode *CntPhi, Value *InitX, Instruction *DefX, const DebugLoc &DL,
+    bool ZeroCheck, bool IsCntPhiUsedOutsideLoop) {
   BranchInst *PreheaderBr = cast<BranchInst>(Preheader->getTerminator());
 
-  // Step 1: Insert the CTLZ instruction at the end of the preheader block
-  //   Count = BitWidth - CTLZ(InitX);
-  // If there are uses of CntPhi create:
-  //   CountPrev = BitWidth - CTLZ(InitX >> 1);
+  // Step 1: Insert the CTLZ/CTTZ instruction at the end of the preheader block
   IRBuilder<> Builder(PreheaderBr);
   Builder.SetCurrentDebugLocation(DL);
-  Value *CTLZ, *Count, *CountPrev, *NewCount, *InitXNext;
+  Value *FFS, *Count, *CountPrev, *NewCount, *InitXNext;
 
+  //   Count = BitWidth - CTLZ(InitX);
+  // If there are uses of CntPhi create:
+  //   CountPrev = BitWidth - CTLZ(InitX >> 1);
   if (IsCntPhiUsedOutsideLoop) {
     if (DefX->getOpcode() == Instruction::AShr)
       InitXNext =
@@ -1580,29 +1606,30 @@ void LoopIdiomRecognize::transformLoopToCountable(
     else if (DefX->getOpcode() == Instruction::LShr)
       InitXNext =
           Builder.CreateLShr(InitX, ConstantInt::get(InitX->getType(), 1));
+    else if (DefX->getOpcode() == Instruction::Shl) // cttz
+      InitXNext =
+          Builder.CreateShl(InitX, ConstantInt::get(InitX->getType(), 1));
     else
       llvm_unreachable("Unexpected opcode!");
   } else
     InitXNext = InitX;
-  CTLZ = createCTLZIntrinsic(Builder, InitXNext, DL, ZeroCheck);
+  FFS = createFFSIntrinsic(Builder, InitXNext, DL, ZeroCheck, IntrinID);
   Count = Builder.CreateSub(
-      ConstantInt::get(CTLZ->getType(),
-                       CTLZ->getType()->getIntegerBitWidth()),
-      CTLZ);
+      ConstantInt::get(FFS->getType(),
+                       FFS->getType()->getIntegerBitWidth()),
+      FFS);
   if (IsCntPhiUsedOutsideLoop) {
     CountPrev = Count;
     Count = Builder.CreateAdd(
         CountPrev,
         ConstantInt::get(CountPrev->getType(), 1));
   }
-  if (IsCntPhiUsedOutsideLoop)
-    NewCount = Builder.CreateZExtOrTrunc(CountPrev,
-        cast<IntegerType>(CntInst->getType()));
-  else
-    NewCount = Builder.CreateZExtOrTrunc(Count,
-        cast<IntegerType>(CntInst->getType()));
 
-  // If the CTLZ counter's initial value is not zero, insert Add Inst.
+  NewCount = Builder.CreateZExtOrTrunc(
+                      IsCntPhiUsedOutsideLoop ? CountPrev : Count,
+                      cast<IntegerType>(CntInst->getType()));
+
+  // If the counter's initial value is not zero, insert Add Inst.
   Value *CntInitVal = CntPhi->getIncomingValueForBlock(Preheader);
   ConstantInt *InitConst = dyn_cast<ConstantInt>(CntInitVal);
   if (!InitConst || !InitConst->isZero())
@@ -1638,8 +1665,7 @@ void LoopIdiomRecognize::transformLoopToCountable(
   LbCond->setOperand(1, ConstantInt::get(Ty, 0));
 
   // Step 3: All the references to the original counter outside
-  //  the loop are replaced with the NewCount -- the value returned from
-  //  __builtin_ctlz(x).
+  //  the loop are replaced with the NewCount
   if (IsCntPhiUsedOutsideLoop)
     CntPhi->replaceUsesOutsideBlock(NewCount, Body);
   else
diff --git a/lib/Transforms/Scalar/LoopInstSimplify.cpp b/lib/Transforms/Scalar/LoopInstSimplify.cpp
index 6f7dc2429c0990abc9c3cf9a739470f2a7a53829..6df32b0129c1a1273a637e8e712cfcd7b9f6fca0 100644
--- a/lib/Transforms/Scalar/LoopInstSimplify.cpp
+++ b/lib/Transforms/Scalar/LoopInstSimplify.cpp
@@ -1,9 +1,8 @@
 //===- LoopInstSimplify.cpp - Loop Instruction Simplification Pass --------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Transforms/Scalar/LoopInterchange.cpp b/lib/Transforms/Scalar/LoopInterchange.cpp
index 766e39b439a0d762fb9f7e39d5267159040f0cc9..aadbc3857e48b37e975b4de06347691752458339 100644
--- a/lib/Transforms/Scalar/LoopInterchange.cpp
+++ b/lib/Transforms/Scalar/LoopInterchange.cpp
@@ -1,9 +1,8 @@
 //===- LoopInterchange.cpp - Loop interchange pass-------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Transforms/Scalar/LoopLoadElimination.cpp b/lib/Transforms/Scalar/LoopLoadElimination.cpp
index 19bd9ebcc15b1b0849fc83f05f42c062ca42d6c7..60d27d3058c4611324006d1f527ee9f5ee90250e 100644
--- a/lib/Transforms/Scalar/LoopLoadElimination.cpp
+++ b/lib/Transforms/Scalar/LoopLoadElimination.cpp
@@ -1,9 +1,8 @@
 //===- LoopLoadElimination.cpp - Loop Load Elimination Pass ---------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -34,6 +33,7 @@
 #include "llvm/Analysis/LoopAccessAnalysis.h"
 #include "llvm/Analysis/LoopAnalysisManager.h"
 #include "llvm/Analysis/LoopInfo.h"
+#include "llvm/Analysis/MemorySSA.h"
 #include "llvm/Analysis/ScalarEvolution.h"
 #include "llvm/Analysis/ScalarEvolutionExpander.h"
 #include "llvm/Analysis/ScalarEvolutionExpressions.h"
@@ -428,9 +428,9 @@ public:
     auto *PH = L->getLoopPreheader();
     Value *InitialPtr = SEE.expandCodeFor(PtrSCEV->getStart(), Ptr->getType(),
                                           PH->getTerminator());
-    Value *Initial =
-        new LoadInst(InitialPtr, "load_initial", /* isVolatile */ false,
-                     Cand.Load->getAlignment(), PH->getTerminator());
+    Value *Initial = new LoadInst(
+        Cand.Load->getType(), InitialPtr, "load_initial",
+        /* isVolatile */ false, Cand.Load->getAlignment(), PH->getTerminator());
 
     PHINode *PHI = PHINode::Create(Initial->getType(), 2, "store_forwarded",
                                    &L->getHeader()->front());
@@ -668,12 +668,14 @@ PreservedAnalyses LoopLoadEliminationPass::run(Function &F,
   auto &TLI = AM.getResult<TargetLibraryAnalysis>(F);
   auto &AA = AM.getResult<AAManager>(F);
   auto &AC = AM.getResult<AssumptionAnalysis>(F);
+  MemorySSA *MSSA = EnableMSSALoopDependency
+                        ? &AM.getResult<MemorySSAAnalysis>(F).getMSSA()
+                        : nullptr;
 
   auto &LAM = AM.getResult<LoopAnalysisManagerFunctionProxy>(F).getManager();
   bool Changed = eliminateLoadsAcrossLoops(
       F, LI, DT, [&](Loop &L) -> const LoopAccessInfo & {
-        LoopStandardAnalysisResults AR = {AA, AC,  DT,  LI,
-                                          SE, TLI, TTI, nullptr};
+        LoopStandardAnalysisResults AR = {AA, AC, DT, LI, SE, TLI, TTI, MSSA};
         return LAM.getResult<LoopAccessAnalysis>(L, AR);
       });
 
diff --git a/lib/Transforms/Scalar/LoopPassManager.cpp b/lib/Transforms/Scalar/LoopPassManager.cpp
index 774ad7b945a0f282fff4a27f16142c0c1d8cce0d..f3bfbd3564ab1b55b7147d6b608aec1bbde8ff15 100644
--- a/lib/Transforms/Scalar/LoopPassManager.cpp
+++ b/lib/Transforms/Scalar/LoopPassManager.cpp
@@ -1,9 +1,8 @@
 //===- LoopPassManager.cpp - Loop pass management -------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Transforms/Scalar/LoopPredication.cpp b/lib/Transforms/Scalar/LoopPredication.cpp
index ccaf10142d51318b660e4a1fc45c0ce460ca1d96..eef048ec36b9e6ff2b606f7a6b0a84a7c9f95a66 100644
--- a/lib/Transforms/Scalar/LoopPredication.cpp
+++ b/lib/Transforms/Scalar/LoopPredication.cpp
@@ -1,9 +1,8 @@
 //===-- LoopPredication.cpp - Guard based loop predication pass -----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -180,6 +179,7 @@
 #include "llvm/Transforms/Scalar/LoopPredication.h"
 #include "llvm/ADT/Statistic.h"
 #include "llvm/Analysis/BranchProbabilityInfo.h"
+#include "llvm/Analysis/GuardUtils.h"
 #include "llvm/Analysis/LoopInfo.h"
 #include "llvm/Analysis/LoopPass.h"
 #include "llvm/Analysis/ScalarEvolution.h"
@@ -221,6 +221,12 @@ static cl::opt<float> LatchExitProbabilityScale(
     cl::desc("scale factor for the latch probability. Value should be greater "
              "than 1. Lower values are ignored"));
 
+static cl::opt<bool> PredicateWidenableBranchGuards(
+    "loop-predication-predicate-widenable-branches-to-deopt", cl::Hidden,
+    cl::desc("Whether or not we should predicate guards "
+             "expressed as widenable branches to deoptimize blocks"),
+    cl::init(true));
+
 namespace {
 class LoopPredication {
   /// Represents an induction variable check:
@@ -272,8 +278,10 @@ class LoopPredication {
                                                         LoopICmp RangeCheck,
                                                         SCEVExpander &Expander,
                                                         IRBuilder<> &Builder);
+  unsigned collectChecks(SmallVectorImpl<Value *> &Checks, Value *Condition,
+                         SCEVExpander &Expander, IRBuilder<> &Builder);
   bool widenGuardConditions(IntrinsicInst *II, SCEVExpander &Expander);
-
+  bool widenWidenableBranchGuardConditions(BranchInst *Guard, SCEVExpander &Expander);
   // If the loop always exits through another block in the loop, we should not
   // predicate based on the latch check. For example, the latch check can be a
   // very coarse grained check and there can be more fine grained exit checks
@@ -573,26 +581,18 @@ Optional<Value *> LoopPredication::widenICmpRangeCheck(ICmpInst *ICI,
   }
 }
 
-bool LoopPredication::widenGuardConditions(IntrinsicInst *Guard,
-                                           SCEVExpander &Expander) {
-  LLVM_DEBUG(dbgs() << "Processing guard:\n");
-  LLVM_DEBUG(Guard->dump());
-
-  TotalConsidered++;
-
-  IRBuilder<> Builder(cast<Instruction>(Preheader->getTerminator()));
-
+unsigned LoopPredication::collectChecks(SmallVectorImpl<Value *> &Checks,
+                                        Value *Condition,
+                                        SCEVExpander &Expander,
+                                        IRBuilder<> &Builder) {
+  unsigned NumWidened = 0;
   // The guard condition is expected to be in form of:
   //   cond1 && cond2 && cond3 ...
   // Iterate over subconditions looking for icmp conditions which can be
   // widened across loop iterations. Widening these conditions remember the
   // resulting list of subconditions in Checks vector.
-  SmallVector<Value *, 4> Worklist(1, Guard->getOperand(0));
+  SmallVector<Value *, 4> Worklist(1, Condition);
   SmallPtrSet<Value *, 4> Visited;
-
-  SmallVector<Value *, 4> Checks;
-
-  unsigned NumWidened = 0;
   do {
     Value *Condition = Worklist.pop_back_val();
     if (!Visited.insert(Condition).second)
@@ -616,8 +616,52 @@ bool LoopPredication::widenGuardConditions(IntrinsicInst *Guard,
 
     // Save the condition as is if we can't widen it
     Checks.push_back(Condition);
-  } while (Worklist.size() != 0);
+  } while (!Worklist.empty());
+  return NumWidened;
+}
+
+bool LoopPredication::widenGuardConditions(IntrinsicInst *Guard,
+                                           SCEVExpander &Expander) {
+  LLVM_DEBUG(dbgs() << "Processing guard:\n");
+  LLVM_DEBUG(Guard->dump());
+
+  TotalConsidered++;
+  SmallVector<Value *, 4> Checks;
+  IRBuilder<> Builder(cast<Instruction>(Preheader->getTerminator()));
+  unsigned NumWidened = collectChecks(Checks, Guard->getOperand(0), Expander,
+                                      Builder);
+  if (NumWidened == 0)
+    return false;
+
+  TotalWidened += NumWidened;
+
+  // Emit the new guard condition
+  Builder.SetInsertPoint(Guard);
+  Value *LastCheck = nullptr;
+  for (auto *Check : Checks)
+    if (!LastCheck)
+      LastCheck = Check;
+    else
+      LastCheck = Builder.CreateAnd(LastCheck, Check);
+  Guard->setOperand(0, LastCheck);
 
+  LLVM_DEBUG(dbgs() << "Widened checks = " << NumWidened << "\n");
+  return true;
+}
+
+bool LoopPredication::widenWidenableBranchGuardConditions(
+    BranchInst *Guard, SCEVExpander &Expander) {
+  assert(isGuardAsWidenableBranch(Guard) && "Must be!");
+  LLVM_DEBUG(dbgs() << "Processing guard:\n");
+  LLVM_DEBUG(Guard->dump());
+
+  TotalConsidered++;
+  SmallVector<Value *, 4> Checks;
+  IRBuilder<> Builder(cast<Instruction>(Preheader->getTerminator()));
+  Value *Condition = nullptr, *WidenableCondition = nullptr;
+  BasicBlock *GBB = nullptr, *DBB = nullptr;
+  parseWidenableBranch(Guard, Condition, WidenableCondition, GBB, DBB);
+  unsigned NumWidened = collectChecks(Checks, Condition, Expander, Builder);
   if (NumWidened == 0)
     return false;
 
@@ -631,7 +675,12 @@ bool LoopPredication::widenGuardConditions(IntrinsicInst *Guard,
       LastCheck = Check;
     else
       LastCheck = Builder.CreateAnd(LastCheck, Check);
+  // Make sure that the check contains widenable condition and therefore can be
+  // further widened.
+  LastCheck = Builder.CreateAnd(LastCheck, WidenableCondition);
   Guard->setOperand(0, LastCheck);
+  assert(isGuardAsWidenableBranch(Guard) &&
+         "Stopped being a guard after transform?");
 
   LLVM_DEBUG(dbgs() << "Widened checks = " << NumWidened << "\n");
   return true;
@@ -794,7 +843,12 @@ bool LoopPredication::runOnLoop(Loop *Loop) {
   // There is nothing to do if the module doesn't use guards
   auto *GuardDecl =
       M->getFunction(Intrinsic::getName(Intrinsic::experimental_guard));
-  if (!GuardDecl || GuardDecl->use_empty())
+  bool HasIntrinsicGuards = GuardDecl && !GuardDecl->use_empty();
+  auto *WCDecl = M->getFunction(
+      Intrinsic::getName(Intrinsic::experimental_widenable_condition));
+  bool HasWidenableConditions =
+      PredicateWidenableBranchGuards && WCDecl && !WCDecl->use_empty();
+  if (!HasIntrinsicGuards && !HasWidenableConditions)
     return false;
 
   DL = &M->getDataLayout();
@@ -818,13 +872,18 @@ bool LoopPredication::runOnLoop(Loop *Loop) {
   // Collect all the guards into a vector and process later, so as not
   // to invalidate the instruction iterator.
   SmallVector<IntrinsicInst *, 4> Guards;
-  for (const auto BB : L->blocks())
+  SmallVector<BranchInst *, 4> GuardsAsWidenableBranches;
+  for (const auto BB : L->blocks()) {
     for (auto &I : *BB)
-      if (auto *II = dyn_cast<IntrinsicInst>(&I))
-        if (II->getIntrinsicID() == Intrinsic::experimental_guard)
-          Guards.push_back(II);
+      if (isGuard(&I))
+        Guards.push_back(cast<IntrinsicInst>(&I));
+    if (PredicateWidenableBranchGuards &&
+        isGuardAsWidenableBranch(BB->getTerminator()))
+      GuardsAsWidenableBranches.push_back(
+          cast<BranchInst>(BB->getTerminator()));
+  }
 
-  if (Guards.empty())
+  if (Guards.empty() && GuardsAsWidenableBranches.empty())
     return false;
 
   SCEVExpander Expander(*SE, *DL, "loop-predication");
@@ -832,6 +891,8 @@ bool LoopPredication::runOnLoop(Loop *Loop) {
   bool Changed = false;
   for (auto *Guard : Guards)
     Changed |= widenGuardConditions(Guard, Expander);
+  for (auto *Guard : GuardsAsWidenableBranches)
+    Changed |= widenWidenableBranchGuardConditions(Guard, Expander);
 
   return Changed;
 }
diff --git a/lib/Transforms/Scalar/LoopRerollPass.cpp b/lib/Transforms/Scalar/LoopRerollPass.cpp
index 9a99e59255723308ea46a02accc785e6af824111..166b57f20b43fadb7a59c01e824dece53b2a37cb 100644
--- a/lib/Transforms/Scalar/LoopRerollPass.cpp
+++ b/lib/Transforms/Scalar/LoopRerollPass.cpp
@@ -1,9 +1,8 @@
 //===- LoopReroll.cpp - Loop rerolling pass -------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -892,12 +891,22 @@ bool LoopReroll::DAGRootTracker::validateRootSet(DAGRootSet &DRS) {
   const auto *ADR = dyn_cast<SCEVAddRecExpr>(SE->getSCEV(DRS.BaseInst));
   if (!ADR)
     return false;
+
+  // Check that the first root is evenly spaced.
   unsigned N = DRS.Roots.size() + 1;
   const SCEV *StepSCEV = SE->getMinusSCEV(SE->getSCEV(DRS.Roots[0]), ADR);
   const SCEV *ScaleSCEV = SE->getConstant(StepSCEV->getType(), N);
   if (ADR->getStepRecurrence(*SE) != SE->getMulExpr(StepSCEV, ScaleSCEV))
     return false;
 
+  // Check that the remainling roots are evenly spaced.
+  for (unsigned i = 1; i < N - 1; ++i) {
+    const SCEV *NewStepSCEV = SE->getMinusSCEV(SE->getSCEV(DRS.Roots[i]),
+                                               SE->getSCEV(DRS.Roots[i-1]));
+    if (NewStepSCEV != StepSCEV)
+      return false;
+  }
+
   return true;
 }
 
diff --git a/lib/Transforms/Scalar/LoopRotation.cpp b/lib/Transforms/Scalar/LoopRotation.cpp
index fd22128f7fe6b849c6083623081a934df2e5a3d5..f9d7840499815aa7b93c6c8430d45badb040ab73 100644
--- a/lib/Transforms/Scalar/LoopRotation.cpp
+++ b/lib/Transforms/Scalar/LoopRotation.cpp
@@ -1,9 +1,8 @@
 //===- LoopRotation.cpp - Loop Rotation Pass ------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Transforms/Scalar/LoopSimplifyCFG.cpp b/lib/Transforms/Scalar/LoopSimplifyCFG.cpp
index c370efa120772714e2d3f56bc4e3c60e3cfcc71d..746ca9ec31b242744b063bd041c0520703110282 100644
--- a/lib/Transforms/Scalar/LoopSimplifyCFG.cpp
+++ b/lib/Transforms/Scalar/LoopSimplifyCFG.cpp
@@ -1,9 +1,8 @@
 //===--------- LoopSimplifyCFG.cpp - Loop CFG Simplification Pass ---------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -21,6 +20,7 @@
 #include "llvm/Analysis/AssumptionCache.h"
 #include "llvm/Analysis/BasicAliasAnalysis.h"
 #include "llvm/Analysis/DependenceAnalysis.h"
+#include "llvm/Analysis/DomTreeUpdater.h"
 #include "llvm/Analysis/GlobalsModRef.h"
 #include "llvm/Analysis/LoopInfo.h"
 #include "llvm/Analysis/LoopPass.h"
@@ -29,7 +29,6 @@
 #include "llvm/Analysis/ScalarEvolution.h"
 #include "llvm/Analysis/ScalarEvolutionAliasAnalysis.h"
 #include "llvm/Analysis/TargetTransformInfo.h"
-#include "llvm/IR/DomTreeUpdater.h"
 #include "llvm/IR/Dominators.h"
 #include "llvm/Transforms/Scalar.h"
 #include "llvm/Transforms/Scalar/LoopPassManager.h"
@@ -46,6 +45,10 @@ static cl::opt<bool> EnableTermFolding("enable-loop-simplifycfg-term-folding",
 
 STATISTIC(NumTerminatorsFolded,
           "Number of terminators folded to unconditional branches");
+STATISTIC(NumLoopBlocksDeleted,
+          "Number of loop blocks deleted");
+STATISTIC(NumLoopExitsDeleted,
+          "Number of loop exiting edges deleted");
 
 /// If \p BB is a switch or a conditional branch, but only one of its successors
 /// can be reached from this block in runtime, return this successor. Otherwise,
@@ -76,6 +79,7 @@ static BasicBlock *getOnlyLiveSuccessor(BasicBlock *BB) {
   return nullptr;
 }
 
+namespace {
 /// Helper class that can turn branches and switches with constant conditions
 /// into unconditional branches.
 class ConstantTerminatorFoldingImpl {
@@ -83,7 +87,10 @@ private:
   Loop &L;
   LoopInfo &LI;
   DominatorTree &DT;
+  ScalarEvolution &SE;
   MemorySSAUpdater *MSSAU;
+  DomTreeUpdater DTU;
+  SmallVector<DominatorTree::UpdateType, 16> DTUpdates;
 
   // Whether or not the current loop has irreducible CFG.
   bool HasIrreducibleCFG = false;
@@ -102,7 +109,7 @@ private:
   SmallPtrSet<BasicBlock *, 8> LiveLoopBlocks;
   // The blocks of the original loop that will become unreachable from entry
   // after the constant folding.
-  SmallPtrSet<BasicBlock *, 8> DeadLoopBlocks;
+  SmallVector<BasicBlock *, 8> DeadLoopBlocks;
   // The exits of the original loop that will still be reachable from entry
   // after the constant folding.
   SmallPtrSet<BasicBlock *, 8> LiveExitBlocks;
@@ -137,7 +144,7 @@ private:
     PrintOutVector("Blocks in which we can constant-fold terminator:",
                    FoldCandidates);
     PrintOutSet("Live blocks from the original loop:", LiveLoopBlocks);
-    PrintOutSet("Dead blocks from the original loop:", DeadLoopBlocks);
+    PrintOutVector("Dead blocks from the original loop:", DeadLoopBlocks);
     PrintOutSet("Live exit blocks:", LiveExitBlocks);
     PrintOutVector("Dead exit blocks:", DeadExitBlocks);
     if (!DeleteCurrentLoop)
@@ -192,7 +199,7 @@ private:
 
       // If a loop block wasn't marked as live so far, then it's dead.
       if (!LiveLoopBlocks.count(BB)) {
-        DeadLoopBlocks.insert(BB);
+        DeadLoopBlocks.push_back(BB);
         continue;
       }
 
@@ -202,12 +209,13 @@ private:
       // folding. Only handle blocks from current loop: branches in child loops
       // are skipped because if they can be folded, they should be folded during
       // the processing of child loops.
-      if (TheOnlySucc && LI.getLoopFor(BB) == &L)
+      bool TakeFoldCandidate = TheOnlySucc && LI.getLoopFor(BB) == &L;
+      if (TakeFoldCandidate)
         FoldCandidates.push_back(BB);
 
       // Handle successors.
       for (BasicBlock *Succ : successors(BB))
-        if (!TheOnlySucc || TheOnlySucc == Succ) {
+        if (!TakeFoldCandidate || TheOnlySucc == Succ) {
           if (L.contains(Succ))
             LiveLoopBlocks.insert(Succ);
           else
@@ -223,8 +231,10 @@ private:
     // Now, all exit blocks that are not marked as live are dead.
     SmallVector<BasicBlock *, 8> ExitBlocks;
     L.getExitBlocks(ExitBlocks);
+    SmallPtrSet<BasicBlock *, 8> UniqueDeadExits;
     for (auto *ExitBlock : ExitBlocks)
-      if (!LiveExitBlocks.count(ExitBlock))
+      if (!LiveExitBlocks.count(ExitBlock) &&
+          UniqueDeadExits.insert(ExitBlock).second)
         DeadExitBlocks.push_back(ExitBlock);
 
     // Whether or not the edge From->To will still be present in graph after the
@@ -233,7 +243,7 @@ private:
       if (!LiveLoopBlocks.count(From))
         return false;
       BasicBlock *TheOnlySucc = getOnlyLiveSuccessor(From);
-      return !TheOnlySucc || TheOnlySucc == To;
+      return !TheOnlySucc || TheOnlySucc == To || LI.getLoopFor(From) != &L;
     };
 
     // The loop will not be destroyed if its latch is live.
@@ -269,11 +279,174 @@ private:
            "All blocks that stay in loop should be live!");
   }
 
+  /// We need to preserve static reachibility of all loop exit blocks (this is)
+  /// required by loop pass manager. In order to do it, we make the following
+  /// trick:
+  ///
+  ///  preheader:
+  ///    <preheader code>
+  ///    br label %loop_header
+  ///
+  ///  loop_header:
+  ///    ...
+  ///    br i1 false, label %dead_exit, label %loop_block
+  ///    ...
+  ///
+  /// We cannot simply remove edge from the loop to dead exit because in this
+  /// case dead_exit (and its successors) may become unreachable. To avoid that,
+  /// we insert the following fictive preheader:
+  ///
+  ///  preheader:
+  ///    <preheader code>
+  ///    switch i32 0, label %preheader-split,
+  ///                  [i32 1, label %dead_exit_1],
+  ///                  [i32 2, label %dead_exit_2],
+  ///                  ...
+  ///                  [i32 N, label %dead_exit_N],
+  ///
+  ///  preheader-split:
+  ///    br label %loop_header
+  ///
+  ///  loop_header:
+  ///    ...
+  ///    br i1 false, label %dead_exit_N, label %loop_block
+  ///    ...
+  ///
+  /// Doing so, we preserve static reachibility of all dead exits and can later
+  /// remove edges from the loop to these blocks.
+  void handleDeadExits() {
+    // If no dead exits, nothing to do.
+    if (DeadExitBlocks.empty())
+      return;
+
+    // Construct split preheader and the dummy switch to thread edges from it to
+    // dead exits.
+    BasicBlock *Preheader = L.getLoopPreheader();
+    BasicBlock *NewPreheader = Preheader->splitBasicBlock(
+        Preheader->getTerminator(),
+        Twine(Preheader->getName()).concat("-split"));
+    DTUpdates.push_back({DominatorTree::Delete, Preheader, L.getHeader()});
+    DTUpdates.push_back({DominatorTree::Insert, NewPreheader, L.getHeader()});
+    DTUpdates.push_back({DominatorTree::Insert, Preheader, NewPreheader});
+    IRBuilder<> Builder(Preheader->getTerminator());
+    SwitchInst *DummySwitch =
+        Builder.CreateSwitch(Builder.getInt32(0), NewPreheader);
+    Preheader->getTerminator()->eraseFromParent();
+
+    unsigned DummyIdx = 1;
+    for (BasicBlock *BB : DeadExitBlocks) {
+      SmallVector<Instruction *, 4> DeadPhis;
+      for (auto &PN : BB->phis())
+        DeadPhis.push_back(&PN);
+
+      // Eliminate all Phis from dead exits.
+      for (Instruction *PN : DeadPhis) {
+        PN->replaceAllUsesWith(UndefValue::get(PN->getType()));
+        PN->eraseFromParent();
+      }
+      assert(DummyIdx != 0 && "Too many dead exits!");
+      DummySwitch->addCase(Builder.getInt32(DummyIdx++), BB);
+      DTUpdates.push_back({DominatorTree::Insert, Preheader, BB});
+      ++NumLoopExitsDeleted;
+    }
+
+    assert(L.getLoopPreheader() == NewPreheader && "Malformed CFG?");
+    if (Loop *OuterLoop = LI.getLoopFor(Preheader)) {
+      OuterLoop->addBasicBlockToLoop(NewPreheader, LI);
+
+      // When we break dead edges, the outer loop may become unreachable from
+      // the current loop. We need to fix loop info accordingly. For this, we
+      // find the most nested loop that still contains L and remove L from all
+      // loops that are inside of it.
+      Loop *StillReachable = nullptr;
+      for (BasicBlock *BB : LiveExitBlocks) {
+        Loop *BBL = LI.getLoopFor(BB);
+        if (BBL && BBL->contains(L.getHeader()))
+          if (!StillReachable ||
+              BBL->getLoopDepth() > StillReachable->getLoopDepth())
+            StillReachable = BBL;
+      }
+
+      // Okay, our loop is no longer in the outer loop (and maybe not in some of
+      // its parents as well). Make the fixup.
+      if (StillReachable != OuterLoop) {
+        LI.changeLoopFor(NewPreheader, StillReachable);
+        for (Loop *NotContaining = OuterLoop; NotContaining != StillReachable;
+             NotContaining = NotContaining->getParentLoop()) {
+          NotContaining->removeBlockFromLoop(NewPreheader);
+          for (auto *BB : L.blocks())
+            NotContaining->removeBlockFromLoop(BB);
+        }
+        OuterLoop->removeChildLoop(&L);
+        if (StillReachable)
+          StillReachable->addChildLoop(&L);
+        else
+          LI.addTopLevelLoop(&L);
+
+        // Some values from loops in [OuterLoop, StillReachable) could be used
+        // in the current loop. Now it is not their child anymore, so such uses
+        // require LCSSA Phis.
+        Loop *FixLCSSALoop = OuterLoop;
+        while (FixLCSSALoop->getParentLoop() != StillReachable)
+          FixLCSSALoop = FixLCSSALoop->getParentLoop();
+        assert(FixLCSSALoop && "Should be a loop!");
+        // We need all DT updates to be done before forming LCSSA.
+        DTU.applyUpdates(DTUpdates);
+        DTUpdates.clear();
+        formLCSSARecursively(*FixLCSSALoop, DT, &LI, &SE);
+      }
+    }
+  }
+
+  /// Delete loop blocks that have become unreachable after folding. Make all
+  /// relevant updates to DT and LI.
+  void deleteDeadLoopBlocks() {
+    if (MSSAU) {
+      SmallPtrSet<BasicBlock *, 8> DeadLoopBlocksSet(DeadLoopBlocks.begin(),
+                                                     DeadLoopBlocks.end());
+      MSSAU->removeBlocks(DeadLoopBlocksSet);
+    }
+
+    // The function LI.erase has some invariants that need to be preserved when
+    // it tries to remove a loop which is not the top-level loop. In particular,
+    // it requires loop's preheader to be strictly in loop's parent. We cannot
+    // just remove blocks one by one, because after removal of preheader we may
+    // break this invariant for the dead loop. So we detatch and erase all dead
+    // loops beforehand.
+    for (auto *BB : DeadLoopBlocks)
+      if (LI.isLoopHeader(BB)) {
+        assert(LI.getLoopFor(BB) != &L && "Attempt to remove current loop!");
+        Loop *DL = LI.getLoopFor(BB);
+        if (DL->getParentLoop()) {
+          for (auto *PL = DL->getParentLoop(); PL; PL = PL->getParentLoop())
+            for (auto *BB : DL->getBlocks())
+              PL->removeBlockFromLoop(BB);
+          DL->getParentLoop()->removeChildLoop(DL);
+          LI.addTopLevelLoop(DL);
+        }
+        LI.erase(DL);
+      }
+
+    for (auto *BB : DeadLoopBlocks) {
+      assert(BB != L.getHeader() &&
+             "Header of the current loop cannot be dead!");
+      LLVM_DEBUG(dbgs() << "Deleting dead loop block " << BB->getName()
+                        << "\n");
+      LI.removeBlock(BB);
+    }
+
+    DetatchDeadBlocks(DeadLoopBlocks, &DTUpdates, /*KeepOneInputPHIs*/true);
+    DTU.applyUpdates(DTUpdates);
+    DTUpdates.clear();
+    for (auto *BB : DeadLoopBlocks)
+      DTU.deleteBB(BB);
+
+    NumLoopBlocksDeleted += DeadLoopBlocks.size();
+  }
+
   /// Constant-fold terminators of blocks acculumated in FoldCandidates into the
   /// unconditional branches.
   void foldTerminators() {
-    DomTreeUpdater DTU(DT, DomTreeUpdater::UpdateStrategy::Eager);
-
     for (BasicBlock *BB : FoldCandidates) {
       assert(LI.getLoopFor(BB) == &L && "Should be a loop block!");
       BasicBlock *TheOnlySucc = getOnlyLiveSuccessor(BB);
@@ -315,7 +488,7 @@ private:
       Term->eraseFromParent();
 
       for (auto *DeadSucc : DeadSuccessors)
-        DTU.deleteEdge(BB, DeadSucc);
+        DTUpdates.push_back({DominatorTree::Delete, BB, DeadSucc});
 
       ++NumTerminatorsFolded;
     }
@@ -323,8 +496,10 @@ private:
 
 public:
   ConstantTerminatorFoldingImpl(Loop &L, LoopInfo &LI, DominatorTree &DT,
+                                ScalarEvolution &SE,
                                 MemorySSAUpdater *MSSAU)
-      : L(L), LI(LI), DT(DT), MSSAU(MSSAU) {}
+      : L(L), LI(LI), DT(DT), SE(SE), MSSAU(MSSAU),
+        DTU(DT, DomTreeUpdater::UpdateStrategy::Eager) {}
   bool run() {
     assert(L.getLoopLatch() && "Should be single latch!");
 
@@ -358,26 +533,10 @@ public:
       return false;
     }
 
-    // TODO: Support deletion of dead loop blocks.
-    if (!DeadLoopBlocks.empty()) {
-      LLVM_DEBUG(dbgs() << "Give up constant terminator folding in loop "
-                        << L.getHeader()->getName()
-                        << ": we don't currently"
-                           " support deletion of dead in-loop blocks.\n");
-      return false;
-    }
-
-    // TODO: Support dead loop exits.
-    if (!DeadExitBlocks.empty()) {
-      LLVM_DEBUG(dbgs() << "Give up constant terminator folding in loop "
-                        << L.getHeader()->getName()
-                        << ": we don't currently support dead loop exits.\n");
-      return false;
-    }
-
     // TODO: Support blocks that are not dead, but also not in loop after the
     // folding.
-    if (BlocksInLoopAfterFolding.size() != L.getNumBlocks()) {
+    if (BlocksInLoopAfterFolding.size() + DeadLoopBlocks.size() !=
+        L.getNumBlocks()) {
       LLVM_DEBUG(
           dbgs() << "Give up constant terminator folding in loop "
                  << L.getHeader()->getName()
@@ -387,6 +546,7 @@ public:
       return false;
     }
 
+    SE.forgetTopmostLoop(&L);
     // Dump analysis results.
     LLVM_DEBUG(dump());
 
@@ -395,11 +555,23 @@ public:
                       << "\n");
 
     // Make the actual transforms.
+    handleDeadExits();
     foldTerminators();
 
+    if (!DeadLoopBlocks.empty()) {
+      LLVM_DEBUG(dbgs() << "Deleting " << DeadLoopBlocks.size()
+                    << " dead blocks in loop " << L.getHeader()->getName()
+                    << "\n");
+      deleteDeadLoopBlocks();
+    } else {
+      // If we didn't do updates inside deleteDeadLoopBlocks, do them here.
+      DTU.applyUpdates(DTUpdates);
+      DTUpdates.clear();
+    }
+
 #ifndef NDEBUG
     // Make sure that we have preserved all data structures after the transform.
-    DT.verify();
+    assert(DT.verify() && "DT broken after transform!");
     assert(DT.isReachableFromEntry(L.getHeader()));
     LI.verify(DT);
 #endif
@@ -407,10 +579,12 @@ public:
     return true;
   }
 };
+} // namespace
 
 /// Turn branches and switches with known constant conditions into unconditional
 /// branches.
 static bool constantFoldTerminators(Loop &L, DominatorTree &DT, LoopInfo &LI,
+                                    ScalarEvolution &SE,
                                     MemorySSAUpdater *MSSAU) {
   if (!EnableTermFolding)
     return false;
@@ -420,7 +594,7 @@ static bool constantFoldTerminators(Loop &L, DominatorTree &DT, LoopInfo &LI,
   if (!L.getLoopLatch())
     return false;
 
-  ConstantTerminatorFoldingImpl BranchFolder(L, LI, DT, MSSAU);
+  ConstantTerminatorFoldingImpl BranchFolder(L, LI, DT, SE, MSSAU);
   return BranchFolder.run();
 }
 
@@ -457,7 +631,7 @@ static bool simplifyLoopCFG(Loop &L, DominatorTree &DT, LoopInfo &LI,
   bool Changed = false;
 
   // Constant-fold terminators with known constant conditions.
-  Changed |= constantFoldTerminators(L, DT, LI, MSSAU);
+  Changed |= constantFoldTerminators(L, DT, LI, SE, MSSAU);
 
   // Eliminate unconditional branches by merging blocks into their predecessors.
   Changed |= mergeBlocksIntoPredecessors(L, DT, LI, MSSAU);
diff --git a/lib/Transforms/Scalar/LoopSink.cpp b/lib/Transforms/Scalar/LoopSink.cpp
index 540d19fc7939199b63c7c753fe1b9880583e4f84..b3cf208ce1e3abfc2c56bcc4da4fd14321373917 100644
--- a/lib/Transforms/Scalar/LoopSink.cpp
+++ b/lib/Transforms/Scalar/LoopSink.cpp
@@ -1,9 +1,8 @@
 //===-- LoopSink.cpp - Loop Sink Pass -------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -304,7 +303,7 @@ static bool sinkLoopInvariantInstructions(Loop &L, AAResults &AA, LoopInfo &LI,
     // No need to check for instruction's operands are loop invariant.
     assert(L.hasLoopInvariantOperands(I) &&
            "Insts in a loop's preheader should have loop invariant operands!");
-    if (!canSinkOrHoistInst(*I, &AA, &DT, &L, &CurAST, false))
+    if (!canSinkOrHoistInst(*I, &AA, &DT, &L, &CurAST, nullptr, false, false))
       continue;
     if (sinkInstruction(L, *I, ColdLoopBBs, LoopBlockNumber, LI, DT, BFI))
       Changed = true;
diff --git a/lib/Transforms/Scalar/LoopStrengthReduce.cpp b/lib/Transforms/Scalar/LoopStrengthReduce.cpp
index 773ffb9df0a2804a7d70898945c73f99dcf8ef90..a1533019128131ed65653520c7a8872020953abe 100644
--- a/lib/Transforms/Scalar/LoopStrengthReduce.cpp
+++ b/lib/Transforms/Scalar/LoopStrengthReduce.cpp
@@ -1,9 +1,8 @@
 //===- LoopStrengthReduce.cpp - Strength Reduce IVs in Loops --------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -155,6 +154,10 @@ static cl::opt<bool> FilterSameScaledReg(
     cl::desc("Narrow LSR search space by filtering non-optimal formulae"
              " with the same ScaledReg and Scale"));
 
+static cl::opt<bool> EnableBackedgeIndexing(
+  "lsr-backedge-indexing", cl::Hidden, cl::init(true),
+  cl::desc("Enable the generation of cross iteration indexed memops"));
+
 static cl::opt<unsigned> ComplexityLimit(
   "lsr-complexity-limit", cl::Hidden,
   cl::init(std::numeric_limits<uint16_t>::max()),
@@ -1053,12 +1056,12 @@ public:
   void dump() const;
 
 private:
-  void RateRegister(const SCEV *Reg,
+  void RateRegister(const Formula &F, const SCEV *Reg,
                     SmallPtrSetImpl<const SCEV *> &Regs,
                     const Loop *L,
                     ScalarEvolution &SE, DominatorTree &DT,
                     const TargetTransformInfo &TTI);
-  void RatePrimaryRegister(const SCEV *Reg,
+  void RatePrimaryRegister(const Formula &F, const SCEV *Reg,
                            SmallPtrSetImpl<const SCEV *> &Regs,
                            const Loop *L,
                            ScalarEvolution &SE, DominatorTree &DT,
@@ -1209,7 +1212,7 @@ static bool isAMCompletelyFolded(const TargetTransformInfo &TTI,
                                  Instruction *Fixup = nullptr);
 
 /// Tally up interesting quantities from the given register.
-void Cost::RateRegister(const SCEV *Reg,
+void Cost::RateRegister(const Formula &F, const SCEV *Reg,
                         SmallPtrSetImpl<const SCEV *> &Regs,
                         const Loop *L,
                         ScalarEvolution &SE, DominatorTree &DT,
@@ -1236,16 +1239,24 @@ void Cost::RateRegister(const SCEV *Reg,
     }
 
     unsigned LoopCost = 1;
-    if (TTI.shouldFavorPostInc()) {
-      const SCEV *LoopStep = AR->getStepRecurrence(SE);
-      if (isa<SCEVConstant>(LoopStep)) {
-        // Check if a post-indexed load/store can be used.
-        if (TTI.isIndexedLoadLegal(TTI.MIM_PostInc, AR->getType()) ||
-            TTI.isIndexedStoreLegal(TTI.MIM_PostInc, AR->getType())) {
+    if (TTI.isIndexedLoadLegal(TTI.MIM_PostInc, AR->getType()) ||
+        TTI.isIndexedStoreLegal(TTI.MIM_PostInc, AR->getType())) {
+
+      // If the step size matches the base offset, we could use pre-indexed
+      // addressing.
+      if (TTI.shouldFavorBackedgeIndex(L)) {
+        if (auto *Step = dyn_cast<SCEVConstant>(AR->getStepRecurrence(SE)))
+          if (Step->getAPInt() == F.BaseOffset)
+            LoopCost = 0;
+      }
+
+      if (TTI.shouldFavorPostInc()) {
+        const SCEV *LoopStep = AR->getStepRecurrence(SE);
+        if (isa<SCEVConstant>(LoopStep)) {
           const SCEV *LoopStart = AR->getStart();
           if (!isa<SCEVConstant>(LoopStart) &&
-            SE.isLoopInvariant(LoopStart, L))
-              LoopCost = 0;
+              SE.isLoopInvariant(LoopStart, L))
+            LoopCost = 0;
         }
       }
     }
@@ -1255,7 +1266,7 @@ void Cost::RateRegister(const SCEV *Reg,
     // TODO: The non-affine case isn't precisely modeled here.
     if (!AR->isAffine() || !isa<SCEVConstant>(AR->getOperand(1))) {
       if (!Regs.count(AR->getOperand(1))) {
-        RateRegister(AR->getOperand(1), Regs, L, SE, DT, TTI);
+        RateRegister(F, AR->getOperand(1), Regs, L, SE, DT, TTI);
         if (isLoser())
           return;
       }
@@ -1279,7 +1290,7 @@ void Cost::RateRegister(const SCEV *Reg,
 /// Record this register in the set. If we haven't seen it before, rate
 /// it. Optional LoserRegs provides a way to declare any formula that refers to
 /// one of those regs an instant loser.
-void Cost::RatePrimaryRegister(const SCEV *Reg,
+void Cost::RatePrimaryRegister(const Formula &F, const SCEV *Reg,
                                SmallPtrSetImpl<const SCEV *> &Regs,
                                const Loop *L,
                                ScalarEvolution &SE, DominatorTree &DT,
@@ -1290,7 +1301,7 @@ void Cost::RatePrimaryRegister(const SCEV *Reg,
     return;
   }
   if (Regs.insert(Reg).second) {
-    RateRegister(Reg, Regs, L, SE, DT, TTI);
+    RateRegister(F, Reg, Regs, L, SE, DT, TTI);
     if (LoserRegs && isLoser())
       LoserRegs->insert(Reg);
   }
@@ -1314,7 +1325,7 @@ void Cost::RateFormula(const TargetTransformInfo &TTI,
       Lose();
       return;
     }
-    RatePrimaryRegister(ScaledReg, Regs, L, SE, DT, LoserRegs, TTI);
+    RatePrimaryRegister(F, ScaledReg, Regs, L, SE, DT, LoserRegs, TTI);
     if (isLoser())
       return;
   }
@@ -1323,7 +1334,7 @@ void Cost::RateFormula(const TargetTransformInfo &TTI,
       Lose();
       return;
     }
-    RatePrimaryRegister(BaseReg, Regs, L, SE, DT, LoserRegs, TTI);
+    RatePrimaryRegister(F, BaseReg, Regs, L, SE, DT, LoserRegs, TTI);
     if (isLoser())
       return;
   }
@@ -1890,6 +1901,7 @@ class LSRInstance {
   LoopInfo &LI;
   const TargetTransformInfo &TTI;
   Loop *const L;
+  bool FavorBackedgeIndex = false;
   bool Changed = false;
 
   /// This is the insert position that the current loop's induction variable
@@ -2804,7 +2816,7 @@ bool IVChain::isProfitableIncrement(const SCEV *OperExpr,
 /// TODO: Consider IVInc free if it's already used in another chains.
 static bool
 isProfitableChain(IVChain &Chain, SmallPtrSetImpl<Instruction*> &Users,
-                  ScalarEvolution &SE, const TargetTransformInfo &TTI) {
+                  ScalarEvolution &SE) {
   if (StressIVChain)
     return true;
 
@@ -3064,7 +3076,7 @@ void LSRInstance::CollectChains() {
   for (unsigned UsersIdx = 0, NChains = IVChainVec.size();
        UsersIdx < NChains; ++UsersIdx) {
     if (!isProfitableChain(IVChainVec[UsersIdx],
-                           ChainUsersVec[UsersIdx].FarUsers, SE, TTI))
+                           ChainUsersVec[UsersIdx].FarUsers, SE))
       continue;
     // Preserve the chain at UsesIdx.
     if (ChainIdx != UsersIdx)
@@ -3078,7 +3090,7 @@ void LSRInstance::CollectChains() {
 void LSRInstance::FinalizeChain(IVChain &Chain) {
   assert(!Chain.Incs.empty() && "empty IV chains are not allowed");
   LLVM_DEBUG(dbgs() << "Final Chain: " << *Chain.Incs[0].UserInst << "\n");
-
+  
   for (const IVInc &Inc : Chain) {
     LLVM_DEBUG(dbgs() << "        Inc: " << *Inc.UserInst << "\n");
     auto UseI = find(Inc.UserInst->operands(), Inc.IVOperand);
@@ -3738,10 +3750,11 @@ void LSRInstance::GenerateSymbolicOffsets(LSRUse &LU, unsigned LUIdx,
 void LSRInstance::GenerateConstantOffsetsImpl(
     LSRUse &LU, unsigned LUIdx, const Formula &Base,
     const SmallVectorImpl<int64_t> &Worklist, size_t Idx, bool IsScaledReg) {
-  const SCEV *G = IsScaledReg ? Base.ScaledReg : Base.BaseRegs[Idx];
-  for (int64_t Offset : Worklist) {
+
+  auto GenerateOffset = [&](const SCEV *G, int64_t Offset) {
     Formula F = Base;
     F.BaseOffset = (uint64_t)Base.BaseOffset - Offset;
+
     if (isLegalUse(TTI, LU.MinOffset - Offset, LU.MaxOffset - Offset, LU.Kind,
                    LU.AccessTy, F)) {
       // Add the offset to the base register.
@@ -3761,7 +3774,35 @@ void LSRInstance::GenerateConstantOffsetsImpl(
 
       (void)InsertFormula(LU, LUIdx, F);
     }
+  };
+
+  const SCEV *G = IsScaledReg ? Base.ScaledReg : Base.BaseRegs[Idx];
+
+  // With constant offsets and constant steps, we can generate pre-inc
+  // accesses by having the offset equal the step. So, for access #0 with a
+  // step of 8, we generate a G - 8 base which would require the first access
+  // to be ((G - 8) + 8),+,8. The pre-indexed access then updates the pointer
+  // for itself and hopefully becomes the base for other accesses. This means
+  // means that a single pre-indexed access can be generated to become the new
+  // base pointer for each iteration of the loop, resulting in no extra add/sub
+  // instructions for pointer updating.
+  if (FavorBackedgeIndex && LU.Kind == LSRUse::Address) {
+    if (auto *GAR = dyn_cast<SCEVAddRecExpr>(G)) {
+      if (auto *StepRec =
+          dyn_cast<SCEVConstant>(GAR->getStepRecurrence(SE))) {
+        const APInt &StepInt = StepRec->getAPInt();
+        int64_t Step = StepInt.isNegative() ?
+          StepInt.getSExtValue() : StepInt.getZExtValue();
+
+        for (int64_t Offset : Worklist) {
+          Offset -= Step;
+          GenerateOffset(G, Offset);
+        }
+      }
+    }
   }
+  for (int64_t Offset : Worklist)
+    GenerateOffset(G, Offset);
 
   int64_t Imm = ExtractImmediate(G, SE);
   if (G->isZero() || Imm == 0)
@@ -3968,9 +4009,27 @@ void LSRInstance::GenerateTruncates(LSRUse &LU, unsigned LUIdx, Formula Base) {
     if (SrcTy != DstTy && TTI.isTruncateFree(SrcTy, DstTy)) {
       Formula F = Base;
 
-      if (F.ScaledReg) F.ScaledReg = SE.getAnyExtendExpr(F.ScaledReg, SrcTy);
-      for (const SCEV *&BaseReg : F.BaseRegs)
-        BaseReg = SE.getAnyExtendExpr(BaseReg, SrcTy);
+      // Sometimes SCEV is able to prove zero during ext transform. It may
+      // happen if SCEV did not do all possible transforms while creating the
+      // initial node (maybe due to depth limitations), but it can do them while
+      // taking ext.
+      if (F.ScaledReg) {
+        const SCEV *NewScaledReg = SE.getAnyExtendExpr(F.ScaledReg, SrcTy);
+        if (NewScaledReg->isZero())
+         continue;
+        F.ScaledReg = NewScaledReg;
+      }
+      bool HasZeroBaseReg = false;
+      for (const SCEV *&BaseReg : F.BaseRegs) {
+        const SCEV *NewBaseReg = SE.getAnyExtendExpr(BaseReg, SrcTy);
+        if (NewBaseReg->isZero()) {
+          HasZeroBaseReg = true;
+          break;
+        }
+        BaseReg = NewBaseReg;
+      }
+      if (HasZeroBaseReg)
+        continue;
 
       // TODO: This assumes we've done basic processing on all uses and
       // have an idea what the register usage is.
@@ -4400,7 +4459,7 @@ void LSRInstance::NarrowSearchSpaceByDetectingSupersets() {
 /// When there are many registers for expressions like A, A+1, A+2, etc.,
 /// allocate a single register for them.
 void LSRInstance::NarrowSearchSpaceByCollapsingUnrolledCode() {
-  if (EstimateSearchSpaceComplexity() < ComplexityLimit)
+  if (EstimateSearchSpaceComplexity() < ComplexityLimit) 
     return;
 
   LLVM_DEBUG(
@@ -5233,7 +5292,7 @@ void LSRInstance::RewriteForPHI(
             NewBB = SplitCriticalEdge(BB, Parent,
                                       CriticalEdgeSplittingOptions(&DT, &LI)
                                           .setMergeIdenticalEdges()
-                                          .setDontDeleteUselessPHIs());
+                                          .setKeepOneInputPHIs());
           } else {
             SmallVector<BasicBlock*, 2> NewBBs;
             SplitLandingPadPredecessors(Parent, BB, "", "", NewBBs, &DT, &LI);
@@ -5361,7 +5420,9 @@ void LSRInstance::ImplementSolution(
 LSRInstance::LSRInstance(Loop *L, IVUsers &IU, ScalarEvolution &SE,
                          DominatorTree &DT, LoopInfo &LI,
                          const TargetTransformInfo &TTI)
-    : IU(IU), SE(SE), DT(DT), LI(LI), TTI(TTI), L(L) {
+    : IU(IU), SE(SE), DT(DT), LI(LI), TTI(TTI), L(L),
+      FavorBackedgeIndex(EnableBackedgeIndexing &&
+                         TTI.shouldFavorBackedgeIndex(L)) {
   // If LoopSimplify form is not available, stay out of trouble.
   if (!L->isLoopSimplifyForm())
     return;
diff --git a/lib/Transforms/Scalar/LoopUnrollAndJamPass.cpp b/lib/Transforms/Scalar/LoopUnrollAndJamPass.cpp
index da46210b6fdd21facd3c53d0c59801e880101619..71e9953aed63a76e22556e85156ce0e2fb43a03f 100644
--- a/lib/Transforms/Scalar/LoopUnrollAndJamPass.cpp
+++ b/lib/Transforms/Scalar/LoopUnrollAndJamPass.cpp
@@ -1,9 +1,8 @@
 //===- LoopUnrollAndJam.cpp - Loop unroll and jam pass --------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Transforms/Scalar/LoopUnrollPass.cpp b/lib/Transforms/Scalar/LoopUnrollPass.cpp
index 38b80f48ed0e2ccf5bb86432db3905d9e6b915b4..55d45904138d75eff0e9788b553301d332f8c4aa 100644
--- a/lib/Transforms/Scalar/LoopUnrollPass.cpp
+++ b/lib/Transforms/Scalar/LoopUnrollPass.cpp
@@ -1,9 +1,8 @@
 //===- LoopUnroll.cpp - Loop unroller pass --------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Transforms/Scalar/LoopUnswitch.cpp b/lib/Transforms/Scalar/LoopUnswitch.cpp
index 4a089dfa7dbf3716ab56fa9d9b867de75576c320..3bff0538be5bf442550de35d3145aa3a9b73606c 100644
--- a/lib/Transforms/Scalar/LoopUnswitch.cpp
+++ b/lib/Transforms/Scalar/LoopUnswitch.cpp
@@ -1,9 +1,8 @@
 //===- LoopUnswitch.cpp - Hoist loop-invariant conditionals in loop -------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Transforms/Scalar/LoopVersioningLICM.cpp b/lib/Transforms/Scalar/LoopVersioningLICM.cpp
index c0c59d24dffc708b61d582e4ef951a24f91ce322..87898df5ec15df567aa3636bf860a19230c995e1 100644
--- a/lib/Transforms/Scalar/LoopVersioningLICM.cpp
+++ b/lib/Transforms/Scalar/LoopVersioningLICM.cpp
@@ -1,9 +1,8 @@
 //===- LoopVersioningLICM.cpp - LICM Loop Versioning ----------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -360,10 +359,11 @@ bool LoopVersioningLICM::legalLoopMemoryAccesses() {
 bool LoopVersioningLICM::instructionSafeForVersioning(Instruction *I) {
   assert(I != nullptr && "Null instruction found!");
   // Check function call safety
-  if (isa<CallInst>(I) && !AA->doesNotAccessMemory(CallSite(I))) {
-    LLVM_DEBUG(dbgs() << "    Unsafe call site found.\n");
-    return false;
-  }
+  if (auto *Call = dyn_cast<CallBase>(I))
+    if (!AA->doesNotAccessMemory(Call)) {
+      LLVM_DEBUG(dbgs() << "    Unsafe call site found.\n");
+      return false;
+    }
   // Avoid loops with possiblity of throw
   if (I->mayThrow()) {
     LLVM_DEBUG(dbgs() << "    May throw instruction found in loop body\n");
@@ -633,6 +633,8 @@ bool LoopVersioningLICM::runOnLoop(Loop *L, LPPassManager &LPM) {
     // Set Loop Versioning metaData for version loop.
     addStringMetadataToLoop(LVer.getVersionedLoop(), LICMVersioningMetaData);
     // Set "llvm.mem.parallel_loop_access" metaData to versioned loop.
+    // FIXME: "llvm.mem.parallel_loop_access" annotates memory access
+    // instructions, not loops.
     addStringMetadataToLoop(LVer.getVersionedLoop(),
                             "llvm.mem.parallel_loop_access");
     // Update version loop with aggressive aliasing assumption.
diff --git a/lib/Transforms/Scalar/LowerAtomic.cpp b/lib/Transforms/Scalar/LowerAtomic.cpp
index c165c5ece95c457789963cd34734440098c4d4ac..f39ca239644680c6fbd339143828435c1d7ab255 100644
--- a/lib/Transforms/Scalar/LowerAtomic.cpp
+++ b/lib/Transforms/Scalar/LowerAtomic.cpp
@@ -1,9 +1,8 @@
 //===- LowerAtomic.cpp - Lower atomic intrinsics --------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -27,7 +26,7 @@ static bool LowerAtomicCmpXchgInst(AtomicCmpXchgInst *CXI) {
   Value *Cmp = CXI->getCompareOperand();
   Value *Val = CXI->getNewValOperand();
 
-  LoadInst *Orig = Builder.CreateLoad(Ptr);
+  LoadInst *Orig = Builder.CreateLoad(Val->getType(), Ptr);
   Value *Equal = Builder.CreateICmpEQ(Orig, Cmp);
   Value *Res = Builder.CreateSelect(Equal, Val, Orig);
   Builder.CreateStore(Res, Ptr);
@@ -45,7 +44,7 @@ static bool LowerAtomicRMWInst(AtomicRMWInst *RMWI) {
   Value *Ptr = RMWI->getPointerOperand();
   Value *Val = RMWI->getValOperand();
 
-  LoadInst *Orig = Builder.CreateLoad(Ptr);
+  LoadInst *Orig = Builder.CreateLoad(Val->getType(), Ptr);
   Value *Res = nullptr;
 
   switch (RMWI->getOperation()) {
diff --git a/lib/Transforms/Scalar/LowerExpectIntrinsic.cpp b/lib/Transforms/Scalar/LowerExpectIntrinsic.cpp
index 68bfa00303956b84d86000494dead973d3674160..0d67c0d740ec995781352357c39810b660c07691 100644
--- a/lib/Transforms/Scalar/LowerExpectIntrinsic.cpp
+++ b/lib/Transforms/Scalar/LowerExpectIntrinsic.cpp
@@ -1,9 +1,8 @@
 //===- LowerExpectIntrinsic.cpp - Lower expect intrinsic ------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Transforms/Scalar/LowerGuardIntrinsic.cpp b/lib/Transforms/Scalar/LowerGuardIntrinsic.cpp
index 4867b33d671f5e0797d81791669da885e42d1f29..9489e01774d64147d84e63f9f28e27223c92eba9 100644
--- a/lib/Transforms/Scalar/LowerGuardIntrinsic.cpp
+++ b/lib/Transforms/Scalar/LowerGuardIntrinsic.cpp
@@ -1,9 +1,8 @@
 //===- LowerGuardIntrinsic.cpp - Lower the guard intrinsic ---------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Transforms/Scalar/LowerWidenableCondition.cpp b/lib/Transforms/Scalar/LowerWidenableCondition.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..5342f2ddcb6b11abd47604b2aa5d3f78467ff565
--- /dev/null
+++ b/lib/Transforms/Scalar/LowerWidenableCondition.cpp
@@ -0,0 +1,85 @@
+//===- LowerWidenableCondition.cpp - Lower the guard intrinsic ---------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This pass lowers the llvm.widenable.condition intrinsic to default value
+// which is i1 true.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Transforms/Scalar/LowerWidenableCondition.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/Analysis/GuardUtils.h"
+#include "llvm/IR/BasicBlock.h"
+#include "llvm/IR/Function.h"
+#include "llvm/IR/InstIterator.h"
+#include "llvm/IR/Instructions.h"
+#include "llvm/IR/Intrinsics.h"
+#include "llvm/IR/Module.h"
+#include "llvm/IR/PatternMatch.h"
+#include "llvm/Pass.h"
+#include "llvm/Transforms/Scalar.h"
+#include "llvm/Transforms/Utils/GuardUtils.h"
+
+using namespace llvm;
+
+namespace {
+struct LowerWidenableConditionLegacyPass : public FunctionPass {
+  static char ID;
+  LowerWidenableConditionLegacyPass() : FunctionPass(ID) {
+    initializeLowerWidenableConditionLegacyPassPass(
+        *PassRegistry::getPassRegistry());
+  }
+
+  bool runOnFunction(Function &F) override;
+};
+}
+
+static bool lowerWidenableCondition(Function &F) {
+  // Check if we can cheaply rule out the possibility of not having any work to
+  // do.
+  auto *WCDecl = F.getParent()->getFunction(
+      Intrinsic::getName(Intrinsic::experimental_widenable_condition));
+  if (!WCDecl || WCDecl->use_empty())
+    return false;
+
+  using namespace llvm::PatternMatch;
+  SmallVector<CallInst *, 8> ToLower;
+  for (auto &I : instructions(F))
+    if (match(&I, m_Intrinsic<Intrinsic::experimental_widenable_condition>()))
+      ToLower.push_back(cast<CallInst>(&I));
+
+  if (ToLower.empty())
+    return false;
+
+  for (auto *CI : ToLower) {
+    CI->replaceAllUsesWith(ConstantInt::getTrue(CI->getContext()));
+    CI->eraseFromParent();
+  }
+  return true;
+}
+
+bool LowerWidenableConditionLegacyPass::runOnFunction(Function &F) {
+  return lowerWidenableCondition(F);
+}
+
+char LowerWidenableConditionLegacyPass::ID = 0;
+INITIALIZE_PASS(LowerWidenableConditionLegacyPass, "lower-widenable-condition",
+                "Lower the widenable condition to default true value", false,
+                false)
+
+Pass *llvm::createLowerWidenableConditionPass() {
+  return new LowerWidenableConditionLegacyPass();
+}
+
+PreservedAnalyses LowerWidenableConditionPass::run(Function &F,
+                                               FunctionAnalysisManager &AM) {
+  if (lowerWidenableCondition(F))
+    return PreservedAnalyses::none();
+
+  return PreservedAnalyses::all();
+}
diff --git a/lib/Transforms/Scalar/MakeGuardsExplicit.cpp b/lib/Transforms/Scalar/MakeGuardsExplicit.cpp
index 1ba3994eba0ed57aac65aa2a1d98357f9cf257d1..789232e0f5ce705c1b3f88da68988f813c0863e6 100644
--- a/lib/Transforms/Scalar/MakeGuardsExplicit.cpp
+++ b/lib/Transforms/Scalar/MakeGuardsExplicit.cpp
@@ -1,9 +1,8 @@
 //===- MakeGuardsExplicit.cpp - Turn guard intrinsics into guard branches -===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Transforms/Scalar/MemCpyOptimizer.cpp b/lib/Transforms/Scalar/MemCpyOptimizer.cpp
index 8756a1afcddfb52f2ffec90e56f3df0ebeb37e40..eeeef58384dbe20b81b70d42e4d5c74ecde7d6d7 100644
--- a/lib/Transforms/Scalar/MemCpyOptimizer.cpp
+++ b/lib/Transforms/Scalar/MemCpyOptimizer.cpp
@@ -1,9 +1,8 @@
 //===- MemCpyOptimizer.cpp - Optimize use of memcpy and friends -----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -546,8 +545,8 @@ static bool moveUp(AliasAnalysis &AA, StoreInst *SI, Instruction *P,
   // Memory locations of lifted instructions.
   SmallVector<MemoryLocation, 8> MemLocs{StoreLoc};
 
-  // Lifted callsites.
-  SmallVector<ImmutableCallSite, 8> CallSites;
+  // Lifted calls.
+  SmallVector<const CallBase *, 8> Calls;
 
   const MemoryLocation LoadLoc = MemoryLocation::get(LI);
 
@@ -565,10 +564,9 @@ static bool moveUp(AliasAnalysis &AA, StoreInst *SI, Instruction *P,
       });
 
       if (!NeedLift)
-        NeedLift =
-            llvm::any_of(CallSites, [C, &AA](const ImmutableCallSite &CS) {
-              return isModOrRefSet(AA.getModRefInfo(C, CS));
-            });
+        NeedLift = llvm::any_of(Calls, [C, &AA](const CallBase *Call) {
+          return isModOrRefSet(AA.getModRefInfo(C, Call));
+        });
     }
 
     if (!NeedLift)
@@ -579,12 +577,12 @@ static bool moveUp(AliasAnalysis &AA, StoreInst *SI, Instruction *P,
       // none of them may modify its source.
       if (isModSet(AA.getModRefInfo(C, LoadLoc)))
         return false;
-      else if (auto CS = ImmutableCallSite(C)) {
+      else if (const auto *Call = dyn_cast<CallBase>(C)) {
         // If we can't lift this before P, it's game over.
-        if (isModOrRefSet(AA.getModRefInfo(P, CS)))
+        if (isModOrRefSet(AA.getModRefInfo(P, Call)))
           return false;
 
-        CallSites.push_back(CS);
+        Calls.push_back(Call);
       } else if (isa<LoadInst>(C) || isa<StoreInst>(C) || isa<VAArgInst>(C)) {
         // If we can't lift this before P, it's game over.
         auto ML = MemoryLocation::get(C);
@@ -675,13 +673,11 @@ bool MemCpyOptPass::processStore(StoreInst *SI, BasicBlock::iterator &BBI) {
           if (UseMemMove)
             M = Builder.CreateMemMove(
                 SI->getPointerOperand(), findStoreAlignment(DL, SI),
-                LI->getPointerOperand(), findLoadAlignment(DL, LI), Size,
-                SI->isVolatile());
+                LI->getPointerOperand(), findLoadAlignment(DL, LI), Size);
           else
             M = Builder.CreateMemCpy(
                 SI->getPointerOperand(), findStoreAlignment(DL, SI),
-                LI->getPointerOperand(), findLoadAlignment(DL, LI), Size,
-                SI->isVolatile());
+                LI->getPointerOperand(), findLoadAlignment(DL, LI), Size);
 
           LLVM_DEBUG(dbgs() << "Promoting " << *LI << " to " << *SI << " => "
                             << *M << "\n");
@@ -770,8 +766,8 @@ bool MemCpyOptPass::processStore(StoreInst *SI, BasicBlock::iterator &BBI) {
       if (!Align)
         Align = DL.getABITypeAlignment(T);
       IRBuilder<> Builder(SI);
-      auto *M = Builder.CreateMemSet(SI->getPointerOperand(), ByteVal,
-                                     Size, Align, SI->isVolatile());
+      auto *M =
+          Builder.CreateMemSet(SI->getPointerOperand(), ByteVal, Size, Align);
 
       LLVM_DEBUG(dbgs() << "Promoting " << *SI << " to " << *M << "\n");
 
@@ -919,8 +915,7 @@ bool MemCpyOptPass::performCallSlotOptzn(Instruction *cpy, Value *cpyDest,
       continue;
     }
     if (const IntrinsicInst *IT = dyn_cast<IntrinsicInst>(U))
-      if (IT->getIntrinsicID() == Intrinsic::lifetime_start ||
-          IT->getIntrinsicID() == Intrinsic::lifetime_end)
+      if (IT->isLifetimeStartOrEnd())
         continue;
 
     if (U != C && U != cpy)
@@ -945,10 +940,10 @@ bool MemCpyOptPass::performCallSlotOptzn(Instruction *cpy, Value *cpyDest,
   // the use analysis, we also need to know that it does not sneakily
   // access dest.  We rely on AA to figure this out for us.
   AliasAnalysis &AA = LookupAliasAnalysis();
-  ModRefInfo MR = AA.getModRefInfo(C, cpyDest, srcSize);
+  ModRefInfo MR = AA.getModRefInfo(C, cpyDest, LocationSize::precise(srcSize));
   // If necessary, perform additional analysis.
   if (isModOrRefSet(MR))
-    MR = AA.callCapturesBefore(C, cpyDest, srcSize, &DT);
+    MR = AA.callCapturesBefore(C, cpyDest, LocationSize::precise(srcSize), &DT);
   if (isModOrRefSet(MR))
     return false;
 
@@ -996,7 +991,8 @@ bool MemCpyOptPass::performCallSlotOptzn(Instruction *cpy, Value *cpyDest,
   // handled here, but combineMetadata doesn't support them yet
   unsigned KnownIDs[] = {LLVMContext::MD_tbaa, LLVMContext::MD_alias_scope,
                          LLVMContext::MD_noalias,
-                         LLVMContext::MD_invariant_group};
+                         LLVMContext::MD_invariant_group,
+                         LLVMContext::MD_access_group};
   combineMetadata(C, cpy, KnownIDs, true);
 
   // Remove the memcpy.
@@ -1059,6 +1055,8 @@ bool MemCpyOptPass::processMemCpyMemCpyDependence(MemCpyInst *M,
     UseMemMove = true;
 
   // If all checks passed, then we can transform M.
+  LLVM_DEBUG(dbgs() << "MemCpyOptPass: Forwarding memcpy->memcpy src:\n"
+                    << *MDep << '\n' << *M << '\n');
 
   // TODO: Is this worth it if we're creating a less aligned memcpy? For
   // example we could be moving from movaps -> movq on x86.
@@ -1136,8 +1134,10 @@ bool MemCpyOptPass::processMemSetMemCpyDependence(MemCpyInst *MemCpy,
   Value *SizeDiff = Builder.CreateSub(DestSize, SrcSize);
   Value *MemsetLen = Builder.CreateSelect(
       Ule, ConstantInt::getNullValue(DestSize->getType()), SizeDiff);
-  Builder.CreateMemSet(Builder.CreateGEP(Dest, SrcSize), MemSet->getOperand(1),
-                       MemsetLen, Align);
+  Builder.CreateMemSet(
+      Builder.CreateGEP(Dest->getType()->getPointerElementType(), Dest,
+                        SrcSize),
+      MemSet->getOperand(1), MemsetLen, Align);
 
   MD->removeInstruction(MemSet);
   MemSet->eraseFromParent();
@@ -1341,7 +1341,7 @@ bool MemCpyOptPass::processByValArgument(CallSite CS, unsigned ArgNo) {
   Type *ByValTy = cast<PointerType>(ByValArg->getType())->getElementType();
   uint64_t ByValSize = DL.getTypeAllocSize(ByValTy);
   MemDepResult DepInfo = MD->getPointerDependencyFrom(
-      MemoryLocation(ByValArg, ByValSize), true,
+      MemoryLocation(ByValArg, LocationSize::precise(ByValSize)), true,
       CS.getInstruction()->getIterator(), CS.getInstruction()->getParent());
   if (!DepInfo.isClobber())
     return false;
diff --git a/lib/Transforms/Scalar/MergeICmps.cpp b/lib/Transforms/Scalar/MergeICmps.cpp
index 69fd8b163a070d1fa483927b27214c28d1b903b5..f7b224e2855afe99721a7b11bd4636c96e351bca 100644
--- a/lib/Transforms/Scalar/MergeICmps.cpp
+++ b/lib/Transforms/Scalar/MergeICmps.cpp
@@ -1,9 +1,8 @@
 //===- MergeICmps.cpp - Optimize chains of integer comparisons ------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Transforms/Scalar/MergedLoadStoreMotion.cpp b/lib/Transforms/Scalar/MergedLoadStoreMotion.cpp
index ee21feca8d2c02deeeb89061015851818f878010..30645f4400e39930b1306db301597cfb94058ab6 100644
--- a/lib/Transforms/Scalar/MergedLoadStoreMotion.cpp
+++ b/lib/Transforms/Scalar/MergedLoadStoreMotion.cpp
@@ -1,9 +1,8 @@
 //===- MergedLoadStoreMotion.cpp - merge and hoist/sink load/stores -------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Transforms/Scalar/NaryReassociate.cpp b/lib/Transforms/Scalar/NaryReassociate.cpp
index 7106ea216ad6f282a8b7bdd56c5e418d2e1a05f2..94436b55752a4b11f8213883f629f547688ae2bb 100644
--- a/lib/Transforms/Scalar/NaryReassociate.cpp
+++ b/lib/Transforms/Scalar/NaryReassociate.cpp
@@ -1,9 +1,8 @@
 //===- NaryReassociate.cpp - Reassociate n-ary expressions ----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -427,8 +426,8 @@ NaryReassociatePass::tryReassociateGEPAtIndex(GetElementPtrInst *GEP,
     RHS = Builder.CreateMul(
         RHS, ConstantInt::get(IntPtrTy, IndexedSize / ElementSize));
   }
-  GetElementPtrInst *NewGEP =
-      cast<GetElementPtrInst>(Builder.CreateGEP(Candidate, RHS));
+  GetElementPtrInst *NewGEP = cast<GetElementPtrInst>(
+      Builder.CreateGEP(GEP->getResultElementType(), Candidate, RHS));
   NewGEP->setIsInBounds(GEP->isInBounds());
   NewGEP->takeName(GEP);
   return NewGEP;
diff --git a/lib/Transforms/Scalar/NewGVN.cpp b/lib/Transforms/Scalar/NewGVN.cpp
index 7cbb0fe70f820762001e702f0b8ab81418e3af37..a77df095cc4a4da1909ff5fc7093186f6c88579a 100644
--- a/lib/Transforms/Scalar/NewGVN.cpp
+++ b/lib/Transforms/Scalar/NewGVN.cpp
@@ -1,9 +1,8 @@
 //===- NewGVN.cpp - Global Value Numbering Pass ---------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Transforms/Scalar/PartiallyInlineLibCalls.cpp b/lib/Transforms/Scalar/PartiallyInlineLibCalls.cpp
index 05ea9144f66caa39507139320b2a2bfde7a1747c..039123218544f72be06cb38651936c2e6734f014 100644
--- a/lib/Transforms/Scalar/PartiallyInlineLibCalls.cpp
+++ b/lib/Transforms/Scalar/PartiallyInlineLibCalls.cpp
@@ -1,9 +1,8 @@
 //===--- PartiallyInlineLibCalls.cpp - Partially inline libcalls ----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Transforms/Scalar/PlaceSafepoints.cpp b/lib/Transforms/Scalar/PlaceSafepoints.cpp
index fd2eb85fd7bf5326ff9c83006e6d9910e8e10ce5..b544f0a39ea87dde1d45fdcd0215fb59f8e14ef8 100644
--- a/lib/Transforms/Scalar/PlaceSafepoints.cpp
+++ b/lib/Transforms/Scalar/PlaceSafepoints.cpp
@@ -1,9 +1,8 @@
 //===- PlaceSafepoints.cpp - Place GC Safepoints --------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -56,7 +55,6 @@
 #include "llvm/Analysis/ScalarEvolution.h"
 #include "llvm/Analysis/TargetLibraryInfo.h"
 #include "llvm/Transforms/Utils/Local.h"
-#include "llvm/IR/CallSite.h"
 #include "llvm/IR/Dominators.h"
 #include "llvm/IR/IntrinsicInst.h"
 #include "llvm/IR/LegacyPassManager.h"
@@ -179,19 +177,18 @@ struct PlaceSafepoints : public FunctionPass {
 // callers job.
 static void
 InsertSafepointPoll(Instruction *InsertBefore,
-                    std::vector<CallSite> &ParsePointsNeeded /*rval*/,
+                    std::vector<CallBase *> &ParsePointsNeeded /*rval*/,
                     const TargetLibraryInfo &TLI);
 
-static bool needsStatepoint(const CallSite &CS, const TargetLibraryInfo &TLI) {
-  if (callsGCLeafFunction(CS, TLI))
+static bool needsStatepoint(CallBase *Call, const TargetLibraryInfo &TLI) {
+  if (callsGCLeafFunction(Call, TLI))
     return false;
-  if (CS.isCall()) {
-    CallInst *call = cast<CallInst>(CS.getInstruction());
-    if (call->isInlineAsm())
+  if (auto *CI = dyn_cast<CallInst>(Call)) {
+    if (CI->isInlineAsm())
       return false;
   }
 
-  return !(isStatepoint(CS) || isGCRelocate(CS) || isGCResult(CS));
+  return !(isStatepoint(Call) || isGCRelocate(Call) || isGCResult(Call));
 }
 
 /// Returns true if this loop is known to contain a call safepoint which
@@ -217,14 +214,14 @@ static bool containsUnconditionalCallSafepoint(Loop *L, BasicBlock *Header,
   BasicBlock *Current = Pred;
   while (true) {
     for (Instruction &I : *Current) {
-      if (auto CS = CallSite(&I))
+      if (auto *Call = dyn_cast<CallBase>(&I))
         // Note: Technically, needing a safepoint isn't quite the right
         // condition here.  We should instead be checking if the target method
         // has an
         // unconditional poll. In practice, this is only a theoretical concern
         // since we don't have any methods with conditional-only safepoint
         // polls.
-        if (needsStatepoint(CS, TLI))
+        if (needsStatepoint(Call, TLI))
           return true;
     }
 
@@ -360,9 +357,8 @@ bool PlaceBackedgeSafepointsImpl::runOnLoop(Loop *L) {
 
 /// Returns true if an entry safepoint is not required before this callsite in
 /// the caller function.
-static bool doesNotRequireEntrySafepointBefore(const CallSite &CS) {
-  Instruction *Inst = CS.getInstruction();
-  if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(Inst)) {
+static bool doesNotRequireEntrySafepointBefore(CallBase *Call) {
+  if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(Call)) {
     switch (II->getIntrinsicID()) {
     case Intrinsic::experimental_gc_statepoint:
     case Intrinsic::experimental_patchpoint_void:
@@ -424,8 +420,8 @@ static Instruction *findLocationForEntrySafepoint(Function &F,
     // which can grow the stack by an unbounded amount.  This isn't required
     // for GC semantics per se, but is a common requirement for languages
     // which detect stack overflow via guard pages and then throw exceptions.
-    if (auto CS = CallSite(Cursor)) {
-      if (doesNotRequireEntrySafepointBefore(CS))
+    if (auto *Call = dyn_cast<CallBase>(Cursor)) {
+      if (doesNotRequireEntrySafepointBefore(Call))
         continue;
       break;
     }
@@ -500,7 +496,7 @@ bool PlaceSafepoints::runOnFunction(Function &F) {
   DT.recalculate(F);
 
   SmallVector<Instruction *, 16> PollsNeeded;
-  std::vector<CallSite> ParsePointNeeded;
+  std::vector<CallBase *> ParsePointNeeded;
 
   if (enableBackedgeSafepoints(F)) {
     // Construct a pass manager to run the LoopPass backedge logic.  We
@@ -589,7 +585,7 @@ bool PlaceSafepoints::runOnFunction(Function &F) {
   // Now that we've identified all the needed safepoint poll locations, insert
   // safepoint polls themselves.
   for (Instruction *PollLocation : PollsNeeded) {
-    std::vector<CallSite> RuntimeCalls;
+    std::vector<CallBase *> RuntimeCalls;
     InsertSafepointPoll(PollLocation, RuntimeCalls, TLI);
     ParsePointNeeded.insert(ParsePointNeeded.end(), RuntimeCalls.begin(),
                             RuntimeCalls.end());
@@ -622,7 +618,7 @@ INITIALIZE_PASS_END(PlaceSafepoints, "place-safepoints", "Place Safepoints",
 
 static void
 InsertSafepointPoll(Instruction *InsertBefore,
-                    std::vector<CallSite> &ParsePointsNeeded /*rval*/,
+                    std::vector<CallBase *> &ParsePointsNeeded /*rval*/,
                     const TargetLibraryInfo &TLI) {
   BasicBlock *OrigBB = InsertBefore->getParent();
   Module *M = InsertBefore->getModule();
@@ -687,7 +683,7 @@ InsertSafepointPoll(Instruction *InsertBefore,
 
     // These are likely runtime calls.  Should we assert that via calling
     // convention or something?
-    ParsePointsNeeded.push_back(CallSite(CI));
+    ParsePointsNeeded.push_back(CI);
   }
   assert(ParsePointsNeeded.size() <= Calls.size());
 }
diff --git a/lib/Transforms/Scalar/Reassociate.cpp b/lib/Transforms/Scalar/Reassociate.cpp
index cb893eab1654f6968cd3812758b26628f0495789..6da95712d6444a3b49997763a81ea221b9356a46 100644
--- a/lib/Transforms/Scalar/Reassociate.cpp
+++ b/lib/Transforms/Scalar/Reassociate.cpp
@@ -1,9 +1,8 @@
 //===- Reassociate.cpp - Reassociate binary expressions -------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Transforms/Scalar/Reg2Mem.cpp b/lib/Transforms/Scalar/Reg2Mem.cpp
index 018feb035a4fe9720bd7322e79fe55b82ce18e12..3296322e00d5fb6ec43d254e9da7c797897d1d8f 100644
--- a/lib/Transforms/Scalar/Reg2Mem.cpp
+++ b/lib/Transforms/Scalar/Reg2Mem.cpp
@@ -1,9 +1,8 @@
 //===- Reg2Mem.cpp - Convert registers to allocas -------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp b/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp
index cf2ce03049afc33edaceb91466c55c93d2a43486..c358258d24cf9eca349b16d4612cceadab60e320 100644
--- a/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp
+++ b/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp
@@ -1,9 +1,8 @@
 //===- RewriteStatepointsForGC.cpp - Make GC relocations explicit ---------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -26,18 +25,17 @@
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/iterator_range.h"
+#include "llvm/Analysis/DomTreeUpdater.h"
 #include "llvm/Analysis/TargetLibraryInfo.h"
 #include "llvm/Analysis/TargetTransformInfo.h"
 #include "llvm/IR/Argument.h"
 #include "llvm/IR/Attributes.h"
 #include "llvm/IR/BasicBlock.h"
-#include "llvm/IR/CallSite.h"
 #include "llvm/IR/CallingConv.h"
 #include "llvm/IR/Constant.h"
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/DataLayout.h"
 #include "llvm/IR/DerivedTypes.h"
-#include "llvm/IR/DomTreeUpdater.h"
 #include "llvm/IR/Dominators.h"
 #include "llvm/IR/Function.h"
 #include "llvm/IR/IRBuilder.h"
@@ -286,9 +284,9 @@ struct PartiallyConstructedSafepointRecord {
 
 } // end anonymous namespace
 
-static ArrayRef<Use> GetDeoptBundleOperands(ImmutableCallSite CS) {
+static ArrayRef<Use> GetDeoptBundleOperands(const CallBase *Call) {
   Optional<OperandBundleUse> DeoptBundle =
-      CS.getOperandBundle(LLVMContext::OB_deopt);
+      Call->getOperandBundle(LLVMContext::OB_deopt);
 
   if (!DeoptBundle.hasValue()) {
     assert(AllowStatepointWithNoDeoptInfo &&
@@ -347,7 +345,7 @@ static bool containsGCPtrType(Type *Ty) {
   if (ArrayType *AT = dyn_cast<ArrayType>(Ty))
     return containsGCPtrType(AT->getElementType());
   if (StructType *ST = dyn_cast<StructType>(Ty))
-    return llvm::any_of(ST->subtypes(), containsGCPtrType);
+    return llvm::any_of(ST->elements(), containsGCPtrType);
   return false;
 }
 
@@ -370,14 +368,11 @@ static std::string suffixed_name_or(Value *V, StringRef Suffix,
 // given instruction. The  analysis is performed immediately before the
 // given instruction. Values defined by that instruction are not considered
 // live.  Values used by that instruction are considered live.
-static void
-analyzeParsePointLiveness(DominatorTree &DT,
-                          GCPtrLivenessData &OriginalLivenessData, CallSite CS,
-                          PartiallyConstructedSafepointRecord &Result) {
-  Instruction *Inst = CS.getInstruction();
-
+static void analyzeParsePointLiveness(
+    DominatorTree &DT, GCPtrLivenessData &OriginalLivenessData, CallBase *Call,
+    PartiallyConstructedSafepointRecord &Result) {
   StatepointLiveSetTy LiveSet;
-  findLiveSetAtInst(Inst, OriginalLivenessData, LiveSet);
+  findLiveSetAtInst(Call, OriginalLivenessData, LiveSet);
 
   if (PrintLiveSet) {
     dbgs() << "Live Variables:\n";
@@ -385,7 +380,7 @@ analyzeParsePointLiveness(DominatorTree &DT,
       dbgs() << " " << V->getName() << " " << *V << "\n";
   }
   if (PrintLiveSetSize) {
-    dbgs() << "Safepoint For: " << CS.getCalledValue()->getName() << "\n";
+    dbgs() << "Safepoint For: " << Call->getCalledValue()->getName() << "\n";
     dbgs() << "Number live values: " << LiveSet.size() << "\n";
   }
   Result.LiveSet = LiveSet;
@@ -1178,7 +1173,7 @@ findBasePointers(const StatepointLiveSetTy &live,
 /// Find the required based pointers (and adjust the live set) for the given
 /// parse point.
 static void findBasePointers(DominatorTree &DT, DefiningValueMapTy &DVCache,
-                             CallSite CS,
+                             CallBase *Call,
                              PartiallyConstructedSafepointRecord &result) {
   MapVector<Value *, Value *> PointerToBase;
   findBasePointers(result.LiveSet, PointerToBase, &DT, DVCache);
@@ -1200,11 +1195,11 @@ static void findBasePointers(DominatorTree &DT, DefiningValueMapTy &DVCache,
 /// Given an updated version of the dataflow liveness results, update the
 /// liveset and base pointer maps for the call site CS.
 static void recomputeLiveInValues(GCPtrLivenessData &RevisedLivenessData,
-                                  CallSite CS,
+                                  CallBase *Call,
                                   PartiallyConstructedSafepointRecord &result);
 
 static void recomputeLiveInValues(
-    Function &F, DominatorTree &DT, ArrayRef<CallSite> toUpdate,
+    Function &F, DominatorTree &DT, ArrayRef<CallBase *> toUpdate,
     MutableArrayRef<struct PartiallyConstructedSafepointRecord> records) {
   // TODO-PERF: reuse the original liveness, then simply run the dataflow
   // again.  The old values are still live and will help it stabilize quickly.
@@ -1307,7 +1302,7 @@ static void CreateGCRelocates(ArrayRef<Value *> LiveVariables,
   // Lazily populated map from input types to the canonicalized form mentioned
   // in the comment above.  This should probably be cached somewhere more
   // broadly.
-  DenseMap<Type*, Value*> TypeToDeclMap;
+  DenseMap<Type *, Function *> TypeToDeclMap;
 
   for (unsigned i = 0; i < LiveVariables.size(); i++) {
     // Generate the gc.relocate call and save the result
@@ -1318,7 +1313,7 @@ static void CreateGCRelocates(ArrayRef<Value *> LiveVariables,
     Type *Ty = LiveVariables[i]->getType();
     if (!TypeToDeclMap.count(Ty))
       TypeToDeclMap[Ty] = getGCRelocateDecl(Ty);
-    Value *GCRelocateDecl = TypeToDeclMap[Ty];
+    Function *GCRelocateDecl = TypeToDeclMap[Ty];
 
     // only specify a debug name if we can give a useful one
     CallInst *Reloc = Builder.CreateCall(
@@ -1399,16 +1394,16 @@ public:
 
 } // end anonymous namespace
 
-static StringRef getDeoptLowering(CallSite CS) {
+static StringRef getDeoptLowering(CallBase *Call) {
   const char *DeoptLowering = "deopt-lowering";
-  if (CS.hasFnAttr(DeoptLowering)) {
-    // FIXME: CallSite has a *really* confusing interface around attributes
+  if (Call->hasFnAttr(DeoptLowering)) {
+    // FIXME: Calls have a *really* confusing interface around attributes
     // with values.
-    const AttributeList &CSAS = CS.getAttributes();
+    const AttributeList &CSAS = Call->getAttributes();
     if (CSAS.hasAttribute(AttributeList::FunctionIndex, DeoptLowering))
       return CSAS.getAttribute(AttributeList::FunctionIndex, DeoptLowering)
           .getValueAsString();
-    Function *F = CS.getCalledFunction();
+    Function *F = Call->getCalledFunction();
     assert(F && F->hasFnAttribute(DeoptLowering));
     return F->getFnAttribute(DeoptLowering).getValueAsString();
   }
@@ -1416,7 +1411,7 @@ static StringRef getDeoptLowering(CallSite CS) {
 }
 
 static void
-makeStatepointExplicitImpl(const CallSite CS, /* to replace */
+makeStatepointExplicitImpl(CallBase *Call, /* to replace */
                            const SmallVectorImpl<Value *> &BasePtrs,
                            const SmallVectorImpl<Value *> &LiveVariables,
                            PartiallyConstructedSafepointRecord &Result,
@@ -1427,19 +1422,18 @@ makeStatepointExplicitImpl(const CallSite CS, /* to replace */
   // immediately before the previous instruction under the assumption that all
   // arguments will be available here.  We can't insert afterwards since we may
   // be replacing a terminator.
-  Instruction *InsertBefore = CS.getInstruction();
-  IRBuilder<> Builder(InsertBefore);
+  IRBuilder<> Builder(Call);
 
   ArrayRef<Value *> GCArgs(LiveVariables);
   uint64_t StatepointID = StatepointDirectives::DefaultStatepointID;
   uint32_t NumPatchBytes = 0;
   uint32_t Flags = uint32_t(StatepointFlags::None);
 
-  ArrayRef<Use> CallArgs(CS.arg_begin(), CS.arg_end());
-  ArrayRef<Use> DeoptArgs = GetDeoptBundleOperands(CS);
+  ArrayRef<Use> CallArgs(Call->arg_begin(), Call->arg_end());
+  ArrayRef<Use> DeoptArgs = GetDeoptBundleOperands(Call);
   ArrayRef<Use> TransitionArgs;
   if (auto TransitionBundle =
-      CS.getOperandBundle(LLVMContext::OB_gc_transition)) {
+          Call->getOperandBundle(LLVMContext::OB_gc_transition)) {
     Flags |= uint32_t(StatepointFlags::GCTransition);
     TransitionArgs = TransitionBundle->Inputs;
   }
@@ -1450,21 +1444,21 @@ makeStatepointExplicitImpl(const CallSite CS, /* to replace */
   bool IsDeoptimize = false;
 
   StatepointDirectives SD =
-      parseStatepointDirectivesFromAttrs(CS.getAttributes());
+      parseStatepointDirectivesFromAttrs(Call->getAttributes());
   if (SD.NumPatchBytes)
     NumPatchBytes = *SD.NumPatchBytes;
   if (SD.StatepointID)
     StatepointID = *SD.StatepointID;
 
   // Pass through the requested lowering if any.  The default is live-through.
-  StringRef DeoptLowering = getDeoptLowering(CS);
+  StringRef DeoptLowering = getDeoptLowering(Call);
   if (DeoptLowering.equals("live-in"))
     Flags |= uint32_t(StatepointFlags::DeoptLiveIn);
   else {
     assert(DeoptLowering.equals("live-through") && "Unsupported value!");
   }
 
-  Value *CallTarget = CS.getCalledValue();
+  Value *CallTarget = Call->getCalledValue();
   if (Function *F = dyn_cast<Function>(CallTarget)) {
     if (F->getIntrinsicID() == Intrinsic::experimental_deoptimize) {
       // Calls to llvm.experimental.deoptimize are lowered to calls to the
@@ -1481,8 +1475,9 @@ makeStatepointExplicitImpl(const CallSite CS, /* to replace */
       // calls to @llvm.experimental.deoptimize with different argument types in
       // the same module.  This is fine -- we assume the frontend knew what it
       // was doing when generating this kind of IR.
-      CallTarget =
-          F->getParent()->getOrInsertFunction("__llvm_deoptimize", FTy);
+      CallTarget = F->getParent()
+                       ->getOrInsertFunction("__llvm_deoptimize", FTy)
+                       .getCallee();
 
       IsDeoptimize = true;
     }
@@ -1490,57 +1485,56 @@ makeStatepointExplicitImpl(const CallSite CS, /* to replace */
 
   // Create the statepoint given all the arguments
   Instruction *Token = nullptr;
-  if (CS.isCall()) {
-    CallInst *ToReplace = cast<CallInst>(CS.getInstruction());
-    CallInst *Call = Builder.CreateGCStatepointCall(
+  if (auto *CI = dyn_cast<CallInst>(Call)) {
+    CallInst *SPCall = Builder.CreateGCStatepointCall(
         StatepointID, NumPatchBytes, CallTarget, Flags, CallArgs,
         TransitionArgs, DeoptArgs, GCArgs, "safepoint_token");
 
-    Call->setTailCallKind(ToReplace->getTailCallKind());
-    Call->setCallingConv(ToReplace->getCallingConv());
+    SPCall->setTailCallKind(CI->getTailCallKind());
+    SPCall->setCallingConv(CI->getCallingConv());
 
     // Currently we will fail on parameter attributes and on certain
     // function attributes.  In case if we can handle this set of attributes -
     // set up function attrs directly on statepoint and return attrs later for
     // gc_result intrinsic.
-    Call->setAttributes(legalizeCallAttributes(ToReplace->getAttributes()));
+    SPCall->setAttributes(legalizeCallAttributes(CI->getAttributes()));
 
-    Token = Call;
+    Token = SPCall;
 
     // Put the following gc_result and gc_relocate calls immediately after the
     // the old call (which we're about to delete)
-    assert(ToReplace->getNextNode() && "Not a terminator, must have next!");
-    Builder.SetInsertPoint(ToReplace->getNextNode());
-    Builder.SetCurrentDebugLocation(ToReplace->getNextNode()->getDebugLoc());
+    assert(CI->getNextNode() && "Not a terminator, must have next!");
+    Builder.SetInsertPoint(CI->getNextNode());
+    Builder.SetCurrentDebugLocation(CI->getNextNode()->getDebugLoc());
   } else {
-    InvokeInst *ToReplace = cast<InvokeInst>(CS.getInstruction());
+    auto *II = cast<InvokeInst>(Call);
 
     // Insert the new invoke into the old block.  We'll remove the old one in a
     // moment at which point this will become the new terminator for the
     // original block.
-    InvokeInst *Invoke = Builder.CreateGCStatepointInvoke(
-        StatepointID, NumPatchBytes, CallTarget, ToReplace->getNormalDest(),
-        ToReplace->getUnwindDest(), Flags, CallArgs, TransitionArgs, DeoptArgs,
-        GCArgs, "statepoint_token");
+    InvokeInst *SPInvoke = Builder.CreateGCStatepointInvoke(
+        StatepointID, NumPatchBytes, CallTarget, II->getNormalDest(),
+        II->getUnwindDest(), Flags, CallArgs, TransitionArgs, DeoptArgs, GCArgs,
+        "statepoint_token");
 
-    Invoke->setCallingConv(ToReplace->getCallingConv());
+    SPInvoke->setCallingConv(II->getCallingConv());
 
     // Currently we will fail on parameter attributes and on certain
     // function attributes.  In case if we can handle this set of attributes -
     // set up function attrs directly on statepoint and return attrs later for
     // gc_result intrinsic.
-    Invoke->setAttributes(legalizeCallAttributes(ToReplace->getAttributes()));
+    SPInvoke->setAttributes(legalizeCallAttributes(II->getAttributes()));
 
-    Token = Invoke;
+    Token = SPInvoke;
 
     // Generate gc relocates in exceptional path
-    BasicBlock *UnwindBlock = ToReplace->getUnwindDest();
+    BasicBlock *UnwindBlock = II->getUnwindDest();
     assert(!isa<PHINode>(UnwindBlock->begin()) &&
            UnwindBlock->getUniquePredecessor() &&
            "can't safely insert in this block!");
 
     Builder.SetInsertPoint(&*UnwindBlock->getFirstInsertionPt());
-    Builder.SetCurrentDebugLocation(ToReplace->getDebugLoc());
+    Builder.SetCurrentDebugLocation(II->getDebugLoc());
 
     // Attach exceptional gc relocates to the landingpad.
     Instruction *ExceptionalToken = UnwindBlock->getLandingPadInst();
@@ -1551,7 +1545,7 @@ makeStatepointExplicitImpl(const CallSite CS, /* to replace */
                       Builder);
 
     // Generate gc relocates and returns for normal block
-    BasicBlock *NormalDest = ToReplace->getNormalDest();
+    BasicBlock *NormalDest = II->getNormalDest();
     assert(!isa<PHINode>(NormalDest->begin()) &&
            NormalDest->getUniquePredecessor() &&
            "can't safely insert in this block!");
@@ -1568,16 +1562,15 @@ makeStatepointExplicitImpl(const CallSite CS, /* to replace */
     // transform the tail-call like structure to a call to a void function
     // followed by unreachable to get better codegen.
     Replacements.push_back(
-        DeferredReplacement::createDeoptimizeReplacement(CS.getInstruction()));
+        DeferredReplacement::createDeoptimizeReplacement(Call));
   } else {
     Token->setName("statepoint_token");
-    if (!CS.getType()->isVoidTy() && !CS.getInstruction()->use_empty()) {
-      StringRef Name =
-          CS.getInstruction()->hasName() ? CS.getInstruction()->getName() : "";
-      CallInst *GCResult = Builder.CreateGCResult(Token, CS.getType(), Name);
+    if (!Call->getType()->isVoidTy() && !Call->use_empty()) {
+      StringRef Name = Call->hasName() ? Call->getName() : "";
+      CallInst *GCResult = Builder.CreateGCResult(Token, Call->getType(), Name);
       GCResult->setAttributes(
           AttributeList::get(GCResult->getContext(), AttributeList::ReturnIndex,
-                             CS.getAttributes().getRetAttributes()));
+                             Call->getAttributes().getRetAttributes()));
 
       // We cannot RAUW or delete CS.getInstruction() because it could be in the
       // live set of some other safepoint, in which case that safepoint's
@@ -1586,10 +1579,9 @@ makeStatepointExplicitImpl(const CallSite CS, /* to replace */
       // after the live sets have been made explicit in the IR, and we no longer
       // have raw pointers to worry about.
       Replacements.emplace_back(
-          DeferredReplacement::createRAUW(CS.getInstruction(), GCResult));
+          DeferredReplacement::createRAUW(Call, GCResult));
     } else {
-      Replacements.emplace_back(
-          DeferredReplacement::createDelete(CS.getInstruction()));
+      Replacements.emplace_back(DeferredReplacement::createDelete(Call));
     }
   }
 
@@ -1606,7 +1598,7 @@ makeStatepointExplicitImpl(const CallSite CS, /* to replace */
 // WARNING: Does not do any fixup to adjust users of the original live
 // values.  That's the callers responsibility.
 static void
-makeStatepointExplicit(DominatorTree &DT, CallSite CS,
+makeStatepointExplicit(DominatorTree &DT, CallBase *Call,
                        PartiallyConstructedSafepointRecord &Result,
                        std::vector<DeferredReplacement> &Replacements) {
   const auto &LiveSet = Result.LiveSet;
@@ -1625,7 +1617,7 @@ makeStatepointExplicit(DominatorTree &DT, CallSite CS,
   assert(LiveVec.size() == BaseVec.size());
 
   // Do the actual rewriting and delete the old statepoint
-  makeStatepointExplicitImpl(CS, BaseVec, LiveVec, Result, Replacements);
+  makeStatepointExplicitImpl(Call, BaseVec, LiveVec, Result, Replacements);
 }
 
 // Helper function for the relocationViaAlloca.
@@ -1636,7 +1628,7 @@ makeStatepointExplicit(DominatorTree &DT, CallSite CS,
 // for sanity checking.
 static void
 insertRelocationStores(iterator_range<Value::user_iterator> GCRelocs,
-                       DenseMap<Value *, Value *> &AllocaMap,
+                       DenseMap<Value *, AllocaInst *> &AllocaMap,
                        DenseSet<Value *> &VisitedLiveValues) {
   for (User *U : GCRelocs) {
     GCRelocateInst *Relocate = dyn_cast<GCRelocateInst>(U);
@@ -1671,7 +1663,7 @@ insertRelocationStores(iterator_range<Value::user_iterator> GCRelocs,
 // "insertRelocationStores" but works for rematerialized values.
 static void insertRematerializationStores(
     const RematerializedValueMapTy &RematerializedValues,
-    DenseMap<Value *, Value *> &AllocaMap,
+    DenseMap<Value *, AllocaInst *> &AllocaMap,
     DenseSet<Value *> &VisitedLiveValues) {
   for (auto RematerializedValuePair: RematerializedValues) {
     Instruction *RematerializedValue = RematerializedValuePair.first;
@@ -1704,7 +1696,7 @@ static void relocationViaAlloca(
 #endif
 
   // TODO-PERF: change data structures, reserve
-  DenseMap<Value *, Value *> AllocaMap;
+  DenseMap<Value *, AllocaInst *> AllocaMap;
   SmallVector<AllocaInst *, 200> PromotableAllocas;
   // Used later to chack that we have enough allocas to store all values
   std::size_t NumRematerializedValues = 0;
@@ -1774,7 +1766,7 @@ static void relocationViaAlloca(
       SmallVector<AllocaInst *, 64> ToClobber;
       for (auto Pair : AllocaMap) {
         Value *Def = Pair.first;
-        AllocaInst *Alloca = cast<AllocaInst>(Pair.second);
+        AllocaInst *Alloca = Pair.second;
 
         // This value was relocated
         if (VisitedLiveValues.count(Def)) {
@@ -1806,7 +1798,7 @@ static void relocationViaAlloca(
   // Update use with load allocas and add store for gc_relocated.
   for (auto Pair : AllocaMap) {
     Value *Def = Pair.first;
-    Value *Alloca = Pair.second;
+    AllocaInst *Alloca = Pair.second;
 
     // We pre-record the uses of allocas so that we dont have to worry about
     // later update that changes the user information..
@@ -1834,13 +1826,15 @@ static void relocationViaAlloca(
         PHINode *Phi = cast<PHINode>(Use);
         for (unsigned i = 0; i < Phi->getNumIncomingValues(); i++) {
           if (Def == Phi->getIncomingValue(i)) {
-            LoadInst *Load = new LoadInst(
-                Alloca, "", Phi->getIncomingBlock(i)->getTerminator());
+            LoadInst *Load =
+                new LoadInst(Alloca->getAllocatedType(), Alloca, "",
+                             Phi->getIncomingBlock(i)->getTerminator());
             Phi->setIncomingValue(i, Load);
           }
         }
       } else {
-        LoadInst *Load = new LoadInst(Alloca, "", Use);
+        LoadInst *Load =
+            new LoadInst(Alloca->getAllocatedType(), Alloca, "", Use);
         Use->replaceUsesOfWith(Def, Load);
       }
     }
@@ -1893,25 +1887,25 @@ template <typename T> static void unique_unsorted(SmallVectorImpl<T> &Vec) {
 
 /// Insert holders so that each Value is obviously live through the entire
 /// lifetime of the call.
-static void insertUseHolderAfter(CallSite &CS, const ArrayRef<Value *> Values,
+static void insertUseHolderAfter(CallBase *Call, const ArrayRef<Value *> Values,
                                  SmallVectorImpl<CallInst *> &Holders) {
   if (Values.empty())
     // No values to hold live, might as well not insert the empty holder
     return;
 
-  Module *M = CS.getInstruction()->getModule();
+  Module *M = Call->getModule();
   // Use a dummy vararg function to actually hold the values live
-  Function *Func = cast<Function>(M->getOrInsertFunction(
-      "__tmp_use", FunctionType::get(Type::getVoidTy(M->getContext()), true)));
-  if (CS.isCall()) {
+  FunctionCallee Func = M->getOrInsertFunction(
+      "__tmp_use", FunctionType::get(Type::getVoidTy(M->getContext()), true));
+  if (isa<CallInst>(Call)) {
     // For call safepoints insert dummy calls right after safepoint
-    Holders.push_back(CallInst::Create(Func, Values, "",
-                                       &*++CS.getInstruction()->getIterator()));
+    Holders.push_back(
+        CallInst::Create(Func, Values, "", &*++Call->getIterator()));
     return;
   }
   // For invoke safepooints insert dummy calls both in normal and
   // exceptional destination blocks
-  auto *II = cast<InvokeInst>(CS.getInstruction());
+  auto *II = cast<InvokeInst>(Call);
   Holders.push_back(CallInst::Create(
       Func, Values, "", &*II->getNormalDest()->getFirstInsertionPt()));
   Holders.push_back(CallInst::Create(
@@ -1919,7 +1913,7 @@ static void insertUseHolderAfter(CallSite &CS, const ArrayRef<Value *> Values,
 }
 
 static void findLiveReferences(
-    Function &F, DominatorTree &DT, ArrayRef<CallSite> toUpdate,
+    Function &F, DominatorTree &DT, ArrayRef<CallBase *> toUpdate,
     MutableArrayRef<struct PartiallyConstructedSafepointRecord> records) {
   GCPtrLivenessData OriginalLivenessData;
   computeLiveInValues(DT, F, OriginalLivenessData);
@@ -2022,7 +2016,7 @@ static bool AreEquivalentPhiNodes(PHINode &OrigRootPhi, PHINode &AlternateRootPh
 // to relocate. Remove this values from the live set, rematerialize them after
 // statepoint and record them in "Info" structure. Note that similar to
 // relocated values we don't do any user adjustments here.
-static void rematerializeLiveValues(CallSite CS,
+static void rematerializeLiveValues(CallBase *Call,
                                     PartiallyConstructedSafepointRecord &Info,
                                     TargetTransformInfo &TTI) {
   const unsigned int ChainLengthThreshold = 10;
@@ -2076,7 +2070,7 @@ static void rematerializeLiveValues(CallSite CS,
 
     // For invokes we need to rematerialize each chain twice - for normal and
     // for unwind basic blocks. Model this by multiplying cost by two.
-    if (CS.isInvoke()) {
+    if (isa<InvokeInst>(Call)) {
       Cost *= 2;
     }
     // If it's too expensive - skip it
@@ -2144,14 +2138,14 @@ static void rematerializeLiveValues(CallSite CS,
 
     // Different cases for calls and invokes. For invokes we need to clone
     // instructions both on normal and unwind path.
-    if (CS.isCall()) {
-      Instruction *InsertBefore = CS.getInstruction()->getNextNode();
+    if (isa<CallInst>(Call)) {
+      Instruction *InsertBefore = Call->getNextNode();
       assert(InsertBefore);
       Instruction *RematerializedValue = rematerializeChain(
           InsertBefore, RootOfChain, Info.PointerToBase[LiveValue]);
       Info.RematerializedValues[RematerializedValue] = LiveValue;
     } else {
-      InvokeInst *Invoke = cast<InvokeInst>(CS.getInstruction());
+      auto *Invoke = cast<InvokeInst>(Call);
 
       Instruction *NormalInsertBefore =
           &*Invoke->getNormalDest()->getFirstInsertionPt();
@@ -2176,25 +2170,25 @@ static void rematerializeLiveValues(CallSite CS,
 
 static bool insertParsePoints(Function &F, DominatorTree &DT,
                               TargetTransformInfo &TTI,
-                              SmallVectorImpl<CallSite> &ToUpdate) {
+                              SmallVectorImpl<CallBase *> &ToUpdate) {
 #ifndef NDEBUG
   // sanity check the input
-  std::set<CallSite> Uniqued;
+  std::set<CallBase *> Uniqued;
   Uniqued.insert(ToUpdate.begin(), ToUpdate.end());
   assert(Uniqued.size() == ToUpdate.size() && "no duplicates please!");
 
-  for (CallSite CS : ToUpdate)
-    assert(CS.getInstruction()->getFunction() == &F);
+  for (CallBase *Call : ToUpdate)
+    assert(Call->getFunction() == &F);
 #endif
 
   // When inserting gc.relocates for invokes, we need to be able to insert at
   // the top of the successor blocks.  See the comment on
   // normalForInvokeSafepoint on exactly what is needed.  Note that this step
   // may restructure the CFG.
-  for (CallSite CS : ToUpdate) {
-    if (!CS.isInvoke())
+  for (CallBase *Call : ToUpdate) {
+    auto *II = dyn_cast<InvokeInst>(Call);
+    if (!II)
       continue;
-    auto *II = cast<InvokeInst>(CS.getInstruction());
     normalizeForInvokeSafepoint(II->getNormalDest(), II->getParent(), DT);
     normalizeForInvokeSafepoint(II->getUnwindDest(), II->getParent(), DT);
   }
@@ -2207,17 +2201,17 @@ static bool insertParsePoints(Function &F, DominatorTree &DT,
   // actual safepoint insertion as arguments.  This ensures reference operands
   // in the deopt argument list are considered live through the safepoint (and
   // thus makes sure they get relocated.)
-  for (CallSite CS : ToUpdate) {
+  for (CallBase *Call : ToUpdate) {
     SmallVector<Value *, 64> DeoptValues;
 
-    for (Value *Arg : GetDeoptBundleOperands(CS)) {
+    for (Value *Arg : GetDeoptBundleOperands(Call)) {
       assert(!isUnhandledGCPointerType(Arg->getType()) &&
              "support for FCA unimplemented");
       if (isHandledGCPointerType(Arg->getType()))
         DeoptValues.push_back(Arg);
     }
 
-    insertUseHolderAfter(CS, DeoptValues, Holders);
+    insertUseHolderAfter(Call, DeoptValues, Holders);
   }
 
   SmallVector<PartiallyConstructedSafepointRecord, 64> Records(ToUpdate.size());
@@ -2319,7 +2313,7 @@ static bool insertParsePoints(Function &F, DominatorTree &DT,
   for (size_t i = 0; i < Records.size(); i++)
     makeStatepointExplicit(DT, ToUpdate[i], Records[i], Replacements);
 
-  ToUpdate.clear(); // prevent accident use of invalid CallSites
+  ToUpdate.clear(); // prevent accident use of invalid calls.
 
   for (auto &PR : Replacements)
     PR.doReplacement();
@@ -2384,7 +2378,7 @@ static bool insertParsePoints(Function &F, DominatorTree &DT,
   return !Records.empty();
 }
 
-// Handles both return values and arguments for Functions and CallSites.
+// Handles both return values and arguments for Functions and calls.
 template <typename AttrHolder>
 static void RemoveNonValidAttrAtIndex(LLVMContext &Ctx, AttrHolder &AH,
                                       unsigned Index) {
@@ -2476,12 +2470,13 @@ static void stripNonValidDataFromBody(Function &F) {
 
     stripInvalidMetadataFromInstruction(I);
 
-    if (CallSite CS = CallSite(&I)) {
-      for (int i = 0, e = CS.arg_size(); i != e; i++)
-        if (isa<PointerType>(CS.getArgument(i)->getType()))
-          RemoveNonValidAttrAtIndex(Ctx, CS, i + AttributeList::FirstArgIndex);
-      if (isa<PointerType>(CS.getType()))
-        RemoveNonValidAttrAtIndex(Ctx, CS, AttributeList::ReturnIndex);
+    if (auto *Call = dyn_cast<CallBase>(&I)) {
+      for (int i = 0, e = Call->arg_size(); i != e; i++)
+        if (isa<PointerType>(Call->getArgOperand(i)->getType()))
+          RemoveNonValidAttrAtIndex(Ctx, *Call,
+                                    i + AttributeList::FirstArgIndex);
+      if (isa<PointerType>(Call->getType()))
+        RemoveNonValidAttrAtIndex(Ctx, *Call, AttributeList::ReturnIndex);
     }
   }
 
@@ -2526,12 +2521,11 @@ bool RewriteStatepointsForGC::runOnFunction(Function &F, DominatorTree &DT,
   assert(shouldRewriteStatepointsIn(F) && "mismatch in rewrite decision");
 
   auto NeedsRewrite = [&TLI](Instruction &I) {
-    if (ImmutableCallSite CS = ImmutableCallSite(&I))
-      return !callsGCLeafFunction(CS, TLI) && !isStatepoint(CS);
+    if (const auto *Call = dyn_cast<CallBase>(&I))
+      return !callsGCLeafFunction(Call, TLI) && !isStatepoint(Call);
     return false;
   };
 
-
   // Delete any unreachable statepoints so that we don't have unrewritten
   // statepoints surviving this pass.  This makes testing easier and the
   // resulting IR less confusing to human readers.
@@ -2543,7 +2537,7 @@ bool RewriteStatepointsForGC::runOnFunction(Function &F, DominatorTree &DT,
   // Gather all the statepoints which need rewritten.  Be careful to only
   // consider those in reachable code since we need to ask dominance queries
   // when rewriting.  We'll delete the unreachable ones in a moment.
-  SmallVector<CallSite, 64> ParsePointNeeded;
+  SmallVector<CallBase *, 64> ParsePointNeeded;
   for (Instruction &I : instructions(F)) {
     // TODO: only the ones with the flag set!
     if (NeedsRewrite(I)) {
@@ -2553,7 +2547,7 @@ bool RewriteStatepointsForGC::runOnFunction(Function &F, DominatorTree &DT,
       // isReachableFromEntry() returns true.
       assert(DT.isReachableFromEntry(I.getParent()) &&
             "no unreachable blocks expected");
-      ParsePointNeeded.push_back(CallSite(&I));
+      ParsePointNeeded.push_back(cast<CallBase>(&I));
     }
   }
 
@@ -2602,6 +2596,33 @@ bool RewriteStatepointsForGC::runOnFunction(Function &F, DominatorTree &DT,
       }
   }
 
+  // Nasty workaround - The base computation code in the main algorithm doesn't
+  // consider the fact that a GEP can be used to convert a scalar to a vector.
+  // The right fix for this is to integrate GEPs into the base rewriting
+  // algorithm properly, this is just a short term workaround to prevent
+  // crashes by canonicalizing such GEPs into fully vector GEPs.
+  for (Instruction &I : instructions(F)) {
+    if (!isa<GetElementPtrInst>(I))
+      continue;
+
+    unsigned VF = 0;
+    for (unsigned i = 0; i < I.getNumOperands(); i++)
+      if (I.getOperand(i)->getType()->isVectorTy()) {
+        assert(VF == 0 ||
+               VF == I.getOperand(i)->getType()->getVectorNumElements());
+        VF = I.getOperand(i)->getType()->getVectorNumElements();
+      }
+
+    // It's the vector to scalar traversal through the pointer operand which
+    // confuses base pointer rewriting, so limit ourselves to that case.
+    if (!I.getOperand(0)->getType()->isVectorTy() && VF != 0) {
+      IRBuilder<> B(&I);
+      auto *Splat = B.CreateVectorSplat(VF, I.getOperand(0));
+      I.setOperand(0, Splat);
+      MadeChange = true;
+    }
+  }
+
   MadeChange |= insertParsePoints(F, DT, TTI, ParsePointNeeded);
   return MadeChange;
 }
@@ -2786,11 +2807,10 @@ static void findLiveSetAtInst(Instruction *Inst, GCPtrLivenessData &Data,
 }
 
 static void recomputeLiveInValues(GCPtrLivenessData &RevisedLivenessData,
-                                  CallSite CS,
+                                  CallBase *Call,
                                   PartiallyConstructedSafepointRecord &Info) {
-  Instruction *Inst = CS.getInstruction();
   StatepointLiveSetTy Updated;
-  findLiveSetAtInst(Inst, RevisedLivenessData, Updated);
+  findLiveSetAtInst(Call, RevisedLivenessData, Updated);
 
   // We may have base pointers which are now live that weren't before.  We need
   // to update the PointerToBase structure to reflect this.
diff --git a/lib/Transforms/Scalar/SCCP.cpp b/lib/Transforms/Scalar/SCCP.cpp
index 2f6ed05c023b1e6bb63835fb59682ebc334b1786..e75a5dbddbae4e9be0707ff87f00d82636b72104 100644
--- a/lib/Transforms/Scalar/SCCP.cpp
+++ b/lib/Transforms/Scalar/SCCP.cpp
@@ -1,9 +1,8 @@
 //===- SCCP.cpp - Sparse Conditional Constant Propagation -----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -639,6 +638,11 @@ private:
     visitTerminator(II);
   }
 
+  void visitCallBrInst    (CallBrInst &CBI) {
+    visitCallSite(&CBI);
+    visitTerminator(CBI);
+  }
+
   void visitCallSite      (CallSite CS);
   void visitResumeInst    (ResumeInst &I) { /*returns void*/ }
   void visitUnreachableInst(UnreachableInst &I) { /*returns void*/ }
@@ -734,6 +738,13 @@ void SCCPSolver::getFeasibleSuccessors(Instruction &TI,
     return;
   }
 
+  // In case of callbr, we pessimistically assume that all successors are
+  // feasible.
+  if (isa<CallBrInst>(&TI)) {
+    Succs.assign(TI.getNumSuccessors(), true);
+    return;
+  }
+
   LLVM_DEBUG(dbgs() << "Unknown terminator instruction: " << TI << '\n');
   llvm_unreachable("SCCP: Don't know how to handle this terminator!");
 }
@@ -1232,7 +1243,7 @@ CallOverdefined:
     // Otherwise, if we have a single return value case, and if the function is
     // a declaration, maybe we can constant fold it.
     if (F && F->isDeclaration() && !I->getType()->isStructTy() &&
-        canConstantFoldCallTo(CS, F)) {
+        canConstantFoldCallTo(cast<CallBase>(CS.getInstruction()), F)) {
       SmallVector<Constant*, 8> Operands;
       for (CallSite::arg_iterator AI = CS.arg_begin(), E = CS.arg_end();
            AI != E; ++AI) {
@@ -1253,7 +1264,8 @@ CallOverdefined:
 
       // If we can constant fold this, mark the result of the call as a
       // constant.
-      if (Constant *C = ConstantFoldCall(CS, F, Operands, TLI)) {
+      if (Constant *C = ConstantFoldCall(cast<CallBase>(CS.getInstruction()), F,
+                                         Operands, TLI)) {
         // call -> undef.
         if (isa<UndefValue>(C))
           return;
@@ -1598,6 +1610,7 @@ bool SCCPSolver::ResolvedUndefsIn(Function &F) {
         return true;
       case Instruction::Call:
       case Instruction::Invoke:
+      case Instruction::CallBr:
         // There are two reasons a call can have an undef result
         // 1. It could be tracked.
         // 2. It could be constant-foldable.
diff --git a/lib/Transforms/Scalar/SROA.cpp b/lib/Transforms/Scalar/SROA.cpp
index 81eea0dee4e4425fb192aae64e92d86bd5efa0ec..7a63bf58b47ed0f0f845cac87764eb5b9084950b 100644
--- a/lib/Transforms/Scalar/SROA.cpp
+++ b/lib/Transforms/Scalar/SROA.cpp
@@ -1,9 +1,8 @@
 //===- SROA.cpp - Scalar Replacement Of Aggregates ------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
@@ -222,13 +221,6 @@ public:
 
 } // end anonymous namespace
 
-namespace llvm {
-
-template <typename T> struct isPodLike;
-template <> struct isPodLike<Slice> { static const bool value = true; };
-
-} // end namespace llvm
-
 /// Representation of the alloca slices.
 ///
 /// This class represents the slices of an alloca which are formed by its
@@ -913,8 +905,7 @@ private:
     if (!IsOffsetKnown)
       return PI.setAborted(&II);
 
-    if (II.getIntrinsicID() == Intrinsic::lifetime_start ||
-        II.getIntrinsicID() == Intrinsic::lifetime_end) {
+    if (II.isLifetimeStartOrEnd()) {
       ConstantInt *Length = cast<ConstantInt>(II.getArgOperand(0));
       uint64_t Size = std::min(AllocSize - Offset.getLimitedValue(),
                                Length->getLimitedValue());
@@ -1240,15 +1231,14 @@ static bool isSafePHIToSpeculate(PHINode &PN) {
 static void speculatePHINodeLoads(PHINode &PN) {
   LLVM_DEBUG(dbgs() << "    original: " << PN << "\n");
 
-  Type *LoadTy = cast<PointerType>(PN.getType())->getElementType();
+  LoadInst *SomeLoad = cast<LoadInst>(PN.user_back());
+  Type *LoadTy = SomeLoad->getType();
   IRBuilderTy PHIBuilder(&PN);
   PHINode *NewPN = PHIBuilder.CreatePHI(LoadTy, PN.getNumIncomingValues(),
                                         PN.getName() + ".sroa.speculated");
 
   // Get the AA tags and alignment to use from one of the loads.  It doesn't
   // matter which one we get and if any differ.
-  LoadInst *SomeLoad = cast<LoadInst>(PN.user_back());
-
   AAMDNodes AATags;
   SomeLoad->getAAMetadata(AATags);
   unsigned Align = SomeLoad->getAlignment();
@@ -1279,7 +1269,8 @@ static void speculatePHINodeLoads(PHINode &PN) {
     IRBuilderTy PredBuilder(TI);
 
     LoadInst *Load = PredBuilder.CreateLoad(
-        InVal, (PN.getName() + ".sroa.speculate.load." + Pred->getName()));
+        LoadTy, InVal,
+        (PN.getName() + ".sroa.speculate.load." + Pred->getName()));
     ++NumLoadsSpeculated;
     Load->setAlignment(Align);
     if (AATags)
@@ -1339,10 +1330,10 @@ static void speculateSelectInstLoads(SelectInst &SI) {
     assert(LI->isSimple() && "We only speculate simple loads");
 
     IRB.SetInsertPoint(LI);
-    LoadInst *TL =
-        IRB.CreateLoad(TV, LI->getName() + ".sroa.speculate.load.true");
-    LoadInst *FL =
-        IRB.CreateLoad(FV, LI->getName() + ".sroa.speculate.load.false");
+    LoadInst *TL = IRB.CreateLoad(LI->getType(), TV,
+                                  LI->getName() + ".sroa.speculate.load.true");
+    LoadInst *FL = IRB.CreateLoad(LI->getType(), FV,
+                                  LI->getName() + ".sroa.speculate.load.false");
     NumLoadsSpeculated += 2;
 
     // Transfer alignment and AA info if present.
@@ -1380,8 +1371,8 @@ static Value *buildGEP(IRBuilderTy &IRB, Value *BasePtr,
   if (Indices.size() == 1 && cast<ConstantInt>(Indices.back())->isZero())
     return BasePtr;
 
-  return IRB.CreateInBoundsGEP(nullptr, BasePtr, Indices,
-                               NamePrefix + "sroa_idx");
+  return IRB.CreateInBoundsGEP(BasePtr->getType()->getPointerElementType(),
+                               BasePtr, Indices, NamePrefix + "sroa_idx");
 }
 
 /// Get a natural GEP off of the BasePtr walking through Ty toward
@@ -1807,8 +1798,7 @@ static bool isVectorPromotionViableForSlice(Partition &P, const Slice &S,
     if (!S.isSplittable())
       return false; // Skip any unsplittable intrinsics.
   } else if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(U->getUser())) {
-    if (II->getIntrinsicID() != Intrinsic::lifetime_start &&
-        II->getIntrinsicID() != Intrinsic::lifetime_end)
+    if (!II->isLifetimeStartOrEnd())
       return false;
   } else if (U->get()->getType()->getPointerElementType()->isStructTy()) {
     // Disable vector promotion when there are loads or stores of an FCA.
@@ -2029,8 +2019,7 @@ static bool isIntegerWideningViableForSlice(const Slice &S,
     if (!S.isSplittable())
       return false; // Skip any unsplittable intrinsics.
   } else if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(U->getUser())) {
-    if (II->getIntrinsicID() != Intrinsic::lifetime_start &&
-        II->getIntrinsicID() != Intrinsic::lifetime_end)
+    if (!II->isLifetimeStartOrEnd())
       return false;
   } else {
     return false;
@@ -2421,14 +2410,16 @@ private:
     unsigned EndIndex = getIndex(NewEndOffset);
     assert(EndIndex > BeginIndex && "Empty vector!");
 
-    Value *V = IRB.CreateAlignedLoad(&NewAI, NewAI.getAlignment(), "load");
+    Value *V = IRB.CreateAlignedLoad(NewAI.getAllocatedType(), &NewAI,
+                                     NewAI.getAlignment(), "load");
     return extractVector(IRB, V, BeginIndex, EndIndex, "vec");
   }
 
   Value *rewriteIntegerLoad(LoadInst &LI) {
     assert(IntTy && "We cannot insert an integer to the alloca");
     assert(!LI.isVolatile());
-    Value *V = IRB.CreateAlignedLoad(&NewAI, NewAI.getAlignment(), "load");
+    Value *V = IRB.CreateAlignedLoad(NewAI.getAllocatedType(), &NewAI,
+                                     NewAI.getAlignment(), "load");
     V = convertValue(DL, IRB, V, IntTy);
     assert(NewBeginOffset >= NewAllocaBeginOffset && "Out of bounds offset");
     uint64_t Offset = NewBeginOffset - NewAllocaBeginOffset;
@@ -2472,7 +2463,8 @@ private:
                (canConvertValue(DL, NewAllocaTy, TargetTy) ||
                 (IsLoadPastEnd && NewAllocaTy->isIntegerTy() &&
                  TargetTy->isIntegerTy()))) {
-      LoadInst *NewLI = IRB.CreateAlignedLoad(&NewAI, NewAI.getAlignment(),
+      LoadInst *NewLI = IRB.CreateAlignedLoad(NewAI.getAllocatedType(), &NewAI,
+                                              NewAI.getAlignment(),
                                               LI.isVolatile(), LI.getName());
       if (AATags)
         NewLI->setAAMetadata(AATags);
@@ -2508,9 +2500,9 @@ private:
           }
     } else {
       Type *LTy = TargetTy->getPointerTo(AS);
-      LoadInst *NewLI = IRB.CreateAlignedLoad(getNewAllocaSlicePtr(IRB, LTy),
-                                              getSliceAlign(TargetTy),
-                                              LI.isVolatile(), LI.getName());
+      LoadInst *NewLI = IRB.CreateAlignedLoad(
+          TargetTy, getNewAllocaSlicePtr(IRB, LTy), getSliceAlign(TargetTy),
+          LI.isVolatile(), LI.getName());
       if (AATags)
         NewLI->setAAMetadata(AATags);
       if (LI.isVolatile())
@@ -2536,8 +2528,8 @@ private:
       // basis for the new value. This allows us to replace the uses of LI with
       // the computed value, and then replace the placeholder with LI, leaving
       // LI only used for this computation.
-      Value *Placeholder =
-          new LoadInst(UndefValue::get(LI.getType()->getPointerTo(AS)));
+      Value *Placeholder = new LoadInst(
+          LI.getType(), UndefValue::get(LI.getType()->getPointerTo(AS)));
       V = insertInteger(DL, IRB, Placeholder, V, NewBeginOffset - BeginOffset,
                         "insert");
       LI.replaceAllUsesWith(V);
@@ -2568,7 +2560,8 @@ private:
         V = convertValue(DL, IRB, V, SliceTy);
 
       // Mix in the existing elements.
-      Value *Old = IRB.CreateAlignedLoad(&NewAI, NewAI.getAlignment(), "load");
+      Value *Old = IRB.CreateAlignedLoad(NewAI.getAllocatedType(), &NewAI,
+                                         NewAI.getAlignment(), "load");
       V = insertVector(IRB, Old, V, BeginIndex, "vec");
     }
     StoreInst *Store = IRB.CreateAlignedStore(V, &NewAI, NewAI.getAlignment());
@@ -2584,8 +2577,8 @@ private:
     assert(IntTy && "We cannot extract an integer from the alloca");
     assert(!SI.isVolatile());
     if (DL.getTypeSizeInBits(V->getType()) != IntTy->getBitWidth()) {
-      Value *Old =
-          IRB.CreateAlignedLoad(&NewAI, NewAI.getAlignment(), "oldload");
+      Value *Old = IRB.CreateAlignedLoad(NewAI.getAllocatedType(), &NewAI,
+                                         NewAI.getAlignment(), "oldload");
       Old = convertValue(DL, IRB, Old, IntTy);
       assert(BeginOffset >= NewAllocaBeginOffset && "Out of bounds offset");
       uint64_t Offset = BeginOffset - NewAllocaBeginOffset;
@@ -2593,7 +2586,8 @@ private:
     }
     V = convertValue(DL, IRB, V, NewAllocaTy);
     StoreInst *Store = IRB.CreateAlignedStore(V, &NewAI, NewAI.getAlignment());
-    Store->copyMetadata(SI, LLVMContext::MD_mem_parallel_loop_access);
+    Store->copyMetadata(SI, {LLVMContext::MD_mem_parallel_loop_access,
+                             LLVMContext::MD_access_group});
     if (AATags)
       Store->setAAMetadata(AATags);
     Pass.DeadInsts.insert(&SI);
@@ -2662,7 +2656,8 @@ private:
       NewSI = IRB.CreateAlignedStore(V, NewPtr, getSliceAlign(V->getType()),
                                      SI.isVolatile());
     }
-    NewSI->copyMetadata(SI, LLVMContext::MD_mem_parallel_loop_access);
+    NewSI->copyMetadata(SI, {LLVMContext::MD_mem_parallel_loop_access,
+                             LLVMContext::MD_access_group});
     if (AATags)
       NewSI->setAAMetadata(AATags);
     if (SI.isVolatile())
@@ -2775,8 +2770,8 @@ private:
       if (NumElements > 1)
         Splat = getVectorSplat(Splat, NumElements);
 
-      Value *Old =
-          IRB.CreateAlignedLoad(&NewAI, NewAI.getAlignment(), "oldload");
+      Value *Old = IRB.CreateAlignedLoad(NewAI.getAllocatedType(), &NewAI,
+                                         NewAI.getAlignment(), "oldload");
       V = insertVector(IRB, Old, Splat, BeginIndex, "vec");
     } else if (IntTy) {
       // If this is a memset on an alloca where we can widen stores, insert the
@@ -2788,8 +2783,8 @@ private:
 
       if (IntTy && (BeginOffset != NewAllocaBeginOffset ||
                     EndOffset != NewAllocaBeginOffset)) {
-        Value *Old =
-            IRB.CreateAlignedLoad(&NewAI, NewAI.getAlignment(), "oldload");
+        Value *Old = IRB.CreateAlignedLoad(NewAI.getAllocatedType(), &NewAI,
+                                           NewAI.getAlignment(), "oldload");
         Old = convertValue(DL, IRB, Old, IntTy);
         uint64_t Offset = NewBeginOffset - NewAllocaBeginOffset;
         V = insertInteger(DL, IRB, Old, V, Offset, "insert");
@@ -2949,18 +2944,18 @@ private:
 
     // Reset the other pointer type to match the register type we're going to
     // use, but using the address space of the original other pointer.
+    Type *OtherTy;
     if (VecTy && !IsWholeAlloca) {
       if (NumElements == 1)
-        OtherPtrTy = VecTy->getElementType();
+        OtherTy = VecTy->getElementType();
       else
-        OtherPtrTy = VectorType::get(VecTy->getElementType(), NumElements);
-
-      OtherPtrTy = OtherPtrTy->getPointerTo(OtherAS);
+        OtherTy = VectorType::get(VecTy->getElementType(), NumElements);
     } else if (IntTy && !IsWholeAlloca) {
-      OtherPtrTy = SubIntTy->getPointerTo(OtherAS);
+      OtherTy = SubIntTy;
     } else {
-      OtherPtrTy = NewAllocaTy->getPointerTo(OtherAS);
+      OtherTy = NewAllocaTy;
     }
+    OtherPtrTy = OtherTy->getPointerTo(OtherAS);
 
     Value *SrcPtr = getAdjustedPtr(IRB, DL, OtherPtr, OtherOffset, OtherPtrTy,
                                    OtherPtr->getName() + ".");
@@ -2974,28 +2969,30 @@ private:
 
     Value *Src;
     if (VecTy && !IsWholeAlloca && !IsDest) {
-      Src = IRB.CreateAlignedLoad(&NewAI, NewAI.getAlignment(), "load");
+      Src = IRB.CreateAlignedLoad(NewAI.getAllocatedType(), &NewAI,
+                                  NewAI.getAlignment(), "load");
       Src = extractVector(IRB, Src, BeginIndex, EndIndex, "vec");
     } else if (IntTy && !IsWholeAlloca && !IsDest) {
-      Src = IRB.CreateAlignedLoad(&NewAI, NewAI.getAlignment(), "load");
+      Src = IRB.CreateAlignedLoad(NewAI.getAllocatedType(), &NewAI,
+                                  NewAI.getAlignment(), "load");
       Src = convertValue(DL, IRB, Src, IntTy);
       uint64_t Offset = NewBeginOffset - NewAllocaBeginOffset;
       Src = extractInteger(DL, IRB, Src, SubIntTy, Offset, "extract");
     } else {
-      LoadInst *Load = IRB.CreateAlignedLoad(SrcPtr, SrcAlign, II.isVolatile(),
-                                             "copyload");
+      LoadInst *Load = IRB.CreateAlignedLoad(OtherTy, SrcPtr, SrcAlign,
+                                             II.isVolatile(), "copyload");
       if (AATags)
         Load->setAAMetadata(AATags);
       Src = Load;
     }
 
     if (VecTy && !IsWholeAlloca && IsDest) {
-      Value *Old =
-          IRB.CreateAlignedLoad(&NewAI, NewAI.getAlignment(), "oldload");
+      Value *Old = IRB.CreateAlignedLoad(NewAI.getAllocatedType(), &NewAI,
+                                         NewAI.getAlignment(), "oldload");
       Src = insertVector(IRB, Old, Src, BeginIndex, "vec");
     } else if (IntTy && !IsWholeAlloca && IsDest) {
-      Value *Old =
-          IRB.CreateAlignedLoad(&NewAI, NewAI.getAlignment(), "oldload");
+      Value *Old = IRB.CreateAlignedLoad(NewAI.getAllocatedType(), &NewAI,
+                                         NewAI.getAlignment(), "oldload");
       Old = convertValue(DL, IRB, Old, IntTy);
       uint64_t Offset = NewBeginOffset - NewAllocaBeginOffset;
       Src = insertInteger(DL, IRB, Old, Src, Offset, "insert");
@@ -3011,8 +3008,7 @@ private:
   }
 
   bool visitIntrinsicInst(IntrinsicInst &II) {
-    assert(II.getIntrinsicID() == Intrinsic::lifetime_start ||
-           II.getIntrinsicID() == Intrinsic::lifetime_end);
+    assert(II.isLifetimeStartOrEnd());
     LLVM_DEBUG(dbgs() << "    original: " << II << "\n");
     assert(II.getArgOperand(1) == OldPtr);
 
@@ -3033,7 +3029,10 @@ private:
     ConstantInt *Size =
         ConstantInt::get(cast<IntegerType>(II.getArgOperand(0)->getType()),
                          NewEndOffset - NewBeginOffset);
-    Value *Ptr = getNewAllocaSlicePtr(IRB, OldPtr->getType());
+    // Lifetime intrinsics always expect an i8* so directly get such a pointer
+    // for the new alloca slice.
+    Type *PointerTy = IRB.getInt8PtrTy(OldPtr->getType()->getPointerAddressSpace());
+    Value *Ptr = getNewAllocaSlicePtr(IRB, PointerTy);
     Value *New;
     if (II.getIntrinsicID() == Intrinsic::lifetime_start)
       New = IRB.CreateLifetimeStart(Ptr, Size);
@@ -3299,8 +3298,8 @@ private:
       assert(Ty->isSingleValueType());
       // Load the single value and insert it using the indices.
       Value *GEP =
-          IRB.CreateInBoundsGEP(nullptr, Ptr, GEPIndices, Name + ".gep");
-      LoadInst *Load = IRB.CreateAlignedLoad(GEP, Align, Name + ".load");
+          IRB.CreateInBoundsGEP(BaseTy, Ptr, GEPIndices, Name + ".gep");
+      LoadInst *Load = IRB.CreateAlignedLoad(Ty, GEP, Align, Name + ".load");
       if (AATags)
         Load->setAAMetadata(AATags);
       Agg = IRB.CreateInsertValue(Agg, Load, Indices, Name + ".insert");
@@ -3344,7 +3343,7 @@ private:
       Value *ExtractValue =
           IRB.CreateExtractValue(Agg, Indices, Name + ".extract");
       Value *InBoundsGEP =
-          IRB.CreateInBoundsGEP(nullptr, Ptr, GEPIndices, Name + ".gep");
+          IRB.CreateInBoundsGEP(BaseTy, Ptr, GEPIndices, Name + ".gep");
       StoreInst *Store =
           IRB.CreateAlignedStore(ExtractValue, InBoundsGEP, Align);
       if (AATags)
@@ -3794,12 +3793,14 @@ bool SROA::presplitLoadsAndStores(AllocaInst &AI, AllocaSlices &AS) {
       auto AS = LI->getPointerAddressSpace();
       auto *PartPtrTy = PartTy->getPointerTo(AS);
       LoadInst *PLoad = IRB.CreateAlignedLoad(
+          PartTy,
           getAdjustedPtr(IRB, DL, BasePtr,
                          APInt(DL.getIndexSizeInBits(AS), PartOffset),
                          PartPtrTy, BasePtr->getName() + "."),
           getAdjustedAlignment(LI, PartOffset, DL), /*IsVolatile*/ false,
           LI->getName());
-      PLoad->copyMetadata(*LI, LLVMContext::MD_mem_parallel_loop_access);
+      PLoad->copyMetadata(*LI, {LLVMContext::MD_mem_parallel_loop_access,
+                                LLVMContext::MD_access_group});
 
       // Append this load onto the list of split loads so we can find it later
       // to rewrite the stores.
@@ -3855,7 +3856,8 @@ bool SROA::presplitLoadsAndStores(AllocaInst &AI, AllocaSlices &AS) {
                            APInt(DL.getIndexSizeInBits(AS), PartOffset),
                            PartPtrTy, StoreBasePtr->getName() + "."),
             getAdjustedAlignment(SI, PartOffset, DL), /*IsVolatile*/ false);
-        PStore->copyMetadata(*LI, LLVMContext::MD_mem_parallel_loop_access);
+        PStore->copyMetadata(*LI, {LLVMContext::MD_mem_parallel_loop_access,
+                                   LLVMContext::MD_access_group});
         LLVM_DEBUG(dbgs() << "      +" << PartOffset << ":" << *PStore << "\n");
       }
 
@@ -3933,6 +3935,7 @@ bool SROA::presplitLoadsAndStores(AllocaInst &AI, AllocaSlices &AS) {
         IRB.SetInsertPoint(LI);
         auto AS = LI->getPointerAddressSpace();
         PLoad = IRB.CreateAlignedLoad(
+            PartTy,
             getAdjustedPtr(IRB, DL, LoadBasePtr,
                            APInt(DL.getIndexSizeInBits(AS), PartOffset),
                            LoadPartPtrTy, LoadBasePtr->getName() + "."),
diff --git a/lib/Transforms/Scalar/Scalar.cpp b/lib/Transforms/Scalar/Scalar.cpp
index 976daf4c78c2fd2c3ab86e755c1e2055aadfd9de..c91ffda6d9807b82ebcc931af651097b7c777836 100644
--- a/lib/Transforms/Scalar/Scalar.cpp
+++ b/lib/Transforms/Scalar/Scalar.cpp
@@ -1,9 +1,8 @@
 //===-- Scalar.cpp --------------------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -81,6 +80,7 @@ void llvm::initializeScalarOpts(PassRegistry &Registry) {
   initializeLowerAtomicLegacyPassPass(Registry);
   initializeLowerExpectIntrinsicPass(Registry);
   initializeLowerGuardIntrinsicLegacyPassPass(Registry);
+  initializeLowerWidenableConditionLegacyPassPass(Registry);
   initializeMemCpyOptLegacyPassPass(Registry);
   initializeMergeICmpsPass(Registry);
   initializeMergedLoadStoreMotionLegacyPassPass(Registry);
diff --git a/lib/Transforms/Scalar/Scalarizer.cpp b/lib/Transforms/Scalar/Scalarizer.cpp
index 3a6f8e6b09595833481e78ff883b3355a0ad3887..3d554f15af2cfd94f4128f269a47506e686e7c9b 100644
--- a/lib/Transforms/Scalar/Scalarizer.cpp
+++ b/lib/Transforms/Scalar/Scalarizer.cpp
@@ -1,9 +1,8 @@
 //===- Scalarizer.cpp - Scalarize vector operations -----------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -246,14 +245,13 @@ Value *Scatterer::operator[](unsigned I) {
     return CV[I];
   IRBuilder<> Builder(BB, BBI);
   if (PtrTy) {
+    Type *ElTy = PtrTy->getElementType()->getVectorElementType();
     if (!CV[0]) {
-      Type *Ty =
-        PointerType::get(PtrTy->getElementType()->getVectorElementType(),
-                         PtrTy->getAddressSpace());
-      CV[0] = Builder.CreateBitCast(V, Ty, V->getName() + ".i0");
+      Type *NewPtrTy = PointerType::get(ElTy, PtrTy->getAddressSpace());
+      CV[0] = Builder.CreateBitCast(V, NewPtrTy, V->getName() + ".i0");
     }
     if (I != 0)
-      CV[I] = Builder.CreateConstGEP1_32(nullptr, CV[0], I,
+      CV[I] = Builder.CreateConstGEP1_32(ElTy, CV[0], I,
                                          V->getName() + ".i" + Twine(I));
   } else {
     // Search through a chain of InsertElementInsts looking for element I.
@@ -379,7 +377,8 @@ bool ScalarizerVisitor::canTransferMetadata(unsigned Tag) {
           || Tag == LLVMContext::MD_invariant_load
           || Tag == LLVMContext::MD_alias_scope
           || Tag == LLVMContext::MD_noalias
-          || Tag == ParallelLoopAccessMDKind);
+          || Tag == ParallelLoopAccessMDKind
+          || Tag == LLVMContext::MD_access_group);
 }
 
 // Transfer metadata from Op to the instructions in CV if it is known
@@ -743,7 +742,8 @@ bool ScalarizerVisitor::visitLoadInst(LoadInst &LI) {
   Res.resize(NumElems);
 
   for (unsigned I = 0; I < NumElems; ++I)
-    Res[I] = Builder.CreateAlignedLoad(Ptr[I], Layout.getElemAlign(I),
+    Res[I] = Builder.CreateAlignedLoad(Layout.VecTy->getElementType(), Ptr[I],
+                                       Layout.getElemAlign(I),
                                        LI.getName() + ".i" + Twine(I));
   gather(&LI, Res);
   return true;
diff --git a/lib/Transforms/Scalar/SeparateConstOffsetFromGEP.cpp b/lib/Transforms/Scalar/SeparateConstOffsetFromGEP.cpp
index 967f4a42a8fb79c410ce7fbba33d352b91e65d34..f6a12fb13142225ead13eab73eb813667adda4f1 100644
--- a/lib/Transforms/Scalar/SeparateConstOffsetFromGEP.cpp
+++ b/lib/Transforms/Scalar/SeparateConstOffsetFromGEP.cpp
@@ -1,9 +1,8 @@
 //===- SeparateConstOffsetFromGEP.cpp -------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp b/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp
index abec9183f0fe13ae92c6e016dade5d2968966f10..db7da8edd51bd5dada117349d00a22b37f9ae052 100644
--- a/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp
+++ b/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp
@@ -1,9 +1,8 @@
 ///===- SimpleLoopUnswitch.cpp - Hoist loop-invariant control flow ---------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -762,7 +761,7 @@ static bool unswitchTrivialSwitch(Loop &L, SwitchInst &SI, DominatorTree &DT,
         continue;
       }
       CommonSuccBB->removePredecessor(BB,
-                                      /*DontDeleteUselessPHIs*/ true);
+                                      /*KeepOneInputPHIs*/ true);
     }
     // Now nuke the switch and replace it with a direct branch.
     SI.eraseFromParent();
@@ -848,6 +847,10 @@ static bool unswitchAllTrivialConditions(Loop &L, DominatorTree &DT,
     // Check if there are any side-effecting instructions (e.g. stores, calls,
     // volatile loads) in the part of the loop that the code *would* execute
     // without unswitching.
+    if (MSSAU) // Possible early exit with MSSA
+      if (auto *Defs = MSSAU->getMemorySSA()->getBlockDefs(CurrentBB))
+        if (!isa<MemoryPhi>(*Defs->begin()) || (++Defs->begin() != Defs->end()))
+          return Changed;
     if (llvm::any_of(*CurrentBB,
                      [](Instruction &I) { return I.mayHaveSideEffects(); }))
       return Changed;
@@ -1066,7 +1069,7 @@ static BasicBlock *buildClonedLoopBlocks(
       continue;
 
     ClonedSuccBB->removePredecessor(ClonedParentBB,
-                                    /*DontDeleteUselessPHIs*/ true);
+                                    /*KeepOneInputPHIs*/ true);
   }
 
   // Replace the cloned branch with an unconditional branch to the cloned
@@ -2075,7 +2078,7 @@ static void unswitchNontrivialInvariants(
              "Only one possible unswitched block for a branch!");
       BasicBlock *UnswitchedSuccBB = *UnswitchedSuccBBs.begin();
       UnswitchedSuccBB->removePredecessor(ParentBB,
-                                          /*DontDeleteUselessPHIs*/ true);
+                                          /*KeepOneInputPHIs*/ true);
       DTUpdates.push_back({DominatorTree::Delete, ParentBB, UnswitchedSuccBB});
     } else {
       // Note that we actually want to remove the parent block as a predecessor
@@ -2090,7 +2093,7 @@ static void unswitchNontrivialInvariants(
       for (auto &Case : NewSI->cases())
         Case.getCaseSuccessor()->removePredecessor(
             ParentBB,
-            /*DontDeleteUselessPHIs*/ true);
+            /*KeepOneInputPHIs*/ true);
 
       // We need to use the set to populate domtree updates as even when there
       // are multiple cases pointing at the same successor we only want to
@@ -2281,7 +2284,10 @@ static void unswitchNontrivialInvariants(
   if (MSSAU && VerifyMemorySSA)
     MSSAU->getMemorySSA()->verifyMemorySSA();
 
-  ++NumBranches;
+  if (BI)
+    ++NumBranches;
+  else
+    ++NumSwitches;
 }
 
 /// Recursively compute the cost of a dominator subtree based on the per-block
diff --git a/lib/Transforms/Scalar/SimplifyCFGPass.cpp b/lib/Transforms/Scalar/SimplifyCFGPass.cpp
index b7b1db76b492379589b1880672aee22d0af1dbaa..4544975a4887b84c7b5f79b9e98dcf9beb0d6051 100644
--- a/lib/Transforms/Scalar/SimplifyCFGPass.cpp
+++ b/lib/Transforms/Scalar/SimplifyCFGPass.cpp
@@ -1,9 +1,8 @@
 //===- SimplifyCFGPass.cpp - CFG Simplification Pass ----------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Transforms/Scalar/Sink.cpp b/lib/Transforms/Scalar/Sink.cpp
index d1cdfabb0cc411326ad998d314cbe7777793d353..90f3a2aa46e161bb3d4f825dc38bb7ec321662a7 100644
--- a/lib/Transforms/Scalar/Sink.cpp
+++ b/lib/Transforms/Scalar/Sink.cpp
@@ -1,9 +1,8 @@
 //===-- Sink.cpp - Code Sinking -------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -76,14 +75,14 @@ static bool isSafeToMove(Instruction *Inst, AliasAnalysis &AA,
       Inst->mayThrow())
     return false;
 
-  if (auto CS = CallSite(Inst)) {
+  if (auto *Call = dyn_cast<CallBase>(Inst)) {
     // Convergent operations cannot be made control-dependent on additional
     // values.
-    if (CS.hasFnAttr(Attribute::Convergent))
+    if (Call->hasFnAttr(Attribute::Convergent))
       return false;
 
     for (Instruction *S : Stores)
-      if (isModSet(AA.getModRefInfo(S, CS)))
+      if (isModSet(AA.getModRefInfo(S, Call)))
         return false;
   }
 
diff --git a/lib/Transforms/Scalar/SpeculateAroundPHIs.cpp b/lib/Transforms/Scalar/SpeculateAroundPHIs.cpp
index c0f75ddddbe0f556ad7fe2b9d460dc11586def51..51e97cf1c782ce6b3a589273b70646053597a237 100644
--- a/lib/Transforms/Scalar/SpeculateAroundPHIs.cpp
+++ b/lib/Transforms/Scalar/SpeculateAroundPHIs.cpp
@@ -1,9 +1,8 @@
 //===- SpeculateAroundPHIs.cpp --------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Transforms/Scalar/SpeculativeExecution.cpp b/lib/Transforms/Scalar/SpeculativeExecution.cpp
index f5e1dd6ed8504b64ae87e854986cbdcfa46ddbe1..79d2a52b3932560f4eab34bf8c51b03537111112 100644
--- a/lib/Transforms/Scalar/SpeculativeExecution.cpp
+++ b/lib/Transforms/Scalar/SpeculativeExecution.cpp
@@ -1,9 +1,8 @@
 //===- SpeculativeExecution.cpp ---------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Transforms/Scalar/StraightLineStrengthReduce.cpp b/lib/Transforms/Scalar/StraightLineStrengthReduce.cpp
index b5089b006bdd1131a32ae03c1ddf088488d8ffd6..a58c32cc58943f3fc6071183eb18b7080cded890 100644
--- a/lib/Transforms/Scalar/StraightLineStrengthReduce.cpp
+++ b/lib/Transforms/Scalar/StraightLineStrengthReduce.cpp
@@ -1,9 +1,8 @@
 //===- StraightLineStrengthReduce.cpp - -----------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -683,9 +682,13 @@ void StraightLineStrengthReduce::rewriteCandidateWithBasis(
         // Canonicalize bump to pointer size.
         Bump = Builder.CreateSExtOrTrunc(Bump, IntPtrTy);
         if (InBounds)
-          Reduced = Builder.CreateInBoundsGEP(nullptr, Basis.Ins, Bump);
+          Reduced = Builder.CreateInBoundsGEP(
+              cast<GetElementPtrInst>(Basis.Ins)->getResultElementType(),
+              Basis.Ins, Bump);
         else
-          Reduced = Builder.CreateGEP(nullptr, Basis.Ins, Bump);
+          Reduced = Builder.CreateGEP(
+              cast<GetElementPtrInst>(Basis.Ins)->getResultElementType(),
+              Basis.Ins, Bump);
       }
       break;
     }
diff --git a/lib/Transforms/Scalar/StructurizeCFG.cpp b/lib/Transforms/Scalar/StructurizeCFG.cpp
index 0db762d846f225add2026a9195db667036a2d2c0..b3ef31cea0a8814ac7f87a61f942f11d646e2eb5 100644
--- a/lib/Transforms/Scalar/StructurizeCFG.cpp
+++ b/lib/Transforms/Scalar/StructurizeCFG.cpp
@@ -1,9 +1,8 @@
 //===- StructurizeCFG.cpp -------------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Transforms/Scalar/TailRecursionElimination.cpp b/lib/Transforms/Scalar/TailRecursionElimination.cpp
index 0f6db21f73b60eb5ea57e7b457b79054e8139df9..8238fad1190d7b56e97385c9c6db6624014c127c 100644
--- a/lib/Transforms/Scalar/TailRecursionElimination.cpp
+++ b/lib/Transforms/Scalar/TailRecursionElimination.cpp
@@ -1,9 +1,8 @@
 //===- TailRecursionElimination.cpp - Eliminate Tail Calls ----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -56,6 +55,7 @@
 #include "llvm/ADT/Statistic.h"
 #include "llvm/Analysis/CFG.h"
 #include "llvm/Analysis/CaptureTracking.h"
+#include "llvm/Analysis/DomTreeUpdater.h"
 #include "llvm/Analysis/GlobalsModRef.h"
 #include "llvm/Analysis/InlineCost.h"
 #include "llvm/Analysis/InstructionSimplify.h"
@@ -69,7 +69,6 @@
 #include "llvm/IR/DataLayout.h"
 #include "llvm/IR/DerivedTypes.h"
 #include "llvm/IR/DiagnosticInfo.h"
-#include "llvm/IR/DomTreeUpdater.h"
 #include "llvm/IR/Dominators.h"
 #include "llvm/IR/Function.h"
 #include "llvm/IR/InstIterator.h"
diff --git a/lib/Transforms/Scalar/WarnMissedTransforms.cpp b/lib/Transforms/Scalar/WarnMissedTransforms.cpp
index 80f761e537747462adcd7bdbf477a8e5c035c957..73687e1e5dfdc315eefd603daa1008efcb7732c1 100644
--- a/lib/Transforms/Scalar/WarnMissedTransforms.cpp
+++ b/lib/Transforms/Scalar/WarnMissedTransforms.cpp
@@ -1,9 +1,8 @@
 //===- LoopTransformWarning.cpp -  ----------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Transforms/Utils/ASanStackFrameLayout.cpp b/lib/Transforms/Utils/ASanStackFrameLayout.cpp
index 364878dc588dd5136f84a54bc06d1a7ceed94ad4..a7de158e65ce1d91bf8476c0a751e4b9e222b70f 100644
--- a/lib/Transforms/Utils/ASanStackFrameLayout.cpp
+++ b/lib/Transforms/Utils/ASanStackFrameLayout.cpp
@@ -1,9 +1,8 @@
 //===-- ASanStackFrameLayout.cpp - helper for AddressSanitizer ------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Transforms/Utils/AddDiscriminators.cpp b/lib/Transforms/Utils/AddDiscriminators.cpp
index e3ef423622231cffeec8eddfed3e2702ab2492c3..ee0973002c47de5f8093cde61923292a623cd80f 100644
--- a/lib/Transforms/Utils/AddDiscriminators.cpp
+++ b/lib/Transforms/Utils/AddDiscriminators.cpp
@@ -1,9 +1,8 @@
 //===- AddDiscriminators.cpp - Insert DWARF path discriminators -----------===//
 //
-//                      The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -209,10 +208,18 @@ static bool addDiscriminators(Function &F) {
       // Only the lowest 7 bits are used to represent a discriminator to fit
       // it in 1 byte ULEB128 representation.
       unsigned Discriminator = R.second ? ++LDM[L] : LDM[L];
-      I.setDebugLoc(DIL->setBaseDiscriminator(Discriminator));
-      LLVM_DEBUG(dbgs() << DIL->getFilename() << ":" << DIL->getLine() << ":"
-                        << DIL->getColumn() << ":" << Discriminator << " " << I
-                        << "\n");
+      auto NewDIL = DIL->cloneWithBaseDiscriminator(Discriminator);
+      if (!NewDIL) {
+        LLVM_DEBUG(dbgs() << "Could not encode discriminator: "
+                          << DIL->getFilename() << ":" << DIL->getLine() << ":"
+                          << DIL->getColumn() << ":" << Discriminator << " "
+                          << I << "\n");
+      } else {
+        I.setDebugLoc(NewDIL.getValue());
+        LLVM_DEBUG(dbgs() << DIL->getFilename() << ":" << DIL->getLine() << ":"
+                   << DIL->getColumn() << ":" << Discriminator << " " << I
+                   << "\n");
+      }
       Changed = true;
     }
   }
@@ -224,23 +231,31 @@ static bool addDiscriminators(Function &F) {
   for (BasicBlock &B : F) {
     LocationSet CallLocations;
     for (auto &I : B.getInstList()) {
-      CallInst *Current = dyn_cast<CallInst>(&I);
       // We bypass intrinsic calls for the following two reasons:
       //  1) We want to avoid a non-deterministic assigment of
       //     discriminators.
       //  2) We want to minimize the number of base discriminators used.
-      if (!Current || isa<IntrinsicInst>(&I))
+      if (!isa<InvokeInst>(I) && (!isa<CallInst>(I) || isa<IntrinsicInst>(I)))  
         continue;
 
-      DILocation *CurrentDIL = Current->getDebugLoc();
+      DILocation *CurrentDIL = I.getDebugLoc();
       if (!CurrentDIL)
         continue;
       Location L =
           std::make_pair(CurrentDIL->getFilename(), CurrentDIL->getLine());
       if (!CallLocations.insert(L).second) {
         unsigned Discriminator = ++LDM[L];
-        Current->setDebugLoc(CurrentDIL->setBaseDiscriminator(Discriminator));
-        Changed = true;
+        auto NewDIL = CurrentDIL->cloneWithBaseDiscriminator(Discriminator);
+        if (!NewDIL) {
+          LLVM_DEBUG(dbgs()
+                     << "Could not encode discriminator: "
+                     << CurrentDIL->getFilename() << ":"
+                     << CurrentDIL->getLine() << ":" << CurrentDIL->getColumn()
+                     << ":" << Discriminator << " " << I << "\n");
+        } else {
+          I.setDebugLoc(NewDIL.getValue());
+          Changed = true;
+        }
       }
     }
   }
diff --git a/lib/Transforms/Utils/BasicBlockUtils.cpp b/lib/Transforms/Utils/BasicBlockUtils.cpp
index 11a0114150ffb29a8964d7ed97440d8443425919..5ca9222cf16b92685e105e6578a41a8bcc463304 100644
--- a/lib/Transforms/Utils/BasicBlockUtils.cpp
+++ b/lib/Transforms/Utils/BasicBlockUtils.cpp
@@ -1,9 +1,8 @@
 //===- BasicBlockUtils.cpp - BasicBlock Utilities --------------------------==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -18,6 +17,7 @@
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/Twine.h"
 #include "llvm/Analysis/CFG.h"
+#include "llvm/Analysis/DomTreeUpdater.h"
 #include "llvm/Analysis/LoopInfo.h"
 #include "llvm/Analysis/MemoryDependenceAnalysis.h"
 #include "llvm/Analysis/MemorySSAUpdater.h"
@@ -26,7 +26,6 @@
 #include "llvm/IR/CFG.h"
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/DebugInfoMetadata.h"
-#include "llvm/IR/DomTreeUpdater.h"
 #include "llvm/IR/Dominators.h"
 #include "llvm/IR/Function.h"
 #include "llvm/IR/InstrTypes.h"
@@ -48,47 +47,67 @@
 
 using namespace llvm;
 
-void llvm::DeleteDeadBlock(BasicBlock *BB, DomTreeUpdater *DTU) {
-  assert((pred_begin(BB) == pred_end(BB) ||
-         // Can delete self loop.
-         BB->getSinglePredecessor() == BB) && "Block is not dead!");
-  Instruction *BBTerm = BB->getTerminator();
-  std::vector<DominatorTree::UpdateType> Updates;
+void llvm::DetatchDeadBlocks(
+    ArrayRef<BasicBlock *> BBs,
+    SmallVectorImpl<DominatorTree::UpdateType> *Updates,
+    bool KeepOneInputPHIs) {
+  for (auto *BB : BBs) {
+    // Loop through all of our successors and make sure they know that one
+    // of their predecessors is going away.
+    SmallPtrSet<BasicBlock *, 4> UniqueSuccessors;
+    for (BasicBlock *Succ : successors(BB)) {
+      Succ->removePredecessor(BB, KeepOneInputPHIs);
+      if (Updates && UniqueSuccessors.insert(Succ).second)
+        Updates->push_back({DominatorTree::Delete, BB, Succ});
+    }
 
-  // Loop through all of our successors and make sure they know that one
-  // of their predecessors is going away.
-  if (DTU)
-    Updates.reserve(BBTerm->getNumSuccessors());
-  for (BasicBlock *Succ : successors(BBTerm)) {
-    Succ->removePredecessor(BB);
-    if (DTU)
-      Updates.push_back({DominatorTree::Delete, BB, Succ});
+    // Zap all the instructions in the block.
+    while (!BB->empty()) {
+      Instruction &I = BB->back();
+      // If this instruction is used, replace uses with an arbitrary value.
+      // Because control flow can't get here, we don't care what we replace the
+      // value with.  Note that since this block is unreachable, and all values
+      // contained within it must dominate their uses, that all uses will
+      // eventually be removed (they are themselves dead).
+      if (!I.use_empty())
+        I.replaceAllUsesWith(UndefValue::get(I.getType()));
+      BB->getInstList().pop_back();
+    }
+    new UnreachableInst(BB->getContext(), BB);
+    assert(BB->getInstList().size() == 1 &&
+           isa<UnreachableInst>(BB->getTerminator()) &&
+           "The successor list of BB isn't empty before "
+           "applying corresponding DTU updates.");
   }
+}
 
-  // Zap all the instructions in the block.
-  while (!BB->empty()) {
-    Instruction &I = BB->back();
-    // If this instruction is used, replace uses with an arbitrary value.
-    // Because control flow can't get here, we don't care what we replace the
-    // value with.  Note that since this block is unreachable, and all values
-    // contained within it must dominate their uses, that all uses will
-    // eventually be removed (they are themselves dead).
-    if (!I.use_empty())
-      I.replaceAllUsesWith(UndefValue::get(I.getType()));
-    BB->getInstList().pop_back();
-  }
-  new UnreachableInst(BB->getContext(), BB);
-  assert(BB->getInstList().size() == 1 &&
-         isa<UnreachableInst>(BB->getTerminator()) &&
-         "The successor list of BB isn't empty before "
-         "applying corresponding DTU updates.");
+void llvm::DeleteDeadBlock(BasicBlock *BB, DomTreeUpdater *DTU,
+                           bool KeepOneInputPHIs) {
+  DeleteDeadBlocks({BB}, DTU, KeepOneInputPHIs);
+}
 
-  if (DTU) {
+void llvm::DeleteDeadBlocks(ArrayRef <BasicBlock *> BBs, DomTreeUpdater *DTU,
+                            bool KeepOneInputPHIs) {
+#ifndef NDEBUG
+  // Make sure that all predecessors of each dead block is also dead.
+  SmallPtrSet<BasicBlock *, 4> Dead(BBs.begin(), BBs.end());
+  assert(Dead.size() == BBs.size() && "Duplicating blocks?");
+  for (auto *BB : Dead)
+    for (BasicBlock *Pred : predecessors(BB))
+      assert(Dead.count(Pred) && "All predecessors must be dead!");
+#endif
+
+  SmallVector<DominatorTree::UpdateType, 4> Updates;
+  DetatchDeadBlocks(BBs, DTU ? &Updates : nullptr, KeepOneInputPHIs);
+
+  if (DTU)
     DTU->applyUpdates(Updates, /*ForceRemoveDuplicates*/ true);
-    DTU->deleteBB(BB);
-  } else {
-    BB->eraseFromParent(); // Zap the block!
-  }
+
+  for (BasicBlock *BB : BBs)
+    if (DTU)
+      DTU->deleteBB(BB);
+    else
+      BB->eraseFromParent();
 }
 
 void llvm::FoldSingleEntryPHINodes(BasicBlock *BB,
@@ -532,6 +551,8 @@ BasicBlock *llvm::SplitBlockPredecessors(BasicBlock *BB,
     // all BlockAddress uses would need to be updated.
     assert(!isa<IndirectBrInst>(Preds[i]->getTerminator()) &&
            "Cannot split an edge from an IndirectBrInst");
+    assert(!isa<CallBrInst>(Preds[i]->getTerminator()) &&
+           "Cannot split an edge from a CallBrInst");
     Preds[i]->getTerminator()->replaceUsesOfWith(BB, NewBB);
   }
 
diff --git a/lib/Transforms/Utils/BreakCriticalEdges.cpp b/lib/Transforms/Utils/BreakCriticalEdges.cpp
index fafc9aaba5c9cc4b095e9d7ca7538ac3aae8e9ae..2944c372425e684160f45623e43f1850fbd7768b 100644
--- a/lib/Transforms/Utils/BreakCriticalEdges.cpp
+++ b/lib/Transforms/Utils/BreakCriticalEdges.cpp
@@ -1,9 +1,8 @@
 //===- BreakCriticalEdges.cpp - Critical Edge Elimination Pass ------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -145,6 +144,10 @@ llvm::SplitCriticalEdge(Instruction *TI, unsigned SuccNum,
   // it in this generic function.
   if (DestBB->isEHPad()) return nullptr;
 
+  // Don't split the non-fallthrough edge from a callbr.
+  if (isa<CallBrInst>(TI) && SuccNum > 0)
+    return nullptr;
+
   // Create a new basic block, linking it into the CFG.
   BasicBlock *NewBB = BasicBlock::Create(TI->getContext(),
                       TIBB->getName() + "." + DestBB->getName() + "_crit_edge");
@@ -189,7 +192,7 @@ llvm::SplitCriticalEdge(Instruction *TI, unsigned SuccNum,
       if (TI->getSuccessor(i) != DestBB) continue;
 
       // Remove an entry for TIBB from DestBB phi nodes.
-      DestBB->removePredecessor(TIBB, Options.DontDeleteUselessPHIs);
+      DestBB->removePredecessor(TIBB, Options.KeepOneInputPHIs);
 
       // We found another edge to DestBB, go to NewBB instead.
       TI->setSuccessor(i, NewBB);
diff --git a/lib/Transforms/Utils/BuildLibCalls.cpp b/lib/Transforms/Utils/BuildLibCalls.cpp
index 3466dedd3236e1b1cc1d37d40f3aa69258996b14..568850876a20a9b576eeda9d292b12e5db5e2674 100644
--- a/lib/Transforms/Utils/BuildLibCalls.cpp
+++ b/lib/Transforms/Utils/BuildLibCalls.cpp
@@ -1,9 +1,8 @@
 //===- BuildLibCalls.cpp - Utility builder for libcalls -------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -798,11 +797,12 @@ Value *llvm::emitStrLen(Value *Ptr, IRBuilder<> &B, const DataLayout &DL,
   Module *M = B.GetInsertBlock()->getModule();
   StringRef StrlenName = TLI->getName(LibFunc_strlen);
   LLVMContext &Context = B.GetInsertBlock()->getContext();
-  Constant *StrLen = M->getOrInsertFunction(StrlenName, DL.getIntPtrType(Context),
-                                            B.getInt8PtrTy());
+  FunctionCallee StrLen = M->getOrInsertFunction(
+      StrlenName, DL.getIntPtrType(Context), B.getInt8PtrTy());
   inferLibFuncAttributes(M, StrlenName, *TLI);
   CallInst *CI = B.CreateCall(StrLen, castToCStr(Ptr, B), StrlenName);
-  if (const Function *F = dyn_cast<Function>(StrLen->stripPointerCasts()))
+  if (const Function *F =
+          dyn_cast<Function>(StrLen.getCallee()->stripPointerCasts()))
     CI->setCallingConv(F->getCallingConv());
 
   return CI;
@@ -817,12 +817,13 @@ Value *llvm::emitStrChr(Value *Ptr, char C, IRBuilder<> &B,
   StringRef StrChrName = TLI->getName(LibFunc_strchr);
   Type *I8Ptr = B.getInt8PtrTy();
   Type *I32Ty = B.getInt32Ty();
-  Constant *StrChr =
+  FunctionCallee StrChr =
       M->getOrInsertFunction(StrChrName, I8Ptr, I8Ptr, I32Ty);
   inferLibFuncAttributes(M, StrChrName, *TLI);
   CallInst *CI = B.CreateCall(
       StrChr, {castToCStr(Ptr, B), ConstantInt::get(I32Ty, C)}, StrChrName);
-  if (const Function *F = dyn_cast<Function>(StrChr->stripPointerCasts()))
+  if (const Function *F =
+          dyn_cast<Function>(StrChr.getCallee()->stripPointerCasts()))
     CI->setCallingConv(F->getCallingConv());
   return CI;
 }
@@ -835,14 +836,15 @@ Value *llvm::emitStrNCmp(Value *Ptr1, Value *Ptr2, Value *Len, IRBuilder<> &B,
   Module *M = B.GetInsertBlock()->getModule();
   StringRef StrNCmpName = TLI->getName(LibFunc_strncmp);
   LLVMContext &Context = B.GetInsertBlock()->getContext();
-  Value *StrNCmp = M->getOrInsertFunction(StrNCmpName, B.getInt32Ty(),
-                                          B.getInt8PtrTy(), B.getInt8PtrTy(),
-                                          DL.getIntPtrType(Context));
+  FunctionCallee StrNCmp =
+      M->getOrInsertFunction(StrNCmpName, B.getInt32Ty(), B.getInt8PtrTy(),
+                             B.getInt8PtrTy(), DL.getIntPtrType(Context));
   inferLibFuncAttributes(M, StrNCmpName, *TLI);
   CallInst *CI = B.CreateCall(
       StrNCmp, {castToCStr(Ptr1, B), castToCStr(Ptr2, B), Len}, StrNCmpName);
 
-  if (const Function *F = dyn_cast<Function>(StrNCmp->stripPointerCasts()))
+  if (const Function *F =
+          dyn_cast<Function>(StrNCmp.getCallee()->stripPointerCasts()))
     CI->setCallingConv(F->getCallingConv());
 
   return CI;
@@ -855,11 +857,12 @@ Value *llvm::emitStrCpy(Value *Dst, Value *Src, IRBuilder<> &B,
 
   Module *M = B.GetInsertBlock()->getModule();
   Type *I8Ptr = B.getInt8PtrTy();
-  Value *StrCpy = M->getOrInsertFunction(Name, I8Ptr, I8Ptr, I8Ptr);
+  FunctionCallee StrCpy = M->getOrInsertFunction(Name, I8Ptr, I8Ptr, I8Ptr);
   inferLibFuncAttributes(M, Name, *TLI);
   CallInst *CI =
       B.CreateCall(StrCpy, {castToCStr(Dst, B), castToCStr(Src, B)}, Name);
-  if (const Function *F = dyn_cast<Function>(StrCpy->stripPointerCasts()))
+  if (const Function *F =
+          dyn_cast<Function>(StrCpy.getCallee()->stripPointerCasts()))
     CI->setCallingConv(F->getCallingConv());
   return CI;
 }
@@ -871,12 +874,13 @@ Value *llvm::emitStrNCpy(Value *Dst, Value *Src, Value *Len, IRBuilder<> &B,
 
   Module *M = B.GetInsertBlock()->getModule();
   Type *I8Ptr = B.getInt8PtrTy();
-  Value *StrNCpy = M->getOrInsertFunction(Name, I8Ptr, I8Ptr, I8Ptr,
-                                          Len->getType());
+  FunctionCallee StrNCpy =
+      M->getOrInsertFunction(Name, I8Ptr, I8Ptr, I8Ptr, Len->getType());
   inferLibFuncAttributes(M, Name, *TLI);
   CallInst *CI = B.CreateCall(
       StrNCpy, {castToCStr(Dst, B), castToCStr(Src, B), Len}, Name);
-  if (const Function *F = dyn_cast<Function>(StrNCpy->stripPointerCasts()))
+  if (const Function *F =
+          dyn_cast<Function>(StrNCpy.getCallee()->stripPointerCasts()))
     CI->setCallingConv(F->getCallingConv());
   return CI;
 }
@@ -892,14 +896,15 @@ Value *llvm::emitMemCpyChk(Value *Dst, Value *Src, Value *Len, Value *ObjSize,
   AS = AttributeList::get(M->getContext(), AttributeList::FunctionIndex,
                           Attribute::NoUnwind);
   LLVMContext &Context = B.GetInsertBlock()->getContext();
-  Value *MemCpy = M->getOrInsertFunction(
+  FunctionCallee MemCpy = M->getOrInsertFunction(
       "__memcpy_chk", AttributeList::get(M->getContext(), AS), B.getInt8PtrTy(),
       B.getInt8PtrTy(), B.getInt8PtrTy(), DL.getIntPtrType(Context),
       DL.getIntPtrType(Context));
   Dst = castToCStr(Dst, B);
   Src = castToCStr(Src, B);
   CallInst *CI = B.CreateCall(MemCpy, {Dst, Src, Len, ObjSize});
-  if (const Function *F = dyn_cast<Function>(MemCpy->stripPointerCasts()))
+  if (const Function *F =
+          dyn_cast<Function>(MemCpy.getCallee()->stripPointerCasts()))
     CI->setCallingConv(F->getCallingConv());
   return CI;
 }
@@ -912,13 +917,14 @@ Value *llvm::emitMemChr(Value *Ptr, Value *Val, Value *Len, IRBuilder<> &B,
   Module *M = B.GetInsertBlock()->getModule();
   StringRef MemChrName = TLI->getName(LibFunc_memchr);
   LLVMContext &Context = B.GetInsertBlock()->getContext();
-  Value *MemChr = M->getOrInsertFunction(MemChrName, B.getInt8PtrTy(),
-                                         B.getInt8PtrTy(), B.getInt32Ty(),
-                                         DL.getIntPtrType(Context));
+  FunctionCallee MemChr =
+      M->getOrInsertFunction(MemChrName, B.getInt8PtrTy(), B.getInt8PtrTy(),
+                             B.getInt32Ty(), DL.getIntPtrType(Context));
   inferLibFuncAttributes(M, MemChrName, *TLI);
   CallInst *CI = B.CreateCall(MemChr, {castToCStr(Ptr, B), Val, Len}, MemChrName);
 
-  if (const Function *F = dyn_cast<Function>(MemChr->stripPointerCasts()))
+  if (const Function *F =
+          dyn_cast<Function>(MemChr.getCallee()->stripPointerCasts()))
     CI->setCallingConv(F->getCallingConv());
 
   return CI;
@@ -932,14 +938,15 @@ Value *llvm::emitMemCmp(Value *Ptr1, Value *Ptr2, Value *Len, IRBuilder<> &B,
   Module *M = B.GetInsertBlock()->getModule();
   StringRef MemCmpName = TLI->getName(LibFunc_memcmp);
   LLVMContext &Context = B.GetInsertBlock()->getContext();
-  Value *MemCmp = M->getOrInsertFunction(MemCmpName, B.getInt32Ty(),
-                                         B.getInt8PtrTy(), B.getInt8PtrTy(),
-                                         DL.getIntPtrType(Context));
+  FunctionCallee MemCmp =
+      M->getOrInsertFunction(MemCmpName, B.getInt32Ty(), B.getInt8PtrTy(),
+                             B.getInt8PtrTy(), DL.getIntPtrType(Context));
   inferLibFuncAttributes(M, MemCmpName, *TLI);
   CallInst *CI = B.CreateCall(
       MemCmp, {castToCStr(Ptr1, B), castToCStr(Ptr2, B), Len}, MemCmpName);
 
-  if (const Function *F = dyn_cast<Function>(MemCmp->stripPointerCasts()))
+  if (const Function *F =
+          dyn_cast<Function>(MemCmp.getCallee()->stripPointerCasts()))
     CI->setCallingConv(F->getCallingConv());
 
   return CI;
@@ -966,8 +973,8 @@ static Value *emitUnaryFloatFnCallHelper(Value *Op, StringRef Name,
   assert((Name != "") && "Must specify Name to emitUnaryFloatFnCall");
 
   Module *M = B.GetInsertBlock()->getModule();
-  Value *Callee = M->getOrInsertFunction(Name, Op->getType(),
-                                         Op->getType());
+  FunctionCallee Callee =
+      M->getOrInsertFunction(Name, Op->getType(), Op->getType());
   CallInst *CI = B.CreateCall(Callee, Op, Name);
 
   // The incoming attribute set may have come from a speculatable intrinsic, but
@@ -976,7 +983,8 @@ static Value *emitUnaryFloatFnCallHelper(Value *Op, StringRef Name,
   CI->setAttributes(Attrs.removeAttribute(B.getContext(),
                                           AttributeList::FunctionIndex,
                                           Attribute::Speculatable));
-  if (const Function *F = dyn_cast<Function>(Callee->stripPointerCasts()))
+  if (const Function *F =
+          dyn_cast<Function>(Callee.getCallee()->stripPointerCasts()))
     CI->setCallingConv(F->getCallingConv());
 
   return CI;
@@ -1009,11 +1017,12 @@ Value *llvm::emitBinaryFloatFnCall(Value *Op1, Value *Op2, StringRef Name,
   appendTypeSuffix(Op1, Name, NameBuffer);
 
   Module *M = B.GetInsertBlock()->getModule();
-  Value *Callee = M->getOrInsertFunction(Name, Op1->getType(), Op1->getType(),
-                                         Op2->getType());
+  FunctionCallee Callee = M->getOrInsertFunction(
+      Name, Op1->getType(), Op1->getType(), Op2->getType());
   CallInst *CI = B.CreateCall(Callee, {Op1, Op2}, Name);
   CI->setAttributes(Attrs);
-  if (const Function *F = dyn_cast<Function>(Callee->stripPointerCasts()))
+  if (const Function *F =
+          dyn_cast<Function>(Callee.getCallee()->stripPointerCasts()))
     CI->setCallingConv(F->getCallingConv());
 
   return CI;
@@ -1026,7 +1035,8 @@ Value *llvm::emitPutChar(Value *Char, IRBuilder<> &B,
 
   Module *M = B.GetInsertBlock()->getModule();
   StringRef PutCharName = TLI->getName(LibFunc_putchar);
-  Value *PutChar = M->getOrInsertFunction(PutCharName, B.getInt32Ty(), B.getInt32Ty());
+  FunctionCallee PutChar =
+      M->getOrInsertFunction(PutCharName, B.getInt32Ty(), B.getInt32Ty());
   inferLibFuncAttributes(M, PutCharName, *TLI);
   CallInst *CI = B.CreateCall(PutChar,
                               B.CreateIntCast(Char,
@@ -1035,7 +1045,8 @@ Value *llvm::emitPutChar(Value *Char, IRBuilder<> &B,
                               "chari"),
                               PutCharName);
 
-  if (const Function *F = dyn_cast<Function>(PutChar->stripPointerCasts()))
+  if (const Function *F =
+          dyn_cast<Function>(PutChar.getCallee()->stripPointerCasts()))
     CI->setCallingConv(F->getCallingConv());
   return CI;
 }
@@ -1047,11 +1058,12 @@ Value *llvm::emitPutS(Value *Str, IRBuilder<> &B,
 
   Module *M = B.GetInsertBlock()->getModule();
   StringRef PutsName = TLI->getName(LibFunc_puts);
-  Value *PutS =
+  FunctionCallee PutS =
       M->getOrInsertFunction(PutsName, B.getInt32Ty(), B.getInt8PtrTy());
   inferLibFuncAttributes(M, PutsName, *TLI);
   CallInst *CI = B.CreateCall(PutS, castToCStr(Str, B), PutsName);
-  if (const Function *F = dyn_cast<Function>(PutS->stripPointerCasts()))
+  if (const Function *F =
+          dyn_cast<Function>(PutS.getCallee()->stripPointerCasts()))
     CI->setCallingConv(F->getCallingConv());
   return CI;
 }
@@ -1063,15 +1075,16 @@ Value *llvm::emitFPutC(Value *Char, Value *File, IRBuilder<> &B,
 
   Module *M = B.GetInsertBlock()->getModule();
   StringRef FPutcName = TLI->getName(LibFunc_fputc);
-  Constant *F = M->getOrInsertFunction(FPutcName, B.getInt32Ty(), B.getInt32Ty(),
-                                       File->getType());
+  FunctionCallee F = M->getOrInsertFunction(FPutcName, B.getInt32Ty(),
+                                            B.getInt32Ty(), File->getType());
   if (File->getType()->isPointerTy())
     inferLibFuncAttributes(M, FPutcName, *TLI);
   Char = B.CreateIntCast(Char, B.getInt32Ty(), /*isSigned*/true,
                          "chari");
   CallInst *CI = B.CreateCall(F, {Char, File}, FPutcName);
 
-  if (const Function *Fn = dyn_cast<Function>(F->stripPointerCasts()))
+  if (const Function *Fn =
+          dyn_cast<Function>(F.getCallee()->stripPointerCasts()))
     CI->setCallingConv(Fn->getCallingConv());
   return CI;
 }
@@ -1083,14 +1096,15 @@ Value *llvm::emitFPutCUnlocked(Value *Char, Value *File, IRBuilder<> &B,
 
   Module *M = B.GetInsertBlock()->getModule();
   StringRef FPutcUnlockedName = TLI->getName(LibFunc_fputc_unlocked);
-  Constant *F = M->getOrInsertFunction(FPutcUnlockedName, B.getInt32Ty(),
-                                       B.getInt32Ty(), File->getType());
+  FunctionCallee F = M->getOrInsertFunction(FPutcUnlockedName, B.getInt32Ty(),
+                                            B.getInt32Ty(), File->getType());
   if (File->getType()->isPointerTy())
     inferLibFuncAttributes(M, FPutcUnlockedName, *TLI);
   Char = B.CreateIntCast(Char, B.getInt32Ty(), /*isSigned*/ true, "chari");
   CallInst *CI = B.CreateCall(F, {Char, File}, FPutcUnlockedName);
 
-  if (const Function *Fn = dyn_cast<Function>(F->stripPointerCasts()))
+  if (const Function *Fn =
+          dyn_cast<Function>(F.getCallee()->stripPointerCasts()))
     CI->setCallingConv(Fn->getCallingConv());
   return CI;
 }
@@ -1102,13 +1116,14 @@ Value *llvm::emitFPutS(Value *Str, Value *File, IRBuilder<> &B,
 
   Module *M = B.GetInsertBlock()->getModule();
   StringRef FPutsName = TLI->getName(LibFunc_fputs);
-  Constant *F = M->getOrInsertFunction(
-      FPutsName, B.getInt32Ty(), B.getInt8PtrTy(), File->getType());
+  FunctionCallee F = M->getOrInsertFunction(FPutsName, B.getInt32Ty(),
+                                            B.getInt8PtrTy(), File->getType());
   if (File->getType()->isPointerTy())
     inferLibFuncAttributes(M, FPutsName, *TLI);
   CallInst *CI = B.CreateCall(F, {castToCStr(Str, B), File}, FPutsName);
 
-  if (const Function *Fn = dyn_cast<Function>(F->stripPointerCasts()))
+  if (const Function *Fn =
+          dyn_cast<Function>(F.getCallee()->stripPointerCasts()))
     CI->setCallingConv(Fn->getCallingConv());
   return CI;
 }
@@ -1120,13 +1135,14 @@ Value *llvm::emitFPutSUnlocked(Value *Str, Value *File, IRBuilder<> &B,
 
   Module *M = B.GetInsertBlock()->getModule();
   StringRef FPutsUnlockedName = TLI->getName(LibFunc_fputs_unlocked);
-  Constant *F = M->getOrInsertFunction(FPutsUnlockedName, B.getInt32Ty(),
-                                       B.getInt8PtrTy(), File->getType());
+  FunctionCallee F = M->getOrInsertFunction(FPutsUnlockedName, B.getInt32Ty(),
+                                            B.getInt8PtrTy(), File->getType());
   if (File->getType()->isPointerTy())
     inferLibFuncAttributes(M, FPutsUnlockedName, *TLI);
   CallInst *CI = B.CreateCall(F, {castToCStr(Str, B), File}, FPutsUnlockedName);
 
-  if (const Function *Fn = dyn_cast<Function>(F->stripPointerCasts()))
+  if (const Function *Fn =
+          dyn_cast<Function>(F.getCallee()->stripPointerCasts()))
     CI->setCallingConv(Fn->getCallingConv());
   return CI;
 }
@@ -1139,7 +1155,7 @@ Value *llvm::emitFWrite(Value *Ptr, Value *Size, Value *File, IRBuilder<> &B,
   Module *M = B.GetInsertBlock()->getModule();
   LLVMContext &Context = B.GetInsertBlock()->getContext();
   StringRef FWriteName = TLI->getName(LibFunc_fwrite);
-  Constant *F = M->getOrInsertFunction(
+  FunctionCallee F = M->getOrInsertFunction(
       FWriteName, DL.getIntPtrType(Context), B.getInt8PtrTy(),
       DL.getIntPtrType(Context), DL.getIntPtrType(Context), File->getType());
 
@@ -1149,7 +1165,8 @@ Value *llvm::emitFWrite(Value *Ptr, Value *Size, Value *File, IRBuilder<> &B,
       B.CreateCall(F, {castToCStr(Ptr, B), Size,
                        ConstantInt::get(DL.getIntPtrType(Context), 1), File});
 
-  if (const Function *Fn = dyn_cast<Function>(F->stripPointerCasts()))
+  if (const Function *Fn =
+          dyn_cast<Function>(F.getCallee()->stripPointerCasts()))
     CI->setCallingConv(Fn->getCallingConv());
   return CI;
 }
@@ -1162,12 +1179,13 @@ Value *llvm::emitMalloc(Value *Num, IRBuilder<> &B, const DataLayout &DL,
   Module *M = B.GetInsertBlock()->getModule();
   StringRef MallocName = TLI->getName(LibFunc_malloc);
   LLVMContext &Context = B.GetInsertBlock()->getContext();
-  Value *Malloc = M->getOrInsertFunction(MallocName, B.getInt8PtrTy(),
-                                         DL.getIntPtrType(Context));
+  FunctionCallee Malloc = M->getOrInsertFunction(MallocName, B.getInt8PtrTy(),
+                                                 DL.getIntPtrType(Context));
   inferLibFuncAttributes(M, MallocName, *TLI);
   CallInst *CI = B.CreateCall(Malloc, Num, MallocName);
 
-  if (const Function *F = dyn_cast<Function>(Malloc->stripPointerCasts()))
+  if (const Function *F =
+          dyn_cast<Function>(Malloc.getCallee()->stripPointerCasts()))
     CI->setCallingConv(F->getCallingConv());
 
   return CI;
@@ -1182,12 +1200,13 @@ Value *llvm::emitCalloc(Value *Num, Value *Size, const AttributeList &Attrs,
   StringRef CallocName = TLI.getName(LibFunc_calloc);
   const DataLayout &DL = M->getDataLayout();
   IntegerType *PtrType = DL.getIntPtrType((B.GetInsertBlock()->getContext()));
-  Value *Calloc = M->getOrInsertFunction(CallocName, Attrs, B.getInt8PtrTy(),
-                                         PtrType, PtrType);
+  FunctionCallee Calloc = M->getOrInsertFunction(
+      CallocName, Attrs, B.getInt8PtrTy(), PtrType, PtrType);
   inferLibFuncAttributes(M, CallocName, TLI);
   CallInst *CI = B.CreateCall(Calloc, {Num, Size}, CallocName);
 
-  if (const auto *F = dyn_cast<Function>(Calloc->stripPointerCasts()))
+  if (const auto *F =
+          dyn_cast<Function>(Calloc.getCallee()->stripPointerCasts()))
     CI->setCallingConv(F->getCallingConv());
 
   return CI;
@@ -1202,7 +1221,7 @@ Value *llvm::emitFWriteUnlocked(Value *Ptr, Value *Size, Value *N, Value *File,
   Module *M = B.GetInsertBlock()->getModule();
   LLVMContext &Context = B.GetInsertBlock()->getContext();
   StringRef FWriteUnlockedName = TLI->getName(LibFunc_fwrite_unlocked);
-  Constant *F = M->getOrInsertFunction(
+  FunctionCallee F = M->getOrInsertFunction(
       FWriteUnlockedName, DL.getIntPtrType(Context), B.getInt8PtrTy(),
       DL.getIntPtrType(Context), DL.getIntPtrType(Context), File->getType());
 
@@ -1210,7 +1229,8 @@ Value *llvm::emitFWriteUnlocked(Value *Ptr, Value *Size, Value *N, Value *File,
     inferLibFuncAttributes(M, FWriteUnlockedName, *TLI);
   CallInst *CI = B.CreateCall(F, {castToCStr(Ptr, B), Size, N, File});
 
-  if (const Function *Fn = dyn_cast<Function>(F->stripPointerCasts()))
+  if (const Function *Fn =
+          dyn_cast<Function>(F.getCallee()->stripPointerCasts()))
     CI->setCallingConv(Fn->getCallingConv());
   return CI;
 }
@@ -1222,13 +1242,14 @@ Value *llvm::emitFGetCUnlocked(Value *File, IRBuilder<> &B,
 
   Module *M = B.GetInsertBlock()->getModule();
   StringRef FGetCUnlockedName = TLI->getName(LibFunc_fgetc_unlocked);
-  Constant *F =
-      M->getOrInsertFunction(FGetCUnlockedName, B.getInt32Ty(), File->getType());
+  FunctionCallee F = M->getOrInsertFunction(FGetCUnlockedName, B.getInt32Ty(),
+                                            File->getType());
   if (File->getType()->isPointerTy())
     inferLibFuncAttributes(M, FGetCUnlockedName, *TLI);
   CallInst *CI = B.CreateCall(F, File, FGetCUnlockedName);
 
-  if (const Function *Fn = dyn_cast<Function>(F->stripPointerCasts()))
+  if (const Function *Fn =
+          dyn_cast<Function>(F.getCallee()->stripPointerCasts()))
     CI->setCallingConv(Fn->getCallingConv());
   return CI;
 }
@@ -1240,14 +1261,15 @@ Value *llvm::emitFGetSUnlocked(Value *Str, Value *Size, Value *File,
 
   Module *M = B.GetInsertBlock()->getModule();
   StringRef FGetSUnlockedName = TLI->getName(LibFunc_fgets_unlocked);
-  Constant *F =
+  FunctionCallee F =
       M->getOrInsertFunction(FGetSUnlockedName, B.getInt8PtrTy(),
                              B.getInt8PtrTy(), B.getInt32Ty(), File->getType());
   inferLibFuncAttributes(M, FGetSUnlockedName, *TLI);
   CallInst *CI =
       B.CreateCall(F, {castToCStr(Str, B), Size, File}, FGetSUnlockedName);
 
-  if (const Function *Fn = dyn_cast<Function>(F->stripPointerCasts()))
+  if (const Function *Fn =
+          dyn_cast<Function>(F.getCallee()->stripPointerCasts()))
     CI->setCallingConv(Fn->getCallingConv());
   return CI;
 }
@@ -1261,7 +1283,7 @@ Value *llvm::emitFReadUnlocked(Value *Ptr, Value *Size, Value *N, Value *File,
   Module *M = B.GetInsertBlock()->getModule();
   LLVMContext &Context = B.GetInsertBlock()->getContext();
   StringRef FReadUnlockedName = TLI->getName(LibFunc_fread_unlocked);
-  Constant *F = M->getOrInsertFunction(
+  FunctionCallee F = M->getOrInsertFunction(
       FReadUnlockedName, DL.getIntPtrType(Context), B.getInt8PtrTy(),
       DL.getIntPtrType(Context), DL.getIntPtrType(Context), File->getType());
 
@@ -1269,7 +1291,8 @@ Value *llvm::emitFReadUnlocked(Value *Ptr, Value *Size, Value *N, Value *File,
     inferLibFuncAttributes(M, FReadUnlockedName, *TLI);
   CallInst *CI = B.CreateCall(F, {castToCStr(Ptr, B), Size, N, File});
 
-  if (const Function *Fn = dyn_cast<Function>(F->stripPointerCasts()))
+  if (const Function *Fn =
+          dyn_cast<Function>(F.getCallee()->stripPointerCasts()))
     CI->setCallingConv(Fn->getCallingConv());
   return CI;
 }
diff --git a/lib/Transforms/Utils/BypassSlowDivision.cpp b/lib/Transforms/Utils/BypassSlowDivision.cpp
index e7828af648a91df8f1153d8e7f3937a6cbe07cab..df299f673f651b17b5e7706aa9269c621eaa59c2 100644
--- a/lib/Transforms/Utils/BypassSlowDivision.cpp
+++ b/lib/Transforms/Utils/BypassSlowDivision.cpp
@@ -1,9 +1,8 @@
 //===- BypassSlowDivision.cpp - Bypass slow division ----------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Transforms/Utils/CMakeLists.txt b/lib/Transforms/Utils/CMakeLists.txt
index eb3bad79129fd0081184c0e94036830ad916960b..cb3dc17c03ad8d6c471959748c3e6858578d224a 100644
--- a/lib/Transforms/Utils/CMakeLists.txt
+++ b/lib/Transforms/Utils/CMakeLists.txt
@@ -6,6 +6,7 @@ add_llvm_library(LLVMTransformUtils
   BuildLibCalls.cpp
   BypassSlowDivision.cpp
   CallPromotionUtils.cpp
+  CanonicalizeAliases.cpp
   CloneFunction.cpp
   CloneModule.cpp
   CodeExtractor.cpp
diff --git a/lib/Transforms/Utils/CallPromotionUtils.cpp b/lib/Transforms/Utils/CallPromotionUtils.cpp
index e58ddcf34667d5bf6be9cab699d216885c5f1162..1cb883517e84834ad1065521d51e42ce6b56fb08 100644
--- a/lib/Transforms/Utils/CallPromotionUtils.cpp
+++ b/lib/Transforms/Utils/CallPromotionUtils.cpp
@@ -1,9 +1,8 @@
 //===- CallPromotionUtils.cpp - Utilities for call promotion ----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -367,8 +366,9 @@ Instruction *llvm::promoteCall(CallSite CS, Function *Callee,
                                CastInst **RetBitCast) {
   assert(!CS.getCalledFunction() && "Only indirect call sites can be promoted");
 
-  // Set the called function of the call site to be the given callee.
-  CS.setCalledFunction(Callee);
+  // Set the called function of the call site to be the given callee (but don't
+  // change the type).
+  cast<CallBase>(CS.getInstruction())->setCalledOperand(Callee);
 
   // Since the call site will no longer be direct, we must clear metadata that
   // is only appropriate for indirect calls. This includes !prof and !callees
diff --git a/lib/Transforms/Utils/CanonicalizeAliases.cpp b/lib/Transforms/Utils/CanonicalizeAliases.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..455fcbb1cf98a4e02bfbac43ad5e55fd5b447ddd
--- /dev/null
+++ b/lib/Transforms/Utils/CanonicalizeAliases.cpp
@@ -0,0 +1,104 @@
+//===- CanonicalizeAliases.cpp - ThinLTO Support: Canonicalize Aliases ----===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Currently this file implements partial alias canonicalization, to
+// flatten chains of aliases (also done by GlobalOpt, but not on for
+// O0 compiles). E.g.
+//  @a = alias i8, i8 *@b
+//  @b = alias i8, i8 *@g
+//
+// will be converted to:
+//  @a = alias i8, i8 *@g  <-- @a is now an alias to base object @g
+//  @b = alias i8, i8 *@g
+//
+// Eventually this file will implement full alias canonicalation, so that
+// all aliasees are private anonymous values. E.g.
+//  @a = alias i8, i8 *@g
+//  @g = global i8 0
+//
+// will be converted to:
+//  @0 = private global
+//  @a = alias i8, i8* @0
+//  @g = alias i8, i8* @0
+//
+// This simplifies optimization and ThinLTO linking of the original symbols.
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Transforms/Utils/CanonicalizeAliases.h"
+
+#include "llvm/IR/Operator.h"
+#include "llvm/IR/ValueHandle.h"
+
+using namespace llvm;
+
+namespace {
+
+static Constant *canonicalizeAlias(Constant *C, bool &Changed) {
+  if (auto *GA = dyn_cast<GlobalAlias>(C)) {
+    auto *NewAliasee = canonicalizeAlias(GA->getAliasee(), Changed);
+    if (NewAliasee != GA->getAliasee()) {
+      GA->setAliasee(NewAliasee);
+      Changed = true;
+    }
+    return NewAliasee;
+  }
+
+  auto *CE = dyn_cast<ConstantExpr>(C);
+  if (!CE)
+    return C;
+
+  std::vector<Constant *> Ops;
+  for (Use &U : CE->operands())
+    Ops.push_back(canonicalizeAlias(cast<Constant>(U), Changed));
+  return CE->getWithOperands(Ops);
+}
+
+/// Convert aliases to canonical form.
+static bool canonicalizeAliases(Module &M) {
+  bool Changed = false;
+  for (auto &GA : M.aliases())
+    canonicalizeAlias(&GA, Changed);
+  return Changed;
+}
+
+// Legacy pass that canonicalizes aliases.
+class CanonicalizeAliasesLegacyPass : public ModulePass {
+
+public:
+  /// Pass identification, replacement for typeid
+  static char ID;
+
+  /// Specify pass name for debug output
+  StringRef getPassName() const override { return "Canonicalize Aliases"; }
+
+  explicit CanonicalizeAliasesLegacyPass() : ModulePass(ID) {}
+
+  bool runOnModule(Module &M) override { return canonicalizeAliases(M); }
+};
+char CanonicalizeAliasesLegacyPass::ID = 0;
+
+} // anonymous namespace
+
+PreservedAnalyses CanonicalizeAliasesPass::run(Module &M,
+                                               ModuleAnalysisManager &AM) {
+  if (!canonicalizeAliases(M))
+    return PreservedAnalyses::all();
+
+  return PreservedAnalyses::none();
+}
+
+INITIALIZE_PASS_BEGIN(CanonicalizeAliasesLegacyPass, "canonicalize-aliases",
+                      "Canonicalize aliases", false, false)
+INITIALIZE_PASS_END(CanonicalizeAliasesLegacyPass, "canonicalize-aliases",
+                    "Canonicalize aliases", false, false)
+
+namespace llvm {
+ModulePass *createCanonicalizeAliasesPass() {
+  return new CanonicalizeAliasesLegacyPass();
+}
+} // namespace llvm
diff --git a/lib/Transforms/Utils/CloneFunction.cpp b/lib/Transforms/Utils/CloneFunction.cpp
index 8f8c601f5f136b9ff86258710ad15357e2003d8d..5dd7a5bd8c72bbaa86db517b0dc9938c22d0443a 100644
--- a/lib/Transforms/Utils/CloneFunction.cpp
+++ b/lib/Transforms/Utils/CloneFunction.cpp
@@ -1,9 +1,8 @@
 //===- CloneFunction.cpp - Clone a function into another function ---------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -16,13 +15,13 @@
 #include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/Analysis/ConstantFolding.h"
+#include "llvm/Analysis/DomTreeUpdater.h"
 #include "llvm/Analysis/InstructionSimplify.h"
 #include "llvm/Analysis/LoopInfo.h"
 #include "llvm/IR/CFG.h"
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/DebugInfo.h"
 #include "llvm/IR/DerivedTypes.h"
-#include "llvm/IR/DomTreeUpdater.h"
 #include "llvm/IR/Function.h"
 #include "llvm/IR/GlobalVariable.h"
 #include "llvm/IR/Instructions.h"
diff --git a/lib/Transforms/Utils/CloneModule.cpp b/lib/Transforms/Utils/CloneModule.cpp
index 659993aa547850cf62ab74b1ac011e5e9cf1597c..7ddf59becba96c30143feee4304f9e85f8fef9d4 100644
--- a/lib/Transforms/Utils/CloneModule.cpp
+++ b/lib/Transforms/Utils/CloneModule.cpp
@@ -1,9 +1,8 @@
 //===- CloneModule.cpp - Clone an entire module ---------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Transforms/Utils/CodeExtractor.cpp b/lib/Transforms/Utils/CodeExtractor.cpp
index a6b01107752e5b5ce6d7a08a2980ed7047e34a6f..2e073e89dde99b084663068380bde388ebffc4b7 100644
--- a/lib/Transforms/Utils/CodeExtractor.cpp
+++ b/lib/Transforms/Utils/CodeExtractor.cpp
@@ -1,9 +1,8 @@
 //===- CodeExtractor.cpp - Pull code region into a new function -----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -21,6 +20,7 @@
 #include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/SmallVector.h"
+#include "llvm/Analysis/AssumptionCache.h"
 #include "llvm/Analysis/BlockFrequencyInfo.h"
 #include "llvm/Analysis/BlockFrequencyInfoImpl.h"
 #include "llvm/Analysis/BranchProbabilityInfo.h"
@@ -44,6 +44,7 @@
 #include "llvm/IR/LLVMContext.h"
 #include "llvm/IR/MDBuilder.h"
 #include "llvm/IR/Module.h"
+#include "llvm/IR/PatternMatch.h"
 #include "llvm/IR/Type.h"
 #include "llvm/IR/User.h"
 #include "llvm/IR/Value.h"
@@ -67,6 +68,7 @@
 #include <vector>
 
 using namespace llvm;
+using namespace llvm::PatternMatch;
 using ProfileCount = Function::ProfileCount;
 
 #define DEBUG_TYPE "code-extractor"
@@ -236,18 +238,20 @@ buildExtractionBlockSet(ArrayRef<BasicBlock *> BBs, DominatorTree *DT,
 
 CodeExtractor::CodeExtractor(ArrayRef<BasicBlock *> BBs, DominatorTree *DT,
                              bool AggregateArgs, BlockFrequencyInfo *BFI,
-                             BranchProbabilityInfo *BPI, bool AllowVarArgs,
-                             bool AllowAlloca, std::string Suffix)
+                             BranchProbabilityInfo *BPI, AssumptionCache *AC,
+                             bool AllowVarArgs, bool AllowAlloca,
+                             std::string Suffix)
     : DT(DT), AggregateArgs(AggregateArgs || AggregateArgsOpt), BFI(BFI),
-      BPI(BPI), AllowVarArgs(AllowVarArgs),
+      BPI(BPI), AC(AC), AllowVarArgs(AllowVarArgs),
       Blocks(buildExtractionBlockSet(BBs, DT, AllowVarArgs, AllowAlloca)),
       Suffix(Suffix) {}
 
 CodeExtractor::CodeExtractor(DominatorTree &DT, Loop &L, bool AggregateArgs,
                              BlockFrequencyInfo *BFI,
-                             BranchProbabilityInfo *BPI, std::string Suffix)
+                             BranchProbabilityInfo *BPI, AssumptionCache *AC,
+                             std::string Suffix)
     : DT(&DT), AggregateArgs(AggregateArgs || AggregateArgsOpt), BFI(BFI),
-      BPI(BPI), AllowVarArgs(false),
+      BPI(BPI), AC(AC), AllowVarArgs(false),
       Blocks(buildExtractionBlockSet(L.getBlocks(), &DT,
                                      /* AllowVarArgs */ false,
                                      /* AllowAlloca */ false)),
@@ -332,8 +336,7 @@ bool CodeExtractor::isLegalToShrinkwrapLifetimeMarkers(
       default: {
         IntrinsicInst *IntrInst = dyn_cast<IntrinsicInst>(&II);
         if (IntrInst) {
-          if (IntrInst->getIntrinsicID() == Intrinsic::lifetime_start ||
-              IntrInst->getIntrinsicID() == Intrinsic::lifetime_end)
+          if (IntrInst->isLifetimeStartOrEnd())
             break;
           return false;
         }
@@ -846,7 +849,8 @@ Function *CodeExtractor::constructFunction(const ValueSet &inputs,
       Instruction *TI = newFunction->begin()->getTerminator();
       GetElementPtrInst *GEP = GetElementPtrInst::Create(
           StructTy, &*AI, Idx, "gep_" + inputs[i]->getName(), TI);
-      RewriteVal = new LoadInst(GEP, "loadgep_" + inputs[i]->getName(), TI);
+      RewriteVal = new LoadInst(StructTy->getElementType(i), GEP,
+                                "loadgep_" + inputs[i]->getName(), TI);
     } else
       RewriteVal = &*AI++;
 
@@ -881,12 +885,99 @@ Function *CodeExtractor::constructFunction(const ValueSet &inputs,
   return newFunction;
 }
 
+/// Erase lifetime.start markers which reference inputs to the extraction
+/// region, and insert the referenced memory into \p LifetimesStart. Do the same
+/// with lifetime.end markers (but insert them into \p LifetimesEnd).
+///
+/// The extraction region is defined by a set of blocks (\p Blocks), and a set
+/// of allocas which will be moved from the caller function into the extracted
+/// function (\p SunkAllocas).
+static void eraseLifetimeMarkersOnInputs(const SetVector<BasicBlock *> &Blocks,
+                                         const SetVector<Value *> &SunkAllocas,
+                                         SetVector<Value *> &LifetimesStart,
+                                         SetVector<Value *> &LifetimesEnd) {
+  for (BasicBlock *BB : Blocks) {
+    for (auto It = BB->begin(), End = BB->end(); It != End;) {
+      auto *II = dyn_cast<IntrinsicInst>(&*It);
+      ++It;
+      if (!II || !II->isLifetimeStartOrEnd())
+        continue;
+
+      // Get the memory operand of the lifetime marker. If the underlying
+      // object is a sunk alloca, or is otherwise defined in the extraction
+      // region, the lifetime marker must not be erased.
+      Value *Mem = II->getOperand(1)->stripInBoundsOffsets();
+      if (SunkAllocas.count(Mem) || definedInRegion(Blocks, Mem))
+        continue;
+
+      if (II->getIntrinsicID() == Intrinsic::lifetime_start)
+        LifetimesStart.insert(Mem);
+      else
+        LifetimesEnd.insert(Mem);
+      II->eraseFromParent();
+    }
+  }
+}
+
+/// Insert lifetime start/end markers surrounding the call to the new function
+/// for objects defined in the caller.
+static void insertLifetimeMarkersSurroundingCall(
+    Module *M, ArrayRef<Value *> LifetimesStart, ArrayRef<Value *> LifetimesEnd,
+    CallInst *TheCall) {
+  LLVMContext &Ctx = M->getContext();
+  auto Int8PtrTy = Type::getInt8PtrTy(Ctx);
+  auto NegativeOne = ConstantInt::getSigned(Type::getInt64Ty(Ctx), -1);
+  Instruction *Term = TheCall->getParent()->getTerminator();
+
+  // The memory argument to a lifetime marker must be a i8*. Cache any bitcasts
+  // needed to satisfy this requirement so they may be reused.
+  DenseMap<Value *, Value *> Bitcasts;
+
+  // Emit lifetime markers for the pointers given in \p Objects. Insert the
+  // markers before the call if \p InsertBefore, and after the call otherwise.
+  auto insertMarkers = [&](Function *MarkerFunc, ArrayRef<Value *> Objects,
+                           bool InsertBefore) {
+    for (Value *Mem : Objects) {
+      assert((!isa<Instruction>(Mem) || cast<Instruction>(Mem)->getFunction() ==
+                                            TheCall->getFunction()) &&
+             "Input memory not defined in original function");
+      Value *&MemAsI8Ptr = Bitcasts[Mem];
+      if (!MemAsI8Ptr) {
+        if (Mem->getType() == Int8PtrTy)
+          MemAsI8Ptr = Mem;
+        else
+          MemAsI8Ptr =
+              CastInst::CreatePointerCast(Mem, Int8PtrTy, "lt.cast", TheCall);
+      }
+
+      auto Marker = CallInst::Create(MarkerFunc, {NegativeOne, MemAsI8Ptr});
+      if (InsertBefore)
+        Marker->insertBefore(TheCall);
+      else
+        Marker->insertBefore(Term);
+    }
+  };
+
+  if (!LifetimesStart.empty()) {
+    auto StartFn = llvm::Intrinsic::getDeclaration(
+        M, llvm::Intrinsic::lifetime_start, Int8PtrTy);
+    insertMarkers(StartFn, LifetimesStart, /*InsertBefore=*/true);
+  }
+
+  if (!LifetimesEnd.empty()) {
+    auto EndFn = llvm::Intrinsic::getDeclaration(
+        M, llvm::Intrinsic::lifetime_end, Int8PtrTy);
+    insertMarkers(EndFn, LifetimesEnd, /*InsertBefore=*/false);
+  }
+}
+
 /// emitCallAndSwitchStatement - This method sets up the caller side by adding
 /// the call instruction, splitting any PHI nodes in the header block as
 /// necessary.
-void CodeExtractor::
-emitCallAndSwitchStatement(Function *newFunction, BasicBlock *codeReplacer,
-                           ValueSet &inputs, ValueSet &outputs) {
+CallInst *CodeExtractor::emitCallAndSwitchStatement(Function *newFunction,
+                                                    BasicBlock *codeReplacer,
+                                                    ValueSet &inputs,
+                                                    ValueSet &outputs) {
   // Emit a call to the new function, passing in: *pointer to struct (if
   // aggregating parameters), or plan inputs and allocated memory for outputs
   std::vector<Value *> params, StructValues, ReloadOutputs, Reloads;
@@ -894,13 +985,21 @@ emitCallAndSwitchStatement(Function *newFunction, BasicBlock *codeReplacer,
   Module *M = newFunction->getParent();
   LLVMContext &Context = M->getContext();
   const DataLayout &DL = M->getDataLayout();
+  CallInst *call = nullptr;
 
   // Add inputs as params, or to be filled into the struct
-  for (Value *input : inputs)
+  unsigned ArgNo = 0;
+  SmallVector<unsigned, 1> SwiftErrorArgs;
+  for (Value *input : inputs) {
     if (AggregateArgs)
       StructValues.push_back(input);
-    else
+    else {
       params.push_back(input);
+      if (input->isSwiftError())
+        SwiftErrorArgs.push_back(ArgNo);
+    }
+    ++ArgNo;
+  }
 
   // Create allocas for the outputs
   for (Value *output : outputs) {
@@ -944,8 +1043,8 @@ emitCallAndSwitchStatement(Function *newFunction, BasicBlock *codeReplacer,
   }
 
   // Emit the call to the function
-  CallInst *call = CallInst::Create(newFunction, params,
-                                    NumExitBlocks > 1 ? "targetBlock" : "");
+  call = CallInst::Create(newFunction, params,
+                          NumExitBlocks > 1 ? "targetBlock" : "");
   // Add debug location to the new call, if the original function has debug
   // info. In that case, the terminator of the entry block of the extracted
   // function contains the first debug location of the extracted function,
@@ -956,13 +1055,18 @@ emitCallAndSwitchStatement(Function *newFunction, BasicBlock *codeReplacer,
   }
   codeReplacer->getInstList().push_back(call);
 
+  // Set swifterror parameter attributes.
+  for (unsigned SwiftErrArgNo : SwiftErrorArgs) {
+    call->addParamAttr(SwiftErrArgNo, Attribute::SwiftError);
+    newFunction->addParamAttr(SwiftErrArgNo, Attribute::SwiftError);
+  }
+
   Function::arg_iterator OutputArgBegin = newFunction->arg_begin();
   unsigned FirstOut = inputs.size();
   if (!AggregateArgs)
     std::advance(OutputArgBegin, inputs.size());
 
   // Reload the outputs passed in by reference.
-  Function::arg_iterator OAI = OutputArgBegin;
   for (unsigned i = 0, e = outputs.size(); i != e; ++i) {
     Value *Output = nullptr;
     if (AggregateArgs) {
@@ -976,7 +1080,8 @@ emitCallAndSwitchStatement(Function *newFunction, BasicBlock *codeReplacer,
     } else {
       Output = ReloadOutputs[i];
     }
-    LoadInst *load = new LoadInst(Output, outputs[i]->getName()+".reload");
+    LoadInst *load = new LoadInst(outputs[i]->getType(), Output,
+                                  outputs[i]->getName() + ".reload");
     Reloads.push_back(load);
     codeReplacer->getInstList().push_back(load);
     std::vector<User *> Users(outputs[i]->user_begin(), outputs[i]->user_end());
@@ -985,40 +1090,6 @@ emitCallAndSwitchStatement(Function *newFunction, BasicBlock *codeReplacer,
       if (!Blocks.count(inst->getParent()))
         inst->replaceUsesOfWith(outputs[i], load);
     }
-
-    // Store to argument right after the definition of output value.
-    auto *OutI = dyn_cast<Instruction>(outputs[i]);
-    if (!OutI)
-      continue;
-
-    // Find proper insertion point.
-    BasicBlock::iterator InsertPt;
-    // In case OutI is an invoke, we insert the store at the beginning in the
-    // 'normal destination' BB. Otherwise we insert the store right after OutI.
-    if (auto *InvokeI = dyn_cast<InvokeInst>(OutI))
-      InsertPt = InvokeI->getNormalDest()->getFirstInsertionPt();
-    else if (auto *Phi = dyn_cast<PHINode>(OutI))
-      InsertPt = Phi->getParent()->getFirstInsertionPt();
-    else
-      InsertPt = std::next(OutI->getIterator());
-
-    assert(OAI != newFunction->arg_end() &&
-           "Number of output arguments should match "
-           "the amount of defined values");
-    if (AggregateArgs) {
-      Value *Idx[2];
-      Idx[0] = Constant::getNullValue(Type::getInt32Ty(Context));
-      Idx[1] = ConstantInt::get(Type::getInt32Ty(Context), FirstOut + i);
-      GetElementPtrInst *GEP = GetElementPtrInst::Create(
-          StructArgTy, &*OAI, Idx, "gep_" + outputs[i]->getName(), &*InsertPt);
-      new StoreInst(outputs[i], GEP, &*InsertPt);
-      // Since there should be only one struct argument aggregating
-      // all the output values, we shouldn't increment OAI, which always
-      // points to the struct argument, in this case.
-    } else {
-      new StoreInst(outputs[i], &*OAI, &*InsertPt);
-      ++OAI;
-    }
   }
 
   // Now we can emit a switch statement using the call as a value.
@@ -1074,6 +1145,50 @@ emitCallAndSwitchStatement(Function *newFunction, BasicBlock *codeReplacer,
       }
   }
 
+  // Store the arguments right after the definition of output value.
+  // This should be proceeded after creating exit stubs to be ensure that invoke
+  // result restore will be placed in the outlined function.
+  Function::arg_iterator OAI = OutputArgBegin;
+  for (unsigned i = 0, e = outputs.size(); i != e; ++i) {
+    auto *OutI = dyn_cast<Instruction>(outputs[i]);
+    if (!OutI)
+      continue;
+
+    // Find proper insertion point.
+    BasicBlock::iterator InsertPt;
+    // In case OutI is an invoke, we insert the store at the beginning in the
+    // 'normal destination' BB. Otherwise we insert the store right after OutI.
+    if (auto *InvokeI = dyn_cast<InvokeInst>(OutI))
+      InsertPt = InvokeI->getNormalDest()->getFirstInsertionPt();
+    else if (auto *Phi = dyn_cast<PHINode>(OutI))
+      InsertPt = Phi->getParent()->getFirstInsertionPt();
+    else
+      InsertPt = std::next(OutI->getIterator());
+
+    Instruction *InsertBefore = &*InsertPt;
+    assert((InsertBefore->getFunction() == newFunction ||
+            Blocks.count(InsertBefore->getParent())) &&
+           "InsertPt should be in new function");
+    assert(OAI != newFunction->arg_end() &&
+           "Number of output arguments should match "
+           "the amount of defined values");
+    if (AggregateArgs) {
+      Value *Idx[2];
+      Idx[0] = Constant::getNullValue(Type::getInt32Ty(Context));
+      Idx[1] = ConstantInt::get(Type::getInt32Ty(Context), FirstOut + i);
+      GetElementPtrInst *GEP = GetElementPtrInst::Create(
+          StructArgTy, &*OAI, Idx, "gep_" + outputs[i]->getName(),
+          InsertBefore);
+      new StoreInst(outputs[i], GEP, InsertBefore);
+      // Since there should be only one struct argument aggregating
+      // all the output values, we shouldn't increment OAI, which always
+      // points to the struct argument, in this case.
+    } else {
+      new StoreInst(outputs[i], &*OAI, InsertBefore);
+      ++OAI;
+    }
+  }
+
   // Now that we've done the deed, simplify the switch instruction.
   Type *OldFnRetTy = TheSwitch->getParent()->getParent()->getReturnType();
   switch (NumExitBlocks) {
@@ -1117,6 +1232,12 @@ emitCallAndSwitchStatement(Function *newFunction, BasicBlock *codeReplacer,
     TheSwitch->removeCase(SwitchInst::CaseIt(TheSwitch, NumExitBlocks-1));
     break;
   }
+
+  // Insert lifetime markers around the reloads of any output values. The
+  // allocas output values are stored in are only in-use in the codeRepl block.
+  insertLifetimeMarkersSurroundingCall(M, ReloadOutputs, ReloadOutputs, call);
+
+  return call;
 }
 
 void CodeExtractor::moveCodeToFunction(Function *newFunction) {
@@ -1130,6 +1251,13 @@ void CodeExtractor::moveCodeToFunction(Function *newFunction) {
 
     // Insert this basic block into the new function
     newBlocks.push_back(Block);
+
+    // Remove @llvm.assume calls that were moved to the new function from the
+    // old function's assumption cache.
+    if (AC)
+      for (auto &I : *Block)
+        if (match(&I, m_Intrinsic<Intrinsic::assume>()))
+          AC->unregisterAssumption(cast<CallInst>(&I));
   }
 }
 
@@ -1292,11 +1420,18 @@ Function *CodeExtractor::extractCodeRegion() {
       cast<Instruction>(II)->moveBefore(TI);
   }
 
+  // Collect objects which are inputs to the extraction region and also
+  // referenced by lifetime start/end markers within it. The effects of these
+  // markers must be replicated in the calling function to prevent the stack
+  // coloring pass from merging slots which store input objects.
+  ValueSet LifetimesStart, LifetimesEnd;
+  eraseLifetimeMarkersOnInputs(Blocks, SinkingCands, LifetimesStart,
+                               LifetimesEnd);
+
   // Construct new function based on inputs/outputs & add allocas for all defs.
-  Function *newFunction = constructFunction(inputs, outputs, header,
-                                            newFuncRoot,
-                                            codeReplacer, oldFunction,
-                                            oldFunction->getParent());
+  Function *newFunction =
+      constructFunction(inputs, outputs, header, newFuncRoot, codeReplacer,
+                        oldFunction, oldFunction->getParent());
 
   // Update the entry count of the function.
   if (BFI) {
@@ -1307,10 +1442,17 @@ Function *CodeExtractor::extractCodeRegion() {
     BFI->setBlockFreq(codeReplacer, EntryFreq.getFrequency());
   }
 
-  emitCallAndSwitchStatement(newFunction, codeReplacer, inputs, outputs);
+  CallInst *TheCall =
+      emitCallAndSwitchStatement(newFunction, codeReplacer, inputs, outputs);
 
   moveCodeToFunction(newFunction);
 
+  // Replicate the effects of any lifetime start/end markers which referenced
+  // input objects in the extraction region by placing markers around the call.
+  insertLifetimeMarkersSurroundingCall(oldFunction->getParent(),
+                                       LifetimesStart.getArrayRef(),
+                                       LifetimesEnd.getArrayRef(), TheCall);
+
   // Propagate personality info to the new function if there is one.
   if (oldFunction->hasPersonalityFn())
     newFunction->setPersonalityFn(oldFunction->getPersonalityFn());
diff --git a/lib/Transforms/Utils/CtorUtils.cpp b/lib/Transforms/Utils/CtorUtils.cpp
index 4e7da7d0449f9213c6c6d0a4fe871789c652d901..069a86f6ab337701bd3f9228cd95ec247c4e1475 100644
--- a/lib/Transforms/Utils/CtorUtils.cpp
+++ b/lib/Transforms/Utils/CtorUtils.cpp
@@ -1,9 +1,8 @@
 //===- CtorUtils.cpp - Helpers for working with global_ctors ----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Transforms/Utils/DemoteRegToStack.cpp b/lib/Transforms/Utils/DemoteRegToStack.cpp
index 975b363859a914d8ceb00cefec519c73f0641db7..5f53d794fe8a16c897118c96f7f9bcfd308e341d 100644
--- a/lib/Transforms/Utils/DemoteRegToStack.cpp
+++ b/lib/Transforms/Utils/DemoteRegToStack.cpp
@@ -1,9 +1,8 @@
 //===- DemoteRegToStack.cpp - Move a virtual register to the stack --------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -73,7 +72,8 @@ AllocaInst *llvm::DemoteRegToStack(Instruction &I, bool VolatileLoads,
           Value *&V = Loads[PN->getIncomingBlock(i)];
           if (!V) {
             // Insert the load into the predecessor block
-            V = new LoadInst(Slot, I.getName()+".reload", VolatileLoads,
+            V = new LoadInst(I.getType(), Slot, I.getName() + ".reload",
+                             VolatileLoads,
                              PN->getIncomingBlock(i)->getTerminator());
           }
           PN->setIncomingValue(i, V);
@@ -81,7 +81,8 @@ AllocaInst *llvm::DemoteRegToStack(Instruction &I, bool VolatileLoads,
 
     } else {
       // If this is a normal instruction, just insert a load.
-      Value *V = new LoadInst(Slot, I.getName()+".reload", VolatileLoads, U);
+      Value *V = new LoadInst(I.getType(), Slot, I.getName() + ".reload",
+                              VolatileLoads, U);
       U->replaceUsesOfWith(&I, V);
     }
   }
@@ -142,7 +143,8 @@ AllocaInst *llvm::DemotePHIToStack(PHINode *P, Instruction *AllocaPoint) {
   for (; isa<PHINode>(InsertPt) || InsertPt->isEHPad(); ++InsertPt)
     /* empty */;   // Don't insert before PHI nodes or landingpad instrs.
 
-  Value *V = new LoadInst(Slot, P->getName() + ".reload", &*InsertPt);
+  Value *V =
+      new LoadInst(P->getType(), Slot, P->getName() + ".reload", &*InsertPt);
   P->replaceAllUsesWith(V);
 
   // Delete PHI.
diff --git a/lib/Transforms/Utils/EntryExitInstrumenter.cpp b/lib/Transforms/Utils/EntryExitInstrumenter.cpp
index 569ea58a30476f05c5b318a69c700922fb1f57f9..4aa40eeadda44171cb8667a025ed1382801cd778 100644
--- a/lib/Transforms/Utils/EntryExitInstrumenter.cpp
+++ b/lib/Transforms/Utils/EntryExitInstrumenter.cpp
@@ -1,9 +1,8 @@
 //===- EntryExitInstrumenter.cpp - Function Entry/Exit Instrumentation ----===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -31,7 +30,7 @@ static void insertCall(Function &CurFn, StringRef Func,
       Func == "__mcount" ||
       Func == "_mcount" ||
       Func == "__cyg_profile_func_enter_bare") {
-    Constant *Fn = M.getOrInsertFunction(Func, Type::getVoidTy(C));
+    FunctionCallee Fn = M.getOrInsertFunction(Func, Type::getVoidTy(C));
     CallInst *Call = CallInst::Create(Fn, "", InsertionPt);
     Call->setDebugLoc(DL);
     return;
@@ -40,7 +39,7 @@ static void insertCall(Function &CurFn, StringRef Func,
   if (Func == "__cyg_profile_func_enter" || Func == "__cyg_profile_func_exit") {
     Type *ArgTypes[] = {Type::getInt8PtrTy(C), Type::getInt8PtrTy(C)};
 
-    Constant *Fn = M.getOrInsertFunction(
+    FunctionCallee Fn = M.getOrInsertFunction(
         Func, FunctionType::get(Type::getVoidTy(C), ArgTypes, false));
 
     Instruction *RetAddr = CallInst::Create(
diff --git a/lib/Transforms/Utils/EscapeEnumerator.cpp b/lib/Transforms/Utils/EscapeEnumerator.cpp
index 762a374c135c81b1651ed49569bcc37e07de3c93..914babeb6829d88cf010332b80685b567922249d 100644
--- a/lib/Transforms/Utils/EscapeEnumerator.cpp
+++ b/lib/Transforms/Utils/EscapeEnumerator.cpp
@@ -1,9 +1,8 @@
 //===- EscapeEnumerator.cpp -----------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -19,7 +18,7 @@
 #include "llvm/IR/Module.h"
 using namespace llvm;
 
-static Constant *getDefaultPersonalityFn(Module *M) {
+static FunctionCallee getDefaultPersonalityFn(Module *M) {
   LLVMContext &C = M->getContext();
   Triple T(M->getTargetTriple());
   EHPersonality Pers = getDefaultEHPersonality(T);
@@ -69,8 +68,8 @@ IRBuilder<> *EscapeEnumerator::Next() {
   BasicBlock *CleanupBB = BasicBlock::Create(C, CleanupBBName, &F);
   Type *ExnTy = StructType::get(Type::getInt8PtrTy(C), Type::getInt32Ty(C));
   if (!F.hasPersonalityFn()) {
-    Constant *PersFn = getDefaultPersonalityFn(F.getParent());
-    F.setPersonalityFn(PersFn);
+    FunctionCallee PersFn = getDefaultPersonalityFn(F.getParent());
+    F.setPersonalityFn(cast<Constant>(PersFn.getCallee()));
   }
 
   if (isScopedEHPersonality(classifyEHPersonality(F.getPersonalityFn()))) {
diff --git a/lib/Transforms/Utils/Evaluator.cpp b/lib/Transforms/Utils/Evaluator.cpp
index 992c8b922f75dcf4ce0a9bb744dc0ef27ec33f1c..fb8cb3ddea66aaf0e770d2a8185c4a5692225042 100644
--- a/lib/Transforms/Utils/Evaluator.cpp
+++ b/lib/Transforms/Utils/Evaluator.cpp
@@ -1,9 +1,8 @@
 //===- Evaluator.cpp - LLVM IR evaluator ----------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -344,18 +343,18 @@ bool Evaluator::EvaluateBlock(BasicBlock::iterator CurInst,
             // into a pointer to its first member.
             // FIXME: This could be extended to support arrays as well.
             if (StructType *STy = dyn_cast<StructType>(NewTy)) {
-              NewTy = STy->getTypeAtIndex(0U);
 
               IntegerType *IdxTy = IntegerType::get(NewTy->getContext(), 32);
               Constant *IdxZero = ConstantInt::get(IdxTy, 0, false);
               Constant * const IdxList[] = {IdxZero, IdxZero};
 
-              Ptr = ConstantExpr::getGetElementPtr(nullptr, Ptr, IdxList);
+              Ptr = ConstantExpr::getGetElementPtr(NewTy, Ptr, IdxList);
               if (auto *FoldedPtr = ConstantFoldConstant(Ptr, DL, TLI))
                 Ptr = FoldedPtr;
+              NewTy = STy->getTypeAtIndex(0U);
 
-            // If we can't improve the situation by introspecting NewTy,
-            // we have to give up.
+              // If we can't improve the situation by introspecting NewTy,
+              // we have to give up.
             } else {
               LLVM_DEBUG(dbgs() << "Failed to bitcast constant ptr, can not "
                                    "evaluate.\n");
@@ -483,8 +482,7 @@ bool Evaluator::EvaluateBlock(BasicBlock::iterator CurInst,
           }
         }
 
-        if (II->getIntrinsicID() == Intrinsic::lifetime_start ||
-            II->getIntrinsicID() == Intrinsic::lifetime_end) {
+        if (II->isLifetimeStartOrEnd()) {
           LLVM_DEBUG(dbgs() << "Ignoring lifetime intrinsic.\n");
           ++CurInst;
           continue;
@@ -542,7 +540,8 @@ bool Evaluator::EvaluateBlock(BasicBlock::iterator CurInst,
 
       if (Callee->isDeclaration()) {
         // If this is a function we can constant fold, do it.
-        if (Constant *C = ConstantFoldCall(CS, Callee, Formals, TLI)) {
+        if (Constant *C = ConstantFoldCall(cast<CallBase>(CS.getInstruction()),
+                                           Callee, Formals, TLI)) {
           InstResult = castCallResultIfNeeded(CS.getCalledValue(), C);
           if (!InstResult)
             return false;
diff --git a/lib/Transforms/Utils/FlattenCFG.cpp b/lib/Transforms/Utils/FlattenCFG.cpp
index d9778f4a1fb73ac1e21a3c10a5a4de327616d000..0c52e6f3703b874480aea11bf3244b20cfdf0e8c 100644
--- a/lib/Transforms/Utils/FlattenCFG.cpp
+++ b/lib/Transforms/Utils/FlattenCFG.cpp
@@ -1,9 +1,8 @@
 //===- FlatternCFG.cpp - Code to perform CFG flattening -------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Transforms/Utils/FunctionComparator.cpp b/lib/Transforms/Utils/FunctionComparator.cpp
index a717d9b728198e56cfe96d2375bf90176b312014..5c400cec573464eba6381aad3ea83de7f83d4745 100644
--- a/lib/Transforms/Utils/FunctionComparator.cpp
+++ b/lib/Transforms/Utils/FunctionComparator.cpp
@@ -1,9 +1,8 @@
 //===- FunctionComparator.h - Function Comparator -------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -557,31 +556,20 @@ int FunctionComparator::cmpOperations(const Instruction *L,
   }
   if (const CmpInst *CI = dyn_cast<CmpInst>(L))
     return cmpNumbers(CI->getPredicate(), cast<CmpInst>(R)->getPredicate());
-  if (const CallInst *CI = dyn_cast<CallInst>(L)) {
-    if (int Res = cmpNumbers(CI->getCallingConv(),
-                             cast<CallInst>(R)->getCallingConv()))
-      return Res;
-    if (int Res =
-            cmpAttrs(CI->getAttributes(), cast<CallInst>(R)->getAttributes()))
-      return Res;
-    if (int Res = cmpOperandBundlesSchema(CI, R))
+  if (auto CSL = CallSite(const_cast<Instruction *>(L))) {
+    auto CSR = CallSite(const_cast<Instruction *>(R));
+    if (int Res = cmpNumbers(CSL.getCallingConv(), CSR.getCallingConv()))
       return Res;
-    return cmpRangeMetadata(
-        CI->getMetadata(LLVMContext::MD_range),
-        cast<CallInst>(R)->getMetadata(LLVMContext::MD_range));
-  }
-  if (const InvokeInst *II = dyn_cast<InvokeInst>(L)) {
-    if (int Res = cmpNumbers(II->getCallingConv(),
-                             cast<InvokeInst>(R)->getCallingConv()))
-      return Res;
-    if (int Res =
-            cmpAttrs(II->getAttributes(), cast<InvokeInst>(R)->getAttributes()))
+    if (int Res = cmpAttrs(CSL.getAttributes(), CSR.getAttributes()))
       return Res;
-    if (int Res = cmpOperandBundlesSchema(II, R))
+    if (int Res = cmpOperandBundlesSchema(L, R))
       return Res;
-    return cmpRangeMetadata(
-        II->getMetadata(LLVMContext::MD_range),
-        cast<InvokeInst>(R)->getMetadata(LLVMContext::MD_range));
+    if (const CallInst *CI = dyn_cast<CallInst>(L))
+      if (int Res = cmpNumbers(CI->getTailCallKind(),
+                               cast<CallInst>(R)->getTailCallKind()))
+        return Res;
+    return cmpRangeMetadata(L->getMetadata(LLVMContext::MD_range),
+                            R->getMetadata(LLVMContext::MD_range));
   }
   if (const InsertValueInst *IVI = dyn_cast<InsertValueInst>(L)) {
     ArrayRef<unsigned> LIndices = IVI->getIndices();
diff --git a/lib/Transforms/Utils/FunctionImportUtils.cpp b/lib/Transforms/Utils/FunctionImportUtils.cpp
index a9772e31da5095ccfa3415c52d1d6a4d6eaa6424..e48fcbdeff48540ca2721a2393251ef6768324ed 100644
--- a/lib/Transforms/Utils/FunctionImportUtils.cpp
+++ b/lib/Transforms/Utils/FunctionImportUtils.cpp
@@ -1,9 +1,8 @@
 //===- lib/Transforms/Utils/FunctionImportUtils.cpp - Importing utilities -===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -249,6 +248,8 @@ void FunctionImportGlobalProcessing::processGlobalForThinLTO(GlobalValue &GV) {
   bool DoPromote = false;
   if (GV.hasLocalLinkage() &&
       ((DoPromote = shouldPromoteLocalToGlobal(&GV)) || isPerformingImport())) {
+    // Save the original name string before we rename GV below.
+    auto Name = GV.getName().str();
     // Once we change the name or linkage it is difficult to determine
     // again whether we should promote since shouldPromoteLocalToGlobal needs
     // to locate the summary (based on GUID from name and linkage). Therefore,
@@ -257,6 +258,12 @@ void FunctionImportGlobalProcessing::processGlobalForThinLTO(GlobalValue &GV) {
     GV.setLinkage(getLinkage(&GV, DoPromote));
     if (!GV.hasLocalLinkage())
       GV.setVisibility(GlobalValue::HiddenVisibility);
+
+    // If we are renaming a COMDAT leader, ensure that we record the COMDAT
+    // for later renaming as well. This is required for COFF.
+    if (const auto *C = GV.getComdat())
+      if (C->getName() == Name)
+        RenamedComdats.try_emplace(C, M.getOrInsertComdat(GV.getName()));
   } else
     GV.setLinkage(getLinkage(&GV, /* DoPromote */ false));
 
@@ -281,6 +288,16 @@ void FunctionImportGlobalProcessing::processGlobalsForThinLTO() {
     processGlobalForThinLTO(SF);
   for (GlobalAlias &GA : M.aliases())
     processGlobalForThinLTO(GA);
+
+  // Replace any COMDATS that required renaming (because the COMDAT leader was
+  // promoted and renamed).
+  if (!RenamedComdats.empty())
+    for (auto &GO : M.global_objects())
+      if (auto *C = GO.getComdat()) {
+        auto Replacement = RenamedComdats.find(C);
+        if (Replacement != RenamedComdats.end())
+          GO.setComdat(Replacement->second);
+      }
 }
 
 bool FunctionImportGlobalProcessing::run() {
diff --git a/lib/Transforms/Utils/GlobalStatus.cpp b/lib/Transforms/Utils/GlobalStatus.cpp
index ff6970db47da499d3e7f7287a56704de6805c817..a2942869130d5184c5ba5d347ec904486cec52e2 100644
--- a/lib/Transforms/Utils/GlobalStatus.cpp
+++ b/lib/Transforms/Utils/GlobalStatus.cpp
@@ -1,9 +1,8 @@
 //===-- GlobalStatus.cpp - Compute status info for globals -----------------==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Transforms/Utils/GuardUtils.cpp b/lib/Transforms/Utils/GuardUtils.cpp
index 08de0a4c53e996de3897e6b6966f9920691e3a2b..34c32d9c0c98803af166c704f73ff229ed586e14 100644
--- a/lib/Transforms/Utils/GuardUtils.cpp
+++ b/lib/Transforms/Utils/GuardUtils.cpp
@@ -1,9 +1,8 @@
 //===-- GuardUtils.cpp - Utils for work with guards -------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 // Utils that are used to perform transformations related to guards and their
diff --git a/lib/Transforms/Utils/ImportedFunctionsInliningStatistics.cpp b/lib/Transforms/Utils/ImportedFunctionsInliningStatistics.cpp
index 02482c5503215f29cee52ec832472875df0bb555..8041e66e6c4c24b742da16a29b5d113b395a8e23 100644
--- a/lib/Transforms/Utils/ImportedFunctionsInliningStatistics.cpp
+++ b/lib/Transforms/Utils/ImportedFunctionsInliningStatistics.cpp
@@ -1,9 +1,8 @@
 //===-- ImportedFunctionsInliningStats.cpp ----------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 // Generating inliner statistics for imported functions, mostly useful for
diff --git a/lib/Transforms/Utils/InlineFunction.cpp b/lib/Transforms/Utils/InlineFunction.cpp
index bda2ee2d8a3f3e5a9b4f3625367bf1071bda8d4a..7443a7f9c5e54807116cbc759b3a9bd4c79487a0 100644
--- a/lib/Transforms/Utils/InlineFunction.cpp
+++ b/lib/Transforms/Utils/InlineFunction.cpp
@@ -1,9 +1,8 @@
 //===- InlineFunction.cpp - Code to perform function inlining -------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -31,6 +30,7 @@
 #include "llvm/Analysis/ProfileSummaryInfo.h"
 #include "llvm/Transforms/Utils/Local.h"
 #include "llvm/Analysis/ValueTracking.h"
+#include "llvm/Analysis/VectorUtils.h"
 #include "llvm/IR/Argument.h"
 #include "llvm/IR/BasicBlock.h"
 #include "llvm/IR/CFG.h"
@@ -770,14 +770,16 @@ static void HandleInlinedEHPad(InvokeInst *II, BasicBlock *FirstNewBlock,
   UnwindDest->removePredecessor(InvokeBB);
 }
 
-/// When inlining a call site that has !llvm.mem.parallel_loop_access metadata,
-/// that metadata should be propagated to all memory-accessing cloned
-/// instructions.
+/// When inlining a call site that has !llvm.mem.parallel_loop_access or
+/// llvm.access.group metadata, that metadata should be propagated to all
+/// memory-accessing cloned instructions.
 static void PropagateParallelLoopAccessMetadata(CallSite CS,
                                                 ValueToValueMapTy &VMap) {
   MDNode *M =
     CS.getInstruction()->getMetadata(LLVMContext::MD_mem_parallel_loop_access);
-  if (!M)
+  MDNode *CallAccessGroup =
+      CS.getInstruction()->getMetadata(LLVMContext::MD_access_group);
+  if (!M && !CallAccessGroup)
     return;
 
   for (ValueToValueMapTy::iterator VMI = VMap.begin(), VMIE = VMap.end();
@@ -789,11 +791,20 @@ static void PropagateParallelLoopAccessMetadata(CallSite CS,
     if (!NI)
       continue;
 
-    if (MDNode *PM = NI->getMetadata(LLVMContext::MD_mem_parallel_loop_access)) {
+    if (M) {
+      if (MDNode *PM =
+              NI->getMetadata(LLVMContext::MD_mem_parallel_loop_access)) {
         M = MDNode::concatenate(PM, M);
       NI->setMetadata(LLVMContext::MD_mem_parallel_loop_access, M);
-    } else if (NI->mayReadOrWriteMemory()) {
-      NI->setMetadata(LLVMContext::MD_mem_parallel_loop_access, M);
+      } else if (NI->mayReadOrWriteMemory()) {
+        NI->setMetadata(LLVMContext::MD_mem_parallel_loop_access, M);
+      }
+    }
+
+    if (NI->mayReadOrWriteMemory()) {
+      MDNode *UnitedAccGroups = uniteAccessGroups(
+          NI->getMetadata(LLVMContext::MD_access_group), CallAccessGroup);
+      NI->setMetadata(LLVMContext::MD_access_group, UnitedAccGroups);
     }
   }
 }
@@ -987,22 +998,22 @@ static void AddAliasScopeMetadata(CallSite CS, ValueToValueMapTy &VMap,
         PtrArgs.push_back(CXI->getPointerOperand());
       else if (const AtomicRMWInst *RMWI = dyn_cast<AtomicRMWInst>(I))
         PtrArgs.push_back(RMWI->getPointerOperand());
-      else if (ImmutableCallSite ICS = ImmutableCallSite(I)) {
+      else if (const auto *Call = dyn_cast<CallBase>(I)) {
         // If we know that the call does not access memory, then we'll still
         // know that about the inlined clone of this call site, and we don't
         // need to add metadata.
-        if (ICS.doesNotAccessMemory())
+        if (Call->doesNotAccessMemory())
           continue;
 
         IsFuncCall = true;
         if (CalleeAAR) {
-          FunctionModRefBehavior MRB = CalleeAAR->getModRefBehavior(ICS);
+          FunctionModRefBehavior MRB = CalleeAAR->getModRefBehavior(Call);
           if (MRB == FMRB_OnlyAccessesArgumentPointees ||
               MRB == FMRB_OnlyReadsArgumentPointees)
             IsArgMemOnlyCall = true;
         }
 
-        for (Value *Arg : ICS.args()) {
+        for (Value *Arg : Call->args()) {
           // We need to check the underlying objects of all arguments, not just
           // the pointer arguments, because we might be passing pointers as
           // integers, etc.
@@ -1308,16 +1319,10 @@ static Value *HandleByValArgument(Value *Arg, Instruction *TheCall,
 
 // Check whether this Value is used by a lifetime intrinsic.
 static bool isUsedByLifetimeMarker(Value *V) {
-  for (User *U : V->users()) {
-    if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(U)) {
-      switch (II->getIntrinsicID()) {
-      default: break;
-      case Intrinsic::lifetime_start:
-      case Intrinsic::lifetime_end:
+  for (User *U : V->users())
+    if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(U))
+      if (II->isLifetimeStartOrEnd())
         return true;
-      }
-    }
-  }
   return false;
 }
 
@@ -1442,47 +1447,45 @@ static void updateCallProfile(Function *Callee, const ValueToValueMapTy &VMap,
       CalleeEntryCount.getCount() < 1)
     return;
   auto CallSiteCount = PSI ? PSI->getProfileCount(TheCall, CallerBFI) : None;
-  uint64_t CallCount =
+  int64_t CallCount =
       std::min(CallSiteCount.hasValue() ? CallSiteCount.getValue() : 0,
                CalleeEntryCount.getCount());
-
-  for (auto const &Entry : VMap)
-    if (isa<CallInst>(Entry.first))
-      if (auto *CI = dyn_cast_or_null<CallInst>(Entry.second))
-        CI->updateProfWeight(CallCount, CalleeEntryCount.getCount());
-  for (BasicBlock &BB : *Callee)
-    // No need to update the callsite if it is pruned during inlining.
-    if (VMap.count(&BB))
-      for (Instruction &I : BB)
-        if (CallInst *CI = dyn_cast<CallInst>(&I))
-          CI->updateProfWeight(CalleeEntryCount.getCount() - CallCount,
-                               CalleeEntryCount.getCount());
+  updateProfileCallee(Callee, -CallCount, &VMap);
 }
 
-/// Update the entry count of callee after inlining.
-///
-/// The callsite's block count is subtracted from the callee's function entry
-/// count.
-static void updateCalleeCount(BlockFrequencyInfo *CallerBFI, BasicBlock *CallBB,
-                              Instruction *CallInst, Function *Callee,
-                              ProfileSummaryInfo *PSI) {
-  // If the callee has a original count of N, and the estimated count of
-  // callsite is M, the new callee count is set to N - M. M is estimated from
-  // the caller's entry count, its entry block frequency and the block frequency
-  // of the callsite.
+void llvm::updateProfileCallee(
+    Function *Callee, int64_t entryDelta,
+    const ValueMap<const Value *, WeakTrackingVH> *VMap) {
   auto CalleeCount = Callee->getEntryCount();
-  if (!CalleeCount.hasValue() || !PSI)
-    return;
-  auto CallCount = PSI->getProfileCount(CallInst, CallerBFI);
-  if (!CallCount.hasValue())
+  if (!CalleeCount.hasValue())
     return;
+
+  uint64_t priorEntryCount = CalleeCount.getCount();
+  uint64_t newEntryCount = priorEntryCount;
+
   // Since CallSiteCount is an estimate, it could exceed the original callee
-  // count and has to be set to 0.
-  if (CallCount.getValue() > CalleeCount.getCount())
-    CalleeCount.setCount(0);
+  // count and has to be set to 0 so guard against underflow.
+  if (entryDelta < 0 && static_cast<uint64_t>(-entryDelta) > priorEntryCount)
+    newEntryCount = 0;
   else
-    CalleeCount.setCount(CalleeCount.getCount() - CallCount.getValue());
-  Callee->setEntryCount(CalleeCount);
+    newEntryCount = priorEntryCount + entryDelta;
+
+  Callee->setEntryCount(newEntryCount);
+
+  // During inlining ?
+  if (VMap) {
+    uint64_t cloneEntryCount = priorEntryCount - newEntryCount;
+    for (auto const &Entry : *VMap)
+      if (isa<CallInst>(Entry.first))
+        if (auto *CI = dyn_cast_or_null<CallInst>(Entry.second))
+          CI->updateProfWeight(cloneEntryCount, priorEntryCount);
+  }
+  for (BasicBlock &BB : *Callee)
+    // No need to update the callsite if it is pruned during inlining.
+    if (!VMap || VMap->count(&BB))
+      for (Instruction &I : BB)
+        if (CallInst *CI = dyn_cast<CallInst>(&I))
+          CI->updateProfWeight(newEntryCount, priorEntryCount);
 }
 
 /// This function inlines the called function into the basic block of the
@@ -1501,6 +1504,10 @@ llvm::InlineResult llvm::InlineFunction(CallSite CS, InlineFunctionInfo &IFI,
   assert(TheCall->getParent() && TheCall->getFunction()
          && "Instruction not in function!");
 
+  // FIXME: we don't inline callbr yet.
+  if (isa<CallBrInst>(TheCall))
+    return false;
+
   // If IFI has any state in it, zap it before we fill it in.
   IFI.reset();
 
@@ -1678,8 +1685,6 @@ llvm::InlineResult llvm::InlineFunction(CallSite CS, InlineFunctionInfo &IFI,
 
     updateCallProfile(CalledFunc, VMap, CalledFunc->getEntryCount(), TheCall,
                       IFI.PSI, IFI.CallerBFI);
-    // Update the profile count of callee.
-    updateCalleeCount(IFI.CallerBFI, OrigBB, TheCall, CalledFunc, IFI.PSI);
 
     // Inject byval arguments initialization.
     for (std::pair<Value*, Value*> &Init : ByValInit)
@@ -1728,6 +1733,8 @@ llvm::InlineResult llvm::InlineFunction(CallSite CS, InlineFunctionInfo &IFI,
         Instruction *NewI = nullptr;
         if (isa<CallInst>(I))
           NewI = CallInst::Create(cast<CallInst>(I), OpDefs, I);
+        else if (isa<CallBrInst>(I))
+          NewI = CallBrInst::Create(cast<CallBrInst>(I), OpDefs, I);
         else
           NewI = InvokeInst::Create(cast<InvokeInst>(I), OpDefs, I);
 
@@ -1863,10 +1870,8 @@ llvm::InlineResult llvm::InlineFunction(CallSite CS, InlineFunctionInfo &IFI,
           // Add VarArgs to existing parameters.
           SmallVector<Value *, 6> Params(CI->arg_operands());
           Params.append(VarArgsToForward.begin(), VarArgsToForward.end());
-          CallInst *NewCI =
-              CallInst::Create(CI->getCalledFunction() ? CI->getCalledFunction()
-                                                       : CI->getCalledValue(),
-                               Params, "", CI);
+          CallInst *NewCI = CallInst::Create(
+              CI->getFunctionType(), CI->getCalledOperand(), Params, "", CI);
           NewCI->setDebugLoc(CI->getDebugLoc());
           NewCI->setAttributes(Attrs);
           NewCI->setCallingConv(CI->getCallingConv());
@@ -2032,6 +2037,8 @@ llvm::InlineResult llvm::InlineFunction(CallSite CS, InlineFunctionInfo &IFI,
         Instruction *NewInst;
         if (CS.isCall())
           NewInst = CallInst::Create(cast<CallInst>(I), OpBundles, I);
+        else if (CS.isCallBr())
+          NewInst = CallBrInst::Create(cast<CallBrInst>(I), OpBundles, I);
         else
           NewInst = InvokeInst::Create(cast<InvokeInst>(I), OpBundles, I);
         NewInst->takeName(I);
diff --git a/lib/Transforms/Utils/InstructionNamer.cpp b/lib/Transforms/Utils/InstructionNamer.cpp
index 003721f2b939737efab59ace32f42364e71e0649..6c4fc1ceb991351e3c66cf37ef50637bccefb66b 100644
--- a/lib/Transforms/Utils/InstructionNamer.cpp
+++ b/lib/Transforms/Utils/InstructionNamer.cpp
@@ -1,9 +1,8 @@
 //===- InstructionNamer.cpp - Give anonymous instructions names -----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Transforms/Utils/IntegerDivision.cpp b/lib/Transforms/Utils/IntegerDivision.cpp
index 4a359b99bebdba816ba733fa37a3158330e0b77a..9082049c82da67ce8a9d5df6219f33917f280070 100644
--- a/lib/Transforms/Utils/IntegerDivision.cpp
+++ b/lib/Transforms/Utils/IntegerDivision.cpp
@@ -1,9 +1,8 @@
 //===-- IntegerDivision.cpp - Expand integer division ---------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Transforms/Utils/LCSSA.cpp b/lib/Transforms/Utils/LCSSA.cpp
index 53d444b309d5d972171d8598d95b8f683521f5f4..73b2e4326c481743b2f27077101c556ad9cb126a 100644
--- a/lib/Transforms/Utils/LCSSA.cpp
+++ b/lib/Transforms/Utils/LCSSA.cpp
@@ -1,9 +1,8 @@
 //===-- LCSSA.cpp - Convert loops into loop-closed SSA form ---------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -198,6 +197,17 @@ bool llvm::formLCSSAForInstructions(SmallVectorImpl<Instruction *> &Worklist,
         continue;
       }
 
+      // If we added a single PHI, it must dominate all uses and we can directly
+      // rename it.
+      if (AddedPHIs.size() == 1) {
+        // Tell the VHs that the uses changed. This updates SCEV's caches.
+        // We might call ValueIsRAUWd multiple times for the same value.
+        if (UseToRewrite->get()->hasValueHandle())
+          ValueHandleBase::ValueIsRAUWd(*UseToRewrite, AddedPHIs[0]);
+        UseToRewrite->set(AddedPHIs[0]);
+        continue;
+      }
+
       // Otherwise, do full PHI insertion.
       SSAUpdate.RewriteUse(*UseToRewrite);
     }
@@ -211,9 +221,12 @@ bool llvm::formLCSSAForInstructions(SmallVectorImpl<Instruction *> &Worklist,
       BasicBlock *UserBB = DVI->getParent();
       if (InstBB == UserBB || L->contains(UserBB))
         continue;
-      // We currently only handle debug values residing in blocks where we have
-      // inserted a PHI instruction.
-      if (Value *V = SSAUpdate.FindValueForBlock(UserBB))
+      // We currently only handle debug values residing in blocks that were
+      // traversed while rewriting the uses. If we inserted just a single PHI,
+      // we will handle all relevant debug values.
+      Value *V = AddedPHIs.size() == 1 ? AddedPHIs[0]
+                                       : SSAUpdate.FindValueForBlock(UserBB);
+      if (V)
         DVI->setOperand(0, MetadataAsValue::get(Ctx, ValueAsMetadata::get(V)));
     }
 
@@ -306,6 +319,12 @@ bool llvm::formLCSSA(Loop &L, DominatorTree &DT, LoopInfo *LI,
                      ScalarEvolution *SE) {
   bool Changed = false;
 
+#ifdef EXPENSIVE_CHECKS
+  // Verify all sub-loops are in LCSSA form already.
+  for (Loop *SubLoop: L)
+    assert(SubLoop->isRecursivelyLCSSAForm(DT, *LI) && "Subloop not in LCSSA!");
+#endif
+
   SmallVector<BasicBlock *, 8> ExitBlocks;
   L.getExitBlocks(ExitBlocks);
   if (ExitBlocks.empty())
@@ -325,6 +344,10 @@ bool llvm::formLCSSA(Loop &L, DominatorTree &DT, LoopInfo *LI,
   // Look at all the instructions in the loop, checking to see if they have uses
   // outside the loop.  If so, put them into the worklist to rewrite those uses.
   for (BasicBlock *BB : BlocksDominatingExits) {
+    // Skip blocks that are part of any sub-loops, they must be in LCSSA
+    // already.
+    if (LI->getLoopFor(BB) != &L)
+      continue;
     for (Instruction &I : *BB) {
       // Reject two common cases fast: instructions with no uses (like stores)
       // and instructions with one use that is in the same block as this.
diff --git a/lib/Transforms/Utils/LLVMBuild.txt b/lib/Transforms/Utils/LLVMBuild.txt
index ece0ad4dbf4404eb8e539c386bf148192391adc8..df9a023b5132b71383d3a26f4895cbb9f5aa4fbd 100644
--- a/lib/Transforms/Utils/LLVMBuild.txt
+++ b/lib/Transforms/Utils/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/Transforms/Utils/LLVMBuild.txt ---------------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Transforms/Utils/LibCallsShrinkWrap.cpp b/lib/Transforms/Utils/LibCallsShrinkWrap.cpp
index e1592c86763653e1f381ce84bcf066127aa3695e..8c67d1dc6eb339e600cfe9e3e15f23e4ad9e4f8f 100644
--- a/lib/Transforms/Utils/LibCallsShrinkWrap.cpp
+++ b/lib/Transforms/Utils/LibCallsShrinkWrap.cpp
@@ -1,9 +1,8 @@
 //===-- LibCallsShrinkWrap.cpp ----------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Transforms/Utils/Local.cpp b/lib/Transforms/Utils/Local.cpp
index 79343a90a0e3af394cefead8ab3d0ccd821e4a3b..12c4ff1e37c23519628401c0692d2d09af7166f9 100644
--- a/lib/Transforms/Utils/Local.cpp
+++ b/lib/Transforms/Utils/Local.cpp
@@ -1,9 +1,8 @@
 //===- Local.cpp - Functions to perform local transformations -------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -27,6 +26,7 @@
 #include "llvm/ADT/Statistic.h"
 #include "llvm/ADT/TinyPtrVector.h"
 #include "llvm/Analysis/ConstantFolding.h"
+#include "llvm/Analysis/DomTreeUpdater.h"
 #include "llvm/Analysis/EHPersonalities.h"
 #include "llvm/Analysis/InstructionSimplify.h"
 #include "llvm/Analysis/LazyValueInfo.h"
@@ -34,6 +34,7 @@
 #include "llvm/Analysis/MemorySSAUpdater.h"
 #include "llvm/Analysis/TargetLibraryInfo.h"
 #include "llvm/Analysis/ValueTracking.h"
+#include "llvm/Analysis/VectorUtils.h"
 #include "llvm/BinaryFormat/Dwarf.h"
 #include "llvm/IR/Argument.h"
 #include "llvm/IR/Attributes.h"
@@ -48,7 +49,6 @@
 #include "llvm/IR/DebugInfoMetadata.h"
 #include "llvm/IR/DebugLoc.h"
 #include "llvm/IR/DerivedTypes.h"
-#include "llvm/IR/DomTreeUpdater.h"
 #include "llvm/IR/Dominators.h"
 #include "llvm/IR/Function.h"
 #include "llvm/IR/GetElementPtrTypeIterator.h"
@@ -392,8 +392,7 @@ bool llvm::wouldInstructionBeTriviallyDead(Instruction *I,
       return true;
 
     // Lifetime intrinsics are dead when their right-hand is undef.
-    if (II->getIntrinsicID() == Intrinsic::lifetime_start ||
-        II->getIntrinsicID() == Intrinsic::lifetime_end)
+    if (II->isLifetimeStartOrEnd())
       return isa<UndefValue>(II->getArgOperand(1));
 
     // Assumptions are dead if their condition is trivially true.  Guards on
@@ -416,8 +415,8 @@ bool llvm::wouldInstructionBeTriviallyDead(Instruction *I,
     if (Constant *C = dyn_cast<Constant>(CI->getArgOperand(0)))
       return C->isNullValue() || isa<UndefValue>(C);
 
-  if (CallSite CS = CallSite(I))
-    if (isMathLibCallNoop(CS, TLI))
+  if (auto *Call = dyn_cast<CallBase>(I))
+    if (isMathLibCallNoop(Call, TLI))
       return true;
 
   return false;
@@ -430,7 +429,7 @@ bool llvm::wouldInstructionBeTriviallyDead(Instruction *I,
 bool llvm::RecursivelyDeleteTriviallyDeadInstructions(
     Value *V, const TargetLibraryInfo *TLI, MemorySSAUpdater *MSSAU) {
   Instruction *I = dyn_cast<Instruction>(V);
-  if (!I || !I->use_empty() || !isInstructionTriviallyDead(I, TLI))
+  if (!I || !isInstructionTriviallyDead(I, TLI))
     return false;
 
   SmallVector<Instruction*, 16> DeadInsts;
@@ -997,6 +996,18 @@ bool llvm::TryToSimplifyUncondBranchFromEmptyBlock(BasicBlock *BB,
     }
   }
 
+  // We cannot fold the block if it's a branch to an already present callbr
+  // successor because that creates duplicate successors.
+  for (auto I = pred_begin(BB), E = pred_end(BB); I != E; ++I) {
+    if (auto *CBI = dyn_cast<CallBrInst>((*I)->getTerminator())) {
+      if (Succ == CBI->getDefaultDest())
+        return false;
+      for (unsigned i = 0, e = CBI->getNumIndirectDests(); i != e; ++i)
+        if (Succ == CBI->getIndirectDest(i))
+          return false;
+    }
+  }
+
   LLVM_DEBUG(dbgs() << "Killing Trivial BB: \n" << *BB);
 
   SmallVector<DominatorTree::UpdateType, 32> Updates;
@@ -1594,120 +1605,120 @@ bool llvm::salvageDebugInfo(Instruction &I) {
   if (DbgUsers.empty())
     return false;
 
-  auto &M = *I.getModule();
-  auto &DL = M.getDataLayout();
+  return salvageDebugInfoForDbgValues(I, DbgUsers);
+}
+
+bool llvm::salvageDebugInfoForDbgValues(
+    Instruction &I, ArrayRef<DbgVariableIntrinsic *> DbgUsers) {
   auto &Ctx = I.getContext();
   auto wrapMD = [&](Value *V) { return wrapValueInMetadata(Ctx, V); };
 
-  auto doSalvage = [&](DbgVariableIntrinsic *DII, SmallVectorImpl<uint64_t> &Ops) {
-    auto *DIExpr = DII->getExpression();
-    if (!Ops.empty()) {
-      // Do not add DW_OP_stack_value for DbgDeclare and DbgAddr, because they
-      // are implicitly pointing out the value as a DWARF memory location
-      // description.
-      bool WithStackValue = isa<DbgValueInst>(DII);
-      DIExpr = DIExpression::prependOpcodes(DIExpr, Ops, WithStackValue);
-    }
+  for (auto *DII : DbgUsers) {
+    // Do not add DW_OP_stack_value for DbgDeclare and DbgAddr, because they
+    // are implicitly pointing out the value as a DWARF memory location
+    // description.
+    bool StackValue = isa<DbgValueInst>(DII);
+
+    DIExpression *DIExpr =
+        salvageDebugInfoImpl(I, DII->getExpression(), StackValue);
+
+    // salvageDebugInfoImpl should fail on examining the first element of
+    // DbgUsers, or none of them.
+    if (!DIExpr)
+      return false;
+
     DII->setOperand(0, wrapMD(I.getOperand(0)));
     DII->setOperand(2, MetadataAsValue::get(Ctx, DIExpr));
     LLVM_DEBUG(dbgs() << "SALVAGE: " << *DII << '\n');
+  }
+
+  return true;
+}
+
+DIExpression *llvm::salvageDebugInfoImpl(Instruction &I,
+                                         DIExpression *SrcDIExpr,
+                                         bool WithStackValue) {
+  auto &M = *I.getModule();
+  auto &DL = M.getDataLayout();
+
+  // Apply a vector of opcodes to the source DIExpression.
+  auto doSalvage = [&](SmallVectorImpl<uint64_t> &Ops) -> DIExpression * {
+    DIExpression *DIExpr = SrcDIExpr;
+    if (!Ops.empty()) {
+      DIExpr = DIExpression::prependOpcodes(DIExpr, Ops, WithStackValue);
+    }
+    return DIExpr;
   };
 
-  auto applyOffset = [&](DbgVariableIntrinsic *DII, uint64_t Offset) {
+  // Apply the given offset to the source DIExpression.
+  auto applyOffset = [&](uint64_t Offset) -> DIExpression * {
     SmallVector<uint64_t, 8> Ops;
     DIExpression::appendOffset(Ops, Offset);
-    doSalvage(DII, Ops);
+    return doSalvage(Ops);
   };
 
-  auto applyOps = [&](DbgVariableIntrinsic *DII,
-                      std::initializer_list<uint64_t> Opcodes) {
+  // initializer-list helper for applying operators to the source DIExpression.
+  auto applyOps =
+      [&](std::initializer_list<uint64_t> Opcodes) -> DIExpression * {
     SmallVector<uint64_t, 8> Ops(Opcodes);
-    doSalvage(DII, Ops);
+    return doSalvage(Ops);
   };
 
   if (auto *CI = dyn_cast<CastInst>(&I)) {
     if (!CI->isNoopCast(DL))
-      return false;
+      return nullptr;
 
     // No-op casts are irrelevant for debug info.
-    MetadataAsValue *CastSrc = wrapMD(I.getOperand(0));
-    for (auto *DII : DbgUsers) {
-      DII->setOperand(0, CastSrc);
-      LLVM_DEBUG(dbgs() << "SALVAGE: " << *DII << '\n');
-    }
-    return true;
+    return SrcDIExpr;
   } else if (auto *GEP = dyn_cast<GetElementPtrInst>(&I)) {
     unsigned BitWidth =
         M.getDataLayout().getIndexSizeInBits(GEP->getPointerAddressSpace());
-    // Rewrite a constant GEP into a DIExpression.  Since we are performing
-    // arithmetic to compute the variable's *value* in the DIExpression, we
-    // need to mark the expression with a DW_OP_stack_value.
+    // Rewrite a constant GEP into a DIExpression.
     APInt Offset(BitWidth, 0);
-    if (GEP->accumulateConstantOffset(M.getDataLayout(), Offset))
-      for (auto *DII : DbgUsers)
-        applyOffset(DII, Offset.getSExtValue());
-    return true;
+    if (GEP->accumulateConstantOffset(M.getDataLayout(), Offset)) {
+      return applyOffset(Offset.getSExtValue());
+    } else {
+      return nullptr;
+    }
   } else if (auto *BI = dyn_cast<BinaryOperator>(&I)) {
     // Rewrite binary operations with constant integer operands.
     auto *ConstInt = dyn_cast<ConstantInt>(I.getOperand(1));
     if (!ConstInt || ConstInt->getBitWidth() > 64)
-      return false;
+      return nullptr;
 
     uint64_t Val = ConstInt->getSExtValue();
-    for (auto *DII : DbgUsers) {
-      switch (BI->getOpcode()) {
-      case Instruction::Add:
-        applyOffset(DII, Val);
-        break;
-      case Instruction::Sub:
-        applyOffset(DII, -int64_t(Val));
-        break;
-      case Instruction::Mul:
-        applyOps(DII, {dwarf::DW_OP_constu, Val, dwarf::DW_OP_mul});
-        break;
-      case Instruction::SDiv:
-        applyOps(DII, {dwarf::DW_OP_constu, Val, dwarf::DW_OP_div});
-        break;
-      case Instruction::SRem:
-        applyOps(DII, {dwarf::DW_OP_constu, Val, dwarf::DW_OP_mod});
-        break;
-      case Instruction::Or:
-        applyOps(DII, {dwarf::DW_OP_constu, Val, dwarf::DW_OP_or});
-        break;
-      case Instruction::And:
-        applyOps(DII, {dwarf::DW_OP_constu, Val, dwarf::DW_OP_and});
-        break;
-      case Instruction::Xor:
-        applyOps(DII, {dwarf::DW_OP_constu, Val, dwarf::DW_OP_xor});
-        break;
-      case Instruction::Shl:
-        applyOps(DII, {dwarf::DW_OP_constu, Val, dwarf::DW_OP_shl});
-        break;
-      case Instruction::LShr:
-        applyOps(DII, {dwarf::DW_OP_constu, Val, dwarf::DW_OP_shr});
-        break;
-      case Instruction::AShr:
-        applyOps(DII, {dwarf::DW_OP_constu, Val, dwarf::DW_OP_shra});
-        break;
-      default:
-        // TODO: Salvage constants from each kind of binop we know about.
-        return false;
-      }
-    }
-    return true;
-  } else if (isa<LoadInst>(&I)) {
-    MetadataAsValue *AddrMD = wrapMD(I.getOperand(0));
-    for (auto *DII : DbgUsers) {
-      // Rewrite the load into DW_OP_deref.
-      auto *DIExpr = DII->getExpression();
-      DIExpr = DIExpression::prepend(DIExpr, DIExpression::WithDeref);
-      DII->setOperand(0, AddrMD);
-      DII->setOperand(2, MetadataAsValue::get(Ctx, DIExpr));
-      LLVM_DEBUG(dbgs() << "SALVAGE:  " << *DII << '\n');
+    switch (BI->getOpcode()) {
+    case Instruction::Add:
+      return applyOffset(Val);
+    case Instruction::Sub:
+      return applyOffset(-int64_t(Val));
+    case Instruction::Mul:
+      return applyOps({dwarf::DW_OP_constu, Val, dwarf::DW_OP_mul});
+    case Instruction::SDiv:
+      return applyOps({dwarf::DW_OP_constu, Val, dwarf::DW_OP_div});
+    case Instruction::SRem:
+      return applyOps({dwarf::DW_OP_constu, Val, dwarf::DW_OP_mod});
+    case Instruction::Or:
+      return applyOps({dwarf::DW_OP_constu, Val, dwarf::DW_OP_or});
+    case Instruction::And:
+      return applyOps({dwarf::DW_OP_constu, Val, dwarf::DW_OP_and});
+    case Instruction::Xor:
+      return applyOps({dwarf::DW_OP_constu, Val, dwarf::DW_OP_xor});
+    case Instruction::Shl:
+      return applyOps({dwarf::DW_OP_constu, Val, dwarf::DW_OP_shl});
+    case Instruction::LShr:
+      return applyOps({dwarf::DW_OP_constu, Val, dwarf::DW_OP_shr});
+    case Instruction::AShr:
+      return applyOps({dwarf::DW_OP_constu, Val, dwarf::DW_OP_shra});
+    default:
+      // TODO: Salvage constants from each kind of binop we know about.
+      return nullptr;
     }
-    return true;
+    // *Not* to do: we should not attempt to salvage load instructions,
+    // because the validity and lifetime of a dbg.value containing
+    // DW_OP_deref becomes difficult to analyze. See PR40628 for examples.
   }
-  return false;
+  return nullptr;
 }
 
 /// A replacement for a dbg.value expression.
@@ -1937,8 +1948,8 @@ static void changeToCall(InvokeInst *II, DomTreeUpdater *DTU = nullptr) {
   SmallVector<Value*, 8> Args(II->arg_begin(), II->arg_end());
   SmallVector<OperandBundleDef, 1> OpBundles;
   II->getOperandBundlesAsDefs(OpBundles);
-  CallInst *NewCall = CallInst::Create(II->getCalledValue(), Args, OpBundles,
-                                       "", II);
+  CallInst *NewCall = CallInst::Create(
+      II->getFunctionType(), II->getCalledValue(), Args, OpBundles, "", II);
   NewCall->takeName(II);
   NewCall->setCallingConv(II->getCallingConv());
   NewCall->setAttributes(II->getAttributes());
@@ -1981,8 +1992,9 @@ BasicBlock *llvm::changeToInvokeAndSplitBasicBlock(CallInst *CI,
   // can potentially be avoided with a cleverer API design that we do not have
   // as of this time.
 
-  InvokeInst *II = InvokeInst::Create(CI->getCalledValue(), Split, UnwindEdge,
-                                      InvokeArgs, OpBundles, CI->getName(), BB);
+  InvokeInst *II =
+      InvokeInst::Create(CI->getFunctionType(), CI->getCalledValue(), Split,
+                         UnwindEdge, InvokeArgs, OpBundles, CI->getName(), BB);
   II->setDebugLoc(CI->getDebugLoc());
   II->setCallingConv(CI->getCallingConv());
   II->setAttributes(CI->getAttributes());
@@ -2297,6 +2309,10 @@ void llvm::combineMetadata(Instruction *K, const Instruction *J,
       case LLVMContext::MD_mem_parallel_loop_access:
         K->setMetadata(Kind, MDNode::intersect(JMD, KMD));
         break;
+      case LLVMContext::MD_access_group:
+        K->setMetadata(LLVMContext::MD_access_group,
+                       intersectAccessGroups(K, J));
+        break;
       case LLVMContext::MD_range:
 
         // If K does move, use most generic range. Otherwise keep the range of
@@ -2353,7 +2369,8 @@ void llvm::combineMetadataForCSE(Instruction *K, const Instruction *J,
       LLVMContext::MD_invariant_load,  LLVMContext::MD_nonnull,
       LLVMContext::MD_invariant_group, LLVMContext::MD_align,
       LLVMContext::MD_dereferenceable,
-      LLVMContext::MD_dereferenceable_or_null};
+      LLVMContext::MD_dereferenceable_or_null,
+      LLVMContext::MD_access_group};
   combineMetadata(K, J, KnownIDs, KDominatesJ);
 }
 
@@ -2384,7 +2401,8 @@ void llvm::patchReplacementInstruction(Instruction *I, Value *Repl) {
       LLVMContext::MD_tbaa,            LLVMContext::MD_alias_scope,
       LLVMContext::MD_noalias,         LLVMContext::MD_range,
       LLVMContext::MD_fpmath,          LLVMContext::MD_invariant_load,
-      LLVMContext::MD_invariant_group, LLVMContext::MD_nonnull};
+      LLVMContext::MD_invariant_group, LLVMContext::MD_nonnull,
+      LLVMContext::MD_access_group};
   combineMetadata(ReplInst, I, KnownIDs, false);
 }
 
@@ -2444,12 +2462,12 @@ unsigned llvm::replaceDominatedUsesWith(Value *From, Value *To,
   return ::replaceDominatedUsesWith(From, To, BB, ProperlyDominates);
 }
 
-bool llvm::callsGCLeafFunction(ImmutableCallSite CS,
+bool llvm::callsGCLeafFunction(const CallBase *Call,
                                const TargetLibraryInfo &TLI) {
   // Check if the function is specifically marked as a gc leaf function.
-  if (CS.hasFnAttr("gc-leaf-function"))
+  if (Call->hasFnAttr("gc-leaf-function"))
     return true;
-  if (const Function *F = CS.getCalledFunction()) {
+  if (const Function *F = Call->getCalledFunction()) {
     if (F->hasFnAttribute("gc-leaf-function"))
       return true;
 
@@ -2463,7 +2481,7 @@ bool llvm::callsGCLeafFunction(ImmutableCallSite CS,
   // marked as 'gc-leaf-function.' All available Libcalls are
   // GC-leaf.
   LibFunc LF;
-  if (TLI.getLibFunc(CS, LF)) {
+  if (TLI.getLibFunc(ImmutableCallSite(Call), LF)) {
     return TLI.has(LF);
   }
 
@@ -2524,13 +2542,13 @@ void llvm::hoistAllInstructionsInto(BasicBlock *DomBlock, Instruction *InsertPt,
                                     BasicBlock *BB) {
   // Since we are moving the instructions out of its basic block, we do not
   // retain their original debug locations (DILocations) and debug intrinsic
-  // instructions (dbg.values).
+  // instructions.
   //
   // Doing so would degrade the debugging experience and adversely affect the
   // accuracy of profiling information.
   //
   // Currently, when hoisting the instructions, we take the following actions:
-  // - Remove their dbg.values.
+  // - Remove their debug intrinsic instructions.
   // - Set their debug locations to the values from the insertion point.
   //
   // As per PR39141 (comment #8), the more fundamental reason why the dbg.values
@@ -2548,7 +2566,7 @@ void llvm::hoistAllInstructionsInto(BasicBlock *DomBlock, Instruction *InsertPt,
     I->dropUnknownNonDebugMetadata();
     if (I->isUsedByMetadata())
       dropDebugUsers(*I);
-    if (isa<DbgVariableIntrinsic>(I)) {
+    if (isa<DbgInfoIntrinsic>(I)) {
       // Remove DbgInfo Intrinsics.
       II = I->eraseFromParent();
       continue;
diff --git a/lib/Transforms/Utils/LoopRotationUtils.cpp b/lib/Transforms/Utils/LoopRotationUtils.cpp
index 41f14a8346178fdd1da9cb50d1a30a8308b01bd6..a38d34932e1f37d245afb6e74c5c1d26bd342964 100644
--- a/lib/Transforms/Utils/LoopRotationUtils.cpp
+++ b/lib/Transforms/Utils/LoopRotationUtils.cpp
@@ -1,9 +1,8 @@
 //===----------------- LoopRotationUtils.cpp -----------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -17,6 +16,7 @@
 #include "llvm/Analysis/AssumptionCache.h"
 #include "llvm/Analysis/BasicAliasAnalysis.h"
 #include "llvm/Analysis/CodeMetrics.h"
+#include "llvm/Analysis/DomTreeUpdater.h"
 #include "llvm/Analysis/GlobalsModRef.h"
 #include "llvm/Analysis/InstructionSimplify.h"
 #include "llvm/Analysis/LoopPass.h"
@@ -28,7 +28,6 @@
 #include "llvm/Analysis/ValueTracking.h"
 #include "llvm/IR/CFG.h"
 #include "llvm/IR/DebugInfoMetadata.h"
-#include "llvm/IR/DomTreeUpdater.h"
 #include "llvm/IR/Dominators.h"
 #include "llvm/IR/Function.h"
 #include "llvm/IR/IntrinsicInst.h"
diff --git a/lib/Transforms/Utils/LoopSimplify.cpp b/lib/Transforms/Utils/LoopSimplify.cpp
index 380f4fca54d9ed12dab65fbf969b455ce7034cef..ca29f6787e22384f7f939f29c3b5dcd567c17d2b 100644
--- a/lib/Transforms/Utils/LoopSimplify.cpp
+++ b/lib/Transforms/Utils/LoopSimplify.cpp
@@ -1,9 +1,8 @@
 //===- LoopSimplify.cpp - Loop Canonicalization Pass ----------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -28,6 +27,9 @@
 // to transform the loop and make these guarantees. Client code should check
 // that these conditions are true before relying on them.
 //
+// Similar complications arise from callbr instructions, particularly in
+// asm-goto where blockaddress expressions are used.
+//
 // Note that the simplifycfg pass will clean up blocks which are split out but
 // end up being unnecessary, so usage of this pass should not pessimize
 // generated code.
@@ -124,10 +126,11 @@ BasicBlock *llvm::InsertPreheaderForLoop(Loop *L, DominatorTree *DT,
        PI != PE; ++PI) {
     BasicBlock *P = *PI;
     if (!L->contains(P)) {         // Coming in from outside the loop?
-      // If the loop is branched to from an indirect branch, we won't
+      // If the loop is branched to from an indirect terminator, we won't
       // be able to fully transform the loop, because it prohibits
       // edge splitting.
-      if (isa<IndirectBrInst>(P->getTerminator())) return nullptr;
+      if (P->getTerminator()->isIndirectTerminator())
+        return nullptr;
 
       // Keep track of it.
       OutsideBlocks.push_back(P);
@@ -236,8 +239,8 @@ static Loop *separateNestedLoop(Loop *L, BasicBlock *Preheader,
   for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i) {
     if (PN->getIncomingValue(i) != PN ||
         !L->contains(PN->getIncomingBlock(i))) {
-      // We can't split indirectbr edges.
-      if (isa<IndirectBrInst>(PN->getIncomingBlock(i)->getTerminator()))
+      // We can't split indirect control flow edges.
+      if (PN->getIncomingBlock(i)->getTerminator()->isIndirectTerminator())
         return nullptr;
       OuterLoopPreds.push_back(PN->getIncomingBlock(i));
     }
@@ -358,8 +361,8 @@ static BasicBlock *insertUniqueBackedgeBlock(Loop *L, BasicBlock *Preheader,
   for (pred_iterator I = pred_begin(Header), E = pred_end(Header); I != E; ++I){
     BasicBlock *P = *I;
 
-    // Indirectbr edges cannot be split, so we must fail if we find one.
-    if (isa<IndirectBrInst>(P->getTerminator()))
+    // Indirect edges cannot be split, so we must fail if we find one.
+    if (P->getTerminator()->isIndirectTerminator())
       return nullptr;
 
     if (P != Preheader) BackedgeBlocks.push_back(P);
@@ -659,9 +662,9 @@ ReprocessLoop:
       DT->eraseNode(ExitingBlock);
 
       BI->getSuccessor(0)->removePredecessor(
-          ExitingBlock, /* DontDeleteUselessPHIs */ PreserveLCSSA);
+          ExitingBlock, /* KeepOneInputPHIs */ PreserveLCSSA);
       BI->getSuccessor(1)->removePredecessor(
-          ExitingBlock, /* DontDeleteUselessPHIs */ PreserveLCSSA);
+          ExitingBlock, /* KeepOneInputPHIs */ PreserveLCSSA);
       ExitingBlock->eraseFromParent();
     }
   }
diff --git a/lib/Transforms/Utils/LoopUnroll.cpp b/lib/Transforms/Utils/LoopUnroll.cpp
index efd8b92e814d0d7cd2651235c90d2fa3293414bc..0d62a81c923f52ad17ec2606d8a2abe29f082da4 100644
--- a/lib/Transforms/Utils/LoopUnroll.cpp
+++ b/lib/Transforms/Utils/LoopUnroll.cpp
@@ -1,9 +1,8 @@
 //===-- UnrollLoop.cpp - Loop unrolling utilities -------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -54,10 +53,10 @@ UnrollRuntimeEpilog("unroll-runtime-epilog", cl::init(false), cl::Hidden,
 static cl::opt<bool>
 UnrollVerifyDomtree("unroll-verify-domtree", cl::Hidden,
                     cl::desc("Verify domtree after unrolling"),
-#ifdef NDEBUG
-    cl::init(false)
-#else
+#ifdef EXPENSIVE_CHECKS
     cl::init(true)
+#else
+    cl::init(false)
 #endif
                     );
 
@@ -598,8 +597,15 @@ LoopUnrollResult llvm::UnrollLoop(
     for (BasicBlock *BB : L->getBlocks())
       for (Instruction &I : *BB)
         if (!isa<DbgInfoIntrinsic>(&I))
-          if (const DILocation *DIL = I.getDebugLoc())
-            I.setDebugLoc(DIL->cloneWithDuplicationFactor(Count));
+          if (const DILocation *DIL = I.getDebugLoc()) {
+            auto NewDIL = DIL->cloneByMultiplyingDuplicationFactor(Count);
+            if (NewDIL)
+              I.setDebugLoc(NewDIL.getValue());
+            else
+              LLVM_DEBUG(dbgs()
+                         << "Failed to create new discriminator: "
+                         << DIL->getFilename() << " Line: " << DIL->getLine());
+          }
 
   for (unsigned It = 1; It != Count; ++It) {
     std::vector<BasicBlock*> NewBlocks;
diff --git a/lib/Transforms/Utils/LoopUnrollAndJam.cpp b/lib/Transforms/Utils/LoopUnrollAndJam.cpp
index b5d80f669fbf669eae07d12f0d3fb598f105b467..d5ccadde3f92f7f6b06377b4af8bbbd9845329e3 100644
--- a/lib/Transforms/Utils/LoopUnrollAndJam.cpp
+++ b/lib/Transforms/Utils/LoopUnrollAndJam.cpp
@@ -1,9 +1,8 @@
 //===-- LoopUnrollAndJam.cpp - Loop unrolling utilities -------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -300,8 +299,15 @@ LoopUnrollResult llvm::UnrollAndJamLoop(
     for (BasicBlock *BB : L->getBlocks())
       for (Instruction &I : *BB)
         if (!isa<DbgInfoIntrinsic>(&I))
-          if (const DILocation *DIL = I.getDebugLoc())
-            I.setDebugLoc(DIL->cloneWithDuplicationFactor(Count));
+          if (const DILocation *DIL = I.getDebugLoc()) {
+            auto NewDIL = DIL->cloneByMultiplyingDuplicationFactor(Count);
+            if (NewDIL)
+              I.setDebugLoc(NewDIL.getValue());
+            else
+              LLVM_DEBUG(dbgs()
+                         << "Failed to create new discriminator: "
+                         << DIL->getFilename() << " Line: " << DIL->getLine());
+          }
 
   // Copy all blocks
   for (unsigned It = 1; It != Count; ++It) {
diff --git a/lib/Transforms/Utils/LoopUnrollPeel.cpp b/lib/Transforms/Utils/LoopUnrollPeel.cpp
index 86ac1a71d5e3c94b001187b8cdccf41dd12f4883..f893921aca35a03fbd9a2fa74ea8d3be74feefab 100644
--- a/lib/Transforms/Utils/LoopUnrollPeel.cpp
+++ b/lib/Transforms/Utils/LoopUnrollPeel.cpp
@@ -1,9 +1,8 @@
 //===- UnrollLoopPeel.cpp - Loop peeling utilities ------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -615,7 +614,9 @@ bool llvm::peelLoop(Loop *L, unsigned PeelCount, LoopInfo *LI,
       // the original loop body.
       if (Iter == 0)
         DT->changeImmediateDominator(Exit, cast<BasicBlock>(LVMap[Latch]));
+#ifdef EXPENSIVE_CHECKS
       assert(DT->verify(DominatorTree::VerificationLevel::Fast));
+#endif
     }
 
     auto *LatchBRCopy = cast<BranchInst>(VMap[LatchBR]);
diff --git a/lib/Transforms/Utils/LoopUnrollRuntime.cpp b/lib/Transforms/Utils/LoopUnrollRuntime.cpp
index 3606ec4b9fcaa9be8edaaf2748f8381dcf0a4cc4..a9d41945b27c1c4a2db6caa4033cfc2955ea18de 100644
--- a/lib/Transforms/Utils/LoopUnrollRuntime.cpp
+++ b/lib/Transforms/Utils/LoopUnrollRuntime.cpp
@@ -1,9 +1,8 @@
 //===-- UnrollLoopRuntime.cpp - Runtime Loop unrolling utilities ----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -70,6 +69,17 @@ static void ConnectProlog(Loop *L, Value *BECount, unsigned Count,
                           BasicBlock *PreHeader, BasicBlock *NewPreHeader,
                           ValueToValueMapTy &VMap, DominatorTree *DT,
                           LoopInfo *LI, bool PreserveLCSSA) {
+  // Loop structure should be the following:
+  // Preheader
+  //  PrologHeader
+  //  ...
+  //  PrologLatch
+  //  PrologExit
+  //   NewPreheader
+  //    Header
+  //    ...
+  //    Latch
+  //      LatchExit
   BasicBlock *Latch = L->getLoopLatch();
   assert(Latch && "Loop must have a latch");
   BasicBlock *PrologLatch = cast<BasicBlock>(VMap[Latch]);
@@ -83,14 +93,21 @@ static void ConnectProlog(Loop *L, Value *BECount, unsigned Count,
     for (PHINode &PN : Succ->phis()) {
       // Add a new PHI node to the prolog end block and add the
       // appropriate incoming values.
+      // TODO: This code assumes that the PrologExit (or the LatchExit block for
+      // prolog loop) contains only one predecessor from the loop, i.e. the
+      // PrologLatch. When supporting multiple-exiting block loops, we can have
+      // two or more blocks that have the LatchExit as the target in the
+      // original loop.
       PHINode *NewPN = PHINode::Create(PN.getType(), 2, PN.getName() + ".unr",
                                        PrologExit->getFirstNonPHI());
       // Adding a value to the new PHI node from the original loop preheader.
       // This is the value that skips all the prolog code.
       if (L->contains(&PN)) {
+        // Succ is loop header.
         NewPN->addIncoming(PN.getIncomingValueForBlock(NewPreHeader),
                            PreHeader);
       } else {
+        // Succ is LatchExit.
         NewPN->addIncoming(UndefValue::get(PN.getType()), PreHeader);
       }
 
@@ -787,10 +804,7 @@ bool llvm::UnrollRuntimeLoopRemainder(Loop *L, unsigned Count,
   // Now the loop blocks are cloned and the other exiting blocks from the
   // remainder are connected to the original Loop's exit blocks. The remaining
   // work is to update the phi nodes in the original loop, and take in the
-  // values from the cloned region. Also update the dominator info for
-  // OtherExits and their immediate successors, since we have new edges into
-  // OtherExits.
-  SmallPtrSet<BasicBlock*, 8> ImmediateSuccessorsOfExitBlocks;
+  // values from the cloned region.
   for (auto *BB : OtherExits) {
    for (auto &II : *BB) {
 
@@ -825,27 +839,30 @@ bool llvm::UnrollRuntimeLoopRemainder(Loop *L, unsigned Count,
              "Breaks the definition of dedicated exits!");
     }
 #endif
-   // Update the dominator info because the immediate dominator is no longer the
-   // header of the original Loop. BB has edges both from L and remainder code.
-   // Since the preheader determines which loop is run (L or directly jump to
-   // the remainder code), we set the immediate dominator as the preheader.
-   if (DT) {
-     DT->changeImmediateDominator(BB, PreHeader);
-     // Also update the IDom for immediate successors of BB.  If the current
-     // IDom is the header, update the IDom to be the preheader because that is
-     // the nearest common dominator of all predecessors of SuccBB.  We need to
-     // check for IDom being the header because successors of exit blocks can
-     // have edges from outside the loop, and we should not incorrectly update
-     // the IDom in that case.
-     for (BasicBlock *SuccBB: successors(BB))
-       if (ImmediateSuccessorsOfExitBlocks.insert(SuccBB).second) {
-         if (DT->getNode(SuccBB)->getIDom()->getBlock() == Header) {
-           assert(!SuccBB->getSinglePredecessor() &&
-                  "BB should be the IDom then!");
-           DT->changeImmediateDominator(SuccBB, PreHeader);
-         }
-       }
+  }
+
+  // Update the immediate dominator of the exit blocks and blocks that are
+  // reachable from the exit blocks. This is needed because we now have paths
+  // from both the original loop and the remainder code reaching the exit
+  // blocks. While the IDom of these exit blocks were from the original loop,
+  // now the IDom is the preheader (which decides whether the original loop or
+  // remainder code should run).
+  if (DT && !L->getExitingBlock()) {
+    SmallVector<BasicBlock *, 16> ChildrenToUpdate;
+    // NB! We have to examine the dom children of all loop blocks, not just
+    // those which are the IDom of the exit blocks. This is because blocks
+    // reachable from the exit blocks can have their IDom as the nearest common
+    // dominator of the exit blocks.
+    for (auto *BB : L->blocks()) {
+      auto *DomNodeBB = DT->getNode(BB);
+      for (auto *DomChild : DomNodeBB->getChildren()) {
+        auto *DomChildBB = DomChild->getBlock();
+        if (!L->contains(LI->getLoopFor(DomChildBB)))
+          ChildrenToUpdate.push_back(DomChildBB);
+      }
     }
+    for (auto *BB : ChildrenToUpdate)
+      DT->changeImmediateDominator(BB, PreHeader);
   }
 
   // Loop structure should be the following:
@@ -909,6 +926,12 @@ bool llvm::UnrollRuntimeLoopRemainder(Loop *L, unsigned Count,
   // of its parent loops, so the Scalar Evolution pass needs to be run again.
   SE->forgetTopmostLoop(L);
 
+  // Verify that the Dom Tree is correct.
+#if defined(EXPENSIVE_CHECKS) && !defined(NDEBUG)
+  if (DT)
+    assert(DT->verify(DominatorTree::VerificationLevel::Full));
+#endif
+
   // Canonicalize to LoopSimplifyForm both original and remainder loops. We
   // cannot rely on the LoopUnrollPass to do this because it only does
   // canonicalization for parent/subloops and not the sibling loops.
diff --git a/lib/Transforms/Utils/LoopUtils.cpp b/lib/Transforms/Utils/LoopUtils.cpp
index 1d4f07ff1b9a9548899fe3f88238664786b288e0..5539ff12e4a15b41e81c40d9ca8f4d1bbc8203b3 100644
--- a/lib/Transforms/Utils/LoopUtils.cpp
+++ b/lib/Transforms/Utils/LoopUtils.cpp
@@ -1,9 +1,8 @@
 //===-- LoopUtils.cpp - Loop Utility functions -------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -15,6 +14,7 @@
 #include "llvm/ADT/ScopeExit.h"
 #include "llvm/Analysis/AliasAnalysis.h"
 #include "llvm/Analysis/BasicAliasAnalysis.h"
+#include "llvm/Analysis/DomTreeUpdater.h"
 #include "llvm/Analysis/GlobalsModRef.h"
 #include "llvm/Analysis/InstructionSimplify.h"
 #include "llvm/Analysis/LoopInfo.h"
@@ -27,7 +27,6 @@
 #include "llvm/Analysis/TargetTransformInfo.h"
 #include "llvm/Analysis/ValueTracking.h"
 #include "llvm/IR/DIBuilder.h"
-#include "llvm/IR/DomTreeUpdater.h"
 #include "llvm/IR/Dominators.h"
 #include "llvm/IR/Instructions.h"
 #include "llvm/IR/IntrinsicInst.h"
@@ -66,6 +65,9 @@ bool llvm::formDedicatedExitBlocks(Loop *L, DominatorTree *DT, LoopInfo *LI,
         if (isa<IndirectBrInst>(PredBB->getTerminator()))
           // We cannot rewrite exiting edges from an indirectbr.
           return false;
+        if (isa<CallBrInst>(PredBB->getTerminator()))
+          // We cannot rewrite exiting edges from a callbr.
+          return false;
 
         InLoopPredecessors.push_back(PredBB);
       } else {
@@ -187,44 +189,14 @@ void llvm::initializeLoopPassPass(PassRegistry &Registry) {
   INITIALIZE_PASS_DEPENDENCY(ScalarEvolutionWrapperPass)
 }
 
-static Optional<MDNode *> findOptionMDForLoopID(MDNode *LoopID,
-                                                StringRef Name) {
-  // Return none if LoopID is false.
-  if (!LoopID)
-    return None;
-
-  // First operand should refer to the loop id itself.
-  assert(LoopID->getNumOperands() > 0 && "requires at least one operand");
-  assert(LoopID->getOperand(0) == LoopID && "invalid loop id");
-
-  // Iterate over LoopID operands and look for MDString Metadata
-  for (unsigned i = 1, e = LoopID->getNumOperands(); i < e; ++i) {
-    MDNode *MD = dyn_cast<MDNode>(LoopID->getOperand(i));
-    if (!MD)
-      continue;
-    MDString *S = dyn_cast<MDString>(MD->getOperand(0));
-    if (!S)
-      continue;
-    // Return true if MDString holds expected MetaData.
-    if (Name.equals(S->getString()))
-      return MD;
-  }
-  return None;
-}
-
-static Optional<MDNode *> findOptionMDForLoop(const Loop *TheLoop,
-                                              StringRef Name) {
-  return findOptionMDForLoopID(TheLoop->getLoopID(), Name);
-}
-
 /// Find string metadata for loop
 ///
 /// If it has a value (e.g. {"llvm.distribute", 1} return the value as an
 /// operand or null otherwise.  If the string metadata is not found return
 /// Optional's not-a-value.
-Optional<const MDOperand *> llvm::findStringMetadataForLoop(Loop *TheLoop,
+Optional<const MDOperand *> llvm::findStringMetadataForLoop(const Loop *TheLoop,
                                                             StringRef Name) {
-  auto MD = findOptionMDForLoop(TheLoop, Name).getValueOr(nullptr);
+  MDNode *MD = findOptionMDForLoop(TheLoop, Name);
   if (!MD)
     return None;
   switch (MD->getNumOperands()) {
@@ -239,19 +211,18 @@ Optional<const MDOperand *> llvm::findStringMetadataForLoop(Loop *TheLoop,
 
 static Optional<bool> getOptionalBoolLoopAttribute(const Loop *TheLoop,
                                                    StringRef Name) {
-  Optional<MDNode *> MD = findOptionMDForLoop(TheLoop, Name);
-  if (!MD.hasValue())
-    return None;
-  MDNode *OptionNode = MD.getValue();
-  if (OptionNode == nullptr)
+  MDNode *MD = findOptionMDForLoop(TheLoop, Name);
+  if (!MD)
     return None;
-  switch (OptionNode->getNumOperands()) {
+  switch (MD->getNumOperands()) {
   case 1:
     // When the value is absent it is interpreted as 'attribute set'.
     return true;
   case 2:
-    return mdconst::extract_or_null<ConstantInt>(
-        OptionNode->getOperand(1).get());
+    if (ConstantInt *IntMD =
+            mdconst::extract_or_null<ConstantInt>(MD->getOperand(1).get()))
+      return IntMD->getZExtValue();
+    return true;
   }
   llvm_unreachable("unexpected number of options");
 }
@@ -325,8 +296,7 @@ Optional<MDNode *> llvm::makeFollowupLoopID(
 
   bool HasAnyFollowup = false;
   for (StringRef OptionName : FollowupOptions) {
-    MDNode *FollowupNode =
-        findOptionMDForLoopID(OrigLoopID, OptionName).getValueOr(nullptr);
+    MDNode *FollowupNode = findOptionMDForLoopID(OrigLoopID, OptionName);
     if (!FollowupNode)
       continue;
 
@@ -411,17 +381,17 @@ TransformationMode llvm::hasVectorizeTransformation(Loop *L) {
   Optional<int> InterleaveCount =
       getOptionalIntLoopAttribute(L, "llvm.loop.interleave.count");
 
-  if (Enable == true) {
-    // 'Forcing' vector width and interleave count to one effectively disables
-    // this tranformation.
-    if (VectorizeWidth == 1 && InterleaveCount == 1)
-      return TM_SuppressedByUser;
-    return TM_ForcedByUser;
-  }
+  // 'Forcing' vector width and interleave count to one effectively disables
+  // this tranformation.
+  if (Enable == true && VectorizeWidth == 1 && InterleaveCount == 1)
+    return TM_SuppressedByUser;
 
   if (getBooleanLoopAttribute(L, "llvm.loop.isvectorized"))
     return TM_Disable;
 
+  if (Enable == true)
+    return TM_ForcedByUser;
+
   if (VectorizeWidth == 1 && InterleaveCount == 1)
     return TM_Disable;
 
@@ -963,3 +933,39 @@ void llvm::propagateIRFlags(Value *I, ArrayRef<Value *> VL, Value *OpValue) {
       VecOp->andIRFlags(V);
   }
 }
+
+bool llvm::isKnownNegativeInLoop(const SCEV *S, const Loop *L,
+                                 ScalarEvolution &SE) {
+  const SCEV *Zero = SE.getZero(S->getType());
+  return SE.isAvailableAtLoopEntry(S, L) &&
+         SE.isLoopEntryGuardedByCond(L, ICmpInst::ICMP_SLT, S, Zero);
+}
+
+bool llvm::isKnownNonNegativeInLoop(const SCEV *S, const Loop *L,
+                                    ScalarEvolution &SE) {
+  const SCEV *Zero = SE.getZero(S->getType());
+  return SE.isAvailableAtLoopEntry(S, L) &&
+         SE.isLoopEntryGuardedByCond(L, ICmpInst::ICMP_SGE, S, Zero);
+}
+
+bool llvm::cannotBeMinInLoop(const SCEV *S, const Loop *L, ScalarEvolution &SE,
+                             bool Signed) {
+  unsigned BitWidth = cast<IntegerType>(S->getType())->getBitWidth();
+  APInt Min = Signed ? APInt::getSignedMinValue(BitWidth) :
+    APInt::getMinValue(BitWidth);
+  auto Predicate = Signed ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT;
+  return SE.isAvailableAtLoopEntry(S, L) &&
+         SE.isLoopEntryGuardedByCond(L, Predicate, S,
+                                     SE.getConstant(Min));
+}
+
+bool llvm::cannotBeMaxInLoop(const SCEV *S, const Loop *L, ScalarEvolution &SE,
+                             bool Signed) {
+  unsigned BitWidth = cast<IntegerType>(S->getType())->getBitWidth();
+  APInt Max = Signed ? APInt::getSignedMaxValue(BitWidth) :
+    APInt::getMaxValue(BitWidth);
+  auto Predicate = Signed ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT;
+  return SE.isAvailableAtLoopEntry(S, L) &&
+         SE.isLoopEntryGuardedByCond(L, Predicate, S,
+                                     SE.getConstant(Max));
+}
diff --git a/lib/Transforms/Utils/LoopVersioning.cpp b/lib/Transforms/Utils/LoopVersioning.cpp
index abbcd5f9e3b8b286015fa7fa75a9ee4c331e7dc2..d59076c2a9f2ae04a4ea13f09f3f41c87dd90a4c 100644
--- a/lib/Transforms/Utils/LoopVersioning.cpp
+++ b/lib/Transforms/Utils/LoopVersioning.cpp
@@ -1,9 +1,8 @@
 //===- LoopVersioning.cpp - Utility to version a loop ---------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Transforms/Utils/LowerInvoke.cpp b/lib/Transforms/Utils/LowerInvoke.cpp
index c852d538b0d19b0bf552ddc5faaefcb399c12504..fe67e191dc62a2e039c26a83b9be6644e6cf78d8 100644
--- a/lib/Transforms/Utils/LowerInvoke.cpp
+++ b/lib/Transforms/Utils/LowerInvoke.cpp
@@ -1,9 +1,8 @@
 //===- LowerInvoke.cpp - Eliminate Invoke instructions --------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -53,7 +52,8 @@ static bool runImpl(Function &F) {
       II->getOperandBundlesAsDefs(OpBundles);
       // Insert a normal call instruction...
       CallInst *NewCall =
-          CallInst::Create(II->getCalledValue(), CallArgs, OpBundles, "", II);
+          CallInst::Create(II->getFunctionType(), II->getCalledValue(),
+                           CallArgs, OpBundles, "", II);
       NewCall->takeName(II);
       NewCall->setCallingConv(II->getCallingConv());
       NewCall->setAttributes(II->getAttributes());
diff --git a/lib/Transforms/Utils/LowerMemIntrinsics.cpp b/lib/Transforms/Utils/LowerMemIntrinsics.cpp
index 661b4fa5bcb70118ff4dbc21170817e44aecbf0d..0cc085dc366c66890a6f3aeb9cfea10ed064a4ee 100644
--- a/lib/Transforms/Utils/LowerMemIntrinsics.cpp
+++ b/lib/Transforms/Utils/LowerMemIntrinsics.cpp
@@ -1,9 +1,8 @@
 //===- LowerMemIntrinsics.cpp ----------------------------------*- C++ -*--===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -73,7 +72,7 @@ void llvm::createMemCpyLoopKnownSize(Instruction *InsertBefore, Value *SrcAddr,
     // Loop Body
     Value *SrcGEP =
         LoopBuilder.CreateInBoundsGEP(LoopOpType, SrcAddr, LoopIndex);
-    Value *Load = LoopBuilder.CreateLoad(SrcGEP, SrcIsVolatile);
+    Value *Load = LoopBuilder.CreateLoad(LoopOpType, SrcGEP, SrcIsVolatile);
     Value *DstGEP =
         LoopBuilder.CreateInBoundsGEP(LoopOpType, DstAddr, LoopIndex);
     LoopBuilder.CreateStore(Load, DstGEP, DstIsVolatile);
@@ -115,7 +114,7 @@ void llvm::createMemCpyLoopKnownSize(Instruction *InsertBefore, Value *SrcAddr,
                              : RBuilder.CreateBitCast(SrcAddr, SrcPtrType);
       Value *SrcGEP = RBuilder.CreateInBoundsGEP(
           OpTy, CastedSrc, ConstantInt::get(TypeOfCopyLen, GepIndex));
-      Value *Load = RBuilder.CreateLoad(SrcGEP, SrcIsVolatile);
+      Value *Load = RBuilder.CreateLoad(OpTy, SrcGEP, SrcIsVolatile);
 
       // Cast destination to operand type and store.
       PointerType *DstPtrType = PointerType::get(OpTy, DstAS);
@@ -182,7 +181,7 @@ void llvm::createMemCpyLoopUnknownSize(Instruction *InsertBefore,
   LoopIndex->addIncoming(ConstantInt::get(CopyLenType, 0U), PreLoopBB);
 
   Value *SrcGEP = LoopBuilder.CreateInBoundsGEP(LoopOpType, SrcAddr, LoopIndex);
-  Value *Load = LoopBuilder.CreateLoad(SrcGEP, SrcIsVolatile);
+  Value *Load = LoopBuilder.CreateLoad(LoopOpType, SrcGEP, SrcIsVolatile);
   Value *DstGEP = LoopBuilder.CreateInBoundsGEP(LoopOpType, DstAddr, LoopIndex);
   LoopBuilder.CreateStore(Load, DstGEP, DstIsVolatile);
 
@@ -235,7 +234,7 @@ void llvm::createMemCpyLoopUnknownSize(Instruction *InsertBefore,
     Value *FullOffset = ResBuilder.CreateAdd(RuntimeBytesCopied, ResidualIndex);
     Value *SrcGEP =
         ResBuilder.CreateInBoundsGEP(Int8Type, SrcAsInt8, FullOffset);
-    Value *Load = ResBuilder.CreateLoad(SrcGEP, SrcIsVolatile);
+    Value *Load = ResBuilder.CreateLoad(Int8Type, SrcGEP, SrcIsVolatile);
     Value *DstGEP =
         ResBuilder.CreateInBoundsGEP(Int8Type, DstAsInt8, FullOffset);
     ResBuilder.CreateStore(Load, DstGEP, DstIsVolatile);
@@ -293,6 +292,8 @@ static void createMemMoveLoop(Instruction *InsertBefore,
   BasicBlock *OrigBB = InsertBefore->getParent();
   Function *F = OrigBB->getParent();
 
+  Type *EltTy = cast<PointerType>(SrcAddr->getType())->getElementType();
+
   // Create the a comparison of src and dst, based on which we jump to either
   // the forward-copy part of the function (if src >= dst) or the backwards-copy
   // part (if src < dst).
@@ -331,9 +332,10 @@ static void createMemMoveLoop(Instruction *InsertBefore,
   Value *IndexPtr = LoopBuilder.CreateSub(
       LoopPhi, ConstantInt::get(TypeOfCopyLen, 1), "index_ptr");
   Value *Element = LoopBuilder.CreateLoad(
-      LoopBuilder.CreateInBoundsGEP(SrcAddr, IndexPtr), "element");
-  LoopBuilder.CreateStore(Element,
-                          LoopBuilder.CreateInBoundsGEP(DstAddr, IndexPtr));
+      EltTy, LoopBuilder.CreateInBoundsGEP(EltTy, SrcAddr, IndexPtr),
+      "element");
+  LoopBuilder.CreateStore(
+      Element, LoopBuilder.CreateInBoundsGEP(EltTy, DstAddr, IndexPtr));
   LoopBuilder.CreateCondBr(
       LoopBuilder.CreateICmpEQ(IndexPtr, ConstantInt::get(TypeOfCopyLen, 0)),
       ExitBB, LoopBB);
@@ -348,9 +350,10 @@ static void createMemMoveLoop(Instruction *InsertBefore,
   IRBuilder<> FwdLoopBuilder(FwdLoopBB);
   PHINode *FwdCopyPhi = FwdLoopBuilder.CreatePHI(TypeOfCopyLen, 0, "index_ptr");
   Value *FwdElement = FwdLoopBuilder.CreateLoad(
-      FwdLoopBuilder.CreateInBoundsGEP(SrcAddr, FwdCopyPhi), "element");
+      EltTy, FwdLoopBuilder.CreateInBoundsGEP(EltTy, SrcAddr, FwdCopyPhi),
+      "element");
   FwdLoopBuilder.CreateStore(
-      FwdElement, FwdLoopBuilder.CreateInBoundsGEP(DstAddr, FwdCopyPhi));
+      FwdElement, FwdLoopBuilder.CreateInBoundsGEP(EltTy, DstAddr, FwdCopyPhi));
   Value *FwdIndexPtr = FwdLoopBuilder.CreateAdd(
       FwdCopyPhi, ConstantInt::get(TypeOfCopyLen, 1), "index_increment");
   FwdLoopBuilder.CreateCondBr(FwdLoopBuilder.CreateICmpEQ(FwdIndexPtr, CopyLen),
diff --git a/lib/Transforms/Utils/LowerSwitch.cpp b/lib/Transforms/Utils/LowerSwitch.cpp
index d019a44fc7053a22642a76cb9a13b90272f4471f..fec13c8c160b0c8f922eaddd1f67241034f1ff87 100644
--- a/lib/Transforms/Utils/LowerSwitch.cpp
+++ b/lib/Transforms/Utils/LowerSwitch.cpp
@@ -1,9 +1,8 @@
 //===- LowerSwitch.cpp - Eliminate Switch instructions --------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Transforms/Utils/Mem2Reg.cpp b/lib/Transforms/Utils/Mem2Reg.cpp
index 23145e5847512af9695d7b82727524c7f4ca4a4c..cd2c81b6abc84bb6402543a46523947830226c67 100644
--- a/lib/Transforms/Utils/Mem2Reg.cpp
+++ b/lib/Transforms/Utils/Mem2Reg.cpp
@@ -1,9 +1,8 @@
 //===- Mem2Reg.cpp - The -mem2reg pass, a wrapper around the Utils lib ----===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Transforms/Utils/MetaRenamer.cpp b/lib/Transforms/Utils/MetaRenamer.cpp
index 88d595ee02abe9695972ab1d347b3a7eb27d6178..c0b7edc547fd3b8e5efa717dcea8338781f712b9 100644
--- a/lib/Transforms/Utils/MetaRenamer.cpp
+++ b/lib/Transforms/Utils/MetaRenamer.cpp
@@ -1,9 +1,8 @@
 //===- MetaRenamer.cpp - Rename everything with metasyntatic names --------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Transforms/Utils/ModuleUtils.cpp b/lib/Transforms/Utils/ModuleUtils.cpp
index ba4b7f3cc2639e6e2ffc1b8ebe89102fdb118950..b076e7503c646a81c1b6da044013b17588ec867e 100644
--- a/lib/Transforms/Utils/ModuleUtils.cpp
+++ b/lib/Transforms/Utils/ModuleUtils.cpp
@@ -1,9 +1,8 @@
 //===-- ModuleUtils.cpp - Functions to manipulate Modules -----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -127,36 +126,24 @@ void llvm::appendToCompilerUsed(Module &M, ArrayRef<GlobalValue *> Values) {
   appendToUsedList(M, "llvm.compiler.used", Values);
 }
 
-Function *llvm::checkSanitizerInterfaceFunction(Constant *FuncOrBitcast) {
-  if (isa<Function>(FuncOrBitcast))
-    return cast<Function>(FuncOrBitcast);
-  FuncOrBitcast->print(errs());
-  errs() << '\n';
-  std::string Err;
-  raw_string_ostream Stream(Err);
-  Stream << "Sanitizer interface function redefined: " << *FuncOrBitcast;
-  report_fatal_error(Err);
-}
-
-Function *llvm::declareSanitizerInitFunction(Module &M, StringRef InitName,
-                                             ArrayRef<Type *> InitArgTypes) {
+FunctionCallee
+llvm::declareSanitizerInitFunction(Module &M, StringRef InitName,
+                                   ArrayRef<Type *> InitArgTypes) {
   assert(!InitName.empty() && "Expected init function name");
-  Function *F = checkSanitizerInterfaceFunction(M.getOrInsertFunction(
+  return M.getOrInsertFunction(
       InitName,
       FunctionType::get(Type::getVoidTy(M.getContext()), InitArgTypes, false),
-      AttributeList()));
-  F->setLinkage(Function::ExternalLinkage);
-  return F;
+      AttributeList());
 }
 
-std::pair<Function *, Function *> llvm::createSanitizerCtorAndInitFunctions(
+std::pair<Function *, FunctionCallee> llvm::createSanitizerCtorAndInitFunctions(
     Module &M, StringRef CtorName, StringRef InitName,
     ArrayRef<Type *> InitArgTypes, ArrayRef<Value *> InitArgs,
     StringRef VersionCheckName) {
   assert(!InitName.empty() && "Expected init function name");
   assert(InitArgs.size() == InitArgTypes.size() &&
          "Sanitizer's init function expects different number of arguments");
-  Function *InitFunction =
+  FunctionCallee InitFunction =
       declareSanitizerInitFunction(M, InitName, InitArgTypes);
   Function *Ctor = Function::Create(
       FunctionType::get(Type::getVoidTy(M.getContext()), false),
@@ -165,15 +152,59 @@ std::pair<Function *, Function *> llvm::createSanitizerCtorAndInitFunctions(
   IRBuilder<> IRB(ReturnInst::Create(M.getContext(), CtorBB));
   IRB.CreateCall(InitFunction, InitArgs);
   if (!VersionCheckName.empty()) {
-    Function *VersionCheckFunction =
-        checkSanitizerInterfaceFunction(M.getOrInsertFunction(
-            VersionCheckName, FunctionType::get(IRB.getVoidTy(), {}, false),
-            AttributeList()));
+    FunctionCallee VersionCheckFunction = M.getOrInsertFunction(
+        VersionCheckName, FunctionType::get(IRB.getVoidTy(), {}, false),
+        AttributeList());
     IRB.CreateCall(VersionCheckFunction, {});
   }
   return std::make_pair(Ctor, InitFunction);
 }
 
+std::pair<Function *, FunctionCallee>
+llvm::getOrCreateSanitizerCtorAndInitFunctions(
+    Module &M, StringRef CtorName, StringRef InitName,
+    ArrayRef<Type *> InitArgTypes, ArrayRef<Value *> InitArgs,
+    function_ref<void(Function *, FunctionCallee)> FunctionsCreatedCallback,
+    StringRef VersionCheckName) {
+  assert(!CtorName.empty() && "Expected ctor function name");
+
+  if (Function *Ctor = M.getFunction(CtorName))
+    // FIXME: Sink this logic into the module, similar to the handling of
+    // globals. This will make moving to a concurrent model much easier.
+    if (Ctor->arg_size() == 0 ||
+        Ctor->getReturnType() == Type::getVoidTy(M.getContext()))
+      return {Ctor, declareSanitizerInitFunction(M, InitName, InitArgTypes)};
+
+  Function *Ctor;
+  FunctionCallee InitFunction;
+  std::tie(Ctor, InitFunction) = llvm::createSanitizerCtorAndInitFunctions(
+      M, CtorName, InitName, InitArgTypes, InitArgs, VersionCheckName);
+  FunctionsCreatedCallback(Ctor, InitFunction);
+  return std::make_pair(Ctor, InitFunction);
+}
+
+Function *llvm::getOrCreateInitFunction(Module &M, StringRef Name) {
+  assert(!Name.empty() && "Expected init function name");
+  if (Function *F = M.getFunction(Name)) {
+    if (F->arg_size() != 0 ||
+        F->getReturnType() != Type::getVoidTy(M.getContext())) {
+      std::string Err;
+      raw_string_ostream Stream(Err);
+      Stream << "Sanitizer interface function defined with wrong type: " << *F;
+      report_fatal_error(Err);
+    }
+    return F;
+  }
+  Function *F =
+      cast<Function>(M.getOrInsertFunction(Name, AttributeList(),
+                                           Type::getVoidTy(M.getContext()))
+                         .getCallee());
+
+  appendToGlobalCtors(M, F, 0);
+
+  return F;
+}
+
 void llvm::filterDeadComdatFunctions(
     Module &M, SmallVectorImpl<Function *> &DeadComdatFunctions) {
   // Build a map from the comdat to the number of entries in that comdat we
diff --git a/lib/Transforms/Utils/NameAnonGlobals.cpp b/lib/Transforms/Utils/NameAnonGlobals.cpp
index 34dc1cccdd5b567adcbfe2bfa227f981863f6ba4..ac8991e9d475a241f157f19e2cc3cfaf26d19247 100644
--- a/lib/Transforms/Utils/NameAnonGlobals.cpp
+++ b/lib/Transforms/Utils/NameAnonGlobals.cpp
@@ -1,9 +1,8 @@
 //===- NameAnonGlobals.cpp - ThinLTO Support: Name Unnamed Globals --------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Transforms/Utils/PredicateInfo.cpp b/lib/Transforms/Utils/PredicateInfo.cpp
index 585ce6b4c1188c0b2395e1396fe749bee0040fef..f1c0792da78cf890037f9b5b76cfb7054e20b066 100644
--- a/lib/Transforms/Utils/PredicateInfo.cpp
+++ b/lib/Transforms/Utils/PredicateInfo.cpp
@@ -1,9 +1,8 @@
 //===-- PredicateInfo.cpp - PredicateInfo Builder--------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------===//
 //
@@ -489,8 +488,10 @@ void PredicateInfo::buildPredicateInfo() {
 // tricky (FIXME).
 static Function *getCopyDeclaration(Module *M, Type *Ty) {
   std::string Name = "llvm.ssa.copy." + utostr((uintptr_t) Ty);
-  return cast<Function>(M->getOrInsertFunction(
-      Name, getType(M->getContext(), Intrinsic::ssa_copy, Ty)));
+  return cast<Function>(
+      M->getOrInsertFunction(Name,
+                             getType(M->getContext(), Intrinsic::ssa_copy, Ty))
+          .getCallee());
 }
 
 // Given the renaming stack, make all the operands currently on the stack real
diff --git a/lib/Transforms/Utils/PromoteMemoryToRegister.cpp b/lib/Transforms/Utils/PromoteMemoryToRegister.cpp
index a53083c31e3f12afa3b8fd7ed98f2020a26b52f3..5748b06a76b296767ea0b3fc0d106031c9d633ab 100644
--- a/lib/Transforms/Utils/PromoteMemoryToRegister.cpp
+++ b/lib/Transforms/Utils/PromoteMemoryToRegister.cpp
@@ -1,9 +1,8 @@
 //===- PromoteMemoryToRegister.cpp - Convert allocas to registers ---------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -82,8 +81,7 @@ bool llvm::isAllocaPromotable(const AllocaInst *AI) {
       if (SI->isVolatile())
         return false;
     } else if (const IntrinsicInst *II = dyn_cast<IntrinsicInst>(U)) {
-      if (II->getIntrinsicID() != Intrinsic::lifetime_start &&
-          II->getIntrinsicID() != Intrinsic::lifetime_end)
+      if (!II->isLifetimeStartOrEnd())
         return false;
     } else if (const BitCastInst *BCI = dyn_cast<BitCastInst>(U)) {
       if (BCI->getType() != Type::getInt8PtrTy(U->getContext(), AS))
diff --git a/lib/Transforms/Utils/SSAUpdater.cpp b/lib/Transforms/Utils/SSAUpdater.cpp
index 9e5fb0e7172d4d2d601dac8fe2c4d240401db0d5..bffdd115d940cdce5aff51d4f7b76b6ac9cbd1b7 100644
--- a/lib/Transforms/Utils/SSAUpdater.cpp
+++ b/lib/Transforms/Utils/SSAUpdater.cpp
@@ -1,9 +1,8 @@
 //===- SSAUpdater.cpp - Unstructured SSA Update Tool ----------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -350,8 +349,7 @@ LoadAndStorePromoter(ArrayRef<const Instruction *> Insts,
   SSA.Initialize(SomeVal->getType(), BaseName);
 }
 
-void LoadAndStorePromoter::
-run(const SmallVectorImpl<Instruction *> &Insts) const {
+void LoadAndStorePromoter::run(const SmallVectorImpl<Instruction *> &Insts) {
   // First step: bucket up uses of the alloca by the block they occur in.
   // This is important because we have to handle multiple defs/uses in a block
   // ourselves: SSAUpdater is purely for cross-block references.
diff --git a/lib/Transforms/Utils/SSAUpdaterBulk.cpp b/lib/Transforms/Utils/SSAUpdaterBulk.cpp
index 397bac2940a4985fd543802f1afc1b0605c33e0d..917d5e0a1ef0a80e6857d5abe4b54cf104d406c3 100644
--- a/lib/Transforms/Utils/SSAUpdaterBulk.cpp
+++ b/lib/Transforms/Utils/SSAUpdaterBulk.cpp
@@ -1,9 +1,8 @@
 //===- SSAUpdaterBulk.cpp - Unstructured SSA Update Tool ------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Transforms/Utils/SanitizerStats.cpp b/lib/Transforms/Utils/SanitizerStats.cpp
index 8c23957ac43e915b27fb894a782f00b84ea8562a..a1313c77ed77ed606101e7b367d26eba34e6fdf8 100644
--- a/lib/Transforms/Utils/SanitizerStats.cpp
+++ b/lib/Transforms/Utils/SanitizerStats.cpp
@@ -1,9 +1,8 @@
 //===- SanitizerStats.cpp - Sanitizer statistics gathering ----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -57,8 +56,8 @@ void SanitizerStatReport::create(IRBuilder<> &B, SanitizerStatKind SK) {
 
   FunctionType *StatReportTy =
       FunctionType::get(B.getVoidTy(), Int8PtrTy, false);
-  Constant *StatReport = M->getOrInsertFunction(
-      "__sanitizer_stat_report", StatReportTy);
+  FunctionCallee StatReport =
+      M->getOrInsertFunction("__sanitizer_stat_report", StatReportTy);
 
   auto InitAddr = ConstantExpr::getGetElementPtr(
       EmptyModuleStatsTy, ModuleStatsGV,
@@ -98,8 +97,8 @@ void SanitizerStatReport::finish() {
   IRBuilder<> B(BB);
 
   FunctionType *StatInitTy = FunctionType::get(VoidTy, Int8PtrTy, false);
-  Constant *StatInit = M->getOrInsertFunction(
-      "__sanitizer_stat_init", StatInitTy);
+  FunctionCallee StatInit =
+      M->getOrInsertFunction("__sanitizer_stat_init", StatInitTy);
 
   B.CreateCall(StatInit, ConstantExpr::getBitCast(NewModuleStatsGV, Int8PtrTy));
   B.CreateRetVoid();
diff --git a/lib/Transforms/Utils/SimplifyCFG.cpp b/lib/Transforms/Utils/SimplifyCFG.cpp
index b98f2fff65eaeb96fd0daad9c6561533ceeed348..dbd8fb720e4f35e2cad9f125baee95a9b148ad61 100644
--- a/lib/Transforms/Utils/SimplifyCFG.cpp
+++ b/lib/Transforms/Utils/SimplifyCFG.cpp
@@ -1,9 +1,8 @@
 //===- SimplifyCFG.cpp - Code to perform CFG simplification ---------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -1266,8 +1265,10 @@ static bool HoistThenElseCodeToIf(BranchInst *BI,
     while (isa<DbgInfoIntrinsic>(I2))
       I2 = &*BB2_Itr++;
   }
+  // FIXME: Can we define a safety predicate for CallBr?
   if (isa<PHINode>(I1) || !I1->isIdenticalToWhenDefined(I2) ||
-      (isa<InvokeInst>(I1) && !isSafeToHoistInvoke(BB1, BB2, I1, I2)))
+      (isa<InvokeInst>(I1) && !isSafeToHoistInvoke(BB1, BB2, I1, I2)) ||
+      isa<CallBrInst>(I1))
     return false;
 
   BasicBlock *BIParent = BI->getParent();
@@ -1321,7 +1322,8 @@ static bool HoistThenElseCodeToIf(BranchInst *BI,
                              LLVMContext::MD_align,
                              LLVMContext::MD_dereferenceable,
                              LLVMContext::MD_dereferenceable_or_null,
-                             LLVMContext::MD_mem_parallel_loop_access};
+                             LLVMContext::MD_mem_parallel_loop_access,
+                             LLVMContext::MD_access_group};
       combineMetadata(I1, I2, KnownIDs, true);
 
       // I1 and I2 are being combined into a single instruction.  Its debug
@@ -1349,9 +1351,14 @@ static bool HoistThenElseCodeToIf(BranchInst *BI,
 
 HoistTerminator:
   // It may not be possible to hoist an invoke.
+  // FIXME: Can we define a safety predicate for CallBr?
   if (isa<InvokeInst>(I1) && !isSafeToHoistInvoke(BB1, BB2, I1, I2))
     return Changed;
 
+  // TODO: callbr hoisting currently disabled pending further study.
+  if (isa<CallBrInst>(I1))
+    return Changed;
+
   for (BasicBlock *Succ : successors(BB1)) {
     for (PHINode &PN : Succ->phis()) {
       Value *BB1V = PN.getIncomingValueForBlock(BB1);
@@ -1372,14 +1379,6 @@ HoistTerminator:
     }
   }
 
-  // As the parent basic block terminator is a branch instruction which is
-  // removed at the end of the current transformation, use its previous
-  // non-debug instruction, as the reference insertion point, which will
-  // provide the debug location for generated select instructions. For BBs
-  // with only debug instructions, use an empty debug location.
-  Instruction *InsertPt =
-      BIParent->getTerminator()->getPrevNonDebugInstruction();
-
   // Okay, it is safe to hoist the terminator.
   Instruction *NT = I1->clone();
   BIParent->getInstList().insert(BI->getIterator(), NT);
@@ -1393,11 +1392,8 @@ HoistTerminator:
   // it involves inlinable calls.
   NT->applyMergedLocation(I1->getDebugLoc(), I2->getDebugLoc());
 
+  // PHIs created below will adopt NT's merged DebugLoc.
   IRBuilder<NoFolder> Builder(NT);
-  // If an earlier instruction in this BB had a location, adopt it, otherwise
-  // clear debug locations.
-  Builder.SetCurrentDebugLocation(InsertPt ? InsertPt->getDebugLoc()
-                                           : DebugLoc());
 
   // Hoisting one of the terminators from our successor is a great thing.
   // Unfortunately, the successors of the if/else blocks may have PHI nodes in
@@ -1454,7 +1450,7 @@ static bool canSinkInstructions(
     // Conservatively return false if I is an inline-asm instruction. Sinking
     // and merging inline-asm instructions can potentially create arguments
     // that cannot satisfy the inline-asm constraints.
-    if (const auto *C = dyn_cast<CallInst>(I))
+    if (const auto *C = dyn_cast<CallBase>(I))
       if (C->isInlineAsm())
         return false;
 
@@ -1517,7 +1513,7 @@ static bool canSinkInstructions(
         // We can't create a PHI from this GEP.
         return false;
       // Don't create indirect calls! The called value is the final operand.
-      if ((isa<CallInst>(I0) || isa<InvokeInst>(I0)) && OI == OE - 1) {
+      if (isa<CallBase>(I0) && OI == OE - 1) {
         // FIXME: if the call was *already* indirect, we should do this.
         return false;
       }
@@ -3440,7 +3436,7 @@ static bool SimplifyTerminatorOnSelect(Instruction *OldTerm, Value *Cond,
       KeepEdge2 = nullptr;
     else
       Succ->removePredecessor(OldTerm->getParent(),
-                              /*DontDeleteUselessPHIs=*/true);
+                              /*KeepOneInputPHIs=*/true);
   }
 
   IRBuilder<> Builder(OldTerm);
@@ -5101,7 +5097,9 @@ Value *SwitchLookupTable::BuildLookup(Value *Index, IRBuilder<> &Builder) {
     Value *GEPIndices[] = {Builder.getInt32(0), Index};
     Value *GEP = Builder.CreateInBoundsGEP(Array->getValueType(), Array,
                                            GEPIndices, "switch.gep");
-    return Builder.CreateLoad(GEP, "switch.load");
+    return Builder.CreateLoad(
+        cast<ArrayType>(Array->getValueType())->getElementType(), GEP,
+        "switch.load");
   }
   }
   llvm_unreachable("Unknown lookup table kind!");
@@ -5435,7 +5433,7 @@ static bool SwitchToLookupTable(SwitchInst *SI, IRBuilder<> &Builder,
     // We cached PHINodes in PHIs. To avoid accessing deleted PHINodes later,
     // do not delete PHINodes here.
     SI->getDefaultDest()->removePredecessor(SI->getParent(),
-                                            /*DontDeleteUselessPHIs=*/true);
+                                            /*KeepOneInputPHIs=*/true);
   }
 
   bool ReturnedEarly = false;
diff --git a/lib/Transforms/Utils/SimplifyIndVar.cpp b/lib/Transforms/Utils/SimplifyIndVar.cpp
index 7faf291e73d9fdbb5e40318dceabb3b59ba187a5..6d634d8e917d9ddc2a9fab4bc8e95bf1a056e158 100644
--- a/lib/Transforms/Utils/SimplifyIndVar.cpp
+++ b/lib/Transforms/Utils/SimplifyIndVar.cpp
@@ -1,9 +1,8 @@
 //===-- SimplifyIndVar.cpp - Induction variable simplification ------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Transforms/Utils/SimplifyLibCalls.cpp b/lib/Transforms/Utils/SimplifyLibCalls.cpp
index a50575b025601c4ecbf8d14bea8a671b06344b3d..da368b2d863b043f3d845c843e1c710d38ce59e1 100644
--- a/lib/Transforms/Utils/SimplifyLibCalls.cpp
+++ b/lib/Transforms/Utils/SimplifyLibCalls.cpp
@@ -1,9 +1,8 @@
 //===------ SimplifyLibCalls.cpp - Library calls simplifier ---------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -334,11 +333,12 @@ Value *LibCallSimplifier::optimizeStrCmp(CallInst *CI, IRBuilder<> &B) {
     return ConstantInt::get(CI->getType(), Str1.compare(Str2));
 
   if (HasStr1 && Str1.empty()) // strcmp("", x) -> -*x
-    return B.CreateNeg(
-        B.CreateZExt(B.CreateLoad(Str2P, "strcmpload"), CI->getType()));
+    return B.CreateNeg(B.CreateZExt(
+        B.CreateLoad(B.getInt8Ty(), Str2P, "strcmpload"), CI->getType()));
 
   if (HasStr2 && Str2.empty()) // strcmp(x,"") -> *x
-    return B.CreateZExt(B.CreateLoad(Str1P, "strcmpload"), CI->getType());
+    return B.CreateZExt(B.CreateLoad(B.getInt8Ty(), Str1P, "strcmpload"),
+                        CI->getType());
 
   // strcmp(P, "x") -> memcmp(P, "x", 2)
   uint64_t Len1 = GetStringLength(Str1P);
@@ -398,11 +398,12 @@ Value *LibCallSimplifier::optimizeStrNCmp(CallInst *CI, IRBuilder<> &B) {
   }
 
   if (HasStr1 && Str1.empty()) // strncmp("", x, n) -> -*x
-    return B.CreateNeg(
-        B.CreateZExt(B.CreateLoad(Str2P, "strcmpload"), CI->getType()));
+    return B.CreateNeg(B.CreateZExt(
+        B.CreateLoad(B.getInt8Ty(), Str2P, "strcmpload"), CI->getType()));
 
   if (HasStr2 && Str2.empty()) // strncmp(x, "", n) -> *x
-    return B.CreateZExt(B.CreateLoad(Str1P, "strcmpload"), CI->getType());
+    return B.CreateZExt(B.CreateLoad(B.getInt8Ty(), Str1P, "strcmpload"),
+                        CI->getType());
 
   uint64_t Len1 = GetStringLength(Str1P);
   uint64_t Len2 = GetStringLength(Str2P);
@@ -591,7 +592,8 @@ Value *LibCallSimplifier::optimizeStringLength(CallInst *CI, IRBuilder<> &B,
   // strlen(x) != 0 --> *x != 0
   // strlen(x) == 0 --> *x == 0
   if (isOnlyUsedInZeroEqualityComparison(CI))
-    return B.CreateZExt(B.CreateLoad(Src, "strlenfirst"), CI->getType());
+    return B.CreateZExt(B.CreateLoad(B.getIntNTy(CharSize), Src, "strlenfirst"),
+                        CI->getType());
 
   return nullptr;
 }
@@ -735,7 +737,8 @@ Value *LibCallSimplifier::optimizeStrStr(CallInst *CI, IRBuilder<> &B) {
 
     // strstr("abcd", "bc") -> gep((char*)"abcd", 1)
     Value *Result = castToCStr(CI->getArgOperand(0), B);
-    Result = B.CreateConstInBoundsGEP1_64(Result, Offset, "strstr");
+    Result =
+        B.CreateConstInBoundsGEP1_64(B.getInt8Ty(), Result, Offset, "strstr");
     return B.CreateBitCast(Result, CI->getType());
   }
 
@@ -798,8 +801,11 @@ Value *LibCallSimplifier::optimizeMemChr(CallInst *CI, IRBuilder<> &B) {
       Bitfield.setBit((unsigned char)C);
     Value *BitfieldC = B.getInt(Bitfield);
 
-    // First check that the bit field access is within bounds.
+    // Adjust width of "C" to the bitfield width, then mask off the high bits.
     Value *C = B.CreateZExtOrTrunc(CI->getArgOperand(1), BitfieldC->getType());
+    C = B.CreateAnd(C, B.getIntN(Width, 0xFF));
+
+    // First check that the bit field access is within bounds.
     Value *Bounds = B.CreateICmp(ICmpInst::ICMP_ULT, C, B.getIntN(Width, Width),
                                  "memchr.bounds");
 
@@ -842,10 +848,12 @@ Value *LibCallSimplifier::optimizeMemCmp(CallInst *CI, IRBuilder<> &B) {
 
   // memcmp(S1,S2,1) -> *(unsigned char*)LHS - *(unsigned char*)RHS
   if (Len == 1) {
-    Value *LHSV = B.CreateZExt(B.CreateLoad(castToCStr(LHS, B), "lhsc"),
-                               CI->getType(), "lhsv");
-    Value *RHSV = B.CreateZExt(B.CreateLoad(castToCStr(RHS, B), "rhsc"),
-                               CI->getType(), "rhsv");
+    Value *LHSV =
+        B.CreateZExt(B.CreateLoad(B.getInt8Ty(), castToCStr(LHS, B), "lhsc"),
+                     CI->getType(), "lhsv");
+    Value *RHSV =
+        B.CreateZExt(B.CreateLoad(B.getInt8Ty(), castToCStr(RHS, B), "rhsc"),
+                     CI->getType(), "rhsv");
     return B.CreateSub(LHSV, RHSV, "chardiff");
   }
 
@@ -875,12 +883,12 @@ Value *LibCallSimplifier::optimizeMemCmp(CallInst *CI, IRBuilder<> &B) {
       if (!LHSV) {
         Type *LHSPtrTy =
             IntType->getPointerTo(LHS->getType()->getPointerAddressSpace());
-        LHSV = B.CreateLoad(B.CreateBitCast(LHS, LHSPtrTy), "lhsv");
+        LHSV = B.CreateLoad(IntType, B.CreateBitCast(LHS, LHSPtrTy), "lhsv");
       }
       if (!RHSV) {
         Type *RHSPtrTy =
             IntType->getPointerTo(RHS->getType()->getPointerAddressSpace());
-        RHSV = B.CreateLoad(B.CreateBitCast(RHS, RHSPtrTy), "rhsv");
+        RHSV = B.CreateLoad(IntType, B.CreateBitCast(RHS, RHSPtrTy), "rhsv");
       }
       return B.CreateZExt(B.CreateICmpNE(LHSV, RHSV), CI->getType(), "memcmp");
     }
@@ -1501,9 +1509,8 @@ Value *LibCallSimplifier::optimizeExp2(CallInst *CI, IRBuilder<> &B) {
         One = ConstantExpr::getFPExtend(One, Op->getType());
 
       Module *M = CI->getModule();
-      Value *NewCallee =
-          M->getOrInsertFunction(TLI->getName(LdExp), Op->getType(),
-                                 Op->getType(), B.getInt32Ty());
+      FunctionCallee NewCallee = M->getOrInsertFunction(
+          TLI->getName(LdExp), Op->getType(), Op->getType(), B.getInt32Ty());
       CallInst *CI = B.CreateCall(NewCallee, {One, LdExpArg});
       if (const Function *F = dyn_cast<Function>(Callee->stripPointerCasts()))
         CI->setCallingConv(F->getCallingConv());
@@ -1651,13 +1658,13 @@ Value *LibCallSimplifier::optimizeSqrt(CallInst *CI, IRBuilder<> &B) {
   // replace it with the fabs of that factor.
   Module *M = Callee->getParent();
   Type *ArgType = I->getType();
-  Value *Fabs = Intrinsic::getDeclaration(M, Intrinsic::fabs, ArgType);
+  Function *Fabs = Intrinsic::getDeclaration(M, Intrinsic::fabs, ArgType);
   Value *FabsCall = B.CreateCall(Fabs, RepeatOp, "fabs");
   if (OtherOp) {
     // If we found a non-repeated factor, we still need to get its square
     // root. We then multiply that by the value that was simplified out
     // of the square root calculation.
-    Value *Sqrt = Intrinsic::getDeclaration(M, Intrinsic::sqrt, ArgType);
+    Function *Sqrt = Intrinsic::getDeclaration(M, Intrinsic::sqrt, ArgType);
     Value *SqrtCall = B.CreateCall(Sqrt, OtherOp, "sqrt");
     return B.CreateFMul(FabsCall, SqrtCall);
   }
@@ -1725,8 +1732,8 @@ static void insertSinCosCall(IRBuilder<> &B, Function *OrigCallee, Value *Arg,
   }
 
   Module *M = OrigCallee->getParent();
-  Value *Callee = M->getOrInsertFunction(Name, OrigCallee->getAttributes(),
-                                         ResTy, ArgTy);
+  FunctionCallee Callee =
+      M->getOrInsertFunction(Name, OrigCallee->getAttributes(), ResTy, ArgTy);
 
   if (Instruction *ArgInst = dyn_cast<Instruction>(Arg)) {
     // If the argument is an instruction, it must dominate all uses so put our
@@ -1837,8 +1844,8 @@ Value *LibCallSimplifier::optimizeFFS(CallInst *CI, IRBuilder<> &B) {
   // ffs(x) -> x != 0 ? (i32)llvm.cttz(x)+1 : 0
   Value *Op = CI->getArgOperand(0);
   Type *ArgType = Op->getType();
-  Value *F = Intrinsic::getDeclaration(CI->getCalledFunction()->getParent(),
-                                       Intrinsic::cttz, ArgType);
+  Function *F = Intrinsic::getDeclaration(CI->getCalledFunction()->getParent(),
+                                          Intrinsic::cttz, ArgType);
   Value *V = B.CreateCall(F, {Op, B.getTrue()}, "cttz");
   V = B.CreateAdd(V, ConstantInt::get(V->getType(), 1));
   V = B.CreateIntCast(V, B.getInt32Ty(), false);
@@ -1851,8 +1858,8 @@ Value *LibCallSimplifier::optimizeFls(CallInst *CI, IRBuilder<> &B) {
   // fls(x) -> (i32)(sizeInBits(x) - llvm.ctlz(x, false))
   Value *Op = CI->getArgOperand(0);
   Type *ArgType = Op->getType();
-  Value *F = Intrinsic::getDeclaration(CI->getCalledFunction()->getParent(),
-                                       Intrinsic::ctlz, ArgType);
+  Function *F = Intrinsic::getDeclaration(CI->getCalledFunction()->getParent(),
+                                          Intrinsic::ctlz, ArgType);
   Value *V = B.CreateCall(F, {Op, B.getFalse()}, "ctlz");
   V = B.CreateSub(ConstantInt::get(V->getType(), ArgType->getIntegerBitWidth()),
                   V);
@@ -2023,7 +2030,7 @@ Value *LibCallSimplifier::optimizePrintF(CallInst *CI, IRBuilder<> &B) {
   // arguments.
   if (TLI->has(LibFunc_iprintf) && !callHasFloatingPointArgument(CI)) {
     Module *M = B.GetInsertBlock()->getParent()->getParent();
-    Constant *IPrintFFn =
+    FunctionCallee IPrintFFn =
         M->getOrInsertFunction("iprintf", FT, Callee->getAttributes());
     CallInst *New = cast<CallInst>(CI->clone());
     New->setCalledFunction(IPrintFFn);
@@ -2102,7 +2109,7 @@ Value *LibCallSimplifier::optimizeSPrintF(CallInst *CI, IRBuilder<> &B) {
   // point arguments.
   if (TLI->has(LibFunc_siprintf) && !callHasFloatingPointArgument(CI)) {
     Module *M = B.GetInsertBlock()->getParent()->getParent();
-    Constant *SIPrintFFn =
+    FunctionCallee SIPrintFFn =
         M->getOrInsertFunction("siprintf", FT, Callee->getAttributes());
     CallInst *New = cast<CallInst>(CI->clone());
     New->setCalledFunction(SIPrintFFn);
@@ -2259,7 +2266,7 @@ Value *LibCallSimplifier::optimizeFPrintF(CallInst *CI, IRBuilder<> &B) {
   // floating point arguments.
   if (TLI->has(LibFunc_fiprintf) && !callHasFloatingPointArgument(CI)) {
     Module *M = B.GetInsertBlock()->getParent()->getParent();
-    Constant *FIPrintFFn =
+    FunctionCallee FIPrintFFn =
         M->getOrInsertFunction("fiprintf", FT, Callee->getAttributes());
     CallInst *New = cast<CallInst>(CI->clone());
     New->setCalledFunction(FIPrintFFn);
@@ -2285,7 +2292,8 @@ Value *LibCallSimplifier::optimizeFWrite(CallInst *CI, IRBuilder<> &B) {
     // If this is writing one byte, turn it into fputc.
     // This optimisation is only valid, if the return value is unused.
     if (Bytes == 1 && CI->use_empty()) { // fwrite(S,1,1,F) -> fputc(S[0],F)
-      Value *Char = B.CreateLoad(castToCStr(CI->getArgOperand(0), B), "char");
+      Value *Char = B.CreateLoad(B.getInt8Ty(),
+                                 castToCStr(CI->getArgOperand(0), B), "char");
       Value *NewCI = emitFPutC(Char, CI->getArgOperand(3), B, TLI);
       return NewCI ? ConstantInt::get(CI->getType(), 1) : nullptr;
     }
diff --git a/lib/Transforms/Utils/SplitModule.cpp b/lib/Transforms/Utils/SplitModule.cpp
index 5db4d2e4df9d6b13d3b09cf49380fd6d37f5236e..e2c387cb89835a7984d3857e7a604ba3a552cd2d 100644
--- a/lib/Transforms/Utils/SplitModule.cpp
+++ b/lib/Transforms/Utils/SplitModule.cpp
@@ -1,9 +1,8 @@
 //===- SplitModule.cpp - Split a module into partitions -------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Transforms/Utils/StripGCRelocates.cpp b/lib/Transforms/Utils/StripGCRelocates.cpp
index ac0b519f4a77f87651c561d2134e502db4df8a50..50844cf9d1c54cc833c1d0d045b99366754c079b 100644
--- a/lib/Transforms/Utils/StripGCRelocates.cpp
+++ b/lib/Transforms/Utils/StripGCRelocates.cpp
@@ -1,9 +1,8 @@
 //===- StripGCRelocates.cpp - Remove gc.relocates inserted by RewriteStatePoints===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Transforms/Utils/StripNonLineTableDebugInfo.cpp b/lib/Transforms/Utils/StripNonLineTableDebugInfo.cpp
index 8956a089a99c0ae85b6486c7b337152018e292df..97a4533fabe5a69a74db4a6e1e767de9eac038f5 100644
--- a/lib/Transforms/Utils/StripNonLineTableDebugInfo.cpp
+++ b/lib/Transforms/Utils/StripNonLineTableDebugInfo.cpp
@@ -1,9 +1,8 @@
 //===- StripNonLineTableDebugInfo.cpp -- Strip parts of Debug Info --------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Transforms/Utils/SymbolRewriter.cpp b/lib/Transforms/Utils/SymbolRewriter.cpp
index fd0da79487f1d61b1c39a592f6d1048c9686381c..456724779b4359b447b4e74ae982d1cb76376b41 100644
--- a/lib/Transforms/Utils/SymbolRewriter.cpp
+++ b/lib/Transforms/Utils/SymbolRewriter.cpp
@@ -1,9 +1,8 @@
 //===- SymbolRewriter.cpp - Symbol Rewriter -------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Transforms/Utils/UnifyFunctionExitNodes.cpp b/lib/Transforms/Utils/UnifyFunctionExitNodes.cpp
index d49b264725485cc294022ba5d52fd44c971abfe1..7f7bdf8a3d6d5cba37c24c53714a1da08ed94296 100644
--- a/lib/Transforms/Utils/UnifyFunctionExitNodes.cpp
+++ b/lib/Transforms/Utils/UnifyFunctionExitNodes.cpp
@@ -1,9 +1,8 @@
 //===- UnifyFunctionExitNodes.cpp - Make all functions have a single exit -===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Transforms/Utils/Utils.cpp b/lib/Transforms/Utils/Utils.cpp
index afd842f59911c8a495aac2d85b07f660ad7d8bfe..bcacd191b794b2d1d2cb5cbc6e9d03b6b188478f 100644
--- a/lib/Transforms/Utils/Utils.cpp
+++ b/lib/Transforms/Utils/Utils.cpp
@@ -1,9 +1,8 @@
 //===-- Utils.cpp - TransformUtils Infrastructure -------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -26,6 +25,7 @@ using namespace llvm;
 void llvm::initializeTransformUtils(PassRegistry &Registry) {
   initializeAddDiscriminatorsLegacyPassPass(Registry);
   initializeBreakCriticalEdgesPass(Registry);
+  initializeCanonicalizeAliasesLegacyPassPass(Registry);
   initializeInstNamerPass(Registry);
   initializeLCSSAWrapperPassPass(Registry);
   initializeLibCallsShrinkWrapLegacyPassPass(Registry);
diff --git a/lib/Transforms/Utils/VNCoercion.cpp b/lib/Transforms/Utils/VNCoercion.cpp
index 948d9bd5baad4df4fea7b1127cfb951b4bd7585d..19e5db4eee2a11a2b42c3c7c31ef24d7b72f9040 100644
--- a/lib/Transforms/Utils/VNCoercion.cpp
+++ b/lib/Transforms/Utils/VNCoercion.cpp
@@ -386,12 +386,12 @@ Value *getLoadValueForLoad(LoadInst *SrcVal, unsigned Offset, Type *LoadTy,
     // memdep queries will find the new load.  We can't easily remove the old
     // load completely because it is already in the value numbering table.
     IRBuilder<> Builder(SrcVal->getParent(), ++BasicBlock::iterator(SrcVal));
-    Type *DestPTy = IntegerType::get(LoadTy->getContext(), NewLoadSize * 8);
-    DestPTy =
-        PointerType::get(DestPTy, PtrVal->getType()->getPointerAddressSpace());
+    Type *DestTy = IntegerType::get(LoadTy->getContext(), NewLoadSize * 8);
+    Type *DestPTy =
+        PointerType::get(DestTy, PtrVal->getType()->getPointerAddressSpace());
     Builder.SetCurrentDebugLocation(SrcVal->getDebugLoc());
     PtrVal = Builder.CreateBitCast(PtrVal, DestPTy);
-    LoadInst *NewLoad = Builder.CreateLoad(PtrVal);
+    LoadInst *NewLoad = Builder.CreateLoad(DestTy, PtrVal);
     NewLoad->takeName(SrcVal);
     NewLoad->setAlignment(SrcVal->getAlignment());
 
diff --git a/lib/Transforms/Utils/ValueMapper.cpp b/lib/Transforms/Utils/ValueMapper.cpp
index 55fff3f3872a608d9d0c1d6023190b444278dac9..1710f97d23401a2df4ba6ddf742d3dcaecd8a261 100644
--- a/lib/Transforms/Utils/ValueMapper.cpp
+++ b/lib/Transforms/Utils/ValueMapper.cpp
@@ -1,9 +1,8 @@
 //===- ValueMapper.cpp - Interface shared by lib/Transforms/Utils ---------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/Transforms/Vectorize/CMakeLists.txt b/lib/Transforms/Vectorize/CMakeLists.txt
index 06eaadf58c3fce1f6b9c3531335d23d6ff06d370..f6bd97ceb645dbe719446c53bab160c6e26bc3e2 100644
--- a/lib/Transforms/Vectorize/CMakeLists.txt
+++ b/lib/Transforms/Vectorize/CMakeLists.txt
@@ -7,6 +7,7 @@ add_llvm_library(LLVMVectorize
   VPlan.cpp
   VPlanHCFGBuilder.cpp
   VPlanHCFGTransforms.cpp
+  VPlanPredicator.cpp
   VPlanSLP.cpp
   VPlanVerifier.cpp
 
diff --git a/lib/Transforms/Vectorize/LLVMBuild.txt b/lib/Transforms/Vectorize/LLVMBuild.txt
index be00294ea44b9bb9fc283af4aa63cf5412a30067..89346652ddeefecd16b8d7bf8b1ecaddec686b6f 100644
--- a/lib/Transforms/Vectorize/LLVMBuild.txt
+++ b/lib/Transforms/Vectorize/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/Transforms/Scalar/LLVMBuild.txt --------------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp b/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp
index 9ff18328c219d4eba51b0f342e5868076ba5caea..e0b793f08fd17f5447b41f9630c63f06434b280f 100644
--- a/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp
+++ b/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp
@@ -1,9 +1,8 @@
 //===- LoadStoreVectorizer.cpp - GPU Load & Store Vectorizer --------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -1182,7 +1181,7 @@ bool Vectorizer::vectorizeLoadChain(
 
   Value *Bitcast =
       Builder.CreateBitCast(L0->getPointerOperand(), VecTy->getPointerTo(AS));
-  LoadInst *LI = Builder.CreateAlignedLoad(Bitcast, Alignment);
+  LoadInst *LI = Builder.CreateAlignedLoad(VecTy, Bitcast, Alignment);
   propagateMetadata(LI, Chain);
 
   if (VecLoadTy) {
diff --git a/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp b/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
index ac989dd66f72d228d974df9f7b045aab38940dec..1d030944815f0c212f3a5df2a2358a849b870dcd 100644
--- a/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
+++ b/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
@@ -1,9 +1,8 @@
 //===- LoopVectorizationLegality.cpp --------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -23,6 +22,8 @@ using namespace llvm;
 #define LV_NAME "loop-vectorize"
 #define DEBUG_TYPE LV_NAME
 
+extern cl::opt<bool> EnableVPlanPredication;
+
 static cl::opt<bool>
     EnableIfConversion("enable-if-conversion", cl::init(true), cl::Hidden,
                        cl::desc("Enable if-conversion during vectorization."));
@@ -103,6 +104,25 @@ LoopVectorizeHints::LoopVectorizeHints(const Loop *L,
              << "LV: Interleaving disabled by the pass manager\n");
 }
 
+void LoopVectorizeHints::setAlreadyVectorized() {
+  LLVMContext &Context = TheLoop->getHeader()->getContext();
+
+  MDNode *IsVectorizedMD = MDNode::get(
+      Context,
+      {MDString::get(Context, "llvm.loop.isvectorized"),
+       ConstantAsMetadata::get(ConstantInt::get(Context, APInt(32, 1)))});
+  MDNode *LoopID = TheLoop->getLoopID();
+  MDNode *NewLoopID =
+      makePostTransformationMetadata(Context, LoopID,
+                                     {Twine(Prefix(), "vectorize.").str(),
+                                      Twine(Prefix(), "interleave.").str()},
+                                     {IsVectorizedMD});
+  TheLoop->setLoopID(NewLoopID);
+
+  // Update internal cache.
+  IsVectorized.Value = 1;
+}
+
 bool LoopVectorizeHints::allowVectorization(
     Function *F, Loop *L, bool VectorizeOnlyWhenForced) const {
   if (getForce() == LoopVectorizeHints::FK_Disabled) {
@@ -230,57 +250,6 @@ void LoopVectorizeHints::setHint(StringRef Name, Metadata *Arg) {
   }
 }
 
-MDNode *LoopVectorizeHints::createHintMetadata(StringRef Name,
-                                               unsigned V) const {
-  LLVMContext &Context = TheLoop->getHeader()->getContext();
-  Metadata *MDs[] = {
-      MDString::get(Context, Name),
-      ConstantAsMetadata::get(ConstantInt::get(Type::getInt32Ty(Context), V))};
-  return MDNode::get(Context, MDs);
-}
-
-bool LoopVectorizeHints::matchesHintMetadataName(MDNode *Node,
-                                                 ArrayRef<Hint> HintTypes) {
-  MDString *Name = dyn_cast<MDString>(Node->getOperand(0));
-  if (!Name)
-    return false;
-
-  for (auto H : HintTypes)
-    if (Name->getString().endswith(H.Name))
-      return true;
-  return false;
-}
-
-void LoopVectorizeHints::writeHintsToMetadata(ArrayRef<Hint> HintTypes) {
-  if (HintTypes.empty())
-    return;
-
-  // Reserve the first element to LoopID (see below).
-  SmallVector<Metadata *, 4> MDs(1);
-  // If the loop already has metadata, then ignore the existing operands.
-  MDNode *LoopID = TheLoop->getLoopID();
-  if (LoopID) {
-    for (unsigned i = 1, ie = LoopID->getNumOperands(); i < ie; ++i) {
-      MDNode *Node = cast<MDNode>(LoopID->getOperand(i));
-      // If node in update list, ignore old value.
-      if (!matchesHintMetadataName(Node, HintTypes))
-        MDs.push_back(Node);
-    }
-  }
-
-  // Now, add the missing hints.
-  for (auto H : HintTypes)
-    MDs.push_back(createHintMetadata(Twine(Prefix(), H.Name).str(), H.Value));
-
-  // Replace current metadata node with new one.
-  LLVMContext &Context = TheLoop->getHeader()->getContext();
-  MDNode *NewLoopID = MDNode::get(Context, MDs);
-  // Set operand 0 to refer to the loop id itself.
-  NewLoopID->replaceOperandWith(0, NewLoopID);
-
-  TheLoop->setLoopID(NewLoopID);
-}
-
 bool LoopVectorizationRequirements::doesNotMeet(
     Function *F, Loop *L, const LoopVectorizeHints &Hints) {
   const char *PassName = Hints.vectorizeAnalysisPassName();
@@ -488,7 +457,10 @@ bool LoopVectorizationLegality::canVectorizeOuterLoop() {
     // Check whether the BranchInst is a supported one. Only unconditional
     // branches, conditional branches with an outer loop invariant condition or
     // backedges are supported.
-    if (Br && Br->isConditional() &&
+    // FIXME: We skip these checks when VPlan predication is enabled as we
+    // want to allow divergent branches. This whole check will be removed
+    // once VPlan predication is on by default.
+    if (!EnableVPlanPredication && Br && Br->isConditional() &&
         !TheLoop->isLoopInvariant(Br->getCondition()) &&
         !LI->isLoopHeader(Br->getSuccessor(0)) &&
         !LI->isLoopHeader(Br->getSuccessor(1))) {
@@ -714,10 +686,30 @@ bool LoopVectorizationLegality::canVectorizeInstrs() {
           !isa<DbgInfoIntrinsic>(CI) &&
           !(CI->getCalledFunction() && TLI &&
             TLI->isFunctionVectorizable(CI->getCalledFunction()->getName()))) {
-        ORE->emit(createMissedAnalysis("CantVectorizeCall", CI)
-                  << "call instruction cannot be vectorized");
+        // If the call is a recognized math libary call, it is likely that
+        // we can vectorize it given loosened floating-point constraints.
+        LibFunc Func;
+        bool IsMathLibCall =
+            TLI && CI->getCalledFunction() &&
+            CI->getType()->isFloatingPointTy() &&
+            TLI->getLibFunc(CI->getCalledFunction()->getName(), Func) &&
+            TLI->hasOptimizedCodeGen(Func);
+
+        if (IsMathLibCall) {
+          // TODO: Ideally, we should not use clang-specific language here,
+          // but it's hard to provide meaningful yet generic advice.
+          // Also, should this be guarded by allowExtraAnalysis() and/or be part
+          // of the returned info from isFunctionVectorizable()?
+          ORE->emit(createMissedAnalysis("CantVectorizeLibcall", CI)
+              << "library call cannot be vectorized. "
+                 "Try compiling with -fno-math-errno, -ffast-math, "
+                 "or similar flags");
+        } else {
+          ORE->emit(createMissedAnalysis("CantVectorizeCall", CI)
+                    << "call instruction cannot be vectorized");
+        }
         LLVM_DEBUG(
-            dbgs() << "LV: Found a non-intrinsic, non-libfunc callsite.\n");
+            dbgs() << "LV: Found a non-intrinsic callsite.\n");
         return false;
       }
 
diff --git a/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index 2aa2190642993b55e22f0bd491ebd1e629911684..7264eb97e1b3db453d7827da090e4e018f3af51d 100644
--- a/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -1,9 +1,8 @@
 //===- LoopVectorizationPlanner.h - Planner for LoopVectorization ---------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
@@ -172,6 +171,9 @@ struct VectorizationFactor {
   unsigned Width;
   // Cost of the loop with that width
   unsigned Cost;
+
+  // Width 1 means no vectorization, cost 0 means uncomputed cost.
+  static VectorizationFactor Disabled() { return {1, 0}; }
 };
 
 /// Planner drives the vectorization process after having passed
@@ -222,8 +224,9 @@ public:
                            LoopVectorizationCostModel &CM)
       : OrigLoop(L), LI(LI), TLI(TLI), TTI(TTI), Legal(Legal), CM(CM) {}
 
-  /// Plan how to best vectorize, return the best VF and its cost.
-  VectorizationFactor plan(bool OptForSize, unsigned UserVF);
+  /// Plan how to best vectorize, return the best VF and its cost, or None if
+  /// vectorization and interleaving should be avoided up front.
+  Optional<VectorizationFactor> plan(bool OptForSize, unsigned UserVF);
 
   /// Use the VPlan-native path to plan how to best vectorize, return the best
   /// VF and its cost.
diff --git a/lib/Transforms/Vectorize/LoopVectorize.cpp b/lib/Transforms/Vectorize/LoopVectorize.cpp
index c74352cf7039b49e70066eac51a10bd250e4b21c..b10ce86b6825769ff9c1b472c903b22b1e4b2adc 100644
--- a/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -1,9 +1,8 @@
 //===- LoopVectorize.cpp - A Loop Vectorizer ------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -59,6 +58,7 @@
 #include "VPRecipeBuilder.h"
 #include "VPlanHCFGBuilder.h"
 #include "VPlanHCFGTransforms.h"
+#include "VPlanPredicator.h"
 #include "llvm/ADT/APInt.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/DenseMap.h"
@@ -86,6 +86,7 @@
 #include "llvm/Analysis/LoopAnalysisManager.h"
 #include "llvm/Analysis/LoopInfo.h"
 #include "llvm/Analysis/LoopIterator.h"
+#include "llvm/Analysis/MemorySSA.h"
 #include "llvm/Analysis/OptimizationRemarkEmitter.h"
 #include "llvm/Analysis/ScalarEvolution.h"
 #include "llvm/Analysis/ScalarEvolutionExpander.h"
@@ -256,6 +257,13 @@ cl::opt<bool> EnableVPlanNativePath(
     cl::desc("Enable VPlan-native vectorization path with "
              "support for outer loop vectorization."));
 
+// FIXME: Remove this switch once we have divergence analysis. Currently we
+// assume divergent non-backedge branches when this switch is true.
+cl::opt<bool> EnableVPlanPredication(
+    "enable-vplan-predication", cl::init(false), cl::Hidden,
+    cl::desc("Enable VPlan-native vectorization path predicator with "
+             "support for outer loop vectorization."));
+
 // This flag enables the stress testing of the VPlan H-CFG construction in the
 // VPlan-native vectorization path. It must be used in conjuction with
 // -enable-vplan-native-path. -vplan-verify-hcfg can also be used to enable the
@@ -759,8 +767,15 @@ void InnerLoopVectorizer::setDebugLocFromInst(IRBuilder<> &B, const Value *Ptr)
   if (const Instruction *Inst = dyn_cast_or_null<Instruction>(Ptr)) {
     const DILocation *DIL = Inst->getDebugLoc();
     if (DIL && Inst->getFunction()->isDebugInfoForProfiling() &&
-        !isa<DbgInfoIntrinsic>(Inst))
-      B.SetCurrentDebugLocation(DIL->cloneWithDuplicationFactor(UF * VF));
+        !isa<DbgInfoIntrinsic>(Inst)) {
+      auto NewDIL = DIL->cloneByMultiplyingDuplicationFactor(UF * VF);
+      if (NewDIL)
+        B.SetCurrentDebugLocation(NewDIL.getValue());
+      else
+        LLVM_DEBUG(dbgs()
+                   << "Failed to create new discriminator: "
+                   << DIL->getFilename() << " Line: " << DIL->getLine());
+    }
     else
       B.SetCurrentDebugLocation(DIL);
   } else
@@ -829,7 +844,7 @@ public:
     AC(AC), ORE(ORE), TheFunction(F), Hints(Hints), InterleaveInfo(IAI) {}
 
   /// \return An upper bound for the vectorization factor, or None if
-  /// vectorization should be avoided up front.
+  /// vectorization and interleaving should be avoided up front.
   Optional<unsigned> computeMaxVF(bool OptForSize);
 
   /// \return The most profitable vectorization factor and the cost of that VF.
@@ -2044,7 +2059,7 @@ void InnerLoopVectorizer::vectorizeInterleaveGroup(Instruction *Instr,
     //       A[i]   = b;     // Member of index 0
     //       A[i+2] = c;     // Member of index 2 (Current instruction)
     // Current pointer is pointed to A[i+2], adjust it to A[i].
-    NewPtr = Builder.CreateGEP(NewPtr, Builder.getInt32(-Index));
+    NewPtr = Builder.CreateGEP(ScalarTy, NewPtr, Builder.getInt32(-Index));
     if (InBounds)
       cast<GetElementPtrInst>(NewPtr)->setIsInBounds(true);
 
@@ -2086,8 +2101,8 @@ void InnerLoopVectorizer::vectorizeInterleaveGroup(Instruction *Instr,
                                      GroupMask, UndefVec, "wide.masked.vec");
       }
       else
-        NewLoad = Builder.CreateAlignedLoad(NewPtrs[Part], 
-          Group->getAlignment(), "wide.vec");
+        NewLoad = Builder.CreateAlignedLoad(VecTy, NewPtrs[Part],
+                                            Group->getAlignment(), "wide.vec");
       Group->addMetadata(NewLoad);
       NewLoads.push_back(NewLoad);
     }
@@ -2232,16 +2247,16 @@ void InnerLoopVectorizer::vectorizeMemoryInstruction(Instruction *Instr,
       // If the address is consecutive but reversed, then the
       // wide store needs to start at the last vector element.
       PartPtr = cast<GetElementPtrInst>(
-          Builder.CreateGEP(Ptr, Builder.getInt32(-Part * VF)));
+          Builder.CreateGEP(ScalarDataTy, Ptr, Builder.getInt32(-Part * VF)));
       PartPtr->setIsInBounds(InBounds);
       PartPtr = cast<GetElementPtrInst>(
-          Builder.CreateGEP(PartPtr, Builder.getInt32(1 - VF)));
+          Builder.CreateGEP(ScalarDataTy, PartPtr, Builder.getInt32(1 - VF)));
       PartPtr->setIsInBounds(InBounds);
       if (isMaskRequired) // Reverse of a null all-one mask is a null mask.
         Mask[Part] = reverseVector(Mask[Part]);
     } else {
       PartPtr = cast<GetElementPtrInst>(
-          Builder.CreateGEP(Ptr, Builder.getInt32(Part * VF)));
+          Builder.CreateGEP(ScalarDataTy, Ptr, Builder.getInt32(Part * VF)));
       PartPtr->setIsInBounds(InBounds);
     }
 
@@ -2298,7 +2313,8 @@ void InnerLoopVectorizer::vectorizeMemoryInstruction(Instruction *Instr,
                                          UndefValue::get(DataTy),
                                          "wide.masked.load");
       else
-        NewLI = Builder.CreateAlignedLoad(VecPtr, Alignment, "wide.load");
+        NewLI =
+            Builder.CreateAlignedLoad(DataTy, VecPtr, Alignment, "wide.load");
 
       // Add metadata to the load, but setVectorValue to the reverse shuffle.
       addMetadata(NewLI, LI);
@@ -2658,7 +2674,7 @@ Value *InnerLoopVectorizer::emitTransformedIndex(
     assert(isa<SCEVConstant>(Step) &&
            "Expected constant step for pointer induction");
     return B.CreateGEP(
-        nullptr, StartValue,
+        StartValue->getType()->getPointerElementType(), StartValue,
         CreateMul(Index, Exp.expandCodeFor(Step, Index->getType(),
                                            &*B.GetInsertPoint())));
   }
@@ -3928,9 +3944,11 @@ void InnerLoopVectorizer::widenInstruction(Instruction &I) {
 
         // Create the new GEP. Note that this GEP may be a scalar if VF == 1,
         // but it should be a vector, otherwise.
-        auto *NewGEP = GEP->isInBounds()
-                           ? Builder.CreateInBoundsGEP(Ptr, Indices)
-                           : Builder.CreateGEP(Ptr, Indices);
+        auto *NewGEP =
+            GEP->isInBounds()
+                ? Builder.CreateInBoundsGEP(GEP->getSourceElementType(), Ptr,
+                                            Indices)
+                : Builder.CreateGEP(GEP->getSourceElementType(), Ptr, Indices);
         assert((VF == 1 || NewGEP->getType()->isVectorTy()) &&
                "NewGEP is not a pointer vector");
         VectorLoopValueMap.setVectorValue(&I, Part, NewGEP);
@@ -6062,9 +6080,6 @@ void LoopVectorizationCostModel::collectValuesToIgnore() {
 VectorizationFactor
 LoopVectorizationPlanner::planInVPlanNativePath(bool OptForSize,
                                                 unsigned UserVF) {
-  // Width 1 means no vectorization, cost 0 means uncomputed cost.
-  const VectorizationFactor NoVectorization = {1U, 0U};
-
   // Outer loop handling: They may require CFG and instruction level
   // transformations before even evaluating whether vectorization is profitable.
   // Since we cannot modify the incoming IR, we need to build VPlan upfront in
@@ -6084,7 +6099,7 @@ LoopVectorizationPlanner::planInVPlanNativePath(bool OptForSize,
 
     // For VPlan build stress testing, we bail out after VPlan construction.
     if (VPlanBuildStressTest)
-      return NoVectorization;
+      return VectorizationFactor::Disabled();
 
     return {UserVF, 0};
   }
@@ -6092,17 +6107,15 @@ LoopVectorizationPlanner::planInVPlanNativePath(bool OptForSize,
   LLVM_DEBUG(
       dbgs() << "LV: Not vectorizing. Inner loops aren't supported in the "
                 "VPlan-native path.\n");
-  return NoVectorization;
+  return VectorizationFactor::Disabled();
 }
 
-VectorizationFactor
-LoopVectorizationPlanner::plan(bool OptForSize, unsigned UserVF) {
+Optional<VectorizationFactor> LoopVectorizationPlanner::plan(bool OptForSize,
+                                                             unsigned UserVF) {
   assert(OrigLoop->empty() && "Inner loop expected.");
-  // Width 1 means no vectorization, cost 0 means uncomputed cost.
-  const VectorizationFactor NoVectorization = {1U, 0U};
   Optional<unsigned> MaybeMaxVF = CM.computeMaxVF(OptForSize);
-  if (!MaybeMaxVF.hasValue()) // Cases considered too costly to vectorize.
-    return NoVectorization;
+  if (!MaybeMaxVF) // Cases that should not to be vectorized nor interleaved.
+    return None;
 
   // Invalidate interleave groups if all blocks of loop will be predicated.
   if (CM.blockNeedsPredication(OrigLoop->getHeader()) &&
@@ -6122,7 +6135,7 @@ LoopVectorizationPlanner::plan(bool OptForSize, unsigned UserVF) {
     CM.selectUserVectorizationFactor(UserVF);
     buildVPlansWithVPRecipes(UserVF, UserVF);
     LLVM_DEBUG(printPlans(dbgs()));
-    return {UserVF, 0};
+    return {{UserVF, 0}};
   }
 
   unsigned MaxVF = MaybeMaxVF.getValue();
@@ -6141,7 +6154,7 @@ LoopVectorizationPlanner::plan(bool OptForSize, unsigned UserVF) {
   buildVPlansWithVPRecipes(1, MaxVF);
   LLVM_DEBUG(printPlans(dbgs()));
   if (MaxVF == 1)
-    return NoVectorization;
+    return VectorizationFactor::Disabled();
 
   // Select the optimal vectorization factor.
   return CM.selectVectorizationFactor(MaxVF);
@@ -6890,13 +6903,22 @@ LoopVectorizationPlanner::buildVPlan(VFRange &Range) {
   VPlanHCFGBuilder HCFGBuilder(OrigLoop, LI, *Plan);
   HCFGBuilder.buildHierarchicalCFG();
 
+  for (unsigned VF = Range.Start; VF < Range.End; VF *= 2)
+    Plan->addVF(VF);
+
+  if (EnableVPlanPredication) {
+    VPlanPredicator VPP(*Plan);
+    VPP.predicate();
+
+    // Avoid running transformation to recipes until masked code generation in
+    // VPlan-native path is in place.
+    return Plan;
+  }
+
   SmallPtrSet<Instruction *, 1> DeadInstructions;
   VPlanHCFGTransforms::VPInstructionsToVPRecipes(
       Plan, Legal->getInductionVars(), DeadInstructions);
 
-  for (unsigned VF = Range.Start; VF < Range.End; VF *= 2)
-    Plan->addVF(VF);
-
   return Plan;
 }
 
@@ -7113,8 +7135,8 @@ static bool processLoopInVPlanNativePath(
   VectorizationFactor VF = LVP.planInVPlanNativePath(OptForSize, UserVF);
 
   // If we are stress testing VPlan builds, do not attempt to generate vector
-  // code.
-  if (VPlanBuildStressTest)
+  // code. Masked vector code generation support will follow soon.
+  if (VPlanBuildStressTest || EnableVPlanPredication)
     return false;
 
   LVP.setBestPlan(VF.Width, 1);
@@ -7297,14 +7319,18 @@ bool LoopVectorizePass::processLoop(Loop *L) {
   unsigned UserVF = Hints.getWidth();
 
   // Plan how to best vectorize, return the best VF and its cost.
-  VectorizationFactor VF = LVP.plan(OptForSize, UserVF);
+  Optional<VectorizationFactor> MaybeVF = LVP.plan(OptForSize, UserVF);
 
-  // Select the interleave count.
-  unsigned IC = CM.selectInterleaveCount(OptForSize, VF.Width, VF.Cost);
-
-  // Get user interleave count.
+  VectorizationFactor VF = VectorizationFactor::Disabled();
+  unsigned IC = 1;
   unsigned UserIC = Hints.getInterleave();
 
+  if (MaybeVF) {
+    VF = *MaybeVF;
+    // Select the interleave count.
+    IC = CM.selectInterleaveCount(OptForSize, VF.Width, VF.Cost);
+  }
+
   // Identify the diagnostic messages that should be produced.
   std::pair<StringRef, std::string> VecDiagMsg, IntDiagMsg;
   bool VectorizeLoop = true, InterleaveLoop = true;
@@ -7323,7 +7349,16 @@ bool LoopVectorizePass::processLoop(Loop *L) {
     VectorizeLoop = false;
   }
 
-  if (IC == 1 && UserIC <= 1) {
+  if (!MaybeVF && UserIC > 1) {
+    // Tell the user interleaving was avoided up-front, despite being explicitly
+    // requested.
+    LLVM_DEBUG(dbgs() << "LV: Ignoring UserIC, because vectorization and "
+                         "interleaving should be avoided up front\n");
+    IntDiagMsg = std::make_pair(
+        "InterleavingAvoided",
+        "Ignoring UserIC, because interleaving was avoided up front");
+    InterleaveLoop = false;
+  } else if (IC == 1 && UserIC <= 1) {
     // Tell the user interleaving is not beneficial.
     LLVM_DEBUG(dbgs() << "LV: Interleaving is not beneficial.\n");
     IntDiagMsg = std::make_pair(
@@ -7520,11 +7555,14 @@ PreservedAnalyses LoopVectorizePass::run(Function &F,
     auto &AC = AM.getResult<AssumptionAnalysis>(F);
     auto &DB = AM.getResult<DemandedBitsAnalysis>(F);
     auto &ORE = AM.getResult<OptimizationRemarkEmitterAnalysis>(F);
+    MemorySSA *MSSA = EnableMSSALoopDependency
+                          ? &AM.getResult<MemorySSAAnalysis>(F).getMSSA()
+                          : nullptr;
 
     auto &LAM = AM.getResult<LoopAnalysisManagerFunctionProxy>(F).getManager();
     std::function<const LoopAccessInfo &(Loop &)> GetLAA =
         [&](Loop &L) -> const LoopAccessInfo & {
-      LoopStandardAnalysisResults AR = {AA, AC, DT, LI, SE, TLI, TTI, nullptr};
+      LoopStandardAnalysisResults AR = {AA, AC, DT, LI, SE, TLI, TTI, MSSA};
       return LAM.getResult<LoopAccessAnalysis>(L, AR);
     };
     bool Changed =
diff --git a/lib/Transforms/Vectorize/SLPVectorizer.cpp b/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 87a861970d10a0e33c9d2ef427c6b50792db24a1..446a78be76926b374e5312ec58cea706e7a39b85 100644
--- a/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -1,9 +1,8 @@
 //===- SLPVectorizer.cpp - A bottom up SLP Vectorizer ---------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -1468,8 +1467,9 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
 
   // If any of the scalars is marked as a value that needs to stay scalar, then
   // we need to gather the scalars.
+  // The reduction nodes (stored in UserIgnoreList) also should stay scalar.
   for (unsigned i = 0, e = VL.size(); i != e; ++i) {
-    if (MustGather.count(VL[i])) {
+    if (MustGather.count(VL[i]) || is_contained(UserIgnoreList, VL[i])) {
       LLVM_DEBUG(dbgs() << "SLP: Gathering due to gathered scalar.\n");
       newTreeEntry(VL, false, UserTreeIdx);
       return;
@@ -2164,7 +2164,7 @@ int BoUpSLP::getEntryCost(TreeEntry *E) {
                 // extractelement/ext pair.
                 DeadCost -= TTI->getExtractWithExtendCost(
                     Ext->getOpcode(), Ext->getType(), VecTy, i);
-                // Add back the cost of s|zext which is subtracted seperately.
+                // Add back the cost of s|zext which is subtracted separately.
                 DeadCost += TTI->getCastInstrCost(
                     Ext->getOpcode(), Ext->getType(), E->getType(), Ext);
                 continue;
@@ -2536,13 +2536,13 @@ int BoUpSLP::getTreeCost() {
     // uses. However, we should not compute the cost of duplicate sequences.
     // For example, if we have a build vector (i.e., insertelement sequence)
     // that is used by more than one vector instruction, we only need to
-    // compute the cost of the insertelement instructions once. The redundent
+    // compute the cost of the insertelement instructions once. The redundant
     // instructions will be eliminated by CSE.
     //
     // We should consider not creating duplicate tree entries for gather
     // sequences, and instead add additional edges to the tree representing
     // their uses. Since such an approach results in fewer total entries,
-    // existing heuristics based on tree size may yeild different results.
+    // existing heuristics based on tree size may yield different results.
     //
     if (TE.NeedToGather &&
         std::any_of(std::next(VectorizableTree.begin(), I + 1),
@@ -3136,7 +3136,7 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
         Builder.SetInsertPoint(LI);
         PointerType *PtrTy = PointerType::get(VecTy, LI->getPointerAddressSpace());
         Value *Ptr = Builder.CreateBitCast(LI->getOperand(0), PtrTy);
-        LoadInst *V = Builder.CreateAlignedLoad(Ptr, LI->getAlignment());
+        LoadInst *V = Builder.CreateAlignedLoad(VecTy, Ptr, LI->getAlignment());
         Value *NewV = propagateMetadata(V, E->Scalars);
         if (!E->ReorderIndices.empty()) {
           OrdersType Mask;
@@ -3341,7 +3341,7 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
         ExternalUses.push_back(ExternalUser(PO, cast<User>(VecPtr), 0));
 
       unsigned Alignment = LI->getAlignment();
-      LI = Builder.CreateLoad(VecPtr);
+      LI = Builder.CreateLoad(VecTy, VecPtr);
       if (!Alignment) {
         Alignment = DL->getABITypeAlignment(ScalarLoadTy);
       }
@@ -4269,7 +4269,7 @@ unsigned BoUpSLP::getVectorElementSize(Value *V) {
     Worklist.push_back(I);
 
   // Traverse the expression tree in bottom-up order looking for loads. If we
-  // encounter an instruciton we don't yet handle, we give up.
+  // encounter an instruction we don't yet handle, we give up.
   auto MaxWidth = 0u;
   auto FoundUnknownInst = false;
   while (!Worklist.empty() && !FoundUnknownInst) {
@@ -4746,38 +4746,29 @@ bool SLPVectorizerPass::vectorizeStores(ArrayRef<StoreInst *> Stores,
   BoUpSLP::ValueSet VectorizedStores;
   bool Changed = false;
 
-  // Do a quadratic search on all of the given stores in reverse order and find
-  // all of the pairs of stores that follow each other.
-  SmallVector<unsigned, 16> IndexQueue;
-  unsigned E = Stores.size();
-  IndexQueue.resize(E - 1);
-  for (unsigned I = E; I > 0; --I) {
-    unsigned Idx = I - 1;
-    // If a store has multiple consecutive store candidates, search Stores
-    // array according to the sequence: Idx-1, Idx+1, Idx-2, Idx+2, ...
-    // This is because usually pairing with immediate succeeding or preceding
-    // candidate create the best chance to find slp vectorization opportunity.
-    unsigned Offset = 1;
-    unsigned Cnt = 0;
-    for (unsigned J = 0; J < E - 1; ++J, ++Offset) {
-      if (Idx >= Offset) {
-        IndexQueue[Cnt] = Idx - Offset;
-        ++Cnt;
-      }
-      if (Idx + Offset < E) {
-        IndexQueue[Cnt] = Idx + Offset;
-        ++Cnt;
-      }
-    }
+  auto &&FindConsecutiveAccess =
+      [this, &Stores, &Heads, &Tails, &ConsecutiveChain] (int K, int Idx) {
+        if (!isConsecutiveAccess(Stores[K], Stores[Idx], *DL, *SE))
+          return false;
 
-    for (auto K : IndexQueue) {
-      if (isConsecutiveAccess(Stores[K], Stores[Idx], *DL, *SE)) {
         Tails.insert(Stores[Idx]);
         Heads.insert(Stores[K]);
         ConsecutiveChain[Stores[K]] = Stores[Idx];
+        return true;
+      };
+
+  // Do a quadratic search on all of the given stores in reverse order and find
+  // all of the pairs of stores that follow each other.
+  int E = Stores.size();
+  for (int Idx = E - 1; Idx >= 0; --Idx) {
+    // If a store has multiple consecutive store candidates, search according
+    // to the sequence: Idx-1, Idx+1, Idx-2, Idx+2, ...
+    // This is because usually pairing with immediate succeeding or preceding
+    // candidate create the best chance to find slp vectorization opportunity.
+    for (int Offset = 1, F = std::max(E - Idx, Idx + 1); Offset < F; ++Offset)
+      if ((Idx >= Offset && FindConsecutiveAccess(Idx - Offset, Idx)) ||
+          (Idx + Offset < E && FindConsecutiveAccess(Idx + Offset, Idx)))
         break;
-      }
-    }
   }
 
   // For stores that start but don't end a link in the chain:
diff --git a/lib/Transforms/Vectorize/VPRecipeBuilder.h b/lib/Transforms/Vectorize/VPRecipeBuilder.h
index 15d38ac9c84c89e8630c54a5676638e66052ba44..bc6b221209953a32b605cbc2dc0e6d8f8fcb3387 100644
--- a/lib/Transforms/Vectorize/VPRecipeBuilder.h
+++ b/lib/Transforms/Vectorize/VPRecipeBuilder.h
@@ -1,9 +1,8 @@
 //===- VPRecipeBuilder.h - Helper class to build recipes --------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/Transforms/Vectorize/VPlan.cpp b/lib/Transforms/Vectorize/VPlan.cpp
index 05a5400beb4e69e49e26fe6ff988150229c33a4e..83d214850c152ba3305191dcc7c04cc6d384d47c 100644
--- a/lib/Transforms/Vectorize/VPlan.cpp
+++ b/lib/Transforms/Vectorize/VPlan.cpp
@@ -1,9 +1,8 @@
 //===- VPlan.cpp - Vectorizer Plan ----------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
@@ -561,6 +560,19 @@ void VPlanPrinter::dumpBasicBlock(const VPBasicBlock *BasicBlock) {
   bumpIndent(1);
   OS << Indent << "\"" << DOT::EscapeString(BasicBlock->getName()) << ":\\n\"";
   bumpIndent(1);
+
+  // Dump the block predicate.
+  const VPValue *Pred = BasicBlock->getPredicate();
+  if (Pred) {
+    OS << " +\n" << Indent << " \"BlockPredicate: ";
+    if (const VPInstruction *PredI = dyn_cast<VPInstruction>(Pred)) {
+      PredI->printAsOperand(OS);
+      OS << " (" << DOT::EscapeString(PredI->getParent()->getName())
+         << ")\\l\"";
+    } else
+      Pred->printAsOperand(OS);
+  }
+
   for (const VPRecipeBase &Recipe : *BasicBlock)
     Recipe.print(OS, Indent);
 
diff --git a/lib/Transforms/Vectorize/VPlan.h b/lib/Transforms/Vectorize/VPlan.h
index 5c1b4a83c30e10991dce1adba2ceac4decba0741..8a06412ad59067015ace9bb9eb9ba9abc1257dbb 100644
--- a/lib/Transforms/Vectorize/VPlan.h
+++ b/lib/Transforms/Vectorize/VPlan.h
@@ -1,9 +1,8 @@
 //===- VPlan.h - Represent A Vectorizer Plan --------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -353,6 +352,9 @@ private:
   /// Successor selector, null for zero or single successor blocks.
   VPValue *CondBit = nullptr;
 
+  /// Current block predicate - null if the block does not need a predicate.
+  VPValue *Predicate = nullptr;
+
   /// Add \p Successor as the last successor to this block.
   void appendSuccessor(VPBlockBase *Successor) {
     assert(Successor && "Cannot add nullptr successor!");
@@ -491,6 +493,12 @@ public:
 
   void setCondBit(VPValue *CV) { CondBit = CV; }
 
+  VPValue *getPredicate() { return Predicate; }
+
+  const VPValue *getPredicate() const { return Predicate; }
+
+  void setPredicate(VPValue *Pred) { Predicate = Pred; }
+
   /// Set a given VPBlockBase \p Successor as the single successor of this
   /// VPBlockBase. This VPBlockBase is not added as predecessor of \p Successor.
   /// This VPBlockBase must have no successors.
@@ -521,6 +529,15 @@ public:
       appendPredecessor(Pred);
   }
 
+  /// Remove all the predecessor of this block.
+  void clearPredecessors() { Predecessors.clear(); }
+
+  /// Remove all the successors of this block and set to null its condition bit
+  void clearSuccessors() {
+    Successors.clear();
+    CondBit = nullptr;
+  }
+
   /// The method which generates the output IR that correspond to this
   /// VPBlockBase, thereby "executing" the VPlan.
   virtual void execute(struct VPTransformState *State) = 0;
@@ -1491,6 +1508,41 @@ public:
     From->removeSuccessor(To);
     To->removePredecessor(From);
   }
+
+  /// Returns true if the edge \p FromBlock -> \p ToBlock is a back-edge.
+  static bool isBackEdge(const VPBlockBase *FromBlock,
+                         const VPBlockBase *ToBlock, const VPLoopInfo *VPLI) {
+    assert(FromBlock->getParent() == ToBlock->getParent() &&
+           FromBlock->getParent() && "Must be in same region");
+    const VPLoop *FromLoop = VPLI->getLoopFor(FromBlock);
+    const VPLoop *ToLoop = VPLI->getLoopFor(ToBlock);
+    if (!FromLoop || !ToLoop || FromLoop != ToLoop)
+      return false;
+
+    // A back-edge is a branch from the loop latch to its header.
+    return ToLoop->isLoopLatch(FromBlock) && ToBlock == ToLoop->getHeader();
+  }
+
+  /// Returns true if \p Block is a loop latch
+  static bool blockIsLoopLatch(const VPBlockBase *Block,
+                               const VPLoopInfo *VPLInfo) {
+    if (const VPLoop *ParentVPL = VPLInfo->getLoopFor(Block))
+      return ParentVPL->isLoopLatch(Block);
+
+    return false;
+  }
+
+  /// Count and return the number of succesors of \p PredBlock excluding any
+  /// backedges.
+  static unsigned countSuccessorsNoBE(VPBlockBase *PredBlock,
+                                      VPLoopInfo *VPLI) {
+    unsigned Count = 0;
+    for (VPBlockBase *SuccBlock : PredBlock->getSuccessors()) {
+      if (!VPBlockUtils::isBackEdge(PredBlock, SuccBlock, VPLI))
+        Count++;
+    }
+    return Count;
+  }
 };
 
 class VPInterleavedAccessInfo {
diff --git a/lib/Transforms/Vectorize/VPlanDominatorTree.h b/lib/Transforms/Vectorize/VPlanDominatorTree.h
index 1b81097b6d319a18845af2c1beb5b62b56b1adb7..19f5d2c00c604568f348fc22a40178c0f915077f 100644
--- a/lib/Transforms/Vectorize/VPlanDominatorTree.h
+++ b/lib/Transforms/Vectorize/VPlanDominatorTree.h
@@ -1,9 +1,8 @@
 //===-- VPlanDominatorTree.h ------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
diff --git a/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp b/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp
index 0f42694e193bc2fa84f63c9c383f028a97c16d30..0550253c21d8cb058f1a8dc60377ee897ceebac8 100644
--- a/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp
+++ b/lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp
@@ -1,9 +1,8 @@
 //===-- VPlanHCFGBuilder.cpp ----------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
diff --git a/lib/Transforms/Vectorize/VPlanHCFGBuilder.h b/lib/Transforms/Vectorize/VPlanHCFGBuilder.h
index 3f11dcb5164d0c08d001482b37d31e0faa05e1d1..238ee7e6347c098dbec3809e54354b1a8e711ffb 100644
--- a/lib/Transforms/Vectorize/VPlanHCFGBuilder.h
+++ b/lib/Transforms/Vectorize/VPlanHCFGBuilder.h
@@ -1,9 +1,8 @@
 //===-- VPlanHCFGBuilder.h --------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
diff --git a/lib/Transforms/Vectorize/VPlanHCFGTransforms.cpp b/lib/Transforms/Vectorize/VPlanHCFGTransforms.cpp
index 3ad7fc7e7b96152e161f00473a9d54a03ceda67f..7ed7d21b6caa88b971aacc5f908ca66e666c5de4 100644
--- a/lib/Transforms/Vectorize/VPlanHCFGTransforms.cpp
+++ b/lib/Transforms/Vectorize/VPlanHCFGTransforms.cpp
@@ -1,9 +1,8 @@
 //===-- VPlanHCFGTransforms.cpp - Utility VPlan to VPlan transforms -------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
diff --git a/lib/Transforms/Vectorize/VPlanHCFGTransforms.h b/lib/Transforms/Vectorize/VPlanHCFGTransforms.h
index ae549c6871b3fdc0d4c0ca616bdc0dab396ffc13..79a23c33184f6389f28b7eb1d20b83da6e74ab89 100644
--- a/lib/Transforms/Vectorize/VPlanHCFGTransforms.h
+++ b/lib/Transforms/Vectorize/VPlanHCFGTransforms.h
@@ -1,9 +1,8 @@
 //===- VPlanHCFGTransforms.h - Utility VPlan to VPlan transforms ----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
diff --git a/lib/Transforms/Vectorize/VPlanLoopInfo.h b/lib/Transforms/Vectorize/VPlanLoopInfo.h
index 5c2485fc2145552492f1d8997d4f5b672afcfac8..5208f2d58e2bee851e3a1eba537fac9efead0106 100644
--- a/lib/Transforms/Vectorize/VPlanLoopInfo.h
+++ b/lib/Transforms/Vectorize/VPlanLoopInfo.h
@@ -1,9 +1,8 @@
 //===-- VPLoopInfo.h --------------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
diff --git a/lib/Transforms/Vectorize/VPlanPredicator.cpp b/lib/Transforms/Vectorize/VPlanPredicator.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..7a80f3ff80a5c3bb48d03a2953fc320d804dc629
--- /dev/null
+++ b/lib/Transforms/Vectorize/VPlanPredicator.cpp
@@ -0,0 +1,248 @@
+//===-- VPlanPredicator.cpp -------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file implements the VPlanPredicator class which contains the public
+/// interfaces to predicate and linearize the VPlan region.
+///
+//===----------------------------------------------------------------------===//
+
+#include "VPlanPredicator.h"
+#include "VPlan.h"
+#include "llvm/ADT/DepthFirstIterator.h"
+#include "llvm/ADT/GraphTraits.h"
+#include "llvm/ADT/PostOrderIterator.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/raw_ostream.h"
+
+#define DEBUG_TYPE "VPlanPredicator"
+
+using namespace llvm;
+
+// Generate VPInstructions at the beginning of CurrBB that calculate the
+// predicate being propagated from PredBB to CurrBB depending on the edge type
+// between them. For example if:
+//  i.  PredBB is controlled by predicate %BP, and
+//  ii. The edge PredBB->CurrBB is the false edge, controlled by the condition
+//  bit value %CBV then this function will generate the following two
+//  VPInstructions at the start of CurrBB:
+//   %IntermediateVal = not %CBV
+//   %FinalVal        = and %BP %IntermediateVal
+// It returns %FinalVal.
+VPValue *VPlanPredicator::getOrCreateNotPredicate(VPBasicBlock *PredBB,
+                                                  VPBasicBlock *CurrBB) {
+  VPValue *CBV = PredBB->getCondBit();
+
+  // Set the intermediate value - this is either 'CBV', or 'not CBV'
+  // depending on the edge type.
+  EdgeType ET = getEdgeTypeBetween(PredBB, CurrBB);
+  VPValue *IntermediateVal = nullptr;
+  switch (ET) {
+  case EdgeType::TRUE_EDGE:
+    // CurrBB is the true successor of PredBB - nothing to do here.
+    IntermediateVal = CBV;
+    break;
+
+  case EdgeType::FALSE_EDGE:
+    // CurrBB is the False successor of PredBB - compute not of CBV.
+    IntermediateVal = Builder.createNot(CBV);
+    break;
+  }
+
+  // Now AND intermediate value with PredBB's block predicate if it has one.
+  VPValue *BP = PredBB->getPredicate();
+  if (BP)
+    return Builder.createAnd(BP, IntermediateVal);
+  else
+    return IntermediateVal;
+}
+
+// Generate a tree of ORs for all IncomingPredicates in  WorkList.
+// Note: This function destroys the original Worklist.
+//
+// P1 P2 P3 P4 P5
+//  \ /   \ /  /
+//  OR1   OR2 /
+//    \    | /
+//     \   +/-+
+//      \  /  |
+//       OR3  |
+//         \  |
+//          OR4 <- Returns this
+//           |
+//
+// The algorithm uses a worklist of predicates as its main data structure.
+// We pop a pair of values from the front (e.g. P1 and P2), generate an OR
+// (in this example OR1), and push it back. In this example the worklist
+// contains {P3, P4, P5, OR1}.
+// The process iterates until we have only one element in the Worklist (OR4).
+// The last element is the root predicate which is returned.
+VPValue *VPlanPredicator::genPredicateTree(std::list<VPValue *> &Worklist) {
+  if (Worklist.empty())
+    return nullptr;
+
+  // The worklist initially contains all the leaf nodes. Initialize the tree
+  // using them.
+  while (Worklist.size() >= 2) {
+    // Pop a pair of values from the front.
+    VPValue *LHS = Worklist.front();
+    Worklist.pop_front();
+    VPValue *RHS = Worklist.front();
+    Worklist.pop_front();
+
+    // Create an OR of these values.
+    VPValue *Or = Builder.createOr(LHS, RHS);
+
+    // Push OR to the back of the worklist.
+    Worklist.push_back(Or);
+  }
+
+  assert(Worklist.size() == 1 && "Expected 1 item in worklist");
+
+  // The root is the last node in the worklist.
+  VPValue *Root = Worklist.front();
+
+  // This root needs to replace the existing block predicate. This is done in
+  // the caller function.
+  return Root;
+}
+
+// Return whether the edge FromBlock -> ToBlock is a TRUE_EDGE or FALSE_EDGE
+VPlanPredicator::EdgeType
+VPlanPredicator::getEdgeTypeBetween(VPBlockBase *FromBlock,
+                                    VPBlockBase *ToBlock) {
+  unsigned Count = 0;
+  for (VPBlockBase *SuccBlock : FromBlock->getSuccessors()) {
+    if (SuccBlock == ToBlock) {
+      assert(Count < 2 && "Switch not supported currently");
+      return (Count == 0) ? EdgeType::TRUE_EDGE : EdgeType::FALSE_EDGE;
+    }
+    Count++;
+  }
+
+  llvm_unreachable("Broken getEdgeTypeBetween");
+}
+
+// Generate all predicates needed for CurrBlock by going through its immediate
+// predecessor blocks.
+void VPlanPredicator::createOrPropagatePredicates(VPBlockBase *CurrBlock,
+                                                  VPRegionBlock *Region) {
+  // Blocks that dominate region exit inherit the predicate from the region.
+  // Return after setting the predicate.
+  if (VPDomTree.dominates(CurrBlock, Region->getExit())) {
+    VPValue *RegionBP = Region->getPredicate();
+    CurrBlock->setPredicate(RegionBP);
+    return;
+  }
+
+  // Collect all incoming predicates in a worklist.
+  std::list<VPValue *> IncomingPredicates;
+
+  // Set the builder's insertion point to the top of the current BB
+  VPBasicBlock *CurrBB = cast<VPBasicBlock>(CurrBlock->getEntryBasicBlock());
+  Builder.setInsertPoint(CurrBB, CurrBB->begin());
+
+  // For each predecessor, generate the VPInstructions required for
+  // computing 'BP AND (not) CBV" at the top of CurrBB.
+  // Collect the outcome of this calculation for all predecessors
+  // into IncomingPredicates.
+  for (VPBlockBase *PredBlock : CurrBlock->getPredecessors()) {
+    // Skip back-edges
+    if (VPBlockUtils::isBackEdge(PredBlock, CurrBlock, VPLI))
+      continue;
+
+    VPValue *IncomingPredicate = nullptr;
+    unsigned NumPredSuccsNoBE =
+        VPBlockUtils::countSuccessorsNoBE(PredBlock, VPLI);
+
+    // If there is an unconditional branch to the currBB, then we don't create
+    // edge predicates. We use the predecessor's block predicate instead.
+    if (NumPredSuccsNoBE == 1)
+      IncomingPredicate = PredBlock->getPredicate();
+    else if (NumPredSuccsNoBE == 2) {
+      // Emit recipes into CurrBlock if required
+      assert(isa<VPBasicBlock>(PredBlock) && "Only BBs have multiple exits");
+      IncomingPredicate =
+          getOrCreateNotPredicate(cast<VPBasicBlock>(PredBlock), CurrBB);
+    } else
+      llvm_unreachable("FIXME: switch statement ?");
+
+    if (IncomingPredicate)
+      IncomingPredicates.push_back(IncomingPredicate);
+  }
+
+  // Logically OR all incoming predicates by building the Predicate Tree.
+  VPValue *Predicate = genPredicateTree(IncomingPredicates);
+
+  // Now update the block's predicate with the new one.
+  CurrBlock->setPredicate(Predicate);
+}
+
+// Generate all predicates needed for Region.
+void VPlanPredicator::predicateRegionRec(VPRegionBlock *Region) {
+  VPBasicBlock *EntryBlock = cast<VPBasicBlock>(Region->getEntry());
+  ReversePostOrderTraversal<VPBlockBase *> RPOT(EntryBlock);
+
+  // Generate edge predicates and append them to the block predicate. RPO is
+  // necessary since the predecessor blocks' block predicate needs to be set
+  // before the current block's block predicate can be computed.
+  for (VPBlockBase *Block : make_range(RPOT.begin(), RPOT.end())) {
+    // TODO: Handle nested regions once we start generating the same.
+    assert(!isa<VPRegionBlock>(Block) && "Nested region not expected");
+    createOrPropagatePredicates(Block, Region);
+  }
+}
+
+// Linearize the CFG within Region.
+// TODO: Predication and linearization need RPOT for every region.
+// This traversal is expensive. Since predication is not adding new
+// blocks, we should be able to compute RPOT once in predication and
+// reuse it here. This becomes even more important once we have nested
+// regions.
+void VPlanPredicator::linearizeRegionRec(VPRegionBlock *Region) {
+  ReversePostOrderTraversal<VPBlockBase *> RPOT(Region->getEntry());
+  VPBlockBase *PrevBlock = nullptr;
+
+  for (VPBlockBase *CurrBlock : make_range(RPOT.begin(), RPOT.end())) {
+    // TODO: Handle nested regions once we start generating the same.
+    assert(!isa<VPRegionBlock>(CurrBlock) && "Nested region not expected");
+
+    // Linearize control flow by adding an unconditional edge between PrevBlock
+    // and CurrBlock skipping loop headers and latches to keep intact loop
+    // header predecessors and loop latch successors.
+    if (PrevBlock && !VPLI->isLoopHeader(CurrBlock) &&
+        !VPBlockUtils::blockIsLoopLatch(PrevBlock, VPLI)) {
+
+      LLVM_DEBUG(dbgs() << "Linearizing: " << PrevBlock->getName() << "->"
+                        << CurrBlock->getName() << "\n");
+
+      PrevBlock->clearSuccessors();
+      CurrBlock->clearPredecessors();
+      VPBlockUtils::connectBlocks(PrevBlock, CurrBlock);
+    }
+
+    PrevBlock = CurrBlock;
+  }
+}
+
+// Entry point. The driver function for the predicator.
+void VPlanPredicator::predicate(void) {
+  // Predicate the blocks within Region.
+  predicateRegionRec(cast<VPRegionBlock>(Plan.getEntry()));
+
+  // Linearlize the blocks with Region.
+  linearizeRegionRec(cast<VPRegionBlock>(Plan.getEntry()));
+}
+
+VPlanPredicator::VPlanPredicator(VPlan &Plan)
+    : Plan(Plan), VPLI(&(Plan.getVPLoopInfo())) {
+  // FIXME: Predicator is currently computing the dominator information for the
+  // top region. Once we start storing dominator information in a VPRegionBlock,
+  // we can avoid this recalculation.
+  VPDomTree.recalculate(*(cast<VPRegionBlock>(Plan.getEntry())));
+}
diff --git a/lib/Transforms/Vectorize/VPlanPredicator.h b/lib/Transforms/Vectorize/VPlanPredicator.h
new file mode 100644
index 0000000000000000000000000000000000000000..692afd2978d5300849fc1ebf3097840319174ad9
--- /dev/null
+++ b/lib/Transforms/Vectorize/VPlanPredicator.h
@@ -0,0 +1,74 @@
+//===-- VPlanPredicator.h ---------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file defines the VPlanPredicator class which contains the public
+/// interfaces to predicate and linearize the VPlan region.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_TRANSFORMS_VECTORIZE_VPLAN_PREDICATOR_H
+#define LLVM_TRANSFORMS_VECTORIZE_VPLAN_PREDICATOR_H
+
+#include "LoopVectorizationPlanner.h"
+#include "VPlan.h"
+#include "VPlanDominatorTree.h"
+
+namespace llvm {
+
+class VPlanPredicator {
+private:
+  enum class EdgeType {
+    TRUE_EDGE,
+    FALSE_EDGE,
+  };
+
+  // VPlan being predicated.
+  VPlan &Plan;
+
+  // VPLoopInfo for Plan's HCFG.
+  VPLoopInfo *VPLI;
+
+  // Dominator tree for Plan's HCFG.
+  VPDominatorTree VPDomTree;
+
+  // VPlan builder used to generate VPInstructions for block predicates.
+  VPBuilder Builder;
+
+  /// Get the type of edge from \p FromBlock to \p ToBlock. Returns TRUE_EDGE if
+  /// \p ToBlock is either the unconditional successor or the conditional true
+  /// successor of \p FromBlock and FALSE_EDGE otherwise.
+  EdgeType getEdgeTypeBetween(VPBlockBase *FromBlock, VPBlockBase *ToBlock);
+
+  /// Create and return VPValue corresponding to the predicate for the edge from
+  /// \p PredBB to \p CurrentBlock.
+  VPValue *getOrCreateNotPredicate(VPBasicBlock *PredBB, VPBasicBlock *CurrBB);
+
+  /// Generate and return the result of ORing all the predicate VPValues in \p
+  /// Worklist.
+  VPValue *genPredicateTree(std::list<VPValue *> &Worklist);
+
+  /// Create or propagate predicate for \p CurrBlock in region \p Region using
+  /// predicate(s) of its predecessor(s)
+  void createOrPropagatePredicates(VPBlockBase *CurrBlock,
+                                   VPRegionBlock *Region);
+
+  /// Predicate the CFG within \p Region.
+  void predicateRegionRec(VPRegionBlock *Region);
+
+  /// Linearize the CFG within \p Region.
+  void linearizeRegionRec(VPRegionBlock *Region);
+
+public:
+  VPlanPredicator(VPlan &Plan);
+
+  /// Predicate Plan's HCFG.
+  void predicate(void);
+};
+} // end namespace llvm
+#endif // LLVM_TRANSFORMS_VECTORIZE_VPLAN_PREDICATOR_H
diff --git a/lib/Transforms/Vectorize/VPlanSLP.cpp b/lib/Transforms/Vectorize/VPlanSLP.cpp
index ad3a85a6f760fca5cc6e8dae88f10fe73018a512..e5ab24e52df6b6447811ebf275a55d9320a5abab 100644
--- a/lib/Transforms/Vectorize/VPlanSLP.cpp
+++ b/lib/Transforms/Vectorize/VPlanSLP.cpp
@@ -1,9 +1,8 @@
 //===- VPlanSLP.cpp - SLP Analysis based on VPlan -------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// This file implements SLP analysis based on VPlan. The analysis is based on
diff --git a/lib/Transforms/Vectorize/VPlanValue.h b/lib/Transforms/Vectorize/VPlanValue.h
index b473579b699fb751b595bc3f0de79a1fd9da5aed..7b6c228c229e3016aa4228c0daed7ed7687bf28e 100644
--- a/lib/Transforms/Vectorize/VPlanValue.h
+++ b/lib/Transforms/Vectorize/VPlanValue.h
@@ -1,9 +1,8 @@
 //===- VPlanValue.h - Represent Values in Vectorizer Plan -----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
diff --git a/lib/Transforms/Vectorize/VPlanVerifier.cpp b/lib/Transforms/Vectorize/VPlanVerifier.cpp
index 054bed4e177f96817f2ea0e5f5569d1ebc87cc35..394b1b93113b5f69474d84a9f2e465838403a154 100644
--- a/lib/Transforms/Vectorize/VPlanVerifier.cpp
+++ b/lib/Transforms/Vectorize/VPlanVerifier.cpp
@@ -1,9 +1,8 @@
 //===-- VPlanVerifier.cpp -------------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
diff --git a/lib/Transforms/Vectorize/VPlanVerifier.h b/lib/Transforms/Vectorize/VPlanVerifier.h
index d2f99d006a66a284011095829fb73160eb09cb30..7d2b26252172392e3de4af9bc680433de7f3560d 100644
--- a/lib/Transforms/Vectorize/VPlanVerifier.h
+++ b/lib/Transforms/Vectorize/VPlanVerifier.h
@@ -1,9 +1,8 @@
 //===-- VPlanVerifier.h -----------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
diff --git a/lib/Transforms/Vectorize/Vectorize.cpp b/lib/Transforms/Vectorize/Vectorize.cpp
index 559ab1968844e4b7af5c9b85a6f1368add3a5ef5..6a4f9169c2af0622cf3ac52a725688e5c4049213 100644
--- a/lib/Transforms/Vectorize/Vectorize.cpp
+++ b/lib/Transforms/Vectorize/Vectorize.cpp
@@ -1,9 +1,8 @@
 //===-- Vectorize.cpp -----------------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/WindowsManifest/LLVMBuild.txt b/lib/WindowsManifest/LLVMBuild.txt
index d5de0275b66b15242272ffc56b8f8275a91ee743..385f20cca5ebea5119872d761d7e54af9ce3a7f0 100644
--- a/lib/WindowsManifest/LLVMBuild.txt
+++ b/lib/WindowsManifest/LLVMBuild.txt
@@ -1,22 +1,21 @@
-;===- ./lib/WindowsManifest/LLVMBuild.txt ----------------------*- Conf -*--===;
-;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
-;
-;===------------------------------------------------------------------------===;
-;
-; This is an LLVMBuild description file for the components in this subdirectory.
-;
-; For more information on the LLVMBuild system, please see:
-;
-;   http://llvm.org/docs/LLVMBuild.html
-;
-;===------------------------------------------------------------------------===;
-
-[component_0]
-type = Library
-name = WindowsManifest
-parent = Libraries
-required_libraries = Support
+;===- ./lib/WindowsManifest/LLVMBuild.txt ----------------------*- Conf -*--===;
+;
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+;
+;===------------------------------------------------------------------------===;
+;
+; This is an LLVMBuild description file for the components in this subdirectory.
+;
+; For more information on the LLVMBuild system, please see:
+;
+;   http://llvm.org/docs/LLVMBuild.html
+;
+;===------------------------------------------------------------------------===;
+
+[component_0]
+type = Library
+name = WindowsManifest
+parent = Libraries
+required_libraries = Support
diff --git a/lib/WindowsManifest/WindowsManifestMerger.cpp b/lib/WindowsManifest/WindowsManifestMerger.cpp
index 0a8abed230d41a9bb819815693167b6f69d890f2..d092ab493c9ba9b24d9a0b237adc85fc555804da 100644
--- a/lib/WindowsManifest/WindowsManifestMerger.cpp
+++ b/lib/WindowsManifest/WindowsManifestMerger.cpp
@@ -1,9 +1,8 @@
 //===-- WindowsManifestMerger.cpp ------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===---------------------------------------------------------------------===//
 //
diff --git a/lib/XRay/BlockIndexer.cpp b/lib/XRay/BlockIndexer.cpp
index 4dbe2d2717ad1cd88a445197870665249a33f164..a99a6815f0d16b7be85d63b9ec46546a3e69193f 100644
--- a/lib/XRay/BlockIndexer.cpp
+++ b/lib/XRay/BlockIndexer.cpp
@@ -1,9 +1,8 @@
 //===- BlockIndexer.cpp - FDR Block Indexing VIsitor ----------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/XRay/BlockPrinter.cpp b/lib/XRay/BlockPrinter.cpp
index 0acebee0cbdd253943be032548bec574c8457808..63a60c3c56a3889cd3337442f6bab818fb245c7f 100644
--- a/lib/XRay/BlockPrinter.cpp
+++ b/lib/XRay/BlockPrinter.cpp
@@ -1,9 +1,8 @@
 //===- BlockPrinter.cpp - FDR Block Pretty Printer Implementation --------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 #include "llvm/XRay/BlockPrinter.h"
diff --git a/lib/XRay/BlockVerifier.cpp b/lib/XRay/BlockVerifier.cpp
index 5e949ec4e46afc3d34d1427335418b2f95a01bf1..9fb49fa9a8609b7b698b488fc59d6b73eb184c02 100644
--- a/lib/XRay/BlockVerifier.cpp
+++ b/lib/XRay/BlockVerifier.cpp
@@ -1,9 +1,8 @@
 //===- BlockVerifier.cpp - FDR Block Verifier -----------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 #include "llvm/XRay/BlockVerifier.h"
diff --git a/lib/XRay/FDRRecordProducer.cpp b/lib/XRay/FDRRecordProducer.cpp
index 25b3ee8af219353a86901ec489a737e87f228417..452bc6c55fb8627d0d0817754b5246d173d76c0d 100644
--- a/lib/XRay/FDRRecordProducer.cpp
+++ b/lib/XRay/FDRRecordProducer.cpp
@@ -1,9 +1,8 @@
 //===- FDRRecordProducer.cpp - XRay FDR Mode Record Producer --------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 #include "llvm/XRay/FDRRecordProducer.h"
diff --git a/lib/XRay/FDRRecords.cpp b/lib/XRay/FDRRecords.cpp
index 2a40d5e0622937d6a42208452a167b6a4f01629b..ff315d35417d6c7fda5c0a5cccf86003180abdb2 100644
--- a/lib/XRay/FDRRecords.cpp
+++ b/lib/XRay/FDRRecords.cpp
@@ -1,9 +1,8 @@
 //===- FDRRecords.cpp -  XRay Flight Data Recorder Mode Records -----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/XRay/FDRTraceExpander.cpp b/lib/XRay/FDRTraceExpander.cpp
index a6e1521da87f340a1e8a82bc85fd8e5de09b0f72..cb7f66bccd7e8e232009dc890ad5a5eefab5ce22 100644
--- a/lib/XRay/FDRTraceExpander.cpp
+++ b/lib/XRay/FDRTraceExpander.cpp
@@ -1,9 +1,8 @@
 //===- FDRTraceExpander.cpp -----------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 #include "llvm/XRay/FDRTraceExpander.h"
diff --git a/lib/XRay/FDRTraceWriter.cpp b/lib/XRay/FDRTraceWriter.cpp
index c5224f4be094d30c7fa551ea08418d884de51457..f50dc19b4be8d91deba67860b6615295f83d9af4 100644
--- a/lib/XRay/FDRTraceWriter.cpp
+++ b/lib/XRay/FDRTraceWriter.cpp
@@ -1,9 +1,8 @@
 //===- FDRTraceWriter.cpp - XRay FDR Trace Writer ---------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/XRay/FileHeaderReader.cpp b/lib/XRay/FileHeaderReader.cpp
index 0b3fb8b6f692db919c4ad96134812949f9cceb0a..3fb021906a6fc93b8cf8b2ad60066d291e7a6542 100644
--- a/lib/XRay/FileHeaderReader.cpp
+++ b/lib/XRay/FileHeaderReader.cpp
@@ -1,9 +1,8 @@
 //===- FileHeaderReader.cpp - XRay File Header Reader  --------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 #include "llvm/XRay/FileHeaderReader.h"
diff --git a/lib/XRay/InstrumentationMap.cpp b/lib/XRay/InstrumentationMap.cpp
index 9f2b179486f07dd1b023947c50d7cb751a41c9d7..862949f578c9f849cd7f6958050e08c1f958a414 100644
--- a/lib/XRay/InstrumentationMap.cpp
+++ b/lib/XRay/InstrumentationMap.cpp
@@ -1,9 +1,8 @@
 //===- InstrumentationMap.cpp - XRay Instrumentation Map ------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/XRay/LLVMBuild.txt b/lib/XRay/LLVMBuild.txt
index 904168dad93974b81279aa2bb3f4d2b5b5c27743..3bf556f0dd2450b210bcac6ac95ca8aad87da01a 100644
--- a/lib/XRay/LLVMBuild.txt
+++ b/lib/XRay/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./lib/XRay/LLVMBuild.txt ---------------------------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/lib/XRay/LogBuilderConsumer.cpp b/lib/XRay/LogBuilderConsumer.cpp
index 88b7d2d728b1eb8f5819dbd316d7a6448d3de9e8..ffb49f9eb4e9832a6484f1b60941ac33f15fbaa9 100644
--- a/lib/XRay/LogBuilderConsumer.cpp
+++ b/lib/XRay/LogBuilderConsumer.cpp
@@ -1,9 +1,8 @@
 //===- FDRRecordConsumer.h - XRay Flight Data Recorder Mode Records -------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 #include "llvm/XRay/FDRRecordConsumer.h"
diff --git a/lib/XRay/Profile.cpp b/lib/XRay/Profile.cpp
index e8a082884d69e84b87999161476a39f9bc63a024..e92eb7e3a7ef408aa0b938c7383084f09c5817b2 100644
--- a/lib/XRay/Profile.cpp
+++ b/lib/XRay/Profile.cpp
@@ -1,9 +1,8 @@
 //===- Profile.cpp - XRay Profile Abstraction -----------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/lib/XRay/RecordInitializer.cpp b/lib/XRay/RecordInitializer.cpp
index f136a1e456b77381c53b765c3ed555aac54d38a9..78163031a8cc81b6ab73703f904c88ccddaf799e 100644
--- a/lib/XRay/RecordInitializer.cpp
+++ b/lib/XRay/RecordInitializer.cpp
@@ -1,9 +1,8 @@
 //===- FDRRecordProducer.cpp - XRay FDR Mode Record Producer --------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 #include "llvm/XRay/FDRRecords.h"
diff --git a/lib/XRay/RecordPrinter.cpp b/lib/XRay/RecordPrinter.cpp
index 71ea7d0e969f0f9ef470a467871beead2b3524b4..32d42104db95cfe67a467c67c0aec5ad2803ed3b 100644
--- a/lib/XRay/RecordPrinter.cpp
+++ b/lib/XRay/RecordPrinter.cpp
@@ -1,9 +1,8 @@
 //===- RecordPrinter.cpp - FDR Record Printer -----------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 #include "llvm/XRay/RecordPrinter.h"
diff --git a/lib/XRay/Trace.cpp b/lib/XRay/Trace.cpp
index 4f28f3f754c1ffaf27522e73d8759def033793c4..eba6ff2c7fb37776d7466513973310d9d71c14c2 100644
--- a/lib/XRay/Trace.cpp
+++ b/lib/XRay/Trace.cpp
@@ -1,9 +1,8 @@
 //===- Trace.cpp - XRay Trace Loading implementation. ---------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/llvm.spec.in b/llvm.spec.in
index d4ef08ae5ad3defe3af354cf27c5e78bfbcae39b..d07864581f103050846bd7e13255a80383be4a3f 100644
--- a/llvm.spec.in
+++ b/llvm.spec.in
@@ -2,7 +2,7 @@ Name: @PACKAGE_NAME@
 Version: @PACKAGE_VERSION@
 Release: 0
 Summary: LLVM (An Optimizing Compiler Infrastructure)
-License: University of Illinois/NCSA Open Source License
+License: Apache-2.0 with LLVM exception
 Vendor: None (open source)
 Group: Development/Compilers
 URL: http://llvm..org/
diff --git a/projects/CMakeLists.txt b/projects/CMakeLists.txt
index 9afc30c8928e3cbd33fda03e6102cae3ea2e2eb7..c98a8824773384a4b289e702131e2bcb0a8abe84 100644
--- a/projects/CMakeLists.txt
+++ b/projects/CMakeLists.txt
@@ -31,6 +31,7 @@ if(${LLVM_BUILD_RUNTIME})
     # dependent projects can see the target names of their dependencies.
     add_llvm_external_project(libunwind)
     add_llvm_external_project(libcxxabi)
+    add_llvm_external_project(pstl)
     add_llvm_external_project(libcxx)
   endif()
   if(NOT LLVM_BUILD_EXTERNAL_COMPILER_RT)
diff --git a/projects/LLVMBuild.txt b/projects/LLVMBuild.txt
index 3c24d1a3bdcaddfaf971dec917ef06e1c67bd0fa..17ffc8a6cc6e038628f0b2c04ba1d9059f9f182a 100644
--- a/projects/LLVMBuild.txt
+++ b/projects/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./projects/LLVMBuild.txt ---------------------------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/runtimes/CMakeLists.txt b/runtimes/CMakeLists.txt
index 42842409dca50a782bd1ed0b7cdb3576c9944936..113afb0cf0dbf55857cccb52285e87611f7530bd 100644
--- a/runtimes/CMakeLists.txt
+++ b/runtimes/CMakeLists.txt
@@ -87,28 +87,21 @@ if(${CMAKE_SOURCE_DIR} STREQUAL ${CMAKE_CURRENT_SOURCE_DIR})
 
   # This variable makes sure that e.g. llvm-lit is found.
   set(LLVM_MAIN_SRC_DIR ${LLVM_BUILD_MAIN_SRC_DIR})
+  set(LLVM_CMAKE_PATH ${LLVM_MAIN_SRC_DIR}/cmake/modules)
 
   if(APPLE)
     set(LLVM_ENABLE_LIBCXX ON CACHE BOOL "")
   endif()
 
-  set(SAFE_CMAKE_REQUIRED_FLAGS ${CMAKE_REQUIRED_FLAGS})
-  set(SAFE_CMAKE_REQUIRED_LIBRARIES ${CMAKE_REQUIRED_LIBRARIES})
-
   include(CheckLibraryExists)
   include(CheckCCompilerFlag)
 
-  check_library_exists(c fopen "" LLVM_HAS_C_LIB)
-  check_c_compiler_flag(-nodefaultlibs LLVM_HAS_NODEFAULTLIBS_FLAG)
-  if(LLVM_HAS_NODEFAULTLIBS_FLAG)
-    set(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} -nodefaultlibs")
-    if(LLVM_HAS_C_LIB)
-      list(APPEND CMAKE_REQUIRED_LIBRARIES c)
-    endif()
-  endif()
+  # We don't have libc++ (yet).
+  set(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} -nostdlib++")
 
   # Avoid checking whether the compiler is working.
   set(LLVM_COMPILER_CHECKED ON)
+
   # Enable warnings, otherwise -w gets added to the cflags by HandleLLVMOptions
   # resulting in unjustified successes by check_cxx_compiler_flag.
   set(LLVM_ENABLE_WARNINGS ON)
@@ -116,9 +109,13 @@ if(${CMAKE_SOURCE_DIR} STREQUAL ${CMAKE_CURRENT_SOURCE_DIR})
   # Handle common options used by all runtimes.
   include(AddLLVM)
   include(HandleLLVMOptions)
+  include(FindPythonInterp)
+
+  # Remove the -nostdlib++ option we've added earlier.
+  string(REPLACE "-nostdlib++" "" CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS}")
 
-  set(CMAKE_REQUIRED_FLAGS ${SAFE_CMAKE_REQUIRED_FLAGS})
-  set(CMAKE_REQUIRED_LIBRARIES ${SAFE_CMAKE_REQUIRED_LIBRARIES})
+  # This can be used to detect whether we're in the runtimes build.
+  set(RUNTIMES_BUILD ON)
 
   foreach(entry ${runtimes})
     get_filename_component(projName ${entry} NAME)
@@ -126,8 +123,6 @@ if(${CMAKE_SOURCE_DIR} STREQUAL ${CMAKE_CURRENT_SOURCE_DIR})
     # TODO: Clean this up as part of an interface standardization
     string(REPLACE "-" "_" canon_name ${projName})
     string(TOUPPER ${canon_name} canon_name)
-    # The subdirectories need to treat this as standalone builds
-    set(${canon_name}_STANDALONE_BUILD On)
 
     if(LLVM_RUNTIMES_LIBDIR_SUFFIX)
       set(${canon_name}_LIBDIR_SUFFIX "${LLVM_RUNTIMES_LIBDIR_SUFFIX}" CACHE STRING "" FORCE)
@@ -135,7 +130,7 @@ if(${CMAKE_SOURCE_DIR} STREQUAL ${CMAKE_CURRENT_SOURCE_DIR})
 
     # Setting a variable to let sub-projects detect which other projects
     # will be included under here.
-    set(HAVE_${canon_name} On)
+    set(HAVE_${canon_name} ON)
   endforeach()
 
   # We do this in two loops so that HAVE_* is set for each runtime before the
diff --git a/test/Analysis/BasicAA/128-bit-ptr.ll b/test/Analysis/BasicAA/128-bit-ptr.ll
new file mode 100644
index 0000000000000000000000000000000000000000..59fc99d3d3779557d3db5c5661ccd58af94efd3b
--- /dev/null
+++ b/test/Analysis/BasicAA/128-bit-ptr.ll
@@ -0,0 +1,60 @@
+; This testcase consists of alias relations on 128-bit pointers that
+; should be completely resolvable by basicaa.
+
+; RUN: opt < %s -basicaa -aa-eval -print-no-aliases -print-may-aliases -print-must-aliases -disable-output 2>&1 | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-i128:128:128-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128-p100:128:64:64-p101:128:64:64"
+
+
+; test0 is similar to SimpleCases.ll
+
+%T = type { i32, [10 x i8] }
+
+; CHECK:     Function: test0
+; CHECK-NOT:   MayAlias:
+define void @test0(%T addrspace(100)* %P) {
+  %A = getelementptr %T, %T addrspace(100)* %P, i64 0
+  %B = getelementptr %T, %T addrspace(100)* %P, i64 0, i32 0
+  %C = getelementptr %T, %T addrspace(100)* %P, i64 0, i32 1
+  %D = getelementptr %T, %T addrspace(100)* %P, i64 0, i32 1, i64 0
+  %E = getelementptr %T, %T addrspace(100)* %P, i64 0, i32 1, i64 5
+  ret void
+}
+
+; test1 checks that >64 bits of index can be considered.
+; If BasicAA is truncating the arithmetic, it will conclude
+; that %A and %B must alias when in fact they must not.
+
+; CHECK:     Function: test1
+; CHECK-NOT:   MustAlias:
+; CHECK:       NoAlias:
+; CHECK-SAME:  %A
+; CHECK-SAME:  %B
+define void @test1(double addrspace(100)* %P, i128 %i) {
+  ; 1180591620717411303424 is 2**70
+  ;  590295810358705651712 is 2**69
+  %i70 = add i128 %i, 1180591620717411303424 
+  %i69 = add i128 %i, 590295810358705651712
+  %A = getelementptr double, double addrspace(100)* %P, i128 %i70
+  %B = getelementptr double, double addrspace(100)* %P, i128 %i69
+  ret void
+}
+
+; test2 checks that >64 bits of index can be considered
+; and computes the same address in two ways to ensure that
+; they are considered equivalent.
+
+; CHECK: Function: test2
+; CHECK: MustAlias:
+; CHECK-SAME: %A
+; CHECK-SAME: %C
+define void @test2(double addrspace(100)* %P, i128 %i) {
+  ; 1180591620717411303424 is 2**70
+  ;  590295810358705651712 is 2**69
+  %i70 = add i128 %i, 1180591620717411303424 
+  %i69 = add i128 %i, 590295810358705651712
+  %j70 = add i128 %i69, 590295810358705651712 
+  %A = getelementptr double, double addrspace(100)* %P, i128 %i70
+  %C = getelementptr double, double addrspace(100)* %P, i128 %j70
+  ret void
+}
diff --git a/test/Analysis/BasicAA/gep-and-alias-64.ll b/test/Analysis/BasicAA/gep-and-alias-64.ll
new file mode 100644
index 0000000000000000000000000000000000000000..c1a53540a6de763301d964abe9fc09c840b5cc37
--- /dev/null
+++ b/test/Analysis/BasicAA/gep-and-alias-64.ll
@@ -0,0 +1,43 @@
+; RUN: opt -S -basicaa -gvn < %s | FileCheck %s
+
+target datalayout = "e-m:o-p:64:64-f64:32:64-f80:128-n8:16:32-S128"
+target triple = "x86_64-apple-macosx10.6.0"
+
+; The load and store address in the loop body could alias so the load
+; can't be hoisted above the store and out of the loop.
+
+declare void @llvm.memset.p0i8.i64(i8* nocapture writeonly, i8, i64, i32, i1)
+
+define i64 @foo(i64 %x, i64 %z, i64 %n) {
+entry:
+  %pool = alloca [59 x i64], align 4
+  %tmp = bitcast [59 x i64]* %pool to i8*
+  call void @llvm.memset.p0i8.i64(i8* nonnull %tmp, i8 0, i64 236, i32 4, i1 false)
+  %cmp3 = icmp eq i64 %n, 0
+  br i1 %cmp3, label %for.end, label %for.body.lr.ph
+
+for.body.lr.ph:                                   ; preds = %entry
+  %add = add i64 %z, %x
+  %and = and i64 %add, 9223372036854775807
+  %sub = add nsw i64 %and, -9223372036844814062
+  %arrayidx = getelementptr inbounds [59 x i64], [59 x i64]* %pool, i64 0, i64 %sub
+  %arrayidx1 = getelementptr inbounds [59 x i64], [59 x i64]* %pool, i64 0, i64 42
+  br label %for.body
+
+for.body:                                         ; preds = %for.body.lr.ph, %for.body
+  %i.04 = phi i64 [ 0, %for.body.lr.ph ], [ %inc, %for.body ]
+  store i64 %i.04, i64* %arrayidx, align 4
+  %tmp1 = load i64, i64* %arrayidx1, align 4
+  %inc = add nuw i64 %i.04, 1
+  %exitcond = icmp ne i64 %inc, %n
+  br i1 %exitcond, label %for.body, label %for.end.loopexit
+
+for.end.loopexit:                                 ; preds = %for.body
+  %lcssa = phi i64 [ %tmp1, %for.body ]
+  br label %for.end
+
+for.end:                                          ; preds = %for.end.loopexit, %entry
+  %s = phi i64 [ 0, %entry ], [ %lcssa, %for.end.loopexit ]
+; CHECK: ret i64 %s
+  ret i64 %s
+}
diff --git a/test/Analysis/BasicAA/gep-and-alias.ll b/test/Analysis/BasicAA/gep-and-alias.ll
index e2e5811d26395854a5d22a4e8c2f01fdafd8a74b..0fa17b68a9d1bd1c83fc51cd7e279504b783bdf8 100644
--- a/test/Analysis/BasicAA/gep-and-alias.ll
+++ b/test/Analysis/BasicAA/gep-and-alias.ll
@@ -1,4 +1,5 @@
 ; RUN: opt -S -basicaa -gvn < %s | FileCheck %s
+; RUN: opt -S -basicaa -gvn -basicaa-force-at-least-64b=0 < %s | FileCheck %s
 
 target datalayout = "e-m:o-p:32:32-f64:32:64-f80:128-n8:16:32-S128"
 target triple = "i386-apple-macosx10.6.0"
diff --git a/test/Analysis/ConstantFolding/bitcount.ll b/test/Analysis/ConstantFolding/bitcount.ll
new file mode 100644
index 0000000000000000000000000000000000000000..6333c4d24d094dcda47ca4415de9c4e8b25c1fe6
--- /dev/null
+++ b/test/Analysis/ConstantFolding/bitcount.ll
@@ -0,0 +1,177 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -constprop -S | FileCheck %s
+
+declare i31 @llvm.ctpop.i31(i31 %val)
+declare i32 @llvm.cttz.i32(i32 %val, i1)
+declare i33 @llvm.ctlz.i33(i33 %val, i1)
+declare <2 x i31> @llvm.ctpop.v2i31(<2 x i31> %val)
+declare <2 x i32> @llvm.cttz.v2i32(<2 x i32> %val, i1)
+declare <2 x i33> @llvm.ctlz.v2i33(<2 x i33> %val, i1)
+
+define i31 @ctpop_const() {
+; CHECK-LABEL: @ctpop_const(
+; CHECK-NEXT:    ret i31 12
+;
+  %x = call i31 @llvm.ctpop.i31(i31 12415124)
+  ret i31 %x
+}
+
+define i32 @cttz_const() {
+; CHECK-LABEL: @cttz_const(
+; CHECK-NEXT:    ret i32 1
+;
+  %x = call i32 @llvm.cttz.i32(i32 87359874, i1 true)
+  ret i32 %x
+}
+
+define i33 @ctlz_const() {
+; CHECK-LABEL: @ctlz_const(
+; CHECK-NEXT:    ret i33 6
+;
+  %x = call i33 @llvm.ctlz.i33(i33 87359874, i1 true)
+  ret i33 %x
+}
+
+define i31 @ctpop_zero() {
+; CHECK-LABEL: @ctpop_zero(
+; CHECK-NEXT:    ret i31 0
+;
+  %x = call i31 @llvm.ctpop.i31(i31 0)
+  ret i31 %x
+}
+
+define i32 @cttz_zero_defined() {
+; CHECK-LABEL: @cttz_zero_defined(
+; CHECK-NEXT:    ret i32 32
+;
+  %x = call i32 @llvm.cttz.i32(i32 0, i1 false)
+  ret i32 %x
+}
+
+define i32 @cttz_zero_undefined() {
+; CHECK-LABEL: @cttz_zero_undefined(
+; CHECK-NEXT:    ret i32 undef
+;
+  %x = call i32 @llvm.cttz.i32(i32 0, i1 true)
+  ret i32 %x
+}
+
+define i33 @ctlz_zero_defined() {
+; CHECK-LABEL: @ctlz_zero_defined(
+; CHECK-NEXT:    ret i33 33
+;
+  %x = call i33 @llvm.ctlz.i33(i33 0, i1 false)
+  ret i33 %x
+}
+
+define i33 @ctlz_zero_undefined() {
+; CHECK-LABEL: @ctlz_zero_undefined(
+; CHECK-NEXT:    ret i33 undef
+;
+  %x = call i33 @llvm.ctlz.i33(i33 0, i1 true)
+  ret i33 %x
+}
+
+define i31 @ctpop_undef() {
+; CHECK-LABEL: @ctpop_undef(
+; CHECK-NEXT:    ret i31 0
+;
+  %x = call i31 @llvm.ctpop.i31(i31 undef)
+  ret i31 %x
+}
+
+define i32 @cttz_undef_defined() {
+; CHECK-LABEL: @cttz_undef_defined(
+; CHECK-NEXT:    ret i32 0
+;
+  %x = call i32 @llvm.cttz.i32(i32 undef, i1 false)
+  ret i32 %x
+}
+
+define i32 @cttz_undef_undefined() {
+; CHECK-LABEL: @cttz_undef_undefined(
+; CHECK-NEXT:    ret i32 undef
+;
+  %x = call i32 @llvm.cttz.i32(i32 undef, i1 true)
+  ret i32 %x
+}
+
+define i33 @ctlz_undef_defined() {
+; CHECK-LABEL: @ctlz_undef_defined(
+; CHECK-NEXT:    ret i33 0
+;
+  %x = call i33 @llvm.ctlz.i33(i33 undef, i1 false)
+  ret i33 %x
+}
+
+define i33 @ctlz_undef_undefined() {
+; CHECK-LABEL: @ctlz_undef_undefined(
+; CHECK-NEXT:    ret i33 undef
+;
+  %x = call i33 @llvm.ctlz.i33(i33 undef, i1 true)
+  ret i33 %x
+}
+
+define <2 x i31> @ctpop_vector() {
+; CHECK-LABEL: @ctpop_vector(
+; CHECK-NEXT:    ret <2 x i31> <i31 8, i31 1>
+;
+  %x = call <2 x i31> @llvm.ctpop.v2i31(<2 x i31> <i31 255, i31 16>)
+  ret <2 x i31> %x
+}
+
+define <2 x i31> @ctpop_vector_undef() {
+; CHECK-LABEL: @ctpop_vector_undef(
+; CHECK-NEXT:    ret <2 x i31> zeroinitializer
+;
+  %x = call <2 x i31> @llvm.ctpop.v2i31(<2 x i31> <i31 0, i31 undef>)
+  ret <2 x i31> %x
+}
+
+define <2 x i32> @cttz_vector() {
+; CHECK-LABEL: @cttz_vector(
+; CHECK-NEXT:    ret <2 x i32> <i32 0, i32 4>
+;
+  %x = call <2 x i32> @llvm.cttz.v2i32(<2 x i32> <i32 255, i32 16>, i1 true)
+  ret <2 x i32> %x
+}
+
+define <2 x i32> @cttz_vector_undef_defined() {
+; CHECK-LABEL: @cttz_vector_undef_defined(
+; CHECK-NEXT:    ret <2 x i32> <i32 32, i32 0>
+;
+  %x = call <2 x i32> @llvm.cttz.v2i32(<2 x i32> <i32 0, i32 undef>, i1 false)
+  ret <2 x i32> %x
+}
+
+define <2 x i32> @cttz_vector_undef_undefined() {
+; CHECK-LABEL: @cttz_vector_undef_undefined(
+; CHECK-NEXT:    ret <2 x i32> undef
+;
+  %x = call <2 x i32> @llvm.cttz.v2i32(<2 x i32> <i32 0, i32 undef>, i1 true)
+  ret <2 x i32> %x
+}
+
+define <2 x i33> @ctlz_vector() {
+; CHECK-LABEL: @ctlz_vector(
+; CHECK-NEXT:    ret <2 x i33> <i33 25, i33 28>
+;
+  %x = call <2 x i33> @llvm.ctlz.v2i33(<2 x i33> <i33 255, i33 16>, i1 true)
+  ret <2 x i33> %x
+}
+
+define <2 x i33> @ctlz_vector_undef_defined() {
+; CHECK-LABEL: @ctlz_vector_undef_defined(
+; CHECK-NEXT:    ret <2 x i33> <i33 33, i33 0>
+;
+  %x = call <2 x i33> @llvm.ctlz.v2i33(<2 x i33> <i33 0, i33 undef>, i1 false)
+  ret <2 x i33> %x
+}
+
+define <2 x i33> @ctlz_vector_undef_undefined() {
+; CHECK-LABEL: @ctlz_vector_undef_undefined(
+; CHECK-NEXT:    ret <2 x i33> undef
+;
+  %x = call <2 x i33> @llvm.ctlz.v2i33(<2 x i33> <i33 0, i33 undef>, i1 true)
+  ret <2 x i33> %x
+}
diff --git a/test/Analysis/ConstantFolding/func-and-folding.ll b/test/Analysis/ConstantFolding/func-and-folding.ll
deleted file mode 100644
index 2dbbe673601d574b9f01b2768cde9f3d5dbe71c0..0000000000000000000000000000000000000000
--- a/test/Analysis/ConstantFolding/func-and-folding.ll
+++ /dev/null
@@ -1,27 +0,0 @@
-; RUN: opt < %s -constprop -S -o - | FileCheck %s
-
-; Function Attrs: minsize norecurse nounwind optsize readnone
-define void @foo1() #0 {
-entry:
-  ret void
-}
-
-; Function Attrs: minsize norecurse nounwind optsize readnone
-define void @foo2() align 4 {
-entry:
-  ret void
-}
-
-; Function Attrs: minsize nounwind optsize
-define i32 @main() local_unnamed_addr #1 {
-entry:
-; CHECK: ptrtoint
-  %call = tail call i32 bitcast (i32 (...)* @process to i32 (i32)*)(i32 and (i32 ptrtoint (void ()* @foo1 to i32), i32 2)) #3
-; CHECK-NEXT: ptrtoint
-  %call2 = tail call i32 bitcast (i32 (...)* @process to i32 (i32)*)(i32 and (i32 ptrtoint (void ()* @foo2 to i32), i32 2)) #3
-  ret i32 0
-}
-
-; Function Attrs: minsize optsize
-declare i32 @process(...) local_unnamed_addr #2
-
diff --git a/test/Analysis/ConstantFolding/funnel-shift.ll b/test/Analysis/ConstantFolding/funnel-shift.ll
index 8ccc714ae5398d9c09f595e6730378990409156e..fae29c939216801ec2f59ca711d0126a4b773f07 100644
--- a/test/Analysis/ConstantFolding/funnel-shift.ll
+++ b/test/Analysis/ConstantFolding/funnel-shift.ll
@@ -81,3 +81,152 @@ define <4 x i8> @fshr_v4i8() {
   ret <4 x i8> %f
 }
 
+; Undef handling
+
+define i32 @fshl_scalar_all_undef() {
+; CHECK-LABEL: @fshl_scalar_all_undef(
+; CHECK-NEXT:    ret i32 undef
+;
+  %f = call i32 @llvm.fshl.i32(i32 undef, i32 undef, i32 undef)
+  ret i32 %f
+}
+
+define i32 @fshr_scalar_all_undef() {
+; CHECK-LABEL: @fshr_scalar_all_undef(
+; CHECK-NEXT:    ret i32 undef
+;
+  %f = call i32 @llvm.fshr.i32(i32 undef, i32 undef, i32 undef)
+  ret i32 %f
+}
+
+define i32 @fshl_scalar_undef_shamt() {
+; CHECK-LABEL: @fshl_scalar_undef_shamt(
+; CHECK-NEXT:    ret i32 1
+;
+  %f = call i32 @llvm.fshl.i32(i32 1, i32 2, i32 undef)
+  ret i32 %f
+}
+
+define i32 @fshr_scalar_undef_shamt() {
+; CHECK-LABEL: @fshr_scalar_undef_shamt(
+; CHECK-NEXT:    ret i32 2
+;
+  %f = call i32 @llvm.fshr.i32(i32 1, i32 2, i32 undef)
+  ret i32 %f
+}
+
+define i32 @fshl_scalar_undef_ops() {
+; CHECK-LABEL: @fshl_scalar_undef_ops(
+; CHECK-NEXT:    ret i32 undef
+;
+  %f = call i32 @llvm.fshl.i32(i32 undef, i32 undef, i32 7)
+  ret i32 %f
+}
+
+define i32 @fshr_scalar_undef_ops() {
+; CHECK-LABEL: @fshr_scalar_undef_ops(
+; CHECK-NEXT:    ret i32 undef
+;
+  %f = call i32 @llvm.fshr.i32(i32 undef, i32 undef, i32 7)
+  ret i32 %f
+}
+
+define i32 @fshl_scalar_undef_op1_zero_shift() {
+; CHECK-LABEL: @fshl_scalar_undef_op1_zero_shift(
+; CHECK-NEXT:    ret i32 undef
+;
+  %f = call i32 @llvm.fshl.i32(i32 undef, i32 1, i32 0)
+  ret i32 %f
+}
+
+define i32 @fshl_scalar_undef_op2_zero_shift() {
+; CHECK-LABEL: @fshl_scalar_undef_op2_zero_shift(
+; CHECK-NEXT:    ret i32 1
+;
+  %f = call i32 @llvm.fshl.i32(i32 1, i32 undef, i32 32)
+  ret i32 %f
+}
+
+define i32 @fshr_scalar_undef_op1_zero_shift() {
+; CHECK-LABEL: @fshr_scalar_undef_op1_zero_shift(
+; CHECK-NEXT:    ret i32 1
+;
+  %f = call i32 @llvm.fshr.i32(i32 undef, i32 1, i32 64)
+  ret i32 %f
+}
+
+define i32 @fshr_scalar_undef_op2_zero_shift() {
+; CHECK-LABEL: @fshr_scalar_undef_op2_zero_shift(
+; CHECK-NEXT:    ret i32 undef
+;
+  %f = call i32 @llvm.fshr.i32(i32 1, i32 undef, i32 0)
+  ret i32 %f
+}
+
+define i32 @fshl_scalar_undef_op1_nonzero_shift() {
+; CHECK-LABEL: @fshl_scalar_undef_op1_nonzero_shift(
+; CHECK-NEXT:    ret i32 255
+;
+  %f = call i32 @llvm.fshl.i32(i32 undef, i32 -1, i32 8)
+  ret i32 %f
+}
+
+define i32 @fshl_scalar_undef_op2_nonzero_shift() {
+; CHECK-LABEL: @fshl_scalar_undef_op2_nonzero_shift(
+; CHECK-NEXT:    ret i32 -256
+;
+  %f = call i32 @llvm.fshl.i32(i32 -1, i32 undef, i32 8)
+  ret i32 %f
+}
+
+define i32 @fshr_scalar_undef_op1_nonzero_shift() {
+; CHECK-LABEL: @fshr_scalar_undef_op1_nonzero_shift(
+; CHECK-NEXT:    ret i32 16777215
+;
+  %f = call i32 @llvm.fshr.i32(i32 undef, i32 -1, i32 8)
+  ret i32 %f
+}
+
+define i32 @fshr_scalar_undef_op2_nonzero_shift() {
+; CHECK-LABEL: @fshr_scalar_undef_op2_nonzero_shift(
+; CHECK-NEXT:    ret i32 -16777216
+;
+  %f = call i32 @llvm.fshr.i32(i32 -1, i32 undef, i32 8)
+  ret i32 %f
+}
+
+; Undef/Undef/Undef; 1/2/Undef; Undef/Undef/3; Undef/1/0
+define <4 x i8> @fshl_vector_mix1() {
+; CHECK-LABEL: @fshl_vector_mix1(
+; CHECK-NEXT:    ret <4 x i8> <i8 undef, i8 1, i8 undef, i8 undef>
+;
+  %f = call <4 x i8> @llvm.fshl.v4i8(<4 x i8> <i8 undef, i8 1, i8 undef, i8 undef>, <4 x i8> <i8 undef, i8 2, i8 undef, i8 1>, <4 x i8> <i8 undef, i8 undef, i8 3, i8 0>)
+  ret <4 x i8> %f
+}
+
+; 1/Undef/8; Undef/-1/2; -1/Undef/2; 7/8/4
+define <4 x i8> @fshl_vector_mix2() {
+; CHECK-LABEL: @fshl_vector_mix2(
+; CHECK-NEXT:    ret <4 x i8> <i8 1, i8 3, i8 -4, i8 112>
+;
+  %f = call <4 x i8> @llvm.fshl.v4i8(<4 x i8> <i8 1, i8 undef, i8 -1, i8 7>, <4 x i8> <i8 undef, i8 -1, i8 undef, i8 8>, <4 x i8> <i8 8, i8 2, i8 2, i8 4>)
+  ret <4 x i8> %f
+}
+
+; Undef/Undef/Undef; 1/2/Undef; Undef/Undef/3; Undef/1/0
+define <4 x i8> @fshr_vector_mix1() {
+; CHECK-LABEL: @fshr_vector_mix1(
+; CHECK-NEXT:    ret <4 x i8> <i8 undef, i8 2, i8 undef, i8 1>
+;
+  %f = call <4 x i8> @llvm.fshr.v4i8(<4 x i8> <i8 undef, i8 1, i8 undef, i8 undef>, <4 x i8> <i8 undef, i8 2, i8 undef, i8 1>, <4 x i8> <i8 undef, i8 undef, i8 3, i8 0>)
+  ret <4 x i8> %f
+}
+
+; 1/Undef/8; Undef/-1/2; -1/Undef/2; 7/8/4
+define <4 x i8> @fshr_vector_mix2() {
+; CHECK-LABEL: @fshr_vector_mix2(
+; CHECK-NEXT:    ret <4 x i8> <i8 undef, i8 63, i8 -64, i8 112>
+;
+  %f = call <4 x i8> @llvm.fshr.v4i8(<4 x i8> <i8 1, i8 undef, i8 -1, i8 7>, <4 x i8> <i8 undef, i8 -1, i8 undef, i8 8>, <4 x i8> <i8 8, i8 2, i8 2, i8 4>)
+  ret <4 x i8> %f
+}
diff --git a/test/Analysis/ConstantFolding/saturating-add-sub.ll b/test/Analysis/ConstantFolding/saturating-add-sub.ll
index 14c6a9fabfade7ad69911da348bfa6ca81221252..7c6fdbf1e1fd152543e369172e56cd466e3c8131 100644
--- a/test/Analysis/ConstantFolding/saturating-add-sub.ll
+++ b/test/Analysis/ConstantFolding/saturating-add-sub.ll
@@ -1,9 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -constprop -S | FileCheck %s
 
-declare void @dummy(i8)
-declare void @dummy_vec(<2 x i8>)
-
 declare i8 @llvm.uadd.sat.i8(i8, i8)
 declare i8 @llvm.sadd.sat.i8(i8, i8)
 declare <2 x i8> @llvm.uadd.sat.v2i8(<2 x i8>, <2 x i8>)
@@ -14,98 +11,356 @@ declare i8 @llvm.ssub.sat.i8(i8, i8)
 declare <2 x i8> @llvm.usub.sat.v2i8(<2 x i8>, <2 x i8>)
 declare <2 x i8> @llvm.ssub.sat.v2i8(<2 x i8>, <2 x i8>)
 
-define void @test_add_scalar() {
-; CHECK-LABEL: @test_add_scalar(
-; CHECK-NEXT:    call void @dummy(i8 30)
-; CHECK-NEXT:    call void @dummy(i8 -1)
-; CHECK-NEXT:    call void @dummy(i8 -10)
-; CHECK-NEXT:    call void @dummy(i8 127)
-; CHECK-NEXT:    call void @dummy(i8 -128)
-; CHECK-NEXT:    ret void
-;
-  %x1 = call i8 @llvm.uadd.sat.i8(i8 10, i8 20)
-  call void @dummy(i8 %x1)
-  %x2 = call i8 @llvm.uadd.sat.i8(i8 250, i8 100)
-  call void @dummy(i8 %x2)
-
-  %y1 = call i8 @llvm.sadd.sat.i8(i8 10, i8 -20)
-  call void @dummy(i8 %y1)
-  %y2 = call i8 @llvm.sadd.sat.i8(i8 120, i8 10)
-  call void @dummy(i8 %y2)
-  %y3 = call i8 @llvm.sadd.sat.i8(i8 -120, i8 -10)
-  call void @dummy(i8 %y3)
-
-  ret void
-}
-
-define void @test_add_vector(<2 x i8> %a) {
-; CHECK-LABEL: @test_add_vector(
-; CHECK-NEXT:    call void @dummy_vec(<2 x i8> <i8 20, i8 30>)
-; CHECK-NEXT:    call void @dummy_vec(<2 x i8> <i8 -1, i8 -1>)
-; CHECK-NEXT:    call void @dummy_vec(<2 x i8> <i8 -10, i8 -30>)
-; CHECK-NEXT:    call void @dummy_vec(<2 x i8> <i8 127, i8 127>)
-; CHECK-NEXT:    call void @dummy_vec(<2 x i8> <i8 -128, i8 -128>)
-; CHECK-NEXT:    ret void
-;
-  %x1 = call <2 x i8> @llvm.uadd.sat.v2i8(<2 x i8> <i8 10, i8 15>, <2 x i8> <i8 10, i8 15>)
-  call void @dummy_vec(<2 x i8> %x1)
-  %x2 = call <2 x i8> @llvm.uadd.sat.v2i8(<2 x i8> <i8 100, i8 200>, <2 x i8> <i8 250, i8 100>)
-  call void @dummy_vec(<2 x i8> %x2)
-
-  %y1 = call <2 x i8> @llvm.sadd.sat.v2i8(<2 x i8> <i8 10, i8 -15>, <2 x i8> <i8 -20, i8 -15>)
-  call void @dummy_vec(<2 x i8> %y1)
-  %y2 = call <2 x i8> @llvm.sadd.sat.v2i8(<2 x i8> <i8 100, i8 10>, <2 x i8> <i8 30, i8 120>)
-  call void @dummy_vec(<2 x i8> %y2)
-  %y3 = call <2 x i8> @llvm.sadd.sat.v2i8(<2 x i8> <i8 -100, i8 -10>, <2 x i8> <i8 -30, i8 -120>)
-  call void @dummy_vec(<2 x i8> %y3)
-
-  ret void
-}
-
-define void @test_usub_ssub_scalar() {
-; CHECK-LABEL: @test_usub_ssub_scalar(
-; CHECK-NEXT:    call void @dummy(i8 10)
-; CHECK-NEXT:    call void @dummy(i8 0)
-; CHECK-NEXT:    call void @dummy(i8 -30)
-; CHECK-NEXT:    call void @dummy(i8 127)
-; CHECK-NEXT:    call void @dummy(i8 -128)
-; CHECK-NEXT:    ret void
-;
-  %x1 = call i8 @llvm.usub.sat.i8(i8 20, i8 10)
-  call void @dummy(i8 %x1)
-  %x2 = call i8 @llvm.usub.sat.i8(i8 200, i8 250)
-  call void @dummy(i8 %x2)
-
-  %y1 = call i8 @llvm.ssub.sat.i8(i8 -10, i8 20)
-  call void @dummy(i8 %y1)
-  %y2 = call i8 @llvm.ssub.sat.i8(i8 120, i8 -10)
-  call void @dummy(i8 %y2)
-  %y3 = call i8 @llvm.ssub.sat.i8(i8 -120, i8 10)
-  call void @dummy(i8 %y3)
-
-  ret void
-}
-
-define void @test_sub_vector(<2 x i8> %a) {
-; CHECK-LABEL: @test_sub_vector(
-; CHECK-NEXT:    call void @dummy_vec(<2 x i8> <i8 10, i8 5>)
-; CHECK-NEXT:    call void @dummy_vec(<2 x i8> zeroinitializer)
-; CHECK-NEXT:    call void @dummy_vec(<2 x i8> <i8 30, i8 0>)
-; CHECK-NEXT:    call void @dummy_vec(<2 x i8> <i8 127, i8 127>)
-; CHECK-NEXT:    call void @dummy_vec(<2 x i8> <i8 -128, i8 -128>)
-; CHECK-NEXT:    ret void
-;
-  %x1 = call <2 x i8> @llvm.usub.sat.v2i8(<2 x i8> <i8 20, i8 15>, <2 x i8> <i8 10, i8 10>)
-  call void @dummy_vec(<2 x i8> %x1)
-  %x2 = call <2 x i8> @llvm.usub.sat.v2i8(<2 x i8> <i8 100, i8 200>, <2 x i8> <i8 150, i8 250>)
-  call void @dummy_vec(<2 x i8> %x2)
-
-  %y1 = call <2 x i8> @llvm.ssub.sat.v2i8(<2 x i8> <i8 10, i8 -15>, <2 x i8> <i8 -20, i8 -15>)
-  call void @dummy_vec(<2 x i8> %y1)
-  %y2 = call <2 x i8> @llvm.ssub.sat.v2i8(<2 x i8> <i8 100, i8 10>, <2 x i8> <i8 -30, i8 -120>)
-  call void @dummy_vec(<2 x i8> %y2)
-  %y3 = call <2 x i8> @llvm.ssub.sat.v2i8(<2 x i8> <i8 -100, i8 -10>, <2 x i8> <i8 30, i8 120>)
-  call void @dummy_vec(<2 x i8> %y3)
-
-  ret void
+define i8 @test_uadd_scalar_no_sat() {
+; CHECK-LABEL: @test_uadd_scalar_no_sat(
+; CHECK-NEXT:    ret i8 30
+;
+  %x = call i8 @llvm.uadd.sat.i8(i8 10, i8 20)
+  ret i8 %x
+}
+
+define i8 @test_uadd_scalar_sat() {
+; CHECK-LABEL: @test_uadd_scalar_sat(
+; CHECK-NEXT:    ret i8 -1
+;
+  %x = call i8 @llvm.uadd.sat.i8(i8 250, i8 100)
+  ret i8 %x
+}
+
+define i8 @test_sadd_scalar_no_sat() {
+; CHECK-LABEL: @test_sadd_scalar_no_sat(
+; CHECK-NEXT:    ret i8 -10
+;
+  %x = call i8 @llvm.sadd.sat.i8(i8 10, i8 -20)
+  ret i8 %x
+}
+
+define i8 @test_sadd_scalar_sat_pos() {
+; CHECK-LABEL: @test_sadd_scalar_sat_pos(
+; CHECK-NEXT:    ret i8 127
+;
+  %x = call i8 @llvm.sadd.sat.i8(i8 120, i8 10)
+  ret i8 %x
+}
+
+define i8 @test_sadd_scalar_sat_neg() {
+; CHECK-LABEL: @test_sadd_scalar_sat_neg(
+; CHECK-NEXT:    ret i8 -128
+;
+  %x = call i8 @llvm.sadd.sat.i8(i8 -120, i8 -10)
+  ret i8 %x
+}
+
+define <2 x i8> @test_uadd_vector_no_sat(<2 x i8> %a) {
+; CHECK-LABEL: @test_uadd_vector_no_sat(
+; CHECK-NEXT:    ret <2 x i8> <i8 20, i8 30>
+;
+  %x = call <2 x i8> @llvm.uadd.sat.v2i8(<2 x i8> <i8 10, i8 15>, <2 x i8> <i8 10, i8 15>)
+  ret <2 x i8> %x
+}
+
+define <2 x i8> @test_uadd_vector_sat(<2 x i8> %a) {
+; CHECK-LABEL: @test_uadd_vector_sat(
+; CHECK-NEXT:    ret <2 x i8> <i8 -1, i8 -1>
+;
+  %x = call <2 x i8> @llvm.uadd.sat.v2i8(<2 x i8> <i8 100, i8 200>, <2 x i8> <i8 250, i8 100>)
+  ret <2 x i8> %x
+}
+
+define <2 x i8> @test_sadd_vector_no_sat(<2 x i8> %a) {
+; CHECK-LABEL: @test_sadd_vector_no_sat(
+; CHECK-NEXT:    ret <2 x i8> <i8 -10, i8 -30>
+;
+  %x = call <2 x i8> @llvm.sadd.sat.v2i8(<2 x i8> <i8 10, i8 -15>, <2 x i8> <i8 -20, i8 -15>)
+  ret <2 x i8> %x
+}
+
+define <2 x i8> @test_sadd_vector_sat_pos(<2 x i8> %a) {
+; CHECK-LABEL: @test_sadd_vector_sat_pos(
+; CHECK-NEXT:    ret <2 x i8> <i8 127, i8 127>
+;
+  %x = call <2 x i8> @llvm.sadd.sat.v2i8(<2 x i8> <i8 100, i8 10>, <2 x i8> <i8 30, i8 120>)
+  ret <2 x i8> %x
+}
+
+define <2 x i8> @test_sadd_vector_sat_neg(<2 x i8> %a) {
+; CHECK-LABEL: @test_sadd_vector_sat_neg(
+; CHECK-NEXT:    ret <2 x i8> <i8 -128, i8 -128>
+;
+  %x = call <2 x i8> @llvm.sadd.sat.v2i8(<2 x i8> <i8 -100, i8 -10>, <2 x i8> <i8 -30, i8 -120>)
+  ret <2 x i8> %x
+}
+
+define i8 @test_usub_scalar_no_sat() {
+; CHECK-LABEL: @test_usub_scalar_no_sat(
+; CHECK-NEXT:    ret i8 10
+;
+  %x = call i8 @llvm.usub.sat.i8(i8 20, i8 10)
+  ret i8 %x
+}
+
+define i8 @test_usub_scalar_sat() {
+; CHECK-LABEL: @test_usub_scalar_sat(
+; CHECK-NEXT:    ret i8 0
+;
+  %x = call i8 @llvm.usub.sat.i8(i8 200, i8 250)
+  ret i8 %x
+}
+
+define i8 @test_ssub_scalar_no_sat() {
+; CHECK-LABEL: @test_ssub_scalar_no_sat(
+; CHECK-NEXT:    ret i8 -30
+;
+  %x = call i8 @llvm.ssub.sat.i8(i8 -10, i8 20)
+  ret i8 %x
+}
+
+define i8 @test_ssub_scalar_sat_pos() {
+; CHECK-LABEL: @test_ssub_scalar_sat_pos(
+; CHECK-NEXT:    ret i8 127
+;
+  %x = call i8 @llvm.ssub.sat.i8(i8 120, i8 -10)
+  ret i8 %x
+}
+
+define i8 @test_ssub_scalar_sat_neg() {
+; CHECK-LABEL: @test_ssub_scalar_sat_neg(
+; CHECK-NEXT:    ret i8 -128
+;
+  %x = call i8 @llvm.ssub.sat.i8(i8 -120, i8 10)
+  ret i8 %x
+}
+
+define <2 x i8> @test_usub_vector_no_sat(<2 x i8> %a) {
+; CHECK-LABEL: @test_usub_vector_no_sat(
+; CHECK-NEXT:    ret <2 x i8> <i8 10, i8 5>
+;
+  %x = call <2 x i8> @llvm.usub.sat.v2i8(<2 x i8> <i8 20, i8 15>, <2 x i8> <i8 10, i8 10>)
+  ret <2 x i8> %x
+}
+
+define <2 x i8> @test_usub_vector_sat(<2 x i8> %a) {
+; CHECK-LABEL: @test_usub_vector_sat(
+; CHECK-NEXT:    ret <2 x i8> zeroinitializer
+;
+  %x = call <2 x i8> @llvm.usub.sat.v2i8(<2 x i8> <i8 100, i8 200>, <2 x i8> <i8 150, i8 250>)
+  ret <2 x i8> %x
+}
+
+define <2 x i8> @test_ssub_vector_no_sat(<2 x i8> %a) {
+; CHECK-LABEL: @test_ssub_vector_no_sat(
+; CHECK-NEXT:    ret <2 x i8> <i8 30, i8 0>
+;
+  %x = call <2 x i8> @llvm.ssub.sat.v2i8(<2 x i8> <i8 10, i8 -15>, <2 x i8> <i8 -20, i8 -15>)
+  ret <2 x i8> %x
+}
+
+define <2 x i8> @test_ssub_vector_sat_pos(<2 x i8> %a) {
+; CHECK-LABEL: @test_ssub_vector_sat_pos(
+; CHECK-NEXT:    ret <2 x i8> <i8 127, i8 127>
+;
+  %x = call <2 x i8> @llvm.ssub.sat.v2i8(<2 x i8> <i8 100, i8 10>, <2 x i8> <i8 -30, i8 -120>)
+  ret <2 x i8> %x
+}
+
+define <2 x i8> @test_ssub_vector_sat_neg(<2 x i8> %a) {
+; CHECK-LABEL: @test_ssub_vector_sat_neg(
+; CHECK-NEXT:    ret <2 x i8> <i8 -128, i8 -128>
+;
+  %x = call <2 x i8> @llvm.ssub.sat.v2i8(<2 x i8> <i8 -100, i8 -10>, <2 x i8> <i8 30, i8 120>)
+  ret <2 x i8> %x
+}
+
+; Tests for undef handling
+
+define i8 @test_uadd_scalar_both_undef() {
+; CHECK-LABEL: @test_uadd_scalar_both_undef(
+; CHECK-NEXT:    ret i8 undef
+;
+  %x = call i8 @llvm.uadd.sat.i8(i8 undef, i8 undef)
+  ret i8 %x
+}
+
+define i8 @test_sadd_scalar_both_undef() {
+; CHECK-LABEL: @test_sadd_scalar_both_undef(
+; CHECK-NEXT:    ret i8 undef
+;
+  %x = call i8 @llvm.sadd.sat.i8(i8 undef, i8 undef)
+  ret i8 %x
+}
+
+define i8 @test_usub_scalar_both_undef() {
+; CHECK-LABEL: @test_usub_scalar_both_undef(
+; CHECK-NEXT:    ret i8 undef
+;
+  %x = call i8 @llvm.usub.sat.i8(i8 undef, i8 undef)
+  ret i8 %x
+}
+
+define i8 @test_ssub_scalar_both_undef() {
+; CHECK-LABEL: @test_ssub_scalar_both_undef(
+; CHECK-NEXT:    ret i8 undef
+;
+  %x = call i8 @llvm.ssub.sat.i8(i8 undef, i8 undef)
+  ret i8 %x
+}
+
+define i8 @test_uadd_scalar_op2_undef() {
+; CHECK-LABEL: @test_uadd_scalar_op2_undef(
+; CHECK-NEXT:    ret i8 -1
+;
+  %x = call i8 @llvm.uadd.sat.i8(i8 10, i8 undef)
+  ret i8 %x
+}
+
+define i8 @test_sadd_scalar_op1_undef() {
+; CHECK-LABEL: @test_sadd_scalar_op1_undef(
+; CHECK-NEXT:    ret i8 -1
+;
+  %x = call i8 @llvm.sadd.sat.i8(i8 undef, i8 10)
+  ret i8 %x
+}
+
+define i8 @test_usub_scalar_op2_undef() {
+; CHECK-LABEL: @test_usub_scalar_op2_undef(
+; CHECK-NEXT:    ret i8 0
+;
+  %x = call i8 @llvm.usub.sat.i8(i8 10, i8 undef)
+  ret i8 %x
+}
+
+define i8 @test_usub_scalar_op1_undef() {
+; CHECK-LABEL: @test_usub_scalar_op1_undef(
+; CHECK-NEXT:    ret i8 0
+;
+  %x = call i8 @llvm.usub.sat.i8(i8 undef, i8 10)
+  ret i8 %x
+}
+
+define <2 x i8> @test_uadd_vector_both_undef_splat() {
+; CHECK-LABEL: @test_uadd_vector_both_undef_splat(
+; CHECK-NEXT:    ret <2 x i8> undef
+;
+  %x = call <2 x i8> @llvm.uadd.sat.v2i8(<2 x i8> undef, <2 x i8> undef)
+  ret <2 x i8> %x
+}
+
+define <2 x i8> @test_sadd_vector_both_undef_splat() {
+; CHECK-LABEL: @test_sadd_vector_both_undef_splat(
+; CHECK-NEXT:    ret <2 x i8> undef
+;
+  %x = call <2 x i8> @llvm.sadd.sat.v2i8(<2 x i8> undef, <2 x i8> undef)
+  ret <2 x i8> %x
+}
+
+define <2 x i8> @test_usub_vector_both_undef_splat() {
+; CHECK-LABEL: @test_usub_vector_both_undef_splat(
+; CHECK-NEXT:    ret <2 x i8> undef
+;
+  %x = call <2 x i8> @llvm.usub.sat.v2i8(<2 x i8> undef, <2 x i8> undef)
+  ret <2 x i8> %x
+}
+
+define <2 x i8> @test_ssub_vector_both_undef_splat() {
+; CHECK-LABEL: @test_ssub_vector_both_undef_splat(
+; CHECK-NEXT:    ret <2 x i8> undef
+;
+  %x = call <2 x i8> @llvm.ssub.sat.v2i8(<2 x i8> undef, <2 x i8> undef)
+  ret <2 x i8> %x
+}
+
+define <2 x i8> @test_uadd_vector_op2_undef_splat() {
+; CHECK-LABEL: @test_uadd_vector_op2_undef_splat(
+; CHECK-NEXT:    ret <2 x i8> <i8 -1, i8 -1>
+;
+  %x = call <2 x i8> @llvm.uadd.sat.v2i8(<2 x i8> <i8 10, i8 20>, <2 x i8> undef)
+  ret <2 x i8> %x
+}
+
+define <2 x i8> @test_sadd_vector_op1_undef_splat() {
+; CHECK-LABEL: @test_sadd_vector_op1_undef_splat(
+; CHECK-NEXT:    ret <2 x i8> <i8 -1, i8 -1>
+;
+  %x = call <2 x i8> @llvm.sadd.sat.v2i8(<2 x i8> undef, <2 x i8> <i8 10, i8 20>)
+  ret <2 x i8> %x
+}
+
+define <2 x i8> @test_usub_vector_op2_undef_splat() {
+; CHECK-LABEL: @test_usub_vector_op2_undef_splat(
+; CHECK-NEXT:    ret <2 x i8> zeroinitializer
+;
+  %x = call <2 x i8> @llvm.usub.sat.v2i8(<2 x i8> <i8 10, i8 20>, <2 x i8> undef)
+  ret <2 x i8> %x
+}
+
+define <2 x i8> @test_ssub_vector_op1_undef_splat() {
+; CHECK-LABEL: @test_ssub_vector_op1_undef_splat(
+; CHECK-NEXT:    ret <2 x i8> zeroinitializer
+;
+  %x = call <2 x i8> @llvm.ssub.sat.v2i8(<2 x i8> undef, <2 x i8> <i8 10, i8 20>)
+  ret <2 x i8> %x
+}
+
+define <2 x i8> @test_uadd_vector_op2_undef_mix1() {
+; CHECK-LABEL: @test_uadd_vector_op2_undef_mix1(
+; CHECK-NEXT:    ret <2 x i8> <i8 30, i8 undef>
+;
+  %x = call <2 x i8> @llvm.uadd.sat.v2i8(<2 x i8> <i8 10, i8 undef>, <2 x i8> <i8 20, i8 undef>)
+  ret <2 x i8> %x
+}
+
+define <2 x i8> @test_uadd_vector_op2_undef_mix2() {
+; CHECK-LABEL: @test_uadd_vector_op2_undef_mix2(
+; CHECK-NEXT:    ret <2 x i8> <i8 -1, i8 -1>
+;
+  %x = call <2 x i8> @llvm.uadd.sat.v2i8(<2 x i8> <i8 10, i8 undef>, <2 x i8> <i8 undef, i8 20>)
+  ret <2 x i8> %x
+}
+
+define <2 x i8> @test_sadd_vector_op1_undef_mix1() {
+; CHECK-LABEL: @test_sadd_vector_op1_undef_mix1(
+; CHECK-NEXT:    ret <2 x i8> <i8 undef, i8 30>
+;
+  %x = call <2 x i8> @llvm.sadd.sat.v2i8(<2 x i8> <i8 undef, i8 10>, <2 x i8> <i8 undef, i8 20>)
+  ret <2 x i8> %x
+}
+
+define <2 x i8> @test_sadd_vector_op1_undef_mix2() {
+; CHECK-LABEL: @test_sadd_vector_op1_undef_mix2(
+; CHECK-NEXT:    ret <2 x i8> <i8 -1, i8 -1>
+;
+  %x = call <2 x i8> @llvm.sadd.sat.v2i8(<2 x i8> <i8 undef, i8 10>, <2 x i8> <i8 20, i8 undef>)
+  ret <2 x i8> %x
+}
+
+define <2 x i8> @test_usub_vector_op2_undef_mix1() {
+; CHECK-LABEL: @test_usub_vector_op2_undef_mix1(
+; CHECK-NEXT:    ret <2 x i8> <i8 0, i8 undef>
+;
+  %x = call <2 x i8> @llvm.usub.sat.v2i8(<2 x i8> <i8 10, i8 undef>, <2 x i8> <i8 20, i8 undef>)
+  ret <2 x i8> %x
+}
+
+define <2 x i8> @test_usub_vector_op2_undef_mix2() {
+; CHECK-LABEL: @test_usub_vector_op2_undef_mix2(
+; CHECK-NEXT:    ret <2 x i8> zeroinitializer
+;
+  %x = call <2 x i8> @llvm.usub.sat.v2i8(<2 x i8> <i8 10, i8 undef>, <2 x i8> <i8 undef, i8 20>)
+  ret <2 x i8> %x
+}
+
+define <2 x i8> @test_ssub_vector_op1_undef_mix1() {
+; CHECK-LABEL: @test_ssub_vector_op1_undef_mix1(
+; CHECK-NEXT:    ret <2 x i8> <i8 undef, i8 -10>
+;
+  %x = call <2 x i8> @llvm.ssub.sat.v2i8(<2 x i8> <i8 undef, i8 10>, <2 x i8> <i8 undef, i8 20>)
+  ret <2 x i8> %x
+}
+
+define <2 x i8> @test_ssub_vector_op1_undef_mix2() {
+; CHECK-LABEL: @test_ssub_vector_op1_undef_mix2(
+; CHECK-NEXT:    ret <2 x i8> zeroinitializer
+;
+  %x = call <2 x i8> @llvm.ssub.sat.v2i8(<2 x i8> <i8 undef, i8 10>, <2 x i8> <i8 20, i8 undef>)
+  ret <2 x i8> %x
 }
diff --git a/test/Analysis/CostModel/PowerPC/p9.ll b/test/Analysis/CostModel/PowerPC/p9.ll
new file mode 100644
index 0000000000000000000000000000000000000000..38ffc077628ca4062a60aa4d00d0990f906009ad
--- /dev/null
+++ b/test/Analysis/CostModel/PowerPC/p9.ll
@@ -0,0 +1,68 @@
+; RUN: opt < %s -cost-model -analyze -mtriple=powerpc64-unknown-linux-gnu -mcpu=pwr7 -mattr=+vsx | FileCheck %s
+; RUN: opt < %s -cost-model -analyze -mtriple=powerpc64-unknown-linux-gnu -mcpu=pwr9 -mattr=+vsx | FileCheck --check-prefix=CHECK-P9 %s
+; RUN: opt < %s -cost-model -analyze -mtriple=powerpc64le-unknown-linux-gnu -mcpu=pwr9 -mattr=+vsx | FileCheck --check-prefix=CHECK-LE %s
+
+define void @testi16(i16 %arg1, i16 %arg2, i16* %arg3) {
+
+  %s1 = add i16 %arg1, %arg2
+  %s2 = zext i16 %arg1 to i32
+  %s3 = load i16, i16* %arg3
+  store i16 %arg2, i16* %arg3
+  %c = icmp eq i16 %arg1, %arg2
+
+  ret void
+  ; CHECK: cost of 1 {{.*}} add
+  ; CHECK: cost of 1 {{.*}} zext
+  ; CHECK: cost of 1 {{.*}} load
+  ; CHECK: cost of 1 {{.*}} store
+  ; CHECK: cost of 1 {{.*}} icmp
+  ; CHECK-P9: cost of 1 {{.*}} add
+  ; CHECK-P9: cost of 1 {{.*}} zext
+  ; CHECK-P9: cost of 1 {{.*}} load
+  ; CHECK-P9: cost of 1 {{.*}} store
+  ; CHECK-P9: cost of 1 {{.*}} icmp
+}
+
+define void @test4xi16(<4 x i16> %arg1, <4 x i16> %arg2) {
+
+  %v1 = add <4 x i16> %arg1, %arg2
+  %v2 = zext <4 x i16> %arg1 to <4 x i32>
+  %v3 = shufflevector <4 x i16> %arg1, <4 x i16> undef, <4 x i32> zeroinitializer
+  %c = icmp eq <4 x i16> %arg1, %arg2
+
+  ret void
+  ; CHECK: cost of 1 {{.*}} add
+  ; CHECK: cost of 1 {{.*}} zext
+  ; CHECK: cost of 1 {{.*}} shufflevector
+  ; CHECK: cost of 1 {{.*}} icmp
+  ; CHECK-P9: cost of 2 {{.*}} add
+  ; CHECK-P9: cost of 2 {{.*}} zext
+  ; CHECK-P9: cost of 2 {{.*}} shufflevector
+  ; CHECK-P9: cost of 2 {{.*}} icmp
+}
+
+define void @test4xi32(<4 x i32> %arg1, <4 x i32> %arg2, <4 x i32>* %arg3) {
+
+  %v1 = load <4 x i32>, <4 x i32>* %arg3
+  store <4 x i32> %arg2, <4 x i32>* %arg3
+
+  ret void
+  ; CHECK: cost of 1 {{.*}} load
+  ; CHECK: cost of 1 {{.*}} store
+  ; CHECK-P9: cost of 2 {{.*}} load
+  ; CHECK-P9: cost of 2 {{.*}} store
+}
+
+define void @test2xdouble(<2 x double> %arg1) {
+  %v1 = extractelement <2 x double> %arg1, i32 0
+  %v2 = extractelement <2 x double> %arg1, i32 1
+
+  ret void
+  ; CHECK: cost of 0 {{.*}} extractelement
+  ; CHECK: cost of 1 {{.*}} extractelement
+  ; CHECK-P9: cost of 0 {{.*}} extractelement
+  ; CHECK-P9: cost of 2 {{.*}} extractelement
+  ; CHECK-LE-LABEL: test2xdouble
+  ; CHECK-LE: cost of 2 {{.*}} extractelement
+  ; CHECK-LE: cost of 0 {{.*}} extractelement
+}
diff --git a/test/Analysis/CostModel/X86/arith-fix.ll b/test/Analysis/CostModel/X86/arith-fix.ll
new file mode 100644
index 0000000000000000000000000000000000000000..b9fa774cb6a511140af8734c950ac1126d31f1ba
--- /dev/null
+++ b/test/Analysis/CostModel/X86/arith-fix.ll
@@ -0,0 +1,138 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
+; RUN: opt < %s -cost-model -analyze -mtriple=x86_64-apple-macosx10.8.0 -mattr=+ssse3 | FileCheck %s --check-prefixes=CHECK,SSE,SSSE3
+; RUN: opt < %s -cost-model -analyze -mtriple=x86_64-apple-macosx10.8.0 -mattr=+sse4.2 | FileCheck %s --check-prefixes=CHECK,SSE,SSE42
+; RUN: opt < %s -cost-model -analyze -mtriple=x86_64-apple-macosx10.8.0 -mattr=+avx | FileCheck %s --check-prefixes=CHECK,AVX,AVX1
+; RUN: opt < %s -cost-model -analyze -mtriple=x86_64-apple-macosx10.8.0 -mattr=+avx2 | FileCheck %s --check-prefixes=CHECK,AVX,AVX2
+; RUN: opt < %s -cost-model -analyze -mtriple=x86_64-apple-macosx10.8.0 -mattr=+avx512f | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512F
+; RUN: opt < %s -cost-model -analyze -mtriple=x86_64-apple-macosx10.8.0 -mattr=+avx512f,+avx512bw | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512BW
+; RUN: opt < %s -cost-model -analyze -mtriple=x86_64-apple-macosx10.8.0 -mattr=+avx512f,+avx512dq | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512DQ
+;
+; RUN: opt < %s -cost-model -analyze -mtriple=x86_64-apple-macosx10.8.0 -mcpu=slm | FileCheck %s --check-prefixes=CHECK,SLM
+; RUN: opt < %s -cost-model -analyze -mtriple=x86_64-apple-macosx10.8.0 -mcpu=goldmont | FileCheck %s --check-prefixes=CHECK,GLM
+; RUN: opt < %s -cost-model -analyze -mtriple=x86_64-apple-macosx10.8.0 -mcpu=btver2 | FileCheck %s --check-prefixes=CHECK,BTVER2
+
+declare i64        @llvm.smul.fix.i64(i64, i64, i32)
+declare <2 x i64>  @llvm.smul.fix.v2i64(<2 x i64>, <2 x i64>, i32)
+declare <4 x i64>  @llvm.smul.fix.v4i64(<4 x i64>, <4 x i64>, i32)
+declare <8 x i64>  @llvm.smul.fix.v8i64(<8 x i64>, <8 x i64>, i32)
+
+declare i32        @llvm.smul.fix.i32(i32, i32, i32)
+declare <4 x i32>  @llvm.smul.fix.v4i32(<4 x i32>, <4 x i32>, i32)
+declare <8 x i32>  @llvm.smul.fix.v8i32(<8 x i32>, <8 x i32>, i32)
+declare <16 x i32> @llvm.smul.fix.v16i32(<16 x i32>, <16 x i32>, i32)
+
+declare i16        @llvm.smul.fix.i16(i16, i16, i32)
+declare <8 x i16>  @llvm.smul.fix.v8i16(<8 x i16>, <8 x i16>, i32)
+declare <16 x i16> @llvm.smul.fix.v16i16(<16 x i16>, <16 x i16>, i32)
+declare <32 x i16> @llvm.smul.fix.v32i16(<32 x i16>, <32 x i16>, i32)
+
+declare i8         @llvm.smul.fix.i8(i8, i8, i32)
+declare <16 x i8>  @llvm.smul.fix.v16i8(<16 x i8>, <16 x i8>, i32)
+declare <32 x i8>  @llvm.smul.fix.v32i8(<32 x i8>, <32 x i8>, i32)
+declare <64 x i8>  @llvm.smul.fix.v64i8(<64 x i8>, <64 x i8>, i32)
+
+define i32 @smul(i32 %arg) {
+; CHECK-LABEL: 'smul'
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.smul.fix.i64(i64 undef, i64 undef, i32 3)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2I64 = call <2 x i64> @llvm.smul.fix.v2i64(<2 x i64> undef, <2 x i64> undef, i32 3)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4I64 = call <4 x i64> @llvm.smul.fix.v4i64(<4 x i64> undef, <4 x i64> undef, i32 3)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8I64 = call <8 x i64> @llvm.smul.fix.v8i64(<8 x i64> undef, <8 x i64> undef, i32 3)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.smul.fix.i32(i32 undef, i32 undef, i32 3)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4I32 = call <4 x i32> @llvm.smul.fix.v4i32(<4 x i32> undef, <4 x i32> undef, i32 3)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8I32 = call <8 x i32> @llvm.smul.fix.v8i32(<8 x i32> undef, <8 x i32> undef, i32 3)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V16I32 = call <16 x i32> @llvm.smul.fix.v16i32(<16 x i32> undef, <16 x i32> undef, i32 3)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.smul.fix.i16(i16 undef, i16 undef, i32 3)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8I16 = call <8 x i16> @llvm.smul.fix.v8i16(<8 x i16> undef, <8 x i16> undef, i32 3)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V16I16 = call <16 x i16> @llvm.smul.fix.v16i16(<16 x i16> undef, <16 x i16> undef, i32 3)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %V32I16 = call <32 x i16> @llvm.smul.fix.v32i16(<32 x i16> undef, <32 x i16> undef, i32 3)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.smul.fix.i8(i8 undef, i8 undef, i32 3)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V16I8 = call <16 x i8> @llvm.smul.fix.v16i8(<16 x i8> undef, <16 x i8> undef, i32 3)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %V32I8 = call <32 x i8> @llvm.smul.fix.v32i8(<32 x i8> undef, <32 x i8> undef, i32 3)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: %V64I8 = call <64 x i8> @llvm.smul.fix.v64i8(<64 x i8> undef, <64 x i8> undef, i32 3)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+  %I64 = call i64 @llvm.smul.fix.i64(i64 undef, i64 undef, i32 3)
+  %V2I64 = call <2 x i64> @llvm.smul.fix.v2i64(<2 x i64> undef, <2 x i64> undef, i32 3)
+  %V4I64 = call <4 x i64> @llvm.smul.fix.v4i64(<4 x i64> undef, <4 x i64> undef, i32 3)
+  %V8I64 = call <8 x i64> @llvm.smul.fix.v8i64(<8 x i64> undef, <8 x i64> undef, i32 3)
+
+  %I32 = call i32 @llvm.smul.fix.i32(i32 undef, i32 undef, i32 3)
+  %V4I32  = call <4 x i32>  @llvm.smul.fix.v4i32(<4 x i32> undef, <4 x i32> undef, i32 3)
+  %V8I32  = call <8 x i32>  @llvm.smul.fix.v8i32(<8 x i32> undef, <8 x i32> undef, i32 3)
+  %V16I32 = call <16 x i32> @llvm.smul.fix.v16i32(<16 x i32> undef, <16 x i32> undef, i32 3)
+
+  %I16 = call i16 @llvm.smul.fix.i16(i16 undef, i16 undef, i32 3)
+  %V8I16  = call <8 x i16>  @llvm.smul.fix.v8i16(<8 x i16> undef, <8 x i16> undef, i32 3)
+  %V16I16 = call <16 x i16> @llvm.smul.fix.v16i16(<16 x i16> undef, <16 x i16> undef, i32 3)
+  %V32I16 = call <32 x i16> @llvm.smul.fix.v32i16(<32 x i16> undef, <32 x i16> undef, i32 3)
+
+  %I8 = call i8 @llvm.smul.fix.i8(i8 undef, i8 undef, i32 3)
+  %V16I8 = call <16 x i8> @llvm.smul.fix.v16i8(<16 x i8> undef, <16 x i8> undef, i32 3)
+  %V32I8 = call <32 x i8> @llvm.smul.fix.v32i8(<32 x i8> undef, <32 x i8> undef, i32 3)
+  %V64I8 = call <64 x i8> @llvm.smul.fix.v64i8(<64 x i8> undef, <64 x i8> undef, i32 3)
+
+  ret i32 undef
+}
+
+declare i64        @llvm.umul.fix.i64(i64, i64, i32)
+declare <2 x i64>  @llvm.umul.fix.v2i64(<2 x i64>, <2 x i64>, i32)
+declare <4 x i64>  @llvm.umul.fix.v4i64(<4 x i64>, <4 x i64>, i32)
+declare <8 x i64>  @llvm.umul.fix.v8i64(<8 x i64>, <8 x i64>, i32)
+
+declare i32        @llvm.umul.fix.i32(i32, i32, i32)
+declare <4 x i32>  @llvm.umul.fix.v4i32(<4 x i32>, <4 x i32>, i32)
+declare <8 x i32>  @llvm.umul.fix.v8i32(<8 x i32>, <8 x i32>, i32)
+declare <16 x i32> @llvm.umul.fix.v16i32(<16 x i32>, <16 x i32>, i32)
+
+declare i16        @llvm.umul.fix.i16(i16, i16, i32)
+declare <8 x i16>  @llvm.umul.fix.v8i16(<8 x i16>, <8 x i16>, i32)
+declare <16 x i16> @llvm.umul.fix.v16i16(<16 x i16>, <16 x i16>, i32)
+declare <32 x i16> @llvm.umul.fix.v32i16(<32 x i16>, <32 x i16>, i32)
+
+declare i8         @llvm.umul.fix.i8(i8, i8, i32)
+declare <16 x i8>  @llvm.umul.fix.v16i8(<16 x i8>, <16 x i8>, i32)
+declare <32 x i8>  @llvm.umul.fix.v32i8(<32 x i8>, <32 x i8>, i32)
+declare <64 x i8>  @llvm.umul.fix.v64i8(<64 x i8>, <64 x i8>, i32)
+
+define i32 @umul(i32 %arg) {
+; CHECK-LABEL: 'umul'
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call i64 @llvm.umul.fix.i64(i64 undef, i64 undef, i32 3)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2I64 = call <2 x i64> @llvm.umul.fix.v2i64(<2 x i64> undef, <2 x i64> undef, i32 3)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4I64 = call <4 x i64> @llvm.umul.fix.v4i64(<4 x i64> undef, <4 x i64> undef, i32 3)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8I64 = call <8 x i64> @llvm.umul.fix.v8i64(<8 x i64> undef, <8 x i64> undef, i32 3)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call i32 @llvm.umul.fix.i32(i32 undef, i32 undef, i32 3)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4I32 = call <4 x i32> @llvm.umul.fix.v4i32(<4 x i32> undef, <4 x i32> undef, i32 3)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8I32 = call <8 x i32> @llvm.umul.fix.v8i32(<8 x i32> undef, <8 x i32> undef, i32 3)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V16I32 = call <16 x i32> @llvm.umul.fix.v16i32(<16 x i32> undef, <16 x i32> undef, i32 3)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call i16 @llvm.umul.fix.i16(i16 undef, i16 undef, i32 3)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8I16 = call <8 x i16> @llvm.umul.fix.v8i16(<8 x i16> undef, <8 x i16> undef, i32 3)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V16I16 = call <16 x i16> @llvm.umul.fix.v16i16(<16 x i16> undef, <16 x i16> undef, i32 3)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %V32I16 = call <32 x i16> @llvm.umul.fix.v32i16(<32 x i16> undef, <32 x i16> undef, i32 3)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call i8 @llvm.umul.fix.i8(i8 undef, i8 undef, i32 3)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V16I8 = call <16 x i8> @llvm.umul.fix.v16i8(<16 x i8> undef, <16 x i8> undef, i32 3)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %V32I8 = call <32 x i8> @llvm.umul.fix.v32i8(<32 x i8> undef, <32 x i8> undef, i32 3)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: %V64I8 = call <64 x i8> @llvm.umul.fix.v64i8(<64 x i8> undef, <64 x i8> undef, i32 3)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+  %I64 = call i64 @llvm.umul.fix.i64(i64 undef, i64 undef, i32 3)
+  %V2I64 = call <2 x i64> @llvm.umul.fix.v2i64(<2 x i64> undef, <2 x i64> undef, i32 3)
+  %V4I64 = call <4 x i64> @llvm.umul.fix.v4i64(<4 x i64> undef, <4 x i64> undef, i32 3)
+  %V8I64 = call <8 x i64> @llvm.umul.fix.v8i64(<8 x i64> undef, <8 x i64> undef, i32 3)
+
+  %I32 = call i32 @llvm.umul.fix.i32(i32 undef, i32 undef, i32 3)
+  %V4I32  = call <4 x i32>  @llvm.umul.fix.v4i32(<4 x i32> undef, <4 x i32> undef, i32 3)
+  %V8I32  = call <8 x i32>  @llvm.umul.fix.v8i32(<8 x i32> undef, <8 x i32> undef, i32 3)
+  %V16I32 = call <16 x i32> @llvm.umul.fix.v16i32(<16 x i32> undef, <16 x i32> undef, i32 3)
+
+  %I16 = call i16 @llvm.umul.fix.i16(i16 undef, i16 undef, i32 3)
+  %V8I16  = call <8 x i16>  @llvm.umul.fix.v8i16(<8 x i16> undef, <8 x i16> undef, i32 3)
+  %V16I16 = call <16 x i16> @llvm.umul.fix.v16i16(<16 x i16> undef, <16 x i16> undef, i32 3)
+  %V32I16 = call <32 x i16> @llvm.umul.fix.v32i16(<32 x i16> undef, <32 x i16> undef, i32 3)
+
+  %I8 = call i8 @llvm.umul.fix.i8(i8 undef, i8 undef, i32 3)
+  %V16I8 = call <16 x i8> @llvm.umul.fix.v16i8(<16 x i8> undef, <16 x i8> undef, i32 3)
+  %V32I8 = call <32 x i8> @llvm.umul.fix.v32i8(<32 x i8> undef, <32 x i8> undef, i32 3)
+  %V64I8 = call <64 x i8> @llvm.umul.fix.v64i8(<64 x i8> undef, <64 x i8> undef, i32 3)
+
+  ret i32 undef
+}
diff --git a/test/Analysis/CostModel/X86/arith-fp.ll b/test/Analysis/CostModel/X86/arith-fp.ll
index a6b5cf61473de8f6229c521363459b43a85f3952..bce7ca8d59f27e1d0628c1d850ad5242f70cbc14 100644
--- a/test/Analysis/CostModel/X86/arith-fp.ll
+++ b/test/Analysis/CostModel/X86/arith-fp.ll
@@ -1,4 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
+; RUN: opt < %s -enable-no-nans-fp-math  -cost-model -analyze -mtriple=x86_64-apple-macosx10.8.0 -mattr=-sse2 | FileCheck %s --check-prefixes=CHECK,SSE,SSE1
 ; RUN: opt < %s -enable-no-nans-fp-math  -cost-model -analyze -mtriple=x86_64-apple-macosx10.8.0 -mattr=+sse2 | FileCheck %s --check-prefixes=CHECK,SSE,SSE2
 ; RUN: opt < %s -enable-no-nans-fp-math  -cost-model -analyze -mtriple=x86_64-apple-macosx10.8.0 -mattr=+sse4.2 | FileCheck %s --check-prefixes=CHECK,SSE,SSE42
 ; RUN: opt < %s -enable-no-nans-fp-math  -cost-model -analyze -mtriple=x86_64-apple-macosx10.8.0 -mattr=+avx | FileCheck %s --check-prefixes=CHECK,AVX,AVX1
@@ -14,6 +15,17 @@ target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f3
 target triple = "x86_64-apple-macosx10.8.0"
 
 define i32 @fadd(i32 %arg) {
+; SSE1-LABEL: 'fadd'
+; SSE1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fadd float undef, undef
+; SSE1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F32 = fadd <4 x float> undef, undef
+; SSE1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F32 = fadd <8 x float> undef, undef
+; SSE1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16F32 = fadd <16 x float> undef, undef
+; SSE1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %F64 = fadd double undef, undef
+; SSE1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2F64 = fadd <2 x double> undef, undef
+; SSE1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4F64 = fadd <4 x double> undef, undef
+; SSE1-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8F64 = fadd <8 x double> undef, undef
+; SSE1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
 ; SSE2-LABEL: 'fadd'
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %F32 = fadd float undef, undef
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F32 = fadd <4 x float> undef, undef
@@ -116,6 +128,17 @@ define i32 @fadd(i32 %arg) {
 }
 
 define i32 @fsub(i32 %arg) {
+; SSE1-LABEL: 'fsub'
+; SSE1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fsub float undef, undef
+; SSE1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F32 = fsub <4 x float> undef, undef
+; SSE1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F32 = fsub <8 x float> undef, undef
+; SSE1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16F32 = fsub <16 x float> undef, undef
+; SSE1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %F64 = fsub double undef, undef
+; SSE1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2F64 = fsub <2 x double> undef, undef
+; SSE1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4F64 = fsub <4 x double> undef, undef
+; SSE1-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8F64 = fsub <8 x double> undef, undef
+; SSE1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
 ; SSE2-LABEL: 'fsub'
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %F32 = fsub float undef, undef
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F32 = fsub <4 x float> undef, undef
@@ -218,6 +241,17 @@ define i32 @fsub(i32 %arg) {
 }
 
 define i32 @fmul(i32 %arg) {
+; SSE1-LABEL: 'fmul'
+; SSE1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %F32 = fmul float undef, undef
+; SSE1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F32 = fmul <4 x float> undef, undef
+; SSE1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F32 = fmul <8 x float> undef, undef
+; SSE1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16F32 = fmul <16 x float> undef, undef
+; SSE1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %F64 = fmul double undef, undef
+; SSE1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2F64 = fmul <2 x double> undef, undef
+; SSE1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4F64 = fmul <4 x double> undef, undef
+; SSE1-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8F64 = fmul <8 x double> undef, undef
+; SSE1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
 ; SSE2-LABEL: 'fmul'
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %F32 = fmul float undef, undef
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F32 = fmul <4 x float> undef, undef
@@ -320,6 +354,17 @@ define i32 @fmul(i32 %arg) {
 }
 
 define i32 @fdiv(i32 %arg) {
+; SSE1-LABEL: 'fdiv'
+; SSE1-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %F32 = fdiv float undef, undef
+; SSE1-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %V4F32 = fdiv <4 x float> undef, undef
+; SSE1-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %V8F32 = fdiv <8 x float> undef, undef
+; SSE1-NEXT:  Cost Model: Found an estimated cost of 136 for instruction: %V16F32 = fdiv <16 x float> undef, undef
+; SSE1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %F64 = fdiv double undef, undef
+; SSE1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2F64 = fdiv <2 x double> undef, undef
+; SSE1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4F64 = fdiv <4 x double> undef, undef
+; SSE1-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8F64 = fdiv <8 x double> undef, undef
+; SSE1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
 ; SSE2-LABEL: 'fdiv'
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %F32 = fdiv float undef, undef
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 39 for instruction: %V4F32 = fdiv <4 x float> undef, undef
@@ -422,16 +467,38 @@ define i32 @fdiv(i32 %arg) {
 }
 
 define i32 @frem(i32 %arg) {
-; SSE-LABEL: 'frem'
-; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %F32 = frem float undef, undef
-; SSE-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4F32 = frem <4 x float> undef, undef
-; SSE-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V8F32 = frem <8 x float> undef, undef
-; SSE-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V16F32 = frem <16 x float> undef, undef
-; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %F64 = frem double undef, undef
-; SSE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2F64 = frem <2 x double> undef, undef
-; SSE-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4F64 = frem <4 x double> undef, undef
-; SSE-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8F64 = frem <8 x double> undef, undef
-; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+; SSE1-LABEL: 'frem'
+; SSE1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %F32 = frem float undef, undef
+; SSE1-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4F32 = frem <4 x float> undef, undef
+; SSE1-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V8F32 = frem <8 x float> undef, undef
+; SSE1-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V16F32 = frem <16 x float> undef, undef
+; SSE1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %F64 = frem double undef, undef
+; SSE1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2F64 = frem <2 x double> undef, undef
+; SSE1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4F64 = frem <4 x double> undef, undef
+; SSE1-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8F64 = frem <8 x double> undef, undef
+; SSE1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSE2-LABEL: 'frem'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %F32 = frem float undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4F32 = frem <4 x float> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V8F32 = frem <8 x float> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V16F32 = frem <16 x float> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %F64 = frem double undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2F64 = frem <2 x double> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4F64 = frem <4 x double> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8F64 = frem <8 x double> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSE42-LABEL: 'frem'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %F32 = frem float undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4F32 = frem <4 x float> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V8F32 = frem <8 x float> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V16F32 = frem <16 x float> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %F64 = frem double undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2F64 = frem <2 x double> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4F64 = frem <4 x double> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8F64 = frem <8 x double> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX-LABEL: 'frem'
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %F32 = frem float undef, undef
@@ -502,6 +569,17 @@ define i32 @frem(i32 %arg) {
 }
 
 define i32 @fsqrt(i32 %arg) {
+; SSE1-LABEL: 'fsqrt'
+; SSE1-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %F32 = call float @llvm.sqrt.f32(float undef)
+; SSE1-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V4F32 = call <4 x float> @llvm.sqrt.v4f32(<4 x float> undef)
+; SSE1-NEXT:  Cost Model: Found an estimated cost of 112 for instruction: %V8F32 = call <8 x float> @llvm.sqrt.v8f32(<8 x float> undef)
+; SSE1-NEXT:  Cost Model: Found an estimated cost of 224 for instruction: %V16F32 = call <16 x float> @llvm.sqrt.v16f32(<16 x float> undef)
+; SSE1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = call double @llvm.sqrt.f64(double undef)
+; SSE1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2F64 = call <2 x double> @llvm.sqrt.v2f64(<2 x double> undef)
+; SSE1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4F64 = call <4 x double> @llvm.sqrt.v4f64(<4 x double> undef)
+; SSE1-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8F64 = call <8 x double> @llvm.sqrt.v8f64(<8 x double> undef)
+; SSE1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
 ; SSE2-LABEL: 'fsqrt'
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %F32 = call float @llvm.sqrt.f32(float undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V4F32 = call <4 x float> @llvm.sqrt.v4f32(<4 x float> undef)
@@ -604,16 +682,38 @@ define i32 @fsqrt(i32 %arg) {
 }
 
 define i32 @fabs(i32 %arg) {
-; SSE-LABEL: 'fabs'
-; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %F32 = call float @llvm.fabs.f32(float undef)
-; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F32 = call <4 x float> @llvm.fabs.v4f32(<4 x float> undef)
-; SSE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F32 = call <8 x float> @llvm.fabs.v8f32(<8 x float> undef)
-; SSE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16F32 = call <16 x float> @llvm.fabs.v16f32(<16 x float> undef)
-; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %F64 = call double @llvm.fabs.f64(double undef)
-; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F64 = call <2 x double> @llvm.fabs.v2f64(<2 x double> undef)
-; SSE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4F64 = call <4 x double> @llvm.fabs.v4f64(<4 x double> undef)
-; SSE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8F64 = call <8 x double> @llvm.fabs.v8f64(<8 x double> undef)
-; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+; SSE1-LABEL: 'fabs'
+; SSE1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %F32 = call float @llvm.fabs.f32(float undef)
+; SSE1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F32 = call <4 x float> @llvm.fabs.v4f32(<4 x float> undef)
+; SSE1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F32 = call <8 x float> @llvm.fabs.v8f32(<8 x float> undef)
+; SSE1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16F32 = call <16 x float> @llvm.fabs.v16f32(<16 x float> undef)
+; SSE1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = call double @llvm.fabs.f64(double undef)
+; SSE1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2F64 = call <2 x double> @llvm.fabs.v2f64(<2 x double> undef)
+; SSE1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4F64 = call <4 x double> @llvm.fabs.v4f64(<4 x double> undef)
+; SSE1-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8F64 = call <8 x double> @llvm.fabs.v8f64(<8 x double> undef)
+; SSE1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSE2-LABEL: 'fabs'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %F32 = call float @llvm.fabs.f32(float undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F32 = call <4 x float> @llvm.fabs.v4f32(<4 x float> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F32 = call <8 x float> @llvm.fabs.v8f32(<8 x float> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16F32 = call <16 x float> @llvm.fabs.v16f32(<16 x float> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %F64 = call double @llvm.fabs.f64(double undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F64 = call <2 x double> @llvm.fabs.v2f64(<2 x double> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4F64 = call <4 x double> @llvm.fabs.v4f64(<4 x double> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8F64 = call <8 x double> @llvm.fabs.v8f64(<8 x double> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSE42-LABEL: 'fabs'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %F32 = call float @llvm.fabs.f32(float undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F32 = call <4 x float> @llvm.fabs.v4f32(<4 x float> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F32 = call <8 x float> @llvm.fabs.v8f32(<8 x float> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16F32 = call <16 x float> @llvm.fabs.v16f32(<16 x float> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %F64 = call double @llvm.fabs.f64(double undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F64 = call <2 x double> @llvm.fabs.v2f64(<2 x double> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4F64 = call <4 x double> @llvm.fabs.v4f64(<4 x double> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8F64 = call <8 x double> @llvm.fabs.v8f64(<8 x double> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX-LABEL: 'fabs'
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %F32 = call float @llvm.fabs.f32(float undef)
@@ -684,16 +784,38 @@ define i32 @fabs(i32 %arg) {
 }
 
 define i32 @fcopysign(i32 %arg) {
-; SSE-LABEL: 'fcopysign'
-; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %F32 = call float @llvm.copysign.f32(float undef, float undef)
-; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F32 = call <4 x float> @llvm.copysign.v4f32(<4 x float> undef, <4 x float> undef)
-; SSE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F32 = call <8 x float> @llvm.copysign.v8f32(<8 x float> undef, <8 x float> undef)
-; SSE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16F32 = call <16 x float> @llvm.copysign.v16f32(<16 x float> undef, <16 x float> undef)
-; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %F64 = call double @llvm.copysign.f64(double undef, double undef)
-; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F64 = call <2 x double> @llvm.copysign.v2f64(<2 x double> undef, <2 x double> undef)
-; SSE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4F64 = call <4 x double> @llvm.copysign.v4f64(<4 x double> undef, <4 x double> undef)
-; SSE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8F64 = call <8 x double> @llvm.copysign.v8f64(<8 x double> undef, <8 x double> undef)
-; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+; SSE1-LABEL: 'fcopysign'
+; SSE1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %F32 = call float @llvm.copysign.f32(float undef, float undef)
+; SSE1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F32 = call <4 x float> @llvm.copysign.v4f32(<4 x float> undef, <4 x float> undef)
+; SSE1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F32 = call <8 x float> @llvm.copysign.v8f32(<8 x float> undef, <8 x float> undef)
+; SSE1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16F32 = call <16 x float> @llvm.copysign.v16f32(<16 x float> undef, <16 x float> undef)
+; SSE1-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %F64 = call double @llvm.copysign.f64(double undef, double undef)
+; SSE1-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V2F64 = call <2 x double> @llvm.copysign.v2f64(<2 x double> undef, <2 x double> undef)
+; SSE1-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %V4F64 = call <4 x double> @llvm.copysign.v4f64(<4 x double> undef, <4 x double> undef)
+; SSE1-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %V8F64 = call <8 x double> @llvm.copysign.v8f64(<8 x double> undef, <8 x double> undef)
+; SSE1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSE2-LABEL: 'fcopysign'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %F32 = call float @llvm.copysign.f32(float undef, float undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F32 = call <4 x float> @llvm.copysign.v4f32(<4 x float> undef, <4 x float> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F32 = call <8 x float> @llvm.copysign.v8f32(<8 x float> undef, <8 x float> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16F32 = call <16 x float> @llvm.copysign.v16f32(<16 x float> undef, <16 x float> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %F64 = call double @llvm.copysign.f64(double undef, double undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F64 = call <2 x double> @llvm.copysign.v2f64(<2 x double> undef, <2 x double> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4F64 = call <4 x double> @llvm.copysign.v4f64(<4 x double> undef, <4 x double> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8F64 = call <8 x double> @llvm.copysign.v8f64(<8 x double> undef, <8 x double> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSE42-LABEL: 'fcopysign'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %F32 = call float @llvm.copysign.f32(float undef, float undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F32 = call <4 x float> @llvm.copysign.v4f32(<4 x float> undef, <4 x float> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F32 = call <8 x float> @llvm.copysign.v8f32(<8 x float> undef, <8 x float> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16F32 = call <16 x float> @llvm.copysign.v16f32(<16 x float> undef, <16 x float> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %F64 = call double @llvm.copysign.f64(double undef, double undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F64 = call <2 x double> @llvm.copysign.v2f64(<2 x double> undef, <2 x double> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4F64 = call <4 x double> @llvm.copysign.v4f64(<4 x double> undef, <4 x double> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8F64 = call <8 x double> @llvm.copysign.v8f64(<8 x double> undef, <8 x double> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX-LABEL: 'fcopysign'
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %F32 = call float @llvm.copysign.f32(float undef, float undef)
@@ -764,16 +886,38 @@ define i32 @fcopysign(i32 %arg) {
 }
 
 define i32 @fma(i32 %arg) {
-; SSE-LABEL: 'fma'
-; SSE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %F32 = call float @llvm.fma.f32(float undef, float undef, float undef)
-; SSE-NEXT:  Cost Model: Found an estimated cost of 43 for instruction: %V4F32 = call <4 x float> @llvm.fma.v4f32(<4 x float> undef, <4 x float> undef, <4 x float> undef)
-; SSE-NEXT:  Cost Model: Found an estimated cost of 86 for instruction: %V8F32 = call <8 x float> @llvm.fma.v8f32(<8 x float> undef, <8 x float> undef, <8 x float> undef)
-; SSE-NEXT:  Cost Model: Found an estimated cost of 172 for instruction: %V16F32 = call <16 x float> @llvm.fma.v16f32(<16 x float> undef, <16 x float> undef, <16 x float> undef)
-; SSE-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %F64 = call double @llvm.fma.f64(double undef, double undef, double undef)
-; SSE-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %V2F64 = call <2 x double> @llvm.fma.v2f64(<2 x double> undef, <2 x double> undef, <2 x double> undef)
-; SSE-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %V4F64 = call <4 x double> @llvm.fma.v4f64(<4 x double> undef, <4 x double> undef, <4 x double> undef)
-; SSE-NEXT:  Cost Model: Found an estimated cost of 84 for instruction: %V8F64 = call <8 x double> @llvm.fma.v8f64(<8 x double> undef, <8 x double> undef, <8 x double> undef)
-; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+; SSE1-LABEL: 'fma'
+; SSE1-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %F32 = call float @llvm.fma.f32(float undef, float undef, float undef)
+; SSE1-NEXT:  Cost Model: Found an estimated cost of 43 for instruction: %V4F32 = call <4 x float> @llvm.fma.v4f32(<4 x float> undef, <4 x float> undef, <4 x float> undef)
+; SSE1-NEXT:  Cost Model: Found an estimated cost of 86 for instruction: %V8F32 = call <8 x float> @llvm.fma.v8f32(<8 x float> undef, <8 x float> undef, <8 x float> undef)
+; SSE1-NEXT:  Cost Model: Found an estimated cost of 172 for instruction: %V16F32 = call <16 x float> @llvm.fma.v16f32(<16 x float> undef, <16 x float> undef, <16 x float> undef)
+; SSE1-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %F64 = call double @llvm.fma.f64(double undef, double undef, double undef)
+; SSE1-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V2F64 = call <2 x double> @llvm.fma.v2f64(<2 x double> undef, <2 x double> undef, <2 x double> undef)
+; SSE1-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %V4F64 = call <4 x double> @llvm.fma.v4f64(<4 x double> undef, <4 x double> undef, <4 x double> undef)
+; SSE1-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %V8F64 = call <8 x double> @llvm.fma.v8f64(<8 x double> undef, <8 x double> undef, <8 x double> undef)
+; SSE1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSE2-LABEL: 'fma'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %F32 = call float @llvm.fma.f32(float undef, float undef, float undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 43 for instruction: %V4F32 = call <4 x float> @llvm.fma.v4f32(<4 x float> undef, <4 x float> undef, <4 x float> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 86 for instruction: %V8F32 = call <8 x float> @llvm.fma.v8f32(<8 x float> undef, <8 x float> undef, <8 x float> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 172 for instruction: %V16F32 = call <16 x float> @llvm.fma.v16f32(<16 x float> undef, <16 x float> undef, <16 x float> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %F64 = call double @llvm.fma.f64(double undef, double undef, double undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %V2F64 = call <2 x double> @llvm.fma.v2f64(<2 x double> undef, <2 x double> undef, <2 x double> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %V4F64 = call <4 x double> @llvm.fma.v4f64(<4 x double> undef, <4 x double> undef, <4 x double> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 84 for instruction: %V8F64 = call <8 x double> @llvm.fma.v8f64(<8 x double> undef, <8 x double> undef, <8 x double> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSE42-LABEL: 'fma'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %F32 = call float @llvm.fma.f32(float undef, float undef, float undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 43 for instruction: %V4F32 = call <4 x float> @llvm.fma.v4f32(<4 x float> undef, <4 x float> undef, <4 x float> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 86 for instruction: %V8F32 = call <8 x float> @llvm.fma.v8f32(<8 x float> undef, <8 x float> undef, <8 x float> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 172 for instruction: %V16F32 = call <16 x float> @llvm.fma.v16f32(<16 x float> undef, <16 x float> undef, <16 x float> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %F64 = call double @llvm.fma.f64(double undef, double undef, double undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %V2F64 = call <2 x double> @llvm.fma.v2f64(<2 x double> undef, <2 x double> undef, <2 x double> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %V4F64 = call <4 x double> @llvm.fma.v4f64(<4 x double> undef, <4 x double> undef, <4 x double> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 84 for instruction: %V8F64 = call <8 x double> @llvm.fma.v8f64(<8 x double> undef, <8 x double> undef, <8 x double> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX-LABEL: 'fma'
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %F32 = call float @llvm.fma.f32(float undef, float undef, float undef)
diff --git a/test/Analysis/CostModel/X86/arith-overflow.ll b/test/Analysis/CostModel/X86/arith-overflow.ll
new file mode 100644
index 0000000000000000000000000000000000000000..873dd88e1f4c529f9889bdb0ba162546eb9ae393
--- /dev/null
+++ b/test/Analysis/CostModel/X86/arith-overflow.ll
@@ -0,0 +1,1098 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
+; RUN: opt < %s -cost-model -analyze -mtriple=x86_64-apple-macosx10.8.0 -mattr=+ssse3 | FileCheck %s --check-prefixes=CHECK,SSE,SSSE3
+; RUN: opt < %s -cost-model -analyze -mtriple=x86_64-apple-macosx10.8.0 -mattr=+sse4.2 | FileCheck %s --check-prefixes=CHECK,SSE,SSE42
+; RUN: opt < %s -cost-model -analyze -mtriple=x86_64-apple-macosx10.8.0 -mattr=+avx | FileCheck %s --check-prefixes=CHECK,AVX,AVX1
+; RUN: opt < %s -cost-model -analyze -mtriple=x86_64-apple-macosx10.8.0 -mattr=+avx2 | FileCheck %s --check-prefixes=CHECK,AVX,AVX2
+; RUN: opt < %s -cost-model -analyze -mtriple=x86_64-apple-macosx10.8.0 -mattr=+avx512f | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512F
+; RUN: opt < %s -cost-model -analyze -mtriple=x86_64-apple-macosx10.8.0 -mattr=+avx512f,+avx512bw | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512BW
+; RUN: opt < %s -cost-model -analyze -mtriple=x86_64-apple-macosx10.8.0 -mattr=+avx512f,+avx512dq | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512DQ
+;
+; RUN: opt < %s -cost-model -analyze -mtriple=x86_64-apple-macosx10.8.0 -mcpu=slm | FileCheck %s --check-prefixes=CHECK,SLM
+; RUN: opt < %s -cost-model -analyze -mtriple=x86_64-apple-macosx10.8.0 -mcpu=goldmont | FileCheck %s --check-prefixes=CHECK,GLM
+; RUN: opt < %s -cost-model -analyze -mtriple=x86_64-apple-macosx10.8.0 -mcpu=btver2 | FileCheck %s --check-prefixes=CHECK,BTVER2
+
+;
+; sadd.with.overflow
+;
+
+declare {i64, i1}              @llvm.sadd.with.overflow.i64(i64, i64)
+declare {<2 x i64>, <2 x i1>}  @llvm.sadd.with.overflow.v2i64(<2 x i64>, <2 x i64>)
+declare {<4 x i64>, <4 x i1>}  @llvm.sadd.with.overflow.v4i64(<4 x i64>, <4 x i64>)
+declare {<8 x i64>, <8 x i1>}  @llvm.sadd.with.overflow.v8i64(<8 x i64>, <8 x i64>)
+
+declare {i32, i1}               @llvm.sadd.with.overflow.i32(i32, i32)
+declare {<4 x i32>, <4 x i1>}   @llvm.sadd.with.overflow.v4i32(<4 x i32>, <4 x i32>)
+declare {<8 x i32>, <8 x i1>}   @llvm.sadd.with.overflow.v8i32(<8 x i32>, <8 x i32>)
+declare {<16 x i32>, <16 x i1>} @llvm.sadd.with.overflow.v16i32(<16 x i32>, <16 x i32>)
+
+declare {i16, i1}               @llvm.sadd.with.overflow.i16(i16, i16)
+declare {<8 x i16>,  <8 x i1>}  @llvm.sadd.with.overflow.v8i16(<8 x i16>, <8 x i16>)
+declare {<16 x i16>, <16 x i1>} @llvm.sadd.with.overflow.v16i16(<16 x i16>, <16 x i16>)
+declare {<32 x i16>, <32 x i1>} @llvm.sadd.with.overflow.v32i16(<32 x i16>, <32 x i16>)
+
+declare {i8, i1}                @llvm.sadd.with.overflow.i8(i8, i8)
+declare {<16 x i8>, <16 x i1>}  @llvm.sadd.with.overflow.v16i8(<16 x i8>, <16 x i8>)
+declare {<32 x i8>, <32 x i1>}  @llvm.sadd.with.overflow.v32i8(<32 x i8>, <32 x i8>)
+declare {<64 x i8>, <64 x i1>}  @llvm.sadd.with.overflow.v64i8(<64 x i8>, <64 x i8>)
+
+define i32 @sadd(i32 %arg) {
+; SSSE3-LABEL: 'sadd'
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call { i64, i1 } @llvm.sadd.with.overflow.i64(i64 undef, i64 undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.sadd.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 53 for instruction: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.sadd.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 103 for instruction: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.sadd.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 undef, i32 undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.sadd.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.sadd.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V16I32 = call { <16 x i32>, <16 x i1> } @llvm.sadd.with.overflow.v16i32(<16 x i32> undef, <16 x i32> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call { i16, i1 } @llvm.sadd.with.overflow.i16(i16 undef, i16 undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V8I16 = call { <8 x i16>, <8 x i1> } @llvm.sadd.with.overflow.v8i16(<8 x i16> undef, <8 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V16I16 = call { <16 x i16>, <16 x i1> } @llvm.sadd.with.overflow.v16i16(<16 x i16> undef, <16 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V32I16 = call { <32 x i16>, <32 x i1> } @llvm.sadd.with.overflow.v32i16(<32 x i16> undef, <32 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call { i8, i1 } @llvm.sadd.with.overflow.i8(i8 undef, i8 undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V16I8 = call { <16 x i8>, <16 x i1> } @llvm.sadd.with.overflow.v16i8(<16 x i8> undef, <16 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V32I8 = call { <32 x i8>, <32 x i1> } @llvm.sadd.with.overflow.v32i8(<32 x i8> undef, <32 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V64I8 = call { <64 x i8>, <64 x i1> } @llvm.sadd.with.overflow.v64i8(<64 x i8> undef, <64 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSE42-LABEL: 'sadd'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call { i64, i1 } @llvm.sadd.with.overflow.i64(i64 undef, i64 undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.sadd.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.sadd.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.sadd.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 undef, i32 undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.sadd.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.sadd.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V16I32 = call { <16 x i32>, <16 x i1> } @llvm.sadd.with.overflow.v16i32(<16 x i32> undef, <16 x i32> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call { i16, i1 } @llvm.sadd.with.overflow.i16(i16 undef, i16 undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V8I16 = call { <8 x i16>, <8 x i1> } @llvm.sadd.with.overflow.v8i16(<8 x i16> undef, <8 x i16> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V16I16 = call { <16 x i16>, <16 x i1> } @llvm.sadd.with.overflow.v16i16(<16 x i16> undef, <16 x i16> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V32I16 = call { <32 x i16>, <32 x i1> } @llvm.sadd.with.overflow.v32i16(<32 x i16> undef, <32 x i16> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call { i8, i1 } @llvm.sadd.with.overflow.i8(i8 undef, i8 undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V16I8 = call { <16 x i8>, <16 x i1> } @llvm.sadd.with.overflow.v16i8(<16 x i8> undef, <16 x i8> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V32I8 = call { <32 x i8>, <32 x i1> } @llvm.sadd.with.overflow.v32i8(<32 x i8> undef, <32 x i8> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V64I8 = call { <64 x i8>, <64 x i1> } @llvm.sadd.with.overflow.v64i8(<64 x i8> undef, <64 x i8> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX1-LABEL: 'sadd'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call { i64, i1 } @llvm.sadd.with.overflow.i64(i64 undef, i64 undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.sadd.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.sadd.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.sadd.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 undef, i32 undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.sadd.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.sadd.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %V16I32 = call { <16 x i32>, <16 x i1> } @llvm.sadd.with.overflow.v16i32(<16 x i32> undef, <16 x i32> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call { i16, i1 } @llvm.sadd.with.overflow.i16(i16 undef, i16 undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V8I16 = call { <8 x i16>, <8 x i1> } @llvm.sadd.with.overflow.v8i16(<8 x i16> undef, <8 x i16> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V16I16 = call { <16 x i16>, <16 x i1> } @llvm.sadd.with.overflow.v16i16(<16 x i16> undef, <16 x i16> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 41 for instruction: %V32I16 = call { <32 x i16>, <32 x i1> } @llvm.sadd.with.overflow.v32i16(<32 x i16> undef, <32 x i16> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call { i8, i1 } @llvm.sadd.with.overflow.i8(i8 undef, i8 undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V16I8 = call { <16 x i8>, <16 x i1> } @llvm.sadd.with.overflow.v16i8(<16 x i8> undef, <16 x i8> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %V32I8 = call { <32 x i8>, <32 x i1> } @llvm.sadd.with.overflow.v32i8(<32 x i8> undef, <32 x i8> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 50 for instruction: %V64I8 = call { <64 x i8>, <64 x i1> } @llvm.sadd.with.overflow.v64i8(<64 x i8> undef, <64 x i8> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX2-LABEL: 'sadd'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call { i64, i1 } @llvm.sadd.with.overflow.i64(i64 undef, i64 undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.sadd.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.sadd.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.sadd.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 undef, i32 undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.sadd.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.sadd.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V16I32 = call { <16 x i32>, <16 x i1> } @llvm.sadd.with.overflow.v16i32(<16 x i32> undef, <16 x i32> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call { i16, i1 } @llvm.sadd.with.overflow.i16(i16 undef, i16 undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V8I16 = call { <8 x i16>, <8 x i1> } @llvm.sadd.with.overflow.v8i16(<8 x i16> undef, <8 x i16> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V16I16 = call { <16 x i16>, <16 x i1> } @llvm.sadd.with.overflow.v16i16(<16 x i16> undef, <16 x i16> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V32I16 = call { <32 x i16>, <32 x i1> } @llvm.sadd.with.overflow.v32i16(<32 x i16> undef, <32 x i16> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call { i8, i1 } @llvm.sadd.with.overflow.i8(i8 undef, i8 undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V16I8 = call { <16 x i8>, <16 x i1> } @llvm.sadd.with.overflow.v16i8(<16 x i8> undef, <16 x i8> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V32I8 = call { <32 x i8>, <32 x i1> } @llvm.sadd.with.overflow.v32i8(<32 x i8> undef, <32 x i8> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V64I8 = call { <64 x i8>, <64 x i1> } @llvm.sadd.with.overflow.v64i8(<64 x i8> undef, <64 x i8> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX512F-LABEL: 'sadd'
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call { i64, i1 } @llvm.sadd.with.overflow.i64(i64 undef, i64 undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.sadd.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.sadd.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.sadd.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 undef, i32 undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.sadd.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.sadd.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V16I32 = call { <16 x i32>, <16 x i1> } @llvm.sadd.with.overflow.v16i32(<16 x i32> undef, <16 x i32> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call { i16, i1 } @llvm.sadd.with.overflow.i16(i16 undef, i16 undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V8I16 = call { <8 x i16>, <8 x i1> } @llvm.sadd.with.overflow.v8i16(<8 x i16> undef, <8 x i16> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V16I16 = call { <16 x i16>, <16 x i1> } @llvm.sadd.with.overflow.v16i16(<16 x i16> undef, <16 x i16> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V32I16 = call { <32 x i16>, <32 x i1> } @llvm.sadd.with.overflow.v32i16(<32 x i16> undef, <32 x i16> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call { i8, i1 } @llvm.sadd.with.overflow.i8(i8 undef, i8 undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V16I8 = call { <16 x i8>, <16 x i1> } @llvm.sadd.with.overflow.v16i8(<16 x i8> undef, <16 x i8> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V32I8 = call { <32 x i8>, <32 x i1> } @llvm.sadd.with.overflow.v32i8(<32 x i8> undef, <32 x i8> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V64I8 = call { <64 x i8>, <64 x i1> } @llvm.sadd.with.overflow.v64i8(<64 x i8> undef, <64 x i8> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX512BW-LABEL: 'sadd'
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call { i64, i1 } @llvm.sadd.with.overflow.i64(i64 undef, i64 undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.sadd.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.sadd.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.sadd.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 undef, i32 undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.sadd.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.sadd.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V16I32 = call { <16 x i32>, <16 x i1> } @llvm.sadd.with.overflow.v16i32(<16 x i32> undef, <16 x i32> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call { i16, i1 } @llvm.sadd.with.overflow.i16(i16 undef, i16 undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V8I16 = call { <8 x i16>, <8 x i1> } @llvm.sadd.with.overflow.v8i16(<8 x i16> undef, <8 x i16> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V16I16 = call { <16 x i16>, <16 x i1> } @llvm.sadd.with.overflow.v16i16(<16 x i16> undef, <16 x i16> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V32I16 = call { <32 x i16>, <32 x i1> } @llvm.sadd.with.overflow.v32i16(<32 x i16> undef, <32 x i16> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call { i8, i1 } @llvm.sadd.with.overflow.i8(i8 undef, i8 undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V16I8 = call { <16 x i8>, <16 x i1> } @llvm.sadd.with.overflow.v16i8(<16 x i8> undef, <16 x i8> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V32I8 = call { <32 x i8>, <32 x i1> } @llvm.sadd.with.overflow.v32i8(<32 x i8> undef, <32 x i8> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V64I8 = call { <64 x i8>, <64 x i1> } @llvm.sadd.with.overflow.v64i8(<64 x i8> undef, <64 x i8> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX512DQ-LABEL: 'sadd'
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call { i64, i1 } @llvm.sadd.with.overflow.i64(i64 undef, i64 undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.sadd.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.sadd.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.sadd.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 undef, i32 undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.sadd.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.sadd.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V16I32 = call { <16 x i32>, <16 x i1> } @llvm.sadd.with.overflow.v16i32(<16 x i32> undef, <16 x i32> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call { i16, i1 } @llvm.sadd.with.overflow.i16(i16 undef, i16 undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V8I16 = call { <8 x i16>, <8 x i1> } @llvm.sadd.with.overflow.v8i16(<8 x i16> undef, <8 x i16> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V16I16 = call { <16 x i16>, <16 x i1> } @llvm.sadd.with.overflow.v16i16(<16 x i16> undef, <16 x i16> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V32I16 = call { <32 x i16>, <32 x i1> } @llvm.sadd.with.overflow.v32i16(<32 x i16> undef, <32 x i16> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call { i8, i1 } @llvm.sadd.with.overflow.i8(i8 undef, i8 undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V16I8 = call { <16 x i8>, <16 x i1> } @llvm.sadd.with.overflow.v16i8(<16 x i8> undef, <16 x i8> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V32I8 = call { <32 x i8>, <32 x i1> } @llvm.sadd.with.overflow.v32i8(<32 x i8> undef, <32 x i8> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V64I8 = call { <64 x i8>, <64 x i1> } @llvm.sadd.with.overflow.v64i8(<64 x i8> undef, <64 x i8> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SLM-LABEL: 'sadd'
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call { i64, i1 } @llvm.sadd.with.overflow.i64(i64 undef, i64 undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.sadd.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.sadd.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.sadd.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 undef, i32 undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.sadd.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.sadd.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V16I32 = call { <16 x i32>, <16 x i1> } @llvm.sadd.with.overflow.v16i32(<16 x i32> undef, <16 x i32> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call { i16, i1 } @llvm.sadd.with.overflow.i16(i16 undef, i16 undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V8I16 = call { <8 x i16>, <8 x i1> } @llvm.sadd.with.overflow.v8i16(<8 x i16> undef, <8 x i16> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V16I16 = call { <16 x i16>, <16 x i1> } @llvm.sadd.with.overflow.v16i16(<16 x i16> undef, <16 x i16> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V32I16 = call { <32 x i16>, <32 x i1> } @llvm.sadd.with.overflow.v32i16(<32 x i16> undef, <32 x i16> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call { i8, i1 } @llvm.sadd.with.overflow.i8(i8 undef, i8 undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V16I8 = call { <16 x i8>, <16 x i1> } @llvm.sadd.with.overflow.v16i8(<16 x i8> undef, <16 x i8> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V32I8 = call { <32 x i8>, <32 x i1> } @llvm.sadd.with.overflow.v32i8(<32 x i8> undef, <32 x i8> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V64I8 = call { <64 x i8>, <64 x i1> } @llvm.sadd.with.overflow.v64i8(<64 x i8> undef, <64 x i8> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; GLM-LABEL: 'sadd'
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call { i64, i1 } @llvm.sadd.with.overflow.i64(i64 undef, i64 undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.sadd.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.sadd.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.sadd.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 undef, i32 undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.sadd.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.sadd.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V16I32 = call { <16 x i32>, <16 x i1> } @llvm.sadd.with.overflow.v16i32(<16 x i32> undef, <16 x i32> undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call { i16, i1 } @llvm.sadd.with.overflow.i16(i16 undef, i16 undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V8I16 = call { <8 x i16>, <8 x i1> } @llvm.sadd.with.overflow.v8i16(<8 x i16> undef, <8 x i16> undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V16I16 = call { <16 x i16>, <16 x i1> } @llvm.sadd.with.overflow.v16i16(<16 x i16> undef, <16 x i16> undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V32I16 = call { <32 x i16>, <32 x i1> } @llvm.sadd.with.overflow.v32i16(<32 x i16> undef, <32 x i16> undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call { i8, i1 } @llvm.sadd.with.overflow.i8(i8 undef, i8 undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V16I8 = call { <16 x i8>, <16 x i1> } @llvm.sadd.with.overflow.v16i8(<16 x i8> undef, <16 x i8> undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V32I8 = call { <32 x i8>, <32 x i1> } @llvm.sadd.with.overflow.v32i8(<32 x i8> undef, <32 x i8> undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V64I8 = call { <64 x i8>, <64 x i1> } @llvm.sadd.with.overflow.v64i8(<64 x i8> undef, <64 x i8> undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; BTVER2-LABEL: 'sadd'
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call { i64, i1 } @llvm.sadd.with.overflow.i64(i64 undef, i64 undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.sadd.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.sadd.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.sadd.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 undef, i32 undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.sadd.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.sadd.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %V16I32 = call { <16 x i32>, <16 x i1> } @llvm.sadd.with.overflow.v16i32(<16 x i32> undef, <16 x i32> undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call { i16, i1 } @llvm.sadd.with.overflow.i16(i16 undef, i16 undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V8I16 = call { <8 x i16>, <8 x i1> } @llvm.sadd.with.overflow.v8i16(<8 x i16> undef, <8 x i16> undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V16I16 = call { <16 x i16>, <16 x i1> } @llvm.sadd.with.overflow.v16i16(<16 x i16> undef, <16 x i16> undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 41 for instruction: %V32I16 = call { <32 x i16>, <32 x i1> } @llvm.sadd.with.overflow.v32i16(<32 x i16> undef, <32 x i16> undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call { i8, i1 } @llvm.sadd.with.overflow.i8(i8 undef, i8 undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V16I8 = call { <16 x i8>, <16 x i1> } @llvm.sadd.with.overflow.v16i8(<16 x i8> undef, <16 x i8> undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %V32I8 = call { <32 x i8>, <32 x i1> } @llvm.sadd.with.overflow.v32i8(<32 x i8> undef, <32 x i8> undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 50 for instruction: %V64I8 = call { <64 x i8>, <64 x i1> } @llvm.sadd.with.overflow.v64i8(<64 x i8> undef, <64 x i8> undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+  %I64 = call {i64, i1} @llvm.sadd.with.overflow.i64(i64 undef, i64 undef)
+  %V2I64 = call {<2 x i64>, <2 x i1>} @llvm.sadd.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
+  %V4I64 = call {<4 x i64>, <4 x i1>} @llvm.sadd.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
+  %V8I64 = call {<8 x i64>, <8 x i1>} @llvm.sadd.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
+
+  %I32 = call {i32, i1} @llvm.sadd.with.overflow.i32(i32 undef, i32 undef)
+  %V4I32  = call {<4 x i32>, <4 x i1>}  @llvm.sadd.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
+  %V8I32  = call {<8 x i32>, <8 x i1>}  @llvm.sadd.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
+  %V16I32 = call {<16 x i32>, <16 x i1>} @llvm.sadd.with.overflow.v16i32(<16 x i32> undef, <16 x i32> undef)
+
+  %I16 = call {i16, i1} @llvm.sadd.with.overflow.i16(i16 undef, i16 undef)
+  %V8I16  = call {<8 x i16>, <8 x i1>}  @llvm.sadd.with.overflow.v8i16(<8 x i16> undef, <8 x i16> undef)
+  %V16I16 = call {<16 x i16>, <16 x i1>} @llvm.sadd.with.overflow.v16i16(<16 x i16> undef, <16 x i16> undef)
+  %V32I16 = call {<32 x i16>, <32 x i1>} @llvm.sadd.with.overflow.v32i16(<32 x i16> undef, <32 x i16> undef)
+
+  %I8 = call {i8, i1} @llvm.sadd.with.overflow.i8(i8 undef, i8 undef)
+  %V16I8 = call {<16 x i8>, <16 x i1>} @llvm.sadd.with.overflow.v16i8(<16 x i8> undef, <16 x i8> undef)
+  %V32I8 = call {<32 x i8>, <32 x i1>} @llvm.sadd.with.overflow.v32i8(<32 x i8> undef, <32 x i8> undef)
+  %V64I8 = call {<64 x i8>, <64 x i1>} @llvm.sadd.with.overflow.v64i8(<64 x i8> undef, <64 x i8> undef)
+
+  ret i32 undef
+}
+
+;
+; uadd.with.overflow
+;
+
+declare {i64, i1}              @llvm.uadd.with.overflow.i64(i64, i64)
+declare {<2 x i64>, <2 x i1>}  @llvm.uadd.with.overflow.v2i64(<2 x i64>, <2 x i64>)
+declare {<4 x i64>, <4 x i1>}  @llvm.uadd.with.overflow.v4i64(<4 x i64>, <4 x i64>)
+declare {<8 x i64>, <8 x i1>}  @llvm.uadd.with.overflow.v8i64(<8 x i64>, <8 x i64>)
+
+declare {i32, i1}               @llvm.uadd.with.overflow.i32(i32, i32)
+declare {<4 x i32>, <4 x i1>}   @llvm.uadd.with.overflow.v4i32(<4 x i32>, <4 x i32>)
+declare {<8 x i32>, <8 x i1>}   @llvm.uadd.with.overflow.v8i32(<8 x i32>, <8 x i32>)
+declare {<16 x i32>, <16 x i1>} @llvm.uadd.with.overflow.v16i32(<16 x i32>, <16 x i32>)
+
+declare {i16, i1}               @llvm.uadd.with.overflow.i16(i16, i16)
+declare {<8 x i16>,  <8 x i1>}  @llvm.uadd.with.overflow.v8i16(<8 x i16>, <8 x i16>)
+declare {<16 x i16>, <16 x i1>} @llvm.uadd.with.overflow.v16i16(<16 x i16>, <16 x i16>)
+declare {<32 x i16>, <32 x i1>} @llvm.uadd.with.overflow.v32i16(<32 x i16>, <32 x i16>)
+
+declare {i8, i1}                @llvm.uadd.with.overflow.i8(i8, i8)
+declare {<16 x i8>, <16 x i1>}  @llvm.uadd.with.overflow.v16i8(<16 x i8>, <16 x i8>)
+declare {<32 x i8>, <32 x i1>}  @llvm.uadd.with.overflow.v32i8(<32 x i8>, <32 x i8>)
+declare {<64 x i8>, <64 x i1>}  @llvm.uadd.with.overflow.v64i8(<64 x i8>, <64 x i8>)
+
+define i32 @uadd(i32 %arg) {
+; SSSE3-LABEL: 'uadd'
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call { i64, i1 } @llvm.uadd.with.overflow.i64(i64 undef, i64 undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.uadd.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.uadd.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.uadd.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 undef, i32 undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.uadd.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.uadd.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I32 = call { <16 x i32>, <16 x i1> } @llvm.uadd.with.overflow.v16i32(<16 x i32> undef, <16 x i32> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call { i16, i1 } @llvm.uadd.with.overflow.i16(i16 undef, i16 undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = call { <8 x i16>, <8 x i1> } @llvm.uadd.with.overflow.v8i16(<8 x i16> undef, <8 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I16 = call { <16 x i16>, <16 x i1> } @llvm.uadd.with.overflow.v16i16(<16 x i16> undef, <16 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I16 = call { <32 x i16>, <32 x i1> } @llvm.uadd.with.overflow.v32i16(<32 x i16> undef, <32 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call { i8, i1 } @llvm.uadd.with.overflow.i8(i8 undef, i8 undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = call { <16 x i8>, <16 x i1> } @llvm.uadd.with.overflow.v16i8(<16 x i8> undef, <16 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I8 = call { <32 x i8>, <32 x i1> } @llvm.uadd.with.overflow.v32i8(<32 x i8> undef, <32 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I8 = call { <64 x i8>, <64 x i1> } @llvm.uadd.with.overflow.v64i8(<64 x i8> undef, <64 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSE42-LABEL: 'uadd'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call { i64, i1 } @llvm.uadd.with.overflow.i64(i64 undef, i64 undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.uadd.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.uadd.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.uadd.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 undef, i32 undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.uadd.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.uadd.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I32 = call { <16 x i32>, <16 x i1> } @llvm.uadd.with.overflow.v16i32(<16 x i32> undef, <16 x i32> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call { i16, i1 } @llvm.uadd.with.overflow.i16(i16 undef, i16 undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = call { <8 x i16>, <8 x i1> } @llvm.uadd.with.overflow.v8i16(<8 x i16> undef, <8 x i16> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I16 = call { <16 x i16>, <16 x i1> } @llvm.uadd.with.overflow.v16i16(<16 x i16> undef, <16 x i16> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I16 = call { <32 x i16>, <32 x i1> } @llvm.uadd.with.overflow.v32i16(<32 x i16> undef, <32 x i16> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call { i8, i1 } @llvm.uadd.with.overflow.i8(i8 undef, i8 undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = call { <16 x i8>, <16 x i1> } @llvm.uadd.with.overflow.v16i8(<16 x i8> undef, <16 x i8> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I8 = call { <32 x i8>, <32 x i1> } @llvm.uadd.with.overflow.v32i8(<32 x i8> undef, <32 x i8> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I8 = call { <64 x i8>, <64 x i1> } @llvm.uadd.with.overflow.v64i8(<64 x i8> undef, <64 x i8> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX1-LABEL: 'uadd'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call { i64, i1 } @llvm.uadd.with.overflow.i64(i64 undef, i64 undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.uadd.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.uadd.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.uadd.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 undef, i32 undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.uadd.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.uadd.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16I32 = call { <16 x i32>, <16 x i1> } @llvm.uadd.with.overflow.v16i32(<16 x i32> undef, <16 x i32> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call { i16, i1 } @llvm.uadd.with.overflow.i16(i16 undef, i16 undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = call { <8 x i16>, <8 x i1> } @llvm.uadd.with.overflow.v8i16(<8 x i16> undef, <8 x i16> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I16 = call { <16 x i16>, <16 x i1> } @llvm.uadd.with.overflow.v16i16(<16 x i16> undef, <16 x i16> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V32I16 = call { <32 x i16>, <32 x i1> } @llvm.uadd.with.overflow.v32i16(<32 x i16> undef, <32 x i16> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call { i8, i1 } @llvm.uadd.with.overflow.i8(i8 undef, i8 undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = call { <16 x i8>, <16 x i1> } @llvm.uadd.with.overflow.v16i8(<16 x i8> undef, <16 x i8> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I8 = call { <32 x i8>, <32 x i1> } @llvm.uadd.with.overflow.v32i8(<32 x i8> undef, <32 x i8> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V64I8 = call { <64 x i8>, <64 x i1> } @llvm.uadd.with.overflow.v64i8(<64 x i8> undef, <64 x i8> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX2-LABEL: 'uadd'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call { i64, i1 } @llvm.uadd.with.overflow.i64(i64 undef, i64 undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.uadd.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.uadd.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.uadd.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 undef, i32 undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.uadd.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.uadd.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I32 = call { <16 x i32>, <16 x i1> } @llvm.uadd.with.overflow.v16i32(<16 x i32> undef, <16 x i32> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call { i16, i1 } @llvm.uadd.with.overflow.i16(i16 undef, i16 undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = call { <8 x i16>, <8 x i1> } @llvm.uadd.with.overflow.v8i16(<8 x i16> undef, <8 x i16> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = call { <16 x i16>, <16 x i1> } @llvm.uadd.with.overflow.v16i16(<16 x i16> undef, <16 x i16> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = call { <32 x i16>, <32 x i1> } @llvm.uadd.with.overflow.v32i16(<32 x i16> undef, <32 x i16> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call { i8, i1 } @llvm.uadd.with.overflow.i8(i8 undef, i8 undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = call { <16 x i8>, <16 x i1> } @llvm.uadd.with.overflow.v16i8(<16 x i8> undef, <16 x i8> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I8 = call { <32 x i8>, <32 x i1> } @llvm.uadd.with.overflow.v32i8(<32 x i8> undef, <32 x i8> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = call { <64 x i8>, <64 x i1> } @llvm.uadd.with.overflow.v64i8(<64 x i8> undef, <64 x i8> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX512F-LABEL: 'uadd'
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call { i64, i1 } @llvm.uadd.with.overflow.i64(i64 undef, i64 undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.uadd.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.uadd.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.uadd.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 undef, i32 undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.uadd.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.uadd.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I32 = call { <16 x i32>, <16 x i1> } @llvm.uadd.with.overflow.v16i32(<16 x i32> undef, <16 x i32> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call { i16, i1 } @llvm.uadd.with.overflow.i16(i16 undef, i16 undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = call { <8 x i16>, <8 x i1> } @llvm.uadd.with.overflow.v8i16(<8 x i16> undef, <8 x i16> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = call { <16 x i16>, <16 x i1> } @llvm.uadd.with.overflow.v16i16(<16 x i16> undef, <16 x i16> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = call { <32 x i16>, <32 x i1> } @llvm.uadd.with.overflow.v32i16(<32 x i16> undef, <32 x i16> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call { i8, i1 } @llvm.uadd.with.overflow.i8(i8 undef, i8 undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = call { <16 x i8>, <16 x i1> } @llvm.uadd.with.overflow.v16i8(<16 x i8> undef, <16 x i8> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I8 = call { <32 x i8>, <32 x i1> } @llvm.uadd.with.overflow.v32i8(<32 x i8> undef, <32 x i8> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = call { <64 x i8>, <64 x i1> } @llvm.uadd.with.overflow.v64i8(<64 x i8> undef, <64 x i8> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX512BW-LABEL: 'uadd'
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call { i64, i1 } @llvm.uadd.with.overflow.i64(i64 undef, i64 undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.uadd.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.uadd.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.uadd.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 undef, i32 undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.uadd.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.uadd.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I32 = call { <16 x i32>, <16 x i1> } @llvm.uadd.with.overflow.v16i32(<16 x i32> undef, <16 x i32> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call { i16, i1 } @llvm.uadd.with.overflow.i16(i16 undef, i16 undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = call { <8 x i16>, <8 x i1> } @llvm.uadd.with.overflow.v8i16(<8 x i16> undef, <8 x i16> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = call { <16 x i16>, <16 x i1> } @llvm.uadd.with.overflow.v16i16(<16 x i16> undef, <16 x i16> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I16 = call { <32 x i16>, <32 x i1> } @llvm.uadd.with.overflow.v32i16(<32 x i16> undef, <32 x i16> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call { i8, i1 } @llvm.uadd.with.overflow.i8(i8 undef, i8 undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = call { <16 x i8>, <16 x i1> } @llvm.uadd.with.overflow.v16i8(<16 x i8> undef, <16 x i8> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I8 = call { <32 x i8>, <32 x i1> } @llvm.uadd.with.overflow.v32i8(<32 x i8> undef, <32 x i8> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64I8 = call { <64 x i8>, <64 x i1> } @llvm.uadd.with.overflow.v64i8(<64 x i8> undef, <64 x i8> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX512DQ-LABEL: 'uadd'
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call { i64, i1 } @llvm.uadd.with.overflow.i64(i64 undef, i64 undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.uadd.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.uadd.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.uadd.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 undef, i32 undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.uadd.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.uadd.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I32 = call { <16 x i32>, <16 x i1> } @llvm.uadd.with.overflow.v16i32(<16 x i32> undef, <16 x i32> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call { i16, i1 } @llvm.uadd.with.overflow.i16(i16 undef, i16 undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = call { <8 x i16>, <8 x i1> } @llvm.uadd.with.overflow.v8i16(<8 x i16> undef, <8 x i16> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = call { <16 x i16>, <16 x i1> } @llvm.uadd.with.overflow.v16i16(<16 x i16> undef, <16 x i16> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = call { <32 x i16>, <32 x i1> } @llvm.uadd.with.overflow.v32i16(<32 x i16> undef, <32 x i16> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call { i8, i1 } @llvm.uadd.with.overflow.i8(i8 undef, i8 undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = call { <16 x i8>, <16 x i1> } @llvm.uadd.with.overflow.v16i8(<16 x i8> undef, <16 x i8> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I8 = call { <32 x i8>, <32 x i1> } @llvm.uadd.with.overflow.v32i8(<32 x i8> undef, <32 x i8> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = call { <64 x i8>, <64 x i1> } @llvm.uadd.with.overflow.v64i8(<64 x i8> undef, <64 x i8> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SLM-LABEL: 'uadd'
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call { i64, i1 } @llvm.uadd.with.overflow.i64(i64 undef, i64 undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.uadd.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.uadd.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.uadd.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 undef, i32 undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.uadd.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.uadd.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I32 = call { <16 x i32>, <16 x i1> } @llvm.uadd.with.overflow.v16i32(<16 x i32> undef, <16 x i32> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call { i16, i1 } @llvm.uadd.with.overflow.i16(i16 undef, i16 undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = call { <8 x i16>, <8 x i1> } @llvm.uadd.with.overflow.v8i16(<8 x i16> undef, <8 x i16> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I16 = call { <16 x i16>, <16 x i1> } @llvm.uadd.with.overflow.v16i16(<16 x i16> undef, <16 x i16> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I16 = call { <32 x i16>, <32 x i1> } @llvm.uadd.with.overflow.v32i16(<32 x i16> undef, <32 x i16> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call { i8, i1 } @llvm.uadd.with.overflow.i8(i8 undef, i8 undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = call { <16 x i8>, <16 x i1> } @llvm.uadd.with.overflow.v16i8(<16 x i8> undef, <16 x i8> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I8 = call { <32 x i8>, <32 x i1> } @llvm.uadd.with.overflow.v32i8(<32 x i8> undef, <32 x i8> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I8 = call { <64 x i8>, <64 x i1> } @llvm.uadd.with.overflow.v64i8(<64 x i8> undef, <64 x i8> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; GLM-LABEL: 'uadd'
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call { i64, i1 } @llvm.uadd.with.overflow.i64(i64 undef, i64 undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.uadd.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.uadd.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.uadd.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 undef, i32 undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.uadd.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.uadd.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I32 = call { <16 x i32>, <16 x i1> } @llvm.uadd.with.overflow.v16i32(<16 x i32> undef, <16 x i32> undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call { i16, i1 } @llvm.uadd.with.overflow.i16(i16 undef, i16 undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = call { <8 x i16>, <8 x i1> } @llvm.uadd.with.overflow.v8i16(<8 x i16> undef, <8 x i16> undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I16 = call { <16 x i16>, <16 x i1> } @llvm.uadd.with.overflow.v16i16(<16 x i16> undef, <16 x i16> undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I16 = call { <32 x i16>, <32 x i1> } @llvm.uadd.with.overflow.v32i16(<32 x i16> undef, <32 x i16> undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call { i8, i1 } @llvm.uadd.with.overflow.i8(i8 undef, i8 undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = call { <16 x i8>, <16 x i1> } @llvm.uadd.with.overflow.v16i8(<16 x i8> undef, <16 x i8> undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I8 = call { <32 x i8>, <32 x i1> } @llvm.uadd.with.overflow.v32i8(<32 x i8> undef, <32 x i8> undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I8 = call { <64 x i8>, <64 x i1> } @llvm.uadd.with.overflow.v64i8(<64 x i8> undef, <64 x i8> undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; BTVER2-LABEL: 'uadd'
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call { i64, i1 } @llvm.uadd.with.overflow.i64(i64 undef, i64 undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.uadd.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.uadd.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.uadd.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 undef, i32 undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.uadd.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.uadd.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16I32 = call { <16 x i32>, <16 x i1> } @llvm.uadd.with.overflow.v16i32(<16 x i32> undef, <16 x i32> undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call { i16, i1 } @llvm.uadd.with.overflow.i16(i16 undef, i16 undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = call { <8 x i16>, <8 x i1> } @llvm.uadd.with.overflow.v8i16(<8 x i16> undef, <8 x i16> undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I16 = call { <16 x i16>, <16 x i1> } @llvm.uadd.with.overflow.v16i16(<16 x i16> undef, <16 x i16> undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V32I16 = call { <32 x i16>, <32 x i1> } @llvm.uadd.with.overflow.v32i16(<32 x i16> undef, <32 x i16> undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call { i8, i1 } @llvm.uadd.with.overflow.i8(i8 undef, i8 undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = call { <16 x i8>, <16 x i1> } @llvm.uadd.with.overflow.v16i8(<16 x i8> undef, <16 x i8> undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I8 = call { <32 x i8>, <32 x i1> } @llvm.uadd.with.overflow.v32i8(<32 x i8> undef, <32 x i8> undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V64I8 = call { <64 x i8>, <64 x i1> } @llvm.uadd.with.overflow.v64i8(<64 x i8> undef, <64 x i8> undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+  %I64 = call {i64, i1} @llvm.uadd.with.overflow.i64(i64 undef, i64 undef)
+  %V2I64 = call {<2 x i64>, <2 x i1>} @llvm.uadd.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
+  %V4I64 = call {<4 x i64>, <4 x i1>} @llvm.uadd.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
+  %V8I64 = call {<8 x i64>, <8 x i1>} @llvm.uadd.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
+
+  %I32 = call {i32, i1} @llvm.uadd.with.overflow.i32(i32 undef, i32 undef)
+  %V4I32  = call {<4 x i32>, <4 x i1>}  @llvm.uadd.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
+  %V8I32  = call {<8 x i32>, <8 x i1>}  @llvm.uadd.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
+  %V16I32 = call {<16 x i32>, <16 x i1>} @llvm.uadd.with.overflow.v16i32(<16 x i32> undef, <16 x i32> undef)
+
+  %I16 = call {i16, i1} @llvm.uadd.with.overflow.i16(i16 undef, i16 undef)
+  %V8I16  = call {<8 x i16>, <8 x i1>}  @llvm.uadd.with.overflow.v8i16(<8 x i16> undef, <8 x i16> undef)
+  %V16I16 = call {<16 x i16>, <16 x i1>} @llvm.uadd.with.overflow.v16i16(<16 x i16> undef, <16 x i16> undef)
+  %V32I16 = call {<32 x i16>, <32 x i1>} @llvm.uadd.with.overflow.v32i16(<32 x i16> undef, <32 x i16> undef)
+
+  %I8 = call {i8, i1} @llvm.uadd.with.overflow.i8(i8 undef, i8 undef)
+  %V16I8 = call {<16 x i8>, <16 x i1>} @llvm.uadd.with.overflow.v16i8(<16 x i8> undef, <16 x i8> undef)
+  %V32I8 = call {<32 x i8>, <32 x i1>} @llvm.uadd.with.overflow.v32i8(<32 x i8> undef, <32 x i8> undef)
+  %V64I8 = call {<64 x i8>, <64 x i1>} @llvm.uadd.with.overflow.v64i8(<64 x i8> undef, <64 x i8> undef)
+
+  ret i32 undef
+}
+
+;
+; ssub.with.overflow
+;
+
+declare {i64, i1}              @llvm.ssub.with.overflow.i64(i64, i64)
+declare {<2 x i64>, <2 x i1>}  @llvm.ssub.with.overflow.v2i64(<2 x i64>, <2 x i64>)
+declare {<4 x i64>, <4 x i1>}  @llvm.ssub.with.overflow.v4i64(<4 x i64>, <4 x i64>)
+declare {<8 x i64>, <8 x i1>}  @llvm.ssub.with.overflow.v8i64(<8 x i64>, <8 x i64>)
+
+declare {i32, i1}               @llvm.ssub.with.overflow.i32(i32, i32)
+declare {<4 x i32>, <4 x i1>}   @llvm.ssub.with.overflow.v4i32(<4 x i32>, <4 x i32>)
+declare {<8 x i32>, <8 x i1>}   @llvm.ssub.with.overflow.v8i32(<8 x i32>, <8 x i32>)
+declare {<16 x i32>, <16 x i1>} @llvm.ssub.with.overflow.v16i32(<16 x i32>, <16 x i32>)
+
+declare {i16, i1}               @llvm.ssub.with.overflow.i16(i16, i16)
+declare {<8 x i16>,  <8 x i1>}  @llvm.ssub.with.overflow.v8i16(<8 x i16>, <8 x i16>)
+declare {<16 x i16>, <16 x i1>} @llvm.ssub.with.overflow.v16i16(<16 x i16>, <16 x i16>)
+declare {<32 x i16>, <32 x i1>} @llvm.ssub.with.overflow.v32i16(<32 x i16>, <32 x i16>)
+
+declare {i8, i1}                @llvm.ssub.with.overflow.i8(i8, i8)
+declare {<16 x i8>, <16 x i1>}  @llvm.ssub.with.overflow.v16i8(<16 x i8>, <16 x i8>)
+declare {<32 x i8>, <32 x i1>}  @llvm.ssub.with.overflow.v32i8(<32 x i8>, <32 x i8>)
+declare {<64 x i8>, <64 x i1>}  @llvm.ssub.with.overflow.v64i8(<64 x i8>, <64 x i8>)
+
+define i32 @ssub(i32 %arg) {
+; SSSE3-LABEL: 'ssub'
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call { i64, i1 } @llvm.ssub.with.overflow.i64(i64 undef, i64 undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.ssub.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 53 for instruction: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.ssub.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 103 for instruction: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.ssub.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call { i32, i1 } @llvm.ssub.with.overflow.i32(i32 undef, i32 undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.ssub.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.ssub.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V16I32 = call { <16 x i32>, <16 x i1> } @llvm.ssub.with.overflow.v16i32(<16 x i32> undef, <16 x i32> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call { i16, i1 } @llvm.ssub.with.overflow.i16(i16 undef, i16 undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V8I16 = call { <8 x i16>, <8 x i1> } @llvm.ssub.with.overflow.v8i16(<8 x i16> undef, <8 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V16I16 = call { <16 x i16>, <16 x i1> } @llvm.ssub.with.overflow.v16i16(<16 x i16> undef, <16 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V32I16 = call { <32 x i16>, <32 x i1> } @llvm.ssub.with.overflow.v32i16(<32 x i16> undef, <32 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call { i8, i1 } @llvm.ssub.with.overflow.i8(i8 undef, i8 undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V16I8 = call { <16 x i8>, <16 x i1> } @llvm.ssub.with.overflow.v16i8(<16 x i8> undef, <16 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V32I8 = call { <32 x i8>, <32 x i1> } @llvm.ssub.with.overflow.v32i8(<32 x i8> undef, <32 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V64I8 = call { <64 x i8>, <64 x i1> } @llvm.ssub.with.overflow.v64i8(<64 x i8> undef, <64 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSE42-LABEL: 'ssub'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call { i64, i1 } @llvm.ssub.with.overflow.i64(i64 undef, i64 undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.ssub.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.ssub.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.ssub.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call { i32, i1 } @llvm.ssub.with.overflow.i32(i32 undef, i32 undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.ssub.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.ssub.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V16I32 = call { <16 x i32>, <16 x i1> } @llvm.ssub.with.overflow.v16i32(<16 x i32> undef, <16 x i32> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call { i16, i1 } @llvm.ssub.with.overflow.i16(i16 undef, i16 undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V8I16 = call { <8 x i16>, <8 x i1> } @llvm.ssub.with.overflow.v8i16(<8 x i16> undef, <8 x i16> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V16I16 = call { <16 x i16>, <16 x i1> } @llvm.ssub.with.overflow.v16i16(<16 x i16> undef, <16 x i16> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V32I16 = call { <32 x i16>, <32 x i1> } @llvm.ssub.with.overflow.v32i16(<32 x i16> undef, <32 x i16> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call { i8, i1 } @llvm.ssub.with.overflow.i8(i8 undef, i8 undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V16I8 = call { <16 x i8>, <16 x i1> } @llvm.ssub.with.overflow.v16i8(<16 x i8> undef, <16 x i8> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V32I8 = call { <32 x i8>, <32 x i1> } @llvm.ssub.with.overflow.v32i8(<32 x i8> undef, <32 x i8> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V64I8 = call { <64 x i8>, <64 x i1> } @llvm.ssub.with.overflow.v64i8(<64 x i8> undef, <64 x i8> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX1-LABEL: 'ssub'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call { i64, i1 } @llvm.ssub.with.overflow.i64(i64 undef, i64 undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.ssub.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.ssub.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.ssub.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call { i32, i1 } @llvm.ssub.with.overflow.i32(i32 undef, i32 undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.ssub.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.ssub.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %V16I32 = call { <16 x i32>, <16 x i1> } @llvm.ssub.with.overflow.v16i32(<16 x i32> undef, <16 x i32> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call { i16, i1 } @llvm.ssub.with.overflow.i16(i16 undef, i16 undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V8I16 = call { <8 x i16>, <8 x i1> } @llvm.ssub.with.overflow.v8i16(<8 x i16> undef, <8 x i16> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V16I16 = call { <16 x i16>, <16 x i1> } @llvm.ssub.with.overflow.v16i16(<16 x i16> undef, <16 x i16> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 41 for instruction: %V32I16 = call { <32 x i16>, <32 x i1> } @llvm.ssub.with.overflow.v32i16(<32 x i16> undef, <32 x i16> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call { i8, i1 } @llvm.ssub.with.overflow.i8(i8 undef, i8 undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V16I8 = call { <16 x i8>, <16 x i1> } @llvm.ssub.with.overflow.v16i8(<16 x i8> undef, <16 x i8> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %V32I8 = call { <32 x i8>, <32 x i1> } @llvm.ssub.with.overflow.v32i8(<32 x i8> undef, <32 x i8> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 50 for instruction: %V64I8 = call { <64 x i8>, <64 x i1> } @llvm.ssub.with.overflow.v64i8(<64 x i8> undef, <64 x i8> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX2-LABEL: 'ssub'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call { i64, i1 } @llvm.ssub.with.overflow.i64(i64 undef, i64 undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.ssub.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.ssub.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.ssub.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call { i32, i1 } @llvm.ssub.with.overflow.i32(i32 undef, i32 undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.ssub.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.ssub.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V16I32 = call { <16 x i32>, <16 x i1> } @llvm.ssub.with.overflow.v16i32(<16 x i32> undef, <16 x i32> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call { i16, i1 } @llvm.ssub.with.overflow.i16(i16 undef, i16 undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V8I16 = call { <8 x i16>, <8 x i1> } @llvm.ssub.with.overflow.v8i16(<8 x i16> undef, <8 x i16> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V16I16 = call { <16 x i16>, <16 x i1> } @llvm.ssub.with.overflow.v16i16(<16 x i16> undef, <16 x i16> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V32I16 = call { <32 x i16>, <32 x i1> } @llvm.ssub.with.overflow.v32i16(<32 x i16> undef, <32 x i16> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call { i8, i1 } @llvm.ssub.with.overflow.i8(i8 undef, i8 undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V16I8 = call { <16 x i8>, <16 x i1> } @llvm.ssub.with.overflow.v16i8(<16 x i8> undef, <16 x i8> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V32I8 = call { <32 x i8>, <32 x i1> } @llvm.ssub.with.overflow.v32i8(<32 x i8> undef, <32 x i8> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V64I8 = call { <64 x i8>, <64 x i1> } @llvm.ssub.with.overflow.v64i8(<64 x i8> undef, <64 x i8> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX512F-LABEL: 'ssub'
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call { i64, i1 } @llvm.ssub.with.overflow.i64(i64 undef, i64 undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.ssub.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.ssub.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.ssub.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call { i32, i1 } @llvm.ssub.with.overflow.i32(i32 undef, i32 undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.ssub.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.ssub.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V16I32 = call { <16 x i32>, <16 x i1> } @llvm.ssub.with.overflow.v16i32(<16 x i32> undef, <16 x i32> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call { i16, i1 } @llvm.ssub.with.overflow.i16(i16 undef, i16 undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V8I16 = call { <8 x i16>, <8 x i1> } @llvm.ssub.with.overflow.v8i16(<8 x i16> undef, <8 x i16> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V16I16 = call { <16 x i16>, <16 x i1> } @llvm.ssub.with.overflow.v16i16(<16 x i16> undef, <16 x i16> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V32I16 = call { <32 x i16>, <32 x i1> } @llvm.ssub.with.overflow.v32i16(<32 x i16> undef, <32 x i16> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call { i8, i1 } @llvm.ssub.with.overflow.i8(i8 undef, i8 undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V16I8 = call { <16 x i8>, <16 x i1> } @llvm.ssub.with.overflow.v16i8(<16 x i8> undef, <16 x i8> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V32I8 = call { <32 x i8>, <32 x i1> } @llvm.ssub.with.overflow.v32i8(<32 x i8> undef, <32 x i8> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V64I8 = call { <64 x i8>, <64 x i1> } @llvm.ssub.with.overflow.v64i8(<64 x i8> undef, <64 x i8> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX512BW-LABEL: 'ssub'
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call { i64, i1 } @llvm.ssub.with.overflow.i64(i64 undef, i64 undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.ssub.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.ssub.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.ssub.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call { i32, i1 } @llvm.ssub.with.overflow.i32(i32 undef, i32 undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.ssub.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.ssub.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V16I32 = call { <16 x i32>, <16 x i1> } @llvm.ssub.with.overflow.v16i32(<16 x i32> undef, <16 x i32> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call { i16, i1 } @llvm.ssub.with.overflow.i16(i16 undef, i16 undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V8I16 = call { <8 x i16>, <8 x i1> } @llvm.ssub.with.overflow.v8i16(<8 x i16> undef, <8 x i16> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V16I16 = call { <16 x i16>, <16 x i1> } @llvm.ssub.with.overflow.v16i16(<16 x i16> undef, <16 x i16> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V32I16 = call { <32 x i16>, <32 x i1> } @llvm.ssub.with.overflow.v32i16(<32 x i16> undef, <32 x i16> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call { i8, i1 } @llvm.ssub.with.overflow.i8(i8 undef, i8 undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V16I8 = call { <16 x i8>, <16 x i1> } @llvm.ssub.with.overflow.v16i8(<16 x i8> undef, <16 x i8> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V32I8 = call { <32 x i8>, <32 x i1> } @llvm.ssub.with.overflow.v32i8(<32 x i8> undef, <32 x i8> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V64I8 = call { <64 x i8>, <64 x i1> } @llvm.ssub.with.overflow.v64i8(<64 x i8> undef, <64 x i8> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX512DQ-LABEL: 'ssub'
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call { i64, i1 } @llvm.ssub.with.overflow.i64(i64 undef, i64 undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.ssub.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.ssub.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.ssub.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call { i32, i1 } @llvm.ssub.with.overflow.i32(i32 undef, i32 undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.ssub.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.ssub.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V16I32 = call { <16 x i32>, <16 x i1> } @llvm.ssub.with.overflow.v16i32(<16 x i32> undef, <16 x i32> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call { i16, i1 } @llvm.ssub.with.overflow.i16(i16 undef, i16 undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V8I16 = call { <8 x i16>, <8 x i1> } @llvm.ssub.with.overflow.v8i16(<8 x i16> undef, <8 x i16> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V16I16 = call { <16 x i16>, <16 x i1> } @llvm.ssub.with.overflow.v16i16(<16 x i16> undef, <16 x i16> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V32I16 = call { <32 x i16>, <32 x i1> } @llvm.ssub.with.overflow.v32i16(<32 x i16> undef, <32 x i16> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call { i8, i1 } @llvm.ssub.with.overflow.i8(i8 undef, i8 undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V16I8 = call { <16 x i8>, <16 x i1> } @llvm.ssub.with.overflow.v16i8(<16 x i8> undef, <16 x i8> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V32I8 = call { <32 x i8>, <32 x i1> } @llvm.ssub.with.overflow.v32i8(<32 x i8> undef, <32 x i8> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V64I8 = call { <64 x i8>, <64 x i1> } @llvm.ssub.with.overflow.v64i8(<64 x i8> undef, <64 x i8> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SLM-LABEL: 'ssub'
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call { i64, i1 } @llvm.ssub.with.overflow.i64(i64 undef, i64 undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.ssub.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.ssub.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.ssub.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call { i32, i1 } @llvm.ssub.with.overflow.i32(i32 undef, i32 undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.ssub.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.ssub.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V16I32 = call { <16 x i32>, <16 x i1> } @llvm.ssub.with.overflow.v16i32(<16 x i32> undef, <16 x i32> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call { i16, i1 } @llvm.ssub.with.overflow.i16(i16 undef, i16 undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V8I16 = call { <8 x i16>, <8 x i1> } @llvm.ssub.with.overflow.v8i16(<8 x i16> undef, <8 x i16> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V16I16 = call { <16 x i16>, <16 x i1> } @llvm.ssub.with.overflow.v16i16(<16 x i16> undef, <16 x i16> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V32I16 = call { <32 x i16>, <32 x i1> } @llvm.ssub.with.overflow.v32i16(<32 x i16> undef, <32 x i16> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call { i8, i1 } @llvm.ssub.with.overflow.i8(i8 undef, i8 undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V16I8 = call { <16 x i8>, <16 x i1> } @llvm.ssub.with.overflow.v16i8(<16 x i8> undef, <16 x i8> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V32I8 = call { <32 x i8>, <32 x i1> } @llvm.ssub.with.overflow.v32i8(<32 x i8> undef, <32 x i8> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V64I8 = call { <64 x i8>, <64 x i1> } @llvm.ssub.with.overflow.v64i8(<64 x i8> undef, <64 x i8> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; GLM-LABEL: 'ssub'
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call { i64, i1 } @llvm.ssub.with.overflow.i64(i64 undef, i64 undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.ssub.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.ssub.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.ssub.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call { i32, i1 } @llvm.ssub.with.overflow.i32(i32 undef, i32 undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.ssub.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.ssub.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V16I32 = call { <16 x i32>, <16 x i1> } @llvm.ssub.with.overflow.v16i32(<16 x i32> undef, <16 x i32> undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call { i16, i1 } @llvm.ssub.with.overflow.i16(i16 undef, i16 undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V8I16 = call { <8 x i16>, <8 x i1> } @llvm.ssub.with.overflow.v8i16(<8 x i16> undef, <8 x i16> undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V16I16 = call { <16 x i16>, <16 x i1> } @llvm.ssub.with.overflow.v16i16(<16 x i16> undef, <16 x i16> undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V32I16 = call { <32 x i16>, <32 x i1> } @llvm.ssub.with.overflow.v32i16(<32 x i16> undef, <32 x i16> undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call { i8, i1 } @llvm.ssub.with.overflow.i8(i8 undef, i8 undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V16I8 = call { <16 x i8>, <16 x i1> } @llvm.ssub.with.overflow.v16i8(<16 x i8> undef, <16 x i8> undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V32I8 = call { <32 x i8>, <32 x i1> } @llvm.ssub.with.overflow.v32i8(<32 x i8> undef, <32 x i8> undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V64I8 = call { <64 x i8>, <64 x i1> } @llvm.ssub.with.overflow.v64i8(<64 x i8> undef, <64 x i8> undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; BTVER2-LABEL: 'ssub'
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call { i64, i1 } @llvm.ssub.with.overflow.i64(i64 undef, i64 undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.ssub.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.ssub.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.ssub.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call { i32, i1 } @llvm.ssub.with.overflow.i32(i32 undef, i32 undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.ssub.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.ssub.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %V16I32 = call { <16 x i32>, <16 x i1> } @llvm.ssub.with.overflow.v16i32(<16 x i32> undef, <16 x i32> undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call { i16, i1 } @llvm.ssub.with.overflow.i16(i16 undef, i16 undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V8I16 = call { <8 x i16>, <8 x i1> } @llvm.ssub.with.overflow.v8i16(<8 x i16> undef, <8 x i16> undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V16I16 = call { <16 x i16>, <16 x i1> } @llvm.ssub.with.overflow.v16i16(<16 x i16> undef, <16 x i16> undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 41 for instruction: %V32I16 = call { <32 x i16>, <32 x i1> } @llvm.ssub.with.overflow.v32i16(<32 x i16> undef, <32 x i16> undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call { i8, i1 } @llvm.ssub.with.overflow.i8(i8 undef, i8 undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V16I8 = call { <16 x i8>, <16 x i1> } @llvm.ssub.with.overflow.v16i8(<16 x i8> undef, <16 x i8> undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %V32I8 = call { <32 x i8>, <32 x i1> } @llvm.ssub.with.overflow.v32i8(<32 x i8> undef, <32 x i8> undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 50 for instruction: %V64I8 = call { <64 x i8>, <64 x i1> } @llvm.ssub.with.overflow.v64i8(<64 x i8> undef, <64 x i8> undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+  %I64 = call {i64, i1} @llvm.ssub.with.overflow.i64(i64 undef, i64 undef)
+  %V2I64 = call {<2 x i64>, <2 x i1>} @llvm.ssub.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
+  %V4I64 = call {<4 x i64>, <4 x i1>} @llvm.ssub.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
+  %V8I64 = call {<8 x i64>, <8 x i1>} @llvm.ssub.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
+
+  %I32 = call {i32, i1} @llvm.ssub.with.overflow.i32(i32 undef, i32 undef)
+  %V4I32  = call {<4 x i32>, <4 x i1>}  @llvm.ssub.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
+  %V8I32  = call {<8 x i32>, <8 x i1>}  @llvm.ssub.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
+  %V16I32 = call {<16 x i32>, <16 x i1>} @llvm.ssub.with.overflow.v16i32(<16 x i32> undef, <16 x i32> undef)
+
+  %I16 = call {i16, i1} @llvm.ssub.with.overflow.i16(i16 undef, i16 undef)
+  %V8I16  = call {<8 x i16>, <8 x i1>}  @llvm.ssub.with.overflow.v8i16(<8 x i16> undef, <8 x i16> undef)
+  %V16I16 = call {<16 x i16>, <16 x i1>} @llvm.ssub.with.overflow.v16i16(<16 x i16> undef, <16 x i16> undef)
+  %V32I16 = call {<32 x i16>, <32 x i1>} @llvm.ssub.with.overflow.v32i16(<32 x i16> undef, <32 x i16> undef)
+
+  %I8 = call {i8, i1} @llvm.ssub.with.overflow.i8(i8 undef, i8 undef)
+  %V16I8 = call {<16 x i8>, <16 x i1>} @llvm.ssub.with.overflow.v16i8(<16 x i8> undef, <16 x i8> undef)
+  %V32I8 = call {<32 x i8>, <32 x i1>} @llvm.ssub.with.overflow.v32i8(<32 x i8> undef, <32 x i8> undef)
+  %V64I8 = call {<64 x i8>, <64 x i1>} @llvm.ssub.with.overflow.v64i8(<64 x i8> undef, <64 x i8> undef)
+
+  ret i32 undef
+}
+
+;
+; usub.with.overflow
+;
+
+declare {i64, i1}              @llvm.usub.with.overflow.i64(i64, i64)
+declare {<2 x i64>, <2 x i1>}  @llvm.usub.with.overflow.v2i64(<2 x i64>, <2 x i64>)
+declare {<4 x i64>, <4 x i1>}  @llvm.usub.with.overflow.v4i64(<4 x i64>, <4 x i64>)
+declare {<8 x i64>, <8 x i1>}  @llvm.usub.with.overflow.v8i64(<8 x i64>, <8 x i64>)
+
+declare {i32, i1}               @llvm.usub.with.overflow.i32(i32, i32)
+declare {<4 x i32>, <4 x i1>}   @llvm.usub.with.overflow.v4i32(<4 x i32>, <4 x i32>)
+declare {<8 x i32>, <8 x i1>}   @llvm.usub.with.overflow.v8i32(<8 x i32>, <8 x i32>)
+declare {<16 x i32>, <16 x i1>} @llvm.usub.with.overflow.v16i32(<16 x i32>, <16 x i32>)
+
+declare {i16, i1}               @llvm.usub.with.overflow.i16(i16, i16)
+declare {<8 x i16>,  <8 x i1>}  @llvm.usub.with.overflow.v8i16(<8 x i16>, <8 x i16>)
+declare {<16 x i16>, <16 x i1>} @llvm.usub.with.overflow.v16i16(<16 x i16>, <16 x i16>)
+declare {<32 x i16>, <32 x i1>} @llvm.usub.with.overflow.v32i16(<32 x i16>, <32 x i16>)
+
+declare {i8, i1}                @llvm.usub.with.overflow.i8(i8, i8)
+declare {<16 x i8>, <16 x i1>}  @llvm.usub.with.overflow.v16i8(<16 x i8>, <16 x i8>)
+declare {<32 x i8>, <32 x i1>}  @llvm.usub.with.overflow.v32i8(<32 x i8>, <32 x i8>)
+declare {<64 x i8>, <64 x i1>}  @llvm.usub.with.overflow.v64i8(<64 x i8>, <64 x i8>)
+
+define i32 @usub(i32 %arg) {
+; SSSE3-LABEL: 'usub'
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call { i64, i1 } @llvm.usub.with.overflow.i64(i64 undef, i64 undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.usub.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.usub.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.usub.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call { i32, i1 } @llvm.usub.with.overflow.i32(i32 undef, i32 undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.usub.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.usub.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I32 = call { <16 x i32>, <16 x i1> } @llvm.usub.with.overflow.v16i32(<16 x i32> undef, <16 x i32> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call { i16, i1 } @llvm.usub.with.overflow.i16(i16 undef, i16 undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = call { <8 x i16>, <8 x i1> } @llvm.usub.with.overflow.v8i16(<8 x i16> undef, <8 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I16 = call { <16 x i16>, <16 x i1> } @llvm.usub.with.overflow.v16i16(<16 x i16> undef, <16 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I16 = call { <32 x i16>, <32 x i1> } @llvm.usub.with.overflow.v32i16(<32 x i16> undef, <32 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call { i8, i1 } @llvm.usub.with.overflow.i8(i8 undef, i8 undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = call { <16 x i8>, <16 x i1> } @llvm.usub.with.overflow.v16i8(<16 x i8> undef, <16 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I8 = call { <32 x i8>, <32 x i1> } @llvm.usub.with.overflow.v32i8(<32 x i8> undef, <32 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I8 = call { <64 x i8>, <64 x i1> } @llvm.usub.with.overflow.v64i8(<64 x i8> undef, <64 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSE42-LABEL: 'usub'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call { i64, i1 } @llvm.usub.with.overflow.i64(i64 undef, i64 undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.usub.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.usub.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.usub.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call { i32, i1 } @llvm.usub.with.overflow.i32(i32 undef, i32 undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.usub.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.usub.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I32 = call { <16 x i32>, <16 x i1> } @llvm.usub.with.overflow.v16i32(<16 x i32> undef, <16 x i32> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call { i16, i1 } @llvm.usub.with.overflow.i16(i16 undef, i16 undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = call { <8 x i16>, <8 x i1> } @llvm.usub.with.overflow.v8i16(<8 x i16> undef, <8 x i16> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I16 = call { <16 x i16>, <16 x i1> } @llvm.usub.with.overflow.v16i16(<16 x i16> undef, <16 x i16> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I16 = call { <32 x i16>, <32 x i1> } @llvm.usub.with.overflow.v32i16(<32 x i16> undef, <32 x i16> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call { i8, i1 } @llvm.usub.with.overflow.i8(i8 undef, i8 undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = call { <16 x i8>, <16 x i1> } @llvm.usub.with.overflow.v16i8(<16 x i8> undef, <16 x i8> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I8 = call { <32 x i8>, <32 x i1> } @llvm.usub.with.overflow.v32i8(<32 x i8> undef, <32 x i8> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I8 = call { <64 x i8>, <64 x i1> } @llvm.usub.with.overflow.v64i8(<64 x i8> undef, <64 x i8> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX1-LABEL: 'usub'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call { i64, i1 } @llvm.usub.with.overflow.i64(i64 undef, i64 undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.usub.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.usub.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.usub.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call { i32, i1 } @llvm.usub.with.overflow.i32(i32 undef, i32 undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.usub.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.usub.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16I32 = call { <16 x i32>, <16 x i1> } @llvm.usub.with.overflow.v16i32(<16 x i32> undef, <16 x i32> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call { i16, i1 } @llvm.usub.with.overflow.i16(i16 undef, i16 undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = call { <8 x i16>, <8 x i1> } @llvm.usub.with.overflow.v8i16(<8 x i16> undef, <8 x i16> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I16 = call { <16 x i16>, <16 x i1> } @llvm.usub.with.overflow.v16i16(<16 x i16> undef, <16 x i16> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V32I16 = call { <32 x i16>, <32 x i1> } @llvm.usub.with.overflow.v32i16(<32 x i16> undef, <32 x i16> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call { i8, i1 } @llvm.usub.with.overflow.i8(i8 undef, i8 undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = call { <16 x i8>, <16 x i1> } @llvm.usub.with.overflow.v16i8(<16 x i8> undef, <16 x i8> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I8 = call { <32 x i8>, <32 x i1> } @llvm.usub.with.overflow.v32i8(<32 x i8> undef, <32 x i8> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V64I8 = call { <64 x i8>, <64 x i1> } @llvm.usub.with.overflow.v64i8(<64 x i8> undef, <64 x i8> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX2-LABEL: 'usub'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call { i64, i1 } @llvm.usub.with.overflow.i64(i64 undef, i64 undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.usub.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.usub.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.usub.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call { i32, i1 } @llvm.usub.with.overflow.i32(i32 undef, i32 undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.usub.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.usub.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I32 = call { <16 x i32>, <16 x i1> } @llvm.usub.with.overflow.v16i32(<16 x i32> undef, <16 x i32> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call { i16, i1 } @llvm.usub.with.overflow.i16(i16 undef, i16 undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = call { <8 x i16>, <8 x i1> } @llvm.usub.with.overflow.v8i16(<8 x i16> undef, <8 x i16> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = call { <16 x i16>, <16 x i1> } @llvm.usub.with.overflow.v16i16(<16 x i16> undef, <16 x i16> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = call { <32 x i16>, <32 x i1> } @llvm.usub.with.overflow.v32i16(<32 x i16> undef, <32 x i16> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call { i8, i1 } @llvm.usub.with.overflow.i8(i8 undef, i8 undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = call { <16 x i8>, <16 x i1> } @llvm.usub.with.overflow.v16i8(<16 x i8> undef, <16 x i8> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I8 = call { <32 x i8>, <32 x i1> } @llvm.usub.with.overflow.v32i8(<32 x i8> undef, <32 x i8> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = call { <64 x i8>, <64 x i1> } @llvm.usub.with.overflow.v64i8(<64 x i8> undef, <64 x i8> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX512F-LABEL: 'usub'
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call { i64, i1 } @llvm.usub.with.overflow.i64(i64 undef, i64 undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.usub.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.usub.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.usub.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call { i32, i1 } @llvm.usub.with.overflow.i32(i32 undef, i32 undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.usub.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.usub.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I32 = call { <16 x i32>, <16 x i1> } @llvm.usub.with.overflow.v16i32(<16 x i32> undef, <16 x i32> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call { i16, i1 } @llvm.usub.with.overflow.i16(i16 undef, i16 undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = call { <8 x i16>, <8 x i1> } @llvm.usub.with.overflow.v8i16(<8 x i16> undef, <8 x i16> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = call { <16 x i16>, <16 x i1> } @llvm.usub.with.overflow.v16i16(<16 x i16> undef, <16 x i16> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = call { <32 x i16>, <32 x i1> } @llvm.usub.with.overflow.v32i16(<32 x i16> undef, <32 x i16> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call { i8, i1 } @llvm.usub.with.overflow.i8(i8 undef, i8 undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = call { <16 x i8>, <16 x i1> } @llvm.usub.with.overflow.v16i8(<16 x i8> undef, <16 x i8> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I8 = call { <32 x i8>, <32 x i1> } @llvm.usub.with.overflow.v32i8(<32 x i8> undef, <32 x i8> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = call { <64 x i8>, <64 x i1> } @llvm.usub.with.overflow.v64i8(<64 x i8> undef, <64 x i8> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX512BW-LABEL: 'usub'
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call { i64, i1 } @llvm.usub.with.overflow.i64(i64 undef, i64 undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.usub.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.usub.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.usub.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call { i32, i1 } @llvm.usub.with.overflow.i32(i32 undef, i32 undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.usub.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.usub.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I32 = call { <16 x i32>, <16 x i1> } @llvm.usub.with.overflow.v16i32(<16 x i32> undef, <16 x i32> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call { i16, i1 } @llvm.usub.with.overflow.i16(i16 undef, i16 undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = call { <8 x i16>, <8 x i1> } @llvm.usub.with.overflow.v8i16(<8 x i16> undef, <8 x i16> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = call { <16 x i16>, <16 x i1> } @llvm.usub.with.overflow.v16i16(<16 x i16> undef, <16 x i16> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I16 = call { <32 x i16>, <32 x i1> } @llvm.usub.with.overflow.v32i16(<32 x i16> undef, <32 x i16> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call { i8, i1 } @llvm.usub.with.overflow.i8(i8 undef, i8 undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = call { <16 x i8>, <16 x i1> } @llvm.usub.with.overflow.v16i8(<16 x i8> undef, <16 x i8> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I8 = call { <32 x i8>, <32 x i1> } @llvm.usub.with.overflow.v32i8(<32 x i8> undef, <32 x i8> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64I8 = call { <64 x i8>, <64 x i1> } @llvm.usub.with.overflow.v64i8(<64 x i8> undef, <64 x i8> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX512DQ-LABEL: 'usub'
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call { i64, i1 } @llvm.usub.with.overflow.i64(i64 undef, i64 undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.usub.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.usub.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.usub.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call { i32, i1 } @llvm.usub.with.overflow.i32(i32 undef, i32 undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.usub.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.usub.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I32 = call { <16 x i32>, <16 x i1> } @llvm.usub.with.overflow.v16i32(<16 x i32> undef, <16 x i32> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call { i16, i1 } @llvm.usub.with.overflow.i16(i16 undef, i16 undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = call { <8 x i16>, <8 x i1> } @llvm.usub.with.overflow.v8i16(<8 x i16> undef, <8 x i16> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = call { <16 x i16>, <16 x i1> } @llvm.usub.with.overflow.v16i16(<16 x i16> undef, <16 x i16> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = call { <32 x i16>, <32 x i1> } @llvm.usub.with.overflow.v32i16(<32 x i16> undef, <32 x i16> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call { i8, i1 } @llvm.usub.with.overflow.i8(i8 undef, i8 undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = call { <16 x i8>, <16 x i1> } @llvm.usub.with.overflow.v16i8(<16 x i8> undef, <16 x i8> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I8 = call { <32 x i8>, <32 x i1> } @llvm.usub.with.overflow.v32i8(<32 x i8> undef, <32 x i8> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = call { <64 x i8>, <64 x i1> } @llvm.usub.with.overflow.v64i8(<64 x i8> undef, <64 x i8> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SLM-LABEL: 'usub'
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call { i64, i1 } @llvm.usub.with.overflow.i64(i64 undef, i64 undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.usub.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.usub.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.usub.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call { i32, i1 } @llvm.usub.with.overflow.i32(i32 undef, i32 undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.usub.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.usub.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I32 = call { <16 x i32>, <16 x i1> } @llvm.usub.with.overflow.v16i32(<16 x i32> undef, <16 x i32> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call { i16, i1 } @llvm.usub.with.overflow.i16(i16 undef, i16 undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = call { <8 x i16>, <8 x i1> } @llvm.usub.with.overflow.v8i16(<8 x i16> undef, <8 x i16> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I16 = call { <16 x i16>, <16 x i1> } @llvm.usub.with.overflow.v16i16(<16 x i16> undef, <16 x i16> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I16 = call { <32 x i16>, <32 x i1> } @llvm.usub.with.overflow.v32i16(<32 x i16> undef, <32 x i16> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call { i8, i1 } @llvm.usub.with.overflow.i8(i8 undef, i8 undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = call { <16 x i8>, <16 x i1> } @llvm.usub.with.overflow.v16i8(<16 x i8> undef, <16 x i8> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I8 = call { <32 x i8>, <32 x i1> } @llvm.usub.with.overflow.v32i8(<32 x i8> undef, <32 x i8> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I8 = call { <64 x i8>, <64 x i1> } @llvm.usub.with.overflow.v64i8(<64 x i8> undef, <64 x i8> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; GLM-LABEL: 'usub'
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call { i64, i1 } @llvm.usub.with.overflow.i64(i64 undef, i64 undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.usub.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.usub.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.usub.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call { i32, i1 } @llvm.usub.with.overflow.i32(i32 undef, i32 undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.usub.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.usub.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I32 = call { <16 x i32>, <16 x i1> } @llvm.usub.with.overflow.v16i32(<16 x i32> undef, <16 x i32> undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call { i16, i1 } @llvm.usub.with.overflow.i16(i16 undef, i16 undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = call { <8 x i16>, <8 x i1> } @llvm.usub.with.overflow.v8i16(<8 x i16> undef, <8 x i16> undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I16 = call { <16 x i16>, <16 x i1> } @llvm.usub.with.overflow.v16i16(<16 x i16> undef, <16 x i16> undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I16 = call { <32 x i16>, <32 x i1> } @llvm.usub.with.overflow.v32i16(<32 x i16> undef, <32 x i16> undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call { i8, i1 } @llvm.usub.with.overflow.i8(i8 undef, i8 undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = call { <16 x i8>, <16 x i1> } @llvm.usub.with.overflow.v16i8(<16 x i8> undef, <16 x i8> undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I8 = call { <32 x i8>, <32 x i1> } @llvm.usub.with.overflow.v32i8(<32 x i8> undef, <32 x i8> undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I8 = call { <64 x i8>, <64 x i1> } @llvm.usub.with.overflow.v64i8(<64 x i8> undef, <64 x i8> undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; BTVER2-LABEL: 'usub'
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call { i64, i1 } @llvm.usub.with.overflow.i64(i64 undef, i64 undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.usub.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.usub.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.usub.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call { i32, i1 } @llvm.usub.with.overflow.i32(i32 undef, i32 undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.usub.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.usub.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16I32 = call { <16 x i32>, <16 x i1> } @llvm.usub.with.overflow.v16i32(<16 x i32> undef, <16 x i32> undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call { i16, i1 } @llvm.usub.with.overflow.i16(i16 undef, i16 undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = call { <8 x i16>, <8 x i1> } @llvm.usub.with.overflow.v8i16(<8 x i16> undef, <8 x i16> undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I16 = call { <16 x i16>, <16 x i1> } @llvm.usub.with.overflow.v16i16(<16 x i16> undef, <16 x i16> undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V32I16 = call { <32 x i16>, <32 x i1> } @llvm.usub.with.overflow.v32i16(<32 x i16> undef, <32 x i16> undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call { i8, i1 } @llvm.usub.with.overflow.i8(i8 undef, i8 undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = call { <16 x i8>, <16 x i1> } @llvm.usub.with.overflow.v16i8(<16 x i8> undef, <16 x i8> undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I8 = call { <32 x i8>, <32 x i1> } @llvm.usub.with.overflow.v32i8(<32 x i8> undef, <32 x i8> undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V64I8 = call { <64 x i8>, <64 x i1> } @llvm.usub.with.overflow.v64i8(<64 x i8> undef, <64 x i8> undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+  %I64 = call {i64, i1} @llvm.usub.with.overflow.i64(i64 undef, i64 undef)
+  %V2I64 = call {<2 x i64>, <2 x i1>} @llvm.usub.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
+  %V4I64 = call {<4 x i64>, <4 x i1>} @llvm.usub.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
+  %V8I64 = call {<8 x i64>, <8 x i1>} @llvm.usub.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
+
+  %I32 = call {i32, i1} @llvm.usub.with.overflow.i32(i32 undef, i32 undef)
+  %V4I32  = call {<4 x i32>, <4 x i1>}  @llvm.usub.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
+  %V8I32  = call {<8 x i32>, <8 x i1>}  @llvm.usub.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
+  %V16I32 = call {<16 x i32>, <16 x i1>} @llvm.usub.with.overflow.v16i32(<16 x i32> undef, <16 x i32> undef)
+
+  %I16 = call {i16, i1} @llvm.usub.with.overflow.i16(i16 undef, i16 undef)
+  %V8I16  = call {<8 x i16>, <8 x i1>}  @llvm.usub.with.overflow.v8i16(<8 x i16> undef, <8 x i16> undef)
+  %V16I16 = call {<16 x i16>, <16 x i1>} @llvm.usub.with.overflow.v16i16(<16 x i16> undef, <16 x i16> undef)
+  %V32I16 = call {<32 x i16>, <32 x i1>} @llvm.usub.with.overflow.v32i16(<32 x i16> undef, <32 x i16> undef)
+
+  %I8 = call {i8, i1} @llvm.usub.with.overflow.i8(i8 undef, i8 undef)
+  %V16I8 = call {<16 x i8>, <16 x i1>} @llvm.usub.with.overflow.v16i8(<16 x i8> undef, <16 x i8> undef)
+  %V32I8 = call {<32 x i8>, <32 x i1>} @llvm.usub.with.overflow.v32i8(<32 x i8> undef, <32 x i8> undef)
+  %V64I8 = call {<64 x i8>, <64 x i1>} @llvm.usub.with.overflow.v64i8(<64 x i8> undef, <64 x i8> undef)
+
+  ret i32 undef
+}
+
+;
+; smul.with.overflow
+;
+
+declare {i64, i1}              @llvm.smul.with.overflow.i64(i64, i64)
+declare {<2 x i64>, <2 x i1>}  @llvm.smul.with.overflow.v2i64(<2 x i64>, <2 x i64>)
+declare {<4 x i64>, <4 x i1>}  @llvm.smul.with.overflow.v4i64(<4 x i64>, <4 x i64>)
+declare {<8 x i64>, <8 x i1>}  @llvm.smul.with.overflow.v8i64(<8 x i64>, <8 x i64>)
+
+declare {i32, i1}               @llvm.smul.with.overflow.i32(i32, i32)
+declare {<4 x i32>, <4 x i1>}   @llvm.smul.with.overflow.v4i32(<4 x i32>, <4 x i32>)
+declare {<8 x i32>, <8 x i1>}   @llvm.smul.with.overflow.v8i32(<8 x i32>, <8 x i32>)
+declare {<16 x i32>, <16 x i1>} @llvm.smul.with.overflow.v16i32(<16 x i32>, <16 x i32>)
+
+declare {i16, i1}               @llvm.smul.with.overflow.i16(i16, i16)
+declare {<8 x i16>,  <8 x i1>}  @llvm.smul.with.overflow.v8i16(<8 x i16>, <8 x i16>)
+declare {<16 x i16>, <16 x i1>} @llvm.smul.with.overflow.v16i16(<16 x i16>, <16 x i16>)
+declare {<32 x i16>, <32 x i1>} @llvm.smul.with.overflow.v32i16(<32 x i16>, <32 x i16>)
+
+declare {i8, i1}                @llvm.smul.with.overflow.i8(i8, i8)
+declare {<16 x i8>, <16 x i1>}  @llvm.smul.with.overflow.v16i8(<16 x i8>, <16 x i8>)
+declare {<32 x i8>, <32 x i1>}  @llvm.smul.with.overflow.v32i8(<32 x i8>, <32 x i8>)
+declare {<64 x i8>, <64 x i1>}  @llvm.smul.with.overflow.v64i8(<64 x i8>, <64 x i8>)
+
+define i32 @smul(i32 %arg) {
+; CHECK-LABEL: 'smul'
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call { i64, i1 } @llvm.smul.with.overflow.i64(i64 undef, i64 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.smul.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.smul.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.smul.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call { i32, i1 } @llvm.smul.with.overflow.i32(i32 undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.smul.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.smul.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 47 for instruction: %V16I32 = call { <16 x i32>, <16 x i1> } @llvm.smul.with.overflow.v16i32(<16 x i32> undef, <16 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call { i16, i1 } @llvm.smul.with.overflow.i16(i16 undef, i16 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %V8I16 = call { <8 x i16>, <8 x i1> } @llvm.smul.with.overflow.v8i16(<8 x i16> undef, <8 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 47 for instruction: %V16I16 = call { <16 x i16>, <16 x i1> } @llvm.smul.with.overflow.v16i16(<16 x i16> undef, <16 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 95 for instruction: %V32I16 = call { <32 x i16>, <32 x i1> } @llvm.smul.with.overflow.v32i16(<32 x i16> undef, <32 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call { i8, i1 } @llvm.smul.with.overflow.i8(i8 undef, i8 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 47 for instruction: %V16I8 = call { <16 x i8>, <16 x i1> } @llvm.smul.with.overflow.v16i8(<16 x i8> undef, <16 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 95 for instruction: %V32I8 = call { <32 x i8>, <32 x i1> } @llvm.smul.with.overflow.v32i8(<32 x i8> undef, <32 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 191 for instruction: %V64I8 = call { <64 x i8>, <64 x i1> } @llvm.smul.with.overflow.v64i8(<64 x i8> undef, <64 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+  %I64 = call {i64, i1} @llvm.smul.with.overflow.i64(i64 undef, i64 undef)
+  %V2I64 = call {<2 x i64>, <2 x i1>} @llvm.smul.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
+  %V4I64 = call {<4 x i64>, <4 x i1>} @llvm.smul.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
+  %V8I64 = call {<8 x i64>, <8 x i1>} @llvm.smul.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
+
+  %I32 = call {i32, i1} @llvm.smul.with.overflow.i32(i32 undef, i32 undef)
+  %V4I32  = call {<4 x i32>, <4 x i1>}  @llvm.smul.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
+  %V8I32  = call {<8 x i32>, <8 x i1>}  @llvm.smul.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
+  %V16I32 = call {<16 x i32>, <16 x i1>} @llvm.smul.with.overflow.v16i32(<16 x i32> undef, <16 x i32> undef)
+
+  %I16 = call {i16, i1} @llvm.smul.with.overflow.i16(i16 undef, i16 undef)
+  %V8I16  = call {<8 x i16>, <8 x i1>}  @llvm.smul.with.overflow.v8i16(<8 x i16> undef, <8 x i16> undef)
+  %V16I16 = call {<16 x i16>, <16 x i1>} @llvm.smul.with.overflow.v16i16(<16 x i16> undef, <16 x i16> undef)
+  %V32I16 = call {<32 x i16>, <32 x i1>} @llvm.smul.with.overflow.v32i16(<32 x i16> undef, <32 x i16> undef)
+
+  %I8 = call {i8, i1} @llvm.smul.with.overflow.i8(i8 undef, i8 undef)
+  %V16I8 = call {<16 x i8>, <16 x i1>} @llvm.smul.with.overflow.v16i8(<16 x i8> undef, <16 x i8> undef)
+  %V32I8 = call {<32 x i8>, <32 x i1>} @llvm.smul.with.overflow.v32i8(<32 x i8> undef, <32 x i8> undef)
+  %V64I8 = call {<64 x i8>, <64 x i1>} @llvm.smul.with.overflow.v64i8(<64 x i8> undef, <64 x i8> undef)
+
+  ret i32 undef
+}
+
+;
+; umul.with.overflow
+;
+
+declare {i64, i1}              @llvm.umul.with.overflow.i64(i64, i64)
+declare {<2 x i64>, <2 x i1>}  @llvm.umul.with.overflow.v2i64(<2 x i64>, <2 x i64>)
+declare {<4 x i64>, <4 x i1>}  @llvm.umul.with.overflow.v4i64(<4 x i64>, <4 x i64>)
+declare {<8 x i64>, <8 x i1>}  @llvm.umul.with.overflow.v8i64(<8 x i64>, <8 x i64>)
+
+declare {i32, i1}               @llvm.umul.with.overflow.i32(i32, i32)
+declare {<4 x i32>, <4 x i1>}   @llvm.umul.with.overflow.v4i32(<4 x i32>, <4 x i32>)
+declare {<8 x i32>, <8 x i1>}   @llvm.umul.with.overflow.v8i32(<8 x i32>, <8 x i32>)
+declare {<16 x i32>, <16 x i1>} @llvm.umul.with.overflow.v16i32(<16 x i32>, <16 x i32>)
+
+declare {i16, i1}               @llvm.umul.with.overflow.i16(i16, i16)
+declare {<8 x i16>,  <8 x i1>}  @llvm.umul.with.overflow.v8i16(<8 x i16>, <8 x i16>)
+declare {<16 x i16>, <16 x i1>} @llvm.umul.with.overflow.v16i16(<16 x i16>, <16 x i16>)
+declare {<32 x i16>, <32 x i1>} @llvm.umul.with.overflow.v32i16(<32 x i16>, <32 x i16>)
+
+declare {i8, i1}                @llvm.umul.with.overflow.i8(i8, i8)
+declare {<16 x i8>, <16 x i1>}  @llvm.umul.with.overflow.v16i8(<16 x i8>, <16 x i8>)
+declare {<32 x i8>, <32 x i1>}  @llvm.umul.with.overflow.v32i8(<32 x i8>, <32 x i8>)
+declare {<64 x i8>, <64 x i1>}  @llvm.umul.with.overflow.v64i8(<64 x i8>, <64 x i8>)
+
+define i32 @umul(i32 %arg) {
+; CHECK-LABEL: 'umul'
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 undef, i64 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.umul.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.umul.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.umul.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call { i32, i1 } @llvm.umul.with.overflow.i32(i32 undef, i32 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.umul.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.umul.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 47 for instruction: %V16I32 = call { <16 x i32>, <16 x i1> } @llvm.umul.with.overflow.v16i32(<16 x i32> undef, <16 x i32> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = call { i16, i1 } @llvm.umul.with.overflow.i16(i16 undef, i16 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %V8I16 = call { <8 x i16>, <8 x i1> } @llvm.umul.with.overflow.v8i16(<8 x i16> undef, <8 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 47 for instruction: %V16I16 = call { <16 x i16>, <16 x i1> } @llvm.umul.with.overflow.v16i16(<16 x i16> undef, <16 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 95 for instruction: %V32I16 = call { <32 x i16>, <32 x i1> } @llvm.umul.with.overflow.v32i16(<32 x i16> undef, <32 x i16> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = call { i8, i1 } @llvm.umul.with.overflow.i8(i8 undef, i8 undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 47 for instruction: %V16I8 = call { <16 x i8>, <16 x i1> } @llvm.umul.with.overflow.v16i8(<16 x i8> undef, <16 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 95 for instruction: %V32I8 = call { <32 x i8>, <32 x i1> } @llvm.umul.with.overflow.v32i8(<32 x i8> undef, <32 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 191 for instruction: %V64I8 = call { <64 x i8>, <64 x i1> } @llvm.umul.with.overflow.v64i8(<64 x i8> undef, <64 x i8> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+  %I64 = call {i64, i1} @llvm.umul.with.overflow.i64(i64 undef, i64 undef)
+  %V2I64 = call {<2 x i64>, <2 x i1>} @llvm.umul.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
+  %V4I64 = call {<4 x i64>, <4 x i1>} @llvm.umul.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
+  %V8I64 = call {<8 x i64>, <8 x i1>} @llvm.umul.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
+
+  %I32 = call {i32, i1} @llvm.umul.with.overflow.i32(i32 undef, i32 undef)
+  %V4I32  = call {<4 x i32>, <4 x i1>}  @llvm.umul.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
+  %V8I32  = call {<8 x i32>, <8 x i1>}  @llvm.umul.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
+  %V16I32 = call {<16 x i32>, <16 x i1>} @llvm.umul.with.overflow.v16i32(<16 x i32> undef, <16 x i32> undef)
+
+  %I16 = call {i16, i1} @llvm.umul.with.overflow.i16(i16 undef, i16 undef)
+  %V8I16  = call {<8 x i16>, <8 x i1>}  @llvm.umul.with.overflow.v8i16(<8 x i16> undef, <8 x i16> undef)
+  %V16I16 = call {<16 x i16>, <16 x i1>} @llvm.umul.with.overflow.v16i16(<16 x i16> undef, <16 x i16> undef)
+  %V32I16 = call {<32 x i16>, <32 x i1>} @llvm.umul.with.overflow.v32i16(<32 x i16> undef, <32 x i16> undef)
+
+  %I8 = call {i8, i1} @llvm.umul.with.overflow.i8(i8 undef, i8 undef)
+  %V16I8 = call {<16 x i8>, <16 x i1>} @llvm.umul.with.overflow.v16i8(<16 x i8> undef, <16 x i8> undef)
+  %V32I8 = call {<32 x i8>, <32 x i1>} @llvm.umul.with.overflow.v32i8(<32 x i8> undef, <32 x i8> undef)
+  %V64I8 = call {<64 x i8>, <64 x i1>} @llvm.umul.with.overflow.v64i8(<64 x i8> undef, <64 x i8> undef)
+
+  ret i32 undef
+}
diff --git a/test/Analysis/CostModel/X86/arith-ssat.ll b/test/Analysis/CostModel/X86/arith-ssat.ll
new file mode 100644
index 0000000000000000000000000000000000000000..d99bba3268d931d9eadf25ef431e628877b60082
--- /dev/null
+++ b/test/Analysis/CostModel/X86/arith-ssat.ll
@@ -0,0 +1,483 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
+; RUN: opt < %s -cost-model -analyze -mtriple=x86_64-apple-macosx10.8.0 -mattr=+ssse3 | FileCheck %s --check-prefixes=CHECK,SSE,SSSE3
+; RUN: opt < %s -cost-model -analyze -mtriple=x86_64-apple-macosx10.8.0 -mattr=+sse4.2 | FileCheck %s --check-prefixes=CHECK,SSE,SSE42
+; RUN: opt < %s -cost-model -analyze -mtriple=x86_64-apple-macosx10.8.0 -mattr=+avx | FileCheck %s --check-prefixes=CHECK,AVX,AVX1
+; RUN: opt < %s -cost-model -analyze -mtriple=x86_64-apple-macosx10.8.0 -mattr=+avx2 | FileCheck %s --check-prefixes=CHECK,AVX,AVX2
+; RUN: opt < %s -cost-model -analyze -mtriple=x86_64-apple-macosx10.8.0 -mattr=+avx512f | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512F
+; RUN: opt < %s -cost-model -analyze -mtriple=x86_64-apple-macosx10.8.0 -mattr=+avx512f,+avx512bw | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512BW
+; RUN: opt < %s -cost-model -analyze -mtriple=x86_64-apple-macosx10.8.0 -mattr=+avx512f,+avx512dq | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512DQ
+;
+; RUN: opt < %s -cost-model -analyze -mtriple=x86_64-apple-macosx10.8.0 -mcpu=slm | FileCheck %s --check-prefixes=SLM
+; RUN: opt < %s -cost-model -analyze -mtriple=x86_64-apple-macosx10.8.0 -mcpu=goldmont | FileCheck %s --check-prefixes=GLM
+; RUN: opt < %s -cost-model -analyze -mtriple=x86_64-apple-macosx10.8.0 -mcpu=btver2 | FileCheck %s --check-prefixes=BTVER2
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.8.0"
+
+declare i64        @llvm.sadd.sat.i64(i64, i64)
+declare <2 x i64>  @llvm.sadd.sat.v2i64(<2 x i64>, <2 x i64>)
+declare <4 x i64>  @llvm.sadd.sat.v4i64(<4 x i64>, <4 x i64>)
+declare <8 x i64>  @llvm.sadd.sat.v8i64(<8 x i64>, <8 x i64>)
+
+declare i32        @llvm.sadd.sat.i32(i32, i32)
+declare <4 x i32>  @llvm.sadd.sat.v4i32(<4 x i32>, <4 x i32>)
+declare <8 x i32>  @llvm.sadd.sat.v8i32(<8 x i32>, <8 x i32>)
+declare <16 x i32> @llvm.sadd.sat.v16i32(<16 x i32>, <16 x i32>)
+
+declare i16        @llvm.sadd.sat.i16(i16, i16)
+declare <8 x i16>  @llvm.sadd.sat.v8i16(<8 x i16>, <8 x i16>)
+declare <16 x i16> @llvm.sadd.sat.v16i16(<16 x i16>, <16 x i16>)
+declare <32 x i16> @llvm.sadd.sat.v32i16(<32 x i16>, <32 x i16>)
+
+declare i8         @llvm.sadd.sat.i8(i8,  i8)
+declare <16 x i8>  @llvm.sadd.sat.v16i8(<16 x i8>, <16 x i8>)
+declare <32 x i8>  @llvm.sadd.sat.v32i8(<32 x i8>, <32 x i8>)
+declare <64 x i8>  @llvm.sadd.sat.v64i8(<64 x i8>, <64 x i8>)
+
+define i32 @add(i32 %arg) {
+; SSSE3-LABEL: 'add'
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I64 = call i64 @llvm.sadd.sat.i64(i64 undef, i64 undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V2I64 = call <2 x i64> @llvm.sadd.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 81 for instruction: %V4I64 = call <4 x i64> @llvm.sadd.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 159 for instruction: %V8I64 = call <8 x i64> @llvm.sadd.sat.v8i64(<8 x i64> undef, <8 x i64> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I32 = call i32 @llvm.sadd.sat.i32(i32 undef, i32 undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I32 = call <4 x i32> @llvm.sadd.sat.v4i32(<4 x i32> undef, <4 x i32> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %V8I32 = call <8 x i32> @llvm.sadd.sat.v8i32(<8 x i32> undef, <8 x i32> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 47 for instruction: %V16I32 = call <16 x i32> @llvm.sadd.sat.v16i32(<16 x i32> undef, <16 x i32> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I16 = call i16 @llvm.sadd.sat.i16(i16 undef, i16 undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16> undef, <8 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = call <16 x i16> @llvm.sadd.sat.v16i16(<16 x i16> undef, <16 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = call <32 x i16> @llvm.sadd.sat.v32i16(<32 x i16> undef, <32 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I8 = call i8 @llvm.sadd.sat.i8(i8 undef, i8 undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = call <16 x i8> @llvm.sadd.sat.v16i8(<16 x i8> undef, <16 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I8 = call <32 x i8> @llvm.sadd.sat.v32i8(<32 x i8> undef, <32 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = call <64 x i8> @llvm.sadd.sat.v64i8(<64 x i8> undef, <64 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSE42-LABEL: 'add'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I64 = call i64 @llvm.sadd.sat.i64(i64 undef, i64 undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V2I64 = call <2 x i64> @llvm.sadd.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V4I64 = call <4 x i64> @llvm.sadd.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %V8I64 = call <8 x i64> @llvm.sadd.sat.v8i64(<8 x i64> undef, <8 x i64> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I32 = call i32 @llvm.sadd.sat.i32(i32 undef, i32 undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V4I32 = call <4 x i32> @llvm.sadd.sat.v4i32(<4 x i32> undef, <4 x i32> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V8I32 = call <8 x i32> @llvm.sadd.sat.v8i32(<8 x i32> undef, <8 x i32> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %V16I32 = call <16 x i32> @llvm.sadd.sat.v16i32(<16 x i32> undef, <16 x i32> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I16 = call i16 @llvm.sadd.sat.i16(i16 undef, i16 undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16> undef, <8 x i16> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = call <16 x i16> @llvm.sadd.sat.v16i16(<16 x i16> undef, <16 x i16> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = call <32 x i16> @llvm.sadd.sat.v32i16(<32 x i16> undef, <32 x i16> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I8 = call i8 @llvm.sadd.sat.i8(i8 undef, i8 undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = call <16 x i8> @llvm.sadd.sat.v16i8(<16 x i8> undef, <16 x i8> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I8 = call <32 x i8> @llvm.sadd.sat.v32i8(<32 x i8> undef, <32 x i8> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = call <64 x i8> @llvm.sadd.sat.v64i8(<64 x i8> undef, <64 x i8> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX1-LABEL: 'add'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I64 = call i64 @llvm.sadd.sat.i64(i64 undef, i64 undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V2I64 = call <2 x i64> @llvm.sadd.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %V4I64 = call <4 x i64> @llvm.sadd.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 47 for instruction: %V8I64 = call <8 x i64> @llvm.sadd.sat.v8i64(<8 x i64> undef, <8 x i64> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I32 = call i32 @llvm.sadd.sat.i32(i32 undef, i32 undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V4I32 = call <4 x i32> @llvm.sadd.sat.v4i32(<4 x i32> undef, <4 x i32> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %V8I32 = call <8 x i32> @llvm.sadd.sat.v8i32(<8 x i32> undef, <8 x i32> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 47 for instruction: %V16I32 = call <16 x i32> @llvm.sadd.sat.v16i32(<16 x i32> undef, <16 x i32> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I16 = call i16 @llvm.sadd.sat.i16(i16 undef, i16 undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16> undef, <8 x i16> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I16 = call <16 x i16> @llvm.sadd.sat.v16i16(<16 x i16> undef, <16 x i16> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I16 = call <32 x i16> @llvm.sadd.sat.v32i16(<32 x i16> undef, <32 x i16> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I8 = call i8 @llvm.sadd.sat.i8(i8 undef, i8 undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = call <16 x i8> @llvm.sadd.sat.v16i8(<16 x i8> undef, <16 x i8> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I8 = call <32 x i8> @llvm.sadd.sat.v32i8(<32 x i8> undef, <32 x i8> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I8 = call <64 x i8> @llvm.sadd.sat.v64i8(<64 x i8> undef, <64 x i8> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX2-LABEL: 'add'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I64 = call i64 @llvm.sadd.sat.i64(i64 undef, i64 undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V2I64 = call <2 x i64> @llvm.sadd.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V4I64 = call <4 x i64> @llvm.sadd.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V8I64 = call <8 x i64> @llvm.sadd.sat.v8i64(<8 x i64> undef, <8 x i64> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I32 = call i32 @llvm.sadd.sat.i32(i32 undef, i32 undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V4I32 = call <4 x i32> @llvm.sadd.sat.v4i32(<4 x i32> undef, <4 x i32> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V8I32 = call <8 x i32> @llvm.sadd.sat.v8i32(<8 x i32> undef, <8 x i32> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V16I32 = call <16 x i32> @llvm.sadd.sat.v16i32(<16 x i32> undef, <16 x i32> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I16 = call i16 @llvm.sadd.sat.i16(i16 undef, i16 undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16> undef, <8 x i16> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.sadd.sat.v16i16(<16 x i16> undef, <16 x i16> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I16 = call <32 x i16> @llvm.sadd.sat.v32i16(<32 x i16> undef, <32 x i16> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I8 = call i8 @llvm.sadd.sat.i8(i8 undef, i8 undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = call <16 x i8> @llvm.sadd.sat.v16i8(<16 x i8> undef, <16 x i8> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32I8 = call <32 x i8> @llvm.sadd.sat.v32i8(<32 x i8> undef, <32 x i8> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64I8 = call <64 x i8> @llvm.sadd.sat.v64i8(<64 x i8> undef, <64 x i8> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX512F-LABEL: 'add'
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I64 = call i64 @llvm.sadd.sat.i64(i64 undef, i64 undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V2I64 = call <2 x i64> @llvm.sadd.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V4I64 = call <4 x i64> @llvm.sadd.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V8I64 = call <8 x i64> @llvm.sadd.sat.v8i64(<8 x i64> undef, <8 x i64> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I32 = call i32 @llvm.sadd.sat.i32(i32 undef, i32 undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V4I32 = call <4 x i32> @llvm.sadd.sat.v4i32(<4 x i32> undef, <4 x i32> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V8I32 = call <8 x i32> @llvm.sadd.sat.v8i32(<8 x i32> undef, <8 x i32> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V16I32 = call <16 x i32> @llvm.sadd.sat.v16i32(<16 x i32> undef, <16 x i32> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I16 = call i16 @llvm.sadd.sat.i16(i16 undef, i16 undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16> undef, <8 x i16> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.sadd.sat.v16i16(<16 x i16> undef, <16 x i16> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I16 = call <32 x i16> @llvm.sadd.sat.v32i16(<32 x i16> undef, <32 x i16> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I8 = call i8 @llvm.sadd.sat.i8(i8 undef, i8 undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = call <16 x i8> @llvm.sadd.sat.v16i8(<16 x i8> undef, <16 x i8> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32I8 = call <32 x i8> @llvm.sadd.sat.v32i8(<32 x i8> undef, <32 x i8> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64I8 = call <64 x i8> @llvm.sadd.sat.v64i8(<64 x i8> undef, <64 x i8> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX512BW-LABEL: 'add'
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I64 = call i64 @llvm.sadd.sat.i64(i64 undef, i64 undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V2I64 = call <2 x i64> @llvm.sadd.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V4I64 = call <4 x i64> @llvm.sadd.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V8I64 = call <8 x i64> @llvm.sadd.sat.v8i64(<8 x i64> undef, <8 x i64> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I32 = call i32 @llvm.sadd.sat.i32(i32 undef, i32 undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V4I32 = call <4 x i32> @llvm.sadd.sat.v4i32(<4 x i32> undef, <4 x i32> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V8I32 = call <8 x i32> @llvm.sadd.sat.v8i32(<8 x i32> undef, <8 x i32> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V16I32 = call <16 x i32> @llvm.sadd.sat.v16i32(<16 x i32> undef, <16 x i32> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I16 = call i16 @llvm.sadd.sat.i16(i16 undef, i16 undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16> undef, <8 x i16> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.sadd.sat.v16i16(<16 x i16> undef, <16 x i16> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.sadd.sat.v32i16(<32 x i16> undef, <32 x i16> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I8 = call i8 @llvm.sadd.sat.i8(i8 undef, i8 undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = call <16 x i8> @llvm.sadd.sat.v16i8(<16 x i8> undef, <16 x i8> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32I8 = call <32 x i8> @llvm.sadd.sat.v32i8(<32 x i8> undef, <32 x i8> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64I8 = call <64 x i8> @llvm.sadd.sat.v64i8(<64 x i8> undef, <64 x i8> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX512DQ-LABEL: 'add'
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I64 = call i64 @llvm.sadd.sat.i64(i64 undef, i64 undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V2I64 = call <2 x i64> @llvm.sadd.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V4I64 = call <4 x i64> @llvm.sadd.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V8I64 = call <8 x i64> @llvm.sadd.sat.v8i64(<8 x i64> undef, <8 x i64> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I32 = call i32 @llvm.sadd.sat.i32(i32 undef, i32 undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V4I32 = call <4 x i32> @llvm.sadd.sat.v4i32(<4 x i32> undef, <4 x i32> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V8I32 = call <8 x i32> @llvm.sadd.sat.v8i32(<8 x i32> undef, <8 x i32> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V16I32 = call <16 x i32> @llvm.sadd.sat.v16i32(<16 x i32> undef, <16 x i32> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I16 = call i16 @llvm.sadd.sat.i16(i16 undef, i16 undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16> undef, <8 x i16> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.sadd.sat.v16i16(<16 x i16> undef, <16 x i16> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I16 = call <32 x i16> @llvm.sadd.sat.v32i16(<32 x i16> undef, <32 x i16> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I8 = call i8 @llvm.sadd.sat.i8(i8 undef, i8 undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = call <16 x i8> @llvm.sadd.sat.v16i8(<16 x i8> undef, <16 x i8> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32I8 = call <32 x i8> @llvm.sadd.sat.v32i8(<32 x i8> undef, <32 x i8> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64I8 = call <64 x i8> @llvm.sadd.sat.v64i8(<64 x i8> undef, <64 x i8> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SLM-LABEL: 'add'
+; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I64 = call i64 @llvm.sadd.sat.i64(i64 undef, i64 undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V2I64 = call <2 x i64> @llvm.sadd.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %V4I64 = call <4 x i64> @llvm.sadd.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 43 for instruction: %V8I64 = call <8 x i64> @llvm.sadd.sat.v8i64(<8 x i64> undef, <8 x i64> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I32 = call i32 @llvm.sadd.sat.i32(i32 undef, i32 undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V4I32 = call <4 x i32> @llvm.sadd.sat.v4i32(<4 x i32> undef, <4 x i32> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V8I32 = call <8 x i32> @llvm.sadd.sat.v8i32(<8 x i32> undef, <8 x i32> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %V16I32 = call <16 x i32> @llvm.sadd.sat.v16i32(<16 x i32> undef, <16 x i32> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I16 = call i16 @llvm.sadd.sat.i16(i16 undef, i16 undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16> undef, <8 x i16> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = call <16 x i16> @llvm.sadd.sat.v16i16(<16 x i16> undef, <16 x i16> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = call <32 x i16> @llvm.sadd.sat.v32i16(<32 x i16> undef, <32 x i16> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I8 = call i8 @llvm.sadd.sat.i8(i8 undef, i8 undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = call <16 x i8> @llvm.sadd.sat.v16i8(<16 x i8> undef, <16 x i8> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I8 = call <32 x i8> @llvm.sadd.sat.v32i8(<32 x i8> undef, <32 x i8> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = call <64 x i8> @llvm.sadd.sat.v64i8(<64 x i8> undef, <64 x i8> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; GLM-LABEL: 'add'
+; GLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I64 = call i64 @llvm.sadd.sat.i64(i64 undef, i64 undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V2I64 = call <2 x i64> @llvm.sadd.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V4I64 = call <4 x i64> @llvm.sadd.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %V8I64 = call <8 x i64> @llvm.sadd.sat.v8i64(<8 x i64> undef, <8 x i64> undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I32 = call i32 @llvm.sadd.sat.i32(i32 undef, i32 undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V4I32 = call <4 x i32> @llvm.sadd.sat.v4i32(<4 x i32> undef, <4 x i32> undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V8I32 = call <8 x i32> @llvm.sadd.sat.v8i32(<8 x i32> undef, <8 x i32> undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %V16I32 = call <16 x i32> @llvm.sadd.sat.v16i32(<16 x i32> undef, <16 x i32> undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I16 = call i16 @llvm.sadd.sat.i16(i16 undef, i16 undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16> undef, <8 x i16> undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = call <16 x i16> @llvm.sadd.sat.v16i16(<16 x i16> undef, <16 x i16> undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = call <32 x i16> @llvm.sadd.sat.v32i16(<32 x i16> undef, <32 x i16> undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I8 = call i8 @llvm.sadd.sat.i8(i8 undef, i8 undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = call <16 x i8> @llvm.sadd.sat.v16i8(<16 x i8> undef, <16 x i8> undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I8 = call <32 x i8> @llvm.sadd.sat.v32i8(<32 x i8> undef, <32 x i8> undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = call <64 x i8> @llvm.sadd.sat.v64i8(<64 x i8> undef, <64 x i8> undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; BTVER2-LABEL: 'add'
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I64 = call i64 @llvm.sadd.sat.i64(i64 undef, i64 undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V2I64 = call <2 x i64> @llvm.sadd.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %V4I64 = call <4 x i64> @llvm.sadd.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 47 for instruction: %V8I64 = call <8 x i64> @llvm.sadd.sat.v8i64(<8 x i64> undef, <8 x i64> undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I32 = call i32 @llvm.sadd.sat.i32(i32 undef, i32 undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V4I32 = call <4 x i32> @llvm.sadd.sat.v4i32(<4 x i32> undef, <4 x i32> undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %V8I32 = call <8 x i32> @llvm.sadd.sat.v8i32(<8 x i32> undef, <8 x i32> undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 47 for instruction: %V16I32 = call <16 x i32> @llvm.sadd.sat.v16i32(<16 x i32> undef, <16 x i32> undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I16 = call i16 @llvm.sadd.sat.i16(i16 undef, i16 undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16> undef, <8 x i16> undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I16 = call <16 x i16> @llvm.sadd.sat.v16i16(<16 x i16> undef, <16 x i16> undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I16 = call <32 x i16> @llvm.sadd.sat.v32i16(<32 x i16> undef, <32 x i16> undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I8 = call i8 @llvm.sadd.sat.i8(i8 undef, i8 undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = call <16 x i8> @llvm.sadd.sat.v16i8(<16 x i8> undef, <16 x i8> undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I8 = call <32 x i8> @llvm.sadd.sat.v32i8(<32 x i8> undef, <32 x i8> undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I8 = call <64 x i8> @llvm.sadd.sat.v64i8(<64 x i8> undef, <64 x i8> undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+  %I64 = call i64 @llvm.sadd.sat.i64(i64 undef, i64 undef)
+  %V2I64 = call <2 x i64> @llvm.sadd.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
+  %V4I64 = call <4 x i64> @llvm.sadd.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
+  %V8I64 = call <8 x i64> @llvm.sadd.sat.v8i64(<8 x i64> undef, <8 x i64> undef)
+
+  %I32 = call i32 @llvm.sadd.sat.i32(i32 undef, i32 undef)
+  %V4I32  = call <4 x i32>  @llvm.sadd.sat.v4i32(<4 x i32> undef, <4 x i32> undef)
+  %V8I32  = call <8 x i32>  @llvm.sadd.sat.v8i32(<8 x i32> undef, <8 x i32> undef)
+  %V16I32 = call <16 x i32> @llvm.sadd.sat.v16i32(<16 x i32> undef, <16 x i32> undef)
+
+  %I16 = call i16 @llvm.sadd.sat.i16(i16 undef, i16 undef)
+  %V8I16  = call <8 x i16>  @llvm.sadd.sat.v8i16(<8 x i16> undef, <8 x i16> undef)
+  %V16I16 = call <16 x i16> @llvm.sadd.sat.v16i16(<16 x i16> undef, <16 x i16> undef)
+  %V32I16 = call <32 x i16> @llvm.sadd.sat.v32i16(<32 x i16> undef, <32 x i16> undef)
+
+  %I8 = call i8 @llvm.sadd.sat.i8(i8 undef, i8 undef)
+  %V16I8 = call <16 x i8> @llvm.sadd.sat.v16i8(<16 x i8> undef, <16 x i8> undef)
+  %V32I8 = call <32 x i8> @llvm.sadd.sat.v32i8(<32 x i8> undef, <32 x i8> undef)
+  %V64I8 = call <64 x i8> @llvm.sadd.sat.v64i8(<64 x i8> undef, <64 x i8> undef)
+
+  ret i32 undef
+}
+
+declare i64        @llvm.ssub.sat.i64(i64, i64)
+declare <2 x i64>  @llvm.ssub.sat.v2i64(<2 x i64>, <2 x i64>)
+declare <4 x i64>  @llvm.ssub.sat.v4i64(<4 x i64>, <4 x i64>)
+declare <8 x i64>  @llvm.ssub.sat.v8i64(<8 x i64>, <8 x i64>)
+
+declare i32        @llvm.ssub.sat.i32(i32, i32)
+declare <4 x i32>  @llvm.ssub.sat.v4i32(<4 x i32>, <4 x i32>)
+declare <8 x i32>  @llvm.ssub.sat.v8i32(<8 x i32>, <8 x i32>)
+declare <16 x i32> @llvm.ssub.sat.v16i32(<16 x i32>, <16 x i32>)
+
+declare i16        @llvm.ssub.sat.i16(i16, i16)
+declare <8 x i16>  @llvm.ssub.sat.v8i16(<8 x i16>, <8 x i16>)
+declare <16 x i16> @llvm.ssub.sat.v16i16(<16 x i16>, <16 x i16>)
+declare <32 x i16> @llvm.ssub.sat.v32i16(<32 x i16>, <32 x i16>)
+
+declare i8         @llvm.ssub.sat.i8(i8,  i8)
+declare <16 x i8>  @llvm.ssub.sat.v16i8(<16 x i8>, <16 x i8>)
+declare <32 x i8>  @llvm.ssub.sat.v32i8(<32 x i8>, <32 x i8>)
+declare <64 x i8>  @llvm.ssub.sat.v64i8(<64 x i8>, <64 x i8>)
+
+define i32 @sub(i32 %arg) {
+; SSSE3-LABEL: 'sub'
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I64 = call i64 @llvm.ssub.sat.i64(i64 undef, i64 undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V2I64 = call <2 x i64> @llvm.ssub.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 81 for instruction: %V4I64 = call <4 x i64> @llvm.ssub.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 159 for instruction: %V8I64 = call <8 x i64> @llvm.ssub.sat.v8i64(<8 x i64> undef, <8 x i64> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I32 = call i32 @llvm.ssub.sat.i32(i32 undef, i32 undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I32 = call <4 x i32> @llvm.ssub.sat.v4i32(<4 x i32> undef, <4 x i32> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %V8I32 = call <8 x i32> @llvm.ssub.sat.v8i32(<8 x i32> undef, <8 x i32> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 47 for instruction: %V16I32 = call <16 x i32> @llvm.ssub.sat.v16i32(<16 x i32> undef, <16 x i32> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I16 = call i16 @llvm.ssub.sat.i16(i16 undef, i16 undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.ssub.sat.v8i16(<8 x i16> undef, <8 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = call <16 x i16> @llvm.ssub.sat.v16i16(<16 x i16> undef, <16 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = call <32 x i16> @llvm.ssub.sat.v32i16(<32 x i16> undef, <32 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I8 = call i8 @llvm.ssub.sat.i8(i8 undef, i8 undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = call <16 x i8> @llvm.ssub.sat.v16i8(<16 x i8> undef, <16 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I8 = call <32 x i8> @llvm.ssub.sat.v32i8(<32 x i8> undef, <32 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = call <64 x i8> @llvm.ssub.sat.v64i8(<64 x i8> undef, <64 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSE42-LABEL: 'sub'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I64 = call i64 @llvm.ssub.sat.i64(i64 undef, i64 undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V2I64 = call <2 x i64> @llvm.ssub.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V4I64 = call <4 x i64> @llvm.ssub.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %V8I64 = call <8 x i64> @llvm.ssub.sat.v8i64(<8 x i64> undef, <8 x i64> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I32 = call i32 @llvm.ssub.sat.i32(i32 undef, i32 undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V4I32 = call <4 x i32> @llvm.ssub.sat.v4i32(<4 x i32> undef, <4 x i32> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V8I32 = call <8 x i32> @llvm.ssub.sat.v8i32(<8 x i32> undef, <8 x i32> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %V16I32 = call <16 x i32> @llvm.ssub.sat.v16i32(<16 x i32> undef, <16 x i32> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I16 = call i16 @llvm.ssub.sat.i16(i16 undef, i16 undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.ssub.sat.v8i16(<8 x i16> undef, <8 x i16> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = call <16 x i16> @llvm.ssub.sat.v16i16(<16 x i16> undef, <16 x i16> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = call <32 x i16> @llvm.ssub.sat.v32i16(<32 x i16> undef, <32 x i16> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I8 = call i8 @llvm.ssub.sat.i8(i8 undef, i8 undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = call <16 x i8> @llvm.ssub.sat.v16i8(<16 x i8> undef, <16 x i8> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I8 = call <32 x i8> @llvm.ssub.sat.v32i8(<32 x i8> undef, <32 x i8> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = call <64 x i8> @llvm.ssub.sat.v64i8(<64 x i8> undef, <64 x i8> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX1-LABEL: 'sub'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I64 = call i64 @llvm.ssub.sat.i64(i64 undef, i64 undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V2I64 = call <2 x i64> @llvm.ssub.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %V4I64 = call <4 x i64> @llvm.ssub.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 47 for instruction: %V8I64 = call <8 x i64> @llvm.ssub.sat.v8i64(<8 x i64> undef, <8 x i64> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I32 = call i32 @llvm.ssub.sat.i32(i32 undef, i32 undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V4I32 = call <4 x i32> @llvm.ssub.sat.v4i32(<4 x i32> undef, <4 x i32> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %V8I32 = call <8 x i32> @llvm.ssub.sat.v8i32(<8 x i32> undef, <8 x i32> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 47 for instruction: %V16I32 = call <16 x i32> @llvm.ssub.sat.v16i32(<16 x i32> undef, <16 x i32> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I16 = call i16 @llvm.ssub.sat.i16(i16 undef, i16 undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.ssub.sat.v8i16(<8 x i16> undef, <8 x i16> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I16 = call <16 x i16> @llvm.ssub.sat.v16i16(<16 x i16> undef, <16 x i16> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I16 = call <32 x i16> @llvm.ssub.sat.v32i16(<32 x i16> undef, <32 x i16> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I8 = call i8 @llvm.ssub.sat.i8(i8 undef, i8 undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = call <16 x i8> @llvm.ssub.sat.v16i8(<16 x i8> undef, <16 x i8> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I8 = call <32 x i8> @llvm.ssub.sat.v32i8(<32 x i8> undef, <32 x i8> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I8 = call <64 x i8> @llvm.ssub.sat.v64i8(<64 x i8> undef, <64 x i8> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX2-LABEL: 'sub'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I64 = call i64 @llvm.ssub.sat.i64(i64 undef, i64 undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V2I64 = call <2 x i64> @llvm.ssub.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V4I64 = call <4 x i64> @llvm.ssub.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V8I64 = call <8 x i64> @llvm.ssub.sat.v8i64(<8 x i64> undef, <8 x i64> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I32 = call i32 @llvm.ssub.sat.i32(i32 undef, i32 undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V4I32 = call <4 x i32> @llvm.ssub.sat.v4i32(<4 x i32> undef, <4 x i32> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V8I32 = call <8 x i32> @llvm.ssub.sat.v8i32(<8 x i32> undef, <8 x i32> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V16I32 = call <16 x i32> @llvm.ssub.sat.v16i32(<16 x i32> undef, <16 x i32> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I16 = call i16 @llvm.ssub.sat.i16(i16 undef, i16 undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.ssub.sat.v8i16(<8 x i16> undef, <8 x i16> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.ssub.sat.v16i16(<16 x i16> undef, <16 x i16> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I16 = call <32 x i16> @llvm.ssub.sat.v32i16(<32 x i16> undef, <32 x i16> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I8 = call i8 @llvm.ssub.sat.i8(i8 undef, i8 undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = call <16 x i8> @llvm.ssub.sat.v16i8(<16 x i8> undef, <16 x i8> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32I8 = call <32 x i8> @llvm.ssub.sat.v32i8(<32 x i8> undef, <32 x i8> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64I8 = call <64 x i8> @llvm.ssub.sat.v64i8(<64 x i8> undef, <64 x i8> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX512F-LABEL: 'sub'
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I64 = call i64 @llvm.ssub.sat.i64(i64 undef, i64 undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V2I64 = call <2 x i64> @llvm.ssub.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V4I64 = call <4 x i64> @llvm.ssub.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V8I64 = call <8 x i64> @llvm.ssub.sat.v8i64(<8 x i64> undef, <8 x i64> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I32 = call i32 @llvm.ssub.sat.i32(i32 undef, i32 undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V4I32 = call <4 x i32> @llvm.ssub.sat.v4i32(<4 x i32> undef, <4 x i32> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V8I32 = call <8 x i32> @llvm.ssub.sat.v8i32(<8 x i32> undef, <8 x i32> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V16I32 = call <16 x i32> @llvm.ssub.sat.v16i32(<16 x i32> undef, <16 x i32> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I16 = call i16 @llvm.ssub.sat.i16(i16 undef, i16 undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.ssub.sat.v8i16(<8 x i16> undef, <8 x i16> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.ssub.sat.v16i16(<16 x i16> undef, <16 x i16> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I16 = call <32 x i16> @llvm.ssub.sat.v32i16(<32 x i16> undef, <32 x i16> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I8 = call i8 @llvm.ssub.sat.i8(i8 undef, i8 undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = call <16 x i8> @llvm.ssub.sat.v16i8(<16 x i8> undef, <16 x i8> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32I8 = call <32 x i8> @llvm.ssub.sat.v32i8(<32 x i8> undef, <32 x i8> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64I8 = call <64 x i8> @llvm.ssub.sat.v64i8(<64 x i8> undef, <64 x i8> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX512BW-LABEL: 'sub'
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I64 = call i64 @llvm.ssub.sat.i64(i64 undef, i64 undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V2I64 = call <2 x i64> @llvm.ssub.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V4I64 = call <4 x i64> @llvm.ssub.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V8I64 = call <8 x i64> @llvm.ssub.sat.v8i64(<8 x i64> undef, <8 x i64> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I32 = call i32 @llvm.ssub.sat.i32(i32 undef, i32 undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V4I32 = call <4 x i32> @llvm.ssub.sat.v4i32(<4 x i32> undef, <4 x i32> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V8I32 = call <8 x i32> @llvm.ssub.sat.v8i32(<8 x i32> undef, <8 x i32> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V16I32 = call <16 x i32> @llvm.ssub.sat.v16i32(<16 x i32> undef, <16 x i32> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I16 = call i16 @llvm.ssub.sat.i16(i16 undef, i16 undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.ssub.sat.v8i16(<8 x i16> undef, <8 x i16> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.ssub.sat.v16i16(<16 x i16> undef, <16 x i16> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.ssub.sat.v32i16(<32 x i16> undef, <32 x i16> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I8 = call i8 @llvm.ssub.sat.i8(i8 undef, i8 undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = call <16 x i8> @llvm.ssub.sat.v16i8(<16 x i8> undef, <16 x i8> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32I8 = call <32 x i8> @llvm.ssub.sat.v32i8(<32 x i8> undef, <32 x i8> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64I8 = call <64 x i8> @llvm.ssub.sat.v64i8(<64 x i8> undef, <64 x i8> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX512DQ-LABEL: 'sub'
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I64 = call i64 @llvm.ssub.sat.i64(i64 undef, i64 undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V2I64 = call <2 x i64> @llvm.ssub.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V4I64 = call <4 x i64> @llvm.ssub.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V8I64 = call <8 x i64> @llvm.ssub.sat.v8i64(<8 x i64> undef, <8 x i64> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I32 = call i32 @llvm.ssub.sat.i32(i32 undef, i32 undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V4I32 = call <4 x i32> @llvm.ssub.sat.v4i32(<4 x i32> undef, <4 x i32> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V8I32 = call <8 x i32> @llvm.ssub.sat.v8i32(<8 x i32> undef, <8 x i32> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V16I32 = call <16 x i32> @llvm.ssub.sat.v16i32(<16 x i32> undef, <16 x i32> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I16 = call i16 @llvm.ssub.sat.i16(i16 undef, i16 undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.ssub.sat.v8i16(<8 x i16> undef, <8 x i16> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.ssub.sat.v16i16(<16 x i16> undef, <16 x i16> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I16 = call <32 x i16> @llvm.ssub.sat.v32i16(<32 x i16> undef, <32 x i16> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I8 = call i8 @llvm.ssub.sat.i8(i8 undef, i8 undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = call <16 x i8> @llvm.ssub.sat.v16i8(<16 x i8> undef, <16 x i8> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32I8 = call <32 x i8> @llvm.ssub.sat.v32i8(<32 x i8> undef, <32 x i8> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64I8 = call <64 x i8> @llvm.ssub.sat.v64i8(<64 x i8> undef, <64 x i8> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SLM-LABEL: 'sub'
+; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I64 = call i64 @llvm.ssub.sat.i64(i64 undef, i64 undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V2I64 = call <2 x i64> @llvm.ssub.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %V4I64 = call <4 x i64> @llvm.ssub.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 43 for instruction: %V8I64 = call <8 x i64> @llvm.ssub.sat.v8i64(<8 x i64> undef, <8 x i64> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I32 = call i32 @llvm.ssub.sat.i32(i32 undef, i32 undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V4I32 = call <4 x i32> @llvm.ssub.sat.v4i32(<4 x i32> undef, <4 x i32> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V8I32 = call <8 x i32> @llvm.ssub.sat.v8i32(<8 x i32> undef, <8 x i32> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %V16I32 = call <16 x i32> @llvm.ssub.sat.v16i32(<16 x i32> undef, <16 x i32> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I16 = call i16 @llvm.ssub.sat.i16(i16 undef, i16 undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.ssub.sat.v8i16(<8 x i16> undef, <8 x i16> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = call <16 x i16> @llvm.ssub.sat.v16i16(<16 x i16> undef, <16 x i16> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = call <32 x i16> @llvm.ssub.sat.v32i16(<32 x i16> undef, <32 x i16> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I8 = call i8 @llvm.ssub.sat.i8(i8 undef, i8 undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = call <16 x i8> @llvm.ssub.sat.v16i8(<16 x i8> undef, <16 x i8> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I8 = call <32 x i8> @llvm.ssub.sat.v32i8(<32 x i8> undef, <32 x i8> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = call <64 x i8> @llvm.ssub.sat.v64i8(<64 x i8> undef, <64 x i8> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; GLM-LABEL: 'sub'
+; GLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I64 = call i64 @llvm.ssub.sat.i64(i64 undef, i64 undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V2I64 = call <2 x i64> @llvm.ssub.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V4I64 = call <4 x i64> @llvm.ssub.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %V8I64 = call <8 x i64> @llvm.ssub.sat.v8i64(<8 x i64> undef, <8 x i64> undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I32 = call i32 @llvm.ssub.sat.i32(i32 undef, i32 undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V4I32 = call <4 x i32> @llvm.ssub.sat.v4i32(<4 x i32> undef, <4 x i32> undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V8I32 = call <8 x i32> @llvm.ssub.sat.v8i32(<8 x i32> undef, <8 x i32> undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %V16I32 = call <16 x i32> @llvm.ssub.sat.v16i32(<16 x i32> undef, <16 x i32> undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I16 = call i16 @llvm.ssub.sat.i16(i16 undef, i16 undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.ssub.sat.v8i16(<8 x i16> undef, <8 x i16> undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = call <16 x i16> @llvm.ssub.sat.v16i16(<16 x i16> undef, <16 x i16> undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = call <32 x i16> @llvm.ssub.sat.v32i16(<32 x i16> undef, <32 x i16> undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I8 = call i8 @llvm.ssub.sat.i8(i8 undef, i8 undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = call <16 x i8> @llvm.ssub.sat.v16i8(<16 x i8> undef, <16 x i8> undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I8 = call <32 x i8> @llvm.ssub.sat.v32i8(<32 x i8> undef, <32 x i8> undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = call <64 x i8> @llvm.ssub.sat.v64i8(<64 x i8> undef, <64 x i8> undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; BTVER2-LABEL: 'sub'
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I64 = call i64 @llvm.ssub.sat.i64(i64 undef, i64 undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V2I64 = call <2 x i64> @llvm.ssub.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %V4I64 = call <4 x i64> @llvm.ssub.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 47 for instruction: %V8I64 = call <8 x i64> @llvm.ssub.sat.v8i64(<8 x i64> undef, <8 x i64> undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I32 = call i32 @llvm.ssub.sat.i32(i32 undef, i32 undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V4I32 = call <4 x i32> @llvm.ssub.sat.v4i32(<4 x i32> undef, <4 x i32> undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %V8I32 = call <8 x i32> @llvm.ssub.sat.v8i32(<8 x i32> undef, <8 x i32> undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 47 for instruction: %V16I32 = call <16 x i32> @llvm.ssub.sat.v16i32(<16 x i32> undef, <16 x i32> undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I16 = call i16 @llvm.ssub.sat.i16(i16 undef, i16 undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.ssub.sat.v8i16(<8 x i16> undef, <8 x i16> undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I16 = call <16 x i16> @llvm.ssub.sat.v16i16(<16 x i16> undef, <16 x i16> undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I16 = call <32 x i16> @llvm.ssub.sat.v32i16(<32 x i16> undef, <32 x i16> undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I8 = call i8 @llvm.ssub.sat.i8(i8 undef, i8 undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = call <16 x i8> @llvm.ssub.sat.v16i8(<16 x i8> undef, <16 x i8> undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I8 = call <32 x i8> @llvm.ssub.sat.v32i8(<32 x i8> undef, <32 x i8> undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I8 = call <64 x i8> @llvm.ssub.sat.v64i8(<64 x i8> undef, <64 x i8> undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+  %I64 = call i64 @llvm.ssub.sat.i64(i64 undef, i64 undef)
+  %V2I64 = call <2 x i64> @llvm.ssub.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
+  %V4I64 = call <4 x i64> @llvm.ssub.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
+  %V8I64 = call <8 x i64> @llvm.ssub.sat.v8i64(<8 x i64> undef, <8 x i64> undef)
+
+  %I32 = call i32 @llvm.ssub.sat.i32(i32 undef, i32 undef)
+  %V4I32  = call <4 x i32>  @llvm.ssub.sat.v4i32(<4 x i32> undef, <4 x i32> undef)
+  %V8I32  = call <8 x i32>  @llvm.ssub.sat.v8i32(<8 x i32> undef, <8 x i32> undef)
+  %V16I32 = call <16 x i32> @llvm.ssub.sat.v16i32(<16 x i32> undef, <16 x i32> undef)
+
+  %I16 = call i16 @llvm.ssub.sat.i16(i16 undef, i16 undef)
+  %V8I16  = call <8 x i16>  @llvm.ssub.sat.v8i16(<8 x i16> undef, <8 x i16> undef)
+  %V16I16 = call <16 x i16> @llvm.ssub.sat.v16i16(<16 x i16> undef, <16 x i16> undef)
+  %V32I16 = call <32 x i16> @llvm.ssub.sat.v32i16(<32 x i16> undef, <32 x i16> undef)
+
+  %I8 = call i8 @llvm.ssub.sat.i8(i8 undef, i8 undef)
+  %V16I8 = call <16 x i8> @llvm.ssub.sat.v16i8(<16 x i8> undef, <16 x i8> undef)
+  %V32I8 = call <32 x i8> @llvm.ssub.sat.v32i8(<32 x i8> undef, <32 x i8> undef)
+  %V64I8 = call <64 x i8> @llvm.ssub.sat.v64i8(<64 x i8> undef, <64 x i8> undef)
+
+  ret i32 undef
+}
diff --git a/test/Analysis/CostModel/X86/arith-usat.ll b/test/Analysis/CostModel/X86/arith-usat.ll
new file mode 100644
index 0000000000000000000000000000000000000000..e7aaba8c0245825abd8f1e1ccacb4513b1c324da
--- /dev/null
+++ b/test/Analysis/CostModel/X86/arith-usat.ll
@@ -0,0 +1,483 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
+; RUN: opt < %s -cost-model -analyze -mtriple=x86_64-apple-macosx10.8.0 -mattr=+ssse3 | FileCheck %s --check-prefixes=CHECK,SSE,SSSE3
+; RUN: opt < %s -cost-model -analyze -mtriple=x86_64-apple-macosx10.8.0 -mattr=+sse4.2 | FileCheck %s --check-prefixes=CHECK,SSE,SSE42
+; RUN: opt < %s -cost-model -analyze -mtriple=x86_64-apple-macosx10.8.0 -mattr=+avx | FileCheck %s --check-prefixes=CHECK,AVX,AVX1
+; RUN: opt < %s -cost-model -analyze -mtriple=x86_64-apple-macosx10.8.0 -mattr=+avx2 | FileCheck %s --check-prefixes=CHECK,AVX,AVX2
+; RUN: opt < %s -cost-model -analyze -mtriple=x86_64-apple-macosx10.8.0 -mattr=+avx512f | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512F
+; RUN: opt < %s -cost-model -analyze -mtriple=x86_64-apple-macosx10.8.0 -mattr=+avx512f,+avx512bw | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512BW
+; RUN: opt < %s -cost-model -analyze -mtriple=x86_64-apple-macosx10.8.0 -mattr=+avx512f,+avx512dq | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512DQ
+;
+; RUN: opt < %s -cost-model -analyze -mtriple=x86_64-apple-macosx10.8.0 -mcpu=slm | FileCheck %s --check-prefixes=SLM
+; RUN: opt < %s -cost-model -analyze -mtriple=x86_64-apple-macosx10.8.0 -mcpu=goldmont | FileCheck %s --check-prefixes=GLM
+; RUN: opt < %s -cost-model -analyze -mtriple=x86_64-apple-macosx10.8.0 -mcpu=btver2 | FileCheck %s --check-prefixes=BTVER2
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.8.0"
+
+declare i64        @llvm.uadd.sat.i64(i64, i64)
+declare <2 x i64>  @llvm.uadd.sat.v2i64(<2 x i64>, <2 x i64>)
+declare <4 x i64>  @llvm.uadd.sat.v4i64(<4 x i64>, <4 x i64>)
+declare <8 x i64>  @llvm.uadd.sat.v8i64(<8 x i64>, <8 x i64>)
+
+declare i32        @llvm.uadd.sat.i32(i32, i32)
+declare <4 x i32>  @llvm.uadd.sat.v4i32(<4 x i32>, <4 x i32>)
+declare <8 x i32>  @llvm.uadd.sat.v8i32(<8 x i32>, <8 x i32>)
+declare <16 x i32> @llvm.uadd.sat.v16i32(<16 x i32>, <16 x i32>)
+
+declare i16        @llvm.uadd.sat.i16(i16, i16)
+declare <8 x i16>  @llvm.uadd.sat.v8i16(<8 x i16>, <8 x i16>)
+declare <16 x i16> @llvm.uadd.sat.v16i16(<16 x i16>, <16 x i16>)
+declare <32 x i16> @llvm.uadd.sat.v32i16(<32 x i16>, <32 x i16>)
+
+declare i8         @llvm.uadd.sat.i8(i8,  i8)
+declare <16 x i8>  @llvm.uadd.sat.v16i8(<16 x i8>, <16 x i8>)
+declare <32 x i8>  @llvm.uadd.sat.v32i8(<32 x i8>, <32 x i8>)
+declare <64 x i8>  @llvm.uadd.sat.v64i8(<64 x i8>, <64 x i8>)
+
+define i32 @add(i32 %arg) {
+; SSSE3-LABEL: 'add'
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I64 = call i64 @llvm.uadd.sat.i64(i64 undef, i64 undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V2I64 = call <2 x i64> @llvm.uadd.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V4I64 = call <4 x i64> @llvm.uadd.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V8I64 = call <8 x i64> @llvm.uadd.sat.v8i64(<8 x i64> undef, <8 x i64> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I32 = call i32 @llvm.uadd.sat.i32(i32 undef, i32 undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V4I32 = call <4 x i32> @llvm.uadd.sat.v4i32(<4 x i32> undef, <4 x i32> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V8I32 = call <8 x i32> @llvm.uadd.sat.v8i32(<8 x i32> undef, <8 x i32> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V16I32 = call <16 x i32> @llvm.uadd.sat.v16i32(<16 x i32> undef, <16 x i32> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I16 = call i16 @llvm.uadd.sat.i16(i16 undef, i16 undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.uadd.sat.v8i16(<8 x i16> undef, <8 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = call <16 x i16> @llvm.uadd.sat.v16i16(<16 x i16> undef, <16 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = call <32 x i16> @llvm.uadd.sat.v32i16(<32 x i16> undef, <32 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I8 = call i8 @llvm.uadd.sat.i8(i8 undef, i8 undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = call <16 x i8> @llvm.uadd.sat.v16i8(<16 x i8> undef, <16 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I8 = call <32 x i8> @llvm.uadd.sat.v32i8(<32 x i8> undef, <32 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = call <64 x i8> @llvm.uadd.sat.v64i8(<64 x i8> undef, <64 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSE42-LABEL: 'add'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I64 = call i64 @llvm.uadd.sat.i64(i64 undef, i64 undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2I64 = call <2 x i64> @llvm.uadd.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4I64 = call <4 x i64> @llvm.uadd.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V8I64 = call <8 x i64> @llvm.uadd.sat.v8i64(<8 x i64> undef, <8 x i64> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I32 = call i32 @llvm.uadd.sat.i32(i32 undef, i32 undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4I32 = call <4 x i32> @llvm.uadd.sat.v4i32(<4 x i32> undef, <4 x i32> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V8I32 = call <8 x i32> @llvm.uadd.sat.v8i32(<8 x i32> undef, <8 x i32> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V16I32 = call <16 x i32> @llvm.uadd.sat.v16i32(<16 x i32> undef, <16 x i32> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I16 = call i16 @llvm.uadd.sat.i16(i16 undef, i16 undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.uadd.sat.v8i16(<8 x i16> undef, <8 x i16> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = call <16 x i16> @llvm.uadd.sat.v16i16(<16 x i16> undef, <16 x i16> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = call <32 x i16> @llvm.uadd.sat.v32i16(<32 x i16> undef, <32 x i16> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I8 = call i8 @llvm.uadd.sat.i8(i8 undef, i8 undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = call <16 x i8> @llvm.uadd.sat.v16i8(<16 x i8> undef, <16 x i8> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I8 = call <32 x i8> @llvm.uadd.sat.v32i8(<32 x i8> undef, <32 x i8> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = call <64 x i8> @llvm.uadd.sat.v64i8(<64 x i8> undef, <64 x i8> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX1-LABEL: 'add'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I64 = call i64 @llvm.uadd.sat.i64(i64 undef, i64 undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2I64 = call <2 x i64> @llvm.uadd.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V4I64 = call <4 x i64> @llvm.uadd.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V8I64 = call <8 x i64> @llvm.uadd.sat.v8i64(<8 x i64> undef, <8 x i64> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I32 = call i32 @llvm.uadd.sat.i32(i32 undef, i32 undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4I32 = call <4 x i32> @llvm.uadd.sat.v4i32(<4 x i32> undef, <4 x i32> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8I32 = call <8 x i32> @llvm.uadd.sat.v8i32(<8 x i32> undef, <8 x i32> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16I32 = call <16 x i32> @llvm.uadd.sat.v16i32(<16 x i32> undef, <16 x i32> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I16 = call i16 @llvm.uadd.sat.i16(i16 undef, i16 undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.uadd.sat.v8i16(<8 x i16> undef, <8 x i16> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I16 = call <16 x i16> @llvm.uadd.sat.v16i16(<16 x i16> undef, <16 x i16> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I16 = call <32 x i16> @llvm.uadd.sat.v32i16(<32 x i16> undef, <32 x i16> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I8 = call i8 @llvm.uadd.sat.i8(i8 undef, i8 undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = call <16 x i8> @llvm.uadd.sat.v16i8(<16 x i8> undef, <16 x i8> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I8 = call <32 x i8> @llvm.uadd.sat.v32i8(<32 x i8> undef, <32 x i8> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I8 = call <64 x i8> @llvm.uadd.sat.v64i8(<64 x i8> undef, <64 x i8> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX2-LABEL: 'add'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I64 = call i64 @llvm.uadd.sat.i64(i64 undef, i64 undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2I64 = call <2 x i64> @llvm.uadd.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4I64 = call <4 x i64> @llvm.uadd.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V8I64 = call <8 x i64> @llvm.uadd.sat.v8i64(<8 x i64> undef, <8 x i64> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I32 = call i32 @llvm.uadd.sat.i32(i32 undef, i32 undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4I32 = call <4 x i32> @llvm.uadd.sat.v4i32(<4 x i32> undef, <4 x i32> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8I32 = call <8 x i32> @llvm.uadd.sat.v8i32(<8 x i32> undef, <8 x i32> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V16I32 = call <16 x i32> @llvm.uadd.sat.v16i32(<16 x i32> undef, <16 x i32> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I16 = call i16 @llvm.uadd.sat.i16(i16 undef, i16 undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.uadd.sat.v8i16(<8 x i16> undef, <8 x i16> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.uadd.sat.v16i16(<16 x i16> undef, <16 x i16> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I16 = call <32 x i16> @llvm.uadd.sat.v32i16(<32 x i16> undef, <32 x i16> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I8 = call i8 @llvm.uadd.sat.i8(i8 undef, i8 undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = call <16 x i8> @llvm.uadd.sat.v16i8(<16 x i8> undef, <16 x i8> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32I8 = call <32 x i8> @llvm.uadd.sat.v32i8(<32 x i8> undef, <32 x i8> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64I8 = call <64 x i8> @llvm.uadd.sat.v64i8(<64 x i8> undef, <64 x i8> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX512F-LABEL: 'add'
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I64 = call i64 @llvm.uadd.sat.i64(i64 undef, i64 undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2I64 = call <2 x i64> @llvm.uadd.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4I64 = call <4 x i64> @llvm.uadd.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8I64 = call <8 x i64> @llvm.uadd.sat.v8i64(<8 x i64> undef, <8 x i64> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I32 = call i32 @llvm.uadd.sat.i32(i32 undef, i32 undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4I32 = call <4 x i32> @llvm.uadd.sat.v4i32(<4 x i32> undef, <4 x i32> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8I32 = call <8 x i32> @llvm.uadd.sat.v8i32(<8 x i32> undef, <8 x i32> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16I32 = call <16 x i32> @llvm.uadd.sat.v16i32(<16 x i32> undef, <16 x i32> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I16 = call i16 @llvm.uadd.sat.i16(i16 undef, i16 undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.uadd.sat.v8i16(<8 x i16> undef, <8 x i16> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.uadd.sat.v16i16(<16 x i16> undef, <16 x i16> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I16 = call <32 x i16> @llvm.uadd.sat.v32i16(<32 x i16> undef, <32 x i16> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I8 = call i8 @llvm.uadd.sat.i8(i8 undef, i8 undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = call <16 x i8> @llvm.uadd.sat.v16i8(<16 x i8> undef, <16 x i8> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32I8 = call <32 x i8> @llvm.uadd.sat.v32i8(<32 x i8> undef, <32 x i8> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64I8 = call <64 x i8> @llvm.uadd.sat.v64i8(<64 x i8> undef, <64 x i8> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX512BW-LABEL: 'add'
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I64 = call i64 @llvm.uadd.sat.i64(i64 undef, i64 undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2I64 = call <2 x i64> @llvm.uadd.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4I64 = call <4 x i64> @llvm.uadd.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8I64 = call <8 x i64> @llvm.uadd.sat.v8i64(<8 x i64> undef, <8 x i64> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I32 = call i32 @llvm.uadd.sat.i32(i32 undef, i32 undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4I32 = call <4 x i32> @llvm.uadd.sat.v4i32(<4 x i32> undef, <4 x i32> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8I32 = call <8 x i32> @llvm.uadd.sat.v8i32(<8 x i32> undef, <8 x i32> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16I32 = call <16 x i32> @llvm.uadd.sat.v16i32(<16 x i32> undef, <16 x i32> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I16 = call i16 @llvm.uadd.sat.i16(i16 undef, i16 undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.uadd.sat.v8i16(<8 x i16> undef, <8 x i16> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.uadd.sat.v16i16(<16 x i16> undef, <16 x i16> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.uadd.sat.v32i16(<32 x i16> undef, <32 x i16> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I8 = call i8 @llvm.uadd.sat.i8(i8 undef, i8 undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = call <16 x i8> @llvm.uadd.sat.v16i8(<16 x i8> undef, <16 x i8> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32I8 = call <32 x i8> @llvm.uadd.sat.v32i8(<32 x i8> undef, <32 x i8> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64I8 = call <64 x i8> @llvm.uadd.sat.v64i8(<64 x i8> undef, <64 x i8> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX512DQ-LABEL: 'add'
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I64 = call i64 @llvm.uadd.sat.i64(i64 undef, i64 undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2I64 = call <2 x i64> @llvm.uadd.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4I64 = call <4 x i64> @llvm.uadd.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8I64 = call <8 x i64> @llvm.uadd.sat.v8i64(<8 x i64> undef, <8 x i64> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I32 = call i32 @llvm.uadd.sat.i32(i32 undef, i32 undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4I32 = call <4 x i32> @llvm.uadd.sat.v4i32(<4 x i32> undef, <4 x i32> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8I32 = call <8 x i32> @llvm.uadd.sat.v8i32(<8 x i32> undef, <8 x i32> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16I32 = call <16 x i32> @llvm.uadd.sat.v16i32(<16 x i32> undef, <16 x i32> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I16 = call i16 @llvm.uadd.sat.i16(i16 undef, i16 undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.uadd.sat.v8i16(<8 x i16> undef, <8 x i16> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.uadd.sat.v16i16(<16 x i16> undef, <16 x i16> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I16 = call <32 x i16> @llvm.uadd.sat.v32i16(<32 x i16> undef, <32 x i16> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I8 = call i8 @llvm.uadd.sat.i8(i8 undef, i8 undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = call <16 x i8> @llvm.uadd.sat.v16i8(<16 x i8> undef, <16 x i8> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32I8 = call <32 x i8> @llvm.uadd.sat.v32i8(<32 x i8> undef, <32 x i8> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64I8 = call <64 x i8> @llvm.uadd.sat.v64i8(<64 x i8> undef, <64 x i8> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SLM-LABEL: 'add'
+; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I64 = call i64 @llvm.uadd.sat.i64(i64 undef, i64 undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.uadd.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I64 = call <4 x i64> @llvm.uadd.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I64 = call <8 x i64> @llvm.uadd.sat.v8i64(<8 x i64> undef, <8 x i64> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I32 = call i32 @llvm.uadd.sat.i32(i32 undef, i32 undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4I32 = call <4 x i32> @llvm.uadd.sat.v4i32(<4 x i32> undef, <4 x i32> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V8I32 = call <8 x i32> @llvm.uadd.sat.v8i32(<8 x i32> undef, <8 x i32> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V16I32 = call <16 x i32> @llvm.uadd.sat.v16i32(<16 x i32> undef, <16 x i32> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I16 = call i16 @llvm.uadd.sat.i16(i16 undef, i16 undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.uadd.sat.v8i16(<8 x i16> undef, <8 x i16> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = call <16 x i16> @llvm.uadd.sat.v16i16(<16 x i16> undef, <16 x i16> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = call <32 x i16> @llvm.uadd.sat.v32i16(<32 x i16> undef, <32 x i16> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I8 = call i8 @llvm.uadd.sat.i8(i8 undef, i8 undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = call <16 x i8> @llvm.uadd.sat.v16i8(<16 x i8> undef, <16 x i8> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I8 = call <32 x i8> @llvm.uadd.sat.v32i8(<32 x i8> undef, <32 x i8> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = call <64 x i8> @llvm.uadd.sat.v64i8(<64 x i8> undef, <64 x i8> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; GLM-LABEL: 'add'
+; GLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I64 = call i64 @llvm.uadd.sat.i64(i64 undef, i64 undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2I64 = call <2 x i64> @llvm.uadd.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4I64 = call <4 x i64> @llvm.uadd.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V8I64 = call <8 x i64> @llvm.uadd.sat.v8i64(<8 x i64> undef, <8 x i64> undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I32 = call i32 @llvm.uadd.sat.i32(i32 undef, i32 undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4I32 = call <4 x i32> @llvm.uadd.sat.v4i32(<4 x i32> undef, <4 x i32> undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V8I32 = call <8 x i32> @llvm.uadd.sat.v8i32(<8 x i32> undef, <8 x i32> undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V16I32 = call <16 x i32> @llvm.uadd.sat.v16i32(<16 x i32> undef, <16 x i32> undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I16 = call i16 @llvm.uadd.sat.i16(i16 undef, i16 undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.uadd.sat.v8i16(<8 x i16> undef, <8 x i16> undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = call <16 x i16> @llvm.uadd.sat.v16i16(<16 x i16> undef, <16 x i16> undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = call <32 x i16> @llvm.uadd.sat.v32i16(<32 x i16> undef, <32 x i16> undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I8 = call i8 @llvm.uadd.sat.i8(i8 undef, i8 undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = call <16 x i8> @llvm.uadd.sat.v16i8(<16 x i8> undef, <16 x i8> undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I8 = call <32 x i8> @llvm.uadd.sat.v32i8(<32 x i8> undef, <32 x i8> undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = call <64 x i8> @llvm.uadd.sat.v64i8(<64 x i8> undef, <64 x i8> undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; BTVER2-LABEL: 'add'
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I64 = call i64 @llvm.uadd.sat.i64(i64 undef, i64 undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2I64 = call <2 x i64> @llvm.uadd.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V4I64 = call <4 x i64> @llvm.uadd.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V8I64 = call <8 x i64> @llvm.uadd.sat.v8i64(<8 x i64> undef, <8 x i64> undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I32 = call i32 @llvm.uadd.sat.i32(i32 undef, i32 undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4I32 = call <4 x i32> @llvm.uadd.sat.v4i32(<4 x i32> undef, <4 x i32> undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8I32 = call <8 x i32> @llvm.uadd.sat.v8i32(<8 x i32> undef, <8 x i32> undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16I32 = call <16 x i32> @llvm.uadd.sat.v16i32(<16 x i32> undef, <16 x i32> undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I16 = call i16 @llvm.uadd.sat.i16(i16 undef, i16 undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.uadd.sat.v8i16(<8 x i16> undef, <8 x i16> undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I16 = call <16 x i16> @llvm.uadd.sat.v16i16(<16 x i16> undef, <16 x i16> undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I16 = call <32 x i16> @llvm.uadd.sat.v32i16(<32 x i16> undef, <32 x i16> undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I8 = call i8 @llvm.uadd.sat.i8(i8 undef, i8 undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = call <16 x i8> @llvm.uadd.sat.v16i8(<16 x i8> undef, <16 x i8> undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I8 = call <32 x i8> @llvm.uadd.sat.v32i8(<32 x i8> undef, <32 x i8> undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I8 = call <64 x i8> @llvm.uadd.sat.v64i8(<64 x i8> undef, <64 x i8> undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+  %I64 = call i64 @llvm.uadd.sat.i64(i64 undef, i64 undef)
+  %V2I64 = call <2 x i64> @llvm.uadd.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
+  %V4I64 = call <4 x i64> @llvm.uadd.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
+  %V8I64 = call <8 x i64> @llvm.uadd.sat.v8i64(<8 x i64> undef, <8 x i64> undef)
+
+  %I32 = call i32 @llvm.uadd.sat.i32(i32 undef, i32 undef)
+  %V4I32  = call <4 x i32>  @llvm.uadd.sat.v4i32(<4 x i32> undef, <4 x i32> undef)
+  %V8I32  = call <8 x i32>  @llvm.uadd.sat.v8i32(<8 x i32> undef, <8 x i32> undef)
+  %V16I32 = call <16 x i32> @llvm.uadd.sat.v16i32(<16 x i32> undef, <16 x i32> undef)
+
+  %I16 = call i16 @llvm.uadd.sat.i16(i16 undef, i16 undef)
+  %V8I16  = call <8 x i16>  @llvm.uadd.sat.v8i16(<8 x i16> undef, <8 x i16> undef)
+  %V16I16 = call <16 x i16> @llvm.uadd.sat.v16i16(<16 x i16> undef, <16 x i16> undef)
+  %V32I16 = call <32 x i16> @llvm.uadd.sat.v32i16(<32 x i16> undef, <32 x i16> undef)
+
+  %I8 = call i8 @llvm.uadd.sat.i8(i8 undef, i8 undef)
+  %V16I8 = call <16 x i8> @llvm.uadd.sat.v16i8(<16 x i8> undef, <16 x i8> undef)
+  %V32I8 = call <32 x i8> @llvm.uadd.sat.v32i8(<32 x i8> undef, <32 x i8> undef)
+  %V64I8 = call <64 x i8> @llvm.uadd.sat.v64i8(<64 x i8> undef, <64 x i8> undef)
+
+  ret i32 undef
+}
+
+declare i64        @llvm.usub.sat.i64(i64, i64)
+declare <2 x i64>  @llvm.usub.sat.v2i64(<2 x i64>, <2 x i64>)
+declare <4 x i64>  @llvm.usub.sat.v4i64(<4 x i64>, <4 x i64>)
+declare <8 x i64>  @llvm.usub.sat.v8i64(<8 x i64>, <8 x i64>)
+
+declare i32        @llvm.usub.sat.i32(i32, i32)
+declare <4 x i32>  @llvm.usub.sat.v4i32(<4 x i32>, <4 x i32>)
+declare <8 x i32>  @llvm.usub.sat.v8i32(<8 x i32>, <8 x i32>)
+declare <16 x i32> @llvm.usub.sat.v16i32(<16 x i32>, <16 x i32>)
+
+declare i16        @llvm.usub.sat.i16(i16, i16)
+declare <8 x i16>  @llvm.usub.sat.v8i16(<8 x i16>, <8 x i16>)
+declare <16 x i16> @llvm.usub.sat.v16i16(<16 x i16>, <16 x i16>)
+declare <32 x i16> @llvm.usub.sat.v32i16(<32 x i16>, <32 x i16>)
+
+declare i8         @llvm.usub.sat.i8(i8,  i8)
+declare <16 x i8>  @llvm.usub.sat.v16i8(<16 x i8>, <16 x i8>)
+declare <32 x i8>  @llvm.usub.sat.v32i8(<32 x i8>, <32 x i8>)
+declare <64 x i8>  @llvm.usub.sat.v64i8(<64 x i8>, <64 x i8>)
+
+define i32 @sub(i32 %arg) {
+; SSSE3-LABEL: 'sub'
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I64 = call i64 @llvm.usub.sat.i64(i64 undef, i64 undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V2I64 = call <2 x i64> @llvm.usub.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V4I64 = call <4 x i64> @llvm.usub.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V8I64 = call <8 x i64> @llvm.usub.sat.v8i64(<8 x i64> undef, <8 x i64> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I32 = call i32 @llvm.usub.sat.i32(i32 undef, i32 undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V4I32 = call <4 x i32> @llvm.usub.sat.v4i32(<4 x i32> undef, <4 x i32> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V8I32 = call <8 x i32> @llvm.usub.sat.v8i32(<8 x i32> undef, <8 x i32> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V16I32 = call <16 x i32> @llvm.usub.sat.v16i32(<16 x i32> undef, <16 x i32> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I16 = call i16 @llvm.usub.sat.i16(i16 undef, i16 undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.usub.sat.v8i16(<8 x i16> undef, <8 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = call <16 x i16> @llvm.usub.sat.v16i16(<16 x i16> undef, <16 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = call <32 x i16> @llvm.usub.sat.v32i16(<32 x i16> undef, <32 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I8 = call i8 @llvm.usub.sat.i8(i8 undef, i8 undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = call <16 x i8> @llvm.usub.sat.v16i8(<16 x i8> undef, <16 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I8 = call <32 x i8> @llvm.usub.sat.v32i8(<32 x i8> undef, <32 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = call <64 x i8> @llvm.usub.sat.v64i8(<64 x i8> undef, <64 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSE42-LABEL: 'sub'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I64 = call i64 @llvm.usub.sat.i64(i64 undef, i64 undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2I64 = call <2 x i64> @llvm.usub.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4I64 = call <4 x i64> @llvm.usub.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V8I64 = call <8 x i64> @llvm.usub.sat.v8i64(<8 x i64> undef, <8 x i64> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I32 = call i32 @llvm.usub.sat.i32(i32 undef, i32 undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = call <4 x i32> @llvm.usub.sat.v4i32(<4 x i32> undef, <4 x i32> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I32 = call <8 x i32> @llvm.usub.sat.v8i32(<8 x i32> undef, <8 x i32> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I32 = call <16 x i32> @llvm.usub.sat.v16i32(<16 x i32> undef, <16 x i32> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I16 = call i16 @llvm.usub.sat.i16(i16 undef, i16 undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.usub.sat.v8i16(<8 x i16> undef, <8 x i16> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = call <16 x i16> @llvm.usub.sat.v16i16(<16 x i16> undef, <16 x i16> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = call <32 x i16> @llvm.usub.sat.v32i16(<32 x i16> undef, <32 x i16> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I8 = call i8 @llvm.usub.sat.i8(i8 undef, i8 undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = call <16 x i8> @llvm.usub.sat.v16i8(<16 x i8> undef, <16 x i8> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I8 = call <32 x i8> @llvm.usub.sat.v32i8(<32 x i8> undef, <32 x i8> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = call <64 x i8> @llvm.usub.sat.v64i8(<64 x i8> undef, <64 x i8> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX1-LABEL: 'sub'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I64 = call i64 @llvm.usub.sat.i64(i64 undef, i64 undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2I64 = call <2 x i64> @llvm.usub.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V4I64 = call <4 x i64> @llvm.usub.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V8I64 = call <8 x i64> @llvm.usub.sat.v8i64(<8 x i64> undef, <8 x i64> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I32 = call i32 @llvm.usub.sat.i32(i32 undef, i32 undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = call <4 x i32> @llvm.usub.sat.v4i32(<4 x i32> undef, <4 x i32> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V8I32 = call <8 x i32> @llvm.usub.sat.v8i32(<8 x i32> undef, <8 x i32> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V16I32 = call <16 x i32> @llvm.usub.sat.v16i32(<16 x i32> undef, <16 x i32> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I16 = call i16 @llvm.usub.sat.i16(i16 undef, i16 undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.usub.sat.v8i16(<8 x i16> undef, <8 x i16> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I16 = call <16 x i16> @llvm.usub.sat.v16i16(<16 x i16> undef, <16 x i16> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I16 = call <32 x i16> @llvm.usub.sat.v32i16(<32 x i16> undef, <32 x i16> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I8 = call i8 @llvm.usub.sat.i8(i8 undef, i8 undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = call <16 x i8> @llvm.usub.sat.v16i8(<16 x i8> undef, <16 x i8> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I8 = call <32 x i8> @llvm.usub.sat.v32i8(<32 x i8> undef, <32 x i8> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I8 = call <64 x i8> @llvm.usub.sat.v64i8(<64 x i8> undef, <64 x i8> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX2-LABEL: 'sub'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I64 = call i64 @llvm.usub.sat.i64(i64 undef, i64 undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2I64 = call <2 x i64> @llvm.usub.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4I64 = call <4 x i64> @llvm.usub.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V8I64 = call <8 x i64> @llvm.usub.sat.v8i64(<8 x i64> undef, <8 x i64> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I32 = call i32 @llvm.usub.sat.i32(i32 undef, i32 undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = call <4 x i32> @llvm.usub.sat.v4i32(<4 x i32> undef, <4 x i32> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = call <8 x i32> @llvm.usub.sat.v8i32(<8 x i32> undef, <8 x i32> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I32 = call <16 x i32> @llvm.usub.sat.v16i32(<16 x i32> undef, <16 x i32> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I16 = call i16 @llvm.usub.sat.i16(i16 undef, i16 undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.usub.sat.v8i16(<8 x i16> undef, <8 x i16> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.usub.sat.v16i16(<16 x i16> undef, <16 x i16> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I16 = call <32 x i16> @llvm.usub.sat.v32i16(<32 x i16> undef, <32 x i16> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I8 = call i8 @llvm.usub.sat.i8(i8 undef, i8 undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = call <16 x i8> @llvm.usub.sat.v16i8(<16 x i8> undef, <16 x i8> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32I8 = call <32 x i8> @llvm.usub.sat.v32i8(<32 x i8> undef, <32 x i8> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64I8 = call <64 x i8> @llvm.usub.sat.v64i8(<64 x i8> undef, <64 x i8> undef)
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX512F-LABEL: 'sub'
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I64 = call i64 @llvm.usub.sat.i64(i64 undef, i64 undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I64 = call <2 x i64> @llvm.usub.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I64 = call <4 x i64> @llvm.usub.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I64 = call <8 x i64> @llvm.usub.sat.v8i64(<8 x i64> undef, <8 x i64> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I32 = call i32 @llvm.usub.sat.i32(i32 undef, i32 undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = call <4 x i32> @llvm.usub.sat.v4i32(<4 x i32> undef, <4 x i32> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = call <8 x i32> @llvm.usub.sat.v8i32(<8 x i32> undef, <8 x i32> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I32 = call <16 x i32> @llvm.usub.sat.v16i32(<16 x i32> undef, <16 x i32> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I16 = call i16 @llvm.usub.sat.i16(i16 undef, i16 undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.usub.sat.v8i16(<8 x i16> undef, <8 x i16> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.usub.sat.v16i16(<16 x i16> undef, <16 x i16> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I16 = call <32 x i16> @llvm.usub.sat.v32i16(<32 x i16> undef, <32 x i16> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I8 = call i8 @llvm.usub.sat.i8(i8 undef, i8 undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = call <16 x i8> @llvm.usub.sat.v16i8(<16 x i8> undef, <16 x i8> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32I8 = call <32 x i8> @llvm.usub.sat.v32i8(<32 x i8> undef, <32 x i8> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64I8 = call <64 x i8> @llvm.usub.sat.v64i8(<64 x i8> undef, <64 x i8> undef)
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX512BW-LABEL: 'sub'
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I64 = call i64 @llvm.usub.sat.i64(i64 undef, i64 undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I64 = call <2 x i64> @llvm.usub.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I64 = call <4 x i64> @llvm.usub.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I64 = call <8 x i64> @llvm.usub.sat.v8i64(<8 x i64> undef, <8 x i64> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I32 = call i32 @llvm.usub.sat.i32(i32 undef, i32 undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = call <4 x i32> @llvm.usub.sat.v4i32(<4 x i32> undef, <4 x i32> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = call <8 x i32> @llvm.usub.sat.v8i32(<8 x i32> undef, <8 x i32> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I32 = call <16 x i32> @llvm.usub.sat.v16i32(<16 x i32> undef, <16 x i32> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I16 = call i16 @llvm.usub.sat.i16(i16 undef, i16 undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.usub.sat.v8i16(<8 x i16> undef, <8 x i16> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.usub.sat.v16i16(<16 x i16> undef, <16 x i16> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = call <32 x i16> @llvm.usub.sat.v32i16(<32 x i16> undef, <32 x i16> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I8 = call i8 @llvm.usub.sat.i8(i8 undef, i8 undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = call <16 x i8> @llvm.usub.sat.v16i8(<16 x i8> undef, <16 x i8> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32I8 = call <32 x i8> @llvm.usub.sat.v32i8(<32 x i8> undef, <32 x i8> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64I8 = call <64 x i8> @llvm.usub.sat.v64i8(<64 x i8> undef, <64 x i8> undef)
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX512DQ-LABEL: 'sub'
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I64 = call i64 @llvm.usub.sat.i64(i64 undef, i64 undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I64 = call <2 x i64> @llvm.usub.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I64 = call <4 x i64> @llvm.usub.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I64 = call <8 x i64> @llvm.usub.sat.v8i64(<8 x i64> undef, <8 x i64> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I32 = call i32 @llvm.usub.sat.i32(i32 undef, i32 undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = call <4 x i32> @llvm.usub.sat.v4i32(<4 x i32> undef, <4 x i32> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = call <8 x i32> @llvm.usub.sat.v8i32(<8 x i32> undef, <8 x i32> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I32 = call <16 x i32> @llvm.usub.sat.v16i32(<16 x i32> undef, <16 x i32> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I16 = call i16 @llvm.usub.sat.i16(i16 undef, i16 undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.usub.sat.v8i16(<8 x i16> undef, <8 x i16> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = call <16 x i16> @llvm.usub.sat.v16i16(<16 x i16> undef, <16 x i16> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I16 = call <32 x i16> @llvm.usub.sat.v32i16(<32 x i16> undef, <32 x i16> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I8 = call i8 @llvm.usub.sat.i8(i8 undef, i8 undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = call <16 x i8> @llvm.usub.sat.v16i8(<16 x i8> undef, <16 x i8> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32I8 = call <32 x i8> @llvm.usub.sat.v32i8(<32 x i8> undef, <32 x i8> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64I8 = call <64 x i8> @llvm.usub.sat.v64i8(<64 x i8> undef, <64 x i8> undef)
+; AVX512DQ-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SLM-LABEL: 'sub'
+; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I64 = call i64 @llvm.usub.sat.i64(i64 undef, i64 undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.usub.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I64 = call <4 x i64> @llvm.usub.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I64 = call <8 x i64> @llvm.usub.sat.v8i64(<8 x i64> undef, <8 x i64> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I32 = call i32 @llvm.usub.sat.i32(i32 undef, i32 undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = call <4 x i32> @llvm.usub.sat.v4i32(<4 x i32> undef, <4 x i32> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I32 = call <8 x i32> @llvm.usub.sat.v8i32(<8 x i32> undef, <8 x i32> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I32 = call <16 x i32> @llvm.usub.sat.v16i32(<16 x i32> undef, <16 x i32> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I16 = call i16 @llvm.usub.sat.i16(i16 undef, i16 undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.usub.sat.v8i16(<8 x i16> undef, <8 x i16> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = call <16 x i16> @llvm.usub.sat.v16i16(<16 x i16> undef, <16 x i16> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = call <32 x i16> @llvm.usub.sat.v32i16(<32 x i16> undef, <32 x i16> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I8 = call i8 @llvm.usub.sat.i8(i8 undef, i8 undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = call <16 x i8> @llvm.usub.sat.v16i8(<16 x i8> undef, <16 x i8> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I8 = call <32 x i8> @llvm.usub.sat.v32i8(<32 x i8> undef, <32 x i8> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = call <64 x i8> @llvm.usub.sat.v64i8(<64 x i8> undef, <64 x i8> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; GLM-LABEL: 'sub'
+; GLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I64 = call i64 @llvm.usub.sat.i64(i64 undef, i64 undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2I64 = call <2 x i64> @llvm.usub.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4I64 = call <4 x i64> @llvm.usub.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V8I64 = call <8 x i64> @llvm.usub.sat.v8i64(<8 x i64> undef, <8 x i64> undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I32 = call i32 @llvm.usub.sat.i32(i32 undef, i32 undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = call <4 x i32> @llvm.usub.sat.v4i32(<4 x i32> undef, <4 x i32> undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I32 = call <8 x i32> @llvm.usub.sat.v8i32(<8 x i32> undef, <8 x i32> undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I32 = call <16 x i32> @llvm.usub.sat.v16i32(<16 x i32> undef, <16 x i32> undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I16 = call i16 @llvm.usub.sat.i16(i16 undef, i16 undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.usub.sat.v8i16(<8 x i16> undef, <8 x i16> undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = call <16 x i16> @llvm.usub.sat.v16i16(<16 x i16> undef, <16 x i16> undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = call <32 x i16> @llvm.usub.sat.v32i16(<32 x i16> undef, <32 x i16> undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I8 = call i8 @llvm.usub.sat.i8(i8 undef, i8 undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = call <16 x i8> @llvm.usub.sat.v16i8(<16 x i8> undef, <16 x i8> undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I8 = call <32 x i8> @llvm.usub.sat.v32i8(<32 x i8> undef, <32 x i8> undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = call <64 x i8> @llvm.usub.sat.v64i8(<64 x i8> undef, <64 x i8> undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; BTVER2-LABEL: 'sub'
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I64 = call i64 @llvm.usub.sat.i64(i64 undef, i64 undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2I64 = call <2 x i64> @llvm.usub.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V4I64 = call <4 x i64> @llvm.usub.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V8I64 = call <8 x i64> @llvm.usub.sat.v8i64(<8 x i64> undef, <8 x i64> undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I32 = call i32 @llvm.usub.sat.i32(i32 undef, i32 undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = call <4 x i32> @llvm.usub.sat.v4i32(<4 x i32> undef, <4 x i32> undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V8I32 = call <8 x i32> @llvm.usub.sat.v8i32(<8 x i32> undef, <8 x i32> undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V16I32 = call <16 x i32> @llvm.usub.sat.v16i32(<16 x i32> undef, <16 x i32> undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I16 = call i16 @llvm.usub.sat.i16(i16 undef, i16 undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = call <8 x i16> @llvm.usub.sat.v8i16(<8 x i16> undef, <8 x i16> undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I16 = call <16 x i16> @llvm.usub.sat.v16i16(<16 x i16> undef, <16 x i16> undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I16 = call <32 x i16> @llvm.usub.sat.v32i16(<32 x i16> undef, <32 x i16> undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I8 = call i8 @llvm.usub.sat.i8(i8 undef, i8 undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = call <16 x i8> @llvm.usub.sat.v16i8(<16 x i8> undef, <16 x i8> undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I8 = call <32 x i8> @llvm.usub.sat.v32i8(<32 x i8> undef, <32 x i8> undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I8 = call <64 x i8> @llvm.usub.sat.v64i8(<64 x i8> undef, <64 x i8> undef)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+  %I64 = call i64 @llvm.usub.sat.i64(i64 undef, i64 undef)
+  %V2I64 = call <2 x i64> @llvm.usub.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
+  %V4I64 = call <4 x i64> @llvm.usub.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
+  %V8I64 = call <8 x i64> @llvm.usub.sat.v8i64(<8 x i64> undef, <8 x i64> undef)
+
+  %I32 = call i32 @llvm.usub.sat.i32(i32 undef, i32 undef)
+  %V4I32  = call <4 x i32>  @llvm.usub.sat.v4i32(<4 x i32> undef, <4 x i32> undef)
+  %V8I32  = call <8 x i32>  @llvm.usub.sat.v8i32(<8 x i32> undef, <8 x i32> undef)
+  %V16I32 = call <16 x i32> @llvm.usub.sat.v16i32(<16 x i32> undef, <16 x i32> undef)
+
+  %I16 = call i16 @llvm.usub.sat.i16(i16 undef, i16 undef)
+  %V8I16  = call <8 x i16>  @llvm.usub.sat.v8i16(<8 x i16> undef, <8 x i16> undef)
+  %V16I16 = call <16 x i16> @llvm.usub.sat.v16i16(<16 x i16> undef, <16 x i16> undef)
+  %V32I16 = call <32 x i16> @llvm.usub.sat.v32i16(<32 x i16> undef, <32 x i16> undef)
+
+  %I8 = call i8 @llvm.usub.sat.i8(i8 undef, i8 undef)
+  %V16I8 = call <16 x i8> @llvm.usub.sat.v16i8(<16 x i8> undef, <16 x i8> undef)
+  %V32I8 = call <32 x i8> @llvm.usub.sat.v32i8(<32 x i8> undef, <32 x i8> undef)
+  %V64I8 = call <64 x i8> @llvm.usub.sat.v64i8(<64 x i8> undef, <64 x i8> undef)
+
+  ret i32 undef
+}
diff --git a/test/Analysis/CostModel/X86/cmp.ll b/test/Analysis/CostModel/X86/cmp.ll
deleted file mode 100644
index 044de0c4f13f761b65907a9bdd87ad47c99b61f2..0000000000000000000000000000000000000000
--- a/test/Analysis/CostModel/X86/cmp.ll
+++ /dev/null
@@ -1,330 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
-; RUN: opt < %s -mtriple=x86_64-apple-macosx10.8.0 -cost-model -analyze -mattr=+sse2 | FileCheck %s --check-prefixes=CHECK,SSE,SSE2
-; RUN: opt < %s -mtriple=x86_64-apple-macosx10.8.0 -cost-model -analyze -mattr=+sse3 | FileCheck %s --check-prefixes=CHECK,SSE,SSE3
-; RUN: opt < %s -mtriple=x86_64-apple-macosx10.8.0 -cost-model -analyze -mattr=+ssse3 | FileCheck %s --check-prefixes=CHECK,SSE,SSSE3
-; RUN: opt < %s -mtriple=x86_64-apple-macosx10.8.0 -cost-model -analyze -mattr=+sse4.1 | FileCheck %s --check-prefixes=CHECK,SSE,SSE41
-; RUN: opt < %s -mtriple=x86_64-apple-macosx10.8.0 -cost-model -analyze -mattr=+sse4.2 | FileCheck %s --check-prefixes=CHECK,SSE,SSE42
-; RUN: opt < %s -mtriple=x86_64-apple-macosx10.8.0 -cost-model -analyze -mattr=+avx | FileCheck %s --check-prefixes=CHECK,AVX,AVX1
-; RUN: opt < %s -mtriple=x86_64-apple-macosx10.8.0 -cost-model -analyze -mattr=+avx2 | FileCheck %s --check-prefixes=CHECK,AVX,AVX2
-; RUN: opt < %s -mtriple=x86_64-apple-macosx10.8.0 -cost-model -analyze -mattr=+avx512f | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512F
-; RUN: opt < %s -mtriple=x86_64-apple-macosx10.8.0 -cost-model -analyze -mattr=+avx512f,+avx512bw | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512BW
-;
-; RUN: opt < %s -mtriple=x86_64-apple-macosx10.8.0 -cost-model -analyze -mcpu=slm | FileCheck %s --check-prefixes=CHECK,SSE,SSE42
-; RUN: opt < %s -mtriple=x86_64-apple-macosx10.8.0 -cost-model -analyze -mcpu=goldmont | FileCheck %s --check-prefixes=CHECK,SSE,SSE42
-; RUN: opt < %s -mtriple=x86_64-apple-macosx10.8.0 -cost-model -analyze -mcpu=btver2 | FileCheck %s --check-prefixes=BTVER2
-
-define i32 @cmp_float(i32 %arg) {
-; SSE2-LABEL: 'cmp_float'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2F32 = fcmp olt <2 x float> undef, undef
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4F32 = fcmp olt <4 x float> undef, undef
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V8F32 = fcmp olt <8 x float> undef, undef
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V16F32 = fcmp olt <16 x float> undef, undef
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2F64 = fcmp olt <2 x double> undef, undef
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4F64 = fcmp olt <4 x double> undef, undef
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V8F64 = fcmp olt <8 x double> undef, undef
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V16F64 = fcmp olt <16 x double> undef, undef
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
-;
-; SSE3-LABEL: 'cmp_float'
-; SSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2F32 = fcmp olt <2 x float> undef, undef
-; SSE3-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4F32 = fcmp olt <4 x float> undef, undef
-; SSE3-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V8F32 = fcmp olt <8 x float> undef, undef
-; SSE3-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V16F32 = fcmp olt <16 x float> undef, undef
-; SSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2F64 = fcmp olt <2 x double> undef, undef
-; SSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4F64 = fcmp olt <4 x double> undef, undef
-; SSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V8F64 = fcmp olt <8 x double> undef, undef
-; SSE3-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V16F64 = fcmp olt <16 x double> undef, undef
-; SSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
-;
-; SSSE3-LABEL: 'cmp_float'
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2F32 = fcmp olt <2 x float> undef, undef
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4F32 = fcmp olt <4 x float> undef, undef
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V8F32 = fcmp olt <8 x float> undef, undef
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V16F32 = fcmp olt <16 x float> undef, undef
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2F64 = fcmp olt <2 x double> undef, undef
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4F64 = fcmp olt <4 x double> undef, undef
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V8F64 = fcmp olt <8 x double> undef, undef
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V16F64 = fcmp olt <16 x double> undef, undef
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
-;
-; SSE41-LABEL: 'cmp_float'
-; SSE41-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2F32 = fcmp olt <2 x float> undef, undef
-; SSE41-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4F32 = fcmp olt <4 x float> undef, undef
-; SSE41-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V8F32 = fcmp olt <8 x float> undef, undef
-; SSE41-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V16F32 = fcmp olt <16 x float> undef, undef
-; SSE41-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2F64 = fcmp olt <2 x double> undef, undef
-; SSE41-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4F64 = fcmp olt <4 x double> undef, undef
-; SSE41-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V8F64 = fcmp olt <8 x double> undef, undef
-; SSE41-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V16F64 = fcmp olt <16 x double> undef, undef
-; SSE41-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
-;
-; SSE42-LABEL: 'cmp_float'
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F32 = fcmp olt <2 x float> undef, undef
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F32 = fcmp olt <4 x float> undef, undef
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8F32 = fcmp olt <8 x float> undef, undef
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16F32 = fcmp olt <16 x float> undef, undef
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F64 = fcmp olt <2 x double> undef, undef
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F64 = fcmp olt <4 x double> undef, undef
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F64 = fcmp olt <8 x double> undef, undef
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16F64 = fcmp olt <16 x double> undef, undef
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
-;
-; AVX-LABEL: 'cmp_float'
-; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F32 = fcmp olt <2 x float> undef, undef
-; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F32 = fcmp olt <4 x float> undef, undef
-; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F32 = fcmp olt <8 x float> undef, undef
-; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16F32 = fcmp olt <16 x float> undef, undef
-; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F64 = fcmp olt <2 x double> undef, undef
-; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F64 = fcmp olt <4 x double> undef, undef
-; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8F64 = fcmp olt <8 x double> undef, undef
-; AVX-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16F64 = fcmp olt <16 x double> undef, undef
-; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
-;
-; AVX512-LABEL: 'cmp_float'
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F32 = fcmp olt <2 x float> undef, undef
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F32 = fcmp olt <4 x float> undef, undef
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F32 = fcmp olt <8 x float> undef, undef
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16F32 = fcmp olt <16 x float> undef, undef
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F64 = fcmp olt <2 x double> undef, undef
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F64 = fcmp olt <4 x double> undef, undef
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F64 = fcmp olt <8 x double> undef, undef
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16F64 = fcmp olt <16 x double> undef, undef
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
-;
-; BTVER2-LABEL: 'cmp_float'
-; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F32 = fcmp olt <2 x float> undef, undef
-; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F32 = fcmp olt <4 x float> undef, undef
-; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F32 = fcmp olt <8 x float> undef, undef
-; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16F32 = fcmp olt <16 x float> undef, undef
-; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F64 = fcmp olt <2 x double> undef, undef
-; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F64 = fcmp olt <4 x double> undef, undef
-; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8F64 = fcmp olt <8 x double> undef, undef
-; BTVER2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16F64 = fcmp olt <16 x double> undef, undef
-; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
-;
-  %V2F32 = fcmp olt <2 x float> undef, undef
-  %V4F32 = fcmp olt <4 x float> undef, undef
-  %V8F32 = fcmp olt <8 x float> undef, undef
-  %V16F32 = fcmp olt <16 x float> undef, undef
-
-  %V2F64 = fcmp olt <2 x double> undef, undef
-  %V4F64 = fcmp olt <4 x double> undef, undef
-  %V8F64 = fcmp olt <8 x double> undef, undef
-  %V16F64 = fcmp olt <16 x double> undef, undef
-
-  ret i32 undef
-}
-
-define i32 @cmp_int(i32 %arg) {
-; SSE2-LABEL: 'cmp_int'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = icmp eq <16 x i8> undef, undef
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I8 = icmp eq <32 x i8> undef, undef
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = icmp eq <64 x i8> undef, undef
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V128I8 = icmp eq <128 x i8> undef, undef
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = icmp eq <8 x i16> undef, undef
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = icmp eq <16 x i16> undef, undef
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = icmp eq <32 x i16> undef, undef
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I16 = icmp eq <64 x i16> undef, undef
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = icmp eq <4 x i32> undef, undef
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = icmp eq <8 x i32> undef, undef
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I32 = icmp eq <16 x i32> undef, undef
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I32 = icmp eq <32 x i32> undef, undef
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2I64 = icmp eq <2 x i64> undef, undef
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V4I64 = icmp eq <4 x i64> undef, undef
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V8I64 = icmp eq <8 x i64> undef, undef
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %V16I64 = icmp eq <16 x i64> undef, undef
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
-;
-; SSE3-LABEL: 'cmp_int'
-; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = icmp eq <16 x i8> undef, undef
-; SSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I8 = icmp eq <32 x i8> undef, undef
-; SSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = icmp eq <64 x i8> undef, undef
-; SSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V128I8 = icmp eq <128 x i8> undef, undef
-; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = icmp eq <8 x i16> undef, undef
-; SSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = icmp eq <16 x i16> undef, undef
-; SSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = icmp eq <32 x i16> undef, undef
-; SSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I16 = icmp eq <64 x i16> undef, undef
-; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = icmp eq <4 x i32> undef, undef
-; SSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = icmp eq <8 x i32> undef, undef
-; SSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I32 = icmp eq <16 x i32> undef, undef
-; SSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I32 = icmp eq <32 x i32> undef, undef
-; SSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2I64 = icmp eq <2 x i64> undef, undef
-; SSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V4I64 = icmp eq <4 x i64> undef, undef
-; SSE3-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V8I64 = icmp eq <8 x i64> undef, undef
-; SSE3-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %V16I64 = icmp eq <16 x i64> undef, undef
-; SSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
-;
-; SSSE3-LABEL: 'cmp_int'
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = icmp eq <16 x i8> undef, undef
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I8 = icmp eq <32 x i8> undef, undef
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = icmp eq <64 x i8> undef, undef
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V128I8 = icmp eq <128 x i8> undef, undef
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = icmp eq <8 x i16> undef, undef
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = icmp eq <16 x i16> undef, undef
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = icmp eq <32 x i16> undef, undef
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I16 = icmp eq <64 x i16> undef, undef
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = icmp eq <4 x i32> undef, undef
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = icmp eq <8 x i32> undef, undef
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I32 = icmp eq <16 x i32> undef, undef
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I32 = icmp eq <32 x i32> undef, undef
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2I64 = icmp eq <2 x i64> undef, undef
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V4I64 = icmp eq <4 x i64> undef, undef
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V8I64 = icmp eq <8 x i64> undef, undef
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %V16I64 = icmp eq <16 x i64> undef, undef
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
-;
-; SSE41-LABEL: 'cmp_int'
-; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = icmp eq <16 x i8> undef, undef
-; SSE41-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I8 = icmp eq <32 x i8> undef, undef
-; SSE41-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = icmp eq <64 x i8> undef, undef
-; SSE41-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V128I8 = icmp eq <128 x i8> undef, undef
-; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = icmp eq <8 x i16> undef, undef
-; SSE41-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = icmp eq <16 x i16> undef, undef
-; SSE41-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = icmp eq <32 x i16> undef, undef
-; SSE41-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I16 = icmp eq <64 x i16> undef, undef
-; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = icmp eq <4 x i32> undef, undef
-; SSE41-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = icmp eq <8 x i32> undef, undef
-; SSE41-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I32 = icmp eq <16 x i32> undef, undef
-; SSE41-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I32 = icmp eq <32 x i32> undef, undef
-; SSE41-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2I64 = icmp eq <2 x i64> undef, undef
-; SSE41-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V4I64 = icmp eq <4 x i64> undef, undef
-; SSE41-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V8I64 = icmp eq <8 x i64> undef, undef
-; SSE41-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %V16I64 = icmp eq <16 x i64> undef, undef
-; SSE41-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
-;
-; SSE42-LABEL: 'cmp_int'
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = icmp eq <16 x i8> undef, undef
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I8 = icmp eq <32 x i8> undef, undef
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = icmp eq <64 x i8> undef, undef
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V128I8 = icmp eq <128 x i8> undef, undef
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = icmp eq <8 x i16> undef, undef
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = icmp eq <16 x i16> undef, undef
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = icmp eq <32 x i16> undef, undef
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I16 = icmp eq <64 x i16> undef, undef
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = icmp eq <4 x i32> undef, undef
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = icmp eq <8 x i32> undef, undef
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I32 = icmp eq <16 x i32> undef, undef
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I32 = icmp eq <32 x i32> undef, undef
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = icmp eq <2 x i64> undef, undef
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I64 = icmp eq <4 x i64> undef, undef
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I64 = icmp eq <8 x i64> undef, undef
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I64 = icmp eq <16 x i64> undef, undef
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
-;
-; AVX1-LABEL: 'cmp_int'
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = icmp eq <16 x i8> undef, undef
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I8 = icmp eq <32 x i8> undef, undef
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I8 = icmp eq <64 x i8> undef, undef
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V128I8 = icmp eq <128 x i8> undef, undef
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = icmp eq <8 x i16> undef, undef
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I16 = icmp eq <16 x i16> undef, undef
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I16 = icmp eq <32 x i16> undef, undef
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V64I16 = icmp eq <64 x i16> undef, undef
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = icmp eq <4 x i32> undef, undef
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I32 = icmp eq <8 x i32> undef, undef
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I32 = icmp eq <16 x i32> undef, undef
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V32I32 = icmp eq <32 x i32> undef, undef
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = icmp eq <2 x i64> undef, undef
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4I64 = icmp eq <4 x i64> undef, undef
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8I64 = icmp eq <8 x i64> undef, undef
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16I64 = icmp eq <16 x i64> undef, undef
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
-;
-; AVX2-LABEL: 'cmp_int'
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = icmp eq <16 x i8> undef, undef
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32I8 = icmp eq <32 x i8> undef, undef
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64I8 = icmp eq <64 x i8> undef, undef
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V128I8 = icmp eq <128 x i8> undef, undef
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = icmp eq <8 x i16> undef, undef
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = icmp eq <16 x i16> undef, undef
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I16 = icmp eq <32 x i16> undef, undef
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I16 = icmp eq <64 x i16> undef, undef
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = icmp eq <4 x i32> undef, undef
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = icmp eq <8 x i32> undef, undef
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I32 = icmp eq <16 x i32> undef, undef
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I32 = icmp eq <32 x i32> undef, undef
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = icmp eq <2 x i64> undef, undef
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = icmp eq <4 x i64> undef, undef
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I64 = icmp eq <8 x i64> undef, undef
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I64 = icmp eq <16 x i64> undef, undef
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
-;
-; AVX512F-LABEL: 'cmp_int'
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = icmp eq <16 x i8> undef, undef
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32I8 = icmp eq <32 x i8> undef, undef
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64I8 = icmp eq <64 x i8> undef, undef
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V128I8 = icmp eq <128 x i8> undef, undef
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = icmp eq <8 x i16> undef, undef
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = icmp eq <16 x i16> undef, undef
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I16 = icmp eq <32 x i16> undef, undef
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I16 = icmp eq <64 x i16> undef, undef
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = icmp eq <4 x i32> undef, undef
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = icmp eq <8 x i32> undef, undef
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I32 = icmp eq <16 x i32> undef, undef
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I32 = icmp eq <32 x i32> undef, undef
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = icmp eq <2 x i64> undef, undef
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = icmp eq <4 x i64> undef, undef
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = icmp eq <8 x i64> undef, undef
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I64 = icmp eq <16 x i64> undef, undef
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
-;
-; AVX512BW-LABEL: 'cmp_int'
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = icmp eq <16 x i8> undef, undef
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32I8 = icmp eq <32 x i8> undef, undef
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64I8 = icmp eq <64 x i8> undef, undef
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128I8 = icmp eq <128 x i8> undef, undef
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = icmp eq <8 x i16> undef, undef
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = icmp eq <16 x i16> undef, undef
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = icmp eq <32 x i16> undef, undef
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64I16 = icmp eq <64 x i16> undef, undef
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = icmp eq <4 x i32> undef, undef
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = icmp eq <8 x i32> undef, undef
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I32 = icmp eq <16 x i32> undef, undef
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I32 = icmp eq <32 x i32> undef, undef
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = icmp eq <2 x i64> undef, undef
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = icmp eq <4 x i64> undef, undef
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = icmp eq <8 x i64> undef, undef
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I64 = icmp eq <16 x i64> undef, undef
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
-;
-; BTVER2-LABEL: 'cmp_int'
-; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = icmp eq <16 x i8> undef, undef
-; BTVER2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I8 = icmp eq <32 x i8> undef, undef
-; BTVER2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I8 = icmp eq <64 x i8> undef, undef
-; BTVER2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V128I8 = icmp eq <128 x i8> undef, undef
-; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = icmp eq <8 x i16> undef, undef
-; BTVER2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I16 = icmp eq <16 x i16> undef, undef
-; BTVER2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I16 = icmp eq <32 x i16> undef, undef
-; BTVER2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V64I16 = icmp eq <64 x i16> undef, undef
-; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = icmp eq <4 x i32> undef, undef
-; BTVER2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I32 = icmp eq <8 x i32> undef, undef
-; BTVER2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I32 = icmp eq <16 x i32> undef, undef
-; BTVER2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V32I32 = icmp eq <32 x i32> undef, undef
-; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = icmp eq <2 x i64> undef, undef
-; BTVER2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4I64 = icmp eq <4 x i64> undef, undef
-; BTVER2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8I64 = icmp eq <8 x i64> undef, undef
-; BTVER2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16I64 = icmp eq <16 x i64> undef, undef
-; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
-;
-  %V16I8 = icmp eq <16 x i8> undef, undef
-  %V32I8 = icmp eq <32 x i8> undef, undef
-  %V64I8 = icmp eq <64 x i8> undef, undef
-  %V128I8 = icmp eq <128 x i8> undef, undef
-
-  %V8I16  = icmp eq <8 x i16> undef, undef
-  %V16I16 = icmp eq <16 x i16> undef, undef
-  %V32I16 = icmp eq <32 x i16> undef, undef
-  %V64I16 = icmp eq <64 x i16> undef, undef
-
-  %V4I32 = icmp eq <4 x i32> undef, undef
-  %V8I32 = icmp eq <8 x i32> undef, undef
-  %V16I32 = icmp eq <16 x i32> undef, undef
-  %V32I32 = icmp eq <32 x i32> undef, undef
-
-  %V2I64 = icmp eq <2 x i64> undef, undef
-  %V4I64 = icmp eq <4 x i64> undef, undef
-  %V8I64 = icmp eq <8 x i64> undef, undef
-  %V16I64 = icmp eq <16 x i64> undef, undef
-
-  ret i32 undef
-}
diff --git a/test/Analysis/CostModel/X86/fcmp.ll b/test/Analysis/CostModel/X86/fcmp.ll
new file mode 100644
index 0000000000000000000000000000000000000000..5ec8928b02bf9641a73ef5ea97fc573aa072a707
--- /dev/null
+++ b/test/Analysis/CostModel/X86/fcmp.ll
@@ -0,0 +1,1934 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
+; RUN: opt < %s -mtriple=x86_64-apple-macosx10.8.0 -cost-model -analyze -mattr=+sse2 | FileCheck %s --check-prefixes=CHECK,SSE,SSE2
+; RUN: opt < %s -mtriple=x86_64-apple-macosx10.8.0 -cost-model -analyze -mattr=+sse3 | FileCheck %s --check-prefixes=CHECK,SSE,SSE3
+; RUN: opt < %s -mtriple=x86_64-apple-macosx10.8.0 -cost-model -analyze -mattr=+ssse3 | FileCheck %s --check-prefixes=CHECK,SSE,SSSE3
+; RUN: opt < %s -mtriple=x86_64-apple-macosx10.8.0 -cost-model -analyze -mattr=+sse4.1 | FileCheck %s --check-prefixes=CHECK,SSE,SSE41
+; RUN: opt < %s -mtriple=x86_64-apple-macosx10.8.0 -cost-model -analyze -mattr=+sse4.2 | FileCheck %s --check-prefixes=CHECK,SSE,SSE42
+; RUN: opt < %s -mtriple=x86_64-apple-macosx10.8.0 -cost-model -analyze -mattr=+avx | FileCheck %s --check-prefixes=CHECK,AVX,AVX1
+; RUN: opt < %s -mtriple=x86_64-apple-macosx10.8.0 -cost-model -analyze -mattr=+avx2 | FileCheck %s --check-prefixes=CHECK,AVX,AVX2
+; RUN: opt < %s -mtriple=x86_64-apple-macosx10.8.0 -cost-model -analyze -mattr=+avx512f | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512F
+; RUN: opt < %s -mtriple=x86_64-apple-macosx10.8.0 -cost-model -analyze -mattr=+avx512f,+avx512bw | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512BW
+;
+; RUN: opt < %s -mtriple=x86_64-apple-macosx10.8.0 -cost-model -analyze -mcpu=slm | FileCheck %s --check-prefixes=CHECK,SSE,SSE42
+; RUN: opt < %s -mtriple=x86_64-apple-macosx10.8.0 -cost-model -analyze -mcpu=goldmont | FileCheck %s --check-prefixes=CHECK,SSE,SSE42
+; RUN: opt < %s -mtriple=x86_64-apple-macosx10.8.0 -cost-model -analyze -mcpu=btver2 | FileCheck %s --check-prefixes=BTVER2
+
+define i32 @cmp_float_oeq(i32 %arg) {
+; SSE2-LABEL: 'cmp_float_oeq'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp oeq float undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F32 = fcmp oeq <2 x float> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F32 = fcmp oeq <4 x float> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F32 = fcmp oeq <8 x float> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16F32 = fcmp oeq <16 x float> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp oeq double undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F64 = fcmp oeq <2 x double> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4F64 = fcmp oeq <4 x double> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8F64 = fcmp oeq <8 x double> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16F64 = fcmp oeq <16 x double> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSE3-LABEL: 'cmp_float_oeq'
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp oeq float undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F32 = fcmp oeq <2 x float> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F32 = fcmp oeq <4 x float> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F32 = fcmp oeq <8 x float> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16F32 = fcmp oeq <16 x float> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp oeq double undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F64 = fcmp oeq <2 x double> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4F64 = fcmp oeq <4 x double> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8F64 = fcmp oeq <8 x double> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16F64 = fcmp oeq <16 x double> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSSE3-LABEL: 'cmp_float_oeq'
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp oeq float undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F32 = fcmp oeq <2 x float> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F32 = fcmp oeq <4 x float> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F32 = fcmp oeq <8 x float> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16F32 = fcmp oeq <16 x float> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp oeq double undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F64 = fcmp oeq <2 x double> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4F64 = fcmp oeq <4 x double> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8F64 = fcmp oeq <8 x double> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16F64 = fcmp oeq <16 x double> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSE41-LABEL: 'cmp_float_oeq'
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp oeq float undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F32 = fcmp oeq <2 x float> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F32 = fcmp oeq <4 x float> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F32 = fcmp oeq <8 x float> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16F32 = fcmp oeq <16 x float> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp oeq double undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F64 = fcmp oeq <2 x double> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4F64 = fcmp oeq <4 x double> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8F64 = fcmp oeq <8 x double> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16F64 = fcmp oeq <16 x double> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSE42-LABEL: 'cmp_float_oeq'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp oeq float undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F32 = fcmp oeq <2 x float> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F32 = fcmp oeq <4 x float> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8F32 = fcmp oeq <8 x float> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16F32 = fcmp oeq <16 x float> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp oeq double undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F64 = fcmp oeq <2 x double> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F64 = fcmp oeq <4 x double> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F64 = fcmp oeq <8 x double> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16F64 = fcmp oeq <16 x double> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX-LABEL: 'cmp_float_oeq'
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp oeq float undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F32 = fcmp oeq <2 x float> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F32 = fcmp oeq <4 x float> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F32 = fcmp oeq <8 x float> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16F32 = fcmp oeq <16 x float> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp oeq double undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F64 = fcmp oeq <2 x double> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F64 = fcmp oeq <4 x double> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8F64 = fcmp oeq <8 x double> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16F64 = fcmp oeq <16 x double> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX512-LABEL: 'cmp_float_oeq'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp oeq float undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F32 = fcmp oeq <2 x float> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F32 = fcmp oeq <4 x float> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F32 = fcmp oeq <8 x float> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16F32 = fcmp oeq <16 x float> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp oeq double undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F64 = fcmp oeq <2 x double> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F64 = fcmp oeq <4 x double> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F64 = fcmp oeq <8 x double> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16F64 = fcmp oeq <16 x double> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; BTVER2-LABEL: 'cmp_float_oeq'
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp oeq float undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F32 = fcmp oeq <2 x float> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F32 = fcmp oeq <4 x float> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F32 = fcmp oeq <8 x float> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16F32 = fcmp oeq <16 x float> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp oeq double undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F64 = fcmp oeq <2 x double> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F64 = fcmp oeq <4 x double> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8F64 = fcmp oeq <8 x double> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16F64 = fcmp oeq <16 x double> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+  %F32 = fcmp oeq float undef, undef
+  %V2F32 = fcmp oeq <2 x float> undef, undef
+  %V4F32 = fcmp oeq <4 x float> undef, undef
+  %V8F32 = fcmp oeq <8 x float> undef, undef
+  %V16F32 = fcmp oeq <16 x float> undef, undef
+
+  %F64 = fcmp oeq double undef, undef
+  %V2F64 = fcmp oeq <2 x double> undef, undef
+  %V4F64 = fcmp oeq <4 x double> undef, undef
+  %V8F64 = fcmp oeq <8 x double> undef, undef
+  %V16F64 = fcmp oeq <16 x double> undef, undef
+
+  ret i32 undef
+}
+
+define i32 @cmp_float_one(i32 %arg) {
+; SSE2-LABEL: 'cmp_float_one'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp one float undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F32 = fcmp one <2 x float> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F32 = fcmp one <4 x float> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F32 = fcmp one <8 x float> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16F32 = fcmp one <16 x float> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp one double undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F64 = fcmp one <2 x double> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4F64 = fcmp one <4 x double> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8F64 = fcmp one <8 x double> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16F64 = fcmp one <16 x double> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSE3-LABEL: 'cmp_float_one'
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp one float undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F32 = fcmp one <2 x float> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F32 = fcmp one <4 x float> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F32 = fcmp one <8 x float> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16F32 = fcmp one <16 x float> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp one double undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F64 = fcmp one <2 x double> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4F64 = fcmp one <4 x double> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8F64 = fcmp one <8 x double> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16F64 = fcmp one <16 x double> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSSE3-LABEL: 'cmp_float_one'
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp one float undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F32 = fcmp one <2 x float> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F32 = fcmp one <4 x float> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F32 = fcmp one <8 x float> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16F32 = fcmp one <16 x float> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp one double undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F64 = fcmp one <2 x double> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4F64 = fcmp one <4 x double> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8F64 = fcmp one <8 x double> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16F64 = fcmp one <16 x double> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSE41-LABEL: 'cmp_float_one'
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp one float undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F32 = fcmp one <2 x float> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F32 = fcmp one <4 x float> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F32 = fcmp one <8 x float> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16F32 = fcmp one <16 x float> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp one double undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F64 = fcmp one <2 x double> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4F64 = fcmp one <4 x double> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8F64 = fcmp one <8 x double> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16F64 = fcmp one <16 x double> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSE42-LABEL: 'cmp_float_one'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp one float undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F32 = fcmp one <2 x float> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F32 = fcmp one <4 x float> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8F32 = fcmp one <8 x float> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16F32 = fcmp one <16 x float> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp one double undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F64 = fcmp one <2 x double> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F64 = fcmp one <4 x double> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F64 = fcmp one <8 x double> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16F64 = fcmp one <16 x double> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX-LABEL: 'cmp_float_one'
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp one float undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F32 = fcmp one <2 x float> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F32 = fcmp one <4 x float> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F32 = fcmp one <8 x float> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16F32 = fcmp one <16 x float> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp one double undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F64 = fcmp one <2 x double> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F64 = fcmp one <4 x double> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8F64 = fcmp one <8 x double> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16F64 = fcmp one <16 x double> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX512-LABEL: 'cmp_float_one'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp one float undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F32 = fcmp one <2 x float> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F32 = fcmp one <4 x float> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F32 = fcmp one <8 x float> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16F32 = fcmp one <16 x float> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp one double undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F64 = fcmp one <2 x double> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F64 = fcmp one <4 x double> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F64 = fcmp one <8 x double> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16F64 = fcmp one <16 x double> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; BTVER2-LABEL: 'cmp_float_one'
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp one float undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F32 = fcmp one <2 x float> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F32 = fcmp one <4 x float> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F32 = fcmp one <8 x float> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16F32 = fcmp one <16 x float> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp one double undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F64 = fcmp one <2 x double> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F64 = fcmp one <4 x double> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8F64 = fcmp one <8 x double> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16F64 = fcmp one <16 x double> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+  %F32 = fcmp one float undef, undef
+  %V2F32 = fcmp one <2 x float> undef, undef
+  %V4F32 = fcmp one <4 x float> undef, undef
+  %V8F32 = fcmp one <8 x float> undef, undef
+  %V16F32 = fcmp one <16 x float> undef, undef
+
+  %F64 = fcmp one double undef, undef
+  %V2F64 = fcmp one <2 x double> undef, undef
+  %V4F64 = fcmp one <4 x double> undef, undef
+  %V8F64 = fcmp one <8 x double> undef, undef
+  %V16F64 = fcmp one <16 x double> undef, undef
+
+  ret i32 undef
+}
+
+define i32 @cmp_float_ord(i32 %arg) {
+; SSE2-LABEL: 'cmp_float_ord'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp ord float undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F32 = fcmp ord <2 x float> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F32 = fcmp ord <4 x float> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F32 = fcmp ord <8 x float> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16F32 = fcmp ord <16 x float> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp ord double undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F64 = fcmp ord <2 x double> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4F64 = fcmp ord <4 x double> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8F64 = fcmp ord <8 x double> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16F64 = fcmp ord <16 x double> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSE3-LABEL: 'cmp_float_ord'
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp ord float undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F32 = fcmp ord <2 x float> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F32 = fcmp ord <4 x float> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F32 = fcmp ord <8 x float> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16F32 = fcmp ord <16 x float> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp ord double undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F64 = fcmp ord <2 x double> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4F64 = fcmp ord <4 x double> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8F64 = fcmp ord <8 x double> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16F64 = fcmp ord <16 x double> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSSE3-LABEL: 'cmp_float_ord'
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp ord float undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F32 = fcmp ord <2 x float> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F32 = fcmp ord <4 x float> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F32 = fcmp ord <8 x float> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16F32 = fcmp ord <16 x float> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp ord double undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F64 = fcmp ord <2 x double> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4F64 = fcmp ord <4 x double> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8F64 = fcmp ord <8 x double> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16F64 = fcmp ord <16 x double> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSE41-LABEL: 'cmp_float_ord'
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp ord float undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F32 = fcmp ord <2 x float> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F32 = fcmp ord <4 x float> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F32 = fcmp ord <8 x float> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16F32 = fcmp ord <16 x float> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp ord double undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F64 = fcmp ord <2 x double> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4F64 = fcmp ord <4 x double> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8F64 = fcmp ord <8 x double> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16F64 = fcmp ord <16 x double> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSE42-LABEL: 'cmp_float_ord'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp ord float undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F32 = fcmp ord <2 x float> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F32 = fcmp ord <4 x float> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8F32 = fcmp ord <8 x float> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16F32 = fcmp ord <16 x float> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp ord double undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F64 = fcmp ord <2 x double> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F64 = fcmp ord <4 x double> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F64 = fcmp ord <8 x double> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16F64 = fcmp ord <16 x double> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX-LABEL: 'cmp_float_ord'
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp ord float undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F32 = fcmp ord <2 x float> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F32 = fcmp ord <4 x float> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F32 = fcmp ord <8 x float> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16F32 = fcmp ord <16 x float> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp ord double undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F64 = fcmp ord <2 x double> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F64 = fcmp ord <4 x double> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8F64 = fcmp ord <8 x double> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16F64 = fcmp ord <16 x double> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX512-LABEL: 'cmp_float_ord'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp ord float undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F32 = fcmp ord <2 x float> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F32 = fcmp ord <4 x float> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F32 = fcmp ord <8 x float> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16F32 = fcmp ord <16 x float> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp ord double undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F64 = fcmp ord <2 x double> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F64 = fcmp ord <4 x double> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F64 = fcmp ord <8 x double> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16F64 = fcmp ord <16 x double> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; BTVER2-LABEL: 'cmp_float_ord'
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp ord float undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F32 = fcmp ord <2 x float> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F32 = fcmp ord <4 x float> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F32 = fcmp ord <8 x float> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16F32 = fcmp ord <16 x float> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp ord double undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F64 = fcmp ord <2 x double> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F64 = fcmp ord <4 x double> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8F64 = fcmp ord <8 x double> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16F64 = fcmp ord <16 x double> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+  %F32 = fcmp ord float undef, undef
+  %V2F32 = fcmp ord <2 x float> undef, undef
+  %V4F32 = fcmp ord <4 x float> undef, undef
+  %V8F32 = fcmp ord <8 x float> undef, undef
+  %V16F32 = fcmp ord <16 x float> undef, undef
+
+  %F64 = fcmp ord double undef, undef
+  %V2F64 = fcmp ord <2 x double> undef, undef
+  %V4F64 = fcmp ord <4 x double> undef, undef
+  %V8F64 = fcmp ord <8 x double> undef, undef
+  %V16F64 = fcmp ord <16 x double> undef, undef
+
+  ret i32 undef
+}
+
+define i32 @cmp_float_oge(i32 %arg) {
+; SSE2-LABEL: 'cmp_float_oge'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp oge float undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F32 = fcmp oge <2 x float> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F32 = fcmp oge <4 x float> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F32 = fcmp oge <8 x float> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16F32 = fcmp oge <16 x float> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp oge double undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F64 = fcmp oge <2 x double> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4F64 = fcmp oge <4 x double> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8F64 = fcmp oge <8 x double> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16F64 = fcmp oge <16 x double> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSE3-LABEL: 'cmp_float_oge'
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp oge float undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F32 = fcmp oge <2 x float> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F32 = fcmp oge <4 x float> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F32 = fcmp oge <8 x float> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16F32 = fcmp oge <16 x float> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp oge double undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F64 = fcmp oge <2 x double> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4F64 = fcmp oge <4 x double> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8F64 = fcmp oge <8 x double> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16F64 = fcmp oge <16 x double> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSSE3-LABEL: 'cmp_float_oge'
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp oge float undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F32 = fcmp oge <2 x float> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F32 = fcmp oge <4 x float> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F32 = fcmp oge <8 x float> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16F32 = fcmp oge <16 x float> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp oge double undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F64 = fcmp oge <2 x double> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4F64 = fcmp oge <4 x double> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8F64 = fcmp oge <8 x double> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16F64 = fcmp oge <16 x double> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSE41-LABEL: 'cmp_float_oge'
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp oge float undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F32 = fcmp oge <2 x float> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F32 = fcmp oge <4 x float> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F32 = fcmp oge <8 x float> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16F32 = fcmp oge <16 x float> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp oge double undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F64 = fcmp oge <2 x double> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4F64 = fcmp oge <4 x double> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8F64 = fcmp oge <8 x double> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16F64 = fcmp oge <16 x double> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSE42-LABEL: 'cmp_float_oge'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp oge float undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F32 = fcmp oge <2 x float> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F32 = fcmp oge <4 x float> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8F32 = fcmp oge <8 x float> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16F32 = fcmp oge <16 x float> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp oge double undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F64 = fcmp oge <2 x double> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F64 = fcmp oge <4 x double> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F64 = fcmp oge <8 x double> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16F64 = fcmp oge <16 x double> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX-LABEL: 'cmp_float_oge'
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp oge float undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F32 = fcmp oge <2 x float> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F32 = fcmp oge <4 x float> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F32 = fcmp oge <8 x float> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16F32 = fcmp oge <16 x float> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp oge double undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F64 = fcmp oge <2 x double> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F64 = fcmp oge <4 x double> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8F64 = fcmp oge <8 x double> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16F64 = fcmp oge <16 x double> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX512-LABEL: 'cmp_float_oge'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp oge float undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F32 = fcmp oge <2 x float> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F32 = fcmp oge <4 x float> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F32 = fcmp oge <8 x float> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16F32 = fcmp oge <16 x float> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp oge double undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F64 = fcmp oge <2 x double> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F64 = fcmp oge <4 x double> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F64 = fcmp oge <8 x double> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16F64 = fcmp oge <16 x double> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; BTVER2-LABEL: 'cmp_float_oge'
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp oge float undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F32 = fcmp oge <2 x float> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F32 = fcmp oge <4 x float> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F32 = fcmp oge <8 x float> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16F32 = fcmp oge <16 x float> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp oge double undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F64 = fcmp oge <2 x double> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F64 = fcmp oge <4 x double> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8F64 = fcmp oge <8 x double> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16F64 = fcmp oge <16 x double> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+  %F32 = fcmp oge float undef, undef
+  %V2F32 = fcmp oge <2 x float> undef, undef
+  %V4F32 = fcmp oge <4 x float> undef, undef
+  %V8F32 = fcmp oge <8 x float> undef, undef
+  %V16F32 = fcmp oge <16 x float> undef, undef
+
+  %F64 = fcmp oge double undef, undef
+  %V2F64 = fcmp oge <2 x double> undef, undef
+  %V4F64 = fcmp oge <4 x double> undef, undef
+  %V8F64 = fcmp oge <8 x double> undef, undef
+  %V16F64 = fcmp oge <16 x double> undef, undef
+
+  ret i32 undef
+}
+
+define i32 @cmp_float_ogt(i32 %arg) {
+; SSE2-LABEL: 'cmp_float_ogt'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp ogt float undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F32 = fcmp ogt <2 x float> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F32 = fcmp ogt <4 x float> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F32 = fcmp ogt <8 x float> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16F32 = fcmp ogt <16 x float> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp ogt double undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F64 = fcmp ogt <2 x double> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4F64 = fcmp ogt <4 x double> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8F64 = fcmp ogt <8 x double> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16F64 = fcmp ogt <16 x double> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSE3-LABEL: 'cmp_float_ogt'
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp ogt float undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F32 = fcmp ogt <2 x float> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F32 = fcmp ogt <4 x float> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F32 = fcmp ogt <8 x float> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16F32 = fcmp ogt <16 x float> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp ogt double undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F64 = fcmp ogt <2 x double> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4F64 = fcmp ogt <4 x double> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8F64 = fcmp ogt <8 x double> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16F64 = fcmp ogt <16 x double> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSSE3-LABEL: 'cmp_float_ogt'
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp ogt float undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F32 = fcmp ogt <2 x float> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F32 = fcmp ogt <4 x float> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F32 = fcmp ogt <8 x float> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16F32 = fcmp ogt <16 x float> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp ogt double undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F64 = fcmp ogt <2 x double> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4F64 = fcmp ogt <4 x double> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8F64 = fcmp ogt <8 x double> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16F64 = fcmp ogt <16 x double> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSE41-LABEL: 'cmp_float_ogt'
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp ogt float undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F32 = fcmp ogt <2 x float> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F32 = fcmp ogt <4 x float> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F32 = fcmp ogt <8 x float> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16F32 = fcmp ogt <16 x float> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp ogt double undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F64 = fcmp ogt <2 x double> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4F64 = fcmp ogt <4 x double> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8F64 = fcmp ogt <8 x double> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16F64 = fcmp ogt <16 x double> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSE42-LABEL: 'cmp_float_ogt'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp ogt float undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F32 = fcmp ogt <2 x float> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F32 = fcmp ogt <4 x float> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8F32 = fcmp ogt <8 x float> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16F32 = fcmp ogt <16 x float> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp ogt double undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F64 = fcmp ogt <2 x double> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F64 = fcmp ogt <4 x double> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F64 = fcmp ogt <8 x double> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16F64 = fcmp ogt <16 x double> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX-LABEL: 'cmp_float_ogt'
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp ogt float undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F32 = fcmp ogt <2 x float> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F32 = fcmp ogt <4 x float> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F32 = fcmp ogt <8 x float> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16F32 = fcmp ogt <16 x float> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp ogt double undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F64 = fcmp ogt <2 x double> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F64 = fcmp ogt <4 x double> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8F64 = fcmp ogt <8 x double> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16F64 = fcmp ogt <16 x double> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX512-LABEL: 'cmp_float_ogt'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp ogt float undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F32 = fcmp ogt <2 x float> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F32 = fcmp ogt <4 x float> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F32 = fcmp ogt <8 x float> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16F32 = fcmp ogt <16 x float> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp ogt double undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F64 = fcmp ogt <2 x double> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F64 = fcmp ogt <4 x double> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F64 = fcmp ogt <8 x double> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16F64 = fcmp ogt <16 x double> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; BTVER2-LABEL: 'cmp_float_ogt'
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp ogt float undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F32 = fcmp ogt <2 x float> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F32 = fcmp ogt <4 x float> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F32 = fcmp ogt <8 x float> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16F32 = fcmp ogt <16 x float> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp ogt double undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F64 = fcmp ogt <2 x double> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F64 = fcmp ogt <4 x double> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8F64 = fcmp ogt <8 x double> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16F64 = fcmp ogt <16 x double> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+  %F32 = fcmp ogt float undef, undef
+  %V2F32 = fcmp ogt <2 x float> undef, undef
+  %V4F32 = fcmp ogt <4 x float> undef, undef
+  %V8F32 = fcmp ogt <8 x float> undef, undef
+  %V16F32 = fcmp ogt <16 x float> undef, undef
+
+  %F64 = fcmp ogt double undef, undef
+  %V2F64 = fcmp ogt <2 x double> undef, undef
+  %V4F64 = fcmp ogt <4 x double> undef, undef
+  %V8F64 = fcmp ogt <8 x double> undef, undef
+  %V16F64 = fcmp ogt <16 x double> undef, undef
+
+  ret i32 undef
+}
+
+define i32 @cmp_float_ole(i32 %arg) {
+; SSE2-LABEL: 'cmp_float_ole'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp ole float undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F32 = fcmp ole <2 x float> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F32 = fcmp ole <4 x float> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F32 = fcmp ole <8 x float> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16F32 = fcmp ole <16 x float> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp ole double undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F64 = fcmp ole <2 x double> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4F64 = fcmp ole <4 x double> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8F64 = fcmp ole <8 x double> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16F64 = fcmp ole <16 x double> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSE3-LABEL: 'cmp_float_ole'
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp ole float undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F32 = fcmp ole <2 x float> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F32 = fcmp ole <4 x float> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F32 = fcmp ole <8 x float> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16F32 = fcmp ole <16 x float> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp ole double undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F64 = fcmp ole <2 x double> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4F64 = fcmp ole <4 x double> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8F64 = fcmp ole <8 x double> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16F64 = fcmp ole <16 x double> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSSE3-LABEL: 'cmp_float_ole'
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp ole float undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F32 = fcmp ole <2 x float> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F32 = fcmp ole <4 x float> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F32 = fcmp ole <8 x float> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16F32 = fcmp ole <16 x float> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp ole double undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F64 = fcmp ole <2 x double> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4F64 = fcmp ole <4 x double> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8F64 = fcmp ole <8 x double> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16F64 = fcmp ole <16 x double> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSE41-LABEL: 'cmp_float_ole'
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp ole float undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F32 = fcmp ole <2 x float> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F32 = fcmp ole <4 x float> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F32 = fcmp ole <8 x float> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16F32 = fcmp ole <16 x float> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp ole double undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F64 = fcmp ole <2 x double> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4F64 = fcmp ole <4 x double> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8F64 = fcmp ole <8 x double> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16F64 = fcmp ole <16 x double> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSE42-LABEL: 'cmp_float_ole'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp ole float undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F32 = fcmp ole <2 x float> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F32 = fcmp ole <4 x float> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8F32 = fcmp ole <8 x float> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16F32 = fcmp ole <16 x float> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp ole double undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F64 = fcmp ole <2 x double> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F64 = fcmp ole <4 x double> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F64 = fcmp ole <8 x double> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16F64 = fcmp ole <16 x double> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX-LABEL: 'cmp_float_ole'
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp ole float undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F32 = fcmp ole <2 x float> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F32 = fcmp ole <4 x float> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F32 = fcmp ole <8 x float> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16F32 = fcmp ole <16 x float> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp ole double undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F64 = fcmp ole <2 x double> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F64 = fcmp ole <4 x double> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8F64 = fcmp ole <8 x double> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16F64 = fcmp ole <16 x double> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX512-LABEL: 'cmp_float_ole'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp ole float undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F32 = fcmp ole <2 x float> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F32 = fcmp ole <4 x float> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F32 = fcmp ole <8 x float> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16F32 = fcmp ole <16 x float> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp ole double undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F64 = fcmp ole <2 x double> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F64 = fcmp ole <4 x double> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F64 = fcmp ole <8 x double> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16F64 = fcmp ole <16 x double> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; BTVER2-LABEL: 'cmp_float_ole'
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp ole float undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F32 = fcmp ole <2 x float> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F32 = fcmp ole <4 x float> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F32 = fcmp ole <8 x float> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16F32 = fcmp ole <16 x float> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp ole double undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F64 = fcmp ole <2 x double> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F64 = fcmp ole <4 x double> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8F64 = fcmp ole <8 x double> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16F64 = fcmp ole <16 x double> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+  %F32 = fcmp ole float undef, undef
+  %V2F32 = fcmp ole <2 x float> undef, undef
+  %V4F32 = fcmp ole <4 x float> undef, undef
+  %V8F32 = fcmp ole <8 x float> undef, undef
+  %V16F32 = fcmp ole <16 x float> undef, undef
+
+  %F64 = fcmp ole double undef, undef
+  %V2F64 = fcmp ole <2 x double> undef, undef
+  %V4F64 = fcmp ole <4 x double> undef, undef
+  %V8F64 = fcmp ole <8 x double> undef, undef
+  %V16F64 = fcmp ole <16 x double> undef, undef
+
+  ret i32 undef
+}
+
+define i32 @cmp_float_olt(i32 %arg) {
+; SSE2-LABEL: 'cmp_float_olt'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp olt float undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F32 = fcmp olt <2 x float> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F32 = fcmp olt <4 x float> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F32 = fcmp olt <8 x float> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16F32 = fcmp olt <16 x float> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp olt double undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F64 = fcmp olt <2 x double> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4F64 = fcmp olt <4 x double> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8F64 = fcmp olt <8 x double> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16F64 = fcmp olt <16 x double> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSE3-LABEL: 'cmp_float_olt'
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp olt float undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F32 = fcmp olt <2 x float> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F32 = fcmp olt <4 x float> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F32 = fcmp olt <8 x float> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16F32 = fcmp olt <16 x float> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp olt double undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F64 = fcmp olt <2 x double> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4F64 = fcmp olt <4 x double> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8F64 = fcmp olt <8 x double> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16F64 = fcmp olt <16 x double> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSSE3-LABEL: 'cmp_float_olt'
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp olt float undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F32 = fcmp olt <2 x float> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F32 = fcmp olt <4 x float> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F32 = fcmp olt <8 x float> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16F32 = fcmp olt <16 x float> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp olt double undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F64 = fcmp olt <2 x double> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4F64 = fcmp olt <4 x double> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8F64 = fcmp olt <8 x double> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16F64 = fcmp olt <16 x double> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSE41-LABEL: 'cmp_float_olt'
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp olt float undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F32 = fcmp olt <2 x float> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F32 = fcmp olt <4 x float> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F32 = fcmp olt <8 x float> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16F32 = fcmp olt <16 x float> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp olt double undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F64 = fcmp olt <2 x double> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4F64 = fcmp olt <4 x double> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8F64 = fcmp olt <8 x double> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16F64 = fcmp olt <16 x double> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSE42-LABEL: 'cmp_float_olt'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp olt float undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F32 = fcmp olt <2 x float> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F32 = fcmp olt <4 x float> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8F32 = fcmp olt <8 x float> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16F32 = fcmp olt <16 x float> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp olt double undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F64 = fcmp olt <2 x double> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F64 = fcmp olt <4 x double> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F64 = fcmp olt <8 x double> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16F64 = fcmp olt <16 x double> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX-LABEL: 'cmp_float_olt'
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp olt float undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F32 = fcmp olt <2 x float> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F32 = fcmp olt <4 x float> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F32 = fcmp olt <8 x float> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16F32 = fcmp olt <16 x float> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp olt double undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F64 = fcmp olt <2 x double> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F64 = fcmp olt <4 x double> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8F64 = fcmp olt <8 x double> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16F64 = fcmp olt <16 x double> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX512-LABEL: 'cmp_float_olt'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp olt float undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F32 = fcmp olt <2 x float> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F32 = fcmp olt <4 x float> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F32 = fcmp olt <8 x float> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16F32 = fcmp olt <16 x float> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp olt double undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F64 = fcmp olt <2 x double> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F64 = fcmp olt <4 x double> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F64 = fcmp olt <8 x double> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16F64 = fcmp olt <16 x double> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; BTVER2-LABEL: 'cmp_float_olt'
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp olt float undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F32 = fcmp olt <2 x float> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F32 = fcmp olt <4 x float> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F32 = fcmp olt <8 x float> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16F32 = fcmp olt <16 x float> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp olt double undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F64 = fcmp olt <2 x double> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F64 = fcmp olt <4 x double> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8F64 = fcmp olt <8 x double> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16F64 = fcmp olt <16 x double> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+  %F32 = fcmp olt float undef, undef
+  %V2F32 = fcmp olt <2 x float> undef, undef
+  %V4F32 = fcmp olt <4 x float> undef, undef
+  %V8F32 = fcmp olt <8 x float> undef, undef
+  %V16F32 = fcmp olt <16 x float> undef, undef
+
+  %F64 = fcmp olt double undef, undef
+  %V2F64 = fcmp olt <2 x double> undef, undef
+  %V4F64 = fcmp olt <4 x double> undef, undef
+  %V8F64 = fcmp olt <8 x double> undef, undef
+  %V16F64 = fcmp olt <16 x double> undef, undef
+
+  ret i32 undef
+}
+
+define i32 @cmp_float_ueq(i32 %arg) {
+; SSE2-LABEL: 'cmp_float_ueq'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp ueq float undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F32 = fcmp ueq <2 x float> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F32 = fcmp ueq <4 x float> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F32 = fcmp ueq <8 x float> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16F32 = fcmp ueq <16 x float> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp ueq double undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F64 = fcmp ueq <2 x double> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4F64 = fcmp ueq <4 x double> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8F64 = fcmp ueq <8 x double> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16F64 = fcmp ueq <16 x double> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSE3-LABEL: 'cmp_float_ueq'
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp ueq float undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F32 = fcmp ueq <2 x float> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F32 = fcmp ueq <4 x float> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F32 = fcmp ueq <8 x float> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16F32 = fcmp ueq <16 x float> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp ueq double undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F64 = fcmp ueq <2 x double> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4F64 = fcmp ueq <4 x double> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8F64 = fcmp ueq <8 x double> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16F64 = fcmp ueq <16 x double> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSSE3-LABEL: 'cmp_float_ueq'
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp ueq float undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F32 = fcmp ueq <2 x float> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F32 = fcmp ueq <4 x float> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F32 = fcmp ueq <8 x float> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16F32 = fcmp ueq <16 x float> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp ueq double undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F64 = fcmp ueq <2 x double> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4F64 = fcmp ueq <4 x double> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8F64 = fcmp ueq <8 x double> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16F64 = fcmp ueq <16 x double> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSE41-LABEL: 'cmp_float_ueq'
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp ueq float undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F32 = fcmp ueq <2 x float> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F32 = fcmp ueq <4 x float> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F32 = fcmp ueq <8 x float> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16F32 = fcmp ueq <16 x float> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp ueq double undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F64 = fcmp ueq <2 x double> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4F64 = fcmp ueq <4 x double> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8F64 = fcmp ueq <8 x double> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16F64 = fcmp ueq <16 x double> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSE42-LABEL: 'cmp_float_ueq'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp ueq float undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F32 = fcmp ueq <2 x float> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F32 = fcmp ueq <4 x float> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8F32 = fcmp ueq <8 x float> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16F32 = fcmp ueq <16 x float> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp ueq double undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F64 = fcmp ueq <2 x double> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F64 = fcmp ueq <4 x double> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F64 = fcmp ueq <8 x double> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16F64 = fcmp ueq <16 x double> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX-LABEL: 'cmp_float_ueq'
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp ueq float undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F32 = fcmp ueq <2 x float> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F32 = fcmp ueq <4 x float> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F32 = fcmp ueq <8 x float> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16F32 = fcmp ueq <16 x float> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp ueq double undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F64 = fcmp ueq <2 x double> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F64 = fcmp ueq <4 x double> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8F64 = fcmp ueq <8 x double> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16F64 = fcmp ueq <16 x double> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX512-LABEL: 'cmp_float_ueq'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp ueq float undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F32 = fcmp ueq <2 x float> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F32 = fcmp ueq <4 x float> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F32 = fcmp ueq <8 x float> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16F32 = fcmp ueq <16 x float> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp ueq double undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F64 = fcmp ueq <2 x double> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F64 = fcmp ueq <4 x double> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F64 = fcmp ueq <8 x double> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16F64 = fcmp ueq <16 x double> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; BTVER2-LABEL: 'cmp_float_ueq'
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp ueq float undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F32 = fcmp ueq <2 x float> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F32 = fcmp ueq <4 x float> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F32 = fcmp ueq <8 x float> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16F32 = fcmp ueq <16 x float> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp ueq double undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F64 = fcmp ueq <2 x double> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F64 = fcmp ueq <4 x double> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8F64 = fcmp ueq <8 x double> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16F64 = fcmp ueq <16 x double> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+  %F32 = fcmp ueq float undef, undef
+  %V2F32 = fcmp ueq <2 x float> undef, undef
+  %V4F32 = fcmp ueq <4 x float> undef, undef
+  %V8F32 = fcmp ueq <8 x float> undef, undef
+  %V16F32 = fcmp ueq <16 x float> undef, undef
+
+  %F64 = fcmp ueq double undef, undef
+  %V2F64 = fcmp ueq <2 x double> undef, undef
+  %V4F64 = fcmp ueq <4 x double> undef, undef
+  %V8F64 = fcmp ueq <8 x double> undef, undef
+  %V16F64 = fcmp ueq <16 x double> undef, undef
+
+  ret i32 undef
+}
+
+define i32 @cmp_float_une(i32 %arg) {
+; SSE2-LABEL: 'cmp_float_une'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp une float undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F32 = fcmp une <2 x float> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F32 = fcmp une <4 x float> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F32 = fcmp une <8 x float> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16F32 = fcmp une <16 x float> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp une double undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F64 = fcmp une <2 x double> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4F64 = fcmp une <4 x double> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8F64 = fcmp une <8 x double> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16F64 = fcmp une <16 x double> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSE3-LABEL: 'cmp_float_une'
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp une float undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F32 = fcmp une <2 x float> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F32 = fcmp une <4 x float> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F32 = fcmp une <8 x float> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16F32 = fcmp une <16 x float> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp une double undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F64 = fcmp une <2 x double> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4F64 = fcmp une <4 x double> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8F64 = fcmp une <8 x double> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16F64 = fcmp une <16 x double> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSSE3-LABEL: 'cmp_float_une'
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp une float undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F32 = fcmp une <2 x float> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F32 = fcmp une <4 x float> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F32 = fcmp une <8 x float> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16F32 = fcmp une <16 x float> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp une double undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F64 = fcmp une <2 x double> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4F64 = fcmp une <4 x double> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8F64 = fcmp une <8 x double> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16F64 = fcmp une <16 x double> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSE41-LABEL: 'cmp_float_une'
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp une float undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F32 = fcmp une <2 x float> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F32 = fcmp une <4 x float> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F32 = fcmp une <8 x float> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16F32 = fcmp une <16 x float> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp une double undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F64 = fcmp une <2 x double> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4F64 = fcmp une <4 x double> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8F64 = fcmp une <8 x double> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16F64 = fcmp une <16 x double> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSE42-LABEL: 'cmp_float_une'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp une float undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F32 = fcmp une <2 x float> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F32 = fcmp une <4 x float> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8F32 = fcmp une <8 x float> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16F32 = fcmp une <16 x float> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp une double undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F64 = fcmp une <2 x double> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F64 = fcmp une <4 x double> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F64 = fcmp une <8 x double> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16F64 = fcmp une <16 x double> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX-LABEL: 'cmp_float_une'
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp une float undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F32 = fcmp une <2 x float> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F32 = fcmp une <4 x float> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F32 = fcmp une <8 x float> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16F32 = fcmp une <16 x float> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp une double undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F64 = fcmp une <2 x double> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F64 = fcmp une <4 x double> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8F64 = fcmp une <8 x double> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16F64 = fcmp une <16 x double> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX512-LABEL: 'cmp_float_une'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp une float undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F32 = fcmp une <2 x float> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F32 = fcmp une <4 x float> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F32 = fcmp une <8 x float> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16F32 = fcmp une <16 x float> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp une double undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F64 = fcmp une <2 x double> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F64 = fcmp une <4 x double> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F64 = fcmp une <8 x double> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16F64 = fcmp une <16 x double> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; BTVER2-LABEL: 'cmp_float_une'
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp une float undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F32 = fcmp une <2 x float> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F32 = fcmp une <4 x float> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F32 = fcmp une <8 x float> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16F32 = fcmp une <16 x float> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp une double undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F64 = fcmp une <2 x double> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F64 = fcmp une <4 x double> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8F64 = fcmp une <8 x double> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16F64 = fcmp une <16 x double> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+  %F32 = fcmp une float undef, undef
+  %V2F32 = fcmp une <2 x float> undef, undef
+  %V4F32 = fcmp une <4 x float> undef, undef
+  %V8F32 = fcmp une <8 x float> undef, undef
+  %V16F32 = fcmp une <16 x float> undef, undef
+
+  %F64 = fcmp une double undef, undef
+  %V2F64 = fcmp une <2 x double> undef, undef
+  %V4F64 = fcmp une <4 x double> undef, undef
+  %V8F64 = fcmp une <8 x double> undef, undef
+  %V16F64 = fcmp une <16 x double> undef, undef
+
+  ret i32 undef
+}
+
+define i32 @cmp_float_uno(i32 %arg) {
+; SSE2-LABEL: 'cmp_float_uno'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp uno float undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F32 = fcmp uno <2 x float> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F32 = fcmp uno <4 x float> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F32 = fcmp uno <8 x float> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16F32 = fcmp uno <16 x float> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp uno double undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F64 = fcmp uno <2 x double> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4F64 = fcmp uno <4 x double> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8F64 = fcmp uno <8 x double> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16F64 = fcmp uno <16 x double> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSE3-LABEL: 'cmp_float_uno'
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp uno float undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F32 = fcmp uno <2 x float> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F32 = fcmp uno <4 x float> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F32 = fcmp uno <8 x float> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16F32 = fcmp uno <16 x float> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp uno double undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F64 = fcmp uno <2 x double> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4F64 = fcmp uno <4 x double> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8F64 = fcmp uno <8 x double> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16F64 = fcmp uno <16 x double> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSSE3-LABEL: 'cmp_float_uno'
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp uno float undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F32 = fcmp uno <2 x float> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F32 = fcmp uno <4 x float> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F32 = fcmp uno <8 x float> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16F32 = fcmp uno <16 x float> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp uno double undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F64 = fcmp uno <2 x double> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4F64 = fcmp uno <4 x double> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8F64 = fcmp uno <8 x double> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16F64 = fcmp uno <16 x double> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSE41-LABEL: 'cmp_float_uno'
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp uno float undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F32 = fcmp uno <2 x float> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F32 = fcmp uno <4 x float> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F32 = fcmp uno <8 x float> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16F32 = fcmp uno <16 x float> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp uno double undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F64 = fcmp uno <2 x double> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4F64 = fcmp uno <4 x double> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8F64 = fcmp uno <8 x double> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16F64 = fcmp uno <16 x double> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSE42-LABEL: 'cmp_float_uno'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp uno float undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F32 = fcmp uno <2 x float> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F32 = fcmp uno <4 x float> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8F32 = fcmp uno <8 x float> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16F32 = fcmp uno <16 x float> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp uno double undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F64 = fcmp uno <2 x double> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F64 = fcmp uno <4 x double> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F64 = fcmp uno <8 x double> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16F64 = fcmp uno <16 x double> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX-LABEL: 'cmp_float_uno'
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp uno float undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F32 = fcmp uno <2 x float> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F32 = fcmp uno <4 x float> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F32 = fcmp uno <8 x float> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16F32 = fcmp uno <16 x float> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp uno double undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F64 = fcmp uno <2 x double> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F64 = fcmp uno <4 x double> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8F64 = fcmp uno <8 x double> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16F64 = fcmp uno <16 x double> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX512-LABEL: 'cmp_float_uno'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp uno float undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F32 = fcmp uno <2 x float> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F32 = fcmp uno <4 x float> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F32 = fcmp uno <8 x float> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16F32 = fcmp uno <16 x float> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp uno double undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F64 = fcmp uno <2 x double> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F64 = fcmp uno <4 x double> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F64 = fcmp uno <8 x double> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16F64 = fcmp uno <16 x double> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; BTVER2-LABEL: 'cmp_float_uno'
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp uno float undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F32 = fcmp uno <2 x float> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F32 = fcmp uno <4 x float> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F32 = fcmp uno <8 x float> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16F32 = fcmp uno <16 x float> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp uno double undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F64 = fcmp uno <2 x double> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F64 = fcmp uno <4 x double> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8F64 = fcmp uno <8 x double> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16F64 = fcmp uno <16 x double> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+  %F32 = fcmp uno float undef, undef
+  %V2F32 = fcmp uno <2 x float> undef, undef
+  %V4F32 = fcmp uno <4 x float> undef, undef
+  %V8F32 = fcmp uno <8 x float> undef, undef
+  %V16F32 = fcmp uno <16 x float> undef, undef
+
+  %F64 = fcmp uno double undef, undef
+  %V2F64 = fcmp uno <2 x double> undef, undef
+  %V4F64 = fcmp uno <4 x double> undef, undef
+  %V8F64 = fcmp uno <8 x double> undef, undef
+  %V16F64 = fcmp uno <16 x double> undef, undef
+
+  ret i32 undef
+}
+
+define i32 @cmp_float_uge(i32 %arg) {
+; SSE2-LABEL: 'cmp_float_uge'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp uge float undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F32 = fcmp uge <2 x float> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F32 = fcmp uge <4 x float> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F32 = fcmp uge <8 x float> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16F32 = fcmp uge <16 x float> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp uge double undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F64 = fcmp uge <2 x double> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4F64 = fcmp uge <4 x double> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8F64 = fcmp uge <8 x double> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16F64 = fcmp uge <16 x double> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSE3-LABEL: 'cmp_float_uge'
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp uge float undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F32 = fcmp uge <2 x float> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F32 = fcmp uge <4 x float> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F32 = fcmp uge <8 x float> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16F32 = fcmp uge <16 x float> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp uge double undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F64 = fcmp uge <2 x double> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4F64 = fcmp uge <4 x double> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8F64 = fcmp uge <8 x double> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16F64 = fcmp uge <16 x double> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSSE3-LABEL: 'cmp_float_uge'
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp uge float undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F32 = fcmp uge <2 x float> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F32 = fcmp uge <4 x float> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F32 = fcmp uge <8 x float> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16F32 = fcmp uge <16 x float> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp uge double undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F64 = fcmp uge <2 x double> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4F64 = fcmp uge <4 x double> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8F64 = fcmp uge <8 x double> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16F64 = fcmp uge <16 x double> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSE41-LABEL: 'cmp_float_uge'
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp uge float undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F32 = fcmp uge <2 x float> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F32 = fcmp uge <4 x float> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F32 = fcmp uge <8 x float> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16F32 = fcmp uge <16 x float> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp uge double undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F64 = fcmp uge <2 x double> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4F64 = fcmp uge <4 x double> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8F64 = fcmp uge <8 x double> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16F64 = fcmp uge <16 x double> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSE42-LABEL: 'cmp_float_uge'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp uge float undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F32 = fcmp uge <2 x float> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F32 = fcmp uge <4 x float> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8F32 = fcmp uge <8 x float> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16F32 = fcmp uge <16 x float> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp uge double undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F64 = fcmp uge <2 x double> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F64 = fcmp uge <4 x double> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F64 = fcmp uge <8 x double> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16F64 = fcmp uge <16 x double> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX-LABEL: 'cmp_float_uge'
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp uge float undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F32 = fcmp uge <2 x float> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F32 = fcmp uge <4 x float> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F32 = fcmp uge <8 x float> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16F32 = fcmp uge <16 x float> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp uge double undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F64 = fcmp uge <2 x double> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F64 = fcmp uge <4 x double> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8F64 = fcmp uge <8 x double> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16F64 = fcmp uge <16 x double> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX512-LABEL: 'cmp_float_uge'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp uge float undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F32 = fcmp uge <2 x float> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F32 = fcmp uge <4 x float> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F32 = fcmp uge <8 x float> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16F32 = fcmp uge <16 x float> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp uge double undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F64 = fcmp uge <2 x double> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F64 = fcmp uge <4 x double> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F64 = fcmp uge <8 x double> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16F64 = fcmp uge <16 x double> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; BTVER2-LABEL: 'cmp_float_uge'
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp uge float undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F32 = fcmp uge <2 x float> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F32 = fcmp uge <4 x float> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F32 = fcmp uge <8 x float> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16F32 = fcmp uge <16 x float> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp uge double undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F64 = fcmp uge <2 x double> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F64 = fcmp uge <4 x double> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8F64 = fcmp uge <8 x double> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16F64 = fcmp uge <16 x double> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+  %F32 = fcmp uge float undef, undef
+  %V2F32 = fcmp uge <2 x float> undef, undef
+  %V4F32 = fcmp uge <4 x float> undef, undef
+  %V8F32 = fcmp uge <8 x float> undef, undef
+  %V16F32 = fcmp uge <16 x float> undef, undef
+
+  %F64 = fcmp uge double undef, undef
+  %V2F64 = fcmp uge <2 x double> undef, undef
+  %V4F64 = fcmp uge <4 x double> undef, undef
+  %V8F64 = fcmp uge <8 x double> undef, undef
+  %V16F64 = fcmp uge <16 x double> undef, undef
+
+  ret i32 undef
+}
+
+define i32 @cmp_float_ugt(i32 %arg) {
+; SSE2-LABEL: 'cmp_float_ugt'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp ugt float undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F32 = fcmp ugt <2 x float> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F32 = fcmp ugt <4 x float> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F32 = fcmp ugt <8 x float> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16F32 = fcmp ugt <16 x float> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp ugt double undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F64 = fcmp ugt <2 x double> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4F64 = fcmp ugt <4 x double> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8F64 = fcmp ugt <8 x double> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16F64 = fcmp ugt <16 x double> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSE3-LABEL: 'cmp_float_ugt'
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp ugt float undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F32 = fcmp ugt <2 x float> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F32 = fcmp ugt <4 x float> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F32 = fcmp ugt <8 x float> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16F32 = fcmp ugt <16 x float> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp ugt double undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F64 = fcmp ugt <2 x double> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4F64 = fcmp ugt <4 x double> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8F64 = fcmp ugt <8 x double> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16F64 = fcmp ugt <16 x double> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSSE3-LABEL: 'cmp_float_ugt'
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp ugt float undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F32 = fcmp ugt <2 x float> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F32 = fcmp ugt <4 x float> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F32 = fcmp ugt <8 x float> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16F32 = fcmp ugt <16 x float> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp ugt double undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F64 = fcmp ugt <2 x double> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4F64 = fcmp ugt <4 x double> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8F64 = fcmp ugt <8 x double> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16F64 = fcmp ugt <16 x double> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSE41-LABEL: 'cmp_float_ugt'
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp ugt float undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F32 = fcmp ugt <2 x float> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F32 = fcmp ugt <4 x float> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F32 = fcmp ugt <8 x float> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16F32 = fcmp ugt <16 x float> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp ugt double undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F64 = fcmp ugt <2 x double> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4F64 = fcmp ugt <4 x double> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8F64 = fcmp ugt <8 x double> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16F64 = fcmp ugt <16 x double> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSE42-LABEL: 'cmp_float_ugt'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp ugt float undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F32 = fcmp ugt <2 x float> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F32 = fcmp ugt <4 x float> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8F32 = fcmp ugt <8 x float> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16F32 = fcmp ugt <16 x float> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp ugt double undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F64 = fcmp ugt <2 x double> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F64 = fcmp ugt <4 x double> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F64 = fcmp ugt <8 x double> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16F64 = fcmp ugt <16 x double> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX-LABEL: 'cmp_float_ugt'
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp ugt float undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F32 = fcmp ugt <2 x float> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F32 = fcmp ugt <4 x float> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F32 = fcmp ugt <8 x float> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16F32 = fcmp ugt <16 x float> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp ugt double undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F64 = fcmp ugt <2 x double> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F64 = fcmp ugt <4 x double> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8F64 = fcmp ugt <8 x double> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16F64 = fcmp ugt <16 x double> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX512-LABEL: 'cmp_float_ugt'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp ugt float undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F32 = fcmp ugt <2 x float> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F32 = fcmp ugt <4 x float> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F32 = fcmp ugt <8 x float> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16F32 = fcmp ugt <16 x float> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp ugt double undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F64 = fcmp ugt <2 x double> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F64 = fcmp ugt <4 x double> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F64 = fcmp ugt <8 x double> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16F64 = fcmp ugt <16 x double> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; BTVER2-LABEL: 'cmp_float_ugt'
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp ugt float undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F32 = fcmp ugt <2 x float> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F32 = fcmp ugt <4 x float> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F32 = fcmp ugt <8 x float> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16F32 = fcmp ugt <16 x float> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp ugt double undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F64 = fcmp ugt <2 x double> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F64 = fcmp ugt <4 x double> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8F64 = fcmp ugt <8 x double> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16F64 = fcmp ugt <16 x double> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+  %F32 = fcmp ugt float undef, undef
+  %V2F32 = fcmp ugt <2 x float> undef, undef
+  %V4F32 = fcmp ugt <4 x float> undef, undef
+  %V8F32 = fcmp ugt <8 x float> undef, undef
+  %V16F32 = fcmp ugt <16 x float> undef, undef
+
+  %F64 = fcmp ugt double undef, undef
+  %V2F64 = fcmp ugt <2 x double> undef, undef
+  %V4F64 = fcmp ugt <4 x double> undef, undef
+  %V8F64 = fcmp ugt <8 x double> undef, undef
+  %V16F64 = fcmp ugt <16 x double> undef, undef
+
+  ret i32 undef
+}
+
+define i32 @cmp_float_ule(i32 %arg) {
+; SSE2-LABEL: 'cmp_float_ule'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp ule float undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F32 = fcmp ule <2 x float> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F32 = fcmp ule <4 x float> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F32 = fcmp ule <8 x float> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16F32 = fcmp ule <16 x float> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp ule double undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F64 = fcmp ule <2 x double> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4F64 = fcmp ule <4 x double> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8F64 = fcmp ule <8 x double> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16F64 = fcmp ule <16 x double> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSE3-LABEL: 'cmp_float_ule'
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp ule float undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F32 = fcmp ule <2 x float> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F32 = fcmp ule <4 x float> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F32 = fcmp ule <8 x float> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16F32 = fcmp ule <16 x float> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp ule double undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F64 = fcmp ule <2 x double> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4F64 = fcmp ule <4 x double> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8F64 = fcmp ule <8 x double> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16F64 = fcmp ule <16 x double> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSSE3-LABEL: 'cmp_float_ule'
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp ule float undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F32 = fcmp ule <2 x float> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F32 = fcmp ule <4 x float> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F32 = fcmp ule <8 x float> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16F32 = fcmp ule <16 x float> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp ule double undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F64 = fcmp ule <2 x double> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4F64 = fcmp ule <4 x double> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8F64 = fcmp ule <8 x double> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16F64 = fcmp ule <16 x double> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSE41-LABEL: 'cmp_float_ule'
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp ule float undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F32 = fcmp ule <2 x float> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F32 = fcmp ule <4 x float> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F32 = fcmp ule <8 x float> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16F32 = fcmp ule <16 x float> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp ule double undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F64 = fcmp ule <2 x double> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4F64 = fcmp ule <4 x double> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8F64 = fcmp ule <8 x double> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16F64 = fcmp ule <16 x double> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSE42-LABEL: 'cmp_float_ule'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp ule float undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F32 = fcmp ule <2 x float> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F32 = fcmp ule <4 x float> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8F32 = fcmp ule <8 x float> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16F32 = fcmp ule <16 x float> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp ule double undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F64 = fcmp ule <2 x double> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F64 = fcmp ule <4 x double> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F64 = fcmp ule <8 x double> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16F64 = fcmp ule <16 x double> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX-LABEL: 'cmp_float_ule'
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp ule float undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F32 = fcmp ule <2 x float> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F32 = fcmp ule <4 x float> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F32 = fcmp ule <8 x float> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16F32 = fcmp ule <16 x float> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp ule double undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F64 = fcmp ule <2 x double> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F64 = fcmp ule <4 x double> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8F64 = fcmp ule <8 x double> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16F64 = fcmp ule <16 x double> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX512-LABEL: 'cmp_float_ule'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp ule float undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F32 = fcmp ule <2 x float> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F32 = fcmp ule <4 x float> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F32 = fcmp ule <8 x float> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16F32 = fcmp ule <16 x float> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp ule double undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F64 = fcmp ule <2 x double> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F64 = fcmp ule <4 x double> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F64 = fcmp ule <8 x double> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16F64 = fcmp ule <16 x double> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; BTVER2-LABEL: 'cmp_float_ule'
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp ule float undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F32 = fcmp ule <2 x float> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F32 = fcmp ule <4 x float> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F32 = fcmp ule <8 x float> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16F32 = fcmp ule <16 x float> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp ule double undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F64 = fcmp ule <2 x double> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F64 = fcmp ule <4 x double> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8F64 = fcmp ule <8 x double> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16F64 = fcmp ule <16 x double> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+  %F32 = fcmp ule float undef, undef
+  %V2F32 = fcmp ule <2 x float> undef, undef
+  %V4F32 = fcmp ule <4 x float> undef, undef
+  %V8F32 = fcmp ule <8 x float> undef, undef
+  %V16F32 = fcmp ule <16 x float> undef, undef
+
+  %F64 = fcmp ule double undef, undef
+  %V2F64 = fcmp ule <2 x double> undef, undef
+  %V4F64 = fcmp ule <4 x double> undef, undef
+  %V8F64 = fcmp ule <8 x double> undef, undef
+  %V16F64 = fcmp ule <16 x double> undef, undef
+
+  ret i32 undef
+}
+
+define i32 @cmp_float_ult(i32 %arg) {
+; SSE2-LABEL: 'cmp_float_ult'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp ult float undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F32 = fcmp ult <2 x float> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F32 = fcmp ult <4 x float> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F32 = fcmp ult <8 x float> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16F32 = fcmp ult <16 x float> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp ult double undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F64 = fcmp ult <2 x double> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4F64 = fcmp ult <4 x double> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8F64 = fcmp ult <8 x double> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16F64 = fcmp ult <16 x double> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSE3-LABEL: 'cmp_float_ult'
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp ult float undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F32 = fcmp ult <2 x float> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F32 = fcmp ult <4 x float> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F32 = fcmp ult <8 x float> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16F32 = fcmp ult <16 x float> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp ult double undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F64 = fcmp ult <2 x double> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4F64 = fcmp ult <4 x double> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8F64 = fcmp ult <8 x double> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16F64 = fcmp ult <16 x double> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSSE3-LABEL: 'cmp_float_ult'
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp ult float undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F32 = fcmp ult <2 x float> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F32 = fcmp ult <4 x float> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F32 = fcmp ult <8 x float> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16F32 = fcmp ult <16 x float> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp ult double undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F64 = fcmp ult <2 x double> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4F64 = fcmp ult <4 x double> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8F64 = fcmp ult <8 x double> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16F64 = fcmp ult <16 x double> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSE41-LABEL: 'cmp_float_ult'
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp ult float undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F32 = fcmp ult <2 x float> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F32 = fcmp ult <4 x float> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F32 = fcmp ult <8 x float> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16F32 = fcmp ult <16 x float> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp ult double undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F64 = fcmp ult <2 x double> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4F64 = fcmp ult <4 x double> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8F64 = fcmp ult <8 x double> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16F64 = fcmp ult <16 x double> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSE42-LABEL: 'cmp_float_ult'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp ult float undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F32 = fcmp ult <2 x float> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F32 = fcmp ult <4 x float> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8F32 = fcmp ult <8 x float> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16F32 = fcmp ult <16 x float> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp ult double undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F64 = fcmp ult <2 x double> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F64 = fcmp ult <4 x double> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F64 = fcmp ult <8 x double> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16F64 = fcmp ult <16 x double> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX-LABEL: 'cmp_float_ult'
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp ult float undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F32 = fcmp ult <2 x float> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F32 = fcmp ult <4 x float> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F32 = fcmp ult <8 x float> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16F32 = fcmp ult <16 x float> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp ult double undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F64 = fcmp ult <2 x double> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F64 = fcmp ult <4 x double> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8F64 = fcmp ult <8 x double> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16F64 = fcmp ult <16 x double> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX512-LABEL: 'cmp_float_ult'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp ult float undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F32 = fcmp ult <2 x float> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F32 = fcmp ult <4 x float> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F32 = fcmp ult <8 x float> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16F32 = fcmp ult <16 x float> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp ult double undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F64 = fcmp ult <2 x double> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F64 = fcmp ult <4 x double> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F64 = fcmp ult <8 x double> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16F64 = fcmp ult <16 x double> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; BTVER2-LABEL: 'cmp_float_ult'
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp ult float undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F32 = fcmp ult <2 x float> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F32 = fcmp ult <4 x float> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F32 = fcmp ult <8 x float> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16F32 = fcmp ult <16 x float> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp ult double undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F64 = fcmp ult <2 x double> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F64 = fcmp ult <4 x double> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8F64 = fcmp ult <8 x double> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16F64 = fcmp ult <16 x double> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+  %F32 = fcmp ult float undef, undef
+  %V2F32 = fcmp ult <2 x float> undef, undef
+  %V4F32 = fcmp ult <4 x float> undef, undef
+  %V8F32 = fcmp ult <8 x float> undef, undef
+  %V16F32 = fcmp ult <16 x float> undef, undef
+
+  %F64 = fcmp ult double undef, undef
+  %V2F64 = fcmp ult <2 x double> undef, undef
+  %V4F64 = fcmp ult <4 x double> undef, undef
+  %V8F64 = fcmp ult <8 x double> undef, undef
+  %V16F64 = fcmp ult <16 x double> undef, undef
+
+  ret i32 undef
+}
+
+define i32 @cmp_float_false(i32 %arg) {
+; SSE2-LABEL: 'cmp_float_false'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp false float undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F32 = fcmp false <2 x float> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F32 = fcmp false <4 x float> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F32 = fcmp false <8 x float> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16F32 = fcmp false <16 x float> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp false double undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F64 = fcmp false <2 x double> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4F64 = fcmp false <4 x double> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8F64 = fcmp false <8 x double> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16F64 = fcmp false <16 x double> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSE3-LABEL: 'cmp_float_false'
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp false float undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F32 = fcmp false <2 x float> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F32 = fcmp false <4 x float> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F32 = fcmp false <8 x float> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16F32 = fcmp false <16 x float> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp false double undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F64 = fcmp false <2 x double> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4F64 = fcmp false <4 x double> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8F64 = fcmp false <8 x double> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16F64 = fcmp false <16 x double> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSSE3-LABEL: 'cmp_float_false'
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp false float undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F32 = fcmp false <2 x float> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F32 = fcmp false <4 x float> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F32 = fcmp false <8 x float> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16F32 = fcmp false <16 x float> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp false double undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F64 = fcmp false <2 x double> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4F64 = fcmp false <4 x double> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8F64 = fcmp false <8 x double> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16F64 = fcmp false <16 x double> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSE41-LABEL: 'cmp_float_false'
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp false float undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F32 = fcmp false <2 x float> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F32 = fcmp false <4 x float> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F32 = fcmp false <8 x float> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16F32 = fcmp false <16 x float> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp false double undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F64 = fcmp false <2 x double> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4F64 = fcmp false <4 x double> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8F64 = fcmp false <8 x double> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16F64 = fcmp false <16 x double> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSE42-LABEL: 'cmp_float_false'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp false float undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F32 = fcmp false <2 x float> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F32 = fcmp false <4 x float> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8F32 = fcmp false <8 x float> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16F32 = fcmp false <16 x float> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp false double undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F64 = fcmp false <2 x double> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F64 = fcmp false <4 x double> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F64 = fcmp false <8 x double> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16F64 = fcmp false <16 x double> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX-LABEL: 'cmp_float_false'
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp false float undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F32 = fcmp false <2 x float> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F32 = fcmp false <4 x float> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F32 = fcmp false <8 x float> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16F32 = fcmp false <16 x float> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp false double undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F64 = fcmp false <2 x double> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F64 = fcmp false <4 x double> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8F64 = fcmp false <8 x double> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16F64 = fcmp false <16 x double> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX512-LABEL: 'cmp_float_false'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp false float undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F32 = fcmp false <2 x float> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F32 = fcmp false <4 x float> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F32 = fcmp false <8 x float> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16F32 = fcmp false <16 x float> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp false double undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F64 = fcmp false <2 x double> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F64 = fcmp false <4 x double> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F64 = fcmp false <8 x double> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16F64 = fcmp false <16 x double> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; BTVER2-LABEL: 'cmp_float_false'
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp false float undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F32 = fcmp false <2 x float> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F32 = fcmp false <4 x float> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F32 = fcmp false <8 x float> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16F32 = fcmp false <16 x float> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp false double undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F64 = fcmp false <2 x double> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F64 = fcmp false <4 x double> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8F64 = fcmp false <8 x double> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16F64 = fcmp false <16 x double> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+  %F32 = fcmp false float undef, undef
+  %V2F32 = fcmp false <2 x float> undef, undef
+  %V4F32 = fcmp false <4 x float> undef, undef
+  %V8F32 = fcmp false <8 x float> undef, undef
+  %V16F32 = fcmp false <16 x float> undef, undef
+
+  %F64 = fcmp false double undef, undef
+  %V2F64 = fcmp false <2 x double> undef, undef
+  %V4F64 = fcmp false <4 x double> undef, undef
+  %V8F64 = fcmp false <8 x double> undef, undef
+  %V16F64 = fcmp false <16 x double> undef, undef
+
+  ret i32 undef
+}
+
+define i32 @cmp_float_true(i32 %arg) {
+; SSE2-LABEL: 'cmp_float_true'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp true float undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F32 = fcmp true <2 x float> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F32 = fcmp true <4 x float> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F32 = fcmp true <8 x float> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16F32 = fcmp true <16 x float> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp true double undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F64 = fcmp true <2 x double> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4F64 = fcmp true <4 x double> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8F64 = fcmp true <8 x double> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16F64 = fcmp true <16 x double> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSE3-LABEL: 'cmp_float_true'
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp true float undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F32 = fcmp true <2 x float> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F32 = fcmp true <4 x float> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F32 = fcmp true <8 x float> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16F32 = fcmp true <16 x float> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp true double undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F64 = fcmp true <2 x double> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4F64 = fcmp true <4 x double> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8F64 = fcmp true <8 x double> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16F64 = fcmp true <16 x double> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSSE3-LABEL: 'cmp_float_true'
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp true float undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F32 = fcmp true <2 x float> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F32 = fcmp true <4 x float> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F32 = fcmp true <8 x float> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16F32 = fcmp true <16 x float> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp true double undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F64 = fcmp true <2 x double> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4F64 = fcmp true <4 x double> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8F64 = fcmp true <8 x double> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16F64 = fcmp true <16 x double> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSE41-LABEL: 'cmp_float_true'
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp true float undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F32 = fcmp true <2 x float> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F32 = fcmp true <4 x float> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F32 = fcmp true <8 x float> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16F32 = fcmp true <16 x float> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp true double undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2F64 = fcmp true <2 x double> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4F64 = fcmp true <4 x double> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8F64 = fcmp true <8 x double> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16F64 = fcmp true <16 x double> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSE42-LABEL: 'cmp_float_true'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp true float undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F32 = fcmp true <2 x float> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F32 = fcmp true <4 x float> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8F32 = fcmp true <8 x float> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16F32 = fcmp true <16 x float> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp true double undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F64 = fcmp true <2 x double> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4F64 = fcmp true <4 x double> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8F64 = fcmp true <8 x double> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16F64 = fcmp true <16 x double> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX-LABEL: 'cmp_float_true'
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp true float undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F32 = fcmp true <2 x float> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F32 = fcmp true <4 x float> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F32 = fcmp true <8 x float> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16F32 = fcmp true <16 x float> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp true double undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F64 = fcmp true <2 x double> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F64 = fcmp true <4 x double> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8F64 = fcmp true <8 x double> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16F64 = fcmp true <16 x double> undef, undef
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX512-LABEL: 'cmp_float_true'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp true float undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F32 = fcmp true <2 x float> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F32 = fcmp true <4 x float> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F32 = fcmp true <8 x float> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16F32 = fcmp true <16 x float> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp true double undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F64 = fcmp true <2 x double> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F64 = fcmp true <4 x double> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F64 = fcmp true <8 x double> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16F64 = fcmp true <16 x double> undef, undef
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; BTVER2-LABEL: 'cmp_float_true'
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F32 = fcmp true float undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F32 = fcmp true <2 x float> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F32 = fcmp true <4 x float> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8F32 = fcmp true <8 x float> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16F32 = fcmp true <16 x float> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F64 = fcmp true double undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2F64 = fcmp true <2 x double> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4F64 = fcmp true <4 x double> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8F64 = fcmp true <8 x double> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16F64 = fcmp true <16 x double> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+  %F32 = fcmp true float undef, undef
+  %V2F32 = fcmp true <2 x float> undef, undef
+  %V4F32 = fcmp true <4 x float> undef, undef
+  %V8F32 = fcmp true <8 x float> undef, undef
+  %V16F32 = fcmp true <16 x float> undef, undef
+
+  %F64 = fcmp true double undef, undef
+  %V2F64 = fcmp true <2 x double> undef, undef
+  %V4F64 = fcmp true <4 x double> undef, undef
+  %V8F64 = fcmp true <8 x double> undef, undef
+  %V16F64 = fcmp true <16 x double> undef, undef
+
+  ret i32 undef
+}
diff --git a/test/Analysis/CostModel/X86/fshl.ll b/test/Analysis/CostModel/X86/fshl.ll
index d5da8e7ae4da41c356aad2139bae9929de1a0fea..38621c35a88685ae24443404463613825d3794da 100644
--- a/test/Analysis/CostModel/X86/fshl.ll
+++ b/test/Analysis/CostModel/X86/fshl.ll
@@ -22,9 +22,9 @@ target triple = "x86_64-apple-macosx10.8.0"
 define void @var_funnel_i64(i64 %a64, <2 x i64> %a128, <4 x i64> %a256, <8 x i64> %a512, i64 %b64, <2 x i64> %b128, <4 x i64> %b256, <8 x i64> %b512, i64 %c64, <2 x i64> %c128, <4 x i64> %c256, <8 x i64> %c512) {
 ; SSSE3-LABEL: 'var_funnel_i64'
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I64 = call i64 @llvm.fshl.i64(i64 %a64, i64 %b64, i64 %c64)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V2I64 = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> %c128)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %V4I64 = call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> %c256)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %V8I64 = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> %c512)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V2I64 = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> %c128)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 44 for instruction: %V4I64 = call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> %c256)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 88 for instruction: %V8I64 = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> %c512)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; SSE42-LABEL: 'var_funnel_i64'
@@ -93,9 +93,9 @@ define void @var_funnel_i64(i64 %a64, <2 x i64> %a128, <4 x i64> %a256, <8 x i64
 define void @var_funnel_i32(i32 %a32, <4 x i32> %a128, <8 x i32> %a256, <16 x i32> %a512, i32 %b32, <4 x i32> %b128, <8 x i32> %b256, <16 x i32> %b512, i32 %c32, <4 x i32> %c128, <8 x i32> %c256, <16 x i32> %c512) {
 ; SSSE3-LABEL: 'var_funnel_i32'
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I32 = call i32 @llvm.fshl.i32(i32 %a32, i32 %b32, i32 %c32)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %V2I32 = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> %c128)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 62 for instruction: %V4I32 = call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> %c256)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 124 for instruction: %V8I32 = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> %c512)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %V2I32 = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> %c128)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %V4I32 = call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> %c256)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 132 for instruction: %V8I32 = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> %c512)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; SSE42-LABEL: 'var_funnel_i32'
@@ -164,9 +164,9 @@ define void @var_funnel_i32(i32 %a32, <4 x i32> %a128, <8 x i32> %a256, <16 x i3
 define void @var_funnel_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <32 x i16> %a512, i16 %b16, <8 x i16> %b128, <16 x i16> %b256, <32 x i16> %b512, i16 %c16, <8 x i16> %c128, <16 x i16> %c256, <32 x i16> %c512) {
 ; SSSE3-LABEL: 'var_funnel_i16'
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I16 = call i16 @llvm.fshl.i16(i16 %a16, i16 %b16, i16 %c16)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 69 for instruction: %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> %c128)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 138 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> %c256)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 276 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> %c512)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 71 for instruction: %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> %c128)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 142 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> %c256)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 284 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> %c512)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; SSE42-LABEL: 'var_funnel_i16'
@@ -179,8 +179,8 @@ define void @var_funnel_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <32 x i
 ; AVX1-LABEL: 'var_funnel_i16'
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I16 = call i16 @llvm.fshl.i16(i16 %a16, i16 %b16, i16 %c16)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> %c128)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 71 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> %c256)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 142 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> %c512)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 73 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> %c256)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 146 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> %c512)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX2-LABEL: 'var_funnel_i16'
@@ -228,15 +228,15 @@ define void @var_funnel_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <32 x i
 ; BDVER2-LABEL: 'var_funnel_i16'
 ; BDVER2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I16 = call i16 @llvm.fshl.i16(i16 %a16, i16 %b16, i16 %c16)
 ; BDVER2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> %c128)
-; BDVER2-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> %c256)
-; BDVER2-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> %c512)
+; BDVER2-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> %c256)
+; BDVER2-NEXT:  Cost Model: Found an estimated cost of 46 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> %c512)
 ; BDVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; BTVER2-LABEL: 'var_funnel_i16'
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I16 = call i16 @llvm.fshl.i16(i16 %a16, i16 %b16, i16 %c16)
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> %c128)
-; BTVER2-NEXT:  Cost Model: Found an estimated cost of 71 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> %c256)
-; BTVER2-NEXT:  Cost Model: Found an estimated cost of 142 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> %c512)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 73 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> %c256)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 146 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> %c512)
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %I16    = call i16 @llvm.fshl.i16(i16 %a16, i16 %b16, i16 %c16)
@@ -249,9 +249,9 @@ define void @var_funnel_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <32 x i
 define void @var_funnel_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x i8> %a512, i8 %b8, <16 x i8> %b128, <32 x i8> %b256, <64 x i8> %b512, i8 %c8, <16 x i8> %c128, <32 x i8> %c256, <64 x i8> %c512) {
 ; SSSE3-LABEL: 'var_funnel_i8'
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %b8, i8 %c8)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 57 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> %c128)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 114 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> %c256)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 228 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> %c512)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 59 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> %c128)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 118 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> %c256)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 236 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> %c512)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; SSE42-LABEL: 'var_funnel_i8'
@@ -264,8 +264,8 @@ define void @var_funnel_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x i8> %
 ; AVX1-LABEL: 'var_funnel_i8'
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %b8, i8 %c8)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> %c128)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 61 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> %c256)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 122 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> %c512)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 63 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> %c256)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 126 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> %c512)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX2-LABEL: 'var_funnel_i8'
@@ -313,15 +313,15 @@ define void @var_funnel_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x i8> %
 ; BDVER2-LABEL: 'var_funnel_i8'
 ; BDVER2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %b8, i8 %c8)
 ; BDVER2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> %c128)
-; BDVER2-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> %c256)
-; BDVER2-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> %c512)
+; BDVER2-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> %c256)
+; BDVER2-NEXT:  Cost Model: Found an estimated cost of 46 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> %c512)
 ; BDVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; BTVER2-LABEL: 'var_funnel_i8'
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %b8, i8 %c8)
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> %c128)
-; BTVER2-NEXT:  Cost Model: Found an estimated cost of 61 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> %c256)
-; BTVER2-NEXT:  Cost Model: Found an estimated cost of 122 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> %c512)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 63 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> %c256)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 126 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> %c512)
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %I8    = call i8 @llvm.fshl.i8(i8 %a8, i8 %b8, i8 %c8)
@@ -340,9 +340,9 @@ define void @splatvar_funnel_i64(i64 %a64, <2 x i64> %a128, <4 x i64> %a256, <8
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <2 x i64> %c128, <2 x i64> undef, <2 x i32> zeroinitializer
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %u256 = shufflevector <4 x i64> %c256, <4 x i64> undef, <4 x i32> zeroinitializer
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %u512 = shufflevector <8 x i64> %c512, <8 x i64> undef, <8 x i32> zeroinitializer
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V2I64 = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> %u128)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V4I64 = call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> %u256)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V8I64 = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> %u512)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V2I64 = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> %u128)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V4I64 = call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> %u256)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %V8I64 = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> %u512)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; SSE42-LABEL: 'splatvar_funnel_i64'
@@ -427,14 +427,23 @@ define void @splatvar_funnel_i64(i64 %a64, <2 x i64> %a128, <4 x i64> %a256, <8
 }
 
 define void @splatvar_funnel_i32(i32 %a32, <4 x i32> %a128, <8 x i32> %a256, <16 x i32> %a512, i32 %b32, <4 x i32> %b128, <8 x i32> %b256, <16 x i32> %b512, i32 %c32, <4 x i32> %c128, <8 x i32> %c256, <16 x i32> %c512) {
-; SSE-LABEL: 'splatvar_funnel_i32'
-; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <4 x i32> %c128, <4 x i32> undef, <4 x i32> zeroinitializer
-; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %u256 = shufflevector <8 x i32> %c256, <8 x i32> undef, <8 x i32> zeroinitializer
-; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %u512 = shufflevector <16 x i32> %c512, <16 x i32> undef, <16 x i32> zeroinitializer
-; SSE-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V2I32 = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> %u128)
-; SSE-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I32 = call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> %u256)
-; SSE-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V8I32 = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> %u512)
-; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; SSSE3-LABEL: 'splatvar_funnel_i32'
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <4 x i32> %c128, <4 x i32> undef, <4 x i32> zeroinitializer
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %u256 = shufflevector <8 x i32> %c256, <8 x i32> undef, <8 x i32> zeroinitializer
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %u512 = shufflevector <16 x i32> %c512, <16 x i32> undef, <16 x i32> zeroinitializer
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V2I32 = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> %u128)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V4I32 = call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> %u256)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %V8I32 = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> %u512)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; SSE42-LABEL: 'splatvar_funnel_i32'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <4 x i32> %c128, <4 x i32> undef, <4 x i32> zeroinitializer
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %u256 = shufflevector <8 x i32> %c256, <8 x i32> undef, <8 x i32> zeroinitializer
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %u512 = shufflevector <16 x i32> %c512, <16 x i32> undef, <16 x i32> zeroinitializer
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V2I32 = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> %u128)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I32 = call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> %u256)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V8I32 = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> %u512)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX1-LABEL: 'splatvar_funnel_i32'
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <4 x i32> %c128, <4 x i32> undef, <4 x i32> zeroinitializer
@@ -509,22 +518,31 @@ define void @splatvar_funnel_i32(i32 %a32, <4 x i32> %a128, <8 x i32> %a256, <16
 }
 
 define void @splatvar_funnel_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <32 x i16> %a512, i16 %b16, <8 x i16> %b128, <16 x i16> %b256, <32 x i16> %b512, i16 %c16, <8 x i16> %c128, <16 x i16> %c256, <32 x i16> %c512) {
-; SSE-LABEL: 'splatvar_funnel_i16'
-; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <8 x i16> %c128, <8 x i16> undef, <8 x i32> zeroinitializer
-; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %u256 = shufflevector <16 x i16> %c256, <16 x i16> undef, <16 x i32> zeroinitializer
-; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %u512 = shufflevector <32 x i16> %c512, <32 x i16> undef, <32 x i32> zeroinitializer
-; SSE-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> %u128)
-; SSE-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> %u256)
-; SSE-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> %u512)
-; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; SSSE3-LABEL: 'splatvar_funnel_i16'
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <8 x i16> %c128, <8 x i16> undef, <8 x i32> zeroinitializer
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %u256 = shufflevector <16 x i16> %c256, <16 x i16> undef, <16 x i32> zeroinitializer
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %u512 = shufflevector <32 x i16> %c512, <32 x i16> undef, <32 x i32> zeroinitializer
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> %u128)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> %u256)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> %u512)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; SSE42-LABEL: 'splatvar_funnel_i16'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <8 x i16> %c128, <8 x i16> undef, <8 x i32> zeroinitializer
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %u256 = shufflevector <16 x i16> %c256, <16 x i16> undef, <16 x i32> zeroinitializer
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %u512 = shufflevector <32 x i16> %c512, <32 x i16> undef, <32 x i32> zeroinitializer
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> %u128)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> %u256)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> %u512)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX1-LABEL: 'splatvar_funnel_i16'
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <8 x i16> %c128, <8 x i16> undef, <8 x i32> zeroinitializer
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %u256 = shufflevector <16 x i16> %c256, <16 x i16> undef, <16 x i32> zeroinitializer
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %u512 = shufflevector <32 x i16> %c512, <32 x i16> undef, <32 x i32> zeroinitializer
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> %u128)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> %u256)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 38 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> %u512)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> %u256)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> %u512)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX2-LABEL: 'splatvar_funnel_i16'
@@ -586,8 +604,8 @@ define void @splatvar_funnel_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <3
 ; BDVER2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %u256 = shufflevector <16 x i16> %c256, <16 x i16> undef, <16 x i32> zeroinitializer
 ; BDVER2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %u512 = shufflevector <32 x i16> %c512, <32 x i16> undef, <32 x i32> zeroinitializer
 ; BDVER2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> %u128)
-; BDVER2-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> %u256)
-; BDVER2-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> %u512)
+; BDVER2-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> %u256)
+; BDVER2-NEXT:  Cost Model: Found an estimated cost of 46 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> %u512)
 ; BDVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; BTVER2-LABEL: 'splatvar_funnel_i16'
@@ -595,8 +613,8 @@ define void @splatvar_funnel_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <3
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %u256 = shufflevector <16 x i16> %c256, <16 x i16> undef, <16 x i32> zeroinitializer
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %u512 = shufflevector <32 x i16> %c512, <32 x i16> undef, <32 x i32> zeroinitializer
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> %u128)
-; BTVER2-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> %u256)
-; BTVER2-NEXT:  Cost Model: Found an estimated cost of 38 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> %u512)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> %u256)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> %u512)
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %u128 = shufflevector <8 x i16> %c128, <8 x i16> undef, <8 x i32> zeroinitializer
@@ -613,9 +631,9 @@ define void @splatvar_funnel_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <16 x i8> %c128, <16 x i8> undef, <16 x i32> zeroinitializer
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %u256 = shufflevector <32 x i8> %c256, <32 x i8> undef, <32 x i32> zeroinitializer
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %u512 = shufflevector <64 x i8> %c512, <64 x i8> undef, <64 x i32> zeroinitializer
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 57 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> %u128)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 114 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> %u256)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 228 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> %u512)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 59 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> %u128)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 118 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> %u256)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 236 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> %u512)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; SSE42-LABEL: 'splatvar_funnel_i8'
@@ -632,8 +650,8 @@ define void @splatvar_funnel_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %u256 = shufflevector <32 x i8> %c256, <32 x i8> undef, <32 x i32> zeroinitializer
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %u512 = shufflevector <64 x i8> %c512, <64 x i8> undef, <64 x i32> zeroinitializer
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> %u128)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 61 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> %u256)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 122 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> %u512)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 63 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> %u256)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 126 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> %u512)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX2-LABEL: 'splatvar_funnel_i8'
@@ -695,8 +713,8 @@ define void @splatvar_funnel_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x
 ; BDVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %u256 = shufflevector <32 x i8> %c256, <32 x i8> undef, <32 x i32> zeroinitializer
 ; BDVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %u512 = shufflevector <64 x i8> %c512, <64 x i8> undef, <64 x i32> zeroinitializer
 ; BDVER2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> %u128)
-; BDVER2-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> %u256)
-; BDVER2-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> %u512)
+; BDVER2-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> %u256)
+; BDVER2-NEXT:  Cost Model: Found an estimated cost of 46 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> %u512)
 ; BDVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; BTVER2-LABEL: 'splatvar_funnel_i8'
@@ -704,8 +722,8 @@ define void @splatvar_funnel_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %u256 = shufflevector <32 x i8> %c256, <32 x i8> undef, <32 x i32> zeroinitializer
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %u512 = shufflevector <64 x i8> %c512, <64 x i8> undef, <64 x i32> zeroinitializer
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> %u128)
-; BTVER2-NEXT:  Cost Model: Found an estimated cost of 61 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> %u256)
-; BTVER2-NEXT:  Cost Model: Found an estimated cost of 122 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> %u512)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 63 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> %u256)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 126 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> %u512)
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %u128 = shufflevector <16 x i8> %c128, <16 x i8> undef, <16 x i32> zeroinitializer
@@ -724,9 +742,9 @@ define void @splatvar_funnel_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x
 define void @constant_funnel_i64(i64 %a64, <2 x i64> %a128, <4 x i64> %a256, <8 x i64> %a512, i64 %b64, <2 x i64> %b128, <4 x i64> %b256, <8 x i64> %b512) {
 ; SSSE3-LABEL: 'constant_funnel_i64'
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I64 = call i64 @llvm.fshl.i64(i64 %a64, i64 %b64, i64 7)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V2I64 = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> <i64 1, i64 7>)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 38 for instruction: %V4I64 = call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> <i64 1, i64 7, i64 15, i64 31>)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 76 for instruction: %V8I64 = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> <i64 1, i64 7, i64 15, i64 31, i64 1, i64 7, i64 15, i64 31>)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %V2I64 = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> <i64 1, i64 7>)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %V4I64 = call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> <i64 1, i64 7, i64 15, i64 31>)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 84 for instruction: %V8I64 = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> <i64 1, i64 7, i64 15, i64 31, i64 1, i64 7, i64 15, i64 31>)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; SSE42-LABEL: 'constant_funnel_i64'
@@ -795,9 +813,9 @@ define void @constant_funnel_i64(i64 %a64, <2 x i64> %a128, <4 x i64> %a256, <8
 define void @constant_funnel_i32(i32 %a32, <4 x i32> %a128, <8 x i32> %a256, <16 x i32> %a512, i32 %b32, <4 x i32> %b128, <8 x i32> %b256, <16 x i32> %b512) {
 ; SSSE3-LABEL: 'constant_funnel_i32'
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I32 = call i32 @llvm.fshl.i32(i32 %a32, i32 %b32, i32 7)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V2I32 = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> <i32 4, i32 5, i32 6, i32 7>)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 52 for instruction: %V4I32 = call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3>)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 104 for instruction: %V8I32 = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3>)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V2I32 = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> <i32 4, i32 5, i32 6, i32 7>)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V4I32 = call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3>)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 112 for instruction: %V8I32 = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3>)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; SSE42-LABEL: 'constant_funnel_i32'
@@ -866,9 +884,9 @@ define void @constant_funnel_i32(i32 %a32, <4 x i32> %a128, <8 x i32> %a256, <16
 define void @constant_funnel_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <32 x i16> %a512, i16 %b16, <8 x i16> %b128, <16 x i16> %b256, <32 x i16> %b512) {
 ; SSSE3-LABEL: 'constant_funnel_i16'
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I16 = call i16 @llvm.fshl.i16(i16 %a16, i16 %b16, i16 7)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 37 for instruction: %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7>)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 74 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7>)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 148 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7>)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 39 for instruction: %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7>)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 78 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7>)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 156 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7>)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; SSE42-LABEL: 'constant_funnel_i16'
@@ -881,8 +899,8 @@ define void @constant_funnel_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <3
 ; AVX1-LABEL: 'constant_funnel_i16'
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I16 = call i16 @llvm.fshl.i16(i16 %a16, i16 %b16, i16 7)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7>)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 44 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7>)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 88 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7>)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 46 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7>)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 92 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7>)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX2-LABEL: 'constant_funnel_i16'
@@ -930,15 +948,15 @@ define void @constant_funnel_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <3
 ; BDVER2-LABEL: 'constant_funnel_i16'
 ; BDVER2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I16 = call i16 @llvm.fshl.i16(i16 %a16, i16 %b16, i16 7)
 ; BDVER2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7>)
-; BDVER2-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7>)
-; BDVER2-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7>)
+; BDVER2-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7>)
+; BDVER2-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7>)
 ; BDVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; BTVER2-LABEL: 'constant_funnel_i16'
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I16 = call i16 @llvm.fshl.i16(i16 %a16, i16 %b16, i16 7)
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7>)
-; BTVER2-NEXT:  Cost Model: Found an estimated cost of 44 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7>)
-; BTVER2-NEXT:  Cost Model: Found an estimated cost of 88 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7>)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 46 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7>)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 92 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7>)
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %I16    = call i16 @llvm.fshl.i16(i16 %a16, i16 %b16, i16 7)
@@ -951,9 +969,9 @@ define void @constant_funnel_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <3
 define void @constant_funnel_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x i8> %a512, i8 %b8, <16 x i8> %b128, <32 x i8> %b256, <64 x i8> %b512) {
 ; SSSE3-LABEL: 'constant_funnel_i8'
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %b8, i8 7)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0>)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 112 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0>)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 224 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0>)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 58 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0>)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 116 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0>)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 232 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0>)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; SSE42-LABEL: 'constant_funnel_i8'
@@ -966,8 +984,8 @@ define void @constant_funnel_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x
 ; AVX1-LABEL: 'constant_funnel_i8'
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %b8, i8 7)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0>)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 60 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0>)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 120 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0>)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 62 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0>)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 124 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0>)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX2-LABEL: 'constant_funnel_i8'
@@ -1015,15 +1033,15 @@ define void @constant_funnel_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x
 ; BDVER2-LABEL: 'constant_funnel_i8'
 ; BDVER2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %b8, i8 7)
 ; BDVER2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0>)
-; BDVER2-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0>)
-; BDVER2-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0>)
+; BDVER2-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0>)
+; BDVER2-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0>)
 ; BDVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; BTVER2-LABEL: 'constant_funnel_i8'
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %b8, i8 7)
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0>)
-; BTVER2-NEXT:  Cost Model: Found an estimated cost of 60 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0>)
-; BTVER2-NEXT:  Cost Model: Found an estimated cost of 120 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0>)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 62 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0>)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 124 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0>)
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %I8    = call i8 @llvm.fshl.i8(i8 %a8, i8 %b8, i8 7)
@@ -1040,9 +1058,9 @@ define void @constant_funnel_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x
 define void @splatconstant_funnel_i64(i64 %a64, <2 x i64> %a128, <4 x i64> %a256, <8 x i64> %a512, i64 %b64, <2 x i64> %b128, <4 x i64> %b256, <8 x i64> %b512) {
 ; SSSE3-LABEL: 'splatconstant_funnel_i64'
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I64 = call i64 @llvm.fshl.i64(i64 %a64, i64 %b64, i64 7)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V2I64 = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> <i64 7, i64 7>)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V4I64 = call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> <i64 7, i64 7, i64 7, i64 7>)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 52 for instruction: %V8I64 = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> <i64 7, i64 7, i64 7, i64 7, i64 7, i64 7, i64 7, i64 7>)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V2I64 = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> <i64 7, i64 7>)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %V4I64 = call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> <i64 7, i64 7, i64 7, i64 7>)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 60 for instruction: %V8I64 = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> <i64 7, i64 7, i64 7, i64 7, i64 7, i64 7, i64 7, i64 7>)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; SSE42-LABEL: 'splatconstant_funnel_i64'
@@ -1109,12 +1127,19 @@ define void @splatconstant_funnel_i64(i64 %a64, <2 x i64> %a128, <4 x i64> %a256
 }
 
 define void @splatconstant_funnel_i32(i32 %a32, <4 x i32> %a128, <8 x i32> %a256, <16 x i32> %a512, i32 %b32, <4 x i32> %b128, <8 x i32> %b256, <16 x i32> %b512) {
-; SSE-LABEL: 'splatconstant_funnel_i32'
-; SSE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I32 = call i32 @llvm.fshl.i32(i32 %a32, i32 %b32, i32 5)
-; SSE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I32 = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> <i32 5, i32 5, i32 5, i32 5>)
-; SSE-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> <i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5>)
-; SSE-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I32 = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> <i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5>)
-; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; SSSE3-LABEL: 'splatconstant_funnel_i32'
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I32 = call i32 @llvm.fshl.i32(i32 %a32, i32 %b32, i32 5)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2I32 = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> <i32 5, i32 5, i32 5, i32 5>)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V4I32 = call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> <i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5>)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V8I32 = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> <i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5>)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; SSE42-LABEL: 'splatconstant_funnel_i32'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I32 = call i32 @llvm.fshl.i32(i32 %a32, i32 %b32, i32 5)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I32 = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> <i32 5, i32 5, i32 5, i32 5>)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> <i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5>)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I32 = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> <i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5>)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX1-LABEL: 'splatconstant_funnel_i32'
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I32 = call i32 @llvm.fshl.i32(i32 %a32, i32 %b32, i32 5)
@@ -1173,18 +1198,25 @@ define void @splatconstant_funnel_i32(i32 %a32, <4 x i32> %a128, <8 x i32> %a256
 }
 
 define void @splatconstant_funnel_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <32 x i16> %a512, i16 %b16, <8 x i16> %b128, <16 x i16> %b256, <32 x i16> %b512) {
-; SSE-LABEL: 'splatconstant_funnel_i16'
-; SSE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I16 = call i16 @llvm.fshl.i16(i16 %a16, i16 %b16, i16 3)
-; SSE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> <i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3>)
-; SSE-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> <i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3>)
-; SSE-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> <i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3>)
-; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; SSSE3-LABEL: 'splatconstant_funnel_i16'
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I16 = call i16 @llvm.fshl.i16(i16 %a16, i16 %b16, i16 3)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> <i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3>)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> <i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3>)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> <i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3>)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; SSE42-LABEL: 'splatconstant_funnel_i16'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I16 = call i16 @llvm.fshl.i16(i16 %a16, i16 %b16, i16 3)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> <i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3>)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> <i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3>)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> <i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3>)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX1-LABEL: 'splatconstant_funnel_i16'
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I16 = call i16 @llvm.fshl.i16(i16 %a16, i16 %b16, i16 3)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> <i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3>)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> <i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3>)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> <i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3>)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> <i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3>)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> <i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3>)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX2-LABEL: 'splatconstant_funnel_i16'
@@ -1232,15 +1264,15 @@ define void @splatconstant_funnel_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a25
 ; BDVER2-LABEL: 'splatconstant_funnel_i16'
 ; BDVER2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I16 = call i16 @llvm.fshl.i16(i16 %a16, i16 %b16, i16 3)
 ; BDVER2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> <i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3>)
-; BDVER2-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> <i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3>)
-; BDVER2-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> <i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3>)
+; BDVER2-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> <i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3>)
+; BDVER2-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> <i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3>)
 ; BDVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; BTVER2-LABEL: 'splatconstant_funnel_i16'
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I16 = call i16 @llvm.fshl.i16(i16 %a16, i16 %b16, i16 3)
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V8I16 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> <i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3>)
-; BTVER2-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> <i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3>)
-; BTVER2-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> <i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3>)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V16I16 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> <i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3>)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %V32I16 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> <i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3>)
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %I16    = call i16 @llvm.fshl.i16(i16 %a16, i16 %b16, i16 3)
@@ -1251,18 +1283,25 @@ define void @splatconstant_funnel_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a25
 }
 
 define void @splatconstant_funnel_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x i8> %a512, i8 %b8, <16 x i8> %b128, <32 x i8> %b256, <64 x i8> %b512) {
-; SSE-LABEL: 'splatconstant_funnel_i8'
-; SSE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %b8, i8 3)
-; SSE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
-; SSE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
-; SSE-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
-; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; SSSE3-LABEL: 'splatconstant_funnel_i8'
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %b8, i8 3)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; SSE42-LABEL: 'splatconstant_funnel_i8'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %b8, i8 3)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX1-LABEL: 'splatconstant_funnel_i8'
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %b8, i8 3)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 44 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX2-LABEL: 'splatconstant_funnel_i8'
@@ -1310,15 +1349,15 @@ define void @splatconstant_funnel_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <
 ; BDVER2-LABEL: 'splatconstant_funnel_i8'
 ; BDVER2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %b8, i8 3)
 ; BDVER2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
-; BDVER2-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
-; BDVER2-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
+; BDVER2-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
+; BDVER2-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
 ; BDVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; BTVER2-LABEL: 'splatconstant_funnel_i8'
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I8 = call i8 @llvm.fshl.i8(i8 %a8, i8 %b8, i8 3)
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I8 = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
-; BTVER2-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
-; BTVER2-NEXT:  Cost Model: Found an estimated cost of 44 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V32I8 = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V64I8 = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %I8    = call i8 @llvm.fshl.i8(i8 %a8, i8 %b8, i8 3)
diff --git a/test/Analysis/CostModel/X86/fshr.ll b/test/Analysis/CostModel/X86/fshr.ll
index 7450fc73d40703a1959b5cf18fb582dd5be21663..31e409cc9a7b823be8919bf934ee68b040ebd13e 100644
--- a/test/Analysis/CostModel/X86/fshr.ll
+++ b/test/Analysis/CostModel/X86/fshr.ll
@@ -22,9 +22,9 @@ target triple = "x86_64-apple-macosx10.8.0"
 define void @var_funnel_i64(i64 %a64, <2 x i64> %a128, <4 x i64> %a256, <8 x i64> %a512, i64 %b64, <2 x i64> %b128, <4 x i64> %b256, <8 x i64> %b512, i64 %c64, <2 x i64> %c128, <4 x i64> %c256, <8 x i64> %c512) {
 ; SSSE3-LABEL: 'var_funnel_i64'
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I64 = call i64 @llvm.fshr.i64(i64 %a64, i64 %b64, i64 %c64)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V2I64 = call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> %c128)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %V4I64 = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> %c256)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %V8I64 = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> %c512)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V2I64 = call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> %c128)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 44 for instruction: %V4I64 = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> %c256)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 88 for instruction: %V8I64 = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> %c512)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; SSE42-LABEL: 'var_funnel_i64'
@@ -93,9 +93,9 @@ define void @var_funnel_i64(i64 %a64, <2 x i64> %a128, <4 x i64> %a256, <8 x i64
 define void @var_funnel_i32(i32 %a32, <4 x i32> %a128, <8 x i32> %a256, <16 x i32> %a512, i32 %b32, <4 x i32> %b128, <8 x i32> %b256, <16 x i32> %b512, i32 %c32, <4 x i32> %c128, <8 x i32> %c256, <16 x i32> %c512) {
 ; SSSE3-LABEL: 'var_funnel_i32'
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I32 = call i32 @llvm.fshr.i32(i32 %a32, i32 %b32, i32 %c32)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %V2I32 = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> %c128)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 62 for instruction: %V4I32 = call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> %c256)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 124 for instruction: %V8I32 = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> %c512)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %V2I32 = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> %c128)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %V4I32 = call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> %c256)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 132 for instruction: %V8I32 = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> %c512)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; SSE42-LABEL: 'var_funnel_i32'
@@ -164,9 +164,9 @@ define void @var_funnel_i32(i32 %a32, <4 x i32> %a128, <8 x i32> %a256, <16 x i3
 define void @var_funnel_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <32 x i16> %a512, i16 %b16, <8 x i16> %b128, <16 x i16> %b256, <32 x i16> %b512, i16 %c16, <8 x i16> %c128, <16 x i16> %c256, <32 x i16> %c512) {
 ; SSSE3-LABEL: 'var_funnel_i16'
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I16 = call i16 @llvm.fshr.i16(i16 %a16, i16 %b16, i16 %c16)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 69 for instruction: %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> %c128)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 138 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> %c256)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 276 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> %c512)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 71 for instruction: %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> %c128)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 142 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> %c256)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 284 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> %c512)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; SSE42-LABEL: 'var_funnel_i16'
@@ -179,8 +179,8 @@ define void @var_funnel_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <32 x i
 ; AVX1-LABEL: 'var_funnel_i16'
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I16 = call i16 @llvm.fshr.i16(i16 %a16, i16 %b16, i16 %c16)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> %c128)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 71 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> %c256)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 142 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> %c512)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 73 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> %c256)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 146 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> %c512)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX2-LABEL: 'var_funnel_i16'
@@ -228,15 +228,15 @@ define void @var_funnel_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <32 x i
 ; BDVER2-LABEL: 'var_funnel_i16'
 ; BDVER2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I16 = call i16 @llvm.fshr.i16(i16 %a16, i16 %b16, i16 %c16)
 ; BDVER2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> %c128)
-; BDVER2-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> %c256)
-; BDVER2-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> %c512)
+; BDVER2-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> %c256)
+; BDVER2-NEXT:  Cost Model: Found an estimated cost of 46 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> %c512)
 ; BDVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; BTVER2-LABEL: 'var_funnel_i16'
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I16 = call i16 @llvm.fshr.i16(i16 %a16, i16 %b16, i16 %c16)
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> %c128)
-; BTVER2-NEXT:  Cost Model: Found an estimated cost of 71 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> %c256)
-; BTVER2-NEXT:  Cost Model: Found an estimated cost of 142 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> %c512)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 73 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> %c256)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 146 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> %c512)
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %I16    = call i16 @llvm.fshr.i16(i16 %a16, i16 %b16, i16 %c16)
@@ -249,9 +249,9 @@ define void @var_funnel_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <32 x i
 define void @var_funnel_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x i8> %a512, i8 %b8, <16 x i8> %b128, <32 x i8> %b256, <64 x i8> %b512, i8 %c8, <16 x i8> %c128, <32 x i8> %c256, <64 x i8> %c512) {
 ; SSSE3-LABEL: 'var_funnel_i8'
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %b8, i8 %c8)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 57 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> %c128)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 114 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> %c256)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 228 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> %c512)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 59 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> %c128)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 118 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> %c256)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 236 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> %c512)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; SSE42-LABEL: 'var_funnel_i8'
@@ -264,8 +264,8 @@ define void @var_funnel_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x i8> %
 ; AVX1-LABEL: 'var_funnel_i8'
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %b8, i8 %c8)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> %c128)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 61 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> %c256)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 122 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> %c512)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 63 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> %c256)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 126 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> %c512)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX2-LABEL: 'var_funnel_i8'
@@ -313,15 +313,15 @@ define void @var_funnel_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x i8> %
 ; BDVER2-LABEL: 'var_funnel_i8'
 ; BDVER2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %b8, i8 %c8)
 ; BDVER2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> %c128)
-; BDVER2-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> %c256)
-; BDVER2-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> %c512)
+; BDVER2-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> %c256)
+; BDVER2-NEXT:  Cost Model: Found an estimated cost of 46 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> %c512)
 ; BDVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; BTVER2-LABEL: 'var_funnel_i8'
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %b8, i8 %c8)
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> %c128)
-; BTVER2-NEXT:  Cost Model: Found an estimated cost of 61 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> %c256)
-; BTVER2-NEXT:  Cost Model: Found an estimated cost of 122 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> %c512)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 63 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> %c256)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 126 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> %c512)
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %I8    = call i8 @llvm.fshr.i8(i8 %a8, i8 %b8, i8 %c8)
@@ -340,9 +340,9 @@ define void @splatvar_funnel_i64(i64 %a64, <2 x i64> %a128, <4 x i64> %a256, <8
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <2 x i64> %c128, <2 x i64> undef, <2 x i32> zeroinitializer
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %u256 = shufflevector <4 x i64> %c256, <4 x i64> undef, <4 x i32> zeroinitializer
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %u512 = shufflevector <8 x i64> %c512, <8 x i64> undef, <8 x i32> zeroinitializer
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V2I64 = call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> %u128)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V4I64 = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> %u256)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V8I64 = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> %u512)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V2I64 = call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> %u128)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V4I64 = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> %u256)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %V8I64 = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> %u512)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; SSE42-LABEL: 'splatvar_funnel_i64'
@@ -427,14 +427,23 @@ define void @splatvar_funnel_i64(i64 %a64, <2 x i64> %a128, <4 x i64> %a256, <8
 }
 
 define void @splatvar_funnel_i32(i32 %a32, <4 x i32> %a128, <8 x i32> %a256, <16 x i32> %a512, i32 %b32, <4 x i32> %b128, <8 x i32> %b256, <16 x i32> %b512, i32 %c32, <4 x i32> %c128, <8 x i32> %c256, <16 x i32> %c512) {
-; SSE-LABEL: 'splatvar_funnel_i32'
-; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <4 x i32> %c128, <4 x i32> undef, <4 x i32> zeroinitializer
-; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %u256 = shufflevector <8 x i32> %c256, <8 x i32> undef, <8 x i32> zeroinitializer
-; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %u512 = shufflevector <16 x i32> %c512, <16 x i32> undef, <16 x i32> zeroinitializer
-; SSE-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V2I32 = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> %u128)
-; SSE-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I32 = call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> %u256)
-; SSE-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V8I32 = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> %u512)
-; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; SSSE3-LABEL: 'splatvar_funnel_i32'
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <4 x i32> %c128, <4 x i32> undef, <4 x i32> zeroinitializer
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %u256 = shufflevector <8 x i32> %c256, <8 x i32> undef, <8 x i32> zeroinitializer
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %u512 = shufflevector <16 x i32> %c512, <16 x i32> undef, <16 x i32> zeroinitializer
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V2I32 = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> %u128)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V4I32 = call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> %u256)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %V8I32 = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> %u512)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; SSE42-LABEL: 'splatvar_funnel_i32'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <4 x i32> %c128, <4 x i32> undef, <4 x i32> zeroinitializer
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %u256 = shufflevector <8 x i32> %c256, <8 x i32> undef, <8 x i32> zeroinitializer
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %u512 = shufflevector <16 x i32> %c512, <16 x i32> undef, <16 x i32> zeroinitializer
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V2I32 = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> %u128)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I32 = call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> %u256)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V8I32 = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> %u512)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX1-LABEL: 'splatvar_funnel_i32'
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <4 x i32> %c128, <4 x i32> undef, <4 x i32> zeroinitializer
@@ -509,22 +518,31 @@ define void @splatvar_funnel_i32(i32 %a32, <4 x i32> %a128, <8 x i32> %a256, <16
 }
 
 define void @splatvar_funnel_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <32 x i16> %a512, i16 %b16, <8 x i16> %b128, <16 x i16> %b256, <32 x i16> %b512, i16 %c16, <8 x i16> %c128, <16 x i16> %c256, <32 x i16> %c512) {
-; SSE-LABEL: 'splatvar_funnel_i16'
-; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <8 x i16> %c128, <8 x i16> undef, <8 x i32> zeroinitializer
-; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %u256 = shufflevector <16 x i16> %c256, <16 x i16> undef, <16 x i32> zeroinitializer
-; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %u512 = shufflevector <32 x i16> %c512, <32 x i16> undef, <32 x i32> zeroinitializer
-; SSE-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> %u128)
-; SSE-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> %u256)
-; SSE-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> %u512)
-; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; SSSE3-LABEL: 'splatvar_funnel_i16'
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <8 x i16> %c128, <8 x i16> undef, <8 x i32> zeroinitializer
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %u256 = shufflevector <16 x i16> %c256, <16 x i16> undef, <16 x i32> zeroinitializer
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %u512 = shufflevector <32 x i16> %c512, <32 x i16> undef, <32 x i32> zeroinitializer
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> %u128)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> %u256)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> %u512)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; SSE42-LABEL: 'splatvar_funnel_i16'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <8 x i16> %c128, <8 x i16> undef, <8 x i32> zeroinitializer
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %u256 = shufflevector <16 x i16> %c256, <16 x i16> undef, <16 x i32> zeroinitializer
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %u512 = shufflevector <32 x i16> %c512, <32 x i16> undef, <32 x i32> zeroinitializer
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> %u128)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> %u256)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> %u512)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX1-LABEL: 'splatvar_funnel_i16'
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <8 x i16> %c128, <8 x i16> undef, <8 x i32> zeroinitializer
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %u256 = shufflevector <16 x i16> %c256, <16 x i16> undef, <16 x i32> zeroinitializer
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %u512 = shufflevector <32 x i16> %c512, <32 x i16> undef, <32 x i32> zeroinitializer
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> %u128)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> %u256)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 38 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> %u512)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> %u256)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> %u512)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX2-LABEL: 'splatvar_funnel_i16'
@@ -586,8 +604,8 @@ define void @splatvar_funnel_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <3
 ; BDVER2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %u256 = shufflevector <16 x i16> %c256, <16 x i16> undef, <16 x i32> zeroinitializer
 ; BDVER2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %u512 = shufflevector <32 x i16> %c512, <32 x i16> undef, <32 x i32> zeroinitializer
 ; BDVER2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> %u128)
-; BDVER2-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> %u256)
-; BDVER2-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> %u512)
+; BDVER2-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> %u256)
+; BDVER2-NEXT:  Cost Model: Found an estimated cost of 46 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> %u512)
 ; BDVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; BTVER2-LABEL: 'splatvar_funnel_i16'
@@ -595,8 +613,8 @@ define void @splatvar_funnel_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <3
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %u256 = shufflevector <16 x i16> %c256, <16 x i16> undef, <16 x i32> zeroinitializer
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %u512 = shufflevector <32 x i16> %c512, <32 x i16> undef, <32 x i32> zeroinitializer
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> %u128)
-; BTVER2-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> %u256)
-; BTVER2-NEXT:  Cost Model: Found an estimated cost of 38 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> %u512)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> %u256)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> %u512)
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %u128 = shufflevector <8 x i16> %c128, <8 x i16> undef, <8 x i32> zeroinitializer
@@ -613,9 +631,9 @@ define void @splatvar_funnel_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <16 x i8> %c128, <16 x i8> undef, <16 x i32> zeroinitializer
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %u256 = shufflevector <32 x i8> %c256, <32 x i8> undef, <32 x i32> zeroinitializer
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %u512 = shufflevector <64 x i8> %c512, <64 x i8> undef, <64 x i32> zeroinitializer
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 57 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> %u128)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 114 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> %u256)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 228 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> %u512)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 59 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> %u128)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 118 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> %u256)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 236 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> %u512)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; SSE42-LABEL: 'splatvar_funnel_i8'
@@ -632,8 +650,8 @@ define void @splatvar_funnel_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %u256 = shufflevector <32 x i8> %c256, <32 x i8> undef, <32 x i32> zeroinitializer
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %u512 = shufflevector <64 x i8> %c512, <64 x i8> undef, <64 x i32> zeroinitializer
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> %u128)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 61 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> %u256)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 122 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> %u512)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 63 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> %u256)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 126 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> %u512)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX2-LABEL: 'splatvar_funnel_i8'
@@ -695,8 +713,8 @@ define void @splatvar_funnel_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x
 ; BDVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %u256 = shufflevector <32 x i8> %c256, <32 x i8> undef, <32 x i32> zeroinitializer
 ; BDVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %u512 = shufflevector <64 x i8> %c512, <64 x i8> undef, <64 x i32> zeroinitializer
 ; BDVER2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> %u128)
-; BDVER2-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> %u256)
-; BDVER2-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> %u512)
+; BDVER2-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> %u256)
+; BDVER2-NEXT:  Cost Model: Found an estimated cost of 46 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> %u512)
 ; BDVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; BTVER2-LABEL: 'splatvar_funnel_i8'
@@ -704,8 +722,8 @@ define void @splatvar_funnel_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %u256 = shufflevector <32 x i8> %c256, <32 x i8> undef, <32 x i32> zeroinitializer
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %u512 = shufflevector <64 x i8> %c512, <64 x i8> undef, <64 x i32> zeroinitializer
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> %u128)
-; BTVER2-NEXT:  Cost Model: Found an estimated cost of 61 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> %u256)
-; BTVER2-NEXT:  Cost Model: Found an estimated cost of 122 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> %u512)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 63 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> %u256)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 126 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> %u512)
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %u128 = shufflevector <16 x i8> %c128, <16 x i8> undef, <16 x i32> zeroinitializer
@@ -724,9 +742,9 @@ define void @splatvar_funnel_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x
 define void @constant_funnel_i64(i64 %a64, <2 x i64> %a128, <4 x i64> %a256, <8 x i64> %a512, i64 %b64, <2 x i64> %b128, <4 x i64> %b256, <8 x i64> %b512) {
 ; SSSE3-LABEL: 'constant_funnel_i64'
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I64 = call i64 @llvm.fshr.i64(i64 %a64, i64 %b64, i64 7)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V2I64 = call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> <i64 1, i64 7>)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 38 for instruction: %V4I64 = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> <i64 1, i64 7, i64 15, i64 31>)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 76 for instruction: %V8I64 = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> <i64 1, i64 7, i64 15, i64 31, i64 1, i64 7, i64 15, i64 31>)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %V2I64 = call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> <i64 1, i64 7>)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %V4I64 = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> <i64 1, i64 7, i64 15, i64 31>)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 84 for instruction: %V8I64 = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> <i64 1, i64 7, i64 15, i64 31, i64 1, i64 7, i64 15, i64 31>)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; SSE42-LABEL: 'constant_funnel_i64'
@@ -795,9 +813,9 @@ define void @constant_funnel_i64(i64 %a64, <2 x i64> %a128, <4 x i64> %a256, <8
 define void @constant_funnel_i32(i32 %a32, <4 x i32> %a128, <8 x i32> %a256, <16 x i32> %a512, i32 %b32, <4 x i32> %b128, <8 x i32> %b256, <16 x i32> %b512) {
 ; SSSE3-LABEL: 'constant_funnel_i32'
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I32 = call i32 @llvm.fshr.i32(i32 %a32, i32 %b32, i32 7)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V2I32 = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> <i32 4, i32 5, i32 6, i32 7>)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 52 for instruction: %V4I32 = call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3>)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 104 for instruction: %V8I32 = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3>)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V2I32 = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> <i32 4, i32 5, i32 6, i32 7>)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V4I32 = call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3>)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 112 for instruction: %V8I32 = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3>)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; SSE42-LABEL: 'constant_funnel_i32'
@@ -866,9 +884,9 @@ define void @constant_funnel_i32(i32 %a32, <4 x i32> %a128, <8 x i32> %a256, <16
 define void @constant_funnel_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <32 x i16> %a512, i16 %b16, <8 x i16> %b128, <16 x i16> %b256, <32 x i16> %b512) {
 ; SSSE3-LABEL: 'constant_funnel_i16'
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I16 = call i16 @llvm.fshr.i16(i16 %a16, i16 %b16, i16 7)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 37 for instruction: %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7>)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 74 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7>)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 148 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7>)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 39 for instruction: %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7>)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 78 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7>)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 156 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7>)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; SSE42-LABEL: 'constant_funnel_i16'
@@ -881,8 +899,8 @@ define void @constant_funnel_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <3
 ; AVX1-LABEL: 'constant_funnel_i16'
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I16 = call i16 @llvm.fshr.i16(i16 %a16, i16 %b16, i16 7)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7>)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 44 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7>)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 88 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7>)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 46 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7>)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 92 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7>)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX2-LABEL: 'constant_funnel_i16'
@@ -930,15 +948,15 @@ define void @constant_funnel_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <3
 ; BDVER2-LABEL: 'constant_funnel_i16'
 ; BDVER2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I16 = call i16 @llvm.fshr.i16(i16 %a16, i16 %b16, i16 7)
 ; BDVER2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7>)
-; BDVER2-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7>)
-; BDVER2-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7>)
+; BDVER2-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7>)
+; BDVER2-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7>)
 ; BDVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; BTVER2-LABEL: 'constant_funnel_i16'
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I16 = call i16 @llvm.fshr.i16(i16 %a16, i16 %b16, i16 7)
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7>)
-; BTVER2-NEXT:  Cost Model: Found an estimated cost of 44 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7>)
-; BTVER2-NEXT:  Cost Model: Found an estimated cost of 88 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7>)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 46 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7>)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 92 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7>)
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %I16    = call i16 @llvm.fshr.i16(i16 %a16, i16 %b16, i16 7)
@@ -951,9 +969,9 @@ define void @constant_funnel_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <3
 define void @constant_funnel_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x i8> %a512, i8 %b8, <16 x i8> %b128, <32 x i8> %b256, <64 x i8> %b512) {
 ; SSSE3-LABEL: 'constant_funnel_i8'
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %b8, i8 7)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0>)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 112 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0>)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 224 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0>)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 58 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0>)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 116 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0>)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 232 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0>)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; SSE42-LABEL: 'constant_funnel_i8'
@@ -966,8 +984,8 @@ define void @constant_funnel_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x
 ; AVX1-LABEL: 'constant_funnel_i8'
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %b8, i8 7)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0>)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 60 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0>)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 120 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0>)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 62 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0>)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 124 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0>)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX2-LABEL: 'constant_funnel_i8'
@@ -1015,15 +1033,15 @@ define void @constant_funnel_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x
 ; BDVER2-LABEL: 'constant_funnel_i8'
 ; BDVER2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %b8, i8 7)
 ; BDVER2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0>)
-; BDVER2-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0>)
-; BDVER2-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0>)
+; BDVER2-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0>)
+; BDVER2-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0>)
 ; BDVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; BTVER2-LABEL: 'constant_funnel_i8'
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %b8, i8 7)
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0>)
-; BTVER2-NEXT:  Cost Model: Found an estimated cost of 60 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0>)
-; BTVER2-NEXT:  Cost Model: Found an estimated cost of 120 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0>)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 62 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0>)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 124 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0>)
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %I8    = call i8 @llvm.fshr.i8(i8 %a8, i8 %b8, i8 7)
@@ -1040,9 +1058,9 @@ define void @constant_funnel_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x
 define void @splatconstant_funnel_i64(i64 %a64, <2 x i64> %a128, <4 x i64> %a256, <8 x i64> %a512, i64 %b64, <2 x i64> %b128, <4 x i64> %b256, <8 x i64> %b512) {
 ; SSSE3-LABEL: 'splatconstant_funnel_i64'
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I64 = call i64 @llvm.fshr.i64(i64 %a64, i64 %b64, i64 7)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V2I64 = call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> <i64 7, i64 7>)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V4I64 = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> <i64 7, i64 7, i64 7, i64 7>)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 52 for instruction: %V8I64 = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> <i64 7, i64 7, i64 7, i64 7, i64 7, i64 7, i64 7, i64 7>)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V2I64 = call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> <i64 7, i64 7>)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %V4I64 = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> <i64 7, i64 7, i64 7, i64 7>)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 60 for instruction: %V8I64 = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> <i64 7, i64 7, i64 7, i64 7, i64 7, i64 7, i64 7, i64 7>)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; SSE42-LABEL: 'splatconstant_funnel_i64'
@@ -1109,12 +1127,19 @@ define void @splatconstant_funnel_i64(i64 %a64, <2 x i64> %a128, <4 x i64> %a256
 }
 
 define void @splatconstant_funnel_i32(i32 %a32, <4 x i32> %a128, <8 x i32> %a256, <16 x i32> %a512, i32 %b32, <4 x i32> %b128, <8 x i32> %b256, <16 x i32> %b512) {
-; SSE-LABEL: 'splatconstant_funnel_i32'
-; SSE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I32 = call i32 @llvm.fshr.i32(i32 %a32, i32 %b32, i32 5)
-; SSE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I32 = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> <i32 5, i32 5, i32 5, i32 5>)
-; SSE-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> <i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5>)
-; SSE-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I32 = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> <i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5>)
-; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; SSSE3-LABEL: 'splatconstant_funnel_i32'
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I32 = call i32 @llvm.fshr.i32(i32 %a32, i32 %b32, i32 5)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2I32 = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> <i32 5, i32 5, i32 5, i32 5>)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V4I32 = call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> <i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5>)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V8I32 = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> <i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5>)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; SSE42-LABEL: 'splatconstant_funnel_i32'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I32 = call i32 @llvm.fshr.i32(i32 %a32, i32 %b32, i32 5)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I32 = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %a128, <4 x i32> %b128, <4 x i32> <i32 5, i32 5, i32 5, i32 5>)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I32 = call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %a256, <8 x i32> %b256, <8 x i32> <i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5>)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I32 = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %a512, <16 x i32> %b512, <16 x i32> <i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5>)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX1-LABEL: 'splatconstant_funnel_i32'
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I32 = call i32 @llvm.fshr.i32(i32 %a32, i32 %b32, i32 5)
@@ -1173,18 +1198,25 @@ define void @splatconstant_funnel_i32(i32 %a32, <4 x i32> %a128, <8 x i32> %a256
 }
 
 define void @splatconstant_funnel_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a256, <32 x i16> %a512, i16 %b16, <8 x i16> %b128, <16 x i16> %b256, <32 x i16> %b512) {
-; SSE-LABEL: 'splatconstant_funnel_i16'
-; SSE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I16 = call i16 @llvm.fshr.i16(i16 %a16, i16 %b16, i16 3)
-; SSE-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> <i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3>)
-; SSE-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> <i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3>)
-; SSE-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> <i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3>)
-; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; SSSE3-LABEL: 'splatconstant_funnel_i16'
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I16 = call i16 @llvm.fshr.i16(i16 %a16, i16 %b16, i16 3)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> <i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3>)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> <i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3>)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> <i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3>)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; SSE42-LABEL: 'splatconstant_funnel_i16'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I16 = call i16 @llvm.fshr.i16(i16 %a16, i16 %b16, i16 3)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> <i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3>)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> <i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3>)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> <i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3>)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX1-LABEL: 'splatconstant_funnel_i16'
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I16 = call i16 @llvm.fshr.i16(i16 %a16, i16 %b16, i16 3)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> <i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3>)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> <i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3>)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> <i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3>)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> <i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3>)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> <i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3>)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX2-LABEL: 'splatconstant_funnel_i16'
@@ -1232,15 +1264,15 @@ define void @splatconstant_funnel_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a25
 ; BDVER2-LABEL: 'splatconstant_funnel_i16'
 ; BDVER2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I16 = call i16 @llvm.fshr.i16(i16 %a16, i16 %b16, i16 3)
 ; BDVER2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> <i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3>)
-; BDVER2-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> <i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3>)
-; BDVER2-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> <i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3>)
+; BDVER2-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> <i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3>)
+; BDVER2-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> <i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3>)
 ; BDVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; BTVER2-LABEL: 'splatconstant_funnel_i16'
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I16 = call i16 @llvm.fshr.i16(i16 %a16, i16 %b16, i16 3)
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V8I16 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %a128, <8 x i16> %b128, <8 x i16> <i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3>)
-; BTVER2-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> <i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3>)
-; BTVER2-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> <i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3>)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V16I16 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %a256, <16 x i16> %b256, <16 x i16> <i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3>)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %V32I16 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %a512, <32 x i16> %b512, <32 x i16> <i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3>)
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %I16    = call i16 @llvm.fshr.i16(i16 %a16, i16 %b16, i16 3)
@@ -1251,18 +1283,25 @@ define void @splatconstant_funnel_i16(i16 %a16, <8 x i16> %a128, <16 x i16> %a25
 }
 
 define void @splatconstant_funnel_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <64 x i8> %a512, i8 %b8, <16 x i8> %b128, <32 x i8> %b256, <64 x i8> %b512) {
-; SSE-LABEL: 'splatconstant_funnel_i8'
-; SSE-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %b8, i8 3)
-; SSE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
-; SSE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
-; SSE-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
-; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; SSSE3-LABEL: 'splatconstant_funnel_i8'
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %b8, i8 3)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; SSE42-LABEL: 'splatconstant_funnel_i8'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %b8, i8 3)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX1-LABEL: 'splatconstant_funnel_i8'
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %b8, i8 3)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 44 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; AVX2-LABEL: 'splatconstant_funnel_i8'
@@ -1310,15 +1349,15 @@ define void @splatconstant_funnel_i8(i8 %a8, <16 x i8> %a128, <32 x i8> %a256, <
 ; BDVER2-LABEL: 'splatconstant_funnel_i8'
 ; BDVER2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %b8, i8 3)
 ; BDVER2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
-; BDVER2-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
-; BDVER2-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
+; BDVER2-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
+; BDVER2-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
 ; BDVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; BTVER2-LABEL: 'splatconstant_funnel_i8'
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I8 = call i8 @llvm.fshr.i8(i8 %a8, i8 %b8, i8 3)
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I8 = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %a128, <16 x i8> %b128, <16 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
-; BTVER2-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
-; BTVER2-NEXT:  Cost Model: Found an estimated cost of 44 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V32I8 = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a256, <32 x i8> %b256, <32 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V64I8 = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a512, <64 x i8> %b512, <64 x i8> <i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3, i8 3>)
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %I8    = call i8 @llvm.fshr.i8(i8 %a8, i8 %b8, i8 3)
diff --git a/test/Analysis/CostModel/X86/icmp.ll b/test/Analysis/CostModel/X86/icmp.ll
new file mode 100644
index 0000000000000000000000000000000000000000..caf20d1b9cb5c7754f75c0fa34da4990cedc7e91
--- /dev/null
+++ b/test/Analysis/CostModel/X86/icmp.ll
@@ -0,0 +1,3057 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
+; RUN: opt < %s -mtriple=x86_64-apple-macosx10.8.0 -cost-model -analyze -mattr=+sse2 | FileCheck %s --check-prefixes=CHECK,SSE,SSE2
+; RUN: opt < %s -mtriple=x86_64-apple-macosx10.8.0 -cost-model -analyze -mattr=+sse3 | FileCheck %s --check-prefixes=CHECK,SSE,SSE3
+; RUN: opt < %s -mtriple=x86_64-apple-macosx10.8.0 -cost-model -analyze -mattr=+ssse3 | FileCheck %s --check-prefixes=CHECK,SSE,SSSE3
+; RUN: opt < %s -mtriple=x86_64-apple-macosx10.8.0 -cost-model -analyze -mattr=+sse4.1 | FileCheck %s --check-prefixes=CHECK,SSE,SSE41
+; RUN: opt < %s -mtriple=x86_64-apple-macosx10.8.0 -cost-model -analyze -mattr=+sse4.2 | FileCheck %s --check-prefixes=CHECK,SSE,SSE42
+; RUN: opt < %s -mtriple=x86_64-apple-macosx10.8.0 -cost-model -analyze -mattr=+avx | FileCheck %s --check-prefixes=CHECK,AVX,AVX1
+; RUN: opt < %s -mtriple=x86_64-apple-macosx10.8.0 -cost-model -analyze -mattr=+avx2 | FileCheck %s --check-prefixes=CHECK,AVX,AVX2
+; RUN: opt < %s -mtriple=x86_64-apple-macosx10.8.0 -cost-model -analyze -mattr=+avx512f | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512F
+; RUN: opt < %s -mtriple=x86_64-apple-macosx10.8.0 -cost-model -analyze -mattr=+avx512f,+avx512bw | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512BW
+; RUN: opt < %s -mtriple=x86_64-apple-macosx10.8.0 -cost-model -analyze -mattr=+xop,+avx | FileCheck %s -check-prefixes=CHECK,AVX,XOPAVX1
+; RUN: opt < %s -mtriple=x86_64-apple-macosx10.8.0 -cost-model -analyze -mattr=+xop,+avx2 | FileCheck %s -check-prefixes=CHECK,AVX,XOPAVX2
+;
+; RUN: opt < %s -mtriple=x86_64-apple-macosx10.8.0 -cost-model -analyze -mcpu=slm | FileCheck %s --check-prefixes=CHECK,SSE,SSE42
+; RUN: opt < %s -mtriple=x86_64-apple-macosx10.8.0 -cost-model -analyze -mcpu=goldmont | FileCheck %s --check-prefixes=CHECK,SSE,SSE42
+; RUN: opt < %s -mtriple=x86_64-apple-macosx10.8.0 -cost-model -analyze -mcpu=btver2 | FileCheck %s --check-prefixes=BTVER2
+
+define i32 @cmp_int_eq(i32 %arg) {
+; SSE2-LABEL: 'cmp_int_eq'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp eq i8 undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = icmp eq <16 x i8> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I8 = icmp eq <32 x i8> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = icmp eq <64 x i8> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V128I8 = icmp eq <128 x i8> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp eq i16 undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = icmp eq <8 x i16> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = icmp eq <16 x i16> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = icmp eq <32 x i16> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I16 = icmp eq <64 x i16> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp eq i32 undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = icmp eq <4 x i32> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = icmp eq <8 x i32> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I32 = icmp eq <16 x i32> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I32 = icmp eq <32 x i32> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp eq i64 undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2I64 = icmp eq <2 x i64> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V4I64 = icmp eq <4 x i64> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V8I64 = icmp eq <8 x i64> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %V16I64 = icmp eq <16 x i64> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSE3-LABEL: 'cmp_int_eq'
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp eq i8 undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = icmp eq <16 x i8> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I8 = icmp eq <32 x i8> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = icmp eq <64 x i8> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V128I8 = icmp eq <128 x i8> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp eq i16 undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = icmp eq <8 x i16> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = icmp eq <16 x i16> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = icmp eq <32 x i16> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I16 = icmp eq <64 x i16> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp eq i32 undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = icmp eq <4 x i32> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = icmp eq <8 x i32> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I32 = icmp eq <16 x i32> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I32 = icmp eq <32 x i32> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp eq i64 undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2I64 = icmp eq <2 x i64> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V4I64 = icmp eq <4 x i64> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V8I64 = icmp eq <8 x i64> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %V16I64 = icmp eq <16 x i64> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSSE3-LABEL: 'cmp_int_eq'
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp eq i8 undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = icmp eq <16 x i8> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I8 = icmp eq <32 x i8> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = icmp eq <64 x i8> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V128I8 = icmp eq <128 x i8> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp eq i16 undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = icmp eq <8 x i16> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = icmp eq <16 x i16> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = icmp eq <32 x i16> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I16 = icmp eq <64 x i16> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp eq i32 undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = icmp eq <4 x i32> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = icmp eq <8 x i32> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I32 = icmp eq <16 x i32> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I32 = icmp eq <32 x i32> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp eq i64 undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2I64 = icmp eq <2 x i64> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V4I64 = icmp eq <4 x i64> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V8I64 = icmp eq <8 x i64> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %V16I64 = icmp eq <16 x i64> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSE41-LABEL: 'cmp_int_eq'
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp eq i8 undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = icmp eq <16 x i8> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I8 = icmp eq <32 x i8> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = icmp eq <64 x i8> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V128I8 = icmp eq <128 x i8> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp eq i16 undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = icmp eq <8 x i16> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = icmp eq <16 x i16> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = icmp eq <32 x i16> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I16 = icmp eq <64 x i16> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp eq i32 undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = icmp eq <4 x i32> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = icmp eq <8 x i32> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I32 = icmp eq <16 x i32> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I32 = icmp eq <32 x i32> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp eq i64 undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2I64 = icmp eq <2 x i64> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V4I64 = icmp eq <4 x i64> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V8I64 = icmp eq <8 x i64> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %V16I64 = icmp eq <16 x i64> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSE42-LABEL: 'cmp_int_eq'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp eq i8 undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = icmp eq <16 x i8> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I8 = icmp eq <32 x i8> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = icmp eq <64 x i8> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V128I8 = icmp eq <128 x i8> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp eq i16 undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = icmp eq <8 x i16> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = icmp eq <16 x i16> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = icmp eq <32 x i16> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I16 = icmp eq <64 x i16> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp eq i32 undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = icmp eq <4 x i32> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = icmp eq <8 x i32> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I32 = icmp eq <16 x i32> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I32 = icmp eq <32 x i32> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp eq i64 undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = icmp eq <2 x i64> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I64 = icmp eq <4 x i64> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I64 = icmp eq <8 x i64> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I64 = icmp eq <16 x i64> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX1-LABEL: 'cmp_int_eq'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp eq i8 undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = icmp eq <16 x i8> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I8 = icmp eq <32 x i8> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I8 = icmp eq <64 x i8> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V128I8 = icmp eq <128 x i8> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp eq i16 undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = icmp eq <8 x i16> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I16 = icmp eq <16 x i16> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I16 = icmp eq <32 x i16> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V64I16 = icmp eq <64 x i16> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp eq i32 undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = icmp eq <4 x i32> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I32 = icmp eq <8 x i32> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I32 = icmp eq <16 x i32> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V32I32 = icmp eq <32 x i32> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp eq i64 undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = icmp eq <2 x i64> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4I64 = icmp eq <4 x i64> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8I64 = icmp eq <8 x i64> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16I64 = icmp eq <16 x i64> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX2-LABEL: 'cmp_int_eq'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp eq i8 undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = icmp eq <16 x i8> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32I8 = icmp eq <32 x i8> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64I8 = icmp eq <64 x i8> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V128I8 = icmp eq <128 x i8> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp eq i16 undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = icmp eq <8 x i16> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = icmp eq <16 x i16> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I16 = icmp eq <32 x i16> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I16 = icmp eq <64 x i16> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp eq i32 undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = icmp eq <4 x i32> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = icmp eq <8 x i32> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I32 = icmp eq <16 x i32> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I32 = icmp eq <32 x i32> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp eq i64 undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = icmp eq <2 x i64> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = icmp eq <4 x i64> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I64 = icmp eq <8 x i64> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I64 = icmp eq <16 x i64> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX512F-LABEL: 'cmp_int_eq'
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp eq i8 undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = icmp eq <16 x i8> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32I8 = icmp eq <32 x i8> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64I8 = icmp eq <64 x i8> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V128I8 = icmp eq <128 x i8> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp eq i16 undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = icmp eq <8 x i16> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = icmp eq <16 x i16> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I16 = icmp eq <32 x i16> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I16 = icmp eq <64 x i16> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp eq i32 undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = icmp eq <4 x i32> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = icmp eq <8 x i32> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I32 = icmp eq <16 x i32> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I32 = icmp eq <32 x i32> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp eq i64 undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = icmp eq <2 x i64> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = icmp eq <4 x i64> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = icmp eq <8 x i64> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I64 = icmp eq <16 x i64> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX512BW-LABEL: 'cmp_int_eq'
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp eq i8 undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = icmp eq <16 x i8> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32I8 = icmp eq <32 x i8> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64I8 = icmp eq <64 x i8> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128I8 = icmp eq <128 x i8> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp eq i16 undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = icmp eq <8 x i16> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = icmp eq <16 x i16> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = icmp eq <32 x i16> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64I16 = icmp eq <64 x i16> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp eq i32 undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = icmp eq <4 x i32> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = icmp eq <8 x i32> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I32 = icmp eq <16 x i32> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I32 = icmp eq <32 x i32> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp eq i64 undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = icmp eq <2 x i64> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = icmp eq <4 x i64> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = icmp eq <8 x i64> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I64 = icmp eq <16 x i64> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; XOPAVX1-LABEL: 'cmp_int_eq'
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp eq i8 undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = icmp eq <16 x i8> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I8 = icmp eq <32 x i8> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I8 = icmp eq <64 x i8> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V128I8 = icmp eq <128 x i8> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp eq i16 undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = icmp eq <8 x i16> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I16 = icmp eq <16 x i16> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I16 = icmp eq <32 x i16> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V64I16 = icmp eq <64 x i16> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp eq i32 undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = icmp eq <4 x i32> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I32 = icmp eq <8 x i32> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I32 = icmp eq <16 x i32> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V32I32 = icmp eq <32 x i32> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp eq i64 undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = icmp eq <2 x i64> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4I64 = icmp eq <4 x i64> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8I64 = icmp eq <8 x i64> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16I64 = icmp eq <16 x i64> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; XOPAVX2-LABEL: 'cmp_int_eq'
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp eq i8 undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = icmp eq <16 x i8> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32I8 = icmp eq <32 x i8> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64I8 = icmp eq <64 x i8> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V128I8 = icmp eq <128 x i8> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp eq i16 undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = icmp eq <8 x i16> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = icmp eq <16 x i16> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I16 = icmp eq <32 x i16> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I16 = icmp eq <64 x i16> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp eq i32 undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = icmp eq <4 x i32> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = icmp eq <8 x i32> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I32 = icmp eq <16 x i32> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I32 = icmp eq <32 x i32> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp eq i64 undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = icmp eq <2 x i64> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = icmp eq <4 x i64> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I64 = icmp eq <8 x i64> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I64 = icmp eq <16 x i64> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; BTVER2-LABEL: 'cmp_int_eq'
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp eq i8 undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = icmp eq <16 x i8> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I8 = icmp eq <32 x i8> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I8 = icmp eq <64 x i8> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V128I8 = icmp eq <128 x i8> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp eq i16 undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = icmp eq <8 x i16> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I16 = icmp eq <16 x i16> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I16 = icmp eq <32 x i16> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V64I16 = icmp eq <64 x i16> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp eq i32 undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = icmp eq <4 x i32> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I32 = icmp eq <8 x i32> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I32 = icmp eq <16 x i32> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V32I32 = icmp eq <32 x i32> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp eq i64 undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = icmp eq <2 x i64> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4I64 = icmp eq <4 x i64> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8I64 = icmp eq <8 x i64> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16I64 = icmp eq <16 x i64> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+  %I8 = icmp eq i8 undef, undef
+  %V16I8 = icmp eq <16 x i8> undef, undef
+  %V32I8 = icmp eq <32 x i8> undef, undef
+  %V64I8 = icmp eq <64 x i8> undef, undef
+  %V128I8 = icmp eq <128 x i8> undef, undef
+
+  %I16 = icmp eq i16 undef, undef
+  %V8I16  = icmp eq <8 x i16> undef, undef
+  %V16I16 = icmp eq <16 x i16> undef, undef
+  %V32I16 = icmp eq <32 x i16> undef, undef
+  %V64I16 = icmp eq <64 x i16> undef, undef
+
+  %I32 = icmp eq i32 undef, undef
+  %V4I32 = icmp eq <4 x i32> undef, undef
+  %V8I32 = icmp eq <8 x i32> undef, undef
+  %V16I32 = icmp eq <16 x i32> undef, undef
+  %V32I32 = icmp eq <32 x i32> undef, undef
+
+  %I64 = icmp eq i64 undef, undef
+  %V2I64 = icmp eq <2 x i64> undef, undef
+  %V4I64 = icmp eq <4 x i64> undef, undef
+  %V8I64 = icmp eq <8 x i64> undef, undef
+  %V16I64 = icmp eq <16 x i64> undef, undef
+
+  ret i32 undef
+}
+
+define i32 @cmp_int_ne(i32 %arg) {
+; SSE2-LABEL: 'cmp_int_ne'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp ne i8 undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = icmp ne <16 x i8> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I8 = icmp ne <32 x i8> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I8 = icmp ne <64 x i8> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V128I8 = icmp ne <128 x i8> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp ne i16 undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = icmp ne <8 x i16> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I16 = icmp ne <16 x i16> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I16 = icmp ne <32 x i16> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V64I16 = icmp ne <64 x i16> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp ne i32 undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = icmp ne <4 x i32> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I32 = icmp ne <8 x i32> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I32 = icmp ne <16 x i32> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V32I32 = icmp ne <32 x i32> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp ne i64 undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V2I64 = icmp ne <2 x i64> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V4I64 = icmp ne <4 x i64> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %V8I64 = icmp ne <8 x i64> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %V16I64 = icmp ne <16 x i64> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSE3-LABEL: 'cmp_int_ne'
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp ne i8 undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = icmp ne <16 x i8> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I8 = icmp ne <32 x i8> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I8 = icmp ne <64 x i8> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V128I8 = icmp ne <128 x i8> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp ne i16 undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = icmp ne <8 x i16> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I16 = icmp ne <16 x i16> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I16 = icmp ne <32 x i16> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V64I16 = icmp ne <64 x i16> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp ne i32 undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = icmp ne <4 x i32> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I32 = icmp ne <8 x i32> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I32 = icmp ne <16 x i32> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V32I32 = icmp ne <32 x i32> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp ne i64 undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V2I64 = icmp ne <2 x i64> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V4I64 = icmp ne <4 x i64> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %V8I64 = icmp ne <8 x i64> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %V16I64 = icmp ne <16 x i64> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSSE3-LABEL: 'cmp_int_ne'
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp ne i8 undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = icmp ne <16 x i8> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I8 = icmp ne <32 x i8> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I8 = icmp ne <64 x i8> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V128I8 = icmp ne <128 x i8> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp ne i16 undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = icmp ne <8 x i16> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I16 = icmp ne <16 x i16> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I16 = icmp ne <32 x i16> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V64I16 = icmp ne <64 x i16> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp ne i32 undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = icmp ne <4 x i32> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I32 = icmp ne <8 x i32> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I32 = icmp ne <16 x i32> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V32I32 = icmp ne <32 x i32> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp ne i64 undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V2I64 = icmp ne <2 x i64> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V4I64 = icmp ne <4 x i64> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %V8I64 = icmp ne <8 x i64> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %V16I64 = icmp ne <16 x i64> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSE41-LABEL: 'cmp_int_ne'
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp ne i8 undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = icmp ne <16 x i8> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I8 = icmp ne <32 x i8> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I8 = icmp ne <64 x i8> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V128I8 = icmp ne <128 x i8> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp ne i16 undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = icmp ne <8 x i16> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I16 = icmp ne <16 x i16> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I16 = icmp ne <32 x i16> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V64I16 = icmp ne <64 x i16> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp ne i32 undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = icmp ne <4 x i32> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I32 = icmp ne <8 x i32> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I32 = icmp ne <16 x i32> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V32I32 = icmp ne <32 x i32> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp ne i64 undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V2I64 = icmp ne <2 x i64> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V4I64 = icmp ne <4 x i64> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %V8I64 = icmp ne <8 x i64> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %V16I64 = icmp ne <16 x i64> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSE42-LABEL: 'cmp_int_ne'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp ne i8 undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = icmp ne <16 x i8> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I8 = icmp ne <32 x i8> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I8 = icmp ne <64 x i8> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V128I8 = icmp ne <128 x i8> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp ne i16 undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = icmp ne <8 x i16> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I16 = icmp ne <16 x i16> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I16 = icmp ne <32 x i16> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V64I16 = icmp ne <64 x i16> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp ne i32 undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = icmp ne <4 x i32> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I32 = icmp ne <8 x i32> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I32 = icmp ne <16 x i32> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V32I32 = icmp ne <32 x i32> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp ne i64 undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I64 = icmp ne <2 x i64> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4I64 = icmp ne <4 x i64> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8I64 = icmp ne <8 x i64> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16I64 = icmp ne <16 x i64> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX1-LABEL: 'cmp_int_ne'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp ne i8 undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = icmp ne <16 x i8> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V32I8 = icmp ne <32 x i8> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V64I8 = icmp ne <64 x i8> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V128I8 = icmp ne <128 x i8> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp ne i16 undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = icmp ne <8 x i16> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V16I16 = icmp ne <16 x i16> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V32I16 = icmp ne <32 x i16> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V64I16 = icmp ne <64 x i16> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp ne i32 undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = icmp ne <4 x i32> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V8I32 = icmp ne <8 x i32> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V16I32 = icmp ne <16 x i32> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V32I32 = icmp ne <32 x i32> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp ne i64 undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I64 = icmp ne <2 x i64> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V4I64 = icmp ne <4 x i64> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V8I64 = icmp ne <8 x i64> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V16I64 = icmp ne <16 x i64> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX2-LABEL: 'cmp_int_ne'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp ne i8 undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = icmp ne <16 x i8> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I8 = icmp ne <32 x i8> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = icmp ne <64 x i8> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V128I8 = icmp ne <128 x i8> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp ne i16 undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = icmp ne <8 x i16> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = icmp ne <16 x i16> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = icmp ne <32 x i16> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I16 = icmp ne <64 x i16> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp ne i32 undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = icmp ne <4 x i32> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = icmp ne <8 x i32> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I32 = icmp ne <16 x i32> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I32 = icmp ne <32 x i32> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp ne i64 undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I64 = icmp ne <2 x i64> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I64 = icmp ne <4 x i64> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I64 = icmp ne <8 x i64> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I64 = icmp ne <16 x i64> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX512F-LABEL: 'cmp_int_ne'
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp ne i8 undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = icmp ne <16 x i8> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I8 = icmp ne <32 x i8> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = icmp ne <64 x i8> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V128I8 = icmp ne <128 x i8> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp ne i16 undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = icmp ne <8 x i16> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = icmp ne <16 x i16> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = icmp ne <32 x i16> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I16 = icmp ne <64 x i16> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp ne i32 undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = icmp ne <4 x i32> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = icmp ne <8 x i32> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I32 = icmp ne <16 x i32> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I32 = icmp ne <32 x i32> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp ne i64 undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = icmp ne <2 x i64> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = icmp ne <4 x i64> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = icmp ne <8 x i64> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I64 = icmp ne <16 x i64> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX512BW-LABEL: 'cmp_int_ne'
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp ne i8 undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = icmp ne <16 x i8> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32I8 = icmp ne <32 x i8> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64I8 = icmp ne <64 x i8> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128I8 = icmp ne <128 x i8> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp ne i16 undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = icmp ne <8 x i16> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = icmp ne <16 x i16> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = icmp ne <32 x i16> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64I16 = icmp ne <64 x i16> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp ne i32 undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = icmp ne <4 x i32> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = icmp ne <8 x i32> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I32 = icmp ne <16 x i32> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I32 = icmp ne <32 x i32> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp ne i64 undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = icmp ne <2 x i64> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = icmp ne <4 x i64> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = icmp ne <8 x i64> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I64 = icmp ne <16 x i64> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; XOPAVX1-LABEL: 'cmp_int_ne'
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp ne i8 undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = icmp ne <16 x i8> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I8 = icmp ne <32 x i8> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I8 = icmp ne <64 x i8> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V128I8 = icmp ne <128 x i8> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp ne i16 undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = icmp ne <8 x i16> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I16 = icmp ne <16 x i16> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I16 = icmp ne <32 x i16> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V64I16 = icmp ne <64 x i16> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp ne i32 undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = icmp ne <4 x i32> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I32 = icmp ne <8 x i32> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I32 = icmp ne <16 x i32> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V32I32 = icmp ne <32 x i32> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp ne i64 undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = icmp ne <2 x i64> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4I64 = icmp ne <4 x i64> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8I64 = icmp ne <8 x i64> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16I64 = icmp ne <16 x i64> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; XOPAVX2-LABEL: 'cmp_int_ne'
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp ne i8 undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = icmp ne <16 x i8> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I8 = icmp ne <32 x i8> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = icmp ne <64 x i8> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V128I8 = icmp ne <128 x i8> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp ne i16 undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = icmp ne <8 x i16> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = icmp ne <16 x i16> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = icmp ne <32 x i16> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I16 = icmp ne <64 x i16> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp ne i32 undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = icmp ne <4 x i32> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = icmp ne <8 x i32> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I32 = icmp ne <16 x i32> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I32 = icmp ne <32 x i32> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp ne i64 undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = icmp ne <2 x i64> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I64 = icmp ne <4 x i64> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I64 = icmp ne <8 x i64> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I64 = icmp ne <16 x i64> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; BTVER2-LABEL: 'cmp_int_ne'
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp ne i8 undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = icmp ne <16 x i8> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V32I8 = icmp ne <32 x i8> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V64I8 = icmp ne <64 x i8> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V128I8 = icmp ne <128 x i8> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp ne i16 undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = icmp ne <8 x i16> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V16I16 = icmp ne <16 x i16> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V32I16 = icmp ne <32 x i16> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V64I16 = icmp ne <64 x i16> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp ne i32 undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = icmp ne <4 x i32> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V8I32 = icmp ne <8 x i32> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V16I32 = icmp ne <16 x i32> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V32I32 = icmp ne <32 x i32> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp ne i64 undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I64 = icmp ne <2 x i64> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V4I64 = icmp ne <4 x i64> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V8I64 = icmp ne <8 x i64> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V16I64 = icmp ne <16 x i64> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+  %I8 = icmp ne i8 undef, undef
+  %V16I8 = icmp ne <16 x i8> undef, undef
+  %V32I8 = icmp ne <32 x i8> undef, undef
+  %V64I8 = icmp ne <64 x i8> undef, undef
+  %V128I8 = icmp ne <128 x i8> undef, undef
+
+  %I16 = icmp ne i16 undef, undef
+  %V8I16  = icmp ne <8 x i16> undef, undef
+  %V16I16 = icmp ne <16 x i16> undef, undef
+  %V32I16 = icmp ne <32 x i16> undef, undef
+  %V64I16 = icmp ne <64 x i16> undef, undef
+
+  %I32 = icmp ne i32 undef, undef
+  %V4I32 = icmp ne <4 x i32> undef, undef
+  %V8I32 = icmp ne <8 x i32> undef, undef
+  %V16I32 = icmp ne <16 x i32> undef, undef
+  %V32I32 = icmp ne <32 x i32> undef, undef
+
+  %I64 = icmp ne i64 undef, undef
+  %V2I64 = icmp ne <2 x i64> undef, undef
+  %V4I64 = icmp ne <4 x i64> undef, undef
+  %V8I64 = icmp ne <8 x i64> undef, undef
+  %V16I64 = icmp ne <16 x i64> undef, undef
+
+  ret i32 undef
+}
+
+define i32 @cmp_int_sge(i32 %arg) {
+; SSE2-LABEL: 'cmp_int_sge'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp sge i8 undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = icmp sge <16 x i8> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I8 = icmp sge <32 x i8> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I8 = icmp sge <64 x i8> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V128I8 = icmp sge <128 x i8> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp sge i16 undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = icmp sge <8 x i16> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I16 = icmp sge <16 x i16> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I16 = icmp sge <32 x i16> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V64I16 = icmp sge <64 x i16> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp sge i32 undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = icmp sge <4 x i32> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I32 = icmp sge <8 x i32> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I32 = icmp sge <16 x i32> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V32I32 = icmp sge <32 x i32> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp sge i64 undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V2I64 = icmp sge <2 x i64> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V4I64 = icmp sge <4 x i64> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %V8I64 = icmp sge <8 x i64> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %V16I64 = icmp sge <16 x i64> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSE3-LABEL: 'cmp_int_sge'
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp sge i8 undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = icmp sge <16 x i8> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I8 = icmp sge <32 x i8> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I8 = icmp sge <64 x i8> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V128I8 = icmp sge <128 x i8> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp sge i16 undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = icmp sge <8 x i16> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I16 = icmp sge <16 x i16> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I16 = icmp sge <32 x i16> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V64I16 = icmp sge <64 x i16> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp sge i32 undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = icmp sge <4 x i32> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I32 = icmp sge <8 x i32> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I32 = icmp sge <16 x i32> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V32I32 = icmp sge <32 x i32> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp sge i64 undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V2I64 = icmp sge <2 x i64> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V4I64 = icmp sge <4 x i64> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %V8I64 = icmp sge <8 x i64> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %V16I64 = icmp sge <16 x i64> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSSE3-LABEL: 'cmp_int_sge'
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp sge i8 undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = icmp sge <16 x i8> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I8 = icmp sge <32 x i8> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I8 = icmp sge <64 x i8> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V128I8 = icmp sge <128 x i8> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp sge i16 undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = icmp sge <8 x i16> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I16 = icmp sge <16 x i16> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I16 = icmp sge <32 x i16> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V64I16 = icmp sge <64 x i16> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp sge i32 undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = icmp sge <4 x i32> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I32 = icmp sge <8 x i32> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I32 = icmp sge <16 x i32> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V32I32 = icmp sge <32 x i32> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp sge i64 undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V2I64 = icmp sge <2 x i64> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V4I64 = icmp sge <4 x i64> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %V8I64 = icmp sge <8 x i64> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %V16I64 = icmp sge <16 x i64> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSE41-LABEL: 'cmp_int_sge'
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp sge i8 undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = icmp sge <16 x i8> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I8 = icmp sge <32 x i8> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I8 = icmp sge <64 x i8> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V128I8 = icmp sge <128 x i8> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp sge i16 undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = icmp sge <8 x i16> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I16 = icmp sge <16 x i16> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I16 = icmp sge <32 x i16> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V64I16 = icmp sge <64 x i16> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp sge i32 undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = icmp sge <4 x i32> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I32 = icmp sge <8 x i32> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I32 = icmp sge <16 x i32> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V32I32 = icmp sge <32 x i32> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp sge i64 undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V2I64 = icmp sge <2 x i64> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V4I64 = icmp sge <4 x i64> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %V8I64 = icmp sge <8 x i64> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %V16I64 = icmp sge <16 x i64> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSE42-LABEL: 'cmp_int_sge'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp sge i8 undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = icmp sge <16 x i8> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I8 = icmp sge <32 x i8> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I8 = icmp sge <64 x i8> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V128I8 = icmp sge <128 x i8> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp sge i16 undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = icmp sge <8 x i16> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I16 = icmp sge <16 x i16> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I16 = icmp sge <32 x i16> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V64I16 = icmp sge <64 x i16> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp sge i32 undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = icmp sge <4 x i32> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I32 = icmp sge <8 x i32> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I32 = icmp sge <16 x i32> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V32I32 = icmp sge <32 x i32> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp sge i64 undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I64 = icmp sge <2 x i64> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4I64 = icmp sge <4 x i64> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8I64 = icmp sge <8 x i64> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16I64 = icmp sge <16 x i64> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX1-LABEL: 'cmp_int_sge'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp sge i8 undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = icmp sge <16 x i8> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V32I8 = icmp sge <32 x i8> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V64I8 = icmp sge <64 x i8> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V128I8 = icmp sge <128 x i8> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp sge i16 undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = icmp sge <8 x i16> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V16I16 = icmp sge <16 x i16> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V32I16 = icmp sge <32 x i16> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V64I16 = icmp sge <64 x i16> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp sge i32 undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = icmp sge <4 x i32> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V8I32 = icmp sge <8 x i32> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V16I32 = icmp sge <16 x i32> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V32I32 = icmp sge <32 x i32> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp sge i64 undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I64 = icmp sge <2 x i64> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V4I64 = icmp sge <4 x i64> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V8I64 = icmp sge <8 x i64> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V16I64 = icmp sge <16 x i64> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX2-LABEL: 'cmp_int_sge'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp sge i8 undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = icmp sge <16 x i8> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I8 = icmp sge <32 x i8> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = icmp sge <64 x i8> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V128I8 = icmp sge <128 x i8> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp sge i16 undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = icmp sge <8 x i16> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = icmp sge <16 x i16> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = icmp sge <32 x i16> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I16 = icmp sge <64 x i16> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp sge i32 undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = icmp sge <4 x i32> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = icmp sge <8 x i32> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I32 = icmp sge <16 x i32> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I32 = icmp sge <32 x i32> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp sge i64 undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I64 = icmp sge <2 x i64> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I64 = icmp sge <4 x i64> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I64 = icmp sge <8 x i64> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I64 = icmp sge <16 x i64> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX512F-LABEL: 'cmp_int_sge'
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp sge i8 undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = icmp sge <16 x i8> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I8 = icmp sge <32 x i8> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = icmp sge <64 x i8> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V128I8 = icmp sge <128 x i8> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp sge i16 undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = icmp sge <8 x i16> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = icmp sge <16 x i16> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = icmp sge <32 x i16> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I16 = icmp sge <64 x i16> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp sge i32 undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = icmp sge <4 x i32> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = icmp sge <8 x i32> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I32 = icmp sge <16 x i32> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I32 = icmp sge <32 x i32> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp sge i64 undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = icmp sge <2 x i64> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = icmp sge <4 x i64> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = icmp sge <8 x i64> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I64 = icmp sge <16 x i64> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX512BW-LABEL: 'cmp_int_sge'
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp sge i8 undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = icmp sge <16 x i8> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32I8 = icmp sge <32 x i8> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64I8 = icmp sge <64 x i8> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128I8 = icmp sge <128 x i8> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp sge i16 undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = icmp sge <8 x i16> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = icmp sge <16 x i16> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = icmp sge <32 x i16> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64I16 = icmp sge <64 x i16> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp sge i32 undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = icmp sge <4 x i32> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = icmp sge <8 x i32> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I32 = icmp sge <16 x i32> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I32 = icmp sge <32 x i32> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp sge i64 undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = icmp sge <2 x i64> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = icmp sge <4 x i64> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = icmp sge <8 x i64> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I64 = icmp sge <16 x i64> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; XOPAVX1-LABEL: 'cmp_int_sge'
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp sge i8 undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = icmp sge <16 x i8> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I8 = icmp sge <32 x i8> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I8 = icmp sge <64 x i8> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V128I8 = icmp sge <128 x i8> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp sge i16 undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = icmp sge <8 x i16> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I16 = icmp sge <16 x i16> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I16 = icmp sge <32 x i16> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V64I16 = icmp sge <64 x i16> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp sge i32 undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = icmp sge <4 x i32> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I32 = icmp sge <8 x i32> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I32 = icmp sge <16 x i32> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V32I32 = icmp sge <32 x i32> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp sge i64 undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = icmp sge <2 x i64> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4I64 = icmp sge <4 x i64> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8I64 = icmp sge <8 x i64> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16I64 = icmp sge <16 x i64> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; XOPAVX2-LABEL: 'cmp_int_sge'
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp sge i8 undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = icmp sge <16 x i8> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I8 = icmp sge <32 x i8> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = icmp sge <64 x i8> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V128I8 = icmp sge <128 x i8> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp sge i16 undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = icmp sge <8 x i16> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = icmp sge <16 x i16> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = icmp sge <32 x i16> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I16 = icmp sge <64 x i16> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp sge i32 undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = icmp sge <4 x i32> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = icmp sge <8 x i32> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I32 = icmp sge <16 x i32> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I32 = icmp sge <32 x i32> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp sge i64 undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = icmp sge <2 x i64> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I64 = icmp sge <4 x i64> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I64 = icmp sge <8 x i64> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I64 = icmp sge <16 x i64> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; BTVER2-LABEL: 'cmp_int_sge'
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp sge i8 undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = icmp sge <16 x i8> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V32I8 = icmp sge <32 x i8> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V64I8 = icmp sge <64 x i8> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V128I8 = icmp sge <128 x i8> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp sge i16 undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = icmp sge <8 x i16> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V16I16 = icmp sge <16 x i16> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V32I16 = icmp sge <32 x i16> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V64I16 = icmp sge <64 x i16> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp sge i32 undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = icmp sge <4 x i32> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V8I32 = icmp sge <8 x i32> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V16I32 = icmp sge <16 x i32> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V32I32 = icmp sge <32 x i32> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp sge i64 undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I64 = icmp sge <2 x i64> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V4I64 = icmp sge <4 x i64> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V8I64 = icmp sge <8 x i64> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V16I64 = icmp sge <16 x i64> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+  %I8 = icmp sge i8 undef, undef
+  %V16I8 = icmp sge <16 x i8> undef, undef
+  %V32I8 = icmp sge <32 x i8> undef, undef
+  %V64I8 = icmp sge <64 x i8> undef, undef
+  %V128I8 = icmp sge <128 x i8> undef, undef
+
+  %I16 = icmp sge i16 undef, undef
+  %V8I16  = icmp sge <8 x i16> undef, undef
+  %V16I16 = icmp sge <16 x i16> undef, undef
+  %V32I16 = icmp sge <32 x i16> undef, undef
+  %V64I16 = icmp sge <64 x i16> undef, undef
+
+  %I32 = icmp sge i32 undef, undef
+  %V4I32 = icmp sge <4 x i32> undef, undef
+  %V8I32 = icmp sge <8 x i32> undef, undef
+  %V16I32 = icmp sge <16 x i32> undef, undef
+  %V32I32 = icmp sge <32 x i32> undef, undef
+
+  %I64 = icmp sge i64 undef, undef
+  %V2I64 = icmp sge <2 x i64> undef, undef
+  %V4I64 = icmp sge <4 x i64> undef, undef
+  %V8I64 = icmp sge <8 x i64> undef, undef
+  %V16I64 = icmp sge <16 x i64> undef, undef
+
+  ret i32 undef
+}
+
+define i32 @cmp_int_uge(i32 %arg) {
+; SSE2-LABEL: 'cmp_int_uge'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp uge i8 undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = icmp uge <16 x i8> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I8 = icmp uge <32 x i8> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I8 = icmp uge <64 x i8> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V128I8 = icmp uge <128 x i8> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp uge i16 undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = icmp uge <8 x i16> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I16 = icmp uge <16 x i16> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I16 = icmp uge <32 x i16> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V64I16 = icmp uge <64 x i16> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp uge i32 undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4I32 = icmp uge <4 x i32> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8I32 = icmp uge <8 x i32> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16I32 = icmp uge <16 x i32> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V32I32 = icmp uge <32 x i32> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp uge i64 undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V2I64 = icmp uge <2 x i64> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V4I64 = icmp uge <4 x i64> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 44 for instruction: %V8I64 = icmp uge <8 x i64> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 88 for instruction: %V16I64 = icmp uge <16 x i64> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSE3-LABEL: 'cmp_int_uge'
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp uge i8 undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = icmp uge <16 x i8> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I8 = icmp uge <32 x i8> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I8 = icmp uge <64 x i8> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V128I8 = icmp uge <128 x i8> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp uge i16 undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = icmp uge <8 x i16> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I16 = icmp uge <16 x i16> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I16 = icmp uge <32 x i16> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V64I16 = icmp uge <64 x i16> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp uge i32 undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4I32 = icmp uge <4 x i32> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8I32 = icmp uge <8 x i32> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16I32 = icmp uge <16 x i32> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V32I32 = icmp uge <32 x i32> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp uge i64 undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V2I64 = icmp uge <2 x i64> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V4I64 = icmp uge <4 x i64> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 44 for instruction: %V8I64 = icmp uge <8 x i64> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 88 for instruction: %V16I64 = icmp uge <16 x i64> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSSE3-LABEL: 'cmp_int_uge'
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp uge i8 undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = icmp uge <16 x i8> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I8 = icmp uge <32 x i8> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I8 = icmp uge <64 x i8> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V128I8 = icmp uge <128 x i8> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp uge i16 undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = icmp uge <8 x i16> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I16 = icmp uge <16 x i16> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I16 = icmp uge <32 x i16> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V64I16 = icmp uge <64 x i16> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp uge i32 undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4I32 = icmp uge <4 x i32> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8I32 = icmp uge <8 x i32> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16I32 = icmp uge <16 x i32> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V32I32 = icmp uge <32 x i32> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp uge i64 undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V2I64 = icmp uge <2 x i64> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V4I64 = icmp uge <4 x i64> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 44 for instruction: %V8I64 = icmp uge <8 x i64> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 88 for instruction: %V16I64 = icmp uge <16 x i64> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSE41-LABEL: 'cmp_int_uge'
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp uge i8 undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = icmp uge <16 x i8> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I8 = icmp uge <32 x i8> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I8 = icmp uge <64 x i8> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V128I8 = icmp uge <128 x i8> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp uge i16 undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = icmp uge <8 x i16> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I16 = icmp uge <16 x i16> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I16 = icmp uge <32 x i16> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V64I16 = icmp uge <64 x i16> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp uge i32 undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = icmp uge <4 x i32> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I32 = icmp uge <8 x i32> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I32 = icmp uge <16 x i32> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V32I32 = icmp uge <32 x i32> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp uge i64 undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V2I64 = icmp uge <2 x i64> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V4I64 = icmp uge <4 x i64> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 44 for instruction: %V8I64 = icmp uge <8 x i64> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 88 for instruction: %V16I64 = icmp uge <16 x i64> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSE42-LABEL: 'cmp_int_uge'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp uge i8 undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = icmp uge <16 x i8> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I8 = icmp uge <32 x i8> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I8 = icmp uge <64 x i8> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V128I8 = icmp uge <128 x i8> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp uge i16 undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = icmp uge <8 x i16> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I16 = icmp uge <16 x i16> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I16 = icmp uge <32 x i16> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V64I16 = icmp uge <64 x i16> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp uge i32 undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = icmp uge <4 x i32> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I32 = icmp uge <8 x i32> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I32 = icmp uge <16 x i32> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V32I32 = icmp uge <32 x i32> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp uge i64 undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2I64 = icmp uge <2 x i64> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4I64 = icmp uge <4 x i64> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8I64 = icmp uge <8 x i64> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V16I64 = icmp uge <16 x i64> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX1-LABEL: 'cmp_int_uge'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp uge i8 undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = icmp uge <16 x i8> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V32I8 = icmp uge <32 x i8> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V64I8 = icmp uge <64 x i8> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V128I8 = icmp uge <128 x i8> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp uge i16 undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = icmp uge <8 x i16> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V16I16 = icmp uge <16 x i16> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V32I16 = icmp uge <32 x i16> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V64I16 = icmp uge <64 x i16> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp uge i32 undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = icmp uge <4 x i32> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V8I32 = icmp uge <8 x i32> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V16I32 = icmp uge <16 x i32> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V32I32 = icmp uge <32 x i32> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp uge i64 undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2I64 = icmp uge <2 x i64> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4I64 = icmp uge <4 x i64> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V8I64 = icmp uge <8 x i64> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V16I64 = icmp uge <16 x i64> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX2-LABEL: 'cmp_int_uge'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp uge i8 undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = icmp uge <16 x i8> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I8 = icmp uge <32 x i8> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = icmp uge <64 x i8> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V128I8 = icmp uge <128 x i8> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp uge i16 undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = icmp uge <8 x i16> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = icmp uge <16 x i16> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = icmp uge <32 x i16> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I16 = icmp uge <64 x i16> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp uge i32 undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = icmp uge <4 x i32> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = icmp uge <8 x i32> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I32 = icmp uge <16 x i32> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I32 = icmp uge <32 x i32> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp uge i64 undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2I64 = icmp uge <2 x i64> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4I64 = icmp uge <4 x i64> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8I64 = icmp uge <8 x i64> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16I64 = icmp uge <16 x i64> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX512F-LABEL: 'cmp_int_uge'
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp uge i8 undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = icmp uge <16 x i8> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I8 = icmp uge <32 x i8> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = icmp uge <64 x i8> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V128I8 = icmp uge <128 x i8> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp uge i16 undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = icmp uge <8 x i16> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = icmp uge <16 x i16> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = icmp uge <32 x i16> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I16 = icmp uge <64 x i16> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp uge i32 undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = icmp uge <4 x i32> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = icmp uge <8 x i32> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I32 = icmp uge <16 x i32> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I32 = icmp uge <32 x i32> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp uge i64 undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = icmp uge <2 x i64> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = icmp uge <4 x i64> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = icmp uge <8 x i64> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I64 = icmp uge <16 x i64> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX512BW-LABEL: 'cmp_int_uge'
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp uge i8 undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = icmp uge <16 x i8> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32I8 = icmp uge <32 x i8> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64I8 = icmp uge <64 x i8> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128I8 = icmp uge <128 x i8> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp uge i16 undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = icmp uge <8 x i16> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = icmp uge <16 x i16> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = icmp uge <32 x i16> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64I16 = icmp uge <64 x i16> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp uge i32 undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = icmp uge <4 x i32> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = icmp uge <8 x i32> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I32 = icmp uge <16 x i32> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I32 = icmp uge <32 x i32> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp uge i64 undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = icmp uge <2 x i64> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = icmp uge <4 x i64> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = icmp uge <8 x i64> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I64 = icmp uge <16 x i64> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; XOPAVX1-LABEL: 'cmp_int_uge'
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp uge i8 undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = icmp uge <16 x i8> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I8 = icmp uge <32 x i8> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I8 = icmp uge <64 x i8> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V128I8 = icmp uge <128 x i8> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp uge i16 undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = icmp uge <8 x i16> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I16 = icmp uge <16 x i16> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I16 = icmp uge <32 x i16> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V64I16 = icmp uge <64 x i16> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp uge i32 undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = icmp uge <4 x i32> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I32 = icmp uge <8 x i32> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I32 = icmp uge <16 x i32> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V32I32 = icmp uge <32 x i32> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp uge i64 undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = icmp uge <2 x i64> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4I64 = icmp uge <4 x i64> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8I64 = icmp uge <8 x i64> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16I64 = icmp uge <16 x i64> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; XOPAVX2-LABEL: 'cmp_int_uge'
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp uge i8 undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = icmp uge <16 x i8> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I8 = icmp uge <32 x i8> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = icmp uge <64 x i8> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V128I8 = icmp uge <128 x i8> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp uge i16 undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = icmp uge <8 x i16> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = icmp uge <16 x i16> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = icmp uge <32 x i16> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I16 = icmp uge <64 x i16> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp uge i32 undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = icmp uge <4 x i32> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = icmp uge <8 x i32> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I32 = icmp uge <16 x i32> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I32 = icmp uge <32 x i32> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp uge i64 undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = icmp uge <2 x i64> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4I64 = icmp uge <4 x i64> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8I64 = icmp uge <8 x i64> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16I64 = icmp uge <16 x i64> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; BTVER2-LABEL: 'cmp_int_uge'
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp uge i8 undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = icmp uge <16 x i8> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V32I8 = icmp uge <32 x i8> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V64I8 = icmp uge <64 x i8> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V128I8 = icmp uge <128 x i8> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp uge i16 undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = icmp uge <8 x i16> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V16I16 = icmp uge <16 x i16> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V32I16 = icmp uge <32 x i16> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V64I16 = icmp uge <64 x i16> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp uge i32 undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = icmp uge <4 x i32> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V8I32 = icmp uge <8 x i32> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V16I32 = icmp uge <16 x i32> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V32I32 = icmp uge <32 x i32> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp uge i64 undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2I64 = icmp uge <2 x i64> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4I64 = icmp uge <4 x i64> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V8I64 = icmp uge <8 x i64> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V16I64 = icmp uge <16 x i64> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+  %I8 = icmp uge i8 undef, undef
+  %V16I8 = icmp uge <16 x i8> undef, undef
+  %V32I8 = icmp uge <32 x i8> undef, undef
+  %V64I8 = icmp uge <64 x i8> undef, undef
+  %V128I8 = icmp uge <128 x i8> undef, undef
+
+  %I16 = icmp uge i16 undef, undef
+  %V8I16  = icmp uge <8 x i16> undef, undef
+  %V16I16 = icmp uge <16 x i16> undef, undef
+  %V32I16 = icmp uge <32 x i16> undef, undef
+  %V64I16 = icmp uge <64 x i16> undef, undef
+
+  %I32 = icmp uge i32 undef, undef
+  %V4I32 = icmp uge <4 x i32> undef, undef
+  %V8I32 = icmp uge <8 x i32> undef, undef
+  %V16I32 = icmp uge <16 x i32> undef, undef
+  %V32I32 = icmp uge <32 x i32> undef, undef
+
+  %I64 = icmp uge i64 undef, undef
+  %V2I64 = icmp uge <2 x i64> undef, undef
+  %V4I64 = icmp uge <4 x i64> undef, undef
+  %V8I64 = icmp uge <8 x i64> undef, undef
+  %V16I64 = icmp uge <16 x i64> undef, undef
+
+  ret i32 undef
+}
+
+define i32 @cmp_int_sgt(i32 %arg) {
+; SSE2-LABEL: 'cmp_int_sgt'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp sgt i8 undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = icmp sgt <16 x i8> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I8 = icmp sgt <32 x i8> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = icmp sgt <64 x i8> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V128I8 = icmp sgt <128 x i8> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp sgt i16 undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = icmp sgt <8 x i16> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = icmp sgt <16 x i16> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = icmp sgt <32 x i16> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I16 = icmp sgt <64 x i16> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp sgt i32 undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = icmp sgt <4 x i32> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = icmp sgt <8 x i32> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I32 = icmp sgt <16 x i32> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I32 = icmp sgt <32 x i32> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp sgt i64 undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2I64 = icmp sgt <2 x i64> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V4I64 = icmp sgt <4 x i64> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V8I64 = icmp sgt <8 x i64> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %V16I64 = icmp sgt <16 x i64> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSE3-LABEL: 'cmp_int_sgt'
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp sgt i8 undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = icmp sgt <16 x i8> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I8 = icmp sgt <32 x i8> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = icmp sgt <64 x i8> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V128I8 = icmp sgt <128 x i8> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp sgt i16 undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = icmp sgt <8 x i16> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = icmp sgt <16 x i16> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = icmp sgt <32 x i16> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I16 = icmp sgt <64 x i16> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp sgt i32 undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = icmp sgt <4 x i32> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = icmp sgt <8 x i32> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I32 = icmp sgt <16 x i32> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I32 = icmp sgt <32 x i32> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp sgt i64 undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2I64 = icmp sgt <2 x i64> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V4I64 = icmp sgt <4 x i64> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V8I64 = icmp sgt <8 x i64> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %V16I64 = icmp sgt <16 x i64> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSSE3-LABEL: 'cmp_int_sgt'
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp sgt i8 undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = icmp sgt <16 x i8> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I8 = icmp sgt <32 x i8> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = icmp sgt <64 x i8> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V128I8 = icmp sgt <128 x i8> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp sgt i16 undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = icmp sgt <8 x i16> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = icmp sgt <16 x i16> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = icmp sgt <32 x i16> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I16 = icmp sgt <64 x i16> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp sgt i32 undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = icmp sgt <4 x i32> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = icmp sgt <8 x i32> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I32 = icmp sgt <16 x i32> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I32 = icmp sgt <32 x i32> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp sgt i64 undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2I64 = icmp sgt <2 x i64> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V4I64 = icmp sgt <4 x i64> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V8I64 = icmp sgt <8 x i64> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %V16I64 = icmp sgt <16 x i64> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSE41-LABEL: 'cmp_int_sgt'
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp sgt i8 undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = icmp sgt <16 x i8> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I8 = icmp sgt <32 x i8> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = icmp sgt <64 x i8> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V128I8 = icmp sgt <128 x i8> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp sgt i16 undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = icmp sgt <8 x i16> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = icmp sgt <16 x i16> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = icmp sgt <32 x i16> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I16 = icmp sgt <64 x i16> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp sgt i32 undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = icmp sgt <4 x i32> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = icmp sgt <8 x i32> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I32 = icmp sgt <16 x i32> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I32 = icmp sgt <32 x i32> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp sgt i64 undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2I64 = icmp sgt <2 x i64> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V4I64 = icmp sgt <4 x i64> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V8I64 = icmp sgt <8 x i64> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %V16I64 = icmp sgt <16 x i64> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSE42-LABEL: 'cmp_int_sgt'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp sgt i8 undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = icmp sgt <16 x i8> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I8 = icmp sgt <32 x i8> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = icmp sgt <64 x i8> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V128I8 = icmp sgt <128 x i8> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp sgt i16 undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = icmp sgt <8 x i16> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = icmp sgt <16 x i16> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = icmp sgt <32 x i16> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I16 = icmp sgt <64 x i16> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp sgt i32 undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = icmp sgt <4 x i32> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = icmp sgt <8 x i32> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I32 = icmp sgt <16 x i32> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I32 = icmp sgt <32 x i32> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp sgt i64 undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = icmp sgt <2 x i64> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I64 = icmp sgt <4 x i64> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I64 = icmp sgt <8 x i64> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I64 = icmp sgt <16 x i64> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX1-LABEL: 'cmp_int_sgt'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp sgt i8 undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = icmp sgt <16 x i8> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I8 = icmp sgt <32 x i8> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I8 = icmp sgt <64 x i8> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V128I8 = icmp sgt <128 x i8> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp sgt i16 undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = icmp sgt <8 x i16> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I16 = icmp sgt <16 x i16> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I16 = icmp sgt <32 x i16> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V64I16 = icmp sgt <64 x i16> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp sgt i32 undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = icmp sgt <4 x i32> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I32 = icmp sgt <8 x i32> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I32 = icmp sgt <16 x i32> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V32I32 = icmp sgt <32 x i32> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp sgt i64 undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = icmp sgt <2 x i64> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4I64 = icmp sgt <4 x i64> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8I64 = icmp sgt <8 x i64> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16I64 = icmp sgt <16 x i64> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX2-LABEL: 'cmp_int_sgt'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp sgt i8 undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = icmp sgt <16 x i8> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32I8 = icmp sgt <32 x i8> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64I8 = icmp sgt <64 x i8> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V128I8 = icmp sgt <128 x i8> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp sgt i16 undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = icmp sgt <8 x i16> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = icmp sgt <16 x i16> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I16 = icmp sgt <32 x i16> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I16 = icmp sgt <64 x i16> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp sgt i32 undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = icmp sgt <4 x i32> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = icmp sgt <8 x i32> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I32 = icmp sgt <16 x i32> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I32 = icmp sgt <32 x i32> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp sgt i64 undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = icmp sgt <2 x i64> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = icmp sgt <4 x i64> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I64 = icmp sgt <8 x i64> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I64 = icmp sgt <16 x i64> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX512F-LABEL: 'cmp_int_sgt'
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp sgt i8 undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = icmp sgt <16 x i8> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32I8 = icmp sgt <32 x i8> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64I8 = icmp sgt <64 x i8> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V128I8 = icmp sgt <128 x i8> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp sgt i16 undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = icmp sgt <8 x i16> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = icmp sgt <16 x i16> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I16 = icmp sgt <32 x i16> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I16 = icmp sgt <64 x i16> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp sgt i32 undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = icmp sgt <4 x i32> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = icmp sgt <8 x i32> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I32 = icmp sgt <16 x i32> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I32 = icmp sgt <32 x i32> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp sgt i64 undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = icmp sgt <2 x i64> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = icmp sgt <4 x i64> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = icmp sgt <8 x i64> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I64 = icmp sgt <16 x i64> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX512BW-LABEL: 'cmp_int_sgt'
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp sgt i8 undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = icmp sgt <16 x i8> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32I8 = icmp sgt <32 x i8> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64I8 = icmp sgt <64 x i8> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128I8 = icmp sgt <128 x i8> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp sgt i16 undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = icmp sgt <8 x i16> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = icmp sgt <16 x i16> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = icmp sgt <32 x i16> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64I16 = icmp sgt <64 x i16> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp sgt i32 undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = icmp sgt <4 x i32> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = icmp sgt <8 x i32> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I32 = icmp sgt <16 x i32> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I32 = icmp sgt <32 x i32> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp sgt i64 undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = icmp sgt <2 x i64> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = icmp sgt <4 x i64> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = icmp sgt <8 x i64> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I64 = icmp sgt <16 x i64> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; XOPAVX1-LABEL: 'cmp_int_sgt'
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp sgt i8 undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = icmp sgt <16 x i8> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I8 = icmp sgt <32 x i8> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I8 = icmp sgt <64 x i8> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V128I8 = icmp sgt <128 x i8> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp sgt i16 undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = icmp sgt <8 x i16> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I16 = icmp sgt <16 x i16> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I16 = icmp sgt <32 x i16> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V64I16 = icmp sgt <64 x i16> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp sgt i32 undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = icmp sgt <4 x i32> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I32 = icmp sgt <8 x i32> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I32 = icmp sgt <16 x i32> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V32I32 = icmp sgt <32 x i32> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp sgt i64 undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = icmp sgt <2 x i64> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4I64 = icmp sgt <4 x i64> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8I64 = icmp sgt <8 x i64> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16I64 = icmp sgt <16 x i64> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; XOPAVX2-LABEL: 'cmp_int_sgt'
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp sgt i8 undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = icmp sgt <16 x i8> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32I8 = icmp sgt <32 x i8> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64I8 = icmp sgt <64 x i8> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V128I8 = icmp sgt <128 x i8> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp sgt i16 undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = icmp sgt <8 x i16> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = icmp sgt <16 x i16> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I16 = icmp sgt <32 x i16> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I16 = icmp sgt <64 x i16> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp sgt i32 undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = icmp sgt <4 x i32> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = icmp sgt <8 x i32> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I32 = icmp sgt <16 x i32> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I32 = icmp sgt <32 x i32> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp sgt i64 undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = icmp sgt <2 x i64> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = icmp sgt <4 x i64> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I64 = icmp sgt <8 x i64> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I64 = icmp sgt <16 x i64> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; BTVER2-LABEL: 'cmp_int_sgt'
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp sgt i8 undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = icmp sgt <16 x i8> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I8 = icmp sgt <32 x i8> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I8 = icmp sgt <64 x i8> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V128I8 = icmp sgt <128 x i8> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp sgt i16 undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = icmp sgt <8 x i16> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I16 = icmp sgt <16 x i16> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I16 = icmp sgt <32 x i16> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V64I16 = icmp sgt <64 x i16> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp sgt i32 undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = icmp sgt <4 x i32> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I32 = icmp sgt <8 x i32> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I32 = icmp sgt <16 x i32> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V32I32 = icmp sgt <32 x i32> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp sgt i64 undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = icmp sgt <2 x i64> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4I64 = icmp sgt <4 x i64> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8I64 = icmp sgt <8 x i64> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16I64 = icmp sgt <16 x i64> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+  %I8 = icmp sgt i8 undef, undef
+  %V16I8 = icmp sgt <16 x i8> undef, undef
+  %V32I8 = icmp sgt <32 x i8> undef, undef
+  %V64I8 = icmp sgt <64 x i8> undef, undef
+  %V128I8 = icmp sgt <128 x i8> undef, undef
+
+  %I16 = icmp sgt i16 undef, undef
+  %V8I16  = icmp sgt <8 x i16> undef, undef
+  %V16I16 = icmp sgt <16 x i16> undef, undef
+  %V32I16 = icmp sgt <32 x i16> undef, undef
+  %V64I16 = icmp sgt <64 x i16> undef, undef
+
+  %I32 = icmp sgt i32 undef, undef
+  %V4I32 = icmp sgt <4 x i32> undef, undef
+  %V8I32 = icmp sgt <8 x i32> undef, undef
+  %V16I32 = icmp sgt <16 x i32> undef, undef
+  %V32I32 = icmp sgt <32 x i32> undef, undef
+
+  %I64 = icmp sgt i64 undef, undef
+  %V2I64 = icmp sgt <2 x i64> undef, undef
+  %V4I64 = icmp sgt <4 x i64> undef, undef
+  %V8I64 = icmp sgt <8 x i64> undef, undef
+  %V16I64 = icmp sgt <16 x i64> undef, undef
+
+  ret i32 undef
+}
+
+define i32 @cmp_int_ugt(i32 %arg) {
+; SSE2-LABEL: 'cmp_int_ugt'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp ugt i8 undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16I8 = icmp ugt <16 x i8> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V32I8 = icmp ugt <32 x i8> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V64I8 = icmp ugt <64 x i8> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V128I8 = icmp ugt <128 x i8> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp ugt i16 undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8I16 = icmp ugt <8 x i16> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V16I16 = icmp ugt <16 x i16> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V32I16 = icmp ugt <32 x i16> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V64I16 = icmp ugt <64 x i16> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp ugt i32 undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4I32 = icmp ugt <4 x i32> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V8I32 = icmp ugt <8 x i32> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V16I32 = icmp ugt <16 x i32> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V32I32 = icmp ugt <32 x i32> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp ugt i64 undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V2I64 = icmp ugt <2 x i64> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V4I64 = icmp ugt <4 x i64> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %V8I64 = icmp ugt <8 x i64> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %V16I64 = icmp ugt <16 x i64> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSE3-LABEL: 'cmp_int_ugt'
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp ugt i8 undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16I8 = icmp ugt <16 x i8> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V32I8 = icmp ugt <32 x i8> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V64I8 = icmp ugt <64 x i8> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V128I8 = icmp ugt <128 x i8> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp ugt i16 undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8I16 = icmp ugt <8 x i16> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V16I16 = icmp ugt <16 x i16> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V32I16 = icmp ugt <32 x i16> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V64I16 = icmp ugt <64 x i16> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp ugt i32 undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4I32 = icmp ugt <4 x i32> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V8I32 = icmp ugt <8 x i32> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V16I32 = icmp ugt <16 x i32> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V32I32 = icmp ugt <32 x i32> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp ugt i64 undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V2I64 = icmp ugt <2 x i64> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V4I64 = icmp ugt <4 x i64> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %V8I64 = icmp ugt <8 x i64> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %V16I64 = icmp ugt <16 x i64> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSSE3-LABEL: 'cmp_int_ugt'
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp ugt i8 undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16I8 = icmp ugt <16 x i8> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V32I8 = icmp ugt <32 x i8> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V64I8 = icmp ugt <64 x i8> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V128I8 = icmp ugt <128 x i8> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp ugt i16 undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8I16 = icmp ugt <8 x i16> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V16I16 = icmp ugt <16 x i16> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V32I16 = icmp ugt <32 x i16> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V64I16 = icmp ugt <64 x i16> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp ugt i32 undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4I32 = icmp ugt <4 x i32> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V8I32 = icmp ugt <8 x i32> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V16I32 = icmp ugt <16 x i32> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V32I32 = icmp ugt <32 x i32> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp ugt i64 undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V2I64 = icmp ugt <2 x i64> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V4I64 = icmp ugt <4 x i64> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %V8I64 = icmp ugt <8 x i64> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %V16I64 = icmp ugt <16 x i64> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSE41-LABEL: 'cmp_int_ugt'
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp ugt i8 undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16I8 = icmp ugt <16 x i8> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V32I8 = icmp ugt <32 x i8> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V64I8 = icmp ugt <64 x i8> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V128I8 = icmp ugt <128 x i8> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp ugt i16 undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8I16 = icmp ugt <8 x i16> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V16I16 = icmp ugt <16 x i16> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V32I16 = icmp ugt <32 x i16> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V64I16 = icmp ugt <64 x i16> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp ugt i32 undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4I32 = icmp ugt <4 x i32> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V8I32 = icmp ugt <8 x i32> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V16I32 = icmp ugt <16 x i32> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V32I32 = icmp ugt <32 x i32> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp ugt i64 undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V2I64 = icmp ugt <2 x i64> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V4I64 = icmp ugt <4 x i64> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %V8I64 = icmp ugt <8 x i64> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %V16I64 = icmp ugt <16 x i64> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSE42-LABEL: 'cmp_int_ugt'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp ugt i8 undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16I8 = icmp ugt <16 x i8> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V32I8 = icmp ugt <32 x i8> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V64I8 = icmp ugt <64 x i8> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V128I8 = icmp ugt <128 x i8> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp ugt i16 undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8I16 = icmp ugt <8 x i16> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V16I16 = icmp ugt <16 x i16> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V32I16 = icmp ugt <32 x i16> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V64I16 = icmp ugt <64 x i16> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp ugt i32 undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4I32 = icmp ugt <4 x i32> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V8I32 = icmp ugt <8 x i32> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V16I32 = icmp ugt <16 x i32> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V32I32 = icmp ugt <32 x i32> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp ugt i64 undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2I64 = icmp ugt <2 x i64> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4I64 = icmp ugt <4 x i64> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V8I64 = icmp ugt <8 x i64> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V16I64 = icmp ugt <16 x i64> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX1-LABEL: 'cmp_int_ugt'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp ugt i8 undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16I8 = icmp ugt <16 x i8> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V32I8 = icmp ugt <32 x i8> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V64I8 = icmp ugt <64 x i8> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V128I8 = icmp ugt <128 x i8> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp ugt i16 undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8I16 = icmp ugt <8 x i16> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V16I16 = icmp ugt <16 x i16> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V32I16 = icmp ugt <32 x i16> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V64I16 = icmp ugt <64 x i16> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp ugt i32 undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4I32 = icmp ugt <4 x i32> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V8I32 = icmp ugt <8 x i32> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V16I32 = icmp ugt <16 x i32> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V32I32 = icmp ugt <32 x i32> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp ugt i64 undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2I64 = icmp ugt <2 x i64> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4I64 = icmp ugt <4 x i64> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V8I64 = icmp ugt <8 x i64> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V16I64 = icmp ugt <16 x i64> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX2-LABEL: 'cmp_int_ugt'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp ugt i8 undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16I8 = icmp ugt <16 x i8> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32I8 = icmp ugt <32 x i8> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V64I8 = icmp ugt <64 x i8> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V128I8 = icmp ugt <128 x i8> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp ugt i16 undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8I16 = icmp ugt <8 x i16> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16I16 = icmp ugt <16 x i16> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V32I16 = icmp ugt <32 x i16> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V64I16 = icmp ugt <64 x i16> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp ugt i32 undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4I32 = icmp ugt <4 x i32> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8I32 = icmp ugt <8 x i32> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V16I32 = icmp ugt <16 x i32> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V32I32 = icmp ugt <32 x i32> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp ugt i64 undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2I64 = icmp ugt <2 x i64> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4I64 = icmp ugt <4 x i64> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V8I64 = icmp ugt <8 x i64> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V16I64 = icmp ugt <16 x i64> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX512F-LABEL: 'cmp_int_ugt'
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp ugt i8 undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16I8 = icmp ugt <16 x i8> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32I8 = icmp ugt <32 x i8> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V64I8 = icmp ugt <64 x i8> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V128I8 = icmp ugt <128 x i8> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp ugt i16 undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8I16 = icmp ugt <8 x i16> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16I16 = icmp ugt <16 x i16> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V32I16 = icmp ugt <32 x i16> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V64I16 = icmp ugt <64 x i16> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp ugt i32 undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = icmp ugt <4 x i32> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = icmp ugt <8 x i32> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I32 = icmp ugt <16 x i32> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I32 = icmp ugt <32 x i32> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp ugt i64 undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = icmp ugt <2 x i64> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = icmp ugt <4 x i64> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = icmp ugt <8 x i64> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I64 = icmp ugt <16 x i64> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX512BW-LABEL: 'cmp_int_ugt'
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp ugt i8 undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = icmp ugt <16 x i8> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32I8 = icmp ugt <32 x i8> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64I8 = icmp ugt <64 x i8> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128I8 = icmp ugt <128 x i8> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp ugt i16 undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = icmp ugt <8 x i16> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = icmp ugt <16 x i16> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = icmp ugt <32 x i16> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64I16 = icmp ugt <64 x i16> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp ugt i32 undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = icmp ugt <4 x i32> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = icmp ugt <8 x i32> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I32 = icmp ugt <16 x i32> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I32 = icmp ugt <32 x i32> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp ugt i64 undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = icmp ugt <2 x i64> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = icmp ugt <4 x i64> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = icmp ugt <8 x i64> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I64 = icmp ugt <16 x i64> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; XOPAVX1-LABEL: 'cmp_int_ugt'
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp ugt i8 undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = icmp ugt <16 x i8> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I8 = icmp ugt <32 x i8> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I8 = icmp ugt <64 x i8> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V128I8 = icmp ugt <128 x i8> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp ugt i16 undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = icmp ugt <8 x i16> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I16 = icmp ugt <16 x i16> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I16 = icmp ugt <32 x i16> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V64I16 = icmp ugt <64 x i16> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp ugt i32 undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = icmp ugt <4 x i32> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I32 = icmp ugt <8 x i32> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I32 = icmp ugt <16 x i32> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V32I32 = icmp ugt <32 x i32> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp ugt i64 undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = icmp ugt <2 x i64> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4I64 = icmp ugt <4 x i64> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8I64 = icmp ugt <8 x i64> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16I64 = icmp ugt <16 x i64> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; XOPAVX2-LABEL: 'cmp_int_ugt'
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp ugt i8 undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = icmp ugt <16 x i8> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32I8 = icmp ugt <32 x i8> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V64I8 = icmp ugt <64 x i8> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V128I8 = icmp ugt <128 x i8> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp ugt i16 undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = icmp ugt <8 x i16> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16I16 = icmp ugt <16 x i16> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V32I16 = icmp ugt <32 x i16> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V64I16 = icmp ugt <64 x i16> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp ugt i32 undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = icmp ugt <4 x i32> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8I32 = icmp ugt <8 x i32> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V16I32 = icmp ugt <16 x i32> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V32I32 = icmp ugt <32 x i32> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp ugt i64 undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = icmp ugt <2 x i64> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4I64 = icmp ugt <4 x i64> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V8I64 = icmp ugt <8 x i64> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V16I64 = icmp ugt <16 x i64> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; BTVER2-LABEL: 'cmp_int_ugt'
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp ugt i8 undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16I8 = icmp ugt <16 x i8> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V32I8 = icmp ugt <32 x i8> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V64I8 = icmp ugt <64 x i8> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V128I8 = icmp ugt <128 x i8> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp ugt i16 undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8I16 = icmp ugt <8 x i16> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V16I16 = icmp ugt <16 x i16> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V32I16 = icmp ugt <32 x i16> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V64I16 = icmp ugt <64 x i16> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp ugt i32 undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4I32 = icmp ugt <4 x i32> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V8I32 = icmp ugt <8 x i32> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V16I32 = icmp ugt <16 x i32> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V32I32 = icmp ugt <32 x i32> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp ugt i64 undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2I64 = icmp ugt <2 x i64> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4I64 = icmp ugt <4 x i64> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V8I64 = icmp ugt <8 x i64> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V16I64 = icmp ugt <16 x i64> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+  %I8 = icmp ugt i8 undef, undef
+  %V16I8 = icmp ugt <16 x i8> undef, undef
+  %V32I8 = icmp ugt <32 x i8> undef, undef
+  %V64I8 = icmp ugt <64 x i8> undef, undef
+  %V128I8 = icmp ugt <128 x i8> undef, undef
+
+  %I16 = icmp ugt i16 undef, undef
+  %V8I16  = icmp ugt <8 x i16> undef, undef
+  %V16I16 = icmp ugt <16 x i16> undef, undef
+  %V32I16 = icmp ugt <32 x i16> undef, undef
+  %V64I16 = icmp ugt <64 x i16> undef, undef
+
+  %I32 = icmp ugt i32 undef, undef
+  %V4I32 = icmp ugt <4 x i32> undef, undef
+  %V8I32 = icmp ugt <8 x i32> undef, undef
+  %V16I32 = icmp ugt <16 x i32> undef, undef
+  %V32I32 = icmp ugt <32 x i32> undef, undef
+
+  %I64 = icmp ugt i64 undef, undef
+  %V2I64 = icmp ugt <2 x i64> undef, undef
+  %V4I64 = icmp ugt <4 x i64> undef, undef
+  %V8I64 = icmp ugt <8 x i64> undef, undef
+  %V16I64 = icmp ugt <16 x i64> undef, undef
+
+  ret i32 undef
+}
+
+define i32 @cmp_int_sle(i32 %arg) {
+; SSE2-LABEL: 'cmp_int_sle'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp sle i8 undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = icmp sle <16 x i8> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I8 = icmp sle <32 x i8> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I8 = icmp sle <64 x i8> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V128I8 = icmp sle <128 x i8> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp sle i16 undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = icmp sle <8 x i16> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I16 = icmp sle <16 x i16> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I16 = icmp sle <32 x i16> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V64I16 = icmp sle <64 x i16> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp sle i32 undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = icmp sle <4 x i32> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I32 = icmp sle <8 x i32> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I32 = icmp sle <16 x i32> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V32I32 = icmp sle <32 x i32> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp sle i64 undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V2I64 = icmp sle <2 x i64> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V4I64 = icmp sle <4 x i64> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %V8I64 = icmp sle <8 x i64> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %V16I64 = icmp sle <16 x i64> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSE3-LABEL: 'cmp_int_sle'
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp sle i8 undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = icmp sle <16 x i8> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I8 = icmp sle <32 x i8> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I8 = icmp sle <64 x i8> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V128I8 = icmp sle <128 x i8> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp sle i16 undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = icmp sle <8 x i16> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I16 = icmp sle <16 x i16> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I16 = icmp sle <32 x i16> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V64I16 = icmp sle <64 x i16> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp sle i32 undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = icmp sle <4 x i32> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I32 = icmp sle <8 x i32> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I32 = icmp sle <16 x i32> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V32I32 = icmp sle <32 x i32> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp sle i64 undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V2I64 = icmp sle <2 x i64> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V4I64 = icmp sle <4 x i64> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %V8I64 = icmp sle <8 x i64> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %V16I64 = icmp sle <16 x i64> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSSE3-LABEL: 'cmp_int_sle'
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp sle i8 undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = icmp sle <16 x i8> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I8 = icmp sle <32 x i8> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I8 = icmp sle <64 x i8> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V128I8 = icmp sle <128 x i8> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp sle i16 undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = icmp sle <8 x i16> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I16 = icmp sle <16 x i16> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I16 = icmp sle <32 x i16> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V64I16 = icmp sle <64 x i16> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp sle i32 undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = icmp sle <4 x i32> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I32 = icmp sle <8 x i32> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I32 = icmp sle <16 x i32> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V32I32 = icmp sle <32 x i32> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp sle i64 undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V2I64 = icmp sle <2 x i64> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V4I64 = icmp sle <4 x i64> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %V8I64 = icmp sle <8 x i64> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %V16I64 = icmp sle <16 x i64> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSE41-LABEL: 'cmp_int_sle'
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp sle i8 undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = icmp sle <16 x i8> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I8 = icmp sle <32 x i8> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I8 = icmp sle <64 x i8> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V128I8 = icmp sle <128 x i8> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp sle i16 undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = icmp sle <8 x i16> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I16 = icmp sle <16 x i16> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I16 = icmp sle <32 x i16> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V64I16 = icmp sle <64 x i16> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp sle i32 undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = icmp sle <4 x i32> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I32 = icmp sle <8 x i32> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I32 = icmp sle <16 x i32> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V32I32 = icmp sle <32 x i32> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp sle i64 undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V2I64 = icmp sle <2 x i64> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V4I64 = icmp sle <4 x i64> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %V8I64 = icmp sle <8 x i64> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 72 for instruction: %V16I64 = icmp sle <16 x i64> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSE42-LABEL: 'cmp_int_sle'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp sle i8 undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = icmp sle <16 x i8> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I8 = icmp sle <32 x i8> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I8 = icmp sle <64 x i8> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V128I8 = icmp sle <128 x i8> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp sle i16 undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = icmp sle <8 x i16> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I16 = icmp sle <16 x i16> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I16 = icmp sle <32 x i16> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V64I16 = icmp sle <64 x i16> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp sle i32 undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = icmp sle <4 x i32> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I32 = icmp sle <8 x i32> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I32 = icmp sle <16 x i32> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V32I32 = icmp sle <32 x i32> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp sle i64 undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I64 = icmp sle <2 x i64> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4I64 = icmp sle <4 x i64> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8I64 = icmp sle <8 x i64> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16I64 = icmp sle <16 x i64> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX1-LABEL: 'cmp_int_sle'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp sle i8 undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = icmp sle <16 x i8> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V32I8 = icmp sle <32 x i8> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V64I8 = icmp sle <64 x i8> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V128I8 = icmp sle <128 x i8> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp sle i16 undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = icmp sle <8 x i16> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V16I16 = icmp sle <16 x i16> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V32I16 = icmp sle <32 x i16> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V64I16 = icmp sle <64 x i16> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp sle i32 undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = icmp sle <4 x i32> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V8I32 = icmp sle <8 x i32> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V16I32 = icmp sle <16 x i32> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V32I32 = icmp sle <32 x i32> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp sle i64 undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I64 = icmp sle <2 x i64> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V4I64 = icmp sle <4 x i64> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V8I64 = icmp sle <8 x i64> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V16I64 = icmp sle <16 x i64> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX2-LABEL: 'cmp_int_sle'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp sle i8 undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = icmp sle <16 x i8> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I8 = icmp sle <32 x i8> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = icmp sle <64 x i8> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V128I8 = icmp sle <128 x i8> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp sle i16 undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = icmp sle <8 x i16> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = icmp sle <16 x i16> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = icmp sle <32 x i16> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I16 = icmp sle <64 x i16> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp sle i32 undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = icmp sle <4 x i32> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = icmp sle <8 x i32> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I32 = icmp sle <16 x i32> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I32 = icmp sle <32 x i32> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp sle i64 undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I64 = icmp sle <2 x i64> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I64 = icmp sle <4 x i64> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I64 = icmp sle <8 x i64> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I64 = icmp sle <16 x i64> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX512F-LABEL: 'cmp_int_sle'
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp sle i8 undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = icmp sle <16 x i8> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I8 = icmp sle <32 x i8> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = icmp sle <64 x i8> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V128I8 = icmp sle <128 x i8> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp sle i16 undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = icmp sle <8 x i16> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = icmp sle <16 x i16> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = icmp sle <32 x i16> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I16 = icmp sle <64 x i16> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp sle i32 undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = icmp sle <4 x i32> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = icmp sle <8 x i32> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I32 = icmp sle <16 x i32> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I32 = icmp sle <32 x i32> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp sle i64 undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = icmp sle <2 x i64> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = icmp sle <4 x i64> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = icmp sle <8 x i64> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I64 = icmp sle <16 x i64> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX512BW-LABEL: 'cmp_int_sle'
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp sle i8 undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = icmp sle <16 x i8> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32I8 = icmp sle <32 x i8> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64I8 = icmp sle <64 x i8> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128I8 = icmp sle <128 x i8> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp sle i16 undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = icmp sle <8 x i16> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = icmp sle <16 x i16> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = icmp sle <32 x i16> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64I16 = icmp sle <64 x i16> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp sle i32 undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = icmp sle <4 x i32> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = icmp sle <8 x i32> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I32 = icmp sle <16 x i32> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I32 = icmp sle <32 x i32> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp sle i64 undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = icmp sle <2 x i64> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = icmp sle <4 x i64> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = icmp sle <8 x i64> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I64 = icmp sle <16 x i64> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; XOPAVX1-LABEL: 'cmp_int_sle'
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp sle i8 undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = icmp sle <16 x i8> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I8 = icmp sle <32 x i8> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I8 = icmp sle <64 x i8> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V128I8 = icmp sle <128 x i8> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp sle i16 undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = icmp sle <8 x i16> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I16 = icmp sle <16 x i16> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I16 = icmp sle <32 x i16> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V64I16 = icmp sle <64 x i16> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp sle i32 undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = icmp sle <4 x i32> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I32 = icmp sle <8 x i32> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I32 = icmp sle <16 x i32> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V32I32 = icmp sle <32 x i32> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp sle i64 undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = icmp sle <2 x i64> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4I64 = icmp sle <4 x i64> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8I64 = icmp sle <8 x i64> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16I64 = icmp sle <16 x i64> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; XOPAVX2-LABEL: 'cmp_int_sle'
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp sle i8 undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = icmp sle <16 x i8> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I8 = icmp sle <32 x i8> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = icmp sle <64 x i8> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V128I8 = icmp sle <128 x i8> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp sle i16 undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = icmp sle <8 x i16> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = icmp sle <16 x i16> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = icmp sle <32 x i16> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I16 = icmp sle <64 x i16> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp sle i32 undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = icmp sle <4 x i32> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = icmp sle <8 x i32> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I32 = icmp sle <16 x i32> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I32 = icmp sle <32 x i32> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp sle i64 undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = icmp sle <2 x i64> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I64 = icmp sle <4 x i64> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I64 = icmp sle <8 x i64> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I64 = icmp sle <16 x i64> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; BTVER2-LABEL: 'cmp_int_sle'
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp sle i8 undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = icmp sle <16 x i8> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V32I8 = icmp sle <32 x i8> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V64I8 = icmp sle <64 x i8> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V128I8 = icmp sle <128 x i8> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp sle i16 undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = icmp sle <8 x i16> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V16I16 = icmp sle <16 x i16> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V32I16 = icmp sle <32 x i16> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V64I16 = icmp sle <64 x i16> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp sle i32 undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = icmp sle <4 x i32> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V8I32 = icmp sle <8 x i32> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V16I32 = icmp sle <16 x i32> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V32I32 = icmp sle <32 x i32> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp sle i64 undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I64 = icmp sle <2 x i64> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V4I64 = icmp sle <4 x i64> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V8I64 = icmp sle <8 x i64> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V16I64 = icmp sle <16 x i64> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+  %I8 = icmp sle i8 undef, undef
+  %V16I8 = icmp sle <16 x i8> undef, undef
+  %V32I8 = icmp sle <32 x i8> undef, undef
+  %V64I8 = icmp sle <64 x i8> undef, undef
+  %V128I8 = icmp sle <128 x i8> undef, undef
+
+  %I16 = icmp sle i16 undef, undef
+  %V8I16  = icmp sle <8 x i16> undef, undef
+  %V16I16 = icmp sle <16 x i16> undef, undef
+  %V32I16 = icmp sle <32 x i16> undef, undef
+  %V64I16 = icmp sle <64 x i16> undef, undef
+
+  %I32 = icmp sle i32 undef, undef
+  %V4I32 = icmp sle <4 x i32> undef, undef
+  %V8I32 = icmp sle <8 x i32> undef, undef
+  %V16I32 = icmp sle <16 x i32> undef, undef
+  %V32I32 = icmp sle <32 x i32> undef, undef
+
+  %I64 = icmp sle i64 undef, undef
+  %V2I64 = icmp sle <2 x i64> undef, undef
+  %V4I64 = icmp sle <4 x i64> undef, undef
+  %V8I64 = icmp sle <8 x i64> undef, undef
+  %V16I64 = icmp sle <16 x i64> undef, undef
+
+  ret i32 undef
+}
+
+define i32 @cmp_int_ule(i32 %arg) {
+; SSE2-LABEL: 'cmp_int_ule'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp ule i8 undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = icmp ule <16 x i8> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I8 = icmp ule <32 x i8> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I8 = icmp ule <64 x i8> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V128I8 = icmp ule <128 x i8> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp ule i16 undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = icmp ule <8 x i16> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I16 = icmp ule <16 x i16> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I16 = icmp ule <32 x i16> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V64I16 = icmp ule <64 x i16> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp ule i32 undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4I32 = icmp ule <4 x i32> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8I32 = icmp ule <8 x i32> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16I32 = icmp ule <16 x i32> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V32I32 = icmp ule <32 x i32> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp ule i64 undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V2I64 = icmp ule <2 x i64> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V4I64 = icmp ule <4 x i64> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 44 for instruction: %V8I64 = icmp ule <8 x i64> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 88 for instruction: %V16I64 = icmp ule <16 x i64> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSE3-LABEL: 'cmp_int_ule'
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp ule i8 undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = icmp ule <16 x i8> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I8 = icmp ule <32 x i8> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I8 = icmp ule <64 x i8> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V128I8 = icmp ule <128 x i8> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp ule i16 undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = icmp ule <8 x i16> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I16 = icmp ule <16 x i16> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I16 = icmp ule <32 x i16> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V64I16 = icmp ule <64 x i16> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp ule i32 undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4I32 = icmp ule <4 x i32> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8I32 = icmp ule <8 x i32> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16I32 = icmp ule <16 x i32> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V32I32 = icmp ule <32 x i32> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp ule i64 undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V2I64 = icmp ule <2 x i64> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V4I64 = icmp ule <4 x i64> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 44 for instruction: %V8I64 = icmp ule <8 x i64> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 88 for instruction: %V16I64 = icmp ule <16 x i64> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSSE3-LABEL: 'cmp_int_ule'
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp ule i8 undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = icmp ule <16 x i8> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I8 = icmp ule <32 x i8> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I8 = icmp ule <64 x i8> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V128I8 = icmp ule <128 x i8> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp ule i16 undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = icmp ule <8 x i16> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I16 = icmp ule <16 x i16> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I16 = icmp ule <32 x i16> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V64I16 = icmp ule <64 x i16> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp ule i32 undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4I32 = icmp ule <4 x i32> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8I32 = icmp ule <8 x i32> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16I32 = icmp ule <16 x i32> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V32I32 = icmp ule <32 x i32> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp ule i64 undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V2I64 = icmp ule <2 x i64> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V4I64 = icmp ule <4 x i64> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 44 for instruction: %V8I64 = icmp ule <8 x i64> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 88 for instruction: %V16I64 = icmp ule <16 x i64> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSE41-LABEL: 'cmp_int_ule'
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp ule i8 undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = icmp ule <16 x i8> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I8 = icmp ule <32 x i8> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I8 = icmp ule <64 x i8> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V128I8 = icmp ule <128 x i8> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp ule i16 undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = icmp ule <8 x i16> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I16 = icmp ule <16 x i16> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I16 = icmp ule <32 x i16> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V64I16 = icmp ule <64 x i16> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp ule i32 undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = icmp ule <4 x i32> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I32 = icmp ule <8 x i32> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I32 = icmp ule <16 x i32> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V32I32 = icmp ule <32 x i32> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp ule i64 undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V2I64 = icmp ule <2 x i64> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V4I64 = icmp ule <4 x i64> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 44 for instruction: %V8I64 = icmp ule <8 x i64> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 88 for instruction: %V16I64 = icmp ule <16 x i64> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSE42-LABEL: 'cmp_int_ule'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp ule i8 undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = icmp ule <16 x i8> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I8 = icmp ule <32 x i8> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I8 = icmp ule <64 x i8> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V128I8 = icmp ule <128 x i8> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp ule i16 undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = icmp ule <8 x i16> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I16 = icmp ule <16 x i16> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I16 = icmp ule <32 x i16> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V64I16 = icmp ule <64 x i16> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp ule i32 undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = icmp ule <4 x i32> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I32 = icmp ule <8 x i32> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I32 = icmp ule <16 x i32> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V32I32 = icmp ule <32 x i32> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp ule i64 undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2I64 = icmp ule <2 x i64> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4I64 = icmp ule <4 x i64> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8I64 = icmp ule <8 x i64> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V16I64 = icmp ule <16 x i64> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX1-LABEL: 'cmp_int_ule'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp ule i8 undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = icmp ule <16 x i8> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V32I8 = icmp ule <32 x i8> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V64I8 = icmp ule <64 x i8> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V128I8 = icmp ule <128 x i8> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp ule i16 undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = icmp ule <8 x i16> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V16I16 = icmp ule <16 x i16> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V32I16 = icmp ule <32 x i16> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V64I16 = icmp ule <64 x i16> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp ule i32 undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = icmp ule <4 x i32> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V8I32 = icmp ule <8 x i32> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V16I32 = icmp ule <16 x i32> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V32I32 = icmp ule <32 x i32> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp ule i64 undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2I64 = icmp ule <2 x i64> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4I64 = icmp ule <4 x i64> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V8I64 = icmp ule <8 x i64> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V16I64 = icmp ule <16 x i64> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX2-LABEL: 'cmp_int_ule'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp ule i8 undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = icmp ule <16 x i8> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I8 = icmp ule <32 x i8> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = icmp ule <64 x i8> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V128I8 = icmp ule <128 x i8> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp ule i16 undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = icmp ule <8 x i16> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = icmp ule <16 x i16> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = icmp ule <32 x i16> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I16 = icmp ule <64 x i16> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp ule i32 undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = icmp ule <4 x i32> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = icmp ule <8 x i32> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I32 = icmp ule <16 x i32> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I32 = icmp ule <32 x i32> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp ule i64 undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2I64 = icmp ule <2 x i64> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4I64 = icmp ule <4 x i64> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8I64 = icmp ule <8 x i64> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16I64 = icmp ule <16 x i64> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX512F-LABEL: 'cmp_int_ule'
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp ule i8 undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = icmp ule <16 x i8> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I8 = icmp ule <32 x i8> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = icmp ule <64 x i8> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V128I8 = icmp ule <128 x i8> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp ule i16 undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = icmp ule <8 x i16> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = icmp ule <16 x i16> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = icmp ule <32 x i16> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I16 = icmp ule <64 x i16> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp ule i32 undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = icmp ule <4 x i32> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = icmp ule <8 x i32> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I32 = icmp ule <16 x i32> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I32 = icmp ule <32 x i32> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp ule i64 undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = icmp ule <2 x i64> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = icmp ule <4 x i64> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = icmp ule <8 x i64> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I64 = icmp ule <16 x i64> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX512BW-LABEL: 'cmp_int_ule'
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp ule i8 undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = icmp ule <16 x i8> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32I8 = icmp ule <32 x i8> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64I8 = icmp ule <64 x i8> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128I8 = icmp ule <128 x i8> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp ule i16 undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = icmp ule <8 x i16> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = icmp ule <16 x i16> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = icmp ule <32 x i16> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64I16 = icmp ule <64 x i16> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp ule i32 undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = icmp ule <4 x i32> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = icmp ule <8 x i32> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I32 = icmp ule <16 x i32> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I32 = icmp ule <32 x i32> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp ule i64 undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = icmp ule <2 x i64> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = icmp ule <4 x i64> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = icmp ule <8 x i64> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I64 = icmp ule <16 x i64> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; XOPAVX1-LABEL: 'cmp_int_ule'
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp ule i8 undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = icmp ule <16 x i8> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I8 = icmp ule <32 x i8> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I8 = icmp ule <64 x i8> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V128I8 = icmp ule <128 x i8> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp ule i16 undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = icmp ule <8 x i16> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I16 = icmp ule <16 x i16> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I16 = icmp ule <32 x i16> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V64I16 = icmp ule <64 x i16> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp ule i32 undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = icmp ule <4 x i32> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I32 = icmp ule <8 x i32> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I32 = icmp ule <16 x i32> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V32I32 = icmp ule <32 x i32> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp ule i64 undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = icmp ule <2 x i64> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4I64 = icmp ule <4 x i64> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8I64 = icmp ule <8 x i64> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16I64 = icmp ule <16 x i64> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; XOPAVX2-LABEL: 'cmp_int_ule'
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp ule i8 undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = icmp ule <16 x i8> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I8 = icmp ule <32 x i8> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = icmp ule <64 x i8> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V128I8 = icmp ule <128 x i8> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp ule i16 undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = icmp ule <8 x i16> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = icmp ule <16 x i16> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = icmp ule <32 x i16> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I16 = icmp ule <64 x i16> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp ule i32 undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = icmp ule <4 x i32> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = icmp ule <8 x i32> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I32 = icmp ule <16 x i32> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I32 = icmp ule <32 x i32> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp ule i64 undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = icmp ule <2 x i64> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4I64 = icmp ule <4 x i64> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8I64 = icmp ule <8 x i64> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16I64 = icmp ule <16 x i64> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; BTVER2-LABEL: 'cmp_int_ule'
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp ule i8 undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = icmp ule <16 x i8> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V32I8 = icmp ule <32 x i8> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V64I8 = icmp ule <64 x i8> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V128I8 = icmp ule <128 x i8> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp ule i16 undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = icmp ule <8 x i16> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V16I16 = icmp ule <16 x i16> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V32I16 = icmp ule <32 x i16> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V64I16 = icmp ule <64 x i16> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp ule i32 undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = icmp ule <4 x i32> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V8I32 = icmp ule <8 x i32> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V16I32 = icmp ule <16 x i32> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V32I32 = icmp ule <32 x i32> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp ule i64 undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2I64 = icmp ule <2 x i64> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4I64 = icmp ule <4 x i64> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V8I64 = icmp ule <8 x i64> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V16I64 = icmp ule <16 x i64> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+  %I8 = icmp ule i8 undef, undef
+  %V16I8 = icmp ule <16 x i8> undef, undef
+  %V32I8 = icmp ule <32 x i8> undef, undef
+  %V64I8 = icmp ule <64 x i8> undef, undef
+  %V128I8 = icmp ule <128 x i8> undef, undef
+
+  %I16 = icmp ule i16 undef, undef
+  %V8I16  = icmp ule <8 x i16> undef, undef
+  %V16I16 = icmp ule <16 x i16> undef, undef
+  %V32I16 = icmp ule <32 x i16> undef, undef
+  %V64I16 = icmp ule <64 x i16> undef, undef
+
+  %I32 = icmp ule i32 undef, undef
+  %V4I32 = icmp ule <4 x i32> undef, undef
+  %V8I32 = icmp ule <8 x i32> undef, undef
+  %V16I32 = icmp ule <16 x i32> undef, undef
+  %V32I32 = icmp ule <32 x i32> undef, undef
+
+  %I64 = icmp ule i64 undef, undef
+  %V2I64 = icmp ule <2 x i64> undef, undef
+  %V4I64 = icmp ule <4 x i64> undef, undef
+  %V8I64 = icmp ule <8 x i64> undef, undef
+  %V16I64 = icmp ule <16 x i64> undef, undef
+
+  ret i32 undef
+}
+
+define i32 @cmp_int_slt(i32 %arg) {
+; SSE2-LABEL: 'cmp_int_slt'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp slt i8 undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = icmp slt <16 x i8> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I8 = icmp slt <32 x i8> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = icmp slt <64 x i8> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V128I8 = icmp slt <128 x i8> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp slt i16 undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = icmp slt <8 x i16> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = icmp slt <16 x i16> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = icmp slt <32 x i16> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I16 = icmp slt <64 x i16> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp slt i32 undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = icmp slt <4 x i32> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = icmp slt <8 x i32> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I32 = icmp slt <16 x i32> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I32 = icmp slt <32 x i32> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp slt i64 undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2I64 = icmp slt <2 x i64> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V4I64 = icmp slt <4 x i64> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V8I64 = icmp slt <8 x i64> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %V16I64 = icmp slt <16 x i64> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSE3-LABEL: 'cmp_int_slt'
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp slt i8 undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = icmp slt <16 x i8> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I8 = icmp slt <32 x i8> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = icmp slt <64 x i8> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V128I8 = icmp slt <128 x i8> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp slt i16 undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = icmp slt <8 x i16> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = icmp slt <16 x i16> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = icmp slt <32 x i16> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I16 = icmp slt <64 x i16> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp slt i32 undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = icmp slt <4 x i32> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = icmp slt <8 x i32> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I32 = icmp slt <16 x i32> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I32 = icmp slt <32 x i32> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp slt i64 undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2I64 = icmp slt <2 x i64> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V4I64 = icmp slt <4 x i64> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V8I64 = icmp slt <8 x i64> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %V16I64 = icmp slt <16 x i64> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSSE3-LABEL: 'cmp_int_slt'
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp slt i8 undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = icmp slt <16 x i8> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I8 = icmp slt <32 x i8> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = icmp slt <64 x i8> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V128I8 = icmp slt <128 x i8> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp slt i16 undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = icmp slt <8 x i16> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = icmp slt <16 x i16> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = icmp slt <32 x i16> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I16 = icmp slt <64 x i16> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp slt i32 undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = icmp slt <4 x i32> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = icmp slt <8 x i32> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I32 = icmp slt <16 x i32> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I32 = icmp slt <32 x i32> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp slt i64 undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2I64 = icmp slt <2 x i64> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V4I64 = icmp slt <4 x i64> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V8I64 = icmp slt <8 x i64> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %V16I64 = icmp slt <16 x i64> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSE41-LABEL: 'cmp_int_slt'
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp slt i8 undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = icmp slt <16 x i8> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I8 = icmp slt <32 x i8> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = icmp slt <64 x i8> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V128I8 = icmp slt <128 x i8> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp slt i16 undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = icmp slt <8 x i16> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = icmp slt <16 x i16> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = icmp slt <32 x i16> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I16 = icmp slt <64 x i16> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp slt i32 undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = icmp slt <4 x i32> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = icmp slt <8 x i32> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I32 = icmp slt <16 x i32> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I32 = icmp slt <32 x i32> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp slt i64 undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2I64 = icmp slt <2 x i64> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V4I64 = icmp slt <4 x i64> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V8I64 = icmp slt <8 x i64> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %V16I64 = icmp slt <16 x i64> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSE42-LABEL: 'cmp_int_slt'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp slt i8 undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = icmp slt <16 x i8> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I8 = icmp slt <32 x i8> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = icmp slt <64 x i8> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V128I8 = icmp slt <128 x i8> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp slt i16 undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = icmp slt <8 x i16> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = icmp slt <16 x i16> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = icmp slt <32 x i16> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I16 = icmp slt <64 x i16> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp slt i32 undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = icmp slt <4 x i32> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = icmp slt <8 x i32> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I32 = icmp slt <16 x i32> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I32 = icmp slt <32 x i32> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp slt i64 undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = icmp slt <2 x i64> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I64 = icmp slt <4 x i64> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I64 = icmp slt <8 x i64> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I64 = icmp slt <16 x i64> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX1-LABEL: 'cmp_int_slt'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp slt i8 undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = icmp slt <16 x i8> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I8 = icmp slt <32 x i8> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I8 = icmp slt <64 x i8> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V128I8 = icmp slt <128 x i8> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp slt i16 undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = icmp slt <8 x i16> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I16 = icmp slt <16 x i16> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I16 = icmp slt <32 x i16> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V64I16 = icmp slt <64 x i16> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp slt i32 undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = icmp slt <4 x i32> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I32 = icmp slt <8 x i32> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I32 = icmp slt <16 x i32> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V32I32 = icmp slt <32 x i32> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp slt i64 undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = icmp slt <2 x i64> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4I64 = icmp slt <4 x i64> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8I64 = icmp slt <8 x i64> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16I64 = icmp slt <16 x i64> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX2-LABEL: 'cmp_int_slt'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp slt i8 undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = icmp slt <16 x i8> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32I8 = icmp slt <32 x i8> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64I8 = icmp slt <64 x i8> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V128I8 = icmp slt <128 x i8> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp slt i16 undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = icmp slt <8 x i16> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = icmp slt <16 x i16> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I16 = icmp slt <32 x i16> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I16 = icmp slt <64 x i16> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp slt i32 undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = icmp slt <4 x i32> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = icmp slt <8 x i32> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I32 = icmp slt <16 x i32> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I32 = icmp slt <32 x i32> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp slt i64 undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = icmp slt <2 x i64> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = icmp slt <4 x i64> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I64 = icmp slt <8 x i64> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I64 = icmp slt <16 x i64> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX512F-LABEL: 'cmp_int_slt'
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp slt i8 undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = icmp slt <16 x i8> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32I8 = icmp slt <32 x i8> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64I8 = icmp slt <64 x i8> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V128I8 = icmp slt <128 x i8> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp slt i16 undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = icmp slt <8 x i16> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = icmp slt <16 x i16> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I16 = icmp slt <32 x i16> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I16 = icmp slt <64 x i16> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp slt i32 undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = icmp slt <4 x i32> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = icmp slt <8 x i32> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I32 = icmp slt <16 x i32> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I32 = icmp slt <32 x i32> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp slt i64 undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = icmp slt <2 x i64> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = icmp slt <4 x i64> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = icmp slt <8 x i64> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I64 = icmp slt <16 x i64> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX512BW-LABEL: 'cmp_int_slt'
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp slt i8 undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = icmp slt <16 x i8> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32I8 = icmp slt <32 x i8> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64I8 = icmp slt <64 x i8> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128I8 = icmp slt <128 x i8> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp slt i16 undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = icmp slt <8 x i16> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = icmp slt <16 x i16> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = icmp slt <32 x i16> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64I16 = icmp slt <64 x i16> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp slt i32 undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = icmp slt <4 x i32> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = icmp slt <8 x i32> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I32 = icmp slt <16 x i32> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I32 = icmp slt <32 x i32> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp slt i64 undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = icmp slt <2 x i64> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = icmp slt <4 x i64> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = icmp slt <8 x i64> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I64 = icmp slt <16 x i64> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; XOPAVX1-LABEL: 'cmp_int_slt'
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp slt i8 undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = icmp slt <16 x i8> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I8 = icmp slt <32 x i8> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I8 = icmp slt <64 x i8> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V128I8 = icmp slt <128 x i8> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp slt i16 undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = icmp slt <8 x i16> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I16 = icmp slt <16 x i16> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I16 = icmp slt <32 x i16> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V64I16 = icmp slt <64 x i16> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp slt i32 undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = icmp slt <4 x i32> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I32 = icmp slt <8 x i32> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I32 = icmp slt <16 x i32> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V32I32 = icmp slt <32 x i32> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp slt i64 undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = icmp slt <2 x i64> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4I64 = icmp slt <4 x i64> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8I64 = icmp slt <8 x i64> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16I64 = icmp slt <16 x i64> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; XOPAVX2-LABEL: 'cmp_int_slt'
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp slt i8 undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = icmp slt <16 x i8> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32I8 = icmp slt <32 x i8> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64I8 = icmp slt <64 x i8> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V128I8 = icmp slt <128 x i8> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp slt i16 undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = icmp slt <8 x i16> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = icmp slt <16 x i16> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I16 = icmp slt <32 x i16> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I16 = icmp slt <64 x i16> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp slt i32 undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = icmp slt <4 x i32> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = icmp slt <8 x i32> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I32 = icmp slt <16 x i32> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I32 = icmp slt <32 x i32> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp slt i64 undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = icmp slt <2 x i64> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = icmp slt <4 x i64> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I64 = icmp slt <8 x i64> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I64 = icmp slt <16 x i64> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; BTVER2-LABEL: 'cmp_int_slt'
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp slt i8 undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = icmp slt <16 x i8> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I8 = icmp slt <32 x i8> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I8 = icmp slt <64 x i8> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V128I8 = icmp slt <128 x i8> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp slt i16 undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = icmp slt <8 x i16> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I16 = icmp slt <16 x i16> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I16 = icmp slt <32 x i16> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V64I16 = icmp slt <64 x i16> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp slt i32 undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = icmp slt <4 x i32> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I32 = icmp slt <8 x i32> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I32 = icmp slt <16 x i32> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V32I32 = icmp slt <32 x i32> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp slt i64 undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = icmp slt <2 x i64> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4I64 = icmp slt <4 x i64> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8I64 = icmp slt <8 x i64> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16I64 = icmp slt <16 x i64> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+  %I8 = icmp slt i8 undef, undef
+  %V16I8 = icmp slt <16 x i8> undef, undef
+  %V32I8 = icmp slt <32 x i8> undef, undef
+  %V64I8 = icmp slt <64 x i8> undef, undef
+  %V128I8 = icmp slt <128 x i8> undef, undef
+
+  %I16 = icmp slt i16 undef, undef
+  %V8I16  = icmp slt <8 x i16> undef, undef
+  %V16I16 = icmp slt <16 x i16> undef, undef
+  %V32I16 = icmp slt <32 x i16> undef, undef
+  %V64I16 = icmp slt <64 x i16> undef, undef
+
+  %I32 = icmp slt i32 undef, undef
+  %V4I32 = icmp slt <4 x i32> undef, undef
+  %V8I32 = icmp slt <8 x i32> undef, undef
+  %V16I32 = icmp slt <16 x i32> undef, undef
+  %V32I32 = icmp slt <32 x i32> undef, undef
+
+  %I64 = icmp slt i64 undef, undef
+  %V2I64 = icmp slt <2 x i64> undef, undef
+  %V4I64 = icmp slt <4 x i64> undef, undef
+  %V8I64 = icmp slt <8 x i64> undef, undef
+  %V16I64 = icmp slt <16 x i64> undef, undef
+
+  ret i32 undef
+}
+
+define i32 @cmp_int_ult(i32 %arg) {
+; SSE2-LABEL: 'cmp_int_ult'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp ult i8 undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16I8 = icmp ult <16 x i8> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V32I8 = icmp ult <32 x i8> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V64I8 = icmp ult <64 x i8> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V128I8 = icmp ult <128 x i8> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp ult i16 undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8I16 = icmp ult <8 x i16> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V16I16 = icmp ult <16 x i16> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V32I16 = icmp ult <32 x i16> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V64I16 = icmp ult <64 x i16> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp ult i32 undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4I32 = icmp ult <4 x i32> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V8I32 = icmp ult <8 x i32> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V16I32 = icmp ult <16 x i32> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V32I32 = icmp ult <32 x i32> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp ult i64 undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V2I64 = icmp ult <2 x i64> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V4I64 = icmp ult <4 x i64> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %V8I64 = icmp ult <8 x i64> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %V16I64 = icmp ult <16 x i64> undef, undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSE3-LABEL: 'cmp_int_ult'
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp ult i8 undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16I8 = icmp ult <16 x i8> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V32I8 = icmp ult <32 x i8> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V64I8 = icmp ult <64 x i8> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V128I8 = icmp ult <128 x i8> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp ult i16 undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8I16 = icmp ult <8 x i16> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V16I16 = icmp ult <16 x i16> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V32I16 = icmp ult <32 x i16> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V64I16 = icmp ult <64 x i16> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp ult i32 undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4I32 = icmp ult <4 x i32> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V8I32 = icmp ult <8 x i32> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V16I32 = icmp ult <16 x i32> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V32I32 = icmp ult <32 x i32> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp ult i64 undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V2I64 = icmp ult <2 x i64> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V4I64 = icmp ult <4 x i64> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %V8I64 = icmp ult <8 x i64> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %V16I64 = icmp ult <16 x i64> undef, undef
+; SSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSSE3-LABEL: 'cmp_int_ult'
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp ult i8 undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16I8 = icmp ult <16 x i8> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V32I8 = icmp ult <32 x i8> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V64I8 = icmp ult <64 x i8> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V128I8 = icmp ult <128 x i8> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp ult i16 undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8I16 = icmp ult <8 x i16> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V16I16 = icmp ult <16 x i16> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V32I16 = icmp ult <32 x i16> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V64I16 = icmp ult <64 x i16> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp ult i32 undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4I32 = icmp ult <4 x i32> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V8I32 = icmp ult <8 x i32> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V16I32 = icmp ult <16 x i32> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V32I32 = icmp ult <32 x i32> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp ult i64 undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V2I64 = icmp ult <2 x i64> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V4I64 = icmp ult <4 x i64> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %V8I64 = icmp ult <8 x i64> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %V16I64 = icmp ult <16 x i64> undef, undef
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSE41-LABEL: 'cmp_int_ult'
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp ult i8 undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16I8 = icmp ult <16 x i8> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V32I8 = icmp ult <32 x i8> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V64I8 = icmp ult <64 x i8> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V128I8 = icmp ult <128 x i8> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp ult i16 undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8I16 = icmp ult <8 x i16> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V16I16 = icmp ult <16 x i16> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V32I16 = icmp ult <32 x i16> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V64I16 = icmp ult <64 x i16> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp ult i32 undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4I32 = icmp ult <4 x i32> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V8I32 = icmp ult <8 x i32> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V16I32 = icmp ult <16 x i32> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V32I32 = icmp ult <32 x i32> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp ult i64 undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V2I64 = icmp ult <2 x i64> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V4I64 = icmp ult <4 x i64> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %V8I64 = icmp ult <8 x i64> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %V16I64 = icmp ult <16 x i64> undef, undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSE42-LABEL: 'cmp_int_ult'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp ult i8 undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16I8 = icmp ult <16 x i8> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V32I8 = icmp ult <32 x i8> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V64I8 = icmp ult <64 x i8> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V128I8 = icmp ult <128 x i8> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp ult i16 undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8I16 = icmp ult <8 x i16> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V16I16 = icmp ult <16 x i16> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V32I16 = icmp ult <32 x i16> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V64I16 = icmp ult <64 x i16> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp ult i32 undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4I32 = icmp ult <4 x i32> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V8I32 = icmp ult <8 x i32> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V16I32 = icmp ult <16 x i32> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V32I32 = icmp ult <32 x i32> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp ult i64 undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2I64 = icmp ult <2 x i64> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4I64 = icmp ult <4 x i64> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V8I64 = icmp ult <8 x i64> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V16I64 = icmp ult <16 x i64> undef, undef
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX1-LABEL: 'cmp_int_ult'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp ult i8 undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16I8 = icmp ult <16 x i8> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V32I8 = icmp ult <32 x i8> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V64I8 = icmp ult <64 x i8> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V128I8 = icmp ult <128 x i8> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp ult i16 undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8I16 = icmp ult <8 x i16> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V16I16 = icmp ult <16 x i16> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V32I16 = icmp ult <32 x i16> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V64I16 = icmp ult <64 x i16> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp ult i32 undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4I32 = icmp ult <4 x i32> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V8I32 = icmp ult <8 x i32> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V16I32 = icmp ult <16 x i32> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V32I32 = icmp ult <32 x i32> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp ult i64 undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2I64 = icmp ult <2 x i64> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4I64 = icmp ult <4 x i64> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V8I64 = icmp ult <8 x i64> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V16I64 = icmp ult <16 x i64> undef, undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX2-LABEL: 'cmp_int_ult'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp ult i8 undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16I8 = icmp ult <16 x i8> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32I8 = icmp ult <32 x i8> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V64I8 = icmp ult <64 x i8> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V128I8 = icmp ult <128 x i8> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp ult i16 undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8I16 = icmp ult <8 x i16> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16I16 = icmp ult <16 x i16> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V32I16 = icmp ult <32 x i16> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V64I16 = icmp ult <64 x i16> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp ult i32 undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4I32 = icmp ult <4 x i32> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8I32 = icmp ult <8 x i32> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V16I32 = icmp ult <16 x i32> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V32I32 = icmp ult <32 x i32> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp ult i64 undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2I64 = icmp ult <2 x i64> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4I64 = icmp ult <4 x i64> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V8I64 = icmp ult <8 x i64> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V16I64 = icmp ult <16 x i64> undef, undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX512F-LABEL: 'cmp_int_ult'
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp ult i8 undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16I8 = icmp ult <16 x i8> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32I8 = icmp ult <32 x i8> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V64I8 = icmp ult <64 x i8> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V128I8 = icmp ult <128 x i8> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp ult i16 undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8I16 = icmp ult <8 x i16> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16I16 = icmp ult <16 x i16> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V32I16 = icmp ult <32 x i16> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V64I16 = icmp ult <64 x i16> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp ult i32 undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = icmp ult <4 x i32> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = icmp ult <8 x i32> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I32 = icmp ult <16 x i32> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I32 = icmp ult <32 x i32> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp ult i64 undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = icmp ult <2 x i64> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = icmp ult <4 x i64> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = icmp ult <8 x i64> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I64 = icmp ult <16 x i64> undef, undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX512BW-LABEL: 'cmp_int_ult'
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp ult i8 undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = icmp ult <16 x i8> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32I8 = icmp ult <32 x i8> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64I8 = icmp ult <64 x i8> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V128I8 = icmp ult <128 x i8> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp ult i16 undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = icmp ult <8 x i16> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = icmp ult <16 x i16> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = icmp ult <32 x i16> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64I16 = icmp ult <64 x i16> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp ult i32 undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = icmp ult <4 x i32> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = icmp ult <8 x i32> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I32 = icmp ult <16 x i32> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I32 = icmp ult <32 x i32> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp ult i64 undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = icmp ult <2 x i64> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = icmp ult <4 x i64> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = icmp ult <8 x i64> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I64 = icmp ult <16 x i64> undef, undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; XOPAVX1-LABEL: 'cmp_int_ult'
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp ult i8 undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = icmp ult <16 x i8> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I8 = icmp ult <32 x i8> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I8 = icmp ult <64 x i8> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V128I8 = icmp ult <128 x i8> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp ult i16 undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = icmp ult <8 x i16> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I16 = icmp ult <16 x i16> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I16 = icmp ult <32 x i16> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V64I16 = icmp ult <64 x i16> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp ult i32 undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = icmp ult <4 x i32> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I32 = icmp ult <8 x i32> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I32 = icmp ult <16 x i32> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V32I32 = icmp ult <32 x i32> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp ult i64 undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = icmp ult <2 x i64> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4I64 = icmp ult <4 x i64> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8I64 = icmp ult <8 x i64> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16I64 = icmp ult <16 x i64> undef, undef
+; XOPAVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; XOPAVX2-LABEL: 'cmp_int_ult'
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp ult i8 undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = icmp ult <16 x i8> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32I8 = icmp ult <32 x i8> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V64I8 = icmp ult <64 x i8> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V128I8 = icmp ult <128 x i8> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp ult i16 undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = icmp ult <8 x i16> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16I16 = icmp ult <16 x i16> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V32I16 = icmp ult <32 x i16> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V64I16 = icmp ult <64 x i16> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp ult i32 undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = icmp ult <4 x i32> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8I32 = icmp ult <8 x i32> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V16I32 = icmp ult <16 x i32> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V32I32 = icmp ult <32 x i32> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp ult i64 undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = icmp ult <2 x i64> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4I64 = icmp ult <4 x i64> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V8I64 = icmp ult <8 x i64> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V16I64 = icmp ult <16 x i64> undef, undef
+; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; BTVER2-LABEL: 'cmp_int_ult'
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp ult i8 undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16I8 = icmp ult <16 x i8> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V32I8 = icmp ult <32 x i8> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V64I8 = icmp ult <64 x i8> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V128I8 = icmp ult <128 x i8> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp ult i16 undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8I16 = icmp ult <8 x i16> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V16I16 = icmp ult <16 x i16> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V32I16 = icmp ult <32 x i16> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V64I16 = icmp ult <64 x i16> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp ult i32 undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4I32 = icmp ult <4 x i32> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V8I32 = icmp ult <8 x i32> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V16I32 = icmp ult <16 x i32> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V32I32 = icmp ult <32 x i32> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp ult i64 undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2I64 = icmp ult <2 x i64> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4I64 = icmp ult <4 x i64> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V8I64 = icmp ult <8 x i64> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V16I64 = icmp ult <16 x i64> undef, undef
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+  %I8 = icmp ult i8 undef, undef
+  %V16I8 = icmp ult <16 x i8> undef, undef
+  %V32I8 = icmp ult <32 x i8> undef, undef
+  %V64I8 = icmp ult <64 x i8> undef, undef
+  %V128I8 = icmp ult <128 x i8> undef, undef
+
+  %I16 = icmp ult i16 undef, undef
+  %V8I16  = icmp ult <8 x i16> undef, undef
+  %V16I16 = icmp ult <16 x i16> undef, undef
+  %V32I16 = icmp ult <32 x i16> undef, undef
+  %V64I16 = icmp ult <64 x i16> undef, undef
+
+  %I32 = icmp ult i32 undef, undef
+  %V4I32 = icmp ult <4 x i32> undef, undef
+  %V8I32 = icmp ult <8 x i32> undef, undef
+  %V16I32 = icmp ult <16 x i32> undef, undef
+  %V32I32 = icmp ult <32 x i32> undef, undef
+
+  %I64 = icmp ult i64 undef, undef
+  %V2I64 = icmp ult <2 x i64> undef, undef
+  %V4I64 = icmp ult <4 x i64> undef, undef
+  %V8I64 = icmp ult <8 x i64> undef, undef
+  %V16I64 = icmp ult <16 x i64> undef, undef
+
+  ret i32 undef
+}
+
diff --git a/test/Analysis/CostModel/X86/masked-intrinsic-cost-widen.ll b/test/Analysis/CostModel/X86/masked-intrinsic-cost-widen.ll
index 7ca1c4ec6df0e701b3e3b8c1f49db25ba8e01f06..a59f36b21db17673b03d71551c433d0e86347d23 100644
--- a/test/Analysis/CostModel/X86/masked-intrinsic-cost-widen.ll
+++ b/test/Analysis/CostModel/X86/masked-intrinsic-cost-widen.ll
@@ -1,30 +1,34 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
-; RUN: opt < %s -S -x86-experimental-vector-widening-legalization -mtriple=x86_64-apple-darwin -mcpu=core-avx2 -cost-model -analyze | FileCheck %s --check-prefix=AVX2
-; RUN: opt < %s -S -x86-experimental-vector-widening-legalization -mtriple=x86_64-apple-darwin -mcpu=skylake -cost-model -analyze | FileCheck %s --check-prefix=SKL
-; RUN: opt < %s -S -x86-experimental-vector-widening-legalization -mtriple=x86_64-apple-darwin -mcpu=knl -cost-model -analyze | FileCheck %s --check-prefix=KNL
-; RUN: opt < %s -S -x86-experimental-vector-widening-legalization -mtriple=x86_64-apple-darwin -mcpu=skx -cost-model -analyze | FileCheck %s --check-prefix=SKX
+; RUN: opt < %s -S -x86-experimental-vector-widening-legalization -mtriple=x86_64-apple-darwin -mattr=+sse2 -cost-model -analyze | FileCheck %s --check-prefixes=CHECK,SSE,SSE2
+; RUN: opt < %s -S -x86-experimental-vector-widening-legalization -mtriple=x86_64-apple-darwin -mattr=+sse4.2 -cost-model -analyze | FileCheck %s --check-prefixes=CHECK,SSE,SSE42
+; RUN: opt < %s -S -x86-experimental-vector-widening-legalization -mtriple=x86_64-apple-darwin -mattr=+avx -cost-model -analyze | FileCheck %s --check-prefixes=CHECK,AVX,AVX1
+; RUN: opt < %s -S -x86-experimental-vector-widening-legalization -mtriple=x86_64-apple-darwin -mattr=+avx2 -cost-model -analyze | FileCheck %s --check-prefixes=CHECK,AVX,AVX2
+;
+; RUN: opt < %s -S -x86-experimental-vector-widening-legalization -mtriple=x86_64-apple-darwin -mcpu=skylake -cost-model -analyze | FileCheck %s --check-prefixes=CHECK,AVX,SKL
+; RUN: opt < %s -S -x86-experimental-vector-widening-legalization -mtriple=x86_64-apple-darwin -mcpu=knl -cost-model -analyze | FileCheck %s --check-prefixes=CHECK,AVX512,KNL
+; RUN: opt < %s -S -x86-experimental-vector-widening-legalization -mtriple=x86_64-apple-darwin -mcpu=skx -cost-model -analyze | FileCheck %s --check-prefixes=CHECK,AVX512,SKX
 
 
 define <2 x double> @test1(<2 x i64> %trigger, <2 x double>* %addr, <2 x double> %dst) {
-; AVX2-LABEL: 'test1'
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask = icmp eq <2 x i64> %trigger, zeroinitializer
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %res = call <2 x double> @llvm.masked.load.v2f64.p0v2f64(<2 x double>* %addr, i32 4, <2 x i1> %mask, <2 x double> %dst)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x double> %res
-;
-; SKL-LABEL: 'test1'
-; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask = icmp eq <2 x i64> %trigger, zeroinitializer
-; SKL-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %res = call <2 x double> @llvm.masked.load.v2f64.p0v2f64(<2 x double>* %addr, i32 4, <2 x i1> %mask, <2 x double> %dst)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x double> %res
-;
-; KNL-LABEL: 'test1'
-; KNL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask = icmp eq <2 x i64> %trigger, zeroinitializer
-; KNL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res = call <2 x double> @llvm.masked.load.v2f64.p0v2f64(<2 x double>* %addr, i32 4, <2 x i1> %mask, <2 x double> %dst)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x double> %res
-;
-; SKX-LABEL: 'test1'
-; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask = icmp eq <2 x i64> %trigger, zeroinitializer
-; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res = call <2 x double> @llvm.masked.load.v2f64.p0v2f64(<2 x double>* %addr, i32 4, <2 x i1> %mask, <2 x double> %dst)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x double> %res
+; SSE2-LABEL: 'test1'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %mask = icmp eq <2 x i64> %trigger, zeroinitializer
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %res = call <2 x double> @llvm.masked.load.v2f64.p0v2f64(<2 x double>* %addr, i32 4, <2 x i1> %mask, <2 x double> %dst)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x double> %res
+;
+; SSE42-LABEL: 'test1'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask = icmp eq <2 x i64> %trigger, zeroinitializer
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %res = call <2 x double> @llvm.masked.load.v2f64.p0v2f64(<2 x double>* %addr, i32 4, <2 x i1> %mask, <2 x double> %dst)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x double> %res
+;
+; AVX-LABEL: 'test1'
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask = icmp eq <2 x i64> %trigger, zeroinitializer
+; AVX-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %res = call <2 x double> @llvm.masked.load.v2f64.p0v2f64(<2 x double>* %addr, i32 4, <2 x i1> %mask, <2 x double> %dst)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x double> %res
+;
+; AVX512-LABEL: 'test1'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask = icmp eq <2 x i64> %trigger, zeroinitializer
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res = call <2 x double> @llvm.masked.load.v2f64.p0v2f64(<2 x double>* %addr, i32 4, <2 x i1> %mask, <2 x double> %dst)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x double> %res
 ;
   %mask = icmp eq <2 x i64> %trigger, zeroinitializer
   %res = call <2 x double> @llvm.masked.load.v2f64.p0v2f64(<2 x double>* %addr, i32 4, <2 x i1>%mask, <2 x double>%dst)
@@ -32,25 +36,20 @@ define <2 x double> @test1(<2 x i64> %trigger, <2 x double>* %addr, <2 x double>
 }
 
 define <4 x i32> @test2(<4 x i32> %trigger, <4 x i32>* %addr, <4 x i32> %dst) {
-; AVX2-LABEL: 'test2'
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask = icmp eq <4 x i32> %trigger, zeroinitializer
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %res = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %addr, i32 4, <4 x i1> %mask, <4 x i32> %dst)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i32> %res
+; SSE-LABEL: 'test2'
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask = icmp eq <4 x i32> %trigger, zeroinitializer
+; SSE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %res = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %addr, i32 4, <4 x i1> %mask, <4 x i32> %dst)
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i32> %res
 ;
-; SKL-LABEL: 'test2'
-; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask = icmp eq <4 x i32> %trigger, zeroinitializer
-; SKL-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %res = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %addr, i32 4, <4 x i1> %mask, <4 x i32> %dst)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i32> %res
+; AVX-LABEL: 'test2'
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask = icmp eq <4 x i32> %trigger, zeroinitializer
+; AVX-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %res = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %addr, i32 4, <4 x i1> %mask, <4 x i32> %dst)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i32> %res
 ;
-; KNL-LABEL: 'test2'
-; KNL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask = icmp eq <4 x i32> %trigger, zeroinitializer
-; KNL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %addr, i32 4, <4 x i1> %mask, <4 x i32> %dst)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i32> %res
-;
-; SKX-LABEL: 'test2'
-; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask = icmp eq <4 x i32> %trigger, zeroinitializer
-; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %addr, i32 4, <4 x i1> %mask, <4 x i32> %dst)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i32> %res
+; AVX512-LABEL: 'test2'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask = icmp eq <4 x i32> %trigger, zeroinitializer
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %addr, i32 4, <4 x i1> %mask, <4 x i32> %dst)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i32> %res
 ;
   %mask = icmp eq <4 x i32> %trigger, zeroinitializer
   %res = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %addr, i32 4, <4 x i1>%mask, <4 x i32>%dst)
@@ -58,25 +57,20 @@ define <4 x i32> @test2(<4 x i32> %trigger, <4 x i32>* %addr, <4 x i32> %dst) {
 }
 
 define void @test3(<4 x i32> %trigger, <4 x i32>* %addr, <4 x i32> %val) {
-; AVX2-LABEL: 'test3'
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask = icmp eq <4 x i32> %trigger, zeroinitializer
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %val, <4 x i32>* %addr, i32 4, <4 x i1> %mask)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; SSE-LABEL: 'test3'
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask = icmp eq <4 x i32> %trigger, zeroinitializer
+; SSE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %val, <4 x i32>* %addr, i32 4, <4 x i1> %mask)
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
-; SKL-LABEL: 'test3'
-; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask = icmp eq <4 x i32> %trigger, zeroinitializer
-; SKL-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %val, <4 x i32>* %addr, i32 4, <4 x i1> %mask)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; KNL-LABEL: 'test3'
-; KNL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask = icmp eq <4 x i32> %trigger, zeroinitializer
-; KNL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %val, <4 x i32>* %addr, i32 4, <4 x i1> %mask)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; AVX-LABEL: 'test3'
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask = icmp eq <4 x i32> %trigger, zeroinitializer
+; AVX-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %val, <4 x i32>* %addr, i32 4, <4 x i1> %mask)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
-; SKX-LABEL: 'test3'
-; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask = icmp eq <4 x i32> %trigger, zeroinitializer
-; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %val, <4 x i32>* %addr, i32 4, <4 x i1> %mask)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; AVX512-LABEL: 'test3'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask = icmp eq <4 x i32> %trigger, zeroinitializer
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %val, <4 x i32>* %addr, i32 4, <4 x i1> %mask)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %mask = icmp eq <4 x i32> %trigger, zeroinitializer
   call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32>%val, <4 x i32>* %addr, i32 4, <4 x i1>%mask)
@@ -84,6 +78,16 @@ define void @test3(<4 x i32> %trigger, <4 x i32>* %addr, <4 x i32> %val) {
 }
 
 define <8 x float> @test4(<8 x i32> %trigger, <8 x float>* %addr, <8 x float> %dst) {
+; SSE-LABEL: 'test4'
+; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %mask = icmp eq <8 x i32> %trigger, zeroinitializer
+; SSE-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %res = call <8 x float> @llvm.masked.load.v8f32.p0v8f32(<8 x float>* %addr, i32 4, <8 x i1> %mask, <8 x float> %dst)
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <8 x float> %res
+;
+; AVX1-LABEL: 'test4'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %mask = icmp eq <8 x i32> %trigger, zeroinitializer
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %res = call <8 x float> @llvm.masked.load.v8f32.p0v8f32(<8 x float>* %addr, i32 4, <8 x i1> %mask, <8 x float> %dst)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <8 x float> %res
+;
 ; AVX2-LABEL: 'test4'
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask = icmp eq <8 x i32> %trigger, zeroinitializer
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %res = call <8 x float> @llvm.masked.load.v8f32.p0v8f32(<8 x float>* %addr, i32 4, <8 x i1> %mask, <8 x float> %dst)
@@ -94,15 +98,10 @@ define <8 x float> @test4(<8 x i32> %trigger, <8 x float>* %addr, <8 x float> %d
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %res = call <8 x float> @llvm.masked.load.v8f32.p0v8f32(<8 x float>* %addr, i32 4, <8 x i1> %mask, <8 x float> %dst)
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <8 x float> %res
 ;
-; KNL-LABEL: 'test4'
-; KNL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask = icmp eq <8 x i32> %trigger, zeroinitializer
-; KNL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res = call <8 x float> @llvm.masked.load.v8f32.p0v8f32(<8 x float>* %addr, i32 4, <8 x i1> %mask, <8 x float> %dst)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <8 x float> %res
-;
-; SKX-LABEL: 'test4'
-; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask = icmp eq <8 x i32> %trigger, zeroinitializer
-; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res = call <8 x float> @llvm.masked.load.v8f32.p0v8f32(<8 x float>* %addr, i32 4, <8 x i1> %mask, <8 x float> %dst)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <8 x float> %res
+; AVX512-LABEL: 'test4'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask = icmp eq <8 x i32> %trigger, zeroinitializer
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res = call <8 x float> @llvm.masked.load.v8f32.p0v8f32(<8 x float>* %addr, i32 4, <8 x i1> %mask, <8 x float> %dst)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <8 x float> %res
 ;
   %mask = icmp eq <8 x i32> %trigger, zeroinitializer
   %res = call <8 x float> @llvm.masked.load.v8f32.p0v8f32(<8 x float>* %addr, i32 4, <8 x i1>%mask, <8 x float>%dst)
@@ -110,25 +109,20 @@ define <8 x float> @test4(<8 x i32> %trigger, <8 x float>* %addr, <8 x float> %d
 }
 
 define void @test5(<2 x i32> %trigger, <2 x float>* %addr, <2 x float> %val) {
-; AVX2-LABEL: 'test5'
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask = icmp eq <2 x i32> %trigger, zeroinitializer
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.store.v2f32.p0v2f32(<2 x float> %val, <2 x float>* %addr, i32 4, <2 x i1> %mask)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; SSE-LABEL: 'test5'
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask = icmp eq <2 x i32> %trigger, zeroinitializer
+; SSE-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: call void @llvm.masked.store.v2f32.p0v2f32(<2 x float> %val, <2 x float>* %addr, i32 4, <2 x i1> %mask)
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
-; SKL-LABEL: 'test5'
-; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask = icmp eq <2 x i32> %trigger, zeroinitializer
-; SKL-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.store.v2f32.p0v2f32(<2 x float> %val, <2 x float>* %addr, i32 4, <2 x i1> %mask)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; AVX-LABEL: 'test5'
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask = icmp eq <2 x i32> %trigger, zeroinitializer
+; AVX-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.store.v2f32.p0v2f32(<2 x float> %val, <2 x float>* %addr, i32 4, <2 x i1> %mask)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
-; KNL-LABEL: 'test5'
-; KNL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask = icmp eq <2 x i32> %trigger, zeroinitializer
-; KNL-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: call void @llvm.masked.store.v2f32.p0v2f32(<2 x float> %val, <2 x float>* %addr, i32 4, <2 x i1> %mask)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; SKX-LABEL: 'test5'
-; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask = icmp eq <2 x i32> %trigger, zeroinitializer
-; SKX-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: call void @llvm.masked.store.v2f32.p0v2f32(<2 x float> %val, <2 x float>* %addr, i32 4, <2 x i1> %mask)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; AVX512-LABEL: 'test5'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask = icmp eq <2 x i32> %trigger, zeroinitializer
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: call void @llvm.masked.store.v2f32.p0v2f32(<2 x float> %val, <2 x float>* %addr, i32 4, <2 x i1> %mask)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %mask = icmp eq <2 x i32> %trigger, zeroinitializer
   call void @llvm.masked.store.v2f32.p0v2f32(<2 x float>%val, <2 x float>* %addr, i32 4, <2 x i1>%mask)
@@ -136,25 +130,20 @@ define void @test5(<2 x i32> %trigger, <2 x float>* %addr, <2 x float> %val) {
 }
 
 define void @test6(<2 x i32> %trigger, <2 x i32>* %addr, <2 x i32> %val) {
-; AVX2-LABEL: 'test6'
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask = icmp eq <2 x i32> %trigger, zeroinitializer
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.store.v2i32.p0v2i32(<2 x i32> %val, <2 x i32>* %addr, i32 4, <2 x i1> %mask)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; SSE-LABEL: 'test6'
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask = icmp eq <2 x i32> %trigger, zeroinitializer
+; SSE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.store.v2i32.p0v2i32(<2 x i32> %val, <2 x i32>* %addr, i32 4, <2 x i1> %mask)
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
-; SKL-LABEL: 'test6'
-; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask = icmp eq <2 x i32> %trigger, zeroinitializer
-; SKL-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.store.v2i32.p0v2i32(<2 x i32> %val, <2 x i32>* %addr, i32 4, <2 x i1> %mask)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; AVX-LABEL: 'test6'
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask = icmp eq <2 x i32> %trigger, zeroinitializer
+; AVX-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.store.v2i32.p0v2i32(<2 x i32> %val, <2 x i32>* %addr, i32 4, <2 x i1> %mask)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
-; KNL-LABEL: 'test6'
-; KNL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask = icmp eq <2 x i32> %trigger, zeroinitializer
-; KNL-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: call void @llvm.masked.store.v2i32.p0v2i32(<2 x i32> %val, <2 x i32>* %addr, i32 4, <2 x i1> %mask)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; SKX-LABEL: 'test6'
-; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask = icmp eq <2 x i32> %trigger, zeroinitializer
-; SKX-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: call void @llvm.masked.store.v2i32.p0v2i32(<2 x i32> %val, <2 x i32>* %addr, i32 4, <2 x i1> %mask)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; AVX512-LABEL: 'test6'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask = icmp eq <2 x i32> %trigger, zeroinitializer
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: call void @llvm.masked.store.v2i32.p0v2i32(<2 x i32> %val, <2 x i32>* %addr, i32 4, <2 x i1> %mask)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %mask = icmp eq <2 x i32> %trigger, zeroinitializer
   call void @llvm.masked.store.v2i32.p0v2i32(<2 x i32>%val, <2 x i32>* %addr, i32 4, <2 x i1>%mask)
@@ -162,25 +151,20 @@ define void @test6(<2 x i32> %trigger, <2 x i32>* %addr, <2 x i32> %val) {
 }
 
 define <2 x float> @test7(<2 x i32> %trigger, <2 x float>* %addr, <2 x float> %dst) {
-; AVX2-LABEL: 'test7'
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask = icmp eq <2 x i32> %trigger, zeroinitializer
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %res = call <2 x float> @llvm.masked.load.v2f32.p0v2f32(<2 x float>* %addr, i32 4, <2 x i1> %mask, <2 x float> %dst)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x float> %res
-;
-; SKL-LABEL: 'test7'
-; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask = icmp eq <2 x i32> %trigger, zeroinitializer
-; SKL-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %res = call <2 x float> @llvm.masked.load.v2f32.p0v2f32(<2 x float>* %addr, i32 4, <2 x i1> %mask, <2 x float> %dst)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x float> %res
-;
-; KNL-LABEL: 'test7'
-; KNL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask = icmp eq <2 x i32> %trigger, zeroinitializer
-; KNL-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %res = call <2 x float> @llvm.masked.load.v2f32.p0v2f32(<2 x float>* %addr, i32 4, <2 x i1> %mask, <2 x float> %dst)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x float> %res
-;
-; SKX-LABEL: 'test7'
-; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask = icmp eq <2 x i32> %trigger, zeroinitializer
-; SKX-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %res = call <2 x float> @llvm.masked.load.v2f32.p0v2f32(<2 x float>* %addr, i32 4, <2 x i1> %mask, <2 x float> %dst)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x float> %res
+; SSE-LABEL: 'test7'
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask = icmp eq <2 x i32> %trigger, zeroinitializer
+; SSE-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %res = call <2 x float> @llvm.masked.load.v2f32.p0v2f32(<2 x float>* %addr, i32 4, <2 x i1> %mask, <2 x float> %dst)
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x float> %res
+;
+; AVX-LABEL: 'test7'
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask = icmp eq <2 x i32> %trigger, zeroinitializer
+; AVX-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %res = call <2 x float> @llvm.masked.load.v2f32.p0v2f32(<2 x float>* %addr, i32 4, <2 x i1> %mask, <2 x float> %dst)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x float> %res
+;
+; AVX512-LABEL: 'test7'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask = icmp eq <2 x i32> %trigger, zeroinitializer
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %res = call <2 x float> @llvm.masked.load.v2f32.p0v2f32(<2 x float>* %addr, i32 4, <2 x i1> %mask, <2 x float> %dst)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x float> %res
 ;
   %mask = icmp eq <2 x i32> %trigger, zeroinitializer
   %res = call <2 x float> @llvm.masked.load.v2f32.p0v2f32(<2 x float>* %addr, i32 4, <2 x i1>%mask, <2 x float>%dst)
@@ -188,25 +172,20 @@ define <2 x float> @test7(<2 x i32> %trigger, <2 x float>* %addr, <2 x float> %d
 }
 
 define <2 x i32> @test8(<2 x i32> %trigger, <2 x i32>* %addr, <2 x i32> %dst) {
-; AVX2-LABEL: 'test8'
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask = icmp eq <2 x i32> %trigger, zeroinitializer
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %res = call <2 x i32> @llvm.masked.load.v2i32.p0v2i32(<2 x i32>* %addr, i32 4, <2 x i1> %mask, <2 x i32> %dst)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x i32> %res
-;
-; SKL-LABEL: 'test8'
-; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask = icmp eq <2 x i32> %trigger, zeroinitializer
-; SKL-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %res = call <2 x i32> @llvm.masked.load.v2i32.p0v2i32(<2 x i32>* %addr, i32 4, <2 x i1> %mask, <2 x i32> %dst)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x i32> %res
-;
-; KNL-LABEL: 'test8'
-; KNL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask = icmp eq <2 x i32> %trigger, zeroinitializer
-; KNL-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %res = call <2 x i32> @llvm.masked.load.v2i32.p0v2i32(<2 x i32>* %addr, i32 4, <2 x i1> %mask, <2 x i32> %dst)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x i32> %res
-;
-; SKX-LABEL: 'test8'
-; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask = icmp eq <2 x i32> %trigger, zeroinitializer
-; SKX-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %res = call <2 x i32> @llvm.masked.load.v2i32.p0v2i32(<2 x i32>* %addr, i32 4, <2 x i1> %mask, <2 x i32> %dst)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x i32> %res
+; SSE-LABEL: 'test8'
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask = icmp eq <2 x i32> %trigger, zeroinitializer
+; SSE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %res = call <2 x i32> @llvm.masked.load.v2i32.p0v2i32(<2 x i32>* %addr, i32 4, <2 x i1> %mask, <2 x i32> %dst)
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x i32> %res
+;
+; AVX-LABEL: 'test8'
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask = icmp eq <2 x i32> %trigger, zeroinitializer
+; AVX-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %res = call <2 x i32> @llvm.masked.load.v2i32.p0v2i32(<2 x i32>* %addr, i32 4, <2 x i1> %mask, <2 x i32> %dst)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x i32> %res
+;
+; AVX512-LABEL: 'test8'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask = icmp eq <2 x i32> %trigger, zeroinitializer
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %res = call <2 x i32> @llvm.masked.load.v2i32.p0v2i32(<2 x i32>* %addr, i32 4, <2 x i1> %mask, <2 x i32> %dst)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x i32> %res
 ;
   %mask = icmp eq <2 x i32> %trigger, zeroinitializer
   %res = call <2 x i32> @llvm.masked.load.v2i32.p0v2i32(<2 x i32>* %addr, i32 4, <2 x i1>%mask, <2 x i32>%dst)
@@ -214,6 +193,14 @@ define <2 x i32> @test8(<2 x i32> %trigger, <2 x i32>* %addr, <2 x i32> %dst) {
 }
 
 define <2 x double> @test_gather_2f64(<2 x double*> %ptrs, <2 x i1> %mask, <2 x double> %src0)  {
+; SSE-LABEL: 'test_gather_2f64'
+; SSE-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %res = call <2 x double> @llvm.masked.gather.v2f64.v2p0f64(<2 x double*> %ptrs, i32 4, <2 x i1> %mask, <2 x double> %src0)
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x double> %res
+;
+; AVX1-LABEL: 'test_gather_2f64'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %res = call <2 x double> @llvm.masked.gather.v2f64.v2p0f64(<2 x double*> %ptrs, i32 4, <2 x i1> %mask, <2 x double> %src0)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x double> %res
+;
 ; AVX2-LABEL: 'test_gather_2f64'
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %res = call <2 x double> @llvm.masked.gather.v2f64.v2p0f64(<2 x double*> %ptrs, i32 4, <2 x i1> %mask, <2 x double> %src0)
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x double> %res
@@ -222,13 +209,9 @@ define <2 x double> @test_gather_2f64(<2 x double*> %ptrs, <2 x i1> %mask, <2 x
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %res = call <2 x double> @llvm.masked.gather.v2f64.v2p0f64(<2 x double*> %ptrs, i32 4, <2 x i1> %mask, <2 x double> %src0)
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x double> %res
 ;
-; KNL-LABEL: 'test_gather_2f64'
-; KNL-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %res = call <2 x double> @llvm.masked.gather.v2f64.v2p0f64(<2 x double*> %ptrs, i32 4, <2 x i1> %mask, <2 x double> %src0)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x double> %res
-;
-; SKX-LABEL: 'test_gather_2f64'
-; SKX-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %res = call <2 x double> @llvm.masked.gather.v2f64.v2p0f64(<2 x double*> %ptrs, i32 4, <2 x i1> %mask, <2 x double> %src0)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x double> %res
+; AVX512-LABEL: 'test_gather_2f64'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %res = call <2 x double> @llvm.masked.gather.v2f64.v2p0f64(<2 x double*> %ptrs, i32 4, <2 x i1> %mask, <2 x double> %src0)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x double> %res
 ;
 
 
@@ -240,6 +223,14 @@ define <2 x double> @test_gather_2f64(<2 x double*> %ptrs, <2 x i1> %mask, <2 x
 declare <2 x double> @llvm.masked.gather.v2f64.v2p0f64(<2 x double*> %ptrs, i32, <2 x i1> %mask, <2 x double> %src0)
 
 define <4 x i32> @test_gather_4i32(<4 x i32*> %ptrs, <4 x i1> %mask, <4 x i32> %src0)  {
+; SSE-LABEL: 'test_gather_4i32'
+; SSE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %res = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> %ptrs, i32 4, <4 x i1> %mask, <4 x i32> %src0)
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i32> %res
+;
+; AVX1-LABEL: 'test_gather_4i32'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %res = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> %ptrs, i32 4, <4 x i1> %mask, <4 x i32> %src0)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i32> %res
+;
 ; AVX2-LABEL: 'test_gather_4i32'
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %res = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> %ptrs, i32 4, <4 x i1> %mask, <4 x i32> %src0)
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i32> %res
@@ -265,6 +256,14 @@ define <4 x i32> @test_gather_4i32(<4 x i32*> %ptrs, <4 x i1> %mask, <4 x i32> %
 }
 
 define <4 x i32> @test_gather_4i32_const_mask(<4 x i32*> %ptrs, <4 x i32> %src0)  {
+; SSE-LABEL: 'test_gather_4i32_const_mask'
+; SSE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %res = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> %ptrs, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> %src0)
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i32> %res
+;
+; AVX1-LABEL: 'test_gather_4i32_const_mask'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %res = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> %ptrs, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> %src0)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i32> %res
+;
 ; AVX2-LABEL: 'test_gather_4i32_const_mask'
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %res = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> %ptrs, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> %src0)
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i32> %res
@@ -291,6 +290,24 @@ define <4 x i32> @test_gather_4i32_const_mask(<4 x i32*> %ptrs, <4 x i32> %src0)
 declare <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> %ptrs, i32, <4 x i1> %mask, <4 x i32> %src0)
 
 define <16 x float> @test_gather_16f32_const_mask(float* %base, <16 x i32> %ind) {
+; SSE2-LABEL: 'test_gather_16f32_const_mask'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %sext_ind = sext <16 x i32> %ind to <16 x i64>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.v = getelementptr float, float* %base, <16 x i64> %sext_ind
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> %gep.v, i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x float> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x float> %res
+;
+; SSE42-LABEL: 'test_gather_16f32_const_mask'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %sext_ind = sext <16 x i32> %ind to <16 x i64>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.v = getelementptr float, float* %base, <16 x i64> %sext_ind
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> %gep.v, i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x float> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x float> %res
+;
+; AVX1-LABEL: 'test_gather_16f32_const_mask'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %sext_ind = sext <16 x i32> %ind to <16 x i64>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.v = getelementptr float, float* %base, <16 x i64> %sext_ind
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> %gep.v, i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x float> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x float> %res
+;
 ; AVX2-LABEL: 'test_gather_16f32_const_mask'
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %sext_ind = sext <16 x i32> %ind to <16 x i64>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.v = getelementptr float, float* %base, <16 x i64> %sext_ind
@@ -303,17 +320,11 @@ define <16 x float> @test_gather_16f32_const_mask(float* %base, <16 x i32> %ind)
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> %gep.v, i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x float> undef)
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x float> %res
 ;
-; KNL-LABEL: 'test_gather_16f32_const_mask'
-; KNL-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %sext_ind = sext <16 x i32> %ind to <16 x i64>
-; KNL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.v = getelementptr float, float* %base, <16 x i64> %sext_ind
-; KNL-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> %gep.v, i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x float> undef)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x float> %res
-;
-; SKX-LABEL: 'test_gather_16f32_const_mask'
-; SKX-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %sext_ind = sext <16 x i32> %ind to <16 x i64>
-; SKX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.v = getelementptr float, float* %base, <16 x i64> %sext_ind
-; SKX-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> %gep.v, i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x float> undef)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x float> %res
+; AVX512-LABEL: 'test_gather_16f32_const_mask'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %sext_ind = sext <16 x i32> %ind to <16 x i64>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.v = getelementptr float, float* %base, <16 x i64> %sext_ind
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> %gep.v, i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x float> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x float> %res
 ;
   %sext_ind = sext <16 x i32> %ind to <16 x i64>
   %gep.v = getelementptr float, float* %base, <16 x i64> %sext_ind
@@ -323,6 +334,24 @@ define <16 x float> @test_gather_16f32_const_mask(float* %base, <16 x i32> %ind)
 }
 
 define <16 x float> @test_gather_16f32_var_mask(float* %base, <16 x i32> %ind, <16 x i1>%mask) {
+; SSE2-LABEL: 'test_gather_16f32_var_mask'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %sext_ind = sext <16 x i32> %ind to <16 x i64>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.v = getelementptr float, float* %base, <16 x i64> %sext_ind
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 60 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> %gep.v, i32 4, <16 x i1> %mask, <16 x float> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x float> %res
+;
+; SSE42-LABEL: 'test_gather_16f32_var_mask'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %sext_ind = sext <16 x i32> %ind to <16 x i64>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.v = getelementptr float, float* %base, <16 x i64> %sext_ind
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 60 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> %gep.v, i32 4, <16 x i1> %mask, <16 x float> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x float> %res
+;
+; AVX1-LABEL: 'test_gather_16f32_var_mask'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %sext_ind = sext <16 x i32> %ind to <16 x i64>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.v = getelementptr float, float* %base, <16 x i64> %sext_ind
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 62 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> %gep.v, i32 4, <16 x i1> %mask, <16 x float> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x float> %res
+;
 ; AVX2-LABEL: 'test_gather_16f32_var_mask'
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %sext_ind = sext <16 x i32> %ind to <16 x i64>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.v = getelementptr float, float* %base, <16 x i64> %sext_ind
@@ -335,17 +364,11 @@ define <16 x float> @test_gather_16f32_var_mask(float* %base, <16 x i32> %ind, <
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> %gep.v, i32 4, <16 x i1> %mask, <16 x float> undef)
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x float> %res
 ;
-; KNL-LABEL: 'test_gather_16f32_var_mask'
-; KNL-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %sext_ind = sext <16 x i32> %ind to <16 x i64>
-; KNL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.v = getelementptr float, float* %base, <16 x i64> %sext_ind
-; KNL-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> %gep.v, i32 4, <16 x i1> %mask, <16 x float> undef)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x float> %res
-;
-; SKX-LABEL: 'test_gather_16f32_var_mask'
-; SKX-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %sext_ind = sext <16 x i32> %ind to <16 x i64>
-; SKX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.v = getelementptr float, float* %base, <16 x i64> %sext_ind
-; SKX-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> %gep.v, i32 4, <16 x i1> %mask, <16 x float> undef)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x float> %res
+; AVX512-LABEL: 'test_gather_16f32_var_mask'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %sext_ind = sext <16 x i32> %ind to <16 x i64>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.v = getelementptr float, float* %base, <16 x i64> %sext_ind
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> %gep.v, i32 4, <16 x i1> %mask, <16 x float> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x float> %res
 ;
   %sext_ind = sext <16 x i32> %ind to <16 x i64>
   %gep.v = getelementptr float, float* %base, <16 x i64> %sext_ind
@@ -355,6 +378,24 @@ define <16 x float> @test_gather_16f32_var_mask(float* %base, <16 x i32> %ind, <
 }
 
 define <16 x float> @test_gather_16f32_ra_var_mask(<16 x float*> %ptrs, <16 x i32> %ind, <16 x i1>%mask) {
+; SSE2-LABEL: 'test_gather_16f32_ra_var_mask'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %sext_ind = sext <16 x i32> %ind to <16 x i64>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.v = getelementptr float, <16 x float*> %ptrs, <16 x i64> %sext_ind
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 60 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> %gep.v, i32 4, <16 x i1> %mask, <16 x float> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x float> %res
+;
+; SSE42-LABEL: 'test_gather_16f32_ra_var_mask'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %sext_ind = sext <16 x i32> %ind to <16 x i64>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.v = getelementptr float, <16 x float*> %ptrs, <16 x i64> %sext_ind
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 60 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> %gep.v, i32 4, <16 x i1> %mask, <16 x float> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x float> %res
+;
+; AVX1-LABEL: 'test_gather_16f32_ra_var_mask'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %sext_ind = sext <16 x i32> %ind to <16 x i64>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.v = getelementptr float, <16 x float*> %ptrs, <16 x i64> %sext_ind
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 62 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> %gep.v, i32 4, <16 x i1> %mask, <16 x float> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x float> %res
+;
 ; AVX2-LABEL: 'test_gather_16f32_ra_var_mask'
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %sext_ind = sext <16 x i32> %ind to <16 x i64>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.v = getelementptr float, <16 x float*> %ptrs, <16 x i64> %sext_ind
@@ -367,17 +408,11 @@ define <16 x float> @test_gather_16f32_ra_var_mask(<16 x float*> %ptrs, <16 x i3
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> %gep.v, i32 4, <16 x i1> %mask, <16 x float> undef)
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x float> %res
 ;
-; KNL-LABEL: 'test_gather_16f32_ra_var_mask'
-; KNL-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %sext_ind = sext <16 x i32> %ind to <16 x i64>
-; KNL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.v = getelementptr float, <16 x float*> %ptrs, <16 x i64> %sext_ind
-; KNL-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> %gep.v, i32 4, <16 x i1> %mask, <16 x float> undef)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x float> %res
-;
-; SKX-LABEL: 'test_gather_16f32_ra_var_mask'
-; SKX-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %sext_ind = sext <16 x i32> %ind to <16 x i64>
-; SKX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.v = getelementptr float, <16 x float*> %ptrs, <16 x i64> %sext_ind
-; SKX-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> %gep.v, i32 4, <16 x i1> %mask, <16 x float> undef)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x float> %res
+; AVX512-LABEL: 'test_gather_16f32_ra_var_mask'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %sext_ind = sext <16 x i32> %ind to <16 x i64>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.v = getelementptr float, <16 x float*> %ptrs, <16 x i64> %sext_ind
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> %gep.v, i32 4, <16 x i1> %mask, <16 x float> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x float> %res
 ;
   %sext_ind = sext <16 x i32> %ind to <16 x i64>
   %gep.v = getelementptr float, <16 x float*> %ptrs, <16 x i64> %sext_ind
@@ -387,6 +422,30 @@ define <16 x float> @test_gather_16f32_ra_var_mask(<16 x float*> %ptrs, <16 x i3
 }
 
 define <16 x float> @test_gather_16f32_const_mask2(float* %base, <16 x i32> %ind) {
+; SSE2-LABEL: 'test_gather_16f32_const_mask2'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %broadcast.splatinsert = insertelement <16 x float*> undef, float* %base, i32 0
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %broadcast.splat = shufflevector <16 x float*> %broadcast.splatinsert, <16 x float*> undef, <16 x i32> zeroinitializer
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %sext_ind = sext <16 x i32> %ind to <16 x i64>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.random = getelementptr float, <16 x float*> %broadcast.splat, <16 x i64> %sext_ind
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> %gep.random, i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x float> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x float> %res
+;
+; SSE42-LABEL: 'test_gather_16f32_const_mask2'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %broadcast.splatinsert = insertelement <16 x float*> undef, float* %base, i32 0
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %broadcast.splat = shufflevector <16 x float*> %broadcast.splatinsert, <16 x float*> undef, <16 x i32> zeroinitializer
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %sext_ind = sext <16 x i32> %ind to <16 x i64>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.random = getelementptr float, <16 x float*> %broadcast.splat, <16 x i64> %sext_ind
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> %gep.random, i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x float> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x float> %res
+;
+; AVX1-LABEL: 'test_gather_16f32_const_mask2'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %broadcast.splatinsert = insertelement <16 x float*> undef, float* %base, i32 0
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %broadcast.splat = shufflevector <16 x float*> %broadcast.splatinsert, <16 x float*> undef, <16 x i32> zeroinitializer
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %sext_ind = sext <16 x i32> %ind to <16 x i64>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.random = getelementptr float, <16 x float*> %broadcast.splat, <16 x i64> %sext_ind
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> %gep.random, i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x float> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x float> %res
+;
 ; AVX2-LABEL: 'test_gather_16f32_const_mask2'
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %broadcast.splatinsert = insertelement <16 x float*> undef, float* %base, i32 0
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %broadcast.splat = shufflevector <16 x float*> %broadcast.splatinsert, <16 x float*> undef, <16 x i32> zeroinitializer
@@ -403,21 +462,13 @@ define <16 x float> @test_gather_16f32_const_mask2(float* %base, <16 x i32> %ind
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> %gep.random, i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x float> undef)
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x float> %res
 ;
-; KNL-LABEL: 'test_gather_16f32_const_mask2'
-; KNL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %broadcast.splatinsert = insertelement <16 x float*> undef, float* %base, i32 0
-; KNL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %broadcast.splat = shufflevector <16 x float*> %broadcast.splatinsert, <16 x float*> undef, <16 x i32> zeroinitializer
-; KNL-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %sext_ind = sext <16 x i32> %ind to <16 x i64>
-; KNL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.random = getelementptr float, <16 x float*> %broadcast.splat, <16 x i64> %sext_ind
-; KNL-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> %gep.random, i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x float> undef)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x float> %res
-;
-; SKX-LABEL: 'test_gather_16f32_const_mask2'
-; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %broadcast.splatinsert = insertelement <16 x float*> undef, float* %base, i32 0
-; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %broadcast.splat = shufflevector <16 x float*> %broadcast.splatinsert, <16 x float*> undef, <16 x i32> zeroinitializer
-; SKX-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %sext_ind = sext <16 x i32> %ind to <16 x i64>
-; SKX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.random = getelementptr float, <16 x float*> %broadcast.splat, <16 x i64> %sext_ind
-; SKX-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> %gep.random, i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x float> undef)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x float> %res
+; AVX512-LABEL: 'test_gather_16f32_const_mask2'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %broadcast.splatinsert = insertelement <16 x float*> undef, float* %base, i32 0
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %broadcast.splat = shufflevector <16 x float*> %broadcast.splatinsert, <16 x float*> undef, <16 x i32> zeroinitializer
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %sext_ind = sext <16 x i32> %ind to <16 x i64>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.random = getelementptr float, <16 x float*> %broadcast.splat, <16 x i64> %sext_ind
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> %gep.random, i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x float> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x float> %res
 ;
   %broadcast.splatinsert = insertelement <16 x float*> undef, float* %base, i32 0
   %broadcast.splat = shufflevector <16 x float*> %broadcast.splatinsert, <16 x float*> undef, <16 x i32> zeroinitializer
@@ -430,6 +481,22 @@ define <16 x float> @test_gather_16f32_const_mask2(float* %base, <16 x i32> %ind
 }
 
 define void @test_scatter_16i32(i32* %base, <16 x i32> %ind, i16 %mask, <16 x i32>%val) {
+; SSE-LABEL: 'test_scatter_16i32'
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %broadcast.splatinsert = insertelement <16 x i32*> undef, i32* %base, i32 0
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %broadcast.splat = shufflevector <16 x i32*> %broadcast.splatinsert, <16 x i32*> undef, <16 x i32> zeroinitializer
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.random = getelementptr i32, <16 x i32*> %broadcast.splat, <16 x i32> %ind
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %imask = bitcast i16 %mask to <16 x i1>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: call void @llvm.masked.scatter.v16i32.v16p0i32(<16 x i32> %val, <16 x i32*> %gep.random, i32 4, <16 x i1> %imask)
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; AVX1-LABEL: 'test_scatter_16i32'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %broadcast.splatinsert = insertelement <16 x i32*> undef, i32* %base, i32 0
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %broadcast.splat = shufflevector <16 x i32*> %broadcast.splatinsert, <16 x i32*> undef, <16 x i32> zeroinitializer
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.random = getelementptr i32, <16 x i32*> %broadcast.splat, <16 x i32> %ind
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %imask = bitcast i16 %mask to <16 x i1>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: call void @llvm.masked.scatter.v16i32.v16p0i32(<16 x i32> %val, <16 x i32*> %gep.random, i32 4, <16 x i1> %imask)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
 ; AVX2-LABEL: 'test_scatter_16i32'
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %broadcast.splatinsert = insertelement <16 x i32*> undef, i32* %base, i32 0
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %broadcast.splat = shufflevector <16 x i32*> %broadcast.splatinsert, <16 x i32*> undef, <16 x i32> zeroinitializer
@@ -446,21 +513,13 @@ define void @test_scatter_16i32(i32* %base, <16 x i32> %ind, i16 %mask, <16 x i3
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: call void @llvm.masked.scatter.v16i32.v16p0i32(<16 x i32> %val, <16 x i32*> %gep.random, i32 4, <16 x i1> %imask)
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
-; KNL-LABEL: 'test_scatter_16i32'
-; KNL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %broadcast.splatinsert = insertelement <16 x i32*> undef, i32* %base, i32 0
-; KNL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %broadcast.splat = shufflevector <16 x i32*> %broadcast.splatinsert, <16 x i32*> undef, <16 x i32> zeroinitializer
-; KNL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.random = getelementptr i32, <16 x i32*> %broadcast.splat, <16 x i32> %ind
-; KNL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %imask = bitcast i16 %mask to <16 x i1>
-; KNL-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: call void @llvm.masked.scatter.v16i32.v16p0i32(<16 x i32> %val, <16 x i32*> %gep.random, i32 4, <16 x i1> %imask)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; SKX-LABEL: 'test_scatter_16i32'
-; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %broadcast.splatinsert = insertelement <16 x i32*> undef, i32* %base, i32 0
-; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %broadcast.splat = shufflevector <16 x i32*> %broadcast.splatinsert, <16 x i32*> undef, <16 x i32> zeroinitializer
-; SKX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.random = getelementptr i32, <16 x i32*> %broadcast.splat, <16 x i32> %ind
-; SKX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %imask = bitcast i16 %mask to <16 x i1>
-; SKX-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: call void @llvm.masked.scatter.v16i32.v16p0i32(<16 x i32> %val, <16 x i32*> %gep.random, i32 4, <16 x i1> %imask)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; AVX512-LABEL: 'test_scatter_16i32'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %broadcast.splatinsert = insertelement <16 x i32*> undef, i32* %base, i32 0
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %broadcast.splat = shufflevector <16 x i32*> %broadcast.splatinsert, <16 x i32*> undef, <16 x i32> zeroinitializer
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.random = getelementptr i32, <16 x i32*> %broadcast.splat, <16 x i32> %ind
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %imask = bitcast i16 %mask to <16 x i1>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: call void @llvm.masked.scatter.v16i32.v16p0i32(<16 x i32> %val, <16 x i32*> %gep.random, i32 4, <16 x i1> %imask)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %broadcast.splatinsert = insertelement <16 x i32*> undef, i32* %base, i32 0
   %broadcast.splat = shufflevector <16 x i32*> %broadcast.splatinsert, <16 x i32*> undef, <16 x i32> zeroinitializer
@@ -472,21 +531,17 @@ define void @test_scatter_16i32(i32* %base, <16 x i32> %ind, i16 %mask, <16 x i3
 }
 
 define void @test_scatter_8i32(<8 x i32>%a1, <8 x i32*> %ptr, <8 x i1>%mask) {
-; AVX2-LABEL: 'test_scatter_8i32'
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: call void @llvm.masked.scatter.v8i32.v8p0i32(<8 x i32> %a1, <8 x i32*> %ptr, i32 4, <8 x i1> %mask)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; SSE-LABEL: 'test_scatter_8i32'
+; SSE-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: call void @llvm.masked.scatter.v8i32.v8p0i32(<8 x i32> %a1, <8 x i32*> %ptr, i32 4, <8 x i1> %mask)
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
-; SKL-LABEL: 'test_scatter_8i32'
-; SKL-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: call void @llvm.masked.scatter.v8i32.v8p0i32(<8 x i32> %a1, <8 x i32*> %ptr, i32 4, <8 x i1> %mask)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; AVX-LABEL: 'test_scatter_8i32'
+; AVX-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: call void @llvm.masked.scatter.v8i32.v8p0i32(<8 x i32> %a1, <8 x i32*> %ptr, i32 4, <8 x i1> %mask)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
-; KNL-LABEL: 'test_scatter_8i32'
-; KNL-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: call void @llvm.masked.scatter.v8i32.v8p0i32(<8 x i32> %a1, <8 x i32*> %ptr, i32 4, <8 x i1> %mask)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; SKX-LABEL: 'test_scatter_8i32'
-; SKX-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: call void @llvm.masked.scatter.v8i32.v8p0i32(<8 x i32> %a1, <8 x i32*> %ptr, i32 4, <8 x i1> %mask)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; AVX512-LABEL: 'test_scatter_8i32'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: call void @llvm.masked.scatter.v8i32.v8p0i32(<8 x i32> %a1, <8 x i32*> %ptr, i32 4, <8 x i1> %mask)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 
   call void @llvm.masked.scatter.v8i32.v8p0i32(<8 x i32> %a1, <8 x i32*> %ptr, i32 4, <8 x i1> %mask)
@@ -496,13 +551,13 @@ define void @test_scatter_8i32(<8 x i32>%a1, <8 x i32*> %ptr, <8 x i1>%mask) {
 declare void @llvm.masked.scatter.v8i32.v8p0i32(<8 x i32> %a1, <8 x i32*> %ptr, i32, <8 x i1> %mask)
 
 define void @test_scatter_4i32(<4 x i32>%a1, <4 x i32*> %ptr, <4 x i1>%mask) {
-; AVX2-LABEL: 'test_scatter_4i32'
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.scatter.v4i32.v4p0i32(<4 x i32> %a1, <4 x i32*> %ptr, i32 4, <4 x i1> %mask)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; SSE-LABEL: 'test_scatter_4i32'
+; SSE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.scatter.v4i32.v4p0i32(<4 x i32> %a1, <4 x i32*> %ptr, i32 4, <4 x i1> %mask)
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
-; SKL-LABEL: 'test_scatter_4i32'
-; SKL-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.scatter.v4i32.v4p0i32(<4 x i32> %a1, <4 x i32*> %ptr, i32 4, <4 x i1> %mask)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; AVX-LABEL: 'test_scatter_4i32'
+; AVX-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.scatter.v4i32.v4p0i32(<4 x i32> %a1, <4 x i32*> %ptr, i32 4, <4 x i1> %mask)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; KNL-LABEL: 'test_scatter_4i32'
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.scatter.v4i32.v4p0i32(<4 x i32> %a1, <4 x i32*> %ptr, i32 4, <4 x i1> %mask)
@@ -517,6 +572,24 @@ define void @test_scatter_4i32(<4 x i32>%a1, <4 x i32*> %ptr, <4 x i1>%mask) {
 }
 
 define <4 x float> @test_gather_4f32(float* %ptr, <4 x i32> %ind, <4 x i1>%mask) {
+; SSE2-LABEL: 'test_gather_4f32'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %sext_ind = sext <4 x i32> %ind to <4 x i64>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.v = getelementptr float, float* %ptr, <4 x i64> %sext_ind
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %res = call <4 x float> @llvm.masked.gather.v4f32.v4p0f32(<4 x float*> %gep.v, i32 4, <4 x i1> %mask, <4 x float> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x float> %res
+;
+; SSE42-LABEL: 'test_gather_4f32'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %sext_ind = sext <4 x i32> %ind to <4 x i64>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.v = getelementptr float, float* %ptr, <4 x i64> %sext_ind
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %res = call <4 x float> @llvm.masked.gather.v4f32.v4p0f32(<4 x float*> %gep.v, i32 4, <4 x i1> %mask, <4 x float> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x float> %res
+;
+; AVX1-LABEL: 'test_gather_4f32'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %sext_ind = sext <4 x i32> %ind to <4 x i64>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.v = getelementptr float, float* %ptr, <4 x i64> %sext_ind
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %res = call <4 x float> @llvm.masked.gather.v4f32.v4p0f32(<4 x float*> %gep.v, i32 4, <4 x i1> %mask, <4 x float> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x float> %res
+;
 ; AVX2-LABEL: 'test_gather_4f32'
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sext_ind = sext <4 x i32> %ind to <4 x i64>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.v = getelementptr float, float* %ptr, <4 x i64> %sext_ind
@@ -549,6 +622,24 @@ define <4 x float> @test_gather_4f32(float* %ptr, <4 x i32> %ind, <4 x i1>%mask)
 }
 
 define <4 x float> @test_gather_4f32_const_mask(float* %ptr, <4 x i32> %ind) {
+; SSE2-LABEL: 'test_gather_4f32_const_mask'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %sext_ind = sext <4 x i32> %ind to <4 x i64>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.v = getelementptr float, float* %ptr, <4 x i64> %sext_ind
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %res = call <4 x float> @llvm.masked.gather.v4f32.v4p0f32(<4 x float*> %gep.v, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x float> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x float> %res
+;
+; SSE42-LABEL: 'test_gather_4f32_const_mask'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %sext_ind = sext <4 x i32> %ind to <4 x i64>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.v = getelementptr float, float* %ptr, <4 x i64> %sext_ind
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %res = call <4 x float> @llvm.masked.gather.v4f32.v4p0f32(<4 x float*> %gep.v, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x float> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x float> %res
+;
+; AVX1-LABEL: 'test_gather_4f32_const_mask'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %sext_ind = sext <4 x i32> %ind to <4 x i64>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.v = getelementptr float, float* %ptr, <4 x i64> %sext_ind
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %res = call <4 x float> @llvm.masked.gather.v4f32.v4p0f32(<4 x float*> %gep.v, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x float> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x float> %res
+;
 ; AVX2-LABEL: 'test_gather_4f32_const_mask'
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sext_ind = sext <4 x i32> %ind to <4 x i64>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.v = getelementptr float, float* %ptr, <4 x i64> %sext_ind
diff --git a/test/Analysis/CostModel/X86/masked-intrinsic-cost.ll b/test/Analysis/CostModel/X86/masked-intrinsic-cost.ll
index 5f2a51c7f938bd8e387cb0746c599cb992fb13a2..7845d1022972dda2b33866f602cfc2023df616d9 100644
--- a/test/Analysis/CostModel/X86/masked-intrinsic-cost.ll
+++ b/test/Analysis/CostModel/X86/masked-intrinsic-cost.ll
@@ -1,30 +1,34 @@
 ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
-; RUN: opt < %s -S -mtriple=x86_64-apple-darwin -mcpu=core-avx2 -cost-model -analyze | FileCheck %s --check-prefix=AVX2
-; RUN: opt < %s -S -mtriple=x86_64-apple-darwin -mcpu=skylake -cost-model -analyze | FileCheck %s --check-prefix=SKL
-; RUN: opt < %s -S -mtriple=x86_64-apple-darwin -mcpu=knl -cost-model -analyze | FileCheck %s --check-prefix=KNL
-; RUN: opt < %s -S -mtriple=x86_64-apple-darwin -mcpu=skx -cost-model -analyze | FileCheck %s --check-prefix=SKX
+; RUN: opt < %s -S -mtriple=x86_64-apple-darwin -mattr=+sse2 -cost-model -analyze | FileCheck %s --check-prefixes=CHECK,SSE,SSE2
+; RUN: opt < %s -S -mtriple=x86_64-apple-darwin -mattr=+sse4.2 -cost-model -analyze | FileCheck %s --check-prefixes=CHECK,SSE,SSE42
+; RUN: opt < %s -S -mtriple=x86_64-apple-darwin -mattr=+avx -cost-model -analyze | FileCheck %s --check-prefixes=CHECK,AVX,AVX1
+; RUN: opt < %s -S -mtriple=x86_64-apple-darwin -mattr=+avx2 -cost-model -analyze | FileCheck %s --check-prefixes=CHECK,AVX,AVX2
+;
+; RUN: opt < %s -S -mtriple=x86_64-apple-darwin -mcpu=skylake -cost-model -analyze | FileCheck %s --check-prefixes=CHECK,AVX,SKL
+; RUN: opt < %s -S -mtriple=x86_64-apple-darwin -mcpu=knl -cost-model -analyze | FileCheck %s --check-prefixes=CHECK,AVX512,KNL
+; RUN: opt < %s -S -mtriple=x86_64-apple-darwin -mcpu=skx -cost-model -analyze | FileCheck %s --check-prefixes=CHECK,AVX512,SKX
 
 
 define <2 x double> @test1(<2 x i64> %trigger, <2 x double>* %addr, <2 x double> %dst) {
-; AVX2-LABEL: 'test1'
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask = icmp eq <2 x i64> %trigger, zeroinitializer
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %res = call <2 x double> @llvm.masked.load.v2f64.p0v2f64(<2 x double>* %addr, i32 4, <2 x i1> %mask, <2 x double> %dst)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x double> %res
-;
-; SKL-LABEL: 'test1'
-; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask = icmp eq <2 x i64> %trigger, zeroinitializer
-; SKL-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %res = call <2 x double> @llvm.masked.load.v2f64.p0v2f64(<2 x double>* %addr, i32 4, <2 x i1> %mask, <2 x double> %dst)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x double> %res
-;
-; KNL-LABEL: 'test1'
-; KNL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask = icmp eq <2 x i64> %trigger, zeroinitializer
-; KNL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res = call <2 x double> @llvm.masked.load.v2f64.p0v2f64(<2 x double>* %addr, i32 4, <2 x i1> %mask, <2 x double> %dst)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x double> %res
-;
-; SKX-LABEL: 'test1'
-; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask = icmp eq <2 x i64> %trigger, zeroinitializer
-; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res = call <2 x double> @llvm.masked.load.v2f64.p0v2f64(<2 x double>* %addr, i32 4, <2 x i1> %mask, <2 x double> %dst)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x double> %res
+; SSE2-LABEL: 'test1'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %mask = icmp eq <2 x i64> %trigger, zeroinitializer
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %res = call <2 x double> @llvm.masked.load.v2f64.p0v2f64(<2 x double>* %addr, i32 4, <2 x i1> %mask, <2 x double> %dst)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x double> %res
+;
+; SSE42-LABEL: 'test1'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask = icmp eq <2 x i64> %trigger, zeroinitializer
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %res = call <2 x double> @llvm.masked.load.v2f64.p0v2f64(<2 x double>* %addr, i32 4, <2 x i1> %mask, <2 x double> %dst)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x double> %res
+;
+; AVX-LABEL: 'test1'
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask = icmp eq <2 x i64> %trigger, zeroinitializer
+; AVX-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %res = call <2 x double> @llvm.masked.load.v2f64.p0v2f64(<2 x double>* %addr, i32 4, <2 x i1> %mask, <2 x double> %dst)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x double> %res
+;
+; AVX512-LABEL: 'test1'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask = icmp eq <2 x i64> %trigger, zeroinitializer
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res = call <2 x double> @llvm.masked.load.v2f64.p0v2f64(<2 x double>* %addr, i32 4, <2 x i1> %mask, <2 x double> %dst)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x double> %res
 ;
   %mask = icmp eq <2 x i64> %trigger, zeroinitializer
   %res = call <2 x double> @llvm.masked.load.v2f64.p0v2f64(<2 x double>* %addr, i32 4, <2 x i1>%mask, <2 x double>%dst)
@@ -32,25 +36,20 @@ define <2 x double> @test1(<2 x i64> %trigger, <2 x double>* %addr, <2 x double>
 }
 
 define <4 x i32> @test2(<4 x i32> %trigger, <4 x i32>* %addr, <4 x i32> %dst) {
-; AVX2-LABEL: 'test2'
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask = icmp eq <4 x i32> %trigger, zeroinitializer
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %res = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %addr, i32 4, <4 x i1> %mask, <4 x i32> %dst)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i32> %res
+; SSE-LABEL: 'test2'
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask = icmp eq <4 x i32> %trigger, zeroinitializer
+; SSE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %res = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %addr, i32 4, <4 x i1> %mask, <4 x i32> %dst)
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i32> %res
 ;
-; SKL-LABEL: 'test2'
-; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask = icmp eq <4 x i32> %trigger, zeroinitializer
-; SKL-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %res = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %addr, i32 4, <4 x i1> %mask, <4 x i32> %dst)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i32> %res
+; AVX-LABEL: 'test2'
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask = icmp eq <4 x i32> %trigger, zeroinitializer
+; AVX-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %res = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %addr, i32 4, <4 x i1> %mask, <4 x i32> %dst)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i32> %res
 ;
-; KNL-LABEL: 'test2'
-; KNL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask = icmp eq <4 x i32> %trigger, zeroinitializer
-; KNL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %addr, i32 4, <4 x i1> %mask, <4 x i32> %dst)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i32> %res
-;
-; SKX-LABEL: 'test2'
-; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask = icmp eq <4 x i32> %trigger, zeroinitializer
-; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %addr, i32 4, <4 x i1> %mask, <4 x i32> %dst)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i32> %res
+; AVX512-LABEL: 'test2'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask = icmp eq <4 x i32> %trigger, zeroinitializer
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %addr, i32 4, <4 x i1> %mask, <4 x i32> %dst)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i32> %res
 ;
   %mask = icmp eq <4 x i32> %trigger, zeroinitializer
   %res = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %addr, i32 4, <4 x i1>%mask, <4 x i32>%dst)
@@ -58,25 +57,20 @@ define <4 x i32> @test2(<4 x i32> %trigger, <4 x i32>* %addr, <4 x i32> %dst) {
 }
 
 define void @test3(<4 x i32> %trigger, <4 x i32>* %addr, <4 x i32> %val) {
-; AVX2-LABEL: 'test3'
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask = icmp eq <4 x i32> %trigger, zeroinitializer
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %val, <4 x i32>* %addr, i32 4, <4 x i1> %mask)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; SSE-LABEL: 'test3'
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask = icmp eq <4 x i32> %trigger, zeroinitializer
+; SSE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %val, <4 x i32>* %addr, i32 4, <4 x i1> %mask)
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
-; SKL-LABEL: 'test3'
-; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask = icmp eq <4 x i32> %trigger, zeroinitializer
-; SKL-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %val, <4 x i32>* %addr, i32 4, <4 x i1> %mask)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; KNL-LABEL: 'test3'
-; KNL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask = icmp eq <4 x i32> %trigger, zeroinitializer
-; KNL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %val, <4 x i32>* %addr, i32 4, <4 x i1> %mask)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; AVX-LABEL: 'test3'
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask = icmp eq <4 x i32> %trigger, zeroinitializer
+; AVX-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %val, <4 x i32>* %addr, i32 4, <4 x i1> %mask)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
-; SKX-LABEL: 'test3'
-; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask = icmp eq <4 x i32> %trigger, zeroinitializer
-; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %val, <4 x i32>* %addr, i32 4, <4 x i1> %mask)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; AVX512-LABEL: 'test3'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask = icmp eq <4 x i32> %trigger, zeroinitializer
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %val, <4 x i32>* %addr, i32 4, <4 x i1> %mask)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %mask = icmp eq <4 x i32> %trigger, zeroinitializer
   call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32>%val, <4 x i32>* %addr, i32 4, <4 x i1>%mask)
@@ -84,6 +78,16 @@ define void @test3(<4 x i32> %trigger, <4 x i32>* %addr, <4 x i32> %val) {
 }
 
 define <8 x float> @test4(<8 x i32> %trigger, <8 x float>* %addr, <8 x float> %dst) {
+; SSE-LABEL: 'test4'
+; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %mask = icmp eq <8 x i32> %trigger, zeroinitializer
+; SSE-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %res = call <8 x float> @llvm.masked.load.v8f32.p0v8f32(<8 x float>* %addr, i32 4, <8 x i1> %mask, <8 x float> %dst)
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <8 x float> %res
+;
+; AVX1-LABEL: 'test4'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %mask = icmp eq <8 x i32> %trigger, zeroinitializer
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %res = call <8 x float> @llvm.masked.load.v8f32.p0v8f32(<8 x float>* %addr, i32 4, <8 x i1> %mask, <8 x float> %dst)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <8 x float> %res
+;
 ; AVX2-LABEL: 'test4'
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask = icmp eq <8 x i32> %trigger, zeroinitializer
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %res = call <8 x float> @llvm.masked.load.v8f32.p0v8f32(<8 x float>* %addr, i32 4, <8 x i1> %mask, <8 x float> %dst)
@@ -94,15 +98,10 @@ define <8 x float> @test4(<8 x i32> %trigger, <8 x float>* %addr, <8 x float> %d
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %res = call <8 x float> @llvm.masked.load.v8f32.p0v8f32(<8 x float>* %addr, i32 4, <8 x i1> %mask, <8 x float> %dst)
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <8 x float> %res
 ;
-; KNL-LABEL: 'test4'
-; KNL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask = icmp eq <8 x i32> %trigger, zeroinitializer
-; KNL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res = call <8 x float> @llvm.masked.load.v8f32.p0v8f32(<8 x float>* %addr, i32 4, <8 x i1> %mask, <8 x float> %dst)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <8 x float> %res
-;
-; SKX-LABEL: 'test4'
-; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask = icmp eq <8 x i32> %trigger, zeroinitializer
-; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res = call <8 x float> @llvm.masked.load.v8f32.p0v8f32(<8 x float>* %addr, i32 4, <8 x i1> %mask, <8 x float> %dst)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <8 x float> %res
+; AVX512-LABEL: 'test4'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask = icmp eq <8 x i32> %trigger, zeroinitializer
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %res = call <8 x float> @llvm.masked.load.v8f32.p0v8f32(<8 x float>* %addr, i32 4, <8 x i1> %mask, <8 x float> %dst)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <8 x float> %res
 ;
   %mask = icmp eq <8 x i32> %trigger, zeroinitializer
   %res = call <8 x float> @llvm.masked.load.v8f32.p0v8f32(<8 x float>* %addr, i32 4, <8 x i1>%mask, <8 x float>%dst)
@@ -110,25 +109,25 @@ define <8 x float> @test4(<8 x i32> %trigger, <8 x float>* %addr, <8 x float> %d
 }
 
 define void @test5(<2 x i32> %trigger, <2 x float>* %addr, <2 x float> %val) {
-; AVX2-LABEL: 'test5'
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask = icmp eq <2 x i32> %trigger, zeroinitializer
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.store.v2f32.p0v2f32(<2 x float> %val, <2 x float>* %addr, i32 4, <2 x i1> %mask)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; SKL-LABEL: 'test5'
-; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask = icmp eq <2 x i32> %trigger, zeroinitializer
-; SKL-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.store.v2f32.p0v2f32(<2 x float> %val, <2 x float>* %addr, i32 4, <2 x i1> %mask)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; KNL-LABEL: 'test5'
-; KNL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask = icmp eq <2 x i32> %trigger, zeroinitializer
-; KNL-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: call void @llvm.masked.store.v2f32.p0v2f32(<2 x float> %val, <2 x float>* %addr, i32 4, <2 x i1> %mask)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; SKX-LABEL: 'test5'
-; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask = icmp eq <2 x i32> %trigger, zeroinitializer
-; SKX-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: call void @llvm.masked.store.v2f32.p0v2f32(<2 x float> %val, <2 x float>* %addr, i32 4, <2 x i1> %mask)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; SSE2-LABEL: 'test5'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %mask = icmp eq <2 x i32> %trigger, zeroinitializer
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: call void @llvm.masked.store.v2f32.p0v2f32(<2 x float> %val, <2 x float>* %addr, i32 4, <2 x i1> %mask)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; SSE42-LABEL: 'test5'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask = icmp eq <2 x i32> %trigger, zeroinitializer
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: call void @llvm.masked.store.v2f32.p0v2f32(<2 x float> %val, <2 x float>* %addr, i32 4, <2 x i1> %mask)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; AVX-LABEL: 'test5'
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask = icmp eq <2 x i32> %trigger, zeroinitializer
+; AVX-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.store.v2f32.p0v2f32(<2 x float> %val, <2 x float>* %addr, i32 4, <2 x i1> %mask)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; AVX512-LABEL: 'test5'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask = icmp eq <2 x i32> %trigger, zeroinitializer
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: call void @llvm.masked.store.v2f32.p0v2f32(<2 x float> %val, <2 x float>* %addr, i32 4, <2 x i1> %mask)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %mask = icmp eq <2 x i32> %trigger, zeroinitializer
   call void @llvm.masked.store.v2f32.p0v2f32(<2 x float>%val, <2 x float>* %addr, i32 4, <2 x i1>%mask)
@@ -136,25 +135,25 @@ define void @test5(<2 x i32> %trigger, <2 x float>* %addr, <2 x float> %val) {
 }
 
 define void @test6(<2 x i32> %trigger, <2 x i32>* %addr, <2 x i32> %val) {
-; AVX2-LABEL: 'test6'
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask = icmp eq <2 x i32> %trigger, zeroinitializer
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: call void @llvm.masked.store.v2i32.p0v2i32(<2 x i32> %val, <2 x i32>* %addr, i32 4, <2 x i1> %mask)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; SKL-LABEL: 'test6'
-; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask = icmp eq <2 x i32> %trigger, zeroinitializer
-; SKL-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: call void @llvm.masked.store.v2i32.p0v2i32(<2 x i32> %val, <2 x i32>* %addr, i32 4, <2 x i1> %mask)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; KNL-LABEL: 'test6'
-; KNL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask = icmp eq <2 x i32> %trigger, zeroinitializer
-; KNL-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: call void @llvm.masked.store.v2i32.p0v2i32(<2 x i32> %val, <2 x i32>* %addr, i32 4, <2 x i1> %mask)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; SKX-LABEL: 'test6'
-; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask = icmp eq <2 x i32> %trigger, zeroinitializer
-; SKX-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: call void @llvm.masked.store.v2i32.p0v2i32(<2 x i32> %val, <2 x i32>* %addr, i32 4, <2 x i1> %mask)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; SSE2-LABEL: 'test6'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %mask = icmp eq <2 x i32> %trigger, zeroinitializer
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.store.v2i32.p0v2i32(<2 x i32> %val, <2 x i32>* %addr, i32 4, <2 x i1> %mask)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; SSE42-LABEL: 'test6'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask = icmp eq <2 x i32> %trigger, zeroinitializer
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: call void @llvm.masked.store.v2i32.p0v2i32(<2 x i32> %val, <2 x i32>* %addr, i32 4, <2 x i1> %mask)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; AVX-LABEL: 'test6'
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask = icmp eq <2 x i32> %trigger, zeroinitializer
+; AVX-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: call void @llvm.masked.store.v2i32.p0v2i32(<2 x i32> %val, <2 x i32>* %addr, i32 4, <2 x i1> %mask)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; AVX512-LABEL: 'test6'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask = icmp eq <2 x i32> %trigger, zeroinitializer
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: call void @llvm.masked.store.v2i32.p0v2i32(<2 x i32> %val, <2 x i32>* %addr, i32 4, <2 x i1> %mask)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %mask = icmp eq <2 x i32> %trigger, zeroinitializer
   call void @llvm.masked.store.v2i32.p0v2i32(<2 x i32>%val, <2 x i32>* %addr, i32 4, <2 x i1>%mask)
@@ -162,25 +161,25 @@ define void @test6(<2 x i32> %trigger, <2 x i32>* %addr, <2 x i32> %val) {
 }
 
 define <2 x float> @test7(<2 x i32> %trigger, <2 x float>* %addr, <2 x float> %dst) {
-; AVX2-LABEL: 'test7'
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask = icmp eq <2 x i32> %trigger, zeroinitializer
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %res = call <2 x float> @llvm.masked.load.v2f32.p0v2f32(<2 x float>* %addr, i32 4, <2 x i1> %mask, <2 x float> %dst)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x float> %res
-;
-; SKL-LABEL: 'test7'
-; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask = icmp eq <2 x i32> %trigger, zeroinitializer
-; SKL-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %res = call <2 x float> @llvm.masked.load.v2f32.p0v2f32(<2 x float>* %addr, i32 4, <2 x i1> %mask, <2 x float> %dst)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x float> %res
-;
-; KNL-LABEL: 'test7'
-; KNL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask = icmp eq <2 x i32> %trigger, zeroinitializer
-; KNL-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %res = call <2 x float> @llvm.masked.load.v2f32.p0v2f32(<2 x float>* %addr, i32 4, <2 x i1> %mask, <2 x float> %dst)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x float> %res
-;
-; SKX-LABEL: 'test7'
-; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask = icmp eq <2 x i32> %trigger, zeroinitializer
-; SKX-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %res = call <2 x float> @llvm.masked.load.v2f32.p0v2f32(<2 x float>* %addr, i32 4, <2 x i1> %mask, <2 x float> %dst)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x float> %res
+; SSE2-LABEL: 'test7'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %mask = icmp eq <2 x i32> %trigger, zeroinitializer
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %res = call <2 x float> @llvm.masked.load.v2f32.p0v2f32(<2 x float>* %addr, i32 4, <2 x i1> %mask, <2 x float> %dst)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x float> %res
+;
+; SSE42-LABEL: 'test7'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask = icmp eq <2 x i32> %trigger, zeroinitializer
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %res = call <2 x float> @llvm.masked.load.v2f32.p0v2f32(<2 x float>* %addr, i32 4, <2 x i1> %mask, <2 x float> %dst)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x float> %res
+;
+; AVX-LABEL: 'test7'
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask = icmp eq <2 x i32> %trigger, zeroinitializer
+; AVX-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %res = call <2 x float> @llvm.masked.load.v2f32.p0v2f32(<2 x float>* %addr, i32 4, <2 x i1> %mask, <2 x float> %dst)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x float> %res
+;
+; AVX512-LABEL: 'test7'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask = icmp eq <2 x i32> %trigger, zeroinitializer
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %res = call <2 x float> @llvm.masked.load.v2f32.p0v2f32(<2 x float>* %addr, i32 4, <2 x i1> %mask, <2 x float> %dst)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x float> %res
 ;
   %mask = icmp eq <2 x i32> %trigger, zeroinitializer
   %res = call <2 x float> @llvm.masked.load.v2f32.p0v2f32(<2 x float>* %addr, i32 4, <2 x i1>%mask, <2 x float>%dst)
@@ -188,25 +187,25 @@ define <2 x float> @test7(<2 x i32> %trigger, <2 x float>* %addr, <2 x float> %d
 }
 
 define <2 x i32> @test8(<2 x i32> %trigger, <2 x i32>* %addr, <2 x i32> %dst) {
-; AVX2-LABEL: 'test8'
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask = icmp eq <2 x i32> %trigger, zeroinitializer
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %res = call <2 x i32> @llvm.masked.load.v2i32.p0v2i32(<2 x i32>* %addr, i32 4, <2 x i1> %mask, <2 x i32> %dst)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x i32> %res
-;
-; SKL-LABEL: 'test8'
-; SKL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask = icmp eq <2 x i32> %trigger, zeroinitializer
-; SKL-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %res = call <2 x i32> @llvm.masked.load.v2i32.p0v2i32(<2 x i32>* %addr, i32 4, <2 x i1> %mask, <2 x i32> %dst)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x i32> %res
-;
-; KNL-LABEL: 'test8'
-; KNL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask = icmp eq <2 x i32> %trigger, zeroinitializer
-; KNL-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %res = call <2 x i32> @llvm.masked.load.v2i32.p0v2i32(<2 x i32>* %addr, i32 4, <2 x i1> %mask, <2 x i32> %dst)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x i32> %res
-;
-; SKX-LABEL: 'test8'
-; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask = icmp eq <2 x i32> %trigger, zeroinitializer
-; SKX-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %res = call <2 x i32> @llvm.masked.load.v2i32.p0v2i32(<2 x i32>* %addr, i32 4, <2 x i1> %mask, <2 x i32> %dst)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x i32> %res
+; SSE2-LABEL: 'test8'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %mask = icmp eq <2 x i32> %trigger, zeroinitializer
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %res = call <2 x i32> @llvm.masked.load.v2i32.p0v2i32(<2 x i32>* %addr, i32 4, <2 x i1> %mask, <2 x i32> %dst)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x i32> %res
+;
+; SSE42-LABEL: 'test8'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask = icmp eq <2 x i32> %trigger, zeroinitializer
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %res = call <2 x i32> @llvm.masked.load.v2i32.p0v2i32(<2 x i32>* %addr, i32 4, <2 x i1> %mask, <2 x i32> %dst)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x i32> %res
+;
+; AVX-LABEL: 'test8'
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask = icmp eq <2 x i32> %trigger, zeroinitializer
+; AVX-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %res = call <2 x i32> @llvm.masked.load.v2i32.p0v2i32(<2 x i32>* %addr, i32 4, <2 x i1> %mask, <2 x i32> %dst)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x i32> %res
+;
+; AVX512-LABEL: 'test8'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %mask = icmp eq <2 x i32> %trigger, zeroinitializer
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %res = call <2 x i32> @llvm.masked.load.v2i32.p0v2i32(<2 x i32>* %addr, i32 4, <2 x i1> %mask, <2 x i32> %dst)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x i32> %res
 ;
   %mask = icmp eq <2 x i32> %trigger, zeroinitializer
   %res = call <2 x i32> @llvm.masked.load.v2i32.p0v2i32(<2 x i32>* %addr, i32 4, <2 x i1>%mask, <2 x i32>%dst)
@@ -214,6 +213,14 @@ define <2 x i32> @test8(<2 x i32> %trigger, <2 x i32>* %addr, <2 x i32> %dst) {
 }
 
 define <2 x double> @test_gather_2f64(<2 x double*> %ptrs, <2 x i1> %mask, <2 x double> %src0)  {
+; SSE-LABEL: 'test_gather_2f64'
+; SSE-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %res = call <2 x double> @llvm.masked.gather.v2f64.v2p0f64(<2 x double*> %ptrs, i32 4, <2 x i1> %mask, <2 x double> %src0)
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x double> %res
+;
+; AVX1-LABEL: 'test_gather_2f64'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %res = call <2 x double> @llvm.masked.gather.v2f64.v2p0f64(<2 x double*> %ptrs, i32 4, <2 x i1> %mask, <2 x double> %src0)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x double> %res
+;
 ; AVX2-LABEL: 'test_gather_2f64'
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %res = call <2 x double> @llvm.masked.gather.v2f64.v2p0f64(<2 x double*> %ptrs, i32 4, <2 x i1> %mask, <2 x double> %src0)
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x double> %res
@@ -222,13 +229,9 @@ define <2 x double> @test_gather_2f64(<2 x double*> %ptrs, <2 x i1> %mask, <2 x
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %res = call <2 x double> @llvm.masked.gather.v2f64.v2p0f64(<2 x double*> %ptrs, i32 4, <2 x i1> %mask, <2 x double> %src0)
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x double> %res
 ;
-; KNL-LABEL: 'test_gather_2f64'
-; KNL-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %res = call <2 x double> @llvm.masked.gather.v2f64.v2p0f64(<2 x double*> %ptrs, i32 4, <2 x i1> %mask, <2 x double> %src0)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x double> %res
-;
-; SKX-LABEL: 'test_gather_2f64'
-; SKX-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %res = call <2 x double> @llvm.masked.gather.v2f64.v2p0f64(<2 x double*> %ptrs, i32 4, <2 x i1> %mask, <2 x double> %src0)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x double> %res
+; AVX512-LABEL: 'test_gather_2f64'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %res = call <2 x double> @llvm.masked.gather.v2f64.v2p0f64(<2 x double*> %ptrs, i32 4, <2 x i1> %mask, <2 x double> %src0)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x double> %res
 ;
 
 
@@ -240,6 +243,14 @@ define <2 x double> @test_gather_2f64(<2 x double*> %ptrs, <2 x i1> %mask, <2 x
 declare <2 x double> @llvm.masked.gather.v2f64.v2p0f64(<2 x double*> %ptrs, i32, <2 x i1> %mask, <2 x double> %src0)
 
 define <4 x i32> @test_gather_4i32(<4 x i32*> %ptrs, <4 x i1> %mask, <4 x i32> %src0)  {
+; SSE-LABEL: 'test_gather_4i32'
+; SSE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %res = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> %ptrs, i32 4, <4 x i1> %mask, <4 x i32> %src0)
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i32> %res
+;
+; AVX1-LABEL: 'test_gather_4i32'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %res = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> %ptrs, i32 4, <4 x i1> %mask, <4 x i32> %src0)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i32> %res
+;
 ; AVX2-LABEL: 'test_gather_4i32'
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %res = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> %ptrs, i32 4, <4 x i1> %mask, <4 x i32> %src0)
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i32> %res
@@ -265,6 +276,14 @@ define <4 x i32> @test_gather_4i32(<4 x i32*> %ptrs, <4 x i1> %mask, <4 x i32> %
 }
 
 define <4 x i32> @test_gather_4i32_const_mask(<4 x i32*> %ptrs, <4 x i32> %src0)  {
+; SSE-LABEL: 'test_gather_4i32_const_mask'
+; SSE-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %res = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> %ptrs, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> %src0)
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i32> %res
+;
+; AVX1-LABEL: 'test_gather_4i32_const_mask'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %res = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> %ptrs, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> %src0)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i32> %res
+;
 ; AVX2-LABEL: 'test_gather_4i32_const_mask'
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %res = call <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> %ptrs, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> %src0)
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i32> %res
@@ -291,6 +310,24 @@ define <4 x i32> @test_gather_4i32_const_mask(<4 x i32*> %ptrs, <4 x i32> %src0)
 declare <4 x i32> @llvm.masked.gather.v4i32.v4p0i32(<4 x i32*> %ptrs, i32, <4 x i1> %mask, <4 x i32> %src0)
 
 define <16 x float> @test_gather_16f32_const_mask(float* %base, <16 x i32> %ind) {
+; SSE2-LABEL: 'test_gather_16f32_const_mask'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %sext_ind = sext <16 x i32> %ind to <16 x i64>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.v = getelementptr float, float* %base, <16 x i64> %sext_ind
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> %gep.v, i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x float> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x float> %res
+;
+; SSE42-LABEL: 'test_gather_16f32_const_mask'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %sext_ind = sext <16 x i32> %ind to <16 x i64>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.v = getelementptr float, float* %base, <16 x i64> %sext_ind
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> %gep.v, i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x float> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x float> %res
+;
+; AVX1-LABEL: 'test_gather_16f32_const_mask'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %sext_ind = sext <16 x i32> %ind to <16 x i64>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.v = getelementptr float, float* %base, <16 x i64> %sext_ind
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> %gep.v, i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x float> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x float> %res
+;
 ; AVX2-LABEL: 'test_gather_16f32_const_mask'
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %sext_ind = sext <16 x i32> %ind to <16 x i64>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.v = getelementptr float, float* %base, <16 x i64> %sext_ind
@@ -303,17 +340,11 @@ define <16 x float> @test_gather_16f32_const_mask(float* %base, <16 x i32> %ind)
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> %gep.v, i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x float> undef)
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x float> %res
 ;
-; KNL-LABEL: 'test_gather_16f32_const_mask'
-; KNL-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %sext_ind = sext <16 x i32> %ind to <16 x i64>
-; KNL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.v = getelementptr float, float* %base, <16 x i64> %sext_ind
-; KNL-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> %gep.v, i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x float> undef)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x float> %res
-;
-; SKX-LABEL: 'test_gather_16f32_const_mask'
-; SKX-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %sext_ind = sext <16 x i32> %ind to <16 x i64>
-; SKX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.v = getelementptr float, float* %base, <16 x i64> %sext_ind
-; SKX-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> %gep.v, i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x float> undef)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x float> %res
+; AVX512-LABEL: 'test_gather_16f32_const_mask'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %sext_ind = sext <16 x i32> %ind to <16 x i64>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.v = getelementptr float, float* %base, <16 x i64> %sext_ind
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> %gep.v, i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x float> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x float> %res
 ;
   %sext_ind = sext <16 x i32> %ind to <16 x i64>
   %gep.v = getelementptr float, float* %base, <16 x i64> %sext_ind
@@ -323,6 +354,24 @@ define <16 x float> @test_gather_16f32_const_mask(float* %base, <16 x i32> %ind)
 }
 
 define <16 x float> @test_gather_16f32_var_mask(float* %base, <16 x i32> %ind, <16 x i1>%mask) {
+; SSE2-LABEL: 'test_gather_16f32_var_mask'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %sext_ind = sext <16 x i32> %ind to <16 x i64>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.v = getelementptr float, float* %base, <16 x i64> %sext_ind
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 60 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> %gep.v, i32 4, <16 x i1> %mask, <16 x float> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x float> %res
+;
+; SSE42-LABEL: 'test_gather_16f32_var_mask'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %sext_ind = sext <16 x i32> %ind to <16 x i64>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.v = getelementptr float, float* %base, <16 x i64> %sext_ind
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 60 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> %gep.v, i32 4, <16 x i1> %mask, <16 x float> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x float> %res
+;
+; AVX1-LABEL: 'test_gather_16f32_var_mask'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %sext_ind = sext <16 x i32> %ind to <16 x i64>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.v = getelementptr float, float* %base, <16 x i64> %sext_ind
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 62 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> %gep.v, i32 4, <16 x i1> %mask, <16 x float> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x float> %res
+;
 ; AVX2-LABEL: 'test_gather_16f32_var_mask'
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %sext_ind = sext <16 x i32> %ind to <16 x i64>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.v = getelementptr float, float* %base, <16 x i64> %sext_ind
@@ -335,17 +384,11 @@ define <16 x float> @test_gather_16f32_var_mask(float* %base, <16 x i32> %ind, <
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> %gep.v, i32 4, <16 x i1> %mask, <16 x float> undef)
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x float> %res
 ;
-; KNL-LABEL: 'test_gather_16f32_var_mask'
-; KNL-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %sext_ind = sext <16 x i32> %ind to <16 x i64>
-; KNL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.v = getelementptr float, float* %base, <16 x i64> %sext_ind
-; KNL-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> %gep.v, i32 4, <16 x i1> %mask, <16 x float> undef)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x float> %res
-;
-; SKX-LABEL: 'test_gather_16f32_var_mask'
-; SKX-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %sext_ind = sext <16 x i32> %ind to <16 x i64>
-; SKX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.v = getelementptr float, float* %base, <16 x i64> %sext_ind
-; SKX-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> %gep.v, i32 4, <16 x i1> %mask, <16 x float> undef)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x float> %res
+; AVX512-LABEL: 'test_gather_16f32_var_mask'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %sext_ind = sext <16 x i32> %ind to <16 x i64>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.v = getelementptr float, float* %base, <16 x i64> %sext_ind
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> %gep.v, i32 4, <16 x i1> %mask, <16 x float> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x float> %res
 ;
   %sext_ind = sext <16 x i32> %ind to <16 x i64>
   %gep.v = getelementptr float, float* %base, <16 x i64> %sext_ind
@@ -355,6 +398,24 @@ define <16 x float> @test_gather_16f32_var_mask(float* %base, <16 x i32> %ind, <
 }
 
 define <16 x float> @test_gather_16f32_ra_var_mask(<16 x float*> %ptrs, <16 x i32> %ind, <16 x i1>%mask) {
+; SSE2-LABEL: 'test_gather_16f32_ra_var_mask'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %sext_ind = sext <16 x i32> %ind to <16 x i64>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.v = getelementptr float, <16 x float*> %ptrs, <16 x i64> %sext_ind
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 60 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> %gep.v, i32 4, <16 x i1> %mask, <16 x float> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x float> %res
+;
+; SSE42-LABEL: 'test_gather_16f32_ra_var_mask'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %sext_ind = sext <16 x i32> %ind to <16 x i64>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.v = getelementptr float, <16 x float*> %ptrs, <16 x i64> %sext_ind
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 60 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> %gep.v, i32 4, <16 x i1> %mask, <16 x float> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x float> %res
+;
+; AVX1-LABEL: 'test_gather_16f32_ra_var_mask'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %sext_ind = sext <16 x i32> %ind to <16 x i64>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.v = getelementptr float, <16 x float*> %ptrs, <16 x i64> %sext_ind
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 62 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> %gep.v, i32 4, <16 x i1> %mask, <16 x float> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x float> %res
+;
 ; AVX2-LABEL: 'test_gather_16f32_ra_var_mask'
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %sext_ind = sext <16 x i32> %ind to <16 x i64>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.v = getelementptr float, <16 x float*> %ptrs, <16 x i64> %sext_ind
@@ -367,17 +428,11 @@ define <16 x float> @test_gather_16f32_ra_var_mask(<16 x float*> %ptrs, <16 x i3
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> %gep.v, i32 4, <16 x i1> %mask, <16 x float> undef)
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x float> %res
 ;
-; KNL-LABEL: 'test_gather_16f32_ra_var_mask'
-; KNL-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %sext_ind = sext <16 x i32> %ind to <16 x i64>
-; KNL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.v = getelementptr float, <16 x float*> %ptrs, <16 x i64> %sext_ind
-; KNL-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> %gep.v, i32 4, <16 x i1> %mask, <16 x float> undef)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x float> %res
-;
-; SKX-LABEL: 'test_gather_16f32_ra_var_mask'
-; SKX-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %sext_ind = sext <16 x i32> %ind to <16 x i64>
-; SKX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.v = getelementptr float, <16 x float*> %ptrs, <16 x i64> %sext_ind
-; SKX-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> %gep.v, i32 4, <16 x i1> %mask, <16 x float> undef)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x float> %res
+; AVX512-LABEL: 'test_gather_16f32_ra_var_mask'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %sext_ind = sext <16 x i32> %ind to <16 x i64>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.v = getelementptr float, <16 x float*> %ptrs, <16 x i64> %sext_ind
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> %gep.v, i32 4, <16 x i1> %mask, <16 x float> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x float> %res
 ;
   %sext_ind = sext <16 x i32> %ind to <16 x i64>
   %gep.v = getelementptr float, <16 x float*> %ptrs, <16 x i64> %sext_ind
@@ -387,6 +442,30 @@ define <16 x float> @test_gather_16f32_ra_var_mask(<16 x float*> %ptrs, <16 x i3
 }
 
 define <16 x float> @test_gather_16f32_const_mask2(float* %base, <16 x i32> %ind) {
+; SSE2-LABEL: 'test_gather_16f32_const_mask2'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %broadcast.splatinsert = insertelement <16 x float*> undef, float* %base, i32 0
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %broadcast.splat = shufflevector <16 x float*> %broadcast.splatinsert, <16 x float*> undef, <16 x i32> zeroinitializer
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %sext_ind = sext <16 x i32> %ind to <16 x i64>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.random = getelementptr float, <16 x float*> %broadcast.splat, <16 x i64> %sext_ind
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> %gep.random, i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x float> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x float> %res
+;
+; SSE42-LABEL: 'test_gather_16f32_const_mask2'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %broadcast.splatinsert = insertelement <16 x float*> undef, float* %base, i32 0
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %broadcast.splat = shufflevector <16 x float*> %broadcast.splatinsert, <16 x float*> undef, <16 x i32> zeroinitializer
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %sext_ind = sext <16 x i32> %ind to <16 x i64>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.random = getelementptr float, <16 x float*> %broadcast.splat, <16 x i64> %sext_ind
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> %gep.random, i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x float> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x float> %res
+;
+; AVX1-LABEL: 'test_gather_16f32_const_mask2'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %broadcast.splatinsert = insertelement <16 x float*> undef, float* %base, i32 0
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %broadcast.splat = shufflevector <16 x float*> %broadcast.splatinsert, <16 x float*> undef, <16 x i32> zeroinitializer
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %sext_ind = sext <16 x i32> %ind to <16 x i64>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.random = getelementptr float, <16 x float*> %broadcast.splat, <16 x i64> %sext_ind
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> %gep.random, i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x float> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x float> %res
+;
 ; AVX2-LABEL: 'test_gather_16f32_const_mask2'
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %broadcast.splatinsert = insertelement <16 x float*> undef, float* %base, i32 0
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %broadcast.splat = shufflevector <16 x float*> %broadcast.splatinsert, <16 x float*> undef, <16 x i32> zeroinitializer
@@ -403,21 +482,13 @@ define <16 x float> @test_gather_16f32_const_mask2(float* %base, <16 x i32> %ind
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> %gep.random, i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x float> undef)
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x float> %res
 ;
-; KNL-LABEL: 'test_gather_16f32_const_mask2'
-; KNL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %broadcast.splatinsert = insertelement <16 x float*> undef, float* %base, i32 0
-; KNL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %broadcast.splat = shufflevector <16 x float*> %broadcast.splatinsert, <16 x float*> undef, <16 x i32> zeroinitializer
-; KNL-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %sext_ind = sext <16 x i32> %ind to <16 x i64>
-; KNL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.random = getelementptr float, <16 x float*> %broadcast.splat, <16 x i64> %sext_ind
-; KNL-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> %gep.random, i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x float> undef)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x float> %res
-;
-; SKX-LABEL: 'test_gather_16f32_const_mask2'
-; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %broadcast.splatinsert = insertelement <16 x float*> undef, float* %base, i32 0
-; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %broadcast.splat = shufflevector <16 x float*> %broadcast.splatinsert, <16 x float*> undef, <16 x i32> zeroinitializer
-; SKX-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %sext_ind = sext <16 x i32> %ind to <16 x i64>
-; SKX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.random = getelementptr float, <16 x float*> %broadcast.splat, <16 x i64> %sext_ind
-; SKX-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> %gep.random, i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x float> undef)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x float> %res
+; AVX512-LABEL: 'test_gather_16f32_const_mask2'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %broadcast.splatinsert = insertelement <16 x float*> undef, float* %base, i32 0
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %broadcast.splat = shufflevector <16 x float*> %broadcast.splatinsert, <16 x float*> undef, <16 x i32> zeroinitializer
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %sext_ind = sext <16 x i32> %ind to <16 x i64>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.random = getelementptr float, <16 x float*> %broadcast.splat, <16 x i64> %sext_ind
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %res = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> %gep.random, i32 4, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x float> undef)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x float> %res
 ;
   %broadcast.splatinsert = insertelement <16 x float*> undef, float* %base, i32 0
   %broadcast.splat = shufflevector <16 x float*> %broadcast.splatinsert, <16 x float*> undef, <16 x i32> zeroinitializer
@@ -430,6 +501,22 @@ define <16 x float> @test_gather_16f32_const_mask2(float* %base, <16 x i32> %ind
 }
 
 define void @test_scatter_16i32(i32* %base, <16 x i32> %ind, i16 %mask, <16 x i32>%val) {
+; SSE-LABEL: 'test_scatter_16i32'
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %broadcast.splatinsert = insertelement <16 x i32*> undef, i32* %base, i32 0
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %broadcast.splat = shufflevector <16 x i32*> %broadcast.splatinsert, <16 x i32*> undef, <16 x i32> zeroinitializer
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.random = getelementptr i32, <16 x i32*> %broadcast.splat, <16 x i32> %ind
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %imask = bitcast i16 %mask to <16 x i1>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: call void @llvm.masked.scatter.v16i32.v16p0i32(<16 x i32> %val, <16 x i32*> %gep.random, i32 4, <16 x i1> %imask)
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; AVX1-LABEL: 'test_scatter_16i32'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %broadcast.splatinsert = insertelement <16 x i32*> undef, i32* %base, i32 0
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %broadcast.splat = shufflevector <16 x i32*> %broadcast.splatinsert, <16 x i32*> undef, <16 x i32> zeroinitializer
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.random = getelementptr i32, <16 x i32*> %broadcast.splat, <16 x i32> %ind
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %imask = bitcast i16 %mask to <16 x i1>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: call void @llvm.masked.scatter.v16i32.v16p0i32(<16 x i32> %val, <16 x i32*> %gep.random, i32 4, <16 x i1> %imask)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
 ; AVX2-LABEL: 'test_scatter_16i32'
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %broadcast.splatinsert = insertelement <16 x i32*> undef, i32* %base, i32 0
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %broadcast.splat = shufflevector <16 x i32*> %broadcast.splatinsert, <16 x i32*> undef, <16 x i32> zeroinitializer
@@ -446,21 +533,13 @@ define void @test_scatter_16i32(i32* %base, <16 x i32> %ind, i16 %mask, <16 x i3
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: call void @llvm.masked.scatter.v16i32.v16p0i32(<16 x i32> %val, <16 x i32*> %gep.random, i32 4, <16 x i1> %imask)
 ; SKL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
-; KNL-LABEL: 'test_scatter_16i32'
-; KNL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %broadcast.splatinsert = insertelement <16 x i32*> undef, i32* %base, i32 0
-; KNL-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %broadcast.splat = shufflevector <16 x i32*> %broadcast.splatinsert, <16 x i32*> undef, <16 x i32> zeroinitializer
-; KNL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.random = getelementptr i32, <16 x i32*> %broadcast.splat, <16 x i32> %ind
-; KNL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %imask = bitcast i16 %mask to <16 x i1>
-; KNL-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: call void @llvm.masked.scatter.v16i32.v16p0i32(<16 x i32> %val, <16 x i32*> %gep.random, i32 4, <16 x i1> %imask)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; SKX-LABEL: 'test_scatter_16i32'
-; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %broadcast.splatinsert = insertelement <16 x i32*> undef, i32* %base, i32 0
-; SKX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %broadcast.splat = shufflevector <16 x i32*> %broadcast.splatinsert, <16 x i32*> undef, <16 x i32> zeroinitializer
-; SKX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.random = getelementptr i32, <16 x i32*> %broadcast.splat, <16 x i32> %ind
-; SKX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %imask = bitcast i16 %mask to <16 x i1>
-; SKX-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: call void @llvm.masked.scatter.v16i32.v16p0i32(<16 x i32> %val, <16 x i32*> %gep.random, i32 4, <16 x i1> %imask)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; AVX512-LABEL: 'test_scatter_16i32'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %broadcast.splatinsert = insertelement <16 x i32*> undef, i32* %base, i32 0
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %broadcast.splat = shufflevector <16 x i32*> %broadcast.splatinsert, <16 x i32*> undef, <16 x i32> zeroinitializer
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.random = getelementptr i32, <16 x i32*> %broadcast.splat, <16 x i32> %ind
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %imask = bitcast i16 %mask to <16 x i1>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: call void @llvm.masked.scatter.v16i32.v16p0i32(<16 x i32> %val, <16 x i32*> %gep.random, i32 4, <16 x i1> %imask)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %broadcast.splatinsert = insertelement <16 x i32*> undef, i32* %base, i32 0
   %broadcast.splat = shufflevector <16 x i32*> %broadcast.splatinsert, <16 x i32*> undef, <16 x i32> zeroinitializer
@@ -472,21 +551,17 @@ define void @test_scatter_16i32(i32* %base, <16 x i32> %ind, i16 %mask, <16 x i3
 }
 
 define void @test_scatter_8i32(<8 x i32>%a1, <8 x i32*> %ptr, <8 x i1>%mask) {
-; AVX2-LABEL: 'test_scatter_8i32'
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: call void @llvm.masked.scatter.v8i32.v8p0i32(<8 x i32> %a1, <8 x i32*> %ptr, i32 4, <8 x i1> %mask)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; SSE-LABEL: 'test_scatter_8i32'
+; SSE-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: call void @llvm.masked.scatter.v8i32.v8p0i32(<8 x i32> %a1, <8 x i32*> %ptr, i32 4, <8 x i1> %mask)
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
-; SKL-LABEL: 'test_scatter_8i32'
-; SKL-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: call void @llvm.masked.scatter.v8i32.v8p0i32(<8 x i32> %a1, <8 x i32*> %ptr, i32 4, <8 x i1> %mask)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; AVX-LABEL: 'test_scatter_8i32'
+; AVX-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: call void @llvm.masked.scatter.v8i32.v8p0i32(<8 x i32> %a1, <8 x i32*> %ptr, i32 4, <8 x i1> %mask)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
-; KNL-LABEL: 'test_scatter_8i32'
-; KNL-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: call void @llvm.masked.scatter.v8i32.v8p0i32(<8 x i32> %a1, <8 x i32*> %ptr, i32 4, <8 x i1> %mask)
-; KNL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
-;
-; SKX-LABEL: 'test_scatter_8i32'
-; SKX-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: call void @llvm.masked.scatter.v8i32.v8p0i32(<8 x i32> %a1, <8 x i32*> %ptr, i32 4, <8 x i1> %mask)
-; SKX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; AVX512-LABEL: 'test_scatter_8i32'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: call void @llvm.masked.scatter.v8i32.v8p0i32(<8 x i32> %a1, <8 x i32*> %ptr, i32 4, <8 x i1> %mask)
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 
   call void @llvm.masked.scatter.v8i32.v8p0i32(<8 x i32> %a1, <8 x i32*> %ptr, i32 4, <8 x i1> %mask)
@@ -496,13 +571,13 @@ define void @test_scatter_8i32(<8 x i32>%a1, <8 x i32*> %ptr, <8 x i1>%mask) {
 declare void @llvm.masked.scatter.v8i32.v8p0i32(<8 x i32> %a1, <8 x i32*> %ptr, i32, <8 x i1> %mask)
 
 define void @test_scatter_4i32(<4 x i32>%a1, <4 x i32*> %ptr, <4 x i1>%mask) {
-; AVX2-LABEL: 'test_scatter_4i32'
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.scatter.v4i32.v4p0i32(<4 x i32> %a1, <4 x i32*> %ptr, i32 4, <4 x i1> %mask)
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; SSE-LABEL: 'test_scatter_4i32'
+; SSE-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.scatter.v4i32.v4p0i32(<4 x i32> %a1, <4 x i32*> %ptr, i32 4, <4 x i1> %mask)
+; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
-; SKL-LABEL: 'test_scatter_4i32'
-; SKL-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.scatter.v4i32.v4p0i32(<4 x i32> %a1, <4 x i32*> %ptr, i32 4, <4 x i1> %mask)
-; SKL-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+; AVX-LABEL: 'test_scatter_4i32'
+; AVX-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.scatter.v4i32.v4p0i32(<4 x i32> %a1, <4 x i32*> %ptr, i32 4, <4 x i1> %mask)
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; KNL-LABEL: 'test_scatter_4i32'
 ; KNL-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: call void @llvm.masked.scatter.v4i32.v4p0i32(<4 x i32> %a1, <4 x i32*> %ptr, i32 4, <4 x i1> %mask)
@@ -517,6 +592,24 @@ define void @test_scatter_4i32(<4 x i32>%a1, <4 x i32*> %ptr, <4 x i1>%mask) {
 }
 
 define <4 x float> @test_gather_4f32(float* %ptr, <4 x i32> %ind, <4 x i1>%mask) {
+; SSE2-LABEL: 'test_gather_4f32'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %sext_ind = sext <4 x i32> %ind to <4 x i64>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.v = getelementptr float, float* %ptr, <4 x i64> %sext_ind
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %res = call <4 x float> @llvm.masked.gather.v4f32.v4p0f32(<4 x float*> %gep.v, i32 4, <4 x i1> %mask, <4 x float> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x float> %res
+;
+; SSE42-LABEL: 'test_gather_4f32'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %sext_ind = sext <4 x i32> %ind to <4 x i64>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.v = getelementptr float, float* %ptr, <4 x i64> %sext_ind
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %res = call <4 x float> @llvm.masked.gather.v4f32.v4p0f32(<4 x float*> %gep.v, i32 4, <4 x i1> %mask, <4 x float> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x float> %res
+;
+; AVX1-LABEL: 'test_gather_4f32'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %sext_ind = sext <4 x i32> %ind to <4 x i64>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.v = getelementptr float, float* %ptr, <4 x i64> %sext_ind
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %res = call <4 x float> @llvm.masked.gather.v4f32.v4p0f32(<4 x float*> %gep.v, i32 4, <4 x i1> %mask, <4 x float> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x float> %res
+;
 ; AVX2-LABEL: 'test_gather_4f32'
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sext_ind = sext <4 x i32> %ind to <4 x i64>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.v = getelementptr float, float* %ptr, <4 x i64> %sext_ind
@@ -549,6 +642,24 @@ define <4 x float> @test_gather_4f32(float* %ptr, <4 x i32> %ind, <4 x i1>%mask)
 }
 
 define <4 x float> @test_gather_4f32_const_mask(float* %ptr, <4 x i32> %ind) {
+; SSE2-LABEL: 'test_gather_4f32_const_mask'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %sext_ind = sext <4 x i32> %ind to <4 x i64>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.v = getelementptr float, float* %ptr, <4 x i64> %sext_ind
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %res = call <4 x float> @llvm.masked.gather.v4f32.v4p0f32(<4 x float*> %gep.v, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x float> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x float> %res
+;
+; SSE42-LABEL: 'test_gather_4f32_const_mask'
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %sext_ind = sext <4 x i32> %ind to <4 x i64>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.v = getelementptr float, float* %ptr, <4 x i64> %sext_ind
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %res = call <4 x float> @llvm.masked.gather.v4f32.v4p0f32(<4 x float*> %gep.v, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x float> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x float> %res
+;
+; AVX1-LABEL: 'test_gather_4f32_const_mask'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %sext_ind = sext <4 x i32> %ind to <4 x i64>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.v = getelementptr float, float* %ptr, <4 x i64> %sext_ind
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %res = call <4 x float> @llvm.masked.gather.v4f32.v4p0f32(<4 x float*> %gep.v, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x float> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x float> %res
+;
 ; AVX2-LABEL: 'test_gather_4f32_const_mask'
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sext_ind = sext <4 x i32> %ind to <4 x i64>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %gep.v = getelementptr float, float* %ptr, <4 x i64> %sext_ind
diff --git a/test/Analysis/CostModel/X86/reduce-smax-widen.ll b/test/Analysis/CostModel/X86/reduce-smax-widen.ll
index 44b01d3707c8ce74607a9e28103558d4594628b2..6eba5c354b9eb30e1097f8b79b77998e75272beb 100644
--- a/test/Analysis/CostModel/X86/reduce-smax-widen.ll
+++ b/test/Analysis/CostModel/X86/reduce-smax-widen.ll
@@ -11,18 +11,18 @@
 define i32 @reduce_i64(i32 %arg) {
 ; SSE2-LABEL: 'reduce_i64'
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1 = call i64 @llvm.experimental.vector.reduce.smax.i64.v1i64(<1 x i64> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V2 = call i64 @llvm.experimental.vector.reduce.smax.i64.v2i64(<2 x i64> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V4 = call i64 @llvm.experimental.vector.reduce.smax.i64.v4i64(<4 x i64> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 38 for instruction: %V8 = call i64 @llvm.experimental.vector.reduce.smax.i64.v8i64(<8 x i64> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 74 for instruction: %V16 = call i64 @llvm.experimental.vector.reduce.smax.i64.v16i64(<16 x i64> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V2 = call i64 @llvm.experimental.vector.reduce.smax.i64.v2i64(<2 x i64> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V4 = call i64 @llvm.experimental.vector.reduce.smax.i64.v4i64(<4 x i64> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 46 for instruction: %V8 = call i64 @llvm.experimental.vector.reduce.smax.i64.v8i64(<8 x i64> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 90 for instruction: %V16 = call i64 @llvm.experimental.vector.reduce.smax.i64.v16i64(<16 x i64> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSSE3-LABEL: 'reduce_i64'
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1 = call i64 @llvm.experimental.vector.reduce.smax.i64.v1i64(<1 x i64> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V2 = call i64 @llvm.experimental.vector.reduce.smax.i64.v2i64(<2 x i64> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V4 = call i64 @llvm.experimental.vector.reduce.smax.i64.v4i64(<4 x i64> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 38 for instruction: %V8 = call i64 @llvm.experimental.vector.reduce.smax.i64.v8i64(<8 x i64> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 74 for instruction: %V16 = call i64 @llvm.experimental.vector.reduce.smax.i64.v16i64(<16 x i64> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V2 = call i64 @llvm.experimental.vector.reduce.smax.i64.v2i64(<2 x i64> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V4 = call i64 @llvm.experimental.vector.reduce.smax.i64.v4i64(<4 x i64> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 46 for instruction: %V8 = call i64 @llvm.experimental.vector.reduce.smax.i64.v8i64(<8 x i64> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 90 for instruction: %V16 = call i64 @llvm.experimental.vector.reduce.smax.i64.v16i64(<16 x i64> undef)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSE42-LABEL: 'reduce_i64'
@@ -67,19 +67,19 @@ define i32 @reduce_i64(i32 %arg) {
 
 define i32 @reduce_i32(i32 %arg) {
 ; SSE2-LABEL: 'reduce_i32'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2 = call i32 @llvm.experimental.vector.reduce.smax.i32.v2i32(<2 x i32> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4 = call i32 @llvm.experimental.vector.reduce.smax.i32.v4i32(<4 x i32> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V8 = call i32 @llvm.experimental.vector.reduce.smax.i32.v8i32(<8 x i32> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V16 = call i32 @llvm.experimental.vector.reduce.smax.i32.v16i32(<16 x i32> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %V32 = call i32 @llvm.experimental.vector.reduce.smax.i32.v32i32(<32 x i32> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2 = call i32 @llvm.experimental.vector.reduce.smax.i32.v2i32(<2 x i32> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V4 = call i32 @llvm.experimental.vector.reduce.smax.i32.v4i32(<4 x i32> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V8 = call i32 @llvm.experimental.vector.reduce.smax.i32.v8i32(<8 x i32> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %V16 = call i32 @llvm.experimental.vector.reduce.smax.i32.v16i32(<16 x i32> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 39 for instruction: %V32 = call i32 @llvm.experimental.vector.reduce.smax.i32.v32i32(<32 x i32> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSSE3-LABEL: 'reduce_i32'
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2 = call i32 @llvm.experimental.vector.reduce.smax.i32.v2i32(<2 x i32> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4 = call i32 @llvm.experimental.vector.reduce.smax.i32.v4i32(<4 x i32> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V8 = call i32 @llvm.experimental.vector.reduce.smax.i32.v8i32(<8 x i32> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V16 = call i32 @llvm.experimental.vector.reduce.smax.i32.v16i32(<16 x i32> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %V32 = call i32 @llvm.experimental.vector.reduce.smax.i32.v32i32(<32 x i32> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2 = call i32 @llvm.experimental.vector.reduce.smax.i32.v2i32(<2 x i32> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V4 = call i32 @llvm.experimental.vector.reduce.smax.i32.v4i32(<4 x i32> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V8 = call i32 @llvm.experimental.vector.reduce.smax.i32.v8i32(<8 x i32> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %V16 = call i32 @llvm.experimental.vector.reduce.smax.i32.v16i32(<16 x i32> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 39 for instruction: %V32 = call i32 @llvm.experimental.vector.reduce.smax.i32.v32i32(<32 x i32> undef)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSE42-LABEL: 'reduce_i32'
@@ -124,21 +124,21 @@ define i32 @reduce_i32(i32 %arg) {
 
 define i32 @reduce_i16(i32 %arg) {
 ; SSE2-LABEL: 'reduce_i16'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.smax.i16.v2i16(<2 x i16> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.smax.i16.v4i16(<4 x i16> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V8 = call i16 @llvm.experimental.vector.reduce.smax.i16.v8i16(<8 x i16> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.smax.i16.v16i16(<16 x i16> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V32 = call i16 @llvm.experimental.vector.reduce.smax.i16.v32i16(<32 x i16> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %V64 = call i16 @llvm.experimental.vector.reduce.smax.i16.v64i16(<64 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.smax.i16.v2i16(<2 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.smax.i16.v4i16(<4 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V8 = call i16 @llvm.experimental.vector.reduce.smax.i16.v8i16(<8 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.smax.i16.v16i16(<16 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %V32 = call i16 @llvm.experimental.vector.reduce.smax.i16.v32i16(<32 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V64 = call i16 @llvm.experimental.vector.reduce.smax.i16.v64i16(<64 x i16> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSSE3-LABEL: 'reduce_i16'
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.smax.i16.v2i16(<2 x i16> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.smax.i16.v4i16(<4 x i16> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V8 = call i16 @llvm.experimental.vector.reduce.smax.i16.v8i16(<8 x i16> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.smax.i16.v16i16(<16 x i16> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V32 = call i16 @llvm.experimental.vector.reduce.smax.i16.v32i16(<32 x i16> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V64 = call i16 @llvm.experimental.vector.reduce.smax.i16.v64i16(<64 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.smax.i16.v2i16(<2 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.smax.i16.v4i16(<4 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8 = call i16 @llvm.experimental.vector.reduce.smax.i16.v8i16(<8 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.smax.i16.v16i16(<16 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V32 = call i16 @llvm.experimental.vector.reduce.smax.i16.v32i16(<32 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 44 for instruction: %V64 = call i16 @llvm.experimental.vector.reduce.smax.i16.v64i16(<64 x i16> undef)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSE42-LABEL: 'reduce_i16'
@@ -154,9 +154,9 @@ define i32 @reduce_i16(i32 %arg) {
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.smax.i16.v2i16(<2 x i16> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.smax.i16.v4i16(<4 x i16> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8 = call i16 @llvm.experimental.vector.reduce.smax.i16.v8i16(<8 x i16> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 53 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.smax.i16.v16i16(<16 x i16> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 58 for instruction: %V32 = call i16 @llvm.experimental.vector.reduce.smax.i16.v32i16(<32 x i16> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %V64 = call i16 @llvm.experimental.vector.reduce.smax.i16.v64i16(<64 x i16> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 61 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.smax.i16.v16i16(<16 x i16> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %V32 = call i16 @llvm.experimental.vector.reduce.smax.i16.v32i16(<32 x i16> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 82 for instruction: %V64 = call i16 @llvm.experimental.vector.reduce.smax.i16.v64i16(<64 x i16> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX2-LABEL: 'reduce_i16'
@@ -206,23 +206,23 @@ define i32 @reduce_i16(i32 %arg) {
 
 define i32 @reduce_i8(i32 %arg) {
 ; SSE2-LABEL: 'reduce_i8'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.smax.i8.v2i8(<2 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.smax.i8.v4i8(<4 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 37 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.smax.i8.v8i8(<8 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 49 for instruction: %V16 = call i8 @llvm.experimental.vector.reduce.smax.i8.v16i8(<16 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.smax.i8.v32i8(<32 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 55 for instruction: %V64 = call i8 @llvm.experimental.vector.reduce.smax.i8.v64i8(<64 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 63 for instruction: %V128 = call i8 @llvm.experimental.vector.reduce.smax.i8.v128i8(<128 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.smax.i8.v2i8(<2 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 29 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.smax.i8.v4i8(<4 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 43 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.smax.i8.v8i8(<8 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 57 for instruction: %V16 = call i8 @llvm.experimental.vector.reduce.smax.i8.v16i8(<16 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 61 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.smax.i8.v32i8(<32 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 69 for instruction: %V64 = call i8 @llvm.experimental.vector.reduce.smax.i8.v64i8(<64 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 85 for instruction: %V128 = call i8 @llvm.experimental.vector.reduce.smax.i8.v128i8(<128 x i8> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSSE3-LABEL: 'reduce_i8'
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.smax.i8.v2i8(<2 x i8> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.smax.i8.v4i8(<4 x i8> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.smax.i8.v8i8(<8 x i8> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V16 = call i8 @llvm.experimental.vector.reduce.smax.i8.v16i8(<16 x i8> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.smax.i8.v32i8(<32 x i8> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V64 = call i8 @llvm.experimental.vector.reduce.smax.i8.v64i8(<64 x i8> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %V128 = call i8 @llvm.experimental.vector.reduce.smax.i8.v128i8(<128 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.smax.i8.v2i8(<2 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.smax.i8.v4i8(<4 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.smax.i8.v8i8(<8 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %V16 = call i8 @llvm.experimental.vector.reduce.smax.i8.v16i8(<16 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.smax.i8.v32i8(<32 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %V64 = call i8 @llvm.experimental.vector.reduce.smax.i8.v64i8(<64 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 49 for instruction: %V128 = call i8 @llvm.experimental.vector.reduce.smax.i8.v128i8(<128 x i8> undef)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSE42-LABEL: 'reduce_i8'
@@ -240,9 +240,9 @@ define i32 @reduce_i8(i32 %arg) {
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.smax.i8.v4i8(<4 x i8> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.smax.i8.v8i8(<8 x i8> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V16 = call i8 @llvm.experimental.vector.reduce.smax.i8.v16i8(<16 x i8> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.smax.i8.v32i8(<32 x i8> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 71 for instruction: %V64 = call i8 @llvm.experimental.vector.reduce.smax.i8.v64i8(<64 x i8> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 81 for instruction: %V128 = call i8 @llvm.experimental.vector.reduce.smax.i8.v128i8(<128 x i8> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 76 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.smax.i8.v32i8(<32 x i8> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 83 for instruction: %V64 = call i8 @llvm.experimental.vector.reduce.smax.i8.v64i8(<64 x i8> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 97 for instruction: %V128 = call i8 @llvm.experimental.vector.reduce.smax.i8.v128i8(<128 x i8> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX2-LABEL: 'reduce_i8'
diff --git a/test/Analysis/CostModel/X86/reduce-smax.ll b/test/Analysis/CostModel/X86/reduce-smax.ll
index b4b9a3ba6379802b4c29809f953d94c51013e1fe..4ea2a704967aec560773f3d853ed4274c20a4611 100644
--- a/test/Analysis/CostModel/X86/reduce-smax.ll
+++ b/test/Analysis/CostModel/X86/reduce-smax.ll
@@ -11,18 +11,18 @@
 define i32 @reduce_i64(i32 %arg) {
 ; SSE2-LABEL: 'reduce_i64'
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1 = call i64 @llvm.experimental.vector.reduce.smax.i64.v1i64(<1 x i64> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V2 = call i64 @llvm.experimental.vector.reduce.smax.i64.v2i64(<2 x i64> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V4 = call i64 @llvm.experimental.vector.reduce.smax.i64.v4i64(<4 x i64> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 38 for instruction: %V8 = call i64 @llvm.experimental.vector.reduce.smax.i64.v8i64(<8 x i64> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 74 for instruction: %V16 = call i64 @llvm.experimental.vector.reduce.smax.i64.v16i64(<16 x i64> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V2 = call i64 @llvm.experimental.vector.reduce.smax.i64.v2i64(<2 x i64> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V4 = call i64 @llvm.experimental.vector.reduce.smax.i64.v4i64(<4 x i64> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 46 for instruction: %V8 = call i64 @llvm.experimental.vector.reduce.smax.i64.v8i64(<8 x i64> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 90 for instruction: %V16 = call i64 @llvm.experimental.vector.reduce.smax.i64.v16i64(<16 x i64> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSSE3-LABEL: 'reduce_i64'
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1 = call i64 @llvm.experimental.vector.reduce.smax.i64.v1i64(<1 x i64> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V2 = call i64 @llvm.experimental.vector.reduce.smax.i64.v2i64(<2 x i64> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V4 = call i64 @llvm.experimental.vector.reduce.smax.i64.v4i64(<4 x i64> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 38 for instruction: %V8 = call i64 @llvm.experimental.vector.reduce.smax.i64.v8i64(<8 x i64> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 74 for instruction: %V16 = call i64 @llvm.experimental.vector.reduce.smax.i64.v16i64(<16 x i64> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V2 = call i64 @llvm.experimental.vector.reduce.smax.i64.v2i64(<2 x i64> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V4 = call i64 @llvm.experimental.vector.reduce.smax.i64.v4i64(<4 x i64> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 46 for instruction: %V8 = call i64 @llvm.experimental.vector.reduce.smax.i64.v8i64(<8 x i64> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 90 for instruction: %V16 = call i64 @llvm.experimental.vector.reduce.smax.i64.v16i64(<16 x i64> undef)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSE42-LABEL: 'reduce_i64'
@@ -67,19 +67,19 @@ define i32 @reduce_i64(i32 %arg) {
 
 define i32 @reduce_i32(i32 %arg) {
 ; SSE2-LABEL: 'reduce_i32'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V2 = call i32 @llvm.experimental.vector.reduce.smax.i32.v2i32(<2 x i32> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4 = call i32 @llvm.experimental.vector.reduce.smax.i32.v4i32(<4 x i32> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V8 = call i32 @llvm.experimental.vector.reduce.smax.i32.v8i32(<8 x i32> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V16 = call i32 @llvm.experimental.vector.reduce.smax.i32.v16i32(<16 x i32> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %V32 = call i32 @llvm.experimental.vector.reduce.smax.i32.v32i32(<32 x i32> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V2 = call i32 @llvm.experimental.vector.reduce.smax.i32.v2i32(<2 x i32> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V4 = call i32 @llvm.experimental.vector.reduce.smax.i32.v4i32(<4 x i32> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V8 = call i32 @llvm.experimental.vector.reduce.smax.i32.v8i32(<8 x i32> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %V16 = call i32 @llvm.experimental.vector.reduce.smax.i32.v16i32(<16 x i32> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 39 for instruction: %V32 = call i32 @llvm.experimental.vector.reduce.smax.i32.v32i32(<32 x i32> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSSE3-LABEL: 'reduce_i32'
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V2 = call i32 @llvm.experimental.vector.reduce.smax.i32.v2i32(<2 x i32> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4 = call i32 @llvm.experimental.vector.reduce.smax.i32.v4i32(<4 x i32> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V8 = call i32 @llvm.experimental.vector.reduce.smax.i32.v8i32(<8 x i32> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V16 = call i32 @llvm.experimental.vector.reduce.smax.i32.v16i32(<16 x i32> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %V32 = call i32 @llvm.experimental.vector.reduce.smax.i32.v32i32(<32 x i32> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V2 = call i32 @llvm.experimental.vector.reduce.smax.i32.v2i32(<2 x i32> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V4 = call i32 @llvm.experimental.vector.reduce.smax.i32.v4i32(<4 x i32> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V8 = call i32 @llvm.experimental.vector.reduce.smax.i32.v8i32(<8 x i32> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %V16 = call i32 @llvm.experimental.vector.reduce.smax.i32.v16i32(<16 x i32> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 39 for instruction: %V32 = call i32 @llvm.experimental.vector.reduce.smax.i32.v32i32(<32 x i32> undef)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSE42-LABEL: 'reduce_i32'
@@ -124,21 +124,21 @@ define i32 @reduce_i32(i32 %arg) {
 
 define i32 @reduce_i16(i32 %arg) {
 ; SSE2-LABEL: 'reduce_i16'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.smax.i16.v2i16(<2 x i16> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.smax.i16.v4i16(<4 x i16> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V8 = call i16 @llvm.experimental.vector.reduce.smax.i16.v8i16(<8 x i16> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.smax.i16.v16i16(<16 x i16> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V32 = call i16 @llvm.experimental.vector.reduce.smax.i16.v32i16(<32 x i16> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %V64 = call i16 @llvm.experimental.vector.reduce.smax.i16.v64i16(<64 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.smax.i16.v2i16(<2 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.smax.i16.v4i16(<4 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V8 = call i16 @llvm.experimental.vector.reduce.smax.i16.v8i16(<8 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.smax.i16.v16i16(<16 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %V32 = call i16 @llvm.experimental.vector.reduce.smax.i16.v32i16(<32 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V64 = call i16 @llvm.experimental.vector.reduce.smax.i16.v64i16(<64 x i16> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSSE3-LABEL: 'reduce_i16'
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.smax.i16.v2i16(<2 x i16> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.smax.i16.v4i16(<4 x i16> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V8 = call i16 @llvm.experimental.vector.reduce.smax.i16.v8i16(<8 x i16> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.smax.i16.v16i16(<16 x i16> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V32 = call i16 @llvm.experimental.vector.reduce.smax.i16.v32i16(<32 x i16> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V64 = call i16 @llvm.experimental.vector.reduce.smax.i16.v64i16(<64 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.smax.i16.v2i16(<2 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.smax.i16.v4i16(<4 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8 = call i16 @llvm.experimental.vector.reduce.smax.i16.v8i16(<8 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.smax.i16.v16i16(<16 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V32 = call i16 @llvm.experimental.vector.reduce.smax.i16.v32i16(<32 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 44 for instruction: %V64 = call i16 @llvm.experimental.vector.reduce.smax.i16.v64i16(<64 x i16> undef)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSE42-LABEL: 'reduce_i16'
@@ -154,9 +154,9 @@ define i32 @reduce_i16(i32 %arg) {
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.smax.i16.v2i16(<2 x i16> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.smax.i16.v4i16(<4 x i16> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8 = call i16 @llvm.experimental.vector.reduce.smax.i16.v8i16(<8 x i16> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 53 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.smax.i16.v16i16(<16 x i16> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 58 for instruction: %V32 = call i16 @llvm.experimental.vector.reduce.smax.i16.v32i16(<32 x i16> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %V64 = call i16 @llvm.experimental.vector.reduce.smax.i16.v64i16(<64 x i16> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 61 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.smax.i16.v16i16(<16 x i16> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %V32 = call i16 @llvm.experimental.vector.reduce.smax.i16.v32i16(<32 x i16> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 82 for instruction: %V64 = call i16 @llvm.experimental.vector.reduce.smax.i16.v64i16(<64 x i16> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX2-LABEL: 'reduce_i16'
@@ -206,23 +206,23 @@ define i32 @reduce_i16(i32 %arg) {
 
 define i32 @reduce_i8(i32 %arg) {
 ; SSE2-LABEL: 'reduce_i8'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.smax.i8.v2i8(<2 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.smax.i8.v4i8(<4 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.smax.i8.v8i8(<8 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 49 for instruction: %V16 = call i8 @llvm.experimental.vector.reduce.smax.i8.v16i8(<16 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.smax.i8.v32i8(<32 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 55 for instruction: %V64 = call i8 @llvm.experimental.vector.reduce.smax.i8.v64i8(<64 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 63 for instruction: %V128 = call i8 @llvm.experimental.vector.reduce.smax.i8.v128i8(<128 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.smax.i8.v2i8(<2 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.smax.i8.v4i8(<4 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.smax.i8.v8i8(<8 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 57 for instruction: %V16 = call i8 @llvm.experimental.vector.reduce.smax.i8.v16i8(<16 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 61 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.smax.i8.v32i8(<32 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 69 for instruction: %V64 = call i8 @llvm.experimental.vector.reduce.smax.i8.v64i8(<64 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 85 for instruction: %V128 = call i8 @llvm.experimental.vector.reduce.smax.i8.v128i8(<128 x i8> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSSE3-LABEL: 'reduce_i8'
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.smax.i8.v2i8(<2 x i8> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.smax.i8.v4i8(<4 x i8> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.smax.i8.v8i8(<8 x i8> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V16 = call i8 @llvm.experimental.vector.reduce.smax.i8.v16i8(<16 x i8> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.smax.i8.v32i8(<32 x i8> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V64 = call i8 @llvm.experimental.vector.reduce.smax.i8.v64i8(<64 x i8> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %V128 = call i8 @llvm.experimental.vector.reduce.smax.i8.v128i8(<128 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.smax.i8.v2i8(<2 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.smax.i8.v4i8(<4 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.smax.i8.v8i8(<8 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %V16 = call i8 @llvm.experimental.vector.reduce.smax.i8.v16i8(<16 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.smax.i8.v32i8(<32 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %V64 = call i8 @llvm.experimental.vector.reduce.smax.i8.v64i8(<64 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 49 for instruction: %V128 = call i8 @llvm.experimental.vector.reduce.smax.i8.v128i8(<128 x i8> undef)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSE42-LABEL: 'reduce_i8'
@@ -240,9 +240,9 @@ define i32 @reduce_i8(i32 %arg) {
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.smax.i8.v4i8(<4 x i8> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.smax.i8.v8i8(<8 x i8> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V16 = call i8 @llvm.experimental.vector.reduce.smax.i8.v16i8(<16 x i8> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.smax.i8.v32i8(<32 x i8> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 71 for instruction: %V64 = call i8 @llvm.experimental.vector.reduce.smax.i8.v64i8(<64 x i8> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 81 for instruction: %V128 = call i8 @llvm.experimental.vector.reduce.smax.i8.v128i8(<128 x i8> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 76 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.smax.i8.v32i8(<32 x i8> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 83 for instruction: %V64 = call i8 @llvm.experimental.vector.reduce.smax.i8.v64i8(<64 x i8> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 97 for instruction: %V128 = call i8 @llvm.experimental.vector.reduce.smax.i8.v128i8(<128 x i8> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX2-LABEL: 'reduce_i8'
diff --git a/test/Analysis/CostModel/X86/reduce-smin-widen.ll b/test/Analysis/CostModel/X86/reduce-smin-widen.ll
index 18dd54cf026ed04f10eb0309e803c0608ef7bdee..afcf9b2a182eb1ff58475a86ef80add33c4f410d 100644
--- a/test/Analysis/CostModel/X86/reduce-smin-widen.ll
+++ b/test/Analysis/CostModel/X86/reduce-smin-widen.ll
@@ -11,18 +11,18 @@
 define i32 @reduce_i64(i32 %arg) {
 ; SSE2-LABEL: 'reduce_i64'
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1 = call i64 @llvm.experimental.vector.reduce.smin.i64.v1i64(<1 x i64> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V2 = call i64 @llvm.experimental.vector.reduce.smin.i64.v2i64(<2 x i64> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V4 = call i64 @llvm.experimental.vector.reduce.smin.i64.v4i64(<4 x i64> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 38 for instruction: %V8 = call i64 @llvm.experimental.vector.reduce.smin.i64.v8i64(<8 x i64> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 74 for instruction: %V16 = call i64 @llvm.experimental.vector.reduce.smin.i64.v16i64(<16 x i64> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V2 = call i64 @llvm.experimental.vector.reduce.smin.i64.v2i64(<2 x i64> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V4 = call i64 @llvm.experimental.vector.reduce.smin.i64.v4i64(<4 x i64> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 46 for instruction: %V8 = call i64 @llvm.experimental.vector.reduce.smin.i64.v8i64(<8 x i64> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 90 for instruction: %V16 = call i64 @llvm.experimental.vector.reduce.smin.i64.v16i64(<16 x i64> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSSE3-LABEL: 'reduce_i64'
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1 = call i64 @llvm.experimental.vector.reduce.smin.i64.v1i64(<1 x i64> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V2 = call i64 @llvm.experimental.vector.reduce.smin.i64.v2i64(<2 x i64> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V4 = call i64 @llvm.experimental.vector.reduce.smin.i64.v4i64(<4 x i64> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 38 for instruction: %V8 = call i64 @llvm.experimental.vector.reduce.smin.i64.v8i64(<8 x i64> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 74 for instruction: %V16 = call i64 @llvm.experimental.vector.reduce.smin.i64.v16i64(<16 x i64> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V2 = call i64 @llvm.experimental.vector.reduce.smin.i64.v2i64(<2 x i64> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V4 = call i64 @llvm.experimental.vector.reduce.smin.i64.v4i64(<4 x i64> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 46 for instruction: %V8 = call i64 @llvm.experimental.vector.reduce.smin.i64.v8i64(<8 x i64> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 90 for instruction: %V16 = call i64 @llvm.experimental.vector.reduce.smin.i64.v16i64(<16 x i64> undef)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSE42-LABEL: 'reduce_i64'
@@ -67,19 +67,19 @@ define i32 @reduce_i64(i32 %arg) {
 
 define i32 @reduce_i32(i32 %arg) {
 ; SSE2-LABEL: 'reduce_i32'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2 = call i32 @llvm.experimental.vector.reduce.smin.i32.v2i32(<2 x i32> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4 = call i32 @llvm.experimental.vector.reduce.smin.i32.v4i32(<4 x i32> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V8 = call i32 @llvm.experimental.vector.reduce.smin.i32.v8i32(<8 x i32> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V16 = call i32 @llvm.experimental.vector.reduce.smin.i32.v16i32(<16 x i32> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %V32 = call i32 @llvm.experimental.vector.reduce.smin.i32.v32i32(<32 x i32> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2 = call i32 @llvm.experimental.vector.reduce.smin.i32.v2i32(<2 x i32> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V4 = call i32 @llvm.experimental.vector.reduce.smin.i32.v4i32(<4 x i32> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V8 = call i32 @llvm.experimental.vector.reduce.smin.i32.v8i32(<8 x i32> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %V16 = call i32 @llvm.experimental.vector.reduce.smin.i32.v16i32(<16 x i32> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 39 for instruction: %V32 = call i32 @llvm.experimental.vector.reduce.smin.i32.v32i32(<32 x i32> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSSE3-LABEL: 'reduce_i32'
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2 = call i32 @llvm.experimental.vector.reduce.smin.i32.v2i32(<2 x i32> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4 = call i32 @llvm.experimental.vector.reduce.smin.i32.v4i32(<4 x i32> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V8 = call i32 @llvm.experimental.vector.reduce.smin.i32.v8i32(<8 x i32> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V16 = call i32 @llvm.experimental.vector.reduce.smin.i32.v16i32(<16 x i32> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %V32 = call i32 @llvm.experimental.vector.reduce.smin.i32.v32i32(<32 x i32> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2 = call i32 @llvm.experimental.vector.reduce.smin.i32.v2i32(<2 x i32> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V4 = call i32 @llvm.experimental.vector.reduce.smin.i32.v4i32(<4 x i32> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V8 = call i32 @llvm.experimental.vector.reduce.smin.i32.v8i32(<8 x i32> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %V16 = call i32 @llvm.experimental.vector.reduce.smin.i32.v16i32(<16 x i32> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 39 for instruction: %V32 = call i32 @llvm.experimental.vector.reduce.smin.i32.v32i32(<32 x i32> undef)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSE42-LABEL: 'reduce_i32'
@@ -124,19 +124,19 @@ define i32 @reduce_i32(i32 %arg) {
 
 define i32 @reduce_i16(i32 %arg) {
 ; SSE2-LABEL: 'reduce_i16'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.smin.i16.v4i16(<4 x i16> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V8 = call i16 @llvm.experimental.vector.reduce.smin.i16.v8i16(<8 x i16> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.smin.i16.v16i16(<16 x i16> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V32 = call i16 @llvm.experimental.vector.reduce.smin.i16.v32i16(<32 x i16> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %V64 = call i16 @llvm.experimental.vector.reduce.smin.i16.v64i16(<64 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.smin.i16.v4i16(<4 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V8 = call i16 @llvm.experimental.vector.reduce.smin.i16.v8i16(<8 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.smin.i16.v16i16(<16 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %V32 = call i16 @llvm.experimental.vector.reduce.smin.i16.v32i16(<32 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V64 = call i16 @llvm.experimental.vector.reduce.smin.i16.v64i16(<64 x i16> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSSE3-LABEL: 'reduce_i16'
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.smin.i16.v4i16(<4 x i16> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V8 = call i16 @llvm.experimental.vector.reduce.smin.i16.v8i16(<8 x i16> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.smin.i16.v16i16(<16 x i16> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V32 = call i16 @llvm.experimental.vector.reduce.smin.i16.v32i16(<32 x i16> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V64 = call i16 @llvm.experimental.vector.reduce.smin.i16.v64i16(<64 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.smin.i16.v4i16(<4 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8 = call i16 @llvm.experimental.vector.reduce.smin.i16.v8i16(<8 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.smin.i16.v16i16(<16 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V32 = call i16 @llvm.experimental.vector.reduce.smin.i16.v32i16(<32 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 44 for instruction: %V64 = call i16 @llvm.experimental.vector.reduce.smin.i16.v64i16(<64 x i16> undef)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSE42-LABEL: 'reduce_i16'
@@ -150,9 +150,9 @@ define i32 @reduce_i16(i32 %arg) {
 ; AVX1-LABEL: 'reduce_i16'
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.smin.i16.v4i16(<4 x i16> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8 = call i16 @llvm.experimental.vector.reduce.smin.i16.v8i16(<8 x i16> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 53 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.smin.i16.v16i16(<16 x i16> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 58 for instruction: %V32 = call i16 @llvm.experimental.vector.reduce.smin.i16.v32i16(<32 x i16> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %V64 = call i16 @llvm.experimental.vector.reduce.smin.i16.v64i16(<64 x i16> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 61 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.smin.i16.v16i16(<16 x i16> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %V32 = call i16 @llvm.experimental.vector.reduce.smin.i16.v32i16(<32 x i16> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 82 for instruction: %V64 = call i16 @llvm.experimental.vector.reduce.smin.i16.v64i16(<64 x i16> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX2-LABEL: 'reduce_i16'
@@ -197,23 +197,23 @@ define i32 @reduce_i16(i32 %arg) {
 
 define i32 @reduce_i8(i32 %arg) {
 ; SSE2-LABEL: 'reduce_i8'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.smin.i8.v2i8(<2 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.smin.i8.v4i8(<4 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 37 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.smin.i8.v8i8(<8 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 49 for instruction: %V16 = call i8 @llvm.experimental.vector.reduce.smin.i8.v16i8(<16 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.smin.i8.v32i8(<32 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 55 for instruction: %V64 = call i8 @llvm.experimental.vector.reduce.smin.i8.v64i8(<64 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 63 for instruction: %V128 = call i8 @llvm.experimental.vector.reduce.smin.i8.v128i8(<128 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.smin.i8.v2i8(<2 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 29 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.smin.i8.v4i8(<4 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 43 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.smin.i8.v8i8(<8 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 57 for instruction: %V16 = call i8 @llvm.experimental.vector.reduce.smin.i8.v16i8(<16 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 61 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.smin.i8.v32i8(<32 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 69 for instruction: %V64 = call i8 @llvm.experimental.vector.reduce.smin.i8.v64i8(<64 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 85 for instruction: %V128 = call i8 @llvm.experimental.vector.reduce.smin.i8.v128i8(<128 x i8> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSSE3-LABEL: 'reduce_i8'
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.smin.i8.v2i8(<2 x i8> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.smin.i8.v4i8(<4 x i8> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.smin.i8.v8i8(<8 x i8> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V16 = call i8 @llvm.experimental.vector.reduce.smin.i8.v16i8(<16 x i8> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.smin.i8.v32i8(<32 x i8> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V64 = call i8 @llvm.experimental.vector.reduce.smin.i8.v64i8(<64 x i8> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %V128 = call i8 @llvm.experimental.vector.reduce.smin.i8.v128i8(<128 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.smin.i8.v2i8(<2 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.smin.i8.v4i8(<4 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.smin.i8.v8i8(<8 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %V16 = call i8 @llvm.experimental.vector.reduce.smin.i8.v16i8(<16 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.smin.i8.v32i8(<32 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %V64 = call i8 @llvm.experimental.vector.reduce.smin.i8.v64i8(<64 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 49 for instruction: %V128 = call i8 @llvm.experimental.vector.reduce.smin.i8.v128i8(<128 x i8> undef)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSE42-LABEL: 'reduce_i8'
@@ -231,9 +231,9 @@ define i32 @reduce_i8(i32 %arg) {
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.smin.i8.v4i8(<4 x i8> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.smin.i8.v8i8(<8 x i8> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V16 = call i8 @llvm.experimental.vector.reduce.smin.i8.v16i8(<16 x i8> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.smin.i8.v32i8(<32 x i8> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 71 for instruction: %V64 = call i8 @llvm.experimental.vector.reduce.smin.i8.v64i8(<64 x i8> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 81 for instruction: %V128 = call i8 @llvm.experimental.vector.reduce.smin.i8.v128i8(<128 x i8> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 76 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.smin.i8.v32i8(<32 x i8> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 83 for instruction: %V64 = call i8 @llvm.experimental.vector.reduce.smin.i8.v64i8(<64 x i8> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 97 for instruction: %V128 = call i8 @llvm.experimental.vector.reduce.smin.i8.v128i8(<128 x i8> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX2-LABEL: 'reduce_i8'
diff --git a/test/Analysis/CostModel/X86/reduce-smin.ll b/test/Analysis/CostModel/X86/reduce-smin.ll
index f420ad736ed35820afd271489c81be1b96f45c29..0d9adef8566550a3a282414040410f51b0072015 100644
--- a/test/Analysis/CostModel/X86/reduce-smin.ll
+++ b/test/Analysis/CostModel/X86/reduce-smin.ll
@@ -11,18 +11,18 @@
 define i32 @reduce_i64(i32 %arg) {
 ; SSE2-LABEL: 'reduce_i64'
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1 = call i64 @llvm.experimental.vector.reduce.smin.i64.v1i64(<1 x i64> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V2 = call i64 @llvm.experimental.vector.reduce.smin.i64.v2i64(<2 x i64> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V4 = call i64 @llvm.experimental.vector.reduce.smin.i64.v4i64(<4 x i64> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 38 for instruction: %V8 = call i64 @llvm.experimental.vector.reduce.smin.i64.v8i64(<8 x i64> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 74 for instruction: %V16 = call i64 @llvm.experimental.vector.reduce.smin.i64.v16i64(<16 x i64> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V2 = call i64 @llvm.experimental.vector.reduce.smin.i64.v2i64(<2 x i64> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V4 = call i64 @llvm.experimental.vector.reduce.smin.i64.v4i64(<4 x i64> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 46 for instruction: %V8 = call i64 @llvm.experimental.vector.reduce.smin.i64.v8i64(<8 x i64> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 90 for instruction: %V16 = call i64 @llvm.experimental.vector.reduce.smin.i64.v16i64(<16 x i64> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSSE3-LABEL: 'reduce_i64'
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1 = call i64 @llvm.experimental.vector.reduce.smin.i64.v1i64(<1 x i64> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V2 = call i64 @llvm.experimental.vector.reduce.smin.i64.v2i64(<2 x i64> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V4 = call i64 @llvm.experimental.vector.reduce.smin.i64.v4i64(<4 x i64> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 38 for instruction: %V8 = call i64 @llvm.experimental.vector.reduce.smin.i64.v8i64(<8 x i64> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 74 for instruction: %V16 = call i64 @llvm.experimental.vector.reduce.smin.i64.v16i64(<16 x i64> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V2 = call i64 @llvm.experimental.vector.reduce.smin.i64.v2i64(<2 x i64> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V4 = call i64 @llvm.experimental.vector.reduce.smin.i64.v4i64(<4 x i64> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 46 for instruction: %V8 = call i64 @llvm.experimental.vector.reduce.smin.i64.v8i64(<8 x i64> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 90 for instruction: %V16 = call i64 @llvm.experimental.vector.reduce.smin.i64.v16i64(<16 x i64> undef)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSE42-LABEL: 'reduce_i64'
@@ -67,19 +67,19 @@ define i32 @reduce_i64(i32 %arg) {
 
 define i32 @reduce_i32(i32 %arg) {
 ; SSE2-LABEL: 'reduce_i32'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V2 = call i32 @llvm.experimental.vector.reduce.smin.i32.v2i32(<2 x i32> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4 = call i32 @llvm.experimental.vector.reduce.smin.i32.v4i32(<4 x i32> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V8 = call i32 @llvm.experimental.vector.reduce.smin.i32.v8i32(<8 x i32> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V16 = call i32 @llvm.experimental.vector.reduce.smin.i32.v16i32(<16 x i32> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %V32 = call i32 @llvm.experimental.vector.reduce.smin.i32.v32i32(<32 x i32> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V2 = call i32 @llvm.experimental.vector.reduce.smin.i32.v2i32(<2 x i32> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V4 = call i32 @llvm.experimental.vector.reduce.smin.i32.v4i32(<4 x i32> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V8 = call i32 @llvm.experimental.vector.reduce.smin.i32.v8i32(<8 x i32> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %V16 = call i32 @llvm.experimental.vector.reduce.smin.i32.v16i32(<16 x i32> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 39 for instruction: %V32 = call i32 @llvm.experimental.vector.reduce.smin.i32.v32i32(<32 x i32> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSSE3-LABEL: 'reduce_i32'
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V2 = call i32 @llvm.experimental.vector.reduce.smin.i32.v2i32(<2 x i32> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4 = call i32 @llvm.experimental.vector.reduce.smin.i32.v4i32(<4 x i32> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V8 = call i32 @llvm.experimental.vector.reduce.smin.i32.v8i32(<8 x i32> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V16 = call i32 @llvm.experimental.vector.reduce.smin.i32.v16i32(<16 x i32> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %V32 = call i32 @llvm.experimental.vector.reduce.smin.i32.v32i32(<32 x i32> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V2 = call i32 @llvm.experimental.vector.reduce.smin.i32.v2i32(<2 x i32> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V4 = call i32 @llvm.experimental.vector.reduce.smin.i32.v4i32(<4 x i32> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V8 = call i32 @llvm.experimental.vector.reduce.smin.i32.v8i32(<8 x i32> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %V16 = call i32 @llvm.experimental.vector.reduce.smin.i32.v16i32(<16 x i32> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 39 for instruction: %V32 = call i32 @llvm.experimental.vector.reduce.smin.i32.v32i32(<32 x i32> undef)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSE42-LABEL: 'reduce_i32'
@@ -124,21 +124,21 @@ define i32 @reduce_i32(i32 %arg) {
 
 define i32 @reduce_i16(i32 %arg) {
 ; SSE2-LABEL: 'reduce_i16'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.smin.i16.v2i16(<2 x i16> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.smin.i16.v4i16(<4 x i16> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V8 = call i16 @llvm.experimental.vector.reduce.smin.i16.v8i16(<8 x i16> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.smin.i16.v16i16(<16 x i16> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V32 = call i16 @llvm.experimental.vector.reduce.smin.i16.v32i16(<32 x i16> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %V64 = call i16 @llvm.experimental.vector.reduce.smin.i16.v64i16(<64 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.smin.i16.v2i16(<2 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.smin.i16.v4i16(<4 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V8 = call i16 @llvm.experimental.vector.reduce.smin.i16.v8i16(<8 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.smin.i16.v16i16(<16 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %V32 = call i16 @llvm.experimental.vector.reduce.smin.i16.v32i16(<32 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V64 = call i16 @llvm.experimental.vector.reduce.smin.i16.v64i16(<64 x i16> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSSE3-LABEL: 'reduce_i16'
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.smin.i16.v2i16(<2 x i16> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.smin.i16.v4i16(<4 x i16> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V8 = call i16 @llvm.experimental.vector.reduce.smin.i16.v8i16(<8 x i16> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.smin.i16.v16i16(<16 x i16> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V32 = call i16 @llvm.experimental.vector.reduce.smin.i16.v32i16(<32 x i16> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V64 = call i16 @llvm.experimental.vector.reduce.smin.i16.v64i16(<64 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.smin.i16.v2i16(<2 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.smin.i16.v4i16(<4 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8 = call i16 @llvm.experimental.vector.reduce.smin.i16.v8i16(<8 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.smin.i16.v16i16(<16 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V32 = call i16 @llvm.experimental.vector.reduce.smin.i16.v32i16(<32 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 44 for instruction: %V64 = call i16 @llvm.experimental.vector.reduce.smin.i16.v64i16(<64 x i16> undef)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSE42-LABEL: 'reduce_i16'
@@ -154,9 +154,9 @@ define i32 @reduce_i16(i32 %arg) {
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.smin.i16.v2i16(<2 x i16> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.smin.i16.v4i16(<4 x i16> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8 = call i16 @llvm.experimental.vector.reduce.smin.i16.v8i16(<8 x i16> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 53 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.smin.i16.v16i16(<16 x i16> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 58 for instruction: %V32 = call i16 @llvm.experimental.vector.reduce.smin.i16.v32i16(<32 x i16> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %V64 = call i16 @llvm.experimental.vector.reduce.smin.i16.v64i16(<64 x i16> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 61 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.smin.i16.v16i16(<16 x i16> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %V32 = call i16 @llvm.experimental.vector.reduce.smin.i16.v32i16(<32 x i16> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 82 for instruction: %V64 = call i16 @llvm.experimental.vector.reduce.smin.i16.v64i16(<64 x i16> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX2-LABEL: 'reduce_i16'
@@ -206,23 +206,23 @@ define i32 @reduce_i16(i32 %arg) {
 
 define i32 @reduce_i8(i32 %arg) {
 ; SSE2-LABEL: 'reduce_i8'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.smin.i8.v2i8(<2 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.smin.i8.v4i8(<4 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.smin.i8.v8i8(<8 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 49 for instruction: %V16 = call i8 @llvm.experimental.vector.reduce.smin.i8.v16i8(<16 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.smin.i8.v32i8(<32 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 55 for instruction: %V64 = call i8 @llvm.experimental.vector.reduce.smin.i8.v64i8(<64 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 63 for instruction: %V128 = call i8 @llvm.experimental.vector.reduce.smin.i8.v128i8(<128 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.smin.i8.v2i8(<2 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.smin.i8.v4i8(<4 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.smin.i8.v8i8(<8 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 57 for instruction: %V16 = call i8 @llvm.experimental.vector.reduce.smin.i8.v16i8(<16 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 61 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.smin.i8.v32i8(<32 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 69 for instruction: %V64 = call i8 @llvm.experimental.vector.reduce.smin.i8.v64i8(<64 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 85 for instruction: %V128 = call i8 @llvm.experimental.vector.reduce.smin.i8.v128i8(<128 x i8> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSSE3-LABEL: 'reduce_i8'
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.smin.i8.v2i8(<2 x i8> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.smin.i8.v4i8(<4 x i8> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.smin.i8.v8i8(<8 x i8> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V16 = call i8 @llvm.experimental.vector.reduce.smin.i8.v16i8(<16 x i8> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.smin.i8.v32i8(<32 x i8> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V64 = call i8 @llvm.experimental.vector.reduce.smin.i8.v64i8(<64 x i8> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %V128 = call i8 @llvm.experimental.vector.reduce.smin.i8.v128i8(<128 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.smin.i8.v2i8(<2 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.smin.i8.v4i8(<4 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.smin.i8.v8i8(<8 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %V16 = call i8 @llvm.experimental.vector.reduce.smin.i8.v16i8(<16 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.smin.i8.v32i8(<32 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %V64 = call i8 @llvm.experimental.vector.reduce.smin.i8.v64i8(<64 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 49 for instruction: %V128 = call i8 @llvm.experimental.vector.reduce.smin.i8.v128i8(<128 x i8> undef)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSE42-LABEL: 'reduce_i8'
@@ -240,9 +240,9 @@ define i32 @reduce_i8(i32 %arg) {
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.smin.i8.v4i8(<4 x i8> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.smin.i8.v8i8(<8 x i8> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V16 = call i8 @llvm.experimental.vector.reduce.smin.i8.v16i8(<16 x i8> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.smin.i8.v32i8(<32 x i8> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 71 for instruction: %V64 = call i8 @llvm.experimental.vector.reduce.smin.i8.v64i8(<64 x i8> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 81 for instruction: %V128 = call i8 @llvm.experimental.vector.reduce.smin.i8.v128i8(<128 x i8> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 76 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.smin.i8.v32i8(<32 x i8> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 83 for instruction: %V64 = call i8 @llvm.experimental.vector.reduce.smin.i8.v64i8(<64 x i8> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 97 for instruction: %V128 = call i8 @llvm.experimental.vector.reduce.smin.i8.v128i8(<128 x i8> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX2-LABEL: 'reduce_i8'
diff --git a/test/Analysis/CostModel/X86/reduce-umax-widen.ll b/test/Analysis/CostModel/X86/reduce-umax-widen.ll
index c48cfdf9778bec8855b978b5988e4f6c84249abe..c3ef8fdd5465fabbb4403bb57e267a8863ebe208 100644
--- a/test/Analysis/CostModel/X86/reduce-umax-widen.ll
+++ b/test/Analysis/CostModel/X86/reduce-umax-widen.ll
@@ -11,18 +11,18 @@
 define i32 @reduce_i64(i32 %arg) {
 ; SSE2-LABEL: 'reduce_i64'
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1 = call i64 @llvm.experimental.vector.reduce.umax.i64.v1i64(<1 x i64> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V2 = call i64 @llvm.experimental.vector.reduce.umax.i64.v2i64(<2 x i64> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V4 = call i64 @llvm.experimental.vector.reduce.umax.i64.v4i64(<4 x i64> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 38 for instruction: %V8 = call i64 @llvm.experimental.vector.reduce.umax.i64.v8i64(<8 x i64> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 74 for instruction: %V16 = call i64 @llvm.experimental.vector.reduce.umax.i64.v16i64(<16 x i64> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V2 = call i64 @llvm.experimental.vector.reduce.umax.i64.v2i64(<2 x i64> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V4 = call i64 @llvm.experimental.vector.reduce.umax.i64.v4i64(<4 x i64> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 46 for instruction: %V8 = call i64 @llvm.experimental.vector.reduce.umax.i64.v8i64(<8 x i64> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 90 for instruction: %V16 = call i64 @llvm.experimental.vector.reduce.umax.i64.v16i64(<16 x i64> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSSE3-LABEL: 'reduce_i64'
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1 = call i64 @llvm.experimental.vector.reduce.umax.i64.v1i64(<1 x i64> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V2 = call i64 @llvm.experimental.vector.reduce.umax.i64.v2i64(<2 x i64> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V4 = call i64 @llvm.experimental.vector.reduce.umax.i64.v4i64(<4 x i64> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 38 for instruction: %V8 = call i64 @llvm.experimental.vector.reduce.umax.i64.v8i64(<8 x i64> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 74 for instruction: %V16 = call i64 @llvm.experimental.vector.reduce.umax.i64.v16i64(<16 x i64> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V2 = call i64 @llvm.experimental.vector.reduce.umax.i64.v2i64(<2 x i64> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V4 = call i64 @llvm.experimental.vector.reduce.umax.i64.v4i64(<4 x i64> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 46 for instruction: %V8 = call i64 @llvm.experimental.vector.reduce.umax.i64.v8i64(<8 x i64> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 90 for instruction: %V16 = call i64 @llvm.experimental.vector.reduce.umax.i64.v16i64(<16 x i64> undef)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSE42-LABEL: 'reduce_i64'
@@ -67,19 +67,19 @@ define i32 @reduce_i64(i32 %arg) {
 
 define i32 @reduce_i32(i32 %arg) {
 ; SSE2-LABEL: 'reduce_i32'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2 = call i32 @llvm.experimental.vector.reduce.umax.i32.v2i32(<2 x i32> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4 = call i32 @llvm.experimental.vector.reduce.umax.i32.v4i32(<4 x i32> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V8 = call i32 @llvm.experimental.vector.reduce.umax.i32.v8i32(<8 x i32> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V16 = call i32 @llvm.experimental.vector.reduce.umax.i32.v16i32(<16 x i32> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %V32 = call i32 @llvm.experimental.vector.reduce.umax.i32.v32i32(<32 x i32> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2 = call i32 @llvm.experimental.vector.reduce.umax.i32.v2i32(<2 x i32> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V4 = call i32 @llvm.experimental.vector.reduce.umax.i32.v4i32(<4 x i32> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V8 = call i32 @llvm.experimental.vector.reduce.umax.i32.v8i32(<8 x i32> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %V16 = call i32 @llvm.experimental.vector.reduce.umax.i32.v16i32(<16 x i32> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 39 for instruction: %V32 = call i32 @llvm.experimental.vector.reduce.umax.i32.v32i32(<32 x i32> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSSE3-LABEL: 'reduce_i32'
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2 = call i32 @llvm.experimental.vector.reduce.umax.i32.v2i32(<2 x i32> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4 = call i32 @llvm.experimental.vector.reduce.umax.i32.v4i32(<4 x i32> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V8 = call i32 @llvm.experimental.vector.reduce.umax.i32.v8i32(<8 x i32> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V16 = call i32 @llvm.experimental.vector.reduce.umax.i32.v16i32(<16 x i32> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %V32 = call i32 @llvm.experimental.vector.reduce.umax.i32.v32i32(<32 x i32> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2 = call i32 @llvm.experimental.vector.reduce.umax.i32.v2i32(<2 x i32> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V4 = call i32 @llvm.experimental.vector.reduce.umax.i32.v4i32(<4 x i32> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V8 = call i32 @llvm.experimental.vector.reduce.umax.i32.v8i32(<8 x i32> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %V16 = call i32 @llvm.experimental.vector.reduce.umax.i32.v16i32(<16 x i32> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 39 for instruction: %V32 = call i32 @llvm.experimental.vector.reduce.umax.i32.v32i32(<32 x i32> undef)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSE42-LABEL: 'reduce_i32'
@@ -124,21 +124,21 @@ define i32 @reduce_i32(i32 %arg) {
 
 define i32 @reduce_i16(i32 %arg) {
 ; SSE2-LABEL: 'reduce_i16'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.umax.i16.v2i16(<2 x i16> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.umax.i16.v4i16(<4 x i16> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V8 = call i16 @llvm.experimental.vector.reduce.umax.i16.v8i16(<8 x i16> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.umax.i16.v16i16(<16 x i16> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V32 = call i16 @llvm.experimental.vector.reduce.umax.i16.v32i16(<32 x i16> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %V64 = call i16 @llvm.experimental.vector.reduce.umax.i16.v64i16(<64 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.umax.i16.v2i16(<2 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.umax.i16.v4i16(<4 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V8 = call i16 @llvm.experimental.vector.reduce.umax.i16.v8i16(<8 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.umax.i16.v16i16(<16 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %V32 = call i16 @llvm.experimental.vector.reduce.umax.i16.v32i16(<32 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V64 = call i16 @llvm.experimental.vector.reduce.umax.i16.v64i16(<64 x i16> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSSE3-LABEL: 'reduce_i16'
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.umax.i16.v2i16(<2 x i16> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.umax.i16.v4i16(<4 x i16> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V8 = call i16 @llvm.experimental.vector.reduce.umax.i16.v8i16(<8 x i16> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.umax.i16.v16i16(<16 x i16> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V32 = call i16 @llvm.experimental.vector.reduce.umax.i16.v32i16(<32 x i16> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V64 = call i16 @llvm.experimental.vector.reduce.umax.i16.v64i16(<64 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.umax.i16.v2i16(<2 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.umax.i16.v4i16(<4 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8 = call i16 @llvm.experimental.vector.reduce.umax.i16.v8i16(<8 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.umax.i16.v16i16(<16 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V32 = call i16 @llvm.experimental.vector.reduce.umax.i16.v32i16(<32 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 44 for instruction: %V64 = call i16 @llvm.experimental.vector.reduce.umax.i16.v64i16(<64 x i16> undef)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSE42-LABEL: 'reduce_i16'
@@ -154,9 +154,9 @@ define i32 @reduce_i16(i32 %arg) {
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.umax.i16.v2i16(<2 x i16> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.umax.i16.v4i16(<4 x i16> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8 = call i16 @llvm.experimental.vector.reduce.umax.i16.v8i16(<8 x i16> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 53 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.umax.i16.v16i16(<16 x i16> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 58 for instruction: %V32 = call i16 @llvm.experimental.vector.reduce.umax.i16.v32i16(<32 x i16> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %V64 = call i16 @llvm.experimental.vector.reduce.umax.i16.v64i16(<64 x i16> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 61 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.umax.i16.v16i16(<16 x i16> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %V32 = call i16 @llvm.experimental.vector.reduce.umax.i16.v32i16(<32 x i16> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 82 for instruction: %V64 = call i16 @llvm.experimental.vector.reduce.umax.i16.v64i16(<64 x i16> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX2-LABEL: 'reduce_i16'
@@ -206,23 +206,23 @@ define i32 @reduce_i16(i32 %arg) {
 
 define i32 @reduce_i8(i32 %arg) {
 ; SSE2-LABEL: 'reduce_i8'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.umax.i8.v2i8(<2 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.umax.i8.v4i8(<4 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 37 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.umax.i8.v8i8(<8 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 49 for instruction: %V16 = call i8 @llvm.experimental.vector.reduce.umax.i8.v16i8(<16 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.umax.i8.v32i8(<32 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 55 for instruction: %V64 = call i8 @llvm.experimental.vector.reduce.umax.i8.v64i8(<64 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 63 for instruction: %V128 = call i8 @llvm.experimental.vector.reduce.umax.i8.v128i8(<128 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.umax.i8.v2i8(<2 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 29 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.umax.i8.v4i8(<4 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 43 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.umax.i8.v8i8(<8 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 57 for instruction: %V16 = call i8 @llvm.experimental.vector.reduce.umax.i8.v16i8(<16 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 61 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.umax.i8.v32i8(<32 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 69 for instruction: %V64 = call i8 @llvm.experimental.vector.reduce.umax.i8.v64i8(<64 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 85 for instruction: %V128 = call i8 @llvm.experimental.vector.reduce.umax.i8.v128i8(<128 x i8> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSSE3-LABEL: 'reduce_i8'
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.umax.i8.v2i8(<2 x i8> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.umax.i8.v4i8(<4 x i8> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.umax.i8.v8i8(<8 x i8> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V16 = call i8 @llvm.experimental.vector.reduce.umax.i8.v16i8(<16 x i8> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.umax.i8.v32i8(<32 x i8> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V64 = call i8 @llvm.experimental.vector.reduce.umax.i8.v64i8(<64 x i8> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %V128 = call i8 @llvm.experimental.vector.reduce.umax.i8.v128i8(<128 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.umax.i8.v2i8(<2 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.umax.i8.v4i8(<4 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.umax.i8.v8i8(<8 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %V16 = call i8 @llvm.experimental.vector.reduce.umax.i8.v16i8(<16 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.umax.i8.v32i8(<32 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %V64 = call i8 @llvm.experimental.vector.reduce.umax.i8.v64i8(<64 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 49 for instruction: %V128 = call i8 @llvm.experimental.vector.reduce.umax.i8.v128i8(<128 x i8> undef)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSE42-LABEL: 'reduce_i8'
@@ -240,9 +240,9 @@ define i32 @reduce_i8(i32 %arg) {
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.umax.i8.v4i8(<4 x i8> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.umax.i8.v8i8(<8 x i8> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V16 = call i8 @llvm.experimental.vector.reduce.umax.i8.v16i8(<16 x i8> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.umax.i8.v32i8(<32 x i8> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 71 for instruction: %V64 = call i8 @llvm.experimental.vector.reduce.umax.i8.v64i8(<64 x i8> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 81 for instruction: %V128 = call i8 @llvm.experimental.vector.reduce.umax.i8.v128i8(<128 x i8> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 76 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.umax.i8.v32i8(<32 x i8> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 83 for instruction: %V64 = call i8 @llvm.experimental.vector.reduce.umax.i8.v64i8(<64 x i8> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 97 for instruction: %V128 = call i8 @llvm.experimental.vector.reduce.umax.i8.v128i8(<128 x i8> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX2-LABEL: 'reduce_i8'
diff --git a/test/Analysis/CostModel/X86/reduce-umax.ll b/test/Analysis/CostModel/X86/reduce-umax.ll
index 490b0e469b1a3264cfca75f7c25b2e48116eb610..e0a5ab12a39ff5c9a2499dc131a77ce33362c93b 100644
--- a/test/Analysis/CostModel/X86/reduce-umax.ll
+++ b/test/Analysis/CostModel/X86/reduce-umax.ll
@@ -11,18 +11,18 @@
 define i32 @reduce_i64(i32 %arg) {
 ; SSE2-LABEL: 'reduce_i64'
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1 = call i64 @llvm.experimental.vector.reduce.umax.i64.v1i64(<1 x i64> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V2 = call i64 @llvm.experimental.vector.reduce.umax.i64.v2i64(<2 x i64> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V4 = call i64 @llvm.experimental.vector.reduce.umax.i64.v4i64(<4 x i64> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 38 for instruction: %V8 = call i64 @llvm.experimental.vector.reduce.umax.i64.v8i64(<8 x i64> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 74 for instruction: %V16 = call i64 @llvm.experimental.vector.reduce.umax.i64.v16i64(<16 x i64> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V2 = call i64 @llvm.experimental.vector.reduce.umax.i64.v2i64(<2 x i64> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V4 = call i64 @llvm.experimental.vector.reduce.umax.i64.v4i64(<4 x i64> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 46 for instruction: %V8 = call i64 @llvm.experimental.vector.reduce.umax.i64.v8i64(<8 x i64> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 90 for instruction: %V16 = call i64 @llvm.experimental.vector.reduce.umax.i64.v16i64(<16 x i64> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSSE3-LABEL: 'reduce_i64'
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1 = call i64 @llvm.experimental.vector.reduce.umax.i64.v1i64(<1 x i64> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V2 = call i64 @llvm.experimental.vector.reduce.umax.i64.v2i64(<2 x i64> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V4 = call i64 @llvm.experimental.vector.reduce.umax.i64.v4i64(<4 x i64> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 38 for instruction: %V8 = call i64 @llvm.experimental.vector.reduce.umax.i64.v8i64(<8 x i64> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 74 for instruction: %V16 = call i64 @llvm.experimental.vector.reduce.umax.i64.v16i64(<16 x i64> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V2 = call i64 @llvm.experimental.vector.reduce.umax.i64.v2i64(<2 x i64> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V4 = call i64 @llvm.experimental.vector.reduce.umax.i64.v4i64(<4 x i64> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 46 for instruction: %V8 = call i64 @llvm.experimental.vector.reduce.umax.i64.v8i64(<8 x i64> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 90 for instruction: %V16 = call i64 @llvm.experimental.vector.reduce.umax.i64.v16i64(<16 x i64> undef)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSE42-LABEL: 'reduce_i64'
@@ -67,19 +67,19 @@ define i32 @reduce_i64(i32 %arg) {
 
 define i32 @reduce_i32(i32 %arg) {
 ; SSE2-LABEL: 'reduce_i32'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V2 = call i32 @llvm.experimental.vector.reduce.umax.i32.v2i32(<2 x i32> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4 = call i32 @llvm.experimental.vector.reduce.umax.i32.v4i32(<4 x i32> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V8 = call i32 @llvm.experimental.vector.reduce.umax.i32.v8i32(<8 x i32> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V16 = call i32 @llvm.experimental.vector.reduce.umax.i32.v16i32(<16 x i32> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %V32 = call i32 @llvm.experimental.vector.reduce.umax.i32.v32i32(<32 x i32> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V2 = call i32 @llvm.experimental.vector.reduce.umax.i32.v2i32(<2 x i32> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V4 = call i32 @llvm.experimental.vector.reduce.umax.i32.v4i32(<4 x i32> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V8 = call i32 @llvm.experimental.vector.reduce.umax.i32.v8i32(<8 x i32> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %V16 = call i32 @llvm.experimental.vector.reduce.umax.i32.v16i32(<16 x i32> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 39 for instruction: %V32 = call i32 @llvm.experimental.vector.reduce.umax.i32.v32i32(<32 x i32> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSSE3-LABEL: 'reduce_i32'
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V2 = call i32 @llvm.experimental.vector.reduce.umax.i32.v2i32(<2 x i32> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4 = call i32 @llvm.experimental.vector.reduce.umax.i32.v4i32(<4 x i32> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V8 = call i32 @llvm.experimental.vector.reduce.umax.i32.v8i32(<8 x i32> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V16 = call i32 @llvm.experimental.vector.reduce.umax.i32.v16i32(<16 x i32> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %V32 = call i32 @llvm.experimental.vector.reduce.umax.i32.v32i32(<32 x i32> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V2 = call i32 @llvm.experimental.vector.reduce.umax.i32.v2i32(<2 x i32> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V4 = call i32 @llvm.experimental.vector.reduce.umax.i32.v4i32(<4 x i32> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V8 = call i32 @llvm.experimental.vector.reduce.umax.i32.v8i32(<8 x i32> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %V16 = call i32 @llvm.experimental.vector.reduce.umax.i32.v16i32(<16 x i32> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 39 for instruction: %V32 = call i32 @llvm.experimental.vector.reduce.umax.i32.v32i32(<32 x i32> undef)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSE42-LABEL: 'reduce_i32'
@@ -124,21 +124,21 @@ define i32 @reduce_i32(i32 %arg) {
 
 define i32 @reduce_i16(i32 %arg) {
 ; SSE2-LABEL: 'reduce_i16'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.umax.i16.v2i16(<2 x i16> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.umax.i16.v4i16(<4 x i16> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V8 = call i16 @llvm.experimental.vector.reduce.umax.i16.v8i16(<8 x i16> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.umax.i16.v16i16(<16 x i16> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V32 = call i16 @llvm.experimental.vector.reduce.umax.i16.v32i16(<32 x i16> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %V64 = call i16 @llvm.experimental.vector.reduce.umax.i16.v64i16(<64 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.umax.i16.v2i16(<2 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.umax.i16.v4i16(<4 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V8 = call i16 @llvm.experimental.vector.reduce.umax.i16.v8i16(<8 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.umax.i16.v16i16(<16 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %V32 = call i16 @llvm.experimental.vector.reduce.umax.i16.v32i16(<32 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V64 = call i16 @llvm.experimental.vector.reduce.umax.i16.v64i16(<64 x i16> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSSE3-LABEL: 'reduce_i16'
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.umax.i16.v2i16(<2 x i16> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.umax.i16.v4i16(<4 x i16> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V8 = call i16 @llvm.experimental.vector.reduce.umax.i16.v8i16(<8 x i16> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.umax.i16.v16i16(<16 x i16> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V32 = call i16 @llvm.experimental.vector.reduce.umax.i16.v32i16(<32 x i16> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V64 = call i16 @llvm.experimental.vector.reduce.umax.i16.v64i16(<64 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.umax.i16.v2i16(<2 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.umax.i16.v4i16(<4 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8 = call i16 @llvm.experimental.vector.reduce.umax.i16.v8i16(<8 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.umax.i16.v16i16(<16 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V32 = call i16 @llvm.experimental.vector.reduce.umax.i16.v32i16(<32 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 44 for instruction: %V64 = call i16 @llvm.experimental.vector.reduce.umax.i16.v64i16(<64 x i16> undef)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSE42-LABEL: 'reduce_i16'
@@ -154,9 +154,9 @@ define i32 @reduce_i16(i32 %arg) {
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.umax.i16.v2i16(<2 x i16> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.umax.i16.v4i16(<4 x i16> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8 = call i16 @llvm.experimental.vector.reduce.umax.i16.v8i16(<8 x i16> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 53 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.umax.i16.v16i16(<16 x i16> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 58 for instruction: %V32 = call i16 @llvm.experimental.vector.reduce.umax.i16.v32i16(<32 x i16> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %V64 = call i16 @llvm.experimental.vector.reduce.umax.i16.v64i16(<64 x i16> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 61 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.umax.i16.v16i16(<16 x i16> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %V32 = call i16 @llvm.experimental.vector.reduce.umax.i16.v32i16(<32 x i16> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 82 for instruction: %V64 = call i16 @llvm.experimental.vector.reduce.umax.i16.v64i16(<64 x i16> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX2-LABEL: 'reduce_i16'
@@ -206,23 +206,23 @@ define i32 @reduce_i16(i32 %arg) {
 
 define i32 @reduce_i8(i32 %arg) {
 ; SSE2-LABEL: 'reduce_i8'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.umax.i8.v2i8(<2 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.umax.i8.v4i8(<4 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.umax.i8.v8i8(<8 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 49 for instruction: %V16 = call i8 @llvm.experimental.vector.reduce.umax.i8.v16i8(<16 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.umax.i8.v32i8(<32 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 55 for instruction: %V64 = call i8 @llvm.experimental.vector.reduce.umax.i8.v64i8(<64 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 63 for instruction: %V128 = call i8 @llvm.experimental.vector.reduce.umax.i8.v128i8(<128 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.umax.i8.v2i8(<2 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.umax.i8.v4i8(<4 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.umax.i8.v8i8(<8 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 57 for instruction: %V16 = call i8 @llvm.experimental.vector.reduce.umax.i8.v16i8(<16 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 61 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.umax.i8.v32i8(<32 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 69 for instruction: %V64 = call i8 @llvm.experimental.vector.reduce.umax.i8.v64i8(<64 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 85 for instruction: %V128 = call i8 @llvm.experimental.vector.reduce.umax.i8.v128i8(<128 x i8> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSSE3-LABEL: 'reduce_i8'
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.umax.i8.v2i8(<2 x i8> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.umax.i8.v4i8(<4 x i8> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.umax.i8.v8i8(<8 x i8> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V16 = call i8 @llvm.experimental.vector.reduce.umax.i8.v16i8(<16 x i8> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.umax.i8.v32i8(<32 x i8> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V64 = call i8 @llvm.experimental.vector.reduce.umax.i8.v64i8(<64 x i8> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %V128 = call i8 @llvm.experimental.vector.reduce.umax.i8.v128i8(<128 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.umax.i8.v2i8(<2 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.umax.i8.v4i8(<4 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.umax.i8.v8i8(<8 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %V16 = call i8 @llvm.experimental.vector.reduce.umax.i8.v16i8(<16 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.umax.i8.v32i8(<32 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %V64 = call i8 @llvm.experimental.vector.reduce.umax.i8.v64i8(<64 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 49 for instruction: %V128 = call i8 @llvm.experimental.vector.reduce.umax.i8.v128i8(<128 x i8> undef)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSE42-LABEL: 'reduce_i8'
@@ -240,9 +240,9 @@ define i32 @reduce_i8(i32 %arg) {
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.umax.i8.v4i8(<4 x i8> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.umax.i8.v8i8(<8 x i8> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V16 = call i8 @llvm.experimental.vector.reduce.umax.i8.v16i8(<16 x i8> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.umax.i8.v32i8(<32 x i8> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 71 for instruction: %V64 = call i8 @llvm.experimental.vector.reduce.umax.i8.v64i8(<64 x i8> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 81 for instruction: %V128 = call i8 @llvm.experimental.vector.reduce.umax.i8.v128i8(<128 x i8> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 76 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.umax.i8.v32i8(<32 x i8> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 83 for instruction: %V64 = call i8 @llvm.experimental.vector.reduce.umax.i8.v64i8(<64 x i8> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 97 for instruction: %V128 = call i8 @llvm.experimental.vector.reduce.umax.i8.v128i8(<128 x i8> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX2-LABEL: 'reduce_i8'
diff --git a/test/Analysis/CostModel/X86/reduce-umin-widen.ll b/test/Analysis/CostModel/X86/reduce-umin-widen.ll
index b59e2f3712ed5d1f8ade8d789b9dbd84305b45a0..71674ad6f785b20775d9477ed74b5712556db866 100644
--- a/test/Analysis/CostModel/X86/reduce-umin-widen.ll
+++ b/test/Analysis/CostModel/X86/reduce-umin-widen.ll
@@ -11,18 +11,18 @@
 define i32 @reduce_i64(i32 %arg) {
 ; SSE2-LABEL: 'reduce_i64'
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1 = call i64 @llvm.experimental.vector.reduce.umin.i64.v1i64(<1 x i64> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V2 = call i64 @llvm.experimental.vector.reduce.umin.i64.v2i64(<2 x i64> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V4 = call i64 @llvm.experimental.vector.reduce.umin.i64.v4i64(<4 x i64> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 38 for instruction: %V8 = call i64 @llvm.experimental.vector.reduce.umin.i64.v8i64(<8 x i64> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 74 for instruction: %V16 = call i64 @llvm.experimental.vector.reduce.umin.i64.v16i64(<16 x i64> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V2 = call i64 @llvm.experimental.vector.reduce.umin.i64.v2i64(<2 x i64> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V4 = call i64 @llvm.experimental.vector.reduce.umin.i64.v4i64(<4 x i64> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 46 for instruction: %V8 = call i64 @llvm.experimental.vector.reduce.umin.i64.v8i64(<8 x i64> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 90 for instruction: %V16 = call i64 @llvm.experimental.vector.reduce.umin.i64.v16i64(<16 x i64> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSSE3-LABEL: 'reduce_i64'
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1 = call i64 @llvm.experimental.vector.reduce.umin.i64.v1i64(<1 x i64> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V2 = call i64 @llvm.experimental.vector.reduce.umin.i64.v2i64(<2 x i64> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V4 = call i64 @llvm.experimental.vector.reduce.umin.i64.v4i64(<4 x i64> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 38 for instruction: %V8 = call i64 @llvm.experimental.vector.reduce.umin.i64.v8i64(<8 x i64> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 74 for instruction: %V16 = call i64 @llvm.experimental.vector.reduce.umin.i64.v16i64(<16 x i64> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V2 = call i64 @llvm.experimental.vector.reduce.umin.i64.v2i64(<2 x i64> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V4 = call i64 @llvm.experimental.vector.reduce.umin.i64.v4i64(<4 x i64> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 46 for instruction: %V8 = call i64 @llvm.experimental.vector.reduce.umin.i64.v8i64(<8 x i64> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 90 for instruction: %V16 = call i64 @llvm.experimental.vector.reduce.umin.i64.v16i64(<16 x i64> undef)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSE42-LABEL: 'reduce_i64'
@@ -67,19 +67,19 @@ define i32 @reduce_i64(i32 %arg) {
 
 define i32 @reduce_i32(i32 %arg) {
 ; SSE2-LABEL: 'reduce_i32'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2 = call i32 @llvm.experimental.vector.reduce.umin.i32.v2i32(<2 x i32> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4 = call i32 @llvm.experimental.vector.reduce.umin.i32.v4i32(<4 x i32> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V8 = call i32 @llvm.experimental.vector.reduce.umin.i32.v8i32(<8 x i32> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V16 = call i32 @llvm.experimental.vector.reduce.umin.i32.v16i32(<16 x i32> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %V32 = call i32 @llvm.experimental.vector.reduce.umin.i32.v32i32(<32 x i32> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2 = call i32 @llvm.experimental.vector.reduce.umin.i32.v2i32(<2 x i32> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V4 = call i32 @llvm.experimental.vector.reduce.umin.i32.v4i32(<4 x i32> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V8 = call i32 @llvm.experimental.vector.reduce.umin.i32.v8i32(<8 x i32> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %V16 = call i32 @llvm.experimental.vector.reduce.umin.i32.v16i32(<16 x i32> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 39 for instruction: %V32 = call i32 @llvm.experimental.vector.reduce.umin.i32.v32i32(<32 x i32> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSSE3-LABEL: 'reduce_i32'
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2 = call i32 @llvm.experimental.vector.reduce.umin.i32.v2i32(<2 x i32> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4 = call i32 @llvm.experimental.vector.reduce.umin.i32.v4i32(<4 x i32> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V8 = call i32 @llvm.experimental.vector.reduce.umin.i32.v8i32(<8 x i32> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V16 = call i32 @llvm.experimental.vector.reduce.umin.i32.v16i32(<16 x i32> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %V32 = call i32 @llvm.experimental.vector.reduce.umin.i32.v32i32(<32 x i32> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2 = call i32 @llvm.experimental.vector.reduce.umin.i32.v2i32(<2 x i32> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V4 = call i32 @llvm.experimental.vector.reduce.umin.i32.v4i32(<4 x i32> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V8 = call i32 @llvm.experimental.vector.reduce.umin.i32.v8i32(<8 x i32> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %V16 = call i32 @llvm.experimental.vector.reduce.umin.i32.v16i32(<16 x i32> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 39 for instruction: %V32 = call i32 @llvm.experimental.vector.reduce.umin.i32.v32i32(<32 x i32> undef)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSE42-LABEL: 'reduce_i32'
@@ -124,21 +124,21 @@ define i32 @reduce_i32(i32 %arg) {
 
 define i32 @reduce_i16(i32 %arg) {
 ; SSE2-LABEL: 'reduce_i16'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.umin.i16.v2i16(<2 x i16> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.umin.i16.v4i16(<4 x i16> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V8 = call i16 @llvm.experimental.vector.reduce.umin.i16.v8i16(<8 x i16> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.umin.i16.v16i16(<16 x i16> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V32 = call i16 @llvm.experimental.vector.reduce.umin.i16.v32i16(<32 x i16> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %V64 = call i16 @llvm.experimental.vector.reduce.umin.i16.v64i16(<64 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.umin.i16.v2i16(<2 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.umin.i16.v4i16(<4 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V8 = call i16 @llvm.experimental.vector.reduce.umin.i16.v8i16(<8 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.umin.i16.v16i16(<16 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %V32 = call i16 @llvm.experimental.vector.reduce.umin.i16.v32i16(<32 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V64 = call i16 @llvm.experimental.vector.reduce.umin.i16.v64i16(<64 x i16> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSSE3-LABEL: 'reduce_i16'
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.umin.i16.v2i16(<2 x i16> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.umin.i16.v4i16(<4 x i16> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V8 = call i16 @llvm.experimental.vector.reduce.umin.i16.v8i16(<8 x i16> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.umin.i16.v16i16(<16 x i16> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V32 = call i16 @llvm.experimental.vector.reduce.umin.i16.v32i16(<32 x i16> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V64 = call i16 @llvm.experimental.vector.reduce.umin.i16.v64i16(<64 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.umin.i16.v2i16(<2 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.umin.i16.v4i16(<4 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8 = call i16 @llvm.experimental.vector.reduce.umin.i16.v8i16(<8 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.umin.i16.v16i16(<16 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V32 = call i16 @llvm.experimental.vector.reduce.umin.i16.v32i16(<32 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 44 for instruction: %V64 = call i16 @llvm.experimental.vector.reduce.umin.i16.v64i16(<64 x i16> undef)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSE42-LABEL: 'reduce_i16'
@@ -154,9 +154,9 @@ define i32 @reduce_i16(i32 %arg) {
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.umin.i16.v2i16(<2 x i16> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.umin.i16.v4i16(<4 x i16> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8 = call i16 @llvm.experimental.vector.reduce.umin.i16.v8i16(<8 x i16> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 53 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.umin.i16.v16i16(<16 x i16> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 58 for instruction: %V32 = call i16 @llvm.experimental.vector.reduce.umin.i16.v32i16(<32 x i16> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %V64 = call i16 @llvm.experimental.vector.reduce.umin.i16.v64i16(<64 x i16> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 61 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.umin.i16.v16i16(<16 x i16> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %V32 = call i16 @llvm.experimental.vector.reduce.umin.i16.v32i16(<32 x i16> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 82 for instruction: %V64 = call i16 @llvm.experimental.vector.reduce.umin.i16.v64i16(<64 x i16> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX2-LABEL: 'reduce_i16'
@@ -206,23 +206,23 @@ define i32 @reduce_i16(i32 %arg) {
 
 define i32 @reduce_i8(i32 %arg) {
 ; SSE2-LABEL: 'reduce_i8'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.umin.i8.v2i8(<2 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.umin.i8.v4i8(<4 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 37 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.umin.i8.v8i8(<8 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 49 for instruction: %V16 = call i8 @llvm.experimental.vector.reduce.umin.i8.v16i8(<16 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.umin.i8.v32i8(<32 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 55 for instruction: %V64 = call i8 @llvm.experimental.vector.reduce.umin.i8.v64i8(<64 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 63 for instruction: %V128 = call i8 @llvm.experimental.vector.reduce.umin.i8.v128i8(<128 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.umin.i8.v2i8(<2 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 29 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.umin.i8.v4i8(<4 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 43 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.umin.i8.v8i8(<8 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 57 for instruction: %V16 = call i8 @llvm.experimental.vector.reduce.umin.i8.v16i8(<16 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 61 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.umin.i8.v32i8(<32 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 69 for instruction: %V64 = call i8 @llvm.experimental.vector.reduce.umin.i8.v64i8(<64 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 85 for instruction: %V128 = call i8 @llvm.experimental.vector.reduce.umin.i8.v128i8(<128 x i8> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSSE3-LABEL: 'reduce_i8'
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.umin.i8.v2i8(<2 x i8> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.umin.i8.v4i8(<4 x i8> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.umin.i8.v8i8(<8 x i8> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V16 = call i8 @llvm.experimental.vector.reduce.umin.i8.v16i8(<16 x i8> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.umin.i8.v32i8(<32 x i8> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V64 = call i8 @llvm.experimental.vector.reduce.umin.i8.v64i8(<64 x i8> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %V128 = call i8 @llvm.experimental.vector.reduce.umin.i8.v128i8(<128 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.umin.i8.v2i8(<2 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.umin.i8.v4i8(<4 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.umin.i8.v8i8(<8 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %V16 = call i8 @llvm.experimental.vector.reduce.umin.i8.v16i8(<16 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.umin.i8.v32i8(<32 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %V64 = call i8 @llvm.experimental.vector.reduce.umin.i8.v64i8(<64 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 49 for instruction: %V128 = call i8 @llvm.experimental.vector.reduce.umin.i8.v128i8(<128 x i8> undef)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSE42-LABEL: 'reduce_i8'
@@ -240,9 +240,9 @@ define i32 @reduce_i8(i32 %arg) {
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.umin.i8.v4i8(<4 x i8> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.umin.i8.v8i8(<8 x i8> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V16 = call i8 @llvm.experimental.vector.reduce.umin.i8.v16i8(<16 x i8> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.umin.i8.v32i8(<32 x i8> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 71 for instruction: %V64 = call i8 @llvm.experimental.vector.reduce.umin.i8.v64i8(<64 x i8> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 81 for instruction: %V128 = call i8 @llvm.experimental.vector.reduce.umin.i8.v128i8(<128 x i8> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 76 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.umin.i8.v32i8(<32 x i8> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 83 for instruction: %V64 = call i8 @llvm.experimental.vector.reduce.umin.i8.v64i8(<64 x i8> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 97 for instruction: %V128 = call i8 @llvm.experimental.vector.reduce.umin.i8.v128i8(<128 x i8> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX2-LABEL: 'reduce_i8'
diff --git a/test/Analysis/CostModel/X86/reduce-umin.ll b/test/Analysis/CostModel/X86/reduce-umin.ll
index f2f519c6023df5b4384c7c020d99bb1af28a4643..4a082150eb0a0749aa77dd50da7a59a153d8af1a 100644
--- a/test/Analysis/CostModel/X86/reduce-umin.ll
+++ b/test/Analysis/CostModel/X86/reduce-umin.ll
@@ -11,18 +11,18 @@
 define i32 @reduce_i64(i32 %arg) {
 ; SSE2-LABEL: 'reduce_i64'
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1 = call i64 @llvm.experimental.vector.reduce.umin.i64.v1i64(<1 x i64> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V2 = call i64 @llvm.experimental.vector.reduce.umin.i64.v2i64(<2 x i64> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V4 = call i64 @llvm.experimental.vector.reduce.umin.i64.v4i64(<4 x i64> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 38 for instruction: %V8 = call i64 @llvm.experimental.vector.reduce.umin.i64.v8i64(<8 x i64> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 74 for instruction: %V16 = call i64 @llvm.experimental.vector.reduce.umin.i64.v16i64(<16 x i64> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V2 = call i64 @llvm.experimental.vector.reduce.umin.i64.v2i64(<2 x i64> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V4 = call i64 @llvm.experimental.vector.reduce.umin.i64.v4i64(<4 x i64> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 46 for instruction: %V8 = call i64 @llvm.experimental.vector.reduce.umin.i64.v8i64(<8 x i64> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 90 for instruction: %V16 = call i64 @llvm.experimental.vector.reduce.umin.i64.v16i64(<16 x i64> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSSE3-LABEL: 'reduce_i64'
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V1 = call i64 @llvm.experimental.vector.reduce.umin.i64.v1i64(<1 x i64> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V2 = call i64 @llvm.experimental.vector.reduce.umin.i64.v2i64(<2 x i64> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V4 = call i64 @llvm.experimental.vector.reduce.umin.i64.v4i64(<4 x i64> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 38 for instruction: %V8 = call i64 @llvm.experimental.vector.reduce.umin.i64.v8i64(<8 x i64> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 74 for instruction: %V16 = call i64 @llvm.experimental.vector.reduce.umin.i64.v16i64(<16 x i64> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V2 = call i64 @llvm.experimental.vector.reduce.umin.i64.v2i64(<2 x i64> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V4 = call i64 @llvm.experimental.vector.reduce.umin.i64.v4i64(<4 x i64> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 46 for instruction: %V8 = call i64 @llvm.experimental.vector.reduce.umin.i64.v8i64(<8 x i64> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 90 for instruction: %V16 = call i64 @llvm.experimental.vector.reduce.umin.i64.v16i64(<16 x i64> undef)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSE42-LABEL: 'reduce_i64'
@@ -67,19 +67,19 @@ define i32 @reduce_i64(i32 %arg) {
 
 define i32 @reduce_i32(i32 %arg) {
 ; SSE2-LABEL: 'reduce_i32'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V2 = call i32 @llvm.experimental.vector.reduce.umin.i32.v2i32(<2 x i32> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4 = call i32 @llvm.experimental.vector.reduce.umin.i32.v4i32(<4 x i32> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V8 = call i32 @llvm.experimental.vector.reduce.umin.i32.v8i32(<8 x i32> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V16 = call i32 @llvm.experimental.vector.reduce.umin.i32.v16i32(<16 x i32> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %V32 = call i32 @llvm.experimental.vector.reduce.umin.i32.v32i32(<32 x i32> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V2 = call i32 @llvm.experimental.vector.reduce.umin.i32.v2i32(<2 x i32> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V4 = call i32 @llvm.experimental.vector.reduce.umin.i32.v4i32(<4 x i32> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V8 = call i32 @llvm.experimental.vector.reduce.umin.i32.v8i32(<8 x i32> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %V16 = call i32 @llvm.experimental.vector.reduce.umin.i32.v16i32(<16 x i32> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 39 for instruction: %V32 = call i32 @llvm.experimental.vector.reduce.umin.i32.v32i32(<32 x i32> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSSE3-LABEL: 'reduce_i32'
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V2 = call i32 @llvm.experimental.vector.reduce.umin.i32.v2i32(<2 x i32> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4 = call i32 @llvm.experimental.vector.reduce.umin.i32.v4i32(<4 x i32> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V8 = call i32 @llvm.experimental.vector.reduce.umin.i32.v8i32(<8 x i32> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V16 = call i32 @llvm.experimental.vector.reduce.umin.i32.v16i32(<16 x i32> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %V32 = call i32 @llvm.experimental.vector.reduce.umin.i32.v32i32(<32 x i32> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V2 = call i32 @llvm.experimental.vector.reduce.umin.i32.v2i32(<2 x i32> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V4 = call i32 @llvm.experimental.vector.reduce.umin.i32.v4i32(<4 x i32> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V8 = call i32 @llvm.experimental.vector.reduce.umin.i32.v8i32(<8 x i32> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %V16 = call i32 @llvm.experimental.vector.reduce.umin.i32.v16i32(<16 x i32> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 39 for instruction: %V32 = call i32 @llvm.experimental.vector.reduce.umin.i32.v32i32(<32 x i32> undef)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSE42-LABEL: 'reduce_i32'
@@ -124,21 +124,21 @@ define i32 @reduce_i32(i32 %arg) {
 
 define i32 @reduce_i16(i32 %arg) {
 ; SSE2-LABEL: 'reduce_i16'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.umin.i16.v2i16(<2 x i16> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.umin.i16.v4i16(<4 x i16> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V8 = call i16 @llvm.experimental.vector.reduce.umin.i16.v8i16(<8 x i16> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.umin.i16.v16i16(<16 x i16> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V32 = call i16 @llvm.experimental.vector.reduce.umin.i16.v32i16(<32 x i16> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %V64 = call i16 @llvm.experimental.vector.reduce.umin.i16.v64i16(<64 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.umin.i16.v2i16(<2 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.umin.i16.v4i16(<4 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V8 = call i16 @llvm.experimental.vector.reduce.umin.i16.v8i16(<8 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.umin.i16.v16i16(<16 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %V32 = call i16 @llvm.experimental.vector.reduce.umin.i16.v32i16(<32 x i16> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V64 = call i16 @llvm.experimental.vector.reduce.umin.i16.v64i16(<64 x i16> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSSE3-LABEL: 'reduce_i16'
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.umin.i16.v2i16(<2 x i16> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.umin.i16.v4i16(<4 x i16> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V8 = call i16 @llvm.experimental.vector.reduce.umin.i16.v8i16(<8 x i16> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.umin.i16.v16i16(<16 x i16> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V32 = call i16 @llvm.experimental.vector.reduce.umin.i16.v32i16(<32 x i16> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V64 = call i16 @llvm.experimental.vector.reduce.umin.i16.v64i16(<64 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.umin.i16.v2i16(<2 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.umin.i16.v4i16(<4 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8 = call i16 @llvm.experimental.vector.reduce.umin.i16.v8i16(<8 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.umin.i16.v16i16(<16 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V32 = call i16 @llvm.experimental.vector.reduce.umin.i16.v32i16(<32 x i16> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 44 for instruction: %V64 = call i16 @llvm.experimental.vector.reduce.umin.i16.v64i16(<64 x i16> undef)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSE42-LABEL: 'reduce_i16'
@@ -154,9 +154,9 @@ define i32 @reduce_i16(i32 %arg) {
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2 = call i16 @llvm.experimental.vector.reduce.umin.i16.v2i16(<2 x i16> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4 = call i16 @llvm.experimental.vector.reduce.umin.i16.v4i16(<4 x i16> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8 = call i16 @llvm.experimental.vector.reduce.umin.i16.v8i16(<8 x i16> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 53 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.umin.i16.v16i16(<16 x i16> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 58 for instruction: %V32 = call i16 @llvm.experimental.vector.reduce.umin.i16.v32i16(<32 x i16> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %V64 = call i16 @llvm.experimental.vector.reduce.umin.i16.v64i16(<64 x i16> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 61 for instruction: %V16 = call i16 @llvm.experimental.vector.reduce.umin.i16.v16i16(<16 x i16> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %V32 = call i16 @llvm.experimental.vector.reduce.umin.i16.v32i16(<32 x i16> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 82 for instruction: %V64 = call i16 @llvm.experimental.vector.reduce.umin.i16.v64i16(<64 x i16> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX2-LABEL: 'reduce_i16'
@@ -206,23 +206,23 @@ define i32 @reduce_i16(i32 %arg) {
 
 define i32 @reduce_i8(i32 %arg) {
 ; SSE2-LABEL: 'reduce_i8'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.umin.i8.v2i8(<2 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.umin.i8.v4i8(<4 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.umin.i8.v8i8(<8 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 49 for instruction: %V16 = call i8 @llvm.experimental.vector.reduce.umin.i8.v16i8(<16 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 51 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.umin.i8.v32i8(<32 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 55 for instruction: %V64 = call i8 @llvm.experimental.vector.reduce.umin.i8.v64i8(<64 x i8> undef)
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 63 for instruction: %V128 = call i8 @llvm.experimental.vector.reduce.umin.i8.v128i8(<128 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.umin.i8.v2i8(<2 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.umin.i8.v4i8(<4 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.umin.i8.v8i8(<8 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 57 for instruction: %V16 = call i8 @llvm.experimental.vector.reduce.umin.i8.v16i8(<16 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 61 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.umin.i8.v32i8(<32 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 69 for instruction: %V64 = call i8 @llvm.experimental.vector.reduce.umin.i8.v64i8(<64 x i8> undef)
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 85 for instruction: %V128 = call i8 @llvm.experimental.vector.reduce.umin.i8.v128i8(<128 x i8> undef)
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSSE3-LABEL: 'reduce_i8'
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.umin.i8.v2i8(<2 x i8> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.umin.i8.v4i8(<4 x i8> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.umin.i8.v8i8(<8 x i8> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V16 = call i8 @llvm.experimental.vector.reduce.umin.i8.v16i8(<16 x i8> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.umin.i8.v32i8(<32 x i8> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V64 = call i8 @llvm.experimental.vector.reduce.umin.i8.v64i8(<64 x i8> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %V128 = call i8 @llvm.experimental.vector.reduce.umin.i8.v128i8(<128 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V2 = call i8 @llvm.experimental.vector.reduce.umin.i8.v2i8(<2 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.umin.i8.v4i8(<4 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.umin.i8.v8i8(<8 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %V16 = call i8 @llvm.experimental.vector.reduce.umin.i8.v16i8(<16 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.umin.i8.v32i8(<32 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %V64 = call i8 @llvm.experimental.vector.reduce.umin.i8.v64i8(<64 x i8> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 49 for instruction: %V128 = call i8 @llvm.experimental.vector.reduce.umin.i8.v128i8(<128 x i8> undef)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSE42-LABEL: 'reduce_i8'
@@ -240,9 +240,9 @@ define i32 @reduce_i8(i32 %arg) {
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4 = call i8 @llvm.experimental.vector.reduce.umin.i8.v4i8(<4 x i8> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8 = call i8 @llvm.experimental.vector.reduce.umin.i8.v8i8(<8 x i8> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V16 = call i8 @llvm.experimental.vector.reduce.umin.i8.v16i8(<16 x i8> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.umin.i8.v32i8(<32 x i8> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 71 for instruction: %V64 = call i8 @llvm.experimental.vector.reduce.umin.i8.v64i8(<64 x i8> undef)
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 81 for instruction: %V128 = call i8 @llvm.experimental.vector.reduce.umin.i8.v128i8(<128 x i8> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 76 for instruction: %V32 = call i8 @llvm.experimental.vector.reduce.umin.i8.v32i8(<32 x i8> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 83 for instruction: %V64 = call i8 @llvm.experimental.vector.reduce.umin.i8.v64i8(<64 x i8> undef)
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 97 for instruction: %V128 = call i8 @llvm.experimental.vector.reduce.umin.i8.v128i8(<128 x i8> undef)
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX2-LABEL: 'reduce_i8'
diff --git a/test/Analysis/CostModel/X86/trunc.ll b/test/Analysis/CostModel/X86/trunc.ll
index e2b2cfa732d4193879272b9bf8bad830d343dcc7..9668da564133e8259b505ddf304aaaedb5c853fa 100644
--- a/test/Analysis/CostModel/X86/trunc.ll
+++ b/test/Analysis/CostModel/X86/trunc.ll
@@ -16,35 +16,41 @@ define i32 @trunc_vXi32() {
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i64 = trunc <2 x i64> undef to <2 x i32>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i64 = trunc <4 x i64> undef to <4 x i32>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8i64 = trunc <8 x i64> undef to <8 x i32>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V16i64 = trunc <16 x i64> undef to <16 x i32>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX1-LABEL: 'trunc_vXi32'
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i64 = trunc <2 x i64> undef to <2 x i32>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4i64 = trunc <4 x i64> undef to <4 x i32>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V8i64 = trunc <8 x i64> undef to <8 x i32>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V16i64 = trunc <16 x i64> undef to <16 x i32>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX2-LABEL: 'trunc_vXi32'
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i64 = trunc <2 x i64> undef to <2 x i32>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4i64 = trunc <4 x i64> undef to <4 x i32>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8i64 = trunc <8 x i64> undef to <8 x i32>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V16i64 = trunc <16 x i64> undef to <16 x i32>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX512-LABEL: 'trunc_vXi32'
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i64 = trunc <2 x i64> undef to <2 x i32>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4i64 = trunc <4 x i64> undef to <4 x i32>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8i64 = trunc <8 x i64> undef to <8 x i32>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16i64 = trunc <16 x i64> undef to <16 x i32>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; BTVER2-LABEL: 'trunc_vXi32'
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i64 = trunc <2 x i64> undef to <2 x i32>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4i64 = trunc <4 x i64> undef to <4 x i32>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V8i64 = trunc <8 x i64> undef to <8 x i32>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V16i64 = trunc <16 x i64> undef to <16 x i32>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
   %V2i64 = trunc <2 x i64> undef to <2 x i32>
   %V4i64 = trunc <4 x i64> undef to <4 x i32>
   %V8i64 = trunc <8 x i64> undef to <8 x i32>
+  %V16i64 = trunc <16 x i64> undef to <16 x i32>
   ret i32 undef
 }
 
@@ -53,79 +59,117 @@ define i32 @trunc_vXi16() {
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i64 = trunc <2 x i64> undef to <2 x i16>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i64 = trunc <4 x i64> undef to <4 x i16>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8i64 = trunc <8 x i64> undef to <8 x i16>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V16i64 = trunc <16 x i64> undef to <16 x i16>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V32i64 = trunc <32 x i64> undef to <32 x i16>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i32 = trunc <2 x i32> undef to <2 x i16>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4i32 = trunc <4 x i32> undef to <4 x i16>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V8i32 = trunc <8 x i32> undef to <8 x i16>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V16i32 = trunc <16 x i32> undef to <16 x i16>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %V32i32 = trunc <32 x i32> undef to <32 x i16>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSSE3-LABEL: 'trunc_vXi16'
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i64 = trunc <2 x i64> undef to <2 x i16>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i64 = trunc <4 x i64> undef to <4 x i16>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8i64 = trunc <8 x i64> undef to <8 x i16>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V16i64 = trunc <16 x i64> undef to <16 x i16>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V32i64 = trunc <32 x i64> undef to <32 x i16>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i32 = trunc <2 x i32> undef to <2 x i16>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4i32 = trunc <4 x i32> undef to <4 x i16>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V8i32 = trunc <8 x i32> undef to <8 x i16>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V16i32 = trunc <16 x i32> undef to <16 x i16>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 21 for instruction: %V32i32 = trunc <32 x i32> undef to <32 x i16>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSE42-LABEL: 'trunc_vXi16'
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i64 = trunc <2 x i64> undef to <2 x i16>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i64 = trunc <4 x i64> undef to <4 x i16>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8i64 = trunc <8 x i64> undef to <8 x i16>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V16i64 = trunc <16 x i64> undef to <16 x i16>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V32i64 = trunc <32 x i64> undef to <32 x i16>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i32 = trunc <2 x i32> undef to <2 x i16>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i32 = trunc <4 x i32> undef to <4 x i16>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8i32 = trunc <8 x i32> undef to <8 x i16>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V16i32 = trunc <16 x i32> undef to <16 x i16>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V32i32 = trunc <32 x i32> undef to <32 x i16>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX1-LABEL: 'trunc_vXi16'
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i64 = trunc <2 x i64> undef to <2 x i16>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4i64 = trunc <4 x i64> undef to <4 x i16>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V8i64 = trunc <8 x i64> undef to <8 x i16>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16i64 = trunc <16 x i64> undef to <16 x i16>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32i64 = trunc <32 x i64> undef to <32 x i16>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i32 = trunc <2 x i32> undef to <2 x i16>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i32 = trunc <4 x i32> undef to <4 x i16>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V8i32 = trunc <8 x i32> undef to <8 x i16>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V16i32 = trunc <16 x i32> undef to <16 x i16>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V32i32 = trunc <32 x i32> undef to <32 x i16>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX2-LABEL: 'trunc_vXi16'
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i64 = trunc <2 x i64> undef to <2 x i16>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4i64 = trunc <4 x i64> undef to <4 x i16>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V8i64 = trunc <8 x i64> undef to <8 x i16>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16i64 = trunc <16 x i64> undef to <16 x i16>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32i64 = trunc <32 x i64> undef to <32 x i16>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i32 = trunc <2 x i32> undef to <2 x i16>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i32 = trunc <4 x i32> undef to <4 x i16>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8i32 = trunc <8 x i32> undef to <8 x i16>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V16i32 = trunc <16 x i32> undef to <16 x i16>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V32i32 = trunc <32 x i32> undef to <32 x i16>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
-; AVX512-LABEL: 'trunc_vXi16'
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i64 = trunc <2 x i64> undef to <2 x i16>
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4i64 = trunc <4 x i64> undef to <4 x i16>
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8i64 = trunc <8 x i64> undef to <8 x i16>
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i32 = trunc <2 x i32> undef to <2 x i16>
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i32 = trunc <4 x i32> undef to <4 x i16>
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8i32 = trunc <8 x i32> undef to <8 x i16>
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16i32 = trunc <16 x i32> undef to <16 x i16>
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+; AVX512F-LABEL: 'trunc_vXi16'
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i64 = trunc <2 x i64> undef to <2 x i16>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4i64 = trunc <4 x i64> undef to <4 x i16>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8i64 = trunc <8 x i64> undef to <8 x i16>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V16i64 = trunc <16 x i64> undef to <16 x i16>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V32i64 = trunc <32 x i64> undef to <32 x i16>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i32 = trunc <2 x i32> undef to <2 x i16>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i32 = trunc <4 x i32> undef to <4 x i16>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8i32 = trunc <8 x i32> undef to <8 x i16>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16i32 = trunc <16 x i32> undef to <16 x i16>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V32i32 = trunc <32 x i32> undef to <32 x i16>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX512BW-LABEL: 'trunc_vXi16'
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i64 = trunc <2 x i64> undef to <2 x i16>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4i64 = trunc <4 x i64> undef to <4 x i16>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8i64 = trunc <8 x i64> undef to <8 x i16>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V16i64 = trunc <16 x i64> undef to <16 x i16>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32i64 = trunc <32 x i64> undef to <32 x i16>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i32 = trunc <2 x i32> undef to <2 x i16>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i32 = trunc <4 x i32> undef to <4 x i16>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8i32 = trunc <8 x i32> undef to <8 x i16>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16i32 = trunc <16 x i32> undef to <16 x i16>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32i32 = trunc <32 x i32> undef to <32 x i16>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; BTVER2-LABEL: 'trunc_vXi16'
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i64 = trunc <2 x i64> undef to <2 x i16>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4i64 = trunc <4 x i64> undef to <4 x i16>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V8i64 = trunc <8 x i64> undef to <8 x i16>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16i64 = trunc <16 x i64> undef to <16 x i16>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32i64 = trunc <32 x i64> undef to <32 x i16>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i32 = trunc <2 x i32> undef to <2 x i16>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i32 = trunc <4 x i32> undef to <4 x i16>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V8i32 = trunc <8 x i32> undef to <8 x i16>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V16i32 = trunc <16 x i32> undef to <16 x i16>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V32i32 = trunc <32 x i32> undef to <32 x i16>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
   %V2i64 = trunc <2 x i64> undef to <2 x i16>
   %V4i64 = trunc <4 x i64> undef to <4 x i16>
   %V8i64 = trunc <8 x i64> undef to <8 x i16>
+  %V16i64 = trunc <16 x i64> undef to <16 x i16>
+  %V32i64 = trunc <32 x i64> undef to <32 x i16>
+
   %V2i32 = trunc <2 x i32> undef to <2 x i16>
   %V4i32 = trunc <4 x i32> undef to <4 x i16>
   %V8i32 = trunc <8 x i32> undef to <8 x i16>
   %V16i32 = trunc <16 x i32> undef to <16 x i16>
+  %V32i32 = trunc <32 x i32> undef to <32 x i16>
   ret i32 undef
 }
 
@@ -134,136 +178,190 @@ define i32 @trunc_vXi8() {
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i64 = trunc <2 x i64> undef to <2 x i8>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i64 = trunc <4 x i64> undef to <4 x i8>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8i64 = trunc <8 x i64> undef to <8 x i8>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V16i64 = trunc <16 x i64> undef to <16 x i8>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V32i64 = trunc <32 x i64> undef to <32 x i8>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %V64i64 = trunc <64 x i64> undef to <64 x i8>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i32 = trunc <2 x i32> undef to <2 x i8>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4i32 = trunc <4 x i32> undef to <4 x i8>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8i32 = trunc <8 x i32> undef to <8 x i8>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V16i32 = trunc <16 x i32> undef to <16 x i8>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V32i32 = trunc <32 x i32> undef to <32 x i8>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %V64i32 = trunc <64 x i32> undef to <64 x i8>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i16 = trunc <2 x i16> undef to <2 x i8>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4i16 = trunc <4 x i16> undef to <4 x i8>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8i16 = trunc <8 x i16> undef to <8 x i8>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16i16 = trunc <16 x i16> undef to <16 x i8>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V32i16 = trunc <32 x i16> undef to <32 x i8>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V64i16 = trunc <64 x i16> undef to <64 x i8>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSSE3-LABEL: 'trunc_vXi8'
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i64 = trunc <2 x i64> undef to <2 x i8>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i64 = trunc <4 x i64> undef to <4 x i8>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8i64 = trunc <8 x i64> undef to <8 x i8>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V16i64 = trunc <16 x i64> undef to <16 x i8>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V32i64 = trunc <32 x i64> undef to <32 x i8>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %V64i64 = trunc <64 x i64> undef to <64 x i8>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i32 = trunc <2 x i32> undef to <2 x i8>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4i32 = trunc <4 x i32> undef to <4 x i8>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8i32 = trunc <8 x i32> undef to <8 x i8>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V16i32 = trunc <16 x i32> undef to <16 x i8>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V32i32 = trunc <32 x i32> undef to <32 x i8>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %V64i32 = trunc <64 x i32> undef to <64 x i8>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i16 = trunc <2 x i16> undef to <2 x i8>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4i16 = trunc <4 x i16> undef to <4 x i8>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8i16 = trunc <8 x i16> undef to <8 x i8>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16i16 = trunc <16 x i16> undef to <16 x i8>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V32i16 = trunc <32 x i16> undef to <32 x i8>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V64i16 = trunc <64 x i16> undef to <64 x i8>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSE42-LABEL: 'trunc_vXi8'
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i64 = trunc <2 x i64> undef to <2 x i8>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i64 = trunc <4 x i64> undef to <4 x i8>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8i64 = trunc <8 x i64> undef to <8 x i8>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V16i64 = trunc <16 x i64> undef to <16 x i8>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V32i64 = trunc <32 x i64> undef to <32 x i8>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %V64i64 = trunc <64 x i64> undef to <64 x i8>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i32 = trunc <2 x i32> undef to <2 x i8>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i32 = trunc <4 x i32> undef to <4 x i8>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8i32 = trunc <8 x i32> undef to <8 x i8>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V16i32 = trunc <16 x i32> undef to <16 x i8>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V32i32 = trunc <32 x i32> undef to <32 x i8>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %V64i32 = trunc <64 x i32> undef to <64 x i8>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i16 = trunc <2 x i16> undef to <2 x i8>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4i16 = trunc <4 x i16> undef to <4 x i8>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8i16 = trunc <8 x i16> undef to <8 x i8>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16i16 = trunc <16 x i16> undef to <16 x i8>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V32i16 = trunc <32 x i16> undef to <32 x i8>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V64i16 = trunc <64 x i16> undef to <64 x i8>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX1-LABEL: 'trunc_vXi8'
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i64 = trunc <2 x i64> undef to <2 x i8>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4i64 = trunc <4 x i64> undef to <4 x i8>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V8i64 = trunc <8 x i64> undef to <8 x i8>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V16i64 = trunc <16 x i64> undef to <16 x i8>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32i64 = trunc <32 x i64> undef to <32 x i8>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64i64 = trunc <64 x i64> undef to <64 x i8>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i32 = trunc <2 x i32> undef to <2 x i8>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i32 = trunc <4 x i32> undef to <4 x i8>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8i32 = trunc <8 x i32> undef to <8 x i8>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V16i32 = trunc <16 x i32> undef to <16 x i8>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V32i32 = trunc <32 x i32> undef to <32 x i8>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %V64i32 = trunc <64 x i32> undef to <64 x i8>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i16 = trunc <2 x i16> undef to <2 x i8>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4i16 = trunc <4 x i16> undef to <4 x i8>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8i16 = trunc <8 x i16> undef to <8 x i8>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16i16 = trunc <16 x i16> undef to <16 x i8>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V32i16 = trunc <32 x i16> undef to <32 x i8>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V64i16 = trunc <64 x i16> undef to <64 x i8>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX2-LABEL: 'trunc_vXi8'
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i64 = trunc <2 x i64> undef to <2 x i8>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4i64 = trunc <4 x i64> undef to <4 x i8>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V8i64 = trunc <8 x i64> undef to <8 x i8>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V16i64 = trunc <16 x i64> undef to <16 x i8>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32i64 = trunc <32 x i64> undef to <32 x i8>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64i64 = trunc <64 x i64> undef to <64 x i8>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i32 = trunc <2 x i32> undef to <2 x i8>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i32 = trunc <4 x i32> undef to <4 x i8>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8i32 = trunc <8 x i32> undef to <8 x i8>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V16i32 = trunc <16 x i32> undef to <16 x i8>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V32i32 = trunc <32 x i32> undef to <32 x i8>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %V64i32 = trunc <64 x i32> undef to <64 x i8>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i16 = trunc <2 x i16> undef to <2 x i8>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4i16 = trunc <4 x i16> undef to <4 x i8>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8i16 = trunc <8 x i16> undef to <8 x i8>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16i16 = trunc <16 x i16> undef to <16 x i8>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V32i16 = trunc <32 x i16> undef to <32 x i8>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V64i16 = trunc <64 x i16> undef to <64 x i8>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX512F-LABEL: 'trunc_vXi8'
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i64 = trunc <2 x i64> undef to <2 x i8>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4i64 = trunc <4 x i64> undef to <4 x i8>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V8i64 = trunc <8 x i64> undef to <8 x i8>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V16i64 = trunc <16 x i64> undef to <16 x i8>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V32i64 = trunc <32 x i64> undef to <32 x i8>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V64i64 = trunc <64 x i64> undef to <64 x i8>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i32 = trunc <2 x i32> undef to <2 x i8>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i32 = trunc <4 x i32> undef to <4 x i8>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8i32 = trunc <8 x i32> undef to <8 x i8>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16i32 = trunc <16 x i32> undef to <16 x i8>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V32i32 = trunc <32 x i32> undef to <32 x i8>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V64i32 = trunc <64 x i32> undef to <64 x i8>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i16 = trunc <2 x i16> undef to <2 x i8>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4i16 = trunc <4 x i16> undef to <4 x i8>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8i16 = trunc <8 x i16> undef to <8 x i8>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16i16 = trunc <16 x i16> undef to <16 x i8>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V32i16 = trunc <32 x i16> undef to <32 x i8>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V64i16 = trunc <64 x i16> undef to <64 x i8>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX512BW-LABEL: 'trunc_vXi8'
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i64 = trunc <2 x i64> undef to <2 x i8>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4i64 = trunc <4 x i64> undef to <4 x i8>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V8i64 = trunc <8 x i64> undef to <8 x i8>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V16i64 = trunc <16 x i64> undef to <16 x i8>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V32i64 = trunc <32 x i64> undef to <32 x i8>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64i64 = trunc <64 x i64> undef to <64 x i8>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i32 = trunc <2 x i32> undef to <2 x i8>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i32 = trunc <4 x i32> undef to <4 x i8>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8i32 = trunc <8 x i32> undef to <8 x i8>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16i32 = trunc <16 x i32> undef to <16 x i8>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V32i32 = trunc <32 x i32> undef to <32 x i8>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64i32 = trunc <64 x i32> undef to <64 x i8>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i16 = trunc <2 x i16> undef to <2 x i8>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4i16 = trunc <4 x i16> undef to <4 x i8>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8i16 = trunc <8 x i16> undef to <8 x i8>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16i16 = trunc <16 x i16> undef to <16 x i8>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V32i16 = trunc <32 x i16> undef to <32 x i8>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64i16 = trunc <64 x i16> undef to <64 x i8>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; BTVER2-LABEL: 'trunc_vXi8'
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i64 = trunc <2 x i64> undef to <2 x i8>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4i64 = trunc <4 x i64> undef to <4 x i8>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V8i64 = trunc <8 x i64> undef to <8 x i8>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V16i64 = trunc <16 x i64> undef to <16 x i8>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32i64 = trunc <32 x i64> undef to <32 x i8>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64i64 = trunc <64 x i64> undef to <64 x i8>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i32 = trunc <2 x i32> undef to <2 x i8>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i32 = trunc <4 x i32> undef to <4 x i8>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8i32 = trunc <8 x i32> undef to <8 x i8>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V16i32 = trunc <16 x i32> undef to <16 x i8>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V32i32 = trunc <32 x i32> undef to <32 x i8>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %V64i32 = trunc <64 x i32> undef to <64 x i8>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i16 = trunc <2 x i16> undef to <2 x i8>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4i16 = trunc <4 x i16> undef to <4 x i8>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8i16 = trunc <8 x i16> undef to <8 x i8>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16i16 = trunc <16 x i16> undef to <16 x i8>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V32i16 = trunc <32 x i16> undef to <32 x i8>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V64i16 = trunc <64 x i16> undef to <64 x i8>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
   %V2i64 = trunc <2 x i64> undef to <2 x i8>
   %V4i64 = trunc <4 x i64> undef to <4 x i8>
   %V8i64 = trunc <8 x i64> undef to <8 x i8>
+  %V16i64 = trunc <16 x i64> undef to <16 x i8>
+  %V32i64 = trunc <32 x i64> undef to <32 x i8>
+  %V64i64 = trunc <64 x i64> undef to <64 x i8>
 
   %V2i32 = trunc <2 x i32> undef to <2 x i8>
   %V4i32 = trunc <4 x i32> undef to <4 x i8>
   %V8i32 = trunc <8 x i32> undef to <8 x i8>
   %V16i32 = trunc <16 x i32> undef to <16 x i8>
+  %V32i32 = trunc <32 x i32> undef to <32 x i8>
+  %V64i32 = trunc <64 x i32> undef to <64 x i8>
 
   %V2i16 = trunc <2 x i16> undef to <2 x i8>
   %V4i16 = trunc <4 x i16> undef to <4 x i8>
   %V8i16 = trunc <8 x i16> undef to <8 x i8>
   %V16i16 = trunc <16 x i16> undef to <16 x i8>
   %V32i16 = trunc <32 x i16> undef to <32 x i8>
+  %V64i16 = trunc <64 x i16> undef to <64 x i8>
 
   ret i32 undef
 }
@@ -273,15 +371,21 @@ define i32 @trunc_vXi1() {
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i64 = trunc <2 x i64> undef to <2 x i1>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i64 = trunc <4 x i64> undef to <4 x i1>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8i64 = trunc <8 x i64> undef to <8 x i1>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V16i64 = trunc <16 x i64> undef to <16 x i1>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V32i64 = trunc <32 x i64> undef to <32 x i1>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %V64i64 = trunc <64 x i64> undef to <64 x i1>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i32 = trunc <2 x i32> undef to <2 x i1>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V4i32 = trunc <4 x i32> undef to <4 x i1>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8i32 = trunc <8 x i32> undef to <8 x i1>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16i32 = trunc <16 x i32> undef to <16 x i1>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V32i32 = trunc <32 x i32> undef to <32 x i1>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V64i32 = trunc <64 x i32> undef to <64 x i1>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i16 = trunc <2 x i16> undef to <2 x i1>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V4i16 = trunc <4 x i16> undef to <4 x i1>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V8i16 = trunc <8 x i16> undef to <8 x i1>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16i16 = trunc <16 x i16> undef to <16 x i1>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32i16 = trunc <32 x i16> undef to <32 x i1>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V64i16 = trunc <64 x i16> undef to <64 x i1>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V32i8 = trunc <32 x i8> undef to <32 x i1>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V64i8 = trunc <64 x i8> undef to <64 x i1>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
@@ -290,15 +394,21 @@ define i32 @trunc_vXi1() {
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i64 = trunc <2 x i64> undef to <2 x i1>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V4i64 = trunc <4 x i64> undef to <4 x i1>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V8i64 = trunc <8 x i64> undef to <8 x i1>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V16i64 = trunc <16 x i64> undef to <16 x i1>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32i64 = trunc <32 x i64> undef to <32 x i1>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64i64 = trunc <64 x i64> undef to <64 x i1>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i32 = trunc <2 x i32> undef to <2 x i1>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V4i32 = trunc <4 x i32> undef to <4 x i1>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V8i32 = trunc <8 x i32> undef to <8 x i1>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V16i32 = trunc <16 x i32> undef to <16 x i1>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32i32 = trunc <32 x i32> undef to <32 x i1>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64i32 = trunc <64 x i32> undef to <64 x i1>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i16 = trunc <2 x i16> undef to <2 x i1>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V4i16 = trunc <4 x i16> undef to <4 x i1>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V8i16 = trunc <8 x i16> undef to <8 x i1>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V16i16 = trunc <16 x i16> undef to <16 x i1>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32i16 = trunc <32 x i16> undef to <32 x i1>
+; AVX-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64i16 = trunc <64 x i16> undef to <64 x i1>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V32i8 = trunc <32 x i8> undef to <32 x i1>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V64i8 = trunc <64 x i8> undef to <64 x i1>
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
@@ -307,15 +417,21 @@ define i32 @trunc_vXi1() {
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i64 = trunc <2 x i64> undef to <2 x i1>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V4i64 = trunc <4 x i64> undef to <4 x i1>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V8i64 = trunc <8 x i64> undef to <8 x i1>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V16i64 = trunc <16 x i64> undef to <16 x i1>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V32i64 = trunc <32 x i64> undef to <32 x i1>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V64i64 = trunc <64 x i64> undef to <64 x i1>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i32 = trunc <2 x i32> undef to <2 x i1>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V4i32 = trunc <4 x i32> undef to <4 x i1>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V8i32 = trunc <8 x i32> undef to <8 x i1>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V16i32 = trunc <16 x i32> undef to <16 x i1>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V32i32 = trunc <32 x i32> undef to <32 x i1>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V64i32 = trunc <64 x i32> undef to <64 x i1>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i16 = trunc <2 x i16> undef to <2 x i1>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V4i16 = trunc <4 x i16> undef to <4 x i1>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V8i16 = trunc <8 x i16> undef to <8 x i1>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V16i16 = trunc <16 x i16> undef to <16 x i1>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V32i16 = trunc <32 x i16> undef to <32 x i1>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V64i16 = trunc <64 x i16> undef to <64 x i1>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V32i8 = trunc <32 x i8> undef to <32 x i1>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V64i8 = trunc <64 x i8> undef to <64 x i1>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
@@ -324,15 +440,21 @@ define i32 @trunc_vXi1() {
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i64 = trunc <2 x i64> undef to <2 x i1>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V4i64 = trunc <4 x i64> undef to <4 x i1>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V8i64 = trunc <8 x i64> undef to <8 x i1>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V16i64 = trunc <16 x i64> undef to <16 x i1>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32i64 = trunc <32 x i64> undef to <32 x i1>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64i64 = trunc <64 x i64> undef to <64 x i1>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i32 = trunc <2 x i32> undef to <2 x i1>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V4i32 = trunc <4 x i32> undef to <4 x i1>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V8i32 = trunc <8 x i32> undef to <8 x i1>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V16i32 = trunc <16 x i32> undef to <16 x i1>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32i32 = trunc <32 x i32> undef to <32 x i1>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64i32 = trunc <64 x i32> undef to <64 x i1>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i16 = trunc <2 x i16> undef to <2 x i1>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V4i16 = trunc <4 x i16> undef to <4 x i1>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V8i16 = trunc <8 x i16> undef to <8 x i1>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V16i16 = trunc <16 x i16> undef to <16 x i1>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32i16 = trunc <32 x i16> undef to <32 x i1>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64i16 = trunc <64 x i16> undef to <64 x i1>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V32i8 = trunc <32 x i8> undef to <32 x i1>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V64i8 = trunc <64 x i8> undef to <64 x i1>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
@@ -340,17 +462,23 @@ define i32 @trunc_vXi1() {
   %V2i64 = trunc <2 x i64> undef to <2 x i1>
   %V4i64 = trunc <4 x i64> undef to <4 x i1>
   %V8i64 = trunc <8 x i64> undef to <8 x i1>
+  %V16i64 = trunc <16 x i64> undef to <16 x i1>
+  %V32i64 = trunc <32 x i64> undef to <32 x i1>
+  %V64i64 = trunc <64 x i64> undef to <64 x i1>
 
   %V2i32 = trunc <2 x i32> undef to <2 x i1>
   %V4i32 = trunc <4 x i32> undef to <4 x i1>
   %V8i32 = trunc <8 x i32> undef to <8 x i1>
   %V16i32 = trunc <16 x i32> undef to <16 x i1>
+  %V32i32 = trunc <32 x i32> undef to <32 x i1>
+  %V64i32 = trunc <64 x i32> undef to <64 x i1>
 
   %V2i16 = trunc <2 x i16> undef to <2 x i1>
   %V4i16 = trunc <4 x i16> undef to <4 x i1>
   %V8i16 = trunc <8 x i16> undef to <8 x i1>
   %V16i16 = trunc <16 x i16> undef to <16 x i1>
   %V32i16 = trunc <32 x i16> undef to <32 x i1>
+  %V64i16 = trunc <64 x i16> undef to <64 x i1>
 
   %V32i8 = trunc <32 x i8> undef to <32 x i1>
   %V64i8 = trunc <64 x i8> undef to <64 x i1>
diff --git a/test/Analysis/CostModel/X86/vselect-cost.ll b/test/Analysis/CostModel/X86/vselect-cost.ll
index e02ebdd591285478f86b9326f5d82334c12ba907..582942355692d77c2a87085efa51b9c05e1edf55 100644
--- a/test/Analysis/CostModel/X86/vselect-cost.ll
+++ b/test/Analysis/CostModel/X86/vselect-cost.ll
@@ -3,53 +3,232 @@
 ; RUN: opt < %s -cost-model -analyze -mtriple=x86_64-unknown-linux-gnu -mattr=+sse4.1 | FileCheck %s -check-prefixes=CHECK,SSE,SSE41
 ; RUN: opt < %s -cost-model -analyze -mtriple=x86_64-unknown-linux-gnu -mattr=+avx | FileCheck %s -check-prefixes=CHECK,AVX,AVX1
 ; RUN: opt < %s -cost-model -analyze -mtriple=x86_64-unknown-linux-gnu -mattr=+avx2 | FileCheck %s -check-prefixes=CHECK,AVX,AVX2
-
+; RUN: opt < %s -cost-model -analyze -mtriple=x86_64-unknown-linux-gnu -mattr=+avx512f,+avx512vl  | FileCheck %s -check-prefixes=CHECK,AVX,AVX512,AVX512F
+; RUN: opt < %s -cost-model -analyze -mtriple=x86_64-unknown-linux-gnu -mattr=+avx512bw,+avx512vl | FileCheck %s -check-prefixes=CHECK,AVX,AVX512,AVX512BW
 
 ; Verify the cost of vector select instructions.
 
+define i32 @test_select() {
+; SSE2-LABEL: 'test_select'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = select i1 undef, i64 undef, i64 undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2I64 = select <2 x i1> undef, <2 x i64> undef, <2 x i64> undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4I64 = select <4 x i1> undef, <4 x i64> undef, <4 x i64> undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V8I64 = select <8 x i1> undef, <8 x i64> undef, <8 x i64> undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = select i1 undef, i32 undef, i32 undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4I32 = select <4 x i1> undef, <4 x i32> undef, <4 x i32> undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V8I32 = select <8 x i1> undef, <8 x i32> undef, <8 x i32> undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V16I32 = select <16 x i1> undef, <16 x i32> undef, <16 x i32> undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = select i1 undef, i16 undef, i16 undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8I16 = select <8 x i1> undef, <8 x i16> undef, <8 x i16> undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V16I16 = select <16 x i1> undef, <16 x i16> undef, <16 x i16> undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V32I16 = select <32 x i1> undef, <32 x i16> undef, <32 x i16> undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = select i1 undef, i8 undef, i8 undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16I8 = select <16 x i1> undef, <16 x i8> undef, <16 x i8> undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V32I8 = select <32 x i1> undef, <32 x i8> undef, <32 x i8> undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V64I8 = select <64 x i1> undef, <64 x i8> undef, <64 x i8> undef
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; SSE41-LABEL: 'test_select'
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = select i1 undef, i64 undef, i64 undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = select <2 x i1> undef, <2 x i64> undef, <2 x i64> undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I64 = select <4 x i1> undef, <4 x i64> undef, <4 x i64> undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I64 = select <8 x i1> undef, <8 x i64> undef, <8 x i64> undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = select i1 undef, i32 undef, i32 undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = select <4 x i1> undef, <4 x i32> undef, <4 x i32> undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = select <8 x i1> undef, <8 x i32> undef, <8 x i32> undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I32 = select <16 x i1> undef, <16 x i32> undef, <16 x i32> undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = select i1 undef, i16 undef, i16 undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = select <8 x i1> undef, <8 x i16> undef, <8 x i16> undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = select <16 x i1> undef, <16 x i16> undef, <16 x i16> undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = select <32 x i1> undef, <32 x i16> undef, <32 x i16> undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = select i1 undef, i8 undef, i8 undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = select <16 x i1> undef, <16 x i8> undef, <16 x i8> undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I8 = select <32 x i1> undef, <32 x i8> undef, <32 x i8> undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = select <64 x i1> undef, <64 x i8> undef, <64 x i8> undef
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX1-LABEL: 'test_select'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = select i1 undef, i64 undef, i64 undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = select <2 x i1> undef, <2 x i64> undef, <2 x i64> undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = select <4 x i1> undef, <4 x i64> undef, <4 x i64> undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I64 = select <8 x i1> undef, <8 x i64> undef, <8 x i64> undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = select i1 undef, i32 undef, i32 undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = select <4 x i1> undef, <4 x i32> undef, <4 x i32> undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = select <8 x i1> undef, <8 x i32> undef, <8 x i32> undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I32 = select <16 x i1> undef, <16 x i32> undef, <16 x i32> undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = select i1 undef, i16 undef, i16 undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = select <8 x i1> undef, <8 x i16> undef, <8 x i16> undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16I16 = select <16 x i1> undef, <16 x i16> undef, <16 x i16> undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V32I16 = select <32 x i1> undef, <32 x i16> undef, <32 x i16> undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = select i1 undef, i8 undef, i8 undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = select <16 x i1> undef, <16 x i8> undef, <16 x i8> undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32I8 = select <32 x i1> undef, <32 x i8> undef, <32 x i8> undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V64I8 = select <64 x i1> undef, <64 x i8> undef, <64 x i8> undef
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX2-LABEL: 'test_select'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = select i1 undef, i64 undef, i64 undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = select <2 x i1> undef, <2 x i64> undef, <2 x i64> undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = select <4 x i1> undef, <4 x i64> undef, <4 x i64> undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I64 = select <8 x i1> undef, <8 x i64> undef, <8 x i64> undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = select i1 undef, i32 undef, i32 undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = select <4 x i1> undef, <4 x i32> undef, <4 x i32> undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = select <8 x i1> undef, <8 x i32> undef, <8 x i32> undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I32 = select <16 x i1> undef, <16 x i32> undef, <16 x i32> undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = select i1 undef, i16 undef, i16 undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = select <8 x i1> undef, <8 x i16> undef, <8 x i16> undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = select <16 x i1> undef, <16 x i16> undef, <16 x i16> undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I16 = select <32 x i1> undef, <32 x i16> undef, <32 x i16> undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = select i1 undef, i8 undef, i8 undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = select <16 x i1> undef, <16 x i8> undef, <16 x i8> undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32I8 = select <32 x i1> undef, <32 x i8> undef, <32 x i8> undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64I8 = select <64 x i1> undef, <64 x i8> undef, <64 x i8> undef
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX512F-LABEL: 'test_select'
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = select i1 undef, i64 undef, i64 undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = select <2 x i1> undef, <2 x i64> undef, <2 x i64> undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = select <4 x i1> undef, <4 x i64> undef, <4 x i64> undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = select <8 x i1> undef, <8 x i64> undef, <8 x i64> undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = select i1 undef, i32 undef, i32 undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = select <4 x i1> undef, <4 x i32> undef, <4 x i32> undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = select <8 x i1> undef, <8 x i32> undef, <8 x i32> undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I32 = select <16 x i1> undef, <16 x i32> undef, <16 x i32> undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = select i1 undef, i16 undef, i16 undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = select <8 x i1> undef, <8 x i16> undef, <8 x i16> undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = select <16 x i1> undef, <16 x i16> undef, <16 x i16> undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I16 = select <32 x i1> undef, <32 x i16> undef, <32 x i16> undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = select i1 undef, i8 undef, i8 undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = select <16 x i1> undef, <16 x i8> undef, <16 x i8> undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32I8 = select <32 x i1> undef, <32 x i8> undef, <32 x i8> undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V64I8 = select <64 x i1> undef, <64 x i8> undef, <64 x i8> undef
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+; AVX512BW-LABEL: 'test_select'
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = select i1 undef, i64 undef, i64 undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2I64 = select <2 x i1> undef, <2 x i64> undef, <2 x i64> undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I64 = select <4 x i1> undef, <4 x i64> undef, <4 x i64> undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I64 = select <8 x i1> undef, <8 x i64> undef, <8 x i64> undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = select i1 undef, i32 undef, i32 undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = select <4 x i1> undef, <4 x i32> undef, <4 x i32> undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I32 = select <8 x i1> undef, <8 x i32> undef, <8 x i32> undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I32 = select <16 x i1> undef, <16 x i32> undef, <16 x i32> undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = select i1 undef, i16 undef, i16 undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = select <8 x i1> undef, <8 x i16> undef, <8 x i16> undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I16 = select <16 x i1> undef, <16 x i16> undef, <16 x i16> undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32I16 = select <32 x i1> undef, <32 x i16> undef, <32 x i16> undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = select i1 undef, i8 undef, i8 undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = select <16 x i1> undef, <16 x i8> undef, <16 x i8> undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32I8 = select <32 x i1> undef, <32 x i8> undef, <32 x i8> undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64I8 = select <64 x i1> undef, <64 x i8> undef, <64 x i8> undef
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
+  %I64 = select i1 undef, i64 undef, i64 undef
+  %V2I64 = select <2 x i1> undef, <2 x i64> undef, <2 x i64> undef
+  %V4I64 = select <4 x i1> undef, <4 x i64> undef, <4 x i64> undef
+  %V8I64 = select <8 x i1> undef, <8 x i64> undef, <8 x i64> undef
+
+  %I32 = select i1 undef, i32 undef, i32 undef
+  %V4I32 = select <4 x i1> undef, <4 x i32> undef, <4 x i32> undef
+  %V8I32 = select <8 x i1> undef, <8 x i32> undef, <8 x i32> undef
+  %V16I32 = select <16 x i1> undef, <16 x i32> undef, <16 x i32> undef
+
+  %I16 = select i1 undef, i16 undef, i16 undef
+  %V8I16 = select <8 x i1> undef, <8 x i16> undef, <8 x i16> undef
+  %V16I16 = select <16 x i1> undef, <16 x i16> undef, <16 x i16> undef
+  %V32I16 = select <32 x i1> undef, <32 x i16> undef, <32 x i16> undef
+
+  %I8 = select i1 undef, i8 undef, i8 undef
+  %V16I8 = select <16 x i1> undef, <16 x i8> undef, <16 x i8> undef
+  %V32I8 = select <32 x i1> undef, <32 x i8> undef, <32 x i8> undef
+  %V64I8 = select <64 x i1> undef, <64 x i8> undef, <64 x i8> undef
+
+  ret i32 undef
+}
+
 ; SSE41 added blend instructions with an immediate for <2 x double> and
 ; <4 x float>. Integers of the same size should also use those instructions.
 
 define <2 x i64> @test_2i64(<2 x i64> %a, <2 x i64> %b) {
-; CHECK-LABEL: 'test_2i64'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sel = select <2 x i1> <i1 true, i1 false>, <2 x i64> %a, <2 x i64> %b
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x i64> %sel
+; SSE2-LABEL: 'test_2i64'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %sel = select <2 x i1> <i1 true, i1 false>, <2 x i64> %a, <2 x i64> %b
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x i64> %sel
+;
+; SSE41-LABEL: 'test_2i64'
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sel = select <2 x i1> <i1 true, i1 false>, <2 x i64> %a, <2 x i64> %b
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x i64> %sel
+;
+; AVX-LABEL: 'test_2i64'
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sel = select <2 x i1> <i1 true, i1 false>, <2 x i64> %a, <2 x i64> %b
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x i64> %sel
 ;
   %sel = select <2 x i1> <i1 true, i1 false>, <2 x i64> %a, <2 x i64> %b
   ret <2 x i64> %sel
 }
 
 define <2 x double> @test_2double(<2 x double> %a, <2 x double> %b) {
-; CHECK-LABEL: 'test_2double'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sel = select <2 x i1> <i1 true, i1 false>, <2 x double> %a, <2 x double> %b
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x double> %sel
+; SSE2-LABEL: 'test_2double'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %sel = select <2 x i1> <i1 true, i1 false>, <2 x double> %a, <2 x double> %b
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x double> %sel
+;
+; SSE41-LABEL: 'test_2double'
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sel = select <2 x i1> <i1 true, i1 false>, <2 x double> %a, <2 x double> %b
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x double> %sel
+;
+; AVX-LABEL: 'test_2double'
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sel = select <2 x i1> <i1 true, i1 false>, <2 x double> %a, <2 x double> %b
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <2 x double> %sel
 ;
   %sel = select <2 x i1> <i1 true, i1 false>, <2 x double> %a, <2 x double> %b
   ret <2 x double> %sel
 }
 
 define <4 x i32> @test_4i32(<4 x i32> %a, <4 x i32> %b) {
-; CHECK-LABEL: 'test_4i32'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sel = select <4 x i1> <i1 true, i1 false, i1 true, i1 false>, <4 x i32> %a, <4 x i32> %b
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i32> %sel
+; SSE2-LABEL: 'test_4i32'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %sel = select <4 x i1> <i1 true, i1 false, i1 true, i1 false>, <4 x i32> %a, <4 x i32> %b
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i32> %sel
+;
+; SSE41-LABEL: 'test_4i32'
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sel = select <4 x i1> <i1 true, i1 false, i1 true, i1 false>, <4 x i32> %a, <4 x i32> %b
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i32> %sel
+;
+; AVX-LABEL: 'test_4i32'
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sel = select <4 x i1> <i1 true, i1 false, i1 true, i1 false>, <4 x i32> %a, <4 x i32> %b
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i32> %sel
 ;
   %sel = select <4 x i1> <i1 true, i1 false, i1 true, i1 false>, <4 x i32> %a, <4 x i32> %b
   ret <4 x i32> %sel
 }
 
 define <4 x float> @test_4float(<4 x float> %a, <4 x float> %b) {
-; CHECK-LABEL: 'test_4float'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sel = select <4 x i1> <i1 true, i1 false, i1 true, i1 true>, <4 x float> %a, <4 x float> %b
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x float> %sel
+; SSE2-LABEL: 'test_4float'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %sel = select <4 x i1> <i1 true, i1 false, i1 true, i1 true>, <4 x float> %a, <4 x float> %b
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x float> %sel
+;
+; SSE41-LABEL: 'test_4float'
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sel = select <4 x i1> <i1 true, i1 false, i1 true, i1 true>, <4 x float> %a, <4 x float> %b
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x float> %sel
+;
+; AVX-LABEL: 'test_4float'
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sel = select <4 x i1> <i1 true, i1 false, i1 true, i1 true>, <4 x float> %a, <4 x float> %b
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x float> %sel
 ;
   %sel = select <4 x i1> <i1 true, i1 false, i1 true, i1 true>, <4 x float> %a, <4 x float> %b
   ret <4 x float> %sel
 }
 
 define <16 x i8> @test_16i8(<16 x i8> %a, <16 x i8> %b) {
-; CHECK-LABEL: 'test_16i8'
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sel = select <16 x i1> <i1 true, i1 false, i1 true, i1 true, i1 true, i1 false, i1 true, i1 true, i1 true, i1 false, i1 true, i1 true, i1 true, i1 false, i1 true, i1 true>, <16 x i8> %a, <16 x i8> %b
-; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i8> %sel
+; SSE2-LABEL: 'test_16i8'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %sel = select <16 x i1> <i1 true, i1 false, i1 true, i1 true, i1 true, i1 false, i1 true, i1 true, i1 true, i1 false, i1 true, i1 true, i1 true, i1 false, i1 true, i1 true>, <16 x i8> %a, <16 x i8> %b
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i8> %sel
+;
+; SSE41-LABEL: 'test_16i8'
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sel = select <16 x i1> <i1 true, i1 false, i1 true, i1 true, i1 true, i1 false, i1 true, i1 true, i1 true, i1 false, i1 true, i1 true, i1 true, i1 false, i1 true, i1 true>, <16 x i8> %a, <16 x i8> %b
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i8> %sel
+;
+; AVX-LABEL: 'test_16i8'
+; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sel = select <16 x i1> <i1 true, i1 false, i1 true, i1 true, i1 true, i1 false, i1 true, i1 true, i1 true, i1 false, i1 true, i1 true, i1 true, i1 false, i1 true, i1 true>, <16 x i8> %a, <16 x i8> %b
+; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i8> %sel
 ;
   %sel = select <16 x i1> <i1 true, i1 false, i1 true, i1 true, i1 true, i1 false, i1 true, i1 true, i1 true, i1 false, i1 true, i1 true, i1 true, i1 false, i1 true, i1 true>, <16 x i8> %a, <16 x i8> %b
   ret <16 x i8> %sel
@@ -58,9 +237,13 @@ define <16 x i8> @test_16i8(<16 x i8> %a, <16 x i8> %b) {
 ; AVX added blend instructions with an immediate for <4 x double> and
 ; <8 x float>. Integers of the same size should also use those instructions.
 define <4 x i64> @test_4i64(<4 x i64> %a, <4 x i64> %b) {
-; SSE-LABEL: 'test_4i64'
-; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %sel = select <4 x i1> <i1 true, i1 false, i1 false, i1 true>, <4 x i64> %a, <4 x i64> %b
-; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i64> %sel
+; SSE2-LABEL: 'test_4i64'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %sel = select <4 x i1> <i1 true, i1 false, i1 false, i1 true>, <4 x i64> %a, <4 x i64> %b
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i64> %sel
+;
+; SSE41-LABEL: 'test_4i64'
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %sel = select <4 x i1> <i1 true, i1 false, i1 false, i1 true>, <4 x i64> %a, <4 x i64> %b
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i64> %sel
 ;
 ; AVX-LABEL: 'test_4i64'
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sel = select <4 x i1> <i1 true, i1 false, i1 false, i1 true>, <4 x i64> %a, <4 x i64> %b
@@ -71,9 +254,13 @@ define <4 x i64> @test_4i64(<4 x i64> %a, <4 x i64> %b) {
 }
 
 define <4 x double> @test_4double(<4 x double> %a, <4 x double> %b) {
-; SSE-LABEL: 'test_4double'
-; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %sel = select <4 x i1> <i1 true, i1 false, i1 true, i1 false>, <4 x double> %a, <4 x double> %b
-; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x double> %sel
+; SSE2-LABEL: 'test_4double'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %sel = select <4 x i1> <i1 true, i1 false, i1 true, i1 false>, <4 x double> %a, <4 x double> %b
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x double> %sel
+;
+; SSE41-LABEL: 'test_4double'
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %sel = select <4 x i1> <i1 true, i1 false, i1 true, i1 false>, <4 x double> %a, <4 x double> %b
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <4 x double> %sel
 ;
 ; AVX-LABEL: 'test_4double'
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sel = select <4 x i1> <i1 true, i1 false, i1 true, i1 false>, <4 x double> %a, <4 x double> %b
@@ -84,9 +271,13 @@ define <4 x double> @test_4double(<4 x double> %a, <4 x double> %b) {
 }
 
 define <8 x i32> @test_8i32(<8 x i32> %a, <8 x i32> %b) {
-; SSE-LABEL: 'test_8i32'
-; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %sel = select <8 x i1> <i1 true, i1 false, i1 true, i1 true, i1 true, i1 false, i1 true, i1 false>, <8 x i32> %a, <8 x i32> %b
-; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i32> %sel
+; SSE2-LABEL: 'test_8i32'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %sel = select <8 x i1> <i1 true, i1 false, i1 true, i1 true, i1 true, i1 false, i1 true, i1 false>, <8 x i32> %a, <8 x i32> %b
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i32> %sel
+;
+; SSE41-LABEL: 'test_8i32'
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %sel = select <8 x i1> <i1 true, i1 false, i1 true, i1 true, i1 true, i1 false, i1 true, i1 false>, <8 x i32> %a, <8 x i32> %b
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i32> %sel
 ;
 ; AVX-LABEL: 'test_8i32'
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sel = select <8 x i1> <i1 true, i1 false, i1 true, i1 true, i1 true, i1 false, i1 true, i1 false>, <8 x i32> %a, <8 x i32> %b
@@ -97,9 +288,13 @@ define <8 x i32> @test_8i32(<8 x i32> %a, <8 x i32> %b) {
 }
 
 define <8 x float> @test_8float(<8 x float> %a, <8 x float> %b) {
-; SSE-LABEL: 'test_8float'
-; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %sel = select <8 x i1> <i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 false, i1 false>, <8 x float> %a, <8 x float> %b
-; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <8 x float> %sel
+; SSE2-LABEL: 'test_8float'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %sel = select <8 x i1> <i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 false, i1 false>, <8 x float> %a, <8 x float> %b
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <8 x float> %sel
+;
+; SSE41-LABEL: 'test_8float'
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %sel = select <8 x i1> <i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 false, i1 false>, <8 x float> %a, <8 x float> %b
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <8 x float> %sel
 ;
 ; AVX-LABEL: 'test_8float'
 ; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sel = select <8 x i1> <i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 false, i1 false>, <8 x float> %a, <8 x float> %b
@@ -111,26 +306,50 @@ define <8 x float> @test_8float(<8 x float> %a, <8 x float> %b) {
 
 ; AVX2
 define <16 x i16> @test_16i16(<16 x i16> %a, <16 x i16> %b) {
-; SSE-LABEL: 'test_16i16'
-; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %sel = select <16 x i1> <i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 false, i1 false>, <16 x i16> %a, <16 x i16> %b
-; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i16> %sel
+; SSE2-LABEL: 'test_16i16'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %sel = select <16 x i1> <i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 false, i1 false>, <16 x i16> %a, <16 x i16> %b
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i16> %sel
+;
+; SSE41-LABEL: 'test_16i16'
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %sel = select <16 x i1> <i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 false, i1 false>, <16 x i16> %a, <16 x i16> %b
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i16> %sel
+;
+; AVX1-LABEL: 'test_16i16'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %sel = select <16 x i1> <i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 false, i1 false>, <16 x i16> %a, <16 x i16> %b
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i16> %sel
 ;
-; AVX-LABEL: 'test_16i16'
-; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sel = select <16 x i1> <i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 false, i1 false>, <16 x i16> %a, <16 x i16> %b
-; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i16> %sel
+; AVX2-LABEL: 'test_16i16'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sel = select <16 x i1> <i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 false, i1 false>, <16 x i16> %a, <16 x i16> %b
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i16> %sel
+;
+; AVX512-LABEL: 'test_16i16'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sel = select <16 x i1> <i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 false, i1 false>, <16 x i16> %a, <16 x i16> %b
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i16> %sel
 ;
   %sel = select <16 x i1> <i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 false, i1 false>, <16 x i16> %a, <16 x i16> %b
   ret <16 x i16> %sel
 }
 
 define <32 x i8> @test_32i8(<32 x i8> %a, <32 x i8> %b) {
-; SSE-LABEL: 'test_32i8'
-; SSE-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %sel = select <32 x i1> <i1 true, i1 false, i1 true, i1 true, i1 true, i1 false, i1 true, i1 true, i1 true, i1 false, i1 true, i1 true, i1 true, i1 false, i1 true, i1 true, i1 true, i1 false, i1 true, i1 true, i1 true, i1 false, i1 true, i1 true, i1 true, i1 false, i1 true, i1 true, i1 true, i1 false, i1 true, i1 true>, <32 x i8> %a, <32 x i8> %b
-; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i8> %sel
+; SSE2-LABEL: 'test_32i8'
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %sel = select <32 x i1> <i1 true, i1 false, i1 true, i1 true, i1 true, i1 false, i1 true, i1 true, i1 true, i1 false, i1 true, i1 true, i1 true, i1 false, i1 true, i1 true, i1 true, i1 false, i1 true, i1 true, i1 true, i1 false, i1 true, i1 true, i1 true, i1 false, i1 true, i1 true, i1 true, i1 false, i1 true, i1 true>, <32 x i8> %a, <32 x i8> %b
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i8> %sel
+;
+; SSE41-LABEL: 'test_32i8'
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %sel = select <32 x i1> <i1 true, i1 false, i1 true, i1 true, i1 true, i1 false, i1 true, i1 true, i1 true, i1 false, i1 true, i1 true, i1 true, i1 false, i1 true, i1 true, i1 true, i1 false, i1 true, i1 true, i1 true, i1 false, i1 true, i1 true, i1 true, i1 false, i1 true, i1 true, i1 true, i1 false, i1 true, i1 true>, <32 x i8> %a, <32 x i8> %b
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i8> %sel
+;
+; AVX1-LABEL: 'test_32i8'
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %sel = select <32 x i1> <i1 true, i1 false, i1 true, i1 true, i1 true, i1 false, i1 true, i1 true, i1 true, i1 false, i1 true, i1 true, i1 true, i1 false, i1 true, i1 true, i1 true, i1 false, i1 true, i1 true, i1 true, i1 false, i1 true, i1 true, i1 true, i1 false, i1 true, i1 true, i1 true, i1 false, i1 true, i1 true>, <32 x i8> %a, <32 x i8> %b
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i8> %sel
+;
+; AVX2-LABEL: 'test_32i8'
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sel = select <32 x i1> <i1 true, i1 false, i1 true, i1 true, i1 true, i1 false, i1 true, i1 true, i1 true, i1 false, i1 true, i1 true, i1 true, i1 false, i1 true, i1 true, i1 true, i1 false, i1 true, i1 true, i1 true, i1 false, i1 true, i1 true, i1 true, i1 false, i1 true, i1 true, i1 true, i1 false, i1 true, i1 true>, <32 x i8> %a, <32 x i8> %b
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i8> %sel
 ;
-; AVX-LABEL: 'test_32i8'
-; AVX-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sel = select <32 x i1> <i1 true, i1 false, i1 true, i1 true, i1 true, i1 false, i1 true, i1 true, i1 true, i1 false, i1 true, i1 true, i1 true, i1 false, i1 true, i1 true, i1 true, i1 false, i1 true, i1 true, i1 true, i1 false, i1 true, i1 true, i1 true, i1 false, i1 true, i1 true, i1 true, i1 false, i1 true, i1 true>, <32 x i8> %a, <32 x i8> %b
-; AVX-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i8> %sel
+; AVX512-LABEL: 'test_32i8'
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %sel = select <32 x i1> <i1 true, i1 false, i1 true, i1 true, i1 true, i1 false, i1 true, i1 true, i1 true, i1 false, i1 true, i1 true, i1 true, i1 false, i1 true, i1 true, i1 true, i1 false, i1 true, i1 true, i1 true, i1 false, i1 true, i1 true, i1 true, i1 false, i1 true, i1 true, i1 true, i1 false, i1 true, i1 true>, <32 x i8> %a, <32 x i8> %b
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret <32 x i8> %sel
 ;
   %sel = select <32 x i1> <i1 true, i1 false, i1 true, i1 true, i1 true, i1 false, i1 true, i1 true, i1 true, i1 false, i1 true, i1 true, i1 true, i1 false, i1 true, i1 true, i1 true, i1 false, i1 true, i1 true, i1 true, i1 false, i1 true, i1 true, i1 true, i1 false, i1 true, i1 true, i1 true, i1 false, i1 true, i1 true>, <32 x i8> %a, <32 x i8> %b
   ret <32 x i8> %sel
diff --git a/test/Analysis/DependenceAnalysis/AA.ll b/test/Analysis/DependenceAnalysis/AA.ll
index 0d213e25e134305f5292612c588ba2303cf4fcea..6f9c828798cc37b3be7889e7ef1eff1d26a7b8d8 100644
--- a/test/Analysis/DependenceAnalysis/AA.ll
+++ b/test/Analysis/DependenceAnalysis/AA.ll
@@ -1,3 +1,5 @@
+; RUN: opt < %s -disable-output "-passes=print<da>"                            \
+; RUN: "-aa-pipeline=basic-aa,type-based-aa" 2>&1 | FileCheck %s
 ; RUN: opt < %s -analyze -basicaa -tbaa -da | FileCheck %s
 
 ; CHECK-LABEL: 'Dependence Analysis' for function 'test_no_noalias'
diff --git a/test/Analysis/DependenceAnalysis/Banerjee.ll b/test/Analysis/DependenceAnalysis/Banerjee.ll
index 8c28231164ed278a55af003f83451a4239076560..5dda3a9030f403edb9d8bf9d6ac87d506fb6924e 100644
--- a/test/Analysis/DependenceAnalysis/Banerjee.ll
+++ b/test/Analysis/DependenceAnalysis/Banerjee.ll
@@ -1,4 +1,8 @@
+; RUN: opt < %s -disable-output -da-delinearize=false "-passes=print<da>"      \
+; RUN: -aa-pipeline=basic-aa 2>&1 | FileCheck %s
 ; RUN: opt < %s -analyze -basicaa -da -da-delinearize=false | FileCheck %s
+; RUN: opt < %s -disable-output "-passes=print<da>" -aa-pipeline=basic-aa 2>&1 \
+; RUN: | FileCheck %s -check-prefix=DELIN
 ; RUN: opt < %s -analyze -basicaa -da | FileCheck %s -check-prefix=DELIN
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
diff --git a/test/Analysis/DependenceAnalysis/BasePtrBug.ll b/test/Analysis/DependenceAnalysis/BasePtrBug.ll
index 8de75df7dbdd7a98c2ed5ddf1a82d1f8569d4082..694e26afa09d457399cfa2340611b5e642d8f8f3 100644
--- a/test/Analysis/DependenceAnalysis/BasePtrBug.ll
+++ b/test/Analysis/DependenceAnalysis/BasePtrBug.ll
@@ -1,3 +1,5 @@
+; RUN: opt < %s -disable-output "-passes=print<da>" -aa-pipeline=basic-aa 2>&1 \
+; RUN: | FileCheck %s
 ; RUN: opt < %s -analyze -basicaa -da  | FileCheck %s
 
 ; Test that the dependence analysis generates the correct results when using
diff --git a/test/Analysis/DependenceAnalysis/Constraints.ll b/test/Analysis/DependenceAnalysis/Constraints.ll
index 42dfac767a7368b30a5338c1a046fd8a718ce6d8..2fbee702e8e5fe1b9ed82bba43dc13fee70d027d 100644
--- a/test/Analysis/DependenceAnalysis/Constraints.ll
+++ b/test/Analysis/DependenceAnalysis/Constraints.ll
@@ -1,3 +1,4 @@
+; RUN: opt < %s -disable-output "-passes=print<da>" -aa-pipeline=basic-aa 2>&1
 ; RUN: opt < %s -analyze -basicaa -da
 ;; Check that this code doesn't abort. Test case is reduced version of lnt Polybench benchmark test case dynprog.
 
diff --git a/test/Analysis/DependenceAnalysis/Coupled.ll b/test/Analysis/DependenceAnalysis/Coupled.ll
index 709567105ba3703ce59ffd578cb93d7a775d8fa2..0480087145b5ef07a9a46122c4fdaad6e6c02cc0 100644
--- a/test/Analysis/DependenceAnalysis/Coupled.ll
+++ b/test/Analysis/DependenceAnalysis/Coupled.ll
@@ -1,3 +1,5 @@
+; RUN: opt < %s -disable-output "-passes=print<da>" -aa-pipeline=basic-aa 2>&1 \
+; RUN: | FileCheck %s
 ; RUN: opt < %s -analyze -basicaa -da | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
diff --git a/test/Analysis/DependenceAnalysis/DADelin.ll b/test/Analysis/DependenceAnalysis/DADelin.ll
index 8438c2bf425932bb00ccf0b9b2decf4bd47f4a7f..5c70624068522e5f382cac83028ac8f436159b2e 100644
--- a/test/Analysis/DependenceAnalysis/DADelin.ll
+++ b/test/Analysis/DependenceAnalysis/DADelin.ll
@@ -1,3 +1,5 @@
+; RUN: opt < %s -disable-output "-passes=print<da>" -aa-pipeline=basic-aa 2>&1 \
+; RUN: | FileCheck %s
 ; RUN: opt < %s -analyze -basicaa -da | FileCheck %s
 
 target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64"
diff --git a/test/Analysis/DependenceAnalysis/ExactRDIV.ll b/test/Analysis/DependenceAnalysis/ExactRDIV.ll
index 5b2488c1bc8ebeab5d565a33cf53dac12e3590a6..44b399cb191bf467036fab8b4cd6721caee7e44e 100644
--- a/test/Analysis/DependenceAnalysis/ExactRDIV.ll
+++ b/test/Analysis/DependenceAnalysis/ExactRDIV.ll
@@ -1,3 +1,5 @@
+; RUN: opt < %s -disable-output "-passes=print<da>" -aa-pipeline=basic-aa 2>&1 \
+; RUN: | FileCheck %s
 ; RUN: opt < %s -analyze -basicaa -da | FileCheck %s
 
 ; ModuleID = 'ExactRDIV.bc'
diff --git a/test/Analysis/DependenceAnalysis/ExactSIV.ll b/test/Analysis/DependenceAnalysis/ExactSIV.ll
index e815799fa994f928d74c700838868c3619bc8632..f3fba8258bdf596e398a2b0ad6c41f691be6c999 100644
--- a/test/Analysis/DependenceAnalysis/ExactSIV.ll
+++ b/test/Analysis/DependenceAnalysis/ExactSIV.ll
@@ -1,3 +1,5 @@
+; RUN: opt < %s -disable-output "-passes=print<da>" -aa-pipeline=basic-aa 2>&1 \
+; RUN: | FileCheck %s
 ; RUN: opt < %s -analyze -basicaa -da | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
diff --git a/test/Analysis/DependenceAnalysis/GCD.ll b/test/Analysis/DependenceAnalysis/GCD.ll
index b0fcda4a6d5b81e58dd889b9c432f5183b4080f1..5f734b70f5c7bb62c51328afe950ae9787c2484a 100644
--- a/test/Analysis/DependenceAnalysis/GCD.ll
+++ b/test/Analysis/DependenceAnalysis/GCD.ll
@@ -1,3 +1,5 @@
+; RUN: opt < %s -disable-output "-passes=print<da>" -aa-pipeline=basic-aa 2>&1 \
+; RUN: | FileCheck %s -check-prefix=DELIN
 ; RUN: opt < %s -analyze -basicaa -da | FileCheck %s -check-prefix=DELIN
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
diff --git a/test/Analysis/DependenceAnalysis/Invariant.ll b/test/Analysis/DependenceAnalysis/Invariant.ll
index c96412771bbc26ebe521f034ad4ac35fb0ef346d..6835582ac4c88ba8cd902af0af67d1e296402127 100644
--- a/test/Analysis/DependenceAnalysis/Invariant.ll
+++ b/test/Analysis/DependenceAnalysis/Invariant.ll
@@ -1,3 +1,5 @@
+; RUN: opt < %s -disable-output "-passes=print<da>" -aa-pipeline=basic-aa 2>&1 \
+; RUN: | FileCheck %s
 ; RUN: opt < %s -analyze -basicaa -da | FileCheck %s
 
 ; Test for a bug, which caused an assert when an invalid
diff --git a/test/Analysis/DependenceAnalysis/MIVCheckConst.ll b/test/Analysis/DependenceAnalysis/MIVCheckConst.ll
index 64e72df55b8578332b561ef4f0d54e9c3384d3ca..d5dd1050b199060b174bba9d8187b89620061427 100644
--- a/test/Analysis/DependenceAnalysis/MIVCheckConst.ll
+++ b/test/Analysis/DependenceAnalysis/MIVCheckConst.ll
@@ -1,4 +1,5 @@
 ; RUN: opt < %s -analyze -basicaa -da
+; RUN: opt < %s -passes="print<da>"
 
 ; Test that the dependence analysis pass does seg-fault due to a null pointer
 ; dereference. The code in gcdMIVTest requires a null check for the result of
diff --git a/test/Analysis/DependenceAnalysis/NonAffineExpr.ll b/test/Analysis/DependenceAnalysis/NonAffineExpr.ll
index f12e20f336a6a43199e107ec40a1f5d7dd43385d..801b3f6dd6ff8fd7bbd2eabb2fe45fbd58c48bbd 100644
--- a/test/Analysis/DependenceAnalysis/NonAffineExpr.ll
+++ b/test/Analysis/DependenceAnalysis/NonAffineExpr.ll
@@ -1,3 +1,4 @@
+; RUN: opt < %s -disable-output "-passes=print<da>" -aa-pipeline=basic-aa 2>&1
 ; RUN: opt < %s -analyze -basicaa -da
 ;
 ; CHECK: da analyze - consistent input [S S]!
diff --git a/test/Analysis/DependenceAnalysis/NonCanonicalizedSubscript.ll b/test/Analysis/DependenceAnalysis/NonCanonicalizedSubscript.ll
index afeb061eb5e5710578e6486a1b66e09b859feb33..2aba9924381dd4326cdeb4d93624af186e38af5a 100644
--- a/test/Analysis/DependenceAnalysis/NonCanonicalizedSubscript.ll
+++ b/test/Analysis/DependenceAnalysis/NonCanonicalizedSubscript.ll
@@ -1,3 +1,5 @@
+; RUN: opt < %s -disable-output "-passes=print<da>" -aa-pipeline=basic-aa 2>&1 \
+; RUN: | FileCheck %s -check-prefix=DELIN
 ; RUN: opt < %s -analyze -basicaa -da | FileCheck %s -check-prefix=DELIN
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
@@ -98,4 +100,4 @@ attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "no-frame-pointe
 
 !llvm.ident = !{!0}
 
-!0 = !{!"clang version 3.7.0 (https://vaivaswatha@bitbucket.org/compilertree/amd_clang.git 93a05fb75ee3411d24e8b2b184fc766a5318403e) (https://vaivaswatha@bitbucket.org/compilertree/amd_llvm.git 166d93d26efc912b517739f64d054a435e8e95cd)"}
+!0 = !{!"clang version 3.7.0"}
diff --git a/test/Analysis/DependenceAnalysis/PR21585.ll b/test/Analysis/DependenceAnalysis/PR21585.ll
index 23af449ad51e7553b0fe640fcbca3a9bff384aaf..ff42ba96b234336f352f5f48c866d2644c0c6fc4 100644
--- a/test/Analysis/DependenceAnalysis/PR21585.ll
+++ b/test/Analysis/DependenceAnalysis/PR21585.ll
@@ -1,3 +1,5 @@
+; RUN: opt < %s -disable-output "-passes=print<da>"                            \
+; RUN: "-aa-pipeline=basic-aa,globals-aa" 2>&1 | FileCheck %s
 ; RUN: opt < %s -analyze -basicaa -globals-aa -da | FileCheck %s
 define void @i32_subscript(i32* %a) {
 entry:
diff --git a/test/Analysis/DependenceAnalysis/Preliminary.ll b/test/Analysis/DependenceAnalysis/Preliminary.ll
index 372d78a1acd3638fd4a1b95db7c7db395dc31dab..2c7e5dd0db3ba18b6539bae9196dc87291460e7a 100644
--- a/test/Analysis/DependenceAnalysis/Preliminary.ll
+++ b/test/Analysis/DependenceAnalysis/Preliminary.ll
@@ -1,3 +1,5 @@
+; RUN: opt < %s -disable-output "-passes=print<da>" -aa-pipeline=basic-aa 2>&1 \
+; RUN: | FileCheck %s
 ; RUN: opt < %s -analyze -basicaa -da | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
diff --git a/test/Analysis/DependenceAnalysis/Propagating.ll b/test/Analysis/DependenceAnalysis/Propagating.ll
index 0aa2567c9969f78cba4aad0b4763dfbfe32a7195..64d6a9cfae04e4acb4ce5a9237b0764318474443 100644
--- a/test/Analysis/DependenceAnalysis/Propagating.ll
+++ b/test/Analysis/DependenceAnalysis/Propagating.ll
@@ -1,3 +1,5 @@
+; RUN: opt < %s -disable-output "-passes=print<da>" -aa-pipeline=basic-aa 2>&1 \
+; RUN: | FileCheck %s
 ; RUN: opt < %s -analyze -basicaa -da | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
diff --git a/test/Analysis/DependenceAnalysis/Separability.ll b/test/Analysis/DependenceAnalysis/Separability.ll
index 8e6c775fedb632d6e53feeb8c8f5a89d350ab95f..b6004b8d5eee57ed6c9c811a7628959fbe3beb26 100644
--- a/test/Analysis/DependenceAnalysis/Separability.ll
+++ b/test/Analysis/DependenceAnalysis/Separability.ll
@@ -1,3 +1,5 @@
+; RUN: opt < %s -disable-output "-passes=print<da>" -aa-pipeline=basic-aa 2>&1 \
+; RUN: | FileCheck %s
 ; RUN: opt < %s -analyze -basicaa -da | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
diff --git a/test/Analysis/DependenceAnalysis/StrongSIV.ll b/test/Analysis/DependenceAnalysis/StrongSIV.ll
index 4a6136e98262b9fc77629205e93f4ec7f878a64e..f7528a52316f902b11be0c25dbd171dd6905a818 100644
--- a/test/Analysis/DependenceAnalysis/StrongSIV.ll
+++ b/test/Analysis/DependenceAnalysis/StrongSIV.ll
@@ -1,3 +1,5 @@
+; RUN: opt < %s -disable-output "-passes=print<da>" -aa-pipeline=basic-aa 2>&1 \
+; RUN: | FileCheck %s
 ; RUN: opt < %s -analyze -basicaa -da | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
diff --git a/test/Analysis/DependenceAnalysis/SymbolicRDIV.ll b/test/Analysis/DependenceAnalysis/SymbolicRDIV.ll
index 6e8b98ca26433c44206509e11441c0af963747bc..34c1415aeb5e93ada29c58672a62799b6684635c 100644
--- a/test/Analysis/DependenceAnalysis/SymbolicRDIV.ll
+++ b/test/Analysis/DependenceAnalysis/SymbolicRDIV.ll
@@ -1,3 +1,5 @@
+; RUN: opt < %s -disable-output "-passes=print<da>" -aa-pipeline=basic-aa 2>&1 \
+; RUN: | FileCheck %s
 ; RUN: opt < %s -analyze -basicaa -da | FileCheck %s
 
 ; ModuleID = 'SymbolicRDIV.bc'
diff --git a/test/Analysis/DependenceAnalysis/SymbolicSIV.ll b/test/Analysis/DependenceAnalysis/SymbolicSIV.ll
index 26c4770907b6d2f1a7af95d18cc096f1a0eda35f..b24547f73807e7362e9508adc76af08de1df7dfe 100644
--- a/test/Analysis/DependenceAnalysis/SymbolicSIV.ll
+++ b/test/Analysis/DependenceAnalysis/SymbolicSIV.ll
@@ -1,3 +1,5 @@
+; RUN: opt < %s -disable-output "-passes=print<da>" -aa-pipeline=basic-aa 2>&1 \
+; RUN: | FileCheck %s
 ; RUN: opt < %s -analyze -basicaa -da | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
diff --git a/test/Analysis/DependenceAnalysis/UsefulGEP.ll b/test/Analysis/DependenceAnalysis/UsefulGEP.ll
index cd46a275034cd0e40015059543f0679a505ebcbe..283fd2ce2e77039bccfe75bb3f3966645ce5b423 100644
--- a/test/Analysis/DependenceAnalysis/UsefulGEP.ll
+++ b/test/Analysis/DependenceAnalysis/UsefulGEP.ll
@@ -1,3 +1,4 @@
+; RUN: opt < %s -disable-output "-passes=print<da>" -aa-pipeline=basic-aa 2>&1
 ; RUN: opt < %s -analyze -basicaa -da
 ;; Check this doesn't crash.
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
diff --git a/test/Analysis/DependenceAnalysis/WeakCrossingSIV.ll b/test/Analysis/DependenceAnalysis/WeakCrossingSIV.ll
index 5b81ec10f9d824b9bed0485187243e436ef5cc78..119ae98bb530c76d9707f7aad82627f5f70005fb 100644
--- a/test/Analysis/DependenceAnalysis/WeakCrossingSIV.ll
+++ b/test/Analysis/DependenceAnalysis/WeakCrossingSIV.ll
@@ -1,3 +1,5 @@
+; RUN: opt < %s -disable-output "-passes=print<da>" -aa-pipeline=basic-aa 2>&1 \
+; RUN: | FileCheck %s
 ; RUN: opt < %s -analyze -basicaa -da | FileCheck %s
 
 ; ModuleID = 'WeakCrossingSIV.bc'
diff --git a/test/Analysis/DependenceAnalysis/WeakZeroDstSIV.ll b/test/Analysis/DependenceAnalysis/WeakZeroDstSIV.ll
index 128eb47018ae9bc33f0f73fed61254ad43abaa4c..37c912ca7165badc983160b68a0fc0e1ae03824f 100644
--- a/test/Analysis/DependenceAnalysis/WeakZeroDstSIV.ll
+++ b/test/Analysis/DependenceAnalysis/WeakZeroDstSIV.ll
@@ -1,3 +1,5 @@
+; RUN: opt < %s -disable-output "-passes=print<da>" -aa-pipeline=basic-aa 2>&1 \
+; RUN: | FileCheck %s
 ; RUN: opt < %s -analyze -basicaa -da | FileCheck %s
 
 ; ModuleID = 'WeakZeroDstSIV.bc'
diff --git a/test/Analysis/DependenceAnalysis/WeakZeroSrcSIV.ll b/test/Analysis/DependenceAnalysis/WeakZeroSrcSIV.ll
index 43c3de16935dfd93605061d59d393dd070e33011..602a79194e95629faad100e46095893528768ae0 100644
--- a/test/Analysis/DependenceAnalysis/WeakZeroSrcSIV.ll
+++ b/test/Analysis/DependenceAnalysis/WeakZeroSrcSIV.ll
@@ -1,3 +1,5 @@
+; RUN: opt < %s -disable-output "-passes=print<da>" -aa-pipeline=basic-aa 2>&1 \
+; RUN: | FileCheck %s
 ; RUN: opt < %s -analyze -basicaa -da | FileCheck %s
 
 ; ModuleID = 'WeakZeroSrcSIV.bc'
diff --git a/test/Analysis/DependenceAnalysis/ZIV.ll b/test/Analysis/DependenceAnalysis/ZIV.ll
index b3216415c8e85f7777332f0c32603b811129910c..c2049a629776092a2ad6d460993ad0675f3d2851 100644
--- a/test/Analysis/DependenceAnalysis/ZIV.ll
+++ b/test/Analysis/DependenceAnalysis/ZIV.ll
@@ -1,3 +1,5 @@
+; RUN: opt < %s -disable-output "-passes=print<da>" -aa-pipeline=basic-aa 2>&1 \
+; RUN: | FileCheck %s
 ; RUN: opt < %s -analyze -basicaa -da | FileCheck %s
 
 ; ModuleID = 'ZIV.bc'
diff --git a/test/Analysis/DependenceAnalysis/new-pm-invalidation.ll b/test/Analysis/DependenceAnalysis/new-pm-invalidation.ll
new file mode 100644
index 0000000000000000000000000000000000000000..c6af10b19c5c204e2ee5d04fc96b2f6766874508
--- /dev/null
+++ b/test/Analysis/DependenceAnalysis/new-pm-invalidation.ll
@@ -0,0 +1,16 @@
+; RUN: opt < %s -passes='require<da>,invalidate<scalar-evolution>,print<da>'   \
+; RUN:   -disable-output -debug-pass-manager 2>&1 | FileCheck %s
+
+; CHECK: Running analysis: DependenceAnalysis on test_no_noalias
+; CHECK: Running analysis: ScalarEvolutionAnalysis on test_no_noalias
+; CHECK: Invalidating analysis: ScalarEvolutionAnalysis on test_no_noalias
+; CHECK: Invalidating analysis: DependenceAnalysis on test_no_noalias
+; CHECK: Running analysis: DependenceAnalysis on test_no_noalias
+; CHECK: da analyze - none!
+; CHECK: da analyze - confused!
+; CHECK: da analyze - none!
+define void @test_no_noalias(i32* %A, i32* %B) {
+  store i32 1, i32* %A
+  store i32 2, i32* %B
+  ret void
+}
diff --git a/test/Analysis/DivergenceAnalysis/AMDGPU/always_uniform.ll b/test/Analysis/DivergenceAnalysis/AMDGPU/always_uniform.ll
index eaac9ce526c4be351ad813c4ce6392cd2e445f32..b5e8e49c68768252dab16c3268a4621b59f50160 100644
--- a/test/Analysis/DivergenceAnalysis/AMDGPU/always_uniform.ll
+++ b/test/Analysis/DivergenceAnalysis/AMDGPU/always_uniform.ll
@@ -1,6 +1,7 @@
 ; RUN: opt  -mtriple amdgcn-unknown-amdhsa -analyze -divergence -use-gpu-divergence-analysis %s | FileCheck %s
 
-define amdgpu_kernel void @workitem_id_x() #1 {
+; CHECK: for function 'readfirstlane':
+define amdgpu_kernel void @readfirstlane() {
   %id.x = call i32 @llvm.amdgcn.workitem.id.x()
 ; CHECK: DIVERGENT:  %id.x = call i32 @llvm.amdgcn.workitem.id.x()
   %first.lane = call i32 @llvm.amdgcn.readfirstlane(i32 %id.x)
@@ -8,7 +9,24 @@ define amdgpu_kernel void @workitem_id_x() #1 {
   ret void
 }
 
+; CHECK: for function 'icmp':
+define amdgpu_kernel void @icmp(i32 inreg %x) {
+; CHECK-NOT: DIVERGENT:  %icmp = call i64 @llvm.amdgcn.icmp.i32
+  %icmp = call i64 @llvm.amdgcn.icmp.i32(i32 %x, i32 0, i32 33)
+  ret void
+}
+
+; CHECK: for function 'fcmp':
+define amdgpu_kernel void @fcmp(float inreg %x, float inreg %y) {
+; CHECK-NOT: DIVERGENT:  %fcmp = call i64 @llvm.amdgcn.fcmp.i32
+  %fcmp = call i64 @llvm.amdgcn.fcmp.i32(float %x, float %y, i32 33)
+  ret void
+}
+
 declare i32 @llvm.amdgcn.workitem.id.x() #0
 declare i32 @llvm.amdgcn.readfirstlane(i32) #0
+declare i64 @llvm.amdgcn.icmp.i32(i32, i32, i32) #1
+declare i64 @llvm.amdgcn.fcmp.i32(float, float, i32) #1
 
 attributes #0 = { nounwind readnone }
+attributes #1 = { nounwind readnone convergent }
diff --git a/test/Analysis/DivergenceAnalysis/AMDGPU/interp_f16.ll b/test/Analysis/DivergenceAnalysis/AMDGPU/interp_f16.ll
new file mode 100644
index 0000000000000000000000000000000000000000..174dd5679782c24c7a4d48cdaf0b3275eeafebd5
--- /dev/null
+++ b/test/Analysis/DivergenceAnalysis/AMDGPU/interp_f16.ll
@@ -0,0 +1,25 @@
+; RUN: opt -mtriple=amdgcn-- -analyze -divergence -use-gpu-divergence-analysis %s | FileCheck %s
+
+; CHECK: for function 'interp_p1_f16'
+; CHECK: DIVERGENT:       %p1 = call float @llvm.amdgcn.interp.p1.f16
+define amdgpu_ps float @interp_p1_f16(float inreg %i, float inreg %j, i32 inreg %m0) #0 {
+main_body:
+  %p1 = call float @llvm.amdgcn.interp.p1.f16(float %i, i32 1, i32 2, i1 0, i32 %m0)
+  ret float %p1
+}
+
+; CHECK: for function 'interp_p2_f16'
+; CHECK: DIVERGENT:       %p2 = call half @llvm.amdgcn.interp.p2.f16
+define amdgpu_ps half @interp_p2_f16(float inreg %i, float inreg %j, i32 inreg %m0) #0 {
+main_body:
+  %p2 = call half @llvm.amdgcn.interp.p2.f16(float %i, float %j, i32 1, i32 2, i1 0, i32 %m0)
+  ret half %p2
+}
+
+; float @llvm.amdgcn.interp.p1.f16(i, attrchan, attr, high, m0)
+declare float @llvm.amdgcn.interp.p1.f16(float, i32, i32, i1, i32) #0
+; half @llvm.amdgcn.interp.p1.f16(p1, j, attrchan, attr, high, m0)
+declare half @llvm.amdgcn.interp.p2.f16(float, float, i32, i32, i1, i32) #0
+declare float @llvm.amdgcn.interp.mov(i32, i32, i32, i32) #0
+
+attributes #0 = { nounwind readnone }
diff --git a/test/Analysis/LoopInfo/annotated-parallel-complex.ll b/test/Analysis/LoopInfo/annotated-parallel-complex.ll
new file mode 100644
index 0000000000000000000000000000000000000000..661e2660de537f653ce163b829a083e1a5a8a635
--- /dev/null
+++ b/test/Analysis/LoopInfo/annotated-parallel-complex.ll
@@ -0,0 +1,91 @@
+; RUN: opt -loops -analyze < %s | FileCheck %s
+;
+; void func(long n, double A[static const restrict 4*n], double B[static const restrict 4*n]) {
+;   for (long i = 0; i < n; i += 1)
+;     for (long j = 0; j < n; j += 1)
+;       for (long k = 0; k < n; k += 1)
+;         for (long l = 0; l < n; l += 1) {
+;           A[i + j + k + l] = 21;
+;           B[i + j + k + l] = 42;
+;         }
+; }
+;
+; Check that isAnnotatedParallel is working as expected.
+;
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+define void @func(i64 %n, double* noalias nonnull %A, double* noalias nonnull %B) {
+entry:
+  br label %for.cond
+
+for.cond:
+  %i.0 = phi i64 [ 0, %entry ], [ %add28, %for.inc27 ]
+  %cmp = icmp slt i64 %i.0, %n
+  br i1 %cmp, label %for.cond2, label %for.end29
+
+for.cond2:
+  %j.0 = phi i64 [ %add25, %for.inc24 ], [ 0, %for.cond ]
+  %cmp3 = icmp slt i64 %j.0, %n
+  br i1 %cmp3, label %for.cond6, label %for.inc27
+
+for.cond6:
+  %k.0 = phi i64 [ %add22, %for.inc21 ], [ 0, %for.cond2 ]
+  %cmp7 = icmp slt i64 %k.0, %n
+  br i1 %cmp7, label %for.cond10, label %for.inc24
+
+for.cond10:
+  %l.0 = phi i64 [ %add20, %for.body13 ], [ 0, %for.cond6 ]
+  %cmp11 = icmp slt i64 %l.0, %n
+  br i1 %cmp11, label %for.body13, label %for.inc21
+
+for.body13:
+  %add = add nuw nsw i64 %i.0, %j.0
+  %add14 = add nuw nsw i64 %add, %k.0
+  %add15 = add nuw nsw i64 %add14, %l.0
+  %arrayidx = getelementptr inbounds double, double* %A, i64 %add15
+  store double 2.100000e+01, double* %arrayidx, align 8, !llvm.access.group !5
+  %add16 = add nuw nsw i64 %i.0, %j.0
+  %add17 = add nuw nsw i64 %add16, %k.0
+  %add18 = add nuw nsw i64 %add17, %l.0
+  %arrayidx19 = getelementptr inbounds double, double* %B, i64 %add18
+  store double 4.200000e+01, double* %arrayidx19, align 8, !llvm.access.group !6
+  %add20 = add nuw nsw i64 %l.0, 1
+  br label %for.cond10, !llvm.loop !11
+
+for.inc21:
+  %add22 = add nuw nsw i64 %k.0, 1
+  br label %for.cond6, !llvm.loop !14
+
+for.inc24:
+  %add25 = add nuw nsw i64 %j.0, 1
+  br label %for.cond2, !llvm.loop !16
+
+for.inc27:
+  %add28 = add nuw nsw i64 %i.0, 1
+  br label %for.cond, !llvm.loop !18
+
+for.end29:
+  ret void
+}
+
+; access groups
+!7 = distinct !{}
+!8 = distinct !{}
+!10 = distinct !{}
+
+; access group lists
+!5 = !{!7, !10}
+!6 = !{!7, !8, !10}
+
+; LoopIDs
+!11 = distinct !{!11, !{!"llvm.loop.parallel_accesses", !10}}
+!14 = distinct !{!14, !{!"llvm.loop.parallel_accesses", !8, !10}}
+!16 = distinct !{!16, !{!"llvm.loop.parallel_accesses", !8}}
+!18 = distinct !{!18, !{!"llvm.loop.parallel_accesses", !7}}
+
+
+; CHECK: Parallel Loop at depth 1
+; CHECK-NOT: Parallel
+; CHECK:     Loop at depth 2
+; CHECK:         Parallel Loop
+; CHECK:             Parallel Loop
diff --git a/test/Analysis/LoopInfo/annotated-parallel-simple.ll b/test/Analysis/LoopInfo/annotated-parallel-simple.ll
new file mode 100644
index 0000000000000000000000000000000000000000..4e25af899799d8a665ab91dd8e19ba55b7febdf0
--- /dev/null
+++ b/test/Analysis/LoopInfo/annotated-parallel-simple.ll
@@ -0,0 +1,37 @@
+; RUN: opt -loops -analyze < %s | FileCheck %s
+;
+; void func(long n, double A[static const restrict n]) {
+;   for (long i = 0; i < n; i += 1)
+;     A[i] = 21;
+; }
+;
+; Check that isAnnotatedParallel is working as expected.
+;
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+define void @func(i64 %n, double* noalias nonnull %A) {
+entry:
+  br label %for.cond
+
+for.cond:
+  %i.0 = phi i64 [ 0, %entry ], [ %add, %for.body ]
+  %cmp = icmp slt i64 %i.0, %n
+  br i1 %cmp, label %for.body, label %for.end
+
+for.body:
+  %arrayidx = getelementptr inbounds double, double* %A, i64 %i.0
+  store double 2.100000e+01, double* %arrayidx, align 8, !llvm.access.group !6
+  %add = add nuw nsw i64 %i.0, 1
+  br label %for.cond, !llvm.loop !7
+
+for.end:
+  ret void
+}
+
+!6 = distinct !{} ; access group
+
+!7 = distinct !{!7, !9} ; LoopID
+!9 = !{!"llvm.loop.parallel_accesses", !6}
+
+
+; CHECK: Parallel Loop
diff --git a/test/Analysis/MemorySSA/pr40509.ll b/test/Analysis/MemorySSA/pr40509.ll
new file mode 100644
index 0000000000000000000000000000000000000000..4827acf1b802b595f98e25eb6c84dba32feecc68
--- /dev/null
+++ b/test/Analysis/MemorySSA/pr40509.ll
@@ -0,0 +1,54 @@
+; REQUIRES: asserts
+; RUN: opt -mtriple=systemz-unknown -march=z13 -O3 -enable-mssa-loop-dependency -disable-output %s
+
+; During transform to LCSSA, an access becomes obfuscated to:
+; (2 = phi (phi(val), val)), which BasicAA fails to analyze.
+; It's currently hard coded in BasicAA to return MayAlias for nested phis.
+; This leads MemorySSA to finding a new (false) clobber for a previously
+; optimized access. With verifyClobber included in verifyMemorySSA, such a
+; transformation will cause MemorySSA verification to fail.
+; If the verifyClobber is re-enabled, this test will crash.
+
+target datalayout = "E-m:e-i1:8:16-i8:8:16-i64:64-f128:64-v128:64-a:8:16-n32:64"
+target triple = "s390x-ibm-linux"
+
+%0 = type <{ i64, i8, i64, i16 }>
+
+@g_54 = external dso_local global i16, align 2
+@g_101 = external dso_local global <{ i64, i8, i64, i8, i8 }>, align 2
+
+declare dso_local void @safe_lshift_func_int16_t_s_s()
+declare dso_local i8 @safe_div_func_int8_t_s_s()
+
+define dso_local void @func_47(%0* %arg) {
+bb:
+  %tmp = alloca i32, align 4
+  br label %bb1
+
+bb1:                                              ; preds = %bb12, %bb
+  %tmp2 = getelementptr inbounds %0, %0* %arg, i32 0, i32 3
+  store i16 undef, i16* %tmp2, align 1
+  %tmp3 = call signext i8 @safe_div_func_int8_t_s_s()
+  %tmp7 = icmp ne i8 %tmp3, 0
+  br i1 %tmp7, label %bb8, label %bb10
+
+bb8:                                              ; preds = %bb1
+  %tmp9 = icmp eq i32 0, 0
+  br i1 %tmp9, label %bb12, label %bb13
+
+bb10:                                             ; preds = %bb10, %bb1
+  call void @safe_lshift_func_int16_t_s_s()
+  %tmp11 = getelementptr inbounds %0, %0* %arg, i32 0, i32 3
+  store i16 0, i16* %tmp11, align 1
+  store i8 0, i8* getelementptr inbounds (%0, %0* bitcast (<{ i64, i8, i64, i8, i8 }>* @g_101 to %0*), i32 0, i32 1), align 2
+  br label %bb10
+
+bb12:                                             ; preds = %bb8
+  store i16 0, i16* @g_54, align 2
+  br label %bb1
+
+bb13:                                             ; preds = %bb8
+  ret void
+}
+
+
diff --git a/test/Analysis/ScalarEvolution/huge_expression_limit.ll b/test/Analysis/ScalarEvolution/huge_expression_limit.ll
new file mode 100644
index 0000000000000000000000000000000000000000..5740915783189e3c8fa754b892dd4b46b597fdfe
--- /dev/null
+++ b/test/Analysis/ScalarEvolution/huge_expression_limit.ll
@@ -0,0 +1,41 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
+; RUN: opt < %s -analyze -scalar-evolution -scalar-evolution-huge-expr-threshold=1 | FileCheck %s
+
+define void @test(i32 %a, i32 %b, i32 %c, i32 %d, i32 %e, i32 %f) {
+; CHECK-LABEL: 'test'
+; CHECK-NEXT:  Classifying expressions for: @test
+; CHECK-NEXT:    %add1 = add i32 %a, %b
+; CHECK-NEXT:    --> (%a + %b) U: full-set S: full-set
+; CHECK-NEXT:    %add2 = add i32 %add1, %c
+; CHECK-NEXT:    --> ((%a + %b) + %c) U: full-set S: full-set
+; CHECK-NEXT:    %add3 = add i32 %add2, %d
+; CHECK-NEXT:    --> (((%a + %b) + %c) + %d) U: full-set S: full-set
+; CHECK-NEXT:    %add4 = add i32 %add3, %e
+; CHECK-NEXT:    --> ((((%a + %b) + %c) + %d) + %e) U: full-set S: full-set
+; CHECK-NEXT:    %add5 = add i32 %add4, %f
+; CHECK-NEXT:    --> (((((%a + %b) + %c) + %d) + %e) + %f) U: full-set S: full-set
+; CHECK-NEXT:    %mul1 = mul i32 %a, %b
+; CHECK-NEXT:    --> (%a * %b) U: full-set S: full-set
+; CHECK-NEXT:    %mul2 = mul i32 %mul1, %c
+; CHECK-NEXT:    --> ((%a * %b) * %c) U: full-set S: full-set
+; CHECK-NEXT:    %mul3 = mul i32 %mul2, %d
+; CHECK-NEXT:    --> (((%a * %b) * %c) * %d) U: full-set S: full-set
+; CHECK-NEXT:    %mul4 = mul i32 %mul3, %e
+; CHECK-NEXT:    --> ((((%a * %b) * %c) * %d) * %e) U: full-set S: full-set
+; CHECK-NEXT:    %mul5 = mul i32 %mul4, %f
+; CHECK-NEXT:    --> (((((%a * %b) * %c) * %d) * %e) * %f) U: full-set S: full-set
+; CHECK-NEXT:  Determining loop execution counts for: @test
+;
+  %add1 = add i32 %a, %b
+  %add2 = add i32 %add1, %c
+  %add3 = add i32 %add2, %d
+  %add4 = add i32 %add3, %e
+  %add5 = add i32 %add4, %f
+
+  %mul1 = mul i32 %a, %b
+  %mul2 = mul i32 %mul1, %c
+  %mul3 = mul i32 %mul2, %d
+  %mul4 = mul i32 %mul3, %e
+  %mul5 = mul i32 %mul4, %f
+  ret void
+}
diff --git a/test/Analysis/ScalarEvolution/pr40420.ll b/test/Analysis/ScalarEvolution/pr40420.ll
new file mode 100644
index 0000000000000000000000000000000000000000..0f57958c24c6da51833b2543a5b943ed616376e6
--- /dev/null
+++ b/test/Analysis/ScalarEvolution/pr40420.ll
@@ -0,0 +1,50 @@
+; RUN: opt < %s -analyze -scalar-evolution 2>&1 | FileCheck %s
+; REQUIRES: asserts
+
+define void @test(i8 %tmp6) {
+
+; CHECK-LABEL: Classifying expressions for: @test
+
+bb:
+  br label %outer_loop
+
+outer_loop:                                              ; preds = %outer_latch, %bb
+  %tmp8 = phi i8 [ %tmp6, %bb ], [ %tmp20.lcssa, %outer_latch ]
+  %tmp9 = phi i64 [ 0, %bb ], [ %tmp11, %outer_latch ]
+  br label %inner_loop
+
+outer_latch:                                             ; preds = %inner_loop
+  %tmp20.lcssa = phi i8 [ %tmp20.6, %inner_loop ]
+  %tmp11 = add i64 %tmp9, 239
+  br label %outer_loop
+
+inner_loop:                                             ; preds = %inner_latch, %outer_loop
+  %tmp13 = phi i8 [ %tmp8, %outer_loop ], [ %tmp20.1, %inner_latch ]
+  %tmp14 = phi i64 [ %tmp9, %outer_loop ], [ %tmp16.7, %inner_latch ]
+  %tmp15 = phi i64 [ 2, %outer_loop ], [ %tmp21.7, %inner_latch ]
+  %tmp16 = add i64 %tmp14, 1
+  %tmp17 = trunc i64 %tmp16 to i8
+  %tmp18 = mul i8 %tmp13, 100
+  %tmp19 = mul i8 %tmp18, %tmp13
+  %tmp20 = mul i8 %tmp19, %tmp17
+  %tmp21 = add nuw nsw i64 %tmp15, 1
+  %tmp16.1 = add i64 %tmp16, 1
+  %tmp17.1 = trunc i64 %tmp16.1 to i8
+  %tmp18.1 = mul i8 %tmp20, 100
+  %tmp19.1 = mul i8 %tmp18.1, %tmp20
+  %tmp20.1 = mul i8 %tmp19.1, %tmp17.1
+  %tmp21.1 = add nuw nsw i64 %tmp21, 1
+  %tmp16.2 = add i64 %tmp16.1, 1
+  %tmp16.3 = add i64 %tmp16.2, 1
+  %tmp16.4 = add i64 %tmp16.3, 1
+  %tmp16.5 = add i64 %tmp16.4, 1
+  %tmp16.6 = add i64 %tmp16.5, 1
+  %tmp20.6 = mul i8 %tmp19.1, %tmp17.1
+  %tmp22.6 = icmp ugt i64 %tmp16.1, 239
+  br i1 %tmp22.6, label %outer_latch, label %inner_latch
+
+inner_latch:                                           ; preds = %inner_loop
+  %tmp16.7 = add i64 %tmp16.6, 1
+  %tmp21.7 = add nuw nsw i64 %tmp21.1, 1
+  br label %inner_loop
+}
diff --git a/test/Analysis/ScalarEvolution/unreachable-code.ll b/test/Analysis/ScalarEvolution/unreachable-code.ll
index 51d93981800a22d19f3285d48f7b281cb4350fb5..69a7e39839a7e11f804dc3798c3b1d6e851253ba 100644
--- a/test/Analysis/ScalarEvolution/unreachable-code.ll
+++ b/test/Analysis/ScalarEvolution/unreachable-code.ll
@@ -1,7 +1,7 @@
 ; RUN: opt < %s -analyze -scalar-evolution | FileCheck %s
 
 ; CHECK: %t = add i64 %t, 1
-; CHECK: -->  %t
+; CHECK: -->  undef
 
 define void @foo() {
 entry:
diff --git a/test/Analysis/ValueTracking/gep-negative-issue.ll b/test/Analysis/ValueTracking/gep-negative-issue.ll
new file mode 100644
index 0000000000000000000000000000000000000000..84088df55ff8ba17712effb01b91fee4eb95a1e6
--- /dev/null
+++ b/test/Analysis/ValueTracking/gep-negative-issue.ll
@@ -0,0 +1,44 @@
+; RUN: opt -gvn -S < %s | FileCheck %s
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128-p100:128:64:64-p101:128:64:64"
+target triple = "x86_64-unknown-linux-gnu"
+
+%ArrayImpl = type { i64, i64 addrspace(100)*, [1 x i64], [1 x i64], [1 x i64], i64, i64, double addrspace(100)*, double addrspace(100)*, i8, i64 }
+%_array = type { i64, %ArrayImpl addrspace(100)*, i8 }
+
+define void @test(i64 %n_chpl) {
+entry:
+  ; First section is some code
+  %0 = getelementptr inbounds %_array, %_array* null, i32 0, i32 1
+  %1 = load %ArrayImpl addrspace(100)*, %ArrayImpl addrspace(100)** %0
+  %2 = getelementptr inbounds %ArrayImpl, %ArrayImpl addrspace(100)* %1, i32 0, i32 8
+  %3 = load double addrspace(100)*, double addrspace(100)* addrspace(100)* %2
+  %4 = getelementptr inbounds double, double addrspace(100)* %3, i64 -1
+  ; Second section is that code repeated
+  %x0 = getelementptr inbounds %_array, %_array* null, i32 0, i32 1
+  %x1 = load %ArrayImpl addrspace(100)*, %ArrayImpl addrspace(100)** %x0
+  %x2 = getelementptr inbounds %ArrayImpl, %ArrayImpl addrspace(100)* %x1, i32 0, i32 8
+  %x3 = load double addrspace(100)*, double addrspace(100)* addrspace(100)* %x2
+  %x4 = getelementptr inbounds double, double addrspace(100)* %x3, i64 -1
+  ; These two stores refer to the same memory location
+  ; Even so, they are expected to remain separate stores here
+  store double 0.000000e+00, double addrspace(100)* %4
+  store double 0.000000e+00, double addrspace(100)* %x4
+  ; Third section is the repeated code again, with a later store
+  ; This third section is necessary to trigger the crash
+  %y1 = load %ArrayImpl addrspace(100)*, %ArrayImpl addrspace(100)** %0
+  %y2 = getelementptr inbounds %ArrayImpl, %ArrayImpl addrspace(100)* %y1, i32 0, i32 8
+  %y3 = load double addrspace(100)*, double addrspace(100)* addrspace(100)* %y2
+  %y4 = getelementptr inbounds double, double addrspace(100)* %y3, i64 -1
+  store double 0.000000e+00, double addrspace(100)* %y4
+  ret void
+; CHECK-LABEL: define void @test
+; CHECK: getelementptr inbounds double, double addrspace(100)* {{%.*}}, i64 -1
+; CHECK-NEXT: store double 0.000000e+00, double addrspace(100)* [[DST:%.*]]
+; CHECK-NEXT: store double 0.000000e+00, double addrspace(100)* [[DST]]
+; CHECK: load
+; CHECK: getelementptr inbounds %ArrayImpl, %ArrayImpl addrspace(100)*
+; CHECK: load
+; CHECK: getelementptr inbounds double, double addrspace(100)* {{%.*}}, i64 -1
+; CHECK: store double 0.000000e+00, double addrspace(100)*
+; CHECK: ret
+}
diff --git a/test/Assembler/2004-03-07-FunctionAddressAlignment.ll b/test/Assembler/2004-03-07-FunctionAddressAlignment.ll
new file mode 100644
index 0000000000000000000000000000000000000000..7fa0802db405c9852be6867c60785bf849fcc49a
--- /dev/null
+++ b/test/Assembler/2004-03-07-FunctionAddressAlignment.ll
@@ -0,0 +1,16 @@
+; RUN: llvm-as < %s | llvm-dis | not grep ptrtoint
+; RUN: verify-uselistorder %s
+; All of these should be eliminable
+
+
+define i32 @foo() {
+	ret i32 and (i32 ptrtoint (i32()* @foo to i32), i32 1)
+}
+
+define i32 @foo2() {
+	ret i32 and (i32 1, i32 ptrtoint (i32()* @foo2 to i32))
+}
+
+define i1 @foo3() {
+	ret i1 icmp ne (i1()* @foo3, i1()* null)
+}
diff --git a/test/Assembler/DIEnumerator.ll b/test/Assembler/DIEnumerator.ll
index 9dc3d7c9ec277c4e5244df88d90e2ba46941d6be..fdb91d451cc23cf5fef4994a4e19695800fafe15 100644
--- a/test/Assembler/DIEnumerator.ll
+++ b/test/Assembler/DIEnumerator.ll
@@ -40,9 +40,9 @@
 ; CHECK: !DIEnumerator(name: "B0", value: 2147483647)
 
 
-!10 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "E1", file: !3, line: 3, baseType: !6, size: 32, flags: DIFlagFixedEnum, elements: !11, identifier: "_ZTS2E1")
+!10 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "E1", file: !3, line: 3, baseType: !6, size: 32, flags: DIFlagEnumClass, elements: !11, identifier: "_ZTS2E1")
 ; CHECK: !DICompositeType(tag: DW_TAG_enumeration_type, name: "E1"{{.*}}, baseType: ![[INT]]
-; CHECK-SAME: DIFlagFixedEnum
+; CHECK-SAME: DIFlagEnumClass
 !11 = !{!12, !13}
 !12 = !DIEnumerator(name: "A1", value: -2147483648)
 !13 = !DIEnumerator(name: "B1", value: 2147483647)
@@ -50,9 +50,9 @@
 ; CHECK: !DIEnumerator(name: "B1", value: 2147483647)
 
 
-!14 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "E2", file: !3, line: 5, baseType: !15, size: 64, flags: DIFlagFixedEnum, elements: !16, identifier: "_ZTS2E2")
+!14 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "E2", file: !3, line: 5, baseType: !15, size: 64, flags: DIFlagEnumClass, elements: !16, identifier: "_ZTS2E2")
 ; CHECK: !DICompositeType(tag: DW_TAG_enumeration_type, name: "E2"{{.*}}, baseType: ![[LONG:[0-9]+]]
-; CHECK-SAME: DIFlagFixedEnum
+; CHECK-SAME: DIFlagEnumClass
 !15 = !DIBasicType(name: "long long int", size: 64, encoding: DW_ATE_signed)
 ; CHECK: ![[LONG]] = !DIBasicType(name: "long long int", size: 64, encoding: DW_ATE_signed)
 !16 = !{!17, !18}
@@ -62,9 +62,9 @@
 ; CHECK: !DIEnumerator(name: "B2", value: 9223372036854775807)
 
 
-!19 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "E3", file: !3, line: 7, baseType: !20, size: 64, flags: DIFlagFixedEnum, elements: !21, identifier: "_ZTS2E3")
+!19 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "E3", file: !3, line: 7, baseType: !20, size: 64, flags: DIFlagEnumClass, elements: !21, identifier: "_ZTS2E3")
 ; CHECK: !DICompositeType(tag: DW_TAG_enumeration_type, name: "E3"{{.*}}, baseType: ![[ULONG:[0-9]+]]
-; CHECK-SAME: DIFlagFixedEnum
+; CHECK-SAME: DIFlagEnumClass
 !20 = !DIBasicType(name: "long long unsigned int", size: 64, encoding: DW_ATE_unsigned)
 ; CHECK: ![[ULONG]] = !DIBasicType(name: "long long unsigned int", size: 64, encoding: DW_ATE_unsigned)
 !21 = !{!22}
diff --git a/test/Assembler/atomic.ll b/test/Assembler/atomic.ll
index a8b527f2f863c4a326ff50989983ddbaf4197063..04ff262074f21dac4ec964fdcea34486794159ef 100644
--- a/test/Assembler/atomic.ll
+++ b/test/Assembler/atomic.ll
@@ -39,3 +39,13 @@ define void @f(i32* %x) {
   fence syncscope("device") seq_cst
   ret void
 }
+
+define void @fp_atomics(float* %x) {
+ ; CHECK: atomicrmw fadd float* %x, float 1.000000e+00 seq_cst
+  atomicrmw fadd float* %x, float 1.0 seq_cst
+
+  ; CHECK: atomicrmw volatile fadd float* %x, float 1.000000e+00 seq_cst
+  atomicrmw volatile fadd float* %x, float 1.0 seq_cst
+
+  ret void
+}
diff --git a/test/Assembler/auto_upgrade_intrinsics.ll b/test/Assembler/auto_upgrade_intrinsics.ll
index 87ad371deaa556e0e062c2b132050c9fdddd6127..bd2a779800a6eba61b649373258129fe6d8947ce 100644
--- a/test/Assembler/auto_upgrade_intrinsics.ll
+++ b/test/Assembler/auto_upgrade_intrinsics.ll
@@ -53,7 +53,7 @@ entry:
 
 define i32 @test.objectsize() {
 ; CHECK-LABEL: @test.objectsize(
-; CHECK: @llvm.objectsize.i32.p0i8(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0), i1 false, i1 false)
+; CHECK: @llvm.objectsize.i32.p0i8(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0), i1 false, i1 false, i1 false)
   %s = call i32 @llvm.objectsize.i32(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0), i1 false)
   ret i32 %s
 }
@@ -61,12 +61,11 @@ define i32 @test.objectsize() {
 declare i64 @llvm.objectsize.i64.p0i8(i8*, i1) nounwind readonly
 define i64 @test.objectsize.2() {
 ; CHECK-LABEL: @test.objectsize.2(
-; CHECK: @llvm.objectsize.i64.p0i8(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0), i1 false, i1 false)
+; CHECK: @llvm.objectsize.i64.p0i8(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0), i1 false, i1 false, i1 false)
   %s = call i64 @llvm.objectsize.i64.p0i8(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0), i1 false)
   ret i64 %s
 }
 
-
 declare <2 x double> @llvm.masked.load.v2f64(<2 x double>* %ptrs, i32, <2 x i1> %mask, <2 x double> %src0)
 
 define <2 x double> @tests.masked.load(<2 x double>* %ptr, <2 x i1> %mask, <2 x double> %passthru)  {
diff --git a/test/Assembler/invalid-atomicrmw-fadd-must-be-fp-type.ll b/test/Assembler/invalid-atomicrmw-fadd-must-be-fp-type.ll
new file mode 100644
index 0000000000000000000000000000000000000000..3185d9505db9a2fdd1013386d0f01e4a05a8979d
--- /dev/null
+++ b/test/Assembler/invalid-atomicrmw-fadd-must-be-fp-type.ll
@@ -0,0 +1,7 @@
+; RUN: not llvm-as -disable-output %s 2>&1 | FileCheck %s
+
+; CHECK: error: atomicrmw fadd operand must be a floating point type
+define void @f(i32* %ptr) {
+  atomicrmw fadd i32* %ptr, i32 2 seq_cst
+  ret void
+}
diff --git a/test/Assembler/invalid-atomicrmw-fsub-must-be-fp-type.ll b/test/Assembler/invalid-atomicrmw-fsub-must-be-fp-type.ll
new file mode 100644
index 0000000000000000000000000000000000000000..cd5bd4beff575322667ff35ae787009ce591bb8f
--- /dev/null
+++ b/test/Assembler/invalid-atomicrmw-fsub-must-be-fp-type.ll
@@ -0,0 +1,7 @@
+; RUN: not llvm-as -disable-output %s 2>&1 | FileCheck %s
+
+; CHECK: error: atomicrmw fsub operand must be a floating point type
+define void @f(i32* %ptr) {
+  atomicrmw fsub i32* %ptr, i32 2 seq_cst
+  ret void
+}
diff --git a/test/Assembler/invalid-atomicrmw-xchg-must-be-integer-or-fp-type.ll b/test/Assembler/invalid-atomicrmw-xchg-must-be-integer-or-fp-type.ll
new file mode 100644
index 0000000000000000000000000000000000000000..d0a794bd0244a028a66ce14eba2aa067b882c872
--- /dev/null
+++ b/test/Assembler/invalid-atomicrmw-xchg-must-be-integer-or-fp-type.ll
@@ -0,0 +1,7 @@
+; RUN: not llvm-as -disable-output %s 2>&1 | FileCheck %s
+
+; CHECK: error: atomicrmw xchg operand must be an integer or floating point type
+define void @f(i32** %ptr) {
+  atomicrmw xchg i32** %ptr, i32* null seq_cst
+  ret void
+}
diff --git a/test/Bindings/llvm-c/echo.ll b/test/Bindings/llvm-c/echo.ll
index 118f822e432f4b6e25f1cef86283c78e5d69dfcb..bb6556e763e5832d67839e0474d3c6628f77d25c 100644
--- a/test/Bindings/llvm-c/echo.ll
+++ b/test/Bindings/llvm-c/echo.ll
@@ -28,6 +28,13 @@ module asm "classical GAS"
 @aliased4 = weak alias i32, i32* @var
 @aliased5 = weak_odr alias i32, i32* @var
 
+@ifunc = ifunc i32 (i32), i64 ()* @ifunc_resolver
+
+define i64 @ifunc_resolver() {
+entry:
+  ret i64 0
+}
+
 define { i64, %S* } @unpackrepack(%S %s) {
   %1 = extractvalue %S %s, 0
   %2 = extractvalue %S %s, 1
diff --git a/test/Bitcode/callbr.ll b/test/Bitcode/callbr.ll
new file mode 100644
index 0000000000000000000000000000000000000000..ecc397ac7546781cadeba575d229e1f28c9dc0ae
--- /dev/null
+++ b/test/Bitcode/callbr.ll
@@ -0,0 +1,14 @@
+; RUN:  llvm-dis < %s.bc | FileCheck %s
+
+; callbr.ll.bc was generated by passing this file to llvm-as.
+
+define i32 @test_asm_goto(i32 %x){
+entry:
+; CHECK:      callbr void asm "", "r,X"(i32 %x, i8* blockaddress(@test_asm_goto, %fail))
+; CHECK-NEXT: to label %normal [label %fail]
+  callbr void asm "", "r,X"(i32 %x, i8* blockaddress(@test_asm_goto, %fail)) to label %normal [label %fail]
+normal:
+  ret i32 1
+fail:
+  ret i32 0
+}
diff --git a/test/Bitcode/callbr.ll.bc b/test/Bitcode/callbr.ll.bc
new file mode 100644
index 0000000000000000000000000000000000000000..d07cba433c191c375f19113644f3532b87db033b
Binary files /dev/null and b/test/Bitcode/callbr.ll.bc differ
diff --git a/test/Bitcode/compatibility.ll b/test/Bitcode/compatibility.ll
index 1d57f759391af70c5daa041a3ab660ed1b2293d5..3c5e86eee018552b481b02126a8bc587189dac3a 100644
--- a/test/Bitcode/compatibility.ll
+++ b/test/Bitcode/compatibility.ll
@@ -761,6 +761,19 @@ define void @atomics(i32* %word) {
   ret void
 }
 
+define void @fp_atomics(float* %word) {
+; CHECK: %atomicrmw.xchg = atomicrmw xchg float* %word, float 1.000000e+00 monotonic
+  %atomicrmw.xchg = atomicrmw xchg float* %word, float 1.0 monotonic
+
+; CHECK: %atomicrmw.fadd = atomicrmw fadd float* %word, float 1.000000e+00 monotonic
+  %atomicrmw.fadd = atomicrmw fadd float* %word, float 1.0 monotonic
+
+; CHECK: %atomicrmw.fsub = atomicrmw fsub float* %word, float 1.000000e+00 monotonic
+  %atomicrmw.fsub = atomicrmw fsub float* %word, float 1.0 monotonic
+
+  ret void
+}
+
 ;; Fast Math Flags
 define void @fastmathflags_unop(float %op1) {
   %f.nnan = fneg nnan float %op1
diff --git a/test/Bitcode/dilocalvariable-3.9.ll b/test/Bitcode/dilocalvariable-3.9.ll
index 99f99db063748292b265d1fb74425fb552238834..a252c355c51870df64a54929824c7068a6a34061 100644
--- a/test/Bitcode/dilocalvariable-3.9.ll
+++ b/test/Bitcode/dilocalvariable-3.9.ll
@@ -7,12 +7,12 @@
 !llvm.module.flags = !{!3, !4}
 !llvm.ident = !{!5}
 
-!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 3.9.1 (http://llvm.org/git/clang.git c3709e72d22432f53f8e2f14354def31a96734fe)", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !2)
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 3.9.1", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !2)
 !1 = !DIFile(filename: "main.c", directory: "/tmp")
 !2 = !{}
 !3 = !{i32 2, !"Dwarf Version", i32 4}
 !4 = !{i32 2, !"Debug Info Version", i32 3}
-!5 = !{!"clang version 3.9.1 (http://llvm.org/git/clang.git c3709e72d22432f53f8e2f14354def31a96734fe)"}
+!5 = !{!"clang version 3.9.1"}
 !6 = distinct !DISubprogram(name: "f", scope: !1, file: !1, line: 1, type: !7, isLocal: false, isDefinition: true, scopeLine: 1, isOptimized: false, unit: !0, retainedNodes: !2)
 !7 = !DISubroutineType(types: !8)
 !8 = !{null}
diff --git a/test/Bitcode/objectsize-upgrade-7.0.ll b/test/Bitcode/objectsize-upgrade-7.0.ll
new file mode 100644
index 0000000000000000000000000000000000000000..a7e6d497676e734f5c7d347ad305c53d01dd770b
--- /dev/null
+++ b/test/Bitcode/objectsize-upgrade-7.0.ll
@@ -0,0 +1,12 @@
+; RUN: llvm-dis < %s.bc | FileCheck %s
+
+; Bitcode compatibility test for 'dynamic' parameter to llvm.objectsize.
+
+define void @callit(i8* %ptr) {
+  %sz = call i64 @llvm.objectsize.i64.p0i8(i8* %ptr, i1 false, i1 true)
+  ; CHECK: %sz = call i64 @llvm.objectsize.i64.p0i8(i8* %ptr, i1 false, i1 true, i1 false)
+  ret void
+}
+
+declare i64 @llvm.objectsize.i64.p0i8(i8*, i1, i1)
+; CHECK: declare i64 @llvm.objectsize.i64.p0i8(i8*, i1, i1, i1)
diff --git a/test/Bitcode/objectsize-upgrade-7.0.ll.bc b/test/Bitcode/objectsize-upgrade-7.0.ll.bc
new file mode 100644
index 0000000000000000000000000000000000000000..115c69dfb9da87f8ccb69d5115060a96b5c55950
Binary files /dev/null and b/test/Bitcode/objectsize-upgrade-7.0.ll.bc differ
diff --git a/test/Bitcode/thinlto-alias.ll b/test/Bitcode/thinlto-alias.ll
index 835d720c69e5d76564c025cca1f31e235b81e034..f3896d2baba769c8d7df9e60246e091e0f6250f0 100644
--- a/test/Bitcode/thinlto-alias.ll
+++ b/test/Bitcode/thinlto-alias.ll
@@ -18,6 +18,7 @@
 ; CHECK-NEXT: <FUNCTION op0=4 op1=7
 ; CHECK:       <GLOBALVAL_SUMMARY_BLOCK
 ; CHECK-NEXT:    <VERSION
+; CHECK-NEXT:    <FLAGS
 ; See if the call to func is registered.
 ; The value id 1 matches the second FUNCTION record above.
 ; CHECK-NEXT:    <PERMODULE {{.*}} op6=1/>
diff --git a/test/Bitcode/thinlto-alias2.ll b/test/Bitcode/thinlto-alias2.ll
index 3d68e3fd4ba6393070f740e1c469b5a13ce173e1..8b04ee75ac5247b0dfb06c9575a34693d2f9f49b 100644
--- a/test/Bitcode/thinlto-alias2.ll
+++ b/test/Bitcode/thinlto-alias2.ll
@@ -4,6 +4,7 @@
 
 ; CHECK:       <GLOBALVAL_SUMMARY_BLOCK
 ; CHECK-NEXT:    <VERSION
+; CHECK-NEXT:    <FLAGS
 ; CHECK-NEXT:    <PERMODULE {{.*}} op4=0 op5=0 op6=[[ALIASID:[0-9]+]]/>
 ; CHECK-NEXT:    <PERMODULE {{.*}} op0=[[ALIASEEID:[0-9]+]]
 ; CHECK-NEXT:    <ALIAS {{.*}} op0=[[ALIASID]] {{.*}} op2=[[ALIASEEID]]/>
diff --git a/test/Bitcode/thinlto-function-summary-callgraph-cast.ll b/test/Bitcode/thinlto-function-summary-callgraph-cast.ll
index 79644403d38cf4edcabc29a52e06d96d8a452194..d4b4d5491c1458182593342910918e28f3eae71e 100644
--- a/test/Bitcode/thinlto-function-summary-callgraph-cast.ll
+++ b/test/Bitcode/thinlto-function-summary-callgraph-cast.ll
@@ -5,6 +5,7 @@
 
 ; CHECK:       <GLOBALVAL_SUMMARY_BLOCK
 ; CHECK-NEXT:    <VERSION
+; CHECK-NEXT:    <FLAGS
 ; "op7" is a call to "callee" function.
 ; CHECK-NEXT:    <PERMODULE {{.*}} op8=3 op9=[[ALIASID:[0-9]+]]/>
 ; "another_caller" has only references but no calls.
diff --git a/test/Bitcode/thinlto-function-summary-callgraph-pgo.ll b/test/Bitcode/thinlto-function-summary-callgraph-pgo.ll
index e332224343e96b52ff8d0e94396ef1065e469565..b9613f732ed81ae4adbd3f5a46ead146b0009bf5 100644
--- a/test/Bitcode/thinlto-function-summary-callgraph-pgo.ll
+++ b/test/Bitcode/thinlto-function-summary-callgraph-pgo.ll
@@ -16,6 +16,7 @@
 ; CHECK-NEXT: <FUNCTION op0=4 op1=4
 ; CHECK:       <GLOBALVAL_SUMMARY_BLOCK
 ; CHECK-NEXT:    <VERSION
+; CHECK-NEXT:    <FLAGS
 ; See if the call to func is registered, using the expected hotness type.
 ; CHECK-NEXT:    <PERMODULE_PROFILE {{.*}} op6=1 op7=2/>
 ; CHECK-NEXT:  </GLOBALVAL_SUMMARY_BLOCK>
diff --git a/test/Bitcode/thinlto-function-summary-callgraph-profile-summary.ll b/test/Bitcode/thinlto-function-summary-callgraph-profile-summary.ll
index 31c99c189ac014ae85e6bd850b9421b2d4ddef36..0cd10982dd5ae964746324d8362850b81b2b4cc0 100644
--- a/test/Bitcode/thinlto-function-summary-callgraph-profile-summary.ll
+++ b/test/Bitcode/thinlto-function-summary-callgraph-profile-summary.ll
@@ -46,6 +46,7 @@
 ; CHECK-NEXT: <FUNCTION op0=42 op1=5
 ; CHECK-LABEL:       <GLOBALVAL_SUMMARY_BLOCK
 ; CHECK-NEXT:    <VERSION
+; CHECK-NEXT:    <FLAGS
 ; CHECK-NEXT:    <VALUE_GUID op0=25 op1=123/>
 ; op4=hot1 op6=cold op8=hot2 op10=hot4 op12=none1 op14=hot3 op16=none2 op18=none3 op20=123
 ; CHECK-NEXT:    <PERMODULE_PROFILE {{.*}} op6=1 op7=3 op8=5 op9=1 op10=2 op11=3 op12=4 op13=1 op14=6 op15=2 op16=3 op17=3 op18=7 op19=2 op20=8 op21=2 op22=25 op23=4/>
diff --git a/test/Bitcode/thinlto-function-summary-callgraph-relbf.ll b/test/Bitcode/thinlto-function-summary-callgraph-relbf.ll
index 6c14465222569d449442e225b0c922c95d740e7c..7c7a6f6c92468d25cf716cfdef537b30df12aad5 100644
--- a/test/Bitcode/thinlto-function-summary-callgraph-relbf.ll
+++ b/test/Bitcode/thinlto-function-summary-callgraph-relbf.ll
@@ -12,6 +12,7 @@
 ; CHECK-NEXT: <FUNCTION op0=17 op1=4
 ; CHECK:       <GLOBALVAL_SUMMARY_BLOCK
 ; CHECK-NEXT:    <VERSION
+; CHECK-NEXT:    <FLAGS
 ; See if the call to func is registered.
 ; CHECK-NEXT:    <PERMODULE_RELBF {{.*}} op4=1 {{.*}} op8=256
 ; CHECK-NEXT:  </GLOBALVAL_SUMMARY_BLOCK>
diff --git a/test/Bitcode/thinlto-function-summary-callgraph-sample-profile-summary.ll b/test/Bitcode/thinlto-function-summary-callgraph-sample-profile-summary.ll
index d1f980ab5f6a7f864ab03af031c94904d73d1f10..8bf65ab5e21b788b075d699d6aca34d7c0bbff38 100644
--- a/test/Bitcode/thinlto-function-summary-callgraph-sample-profile-summary.ll
+++ b/test/Bitcode/thinlto-function-summary-callgraph-sample-profile-summary.ll
@@ -29,6 +29,7 @@
 
 ; CHECK-LABEL:       <GLOBALVAL_SUMMARY_BLOCK
 ; CHECK-NEXT:    <VERSION
+; CHECK-NEXT:    <FLAGS
 ; CHECK-NEXT:    <VALUE_GUID op0=26 op1=123/>
 ; op4=none1 op6=hot1 op8=cold1 op10=none2 op12=hot2 op14=cold2 op16=none3 op18=hot3 op20=cold3 op22=123
 ; CHECK-NEXT:    <PERMODULE_PROFILE {{.*}} op6=7 op7=0 op8=1 op9=3 op10=4 op11=1 op12=8 op13=0 op14=2 op15=3 op16=5 op17=1 op18=9 op19=0 op20=3 op21=3 op22=6 op23=1 op24=26 op25=4/>
diff --git a/test/Bitcode/thinlto-function-summary-callgraph.ll b/test/Bitcode/thinlto-function-summary-callgraph.ll
index a605b7ec2219b8c79bf73980057b71b3e5b066a9..0969b849bc01218cb9f903d03d4a247994ace760 100644
--- a/test/Bitcode/thinlto-function-summary-callgraph.ll
+++ b/test/Bitcode/thinlto-function-summary-callgraph.ll
@@ -17,6 +17,7 @@
 ; CHECK-NEXT: <FUNCTION op0=17 op1=4
 ; CHECK:       <GLOBALVAL_SUMMARY_BLOCK
 ; CHECK-NEXT:    <VERSION
+; CHECK-NEXT:    <FLAGS
 ; See if the call to func is registered
 ; CHECK-NEXT:    <PERMODULE {{.*}} op4=1
 ; CHECK-NEXT:  </GLOBALVAL_SUMMARY_BLOCK>
diff --git a/test/Bitcode/thinlto-function-summary.ll b/test/Bitcode/thinlto-function-summary.ll
index be7e9749086b06337d34ee86909011316e315b0b..67c50379e7abd70784b71f878168d56bd0b5bbda 100644
--- a/test/Bitcode/thinlto-function-summary.ll
+++ b/test/Bitcode/thinlto-function-summary.ll
@@ -19,6 +19,7 @@
 ; BC-NEXT: <ALIAS op0=67 op1=1
 ; BC: <GLOBALVAL_SUMMARY_BLOCK
 ; BC-NEXT: <VERSION
+; BC-NEXT: <FLAGS
 ; BC-NEXT: <PERMODULE {{.*}} op0=1 op1=0
 ; BC-NEXT: <PERMODULE {{.*}} op0=2 op1=0
 ; BC-NEXT: <PERMODULE {{.*}} op0=3 op1=7
diff --git a/test/BugPoint/compile-custom.ll.py b/test/BugPoint/compile-custom.ll.py
index 4b9b30caadc8149799ea9c791ef28d9e19bcab72..b0062ac0b74f30b53496a07af12809e0dbf99f64 100755
--- a/test/BugPoint/compile-custom.ll.py
+++ b/test/BugPoint/compile-custom.ll.py
@@ -1,5 +1,7 @@
 #!/usr/bin/env python
 
+from __future__ import print_function
+
 import sys
 
 # Currently any print-out from the custom tool is interpreted as a crash
diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
index b39086a4e9d06cf42d95cbc19e70276f4b43f931..e69ceebabe0da4938f4809d0dbce346d7d2ccb30 100644
--- a/test/CMakeLists.txt
+++ b/test/CMakeLists.txt
@@ -1,16 +1,17 @@
 llvm_canonicalize_cmake_booleans(
-  LLVM_TOOL_LTO_BUILD
+  BUILD_SHARED_LIBS
+  HAVE_LIBXAR
+  HAVE_LIBZ
   HAVE_OCAMLOPT
   HAVE_OCAML_OUNIT
-  LLVM_INCLUDE_GO_TESTS
-  LLVM_USE_INTEL_JITEVENTS
-  HAVE_LIBZ
-  HAVE_LIBXAR
   LLVM_ENABLE_DIA_SDK
   LLVM_ENABLE_FFI
   LLVM_ENABLE_THREADS
-  BUILD_SHARED_LIBS
+  LLVM_INCLUDE_GO_TESTS
+  LLVM_LIBXML2_ENABLED
   LLVM_LINK_LLVM_DYLIB
+  LLVM_TOOL_LTO_BUILD
+  LLVM_USE_INTEL_JITEVENTS
   )
 
 configure_lit_site_cfg(
@@ -61,6 +62,7 @@ set(LLVM_TEST_DEPENDS
           dsymutil
           llvm-dwarfdump
           llvm-dwp
+          llvm-elfabi
           llvm-exegesis
           llvm-extract
           llvm-isel-fuzzer
diff --git a/test/CodeGen/AArch64/GlobalISel/arm64-callingconv-ios.ll b/test/CodeGen/AArch64/GlobalISel/arm64-callingconv-ios.ll
index 256bcf28e66cc790a1dae8d1bac5b9f8022eb19c..f06cd14abe753fbf59f6bb578f1be3127520179e 100644
--- a/test/CodeGen/AArch64/GlobalISel/arm64-callingconv-ios.ll
+++ b/test/CodeGen/AArch64/GlobalISel/arm64-callingconv-ios.ll
@@ -17,13 +17,13 @@ target triple = "aarch64-apple-ios9.0"
 ; CHECK: $d0 = COPY [[D_ONE]]
 ; CHECK: $x1 = COPY [[TWELVE]]
 ; CHECK: [[THREE_EXT:%[0-9]+]]:_(s64) = G_ANYEXT [[THREE]]
-; CHECK: G_STORE [[THREE_EXT]](s64), {{%[0-9]+}}(p0) :: (store 8 into stack, align 0)
+; CHECK: G_STORE [[THREE_EXT]](s64), {{%[0-9]+}}(p0) :: (store 8 into stack, align 1)
 ; CHECK: [[ONE_EXT:%[0-9]+]]:_(s64) = G_ANYEXT [[ONE]]
-; CHECK: G_STORE [[ONE_EXT]](s64), {{%[0-9]+}}(p0) :: (store 8 into stack + 8, align 0)
+; CHECK: G_STORE [[ONE_EXT]](s64), {{%[0-9]+}}(p0) :: (store 8 into stack + 8, align 1)
 ; CHECK: [[FOUR_EXT:%[0-9]+]]:_(s64) = G_ANYEXT [[FOUR]]
-; CHECK: G_STORE [[FOUR_EXT]](s64), {{%[0-9]+}}(p0) :: (store 8 into stack + 16, align 0)
-; CHECK: G_STORE [[F_ONE]](s32), {{%[0-9]+}}(p0) :: (store 4 into stack + 24, align 0)
-; CHECK: G_STORE [[TWO]](s64), {{%[0-9]+}}(p0) :: (store 8 into stack + 32, align 0)
+; CHECK: G_STORE [[FOUR_EXT]](s64), {{%[0-9]+}}(p0) :: (store 8 into stack + 16, align 1)
+; CHECK: G_STORE [[F_ONE]](s32), {{%[0-9]+}}(p0) :: (store 4 into stack + 24, align 1)
+; CHECK: G_STORE [[TWO]](s64), {{%[0-9]+}}(p0) :: (store 8 into stack + 32, align 1)
 declare void @varargs(i32, double, i64, ...)
 define void @test_varargs() {
   call void(i32, double, i64, ...) @varargs(i32 42, double 1.0, i64 12, i8 3, i16 1, i32 4, float 1.0, double 2.0)
diff --git a/test/CodeGen/AArch64/GlobalISel/arm64-fallback.ll b/test/CodeGen/AArch64/GlobalISel/arm64-fallback.ll
index da3aa3c1009037480081dc85923b64c9e2352c9e..c75771a8425baefae747b9ddbb0070b36e2ad1fd 100644
--- a/test/CodeGen/AArch64/GlobalISel/arm64-fallback.ll
+++ b/test/CodeGen/AArch64/GlobalISel/arm64-fallback.ll
@@ -205,7 +205,7 @@ define void @nonpow2_load_narrowing() {
   ret void
 }
 
-; FALLBACK-WITH-REPORT-ERR: remark: <unknown>:0:0: unable to legalize instruction: G_STORE %3:_(s96), %0:_(p0) :: (store 12 into %ir.c, align 16) (in function: nonpow2_store_narrowing
+; FALLBACK-WITH-REPORT-ERR: remark: <unknown>:0:0: unable to legalize instruction: %4:_(s64) = G_EXTRACT %3:_(s96), 0 (in function: nonpow2_store_narrowing)
 ; FALLBACK-WITH-REPORT-ERR: warning: Instruction selection used fallback path for nonpow2_store_narrowing
 ; FALLBACK-WITH-REPORT-OUT-LABEL: nonpow2_store_narrowing:
 define void @nonpow2_store_narrowing(i96* %c) {
@@ -215,7 +215,7 @@ define void @nonpow2_store_narrowing(i96* %c) {
   ret void
 }
 
-; FALLBACK-WITH-REPORT-ERR: remark: <unknown>:0:0: unable to legalize instruction: G_STORE %0:_(s96), %1:_(p0) :: (store 12 into `i96* undef`, align 16) (in function: nonpow2_constant_narrowing)
+; FALLBACK-WITH-REPORT-ERR: remark: <unknown>:0:0: unable to legalize instruction: %0:_(s96) = G_CONSTANT i96 0 (in function: nonpow2_constant_narrowing)
 ; FALLBACK-WITH-REPORT-ERR: warning: Instruction selection used fallback path for nonpow2_constant_narrowing
 ; FALLBACK-WITH-REPORT-OUT-LABEL: nonpow2_constant_narrowing:
 define void @nonpow2_constant_narrowing() {
diff --git a/test/CodeGen/AArch64/GlobalISel/arm64-irtranslator.ll b/test/CodeGen/AArch64/GlobalISel/arm64-irtranslator.ll
index 74de033447fcf5cc4ba95aea45da8cc81213a8f4..81505b8839096a6b9765704a2fa1f0a4fa8c1e02 100644
--- a/test/CodeGen/AArch64/GlobalISel/arm64-irtranslator.ll
+++ b/test/CodeGen/AArch64/GlobalISel/arm64-irtranslator.ll
@@ -1,4 +1,5 @@
 ; RUN: llc -O0 -aarch64-enable-atomic-cfg-tidy=0 -stop-after=irtranslator -global-isel -verify-machineinstrs %s -o - 2>&1 | FileCheck %s
+; RUN: llc -O3 -aarch64-enable-atomic-cfg-tidy=0 -stop-after=irtranslator -global-isel -verify-machineinstrs %s -o - 2>&1 | FileCheck %s --check-prefix=O3
 
 ; This file checks that the translation from llvm IR to generic MachineInstr
 ; is correct.
@@ -824,6 +825,16 @@ define float @test_frem(float %arg1, float %arg2) {
   ret float %res
 }
 
+; CHECK-LABEL: name: test_fneg
+; CHECK: [[ARG1:%[0-9]+]]:_(s32) = COPY $s0
+; CHECK-NEXT: [[RES:%[0-9]+]]:_(s32) = G_FNEG [[ARG1]]
+; CHECK-NEXT: $s0 = COPY [[RES]]
+; CHECK-NEXT: RET_ReallyLR implicit $s0
+define float @test_fneg(float %arg1) {
+  %res = fneg float %arg1
+  ret float %res
+}
+
 ; CHECK-LABEL: name: test_sadd_overflow
 ; CHECK: [[LHS:%[0-9]+]]:_(s32) = COPY $w0
 ; CHECK: [[RHS:%[0-9]+]]:_(s32) = COPY $w1
@@ -1416,6 +1427,16 @@ define float @test_fabs_intrin(float %a) {
   ret float %res
 }
 
+declare float @llvm.canonicalize.f32(float)
+define float @test_fcanonicalize_intrin(float %a) {
+; CHECK-LABEL: name: test_fcanonicalize_intrin
+; CHECK: [[A:%[0-9]+]]:_(s32) = COPY $s0
+; CHECK: [[RES:%[0-9]+]]:_(s32) = nnan ninf nsz arcp contract afn reassoc G_FCANONICALIZE [[A]]
+; CHECK: $s0 = COPY [[RES]]
+  %res = call nnan ninf nsz arcp contract afn reassoc float @llvm.canonicalize.f32(float %a)
+  ret float %res
+}
+
 declare float @llvm.trunc.f32(float)
 define float @test_intrinsic_trunc(float %a) {
 ; CHECK-LABEL: name: test_intrinsic_trunc
@@ -1471,6 +1492,11 @@ declare void @llvm.lifetime.end.p0i8(i64, i8*)
 define void @test_lifetime_intrin() {
 ; CHECK-LABEL: name: test_lifetime_intrin
 ; CHECK: RET_ReallyLR
+; O3-LABEL: name: test_lifetime_intrin
+; O3: {{%[0-9]+}}:_(p0) = G_FRAME_INDEX %stack.0.slot
+; O3-NEXT: LIFETIME_START %stack.0.slot
+; O3-NEXT: LIFETIME_END %stack.0.slot
+; O3-NEXT: RET_ReallyLR
   %slot = alloca i8, i32 4
   call void @llvm.lifetime.start.p0i8(i64 0, i8* %slot)
   call void @llvm.lifetime.end.p0i8(i64 0, i8* %slot)
@@ -2242,3 +2268,85 @@ define void @test_invariant_intrin() {
   call void @llvm.invariant.end.p0i8({}* %inv, i64 8, i8* %y)
   ret void
 }
+
+declare float @llvm.ceil.f32(float)
+define float @test_ceil_f32(float %x) {
+  ; CHECK-LABEL: name:            test_ceil_f32
+  ; CHECK: %{{[0-9]+}}:_(s32) = G_FCEIL %{{[0-9]+}}
+  %y = call float @llvm.ceil.f32(float %x)
+  ret float %y
+}
+
+declare double @llvm.ceil.f64(double)
+define double @test_ceil_f64(double %x) {
+  ; CHECK-LABEL: name:            test_ceil_f64
+  ; CHECK: %{{[0-9]+}}:_(s64) = G_FCEIL %{{[0-9]+}}
+  %y = call double @llvm.ceil.f64(double %x)
+  ret double %y
+}
+
+declare <2 x float> @llvm.ceil.v2f32(<2 x float>)
+define <2 x float> @test_ceil_v2f32(<2 x float> %x) {
+  ; CHECK-LABEL: name:            test_ceil_v2f32
+  ; CHECK: %{{[0-9]+}}:_(<2 x s32>) = G_FCEIL %{{[0-9]+}}
+  %y = call <2 x float> @llvm.ceil.v2f32(<2 x float> %x)
+  ret <2 x float> %y
+}
+
+declare <4 x float> @llvm.ceil.v4f32(<4 x float>)
+define <4 x float> @test_ceil_v4f32(<4 x float> %x) {
+  ; CHECK-LABEL: name:            test_ceil_v4f32
+  ; CHECK: %{{[0-9]+}}:_(<4 x s32>) = G_FCEIL %{{[0-9]+}}
+  ; SELECT: %{{[0-9]+}}:fpr128 = FRINTPv4f32 %{{[0-9]+}}
+  %y = call <4 x float> @llvm.ceil.v4f32(<4 x float> %x)
+  ret <4 x float> %y
+}
+
+declare <2 x double> @llvm.ceil.v2f64(<2 x double>)
+define <2 x double> @test_ceil_v2f64(<2 x double> %x) {
+  ; CHECK-LABEL: name:            test_ceil_v2f64
+  ; CHECK: %{{[0-9]+}}:_(<2 x s64>) = G_FCEIL %{{[0-9]+}}
+  %y = call <2 x double> @llvm.ceil.v2f64(<2 x double> %x)
+  ret <2 x double> %y
+}
+
+declare float @llvm.cos.f32(float)
+define float @test_cos_f32(float %x) {
+  ; CHECK-LABEL: name:            test_cos_f32
+  ; CHECK: %{{[0-9]+}}:_(s32) = G_FCOS %{{[0-9]+}}
+  %y = call float @llvm.cos.f32(float %x)
+  ret float %y
+}
+
+declare float @llvm.sin.f32(float)
+define float @test_sin_f32(float %x) {
+  ; CHECK-LABEL: name:            test_sin_f32
+  ; CHECK: %{{[0-9]+}}:_(s32) = G_FSIN %{{[0-9]+}}
+  %y = call float @llvm.sin.f32(float %x)
+  ret float %y
+}
+
+declare float @llvm.sqrt.f32(float)
+define float @test_sqrt_f32(float %x) {
+  ; CHECK-LABEL: name:            test_sqrt_f32
+  ; CHECK: %{{[0-9]+}}:_(s32) = G_FSQRT %{{[0-9]+}}
+  %y = call float @llvm.sqrt.f32(float %x)
+  ret float %y
+}
+
+declare float @llvm.floor.f32(float)
+define float @test_floor_f32(float %x) {
+  ; CHECK-LABEL: name:            test_floor_f32
+  ; CHECK: %{{[0-9]+}}:_(s32) = G_FFLOOR %{{[0-9]+}}
+  %y = call float @llvm.floor.f32(float %x)
+  ret float %y
+}
+
+; CHECK-LABEL: name: test_llvm.aarch64.neon.ld3.v4i32.p0i32
+; CHECK: %1:_(s384) = G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.aarch64.neon.ld3), %0(p0) :: (load 48 from %ir.ptr, align 64)
+define void @test_llvm.aarch64.neon.ld3.v4i32.p0i32(i32* %ptr) {
+  %arst = call { <4 x i32>, <4 x i32>, <4 x i32> } @llvm.aarch64.neon.ld3.v4i32.p0i32(i32* %ptr)
+  ret void
+}
+
+declare { <4 x i32>, <4 x i32>, <4 x i32> } @llvm.aarch64.neon.ld3.v4i32.p0i32(i32*) #3
diff --git a/test/CodeGen/AArch64/GlobalISel/call-translator-cse.ll b/test/CodeGen/AArch64/GlobalISel/call-translator-cse.ll
new file mode 100644
index 0000000000000000000000000000000000000000..9120c109f5bd774a8b29d7b342a1b2223a454264
--- /dev/null
+++ b/test/CodeGen/AArch64/GlobalISel/call-translator-cse.ll
@@ -0,0 +1,34 @@
+; RUN: llc -mtriple=aarch64-linux-gnu -stop-after=irtranslator -enable-cse-in-irtranslator=1 -global-isel -verify-machineinstrs %s -o - 2>&1 | FileCheck %s
+
+; CHECK-LABEL: name: test_split_struct
+; CHECK: [[ADDR:%[0-9]+]]:_(p0) = COPY $x0
+; CHECK: [[LO:%[0-9]+]]:_(s64) = G_LOAD %0(p0) :: (load 8 from %ir.ptr)
+; CHECK: [[CST:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
+; CHECK: [[GEP:%[0-9]+]]:_(p0) = G_GEP [[ADDR]], [[CST]](s64)
+; CHECK: [[HI:%[0-9]+]]:_(s64) = G_LOAD [[GEP]](p0) :: (load 8 from %ir.ptr + 8)
+
+; CHECK: [[IMPDEF:%[0-9]+]]:_(s128) = G_IMPLICIT_DEF
+; CHECK: [[INS1:%[0-9]+]]:_(s128) = G_INSERT [[IMPDEF]], [[LO]](s64), 0
+; CHECK: [[INS2:%[0-9]+]]:_(s128) = G_INSERT [[INS1]], [[HI]](s64), 64
+; CHECK: [[EXTLO:%[0-9]+]]:_(s64) = G_EXTRACT [[INS2]](s128), 0
+; CHECK: [[EXTHI:%[0-9]+]]:_(s64) = G_EXTRACT [[INS2]](s128), 64
+
+; CHECK: [[SP:%[0-9]+]]:_(p0) = COPY $sp
+; CHECK: [[CST2:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+; CHECK: [[GEP2:%[0-9]+]]:_(p0) = G_GEP [[SP]], [[CST2]](s64)
+; CHECK: G_STORE [[EXTLO]](s64), [[GEP2]](p0) :: (store 8 into stack, align 1)
+; CHECK: [[SP:%[0-9]+]]:_(p0) = COPY $sp
+; CHECK: [[CST3:%[0-9]+]]:_(s64) = COPY [[CST]]
+; CHECK: [[GEP3:%[0-9]+]]:_(p0) = G_GEP [[SP]], [[CST3]](s64)
+; CHECK: G_STORE [[EXTHI]](s64), [[GEP3]](p0) :: (store 8 into stack + 8, align 1)
+define void @test_split_struct([2 x i64]* %ptr) {
+  %struct = load [2 x i64], [2 x i64]* %ptr
+  call void @take_split_struct([2 x i64]* null, i64 1, i64 2, i64 3,
+                               i64 4, i64 5, i64 6,
+                               [2 x i64] %struct)
+  ret void
+}
+
+declare void @take_split_struct([2 x i64]* %ptr, i64, i64, i64,
+                               i64, i64, i64,
+                               [2 x i64] %in) ;
diff --git a/test/CodeGen/AArch64/GlobalISel/call-translator-ios.ll b/test/CodeGen/AArch64/GlobalISel/call-translator-ios.ll
index 1cb2e9aa186099d848b72ca5682290a270a16ec4..429c08bfec393b445a63008f60ced339eacb7293 100644
--- a/test/CodeGen/AArch64/GlobalISel/call-translator-ios.ll
+++ b/test/CodeGen/AArch64/GlobalISel/call-translator-ios.ll
@@ -6,9 +6,9 @@
 ; CHECK-DAG:  - { id: [[STACK0:[0-9]+]], type: default, offset: 0, size: 1,
 ; CHECK-DAG:  - { id: [[STACK8:[0-9]+]], type: default, offset: 1, size: 1,
 ; CHECK: [[LHS_ADDR:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.[[STACK0]]
-; CHECK: [[LHS:%[0-9]+]]:_(s8) = G_LOAD [[LHS_ADDR]](p0) :: (invariant load 1 from %fixed-stack.[[STACK0]], align 0)
+; CHECK: [[LHS:%[0-9]+]]:_(s8) = G_LOAD [[LHS_ADDR]](p0) :: (invariant load 1 from %fixed-stack.[[STACK0]])
 ; CHECK: [[RHS_ADDR:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.[[STACK8]]
-; CHECK: [[RHS:%[0-9]+]]:_(s8) = G_LOAD [[RHS_ADDR]](p0) :: (invariant load 1 from %fixed-stack.[[STACK8]], align 0)
+; CHECK: [[RHS:%[0-9]+]]:_(s8) = G_LOAD [[RHS_ADDR]](p0) :: (invariant load 1 from %fixed-stack.[[STACK8]])
 ; CHECK: [[SUM:%[0-9]+]]:_(s8) = G_ADD [[LHS]], [[RHS]]
 ; CHECK: [[SUM32:%[0-9]+]]:_(s32) = G_SEXT [[SUM]](s8)
 ; CHECK: $w0 = COPY [[SUM32]](s32)
@@ -23,11 +23,11 @@ define signext i8 @test_stack_slots([8 x i64], i8 signext %lhs, i8 signext %rhs)
 ; CHECK: [[SP:%[0-9]+]]:_(p0) = COPY $sp
 ; CHECK: [[C42_OFFS:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
 ; CHECK: [[C42_LOC:%[0-9]+]]:_(p0) = G_GEP [[SP]], [[C42_OFFS]](s64)
-; CHECK: G_STORE [[C42]](s8), [[C42_LOC]](p0) :: (store 1 into stack, align 0)
+; CHECK: G_STORE [[C42]](s8), [[C42_LOC]](p0) :: (store 1 into stack)
 ; CHECK: [[SP:%[0-9]+]]:_(p0) = COPY $sp
 ; CHECK: [[C12_OFFS:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
 ; CHECK: [[C12_LOC:%[0-9]+]]:_(p0) = G_GEP [[SP]], [[C12_OFFS]](s64)
-; CHECK: G_STORE [[C12]](s8), [[C12_LOC]](p0) :: (store 1 into stack + 1, align 0)
+; CHECK: G_STORE [[C12]](s8), [[C12_LOC]](p0) :: (store 1 into stack + 1)
 ; CHECK: BL @test_stack_slots
 define void @test_call_stack() {
   call signext i8 @test_stack_slots([8 x i64] undef, i8 signext 42, i8 signext 12)
@@ -68,12 +68,12 @@ define void @take_128bit_struct([2 x i64]* %ptr, [2 x i64] %in) {
 ; CHECK: [[SP:%[0-9]+]]:_(p0) = COPY $sp
 ; CHECK: [[OFF:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
 ; CHECK: [[ADDR:%[0-9]+]]:_(p0) = G_GEP [[SP]], [[OFF]](s64)
-; CHECK: G_STORE [[EXT1]](s64), [[ADDR]](p0) :: (store 8 into stack, align 0)
+; CHECK: G_STORE [[EXT1]](s64), [[ADDR]](p0) :: (store 8 into stack, align 1)
 
 ; CHECK: [[SP:%[0-9]+]]:_(p0) = COPY $sp
 ; CHECK: [[OFF:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
 ; CHECK: [[ADDR:%[0-9]+]]:_(p0) = G_GEP [[SP]], [[OFF]]
-; CHECK: G_STORE [[EXT2]](s64), [[ADDR]](p0) :: (store 8 into stack + 8, align 0)
+; CHECK: G_STORE [[EXT2]](s64), [[ADDR]](p0) :: (store 8 into stack + 8, align 1)
 define void @test_split_struct([2 x i64]* %ptr) {
   %struct = load [2 x i64], [2 x i64]* %ptr
   call void @take_split_struct([2 x i64]* null, i64 1, i64 2, i64 3,
@@ -88,10 +88,10 @@ define void @test_split_struct([2 x i64]* %ptr) {
 ; CHECK-DAG:   - { id: [[HI_FRAME:[0-9]+]], type: default, offset: 8, size: 8
 
 ; CHECK: [[LOPTR:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.[[LO_FRAME]]
-; CHECK: [[LO:%[0-9]+]]:_(s64) = G_LOAD [[LOPTR]](p0) :: (invariant load 8 from %fixed-stack.[[LO_FRAME]], align 0)
+; CHECK: [[LO:%[0-9]+]]:_(s64) = G_LOAD [[LOPTR]](p0) :: (invariant load 8 from %fixed-stack.[[LO_FRAME]], align 1)
 
 ; CHECK: [[HIPTR:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.[[HI_FRAME]]
-; CHECK: [[HI:%[0-9]+]]:_(s64) = G_LOAD [[HIPTR]](p0) :: (invariant load 8 from %fixed-stack.[[HI_FRAME]], align 0)
+; CHECK: [[HI:%[0-9]+]]:_(s64) = G_LOAD [[HIPTR]](p0) :: (invariant load 8 from %fixed-stack.[[HI_FRAME]], align 1)
 define void @take_split_struct([2 x i64]* %ptr, i64, i64, i64,
                                i64, i64, i64,
                                [2 x i64] %in) {
diff --git a/test/CodeGen/AArch64/GlobalISel/call-translator.ll b/test/CodeGen/AArch64/GlobalISel/call-translator.ll
index e3995b2cbad016fe3a122dc420d8610a45f5504e..9072873e58db95536641703d112f0be58ad9a667 100644
--- a/test/CodeGen/AArch64/GlobalISel/call-translator.ll
+++ b/test/CodeGen/AArch64/GlobalISel/call-translator.ll
@@ -196,11 +196,11 @@ define zeroext i8 @test_abi_zext_ret(i8* %addr) {
 ; CHECK-DAG:  - { id: [[STACK8:[0-9]+]], type: default, offset: 8, size: 8,
 ; CHECK-DAG:  - { id: [[STACK16:[0-9]+]], type: default, offset: 16, size: 8,
 ; CHECK: [[LHS_ADDR:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.[[STACK0]]
-; CHECK: [[LHS:%[0-9]+]]:_(s64) = G_LOAD [[LHS_ADDR]](p0) :: (invariant load 8 from %fixed-stack.[[STACK0]], align 0)
+; CHECK: [[LHS:%[0-9]+]]:_(s64) = G_LOAD [[LHS_ADDR]](p0) :: (invariant load 8 from %fixed-stack.[[STACK0]], align 1)
 ; CHECK: [[RHS_ADDR:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.[[STACK8]]
-; CHECK: [[RHS:%[0-9]+]]:_(s64) = G_LOAD [[RHS_ADDR]](p0) :: (invariant load 8 from %fixed-stack.[[STACK8]], align 0)
+; CHECK: [[RHS:%[0-9]+]]:_(s64) = G_LOAD [[RHS_ADDR]](p0) :: (invariant load 8 from %fixed-stack.[[STACK8]], align 1)
 ; CHECK: [[ADDR_ADDR:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.[[STACK16]]
-; CHECK: [[ADDR:%[0-9]+]]:_(p0) = G_LOAD [[ADDR_ADDR]](p0) :: (invariant load 8 from %fixed-stack.[[STACK16]], align 0)
+; CHECK: [[ADDR:%[0-9]+]]:_(p0) = G_LOAD [[ADDR_ADDR]](p0) :: (invariant load 8 from %fixed-stack.[[STACK16]], align 1)
 ; CHECK: [[SUM:%[0-9]+]]:_(s64) = G_ADD [[LHS]], [[RHS]]
 ; CHECK: G_STORE [[SUM]](s64), [[ADDR]](p0)
 define void @test_stack_slots([8 x i64], i64 %lhs, i64 %rhs, i64* %addr) {
@@ -218,15 +218,15 @@ define void @test_stack_slots([8 x i64], i64 %lhs, i64 %rhs, i64* %addr) {
 ; CHECK: [[SP:%[0-9]+]]:_(p0) = COPY $sp
 ; CHECK: [[C42_OFFS:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
 ; CHECK: [[C42_LOC:%[0-9]+]]:_(p0) = G_GEP [[SP]], [[C42_OFFS]](s64)
-; CHECK: G_STORE [[C42]](s64), [[C42_LOC]](p0) :: (store 8 into stack, align 0)
+; CHECK: G_STORE [[C42]](s64), [[C42_LOC]](p0) :: (store 8 into stack, align 1)
 ; CHECK: [[SP:%[0-9]+]]:_(p0) = COPY $sp
 ; CHECK: [[C12_OFFS:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
 ; CHECK: [[C12_LOC:%[0-9]+]]:_(p0) = G_GEP [[SP]], [[C12_OFFS]](s64)
-; CHECK: G_STORE [[C12]](s64), [[C12_LOC]](p0) :: (store 8 into stack + 8, align 0)
+; CHECK: G_STORE [[C12]](s64), [[C12_LOC]](p0) :: (store 8 into stack + 8, align 1)
 ; CHECK: [[SP:%[0-9]+]]:_(p0) = COPY $sp
 ; CHECK: [[PTR_OFFS:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
 ; CHECK: [[PTR_LOC:%[0-9]+]]:_(p0) = G_GEP [[SP]], [[PTR_OFFS]](s64)
-; CHECK: G_STORE [[PTR]](p0), [[PTR_LOC]](p0) :: (store 8 into stack + 16, align 0)
+; CHECK: G_STORE [[PTR]](p0), [[PTR_LOC]](p0) :: (store 8 into stack + 16, align 1)
 ; CHECK: BL @test_stack_slots
 ; CHECK: ADJCALLSTACKUP 24, 0, implicit-def $sp, implicit $sp
 define void @test_call_stack() {
@@ -239,7 +239,7 @@ define void @test_call_stack() {
 ; CHECK-NEXT: - { id: [[SLOT:[0-9]+]], type: default, offset: 0, size: 1, alignment: 16, stack-id: 0,
 ; CHECK-NEXT: isImmutable: true,
 ; CHECK: [[ADDR:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.[[SLOT]]
-; CHECK: {{%[0-9]+}}:_(s1) = G_LOAD [[ADDR]](p0) :: (invariant load 1 from %fixed-stack.[[SLOT]], align 0)
+; CHECK: {{%[0-9]+}}:_(s1) = G_LOAD [[ADDR]](p0) :: (invariant load 1 from %fixed-stack.[[SLOT]])
 define void @test_mem_i1([8 x i64], i1 %in) {
   ret void
 }
@@ -280,11 +280,11 @@ define void @take_128bit_struct([2 x i64]* %ptr, [2 x i64] %in) {
 ; CHECK: [[SP:%[0-9]+]]:_(p0) = COPY $sp
 ; CHECK: [[CST2:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
 ; CHECK: [[GEP2:%[0-9]+]]:_(p0) = G_GEP [[SP]], [[CST2]](s64)
-; CHECK: G_STORE [[EXTLO]](s64), [[GEP2]](p0) :: (store 8 into stack, align 0)
+; CHECK: G_STORE [[EXTLO]](s64), [[GEP2]](p0) :: (store 8 into stack, align 1)
 ; CHECK: [[SP:%[0-9]+]]:_(p0) = COPY $sp
 ; CHECK: [[CST3:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
 ; CHECK: [[GEP3:%[0-9]+]]:_(p0) = G_GEP [[SP]], [[CST3]](s64)
-; CHECK: G_STORE [[EXTHI]](s64), [[GEP3]](p0) :: (store 8 into stack + 8, align 0)
+; CHECK: G_STORE [[EXTHI]](s64), [[GEP3]](p0) :: (store 8 into stack + 8, align 1)
 define void @test_split_struct([2 x i64]* %ptr) {
   %struct = load [2 x i64], [2 x i64]* %ptr
   call void @take_split_struct([2 x i64]* null, i64 1, i64 2, i64 3,
@@ -299,10 +299,10 @@ define void @test_split_struct([2 x i64]* %ptr) {
 ; CHECK-DAG:   - { id: [[HI_FRAME:[0-9]+]], type: default, offset: 8, size: 8
 
 ; CHECK: [[LOPTR:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.[[LO_FRAME]]
-; CHECK: [[LO:%[0-9]+]]:_(s64) = G_LOAD [[LOPTR]](p0) :: (invariant load 8 from %fixed-stack.[[LO_FRAME]], align 0)
+; CHECK: [[LO:%[0-9]+]]:_(s64) = G_LOAD [[LOPTR]](p0) :: (invariant load 8 from %fixed-stack.[[LO_FRAME]], align 1)
 
 ; CHECK: [[HIPTR:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.[[HI_FRAME]]
-; CHECK: [[HI:%[0-9]+]]:_(s64) = G_LOAD [[HIPTR]](p0) :: (invariant load 8 from %fixed-stack.[[HI_FRAME]], align 0)
+; CHECK: [[HI:%[0-9]+]]:_(s64) = G_LOAD [[HIPTR]](p0) :: (invariant load 8 from %fixed-stack.[[HI_FRAME]], align 1)
 define void @take_split_struct([2 x i64]* %ptr, i64, i64, i64,
                                i64, i64, i64,
                                [2 x i64] %in) {
diff --git a/test/CodeGen/AArch64/GlobalISel/dynamic-alloca-lifetime.ll b/test/CodeGen/AArch64/GlobalISel/dynamic-alloca-lifetime.ll
new file mode 100644
index 0000000000000000000000000000000000000000..3ce873f2f6e8d94768b74ba5c85cd2028a034e5d
--- /dev/null
+++ b/test/CodeGen/AArch64/GlobalISel/dynamic-alloca-lifetime.ll
@@ -0,0 +1,47 @@
+; RUN: llc -mtriple aarch64-unknown-unknown -global-isel \
+; RUN:     -no-stack-coloring=false -pass-remarks-missed=gisel* < %s \
+; RUN:      2>&1 | FileCheck %s
+
+; Same as the dynamic-alloca-lifetime.ll X86 test, which was used to fix a bug
+; in stack colouring + lifetime markers.
+
+; This test crashed in PEI because the stack protector was dead.
+; This was due to it being colored, which was in turn due to incorrect
+; lifetimes being applied to the stack protector frame index.
+
+; CHECK: stack_chk_guard
+; CHECK-NOT: remark{{.*}}foo
+
+; Function Attrs: nounwind
+declare void @llvm.lifetime.start.p0i8(i64, i8* nocapture) #0
+
+; Function Attrs: nounwind
+declare void @llvm.lifetime.end.p0i8(i64, i8* nocapture) #0
+
+; Function Attrs: ssp
+define void @foo(i1 %cond1, i1 %cond2) #1 {
+entry:
+  %bitmapBuffer = alloca [8192 x i8], align 1
+  br i1 %cond1, label %end1, label %bb1
+
+bb1:
+  %bitmapBuffer229 = alloca [8192 x i8], align 1
+  br i1 %cond2, label %end1, label %if.else130
+
+end1:
+  ret void
+
+if.else130:                                       ; preds = %bb1
+  %tmp = getelementptr inbounds [8192 x i8], [8192 x i8]* %bitmapBuffer, i32 0, i32 0
+  call void @llvm.lifetime.start.p0i8(i64 8192, i8* %tmp) #0
+  call void @llvm.lifetime.end.p0i8(i64 8192, i8* %tmp) #0
+  %tmp25 = getelementptr inbounds [8192 x i8], [8192 x i8]* %bitmapBuffer229, i32 0, i32 0
+  call void @llvm.lifetime.start.p0i8(i64 8192, i8* %tmp25) #0
+  call void @llvm.lifetime.end.p0i8(i64 8192, i8* %tmp25) #0
+  br label %end1
+}
+
+declare void @bar()
+
+attributes #0 = { nounwind }
+attributes #1 = { ssp }
diff --git a/test/CodeGen/AArch64/GlobalISel/gisel-commandline-option-fastisel.ll b/test/CodeGen/AArch64/GlobalISel/gisel-commandline-option-fastisel.ll
new file mode 100644
index 0000000000000000000000000000000000000000..73158f54325f916bb4bab592f6c526756f67886f
--- /dev/null
+++ b/test/CodeGen/AArch64/GlobalISel/gisel-commandline-option-fastisel.ll
@@ -0,0 +1,35 @@
+; REQUIRES: asserts
+
+; RUN: llc -mtriple=aarch64-- -debug-pass=Structure %s -o /dev/null 2>&1 \
+; RUN:   -verify-machineinstrs=0 -O0 -global-isel=false -debug-only=isel \
+; RUN:   | FileCheck %s --check-prefixes=DISABLED,FASTISEL
+
+; RUN: llc -mtriple=aarch64-- -debug-pass=Structure %s -o /dev/null 2>&1 \
+; RUN:   -verify-machineinstrs=0 -O1 -global-isel=false -debug-only=isel \
+; RUN:   | FileCheck %s --check-prefixes=DISABLED,NOFASTISEL
+
+; RUN: llc -mtriple=aarch64-- -debug-pass=Structure %s -o /dev/null 2>&1 \
+; RUN:   -verify-machineinstrs=0 -O0 -fast-isel=false -global-isel=false \
+; RUN:   -debug-only=isel \
+; RUN:   | FileCheck %s --check-prefixes=DISABLED,NOFASTISEL
+
+; RUN: llc -mtriple=aarch64-- -debug-pass=Structure %s -o /dev/null 2>&1 \
+; RUN:   -verify-machineinstrs=0 -O1 -fast-isel=false -global-isel=false \
+; RUN:   -debug-only=isel \
+; RUN:   | FileCheck %s --check-prefixes=DISABLED,NOFASTISEL
+
+; Check that the right instruction selector is chosen when using
+; -global-isel=false. FastISel should be used at -O0 (unless -fast-isel=false is
+; also present) and SelectionDAG otherwise.
+
+; DISABLED-NOT: IRTranslator
+
+; DISABLED: AArch64 Instruction Selection
+; DISABLED: Expand ISel Pseudo-instructions
+
+; FASTISEL: Enabling fast-isel
+; NOFASTISEL-NOT: Enabling fast-isel
+
+define void @empty() {
+  ret void
+}
diff --git a/test/CodeGen/AArch64/GlobalISel/gisel-commandline-option.ll b/test/CodeGen/AArch64/GlobalISel/gisel-commandline-option.ll
index 87425b8b6cb0da8d0e350a51f1d3ed73435d68bf..a044b3492f4dc4233393705da397eaa109c688ab 100644
--- a/test/CodeGen/AArch64/GlobalISel/gisel-commandline-option.ll
+++ b/test/CodeGen/AArch64/GlobalISel/gisel-commandline-option.ll
@@ -45,6 +45,7 @@
 ; VERIFY-NEXT:   Verify generated machine code
 ; ENABLED-NEXT:  PreLegalizerCombiner
 ; VERIFY-NEXT:   Verify generated machine code
+; ENABLED-NEXT:  Analysis containing CSE Info
 ; ENABLED-NEXT:  Legalizer
 ; VERIFY-NEXT:   Verify generated machine code
 ; ENABLED-NEXT:  RegBankSelect
diff --git a/test/CodeGen/AArch64/GlobalISel/irtranslator-max-address-space.ll b/test/CodeGen/AArch64/GlobalISel/irtranslator-max-address-space.ll
new file mode 100644
index 0000000000000000000000000000000000000000..5bbc0ede90c8fffcfe0dc15c4d2ea070fd5ed791
--- /dev/null
+++ b/test/CodeGen/AArch64/GlobalISel/irtranslator-max-address-space.ll
@@ -0,0 +1,26 @@
+; RUN: llc -O0 -mtriple=aarch64-apple-ios -global-isel -stop-after=irtranslator %s -o - | FileCheck %s
+
+; CHECK-LABEL: name: store_max_address_space
+; CHECK: %0:_(p16777215) = COPY $x0
+; CHECK: G_STORE %1(s32), %0(p16777215) :: (store 4 into %ir.ptr, addrspace 16777215)
+define void @store_max_address_space(i32 addrspace(16777215)* %ptr) {
+  store i32 0, i32 addrspace(16777215)* %ptr
+  ret void
+}
+
+; CHECK-LABEL: name: store_max_address_space_vector
+; CHECK: %0:_(<2 x p16777215>) = COPY $q0
+; CHECK: %1:_(p16777215) = G_EXTRACT_VECTOR_ELT %0(<2 x p16777215>), %2(s64)
+; CHECK: %1(p16777215) :: (store 4 into %ir.elt0, addrspace 16777215)
+define void @store_max_address_space_vector(<2 x i32 addrspace(16777215)*> %vptr) {
+  %elt0 = extractelement <2 x i32 addrspace(16777215)*> %vptr, i32 0
+  store i32 0, i32 addrspace(16777215)* %elt0
+  ret void
+}
+
+; CHECK-LABEL: name: max_address_space_vector_max_num_elts
+; CHECK: %0:_(<65535 x p16777215>) = G_LOAD %1(p0) :: (volatile load 524280 from `<65535 x i32 addrspace(16777215)*>* undef`, align 524288)
+define void @max_address_space_vector_max_num_elts() {
+  %load = load volatile <65535 x i32 addrspace(16777215)*>, <65535 x i32 addrspace(16777215)*>* undef
+  ret void
+}
diff --git a/test/CodeGen/AArch64/GlobalISel/legalize-ceil.mir b/test/CodeGen/AArch64/GlobalISel/legalize-ceil.mir
new file mode 100644
index 0000000000000000000000000000000000000000..bb0063b167fe044004a72da16aea4f94acf2bb91
--- /dev/null
+++ b/test/CodeGen/AArch64/GlobalISel/legalize-ceil.mir
@@ -0,0 +1,86 @@
+# RUN: llc -mtriple=arm64-unknown-unknown -global-isel -O0 -mattr=-fullfp16 -run-pass=legalizer %s -o - | FileCheck %s
+
+--- |
+  define <8 x half> @test_v8f16.ceil(<8 x half> %a) {
+    ret <8 x half> %a
+  }
+
+  define <4 x half> @test_v4f16.ceil(<4 x half> %a) {
+    ret <4 x half> %a
+  }
+
+...
+---
+name:            test_v8f16.ceil
+alignment:       2
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+body:             |
+  bb.1 (%ir-block.0):
+    liveins: $q0
+    ; CHECK-LABEL: name:            test_v8f16.ceil
+    %0:_(<8 x s16>) = COPY $q0
+    ; CHECK: %{{[0-9]+}}:_(s16), %{{[0-9]+}}:_(s16), %{{[0-9]+}}:_(s16), %{{[0-9]+}}:_(s16), %{{[0-9]+}}:_(s16), %{{[0-9]+}}:_(s16), %{{[0-9]+}}:_(s16), %{{[0-9]+}}:_(s16) = G_UNMERGE_VALUES %{{[0-9]+}}(<8 x s16>)
+    ; CHECK: %{{[0-9]+}}:_(s32) = G_FPEXT %{{[0-9]+}}(s16)
+    ; CHECK: %{{[0-9]+}}:_(s32) = G_FCEIL %{{[0-9]+}}
+    ; CHECK: %{{[0-9]+}}:_(s16) = G_FPTRUNC %{{[0-9]+}}(s32)
+    ; CHECK: %{{[0-9]+}}:_(s32) = G_FPEXT %{{[0-9]+}}(s16)
+    ; CHECK: %{{[0-9]+}}:_(s32) = G_FCEIL %{{[0-9]+}}
+    ; CHECK: %{{[0-9]+}}:_(s16) = G_FPTRUNC %{{[0-9]+}}(s32)
+    ; CHECK: %{{[0-9]+}}:_(s32) = G_FPEXT %{{[0-9]+}}(s16)
+    ; CHECK: %{{[0-9]+}}:_(s32) = G_FCEIL %{{[0-9]+}}
+    ; CHECK: %{{[0-9]+}}:_(s16) = G_FPTRUNC %{{[0-9]+}}(s32)
+    ; CHECK: %{{[0-9]+}}:_(s32) = G_FPEXT %{{[0-9]+}}(s16)
+    ; CHECK: %{{[0-9]+}}:_(s32) = G_FCEIL %{{[0-9]+}}
+    ; CHECK: %{{[0-9]+}}:_(s16) = G_FPTRUNC %{{[0-9]+}}(s32)
+    ; CHECK: %{{[0-9]+}}:_(s32) = G_FPEXT %{{[0-9]+}}(s16)
+    ; CHECK: %{{[0-9]+}}:_(s32) = G_FCEIL %{{[0-9]+}}
+    ; CHECK: %{{[0-9]+}}:_(s16) = G_FPTRUNC %{{[0-9]+}}(s32)
+    ; CHECK: %{{[0-9]+}}:_(s32) = G_FPEXT %{{[0-9]+}}(s16)
+    ; CHECK: %{{[0-9]+}}:_(s32) = G_FCEIL %{{[0-9]+}}
+    ; CHECK: %{{[0-9]+}}:_(s16) = G_FPTRUNC %{{[0-9]+}}(s32)
+    ; CHECK: %{{[0-9]+}}:_(s32) = G_FPEXT %{{[0-9]+}}(s16)
+    ; CHECK: %{{[0-9]+}}:_(s32) = G_FCEIL %{{[0-9]+}}
+    ; CHECK: %{{[0-9]+}}:_(s16) = G_FPTRUNC %{{[0-9]+}}(s32)
+    ; CHECK: %{{[0-9]+}}:_(s32) = G_FPEXT %{{[0-9]+}}(s16)
+    ; CHECK: %{{[0-9]+}}:_(s32) = G_FCEIL %{{[0-9]+}}
+    ; CHECK: %{{[0-9]+}}:_(s16) = G_FPTRUNC %{{[0-9]+}}(s32)
+    ; CHECK: %{{[0-9]+}}:_(<8 x s16>) = G_BUILD_VECTOR %{{[0-9]+}}(s16), %{{[0-9]+}}(s16), %{{[0-9]+}}(s16), %{{[0-9]+}}(s16), %{{[0-9]+}}(s16), %{{[0-9]+}}(s16), %{{[0-9]+}}(s16), %{{[0-9]+}}(s16)
+    %1:_(<8 x s16>) = G_FCEIL %0
+    $q0 = COPY %1(<8 x s16>)
+    RET_ReallyLR implicit $q0
+
+...
+---
+name:            test_v4f16.ceil
+alignment:       2
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+body:             |
+  bb.1 (%ir-block.0):
+    liveins: $d0
+    ; CHECK-LABEL: name:            test_v4f16.ceil
+    %0:_(<4 x s16>) = COPY $d0
+    ; CHECK: %{{[0-9]+}}:_(s16), %{{[0-9]+}}:_(s16), %{{[0-9]+}}:_(s16)  = G_UNMERGE_VALUES %{{[0-9]+}}(<4 x s16>)
+    ; CHECK: %{{[0-9]+}}:_(s32) = G_FPEXT %{{[0-9]+}}(s16)
+    ; CHECK: %{{[0-9]+}}:_(s32) = G_FCEIL %{{[0-9]+}}
+    ; CHECK: %{{[0-9]+}}:_(s16) = G_FPTRUNC %{{[0-9]+}}(s32)
+    ; CHECK: %{{[0-9]+}}:_(s32) = G_FPEXT %{{[0-9]+}}(s16)
+    ; CHECK: %{{[0-9]+}}:_(s32) = G_FCEIL %{{[0-9]+}}
+    ; CHECK: %{{[0-9]+}}:_(s16) = G_FPTRUNC %{{[0-9]+}}(s32)
+    ; CHECK: %{{[0-9]+}}:_(s32) = G_FPEXT %{{[0-9]+}}(s16)
+    ; CHECK: %{{[0-9]+}}:_(s32) = G_FCEIL %{{[0-9]+}}
+    ; CHECK: %{{[0-9]+}}:_(s16) = G_FPTRUNC %{{[0-9]+}}(s32)
+    ; CHECK: %{{[0-9]+}}:_(s32) = G_FPEXT %{{[0-9]+}}(s16)
+    ; CHECK: %{{[0-9]+}}:_(s32) = G_FCEIL %{{[0-9]+}}
+    ; CHECK: %{{[0-9]+}}:_(s16) = G_FPTRUNC %{{[0-9]+}}(s32)
+    ; CHECK: %{{[0-9]+}}:_(<4 x s16>) = G_BUILD_VECTOR %{{[0-9]+}}(s16), %{{[0-9]+}}(s16), %{{[0-9]+}}(s16), %{{[0-9]+}}(s16)
+    %1:_(<4 x s16>) = G_FCEIL %0
+    $d0 = COPY %1(<4 x s16>)
+    RET_ReallyLR implicit $d0
+
+...
diff --git a/test/CodeGen/AArch64/GlobalISel/legalize-cmp.mir b/test/CodeGen/AArch64/GlobalISel/legalize-cmp.mir
index ef86df6a5c15188cbf0b59dce9fb8523086b26ea..209d8e9b37ff876ea8d2014368c51b7345556037 100644
--- a/test/CodeGen/AArch64/GlobalISel/legalize-cmp.mir
+++ b/test/CodeGen/AArch64/GlobalISel/legalize-cmp.mir
@@ -1,5 +1,5 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -march=aarch64 -run-pass=legalizer %s -o - | FileCheck %s
+# RUN: llc -O0 -march=aarch64 -run-pass=legalizer %s -o - | FileCheck %s
 ---
 name:            test_icmp
 body:             |
diff --git a/test/CodeGen/AArch64/GlobalISel/legalize-combines.mir b/test/CodeGen/AArch64/GlobalISel/legalize-combines.mir
index 3d0bd02a9811b36245b38ddc31f160df1a7d0103..14ae981c2ede1997121813fd112c9d947f10c743 100644
--- a/test/CodeGen/AArch64/GlobalISel/legalize-combines.mir
+++ b/test/CodeGen/AArch64/GlobalISel/legalize-combines.mir
@@ -22,8 +22,11 @@ body: |
     ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $w0
     ; CHECK: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[COPY]], [[COPY]]
     ; CHECK: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[COPY]](s32), [[ADD]](s32)
-    ; CHECK: [[EXTRACT:%[0-9]+]]:_(s1) = G_EXTRACT [[MV]](s64), 0
-    ; CHECK: [[EXTRACT1:%[0-9]+]]:_(s64) = COPY [[MV]](s64)
+    ; CHECK: [[EXTRACT:%[0-9]+]]:_(s1) = G_EXTRACT [[COPY]](s32), 0
+    ; CHECK: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[EXTRACT]](s1)
+    ; CHECK: $w0 = COPY [[ANYEXT]](s32)
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s64) = COPY [[MV]](s64)
+    ; CHECK: $x0 = COPY [[COPY1]](s64)
     %0:_(s32) = COPY $w0
 
     %1:_(s32) = G_ADD %0, %0
@@ -45,6 +48,7 @@ body: |
     ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $w0
     ; CHECK: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[COPY]], [[COPY]]
     ; CHECK: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[COPY]], [[ADD]]
+    ; CHECK: $w0 = COPY [[ADD1]](s32)
     %0:_(s32) = COPY $w0
 
     %1:_(s32) = G_ADD %0, %0
@@ -64,6 +68,7 @@ body: |
     ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY $x0
     ; CHECK: [[COPY1:%[0-9]+]]:_(s64) = COPY [[COPY]](s64)
     ; CHECK: [[ADD:%[0-9]+]]:_(s64) = G_ADD [[COPY1]], [[COPY1]]
+    ; CHECK: $x0 = COPY [[ADD]](s64)
     %0:_(s64) = COPY $x0
 
     %1:_(s128) = G_MERGE_VALUES %0, %0
@@ -82,6 +87,7 @@ body: |
     ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $w0
     ; CHECK: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[COPY]], [[COPY]]
     ; CHECK: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[COPY]], [[ADD]]
+    ; CHECK: $w0 = COPY [[ADD1]](s32)
     %0:_(s32) = COPY $w0
 
     %1:_(s32) = G_ADD %0, %0
diff --git a/test/CodeGen/AArch64/GlobalISel/legalize-cos.mir b/test/CodeGen/AArch64/GlobalISel/legalize-cos.mir
new file mode 100644
index 0000000000000000000000000000000000000000..a2e8ee172324af7aece2ff1bdacbd3dcbdb97fc0
--- /dev/null
+++ b/test/CodeGen/AArch64/GlobalISel/legalize-cos.mir
@@ -0,0 +1,227 @@
+# RUN: llc -verify-machineinstrs -mtriple aarch64--- \
+# RUN: -run-pass=legalizer -mattr=+fullfp16 -global-isel %s -o - \
+# RUN: | FileCheck %s
+...
+---
+name:            test_v4f16.cos
+alignment:       2
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+body:             |
+  bb.0:
+    liveins: $d0
+    ; CHECK-LABEL: name:            test_v4f16.cos
+    ; CHECK: [[V1:%[0-9]+]]:_(s16), [[V2:%[0-9]+]]:_(s16), [[V3:%[0-9]+]]:_(s16), [[V4:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES %{{[0-9]+}}(<4 x s16>)
+
+    ; CHECK-DAG: [[V1_S32:%[0-9]+]]:_(s32) = G_FPEXT [[V1]](s16)
+    ; CHECK-NEXT: ADJCALLSTACKDOWN
+    ; CHECK-NEXT: $s0 = COPY [[V1_S32]](s32)
+    ; CHECK-NEXT: BL &cosf
+    ; CHECK-NEXT: [[ELT1_S32:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-NEXT: ADJCALLSTACKUP
+    ; CHECK-NEXT: [[ELT1:%[0-9]+]]:_(s16) = G_FPTRUNC [[ELT1_S32]](s32)
+
+    ; CHECK-DAG: [[V2_S32:%[0-9]+]]:_(s32) = G_FPEXT [[V2]](s16)
+    ; CHECK-NEXT: ADJCALLSTACKDOWN
+    ; CHECK-NEXT: $s0 = COPY [[V2_S32]](s32)
+    ; CHECK-NEXT: BL &cosf
+    ; CHECK-NEXT: [[ELT2_S32:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-NEXT: ADJCALLSTACKUP
+    ; CHECK-NEXT: [[ELT2:%[0-9]+]]:_(s16) = G_FPTRUNC [[ELT2_S32]](s32)
+
+    ; CHECK-DAG: [[V3_S32:%[0-9]+]]:_(s32) = G_FPEXT [[V3]](s16)
+    ; CHECK-NEXT: ADJCALLSTACKDOWN
+    ; CHECK-NEXT: $s0 = COPY [[V3_S32]](s32)
+    ; CHECK-NEXT: BL &cosf
+    ; CHECK-NEXT: [[ELT3_S32:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-NEXT: ADJCALLSTACKUP
+    ; CHECK-NEXT: [[ELT3:%[0-9]+]]:_(s16) = G_FPTRUNC [[ELT3_S32]](s32)
+
+    ; CHECK-DAG: [[V4_S32:%[0-9]+]]:_(s32) = G_FPEXT [[V4]](s16)
+    ; CHECK-NEXT: ADJCALLSTACKDOWN
+    ; CHECK-NEXT: $s0 = COPY [[V4_S32]](s32)
+    ; CHECK-NEXT: BL &cosf
+    ; CHECK-NEXT: [[ELT4_S32:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-NEXT: ADJCALLSTACKUP
+    ; CHECK-NEXT: [[ELT4:%[0-9]+]]:_(s16) = G_FPTRUNC [[ELT4_S32]](s32)
+
+    ; CHECK-DAG: %{{[0-9]+}}:_(<4 x s16>) = G_BUILD_VECTOR [[ELT1]](s16), [[ELT2]](s16), [[ELT3]](s16), [[ELT4]](s16)
+
+    %0:_(<4 x s16>) = COPY $d0
+    %1:_(<4 x s16>) = G_FCOS %0
+    $d0 = COPY %1(<4 x s16>)
+    RET_ReallyLR implicit $d0
+
+...
+---
+name:            test_v8f16.cos
+alignment:       2
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+body:             |
+  bb.0:
+    liveins: $q0
+
+    ; CHECK-LABEL: name:            test_v8f16.cos
+
+    ; This is big, so let's just check for the 8 calls to cosf, the the
+    ; G_UNMERGE_VALUES, and the G_BUILD_VECTOR. The other instructions ought
+    ; to be covered by the other tests.
+
+    ; CHECK: G_UNMERGE_VALUES
+    ; CHECK: BL &cosf
+    ; CHECK: BL &cosf
+    ; CHECK: BL &cosf
+    ; CHECK: BL &cosf
+    ; CHECK: BL &cosf
+    ; CHECK: BL &cosf
+    ; CHECK: BL &cosf
+    ; CHECK: BL &cosf
+    ; CHECK: G_BUILD_VECTOR
+
+    %0:_(<8 x s16>) = COPY $q0
+    %1:_(<8 x s16>) = G_FCOS %0
+    $q0 = COPY %1(<8 x s16>)
+    RET_ReallyLR implicit $q0
+
+...
+---
+name:            test_v2f32.cos
+alignment:       2
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+body:             |
+  bb.0:
+    liveins: $d0
+
+    ; CHECK-LABEL: name:            test_v2f32.cos
+    ; CHECK: [[V1:%[0-9]+]]:_(s32), [[V2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES %{{[0-9]+}}(<2 x s32>)
+
+    ; CHECK-NEXT: ADJCALLSTACKDOWN
+    ; CHECK-DAG: $s0 = COPY [[V1]](s32)
+    ; CHECK-DAG: BL &cosf
+    ; CHECK-DAG: [[ELT1:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-DAG: ADJCALLSTACKUP
+
+    ; CHECK-DAG: ADJCALLSTACKDOWN
+    ; CHECK-DAG: $s0 = COPY [[V2]](s32)
+    ; CHECK-DAG: BL &cosf
+    ; CHECK-DAG: [[ELT2:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-DAG: ADJCALLSTACKUP
+
+    ; CHECK-DAG: %1:_(<2 x s32>) = G_BUILD_VECTOR [[ELT1]](s32), [[ELT2]](s32)
+
+    %0:_(<2 x s32>) = COPY $d0
+    %1:_(<2 x s32>) = G_FCOS %0
+    $d0 = COPY %1(<2 x s32>)
+    RET_ReallyLR implicit $d0
+
+...
+---
+name:            test_v4f32.cos
+alignment:       2
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+body:             |
+  bb.0:
+    liveins: $q0
+    ; CHECK-LABEL: name:            test_v4f32.cos
+    ; CHECK: [[V1:%[0-9]+]]:_(s32), [[V2:%[0-9]+]]:_(s32), [[V3:%[0-9]+]]:_(s32), [[V4:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES %{{[0-9]+}}(<4 x s32>)
+
+    ; CHECK-NEXT: ADJCALLSTACKDOWN
+    ; CHECK-DAG: $s0 = COPY [[V1]](s32)
+    ; CHECK-DAG: BL &cosf
+    ; CHECK-DAG: [[ELT1:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-DAG: ADJCALLSTACKUP
+
+    ; CHECK-DAG: ADJCALLSTACKDOWN
+    ; CHECK-DAG: $s0 = COPY [[V2]](s32)
+    ; CHECK-DAG: BL &cosf
+    ; CHECK-DAG: [[ELT2:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-DAG: ADJCALLSTACKUP
+
+    ; CHECK-DAG: ADJCALLSTACKDOWN
+    ; CHECK-DAG: $s0 = COPY [[V3]](s32)
+    ; CHECK-DAG: BL &cosf
+    ; CHECK-DAG: [[ELT3:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-DAG: ADJCALLSTACKUP
+
+    ; CHECK-DAG: ADJCALLSTACKDOWN
+    ; CHECK-DAG: $s0 = COPY [[V4]](s32)
+    ; CHECK-DAG: BL &cosf
+    ; CHECK-DAG: [[ELT4:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-DAG: ADJCALLSTACKUP
+
+    ; CHECK-DAG: %1:_(<4 x s32>) = G_BUILD_VECTOR [[ELT1]](s32), [[ELT2]](s32), [[ELT3]](s32), [[ELT4]](s32)
+
+    %0:_(<4 x s32>) = COPY $q0
+    %1:_(<4 x s32>) = G_FCOS %0
+    $q0 = COPY %1(<4 x s32>)
+    RET_ReallyLR implicit $q0
+
+...
+---
+name:            test_v2f64.cos
+alignment:       2
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+body:             |
+  bb.0:
+    liveins: $q0
+
+    ; CHECK-LABEL: name:            test_v2f64.cos
+    ; CHECK: [[V1:%[0-9]+]]:_(s64), [[V2:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES %{{[0-9]+}}(<2 x s64>)
+
+    ; CHECK-NEXT: ADJCALLSTACKDOWN
+    ; CHECK-DAG: $d0 = COPY [[V1]](s64)
+    ; CHECK-DAG: BL &cos
+    ; CHECK-DAG: [[ELT1:%[0-9]+]]:_(s64) = COPY $d0
+    ; CHECK-DAG: ADJCALLSTACKUP
+
+    ; CHECK-DAG: ADJCALLSTACKDOWN
+    ; CHECK-DAG: $d0 = COPY [[V2]](s64)
+    ; CHECK-DAG: BL &cos
+    ; CHECK-DAG: [[ELT2:%[0-9]+]]:_(s64) = COPY $d0
+    ; CHECK-DAG: ADJCALLSTACKUP
+
+    ; CHECK-DAG: %1:_(<2 x s64>) = G_BUILD_VECTOR [[ELT1]](s64), [[ELT2]](s64)
+
+    %0:_(<2 x s64>) = COPY $q0
+    %1:_(<2 x s64>) = G_FCOS %0
+    $q0 = COPY %1(<2 x s64>)
+    RET_ReallyLR implicit $q0
+
+...
+---
+name:            test_cos_half
+alignment:       2
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+body:             |
+  bb.0:
+    liveins: $h0
+    ; CHECK-LABEL: name:            test_cos_half
+    ; CHECK: [[REG1:%[0-9]+]]:_(s32) = G_FPEXT %0(s16)
+    ; CHECK-NEXT: ADJCALLSTACKDOWN
+    ; CHECK-NEXT: $s0 = COPY [[REG1]](s32)
+    ; CHECK-NEXT: BL &cosf
+    ; CHECK-NEXT: [[REG2:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-NEXT: ADJCALLSTACKUP
+    ; CHECK-NEXT: [[RES:%[0-9]+]]:_(s16) = G_FPTRUNC [[REG2]](s32)
+
+    %0:_(s16) = COPY $h0
+    %1:_(s16) = G_FCOS %0
+    $h0 = COPY %1(s16)
+    RET_ReallyLR implicit $h0
diff --git a/test/CodeGen/AArch64/GlobalISel/legalize-div.mir b/test/CodeGen/AArch64/GlobalISel/legalize-div.mir
index 4753e17ca1cc4eaf34da05e79df6eae2976dc2fb..4f4ff0a7feeb9d97150fcb4b40eb91f4c205b8fd 100644
--- a/test/CodeGen/AArch64/GlobalISel/legalize-div.mir
+++ b/test/CodeGen/AArch64/GlobalISel/legalize-div.mir
@@ -1,5 +1,5 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -march=aarch64 -run-pass=legalizer %s -o - | FileCheck %s
+# RUN: llc -O0 -march=aarch64 -run-pass=legalizer %s -o - | FileCheck %s
 ---
 name:            test_div
 body:             |
diff --git a/test/CodeGen/AArch64/GlobalISel/legalize-exp.mir b/test/CodeGen/AArch64/GlobalISel/legalize-exp.mir
new file mode 100644
index 0000000000000000000000000000000000000000..d299d6e7b6da85c4aac21940b8999a6bd3200394
--- /dev/null
+++ b/test/CodeGen/AArch64/GlobalISel/legalize-exp.mir
@@ -0,0 +1,227 @@
+# RUN: llc -verify-machineinstrs -mtriple aarch64--- \
+# RUN: -run-pass=legalizer -mattr=+fullfp16 -global-isel %s -o - \
+# RUN: | FileCheck %s
+...
+---
+name:            test_v4f16.exp
+alignment:       2
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+body:             |
+  bb.0:
+    liveins: $d0
+    ; CHECK-LABEL: name:            test_v4f16.exp
+    ; CHECK: [[V1:%[0-9]+]]:_(s16), [[V2:%[0-9]+]]:_(s16), [[V3:%[0-9]+]]:_(s16), [[V4:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES %{{[0-9]+}}(<4 x s16>)
+
+    ; CHECK-DAG: [[V1_S32:%[0-9]+]]:_(s32) = G_FPEXT [[V1]](s16)
+    ; CHECK-NEXT: ADJCALLSTACKDOWN
+    ; CHECK-NEXT: $s0 = COPY [[V1_S32]](s32)
+    ; CHECK-NEXT: BL &expf
+    ; CHECK-NEXT: [[ELT1_S32:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-NEXT: ADJCALLSTACKUP
+    ; CHECK-NEXT: [[ELT1:%[0-9]+]]:_(s16) = G_FPTRUNC [[ELT1_S32]](s32)
+
+    ; CHECK-DAG: [[V2_S32:%[0-9]+]]:_(s32) = G_FPEXT [[V2]](s16)
+    ; CHECK-NEXT: ADJCALLSTACKDOWN
+    ; CHECK-NEXT: $s0 = COPY [[V2_S32]](s32)
+    ; CHECK-NEXT: BL &expf
+    ; CHECK-NEXT: [[ELT2_S32:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-NEXT: ADJCALLSTACKUP
+    ; CHECK-NEXT: [[ELT2:%[0-9]+]]:_(s16) = G_FPTRUNC [[ELT2_S32]](s32)
+
+    ; CHECK-DAG: [[V3_S32:%[0-9]+]]:_(s32) = G_FPEXT [[V3]](s16)
+    ; CHECK-NEXT: ADJCALLSTACKDOWN
+    ; CHECK-NEXT: $s0 = COPY [[V3_S32]](s32)
+    ; CHECK-NEXT: BL &expf
+    ; CHECK-NEXT: [[ELT3_S32:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-NEXT: ADJCALLSTACKUP
+    ; CHECK-NEXT: [[ELT3:%[0-9]+]]:_(s16) = G_FPTRUNC [[ELT3_S32]](s32)
+
+    ; CHECK-DAG: [[V4_S32:%[0-9]+]]:_(s32) = G_FPEXT [[V4]](s16)
+    ; CHECK-NEXT: ADJCALLSTACKDOWN
+    ; CHECK-NEXT: $s0 = COPY [[V4_S32]](s32)
+    ; CHECK-NEXT: BL &expf
+    ; CHECK-NEXT: [[ELT4_S32:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-NEXT: ADJCALLSTACKUP
+    ; CHECK-NEXT: [[ELT4:%[0-9]+]]:_(s16) = G_FPTRUNC [[ELT4_S32]](s32)
+
+    ; CHECK-DAG: %{{[0-9]+}}:_(<4 x s16>) = G_BUILD_VECTOR [[ELT1]](s16), [[ELT2]](s16), [[ELT3]](s16), [[ELT4]](s16)
+
+    %0:_(<4 x s16>) = COPY $d0
+    %1:_(<4 x s16>) = G_FEXP %0
+    $d0 = COPY %1(<4 x s16>)
+    RET_ReallyLR implicit $d0
+
+...
+---
+name:            test_v8f16.exp
+alignment:       2
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+body:             |
+  bb.0:
+    liveins: $q0
+
+    ; CHECK-LABEL: name:            test_v8f16.exp
+
+    ; This is big, so let's just check for the 8 calls to expf, the the
+    ; G_UNMERGE_VALUES, and the G_BUILD_VECTOR. The other instructions ought
+    ; to be covered by the other tests.
+
+    ; CHECK: G_UNMERGE_VALUES
+    ; CHECK: BL &expf
+    ; CHECK: BL &expf
+    ; CHECK: BL &expf
+    ; CHECK: BL &expf
+    ; CHECK: BL &expf
+    ; CHECK: BL &expf
+    ; CHECK: BL &expf
+    ; CHECK: BL &expf
+    ; CHECK-DAG: G_BUILD_VECTOR
+
+    %0:_(<8 x s16>) = COPY $q0
+    %1:_(<8 x s16>) = G_FEXP %0
+    $q0 = COPY %1(<8 x s16>)
+    RET_ReallyLR implicit $q0
+
+...
+---
+name:            test_v2f32.exp
+alignment:       2
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+body:             |
+  bb.0:
+    liveins: $d0
+
+    ; CHECK-LABEL: name:            test_v2f32.exp
+    ; CHECK: [[V1:%[0-9]+]]:_(s32), [[V2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES %{{[0-9]+}}(<2 x s32>)
+
+    ; CHECK-NEXT: ADJCALLSTACKDOWN
+    ; CHECK-DAG: $s0 = COPY [[V1]](s32)
+    ; CHECK-DAG: BL &expf
+    ; CHECK-DAG: [[ELT1:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-DAG: ADJCALLSTACKUP
+
+    ; CHECK-DAG: ADJCALLSTACKDOWN
+    ; CHECK-DAG: $s0 = COPY [[V2]](s32)
+    ; CHECK-DAG: BL &expf
+    ; CHECK-DAG: [[ELT2:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-DAG: ADJCALLSTACKUP
+
+    ; CHECK-DAG: %1:_(<2 x s32>) = G_BUILD_VECTOR [[ELT1]](s32), [[ELT2]](s32)
+
+    %0:_(<2 x s32>) = COPY $d0
+    %1:_(<2 x s32>) = G_FEXP %0
+    $d0 = COPY %1(<2 x s32>)
+    RET_ReallyLR implicit $d0
+
+...
+---
+name:            test_v4f32.exp
+alignment:       2
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+body:             |
+  bb.0:
+    liveins: $q0
+    ; CHECK-LABEL: name:            test_v4f32.exp
+    ; CHECK: [[V1:%[0-9]+]]:_(s32), [[V2:%[0-9]+]]:_(s32), [[V3:%[0-9]+]]:_(s32), [[V4:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES %{{[0-9]+}}(<4 x s32>)
+
+    ; CHECK-NEXT: ADJCALLSTACKDOWN
+    ; CHECK-DAG: $s0 = COPY [[V1]](s32)
+    ; CHECK-DAG: BL &expf
+    ; CHECK-DAG: [[ELT1:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-DAG: ADJCALLSTACKUP
+
+    ; CHECK-DAG: ADJCALLSTACKDOWN
+    ; CHECK-DAG: $s0 = COPY [[V2]](s32)
+    ; CHECK-DAG: BL &expf
+    ; CHECK-DAG: [[ELT2:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-DAG: ADJCALLSTACKUP
+
+    ; CHECK-DAG: ADJCALLSTACKDOWN
+    ; CHECK-DAG: $s0 = COPY [[V3]](s32)
+    ; CHECK-DAG: BL &expf
+    ; CHECK-DAG: [[ELT3:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-DAG: ADJCALLSTACKUP
+
+    ; CHECK-DAG: ADJCALLSTACKDOWN
+    ; CHECK-DAG: $s0 = COPY [[V4]](s32)
+    ; CHECK-DAG: BL &expf
+    ; CHECK-DAG: [[ELT4:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-DAG: ADJCALLSTACKUP
+
+    ; CHECK-DAG: %1:_(<4 x s32>) = G_BUILD_VECTOR [[ELT1]](s32), [[ELT2]](s32), [[ELT3]](s32), [[ELT4]](s32)
+
+    %0:_(<4 x s32>) = COPY $q0
+    %1:_(<4 x s32>) = G_FEXP %0
+    $q0 = COPY %1(<4 x s32>)
+    RET_ReallyLR implicit $q0
+
+...
+---
+name:            test_v2f64.exp
+alignment:       2
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+body:             |
+  bb.0:
+    liveins: $q0
+
+    ; CHECK-LABEL: name:            test_v2f64.exp
+    ; CHECK: [[V1:%[0-9]+]]:_(s64), [[V2:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES %{{[0-9]+}}(<2 x s64>)
+
+    ; CHECK-NEXT: ADJCALLSTACKDOWN
+    ; CHECK-DAG: $d0 = COPY [[V1]](s64)
+    ; CHECK-DAG: BL &exp
+    ; CHECK-DAG: [[ELT1:%[0-9]+]]:_(s64) = COPY $d0
+    ; CHECK-DAG: ADJCALLSTACKUP
+
+    ; CHECK-DAG: ADJCALLSTACKDOWN
+    ; CHECK-DAG: $d0 = COPY [[V2]](s64)
+    ; CHECK-DAG: BL &exp
+    ; CHECK-DAG: [[ELT2:%[0-9]+]]:_(s64) = COPY $d0
+    ; CHECK-DAG: ADJCALLSTACKUP
+
+    ; CHECK-DAG: %1:_(<2 x s64>) = G_BUILD_VECTOR [[ELT1]](s64), [[ELT2]](s64)
+
+    %0:_(<2 x s64>) = COPY $q0
+    %1:_(<2 x s64>) = G_FEXP %0
+    $q0 = COPY %1(<2 x s64>)
+    RET_ReallyLR implicit $q0
+
+...
+---
+name:            test_exp_half
+alignment:       2
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+body:             |
+  bb.0:
+    liveins: $h0
+    ; CHECK-LABEL: name:            test_exp_half
+    ; CHECK: [[REG1:%[0-9]+]]:_(s32) = G_FPEXT %0(s16)
+    ; CHECK-NEXT: ADJCALLSTACKDOWN
+    ; CHECK-NEXT: $s0 = COPY [[REG1]](s32)
+    ; CHECK-NEXT: BL &expf
+    ; CHECK-NEXT: [[REG2:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-NEXT: ADJCALLSTACKUP
+    ; CHECK-NEXT: [[RES:%[0-9]+]]:_(s16) = G_FPTRUNC [[REG2]](s32)
+
+    %0:_(s16) = COPY $h0
+    %1:_(s16) = G_FEXP %0
+    $h0 = COPY %1(s16)
+    RET_ReallyLR implicit $h0
diff --git a/test/CodeGen/AArch64/GlobalISel/legalize-ext-cse.mir b/test/CodeGen/AArch64/GlobalISel/legalize-ext-cse.mir
new file mode 100644
index 0000000000000000000000000000000000000000..800ac8d33a0392741218c614c9b7651ae760b0f5
--- /dev/null
+++ b/test/CodeGen/AArch64/GlobalISel/legalize-ext-cse.mir
@@ -0,0 +1,21 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -march=aarch64 -run-pass=legalizer %s -o - -enable-cse-in-legalizer=1 | FileCheck %s
+---
+name:            test_cse_in_legalizer
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: test_cse_in_legalizer
+    ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY $x0
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; CHECK: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
+    ; CHECK: [[AND:%[0-9]+]]:_(s32) = G_AND [[TRUNC]], [[C]]
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY [[AND]](s32)
+    ; CHECK: $w0 = COPY [[COPY1]](s32)
+    ; CHECK: $w0 = COPY [[AND]](s32)
+    %0:_(s64) = COPY $x0
+    %1:_(s8) = G_TRUNC %0(s64)
+    %19:_(s32) = G_ZEXT %1(s8)
+    $w0 = COPY %19(s32)
+    %2:_(s8) = G_TRUNC %0(s64)
+    %20:_(s32) = G_ZEXT %2(s8)
+    $w0 = COPY %20(s32)
diff --git a/test/CodeGen/AArch64/GlobalISel/legalize-ext-csedebug-output.mir b/test/CodeGen/AArch64/GlobalISel/legalize-ext-csedebug-output.mir
new file mode 100644
index 0000000000000000000000000000000000000000..00db342a1edee5f7ec22bc23c147ede0f7eb8bf3
--- /dev/null
+++ b/test/CodeGen/AArch64/GlobalISel/legalize-ext-csedebug-output.mir
@@ -0,0 +1,31 @@
+# RUN: llc -march=aarch64 -run-pass=legalizer %s -o - -enable-cse-in-legalizer=1 -debug-only=cseinfo 2>&1 | FileCheck %s
+# Test the -debug-only=cseinfo option here to make sure we're recording new instructions and printing the hit count
+# at the end.
+# REQUIRES: asserts
+---
+name:            test_cse_in_legalizer
+body:             |
+  bb.0.entry:
+    ; CHECK: CSEInfo::Add MI: %{{[0-9]+}}:_(s8) = G_TRUNC
+    ; CHECK: CSEInfo::Add MI: %{{[0-9]+}}:_(s32) = G_ZEXT
+    ; CHECK: CSEInfo::Add MI: %{{[0-9]+}}:_(s8) = G_TRUNC
+    ; CHECK: CSEInfo::Add MI: %{{[0-9]+}}:_(s32) = G_ZEXT
+    ; CHECK: CSEInfo::Recording new MI G_CONSTANT
+    ; CHECK: CSEInfo::Recording new MI G_CONSTANT
+    ; CHECK: CSEInfo::Recording new MI G_TRUNC
+    ; CHECK: CSEInfo::Recording new MI G_TRUNC
+    ; CHECK: CSEInfo::Recording new MI G_AND
+    ; CHECK: CSEInfo::Recording new MI G_AND
+    ; CHECK: CSEInfo::Found Instr %{{[0-9]+}}:_(s32) = G_CONSTANT
+    ; CHECK: CSEInfo::Found Instr %{{[0-9]+}}:_(s32) = G_TRUNC
+    ; CHECK: CSEInfo::Found Instr %{{[0-9]+}}:_(s32) = G_AND
+    ; CHECK: CSEInfo::CSE Hit for Opc {{[0-9]+}} : 1
+    ; CHECK: CSEInfo::CSE Hit for Opc {{[0-9]+}} : 1
+    ; CHECK: CSEInfo::CSE Hit for Opc {{[0-9]+}} : 1
+    %0:_(s64) = COPY $x0
+    %1:_(s8) = G_TRUNC %0(s64)
+    %19:_(s32) = G_ZEXT %1(s8)
+    $w0 = COPY %19(s32)
+    %2:_(s8) = G_TRUNC %0(s64)
+    %20:_(s32) = G_ZEXT %2(s8)
+    $w0 = COPY %20(s32)
diff --git a/test/CodeGen/AArch64/GlobalISel/legalize-ext.mir b/test/CodeGen/AArch64/GlobalISel/legalize-ext.mir
index a8926832e096ca586fb2a631d4197c955cbca50a..09ccc28507b281b3f173edfc077fb72320e2c399 100644
--- a/test/CodeGen/AArch64/GlobalISel/legalize-ext.mir
+++ b/test/CodeGen/AArch64/GlobalISel/legalize-ext.mir
@@ -1,5 +1,5 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -march=aarch64 -run-pass=legalizer %s -o - | FileCheck %s
+# RUN: llc -O0 -march=aarch64 -run-pass=legalizer %s -o - | FileCheck %s
 ---
 name:            test_ext
 body:             |
diff --git a/test/CodeGen/AArch64/GlobalISel/legalize-extracts.mir b/test/CodeGen/AArch64/GlobalISel/legalize-extracts.mir
index 3444254d070db65f26b76d274d33c32dd3c1e785..86eea78edd94ebcdfea4ef012c58660cd1f439d5 100644
--- a/test/CodeGen/AArch64/GlobalISel/legalize-extracts.mir
+++ b/test/CodeGen/AArch64/GlobalISel/legalize-extracts.mir
@@ -11,13 +11,15 @@ body: |
     ; value stored is forwarded directly from first load.
 
     ; CHECK-LABEL: name: test_extracts_1
-    ; CHECK: [[COPY:%[0-9]+]]:_(p0) = COPY $x2
-    ; CHECK: [[LOAD:%[0-9]+]]:_(s64) = G_LOAD [[COPY]](p0) :: (load 8, align 16)
+    ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY $x0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $w1
+    ; CHECK: [[COPY2:%[0-9]+]]:_(p0) = COPY $x2
+    ; CHECK: [[LOAD:%[0-9]+]]:_(s64) = G_LOAD [[COPY2]](p0) :: (load 8, align 16)
     ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
-    ; CHECK: [[GEP:%[0-9]+]]:_(p0) = G_GEP [[COPY]], [[C]](s64)
+    ; CHECK: [[GEP:%[0-9]+]]:_(p0) = G_GEP [[COPY2]], [[C]](s64)
     ; CHECK: [[LOAD1:%[0-9]+]]:_(s64) = G_LOAD [[GEP]](p0) :: (load 8)
-    ; CHECK: [[COPY1:%[0-9]+]]:_(s64) = COPY [[LOAD]](s64)
-    ; CHECK: G_STORE [[COPY1]](s64), [[COPY]](p0) :: (store 8)
+    ; CHECK: [[COPY3:%[0-9]+]]:_(s64) = COPY [[LOAD]](s64)
+    ; CHECK: G_STORE [[COPY3]](s64), [[COPY2]](p0) :: (store 8)
     ; CHECK: RET_ReallyLR
     %0:_(s64) = COPY $x0
     %1:_(s32) = COPY $w1
@@ -36,16 +38,17 @@ body: |
 
       ; Low extraction wipes takes whole low register. High extraction is real.
     ; CHECK-LABEL: name: test_extracts_2
-    ; CHECK: [[COPY:%[0-9]+]]:_(p0) = COPY $x2
-    ; CHECK: [[LOAD:%[0-9]+]]:_(s64) = G_LOAD [[COPY]](p0) :: (load 8, align 16)
+    ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY $x0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $w1
+    ; CHECK: [[COPY2:%[0-9]+]]:_(p0) = COPY $x2
+    ; CHECK: [[LOAD:%[0-9]+]]:_(s64) = G_LOAD [[COPY2]](p0) :: (load 8, align 16)
     ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
-    ; CHECK: [[GEP:%[0-9]+]]:_(p0) = G_GEP [[COPY]], [[C]](s64)
+    ; CHECK: [[GEP:%[0-9]+]]:_(p0) = G_GEP [[COPY2]], [[C]](s64)
     ; CHECK: [[LOAD1:%[0-9]+]]:_(s64) = G_LOAD [[GEP]](p0) :: (load 8)
-    ; CHECK: [[COPY1:%[0-9]+]]:_(s64) = COPY [[LOAD]](s64)
+    ; CHECK: [[COPY3:%[0-9]+]]:_(s64) = COPY [[LOAD]](s64)
     ; CHECK: [[EXTRACT:%[0-9]+]]:_(s32) = G_EXTRACT [[LOAD1]](s64), 0
-    ; CHECK: [[COPY2:%[0-9]+]]:_(s32) = COPY [[EXTRACT]](s32)
-    ; CHECK: G_STORE [[COPY1]](s64), [[COPY]](p0) :: (store 8)
-    ; CHECK: G_STORE [[COPY2]](s32), [[COPY]](p0) :: (store 4)
+    ; CHECK: G_STORE [[COPY3]](s64), [[COPY2]](p0) :: (store 8)
+    ; CHECK: G_STORE [[EXTRACT]](s32), [[COPY2]](p0) :: (store 4)
     ; CHECK: RET_ReallyLR
     %0:_(s64) = COPY $x0
     %1:_(s32) = COPY $w1
@@ -90,9 +93,9 @@ body: |
 
     ; CHECK-LABEL: name: test_extracts_4
     ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY $x0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s64) = COPY $x1
     ; CHECK: [[EXTRACT:%[0-9]+]]:_(s32) = G_EXTRACT [[COPY]](s64), 32
-    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY [[EXTRACT]](s32)
-    ; CHECK: $w0 = COPY [[COPY1]](s32)
+    ; CHECK: $w0 = COPY [[EXTRACT]](s32)
     ; CHECK: RET_ReallyLR
     %0:_(s64) = COPY $x0
     %1:_(s64) = COPY $x1
diff --git a/test/CodeGen/AArch64/GlobalISel/legalize-fp-arith.mir b/test/CodeGen/AArch64/GlobalISel/legalize-fp-arith.mir
new file mode 100644
index 0000000000000000000000000000000000000000..a0be636dfe6efa5a60b25c675fc1d1e9ca20d426
--- /dev/null
+++ b/test/CodeGen/AArch64/GlobalISel/legalize-fp-arith.mir
@@ -0,0 +1,75 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -march=aarch64 -run-pass=legalizer %s -o - | FileCheck %s
+---
+name:            test_fadd_v2s64
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: test_fadd_v2s64
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s64>) = COPY $q0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(<2 x s64>) = COPY $q1
+    ; CHECK: [[FADD:%[0-9]+]]:_(<2 x s64>) = G_FADD [[COPY]], [[COPY1]]
+    ; CHECK: $q0 = COPY [[FADD]](<2 x s64>)
+    %0:_(<2 x s64>) = COPY $q0
+    %1:_(<2 x s64>) = COPY $q1
+    %2:_(<2 x s64>) = G_FADD %0, %1
+    $q0 = COPY %2(<2 x s64>)
+
+...
+---
+name:            test_fdiv_v2s32
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: test_fdiv_v2s32
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $d0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $d1
+    ; CHECK: [[FDIV:%[0-9]+]]:_(<2 x s32>) = G_FDIV [[COPY]], [[COPY1]]
+    ; CHECK: $d0 = COPY [[FDIV]](<2 x s32>)
+    %0:_(<2 x s32>) = COPY $d0
+    %1:_(<2 x s32>) = COPY $d1
+    %2:_(<2 x s32>) = G_FDIV %0, %1
+    $d0 = COPY %2(<2 x s32>)
+
+...
+---
+name:            test_fsub_v2s32
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: test_fsub_v2s32
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $d0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $d1
+    ; CHECK: [[FSUB:%[0-9]+]]:_(<2 x s32>) = G_FSUB [[COPY]], [[COPY1]]
+    ; CHECK: $d0 = COPY [[FSUB]](<2 x s32>)
+    %0:_(<2 x s32>) = COPY $d0
+    %1:_(<2 x s32>) = COPY $d1
+    %2:_(<2 x s32>) = G_FSUB %0, %1
+    $d0 = COPY %2(<2 x s32>)
+
+...
+---
+name:            test_fneg_v2s32
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: test_fneg_v2s32
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $d0
+    ; CHECK: [[FNEG:%[0-9]+]]:_(<2 x s32>) = G_FNEG [[COPY]]
+    ; CHECK: $d0 = COPY [[FNEG]](<2 x s32>)
+    %0:_(<2 x s32>) = COPY $d0
+    %1:_(<2 x s32>) = G_FNEG %0
+    $d0 = COPY %1(<2 x s32>)
+
+...
+---
+name:            test_fmul_v4s32
+body:             |
+  bb.0.entry:
+    ; CHECK-LABEL: name: test_fmul_v4s32
+    ; CHECK: [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $q0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(<4 x s32>) = COPY $q1
+    ; CHECK: [[FMUL:%[0-9]+]]:_(<4 x s32>) = G_FMUL [[COPY]], [[COPY1]]
+    ; CHECK: $q0 = COPY [[FMUL]](<4 x s32>)
+    %0:_(<4 x s32>) = COPY $q0
+    %1:_(<4 x s32>) = COPY $q1
+    %2:_(<4 x s32>) = G_FMUL %0, %1
+    $q0 = COPY %2(<4 x s32>)
+
+...
diff --git a/test/CodeGen/AArch64/GlobalISel/legalize-fptoi.mir b/test/CodeGen/AArch64/GlobalISel/legalize-fptoi.mir
index d20016ced5745aa55b369d3a13b649263daf2259..9bc639679bea17eeb78c410589f4e9079a79685b 100644
--- a/test/CodeGen/AArch64/GlobalISel/legalize-fptoi.mir
+++ b/test/CodeGen/AArch64/GlobalISel/legalize-fptoi.mir
@@ -23,6 +23,9 @@
 
   define void @test_fptosi_s16_s32() { ret void }
   define void @test_fptoui_s16_s32() { ret void }
+
+  define void @test_fptoui_v4s32() { ret void }
+  define void @test_fptosi_v4s32() { ret void }
 ...
 
 ---
@@ -234,3 +237,31 @@ body: |
     %2:_(s32) = G_ANYEXT %1
     $w0 = COPY %2
 ...
+
+---
+name:            test_fptoui_v4s32
+body: |
+  bb.0:
+    liveins: $q0
+    ; CHECK-LABEL: name: test_fptoui_v4s32
+    ; CHECK: [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $q0
+    ; CHECK: [[FPTOUI:%[0-9]+]]:_(<4 x s32>) = G_FPTOUI [[COPY]](<4 x s32>)
+    ; CHECK: $q0 = COPY [[FPTOUI]](<4 x s32>)
+    %0:_(<4 x s32>) = COPY $q0
+    %1:_(<4 x s32>) = G_FPTOUI %0
+    $q0 = COPY %1
+...
+
+---
+name:            test_fptosi_v4s32
+body: |
+  bb.0:
+    liveins: $q0
+    ; CHECK-LABEL: name: test_fptosi_v4s32
+    ; CHECK: [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $q0
+    ; CHECK: [[FPTOSI:%[0-9]+]]:_(<4 x s32>) = G_FPTOSI [[COPY]](<4 x s32>)
+    ; CHECK: $q0 = COPY [[FPTOSI]](<4 x s32>)
+    %0:_(<4 x s32>) = COPY $q0
+    %1:_(<4 x s32>) = G_FPTOSI %0
+    $q0 = COPY %1
+...
diff --git a/test/CodeGen/AArch64/GlobalISel/legalize-inttoptr-xfail-1.mir b/test/CodeGen/AArch64/GlobalISel/legalize-inttoptr-xfail-1.mir
index 32c4ed82bb37db834bc719fa727064748e54071e..fb0bc27f8e1a94d10a199f2817dbda6d0d29c843 100644
--- a/test/CodeGen/AArch64/GlobalISel/legalize-inttoptr-xfail-1.mir
+++ b/test/CodeGen/AArch64/GlobalISel/legalize-inttoptr-xfail-1.mir
@@ -15,9 +15,9 @@
 # and fix the mistake: check that type index 0 is p0 and type index 1
 # is s64 (in that order).
 
-# CHECK:      LLVM ERROR: unable to legalize instruction:
-# CHECK-SAME: %{{[0-9]+}}:_(s64) = G_INTTOPTR %{{[0-9]+}}:_(p0)
-# CHECK-SAME: (in function: broken)
+# CHECK:  Bad machine code: inttoptr result type must be a pointer
+# CHECK: Bad machine code: inttoptr source type must not be a pointer
+# CHECK: LLVM ERROR: Found 2 machine code errors.
 
 ---
 name:            broken
diff --git a/test/CodeGen/AArch64/GlobalISel/legalize-inttoptr-xfail-2.mir b/test/CodeGen/AArch64/GlobalISel/legalize-inttoptr-xfail-2.mir
index 3584bfd12e664eb0c5c52a0d8b1f18a13d10a513..3956f3a6c351945a2824bfe7898ce0048390261a 100644
--- a/test/CodeGen/AArch64/GlobalISel/legalize-inttoptr-xfail-2.mir
+++ b/test/CodeGen/AArch64/GlobalISel/legalize-inttoptr-xfail-2.mir
@@ -19,9 +19,8 @@
 # and fix the mistake: check that type index 0 is p0 and type index 1
 # is s64.
 
-# CHECK:      LLVM ERROR: unable to legalize instruction:
-# CHECK-SAME: %{{[0-9]+}}:_(p0) = G_INTTOPTR %{{[0-9]+}}:_(<4 x s16>)
-# CHECK-SAME: (in function: broken)
+# CHECK: Bad machine code: operand types must be all-vector or all-scalar
+# CHECK: LLVM ERROR: Found 1 machine code errors.
 
 ---
 name:            broken
diff --git a/test/CodeGen/AArch64/GlobalISel/legalize-itofp.mir b/test/CodeGen/AArch64/GlobalISel/legalize-itofp.mir
index 9539c54c5b09190f9b923f8a473b0fceaa092ac3..48a36566dfa9c4411539a40948ae936f3e83ceb1 100644
--- a/test/CodeGen/AArch64/GlobalISel/legalize-itofp.mir
+++ b/test/CodeGen/AArch64/GlobalISel/legalize-itofp.mir
@@ -21,6 +21,9 @@
   define void @test_sitofp_s64_s8() { ret void }
   define void @test_uitofp_s64_s8() { ret void }
 
+  define void @test_sitofp_v4s32() { ret void }
+  define void @test_uitofp_v4s32() { ret void }
+
   define void @test_sitofp_s32_s16() { ret void }
   define void @test_uitofp_s32_s16() { ret void }
 ...
@@ -33,6 +36,7 @@ body: |
     ; CHECK-LABEL: name: test_sitofp_s32_s32
     ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $w0
     ; CHECK: [[SITOFP:%[0-9]+]]:_(s32) = G_SITOFP [[COPY]](s32)
+    ; CHECK: $w0 = COPY [[SITOFP]](s32)
     %0:_(s32) = COPY $w0
     %1:_(s32) = G_SITOFP %0
     $w0 = COPY %1
@@ -46,6 +50,7 @@ body: |
     ; CHECK-LABEL: name: test_uitofp_s32_s32
     ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $w0
     ; CHECK: [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[COPY]](s32)
+    ; CHECK: $w0 = COPY [[UITOFP]](s32)
     %0:_(s32) = COPY $w0
     %1:_(s32) = G_UITOFP %0
     $w0 = COPY %1
@@ -59,6 +64,7 @@ body: |
     ; CHECK-LABEL: name: test_sitofp_s32_s64
     ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY $x0
     ; CHECK: [[SITOFP:%[0-9]+]]:_(s32) = G_SITOFP [[COPY]](s64)
+    ; CHECK: $w0 = COPY [[SITOFP]](s32)
     %0:_(s64) = COPY $x0
     %1:_(s32) = G_SITOFP %0
     $w0 = COPY %1
@@ -72,6 +78,7 @@ body: |
     ; CHECK-LABEL: name: test_uitofp_s32_s64
     ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY $x0
     ; CHECK: [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[COPY]](s64)
+    ; CHECK: $w0 = COPY [[UITOFP]](s32)
     %0:_(s64) = COPY $x0
     %1:_(s32) = G_UITOFP %0
     $w0 = COPY %1
@@ -85,6 +92,7 @@ body: |
     ; CHECK-LABEL: name: test_sitofp_s64_s32
     ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $w0
     ; CHECK: [[SITOFP:%[0-9]+]]:_(s64) = G_SITOFP [[COPY]](s32)
+    ; CHECK: $x0 = COPY [[SITOFP]](s64)
     %0:_(s32) = COPY $w0
     %1:_(s64) = G_SITOFP %0
     $x0 = COPY %1
@@ -98,6 +106,7 @@ body: |
     ; CHECK-LABEL: name: test_uitofp_s64_s32
     ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $w0
     ; CHECK: [[UITOFP:%[0-9]+]]:_(s64) = G_UITOFP [[COPY]](s32)
+    ; CHECK: $x0 = COPY [[UITOFP]](s64)
     %0:_(s32) = COPY $w0
     %1:_(s64) = G_UITOFP %0
     $x0 = COPY %1
@@ -111,6 +120,7 @@ body: |
     ; CHECK-LABEL: name: test_sitofp_s64_s64
     ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY $x0
     ; CHECK: [[SITOFP:%[0-9]+]]:_(s64) = G_SITOFP [[COPY]](s64)
+    ; CHECK: $x0 = COPY [[SITOFP]](s64)
     %0:_(s64) = COPY $x0
     %1:_(s64) = G_SITOFP %0
     $x0 = COPY %1
@@ -124,6 +134,7 @@ body: |
     ; CHECK-LABEL: name: test_uitofp_s64_s64
     ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY $x0
     ; CHECK: [[UITOFP:%[0-9]+]]:_(s64) = G_UITOFP [[COPY]](s64)
+    ; CHECK: $x0 = COPY [[UITOFP]](s64)
     %0:_(s64) = COPY $x0
     %1:_(s64) = G_UITOFP %0
     $x0 = COPY %1
@@ -139,9 +150,10 @@ body: |
     ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $w0
     ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 31
     ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
-    ; CHECK: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY1]], [[C]]
-    ; CHECK: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C]]
+    ; CHECK: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY1]], [[C]](s32)
+    ; CHECK: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C]](s32)
     ; CHECK: [[SITOFP:%[0-9]+]]:_(s32) = G_SITOFP [[ASHR]](s32)
+    ; CHECK: $w0 = COPY [[SITOFP]](s32)
     %0:_(s32) = COPY $w0
     %1:_(s1) = G_TRUNC %0
     %2:_(s32) = G_SITOFP %1
@@ -159,6 +171,7 @@ body: |
     ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
     ; CHECK: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
     ; CHECK: [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[AND]](s32)
+    ; CHECK: $w0 = COPY [[UITOFP]](s32)
     %0:_(s32) = COPY $w0
     %1:_(s1) = G_TRUNC %0
     %2:_(s32) = G_UITOFP %1
@@ -174,9 +187,10 @@ body: |
     ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $w0
     ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
     ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
-    ; CHECK: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY1]], [[C]]
-    ; CHECK: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C]]
+    ; CHECK: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY1]], [[C]](s32)
+    ; CHECK: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C]](s32)
     ; CHECK: [[SITOFP:%[0-9]+]]:_(s64) = G_SITOFP [[ASHR]](s32)
+    ; CHECK: $x0 = COPY [[SITOFP]](s64)
     %0:_(s32) = COPY $w0
     %1:_(s8) = G_TRUNC %0
     %2:_(s64) = G_SITOFP %1
@@ -194,12 +208,41 @@ body: |
     ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
     ; CHECK: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
     ; CHECK: [[UITOFP:%[0-9]+]]:_(s64) = G_UITOFP [[AND]](s32)
+    ; CHECK: $x0 = COPY [[UITOFP]](s64)
     %0:_(s32) = COPY $w0
     %1:_(s8) = G_TRUNC %0
     %2:_(s64) = G_UITOFP %1
     $x0 = COPY %2
 ...
 
+---
+name:            test_sitofp_v4s32
+body: |
+  bb.0:
+    liveins: $q0
+    ; CHECK-LABEL: name: test_sitofp_v4s32
+    ; CHECK: [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $q0
+    ; CHECK: [[SITOFP:%[0-9]+]]:_(<4 x s32>) = G_SITOFP [[SITOFP]](<4 x s32>)
+    ; CHECK: $q0 = COPY [[SITOFP]](<4 x s32>)
+    %0:_(<4 x s32>) = COPY $q0
+    %1:_(<4 x s32>) = G_SITOFP %1
+    $q0 = COPY %1
+...
+
+---
+name:            test_uitofp_v4s32
+body: |
+  bb.0:
+    liveins: $q0
+    ; CHECK-LABEL: name: test_uitofp_v4s32
+    ; CHECK: [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $q0
+    ; CHECK: [[UITOFP:%[0-9]+]]:_(<4 x s32>) = G_UITOFP [[UITOFP]](<4 x s32>)
+    ; CHECK: $q0 = COPY [[UITOFP]](<4 x s32>)
+    %0:_(<4 x s32>) = COPY $q0
+    %1:_(<4 x s32>) = G_UITOFP %1
+    $q0 = COPY %1
+...
+
 ---
 name:            test_sitofp_s32_s16
 body: |
@@ -209,9 +252,10 @@ body: |
     ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $w0
     ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
     ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
-    ; CHECK: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY1]], [[C]]
-    ; CHECK: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C]]
+    ; CHECK: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY1]], [[C]](s32)
+    ; CHECK: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C]](s32)
     ; CHECK: [[SITOFP:%[0-9]+]]:_(s32) = G_SITOFP [[ASHR]](s32)
+    ; CHECK: $w0 = COPY [[SITOFP]](s32)
     %0:_(s32) = COPY $w0
     %1:_(s16) = G_TRUNC %0
     %2:_(s32) = G_SITOFP %1
@@ -229,6 +273,7 @@ body: |
     ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
     ; CHECK: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
     ; CHECK: [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[AND]](s32)
+    ; CHECK: $w0 = COPY [[UITOFP]](s32)
     %0:_(s32) = COPY $w0
     %1:_(s16) = G_TRUNC %0
     %2:_(s32) = G_UITOFP %1
diff --git a/test/CodeGen/AArch64/GlobalISel/legalize-load-store-fewerElts.mir b/test/CodeGen/AArch64/GlobalISel/legalize-load-store-fewerElts.mir
index 27791881ef3b18cb09010e8e914d6ad2e1776a23..33ec198d37e6a562412cb7961f2e129c0df5e643 100644
--- a/test/CodeGen/AArch64/GlobalISel/legalize-load-store-fewerElts.mir
+++ b/test/CodeGen/AArch64/GlobalISel/legalize-load-store-fewerElts.mir
@@ -1,5 +1,5 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -march=aarch64 -o - -run-pass=legalizer %s | FileCheck %s
+# RUN: llc -O0 -march=aarch64 -o - -run-pass=legalizer %s | FileCheck %s
 ---
 name:            load_v4s32
 legalized:       false
diff --git a/test/CodeGen/AArch64/GlobalISel/legalize-load-store-s128-unaligned.mir b/test/CodeGen/AArch64/GlobalISel/legalize-load-store-s128-unaligned.mir
index 33a6c23eb36786a07d3b1f56d7f558257b1a092b..dc05f6a511a712c14074f77c879fe27294f7675a 100644
--- a/test/CodeGen/AArch64/GlobalISel/legalize-load-store-s128-unaligned.mir
+++ b/test/CodeGen/AArch64/GlobalISel/legalize-load-store-s128-unaligned.mir
@@ -1,5 +1,5 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -march=aarch64 -o - -run-pass=legalizer %s | FileCheck %s
+# RUN: llc -O0 -march=aarch64 -o - -run-pass=legalizer %s | FileCheck %s
 ---
 name:            loadstore128_align4
 exposesReturnsTwice: false
diff --git a/test/CodeGen/AArch64/GlobalISel/legalize-load-store.mir b/test/CodeGen/AArch64/GlobalISel/legalize-load-store.mir
index 7a41cb0cd797dc1e156f84b217fd73e06d5268bc..25487402edb2cb5afdcafcc17d7178b20be2e423 100644
--- a/test/CodeGen/AArch64/GlobalISel/legalize-load-store.mir
+++ b/test/CodeGen/AArch64/GlobalISel/legalize-load-store.mir
@@ -1,5 +1,5 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -march=aarch64 -run-pass=legalizer %s -o - | FileCheck %s
+# RUN: llc -O0 -march=aarch64 -run-pass=legalizer %s -o - | FileCheck %s
 
 ---
 name:            test_load
diff --git a/test/CodeGen/AArch64/GlobalISel/legalize-log.mir b/test/CodeGen/AArch64/GlobalISel/legalize-log.mir
new file mode 100644
index 0000000000000000000000000000000000000000..c70f48b30cabcaa42400a3b23e744d6374633e04
--- /dev/null
+++ b/test/CodeGen/AArch64/GlobalISel/legalize-log.mir
@@ -0,0 +1,227 @@
+# RUN: llc -verify-machineinstrs -mtriple aarch64--- \
+# RUN: -run-pass=legalizer -mattr=+fullfp16 -global-isel %s -o - \
+# RUN: | FileCheck %s
+...
+---
+name:            test_v4f16.log
+alignment:       2
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+body:             |
+  bb.0:
+    liveins: $d0
+    ; CHECK-LABEL: name:            test_v4f16.log
+    ; CHECK: [[V1:%[0-9]+]]:_(s16), [[V2:%[0-9]+]]:_(s16), [[V3:%[0-9]+]]:_(s16), [[V4:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES %{{[0-9]+}}(<4 x s16>)
+
+    ; CHECK-DAG: [[V1_S32:%[0-9]+]]:_(s32) = G_FPEXT [[V1]](s16)
+    ; CHECK-NEXT: ADJCALLSTACKDOWN
+    ; CHECK-NEXT: $s0 = COPY [[V1_S32]](s32)
+    ; CHECK-NEXT: BL &logf
+    ; CHECK-NEXT: [[ELT1_S32:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-NEXT: ADJCALLSTACKUP
+    ; CHECK-NEXT: [[ELT1:%[0-9]+]]:_(s16) = G_FPTRUNC [[ELT1_S32]](s32)
+
+    ; CHECK-DAG: [[V2_S32:%[0-9]+]]:_(s32) = G_FPEXT [[V2]](s16)
+    ; CHECK-NEXT: ADJCALLSTACKDOWN
+    ; CHECK-NEXT: $s0 = COPY [[V2_S32]](s32)
+    ; CHECK-NEXT: BL &logf
+    ; CHECK-NEXT: [[ELT2_S32:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-NEXT: ADJCALLSTACKUP
+    ; CHECK-NEXT: [[ELT2:%[0-9]+]]:_(s16) = G_FPTRUNC [[ELT2_S32]](s32)
+
+    ; CHECK-DAG: [[V3_S32:%[0-9]+]]:_(s32) = G_FPEXT [[V3]](s16)
+    ; CHECK-NEXT: ADJCALLSTACKDOWN
+    ; CHECK-NEXT: $s0 = COPY [[V3_S32]](s32)
+    ; CHECK-NEXT: BL &logf
+    ; CHECK-NEXT: [[ELT3_S32:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-NEXT: ADJCALLSTACKUP
+    ; CHECK-NEXT: [[ELT3:%[0-9]+]]:_(s16) = G_FPTRUNC [[ELT3_S32]](s32)
+
+    ; CHECK-DAG: [[V4_S32:%[0-9]+]]:_(s32) = G_FPEXT [[V4]](s16)
+    ; CHECK-NEXT: ADJCALLSTACKDOWN
+    ; CHECK-NEXT: $s0 = COPY [[V4_S32]](s32)
+    ; CHECK-NEXT: BL &logf
+    ; CHECK-NEXT: [[ELT4_S32:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-NEXT: ADJCALLSTACKUP
+    ; CHECK-NEXT: [[ELT4:%[0-9]+]]:_(s16) = G_FPTRUNC [[ELT4_S32]](s32)
+
+    ; CHECK-DAG: %{{[0-9]+}}:_(<4 x s16>) = G_BUILD_VECTOR [[ELT1]](s16), [[ELT2]](s16), [[ELT3]](s16), [[ELT4]](s16)
+
+    %0:_(<4 x s16>) = COPY $d0
+    %1:_(<4 x s16>) = G_FLOG %0
+    $d0 = COPY %1(<4 x s16>)
+    RET_ReallyLR implicit $d0
+
+...
+---
+name:            test_v8f16.log
+alignment:       2
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+body:             |
+  bb.0:
+    liveins: $q0
+
+    ; CHECK-LABEL: name:            test_v8f16.log
+
+    ; This is big, so let's just check for the 8 calls to logf, the the
+    ; G_UNMERGE_VALUES, and the G_BUILD_VECTOR. The other instructions ought
+    ; to be covered by the other tests.
+
+    ; CHECK: G_UNMERGE_VALUES
+    ; CHECK: BL &logf
+    ; CHECK: BL &logf
+    ; CHECK: BL &logf
+    ; CHECK: BL &logf
+    ; CHECK: BL &logf
+    ; CHECK: BL &logf
+    ; CHECK: BL &logf
+    ; CHECK: BL &logf
+    ; CHECK-DAG: G_BUILD_VECTOR
+
+    %0:_(<8 x s16>) = COPY $q0
+    %1:_(<8 x s16>) = G_FLOG %0
+    $q0 = COPY %1(<8 x s16>)
+    RET_ReallyLR implicit $q0
+
+...
+---
+name:            test_v2f32.log
+alignment:       2
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+body:             |
+  bb.0:
+    liveins: $d0
+
+    ; CHECK-LABEL: name:            test_v2f32.log
+    ; CHECK: [[V1:%[0-9]+]]:_(s32), [[V2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES %{{[0-9]+}}(<2 x s32>)
+
+    ; CHECK-NEXT: ADJCALLSTACKDOWN
+    ; CHECK-DAG: $s0 = COPY [[V1]](s32)
+    ; CHECK-DAG: BL &logf
+    ; CHECK-DAG: [[ELT1:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-DAG: ADJCALLSTACKUP
+
+    ; CHECK-DAG: ADJCALLSTACKDOWN
+    ; CHECK-DAG: $s0 = COPY [[V2]](s32)
+    ; CHECK-DAG: BL &logf
+    ; CHECK-DAG: [[ELT2:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-DAG: ADJCALLSTACKUP
+
+    ; CHECK-DAG: %1:_(<2 x s32>) = G_BUILD_VECTOR [[ELT1]](s32), [[ELT2]](s32)
+
+    %0:_(<2 x s32>) = COPY $d0
+    %1:_(<2 x s32>) = G_FLOG %0
+    $d0 = COPY %1(<2 x s32>)
+    RET_ReallyLR implicit $d0
+
+...
+---
+name:            test_v4f32.log
+alignment:       2
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+body:             |
+  bb.0:
+    liveins: $q0
+    ; CHECK-LABEL: name:            test_v4f32.log
+    ; CHECK: [[V1:%[0-9]+]]:_(s32), [[V2:%[0-9]+]]:_(s32), [[V3:%[0-9]+]]:_(s32), [[V4:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES %{{[0-9]+}}(<4 x s32>)
+
+    ; CHECK-NEXT: ADJCALLSTACKDOWN
+    ; CHECK-DAG: $s0 = COPY [[V1]](s32)
+    ; CHECK-DAG: BL &logf
+    ; CHECK-DAG: [[ELT1:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-DAG: ADJCALLSTACKUP
+
+    ; CHECK-DAG: ADJCALLSTACKDOWN
+    ; CHECK-DAG: $s0 = COPY [[V2]](s32)
+    ; CHECK-DAG: BL &logf
+    ; CHECK-DAG: [[ELT2:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-DAG: ADJCALLSTACKUP
+
+    ; CHECK-DAG: ADJCALLSTACKDOWN
+    ; CHECK-DAG: $s0 = COPY [[V3]](s32)
+    ; CHECK-DAG: BL &logf
+    ; CHECK-DAG: [[ELT3:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-DAG: ADJCALLSTACKUP
+
+    ; CHECK-DAG: ADJCALLSTACKDOWN
+    ; CHECK-DAG: $s0 = COPY [[V4]](s32)
+    ; CHECK-DAG: BL &logf
+    ; CHECK-DAG: [[ELT4:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-DAG: ADJCALLSTACKUP
+
+    ; CHECK-DAG: %1:_(<4 x s32>) = G_BUILD_VECTOR [[ELT1]](s32), [[ELT2]](s32), [[ELT3]](s32), [[ELT4]](s32)
+
+    %0:_(<4 x s32>) = COPY $q0
+    %1:_(<4 x s32>) = G_FLOG %0
+    $q0 = COPY %1(<4 x s32>)
+    RET_ReallyLR implicit $q0
+
+...
+---
+name:            test_v2f64.log
+alignment:       2
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+body:             |
+  bb.0:
+    liveins: $q0
+
+    ; CHECK-LABEL: name:            test_v2f64.log
+    ; CHECK: [[V1:%[0-9]+]]:_(s64), [[V2:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES %{{[0-9]+}}(<2 x s64>)
+
+    ; CHECK-NEXT: ADJCALLSTACKDOWN
+    ; CHECK-DAG: $d0 = COPY [[V1]](s64)
+    ; CHECK-DAG: BL &log
+    ; CHECK-DAG: [[ELT1:%[0-9]+]]:_(s64) = COPY $d0
+    ; CHECK-DAG: ADJCALLSTACKUP
+
+    ; CHECK-DAG: ADJCALLSTACKDOWN
+    ; CHECK-DAG: $d0 = COPY [[V2]](s64)
+    ; CHECK-DAG: BL &log
+    ; CHECK-DAG: [[ELT2:%[0-9]+]]:_(s64) = COPY $d0
+    ; CHECK-DAG: ADJCALLSTACKUP
+
+    ; CHECK-DAG: %1:_(<2 x s64>) = G_BUILD_VECTOR [[ELT1]](s64), [[ELT2]](s64)
+
+    %0:_(<2 x s64>) = COPY $q0
+    %1:_(<2 x s64>) = G_FLOG %0
+    $q0 = COPY %1(<2 x s64>)
+    RET_ReallyLR implicit $q0
+
+...
+---
+name:            test_log_half
+alignment:       2
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+body:             |
+  bb.0:
+    liveins: $h0
+    ; CHECK-LABEL: name:            test_log_half
+    ; CHECK: [[REG1:%[0-9]+]]:_(s32) = G_FPEXT %0(s16)
+    ; CHECK-NEXT: ADJCALLSTACKDOWN
+    ; CHECK-NEXT: $s0 = COPY [[REG1]](s32)
+    ; CHECK-NEXT: BL &logf
+    ; CHECK-NEXT: [[REG2:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-NEXT: ADJCALLSTACKUP
+    ; CHECK-NEXT: [[RES:%[0-9]+]]:_(s16) = G_FPTRUNC [[REG2]](s32)
+
+    %0:_(s16) = COPY $h0
+    %1:_(s16) = G_FLOG %0
+    $h0 = COPY %1(s16)
+    RET_ReallyLR implicit $h0
diff --git a/test/CodeGen/AArch64/GlobalISel/legalize-log10.mir b/test/CodeGen/AArch64/GlobalISel/legalize-log10.mir
new file mode 100644
index 0000000000000000000000000000000000000000..08e1a4f53d15b110464d2072d375c3d6d1ee393c
--- /dev/null
+++ b/test/CodeGen/AArch64/GlobalISel/legalize-log10.mir
@@ -0,0 +1,227 @@
+# RUN: llc -verify-machineinstrs -mtriple aarch64--- \
+# RUN: -run-pass=legalizer -mattr=+fullfp16 -global-isel %s -o - \
+# RUN: | FileCheck %s
+...
+---
+name:            test_v4f16.log10
+alignment:       2
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+body:             |
+  bb.0:
+    liveins: $d0
+    ; CHECK-LABEL: name:            test_v4f16.log10
+    ; CHECK: [[V1:%[0-9]+]]:_(s16), [[V2:%[0-9]+]]:_(s16), [[V3:%[0-9]+]]:_(s16), [[V4:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES %{{[0-9]+}}(<4 x s16>)
+
+    ; CHECK-DAG: [[V1_S32:%[0-9]+]]:_(s32) = G_FPEXT [[V1]](s16)
+    ; CHECK-NEXT: ADJCALLSTACKDOWN
+    ; CHECK-NEXT: $s0 = COPY [[V1_S32]](s32)
+    ; CHECK-NEXT: BL &log10
+    ; CHECK-NEXT: [[ELT1_S32:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-NEXT: ADJCALLSTACKUP
+    ; CHECK-NEXT: [[ELT1:%[0-9]+]]:_(s16) = G_FPTRUNC [[ELT1_S32]](s32)
+
+    ; CHECK-DAG: [[V2_S32:%[0-9]+]]:_(s32) = G_FPEXT [[V2]](s16)
+    ; CHECK-NEXT: ADJCALLSTACKDOWN
+    ; CHECK-NEXT: $s0 = COPY [[V2_S32]](s32)
+    ; CHECK-NEXT: BL &log10
+    ; CHECK-NEXT: [[ELT2_S32:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-NEXT: ADJCALLSTACKUP
+    ; CHECK-NEXT: [[ELT2:%[0-9]+]]:_(s16) = G_FPTRUNC [[ELT2_S32]](s32)
+
+    ; CHECK-DAG: [[V3_S32:%[0-9]+]]:_(s32) = G_FPEXT [[V3]](s16)
+    ; CHECK-NEXT: ADJCALLSTACKDOWN
+    ; CHECK-NEXT: $s0 = COPY [[V3_S32]](s32)
+    ; CHECK-NEXT: BL &log10
+    ; CHECK-NEXT: [[ELT3_S32:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-NEXT: ADJCALLSTACKUP
+    ; CHECK-NEXT: [[ELT3:%[0-9]+]]:_(s16) = G_FPTRUNC [[ELT3_S32]](s32)
+
+    ; CHECK-DAG: [[V4_S32:%[0-9]+]]:_(s32) = G_FPEXT [[V4]](s16)
+    ; CHECK-NEXT: ADJCALLSTACKDOWN
+    ; CHECK-NEXT: $s0 = COPY [[V4_S32]](s32)
+    ; CHECK-NEXT: BL &log10
+    ; CHECK-NEXT: [[ELT4_S32:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-NEXT: ADJCALLSTACKUP
+    ; CHECK-NEXT: [[ELT4:%[0-9]+]]:_(s16) = G_FPTRUNC [[ELT4_S32]](s32)
+
+    ; CHECK-DAG: %{{[0-9]+}}:_(<4 x s16>) = G_BUILD_VECTOR [[ELT1]](s16), [[ELT2]](s16), [[ELT3]](s16), [[ELT4]](s16)
+
+    %0:_(<4 x s16>) = COPY $d0
+    %1:_(<4 x s16>) = G_FLOG10 %0
+    $d0 = COPY %1(<4 x s16>)
+    RET_ReallyLR implicit $d0
+
+...
+---
+name:            test_v8f16.log10
+alignment:       2
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+body:             |
+  bb.0:
+    liveins: $q0
+
+    ; CHECK-LABEL: name:            test_v8f16.log10
+
+    ; This is big, so let's just check for the 8 calls to log10, the the
+    ; G_UNMERGE_VALUES, and the G_BUILD_VECTOR. The other instructions ought
+    ; to be covered by the other tests.
+
+    ; CHECK: G_UNMERGE_VALUES
+    ; CHECK: BL &log10
+    ; CHECK: BL &log10
+    ; CHECK: BL &log10
+    ; CHECK: BL &log10
+    ; CHECK: BL &log10
+    ; CHECK: BL &log10
+    ; CHECK: BL &log10
+    ; CHECK: BL &log10
+    ; CHECK-DAG: G_BUILD_VECTOR
+
+    %0:_(<8 x s16>) = COPY $q0
+    %1:_(<8 x s16>) = G_FLOG10 %0
+    $q0 = COPY %1(<8 x s16>)
+    RET_ReallyLR implicit $q0
+
+...
+---
+name:            test_v2f32.log10
+alignment:       2
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+body:             |
+  bb.0:
+    liveins: $d0
+
+    ; CHECK-LABEL: name:            test_v2f32.log10
+    ; CHECK: [[V1:%[0-9]+]]:_(s32), [[V2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES %{{[0-9]+}}(<2 x s32>)
+
+    ; CHECK-NEXT: ADJCALLSTACKDOWN
+    ; CHECK-DAG: $s0 = COPY [[V1]](s32)
+    ; CHECK-DAG: BL &log10
+    ; CHECK-DAG: [[ELT1:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-DAG: ADJCALLSTACKUP
+
+    ; CHECK-DAG: ADJCALLSTACKDOWN
+    ; CHECK-DAG: $s0 = COPY [[V2]](s32)
+    ; CHECK-DAG: BL &log10
+    ; CHECK-DAG: [[ELT2:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-DAG: ADJCALLSTACKUP
+
+    ; CHECK-DAG: %1:_(<2 x s32>) = G_BUILD_VECTOR [[ELT1]](s32), [[ELT2]](s32)
+
+    %0:_(<2 x s32>) = COPY $d0
+    %1:_(<2 x s32>) = G_FLOG10 %0
+    $d0 = COPY %1(<2 x s32>)
+    RET_ReallyLR implicit $d0
+
+...
+---
+name:            test_v4f32.log10
+alignment:       2
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+body:             |
+  bb.0:
+    liveins: $q0
+    ; CHECK-LABEL: name:            test_v4f32.log10
+    ; CHECK: [[V1:%[0-9]+]]:_(s32), [[V2:%[0-9]+]]:_(s32), [[V3:%[0-9]+]]:_(s32), [[V4:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES %{{[0-9]+}}(<4 x s32>)
+
+    ; CHECK-NEXT: ADJCALLSTACKDOWN
+    ; CHECK-DAG: $s0 = COPY [[V1]](s32)
+    ; CHECK-DAG: BL &log10
+    ; CHECK-DAG: [[ELT1:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-DAG: ADJCALLSTACKUP
+
+    ; CHECK-DAG: ADJCALLSTACKDOWN
+    ; CHECK-DAG: $s0 = COPY [[V2]](s32)
+    ; CHECK-DAG: BL &log10
+    ; CHECK-DAG: [[ELT2:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-DAG: ADJCALLSTACKUP
+
+    ; CHECK-DAG: ADJCALLSTACKDOWN
+    ; CHECK-DAG: $s0 = COPY [[V3]](s32)
+    ; CHECK-DAG: BL &log10
+    ; CHECK-DAG: [[ELT3:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-DAG: ADJCALLSTACKUP
+
+    ; CHECK-DAG: ADJCALLSTACKDOWN
+    ; CHECK-DAG: $s0 = COPY [[V4]](s32)
+    ; CHECK-DAG: BL &log10
+    ; CHECK-DAG: [[ELT4:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-DAG: ADJCALLSTACKUP
+
+    ; CHECK-DAG: %1:_(<4 x s32>) = G_BUILD_VECTOR [[ELT1]](s32), [[ELT2]](s32), [[ELT3]](s32), [[ELT4]](s32)
+
+    %0:_(<4 x s32>) = COPY $q0
+    %1:_(<4 x s32>) = G_FLOG10 %0
+    $q0 = COPY %1(<4 x s32>)
+    RET_ReallyLR implicit $q0
+
+...
+---
+name:            test_v2f64.log10
+alignment:       2
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+body:             |
+  bb.0:
+    liveins: $q0
+
+    ; CHECK-LABEL: name:            test_v2f64.log10
+    ; CHECK: [[V1:%[0-9]+]]:_(s64), [[V2:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES %{{[0-9]+}}(<2 x s64>)
+
+    ; CHECK-NEXT: ADJCALLSTACKDOWN
+    ; CHECK-DAG: $d0 = COPY [[V1]](s64)
+    ; CHECK-DAG: BL &log10
+    ; CHECK-DAG: [[ELT1:%[0-9]+]]:_(s64) = COPY $d0
+    ; CHECK-DAG: ADJCALLSTACKUP
+
+    ; CHECK-DAG: ADJCALLSTACKDOWN
+    ; CHECK-DAG: $d0 = COPY [[V2]](s64)
+    ; CHECK-DAG: BL &log10
+    ; CHECK-DAG: [[ELT2:%[0-9]+]]:_(s64) = COPY $d0
+    ; CHECK-DAG: ADJCALLSTACKUP
+
+    ; CHECK-DAG: %1:_(<2 x s64>) = G_BUILD_VECTOR [[ELT1]](s64), [[ELT2]](s64)
+
+    %0:_(<2 x s64>) = COPY $q0
+    %1:_(<2 x s64>) = G_FLOG10 %0
+    $q0 = COPY %1(<2 x s64>)
+    RET_ReallyLR implicit $q0
+
+...
+---
+name:            test_log10_half
+alignment:       2
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+body:             |
+  bb.0:
+    liveins: $h0
+    ; CHECK-LABEL: name:            test_log10_half
+    ; CHECK: [[REG1:%[0-9]+]]:_(s32) = G_FPEXT %0(s16)
+    ; CHECK-NEXT: ADJCALLSTACKDOWN
+    ; CHECK-NEXT: $s0 = COPY [[REG1]](s32)
+    ; CHECK-NEXT: BL &log10
+    ; CHECK-NEXT: [[REG2:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-NEXT: ADJCALLSTACKUP
+    ; CHECK-NEXT: [[RES:%[0-9]+]]:_(s16) = G_FPTRUNC [[REG2]](s32)
+
+    %0:_(s16) = COPY $h0
+    %1:_(s16) = G_FLOG10 %0
+    $h0 = COPY %1(s16)
+    RET_ReallyLR implicit $h0
diff --git a/test/CodeGen/AArch64/GlobalISel/legalize-log2.mir b/test/CodeGen/AArch64/GlobalISel/legalize-log2.mir
new file mode 100644
index 0000000000000000000000000000000000000000..9ec78992f0e7deb20d9b51bfdc85051b9b2a8f87
--- /dev/null
+++ b/test/CodeGen/AArch64/GlobalISel/legalize-log2.mir
@@ -0,0 +1,227 @@
+# RUN: llc -verify-machineinstrs -mtriple aarch64--- \
+# RUN: -run-pass=legalizer -mattr=+fullfp16 -global-isel %s -o - \
+# RUN: | FileCheck %s
+...
+---
+name:            test_v4f16.log2
+alignment:       2
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+body:             |
+  bb.0:
+    liveins: $d0
+    ; CHECK-LABEL: name:            test_v4f16.log2
+    ; CHECK: [[V1:%[0-9]+]]:_(s16), [[V2:%[0-9]+]]:_(s16), [[V3:%[0-9]+]]:_(s16), [[V4:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES %{{[0-9]+}}(<4 x s16>)
+
+    ; CHECK-DAG: [[V1_S32:%[0-9]+]]:_(s32) = G_FPEXT [[V1]](s16)
+    ; CHECK-NEXT: ADJCALLSTACKDOWN
+    ; CHECK-NEXT: $s0 = COPY [[V1_S32]](s32)
+    ; CHECK-NEXT: BL &log2
+    ; CHECK-NEXT: [[ELT1_S32:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-NEXT: ADJCALLSTACKUP
+    ; CHECK-NEXT: [[ELT1:%[0-9]+]]:_(s16) = G_FPTRUNC [[ELT1_S32]](s32)
+
+    ; CHECK-DAG: [[V2_S32:%[0-9]+]]:_(s32) = G_FPEXT [[V2]](s16)
+    ; CHECK-NEXT: ADJCALLSTACKDOWN
+    ; CHECK-NEXT: $s0 = COPY [[V2_S32]](s32)
+    ; CHECK-NEXT: BL &log2
+    ; CHECK-NEXT: [[ELT2_S32:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-NEXT: ADJCALLSTACKUP
+    ; CHECK-NEXT: [[ELT2:%[0-9]+]]:_(s16) = G_FPTRUNC [[ELT2_S32]](s32)
+
+    ; CHECK-DAG: [[V3_S32:%[0-9]+]]:_(s32) = G_FPEXT [[V3]](s16)
+    ; CHECK-NEXT: ADJCALLSTACKDOWN
+    ; CHECK-NEXT: $s0 = COPY [[V3_S32]](s32)
+    ; CHECK-NEXT: BL &log2
+    ; CHECK-NEXT: [[ELT3_S32:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-NEXT: ADJCALLSTACKUP
+    ; CHECK-NEXT: [[ELT3:%[0-9]+]]:_(s16) = G_FPTRUNC [[ELT3_S32]](s32)
+
+    ; CHECK-DAG: [[V4_S32:%[0-9]+]]:_(s32) = G_FPEXT [[V4]](s16)
+    ; CHECK-NEXT: ADJCALLSTACKDOWN
+    ; CHECK-NEXT: $s0 = COPY [[V4_S32]](s32)
+    ; CHECK-NEXT: BL &log2
+    ; CHECK-NEXT: [[ELT4_S32:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-NEXT: ADJCALLSTACKUP
+    ; CHECK-NEXT: [[ELT4:%[0-9]+]]:_(s16) = G_FPTRUNC [[ELT4_S32]](s32)
+
+    ; CHECK-DAG: %{{[0-9]+}}:_(<4 x s16>) = G_BUILD_VECTOR [[ELT1]](s16), [[ELT2]](s16), [[ELT3]](s16), [[ELT4]](s16)
+
+    %0:_(<4 x s16>) = COPY $d0
+    %1:_(<4 x s16>) = G_FLOG2 %0
+    $d0 = COPY %1(<4 x s16>)
+    RET_ReallyLR implicit $d0
+
+...
+---
+name:            test_v8f16.log2
+alignment:       2
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+body:             |
+  bb.0:
+    liveins: $q0
+
+    ; CHECK-LABEL: name:            test_v8f16.log2
+
+    ; This is big, so let's just check for the 8 calls to log2, the the
+    ; G_UNMERGE_VALUES, and the G_BUILD_VECTOR. The other instructions ought
+    ; to be covered by the other tests.
+
+    ; CHECK: G_UNMERGE_VALUES
+    ; CHECK-DAG: BL &log2
+    ; CHECK-DAG: BL &log2
+    ; CHECK-DAG: BL &log2
+    ; CHECK-DAG: BL &log2
+    ; CHECK-DAG: BL &log2
+    ; CHECK-DAG: BL &log2
+    ; CHECK-DAG: BL &log2
+    ; CHECK-DAG: BL &log2
+    ; CHECK-DAG: G_BUILD_VECTOR
+
+    %0:_(<8 x s16>) = COPY $q0
+    %1:_(<8 x s16>) = G_FLOG2 %0
+    $q0 = COPY %1(<8 x s16>)
+    RET_ReallyLR implicit $q0
+
+...
+---
+name:            test_v2f32.log2
+alignment:       2
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+body:             |
+  bb.0:
+    liveins: $d0
+
+    ; CHECK-LABEL: name:            test_v2f32.log2
+    ; CHECK: [[V1:%[0-9]+]]:_(s32), [[V2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES %{{[0-9]+}}(<2 x s32>)
+
+    ; CHECK-NEXT: ADJCALLSTACKDOWN
+    ; CHECK-DAG: $s0 = COPY [[V1]](s32)
+    ; CHECK-DAG: BL &log2
+    ; CHECK-DAG: [[ELT1:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-DAG: ADJCALLSTACKUP
+
+    ; CHECK-DAG: ADJCALLSTACKDOWN
+    ; CHECK-DAG: $s0 = COPY [[V2]](s32)
+    ; CHECK-DAG: BL &log2
+    ; CHECK-DAG: [[ELT2:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-DAG: ADJCALLSTACKUP
+
+    ; CHECK-DAG: %1:_(<2 x s32>) = G_BUILD_VECTOR [[ELT1]](s32), [[ELT2]](s32)
+
+    %0:_(<2 x s32>) = COPY $d0
+    %1:_(<2 x s32>) = G_FLOG2 %0
+    $d0 = COPY %1(<2 x s32>)
+    RET_ReallyLR implicit $d0
+
+...
+---
+name:            test_v4f32.log2
+alignment:       2
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+body:             |
+  bb.0:
+    liveins: $q0
+    ; CHECK-LABEL: name:            test_v4f32.log2
+    ; CHECK: [[V1:%[0-9]+]]:_(s32), [[V2:%[0-9]+]]:_(s32), [[V3:%[0-9]+]]:_(s32), [[V4:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES %{{[0-9]+}}(<4 x s32>)
+
+    ; CHECK-NEXT: ADJCALLSTACKDOWN
+    ; CHECK-DAG: $s0 = COPY [[V1]](s32)
+    ; CHECK-DAG: BL &log2
+    ; CHECK-DAG: [[ELT1:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-DAG: ADJCALLSTACKUP
+
+    ; CHECK-DAG: ADJCALLSTACKDOWN
+    ; CHECK-DAG: $s0 = COPY [[V2]](s32)
+    ; CHECK-DAG: BL &log2
+    ; CHECK-DAG: [[ELT2:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-DAG: ADJCALLSTACKUP
+
+    ; CHECK-DAG: ADJCALLSTACKDOWN
+    ; CHECK-DAG: $s0 = COPY [[V3]](s32)
+    ; CHECK-DAG: BL &log2
+    ; CHECK-DAG: [[ELT3:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-DAG: ADJCALLSTACKUP
+
+    ; CHECK-DAG: ADJCALLSTACKDOWN
+    ; CHECK-DAG: $s0 = COPY [[V4]](s32)
+    ; CHECK-DAG: BL &log2
+    ; CHECK-DAG: [[ELT4:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-DAG: ADJCALLSTACKUP
+
+    ; CHECK-DAG: %1:_(<4 x s32>) = G_BUILD_VECTOR [[ELT1]](s32), [[ELT2]](s32), [[ELT3]](s32), [[ELT4]](s32)
+
+    %0:_(<4 x s32>) = COPY $q0
+    %1:_(<4 x s32>) = G_FLOG2 %0
+    $q0 = COPY %1(<4 x s32>)
+    RET_ReallyLR implicit $q0
+
+...
+---
+name:            test_v2f64.log2
+alignment:       2
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+body:             |
+  bb.0:
+    liveins: $q0
+
+    ; CHECK-LABEL: name:            test_v2f64.log2
+    ; CHECK: [[V1:%[0-9]+]]:_(s64), [[V2:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES %{{[0-9]+}}(<2 x s64>)
+
+    ; CHECK-NEXT: ADJCALLSTACKDOWN
+    ; CHECK-DAG: $d0 = COPY [[V1]](s64)
+    ; CHECK-DAG: BL &log2
+    ; CHECK-DAG: [[ELT1:%[0-9]+]]:_(s64) = COPY $d0
+    ; CHECK-DAG: ADJCALLSTACKUP
+
+    ; CHECK-DAG: ADJCALLSTACKDOWN
+    ; CHECK-DAG: $d0 = COPY [[V2]](s64)
+    ; CHECK-DAG: BL &log2
+    ; CHECK-DAG: [[ELT2:%[0-9]+]]:_(s64) = COPY $d0
+    ; CHECK-DAG: ADJCALLSTACKUP
+
+    ; CHECK-DAG: %1:_(<2 x s64>) = G_BUILD_VECTOR [[ELT1]](s64), [[ELT2]](s64)
+
+    %0:_(<2 x s64>) = COPY $q0
+    %1:_(<2 x s64>) = G_FLOG2 %0
+    $q0 = COPY %1(<2 x s64>)
+    RET_ReallyLR implicit $q0
+
+...
+---
+name:            test_log2_half
+alignment:       2
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+body:             |
+  bb.0:
+    liveins: $h0
+    ; CHECK-LABEL: name:            test_log2_half
+    ; CHECK: [[REG1:%[0-9]+]]:_(s32) = G_FPEXT %0(s16)
+    ; CHECK-NEXT: ADJCALLSTACKDOWN
+    ; CHECK-NEXT: $s0 = COPY [[REG1]](s32)
+    ; CHECK-NEXT: BL &log2
+    ; CHECK-NEXT: [[REG2:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-NEXT: ADJCALLSTACKUP
+    ; CHECK-NEXT: [[RES:%[0-9]+]]:_(s16) = G_FPTRUNC [[REG2]](s32)
+
+    %0:_(s16) = COPY $h0
+    %1:_(s16) = G_FLOG2 %0
+    $h0 = COPY %1(s16)
+    RET_ReallyLR implicit $h0
diff --git a/test/CodeGen/AArch64/GlobalISel/legalize-merge-values.mir b/test/CodeGen/AArch64/GlobalISel/legalize-merge-values.mir
index 69612eb0f1795b0d6352d603cb0aa39bec402f64..06a10d4a86f7a2ae87ad2922010ad1554f45c0bf 100644
--- a/test/CodeGen/AArch64/GlobalISel/legalize-merge-values.mir
+++ b/test/CodeGen/AArch64/GlobalISel/legalize-merge-values.mir
@@ -1,30 +1,34 @@
-# RUN: llc -O0 -run-pass=legalizer -global-isel-abort=0 -pass-remarks-missed='gisel*' %s -o - 2>&1 | FileCheck %s
-
---- |
-  target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
-  target triple = "aarch64--"
-  define void @test_merge_s4() {
-    ret void
-  }
-...
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc  -march=aarch64 -O0 -run-pass=legalizer  %s -o - | FileCheck %s
 
 ---
 name:            test_merge_s4
-registers:
-  - { id: 0, class: _ }
-  - { id: 1, class: _ }
-  - { id: 2, class: _ }
-  - { id: 3, class: _ }
-  - { id: 4, class: _ }
 body: |
   bb.0:
-    %0(s64) = G_CONSTANT i64 0
-    %1(s4) = G_TRUNC %0(s64)
-    ; Previously, LegalizerInfo was assuming all G_MERGE_VALUES and G_UNMERGE_VALUES
-    ; instructions are legal. Make sure that is no longer happening.
-    ; CHECK: unable to legalize instruction: {{.*}} G_MERGE_VALUES
-    %2(s8) = G_MERGE_VALUES %1(s4), %1(s4)
-    %3(s8) = COPY %2(s8)
-    %4(s64) = G_ANYEXT %3(s8)
-    $x0 = COPY %4(s64)
+    ; CHECK-LABEL: name: test_merge_s4
+    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; CHECK: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+    ; CHECK: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY [[C1]](s32)
+    ; CHECK: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C2]]
+    ; CHECK: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 15
+    ; CHECK: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[C]](s64)
+    ; CHECK: [[AND1:%[0-9]+]]:_(s32) = G_AND [[TRUNC]], [[C3]]
+    ; CHECK: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[AND]](s32)
+    ; CHECK: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 15
+    ; CHECK: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[C]](s64)
+    ; CHECK: [[AND2:%[0-9]+]]:_(s32) = G_AND [[TRUNC1]], [[C4]]
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY [[SHL]](s32)
+    ; CHECK: [[OR:%[0-9]+]]:_(s32) = G_OR [[AND2]], [[COPY1]]
+    ; CHECK: [[TRUNC2:%[0-9]+]]:_(s8) = G_TRUNC [[OR]](s32)
+    ; CHECK: [[COPY2:%[0-9]+]]:_(s8) = COPY [[TRUNC2]](s8)
+    ; CHECK: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[COPY2]](s8)
+    ; CHECK: $x0 = COPY [[ANYEXT]](s64)
+    %0:_(s64) = G_CONSTANT i64 0
+    %1:_(s4) = G_TRUNC %0
+
+    %2:_(s8) = G_MERGE_VALUES %1, %1
+    %3:_(s8) = COPY %2
+    %4:_(s64) = G_ANYEXT %3
+    $x0 = COPY %4
 ...
diff --git a/test/CodeGen/AArch64/GlobalISel/legalize-phi.mir b/test/CodeGen/AArch64/GlobalISel/legalize-phi.mir
index d8f2542d907dc594f8b6723533f16e9d6009aa3c..484224f842c0089d64d9cc4f799d41f15fac909a 100644
--- a/test/CodeGen/AArch64/GlobalISel/legalize-phi.mir
+++ b/test/CodeGen/AArch64/GlobalISel/legalize-phi.mir
@@ -1,4 +1,4 @@
-# RUN: llc -mtriple=aarch64-unknown-unknown -verify-machineinstrs -run-pass=legalizer %s -o - | FileCheck %s
+# RUN: llc -O0 -mtriple=aarch64-unknown-unknown -verify-machineinstrs -run-pass=legalizer %s -o - | FileCheck %s
 --- |
   ; ModuleID = '/tmp/test.ll'
   source_filename = "/tmp/test.ll"
diff --git a/test/CodeGen/AArch64/GlobalISel/legalize-rem.mir b/test/CodeGen/AArch64/GlobalISel/legalize-rem.mir
index 69d1b6d761d5acd9ba2696e1d337f4010c8cd0ef..647b413970638b204d46679b62218768d5d549b2 100644
--- a/test/CodeGen/AArch64/GlobalISel/legalize-rem.mir
+++ b/test/CodeGen/AArch64/GlobalISel/legalize-rem.mir
@@ -1,5 +1,5 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -march=aarch64 -run-pass=legalizer %s -o - | FileCheck %s
+# RUN: llc -O0 -march=aarch64 -run-pass=legalizer %s -o - | FileCheck %s
 ---
 name:            test_urem_64
 body:             |
diff --git a/test/CodeGen/AArch64/GlobalISel/legalize-shift.mir b/test/CodeGen/AArch64/GlobalISel/legalize-shift.mir
index ad1f431c160033d2b5e240479d86b38bef1b7fa0..3fb6a4f6e8abf102108663356f36bbb89c1619da 100644
--- a/test/CodeGen/AArch64/GlobalISel/legalize-shift.mir
+++ b/test/CodeGen/AArch64/GlobalISel/legalize-shift.mir
@@ -1,5 +1,5 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -march=aarch64 -run-pass=legalizer %s -o - | FileCheck %s
+# RUN: llc -O0 -march=aarch64 -run-pass=legalizer %s -o - | FileCheck %s
 ---
 name:            test_shift
 body:             |
@@ -7,30 +7,30 @@ body:             |
     ; CHECK-LABEL: name: test_shift
     ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY $x0
     ; CHECK: [[COPY1:%[0-9]+]]:_(s64) = COPY $x1
-    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
-    ; CHECK: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
-    ; CHECK: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[TRUNC]], [[C]]
-    ; CHECK: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C]]
-    ; CHECK: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
-    ; CHECK: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[COPY1]](s64)
-    ; CHECK: [[AND:%[0-9]+]]:_(s32) = G_AND [[TRUNC1]], [[C1]]
-    ; CHECK: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[ASHR]], [[AND]]
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; CHECK: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY1]](s64)
+    ; CHECK: [[AND:%[0-9]+]]:_(s32) = G_AND [[TRUNC]], [[C]]
+    ; CHECK: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; CHECK: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
+    ; CHECK: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[TRUNC1]], [[C1]](s32)
+    ; CHECK: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C1]](s32)
+    ; CHECK: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[ASHR]], [[AND]](s32)
     ; CHECK: [[COPY2:%[0-9]+]]:_(s32) = COPY [[ASHR1]](s32)
     ; CHECK: $w0 = COPY [[COPY2]](s32)
     ; CHECK: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
-    ; CHECK: [[TRUNC2:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
+    ; CHECK: [[TRUNC2:%[0-9]+]]:_(s32) = G_TRUNC [[COPY1]](s64)
     ; CHECK: [[AND1:%[0-9]+]]:_(s32) = G_AND [[TRUNC2]], [[C2]]
     ; CHECK: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
-    ; CHECK: [[TRUNC3:%[0-9]+]]:_(s32) = G_TRUNC [[COPY1]](s64)
+    ; CHECK: [[TRUNC3:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
     ; CHECK: [[AND2:%[0-9]+]]:_(s32) = G_AND [[TRUNC3]], [[C3]]
-    ; CHECK: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[AND1]], [[AND2]]
+    ; CHECK: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[AND2]], [[AND1]](s32)
     ; CHECK: [[COPY3:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
     ; CHECK: $w0 = COPY [[COPY3]](s32)
-    ; CHECK: [[TRUNC4:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
     ; CHECK: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
-    ; CHECK: [[TRUNC5:%[0-9]+]]:_(s32) = G_TRUNC [[COPY1]](s64)
-    ; CHECK: [[AND3:%[0-9]+]]:_(s32) = G_AND [[TRUNC5]], [[C4]]
-    ; CHECK: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[TRUNC4]], [[AND3]]
+    ; CHECK: [[TRUNC4:%[0-9]+]]:_(s32) = G_TRUNC [[COPY1]](s64)
+    ; CHECK: [[AND3:%[0-9]+]]:_(s32) = G_AND [[TRUNC4]], [[C4]]
+    ; CHECK: [[TRUNC5:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64)
+    ; CHECK: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[TRUNC5]], [[AND3]](s32)
     ; CHECK: [[COPY4:%[0-9]+]]:_(s32) = COPY [[SHL1]](s32)
     ; CHECK: $w0 = COPY [[COPY4]](s32)
     %0:_(s64) = COPY $x0
@@ -48,3 +48,157 @@ body:             |
     $w0 = COPY %9(s32)
 
 ...
+---
+name: test_shl_i64_i32
+body:             |
+  bb.0:
+    ; CHECK-LABEL: name: test_shl_i64_i32
+    ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY $x0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $w1
+    ; CHECK: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[COPY1]](s32)
+    ; CHECK: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[COPY]], [[ZEXT]](s64)
+    ; CHECK: $x0 = COPY [[SHL]](s64)
+    %0:_(s64) = COPY $x0
+    %1:_(s32) = COPY $w1
+    %2:_(s64) = G_SHL %0, %1
+    $x0 = COPY %2(s64)
+
+...
+---
+name: test_ashr_i64_i32
+body:             |
+  bb.0:
+    ; CHECK-LABEL: name: test_ashr_i64_i32
+    ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY $x0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $w1
+    ; CHECK: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[COPY1]](s32)
+    ; CHECK: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[COPY]], [[ZEXT]](s64)
+    ; CHECK: $x0 = COPY [[ASHR]](s64)
+    %0:_(s64) = COPY $x0
+    %1:_(s32) = COPY $w1
+    %2:_(s64) = G_ASHR %0, %1
+    $x0 = COPY %2(s64)
+
+...
+---
+name: test_lshr_i64_i32
+body:             |
+  bb.0:
+    ; CHECK-LABEL: name: test_lshr_i64_i32
+    ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY $x0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $w1
+    ; CHECK: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[COPY1]](s32)
+    ; CHECK: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[COPY]], [[ZEXT]](s64)
+    ; CHECK: $x0 = COPY [[LSHR]](s64)
+    %0:_(s64) = COPY $x0
+    %1:_(s32) = COPY $w1
+    %2:_(s64) = G_LSHR %0, %1
+    $x0 = COPY %2(s64)
+
+...
+
+---
+name: test_shl_s128_s128
+body:             |
+  bb.0:
+    ; CHECK-LABEL: name: test_shl_s128_s128
+    ; CHECK: [[COPY:%[0-9]+]]:_(s128) = COPY $q0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s128) = COPY $q1
+    ; CHECK: [[TRUNC:%[0-9]+]]:_(s64) = G_TRUNC [[COPY1]](s128)
+    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 64
+    ; CHECK: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](s128)
+    ; CHECK: [[SUB:%[0-9]+]]:_(s64) = G_SUB [[TRUNC]], [[C]]
+    ; CHECK: [[SUB1:%[0-9]+]]:_(s64) = G_SUB [[C]], [[TRUNC]]
+    ; CHECK: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; CHECK: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), [[TRUNC]](s64), [[C]]
+    ; CHECK: [[TRUNC1:%[0-9]+]]:_(s1) = G_TRUNC [[ICMP]](s32)
+    ; CHECK: [[ICMP1:%[0-9]+]]:_(s32) = G_ICMP intpred(eq), [[TRUNC]](s64), [[C1]]
+    ; CHECK: [[TRUNC2:%[0-9]+]]:_(s1) = G_TRUNC [[ICMP1]](s32)
+    ; CHECK: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[UV1]], [[TRUNC]](s64)
+    ; CHECK: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[UV1]], [[TRUNC]](s64)
+    ; CHECK: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[UV]], [[SUB1]](s64)
+    ; CHECK: [[OR:%[0-9]+]]:_(s64) = G_OR [[SHL1]], [[LSHR]]
+    ; CHECK: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; CHECK: [[SHL2:%[0-9]+]]:_(s64) = G_SHL [[UV]], [[SUB]](s64)
+    ; CHECK: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[TRUNC1]](s1), [[SHL]], [[C2]]
+    ; CHECK: [[SELECT1:%[0-9]+]]:_(s64) = G_SELECT [[TRUNC1]](s1), [[OR]], [[SHL2]]
+    ; CHECK: [[SELECT2:%[0-9]+]]:_(s64) = G_SELECT [[TRUNC2]](s1), [[UV1]], [[SELECT1]]
+    ; CHECK: [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[SELECT]](s64), [[SELECT2]](s64)
+    ; CHECK: $q0 = COPY [[MV]](s128)
+    %0:_(s128) = COPY $q0
+    %1:_(s128) = COPY $q1
+    %2:_(s128) = G_SHL %0, %1
+    $q0 = COPY %2
+
+...
+
+---
+name: test_lshr_s128_s128
+body:             |
+  bb.0:
+    ; CHECK-LABEL: name: test_lshr_s128_s128
+    ; CHECK: [[COPY:%[0-9]+]]:_(s128) = COPY $q0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s128) = COPY $q1
+    ; CHECK: [[TRUNC:%[0-9]+]]:_(s64) = G_TRUNC [[COPY1]](s128)
+    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 64
+    ; CHECK: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](s128)
+    ; CHECK: [[SUB:%[0-9]+]]:_(s64) = G_SUB [[TRUNC]], [[C]]
+    ; CHECK: [[SUB1:%[0-9]+]]:_(s64) = G_SUB [[C]], [[TRUNC]]
+    ; CHECK: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; CHECK: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), [[TRUNC]](s64), [[C]]
+    ; CHECK: [[TRUNC1:%[0-9]+]]:_(s1) = G_TRUNC [[ICMP]](s32)
+    ; CHECK: [[ICMP1:%[0-9]+]]:_(s32) = G_ICMP intpred(eq), [[TRUNC]](s64), [[C1]]
+    ; CHECK: [[TRUNC2:%[0-9]+]]:_(s1) = G_TRUNC [[ICMP1]](s32)
+    ; CHECK: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[UV1]], [[TRUNC]](s64)
+    ; CHECK: [[LSHR1:%[0-9]+]]:_(s64) = G_LSHR [[UV]], [[TRUNC]](s64)
+    ; CHECK: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[UV1]], [[SUB1]](s64)
+    ; CHECK: [[OR:%[0-9]+]]:_(s64) = G_OR [[LSHR1]], [[SHL]]
+    ; CHECK: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; CHECK: [[LSHR2:%[0-9]+]]:_(s64) = G_LSHR [[UV1]], [[SUB]](s64)
+    ; CHECK: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[TRUNC1]](s1), [[OR]], [[LSHR2]]
+    ; CHECK: [[SELECT1:%[0-9]+]]:_(s64) = G_SELECT [[TRUNC2]](s1), [[UV]], [[SELECT]]
+    ; CHECK: [[SELECT2:%[0-9]+]]:_(s64) = G_SELECT [[TRUNC1]](s1), [[LSHR]], [[C2]]
+    ; CHECK: [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[SELECT1]](s64), [[SELECT2]](s64)
+    ; CHECK: $q0 = COPY [[MV]](s128)
+    %0:_(s128) = COPY $q0
+    %1:_(s128) = COPY $q1
+    %2:_(s128) = G_LSHR %0, %1
+    $q0 = COPY %2
+
+...
+
+---
+name: test_ashr_s128_s128
+body:             |
+  bb.0:
+    ; CHECK-LABEL: name: test_ashr_s128_s128
+    ; CHECK: [[COPY:%[0-9]+]]:_(s128) = COPY $q0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s128) = COPY $q1
+    ; CHECK: [[TRUNC:%[0-9]+]]:_(s64) = G_TRUNC [[COPY1]](s128)
+    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 64
+    ; CHECK: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](s128)
+    ; CHECK: [[SUB:%[0-9]+]]:_(s64) = G_SUB [[TRUNC]], [[C]]
+    ; CHECK: [[SUB1:%[0-9]+]]:_(s64) = G_SUB [[C]], [[TRUNC]]
+    ; CHECK: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; CHECK: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), [[TRUNC]](s64), [[C]]
+    ; CHECK: [[TRUNC1:%[0-9]+]]:_(s1) = G_TRUNC [[ICMP]](s32)
+    ; CHECK: [[ICMP1:%[0-9]+]]:_(s32) = G_ICMP intpred(eq), [[TRUNC]](s64), [[C1]]
+    ; CHECK: [[TRUNC2:%[0-9]+]]:_(s1) = G_TRUNC [[ICMP1]](s32)
+    ; CHECK: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[UV1]], [[TRUNC]](s64)
+    ; CHECK: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[UV]], [[TRUNC]](s64)
+    ; CHECK: [[LSHR1:%[0-9]+]]:_(s64) = G_LSHR [[UV1]], [[SUB1]](s64)
+    ; CHECK: [[OR:%[0-9]+]]:_(s64) = G_OR [[LSHR]], [[LSHR1]]
+    ; CHECK: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 63
+    ; CHECK: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[UV1]], [[C2]](s64)
+    ; CHECK: [[ASHR2:%[0-9]+]]:_(s64) = G_ASHR [[UV1]], [[SUB]](s64)
+    ; CHECK: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[TRUNC1]](s1), [[OR]], [[ASHR2]]
+    ; CHECK: [[SELECT1:%[0-9]+]]:_(s64) = G_SELECT [[TRUNC2]](s1), [[UV]], [[SELECT]]
+    ; CHECK: [[SELECT2:%[0-9]+]]:_(s64) = G_SELECT [[TRUNC1]](s1), [[ASHR]], [[ASHR1]]
+    ; CHECK: [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[SELECT1]](s64), [[SELECT2]](s64)
+    ; CHECK: $q0 = COPY [[MV]](s128)
+    %0:_(s128) = COPY $q0
+    %1:_(s128) = COPY $q1
+    %2:_(s128) = G_ASHR %0, %1
+    $q0 = COPY %2
+
+...
diff --git a/test/CodeGen/AArch64/GlobalISel/legalize-simple.mir b/test/CodeGen/AArch64/GlobalISel/legalize-simple.mir
index 51cda7d793e3e9dc53d509a446e6c2a3249c94ae..5b76737edc2ba208b55dd6a432573c039ec7a6d5 100644
--- a/test/CodeGen/AArch64/GlobalISel/legalize-simple.mir
+++ b/test/CodeGen/AArch64/GlobalISel/legalize-simple.mir
@@ -1,5 +1,5 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -march=aarch64 -run-pass=legalizer %s -o - | FileCheck %s
+# RUN: llc -O0 -march=aarch64 -run-pass=legalizer %s -o - | FileCheck %s
 ---
 name:            test_simple
 body:             |
@@ -37,10 +37,10 @@ body:             |
   ; CHECK:   $x0 = COPY [[BITCAST1]](s64)
   ; CHECK:   [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[SELECT3]](s32)
   ; CHECK:   $w0 = COPY [[BITCAST2]](s32)
-  ; CHECK:   [[BITCAST3:%[0-9]+]]:_(<4 x s8>) = G_BITCAST [[COPY]](s64)
+  ; CHECK:   [[BITCAST3:%[0-9]+]]:_(<4 x s8>) = G_BITCAST [[TRUNC1]](s32)
   ; CHECK:   [[BITCAST4:%[0-9]+]]:_(s32) = G_BITCAST [[BITCAST3]](<4 x s8>)
   ; CHECK:   $w0 = COPY [[BITCAST4]](s32)
-  ; CHECK:   [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[COPY]](s64)
+  ; CHECK:   [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[TRUNC1]](s32)
   ; CHECK:   [[BITCAST6:%[0-9]+]]:_(s32) = G_BITCAST [[BITCAST5]](<2 x s16>)
   ; CHECK:   $w0 = COPY [[BITCAST6]](s32)
   bb.0.entry:
@@ -74,10 +74,10 @@ body:             |
     $x0 = COPY %13(s64)
     %14:_(s32) = G_BITCAST %10(s32)
     $w0 = COPY %14(s32)
-    %15:_(<4 x s8>) = G_BITCAST %0(s64)
+    %15:_(<4 x s8>) = G_BITCAST %4(s32)
     %20:_(s32) = G_BITCAST %15(<4 x s8>)
     $w0 = COPY %20(s32)
-    %16:_(<2 x s16>) = G_BITCAST %0(s64)
+    %16:_(<2 x s16>) = G_BITCAST %4(s32)
     %21:_(s32) = G_BITCAST %16(<2 x s16>)
     $w0 = COPY %21(s32)
 
diff --git a/test/CodeGen/AArch64/GlobalISel/legalize-sin.mir b/test/CodeGen/AArch64/GlobalISel/legalize-sin.mir
new file mode 100644
index 0000000000000000000000000000000000000000..f1fec69fa41092aa3c125b444792e2b46c2c9319
--- /dev/null
+++ b/test/CodeGen/AArch64/GlobalISel/legalize-sin.mir
@@ -0,0 +1,227 @@
+# RUN: llc -verify-machineinstrs -mtriple aarch64--- \
+# RUN: -run-pass=legalizer -mattr=+fullfp16 -global-isel %s -o - \
+# RUN: | FileCheck %s
+...
+---
+name:            test_v4f16.sin
+alignment:       2
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+body:             |
+  bb.0:
+    liveins: $d0
+    ; CHECK-LABEL: name:            test_v4f16.sin
+    ; CHECK: [[V1:%[0-9]+]]:_(s16), [[V2:%[0-9]+]]:_(s16), [[V3:%[0-9]+]]:_(s16), [[V4:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES %{{[0-9]+}}(<4 x s16>)
+
+    ; CHECK-DAG: [[V1_S32:%[0-9]+]]:_(s32) = G_FPEXT [[V1]](s16)
+    ; CHECK-NEXT: ADJCALLSTACKDOWN
+    ; CHECK-NEXT: $s0 = COPY [[V1_S32]](s32)
+    ; CHECK-NEXT: BL &sinf
+    ; CHECK-NEXT: [[ELT1_S32:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-NEXT: ADJCALLSTACKUP
+    ; CHECK-NEXT: [[ELT1:%[0-9]+]]:_(s16) = G_FPTRUNC [[ELT1_S32]](s32)
+
+    ; CHECK-DAG: [[V2_S32:%[0-9]+]]:_(s32) = G_FPEXT [[V2]](s16)
+    ; CHECK-NEXT: ADJCALLSTACKDOWN
+    ; CHECK-NEXT: $s0 = COPY [[V2_S32]](s32)
+    ; CHECK-NEXT: BL &sinf
+    ; CHECK-NEXT: [[ELT2_S32:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-NEXT: ADJCALLSTACKUP
+    ; CHECK-NEXT: [[ELT2:%[0-9]+]]:_(s16) = G_FPTRUNC [[ELT2_S32]](s32)
+
+    ; CHECK-DAG: [[V3_S32:%[0-9]+]]:_(s32) = G_FPEXT [[V3]](s16)
+    ; CHECK-NEXT: ADJCALLSTACKDOWN
+    ; CHECK-NEXT: $s0 = COPY [[V3_S32]](s32)
+    ; CHECK-NEXT: BL &sinf
+    ; CHECK-NEXT: [[ELT3_S32:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-NEXT: ADJCALLSTACKUP
+    ; CHECK-NEXT: [[ELT3:%[0-9]+]]:_(s16) = G_FPTRUNC [[ELT3_S32]](s32)
+
+    ; CHECK-DAG: [[V4_S32:%[0-9]+]]:_(s32) = G_FPEXT [[V4]](s16)
+    ; CHECK-NEXT: ADJCALLSTACKDOWN
+    ; CHECK-NEXT: $s0 = COPY [[V4_S32]](s32)
+    ; CHECK-NEXT: BL &sinf
+    ; CHECK-NEXT: [[ELT4_S32:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-NEXT: ADJCALLSTACKUP
+    ; CHECK-NEXT: [[ELT4:%[0-9]+]]:_(s16) = G_FPTRUNC [[ELT4_S32]](s32)
+
+    ; CHECK-DAG: %{{[0-9]+}}:_(<4 x s16>) = G_BUILD_VECTOR [[ELT1]](s16), [[ELT2]](s16), [[ELT3]](s16), [[ELT4]](s16)
+
+    %0:_(<4 x s16>) = COPY $d0
+    %1:_(<4 x s16>) = G_FSIN %0
+    $d0 = COPY %1(<4 x s16>)
+    RET_ReallyLR implicit $d0
+
+...
+---
+name:            test_v8f16.sin
+alignment:       2
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+body:             |
+  bb.0:
+    liveins: $q0
+
+    ; CHECK-LABEL: name:            test_v8f16.sin
+
+    ; This is big, so let's just check for the 8 calls to sinf, the the
+    ; G_UNMERGE_VALUES, and the G_BUILD_VECTOR. The other instructions ought
+    ; to be covered by the other tests.
+
+    ; CHECK: G_UNMERGE_VALUES
+    ; CHECK-DAG: BL &sinf
+    ; CHECK-DAG: BL &sinf
+    ; CHECK-DAG: BL &sinf
+    ; CHECK-DAG: BL &sinf
+    ; CHECK-DAG: BL &sinf
+    ; CHECK-DAG: BL &sinf
+    ; CHECK-DAG: BL &sinf
+    ; CHECK-DAG: BL &sinf
+    ; CHECK-DAG: G_BUILD_VECTOR
+
+    %0:_(<8 x s16>) = COPY $q0
+    %1:_(<8 x s16>) = G_FSIN %0
+    $q0 = COPY %1(<8 x s16>)
+    RET_ReallyLR implicit $q0
+
+...
+---
+name:            test_v2f32.sin
+alignment:       2
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+body:             |
+  bb.0:
+    liveins: $d0
+
+    ; CHECK-LABEL: name:            test_v2f32.sin
+    ; CHECK: [[V1:%[0-9]+]]:_(s32), [[V2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES %{{[0-9]+}}(<2 x s32>)
+
+    ; CHECK-NEXT: ADJCALLSTACKDOWN
+    ; CHECK-DAG: $s0 = COPY [[V1]](s32)
+    ; CHECK-DAG: BL &sinf
+    ; CHECK-DAG: [[ELT1:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-DAG: ADJCALLSTACKUP
+
+    ; CHECK-DAG: ADJCALLSTACKDOWN
+    ; CHECK-DAG: $s0 = COPY [[V2]](s32)
+    ; CHECK-DAG: BL &sinf
+    ; CHECK-DAG: [[ELT2:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-DAG: ADJCALLSTACKUP
+
+    ; CHECK-DAG: %1:_(<2 x s32>) = G_BUILD_VECTOR [[ELT1]](s32), [[ELT2]](s32)
+
+    %0:_(<2 x s32>) = COPY $d0
+    %1:_(<2 x s32>) = G_FSIN %0
+    $d0 = COPY %1(<2 x s32>)
+    RET_ReallyLR implicit $d0
+
+...
+---
+name:            test_v4f32.sin
+alignment:       2
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+body:             |
+  bb.0:
+    liveins: $q0
+    ; CHECK-LABEL: name:            test_v4f32.sin
+    ; CHECK: [[V1:%[0-9]+]]:_(s32), [[V2:%[0-9]+]]:_(s32), [[V3:%[0-9]+]]:_(s32), [[V4:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES %{{[0-9]+}}(<4 x s32>)
+
+    ; CHECK-NEXT: ADJCALLSTACKDOWN
+    ; CHECK-DAG: $s0 = COPY [[V1]](s32)
+    ; CHECK-DAG: BL &sinf
+    ; CHECK-DAG: [[ELT1:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-DAG: ADJCALLSTACKUP
+
+    ; CHECK-DAG: ADJCALLSTACKDOWN
+    ; CHECK-DAG: $s0 = COPY [[V2]](s32)
+    ; CHECK-DAG: BL &sinf
+    ; CHECK-DAG: [[ELT2:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-DAG: ADJCALLSTACKUP
+
+    ; CHECK-DAG: ADJCALLSTACKDOWN
+    ; CHECK-DAG: $s0 = COPY [[V3]](s32)
+    ; CHECK-DAG: BL &sinf
+    ; CHECK-DAG: [[ELT3:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-DAG: ADJCALLSTACKUP
+
+    ; CHECK-DAG: ADJCALLSTACKDOWN
+    ; CHECK-DAG: $s0 = COPY [[V4]](s32)
+    ; CHECK-DAG: BL &sinf
+    ; CHECK-DAG: [[ELT4:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-DAG: ADJCALLSTACKUP
+
+    ; CHECK-DAG: %1:_(<4 x s32>) = G_BUILD_VECTOR [[ELT1]](s32), [[ELT2]](s32), [[ELT3]](s32), [[ELT4]](s32)
+
+    %0:_(<4 x s32>) = COPY $q0
+    %1:_(<4 x s32>) = G_FSIN %0
+    $q0 = COPY %1(<4 x s32>)
+    RET_ReallyLR implicit $q0
+
+...
+---
+name:            test_v2f64.sin
+alignment:       2
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+body:             |
+  bb.0:
+    liveins: $q0
+
+    ; CHECK-LABEL: name:            test_v2f64.sin
+    ; CHECK: [[V1:%[0-9]+]]:_(s64), [[V2:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES %{{[0-9]+}}(<2 x s64>)
+
+    ; CHECK-NEXT: ADJCALLSTACKDOWN
+    ; CHECK-DAG: $d0 = COPY [[V1]](s64)
+    ; CHECK-DAG: BL &sin
+    ; CHECK-DAG: [[ELT1:%[0-9]+]]:_(s64) = COPY $d0
+    ; CHECK-DAG: ADJCALLSTACKUP
+
+    ; CHECK-DAG: ADJCALLSTACKDOWN
+    ; CHECK-DAG: $d0 = COPY [[V2]](s64)
+    ; CHECK-DAG: BL &sin
+    ; CHECK-DAG: [[ELT2:%[0-9]+]]:_(s64) = COPY $d0
+    ; CHECK-DAG: ADJCALLSTACKUP
+
+    ; CHECK-DAG: %1:_(<2 x s64>) = G_BUILD_VECTOR [[ELT1]](s64), [[ELT2]](s64)
+
+    %0:_(<2 x s64>) = COPY $q0
+    %1:_(<2 x s64>) = G_FSIN %0
+    $q0 = COPY %1(<2 x s64>)
+    RET_ReallyLR implicit $q0
+
+...
+---
+name:            test_sin_half
+alignment:       2
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+body:             |
+  bb.0:
+    liveins: $h0
+    ; CHECK-LABEL: name:            test_sin_half
+    ; CHECK: [[REG1:%[0-9]+]]:_(s32) = G_FPEXT %0(s16)
+    ; CHECK-NEXT: ADJCALLSTACKDOWN
+    ; CHECK-NEXT: $s0 = COPY [[REG1]](s32)
+    ; CHECK-NEXT: BL &sinf
+    ; CHECK-NEXT: [[REG2:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK-NEXT: ADJCALLSTACKUP
+    ; CHECK-NEXT: [[RES:%[0-9]+]]:_(s16) = G_FPTRUNC [[REG2]](s32)
+
+    %0:_(s16) = COPY $h0
+    %1:_(s16) = G_FSIN %0
+    $h0 = COPY %1(s16)
+    RET_ReallyLR implicit $h0
diff --git a/test/CodeGen/AArch64/GlobalISel/legalize-sqrt.mir b/test/CodeGen/AArch64/GlobalISel/legalize-sqrt.mir
new file mode 100644
index 0000000000000000000000000000000000000000..9a6fe808ea3ee092660b9e6ab0a7e7be0888494c
--- /dev/null
+++ b/test/CodeGen/AArch64/GlobalISel/legalize-sqrt.mir
@@ -0,0 +1,86 @@
+# RUN: llc -mtriple=arm64-unknown-unknown -global-isel -O0 -mattr=-fullfp16 -run-pass=legalizer %s -o - | FileCheck %s
+
+--- |
+  define <8 x half> @test_v8f16.sqrt(<8 x half> %a) {
+    ret <8 x half> %a
+  }
+
+  define <4 x half> @test_v4f16.sqrt(<4 x half> %a) {
+    ret <4 x half> %a
+  }
+
+...
+---
+name:            test_v8f16.sqrt
+alignment:       2
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+body:             |
+  bb.1 (%ir-block.0):
+    liveins: $q0
+    ; CHECK-LABEL: name:            test_v8f16.sqrt
+    %0:_(<8 x s16>) = COPY $q0
+    ; CHECK: %{{[0-9]+}}:_(s16), %{{[0-9]+}}:_(s16), %{{[0-9]+}}:_(s16), %{{[0-9]+}}:_(s16), %{{[0-9]+}}:_(s16), %{{[0-9]+}}:_(s16), %{{[0-9]+}}:_(s16), %{{[0-9]+}}:_(s16) = G_UNMERGE_VALUES %{{[0-9]+}}(<8 x s16>)
+    ; CHECK: %{{[0-9]+}}:_(s32) = G_FPEXT %{{[0-9]+}}(s16)
+    ; CHECK: %{{[0-9]+}}:_(s32) = G_FSQRT %{{[0-9]+}}
+    ; CHECK: %{{[0-9]+}}:_(s16) = G_FPTRUNC %{{[0-9]+}}(s32)
+    ; CHECK: %{{[0-9]+}}:_(s32) = G_FPEXT %{{[0-9]+}}(s16)
+    ; CHECK: %{{[0-9]+}}:_(s32) = G_FSQRT %{{[0-9]+}}
+    ; CHECK: %{{[0-9]+}}:_(s16) = G_FPTRUNC %{{[0-9]+}}(s32)
+    ; CHECK: %{{[0-9]+}}:_(s32) = G_FPEXT %{{[0-9]+}}(s16)
+    ; CHECK: %{{[0-9]+}}:_(s32) = G_FSQRT %{{[0-9]+}}
+    ; CHECK: %{{[0-9]+}}:_(s16) = G_FPTRUNC %{{[0-9]+}}(s32)
+    ; CHECK: %{{[0-9]+}}:_(s32) = G_FPEXT %{{[0-9]+}}(s16)
+    ; CHECK: %{{[0-9]+}}:_(s32) = G_FSQRT %{{[0-9]+}}
+    ; CHECK: %{{[0-9]+}}:_(s16) = G_FPTRUNC %{{[0-9]+}}(s32)
+    ; CHECK: %{{[0-9]+}}:_(s32) = G_FPEXT %{{[0-9]+}}(s16)
+    ; CHECK: %{{[0-9]+}}:_(s32) = G_FSQRT %{{[0-9]+}}
+    ; CHECK: %{{[0-9]+}}:_(s16) = G_FPTRUNC %{{[0-9]+}}(s32)
+    ; CHECK: %{{[0-9]+}}:_(s32) = G_FPEXT %{{[0-9]+}}(s16)
+    ; CHECK: %{{[0-9]+}}:_(s32) = G_FSQRT %{{[0-9]+}}
+    ; CHECK: %{{[0-9]+}}:_(s16) = G_FPTRUNC %{{[0-9]+}}(s32)
+    ; CHECK: %{{[0-9]+}}:_(s32) = G_FPEXT %{{[0-9]+}}(s16)
+    ; CHECK: %{{[0-9]+}}:_(s32) = G_FSQRT %{{[0-9]+}}
+    ; CHECK: %{{[0-9]+}}:_(s16) = G_FPTRUNC %{{[0-9]+}}(s32)
+    ; CHECK: %{{[0-9]+}}:_(s32) = G_FPEXT %{{[0-9]+}}(s16)
+    ; CHECK: %{{[0-9]+}}:_(s32) = G_FSQRT %{{[0-9]+}}
+    ; CHECK: %{{[0-9]+}}:_(s16) = G_FPTRUNC %{{[0-9]+}}(s32)
+    ; CHECK: %{{[0-9]+}}:_(<8 x s16>) = G_BUILD_VECTOR %{{[0-9]+}}(s16), %{{[0-9]+}}(s16), %{{[0-9]+}}(s16), %{{[0-9]+}}(s16), %{{[0-9]+}}(s16), %{{[0-9]+}}(s16), %{{[0-9]+}}(s16), %{{[0-9]+}}(s16)
+    %1:_(<8 x s16>) = G_FSQRT %0
+    $q0 = COPY %1(<8 x s16>)
+    RET_ReallyLR implicit $q0
+
+...
+---
+name:            test_v4f16.sqrt
+alignment:       2
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+body:             |
+  bb.1 (%ir-block.0):
+    liveins: $d0
+    ; CHECK-LABEL: name:            test_v4f16.sqrt
+    %0:_(<4 x s16>) = COPY $d0
+    ; CHECK: %{{[0-9]+}}:_(s16), %{{[0-9]+}}:_(s16), %{{[0-9]+}}:_(s16)  = G_UNMERGE_VALUES %{{[0-9]+}}(<4 x s16>)
+    ; CHECK: %{{[0-9]+}}:_(s32) = G_FPEXT %{{[0-9]+}}(s16)
+    ; CHECK: %{{[0-9]+}}:_(s32) = G_FSQRT %{{[0-9]+}}
+    ; CHECK: %{{[0-9]+}}:_(s16) = G_FPTRUNC %{{[0-9]+}}(s32)
+    ; CHECK: %{{[0-9]+}}:_(s32) = G_FPEXT %{{[0-9]+}}(s16)
+    ; CHECK: %{{[0-9]+}}:_(s32) = G_FSQRT %{{[0-9]+}}
+    ; CHECK: %{{[0-9]+}}:_(s16) = G_FPTRUNC %{{[0-9]+}}(s32)
+    ; CHECK: %{{[0-9]+}}:_(s32) = G_FPEXT %{{[0-9]+}}(s16)
+    ; CHECK: %{{[0-9]+}}:_(s32) = G_FSQRT %{{[0-9]+}}
+    ; CHECK: %{{[0-9]+}}:_(s16) = G_FPTRUNC %{{[0-9]+}}(s32)
+    ; CHECK: %{{[0-9]+}}:_(s32) = G_FPEXT %{{[0-9]+}}(s16)
+    ; CHECK: %{{[0-9]+}}:_(s32) = G_FSQRT %{{[0-9]+}}
+    ; CHECK: %{{[0-9]+}}:_(s16) = G_FPTRUNC %{{[0-9]+}}(s32)
+    ; CHECK: %{{[0-9]+}}:_(<4 x s16>) = G_BUILD_VECTOR %{{[0-9]+}}(s16), %{{[0-9]+}}(s16), %{{[0-9]+}}(s16), %{{[0-9]+}}(s16)
+    %1:_(<4 x s16>) = G_FSQRT %0
+    $d0 = COPY %1(<4 x s16>)
+    RET_ReallyLR implicit $d0
+
+...
diff --git a/test/CodeGen/AArch64/GlobalISel/legalize-undef.mir b/test/CodeGen/AArch64/GlobalISel/legalize-undef.mir
index e46c9ad79c6748ab4d5afe26506b980b3b5d43d6..aaa1417e3c22108a9035849e72835712edfe97a7 100644
--- a/test/CodeGen/AArch64/GlobalISel/legalize-undef.mir
+++ b/test/CodeGen/AArch64/GlobalISel/legalize-undef.mir
@@ -1,5 +1,5 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -march=aarch64 -run-pass=legalizer %s -o - | FileCheck %s
+# RUN: llc -march=aarch64 -run-pass=legalizer -O0 %s -o - | FileCheck %s
 ---
 name:            test_implicit_def
 body: |
@@ -16,3 +16,58 @@ body: |
     %1:_(s64) = G_TRUNC %0(s128)
     $x0 = COPY %1(s64)
 ...
+
+---
+name: test_implicit_def_s3
+body: |
+  bb.0:
+    liveins:
+
+    ; CHECK-LABEL: name: test_implicit_def_s3
+    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 61
+    ; CHECK: [[DEF:%[0-9]+]]:_(s64) = G_IMPLICIT_DEF
+    ; CHECK: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[DEF]], [[C]](s64)
+    ; CHECK: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[SHL]], [[C]](s64)
+    ; CHECK: $x0 = COPY [[ASHR]](s64)
+    %0:_(s3) = G_IMPLICIT_DEF
+    %1:_(s64) = G_SEXT %0
+    $x0 = COPY %1(s64)
+...
+
+# FIXME: s2 not correctly handled
+
+---
+name: test_implicit_def_v2s32
+body: |
+  bb.0:
+
+    ; CHECK-LABEL: name: test_implicit_def_v2s32
+    ; CHECK: [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
+    ; CHECK: [[DEF1:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
+    ; CHECK: [[DEF2:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
+    ; CHECK: [[DEF3:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[DEF]](s32), [[DEF1]](s32), [[DEF2]](s32), [[DEF3]](s32)
+    ; CHECK: [[UV:%[0-9]+]]:_(<2 x s32>), [[UV1:%[0-9]+]]:_(<2 x s32>) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<4 x s32>)
+    ; CHECK: $x0 = COPY [[UV]](<2 x s32>)
+    ; CHECK: $x1 = COPY [[UV1]](<2 x s32>)
+    %0:_(<4 x s32>) = G_IMPLICIT_DEF
+    %1:_(<2 x s32> ), %2:_(<2 x s32>) = G_UNMERGE_VALUES %0
+    $x0 = COPY %1
+    $x1 = COPY %2
+...
+
+---
+name: test_implicit_def_v4s64
+body: |
+  bb.0:
+
+    ; CHECK-LABEL: name: test_implicit_def_v4s64
+    ; CHECK: [[DEF:%[0-9]+]]:_(<2 x s64>) = G_IMPLICIT_DEF
+    ; CHECK: [[DEF1:%[0-9]+]]:_(<2 x s64>) = G_IMPLICIT_DEF
+    ; CHECK: $q0 = COPY [[DEF]](<2 x s64>)
+    ; CHECK: $q1 = COPY [[DEF1]](<2 x s64>)
+    %0:_(<4 x s64>) = G_IMPLICIT_DEF
+    %1:_(<2 x s64> ), %2:_(<2 x s64>) = G_UNMERGE_VALUES %0
+    $q0 = COPY %1
+    $q1 = COPY %2
+...
diff --git a/test/CodeGen/AArch64/GlobalISel/legalize-unmerge-values.mir b/test/CodeGen/AArch64/GlobalISel/legalize-unmerge-values.mir
index 12e7bb489d6e94451caebb974e207c75a37975b1..39198da0ec2059f48aa765e3d78ccf4c96919596 100644
--- a/test/CodeGen/AArch64/GlobalISel/legalize-unmerge-values.mir
+++ b/test/CodeGen/AArch64/GlobalISel/legalize-unmerge-values.mir
@@ -1,28 +1,34 @@
-# RUN: llc -O0 -run-pass=legalizer -global-isel-abort=0 -pass-remarks-missed='gisel*' %s -o - 2>&1 | FileCheck %s
-
---- |
-  target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
-  target triple = "aarch64--"
-  define void @test_unmerge_s4() {
-    ret void
-  }
-...
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -march=aarch64 -O0 -run-pass=legalizer -global-isel-abort=0 %s -o -  | FileCheck %s
+# RUN: llc -march=aarch64 -O0 -run-pass=legalizer -global-isel-abort=0 -pass-remarks-missed='gisel*'  %s -o /dev/null 2>&1  | FileCheck -check-prefix=ERROR %s
+# ERROR: unable to legalize instruction: %3:_(s64) = G_ANYEXT %1:_(s4) (in function: test_unmerge_s4)
 
 ---
-name:            test_unmerge_s4
-registers:
-  - { id: 0, class: _ }
-  - { id: 1, class: _ }
-  - { id: 2, class: _ }
-  - { id: 3, class: _ }
+name:  test_unmerge_s4
 body: |
   bb.0:
-    %0(s8) = G_CONSTANT i8 0
-    ; Previously, LegalizerInfo was assuming all G_MERGE_VALUES and G_UNMERGE_VALUES
-    ; instructions are legal. Make sure that is no longer happening.
-    ; CHECK: unable to legalize instruction: {{.*}} G_UNMERGE_VALUES
-    %1(s4), %2(s4)= G_UNMERGE_VALUES %0(s8)
-    %3(s64) = G_ANYEXT %1(s4)
-    $x0 = COPY %3(s64)
+    ; CHECK-LABEL: name: test_unmerge_s4
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; CHECK: [[TRUNC:%[0-9]+]]:_(s8) = G_TRUNC [[C]](s32)
+    ; CHECK: [[ZEXT:%[0-9]+]]:_(s16) = G_ZEXT [[TRUNC]](s8)
+    ; CHECK: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+    ; CHECK: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[C1]](s32)
+    ; CHECK: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[TRUNC1]](s16)
+    ; CHECK: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[ZEXT]](s16)
+    ; CHECK: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ANYEXT]], [[ZEXT1]](s32)
+    ; CHECK: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[SHL]](s32)
+    ; CHECK: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[ZEXT]](s16)
+    ; CHECK: [[ANYEXT2:%[0-9]+]]:_(s32) = G_ANYEXT [[TRUNC2]](s16)
+    ; CHECK: [[OR:%[0-9]+]]:_(s32) = G_OR [[ANYEXT1]], [[ANYEXT2]]
+    ; CHECK: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[OR]](s32)
+    ; CHECK: [[UV:%[0-9]+]]:_(s8), [[UV1:%[0-9]+]]:_(s8) = G_UNMERGE_VALUES [[TRUNC3]](s16)
+    ; CHECK: [[TRUNC4:%[0-9]+]]:_(s4) = G_TRUNC [[UV]](s8)
+    ; CHECK: [[ANYEXT3:%[0-9]+]]:_(s64) = G_ANYEXT [[TRUNC4]](s4)
+    ; CHECK: [[TRUNC5:%[0-9]+]]:_(s4) = G_TRUNC [[UV1]](s8)
+    ; CHECK: $x0 = COPY [[ANYEXT3]](s64)
+    %0:_(s8) = G_CONSTANT i8 0
+    %1:_(s4), %2:_(s4)= G_UNMERGE_VALUES %0
+    %3:_(s64) = G_ANYEXT %1
+    $x0 = COPY %3
 
 ...
diff --git a/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir b/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir
index 0c75cd3ce3b60ef9686079fa371d88b7a5eeae3c..63d74b15b76cc1d012a411e4ee4c99af57a1f956 100644
--- a/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir
+++ b/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir
@@ -7,7 +7,7 @@
 # REQUIRES: asserts
 
 # The main purpose of this test is to make sure we don't over-relax
-# LegalizerInfo validation and loose its ability to catch bugs.
+# LegalizerInfo validation and lose its ability to catch bugs.
 #
 # Watch out for every "SKIPPED: user-defined predicate detected" in the
 # check-lines below and keep each and every one of them justified.
@@ -46,7 +46,7 @@
 # DEBUG:      .. the first uncovered type index: 1, OK
 #
 # DEBUG-NEXT: G_IMPLICIT_DEF (opcode {{[0-9]+}}): 1 type index
-# DEBUG:      .. the first uncovered type index: 1, OK
+# DEBUG: .. type index coverage check SKIPPED: user-defined predicate detected
 #
 # DEBUG-NEXT: G_PHI (opcode {{[0-9]+}}): 1 type index
 # DEBUG:      .. the first uncovered type index: 1, OK
@@ -180,14 +180,14 @@
 # DEBUG-NEXT: G_ZEXT (opcode {{[0-9]+}}): 2 type indices
 # DEBUG:      .. the first uncovered type index: 2, OK
 #
-# DEBUG-NEXT: G_SHL (opcode {{[0-9]+}}): 1 type index
-# DEBUG:      .. the first uncovered type index: 1, OK
+# DEBUG-NEXT: G_SHL (opcode {{[0-9]+}}): 2 type indices
+# DEBUG:.. type index coverage check SKIPPED: user-defined predicate detected
 #
-# DEBUG-NEXT: G_LSHR (opcode {{[0-9]+}}): 1 type index
-# DEBUG:      .. the first uncovered type index: 1, OK
+# DEBUG-NEXT: G_LSHR (opcode {{[0-9]+}}): 2 type indices
+# DEBUG: .. type index coverage check SKIPPED: user-defined predicate detected
 #
-# DEBUG-NEXT: G_ASHR (opcode {{[0-9]+}}): 1 type index
-# DEBUG:      .. the first uncovered type index: 1, OK
+# DEBUG-NEXT: G_ASHR (opcode {{[0-9]+}}): 2 type indices
+# DEBUG: .. type index coverage check SKIPPED: user-defined predicate detected
 #
 # DEBUG-NEXT: G_ICMP (opcode {{[0-9]+}}): 2 type indices
 # DEBUG:      .. the first uncovered type index: 2, OK
@@ -256,22 +256,22 @@
 # DEBUG:      .. the first uncovered type index: 1, OK
 #
 # DEBUG-NEXT: G_FEXP (opcode {{[0-9]+}}): 1 type index
-# DEBUG:      .. type index coverage check SKIPPED: no rules defined
+# DEBUG:      .. the first uncovered type index: 1, OK
 #
 # DEBUG-NEXT: G_FEXP2 (opcode {{[0-9]+}}): 1 type index
 # DEBUG:      .. type index coverage check SKIPPED: no rules defined
 #
 # DEBUG-NEXT: G_FLOG (opcode {{[0-9]+}}): 1 type index
-# DEBUG:      .. type index coverage check SKIPPED: no rules defined
+# DEBUG:      .. the first uncovered type index: 1, OK
 #
 # DEBUG-NEXT: G_FLOG2 (opcode {{[0-9]+}}): 1 type index
-# DEBUG:      .. type index coverage check SKIPPED: no rules defined
+# DEBUG:      .. the first uncovered type index: 1, OK
 #
 # DEBUG-NEXT: G_FLOG10 (opcode {{[0-9]+}}): 1 type index
-# DEBUG:      .. type index coverage check SKIPPED: no rules defined
+# DEBUG:      .. the first uncovered type index: 1, OK
 #
 # DEBUG-NEXT: G_FNEG (opcode {{[0-9]+}}): 1 type index
-# DEBUG:      .. type index coverage check SKIPPED: no rules defined
+# DEBUG:      .. the first uncovered type index: 1, OK
 #
 # DEBUG-NEXT: G_FPEXT (opcode {{[0-9]+}}): 2 type indices
 # DEBUG:      .. the first uncovered type index: 2, OK
@@ -292,7 +292,10 @@
 # DEBUG:      .. the first uncovered type index: 2, OK
 #
 # DEBUG-NEXT: G_FABS (opcode {{[0-9]+}}): 1 type index
-# DEBUG:      .. type index coverage check SKIPPED: no rules defined
+# DEBUG:      .. type index coverage check SKIPPED: user-defined predicate detected
+#
+# DEBUG-NEXT: G_FCANONICALIZE (opcode 128): 1 type index
+# DEBUG: .. type index coverage check SKIPPED: no rules defined
 #
 # DEBUG-NEXT: G_GEP (opcode {{[0-9]+}}): 2 type indices
 # DEBUG:      .. the first uncovered type index: 2, OK
@@ -312,23 +315,38 @@
 # DEBUG-NEXT: G_SHUFFLE_VECTOR (opcode {{[0-9]+}}): 3 type indices
 # DEBUG:      .. type index coverage check SKIPPED: no rules defined
 #
-# DEBUG-NEXT: G_CTTZ (opcode {{[0-9]+}}): 1 type index
+# DEBUG-NEXT: G_CTTZ (opcode {{[0-9]+}}): 2 type indices
 # DEBUG:      .. type index coverage check SKIPPED: no rules defined
 #
-# DEBUG-NEXT: G_CTTZ_ZERO_UNDEF (opcode {{[0-9]+}}): 1 type index
+# DEBUG-NEXT: G_CTTZ_ZERO_UNDEF (opcode {{[0-9]+}}): 2 type indices
 # DEBUG:      .. type index coverage check SKIPPED: no rules defined
 #
-# DEBUG-NEXT: G_CTLZ (opcode {{[0-9]+}}): 1 type index
+# DEBUG-NEXT: G_CTLZ (opcode {{[0-9]+}}): 2 type indices
 # DEBUG:      .. type index coverage check SKIPPED: no rules defined
 #
-# DEBUG-NEXT: G_CTLZ_ZERO_UNDEF (opcode {{[0-9]+}}): 1 type index
+# DEBUG-NEXT: G_CTLZ_ZERO_UNDEF (opcode {{[0-9]+}}): 2 type indices
 # DEBUG:      .. type index coverage check SKIPPED: no rules defined
 #
-# DEBUG-NEXT: G_CTPOP (opcode {{[0-9]+}}): 1 type index
+# DEBUG-NEXT: G_CTPOP (opcode {{[0-9]+}}): 2 type indices
 # DEBUG:      .. type index coverage check SKIPPED: no rules defined
 #
 # DEBUG-NEXT: G_BSWAP (opcode {{[0-9]+}}): 1 type index
 # DEBUG:      .. the first uncovered type index: 1, OK
+#
+# DEBUG-NEXT: G_FCEIL (opcode {{[0-9]+}}): 1 type index
+# DEBUG: .. type index coverage check SKIPPED: user-defined predicate detected
+#
+# DEBUG-NEXT: G_FCOS (opcode {{[0-9]+}}): 1 type index
+# DEBUG: .. the first uncovered type index: 1, OK
+#
+# DEBUG-NEXT: G_FSIN (opcode {{[0-9]+}}): 1 type index
+# DEBUG: .. the first uncovered type index: 1, OK
+#
+# DEBUG-NEXT: G_FSQRT (opcode {{[0-9]+}}): 1 type index
+# DEBUG: .. type index coverage check SKIPPED: user-defined predicate detected
+#
+# DEBUG-NEXT: G_FFLOOR (opcode {{[0-9]+}}): 1 type index
+# DEBUG: .. type index coverage check SKIPPED: no rules defined
 
 # CHECK-NOT: ill-defined
 
diff --git a/test/CodeGen/AArch64/GlobalISel/localizer.mir b/test/CodeGen/AArch64/GlobalISel/localizer.mir
index 33a1cf3112e12aed63565f3b589e62da0669cc75..e4648a868f222a5ab2d21f09b4af513d3fb728f3 100644
--- a/test/CodeGen/AArch64/GlobalISel/localizer.mir
+++ b/test/CodeGen/AArch64/GlobalISel/localizer.mir
@@ -24,9 +24,9 @@ regBankSelected: true
 body:             |
   bb.0:
     ; CHECK-LABEL: name: local_use
-    ; CHECK: [[C:%[0-9]+]]:gpr(s32) = G_CONSTANT 1
+    ; CHECK: [[C:%[0-9]+]]:gpr(s32) = G_CONSTANT i32 1
     ; CHECK: [[ADD:%[0-9]+]]:gpr(s32) = G_ADD [[C]], [[C]]
-    %0:gpr(s32) = G_CONSTANT 1
+    %0:gpr(s32) = G_CONSTANT i32 1
     %1:gpr(s32) = G_ADD %0, %0
 ...
 
@@ -38,10 +38,10 @@ body:             |
   ; CHECK-LABEL: name: non_local_1use
   ; CHECK: bb.0:
   ; CHECK:   successors: %bb.1(0x80000000)
-  ; CHECK:   [[C:%[0-9]+]]:gpr(s32) = G_CONSTANT 1
+  ; CHECK:   [[C:%[0-9]+]]:gpr(s32) = G_CONSTANT i32 1
   ; CHECK:   [[ADD:%[0-9]+]]:gpr(s32) = G_ADD [[C]], [[C]]
   ; CHECK: bb.1:
-  ; CHECK:   [[C1:%[0-9]+]]:gpr(s32) = G_CONSTANT 1
+  ; CHECK:   [[C1:%[0-9]+]]:gpr(s32) = G_CONSTANT i32 1
   ; CHECK:   [[ADD1:%[0-9]+]]:gpr(s32) = G_ADD [[C1]], [[ADD]]
 
   ; Existing registers should be left untouched
@@ -50,7 +50,7 @@ body:             |
   bb.0:
     successors: %bb.1
 
-    %0:gpr(s32) = G_CONSTANT 1
+    %0:gpr(s32) = G_CONSTANT i32 1
     %1:gpr(s32) = G_ADD %0, %0
 
   bb.1:
@@ -65,10 +65,10 @@ body:             |
   ; CHECK-LABEL: name: non_local_2uses
   ; CHECK: bb.0:
   ; CHECK:   successors: %bb.1(0x80000000)
-  ; CHECK:   [[C:%[0-9]+]]:gpr(s32) = G_CONSTANT 1
+  ; CHECK:   [[C:%[0-9]+]]:gpr(s32) = G_CONSTANT i32 1
   ; CHECK:   [[ADD:%[0-9]+]]:gpr(s32) = G_ADD [[C]], [[C]]
   ; CHECK: bb.1:
-  ; CHECK:   [[C1:%[0-9]+]]:gpr(s32) = G_CONSTANT 1
+  ; CHECK:   [[C1:%[0-9]+]]:gpr(s32) = G_CONSTANT i32 1
   ; CHECK:   [[ADD1:%[0-9]+]]:gpr(s32) = G_ADD [[C1]], [[C1]]
 
   ; Existing registers should be left untouched
@@ -77,7 +77,7 @@ body:             |
   bb.0:
     successors: %bb.1
 
-    %0:gpr(s32) = G_CONSTANT 1
+    %0:gpr(s32) = G_CONSTANT i32 1
     %1:gpr(s32) = G_ADD %0, %0
 
   bb.1:
@@ -93,11 +93,11 @@ body:             |
   ; CHECK-LABEL: name: non_local_phi_use
   ; CHECK: bb.0:
   ; CHECK:   successors: %bb.1(0x80000000)
-  ; CHECK:   [[C:%[0-9]+]]:gpr(s32) = G_CONSTANT 1
+  ; CHECK:   [[C:%[0-9]+]]:gpr(s32) = G_CONSTANT i32 1
   ; CHECK:   [[ADD:%[0-9]+]]:gpr(s32) = G_ADD [[C]], [[C]]
   ; CHECK: bb.1:
   ; CHECK:   successors: %bb.2(0x80000000)
-  ; CHECK:   [[C1:%[0-9]+]]:gpr(s32) = G_CONSTANT 1
+  ; CHECK:   [[C1:%[0-9]+]]:gpr(s32) = G_CONSTANT i32 1
   ; CHECK: bb.2:
   ; CHECK:   [[PHI:%[0-9]+]]:gpr(s32) = PHI [[C1]](s32), %bb.1
   ; CHECK:   [[ADD1:%[0-9]+]]:gpr(s32) = G_ADD [[PHI]], [[PHI]]
@@ -108,7 +108,7 @@ body:             |
   bb.0:
     successors: %bb.1
 
-    %0:gpr(s32) = G_CONSTANT 1
+    %0:gpr(s32) = G_CONSTANT i32 1
     %1:gpr(s32) = G_ADD %0, %0
 
   bb.1:
@@ -128,14 +128,14 @@ body:             |
   ; CHECK-LABEL: name: non_local_phi_use_followed_by_use
   ; CHECK: bb.0:
   ; CHECK:   successors: %bb.1(0x80000000)
-  ; CHECK:   [[C:%[0-9]+]]:gpr(s32) = G_CONSTANT 1
+  ; CHECK:   [[C:%[0-9]+]]:gpr(s32) = G_CONSTANT i32 1
   ; CHECK:   [[ADD:%[0-9]+]]:gpr(s32) = G_ADD [[C]], [[C]]
   ; CHECK: bb.1:
   ; CHECK:   successors: %bb.2(0x80000000)
-  ; CHECK:   [[C1:%[0-9]+]]:gpr(s32) = G_CONSTANT 1
+  ; CHECK:   [[C1:%[0-9]+]]:gpr(s32) = G_CONSTANT i32 1
   ; CHECK: bb.2:
   ; CHECK:   [[PHI:%[0-9]+]]:gpr(s32) = PHI [[C1]](s32), %bb.1
-  ; CHECK:   [[C2:%[0-9]+]]:gpr(s32) = G_CONSTANT 1
+  ; CHECK:   [[C2:%[0-9]+]]:gpr(s32) = G_CONSTANT i32 1
   ; CHECK:   [[ADD1:%[0-9]+]]:gpr(s32) = G_ADD [[PHI]], [[C2]]
 
   ; Existing registers should be left untouched
@@ -144,7 +144,7 @@ body:             |
   bb.0:
     successors: %bb.1
 
-    %0:gpr(s32) = G_CONSTANT 1
+    %0:gpr(s32) = G_CONSTANT i32 1
     %1:gpr(s32) = G_ADD %0, %0
 
   bb.1:
diff --git a/test/CodeGen/AArch64/GlobalISel/prelegalizercombiner-extending-loads-cornercases.mir b/test/CodeGen/AArch64/GlobalISel/prelegalizercombiner-extending-loads-cornercases.mir
index c41405c5a0dd968b4f6ee1cce39e2b1d7b070015..9128cb88c6b6ee58a5cd9c33c87f8f9edc32879f 100644
--- a/test/CodeGen/AArch64/GlobalISel/prelegalizercombiner-extending-loads-cornercases.mir
+++ b/test/CodeGen/AArch64/GlobalISel/prelegalizercombiner-extending-loads-cornercases.mir
@@ -119,7 +119,7 @@ body: |
 # CHECK-WORKLIST-DAG:   Changed: [[IN3]]:_(s8) = G_ADD [[NEW1:%[0-9]+]]:_, [[IN4]]:_
 # CHECK-WORKLIST-DAG:   Changing: [[IN5:%[0-9]+]]:_(s8) = G_SUB [[IN0]]:_, [[IN6:%[0-9]+]]:_
 # CHECK-WORKLIST-DAG:   Changed: [[IN5]]:_(s8) = G_SUB [[NEW2:%[0-9]+]]:_, [[IN6]]:_
-# CHECK-WORKLIST-DAG:   Erased: [[IN2]]:_(s32) = G_SEXT [[IN0]]:_(s8)
+# CHECK-WORKLIST-DAG:   Erasing: [[IN2]]:_(s32) = G_SEXT [[IN0]]:_(s8)
 # CHECK-WORKLIST-DAG:   Changed: [[IN2]]:_(s32) = G_SEXTLOAD [[IN1]]:_(p0) :: (load 1 from %ir.addr)
 # CHECK-WORKLIST-DAG:   Created: [[NEW1]]:_(s8) = G_TRUNC [[IN2]]:_(s32)
 # CHECK-WORKLIST-DAG:   Created: [[NEW2]]:_(s8) = G_TRUNC [[IN2]]:_(s32)
@@ -220,7 +220,7 @@ body: |
 # CHECK-WORKLIST-DAG:   Creating: G_TRUNC
 # CHECK-WORKLIST-DAG:   Changing: [[IN5:%[0-9]+]]:_(s8) = G_SUB [[IN0]]:_, [[IN6:%[0-9]+]]:_
 # CHECK-WORKLIST-DAG:   Changed: [[IN5]]:_(s8) = G_SUB [[OUT2:%[0-9]+]]:_, [[IN6]]:_
-# CHECK-WORKLIST-DAG:   Erased: [[IN2]]:_(s32) = G_SEXT [[IN0]]:_(s8)
+# CHECK-WORKLIST-DAG:   Erasing: [[IN2]]:_(s32) = G_SEXT [[IN0]]:_(s8)
 # CHECK-WORKLIST-DAG:   Changed: [[IN2]]:_(s32) = G_SEXTLOAD [[IN1]]:_(p0) :: (load 1 from %ir.addr)
 # CHECK-WORKLIST-DAG:   Created: [[OUT1]]:_(s8) = G_TRUNC [[IN2]]:_(s32)
 # CHECK-WORKLIST-DAG:   Created: [[OUT2]]:_(s8) = G_TRUNC [[IN2]]:_(s32)
diff --git a/test/CodeGen/AArch64/GlobalISel/prelegalizercombiner-extending-loads-s1.mir b/test/CodeGen/AArch64/GlobalISel/prelegalizercombiner-extending-loads-s1.mir
new file mode 100644
index 0000000000000000000000000000000000000000..e8decdc21e5b67bf2f9d84dee5ef89cd61623ea8
--- /dev/null
+++ b/test/CodeGen/AArch64/GlobalISel/prelegalizercombiner-extending-loads-s1.mir
@@ -0,0 +1,39 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple aarch64 -O0 -run-pass=aarch64-prelegalizer-combiner -global-isel -verify-machineinstrs %s -o - | FileCheck %s
+
+# Check we don't try to combine a load of < s8 as that will end up creating a illegal non-extending load.
+--- |
+  define i8 @test(i1* %ptr) {
+    ret i8 undef
+  }
+
+...
+---
+name:            test
+alignment:       2
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+  - { id: 2, class: _ }
+  - { id: 3, class: _ }
+body:             |
+  bb.1 (%ir-block.0):
+    liveins: $x0
+
+    ; CHECK-LABEL: name: test
+    ; CHECK: liveins: $x0
+    ; CHECK: [[COPY:%[0-9]+]]:_(p0) = COPY $x0
+    ; CHECK: [[LOAD:%[0-9]+]]:_(s1) = G_LOAD [[COPY]](p0) :: (load 1 from %ir.ptr)
+    ; CHECK: [[ZEXT:%[0-9]+]]:_(s8) = G_ZEXT [[LOAD]](s1)
+    ; CHECK: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[ZEXT]](s8)
+    ; CHECK: $w0 = COPY [[ANYEXT]](s32)
+    ; CHECK: RET_ReallyLR implicit $w0
+    %0:_(p0) = COPY $x0
+    %1:_(s1) = G_LOAD %0(p0) :: (load 1 from %ir.ptr)
+    %2:_(s8) = G_ZEXT %1(s1)
+    %3:_(s32) = G_ANYEXT %2(s8)
+    $w0 = COPY %3(s32)
+    RET_ReallyLR implicit $w0
+
+...
diff --git a/test/CodeGen/AArch64/GlobalISel/prelegalizercombiner-extending-loads.mir b/test/CodeGen/AArch64/GlobalISel/prelegalizercombiner-extending-loads.mir
index 47d7156789f8d7c2458643696ee58a3f783ddb9b..bc5a2eaee87681d54e0c4f5575337896f773be79 100644
--- a/test/CodeGen/AArch64/GlobalISel/prelegalizercombiner-extending-loads.mir
+++ b/test/CodeGen/AArch64/GlobalISel/prelegalizercombiner-extending-loads.mir
@@ -1,4 +1,4 @@
-# RUN: llc -O0 -run-pass=aarch64-prelegalizer-combiner -global-isel %s -o - | FileCheck %s
+# RUN: llc -O0 -run-pass=aarch64-prelegalizer-combiner -global-isel -verify-machineinstrs %s -o - | FileCheck %s
 
 --- |
   target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
@@ -193,7 +193,7 @@ body: |
     ; CHECK: $w1 = COPY [[T1]](s32)
     %0:_(p0) = COPY $x0
     %1:_(s8) = G_LOAD %0 :: (load 1 from %ir.addr)
-    %2:_(s8) = G_CONSTANT i32 -1
+    %2:_(s8) = G_CONSTANT i8 -1
     %3:_(s8) = G_XOR %1, %2
     %5:_(s32) = G_ANYEXT %3
     %6:_(s32) = G_SEXT %1
@@ -218,7 +218,7 @@ body: |
     $w0 = COPY %2
     $w1 = COPY %3
 ...
- 
+
 ---
 name:            test_1signext_1zeroext
 body: |
diff --git a/test/CodeGen/AArch64/GlobalISel/regbank-ceil.ll b/test/CodeGen/AArch64/GlobalISel/regbank-ceil.ll
new file mode 100644
index 0000000000000000000000000000000000000000..b7bc230fb4710e0869f9713bed06287f870d94a0
--- /dev/null
+++ b/test/CodeGen/AArch64/GlobalISel/regbank-ceil.ll
@@ -0,0 +1,16 @@
+; RUN: llc -O=0 -verify-machineinstrs -mtriple aarch64--- \
+; RUN: -stop-before=instruction-select -global-isel %s -o - | FileCheck %s
+
+; Make sure that we choose a FPR for the G_FCEIL and G_LOAD instead of a GPR.
+
+declare float @llvm.ceil.f32(float)
+
+; CHECK-LABEL: name:            foo
+define float @foo(float) {
+  store float %0, float* undef, align 4
+  ; CHECK: %2:fpr(s32) = G_LOAD %1(p0)
+  ; CHECK-NEXT: %3:fpr(s32) = G_FCEIL %2
+  %2 = load float, float* undef, align 4
+  %3 = call float @llvm.ceil.f32(float %2)
+  ret float %3
+}
diff --git a/test/CodeGen/AArch64/GlobalISel/regbankselect-default.mir b/test/CodeGen/AArch64/GlobalISel/regbankselect-default.mir
index dc124096190fc99d4b7c41bf44b3c4858500858a..e226c0fbae7d32d9c6b5294fca2993f74a31d27f 100644
--- a/test/CodeGen/AArch64/GlobalISel/regbankselect-default.mir
+++ b/test/CodeGen/AArch64/GlobalISel/regbankselect-default.mir
@@ -59,14 +59,19 @@
   define void @test_fptrunc_s32_s64() { ret void }
 
   define void @test_fconstant_s32() { ret void }
+  define void @test_fneg_s32() { ret void }
 
   define void @test_fcmp_s32() { ret void }
 
   define void @test_sitofp_s64_s32() { ret void }
+  define void @test_sitofp_v4s32() { ret void }
   define void @test_uitofp_s32_s64() { ret void }
+  define void @test_uitofp_v4s32() { ret void }
 
   define void @test_fptosi_s64_s32() { ret void }
+  define void @test_fptosi_v4s32() { ret void }
   define void @test_fptoui_s32_s64() { ret void }
+  define void @test_fptoui_v4s32() { ret void }
 
   define void @test_gphi_ptr() { ret void }
 
@@ -275,7 +280,7 @@ body: |
     liveins: $w0
     ; CHECK-LABEL: name: test_shl_s32
     ; CHECK: [[COPY:%[0-9]+]]:gpr(s32) = COPY $w0
-    ; CHECK: [[SHL:%[0-9]+]]:gpr(s32) = G_SHL [[COPY]], [[COPY]]
+    ; CHECK: [[SHL:%[0-9]+]]:gpr(s32) = G_SHL [[COPY]], [[COPY]](s32)
     %0(s32) = COPY $w0
     %1(s32) = G_SHL %0, %0
 ...
@@ -291,7 +296,7 @@ body: |
     liveins: $q0
     ; CHECK-LABEL: name: test_shl_v4s32
     ; CHECK: [[COPY:%[0-9]+]]:fpr(<4 x s32>) = COPY $q0
-    ; CHECK: [[SHL:%[0-9]+]]:fpr(<4 x s32>) = G_SHL [[COPY]], [[COPY]]
+    ; CHECK: [[SHL:%[0-9]+]]:fpr(<4 x s32>) = G_SHL [[COPY]], [[COPY]](<4 x s32>)
     %0(<4 x s32>) = COPY $q0
     %1(<4 x s32>) = G_SHL %0, %0
 ...
@@ -307,7 +312,7 @@ body: |
     liveins: $w0
     ; CHECK-LABEL: name: test_lshr_s32
     ; CHECK: [[COPY:%[0-9]+]]:gpr(s32) = COPY $w0
-    ; CHECK: [[LSHR:%[0-9]+]]:gpr(s32) = G_LSHR [[COPY]], [[COPY]]
+    ; CHECK: [[LSHR:%[0-9]+]]:gpr(s32) = G_LSHR [[COPY]], [[COPY]](s32)
     %0(s32) = COPY $w0
     %1(s32) = G_LSHR %0, %0
 ...
@@ -323,7 +328,7 @@ body: |
     liveins: $w0
     ; CHECK-LABEL: name: test_ashr_s32
     ; CHECK: [[COPY:%[0-9]+]]:gpr(s32) = COPY $w0
-    ; CHECK: [[ASHR:%[0-9]+]]:gpr(s32) = G_ASHR [[COPY]], [[COPY]]
+    ; CHECK: [[ASHR:%[0-9]+]]:gpr(s32) = G_ASHR [[COPY]], [[COPY]](s32)
     %0(s32) = COPY $w0
     %1(s32) = G_ASHR %0, %0
 ...
@@ -432,8 +437,8 @@ registers:
 body: |
   bb.0:
     ; CHECK-LABEL: name: test_constant_s32
-    ; CHECK: [[C:%[0-9]+]]:gpr(s32) = G_CONSTANT 123
-    %0(s32) = G_CONSTANT 123
+    ; CHECK: [[C:%[0-9]+]]:gpr(s32) = G_CONSTANT i32 123
+    %0(s32) = G_CONSTANT i32 123
 ...
 
 ---
@@ -444,8 +449,8 @@ registers:
 body: |
   bb.0:
     ; CHECK-LABEL: name: test_constant_p0
-    ; CHECK: [[C:%[0-9]+]]:gpr(p0) = G_CONSTANT 0
-    %0(p0) = G_CONSTANT 0
+    ; CHECK: [[C:%[0-9]+]]:gpr(p0) = G_CONSTANT i64 0
+    %0(p0) = G_CONSTANT i64 0
 ...
 
 ---
@@ -674,6 +679,19 @@ body: |
     %0(s32) = G_FCONSTANT float 1.0
 ...
 
+---
+name:            test_fneg_s32
+legalized:       true
+body: |
+  bb.0:
+  liveins: $s0
+    ; CHECK-LABEL: name: test_fneg_s32
+    ; CHECK: [[COPY:%[0-9]+]]:fpr(s32) = COPY $s0
+    ; CHECK: [[FNEG:%[0-9]+]]:fpr(s32) = G_FNEG [[COPY]]
+    %0:_(s32) = COPY $s0
+    %1:_(s32) = G_FNEG %0(s32)
+...
+
 ---
 name:            test_fcmp_s32
 legalized:       true
@@ -709,6 +727,22 @@ body: |
     %1(s64) = G_SITOFP %0
 ...
 
+---
+name:            test_sitofp_v4s32
+legalized:       true
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+body: |
+  bb.0:
+    liveins: $q0
+    ; CHECK-LABEL: name: test_sitofp_v4s32
+    ; CHECK: [[COPY:%[0-9]+]]:fpr(<4 x s32>) = COPY $q0
+    ; CHECK: [[SITOFP:%[0-9]+]]:fpr(<4 x s32>) = G_SITOFP [[COPY]](<4 x s32>)
+    %0(<4 x s32>) = COPY $q0
+    %1(<4 x s32>) = G_SITOFP %0
+...
+
 ---
 name:            test_uitofp_s32_s64
 legalized:       true
@@ -725,6 +759,22 @@ body: |
     %1(s32) = G_UITOFP %0
 ...
 
+---
+name:            test_uitofp_v4s32
+legalized:       true
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+body: |
+  bb.0:
+    liveins: $q0
+    ; CHECK-LABEL: name: test_uitofp_v4s32
+    ; CHECK: [[COPY:%[0-9]+]]:fpr(<4 x s32>) = COPY $q0
+    ; CHECK: [[UITOFP:%[0-9]+]]:fpr(<4 x s32>) = G_UITOFP [[COPY]](<4 x s32>)
+    %0(<4 x s32>) = COPY $q0
+    %1(<4 x s32>) = G_UITOFP %0
+...
+
 ---
 name:            test_fptosi_s64_s32
 legalized:       true
@@ -741,6 +791,22 @@ body: |
     %1(s64) = G_FPTOSI %0
 ...
 
+---
+name:            test_fptosi_v4s32
+legalized:       true
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+body: |
+  bb.0:
+    liveins: $q0
+    ; CHECK-LABEL: name: test_fptosi_v4s32
+    ; CHECK: [[COPY:%[0-9]+]]:fpr(<4 x s32>) = COPY $q0
+    ; CHECK: [[FPTOSI:%[0-9]+]]:fpr(<4 x s32>) = G_FPTOSI [[COPY]](<4 x s32>)
+    %0(<4 x s32>) = COPY $q0
+    %1(<4 x s32>) = G_FPTOSI %0
+...
+
 ---
 name:            test_fptoui_s32_s64
 legalized:       true
@@ -757,6 +823,22 @@ body: |
     %1(s32) = G_FPTOUI %0
 ...
 
+---
+name:            test_fptoui_v4s32
+legalized:       true
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+body: |
+  bb.0:
+    liveins: $q0
+    ; CHECK-LABEL: name: test_fptoui_v4s32
+    ; CHECK: [[COPY:%[0-9]+]]:fpr(<4 x s32>) = COPY $q0
+    ; CHECK: [[FPTOUI:%[0-9]+]]:fpr(<4 x s32>) = G_FPTOUI [[COPY]](<4 x s32>)
+    %0(<4 x s32>) = COPY $q0
+    %1(<4 x s32>) = G_FPTOUI %0
+...
+
 ---
 name:            test_gphi_ptr
 legalized:       true
diff --git a/test/CodeGen/AArch64/GlobalISel/regbankselect-unmerge-vec.mir b/test/CodeGen/AArch64/GlobalISel/regbankselect-unmerge-vec.mir
new file mode 100644
index 0000000000000000000000000000000000000000..3123de27c1e20017e7e0504eda2ccfdf2eba5903
--- /dev/null
+++ b/test/CodeGen/AArch64/GlobalISel/regbankselect-unmerge-vec.mir
@@ -0,0 +1,26 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -O0 -mtriple arm64-- -run-pass=regbankselect -verify-machineinstrs %s -o - | FileCheck %s
+---
+name:            unmerge
+alignment:       2
+legalized:       true
+tracksRegLiveness: true
+frameInfo:
+  maxCallFrameSize: 0
+body:             |
+  bb.0:
+    liveins: $q0
+
+    ; Ensure that the dest regs have FPR since we're unmerging from a vector
+    ; CHECK-LABEL: name: unmerge
+    ; CHECK: liveins: $q0
+    ; CHECK: [[COPY:%[0-9]+]]:fpr(<2 x s64>) = COPY $q0
+    ; CHECK: [[UV:%[0-9]+]]:fpr(s64), [[UV1:%[0-9]+]]:fpr(s64) = G_UNMERGE_VALUES [[COPY]](<2 x s64>)
+    ; CHECK: $x0 = COPY [[UV]](s64)
+    ; CHECK: RET_ReallyLR implicit $x0
+    %0:_(<2 x s64>) = COPY $q0
+    %1:_(s64), %2:_(s64) = G_UNMERGE_VALUES %0(<2 x s64>)
+    $x0 = COPY %1(s64)
+    RET_ReallyLR implicit $x0
+
+...
diff --git a/test/CodeGen/AArch64/GlobalISel/select-binop.mir b/test/CodeGen/AArch64/GlobalISel/select-binop.mir
index f836a149fb2251e44375f7e16bf3cc9c3ebce83f..8c858bcdb3ee902f382ce8c21293034ea94be369 100644
--- a/test/CodeGen/AArch64/GlobalISel/select-binop.mir
+++ b/test/CodeGen/AArch64/GlobalISel/select-binop.mir
@@ -152,7 +152,7 @@ body:             |
     ; CHECK: [[ADDXri:%[0-9]+]]:gpr64sp = ADDXri [[COPY]], 1, 0
     ; CHECK: $x0 = COPY [[ADDXri]]
     %0(s64) = COPY $x0
-    %1(s64) = G_CONSTANT i32 1
+    %1(s64) = G_CONSTANT i64 1
     %2(s64) = G_ADD %0, %1
     $x0 = COPY %2(s64)
 ...
diff --git a/test/CodeGen/AArch64/GlobalISel/select-ceil.mir b/test/CodeGen/AArch64/GlobalISel/select-ceil.mir
new file mode 100644
index 0000000000000000000000000000000000000000..9bacb7c75b9b36961b808db7afa046a7faeaab32
--- /dev/null
+++ b/test/CodeGen/AArch64/GlobalISel/select-ceil.mir
@@ -0,0 +1,130 @@
+# RUN: llc -verify-machineinstrs -mtriple aarch64--- \
+# RUN: -run-pass=instruction-select -mattr=+fullfp16 -global-isel %s -o - \
+# RUN: | FileCheck %s
+...
+---
+name:            ceil_float
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: fpr }
+  - { id: 1, class: fpr }
+body:             |
+  bb.0:
+    ; CHECK-LABEL: name:            ceil_float
+    ; CHECK: %{{[0-9]+}}:fpr32 = FRINTPSr %{{[0-9]+}}
+    liveins: $s0
+    %0:fpr(s32) = COPY $s0
+    %1:fpr(s32) = G_FCEIL %0
+    $s0 = COPY %1(s32)
+
+...
+---
+name:            ceil_double
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: fpr }
+  - { id: 1, class: fpr }
+body:             |
+  bb.0:
+    ; CHECK-LABEL: name:            ceil_double
+    ; CHECK: %{{[0-9]+}}:fpr64 = FRINTPDr %{{[0-9]+}}
+    liveins: $d0
+    %0:fpr(s64) = COPY $d0
+    %1:fpr(s64) = G_FCEIL %0
+    $d0 = COPY %1(s64)
+
+...
+---
+name:            ceil_v2f32
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: fpr }
+  - { id: 1, class: fpr }
+body:             |
+  bb.0:
+    ; CHECK-LABEL: name:            ceil_v2f32
+    ; CHECK: %{{[0-9]+}}:fpr64 = FRINTPv2f32 %{{[0-9]+}}
+    liveins: $d0
+    %0:fpr(<2 x s32>) = COPY $d0
+    %1:fpr(<2 x s32>) = G_FCEIL %0
+    $d0 = COPY %1(<2 x s32>)
+
+...
+---
+name:            ceil_v4f32
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: fpr }
+  - { id: 1, class: fpr }
+body:             |
+  bb.0:
+    ; CHECK-LABEL: name:            ceil_v4f32
+    ; CHECK: %{{[0-9]+}}:fpr128 = FRINTPv4f32 %{{[0-9]+}}
+    liveins: $q0
+    %0:fpr(<4 x s32>) = COPY $q0
+    %1:fpr(<4 x s32>) = G_FCEIL %0
+    $q0 = COPY %1(<4 x s32>)
+
+...
+---
+name:            ceil_v2f64
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: fpr }
+  - { id: 1, class: fpr }
+body:             |
+  bb.0:
+    ; CHECK-LABEL: name:            ceil_v2f64
+    ; CHECK: %{{[0-9]+}}:fpr128 = FRINTPv2f64 %{{[0-9]+}}
+    liveins: $q0
+    %0:fpr(<2 x s64>) = COPY $q0
+    %1:fpr(<2 x s64>) = G_FCEIL %0
+    $q0 = COPY %1(<2 x s64>)
+
+...
+---
+name:            ceil_v4f16
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: fpr }
+  - { id: 1, class: fpr }
+body:             |
+  bb.0:
+    ; CHECK-LABEL: name:            ceil_v4f16
+    ; CHECK: %{{[0-9]+}}:fpr64 = FRINTPv4f16 %{{[0-9]+}}
+    liveins: $d0
+    %0:fpr(<4 x s16>) = COPY $d0
+    %1:fpr(<4 x s16>) = G_FCEIL %0
+    $d0 = COPY %1(<4 x s16>)
+
+...
+---
+name:            ceil_v8f16
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: fpr }
+  - { id: 1, class: fpr }
+body:             |
+  bb.0:
+    ; CHECK-LABEL: name:            ceil_v8f16
+    ; CHECK: %{{[0-9]+}}:fpr128 = FRINTPv8f16 %{{[0-9]+}}
+    liveins: $q0
+    %0:fpr(<8 x s16>) = COPY $q0
+    %1:fpr(<8 x s16>) = G_FCEIL %0
+    $q0 = COPY %1(<8 x s16>)
+
+...
diff --git a/test/CodeGen/AArch64/GlobalISel/select-fabs.mir b/test/CodeGen/AArch64/GlobalISel/select-fabs.mir
new file mode 100644
index 0000000000000000000000000000000000000000..e7d74cc4baa605433c58a061a7d9bb68c6e49917
--- /dev/null
+++ b/test/CodeGen/AArch64/GlobalISel/select-fabs.mir
@@ -0,0 +1,130 @@
+# RUN: llc -verify-machineinstrs -mtriple aarch64--- \
+# RUN: -run-pass=instruction-select -mattr=+fullfp16 -global-isel %s -o - \
+# RUN: | FileCheck %s
+...
+---
+name:            fabs_float
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: fpr }
+  - { id: 1, class: fpr }
+body:             |
+  bb.0:
+    ; CHECK-LABEL: name:            fabs_float
+    ; CHECK: %{{[0-9]+}}:fpr32 = FABSSr %{{[0-9]+}}
+    liveins: $s0
+    %0:fpr(s32) = COPY $s0
+    %1:fpr(s32) = G_FABS %0
+    $s0 = COPY %1(s32)
+
+...
+---
+name:            fabs_double
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: fpr }
+  - { id: 1, class: fpr }
+body:             |
+  bb.0:
+    ; CHECK-LABEL: name:            fabs_double
+    ; CHECK: %{{[0-9]+}}:fpr64 = FABSDr %{{[0-9]+}}
+    liveins: $d0
+    %0:fpr(s64) = COPY $d0
+    %1:fpr(s64) = G_FABS %0
+    $d0 = COPY %1(s64)
+
+...
+---
+name:            fabs_v2f32
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: fpr }
+  - { id: 1, class: fpr }
+body:             |
+  bb.0:
+    ; CHECK-LABEL: name:            fabs_v2f32
+    ; CHECK: %{{[0-9]+}}:fpr64 = FABSv2f32 %{{[0-9]+}}
+    liveins: $d0
+    %0:fpr(<2 x s32>) = COPY $d0
+    %1:fpr(<2 x s32>) = G_FABS %0
+    $d0 = COPY %1(<2 x s32>)
+
+...
+---
+name:            fabs_v4f32
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: fpr }
+  - { id: 1, class: fpr }
+body:             |
+  bb.0:
+    ; CHECK-LABEL: name:            fabs_v4f32
+    ; CHECK: %{{[0-9]+}}:fpr128 = FABSv4f32 %{{[0-9]+}}
+    liveins: $q0
+    %0:fpr(<4 x s32>) = COPY $q0
+    %1:fpr(<4 x s32>) = G_FABS %0
+    $q0 = COPY %1(<4 x s32>)
+
+...
+---
+name:            fabs_v2f64
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: fpr }
+  - { id: 1, class: fpr }
+body:             |
+  bb.0:
+    ; CHECK-LABEL: name:            fabs_v2f64
+    ; CHECK: %{{[0-9]+}}:fpr128 = FABSv2f64 %{{[0-9]+}}
+    liveins: $q0
+    %0:fpr(<2 x s64>) = COPY $q0
+    %1:fpr(<2 x s64>) = G_FABS %0
+    $q0 = COPY %1(<2 x s64>)
+
+...
+---
+name:            fabs_v4f16
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: fpr }
+  - { id: 1, class: fpr }
+body:             |
+  bb.0:
+    ; CHECK-LABEL: name:            fabs_v4f16
+    ; CHECK: %{{[0-9]+}}:fpr64 = FABSv4f16 %{{[0-9]+}}
+    liveins: $d0
+    %0:fpr(<4 x s16>) = COPY $d0
+    %1:fpr(<4 x s16>) = G_FABS %0
+    $d0 = COPY %1(<4 x s16>)
+
+...
+---
+name:            fabs_v8f16
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: fpr }
+  - { id: 1, class: fpr }
+body:             |
+  bb.0:
+    ; CHECK-LABEL: name:            fabs_v8f16
+    ; CHECK: %{{[0-9]+}}:fpr128 = FABSv8f16 %{{[0-9]+}}
+    liveins: $q0
+    %0:fpr(<8 x s16>) = COPY $q0
+    %1:fpr(<8 x s16>) = G_FABS %0
+    $q0 = COPY %1(<8 x s16>)
+
+...
diff --git a/test/CodeGen/AArch64/GlobalISel/select-floor.mir b/test/CodeGen/AArch64/GlobalISel/select-floor.mir
new file mode 100644
index 0000000000000000000000000000000000000000..23b95cf844f8abb5cdb2714764ac01677342da57
--- /dev/null
+++ b/test/CodeGen/AArch64/GlobalISel/select-floor.mir
@@ -0,0 +1,130 @@
+# RUN: llc -verify-machineinstrs -mtriple aarch64--- \
+# RUN: -run-pass=instruction-select -mattr=+fullfp16 -global-isel %s -o - \
+# RUN: | FileCheck %s
+...
+---
+name:            floor_float
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: fpr }
+  - { id: 1, class: fpr }
+body:             |
+  bb.0:
+    ; CHECK-LABEL: name:            floor_float
+    ; CHECK: %{{[0-9]+}}:fpr32 = FRINTMSr %{{[0-9]+}}
+    liveins: $s0
+    %0:fpr(s32) = COPY $s0
+    %1:fpr(s32) = G_FFLOOR %0
+    $s0 = COPY %1(s32)
+
+...
+---
+name:            floor_double
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: fpr }
+  - { id: 1, class: fpr }
+body:             |
+  bb.0:
+    ; CHECK-LABEL: name:            floor_double
+    ; CHECK: %{{[0-9]+}}:fpr64 = FRINTMDr %{{[0-9]+}}
+    liveins: $d0
+    %0:fpr(s64) = COPY $d0
+    %1:fpr(s64) = G_FFLOOR %0
+    $d0 = COPY %1(s64)
+
+...
+---
+name:            floor_v2f32
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: fpr }
+  - { id: 1, class: fpr }
+body:             |
+  bb.0:
+    ; CHECK-LABEL: name:            floor_v2f32
+    ; CHECK: %{{[0-9]+}}:fpr64 = FRINTMv2f32 %{{[0-9]+}}
+    liveins: $d0
+    %0:fpr(<2 x s32>) = COPY $d0
+    %1:fpr(<2 x s32>) = G_FFLOOR %0
+    $d0 = COPY %1(<2 x s32>)
+
+...
+---
+name:            floor_v4f32
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: fpr }
+  - { id: 1, class: fpr }
+body:             |
+  bb.0:
+    ; CHECK-LABEL: name:            floor_v4f32
+    ; CHECK: %{{[0-9]+}}:fpr128 = FRINTMv4f32 %{{[0-9]+}}
+    liveins: $q0
+    %0:fpr(<4 x s32>) = COPY $q0
+    %1:fpr(<4 x s32>) = G_FFLOOR %0
+    $q0 = COPY %1(<4 x s32>)
+
+...
+---
+name:            floor_v2f64
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: fpr }
+  - { id: 1, class: fpr }
+body:             |
+  bb.0:
+    ; CHECK-LABEL: name:            floor_v2f64
+    ; CHECK: %{{[0-9]+}}:fpr128 = FRINTMv2f64 %{{[0-9]+}}
+    liveins: $q0
+    %0:fpr(<2 x s64>) = COPY $q0
+    %1:fpr(<2 x s64>) = G_FFLOOR %0
+    $q0 = COPY %1(<2 x s64>)
+
+...
+---
+name:            floor_v4f16
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: fpr }
+  - { id: 1, class: fpr }
+body:             |
+  bb.0:
+    ; CHECK-LABEL: name:            floor_v4f16
+    ; CHECK: %{{[0-9]+}}:fpr64 = FRINTMv4f16 %{{[0-9]+}}
+    liveins: $d0
+    %0:fpr(<4 x s16>) = COPY $d0
+    %1:fpr(<4 x s16>) = G_FFLOOR %0
+    $d0 = COPY %1(<4 x s16>)
+
+...
+---
+name:            floor_v8f16
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: fpr }
+  - { id: 1, class: fpr }
+body:             |
+  bb.0:
+    ; CHECK-LABEL: name:            floor_v8f16
+    ; CHECK: %{{[0-9]+}}:fpr128 = FRINTMv8f16 %{{[0-9]+}}
+    liveins: $q0
+    %0:fpr(<8 x s16>) = COPY $q0
+    %1:fpr(<8 x s16>) = G_FFLOOR %0
+    $q0 = COPY %1(<8 x s16>)
+
+...
diff --git a/test/CodeGen/AArch64/GlobalISel/select-fp-casts.mir b/test/CodeGen/AArch64/GlobalISel/select-fp-casts.mir
index c186ccc3649713aacefe58c6970533aab866a790..aea10c5c6c9dd5faf5a159e16ae5e646a7e8ea08 100644
--- a/test/CodeGen/AArch64/GlobalISel/select-fp-casts.mir
+++ b/test/CodeGen/AArch64/GlobalISel/select-fp-casts.mir
@@ -1,38 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
 # RUN: llc -mtriple=aarch64-- -run-pass=instruction-select -verify-machineinstrs %s -o - | FileCheck %s
 
---- |
-  target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
-
-  define void @fptrunc_s16_s32_fpr() { ret void }
-  define void @fptrunc_s16_s64_fpr() { ret void }
-  define void @fptrunc_s32_s64_fpr() { ret void }
-
-  define void @fpext_s32_s16_fpr() { ret void }
-  define void @fpext_s64_s16_fpr() { ret void }
-  define void @fpext_s64_s32_fpr() { ret void }
-
-  define void @sitofp_s32_s32_fpr() { ret void }
-  define void @sitofp_s32_s64_fpr() { ret void }
-  define void @sitofp_s64_s32_fpr() { ret void }
-  define void @sitofp_s64_s64_fpr() { ret void }
-
-  define void @uitofp_s32_s32_fpr() { ret void }
-  define void @uitofp_s32_s64_fpr() { ret void }
-  define void @uitofp_s64_s32_fpr() { ret void }
-  define void @uitofp_s64_s64_fpr() { ret void }
-
-  define void @fptosi_s32_s32_gpr() { ret void }
-  define void @fptosi_s32_s64_gpr() { ret void }
-  define void @fptosi_s64_s32_gpr() { ret void }
-  define void @fptosi_s64_s64_gpr() { ret void }
-
-  define void @fptoui_s32_s32_gpr() { ret void }
-  define void @fptoui_s32_s64_gpr() { ret void }
-  define void @fptoui_s64_s32_gpr() { ret void }
-  define void @fptoui_s64_s64_gpr() { ret void }
-...
-
 ---
 name:            fptrunc_s16_s32_fpr
 legalized:       true
@@ -99,6 +67,48 @@ body:             |
     $s0 = COPY %1(s32)
 ...
 
+---
+name:            fptrunc_v4s16_v4s32_fpr
+legalized:       true
+regBankSelected: true
+
+registers:
+  - { id: 0, class: fpr }
+  - { id: 1, class: fpr }
+
+body:             |
+  bb.0:
+    liveins: $d0
+    ; CHECK-LABEL: name: fptrunc_v4s16_v4s32_fpr
+    ; CHECK: [[COPY:%[0-9]+]]:fpr128 = COPY $q0
+    ; CHECK: [[FCVTNv4i16:%[0-9]+]]:fpr64 = FCVTNv4i16 [[COPY]]
+    ; CHECK: $d0 = COPY [[FCVTNv4i16]]
+    %0(<4 x s32>) = COPY $q0
+    %1(<4 x s16>) = G_FPTRUNC %0
+    $d0 = COPY %1(<4 x s16>)
+...
+
+---
+name:            fptrunc_v2s32_v2s64_fpr
+legalized:       true
+regBankSelected: true
+
+registers:
+  - { id: 0, class: fpr }
+  - { id: 1, class: fpr }
+
+body:             |
+  bb.0:
+    liveins: $q0
+    ; CHECK-LABEL: name: fptrunc_v2s32_v2s64_fpr
+    ; CHECK: [[COPY:%[0-9]+]]:fpr128 = COPY $q0
+    ; CHECK: [[FCVTNv2i32:%[0-9]+]]:fpr64 = FCVTNv2i32 [[COPY]]
+    ; CHECK: $d0 = COPY [[FCVTNv2i32]]
+    %0(<2 x s64>) = COPY $q0
+    %1(<2 x s32>) = G_FPTRUNC %0
+    $d0 = COPY %1(<2 x s32>)
+...
+
 ---
 name:            fpext_s32_s16_fpr
 legalized:       true
@@ -165,6 +175,48 @@ body:             |
     $d0 = COPY %1(s64)
 ...
 
+---
+name:            fpext_v4s32_v4s16_fpr
+legalized:       true
+regBankSelected: true
+
+registers:
+  - { id: 0, class: fpr }
+  - { id: 1, class: fpr }
+
+body:             |
+  bb.0:
+    liveins: $d0
+    ; CHECK-LABEL: name: fpext_v4s32_v4s16_fpr
+    ; CHECK: [[COPY:%[0-9]+]]:fpr64 = COPY $d0
+    ; CHECK: [[FCVTLv4i16:%[0-9]+]]:fpr128 = FCVTLv4i16 [[COPY]]
+    ; CHECK: $q0 = COPY [[FCVTLv4i16]]
+    %0(<4 x s16>) = COPY $d0
+    %1(<4 x s32>) = G_FPEXT %0
+    $q0 = COPY %1(<4 x s32>)
+...
+
+---
+name:            fpext_v2s64_v2s32_fpr
+legalized:       true
+regBankSelected: true
+
+registers:
+  - { id: 0, class: fpr }
+  - { id: 1, class: fpr }
+
+body:             |
+  bb.0:
+    liveins: $d0
+    ; CHECK-LABEL: name: fpext_v2s64_v2s32_fpr
+    ; CHECK: [[COPY:%[0-9]+]]:fpr64 = COPY $d0
+    ; CHECK: [[FCVTLv2i32:%[0-9]+]]:fpr128 = FCVTLv2i32 [[COPY]]
+    ; CHECK: $q0 = COPY [[FCVTLv2i32]]
+    %0(<2 x s32>) = COPY $d0
+    %1(<2 x s64>) = G_FPEXT %0
+    $q0 = COPY %1(<2 x s64>)
+...
+
 ---
 name:            sitofp_s32_s32_fpr
 legalized:       true
diff --git a/test/CodeGen/AArch64/GlobalISel/select-scalar-merge.mir b/test/CodeGen/AArch64/GlobalISel/select-scalar-merge.mir
new file mode 100644
index 0000000000000000000000000000000000000000..dafeee4d981f8c5d53169162e5c544c9857fe9b1
--- /dev/null
+++ b/test/CodeGen/AArch64/GlobalISel/select-scalar-merge.mir
@@ -0,0 +1,34 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -O0 -mtriple=aarch64-- -run-pass=instruction-select -verify-machineinstrs %s -o - | FileCheck %s
+
+--- |
+  target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
+
+  define void @gmerge_s64_s32() { ret void }
+...
+
+---
+name:            gmerge_s64_s32
+legalized:       true
+regBankSelected: true
+registers:
+  - { id: 0, class: gpr }
+  - { id: 1, class: gpr }
+  - { id: 2, class: gpr }
+
+body:             |
+  bb.0:
+    liveins: $w0, $w1
+
+    ; CHECK-LABEL: name: gmerge_s64_s32
+    ; CHECK: [[COPY:%[0-9]+]]:gpr32all = COPY $w0
+    ; CHECK: [[COPY1:%[0-9]+]]:gpr32all = COPY $w1
+    ; CHECK: [[SUBREG_TO_REG:%[0-9]+]]:gpr64 = SUBREG_TO_REG 0, [[COPY]], %subreg.sub_32
+    ; CHECK: [[SUBREG_TO_REG1:%[0-9]+]]:gpr64 = SUBREG_TO_REG 0, [[COPY1]], %subreg.sub_32
+    ; CHECK: [[BFMXri:%[0-9]+]]:gpr64 = BFMXri [[SUBREG_TO_REG]], [[SUBREG_TO_REG1]], 32, 31
+    ; CHECK: $x0 = COPY [[BFMXri]]
+    %0(s32) = COPY $w0
+    %1(s32) = COPY $w1
+    %2(s64) = G_MERGE_VALUES %0(s32), %1(s32)
+    $x0 = COPY %2(s64)
+...
diff --git a/test/CodeGen/AArch64/GlobalISel/select-sqrt.mir b/test/CodeGen/AArch64/GlobalISel/select-sqrt.mir
new file mode 100644
index 0000000000000000000000000000000000000000..250fc213f830e4f2501fcd87fbbdef1e85ba08cb
--- /dev/null
+++ b/test/CodeGen/AArch64/GlobalISel/select-sqrt.mir
@@ -0,0 +1,130 @@
+# RUN: llc -verify-machineinstrs -mtriple aarch64--- \
+# RUN: -run-pass=instruction-select -mattr=+fullfp16 -global-isel %s -o - \
+# RUN: | FileCheck %s
+...
+---
+name:            sqrt_float
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: fpr }
+  - { id: 1, class: fpr }
+body:             |
+  bb.0:
+    ; CHECK-LABEL: name:            sqrt_float
+    ; CHECK: %{{[0-9]+}}:fpr32 = FSQRTSr %{{[0-9]+}}
+    liveins: $s0
+    %0:fpr(s32) = COPY $s0
+    %1:fpr(s32) = G_FSQRT %0
+    $s0 = COPY %1(s32)
+
+...
+---
+name:            sqrt_double
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: fpr }
+  - { id: 1, class: fpr }
+body:             |
+  bb.0:
+    ; CHECK-LABEL: name:            sqrt_double
+    ; CHECK: %{{[0-9]+}}:fpr64 = FSQRTDr %{{[0-9]+}}
+    liveins: $d0
+    %0:fpr(s64) = COPY $d0
+    %1:fpr(s64) = G_FSQRT %0
+    $d0 = COPY %1(s64)
+
+...
+---
+name:            sqrt_v2f32
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: fpr }
+  - { id: 1, class: fpr }
+body:             |
+  bb.0:
+    ; CHECK-LABEL: name:            sqrt_v2f32
+    ; CHECK: %{{[0-9]+}}:fpr64 = FSQRTv2f32 %{{[0-9]+}}
+    liveins: $d0
+    %0:fpr(<2 x s32>) = COPY $d0
+    %1:fpr(<2 x s32>) = G_FSQRT %0
+    $d0 = COPY %1(<2 x s32>)
+
+...
+---
+name:            sqrt_v4f32
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: fpr }
+  - { id: 1, class: fpr }
+body:             |
+  bb.0:
+    ; CHECK-LABEL: name:            sqrt_v4f32
+    ; CHECK: %{{[0-9]+}}:fpr128 = FSQRTv4f32 %{{[0-9]+}}
+    liveins: $q0
+    %0:fpr(<4 x s32>) = COPY $q0
+    %1:fpr(<4 x s32>) = G_FSQRT %0
+    $q0 = COPY %1(<4 x s32>)
+
+...
+---
+name:            sqrt_v2f64
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: fpr }
+  - { id: 1, class: fpr }
+body:             |
+  bb.0:
+    ; CHECK-LABEL: name:            sqrt_v2f64
+    ; CHECK: %{{[0-9]+}}:fpr128 = FSQRTv2f64 %{{[0-9]+}}
+    liveins: $q0
+    %0:fpr(<2 x s64>) = COPY $q0
+    %1:fpr(<2 x s64>) = G_FSQRT %0
+    $q0 = COPY %1(<2 x s64>)
+
+...
+---
+name:            sqrt_v4f16
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: fpr }
+  - { id: 1, class: fpr }
+body:             |
+  bb.0:
+    ; CHECK-LABEL: name:            sqrt_v4f16
+    ; CHECK: %{{[0-9]+}}:fpr64 = FSQRTv4f16 %{{[0-9]+}}
+    liveins: $d0
+    %0:fpr(<4 x s16>) = COPY $d0
+    %1:fpr(<4 x s16>) = G_FSQRT %0
+    $d0 = COPY %1(<4 x s16>)
+
+...
+---
+name:            sqrt_v8f16
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: fpr }
+  - { id: 1, class: fpr }
+body:             |
+  bb.0:
+    ; CHECK-LABEL: name:            sqrt_v8f16
+    ; CHECK: %{{[0-9]+}}:fpr128 = FSQRTv8f16 %{{[0-9]+}}
+    liveins: $q0
+    %0:fpr(<8 x s16>) = COPY $q0
+    %1:fpr(<8 x s16>) = G_FSQRT %0
+    $q0 = COPY %1(<8 x s16>)
+
+...
diff --git a/test/CodeGen/AArch64/GlobalISel/select-unmerge.mir b/test/CodeGen/AArch64/GlobalISel/select-unmerge.mir
new file mode 100644
index 0000000000000000000000000000000000000000..6814b99339498cc91fa6ba2d0e9e6e77c8bc2b76
--- /dev/null
+++ b/test/CodeGen/AArch64/GlobalISel/select-unmerge.mir
@@ -0,0 +1,154 @@
+
+# RUN: llc -O0 -mattr=-fullfp16 -mtriple=aarch64-- \
+# RUN: -run-pass=instruction-select -verify-machineinstrs %s -o - | FileCheck %s
+
+--- |
+  define <2 x double> @test_v2s64_unmerge(<2 x double> %a) {
+    ret <2 x double> %a
+  }
+
+  define <4 x float> @test_v4s32_unmerge(<4 x float> %a) {
+    ret <4 x float> %a
+  }
+
+  define <4 x half> @test_v4s16_unmerge(<4 x half> %a) {
+    ret <4 x half> %a
+  }
+
+  define <8 x half> @test_v8s16_unmerge(<8 x half> %a) {
+    ret <8 x half> %a
+  }
+
+...
+---
+name:            test_v2s64_unmerge
+alignment:       2
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: fpr }
+  - { id: 1, class: fpr }
+  - { id: 2, class: fpr }
+  - { id: 3, class: fpr }
+body:             |
+  bb.1 (%ir-block.0):
+    liveins: $q0
+    ; CHECK-LABEL: name:            test_v2s64_unmerge
+    %0:fpr(<2 x s64>) = COPY $q0
+
+    ; Since 2 * 64 = 128, we can just directly copy.
+    ; CHECK: %2:fpr64 = COPY %0.dsub
+    ; CHECK: %3:fpr64 = CPYi64 %0, 1
+    %2:fpr(s64), %3:fpr(s64) = G_UNMERGE_VALUES %0(<2 x s64>)
+
+    %1:fpr(<2 x s64>) = G_BUILD_VECTOR %2(s64), %3(s64)
+    $q0 = COPY %1(<2 x s64>)
+    RET_ReallyLR implicit $q0
+...
+---
+name:            test_v4s32_unmerge
+alignment:       2
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: fpr }
+  - { id: 1, class: fpr }
+  - { id: 2, class: fpr }
+  - { id: 3, class: fpr }
+  - { id: 4, class: fpr }
+  - { id: 5, class: fpr }
+body:             |
+  bb.1 (%ir-block.0):
+    liveins: $q0
+    ; CHECK-LABEL: name:            test_v4s32_unmerge
+    %0:fpr(<4 x s32>) = COPY $q0
+
+    ; Since 4 * 32 = 128, we can just directly copy.
+    ; CHECK: %2:fpr32 = COPY %0.ssub
+    ; CHECK: %3:fpr32 = CPYi32 %0, 1
+    ; CHECK: %4:fpr32 = CPYi32 %0, 2
+    ; CHECK: %5:fpr32 = CPYi32 %0, 3
+    %2:fpr(s32), %3:fpr(s32), %4:fpr(s32), %5:fpr(s32) = G_UNMERGE_VALUES %0(<4 x s32>)
+
+    %1:fpr(<4 x s32>) = G_BUILD_VECTOR %2(s32), %3(s32), %4(s32), %5(s32)
+    $q0 = COPY %1(<4 x s32>)
+    RET_ReallyLR implicit $q0
+...
+---
+name:            test_v4s16_unmerge
+alignment:       2
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: fpr }
+  - { id: 1, class: fpr }
+  - { id: 2, class: fpr }
+  - { id: 3, class: fpr }
+  - { id: 4, class: fpr }
+  - { id: 5, class: fpr }
+body:             |
+  bb.1 (%ir-block.0):
+    liveins: $d0
+    ; CHECK-LABEL: name:            test_v4s16_unmerge
+    %0:fpr(<4 x s16>) = COPY $d0
+
+    ; Since 4 * 16 != 128, we need to widen using implicit defs.
+    ; Note that we expect to reuse one of the INSERT_SUBREG results, as CPYi16
+    ; expects a lane > 0.
+    ; CHECK-DAG: [[IMPDEF1:%[0-9]+]]:fpr128 = IMPLICIT_DEF
+    ; CHECK-NEXT: [[INS_SHARED:%[0-9]+]]:fpr128 = INSERT_SUBREG [[IMPDEF1]], %0, %subreg.dsub
+    ; CHECK: [[IMPDEF2:%[0-9]+]]:fpr128 = IMPLICIT_DEF
+    ; CHECK-NEXT: [[INS2:%[0-9]+]]:fpr128 = INSERT_SUBREG [[IMPDEF2]], %0, %subreg.dsub
+    ; CHECK: [[IMPDEF3:%[0-9]+]]:fpr128 = IMPLICIT_DEF
+    ; CHECK-NEXT: [[INS3:%[0-9]+]]:fpr128 = INSERT_SUBREG [[IMPDEF3]], %0, %subreg.dsub
+    ; CHECK: %2:fpr16 = COPY [[INS_SHARED]].hsub
+    ; CHECK: %3:fpr16 = CPYi16 [[INS_SHARED]], 1
+    ; CHECK: %4:fpr16 = CPYi16 [[INS2]], 2
+    ; CHECK: %5:fpr16 = CPYi16 [[INS3]], 3
+    %2:fpr(s16), %3:fpr(s16), %4:fpr(s16), %5:fpr(s16) = G_UNMERGE_VALUES %0(<4 x s16>)
+
+    %1:fpr(<4 x s16>) = G_BUILD_VECTOR %2(s16), %3(s16), %4(s16), %5(s16)
+    $d0 = COPY %1(<4 x s16>)
+    RET_ReallyLR implicit $d0
+...
+---
+name:            test_v8s16_unmerge
+alignment:       2
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: fpr }
+  - { id: 1, class: fpr }
+  - { id: 2, class: fpr }
+  - { id: 3, class: fpr }
+  - { id: 4, class: fpr }
+  - { id: 5, class: fpr }
+  - { id: 6, class: fpr }
+  - { id: 7, class: fpr }
+  - { id: 8, class: fpr }
+  - { id: 9, class: fpr }
+body:             |
+  bb.1 (%ir-block.0):
+    liveins: $q0
+    ; CHECK-LABEL: name:            test_v8s16_unmerge
+    %0:fpr(<8 x s16>) = COPY $q0
+
+    ; Since 8 * 16 = 128, we can just directly copy.
+    ; CHECK: %2:fpr16 = COPY %0.hsub
+    ; CHECK: %3:fpr16 = CPYi16 %0, 1
+    ; CHECK: %4:fpr16 = CPYi16 %0, 2
+    ; CHECK: %5:fpr16 = CPYi16 %0, 3
+    ; CHECK: %6:fpr16 = CPYi16 %0, 4
+    ; CHECK: %7:fpr16 = CPYi16 %0, 5
+    ; CHECK: %8:fpr16 = CPYi16 %0, 6
+    ; CHECK: %9:fpr16 = CPYi16 %0, 7
+    %2:fpr(s16), %3:fpr(s16), %4:fpr(s16), %5:fpr(s16), %6:fpr(s16), %7:fpr(s16), %8:fpr(s16), %9:fpr(s16) = G_UNMERGE_VALUES %0(<8 x s16>)
+
+    %1:fpr(<8 x s16>) = G_BUILD_VECTOR %2:fpr(s16), %3:fpr(s16), %4:fpr(s16), %5:fpr(s16), %6:fpr(s16), %7:fpr(s16), %8:fpr(s16), %9:fpr(s16)
+    $q0 = COPY %1(<8 x s16>)
+    RET_ReallyLR implicit $q0
+...
diff --git a/test/CodeGen/AArch64/GlobalISel/varargs-ios-translator.ll b/test/CodeGen/AArch64/GlobalISel/varargs-ios-translator.ll
index 770985a530c4b9d6283e30444f3ce942b8ed521b..5f861f4ed0eeb65d127568c4598da44ebbfca5b8 100644
--- a/test/CodeGen/AArch64/GlobalISel/varargs-ios-translator.ll
+++ b/test/CodeGen/AArch64/GlobalISel/varargs-ios-translator.ll
@@ -8,7 +8,7 @@ define void @test_varargs_sentinel(i8* %list, i64, i64, i64, i64, i64, i64, i64,
 ; CHECK: body:
 ; CHECK:   [[LIST:%[0-9]+]]:gpr64sp = COPY $x0
 ; CHECK:   [[VARARGS_AREA:%[0-9]+]]:gpr64common = ADDXri %fixed-stack.[[VARARGS_SLOT]], 0, 0
-; CHECK:   STRXui [[VARARGS_AREA]], [[LIST]], 0 :: (store 8 into %ir.list, align 0)
+; CHECK:   STRXui [[VARARGS_AREA]], [[LIST]], 0 :: (store 8 into %ir.list, align 1)
   call void @llvm.va_start(i8* %list)
   ret void
 }
diff --git a/test/CodeGen/AArch64/GlobalISel/vastart.ll b/test/CodeGen/AArch64/GlobalISel/vastart.ll
index 9e83f030a0589f4fb347b3a81226ccd27534dfbf..32d1ac7194901631c2c5609a6b34b6f6d46e8274 100644
--- a/test/CodeGen/AArch64/GlobalISel/vastart.ll
+++ b/test/CodeGen/AArch64/GlobalISel/vastart.ll
@@ -6,8 +6,8 @@ declare void @llvm.va_start(i8*)
 define void @test_va_start(i8* %list) {
 ; CHECK-LABEL: name: test_va_start
 ; CHECK: [[LIST:%[0-9]+]]:_(p0) = COPY $x0
-; CHECK-IOS: G_VASTART [[LIST]](p0) :: (store 8 into %ir.list, align 0)
-; CHECK-LINUX: G_VASTART [[LIST]](p0) :: (store 32 into %ir.list, align 0)
+; CHECK-IOS: G_VASTART [[LIST]](p0) :: (store 8 into %ir.list, align 1)
+; CHECK-LINUX: G_VASTART [[LIST]](p0) :: (store 32 into %ir.list, align 1)
   call void @llvm.va_start(i8* %list)
   ret void
 }
diff --git a/test/CodeGen/AArch64/O0-pipeline.ll b/test/CodeGen/AArch64/O0-pipeline.ll
index 6d0aa91272bd058c71155db579e6d9160a3ea4fe..aa9b1d0c0f79688fa0591533d7a034b545b095f4 100644
--- a/test/CodeGen/AArch64/O0-pipeline.ll
+++ b/test/CodeGen/AArch64/O0-pipeline.ll
@@ -32,8 +32,10 @@
 ; CHECK-NEXT:       Safe Stack instrumentation pass
 ; CHECK-NEXT:       Insert stack protectors
 ; CHECK-NEXT:       Module Verifier
+; CHECK-NEXT:       Analysis containing CSE Info
 ; CHECK-NEXT:       IRTranslator
 ; CHECK-NEXT:       AArch64PreLegalizerCombiner
+; CHECK-NEXT:       Analysis containing CSE Info
 ; CHECK-NEXT:       Legalizer
 ; CHECK-NEXT:       RegBankSelect
 ; CHECK-NEXT:       Localizer
diff --git a/test/CodeGen/AArch64/aarch64-dynamic-stack-layout.ll b/test/CodeGen/AArch64/aarch64-dynamic-stack-layout.ll
index 708ae083eb865be6caf4a2baa6c45d11f08ae022..09eb5fe429da8f2cbdd606608ab439fec0cd088f 100644
--- a/test/CodeGen/AArch64/aarch64-dynamic-stack-layout.ll
+++ b/test/CodeGen/AArch64/aarch64-dynamic-stack-layout.ll
@@ -1,5 +1,5 @@
 ; RUN: llc -verify-machineinstrs -mtriple=aarch64-none-linux-gnu -disable-post-ra < %s | FileCheck %s
-; RUN: llc -verify-machineinstrs -mtriple=arm64-apple-ios -disable-fp-elim -disable-post-ra < %s | FileCheck %s --check-prefix=CHECK-MACHO
+; RUN: llc -verify-machineinstrs -mtriple=arm64-apple-ios -frame-pointer=all -disable-post-ra < %s | FileCheck %s --check-prefix=CHECK-MACHO
 
 ; This test aims to check basic correctness of frame layout &
 ; frame access code. There are 8 functions in this test file,
diff --git a/test/CodeGen/AArch64/addr-of-ret-addr.ll b/test/CodeGen/AArch64/addr-of-ret-addr.ll
index 247b2825e15509f7f6e49fd5b0e104888800d8d8..b099b18362ed637b908ea89f0520a0a0ce8c40b1 100644
--- a/test/CodeGen/AArch64/addr-of-ret-addr.ll
+++ b/test/CodeGen/AArch64/addr-of-ret-addr.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -disable-fp-elim -mtriple=arm64-windows | FileCheck %s
+; RUN: llc < %s -frame-pointer=all -mtriple=arm64-windows | FileCheck %s
 
 ; Test generated from C code:
 ; #include <stdarg.h>
diff --git a/test/CodeGen/AArch64/apple-latest-cpu.ll b/test/CodeGen/AArch64/apple-latest-cpu.ll
new file mode 100644
index 0000000000000000000000000000000000000000..033c3772af051dee9ca2411e39e1fc9a924be429
--- /dev/null
+++ b/test/CodeGen/AArch64/apple-latest-cpu.ll
@@ -0,0 +1,7 @@
+; RUN: llc -mtriple=arm64-apple-ios -mcpu=apple-latest -stop-before=expand-isel-pseudos -o - 2>&1 < %s | FileCheck %s
+
+; CHECK-LABEL: @dummy
+; CHECK: "target-cpu"="apple-latest"
+define void @dummy() {
+  ret void
+}
diff --git a/test/CodeGen/AArch64/arm64-2011-03-17-AsmPrinterCrash.ll b/test/CodeGen/AArch64/arm64-2011-03-17-AsmPrinterCrash.ll
index 294680a6d3a90a14f2ff94f04ddb737bc707adc9..d9d12c3f43e78be05d794e0a28536f8cafd15221 100644
--- a/test/CodeGen/AArch64/arm64-2011-03-17-AsmPrinterCrash.ll
+++ b/test/CodeGen/AArch64/arm64-2011-03-17-AsmPrinterCrash.ll
@@ -28,7 +28,7 @@ attributes #1 = { nounwind readnone }
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!9, !10}
 
-!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 3.0 (http://llvm.org/git/clang.git git:/git/puzzlebox/clang.git/ c4d1aea01c4444eb81bdbf391f1be309127c3cf1)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, globals: !2)
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 3.0", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, globals: !2)
 !1 = !DIFile(filename: "print.i", directory: "/Volumes/Ebi/echeng/radars/r9146594")
 !2 = !{!3}
 !3 = !DIGlobalVariableExpression(var: !4, expr: !DIExpression())
diff --git a/test/CodeGen/AArch64/arm64-abi_align.ll b/test/CodeGen/AArch64/arm64-abi_align.ll
index 85fd1eaa9c0b4e15064dab5d9c8a434ff5276f64..836b7b8adc55373e829c707b342d213a77173942 100644
--- a/test/CodeGen/AArch64/arm64-abi_align.ll
+++ b/test/CodeGen/AArch64/arm64-abi_align.ll
@@ -1,5 +1,5 @@
-; RUN: llc -fast-isel-sink-local-values < %s -mtriple=arm64-apple-darwin -mcpu=cyclone -enable-misched=false -disable-fp-elim | FileCheck %s
-; RUN: llc -fast-isel-sink-local-values < %s -mtriple=arm64-apple-darwin -O0 -disable-fp-elim -fast-isel | FileCheck -check-prefix=FAST %s
+; RUN: llc -fast-isel-sink-local-values < %s -mtriple=arm64-apple-darwin -mcpu=cyclone -enable-misched=false -frame-pointer=all | FileCheck %s
+; RUN: llc -fast-isel-sink-local-values < %s -mtriple=arm64-apple-darwin -O0 -frame-pointer=all -fast-isel | FileCheck -check-prefix=FAST %s
 
 ; rdar://12648441
 ; Generated from arm64-arguments.c with -O2.
diff --git a/test/CodeGen/AArch64/arm64-fast-isel-alloca.ll b/test/CodeGen/AArch64/arm64-fast-isel-alloca.ll
index 256db180d9114de0f4eff759e199aab0123626a4..1b65d9cbce14a586b93ef680a6084ea2910457ae 100644
--- a/test/CodeGen/AArch64/arm64-fast-isel-alloca.ll
+++ b/test/CodeGen/AArch64/arm64-fast-isel-alloca.ll
@@ -1,5 +1,5 @@
 ; This test should cause the TargetMaterializeAlloca to be invoked
-; RUN: llc -O0 -fast-isel -fast-isel-abort=1 -verify-machineinstrs -mtriple=arm64-apple-darwin -disable-fp-elim < %s | FileCheck %s
+; RUN: llc -O0 -fast-isel -fast-isel-abort=1 -verify-machineinstrs -mtriple=arm64-apple-darwin -frame-pointer=all < %s | FileCheck %s
 
 %struct.S1Ty = type { i64 }
 %struct.S2Ty = type { %struct.S1Ty, %struct.S1Ty }
diff --git a/test/CodeGen/AArch64/arm64-fast-isel-call.ll b/test/CodeGen/AArch64/arm64-fast-isel-call.ll
index abbe6554a54e8b8f908ebb7a1d8b0a744d1afdac..dc1aac8409cc26c325db689eec78728b6b9cfda6 100644
--- a/test/CodeGen/AArch64/arm64-fast-isel-call.ll
+++ b/test/CodeGen/AArch64/arm64-fast-isel-call.ll
@@ -1,6 +1,6 @@
-; RUN: llc -fast-isel-sink-local-values -O0 -fast-isel -fast-isel-abort=2 -code-model=small -verify-machineinstrs -disable-fp-elim -mtriple=arm64-apple-darwin   < %s | FileCheck %s
-; RUN: llc -fast-isel-sink-local-values -O0 -fast-isel -fast-isel-abort=2 -code-model=large -verify-machineinstrs -disable-fp-elim -mtriple=arm64-apple-darwin   < %s | FileCheck %s --check-prefix=LARGE
-; RUN: llc -fast-isel-sink-local-values -O0 -fast-isel -fast-isel-abort=2 -code-model=small -verify-machineinstrs -disable-fp-elim -mtriple=aarch64_be-linux-gnu < %s | FileCheck %s --check-prefix=CHECK-BE
+; RUN: llc -fast-isel-sink-local-values -O0 -fast-isel -fast-isel-abort=2 -code-model=small -verify-machineinstrs -frame-pointer=all -mtriple=arm64-apple-darwin   < %s | FileCheck %s
+; RUN: llc -fast-isel-sink-local-values -O0 -fast-isel -fast-isel-abort=2 -code-model=large -verify-machineinstrs -frame-pointer=all -mtriple=arm64-apple-darwin   < %s | FileCheck %s --check-prefix=LARGE
+; RUN: llc -fast-isel-sink-local-values -O0 -fast-isel -fast-isel-abort=2 -code-model=small -verify-machineinstrs -frame-pointer=all -mtriple=aarch64_be-linux-gnu < %s | FileCheck %s --check-prefix=CHECK-BE
 
 define void @call0() nounwind {
 entry:
diff --git a/test/CodeGen/AArch64/arm64-hello.ll b/test/CodeGen/AArch64/arm64-hello.ll
index a8d1c24825206e2439f84339888eb6a899c41a96..5e1bd9d499202faf3a93a2342589610265534d1f 100644
--- a/test/CodeGen/AArch64/arm64-hello.ll
+++ b/test/CodeGen/AArch64/arm64-hello.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -mtriple=arm64-apple-ios7.0 -disable-post-ra -disable-fp-elim | FileCheck %s
+; RUN: llc < %s -mtriple=arm64-apple-ios7.0 -disable-post-ra -frame-pointer=all | FileCheck %s
 ; RUN: llc < %s -mtriple=arm64-linux-gnu -disable-post-ra | FileCheck %s --check-prefix=CHECK-LINUX
 
 ; CHECK-LABEL: main:
diff --git a/test/CodeGen/AArch64/arm64-large-frame.ll b/test/CodeGen/AArch64/arm64-large-frame.ll
index d1244e73b0f333c7f82e0ce452155b5e6dc4990d..cfda00ceb5e3e0556d6531cef542a2aa9f32138f 100644
--- a/test/CodeGen/AArch64/arm64-large-frame.ll
+++ b/test/CodeGen/AArch64/arm64-large-frame.ll
@@ -1,4 +1,4 @@
-; RUN: llc -verify-machineinstrs -mtriple=arm64-none-linux-gnu -disable-fp-elim -disable-post-ra < %s | FileCheck %s
+; RUN: llc -verify-machineinstrs -mtriple=arm64-none-linux-gnu -frame-pointer=all -disable-post-ra < %s | FileCheck %s
 declare void @use_addr(i8*)
 
 @addr = global i8* null
diff --git a/test/CodeGen/AArch64/arm64-platform-reg.ll b/test/CodeGen/AArch64/arm64-platform-reg.ll
index 73fe36d761110edf15e067ece3f29badcb3ecbef..42448fcce56cf3a06374ed135be0af91793226fa 100644
--- a/test/CodeGen/AArch64/arm64-platform-reg.ll
+++ b/test/CodeGen/AArch64/arm64-platform-reg.ll
@@ -1,7 +1,5 @@
 ; RUN: llc -mtriple=arm64-apple-ios -mattr=+reserve-x18 -o - %s | FileCheck %s --check-prefix=CHECK-RESERVE --check-prefix=CHECK-RESERVE-X18
 ; RUN: llc -mtriple=arm64-freebsd-gnu -mattr=+reserve-x18 -o - %s | FileCheck %s --check-prefix=CHECK-RESERVE --check-prefix=CHECK-RESERVE-X18
-; RUN: llc -mtriple=aarch64-fuchsia -mattr=+reserve-x20 -o - %s | FileCheck %s --check-prefix=CHECK-RESERVE --check-prefix=CHECK-RESERVE-X20
-; RUN: llc -mtriple=aarch64-fuchsia -mattr=+reserve-x18,+reserve-x20 -o - %s | FileCheck %s --check-prefix=CHECK-RESERVE --check-prefix=CHECK-RESERVE-X18 --check-prefix=CHECK-RESERVE-X20
 ; RUN: llc -mtriple=arm64-linux-gnu -o - %s | FileCheck %s
 ; RUN: llc -mtriple=aarch64-linux-android -o - %s | FileCheck %s --check-prefix=CHECK-RESERVE --check-prefix=CHECK-RESERVE-X18
 ; RUN: llc -mtriple=aarch64-fuchsia -o - %s | FileCheck %s --check-prefix=CHECK-RESERVE --check-prefix=CHECK-RESERVE-X18
@@ -15,6 +13,22 @@
 ; RUN: llc -mtriple=arm64-linux-gnu -mattr=+reserve-x5 -o - %s | FileCheck %s --check-prefixes=CHECK-RESERVE,CHECK-RESERVE-X5
 ; RUN: llc -mtriple=arm64-linux-gnu -mattr=+reserve-x6 -o - %s | FileCheck %s --check-prefixes=CHECK-RESERVE,CHECK-RESERVE-X6
 ; RUN: llc -mtriple=arm64-linux-gnu -mattr=+reserve-x7 -o - %s | FileCheck %s --check-prefixes=CHECK-RESERVE,CHECK-RESERVE-X7
+; RUN: llc -mtriple=arm64-linux-gnu -mattr=+reserve-x9 -o - %s | FileCheck %s --check-prefixes=CHECK-RESERVE,CHECK-RESERVE-X9
+; RUN: llc -mtriple=arm64-linux-gnu -mattr=+reserve-x10 -o - %s | FileCheck %s --check-prefixes=CHECK-RESERVE,CHECK-RESERVE-X10
+; RUN: llc -mtriple=arm64-linux-gnu -mattr=+reserve-x11 -o - %s | FileCheck %s --check-prefixes=CHECK-RESERVE,CHECK-RESERVE-X11
+; RUN: llc -mtriple=arm64-linux-gnu -mattr=+reserve-x12 -o - %s | FileCheck %s --check-prefixes=CHECK-RESERVE,CHECK-RESERVE-X12
+; RUN: llc -mtriple=arm64-linux-gnu -mattr=+reserve-x13 -o - %s | FileCheck %s --check-prefixes=CHECK-RESERVE,CHECK-RESERVE-X13
+; RUN: llc -mtriple=arm64-linux-gnu -mattr=+reserve-x14 -o - %s | FileCheck %s --check-prefixes=CHECK-RESERVE,CHECK-RESERVE-X14
+; RUN: llc -mtriple=arm64-linux-gnu -mattr=+reserve-x15 -o - %s | FileCheck %s --check-prefixes=CHECK-RESERVE,CHECK-RESERVE-X15
+; RUN: llc -mtriple=arm64-linux-gnu -mattr=+reserve-x20 -o - %s | FileCheck %s --check-prefixes=CHECK-RESERVE,CHECK-RESERVE-X20
+; RUN: llc -mtriple=arm64-linux-gnu -mattr=+reserve-x21 -o - %s | FileCheck %s --check-prefixes=CHECK-RESERVE,CHECK-RESERVE-X21
+; RUN: llc -mtriple=arm64-linux-gnu -mattr=+reserve-x22 -o - %s | FileCheck %s --check-prefixes=CHECK-RESERVE,CHECK-RESERVE-X22
+; RUN: llc -mtriple=arm64-linux-gnu -mattr=+reserve-x23 -o - %s | FileCheck %s --check-prefixes=CHECK-RESERVE,CHECK-RESERVE-X23
+; RUN: llc -mtriple=arm64-linux-gnu -mattr=+reserve-x24 -o - %s | FileCheck %s --check-prefixes=CHECK-RESERVE,CHECK-RESERVE-X24
+; RUN: llc -mtriple=arm64-linux-gnu -mattr=+reserve-x25 -o - %s | FileCheck %s --check-prefixes=CHECK-RESERVE,CHECK-RESERVE-X25
+; RUN: llc -mtriple=arm64-linux-gnu -mattr=+reserve-x26 -o - %s | FileCheck %s --check-prefixes=CHECK-RESERVE,CHECK-RESERVE-X26
+; RUN: llc -mtriple=arm64-linux-gnu -mattr=+reserve-x27 -o - %s | FileCheck %s --check-prefixes=CHECK-RESERVE,CHECK-RESERVE-X27
+; RUN: llc -mtriple=arm64-linux-gnu -mattr=+reserve-x28 -o - %s | FileCheck %s --check-prefixes=CHECK-RESERVE,CHECK-RESERVE-X28
 
 ; Test multiple of reserve-x# options together.
 ; RUN: llc -mtriple=arm64-linux-gnu \
@@ -36,8 +50,23 @@
 ; RUN: -mattr=+reserve-x5 \
 ; RUN: -mattr=+reserve-x6 \
 ; RUN: -mattr=+reserve-x7 \
+; RUN: -mattr=+reserve-x9 \
+; RUN: -mattr=+reserve-x10 \
+; RUN: -mattr=+reserve-x11 \
+; RUN: -mattr=+reserve-x12 \
+; RUN: -mattr=+reserve-x13 \
+; RUN: -mattr=+reserve-x14 \
+; RUN: -mattr=+reserve-x15 \
 ; RUN: -mattr=+reserve-x18 \
 ; RUN: -mattr=+reserve-x20 \
+; RUN: -mattr=+reserve-x21 \
+; RUN: -mattr=+reserve-x22 \
+; RUN: -mattr=+reserve-x23 \
+; RUN: -mattr=+reserve-x24 \
+; RUN: -mattr=+reserve-x25 \
+; RUN: -mattr=+reserve-x26 \
+; RUN: -mattr=+reserve-x27 \
+; RUN: -mattr=+reserve-x28 \
 ; RUN: -o - %s | FileCheck %s \
 ; RUN: --check-prefix=CHECK-RESERVE \
 ; RUN: --check-prefix=CHECK-RESERVE-X1 \
@@ -47,8 +76,23 @@
 ; RUN: --check-prefix=CHECK-RESERVE-X5 \
 ; RUN: --check-prefix=CHECK-RESERVE-X6 \
 ; RUN: --check-prefix=CHECK-RESERVE-X7 \
+; RUN: --check-prefix=CHECK-RESERVE-X9 \
+; RUN: --check-prefix=CHECK-RESERVE-X10 \
+; RUN: --check-prefix=CHECK-RESERVE-X11 \
+; RUN: --check-prefix=CHECK-RESERVE-X12 \
+; RUN: --check-prefix=CHECK-RESERVE-X13 \
+; RUN: --check-prefix=CHECK-RESERVE-X14 \
+; RUN: --check-prefix=CHECK-RESERVE-X15 \
 ; RUN: --check-prefix=CHECK-RESERVE-X18 \
-; RUN: --check-prefix=CHECK-RESERVE-X20
+; RUN: --check-prefix=CHECK-RESERVE-X20 \
+; RUN: --check-prefix=CHECK-RESERVE-X21 \
+; RUN: --check-prefix=CHECK-RESERVE-X22 \
+; RUN: --check-prefix=CHECK-RESERVE-X23 \
+; RUN: --check-prefix=CHECK-RESERVE-X24 \
+; RUN: --check-prefix=CHECK-RESERVE-X25 \
+; RUN: --check-prefix=CHECK-RESERVE-X26 \
+; RUN: --check-prefix=CHECK-RESERVE-X27 \
+; RUN: --check-prefix=CHECK-RESERVE-X28
 
 ; x18 is reserved as a platform register on Darwin but not on other
 ; systems. Create loads of register pressure and make sure this is respected.
@@ -73,8 +117,23 @@ define void @keep_live() {
 ; CHECK-RESERVE-X5-NOT: ldr x5,
 ; CHECK-RESERVE-X6-NOT: ldr x6,
 ; CHECK-RESERVE-X7-NOT: ldr x7,
+; CHECK-RESERVE-X9-NOT: ldr x9,
+; CHECK-RESERVE-X10-NOT: ldr x10,
+; CHECK-RESERVE-X11-NOT: ldr x11,
+; CHECK-RESERVE-X12-NOT: ldr x12,
+; CHECK-RESERVE-X13-NOT: ldr x13,
+; CHECK-RESERVE-X14-NOT: ldr x14,
+; CHECK-RESERVE-X15-NOT: ldr x15,
 ; CHECK-RESERVE-X18-NOT: ldr x18
 ; CHECK-RESERVE-X20-NOT: ldr x20
+; CHECK-RESERVE-X21-NOT: ldr x21
+; CHECK-RESERVE-X22-NOT: ldr x22
+; CHECK-RESERVE-X23-NOT: ldr x23
+; CHECK-RESERVE-X24-NOT: ldr x24
+; CHECK-RESERVE-X25-NOT: ldr x25
+; CHECK-RESERVE-X26-NOT: ldr x26
+; CHECK-RESERVE-X27-NOT: ldr x27
+; CHECK-RESERVE-X28-NOT: ldr x28
 ; CHECK-RESERVE: Spill
 ; CHECK-RESERVE-NOT: ldr fp
 ; CHECK-RESERVE-X1-NOT: ldr x1,
@@ -84,8 +143,23 @@ define void @keep_live() {
 ; CHECK-RESERVE-X5-NOT: ldr x5,
 ; CHECK-RESERVE-X6-NOT: ldr x6,
 ; CHECK-RESERVE-X7-NOT: ldr x7,
+; CHECK-RESERVE-X9-NOT: ldr x9,
+; CHECK-RESERVE-X10-NOT: ldr x10,
+; CHECK-RESERVE-X11-NOT: ldr x11,
+; CHECK-RESERVE-X12-NOT: ldr x12,
+; CHECK-RESERVE-X13-NOT: ldr x13,
+; CHECK-RESERVE-X14-NOT: ldr x14,
+; CHECK-RESERVE-X15-NOT: ldr x15,
 ; CHECK-RESERVE-X18-NOT: ldr x18
 ; CHECK-RESERVE-X20-NOT: ldr x20
+; CHECK-RESERVE-X21-NOT: ldr x21
+; CHECK-RESERVE-X22-NOT: ldr x22
+; CHECK-RESERVE-X23-NOT: ldr x23
+; CHECK-RESERVE-X24-NOT: ldr x24
+; CHECK-RESERVE-X25-NOT: ldr x25
+; CHECK-RESERVE-X26-NOT: ldr x26
+; CHECK-RESERVE-X27-NOT: ldr x27
+; CHECK-RESERVE-X28-NOT: ldr x28
 ; CHECK-RESERVE: ret
   ret void
 }
diff --git a/test/CodeGen/AArch64/arm64-reserve-call-saved-reg.ll b/test/CodeGen/AArch64/arm64-reserve-call-saved-reg.ll
index 050e3033fcb83955e806bdb86de375d5daa54cca..ce9d2c7171d2e627e7aa753c65a4794e6b84acfd 100644
--- a/test/CodeGen/AArch64/arm64-reserve-call-saved-reg.ll
+++ b/test/CodeGen/AArch64/arm64-reserve-call-saved-reg.ll
@@ -1,13 +1,34 @@
-; RUN: llc -mtriple=arm64-linux-gnu -mattr=+reserve-x18 -mattr=+call-saved-x18 \
-; RUN: -o - %s | FileCheck %s
+; RUN: llc -mtriple=arm64-linux-gnu -mattr=+reserve-x9 -mattr=+call-saved-x9 -o - %s | FileCheck %s --check-prefixes=CHECK-X9
+; RUN: llc -mtriple=arm64-linux-gnu -mattr=+reserve-x9 -mattr=+call-saved-x9 -global-isel -o - %s | FileCheck %s --check-prefixes=CHECK-X9
+; RUN: llc -mtriple=arm64-linux-gnu -mattr=+reserve-x9 -mattr=+call-saved-x9 -fast-isel -o - %s | FileCheck %s --check-prefixes=CHECK-X9
 
-; RUN: llc -mtriple=arm64-linux-gnu -mattr=+reserve-x18 -mattr=+call-saved-x18 \
-; RUN: -global-isel \
-; RUN: -o - %s | FileCheck %s
+; RUN: llc -mtriple=arm64-linux-gnu -mattr=+reserve-x10 -mattr=+call-saved-x10 -o - %s | FileCheck %s --check-prefixes=CHECK-X10
+; RUN: llc -mtriple=arm64-linux-gnu -mattr=+reserve-x10 -mattr=+call-saved-x10 -global-isel -o - %s | FileCheck %s --check-prefixes=CHECK-X10
+; RUN: llc -mtriple=arm64-linux-gnu -mattr=+reserve-x10 -mattr=+call-saved-x10 -fast-isel -o - %s | FileCheck %s --check-prefixes=CHECK-X10
 
-; RUN: llc -mtriple=arm64-linux-gnu -mattr=+reserve-x18 -mattr=+call-saved-x18 \
-; RUN: -fast-isel \
-; RUN: -o - %s | FileCheck %s
+; RUN: llc -mtriple=arm64-linux-gnu -mattr=+reserve-x11 -mattr=+call-saved-x11 -o - %s | FileCheck %s --check-prefixes=CHECK-X11
+; RUN: llc -mtriple=arm64-linux-gnu -mattr=+reserve-x11 -mattr=+call-saved-x11 -global-isel -o - %s | FileCheck %s --check-prefixes=CHECK-X11
+; RUN: llc -mtriple=arm64-linux-gnu -mattr=+reserve-x11 -mattr=+call-saved-x11 -fast-isel -o - %s | FileCheck %s --check-prefixes=CHECK-X11
+
+; RUN: llc -mtriple=arm64-linux-gnu -mattr=+reserve-x12 -mattr=+call-saved-x12 -o - %s | FileCheck %s --check-prefixes=CHECK-X12
+; RUN: llc -mtriple=arm64-linux-gnu -mattr=+reserve-x12 -mattr=+call-saved-x12 -global-isel -o - %s | FileCheck %s --check-prefixes=CHECK-X12
+; RUN: llc -mtriple=arm64-linux-gnu -mattr=+reserve-x12 -mattr=+call-saved-x12 -fast-isel -o - %s | FileCheck %s --check-prefixes=CHECK-X12
+
+; RUN: llc -mtriple=arm64-linux-gnu -mattr=+reserve-x13 -mattr=+call-saved-x13 -o - %s | FileCheck %s --check-prefixes=CHECK-X13
+; RUN: llc -mtriple=arm64-linux-gnu -mattr=+reserve-x13 -mattr=+call-saved-x13 -global-isel -o - %s | FileCheck %s --check-prefixes=CHECK-X13
+; RUN: llc -mtriple=arm64-linux-gnu -mattr=+reserve-x13 -mattr=+call-saved-x13 -fast-isel -o - %s | FileCheck %s --check-prefixes=CHECK-X13
+
+; RUN: llc -mtriple=arm64-linux-gnu -mattr=+reserve-x14 -mattr=+call-saved-x14 -o - %s | FileCheck %s --check-prefixes=CHECK-X14
+; RUN: llc -mtriple=arm64-linux-gnu -mattr=+reserve-x14 -mattr=+call-saved-x14 -global-isel -o - %s | FileCheck %s --check-prefixes=CHECK-X14
+; RUN: llc -mtriple=arm64-linux-gnu -mattr=+reserve-x14 -mattr=+call-saved-x14 -fast-isel -o - %s | FileCheck %s --check-prefixes=CHECK-X14
+
+; RUN: llc -mtriple=arm64-linux-gnu -mattr=+reserve-x15 -mattr=+call-saved-x15 -o - %s | FileCheck %s --check-prefixes=CHECK-X15
+; RUN: llc -mtriple=arm64-linux-gnu -mattr=+reserve-x15 -mattr=+call-saved-x15 -global-isel -o - %s | FileCheck %s --check-prefixes=CHECK-X15
+; RUN: llc -mtriple=arm64-linux-gnu -mattr=+reserve-x15 -mattr=+call-saved-x15 -fast-isel -o - %s | FileCheck %s --check-prefixes=CHECK-X15
+
+; RUN: llc -mtriple=arm64-linux-gnu -mattr=+reserve-x18 -mattr=+call-saved-x18 -o - %s | FileCheck %s --check-prefixes=CHECK-X18
+; RUN: llc -mtriple=arm64-linux-gnu -mattr=+reserve-x18 -mattr=+call-saved-x18 -global-isel -o - %s | FileCheck %s --check-prefixes=CHECK-X18
+; RUN: llc -mtriple=arm64-linux-gnu -mattr=+reserve-x18 -mattr=+call-saved-x18 -fast-isel -o - %s | FileCheck %s --check-prefixes=CHECK-X18
 
 ; Used to exhaust the supply of GPRs.
 @var = global [30 x i64] zeroinitializer
@@ -15,14 +36,42 @@
 ; If a register is specified to be both reserved and callee-saved, then it
 ; should not be allocated and should not be spilled onto the stack.
 define void @foo() {
-; CHECK-NOT: str x18, [sp
+; CHECK-X9-NOT: str x9, [sp
+; CHECK-X10-NOT: str x10, [sp
+; CHECK-X11-NOT: str x11, [sp
+; CHECK-X12-NOT: str x12, [sp
+; CHECK-X13-NOT: str x13, [sp
+; CHECK-X14-NOT: str x14, [sp
+; CHECK-X15-NOT: str x15, [sp
+; CHECK-X18-NOT: str x18, [sp
 
   %val = load volatile [30 x i64], [30 x i64]* @var
   store volatile [30 x i64] %val, [30 x i64]* @var
 
-; CHECK-NOT: ldr x18
-; CHECK-NOT: str x18
+; CHECK-X9-NOT: ldr x9
+; CHECK-X10-NOT: ldr x10
+; CHECK-X11-NOT: ldr x11
+; CHECK-X12-NOT: ldr x12
+; CHECK-X13-NOT: ldr x13
+; CHECK-X14-NOT: ldr x14
+; CHECK-X15-NOT: ldr x15
+; CHECK-X18-NOT: ldr x18
+; CHECK-X9-NOT: str x9
+; CHECK-X10-NOT: str x10
+; CHECK-X11-NOT: str x11
+; CHECK-X12-NOT: str x12
+; CHECK-X13-NOT: str x13
+; CHECK-X14-NOT: str x14
+; CHECK-X15-NOT: str x15
+; CHECK-X18-NOT: str x18
 
-; CHECK-NOT: ldr x18, [sp
+; CHECK-X9-NOT: ldr x9, [sp
+; CHECK-X10-NOT: ldr x10, [sp
+; CHECK-X11-NOT: ldr x11, [sp
+; CHECK-X12-NOT: ldr x12, [sp
+; CHECK-X13-NOT: ldr x13, [sp
+; CHECK-X14-NOT: ldr x14, [sp
+; CHECK-X15-NOT: ldr x15, [sp
+; CHECK-X18-NOT: ldr x18, [sp
   ret void
 }
diff --git a/test/CodeGen/AArch64/arm64-rev.ll b/test/CodeGen/AArch64/arm64-rev.ll
index 1ce5ab44e292a63c72e1ac5637b6bd28fd5d84e4..9b864c3a66cc78a285ca0ba74073637e3a4e9354 100644
--- a/test/CodeGen/AArch64/arm64-rev.ll
+++ b/test/CodeGen/AArch64/arm64-rev.ll
@@ -1,17 +1,33 @@
-; RUN: llc < %s -mtriple=arm64-eabi -aarch64-neon-syntax=apple | FileCheck %s
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=aarch64-eabi -aarch64-neon-syntax=apple | FileCheck %s
+; RUN: llc < %s -global-isel -global-isel-abort=2 -pass-remarks-missed=gisel* -mtriple=aarch64-eabi -aarch64-neon-syntax=apple | FileCheck %s --check-prefixes=FALLBACK,GISEL
 
+; FALLBACK-NOT: remark{{.*}}test_rev_w
 define i32 @test_rev_w(i32 %a) nounwind {
-entry:
 ; CHECK-LABEL: test_rev_w:
-; CHECK: rev w0, w0
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    rev w0, w0
+; CHECK-NEXT:    ret
+; GISEL-LABEL: test_rev_w:
+; GISEL:       // %bb.0: // %entry
+; GISEL-NEXT:    rev w0, w0
+; GISEL-NEXT:    ret
+entry:
   %0 = tail call i32 @llvm.bswap.i32(i32 %a)
   ret i32 %0
 }
 
+; FALLBACK-NOT: remark{{.*}}test_rev_x
 define i64 @test_rev_x(i64 %a) nounwind {
-entry:
 ; CHECK-LABEL: test_rev_x:
-; CHECK: rev x0, x0
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    rev x0, x0
+; CHECK-NEXT:    ret
+; GISEL-LABEL: test_rev_x:
+; GISEL:       // %bb.0: // %entry
+; GISEL-NEXT:    rev x0, x0
+; GISEL-NEXT:    ret
+entry:
   %0 = tail call i64 @llvm.bswap.i64(i64 %a)
   ret i64 %0
 }
@@ -19,11 +35,12 @@ entry:
 ; Canonicalize (srl (bswap x), 16) to (rotr (bswap x), 16) if the high 16-bits
 ; of %a are zero. This optimizes rev + lsr 16 to rev16.
 define i32 @test_rev_w_srl16(i16 %a) {
-entry:
 ; CHECK-LABEL: test_rev_w_srl16:
-; CHECK: and [[REG:w[0-9]+]], w0, #0xffff
-; CHECK: rev16 w0, [[REG]]
-; CHECK-NOT: lsr
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    and w8, w0, #0xffff
+; CHECK-NEXT:    rev16 w0, w8
+; CHECK-NEXT:    ret
+entry:
   %0 = zext i16 %a to i32
   %1 = tail call i32 @llvm.bswap.i32(i32 %0)
   %2 = lshr i32 %1, 16
@@ -33,10 +50,12 @@ entry:
 ; Canonicalize (srl (bswap x), 32) to (rotr (bswap x), 32) if the high 32-bits
 ; of %a are zero. This optimizes rev + lsr 32 to rev32.
 define i64 @test_rev_x_srl32(i32 %a) {
-entry:
 ; CHECK-LABEL: test_rev_x_srl32:
-; CHECK: rev32 x0, {{x[0-9]+}}
-; CHECK-NOT: lsr
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov w8, w0
+; CHECK-NEXT:    rev32 x0, x8
+; CHECK-NEXT:    ret
+entry:
   %0 = zext i32 %a to i64
   %1 = tail call i64 @llvm.bswap.i64(i64 %0)
   %2 = lshr i64 %1, 32
@@ -47,9 +66,11 @@ declare i32 @llvm.bswap.i32(i32) nounwind readnone
 declare i64 @llvm.bswap.i64(i64) nounwind readnone
 
 define i32 @test_rev16_w(i32 %X) nounwind {
-entry:
 ; CHECK-LABEL: test_rev16_w:
-; CHECK: rev16 w0, w0
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    rev16 w0, w0
+; CHECK-NEXT:    ret
+entry:
   %tmp1 = lshr i32 %X, 8
   %X15 = bitcast i32 %X to i32
   %tmp4 = shl i32 %X15, 8
@@ -67,9 +88,12 @@ entry:
 ;   01234567 ->(bswap) 76543210 ->(rotr) 10765432
 ;   01234567 ->(rev16) 10325476
 define i64 @test_rev16_x(i64 %a) nounwind {
-entry:
 ; CHECK-LABEL: test_rev16_x:
-; CHECK-NOT: rev16 x0, x0
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    rev x8, x0
+; CHECK-NEXT:    ror x0, x8, #16
+; CHECK-NEXT:    ret
+entry:
   %0 = tail call i64 @llvm.bswap.i64(i64 %a)
   %1 = lshr i64 %0, 16
   %2 = shl i64 %0, 48
@@ -78,9 +102,11 @@ entry:
 }
 
 define i64 @test_rev32_x(i64 %a) nounwind {
-entry:
 ; CHECK-LABEL: test_rev32_x:
-; CHECK: rev32 x0, x0
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    rev32 x0, x0
+; CHECK-NEXT:    ret
+entry:
   %0 = tail call i64 @llvm.bswap.i64(i64 %a)
   %1 = lshr i64 %0, 32
   %2 = shl i64 %0, 32
@@ -89,112 +115,154 @@ entry:
 }
 
 define <8 x i8> @test_vrev64D8(<8 x i8>* %A) nounwind {
-;CHECK-LABEL: test_vrev64D8:
-;CHECK: rev64.8b
+; CHECK-LABEL: test_vrev64D8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr d0, [x0]
+; CHECK-NEXT:    rev64.8b v0, v0
+; CHECK-NEXT:    ret
 	%tmp1 = load <8 x i8>, <8 x i8>* %A
 	%tmp2 = shufflevector <8 x i8> %tmp1, <8 x i8> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
 	ret <8 x i8> %tmp2
 }
 
 define <4 x i16> @test_vrev64D16(<4 x i16>* %A) nounwind {
-;CHECK-LABEL: test_vrev64D16:
-;CHECK: rev64.4h
+; CHECK-LABEL: test_vrev64D16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr d0, [x0]
+; CHECK-NEXT:    rev64.4h v0, v0
+; CHECK-NEXT:    ret
 	%tmp1 = load <4 x i16>, <4 x i16>* %A
 	%tmp2 = shufflevector <4 x i16> %tmp1, <4 x i16> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
 	ret <4 x i16> %tmp2
 }
 
 define <2 x i32> @test_vrev64D32(<2 x i32>* %A) nounwind {
-;CHECK-LABEL: test_vrev64D32:
-;CHECK: rev64.2s
+; CHECK-LABEL: test_vrev64D32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr d0, [x0]
+; CHECK-NEXT:    rev64.2s v0, v0
+; CHECK-NEXT:    ret
 	%tmp1 = load <2 x i32>, <2 x i32>* %A
 	%tmp2 = shufflevector <2 x i32> %tmp1, <2 x i32> undef, <2 x i32> <i32 1, i32 0>
 	ret <2 x i32> %tmp2
 }
 
 define <2 x float> @test_vrev64Df(<2 x float>* %A) nounwind {
-;CHECK-LABEL: test_vrev64Df:
-;CHECK: rev64.2s
+; CHECK-LABEL: test_vrev64Df:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr d0, [x0]
+; CHECK-NEXT:    rev64.2s v0, v0
+; CHECK-NEXT:    ret
 	%tmp1 = load <2 x float>, <2 x float>* %A
 	%tmp2 = shufflevector <2 x float> %tmp1, <2 x float> undef, <2 x i32> <i32 1, i32 0>
 	ret <2 x float> %tmp2
 }
 
 define <16 x i8> @test_vrev64Q8(<16 x i8>* %A) nounwind {
-;CHECK-LABEL: test_vrev64Q8:
-;CHECK: rev64.16b
+; CHECK-LABEL: test_vrev64Q8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr q0, [x0]
+; CHECK-NEXT:    rev64.16b v0, v0
+; CHECK-NEXT:    ret
 	%tmp1 = load <16 x i8>, <16 x i8>* %A
 	%tmp2 = shufflevector <16 x i8> %tmp1, <16 x i8> undef, <16 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8>
 	ret <16 x i8> %tmp2
 }
 
 define <8 x i16> @test_vrev64Q16(<8 x i16>* %A) nounwind {
-;CHECK-LABEL: test_vrev64Q16:
-;CHECK: rev64.8h
+; CHECK-LABEL: test_vrev64Q16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr q0, [x0]
+; CHECK-NEXT:    rev64.8h v0, v0
+; CHECK-NEXT:    ret
 	%tmp1 = load <8 x i16>, <8 x i16>* %A
 	%tmp2 = shufflevector <8 x i16> %tmp1, <8 x i16> undef, <8 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4>
 	ret <8 x i16> %tmp2
 }
 
 define <4 x i32> @test_vrev64Q32(<4 x i32>* %A) nounwind {
-;CHECK-LABEL: test_vrev64Q32:
-;CHECK: rev64.4s
+; CHECK-LABEL: test_vrev64Q32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr q0, [x0]
+; CHECK-NEXT:    rev64.4s v0, v0
+; CHECK-NEXT:    ret
 	%tmp1 = load <4 x i32>, <4 x i32>* %A
 	%tmp2 = shufflevector <4 x i32> %tmp1, <4 x i32> undef, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
 	ret <4 x i32> %tmp2
 }
 
 define <4 x float> @test_vrev64Qf(<4 x float>* %A) nounwind {
-;CHECK-LABEL: test_vrev64Qf:
-;CHECK: rev64.4s
+; CHECK-LABEL: test_vrev64Qf:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr q0, [x0]
+; CHECK-NEXT:    rev64.4s v0, v0
+; CHECK-NEXT:    ret
 	%tmp1 = load <4 x float>, <4 x float>* %A
 	%tmp2 = shufflevector <4 x float> %tmp1, <4 x float> undef, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
 	ret <4 x float> %tmp2
 }
 
 define <8 x i8> @test_vrev32D8(<8 x i8>* %A) nounwind {
-;CHECK-LABEL: test_vrev32D8:
-;CHECK: rev32.8b
+; CHECK-LABEL: test_vrev32D8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr d0, [x0]
+; CHECK-NEXT:    rev32.8b v0, v0
+; CHECK-NEXT:    ret
 	%tmp1 = load <8 x i8>, <8 x i8>* %A
 	%tmp2 = shufflevector <8 x i8> %tmp1, <8 x i8> undef, <8 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4>
 	ret <8 x i8> %tmp2
 }
 
 define <4 x i16> @test_vrev32D16(<4 x i16>* %A) nounwind {
-;CHECK-LABEL: test_vrev32D16:
-;CHECK: rev32.4h
+; CHECK-LABEL: test_vrev32D16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr d0, [x0]
+; CHECK-NEXT:    rev32.4h v0, v0
+; CHECK-NEXT:    ret
 	%tmp1 = load <4 x i16>, <4 x i16>* %A
 	%tmp2 = shufflevector <4 x i16> %tmp1, <4 x i16> undef, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
 	ret <4 x i16> %tmp2
 }
 
 define <16 x i8> @test_vrev32Q8(<16 x i8>* %A) nounwind {
-;CHECK-LABEL: test_vrev32Q8:
-;CHECK: rev32.16b
+; CHECK-LABEL: test_vrev32Q8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr q0, [x0]
+; CHECK-NEXT:    rev32.16b v0, v0
+; CHECK-NEXT:    ret
 	%tmp1 = load <16 x i8>, <16 x i8>* %A
 	%tmp2 = shufflevector <16 x i8> %tmp1, <16 x i8> undef, <16 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4, i32 11, i32 10, i32 9, i32 8, i32 15, i32 14, i32 13, i32 12>
 	ret <16 x i8> %tmp2
 }
 
 define <8 x i16> @test_vrev32Q16(<8 x i16>* %A) nounwind {
-;CHECK-LABEL: test_vrev32Q16:
-;CHECK: rev32.8h
+; CHECK-LABEL: test_vrev32Q16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr q0, [x0]
+; CHECK-NEXT:    rev32.8h v0, v0
+; CHECK-NEXT:    ret
 	%tmp1 = load <8 x i16>, <8 x i16>* %A
 	%tmp2 = shufflevector <8 x i16> %tmp1, <8 x i16> undef, <8 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6>
 	ret <8 x i16> %tmp2
 }
 
 define <8 x i8> @test_vrev16D8(<8 x i8>* %A) nounwind {
-;CHECK-LABEL: test_vrev16D8:
-;CHECK: rev16.8b
+; CHECK-LABEL: test_vrev16D8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr d0, [x0]
+; CHECK-NEXT:    rev16.8b v0, v0
+; CHECK-NEXT:    ret
 	%tmp1 = load <8 x i8>, <8 x i8>* %A
 	%tmp2 = shufflevector <8 x i8> %tmp1, <8 x i8> undef, <8 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6>
 	ret <8 x i8> %tmp2
 }
 
 define <16 x i8> @test_vrev16Q8(<16 x i8>* %A) nounwind {
-;CHECK-LABEL: test_vrev16Q8:
-;CHECK: rev16.16b
+; CHECK-LABEL: test_vrev16Q8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr q0, [x0]
+; CHECK-NEXT:    rev16.16b v0, v0
+; CHECK-NEXT:    ret
 	%tmp1 = load <16 x i8>, <16 x i8>* %A
 	%tmp2 = shufflevector <16 x i8> %tmp1, <16 x i8> undef, <16 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6, i32 9, i32 8, i32 11, i32 10, i32 13, i32 12, i32 15, i32 14>
 	ret <16 x i8> %tmp2
@@ -203,16 +271,22 @@ define <16 x i8> @test_vrev16Q8(<16 x i8>* %A) nounwind {
 ; Undef shuffle indices should not prevent matching to VREV:
 
 define <8 x i8> @test_vrev64D8_undef(<8 x i8>* %A) nounwind {
-;CHECK-LABEL: test_vrev64D8_undef:
-;CHECK: rev64.8b
+; CHECK-LABEL: test_vrev64D8_undef:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr d0, [x0]
+; CHECK-NEXT:    rev64.8b v0, v0
+; CHECK-NEXT:    ret
 	%tmp1 = load <8 x i8>, <8 x i8>* %A
 	%tmp2 = shufflevector <8 x i8> %tmp1, <8 x i8> undef, <8 x i32> <i32 7, i32 undef, i32 undef, i32 4, i32 3, i32 2, i32 1, i32 0>
 	ret <8 x i8> %tmp2
 }
 
 define <8 x i16> @test_vrev32Q16_undef(<8 x i16>* %A) nounwind {
-;CHECK-LABEL: test_vrev32Q16_undef:
-;CHECK: rev32.8h
+; CHECK-LABEL: test_vrev32Q16_undef:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr q0, [x0]
+; CHECK-NEXT:    rev32.8h v0, v0
+; CHECK-NEXT:    ret
 	%tmp1 = load <8 x i16>, <8 x i16>* %A
 	%tmp2 = shufflevector <8 x i16> %tmp1, <8 x i16> undef, <8 x i32> <i32 undef, i32 0, i32 undef, i32 2, i32 5, i32 4, i32 7, i32 undef>
 	ret <8 x i16> %tmp2
@@ -221,9 +295,12 @@ define <8 x i16> @test_vrev32Q16_undef(<8 x i16>* %A) nounwind {
 ; vrev <4 x i16> should use REV32 and not REV64
 define void @test_vrev64(<4 x i16>* nocapture %source, <2 x i16>* nocapture %dst) nounwind ssp {
 ; CHECK-LABEL: test_vrev64:
-; CHECK: ldr [[DEST:q[0-9]+]],
-; CHECK: st1.h
-; CHECK: st1.h
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ldr q0, [x0]
+; CHECK-NEXT:    add x8, x1, #2 // =2
+; CHECK-NEXT:    st1.h { v0 }[5], [x8]
+; CHECK-NEXT:    st1.h { v0 }[6], [x1]
+; CHECK-NEXT:    ret
 entry:
   %0 = bitcast <4 x i16>* %source to <8 x i16>*
   %tmp2 = load <8 x i16>, <8 x i16>* %0, align 4
@@ -237,9 +314,15 @@ entry:
 
 ; Test vrev of float4
 define void @float_vrev64(float* nocapture %source, <4 x float>* nocapture %dest) nounwind noinline ssp {
-; CHECK: float_vrev64
-; CHECK: ldr [[DEST:q[0-9]+]],
-; CHECK: rev64.4s
+; CHECK-LABEL: float_vrev64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ldr q0, [x0]
+; CHECK-NEXT:    movi.2d v1, #0000000000000000
+; CHECK-NEXT:    dup.4s v1, v1[0]
+; CHECK-NEXT:    ext.16b v0, v0, v1, #12
+; CHECK-NEXT:    rev64.4s v0, v0
+; CHECK-NEXT:    str q0, [x1, #176]
+; CHECK-NEXT:    ret
 entry:
   %0 = bitcast float* %source to <4 x float>*
   %tmp2 = load <4 x float>, <4 x float>* %0, align 4
@@ -252,9 +335,9 @@ entry:
 
 define <4 x i32> @test_vrev32_bswap(<4 x i32> %source) nounwind {
 ; CHECK-LABEL: test_vrev32_bswap:
-; CHECK: rev32.16b
-; CHECK-NOT: rev
-; CHECK: ret
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    rev32.16b v0, v0
+; CHECK-NEXT:    ret
   %bswap = call <4 x i32> @llvm.bswap.v4i32(<4 x i32> %source)
   ret <4 x i32> %bswap
 }
diff --git a/test/CodeGen/AArch64/arm64-shrink-wrapping.ll b/test/CodeGen/AArch64/arm64-shrink-wrapping.ll
index 7ae739ebf4d5f5a8296c3c77a0064258ec6874ad..b98cb7a6676d533da3f8dc5ad47a60054e47b98e 100644
--- a/test/CodeGen/AArch64/arm64-shrink-wrapping.ll
+++ b/test/CodeGen/AArch64/arm64-shrink-wrapping.ll
@@ -1,5 +1,5 @@
-; RUN: llc %s -o - -enable-shrink-wrap=true -disable-post-ra -disable-fp-elim | FileCheck %s --check-prefix=CHECK --check-prefix=ENABLE
-; RUN: llc %s -o - -enable-shrink-wrap=false -disable-post-ra -disable-fp-elim | FileCheck %s --check-prefix=CHECK --check-prefix=DISABLE
+; RUN: llc %s -o - -enable-shrink-wrap=true -disable-post-ra -frame-pointer=all | FileCheck %s --check-prefix=CHECK --check-prefix=ENABLE
+; RUN: llc %s -o - -enable-shrink-wrap=false -disable-post-ra -frame-pointer=all | FileCheck %s --check-prefix=CHECK --check-prefix=DISABLE
 target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
 target triple = "arm64-apple-ios"
 
diff --git a/test/CodeGen/AArch64/arm64-vabs.ll b/test/CodeGen/AArch64/arm64-vabs.ll
index 53669a15b9ec978dfeafba9cbe048cfcb2b8fcc7..c6af680bb73b62ee424cfd45b2f5ee2acfe930d2 100644
--- a/test/CodeGen/AArch64/arm64-vabs.ll
+++ b/test/CodeGen/AArch64/arm64-vabs.ll
@@ -885,6 +885,20 @@ declare double @llvm.fabs.f64(double) nounwind readnone
 define <2 x i64> @uabdl_from_extract_dup(<4 x i32> %lhs, i32 %rhs) {
 ; CHECK-LABEL: uabdl_from_extract_dup:
 ; CHECK-NOT: ext.16b
+; CHECK: uabdl.2d
+  %rhsvec.tmp = insertelement <2 x i32> undef, i32 %rhs, i32 0
+  %rhsvec = insertelement <2 x i32> %rhsvec.tmp, i32 %rhs, i32 1
+
+  %lhs.high = shufflevector <4 x i32> %lhs, <4 x i32> undef, <2 x i32> <i32 0, i32 1>
+
+  %res = tail call <2 x i32> @llvm.aarch64.neon.uabd.v2i32(<2 x i32> %lhs.high, <2 x i32> %rhsvec) nounwind
+  %res1 = zext <2 x i32> %res to <2 x i64>
+  ret <2 x i64> %res1
+}
+
+define <2 x i64> @uabdl2_from_extract_dup(<4 x i32> %lhs, i32 %rhs) {
+; CHECK-LABEL: uabdl2_from_extract_dup:
+; CHECK-NOT: ext.16b
 ; CHECK: uabdl2.2d
   %rhsvec.tmp = insertelement <2 x i32> undef, i32 %rhs, i32 0
   %rhsvec = insertelement <2 x i32> %rhsvec.tmp, i32 %rhs, i32 1
@@ -899,6 +913,20 @@ define <2 x i64> @uabdl_from_extract_dup(<4 x i32> %lhs, i32 %rhs) {
 define <2 x i64> @sabdl_from_extract_dup(<4 x i32> %lhs, i32 %rhs) {
 ; CHECK-LABEL: sabdl_from_extract_dup:
 ; CHECK-NOT: ext.16b
+; CHECK: sabdl.2d
+  %rhsvec.tmp = insertelement <2 x i32> undef, i32 %rhs, i32 0
+  %rhsvec = insertelement <2 x i32> %rhsvec.tmp, i32 %rhs, i32 1
+
+  %lhs.high = shufflevector <4 x i32> %lhs, <4 x i32> undef, <2 x i32> <i32 0, i32 1>
+
+  %res = tail call <2 x i32> @llvm.aarch64.neon.sabd.v2i32(<2 x i32> %lhs.high, <2 x i32> %rhsvec) nounwind
+  %res1 = zext <2 x i32> %res to <2 x i64>
+  ret <2 x i64> %res1
+}
+
+define <2 x i64> @sabdl2_from_extract_dup(<4 x i32> %lhs, i32 %rhs) {
+; CHECK-LABEL: sabdl2_from_extract_dup:
+; CHECK-NOT: ext.16b
 ; CHECK: sabdl2.2d
   %rhsvec.tmp = insertelement <2 x i32> undef, i32 %rhs, i32 0
   %rhsvec = insertelement <2 x i32> %rhsvec.tmp, i32 %rhs, i32 1
diff --git a/test/CodeGen/AArch64/arm64-vadd.ll b/test/CodeGen/AArch64/arm64-vadd.ll
index 2a25538250e43823188af1e9b72a8a030e69d264..a244a0714c03995a5f9cfeb21ec70befc54d2e9e 100644
--- a/test/CodeGen/AArch64/arm64-vadd.ll
+++ b/test/CodeGen/AArch64/arm64-vadd.ll
@@ -738,6 +738,22 @@ declare <2 x float> @llvm.aarch64.neon.addp.v2f32(<2 x float>, <2 x float>) noun
 declare <4 x float> @llvm.aarch64.neon.addp.v4f32(<4 x float>, <4 x float>) nounwind readnone
 declare <2 x double> @llvm.aarch64.neon.addp.v2f64(<2 x double>, <2 x double>) nounwind readnone
 
+define <2 x i64> @uaddl_duprhs(<4 x i32> %lhs, i32 %rhs) {
+; CHECK-LABEL: uaddl_duprhs
+; CHECK-NOT: ext.16b
+; CHECK: uaddl.2d
+  %rhsvec.tmp = insertelement <2 x i32> undef, i32 %rhs, i32 0
+  %rhsvec = insertelement <2 x i32> %rhsvec.tmp, i32 %rhs, i32 1
+
+  %lhs.high = shufflevector <4 x i32> %lhs, <4 x i32> undef, <2 x i32> <i32 0, i32 1>
+
+  %lhs.ext = zext <2 x i32> %lhs.high to <2 x i64>
+  %rhs.ext = zext <2 x i32> %rhsvec to <2 x i64>
+
+  %res = add <2 x i64> %lhs.ext, %rhs.ext
+  ret <2 x i64> %res
+}
+
 define <2 x i64> @uaddl2_duprhs(<4 x i32> %lhs, i32 %rhs) {
 ; CHECK-LABEL: uaddl2_duprhs
 ; CHECK-NOT: ext.16b
@@ -754,6 +770,22 @@ define <2 x i64> @uaddl2_duprhs(<4 x i32> %lhs, i32 %rhs) {
   ret <2 x i64> %res
 }
 
+define <2 x i64> @saddl_duplhs(i32 %lhs, <4 x i32> %rhs) {
+; CHECK-LABEL: saddl_duplhs
+; CHECK-NOT: ext.16b
+; CHECK: saddl.2d
+  %lhsvec.tmp = insertelement <2 x i32> undef, i32 %lhs, i32 0
+  %lhsvec = insertelement <2 x i32> %lhsvec.tmp, i32 %lhs, i32 1
+
+  %rhs.high = shufflevector <4 x i32> %rhs, <4 x i32> undef, <2 x i32> <i32 0, i32 1>
+
+  %lhs.ext = sext <2 x i32> %lhsvec to <2 x i64>
+  %rhs.ext = sext <2 x i32> %rhs.high to <2 x i64>
+
+  %res = add <2 x i64> %lhs.ext, %rhs.ext
+  ret <2 x i64> %res
+}
+
 define <2 x i64> @saddl2_duplhs(i32 %lhs, <4 x i32> %rhs) {
 ; CHECK-LABEL: saddl2_duplhs
 ; CHECK-NOT: ext.16b
@@ -770,6 +802,22 @@ define <2 x i64> @saddl2_duplhs(i32 %lhs, <4 x i32> %rhs) {
   ret <2 x i64> %res
 }
 
+define <2 x i64> @usubl_duprhs(<4 x i32> %lhs, i32 %rhs) {
+; CHECK-LABEL: usubl_duprhs
+; CHECK-NOT: ext.16b
+; CHECK: usubl.2d
+  %rhsvec.tmp = insertelement <2 x i32> undef, i32 %rhs, i32 0
+  %rhsvec = insertelement <2 x i32> %rhsvec.tmp, i32 %rhs, i32 1
+
+  %lhs.high = shufflevector <4 x i32> %lhs, <4 x i32> undef, <2 x i32> <i32 0, i32 1>
+
+  %lhs.ext = zext <2 x i32> %lhs.high to <2 x i64>
+  %rhs.ext = zext <2 x i32> %rhsvec to <2 x i64>
+
+  %res = sub <2 x i64> %lhs.ext, %rhs.ext
+  ret <2 x i64> %res
+}
+
 define <2 x i64> @usubl2_duprhs(<4 x i32> %lhs, i32 %rhs) {
 ; CHECK-LABEL: usubl2_duprhs
 ; CHECK-NOT: ext.16b
@@ -786,8 +834,24 @@ define <2 x i64> @usubl2_duprhs(<4 x i32> %lhs, i32 %rhs) {
   ret <2 x i64> %res
 }
 
+define <2 x i64> @ssubl_duplhs(i32 %lhs, <4 x i32> %rhs) {
+; CHECK-LABEL: ssubl_duplhs:
+; CHECK-NOT: ext.16b
+; CHECK: ssubl.2d
+  %lhsvec.tmp = insertelement <2 x i32> undef, i32 %lhs, i32 0
+  %lhsvec = insertelement <2 x i32> %lhsvec.tmp, i32 %lhs, i32 1
+
+  %rhs.high = shufflevector <4 x i32> %rhs, <4 x i32> undef, <2 x i32> <i32 0, i32 1>
+
+  %lhs.ext = sext <2 x i32> %lhsvec to <2 x i64>
+  %rhs.ext = sext <2 x i32> %rhs.high to <2 x i64>
+
+  %res = sub <2 x i64> %lhs.ext, %rhs.ext
+  ret <2 x i64> %res
+}
+
 define <2 x i64> @ssubl2_duplhs(i32 %lhs, <4 x i32> %rhs) {
-; CHECK-LABEL: ssubl2_duplhs
+; CHECK-LABEL: ssubl2_duplhs:
 ; CHECK-NOT: ext.16b
 ; CHECK: ssubl2.2d
   %lhsvec.tmp = insertelement <2 x i32> undef, i32 %lhs, i32 0
diff --git a/test/CodeGen/AArch64/arm64-vcvt.ll b/test/CodeGen/AArch64/arm64-vcvt.ll
index f7437bc27ec2fc15b9fd5930b04c9c3b8f0aaad7..d236aeaf32a709991f7ec33c319db805bd81d78b 100644
--- a/test/CodeGen/AArch64/arm64-vcvt.ll
+++ b/test/CodeGen/AArch64/arm64-vcvt.ll
@@ -1,4 +1,7 @@
 ; RUN: llc < %s -mtriple=arm64-eabi -aarch64-neon-syntax=apple | FileCheck %s
+; RUN: llc < %s -mtriple=arm64-eabi -pass-remarks-missed=gisel-* \
+; RUN: -aarch64-neon-syntax=apple -global-isel -global-isel-abort=2 2>&1 | \
+; RUN: FileCheck %s --check-prefixes=FALLBACK,CHECK
 
 define <2 x i32> @fcvtas_2s(<2 x float> %A) nounwind {
 ;CHECK-LABEL: fcvtas_2s:
@@ -427,6 +430,7 @@ declare <2 x float> @llvm.aarch64.neon.frintn.v2f32(<2 x float>) nounwind readno
 declare <4 x float> @llvm.aarch64.neon.frintn.v4f32(<4 x float>) nounwind readnone
 declare <2 x double> @llvm.aarch64.neon.frintn.v2f64(<2 x double>) nounwind readnone
 
+; FALLBACK-NOT: remark{{.*}}frintp_2s
 define <2 x float> @frintp_2s(<2 x float> %A) nounwind {
 ;CHECK-LABEL: frintp_2s:
 ;CHECK-NOT: ld1
@@ -436,6 +440,7 @@ define <2 x float> @frintp_2s(<2 x float> %A) nounwind {
 	ret <2 x float> %tmp3
 }
 
+; FALLBACK-NOT: remark{{.*}}frintp_4s
 define <4 x float> @frintp_4s(<4 x float> %A) nounwind {
 ;CHECK-LABEL: frintp_4s:
 ;CHECK-NOT: ld1
@@ -445,6 +450,7 @@ define <4 x float> @frintp_4s(<4 x float> %A) nounwind {
 	ret <4 x float> %tmp3
 }
 
+; FALLBACK-NOT: remark{{.*}}frintp_2d
 define <2 x double> @frintp_2d(<2 x double> %A) nounwind {
 ;CHECK-LABEL: frintp_2d:
 ;CHECK-NOT: ld1
diff --git a/test/CodeGen/AArch64/arm64-vcvt_f.ll b/test/CodeGen/AArch64/arm64-vcvt_f.ll
index 90cc2d37882c7df89f8e315a82316b4ea976d239..8a1d675f9162c44a4a79e35885ba48e601342e77 100644
--- a/test/CodeGen/AArch64/arm64-vcvt_f.ll
+++ b/test/CodeGen/AArch64/arm64-vcvt_f.ll
@@ -1,39 +1,61 @@
 ; RUN: llc < %s -mtriple=arm64-eabi -aarch64-neon-syntax=apple | FileCheck %s
 ; RUN: llc < %s -O0 -fast-isel -mtriple=arm64-eabi -aarch64-neon-syntax=apple | FileCheck %s
+; RUN: llc < %s -global-isel -global-isel-abort=2 -pass-remarks-missed=gisel* \
+; RUN:          -mtriple=arm64-eabi -aarch64-neon-syntax=apple \
+; RUN:          | FileCheck %s --check-prefixes=GISEL,FALLBACK
 
+; FALLBACK-NOT: remark{{.*}}G_FPEXT{{.*}}(in function: test_vcvt_f64_f32)
+; FALLBACK-NOT: remark{{.*}}fpext{{.*}}(in function: test_vcvt_f64_f32)
 define <2 x double> @test_vcvt_f64_f32(<2 x float> %x) nounwind readnone ssp {
 ; CHECK-LABEL: test_vcvt_f64_f32:
+; GISEL-LABEL: test_vcvt_f64_f32:
   %vcvt1.i = fpext <2 x float> %x to <2 x double>
 ; CHECK: fcvtl	v0.2d, v0.2s
+; GISEL: fcvtl	v0.2d, v0.2s
   ret <2 x double> %vcvt1.i
 ; CHECK: ret
+; GISEL: ret
 }
 
+; FALLBACK-NOT: remark{{.*}}G_FPEXT{{.*}}(in function: test_vcvt_high_f64_f32)
+; FALLBACK-NOT: remark{{.*}}fpext{{.*}}(in function: test_vcvt_high_f64_f32)
 define <2 x double> @test_vcvt_high_f64_f32(<4 x float> %x) nounwind readnone ssp {
 ; CHECK-LABEL: test_vcvt_high_f64_f32:
+; GISEL-LABEL: test_vcvt_high_f64_f32:
   %cvt_in = shufflevector <4 x float> %x, <4 x float> undef, <2 x i32> <i32 2, i32 3>
   %vcvt1.i = fpext <2 x float> %cvt_in to <2 x double>
 ; CHECK: fcvtl2	v0.2d, v0.4s
+; GISEL: fcvtl2	v0.2d, v0.4s
   ret <2 x double> %vcvt1.i
 ; CHECK: ret
+; GISEL: ret
 }
 
+; FALLBACK-NOT: remark{{.*}}G_FPEXT{{.*}}(in function: test_vcvt_f32_f64)
+; FALLBACK-NOT: remark{{.*}}fpext{{.*}}(in function: test_vcvt_f32_f64)
 define <2 x float> @test_vcvt_f32_f64(<2 x double> %v) nounwind readnone ssp {
 ; CHECK-LABEL: test_vcvt_f32_f64:
+; GISEL-LABEL: test_vcvt_f32_f64:
   %vcvt1.i = fptrunc <2 x double> %v to <2 x float>
 ; CHECK: fcvtn
+; GISEL: fcvtn
   ret <2 x float> %vcvt1.i
 ; CHECK: ret
+; GISEL: ret
 }
 
+; FALLBACK-NOT: remark{{.*}}G_FPEXT{{.*}}(in function: test_vcvt_high_f32_f64)
+; FALLBACK-NOT: remark{{.*}}fpext{{.*}}(in function: test_vcvt_high_f32_f64)
 define <4 x float> @test_vcvt_high_f32_f64(<2 x float> %x, <2 x double> %v) nounwind readnone ssp {
 ; CHECK-LABEL: test_vcvt_high_f32_f64:
-
+; GISEL-LABEL: test_vcvt_high_f32_f64:
   %cvt = fptrunc <2 x double> %v to <2 x float>
   %vcvt2.i = shufflevector <2 x float> %x, <2 x float> %cvt, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
 ; CHECK: fcvtn2
+; GISEL: fcvtn2
   ret <4 x float> %vcvt2.i
 ; CHECK: ret
+; GISEL: ret
 }
 
 define <2 x float> @test_vcvtx_f32_f64(<2 x double> %v) nounwind readnone ssp {
diff --git a/test/CodeGen/AArch64/arm64-vfloatintrinsics.ll b/test/CodeGen/AArch64/arm64-vfloatintrinsics.ll
index 4e1de87f586557fb1bb6c2ea20b2e94b01c3ac6d..fef09ff49799c0c5264e90cc41b14c74c95dac36 100644
--- a/test/CodeGen/AArch64/arm64-vfloatintrinsics.ll
+++ b/test/CodeGen/AArch64/arm64-vfloatintrinsics.ll
@@ -1,57 +1,467 @@
-; RUN: llc < %s -mtriple=arm64-eabi -aarch64-neon-syntax=apple -mattr=-fullfp16 | FileCheck %s
-; RUN: llc < %s -mtriple=arm64-eabi -aarch64-neon-syntax=apple -mattr=+fullfp16 | FileCheck %s --check-prefix=CHECK-FP16
+; RUN: llc < %s -mtriple=arm64-eabi -aarch64-neon-syntax=apple -mattr=-fullfp16 \
+; RUN:     | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-NOFP16
+; RUN: llc < %s -mtriple=arm64-eabi -aarch64-neon-syntax=apple -mattr=+fullfp16 \
+; RUN:     | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-FP16
 
-;;; Float vectors
+; RUN: llc < %s -mtriple=arm64-eabi -aarch64-neon-syntax=apple -mattr=-fullfp16 \
+; RUN:     -global-isel -global-isel-abort=2 -pass-remarks-missed=gisel* \
+; RUN:     2>&1 | FileCheck %s --check-prefixes=GISEL,GISEL-NOFP16,FALLBACK
+; RUN: llc < %s -mtriple=arm64-eabi -aarch64-neon-syntax=apple -mattr=+fullfp16 \
+; RUN:     -global-isel -global-isel-abort=2 -pass-remarks-missed=gisel* \
+; RUN:     2>&1 | FileCheck %s --check-prefixes=GISEL,GISEL-FP16,FALLBACK
+
+;;; Half vectors
 
-%v2f32 = type <2 x float>
 %v4f16 = type <4 x half>
-%v8f16 = type <8 x half>
 
-; CHECK-LABEL: test_v2f32.sqrt:
-define %v2f32 @test_v2f32.sqrt(%v2f32 %a) {
-  ; CHECK: fsqrt.2s
-  %1 = call %v2f32 @llvm.sqrt.v2f32(%v2f32 %a)
-  ret %v2f32 %1
-}
+; FALLBACK-NOT: remark{{.*}}test_v4f16.sqrt
 define %v4f16 @test_v4f16.sqrt(%v4f16 %a) {
-; CHECK-LABEL: test_v4f16.sqrt:
-; CHECK:       fsqrt s{{.}}, s{{.}}
-; CHECK:       fsqrt s{{.}}, s{{.}}
-; CHECK:       fsqrt s{{.}}, s{{.}}
-; CHECK:       fsqrt s{{.}}, s{{.}}
-
-; CHECK-FP16-LABEL: test_v4f16.sqrt:
-; CHECK-FP16-NOT:   fcvt
-; CHECK-FP16:       fsqrt.4h
-; CHECK-FP16-NEXT:  ret
+  ; CHECK-LABEL:          test_v4f16.sqrt:
+  ; CHECK-NOFP16-COUNT-4: fsqrt s{{[0-9]+}}, s{{[0-9]+}}
+  ; CHECK-FP16-NOT:       fcvt
+  ; CHECK-FP16:           fsqrt.4h
+  ; CHECK-FP16-NEXT:      ret
+  ; GISEL-LABEL:          test_v4f16.sqrt:
+  ; GISEL-NOFP16-COUNT-4: fsqrt s{{[0-9]+}}, s{{[0-9]+}}
+  ; GISEL-FP16-NOT:       fcvt
+  ; GISEL-FP16:           fsqrt.4h
+  ; GISEL-FP16-NEXT:      ret
   %1 = call %v4f16 @llvm.sqrt.v4f16(%v4f16 %a)
   ret %v4f16 %1
 }
-define %v8f16 @test_v8f16.sqrt(%v8f16 %a) {
-; Filechecks are unwieldy with 16 fcvt and 8 fsqrt tests, so skipped for -fullfp16.
+define %v4f16 @test_v4f16.powi(%v4f16 %a, i32 %b) {
+  ; This operation is expanded, whether with or without +fullfp16.
+  ; CHECK-LABEL:   test_v4f16.powi:
+  ; CHECK-COUNT-4: bl __powi
+  %1 = call %v4f16 @llvm.powi.v4f16(%v4f16 %a, i32 %b)
+  ret %v4f16 %1
+}
+
+; FALLBACK-NOT: remark{{.*}}test_v4f16.sin
+define %v4f16 @test_v4f16.sin(%v4f16 %a) {
+  ; This operation is expanded, whether with or without +fullfp16.
+  ; CHECK-LABEL:   test_v4f16.sin:
+  ; CHECK-COUNT-4: bl sinf
+  ; GISEL-LABEL:   test_v4f16.sin:
+  ; GISEL-COUNT-4: bl sinf
+  %1 = call %v4f16 @llvm.sin.v4f16(%v4f16 %a)
+  ret %v4f16 %1
+}
+
+; FALLBACK-NOT: remark{{.*}}test_v4f16.cos
+define %v4f16 @test_v4f16.cos(%v4f16 %a) {
+  ; This operation is expanded, whether with or without +fullfp16.
+  ; CHECK-LABEL:   test_v4f16.cos:
+  ; CHECK-COUNT-4: bl cosf
+  ; GISEL-LABEL:   test_v4f16.cos:
+  ; GISEL-COUNT-4: bl cosf
+  %1 = call %v4f16 @llvm.cos.v4f16(%v4f16 %a)
+  ret %v4f16 %1
+}
+define %v4f16 @test_v4f16.pow(%v4f16 %a, %v4f16 %b) {
+  ; This operation is expanded, whether with or without +fullfp16.
+  ; CHECK-LABEL:   test_v4f16.pow:
+  ; CHECK-COUNT-4: bl pow
+  %1 = call %v4f16 @llvm.pow.v4f16(%v4f16 %a, %v4f16 %b)
+  ret %v4f16 %1
+}
+
+; FALLBACK-NOT: remark{{.*}}test_v4f16.exp
+define %v4f16 @test_v4f16.exp(%v4f16 %a) {
+  ; This operation is expanded, whether with or without +fullfp16.
+  ; CHECK-LABEL:   test_v4f16.exp:
+  ; CHECK-COUNT-4: bl exp
+  ; GISEL-LABEL:   test_v4f16.exp:
+  ; GISEL-COUNT-4: bl exp
+  %1 = call %v4f16 @llvm.exp.v4f16(%v4f16 %a)
+  ret %v4f16 %1
+}
+define %v4f16 @test_v4f16.exp2(%v4f16 %a) {
+  ; This operation is expanded, whether with or without +fullfp16.
+  ; CHECK-LABEL:   test_v4f16.exp2:
+  ; CHECK-COUNT-4: bl exp2
+  %1 = call %v4f16 @llvm.exp2.v4f16(%v4f16 %a)
+  ret %v4f16 %1
+}
+
+; FALLBACK-NOT: remark{{.*}}test_v4f16.log
+define %v4f16 @test_v4f16.log(%v4f16 %a) {
+  ; This operation is expanded, whether with or without +fullfp16.
+  ; CHECK-LABEL:   test_v4f16.log:
+  ; CHECK-COUNT-4: bl log
+  ; GISEL-LABEL:   test_v4f16.log:
+  ; GISEL-COUNT-4: bl log
+  %1 = call %v4f16 @llvm.log.v4f16(%v4f16 %a)
+  ret %v4f16 %1
+}
+
+; FALLBACK-NOT: remark{{.*}}test_v4f16.log10
+define %v4f16 @test_v4f16.log10(%v4f16 %a) {
+  ; This operation is expanded, whether with or without +fullfp16.
+  ; CHECK-LABEL:   test_v4f16.log10:
+  ; CHECK-COUNT-4: bl log10
+  ; GISEL-LABEL:   test_v4f16.log10:
+  ; GISEL-COUNT-4: bl log10
+  %1 = call %v4f16 @llvm.log10.v4f16(%v4f16 %a)
+  ret %v4f16 %1
+}
+
+; FALLBACK-NOT: remark{{.*}}test_v4f16.log2
+define %v4f16 @test_v4f16.log2(%v4f16 %a) {
+  ; This operation is expanded, whether with or without +fullfp16.
+  ; CHECK-LABEL:   test_v4f16.log2:
+  ; CHECK-COUNT-4: bl log2
+  ; GISEL-LABEL:   test_v4f16.log2:
+  ; GISEL-COUNT-4: bl log2
+  %1 = call %v4f16 @llvm.log2.v4f16(%v4f16 %a)
+  ret %v4f16 %1
+}
+define %v4f16 @test_v4f16.fma(%v4f16 %a, %v4f16 %b, %v4f16 %c) {
+  ; CHECK-LABEL:          test_v4f16.fma:
+  ; CHECK-NOFP16-COUNT-4: fmadd s{{[0-9]+}}, s{{[0-9]+}}, s{{[0-9]+}}, s{{[0-9]+}}
+  ; CHECK-FP16-NOT:       fcvt
+  ; CHECK-FP16:           fmla.4h
+  %1 = call %v4f16 @llvm.fma.v4f16(%v4f16 %a, %v4f16 %b, %v4f16 %c)
+  ret %v4f16 %1
+}
 
-; CHECK-FP16-LABEL: test_v8f16.sqrt:
-; CHECK-FP16-NOT:   fcvt
-; CHECK-FP16:       fsqrt.8h
-; CHECK-FP16-NEXT:  ret
+; FALLBACK-NOT: remark{{.*}}test_v4f16.fabs
+define %v4f16 @test_v4f16.fabs(%v4f16 %a) {
+  ; CHECK-LABEL:          test_v4f16.fabs:
+  ; CHECK-NOFP16-COUNT-4: fabs s{{[0-9]+}}, s{{[0-9]+}}
+  ; CHECK-FP16-NOT:       fcvt
+  ; CHECK-FP16:           fabs.4h
+  ; CHECK-FP16-NEXT:      ret
+
+  ; GISEL-LABEL:          test_v4f16.fabs:
+  ; GISEL-NOFP16-COUNT-4: fabs s{{[0-9]+}}, s{{[0-9]+}}
+  ; GISEL-FP16-NOT:       fcvt
+  ; GISEL-FP16:           fabs.4h
+  ; GISEL-FP16-NEXT:      ret
+  %1 = call %v4f16 @llvm.fabs.v4f16(%v4f16 %a)
+  ret %v4f16 %1
+}
+
+; FALLBACK-NOT: remark{{.*}}test_v4f16.floor
+define %v4f16 @test_v4f16.floor(%v4f16 %a) {
+  ; CHECK-LABEL:          test_v4f16.floor:
+  ; CHECK-NOFP16-COUNT-4: frintm s{{[0-9]+}}, s{{[0-9]+}}
+  ; CHECK-FP16-NOT:       fcvt
+  ; CHECK-FP16:           frintm.4h
+  ; CHECK-FP16-NEXT:      ret
+
+  ; GISEL-LABEL:          test_v4f16.floor:
+  ; GISEL-NOFP16-COUNT-4: frintm s{{[0-9]+}}, s{{[0-9]+}}
+  ; GISEL-FP16-NOT:       fcvt
+  ; GISEL-FP16:           frintm.4h
+  ; GISEL-FP16-NEXT:      ret
+  %1 = call %v4f16 @llvm.floor.v4f16(%v4f16 %a)
+  ret %v4f16 %1
+}
+define %v4f16 @test_v4f16.ceil(%v4f16 %a) {
+  ; CHECK-LABEL:          test_v4f16.ceil:
+  ; CHECK-NOFP16-COUNT-4: frintp s{{[0-9]+}}, s{{[0-9]+}}
+  ; CHECK-FP16-NOT:       fcvt
+  ; CHECK-FP16:           frintp.4h
+  ; CHECK-FP16-NEXT:      ret
+  ; FALLBACK-NOT: remark{{.*}}test_v4f16.ceil:
+  ; GISEL-LABEL:          test_v4f16.ceil:
+  ; GISEL-NOFP16-COUNT-4: frintp s{{[0-9]+}}, s{{[0-9]+}}
+  ; GISEL-FP16-NOT:       fcvt
+  ; GISEL-FP16:           frintp.4h
+  ; GISEL-FP16-NEXT:      ret
+  %1 = call %v4f16 @llvm.ceil.v4f16(%v4f16 %a)
+  ret %v4f16 %1
+}
+define %v4f16 @test_v4f16.trunc(%v4f16 %a) {
+  ; CHECK-LABEL:          test_v4f16.trunc:
+  ; CHECK-NOFP16-COUNT-4: frintz s{{[0-9]+}}, s{{[0-9]+}}
+  ; CHECK-FP16-NOT:       fcvt
+  ; CHECK-FP16:           frintz.4h
+  ; CHECK-FP16-NEXT:      ret
+  %1 = call %v4f16 @llvm.trunc.v4f16(%v4f16 %a)
+  ret %v4f16 %1
+}
+define %v4f16 @test_v4f16.rint(%v4f16 %a) {
+  ; CHECK-LABEL:          test_v4f16.rint:
+  ; CHECK-NOFP16-COUNT-4: frintx s{{[0-9]+}}, s{{[0-9]+}}
+  ; CHECK-FP16-NOT:       fcvt
+  ; CHECK-FP16:           frintx.4h
+  ; CHECK-FP16-NEXT:      ret
+  %1 = call %v4f16 @llvm.rint.v4f16(%v4f16 %a)
+  ret %v4f16 %1
+}
+define %v4f16 @test_v4f16.nearbyint(%v4f16 %a) {
+  ; CHECK-LABEL:          test_v4f16.nearbyint:
+  ; CHECK-NOFP16-COUNT-4: frinti s{{[0-9]+}}, s{{[0-9]+}}
+  ; CHECK-FP16-NOT:       fcvt
+  ; CHECK-FP16:           frinti.4h
+  ; CHECK-FP16-NEXT:      ret
+  %1 = call %v4f16 @llvm.nearbyint.v4f16(%v4f16 %a)
+  ret %v4f16 %1
+}
+
+declare %v4f16 @llvm.sqrt.v4f16(%v4f16) #0
+declare %v4f16 @llvm.powi.v4f16(%v4f16, i32) #0
+declare %v4f16 @llvm.sin.v4f16(%v4f16) #0
+declare %v4f16 @llvm.cos.v4f16(%v4f16) #0
+declare %v4f16 @llvm.pow.v4f16(%v4f16, %v4f16) #0
+declare %v4f16 @llvm.exp.v4f16(%v4f16) #0
+declare %v4f16 @llvm.exp2.v4f16(%v4f16) #0
+declare %v4f16 @llvm.log.v4f16(%v4f16) #0
+declare %v4f16 @llvm.log10.v4f16(%v4f16) #0
+declare %v4f16 @llvm.log2.v4f16(%v4f16) #0
+declare %v4f16 @llvm.fma.v4f16(%v4f16, %v4f16, %v4f16) #0
+declare %v4f16 @llvm.fabs.v4f16(%v4f16) #0
+declare %v4f16 @llvm.floor.v4f16(%v4f16) #0
+declare %v4f16 @llvm.ceil.v4f16(%v4f16) #0
+declare %v4f16 @llvm.trunc.v4f16(%v4f16) #0
+declare %v4f16 @llvm.rint.v4f16(%v4f16) #0
+declare %v4f16 @llvm.nearbyint.v4f16(%v4f16) #0
+
+;;;
+
+%v8f16 = type <8 x half>
+
+; FALLBACK-NOT: remark{{.*}}test_v8f16.sqrt
+define %v8f16 @test_v8f16.sqrt(%v8f16 %a) {
+  ; CHECK-LABEL:          test_v8f16.sqrt:
+  ; CHECK-NOFP16-COUNT-8: fsqrt s{{[0-9]+}}, s{{[0-9]+}}
+  ; CHECK-FP16-NOT:       fcvt
+  ; CHECK-FP16:           fsqrt.8h
+  ; CHECK-FP16-NEXT:      ret
+  ; GISEL-LABEL:          test_v8f16.sqrt:
+  ; GISEL-NOFP16-COUNT-8: fsqrt s{{[0-9]+}}, s{{[0-9]+}}
+  ; GISEL-FP16-NOT:       fcvt
+  ; GISEL-FP16:           fsqrt.8h
+  ; GISEL-FP16-NEXT:      ret
   %1 = call %v8f16 @llvm.sqrt.v8f16(%v8f16 %a)
   ret %v8f16 %1
 }
+define %v8f16 @test_v8f16.powi(%v8f16 %a, i32 %b) {
+  ; This operation is expanded, whether with or without +fullfp16.
+  ; CHECK-LABEL:   test_v8f16.powi:
+  ; CHECK-COUNT-8: bl __powi
+  %1 = call %v8f16 @llvm.powi.v8f16(%v8f16 %a, i32 %b)
+  ret %v8f16 %1
+}
+
+; FALLBACK-NOT: remark{{.*}}test_v8f16.sin
+define %v8f16 @test_v8f16.sin(%v8f16 %a) {
+  ; This operation is expanded, whether with or without +fullfp16.
+  ; CHECK-LABEL:   test_v8f16.sin:
+  ; CHECK-COUNT-8: bl sinf
+  ; GISEL-LABEL:   test_v8f16.sin:
+  ; GISEL-COUNT-8: bl sinf
+  %1 = call %v8f16 @llvm.sin.v8f16(%v8f16 %a)
+  ret %v8f16 %1
+}
+
+; FALLBACK-NOT: remark{{.*}}test_v8f16.cos
+define %v8f16 @test_v8f16.cos(%v8f16 %a) {
+  ; This operation is expanded, whether with or without +fullfp16.
+  ; CHECK-LABEL:   test_v8f16.cos:
+  ; CHECK-COUNT-8: bl cosf
+  ; GISEL-LABEL:   test_v8f16.cos:
+  ; GISEL-COUNT-8: bl cosf
+  %1 = call %v8f16 @llvm.cos.v8f16(%v8f16 %a)
+  ret %v8f16 %1
+}
+define %v8f16 @test_v8f16.pow(%v8f16 %a, %v8f16 %b) {
+  ; This operation is expanded, whether with or without +fullfp16.
+  ; CHECK-LABEL:   test_v8f16.pow:
+  ; CHECK-COUNT-8: bl pow
+  %1 = call %v8f16 @llvm.pow.v8f16(%v8f16 %a, %v8f16 %b)
+  ret %v8f16 %1
+}
+
+; FALLBACK-NOT: remark{{.*}}test_v8f16.exp
+define %v8f16 @test_v8f16.exp(%v8f16 %a) {
+  ; This operation is expanded, whether with or without +fullfp16.
+  ; CHECK-LABEL:   test_v8f16.exp:
+  ; CHECK-COUNT-8: bl exp
+  ; GISEL-LABEL:   test_v8f16.exp:
+  ; GISEL-COUNT-8: bl exp
+  %1 = call %v8f16 @llvm.exp.v8f16(%v8f16 %a)
+  ret %v8f16 %1
+}
+define %v8f16 @test_v8f16.exp2(%v8f16 %a) {
+  ; This operation is expanded, whether with or without +fullfp16.
+  ; CHECK-LABEL:   test_v8f16.exp2:
+  ; CHECK-COUNT-8: bl exp2
+  %1 = call %v8f16 @llvm.exp2.v8f16(%v8f16 %a)
+  ret %v8f16 %1
+}
+
+; FALLBACK-NOT: remark{{.*}}test_v8f16.log
+define %v8f16 @test_v8f16.log(%v8f16 %a) {
+  ; This operation is expanded, whether with or without +fullfp16.
+  ; CHECK-LABEL:   test_v8f16.log:
+  ; CHECK-COUNT-8: bl log
+  ; GISEL-LABEL:   test_v8f16.log:
+  ; GISEL-COUNT-8: bl log
+  %1 = call %v8f16 @llvm.log.v8f16(%v8f16 %a)
+  ret %v8f16 %1
+}
+
+; FALLBACK-NOT: remark{{.*}}test_v8f16.log10
+define %v8f16 @test_v8f16.log10(%v8f16 %a) {
+  ; This operation is expanded, whether with or without +fullfp16.
+  ; CHECK-LABEL:   test_v8f16.log10:
+  ; CHECK-COUNT-8: bl log10
+  ; GISEL-LABEL:   test_v8f16.log10:
+  ; GISEL-COUNT-8: bl log10
+  %1 = call %v8f16 @llvm.log10.v8f16(%v8f16 %a)
+  ret %v8f16 %1
+}
+
+; FALLBACK-NOT: remark{{.*}}test_v8f16.log2
+define %v8f16 @test_v8f16.log2(%v8f16 %a) {
+  ; This operation is expanded, whether with or without +fullfp16.
+  ; CHECK-LABEL:   test_v8f16.log2:
+  ; CHECK-COUNT-8: bl log2
+  ; GISEL-LABEL:   test_v8f16.log2:
+  ; GISEL-COUNT-8: bl log2
+  %1 = call %v8f16 @llvm.log2.v8f16(%v8f16 %a)
+  ret %v8f16 %1
+}
+define %v8f16 @test_v8f16.fma(%v8f16 %a, %v8f16 %b, %v8f16 %c) {
+  ; CHECK-LABEL:          test_v8f16.fma:
+  ; CHECK-NOFP16-COUNT-8: fmadd s{{[0-9]+}}, s{{[0-9]+}}, s{{[0-9]+}}, s{{[0-9]+}}
+  ; CHECK-FP16-NOT:       fcvt
+  ; CHECK-FP16:           fmla.8h
+  %1 = call %v8f16 @llvm.fma.v8f16(%v8f16 %a, %v8f16 %b, %v8f16 %c)
+  ret %v8f16 %1
+}
+
+; FALLBACK-NOT: remark{{.*}}test_v8f16.fabs
+define %v8f16 @test_v8f16.fabs(%v8f16 %a) {
+  ; CHECK-LABEL:          test_v8f16.fabs:
+  ; CHECK-NOFP16-COUNT-8: fabs s{{[0-9]+}}, s{{[0-9]+}}
+  ; CHECK-FP16-NOT:       fcvt
+  ; CHECK-FP16:           fabs.8h
+  ; CHECK-FP16-NEXT:      ret
+
+  ; GISEL-LABEL:          test_v8f16.fabs:
+  ; GISEL-NOFP16-COUNT-8: fabs s{{[0-9]+}}, s{{[0-9]+}}
+  ; GISEL-FP16-NOT:       fcvt
+  ; GISEL-FP16:           fabs.8h
+  ; GISEL-FP16-NEXT:      ret
+  %1 = call %v8f16 @llvm.fabs.v8f16(%v8f16 %a)
+  ret %v8f16 %1
+}
+
+; FALLBACK-NOT: remark{{.*}}test_v8f16.floor
+define %v8f16 @test_v8f16.floor(%v8f16 %a) {
+  ; CHECK-LABEL:     		  test_v8f16.floor:
+  ; CHECK-NOFP16-COUNT-8: frintm s{{[0-9]+}}, s{{[0-9]+}}
+  ; CHECK-FP16-NOT:       fcvt
+  ; CHECK-FP16:           frintm.8h
+  ; CHECK-FP16-NEXT:      ret
+
+  ; GISEL-LABEL:     		  test_v8f16.floor:
+  ; GISEL-NOFP16-COUNT-8: frintm s{{[0-9]+}}, s{{[0-9]+}}
+  ; GISEL-FP16-NOT:       fcvt
+  ; GISEL-FP16:           frintm.8h
+  ; GISEL-FP16-NEXT:      ret
+  %1 = call %v8f16 @llvm.floor.v8f16(%v8f16 %a)
+  ret %v8f16 %1
+}
+define %v8f16 @test_v8f16.ceil(%v8f16 %a) {
+  ; CHECK-LABEL:          test_v8f16.ceil:
+  ; CHECK-NOFP16-COUNT-8: frintp s{{[0-9]+}}, s{{[0-9]+}}
+  ; CHECK-FP16-NOT:       fcvt
+  ; CHECK-FP16:           frintp.8h
+  ; CHECK-FP16-NEXT:      ret
+  ; FALLBACK-NOT:         remark{{.*}}test_v8f16.ceil:
+  ; GISEL-LABEL:          test_v8f16.ceil:
+  ; GISEL-NOFP16-COUNT-8: frintp s{{[0-9]+}}, s{{[0-9]+}}
+  ; GISEL-FP16-NOT:       fcvt
+  ; GISEL-FP16:           frintp.8h
+  ; GISEL-FP16-NEXT:      ret
+  %1 = call %v8f16 @llvm.ceil.v8f16(%v8f16 %a)
+  ret %v8f16 %1
+}
+define %v8f16 @test_v8f16.trunc(%v8f16 %a) {
+  ; CHECK-LABEL:          test_v8f16.trunc:
+  ; CHECK-NOFP16-COUNT-8: frintz s{{[0-9]+}}, s{{[0-9]+}}
+  ; CHECK-FP16-NOT:       fcvt
+  ; CHECK-FP16:           frintz.8h
+  ; CHECK-FP16-NEXT:      ret
+  %1 = call %v8f16 @llvm.trunc.v8f16(%v8f16 %a)
+  ret %v8f16 %1
+}
+define %v8f16 @test_v8f16.rint(%v8f16 %a) {
+  ; CHECK-LABEL:          test_v8f16.rint:
+  ; CHECK-NOFP16-COUNT-8: frintx s{{[0-9]+}}, s{{[0-9]+}}
+  ; CHECK-FP16-NOT:       fcvt
+  ; CHECK-FP16:           frintx.8h
+  ; CHECK-FP16-NEXT:      ret
+  %1 = call %v8f16 @llvm.rint.v8f16(%v8f16 %a)
+  ret %v8f16 %1
+}
+define %v8f16 @test_v8f16.nearbyint(%v8f16 %a) {
+  ; CHECK-LABEL:          test_v8f16.nearbyint:
+  ; CHECK-NOFP16-COUNT-8: frinti s{{[0-9]+}}, s{{[0-9]+}}
+  ; CHECK-FP16-NOT:       fcvt
+  ; CHECK-FP16:           frinti.8h
+  ; CHECK-FP16-NEXT:      ret
+  %1 = call %v8f16 @llvm.nearbyint.v8f16(%v8f16 %a)
+  ret %v8f16 %1
+}
+
+declare %v8f16 @llvm.sqrt.v8f16(%v8f16) #0
+declare %v8f16 @llvm.powi.v8f16(%v8f16, i32) #0
+declare %v8f16 @llvm.sin.v8f16(%v8f16) #0
+declare %v8f16 @llvm.cos.v8f16(%v8f16) #0
+declare %v8f16 @llvm.pow.v8f16(%v8f16, %v8f16) #0
+declare %v8f16 @llvm.exp.v8f16(%v8f16) #0
+declare %v8f16 @llvm.exp2.v8f16(%v8f16) #0
+declare %v8f16 @llvm.log.v8f16(%v8f16) #0
+declare %v8f16 @llvm.log10.v8f16(%v8f16) #0
+declare %v8f16 @llvm.log2.v8f16(%v8f16) #0
+declare %v8f16 @llvm.fma.v8f16(%v8f16, %v8f16, %v8f16) #0
+declare %v8f16 @llvm.fabs.v8f16(%v8f16) #0
+declare %v8f16 @llvm.floor.v8f16(%v8f16) #0
+declare %v8f16 @llvm.ceil.v8f16(%v8f16) #0
+declare %v8f16 @llvm.trunc.v8f16(%v8f16) #0
+declare %v8f16 @llvm.rint.v8f16(%v8f16) #0
+declare %v8f16 @llvm.nearbyint.v8f16(%v8f16) #0
+
+;;; Float vectors
+
+%v2f32 = type <2 x float>
+
+; FALLBACK-NOT: remark{{.*}}test_v2f32.sqrt
+; CHECK-LABEL: test_v2f32.sqrt:
+; GISEL-LABEL: test_v2f32.sqrt:
+define %v2f32 @test_v2f32.sqrt(%v2f32 %a) {
+  ; CHECK: fsqrt.2s
+  ; GISEL: fsqrt.2s
+  %1 = call %v2f32 @llvm.sqrt.v2f32(%v2f32 %a)
+  ret %v2f32 %1
+}
 ; CHECK: test_v2f32.powi:
 define %v2f32 @test_v2f32.powi(%v2f32 %a, i32 %b) {
   ; CHECK: pow
   %1 = call %v2f32 @llvm.powi.v2f32(%v2f32 %a, i32 %b)
   ret %v2f32 %1
 }
+
+; FALLBACK-NOT: remark{{.*}}test_v2f32.sin
 ; CHECK: test_v2f32.sin:
 define %v2f32 @test_v2f32.sin(%v2f32 %a) {
   ; CHECK: sin
+  ; GISEL: sin
   %1 = call %v2f32 @llvm.sin.v2f32(%v2f32 %a)
   ret %v2f32 %1
 }
+
+; FALLBACK-NOT: remark{{.*}}test_v2f32.cos
 ; CHECK: test_v2f32.cos:
 define %v2f32 @test_v2f32.cos(%v2f32 %a) {
   ; CHECK: cos
+  ; GISEL: cos
   %1 = call %v2f32 @llvm.cos.v2f32(%v2f32 %a)
   ret %v2f32 %1
 }
@@ -61,9 +471,13 @@ define %v2f32 @test_v2f32.pow(%v2f32 %a, %v2f32 %b) {
   %1 = call %v2f32 @llvm.pow.v2f32(%v2f32 %a, %v2f32 %b)
   ret %v2f32 %1
 }
+
+; FALLBACK-NOT: remark{{.*}}test_v2f32.exp
 ; CHECK: test_v2f32.exp:
+; GISEL: test_v2f32.exp:
 define %v2f32 @test_v2f32.exp(%v2f32 %a) {
   ; CHECK: exp
+  ; GISEL: exp
   %1 = call %v2f32 @llvm.exp.v2f32(%v2f32 %a)
   ret %v2f32 %1
 }
@@ -73,21 +487,32 @@ define %v2f32 @test_v2f32.exp2(%v2f32 %a) {
   %1 = call %v2f32 @llvm.exp2.v2f32(%v2f32 %a)
   ret %v2f32 %1
 }
+
+; FALLBACK-NOT: remark{{.*}}test_v2f32.log
 ; CHECK: test_v2f32.log:
 define %v2f32 @test_v2f32.log(%v2f32 %a) {
   ; CHECK: log
+  ; GISEL: log
   %1 = call %v2f32 @llvm.log.v2f32(%v2f32 %a)
   ret %v2f32 %1
 }
+
+; FALLBACK-NOT: remark{{.*}}test_v2f32.log10
 ; CHECK: test_v2f32.log10:
+; GISEL: test_v2f32.log10:
 define %v2f32 @test_v2f32.log10(%v2f32 %a) {
   ; CHECK: log
+  ; GISEL: log
   %1 = call %v2f32 @llvm.log10.v2f32(%v2f32 %a)
   ret %v2f32 %1
 }
+
+; FALLBACK-NOT: remark{{.*}}test_v2f32.log2
 ; CHECK: test_v2f32.log2:
+; GISEL: test_v2f32.log2:
 define %v2f32 @test_v2f32.log2(%v2f32 %a) {
   ; CHECK: log
+  ; GISEL: log
   %1 = call %v2f32 @llvm.log2.v2f32(%v2f32 %a)
   ret %v2f32 %1
 }
@@ -97,211 +522,55 @@ define %v2f32 @test_v2f32.fma(%v2f32 %a, %v2f32 %b, %v2f32 %c) {
   %1 = call %v2f32 @llvm.fma.v2f32(%v2f32 %a, %v2f32 %b, %v2f32 %c)
   ret %v2f32 %1
 }
-define %v4f16 @test_v4f16.fma(%v4f16 %a, %v4f16 %b, %v4f16 %c) {
-; CHECK-LABEL: test_v4f16.fma:
-; CHECK:       fmadd s{{.}}, s{{.}}, s{{.}}, s{{.}}
-; CHECK:       fmadd s{{.}}, s{{.}}, s{{.}}, s{{.}}
-; CHECK:       fmadd s{{.}}, s{{.}}, s{{.}}, s{{.}}
-; CHECK:       fmadd s{{.}}, s{{.}}, s{{.}}, s{{.}}
-
-; CHECK-FP16-LABEL: test_v4f16.fma:
-; CHECK-FP16-NOT:   fcvt
-; CHECK-FP16:       fmla.4h
-  %1 = call %v4f16 @llvm.fma.v4f16(%v4f16 %a, %v4f16 %b, %v4f16 %c)
-  ret %v4f16 %1
-}
-define %v8f16 @test_v8f16.fma(%v8f16 %a, %v8f16 %b, %v8f16 %c) {
-; Filechecks are unwieldy with 16 fcvt and 8 fma tests, so skipped for -fullfp16.
 
-; CHECK-FP16-LABEL: test_v8f16.fma:
-; CHECK-FP16-NOT:   fcvt
-; CHECK-FP16:       fmla.8h
-  %1 = call %v8f16 @llvm.fma.v8f16(%v8f16 %a, %v8f16 %b, %v8f16 %c)
-  ret %v8f16 %1
-}
+; FALLBACK-NOT: remark{{.*}}test_v2f32.fabs
 ; CHECK-LABEL: test_v2f32.fabs:
+; GISEL-LABEL: test_v2f32.fabs:
 define %v2f32 @test_v2f32.fabs(%v2f32 %a) {
   ; CHECK: fabs.2s
+  ; GISEL: fabs.2s
   %1 = call %v2f32 @llvm.fabs.v2f32(%v2f32 %a)
   ret %v2f32 %1
 }
-define %v4f16 @test_v4f16.fabs(%v4f16 %a) {
-; CHECK-LABEL: test_v4f16.fabs:
-; CHECK:       fabs s{{.}}, s{{.}}
-; CHECK:       fabs s{{.}}, s{{.}}
-; CHECK:       fabs s{{.}}, s{{.}}
-; CHECK:       fabs s{{.}}, s{{.}}
-
-; CHECK-FP16-LABEL: test_v4f16.fabs:
-; CHECK-FP16-NOT:   fcvt
-; CHECK-FP16:       fabs.4h
-; CHECK-FP16-NEXT:  ret
-  %1 = call %v4f16 @llvm.fabs.v4f16(%v4f16 %a)
-  ret %v4f16 %1
-}
-define %v8f16 @test_v8f16.fabs(%v8f16 %a) {
-; Filechecks are unwieldy with 16 fcvt and 8 fabs tests, so skipped for -fullfp16.
 
-; CHECK-FP16-LABEL: test_v8f16.fabs:
-; CHECK-FP16-NOT:   fcvt
-; CHECK-FP16:       fabs.8h
-; CHECK-FP16-NEXT:  ret
-  %1 = call %v8f16 @llvm.fabs.v8f16(%v8f16 %a)
-  ret %v8f16 %1
-}
+; FALLBACK-NOT: remark{{.*}}test_v2f32.floor
 ; CHECK-LABEL: test_v2f32.floor:
+; GISEL-LABEL: test_v2f32.floor:
 define %v2f32 @test_v2f32.floor(%v2f32 %a) {
   ; CHECK: frintm.2s
+  ; GISEL: frintm.2s
   %1 = call %v2f32 @llvm.floor.v2f32(%v2f32 %a)
   ret %v2f32 %1
 }
-define %v4f16 @test_v4f16.floor(%v4f16 %a) {
-; CHECK-LABEL: test_v4f16.floor:
-; CHECK:       frintm s{{.}}, s{{.}}
-; CHECK:       frintm s{{.}}, s{{.}}
-; CHECK:       frintm s{{.}}, s{{.}}
-; CHECK:       frintm s{{.}}, s{{.}}
-
-; CHECK-FP16-LABEL: test_v4f16.floor:
-; CHECK-FP16-NOT:   fcvt
-; CHECK-FP16:       frintm.4h
-; CHECK-FP16-NEXT:  ret
-  %1 = call %v4f16 @llvm.floor.v4f16(%v4f16 %a)
-  ret %v4f16 %1
-}
-define %v8f16 @test_v8f16.floor(%v8f16 %a) {
-; Filechecks are unwieldy with 16 fcvt and 8 frintm tests, so skipped for -fullfp16.
-
-; CHECK-FP16-LABEL: test_v8f16.floor:
-; CHECK-FP16-NOT:   fcvt
-; CHECK-FP16:       frintm.8h
-; CHECK-FP16-NEXT:  ret
-  %1 = call %v8f16 @llvm.floor.v8f16(%v8f16 %a)
-  ret %v8f16 %1
-}
 ; CHECK-LABEL: test_v2f32.ceil:
+; FALLBACK-NOT: remark{{.*}}test_v2f32.ceil
+; GISEL-LABEL: test_v2f32.ceil:
 define %v2f32 @test_v2f32.ceil(%v2f32 %a) {
   ; CHECK: frintp.2s
+  ; GISEL: frintp.2s
   %1 = call %v2f32 @llvm.ceil.v2f32(%v2f32 %a)
   ret %v2f32 %1
 }
-define %v4f16 @test_v4f16.ceil(%v4f16 %a) {
-; CHECK-LABEL: test_v4f16.ceil:
-; CHECK:       frintp s{{.}}, s{{.}}
-; CHECK:       frintp s{{.}}, s{{.}}
-; CHECK:       frintp s{{.}}, s{{.}}
-; CHECK:       frintp s{{.}}, s{{.}}
-
-; CHECK-FP16-LABEL: test_v4f16.ceil:
-; CHECK-FP16-NOT:   fcvt
-; CHECK-FP16:       frintp.4h
-; CHECK-FP16-NEXT:  ret
-  %1 = call %v4f16 @llvm.ceil.v4f16(%v4f16 %a)
-  ret %v4f16 %1
-}
-define %v8f16 @test_v8f16.ceil(%v8f16 %a) {
-; Filechecks are unwieldy with 16 fcvt and 8 frint tests, so skipped for -fullfp16.
-
-; CHECK-FP16-LABEL: test_v8f16.ceil:
-; CHECK-FP16-NOT:   fcvt
-; CHECK-FP16:       frintp.8h
-; CHECK-FP16-NEXT:  ret
-  %1 = call %v8f16 @llvm.ceil.v8f16(%v8f16 %a)
-  ret %v8f16 %1
-}
 ; CHECK-LABEL: test_v2f32.trunc:
 define %v2f32 @test_v2f32.trunc(%v2f32 %a) {
   ; CHECK: frintz.2s
   %1 = call %v2f32 @llvm.trunc.v2f32(%v2f32 %a)
   ret %v2f32 %1
 }
-define %v4f16 @test_v4f16.trunc(%v4f16 %a) {
-; CHECK-LABEL: test_v4f16.trunc:
-; CHECK:       frintz s{{.}}, s{{.}}
-; CHECK:       frintz s{{.}}, s{{.}}
-; CHECK:       frintz s{{.}}, s{{.}}
-; CHECK:       frintz s{{.}}, s{{.}}
-
-; CHECK-FP16-LABEL: test_v4f16.trunc:
-; CHECK-FP16:       frintz.4h
-; CHECK-FP16-NEXT:  ret
-  %1 = call %v4f16 @llvm.trunc.v4f16(%v4f16 %a)
-  ret %v4f16 %1
-}
-define %v8f16 @test_v8f16.trunc(%v8f16 %a) {
-; Filechecks are unwieldy with 16 fcvt and 8 frint tests, so skipped for -fullfp16.
-
-; CHECK-FP16-LABEL: test_v8f16.trunc:
-; CHECK-FP16-NOT:   fcvt
-; CHECK-FP16:       frintz.8h
-; CHECK-FP16-NEXT:  ret
-  %1 = call %v8f16 @llvm.trunc.v8f16(%v8f16 %a)
-  ret %v8f16 %1
-}
 ; CHECK-LABEL: test_v2f32.rint:
 define %v2f32 @test_v2f32.rint(%v2f32 %a) {
   ; CHECK: frintx.2s
   %1 = call %v2f32 @llvm.rint.v2f32(%v2f32 %a)
   ret %v2f32 %1
 }
-define %v4f16 @test_v4f16.rint(%v4f16 %a) {
-; CHECK-LABEL: test_v4f16.rint:
-; CHECK:       frintx s{{.}}, s{{.}}
-; CHECK:       frintx s{{.}}, s{{.}}
-; CHECK:       frintx s{{.}}, s{{.}}
-; CHECK:       frintx s{{.}}, s{{.}}
-
-; CHECK-FP16-LABEL: test_v4f16.rint:
-; CHECK-FP16-NOT:   fcvt
-; CHECK-FP16:       frintx.4h
-; CHECK-FP16-NEXT:  ret
-  %1 = call %v4f16 @llvm.rint.v4f16(%v4f16 %a)
-  ret %v4f16 %1
-}
-define %v8f16 @test_v8f16.rint(%v8f16 %a) {
-; Filechecks are unwieldy with 16 fcvt and 8 frint tests, so skipped for -fullfp16.
-
-; CHECK-FP16-LABEL: test_v8f16.rint:
-; CHECK-FP16:       frintx.8h
-; CHECK-FP16-NEXT:  ret
-  %1 = call %v8f16 @llvm.rint.v8f16(%v8f16 %a)
-  ret %v8f16 %1
-}
 ; CHECK-LABEL: test_v2f32.nearbyint:
 define %v2f32 @test_v2f32.nearbyint(%v2f32 %a) {
   ; CHECK: frinti.2s
   %1 = call %v2f32 @llvm.nearbyint.v2f32(%v2f32 %a)
   ret %v2f32 %1
 }
-define %v4f16 @test_v4f16.nearbyint(%v4f16 %a) {
-; CHECK-LABEL: test_v4f16.nearbyint:
-; CHECK:       frinti s{{.}}, s{{.}}
-; CHECK:       frinti s{{.}}, s{{.}}
-; CHECK:       frinti s{{.}}, s{{.}}
-; CHECK:       frinti s{{.}}, s{{.}}
-
-; CHECK-FP16-LABEL: test_v4f16.nearbyint:
-; CHECK-FP16-NOT:   fcvt
-; CHECK-FP16:       frinti.4h
-; CHECK-FP16-NEXT:  ret
-  %1 = call %v4f16 @llvm.nearbyint.v4f16(%v4f16 %a)
-  ret %v4f16 %1
-}
-define %v8f16 @test_v8f16.nearbyint(%v8f16 %a) {
-; Filechecks are unwieldy with 16 fcvt and 8 frint tests, so skipped for -fullfp16.
-
-; CHECK-FP16-LABEL: test_v8f16.nearbyint:
-; CHECK-FP16-NOT:   fcvt
-; CHECK-FP16:       frinti.8h
-; CHECK-FP16-NEXT:  ret
-  %1 = call %v8f16 @llvm.nearbyint.v8f16(%v8f16 %a)
-  ret %v8f16 %1
-}
 
 declare %v2f32 @llvm.sqrt.v2f32(%v2f32) #0
-declare %v4f16 @llvm.sqrt.v4f16(%v4f16) #0
-declare %v8f16 @llvm.sqrt.v8f16(%v8f16) #0
-
 declare %v2f32 @llvm.powi.v2f32(%v2f32, i32) #0
 declare %v2f32 @llvm.sin.v2f32(%v2f32) #0
 declare %v2f32 @llvm.cos.v2f32(%v2f32) #0
@@ -311,41 +580,24 @@ declare %v2f32 @llvm.exp2.v2f32(%v2f32) #0
 declare %v2f32 @llvm.log.v2f32(%v2f32) #0
 declare %v2f32 @llvm.log10.v2f32(%v2f32) #0
 declare %v2f32 @llvm.log2.v2f32(%v2f32) #0
-
 declare %v2f32 @llvm.fma.v2f32(%v2f32, %v2f32, %v2f32) #0
-declare %v4f16 @llvm.fma.v4f16(%v4f16, %v4f16, %v4f16) #0
-declare %v8f16 @llvm.fma.v8f16(%v8f16, %v8f16, %v8f16) #0
-
 declare %v2f32 @llvm.fabs.v2f32(%v2f32) #0
-declare %v4f16 @llvm.fabs.v4f16(%v4f16) #0
-declare %v8f16 @llvm.fabs.v8f16(%v8f16) #0
-
 declare %v2f32 @llvm.floor.v2f32(%v2f32) #0
-declare %v4f16 @llvm.floor.v4f16(%v4f16) #0
-declare %v8f16 @llvm.floor.v8f16(%v8f16) #0
-
 declare %v2f32 @llvm.ceil.v2f32(%v2f32) #0
-declare %v4f16 @llvm.ceil.v4f16(%v4f16) #0
-declare %v8f16 @llvm.ceil.v8f16(%v8f16) #0
-
 declare %v2f32 @llvm.trunc.v2f32(%v2f32) #0
-declare %v4f16 @llvm.trunc.v4f16(%v4f16) #0
-declare %v8f16 @llvm.trunc.v8f16(%v8f16) #0
-
 declare %v2f32 @llvm.rint.v2f32(%v2f32) #0
-declare %v4f16 @llvm.rint.v4f16(%v4f16) #0
-declare %v8f16 @llvm.rint.v8f16(%v8f16) #0
-
 declare %v2f32 @llvm.nearbyint.v2f32(%v2f32) #0
-declare %v4f16 @llvm.nearbyint.v4f16(%v4f16) #0
-declare %v8f16 @llvm.nearbyint.v8f16(%v8f16) #0
 
 ;;;
 
 %v4f32 = type <4 x float>
+
+; FALLBACK-NOT: remark{{.*}}test_v4f32.sqrt
 ; CHECK: test_v4f32.sqrt:
+; GISEL: test_v4f32.sqrt:
 define %v4f32 @test_v4f32.sqrt(%v4f32 %a) {
   ; CHECK: fsqrt.4s
+  ; GISEL: fsqrt.4s
   %1 = call %v4f32 @llvm.sqrt.v4f32(%v4f32 %a)
   ret %v4f32 %1
 }
@@ -355,15 +607,21 @@ define %v4f32 @test_v4f32.powi(%v4f32 %a, i32 %b) {
   %1 = call %v4f32 @llvm.powi.v4f32(%v4f32 %a, i32 %b)
   ret %v4f32 %1
 }
+
+; FALLBACK-NOT: remark{{.*}}test_v4f32.sin
 ; CHECK: test_v4f32.sin:
 define %v4f32 @test_v4f32.sin(%v4f32 %a) {
   ; CHECK: sin
+  ; GISEL: sin
   %1 = call %v4f32 @llvm.sin.v4f32(%v4f32 %a)
   ret %v4f32 %1
 }
+
+; FALLBACK-NOT: remark{{.*}}test_v4f32.cos
 ; CHECK: test_v4f32.cos:
 define %v4f32 @test_v4f32.cos(%v4f32 %a) {
   ; CHECK: cos
+  ; GISEL: cos
   %1 = call %v4f32 @llvm.cos.v4f32(%v4f32 %a)
   ret %v4f32 %1
 }
@@ -373,9 +631,13 @@ define %v4f32 @test_v4f32.pow(%v4f32 %a, %v4f32 %b) {
   %1 = call %v4f32 @llvm.pow.v4f32(%v4f32 %a, %v4f32 %b)
   ret %v4f32 %1
 }
+
+; FALLBACK-NOT: remark{{.*}}test_v4f32.exp
 ; CHECK: test_v4f32.exp:
+; GISEL: test_v4f32.exp:
 define %v4f32 @test_v4f32.exp(%v4f32 %a) {
   ; CHECK: exp
+  ; GISEL: exp
   %1 = call %v4f32 @llvm.exp.v4f32(%v4f32 %a)
   ret %v4f32 %1
 }
@@ -385,21 +647,31 @@ define %v4f32 @test_v4f32.exp2(%v4f32 %a) {
   %1 = call %v4f32 @llvm.exp2.v4f32(%v4f32 %a)
   ret %v4f32 %1
 }
+
+; FALLBACK-NOT: remark{{.*}}test_v4f32.log
 ; CHECK: test_v4f32.log:
 define %v4f32 @test_v4f32.log(%v4f32 %a) {
   ; CHECK: log
+  ; GISEL: log
   %1 = call %v4f32 @llvm.log.v4f32(%v4f32 %a)
   ret %v4f32 %1
 }
+
+; FALLBACK-NOT: remark{{.*}}test_v4f32.log10
 ; CHECK: test_v4f32.log10:
 define %v4f32 @test_v4f32.log10(%v4f32 %a) {
   ; CHECK: log
+  ; GISEL: log
   %1 = call %v4f32 @llvm.log10.v4f32(%v4f32 %a)
   ret %v4f32 %1
 }
+
+; FALLBACK-NOT: remark{{.*}}test_v4f32.log2
 ; CHECK: test_v4f32.log2:
+; GISEL: test_v4f32.log2:
 define %v4f32 @test_v4f32.log2(%v4f32 %a) {
   ; CHECK: log
+  ; GISEL: log
   %1 = call %v4f32 @llvm.log2.v4f32(%v4f32 %a)
   ret %v4f32 %1
 }
@@ -409,21 +681,32 @@ define %v4f32 @test_v4f32.fma(%v4f32 %a, %v4f32 %b, %v4f32 %c) {
   %1 = call %v4f32 @llvm.fma.v4f32(%v4f32 %a, %v4f32 %b, %v4f32 %c)
   ret %v4f32 %1
 }
+
+; FALLBACK-NOT: remark{{.*}}test_v4f32.fabs
 ; CHECK: test_v4f32.fabs:
+; GISEL: test_v4f32.fabs:
 define %v4f32 @test_v4f32.fabs(%v4f32 %a) {
   ; CHECK: fabs
+  ; GISEL: fabs
   %1 = call %v4f32 @llvm.fabs.v4f32(%v4f32 %a)
   ret %v4f32 %1
 }
+
+; FALLBACK-NOT: remark{{.*}}test_v4f32.floor
 ; CHECK: test_v4f32.floor:
+; GISEL: test_v4f32.floor:
 define %v4f32 @test_v4f32.floor(%v4f32 %a) {
   ; CHECK: frintm.4s
+  ; GISEL frintm.4s
   %1 = call %v4f32 @llvm.floor.v4f32(%v4f32 %a)
   ret %v4f32 %1
 }
 ; CHECK: test_v4f32.ceil:
+; FALLBACK-NOT: remark{{.*}}test_v4f32.ceil
+; GISEL-LABEL: test_v4f32.ceil:
 define %v4f32 @test_v4f32.ceil(%v4f32 %a) {
   ; CHECK: frintp.4s
+  ; GISEL: frintp.4s
   %1 = call %v4f32 @llvm.ceil.v4f32(%v4f32 %a)
   ret %v4f32 %1
 }
@@ -467,9 +750,12 @@ declare %v4f32 @llvm.nearbyint.v4f32(%v4f32) #0
 ;;; Double vector
 
 %v2f64 = type <2 x double>
+; FALLBACK-NOT: remark{{.*}}test_v2f64.sqrt
 ; CHECK: test_v2f64.sqrt:
+; GISEL: test_v2f64.sqrt:
 define %v2f64 @test_v2f64.sqrt(%v2f64 %a) {
   ; CHECK: fsqrt.2d
+  ; GISEL: fsqrt.2d
   %1 = call %v2f64 @llvm.sqrt.v2f64(%v2f64 %a)
   ret %v2f64 %1
 }
@@ -479,15 +765,21 @@ define %v2f64 @test_v2f64.powi(%v2f64 %a, i32 %b) {
   %1 = call %v2f64 @llvm.powi.v2f64(%v2f64 %a, i32 %b)
   ret %v2f64 %1
 }
+
+; FALLBACK-NOT: remark{{.*}}test_v2f64.sin
 ; CHECK: test_v2f64.sin:
 define %v2f64 @test_v2f64.sin(%v2f64 %a) {
   ; CHECK: sin
+  ; GISEL: sin
   %1 = call %v2f64 @llvm.sin.v2f64(%v2f64 %a)
   ret %v2f64 %1
 }
+
+; FALLBACK-NOT: remark{{.*}}test_v2f64.cos
 ; CHECK: test_v2f64.cos:
 define %v2f64 @test_v2f64.cos(%v2f64 %a) {
   ; CHECK: cos
+  ; GISEL: cos
   %1 = call %v2f64 @llvm.cos.v2f64(%v2f64 %a)
   ret %v2f64 %1
 }
@@ -497,9 +789,13 @@ define %v2f64 @test_v2f64.pow(%v2f64 %a, %v2f64 %b) {
   %1 = call %v2f64 @llvm.pow.v2f64(%v2f64 %a, %v2f64 %b)
   ret %v2f64 %1
 }
+
+; FALLBACK-NOT: remark{{.*}}test_v2f64.exp
 ; CHECK: test_v2f64.exp:
+; GISEL: test_v2f64.exp:
 define %v2f64 @test_v2f64.exp(%v2f64 %a) {
   ; CHECK: exp
+  ; GISEL: exp
   %1 = call %v2f64 @llvm.exp.v2f64(%v2f64 %a)
   ret %v2f64 %1
 }
@@ -509,21 +805,32 @@ define %v2f64 @test_v2f64.exp2(%v2f64 %a) {
   %1 = call %v2f64 @llvm.exp2.v2f64(%v2f64 %a)
   ret %v2f64 %1
 }
+
+; FALLBACK-NOT: remark{{.*}}test_v2f64.log
 ; CHECK: test_v2f64.log:
 define %v2f64 @test_v2f64.log(%v2f64 %a) {
   ; CHECK: log
+  ; GISEL: log
   %1 = call %v2f64 @llvm.log.v2f64(%v2f64 %a)
   ret %v2f64 %1
 }
+
+; FALLBACK-NOT: remark{{.*}}test_v2f64.log10
 ; CHECK: test_v2f64.log10:
+; GISEL: test_v2f64.log10:
 define %v2f64 @test_v2f64.log10(%v2f64 %a) {
   ; CHECK: log
+  ; GISEL: log
   %1 = call %v2f64 @llvm.log10.v2f64(%v2f64 %a)
   ret %v2f64 %1
 }
+
+; FALLBACK-NOT: remark{{.*}}test_v2f64.log2
 ; CHECK: test_v2f64.log2:
+; GISEL: test_v2f64.log2:
 define %v2f64 @test_v2f64.log2(%v2f64 %a) {
   ; CHECK: log
+  ; GISEL: log
   %1 = call %v2f64 @llvm.log2.v2f64(%v2f64 %a)
   ret %v2f64 %1
 }
@@ -533,21 +840,32 @@ define %v2f64 @test_v2f64.fma(%v2f64 %a, %v2f64 %b, %v2f64 %c) {
   %1 = call %v2f64 @llvm.fma.v2f64(%v2f64 %a, %v2f64 %b, %v2f64 %c)
   ret %v2f64 %1
 }
+
+; FALLBACK-NOT: remark{{.*}}test_v2f64.fabs
 ; CHECK: test_v2f64.fabs:
+; GISEL: test_v2f64.fabs:
 define %v2f64 @test_v2f64.fabs(%v2f64 %a) {
   ; CHECK: fabs
+  ; GISEL: fabs
   %1 = call %v2f64 @llvm.fabs.v2f64(%v2f64 %a)
   ret %v2f64 %1
 }
+
+; FALLBACK-NOT: remark{{.*}}test_v2f64.floor
 ; CHECK: test_v2f64.floor:
+; GISEL: test_v2f64.floor:
 define %v2f64 @test_v2f64.floor(%v2f64 %a) {
   ; CHECK: frintm.2d
+  ; GISEL: frintm.2d
   %1 = call %v2f64 @llvm.floor.v2f64(%v2f64 %a)
   ret %v2f64 %1
 }
 ; CHECK: test_v2f64.ceil:
+; FALLBACK-NOT: remark{{.*}}test_v2f64.ceil
+; GISEL-LABEL: test_v2f64.ceil:
 define %v2f64 @test_v2f64.ceil(%v2f64 %a) {
   ; CHECK: frintp.2d
+  ; GISEL: frintp.2d
   %1 = call %v2f64 @llvm.ceil.v2f64(%v2f64 %a)
   ret %v2f64 %1
 }
diff --git a/test/CodeGen/AArch64/arm64-vmul.ll b/test/CodeGen/AArch64/arm64-vmul.ll
index f70ed9a43427869581674ccca76545039152e88d..6d795dbcff810e34b32ce58035cfc07b90e7f786 100644
--- a/test/CodeGen/AArch64/arm64-vmul.ll
+++ b/test/CodeGen/AArch64/arm64-vmul.ll
@@ -1338,6 +1338,19 @@ entry:
   ret <4 x i32> %vmull2.i
 }
 
+define <4 x i32> @foo6a(<4 x i32> %a, <8 x i16> %b, <4 x i16> %c) nounwind readnone optsize ssp {
+; CHECK-LABEL: foo6a:
+; CHECK-NEXT: smull.4s v0, v1, v2[1]
+; CHECK-NEXT: ret
+entry:
+  %0 = bitcast <8 x i16> %b to <2 x i64>
+  %shuffle.i = shufflevector <2 x i64> %0, <2 x i64> undef, <1 x i32> <i32 0>
+  %1 = bitcast <1 x i64> %shuffle.i to <4 x i16>
+  %shuffle = shufflevector <4 x i16> %c, <4 x i16> undef, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+  %vmull2.i = tail call <4 x i32> @llvm.aarch64.neon.smull.v4i32(<4 x i16> %1, <4 x i16> %shuffle) nounwind
+  ret <4 x i32> %vmull2.i
+}
+
 define <2 x i64> @foo7(<2 x i64> %a, <4 x i32> %b, <2 x i32> %c) nounwind readnone optsize ssp {
 ; CHECK-LABEL: foo7:
 ; CHECK-NEXT: smull2.2d v0, v1, v2[1]
@@ -1351,6 +1364,20 @@ entry:
   ret <2 x i64> %vmull2.i
 }
 
+define <2 x i64> @foo7a(<2 x i64> %a, <4 x i32> %b, <2 x i32> %c) nounwind readnone optsize ssp {
+; CHECK-LABEL: foo7a:
+; CHECK-NEXT: smull.2d v0, v1, v2[1]
+; CHECK-NEXT: ret
+entry:
+  %0 = bitcast <4 x i32> %b to <2 x i64>
+  %shuffle.i = shufflevector <2 x i64> %0, <2 x i64> undef, <1 x i32> <i32 0>
+  %1 = bitcast <1 x i64> %shuffle.i to <2 x i32>
+  %shuffle = shufflevector <2 x i32> %c, <2 x i32> undef, <2 x i32> <i32 1, i32 1>
+  %vmull2.i = tail call <2 x i64> @llvm.aarch64.neon.smull.v2i64(<2 x i32> %1, <2 x i32> %shuffle) nounwind
+  ret <2 x i64> %vmull2.i
+}
+
+
 define <4 x i32> @foo8(<4 x i32> %a, <8 x i16> %b, <4 x i16> %c) nounwind readnone optsize ssp {
 ; CHECK-LABEL: foo8:
 ; CHECK-NEXT: umull2.4s v0, v1, v2[1]
@@ -1364,6 +1391,19 @@ entry:
   ret <4 x i32> %vmull2.i
 }
 
+define <4 x i32> @foo8a(<4 x i32> %a, <8 x i16> %b, <4 x i16> %c) nounwind readnone optsize ssp {
+; CHECK-LABEL: foo8a:
+; CHECK-NEXT: umull.4s v0, v1, v2[1]
+; CHECK-NEXT: ret
+entry:
+  %0 = bitcast <8 x i16> %b to <2 x i64>
+  %shuffle.i = shufflevector <2 x i64> %0, <2 x i64> undef, <1 x i32> <i32 0>
+  %1 = bitcast <1 x i64> %shuffle.i to <4 x i16>
+  %shuffle = shufflevector <4 x i16> %c, <4 x i16> undef, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+  %vmull2.i = tail call <4 x i32> @llvm.aarch64.neon.umull.v4i32(<4 x i16> %1, <4 x i16> %shuffle) nounwind
+  ret <4 x i32> %vmull2.i
+}
+
 define <2 x i64> @foo9(<2 x i64> %a, <4 x i32> %b, <2 x i32> %c) nounwind readnone optsize ssp {
 ; CHECK-LABEL: foo9:
 ; CHECK-NEXT: umull2.2d v0, v1, v2[1]
@@ -1377,6 +1417,19 @@ entry:
   ret <2 x i64> %vmull2.i
 }
 
+define <2 x i64> @foo9a(<2 x i64> %a, <4 x i32> %b, <2 x i32> %c) nounwind readnone optsize ssp {
+; CHECK-LABEL: foo9a:
+; CHECK-NEXT: umull.2d v0, v1, v2[1]
+; CHECK-NEXT: ret
+entry:
+  %0 = bitcast <4 x i32> %b to <2 x i64>
+  %shuffle.i = shufflevector <2 x i64> %0, <2 x i64> undef, <1 x i32> <i32 0>
+  %1 = bitcast <1 x i64> %shuffle.i to <2 x i32>
+  %shuffle = shufflevector <2 x i32> %c, <2 x i32> undef, <2 x i32> <i32 1, i32 1>
+  %vmull2.i = tail call <2 x i64> @llvm.aarch64.neon.umull.v2i64(<2 x i32> %1, <2 x i32> %shuffle) nounwind
+  ret <2 x i64> %vmull2.i
+}
+
 define <8 x i16> @bar0(<8 x i16> %a, <16 x i8> %b, <16 x i8> %c) nounwind {
 ; CHECK-LABEL: bar0:
 ; CHECK: smlal2.8h v0, v1, v2
@@ -1667,6 +1720,24 @@ entry:
   ret <2 x i64> %vmull2.i
 }
 
+define <4 x i32> @vmull_low_n_s16_test(<4 x i32> %a, <8 x i16> %b, <4 x i16> %c, i32 %d) nounwind readnone optsize ssp {
+entry:
+; CHECK: vmull_low_n_s16_test
+; CHECK-NOT: ext
+; CHECK: smull.4s
+; CHECK-NEXT: ret
+  %conv = trunc i32 %d to i16
+  %0 = bitcast <8 x i16> %b to <2 x i64>
+  %shuffle.i.i = shufflevector <2 x i64> %0, <2 x i64> undef, <1 x i32> <i32 0>
+  %1 = bitcast <1 x i64> %shuffle.i.i to <4 x i16>
+  %vecinit.i = insertelement <4 x i16> undef, i16 %conv, i32 0
+  %vecinit1.i = insertelement <4 x i16> %vecinit.i, i16 %conv, i32 1
+  %vecinit2.i = insertelement <4 x i16> %vecinit1.i, i16 %conv, i32 2
+  %vecinit3.i = insertelement <4 x i16> %vecinit2.i, i16 %conv, i32 3
+  %vmull2.i.i = tail call <4 x i32> @llvm.aarch64.neon.smull.v4i32(<4 x i16> %1, <4 x i16> %vecinit3.i) nounwind
+  ret <4 x i32> %vmull2.i.i
+}
+
 define <4 x i32> @vmull_high_n_s16_test(<4 x i32> %a, <8 x i16> %b, <4 x i16> %c, i32 %d) nounwind readnone optsize ssp {
 entry:
 ; CHECK: vmull_high_n_s16_test
@@ -1804,8 +1875,21 @@ define <2 x i64> @mlal_from_two_extracts(<2 x i64> %accum, <4 x i32> %lhs, <4 x
   ret <2 x i64> %sum
 }
 
-define <2 x i64> @mull_from_extract_dup(<4 x i32> %lhs, i32 %rhs) {
-; CHECK-LABEL: mull_from_extract_dup:
+define <2 x i64> @mull_from_extract_dup_low(<4 x i32> %lhs, i32 %rhs) {
+; CHECK-LABEL: mull_from_extract_dup_low:
+; CHECK-NOT: ext
+; CHECK: sqdmull.2d
+  %rhsvec.tmp = insertelement <2 x i32> undef, i32 %rhs, i32 0
+  %rhsvec = insertelement <2 x i32> %rhsvec.tmp, i32 %rhs, i32 1
+
+  %lhs.high = shufflevector <4 x i32> %lhs, <4 x i32> undef, <2 x i32> <i32 0, i32 1>
+
+  %res = tail call <2 x i64> @llvm.aarch64.neon.sqdmull.v2i64(<2 x i32> %lhs.high, <2 x i32> %rhsvec) nounwind
+  ret <2 x i64> %res
+}
+
+define <2 x i64> @mull_from_extract_dup_high(<4 x i32> %lhs, i32 %rhs) {
+; CHECK-LABEL: mull_from_extract_dup_high:
 ; CHECK-NOT: ext
 ; CHECK: sqdmull2.2d
   %rhsvec.tmp = insertelement <2 x i32> undef, i32 %rhs, i32 0
@@ -1817,8 +1901,21 @@ define <2 x i64> @mull_from_extract_dup(<4 x i32> %lhs, i32 %rhs) {
   ret <2 x i64> %res
 }
 
-define <8 x i16> @pmull_from_extract_dup(<16 x i8> %lhs, i8 %rhs) {
-; CHECK-LABEL: pmull_from_extract_dup:
+define <8 x i16> @pmull_from_extract_dup_low(<16 x i8> %lhs, i8 %rhs) {
+; CHECK-LABEL: pmull_from_extract_dup_low:
+; CHECK-NOT: ext
+; CHECK: pmull.8h
+  %rhsvec.0 = insertelement <8 x i8> undef, i8 %rhs, i32 0
+  %rhsvec = shufflevector <8 x i8> %rhsvec.0, <8 x i8> undef, <8 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0>
+
+  %lhs.high = shufflevector <16 x i8> %lhs, <16 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+
+  %res = tail call <8 x i16> @llvm.aarch64.neon.pmull.v8i16(<8 x i8> %lhs.high, <8 x i8> %rhsvec) nounwind
+  ret <8 x i16> %res
+}
+
+define <8 x i16> @pmull_from_extract_dup_high(<16 x i8> %lhs, i8 %rhs) {
+; CHECK-LABEL: pmull_from_extract_dup_high:
 ; CHECK-NOT: ext
 ; CHECK: pmull2.8h
   %rhsvec.0 = insertelement <8 x i8> undef, i8 %rhs, i32 0
@@ -1830,8 +1927,20 @@ define <8 x i16> @pmull_from_extract_dup(<16 x i8> %lhs, i8 %rhs) {
   ret <8 x i16> %res
 }
 
-define <8 x i16> @pmull_from_extract_duplane(<16 x i8> %lhs, <8 x i8> %rhs) {
-; CHECK-LABEL: pmull_from_extract_duplane:
+define <8 x i16> @pmull_from_extract_duplane_low(<16 x i8> %lhs, <8 x i8> %rhs) {
+; CHECK-LABEL: pmull_from_extract_duplane_low:
+; CHECK-NOT: ext
+; CHECK: pmull.8h
+
+  %lhs.high = shufflevector <16 x i8> %lhs, <16 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %rhs.high = shufflevector <8 x i8> %rhs, <8 x i8> undef, <8 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0>
+
+  %res = tail call <8 x i16> @llvm.aarch64.neon.pmull.v8i16(<8 x i8> %lhs.high, <8 x i8> %rhs.high) nounwind
+  ret <8 x i16> %res
+}
+
+define <8 x i16> @pmull_from_extract_duplane_high(<16 x i8> %lhs, <8 x i8> %rhs) {
+; CHECK-LABEL: pmull_from_extract_duplane_high:
 ; CHECK-NOT: ext
 ; CHECK: pmull2.8h
 
@@ -1842,8 +1951,20 @@ define <8 x i16> @pmull_from_extract_duplane(<16 x i8> %lhs, <8 x i8> %rhs) {
   ret <8 x i16> %res
 }
 
-define <2 x i64> @sqdmull_from_extract_duplane(<4 x i32> %lhs, <4 x i32> %rhs) {
-; CHECK-LABEL: sqdmull_from_extract_duplane:
+define <2 x i64> @sqdmull_from_extract_duplane_low(<4 x i32> %lhs, <4 x i32> %rhs) {
+; CHECK-LABEL: sqdmull_from_extract_duplane_low:
+; CHECK-NOT: ext
+; CHECK: sqdmull.2d
+
+  %lhs.high = shufflevector <4 x i32> %lhs, <4 x i32> undef, <2 x i32> <i32 0, i32 1>
+  %rhs.high = shufflevector <4 x i32> %rhs, <4 x i32> undef, <2 x i32> <i32 0, i32 0>
+
+  %res = tail call <2 x i64> @llvm.aarch64.neon.sqdmull.v2i64(<2 x i32> %lhs.high, <2 x i32> %rhs.high) nounwind
+  ret <2 x i64> %res
+}
+
+define <2 x i64> @sqdmull_from_extract_duplane_high(<4 x i32> %lhs, <4 x i32> %rhs) {
+; CHECK-LABEL: sqdmull_from_extract_duplane_high:
 ; CHECK-NOT: ext
 ; CHECK: sqdmull2.2d
 
@@ -1854,8 +1975,21 @@ define <2 x i64> @sqdmull_from_extract_duplane(<4 x i32> %lhs, <4 x i32> %rhs) {
   ret <2 x i64> %res
 }
 
-define <2 x i64> @sqdmlal_from_extract_duplane(<2 x i64> %accum, <4 x i32> %lhs, <4 x i32> %rhs) {
-; CHECK-LABEL: sqdmlal_from_extract_duplane:
+define <2 x i64> @sqdmlal_from_extract_duplane_low(<2 x i64> %accum, <4 x i32> %lhs, <4 x i32> %rhs) {
+; CHECK-LABEL: sqdmlal_from_extract_duplane_low:
+; CHECK-NOT: ext
+; CHECK: sqdmlal.2d
+
+  %lhs.high = shufflevector <4 x i32> %lhs, <4 x i32> undef, <2 x i32> <i32 0, i32 1>
+  %rhs.high = shufflevector <4 x i32> %rhs, <4 x i32> undef, <2 x i32> <i32 0, i32 0>
+
+  %res = tail call <2 x i64> @llvm.aarch64.neon.sqdmull.v2i64(<2 x i32> %lhs.high, <2 x i32> %rhs.high) nounwind
+  %sum = call <2 x i64> @llvm.aarch64.neon.sqadd.v2i64(<2 x i64> %accum, <2 x i64> %res)
+  ret <2 x i64> %sum
+}
+
+define <2 x i64> @sqdmlal_from_extract_duplane_high(<2 x i64> %accum, <4 x i32> %lhs, <4 x i32> %rhs) {
+; CHECK-LABEL: sqdmlal_from_extract_duplane_high:
 ; CHECK-NOT: ext
 ; CHECK: sqdmlal2.2d
 
@@ -1867,8 +2001,21 @@ define <2 x i64> @sqdmlal_from_extract_duplane(<2 x i64> %accum, <4 x i32> %lhs,
   ret <2 x i64> %sum
 }
 
-define <2 x i64> @umlal_from_extract_duplane(<2 x i64> %accum, <4 x i32> %lhs, <4 x i32> %rhs) {
-; CHECK-LABEL: umlal_from_extract_duplane:
+define <2 x i64> @umlal_from_extract_duplane_low(<2 x i64> %accum, <4 x i32> %lhs, <4 x i32> %rhs) {
+; CHECK-LABEL: umlal_from_extract_duplane_low:
+; CHECK-NOT: ext
+; CHECK: umlal.2d
+
+  %lhs.high = shufflevector <4 x i32> %lhs, <4 x i32> undef, <2 x i32> <i32 0, i32 1>
+  %rhs.high = shufflevector <4 x i32> %rhs, <4 x i32> undef, <2 x i32> <i32 0, i32 0>
+
+  %res = tail call <2 x i64> @llvm.aarch64.neon.umull.v2i64(<2 x i32> %lhs.high, <2 x i32> %rhs.high) nounwind
+  %sum = add <2 x i64> %accum, %res
+  ret <2 x i64> %sum
+}
+
+define <2 x i64> @umlal_from_extract_duplane_high(<2 x i64> %accum, <4 x i32> %lhs, <4 x i32> %rhs) {
+; CHECK-LABEL: umlal_from_extract_duplane_high:
 ; CHECK-NOT: ext
 ; CHECK: umlal2.2d
 
diff --git a/test/CodeGen/AArch64/bool-ext-inc.ll b/test/CodeGen/AArch64/bool-ext-inc.ll
new file mode 100644
index 0000000000000000000000000000000000000000..8f45e97924118f84dab8c51fb6437f3ef6f770ef
--- /dev/null
+++ b/test/CodeGen/AArch64/bool-ext-inc.ll
@@ -0,0 +1,57 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=aarch64-- | FileCheck %s
+
+define <4 x i32> @sextbool_add_vector(<4 x i32> %c1, <4 x i32> %c2, <4 x i32> %x) {
+; CHECK-LABEL: sextbool_add_vector:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmeq v0.4s, v0.4s, v1.4s
+; CHECK-NEXT:    add v0.4s, v2.4s, v0.4s
+; CHECK-NEXT:    ret
+  %c = icmp eq <4 x i32> %c1, %c2
+  %b = sext <4 x i1> %c to <4 x i32>
+  %s = add <4 x i32> %x, %b
+  ret <4 x i32> %s
+}
+
+define <4 x i32> @zextbool_sub_vector(<4 x i32> %c1, <4 x i32> %c2, <4 x i32> %x) {
+; CHECK-LABEL: zextbool_sub_vector:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmeq v0.4s, v0.4s, v1.4s
+; CHECK-NEXT:    add v0.4s, v2.4s, v0.4s
+; CHECK-NEXT:    ret
+  %c = icmp eq <4 x i32> %c1, %c2
+  %b = zext <4 x i1> %c to <4 x i32>
+  %s = sub <4 x i32> %x, %b
+  ret <4 x i32> %s
+}
+
+define i32 @assertsext_sub_1(i1 signext %cond, i32 %y) {
+; CHECK-LABEL: assertsext_sub_1:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    add w0, w1, w0
+; CHECK-NEXT:    ret
+  %e = zext i1 %cond to i32
+  %r = sub i32 %y, %e
+  ret i32 %r
+}
+
+define i32 @assertsext_add_1(i1 signext %cond, i32 %y) {
+; CHECK-LABEL: assertsext_add_1:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sub w0, w1, w0
+; CHECK-NEXT:    ret
+  %e = zext i1 %cond to i32
+  %r = add i32 %e, %y
+  ret i32 %r
+}
+
+define i32 @assertsext_add_1_commute(i1 signext %cond, i32 %y) {
+; CHECK-LABEL: assertsext_add_1_commute:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sub w0, w1, w0
+; CHECK-NEXT:    ret
+  %e = zext i1 %cond to i32
+  %r = add i32 %y, %e
+  ret i32 %r
+}
+
diff --git a/test/CodeGen/AArch64/build-vector-extract.ll b/test/CodeGen/AArch64/build-vector-extract.ll
new file mode 100644
index 0000000000000000000000000000000000000000..a785533e8db9be3176bfc72aeb259377eefa4cc0
--- /dev/null
+++ b/test/CodeGen/AArch64/build-vector-extract.ll
@@ -0,0 +1,441 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=aarch64-- | FileCheck %s
+
+define <2 x i64> @extract0_i32_zext_insert0_i64_undef(<4 x i32> %x) {
+; CHECK-LABEL: extract0_i32_zext_insert0_i64_undef:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    movi v1.2d, #0000000000000000
+; CHECK-NEXT:    zip1 v0.4s, v0.4s, v1.4s
+; CHECK-NEXT:    ret
+  %e = extractelement <4 x i32> %x, i32 0
+  %z = zext i32 %e to i64
+  %r = insertelement <2 x i64> undef, i64 %z, i32 0
+  ret <2 x i64> %r
+}
+
+define <2 x i64> @extract0_i32_zext_insert0_i64_zero(<4 x i32> %x) {
+; CHECK-LABEL: extract0_i32_zext_insert0_i64_zero:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fmov w8, s0
+; CHECK-NEXT:    movi v0.2d, #0000000000000000
+; CHECK-NEXT:    mov v0.d[0], x8
+; CHECK-NEXT:    ret
+  %e = extractelement <4 x i32> %x, i32 0
+  %z = zext i32 %e to i64
+  %r = insertelement <2 x i64> zeroinitializer, i64 %z, i32 0
+  ret <2 x i64> %r
+}
+
+define <2 x i64> @extract1_i32_zext_insert0_i64_undef(<4 x i32> %x) {
+; CHECK-LABEL: extract1_i32_zext_insert0_i64_undef:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    zip1 v0.4s, v0.4s, v0.4s
+; CHECK-NEXT:    movi v1.2d, #0000000000000000
+; CHECK-NEXT:    ext v0.16b, v0.16b, v1.16b, #12
+; CHECK-NEXT:    ret
+  %e = extractelement <4 x i32> %x, i32 1
+  %z = zext i32 %e to i64
+  %r = insertelement <2 x i64> undef, i64 %z, i32 0
+  ret <2 x i64> %r
+}
+
+define <2 x i64> @extract1_i32_zext_insert0_i64_zero(<4 x i32> %x) {
+; CHECK-LABEL: extract1_i32_zext_insert0_i64_zero:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, v0.s[1]
+; CHECK-NEXT:    movi v0.2d, #0000000000000000
+; CHECK-NEXT:    mov v0.d[0], x8
+; CHECK-NEXT:    ret
+  %e = extractelement <4 x i32> %x, i32 1
+  %z = zext i32 %e to i64
+  %r = insertelement <2 x i64> zeroinitializer, i64 %z, i32 0
+  ret <2 x i64> %r
+}
+
+define <2 x i64> @extract2_i32_zext_insert0_i64_undef(<4 x i32> %x) {
+; CHECK-LABEL: extract2_i32_zext_insert0_i64_undef:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    uzp1 v0.4s, v0.4s, v0.4s
+; CHECK-NEXT:    movi v1.2d, #0000000000000000
+; CHECK-NEXT:    ext v0.16b, v0.16b, v1.16b, #12
+; CHECK-NEXT:    ret
+  %e = extractelement <4 x i32> %x, i32 2
+  %z = zext i32 %e to i64
+  %r = insertelement <2 x i64> undef, i64 %z, i32 0
+  ret <2 x i64> %r
+}
+
+define <2 x i64> @extract2_i32_zext_insert0_i64_zero(<4 x i32> %x) {
+; CHECK-LABEL: extract2_i32_zext_insert0_i64_zero:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, v0.s[2]
+; CHECK-NEXT:    movi v0.2d, #0000000000000000
+; CHECK-NEXT:    mov v0.d[0], x8
+; CHECK-NEXT:    ret
+  %e = extractelement <4 x i32> %x, i32 2
+  %z = zext i32 %e to i64
+  %r = insertelement <2 x i64> zeroinitializer, i64 %z, i32 0
+  ret <2 x i64> %r
+}
+
+define <2 x i64> @extract3_i32_zext_insert0_i64_undef(<4 x i32> %x) {
+; CHECK-LABEL: extract3_i32_zext_insert0_i64_undef:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    movi v1.2d, #0000000000000000
+; CHECK-NEXT:    ext v0.16b, v0.16b, v1.16b, #12
+; CHECK-NEXT:    ret
+  %e = extractelement <4 x i32> %x, i32 3
+  %z = zext i32 %e to i64
+  %r = insertelement <2 x i64> undef, i64 %z, i32 0
+  ret <2 x i64> %r
+}
+
+define <2 x i64> @extract3_i32_zext_insert0_i64_zero(<4 x i32> %x) {
+; CHECK-LABEL: extract3_i32_zext_insert0_i64_zero:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, v0.s[3]
+; CHECK-NEXT:    movi v0.2d, #0000000000000000
+; CHECK-NEXT:    mov v0.d[0], x8
+; CHECK-NEXT:    ret
+  %e = extractelement <4 x i32> %x, i32 3
+  %z = zext i32 %e to i64
+  %r = insertelement <2 x i64> zeroinitializer, i64 %z, i32 0
+  ret <2 x i64> %r
+}
+
+define <2 x i64> @extract0_i32_zext_insert1_i64_undef(<4 x i32> %x) {
+; CHECK-LABEL: extract0_i32_zext_insert1_i64_undef:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    movi v1.2d, #0000000000000000
+; CHECK-NEXT:    zip1 v1.4s, v0.4s, v1.4s
+; CHECK-NEXT:    ext v0.16b, v0.16b, v1.16b, #8
+; CHECK-NEXT:    ret
+  %e = extractelement <4 x i32> %x, i32 0
+  %z = zext i32 %e to i64
+  %r = insertelement <2 x i64> undef, i64 %z, i32 1
+  ret <2 x i64> %r
+}
+
+define <2 x i64> @extract0_i32_zext_insert1_i64_zero(<4 x i32> %x) {
+; CHECK-LABEL: extract0_i32_zext_insert1_i64_zero:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fmov w8, s0
+; CHECK-NEXT:    movi v0.2d, #0000000000000000
+; CHECK-NEXT:    mov v0.d[1], x8
+; CHECK-NEXT:    ret
+  %e = extractelement <4 x i32> %x, i32 0
+  %z = zext i32 %e to i64
+  %r = insertelement <2 x i64> zeroinitializer, i64 %z, i32 1
+  ret <2 x i64> %r
+}
+
+define <2 x i64> @extract1_i32_zext_insert1_i64_undef(<4 x i32> %x) {
+; CHECK-LABEL: extract1_i32_zext_insert1_i64_undef:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-NEXT:    movi v1.2d, #0000000000000000
+; CHECK-NEXT:    ext v0.16b, v0.16b, v1.16b, #4
+; CHECK-NEXT:    ret
+  %e = extractelement <4 x i32> %x, i32 1
+  %z = zext i32 %e to i64
+  %r = insertelement <2 x i64> undef, i64 %z, i32 1
+  ret <2 x i64> %r
+}
+
+define <2 x i64> @extract1_i32_zext_insert1_i64_zero(<4 x i32> %x) {
+; CHECK-LABEL: extract1_i32_zext_insert1_i64_zero:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, v0.s[1]
+; CHECK-NEXT:    movi v0.2d, #0000000000000000
+; CHECK-NEXT:    mov v0.d[1], x8
+; CHECK-NEXT:    ret
+  %e = extractelement <4 x i32> %x, i32 1
+  %z = zext i32 %e to i64
+  %r = insertelement <2 x i64> zeroinitializer, i64 %z, i32 1
+  ret <2 x i64> %r
+}
+
+define <2 x i64> @extract2_i32_zext_insert1_i64_undef(<4 x i32> %x) {
+; CHECK-LABEL: extract2_i32_zext_insert1_i64_undef:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov v0.s[3], wzr
+; CHECK-NEXT:    ret
+  %e = extractelement <4 x i32> %x, i32 2
+  %z = zext i32 %e to i64
+  %r = insertelement <2 x i64> undef, i64 %z, i32 1
+  ret <2 x i64> %r
+}
+
+define <2 x i64> @extract2_i32_zext_insert1_i64_zero(<4 x i32> %x) {
+; CHECK-LABEL: extract2_i32_zext_insert1_i64_zero:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, v0.s[2]
+; CHECK-NEXT:    movi v0.2d, #0000000000000000
+; CHECK-NEXT:    mov v0.d[1], x8
+; CHECK-NEXT:    ret
+  %e = extractelement <4 x i32> %x, i32 2
+  %z = zext i32 %e to i64
+  %r = insertelement <2 x i64> zeroinitializer, i64 %z, i32 1
+  ret <2 x i64> %r
+}
+
+define <2 x i64> @extract3_i32_zext_insert1_i64_undef(<4 x i32> %x) {
+; CHECK-LABEL: extract3_i32_zext_insert1_i64_undef:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    movi v1.2d, #0000000000000000
+; CHECK-NEXT:    ext v0.16b, v0.16b, v1.16b, #4
+; CHECK-NEXT:    ret
+  %e = extractelement <4 x i32> %x, i32 3
+  %z = zext i32 %e to i64
+  %r = insertelement <2 x i64> undef, i64 %z, i32 1
+  ret <2 x i64> %r
+}
+
+define <2 x i64> @extract3_i32_zext_insert1_i64_zero(<4 x i32> %x) {
+; CHECK-LABEL: extract3_i32_zext_insert1_i64_zero:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, v0.s[3]
+; CHECK-NEXT:    movi v0.2d, #0000000000000000
+; CHECK-NEXT:    mov v0.d[1], x8
+; CHECK-NEXT:    ret
+  %e = extractelement <4 x i32> %x, i32 3
+  %z = zext i32 %e to i64
+  %r = insertelement <2 x i64> zeroinitializer, i64 %z, i32 1
+  ret <2 x i64> %r
+}
+
+define <2 x i64> @extract0_i16_zext_insert0_i64_undef(<8 x i16> %x) {
+; CHECK-LABEL: extract0_i16_zext_insert0_i64_undef:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    umov w8, v0.h[0]
+; CHECK-NEXT:    and x8, x8, #0xffff
+; CHECK-NEXT:    fmov d0, x8
+; CHECK-NEXT:    ret
+  %e = extractelement <8 x i16> %x, i32 0
+  %z = zext i16 %e to i64
+  %r = insertelement <2 x i64> undef, i64 %z, i32 0
+  ret <2 x i64> %r
+}
+
+define <2 x i64> @extract0_i16_zext_insert0_i64_zero(<8 x i16> %x) {
+; CHECK-LABEL: extract0_i16_zext_insert0_i64_zero:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    umov w8, v0.h[0]
+; CHECK-NEXT:    and x8, x8, #0xffff
+; CHECK-NEXT:    movi v0.2d, #0000000000000000
+; CHECK-NEXT:    mov v0.d[0], x8
+; CHECK-NEXT:    ret
+  %e = extractelement <8 x i16> %x, i32 0
+  %z = zext i16 %e to i64
+  %r = insertelement <2 x i64> zeroinitializer, i64 %z, i32 0
+  ret <2 x i64> %r
+}
+
+define <2 x i64> @extract1_i16_zext_insert0_i64_undef(<8 x i16> %x) {
+; CHECK-LABEL: extract1_i16_zext_insert0_i64_undef:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    umov w8, v0.h[1]
+; CHECK-NEXT:    and x8, x8, #0xffff
+; CHECK-NEXT:    fmov d0, x8
+; CHECK-NEXT:    ret
+  %e = extractelement <8 x i16> %x, i32 1
+  %z = zext i16 %e to i64
+  %r = insertelement <2 x i64> undef, i64 %z, i32 0
+  ret <2 x i64> %r
+}
+
+define <2 x i64> @extract1_i16_zext_insert0_i64_zero(<8 x i16> %x) {
+; CHECK-LABEL: extract1_i16_zext_insert0_i64_zero:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    umov w8, v0.h[1]
+; CHECK-NEXT:    and x8, x8, #0xffff
+; CHECK-NEXT:    movi v0.2d, #0000000000000000
+; CHECK-NEXT:    mov v0.d[0], x8
+; CHECK-NEXT:    ret
+  %e = extractelement <8 x i16> %x, i32 1
+  %z = zext i16 %e to i64
+  %r = insertelement <2 x i64> zeroinitializer, i64 %z, i32 0
+  ret <2 x i64> %r
+}
+
+define <2 x i64> @extract2_i16_zext_insert0_i64_undef(<8 x i16> %x) {
+; CHECK-LABEL: extract2_i16_zext_insert0_i64_undef:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    umov w8, v0.h[2]
+; CHECK-NEXT:    and x8, x8, #0xffff
+; CHECK-NEXT:    fmov d0, x8
+; CHECK-NEXT:    ret
+  %e = extractelement <8 x i16> %x, i32 2
+  %z = zext i16 %e to i64
+  %r = insertelement <2 x i64> undef, i64 %z, i32 0
+  ret <2 x i64> %r
+}
+
+define <2 x i64> @extract2_i16_zext_insert0_i64_zero(<8 x i16> %x) {
+; CHECK-LABEL: extract2_i16_zext_insert0_i64_zero:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    umov w8, v0.h[2]
+; CHECK-NEXT:    and x8, x8, #0xffff
+; CHECK-NEXT:    movi v0.2d, #0000000000000000
+; CHECK-NEXT:    mov v0.d[0], x8
+; CHECK-NEXT:    ret
+  %e = extractelement <8 x i16> %x, i32 2
+  %z = zext i16 %e to i64
+  %r = insertelement <2 x i64> zeroinitializer, i64 %z, i32 0
+  ret <2 x i64> %r
+}
+
+define <2 x i64> @extract3_i16_zext_insert0_i64_undef(<8 x i16> %x) {
+; CHECK-LABEL: extract3_i16_zext_insert0_i64_undef:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    umov w8, v0.h[3]
+; CHECK-NEXT:    and x8, x8, #0xffff
+; CHECK-NEXT:    fmov d0, x8
+; CHECK-NEXT:    ret
+  %e = extractelement <8 x i16> %x, i32 3
+  %z = zext i16 %e to i64
+  %r = insertelement <2 x i64> undef, i64 %z, i32 0
+  ret <2 x i64> %r
+}
+
+define <2 x i64> @extract3_i16_zext_insert0_i64_zero(<8 x i16> %x) {
+; CHECK-LABEL: extract3_i16_zext_insert0_i64_zero:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    umov w8, v0.h[3]
+; CHECK-NEXT:    and x8, x8, #0xffff
+; CHECK-NEXT:    movi v0.2d, #0000000000000000
+; CHECK-NEXT:    mov v0.d[0], x8
+; CHECK-NEXT:    ret
+  %e = extractelement <8 x i16> %x, i32 3
+  %z = zext i16 %e to i64
+  %r = insertelement <2 x i64> zeroinitializer, i64 %z, i32 0
+  ret <2 x i64> %r
+}
+
+define <2 x i64> @extract0_i16_zext_insert1_i64_undef(<8 x i16> %x) {
+; CHECK-LABEL: extract0_i16_zext_insert1_i64_undef:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    umov w8, v0.h[0]
+; CHECK-NEXT:    and x8, x8, #0xffff
+; CHECK-NEXT:    dup v0.2d, x8
+; CHECK-NEXT:    ret
+  %e = extractelement <8 x i16> %x, i32 0
+  %z = zext i16 %e to i64
+  %r = insertelement <2 x i64> undef, i64 %z, i32 1
+  ret <2 x i64> %r
+}
+
+define <2 x i64> @extract0_i16_zext_insert1_i64_zero(<8 x i16> %x) {
+; CHECK-LABEL: extract0_i16_zext_insert1_i64_zero:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    umov w8, v0.h[0]
+; CHECK-NEXT:    and x8, x8, #0xffff
+; CHECK-NEXT:    movi v0.2d, #0000000000000000
+; CHECK-NEXT:    mov v0.d[1], x8
+; CHECK-NEXT:    ret
+  %e = extractelement <8 x i16> %x, i32 0
+  %z = zext i16 %e to i64
+  %r = insertelement <2 x i64> zeroinitializer, i64 %z, i32 1
+  ret <2 x i64> %r
+}
+
+define <2 x i64> @extract1_i16_zext_insert1_i64_undef(<8 x i16> %x) {
+; CHECK-LABEL: extract1_i16_zext_insert1_i64_undef:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    umov w8, v0.h[1]
+; CHECK-NEXT:    and x8, x8, #0xffff
+; CHECK-NEXT:    dup v0.2d, x8
+; CHECK-NEXT:    ret
+  %e = extractelement <8 x i16> %x, i32 1
+  %z = zext i16 %e to i64
+  %r = insertelement <2 x i64> undef, i64 %z, i32 1
+  ret <2 x i64> %r
+}
+
+define <2 x i64> @extract1_i16_zext_insert1_i64_zero(<8 x i16> %x) {
+; CHECK-LABEL: extract1_i16_zext_insert1_i64_zero:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    umov w8, v0.h[1]
+; CHECK-NEXT:    and x8, x8, #0xffff
+; CHECK-NEXT:    movi v0.2d, #0000000000000000
+; CHECK-NEXT:    mov v0.d[1], x8
+; CHECK-NEXT:    ret
+  %e = extractelement <8 x i16> %x, i32 1
+  %z = zext i16 %e to i64
+  %r = insertelement <2 x i64> zeroinitializer, i64 %z, i32 1
+  ret <2 x i64> %r
+}
+
+define <2 x i64> @extract2_i16_zext_insert1_i64_undef(<8 x i16> %x) {
+; CHECK-LABEL: extract2_i16_zext_insert1_i64_undef:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    umov w8, v0.h[2]
+; CHECK-NEXT:    and x8, x8, #0xffff
+; CHECK-NEXT:    dup v0.2d, x8
+; CHECK-NEXT:    ret
+  %e = extractelement <8 x i16> %x, i32 2
+  %z = zext i16 %e to i64
+  %r = insertelement <2 x i64> undef, i64 %z, i32 1
+  ret <2 x i64> %r
+}
+
+define <2 x i64> @extract2_i16_zext_insert1_i64_zero(<8 x i16> %x) {
+; CHECK-LABEL: extract2_i16_zext_insert1_i64_zero:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    umov w8, v0.h[2]
+; CHECK-NEXT:    and x8, x8, #0xffff
+; CHECK-NEXT:    movi v0.2d, #0000000000000000
+; CHECK-NEXT:    mov v0.d[1], x8
+; CHECK-NEXT:    ret
+  %e = extractelement <8 x i16> %x, i32 2
+  %z = zext i16 %e to i64
+  %r = insertelement <2 x i64> zeroinitializer, i64 %z, i32 1
+  ret <2 x i64> %r
+}
+
+define <2 x i64> @extract3_i16_zext_insert1_i64_undef(<8 x i16> %x) {
+; CHECK-LABEL: extract3_i16_zext_insert1_i64_undef:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    umov w8, v0.h[3]
+; CHECK-NEXT:    and x8, x8, #0xffff
+; CHECK-NEXT:    dup v0.2d, x8
+; CHECK-NEXT:    ret
+  %e = extractelement <8 x i16> %x, i32 3
+  %z = zext i16 %e to i64
+  %r = insertelement <2 x i64> undef, i64 %z, i32 1
+  ret <2 x i64> %r
+}
+
+define <2 x i64> @extract3_i16_zext_insert1_i64_zero(<8 x i16> %x) {
+; CHECK-LABEL: extract3_i16_zext_insert1_i64_zero:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    umov w8, v0.h[3]
+; CHECK-NEXT:    and x8, x8, #0xffff
+; CHECK-NEXT:    movi v0.2d, #0000000000000000
+; CHECK-NEXT:    mov v0.d[1], x8
+; CHECK-NEXT:    ret
+  %e = extractelement <8 x i16> %x, i32 3
+  %z = zext i16 %e to i64
+  %r = insertelement <2 x i64> zeroinitializer, i64 %z, i32 1
+  ret <2 x i64> %r
+}
+
+; This would crash because we did not expect to create
+; a shuffle for a vector where the source operand is
+; not the same size as the result.
+; TODO: Should we handle this pattern? Ie, is moving to/from
+; registers the optimal code?
+
+define <4 x i32> @larger_bv_than_source(<4 x i16> %t0) {
+; CHECK-LABEL: larger_bv_than_source:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    umov w8, v0.h[2]
+; CHECK-NEXT:    fmov s0, w8
+; CHECK-NEXT:    ret
+  %t1 = extractelement <4 x i16> %t0, i32 2
+  %vgetq_lane = zext i16 %t1 to i32
+  %t2 = insertelement <4 x i32> undef, i32 %vgetq_lane, i64 0
+  ret <4 x i32> %t2
+}
+
diff --git a/test/CodeGen/AArch64/cgp-usubo.ll b/test/CodeGen/AArch64/cgp-usubo.ll
new file mode 100644
index 0000000000000000000000000000000000000000..1afa585822761dc1952661affa01cec8ce1a1d45
--- /dev/null
+++ b/test/CodeGen/AArch64/cgp-usubo.ll
@@ -0,0 +1,197 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=aarch64-- | FileCheck %s
+
+; CodeGenPrepare is expected to form overflow intrinsics to improve DAG/isel.
+
+define i1 @usubo_ult_i64(i64 %x, i64 %y, i64* %p) nounwind {
+; CHECK-LABEL: usubo_ult_i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    subs x8, x0, x1
+; CHECK-NEXT:    cset w0, lo
+; CHECK-NEXT:    str x8, [x2]
+; CHECK-NEXT:    ret
+  %s = sub i64 %x, %y
+  store i64 %s, i64* %p
+  %ov = icmp ult i64 %x, %y
+  ret i1 %ov
+}
+
+; Verify insertion point for single-BB. Toggle predicate.
+
+define i1 @usubo_ugt_i32(i32 %x, i32 %y, i32* %p) nounwind {
+; CHECK-LABEL: usubo_ugt_i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmp w1, w0
+; CHECK-NEXT:    cset w8, hi
+; CHECK-NEXT:    sub w9, w0, w1
+; CHECK-NEXT:    mov w0, w8
+; CHECK-NEXT:    str w9, [x2]
+; CHECK-NEXT:    ret
+  %ov = icmp ugt i32 %y, %x
+  %s = sub i32 %x, %y
+  store i32 %s, i32* %p
+  ret i1 %ov
+}
+
+; Constant operand should match.
+
+define i1 @usubo_ugt_constant_op0_i8(i8 %x, i8* %p) nounwind {
+; CHECK-LABEL: usubo_ugt_constant_op0_i8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    and w8, w0, #0xff
+; CHECK-NEXT:    mov w9, #42
+; CHECK-NEXT:    cmp w8, #42 // =42
+; CHECK-NEXT:    sub w9, w9, w0
+; CHECK-NEXT:    cset w0, hi
+; CHECK-NEXT:    strb w9, [x1]
+; CHECK-NEXT:    ret
+  %s = sub i8 42, %x
+  %ov = icmp ugt i8 %x, 42
+  store i8 %s, i8* %p
+  ret i1 %ov
+}
+
+; Compare with constant operand 0 is canonicalized by commuting, but verify match for non-canonical form.
+
+define i1 @usubo_ult_constant_op0_i16(i16 %x, i16* %p) nounwind {
+; CHECK-LABEL: usubo_ult_constant_op0_i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    and w8, w0, #0xffff
+; CHECK-NEXT:    mov w9, #43
+; CHECK-NEXT:    cmp w8, #43 // =43
+; CHECK-NEXT:    sub w9, w9, w0
+; CHECK-NEXT:    cset w0, hi
+; CHECK-NEXT:    strh w9, [x1]
+; CHECK-NEXT:    ret
+  %s = sub i16 43, %x
+  %ov = icmp ult i16 43, %x
+  store i16 %s, i16* %p
+  ret i1 %ov
+}
+
+; Subtract with constant operand 1 is canonicalized to add.
+
+define i1 @usubo_ult_constant_op1_i16(i16 %x, i16* %p) nounwind {
+; CHECK-LABEL: usubo_ult_constant_op1_i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    and w8, w0, #0xffff
+; CHECK-NEXT:    cmp w8, #44 // =44
+; CHECK-NEXT:    sub w9, w0, #44 // =44
+; CHECK-NEXT:    cset w0, lo
+; CHECK-NEXT:    strh w9, [x1]
+; CHECK-NEXT:    ret
+  %s = add i16 %x, -44
+  %ov = icmp ult i16 %x, 44
+  store i16 %s, i16* %p
+  ret i1 %ov
+}
+
+define i1 @usubo_ugt_constant_op1_i8(i8 %x, i8* %p) nounwind {
+; CHECK-LABEL: usubo_ugt_constant_op1_i8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    and w8, w0, #0xff
+; CHECK-NEXT:    cmp w8, #45 // =45
+; CHECK-NEXT:    cset w8, lo
+; CHECK-NEXT:    sub w9, w0, #45 // =45
+; CHECK-NEXT:    mov w0, w8
+; CHECK-NEXT:    strb w9, [x1]
+; CHECK-NEXT:    ret
+  %ov = icmp ugt i8 45, %x
+  %s = add i8 %x, -45
+  store i8 %s, i8* %p
+  ret i1 %ov
+}
+
+; Special-case: subtract 1 changes the compare predicate and constant.
+
+define i1 @usubo_eq_constant1_op1_i32(i32 %x, i32* %p) nounwind {
+; CHECK-LABEL: usubo_eq_constant1_op1_i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    cmp w0, #0 // =0
+; CHECK-NEXT:    sub w8, w0, #1 // =1
+; CHECK-NEXT:    cset w0, eq
+; CHECK-NEXT:    str w8, [x1]
+; CHECK-NEXT:    ret
+  %s = add i32 %x, -1
+  %ov = icmp eq i32 %x, 0
+  store i32 %s, i32* %p
+  ret i1 %ov
+}
+
+; Verify insertion point for multi-BB.
+
+declare void @call(i1)
+
+define i1 @usubo_ult_sub_dominates_i64(i64 %x, i64 %y, i64* %p, i1 %cond) nounwind {
+; CHECK-LABEL: usubo_ult_sub_dominates_i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    tbz w3, #0, .LBB7_2
+; CHECK-NEXT:  // %bb.1: // %t
+; CHECK-NEXT:    subs x8, x0, x1
+; CHECK-NEXT:    cset w0, lo
+; CHECK-NEXT:    str x8, [x2]
+; CHECK-NEXT:    ret
+; CHECK-NEXT:  .LBB7_2: // %f
+; CHECK-NEXT:    and w0, w3, #0x1
+; CHECK-NEXT:    ret
+entry:
+  br i1 %cond, label %t, label %f
+
+t:
+  %s = sub i64 %x, %y
+  store i64 %s, i64* %p
+  br i1 %cond, label %end, label %f
+
+f:
+  ret i1 %cond
+
+end:
+  %ov = icmp ult i64 %x, %y
+  ret i1 %ov
+}
+
+define i1 @usubo_ult_cmp_dominates_i64(i64 %x, i64 %y, i64* %p, i1 %cond) nounwind {
+; CHECK-LABEL: usubo_ult_cmp_dominates_i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    str x22, [sp, #-48]! // 8-byte Folded Spill
+; CHECK-NEXT:    stp x21, x20, [sp, #16] // 16-byte Folded Spill
+; CHECK-NEXT:    mov w20, w3
+; CHECK-NEXT:    stp x19, x30, [sp, #32] // 16-byte Folded Spill
+; CHECK-NEXT:    tbz w3, #0, .LBB8_3
+; CHECK-NEXT:  // %bb.1: // %t
+; CHECK-NEXT:    cmp x0, x1
+; CHECK-NEXT:    mov x22, x0
+; CHECK-NEXT:    cset w0, lo
+; CHECK-NEXT:    mov x19, x2
+; CHECK-NEXT:    mov x21, x1
+; CHECK-NEXT:    bl call
+; CHECK-NEXT:    subs x8, x22, x21
+; CHECK-NEXT:    b.hs .LBB8_3
+; CHECK-NEXT:  // %bb.2: // %end
+; CHECK-NEXT:    cset w0, lo
+; CHECK-NEXT:    str x8, [x19]
+; CHECK-NEXT:    b .LBB8_4
+; CHECK-NEXT:  .LBB8_3: // %f
+; CHECK-NEXT:    and w0, w20, #0x1
+; CHECK-NEXT:  .LBB8_4: // %f
+; CHECK-NEXT:    ldp x19, x30, [sp, #32] // 16-byte Folded Reload
+; CHECK-NEXT:    ldp x21, x20, [sp, #16] // 16-byte Folded Reload
+; CHECK-NEXT:    ldr x22, [sp], #48 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+entry:
+  br i1 %cond, label %t, label %f
+
+t:
+  %ov = icmp ult i64 %x, %y
+  call void @call(i1 %ov)
+  br i1 %ov, label %end, label %f
+
+f:
+  ret i1 %cond
+
+end:
+  %s = sub i64 %x, %y
+  store i64 %s, i64* %p
+  ret i1 %ov
+}
+
diff --git a/test/CodeGen/AArch64/cmpxchg-lse-even-regs.ll b/test/CodeGen/AArch64/cmpxchg-lse-even-regs.ll
new file mode 100644
index 0000000000000000000000000000000000000000..9c6d8cc2053312177bf37070c12e97201d737cb8
--- /dev/null
+++ b/test/CodeGen/AArch64/cmpxchg-lse-even-regs.ll
@@ -0,0 +1,17 @@
+; RUN: llc -mtriple arm64-apple-ios -mattr=+lse %s -o - | FileCheck %s
+
+; Only "even,even+1" pairs are valid for CASP instructions. Make sure LLVM
+; doesn't allocate odd ones and that it can copy them around properly. N.b. we
+; don't actually check that they're sequential because FileCheck can't; odd/even
+; will have to be good enough.
+define void @test_atomic_cmpxchg_i128_register_shuffling(i128* %addr, i128 %desired, i128 %new) nounwind {
+; CHECK-LABEL: test_atomic_cmpxchg_i128_register_shuffling:
+; CHECK-DAG: mov [[DESIRED_LO:x[0-9]*[02468]]], x1
+; CHECK-DAG: mov [[DESIRED_HI:x[0-9]*[13579]]], x2
+; CHECK-DAG: mov [[NEW_LO:x[0-9]*[02468]]], x3
+; CHECK-DAG: mov [[NEW_HI:x[0-9]*[13579]]], x4
+; CHECK: caspal [[DESIRED_LO]], [[DESIRED_HI]], [[NEW_LO]], [[NEW_HI]], [x0]
+
+  %res = cmpxchg i128* %addr, i128 %desired, i128 %new seq_cst seq_cst
+  ret void
+}
diff --git a/test/CodeGen/AArch64/cpus.ll b/test/CodeGen/AArch64/cpus.ll
index d4a32bd278caa513c0c74bc91beb41523a7ffb14..c639ee0beff2e1ac2349de0777afb2976638d889 100644
--- a/test/CodeGen/AArch64/cpus.ll
+++ b/test/CodeGen/AArch64/cpus.ll
@@ -18,6 +18,7 @@
 ; RUN: llc < %s -mtriple=arm64-unknown-unknown -mcpu=kryo 2>&1 | FileCheck %s
 ; RUN: llc < %s -mtriple=arm64-unknown-unknown -mcpu=thunderx2t99 2>&1 | FileCheck %s
 ; RUN: llc < %s -mtriple=arm64-unknown-unknown -mcpu=tsv110 2>&1 | FileCheck %s
+; RUN: llc < %s -mtriple=arm64-unknown-unknown -mcpu=apple-latest 2>&1 | FileCheck %s
 ; RUN: llc < %s -mtriple=arm64-unknown-unknown -mcpu=invalidcpu 2>&1 | FileCheck %s --check-prefix=INVALID
 
 ; CHECK-NOT: {{.*}}  is not a recognized processor for this target
diff --git a/test/CodeGen/AArch64/dp1.ll b/test/CodeGen/AArch64/dp1.ll
index 4247afa11b7608aa09685bc21a5f179fdd13c3ae..3250027c520fd2158316c032600d5a0cadae3c6f 100644
--- a/test/CodeGen/AArch64/dp1.ll
+++ b/test/CodeGen/AArch64/dp1.ll
@@ -1,22 +1,29 @@
 ; RUN: llc -verify-machineinstrs -o - %s -mtriple=aarch64-linux-gnu | FileCheck %s
+; RUN: llc -global-isel -global-isel-abort=2 -pass-remarks-missed=gisel* -verify-machineinstrs -o - %s -mtriple=aarch64-linux-gnu | FileCheck %s --check-prefixes=FALLBACK,GISEL
 
 @var32 = global i32 0
 @var64 = global i64 0
 
+; FALLBACK-NOT: remark{{.*}}rev_i32
 define void @rev_i32() {
 ; CHECK-LABEL: rev_i32:
+; GISEL-LABEL: rev_i32:
     %val0_tmp = load i32, i32* @var32
     %val1_tmp = call i32 @llvm.bswap.i32(i32 %val0_tmp)
 ; CHECK: rev	{{w[0-9]+}}, {{w[0-9]+}}
+; GISEL: rev	{{w[0-9]+}}, {{w[0-9]+}}
     store volatile i32 %val1_tmp, i32* @var32
     ret void
 }
 
+; FALLBACK-NOT: remark{{.*}}rev_i64
 define void @rev_i64() {
 ; CHECK-LABEL: rev_i64:
+; GISEL-LABEL: rev_i64:
     %val0_tmp = load i64, i64* @var64
     %val1_tmp = call i64 @llvm.bswap.i64(i64 %val0_tmp)
 ; CHECK: rev	{{x[0-9]+}}, {{x[0-9]+}}
+; GISEL: rev	{{x[0-9]+}}, {{x[0-9]+}}
     store volatile i64 %val1_tmp, i64* @var64
     ret void
 }
@@ -149,4 +156,3 @@ declare i32  @llvm.cttz.i32 (i32, i1)
 declare i64  @llvm.cttz.i64 (i64, i1)
 declare i32  @llvm.ctpop.i32 (i32)
 declare i64  @llvm.ctpop.i64 (i64)
-
diff --git a/test/CodeGen/AArch64/eh_recoverfp.ll b/test/CodeGen/AArch64/eh_recoverfp.ll
new file mode 100644
index 0000000000000000000000000000000000000000..777bcee543827bfe6e32b4bc8e140d34cad6d4aa
--- /dev/null
+++ b/test/CodeGen/AArch64/eh_recoverfp.ll
@@ -0,0 +1,11 @@
+; RUN: llc -mtriple arm64-windows %s -o - 2>&1 | FileCheck %s
+
+define i8* @foo(i8* %a) {
+; CHECK-LABEL: foo
+; CHECK-NOT: llvm.x86.seh.recoverfp
+  %1 = call i8* @llvm.x86.seh.recoverfp(i8* bitcast (i32 ()* @f to i8*), i8* %a)
+  ret i8* %1
+}
+
+declare i8* @llvm.x86.seh.recoverfp(i8*, i8*)
+declare i32 @f()
diff --git a/test/CodeGen/AArch64/emutls.ll b/test/CodeGen/AArch64/emutls.ll
index c322058a5f617da8ad17915155ada5270b67a62e..25be391bbfaa456cc9ee7fe1e58ed509cd647253 100644
--- a/test/CodeGen/AArch64/emutls.ll
+++ b/test/CodeGen/AArch64/emutls.ll
@@ -1,7 +1,7 @@
 ; RUN: llc -emulated-tls -mtriple=aarch64-linux-android \
-; RUN:     -relocation-model=pic -disable-fp-elim < %s | FileCheck -check-prefix=ARM64 %s
+; RUN:     -relocation-model=pic -frame-pointer=all < %s | FileCheck -check-prefix=ARM64 %s
 ; RUN: llc -mtriple=aarch64-linux-android \
-; RUN:     -relocation-model=pic -disable-fp-elim < %s | FileCheck -check-prefix=ARM64 %s
+; RUN:     -relocation-model=pic -frame-pointer=all < %s | FileCheck -check-prefix=ARM64 %s
 
 ; Copied from X86/emutls.ll
 
diff --git a/test/CodeGen/AArch64/extract-bits.ll b/test/CodeGen/AArch64/extract-bits.ll
index 5dbb71939bb4f780812b98d121e81f7bf23bef6f..b1a203445a7986c3dcc07f3a1f8359a083cb0923 100644
--- a/test/CodeGen/AArch64/extract-bits.ll
+++ b/test/CodeGen/AArch64/extract-bits.ll
@@ -232,6 +232,64 @@ define i64 @bextr64_a4_commutative(i64 %val, i64 %numskipbits, i64 %numlowbits)
   ret i64 %masked
 }
 
+; 64-bit, but with 32-bit output
+
+; Everything done in 64-bit, truncation happens last.
+define i32 @bextr64_32_a0(i64 %val, i64 %numskipbits, i64 %numlowbits) nounwind {
+; CHECK-LABEL: bextr64_32_a0:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    orr w9, wzr, #0x1
+; CHECK-NEXT:    lsl x9, x9, x2
+; CHECK-NEXT:    lsr x8, x0, x1
+; CHECK-NEXT:    sub w9, w9, #1 // =1
+; CHECK-NEXT:    and w0, w9, w8
+; CHECK-NEXT:    ret
+  %shifted = lshr i64 %val, %numskipbits
+  %onebit = shl i64 1, %numlowbits
+  %mask = add nsw i64 %onebit, -1
+  %masked = and i64 %mask, %shifted
+  %res = trunc i64 %masked to i32
+  ret i32 %res
+}
+
+; Shifting happens in 64-bit, then truncation. Masking is 32-bit.
+define i32 @bextr64_32_a1(i64 %val, i64 %numskipbits, i32 %numlowbits) nounwind {
+; CHECK-LABEL: bextr64_32_a1:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    orr w9, wzr, #0x1
+; CHECK-NEXT:    lsl w9, w9, w2
+; CHECK-NEXT:    lsr x8, x0, x1
+; CHECK-NEXT:    sub w9, w9, #1 // =1
+; CHECK-NEXT:    and w0, w9, w8
+; CHECK-NEXT:    ret
+  %shifted = lshr i64 %val, %numskipbits
+  %truncshifted = trunc i64 %shifted to i32
+  %onebit = shl i32 1, %numlowbits
+  %mask = add nsw i32 %onebit, -1
+  %masked = and i32 %mask, %truncshifted
+  ret i32 %masked
+}
+
+; Shifting happens in 64-bit. Mask is 32-bit, but extended to 64-bit.
+; Masking is 64-bit. Then truncation.
+define i32 @bextr64_32_a2(i64 %val, i64 %numskipbits, i32 %numlowbits) nounwind {
+; CHECK-LABEL: bextr64_32_a2:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    orr w9, wzr, #0x1
+; CHECK-NEXT:    lsl w9, w9, w2
+; CHECK-NEXT:    lsr x8, x0, x1
+; CHECK-NEXT:    sub w9, w9, #1 // =1
+; CHECK-NEXT:    and w0, w9, w8
+; CHECK-NEXT:    ret
+  %shifted = lshr i64 %val, %numskipbits
+  %onebit = shl i32 1, %numlowbits
+  %mask = add nsw i32 %onebit, -1
+  %zextmask = zext i32 %mask to i64
+  %masked = and i64 %zextmask, %shifted
+  %truncmasked = trunc i64 %masked to i32
+  ret i32 %truncmasked
+}
+
 ; ---------------------------------------------------------------------------- ;
 ; Pattern b. 32-bit
 ; ---------------------------------------------------------------------------- ;
@@ -408,6 +466,67 @@ define i64 @bextr64_b4_commutative(i64 %val, i64 %numskipbits, i64 %numlowbits)
   ret i64 %masked
 }
 
+; 64-bit, but with 32-bit output
+
+; Everything done in 64-bit, truncation happens last.
+define i32 @bextr64_32_b0(i64 %val, i64 %numskipbits, i8 %numlowbits) nounwind {
+; CHECK-LABEL: bextr64_32_b0:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov x9, #-1
+; CHECK-NEXT:    // kill: def $w2 killed $w2 def $x2
+; CHECK-NEXT:    lsr x8, x0, x1
+; CHECK-NEXT:    lsl x9, x9, x2
+; CHECK-NEXT:    bic w0, w8, w9
+; CHECK-NEXT:    ret
+  %shiftedval = lshr i64 %val, %numskipbits
+  %widenumlowbits = zext i8 %numlowbits to i64
+  %notmask = shl nsw i64 -1, %widenumlowbits
+  %mask = xor i64 %notmask, -1
+  %wideres = and i64 %shiftedval, %mask
+  %res = trunc i64 %wideres to i32
+  ret i32 %res
+}
+
+; Shifting happens in 64-bit, then truncation. Masking is 32-bit.
+define i32 @bextr64_32_b1(i64 %val, i64 %numskipbits, i8 %numlowbits) nounwind {
+; CHECK-LABEL: bextr64_32_b1:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w9, #-1
+; CHECK-NEXT:    // kill: def $w2 killed $w2 def $x2
+; CHECK-NEXT:    lsr x8, x0, x1
+; CHECK-NEXT:    lsl w9, w9, w2
+; CHECK-NEXT:    bic w0, w8, w9
+; CHECK-NEXT:    ret
+  %shiftedval = lshr i64 %val, %numskipbits
+  %truncshiftedval = trunc i64 %shiftedval to i32
+  %widenumlowbits = zext i8 %numlowbits to i32
+  %notmask = shl nsw i32 -1, %widenumlowbits
+  %mask = xor i32 %notmask, -1
+  %res = and i32 %truncshiftedval, %mask
+  ret i32 %res
+}
+
+; Shifting happens in 64-bit. Mask is 32-bit, but extended to 64-bit.
+; Masking is 64-bit. Then truncation.
+define i32 @bextr64_32_b2(i64 %val, i64 %numskipbits, i8 %numlowbits) nounwind {
+; CHECK-LABEL: bextr64_32_b2:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w9, #-1
+; CHECK-NEXT:    // kill: def $w2 killed $w2 def $x2
+; CHECK-NEXT:    lsr x8, x0, x1
+; CHECK-NEXT:    lsl w9, w9, w2
+; CHECK-NEXT:    bic w0, w8, w9
+; CHECK-NEXT:    ret
+  %shiftedval = lshr i64 %val, %numskipbits
+  %widenumlowbits = zext i8 %numlowbits to i32
+  %notmask = shl nsw i32 -1, %widenumlowbits
+  %mask = xor i32 %notmask, -1
+  %zextmask = zext i32 %mask to i64
+  %wideres = and i64 %shiftedval, %zextmask
+  %res = trunc i64 %wideres to i32
+  ret i32 %res
+}
+
 ; ---------------------------------------------------------------------------- ;
 ; Pattern c. 32-bit
 ; ---------------------------------------------------------------------------- ;
@@ -598,6 +717,64 @@ define i64 @bextr64_c4_commutative(i64 %val, i64 %numskipbits, i64 %numlowbits)
   ret i64 %masked
 }
 
+; 64-bit, but with 32-bit output
+
+; Everything done in 64-bit, truncation happens last.
+define i32 @bextr64_32_c0(i64 %val, i64 %numskipbits, i64 %numlowbits) nounwind {
+; CHECK-LABEL: bextr64_32_c0:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    neg x9, x2
+; CHECK-NEXT:    mov x10, #-1
+; CHECK-NEXT:    lsr x8, x0, x1
+; CHECK-NEXT:    lsr x9, x10, x9
+; CHECK-NEXT:    and w0, w9, w8
+; CHECK-NEXT:    ret
+  %shifted = lshr i64 %val, %numskipbits
+  %numhighbits = sub i64 64, %numlowbits
+  %mask = lshr i64 -1, %numhighbits
+  %masked = and i64 %mask, %shifted
+  %res = trunc i64 %masked to i32
+  ret i32 %res
+}
+
+; Shifting happens in 64-bit, then truncation. Masking is 32-bit.
+define i32 @bextr64_32_c1(i64 %val, i64 %numskipbits, i32 %numlowbits) nounwind {
+; CHECK-LABEL: bextr64_32_c1:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    neg w9, w2
+; CHECK-NEXT:    mov w10, #-1
+; CHECK-NEXT:    lsr x8, x0, x1
+; CHECK-NEXT:    lsr w9, w10, w9
+; CHECK-NEXT:    and w0, w9, w8
+; CHECK-NEXT:    ret
+  %shifted = lshr i64 %val, %numskipbits
+  %truncshifted = trunc i64 %shifted to i32
+  %numhighbits = sub i32 32, %numlowbits
+  %mask = lshr i32 -1, %numhighbits
+  %masked = and i32 %mask, %truncshifted
+  ret i32 %masked
+}
+
+; Shifting happens in 64-bit. Mask is 32-bit, but extended to 64-bit.
+; Masking is 64-bit. Then truncation.
+define i32 @bextr64_32_c2(i64 %val, i64 %numskipbits, i32 %numlowbits) nounwind {
+; CHECK-LABEL: bextr64_32_c2:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    neg w9, w2
+; CHECK-NEXT:    mov w10, #-1
+; CHECK-NEXT:    lsr x8, x0, x1
+; CHECK-NEXT:    lsr w9, w10, w9
+; CHECK-NEXT:    and w0, w9, w8
+; CHECK-NEXT:    ret
+  %shifted = lshr i64 %val, %numskipbits
+  %numhighbits = sub i32 32, %numlowbits
+  %mask = lshr i32 -1, %numhighbits
+  %zextmask = zext i32 %mask to i64
+  %masked = and i64 %zextmask, %shifted
+  %truncmasked = trunc i64 %masked to i32
+  ret i32 %truncmasked
+}
+
 ; ---------------------------------------------------------------------------- ;
 ; Pattern d. 32-bit.
 ; ---------------------------------------------------------------------------- ;
@@ -748,12 +925,49 @@ define i64 @bextr64_d3_load_indexzext(i64* %w, i8 %numskipbits, i8 %numlowbits)
   ret i64 %masked
 }
 
+; 64-bit, but with 32-bit output
+
+; Everything done in 64-bit, truncation happens last.
+define i32 @bextr64_32_d0(i64 %val, i64 %numskipbits, i64 %numlowbits) nounwind {
+; CHECK-LABEL: bextr64_32_d0:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    lsr x8, x0, x1
+; CHECK-NEXT:    neg x9, x2
+; CHECK-NEXT:    lsl x8, x8, x9
+; CHECK-NEXT:    lsr x0, x8, x9
+; CHECK-NEXT:    // kill: def $w0 killed $w0 killed $x0
+; CHECK-NEXT:    ret
+  %shifted = lshr i64 %val, %numskipbits
+  %numhighbits = sub i64 64, %numlowbits
+  %highbitscleared = shl i64 %shifted, %numhighbits
+  %masked = lshr i64 %highbitscleared, %numhighbits
+  %res = trunc i64 %masked to i32
+  ret i32 %res
+}
+
+; Shifting happens in 64-bit, then truncation. Masking is 32-bit.
+define i32 @bextr64_32_d1(i64 %val, i64 %numskipbits, i32 %numlowbits) nounwind {
+; CHECK-LABEL: bextr64_32_d1:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    lsr x8, x0, x1
+; CHECK-NEXT:    neg w9, w2
+; CHECK-NEXT:    lsl w8, w8, w9
+; CHECK-NEXT:    lsr w0, w8, w9
+; CHECK-NEXT:    ret
+  %shifted = lshr i64 %val, %numskipbits
+  %truncshifted = trunc i64 %shifted to i32
+  %numhighbits = sub i32 32, %numlowbits
+  %highbitscleared = shl i32 %truncshifted, %numhighbits
+  %masked = lshr i32 %highbitscleared, %numhighbits
+  ret i32 %masked
+}
+
 ; ---------------------------------------------------------------------------- ;
 ; Constant
 ; ---------------------------------------------------------------------------- ;
 
 ; https://bugs.llvm.org/show_bug.cgi?id=38938
-define void @pr38938(i32* %a0, i64* %a1) {
+define void @pr38938(i32* %a0, i64* %a1) nounwind {
 ; CHECK-LABEL: pr38938:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ldr x8, [x1]
@@ -774,7 +988,7 @@ define void @pr38938(i32* %a0, i64* %a1) {
 }
 
 ; The most canonical variant
-define i32 @c0_i32(i32 %arg) {
+define i32 @c0_i32(i32 %arg) nounwind {
 ; CHECK-LABEL: c0_i32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ubfx w0, w0, #19, #10
@@ -785,7 +999,7 @@ define i32 @c0_i32(i32 %arg) {
 }
 
 ; Should be still fine, but the mask is shifted
-define i32 @c1_i32(i32 %arg) {
+define i32 @c1_i32(i32 %arg) nounwind {
 ; CHECK-LABEL: c1_i32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    lsr w8, w0, #19
@@ -797,7 +1011,7 @@ define i32 @c1_i32(i32 %arg) {
 }
 
 ; Should be still fine, but the result is shifted left afterwards
-define i32 @c2_i32(i32 %arg) {
+define i32 @c2_i32(i32 %arg) nounwind {
 ; CHECK-LABEL: c2_i32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ubfx w8, w0, #19, #10
@@ -810,7 +1024,7 @@ define i32 @c2_i32(i32 %arg) {
 }
 
 ; The mask covers newly shifted-in bit
-define i32 @c4_i32_bad(i32 %arg) {
+define i32 @c4_i32_bad(i32 %arg) nounwind {
 ; CHECK-LABEL: c4_i32_bad:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    lsr w8, w0, #19
@@ -824,7 +1038,7 @@ define i32 @c4_i32_bad(i32 %arg) {
 ; i64
 
 ; The most canonical variant
-define i64 @c0_i64(i64 %arg) {
+define i64 @c0_i64(i64 %arg) nounwind {
 ; CHECK-LABEL: c0_i64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ubfx x0, x0, #51, #10
@@ -835,7 +1049,7 @@ define i64 @c0_i64(i64 %arg) {
 }
 
 ; Should be still fine, but the mask is shifted
-define i64 @c1_i64(i64 %arg) {
+define i64 @c1_i64(i64 %arg) nounwind {
 ; CHECK-LABEL: c1_i64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    lsr x8, x0, #51
@@ -847,7 +1061,7 @@ define i64 @c1_i64(i64 %arg) {
 }
 
 ; Should be still fine, but the result is shifted left afterwards
-define i64 @c2_i64(i64 %arg) {
+define i64 @c2_i64(i64 %arg) nounwind {
 ; CHECK-LABEL: c2_i64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ubfx x8, x0, #51, #10
@@ -860,7 +1074,7 @@ define i64 @c2_i64(i64 %arg) {
 }
 
 ; The mask covers newly shifted-in bit
-define i64 @c4_i64_bad(i64 %arg) {
+define i64 @c4_i64_bad(i64 %arg) nounwind {
 ; CHECK-LABEL: c4_i64_bad:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    lsr x8, x0, #51
@@ -878,7 +1092,7 @@ define i64 @c4_i64_bad(i64 %arg) {
 ; i32
 
 ; The most canonical variant
-define void @c5_i32(i32 %arg, i32* %ptr) {
+define void @c5_i32(i32 %arg, i32* %ptr) nounwind {
 ; CHECK-LABEL: c5_i32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ubfx w8, w0, #19, #10
@@ -891,7 +1105,7 @@ define void @c5_i32(i32 %arg, i32* %ptr) {
 }
 
 ; Should be still fine, but the mask is shifted
-define void @c6_i32(i32 %arg, i32* %ptr) {
+define void @c6_i32(i32 %arg, i32* %ptr) nounwind {
 ; CHECK-LABEL: c6_i32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ubfx w8, w0, #19, #12
@@ -904,7 +1118,7 @@ define void @c6_i32(i32 %arg, i32* %ptr) {
 }
 
 ; Should be still fine, but the result is shifted left afterwards
-define void @c7_i32(i32 %arg, i32* %ptr) {
+define void @c7_i32(i32 %arg, i32* %ptr) nounwind {
 ; CHECK-LABEL: c7_i32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ubfx w8, w0, #19, #10
@@ -921,7 +1135,7 @@ define void @c7_i32(i32 %arg, i32* %ptr) {
 ; i64
 
 ; The most canonical variant
-define void @c5_i64(i64 %arg, i64* %ptr) {
+define void @c5_i64(i64 %arg, i64* %ptr) nounwind {
 ; CHECK-LABEL: c5_i64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ubfx x8, x0, #51, #10
@@ -934,7 +1148,7 @@ define void @c5_i64(i64 %arg, i64* %ptr) {
 }
 
 ; Should be still fine, but the mask is shifted
-define void @c6_i64(i64 %arg, i64* %ptr) {
+define void @c6_i64(i64 %arg, i64* %ptr) nounwind {
 ; CHECK-LABEL: c6_i64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ubfx x8, x0, #51, #12
@@ -947,7 +1161,7 @@ define void @c6_i64(i64 %arg, i64* %ptr) {
 }
 
 ; Should be still fine, but the result is shifted left afterwards
-define void @c7_i64(i64 %arg, i64* %ptr) {
+define void @c7_i64(i64 %arg, i64* %ptr) nounwind {
 ; CHECK-LABEL: c7_i64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ubfx x8, x0, #51, #10
diff --git a/test/CodeGen/AArch64/f16-instructions.ll b/test/CodeGen/AArch64/f16-instructions.ll
index 352a27539032bffa303a45807cf3131e9f00f9a6..173f296d3be0590da0539d47a5d469a0c26c0444 100644
--- a/test/CodeGen/AArch64/f16-instructions.ll
+++ b/test/CodeGen/AArch64/f16-instructions.ll
@@ -1,5 +1,16 @@
-; RUN: llc < %s -mtriple aarch64-unknown-unknown -aarch64-neon-syntax=apple -asm-verbose=false -disable-post-ra -disable-fp-elim | FileCheck %s --check-prefix=CHECK-CVT --check-prefix=CHECK-COMMON
-; RUN: llc < %s -mtriple aarch64-unknown-unknown -mattr=+fullfp16 -aarch64-neon-syntax=apple -asm-verbose=false -disable-post-ra -disable-fp-elim | FileCheck %s --check-prefix=CHECK-COMMON --check-prefix=CHECK-FP16
+; RUN: llc < %s -mtriple aarch64-unknown-unknown -aarch64-neon-syntax=apple -asm-verbose=false -disable-post-ra -frame-pointer=all | FileCheck %s --check-prefix=CHECK-CVT --check-prefix=CHECK-COMMON
+; RUN: llc < %s -mtriple aarch64-unknown-unknown -mattr=+fullfp16 -aarch64-neon-syntax=apple -asm-verbose=false -disable-post-ra -frame-pointer=all | FileCheck %s --check-prefix=CHECK-COMMON --check-prefix=CHECK-FP16
+
+; RUN: llc < %s -mtriple aarch64-unknown-unknown -aarch64-neon-syntax=apple \
+; RUN: -asm-verbose=false -disable-post-ra -frame-pointer=all -global-isel \
+; RUN: -global-isel-abort=2 -pass-remarks-missed=gisel-* 2>&1 | FileCheck %s \
+; RUN: --check-prefixes=FALLBACK,GISEL-CVT,GISEL
+
+; RUN: llc < %s -mtriple aarch64-unknown-unknown -mattr=+fullfp16 \
+; RUN: -aarch64-neon-syntax=apple -asm-verbose=false -disable-post-ra \
+; RUN: -frame-pointer=all -global-isel -global-isel-abort=2 \
+; RUN: -pass-remarks-missed=gisel-* 2>&1 | FileCheck %s \
+; RUN: --check-prefixes=FALLBACK-FP16,GISEL-FP16,GISEL
 
 target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
 
@@ -770,6 +781,9 @@ declare half @llvm.aarch64.neon.frecpe.f16(half %a) #0
 declare half @llvm.aarch64.neon.frecpx.f16(half %a) #0
 declare half @llvm.aarch64.neon.frsqrte.f16(half %a) #0
 
+; FALLBACK-NOT: remark:{{.*}}test_sqrt
+; FALLBACK-FP16-NOT: remark:{{.*}}test_sqrt
+
 ; CHECK-CVT-LABEL: test_sqrt:
 ; CHECK-CVT-NEXT: fcvt s0, h0
 ; CHECK-CVT-NEXT: fsqrt s0, s0
@@ -780,6 +794,16 @@ declare half @llvm.aarch64.neon.frsqrte.f16(half %a) #0
 ; CHECK-FP16-NEXT: fsqrt h0, h0
 ; CHECK-FP16-NEXT: ret
 
+; GISEL-CVT-LABEL: test_sqrt:
+; GISEL-CVT-NEXT: fcvt s0, h0
+; GISEL-CVT-NEXT: fsqrt s0, s0
+; GISEL-CVT-NEXT: fcvt h0, s0
+; GISEL-CVT-NEXT: ret
+
+; GISEL-FP16-LABEL: test_sqrt:
+; GISEL-FP16-NEXT: fsqrt h0, h0
+; GISEL-FP16-NEXT: ret
+
 define half @test_sqrt(half %a) #0 {
   %r = call half @llvm.sqrt.f16(half %a)
   ret half %r
@@ -798,6 +822,9 @@ define half @test_powi(half %a, i32 %b) #0 {
   ret half %r
 }
 
+; FALLBACK-NOT: remark:{{.*}}test_sin
+; FALLBACK-FP16-NOT: remark:{{.*}}test_sin
+
 ; CHECK-COMMON-LABEL: test_sin:
 ; CHECK-COMMON-NEXT: stp x29, x30, [sp, #-16]!
 ; CHECK-COMMON-NEXT: mov  x29, sp
@@ -806,11 +833,23 @@ define half @test_powi(half %a, i32 %b) #0 {
 ; CHECK-COMMON-NEXT: fcvt h0, s0
 ; CHECK-COMMON-NEXT: ldp x29, x30, [sp], #16
 ; CHECK-COMMON-NEXT: ret
+
+; GISEL-LABEL: test_sin:
+; GISEL-NEXT: stp x29, x30, [sp, #-16]!
+; GISEL-NEXT: mov  x29, sp
+; GISEL-NEXT: fcvt s0, h0
+; GISEL-NEXT: bl {{_?}}sinf
+; GISEL-NEXT: fcvt h0, s0
+; GISEL-NEXT: ldp x29, x30, [sp], #16
+; GISEL-NEXT: ret
 define half @test_sin(half %a) #0 {
   %r = call half @llvm.sin.f16(half %a)
   ret half %r
 }
 
+; FALLBACK-NOT: remark:{{.*}}test_cos
+; FALLBACK-FP16-NOT: remark:{{.*}}test_cos
+
 ; CHECK-COMMON-LABEL: test_cos:
 ; CHECK-COMMON-NEXT: stp x29, x30, [sp, #-16]!
 ; CHECK-COMMON-NEXT: mov  x29, sp
@@ -819,6 +858,15 @@ define half @test_sin(half %a) #0 {
 ; CHECK-COMMON-NEXT: fcvt h0, s0
 ; CHECK-COMMON-NEXT: ldp x29, x30, [sp], #16
 ; CHECK-COMMON-NEXT: ret
+
+; GISEL-LABEL: test_cos:
+; GISEL-NEXT: stp x29, x30, [sp, #-16]!
+; GISEL-NEXT: mov  x29, sp
+; GISEL-NEXT: fcvt s0, h0
+; GISEL-NEXT: bl {{_?}}cosf
+; GISEL-NEXT: fcvt h0, s0
+; GISEL-NEXT: ldp x29, x30, [sp], #16
+; GISEL-NEXT: ret
 define half @test_cos(half %a) #0 {
   %r = call half @llvm.cos.f16(half %a)
   ret half %r
@@ -838,6 +886,9 @@ define half @test_pow(half %a, half %b) #0 {
   ret half %r
 }
 
+; FALLBACK-NOT: remark:{{.*}}test_exp
+; FALLBACK-FP16-NOT: remark:{{.*}}test_exp
+
 ; CHECK-COMMON-LABEL: test_exp:
 ; CHECK-COMMON-NEXT: stp x29, x30, [sp, #-16]!
 ; CHECK-COMMON-NEXT: mov  x29, sp
@@ -846,6 +897,15 @@ define half @test_pow(half %a, half %b) #0 {
 ; CHECK-COMMON-NEXT: fcvt h0, s0
 ; CHECK-COMMON-NEXT: ldp x29, x30, [sp], #16
 ; CHECK-COMMON-NEXT: ret
+
+; GISEL-LABEL: test_exp:
+; GISEL-NEXT: stp x29, x30, [sp, #-16]!
+; GISEL-NEXT: mov  x29, sp
+; GISEL-NEXT: fcvt s0, h0
+; GISEL-NEXT: bl {{_?}}expf
+; GISEL-NEXT: fcvt h0, s0
+; GISEL-NEXT: ldp x29, x30, [sp], #16
+; GISEL-NEXT: ret
 define half @test_exp(half %a) #0 {
   %r = call half @llvm.exp.f16(half %a)
   ret half %r
@@ -864,6 +924,9 @@ define half @test_exp2(half %a) #0 {
   ret half %r
 }
 
+; FALLBACK-NOT: remark:{{.*}}test_log
+; FALLBACK-FP16-NOT: remark:{{.*}}test_log
+
 ; CHECK-COMMON-LABEL: test_log:
 ; CHECK-COMMON-NEXT: stp x29, x30, [sp, #-16]!
 ; CHECK-COMMON-NEXT: mov  x29, sp
@@ -872,11 +935,24 @@ define half @test_exp2(half %a) #0 {
 ; CHECK-COMMON-NEXT: fcvt h0, s0
 ; CHECK-COMMON-NEXT: ldp x29, x30, [sp], #16
 ; CHECK-COMMON-NEXT: ret
+
+; GISEL-LABEL: test_log:
+; GISEL: stp x29, x30, [sp, #-16]!
+; GISEL-NEXT: mov  x29, sp
+; GISEL-NEXT: fcvt s0, h0
+; GISEL-NEXT: bl {{_?}}logf
+; GISEL-NEXT: fcvt h0, s0
+; GISEL-NEXT: ldp x29, x30, [sp], #16
+; GISEL-NEXT: ret
+
 define half @test_log(half %a) #0 {
   %r = call half @llvm.log.f16(half %a)
   ret half %r
 }
 
+; FALLBACK-NOT: remark:{{.*}}test_log10
+; FALLBACK-FP16-NOT: remark:{{.*}}test_log10
+
 ; CHECK-COMMON-LABEL: test_log10:
 ; CHECK-COMMON-NEXT: stp x29, x30, [sp, #-16]!
 ; CHECK-COMMON-NEXT: mov  x29, sp
@@ -885,11 +961,24 @@ define half @test_log(half %a) #0 {
 ; CHECK-COMMON-NEXT: fcvt h0, s0
 ; CHECK-COMMON-NEXT: ldp x29, x30, [sp], #16
 ; CHECK-COMMON-NEXT: ret
+
+; GISEL-LABEL: test_log10:
+; GISEL-NEXT: stp x29, x30, [sp, #-16]!
+; GISEL-NEXT: mov  x29, sp
+; GISEL-NEXT: fcvt s0, h0
+; GISEL-NEXT: bl {{_?}}log10f
+; GISEL-NEXT: fcvt h0, s0
+; GISEL-NEXT: ldp x29, x30, [sp], #16
+; GISEL-NEXT: ret
+
 define half @test_log10(half %a) #0 {
   %r = call half @llvm.log10.f16(half %a)
   ret half %r
 }
 
+; FALLBACK-NOT: remark:{{.*}}test_log2
+; FALLBACK-FP16-NOT: remark:{{.*}}test_log2
+
 ; CHECK-COMMON-LABEL: test_log2:
 ; CHECK-COMMON-NEXT: stp x29, x30, [sp, #-16]!
 ; CHECK-COMMON-NEXT: mov  x29, sp
@@ -898,6 +987,16 @@ define half @test_log10(half %a) #0 {
 ; CHECK-COMMON-NEXT: fcvt h0, s0
 ; CHECK-COMMON-NEXT: ldp x29, x30, [sp], #16
 ; CHECK-COMMON-NEXT: ret
+
+; GISEL-LABEL: test_log2:
+; GISEL-NEXT: stp x29, x30, [sp, #-16]!
+; GISEL-NEXT: mov  x29, sp
+; GISEL-NEXT: fcvt s0, h0
+; GISEL-NEXT: bl {{_?}}log2f
+; GISEL-NEXT: fcvt h0, s0
+; GISEL-NEXT: ldp x29, x30, [sp], #16
+; GISEL-NEXT: ret
+
 define half @test_log2(half %a) #0 {
   %r = call half @llvm.log2.f16(half %a)
   ret half %r
@@ -930,6 +1029,19 @@ define half @test_fma(half %a, half %b, half %c) #0 {
 ; CHECK-FP16-NEXT: fabs h0, h0
 ; CHECK-FP16-NEXT: ret
 
+; FALLBACK-NOT: remark:{{.*}}test_fabs
+; FALLBACK-FP16-NOT: remark:{{.*}}test_fabs
+
+; GISEL-CVT-LABEL: test_fabs:
+; GISEL-CVT-NEXT: fcvt s0, h0
+; GISEL-CVT-NEXT: fabs s0, s0
+; GISEL-CVT-NEXT: fcvt h0, s0
+; GISEL-CVT-NEXT: ret
+
+; GISEL-FP16-LABEL: test_fabs:
+; GISEL-FP16-NEXT: fabs h0, h0
+; GISEL-FP16-NEXT: ret
+
 define half @test_fabs(half %a) #0 {
   %r = call half @llvm.fabs.f16(half %a)
   ret half %r
@@ -1056,6 +1168,19 @@ define float @test_copysign_extended(half %a, half %b) #0 {
 ; CHECK-FP16-NEXT: frintm h0, h0
 ; CHECK-FP16-NEXT: ret
 
+; FALLBACK-NOT: remark:{{.*}}test_floor
+; FALLBACK-FP16-NOT: remark:{{.*}}test_floor
+
+; GISEL-CVT-LABEL: test_floor:
+; GISEL-CVT-NEXT: fcvt [[FLOAT32:s[0-9]+]], h0
+; GISEL-CVT-NEXT: frintm [[INT32:s[0-9]+]], [[FLOAT32]]
+; GISEL-CVT-NEXT: fcvt h0, [[INT32]]
+; GISEL-CVT-NEXT: ret
+
+; GISEL-FP16-LABEL: test_floor:
+; GISEL-FP16-NEXT: frintm h0, h0
+; GISEL-FP16-NEXT: ret
+
 define half @test_floor(half %a) #0 {
   %r = call half @llvm.floor.f16(half %a)
   ret half %r
@@ -1071,6 +1196,18 @@ define half @test_floor(half %a) #0 {
 ; CHECK-FP16-NEXT: frintp h0, h0
 ; CHECK-FP16-NEXT: ret
 
+; FALLBACK-NOT: remark:{{.*}}test_ceil
+; FALLBACK-FP16-NOT: remark:{{.*}}test_ceil
+
+; GISEL-CVT-LABEL: test_ceil:
+; GISEL-CVT-NEXT: fcvt [[FLOAT32:s[0-9]+]], h0
+; GISEL-CVT-NEXT: frintp [[INT32:s[0-9]+]], [[FLOAT32]]
+; GISEL-CVT-NEXT: fcvt h0, [[INT32]]
+; GISEL-CVT-NEXT: ret
+
+; GISEL-FP16-LABEL: test_ceil:
+; GISEL-FP16-NEXT: frintp h0, h0
+; GISEL-FP16-NEXT: ret
 define half @test_ceil(half %a) #0 {
   %r = call half @llvm.ceil.f16(half %a)
   ret half %r
diff --git a/test/CodeGen/AArch64/fabs.ll b/test/CodeGen/AArch64/fabs.ll
index cd315ab3631ddb74cf7d90cb18d9c37f00d9d4cd..58f047afa77736b0bc9569188dc626ec7763dea4 100644
--- a/test/CodeGen/AArch64/fabs.ll
+++ b/test/CodeGen/AArch64/fabs.ll
@@ -22,11 +22,11 @@ define double @not_fabs(double %x) #0 {
 define float @still_not_fabs(float %x) #0 {
 ; CHECK-LABEL: still_not_fabs:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    adrp x8, .LCPI1_0
-; CHECK-NEXT:    ldr s1, [x8, :lo12:.LCPI1_0]
-; CHECK-NEXT:    fneg s2, s0
-; CHECK-NEXT:    fcmp s0, s1
-; CHECK-NEXT:    fcsel s0, s0, s2, ge
+; CHECK-NEXT:    orr w8, wzr, #0x80000000
+; CHECK-NEXT:    fmov s2, w8
+; CHECK-NEXT:    fneg s1, s0
+; CHECK-NEXT:    fcmp s0, s2
+; CHECK-NEXT:    fcsel s0, s0, s1, ge
 ; CHECK-NEXT:    ret
   %cmp = fcmp nnan oge float %x, -0.0
   %sub = fsub nnan float -0.0, %x
diff --git a/test/CodeGen/AArch64/fadd-combines.ll b/test/CodeGen/AArch64/fadd-combines.ll
index 7332101a481e610c037cf34bc77cfd237cd4900d..a204684db0f6aa0f20aaa704caaf13eb2dfb8f8b 100644
--- a/test/CodeGen/AArch64/fadd-combines.ll
+++ b/test/CodeGen/AArch64/fadd-combines.ll
@@ -132,13 +132,13 @@ define double @test7(double %a, double %b) nounwind {
 define float @fadd_const_multiuse_fmf(float %x) {
 ; CHECK-LABEL: fadd_const_multiuse_fmf:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    adrp x8, .LCPI10_0
-; CHECK-NEXT:    adrp x9, .LCPI10_1
-; CHECK-NEXT:    ldr s1, [x8, :lo12:.LCPI10_0]
-; CHECK-NEXT:    ldr s2, [x9, :lo12:.LCPI10_1]
-; CHECK-NEXT:    fadd s1, s0, s1
-; CHECK-NEXT:    fadd s0, s0, s2
-; CHECK-NEXT:    fadd s0, s1, s0
+; CHECK-DAG:     mov  [[W59:w[0-9]+]], #1114374144
+; CHECK-DAG:     mov  [[W42:w[0-9]+]], #1109917696
+; CHECK-DAG:     fmov [[FP59:s[0-9]+]], [[W59]]
+; CHECK-DAG:     fmov [[FP42:s[0-9]+]], [[W42]]
+; CHECK-NEXT:    fadd [[TMP1:s[0-9]+]], s0, [[FP42]]
+; CHECK-NEXT:    fadd [[TMP2:s[0-9]+]], s0, [[FP59]]
+; CHECK-NEXT:    fadd s0, [[TMP1]], [[TMP2]]
 ; CHECK-NEXT:    ret
   %a1 = fadd float %x, 42.0
   %a2 = fadd nsz reassoc float %a1, 17.0
@@ -153,13 +153,13 @@ define float @fadd_const_multiuse_fmf(float %x) {
 define float @fadd_const_multiuse_attr(float %x) #0 {
 ; CHECK-LABEL: fadd_const_multiuse_attr:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    adrp x9, .LCPI11_1
-; CHECK-NEXT:    adrp x8, .LCPI11_0
-; CHECK-NEXT:    ldr s1, [x9, :lo12:.LCPI11_1]
-; CHECK-NEXT:    ldr s2, [x8, :lo12:.LCPI11_0]
-; CHECK-NEXT:    fadd s1, s0, s1
-; CHECK-NEXT:    fadd s1, s2, s1
-; CHECK-NEXT:    fadd s0, s0, s1
+; CHECK-DAG:     mov  [[W59:w[0-9]+]], #1114374144
+; CHECK-DAG:     mov  [[W17:w[0-9]+]], #1109917696
+; CHECK-NEXT:    fmov [[FP59:s[0-9]+]], [[W59]]
+; CHECK-NEXT:    fmov [[FP17:s[0-9]+]], [[W17]]
+; CHECK-NEXT:    fadd [[TMP1:s[0-9]+]], s0, [[FP59]]
+; CHECK-NEXT:    fadd [[TMP2:s[0-9]+]], [[FP17]], [[TMP1]]
+; CHECK-NEXT:    fadd s0, s0, [[TMP2]]
 ; CHECK-NEXT:    ret
   %a1 = fadd float %x, 42.0
   %a2 = fadd float %a1, 17.0
diff --git a/test/CodeGen/AArch64/fastcc.ll b/test/CodeGen/AArch64/fastcc.ll
index 3ea6df5be49c5ac87d922cd959714c27675389e8..d4e116134cd142f9ab15436e55ee5d3f5801ec62 100644
--- a/test/CodeGen/AArch64/fastcc.ll
+++ b/test/CodeGen/AArch64/fastcc.ll
@@ -1,6 +1,6 @@
-; RUN: llc -verify-machineinstrs < %s -mtriple=aarch64-none-linux-gnu -disable-fp-elim -tailcallopt | FileCheck %s -check-prefix CHECK-TAIL
-; RUN: llc -verify-machineinstrs < %s -mtriple=aarch64-none-linux-gnu -disable-fp-elim | FileCheck %s
-; RUN: llc -verify-machineinstrs < %s -mtriple=aarch64-none-linux-gnu -disable-fp-elim -tailcallopt -aarch64-redzone | FileCheck %s -check-prefix CHECK-TAIL-RZ
+; RUN: llc -verify-machineinstrs < %s -mtriple=aarch64-none-linux-gnu -frame-pointer=all -tailcallopt | FileCheck %s -check-prefix CHECK-TAIL
+; RUN: llc -verify-machineinstrs < %s -mtriple=aarch64-none-linux-gnu -frame-pointer=all | FileCheck %s
+; RUN: llc -verify-machineinstrs < %s -mtriple=aarch64-none-linux-gnu -frame-pointer=all -tailcallopt -aarch64-redzone | FileCheck %s -check-prefix CHECK-TAIL-RZ
 
 ; Without tailcallopt fastcc still means the caller cleans up the
 ; stack, so try to make sure this is respected.
diff --git a/test/CodeGen/AArch64/fpimm.ll b/test/CodeGen/AArch64/fpimm.ll
index de15bc0e2fa874f11f32129a566b236c8d9fca75..a09d5d05de7fe1b336de73efa6644bfb0689400a 100644
--- a/test/CodeGen/AArch64/fpimm.ll
+++ b/test/CodeGen/AArch64/fpimm.ll
@@ -18,8 +18,10 @@ define void @check_float() {
 
   %newval2 = fadd float %val, 128.0
   store volatile float %newval2, float* @varf32
-; CHECK-DAG: ldr {{s[0-9]+}}, [{{x[0-9]+}}, {{#?}}:lo12:.LCPI0_0
-; TINY-DAG: ldr {{s[0-9]+}}, .LCPI0_0
+; CHECK-DAG: mov [[W128:w[0-9]+]], #1124073472
+; CHECK-DAG: fmov {{s[0-9]+}}, [[W128]]
+; TINY-DAG: mov [[W128:w[0-9]+]], #1124073472
+; TINY-DAG: fmov {{s[0-9]+}}, [[W128]]
 
 ; CHECK: ret
 ; TINY: ret
@@ -38,8 +40,10 @@ define void @check_double() {
 
   %newval2 = fadd double %val, 128.0
   store volatile double %newval2, double* @varf64
-; CHECK-DAG: ldr {{d[0-9]+}}, [{{x[0-9]+}}, {{#?}}:lo12:.LCPI1_0
-; TINY-DAG: ldr {{d[0-9]+}}, .LCPI1_0
+; CHECK-DAG: mov [[X128:x[0-9]+]], #4638707616191610880
+; CHECK-DAG: fmov {{d[0-9]+}}, [[X128]]
+; TINY-DAG: mov [[X128:x[0-9]+]], #4638707616191610880
+; TINY-DAG: fmov {{d[0-9]+}}, [[X128]]
 
 ; CHECK: ret
 ; TINY: ret
diff --git a/test/CodeGen/AArch64/hwasan-check-memaccess.ll b/test/CodeGen/AArch64/hwasan-check-memaccess.ll
new file mode 100644
index 0000000000000000000000000000000000000000..3d5340b4aa06e07a2683793c7fb5a1056ba4d12f
--- /dev/null
+++ b/test/CodeGen/AArch64/hwasan-check-memaccess.ll
@@ -0,0 +1,63 @@
+; RUN: llc < %s | FileCheck %s
+
+target triple = "aarch64--linux-android"
+
+define i8* @f1(i8* %x0, i8* %x1) {
+  ; CHECK: f1:
+  ; CHECK: str x30, [sp, #-16]!
+  ; CHECK-NEXT: .cfi_def_cfa_offset 16
+  ; CHECK-NEXT: .cfi_offset w30, -16
+  ; CHECK-NEXT: mov x9, x0
+  ; CHECK-NEXT: bl __hwasan_check_x1_123
+  ; CHECK-NEXT: mov x0, x1
+  ; CHECK-NEXT: ldr x30, [sp], #16
+  ; CHECK-NEXT: ret
+  call void @llvm.hwasan.check.memaccess(i8* %x0, i8* %x1, i32 123)
+  ret i8* %x1
+}
+
+define i8* @f2(i8* %x0, i8* %x1) {
+  ; CHECK: f2:
+  ; CHECK: str x30, [sp, #-16]!
+  ; CHECK-NEXT: .cfi_def_cfa_offset 16
+  ; CHECK-NEXT: .cfi_offset w30, -16
+  ; CHECK-NEXT: mov x9, x1
+  ; CHECK-NEXT: bl __hwasan_check_x0_456
+  ; CHECK-NEXT: ldr x30, [sp], #16
+  ; CHECK-NEXT: ret
+  call void @llvm.hwasan.check.memaccess(i8* %x1, i8* %x0, i32 456)
+  ret i8* %x0
+}
+
+declare void @llvm.hwasan.check.memaccess(i8*, i8*, i32)
+
+; CHECK:      .section .text.hot,"axG",@progbits,__hwasan_check_x0_456,comdat
+; CHECK-NEXT: .type __hwasan_check_x0_456,@function
+; CHECK-NEXT: .weak __hwasan_check_x0_456
+; CHECK-NEXT: .hidden __hwasan_check_x0_456
+; CHECK-NEXT: __hwasan_check_x0_456:
+; CHECK-NEXT: ubfx x16, x0, #4, #52
+; CHECK-NEXT: ldrb w16, [x9, x16]
+; CHECK-NEXT: lsr x17, x0, #56
+; CHECK-NEXT: cmp w16, w17
+; CHECK-NEXT: b.ne .Ltmp0
+; CHECK-NEXT: ret
+; CHECK-NEXT: .Ltmp0:
+; CHECK-NEXT: mov x1, #456
+; CHECK-NEXT: b __hwasan_tag_mismatch
+
+; CHECK:      .section .text.hot,"axG",@progbits,__hwasan_check_x1_123,comdat
+; CHECK-NEXT: .type __hwasan_check_x1_123,@function
+; CHECK-NEXT: .weak __hwasan_check_x1_123
+; CHECK-NEXT: .hidden __hwasan_check_x1_123
+; CHECK-NEXT: __hwasan_check_x1_123:
+; CHECK-NEXT: ubfx x16, x1, #4, #52
+; CHECK-NEXT: ldrb w16, [x9, x16]
+; CHECK-NEXT: lsr x17, x1, #56
+; CHECK-NEXT: cmp w16, w17
+; CHECK-NEXT: b.ne .Ltmp1
+; CHECK-NEXT: ret
+; CHECK-NEXT: .Ltmp1:
+; CHECK-NEXT: mov x0, x1
+; CHECK-NEXT: mov x1, #123
+; CHECK-NEXT: b __hwasan_tag_mismatch
diff --git a/test/CodeGen/AArch64/isinf.ll b/test/CodeGen/AArch64/isinf.ll
new file mode 100644
index 0000000000000000000000000000000000000000..e4607d08b66662b55156fc172ea2f230b1bd0356
--- /dev/null
+++ b/test/CodeGen/AArch64/isinf.ll
@@ -0,0 +1,62 @@
+; RUN: llc -mtriple=aarch64-none-linux-gnu -mattr=+neon,+fullfp16 < %s -o -| FileCheck %s
+
+declare half   @llvm.fabs.f16(half)
+declare float  @llvm.fabs.f32(float)
+declare double @llvm.fabs.f64(double)
+declare fp128  @llvm.fabs.f128(fp128)
+
+; INFINITY requires loading the constant for _Float16
+define i32 @replace_isinf_call_f16(half %x) {
+; CHECK-LABEL: replace_isinf_call_f16:
+; CHECK:       adrp    [[ADDR:x[0-9]+]], [[CSTLABEL:.LCP.*]]
+; CHECK:       ldr     [[INFINITY:h[0-9]+]], {{[[]}}[[ADDR]], :lo12:[[CSTLABEL]]{{[]]}}
+; CHECK-NEXT:  fabs    [[ABS:h[0-9]+]], h0
+; CHECK-NEXT:  fcmp    [[ABS]], [[INFINITY]]
+; CHECK-NEXT:  cset    w0, eq
+  %abs = tail call half @llvm.fabs.f16(half %x)
+  %cmpinf = fcmp oeq half %abs, 0xH7C00
+  %ret = zext i1 %cmpinf to i32
+  ret i32 %ret
+}
+
+; Check if INFINITY for float is materialized
+define i32 @replace_isinf_call_f32(float %x) {
+; CHECK-LABEL: replace_isinf_call_f32:
+; CHECK:       orr    [[INFSCALARREG:w[0-9]+]], wzr, #0x7f800000
+; CHECK-NEXT:  fabs   [[ABS:s[0-9]+]], s0
+; CHECK-NEXT:  fmov   [[INFREG:s[0-9]+]], [[INFSCALARREG]]
+; CHECK-NEXT:  fcmp   [[ABS]], [[INFREG]]
+; CHECK-NEXT:  cset   w0, eq
+  %abs = tail call float @llvm.fabs.f32(float %x)
+  %cmpinf = fcmp oeq float %abs, 0x7FF0000000000000
+  %ret = zext i1 %cmpinf to i32
+  ret i32 %ret
+}
+
+; Check if INFINITY for double is materialized
+define i32 @replace_isinf_call_f64(double %x) {
+; CHECK-LABEL: replace_isinf_call_f64:
+; CHECK:       orr    [[INFSCALARREG:x[0-9]+]], xzr, #0x7ff0000000000000
+; CHECK-NEXT:  fabs   [[ABS:d[0-9]+]], d0
+; CHECK-NEXT:  fmov   [[INFREG:d[0-9]+]], [[INFSCALARREG]]
+; CHECK-NEXT:  fcmp   [[ABS]], [[INFREG]]
+; CHECK-NEXT:  cset   w0, eq
+  %abs = tail call double @llvm.fabs.f64(double %x)
+  %cmpinf = fcmp oeq double %abs, 0x7FF0000000000000
+  %ret = zext i1 %cmpinf to i32
+  ret i32 %ret
+}
+
+; For long double it still requires loading the constant.
+define i32 @replace_isinf_call_f128(fp128 %x) {
+; CHECK-LABEL: replace_isinf_call_f128:
+; CHECK:       adrp    [[ADDR:x[0-9]+]], [[CSTLABEL:.LCP.*]]
+; CHECK:       ldr     q1, {{[[]}}[[ADDR]], :lo12:[[CSTLABEL]]{{[]]}}
+; CHECK:       bl      __eqtf2
+; CHECK:       cmp     w0, #0
+; CHECK:       cset    w0, eq
+  %abs = tail call fp128 @llvm.fabs.f128(fp128 %x)
+  %cmpinf = fcmp oeq fp128 %abs, 0xL00000000000000007FFF000000000000
+  %ret = zext i1 %cmpinf to i32
+  ret i32 %ret
+}
diff --git a/test/CodeGen/AArch64/known-never-nan.ll b/test/CodeGen/AArch64/known-never-nan.ll
index 1ffbb9bda44edfeb8e78b2218352be88120c1cb8..ef9fa5faac60c05ced586562b51588f64b876b29 100644
--- a/test/CodeGen/AArch64/known-never-nan.ll
+++ b/test/CodeGen/AArch64/known-never-nan.ll
@@ -28,13 +28,13 @@ define float @fmaxnm(i32 %i1, i32 %i2) #0 {
 define float @not_fmaxnm_maybe_nan(i32 %i1, i32 %i2) #0 {
 ; CHECK-LABEL: not_fmaxnm_maybe_nan:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    adrp x8, .LCPI1_0
-; CHECK-NEXT:    ldr s0, [x8, :lo12:.LCPI1_0]
-; CHECK-NEXT:    ucvtf s1, w0
-; CHECK-NEXT:    ucvtf s2, w1
-; CHECK-NEXT:    fmov s3, #17.00000000
-; CHECK-NEXT:    fmul s0, s1, s0
-; CHECK-NEXT:    fadd s1, s2, s3
+; CHECK-NEXT:    orr w8, wzr, #0xff800000
+; CHECK-NEXT:    ucvtf s0, w0
+; CHECK-NEXT:    ucvtf s1, w1
+; CHECK-NEXT:    fmov s2, #17.00000000
+; CHECK-NEXT:    fmov s3, w8
+; CHECK-NEXT:    fmul s0, s0, s3
+; CHECK-NEXT:    fadd s1, s1, s2
 ; CHECK-NEXT:    fcmp s0, s1
 ; CHECK-NEXT:    fcsel s0, s0, s1, pl
 ; CHECK-NEXT:    ret
diff --git a/test/CodeGen/AArch64/landingpad-ifcvt.ll b/test/CodeGen/AArch64/landingpad-ifcvt.ll
new file mode 100644
index 0000000000000000000000000000000000000000..4437970e1660065cd774ea7b7b68a48f94bd6b30
--- /dev/null
+++ b/test/CodeGen/AArch64/landingpad-ifcvt.ll
@@ -0,0 +1,29 @@
+; RUN: llc < %s | FileCheck %s
+
+; Make sure this doesn't crash (and the output is sane).
+; CHECK: ; %__except.ret
+; CHECK-NEXT: mov     x0, xzr
+
+target datalayout = "e-m:w-p:64:64-i32:32-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64-pc-windows-msvc19.11.0"
+
+define i64 @f(i32* %hwnd, i32 %message, i64 %wparam, i64 %lparam) personality i8* bitcast (i32 (...)* @__C_specific_handler to i8*) {
+entry:
+  %call = invoke i64 @callee(i32* %hwnd, i32 %message, i64 %wparam, i64 %lparam)
+          to label %__try.cont unwind label %catch.dispatch
+
+catch.dispatch:                                   ; preds = %entry
+  %0 = catchswitch within none [label %__except.ret] unwind to caller
+
+__except.ret:                                     ; preds = %catch.dispatch
+  %1 = catchpad within %0 [i8* bitcast (i32 (i8*, i8*)* @filt to i8*)]
+  catchret from %1 to label %__try.cont
+
+__try.cont:                                       ; preds = %__except.ret, %entry
+  %rv.0 = phi i64 [ 0, %__except.ret ], [ %call, %entry ]
+  ret i64 %rv.0
+}
+
+declare dso_local i64 @callee(i32*, i32, i64, i64)
+declare i32 @filt(i8*, i8* nocapture readnone)
+declare dso_local i32 @__C_specific_handler(...)
diff --git a/test/CodeGen/AArch64/ldst-opt.ll b/test/CodeGen/AArch64/ldst-opt.ll
index 7f6cba2133f05a69f2c6885e7f211f3f1979d539..fe55806e56f4826b8b593ac05050b120b6ce6bab 100644
--- a/test/CodeGen/AArch64/ldst-opt.ll
+++ b/test/CodeGen/AArch64/ldst-opt.ll
@@ -1681,3 +1681,19 @@ entry:
   %add = add i64 %ld, 1
   ret i64 %add
 }
+
+; CHECK-LABEL: trunc_splat_zero:
+; CHECK-DAG: strh wzr, [x0]
+define void @trunc_splat_zero(<2 x i8>* %ptr) {
+  store <2 x i8> zeroinitializer, <2 x i8>* %ptr, align 2
+  ret void
+}
+
+; CHECK-LABEL: trunc_splat:
+; CHECK: mov [[VAL:w[0-9]+]], #42
+; CHECK: movk [[VAL]], #42, lsl #16
+; CHECK: str [[VAL]], [x0]
+define void @trunc_splat(<2 x i16>* %ptr) {
+  store <2 x i16> <i16 42, i16 42>, <2 x i16>* %ptr, align 4
+  ret void
+}
diff --git a/test/CodeGen/AArch64/literal_pools_float.ll b/test/CodeGen/AArch64/literal_pools_float.ll
index 3c7063481e4835dc3be76d519f2c6b83c5f120b1..1a673949d7f52ef48fa99a5e2e69d162a87f501f 100644
--- a/test/CodeGen/AArch64/literal_pools_float.ll
+++ b/test/CodeGen/AArch64/literal_pools_float.ll
@@ -13,18 +13,16 @@ define void @floating_lits() {
 
   %floatval = load float, float* @varfloat
   %newfloat = fadd float %floatval, 128.0
-; CHECK: adrp x[[LITBASE:[0-9]+]], [[CURLIT:.LCPI[0-9]+_[0-9]+]]
-; CHECK: ldr [[LIT128:s[0-9]+]], [x[[LITBASE]], {{#?}}:lo12:[[CURLIT]]]
+; CHECK: mov [[W128:w[0-9]+]], #1124073472
+; CHECK: fmov [[LIT128:s[0-9]+]], [[W128]]
 ; CHECK-NOFP-NOT: ldr {{s[0-9]+}},
 
-; CHECK-TINY: ldr [[LIT128:s[0-9]+]], [[CURLIT:.LCPI[0-9]+_[0-9]+]]
+; CHECK-TINY: mov [[W128:w[0-9]+]], #1124073472
+; CHECK-TINE: fmov [[LIT128:s[0-9]+]], [[W128]]
 ; CHECK-NOFP-TINY-NOT: ldr {{s[0-9]+}},
 
-; CHECK-LARGE: movz x[[LITADDR:[0-9]+]], #:abs_g0_nc:[[CURLIT:.LCPI[0-9]+_[0-9]+]]
-; CHECK-LARGE: movk x[[LITADDR]], #:abs_g1_nc:[[CURLIT]]
-; CHECK-LARGE: movk x[[LITADDR]], #:abs_g2_nc:[[CURLIT]]
-; CHECK-LARGE: movk x[[LITADDR]], #:abs_g3:[[CURLIT]]
-; CHECK-LARGE: ldr {{s[0-9]+}}, [x[[LITADDR]]]
+; CHECK-LARGE: mov [[W128:w[0-9]+]], #1124073472
+; CHECK-LARGE: fmov [[LIT128:s[0-9]+]], [[W128]]
 ; CHECK-LARGE: fadd
 ; CHECK-NOFP-LARGE-NOT: ldr {{s[0-9]+}},
 ; CHECK-NOFP-LARGE-NOT: fadd
diff --git a/test/CodeGen/AArch64/local_vars.ll b/test/CodeGen/AArch64/local_vars.ll
index a479572d2a310168a5ee397e5d0d8ced55713de0..cf5bdac552794d2e59eb6a9c142164e69254bb28 100644
--- a/test/CodeGen/AArch64/local_vars.ll
+++ b/test/CodeGen/AArch64/local_vars.ll
@@ -1,5 +1,5 @@
 ; RUN: llc -verify-machineinstrs < %s -mtriple=aarch64-none-linux-gnu | FileCheck %s
-; RUN: llc -verify-machineinstrs < %s -mtriple=aarch64-none-linux-gnu -disable-fp-elim | FileCheck -check-prefix CHECK-WITHFP-ARM64 %s
+; RUN: llc -verify-machineinstrs < %s -mtriple=aarch64-none-linux-gnu -frame-pointer=all | FileCheck -check-prefix CHECK-WITHFP-ARM64 %s
 
 ; Make sure a reasonably sane prologue and epilogue are
 ; generated. This test is not robust in the face of an frame-handling
diff --git a/test/CodeGen/AArch64/machine-outliner-outline-bti.ll b/test/CodeGen/AArch64/machine-outliner-outline-bti.ll
new file mode 100644
index 0000000000000000000000000000000000000000..bc1521c6c80d7b60aa6d12965f107f0ace15fa26
--- /dev/null
+++ b/test/CodeGen/AArch64/machine-outliner-outline-bti.ll
@@ -0,0 +1,22 @@
+; RUN: llc -mtriple aarch64--none-eabi < %s | FileCheck %s
+
+; The BTI instruction cannot be outlined, because it needs to be the very first
+; instruction executed after an indirect call.
+
+@g = hidden global i32 0, align 4
+
+define hidden void @foo() minsize "branch-target-enforcement" {
+entry:
+; CHECK: hint #34
+; CHECK: b       OUTLINED_FUNCTION_0
+  store volatile i32 1, i32* @g, align 4
+  ret void
+}
+
+define hidden void @bar() minsize "branch-target-enforcement" {
+entry:
+; CHECK: hint #34
+; CHECK: b       OUTLINED_FUNCTION_0
+  store volatile i32 1, i32* @g, align 4
+  ret void
+}
diff --git a/test/CodeGen/AArch64/mingw-refptr.ll b/test/CodeGen/AArch64/mingw-refptr.ll
index 3a76d8586270a9132a9c0e59eb43826def01c967..dde1b895257811e41b9ed4228c3998d775b48c8f 100644
--- a/test/CodeGen/AArch64/mingw-refptr.ll
+++ b/test/CodeGen/AArch64/mingw-refptr.ll
@@ -1,4 +1,6 @@
 ; RUN: llc < %s -mtriple=aarch64-w64-mingw32 | FileCheck %s
+; RUN: llc < %s -global-isel -global-isel-abort=2 -pass-remarks-missed=gisel* \
+; RUN:     -mtriple=aarch64-w64-mingw32 2>&1| FileCheck %s --check-prefixes=GISEL,FALLBACK
 
 @var = external local_unnamed_addr global i32, align 4
 @dsolocalvar = external dso_local local_unnamed_addr global i32, align 4
@@ -68,11 +70,16 @@ entry:
 
 declare dso_local void @otherFunc()
 
+; FALLBACK-NOT: remark:{{.*}}sspFunc
 define dso_local void @sspFunc() #0 {
 ; CHECK-LABEL: sspFunc:
 ; CHECK:    adrp x8, .refptr.__stack_chk_guard
 ; CHECK:    ldr  x8, [x8, .refptr.__stack_chk_guard]
 ; CHECK:    ldr  x8, [x8]
+; GISEL-LABEL: sspFunc:
+; GISEL:    adrp x8, .refptr.__stack_chk_guard
+; GISEL:    ldr  x8, [x8, .refptr.__stack_chk_guard]
+; GISEL:    ldr  x8, [x8]
 entry:
   %c = alloca i8, align 1
   call void @llvm.lifetime.start.p0i8(i64 1, i8* nonnull %c)
diff --git a/test/CodeGen/AArch64/misched-fusion-arith-logic.mir b/test/CodeGen/AArch64/misched-fusion-arith-logic.mir
new file mode 100644
index 0000000000000000000000000000000000000000..62276779d1423991acbd81d09cf4071a30b1015b
--- /dev/null
+++ b/test/CodeGen/AArch64/misched-fusion-arith-logic.mir
@@ -0,0 +1,112 @@
+# RUN: llc -o /dev/null 2>&1 %s -mtriple aarch64-unknown -mattr=fuse-arith-logic -run-pass=machine-scheduler -misched-print-dags | FileCheck %s
+# RUN: llc -o /dev/null 2>&1 %s -mtriple aarch64-unknown -mcpu=exynos-m4         -run-pass=machine-scheduler -misched-print-dags | FileCheck %s
+# REQUIRES: asserts
+
+---
+name: arith
+body: |
+  bb.0.entry:
+    %0:gpr32 = SUBWrr undef $w0, undef $w1
+    %1:gpr32 = ADDWrr undef $w1, undef $w2
+    %2:gpr32 = SUBWrs %0, undef $w2, 0
+    %3:gpr32 = ADDWrs %1, undef $w3, 0
+
+    ; CHECK: SU(0): %0:gpr32 = SUBWrr undef $w0, undef $w1
+    ; CHECK: Successors:
+    ; CHECK: SU(2): Ord  Latency=0 Cluster
+    ; CHECK: SU(1): %1:gpr32 = ADDWrr undef $w1, undef $w2
+    ; CHECK: Successors:
+    ; CHECK: SU(3): Ord  Latency=0 Cluster
+    ; CHECK: SU(2): dead %2:gpr32 = SUBWrs %0:gpr32, undef $w2, 0
+    ; CHECK: Predecessors:
+    ; CHECK: SU(0): Ord  Latency=0 Cluster
+    ; CHECK: SU(3): dead %3:gpr32 = ADDWrs %1:gpr32, undef $w3, 0
+    ; CHECK: Predecessors:
+    ; CHECK: SU(1): Ord  Latency=0 Cluster
+...
+---
+name: compare
+body: |
+  bb.0.entry:
+    %0:gpr64 = ADDXrr undef $x0, undef $x1
+    %1:gpr64 = SUBXrs undef $x1, undef $x2, 0
+    %2:gpr64 = ADDSXrr %0, undef $x3, implicit-def $nzcv
+    %3:gpr64 = SUBSXrs %1, undef $x4, 0, implicit-def $nzcv
+
+    ; CHECK: SU(0): %0:gpr64 = ADDXrr undef $x0, undef $x1
+    ; CHECK: Successors:
+    ; CHECK: SU(2): Ord  Latency=0 Cluster
+    ; CHECK: SU(1): %1:gpr64 = SUBXrs undef $x1, undef $x2, 0
+    ; CHECK: Successors:
+    ; CHECK: SU(3): Ord  Latency=0 Cluster
+    ; CHECK: SU(2): dead %2:gpr64 = ADDSXrr %0:gpr64, undef $x3, implicit-def $nzcv
+    ; CHECK: Predecessors:
+    ; CHECK: SU(0): Ord  Latency=0 Cluster
+    ; CHECK: SU(3): dead %3:gpr64 = SUBSXrs %1:gpr64, undef $x4, 0, implicit-def $nzcv
+    ; CHECK: Predecessors:
+    ; CHECK: SU(1): Ord  Latency=0 Cluster
+...
+---
+name: logic
+body: |
+  bb.0.entry:
+    %0:gpr32 = ADDWrr undef $w0, undef $w1
+    %1:gpr64 = SUBXrs undef $x1, undef $x2, 0
+    %3:gpr32 = ANDWrs %0, undef $w3, 0
+    %4:gpr64 = ORRXrr %1, undef $x4
+
+    ; CHECK: SU(0): %0:gpr32 = ADDWrr undef $w0, undef $w1
+    ; CHECK: Successors:
+    ; CHECK: SU(2): Ord  Latency=0 Cluster
+    ; CHECK: SU(1): %1:gpr64 = SUBXrs undef $x1, undef $x2, 0
+    ; CHECK: Successors:
+    ; CHECK: SU(3): Ord  Latency=0 Cluster
+    ; CHECK: SU(2): dead %2:gpr32 = ANDWrs %0:gpr32, undef $w3, 0
+    ; CHECK: Predecessors:
+    ; CHECK: SU(0): Ord  Latency=0 Cluster
+    ; CHECK: SU(3): dead %3:gpr64 = ORRXrr %1:gpr64, undef $x4
+    ; CHECK: Predecessors:
+    ; CHECK: SU(1): Ord  Latency=0 Cluster
+...
+---
+name: nope
+body: |
+  bb.0.entry:
+    ; Shifted register.
+    %0:gpr32 = SUBWrr undef $w0, undef $w1
+    %1:gpr32 = SUBWrs %0, undef $w2, 1
+    ; CHECK: SU(0): %0:gpr32 = SUBWrr undef $w0, undef $w1
+    ; CHECK: Successors:
+    ; CHECK-NOT: SU(1): Ord  Latency=0 Cluster
+    ; CHECK: SU(1): dead %1:gpr32 = SUBWrs %0:gpr32, undef $w2, 1
+
+    ; Multiple successors.
+    %2:gpr64 = ADDXrr undef $x0, undef $x1
+    %3:gpr32 = EXTRACT_SUBREG %2, %subreg.sub_32
+    %4:gpr32 = ANDWrs %3, undef $w2, 0
+    %5:gpr64 = ADDSXrr %2, undef $x3, implicit-def $nzcv
+    ; CHECK: SU(2): %2:gpr64 = ADDXrr undef $x0, undef $x1
+    ; CHECK: Successors:
+    ; CHECK-NOT: SU(3): Ord  Latency=0 Cluster
+    ; CHECK: SU(5): Ord  Latency=0 Cluster
+    ; CHECK: SU(3): %3:gpr32 = EXTRACT_SUBREG %2:gpr64, %subreg.sub_32
+    ; CHECK: SU(5): dead %5:gpr64 = ADDSXrr %2:gpr64, undef $x3, implicit-def $nzcv
+
+    ; Different register sizes.
+    %6:gpr32 = SUBWrr undef $w0, undef $w1
+    %7:gpr64 = ADDXrr undef $x1, undef $x2
+    %8:gpr64 = SUBXrr %7, undef $x3
+    %9:gpr32 = ADDWrr %6, undef $w4
+    ; CHECK: SU(6): %6:gpr32 = SUBWrr undef $w0, undef $w1
+    ; CHECK: Successors:
+    ; CHECK-NOT: SU(8): Ord  Latency=0 Cluster
+    ; CHECK: SU(7): %7:gpr64 = ADDXrr undef $x1, undef $x2
+    ; CHECK: Successors:
+    ; CHECK-NOT: SU(9): Ord  Latency=0 Cluster
+    ; CHECK: SU(8): dead %8:gpr64 = SUBXrr %7:gpr64, undef $x3
+    ; CHECK: Predecessors:
+    ; CHECK: SU(7): Ord  Latency=0 Cluster
+    ; CHECK: SU(9): dead %9:gpr32 = ADDWrr %6:gpr32, undef $w4
+    ; CHECK: Predecessors:
+    ; CHECK: SU(6): Ord  Latency=0 Cluster
+...
diff --git a/test/CodeGen/AArch64/pow.75.ll b/test/CodeGen/AArch64/pow.75.ll
new file mode 100644
index 0000000000000000000000000000000000000000..4ceb7d9e23e15081540bda498bba18d54536ff93
--- /dev/null
+++ b/test/CodeGen/AArch64/pow.75.ll
@@ -0,0 +1,48 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=aarch64-- -debug 2>&1 | FileCheck %s
+; REQUIRES: asserts
+
+declare float @llvm.pow.f32(float, float)
+declare <4 x float> @llvm.pow.v4f32(<4 x float>, <4 x float>)
+declare double @llvm.pow.f64(double, double)
+declare <2 x double> @llvm.pow.v2f64(<2 x double>, <2 x double>)
+
+define float @pow_f32_three_fourth_fmf(float %x) nounwind {
+; CHECK: Combining: {{.*}}: f32 = fpow ninf nsz afn [[X:t[0-9]+]], ConstantFP:f32<7.500000e-01>
+; CHECK-NEXT: Creating new node: [[SQRT:t[0-9]+]]: f32 = fsqrt ninf nsz afn [[X]]
+; CHECK-NEXT: Creating new node: [[SQRTSQRT:t[0-9]+]]: f32 = fsqrt ninf nsz afn [[SQRT]]
+; CHECK-NEXT: Creating new node: [[R:t[0-9]+]]: f32 = fmul ninf nsz afn [[SQRT]], [[SQRTSQRT]]
+; CHECK-NEXT:  ... into: [[R]]: f32 = fmul ninf nsz afn [[SQRT]], [[SQRTSQRT]]
+  %r = call nsz ninf afn float @llvm.pow.f32(float %x, float 7.5e-01)
+  ret float %r
+}
+
+define double @pow_f64_three_fourth_fmf(double %x) nounwind {
+; CHECK: Combining: {{.*}}: f64 = fpow ninf nsz afn [[X:t[0-9]+]], ConstantFP:f64<7.500000e-01>
+; CHECK-NEXT: Creating new node: [[SQRT:t[0-9]+]]: f64 = fsqrt ninf nsz afn [[X]]
+; CHECK-NEXT: Creating new node: [[SQRTSQRT:t[0-9]+]]: f64 = fsqrt ninf nsz afn [[SQRT]]
+; CHECK-NEXT: Creating new node: [[R:t[0-9]+]]: f64 = fmul ninf nsz afn [[SQRT]], [[SQRTSQRT]]
+; CHECK-NEXT:  ... into: [[R]]: f64 = fmul ninf nsz afn [[SQRT]], [[SQRTSQRT]]
+  %r = call nsz ninf afn double @llvm.pow.f64(double %x, double 7.5e-01)
+  ret double %r
+}
+
+define <4 x float> @pow_v4f32_three_fourth_fmf(<4 x float> %x) nounwind {
+; CHECK: Combining: {{.*}}: v4f32 = fpow nnan ninf nsz arcp contract afn reassoc [[X:t[0-9]+]], {{.*}}
+; CHECK-NEXT: Creating new node: [[SQRT:t[0-9]+]]: v4f32 = fsqrt nnan ninf nsz arcp contract afn reassoc [[X]]
+; CHECK-NEXT: Creating new node: [[SQRTSQRT:t[0-9]+]]: v4f32 = fsqrt nnan ninf nsz arcp contract afn reassoc [[SQRT]]
+; CHECK-NEXT: Creating new node: [[R:t[0-9]+]]: v4f32 = fmul nnan ninf nsz arcp contract afn reassoc [[SQRT]], [[SQRTSQRT]]
+; CHECK-NEXT:  ... into: [[R]]: v4f32 = fmul nnan ninf nsz arcp contract afn reassoc [[SQRT]], [[SQRTSQRT]]
+  %r = call fast <4 x float> @llvm.pow.v4f32(<4 x float> %x, <4 x float> <float 7.5e-1, float 7.5e-1, float 7.5e-01, float 7.5e-01>)
+  ret <4 x float> %r
+}
+
+define <2 x double> @pow_v2f64_three_fourth_fmf(<2 x double> %x) nounwind {
+; CHECK: Combining: {{.*}}: v2f64 = fpow nnan ninf nsz arcp contract afn reassoc [[X:t[0-9]+]], {{.*}}
+; CHECK-NEXT: Creating new node: [[SQRT:t[0-9]+]]: v2f64 = fsqrt nnan ninf nsz arcp contract afn reassoc [[X]]
+; CHECK-NEXT: Creating new node: [[SQRTSQRT:t[0-9]+]]: v2f64 = fsqrt nnan ninf nsz arcp contract afn reassoc [[SQRT]]
+; CHECK-NEXT: Creating new node: [[R:t[0-9]+]]: v2f64 = fmul nnan ninf nsz arcp contract afn reassoc [[SQRT]], [[SQRTSQRT]]
+; CHECK-NEXT:  ... into: [[R]]: v2f64 = fmul nnan ninf nsz arcp contract afn reassoc [[SQRT]], [[SQRTSQRT]]
+  %r = call fast <2 x double> @llvm.pow.v2f64(<2 x double> %x, <2 x double> <double 7.5e-1, double 7.5e-1>)
+  ret <2 x double> %r
+}
diff --git a/test/CodeGen/AArch64/pr40091.ll b/test/CodeGen/AArch64/pr40091.ll
index 8cf51f4beb5f44a87cfecda9360103abe108a20c..b70ae8a39b7ea0220628edfe8cb667c2b51ec19b 100644
--- a/test/CodeGen/AArch64/pr40091.ll
+++ b/test/CodeGen/AArch64/pr40091.ll
@@ -4,7 +4,7 @@
 define i64 @test(i64 %aa) {
 ; CHECK-LABEL: test:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    movi v0.2d, #0xffffffffffffffff
+; CHECK-NEXT:    movi v0.8b, #137
 ; CHECK-NEXT:    fmov x0, d0
 ; CHECK-NEXT:    ret
 entry:
diff --git a/test/CodeGen/AArch64/regress-tblgen-chains.ll b/test/CodeGen/AArch64/regress-tblgen-chains.ll
index 50da7d139f177b7943133afa374bac42ee1ae59d..bf2dece22508ea230e802f7abe299d2cf127ef35 100644
--- a/test/CodeGen/AArch64/regress-tblgen-chains.ll
+++ b/test/CodeGen/AArch64/regress-tblgen-chains.ll
@@ -1,4 +1,4 @@
-; RUN: llc -verify-machineinstrs -mtriple=arm64-apple-ios7.0 -disable-fp-elim -o - %s | FileCheck %s
+; RUN: llc -verify-machineinstrs -mtriple=arm64-apple-ios7.0 -frame-pointer=all -o - %s | FileCheck %s
 
 ; When generating DAG selection tables, TableGen used to only flag an
 ; instruction as needing a chain on its own account if it had a built-in pattern
diff --git a/test/CodeGen/AArch64/regress-w29-reserved-with-fp.ll b/test/CodeGen/AArch64/regress-w29-reserved-with-fp.ll
index 66a5ed6e5277a5c3cda1422f6bb26deff9645ec6..2cbcad6c70cbab48cb352e1ff727efd2432203df 100644
--- a/test/CodeGen/AArch64/regress-w29-reserved-with-fp.ll
+++ b/test/CodeGen/AArch64/regress-w29-reserved-with-fp.ll
@@ -1,4 +1,4 @@
-; RUN: llc -mtriple=aarch64-none-linux-gnu -disable-fp-elim < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-none-linux-gnu -frame-pointer=all < %s | FileCheck %s
 @var = global i32 0
 
 declare void @bar()
diff --git a/test/CodeGen/AArch64/reloc-specifiers.mir b/test/CodeGen/AArch64/reloc-specifiers.mir
new file mode 100644
index 0000000000000000000000000000000000000000..374a4759b4cd99c71b7900dea282f554d42fc7a6
--- /dev/null
+++ b/test/CodeGen/AArch64/reloc-specifiers.mir
@@ -0,0 +1,21 @@
+# RUN: llc -mtriple=arm64-windows -start-after=prologepilog -show-mc-encoding \
+# RUN: -o - %s | FileCheck %s
+
+--- |
+  define void @bar() { ret void }
+...
+
+---
+name:            bar
+body: |
+  bb.0:
+    ; CHECK-LABEL: bar
+
+    ; CHECK: movz    x0, #:abs_g1_s:.Lfoo$frame_escape_0 ; encoding: [0bAAA00000,A,0b101AAAAA,0xd2]
+    ; CHECK: fixup A - offset: 0, value: :abs_g1_s:.Lfoo$frame_escape_0, kind: fixup_aarch64_movw
+    renamable $x0 = MOVZXi target-flags(aarch64-g1, aarch64-s) <mcsymbol .Lfoo$frame_escape_0>, 16
+
+    ; CHECK: movk    x0, #:abs_g0_nc:.Lfoo$frame_escape_0 ; encoding: [0bAAA00000,A,0b100AAAAA,0xf2]
+    ; CHECK: fixup A - offset: 0, value: :abs_g0_nc:.Lfoo$frame_escape_0, kind: fixup_aarch64_movw
+    renamable $x0 = MOVKXi $x0, target-flags(aarch64-g0, aarch64-nc) <mcsymbol .Lfoo$frame_escape_0>, 0
+...
diff --git a/test/CodeGen/AArch64/seh-finally.ll b/test/CodeGen/AArch64/seh-finally.ll
new file mode 100644
index 0000000000000000000000000000000000000000..2a7b2a925f3784ca244c29ca84eb900491957fc8
--- /dev/null
+++ b/test/CodeGen/AArch64/seh-finally.ll
@@ -0,0 +1,283 @@
+; RUN: llc -mtriple arm64-windows -o - %s | FileCheck %s
+
+; struct S { int x; };
+; void foo(int n);
+; void foo(struct S o);
+; void simple_seh() {
+;   struct S o;
+; 
+;   __try { foo(o.x); }
+;   __finally { foo(o.x); }
+; }
+; void stack_realign() {
+;   struct S __declspec(align(32)) o;
+; 
+;   __try { foo(o.x); }
+;   __finally { foo(o.x); }
+; }
+; void vla_present(int n) {
+;   int vla[n];
+; 
+;   __try { foo(n); }
+;   __finally { foo(n); }
+; }
+; void vla_and_realign(int n) {
+;   struct S __declspec(align(32)) o;
+;   int vla[n];
+; 
+;   __try { foo(o.x); }
+;   __finally { foo(o.x); }
+; }
+
+%struct.S = type { i32 }
+
+; Test simple SEH (__try/__finally).
+define void @simple_seh() #0 personality i8* bitcast (i32 (...)* @__C_specific_handler to i8*) {
+entry:
+; CHECK-LABEL: simple_seh
+; CHECK: add     x29, sp, #16
+; CHECK: orr     x1, xzr, #0xfffffffffffffffe
+; CHECK: stur    x1, [x29, #-16]
+; CHECK: .set .Lsimple_seh$frame_escape_0, -8
+; CHECK: ldur    w0, [x29, #-8]
+; CHECK: bl      foo
+
+  %o = alloca %struct.S, align 4
+  call void (...) @llvm.localescape(%struct.S* %o)
+  %x = getelementptr inbounds %struct.S, %struct.S* %o, i32 0, i32 0
+  %0 = load i32, i32* %x, align 4
+  invoke void @foo(i32 %0) #5
+          to label %invoke.cont unwind label %ehcleanup
+
+invoke.cont:                                      ; preds = %entry
+  %1 = call i8* @llvm.localaddress()
+  call void @fin_simple_seh(i8 0, i8* %1)
+  ret void
+
+ehcleanup:                                        ; preds = %entry
+  %2 = cleanuppad within none []
+  %3 = call i8* @llvm.localaddress()
+  call void @fin_simple_seh(i8 1, i8* %3) [ "funclet"(token %2) ]
+  cleanupret from %2 unwind to caller
+}
+
+define void @fin_simple_seh(i8 %abnormal_termination, i8* %frame_pointer) {
+entry:
+; CHECK-LABEL: fin_simple_seh
+; CHECK: movz    x8, #:abs_g1_s:.Lsimple_seh$frame_escape_0
+; CHECK: movk    x8, #:abs_g0_nc:.Lsimple_seh$frame_escape_0
+; CHECK: strb    w0, [sp, #15]
+; CHECK: ldr     w0, [x1, x8]
+; CHECK: bl      foo
+
+  %frame_pointer.addr = alloca i8*, align 8
+  %abnormal_termination.addr = alloca i8, align 1
+  %0 = call i8* @llvm.localrecover(i8* bitcast (void ()* @simple_seh to i8*), i8* %frame_pointer, i32 0)
+  %o = bitcast i8* %0 to %struct.S*
+  store i8* %frame_pointer, i8** %frame_pointer.addr, align 8
+  store i8 %abnormal_termination, i8* %abnormal_termination.addr, align 1
+  %x = getelementptr inbounds %struct.S, %struct.S* %o, i32 0, i32 0
+  %1 = load i32, i32* %x, align 4
+  call void @foo(i32 %1)
+  ret void
+}
+
+; Test SEH when stack realignment is needed in case highly aligned stack objects are present.
+define void @stack_realign() #0 personality i8* bitcast (i32 (...)* @__C_specific_handler to i8*) {
+entry:
+; CHECK-LABEL: stack_realign
+; CHECK: add     x29, sp, #16
+; CHECK: sub     x9, sp, #64
+; CHECK: and     sp, x9, #0xffffffffffffffe0
+; CHECK: mov     x19, sp
+; CHECK: orr     x1, xzr, #0xfffffffffffffffe
+; CHECK: stur    x1, [x19, #16]
+; CHECK: .set .Lstack_realign$frame_escape_0, 32
+; CHECK: ldr     w0, [x19, #32]
+; CHECK: bl      foo
+
+  %o = alloca %struct.S, align 32
+  call void (...) @llvm.localescape(%struct.S* %o)
+  %x = getelementptr inbounds %struct.S, %struct.S* %o, i32 0, i32 0
+  %0 = load i32, i32* %x, align 32
+  invoke void @foo(i32 %0) #5
+          to label %invoke.cont unwind label %ehcleanup
+
+invoke.cont:                                      ; preds = %entry
+  %1 = call i8* @llvm.localaddress()
+  call void @fin_stack_realign(i8 0, i8* %1)
+  ret void
+
+ehcleanup:                                        ; preds = %entry
+  %2 = cleanuppad within none []
+  %3 = call i8* @llvm.localaddress()
+  call void @fin_stack_realign(i8 1, i8* %3) [ "funclet"(token %2) ]
+  cleanupret from %2 unwind to caller
+}
+
+define void @fin_stack_realign(i8 %abnormal_termination, i8* %frame_pointer) {
+entry:
+; CHECK-LABEL: fin_stack_realign
+; CHECK: movz    x8, #:abs_g1_s:.Lstack_realign$frame_escape_0
+; CHECK: movk    x8, #:abs_g0_nc:.Lstack_realign$frame_escape_0
+; CHECK: strb    w0, [sp, #15]
+; CHECK: ldr     w0, [x1, x8]
+; CHECK: bl      foo
+
+  %frame_pointer.addr = alloca i8*, align 8
+  %abnormal_termination.addr = alloca i8, align 1
+  %0 = call i8* @llvm.localrecover(i8* bitcast (void ()* @stack_realign to i8*), i8* %frame_pointer, i32 0)
+  %o = bitcast i8* %0 to %struct.S*
+  store i8* %frame_pointer, i8** %frame_pointer.addr, align 8
+  store i8 %abnormal_termination, i8* %abnormal_termination.addr, align 1
+  %x = getelementptr inbounds %struct.S, %struct.S* %o, i32 0, i32 0
+  %1 = load i32, i32* %x, align 32
+  call void @foo(i32 %1)
+  ret void
+}
+
+; Test SEH when variable size objects are present on the stack. Note: Escaped vla's are current not supported by SEH.
+define void @vla_present(i32 %n) #0 personality i8* bitcast (i32 (...)* @__C_specific_handler to i8*) {
+entry:
+; CHECK-LABEL: vla_present
+; CHECK: add     x29, sp, #32
+; CHECK: orr     x1, xzr, #0xfffffffffffffffe
+; CHECK: stur    x1, [x29, #-32]
+; CHECK: .set .Lvla_present$frame_escape_0, -4
+; CHECK: stur    w0, [x29, #-4]
+; CHECK: ldur    w8, [x29, #-4]
+; CHECK: mov     x9, sp
+; CHECK: stur    x9, [x29, #-16]
+; CHECK: stur    x8, [x29, #-24]
+; CHECK: ldur    w0, [x29, #-4]
+; CHECK: bl      foo
+
+  %n.addr = alloca i32, align 4
+  %saved_stack = alloca i8*, align 8
+  %__vla_expr0 = alloca i64, align 8
+  call void (...) @llvm.localescape(i32* %n.addr)
+  store i32 %n, i32* %n.addr, align 4
+  %0 = load i32, i32* %n.addr, align 4
+  %1 = zext i32 %0 to i64
+  %2 = call i8* @llvm.stacksave()
+  store i8* %2, i8** %saved_stack, align 8
+  %vla = alloca i32, i64 %1, align 4
+  store i64 %1, i64* %__vla_expr0, align 8
+  %3 = load i32, i32* %n.addr, align 4
+  invoke void @foo(i32 %3) #5
+          to label %invoke.cont unwind label %ehcleanup
+
+invoke.cont:                                      ; preds = %entry
+  %4 = call i8* @llvm.localaddress()
+  call void @fin_vla_present(i8 0, i8* %4)
+  %5 = load i8*, i8** %saved_stack, align 8
+  call void @llvm.stackrestore(i8* %5)
+  ret void
+
+ehcleanup:                                        ; preds = %entry
+  %6 = cleanuppad within none []
+  %7 = call i8* @llvm.localaddress()
+  call void @fin_vla_present(i8 1, i8* %7) [ "funclet"(token %6) ]
+  cleanupret from %6 unwind to caller
+}
+
+define void @fin_vla_present(i8 %abnormal_termination, i8* %frame_pointer) {
+entry:
+; CHECK-LABEL: fin_vla_present
+; CHECK: movz    x8, #:abs_g1_s:.Lvla_present$frame_escape_0
+; CHECK: movk    x8, #:abs_g0_nc:.Lvla_present$frame_escape_0
+; CHECK: strb    w0, [sp, #15]
+; CHECK: ldr     w0, [x1, x8]
+; CHECK: bl      foo
+
+  %frame_pointer.addr = alloca i8*, align 8
+  %abnormal_termination.addr = alloca i8, align 1
+  %0 = call i8* @llvm.localrecover(i8* bitcast (void (i32)* @vla_present to i8*), i8* %frame_pointer, i32 0)
+  %n.addr = bitcast i8* %0 to i32*
+  store i8* %frame_pointer, i8** %frame_pointer.addr, align 8
+  store i8 %abnormal_termination, i8* %abnormal_termination.addr, align 1
+  %1 = load i32, i32* %n.addr, align 4
+  call void @foo(i32 %1)
+  ret void
+}
+
+; Test when both vla's and highly aligned objects are present on stack.
+define void @vla_and_realign(i32 %n) #0 personality i8* bitcast (i32 (...)* @__C_specific_handler to i8*) {
+entry:
+; CHECK-LABEL: vla_and_realign
+; CHECK: add     x29, sp, #16
+; CHECK: sub     x9, sp, #64
+; CHECK: and     sp, x9, #0xffffffffffffffe0
+; CHECK: mov     x19, sp
+; CHECK: orr     x1, xzr, #0xfffffffffffffffe
+; CHECK: stur    x1, [x19]
+; CHECK: .set .Lvla_and_realign$frame_escape_0, 32
+; CHECK: stur    w0, [x29, #-4]
+; CHECK: ldur    w8, [x29, #-4]
+; CHECK: mov     x9, sp
+; CHECK: str     x9, [x19, #24]
+; CHECK: str     x8, [x19, #16]
+; CHECK: ldr     w0, [x19, #32]
+; CHECK: bl      foo
+
+  %n.addr = alloca i32, align 4
+  %o = alloca %struct.S, align 32
+  %saved_stack = alloca i8*, align 8
+  %__vla_expr0 = alloca i64, align 8
+  call void (...) @llvm.localescape(%struct.S* %o)
+  store i32 %n, i32* %n.addr, align 4
+  %0 = load i32, i32* %n.addr, align 4
+  %1 = zext i32 %0 to i64
+  %2 = call i8* @llvm.stacksave()
+  store i8* %2, i8** %saved_stack, align 8
+  %vla = alloca i32, i64 %1, align 4
+  store i64 %1, i64* %__vla_expr0, align 8
+  %x = getelementptr inbounds %struct.S, %struct.S* %o, i32 0, i32 0
+  %3 = load i32, i32* %x, align 32
+  invoke void @foo(i32 %3) #5
+          to label %invoke.cont unwind label %ehcleanup
+
+invoke.cont:                                      ; preds = %entry
+  %4 = call i8* @llvm.localaddress()
+  call void @fin_vla_and_realign(i8 0, i8* %4)
+  %5 = load i8*, i8** %saved_stack, align 8
+  call void @llvm.stackrestore(i8* %5)
+  ret void
+
+ehcleanup:                                        ; preds = %entry
+  %6 = cleanuppad within none []
+  %7 = call i8* @llvm.localaddress()
+  call void @fin_vla_and_realign(i8 1, i8* %7) [ "funclet"(token %6) ]
+  cleanupret from %6 unwind to caller
+}
+
+define void @fin_vla_and_realign(i8 %abnormal_termination, i8* %frame_pointer) {
+entry:
+; CHECK-LABEL: fin_vla_and_realign
+; CHECK: movz    x8, #:abs_g1_s:.Lvla_and_realign$frame_escape_0
+; CHECK: movk    x8, #:abs_g0_nc:.Lvla_and_realign$frame_escape_0
+; CHECK: strb    w0, [sp, #15]
+; CHECK: ldr     w0, [x1, x8]
+; CHECK: bl      foo
+
+  %frame_pointer.addr = alloca i8*, align 8
+  %abnormal_termination.addr = alloca i8, align 1
+  %0 = call i8* @llvm.localrecover(i8* bitcast (void (i32)* @vla_and_realign to i8*), i8* %frame_pointer, i32 0)
+  %o = bitcast i8* %0 to %struct.S*
+  store i8* %frame_pointer, i8** %frame_pointer.addr, align 8
+  store i8 %abnormal_termination, i8* %abnormal_termination.addr, align 1
+  %x = getelementptr inbounds %struct.S, %struct.S* %o, i32 0, i32 0
+  %1 = load i32, i32* %x, align 32
+  call void @foo(i32 %1)
+  ret void
+}
+
+declare void @foo(i32)
+declare void @llvm.stackrestore(i8*)
+declare i8* @llvm.stacksave()
+declare i8* @llvm.localrecover(i8*, i8*, i32)
+declare i8* @llvm.localaddress()
+declare void @llvm.localescape(...)
+declare i32 @__C_specific_handler(...)
+
+attributes #0 = { noinline optnone }
diff --git a/test/CodeGen/AArch64/seh_funclet_x1.ll b/test/CodeGen/AArch64/seh_funclet_x1.ll
new file mode 100644
index 0000000000000000000000000000000000000000..8f2a4dade30909d1d0b4cda3fc84b78b29a2a059
--- /dev/null
+++ b/test/CodeGen/AArch64/seh_funclet_x1.ll
@@ -0,0 +1,121 @@
+; RUN: llc -o - %s -verify-machineinstrs -mtriple=aarch64-windows | FileCheck %s
+
+; Windows runtime passes the establisher frame as the second argument to the
+; termination handler.  Check that we copy it into fp.
+
+; CHECK:      ?dtor$3@?0?main@4HA":
+; CHECK:      .seh_proc "?dtor$3@?0?main@4HA"
+; CHECK:      stp     x29, x30, [sp, #-16]!   ; 16-byte Folded Spill
+; CHECK-NEXT: mov     x29, x1
+
+target datalayout = "e-m:w-p:64:64-i32:32-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64-unknown-windows-msvc19.15.26732"
+
+; Function Attrs: noinline nounwind optnone uwtable
+define dso_local i32 @main() #0 personality i8* bitcast (i32 (...)* @__C_specific_handler to i8*) {
+entry:
+  %retval = alloca i32, align 4
+  %Counter = alloca i32, align 4
+  %__exception_code = alloca i32, align 4
+  call void (...) @llvm.localescape(i32* %Counter)
+  store i32 0, i32* %retval, align 4
+  store i32 0, i32* %Counter, align 4
+  %call = invoke i32 bitcast (i32 (...)* @RaiseStatus to i32 (i32)*)(i32 -1073741675) #3
+          to label %invoke.cont unwind label %ehcleanup
+
+invoke.cont:                                      ; preds = %entry
+  %0 = call i8* @llvm.localaddress()
+  invoke void @"?fin$0@0@main@@"(i8 0, i8* %0) #3
+          to label %invoke.cont1 unwind label %catch.dispatch
+
+invoke.cont1:                                     ; preds = %invoke.cont
+  br label %__try.cont
+
+ehcleanup:                                        ; preds = %entry
+  %1 = cleanuppad within none []
+  %2 = call i8* @llvm.localaddress()
+  invoke void @"?fin$0@0@main@@"(i8 1, i8* %2) #3 [ "funclet"(token %1) ]
+          to label %invoke.cont2 unwind label %catch.dispatch
+
+invoke.cont2:                                     ; preds = %ehcleanup
+  cleanupret from %1 unwind label %catch.dispatch
+
+catch.dispatch:                                   ; preds = %invoke.cont2, %ehcleanup, %invoke.cont
+  %3 = catchswitch within none [label %__except] unwind to caller
+
+__except:                                         ; preds = %catch.dispatch
+  %4 = catchpad within %3 [i8* null]
+  catchret from %4 to label %__except3
+
+__except3:                                        ; preds = %__except
+  %5 = call i32 @llvm.eh.exceptioncode(token %4)
+  store i32 %5, i32* %__exception_code, align 4
+  %6 = load i32, i32* %Counter, align 4
+  %add = add nsw i32 %6, 5
+  store i32 %add, i32* %Counter, align 4
+  br label %__try.cont
+
+__try.cont:                                       ; preds = %__except3, %invoke.cont1
+  %7 = load i32, i32* %retval, align 4
+  ret i32 %7
+}
+
+define internal void @"?fin$0@0@main@@"(i8 %abnormal_termination, i8* %frame_pointer) {
+entry:
+  %frame_pointer.addr = alloca i8*, align 8
+  %abnormal_termination.addr = alloca i8, align 1
+  %0 = call i8* @llvm.localrecover(i8* bitcast (i32 ()* @main to i8*), i8* %frame_pointer, i32 0)
+  %Counter = bitcast i8* %0 to i32*
+  store i8* %frame_pointer, i8** %frame_pointer.addr, align 8
+  store i8 %abnormal_termination, i8* %abnormal_termination.addr, align 1
+  store i32 3, i32* %Counter, align 4
+  call void @"?fin$1@0@main@@"(i8 0, i8* %frame_pointer)
+  %1 = load i32, i32* %Counter, align 4
+  %add = add nsw i32 %1, 2
+  store i32 %add, i32* %Counter, align 4
+  %call = call i32 bitcast (i32 (...)* @RaiseStatus to i32 (i32)*)(i32 -1073741675)
+  ret void
+}
+
+; Function Attrs: nounwind readnone
+declare i8* @llvm.localrecover(i8*, i8*, i32) #1
+
+define internal void @"?fin$1@0@main@@"(i8 %abnormal_termination, i8* %frame_pointer) {
+entry:
+  %frame_pointer.addr = alloca i8*, align 8
+  %abnormal_termination.addr = alloca i8, align 1
+  %0 = call i8* @llvm.localrecover(i8* bitcast (i32 ()* @main to i8*), i8* %frame_pointer, i32 0)
+  %Counter = bitcast i8* %0 to i32*
+  store i8* %frame_pointer, i8** %frame_pointer.addr, align 8
+  store i8 %abnormal_termination, i8* %abnormal_termination.addr, align 1
+  %1 = load i32, i32* %Counter, align 4
+  %cmp = icmp eq i32 %1, 3
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  %2 = load i32, i32* %Counter, align 4
+  %add = add nsw i32 %2, 1
+  store i32 %add, i32* %Counter, align 4
+  br label %if.end
+
+if.end:                                           ; preds = %if.then, %entry
+  ret void
+}
+
+declare dso_local i32 @RaiseStatus(...)
+
+declare dso_local i32 @__C_specific_handler(...)
+
+; Function Attrs: nounwind readnone
+declare i8* @llvm.localaddress() #1
+
+; Function Attrs: nounwind readnone
+declare i32 @llvm.eh.exceptioncode(token) #1
+
+; Function Attrs: nounwind
+declare void @llvm.localescape(...) #2
+
+attributes #0 = { noinline nounwind optnone uwtable }
+attributes #1 = { nounwind readnone }
+attributes #2 = { nounwind }
+attributes #3 = { noinline }
diff --git a/test/CodeGen/AArch64/seqpaircopy.mir b/test/CodeGen/AArch64/seqpaircopy.mir
new file mode 100644
index 0000000000000000000000000000000000000000..89511cbf726bd069a3e7944a1538fa73d26fa3e7
--- /dev/null
+++ b/test/CodeGen/AArch64/seqpaircopy.mir
@@ -0,0 +1,23 @@
+# RUN: llc -o - %s -mtriple=aarch64-- -mattr=+v8.1a -run-pass=postrapseudos | FileCheck %s
+---
+# CHECK-LABEL: name: copy_xseqpairs
+name: copy_xseqpairs
+body: |
+  bb.0:
+    ; CHECK: $x4_x5 = CASPALX $x4_x5, $x2_x3, $x0
+    ; CHECK: $x0 = ORRXrs $xzr, $x4, 0
+    ; CHECK: $x1 = ORRXrs $xzr, $x5, 0
+    $x4_x5 = CASPALX $x4_x5, $x2_x3, $x0
+    $x0_x1 = COPY $x4_x5
+...
+---
+# CHECK-LABEL: name: copy_wseqpairs
+name: copy_wseqpairs
+body: |
+  bb.0:
+    ; CHECK: $w4_w5 = CASPALW $w4_w5, $w2_w3, $x0
+    ; CHECK: $w0 = ORRWrs $wzr, $w4, 0
+    ; CHECK: $w1 = ORRWrs $wzr, $w5, 0
+    $w4_w5 = CASPALW $w4_w5, $w2_w3, $x0
+    $w0_w1 = COPY $w4_w5
+...
diff --git a/test/CodeGen/AArch64/shift-mod.ll b/test/CodeGen/AArch64/shift-mod.ll
index fdf1fa49ba3dd11cb6e3fe0b0a67077d92d7e9cc..8eba4ab1aab5154432968d78e9917c1a8c96ed3c 100644
--- a/test/CodeGen/AArch64/shift-mod.ll
+++ b/test/CodeGen/AArch64/shift-mod.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=aarch64 < %s | FileCheck %s
 
 ; Check that we optimize out AND instructions and ADD/SUB instructions
@@ -6,8 +7,9 @@
 
 define i32 @test1(i32 %x, i64 %y) {
 ; CHECK-LABEL: test1:
-; CHECK-NOT: and
-; CHECK: lsr
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    lsr w0, w0, w1
+; CHECK-NEXT:    ret
   %sh_prom = trunc i64 %y to i32
   %shr = lshr i32 %x, %sh_prom
   ret i32 %shr
@@ -15,10 +17,10 @@ define i32 @test1(i32 %x, i64 %y) {
 
 define i64 @test2(i32 %x, i64 %y) {
 ; CHECK-LABEL: test2:
-; CHECK-NOT: orr
-; CHECK-NOT: sub
-; CHECK: neg
-; CHECK: asr
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    neg w[[REG:[0-9]+]], w0
+; CHECK-NEXT:    asr x0, x1, x[[REG]]
+; CHECK-NEXT:    ret
   %sub9 = sub nsw i32 64, %x
   %sh_prom12.i = zext i32 %sub9 to i64
   %shr.i = ashr i64 %y, %sh_prom12.i
@@ -27,9 +29,46 @@ define i64 @test2(i32 %x, i64 %y) {
 
 define i64 @test3(i64 %x, i64 %y) {
 ; CHECK-LABEL: test3:
-; CHECK-NOT: add
-; CHECK: lsl
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    lsl x0, x1, x0
+; CHECK-NEXT:    ret
   %add = add nsw i64 64, %x
   %shl = shl i64 %y, %add
   ret i64 %shl
-}
\ No newline at end of file
+}
+
+define i64 @test4(i64 %y, i32 %s) {
+; CHECK-LABEL: test4:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-NEXT:    asr x0, x0, x1
+; CHECK-NEXT:    ret
+entry:
+  %sh_prom = zext i32 %s to i64
+  %shr = ashr i64 %y, %sh_prom
+  ret i64 %shr
+}
+
+define i64 @test5(i64 %y, i32 %s) {
+; CHECK-LABEL: test5:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-NEXT:    asr x0, x0, x1
+; CHECK-NEXT:    ret
+entry:
+  %sh_prom = sext i32 %s to i64
+  %shr = ashr i64 %y, %sh_prom
+  ret i64 %shr
+}
+
+define i64 @test6(i64 %y, i32 %s) {
+; CHECK-LABEL: test6:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
+; CHECK-NEXT:    lsl x0, x0, x1
+; CHECK-NEXT:    ret
+entry:
+  %sh_prom = sext i32 %s to i64
+  %shr = shl i64 %y, %sh_prom
+  ret i64 %shr
+}
diff --git a/test/CodeGen/AArch64/shift_minsize.ll b/test/CodeGen/AArch64/shift_minsize.ll
new file mode 100644
index 0000000000000000000000000000000000000000..d1b95e87577b6e55ce01ce3dcb2ef6d8f42a98ce
--- /dev/null
+++ b/test/CodeGen/AArch64/shift_minsize.ll
@@ -0,0 +1,122 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=aarch64-unknown-unknown | FileCheck %s
+
+define i64 @f0(i64 %val, i64 %amt) minsize optsize {
+; CHECK-LABEL: f0:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    lsl x0, x0, x1
+; CHECK-NEXT:    ret
+  %res = shl i64 %val, %amt
+  ret i64 %res
+}
+
+define i32 @f1(i64 %x, i64 %y) minsize optsize {
+; CHECK-LABEL: f1:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    lsl x0, x0, x1
+; CHECK-NEXT:    // kill: def $w0 killed $w0 killed $x0
+; CHECK-NEXT:    ret
+	%a = shl i64 %x, %y
+	%b = trunc i64 %a to i32
+	ret i32 %b
+}
+
+define i32 @f2(i64 %x, i64 %y) minsize optsize {
+; CHECK-LABEL: f2:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    asr x0, x0, x1
+; CHECK-NEXT:    // kill: def $w0 killed $w0 killed $x0
+; CHECK-NEXT:    ret
+	%a = ashr i64 %x, %y
+	%b = trunc i64 %a to i32
+	ret i32 %b
+}
+
+define i32 @f3(i64 %x, i64 %y) minsize optsize {
+; CHECK-LABEL: f3:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    lsr x0, x0, x1
+; CHECK-NEXT:    // kill: def $w0 killed $w0 killed $x0
+; CHECK-NEXT:    ret
+	%a = lshr i64 %x, %y
+	%b = trunc i64 %a to i32
+	ret i32 %b
+}
+
+define dso_local { i64, i64 } @shl128(i64 %x.coerce0, i64 %x.coerce1, i8 signext %y) minsize optsize {
+; CHECK-LABEL: shl128:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    .cfi_offset w30, -16
+; CHECK-NEXT:    // kill: def $w2 killed $w2 def $x2
+; CHECK-NEXT:    bl __ashlti3
+; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+entry:
+  %x.sroa.2.0.insert.ext = zext i64 %x.coerce1 to i128
+  %x.sroa.2.0.insert.shift = shl nuw i128 %x.sroa.2.0.insert.ext, 64
+  %x.sroa.0.0.insert.ext = zext i64 %x.coerce0 to i128
+  %x.sroa.0.0.insert.insert = or i128 %x.sroa.2.0.insert.shift, %x.sroa.0.0.insert.ext
+  %conv = sext i8 %y to i32
+  %sh_prom = zext i32 %conv to i128
+  %shl = shl i128 %x.sroa.0.0.insert.insert, %sh_prom
+  %retval.sroa.0.0.extract.trunc = trunc i128 %shl to i64
+  %retval.sroa.2.0.extract.shift = lshr i128 %shl, 64
+  %retval.sroa.2.0.extract.trunc = trunc i128 %retval.sroa.2.0.extract.shift to i64
+  %.fca.0.insert = insertvalue { i64, i64 } undef, i64 %retval.sroa.0.0.extract.trunc, 0
+  %.fca.1.insert = insertvalue { i64, i64 } %.fca.0.insert, i64 %retval.sroa.2.0.extract.trunc, 1
+  ret { i64, i64 } %.fca.1.insert
+}
+
+define dso_local { i64, i64 } @ashr128(i64 %x.coerce0, i64 %x.coerce1, i8 signext %y) minsize optsize {
+; CHECK-LABEL: ashr128:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    .cfi_offset w30, -16
+; CHECK-NEXT:    // kill: def $w2 killed $w2 def $x2
+; CHECK-NEXT:    bl __ashrti3
+; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+entry:
+  %x.sroa.2.0.insert.ext = zext i64 %x.coerce1 to i128
+  %x.sroa.2.0.insert.shift = shl nuw i128 %x.sroa.2.0.insert.ext, 64
+  %x.sroa.0.0.insert.ext = zext i64 %x.coerce0 to i128
+  %x.sroa.0.0.insert.insert = or i128 %x.sroa.2.0.insert.shift, %x.sroa.0.0.insert.ext
+  %conv = sext i8 %y to i32
+  %sh_prom = zext i32 %conv to i128
+  %shr = ashr i128 %x.sroa.0.0.insert.insert, %sh_prom
+  %retval.sroa.0.0.extract.trunc = trunc i128 %shr to i64
+  %retval.sroa.2.0.extract.shift = lshr i128 %shr, 64
+  %retval.sroa.2.0.extract.trunc = trunc i128 %retval.sroa.2.0.extract.shift to i64
+  %.fca.0.insert = insertvalue { i64, i64 } undef, i64 %retval.sroa.0.0.extract.trunc, 0
+  %.fca.1.insert = insertvalue { i64, i64 } %.fca.0.insert, i64 %retval.sroa.2.0.extract.trunc, 1
+  ret { i64, i64 } %.fca.1.insert
+}
+
+define dso_local { i64, i64 } @lshr128(i64 %x.coerce0, i64 %x.coerce1, i8 signext %y) minsize optsize {
+; CHECK-LABEL: lshr128:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    .cfi_offset w30, -16
+; CHECK-NEXT:    // kill: def $w2 killed $w2 def $x2
+; CHECK-NEXT:    bl __lshrti3
+; CHECK-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+entry:
+  %x.sroa.2.0.insert.ext = zext i64 %x.coerce1 to i128
+  %x.sroa.2.0.insert.shift = shl nuw i128 %x.sroa.2.0.insert.ext, 64
+  %x.sroa.0.0.insert.ext = zext i64 %x.coerce0 to i128
+  %x.sroa.0.0.insert.insert = or i128 %x.sroa.2.0.insert.shift, %x.sroa.0.0.insert.ext
+  %conv = sext i8 %y to i32
+  %sh_prom = zext i32 %conv to i128
+  %shr = lshr i128 %x.sroa.0.0.insert.insert, %sh_prom
+  %retval.sroa.0.0.extract.trunc = trunc i128 %shr to i64
+  %retval.sroa.2.0.extract.shift = lshr i128 %shr, 64
+  %retval.sroa.2.0.extract.trunc = trunc i128 %retval.sroa.2.0.extract.shift to i64
+  %.fca.0.insert = insertvalue { i64, i64 } undef, i64 %retval.sroa.0.0.extract.trunc, 0
+  %.fca.1.insert = insertvalue { i64, i64 } %.fca.0.insert, i64 %retval.sroa.2.0.extract.trunc, 1
+  ret { i64, i64 } %.fca.1.insert
+}
diff --git a/test/CodeGen/AArch64/shrink-constant-multiple-users.ll b/test/CodeGen/AArch64/shrink-constant-multiple-users.ll
new file mode 100644
index 0000000000000000000000000000000000000000..d787d3696c11013ea66781b7becc40d5a8fd73e0
--- /dev/null
+++ b/test/CodeGen/AArch64/shrink-constant-multiple-users.ll
@@ -0,0 +1,18 @@
+; RUN: llc -mtriple arm64-ios- %s -o - | FileCheck %s
+
+; Check the -8 constant is shrunk if there are multiple users of the AND instruction.
+
+; CHECK-LABEL:  _test:
+; CHECK:          and x0, x0, #0xfffffff8
+; CHECK-NEXT:     add x19, x0, #10
+; CHECK-NEXT:     bl  _user
+
+define i64 @test(i32 %a) {
+  %ext = zext i32 %a to i64
+  %v1 = and i64 %ext, -8
+  %v2 = add i64 %v1, 10
+  call void @user(i64 %v1)
+  ret i64 %v2
+}
+
+declare void @user(i64)
diff --git a/test/CodeGen/AArch64/sincospow-vector-expansion.ll b/test/CodeGen/AArch64/sincospow-vector-expansion.ll
index 22f33a83394bb495af5f85fc107c1ec5fcd2c22c..b70f9b98916af52137f58f9a08d5f4ed8c2f6b78 100644
--- a/test/CodeGen/AArch64/sincospow-vector-expansion.ll
+++ b/test/CodeGen/AArch64/sincospow-vector-expansion.ll
@@ -1,19 +1,30 @@
 ; RUN: llc -o - %s -verify-machineinstrs -mtriple=aarch64-linux-gnu -mattr=+neon | FileCheck %s
+; RUN: llc -o - %s -verify-machineinstrs -mtriple=aarch64-linux-gnu \
+; RUN:     -mattr=+neon -global-isel -global-isel-abort=2 \
+; RUN:     -pass-remarks-missed=gisel* \
+; RUN:     2>&1 | FileCheck %s  --check-prefixes=FALLBACK,GISEL
 
-
+; FALLBACK-NOT: remark{{.*}}test_cos_v2f32
 define <2 x float> @test_cos_v2f64(<2 x double> %v1) {
 ; CHECK-LABEL: test_cos_v2f64:
 ; CHECK: bl cos
 ; CHECK: bl cos
+; GISEL-LABEL: test_cos_v2f64:
+; GISEL: bl cos
+; GISEL: bl cos
   %1 = call <2 x double> @llvm.cos.v2f64(<2 x double> %v1)
   %2 = fptrunc <2 x double> %1 to <2 x float>
   ret <2 x float> %2
 }
 
+; FALLBACK-NOT: remark{{.*}}test_sin_v2f32
 define <2 x float> @test_sin_v2f64(<2 x double> %v1) {
 ; CHECK-LABEL: test_sin_v2f64:
 ; CHECK: bl sin
 ; CHECK: bl sin
+; GISEL-LABEL: test_sin_v2f64:
+; GISEL: bl sin
+; GISEL: bl sin
   %1 = call <2 x double> @llvm.sin.v2f64(<2 x double> %v1)
   %2 = fptrunc <2 x double> %1 to <2 x float>
   ret <2 x float> %2
@@ -32,18 +43,26 @@ declare <2 x double> @llvm.cos.v2f64(<2 x double>)
 declare <2 x double> @llvm.sin.v2f64(<2 x double>)
 declare <2 x double> @llvm.pow.v2f64(<2 x double>, <2 x double>)
 
+; FALLBACK-NOT: remark{{.*}}test_cos_v2f32
 define <2 x float> @test_cos_v2f32(<2 x float> %v1) {
 ; CHECK-LABEL: test_cos_v2f32:
 ; CHECK: bl cos
 ; CHECK: bl cos
+; GISEL-LABEL: test_cos_v2f32:
+; GISEL: bl cos
+; GISEL: bl cos
   %1 = call <2 x float> @llvm.cos.v2f32(<2 x float> %v1)
   ret <2 x float> %1
 }
 
+; FALLBACK-NOT: remark{{.*}}test_sin_v2f32
 define <2 x float> @test_sin_v2f32(<2 x float> %v1) {
 ; CHECK-LABEL: test_sin_v2f32:
 ; CHECK: bl sin
 ; CHECK: bl sin
+; GISEL-LABEL: test_sin_v2f32:
+; GISEL: bl sin
+; GISEL: bl sin
   %1 = call <2 x float> @llvm.sin.v2f32(<2 x float> %v1)
   ret <2 x float> %1
 }
@@ -60,22 +79,34 @@ declare <2 x float> @llvm.cos.v2f32(<2 x float>)
 declare <2 x float> @llvm.sin.v2f32(<2 x float>)
 declare <2 x float> @llvm.pow.v2f32(<2 x float>, <2 x float>)
 
+; FALLBACK-NOT: remark{{.*}}test_cos_v4f32
 define <4 x float> @test_cos_v4f32(<4 x float> %v1) {
 ; CHECK-LABEL: test_cos_v4f32:
 ; CHECK: bl cos
 ; CHECK: bl cos
 ; CHECK: bl cos
 ; CHECK: bl cos
+; GISEL-LABEL: test_cos_v4f32:
+; GISEL: bl cos
+; GISEL: bl cos
+; GISEL: bl cos
+; GISEL: bl cos
   %1 = call <4 x float> @llvm.cos.v4f32(<4 x float> %v1)
   ret <4 x float> %1
 }
 
+; FALLBACK-NOT: remark{{.*}}test_sin_v4f32
 define <4 x float> @test_sin_v4f32(<4 x float> %v1) {
 ; CHECK-LABEL: test_sin_v4f32:
 ; CHECK: bl sin
 ; CHECK: bl sin
 ; CHECK: bl sin
 ; CHECK: bl sin
+; GISEL-LABEL: test_sin_v4f32:
+; GISEL: bl sin
+; GISEL: bl sin
+; GISEL: bl sin
+; GISEL: bl sin
   %1 = call <4 x float> @llvm.sin.v4f32(<4 x float> %v1)
   ret <4 x float> %1
 }
@@ -93,4 +124,3 @@ define <4 x float> @test_pow_v4f32(<4 x float> %v1, <4 x float> %v2) {
 declare <4 x float> @llvm.cos.v4f32(<4 x float>)
 declare <4 x float> @llvm.sin.v4f32(<4 x float>)
 declare <4 x float> @llvm.pow.v4f32(<4 x float>, <4 x float>)
-
diff --git a/test/CodeGen/AArch64/speculation-hardening-loads.ll b/test/CodeGen/AArch64/speculation-hardening-loads.ll
new file mode 100644
index 0000000000000000000000000000000000000000..e90fb19a522a1fb4c06f5d0620987d7e1a462d23
--- /dev/null
+++ b/test/CodeGen/AArch64/speculation-hardening-loads.ll
@@ -0,0 +1,157 @@
+; RUN: llc < %s -verify-machineinstrs -mtriple=aarch64-none-linux-gnu | FileCheck %s --dump-input-on-failure
+
+define i128 @ldp_single_csdb(i128* %p) speculative_load_hardening {
+entry:
+  %0 = load i128, i128* %p, align 16
+  ret i128 %0
+; CHECK-LABEL: ldp_single_csdb
+; CHECK:      ldp   x8, x1, [x0]
+; CHECK-NEXT: cmp sp, #0
+; CHECK-NEXT: csetm x16, ne
+; CHECK-NEXT: and   x8, x8, x16
+; CHECK-NEXT: and   x1, x1, x16
+; CHECK-NEXT: csdb
+; CHECK-NEXT: mov [[TMPREG:x[0-9]+]], sp
+; CHECK-NEXT: and [[TMPREG]], [[TMPREG]], x16
+; CHECK-NEXT: mov x0, x8
+; CHECK-NEXT: mov sp, [[TMPREG]]
+; CHECK-NEXT: ret
+}
+
+define double @ld_double(double* %p) speculative_load_hardening {
+entry:
+  %0 = load double, double* %p, align 8
+  ret double %0
+; Checking that the address laoded from is masked for a floating point load.
+; CHECK-LABEL: ld_double
+; CHECK:      cmp sp, #0
+; CHECK-NEXT: csetm x16, ne
+; CHECK-NEXT: and   x0, x0, x16
+; CHECK-NEXT: csdb
+; CHECK-NEXT: ldr   d0, [x0]
+; CHECK-NEXT: mov [[TMPREG:x[0-9]+]], sp
+; CHECK-NEXT: and [[TMPREG]], [[TMPREG]], x16
+; CHECK-NEXT: mov sp, [[TMPREG]]
+; CHECK-NEXT: ret
+}
+
+define i32 @csdb_emitted_for_subreg_use(i64* %p, i32 %b) speculative_load_hardening {
+entry:
+  %X = load i64, i64* %p, align 8
+  %X_trunc = trunc i64 %X to i32
+  %add = add i32 %b, %X_trunc
+  %iszero = icmp eq i64 %X, 0
+  %ret = select i1 %iszero, i32 %b, i32 %add
+  ret i32 %ret
+; Checking that the address laoded from is masked for a floating point load.
+; CHECK-LABEL: csdb_emitted_for_subreg_use
+; CHECK:      ldr x8, [x0]
+; CHECK-NEXT: cmp sp, #0
+; CHECK-NEXT: csetm x16, ne
+; CHECK-NEXT: and x8, x8, x16
+; csdb instruction must occur before the add instruction with w8 as operand.
+; CHECK-NEXT: csdb
+; CHECK-NEXT: add w9, w1, w8
+; CHECK-NEXT: cmp x8, #0
+; CHECK-NEXT: csel w0, w1, w9, eq
+; CHECK-NEXT: mov [[TMPREG:x[0-9]+]], sp
+; CHECK-NEXT: and [[TMPREG]], [[TMPREG]], x16
+; CHECK-NEXT: mov sp, [[TMPREG]]
+; CHECK-NEXT: ret
+}
+
+define i64 @csdb_emitted_for_superreg_use(i32* %p, i64 %b) speculative_load_hardening {
+entry:
+  %X = load i32, i32* %p, align 4
+  %X_ext = zext i32 %X to i64
+  %add = add i64 %b, %X_ext
+  %iszero = icmp eq i32 %X, 0
+  %ret = select i1 %iszero, i64 %b, i64 %add
+  ret i64 %ret
+; Checking that the address laoded from is masked for a floating point load.
+; CHECK-LABEL: csdb_emitted_for_superreg_use
+; CHECK:      ldr w8, [x0]
+; CHECK-NEXT: cmp sp, #0
+; CHECK-NEXT: csetm x16, ne
+; CHECK-NEXT: and w8, w8, w16
+; csdb instruction must occur before the add instruction with x8 as operand.
+; CHECK-NEXT: csdb
+; CHECK-NEXT: add x9, x1, x8
+; CHECK-NEXT: cmp w8, #0
+; CHECK-NEXT: csel x0, x1, x9, eq
+; CHECK-NEXT: mov [[TMPREG:x[0-9]+]], sp
+; CHECK-NEXT: and [[TMPREG]], [[TMPREG]], x16
+; CHECK-NEXT: mov sp, [[TMPREG]]
+; CHECK-NEXT: ret
+}
+
+define i64 @no_masking_with_full_control_flow_barriers(i64 %a, i64 %b, i64* %p) speculative_load_hardening {
+; CHECK-LABEL: no_masking_with_full_control_flow_barriers
+; CHECK: dsb sy
+; CHECK: isb
+entry:
+  %0 = tail call i64 asm "autia1716", "={x17},{x16},0"(i64 %b, i64 %a)
+  %X = load i64, i64* %p, align 8
+  %ret = add i64 %X, %0
+; CHECK-NOT: csdb
+; CHECK-NOT: and
+; CHECK: ret
+  ret i64 %ret
+}
+
+define void @f_implicitdef_vector_load(<4 x i32>* %dst, <2 x i32>* %src) speculative_load_hardening
+{
+entry:
+  %0 = load <2 x i32>, <2 x i32>* %src, align 8
+  %shuffle = shufflevector <2 x i32> %0, <2 x i32> undef, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
+  store <4 x i32> %shuffle, <4 x i32>* %dst, align 4
+  ret void
+; CHECK-LABEL: f_implicitdef_vector_load
+; CHECK:       cmp     sp, #0
+; CHECK-NEXT:  csetm   x16, ne
+; CHECK-NEXT:  and     x1, x1, x16
+; CHECK-NEXT:  csdb
+; CHECK-NEXT:  ldr     d0, [x1]
+; CHECK-NEXT:  mov     v0.d[1], v0.d[0]
+; CHECK-NEXT:  str     q0, [x0]
+; CHECK-NEXT:  mov     [[TMPREG:x[0-9]+]], sp
+; CHECK-NEXT:  and     [[TMPREG]], [[TMPREG]], x16
+; CHECK-NEXT:  mov     sp, [[TMPREG]]
+; CHECK-NEXT:  ret
+}
+
+define <2 x double> @f_usedefvectorload(double* %a, double* %b) speculative_load_hardening {
+entry:
+; CHECK-LABEL: f_usedefvectorload
+; CHECK:       cmp     sp, #0
+; CHECK-NEXT:  csetm   x16, ne
+; CHECK-NEXT:  movi    v0.2d, #0000000000000000
+; CHECK-NEXT:  and     x1, x1, x16
+; CHECK-NEXT:  csdb
+; CHECK-NEXT:  ld1     { v0.d }[0], [x1]
+; CHECK-NEXT:  mov     [[TMPREG:x[0-9]+]], sp
+; CHECK-NEXT:  and     [[TMPREG]], [[TMPREG]], x16
+; CHECK-NEXT:  mov     sp, [[TMPREG]]
+; CHECK-NEXT:  ret
+  %0 = load double, double* %b, align 16
+  %vld1_lane = insertelement <2 x double> <double undef, double 0.000000e+00>, double %0, i32 0
+  ret <2 x double> %vld1_lane
+}
+
+define i32 @deadload() speculative_load_hardening {
+entry:
+; CHECK-LABEL: deadload
+; CHECK:       cmp     sp, #0
+; CHECK-NEXT:  csetm   x16, ne
+; CHECK-NEXT:  sub     sp, sp, #16
+; CHECK-NEXT:  .cfi_def_cfa_offset 16
+; CHECK-NEXT:  ldr     w8, [sp, #12]
+; CHECK-NEXT:  add     sp, sp, #16
+; CHECK-NEXT:  mov     [[TMPREG:x[0-9]+]], sp
+; CHECK-NEXT:  and     [[TMPREG]], [[TMPREG]], x16
+; CHECK-NEXT:  mov     sp, [[TMPREG]]
+; CHECK-NEXT:  ret
+  %a = alloca i32, align 4
+  %val = load volatile i32, i32* %a, align 4
+  ret i32 undef
+}
diff --git a/test/CodeGen/AArch64/speculation-hardening.ll b/test/CodeGen/AArch64/speculation-hardening.ll
index 3535b63c32cc8f11bd11845c7b68840e7ab3fcc0..51156f68dec8c93d70bf9469831935b5b4d27c38 100644
--- a/test/CodeGen/AArch64/speculation-hardening.ll
+++ b/test/CodeGen/AArch64/speculation-hardening.ll
@@ -1,9 +1,9 @@
-; RUN: sed -e 's/SLHATTR/speculative_load_hardening/' %s | llc -verify-machineinstrs -mtriple=aarch64-none-linux-gnu | FileCheck %s --check-prefixes=CHECK,SLH --dump-input-on-failure
-; RUN: sed -e 's/SLHATTR//' %s | llc -verify-machineinstrs -mtriple=aarch64-none-linux-gnu | FileCheck %s --check-prefixes=CHECK,NOSLH --dump-input-on-failure
-; RUN: sed -e 's/SLHATTR/speculative_load_hardening/' %s | llc -verify-machineinstrs -mtriple=aarch64-none-linux-gnu -global-isel | FileCheck %s --check-prefixes=CHECK,SLH --dump-input-on-failure
-; RUN sed -e 's/SLHATTR//' %s | llc -verify-machineinstrs -mtriple=aarch64-none-linux-gnu -global-isel | FileCheck %s --check-prefixes=CHECK,NOSLH --dump-input-on-failure
-; RUN: sed -e 's/SLHATTR/speculative_load_hardening/' %s | llc -verify-machineinstrs -mtriple=aarch64-none-linux-gnu -fast-isel | FileCheck %s --check-prefixes=CHECK,SLH --dump-input-on-failure
-; RUN: sed -e 's/SLHATTR//' %s | llc -verify-machineinstrs -mtriple=aarch64-none-linux-gnu -fast-isel | FileCheck %s --check-prefixes=CHECK,NOSLH --dump-input-on-failure
+; RUN: sed -e 's/SLHATTR/speculative_load_hardening/' %s | llc -verify-machineinstrs -mtriple=aarch64-none-linux-gnu | FileCheck %s --check-prefixes=CHECK,SLH,NOGISELSLH --dump-input-on-failure
+; RUN: sed -e 's/SLHATTR//' %s | llc -verify-machineinstrs -mtriple=aarch64-none-linux-gnu | FileCheck %s --check-prefixes=CHECK,NOSLH,NOGISELNOSLH --dump-input-on-failure
+; RUN: sed -e 's/SLHATTR/speculative_load_hardening/' %s | llc -verify-machineinstrs -mtriple=aarch64-none-linux-gnu -global-isel | FileCheck %s --check-prefixes=CHECK,SLH,GISELSLH --dump-input-on-failure
+; RUN sed -e 's/SLHATTR//' %s | llc -verify-machineinstrs -mtriple=aarch64-none-linux-gnu -global-isel | FileCheck %s --check-prefixes=CHECK,NOSLH,GISELNOSLH --dump-input-on-failure
+; RUN: sed -e 's/SLHATTR/speculative_load_hardening/' %s | llc -verify-machineinstrs -mtriple=aarch64-none-linux-gnu -fast-isel | FileCheck %s --check-prefixes=CHECK,SLH,NOGISELSLH --dump-input-on-failure
+; RUN: sed -e 's/SLHATTR//' %s | llc -verify-machineinstrs -mtriple=aarch64-none-linux-gnu -fast-isel | FileCheck %s --check-prefixes=CHECK,NOSLH,NOGISELNOSLH --dump-input-on-failure
 
 define i32 @f(i8* nocapture readonly %p, i32 %i, i32 %N) local_unnamed_addr SLHATTR {
 ; CHECK-LABEL: f
@@ -13,12 +13,12 @@ entry:
 ; NOSLH-NOT:  cmp sp, #0
 ; NOSLH-NOT:  csetm x16, ne
 
-; SLH:  mov x17, sp
-; SLH:  and x17, x17, x16
-; SLH:  mov sp, x17
-; NOSLH-NOT:  mov x17, sp
-; NOSLH-NOT:  and x17, x17, x16
-; NOSLH-NOT:  mov sp, x17
+; SLH:  mov [[TMPREG:x[0-9]+]], sp
+; SLH:  and [[TMPREG]], [[TMPREG]], x16
+; SLH:  mov sp, [[TMPREG]]
+; NOSLH-NOT:  mov [[TMPREG:x[0-9]+]], sp
+; NOSLH-NOT:  and [[TMPREG]], [[TMPREG]], x16
+; NOSLH-NOT:  mov sp, [[TMPREG]]
   %call = tail call i32 @tail_callee(i32 %i)
 ; SLH:  cmp sp, #0
 ; SLH:  csetm x16, ne
@@ -43,29 +43,37 @@ if.then:                                          ; preds = %entry
 ; NOSLH-NOT: csel x16, x16, xzr, [[COND]]
 return:                                           ; preds = %entry, %if.then
   %retval.0 = phi i32 [ %conv, %if.then ], [ 0, %entry ]
-; SLH:  mov x17, sp
-; SLH:  and x17, x17, x16
-; SLH:  mov sp, x17
-; NOSLH-NOT:  mov x17, sp
-; NOSLH-NOT:  and x17, x17, x16
-; NOSLH-NOT:  mov sp, x17
+; SLH:  mov [[TMPREG:x[0-9]+]], sp
+; SLH:  and [[TMPREG]], [[TMPREG]], x16
+; SLH:  mov sp, [[TMPREG]]
+; NOSLH-NOT:  mov [[TMPREG:x[0-9]+]], sp
+; NOSLH-NOT:  and [[TMPREG]], [[TMPREG]], x16
+; NOSLH-NOT:  mov sp, [[TMPREG]]
   ret i32 %retval.0
 }
 
 ; Make sure that for a tail call, taint doesn't get put into SP twice.
 define i32 @tail_caller(i32 %a) local_unnamed_addr SLHATTR {
 ; CHECK-LABEL: tail_caller:
-; SLH:     mov     x17, sp
-; SLH:     and     x17, x17, x16
-; SLH:     mov     sp, x17
-; NOSLH-NOT:     mov     x17, sp
-; NOSLH-NOT:     and     x17, x17, x16
-; NOSLH-NOT:     mov     sp, x17
+; NOGISELSLH:     mov [[TMPREG:x[0-9]+]], sp
+; NOGISELSLH:     and [[TMPREG]], [[TMPREG]], x16
+; NOGISELSLH:     mov sp, [[TMPREG]]
+; NOGISELNOSLH-NOT:     mov [[TMPREG:x[0-9]+]], sp
+; NOGISELNOSLH-NOT:     and [[TMPREG]], [[TMPREG]], x16
+; NOGISELNOSLH-NOT:     mov sp, [[TMPREG]]
+; GISELSLH:     mov [[TMPREG:x[0-9]+]], sp
+; GISELSLH:     and [[TMPREG]], [[TMPREG]], x16
+; GISELSLH:     mov sp, [[TMPREG]]
+; GISELNOSLH-NOT:     mov [[TMPREG:x[0-9]+]], sp
+; GISELNOSLH-NOT:     and [[TMPREG]], [[TMPREG]], x16
+; GISELNOSLH-NOT:     mov sp, [[TMPREG]]
 ;  GlobalISel doesn't optimize tail calls (yet?), so only check that
 ;  cross-call taint register setup code is missing if a tail call was
 ;  actually produced.
-; SLH:     {{(bl tail_callee[[:space:]] cmp sp, #0)|(b tail_callee)}}
-; SLH-NOT: cmp sp, #0
+; NOGISELSLH:     b tail_callee
+; GISELSLH:       bl tail_callee
+; GISELSLH:       cmp sp, #0
+; SLH-NOT:        cmp sp, #0
   %call = tail call i32 @tail_callee(i32 %a)
   ret i32 %call
 }
diff --git a/test/CodeGen/AArch64/speculation-hardening.mir b/test/CodeGen/AArch64/speculation-hardening.mir
index cf8357d9558b0064b54c9f618e97411ff830bde9..5991c4df0407f7ec701d89cc0608962cbcf45f9f 100644
--- a/test/CodeGen/AArch64/speculation-hardening.mir
+++ b/test/CodeGen/AArch64/speculation-hardening.mir
@@ -25,6 +25,22 @@
   define void @indirectbranch(i32 %a, i32 %b) speculative_load_hardening {
    ret void
   }
+  ; Also check that a non-default temporary register gets picked correctly to
+  ; transfer the SP to to and it with the taint register when the default
+  ; temporary isn't available.
+  define void @indirect_call_x17(i32 %a, i32 %b) speculative_load_hardening {
+   ret void
+  }
+  @g = common dso_local local_unnamed_addr global i64 (...)* null, align 8
+  define void @indirect_tailcall_x17(i32 %a, i32 %b) speculative_load_hardening {
+   ret void
+  }
+  define void @indirect_call_lr(i32 %a, i32 %b) speculative_load_hardening {
+   ret void
+  }
+  define void @RS_cannot_find_available_regs() speculative_load_hardening {
+   ret void
+  }
 ...
 ---
 name:            nobranch_fallthrough
@@ -115,3 +131,72 @@ body:             |
   ; CHECK-NOT: csel
    RET undef $lr, implicit $x0
 ...
+---
+name:            indirect_call_x17
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $x17
+    ; CHECK-LABEL: indirect_call_x17
+    ; CHECK:       mov x0, sp
+    ; CHECK:       and x0, x0, x16
+    ; CHECK:       mov sp, x0
+    ; CHECK:       blr x17
+    BLR killed renamable $x17, implicit-def dead $lr, implicit $sp
+    RET undef $lr, implicit undef $w0
+...
+---
+name:           indirect_tailcall_x17
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $x0
+    ; CHECK-LABEL: indirect_tailcall_x17
+    ; CHECK:       mov x1, sp
+    ; CHECK:       and x1, x1, x16
+    ; CHECK:       mov sp, x1
+    ; CHECK:       br x17
+    $x8 = ADRP target-flags(aarch64-page) @g
+    $x17 = LDRXui killed $x8, target-flags(aarch64-pageoff, aarch64-nc) @g
+    TCRETURNri killed $x17, 0, implicit $sp, implicit $x0
+...
+---
+name:           indirect_call_lr
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    ; CHECK-LABEL: indirect_call_lr
+    ; CHECK:            mov x1, sp
+    ; CHECK-NEXT:       and x1, x1, x16
+    ; CHECK-NEXT:       mov sp, x1
+    ; CHECK-NEXT:       blr x30
+    liveins: $x0, $lr
+    BLR killed renamable $lr, implicit-def dead $lr, implicit $sp, implicit-def $sp, implicit-def $w0
+    $w0 = nsw ADDWri killed $w0, 1, 0
+    RET undef $lr, implicit $w0
+...
+---
+name:           RS_cannot_find_available_regs
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    ; In the rare case when no free temporary register is available for the
+    ; propagate taint-to-sp operation, just put in a full speculation barrier
+    ; (isb+dsb sy) at the start of the basic block. And don't put masks on
+    ; instructions for the rest of the basic block, since speculation in that
+    ; basic block was already done, so no need to do masking.
+    ; CHECK-LABEL: RS_cannot_find_available_regs
+    ; CHECK:       dsb sy
+    ; CHECK-NEXT:  isb
+    ; CHECK-NEXT:  ldr x0, [x0]
+    ; The following 2 instructions come from propagating the taint encoded in
+    ; sp at function entry to x16. It turns out the taint info in x16 is not
+    ; used in this function, so those instructions could be optimized away. An
+    ; optimization for later if it turns out this situation occurs often enough.
+    ; CHECK-NEXT:  cmp sp, #0
+    ; CHECK-NEXT:  csetm x16, ne
+    ; CHECK-NEXT:  ret
+    liveins: $x0, $x1, $x2, $x3, $x4, $x5, $x6, $x7, $x8, $x9, $x10, $x11, $x12, $x13, $x14, $x15, $x17, $x18, $x19, $x20, $x21, $x22, $x23, $x24, $x25, $x26, $x27, $x28, $fp, $lr
+     $x0 = LDRXui killed $x0, 0
+     RET undef $lr, implicit $x0
+...
diff --git a/test/CodeGen/AArch64/sponentry.ll b/test/CodeGen/AArch64/sponentry.ll
index 5b3638a1d862a82766a5ad41be1c924fa144b2aa..8bd995f22fc04e02dd80a00d397fc6d7e796e2fe 100644
--- a/test/CodeGen/AArch64/sponentry.ll
+++ b/test/CodeGen/AArch64/sponentry.ll
@@ -1,5 +1,5 @@
-; RUN: llc -mtriple=aarch64-windows-msvc -disable-fp-elim %s -o - | FileCheck %s
-; RUN: llc -mtriple=aarch64-windows-msvc -fast-isel -disable-fp-elim %s -o - | FileCheck %s
+; RUN: llc -mtriple=aarch64-windows-msvc -frame-pointer=all %s -o - | FileCheck %s
+; RUN: llc -mtriple=aarch64-windows-msvc -fast-isel -frame-pointer=all %s -o - | FileCheck %s
 ; RUN: llc -mtriple=aarch64-windows-msvc %s -o - | FileCheck %s --check-prefix=NOFP
 ; RUN: llc -mtriple=aarch64-windows-msvc -fast-isel %s -o - | FileCheck %s --check-prefix=NOFP
 
diff --git a/test/CodeGen/AArch64/stack-guard-remat-bitcast.ll b/test/CodeGen/AArch64/stack-guard-remat-bitcast.ll
index 8f1a7b41c010042a772c30eae68ca545b5be0b09..946d1cc3171645a848a9a1844f982b3fca9c67bc 100644
--- a/test/CodeGen/AArch64/stack-guard-remat-bitcast.ll
+++ b/test/CodeGen/AArch64/stack-guard-remat-bitcast.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -mtriple=arm64-apple-ios -relocation-model=pic -disable-fp-elim | FileCheck %s
+; RUN: llc < %s -mtriple=arm64-apple-ios -relocation-model=pic -frame-pointer=all | FileCheck %s
 
 @__stack_chk_guard = external global i64*
 
diff --git a/test/CodeGen/AArch64/stack_guard_remat.ll b/test/CodeGen/AArch64/stack_guard_remat.ll
index 2b7b3485311aee240e1e8f386e3fde32bf86f1be..3a87a37316ba0e124dbc372150547e0ba56e6e16 100644
--- a/test/CodeGen/AArch64/stack_guard_remat.ll
+++ b/test/CodeGen/AArch64/stack_guard_remat.ll
@@ -4,6 +4,12 @@
 ; RUN: llc < %s -mtriple=aarch64-linux-gnu -relocation-model=static -code-model=large -no-integrated-as | FileCheck %s -check-prefix=STATIC-LARGE
 ; RUN: llc < %s -mtriple=aarch64-linux-gnu -relocation-model=static -code-model=small -no-integrated-as | FileCheck %s -check-prefix=STATIC-SMALL
 
+; RUN: llc < %s -global-isel -global-isel-abort=2 -pass-remarks-missed=gisel* -mtriple=arm64-apple-ios -relocation-model=pic -no-integrated-as 2>&1 | FileCheck %s -check-prefixes=DARWIN,FALLBACK
+; RUN: llc < %s -global-isel -global-isel-abort=2 -pass-remarks-missed=gisel* -mtriple=arm64-apple-ios -relocation-model=static -no-integrated-as 2>&1 | FileCheck %s -check-prefixes=DARWIN,FALLBACK
+; RUN: llc < %s -global-isel -global-isel-abort=2 -pass-remarks-missed=gisel* -mtriple=aarch64-linux-gnu -relocation-model=pic -no-integrated-as 2>&1 | FileCheck %s -check-prefixes=PIC-LINUX,FALLBACK
+; RUN: llc < %s -global-isel -global-isel-abort=2 -pass-remarks-missed=gisel* -mtriple=aarch64-linux-gnu -relocation-model=static -code-model=large -no-integrated-as 2>&1 | FileCheck %s -check-prefixes=STATIC-LARGE,FALLBACK
+; RUN: llc < %s -global-isel -global-isel-abort=2 -pass-remarks-missed=gisel* -mtriple=aarch64-linux-gnu -relocation-model=static -code-model=small -no-integrated-as 2>&1 | FileCheck %s -check-prefixes=STATIC-SMALL,FALLBACK
+
 ; DARWIN: foo2
 ; DARWIN: adrp [[R0:x[0-9]+]], ___stack_chk_guard@GOTPAGE
 ; DARWIN: ldr [[R1:x[0-9]+]], {{\[}}[[R0]], ___stack_chk_guard@GOTPAGEOFF{{\]}}
@@ -25,6 +31,8 @@
 ; STATIC-SMALL: adrp [[R0:x[0-9]+]], __stack_chk_guard
 ; STATIC-SMALL: ldr {{x[0-9]+}}, {{\[}}[[R0]], :lo12:__stack_chk_guard{{\]}}
 
+; FALLBACK-NOT: remark:{{.*}}llvm.lifetime.end
+; FALLBACK-NOT: remark:{{.*}}llvm.lifetime.start
 define i32 @test_stack_guard_remat() #0 {
 entry:
   %a1 = alloca [256 x i32], align 4
diff --git a/test/CodeGen/AArch64/swifterror.ll b/test/CodeGen/AArch64/swifterror.ll
index 8ea89464ab0e9107bba35f260f754ed5e22e9e8f..3c3ab607df4b52d8c8cec1e0d880a8b213efcb0d 100644
--- a/test/CodeGen/AArch64/swifterror.ll
+++ b/test/CodeGen/AArch64/swifterror.ll
@@ -1,5 +1,5 @@
-; RUN: llc -fast-isel-sink-local-values -verify-machineinstrs -disable-fp-elim -enable-shrink-wrap=false < %s -mtriple=aarch64-apple-ios -disable-post-ra | FileCheck -allow-deprecated-dag-overlap --check-prefix=CHECK-APPLE %s
-; RUN: llc -fast-isel-sink-local-values -verify-machineinstrs -disable-fp-elim -O0 -fast-isel < %s -mtriple=aarch64-apple-ios -disable-post-ra | FileCheck -allow-deprecated-dag-overlap --check-prefix=CHECK-O0 %s
+; RUN: llc -fast-isel-sink-local-values -verify-machineinstrs -frame-pointer=all -enable-shrink-wrap=false < %s -mtriple=aarch64-apple-ios -disable-post-ra | FileCheck -allow-deprecated-dag-overlap --check-prefix=CHECK-APPLE %s
+; RUN: llc -fast-isel-sink-local-values -verify-machineinstrs -frame-pointer=all -O0 -fast-isel < %s -mtriple=aarch64-apple-ios -disable-post-ra | FileCheck -allow-deprecated-dag-overlap --check-prefix=CHECK-O0 %s
 
 declare i8* @malloc(i64)
 declare void @free(i8*)
diff --git a/test/CodeGen/AArch64/tail-call-unused-zext.ll b/test/CodeGen/AArch64/tail-call-unused-zext.ll
new file mode 100644
index 0000000000000000000000000000000000000000..1617d13f7e110c4089b1eafc39767a69e58ff42f
--- /dev/null
+++ b/test/CodeGen/AArch64/tail-call-unused-zext.ll
@@ -0,0 +1,36 @@
+; RUN: llc -mtriple=arm64--- -stop-after=expand-isel-pseudos -o - %s | FileCheck %s
+
+; Check that we ignore the zeroext attribute on the return type of the tail
+; call, since the return value is unused. This happens during CodeGenPrepare in
+; dupRetToEnableTailCallOpts, which calls attributesPermitTailCall to check if
+; the attributes of the caller and the callee match.
+
+declare zeroext i1 @zcallee()
+define void @zcaller() {
+; CHECK-LABEL: name: zcaller
+entry:
+  br i1 undef, label %calllabel, label %retlabel
+calllabel:
+; CHECK: bb.1.calllabel:
+; CHECK-NOT: BL @zcallee
+; CHECK-NEXT: TCRETURNdi @zcallee
+  %unused_result = tail call zeroext i1 @zcallee()
+  br label %retlabel
+retlabel:
+  ret void
+}
+
+declare signext i1 @scallee()
+define void @scaller() {
+; CHECK-LABEL: name: scaller
+entry:
+  br i1 undef, label %calllabel, label %retlabel
+calllabel:
+; CHECK: bb.1.calllabel:
+; CHECK-NOT: BL @scallee
+; CHECK-NEXT: TCRETURNdi @scallee
+  %unused_result = tail call signext i1 @scallee()
+  br label %retlabel
+retlabel:
+  ret void
+}
diff --git a/test/CodeGen/AArch64/urem-seteq-vec-splat.ll b/test/CodeGen/AArch64/urem-seteq-vec-splat.ll
index 2a34d085fc1e66b5042398156c03a48229401a28..6f28581d59e61b32010dc95a024c54b7e9b74aa3 100644
--- a/test/CodeGen/AArch64/urem-seteq-vec-splat.ll
+++ b/test/CodeGen/AArch64/urem-seteq-vec-splat.ll
@@ -25,45 +25,28 @@ define <4 x i32> @test_urem_odd_vec_i32(<4 x i32> %X) nounwind readnone {
   ret <4 x i32> %ret
 }
 
-; Like test_urem_odd_vec_i32, but with 4 x i16 vectors.
-define <4 x i16> @test_urem_odd_vec_i16(<4 x i16> %X) nounwind readnone {
+; Like test_urem_odd_vec_i32, but with 8 x i16 vectors.
+define <8 x i16> @test_urem_odd_vec_i16(<8 x i16> %X) nounwind readnone {
 ; CHECK-LABEL: test_urem_odd_vec_i16:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w9, #52429
-; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT:    umov w8, v0.h[1]
-; CHECK-NEXT:    movk w9, #52428, lsl #16
-; CHECK-NEXT:    umull x12, w8, w9
-; CHECK-NEXT:    lsr x12, x12, #34
-; CHECK-NEXT:    umov w10, v0.h[0]
-; CHECK-NEXT:    add w12, w12, w12, lsl #2
-; CHECK-NEXT:    sub w8, w8, w12
-; CHECK-NEXT:    umull x12, w10, w9
-; CHECK-NEXT:    lsr x12, x12, #34
-; CHECK-NEXT:    umov w11, v0.h[2]
-; CHECK-NEXT:    add w12, w12, w12, lsl #2
-; CHECK-NEXT:    sub w10, w10, w12
-; CHECK-NEXT:    umull x12, w11, w9
-; CHECK-NEXT:    lsr x12, x12, #34
-; CHECK-NEXT:    add w12, w12, w12, lsl #2
-; CHECK-NEXT:    sub w11, w11, w12
-; CHECK-NEXT:    umov w12, v0.h[3]
-; CHECK-NEXT:    umull x9, w12, w9
-; CHECK-NEXT:    lsr x9, x9, #34
-; CHECK-NEXT:    fmov s0, w10
-; CHECK-NEXT:    add w9, w9, w9, lsl #2
-; CHECK-NEXT:    mov v0.h[1], w8
-; CHECK-NEXT:    sub w9, w12, w9
-; CHECK-NEXT:    mov v0.h[2], w11
-; CHECK-NEXT:    mov v0.h[3], w9
-; CHECK-NEXT:    cmeq v0.4h, v0.4h, #0
-; CHECK-NEXT:    movi v1.4h, #1
-; CHECK-NEXT:    and v0.8b, v0.8b, v1.8b
+; CHECK-NEXT:    mov w8, #52429
+; CHECK-NEXT:    dup v2.8h, w8
+; CHECK-NEXT:    umull2 v3.4s, v0.8h, v2.8h
+; CHECK-NEXT:    umull v2.4s, v0.4h, v2.4h
+; CHECK-NEXT:    uzp2 v2.8h, v2.8h, v3.8h
+; CHECK-NEXT:    movi v1.8h, #5
+; CHECK-NEXT:    ushr v2.8h, v2.8h, #2
+; CHECK-NEXT:    mls v0.8h, v2.8h, v1.8h
+; CHECK-NEXT:    cmeq v0.8h, v0.8h, #0
+; CHECK-NEXT:    movi v1.8h, #1
+; CHECK-NEXT:    and v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    ret
-  %urem = urem <4 x i16> %X, <i16 5, i16 5, i16 5, i16 5>
-  %cmp = icmp eq <4 x i16> %urem, <i16 0, i16 0, i16 0, i16 0>
-  %ret = zext <4 x i1> %cmp to <4 x i16>
-  ret <4 x i16> %ret
+  %urem = urem <8 x i16> %X, <i16 5, i16 5, i16 5, i16 5,
+                              i16 5, i16 5, i16 5, i16 5>
+  %cmp = icmp eq <8 x i16> %urem, <i16 0, i16 0, i16 0, i16 0,
+                                   i16 0, i16 0, i16 0, i16 0>
+  %ret = zext <8 x i1> %cmp to <8 x i16>
+  ret <8 x i16> %ret
 }
 
 ; Tests BuildUREMEqFold for 4 x i32 splat vectors with even divisor.
@@ -92,48 +75,31 @@ define <4 x i32> @test_urem_even_vec_i32(<4 x i32> %X) nounwind readnone {
   ret <4 x i32> %ret
 }
 
-; Like test_urem_even_vec_i32, but with 4 x i16 vectors.
+; Like test_urem_even_vec_i32, but with 8 x i16 vectors.
 ; i16 is not legal for ROTR on AArch64, but ROTR also cannot be promoted to i32,
 ; so this would crash if BuildUREMEqFold was applied.
-define <4 x i16> @test_urem_even_vec_i16(<4 x i16> %X) nounwind readnone {
+define <8 x i16> @test_urem_even_vec_i16(<8 x i16> %X) nounwind readnone {
 ; CHECK-LABEL: test_urem_even_vec_i16:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT:    umov w8, v0.h[1]
-; CHECK-NEXT:    mov w9, #9363
-; CHECK-NEXT:    movk w9, #37449, lsl #16
-; CHECK-NEXT:    umov w10, v0.h[0]
-; CHECK-NEXT:    umov w11, v0.h[2]
-; CHECK-NEXT:    umov w12, v0.h[3]
-; CHECK-NEXT:    ubfx w13, w8, #1, #15
-; CHECK-NEXT:    ubfx w14, w10, #1, #15
-; CHECK-NEXT:    ubfx w15, w11, #1, #15
-; CHECK-NEXT:    ubfx w16, w12, #1, #15
-; CHECK-NEXT:    umull x13, w13, w9
-; CHECK-NEXT:    umull x14, w14, w9
-; CHECK-NEXT:    umull x15, w15, w9
-; CHECK-NEXT:    umull x9, w16, w9
-; CHECK-NEXT:    orr w16, wzr, #0xe
-; CHECK-NEXT:    lsr x13, x13, #34
-; CHECK-NEXT:    msub w8, w13, w16, w8
-; CHECK-NEXT:    lsr x13, x14, #34
-; CHECK-NEXT:    msub w10, w13, w16, w10
-; CHECK-NEXT:    lsr x13, x15, #34
-; CHECK-NEXT:    fmov s0, w10
-; CHECK-NEXT:    msub w11, w13, w16, w11
-; CHECK-NEXT:    lsr x9, x9, #34
-; CHECK-NEXT:    mov v0.h[1], w8
-; CHECK-NEXT:    msub w9, w9, w16, w12
-; CHECK-NEXT:    mov v0.h[2], w11
-; CHECK-NEXT:    mov v0.h[3], w9
-; CHECK-NEXT:    cmeq v0.4h, v0.4h, #0
-; CHECK-NEXT:    movi v1.4h, #1
-; CHECK-NEXT:    and v0.8b, v0.8b, v1.8b
+; CHECK-NEXT:    mov w8, #18725
+; CHECK-NEXT:    ushr v1.8h, v0.8h, #1
+; CHECK-NEXT:    dup v3.8h, w8
+; CHECK-NEXT:    umull2 v4.4s, v1.8h, v3.8h
+; CHECK-NEXT:    umull v1.4s, v1.4h, v3.4h
+; CHECK-NEXT:    uzp2 v1.8h, v1.8h, v4.8h
+; CHECK-NEXT:    movi v2.8h, #14
+; CHECK-NEXT:    ushr v1.8h, v1.8h, #1
+; CHECK-NEXT:    mls v0.8h, v1.8h, v2.8h
+; CHECK-NEXT:    cmeq v0.8h, v0.8h, #0
+; CHECK-NEXT:    movi v1.8h, #1
+; CHECK-NEXT:    and v0.16b, v0.16b, v1.16b
 ; CHECK-NEXT:    ret
-  %urem = urem <4 x i16> %X, <i16 14, i16 14, i16 14, i16 14>
-  %cmp = icmp eq <4 x i16> %urem, <i16 0, i16 0, i16 0, i16 0>
-  %ret = zext <4 x i1> %cmp to <4 x i16>
-  ret <4 x i16> %ret
+  %urem = urem <8 x i16> %X, <i16 14, i16 14, i16 14, i16 14,
+                              i16 14, i16 14, i16 14, i16 14>
+  %cmp = icmp eq <8 x i16> %urem, <i16 0, i16 0, i16 0, i16 0,
+                                   i16 0, i16 0, i16 0, i16 0>
+  %ret = zext <8 x i1> %cmp to <8 x i16>
+  ret <8 x i16> %ret
 }
 
 ; We should not proceed with this fold if the divisor is 1 or -1
diff --git a/test/CodeGen/AArch64/vec_cttz.ll b/test/CodeGen/AArch64/vec_cttz.ll
new file mode 100644
index 0000000000000000000000000000000000000000..68efa0de9f8dfe0fbc4c042997e8860747cf90e6
--- /dev/null
+++ b/test/CodeGen/AArch64/vec_cttz.ll
@@ -0,0 +1,124 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=aarch64-none-linux-gnu -mattr=+neon | FileCheck %s --check-prefix=CHECK
+
+declare <8 x i8> @llvm.cttz.v8i8(<8 x i8>, i1)
+declare <4 x i16> @llvm.cttz.v4i16(<4 x i16>, i1)
+declare <2 x i32> @llvm.cttz.v2i32(<2 x i32>, i1)
+declare <1 x i64> @llvm.cttz.v1i64(<1 x i64>, i1)
+
+declare <16 x i8> @llvm.cttz.v16i8(<16 x i8>, i1)
+declare <8 x i16> @llvm.cttz.v8i16(<8 x i16>, i1)
+declare <4 x i32> @llvm.cttz.v4i32(<4 x i32>, i1)
+declare <2 x i64> @llvm.cttz.v2i64(<2 x i64>, i1)
+
+define <8 x i8> @cttz_v8i8(<8 x i8> %a) nounwind {
+; CHECK-LABEL: cttz_v8i8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    movi v1.8b, #1
+; CHECK-NEXT:    sub v1.8b, v0.8b, v1.8b
+; CHECK-NEXT:    bic v0.8b, v1.8b, v0.8b
+; CHECK-NEXT:    cnt v0.8b, v0.8b
+; CHECK-NEXT:    ret
+    %b = call <8 x i8> @llvm.cttz.v8i8(<8 x i8> %a, i1 true)
+    ret <8 x i8> %b
+}
+
+define <4 x i16> @cttz_v4i16(<4 x i16> %a) nounwind {
+; CHECK-LABEL: cttz_v4i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    movi v1.4h, #1
+; CHECK-NEXT:    sub v1.4h, v0.4h, v1.4h
+; CHECK-NEXT:    bic v0.8b, v1.8b, v0.8b
+; CHECK-NEXT:    clz v0.4h, v0.4h
+; CHECK-NEXT:    movi v1.4h, #16
+; CHECK-NEXT:    sub v0.4h, v1.4h, v0.4h
+; CHECK-NEXT:    ret
+    %b = call <4 x i16> @llvm.cttz.v4i16(<4 x i16> %a, i1 true)
+    ret <4 x i16> %b
+}
+
+define <2 x i32> @cttz_v2i32(<2 x i32> %a) nounwind {
+; CHECK-LABEL: cttz_v2i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    movi v1.2s, #1
+; CHECK-NEXT:    sub v1.2s, v0.2s, v1.2s
+; CHECK-NEXT:    bic v0.8b, v1.8b, v0.8b
+; CHECK-NEXT:    clz v0.2s, v0.2s
+; CHECK-NEXT:    movi v1.2s, #32
+; CHECK-NEXT:    sub v0.2s, v1.2s, v0.2s
+; CHECK-NEXT:    ret
+    %b = call <2 x i32> @llvm.cttz.v2i32(<2 x i32> %a, i1 true)
+    ret <2 x i32> %b
+}
+
+define <1 x i64> @cttz_v1i64(<1 x i64> %a) nounwind {
+; CHECK-LABEL: cttz_v1i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    orr w8, wzr, #0x1
+; CHECK-NEXT:    fmov d1, x8
+; CHECK-NEXT:    sub d1, d0, d1
+; CHECK-NEXT:    bic v0.8b, v1.8b, v0.8b
+; CHECK-NEXT:    cnt v0.8b, v0.8b
+; CHECK-NEXT:    uaddlp v0.4h, v0.8b
+; CHECK-NEXT:    uaddlp v0.2s, v0.4h
+; CHECK-NEXT:    uaddlp v0.1d, v0.2s
+; CHECK-NEXT:    ret
+    %b = call <1 x i64> @llvm.cttz.v1i64(<1 x i64> %a, i1 true)
+    ret <1 x i64> %b
+}
+
+define <16 x i8> @cttz_v16i8(<16 x i8> %a) nounwind {
+; CHECK-LABEL: cttz_v16i8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    movi v1.16b, #1
+; CHECK-NEXT:    sub v1.16b, v0.16b, v1.16b
+; CHECK-NEXT:    bic v0.16b, v1.16b, v0.16b
+; CHECK-NEXT:    cnt v0.16b, v0.16b
+; CHECK-NEXT:    ret
+    %b = call <16 x i8> @llvm.cttz.v16i8(<16 x i8> %a, i1 true)
+    ret <16 x i8> %b
+}
+
+define <8 x i16> @cttz_v8i16(<8 x i16> %a) nounwind {
+; CHECK-LABEL: cttz_v8i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    movi v1.8h, #1
+; CHECK-NEXT:    sub v1.8h, v0.8h, v1.8h
+; CHECK-NEXT:    bic v0.16b, v1.16b, v0.16b
+; CHECK-NEXT:    clz v0.8h, v0.8h
+; CHECK-NEXT:    movi v1.8h, #16
+; CHECK-NEXT:    sub v0.8h, v1.8h, v0.8h
+; CHECK-NEXT:    ret
+    %b = call <8 x i16> @llvm.cttz.v8i16(<8 x i16> %a, i1 true)
+    ret <8 x i16> %b
+}
+
+define <4 x i32> @cttz_v4i32(<4 x i32> %a) nounwind {
+; CHECK-LABEL: cttz_v4i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    movi v1.4s, #1
+; CHECK-NEXT:    sub v1.4s, v0.4s, v1.4s
+; CHECK-NEXT:    bic v0.16b, v1.16b, v0.16b
+; CHECK-NEXT:    clz v0.4s, v0.4s
+; CHECK-NEXT:    movi v1.4s, #32
+; CHECK-NEXT:    sub v0.4s, v1.4s, v0.4s
+; CHECK-NEXT:    ret
+    %b = call <4 x i32> @llvm.cttz.v4i32(<4 x i32> %a, i1 true)
+    ret <4 x i32> %b
+}
+
+define <2 x i64> @cttz_v2i64(<2 x i64> %a) nounwind {
+; CHECK-LABEL: cttz_v2i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    orr w8, wzr, #0x1
+; CHECK-NEXT:    dup v1.2d, x8
+; CHECK-NEXT:    sub v1.2d, v0.2d, v1.2d
+; CHECK-NEXT:    bic v0.16b, v1.16b, v0.16b
+; CHECK-NEXT:    cnt v0.16b, v0.16b
+; CHECK-NEXT:    uaddlp v0.8h, v0.16b
+; CHECK-NEXT:    uaddlp v0.4s, v0.8h
+; CHECK-NEXT:    uaddlp v0.2d, v0.4s
+; CHECK-NEXT:    ret
+    %b = call <2 x i64> @llvm.cttz.v2i64(<2 x i64> %a, i1 true)
+    ret <2 x i64> %b
+}
diff --git a/test/CodeGen/AArch64/vec_uaddo.ll b/test/CodeGen/AArch64/vec_uaddo.ll
new file mode 100644
index 0000000000000000000000000000000000000000..6c2f07f8dc9bc6d5f0737caa2eba2ba042cad86a
--- /dev/null
+++ b/test/CodeGen/AArch64/vec_uaddo.ll
@@ -0,0 +1,319 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=aarch64-none-linux-gnu -mattr=+neon | FileCheck %s --check-prefix=CHECK
+
+declare {<1 x i32>, <1 x i1>} @llvm.uadd.with.overflow.v1i32(<1 x i32>, <1 x i32>)
+declare {<2 x i32>, <2 x i1>} @llvm.uadd.with.overflow.v2i32(<2 x i32>, <2 x i32>)
+declare {<3 x i32>, <3 x i1>} @llvm.uadd.with.overflow.v3i32(<3 x i32>, <3 x i32>)
+declare {<4 x i32>, <4 x i1>} @llvm.uadd.with.overflow.v4i32(<4 x i32>, <4 x i32>)
+declare {<6 x i32>, <6 x i1>} @llvm.uadd.with.overflow.v6i32(<6 x i32>, <6 x i32>)
+declare {<8 x i32>, <8 x i1>} @llvm.uadd.with.overflow.v8i32(<8 x i32>, <8 x i32>)
+
+declare {<16 x i8>, <16 x i1>} @llvm.uadd.with.overflow.v16i8(<16 x i8>, <16 x i8>)
+declare {<8 x i16>, <8 x i1>} @llvm.uadd.with.overflow.v8i16(<8 x i16>, <8 x i16>)
+declare {<2 x i64>, <2 x i1>} @llvm.uadd.with.overflow.v2i64(<2 x i64>, <2 x i64>)
+
+declare {<4 x i24>, <4 x i1>} @llvm.uadd.with.overflow.v4i24(<4 x i24>, <4 x i24>)
+declare {<4 x i1>, <4 x i1>} @llvm.uadd.with.overflow.v4i1(<4 x i1>, <4 x i1>)
+declare {<2 x i128>, <2 x i1>} @llvm.uadd.with.overflow.v2i128(<2 x i128>, <2 x i128>)
+
+define <1 x i32> @uaddo_v1i32(<1 x i32> %a0, <1 x i32> %a1, <1 x i32>* %p2) nounwind {
+; CHECK-LABEL: uaddo_v1i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    add v1.2s, v0.2s, v1.2s
+; CHECK-NEXT:    cmhi v0.2s, v0.2s, v1.2s
+; CHECK-NEXT:    str s1, [x0]
+; CHECK-NEXT:    ret
+  %t = call {<1 x i32>, <1 x i1>} @llvm.uadd.with.overflow.v1i32(<1 x i32> %a0, <1 x i32> %a1)
+  %val = extractvalue {<1 x i32>, <1 x i1>} %t, 0
+  %obit = extractvalue {<1 x i32>, <1 x i1>} %t, 1
+  %res = sext <1 x i1> %obit to <1 x i32>
+  store <1 x i32> %val, <1 x i32>* %p2
+  ret <1 x i32> %res
+}
+
+define <2 x i32> @uaddo_v2i32(<2 x i32> %a0, <2 x i32> %a1, <2 x i32>* %p2) nounwind {
+; CHECK-LABEL: uaddo_v2i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    add v1.2s, v0.2s, v1.2s
+; CHECK-NEXT:    cmhi v0.2s, v0.2s, v1.2s
+; CHECK-NEXT:    str d1, [x0]
+; CHECK-NEXT:    ret
+  %t = call {<2 x i32>, <2 x i1>} @llvm.uadd.with.overflow.v2i32(<2 x i32> %a0, <2 x i32> %a1)
+  %val = extractvalue {<2 x i32>, <2 x i1>} %t, 0
+  %obit = extractvalue {<2 x i32>, <2 x i1>} %t, 1
+  %res = sext <2 x i1> %obit to <2 x i32>
+  store <2 x i32> %val, <2 x i32>* %p2
+  ret <2 x i32> %res
+}
+
+define <3 x i32> @uaddo_v3i32(<3 x i32> %a0, <3 x i32> %a1, <3 x i32>* %p2) nounwind {
+; CHECK-LABEL: uaddo_v3i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    add v1.4s, v0.4s, v1.4s
+; CHECK-NEXT:    cmhi v0.4s, v0.4s, v1.4s
+; CHECK-NEXT:    xtn v0.4h, v0.4s
+; CHECK-NEXT:    add x8, x0, #8 // =8
+; CHECK-NEXT:    sshll v0.4s, v0.4h, #0
+; CHECK-NEXT:    st1 { v1.s }[2], [x8]
+; CHECK-NEXT:    str d1, [x0]
+; CHECK-NEXT:    ret
+  %t = call {<3 x i32>, <3 x i1>} @llvm.uadd.with.overflow.v3i32(<3 x i32> %a0, <3 x i32> %a1)
+  %val = extractvalue {<3 x i32>, <3 x i1>} %t, 0
+  %obit = extractvalue {<3 x i32>, <3 x i1>} %t, 1
+  %res = sext <3 x i1> %obit to <3 x i32>
+  store <3 x i32> %val, <3 x i32>* %p2
+  ret <3 x i32> %res
+}
+
+define <4 x i32> @uaddo_v4i32(<4 x i32> %a0, <4 x i32> %a1, <4 x i32>* %p2) nounwind {
+; CHECK-LABEL: uaddo_v4i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    add v1.4s, v0.4s, v1.4s
+; CHECK-NEXT:    cmhi v0.4s, v0.4s, v1.4s
+; CHECK-NEXT:    xtn v0.4h, v0.4s
+; CHECK-NEXT:    sshll v0.4s, v0.4h, #0
+; CHECK-NEXT:    str q1, [x0]
+; CHECK-NEXT:    ret
+  %t = call {<4 x i32>, <4 x i1>} @llvm.uadd.with.overflow.v4i32(<4 x i32> %a0, <4 x i32> %a1)
+  %val = extractvalue {<4 x i32>, <4 x i1>} %t, 0
+  %obit = extractvalue {<4 x i32>, <4 x i1>} %t, 1
+  %res = sext <4 x i1> %obit to <4 x i32>
+  store <4 x i32> %val, <4 x i32>* %p2
+  ret <4 x i32> %res
+}
+
+define <6 x i32> @uaddo_v6i32(<6 x i32> %a0, <6 x i32> %a1, <6 x i32>* %p2) nounwind {
+; CHECK-LABEL: uaddo_v6i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fmov s0, w6
+; CHECK-NEXT:    mov x8, sp
+; CHECK-NEXT:    mov v0.s[1], w7
+; CHECK-NEXT:    ldr s2, [sp, #16]
+; CHECK-NEXT:    ld1 { v0.s }[2], [x8]
+; CHECK-NEXT:    add x9, sp, #8 // =8
+; CHECK-NEXT:    add x10, sp, #24 // =24
+; CHECK-NEXT:    fmov s1, w0
+; CHECK-NEXT:    ld1 { v2.s }[1], [x10]
+; CHECK-NEXT:    ld1 { v0.s }[3], [x9]
+; CHECK-NEXT:    mov v1.s[1], w1
+; CHECK-NEXT:    fmov s3, w4
+; CHECK-NEXT:    ldr x11, [sp, #32]
+; CHECK-NEXT:    mov v1.s[2], w2
+; CHECK-NEXT:    mov v3.s[1], w5
+; CHECK-NEXT:    mov v1.s[3], w3
+; CHECK-NEXT:    add v2.4s, v3.4s, v2.4s
+; CHECK-NEXT:    add v0.4s, v1.4s, v0.4s
+; CHECK-NEXT:    cmhi v3.4s, v3.4s, v2.4s
+; CHECK-NEXT:    cmhi v1.4s, v1.4s, v0.4s
+; CHECK-NEXT:    str d2, [x11, #16]
+; CHECK-NEXT:    xtn v2.4h, v3.4s
+; CHECK-NEXT:    xtn v1.4h, v1.4s
+; CHECK-NEXT:    sshll v2.4s, v2.4h, #0
+; CHECK-NEXT:    sshll v1.4s, v1.4h, #0
+; CHECK-NEXT:    mov w5, v2.s[1]
+; CHECK-NEXT:    mov w1, v1.s[1]
+; CHECK-NEXT:    mov w2, v1.s[2]
+; CHECK-NEXT:    mov w3, v1.s[3]
+; CHECK-NEXT:    fmov w4, s2
+; CHECK-NEXT:    fmov w0, s1
+; CHECK-NEXT:    str q0, [x11]
+; CHECK-NEXT:    ret
+  %t = call {<6 x i32>, <6 x i1>} @llvm.uadd.with.overflow.v6i32(<6 x i32> %a0, <6 x i32> %a1)
+  %val = extractvalue {<6 x i32>, <6 x i1>} %t, 0
+  %obit = extractvalue {<6 x i32>, <6 x i1>} %t, 1
+  %res = sext <6 x i1> %obit to <6 x i32>
+  store <6 x i32> %val, <6 x i32>* %p2
+  ret <6 x i32> %res
+}
+
+define <8 x i32> @uaddo_v8i32(<8 x i32> %a0, <8 x i32> %a1, <8 x i32>* %p2) nounwind {
+; CHECK-LABEL: uaddo_v8i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    add v3.4s, v1.4s, v3.4s
+; CHECK-NEXT:    add v2.4s, v0.4s, v2.4s
+; CHECK-NEXT:    cmhi v1.4s, v1.4s, v3.4s
+; CHECK-NEXT:    cmhi v0.4s, v0.4s, v2.4s
+; CHECK-NEXT:    xtn v1.4h, v1.4s
+; CHECK-NEXT:    xtn v0.4h, v0.4s
+; CHECK-NEXT:    sshll v0.4s, v0.4h, #0
+; CHECK-NEXT:    sshll v1.4s, v1.4h, #0
+; CHECK-NEXT:    stp q2, q3, [x0]
+; CHECK-NEXT:    ret
+  %t = call {<8 x i32>, <8 x i1>} @llvm.uadd.with.overflow.v8i32(<8 x i32> %a0, <8 x i32> %a1)
+  %val = extractvalue {<8 x i32>, <8 x i1>} %t, 0
+  %obit = extractvalue {<8 x i32>, <8 x i1>} %t, 1
+  %res = sext <8 x i1> %obit to <8 x i32>
+  store <8 x i32> %val, <8 x i32>* %p2
+  ret <8 x i32> %res
+}
+
+define <16 x i32> @uaddo_v16i8(<16 x i8> %a0, <16 x i8> %a1, <16 x i8>* %p2) nounwind {
+; CHECK-LABEL: uaddo_v16i8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    add v4.16b, v0.16b, v1.16b
+; CHECK-NEXT:    cmhi v0.16b, v0.16b, v4.16b
+; CHECK-NEXT:    zip1 v1.8b, v0.8b, v0.8b
+; CHECK-NEXT:    zip2 v2.8b, v0.8b, v0.8b
+; CHECK-NEXT:    ext v0.16b, v0.16b, v0.16b, #8
+; CHECK-NEXT:    ushll v1.4s, v1.4h, #0
+; CHECK-NEXT:    ushll v2.4s, v2.4h, #0
+; CHECK-NEXT:    zip1 v3.8b, v0.8b, v0.8b
+; CHECK-NEXT:    zip2 v0.8b, v0.8b, v0.8b
+; CHECK-NEXT:    shl v1.4s, v1.4s, #31
+; CHECK-NEXT:    shl v2.4s, v2.4s, #31
+; CHECK-NEXT:    ushll v3.4s, v3.4h, #0
+; CHECK-NEXT:    ushll v5.4s, v0.4h, #0
+; CHECK-NEXT:    sshr v0.4s, v1.4s, #31
+; CHECK-NEXT:    sshr v1.4s, v2.4s, #31
+; CHECK-NEXT:    shl v2.4s, v3.4s, #31
+; CHECK-NEXT:    shl v3.4s, v5.4s, #31
+; CHECK-NEXT:    sshr v2.4s, v2.4s, #31
+; CHECK-NEXT:    sshr v3.4s, v3.4s, #31
+; CHECK-NEXT:    str q4, [x0]
+; CHECK-NEXT:    ret
+  %t = call {<16 x i8>, <16 x i1>} @llvm.uadd.with.overflow.v16i8(<16 x i8> %a0, <16 x i8> %a1)
+  %val = extractvalue {<16 x i8>, <16 x i1>} %t, 0
+  %obit = extractvalue {<16 x i8>, <16 x i1>} %t, 1
+  %res = sext <16 x i1> %obit to <16 x i32>
+  store <16 x i8> %val, <16 x i8>* %p2
+  ret <16 x i32> %res
+}
+
+define <8 x i32> @uaddo_v8i16(<8 x i16> %a0, <8 x i16> %a1, <8 x i16>* %p2) nounwind {
+; CHECK-LABEL: uaddo_v8i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    add v2.8h, v0.8h, v1.8h
+; CHECK-NEXT:    cmhi v0.8h, v0.8h, v2.8h
+; CHECK-NEXT:    xtn v0.8b, v0.8h
+; CHECK-NEXT:    zip1 v1.8b, v0.8b, v0.8b
+; CHECK-NEXT:    zip2 v0.8b, v0.8b, v0.8b
+; CHECK-NEXT:    ushll v1.4s, v1.4h, #0
+; CHECK-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-NEXT:    shl v1.4s, v1.4s, #31
+; CHECK-NEXT:    shl v3.4s, v0.4s, #31
+; CHECK-NEXT:    sshr v0.4s, v1.4s, #31
+; CHECK-NEXT:    sshr v1.4s, v3.4s, #31
+; CHECK-NEXT:    str q2, [x0]
+; CHECK-NEXT:    ret
+  %t = call {<8 x i16>, <8 x i1>} @llvm.uadd.with.overflow.v8i16(<8 x i16> %a0, <8 x i16> %a1)
+  %val = extractvalue {<8 x i16>, <8 x i1>} %t, 0
+  %obit = extractvalue {<8 x i16>, <8 x i1>} %t, 1
+  %res = sext <8 x i1> %obit to <8 x i32>
+  store <8 x i16> %val, <8 x i16>* %p2
+  ret <8 x i32> %res
+}
+
+define <2 x i32> @uaddo_v2i64(<2 x i64> %a0, <2 x i64> %a1, <2 x i64>* %p2) nounwind {
+; CHECK-LABEL: uaddo_v2i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    add v1.2d, v0.2d, v1.2d
+; CHECK-NEXT:    cmhi v0.2d, v0.2d, v1.2d
+; CHECK-NEXT:    xtn v0.2s, v0.2d
+; CHECK-NEXT:    str q1, [x0]
+; CHECK-NEXT:    ret
+  %t = call {<2 x i64>, <2 x i1>} @llvm.uadd.with.overflow.v2i64(<2 x i64> %a0, <2 x i64> %a1)
+  %val = extractvalue {<2 x i64>, <2 x i1>} %t, 0
+  %obit = extractvalue {<2 x i64>, <2 x i1>} %t, 1
+  %res = sext <2 x i1> %obit to <2 x i32>
+  store <2 x i64> %val, <2 x i64>* %p2
+  ret <2 x i32> %res
+}
+
+define <4 x i32> @uaddo_v4i24(<4 x i24> %a0, <4 x i24> %a1, <4 x i24>* %p2) nounwind {
+; CHECK-LABEL: uaddo_v4i24:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    bic v1.4s, #255, lsl #24
+; CHECK-NEXT:    bic v0.4s, #255, lsl #24
+; CHECK-NEXT:    add v0.4s, v0.4s, v1.4s
+; CHECK-NEXT:    mov v1.16b, v0.16b
+; CHECK-NEXT:    mov w8, v0.s[3]
+; CHECK-NEXT:    bic v1.4s, #255, lsl #24
+; CHECK-NEXT:    mov w9, v0.s[2]
+; CHECK-NEXT:    mov w10, v0.s[1]
+; CHECK-NEXT:    sturh w8, [x0, #9]
+; CHECK-NEXT:    lsr w8, w8, #16
+; CHECK-NEXT:    cmeq v1.4s, v1.4s, v0.4s
+; CHECK-NEXT:    fmov w11, s0
+; CHECK-NEXT:    strh w9, [x0, #6]
+; CHECK-NEXT:    sturh w10, [x0, #3]
+; CHECK-NEXT:    lsr w9, w9, #16
+; CHECK-NEXT:    lsr w10, w10, #16
+; CHECK-NEXT:    strb w8, [x0, #11]
+; CHECK-NEXT:    mvn v0.16b, v1.16b
+; CHECK-NEXT:    lsr w8, w11, #16
+; CHECK-NEXT:    strh w11, [x0]
+; CHECK-NEXT:    strb w9, [x0, #8]
+; CHECK-NEXT:    strb w10, [x0, #5]
+; CHECK-NEXT:    strb w8, [x0, #2]
+; CHECK-NEXT:    ret
+  %t = call {<4 x i24>, <4 x i1>} @llvm.uadd.with.overflow.v4i24(<4 x i24> %a0, <4 x i24> %a1)
+  %val = extractvalue {<4 x i24>, <4 x i1>} %t, 0
+  %obit = extractvalue {<4 x i24>, <4 x i1>} %t, 1
+  %res = sext <4 x i1> %obit to <4 x i32>
+  store <4 x i24> %val, <4 x i24>* %p2
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @uaddo_v4i1(<4 x i1> %a0, <4 x i1> %a1, <4 x i1>* %p2) nounwind {
+; CHECK-LABEL: uaddo_v4i1:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    movi v2.4h, #1
+; CHECK-NEXT:    and v1.8b, v1.8b, v2.8b
+; CHECK-NEXT:    and v0.8b, v0.8b, v2.8b
+; CHECK-NEXT:    add v1.4h, v0.4h, v1.4h
+; CHECK-NEXT:    umov w9, v1.h[1]
+; CHECK-NEXT:    umov w8, v1.h[0]
+; CHECK-NEXT:    and w9, w9, #0x1
+; CHECK-NEXT:    bfi w8, w9, #1, #1
+; CHECK-NEXT:    umov w9, v1.h[2]
+; CHECK-NEXT:    and v0.8b, v1.8b, v2.8b
+; CHECK-NEXT:    and w9, w9, #0x1
+; CHECK-NEXT:    cmeq v0.4h, v0.4h, v1.4h
+; CHECK-NEXT:    bfi w8, w9, #2, #1
+; CHECK-NEXT:    umov w9, v1.h[3]
+; CHECK-NEXT:    mvn v0.8b, v0.8b
+; CHECK-NEXT:    bfi w8, w9, #3, #29
+; CHECK-NEXT:    sshll v0.4s, v0.4h, #0
+; CHECK-NEXT:    and w8, w8, #0xf
+; CHECK-NEXT:    strb w8, [x0]
+; CHECK-NEXT:    ret
+  %t = call {<4 x i1>, <4 x i1>} @llvm.uadd.with.overflow.v4i1(<4 x i1> %a0, <4 x i1> %a1)
+  %val = extractvalue {<4 x i1>, <4 x i1>} %t, 0
+  %obit = extractvalue {<4 x i1>, <4 x i1>} %t, 1
+  %res = sext <4 x i1> %obit to <4 x i32>
+  store <4 x i1> %val, <4 x i1>* %p2
+  ret <4 x i32> %res
+}
+
+define <2 x i32> @uaddo_v2i128(<2 x i128> %a0, <2 x i128> %a1, <2 x i128>* %p2) nounwind {
+; CHECK-LABEL: uaddo_v2i128:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    adds x9, x2, x6
+; CHECK-NEXT:    adcs x10, x3, x7
+; CHECK-NEXT:    cmp x9, x2
+; CHECK-NEXT:    cset w11, lo
+; CHECK-NEXT:    cmp x10, x3
+; CHECK-NEXT:    cset w12, lo
+; CHECK-NEXT:    csel w11, w11, w12, eq
+; CHECK-NEXT:    adds x12, x0, x4
+; CHECK-NEXT:    adcs x13, x1, x5
+; CHECK-NEXT:    cmp x12, x0
+; CHECK-NEXT:    cset w14, lo
+; CHECK-NEXT:    cmp x13, x1
+; CHECK-NEXT:    cset w15, lo
+; CHECK-NEXT:    csel w14, w14, w15, eq
+; CHECK-NEXT:    ldr x8, [sp]
+; CHECK-NEXT:    fmov s0, w14
+; CHECK-NEXT:    mov v0.s[1], w11
+; CHECK-NEXT:    shl v0.2s, v0.2s, #31
+; CHECK-NEXT:    sshr v0.2s, v0.2s, #31
+; CHECK-NEXT:    stp x9, x10, [x8, #16]
+; CHECK-NEXT:    stp x12, x13, [x8]
+; CHECK-NEXT:    ret
+  %t = call {<2 x i128>, <2 x i1>} @llvm.uadd.with.overflow.v2i128(<2 x i128> %a0, <2 x i128> %a1)
+  %val = extractvalue {<2 x i128>, <2 x i1>} %t, 0
+  %obit = extractvalue {<2 x i128>, <2 x i1>} %t, 1
+  %res = sext <2 x i1> %obit to <2 x i32>
+  store <2 x i128> %val, <2 x i128>* %p2
+  ret <2 x i32> %res
+}
diff --git a/test/CodeGen/AArch64/win64-jumptable.ll b/test/CodeGen/AArch64/win64-jumptable.ll
new file mode 100644
index 0000000000000000000000000000000000000000..8148a593c91b78278cfaf41a53c1507d14eddf0a
--- /dev/null
+++ b/test/CodeGen/AArch64/win64-jumptable.ll
@@ -0,0 +1,48 @@
+; RUN: llc -o - %s -mtriple=aarch64-windows -aarch64-enable-compress-jump-tables=0 | FileCheck %s
+
+define void @f(i32 %x) {
+entry:
+  switch i32 %x, label %sw.epilog [
+    i32 0, label %sw.bb
+    i32 1, label %sw.bb1
+    i32 2, label %sw.bb2
+    i32 3, label %sw.bb3
+  ]
+
+sw.bb:                                            ; preds = %entry
+  tail call void @g(i32 0) #2
+  br label %sw.epilog
+
+sw.bb1:                                           ; preds = %entry
+  tail call void @g(i32 1) #2
+  br label %sw.epilog
+
+sw.bb2:                                           ; preds = %entry
+  tail call void @g(i32 2) #2
+  br label %sw.epilog
+
+sw.bb3:                                           ; preds = %entry
+  tail call void @g(i32 3) #2
+  br label %sw.epilog
+
+sw.epilog:                                        ; preds = %entry, %sw.bb3, %sw.bb2, %sw.bb1, %sw.bb
+  tail call void @g(i32 10) #2
+  ret void
+}
+
+declare void @g(i32)
+
+; CHECK:		.text
+; CHECK:		f:
+; CHECK:		.seh_proc f
+; CHECK:		b	g
+; CHECK-NEXT:	.p2align	2
+; CHECK-NEXT:	.LJTI0_0:
+; CHECK:		.word	.LBB0_2-.LJTI0_0
+; CHECK:		.word	.LBB0_3-.LJTI0_0
+; CHECK:		.word	.LBB0_4-.LJTI0_0
+; CHECK:		.word	.LBB0_5-.LJTI0_0
+; CHECK:		.section	.xdata,"dr"
+; CHECK:		.seh_handlerdata
+; CHECK:		.text
+; CHECK:		.seh_endproc
diff --git a/test/CodeGen/AArch64/win_cst_pool.ll b/test/CodeGen/AArch64/win_cst_pool.ll
index 5bcc9194c7962fe423b1695b3faf1cb4a94d7eaf..f26e7aab3e52c19cb6cca922fef2926dbf371ccc 100644
--- a/test/CodeGen/AArch64/win_cst_pool.ll
+++ b/test/CodeGen/AArch64/win_cst_pool.ll
@@ -2,22 +2,22 @@
 ; RUN: llc < %s -mtriple=aarch64-win32-gnu | FileCheck -check-prefix=MINGW %s
 
 define double @double() {
-  ret double 0x0000000000800000
+  ret double 0x0000000000800001
 }
-; CHECK:              .globl  __real@0000000000800000
-; CHECK-NEXT:         .section        .rdata,"dr",discard,__real@0000000000800000
+; CHECK:              .globl  __real@0000000000800001
+; CHECK-NEXT:         .section        .rdata,"dr",discard,__real@0000000000800001
 ; CHECK-NEXT:         .p2align  3
-; CHECK-NEXT: __real@0000000000800000:
-; CHECK-NEXT:         .xword   8388608
+; CHECK-NEXT: __real@0000000000800001:
+; CHECK-NEXT:         .xword   8388609
 ; CHECK:      double:
-; CHECK:               adrp    x8, __real@0000000000800000
-; CHECK-NEXT:          ldr     d0, [x8, __real@0000000000800000]
+; CHECK:               adrp    x8, __real@0000000000800001
+; CHECK-NEXT:          ldr     d0, [x8, __real@0000000000800001]
 ; CHECK-NEXT:          ret
 
 ; MINGW:              .section        .rdata,"dr"
 ; MINGW-NEXT:         .p2align  3
 ; MINGW-NEXT: [[LABEL:\.LC.*]]:
-; MINGW-NEXT:         .xword   8388608
+; MINGW-NEXT:         .xword   8388609
 ; MINGW:      double:
 ; MINGW:               adrp    x8, [[LABEL]]
 ; MINGW-NEXT:          ldr     d0, [x8, [[LABEL]]]
diff --git a/test/CodeGen/AArch64/wineh4.mir b/test/CodeGen/AArch64/wineh4.mir
index 4d4cc892c2e82f7809a569c7a04842f103191e9a..3a324324431c7c41ee607a53f96fe7e750a8f084 100644
--- a/test/CodeGen/AArch64/wineh4.mir
+++ b/test/CodeGen/AArch64/wineh4.mir
@@ -1,7 +1,7 @@
 # RUN: llc -o - %s -mtriple=aarch64-windows -start-after=prologepilog \
 # RUN:   -disable-branch-fold  -filetype=obj \
 # RUN: | llvm-readobj -unwind | FileCheck %s
-# Check that multiple epilgoues are correctly placed in .xdata.
+# Check that identical multiple epilgoues are correctly shared in .xdata.
 
 # CHECK:        ExceptionData {
 # CHECK-NEXT:      FunctionLength: 164
@@ -9,7 +9,7 @@
 # CHECK-NEXT:      ExceptionData: No
 # CHECK-NEXT:      EpiloguePacked: No
 # CHECK-NEXT:      EpilogueScopes: 2
-# CHECK-NEXT:      ByteCodeLength: 48
+# CHECK-NEXT:      ByteCodeLength: 32
 # CHECK-NEXT:      Prologue [
 # CHECK-NEXT:        0xc80c              ; stp x19, x20, [sp, #96]
 # CHECK-NEXT:        0xc88a              ; stp x21, x22, [sp, #80]
@@ -37,7 +37,7 @@
 # CHECK-NEXT:        }
 # CHECK-NEXT:        EpilogueScope {
 # CHECK-NEXT:          StartOffset: 33
-# CHECK-NEXT:          EpilogueStartIndex: 30
+# CHECK-NEXT:          EpilogueStartIndex: 15
 # CHECK-NEXT:          Opcodes [
 # CHECK-NEXT:            0xc80c              ; ldp x19, x20, [sp, #96]
 # CHECK-NEXT:            0xc88a              ; ldp x21, x22, [sp, #80]
diff --git a/test/CodeGen/AArch64/wineh8.mir b/test/CodeGen/AArch64/wineh8.mir
new file mode 100644
index 0000000000000000000000000000000000000000..606bc140b232dc24b1e3e1923dd483c29d84cac6
--- /dev/null
+++ b/test/CodeGen/AArch64/wineh8.mir
@@ -0,0 +1,225 @@
+# RUN: llc -o - %s -mtriple=aarch64-windows -start-after=prologepilog \
+# RUN:   -disable-branch-fold  -filetype=obj \
+# RUN: | llvm-readobj -unwind | FileCheck %s
+# Check that non-identical multiple epilgoues are correctly shared in .xdata.
+
+# CHECK:        ExceptionData {
+# CHECK-NEXT:      FunctionLength: 160
+# CHECK-NEXT:      Version: 0
+# CHECK-NEXT:      ExceptionData: No
+# CHECK-NEXT:      EpiloguePacked: No
+# CHECK-NEXT:      EpilogueScopes: 2
+# CHECK-NEXT:      ByteCodeLength: 44
+# CHECK-NEXT:      Prologue [
+# CHECK-NEXT:        0xc80c              ; stp x19, x20, [sp, #96]
+# CHECK-NEXT:        0xc88a              ; stp x21, x22, [sp, #80]
+# CHECK-NEXT:        0xc908              ; stp x23, x24, [sp, #64]
+# CHECK-NEXT:        0xc986              ; stp x25, x26, [sp, #48]
+# CHECK-NEXT:        0xca04              ; stp x27, x28, [sp, #32]
+# CHECK-NEXT:        0xd802              ; stp d8, d9, [sp, #16]
+# CHECK-NEXT:        0xda8d              ; stp d10, d11, [sp, #-112]!
+# CHECK-NEXT:        0xe4                ; end
+# CHECK-NEXT:      ]
+# CHECK-NEXT:      EpilogueScopes [
+# CHECK-NEXT:        EpilogueScope {
+# CHECK-NEXT:          StartOffset: 16
+# CHECK-NEXT:          EpilogueStartIndex: 15
+# CHECK-NEXT:          Opcodes [
+# CHECK-NEXT:            0xc80c              ; ldp x19, x20, [sp, #96]
+# CHECK-NEXT:            0xc88a              ; ldp x21, x22, [sp, #80]
+# CHECK-NEXT:            0xc908              ; ldp x23, x24, [sp, #64]
+# CHECK-NEXT:            0xc986              ; ldp x25, x26, [sp, #48]
+# CHECK-NEXT:            0xd802              ; ldp d8, d9, [sp, #16]
+# CHECK-NEXT:            0xda8d              ; ldp d10, d11, [sp], #112
+# CHECK-NEXT:            0xe4                ; end
+# CHECK-NEXT:          ]
+# CHECK-NEXT:        }
+# CHECK-NEXT:        EpilogueScope {
+# CHECK-NEXT:          StartOffset: 32
+# CHECK-NEXT:          EpilogueStartIndex: 28
+# CHECK-NEXT:          Opcodes [
+# CHECK-NEXT:            0xc80c              ; ldp x19, x20, [sp, #96]
+# CHECK-NEXT:            0xc88a              ; ldp x21, x22, [sp, #80]
+# CHECK-NEXT:            0xc908              ; ldp x23, x24, [sp, #64]
+# CHECK-NEXT:            0xc986              ; ldp x25, x26, [sp, #48]
+# CHECK-NEXT:            0xca04              ; ldp x27, x28, [sp, #32]
+# CHECK-NEXT:            0xd802              ; ldp d8, d9, [sp, #16]
+# CHECK-NEXT:            0xda8d              ; ldp d10, d11, [sp], #112
+# CHECK-NEXT:            0xe4                ; end
+# CHECK-NEXT:          ]
+# CHECK-NEXT:        }
+# CHECK-NEXT:      ]
+# CHECK-NEXT:    }
+...
+---
+name:            test
+alignment:       2
+exposesReturnsTwice: false
+legalized:       false
+regBankSelected: false
+selected:        false
+failedISel:      false
+tracksRegLiveness: true
+hasWinCFI:       true
+registers:
+liveins:
+  - { reg: '$w0', virtual-reg: '' }
+frameInfo:
+  isFrameAddressTaken: false
+  isReturnAddressTaken: false
+  hasStackMap:     false
+  hasPatchPoint:   false
+  stackSize:       112
+  offsetAdjustment: 0
+  maxAlignment:    8
+  adjustsStack:    false
+  hasCalls:        false
+  stackProtector:  ''
+  maxCallFrameSize: 0
+  hasOpaqueSPAdjustment: true
+  hasVAStart:      false
+  hasMustTailInVarArgFunc: false
+  localFrameSize:  0
+  savePoint:       ''
+  restorePoint:    ''
+fixedStack:
+stack:
+  - { id: 0, name: '', type: spill-slot, offset: -8, size: 8, alignment: 8,
+      stack-id: 0, callee-saved-register: '$x19', callee-saved-restored: true,
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+  - { id: 1, name: '', type: spill-slot, offset: -16, size: 8, alignment: 8,
+      stack-id: 0, callee-saved-register: '$x20', callee-saved-restored: true,
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+  - { id: 2, name: '', type: spill-slot, offset: -24, size: 8, alignment: 8,
+      stack-id: 0, callee-saved-register: '$x21', callee-saved-restored: true,
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+  - { id: 3, name: '', type: spill-slot, offset: -32, size: 8, alignment: 8,
+      stack-id: 0, callee-saved-register: '$x22', callee-saved-restored: true,
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+  - { id: 4, name: '', type: spill-slot, offset: -40, size: 8, alignment: 8,
+      stack-id: 0, callee-saved-register: '$x23', callee-saved-restored: true,
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+  - { id: 5, name: '', type: spill-slot, offset: -48, size: 8, alignment: 8,
+      stack-id: 0, callee-saved-register: '$x24', callee-saved-restored: true,
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+  - { id: 6, name: '', type: spill-slot, offset: -56, size: 8, alignment: 8,
+      stack-id: 0, callee-saved-register: '$x25', callee-saved-restored: true,
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+  - { id: 7, name: '', type: spill-slot, offset: -64, size: 8, alignment: 8,
+      stack-id: 0, callee-saved-register: '$x26', callee-saved-restored: true,
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+  - { id: 8, name: '', type: spill-slot, offset: -72, size: 8, alignment: 8,
+      stack-id: 0, callee-saved-register: '$x27', callee-saved-restored: true,
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+  - { id: 9, name: '', type: spill-slot, offset: -80, size: 8, alignment: 8,
+      stack-id: 0, callee-saved-register: '$x28', callee-saved-restored: true,
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+  - { id: 10, name: '', type: spill-slot, offset: -88, size: 8, alignment: 8,
+      stack-id: 0, callee-saved-register: '$d8', callee-saved-restored: true,
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+  - { id: 11, name: '', type: spill-slot, offset: -96, size: 8, alignment: 8,
+      stack-id: 0, callee-saved-register: '$d9', callee-saved-restored: true,
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+  - { id: 12, name: '', type: spill-slot, offset: -104, size: 8, alignment: 8,
+      stack-id: 0, callee-saved-register: '$d10', callee-saved-restored: true,
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+  - { id: 13, name: '', type: spill-slot, offset: -112, size: 8, alignment: 8,
+      stack-id: 0, callee-saved-register: '$d11', callee-saved-restored: true,
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+constants:
+body:             |
+  bb.0.entry:
+    successors: %bb.2(0x40000000), %bb.1(0x40000000)
+    liveins: $x0, $x1, $d0, $d1, $d10, $d11, $d8, $d9, $x27, $x28, $x25, $x26, $x23, $x24, $x21, $x22, $x19, $x20
+
+    early-clobber $sp = frame-setup STPDpre killed $d10, killed $d11, $sp, -14 :: (store 8 into %stack.12), (store 8 into %stack.13)
+    frame-setup SEH_SaveFRegP_X 10, 11, -112
+    frame-setup STPDi killed $d8, killed $d9, $sp, 2 :: (store 8 into %stack.10), (store 8 into %stack.11)
+    frame-setup SEH_SaveFRegP 8, 9, 16
+    frame-setup STPXi killed $x27, killed $x28, $sp, 4 :: (store 8 into %stack.8), (store 8 into %stack.9)
+    frame-setup SEH_SaveRegP 27, 28, 32
+    frame-setup STPXi killed $x25, killed $x26, $sp, 6 :: (store 8 into %stack.6), (store 8 into %stack.7)
+    frame-setup SEH_SaveRegP 25, 26, 48
+    frame-setup STPXi killed $x23, killed $x24, $sp, 8 :: (store 8 into %stack.4), (store 8 into %stack.5)
+    frame-setup SEH_SaveRegP 23, 24, 64
+    frame-setup STPXi killed $x21, killed $x22, $sp, 10 :: (store 8 into %stack.2), (store 8 into %stack.3)
+    frame-setup SEH_SaveRegP 21, 22, 80
+    frame-setup STPXi killed $x19, killed $x20, $sp, 12 :: (store 8 into %stack.0), (store 8 into %stack.1)
+    frame-setup SEH_SaveRegP 19, 20, 96
+    frame-setup SEH_PrologEnd
+    frame-setup CFI_INSTRUCTION def_cfa_offset 112
+    frame-setup CFI_INSTRUCTION offset $w19, -8
+    frame-setup CFI_INSTRUCTION offset $w20, -16
+    frame-setup CFI_INSTRUCTION offset $w21, -24
+    frame-setup CFI_INSTRUCTION offset $w22, -32
+    frame-setup CFI_INSTRUCTION offset $w23, -40
+    frame-setup CFI_INSTRUCTION offset $w24, -48
+    frame-setup CFI_INSTRUCTION offset $w25, -56
+    frame-setup CFI_INSTRUCTION offset $w26, -64
+    frame-setup CFI_INSTRUCTION offset $w27, -72
+    frame-setup CFI_INSTRUCTION offset $w28, -80
+    frame-setup CFI_INSTRUCTION offset $b8, -88
+    frame-setup CFI_INSTRUCTION offset $b9, -96
+    frame-setup CFI_INSTRUCTION offset $b10, -104
+    frame-setup CFI_INSTRUCTION offset $b11, -112
+    $x19 = ADDXrr $x0, killed $x1
+    $d8 = FADDDrr killed $d0, $d1
+    $d9 = FADDDrr $d8, $d1
+    $d10 = FADDDrr $d9, $d8
+    $d11 = FADDDrr killed $d9, $d10
+    $x20 = SUBSXrr $x19, killed $x0, implicit-def $nzcv
+    Bcc 1, %bb.2, implicit killed $nzcv
+    B %bb.1
+
+  bb.1:
+    liveins: $x19, $x20
+
+    $x21 = ADDXrr $x20, killed $x19
+    $x22 = ADDXrr $x21, killed $x20
+    $x23 = ADDXrr $x22, killed $x21
+    $x24 = ADDXrr $x23, killed $x22
+    $x25 = ADDXrr $x24, killed $x23
+    $x26 = ADDXrr $x25, killed $x24
+    $x27 = ADDXrr $x26, killed $x25
+    $x28 = ADDXrr $x27, killed $x26
+    $x0 = COPY $x28
+    frame-destroy SEH_EpilogStart
+    $x19, $x20 = frame-destroy LDPXi $sp, 12 :: (load 8 from %stack.0), (load 8 from %stack.1)
+    frame-destroy SEH_SaveRegP 19, 20, 96
+    $x21, $x22 = frame-destroy LDPXi $sp, 10 :: (load 8 from %stack.2), (load 8 from %stack.3)
+    frame-destroy SEH_SaveRegP 21, 22, 80
+    $x23, $x24 = frame-destroy LDPXi $sp, 8 :: (load 8 from %stack.4), (load 8 from %stack.5)
+    frame-destroy SEH_SaveRegP 23, 24, 64
+    $x25, $x26 = frame-destroy LDPXi $sp, 6 :: (load 8 from %stack.6), (load 8 from %stack.7)
+    frame-destroy SEH_SaveRegP 25, 26, 48
+    $x27, $x28 = frame-destroy LDPXi $sp, 4 :: (load 8 from %stack.8), (load 8 from %stack.9)
+    frame-destroy SEH_SaveRegP 27, 28, 32
+    $d8, $d9 = frame-destroy LDPDi $sp, 2 :: (load 8 from %stack.10), (load 8 from %stack.11)
+    frame-destroy SEH_SaveFRegP 8, 9, 16
+    early-clobber $sp, $d10, $d11 = frame-destroy LDPDpost $sp, 14 :: (load 8 from %stack.12), (load 8 from %stack.13)
+    frame-destroy SEH_SaveFRegP_X 10, 11, -112
+    frame-destroy SEH_EpilogEnd
+    RET_ReallyLR implicit $x0
+
+  bb.2:
+    liveins: $x28, $d11
+
+    $x0 = COPY $d11
+    $x0 = ADDXrr $x0, killed $x28
+    frame-destroy SEH_EpilogStart
+    $x19, $x20 = frame-destroy LDPXi $sp, 12 :: (load 8 from %stack.0), (load 8 from %stack.1)
+    frame-destroy SEH_SaveRegP 19, 20, 96
+    $x21, $x22 = frame-destroy LDPXi $sp, 10 :: (load 8 from %stack.2), (load 8 from %stack.3)
+    frame-destroy SEH_SaveRegP 21, 22, 80
+    $x23, $x24 = frame-destroy LDPXi $sp, 8 :: (load 8 from %stack.4), (load 8 from %stack.5)
+    frame-destroy SEH_SaveRegP 23, 24, 64
+    $x25, $x26 = frame-destroy LDPXi $sp, 6 :: (load 8 from %stack.6), (load 8 from %stack.7)
+    frame-destroy SEH_SaveRegP 25, 26, 48
+    $d8, $d9 = frame-destroy LDPDi $sp, 2 :: (load 8 from %stack.10), (load 8 from %stack.11)
+    frame-destroy SEH_SaveFRegP 8, 9, 16
+    early-clobber $sp, $d10, $d11 = frame-destroy LDPDpost $sp, 14 :: (load 8 from %stack.12), (load 8 from %stack.13)
+    frame-destroy SEH_SaveFRegP_X 10, 11, -112
+    frame-destroy SEH_EpilogEnd
+    RET_ReallyLR implicit $x0
+
+...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/artifact-combiner-anyext.mir b/test/CodeGen/AMDGPU/GlobalISel/artifact-combiner-anyext.mir
new file mode 100644
index 0000000000000000000000000000000000000000..a2bac737d3764699deaa82cfde4ee7b3753e3c46
--- /dev/null
+++ b/test/CodeGen/AMDGPU/GlobalISel/artifact-combiner-anyext.mir
@@ -0,0 +1,70 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -O0 -mtriple=amdgcn-mesa-mesa3d -mcpu=tahiti -run-pass=legalizer  -global-isel-abort=0 %s -o - | FileCheck %s
+# FIXME: Remove -global-isel-abort=0 when G_TRUNC legality handled
+
+---
+name: test_anyext_trunc_v2s32_to_v2s16_to_v2s32
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; CHECK-LABEL: name: test_anyext_trunc_v2s32_to_v2s16_to_v2s32
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    ; CHECK: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY [[COPY]](<2 x s32>)
+    ; CHECK: $vgpr0_vgpr1 = COPY [[COPY1]](<2 x s32>)
+    %0:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    %1:_(<2 x s16>) = G_TRUNC %0
+    %2:_(<2 x s32>) = G_ANYEXT %1
+    $vgpr0_vgpr1 = COPY %2
+...
+
+---
+name: test_anyext_trunc_v2s32_to_v2s16_to_v2s64
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; CHECK-LABEL: name: test_anyext_trunc_v2s32_to_v2s16_to_v2s64
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    ; CHECK: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
+    ; CHECK: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[UV]](s32)
+    ; CHECK: [[ANYEXT1:%[0-9]+]]:_(s64) = G_ANYEXT [[UV1]](s32)
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[ANYEXT]](s64), [[ANYEXT1]](s64)
+    ; CHECK: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>)
+    %0:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    %1:_(<2 x s16>) = G_TRUNC %0
+    %2:_(<2 x s64>) = G_ANYEXT %1
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %2
+...
+
+---
+name: test_anyext_trunc_v2s32_to_v2s8_to_v2s16
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; CHECK-LABEL: name: test_anyext_trunc_v2s32_to_v2s8_to_v2s16
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    ; CHECK: [[TRUNC:%[0-9]+]]:_(<2 x s16>) = G_TRUNC [[COPY]](<2 x s32>)
+    ; CHECK: $vgpr0 = COPY [[TRUNC]](<2 x s16>)
+    %0:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    %1:_(<2 x s8>) = G_TRUNC %0
+    %2:_(<2 x s16>) = G_ANYEXT %1
+    $vgpr0 = COPY %2
+...
+
+---
+name: test_anyext_trunc_v3s32_to_v3s16_to_v3s32
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2
+
+    ; CHECK-LABEL: name: test_anyext_trunc_v3s32_to_v3s16_to_v3s32
+    ; CHECK: [[COPY:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr0_vgpr1_vgpr2
+    ; CHECK: [[COPY1:%[0-9]+]]:_(<3 x s32>) = COPY [[COPY]](<3 x s32>)
+    ; CHECK: $vgpr0_vgpr1_vgpr2 = COPY [[COPY1]](<3 x s32>)
+    %0:_(<3 x s32>) = COPY $vgpr0_vgpr1_vgpr2
+    %1:_(<3 x s16>) = G_TRUNC %0
+    %2:_(<3 x s32>) = G_ANYEXT %1
+    $vgpr0_vgpr1_vgpr2 = COPY %2
+...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/artifact-combiner-extract.mir b/test/CodeGen/AMDGPU/GlobalISel/artifact-combiner-extract.mir
new file mode 100644
index 0000000000000000000000000000000000000000..4202c4d1348efa862d238282e7811989283b34d3
--- /dev/null
+++ b/test/CodeGen/AMDGPU/GlobalISel/artifact-combiner-extract.mir
@@ -0,0 +1,470 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -O0 -mtriple=amdgcn-mesa-mesa3d -mcpu=tahiti -run-pass=legalizer -global-isel-abort=0 %s -o - | FileCheck %s
+
+---
+name: extract_s32_merge_s64_s32_s32_offset0
+
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: extract_s32_merge_s64_s32_s32_offset0
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; CHECK: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY [[C]](s32)
+    ; CHECK: $vgpr0 = COPY [[COPY]](s32)
+    %0:_(s32) = G_CONSTANT i32 0
+    %1:_(s32) = G_CONSTANT i32 1
+    %2:_(s64) = G_MERGE_VALUES %0, %1
+    %3:_(s32) = G_EXTRACT %2, 0
+    $vgpr0 = COPY %3
+...
+
+---
+name: extract_s32_merge_s64_s32_s32_offset32
+
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: extract_s32_merge_s64_s32_s32_offset32
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; CHECK: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY [[C1]](s32)
+    ; CHECK: $vgpr0 = COPY [[COPY]](s32)
+    %0:_(s32) = G_CONSTANT i32 0
+    %1:_(s32) = G_CONSTANT i32 1
+    %2:_(s64) = G_MERGE_VALUES %0, %1
+    %3:_(s32) = G_EXTRACT %2, 32
+    $vgpr0 = COPY %3
+...
+
+---
+name: extract_s64_merge_s128_s64_s64_offset0
+
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: extract_s64_merge_s128_s64_s64_offset0
+    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; CHECK: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
+    ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY [[C]](s64)
+    ; CHECK: $vgpr0_vgpr1 = COPY [[COPY]](s64)
+    %0:_(s64) = G_CONSTANT i64 0
+    %1:_(s64) = G_CONSTANT i64 1
+    %2:_(s128) = G_MERGE_VALUES %0, %1
+    %3:_(s64) = G_EXTRACT %2, 0
+    $vgpr0_vgpr1 = COPY %3
+...
+
+---
+name: extract_s64_merge_s128_s64_s64_offset64
+
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: extract_s64_merge_s128_s64_s64_offset64
+    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; CHECK: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
+    ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY [[C1]](s64)
+    ; CHECK: $vgpr0_vgpr1 = COPY [[COPY]](s64)
+    %0:_(s64) = G_CONSTANT i64 0
+    %1:_(s64) = G_CONSTANT i64 1
+    %2:_(s128) = G_MERGE_VALUES %0, %1
+    %3:_(s64) = G_EXTRACT %2, 64
+    $vgpr0_vgpr1 = COPY %3
+...
+
+---
+name: extract_s32_merge_s128_s64_s64_offset0
+
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: extract_s32_merge_s128_s64_s64_offset0
+    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; CHECK: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
+    ; CHECK: [[EXTRACT:%[0-9]+]]:_(s32) = G_EXTRACT [[C]](s64), 0
+    ; CHECK: $vgpr0 = COPY [[EXTRACT]](s32)
+    %0:_(s64) = G_CONSTANT i64 0
+    %1:_(s64) = G_CONSTANT i64 1
+    %2:_(s128) = G_MERGE_VALUES %0, %1
+    %3:_(s32) = G_EXTRACT %2, 0
+    $vgpr0 = COPY %3
+...
+
+---
+name: extract_s32_merge_s128_s64_s64_offset32
+
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: extract_s32_merge_s128_s64_s64_offset32
+    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; CHECK: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
+    ; CHECK: [[EXTRACT:%[0-9]+]]:_(s32) = G_EXTRACT [[C]](s64), 32
+    ; CHECK: $vgpr0 = COPY [[EXTRACT]](s32)
+    %0:_(s64) = G_CONSTANT i64 0
+    %1:_(s64) = G_CONSTANT i64 1
+    %2:_(s128) = G_MERGE_VALUES %0, %1
+    %3:_(s32) = G_EXTRACT %2, 32
+    $vgpr0 = COPY %3
+...
+
+---
+name: extract_s32_merge_s128_s64_s64_offset64
+
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: extract_s32_merge_s128_s64_s64_offset64
+    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; CHECK: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
+    ; CHECK: [[EXTRACT:%[0-9]+]]:_(s32) = G_EXTRACT [[C1]](s64), 0
+    ; CHECK: $vgpr0 = COPY [[EXTRACT]](s32)
+    %0:_(s64) = G_CONSTANT i64 0
+    %1:_(s64) = G_CONSTANT i64 1
+    %2:_(s128) = G_MERGE_VALUES %0, %1
+    %3:_(s32) = G_EXTRACT %2, 64
+    $vgpr0 = COPY %3
+...
+
+---
+name: extract_s32_merge_s128_s64_s64_offset96
+
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: extract_s32_merge_s128_s64_s64_offset96
+    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; CHECK: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
+    ; CHECK: [[EXTRACT:%[0-9]+]]:_(s32) = G_EXTRACT [[C1]](s64), 32
+    ; CHECK: $vgpr0 = COPY [[EXTRACT]](s32)
+    %0:_(s64) = G_CONSTANT i64 0
+    %1:_(s64) = G_CONSTANT i64 1
+    %2:_(s128) = G_MERGE_VALUES %0, %1
+    %3:_(s32) = G_EXTRACT %2, 96
+    $vgpr0 = COPY %3
+...
+
+# Destination size fits, but is skewed from the start of the register.
+---
+name: extract_s16_merge_s128_s64_s64_offset18
+
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: extract_s16_merge_s128_s64_s64_offset18
+    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; CHECK: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
+    ; CHECK: [[EXTRACT:%[0-9]+]]:_(s16) = G_EXTRACT [[C]](s64), 18
+    ; CHECK: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[EXTRACT]](s16)
+    ; CHECK: $vgpr0 = COPY [[ANYEXT]](s32)
+    %0:_(s64) = G_CONSTANT i64 0
+    %1:_(s64) = G_CONSTANT i64 1
+    %2:_(s128) = G_MERGE_VALUES %0, %1
+    %3:_(s16) = G_EXTRACT %2, 18
+    %4:_(s32) = G_ANYEXT %3
+    $vgpr0 = COPY %4
+...
+
+# Destination size fits, but is skewed from the start of the register.
+---
+name: extract_s16_merge_s128_s64_s64_offset82
+
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: extract_s16_merge_s128_s64_s64_offset82
+    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; CHECK: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
+    ; CHECK: [[EXTRACT:%[0-9]+]]:_(s16) = G_EXTRACT [[C1]](s64), 18
+    ; CHECK: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[EXTRACT]](s16)
+    ; CHECK: $vgpr0 = COPY [[ANYEXT]](s32)
+    %0:_(s64) = G_CONSTANT i64 0
+    %1:_(s64) = G_CONSTANT i64 1
+    %2:_(s128) = G_MERGE_VALUES %0, %1
+    %3:_(s16) = G_EXTRACT %2, 82
+    %4:_(s32) = G_ANYEXT %3
+    $vgpr0 = COPY %4
+...
+
+
+# Can't handle this since it spans two registers
+---
+name: extract_s64_merge_s128_s64_s64_offset32
+
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: extract_s64_merge_s128_s64_s64_offset32
+    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; CHECK: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
+    ; CHECK: [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[C]](s64), [[C1]](s64)
+    ; CHECK: [[EXTRACT:%[0-9]+]]:_(s64) = G_EXTRACT [[MV]](s128), 32
+    ; CHECK: $vgpr0_vgpr1 = COPY [[EXTRACT]](s64)
+    %0:_(s64) = G_CONSTANT i64 0
+    %1:_(s64) = G_CONSTANT i64 1
+    %2:_(s128) = G_MERGE_VALUES %0, %1
+    %3:_(s64) = G_EXTRACT %2, 32
+    $vgpr0_vgpr1 = COPY %3
+...
+
+
+# Only the last bit spans to another register
+---
+name: extract_s16_merge_s32_s32_offset1
+
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: extract_s16_merge_s32_s32_offset1
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; CHECK: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; CHECK: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[C]](s32), [[C1]](s32)
+    ; CHECK: [[EXTRACT:%[0-9]+]]:_(s32) = G_EXTRACT [[MV]](s64), 1
+    ; CHECK: $vgpr0 = COPY [[EXTRACT]](s32)
+    %0:_(s32) = G_CONSTANT i32 0
+    %1:_(s32) = G_CONSTANT i32 1
+    %2:_(s64) = G_MERGE_VALUES %0, %1
+    %3:_(s32) = G_EXTRACT %2, 1
+    $vgpr0 = COPY %3
+...
+
+
+# Test with some merges with 3 operands
+
+---
+name: extract_s32_merge_s96_s32_s32_s32_offset0
+
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: extract_s32_merge_s96_s32_s32_s32_offset0
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; CHECK: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; CHECK: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY [[C]](s32)
+    ; CHECK: $vgpr0 = COPY [[COPY]](s32)
+    %0:_(s32) = G_CONSTANT i32 0
+    %1:_(s32) = G_CONSTANT i32 1
+    %2:_(s32) = G_CONSTANT i32 1
+    %3:_(s96) = G_MERGE_VALUES %0, %1, %2
+    %4:_(s32) = G_EXTRACT %3, 0
+    $vgpr0 = COPY %4
+...
+
+---
+name: extract_s32_merge_s96_s32_s32_s32_offset64
+
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: extract_s32_merge_s96_s32_s32_s32_offset64
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; CHECK: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; CHECK: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY [[C2]](s32)
+    ; CHECK: $vgpr0 = COPY [[COPY]](s32)
+    %0:_(s32) = G_CONSTANT i32 0
+    %1:_(s32) = G_CONSTANT i32 1
+    %2:_(s32) = G_CONSTANT i32 1
+    %3:_(s96) = G_MERGE_VALUES %0, %1, %2
+    %4:_(s32) = G_EXTRACT %3, 64
+    $vgpr0 = COPY %4
+...
+
+---
+name: extract_s64_merge_s96_s32_s32_s32_offset0
+
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: extract_s64_merge_s96_s32_s32_s32_offset0
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; CHECK: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; CHECK: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; CHECK: [[MV:%[0-9]+]]:_(s96) = G_MERGE_VALUES [[C]](s32), [[C1]](s32), [[C2]](s32)
+    ; CHECK: [[EXTRACT:%[0-9]+]]:_(s64) = G_EXTRACT [[MV]](s96), 0
+    ; CHECK: $vgpr0_vgpr1 = COPY [[EXTRACT]](s64)
+    %0:_(s32) = G_CONSTANT i32 0
+    %1:_(s32) = G_CONSTANT i32 1
+    %2:_(s32) = G_CONSTANT i32 1
+    %3:_(s96) = G_MERGE_VALUES %0, %1, %2
+    %4:_(s64) = G_EXTRACT %3, 0
+    $vgpr0_vgpr1 = COPY %4
+...
+
+---
+name: extract_s64_merge_s96_s32_s32_s32_offset32
+
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: extract_s64_merge_s96_s32_s32_s32_offset32
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; CHECK: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; CHECK: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; CHECK: [[MV:%[0-9]+]]:_(s96) = G_MERGE_VALUES [[C]](s32), [[C1]](s32), [[C2]](s32)
+    ; CHECK: [[EXTRACT:%[0-9]+]]:_(s64) = G_EXTRACT [[MV]](s96), 32
+    ; CHECK: $vgpr0_vgpr1 = COPY [[EXTRACT]](s64)
+    %0:_(s32) = G_CONSTANT i32 0
+    %1:_(s32) = G_CONSTANT i32 1
+    %2:_(s32) = G_CONSTANT i32 1
+    %3:_(s96) = G_MERGE_VALUES %0, %1, %2
+    %4:_(s64) = G_EXTRACT %3, 32
+    $vgpr0_vgpr1 = COPY %4
+...
+
+# Test build_vector sources
+---
+name: extract_s64_build_vector_v2s64_s64_s64_offset0
+
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: extract_s64_build_vector_v2s64_s64_s64_offset0
+    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; CHECK: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
+    ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY [[C]](s64)
+    ; CHECK: $vgpr0_vgpr1 = COPY [[COPY]](s64)
+    %0:_(s64) = G_CONSTANT i64 0
+    %1:_(s64) = G_CONSTANT i64 1
+    %2:_(<2 x s64>) = G_BUILD_VECTOR %0, %1
+    %3:_(s64) = G_EXTRACT %2, 0
+    $vgpr0_vgpr1 = COPY %3
+...
+
+---
+name: extract_s64_build_vector_v2s64_s64_s64_offset64
+
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: extract_s64_build_vector_v2s64_s64_s64_offset64
+    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; CHECK: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
+    ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY [[C1]](s64)
+    ; CHECK: $vgpr0_vgpr1 = COPY [[COPY]](s64)
+    %0:_(s64) = G_CONSTANT i64 0
+    %1:_(s64) = G_CONSTANT i64 1
+    %2:_(<2 x s64>) = G_BUILD_VECTOR %0, %1
+    %3:_(s64) = G_EXTRACT %2, 64
+    $vgpr0_vgpr1 = COPY %3
+...
+
+---
+name: extract_s64_build_vector_v2s64_s64_s64_offset32
+
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: extract_s64_build_vector_v2s64_s64_s64_offset32
+    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; CHECK: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[C]](s64), [[C1]](s64)
+    ; CHECK: [[EXTRACT:%[0-9]+]]:_(s64) = G_EXTRACT [[BUILD_VECTOR]](<2 x s64>), 32
+    ; CHECK: $vgpr0_vgpr1 = COPY [[EXTRACT]](s64)
+    %0:_(s64) = G_CONSTANT i64 0
+    %1:_(s64) = G_CONSTANT i64 1
+    %2:_(<2 x s64>) = G_BUILD_VECTOR %0, %1
+    %3:_(s64) = G_EXTRACT %2, 32
+    $vgpr0_vgpr1 = COPY %3
+...
+
+# Test extracting something smaller than the element size
+---
+name: extract_s32_build_vector_v2s64_s64_s64_offset64
+
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: extract_s32_build_vector_v2s64_s64_s64_offset64
+    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; CHECK: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
+    ; CHECK: [[EXTRACT:%[0-9]+]]:_(s32) = G_EXTRACT [[C1]](s64), 0
+    ; CHECK: $vgpr0 = COPY [[EXTRACT]](s32)
+    %0:_(s64) = G_CONSTANT i64 0
+    %1:_(s64) = G_CONSTANT i64 1
+    %2:_(<2 x s64>) = G_BUILD_VECTOR %0, %1
+    %3:_(s32) = G_EXTRACT %2, 64
+    $vgpr0 = COPY %3
+
+...
+
+# Test concat_vector sources
+---
+name: extract_v2s16_build_vector_v2s64_v2s16_v2s16_offset0
+
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+    ; CHECK-LABEL: name: extract_v2s16_build_vector_v2s64_v2s16_v2s16_offset0
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
+    ; CHECK: [[COPY2:%[0-9]+]]:_(<2 x s16>) = COPY [[COPY]](<2 x s16>)
+    ; CHECK: $vgpr0 = COPY [[COPY2]](<2 x s16>)
+    %0:_(<2 x s16>) = COPY $vgpr0
+    %1:_(<2 x s16>) = COPY $vgpr1
+    %2:_(<4 x s16>) = G_CONCAT_VECTORS %0, %1
+    %3:_(<2 x s16>) = G_EXTRACT %2, 0
+    $vgpr0 = COPY %3
+...
+
+---
+name: extract_v2s16_build_vector_v2s64_v2s16_v2s16_offset32
+
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+    ; CHECK-LABEL: name: extract_v2s16_build_vector_v2s64_v2s16_v2s16_offset32
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
+    ; CHECK: [[COPY2:%[0-9]+]]:_(<2 x s16>) = COPY [[COPY1]](<2 x s16>)
+    ; CHECK: $vgpr0 = COPY [[COPY2]](<2 x s16>)
+    %0:_(<2 x s16>) = COPY $vgpr0
+    %1:_(<2 x s16>) = COPY $vgpr1
+    %2:_(<4 x s16>) = G_CONCAT_VECTORS %0, %1
+    %3:_(<2 x s16>) = G_EXTRACT %2, 32
+    $vgpr0 = COPY %3
+...
+
+# Test extracting only a single element, not a subvector
+---
+name: extract_s16_build_vector_v2s64_v2s16_v2s16_offset32
+
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+    ; CHECK-LABEL: name: extract_s16_build_vector_v2s64_v2s16_v2s16_offset32
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
+    ; CHECK: [[EXTRACT:%[0-9]+]]:_(s16) = G_EXTRACT [[COPY1]](<2 x s16>), 0
+    ; CHECK: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[EXTRACT]](s16)
+    ; CHECK: $vgpr0 = COPY [[ANYEXT]](s32)
+    %0:_(<2 x s16>) = COPY $vgpr0
+    %1:_(<2 x s16>) = COPY $vgpr1
+    %2:_(<4 x s16>) = G_CONCAT_VECTORS %0, %1
+    %3:_(s16) = G_EXTRACT %2, 32
+    %4:_(s32) = G_ANYEXT %3
+    $vgpr0 = COPY %4
+...
+
+---
+name: extract_s16_build_vector_v2s64_v2s16_v2s16_offset48
+
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+    ; CHECK-LABEL: name: extract_s16_build_vector_v2s64_v2s16_v2s16_offset48
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
+    ; CHECK: [[EXTRACT:%[0-9]+]]:_(s16) = G_EXTRACT [[COPY1]](<2 x s16>), 16
+    ; CHECK: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[EXTRACT]](s16)
+    ; CHECK: $vgpr0 = COPY [[ANYEXT]](s32)
+    %0:_(<2 x s16>) = COPY $vgpr0
+    %1:_(<2 x s16>) = COPY $vgpr1
+    %2:_(<4 x s16>) = G_CONCAT_VECTORS %0, %1
+    %3:_(s16) = G_EXTRACT %2, 48
+    %4:_(s32) = G_ANYEXT %3
+    $vgpr0 = COPY %4
+...
+
+# Test extracting less than an element
+---
+name: extract_s8_build_vector_v2s64_v2s16_v2s16_offset48
+
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+    ; CHECK-LABEL: name: extract_s8_build_vector_v2s64_v2s16_v2s16_offset48
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
+    ; CHECK: [[EXTRACT:%[0-9]+]]:_(s8) = G_EXTRACT [[COPY1]](<2 x s16>), 16
+    ; CHECK: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[EXTRACT]](s8)
+    ; CHECK: $vgpr0 = COPY [[ANYEXT]](s32)
+    %0:_(<2 x s16>) = COPY $vgpr0
+    %1:_(<2 x s16>) = COPY $vgpr1
+    %2:_(<4 x s16>) = G_CONCAT_VECTORS %0, %1
+    %3:_(s8) = G_EXTRACT %2, 48
+    %4:_(s32) = G_ANYEXT %3
+    $vgpr0 = COPY %4
+...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/artifact-combiner-sext.mir b/test/CodeGen/AMDGPU/GlobalISel/artifact-combiner-sext.mir
new file mode 100644
index 0000000000000000000000000000000000000000..821b7a5063a36830966f27990c058505eac90ba0
--- /dev/null
+++ b/test/CodeGen/AMDGPU/GlobalISel/artifact-combiner-sext.mir
@@ -0,0 +1,99 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -O0 -mtriple=amdgcn-mesa-mesa3d -mcpu=tahiti -run-pass=legalizer  -global-isel-abort=0 %s -o - | FileCheck %s
+# FIXME: Remove -global-isel-abort=0 when G_TRUNC legality handled
+
+---
+name: test_sext_trunc_v2s32_to_v2s16_to_v2s32
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; CHECK-LABEL: name: test_sext_trunc_v2s32_to_v2s16_to_v2s32
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; CHECK: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY [[COPY]](<2 x s32>)
+    ; CHECK: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<2 x s32>)
+    ; CHECK: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[UV]], [[C]](s32)
+    ; CHECK: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[UV1]], [[C]](s32)
+    ; CHECK: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C]](s32)
+    ; CHECK: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[C]](s32)
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[ASHR]](s32), [[ASHR1]](s32)
+    ; CHECK: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
+    %0:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    %1:_(<2 x s16>) = G_TRUNC %0
+    %2:_(<2 x s32>) = G_SEXT %1
+    $vgpr0_vgpr1 = COPY %2
+...
+
+---
+name: test_sext_trunc_v2s32_to_v2s16_to_v2s64
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; CHECK-LABEL: name: test_sext_trunc_v2s32_to_v2s16_to_v2s64
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 48
+    ; CHECK: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
+    ; CHECK: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[UV]](s32)
+    ; CHECK: [[ANYEXT1:%[0-9]+]]:_(s64) = G_ANYEXT [[UV1]](s32)
+    ; CHECK: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[C]](s64)
+    ; CHECK: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[ANYEXT]], [[TRUNC]](s32)
+    ; CHECK: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[C]](s64)
+    ; CHECK: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[ANYEXT1]], [[TRUNC1]](s32)
+    ; CHECK: [[TRUNC2:%[0-9]+]]:_(s32) = G_TRUNC [[C]](s64)
+    ; CHECK: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[SHL]], [[TRUNC2]](s32)
+    ; CHECK: [[TRUNC3:%[0-9]+]]:_(s32) = G_TRUNC [[C]](s64)
+    ; CHECK: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[SHL1]], [[TRUNC3]](s32)
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[ASHR]](s64), [[ASHR1]](s64)
+    ; CHECK: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>)
+    %0:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    %1:_(<2 x s16>) = G_TRUNC %0
+    %2:_(<2 x s64>) = G_SEXT %1
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %2
+...
+
+---
+name: test_sext_trunc_v2s32_to_v2s8_to_v2s16
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; CHECK-LABEL: name: test_sext_trunc_v2s32_to_v2s8_to_v2s16
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    ; CHECK: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[C]](s16), [[C]](s16)
+    ; CHECK: [[TRUNC:%[0-9]+]]:_(<2 x s16>) = G_TRUNC [[COPY]](<2 x s32>)
+    ; CHECK: [[SHL:%[0-9]+]]:_(<2 x s16>) = G_SHL [[TRUNC]], [[BUILD_VECTOR]](<2 x s16>)
+    ; CHECK: [[ASHR:%[0-9]+]]:_(<2 x s16>) = G_ASHR [[SHL]], [[BUILD_VECTOR]](<2 x s16>)
+    ; CHECK: $vgpr0 = COPY [[ASHR]](<2 x s16>)
+    %0:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    %1:_(<2 x s8>) = G_TRUNC %0
+    %2:_(<2 x s16>) = G_SEXT %1
+    $vgpr0 = COPY %2
+...
+
+---
+name: test_sext_trunc_v3s32_to_v3s16_to_v3s32
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2
+
+    ; CHECK-LABEL: name: test_sext_trunc_v3s32_to_v3s16_to_v3s32
+    ; CHECK: [[COPY:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr0_vgpr1_vgpr2
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; CHECK: [[COPY1:%[0-9]+]]:_(<3 x s32>) = COPY [[COPY]](<3 x s32>)
+    ; CHECK: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<3 x s32>)
+    ; CHECK: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[UV]], [[C]](s32)
+    ; CHECK: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[UV1]], [[C]](s32)
+    ; CHECK: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[UV2]], [[C]](s32)
+    ; CHECK: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C]](s32)
+    ; CHECK: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[C]](s32)
+    ; CHECK: [[ASHR2:%[0-9]+]]:_(s32) = G_ASHR [[SHL2]], [[C]](s32)
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[ASHR]](s32), [[ASHR1]](s32), [[ASHR2]](s32)
+    ; CHECK: $vgpr0_vgpr1_vgpr2 = COPY [[BUILD_VECTOR]](<3 x s32>)
+    %0:_(<3 x s32>) = COPY $vgpr0_vgpr1_vgpr2
+    %1:_(<3 x s16>) = G_TRUNC %0
+    %2:_(<3 x s32>) = G_SEXT %1
+    $vgpr0_vgpr1_vgpr2 = COPY %2
+...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/artifact-combiner-zext.mir b/test/CodeGen/AMDGPU/GlobalISel/artifact-combiner-zext.mir
new file mode 100644
index 0000000000000000000000000000000000000000..6b9144617b264b4077f36a79648bad44efe2701f
--- /dev/null
+++ b/test/CodeGen/AMDGPU/GlobalISel/artifact-combiner-zext.mir
@@ -0,0 +1,85 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -O0 -mtriple=amdgcn-mesa-mesa3d -mcpu=tahiti -run-pass=legalizer  -global-isel-abort=0 %s -o - | FileCheck %s
+# FIXME: Remove -global-isel-abort=0 when G_TRUNC legality handled
+
+---
+name: test_zext_trunc_v2s32_to_v2s16_to_v2s32
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; CHECK-LABEL: name: test_zext_trunc_v2s32_to_v2s16_to_v2s32
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[C]](s32), [[C]](s32)
+    ; CHECK: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY [[COPY]](<2 x s32>)
+    ; CHECK: [[AND:%[0-9]+]]:_(<2 x s32>) = G_AND [[COPY1]], [[BUILD_VECTOR]]
+    ; CHECK: $vgpr0_vgpr1 = COPY [[AND]](<2 x s32>)
+    %0:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    %1:_(<2 x s16>) = G_TRUNC %0
+    %2:_(<2 x s32>) = G_ZEXT %1
+    $vgpr0_vgpr1 = COPY %2
+...
+
+---
+name: test_zext_trunc_v2s32_to_v2s16_to_v2s64
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; CHECK-LABEL: name: test_zext_trunc_v2s32_to_v2s16_to_v2s64
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 65535
+    ; CHECK: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
+    ; CHECK: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[UV]](s32)
+    ; CHECK: [[ANYEXT1:%[0-9]+]]:_(s64) = G_ANYEXT [[UV1]](s32)
+    ; CHECK: [[AND:%[0-9]+]]:_(s64) = G_AND [[ANYEXT]], [[C]]
+    ; CHECK: [[AND1:%[0-9]+]]:_(s64) = G_AND [[ANYEXT1]], [[C]]
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[AND]](s64), [[AND1]](s64)
+    ; CHECK: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>)
+    %0:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    %1:_(<2 x s16>) = G_TRUNC %0
+    %2:_(<2 x s64>) = G_ZEXT %1
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %2
+...
+
+---
+name: test_zext_trunc_v2s32_to_v2s8_to_v2s16
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; CHECK-LABEL: name: test_zext_trunc_v2s32_to_v2s8_to_v2s16
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    ; CHECK: [[C:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[C]](s16), [[C]](s16)
+    ; CHECK: [[TRUNC:%[0-9]+]]:_(<2 x s16>) = G_TRUNC [[COPY]](<2 x s32>)
+    ; CHECK: [[AND:%[0-9]+]]:_(<2 x s16>) = G_AND [[TRUNC]], [[BUILD_VECTOR]]
+    ; CHECK: $vgpr0 = COPY [[AND]](<2 x s16>)
+    %0:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    %1:_(<2 x s8>) = G_TRUNC %0
+    %2:_(<2 x s16>) = G_ZEXT %1
+    $vgpr0 = COPY %2
+...
+
+---
+name: test_zext_trunc_v3s32_to_v3s16_to_v3s32
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2
+
+    ; CHECK-LABEL: name: test_zext_trunc_v3s32_to_v3s16_to_v3s32
+    ; CHECK: [[COPY:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr0_vgpr1_vgpr2
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; CHECK: [[COPY1:%[0-9]+]]:_(<3 x s32>) = COPY [[COPY]](<3 x s32>)
+    ; CHECK: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<3 x s32>)
+    ; CHECK: [[AND:%[0-9]+]]:_(s32) = G_AND [[UV]], [[C]]
+    ; CHECK: [[AND1:%[0-9]+]]:_(s32) = G_AND [[UV1]], [[C]]
+    ; CHECK: [[AND2:%[0-9]+]]:_(s32) = G_AND [[UV2]], [[C]]
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[AND]](s32), [[AND1]](s32), [[AND2]](s32)
+    ; CHECK: $vgpr0_vgpr1_vgpr2 = COPY [[BUILD_VECTOR]](<3 x s32>)
+    %0:_(<3 x s32>) = COPY $vgpr0_vgpr1_vgpr2
+    %1:_(<3 x s16>) = G_TRUNC %0
+    %2:_(<3 x s32>) = G_ZEXT %1
+    $vgpr0_vgpr1_vgpr2 = COPY %2
+...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/combine-ext-legalizer.mir b/test/CodeGen/AMDGPU/GlobalISel/combine-ext-legalizer.mir
new file mode 100644
index 0000000000000000000000000000000000000000..8eeb6bd6a24a88497881ab571679fb107d3e433e
--- /dev/null
+++ b/test/CodeGen/AMDGPU/GlobalISel/combine-ext-legalizer.mir
@@ -0,0 +1,41 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -O0 -run-pass=legalizer %s -o - | FileCheck %s
+
+---
+name: test_sext_trunc_i64_i32_i64
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; CHECK-LABEL: name: test_sext_trunc_i64_i32_i64
+    ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
+    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 32
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s64) = COPY [[COPY]](s64)
+    ; CHECK: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[C]](s64)
+    ; CHECK: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[COPY1]], [[TRUNC]](s32)
+    ; CHECK: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[C]](s64)
+    ; CHECK: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[SHL]], [[TRUNC1]](s32)
+    ; CHECK: $vgpr0_vgpr1 = COPY [[ASHR]](s64)
+    %0:_(s64) = COPY $vgpr0_vgpr1
+    %1:_(s32) = G_TRUNC %0
+    %2:_(s64) = G_SEXT %1
+    $vgpr0_vgpr1 = COPY %2
+...
+
+---
+name: test_zext_trunc_i64_i32_i64
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; CHECK-LABEL: name: test_zext_trunc_i64_i32_i64
+    ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
+    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 4294967295
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s64) = COPY [[COPY]](s64)
+    ; CHECK: [[AND:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[C]]
+    ; CHECK: $vgpr0_vgpr1 = COPY [[AND]](s64)
+    %0:_(s64) = COPY $vgpr0_vgpr1
+    %1:_(s32) = G_TRUNC %0
+    %2:_(s64) = G_ZEXT %1
+    $vgpr0_vgpr1 = COPY %2
+...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/inst-select-amdgcn.cvt.pkrtz.mir b/test/CodeGen/AMDGPU/GlobalISel/inst-select-amdgcn.cvt.pkrtz.mir
index f1c19f124ffad6240a7d65637dd41cbabcd157f8..f241c2dad1e9cd3e2607a885d6a6eff145fb19e8 100644
--- a/test/CodeGen/AMDGPU/GlobalISel/inst-select-amdgcn.cvt.pkrtz.mir
+++ b/test/CodeGen/AMDGPU/GlobalISel/inst-select-amdgcn.cvt.pkrtz.mir
@@ -21,7 +21,7 @@ body: |
     %1:vgpr(s32) = COPY $vgpr0
     ; GCN: [[VGPR1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
     %2:vgpr(s32) = COPY $vgpr1
-    %3:vgpr(s64) = COPY $vgpr3_vgpr4
+    %3:vgpr(p1) = COPY $vgpr3_vgpr4
 
     ; cvt_pkrtz vs
     ; GCN: V_CVT_PKRTZ_F16_F32_e64 0, [[VGPR0]], 0, [[SGPR0]]
@@ -38,8 +38,8 @@ body: |
     %7:vgpr(s32) = G_BITCAST %4
     %8:vgpr(s32) = G_BITCAST %5
     %9:vgpr(s32) = G_BITCAST %6
-    G_STORE %7, %3 :: (store 4 into %ir.global0)
-    G_STORE %8, %3 :: (store 4 into %ir.global0)
-    G_STORE %9, %3 :: (store 4 into %ir.global0)
+    G_STORE %7, %3 :: (store 4 into %ir.global0, addrspace 1)
+    G_STORE %8, %3 :: (store 4 into %ir.global0, addrspace 1)
+    G_STORE %9, %3 :: (store 4 into %ir.global0, addrspace 1)
 ...
 ---
diff --git a/test/CodeGen/AMDGPU/GlobalISel/inst-select-ashr.mir b/test/CodeGen/AMDGPU/GlobalISel/inst-select-ashr.mir
index 86692c3e1d626582e6db84462db6fca4341c9f1e..a9126854c21b4753dadaeec16ac24a51a0ffdf11 100644
--- a/test/CodeGen/AMDGPU/GlobalISel/inst-select-ashr.mir
+++ b/test/CodeGen/AMDGPU/GlobalISel/inst-select-ashr.mir
@@ -20,7 +20,7 @@ body: |
     %0:sgpr(s32) = COPY $sgpr0
     %1:sgpr(s32) = COPY $sgpr1
     %2:vgpr(s32) = COPY $vgpr0
-    %3:vgpr(s64) = COPY $vgpr3_vgpr4
+    %3:vgpr(p1) = COPY $vgpr3_vgpr4
 
     ; GCN: [[C1:%[0-9]+]]:sreg_32_xm0 = S_MOV_B32 1
     ; GCN: [[C4096:%[0-9]+]]:sreg_32_xm0 = S_MOV_B32 4096
@@ -80,7 +80,7 @@ body: |
     %17:vgpr(s32) = G_ASHR %16, %5
 
 
-    G_STORE %17, %3 :: (store 4 into %ir.global0)
+    G_STORE %17, %3 :: (store 4 into %ir.global0, addrspace 1)
 
 ...
 ---
diff --git a/test/CodeGen/AMDGPU/GlobalISel/inst-select-bitcast.mir b/test/CodeGen/AMDGPU/GlobalISel/inst-select-bitcast.mir
index 88f811160c0dc5a4c0cec4b0d3b1a0524d55e0a6..157e7b703d787b98a9c15df031bb9e116a1489d0 100644
--- a/test/CodeGen/AMDGPU/GlobalISel/inst-select-bitcast.mir
+++ b/test/CodeGen/AMDGPU/GlobalISel/inst-select-bitcast.mir
@@ -18,9 +18,9 @@ body: |
   bb.0:
     liveins: $sgpr0, $vgpr3_vgpr4
     %0:vgpr(s32) = COPY $vgpr0
-    %1:vgpr(s64) = COPY $vgpr3_vgpr4
+    %1:vgpr(p1) = COPY $vgpr3_vgpr4
     %2:vgpr(<2 x s16>) = G_BITCAST %0
     %3:vgpr(s32) = G_BITCAST %2
-    G_STORE %3, %1 :: (store 4 into %ir.global0)
+    G_STORE %3, %1 :: (store 4 into %ir.global0, addrspace 1)
 ...
 ---
diff --git a/test/CodeGen/AMDGPU/GlobalISel/inst-select-constant.mir b/test/CodeGen/AMDGPU/GlobalISel/inst-select-constant.mir
index f848edaf6675af20d13784e6d3dac04dcef1d920..19ad8729a02bdd60f64caa5503d5bd62d4a9abab 100644
--- a/test/CodeGen/AMDGPU/GlobalISel/inst-select-constant.mir
+++ b/test/CodeGen/AMDGPU/GlobalISel/inst-select-constant.mir
@@ -14,8 +14,8 @@ body: |
   bb.0:
     liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
     ; GCN-LABEL: name: constant
-    %0:vgpr(s64) = COPY $vgpr0_vgpr1
-    %1:vgpr(s64) = COPY $vgpr2_vgpr3
+    %0:vgpr(p1) = COPY $vgpr0_vgpr1
+    %1:vgpr(p1) = COPY $vgpr2_vgpr3
 
     ; GCN: %{{[0-9]+}}:sreg_32 = S_MOV_B32 1
     %2:sreg_32(s32) = G_CONSTANT i32 1
@@ -49,13 +49,13 @@ body: |
     ; GCN: %{{[0-9]+}}:vreg_64 = REG_SEQUENCE [[LO3]], %subreg.sub0, [[HI3]], %subreg.sub1
     %9:vgpr(s64) = G_FCONSTANT double 1.0
 
-    G_STORE %2, %0 :: (volatile store 4 into %ir.global0)
-    G_STORE %4, %0 :: (volatile store 4 into %ir.global0)
-    G_STORE %6, %0 :: (volatile store 4 into %ir.global0)
-    G_STORE %8, %0 :: (volatile store 4 into %ir.global0)
-    G_STORE %3, %1 :: (volatile store 8 into %ir.global1)
-    G_STORE %5, %1 :: (volatile store 8 into %ir.global1)
-    G_STORE %7, %1 :: (volatile store 8 into %ir.global1)
-    G_STORE %9, %1 :: (volatile store 8 into %ir.global1)
+    G_STORE %2, %0 :: (volatile store 4 into %ir.global0, addrspace 1)
+    G_STORE %4, %0 :: (volatile store 4 into %ir.global0, addrspace 1)
+    G_STORE %6, %0 :: (volatile store 4 into %ir.global0, addrspace 1)
+    G_STORE %8, %0 :: (volatile store 4 into %ir.global0, addrspace 1)
+    G_STORE %3, %1 :: (volatile store 8 into %ir.global1, addrspace 1)
+    G_STORE %5, %1 :: (volatile store 8 into %ir.global1, addrspace 1)
+    G_STORE %7, %1 :: (volatile store 8 into %ir.global1, addrspace 1)
+    G_STORE %9, %1 :: (volatile store 8 into %ir.global1, addrspace 1)
 ...
 ---
diff --git a/test/CodeGen/AMDGPU/GlobalISel/inst-select-copy.mir b/test/CodeGen/AMDGPU/GlobalISel/inst-select-copy.mir
index 495acf634726dfa9528467cd050fe3cafd5e5696..5c169ca6b613c21b506cba230e1de5694f9624b8 100644
--- a/test/CodeGen/AMDGPU/GlobalISel/inst-select-copy.mir
+++ b/test/CodeGen/AMDGPU/GlobalISel/inst-select-copy.mir
@@ -19,8 +19,8 @@ body: |
     ; GCN: [[COPY1:%[0-9]+]]:vreg_64 = COPY [[COPY]]
     ; GCN: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
     ; GCN: FLAT_STORE_DWORD [[COPY1]], [[DEF]], 0, 0, 0, implicit $exec, implicit $flat_scr
-    %0:sgpr(s64) = COPY $sgpr2_sgpr3
-    %1:vgpr(s64) = COPY %0
+    %0:sgpr(p1) = COPY $sgpr2_sgpr3
+    %1:vgpr(p1) = COPY %0
     %2:vgpr(s32) = G_IMPLICIT_DEF
     G_STORE %2, %1 :: (store 4 into %ir.global0)
 ...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/inst-select-fadd.mir b/test/CodeGen/AMDGPU/GlobalISel/inst-select-fadd.mir
index 01a59f05a610e242ad35974fbb4985f9e97de9ee..74c83e76aec1f4ba8cc55510126ad5ca7a49d82a 100644
--- a/test/CodeGen/AMDGPU/GlobalISel/inst-select-fadd.mir
+++ b/test/CodeGen/AMDGPU/GlobalISel/inst-select-fadd.mir
@@ -16,7 +16,7 @@ body: |
     %0:sgpr(s32) = COPY $sgpr0
     %1:vgpr(s32) = COPY $vgpr0
     %2:vgpr(s32) = COPY $vgpr1
-    %3:vgpr(s64) = COPY $vgpr3_vgpr4
+    %3:vgpr(p1) = COPY $vgpr3_vgpr4
 
     ; fadd vs
     ; GCN: V_ADD_F32_e64
diff --git a/test/CodeGen/AMDGPU/GlobalISel/inst-select-fmul.mir b/test/CodeGen/AMDGPU/GlobalISel/inst-select-fmul.mir
index b7e472abc93f372ab07fe36b5e7ad5f1f357f484..f28caf2de08b33fb84c5f14ff8b1470cf16da94c 100644
--- a/test/CodeGen/AMDGPU/GlobalISel/inst-select-fmul.mir
+++ b/test/CodeGen/AMDGPU/GlobalISel/inst-select-fmul.mir
@@ -16,7 +16,7 @@ body: |
     %0:sgpr(s32) = COPY $sgpr0
     %1:vgpr(s32) = COPY $vgpr0
     %2:vgpr(s32) = COPY $vgpr1
-    %3:vgpr(s64) = COPY $vgpr3_vgpr4
+    %3:vgpr(p1) = COPY $vgpr3_vgpr4
 
     ; fmul vs
     ; GCN: V_MUL_F32_e64
@@ -30,8 +30,8 @@ body: |
     ; GCN: V_MUL_F32_e64
     %6:vgpr(s32) = G_FMUL %1, %2
 
-    G_STORE %4, %3 :: (store 4 into %ir.global0)
-    G_STORE %5, %3 :: (store 4 into %ir.global0)
-    G_STORE %6, %3 :: (store 4 into %ir.global0)
+    G_STORE %4, %3 :: (store 4 into %ir.global0, addrspace 1)
+    G_STORE %5, %3 :: (store 4 into %ir.global0, addrspace 1)
+    G_STORE %6, %3 :: (store 4 into %ir.global0, addrspace 1)
 ...
 ---
diff --git a/test/CodeGen/AMDGPU/GlobalISel/inst-select-fptoui.mir b/test/CodeGen/AMDGPU/GlobalISel/inst-select-fptoui.mir
index 07f19c4b34d6c735caa98d32b8c49389d18bf713..950e7a508977a50f8f0e06a04e1101ef02bf9f04 100644
--- a/test/CodeGen/AMDGPU/GlobalISel/inst-select-fptoui.mir
+++ b/test/CodeGen/AMDGPU/GlobalISel/inst-select-fptoui.mir
@@ -20,7 +20,7 @@ body: |
     ; GCN: [[VGPR:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     %1:vgpr(s32) = COPY $vgpr0
 
-    %2:vgpr(s64) = COPY $vgpr3_vgpr4
+    %2:vgpr(p1) = COPY $vgpr3_vgpr4
 
     ; fptoui s
     ; GCN: V_CVT_U32_F32_e64 0, [[SGPR]], 0, 0
diff --git a/test/CodeGen/AMDGPU/GlobalISel/inst-select-implicit-def.mir b/test/CodeGen/AMDGPU/GlobalISel/inst-select-implicit-def.mir
index 0596d8321a6aa1d35342131f1f4f51f2c47d0d9d..b7af7a919711d46bf7f6bee7f661ed93d385b85e 100644
--- a/test/CodeGen/AMDGPU/GlobalISel/inst-select-implicit-def.mir
+++ b/test/CodeGen/AMDGPU/GlobalISel/inst-select-implicit-def.mir
@@ -14,9 +14,9 @@ body: |
     ; GCN: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr3_vgpr4
     ; GCN: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
     ; GCN: FLAT_STORE_DWORD [[COPY]], [[DEF]], 0, 0, 0, implicit $exec, implicit $flat_scr
-    %0:vgpr(s64) = COPY $vgpr3_vgpr4
+    %0:vgpr(p1) = COPY $vgpr3_vgpr4
     %1:vgpr(s32) = G_IMPLICIT_DEF
-    G_STORE %1, %0 :: (store 4)
+    G_STORE %1, %0 :: (store 4, addrspace 1)
 ...
 ---
 
@@ -31,9 +31,9 @@ body: |
     ; GCN: [[COPY:%[0-9]+]]:vreg_64 = COPY $vgpr3_vgpr4
     ; GCN: [[DEF:%[0-9]+]]:vreg_64 = IMPLICIT_DEF
     ; GCN: FLAT_STORE_DWORDX2 [[COPY]], [[DEF]], 0, 0, 0, implicit $exec, implicit $flat_scr
-    %0:vgpr(s64) = COPY $vgpr3_vgpr4
+    %0:vgpr(p1) = COPY $vgpr3_vgpr4
     %1:vgpr(s64) = G_IMPLICIT_DEF
-    G_STORE %1, %0 :: (store 8)
+    G_STORE %1, %0 :: (store 8, addrspace 1)
 ---
 
 ---
@@ -45,7 +45,7 @@ regBankSelected: true
 body: |
   bb.0:
     %0:vgpr(p0) = G_IMPLICIT_DEF
-    %1:vgpr(s32) = G_CONSTANT 4
+    %1:vgpr(s32) = G_CONSTANT i32 4
     G_STORE %1, %0 :: (store 4)
 ...
 
@@ -62,8 +62,8 @@ body: |
     ; GCN: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4, implicit $exec
     ; GCN: FLAT_STORE_DWORD [[DEF]], [[V_MOV_B32_e32_]], 0, 0, 0, implicit $exec, implicit $flat_scr
     %0:vgpr(p1) = G_IMPLICIT_DEF
-    %1:vgpr(s32) = G_CONSTANT 4
-    G_STORE %1, %0 :: (store 4)
+    %1:vgpr(s32) = G_CONSTANT i32 4
+    G_STORE %1, %0 :: (store 4, addrspace 1)
 ...
 ---
 
@@ -78,8 +78,8 @@ body: |
     ; GCN: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4, implicit $exec
     ; GCN: FLAT_STORE_DWORD [[DEF]], [[V_MOV_B32_e32_]], 0, 0, 0, implicit $exec, implicit $flat_scr
     %0:vgpr(p3) = G_IMPLICIT_DEF
-    %1:vgpr(s32) = G_CONSTANT 4
-    G_STORE %1, %0 :: (store 4)
+    %1:vgpr(s32) = G_CONSTANT i32 4
+    G_STORE %1, %0 :: (store 4, addrspace 1)
 ...
 ---
 
@@ -94,6 +94,6 @@ body: |
     ; GCN: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4, implicit $exec
     ; GCN: FLAT_STORE_DWORD [[DEF]], [[V_MOV_B32_e32_]], 0, 0, 0, implicit $exec, implicit $flat_scr
     %0:vgpr(p4) = G_IMPLICIT_DEF
-    %1:vgpr(s32) = G_CONSTANT 4
-    G_STORE %1, %0 :: (store 4)
+    %1:vgpr(s32) = G_CONSTANT i32 4
+    G_STORE %1, %0 :: (store 4, addrspace 1)
 ...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/inst-select-maxnum.mir b/test/CodeGen/AMDGPU/GlobalISel/inst-select-maxnum.mir
index a473259201dc26ccfc3b52c9c0ac122ae5742a02..67fe61df3e3b16f341b8ed729da7eaad4f8d74a3 100644
--- a/test/CodeGen/AMDGPU/GlobalISel/inst-select-maxnum.mir
+++ b/test/CodeGen/AMDGPU/GlobalISel/inst-select-maxnum.mir
@@ -19,7 +19,7 @@ body: |
     %0:sgpr(s32) = COPY $sgpr0
     %1:vgpr(s32) = COPY $vgpr0
     %2:vgpr(s32) = COPY $vgpr1
-    %3:vgpr(s64) = COPY $vgpr3_vgpr4
+    %3:vgpr(p1) = COPY $vgpr3_vgpr4
 
     ; GCN: [[SGPR64_0:%[0-9]+]]:sreg_64_xexec = COPY $sgpr10_sgpr11
     ; GCN: [[VGPR64_0:%[0-9]+]]:vreg_64 = COPY $vgpr10_vgpr11
diff --git a/test/CodeGen/AMDGPU/GlobalISel/inst-select-minnum.mir b/test/CodeGen/AMDGPU/GlobalISel/inst-select-minnum.mir
index 0bfe9bb7217b067a67a847c53bf10a7d27454b97..f8132d7a46e6dc131b346785a3e9d710e47d2ab4 100644
--- a/test/CodeGen/AMDGPU/GlobalISel/inst-select-minnum.mir
+++ b/test/CodeGen/AMDGPU/GlobalISel/inst-select-minnum.mir
@@ -19,7 +19,7 @@ body: |
     %0:sgpr(s32) = COPY $sgpr0
     %1:vgpr(s32) = COPY $vgpr0
     %2:vgpr(s32) = COPY $vgpr1
-    %3:vgpr(s64) = COPY $vgpr3_vgpr4
+    %3:vgpr(p1) = COPY $vgpr3_vgpr4
 
     ; GCN: [[SGPR64_0:%[0-9]+]]:sreg_64_xexec = COPY $sgpr10_sgpr11
     ; GCN: [[VGPR64_0:%[0-9]+]]:vreg_64 = COPY $vgpr10_vgpr11
diff --git a/test/CodeGen/AMDGPU/GlobalISel/inst-select-or.mir b/test/CodeGen/AMDGPU/GlobalISel/inst-select-or.mir
index 60e1a4cdd5ceb2bf49a88b9f4b85f5c74706c66c..4d0de1dff5a66e4a132af2258d73e3154d6ed35e 100644
--- a/test/CodeGen/AMDGPU/GlobalISel/inst-select-or.mir
+++ b/test/CodeGen/AMDGPU/GlobalISel/inst-select-or.mir
@@ -19,7 +19,7 @@ body: |
     %0:sgpr(s32) = COPY $sgpr0
     %1:sgpr(s32) = COPY $sgpr1
     %2:vgpr(s32) = COPY $vgpr0
-    %3:vgpr(s64) = COPY $vgpr3_vgpr4
+    %3:vgpr(p1) = COPY $vgpr3_vgpr4
     %4:sgpr(s32) = G_CONSTANT i32 1
     %5:sgpr(s32) = G_CONSTANT i32 4096
 
@@ -39,7 +39,7 @@ body: |
     ; GCN: [[VV:%[0-9]+]]:vgpr_32 = V_OR_B32_e32 [[SV]], [[VGPR0]]
     %9:vgpr(s32) = G_OR %8, %2
 
-    G_STORE %9, %3 :: (store 4 into %ir.global0)
+    G_STORE %9, %3 :: (store 4 into %ir.global0, addrspace 1)
 
 ...
 ---
diff --git a/test/CodeGen/AMDGPU/GlobalISel/inst-select-sitofp.mir b/test/CodeGen/AMDGPU/GlobalISel/inst-select-sitofp.mir
index 42fc095985aeefaabdff0b8cf971268e4655541c..bedcaf01f36d9f7e26886626a6f8ba4244084d4c 100644
--- a/test/CodeGen/AMDGPU/GlobalISel/inst-select-sitofp.mir
+++ b/test/CodeGen/AMDGPU/GlobalISel/inst-select-sitofp.mir
@@ -20,7 +20,7 @@ body: |
     ; GCN: [[VGPR:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     %1:vgpr(s32) = COPY $vgpr0
 
-    %2:vgpr(s64) = COPY $vgpr3_vgpr4
+    %2:vgpr(p1) = COPY $vgpr3_vgpr4
 
     ; sitofp s
     ; GCN: V_CVT_F32_I32_e64 [[SGPR]], 0, 0
diff --git a/test/CodeGen/AMDGPU/GlobalISel/legalize-add.mir b/test/CodeGen/AMDGPU/GlobalISel/legalize-add.mir
index 57e1148bc59eefbce645dcac8a2a43023a156fb8..69b0b8b7be338d77dbf77c9ef6d281aa4d8f03fd 100644
--- a/test/CodeGen/AMDGPU/GlobalISel/legalize-add.mir
+++ b/test/CodeGen/AMDGPU/GlobalISel/legalize-add.mir
@@ -2,17 +2,39 @@
 # RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -O0 -run-pass=legalizer %s -o - | FileCheck %s
 
 ---
-name:            test_add
+name: test_add_s32
 body: |
   bb.0:
     liveins: $vgpr0, $vgpr1
 
-    ; CHECK-LABEL: name: test_add
+    ; CHECK-LABEL: name: test_add_s32
     ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
     ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
     ; CHECK: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[COPY]], [[COPY1]]
+    ; CHECK: $vgpr0 = COPY [[ADD]](s32)
     %0:_(s32) = COPY $vgpr0
     %1:_(s32) = COPY $vgpr1
     %2:_(s32) = G_ADD %0, %1
     $vgpr0 = COPY %2
 ...
+
+---
+name: test_add_v2s32
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
+
+    ; CHECK-LABEL: name: test_add_v2s32
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    ; CHECK: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr2_vgpr3
+    ; CHECK: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
+    ; CHECK: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<2 x s32>)
+    ; CHECK: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[UV]], [[UV2]]
+    ; CHECK: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[UV1]], [[UV3]]
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[ADD]](s32), [[ADD1]](s32)
+    ; CHECK: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
+    %0:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    %1:_(<2 x s32>) = COPY $vgpr2_vgpr3
+    %2:_(<2 x s32>) = G_ADD %0, %1
+    $vgpr0_vgpr1 = COPY %2
+...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/legalize-addrspacecast.mir b/test/CodeGen/AMDGPU/GlobalISel/legalize-addrspacecast.mir
new file mode 100644
index 0000000000000000000000000000000000000000..af039bcada8ee8b079d45617066942207db4a696
--- /dev/null
+++ b/test/CodeGen/AMDGPU/GlobalISel/legalize-addrspacecast.mir
@@ -0,0 +1,393 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -march=amdgcn -mcpu=fiji -O0 -run-pass=legalizer -global-isel-abort=0 -o - %s  | FileCheck -check-prefix=VI %s
+# RUN: llc -march=amdgcn -mcpu=gfx900 -O0 -run-pass=legalizer -global-isel-abort=0 -o - %s  | FileCheck -check-prefix=GFX9 %s
+
+---
+name: test_addrspacecast_p0_to_p1
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; VI-LABEL: name: test_addrspacecast_p0_to_p1
+    ; VI: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
+    ; VI: [[BITCAST:%[0-9]+]]:_(p1) = G_BITCAST [[COPY]](p0)
+    ; VI: $vgpr0_vgpr1 = COPY [[BITCAST]](p1)
+    ; GFX9-LABEL: name: test_addrspacecast_p0_to_p1
+    ; GFX9: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
+    ; GFX9: [[BITCAST:%[0-9]+]]:_(p1) = G_BITCAST [[COPY]](p0)
+    ; GFX9: $vgpr0_vgpr1 = COPY [[BITCAST]](p1)
+    %0:_(p0) = COPY $vgpr0_vgpr1
+    %1:_(p1) = G_ADDRSPACE_CAST %0
+    $vgpr0_vgpr1 = COPY %1
+...
+
+---
+name: test_addrspacecast_p1_to_p0
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; VI-LABEL: name: test_addrspacecast_p1_to_p0
+    ; VI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
+    ; VI: [[BITCAST:%[0-9]+]]:_(p0) = G_BITCAST [[COPY]](p1)
+    ; VI: $vgpr0_vgpr1 = COPY [[BITCAST]](p0)
+    ; GFX9-LABEL: name: test_addrspacecast_p1_to_p0
+    ; GFX9: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
+    ; GFX9: [[BITCAST:%[0-9]+]]:_(p0) = G_BITCAST [[COPY]](p1)
+    ; GFX9: $vgpr0_vgpr1 = COPY [[BITCAST]](p0)
+    %0:_(p1) = COPY $vgpr0_vgpr1
+    %1:_(p0) = G_ADDRSPACE_CAST %0
+    $vgpr0_vgpr1 = COPY %1
+...
+
+---
+name: test_addrspacecast_p0_to_p4
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; VI-LABEL: name: test_addrspacecast_p0_to_p4
+    ; VI: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
+    ; VI: [[BITCAST:%[0-9]+]]:_(p4) = G_BITCAST [[COPY]](p0)
+    ; VI: $vgpr0_vgpr1 = COPY [[BITCAST]](p4)
+    ; GFX9-LABEL: name: test_addrspacecast_p0_to_p4
+    ; GFX9: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
+    ; GFX9: [[BITCAST:%[0-9]+]]:_(p4) = G_BITCAST [[COPY]](p0)
+    ; GFX9: $vgpr0_vgpr1 = COPY [[BITCAST]](p4)
+    %0:_(p0) = COPY $vgpr0_vgpr1
+    %1:_(p4) = G_ADDRSPACE_CAST %0
+    $vgpr0_vgpr1 = COPY %1
+...
+
+---
+name: test_addrspacecast_p4_to_p0
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; VI-LABEL: name: test_addrspacecast_p4_to_p0
+    ; VI: [[COPY:%[0-9]+]]:_(p4) = COPY $vgpr0_vgpr1
+    ; VI: [[BITCAST:%[0-9]+]]:_(p0) = G_BITCAST [[COPY]](p4)
+    ; VI: $vgpr0_vgpr1 = COPY [[BITCAST]](p0)
+    ; GFX9-LABEL: name: test_addrspacecast_p4_to_p0
+    ; GFX9: [[COPY:%[0-9]+]]:_(p4) = COPY $vgpr0_vgpr1
+    ; GFX9: [[BITCAST:%[0-9]+]]:_(p0) = G_BITCAST [[COPY]](p4)
+    ; GFX9: $vgpr0_vgpr1 = COPY [[BITCAST]](p0)
+    %0:_(p4) = COPY $vgpr0_vgpr1
+    %1:_(p0) = G_ADDRSPACE_CAST %0
+    $vgpr0_vgpr1 = COPY %1
+...
+
+---
+name: test_addrspacecast_p0_to_p999
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; VI-LABEL: name: test_addrspacecast_p0_to_p999
+    ; VI: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
+    ; VI: [[BITCAST:%[0-9]+]]:_(p999) = G_BITCAST [[COPY]](p0)
+    ; VI: $vgpr0_vgpr1 = COPY [[BITCAST]](p999)
+    ; GFX9-LABEL: name: test_addrspacecast_p0_to_p999
+    ; GFX9: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
+    ; GFX9: [[BITCAST:%[0-9]+]]:_(p999) = G_BITCAST [[COPY]](p0)
+    ; GFX9: $vgpr0_vgpr1 = COPY [[BITCAST]](p999)
+    %0:_(p0) = COPY $vgpr0_vgpr1
+    %1:_(p999) = G_ADDRSPACE_CAST %0
+    $vgpr0_vgpr1 = COPY %1
+...
+
+---
+name: test_addrspacecast_p999_to_p0
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; VI-LABEL: name: test_addrspacecast_p999_to_p0
+    ; VI: [[COPY:%[0-9]+]]:_(p999) = COPY $vgpr0_vgpr1
+    ; VI: [[BITCAST:%[0-9]+]]:_(p0) = G_BITCAST [[COPY]](p999)
+    ; VI: $vgpr0_vgpr1 = COPY [[BITCAST]](p0)
+    ; GFX9-LABEL: name: test_addrspacecast_p999_to_p0
+    ; GFX9: [[COPY:%[0-9]+]]:_(p999) = COPY $vgpr0_vgpr1
+    ; GFX9: [[BITCAST:%[0-9]+]]:_(p0) = G_BITCAST [[COPY]](p999)
+    ; GFX9: $vgpr0_vgpr1 = COPY [[BITCAST]](p0)
+    %0:_(p999) = COPY $vgpr0_vgpr1
+    %1:_(p0) = G_ADDRSPACE_CAST %0
+    $vgpr0_vgpr1 = COPY %1
+...
+
+---
+name: test_addrspacecast_p5_to_p0
+body: |
+  bb.0:
+    liveins: $vgpr0
+
+    ; VI-LABEL: name: test_addrspacecast_p5_to_p0
+    ; VI: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
+    ; VI: [[C:%[0-9]+]]:_(p5) = G_CONSTANT i32 0
+    ; VI: [[C1:%[0-9]+]]:_(p0) = G_CONSTANT i64 0
+    ; VI: [[C2:%[0-9]+]]:_(p4) = G_CONSTANT i64 3735928559
+    ; VI: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 68
+    ; VI: [[GEP:%[0-9]+]]:_(p4) = G_GEP [[C2]], [[C3]](s64)
+    ; VI: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[GEP]](p4) :: (dereferenceable invariant load 4 from `i8 addrspace(4)* undef` + 68, addrspace 4)
+    ; VI: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[COPY]](p5), [[C]]
+    ; VI: [[PTRTOINT:%[0-9]+]]:_(s32) = G_PTRTOINT [[COPY]](p5)
+    ; VI: [[MV:%[0-9]+]]:_(p0) = G_MERGE_VALUES [[PTRTOINT]](s32), [[LOAD]](s32)
+    ; VI: [[SELECT:%[0-9]+]]:_(p0) = G_SELECT [[ICMP]](s1), [[MV]], [[C1]]
+    ; VI: $vgpr0_vgpr1 = COPY [[SELECT]](p0)
+    ; GFX9-LABEL: name: test_addrspacecast_p5_to_p0
+    ; GFX9: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
+    ; GFX9: [[C:%[0-9]+]]:_(p5) = G_CONSTANT i32 0
+    ; GFX9: [[C1:%[0-9]+]]:_(p0) = G_CONSTANT i64 0
+    ; GFX9: [[S_GETREG_B32_:%[0-9]+]]:sreg_32(s32) = S_GETREG_B32 30735
+    ; GFX9: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; GFX9: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[S_GETREG_B32_]], [[C2]](s32)
+    ; GFX9: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[COPY]](p5), [[C]]
+    ; GFX9: [[PTRTOINT:%[0-9]+]]:_(s32) = G_PTRTOINT [[COPY]](p5)
+    ; GFX9: [[MV:%[0-9]+]]:_(p0) = G_MERGE_VALUES [[PTRTOINT]](s32), [[SHL]](s32)
+    ; GFX9: [[SELECT:%[0-9]+]]:_(p0) = G_SELECT [[ICMP]](s1), [[MV]], [[C1]]
+    ; GFX9: $vgpr0_vgpr1 = COPY [[SELECT]](p0)
+    %0:_(p5) = COPY $vgpr0
+    %1:_(p0) = G_ADDRSPACE_CAST %0
+    $vgpr0_vgpr1 = COPY %1
+...
+
+---
+name: test_addrspacecast_p0_to_p5
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; VI-LABEL: name: test_addrspacecast_p0_to_p5
+    ; VI: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
+    ; VI: [[C:%[0-9]+]]:_(p5) = G_CONSTANT i32 0
+    ; VI: [[C1:%[0-9]+]]:_(p0) = G_CONSTANT i64 0
+    ; VI: [[EXTRACT:%[0-9]+]]:_(p5) = G_EXTRACT [[COPY]](p0), 0
+    ; VI: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[COPY]](p0), [[C1]]
+    ; VI: [[SELECT:%[0-9]+]]:_(p5) = G_SELECT [[ICMP]](s1), [[EXTRACT]], [[C]]
+    ; VI: $vgpr0 = COPY [[SELECT]](p5)
+    ; GFX9-LABEL: name: test_addrspacecast_p0_to_p5
+    ; GFX9: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
+    ; GFX9: [[C:%[0-9]+]]:_(p5) = G_CONSTANT i32 0
+    ; GFX9: [[C1:%[0-9]+]]:_(p0) = G_CONSTANT i64 0
+    ; GFX9: [[EXTRACT:%[0-9]+]]:_(p5) = G_EXTRACT [[COPY]](p0), 0
+    ; GFX9: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[COPY]](p0), [[C1]]
+    ; GFX9: [[SELECT:%[0-9]+]]:_(p5) = G_SELECT [[ICMP]](s1), [[EXTRACT]], [[C]]
+    ; GFX9: $vgpr0 = COPY [[SELECT]](p5)
+    %0:_(p0) = COPY $vgpr0_vgpr1
+    %1:_(p5) = G_ADDRSPACE_CAST %0
+    $vgpr0 = COPY %1
+...
+
+---
+name: test_addrspacecast_p3_to_p0
+body: |
+  bb.0:
+    liveins: $vgpr0
+
+    ; VI-LABEL: name: test_addrspacecast_p3_to_p0
+    ; VI: [[COPY:%[0-9]+]]:_(p3) = COPY $vgpr0
+    ; VI: [[C:%[0-9]+]]:_(p3) = G_CONSTANT i32 -1
+    ; VI: [[C1:%[0-9]+]]:_(p0) = G_CONSTANT i64 0
+    ; VI: [[C2:%[0-9]+]]:_(p4) = G_CONSTANT i64 3735928559
+    ; VI: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 68
+    ; VI: [[GEP:%[0-9]+]]:_(p4) = G_GEP [[C2]], [[C3]](s64)
+    ; VI: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[GEP]](p4) :: (dereferenceable invariant load 4 from `i8 addrspace(4)* undef` + 68, addrspace 4)
+    ; VI: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[COPY]](p3), [[C]]
+    ; VI: [[PTRTOINT:%[0-9]+]]:_(s32) = G_PTRTOINT [[COPY]](p3)
+    ; VI: [[MV:%[0-9]+]]:_(p0) = G_MERGE_VALUES [[PTRTOINT]](s32), [[LOAD]](s32)
+    ; VI: [[SELECT:%[0-9]+]]:_(p0) = G_SELECT [[ICMP]](s1), [[MV]], [[C1]]
+    ; VI: $vgpr0_vgpr1 = COPY [[SELECT]](p0)
+    ; GFX9-LABEL: name: test_addrspacecast_p3_to_p0
+    ; GFX9: [[COPY:%[0-9]+]]:_(p3) = COPY $vgpr0
+    ; GFX9: [[C:%[0-9]+]]:_(p3) = G_CONSTANT i32 -1
+    ; GFX9: [[C1:%[0-9]+]]:_(p0) = G_CONSTANT i64 0
+    ; GFX9: [[S_GETREG_B32_:%[0-9]+]]:sreg_32(s32) = S_GETREG_B32 30735
+    ; GFX9: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; GFX9: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[S_GETREG_B32_]], [[C2]](s32)
+    ; GFX9: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[COPY]](p3), [[C]]
+    ; GFX9: [[PTRTOINT:%[0-9]+]]:_(s32) = G_PTRTOINT [[COPY]](p3)
+    ; GFX9: [[MV:%[0-9]+]]:_(p0) = G_MERGE_VALUES [[PTRTOINT]](s32), [[SHL]](s32)
+    ; GFX9: [[SELECT:%[0-9]+]]:_(p0) = G_SELECT [[ICMP]](s1), [[MV]], [[C1]]
+    ; GFX9: $vgpr0_vgpr1 = COPY [[SELECT]](p0)
+    %0:_(p3) = COPY $vgpr0
+    %1:_(p0) = G_ADDRSPACE_CAST %0
+    $vgpr0_vgpr1 = COPY %1
+...
+
+---
+name: test_addrspacecast_p0_to_p3
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; VI-LABEL: name: test_addrspacecast_p0_to_p3
+    ; VI: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
+    ; VI: [[C:%[0-9]+]]:_(p3) = G_CONSTANT i32 -1
+    ; VI: [[C1:%[0-9]+]]:_(p0) = G_CONSTANT i64 0
+    ; VI: [[EXTRACT:%[0-9]+]]:_(p3) = G_EXTRACT [[COPY]](p0), 0
+    ; VI: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[COPY]](p0), [[C1]]
+    ; VI: [[SELECT:%[0-9]+]]:_(p3) = G_SELECT [[ICMP]](s1), [[EXTRACT]], [[C]]
+    ; VI: $vgpr0 = COPY [[SELECT]](p3)
+    ; GFX9-LABEL: name: test_addrspacecast_p0_to_p3
+    ; GFX9: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
+    ; GFX9: [[C:%[0-9]+]]:_(p3) = G_CONSTANT i32 -1
+    ; GFX9: [[C1:%[0-9]+]]:_(p0) = G_CONSTANT i64 0
+    ; GFX9: [[EXTRACT:%[0-9]+]]:_(p3) = G_EXTRACT [[COPY]](p0), 0
+    ; GFX9: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[COPY]](p0), [[C1]]
+    ; GFX9: [[SELECT:%[0-9]+]]:_(p3) = G_SELECT [[ICMP]](s1), [[EXTRACT]], [[C]]
+    ; GFX9: $vgpr0 = COPY [[SELECT]](p3)
+    %0:_(p0) = COPY $vgpr0_vgpr1
+    %1:_(p3) = G_ADDRSPACE_CAST %0
+    $vgpr0 = COPY %1
+...
+
+---
+name: test_addrspacecast_v2p0_to_v2p1
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3
+
+    ; VI-LABEL: name: test_addrspacecast_v2p0_to_v2p1
+    ; VI: [[COPY:%[0-9]+]]:_(<2 x p0>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; VI: [[UV:%[0-9]+]]:_(p0), [[UV1:%[0-9]+]]:_(p0) = G_UNMERGE_VALUES [[COPY]](<2 x p0>)
+    ; VI: [[BITCAST:%[0-9]+]]:_(p1) = G_BITCAST [[UV]](p0)
+    ; VI: [[BITCAST1:%[0-9]+]]:_(p1) = G_BITCAST [[UV1]](p0)
+    ; VI: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x p1>) = G_BUILD_VECTOR [[BITCAST]](p1), [[BITCAST1]](p1)
+    ; VI: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x p1>)
+    ; GFX9-LABEL: name: test_addrspacecast_v2p0_to_v2p1
+    ; GFX9: [[COPY:%[0-9]+]]:_(<2 x p0>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; GFX9: [[UV:%[0-9]+]]:_(p0), [[UV1:%[0-9]+]]:_(p0) = G_UNMERGE_VALUES [[COPY]](<2 x p0>)
+    ; GFX9: [[BITCAST:%[0-9]+]]:_(p1) = G_BITCAST [[UV]](p0)
+    ; GFX9: [[BITCAST1:%[0-9]+]]:_(p1) = G_BITCAST [[UV1]](p0)
+    ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x p1>) = G_BUILD_VECTOR [[BITCAST]](p1), [[BITCAST1]](p1)
+    ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x p1>)
+    %0:_(<2 x p0>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    %1:_(<2 x p1>) = G_ADDRSPACE_CAST %0
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %1
+...
+
+---
+name: test_addrspacecast_v2p1_to_v2p0
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3
+
+    ; VI-LABEL: name: test_addrspacecast_v2p1_to_v2p0
+    ; VI: [[COPY:%[0-9]+]]:_(<2 x p1>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; VI: [[UV:%[0-9]+]]:_(p1), [[UV1:%[0-9]+]]:_(p1) = G_UNMERGE_VALUES [[COPY]](<2 x p1>)
+    ; VI: [[BITCAST:%[0-9]+]]:_(p0) = G_BITCAST [[UV]](p1)
+    ; VI: [[BITCAST1:%[0-9]+]]:_(p0) = G_BITCAST [[UV1]](p1)
+    ; VI: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x p0>) = G_BUILD_VECTOR [[BITCAST]](p0), [[BITCAST1]](p0)
+    ; VI: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x p0>)
+    ; GFX9-LABEL: name: test_addrspacecast_v2p1_to_v2p0
+    ; GFX9: [[COPY:%[0-9]+]]:_(<2 x p1>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; GFX9: [[UV:%[0-9]+]]:_(p1), [[UV1:%[0-9]+]]:_(p1) = G_UNMERGE_VALUES [[COPY]](<2 x p1>)
+    ; GFX9: [[BITCAST:%[0-9]+]]:_(p0) = G_BITCAST [[UV]](p1)
+    ; GFX9: [[BITCAST1:%[0-9]+]]:_(p0) = G_BITCAST [[UV1]](p1)
+    ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x p0>) = G_BUILD_VECTOR [[BITCAST]](p0), [[BITCAST1]](p0)
+    ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x p0>)
+    %0:_(<2 x p1>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    %1:_(<2 x p0>) = G_ADDRSPACE_CAST %0
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %1
+...
+
+---
+name: test_addrspacecast_v2p0_to_v2p3
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3
+
+    ; VI-LABEL: name: test_addrspacecast_v2p0_to_v2p3
+    ; VI: [[COPY:%[0-9]+]]:_(<2 x p0>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; VI: [[UV:%[0-9]+]]:_(p0), [[UV1:%[0-9]+]]:_(p0) = G_UNMERGE_VALUES [[COPY]](<2 x p0>)
+    ; VI: [[C:%[0-9]+]]:_(p3) = G_CONSTANT i32 -1
+    ; VI: [[C1:%[0-9]+]]:_(p0) = G_CONSTANT i64 0
+    ; VI: [[EXTRACT:%[0-9]+]]:_(p3) = G_EXTRACT [[UV]](p0), 0
+    ; VI: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[UV]](p0), [[C1]]
+    ; VI: [[SELECT:%[0-9]+]]:_(p3) = G_SELECT [[ICMP]](s1), [[EXTRACT]], [[C]]
+    ; VI: [[C2:%[0-9]+]]:_(p3) = G_CONSTANT i32 -1
+    ; VI: [[C3:%[0-9]+]]:_(p0) = G_CONSTANT i64 0
+    ; VI: [[EXTRACT1:%[0-9]+]]:_(p3) = G_EXTRACT [[UV1]](p0), 0
+    ; VI: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[UV1]](p0), [[C3]]
+    ; VI: [[SELECT1:%[0-9]+]]:_(p3) = G_SELECT [[ICMP1]](s1), [[EXTRACT1]], [[C2]]
+    ; VI: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x p3>) = G_BUILD_VECTOR [[SELECT]](p3), [[SELECT1]](p3)
+    ; VI: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x p3>)
+    ; GFX9-LABEL: name: test_addrspacecast_v2p0_to_v2p3
+    ; GFX9: [[COPY:%[0-9]+]]:_(<2 x p0>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; GFX9: [[UV:%[0-9]+]]:_(p0), [[UV1:%[0-9]+]]:_(p0) = G_UNMERGE_VALUES [[COPY]](<2 x p0>)
+    ; GFX9: [[C:%[0-9]+]]:_(p3) = G_CONSTANT i32 -1
+    ; GFX9: [[C1:%[0-9]+]]:_(p0) = G_CONSTANT i64 0
+    ; GFX9: [[EXTRACT:%[0-9]+]]:_(p3) = G_EXTRACT [[UV]](p0), 0
+    ; GFX9: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[UV]](p0), [[C1]]
+    ; GFX9: [[SELECT:%[0-9]+]]:_(p3) = G_SELECT [[ICMP]](s1), [[EXTRACT]], [[C]]
+    ; GFX9: [[C2:%[0-9]+]]:_(p3) = G_CONSTANT i32 -1
+    ; GFX9: [[C3:%[0-9]+]]:_(p0) = G_CONSTANT i64 0
+    ; GFX9: [[EXTRACT1:%[0-9]+]]:_(p3) = G_EXTRACT [[UV1]](p0), 0
+    ; GFX9: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[UV1]](p0), [[C3]]
+    ; GFX9: [[SELECT1:%[0-9]+]]:_(p3) = G_SELECT [[ICMP1]](s1), [[EXTRACT1]], [[C2]]
+    ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x p3>) = G_BUILD_VECTOR [[SELECT]](p3), [[SELECT1]](p3)
+    ; GFX9: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x p3>)
+    %0:_(<2 x p0>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    %1:_(<2 x p3>) = G_ADDRSPACE_CAST %0
+    $vgpr0_vgpr1 = COPY %1
+...
+
+---
+name: test_addrspacecast_v2p3_to_v2p0
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; VI-LABEL: name: test_addrspacecast_v2p3_to_v2p0
+    ; VI: [[COPY:%[0-9]+]]:_(<2 x p3>) = COPY $vgpr0_vgpr1
+    ; VI: [[UV:%[0-9]+]]:_(p3), [[UV1:%[0-9]+]]:_(p3) = G_UNMERGE_VALUES [[COPY]](<2 x p3>)
+    ; VI: [[C:%[0-9]+]]:_(p3) = G_CONSTANT i32 -1
+    ; VI: [[C1:%[0-9]+]]:_(p0) = G_CONSTANT i64 0
+    ; VI: [[C2:%[0-9]+]]:_(p4) = G_CONSTANT i64 3735928559
+    ; VI: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 68
+    ; VI: [[GEP:%[0-9]+]]:_(p4) = G_GEP [[C2]], [[C3]](s64)
+    ; VI: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[GEP]](p4) :: (dereferenceable invariant load 4 from `i8 addrspace(4)* undef` + 68, addrspace 4)
+    ; VI: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[UV]](p3), [[C]]
+    ; VI: [[PTRTOINT:%[0-9]+]]:_(s32) = G_PTRTOINT [[UV]](p3)
+    ; VI: [[MV:%[0-9]+]]:_(p0) = G_MERGE_VALUES [[PTRTOINT]](s32), [[LOAD]](s32)
+    ; VI: [[SELECT:%[0-9]+]]:_(p0) = G_SELECT [[ICMP]](s1), [[MV]], [[C1]]
+    ; VI: [[C4:%[0-9]+]]:_(p3) = G_CONSTANT i32 -1
+    ; VI: [[C5:%[0-9]+]]:_(p0) = G_CONSTANT i64 0
+    ; VI: [[C6:%[0-9]+]]:_(p4) = G_CONSTANT i64 3735928559
+    ; VI: [[C7:%[0-9]+]]:_(s64) = G_CONSTANT i64 68
+    ; VI: [[GEP1:%[0-9]+]]:_(p4) = G_GEP [[C6]], [[C7]](s64)
+    ; VI: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[GEP1]](p4) :: (dereferenceable invariant load 4 from `i8 addrspace(4)* undef` + 68, addrspace 4)
+    ; VI: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[UV1]](p3), [[C4]]
+    ; VI: [[PTRTOINT1:%[0-9]+]]:_(s32) = G_PTRTOINT [[UV1]](p3)
+    ; VI: [[MV1:%[0-9]+]]:_(p0) = G_MERGE_VALUES [[PTRTOINT1]](s32), [[LOAD1]](s32)
+    ; VI: [[SELECT1:%[0-9]+]]:_(p0) = G_SELECT [[ICMP1]](s1), [[MV1]], [[C5]]
+    ; VI: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x p0>) = G_BUILD_VECTOR [[SELECT]](p0), [[SELECT1]](p0)
+    ; VI: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x p0>)
+    ; GFX9-LABEL: name: test_addrspacecast_v2p3_to_v2p0
+    ; GFX9: [[COPY:%[0-9]+]]:_(<2 x p3>) = COPY $vgpr0_vgpr1
+    ; GFX9: [[UV:%[0-9]+]]:_(p3), [[UV1:%[0-9]+]]:_(p3) = G_UNMERGE_VALUES [[COPY]](<2 x p3>)
+    ; GFX9: [[C:%[0-9]+]]:_(p3) = G_CONSTANT i32 -1
+    ; GFX9: [[C1:%[0-9]+]]:_(p0) = G_CONSTANT i64 0
+    ; GFX9: [[S_GETREG_B32_:%[0-9]+]]:sreg_32(s32) = S_GETREG_B32 30735
+    ; GFX9: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; GFX9: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[S_GETREG_B32_]], [[C2]](s32)
+    ; GFX9: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[UV]](p3), [[C]]
+    ; GFX9: [[PTRTOINT:%[0-9]+]]:_(s32) = G_PTRTOINT [[UV]](p3)
+    ; GFX9: [[MV:%[0-9]+]]:_(p0) = G_MERGE_VALUES [[PTRTOINT]](s32), [[SHL]](s32)
+    ; GFX9: [[SELECT:%[0-9]+]]:_(p0) = G_SELECT [[ICMP]](s1), [[MV]], [[C1]]
+    ; GFX9: [[C3:%[0-9]+]]:_(p3) = G_CONSTANT i32 -1
+    ; GFX9: [[C4:%[0-9]+]]:_(p0) = G_CONSTANT i64 0
+    ; GFX9: [[S_GETREG_B32_1:%[0-9]+]]:sreg_32(s32) = S_GETREG_B32 30735
+    ; GFX9: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; GFX9: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[S_GETREG_B32_1]], [[C5]](s32)
+    ; GFX9: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[UV1]](p3), [[C3]]
+    ; GFX9: [[PTRTOINT1:%[0-9]+]]:_(s32) = G_PTRTOINT [[UV1]](p3)
+    ; GFX9: [[MV1:%[0-9]+]]:_(p0) = G_MERGE_VALUES [[PTRTOINT1]](s32), [[SHL1]](s32)
+    ; GFX9: [[SELECT1:%[0-9]+]]:_(p0) = G_SELECT [[ICMP1]](s1), [[MV1]], [[C4]]
+    ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x p0>) = G_BUILD_VECTOR [[SELECT]](p0), [[SELECT1]](p0)
+    ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x p0>)
+    %0:_(<2 x p3>) = COPY $vgpr0_vgpr1
+    %1:_(<2 x p0>) = G_ADDRSPACE_CAST %0
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %1
+...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/legalize-and.mir b/test/CodeGen/AMDGPU/GlobalISel/legalize-and.mir
index f561cc7d092d6fae8ad64c306f850b86284a27f5..6cc198223220bda91755818f571d455e227b27d7 100644
--- a/test/CodeGen/AMDGPU/GlobalISel/legalize-and.mir
+++ b/test/CodeGen/AMDGPU/GlobalISel/legalize-and.mir
@@ -2,17 +2,327 @@
 # RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -O0 -run-pass=legalizer %s -o - | FileCheck %s
 
 ---
-name:            test_and
+name: test_and_s32
 body: |
   bb.0:
     liveins: $vgpr0, $vgpr1
 
-    ; CHECK-LABEL: name: test_and
+    ; CHECK-LABEL: name: test_and_s32
     ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
     ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
     ; CHECK: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[COPY1]]
+    ; CHECK: $vgpr0 = COPY [[AND]](s32)
     %0:_(s32) = COPY $vgpr0
     %1:_(s32) = COPY $vgpr1
     %2:_(s32) = G_AND %0, %1
     $vgpr0 = COPY %2
 ...
+
+---
+name: test_and_s1
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+
+    ; CHECK-LABEL: name: test_and_s1
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; CHECK: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[COPY1]]
+    ; CHECK: S_NOP 0, implicit [[AND]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s32) = G_CONSTANT i32 0
+    %3:_(s1) = G_ICMP intpred(ne), %0, %2
+    %4:_(s1) = G_ICMP intpred(ne), %1, %2
+    %5:_(s32) = G_AND %0, %1
+    S_NOP 0, implicit %5
+...
+
+---
+name: test_and_s64
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
+
+    ; CHECK-LABEL: name: test_and_s64
+    ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s64) = COPY $vgpr2_vgpr3
+    ; CHECK: [[AND:%[0-9]+]]:_(s64) = G_AND [[COPY]], [[COPY1]]
+    ; CHECK: $vgpr0_vgpr1 = COPY [[AND]](s64)
+    %0:_(s64) = COPY $vgpr0_vgpr1
+    %1:_(s64) = COPY $vgpr2_vgpr3
+    %2:_(s64) = G_AND %0, %1
+    $vgpr0_vgpr1 = COPY %2
+...
+
+---
+name: test_and_s7
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+
+    ; CHECK-LABEL: name: test_and_s7
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; CHECK: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; CHECK: [[COPY3:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+    ; CHECK: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[COPY3]]
+    ; CHECK: [[COPY4:%[0-9]+]]:_(s32) = COPY [[AND]](s32)
+    ; CHECK: $vgpr0 = COPY [[COPY4]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s7) = G_TRUNC %0
+    %3:_(s7) = G_TRUNC %1
+    %4:_(s7) = G_AND %2, %3
+    %5:_(s32) = G_ANYEXT %4
+    $vgpr0 = COPY %5
+...
+
+---
+name: test_and_s8
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+
+    ; CHECK-LABEL: name: test_and_s8
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; CHECK: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; CHECK: [[COPY3:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+    ; CHECK: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[COPY3]]
+    ; CHECK: [[COPY4:%[0-9]+]]:_(s32) = COPY [[AND]](s32)
+    ; CHECK: $vgpr0 = COPY [[COPY4]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s8) = G_TRUNC %0
+    %3:_(s8) = G_TRUNC %1
+    %4:_(s8) = G_AND %2, %3
+    %5:_(s32) = G_ANYEXT %4
+    $vgpr0 = COPY %5
+...
+
+---
+name: test_and_s16
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+
+    ; CHECK-LABEL: name: test_and_s16
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; CHECK: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; CHECK: [[COPY3:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+    ; CHECK: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[COPY3]]
+    ; CHECK: [[COPY4:%[0-9]+]]:_(s32) = COPY [[AND]](s32)
+    ; CHECK: $vgpr0 = COPY [[COPY4]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s16) = G_TRUNC %0
+    %3:_(s16) = G_TRUNC %1
+    %4:_(s16) = G_AND %2, %3
+    %5:_(s32) = G_ANYEXT %4
+    $vgpr0 = COPY %5
+...
+
+---
+name: test_and_s24
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+
+    ; CHECK-LABEL: name: test_and_s24
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; CHECK: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; CHECK: [[COPY3:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+    ; CHECK: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[COPY3]]
+    ; CHECK: [[COPY4:%[0-9]+]]:_(s32) = COPY [[AND]](s32)
+    ; CHECK: $vgpr0 = COPY [[COPY4]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s24) = G_TRUNC %0
+    %3:_(s24) = G_TRUNC %1
+    %4:_(s24) = G_AND %2, %3
+    %5:_(s32) = G_ANYEXT %4
+    $vgpr0 = COPY %5
+...
+
+---
+name: test_and_v2s32
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
+
+    ; CHECK-LABEL: name: test_and_v2s32
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    ; CHECK: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr2_vgpr3
+    ; CHECK: [[AND:%[0-9]+]]:_(<2 x s32>) = G_AND [[COPY]], [[COPY1]]
+    ; CHECK: $vgpr0_vgpr1 = COPY [[AND]](<2 x s32>)
+    %0:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    %1:_(<2 x s32>) = COPY $vgpr2_vgpr3
+    %2:_(<2 x s32>) = G_AND %0, %1
+    $vgpr0_vgpr1 = COPY %2
+...
+
+---
+name: test_and_v3i32
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2, $vgpr3_vgpr4_vgpr5
+
+    ; CHECK-LABEL: name: test_and_v3i32
+    ; CHECK: [[COPY:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr0_vgpr1_vgpr2
+    ; CHECK: [[COPY1:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr3_vgpr4_vgpr5
+    ; CHECK: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<3 x s32>)
+    ; CHECK: [[UV3:%[0-9]+]]:_(s32), [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<3 x s32>)
+    ; CHECK: [[AND:%[0-9]+]]:_(s32) = G_AND [[UV]], [[UV3]]
+    ; CHECK: [[AND1:%[0-9]+]]:_(s32) = G_AND [[UV1]], [[UV4]]
+    ; CHECK: [[AND2:%[0-9]+]]:_(s32) = G_AND [[UV2]], [[UV5]]
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[AND]](s32), [[AND1]](s32), [[AND2]](s32)
+    ; CHECK: $vgpr0_vgpr1_vgpr2 = COPY [[BUILD_VECTOR]](<3 x s32>)
+    %0:_(<3 x s32>) = COPY $vgpr0_vgpr1_vgpr2
+    %1:_(<3 x s32>) = COPY $vgpr3_vgpr4_vgpr5
+    %2:_(<3 x s32>) = G_AND %0, %1
+    $vgpr0_vgpr1_vgpr2 = COPY %2
+...
+
+---
+name: test_and_v4i32
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7
+
+    ; CHECK-LABEL: name: test_and_v4i32
+    ; CHECK: [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; CHECK: [[COPY1:%[0-9]+]]:_(<4 x s32>) = COPY $vgpr4_vgpr5_vgpr6_vgpr7
+    ; CHECK: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<4 x s32>)
+    ; CHECK: [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32), [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<4 x s32>)
+    ; CHECK: [[AND:%[0-9]+]]:_(s32) = G_AND [[UV]], [[UV4]]
+    ; CHECK: [[AND1:%[0-9]+]]:_(s32) = G_AND [[UV1]], [[UV5]]
+    ; CHECK: [[AND2:%[0-9]+]]:_(s32) = G_AND [[UV2]], [[UV6]]
+    ; CHECK: [[AND3:%[0-9]+]]:_(s32) = G_AND [[UV3]], [[UV7]]
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[AND]](s32), [[AND1]](s32), [[AND2]](s32), [[AND3]](s32)
+    ; CHECK: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<4 x s32>)
+    %0:_(<4 x s32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    %1:_(<4 x s32>) = COPY $vgpr4_vgpr5_vgpr6_vgpr7
+    %2:_(<4 x s32>) = G_AND %0, %1
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %2
+...
+
+---
+name: test_and_v5i32
+body: |
+  bb.0:
+
+    ; CHECK-LABEL: name: test_and_v5i32
+    ; CHECK: [[DEF:%[0-9]+]]:_(<5 x s32>) = G_IMPLICIT_DEF
+    ; CHECK: [[DEF1:%[0-9]+]]:_(<5 x s32>) = G_IMPLICIT_DEF
+    ; CHECK: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32), [[UV4:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[DEF]](<5 x s32>)
+    ; CHECK: [[UV5:%[0-9]+]]:_(s32), [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32), [[UV8:%[0-9]+]]:_(s32), [[UV9:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[DEF1]](<5 x s32>)
+    ; CHECK: [[AND:%[0-9]+]]:_(s32) = G_AND [[UV]], [[UV5]]
+    ; CHECK: [[AND1:%[0-9]+]]:_(s32) = G_AND [[UV1]], [[UV6]]
+    ; CHECK: [[AND2:%[0-9]+]]:_(s32) = G_AND [[UV2]], [[UV7]]
+    ; CHECK: [[AND3:%[0-9]+]]:_(s32) = G_AND [[UV3]], [[UV8]]
+    ; CHECK: [[AND4:%[0-9]+]]:_(s32) = G_AND [[UV4]], [[UV9]]
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<5 x s32>) = G_BUILD_VECTOR [[AND]](s32), [[AND1]](s32), [[AND2]](s32), [[AND3]](s32), [[AND4]](s32)
+    ; CHECK: [[DEF2:%[0-9]+]]:_(<8 x s32>) = G_IMPLICIT_DEF
+    ; CHECK: [[INSERT:%[0-9]+]]:_(<8 x s32>) = G_INSERT [[DEF2]], [[BUILD_VECTOR]](<5 x s32>), 0
+    ; CHECK: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY [[INSERT]](<8 x s32>)
+    %0:_(<5 x s32>) = G_IMPLICIT_DEF
+    %1:_(<5 x s32>) = G_IMPLICIT_DEF
+    %2:_(<5 x s32>) = G_AND %0, %1
+    %3:_(<8 x s32>) = G_IMPLICIT_DEF
+    %4:_(<8 x s32>) = G_INSERT %3, %2, 0
+    $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY %4
+...
+
+---
+name: test_and_v2s64
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7
+
+    ; CHECK-LABEL: name: test_and_v2s64
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; CHECK: [[COPY1:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr4_vgpr5_vgpr6_vgpr7
+    ; CHECK: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](<2 x s64>)
+    ; CHECK: [[UV2:%[0-9]+]]:_(s64), [[UV3:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY1]](<2 x s64>)
+    ; CHECK: [[AND:%[0-9]+]]:_(s64) = G_AND [[UV]], [[UV2]]
+    ; CHECK: [[AND1:%[0-9]+]]:_(s64) = G_AND [[UV1]], [[UV3]]
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[AND]](s64), [[AND1]](s64)
+    ; CHECK: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>)
+    %0:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    %1:_(<2 x s64>) = COPY $vgpr4_vgpr5_vgpr6_vgpr7
+    %2:_(<2 x s64>) = G_AND %0, %1
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %2
+...
+
+---
+name: test_and_v2s16
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+
+    ; CHECK-LABEL: name: test_and_v2s16
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
+    ; CHECK: [[AND:%[0-9]+]]:_(<2 x s16>) = G_AND [[COPY]], [[COPY1]]
+    ; CHECK: $vgpr0 = COPY [[AND]](<2 x s16>)
+    %0:_(<2 x s16>) = COPY $vgpr0
+    %1:_(<2 x s16>) = COPY $vgpr1
+    %2:_(<2 x s16>) = G_AND %0, %1
+    $vgpr0 = COPY %2
+...
+
+---
+name: test_and_v3s16
+body: |
+  bb.0:
+
+    ; CHECK-LABEL: name: test_and_v3s16
+    ; CHECK: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; CHECK: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
+    ; CHECK: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; CHECK: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF1]](<4 x s16>), 0
+    ; CHECK: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16), [[UV2:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[EXTRACT]](<3 x s16>)
+    ; CHECK: [[UV3:%[0-9]+]]:_(s16), [[UV4:%[0-9]+]]:_(s16), [[UV5:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[EXTRACT1]](<3 x s16>)
+    ; CHECK: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[UV]](s16)
+    ; CHECK: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[UV3]](s16)
+    ; CHECK: [[AND:%[0-9]+]]:_(s32) = G_AND [[ANYEXT]], [[ANYEXT1]]
+    ; CHECK: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[AND]](s32)
+    ; CHECK: [[ANYEXT2:%[0-9]+]]:_(s32) = G_ANYEXT [[UV1]](s16)
+    ; CHECK: [[ANYEXT3:%[0-9]+]]:_(s32) = G_ANYEXT [[UV4]](s16)
+    ; CHECK: [[AND1:%[0-9]+]]:_(s32) = G_AND [[ANYEXT2]], [[ANYEXT3]]
+    ; CHECK: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[AND1]](s32)
+    ; CHECK: [[ANYEXT4:%[0-9]+]]:_(s32) = G_ANYEXT [[UV2]](s16)
+    ; CHECK: [[ANYEXT5:%[0-9]+]]:_(s32) = G_ANYEXT [[UV5]](s16)
+    ; CHECK: [[AND2:%[0-9]+]]:_(s32) = G_AND [[ANYEXT4]], [[ANYEXT5]]
+    ; CHECK: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[AND2]](s32)
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16), [[TRUNC2]](s16)
+    ; CHECK: [[DEF2:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; CHECK: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF2]], [[BUILD_VECTOR]](<3 x s16>), 0
+    ; CHECK: $vgpr0_vgpr1 = COPY [[INSERT]](<4 x s16>)
+    %0:_(<3 x s16>) = G_IMPLICIT_DEF
+    %1:_(<3 x s16>) = G_IMPLICIT_DEF
+    %2:_(<3 x s16>) = G_AND %0, %1
+    %4:_(<4 x s16>) = G_IMPLICIT_DEF
+    %5:_(<4 x s16>) = G_INSERT %4, %2, 0
+    $vgpr0_vgpr1 = COPY %5
+...
+
+---
+name: test_and_v4s16
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
+
+    ; CHECK-LABEL: name: test_and_v4s16
+    ; CHECK: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
+    ; CHECK: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
+    ; CHECK: [[AND:%[0-9]+]]:_(<4 x s16>) = G_AND [[COPY]], [[COPY1]]
+    ; CHECK: $vgpr0_vgpr1 = COPY [[AND]](<4 x s16>)
+    %0:_(<4 x s16>) = COPY $vgpr0_vgpr1
+    %1:_(<4 x s16>) = COPY $vgpr2_vgpr3
+    %2:_(<4 x s16>) = G_AND %0, %1
+    $vgpr0_vgpr1 = COPY %2
+...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/legalize-anyext.mir b/test/CodeGen/AMDGPU/GlobalISel/legalize-anyext.mir
new file mode 100644
index 0000000000000000000000000000000000000000..a7b202a3b0a6f027a53627cd7f607e0d7215a6fc
--- /dev/null
+++ b/test/CodeGen/AMDGPU/GlobalISel/legalize-anyext.mir
@@ -0,0 +1,188 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -run-pass=legalizer %s -o - | FileCheck %s
+
+---
+name: test_anyext_s32_to_s64
+body: |
+  bb.0:
+    liveins: $vgpr0
+
+    ; CHECK-LABEL: name: test_anyext_s32_to_s64
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[COPY]](s32)
+    ; CHECK: $vgpr0_vgpr1 = COPY [[ANYEXT]](s64)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s64) = G_ANYEXT %0
+    $vgpr0_vgpr1 = COPY %1
+...
+
+---
+name: test_anyext_s16_to_s64
+body: |
+  bb.0:
+    liveins: $vgpr0
+
+    ; CHECK-LABEL: name: test_anyext_s16_to_s64
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[COPY]](s32)
+    ; CHECK: $vgpr0_vgpr1 = COPY [[ANYEXT]](s64)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s16) = G_TRUNC %0
+    %2:_(s64) = G_ANYEXT %1
+    $vgpr0_vgpr1 = COPY %2
+...
+
+---
+name: test_anyext_s16_to_s32
+body: |
+  bb.0:
+    liveins: $vgpr0
+
+    ; CHECK-LABEL: name: test_anyext_s16_to_s32
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; CHECK: $vgpr0 = COPY [[COPY1]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s16) = G_TRUNC %0
+    %2:_(s32) = G_ANYEXT %1
+    $vgpr0 = COPY %2
+...
+
+---
+name: test_anyext_s1_to_s32
+body: |
+  bb.0:
+
+    ; CHECK-LABEL: name: test_anyext_s1_to_s32
+    ; CHECK: [[C:%[0-9]+]]:_(s1) = G_CONSTANT i1 false
+    ; CHECK: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[C]](s1)
+    ; CHECK: $vgpr0 = COPY [[ANYEXT]](s32)
+    %0:_(s1) = G_CONSTANT i1 0
+    %1:_(s32) = G_ANYEXT %0
+    $vgpr0 = COPY %1
+...
+
+---
+name: test_anyext_s1_to_s64
+body: |
+  bb.0:
+
+    ; CHECK-LABEL: name: test_anyext_s1_to_s64
+    ; CHECK: [[C:%[0-9]+]]:_(s1) = G_CONSTANT i1 false
+    ; CHECK: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[C]](s1)
+    ; CHECK: $vgpr0_vgpr1 = COPY [[ANYEXT]](s64)
+    %0:_(s1) = G_CONSTANT i1 0
+    %1:_(s64) = G_ANYEXT %0
+    $vgpr0_vgpr1 = COPY %1
+...
+
+---
+name: test_anyext_v2s16_to_v2s32
+body: |
+  bb.0:
+    liveins: $vgpr0
+
+    ; CHECK-LABEL: name: test_anyext_v2s16_to_v2s32
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
+    ; CHECK: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY]](<2 x s16>)
+    ; CHECK: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[UV]](s16)
+    ; CHECK: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[UV1]](s16)
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[ANYEXT]](s32), [[ANYEXT1]](s32)
+    ; CHECK: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
+    %0:_(<2 x s16>) = COPY $vgpr0
+    %1:_(<2 x s32>) = G_ANYEXT %0
+    $vgpr0_vgpr1 = COPY %1
+...
+
+---
+name: test_anyext_v3s16_to_v3s32
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; CHECK-LABEL: name: test_anyext_v3s16_to_v3s32
+    ; CHECK: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
+    ; CHECK: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[COPY]](<4 x s16>), 0
+    ; CHECK: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16), [[UV2:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[EXTRACT]](<3 x s16>)
+    ; CHECK: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[UV]](s16)
+    ; CHECK: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[UV1]](s16)
+    ; CHECK: [[ANYEXT2:%[0-9]+]]:_(s32) = G_ANYEXT [[UV2]](s16)
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[ANYEXT]](s32), [[ANYEXT1]](s32), [[ANYEXT2]](s32)
+    ; CHECK: $vgpr0_vgpr1_vgpr2 = COPY [[BUILD_VECTOR]](<3 x s32>)
+    %0:_(<4 x s16>) = COPY $vgpr0_vgpr1
+    %1:_(<3 x s16>) = G_EXTRACT %0, 0
+    %2:_(<3 x s32>) = G_ANYEXT %1
+    $vgpr0_vgpr1_vgpr2 = COPY %2
+...
+
+---
+name: test_anyext_v4s16_to_v4s32
+body: |
+  bb.0:
+    liveins: $vgpr0
+
+    ; CHECK-LABEL: name: test_anyext_v4s16_to_v4s32
+    ; CHECK: [[DEF:%[0-9]+]]:_(<4 x s32>) = G_IMPLICIT_DEF
+    ; CHECK: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[DEF]](<4 x s32>)
+    %0:_(<4 x s16>) = G_IMPLICIT_DEF
+    %1:_(<4 x s32>) = G_ANYEXT %0
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %1
+...
+
+---
+name: test_anyext_v2s32_to_v2s64
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; CHECK-LABEL: name: test_anyext_v2s32_to_v2s64
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    ; CHECK: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
+    ; CHECK: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[UV]](s32)
+    ; CHECK: [[ANYEXT1:%[0-9]+]]:_(s64) = G_ANYEXT [[UV1]](s32)
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[ANYEXT]](s64), [[ANYEXT1]](s64)
+    ; CHECK: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>)
+    %0:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    %1:_(<2 x s64>) = G_ANYEXT %0
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %1
+...
+
+---
+name: test_anyext_v3s32_to_v3s64
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2
+
+    ; CHECK-LABEL: name: test_anyext_v3s32_to_v3s64
+    ; CHECK: [[COPY:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr0_vgpr1_vgpr2
+    ; CHECK: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<3 x s32>)
+    ; CHECK: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[UV]](s32)
+    ; CHECK: [[ANYEXT1:%[0-9]+]]:_(s64) = G_ANYEXT [[UV1]](s32)
+    ; CHECK: [[ANYEXT2:%[0-9]+]]:_(s64) = G_ANYEXT [[UV2]](s32)
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s64>) = G_BUILD_VECTOR [[ANYEXT]](s64), [[ANYEXT1]](s64), [[ANYEXT2]](s64)
+    ; CHECK: S_NOP 0, implicit [[BUILD_VECTOR]](<3 x s64>)
+    %0:_(<3 x s32>) = COPY $vgpr0_vgpr1_vgpr2
+    %1:_(<3 x s64>) = G_ANYEXT %0
+    S_NOP 0, implicit %1
+
+...
+
+---
+name: test_anyext_v4s32_to_v4s64
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3
+
+    ; CHECK-LABEL: name: test_anyext_v4s32_to_v4s64
+    ; CHECK: [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; CHECK: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<4 x s32>)
+    ; CHECK: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[UV]](s32)
+    ; CHECK: [[ANYEXT1:%[0-9]+]]:_(s64) = G_ANYEXT [[UV1]](s32)
+    ; CHECK: [[ANYEXT2:%[0-9]+]]:_(s64) = G_ANYEXT [[UV2]](s32)
+    ; CHECK: [[ANYEXT3:%[0-9]+]]:_(s64) = G_ANYEXT [[UV3]](s32)
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s64>) = G_BUILD_VECTOR [[ANYEXT]](s64), [[ANYEXT1]](s64), [[ANYEXT2]](s64), [[ANYEXT3]](s64)
+    ; CHECK: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY [[BUILD_VECTOR]](<4 x s64>)
+    %0:_(<4 x s32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    %1:_(<4 x s64>) = G_ANYEXT %0
+    $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY %1
+...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/legalize-ashr.mir b/test/CodeGen/AMDGPU/GlobalISel/legalize-ashr.mir
index 59b5ac993ce47fc2c2708c0bc170eec6bd79c90b..a3d1c32ea35e80a32254544a61818a2b45565fc9 100644
--- a/test/CodeGen/AMDGPU/GlobalISel/legalize-ashr.mir
+++ b/test/CodeGen/AMDGPU/GlobalISel/legalize-ashr.mir
@@ -1,18 +1,1548 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn-mesa-mesa3d -O0 -run-pass=legalizer -global-isel %s -o - | FileCheck %s
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=tahiti -O0 -run-pass=legalizer -global-isel-abort=0 -o - %s | FileCheck -check-prefix=SI %s
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -O0 -run-pass=legalizer -global-isel-abort=0 -o - %s | FileCheck -check-prefix=VI %s
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx900 -O0 -run-pass=legalizer -global-isel-abort=0 -o - %s | FileCheck -check-prefix=GFX9  %s
 
 ---
-name:            test_ashr
+name: test_ashr_s32_s32
 body: |
-  bb.0.entry:
+  bb.0:
     liveins: $vgpr0, $vgpr1
 
-    ; CHECK-LABEL: name: test_ashr
-    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
-    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
-    ; CHECK: [[SHL:%[0-9]+]]:_(s32) = G_ASHR [[COPY]], [[COPY1]]
+    ; SI-LABEL: name: test_ashr_s32_s32
+    ; SI: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; SI: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; SI: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[COPY]], [[COPY1]](s32)
+    ; SI: $vgpr0 = COPY [[ASHR]](s32)
+    ; VI-LABEL: name: test_ashr_s32_s32
+    ; VI: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; VI: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; VI: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[COPY]], [[COPY1]](s32)
+    ; VI: $vgpr0 = COPY [[ASHR]](s32)
+    ; GFX9-LABEL: name: test_ashr_s32_s32
+    ; GFX9: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; GFX9: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; GFX9: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[COPY]], [[COPY1]](s32)
+    ; GFX9: $vgpr0 = COPY [[ASHR]](s32)
     %0:_(s32) = COPY $vgpr0
     %1:_(s32) = COPY $vgpr1
     %2:_(s32) = G_ASHR %0, %1
     $vgpr0 = COPY %2
 ...
+---
+name: test_ashr_s64_s64
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
+
+    ; SI-LABEL: name: test_ashr_s64_s64
+    ; SI: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
+    ; SI: [[COPY1:%[0-9]+]]:_(s64) = COPY $vgpr2_vgpr3
+    ; SI: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY1]](s64)
+    ; SI: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[COPY]], [[TRUNC]](s32)
+    ; SI: $vgpr0_vgpr1 = COPY [[ASHR]](s64)
+    ; VI-LABEL: name: test_ashr_s64_s64
+    ; VI: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
+    ; VI: [[COPY1:%[0-9]+]]:_(s64) = COPY $vgpr2_vgpr3
+    ; VI: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY1]](s64)
+    ; VI: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[COPY]], [[TRUNC]](s32)
+    ; VI: $vgpr0_vgpr1 = COPY [[ASHR]](s64)
+    ; GFX9-LABEL: name: test_ashr_s64_s64
+    ; GFX9: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
+    ; GFX9: [[COPY1:%[0-9]+]]:_(s64) = COPY $vgpr2_vgpr3
+    ; GFX9: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY1]](s64)
+    ; GFX9: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[COPY]], [[TRUNC]](s32)
+    ; GFX9: $vgpr0_vgpr1 = COPY [[ASHR]](s64)
+    %0:_(s64) = COPY $vgpr0_vgpr1
+    %1:_(s64) = COPY $vgpr2_vgpr3
+    %2:_(s64) = G_ASHR %0, %1
+    $vgpr0_vgpr1 = COPY %2
+...
+---
+name: test_ashr_s64_s32
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2
+
+    ; SI-LABEL: name: test_ashr_s64_s32
+    ; SI: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
+    ; SI: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr2
+    ; SI: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[COPY]], [[COPY1]](s32)
+    ; SI: $vgpr0_vgpr1 = COPY [[ASHR]](s64)
+    ; VI-LABEL: name: test_ashr_s64_s32
+    ; VI: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
+    ; VI: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr2
+    ; VI: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[COPY]], [[COPY1]](s32)
+    ; VI: $vgpr0_vgpr1 = COPY [[ASHR]](s64)
+    ; GFX9-LABEL: name: test_ashr_s64_s32
+    ; GFX9: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
+    ; GFX9: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr2
+    ; GFX9: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[COPY]], [[COPY1]](s32)
+    ; GFX9: $vgpr0_vgpr1 = COPY [[ASHR]](s64)
+    %0:_(s64) = COPY $vgpr0_vgpr1
+    %1:_(s32) = COPY $vgpr2
+    %2:_(s64) = G_ASHR %0, %1
+    $vgpr0_vgpr1 = COPY %2
+...
+---
+name: test_ashr_s64_s16
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2
+
+    ; SI-LABEL: name: test_ashr_s64_s16
+    ; SI: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
+    ; SI: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr2
+    ; SI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; SI: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+    ; SI: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C]]
+    ; SI: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[COPY]], [[AND]](s32)
+    ; SI: $vgpr0_vgpr1 = COPY [[ASHR]](s64)
+    ; VI-LABEL: name: test_ashr_s64_s16
+    ; VI: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
+    ; VI: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr2
+    ; VI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32)
+    ; VI: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[COPY]], [[TRUNC]](s16)
+    ; VI: $vgpr0_vgpr1 = COPY [[ASHR]](s64)
+    ; GFX9-LABEL: name: test_ashr_s64_s16
+    ; GFX9: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
+    ; GFX9: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr2
+    ; GFX9: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32)
+    ; GFX9: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[COPY]], [[TRUNC]](s16)
+    ; GFX9: $vgpr0_vgpr1 = COPY [[ASHR]](s64)
+    %0:_(s64) = COPY $vgpr0_vgpr1
+    %1:_(s32) = COPY $vgpr2
+    %2:_(s16) = G_TRUNC %1
+    %3:_(s64) = G_ASHR %0, %2
+    $vgpr0_vgpr1 = COPY %3
+...
+
+---
+name: test_ashr_s16_s32
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+
+    ; SI-LABEL: name: test_ashr_s16_s32
+    ; SI: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; SI: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; SI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; SI: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; SI: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY2]], [[C]](s32)
+    ; SI: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C]](s32)
+    ; SI: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[ASHR]], [[COPY1]](s32)
+    ; SI: [[COPY3:%[0-9]+]]:_(s32) = COPY [[ASHR1]](s32)
+    ; SI: $vgpr0 = COPY [[COPY3]](s32)
+    ; VI-LABEL: name: test_ashr_s16_s32
+    ; VI: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; VI: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; VI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+    ; VI: [[ASHR:%[0-9]+]]:_(s16) = G_ASHR [[TRUNC]], [[COPY1]](s32)
+    ; VI: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[ASHR]](s16)
+    ; VI: $vgpr0 = COPY [[ANYEXT]](s32)
+    ; GFX9-LABEL: name: test_ashr_s16_s32
+    ; GFX9: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; GFX9: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; GFX9: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+    ; GFX9: [[ASHR:%[0-9]+]]:_(s16) = G_ASHR [[TRUNC]], [[COPY1]](s32)
+    ; GFX9: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[ASHR]](s16)
+    ; GFX9: $vgpr0 = COPY [[ANYEXT]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s16) = G_TRUNC %0
+    %3:_(s16) = G_ASHR %2, %1
+    %4:_(s32) = G_ANYEXT %3
+    $vgpr0 = COPY %4
+...
+
+---
+name: test_ashr_s16_s16
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+
+    ; SI-LABEL: name: test_ashr_s16_s16
+    ; SI: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; SI: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; SI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; SI: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+    ; SI: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C]]
+    ; SI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; SI: [[COPY3:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; SI: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY3]], [[C1]](s32)
+    ; SI: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C1]](s32)
+    ; SI: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[ASHR]], [[AND]](s32)
+    ; SI: [[COPY4:%[0-9]+]]:_(s32) = COPY [[ASHR1]](s32)
+    ; SI: $vgpr0 = COPY [[COPY4]](s32)
+    ; VI-LABEL: name: test_ashr_s16_s16
+    ; VI: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; VI: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; VI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+    ; VI: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32)
+    ; VI: [[ASHR:%[0-9]+]]:_(s16) = G_ASHR [[TRUNC]], [[TRUNC1]](s16)
+    ; VI: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[ASHR]](s16)
+    ; VI: $vgpr0 = COPY [[ANYEXT]](s32)
+    ; GFX9-LABEL: name: test_ashr_s16_s16
+    ; GFX9: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; GFX9: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; GFX9: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+    ; GFX9: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32)
+    ; GFX9: [[ASHR:%[0-9]+]]:_(s16) = G_ASHR [[TRUNC]], [[TRUNC1]](s16)
+    ; GFX9: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[ASHR]](s16)
+    ; GFX9: $vgpr0 = COPY [[ANYEXT]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s16) = G_TRUNC %0
+    %3:_(s16) = G_TRUNC %1
+    %4:_(s16) = G_ASHR %2, %3
+    %5:_(s32) = G_ANYEXT %4
+    $vgpr0 = COPY %5
+...
+
+---
+name: test_ashr_s16_i8
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+
+    ; SI-LABEL: name: test_ashr_s16_i8
+    ; SI: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; SI: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; SI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; SI: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+    ; SI: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C]]
+    ; SI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; SI: [[COPY3:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; SI: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY3]], [[C1]](s32)
+    ; SI: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C1]](s32)
+    ; SI: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[ASHR]], [[AND]](s32)
+    ; SI: [[COPY4:%[0-9]+]]:_(s32) = COPY [[ASHR1]](s32)
+    ; SI: $vgpr0 = COPY [[COPY4]](s32)
+    ; VI-LABEL: name: test_ashr_s16_i8
+    ; VI: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; VI: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; VI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+    ; VI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; VI: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+    ; VI: [[COPY3:%[0-9]+]]:_(s32) = COPY [[C]](s32)
+    ; VI: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[COPY3]]
+    ; VI: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[AND]](s32)
+    ; VI: [[ASHR:%[0-9]+]]:_(s16) = G_ASHR [[TRUNC]], [[TRUNC1]](s16)
+    ; VI: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[ASHR]](s16)
+    ; VI: $vgpr0 = COPY [[ANYEXT]](s32)
+    ; GFX9-LABEL: name: test_ashr_s16_i8
+    ; GFX9: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; GFX9: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; GFX9: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; GFX9: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+    ; GFX9: [[COPY3:%[0-9]+]]:_(s32) = COPY [[C]](s32)
+    ; GFX9: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[COPY3]]
+    ; GFX9: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[AND]](s32)
+    ; GFX9: [[ASHR:%[0-9]+]]:_(s16) = G_ASHR [[TRUNC]], [[TRUNC1]](s16)
+    ; GFX9: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[ASHR]](s16)
+    ; GFX9: $vgpr0 = COPY [[ANYEXT]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s16) = G_TRUNC %0
+    %3:_(s8) = G_TRUNC %1
+    %4:_(s16) = G_ASHR %2, %3
+    %5:_(s32) = G_ANYEXT %4
+    $vgpr0 = COPY %5
+...
+
+---
+name: test_ashr_i8_i8
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+
+    ; SI-LABEL: name: test_ashr_i8_i8
+    ; SI: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; SI: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; SI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; SI: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+    ; SI: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C]]
+    ; SI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; SI: [[COPY3:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; SI: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY3]], [[C1]](s32)
+    ; SI: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C1]](s32)
+    ; SI: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[ASHR]], [[AND]](s32)
+    ; SI: [[COPY4:%[0-9]+]]:_(s32) = COPY [[ASHR1]](s32)
+    ; SI: $vgpr0 = COPY [[COPY4]](s32)
+    ; VI-LABEL: name: test_ashr_i8_i8
+    ; VI: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; VI: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; VI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; VI: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+    ; VI: [[COPY3:%[0-9]+]]:_(s32) = COPY [[C]](s32)
+    ; VI: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[COPY3]]
+    ; VI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[AND]](s32)
+    ; VI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; VI: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[C1]](s32)
+    ; VI: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+    ; VI: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[TRUNC2]], [[TRUNC1]](s16)
+    ; VI: [[ASHR:%[0-9]+]]:_(s16) = G_ASHR [[SHL]], [[TRUNC1]](s16)
+    ; VI: [[ASHR1:%[0-9]+]]:_(s16) = G_ASHR [[ASHR]], [[TRUNC]](s16)
+    ; VI: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[ASHR1]](s16)
+    ; VI: $vgpr0 = COPY [[ANYEXT]](s32)
+    ; GFX9-LABEL: name: test_ashr_i8_i8
+    ; GFX9: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; GFX9: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; GFX9: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+    ; GFX9: [[COPY3:%[0-9]+]]:_(s32) = COPY [[C]](s32)
+    ; GFX9: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[COPY3]]
+    ; GFX9: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[AND]](s32)
+    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; GFX9: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[C1]](s32)
+    ; GFX9: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+    ; GFX9: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[TRUNC2]], [[TRUNC1]](s16)
+    ; GFX9: [[ASHR:%[0-9]+]]:_(s16) = G_ASHR [[SHL]], [[TRUNC1]](s16)
+    ; GFX9: [[ASHR1:%[0-9]+]]:_(s16) = G_ASHR [[ASHR]], [[TRUNC]](s16)
+    ; GFX9: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[ASHR1]](s16)
+    ; GFX9: $vgpr0 = COPY [[ANYEXT]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s8) = G_TRUNC %0
+    %3:_(s8) = G_TRUNC %1
+    %4:_(s8) = G_ASHR %2, %3
+    %5:_(s32) = G_ANYEXT %4
+    $vgpr0 = COPY %5
+...
+
+---
+name: test_ashr_v2s32_v2s32
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
+
+    ; SI-LABEL: name: test_ashr_v2s32_v2s32
+    ; SI: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    ; SI: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr2_vgpr3
+    ; SI: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
+    ; SI: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<2 x s32>)
+    ; SI: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[UV]], [[UV2]](s32)
+    ; SI: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[UV1]], [[UV3]](s32)
+    ; SI: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[ASHR]](s32), [[ASHR1]](s32)
+    ; SI: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
+    ; VI-LABEL: name: test_ashr_v2s32_v2s32
+    ; VI: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    ; VI: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr2_vgpr3
+    ; VI: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
+    ; VI: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<2 x s32>)
+    ; VI: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[UV]], [[UV2]](s32)
+    ; VI: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[UV1]], [[UV3]](s32)
+    ; VI: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[ASHR]](s32), [[ASHR1]](s32)
+    ; VI: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
+    ; GFX9-LABEL: name: test_ashr_v2s32_v2s32
+    ; GFX9: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    ; GFX9: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr2_vgpr3
+    ; GFX9: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
+    ; GFX9: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<2 x s32>)
+    ; GFX9: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[UV]], [[UV2]](s32)
+    ; GFX9: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[UV1]], [[UV3]](s32)
+    ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[ASHR]](s32), [[ASHR1]](s32)
+    ; GFX9: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
+    %0:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    %1:_(<2 x s32>) = COPY $vgpr2_vgpr3
+    %2:_(<2 x s32>) = G_ASHR %0, %1
+    $vgpr0_vgpr1 = COPY %2
+...
+
+---
+name: test_ashr_v3s32_v3s32
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2, $vgpr3_vgpr4_vgpr5
+
+    ; SI-LABEL: name: test_ashr_v3s32_v3s32
+    ; SI: [[COPY:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr0_vgpr1_vgpr2
+    ; SI: [[COPY1:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr3_vgpr4_vgpr5
+    ; SI: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<3 x s32>)
+    ; SI: [[UV3:%[0-9]+]]:_(s32), [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<3 x s32>)
+    ; SI: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[UV]], [[UV3]](s32)
+    ; SI: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[UV1]], [[UV4]](s32)
+    ; SI: [[ASHR2:%[0-9]+]]:_(s32) = G_ASHR [[UV2]], [[UV5]](s32)
+    ; SI: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[ASHR]](s32), [[ASHR1]](s32), [[ASHR2]](s32)
+    ; SI: $vgpr0_vgpr1_vgpr2 = COPY [[BUILD_VECTOR]](<3 x s32>)
+    ; VI-LABEL: name: test_ashr_v3s32_v3s32
+    ; VI: [[COPY:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr0_vgpr1_vgpr2
+    ; VI: [[COPY1:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr3_vgpr4_vgpr5
+    ; VI: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<3 x s32>)
+    ; VI: [[UV3:%[0-9]+]]:_(s32), [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<3 x s32>)
+    ; VI: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[UV]], [[UV3]](s32)
+    ; VI: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[UV1]], [[UV4]](s32)
+    ; VI: [[ASHR2:%[0-9]+]]:_(s32) = G_ASHR [[UV2]], [[UV5]](s32)
+    ; VI: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[ASHR]](s32), [[ASHR1]](s32), [[ASHR2]](s32)
+    ; VI: $vgpr0_vgpr1_vgpr2 = COPY [[BUILD_VECTOR]](<3 x s32>)
+    ; GFX9-LABEL: name: test_ashr_v3s32_v3s32
+    ; GFX9: [[COPY:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr0_vgpr1_vgpr2
+    ; GFX9: [[COPY1:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr3_vgpr4_vgpr5
+    ; GFX9: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<3 x s32>)
+    ; GFX9: [[UV3:%[0-9]+]]:_(s32), [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<3 x s32>)
+    ; GFX9: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[UV]], [[UV3]](s32)
+    ; GFX9: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[UV1]], [[UV4]](s32)
+    ; GFX9: [[ASHR2:%[0-9]+]]:_(s32) = G_ASHR [[UV2]], [[UV5]](s32)
+    ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[ASHR]](s32), [[ASHR1]](s32), [[ASHR2]](s32)
+    ; GFX9: $vgpr0_vgpr1_vgpr2 = COPY [[BUILD_VECTOR]](<3 x s32>)
+    %0:_(<3 x s32>) = COPY $vgpr0_vgpr1_vgpr2
+    %1:_(<3 x s32>) = COPY $vgpr3_vgpr4_vgpr5
+    %2:_(<3 x s32>) = G_ASHR %0, %1
+    $vgpr0_vgpr1_vgpr2 = COPY %2
+...
+
+---
+name: test_ashr_v2s64_v2s32
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5
+
+    ; SI-LABEL: name: test_ashr_v2s64_v2s32
+    ; SI: [[COPY:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; SI: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr4_vgpr5
+    ; SI: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](<2 x s64>)
+    ; SI: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<2 x s32>)
+    ; SI: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[UV]], [[UV2]](s32)
+    ; SI: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[UV1]], [[UV3]](s32)
+    ; SI: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[ASHR]](s64), [[ASHR1]](s64)
+    ; SI: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>)
+    ; VI-LABEL: name: test_ashr_v2s64_v2s32
+    ; VI: [[COPY:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; VI: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr4_vgpr5
+    ; VI: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](<2 x s64>)
+    ; VI: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<2 x s32>)
+    ; VI: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[UV]], [[UV2]](s32)
+    ; VI: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[UV1]], [[UV3]](s32)
+    ; VI: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[ASHR]](s64), [[ASHR1]](s64)
+    ; VI: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>)
+    ; GFX9-LABEL: name: test_ashr_v2s64_v2s32
+    ; GFX9: [[COPY:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; GFX9: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr4_vgpr5
+    ; GFX9: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](<2 x s64>)
+    ; GFX9: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<2 x s32>)
+    ; GFX9: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[UV]], [[UV2]](s32)
+    ; GFX9: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[UV1]], [[UV3]](s32)
+    ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[ASHR]](s64), [[ASHR1]](s64)
+    ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>)
+    %0:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    %1:_(<2 x s32>) = COPY $vgpr4_vgpr5
+    %2:_(<2 x s64>) = G_ASHR %0, %1
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %2
+...
+
+---
+name: test_ashr_v3s64_v3s32
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10
+
+    ; SI-LABEL: name: test_ashr_v3s64_v3s32
+    ; SI: [[COPY:%[0-9]+]]:_(<4 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+    ; SI: [[EXTRACT:%[0-9]+]]:_(<3 x s64>) = G_EXTRACT [[COPY]](<4 x s64>), 0
+    ; SI: [[COPY1:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr8_vgpr9_vgpr10
+    ; SI: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64), [[UV2:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[EXTRACT]](<3 x s64>)
+    ; SI: [[UV3:%[0-9]+]]:_(s32), [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<3 x s32>)
+    ; SI: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[UV]], [[UV3]](s32)
+    ; SI: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[UV1]], [[UV4]](s32)
+    ; SI: [[ASHR2:%[0-9]+]]:_(s64) = G_ASHR [[UV2]], [[UV5]](s32)
+    ; SI: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s64>) = G_BUILD_VECTOR [[ASHR]](s64), [[ASHR1]](s64), [[ASHR2]](s64)
+    ; SI: [[DEF:%[0-9]+]]:_(<4 x s64>) = G_IMPLICIT_DEF
+    ; SI: [[INSERT:%[0-9]+]]:_(<4 x s64>) = G_INSERT [[DEF]], [[BUILD_VECTOR]](<3 x s64>), 0
+    ; SI: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY [[INSERT]](<4 x s64>)
+    ; VI-LABEL: name: test_ashr_v3s64_v3s32
+    ; VI: [[COPY:%[0-9]+]]:_(<4 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+    ; VI: [[EXTRACT:%[0-9]+]]:_(<3 x s64>) = G_EXTRACT [[COPY]](<4 x s64>), 0
+    ; VI: [[COPY1:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr8_vgpr9_vgpr10
+    ; VI: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64), [[UV2:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[EXTRACT]](<3 x s64>)
+    ; VI: [[UV3:%[0-9]+]]:_(s32), [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<3 x s32>)
+    ; VI: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[UV]], [[UV3]](s32)
+    ; VI: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[UV1]], [[UV4]](s32)
+    ; VI: [[ASHR2:%[0-9]+]]:_(s64) = G_ASHR [[UV2]], [[UV5]](s32)
+    ; VI: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s64>) = G_BUILD_VECTOR [[ASHR]](s64), [[ASHR1]](s64), [[ASHR2]](s64)
+    ; VI: [[DEF:%[0-9]+]]:_(<4 x s64>) = G_IMPLICIT_DEF
+    ; VI: [[INSERT:%[0-9]+]]:_(<4 x s64>) = G_INSERT [[DEF]], [[BUILD_VECTOR]](<3 x s64>), 0
+    ; VI: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY [[INSERT]](<4 x s64>)
+    ; GFX9-LABEL: name: test_ashr_v3s64_v3s32
+    ; GFX9: [[COPY:%[0-9]+]]:_(<4 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+    ; GFX9: [[EXTRACT:%[0-9]+]]:_(<3 x s64>) = G_EXTRACT [[COPY]](<4 x s64>), 0
+    ; GFX9: [[COPY1:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr8_vgpr9_vgpr10
+    ; GFX9: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64), [[UV2:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[EXTRACT]](<3 x s64>)
+    ; GFX9: [[UV3:%[0-9]+]]:_(s32), [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<3 x s32>)
+    ; GFX9: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[UV]], [[UV3]](s32)
+    ; GFX9: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[UV1]], [[UV4]](s32)
+    ; GFX9: [[ASHR2:%[0-9]+]]:_(s64) = G_ASHR [[UV2]], [[UV5]](s32)
+    ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s64>) = G_BUILD_VECTOR [[ASHR]](s64), [[ASHR1]](s64), [[ASHR2]](s64)
+    ; GFX9: [[DEF:%[0-9]+]]:_(<4 x s64>) = G_IMPLICIT_DEF
+    ; GFX9: [[INSERT:%[0-9]+]]:_(<4 x s64>) = G_INSERT [[DEF]], [[BUILD_VECTOR]](<3 x s64>), 0
+    ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY [[INSERT]](<4 x s64>)
+    %0:_(<4 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+    %1:_(<3 x s64>) = G_EXTRACT %0, 0
+    %2:_(<3 x s32>) = COPY $vgpr8_vgpr9_vgpr10
+    %3:_(<3 x s64>) = G_ASHR %1, %2
+    %4:_(<4 x s64>) = G_IMPLICIT_DEF
+    %5:_(<4 x s64>) = G_INSERT %4, %3, 0
+    $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY %5
+...
+
+---
+name: test_ashr_v2s16_v2s16
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+
+    ; SI-LABEL: name: test_ashr_v2s16_v2s16
+    ; SI: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
+    ; SI: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
+    ; SI: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY]](<2 x s16>)
+    ; SI: [[UV2:%[0-9]+]]:_(s16), [[UV3:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY1]](<2 x s16>)
+    ; SI: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[UV2]](s16)
+    ; SI: [[SEXT:%[0-9]+]]:_(s32) = G_SEXT [[UV]](s16)
+    ; SI: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SEXT]], [[ZEXT]](s32)
+    ; SI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[ASHR]](s32)
+    ; SI: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[UV3]](s16)
+    ; SI: [[SEXT1:%[0-9]+]]:_(s32) = G_SEXT [[UV1]](s16)
+    ; SI: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SEXT1]], [[ZEXT1]](s32)
+    ; SI: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[ASHR1]](s32)
+    ; SI: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
+    ; SI: $vgpr0 = COPY [[BUILD_VECTOR]](<2 x s16>)
+    ; VI-LABEL: name: test_ashr_v2s16_v2s16
+    ; VI: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
+    ; VI: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
+    ; VI: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY]](<2 x s16>)
+    ; VI: [[UV2:%[0-9]+]]:_(s16), [[UV3:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY1]](<2 x s16>)
+    ; VI: [[ASHR:%[0-9]+]]:_(s16) = G_ASHR [[UV]], [[UV2]](s16)
+    ; VI: [[ASHR1:%[0-9]+]]:_(s16) = G_ASHR [[UV1]], [[UV3]](s16)
+    ; VI: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[ASHR]](s16), [[ASHR1]](s16)
+    ; VI: $vgpr0 = COPY [[BUILD_VECTOR]](<2 x s16>)
+    ; GFX9-LABEL: name: test_ashr_v2s16_v2s16
+    ; GFX9: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
+    ; GFX9: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
+    ; GFX9: [[ASHR:%[0-9]+]]:_(<2 x s16>) = G_ASHR [[COPY]], [[COPY1]](<2 x s16>)
+    ; GFX9: $vgpr0 = COPY [[ASHR]](<2 x s16>)
+    %0:_(<2 x s16>) = COPY $vgpr0
+    %1:_(<2 x s16>) = COPY $vgpr1
+    %2:_(<2 x s16>) = G_ASHR %0, %1
+    $vgpr0 = COPY %2
+...
+
+---
+name: test_ashr_v2s16_v2s32
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr0_vgpr1
+
+    ; SI-LABEL: name: test_ashr_v2s16_v2s32
+    ; SI: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
+    ; SI: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    ; SI: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY]](<2 x s16>)
+    ; SI: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<2 x s32>)
+    ; SI: [[SEXT:%[0-9]+]]:_(s32) = G_SEXT [[UV]](s16)
+    ; SI: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SEXT]], [[UV2]](s32)
+    ; SI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[ASHR]](s32)
+    ; SI: [[SEXT1:%[0-9]+]]:_(s32) = G_SEXT [[UV1]](s16)
+    ; SI: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SEXT1]], [[UV3]](s32)
+    ; SI: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[ASHR1]](s32)
+    ; SI: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
+    ; SI: $vgpr0 = COPY [[BUILD_VECTOR]](<2 x s16>)
+    ; VI-LABEL: name: test_ashr_v2s16_v2s32
+    ; VI: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
+    ; VI: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    ; VI: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY]](<2 x s16>)
+    ; VI: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<2 x s32>)
+    ; VI: [[ASHR:%[0-9]+]]:_(s16) = G_ASHR [[UV]], [[UV2]](s32)
+    ; VI: [[ASHR1:%[0-9]+]]:_(s16) = G_ASHR [[UV1]], [[UV3]](s32)
+    ; VI: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[ASHR]](s16), [[ASHR1]](s16)
+    ; VI: $vgpr0 = COPY [[BUILD_VECTOR]](<2 x s16>)
+    ; GFX9-LABEL: name: test_ashr_v2s16_v2s32
+    ; GFX9: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
+    ; GFX9: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    ; GFX9: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY]](<2 x s16>)
+    ; GFX9: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<2 x s32>)
+    ; GFX9: [[ASHR:%[0-9]+]]:_(s16) = G_ASHR [[UV]], [[UV2]](s32)
+    ; GFX9: [[ASHR1:%[0-9]+]]:_(s16) = G_ASHR [[UV1]], [[UV3]](s32)
+    ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[ASHR]](s16), [[ASHR1]](s16)
+    ; GFX9: $vgpr0 = COPY [[BUILD_VECTOR]](<2 x s16>)
+    %0:_(<2 x s16>) = COPY $vgpr0
+    %1:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    %2:_(<2 x s16>) = G_ASHR %0, %1
+    $vgpr0 = COPY %2
+...
+
+---
+name: test_ashr_v3s16_v3s16
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
+
+    ; SI-LABEL: name: test_ashr_v3s16_v3s16
+    ; SI: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
+    ; SI: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
+    ; SI: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[COPY]](<4 x s16>), 0
+    ; SI: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[COPY1]](<4 x s16>), 0
+    ; SI: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16), [[UV2:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[EXTRACT]](<3 x s16>)
+    ; SI: [[UV3:%[0-9]+]]:_(s16), [[UV4:%[0-9]+]]:_(s16), [[UV5:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[EXTRACT1]](<3 x s16>)
+    ; SI: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[UV3]](s16)
+    ; SI: [[SEXT:%[0-9]+]]:_(s32) = G_SEXT [[UV]](s16)
+    ; SI: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SEXT]], [[ZEXT]](s32)
+    ; SI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[ASHR]](s32)
+    ; SI: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[UV4]](s16)
+    ; SI: [[SEXT1:%[0-9]+]]:_(s32) = G_SEXT [[UV1]](s16)
+    ; SI: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SEXT1]], [[ZEXT1]](s32)
+    ; SI: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[ASHR1]](s32)
+    ; SI: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[UV5]](s16)
+    ; SI: [[SEXT2:%[0-9]+]]:_(s32) = G_SEXT [[UV2]](s16)
+    ; SI: [[ASHR2:%[0-9]+]]:_(s32) = G_ASHR [[SEXT2]], [[ZEXT2]](s32)
+    ; SI: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[ASHR2]](s32)
+    ; SI: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16), [[TRUNC2]](s16)
+    ; SI: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; SI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[BUILD_VECTOR]](<3 x s16>), 0
+    ; SI: $vgpr0_vgpr1 = COPY [[INSERT]](<4 x s16>)
+    ; VI-LABEL: name: test_ashr_v3s16_v3s16
+    ; VI: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
+    ; VI: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
+    ; VI: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[COPY]](<4 x s16>), 0
+    ; VI: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[COPY1]](<4 x s16>), 0
+    ; VI: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16), [[UV2:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[EXTRACT]](<3 x s16>)
+    ; VI: [[UV3:%[0-9]+]]:_(s16), [[UV4:%[0-9]+]]:_(s16), [[UV5:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[EXTRACT1]](<3 x s16>)
+    ; VI: [[ASHR:%[0-9]+]]:_(s16) = G_ASHR [[UV]], [[UV3]](s16)
+    ; VI: [[ASHR1:%[0-9]+]]:_(s16) = G_ASHR [[UV1]], [[UV4]](s16)
+    ; VI: [[ASHR2:%[0-9]+]]:_(s16) = G_ASHR [[UV2]], [[UV5]](s16)
+    ; VI: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s16>) = G_BUILD_VECTOR [[ASHR]](s16), [[ASHR1]](s16), [[ASHR2]](s16)
+    ; VI: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; VI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[BUILD_VECTOR]](<3 x s16>), 0
+    ; VI: $vgpr0_vgpr1 = COPY [[INSERT]](<4 x s16>)
+    ; GFX9-LABEL: name: test_ashr_v3s16_v3s16
+    ; GFX9: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
+    ; GFX9: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
+    ; GFX9: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[COPY]](<4 x s16>), 0
+    ; GFX9: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[COPY1]](<4 x s16>), 0
+    ; GFX9: [[EXTRACT2:%[0-9]+]]:_(<2 x s16>) = G_EXTRACT [[EXTRACT]](<3 x s16>), 0
+    ; GFX9: [[EXTRACT3:%[0-9]+]]:_(s16) = G_EXTRACT [[EXTRACT]](<3 x s16>), 32
+    ; GFX9: [[EXTRACT4:%[0-9]+]]:_(<2 x s16>) = G_EXTRACT [[EXTRACT1]](<3 x s16>), 0
+    ; GFX9: [[EXTRACT5:%[0-9]+]]:_(s16) = G_EXTRACT [[EXTRACT1]](<3 x s16>), 32
+    ; GFX9: [[ASHR:%[0-9]+]]:_(<2 x s16>) = G_ASHR [[EXTRACT2]], [[EXTRACT4]](<2 x s16>)
+    ; GFX9: [[ASHR1:%[0-9]+]]:_(s16) = G_ASHR [[EXTRACT3]], [[EXTRACT5]](s16)
+    ; GFX9: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; GFX9: [[EXTRACT6:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
+    ; GFX9: [[INSERT:%[0-9]+]]:_(<3 x s16>) = G_INSERT [[EXTRACT6]], [[ASHR]](<2 x s16>), 0
+    ; GFX9: [[INSERT1:%[0-9]+]]:_(<3 x s16>) = G_INSERT [[INSERT]], [[ASHR1]](s16), 32
+    ; GFX9: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; GFX9: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[INSERT1]](<3 x s16>), 0
+    ; GFX9: $vgpr0_vgpr1 = COPY [[INSERT2]](<4 x s16>)
+    %0:_(<4 x s16>) = COPY $vgpr0_vgpr1
+    %1:_(<4 x s16>) = COPY $vgpr2_vgpr3
+    %2:_(<3 x s16>) = G_EXTRACT %0, 0
+    %3:_(<3 x s16>) = G_EXTRACT %1, 0
+    %4:_(<3 x s16>) = G_ASHR %2, %3
+    %5:_(<4 x s16>) = G_IMPLICIT_DEF
+    %6:_(<4 x s16>) = G_INSERT %5, %4, 0
+    $vgpr0_vgpr1 = COPY %6
+...
+
+---
+name: test_ashr_v4s16_v4s16
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
+
+    ; SI-LABEL: name: test_ashr_v4s16_v4s16
+    ; SI: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
+    ; SI: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
+    ; SI: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16), [[UV2:%[0-9]+]]:_(s16), [[UV3:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
+    ; SI: [[UV4:%[0-9]+]]:_(s16), [[UV5:%[0-9]+]]:_(s16), [[UV6:%[0-9]+]]:_(s16), [[UV7:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
+    ; SI: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[UV4]](s16)
+    ; SI: [[SEXT:%[0-9]+]]:_(s32) = G_SEXT [[UV]](s16)
+    ; SI: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SEXT]], [[ZEXT]](s32)
+    ; SI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[ASHR]](s32)
+    ; SI: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[UV5]](s16)
+    ; SI: [[SEXT1:%[0-9]+]]:_(s32) = G_SEXT [[UV1]](s16)
+    ; SI: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SEXT1]], [[ZEXT1]](s32)
+    ; SI: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[ASHR1]](s32)
+    ; SI: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[UV6]](s16)
+    ; SI: [[SEXT2:%[0-9]+]]:_(s32) = G_SEXT [[UV2]](s16)
+    ; SI: [[ASHR2:%[0-9]+]]:_(s32) = G_ASHR [[SEXT2]], [[ZEXT2]](s32)
+    ; SI: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[ASHR2]](s32)
+    ; SI: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[UV7]](s16)
+    ; SI: [[SEXT3:%[0-9]+]]:_(s32) = G_SEXT [[UV3]](s16)
+    ; SI: [[ASHR3:%[0-9]+]]:_(s32) = G_ASHR [[SEXT3]], [[ZEXT3]](s32)
+    ; SI: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[ASHR3]](s32)
+    ; SI: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16), [[TRUNC2]](s16), [[TRUNC3]](s16)
+    ; SI: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<4 x s16>)
+    ; VI-LABEL: name: test_ashr_v4s16_v4s16
+    ; VI: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
+    ; VI: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
+    ; VI: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16), [[UV2:%[0-9]+]]:_(s16), [[UV3:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
+    ; VI: [[UV4:%[0-9]+]]:_(s16), [[UV5:%[0-9]+]]:_(s16), [[UV6:%[0-9]+]]:_(s16), [[UV7:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
+    ; VI: [[ASHR:%[0-9]+]]:_(s16) = G_ASHR [[UV]], [[UV4]](s16)
+    ; VI: [[ASHR1:%[0-9]+]]:_(s16) = G_ASHR [[UV1]], [[UV5]](s16)
+    ; VI: [[ASHR2:%[0-9]+]]:_(s16) = G_ASHR [[UV2]], [[UV6]](s16)
+    ; VI: [[ASHR3:%[0-9]+]]:_(s16) = G_ASHR [[UV3]], [[UV7]](s16)
+    ; VI: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s16>) = G_BUILD_VECTOR [[ASHR]](s16), [[ASHR1]](s16), [[ASHR2]](s16), [[ASHR3]](s16)
+    ; VI: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<4 x s16>)
+    ; GFX9-LABEL: name: test_ashr_v4s16_v4s16
+    ; GFX9: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
+    ; GFX9: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
+    ; GFX9: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
+    ; GFX9: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
+    ; GFX9: [[ASHR:%[0-9]+]]:_(<2 x s16>) = G_ASHR [[UV]], [[UV2]](<2 x s16>)
+    ; GFX9: [[ASHR1:%[0-9]+]]:_(<2 x s16>) = G_ASHR [[UV1]], [[UV3]](<2 x s16>)
+    ; GFX9: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[ASHR]](<2 x s16>), [[ASHR1]](<2 x s16>)
+    ; GFX9: $vgpr0_vgpr1 = COPY [[CONCAT_VECTORS]](<4 x s16>)
+    %0:_(<4 x s16>) = COPY $vgpr0_vgpr1
+    %1:_(<4 x s16>) = COPY $vgpr2_vgpr3
+    %2:_(<4 x s16>) = G_ASHR %0, %1
+    $vgpr0_vgpr1 = COPY %2
+...
+
+---
+name: test_ashr_s128_s128
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4
+
+    ; SI-LABEL: name: test_ashr_s128_s128
+    ; SI: [[COPY:%[0-9]+]]:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; SI: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr4
+    ; SI: [[ZEXT:%[0-9]+]]:_(s128) = G_ZEXT [[COPY1]](s32)
+    ; SI: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[ZEXT]](s128)
+    ; SI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 64
+    ; SI: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](s128)
+    ; SI: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[TRUNC]], [[C]]
+    ; SI: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[C]], [[TRUNC]]
+    ; SI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; SI: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[TRUNC]](s32), [[C]]
+    ; SI: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[TRUNC]](s32), [[C1]]
+    ; SI: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[UV1]], [[TRUNC]](s32)
+    ; SI: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[UV]], [[TRUNC]](s32)
+    ; SI: [[LSHR1:%[0-9]+]]:_(s64) = G_LSHR [[UV1]], [[SUB1]](s32)
+    ; SI: [[OR:%[0-9]+]]:_(s64) = G_OR [[LSHR]], [[LSHR1]]
+    ; SI: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 63
+    ; SI: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[UV1]], [[C2]](s32)
+    ; SI: [[ASHR2:%[0-9]+]]:_(s64) = G_ASHR [[UV1]], [[SUB]](s32)
+    ; SI: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[OR]], [[ASHR2]]
+    ; SI: [[SELECT1:%[0-9]+]]:_(s64) = G_SELECT [[ICMP1]](s1), [[UV]], [[SELECT]]
+    ; SI: [[SELECT2:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[ASHR]], [[ASHR1]]
+    ; SI: [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[SELECT1]](s64), [[SELECT2]](s64)
+    ; SI: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[MV]](s128)
+    ; VI-LABEL: name: test_ashr_s128_s128
+    ; VI: [[COPY:%[0-9]+]]:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; VI: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr4
+    ; VI: [[ZEXT:%[0-9]+]]:_(s128) = G_ZEXT [[COPY1]](s32)
+    ; VI: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[ZEXT]](s128)
+    ; VI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 64
+    ; VI: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](s128)
+    ; VI: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[TRUNC]], [[C]]
+    ; VI: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[C]], [[TRUNC]]
+    ; VI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; VI: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[TRUNC]](s32), [[C]]
+    ; VI: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[TRUNC]](s32), [[C1]]
+    ; VI: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[UV1]], [[TRUNC]](s32)
+    ; VI: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[UV]], [[TRUNC]](s32)
+    ; VI: [[LSHR1:%[0-9]+]]:_(s64) = G_LSHR [[UV1]], [[SUB1]](s32)
+    ; VI: [[OR:%[0-9]+]]:_(s64) = G_OR [[LSHR]], [[LSHR1]]
+    ; VI: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 63
+    ; VI: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[UV1]], [[C2]](s32)
+    ; VI: [[ASHR2:%[0-9]+]]:_(s64) = G_ASHR [[UV1]], [[SUB]](s32)
+    ; VI: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[OR]], [[ASHR2]]
+    ; VI: [[SELECT1:%[0-9]+]]:_(s64) = G_SELECT [[ICMP1]](s1), [[UV]], [[SELECT]]
+    ; VI: [[SELECT2:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[ASHR]], [[ASHR1]]
+    ; VI: [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[SELECT1]](s64), [[SELECT2]](s64)
+    ; VI: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[MV]](s128)
+    ; GFX9-LABEL: name: test_ashr_s128_s128
+    ; GFX9: [[COPY:%[0-9]+]]:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; GFX9: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr4
+    ; GFX9: [[ZEXT:%[0-9]+]]:_(s128) = G_ZEXT [[COPY1]](s32)
+    ; GFX9: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[ZEXT]](s128)
+    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 64
+    ; GFX9: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](s128)
+    ; GFX9: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[TRUNC]], [[C]]
+    ; GFX9: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[C]], [[TRUNC]]
+    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; GFX9: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[TRUNC]](s32), [[C]]
+    ; GFX9: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[TRUNC]](s32), [[C1]]
+    ; GFX9: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[UV1]], [[TRUNC]](s32)
+    ; GFX9: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[UV]], [[TRUNC]](s32)
+    ; GFX9: [[LSHR1:%[0-9]+]]:_(s64) = G_LSHR [[UV1]], [[SUB1]](s32)
+    ; GFX9: [[OR:%[0-9]+]]:_(s64) = G_OR [[LSHR]], [[LSHR1]]
+    ; GFX9: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 63
+    ; GFX9: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[UV1]], [[C2]](s32)
+    ; GFX9: [[ASHR2:%[0-9]+]]:_(s64) = G_ASHR [[UV1]], [[SUB]](s32)
+    ; GFX9: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[OR]], [[ASHR2]]
+    ; GFX9: [[SELECT1:%[0-9]+]]:_(s64) = G_SELECT [[ICMP1]](s1), [[UV]], [[SELECT]]
+    ; GFX9: [[SELECT2:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[ASHR]], [[ASHR1]]
+    ; GFX9: [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[SELECT1]](s64), [[SELECT2]](s64)
+    ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[MV]](s128)
+    %0:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    %1:_(s32) = COPY $vgpr4
+    %2:_(s128) = G_ZEXT %1
+    %3:_(s128) = G_ASHR %0, %2
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %3
+...
+
+---
+name: test_ashr_s128_s132
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4
+
+    ; SI-LABEL: name: test_ashr_s128_s132
+    ; SI: [[COPY:%[0-9]+]]:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; SI: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr4
+    ; SI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 64
+    ; SI: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](s128)
+    ; SI: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[COPY1]], [[C]]
+    ; SI: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[C]], [[COPY1]]
+    ; SI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; SI: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[COPY1]](s32), [[C]]
+    ; SI: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[COPY1]](s32), [[C1]]
+    ; SI: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[UV1]], [[COPY1]](s32)
+    ; SI: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[UV]], [[COPY1]](s32)
+    ; SI: [[LSHR1:%[0-9]+]]:_(s64) = G_LSHR [[UV1]], [[SUB1]](s32)
+    ; SI: [[OR:%[0-9]+]]:_(s64) = G_OR [[LSHR]], [[LSHR1]]
+    ; SI: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 63
+    ; SI: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[UV1]], [[C2]](s32)
+    ; SI: [[ASHR2:%[0-9]+]]:_(s64) = G_ASHR [[UV1]], [[SUB]](s32)
+    ; SI: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[OR]], [[ASHR2]]
+    ; SI: [[SELECT1:%[0-9]+]]:_(s64) = G_SELECT [[ICMP1]](s1), [[UV]], [[SELECT]]
+    ; SI: [[SELECT2:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[ASHR]], [[ASHR1]]
+    ; SI: [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[SELECT1]](s64), [[SELECT2]](s64)
+    ; SI: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[MV]](s128)
+    ; VI-LABEL: name: test_ashr_s128_s132
+    ; VI: [[COPY:%[0-9]+]]:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; VI: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr4
+    ; VI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 64
+    ; VI: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](s128)
+    ; VI: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[COPY1]], [[C]]
+    ; VI: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[C]], [[COPY1]]
+    ; VI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; VI: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[COPY1]](s32), [[C]]
+    ; VI: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[COPY1]](s32), [[C1]]
+    ; VI: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[UV1]], [[COPY1]](s32)
+    ; VI: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[UV]], [[COPY1]](s32)
+    ; VI: [[LSHR1:%[0-9]+]]:_(s64) = G_LSHR [[UV1]], [[SUB1]](s32)
+    ; VI: [[OR:%[0-9]+]]:_(s64) = G_OR [[LSHR]], [[LSHR1]]
+    ; VI: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 63
+    ; VI: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[UV1]], [[C2]](s32)
+    ; VI: [[ASHR2:%[0-9]+]]:_(s64) = G_ASHR [[UV1]], [[SUB]](s32)
+    ; VI: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[OR]], [[ASHR2]]
+    ; VI: [[SELECT1:%[0-9]+]]:_(s64) = G_SELECT [[ICMP1]](s1), [[UV]], [[SELECT]]
+    ; VI: [[SELECT2:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[ASHR]], [[ASHR1]]
+    ; VI: [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[SELECT1]](s64), [[SELECT2]](s64)
+    ; VI: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[MV]](s128)
+    ; GFX9-LABEL: name: test_ashr_s128_s132
+    ; GFX9: [[COPY:%[0-9]+]]:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; GFX9: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr4
+    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 64
+    ; GFX9: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](s128)
+    ; GFX9: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[COPY1]], [[C]]
+    ; GFX9: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[C]], [[COPY1]]
+    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; GFX9: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[COPY1]](s32), [[C]]
+    ; GFX9: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[COPY1]](s32), [[C1]]
+    ; GFX9: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[UV1]], [[COPY1]](s32)
+    ; GFX9: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[UV]], [[COPY1]](s32)
+    ; GFX9: [[LSHR1:%[0-9]+]]:_(s64) = G_LSHR [[UV1]], [[SUB1]](s32)
+    ; GFX9: [[OR:%[0-9]+]]:_(s64) = G_OR [[LSHR]], [[LSHR1]]
+    ; GFX9: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 63
+    ; GFX9: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[UV1]], [[C2]](s32)
+    ; GFX9: [[ASHR2:%[0-9]+]]:_(s64) = G_ASHR [[UV1]], [[SUB]](s32)
+    ; GFX9: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[OR]], [[ASHR2]]
+    ; GFX9: [[SELECT1:%[0-9]+]]:_(s64) = G_SELECT [[ICMP1]](s1), [[UV]], [[SELECT]]
+    ; GFX9: [[SELECT2:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[ASHR]], [[ASHR1]]
+    ; GFX9: [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[SELECT1]](s64), [[SELECT2]](s64)
+    ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[MV]](s128)
+    %0:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    %1:_(s32) = COPY $vgpr4
+    %2:_(s128) = G_ASHR %0, %1
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %2
+...
+
+---
+name: test_ashr_s128_s32_0
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4
+
+    ; SI-LABEL: name: test_ashr_s128_s32_0
+    ; SI: [[COPY:%[0-9]+]]:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; SI: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](s128)
+    ; SI: [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[UV]](s64), [[UV1]](s64)
+    ; SI: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[MV]](s128)
+    ; VI-LABEL: name: test_ashr_s128_s32_0
+    ; VI: [[COPY:%[0-9]+]]:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; VI: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](s128)
+    ; VI: [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[UV]](s64), [[UV1]](s64)
+    ; VI: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[MV]](s128)
+    ; GFX9-LABEL: name: test_ashr_s128_s32_0
+    ; GFX9: [[COPY:%[0-9]+]]:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; GFX9: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](s128)
+    ; GFX9: [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[UV]](s64), [[UV1]](s64)
+    ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[MV]](s128)
+    %0:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    %1:_(s32) = G_CONSTANT i32 0
+    %3:_(s128) = G_ASHR %0, %1
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %3
+...
+
+
+---
+name: test_ashr_s128_s32_23
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4
+
+    ; SI-LABEL: name: test_ashr_s128_s32_23
+    ; SI: [[COPY:%[0-9]+]]:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; SI: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](s128)
+    ; SI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 23
+    ; SI: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[UV]], [[C]](s32)
+    ; SI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 41
+    ; SI: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[UV1]], [[C1]](s32)
+    ; SI: [[OR:%[0-9]+]]:_(s64) = G_OR [[LSHR]], [[SHL]]
+    ; SI: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[UV1]], [[C]](s32)
+    ; SI: [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[OR]](s64), [[ASHR]](s64)
+    ; SI: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[MV]](s128)
+    ; VI-LABEL: name: test_ashr_s128_s32_23
+    ; VI: [[COPY:%[0-9]+]]:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; VI: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](s128)
+    ; VI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 23
+    ; VI: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[UV]], [[C]](s32)
+    ; VI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 41
+    ; VI: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[UV1]], [[C1]](s32)
+    ; VI: [[OR:%[0-9]+]]:_(s64) = G_OR [[LSHR]], [[SHL]]
+    ; VI: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[UV1]], [[C]](s32)
+    ; VI: [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[OR]](s64), [[ASHR]](s64)
+    ; VI: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[MV]](s128)
+    ; GFX9-LABEL: name: test_ashr_s128_s32_23
+    ; GFX9: [[COPY:%[0-9]+]]:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; GFX9: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](s128)
+    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 23
+    ; GFX9: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[UV]], [[C]](s32)
+    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 41
+    ; GFX9: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[UV1]], [[C1]](s32)
+    ; GFX9: [[OR:%[0-9]+]]:_(s64) = G_OR [[LSHR]], [[SHL]]
+    ; GFX9: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[UV1]], [[C]](s32)
+    ; GFX9: [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[OR]](s64), [[ASHR]](s64)
+    ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[MV]](s128)
+    %0:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    %1:_(s32) = G_CONSTANT i32 23
+    %3:_(s128) = G_ASHR %0, %1
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %3
+...
+
+---
+name: test_ashr_s128_s32_31
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4
+
+    ; SI-LABEL: name: test_ashr_s128_s32_31
+    ; SI: [[COPY:%[0-9]+]]:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; SI: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](s128)
+    ; SI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 31
+    ; SI: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[UV]], [[C]](s32)
+    ; SI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 33
+    ; SI: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[UV1]], [[C1]](s32)
+    ; SI: [[OR:%[0-9]+]]:_(s64) = G_OR [[LSHR]], [[SHL]]
+    ; SI: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[UV1]], [[C]](s32)
+    ; SI: [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[OR]](s64), [[ASHR]](s64)
+    ; SI: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[MV]](s128)
+    ; VI-LABEL: name: test_ashr_s128_s32_31
+    ; VI: [[COPY:%[0-9]+]]:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; VI: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](s128)
+    ; VI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 31
+    ; VI: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[UV]], [[C]](s32)
+    ; VI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 33
+    ; VI: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[UV1]], [[C1]](s32)
+    ; VI: [[OR:%[0-9]+]]:_(s64) = G_OR [[LSHR]], [[SHL]]
+    ; VI: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[UV1]], [[C]](s32)
+    ; VI: [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[OR]](s64), [[ASHR]](s64)
+    ; VI: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[MV]](s128)
+    ; GFX9-LABEL: name: test_ashr_s128_s32_31
+    ; GFX9: [[COPY:%[0-9]+]]:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; GFX9: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](s128)
+    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 31
+    ; GFX9: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[UV]], [[C]](s32)
+    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 33
+    ; GFX9: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[UV1]], [[C1]](s32)
+    ; GFX9: [[OR:%[0-9]+]]:_(s64) = G_OR [[LSHR]], [[SHL]]
+    ; GFX9: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[UV1]], [[C]](s32)
+    ; GFX9: [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[OR]](s64), [[ASHR]](s64)
+    ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[MV]](s128)
+    %0:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    %1:_(s32) = G_CONSTANT i32 31
+    %3:_(s128) = G_ASHR %0, %1
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %3
+...
+
+---
+name: test_ashr_s128_s32_32
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4
+
+    ; SI-LABEL: name: test_ashr_s128_s32_32
+    ; SI: [[COPY:%[0-9]+]]:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; SI: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](s128)
+    ; SI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 32
+    ; SI: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[UV]], [[C]](s32)
+    ; SI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 32
+    ; SI: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[UV1]], [[C1]](s32)
+    ; SI: [[OR:%[0-9]+]]:_(s64) = G_OR [[LSHR]], [[SHL]]
+    ; SI: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[UV1]], [[C]](s32)
+    ; SI: [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[OR]](s64), [[ASHR]](s64)
+    ; SI: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[MV]](s128)
+    ; VI-LABEL: name: test_ashr_s128_s32_32
+    ; VI: [[COPY:%[0-9]+]]:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; VI: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](s128)
+    ; VI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 32
+    ; VI: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[UV]], [[C]](s32)
+    ; VI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 32
+    ; VI: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[UV1]], [[C1]](s32)
+    ; VI: [[OR:%[0-9]+]]:_(s64) = G_OR [[LSHR]], [[SHL]]
+    ; VI: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[UV1]], [[C]](s32)
+    ; VI: [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[OR]](s64), [[ASHR]](s64)
+    ; VI: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[MV]](s128)
+    ; GFX9-LABEL: name: test_ashr_s128_s32_32
+    ; GFX9: [[COPY:%[0-9]+]]:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; GFX9: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](s128)
+    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 32
+    ; GFX9: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[UV]], [[C]](s32)
+    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 32
+    ; GFX9: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[UV1]], [[C1]](s32)
+    ; GFX9: [[OR:%[0-9]+]]:_(s64) = G_OR [[LSHR]], [[SHL]]
+    ; GFX9: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[UV1]], [[C]](s32)
+    ; GFX9: [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[OR]](s64), [[ASHR]](s64)
+    ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[MV]](s128)
+    %0:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    %1:_(s32) = G_CONSTANT i32 32
+    %3:_(s128) = G_ASHR %0, %1
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %3
+...
+
+---
+name: test_ashr_s128_s32_33
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4
+
+    ; SI-LABEL: name: test_ashr_s128_s32_33
+    ; SI: [[COPY:%[0-9]+]]:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; SI: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](s128)
+    ; SI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 33
+    ; SI: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[UV]], [[C]](s32)
+    ; SI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 31
+    ; SI: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[UV1]], [[C1]](s32)
+    ; SI: [[OR:%[0-9]+]]:_(s64) = G_OR [[LSHR]], [[SHL]]
+    ; SI: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[UV1]], [[C]](s32)
+    ; SI: [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[OR]](s64), [[ASHR]](s64)
+    ; SI: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[MV]](s128)
+    ; VI-LABEL: name: test_ashr_s128_s32_33
+    ; VI: [[COPY:%[0-9]+]]:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; VI: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](s128)
+    ; VI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 33
+    ; VI: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[UV]], [[C]](s32)
+    ; VI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 31
+    ; VI: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[UV1]], [[C1]](s32)
+    ; VI: [[OR:%[0-9]+]]:_(s64) = G_OR [[LSHR]], [[SHL]]
+    ; VI: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[UV1]], [[C]](s32)
+    ; VI: [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[OR]](s64), [[ASHR]](s64)
+    ; VI: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[MV]](s128)
+    ; GFX9-LABEL: name: test_ashr_s128_s32_33
+    ; GFX9: [[COPY:%[0-9]+]]:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; GFX9: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](s128)
+    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 33
+    ; GFX9: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[UV]], [[C]](s32)
+    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 31
+    ; GFX9: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[UV1]], [[C1]](s32)
+    ; GFX9: [[OR:%[0-9]+]]:_(s64) = G_OR [[LSHR]], [[SHL]]
+    ; GFX9: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[UV1]], [[C]](s32)
+    ; GFX9: [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[OR]](s64), [[ASHR]](s64)
+    ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[MV]](s128)
+    %0:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    %1:_(s32) = G_CONSTANT i32 33
+    %3:_(s128) = G_ASHR %0, %1
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %3
+...
+
+---
+name: test_ashr_s128_s32_127
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4
+
+    ; SI-LABEL: name: test_ashr_s128_s32_127
+    ; SI: [[COPY:%[0-9]+]]:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; SI: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](s128)
+    ; SI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 63
+    ; SI: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[UV1]], [[C]](s32)
+    ; SI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 63
+    ; SI: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[UV1]], [[C1]](s32)
+    ; SI: [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[ASHR]](s64), [[ASHR1]](s64)
+    ; SI: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[MV]](s128)
+    ; VI-LABEL: name: test_ashr_s128_s32_127
+    ; VI: [[COPY:%[0-9]+]]:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; VI: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](s128)
+    ; VI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 63
+    ; VI: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[UV1]], [[C]](s32)
+    ; VI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 63
+    ; VI: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[UV1]], [[C1]](s32)
+    ; VI: [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[ASHR]](s64), [[ASHR1]](s64)
+    ; VI: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[MV]](s128)
+    ; GFX9-LABEL: name: test_ashr_s128_s32_127
+    ; GFX9: [[COPY:%[0-9]+]]:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; GFX9: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](s128)
+    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 63
+    ; GFX9: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[UV1]], [[C]](s32)
+    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 63
+    ; GFX9: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[UV1]], [[C1]](s32)
+    ; GFX9: [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[ASHR]](s64), [[ASHR1]](s64)
+    ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[MV]](s128)
+    %0:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    %1:_(s32) = G_CONSTANT i32 127
+    %3:_(s128) = G_ASHR %0, %1
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %3
+...
+
+---
+name: test_ashr_s256_s256
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8
+
+    ; SI-LABEL: name: test_ashr_s256_s256
+    ; SI: [[COPY:%[0-9]+]]:_(s256) = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+    ; SI: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr8
+    ; SI: [[ZEXT:%[0-9]+]]:_(s256) = G_ZEXT [[COPY1]](s32)
+    ; SI: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[ZEXT]](s256)
+    ; SI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 128
+    ; SI: [[UV:%[0-9]+]]:_(s128), [[UV1:%[0-9]+]]:_(s128) = G_UNMERGE_VALUES [[COPY]](s256)
+    ; SI: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[TRUNC]], [[C]]
+    ; SI: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[C]], [[TRUNC]]
+    ; SI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; SI: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[TRUNC]](s32), [[C]]
+    ; SI: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[TRUNC]](s32), [[C1]]
+    ; SI: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 64
+    ; SI: [[UV2:%[0-9]+]]:_(s64), [[UV3:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[UV1]](s128)
+    ; SI: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[TRUNC]], [[C2]]
+    ; SI: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[TRUNC]]
+    ; SI: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; SI: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[TRUNC]](s32), [[C2]]
+    ; SI: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[TRUNC]](s32), [[C3]]
+    ; SI: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[UV3]], [[TRUNC]](s32)
+    ; SI: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[UV2]], [[TRUNC]](s32)
+    ; SI: [[LSHR1:%[0-9]+]]:_(s64) = G_LSHR [[UV3]], [[SUB3]](s32)
+    ; SI: [[OR:%[0-9]+]]:_(s64) = G_OR [[LSHR]], [[LSHR1]]
+    ; SI: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 63
+    ; SI: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[UV3]], [[C4]](s32)
+    ; SI: [[ASHR2:%[0-9]+]]:_(s64) = G_ASHR [[UV3]], [[SUB2]](s32)
+    ; SI: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[ICMP2]](s1), [[OR]], [[ASHR2]]
+    ; SI: [[SELECT1:%[0-9]+]]:_(s64) = G_SELECT [[ICMP3]](s1), [[UV2]], [[SELECT]]
+    ; SI: [[SELECT2:%[0-9]+]]:_(s64) = G_SELECT [[ICMP2]](s1), [[ASHR]], [[ASHR1]]
+    ; SI: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 64
+    ; SI: [[UV4:%[0-9]+]]:_(s64), [[UV5:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[UV]](s128)
+    ; SI: [[SUB4:%[0-9]+]]:_(s32) = G_SUB [[TRUNC]], [[C5]]
+    ; SI: [[SUB5:%[0-9]+]]:_(s32) = G_SUB [[C5]], [[TRUNC]]
+    ; SI: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; SI: [[ICMP4:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[TRUNC]](s32), [[C5]]
+    ; SI: [[ICMP5:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[TRUNC]](s32), [[C6]]
+    ; SI: [[LSHR2:%[0-9]+]]:_(s64) = G_LSHR [[UV5]], [[TRUNC]](s32)
+    ; SI: [[LSHR3:%[0-9]+]]:_(s64) = G_LSHR [[UV4]], [[TRUNC]](s32)
+    ; SI: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[UV5]], [[SUB5]](s32)
+    ; SI: [[OR1:%[0-9]+]]:_(s64) = G_OR [[LSHR3]], [[SHL]]
+    ; SI: [[C7:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; SI: [[LSHR4:%[0-9]+]]:_(s64) = G_LSHR [[UV5]], [[SUB4]](s32)
+    ; SI: [[SELECT3:%[0-9]+]]:_(s64) = G_SELECT [[ICMP4]](s1), [[OR1]], [[LSHR4]]
+    ; SI: [[SELECT4:%[0-9]+]]:_(s64) = G_SELECT [[ICMP5]](s1), [[UV4]], [[SELECT3]]
+    ; SI: [[SELECT5:%[0-9]+]]:_(s64) = G_SELECT [[ICMP4]](s1), [[LSHR2]], [[C7]]
+    ; SI: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 64
+    ; SI: [[UV6:%[0-9]+]]:_(s64), [[UV7:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[UV1]](s128)
+    ; SI: [[SUB6:%[0-9]+]]:_(s32) = G_SUB [[SUB1]], [[C8]]
+    ; SI: [[SUB7:%[0-9]+]]:_(s32) = G_SUB [[C8]], [[SUB1]]
+    ; SI: [[C9:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; SI: [[ICMP6:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[SUB1]](s32), [[C8]]
+    ; SI: [[ICMP7:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[SUB1]](s32), [[C9]]
+    ; SI: [[LSHR5:%[0-9]+]]:_(s64) = G_LSHR [[UV7]], [[SUB1]](s32)
+    ; SI: [[LSHR6:%[0-9]+]]:_(s64) = G_LSHR [[UV6]], [[SUB1]](s32)
+    ; SI: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[UV7]], [[SUB7]](s32)
+    ; SI: [[OR2:%[0-9]+]]:_(s64) = G_OR [[LSHR6]], [[SHL1]]
+    ; SI: [[C10:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; SI: [[LSHR7:%[0-9]+]]:_(s64) = G_LSHR [[UV7]], [[SUB6]](s32)
+    ; SI: [[SELECT6:%[0-9]+]]:_(s64) = G_SELECT [[ICMP6]](s1), [[OR2]], [[LSHR7]]
+    ; SI: [[SELECT7:%[0-9]+]]:_(s64) = G_SELECT [[ICMP7]](s1), [[UV6]], [[SELECT6]]
+    ; SI: [[SELECT8:%[0-9]+]]:_(s64) = G_SELECT [[ICMP6]](s1), [[LSHR5]], [[C10]]
+    ; SI: [[OR3:%[0-9]+]]:_(s64) = G_OR [[SELECT4]], [[SELECT7]]
+    ; SI: [[OR4:%[0-9]+]]:_(s64) = G_OR [[SELECT5]], [[SELECT8]]
+    ; SI: [[UV8:%[0-9]+]]:_(s64), [[UV9:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[UV1]](s128)
+    ; SI: [[C11:%[0-9]+]]:_(s32) = G_CONSTANT i32 63
+    ; SI: [[ASHR3:%[0-9]+]]:_(s64) = G_ASHR [[UV9]], [[C11]](s32)
+    ; SI: [[C12:%[0-9]+]]:_(s32) = G_CONSTANT i32 63
+    ; SI: [[ASHR4:%[0-9]+]]:_(s64) = G_ASHR [[UV9]], [[C12]](s32)
+    ; SI: [[C13:%[0-9]+]]:_(s32) = G_CONSTANT i32 64
+    ; SI: [[UV10:%[0-9]+]]:_(s64), [[UV11:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[UV1]](s128)
+    ; SI: [[SUB8:%[0-9]+]]:_(s32) = G_SUB [[SUB]], [[C13]]
+    ; SI: [[SUB9:%[0-9]+]]:_(s32) = G_SUB [[C13]], [[SUB]]
+    ; SI: [[C14:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; SI: [[ICMP8:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[SUB]](s32), [[C13]]
+    ; SI: [[ICMP9:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[SUB]](s32), [[C14]]
+    ; SI: [[ASHR5:%[0-9]+]]:_(s64) = G_ASHR [[UV11]], [[SUB]](s32)
+    ; SI: [[LSHR8:%[0-9]+]]:_(s64) = G_LSHR [[UV10]], [[SUB]](s32)
+    ; SI: [[LSHR9:%[0-9]+]]:_(s64) = G_LSHR [[UV11]], [[SUB9]](s32)
+    ; SI: [[OR5:%[0-9]+]]:_(s64) = G_OR [[LSHR8]], [[LSHR9]]
+    ; SI: [[C15:%[0-9]+]]:_(s32) = G_CONSTANT i32 63
+    ; SI: [[ASHR6:%[0-9]+]]:_(s64) = G_ASHR [[UV11]], [[C15]](s32)
+    ; SI: [[ASHR7:%[0-9]+]]:_(s64) = G_ASHR [[UV11]], [[SUB8]](s32)
+    ; SI: [[SELECT9:%[0-9]+]]:_(s64) = G_SELECT [[ICMP8]](s1), [[OR5]], [[ASHR7]]
+    ; SI: [[SELECT10:%[0-9]+]]:_(s64) = G_SELECT [[ICMP9]](s1), [[UV10]], [[SELECT9]]
+    ; SI: [[SELECT11:%[0-9]+]]:_(s64) = G_SELECT [[ICMP8]](s1), [[ASHR5]], [[ASHR6]]
+    ; SI: [[SELECT12:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[OR3]], [[SELECT10]]
+    ; SI: [[SELECT13:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[OR4]], [[SELECT11]]
+    ; SI: [[UV12:%[0-9]+]]:_(s64), [[UV13:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[UV]](s128)
+    ; SI: [[SELECT14:%[0-9]+]]:_(s64) = G_SELECT [[ICMP1]](s1), [[UV12]], [[SELECT12]]
+    ; SI: [[SELECT15:%[0-9]+]]:_(s64) = G_SELECT [[ICMP1]](s1), [[UV13]], [[SELECT13]]
+    ; SI: [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[SELECT14]](s64), [[SELECT15]](s64)
+    ; SI: [[SELECT16:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[SELECT1]], [[ASHR3]]
+    ; SI: [[SELECT17:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[SELECT2]], [[ASHR4]]
+    ; SI: [[MV1:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[SELECT16]](s64), [[SELECT17]](s64)
+    ; SI: [[MV2:%[0-9]+]]:_(s256) = G_MERGE_VALUES [[MV]](s128), [[MV1]](s128)
+    ; SI: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY [[MV2]](s256)
+    ; VI-LABEL: name: test_ashr_s256_s256
+    ; VI: [[COPY:%[0-9]+]]:_(s256) = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+    ; VI: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr8
+    ; VI: [[ZEXT:%[0-9]+]]:_(s256) = G_ZEXT [[COPY1]](s32)
+    ; VI: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[ZEXT]](s256)
+    ; VI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 128
+    ; VI: [[UV:%[0-9]+]]:_(s128), [[UV1:%[0-9]+]]:_(s128) = G_UNMERGE_VALUES [[COPY]](s256)
+    ; VI: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[TRUNC]], [[C]]
+    ; VI: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[C]], [[TRUNC]]
+    ; VI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; VI: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[TRUNC]](s32), [[C]]
+    ; VI: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[TRUNC]](s32), [[C1]]
+    ; VI: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 64
+    ; VI: [[UV2:%[0-9]+]]:_(s64), [[UV3:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[UV1]](s128)
+    ; VI: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[TRUNC]], [[C2]]
+    ; VI: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[TRUNC]]
+    ; VI: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; VI: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[TRUNC]](s32), [[C2]]
+    ; VI: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[TRUNC]](s32), [[C3]]
+    ; VI: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[UV3]], [[TRUNC]](s32)
+    ; VI: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[UV2]], [[TRUNC]](s32)
+    ; VI: [[LSHR1:%[0-9]+]]:_(s64) = G_LSHR [[UV3]], [[SUB3]](s32)
+    ; VI: [[OR:%[0-9]+]]:_(s64) = G_OR [[LSHR]], [[LSHR1]]
+    ; VI: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 63
+    ; VI: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[UV3]], [[C4]](s32)
+    ; VI: [[ASHR2:%[0-9]+]]:_(s64) = G_ASHR [[UV3]], [[SUB2]](s32)
+    ; VI: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[ICMP2]](s1), [[OR]], [[ASHR2]]
+    ; VI: [[SELECT1:%[0-9]+]]:_(s64) = G_SELECT [[ICMP3]](s1), [[UV2]], [[SELECT]]
+    ; VI: [[SELECT2:%[0-9]+]]:_(s64) = G_SELECT [[ICMP2]](s1), [[ASHR]], [[ASHR1]]
+    ; VI: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 64
+    ; VI: [[UV4:%[0-9]+]]:_(s64), [[UV5:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[UV]](s128)
+    ; VI: [[SUB4:%[0-9]+]]:_(s32) = G_SUB [[TRUNC]], [[C5]]
+    ; VI: [[SUB5:%[0-9]+]]:_(s32) = G_SUB [[C5]], [[TRUNC]]
+    ; VI: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; VI: [[ICMP4:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[TRUNC]](s32), [[C5]]
+    ; VI: [[ICMP5:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[TRUNC]](s32), [[C6]]
+    ; VI: [[LSHR2:%[0-9]+]]:_(s64) = G_LSHR [[UV5]], [[TRUNC]](s32)
+    ; VI: [[LSHR3:%[0-9]+]]:_(s64) = G_LSHR [[UV4]], [[TRUNC]](s32)
+    ; VI: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[UV5]], [[SUB5]](s32)
+    ; VI: [[OR1:%[0-9]+]]:_(s64) = G_OR [[LSHR3]], [[SHL]]
+    ; VI: [[C7:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; VI: [[LSHR4:%[0-9]+]]:_(s64) = G_LSHR [[UV5]], [[SUB4]](s32)
+    ; VI: [[SELECT3:%[0-9]+]]:_(s64) = G_SELECT [[ICMP4]](s1), [[OR1]], [[LSHR4]]
+    ; VI: [[SELECT4:%[0-9]+]]:_(s64) = G_SELECT [[ICMP5]](s1), [[UV4]], [[SELECT3]]
+    ; VI: [[SELECT5:%[0-9]+]]:_(s64) = G_SELECT [[ICMP4]](s1), [[LSHR2]], [[C7]]
+    ; VI: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 64
+    ; VI: [[UV6:%[0-9]+]]:_(s64), [[UV7:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[UV1]](s128)
+    ; VI: [[SUB6:%[0-9]+]]:_(s32) = G_SUB [[SUB1]], [[C8]]
+    ; VI: [[SUB7:%[0-9]+]]:_(s32) = G_SUB [[C8]], [[SUB1]]
+    ; VI: [[C9:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; VI: [[ICMP6:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[SUB1]](s32), [[C8]]
+    ; VI: [[ICMP7:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[SUB1]](s32), [[C9]]
+    ; VI: [[LSHR5:%[0-9]+]]:_(s64) = G_LSHR [[UV7]], [[SUB1]](s32)
+    ; VI: [[LSHR6:%[0-9]+]]:_(s64) = G_LSHR [[UV6]], [[SUB1]](s32)
+    ; VI: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[UV7]], [[SUB7]](s32)
+    ; VI: [[OR2:%[0-9]+]]:_(s64) = G_OR [[LSHR6]], [[SHL1]]
+    ; VI: [[C10:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; VI: [[LSHR7:%[0-9]+]]:_(s64) = G_LSHR [[UV7]], [[SUB6]](s32)
+    ; VI: [[SELECT6:%[0-9]+]]:_(s64) = G_SELECT [[ICMP6]](s1), [[OR2]], [[LSHR7]]
+    ; VI: [[SELECT7:%[0-9]+]]:_(s64) = G_SELECT [[ICMP7]](s1), [[UV6]], [[SELECT6]]
+    ; VI: [[SELECT8:%[0-9]+]]:_(s64) = G_SELECT [[ICMP6]](s1), [[LSHR5]], [[C10]]
+    ; VI: [[OR3:%[0-9]+]]:_(s64) = G_OR [[SELECT4]], [[SELECT7]]
+    ; VI: [[OR4:%[0-9]+]]:_(s64) = G_OR [[SELECT5]], [[SELECT8]]
+    ; VI: [[UV8:%[0-9]+]]:_(s64), [[UV9:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[UV1]](s128)
+    ; VI: [[C11:%[0-9]+]]:_(s32) = G_CONSTANT i32 63
+    ; VI: [[ASHR3:%[0-9]+]]:_(s64) = G_ASHR [[UV9]], [[C11]](s32)
+    ; VI: [[C12:%[0-9]+]]:_(s32) = G_CONSTANT i32 63
+    ; VI: [[ASHR4:%[0-9]+]]:_(s64) = G_ASHR [[UV9]], [[C12]](s32)
+    ; VI: [[C13:%[0-9]+]]:_(s32) = G_CONSTANT i32 64
+    ; VI: [[UV10:%[0-9]+]]:_(s64), [[UV11:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[UV1]](s128)
+    ; VI: [[SUB8:%[0-9]+]]:_(s32) = G_SUB [[SUB]], [[C13]]
+    ; VI: [[SUB9:%[0-9]+]]:_(s32) = G_SUB [[C13]], [[SUB]]
+    ; VI: [[C14:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; VI: [[ICMP8:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[SUB]](s32), [[C13]]
+    ; VI: [[ICMP9:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[SUB]](s32), [[C14]]
+    ; VI: [[ASHR5:%[0-9]+]]:_(s64) = G_ASHR [[UV11]], [[SUB]](s32)
+    ; VI: [[LSHR8:%[0-9]+]]:_(s64) = G_LSHR [[UV10]], [[SUB]](s32)
+    ; VI: [[LSHR9:%[0-9]+]]:_(s64) = G_LSHR [[UV11]], [[SUB9]](s32)
+    ; VI: [[OR5:%[0-9]+]]:_(s64) = G_OR [[LSHR8]], [[LSHR9]]
+    ; VI: [[C15:%[0-9]+]]:_(s32) = G_CONSTANT i32 63
+    ; VI: [[ASHR6:%[0-9]+]]:_(s64) = G_ASHR [[UV11]], [[C15]](s32)
+    ; VI: [[ASHR7:%[0-9]+]]:_(s64) = G_ASHR [[UV11]], [[SUB8]](s32)
+    ; VI: [[SELECT9:%[0-9]+]]:_(s64) = G_SELECT [[ICMP8]](s1), [[OR5]], [[ASHR7]]
+    ; VI: [[SELECT10:%[0-9]+]]:_(s64) = G_SELECT [[ICMP9]](s1), [[UV10]], [[SELECT9]]
+    ; VI: [[SELECT11:%[0-9]+]]:_(s64) = G_SELECT [[ICMP8]](s1), [[ASHR5]], [[ASHR6]]
+    ; VI: [[SELECT12:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[OR3]], [[SELECT10]]
+    ; VI: [[SELECT13:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[OR4]], [[SELECT11]]
+    ; VI: [[UV12:%[0-9]+]]:_(s64), [[UV13:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[UV]](s128)
+    ; VI: [[SELECT14:%[0-9]+]]:_(s64) = G_SELECT [[ICMP1]](s1), [[UV12]], [[SELECT12]]
+    ; VI: [[SELECT15:%[0-9]+]]:_(s64) = G_SELECT [[ICMP1]](s1), [[UV13]], [[SELECT13]]
+    ; VI: [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[SELECT14]](s64), [[SELECT15]](s64)
+    ; VI: [[SELECT16:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[SELECT1]], [[ASHR3]]
+    ; VI: [[SELECT17:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[SELECT2]], [[ASHR4]]
+    ; VI: [[MV1:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[SELECT16]](s64), [[SELECT17]](s64)
+    ; VI: [[MV2:%[0-9]+]]:_(s256) = G_MERGE_VALUES [[MV]](s128), [[MV1]](s128)
+    ; VI: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY [[MV2]](s256)
+    ; GFX9-LABEL: name: test_ashr_s256_s256
+    ; GFX9: [[COPY:%[0-9]+]]:_(s256) = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+    ; GFX9: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr8
+    ; GFX9: [[ZEXT:%[0-9]+]]:_(s256) = G_ZEXT [[COPY1]](s32)
+    ; GFX9: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[ZEXT]](s256)
+    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 128
+    ; GFX9: [[UV:%[0-9]+]]:_(s128), [[UV1:%[0-9]+]]:_(s128) = G_UNMERGE_VALUES [[COPY]](s256)
+    ; GFX9: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[TRUNC]], [[C]]
+    ; GFX9: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[C]], [[TRUNC]]
+    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; GFX9: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[TRUNC]](s32), [[C]]
+    ; GFX9: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[TRUNC]](s32), [[C1]]
+    ; GFX9: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 64
+    ; GFX9: [[UV2:%[0-9]+]]:_(s64), [[UV3:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[UV1]](s128)
+    ; GFX9: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[TRUNC]], [[C2]]
+    ; GFX9: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[TRUNC]]
+    ; GFX9: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; GFX9: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[TRUNC]](s32), [[C2]]
+    ; GFX9: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[TRUNC]](s32), [[C3]]
+    ; GFX9: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[UV3]], [[TRUNC]](s32)
+    ; GFX9: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[UV2]], [[TRUNC]](s32)
+    ; GFX9: [[LSHR1:%[0-9]+]]:_(s64) = G_LSHR [[UV3]], [[SUB3]](s32)
+    ; GFX9: [[OR:%[0-9]+]]:_(s64) = G_OR [[LSHR]], [[LSHR1]]
+    ; GFX9: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 63
+    ; GFX9: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[UV3]], [[C4]](s32)
+    ; GFX9: [[ASHR2:%[0-9]+]]:_(s64) = G_ASHR [[UV3]], [[SUB2]](s32)
+    ; GFX9: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[ICMP2]](s1), [[OR]], [[ASHR2]]
+    ; GFX9: [[SELECT1:%[0-9]+]]:_(s64) = G_SELECT [[ICMP3]](s1), [[UV2]], [[SELECT]]
+    ; GFX9: [[SELECT2:%[0-9]+]]:_(s64) = G_SELECT [[ICMP2]](s1), [[ASHR]], [[ASHR1]]
+    ; GFX9: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 64
+    ; GFX9: [[UV4:%[0-9]+]]:_(s64), [[UV5:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[UV]](s128)
+    ; GFX9: [[SUB4:%[0-9]+]]:_(s32) = G_SUB [[TRUNC]], [[C5]]
+    ; GFX9: [[SUB5:%[0-9]+]]:_(s32) = G_SUB [[C5]], [[TRUNC]]
+    ; GFX9: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; GFX9: [[ICMP4:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[TRUNC]](s32), [[C5]]
+    ; GFX9: [[ICMP5:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[TRUNC]](s32), [[C6]]
+    ; GFX9: [[LSHR2:%[0-9]+]]:_(s64) = G_LSHR [[UV5]], [[TRUNC]](s32)
+    ; GFX9: [[LSHR3:%[0-9]+]]:_(s64) = G_LSHR [[UV4]], [[TRUNC]](s32)
+    ; GFX9: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[UV5]], [[SUB5]](s32)
+    ; GFX9: [[OR1:%[0-9]+]]:_(s64) = G_OR [[LSHR3]], [[SHL]]
+    ; GFX9: [[C7:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; GFX9: [[LSHR4:%[0-9]+]]:_(s64) = G_LSHR [[UV5]], [[SUB4]](s32)
+    ; GFX9: [[SELECT3:%[0-9]+]]:_(s64) = G_SELECT [[ICMP4]](s1), [[OR1]], [[LSHR4]]
+    ; GFX9: [[SELECT4:%[0-9]+]]:_(s64) = G_SELECT [[ICMP5]](s1), [[UV4]], [[SELECT3]]
+    ; GFX9: [[SELECT5:%[0-9]+]]:_(s64) = G_SELECT [[ICMP4]](s1), [[LSHR2]], [[C7]]
+    ; GFX9: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 64
+    ; GFX9: [[UV6:%[0-9]+]]:_(s64), [[UV7:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[UV1]](s128)
+    ; GFX9: [[SUB6:%[0-9]+]]:_(s32) = G_SUB [[SUB1]], [[C8]]
+    ; GFX9: [[SUB7:%[0-9]+]]:_(s32) = G_SUB [[C8]], [[SUB1]]
+    ; GFX9: [[C9:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; GFX9: [[ICMP6:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[SUB1]](s32), [[C8]]
+    ; GFX9: [[ICMP7:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[SUB1]](s32), [[C9]]
+    ; GFX9: [[LSHR5:%[0-9]+]]:_(s64) = G_LSHR [[UV7]], [[SUB1]](s32)
+    ; GFX9: [[LSHR6:%[0-9]+]]:_(s64) = G_LSHR [[UV6]], [[SUB1]](s32)
+    ; GFX9: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[UV7]], [[SUB7]](s32)
+    ; GFX9: [[OR2:%[0-9]+]]:_(s64) = G_OR [[LSHR6]], [[SHL1]]
+    ; GFX9: [[C10:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; GFX9: [[LSHR7:%[0-9]+]]:_(s64) = G_LSHR [[UV7]], [[SUB6]](s32)
+    ; GFX9: [[SELECT6:%[0-9]+]]:_(s64) = G_SELECT [[ICMP6]](s1), [[OR2]], [[LSHR7]]
+    ; GFX9: [[SELECT7:%[0-9]+]]:_(s64) = G_SELECT [[ICMP7]](s1), [[UV6]], [[SELECT6]]
+    ; GFX9: [[SELECT8:%[0-9]+]]:_(s64) = G_SELECT [[ICMP6]](s1), [[LSHR5]], [[C10]]
+    ; GFX9: [[OR3:%[0-9]+]]:_(s64) = G_OR [[SELECT4]], [[SELECT7]]
+    ; GFX9: [[OR4:%[0-9]+]]:_(s64) = G_OR [[SELECT5]], [[SELECT8]]
+    ; GFX9: [[UV8:%[0-9]+]]:_(s64), [[UV9:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[UV1]](s128)
+    ; GFX9: [[C11:%[0-9]+]]:_(s32) = G_CONSTANT i32 63
+    ; GFX9: [[ASHR3:%[0-9]+]]:_(s64) = G_ASHR [[UV9]], [[C11]](s32)
+    ; GFX9: [[C12:%[0-9]+]]:_(s32) = G_CONSTANT i32 63
+    ; GFX9: [[ASHR4:%[0-9]+]]:_(s64) = G_ASHR [[UV9]], [[C12]](s32)
+    ; GFX9: [[C13:%[0-9]+]]:_(s32) = G_CONSTANT i32 64
+    ; GFX9: [[UV10:%[0-9]+]]:_(s64), [[UV11:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[UV1]](s128)
+    ; GFX9: [[SUB8:%[0-9]+]]:_(s32) = G_SUB [[SUB]], [[C13]]
+    ; GFX9: [[SUB9:%[0-9]+]]:_(s32) = G_SUB [[C13]], [[SUB]]
+    ; GFX9: [[C14:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; GFX9: [[ICMP8:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[SUB]](s32), [[C13]]
+    ; GFX9: [[ICMP9:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[SUB]](s32), [[C14]]
+    ; GFX9: [[ASHR5:%[0-9]+]]:_(s64) = G_ASHR [[UV11]], [[SUB]](s32)
+    ; GFX9: [[LSHR8:%[0-9]+]]:_(s64) = G_LSHR [[UV10]], [[SUB]](s32)
+    ; GFX9: [[LSHR9:%[0-9]+]]:_(s64) = G_LSHR [[UV11]], [[SUB9]](s32)
+    ; GFX9: [[OR5:%[0-9]+]]:_(s64) = G_OR [[LSHR8]], [[LSHR9]]
+    ; GFX9: [[C15:%[0-9]+]]:_(s32) = G_CONSTANT i32 63
+    ; GFX9: [[ASHR6:%[0-9]+]]:_(s64) = G_ASHR [[UV11]], [[C15]](s32)
+    ; GFX9: [[ASHR7:%[0-9]+]]:_(s64) = G_ASHR [[UV11]], [[SUB8]](s32)
+    ; GFX9: [[SELECT9:%[0-9]+]]:_(s64) = G_SELECT [[ICMP8]](s1), [[OR5]], [[ASHR7]]
+    ; GFX9: [[SELECT10:%[0-9]+]]:_(s64) = G_SELECT [[ICMP9]](s1), [[UV10]], [[SELECT9]]
+    ; GFX9: [[SELECT11:%[0-9]+]]:_(s64) = G_SELECT [[ICMP8]](s1), [[ASHR5]], [[ASHR6]]
+    ; GFX9: [[SELECT12:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[OR3]], [[SELECT10]]
+    ; GFX9: [[SELECT13:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[OR4]], [[SELECT11]]
+    ; GFX9: [[UV12:%[0-9]+]]:_(s64), [[UV13:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[UV]](s128)
+    ; GFX9: [[SELECT14:%[0-9]+]]:_(s64) = G_SELECT [[ICMP1]](s1), [[UV12]], [[SELECT12]]
+    ; GFX9: [[SELECT15:%[0-9]+]]:_(s64) = G_SELECT [[ICMP1]](s1), [[UV13]], [[SELECT13]]
+    ; GFX9: [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[SELECT14]](s64), [[SELECT15]](s64)
+    ; GFX9: [[SELECT16:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[SELECT1]], [[ASHR3]]
+    ; GFX9: [[SELECT17:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[SELECT2]], [[ASHR4]]
+    ; GFX9: [[MV1:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[SELECT16]](s64), [[SELECT17]](s64)
+    ; GFX9: [[MV2:%[0-9]+]]:_(s256) = G_MERGE_VALUES [[MV]](s128), [[MV1]](s128)
+    ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY [[MV2]](s256)
+    %0:_(s256) = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+    %1:_(s32) = COPY $vgpr8
+    %2:_(s256) = G_ZEXT %1
+    %3:_(s256) = G_ASHR %0, %2
+    $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7  = COPY %3
+...
+
+---
+name: test_ashr_v2s128_v2s32
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, $vgpr4_vgpr5
+
+    ; SI-LABEL: name: test_ashr_v2s128_v2s32
+    ; SI: [[COPY:%[0-9]+]]:_(<2 x s128>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+    ; SI: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr4_vgpr5
+    ; SI: [[UV:%[0-9]+]]:_(s128), [[UV1:%[0-9]+]]:_(s128) = G_UNMERGE_VALUES [[COPY]](<2 x s128>)
+    ; SI: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<2 x s32>)
+    ; SI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 64
+    ; SI: [[UV4:%[0-9]+]]:_(s64), [[UV5:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[UV]](s128)
+    ; SI: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[UV2]], [[C]]
+    ; SI: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[C]], [[UV2]]
+    ; SI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; SI: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[UV2]](s32), [[C]]
+    ; SI: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[UV2]](s32), [[C1]]
+    ; SI: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[UV5]], [[UV2]](s32)
+    ; SI: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[UV4]], [[UV2]](s32)
+    ; SI: [[LSHR1:%[0-9]+]]:_(s64) = G_LSHR [[UV5]], [[SUB1]](s32)
+    ; SI: [[OR:%[0-9]+]]:_(s64) = G_OR [[LSHR]], [[LSHR1]]
+    ; SI: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 63
+    ; SI: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[UV5]], [[C2]](s32)
+    ; SI: [[ASHR2:%[0-9]+]]:_(s64) = G_ASHR [[UV5]], [[SUB]](s32)
+    ; SI: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[OR]], [[ASHR2]]
+    ; SI: [[SELECT1:%[0-9]+]]:_(s64) = G_SELECT [[ICMP1]](s1), [[UV4]], [[SELECT]]
+    ; SI: [[SELECT2:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[ASHR]], [[ASHR1]]
+    ; SI: [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[SELECT1]](s64), [[SELECT2]](s64)
+    ; SI: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 64
+    ; SI: [[UV6:%[0-9]+]]:_(s64), [[UV7:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[UV1]](s128)
+    ; SI: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[UV3]], [[C3]]
+    ; SI: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[C3]], [[UV3]]
+    ; SI: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; SI: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[UV3]](s32), [[C3]]
+    ; SI: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[UV3]](s32), [[C4]]
+    ; SI: [[ASHR3:%[0-9]+]]:_(s64) = G_ASHR [[UV7]], [[UV3]](s32)
+    ; SI: [[LSHR2:%[0-9]+]]:_(s64) = G_LSHR [[UV6]], [[UV3]](s32)
+    ; SI: [[LSHR3:%[0-9]+]]:_(s64) = G_LSHR [[UV7]], [[SUB3]](s32)
+    ; SI: [[OR1:%[0-9]+]]:_(s64) = G_OR [[LSHR2]], [[LSHR3]]
+    ; SI: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 63
+    ; SI: [[ASHR4:%[0-9]+]]:_(s64) = G_ASHR [[UV7]], [[C5]](s32)
+    ; SI: [[ASHR5:%[0-9]+]]:_(s64) = G_ASHR [[UV7]], [[SUB2]](s32)
+    ; SI: [[SELECT3:%[0-9]+]]:_(s64) = G_SELECT [[ICMP2]](s1), [[OR1]], [[ASHR5]]
+    ; SI: [[SELECT4:%[0-9]+]]:_(s64) = G_SELECT [[ICMP3]](s1), [[UV6]], [[SELECT3]]
+    ; SI: [[SELECT5:%[0-9]+]]:_(s64) = G_SELECT [[ICMP2]](s1), [[ASHR3]], [[ASHR4]]
+    ; SI: [[MV1:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[SELECT4]](s64), [[SELECT5]](s64)
+    ; SI: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s128>) = G_BUILD_VECTOR [[MV]](s128), [[MV1]](s128)
+    ; SI: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY [[BUILD_VECTOR]](<2 x s128>)
+    ; VI-LABEL: name: test_ashr_v2s128_v2s32
+    ; VI: [[COPY:%[0-9]+]]:_(<2 x s128>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+    ; VI: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr4_vgpr5
+    ; VI: [[UV:%[0-9]+]]:_(s128), [[UV1:%[0-9]+]]:_(s128) = G_UNMERGE_VALUES [[COPY]](<2 x s128>)
+    ; VI: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<2 x s32>)
+    ; VI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 64
+    ; VI: [[UV4:%[0-9]+]]:_(s64), [[UV5:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[UV]](s128)
+    ; VI: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[UV2]], [[C]]
+    ; VI: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[C]], [[UV2]]
+    ; VI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; VI: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[UV2]](s32), [[C]]
+    ; VI: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[UV2]](s32), [[C1]]
+    ; VI: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[UV5]], [[UV2]](s32)
+    ; VI: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[UV4]], [[UV2]](s32)
+    ; VI: [[LSHR1:%[0-9]+]]:_(s64) = G_LSHR [[UV5]], [[SUB1]](s32)
+    ; VI: [[OR:%[0-9]+]]:_(s64) = G_OR [[LSHR]], [[LSHR1]]
+    ; VI: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 63
+    ; VI: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[UV5]], [[C2]](s32)
+    ; VI: [[ASHR2:%[0-9]+]]:_(s64) = G_ASHR [[UV5]], [[SUB]](s32)
+    ; VI: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[OR]], [[ASHR2]]
+    ; VI: [[SELECT1:%[0-9]+]]:_(s64) = G_SELECT [[ICMP1]](s1), [[UV4]], [[SELECT]]
+    ; VI: [[SELECT2:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[ASHR]], [[ASHR1]]
+    ; VI: [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[SELECT1]](s64), [[SELECT2]](s64)
+    ; VI: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 64
+    ; VI: [[UV6:%[0-9]+]]:_(s64), [[UV7:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[UV1]](s128)
+    ; VI: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[UV3]], [[C3]]
+    ; VI: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[C3]], [[UV3]]
+    ; VI: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; VI: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[UV3]](s32), [[C3]]
+    ; VI: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[UV3]](s32), [[C4]]
+    ; VI: [[ASHR3:%[0-9]+]]:_(s64) = G_ASHR [[UV7]], [[UV3]](s32)
+    ; VI: [[LSHR2:%[0-9]+]]:_(s64) = G_LSHR [[UV6]], [[UV3]](s32)
+    ; VI: [[LSHR3:%[0-9]+]]:_(s64) = G_LSHR [[UV7]], [[SUB3]](s32)
+    ; VI: [[OR1:%[0-9]+]]:_(s64) = G_OR [[LSHR2]], [[LSHR3]]
+    ; VI: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 63
+    ; VI: [[ASHR4:%[0-9]+]]:_(s64) = G_ASHR [[UV7]], [[C5]](s32)
+    ; VI: [[ASHR5:%[0-9]+]]:_(s64) = G_ASHR [[UV7]], [[SUB2]](s32)
+    ; VI: [[SELECT3:%[0-9]+]]:_(s64) = G_SELECT [[ICMP2]](s1), [[OR1]], [[ASHR5]]
+    ; VI: [[SELECT4:%[0-9]+]]:_(s64) = G_SELECT [[ICMP3]](s1), [[UV6]], [[SELECT3]]
+    ; VI: [[SELECT5:%[0-9]+]]:_(s64) = G_SELECT [[ICMP2]](s1), [[ASHR3]], [[ASHR4]]
+    ; VI: [[MV1:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[SELECT4]](s64), [[SELECT5]](s64)
+    ; VI: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s128>) = G_BUILD_VECTOR [[MV]](s128), [[MV1]](s128)
+    ; VI: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY [[BUILD_VECTOR]](<2 x s128>)
+    ; GFX9-LABEL: name: test_ashr_v2s128_v2s32
+    ; GFX9: [[COPY:%[0-9]+]]:_(<2 x s128>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+    ; GFX9: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr4_vgpr5
+    ; GFX9: [[UV:%[0-9]+]]:_(s128), [[UV1:%[0-9]+]]:_(s128) = G_UNMERGE_VALUES [[COPY]](<2 x s128>)
+    ; GFX9: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<2 x s32>)
+    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 64
+    ; GFX9: [[UV4:%[0-9]+]]:_(s64), [[UV5:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[UV]](s128)
+    ; GFX9: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[UV2]], [[C]]
+    ; GFX9: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[C]], [[UV2]]
+    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; GFX9: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[UV2]](s32), [[C]]
+    ; GFX9: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[UV2]](s32), [[C1]]
+    ; GFX9: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[UV5]], [[UV2]](s32)
+    ; GFX9: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[UV4]], [[UV2]](s32)
+    ; GFX9: [[LSHR1:%[0-9]+]]:_(s64) = G_LSHR [[UV5]], [[SUB1]](s32)
+    ; GFX9: [[OR:%[0-9]+]]:_(s64) = G_OR [[LSHR]], [[LSHR1]]
+    ; GFX9: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 63
+    ; GFX9: [[ASHR1:%[0-9]+]]:_(s64) = G_ASHR [[UV5]], [[C2]](s32)
+    ; GFX9: [[ASHR2:%[0-9]+]]:_(s64) = G_ASHR [[UV5]], [[SUB]](s32)
+    ; GFX9: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[OR]], [[ASHR2]]
+    ; GFX9: [[SELECT1:%[0-9]+]]:_(s64) = G_SELECT [[ICMP1]](s1), [[UV4]], [[SELECT]]
+    ; GFX9: [[SELECT2:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[ASHR]], [[ASHR1]]
+    ; GFX9: [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[SELECT1]](s64), [[SELECT2]](s64)
+    ; GFX9: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 64
+    ; GFX9: [[UV6:%[0-9]+]]:_(s64), [[UV7:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[UV1]](s128)
+    ; GFX9: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[UV3]], [[C3]]
+    ; GFX9: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[C3]], [[UV3]]
+    ; GFX9: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; GFX9: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[UV3]](s32), [[C3]]
+    ; GFX9: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[UV3]](s32), [[C4]]
+    ; GFX9: [[ASHR3:%[0-9]+]]:_(s64) = G_ASHR [[UV7]], [[UV3]](s32)
+    ; GFX9: [[LSHR2:%[0-9]+]]:_(s64) = G_LSHR [[UV6]], [[UV3]](s32)
+    ; GFX9: [[LSHR3:%[0-9]+]]:_(s64) = G_LSHR [[UV7]], [[SUB3]](s32)
+    ; GFX9: [[OR1:%[0-9]+]]:_(s64) = G_OR [[LSHR2]], [[LSHR3]]
+    ; GFX9: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 63
+    ; GFX9: [[ASHR4:%[0-9]+]]:_(s64) = G_ASHR [[UV7]], [[C5]](s32)
+    ; GFX9: [[ASHR5:%[0-9]+]]:_(s64) = G_ASHR [[UV7]], [[SUB2]](s32)
+    ; GFX9: [[SELECT3:%[0-9]+]]:_(s64) = G_SELECT [[ICMP2]](s1), [[OR1]], [[ASHR5]]
+    ; GFX9: [[SELECT4:%[0-9]+]]:_(s64) = G_SELECT [[ICMP3]](s1), [[UV6]], [[SELECT3]]
+    ; GFX9: [[SELECT5:%[0-9]+]]:_(s64) = G_SELECT [[ICMP2]](s1), [[ASHR3]], [[ASHR4]]
+    ; GFX9: [[MV1:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[SELECT4]](s64), [[SELECT5]](s64)
+    ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s128>) = G_BUILD_VECTOR [[MV]](s128), [[MV1]](s128)
+    ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY [[BUILD_VECTOR]](<2 x s128>)
+    %0:_(<2 x s128>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+    %1:_(<2 x s32>) = COPY $vgpr4_vgpr5
+    %2:_(<2 x s128>) = G_ASHR %0, %1
+    $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7  = COPY %2
+...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/legalize-atomic-cmpxchg.mir b/test/CodeGen/AMDGPU/GlobalISel/legalize-atomic-cmpxchg.mir
new file mode 100644
index 0000000000000000000000000000000000000000..738106de9f79a25896da5f47215c62a3cea2d5f7
--- /dev/null
+++ b/test/CodeGen/AMDGPU/GlobalISel/legalize-atomic-cmpxchg.mir
@@ -0,0 +1,71 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=tahiti -O0 -run-pass=legalizer %s -o - | FileCheck %s
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -O0 -run-pass=legalizer %s -o - | FileCheck %s
+
+---
+name: atomic_cmpxchg_global_i32
+
+body: |
+  bb.0:
+    liveins: $sgpr0_sgpr1, $sgpr2, $sgpr3
+    ; CHECK-LABEL: name: atomic_cmpxchg_global_i32
+    ; CHECK: [[COPY:%[0-9]+]]:_(p1) = COPY $sgpr0_sgpr1
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr2
+    ; CHECK: [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr3
+    ; CHECK: [[ATOMIC_CMPXCHG:%[0-9]+]]:_(s32) = G_ATOMIC_CMPXCHG [[COPY]](p1), [[COPY1]], [[COPY2]] :: (load store seq_cst 4, addrspace 1)
+    %0:_(p1) = COPY $sgpr0_sgpr1
+    %1:_(s32) = COPY $sgpr2
+    %2:_(s32) = COPY $sgpr3
+    %3:_(s32) = G_ATOMIC_CMPXCHG %0, %1, %2 :: (load store seq_cst 4, addrspace 1)
+...
+
+---
+name: atomic_cmpxchg_local_i32
+
+body: |
+  bb.0:
+    liveins: $sgpr0, $sgpr1, $sgpr2
+    ; CHECK-LABEL: name: atomic_cmpxchg_local_i32
+    ; CHECK: [[COPY:%[0-9]+]]:_(p3) = COPY $sgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr1
+    ; CHECK: [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr2
+    ; CHECK: [[ATOMIC_CMPXCHG:%[0-9]+]]:_(s32) = G_ATOMIC_CMPXCHG [[COPY]](p3), [[COPY1]], [[COPY2]] :: (load store seq_cst 4, addrspace 3)
+    %0:_(p3) = COPY $sgpr0
+    %1:_(s32) = COPY $sgpr1
+    %2:_(s32) = COPY $sgpr2
+    %3:_(s32) = G_ATOMIC_CMPXCHG %0, %1, %2 :: (load store seq_cst 4, addrspace 3)
+...
+
+---
+name: atomic_cmpxchg_global_i64
+
+body: |
+  bb.0:
+    liveins: $sgpr0_sgpr1, $sgpr2, $sgpr3
+    ; CHECK-LABEL: name: atomic_cmpxchg_global_i64
+    ; CHECK: [[COPY:%[0-9]+]]:_(p1) = COPY $sgpr0_sgpr1
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr2
+    ; CHECK: [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr3
+    ; CHECK: [[ATOMIC_CMPXCHG:%[0-9]+]]:_(s32) = G_ATOMIC_CMPXCHG [[COPY]](p1), [[COPY1]], [[COPY2]] :: (load store seq_cst 4, addrspace 1)
+    %0:_(p1) = COPY $sgpr0_sgpr1
+    %1:_(s32) = COPY $sgpr2
+    %2:_(s32) = COPY $sgpr3
+    %3:_(s32) = G_ATOMIC_CMPXCHG %0, %1, %2 :: (load store seq_cst 4, addrspace 1)
+...
+
+---
+name: atomic_cmpxchg_local_i64
+
+body: |
+  bb.0:
+    liveins: $sgpr0, $sgpr1, $sgpr2
+    ; CHECK-LABEL: name: atomic_cmpxchg_local_i64
+    ; CHECK: [[COPY:%[0-9]+]]:_(p3) = COPY $sgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr1
+    ; CHECK: [[COPY2:%[0-9]+]]:_(s32) = COPY $sgpr2
+    ; CHECK: [[ATOMIC_CMPXCHG:%[0-9]+]]:_(s32) = G_ATOMIC_CMPXCHG [[COPY]](p3), [[COPY1]], [[COPY2]] :: (load store seq_cst 4, addrspace 3)
+    %0:_(p3) = COPY $sgpr0
+    %1:_(s32) = COPY $sgpr1
+    %2:_(s32) = COPY $sgpr2
+    %3:_(s32) = G_ATOMIC_CMPXCHG %0, %1, %2 :: (load store seq_cst 4, addrspace 3)
+...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/legalize-atomicrmw-add.mir b/test/CodeGen/AMDGPU/GlobalISel/legalize-atomicrmw-add.mir
new file mode 100644
index 0000000000000000000000000000000000000000..a28718de322f7ef1d8366ec3a2b299e67e2f79df
--- /dev/null
+++ b/test/CodeGen/AMDGPU/GlobalISel/legalize-atomicrmw-add.mir
@@ -0,0 +1,63 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=tahiti -O0 -run-pass=legalizer %s -o - | FileCheck %s
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -O0 -run-pass=legalizer %s -o - | FileCheck %s
+
+---
+name: atomicrmw_add_global_i32
+
+body: |
+  bb.0:
+    liveins: $sgpr0_sgpr1, $sgpr2
+    ; CHECK-LABEL: name: atomicrmw_add_global_i32
+    ; CHECK: [[COPY:%[0-9]+]]:_(p1) = COPY $sgpr0_sgpr1
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr2
+    ; CHECK: [[ATOMICRMW_ADD:%[0-9]+]]:_(s32) = G_ATOMICRMW_ADD [[COPY]](p1), [[COPY1]] :: (load store seq_cst 4, addrspace 1)
+    %0:_(p1) = COPY $sgpr0_sgpr1
+    %1:_(s32) = COPY $sgpr2
+    %2:_(s32) = G_ATOMICRMW_ADD %0, %1 :: (load store seq_cst 4, addrspace 1)
+...
+
+---
+name: atomicrmw_add_local_i32
+
+body: |
+  bb.0:
+    liveins: $sgpr0, $sgpr1
+    ; CHECK-LABEL: name: atomicrmw_add_local_i32
+    ; CHECK: [[COPY:%[0-9]+]]:_(p3) = COPY $sgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr1
+    ; CHECK: [[ATOMICRMW_ADD:%[0-9]+]]:_(s32) = G_ATOMICRMW_ADD [[COPY]](p3), [[COPY1]] :: (load store seq_cst 4, addrspace 3)
+    %0:_(p3) = COPY $sgpr0
+    %1:_(s32) = COPY $sgpr1
+    %2:_(s32) = G_ATOMICRMW_ADD %0, %1 :: (load store seq_cst 4, addrspace 3)
+...
+
+---
+name: atomicrmw_add_global_i64
+
+body: |
+  bb.0:
+    liveins: $sgpr0_sgpr1, $sgpr2
+    ; CHECK-LABEL: name: atomicrmw_add_global_i64
+    ; CHECK: [[COPY:%[0-9]+]]:_(p1) = COPY $sgpr0_sgpr1
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr2
+    ; CHECK: [[ATOMICRMW_ADD:%[0-9]+]]:_(s32) = G_ATOMICRMW_ADD [[COPY]](p1), [[COPY1]] :: (load store seq_cst 4, addrspace 1)
+    %0:_(p1) = COPY $sgpr0_sgpr1
+    %1:_(s32) = COPY $sgpr2
+    %2:_(s32) = G_ATOMICRMW_ADD %0, %1 :: (load store seq_cst 4, addrspace 1)
+...
+
+---
+name: atomicrmw_add_local_i64
+
+body: |
+  bb.0:
+    liveins: $sgpr0, $sgpr1
+    ; CHECK-LABEL: name: atomicrmw_add_local_i64
+    ; CHECK: [[COPY:%[0-9]+]]:_(p3) = COPY $sgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr1
+    ; CHECK: [[ATOMICRMW_ADD:%[0-9]+]]:_(s32) = G_ATOMICRMW_ADD [[COPY]](p3), [[COPY1]] :: (load store seq_cst 4, addrspace 3)
+    %0:_(p3) = COPY $sgpr0
+    %1:_(s32) = COPY $sgpr1
+    %2:_(s32) = G_ATOMICRMW_ADD %0, %1 :: (load store seq_cst 4, addrspace 3)
+...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/legalize-atomicrmw-and.mir b/test/CodeGen/AMDGPU/GlobalISel/legalize-atomicrmw-and.mir
new file mode 100644
index 0000000000000000000000000000000000000000..5ea7c58dbae9c0ff5df18a66cb668516d7bcaecf
--- /dev/null
+++ b/test/CodeGen/AMDGPU/GlobalISel/legalize-atomicrmw-and.mir
@@ -0,0 +1,63 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=tahiti -O0 -run-pass=legalizer %s -o - | FileCheck %s
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -O0 -run-pass=legalizer %s -o - | FileCheck %s
+
+---
+name: atomicrmw_and_global_i32
+
+body: |
+  bb.0:
+    liveins: $sgpr0_sgpr1, $sgpr2
+    ; CHECK-LABEL: name: atomicrmw_and_global_i32
+    ; CHECK: [[COPY:%[0-9]+]]:_(p1) = COPY $sgpr0_sgpr1
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr2
+    ; CHECK: [[ATOMICRMW_AND:%[0-9]+]]:_(s32) = G_ATOMICRMW_AND [[COPY]](p1), [[COPY1]] :: (load store seq_cst 4, addrspace 1)
+    %0:_(p1) = COPY $sgpr0_sgpr1
+    %1:_(s32) = COPY $sgpr2
+    %2:_(s32) = G_ATOMICRMW_AND %0, %1 :: (load store seq_cst 4, addrspace 1)
+...
+
+---
+name: atomicrmw_and_local_i32
+
+body: |
+  bb.0:
+    liveins: $sgpr0, $sgpr1
+    ; CHECK-LABEL: name: atomicrmw_and_local_i32
+    ; CHECK: [[COPY:%[0-9]+]]:_(p3) = COPY $sgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr1
+    ; CHECK: [[ATOMICRMW_AND:%[0-9]+]]:_(s32) = G_ATOMICRMW_AND [[COPY]](p3), [[COPY1]] :: (load store seq_cst 4, addrspace 3)
+    %0:_(p3) = COPY $sgpr0
+    %1:_(s32) = COPY $sgpr1
+    %2:_(s32) = G_ATOMICRMW_AND %0, %1 :: (load store seq_cst 4, addrspace 3)
+...
+
+---
+name: atomicrmw_and_global_i64
+
+body: |
+  bb.0:
+    liveins: $sgpr0_sgpr1, $sgpr2
+    ; CHECK-LABEL: name: atomicrmw_and_global_i64
+    ; CHECK: [[COPY:%[0-9]+]]:_(p1) = COPY $sgpr0_sgpr1
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr2
+    ; CHECK: [[ATOMICRMW_AND:%[0-9]+]]:_(s32) = G_ATOMICRMW_AND [[COPY]](p1), [[COPY1]] :: (load store seq_cst 4, addrspace 1)
+    %0:_(p1) = COPY $sgpr0_sgpr1
+    %1:_(s32) = COPY $sgpr2
+    %2:_(s32) = G_ATOMICRMW_AND %0, %1 :: (load store seq_cst 4, addrspace 1)
+...
+
+---
+name: atomicrmw_and_local_i64
+
+body: |
+  bb.0:
+    liveins: $sgpr0, $sgpr1
+    ; CHECK-LABEL: name: atomicrmw_and_local_i64
+    ; CHECK: [[COPY:%[0-9]+]]:_(p3) = COPY $sgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr1
+    ; CHECK: [[ATOMICRMW_AND:%[0-9]+]]:_(s32) = G_ATOMICRMW_AND [[COPY]](p3), [[COPY1]] :: (load store seq_cst 4, addrspace 3)
+    %0:_(p3) = COPY $sgpr0
+    %1:_(s32) = COPY $sgpr1
+    %2:_(s32) = G_ATOMICRMW_AND %0, %1 :: (load store seq_cst 4, addrspace 3)
+...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/legalize-atomicrmw-max.mir b/test/CodeGen/AMDGPU/GlobalISel/legalize-atomicrmw-max.mir
new file mode 100644
index 0000000000000000000000000000000000000000..c1b4648ae0ccc57acedb5240fd09e4a9c9bcb712
--- /dev/null
+++ b/test/CodeGen/AMDGPU/GlobalISel/legalize-atomicrmw-max.mir
@@ -0,0 +1,63 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=tahiti -O0 -run-pass=legalizer %s -o - | FileCheck %s
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -O0 -run-pass=legalizer %s -o - | FileCheck %s
+
+---
+name: atomicrmw_max_global_i32
+
+body: |
+  bb.0:
+    liveins: $sgpr0_sgpr1, $sgpr2
+    ; CHECK-LABEL: name: atomicrmw_max_global_i32
+    ; CHECK: [[COPY:%[0-9]+]]:_(p1) = COPY $sgpr0_sgpr1
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr2
+    ; CHECK: [[ATOMICRMW_MAX:%[0-9]+]]:_(s32) = G_ATOMICRMW_MAX [[COPY]](p1), [[COPY1]] :: (load store seq_cst 4, addrspace 1)
+    %0:_(p1) = COPY $sgpr0_sgpr1
+    %1:_(s32) = COPY $sgpr2
+    %2:_(s32) = G_ATOMICRMW_MAX %0, %1 :: (load store seq_cst 4, addrspace 1)
+...
+
+---
+name: atomicrmw_max_local_i32
+
+body: |
+  bb.0:
+    liveins: $sgpr0, $sgpr1
+    ; CHECK-LABEL: name: atomicrmw_max_local_i32
+    ; CHECK: [[COPY:%[0-9]+]]:_(p3) = COPY $sgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr1
+    ; CHECK: [[ATOMICRMW_MAX:%[0-9]+]]:_(s32) = G_ATOMICRMW_MAX [[COPY]](p3), [[COPY1]] :: (load store seq_cst 4, addrspace 3)
+    %0:_(p3) = COPY $sgpr0
+    %1:_(s32) = COPY $sgpr1
+    %2:_(s32) = G_ATOMICRMW_MAX %0, %1 :: (load store seq_cst 4, addrspace 3)
+...
+
+---
+name: atomicrmw_max_global_i64
+
+body: |
+  bb.0:
+    liveins: $sgpr0_sgpr1, $sgpr2
+    ; CHECK-LABEL: name: atomicrmw_max_global_i64
+    ; CHECK: [[COPY:%[0-9]+]]:_(p1) = COPY $sgpr0_sgpr1
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr2
+    ; CHECK: [[ATOMICRMW_MAX:%[0-9]+]]:_(s32) = G_ATOMICRMW_MAX [[COPY]](p1), [[COPY1]] :: (load store seq_cst 4, addrspace 1)
+    %0:_(p1) = COPY $sgpr0_sgpr1
+    %1:_(s32) = COPY $sgpr2
+    %2:_(s32) = G_ATOMICRMW_MAX %0, %1 :: (load store seq_cst 4, addrspace 1)
+...
+
+---
+name: atomicrmw_max_local_i64
+
+body: |
+  bb.0:
+    liveins: $sgpr0, $sgpr1
+    ; CHECK-LABEL: name: atomicrmw_max_local_i64
+    ; CHECK: [[COPY:%[0-9]+]]:_(p3) = COPY $sgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr1
+    ; CHECK: [[ATOMICRMW_MAX:%[0-9]+]]:_(s32) = G_ATOMICRMW_MAX [[COPY]](p3), [[COPY1]] :: (load store seq_cst 4, addrspace 3)
+    %0:_(p3) = COPY $sgpr0
+    %1:_(s32) = COPY $sgpr1
+    %2:_(s32) = G_ATOMICRMW_MAX %0, %1 :: (load store seq_cst 4, addrspace 3)
+...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/legalize-atomicrmw-min.mir b/test/CodeGen/AMDGPU/GlobalISel/legalize-atomicrmw-min.mir
new file mode 100644
index 0000000000000000000000000000000000000000..08ec0a0aa14db134347fb3ad855ae34ac7caca23
--- /dev/null
+++ b/test/CodeGen/AMDGPU/GlobalISel/legalize-atomicrmw-min.mir
@@ -0,0 +1,63 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=tahiti -O0 -run-pass=legalizer %s -o - | FileCheck %s
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -O0 -run-pass=legalizer %s -o - | FileCheck %s
+
+---
+name: atomicrmw_min_global_i32
+
+body: |
+  bb.0:
+    liveins: $sgpr0_sgpr1, $sgpr2
+    ; CHECK-LABEL: name: atomicrmw_min_global_i32
+    ; CHECK: [[COPY:%[0-9]+]]:_(p1) = COPY $sgpr0_sgpr1
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr2
+    ; CHECK: [[ATOMICRMW_MIN:%[0-9]+]]:_(s32) = G_ATOMICRMW_MIN [[COPY]](p1), [[COPY1]] :: (load store seq_cst 4, addrspace 1)
+    %0:_(p1) = COPY $sgpr0_sgpr1
+    %1:_(s32) = COPY $sgpr2
+    %2:_(s32) = G_ATOMICRMW_MIN %0, %1 :: (load store seq_cst 4, addrspace 1)
+...
+
+---
+name: atomicrmw_min_local_i32
+
+body: |
+  bb.0:
+    liveins: $sgpr0, $sgpr1
+    ; CHECK-LABEL: name: atomicrmw_min_local_i32
+    ; CHECK: [[COPY:%[0-9]+]]:_(p3) = COPY $sgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr1
+    ; CHECK: [[ATOMICRMW_MIN:%[0-9]+]]:_(s32) = G_ATOMICRMW_MIN [[COPY]](p3), [[COPY1]] :: (load store seq_cst 4, addrspace 3)
+    %0:_(p3) = COPY $sgpr0
+    %1:_(s32) = COPY $sgpr1
+    %2:_(s32) = G_ATOMICRMW_MIN %0, %1 :: (load store seq_cst 4, addrspace 3)
+...
+
+---
+name: atomicrmw_min_global_i64
+
+body: |
+  bb.0:
+    liveins: $sgpr0_sgpr1, $sgpr2
+    ; CHECK-LABEL: name: atomicrmw_min_global_i64
+    ; CHECK: [[COPY:%[0-9]+]]:_(p1) = COPY $sgpr0_sgpr1
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr2
+    ; CHECK: [[ATOMICRMW_MIN:%[0-9]+]]:_(s32) = G_ATOMICRMW_MIN [[COPY]](p1), [[COPY1]] :: (load store seq_cst 4, addrspace 1)
+    %0:_(p1) = COPY $sgpr0_sgpr1
+    %1:_(s32) = COPY $sgpr2
+    %2:_(s32) = G_ATOMICRMW_MIN %0, %1 :: (load store seq_cst 4, addrspace 1)
+...
+
+---
+name: atomicrmw_min_local_i64
+
+body: |
+  bb.0:
+    liveins: $sgpr0, $sgpr1
+    ; CHECK-LABEL: name: atomicrmw_min_local_i64
+    ; CHECK: [[COPY:%[0-9]+]]:_(p3) = COPY $sgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr1
+    ; CHECK: [[ATOMICRMW_MIN:%[0-9]+]]:_(s32) = G_ATOMICRMW_MIN [[COPY]](p3), [[COPY1]] :: (load store seq_cst 4, addrspace 3)
+    %0:_(p3) = COPY $sgpr0
+    %1:_(s32) = COPY $sgpr1
+    %2:_(s32) = G_ATOMICRMW_MIN %0, %1 :: (load store seq_cst 4, addrspace 3)
+...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/legalize-atomicrmw-nand.mir b/test/CodeGen/AMDGPU/GlobalISel/legalize-atomicrmw-nand.mir
new file mode 100644
index 0000000000000000000000000000000000000000..ccce93db415f376df395393b730f1bb6d7f0251e
--- /dev/null
+++ b/test/CodeGen/AMDGPU/GlobalISel/legalize-atomicrmw-nand.mir
@@ -0,0 +1,22 @@
+# RUN: not llc -mtriple=amdgcn-mesa-mesa3d -mcpu=tahiti -O0 -run-pass=legalizer -o - %s 2>&1| FileCheck -check-prefix=ERROR %s
+
+# This needs to be expanded into a cmpxchg loop.
+# TODO: Will AtomicExpand still do this?
+
+# ERROR: LLVM ERROR: unable to legalize instruction: %2:_(s32) = G_ATOMICRMW_NAND %0:_(p1), %1:_ :: (load store seq_cst 4, addrspace 1) (in function: atomicrmw_nand_global_i32)
+
+---
+name: atomicrmw_nand_global_i32
+
+body: |
+  bb.0:
+    liveins: $sgpr0_sgpr1, $sgpr2
+    ; CHECK-LABEL: name: atomicrmw_nand_global_i32
+    ; CHECK: [[COPY:%[0-9]+]]:_(p1) = COPY $sgpr0_sgpr1
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr2
+    ; CHECK: [[ATOMICRMW_NAND:%[0-9]+]]:_(s32) = G_ATOMICRMW_NAND [[COPY]](p1), [[COPY1]] :: (load store seq_cst 4, addrspace 1)
+    %0:_(p1) = COPY $sgpr0_sgpr1
+    %1:_(s32) = COPY $sgpr2
+    %2:_(s32) = G_ATOMICRMW_NAND %0, %1 :: (load store seq_cst 4, addrspace 1)
+...
+
diff --git a/test/CodeGen/AMDGPU/GlobalISel/legalize-atomicrmw-or.mir b/test/CodeGen/AMDGPU/GlobalISel/legalize-atomicrmw-or.mir
new file mode 100644
index 0000000000000000000000000000000000000000..e214ee6c8d2e761deb40c09cbc63241a490087e1
--- /dev/null
+++ b/test/CodeGen/AMDGPU/GlobalISel/legalize-atomicrmw-or.mir
@@ -0,0 +1,63 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=tahiti -O0 -run-pass=legalizer %s -o - | FileCheck %s
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -O0 -run-pass=legalizer %s -o - | FileCheck %s
+
+---
+name: atomicrmw_or_global_i32
+
+body: |
+  bb.0:
+    liveins: $sgpr0_sgpr1, $sgpr2
+    ; CHECK-LABEL: name: atomicrmw_or_global_i32
+    ; CHECK: [[COPY:%[0-9]+]]:_(p1) = COPY $sgpr0_sgpr1
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr2
+    ; CHECK: [[ATOMICRMW_OR:%[0-9]+]]:_(s32) = G_ATOMICRMW_OR [[COPY]](p1), [[COPY1]] :: (load store seq_cst 4, addrspace 1)
+    %0:_(p1) = COPY $sgpr0_sgpr1
+    %1:_(s32) = COPY $sgpr2
+    %2:_(s32) = G_ATOMICRMW_OR %0, %1 :: (load store seq_cst 4, addrspace 1)
+...
+
+---
+name: atomicrmw_or_local_i32
+
+body: |
+  bb.0:
+    liveins: $sgpr0, $sgpr1
+    ; CHECK-LABEL: name: atomicrmw_or_local_i32
+    ; CHECK: [[COPY:%[0-9]+]]:_(p3) = COPY $sgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr1
+    ; CHECK: [[ATOMICRMW_OR:%[0-9]+]]:_(s32) = G_ATOMICRMW_OR [[COPY]](p3), [[COPY1]] :: (load store seq_cst 4, addrspace 3)
+    %0:_(p3) = COPY $sgpr0
+    %1:_(s32) = COPY $sgpr1
+    %2:_(s32) = G_ATOMICRMW_OR %0, %1 :: (load store seq_cst 4, addrspace 3)
+...
+
+---
+name: atomicrmw_or_global_i64
+
+body: |
+  bb.0:
+    liveins: $sgpr0_sgpr1, $sgpr2
+    ; CHECK-LABEL: name: atomicrmw_or_global_i64
+    ; CHECK: [[COPY:%[0-9]+]]:_(p1) = COPY $sgpr0_sgpr1
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr2
+    ; CHECK: [[ATOMICRMW_OR:%[0-9]+]]:_(s32) = G_ATOMICRMW_OR [[COPY]](p1), [[COPY1]] :: (load store seq_cst 4, addrspace 1)
+    %0:_(p1) = COPY $sgpr0_sgpr1
+    %1:_(s32) = COPY $sgpr2
+    %2:_(s32) = G_ATOMICRMW_OR %0, %1 :: (load store seq_cst 4, addrspace 1)
+...
+
+---
+name: atomicrmw_or_local_i64
+
+body: |
+  bb.0:
+    liveins: $sgpr0, $sgpr1
+    ; CHECK-LABEL: name: atomicrmw_or_local_i64
+    ; CHECK: [[COPY:%[0-9]+]]:_(p3) = COPY $sgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr1
+    ; CHECK: [[ATOMICRMW_OR:%[0-9]+]]:_(s32) = G_ATOMICRMW_OR [[COPY]](p3), [[COPY1]] :: (load store seq_cst 4, addrspace 3)
+    %0:_(p3) = COPY $sgpr0
+    %1:_(s32) = COPY $sgpr1
+    %2:_(s32) = G_ATOMICRMW_OR %0, %1 :: (load store seq_cst 4, addrspace 3)
+...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/legalize-atomicrmw-sub.mir b/test/CodeGen/AMDGPU/GlobalISel/legalize-atomicrmw-sub.mir
new file mode 100644
index 0000000000000000000000000000000000000000..974cd3f12e3b68def0a9c7d547ffdf5f931dd532
--- /dev/null
+++ b/test/CodeGen/AMDGPU/GlobalISel/legalize-atomicrmw-sub.mir
@@ -0,0 +1,63 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=tahiti -O0 -run-pass=legalizer %s -o - | FileCheck %s
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -O0 -run-pass=legalizer %s -o - | FileCheck %s
+
+---
+name: atomicrmw_sub_global_i32
+
+body: |
+  bb.0:
+    liveins: $sgpr0_sgpr1, $sgpr2
+    ; CHECK-LABEL: name: atomicrmw_sub_global_i32
+    ; CHECK: [[COPY:%[0-9]+]]:_(p1) = COPY $sgpr0_sgpr1
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr2
+    ; CHECK: [[ATOMICRMW_SUB:%[0-9]+]]:_(s32) = G_ATOMICRMW_SUB [[COPY]](p1), [[COPY1]] :: (load store seq_cst 4, addrspace 1)
+    %0:_(p1) = COPY $sgpr0_sgpr1
+    %1:_(s32) = COPY $sgpr2
+    %2:_(s32) = G_ATOMICRMW_SUB %0, %1 :: (load store seq_cst 4, addrspace 1)
+...
+
+---
+name: atomicrmw_sub_local_i32
+
+body: |
+  bb.0:
+    liveins: $sgpr0, $sgpr1
+    ; CHECK-LABEL: name: atomicrmw_sub_local_i32
+    ; CHECK: [[COPY:%[0-9]+]]:_(p3) = COPY $sgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr1
+    ; CHECK: [[ATOMICRMW_SUB:%[0-9]+]]:_(s32) = G_ATOMICRMW_SUB [[COPY]](p3), [[COPY1]] :: (load store seq_cst 4, addrspace 3)
+    %0:_(p3) = COPY $sgpr0
+    %1:_(s32) = COPY $sgpr1
+    %2:_(s32) = G_ATOMICRMW_SUB %0, %1 :: (load store seq_cst 4, addrspace 3)
+...
+
+---
+name: atomicrmw_sub_global_i64
+
+body: |
+  bb.0:
+    liveins: $sgpr0_sgpr1, $sgpr2
+    ; CHECK-LABEL: name: atomicrmw_sub_global_i64
+    ; CHECK: [[COPY:%[0-9]+]]:_(p1) = COPY $sgpr0_sgpr1
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr2
+    ; CHECK: [[ATOMICRMW_SUB:%[0-9]+]]:_(s32) = G_ATOMICRMW_SUB [[COPY]](p1), [[COPY1]] :: (load store seq_cst 4, addrspace 1)
+    %0:_(p1) = COPY $sgpr0_sgpr1
+    %1:_(s32) = COPY $sgpr2
+    %2:_(s32) = G_ATOMICRMW_SUB %0, %1 :: (load store seq_cst 4, addrspace 1)
+...
+
+---
+name: atomicrmw_sub_local_i64
+
+body: |
+  bb.0:
+    liveins: $sgpr0, $sgpr1
+    ; CHECK-LABEL: name: atomicrmw_sub_local_i64
+    ; CHECK: [[COPY:%[0-9]+]]:_(p3) = COPY $sgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr1
+    ; CHECK: [[ATOMICRMW_SUB:%[0-9]+]]:_(s32) = G_ATOMICRMW_SUB [[COPY]](p3), [[COPY1]] :: (load store seq_cst 4, addrspace 3)
+    %0:_(p3) = COPY $sgpr0
+    %1:_(s32) = COPY $sgpr1
+    %2:_(s32) = G_ATOMICRMW_SUB %0, %1 :: (load store seq_cst 4, addrspace 3)
+...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/legalize-atomicrmw-umax.mir b/test/CodeGen/AMDGPU/GlobalISel/legalize-atomicrmw-umax.mir
new file mode 100644
index 0000000000000000000000000000000000000000..0fc62a4ec895dccf14b346c330468e1ea52c2889
--- /dev/null
+++ b/test/CodeGen/AMDGPU/GlobalISel/legalize-atomicrmw-umax.mir
@@ -0,0 +1,63 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=tahiti -O0 -run-pass=legalizer %s -o - | FileCheck %s
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -O0 -run-pass=legalizer %s -o - | FileCheck %s
+
+---
+name: atomicrmw_umax_global_i32
+
+body: |
+  bb.0:
+    liveins: $sgpr0_sgpr1, $sgpr2
+    ; CHECK-LABEL: name: atomicrmw_umax_global_i32
+    ; CHECK: [[COPY:%[0-9]+]]:_(p1) = COPY $sgpr0_sgpr1
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr2
+    ; CHECK: [[ATOMICRMW_UMAX:%[0-9]+]]:_(s32) = G_ATOMICRMW_UMAX [[COPY]](p1), [[COPY1]] :: (load store seq_cst 4, addrspace 1)
+    %0:_(p1) = COPY $sgpr0_sgpr1
+    %1:_(s32) = COPY $sgpr2
+    %2:_(s32) = G_ATOMICRMW_UMAX %0, %1 :: (load store seq_cst 4, addrspace 1)
+...
+
+---
+name: atomicrmw_umax_local_i32
+
+body: |
+  bb.0:
+    liveins: $sgpr0, $sgpr1
+    ; CHECK-LABEL: name: atomicrmw_umax_local_i32
+    ; CHECK: [[COPY:%[0-9]+]]:_(p3) = COPY $sgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr1
+    ; CHECK: [[ATOMICRMW_UMAX:%[0-9]+]]:_(s32) = G_ATOMICRMW_UMAX [[COPY]](p3), [[COPY1]] :: (load store seq_cst 4, addrspace 3)
+    %0:_(p3) = COPY $sgpr0
+    %1:_(s32) = COPY $sgpr1
+    %2:_(s32) = G_ATOMICRMW_UMAX %0, %1 :: (load store seq_cst 4, addrspace 3)
+...
+
+---
+name: atomicrmw_umax_global_i64
+
+body: |
+  bb.0:
+    liveins: $sgpr0_sgpr1, $sgpr2
+    ; CHECK-LABEL: name: atomicrmw_umax_global_i64
+    ; CHECK: [[COPY:%[0-9]+]]:_(p1) = COPY $sgpr0_sgpr1
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr2
+    ; CHECK: [[ATOMICRMW_UMAX:%[0-9]+]]:_(s32) = G_ATOMICRMW_UMAX [[COPY]](p1), [[COPY1]] :: (load store seq_cst 4, addrspace 1)
+    %0:_(p1) = COPY $sgpr0_sgpr1
+    %1:_(s32) = COPY $sgpr2
+    %2:_(s32) = G_ATOMICRMW_UMAX %0, %1 :: (load store seq_cst 4, addrspace 1)
+...
+
+---
+name: atomicrmw_umax_local_i64
+
+body: |
+  bb.0:
+    liveins: $sgpr0, $sgpr1
+    ; CHECK-LABEL: name: atomicrmw_umax_local_i64
+    ; CHECK: [[COPY:%[0-9]+]]:_(p3) = COPY $sgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr1
+    ; CHECK: [[ATOMICRMW_UMAX:%[0-9]+]]:_(s32) = G_ATOMICRMW_UMAX [[COPY]](p3), [[COPY1]] :: (load store seq_cst 4, addrspace 3)
+    %0:_(p3) = COPY $sgpr0
+    %1:_(s32) = COPY $sgpr1
+    %2:_(s32) = G_ATOMICRMW_UMAX %0, %1 :: (load store seq_cst 4, addrspace 3)
+...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/legalize-atomicrmw-umin.mir b/test/CodeGen/AMDGPU/GlobalISel/legalize-atomicrmw-umin.mir
new file mode 100644
index 0000000000000000000000000000000000000000..a72b6d7f152b1d252eb02fb7bb21ecac675a4d61
--- /dev/null
+++ b/test/CodeGen/AMDGPU/GlobalISel/legalize-atomicrmw-umin.mir
@@ -0,0 +1,63 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=tahiti -O0 -run-pass=legalizer %s -o - | FileCheck %s
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -O0 -run-pass=legalizer %s -o - | FileCheck %s
+
+---
+name: atomicrmw_umin_global_i32
+
+body: |
+  bb.0:
+    liveins: $sgpr0_sgpr1, $sgpr2
+    ; CHECK-LABEL: name: atomicrmw_umin_global_i32
+    ; CHECK: [[COPY:%[0-9]+]]:_(p1) = COPY $sgpr0_sgpr1
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr2
+    ; CHECK: [[ATOMICRMW_UMIN:%[0-9]+]]:_(s32) = G_ATOMICRMW_UMIN [[COPY]](p1), [[COPY1]] :: (load store seq_cst 4, addrspace 1)
+    %0:_(p1) = COPY $sgpr0_sgpr1
+    %1:_(s32) = COPY $sgpr2
+    %2:_(s32) = G_ATOMICRMW_UMIN %0, %1 :: (load store seq_cst 4, addrspace 1)
+...
+
+---
+name: atomicrmw_umin_local_i32
+
+body: |
+  bb.0:
+    liveins: $sgpr0, $sgpr1
+    ; CHECK-LABEL: name: atomicrmw_umin_local_i32
+    ; CHECK: [[COPY:%[0-9]+]]:_(p3) = COPY $sgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr1
+    ; CHECK: [[ATOMICRMW_UMIN:%[0-9]+]]:_(s32) = G_ATOMICRMW_UMIN [[COPY]](p3), [[COPY1]] :: (load store seq_cst 4, addrspace 3)
+    %0:_(p3) = COPY $sgpr0
+    %1:_(s32) = COPY $sgpr1
+    %2:_(s32) = G_ATOMICRMW_UMIN %0, %1 :: (load store seq_cst 4, addrspace 3)
+...
+
+---
+name: atomicrmw_umin_global_i64
+
+body: |
+  bb.0:
+    liveins: $sgpr0_sgpr1, $sgpr2
+    ; CHECK-LABEL: name: atomicrmw_umin_global_i64
+    ; CHECK: [[COPY:%[0-9]+]]:_(p1) = COPY $sgpr0_sgpr1
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr2
+    ; CHECK: [[ATOMICRMW_UMIN:%[0-9]+]]:_(s32) = G_ATOMICRMW_UMIN [[COPY]](p1), [[COPY1]] :: (load store seq_cst 4, addrspace 1)
+    %0:_(p1) = COPY $sgpr0_sgpr1
+    %1:_(s32) = COPY $sgpr2
+    %2:_(s32) = G_ATOMICRMW_UMIN %0, %1 :: (load store seq_cst 4, addrspace 1)
+...
+
+---
+name: atomicrmw_umin_local_i64
+
+body: |
+  bb.0:
+    liveins: $sgpr0, $sgpr1
+    ; CHECK-LABEL: name: atomicrmw_umin_local_i64
+    ; CHECK: [[COPY:%[0-9]+]]:_(p3) = COPY $sgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr1
+    ; CHECK: [[ATOMICRMW_UMIN:%[0-9]+]]:_(s32) = G_ATOMICRMW_UMIN [[COPY]](p3), [[COPY1]] :: (load store seq_cst 4, addrspace 3)
+    %0:_(p3) = COPY $sgpr0
+    %1:_(s32) = COPY $sgpr1
+    %2:_(s32) = G_ATOMICRMW_UMIN %0, %1 :: (load store seq_cst 4, addrspace 3)
+...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/legalize-atomicrmw-xchg-flat.mir b/test/CodeGen/AMDGPU/GlobalISel/legalize-atomicrmw-xchg-flat.mir
new file mode 100644
index 0000000000000000000000000000000000000000..96b33c1b768a462889d09b9603b2ee50ed24ea18
--- /dev/null
+++ b/test/CodeGen/AMDGPU/GlobalISel/legalize-atomicrmw-xchg-flat.mir
@@ -0,0 +1,36 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=bonaire -O0 -run-pass=legalizer -o - %s | FileCheck %s
+# RUN: not llc -mtriple=amdgcn-mesa-mesa3d -mcpu=tahiti -O0 -run-pass=legalizer -o /dev/null %s 2>&1 | FileCheck -check-prefix=ERROR %s
+
+# ERROR: LLVM ERROR: unable to legalize instruction: %2:_(s32) = G_ATOMICRMW_XCHG %0:_(p0), %1:_ :: (load store seq_cst 4) (in function: atomicrmw_xchg_flat_i32)
+
+
+---
+name: atomicrmw_xchg_flat_i32
+
+body: |
+  bb.0:
+    liveins: $sgpr0_sgpr1, $sgpr2
+    ; CHECK-LABEL: name: atomicrmw_xchg_flat_i32
+    ; CHECK: [[COPY:%[0-9]+]]:_(p0) = COPY $sgpr0_sgpr1
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr2
+    ; CHECK: [[ATOMICRMW_XCHG:%[0-9]+]]:_(s32) = G_ATOMICRMW_XCHG [[COPY]](p0), [[COPY1]] :: (load store seq_cst 4)
+    %0:_(p0) = COPY $sgpr0_sgpr1
+    %1:_(s32) = COPY $sgpr2
+    %2:_(s32) = G_ATOMICRMW_XCHG %0, %1 :: (load store seq_cst 4, addrspace 0)
+...
+
+---
+name: atomicrmw_xchg_flat_i64
+
+body: |
+  bb.0:
+    liveins: $sgpr0_sgpr1, $sgpr2
+    ; CHECK-LABEL: name: atomicrmw_xchg_flat_i64
+    ; CHECK: [[COPY:%[0-9]+]]:_(p0) = COPY $sgpr0_sgpr1
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr2
+    ; CHECK: [[ATOMICRMW_XCHG:%[0-9]+]]:_(s32) = G_ATOMICRMW_XCHG [[COPY]](p0), [[COPY1]] :: (load store seq_cst 4)
+    %0:_(p0) = COPY $sgpr0_sgpr1
+    %1:_(s32) = COPY $sgpr2
+    %2:_(s32) = G_ATOMICRMW_XCHG %0, %1 :: (load store seq_cst 4, addrspace 0)
+...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/legalize-atomicrmw-xchg.mir b/test/CodeGen/AMDGPU/GlobalISel/legalize-atomicrmw-xchg.mir
new file mode 100644
index 0000000000000000000000000000000000000000..424ac334cc682afc25c33dc81f6a951943e6de6c
--- /dev/null
+++ b/test/CodeGen/AMDGPU/GlobalISel/legalize-atomicrmw-xchg.mir
@@ -0,0 +1,63 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=tahiti -O0 -run-pass=legalizer %s -o - | FileCheck %s
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -O0 -run-pass=legalizer %s -o - | FileCheck %s
+
+---
+name: atomicrmw_xchg_global_i32
+
+body: |
+  bb.0:
+    liveins: $sgpr0_sgpr1, $sgpr2
+    ; CHECK-LABEL: name: atomicrmw_xchg_global_i32
+    ; CHECK: [[COPY:%[0-9]+]]:_(p1) = COPY $sgpr0_sgpr1
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr2
+    ; CHECK: [[ATOMICRMW_XCHG:%[0-9]+]]:_(s32) = G_ATOMICRMW_XCHG [[COPY]](p1), [[COPY1]] :: (load store seq_cst 4, addrspace 1)
+    %0:_(p1) = COPY $sgpr0_sgpr1
+    %1:_(s32) = COPY $sgpr2
+    %2:_(s32) = G_ATOMICRMW_XCHG %0, %1 :: (load store seq_cst 4, addrspace 1)
+...
+
+---
+name: atomicrmw_xchg_local_i32
+
+body: |
+  bb.0:
+    liveins: $sgpr0, $sgpr1
+    ; CHECK-LABEL: name: atomicrmw_xchg_local_i32
+    ; CHECK: [[COPY:%[0-9]+]]:_(p3) = COPY $sgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr1
+    ; CHECK: [[ATOMICRMW_XCHG:%[0-9]+]]:_(s32) = G_ATOMICRMW_XCHG [[COPY]](p3), [[COPY1]] :: (load store seq_cst 4, addrspace 3)
+    %0:_(p3) = COPY $sgpr0
+    %1:_(s32) = COPY $sgpr1
+    %2:_(s32) = G_ATOMICRMW_XCHG %0, %1 :: (load store seq_cst 4, addrspace 3)
+...
+
+---
+name: atomicrmw_xchg_global_i64
+
+body: |
+  bb.0:
+    liveins: $sgpr0_sgpr1, $sgpr2
+    ; CHECK-LABEL: name: atomicrmw_xchg_global_i64
+    ; CHECK: [[COPY:%[0-9]+]]:_(p1) = COPY $sgpr0_sgpr1
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr2
+    ; CHECK: [[ATOMICRMW_XCHG:%[0-9]+]]:_(s32) = G_ATOMICRMW_XCHG [[COPY]](p1), [[COPY1]] :: (load store seq_cst 4, addrspace 1)
+    %0:_(p1) = COPY $sgpr0_sgpr1
+    %1:_(s32) = COPY $sgpr2
+    %2:_(s32) = G_ATOMICRMW_XCHG %0, %1 :: (load store seq_cst 4, addrspace 1)
+...
+
+---
+name: atomicrmw_xchg_local_i64
+
+body: |
+  bb.0:
+    liveins: $sgpr0, $sgpr1
+    ; CHECK-LABEL: name: atomicrmw_xchg_local_i64
+    ; CHECK: [[COPY:%[0-9]+]]:_(p3) = COPY $sgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr1
+    ; CHECK: [[ATOMICRMW_XCHG:%[0-9]+]]:_(s32) = G_ATOMICRMW_XCHG [[COPY]](p3), [[COPY1]] :: (load store seq_cst 4, addrspace 3)
+    %0:_(p3) = COPY $sgpr0
+    %1:_(s32) = COPY $sgpr1
+    %2:_(s32) = G_ATOMICRMW_XCHG %0, %1 :: (load store seq_cst 4, addrspace 3)
+...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/legalize-atomicrmw-xor.mir b/test/CodeGen/AMDGPU/GlobalISel/legalize-atomicrmw-xor.mir
new file mode 100644
index 0000000000000000000000000000000000000000..ce5f78f6513776fdefd735df804a48963aa9a86c
--- /dev/null
+++ b/test/CodeGen/AMDGPU/GlobalISel/legalize-atomicrmw-xor.mir
@@ -0,0 +1,63 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=tahiti -O0 -run-pass=legalizer %s -o - | FileCheck %s
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -O0 -run-pass=legalizer %s -o - | FileCheck %s
+
+---
+name: atomicrmw_xor_global_i32
+
+body: |
+  bb.0:
+    liveins: $sgpr0_sgpr1, $sgpr2
+    ; CHECK-LABEL: name: atomicrmw_xor_global_i32
+    ; CHECK: [[COPY:%[0-9]+]]:_(p1) = COPY $sgpr0_sgpr1
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr2
+    ; CHECK: [[ATOMICRMW_XOR:%[0-9]+]]:_(s32) = G_ATOMICRMW_XOR [[COPY]](p1), [[COPY1]] :: (load store seq_cst 4, addrspace 1)
+    %0:_(p1) = COPY $sgpr0_sgpr1
+    %1:_(s32) = COPY $sgpr2
+    %2:_(s32) = G_ATOMICRMW_XOR %0, %1 :: (load store seq_cst 4, addrspace 1)
+...
+
+---
+name: atomicrmw_xor_local_i32
+
+body: |
+  bb.0:
+    liveins: $sgpr0, $sgpr1
+    ; CHECK-LABEL: name: atomicrmw_xor_local_i32
+    ; CHECK: [[COPY:%[0-9]+]]:_(p3) = COPY $sgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr1
+    ; CHECK: [[ATOMICRMW_XOR:%[0-9]+]]:_(s32) = G_ATOMICRMW_XOR [[COPY]](p3), [[COPY1]] :: (load store seq_cst 4, addrspace 3)
+    %0:_(p3) = COPY $sgpr0
+    %1:_(s32) = COPY $sgpr1
+    %2:_(s32) = G_ATOMICRMW_XOR %0, %1 :: (load store seq_cst 4, addrspace 3)
+...
+
+---
+name: atomicrmw_xor_global_i64
+
+body: |
+  bb.0:
+    liveins: $sgpr0_sgpr1, $sgpr2
+    ; CHECK-LABEL: name: atomicrmw_xor_global_i64
+    ; CHECK: [[COPY:%[0-9]+]]:_(p1) = COPY $sgpr0_sgpr1
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr2
+    ; CHECK: [[ATOMICRMW_XOR:%[0-9]+]]:_(s32) = G_ATOMICRMW_XOR [[COPY]](p1), [[COPY1]] :: (load store seq_cst 4, addrspace 1)
+    %0:_(p1) = COPY $sgpr0_sgpr1
+    %1:_(s32) = COPY $sgpr2
+    %2:_(s32) = G_ATOMICRMW_XOR %0, %1 :: (load store seq_cst 4, addrspace 1)
+...
+
+---
+name: atomicrmw_xor_local_i64
+
+body: |
+  bb.0:
+    liveins: $sgpr0, $sgpr1
+    ; CHECK-LABEL: name: atomicrmw_xor_local_i64
+    ; CHECK: [[COPY:%[0-9]+]]:_(p3) = COPY $sgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr1
+    ; CHECK: [[ATOMICRMW_XOR:%[0-9]+]]:_(s32) = G_ATOMICRMW_XOR [[COPY]](p3), [[COPY1]] :: (load store seq_cst 4, addrspace 3)
+    %0:_(p3) = COPY $sgpr0
+    %1:_(s32) = COPY $sgpr1
+    %2:_(s32) = G_ATOMICRMW_XOR %0, %1 :: (load store seq_cst 4, addrspace 3)
+...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/legalize-bitcast.mir b/test/CodeGen/AMDGPU/GlobalISel/legalize-bitcast.mir
index 492da48a93f0913aeb20395be257f8b245eaa0ca..2c3995a49d1c60050a163f8fa263cbb474010283 100644
--- a/test/CodeGen/AMDGPU/GlobalISel/legalize-bitcast.mir
+++ b/test/CodeGen/AMDGPU/GlobalISel/legalize-bitcast.mir
@@ -2,17 +2,181 @@
 # RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -run-pass=legalizer %s -o - | FileCheck %s
 
 ---
-name:            test_bitcast
+name: test_bitcast_s32_to_v2s16
 body: |
   bb.0:
     liveins: $vgpr0
 
-    ; CHECK-LABEL: name: test_bitcast
+    ; CHECK-LABEL: name: test_bitcast_s32_to_v2s16
     ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
     ; CHECK: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[COPY]](s32)
-    ; CHECK: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[BITCAST]](<2 x s16>)
+    ; CHECK: $vgpr0 = COPY [[BITCAST]](<2 x s16>)
     %0:_(s32) = COPY $vgpr0
     %1:_(<2 x s16>) = G_BITCAST %0
-    %2:_(s32) = G_BITCAST %1
-    $vgpr0 = COPY %2
+    $vgpr0 = COPY %1
+...
+
+---
+name: test_bitcast_v2s16_to_s32
+body: |
+  bb.0:
+    liveins: $vgpr0
+
+    ; CHECK-LABEL: name: test_bitcast_v2s16_to_s32
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
+    ; CHECK: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
+    ; CHECK: $vgpr0 = COPY [[BITCAST]](s32)
+    %0:_(<2 x s16>) = COPY $vgpr0
+    %1:_(s32) = G_BITCAST %0
+    $vgpr0 = COPY %1
+...
+
+---
+name: test_bitcast_v2s32_to_s64
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; CHECK-LABEL: name: test_bitcast_v2s32_to_s64
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    ; CHECK: [[BITCAST:%[0-9]+]]:_(s64) = G_BITCAST [[COPY]](<2 x s32>)
+    ; CHECK: $vgpr0_vgpr1 = COPY [[BITCAST]](s64)
+    %0:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    %1:_(s64) = G_BITCAST %0
+    $vgpr0_vgpr1 = COPY %1
+...
+
+---
+name: test_bitcast_s64_to_v2s32
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; CHECK-LABEL: name: test_bitcast_s64_to_v2s32
+    ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
+    ; CHECK: [[BITCAST:%[0-9]+]]:_(<2 x s32>) = G_BITCAST [[COPY]](s64)
+    ; CHECK: $vgpr0_vgpr1 = COPY [[BITCAST]](<2 x s32>)
+    %0:_(s64) = COPY $vgpr0_vgpr1
+    %1:_(<2 x s32>) = G_BITCAST %0
+    $vgpr0_vgpr1 = COPY %1
+...
+
+---
+name: test_bitcast_v2s64_to_v4s32
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3
+
+    ; CHECK-LABEL: name: test_bitcast_v2s64_to_v4s32
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; CHECK: [[BITCAST:%[0-9]+]]:_(<4 x s32>) = G_BITCAST [[COPY]](<2 x s64>)
+    ; CHECK: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BITCAST]](<4 x s32>)
+    %0:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    %1:_(<4 x s32>) = G_BITCAST %0
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %1
+...
+
+---
+name: test_bitcast_v4s32_to_v2s64
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3
+
+    ; CHECK-LABEL: name: test_bitcast_v4s32_to_v2s64
+    ; CHECK: [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; CHECK: [[BITCAST:%[0-9]+]]:_(<2 x s64>) = G_BITCAST [[COPY]](<4 x s32>)
+    ; CHECK: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BITCAST]](<2 x s64>)
+    %0:_(<4 x s32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    %1:_(<2 x s64>) = G_BITCAST %0
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %1
+...
+
+---
+name: test_bitcast_v4s16_to_s64
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; CHECK-LABEL: name: test_bitcast_v4s16_to_s64
+    ; CHECK: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
+    ; CHECK: [[BITCAST:%[0-9]+]]:_(s64) = G_BITCAST [[COPY]](<4 x s16>)
+    ; CHECK: $vgpr0_vgpr1 = COPY [[BITCAST]](s64)
+    %0:_(<4 x s16>) = COPY $vgpr0_vgpr1
+    %1:_(s64) = G_BITCAST %0
+    $vgpr0_vgpr1 = COPY %1
+...
+
+---
+name: test_bitcast_s64_to_v4s16
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; CHECK-LABEL: name: test_bitcast_s64_to_v4s16
+    ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
+    ; CHECK: [[BITCAST:%[0-9]+]]:_(<4 x s16>) = G_BITCAST [[COPY]](s64)
+    ; CHECK: $vgpr0_vgpr1 = COPY [[BITCAST]](<4 x s16>)
+    %0:_(s64) = COPY $vgpr0_vgpr1
+    %1:_(<4 x s16>) = G_BITCAST %0
+    $vgpr0_vgpr1 = COPY %1
+...
+
+---
+name: test_bitcast_p0_to_p1
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; CHECK-LABEL: name: test_bitcast_p0_to_p1
+    ; CHECK: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
+    ; CHECK: [[BITCAST:%[0-9]+]]:_(p1) = G_BITCAST [[COPY]](p0)
+    ; CHECK: $vgpr0_vgpr1 = COPY [[BITCAST]](p1)
+    %0:_(p0) = COPY $vgpr0_vgpr1
+    %1:_(p1) = G_BITCAST %0
+    $vgpr0_vgpr1 = COPY %1
+...
+
+---
+name: test_bitcast_p1_to_p0
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; CHECK-LABEL: name: test_bitcast_p1_to_p0
+    ; CHECK: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
+    ; CHECK: [[BITCAST:%[0-9]+]]:_(p0) = G_BITCAST [[COPY]](p1)
+    ; CHECK: $vgpr0_vgpr1 = COPY [[BITCAST]](p0)
+    %0:_(p1) = COPY $vgpr0_vgpr1
+    %1:_(p0) = G_BITCAST %0
+    $vgpr0_vgpr1 = COPY %1
+...
+
+---
+name: test_bitcast_p999_to_p0
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; CHECK-LABEL: name: test_bitcast_p999_to_p0
+    ; CHECK: [[COPY:%[0-9]+]]:_(p999) = COPY $vgpr0_vgpr1
+    ; CHECK: [[BITCAST:%[0-9]+]]:_(p0) = G_BITCAST [[COPY]](p999)
+    ; CHECK: $vgpr0_vgpr1 = COPY [[BITCAST]](p0)
+    %0:_(p999) = COPY $vgpr0_vgpr1
+    %1:_(p0) = G_BITCAST %0
+    $vgpr0_vgpr1 = COPY %1
+...
+
+---
+name: test_bitcast_p123_to_p999
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; CHECK-LABEL: name: test_bitcast_p123_to_p999
+    ; CHECK: [[COPY:%[0-9]+]]:_(p123) = COPY $vgpr0_vgpr1
+    ; CHECK: [[BITCAST:%[0-9]+]]:_(p999) = G_BITCAST [[COPY]](p123)
+    ; CHECK: $vgpr0_vgpr1 = COPY [[BITCAST]](p999)
+    %0:_(p123) = COPY $vgpr0_vgpr1
+    %1:_(p999) = G_BITCAST %0
+    $vgpr0_vgpr1 = COPY %1
 ...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/legalize-brcond.mir b/test/CodeGen/AMDGPU/GlobalISel/legalize-brcond.mir
new file mode 100644
index 0000000000000000000000000000000000000000..eb8b34915f50672e5635e7ae186896e8bb0676c7
--- /dev/null
+++ b/test/CodeGen/AMDGPU/GlobalISel/legalize-brcond.mir
@@ -0,0 +1,24 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -O0 -run-pass=legalizer %s -o - | FileCheck %s
+
+---
+name: legal_brcond
+body:             |
+  ; CHECK-LABEL: name: legal_brcond
+  ; CHECK: bb.0.entry:
+  ; CHECK:   successors: %bb.1(0x80000000)
+  ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; CHECK:   [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[COPY]](s32), [[COPY1]]
+  ; CHECK:   G_BRCOND [[ICMP]](s1), %bb.1
+  ; CHECK: bb.1:
+  bb.0.entry:
+    successors: %bb.1
+    liveins: $vgpr0, $vgpr1
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s1) = G_ICMP intpred(ne), %0, %1
+    G_BRCOND %2, %bb.1
+
+  bb.1:
+...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/legalize-bswap.mir b/test/CodeGen/AMDGPU/GlobalISel/legalize-bswap.mir
new file mode 100644
index 0000000000000000000000000000000000000000..ed45be908c3b45f248c180179c46424a05ffdcb0
--- /dev/null
+++ b/test/CodeGen/AMDGPU/GlobalISel/legalize-bswap.mir
@@ -0,0 +1,125 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -O0 -run-pass=legalizer %s -o - | FileCheck %s
+
+---
+name: bswap_s8
+
+body: |
+  bb.0:
+    liveins: $vgpr0
+    ; CHECK-LABEL: name: bswap_s8
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; CHECK: [[BSWAP:%[0-9]+]]:_(s32) = G_BSWAP [[COPY1]]
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; CHECK: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BSWAP]], [[C]](s32)
+    ; CHECK: [[COPY2:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; CHECK: $vgpr0 = COPY [[COPY2]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s8) = G_TRUNC %0
+    %2:_(s8) = G_BSWAP %1
+    %3:_(s32) = G_ANYEXT %2
+    $vgpr0 = COPY %3
+...
+
+---
+name: bswap_s16
+
+body: |
+  bb.0:
+    liveins: $vgpr0
+    ; CHECK-LABEL: name: bswap_s16
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; CHECK: [[BSWAP:%[0-9]+]]:_(s32) = G_BSWAP [[COPY1]]
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; CHECK: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BSWAP]], [[C]](s32)
+    ; CHECK: [[COPY2:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; CHECK: $vgpr0 = COPY [[COPY2]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s16) = G_TRUNC %0
+    %2:_(s16) = G_BSWAP %1
+    %3:_(s32) = G_ANYEXT %2
+    $vgpr0 = COPY %3
+...
+
+---
+name: bswap_s24
+
+body: |
+  bb.0:
+    liveins: $vgpr0
+    ; CHECK-LABEL: name: bswap_s24
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; CHECK: [[BSWAP:%[0-9]+]]:_(s32) = G_BSWAP [[COPY1]]
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; CHECK: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BSWAP]], [[C]](s32)
+    ; CHECK: [[COPY2:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; CHECK: $vgpr0 = COPY [[COPY2]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s24) = G_TRUNC %0
+    %2:_(s24) = G_BSWAP %1
+    %3:_(s32) = G_ANYEXT %2
+    $vgpr0 = COPY %3
+...
+
+---
+name: bswap_s32
+
+body: |
+  bb.0:
+    liveins: $vgpr0
+    ; CHECK-LABEL: name: bswap_s32
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[BSWAP:%[0-9]+]]:_(s32) = G_BSWAP [[COPY]]
+    ; CHECK: $vgpr0 = COPY [[BSWAP]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = G_BSWAP %0
+    $vgpr0 = COPY %1
+...
+
+---
+name: bswap_v2s16
+
+body: |
+  bb.0:
+    liveins: $vgpr0
+    ; CHECK-LABEL: name: bswap_v2s16
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
+    ; CHECK: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY]](<2 x s16>)
+    ; CHECK: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[UV]](s16)
+    ; CHECK: [[BSWAP:%[0-9]+]]:_(s32) = G_BSWAP [[ANYEXT]]
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; CHECK: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BSWAP]], [[C]](s32)
+    ; CHECK: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+    ; CHECK: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[UV1]](s16)
+    ; CHECK: [[BSWAP1:%[0-9]+]]:_(s32) = G_BSWAP [[ANYEXT1]]
+    ; CHECK: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; CHECK: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BSWAP1]], [[C1]](s32)
+    ; CHECK: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
+    ; CHECK: $vgpr0 = COPY [[BUILD_VECTOR]](<2 x s16>)
+    %0:_(<2 x s16>) = COPY $vgpr0
+    %1:_(<2 x s16>) = G_BSWAP %0
+    $vgpr0 = COPY %1
+...
+
+---
+name: bswap_v2s32
+
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+    ; CHECK-LABEL: name: bswap_v2s32
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    ; CHECK: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
+    ; CHECK: [[BSWAP:%[0-9]+]]:_(s32) = G_BSWAP [[UV]]
+    ; CHECK: [[BSWAP1:%[0-9]+]]:_(s32) = G_BSWAP [[UV1]]
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[BSWAP]](s32), [[BSWAP1]](s32)
+    ; CHECK: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
+    %0:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    %1:_(<2 x s32>) = G_BSWAP %0
+    $vgpr0_vgpr1 = COPY %1
+...
+
diff --git a/test/CodeGen/AMDGPU/GlobalISel/legalize-build-vector.mir b/test/CodeGen/AMDGPU/GlobalISel/legalize-build-vector.mir
new file mode 100644
index 0000000000000000000000000000000000000000..2768ff87dc5cb107a9d0c7f3c4b177139432d3eb
--- /dev/null
+++ b/test/CodeGen/AMDGPU/GlobalISel/legalize-build-vector.mir
@@ -0,0 +1,585 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -O0 -run-pass=legalizer %s -o - | FileCheck %s
+
+---
+name: legal_v2s32
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+    ; CHECK-LABEL: name: legal_v2s32
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32)
+    ; CHECK: S_NOP 0, implicit [[BUILD_VECTOR]](<2 x s32>)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(<2 x s32>) = G_BUILD_VECTOR %0, %1
+    S_NOP 0, implicit %2
+...
+---
+name: legal_v3s32
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2
+    ; CHECK-LABEL: name: legal_v3s32
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; CHECK: [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32)
+    ; CHECK: S_NOP 0, implicit [[BUILD_VECTOR]](<3 x s32>)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s32) = COPY $vgpr2
+    %3:_(<3 x s32>) = G_BUILD_VECTOR %0, %1, %2
+    S_NOP 0, implicit %3
+...
+---
+name: legal_v4s32
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3
+    ; CHECK-LABEL: name: legal_v4s32
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; CHECK: [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+    ; CHECK: [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr3
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32)
+    ; CHECK: S_NOP 0, implicit [[BUILD_VECTOR]](<4 x s32>)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s32) = COPY $vgpr2
+    %3:_(s32) = COPY $vgpr3
+    %4:_(<4 x s32>) = G_BUILD_VECTOR %0, %1, %2, %3
+    S_NOP 0, implicit %4
+...
+---
+name: legal_v5s32
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+    ; CHECK-LABEL: name: legal_v5s32
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; CHECK: [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+    ; CHECK: [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr3
+    ; CHECK: [[COPY4:%[0-9]+]]:_(s32) = COPY $vgpr4
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<5 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32)
+    ; CHECK: S_NOP 0, implicit [[BUILD_VECTOR]](<5 x s32>)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s32) = COPY $vgpr2
+    %3:_(s32) = COPY $vgpr3
+    %4:_(s32) = COPY $vgpr4
+    %5:_(<5 x s32>) = G_BUILD_VECTOR %0, %1, %2, %3, %4
+    S_NOP 0, implicit %5
+...
+---
+name: legal_v6s32
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+    ; CHECK-LABEL: name: legal_v6s32
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; CHECK: [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+    ; CHECK: [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr3
+    ; CHECK: [[COPY4:%[0-9]+]]:_(s32) = COPY $vgpr4
+    ; CHECK: [[COPY5:%[0-9]+]]:_(s32) = COPY $vgpr5
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<6 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32)
+    ; CHECK: S_NOP 0, implicit [[BUILD_VECTOR]](<6 x s32>)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s32) = COPY $vgpr2
+    %3:_(s32) = COPY $vgpr3
+    %4:_(s32) = COPY $vgpr4
+    %5:_(s32) = COPY $vgpr5
+    %6:_(<6 x s32>) = G_BUILD_VECTOR %0, %1, %2, %3, %4, %5
+    S_NOP 0, implicit %6
+...
+---
+name: legal_v7s32
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+    ; CHECK-LABEL: name: legal_v7s32
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; CHECK: [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+    ; CHECK: [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr3
+    ; CHECK: [[COPY4:%[0-9]+]]:_(s32) = COPY $vgpr4
+    ; CHECK: [[COPY5:%[0-9]+]]:_(s32) = COPY $vgpr5
+    ; CHECK: [[COPY6:%[0-9]+]]:_(s32) = COPY $vgpr6
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<7 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32)
+    ; CHECK: S_NOP 0, implicit [[BUILD_VECTOR]](<7 x s32>)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s32) = COPY $vgpr2
+    %3:_(s32) = COPY $vgpr3
+    %4:_(s32) = COPY $vgpr4
+    %5:_(s32) = COPY $vgpr5
+    %6:_(s32) = COPY $vgpr6
+    %7:_(<7 x s32>) = G_BUILD_VECTOR %0, %1, %2, %3, %4, %5, %6
+    S_NOP 0, implicit %7
+...
+---
+name: legal_v8s32
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7
+    ; CHECK-LABEL: name: legal_v8s32
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; CHECK: [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+    ; CHECK: [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr3
+    ; CHECK: [[COPY4:%[0-9]+]]:_(s32) = COPY $vgpr4
+    ; CHECK: [[COPY5:%[0-9]+]]:_(s32) = COPY $vgpr5
+    ; CHECK: [[COPY6:%[0-9]+]]:_(s32) = COPY $vgpr6
+    ; CHECK: [[COPY7:%[0-9]+]]:_(s32) = COPY $vgpr7
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
+    ; CHECK: S_NOP 0, implicit [[BUILD_VECTOR]](<8 x s32>)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s32) = COPY $vgpr2
+    %3:_(s32) = COPY $vgpr3
+    %4:_(s32) = COPY $vgpr4
+    %5:_(s32) = COPY $vgpr5
+    %6:_(s32) = COPY $vgpr6
+    %7:_(s32) = COPY $vgpr7
+    %8:_(<8 x s32>) = G_BUILD_VECTOR %0, %1, %2, %3, %4, %5, %6, %7
+    S_NOP 0, implicit %8
+...
+---
+name: legal_v9s32
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8
+    ; CHECK-LABEL: name: legal_v9s32
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; CHECK: [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+    ; CHECK: [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr3
+    ; CHECK: [[COPY4:%[0-9]+]]:_(s32) = COPY $vgpr4
+    ; CHECK: [[COPY5:%[0-9]+]]:_(s32) = COPY $vgpr5
+    ; CHECK: [[COPY6:%[0-9]+]]:_(s32) = COPY $vgpr6
+    ; CHECK: [[COPY7:%[0-9]+]]:_(s32) = COPY $vgpr7
+    ; CHECK: [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr8
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<9 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32), [[COPY8]](s32)
+    ; CHECK: S_NOP 0, implicit [[BUILD_VECTOR]](<9 x s32>)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s32) = COPY $vgpr2
+    %3:_(s32) = COPY $vgpr3
+    %4:_(s32) = COPY $vgpr4
+    %5:_(s32) = COPY $vgpr5
+    %6:_(s32) = COPY $vgpr6
+    %7:_(s32) = COPY $vgpr7
+    %8:_(s32) = COPY $vgpr8
+    %9:_(<9 x s32>) = G_BUILD_VECTOR %0, %1, %2, %3, %4, %5, %6, %7, %8
+    S_NOP 0, implicit %9
+...
+---
+name: legal_v10s32
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8, $vgpr9
+    ; CHECK-LABEL: name: legal_v10s32
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; CHECK: [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+    ; CHECK: [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr3
+    ; CHECK: [[COPY4:%[0-9]+]]:_(s32) = COPY $vgpr4
+    ; CHECK: [[COPY5:%[0-9]+]]:_(s32) = COPY $vgpr5
+    ; CHECK: [[COPY6:%[0-9]+]]:_(s32) = COPY $vgpr6
+    ; CHECK: [[COPY7:%[0-9]+]]:_(s32) = COPY $vgpr7
+    ; CHECK: [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr8
+    ; CHECK: [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr9
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<10 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32), [[COPY8]](s32), [[COPY9]](s32)
+    ; CHECK: S_NOP 0, implicit [[BUILD_VECTOR]](<10 x s32>)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s32) = COPY $vgpr2
+    %3:_(s32) = COPY $vgpr3
+    %4:_(s32) = COPY $vgpr4
+    %5:_(s32) = COPY $vgpr5
+    %6:_(s32) = COPY $vgpr6
+    %7:_(s32) = COPY $vgpr7
+    %8:_(s32) = COPY $vgpr8
+    %9:_(s32) = COPY $vgpr9
+    %10:_(<10 x s32>) = G_BUILD_VECTOR %0, %1, %2, %3, %4, %5, %6, %7, %8, %9
+    S_NOP 0, implicit %10
+...
+---
+name: legal_v11s32
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8, $vgpr9, $vgpr10
+    ; CHECK-LABEL: name: legal_v11s32
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; CHECK: [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+    ; CHECK: [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr3
+    ; CHECK: [[COPY4:%[0-9]+]]:_(s32) = COPY $vgpr4
+    ; CHECK: [[COPY5:%[0-9]+]]:_(s32) = COPY $vgpr5
+    ; CHECK: [[COPY6:%[0-9]+]]:_(s32) = COPY $vgpr6
+    ; CHECK: [[COPY7:%[0-9]+]]:_(s32) = COPY $vgpr7
+    ; CHECK: [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr8
+    ; CHECK: [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr9
+    ; CHECK: [[COPY10:%[0-9]+]]:_(s32) = COPY $vgpr10
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<11 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32), [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32)
+    ; CHECK: S_NOP 0, implicit [[BUILD_VECTOR]](<11 x s32>)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s32) = COPY $vgpr2
+    %3:_(s32) = COPY $vgpr3
+    %4:_(s32) = COPY $vgpr4
+    %5:_(s32) = COPY $vgpr5
+    %6:_(s32) = COPY $vgpr6
+    %7:_(s32) = COPY $vgpr7
+    %8:_(s32) = COPY $vgpr8
+    %9:_(s32) = COPY $vgpr9
+    %10:_(s32) = COPY $vgpr10
+    %11:_(<11 x s32>) = G_BUILD_VECTOR %0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10
+    S_NOP 0, implicit %11
+...
+---
+name: legal_v12s32
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8, $vgpr9, $vgpr10, $vgpr11
+    ; CHECK-LABEL: name: legal_v12s32
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; CHECK: [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+    ; CHECK: [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr3
+    ; CHECK: [[COPY4:%[0-9]+]]:_(s32) = COPY $vgpr4
+    ; CHECK: [[COPY5:%[0-9]+]]:_(s32) = COPY $vgpr5
+    ; CHECK: [[COPY6:%[0-9]+]]:_(s32) = COPY $vgpr6
+    ; CHECK: [[COPY7:%[0-9]+]]:_(s32) = COPY $vgpr7
+    ; CHECK: [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr8
+    ; CHECK: [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr9
+    ; CHECK: [[COPY10:%[0-9]+]]:_(s32) = COPY $vgpr10
+    ; CHECK: [[COPY11:%[0-9]+]]:_(s32) = COPY $vgpr11
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<12 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32), [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
+    ; CHECK: S_NOP 0, implicit [[BUILD_VECTOR]](<12 x s32>)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s32) = COPY $vgpr2
+    %3:_(s32) = COPY $vgpr3
+    %4:_(s32) = COPY $vgpr4
+    %5:_(s32) = COPY $vgpr5
+    %6:_(s32) = COPY $vgpr6
+    %7:_(s32) = COPY $vgpr7
+    %8:_(s32) = COPY $vgpr8
+    %9:_(s32) = COPY $vgpr9
+    %10:_(s32) = COPY $vgpr10
+    %11:_(s32) = COPY $vgpr11
+    %12:_(<12 x s32>) = G_BUILD_VECTOR %0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11
+    S_NOP 0, implicit %12
+...
+---
+name: legal_v13s32
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8, $vgpr9, $vgpr10, $vgpr11, $vgpr12
+    ; CHECK-LABEL: name: legal_v13s32
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; CHECK: [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+    ; CHECK: [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr3
+    ; CHECK: [[COPY4:%[0-9]+]]:_(s32) = COPY $vgpr4
+    ; CHECK: [[COPY5:%[0-9]+]]:_(s32) = COPY $vgpr5
+    ; CHECK: [[COPY6:%[0-9]+]]:_(s32) = COPY $vgpr6
+    ; CHECK: [[COPY7:%[0-9]+]]:_(s32) = COPY $vgpr7
+    ; CHECK: [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr8
+    ; CHECK: [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr9
+    ; CHECK: [[COPY10:%[0-9]+]]:_(s32) = COPY $vgpr10
+    ; CHECK: [[COPY11:%[0-9]+]]:_(s32) = COPY $vgpr11
+    ; CHECK: [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr12
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<13 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32), [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32), [[COPY12]](s32)
+    ; CHECK: S_NOP 0, implicit [[BUILD_VECTOR]](<13 x s32>)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s32) = COPY $vgpr2
+    %3:_(s32) = COPY $vgpr3
+    %4:_(s32) = COPY $vgpr4
+    %5:_(s32) = COPY $vgpr5
+    %6:_(s32) = COPY $vgpr6
+    %7:_(s32) = COPY $vgpr7
+    %8:_(s32) = COPY $vgpr8
+    %9:_(s32) = COPY $vgpr9
+    %10:_(s32) = COPY $vgpr10
+    %11:_(s32) = COPY $vgpr11
+    %12:_(s32) = COPY $vgpr12
+    %13:_(<13 x s32>) = G_BUILD_VECTOR %0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12
+    S_NOP 0, implicit %13
+...
+---
+name: legal_v14s32
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8, $vgpr9, $vgpr10, $vgpr11, $vgpr12, $vgpr13
+    ; CHECK-LABEL: name: legal_v14s32
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; CHECK: [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+    ; CHECK: [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr3
+    ; CHECK: [[COPY4:%[0-9]+]]:_(s32) = COPY $vgpr4
+    ; CHECK: [[COPY5:%[0-9]+]]:_(s32) = COPY $vgpr5
+    ; CHECK: [[COPY6:%[0-9]+]]:_(s32) = COPY $vgpr6
+    ; CHECK: [[COPY7:%[0-9]+]]:_(s32) = COPY $vgpr7
+    ; CHECK: [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr8
+    ; CHECK: [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr9
+    ; CHECK: [[COPY10:%[0-9]+]]:_(s32) = COPY $vgpr10
+    ; CHECK: [[COPY11:%[0-9]+]]:_(s32) = COPY $vgpr11
+    ; CHECK: [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr12
+    ; CHECK: [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr13
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<14 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32), [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32), [[COPY12]](s32), [[COPY13]](s32)
+    ; CHECK: S_NOP 0, implicit [[BUILD_VECTOR]](<14 x s32>)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s32) = COPY $vgpr2
+    %3:_(s32) = COPY $vgpr3
+    %4:_(s32) = COPY $vgpr4
+    %5:_(s32) = COPY $vgpr5
+    %6:_(s32) = COPY $vgpr6
+    %7:_(s32) = COPY $vgpr7
+    %8:_(s32) = COPY $vgpr8
+    %9:_(s32) = COPY $vgpr9
+    %10:_(s32) = COPY $vgpr10
+    %11:_(s32) = COPY $vgpr11
+    %12:_(s32) = COPY $vgpr12
+    %13:_(s32) = COPY $vgpr13
+    %14:_(<14 x s32>) = G_BUILD_VECTOR %0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13
+    S_NOP 0, implicit %14
+...
+---
+name: legal_v15s32
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8, $vgpr9, $vgpr10, $vgpr11, $vgpr12, $vgpr13, $vgpr14
+    ; CHECK-LABEL: name: legal_v15s32
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; CHECK: [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+    ; CHECK: [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr3
+    ; CHECK: [[COPY4:%[0-9]+]]:_(s32) = COPY $vgpr4
+    ; CHECK: [[COPY5:%[0-9]+]]:_(s32) = COPY $vgpr5
+    ; CHECK: [[COPY6:%[0-9]+]]:_(s32) = COPY $vgpr6
+    ; CHECK: [[COPY7:%[0-9]+]]:_(s32) = COPY $vgpr7
+    ; CHECK: [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr8
+    ; CHECK: [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr9
+    ; CHECK: [[COPY10:%[0-9]+]]:_(s32) = COPY $vgpr10
+    ; CHECK: [[COPY11:%[0-9]+]]:_(s32) = COPY $vgpr11
+    ; CHECK: [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr12
+    ; CHECK: [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr13
+    ; CHECK: [[COPY14:%[0-9]+]]:_(s32) = COPY $vgpr14
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<15 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32), [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32), [[COPY12]](s32), [[COPY13]](s32), [[COPY14]](s32)
+    ; CHECK: S_NOP 0, implicit [[BUILD_VECTOR]](<15 x s32>)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s32) = COPY $vgpr2
+    %3:_(s32) = COPY $vgpr3
+    %4:_(s32) = COPY $vgpr4
+    %5:_(s32) = COPY $vgpr5
+    %6:_(s32) = COPY $vgpr6
+    %7:_(s32) = COPY $vgpr7
+    %8:_(s32) = COPY $vgpr8
+    %9:_(s32) = COPY $vgpr9
+    %10:_(s32) = COPY $vgpr10
+    %11:_(s32) = COPY $vgpr11
+    %12:_(s32) = COPY $vgpr12
+    %13:_(s32) = COPY $vgpr13
+    %14:_(s32) = COPY $vgpr14
+    %15:_(<15 x s32>) = G_BUILD_VECTOR %0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14
+    S_NOP 0, implicit %15
+...
+---
+name: legal_v16s32
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8, $vgpr9, $vgpr10, $vgpr11, $vgpr12, $vgpr13, $vgpr14, $vgpr15
+    ; CHECK-LABEL: name: legal_v16s32
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; CHECK: [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+    ; CHECK: [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr3
+    ; CHECK: [[COPY4:%[0-9]+]]:_(s32) = COPY $vgpr4
+    ; CHECK: [[COPY5:%[0-9]+]]:_(s32) = COPY $vgpr5
+    ; CHECK: [[COPY6:%[0-9]+]]:_(s32) = COPY $vgpr6
+    ; CHECK: [[COPY7:%[0-9]+]]:_(s32) = COPY $vgpr7
+    ; CHECK: [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr8
+    ; CHECK: [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr9
+    ; CHECK: [[COPY10:%[0-9]+]]:_(s32) = COPY $vgpr10
+    ; CHECK: [[COPY11:%[0-9]+]]:_(s32) = COPY $vgpr11
+    ; CHECK: [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr12
+    ; CHECK: [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr13
+    ; CHECK: [[COPY14:%[0-9]+]]:_(s32) = COPY $vgpr14
+    ; CHECK: [[COPY15:%[0-9]+]]:_(s32) = COPY $vgpr15
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<16 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32), [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32), [[COPY12]](s32), [[COPY13]](s32), [[COPY14]](s32), [[COPY15]](s32)
+    ; CHECK: S_NOP 0, implicit [[BUILD_VECTOR]](<16 x s32>)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s32) = COPY $vgpr2
+    %3:_(s32) = COPY $vgpr3
+    %4:_(s32) = COPY $vgpr4
+    %5:_(s32) = COPY $vgpr5
+    %6:_(s32) = COPY $vgpr6
+    %7:_(s32) = COPY $vgpr7
+    %8:_(s32) = COPY $vgpr8
+    %9:_(s32) = COPY $vgpr9
+    %10:_(s32) = COPY $vgpr10
+    %11:_(s32) = COPY $vgpr11
+    %12:_(s32) = COPY $vgpr12
+    %13:_(s32) = COPY $vgpr13
+    %14:_(s32) = COPY $vgpr14
+    %15:_(s32) = COPY $vgpr15
+    %16:_(<16 x s32>) = G_BUILD_VECTOR %0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15
+    S_NOP 0, implicit %16
+...
+---
+name: legal_v2s64
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
+    ; CHECK-LABEL: name: legal_v2s64
+    ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s64) = COPY $vgpr2_vgpr3
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[COPY]](s64), [[COPY1]](s64)
+    ; CHECK: S_NOP 0, implicit [[BUILD_VECTOR]](<2 x s64>)
+    %0:_(s64) = COPY $vgpr0_vgpr1
+    %1:_(s64) = COPY $vgpr2_vgpr3
+    %2:_(<2 x s64>) = G_BUILD_VECTOR %0, %1
+    S_NOP 0, implicit %2
+...
+---
+name: legal_v3s64
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2_vgpr3, $vgpr4_vgpr5
+    ; CHECK-LABEL: name: legal_v3s64
+    ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s64) = COPY $vgpr2_vgpr3
+    ; CHECK: [[COPY2:%[0-9]+]]:_(s64) = COPY $vgpr4_vgpr5
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s64>) = G_BUILD_VECTOR [[COPY]](s64), [[COPY1]](s64), [[COPY2]](s64)
+    ; CHECK: S_NOP 0, implicit [[BUILD_VECTOR]](<3 x s64>)
+    %0:_(s64) = COPY $vgpr0_vgpr1
+    %1:_(s64) = COPY $vgpr2_vgpr3
+    %2:_(s64) = COPY $vgpr4_vgpr5
+    %3:_(<3 x s64>) = G_BUILD_VECTOR %0, %1, %2
+    S_NOP 0, implicit %3
+...
+---
+name: legal_v4s64
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2_vgpr3, $vgpr4_vgpr5, $vgpr6_vgpr7
+    ; CHECK-LABEL: name: legal_v4s64
+    ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s64) = COPY $vgpr2_vgpr3
+    ; CHECK: [[COPY2:%[0-9]+]]:_(s64) = COPY $vgpr4_vgpr5
+    ; CHECK: [[COPY3:%[0-9]+]]:_(s64) = COPY $vgpr6_vgpr7
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s64>) = G_BUILD_VECTOR [[COPY]](s64), [[COPY1]](s64), [[COPY2]](s64), [[COPY3]](s64)
+    ; CHECK: S_NOP 0, implicit [[BUILD_VECTOR]](<4 x s64>)
+    %0:_(s64) = COPY $vgpr0_vgpr1
+    %1:_(s64) = COPY $vgpr2_vgpr3
+    %2:_(s64) = COPY $vgpr4_vgpr5
+    %3:_(s64) = COPY $vgpr6_vgpr7
+    %4:_(<4 x s64>) = G_BUILD_VECTOR %0, %1, %2, %3
+    S_NOP 0, implicit %4
+...
+---
+name: legal_v5s64
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2_vgpr3, $vgpr4_vgpr5, $vgpr6_vgpr7, $vgpr8_vgpr9
+    ; CHECK-LABEL: name: legal_v5s64
+    ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s64) = COPY $vgpr2_vgpr3
+    ; CHECK: [[COPY2:%[0-9]+]]:_(s64) = COPY $vgpr4_vgpr5
+    ; CHECK: [[COPY3:%[0-9]+]]:_(s64) = COPY $vgpr6_vgpr7
+    ; CHECK: [[COPY4:%[0-9]+]]:_(s64) = COPY $vgpr8_vgpr9
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<5 x s64>) = G_BUILD_VECTOR [[COPY]](s64), [[COPY1]](s64), [[COPY2]](s64), [[COPY3]](s64), [[COPY4]](s64)
+    ; CHECK: S_NOP 0, implicit [[BUILD_VECTOR]](<5 x s64>)
+    %0:_(s64) = COPY $vgpr0_vgpr1
+    %1:_(s64) = COPY $vgpr2_vgpr3
+    %2:_(s64) = COPY $vgpr4_vgpr5
+    %3:_(s64) = COPY $vgpr6_vgpr7
+    %4:_(s64) = COPY $vgpr8_vgpr9
+    %5:_(<5 x s64>) = G_BUILD_VECTOR %0, %1, %2, %3, %4
+    S_NOP 0, implicit %5
+...
+---
+name: legal_v6s64
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2_vgpr3, $vgpr4_vgpr5, $vgpr6_vgpr7, $vgpr8_vgpr9, $vgpr10_vgpr11
+    ; CHECK-LABEL: name: legal_v6s64
+    ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s64) = COPY $vgpr2_vgpr3
+    ; CHECK: [[COPY2:%[0-9]+]]:_(s64) = COPY $vgpr4_vgpr5
+    ; CHECK: [[COPY3:%[0-9]+]]:_(s64) = COPY $vgpr6_vgpr7
+    ; CHECK: [[COPY4:%[0-9]+]]:_(s64) = COPY $vgpr8_vgpr9
+    ; CHECK: [[COPY5:%[0-9]+]]:_(s64) = COPY $vgpr10_vgpr11
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<6 x s64>) = G_BUILD_VECTOR [[COPY]](s64), [[COPY1]](s64), [[COPY2]](s64), [[COPY3]](s64), [[COPY4]](s64), [[COPY5]](s64)
+    ; CHECK: S_NOP 0, implicit [[BUILD_VECTOR]](<6 x s64>)
+    %0:_(s64) = COPY $vgpr0_vgpr1
+    %1:_(s64) = COPY $vgpr2_vgpr3
+    %2:_(s64) = COPY $vgpr4_vgpr5
+    %3:_(s64) = COPY $vgpr6_vgpr7
+    %4:_(s64) = COPY $vgpr8_vgpr9
+    %5:_(s64) = COPY $vgpr10_vgpr11
+    %6:_(<6 x s64>) = G_BUILD_VECTOR %0, %1, %2, %3, %4, %5
+    S_NOP 0, implicit %6
+...
+---
+name: legal_v7s64
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2_vgpr3, $vgpr4_vgpr5, $vgpr6_vgpr7, $vgpr8_vgpr9, $vgpr10_vgpr11, $vgpr12_vgpr13
+    ; CHECK-LABEL: name: legal_v7s64
+    ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s64) = COPY $vgpr2_vgpr3
+    ; CHECK: [[COPY2:%[0-9]+]]:_(s64) = COPY $vgpr4_vgpr5
+    ; CHECK: [[COPY3:%[0-9]+]]:_(s64) = COPY $vgpr6_vgpr7
+    ; CHECK: [[COPY4:%[0-9]+]]:_(s64) = COPY $vgpr8_vgpr9
+    ; CHECK: [[COPY5:%[0-9]+]]:_(s64) = COPY $vgpr10_vgpr11
+    ; CHECK: [[COPY6:%[0-9]+]]:_(s64) = COPY $vgpr12_vgpr13
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<7 x s64>) = G_BUILD_VECTOR [[COPY]](s64), [[COPY1]](s64), [[COPY2]](s64), [[COPY3]](s64), [[COPY4]](s64), [[COPY5]](s64), [[COPY6]](s64)
+    ; CHECK: S_NOP 0, implicit [[BUILD_VECTOR]](<7 x s64>)
+    %0:_(s64) = COPY $vgpr0_vgpr1
+    %1:_(s64) = COPY $vgpr2_vgpr3
+    %2:_(s64) = COPY $vgpr4_vgpr5
+    %3:_(s64) = COPY $vgpr6_vgpr7
+    %4:_(s64) = COPY $vgpr8_vgpr9
+    %5:_(s64) = COPY $vgpr10_vgpr11
+    %6:_(s64) = COPY $vgpr12_vgpr13
+    %7:_(<7 x s64>) = G_BUILD_VECTOR %0, %1, %2, %3, %4, %5, %6
+    S_NOP 0, implicit %7
+...
+---
+name: legal_v8s64
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2_vgpr3, $vgpr4_vgpr5, $vgpr6_vgpr7, $vgpr8_vgpr9, $vgpr10_vgpr11, $vgpr12_vgpr13, $vgpr14_vgpr15
+    ; CHECK-LABEL: name: legal_v8s64
+    ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s64) = COPY $vgpr2_vgpr3
+    ; CHECK: [[COPY2:%[0-9]+]]:_(s64) = COPY $vgpr4_vgpr5
+    ; CHECK: [[COPY3:%[0-9]+]]:_(s64) = COPY $vgpr6_vgpr7
+    ; CHECK: [[COPY4:%[0-9]+]]:_(s64) = COPY $vgpr8_vgpr9
+    ; CHECK: [[COPY5:%[0-9]+]]:_(s64) = COPY $vgpr10_vgpr11
+    ; CHECK: [[COPY6:%[0-9]+]]:_(s64) = COPY $vgpr12_vgpr13
+    ; CHECK: [[COPY7:%[0-9]+]]:_(s64) = COPY $vgpr14_vgpr15
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s64>) = G_BUILD_VECTOR [[COPY]](s64), [[COPY1]](s64), [[COPY2]](s64), [[COPY3]](s64), [[COPY4]](s64), [[COPY5]](s64), [[COPY6]](s64), [[COPY7]](s64)
+    ; CHECK: S_NOP 0, implicit [[BUILD_VECTOR]](<8 x s64>)
+    %0:_(s64) = COPY $vgpr0_vgpr1
+    %1:_(s64) = COPY $vgpr2_vgpr3
+    %2:_(s64) = COPY $vgpr4_vgpr5
+    %3:_(s64) = COPY $vgpr6_vgpr7
+    %4:_(s64) = COPY $vgpr8_vgpr9
+    %5:_(s64) = COPY $vgpr10_vgpr11
+    %6:_(s64) = COPY $vgpr12_vgpr13
+    %7:_(s64) = COPY $vgpr14_vgpr15
+    %8:_(<8 x s64>) = G_BUILD_VECTOR %0, %1, %2, %3, %4, %5, %6, %7
+    S_NOP 0, implicit %8
+...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/legalize-concat-vectors.mir b/test/CodeGen/AMDGPU/GlobalISel/legalize-concat-vectors.mir
new file mode 100644
index 0000000000000000000000000000000000000000..c3e1bca6003a77c4008d2d0ef9bb9f2174303e78
--- /dev/null
+++ b/test/CodeGen/AMDGPU/GlobalISel/legalize-concat-vectors.mir
@@ -0,0 +1,129 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -O0 -run-pass=legalizer %s -o - | FileCheck %s
+
+---
+name: concat_vectors_v2s32_v2s32
+
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
+    ; CHECK-LABEL: name: concat_vectors_v2s32_v2s32
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    ; CHECK: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr2_vgpr3
+    ; CHECK: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s32>) = G_CONCAT_VECTORS [[COPY]](<2 x s32>), [[COPY1]](<2 x s32>)
+    ; CHECK: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[CONCAT_VECTORS]](<4 x s32>)
+    %0:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    %1:_(<2 x s32>) = COPY $vgpr2_vgpr3
+    %2:_(<4 x s32>) = G_CONCAT_VECTORS %0, %1
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %2
+...
+
+---
+name: concat_vectors_v2s16_v2s16
+
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+    ; CHECK-LABEL: name: concat_vectors_v2s16_v2s16
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
+    ; CHECK: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[COPY]](<2 x s16>), [[COPY1]](<2 x s16>)
+    ; CHECK: $vgpr0_vgpr1 = COPY [[CONCAT_VECTORS]](<4 x s16>)
+    %0:_(<2 x s16>) = COPY $vgpr0
+    %1:_(<2 x s16>) = COPY $vgpr1
+    %2:_(<4 x s16>) = G_CONCAT_VECTORS %0, %1
+    $vgpr0_vgpr1 = COPY %2
+...
+
+---
+name: concat_vectors_v2s16_v2s16_v2s16_v2s16
+
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3
+    ; CHECK-LABEL: name: concat_vectors_v2s16_v2s16_v2s16_v2s16
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
+    ; CHECK: [[COPY2:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr2
+    ; CHECK: [[COPY3:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr3
+    ; CHECK: [[CONCAT_VECTORS:%[0-9]+]]:_(<8 x s16>) = G_CONCAT_VECTORS [[COPY]](<2 x s16>), [[COPY1]](<2 x s16>), [[COPY2]](<2 x s16>), [[COPY3]](<2 x s16>)
+    ; CHECK: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[CONCAT_VECTORS]](<8 x s16>)
+    %0:_(<2 x s16>) = COPY $vgpr0
+    %1:_(<2 x s16>) = COPY $vgpr1
+    %2:_(<2 x s16>) = COPY $vgpr2
+    %3:_(<2 x s16>) = COPY $vgpr3
+    %4:_(<8 x s16>) = G_CONCAT_VECTORS %0, %1, %2, %3
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %4
+...
+
+---
+name: concat_vectors_v4s16_v4s16
+
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
+    ; CHECK-LABEL: name: concat_vectors_v4s16_v4s16
+    ; CHECK: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
+    ; CHECK: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
+    ; CHECK: [[CONCAT_VECTORS:%[0-9]+]]:_(<8 x s16>) = G_CONCAT_VECTORS [[COPY]](<4 x s16>), [[COPY1]](<4 x s16>)
+    ; CHECK: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[CONCAT_VECTORS]](<8 x s16>)
+    %0:_(<4 x s16>) = COPY $vgpr0_vgpr1
+    %1:_(<4 x s16>) = COPY $vgpr2_vgpr3
+    %2:_(<8 x s16>) = G_CONCAT_VECTORS %0, %1
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %2
+...
+
+---
+name: concat_vectors_v4s32_v4s32
+
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7
+    ; CHECK-LABEL: name: concat_vectors_v4s32_v4s32
+    ; CHECK: [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; CHECK: [[COPY1:%[0-9]+]]:_(<4 x s32>) = COPY $vgpr4_vgpr5_vgpr6_vgpr7
+    ; CHECK: [[CONCAT_VECTORS:%[0-9]+]]:_(<8 x s32>) = G_CONCAT_VECTORS [[COPY]](<4 x s32>), [[COPY1]](<4 x s32>)
+    ; CHECK: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY [[CONCAT_VECTORS]](<8 x s32>)
+    %0:_(<4 x s32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3,
+    %1:_(<4 x s32>) = COPY $vgpr4_vgpr5_vgpr6_vgpr7
+    %2:_(<8 x s32>) = G_CONCAT_VECTORS %0, %1
+    $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY %2
+...
+
+---
+name: concat_vectors_v2s32_v2s32_v2s32_v2s32
+
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2_vgpr3, $vgpr4_vgpr5, $vgpr6_vgpr7
+    ; CHECK-LABEL: name: concat_vectors_v2s32_v2s32_v2s32_v2s32
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    ; CHECK: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr2_vgpr3
+    ; CHECK: [[COPY2:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr4_vgpr5
+    ; CHECK: [[COPY3:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr6_vgpr7
+    ; CHECK: [[CONCAT_VECTORS:%[0-9]+]]:_(<8 x s32>) = G_CONCAT_VECTORS [[COPY]](<2 x s32>), [[COPY1]](<2 x s32>), [[COPY2]](<2 x s32>), [[COPY3]](<2 x s32>)
+    ; CHECK: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY [[CONCAT_VECTORS]](<8 x s32>)
+    %0:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    %1:_(<2 x s32>) = COPY $vgpr2_vgpr3
+    %2:_(<2 x s32>) = COPY $vgpr4_vgpr5
+    %3:_(<2 x s32>) = COPY $vgpr6_vgpr7
+
+    %4:_(<8 x s32>) = G_CONCAT_VECTORS %0, %1, %2, %3
+    $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY %4
+...
+---
+name: concat_vectors_v2s64_v2s64
+
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7
+    ; CHECK-LABEL: name: concat_vectors_v2s64_v2s64
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; CHECK: [[COPY1:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr4_vgpr5_vgpr6_vgpr7
+    ; CHECK: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s64>) = G_CONCAT_VECTORS [[COPY]](<2 x s64>), [[COPY1]](<2 x s64>)
+    ; CHECK: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY [[CONCAT_VECTORS]](<4 x s64>)
+    %0:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    %1:_(<2 x s64>) = COPY $vgpr4_vgpr5_vgpr6_vgpr7
+    %2:_(<4 x s64>) = G_CONCAT_VECTORS %0, %1
+    $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY %2
+...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/legalize-constant.mir b/test/CodeGen/AMDGPU/GlobalISel/legalize-constant.mir
index 56da4e3add5c28a265c2249321417ff166cd9a12..a38ef998a9ec714a5656ab2123a10c07b0082409 100644
--- a/test/CodeGen/AMDGPU/GlobalISel/legalize-constant.mir
+++ b/test/CodeGen/AMDGPU/GlobalISel/legalize-constant.mir
@@ -2,59 +2,177 @@
 # RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -O0 -run-pass=legalizer %s -o - | FileCheck %s
 
 ---
-name:            test_constant_i32
+name:            test_constant_s32
 body: |
   bb.0:
 
-    ; CHECK-LABEL: name: test_constant_i32
+    ; CHECK-LABEL: name: test_constant_s32
     ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 5
     ; CHECK: $vgpr0 = COPY [[C]](s32)
     %0:_(s32) = G_CONSTANT i32 5
     $vgpr0 = COPY %0
 ...
 ---
-name:            test_constant_i64
+name:            test_constant_s64
 body: |
   bb.0:
 
-    ; CHECK-LABEL: name: test_constant_i64
+    ; CHECK-LABEL: name: test_constant_s64
     ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 5
     ; CHECK: $vgpr0_vgpr1 = COPY [[C]](s64)
     %0:_(s64) = G_CONSTANT i64 5
     $vgpr0_vgpr1 = COPY %0
 
 ...
+
 ---
-name:            test_fconstant_f32
+name:            test_constant_s1
 body: |
   bb.0:
 
-    ; CHECK-LABEL: name: test_fconstant_f32
-    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_FCONSTANT float 1.000000e+00
-    ; CHECK: $vgpr0 = COPY [[C]](s32)
-    %0:_(s32) = G_FCONSTANT float 1.0
+    ; CHECK-LABEL: name: test_constant_s1
+    ; CHECK: [[C:%[0-9]+]]:_(s1) = G_CONSTANT i1 false
+    ; CHECK: S_ENDPGM implicit [[C]](s1)
+    %1:_(s1) = G_CONSTANT i1 0
+    S_ENDPGM implicit %1
+...
+
+---
+name:            test_constant_s7
+body: |
+  bb.0:
+
+    ; CHECK-LABEL: name: test_constant_s7
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 5
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY [[C]](s32)
+    ; CHECK: $vgpr0 = COPY [[COPY]](s32)
+    %0:_(s7) = G_CONSTANT i7 5
+    %1:_(s32) = G_ANYEXT %0
+    $vgpr0 = COPY %1
+...
+
+---
+name:            test_constant_s8
+body: |
+  bb.0:
+
+    ; CHECK-LABEL: name: test_constant_s8
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 5
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY [[C]](s32)
+    ; CHECK: $vgpr0 = COPY [[COPY]](s32)
+    %0:_(s8) = G_CONSTANT i8 5
+    %1:_(s32) = G_ANYEXT %0
+    $vgpr0 = COPY %1
+...
+
+---
+name:            test_constant_s16
+body: |
+  bb.0:
+
+    ; CHECK-LABEL: name: test_constant_s16
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 5
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY [[C]](s32)
+    ; CHECK: $vgpr0 = COPY [[COPY]](s32)
+    %0:_(s16) = G_CONSTANT i16 5
+    %1:_(s32) = G_ANYEXT %0
+    $vgpr0 = COPY %1
+...
+
+---
+name:            test_constant_s128
+body: |
+  bb.0:
+
+    ; CHECK-LABEL: name: test_constant_s128
+    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 5
+    ; CHECK: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; CHECK: [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[C]](s64), [[C1]](s64)
+    ; CHECK: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[MV]](s128)
+    %0:_(s128) = G_CONSTANT i128 5
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %0
+...
+
+---
+name: test_constant_p0
+body: |
+  bb.0:
+
+    ; CHECK-LABEL: name: test_constant_p0
+    ; CHECK: [[C:%[0-9]+]]:_(p0) = G_CONSTANT i64 0
+    ; CHECK: $vgpr0_vgpr1 = COPY [[C]](p0)
+    %0:_(p0) = G_CONSTANT i64 0
+    $vgpr0_vgpr1 = COPY %0
+...
+
+---
+name: test_constant_p1
+body: |
+  bb.0:
+
+    ; CHECK-LABEL: name: test_constant_p1
+    ; CHECK: [[C:%[0-9]+]]:_(p1) = G_CONSTANT i64 0
+    ; CHECK: $vgpr0_vgpr1 = COPY [[C]](p1)
+    %0:_(p1) = G_CONSTANT i64 0
+    $vgpr0_vgpr1 = COPY %0
+...
+
+---
+name: test_constant_p2
+body: |
+  bb.0:
+
+    ; CHECK-LABEL: name: test_constant_p2
+    ; CHECK: [[C:%[0-9]+]]:_(p2) = G_CONSTANT i32 0
+    ; CHECK: $vgpr0 = COPY [[C]](p2)
+    %0:_(p2) = G_CONSTANT i32 0
     $vgpr0 = COPY %0
 ...
+
 ---
-name:            test_fconstant_f64
+name: test_constant_p3
 body: |
   bb.0:
 
-    ; CHECK-LABEL: name: test_fconstant_f64
-    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_FCONSTANT double 1.000000e+00
-    ; CHECK: $vgpr0_vgpr1 = COPY [[C]](s64)
-    %0:_(s64) = G_FCONSTANT double 1.0
+    ; CHECK-LABEL: name: test_constant_p3
+    ; CHECK: [[C:%[0-9]+]]:_(p2) = G_CONSTANT i32 0
+    ; CHECK: $vgpr0 = COPY [[C]](p2)
+    %0:_(p2) = G_CONSTANT i32 0
+    $vgpr0 = COPY %0
+...
+
+---
+name: test_constant_p4
+body: |
+  bb.0:
+
+    ; CHECK-LABEL: name: test_constant_p4
+    ; CHECK: [[C:%[0-9]+]]:_(p4) = G_CONSTANT i64 0
+    ; CHECK: $vgpr0_vgpr1 = COPY [[C]](p4)
+    %0:_(p4) = G_CONSTANT i64 0
     $vgpr0_vgpr1 = COPY %0
 ...
 
 ---
-name:            test_constant_1
+name: test_constant_p5
 body: |
   bb.0:
 
-    ; CHECK-LABEL: name: test_constant_1
-    ; CHECK: [[C:%[0-9]+]]:_(s1) = G_CONSTANT i1 false
-    ; CHECK: S_ENDPGM implicit [[C]](s1)
-    %1:_(s1) = G_CONSTANT i1 0
-    S_ENDPGM implicit %1
+    ; CHECK-LABEL: name: test_constant_p5
+    ; CHECK: [[C:%[0-9]+]]:_(p5) = G_CONSTANT i32 0
+    ; CHECK: $vgpr0 = COPY [[C]](p5)
+    %0:_(p5) = G_CONSTANT i32 0
+    $vgpr0 = COPY %0
+...
+
+---
+name: test_constant_p999
+body: |
+  bb.0:
+
+    ; CHECK-LABEL: name: test_constant_p999
+    ; CHECK: [[C:%[0-9]+]]:_(p999) = G_CONSTANT i64 0
+    ; CHECK: $vgpr0_vgpr1 = COPY [[C]](p999)
+    %0:_(p999) = G_CONSTANT i64 0
+    $vgpr0_vgpr1 = COPY %0
 ...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/legalize-ctlz-zero-undef.mir b/test/CodeGen/AMDGPU/GlobalISel/legalize-ctlz-zero-undef.mir
new file mode 100644
index 0000000000000000000000000000000000000000..aa01879f7bf6f8c61cfaecd41f9308ca07a23d9e
--- /dev/null
+++ b/test/CodeGen/AMDGPU/GlobalISel/legalize-ctlz-zero-undef.mir
@@ -0,0 +1,93 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -O0 -run-pass=legalizer %s -o - | FileCheck %s
+
+---
+name: ctlz_zero_undef_s32_s32
+
+body: |
+  bb.0:
+    liveins: $vgpr0
+    ; CHECK-LABEL: name: ctlz_zero_undef_s32_s32
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[CTLZ_ZERO_UNDEF:%[0-9]+]]:_(s32) = G_CTLZ_ZERO_UNDEF [[COPY]](s32)
+    ; CHECK: $vgpr0 = COPY [[CTLZ_ZERO_UNDEF]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = G_CTLZ_ZERO_UNDEF %0
+    $vgpr0 = COPY %1
+...
+
+---
+name: ctlz_zero_undef_s32_s64
+
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+    ; CHECK-LABEL: name: ctlz_zero_undef_s32_s64
+    ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
+    ; CHECK: [[CTLZ_ZERO_UNDEF:%[0-9]+]]:_(s32) = G_CTLZ_ZERO_UNDEF [[COPY]](s64)
+    ; CHECK: $vgpr0 = COPY [[CTLZ_ZERO_UNDEF]](s32)
+    %0:_(s64) = COPY $vgpr0_vgpr1
+    %1:_(s32) = G_CTLZ_ZERO_UNDEF %0
+    $vgpr0 = COPY %1
+...
+
+---
+name: ctlz_zero_undef_s64_s64
+
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+    ; CHECK-LABEL: name: ctlz_zero_undef_s64_s64
+    ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
+    ; CHECK: [[CTLZ_ZERO_UNDEF:%[0-9]+]]:_(s32) = G_CTLZ_ZERO_UNDEF [[COPY]](s64)
+    ; CHECK: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[CTLZ_ZERO_UNDEF]](s32)
+    ; CHECK: $vgpr0_vgpr1 = COPY [[ZEXT]](s64)
+    %0:_(s64) = COPY $vgpr0_vgpr1
+    %1:_(s64) = G_CTLZ_ZERO_UNDEF %0
+    $vgpr0_vgpr1 = COPY %1
+...
+
+---
+name: ctlz_zero_undef_s16_s32
+
+body: |
+  bb.0:
+    liveins: $vgpr0
+    ; CHECK-LABEL: name: ctlz_zero_undef_s16_s32
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[CTLZ_ZERO_UNDEF:%[0-9]+]]:_(s32) = G_CTLZ_ZERO_UNDEF [[COPY]](s32)
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY [[CTLZ_ZERO_UNDEF]](s32)
+    ; CHECK: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
+    ; CHECK: $vgpr0 = COPY [[AND]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s16) = G_CTLZ_ZERO_UNDEF %0
+    %2:_(s32) = G_ZEXT %1
+    $vgpr0 = COPY %2
+...
+
+---
+name: ctlz_zero_undef_s16_s16
+
+body: |
+  bb.0:
+    liveins: $vgpr0
+    ; CHECK-LABEL: name: ctlz_zero_undef_s16_s16
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; CHECK: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
+    ; CHECK: [[CTLZ_ZERO_UNDEF:%[0-9]+]]:_(s32) = G_CTLZ_ZERO_UNDEF [[AND]](s32)
+    ; CHECK: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; CHECK: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[CTLZ_ZERO_UNDEF]], [[C1]]
+    ; CHECK: [[COPY2:%[0-9]+]]:_(s32) = COPY [[SUB]](s32)
+    ; CHECK: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; CHECK: [[COPY3:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+    ; CHECK: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[C2]]
+    ; CHECK: $vgpr0 = COPY [[AND1]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s16) = G_TRUNC %0
+    %2:_(s16) = G_CTLZ_ZERO_UNDEF %1
+    %3:_(s32) = G_ZEXT %2
+    $vgpr0 = COPY %3
+...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/legalize-ctlz.mir b/test/CodeGen/AMDGPU/GlobalISel/legalize-ctlz.mir
new file mode 100644
index 0000000000000000000000000000000000000000..acdfc3abd5751fe6715afd4b037b9e6d4db61c33
--- /dev/null
+++ b/test/CodeGen/AMDGPU/GlobalISel/legalize-ctlz.mir
@@ -0,0 +1,93 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -O0 -run-pass=legalizer %s -o - | FileCheck %s
+
+---
+name: ctlz_s32_s32
+
+body: |
+  bb.0:
+    liveins: $vgpr0
+    ; CHECK-LABEL: name: ctlz_s32_s32
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[CTLZ:%[0-9]+]]:_(s32) = G_CTLZ [[COPY]](s32)
+    ; CHECK: $vgpr0 = COPY [[CTLZ]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = G_CTLZ %0
+    $vgpr0 = COPY %1
+...
+
+---
+name: ctlz_s32_s64
+
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+    ; CHECK-LABEL: name: ctlz_s32_s64
+    ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
+    ; CHECK: [[CTLZ:%[0-9]+]]:_(s32) = G_CTLZ [[COPY]](s64)
+    ; CHECK: $vgpr0 = COPY [[CTLZ]](s32)
+    %0:_(s64) = COPY $vgpr0_vgpr1
+    %1:_(s32) = G_CTLZ %0
+    $vgpr0 = COPY %1
+...
+
+---
+name: ctlz_s64_s64
+
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+    ; CHECK-LABEL: name: ctlz_s64_s64
+    ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
+    ; CHECK: [[CTLZ:%[0-9]+]]:_(s32) = G_CTLZ [[COPY]](s64)
+    ; CHECK: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[CTLZ]](s32)
+    ; CHECK: $vgpr0_vgpr1 = COPY [[ZEXT]](s64)
+    %0:_(s64) = COPY $vgpr0_vgpr1
+    %1:_(s64) = G_CTLZ %0
+    $vgpr0_vgpr1 = COPY %1
+...
+
+---
+name: ctlz_s16_s32
+
+body: |
+  bb.0:
+    liveins: $vgpr0
+    ; CHECK-LABEL: name: ctlz_s16_s32
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[CTLZ:%[0-9]+]]:_(s32) = G_CTLZ [[COPY]](s32)
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY [[CTLZ]](s32)
+    ; CHECK: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
+    ; CHECK: $vgpr0 = COPY [[AND]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s16) = G_CTLZ %0
+    %2:_(s32) = G_ZEXT %1
+    $vgpr0 = COPY %2
+...
+
+---
+name: ctlz_s16_s16
+
+body: |
+  bb.0:
+    liveins: $vgpr0
+    ; CHECK-LABEL: name: ctlz_s16_s16
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; CHECK: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
+    ; CHECK: [[CTLZ:%[0-9]+]]:_(s32) = G_CTLZ [[AND]](s32)
+    ; CHECK: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; CHECK: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[CTLZ]], [[C1]]
+    ; CHECK: [[COPY2:%[0-9]+]]:_(s32) = COPY [[SUB]](s32)
+    ; CHECK: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; CHECK: [[COPY3:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+    ; CHECK: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[C2]]
+    ; CHECK: $vgpr0 = COPY [[AND1]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s16) = G_TRUNC %0
+    %2:_(s16) = G_CTLZ %1
+    %3:_(s32) = G_ZEXT %2
+    $vgpr0 = COPY %3
+...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/legalize-ctpop.mir b/test/CodeGen/AMDGPU/GlobalISel/legalize-ctpop.mir
new file mode 100644
index 0000000000000000000000000000000000000000..d0a272ea233937574c9c26e313ced028bc7f429e
--- /dev/null
+++ b/test/CodeGen/AMDGPU/GlobalISel/legalize-ctpop.mir
@@ -0,0 +1,91 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -O0 -run-pass=legalizer %s -o - | FileCheck %s
+
+---
+name: ctpop_s32_s32
+
+body: |
+  bb.0:
+    liveins: $vgpr0
+    ; CHECK-LABEL: name: ctpop_s32_s32
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[CTPOP:%[0-9]+]]:_(s32) = G_CTPOP [[COPY]](s32)
+    ; CHECK: $vgpr0 = COPY [[CTPOP]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = G_CTPOP %0
+    $vgpr0 = COPY %1
+...
+
+---
+name: ctpop_s32_s64
+
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+    ; CHECK-LABEL: name: ctpop_s32_s64
+    ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
+    ; CHECK: [[CTPOP:%[0-9]+]]:_(s32) = G_CTPOP [[COPY]](s64)
+    ; CHECK: $vgpr0 = COPY [[CTPOP]](s32)
+    %0:_(s64) = COPY $vgpr0_vgpr1
+    %1:_(s32) = G_CTPOP %0
+    $vgpr0 = COPY %1
+...
+
+---
+name: ctpop_s64_s64
+
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+    ; CHECK-LABEL: name: ctpop_s64_s64
+    ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
+    ; CHECK: [[CTPOP:%[0-9]+]]:_(s32) = G_CTPOP [[COPY]](s64)
+    ; CHECK: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[CTPOP]](s32)
+    ; CHECK: $vgpr0_vgpr1 = COPY [[ZEXT]](s64)
+    %0:_(s64) = COPY $vgpr0_vgpr1
+    %1:_(s64) = G_CTPOP %0
+    $vgpr0_vgpr1 = COPY %1
+...
+
+---
+name: ctpop_s16_s32
+
+body: |
+  bb.0:
+    liveins: $vgpr0
+    ; CHECK-LABEL: name: ctpop_s16_s32
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[CTPOP:%[0-9]+]]:_(s32) = G_CTPOP [[COPY]](s32)
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY [[CTPOP]](s32)
+    ; CHECK: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
+    ; CHECK: $vgpr0 = COPY [[AND]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s16) = G_CTPOP %0
+    %2:_(s32) = G_ZEXT %1
+    $vgpr0 = COPY %2
+...
+
+---
+name: ctpop_s16_s16
+
+body: |
+  bb.0:
+    liveins: $vgpr0
+    ; CHECK-LABEL: name: ctpop_s16_s16
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; CHECK: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
+    ; CHECK: [[CTPOP:%[0-9]+]]:_(s32) = G_CTPOP [[AND]](s32)
+    ; CHECK: [[COPY2:%[0-9]+]]:_(s32) = COPY [[CTPOP]](s32)
+    ; CHECK: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; CHECK: [[COPY3:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+    ; CHECK: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[C1]]
+    ; CHECK: $vgpr0 = COPY [[AND1]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s16) = G_TRUNC %0
+    %2:_(s16) = G_CTPOP %1
+    %3:_(s32) = G_ZEXT %2
+    $vgpr0 = COPY %3
+...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/legalize-cttz-zero-undef.mir b/test/CodeGen/AMDGPU/GlobalISel/legalize-cttz-zero-undef.mir
new file mode 100644
index 0000000000000000000000000000000000000000..cb474c59ee101bf9a0e7c80e80b93c2946de8adf
--- /dev/null
+++ b/test/CodeGen/AMDGPU/GlobalISel/legalize-cttz-zero-undef.mir
@@ -0,0 +1,91 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -O0 -run-pass=legalizer %s -o - | FileCheck %s
+
+---
+name: cttz_zero_undef_s32_s32
+
+body: |
+  bb.0:
+    liveins: $vgpr0
+    ; CHECK-LABEL: name: cttz_zero_undef_s32_s32
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[CTTZ_ZERO_UNDEF:%[0-9]+]]:_(s32) = G_CTTZ_ZERO_UNDEF [[COPY]](s32)
+    ; CHECK: $vgpr0 = COPY [[CTTZ_ZERO_UNDEF]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = G_CTTZ_ZERO_UNDEF %0
+    $vgpr0 = COPY %1
+...
+
+---
+name: cttz_zero_undef_s32_s64
+
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+    ; CHECK-LABEL: name: cttz_zero_undef_s32_s64
+    ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
+    ; CHECK: [[CTTZ_ZERO_UNDEF:%[0-9]+]]:_(s32) = G_CTTZ_ZERO_UNDEF [[COPY]](s64)
+    ; CHECK: $vgpr0 = COPY [[CTTZ_ZERO_UNDEF]](s32)
+    %0:_(s64) = COPY $vgpr0_vgpr1
+    %1:_(s32) = G_CTTZ_ZERO_UNDEF %0
+    $vgpr0 = COPY %1
+...
+
+---
+name: cttz_zero_undef_s64_s64
+
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+    ; CHECK-LABEL: name: cttz_zero_undef_s64_s64
+    ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
+    ; CHECK: [[CTTZ_ZERO_UNDEF:%[0-9]+]]:_(s32) = G_CTTZ_ZERO_UNDEF [[COPY]](s64)
+    ; CHECK: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[CTTZ_ZERO_UNDEF]](s32)
+    ; CHECK: $vgpr0_vgpr1 = COPY [[ZEXT]](s64)
+    %0:_(s64) = COPY $vgpr0_vgpr1
+    %1:_(s64) = G_CTTZ_ZERO_UNDEF %0
+    $vgpr0_vgpr1 = COPY %1
+...
+
+---
+name: cttz_zero_undef_s16_s32
+
+body: |
+  bb.0:
+    liveins: $vgpr0
+    ; CHECK-LABEL: name: cttz_zero_undef_s16_s32
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[CTTZ_ZERO_UNDEF:%[0-9]+]]:_(s32) = G_CTTZ_ZERO_UNDEF [[COPY]](s32)
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY [[CTTZ_ZERO_UNDEF]](s32)
+    ; CHECK: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
+    ; CHECK: $vgpr0 = COPY [[AND]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s16) = G_CTTZ_ZERO_UNDEF %0
+    %2:_(s32) = G_ZEXT %1
+    $vgpr0 = COPY %2
+...
+
+---
+name: cttz_zero_undef_s16_s16
+
+body: |
+  bb.0:
+    liveins: $vgpr0
+    ; CHECK-LABEL: name: cttz_zero_undef_s16_s16
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; CHECK: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
+    ; CHECK: [[CTTZ_ZERO_UNDEF:%[0-9]+]]:_(s32) = G_CTTZ_ZERO_UNDEF [[AND]](s32)
+    ; CHECK: [[COPY2:%[0-9]+]]:_(s32) = COPY [[CTTZ_ZERO_UNDEF]](s32)
+    ; CHECK: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; CHECK: [[COPY3:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+    ; CHECK: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[C1]]
+    ; CHECK: $vgpr0 = COPY [[AND1]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s16) = G_TRUNC %0
+    %2:_(s16) = G_CTTZ_ZERO_UNDEF %1
+    %3:_(s32) = G_ZEXT %2
+    $vgpr0 = COPY %3
+...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/legalize-cttz.mir b/test/CodeGen/AMDGPU/GlobalISel/legalize-cttz.mir
new file mode 100644
index 0000000000000000000000000000000000000000..66e5951198f834f816692fcea5b7f531542507f8
--- /dev/null
+++ b/test/CodeGen/AMDGPU/GlobalISel/legalize-cttz.mir
@@ -0,0 +1,93 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -O0 -run-pass=legalizer %s -o - | FileCheck %s
+
+---
+name: cttz_s32_s32
+
+body: |
+  bb.0:
+    liveins: $vgpr0
+    ; CHECK-LABEL: name: cttz_s32_s32
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[CTTZ:%[0-9]+]]:_(s32) = G_CTTZ [[COPY]](s32)
+    ; CHECK: $vgpr0 = COPY [[CTTZ]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = G_CTTZ %0
+    $vgpr0 = COPY %1
+...
+
+---
+name: cttz_s32_s64
+
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+    ; CHECK-LABEL: name: cttz_s32_s64
+    ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
+    ; CHECK: [[CTTZ:%[0-9]+]]:_(s32) = G_CTTZ [[COPY]](s64)
+    ; CHECK: $vgpr0 = COPY [[CTTZ]](s32)
+    %0:_(s64) = COPY $vgpr0_vgpr1
+    %1:_(s32) = G_CTTZ %0
+    $vgpr0 = COPY %1
+...
+
+---
+name: cttz_s64_s64
+
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+    ; CHECK-LABEL: name: cttz_s64_s64
+    ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
+    ; CHECK: [[CTTZ:%[0-9]+]]:_(s32) = G_CTTZ [[COPY]](s64)
+    ; CHECK: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[CTTZ]](s32)
+    ; CHECK: $vgpr0_vgpr1 = COPY [[ZEXT]](s64)
+    %0:_(s64) = COPY $vgpr0_vgpr1
+    %1:_(s64) = G_CTTZ %0
+    $vgpr0_vgpr1 = COPY %1
+...
+
+---
+name: cttz_s16_s32
+
+body: |
+  bb.0:
+    liveins: $vgpr0
+    ; CHECK-LABEL: name: cttz_s16_s32
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[CTTZ:%[0-9]+]]:_(s32) = G_CTTZ [[COPY]](s32)
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY [[CTTZ]](s32)
+    ; CHECK: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
+    ; CHECK: $vgpr0 = COPY [[AND]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s16) = G_CTTZ %0
+    %2:_(s32) = G_ZEXT %1
+    $vgpr0 = COPY %2
+...
+
+---
+name: cttz_s16_s16
+
+body: |
+  bb.0:
+    liveins: $vgpr0
+    ; CHECK-LABEL: name: cttz_s16_s16
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; CHECK: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
+    ; CHECK: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65536
+    ; CHECK: [[OR:%[0-9]+]]:_(s32) = G_OR [[AND]], [[C1]]
+    ; CHECK: [[CTTZ:%[0-9]+]]:_(s32) = G_CTTZ [[OR]](s32)
+    ; CHECK: [[COPY2:%[0-9]+]]:_(s32) = COPY [[CTTZ]](s32)
+    ; CHECK: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; CHECK: [[COPY3:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+    ; CHECK: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[C2]]
+    ; CHECK: $vgpr0 = COPY [[AND1]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s16) = G_TRUNC %0
+    %2:_(s16) = G_CTTZ %1
+    %3:_(s32) = G_ZEXT %2
+    $vgpr0 = COPY %3
+...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/legalize-extract-vector-elt.mir b/test/CodeGen/AMDGPU/GlobalISel/legalize-extract-vector-elt.mir
index ec609f296391f119fa0eb49a2be7235a95e43279..cc8471a39c4d569aee272b73ce2c999608fd498a 100644
--- a/test/CodeGen/AMDGPU/GlobalISel/legalize-extract-vector-elt.mir
+++ b/test/CodeGen/AMDGPU/GlobalISel/legalize-extract-vector-elt.mir
@@ -1,5 +1,5 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -run-pass=legalizer %s -o - | FileCheck %s
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -run-pass=legalizer -global-isel-abort=0 %s -o - | FileCheck %s
 
 ---
 name: extract_vector_elt_0_v2i32
@@ -58,9 +58,9 @@ body: |
     liveins: $vgpr0
     ; CHECK-LABEL: name: extract_vector_elt_0_v5i32
     ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
-    ; CHECK: [[MV:%[0-9]+]]:_(<5 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY]](s32), [[COPY]](s32), [[COPY]](s32), [[COPY]](s32)
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<5 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY]](s32), [[COPY]](s32), [[COPY]](s32), [[COPY]](s32)
     ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; CHECK: [[EVEC:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[MV]](<5 x s32>), [[C]](s32)
+    ; CHECK: [[EVEC:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[BUILD_VECTOR]](<5 x s32>), [[C]](s32)
     ; CHECK: $vgpr0 = COPY [[EVEC]](s32)
     %0:_(s32) = COPY $vgpr0
     %1:_(<5 x s32>) = G_BUILD_VECTOR %0, %0, %0, %0, %0
@@ -77,9 +77,9 @@ body: |
     liveins: $vgpr0
     ; CHECK-LABEL: name: extract_vector_elt_0_v6i32
     ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
-    ; CHECK: [[MV:%[0-9]+]]:_(<6 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY]](s32), [[COPY]](s32), [[COPY]](s32), [[COPY]](s32), [[COPY]](s32)
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<6 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY]](s32), [[COPY]](s32), [[COPY]](s32), [[COPY]](s32), [[COPY]](s32)
     ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; CHECK: [[EVEC:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[MV]](<6 x s32>), [[C]](s32)
+    ; CHECK: [[EVEC:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[BUILD_VECTOR]](<6 x s32>), [[C]](s32)
     ; CHECK: $vgpr0 = COPY [[EVEC]](s32)
     %0:_(s32) = COPY $vgpr0
     %1:_(<6 x s32>) = G_BUILD_VECTOR %0, %0, %0, %0, %0, %0
@@ -96,9 +96,9 @@ body: |
     liveins: $vgpr0
     ; CHECK-LABEL: name: extract_vector_elt_0_v7i32
     ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
-    ; CHECK: [[MV:%[0-9]+]]:_(<7 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY]](s32), [[COPY]](s32), [[COPY]](s32), [[COPY]](s32), [[COPY]](s32), [[COPY]](s32)
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<7 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY]](s32), [[COPY]](s32), [[COPY]](s32), [[COPY]](s32), [[COPY]](s32), [[COPY]](s32)
     ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; CHECK: [[EVEC:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[MV]](<7 x s32>), [[C]](s32)
+    ; CHECK: [[EVEC:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[BUILD_VECTOR]](<7 x s32>), [[C]](s32)
     ; CHECK: $vgpr0 = COPY [[EVEC]](s32)
     %0:_(s32) = COPY $vgpr0
     %1:_(<7 x s32>) = G_BUILD_VECTOR %0, %0, %0, %0, %0, %0, %0
@@ -115,9 +115,9 @@ body: |
     liveins: $vgpr0
     ; CHECK-LABEL: name: extract_vector_elt_0_v8i32
     ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
-    ; CHECK: [[MV:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY]](s32), [[COPY]](s32), [[COPY]](s32), [[COPY]](s32), [[COPY]](s32), [[COPY]](s32), [[COPY]](s32)
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY]](s32), [[COPY]](s32), [[COPY]](s32), [[COPY]](s32), [[COPY]](s32), [[COPY]](s32), [[COPY]](s32)
     ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; CHECK: [[EVEC:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[MV]](<8 x s32>), [[C]](s32)
+    ; CHECK: [[EVEC:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[BUILD_VECTOR]](<8 x s32>), [[C]](s32)
     ; CHECK: $vgpr0 = COPY [[EVEC]](s32)
     %0:_(s32) = COPY $vgpr0
     %1:_(<8 x s32>) = G_BUILD_VECTOR %0, %0, %0, %0, %0, %0, %0, %0
@@ -134,9 +134,9 @@ body: |
     liveins: $vgpr0
     ; CHECK-LABEL: name: extract_vector_elt_0_v16i32
     ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
-    ; CHECK: [[MV:%[0-9]+]]:_(<16 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY]](s32), [[COPY]](s32), [[COPY]](s32), [[COPY]](s32), [[COPY]](s32), [[COPY]](s32), [[COPY]](s32), [[COPY]](s32), [[COPY]](s32), [[COPY]](s32), [[COPY]](s32), [[COPY]](s32), [[COPY]](s32), [[COPY]](s32), [[COPY]](s32)
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<16 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY]](s32), [[COPY]](s32), [[COPY]](s32), [[COPY]](s32), [[COPY]](s32), [[COPY]](s32), [[COPY]](s32), [[COPY]](s32), [[COPY]](s32), [[COPY]](s32), [[COPY]](s32), [[COPY]](s32), [[COPY]](s32), [[COPY]](s32), [[COPY]](s32)
     ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; CHECK: [[EVEC:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[MV]](<16 x s32>), [[C]](s32)
+    ; CHECK: [[EVEC:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[BUILD_VECTOR]](<16 x s32>), [[C]](s32)
     ; CHECK: $vgpr0 = COPY [[EVEC]](s32)
     %0:_(s32) = COPY $vgpr0
     %1:_(<16 x s32>) = G_BUILD_VECTOR %0, %0, %0, %0, %0, %0, %0, %0, %0, %0, %0, %0, %0, %0, %0, %0
@@ -178,3 +178,105 @@ body: |
     %2:_(s32) = G_EXTRACT_VECTOR_ELT %0, %1
     $vgpr0 = COPY %2
 ...
+
+
+---
+name: extract_vector_elt_0_v2i8_i32
+
+body: |
+  bb.0:
+
+    ; CHECK-LABEL: name: extract_vector_elt_0_v2i8_i32
+    ; CHECK: [[DEF:%[0-9]+]]:_(<2 x s32>) = G_IMPLICIT_DEF
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; CHECK: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY [[DEF]](<2 x s32>)
+    ; CHECK: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
+    ; CHECK: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[UV]], [[C1]](s32)
+    ; CHECK: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[UV1]], [[C1]](s32)
+    ; CHECK: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C1]](s32)
+    ; CHECK: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[C1]](s32)
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[ASHR]](s32), [[ASHR1]](s32)
+    ; CHECK: [[EVEC:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[BUILD_VECTOR]](<2 x s32>), [[C]](s32)
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY [[EVEC]](s32)
+    ; CHECK: $vgpr0 = COPY [[COPY1]](s32)
+    %0:_(<2 x s8>) = G_IMPLICIT_DEF
+    %1:_(s32) = G_CONSTANT i32 0
+    %2:_(s8) = G_EXTRACT_VECTOR_ELT %0, %1
+    %3:_(s32) = G_ANYEXT %2
+    $vgpr0 = COPY %3
+...
+
+---
+name: extract_vector_elt_0_v2i16_i32
+
+body: |
+  bb.0:
+
+    ; CHECK-LABEL: name: extract_vector_elt_0_v2i16_i32
+    ; CHECK: [[DEF:%[0-9]+]]:_(<2 x s16>) = G_IMPLICIT_DEF
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; CHECK: [[EVEC:%[0-9]+]]:_(s16) = G_EXTRACT_VECTOR_ELT [[DEF]](<2 x s16>), [[C]](s32)
+    ; CHECK: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[EVEC]](s16)
+    ; CHECK: $vgpr0 = COPY [[ANYEXT]](s32)
+    %0:_(<2 x s16>) = G_IMPLICIT_DEF
+    %1:_(s32) = G_CONSTANT i32 0
+    %2:_(s16) = G_EXTRACT_VECTOR_ELT %0, %1
+    %3:_(s32) = G_ANYEXT %2
+    $vgpr0 = COPY %3
+...
+
+---
+name: extract_vector_elt_0_v2i1_i32
+
+body: |
+  bb.0:
+
+    ; CHECK-LABEL: name: extract_vector_elt_0_v2i1_i32
+    ; CHECK: [[DEF:%[0-9]+]]:_(<2 x s32>) = G_IMPLICIT_DEF
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; CHECK: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 31
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY [[DEF]](<2 x s32>)
+    ; CHECK: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
+    ; CHECK: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[UV]], [[C1]](s32)
+    ; CHECK: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[UV1]], [[C1]](s32)
+    ; CHECK: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C1]](s32)
+    ; CHECK: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[C1]](s32)
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[ASHR]](s32), [[ASHR1]](s32)
+    ; CHECK: [[EVEC:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[BUILD_VECTOR]](<2 x s32>), [[C]](s32)
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY [[EVEC]](s32)
+    ; CHECK: $vgpr0 = COPY [[COPY1]](s32)
+    %0:_(<2 x s1>) = G_IMPLICIT_DEF
+    %1:_(s32) = G_CONSTANT i32 0
+    %2:_(s1) = G_EXTRACT_VECTOR_ELT %0, %1
+    %3:_(s32) = G_ANYEXT %2
+    $vgpr0 = COPY %3
+...
+
+---
+name: extract_vector_elt_0_v2i1_i1
+
+body: |
+  bb.0:
+
+    ; CHECK-LABEL: name: extract_vector_elt_0_v2i1_i1
+    ; CHECK: [[DEF:%[0-9]+]]:_(<2 x s32>) = G_IMPLICIT_DEF
+    ; CHECK: [[C:%[0-9]+]]:_(s1) = G_CONSTANT i1 false
+    ; CHECK: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 31
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY [[DEF]](<2 x s32>)
+    ; CHECK: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
+    ; CHECK: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[UV]], [[C1]](s32)
+    ; CHECK: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[UV1]], [[C1]](s32)
+    ; CHECK: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C1]](s32)
+    ; CHECK: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SHL1]], [[C1]](s32)
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[ASHR]](s32), [[ASHR1]](s32)
+    ; CHECK: [[SEXT:%[0-9]+]]:_(s32) = G_SEXT [[C]](s1)
+    ; CHECK: [[EVEC:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[BUILD_VECTOR]](<2 x s32>), [[SEXT]](s32)
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY [[EVEC]](s32)
+    ; CHECK: $vgpr0 = COPY [[COPY1]](s32)
+    %0:_(<2 x s1>) = G_IMPLICIT_DEF
+    %1:_(s1) = G_CONSTANT i1 false
+    %2:_(s1) = G_EXTRACT_VECTOR_ELT %0, %1
+    %3:_(s32) = G_ANYEXT %2
+    $vgpr0 = COPY %3
+...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/legalize-extract.mir b/test/CodeGen/AMDGPU/GlobalISel/legalize-extract.mir
index 3aa27eeff6caf9232f4ffb2b496517b8f612de4f..c049231a3a436bbae9b7b6e6bd3deb50ca5c91b4 100644
--- a/test/CodeGen/AMDGPU/GlobalISel/legalize-extract.mir
+++ b/test/CodeGen/AMDGPU/GlobalISel/legalize-extract.mir
@@ -1,105 +1,525 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -run-pass=legalizer %s -o - | FileCheck %s
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -run-pass=legalizer -global-isel-abort=0 %s -o - | FileCheck %s
 
 ---
-name: test_extract_lo32_i64
+name: test_extract_s32_s64_offset0
 body: |
-  bb.0.entry:
+  bb.0:
     liveins: $vgpr0_vgpr1
 
-    ; CHECK-LABEL: name: test_extract_lo32_i64
+    ; CHECK-LABEL: name: test_extract_s32_s64_offset0
     ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
     ; CHECK: [[EXTRACT:%[0-9]+]]:_(s32) = G_EXTRACT [[COPY]](s64), 0
-    ; CHECK: S_ENDPGM implicit [[EXTRACT]](s32)
+    ; CHECK: $vgpr0 = COPY [[EXTRACT]](s32)
     %0:_(s64) = COPY $vgpr0_vgpr1
     %1:_(s32) = G_EXTRACT %0, 0
-    S_ENDPGM implicit %1
+    $vgpr0 = COPY %1
 ...
 ---
-name: test_extract_hi32_i64
+name: test_extract_s32_s64_offset32
 body: |
-  bb.0.entry:
+  bb.0:
     liveins: $vgpr0_vgpr1
 
-    ; CHECK-LABEL: name: test_extract_hi32_i64
+    ; CHECK-LABEL: name: test_extract_s32_s64_offset32
     ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
     ; CHECK: [[EXTRACT:%[0-9]+]]:_(s32) = G_EXTRACT [[COPY]](s64), 32
-    ; CHECK: S_ENDPGM implicit [[EXTRACT]](s32)
+    ; CHECK: $vgpr0 = COPY [[EXTRACT]](s32)
     %0:_(s64) = COPY $vgpr0_vgpr1
     %1:_(s32) = G_EXTRACT %0, 32
-     S_ENDPGM implicit %1
+     $vgpr0 = COPY %1
 ...
-
 ---
-name: test_extract_dword0_i128
+name: test_extract_s32_s96_offset0
 body: |
-  bb.0.entry:
-    liveins: $vgpr0_vgpr1
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2
 
-    ; CHECK-LABEL: name: test_extract_dword0_i128
-    ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
-    ; CHECK: [[EXTRACT:%[0-9]+]]:_(s32) = G_EXTRACT [[COPY]](s64), 0
-    ; CHECK: S_ENDPGM implicit [[EXTRACT]](s32)
-    %0:_(s64) = COPY $vgpr0_vgpr1
+    ; CHECK-LABEL: name: test_extract_s32_s96_offset0
+    ; CHECK: [[COPY:%[0-9]+]]:_(s96) = COPY $vgpr0_vgpr1_vgpr2
+    ; CHECK: [[EXTRACT:%[0-9]+]]:_(s32) = G_EXTRACT [[COPY]](s96), 0
+    ; CHECK: $vgpr0 = COPY [[EXTRACT]](s32)
+    %0:_(s96) = COPY $vgpr0_vgpr1_vgpr2
     %1:_(s32) = G_EXTRACT %0, 0
-    S_ENDPGM implicit %1
+    $vgpr0 = COPY %1
 ...
+---
+name: test_extract_s32_s96_offset32
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2
 
+    ; CHECK-LABEL: name: test_extract_s32_s96_offset32
+    ; CHECK: [[COPY:%[0-9]+]]:_(s96) = COPY $vgpr0_vgpr1_vgpr2
+    ; CHECK: [[EXTRACT:%[0-9]+]]:_(s32) = G_EXTRACT [[COPY]](s96), 32
+    ; CHECK: $vgpr0 = COPY [[EXTRACT]](s32)
+    %0:_(s96) = COPY $vgpr0_vgpr1_vgpr2
+    %1:_(s32) = G_EXTRACT %0, 32
+    $vgpr0 = COPY %1
+...
 ---
-name: test_extract_dword1_i128
+name: test_extract_s32_s96_offset64
 body: |
-  bb.0.entry:
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2
+
+    ; CHECK-LABEL: name: test_extract_s32_s96_offset64
+    ; CHECK: [[COPY:%[0-9]+]]:_(s96) = COPY $vgpr0_vgpr1_vgpr2
+    ; CHECK: [[EXTRACT:%[0-9]+]]:_(s32) = G_EXTRACT [[COPY]](s96), 64
+    ; CHECK: $vgpr0 = COPY [[EXTRACT]](s32)
+    %0:_(s96) = COPY $vgpr0_vgpr1_vgpr2
+    %1:_(s32) = G_EXTRACT %0, 64
+    $vgpr0 = COPY %1
+...
+---
+name: test_extract_s32_s128_offset0
+body: |
+  bb.0:
     liveins: $vgpr0_vgpr1_vgpr2_vgpr3
 
-    ; CHECK-LABEL: name: test_extract_dword1_i128
+    ; CHECK-LABEL: name: test_extract_s32_s128_offset0
+    ; CHECK: [[COPY:%[0-9]+]]:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; CHECK: [[EXTRACT:%[0-9]+]]:_(s32) = G_EXTRACT [[COPY]](s128), 0
+    ; CHECK: $vgpr0 = COPY [[EXTRACT]](s32)
+    %0:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    %1:_(s32) = G_EXTRACT %0, 0
+    $vgpr0 = COPY %1
+...
+---
+name: test_extract_s32_s128_offset32
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3
+
+    ; CHECK-LABEL: name: test_extract_s32_s128_offset32
     ; CHECK: [[COPY:%[0-9]+]]:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
     ; CHECK: [[EXTRACT:%[0-9]+]]:_(s32) = G_EXTRACT [[COPY]](s128), 32
-    ; CHECK: S_ENDPGM implicit [[EXTRACT]](s32)
+    ; CHECK: $vgpr0 = COPY [[EXTRACT]](s32)
     %0:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
     %1:_(s32) = G_EXTRACT %0, 32
-    S_ENDPGM implicit %1
+    $vgpr0 = COPY %1
 ...
-
 ---
-name: test_extract_dword2_i128
+name: test_extract_s32_s128_offset64
 body: |
-  bb.0.entry:
+  bb.0:
     liveins: $vgpr0_vgpr1_vgpr2_vgpr3
 
-    ; CHECK-LABEL: name: test_extract_dword2_i128
+    ; CHECK-LABEL: name: test_extract_s32_s128_offset64
     ; CHECK: [[COPY:%[0-9]+]]:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
     ; CHECK: [[EXTRACT:%[0-9]+]]:_(s32) = G_EXTRACT [[COPY]](s128), 64
-    ; CHECK: S_ENDPGM implicit [[EXTRACT]](s32)
+    ; CHECK: $vgpr0 = COPY [[EXTRACT]](s32)
     %0:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
     %1:_(s32) = G_EXTRACT %0, 64
-    S_ENDPGM implicit %1
+    $vgpr0 = COPY %1
 ...
-
 ---
-name: test_extract_dword3_i128
+name: test_extract_s32_s128_offset96
 body: |
-  bb.0.entry:
+  bb.0:
     liveins: $vgpr0_vgpr1_vgpr2_vgpr3
 
-    ; CHECK-LABEL: name: test_extract_dword3_i128
+    ; CHECK-LABEL: name: test_extract_s32_s128_offset96
     ; CHECK: [[COPY:%[0-9]+]]:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
     ; CHECK: [[EXTRACT:%[0-9]+]]:_(s32) = G_EXTRACT [[COPY]](s128), 96
-    ; CHECK: S_ENDPGM implicit [[EXTRACT]](s32)
+    ; CHECK: $vgpr0 = COPY [[EXTRACT]](s32)
     %0:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
     %1:_(s32) = G_EXTRACT %0, 96
-     S_ENDPGM implicit %1
+    $vgpr0 = COPY %1
 ...
+
 ---
-name: test_extract_dword2_i96
+name: test_extract_s32_v2s32_offset0
 body: |
-  bb.0.entry:
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; CHECK-LABEL: name: test_extract_s32_v2s32_offset0
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    ; CHECK: [[EXTRACT:%[0-9]+]]:_(s32) = G_EXTRACT [[COPY]](<2 x s32>), 0
+    ; CHECK: $vgpr0 = COPY [[EXTRACT]](s32)
+    %0:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    %1:_(s32) = G_EXTRACT %0, 0
+    $vgpr0 = COPY %1
+...
+---
+name: test_extract_s32_v2s32_offset32
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; CHECK-LABEL: name: test_extract_s32_v2s32_offset32
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    ; CHECK: [[EXTRACT:%[0-9]+]]:_(s32) = G_EXTRACT [[COPY]](<2 x s32>), 32
+    ; CHECK: $vgpr0 = COPY [[EXTRACT]](s32)
+    %0:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    %1:_(s32) = G_EXTRACT %0, 32
+     $vgpr0 = COPY %1
+...
+---
+name: test_extract_s32_v3s32_offset0
+body: |
+  bb.0:
     liveins: $vgpr0_vgpr1_vgpr2
 
-    ; CHECK-LABEL: name: test_extract_dword2_i96
-    ; CHECK: [[COPY:%[0-9]+]]:_(s96) = COPY $vgpr0_vgpr1_vgpr2
-    ; CHECK: [[EXTRACT:%[0-9]+]]:_(s32) = G_EXTRACT [[COPY]](s96), 64
-    ; CHECK: S_ENDPGM implicit [[EXTRACT]](s32)
-    %0:_(s96) = COPY $vgpr0_vgpr1_vgpr2
+    ; CHECK-LABEL: name: test_extract_s32_v3s32_offset0
+    ; CHECK: [[COPY:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr0_vgpr1_vgpr2
+    ; CHECK: [[EXTRACT:%[0-9]+]]:_(s32) = G_EXTRACT [[COPY]](<3 x s32>), 0
+    ; CHECK: $vgpr0 = COPY [[EXTRACT]](s32)
+    %0:_(<3 x s32>) = COPY $vgpr0_vgpr1_vgpr2
+    %1:_(s32) = G_EXTRACT %0, 0
+    $vgpr0 = COPY %1
+...
+---
+name: test_extract_s32_v3s32_offset32
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2
+
+    ; CHECK-LABEL: name: test_extract_s32_v3s32_offset32
+    ; CHECK: [[COPY:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr0_vgpr1_vgpr2
+    ; CHECK: [[EXTRACT:%[0-9]+]]:_(s32) = G_EXTRACT [[COPY]](<3 x s32>), 32
+    ; CHECK: $vgpr0 = COPY [[EXTRACT]](s32)
+    %0:_(<3 x s32>) = COPY $vgpr0_vgpr1_vgpr2
+    %1:_(s32) = G_EXTRACT %0, 32
+    $vgpr0 = COPY %1
+...
+---
+name: test_extract_s32_v3s32_offset64
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2
+
+    ; CHECK-LABEL: name: test_extract_s32_v3s32_offset64
+    ; CHECK: [[COPY:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr0_vgpr1_vgpr2
+    ; CHECK: [[EXTRACT:%[0-9]+]]:_(s32) = G_EXTRACT [[COPY]](<3 x s32>), 64
+    ; CHECK: $vgpr0 = COPY [[EXTRACT]](s32)
+    %0:_(<3 x s32>) = COPY $vgpr0_vgpr1_vgpr2
+    %1:_(s32) = G_EXTRACT %0, 64
+    $vgpr0 = COPY %1
+...
+---
+name: test_extract_s32_v4s32_offset0
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3
+
+    ; CHECK-LABEL: name: test_extract_s32_v4s32_offset0
+    ; CHECK: [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; CHECK: [[EXTRACT:%[0-9]+]]:_(s32) = G_EXTRACT [[COPY]](<4 x s32>), 0
+    ; CHECK: $vgpr0 = COPY [[EXTRACT]](s32)
+    %0:_(<4 x s32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    %1:_(s32) = G_EXTRACT %0, 0
+    $vgpr0 = COPY %1
+...
+---
+name: test_extract_s32_v4s32_offset32
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3
+
+    ; CHECK-LABEL: name: test_extract_s32_v4s32_offset32
+    ; CHECK: [[COPY:%[0-9]+]]:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; CHECK: [[EXTRACT:%[0-9]+]]:_(s32) = G_EXTRACT [[COPY]](s128), 32
+    ; CHECK: $vgpr0 = COPY [[EXTRACT]](s32)
+    %0:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    %1:_(s32) = G_EXTRACT %0, 32
+    $vgpr0 = COPY %1
+...
+---
+name: test_extract_s32_v4s32_offset64
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3
+
+    ; CHECK-LABEL: name: test_extract_s32_v4s32_offset64
+    ; CHECK: [[COPY:%[0-9]+]]:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; CHECK: [[EXTRACT:%[0-9]+]]:_(s32) = G_EXTRACT [[COPY]](s128), 64
+    ; CHECK: $vgpr0 = COPY [[EXTRACT]](s32)
+    %0:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
     %1:_(s32) = G_EXTRACT %0, 64
-    S_ENDPGM implicit %1
+    $vgpr0 = COPY %1
+...
+---
+name: test_extract_s32_v4s32_offset96
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3
+
+    ; CHECK-LABEL: name: test_extract_s32_v4s32_offset96
+    ; CHECK: [[COPY:%[0-9]+]]:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; CHECK: [[EXTRACT:%[0-9]+]]:_(s32) = G_EXTRACT [[COPY]](s128), 96
+    ; CHECK: $vgpr0 = COPY [[EXTRACT]](s32)
+    %0:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    %1:_(s32) = G_EXTRACT %0, 96
+    $vgpr0 = COPY %1
+...
+---
+name: test_extract_v2s32_v4s32_offset0
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3
+
+    ; CHECK-LABEL: name: test_extract_v2s32_v4s32_offset0
+    ; CHECK: [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; CHECK: [[EXTRACT:%[0-9]+]]:_(<2 x s32>) = G_EXTRACT [[COPY]](<4 x s32>), 0
+    ; CHECK: $vgpr0_vgpr1 = COPY [[EXTRACT]](<2 x s32>)
+    %0:_(<4 x s32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    %1:_(<2 x s32>) = G_EXTRACT %0, 0
+    $vgpr0_vgpr1 = COPY %1
+...
+
+---
+name: test_extract_v2s32_v4s32_offset32
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3
+
+    ; CHECK-LABEL: name: test_extract_v2s32_v4s32_offset32
+    ; CHECK: [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; CHECK: [[EXTRACT:%[0-9]+]]:_(<2 x s32>) = G_EXTRACT [[COPY]](<4 x s32>), 32
+    ; CHECK: $vgpr0_vgpr1 = COPY [[EXTRACT]](<2 x s32>)
+    %0:_(<4 x s32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    %1:_(<2 x s32>) = G_EXTRACT %0, 32
+    $vgpr0_vgpr1 = COPY %1
+...
+
+---
+name: test_extract_v2s32_v4s32_offset64
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3
+
+    ; CHECK-LABEL: name: test_extract_v2s32_v4s32_offset64
+    ; CHECK: [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; CHECK: [[EXTRACT:%[0-9]+]]:_(<2 x s32>) = G_EXTRACT [[COPY]](<4 x s32>), 64
+    ; CHECK: $vgpr0_vgpr1 = COPY [[EXTRACT]](<2 x s32>)
+    %0:_(<4 x s32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    %1:_(<2 x s32>) = G_EXTRACT %0, 64
+    $vgpr0_vgpr1 = COPY %1
+...
+---
+name: test_extract_s64_v4s32_offset0
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3
+
+    ; CHECK-LABEL: name: test_extract_s64_v4s32_offset0
+    ; CHECK: [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; CHECK: [[EXTRACT:%[0-9]+]]:_(s64) = G_EXTRACT [[COPY]](<4 x s32>), 0
+    ; CHECK: $vgpr0_vgpr1 = COPY [[EXTRACT]](s64)
+    %0:_(<4 x s32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    %1:_(s64) = G_EXTRACT %0, 0
+    $vgpr0_vgpr1 = COPY %1
+...
+
+---
+name: test_extract_s64_v4s32_offset32
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3
+
+    ; CHECK-LABEL: name: test_extract_s64_v4s32_offset32
+    ; CHECK: [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; CHECK: [[EXTRACT:%[0-9]+]]:_(s64) = G_EXTRACT [[COPY]](<4 x s32>), 32
+    ; CHECK: $vgpr0_vgpr1 = COPY [[EXTRACT]](s64)
+    %0:_(<4 x s32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    %1:_(s64) = G_EXTRACT %0, 32
+    $vgpr0_vgpr1 = COPY %1
+...
+
+---
+name: test_extract_s64_v4s32_offset64
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3
+
+    ; CHECK-LABEL: name: test_extract_s64_v4s32_offset64
+    ; CHECK: [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; CHECK: [[EXTRACT:%[0-9]+]]:_(s64) = G_EXTRACT [[COPY]](<4 x s32>), 64
+    ; CHECK: $vgpr0_vgpr1 = COPY [[EXTRACT]](s64)
+    %0:_(<4 x s32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    %1:_(s64) = G_EXTRACT %0, 64
+    $vgpr0_vgpr1 = COPY %1
+...
+---
+name: test_extract_p0_v4s32_offset0
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3
+
+    ; CHECK-LABEL: name: test_extract_p0_v4s32_offset0
+    ; CHECK: [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; CHECK: [[EXTRACT:%[0-9]+]]:_(p0) = G_EXTRACT [[COPY]](<4 x s32>), 0
+    ; CHECK: $vgpr0_vgpr1 = COPY [[EXTRACT]](p0)
+    %0:_(<4 x s32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    %1:_(p0) = G_EXTRACT %0, 0
+    $vgpr0_vgpr1 = COPY %1
+...
+
+---
+name: test_extract_p0_v4s32_offset32
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3
+
+    ; CHECK-LABEL: name: test_extract_p0_v4s32_offset32
+    ; CHECK: [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; CHECK: [[EXTRACT:%[0-9]+]]:_(p0) = G_EXTRACT [[COPY]](<4 x s32>), 32
+    ; CHECK: $vgpr0_vgpr1 = COPY [[EXTRACT]](p0)
+    %0:_(<4 x s32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    %1:_(p0) = G_EXTRACT %0, 32
+    $vgpr0_vgpr1 = COPY %1
+...
+
+---
+name: test_extract_p0_v4s32_offset64
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3
+
+    ; CHECK-LABEL: name: test_extract_p0_v4s32_offset64
+    ; CHECK: [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; CHECK: [[EXTRACT:%[0-9]+]]:_(p0) = G_EXTRACT [[COPY]](<4 x s32>), 64
+    ; CHECK: $vgpr0_vgpr1 = COPY [[EXTRACT]](p0)
+    %0:_(<4 x s32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    %1:_(p0) = G_EXTRACT %0, 64
+    $vgpr0_vgpr1 = COPY %1
+...
+
+---
+name: extract_s8_v4s8_offset0
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: extract_s8_v4s8_offset0
+    ; CHECK: [[DEF:%[0-9]+]]:_(<4 x s32>) = G_IMPLICIT_DEF
+    ; CHECK: [[TRUNC:%[0-9]+]]:_(<4 x s8>) = G_TRUNC [[DEF]](<4 x s32>)
+    ; CHECK: [[ANYEXT:%[0-9]+]]:_(<4 x s16>) = G_ANYEXT [[TRUNC]](<4 x s8>)
+    ; CHECK: [[EXTRACT:%[0-9]+]]:_(s16) = G_EXTRACT [[ANYEXT]](<4 x s16>), 0
+    ; CHECK: [[TRUNC1:%[0-9]+]]:_(s8) = G_TRUNC [[EXTRACT]](s16)
+    ; CHECK: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[TRUNC1]](s8)
+    ; CHECK: $vgpr0 = COPY [[ANYEXT1]](s32)
+    %0:_(<4 x s8>) = G_IMPLICIT_DEF
+    %1:_(s8) = G_EXTRACT %0, 0
+    %2:_(s32) = G_ANYEXT %1
+    $vgpr0 = COPY %2
+...
+
+---
+name: extract_s8_v4s8_offset8
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: extract_s8_v4s8_offset8
+    ; CHECK: [[DEF:%[0-9]+]]:_(<4 x s32>) = G_IMPLICIT_DEF
+    ; CHECK: [[TRUNC:%[0-9]+]]:_(<4 x s8>) = G_TRUNC [[DEF]](<4 x s32>)
+    ; CHECK: [[ANYEXT:%[0-9]+]]:_(<4 x s16>) = G_ANYEXT [[TRUNC]](<4 x s8>)
+    ; CHECK: [[EXTRACT:%[0-9]+]]:_(s16) = G_EXTRACT [[ANYEXT]](<4 x s16>), 16
+    ; CHECK: [[TRUNC1:%[0-9]+]]:_(s8) = G_TRUNC [[EXTRACT]](s16)
+    ; CHECK: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[TRUNC1]](s8)
+    ; CHECK: $vgpr0 = COPY [[ANYEXT1]](s32)
+    %0:_(<4 x s8>) = G_IMPLICIT_DEF
+    %1:_(s8) = G_EXTRACT %0, 8
+    %2:_(s32) = G_ANYEXT %1
+    $vgpr0 = COPY %2
+...
+
+---
+name: extract_s8_v4s8_offset16
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: extract_s8_v4s8_offset16
+    ; CHECK: [[DEF:%[0-9]+]]:_(<4 x s32>) = G_IMPLICIT_DEF
+    ; CHECK: [[TRUNC:%[0-9]+]]:_(<4 x s8>) = G_TRUNC [[DEF]](<4 x s32>)
+    ; CHECK: [[ANYEXT:%[0-9]+]]:_(<4 x s16>) = G_ANYEXT [[TRUNC]](<4 x s8>)
+    ; CHECK: [[EXTRACT:%[0-9]+]]:_(s16) = G_EXTRACT [[ANYEXT]](<4 x s16>), 32
+    ; CHECK: [[TRUNC1:%[0-9]+]]:_(s8) = G_TRUNC [[EXTRACT]](s16)
+    ; CHECK: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[TRUNC1]](s8)
+    ; CHECK: $vgpr0 = COPY [[ANYEXT1]](s32)
+    %0:_(<4 x s8>) = G_IMPLICIT_DEF
+    %1:_(s8) = G_EXTRACT %0, 16
+    %2:_(s32) = G_ANYEXT %1
+    $vgpr0 = COPY %2
+...
+
+---
+name: extract_s8_v4s8_offset24
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: extract_s8_v4s8_offset24
+    ; CHECK: [[DEF:%[0-9]+]]:_(<4 x s32>) = G_IMPLICIT_DEF
+    ; CHECK: [[TRUNC:%[0-9]+]]:_(<4 x s8>) = G_TRUNC [[DEF]](<4 x s32>)
+    ; CHECK: [[ANYEXT:%[0-9]+]]:_(<4 x s16>) = G_ANYEXT [[TRUNC]](<4 x s8>)
+    ; CHECK: [[EXTRACT:%[0-9]+]]:_(s16) = G_EXTRACT [[ANYEXT]](<4 x s16>), 48
+    ; CHECK: [[TRUNC1:%[0-9]+]]:_(s8) = G_TRUNC [[EXTRACT]](s16)
+    ; CHECK: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[TRUNC1]](s8)
+    ; CHECK: $vgpr0 = COPY [[ANYEXT1]](s32)
+    %0:_(<4 x s8>) = G_IMPLICIT_DEF
+    %1:_(s8) = G_EXTRACT %0, 24
+    %2:_(s32) = G_ANYEXT %1
+    $vgpr0 = COPY %2
+...
+
+---
+name: extract_s8_v3s8_offset16
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: extract_s8_v3s8_offset16
+    ; CHECK: [[DEF:%[0-9]+]]:_(<4 x s32>) = G_IMPLICIT_DEF
+    ; CHECK: [[TRUNC:%[0-9]+]]:_(<4 x s8>) = G_TRUNC [[DEF]](<4 x s32>)
+    ; CHECK: [[EXTRACT:%[0-9]+]]:_(<3 x s8>) = G_EXTRACT [[TRUNC]](<4 x s8>), 0
+    ; CHECK: [[ANYEXT:%[0-9]+]]:_(<3 x s16>) = G_ANYEXT [[EXTRACT]](<3 x s8>)
+    ; CHECK: [[EXTRACT1:%[0-9]+]]:_(s16) = G_EXTRACT [[ANYEXT]](<3 x s16>), 32
+    ; CHECK: [[TRUNC1:%[0-9]+]]:_(s8) = G_TRUNC [[EXTRACT1]](s16)
+    ; CHECK: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[TRUNC1]](s8)
+    ; CHECK: $vgpr0 = COPY [[ANYEXT1]](s32)
+    %0:_(<3 x s8>) = G_IMPLICIT_DEF
+    %1:_(s8) = G_EXTRACT %0, 16
+    %2:_(s32) = G_ANYEXT %1
+    $vgpr0 = COPY %2
+...
+
+---
+name: extract_s8_v5s1_offset4
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: extract_s8_v5s1_offset4
+    ; CHECK: [[DEF:%[0-9]+]]:_(<6 x s32>) = G_IMPLICIT_DEF
+    ; CHECK: [[TRUNC:%[0-9]+]]:_(<6 x s1>) = G_TRUNC [[DEF]](<6 x s32>)
+    ; CHECK: [[EXTRACT:%[0-9]+]]:_(<5 x s1>) = G_EXTRACT [[TRUNC]](<6 x s1>), 0
+    ; CHECK: [[ANYEXT:%[0-9]+]]:_(<5 x s16>) = G_ANYEXT [[EXTRACT]](<5 x s1>)
+    ; CHECK: [[EXTRACT1:%[0-9]+]]:_(s16) = G_EXTRACT [[ANYEXT]](<5 x s16>), 64
+    ; CHECK: [[TRUNC1:%[0-9]+]]:_(s1) = G_TRUNC [[EXTRACT1]](s16)
+    ; CHECK: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[TRUNC1]](s1)
+    ; CHECK: $vgpr0 = COPY [[ANYEXT1]](s32)
+    %0:_(<5 x s1>) = G_IMPLICIT_DEF
+    %1:_(s1) = G_EXTRACT %0, 4
+    %2:_(s32) = G_ANYEXT %1
+    $vgpr0 = COPY %2
+...
+
+---
+name: extract_v2s16_v4s16_offset32
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: extract_v2s16_v4s16_offset32
+    ; CHECK: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; CHECK: [[EXTRACT:%[0-9]+]]:_(<2 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 32
+    ; CHECK: $vgpr0 = COPY [[EXTRACT]](<2 x s16>)
+    %0:_(<4 x s16>) = G_IMPLICIT_DEF
+    %1:_(<2 x s16>) = G_EXTRACT %0, 32
+    $vgpr0 = COPY %1
+...
+
+---
+name: extract_v2s16_v6s16_offset32
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: extract_v2s16_v6s16_offset32
+    ; CHECK: [[DEF:%[0-9]+]]:_(<6 x s32>) = G_IMPLICIT_DEF
+    ; CHECK: [[TRUNC:%[0-9]+]]:_(<6 x s16>) = G_TRUNC [[DEF]](<6 x s32>)
+    ; CHECK: [[EXTRACT:%[0-9]+]]:_(<2 x s16>) = G_EXTRACT [[TRUNC]](<6 x s16>), 32
+    ; CHECK: $vgpr0 = COPY [[EXTRACT]](<2 x s16>)
+    %0:_(<6 x s16>) = G_IMPLICIT_DEF
+    %1:_(<2 x s16>) = G_EXTRACT %0, 32
+    $vgpr0 = COPY %1
 ...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/legalize-fabs.mir b/test/CodeGen/AMDGPU/GlobalISel/legalize-fabs.mir
index 31f1e41775fe73bc48c788c2b378d53d692cd958..7f19e50dc24a3796cf4e608616b056aeb52c0712 100644
--- a/test/CodeGen/AMDGPU/GlobalISel/legalize-fabs.mir
+++ b/test/CodeGen/AMDGPU/GlobalISel/legalize-fabs.mir
@@ -1,25 +1,305 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=fiji -run-pass=legalizer -global-isel %s -o - | FileCheck  %s
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=tahiti -run-pass=legalizer %s -o - | FileCheck -check-prefix=SI  %s
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -run-pass=legalizer %s -o - | FileCheck -check-prefix=VI %s
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx900 -run-pass=legalizer %s -o - | FileCheck -check-prefix=GFX9  %s
+
 
 ---
-name: test_fabs_f32
+name: test_fabs_s32
 body: |
   bb.0:
     liveins: $vgpr0
 
-    ; CHECK-LABEL: name: test_fabs_f32
-    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; SI-LABEL: name: test_fabs_s32
+    ; SI: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; SI: [[FABS:%[0-9]+]]:_(s32) = G_FABS [[COPY]]
+    ; SI: $vgpr0 = COPY [[FABS]](s32)
+    ; VI-LABEL: name: test_fabs_s32
+    ; VI: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; VI: [[FABS:%[0-9]+]]:_(s32) = G_FABS [[COPY]]
+    ; VI: $vgpr0 = COPY [[FABS]](s32)
+    ; GFX9-LABEL: name: test_fabs_s32
+    ; GFX9: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; GFX9: [[FABS:%[0-9]+]]:_(s32) = G_FABS [[COPY]]
+    ; GFX9: $vgpr0 = COPY [[FABS]](s32)
     %0:_(s32) = COPY $vgpr0
     %1:_(s32) = G_FABS %0
+    $vgpr0 = COPY %1
+
 ...
 ---
-name: test_fabs_f64
+name: test_fabs_s64
 body: |
   bb.0:
     liveins: $vgpr0
 
-    ; CHECK-LABEL: name: test_fabs_f64
-    ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
+    ; SI-LABEL: name: test_fabs_s64
+    ; SI: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
+    ; SI: [[FABS:%[0-9]+]]:_(s64) = G_FABS [[COPY]]
+    ; SI: $vgpr0_vgpr1 = COPY [[FABS]](s64)
+    ; VI-LABEL: name: test_fabs_s64
+    ; VI: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
+    ; VI: [[FABS:%[0-9]+]]:_(s64) = G_FABS [[COPY]]
+    ; VI: $vgpr0_vgpr1 = COPY [[FABS]](s64)
+    ; GFX9-LABEL: name: test_fabs_s64
+    ; GFX9: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
+    ; GFX9: [[FABS:%[0-9]+]]:_(s64) = G_FABS [[COPY]]
+    ; GFX9: $vgpr0_vgpr1 = COPY [[FABS]](s64)
     %0:_(s64) = COPY $vgpr0_vgpr1
     %1:_(s64) = G_FABS %0
+    $vgpr0_vgpr1 = COPY %1
+...
+---
+name: test_fabs_s16
+body: |
+  bb.0:
+    liveins: $vgpr0
+
+    ; SI-LABEL: name: test_fabs_s16
+    ; SI: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; SI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+    ; SI: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC]](s16)
+    ; SI: [[FABS:%[0-9]+]]:_(s32) = G_FABS [[FPEXT]]
+    ; SI: [[FPTRUNC:%[0-9]+]]:_(s16) = G_FPTRUNC [[FABS]](s32)
+    ; SI: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[FPTRUNC]](s16)
+    ; SI: $vgpr0 = COPY [[ANYEXT]](s32)
+    ; VI-LABEL: name: test_fabs_s16
+    ; VI: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; VI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+    ; VI: [[FABS:%[0-9]+]]:_(s16) = G_FABS [[TRUNC]]
+    ; VI: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[FABS]](s16)
+    ; VI: $vgpr0 = COPY [[ANYEXT]](s32)
+    ; GFX9-LABEL: name: test_fabs_s16
+    ; GFX9: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; GFX9: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+    ; GFX9: [[FABS:%[0-9]+]]:_(s16) = G_FABS [[TRUNC]]
+    ; GFX9: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[FABS]](s16)
+    ; GFX9: $vgpr0 = COPY [[ANYEXT]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s16) = G_TRUNC %0
+    %2:_(s16) = G_FABS %1
+    %3:_(s32) = G_ANYEXT %2
+    $vgpr0 = COPY %3
+...
+
+---
+name: test_fabs_v2s32
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; SI-LABEL: name: test_fabs_v2s32
+    ; SI: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    ; SI: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
+    ; SI: [[FABS:%[0-9]+]]:_(s32) = G_FABS [[UV]]
+    ; SI: [[FABS1:%[0-9]+]]:_(s32) = G_FABS [[UV1]]
+    ; SI: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[FABS]](s32), [[FABS1]](s32)
+    ; SI: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
+    ; VI-LABEL: name: test_fabs_v2s32
+    ; VI: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    ; VI: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
+    ; VI: [[FABS:%[0-9]+]]:_(s32) = G_FABS [[UV]]
+    ; VI: [[FABS1:%[0-9]+]]:_(s32) = G_FABS [[UV1]]
+    ; VI: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[FABS]](s32), [[FABS1]](s32)
+    ; VI: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
+    ; GFX9-LABEL: name: test_fabs_v2s32
+    ; GFX9: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    ; GFX9: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
+    ; GFX9: [[FABS:%[0-9]+]]:_(s32) = G_FABS [[UV]]
+    ; GFX9: [[FABS1:%[0-9]+]]:_(s32) = G_FABS [[UV1]]
+    ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[FABS]](s32), [[FABS1]](s32)
+    ; GFX9: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
+    %0:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    %1:_(<2 x s32>) = G_FABS %0
+    $vgpr0_vgpr1 = COPY %1
+...
+
+---
+name: test_fabs_v3s32
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2
+
+    ; SI-LABEL: name: test_fabs_v3s32
+    ; SI: [[COPY:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr0_vgpr1_vgpr2
+    ; SI: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<3 x s32>)
+    ; SI: [[FABS:%[0-9]+]]:_(s32) = G_FABS [[UV]]
+    ; SI: [[FABS1:%[0-9]+]]:_(s32) = G_FABS [[UV1]]
+    ; SI: [[FABS2:%[0-9]+]]:_(s32) = G_FABS [[UV2]]
+    ; SI: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[FABS]](s32), [[FABS1]](s32), [[FABS2]](s32)
+    ; SI: $vgpr0_vgpr1_vgpr2 = COPY [[BUILD_VECTOR]](<3 x s32>)
+    ; VI-LABEL: name: test_fabs_v3s32
+    ; VI: [[COPY:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr0_vgpr1_vgpr2
+    ; VI: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<3 x s32>)
+    ; VI: [[FABS:%[0-9]+]]:_(s32) = G_FABS [[UV]]
+    ; VI: [[FABS1:%[0-9]+]]:_(s32) = G_FABS [[UV1]]
+    ; VI: [[FABS2:%[0-9]+]]:_(s32) = G_FABS [[UV2]]
+    ; VI: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[FABS]](s32), [[FABS1]](s32), [[FABS2]](s32)
+    ; VI: $vgpr0_vgpr1_vgpr2 = COPY [[BUILD_VECTOR]](<3 x s32>)
+    ; GFX9-LABEL: name: test_fabs_v3s32
+    ; GFX9: [[COPY:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr0_vgpr1_vgpr2
+    ; GFX9: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<3 x s32>)
+    ; GFX9: [[FABS:%[0-9]+]]:_(s32) = G_FABS [[UV]]
+    ; GFX9: [[FABS1:%[0-9]+]]:_(s32) = G_FABS [[UV1]]
+    ; GFX9: [[FABS2:%[0-9]+]]:_(s32) = G_FABS [[UV2]]
+    ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[FABS]](s32), [[FABS1]](s32), [[FABS2]](s32)
+    ; GFX9: $vgpr0_vgpr1_vgpr2 = COPY [[BUILD_VECTOR]](<3 x s32>)
+    %0:_(<3 x s32>) = COPY $vgpr0_vgpr1_vgpr2
+    %1:_(<3 x  s32>) = G_FABS %0
+    $vgpr0_vgpr1_vgpr2 = COPY %1
+...
+
+---
+name: test_fabs_v2s64
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3
+
+    ; SI-LABEL: name: test_fabs_v2s64
+    ; SI: [[COPY:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; SI: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](<2 x s64>)
+    ; SI: [[FABS:%[0-9]+]]:_(s64) = G_FABS [[UV]]
+    ; SI: [[FABS1:%[0-9]+]]:_(s64) = G_FABS [[UV1]]
+    ; SI: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[FABS]](s64), [[FABS1]](s64)
+    ; SI: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>)
+    ; VI-LABEL: name: test_fabs_v2s64
+    ; VI: [[COPY:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; VI: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](<2 x s64>)
+    ; VI: [[FABS:%[0-9]+]]:_(s64) = G_FABS [[UV]]
+    ; VI: [[FABS1:%[0-9]+]]:_(s64) = G_FABS [[UV1]]
+    ; VI: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[FABS]](s64), [[FABS1]](s64)
+    ; VI: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>)
+    ; GFX9-LABEL: name: test_fabs_v2s64
+    ; GFX9: [[COPY:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; GFX9: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](<2 x s64>)
+    ; GFX9: [[FABS:%[0-9]+]]:_(s64) = G_FABS [[UV]]
+    ; GFX9: [[FABS1:%[0-9]+]]:_(s64) = G_FABS [[UV1]]
+    ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[FABS]](s64), [[FABS1]](s64)
+    ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>)
+    %0:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    %1:_(<2 x s64>) = G_FABS %0
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %1
+...
+
+---
+name: test_fabs_v2s16
+body: |
+  bb.0:
+    liveins: $vgpr0
+
+    ; SI-LABEL: name: test_fabs_v2s16
+    ; SI: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
+    ; SI: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY]](<2 x s16>)
+    ; SI: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT [[UV]](s16)
+    ; SI: [[FABS:%[0-9]+]]:_(s32) = G_FABS [[FPEXT]]
+    ; SI: [[FPTRUNC:%[0-9]+]]:_(s16) = G_FPTRUNC [[FABS]](s32)
+    ; SI: [[FPEXT1:%[0-9]+]]:_(s32) = G_FPEXT [[UV1]](s16)
+    ; SI: [[FABS1:%[0-9]+]]:_(s32) = G_FABS [[FPEXT1]]
+    ; SI: [[FPTRUNC1:%[0-9]+]]:_(s16) = G_FPTRUNC [[FABS1]](s32)
+    ; SI: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[FPTRUNC]](s16), [[FPTRUNC1]](s16)
+    ; SI: $vgpr0 = COPY [[BUILD_VECTOR]](<2 x s16>)
+    ; VI-LABEL: name: test_fabs_v2s16
+    ; VI: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
+    ; VI: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY]](<2 x s16>)
+    ; VI: [[FABS:%[0-9]+]]:_(s16) = G_FABS [[UV]]
+    ; VI: [[FABS1:%[0-9]+]]:_(s16) = G_FABS [[UV1]]
+    ; VI: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[FABS]](s16), [[FABS1]](s16)
+    ; VI: $vgpr0 = COPY [[BUILD_VECTOR]](<2 x s16>)
+    ; GFX9-LABEL: name: test_fabs_v2s16
+    ; GFX9: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
+    ; GFX9: [[FABS:%[0-9]+]]:_(<2 x s16>) = G_FABS [[COPY]]
+    ; GFX9: $vgpr0 = COPY [[FABS]](<2 x s16>)
+    %0:_(<2 x s16>) = COPY $vgpr0
+    %1:_(<2 x s16>) = G_FABS %0
+    $vgpr0 = COPY %1
+...
+
+---
+name: test_fabs_v3s16
+body: |
+  bb.0:
+
+    ; SI-LABEL: name: test_fabs_v3s16
+    ; SI: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; SI: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
+    ; SI: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16), [[UV2:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[EXTRACT]](<3 x s16>)
+    ; SI: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT [[UV]](s16)
+    ; SI: [[FABS:%[0-9]+]]:_(s32) = G_FABS [[FPEXT]]
+    ; SI: [[FPTRUNC:%[0-9]+]]:_(s16) = G_FPTRUNC [[FABS]](s32)
+    ; SI: [[FPEXT1:%[0-9]+]]:_(s32) = G_FPEXT [[UV1]](s16)
+    ; SI: [[FABS1:%[0-9]+]]:_(s32) = G_FABS [[FPEXT1]]
+    ; SI: [[FPTRUNC1:%[0-9]+]]:_(s16) = G_FPTRUNC [[FABS1]](s32)
+    ; SI: [[FPEXT2:%[0-9]+]]:_(s32) = G_FPEXT [[UV2]](s16)
+    ; SI: [[FABS2:%[0-9]+]]:_(s32) = G_FABS [[FPEXT2]]
+    ; SI: [[FPTRUNC2:%[0-9]+]]:_(s16) = G_FPTRUNC [[FABS2]](s32)
+    ; SI: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s16>) = G_BUILD_VECTOR [[FPTRUNC]](s16), [[FPTRUNC1]](s16), [[FPTRUNC2]](s16)
+    ; SI: S_NOP 0, implicit [[BUILD_VECTOR]](<3 x s16>)
+    ; VI-LABEL: name: test_fabs_v3s16
+    ; VI: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; VI: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
+    ; VI: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16), [[UV2:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[EXTRACT]](<3 x s16>)
+    ; VI: [[FABS:%[0-9]+]]:_(s16) = G_FABS [[UV]]
+    ; VI: [[FABS1:%[0-9]+]]:_(s16) = G_FABS [[UV1]]
+    ; VI: [[FABS2:%[0-9]+]]:_(s16) = G_FABS [[UV2]]
+    ; VI: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s16>) = G_BUILD_VECTOR [[FABS]](s16), [[FABS1]](s16), [[FABS2]](s16)
+    ; VI: S_NOP 0, implicit [[BUILD_VECTOR]](<3 x s16>)
+    ; GFX9-LABEL: name: test_fabs_v3s16
+    ; GFX9: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; GFX9: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
+    ; GFX9: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; GFX9: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF1]](<4 x s16>), 0
+    ; GFX9: [[EXTRACT2:%[0-9]+]]:_(<2 x s16>) = G_EXTRACT [[EXTRACT]](<3 x s16>), 0
+    ; GFX9: [[FABS:%[0-9]+]]:_(<2 x s16>) = G_FABS [[EXTRACT2]]
+    ; GFX9: [[INSERT:%[0-9]+]]:_(<3 x s16>) = G_INSERT [[EXTRACT1]], [[FABS]](<2 x s16>), 0
+    ; GFX9: [[EXTRACT3:%[0-9]+]]:_(s16) = G_EXTRACT [[EXTRACT]](<3 x s16>), 32
+    ; GFX9: [[FABS1:%[0-9]+]]:_(s16) = G_FABS [[EXTRACT3]]
+    ; GFX9: [[INSERT1:%[0-9]+]]:_(<3 x s16>) = G_INSERT [[INSERT]], [[FABS1]](s16), 32
+    ; GFX9: S_NOP 0, implicit [[INSERT1]](<3 x s16>)
+    %0:_(<3 x s16>) = G_IMPLICIT_DEF
+    %1:_(<3 x s16>) = G_FABS %0
+    S_NOP 0, implicit %1
+...
+
+---
+name: test_fabs_v4s16
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; SI-LABEL: name: test_fabs_v4s16
+    ; SI: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
+    ; SI: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16), [[UV2:%[0-9]+]]:_(s16), [[UV3:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
+    ; SI: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT [[UV]](s16)
+    ; SI: [[FABS:%[0-9]+]]:_(s32) = G_FABS [[FPEXT]]
+    ; SI: [[FPTRUNC:%[0-9]+]]:_(s16) = G_FPTRUNC [[FABS]](s32)
+    ; SI: [[FPEXT1:%[0-9]+]]:_(s32) = G_FPEXT [[UV1]](s16)
+    ; SI: [[FABS1:%[0-9]+]]:_(s32) = G_FABS [[FPEXT1]]
+    ; SI: [[FPTRUNC1:%[0-9]+]]:_(s16) = G_FPTRUNC [[FABS1]](s32)
+    ; SI: [[FPEXT2:%[0-9]+]]:_(s32) = G_FPEXT [[UV2]](s16)
+    ; SI: [[FABS2:%[0-9]+]]:_(s32) = G_FABS [[FPEXT2]]
+    ; SI: [[FPTRUNC2:%[0-9]+]]:_(s16) = G_FPTRUNC [[FABS2]](s32)
+    ; SI: [[FPEXT3:%[0-9]+]]:_(s32) = G_FPEXT [[UV3]](s16)
+    ; SI: [[FABS3:%[0-9]+]]:_(s32) = G_FABS [[FPEXT3]]
+    ; SI: [[FPTRUNC3:%[0-9]+]]:_(s16) = G_FPTRUNC [[FABS3]](s32)
+    ; SI: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s16>) = G_BUILD_VECTOR [[FPTRUNC]](s16), [[FPTRUNC1]](s16), [[FPTRUNC2]](s16), [[FPTRUNC3]](s16)
+    ; SI: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<4 x s16>)
+    ; VI-LABEL: name: test_fabs_v4s16
+    ; VI: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
+    ; VI: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16), [[UV2:%[0-9]+]]:_(s16), [[UV3:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
+    ; VI: [[FABS:%[0-9]+]]:_(s16) = G_FABS [[UV]]
+    ; VI: [[FABS1:%[0-9]+]]:_(s16) = G_FABS [[UV1]]
+    ; VI: [[FABS2:%[0-9]+]]:_(s16) = G_FABS [[UV2]]
+    ; VI: [[FABS3:%[0-9]+]]:_(s16) = G_FABS [[UV3]]
+    ; VI: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s16>) = G_BUILD_VECTOR [[FABS]](s16), [[FABS1]](s16), [[FABS2]](s16), [[FABS3]](s16)
+    ; VI: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<4 x s16>)
+    ; GFX9-LABEL: name: test_fabs_v4s16
+    ; GFX9: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
+    ; GFX9: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
+    ; GFX9: [[FABS:%[0-9]+]]:_(<2 x s16>) = G_FABS [[UV]]
+    ; GFX9: [[FABS1:%[0-9]+]]:_(<2 x s16>) = G_FABS [[UV1]]
+    ; GFX9: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[FABS]](<2 x s16>), [[FABS1]](<2 x s16>)
+    ; GFX9: $vgpr0_vgpr1 = COPY [[CONCAT_VECTORS]](<4 x s16>)
+    %0:_(<4 x s16>) = COPY $vgpr0_vgpr1
+    %1:_(<4 x s16>) = G_FABS %0
+    $vgpr0_vgpr1 = COPY %1
 ...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/legalize-fadd.mir b/test/CodeGen/AMDGPU/GlobalISel/legalize-fadd.mir
index 85ed234bf47fb51790d14ad8fcbcad6293ff2e29..6bc53a5368c766fff926773053364bd489638669 100644
--- a/test/CodeGen/AMDGPU/GlobalISel/legalize-fadd.mir
+++ b/test/CodeGen/AMDGPU/GlobalISel/legalize-fadd.mir
@@ -1,26 +1,417 @@
-# RUN: llc -mtriple=amdgcn-mesa-mesa3d -run-pass=legalizer %s -o - | FileCheck %s
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=tahiti -run-pass=legalizer %s -o - | FileCheck -check-prefix=SI  %s
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -run-pass=legalizer %s -o - | FileCheck -check-prefix=VI %s
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx900 -run-pass=legalizer %s -o - | FileCheck -check-prefix=GFX9  %s
 
 ---
-name: test_fadd_f32
+name: test_fadd_s32
 body: |
-  bb.0.entry:
+  bb.0:
     liveins: $vgpr0, $vgpr1
-    ; CHECK-LABEL: name: test_fadd
-    ; CHECK: %2:_(s32) = G_FADD %0, %1
 
+    ; SI-LABEL: name: test_fadd_s32
+    ; SI: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; SI: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; SI: [[FADD:%[0-9]+]]:_(s32) = G_FADD [[COPY]], [[COPY1]]
+    ; SI: $vgpr0 = COPY [[FADD]](s32)
+    ; VI-LABEL: name: test_fadd_s32
+    ; VI: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; VI: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; VI: [[FADD:%[0-9]+]]:_(s32) = G_FADD [[COPY]], [[COPY1]]
+    ; VI: $vgpr0 = COPY [[FADD]](s32)
+    ; GFX9-LABEL: name: test_fadd_s32
+    ; GFX9: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; GFX9: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; GFX9: [[FADD:%[0-9]+]]:_(s32) = G_FADD [[COPY]], [[COPY1]]
+    ; GFX9: $vgpr0 = COPY [[FADD]](s32)
     %0:_(s32) = COPY $vgpr0
     %1:_(s32) = COPY $vgpr1
     %2:_(s32) = G_FADD %0, %1
     $vgpr0 = COPY %2
 ...
+
 ---
-name: test_fadd_f64
+name: test_fadd_s64
 body: |
-  bb.0.entry:
+  bb.0:
     liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
 
+    ; SI-LABEL: name: test_fadd_s64
+    ; SI: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
+    ; SI: [[COPY1:%[0-9]+]]:_(s64) = COPY $vgpr2_vgpr3
+    ; SI: [[FADD:%[0-9]+]]:_(s64) = G_FADD [[COPY]], [[COPY1]]
+    ; SI: $vgpr0_vgpr1 = COPY [[FADD]](s64)
+    ; VI-LABEL: name: test_fadd_s64
+    ; VI: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
+    ; VI: [[COPY1:%[0-9]+]]:_(s64) = COPY $vgpr2_vgpr3
+    ; VI: [[FADD:%[0-9]+]]:_(s64) = G_FADD [[COPY]], [[COPY1]]
+    ; VI: $vgpr0_vgpr1 = COPY [[FADD]](s64)
+    ; GFX9-LABEL: name: test_fadd_s64
+    ; GFX9: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
+    ; GFX9: [[COPY1:%[0-9]+]]:_(s64) = COPY $vgpr2_vgpr3
+    ; GFX9: [[FADD:%[0-9]+]]:_(s64) = G_FADD [[COPY]], [[COPY1]]
+    ; GFX9: $vgpr0_vgpr1 = COPY [[FADD]](s64)
     %0:_(s64) = COPY $vgpr0_vgpr1
     %1:_(s64) = COPY $vgpr2_vgpr3
     %2:_(s64) = G_FADD %0, %1
     $vgpr0_vgpr1 = COPY %2
 ...
+
+---
+name: test_fadd_s16
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+
+    ; SI-LABEL: name: test_fadd_s16
+    ; SI: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; SI: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; SI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+    ; SI: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32)
+    ; SI: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC]](s16)
+    ; SI: [[FPEXT1:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC1]](s16)
+    ; SI: [[FADD:%[0-9]+]]:_(s32) = G_FADD [[FPEXT]], [[FPEXT1]]
+    ; SI: [[FPTRUNC:%[0-9]+]]:_(s16) = G_FPTRUNC [[FADD]](s32)
+    ; SI: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[FPTRUNC]](s16)
+    ; SI: $vgpr0 = COPY [[ANYEXT]](s32)
+    ; VI-LABEL: name: test_fadd_s16
+    ; VI: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; VI: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; VI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+    ; VI: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32)
+    ; VI: [[FADD:%[0-9]+]]:_(s16) = G_FADD [[TRUNC]], [[TRUNC1]]
+    ; VI: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[FADD]](s16)
+    ; VI: $vgpr0 = COPY [[ANYEXT]](s32)
+    ; GFX9-LABEL: name: test_fadd_s16
+    ; GFX9: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; GFX9: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; GFX9: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+    ; GFX9: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32)
+    ; GFX9: [[FADD:%[0-9]+]]:_(s16) = G_FADD [[TRUNC]], [[TRUNC1]]
+    ; GFX9: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[FADD]](s16)
+    ; GFX9: $vgpr0 = COPY [[ANYEXT]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s16) = G_TRUNC %0
+    %3:_(s16) = G_TRUNC %1
+
+    %4:_(s16) = G_FADD %2, %3
+    %5:_(s32) = G_ANYEXT %4
+    $vgpr0 = COPY %5
+...
+
+---
+name: test_fadd_v2s32
+body: |
+  bb.0.entry:
+    liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
+
+    ; SI-LABEL: name: test_fadd_v2s32
+    ; SI: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    ; SI: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr2_vgpr3
+    ; SI: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
+    ; SI: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<2 x s32>)
+    ; SI: [[FADD:%[0-9]+]]:_(s32) = G_FADD [[UV]], [[UV2]]
+    ; SI: [[FADD1:%[0-9]+]]:_(s32) = G_FADD [[UV1]], [[UV3]]
+    ; SI: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[FADD]](s32), [[FADD1]](s32)
+    ; SI: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
+    ; VI-LABEL: name: test_fadd_v2s32
+    ; VI: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    ; VI: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr2_vgpr3
+    ; VI: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
+    ; VI: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<2 x s32>)
+    ; VI: [[FADD:%[0-9]+]]:_(s32) = G_FADD [[UV]], [[UV2]]
+    ; VI: [[FADD1:%[0-9]+]]:_(s32) = G_FADD [[UV1]], [[UV3]]
+    ; VI: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[FADD]](s32), [[FADD1]](s32)
+    ; VI: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
+    ; GFX9-LABEL: name: test_fadd_v2s32
+    ; GFX9: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    ; GFX9: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr2_vgpr3
+    ; GFX9: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
+    ; GFX9: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<2 x s32>)
+    ; GFX9: [[FADD:%[0-9]+]]:_(s32) = G_FADD [[UV]], [[UV2]]
+    ; GFX9: [[FADD1:%[0-9]+]]:_(s32) = G_FADD [[UV1]], [[UV3]]
+    ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[FADD]](s32), [[FADD1]](s32)
+    ; GFX9: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
+    %0:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    %1:_(<2 x s32>) = COPY $vgpr2_vgpr3
+    %2:_(<2 x s32>) = G_FADD %0, %1
+    $vgpr0_vgpr1 = COPY %2
+...
+
+---
+name: test_fadd_v2s32_flags
+body: |
+  bb.0.entry:
+    liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
+
+    ; SI-LABEL: name: test_fadd_v2s32_flags
+    ; SI: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    ; SI: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr2_vgpr3
+    ; SI: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
+    ; SI: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<2 x s32>)
+    ; SI: %7:_(s32) = nnan G_FADD [[UV]], [[UV2]]
+    ; SI: %8:_(s32) = nnan G_FADD [[UV1]], [[UV3]]
+    ; SI: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR %7(s32), %8(s32)
+    ; SI: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
+    ; VI-LABEL: name: test_fadd_v2s32_flags
+    ; VI: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    ; VI: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr2_vgpr3
+    ; VI: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
+    ; VI: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<2 x s32>)
+    ; VI: %7:_(s32) = nnan G_FADD [[UV]], [[UV2]]
+    ; VI: %8:_(s32) = nnan G_FADD [[UV1]], [[UV3]]
+    ; VI: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR %7(s32), %8(s32)
+    ; VI: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
+    ; GFX9-LABEL: name: test_fadd_v2s32_flags
+    ; GFX9: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    ; GFX9: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr2_vgpr3
+    ; GFX9: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
+    ; GFX9: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<2 x s32>)
+    ; GFX9: %7:_(s32) = nnan G_FADD [[UV]], [[UV2]]
+    ; GFX9: %8:_(s32) = nnan G_FADD [[UV1]], [[UV3]]
+    ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR %7(s32), %8(s32)
+    ; GFX9: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
+    %0:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    %1:_(<2 x s32>) = COPY $vgpr2_vgpr3
+    %2:_(<2 x s32>) = nnan G_FADD %0, %1
+    $vgpr0_vgpr1 = COPY %2
+...
+
+---
+name: test_fadd_v3s32
+body: |
+  bb.0.entry:
+    liveins: $vgpr0_vgpr1_vgpr2, $vgpr3_vgpr4_vgpr5
+
+    ; SI-LABEL: name: test_fadd_v3s32
+    ; SI: [[COPY:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr0_vgpr1_vgpr2
+    ; SI: [[COPY1:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr3_vgpr4_vgpr5
+    ; SI: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<3 x s32>)
+    ; SI: [[UV3:%[0-9]+]]:_(s32), [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<3 x s32>)
+    ; SI: [[FADD:%[0-9]+]]:_(s32) = G_FADD [[UV]], [[UV3]]
+    ; SI: [[FADD1:%[0-9]+]]:_(s32) = G_FADD [[UV1]], [[UV4]]
+    ; SI: [[FADD2:%[0-9]+]]:_(s32) = G_FADD [[UV2]], [[UV5]]
+    ; SI: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[FADD]](s32), [[FADD1]](s32), [[FADD2]](s32)
+    ; SI: $vgpr0_vgpr1_vgpr2 = COPY [[BUILD_VECTOR]](<3 x s32>)
+    ; VI-LABEL: name: test_fadd_v3s32
+    ; VI: [[COPY:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr0_vgpr1_vgpr2
+    ; VI: [[COPY1:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr3_vgpr4_vgpr5
+    ; VI: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<3 x s32>)
+    ; VI: [[UV3:%[0-9]+]]:_(s32), [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<3 x s32>)
+    ; VI: [[FADD:%[0-9]+]]:_(s32) = G_FADD [[UV]], [[UV3]]
+    ; VI: [[FADD1:%[0-9]+]]:_(s32) = G_FADD [[UV1]], [[UV4]]
+    ; VI: [[FADD2:%[0-9]+]]:_(s32) = G_FADD [[UV2]], [[UV5]]
+    ; VI: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[FADD]](s32), [[FADD1]](s32), [[FADD2]](s32)
+    ; VI: $vgpr0_vgpr1_vgpr2 = COPY [[BUILD_VECTOR]](<3 x s32>)
+    ; GFX9-LABEL: name: test_fadd_v3s32
+    ; GFX9: [[COPY:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr0_vgpr1_vgpr2
+    ; GFX9: [[COPY1:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr3_vgpr4_vgpr5
+    ; GFX9: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<3 x s32>)
+    ; GFX9: [[UV3:%[0-9]+]]:_(s32), [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<3 x s32>)
+    ; GFX9: [[FADD:%[0-9]+]]:_(s32) = G_FADD [[UV]], [[UV3]]
+    ; GFX9: [[FADD1:%[0-9]+]]:_(s32) = G_FADD [[UV1]], [[UV4]]
+    ; GFX9: [[FADD2:%[0-9]+]]:_(s32) = G_FADD [[UV2]], [[UV5]]
+    ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[FADD]](s32), [[FADD1]](s32), [[FADD2]](s32)
+    ; GFX9: $vgpr0_vgpr1_vgpr2 = COPY [[BUILD_VECTOR]](<3 x s32>)
+    %0:_(<3 x s32>) = COPY $vgpr0_vgpr1_vgpr2
+    %1:_(<3 x s32>) = COPY $vgpr3_vgpr4_vgpr5
+    %2:_(<3 x s32>) = G_FADD %0, %1
+    $vgpr0_vgpr1_vgpr2 = COPY %2
+...
+
+---
+name: test_fadd_v2s64
+body: |
+  bb.0.entry:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7
+
+    ; SI-LABEL: name: test_fadd_v2s64
+    ; SI: [[COPY:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; SI: [[COPY1:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr4_vgpr5_vgpr6_vgpr7
+    ; SI: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](<2 x s64>)
+    ; SI: [[UV2:%[0-9]+]]:_(s64), [[UV3:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY1]](<2 x s64>)
+    ; SI: [[FADD:%[0-9]+]]:_(s64) = G_FADD [[UV]], [[UV2]]
+    ; SI: [[FADD1:%[0-9]+]]:_(s64) = G_FADD [[UV1]], [[UV3]]
+    ; SI: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[FADD]](s64), [[FADD1]](s64)
+    ; SI: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>)
+    ; VI-LABEL: name: test_fadd_v2s64
+    ; VI: [[COPY:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; VI: [[COPY1:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr4_vgpr5_vgpr6_vgpr7
+    ; VI: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](<2 x s64>)
+    ; VI: [[UV2:%[0-9]+]]:_(s64), [[UV3:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY1]](<2 x s64>)
+    ; VI: [[FADD:%[0-9]+]]:_(s64) = G_FADD [[UV]], [[UV2]]
+    ; VI: [[FADD1:%[0-9]+]]:_(s64) = G_FADD [[UV1]], [[UV3]]
+    ; VI: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[FADD]](s64), [[FADD1]](s64)
+    ; VI: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>)
+    ; GFX9-LABEL: name: test_fadd_v2s64
+    ; GFX9: [[COPY:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; GFX9: [[COPY1:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr4_vgpr5_vgpr6_vgpr7
+    ; GFX9: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](<2 x s64>)
+    ; GFX9: [[UV2:%[0-9]+]]:_(s64), [[UV3:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY1]](<2 x s64>)
+    ; GFX9: [[FADD:%[0-9]+]]:_(s64) = G_FADD [[UV]], [[UV2]]
+    ; GFX9: [[FADD1:%[0-9]+]]:_(s64) = G_FADD [[UV1]], [[UV3]]
+    ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[FADD]](s64), [[FADD1]](s64)
+    ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>)
+    %0:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    %1:_(<2 x s64>) = COPY $vgpr4_vgpr5_vgpr6_vgpr7
+    %2:_(<2 x s64>) = G_FADD %0, %1
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %2
+...
+
+---
+name: test_fadd_v2s16
+body: |
+  bb.0.entry:
+    liveins: $vgpr0, $vgpr1
+
+    ; SI-LABEL: name: test_fadd_v2s16
+    ; SI: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
+    ; SI: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
+    ; SI: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY]](<2 x s16>)
+    ; SI: [[UV2:%[0-9]+]]:_(s16), [[UV3:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY1]](<2 x s16>)
+    ; SI: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT [[UV]](s16)
+    ; SI: [[FPEXT1:%[0-9]+]]:_(s32) = G_FPEXT [[UV2]](s16)
+    ; SI: [[FADD:%[0-9]+]]:_(s32) = G_FADD [[FPEXT]], [[FPEXT1]]
+    ; SI: [[FPTRUNC:%[0-9]+]]:_(s16) = G_FPTRUNC [[FADD]](s32)
+    ; SI: [[FPEXT2:%[0-9]+]]:_(s32) = G_FPEXT [[UV1]](s16)
+    ; SI: [[FPEXT3:%[0-9]+]]:_(s32) = G_FPEXT [[UV3]](s16)
+    ; SI: [[FADD1:%[0-9]+]]:_(s32) = G_FADD [[FPEXT2]], [[FPEXT3]]
+    ; SI: [[FPTRUNC1:%[0-9]+]]:_(s16) = G_FPTRUNC [[FADD1]](s32)
+    ; SI: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[FPTRUNC]](s16), [[FPTRUNC1]](s16)
+    ; SI: $vgpr0 = COPY [[BUILD_VECTOR]](<2 x s16>)
+    ; VI-LABEL: name: test_fadd_v2s16
+    ; VI: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
+    ; VI: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
+    ; VI: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY]](<2 x s16>)
+    ; VI: [[UV2:%[0-9]+]]:_(s16), [[UV3:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY1]](<2 x s16>)
+    ; VI: [[FADD:%[0-9]+]]:_(s16) = G_FADD [[UV]], [[UV2]]
+    ; VI: [[FADD1:%[0-9]+]]:_(s16) = G_FADD [[UV1]], [[UV3]]
+    ; VI: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[FADD]](s16), [[FADD1]](s16)
+    ; VI: $vgpr0 = COPY [[BUILD_VECTOR]](<2 x s16>)
+    ; GFX9-LABEL: name: test_fadd_v2s16
+    ; GFX9: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
+    ; GFX9: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
+    ; GFX9: [[FADD:%[0-9]+]]:_(<2 x s16>) = G_FADD [[COPY]], [[COPY1]]
+    ; GFX9: $vgpr0 = COPY [[FADD]](<2 x s16>)
+    %0:_(<2 x s16>) = COPY $vgpr0
+    %1:_(<2 x s16>) = COPY $vgpr1
+    %2:_(<2 x s16>) = G_FADD %0, %1
+    $vgpr0 = COPY %2
+...
+
+---
+name: test_fadd_v3s16
+body: |
+  bb.0.entry:
+    liveins: $vgpr0, $vgpr1
+
+    ; SI-LABEL: name: test_fadd_v3s16
+    ; SI: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; SI: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
+    ; SI: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; SI: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF1]](<4 x s16>), 0
+    ; SI: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16), [[UV2:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[EXTRACT]](<3 x s16>)
+    ; SI: [[UV3:%[0-9]+]]:_(s16), [[UV4:%[0-9]+]]:_(s16), [[UV5:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[EXTRACT1]](<3 x s16>)
+    ; SI: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT [[UV]](s16)
+    ; SI: [[FPEXT1:%[0-9]+]]:_(s32) = G_FPEXT [[UV3]](s16)
+    ; SI: [[FADD:%[0-9]+]]:_(s32) = G_FADD [[FPEXT]], [[FPEXT1]]
+    ; SI: [[FPTRUNC:%[0-9]+]]:_(s16) = G_FPTRUNC [[FADD]](s32)
+    ; SI: [[FPEXT2:%[0-9]+]]:_(s32) = G_FPEXT [[UV1]](s16)
+    ; SI: [[FPEXT3:%[0-9]+]]:_(s32) = G_FPEXT [[UV4]](s16)
+    ; SI: [[FADD1:%[0-9]+]]:_(s32) = G_FADD [[FPEXT2]], [[FPEXT3]]
+    ; SI: [[FPTRUNC1:%[0-9]+]]:_(s16) = G_FPTRUNC [[FADD1]](s32)
+    ; SI: [[FPEXT4:%[0-9]+]]:_(s32) = G_FPEXT [[UV2]](s16)
+    ; SI: [[FPEXT5:%[0-9]+]]:_(s32) = G_FPEXT [[UV5]](s16)
+    ; SI: [[FADD2:%[0-9]+]]:_(s32) = G_FADD [[FPEXT4]], [[FPEXT5]]
+    ; SI: [[FPTRUNC2:%[0-9]+]]:_(s16) = G_FPTRUNC [[FADD2]](s32)
+    ; SI: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s16>) = G_BUILD_VECTOR [[FPTRUNC]](s16), [[FPTRUNC1]](s16), [[FPTRUNC2]](s16)
+    ; SI: S_NOP 0, implicit [[BUILD_VECTOR]](<3 x s16>)
+    ; VI-LABEL: name: test_fadd_v3s16
+    ; VI: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; VI: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
+    ; VI: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; VI: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF1]](<4 x s16>), 0
+    ; VI: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16), [[UV2:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[EXTRACT]](<3 x s16>)
+    ; VI: [[UV3:%[0-9]+]]:_(s16), [[UV4:%[0-9]+]]:_(s16), [[UV5:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[EXTRACT1]](<3 x s16>)
+    ; VI: [[FADD:%[0-9]+]]:_(s16) = G_FADD [[UV]], [[UV3]]
+    ; VI: [[FADD1:%[0-9]+]]:_(s16) = G_FADD [[UV1]], [[UV4]]
+    ; VI: [[FADD2:%[0-9]+]]:_(s16) = G_FADD [[UV2]], [[UV5]]
+    ; VI: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s16>) = G_BUILD_VECTOR [[FADD]](s16), [[FADD1]](s16), [[FADD2]](s16)
+    ; VI: S_NOP 0, implicit [[BUILD_VECTOR]](<3 x s16>)
+    ; GFX9-LABEL: name: test_fadd_v3s16
+    ; GFX9: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; GFX9: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
+    ; GFX9: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; GFX9: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF1]](<4 x s16>), 0
+    ; GFX9: [[DEF2:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; GFX9: [[EXTRACT2:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF2]](<4 x s16>), 0
+    ; GFX9: [[EXTRACT3:%[0-9]+]]:_(<2 x s16>) = G_EXTRACT [[EXTRACT]](<3 x s16>), 0
+    ; GFX9: [[EXTRACT4:%[0-9]+]]:_(<2 x s16>) = G_EXTRACT [[EXTRACT1]](<3 x s16>), 0
+    ; GFX9: [[FADD:%[0-9]+]]:_(<2 x s16>) = G_FADD [[EXTRACT3]], [[EXTRACT4]]
+    ; GFX9: [[INSERT:%[0-9]+]]:_(<3 x s16>) = G_INSERT [[EXTRACT2]], [[FADD]](<2 x s16>), 0
+    ; GFX9: [[EXTRACT5:%[0-9]+]]:_(s16) = G_EXTRACT [[EXTRACT]](<3 x s16>), 32
+    ; GFX9: [[EXTRACT6:%[0-9]+]]:_(s16) = G_EXTRACT [[EXTRACT1]](<3 x s16>), 32
+    ; GFX9: [[FADD1:%[0-9]+]]:_(s16) = G_FADD [[EXTRACT5]], [[EXTRACT6]]
+    ; GFX9: [[INSERT1:%[0-9]+]]:_(<3 x s16>) = G_INSERT [[INSERT]], [[FADD1]](s16), 32
+    ; GFX9: S_NOP 0, implicit [[INSERT1]](<3 x s16>)
+    %0:_(<3 x s16>) = G_IMPLICIT_DEF
+    %1:_(<3 x s16>) = G_IMPLICIT_DEF
+    %2:_(<3 x s16>) = G_FADD %0, %1
+    S_NOP 0, implicit %2
+...
+
+---
+name: test_fadd_v4s16
+body: |
+  bb.0.entry:
+    liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
+
+    ; SI-LABEL: name: test_fadd_v4s16
+    ; SI: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
+    ; SI: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
+    ; SI: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16), [[UV2:%[0-9]+]]:_(s16), [[UV3:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
+    ; SI: [[UV4:%[0-9]+]]:_(s16), [[UV5:%[0-9]+]]:_(s16), [[UV6:%[0-9]+]]:_(s16), [[UV7:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
+    ; SI: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT [[UV]](s16)
+    ; SI: [[FPEXT1:%[0-9]+]]:_(s32) = G_FPEXT [[UV4]](s16)
+    ; SI: [[FADD:%[0-9]+]]:_(s32) = G_FADD [[FPEXT]], [[FPEXT1]]
+    ; SI: [[FPTRUNC:%[0-9]+]]:_(s16) = G_FPTRUNC [[FADD]](s32)
+    ; SI: [[FPEXT2:%[0-9]+]]:_(s32) = G_FPEXT [[UV1]](s16)
+    ; SI: [[FPEXT3:%[0-9]+]]:_(s32) = G_FPEXT [[UV5]](s16)
+    ; SI: [[FADD1:%[0-9]+]]:_(s32) = G_FADD [[FPEXT2]], [[FPEXT3]]
+    ; SI: [[FPTRUNC1:%[0-9]+]]:_(s16) = G_FPTRUNC [[FADD1]](s32)
+    ; SI: [[FPEXT4:%[0-9]+]]:_(s32) = G_FPEXT [[UV2]](s16)
+    ; SI: [[FPEXT5:%[0-9]+]]:_(s32) = G_FPEXT [[UV6]](s16)
+    ; SI: [[FADD2:%[0-9]+]]:_(s32) = G_FADD [[FPEXT4]], [[FPEXT5]]
+    ; SI: [[FPTRUNC2:%[0-9]+]]:_(s16) = G_FPTRUNC [[FADD2]](s32)
+    ; SI: [[FPEXT6:%[0-9]+]]:_(s32) = G_FPEXT [[UV3]](s16)
+    ; SI: [[FPEXT7:%[0-9]+]]:_(s32) = G_FPEXT [[UV7]](s16)
+    ; SI: [[FADD3:%[0-9]+]]:_(s32) = G_FADD [[FPEXT6]], [[FPEXT7]]
+    ; SI: [[FPTRUNC3:%[0-9]+]]:_(s16) = G_FPTRUNC [[FADD3]](s32)
+    ; SI: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s16>) = G_BUILD_VECTOR [[FPTRUNC]](s16), [[FPTRUNC1]](s16), [[FPTRUNC2]](s16), [[FPTRUNC3]](s16)
+    ; SI: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<4 x s16>)
+    ; VI-LABEL: name: test_fadd_v4s16
+    ; VI: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
+    ; VI: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
+    ; VI: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16), [[UV2:%[0-9]+]]:_(s16), [[UV3:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
+    ; VI: [[UV4:%[0-9]+]]:_(s16), [[UV5:%[0-9]+]]:_(s16), [[UV6:%[0-9]+]]:_(s16), [[UV7:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
+    ; VI: [[FADD:%[0-9]+]]:_(s16) = G_FADD [[UV]], [[UV4]]
+    ; VI: [[FADD1:%[0-9]+]]:_(s16) = G_FADD [[UV1]], [[UV5]]
+    ; VI: [[FADD2:%[0-9]+]]:_(s16) = G_FADD [[UV2]], [[UV6]]
+    ; VI: [[FADD3:%[0-9]+]]:_(s16) = G_FADD [[UV3]], [[UV7]]
+    ; VI: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s16>) = G_BUILD_VECTOR [[FADD]](s16), [[FADD1]](s16), [[FADD2]](s16), [[FADD3]](s16)
+    ; VI: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<4 x s16>)
+    ; GFX9-LABEL: name: test_fadd_v4s16
+    ; GFX9: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
+    ; GFX9: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
+    ; GFX9: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
+    ; GFX9: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
+    ; GFX9: [[FADD:%[0-9]+]]:_(<2 x s16>) = G_FADD [[UV]], [[UV2]]
+    ; GFX9: [[FADD1:%[0-9]+]]:_(<2 x s16>) = G_FADD [[UV1]], [[UV3]]
+    ; GFX9: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[FADD]](<2 x s16>), [[FADD1]](<2 x s16>)
+    ; GFX9: $vgpr0_vgpr1 = COPY [[CONCAT_VECTORS]](<4 x s16>)
+    %0:_(<4 x s16>) = COPY $vgpr0_vgpr1
+    %1:_(<4 x s16>) = COPY $vgpr2_vgpr3
+    %2:_(<4 x s16>) = G_FADD %0, %1
+    $vgpr0_vgpr1 = COPY %2
+...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/legalize-fcanonicalize.mir b/test/CodeGen/AMDGPU/GlobalISel/legalize-fcanonicalize.mir
new file mode 100644
index 0000000000000000000000000000000000000000..3bd89850af109f03bd4e3880993381e52c0fe80e
--- /dev/null
+++ b/test/CodeGen/AMDGPU/GlobalISel/legalize-fcanonicalize.mir
@@ -0,0 +1,290 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=tahiti -run-pass=legalizer %s -o - | FileCheck -check-prefix=SI  %s
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -run-pass=legalizer %s -o - | FileCheck -check-prefix=VI %s
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx900 -run-pass=legalizer %s -o - | FileCheck -check-prefix=GFX9  %s
+
+
+---
+name: test_fcanonicalize_s32
+body: |
+  bb.0:
+    liveins: $vgpr0
+
+    ; SI-LABEL: name: test_fcanonicalize_s32
+    ; SI: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; VI-LABEL: name: test_fcanonicalize_s32
+    ; VI: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; GFX9-LABEL: name: test_fcanonicalize_s32
+    ; GFX9: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = G_FCANONICALIZE %0
+...
+---
+name: test_fcanonicalize_s64
+body: |
+  bb.0:
+    liveins: $vgpr0
+
+    ; SI-LABEL: name: test_fcanonicalize_s64
+    ; SI: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
+    ; VI-LABEL: name: test_fcanonicalize_s64
+    ; VI: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
+    ; GFX9-LABEL: name: test_fcanonicalize_s64
+    ; GFX9: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
+    %0:_(s64) = COPY $vgpr0_vgpr1
+    %1:_(s64) = G_FCANONICALIZE %0
+...
+---
+name: test_fcanonicalize_s16
+body: |
+  bb.0:
+    liveins: $vgpr0
+
+    ; SI-LABEL: name: test_fcanonicalize_s16
+    ; SI: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; SI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+    ; SI: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC]](s16)
+    ; SI: [[FCANONICALIZE:%[0-9]+]]:_(s32) = G_FCANONICALIZE [[FPEXT]]
+    ; SI: [[FPTRUNC:%[0-9]+]]:_(s16) = G_FPTRUNC [[FCANONICALIZE]](s32)
+    ; SI: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[FPTRUNC]](s16)
+    ; SI: $vgpr0 = COPY [[ANYEXT]](s32)
+    ; VI-LABEL: name: test_fcanonicalize_s16
+    ; VI: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; VI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+    ; VI: [[FCANONICALIZE:%[0-9]+]]:_(s16) = G_FCANONICALIZE [[TRUNC]]
+    ; VI: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[FCANONICALIZE]](s16)
+    ; VI: $vgpr0 = COPY [[ANYEXT]](s32)
+    ; GFX9-LABEL: name: test_fcanonicalize_s16
+    ; GFX9: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; GFX9: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+    ; GFX9: [[FCANONICALIZE:%[0-9]+]]:_(s16) = G_FCANONICALIZE [[TRUNC]]
+    ; GFX9: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[FCANONICALIZE]](s16)
+    ; GFX9: $vgpr0 = COPY [[ANYEXT]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s16) = G_TRUNC %0
+    %2:_(s16) = G_FCANONICALIZE %1
+    %3:_(s32) = G_ANYEXT %2
+    $vgpr0 = COPY %3
+...
+
+---
+name: test_fcanonicalize_v2s32
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; SI-LABEL: name: test_fcanonicalize_v2s32
+    ; SI: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    ; SI: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
+    ; SI: [[FCANONICALIZE:%[0-9]+]]:_(s32) = G_FCANONICALIZE [[UV]]
+    ; SI: [[FCANONICALIZE1:%[0-9]+]]:_(s32) = G_FCANONICALIZE [[UV1]]
+    ; SI: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[FCANONICALIZE]](s32), [[FCANONICALIZE1]](s32)
+    ; SI: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
+    ; VI-LABEL: name: test_fcanonicalize_v2s32
+    ; VI: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    ; VI: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
+    ; VI: [[FCANONICALIZE:%[0-9]+]]:_(s32) = G_FCANONICALIZE [[UV]]
+    ; VI: [[FCANONICALIZE1:%[0-9]+]]:_(s32) = G_FCANONICALIZE [[UV1]]
+    ; VI: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[FCANONICALIZE]](s32), [[FCANONICALIZE1]](s32)
+    ; VI: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
+    ; GFX9-LABEL: name: test_fcanonicalize_v2s32
+    ; GFX9: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    ; GFX9: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
+    ; GFX9: [[FCANONICALIZE:%[0-9]+]]:_(s32) = G_FCANONICALIZE [[UV]]
+    ; GFX9: [[FCANONICALIZE1:%[0-9]+]]:_(s32) = G_FCANONICALIZE [[UV1]]
+    ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[FCANONICALIZE]](s32), [[FCANONICALIZE1]](s32)
+    ; GFX9: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
+    %0:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    %1:_(<2 x s32>) = G_FCANONICALIZE %0
+    $vgpr0_vgpr1 = COPY %1
+...
+
+---
+name: test_fcanonicalize_v3s32
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2
+
+    ; SI-LABEL: name: test_fcanonicalize_v3s32
+    ; SI: [[COPY:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr0_vgpr1_vgpr2
+    ; SI: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<3 x s32>)
+    ; SI: [[FCANONICALIZE:%[0-9]+]]:_(s32) = G_FCANONICALIZE [[UV]]
+    ; SI: [[FCANONICALIZE1:%[0-9]+]]:_(s32) = G_FCANONICALIZE [[UV1]]
+    ; SI: [[FCANONICALIZE2:%[0-9]+]]:_(s32) = G_FCANONICALIZE [[UV2]]
+    ; SI: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[FCANONICALIZE]](s32), [[FCANONICALIZE1]](s32), [[FCANONICALIZE2]](s32)
+    ; SI: $vgpr0_vgpr1_vgpr2 = COPY [[BUILD_VECTOR]](<3 x s32>)
+    ; VI-LABEL: name: test_fcanonicalize_v3s32
+    ; VI: [[COPY:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr0_vgpr1_vgpr2
+    ; VI: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<3 x s32>)
+    ; VI: [[FCANONICALIZE:%[0-9]+]]:_(s32) = G_FCANONICALIZE [[UV]]
+    ; VI: [[FCANONICALIZE1:%[0-9]+]]:_(s32) = G_FCANONICALIZE [[UV1]]
+    ; VI: [[FCANONICALIZE2:%[0-9]+]]:_(s32) = G_FCANONICALIZE [[UV2]]
+    ; VI: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[FCANONICALIZE]](s32), [[FCANONICALIZE1]](s32), [[FCANONICALIZE2]](s32)
+    ; VI: $vgpr0_vgpr1_vgpr2 = COPY [[BUILD_VECTOR]](<3 x s32>)
+    ; GFX9-LABEL: name: test_fcanonicalize_v3s32
+    ; GFX9: [[COPY:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr0_vgpr1_vgpr2
+    ; GFX9: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<3 x s32>)
+    ; GFX9: [[FCANONICALIZE:%[0-9]+]]:_(s32) = G_FCANONICALIZE [[UV]]
+    ; GFX9: [[FCANONICALIZE1:%[0-9]+]]:_(s32) = G_FCANONICALIZE [[UV1]]
+    ; GFX9: [[FCANONICALIZE2:%[0-9]+]]:_(s32) = G_FCANONICALIZE [[UV2]]
+    ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[FCANONICALIZE]](s32), [[FCANONICALIZE1]](s32), [[FCANONICALIZE2]](s32)
+    ; GFX9: $vgpr0_vgpr1_vgpr2 = COPY [[BUILD_VECTOR]](<3 x s32>)
+    %0:_(<3 x s32>) = COPY $vgpr0_vgpr1_vgpr2
+    %1:_(<3 x  s32>) = G_FCANONICALIZE %0
+    $vgpr0_vgpr1_vgpr2 = COPY %1
+...
+
+---
+name: test_fcanonicalize_v2s64
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3
+
+    ; SI-LABEL: name: test_fcanonicalize_v2s64
+    ; SI: [[COPY:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; SI: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](<2 x s64>)
+    ; SI: [[FCANONICALIZE:%[0-9]+]]:_(s64) = G_FCANONICALIZE [[UV]]
+    ; SI: [[FCANONICALIZE1:%[0-9]+]]:_(s64) = G_FCANONICALIZE [[UV1]]
+    ; SI: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[FCANONICALIZE]](s64), [[FCANONICALIZE1]](s64)
+    ; SI: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>)
+    ; VI-LABEL: name: test_fcanonicalize_v2s64
+    ; VI: [[COPY:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; VI: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](<2 x s64>)
+    ; VI: [[FCANONICALIZE:%[0-9]+]]:_(s64) = G_FCANONICALIZE [[UV]]
+    ; VI: [[FCANONICALIZE1:%[0-9]+]]:_(s64) = G_FCANONICALIZE [[UV1]]
+    ; VI: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[FCANONICALIZE]](s64), [[FCANONICALIZE1]](s64)
+    ; VI: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>)
+    ; GFX9-LABEL: name: test_fcanonicalize_v2s64
+    ; GFX9: [[COPY:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; GFX9: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](<2 x s64>)
+    ; GFX9: [[FCANONICALIZE:%[0-9]+]]:_(s64) = G_FCANONICALIZE [[UV]]
+    ; GFX9: [[FCANONICALIZE1:%[0-9]+]]:_(s64) = G_FCANONICALIZE [[UV1]]
+    ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[FCANONICALIZE]](s64), [[FCANONICALIZE1]](s64)
+    ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>)
+    %0:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    %1:_(<2 x s64>) = G_FCANONICALIZE %0
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %1
+...
+
+---
+name: test_fcanonicalize_v2s16
+body: |
+  bb.0:
+    liveins: $vgpr0
+
+    ; SI-LABEL: name: test_fcanonicalize_v2s16
+    ; SI: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
+    ; SI: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY]](<2 x s16>)
+    ; SI: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT [[UV]](s16)
+    ; SI: [[FCANONICALIZE:%[0-9]+]]:_(s32) = G_FCANONICALIZE [[FPEXT]]
+    ; SI: [[FPTRUNC:%[0-9]+]]:_(s16) = G_FPTRUNC [[FCANONICALIZE]](s32)
+    ; SI: [[FPEXT1:%[0-9]+]]:_(s32) = G_FPEXT [[UV1]](s16)
+    ; SI: [[FCANONICALIZE1:%[0-9]+]]:_(s32) = G_FCANONICALIZE [[FPEXT1]]
+    ; SI: [[FPTRUNC1:%[0-9]+]]:_(s16) = G_FPTRUNC [[FCANONICALIZE1]](s32)
+    ; SI: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[FPTRUNC]](s16), [[FPTRUNC1]](s16)
+    ; SI: $vgpr0 = COPY [[BUILD_VECTOR]](<2 x s16>)
+    ; VI-LABEL: name: test_fcanonicalize_v2s16
+    ; VI: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
+    ; VI: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY]](<2 x s16>)
+    ; VI: [[FCANONICALIZE:%[0-9]+]]:_(s16) = G_FCANONICALIZE [[UV]]
+    ; VI: [[FCANONICALIZE1:%[0-9]+]]:_(s16) = G_FCANONICALIZE [[UV1]]
+    ; VI: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[FCANONICALIZE]](s16), [[FCANONICALIZE1]](s16)
+    ; VI: $vgpr0 = COPY [[BUILD_VECTOR]](<2 x s16>)
+    ; GFX9-LABEL: name: test_fcanonicalize_v2s16
+    ; GFX9: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
+    ; GFX9: [[FCANONICALIZE:%[0-9]+]]:_(<2 x s16>) = G_FCANONICALIZE [[COPY]]
+    ; GFX9: $vgpr0 = COPY [[FCANONICALIZE]](<2 x s16>)
+    %0:_(<2 x s16>) = COPY $vgpr0
+    %1:_(<2 x s16>) = G_FCANONICALIZE %0
+    $vgpr0 = COPY %1
+...
+
+---
+name: test_fcanonicalize_v3s16
+body: |
+  bb.0:
+
+    ; SI-LABEL: name: test_fcanonicalize_v3s16
+    ; SI: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; SI: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
+    ; SI: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16), [[UV2:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[EXTRACT]](<3 x s16>)
+    ; SI: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT [[UV]](s16)
+    ; SI: [[FCANONICALIZE:%[0-9]+]]:_(s32) = G_FCANONICALIZE [[FPEXT]]
+    ; SI: [[FPTRUNC:%[0-9]+]]:_(s16) = G_FPTRUNC [[FCANONICALIZE]](s32)
+    ; SI: [[FPEXT1:%[0-9]+]]:_(s32) = G_FPEXT [[UV1]](s16)
+    ; SI: [[FCANONICALIZE1:%[0-9]+]]:_(s32) = G_FCANONICALIZE [[FPEXT1]]
+    ; SI: [[FPTRUNC1:%[0-9]+]]:_(s16) = G_FPTRUNC [[FCANONICALIZE1]](s32)
+    ; SI: [[FPEXT2:%[0-9]+]]:_(s32) = G_FPEXT [[UV2]](s16)
+    ; SI: [[FCANONICALIZE2:%[0-9]+]]:_(s32) = G_FCANONICALIZE [[FPEXT2]]
+    ; SI: [[FPTRUNC2:%[0-9]+]]:_(s16) = G_FPTRUNC [[FCANONICALIZE2]](s32)
+    ; SI: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s16>) = G_BUILD_VECTOR [[FPTRUNC]](s16), [[FPTRUNC1]](s16), [[FPTRUNC2]](s16)
+    ; SI: S_NOP 0, implicit [[BUILD_VECTOR]](<3 x s16>)
+    ; VI-LABEL: name: test_fcanonicalize_v3s16
+    ; VI: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; VI: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
+    ; VI: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16), [[UV2:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[EXTRACT]](<3 x s16>)
+    ; VI: [[FCANONICALIZE:%[0-9]+]]:_(s16) = G_FCANONICALIZE [[UV]]
+    ; VI: [[FCANONICALIZE1:%[0-9]+]]:_(s16) = G_FCANONICALIZE [[UV1]]
+    ; VI: [[FCANONICALIZE2:%[0-9]+]]:_(s16) = G_FCANONICALIZE [[UV2]]
+    ; VI: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s16>) = G_BUILD_VECTOR [[FCANONICALIZE]](s16), [[FCANONICALIZE1]](s16), [[FCANONICALIZE2]](s16)
+    ; VI: S_NOP 0, implicit [[BUILD_VECTOR]](<3 x s16>)
+    ; GFX9-LABEL: name: test_fcanonicalize_v3s16
+    ; GFX9: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; GFX9: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
+    ; GFX9: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; GFX9: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF1]](<4 x s16>), 0
+    ; GFX9: [[EXTRACT2:%[0-9]+]]:_(<2 x s16>) = G_EXTRACT [[EXTRACT]](<3 x s16>), 0
+    ; GFX9: [[FCANONICALIZE:%[0-9]+]]:_(<2 x s16>) = G_FCANONICALIZE [[EXTRACT2]]
+    ; GFX9: [[INSERT:%[0-9]+]]:_(<3 x s16>) = G_INSERT [[EXTRACT1]], [[FCANONICALIZE]](<2 x s16>), 0
+    ; GFX9: [[EXTRACT3:%[0-9]+]]:_(s16) = G_EXTRACT [[EXTRACT]](<3 x s16>), 32
+    ; GFX9: [[FCANONICALIZE1:%[0-9]+]]:_(s16) = G_FCANONICALIZE [[EXTRACT3]]
+    ; GFX9: [[INSERT1:%[0-9]+]]:_(<3 x s16>) = G_INSERT [[INSERT]], [[FCANONICALIZE1]](s16), 32
+    ; GFX9: S_NOP 0, implicit [[INSERT1]](<3 x s16>)
+    %0:_(<3 x s16>) = G_IMPLICIT_DEF
+    %1:_(<3 x s16>) = G_FCANONICALIZE %0
+    S_NOP 0, implicit %1
+...
+
+---
+name: test_fcanonicalize_v4s16
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; SI-LABEL: name: test_fcanonicalize_v4s16
+    ; SI: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
+    ; SI: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16), [[UV2:%[0-9]+]]:_(s16), [[UV3:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
+    ; SI: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT [[UV]](s16)
+    ; SI: [[FCANONICALIZE:%[0-9]+]]:_(s32) = G_FCANONICALIZE [[FPEXT]]
+    ; SI: [[FPTRUNC:%[0-9]+]]:_(s16) = G_FPTRUNC [[FCANONICALIZE]](s32)
+    ; SI: [[FPEXT1:%[0-9]+]]:_(s32) = G_FPEXT [[UV1]](s16)
+    ; SI: [[FCANONICALIZE1:%[0-9]+]]:_(s32) = G_FCANONICALIZE [[FPEXT1]]
+    ; SI: [[FPTRUNC1:%[0-9]+]]:_(s16) = G_FPTRUNC [[FCANONICALIZE1]](s32)
+    ; SI: [[FPEXT2:%[0-9]+]]:_(s32) = G_FPEXT [[UV2]](s16)
+    ; SI: [[FCANONICALIZE2:%[0-9]+]]:_(s32) = G_FCANONICALIZE [[FPEXT2]]
+    ; SI: [[FPTRUNC2:%[0-9]+]]:_(s16) = G_FPTRUNC [[FCANONICALIZE2]](s32)
+    ; SI: [[FPEXT3:%[0-9]+]]:_(s32) = G_FPEXT [[UV3]](s16)
+    ; SI: [[FCANONICALIZE3:%[0-9]+]]:_(s32) = G_FCANONICALIZE [[FPEXT3]]
+    ; SI: [[FPTRUNC3:%[0-9]+]]:_(s16) = G_FPTRUNC [[FCANONICALIZE3]](s32)
+    ; SI: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s16>) = G_BUILD_VECTOR [[FPTRUNC]](s16), [[FPTRUNC1]](s16), [[FPTRUNC2]](s16), [[FPTRUNC3]](s16)
+    ; SI: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<4 x s16>)
+    ; VI-LABEL: name: test_fcanonicalize_v4s16
+    ; VI: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
+    ; VI: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16), [[UV2:%[0-9]+]]:_(s16), [[UV3:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
+    ; VI: [[FCANONICALIZE:%[0-9]+]]:_(s16) = G_FCANONICALIZE [[UV]]
+    ; VI: [[FCANONICALIZE1:%[0-9]+]]:_(s16) = G_FCANONICALIZE [[UV1]]
+    ; VI: [[FCANONICALIZE2:%[0-9]+]]:_(s16) = G_FCANONICALIZE [[UV2]]
+    ; VI: [[FCANONICALIZE3:%[0-9]+]]:_(s16) = G_FCANONICALIZE [[UV3]]
+    ; VI: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s16>) = G_BUILD_VECTOR [[FCANONICALIZE]](s16), [[FCANONICALIZE1]](s16), [[FCANONICALIZE2]](s16), [[FCANONICALIZE3]](s16)
+    ; VI: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<4 x s16>)
+    ; GFX9-LABEL: name: test_fcanonicalize_v4s16
+    ; GFX9: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
+    ; GFX9: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
+    ; GFX9: [[FCANONICALIZE:%[0-9]+]]:_(<2 x s16>) = G_FCANONICALIZE [[UV]]
+    ; GFX9: [[FCANONICALIZE1:%[0-9]+]]:_(<2 x s16>) = G_FCANONICALIZE [[UV1]]
+    ; GFX9: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[FCANONICALIZE]](<2 x s16>), [[FCANONICALIZE1]](<2 x s16>)
+    ; GFX9: $vgpr0_vgpr1 = COPY [[CONCAT_VECTORS]](<4 x s16>)
+    %0:_(<4 x s16>) = COPY $vgpr0_vgpr1
+    %1:_(<4 x s16>) = G_FCANONICALIZE %0
+    $vgpr0_vgpr1 = COPY %1
+...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/legalize-fcmp.mir b/test/CodeGen/AMDGPU/GlobalISel/legalize-fcmp.mir
index c09903f25730767a8b278415c455e441168bc204..be0b944b6a6487a4e4f93e1723672cb337e44d83 100644
--- a/test/CodeGen/AMDGPU/GlobalISel/legalize-fcmp.mir
+++ b/test/CodeGen/AMDGPU/GlobalISel/legalize-fcmp.mir
@@ -1,27 +1,153 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -march=amdgcn -mcpu=fiji  -run-pass=legalizer %s -o - | FileCheck %s
+# RUN: llc -O0 -march=amdgcn -mcpu=fiji  -run-pass=legalizer %s -o - | FileCheck %s
 
 ---
-name: test_fcmp_f32
+name: test_fcmp_s32
 body: |
-  bb.0:
+  bb.0.entry:
     liveins: $vgpr0
-    ; CHECK-LABEL: name: test_fcmp_f32
+    ; CHECK-LABEL: name: test_fcmp_s32
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
     ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[FCMP:%[0-9]+]]:_(s1) = G_FCMP floatpred(oeq), [[C]](s32), [[COPY]]
+    ; CHECK: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[FCMP]](s1), [[C]], [[COPY]]
+    ; CHECK: $vgpr0 = COPY [[SELECT]](s32)
     %0:_(s32) = G_CONSTANT i32 0
     %1:_(s32) = COPY $vgpr0
-
-    %2:_(s1) = G_FCMP floatpred(uge), %0, %1
+    %2:_(s1) = G_FCMP floatpred(oeq), %0, %1
+    %3:_(s32) = G_SELECT %2, %0, %1
+    $vgpr0 = COPY %3
 ...
+
 ---
-name: test_fcmp_f64
+name: test_fcmp_s64
 body: |
-  bb.0:
+  bb.0.entry:
     liveins: $vgpr0_vgpr1
-    ; CHECK-LABEL: name: test_fcmp_f64
+    ; CHECK-LABEL: name: test_fcmp_s64
+    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
     ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
+    ; CHECK: [[FCMP:%[0-9]+]]:_(s1) = G_FCMP floatpred(oeq), [[C]](s64), [[COPY]]
+    ; CHECK: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[FCMP]](s1), [[C]], [[COPY]]
+    ; CHECK: $vgpr0_vgpr1 = COPY [[SELECT]](s64)
     %0:_(s64) = G_CONSTANT i64 0
     %1:_(s64) = COPY $vgpr0_vgpr1
+    %2:_(s1) = G_FCMP floatpred(oeq), %0, %1
+    %3:_(s64) = G_SELECT %2, %0, %1
+    $vgpr0_vgpr1 = COPY %3
+...
+
+---
+name: test_fcmp_s16
+body: |
+  bb.0.entry:
+    liveins: $vgpr0
+    ; CHECK-LABEL: name: test_fcmp_s16
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; CHECK: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[C]](s32)
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+    ; CHECK: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC]](s16)
+    ; CHECK: [[FPEXT1:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC1]](s16)
+    ; CHECK: [[FCMP:%[0-9]+]]:_(s1) = G_FCMP floatpred(oeq), [[FPEXT]](s32), [[FPEXT1]]
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY [[C]](s32)
+    ; CHECK: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; CHECK: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[FCMP]](s1), [[COPY1]], [[COPY2]]
+    ; CHECK: [[COPY3:%[0-9]+]]:_(s32) = COPY [[SELECT]](s32)
+    ; CHECK: $vgpr0 = COPY [[COPY3]](s32)
+    %0:_(s16) = G_CONSTANT i16 0
+    %1:_(s32) = COPY $vgpr0
+    %2:_(s16) = G_TRUNC %1
+    %3:_(s1) = G_FCMP floatpred(oeq), %0, %2
+    %4:_(s16) = G_SELECT %3, %0, %2
+    %5:_(s32) = G_ANYEXT %4
+    $vgpr0 = COPY %5
+...
+
+---
+name: test_fcmp_v2s32
+body: |
+  bb.0.entry:
+    liveins: $vgpr0_vgpr1
+    ; CHECK-LABEL: name: test_fcmp_v2s32
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    ; CHECK: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
+    ; CHECK: [[FCMP:%[0-9]+]]:_(s1) = G_FCMP floatpred(oeq), [[C]](s32), [[UV]]
+    ; CHECK: [[FCMP1:%[0-9]+]]:_(s1) = G_FCMP floatpred(oeq), [[C]](s32), [[UV1]]
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s1>) = G_BUILD_VECTOR [[FCMP]](s1), [[FCMP1]](s1)
+    ; CHECK: S_NOP 0, implicit [[BUILD_VECTOR]](<2 x s1>)
+    %0:_(s32) = G_CONSTANT i32 0
+    %1:_(<2 x s32>) = G_BUILD_VECTOR %0, %0
+    %2:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    %3:_(<2 x s1>) = G_FCMP floatpred(oeq), %1, %2
+    S_NOP 0, implicit %3
+...
+
+---
+name: test_fcmp_v2s32_flags
+body: |
+  bb.0.entry:
+    liveins: $vgpr0_vgpr1
+    ; CHECK-LABEL: name: test_fcmp_v2s32_flags
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    ; CHECK: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
+    ; CHECK: %8:_(s1) = nnan G_FCMP floatpred(oeq), [[C]](s32), [[UV]]
+    ; CHECK: %9:_(s1) = nnan G_FCMP floatpred(oeq), [[C]](s32), [[UV1]]
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s1>) = G_BUILD_VECTOR %8(s1), %9(s1)
+    ; CHECK: S_NOP 0, implicit [[BUILD_VECTOR]](<2 x s1>)
+    %0:_(s32) = G_CONSTANT i32 0
+    %1:_(<2 x s32>) = G_BUILD_VECTOR %0, %0
+    %2:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    %3:_(<2 x s1>) = nnan G_FCMP floatpred(oeq), %1, %2
+    S_NOP 0, implicit %3
+...
+
+---
+name: test_fcmp_v3s32
+body: |
+  bb.0.entry:
+    liveins: $vgpr0_vgpr1_vgpr2
+
+    ; CHECK-LABEL: name: test_fcmp_v3s32
+    ; CHECK: [[DEF:%[0-9]+]]:_(<3 x s32>) = G_IMPLICIT_DEF
+    ; CHECK: [[COPY:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr0_vgpr1_vgpr2
+    ; CHECK: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[DEF]](<3 x s32>)
+    ; CHECK: [[UV3:%[0-9]+]]:_(s32), [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<3 x s32>)
+    ; CHECK: [[FCMP:%[0-9]+]]:_(s1) = G_FCMP floatpred(oeq), [[UV]](s32), [[UV3]]
+    ; CHECK: [[FCMP1:%[0-9]+]]:_(s1) = G_FCMP floatpred(oeq), [[UV1]](s32), [[UV4]]
+    ; CHECK: [[FCMP2:%[0-9]+]]:_(s1) = G_FCMP floatpred(oeq), [[UV2]](s32), [[UV5]]
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s1>) = G_BUILD_VECTOR [[FCMP]](s1), [[FCMP1]](s1), [[FCMP2]](s1)
+    ; CHECK: S_NOP 0, implicit [[BUILD_VECTOR]](<3 x s1>)
+    %0:_(<3 x s32>) = G_IMPLICIT_DEF
+    %1:_(<3 x s32>) = COPY $vgpr0_vgpr1_vgpr2
+    %2:_(<3 x s1>) = G_FCMP floatpred(oeq), %0, %1
+    S_NOP 0, implicit %2
+...
+
+---
+name: test_fcmp_v4s32
+body: |
+  bb.0.entry:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3
+
+    ; CHECK-LABEL: name: test_fcmp_v4s32
+    ; CHECK: [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+    ; CHECK: [[LOAD:%[0-9]+]]:_(<4 x s32>) = G_LOAD [[DEF]](p1) :: (volatile load 16)
+    ; CHECK: [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; CHECK: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[LOAD]](<4 x s32>)
+    ; CHECK: [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32), [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<4 x s32>)
+    ; CHECK: [[FCMP:%[0-9]+]]:_(s1) = G_FCMP floatpred(oeq), [[UV]](s32), [[UV4]]
+    ; CHECK: [[FCMP1:%[0-9]+]]:_(s1) = G_FCMP floatpred(oeq), [[UV1]](s32), [[UV5]]
+    ; CHECK: [[FCMP2:%[0-9]+]]:_(s1) = G_FCMP floatpred(oeq), [[UV2]](s32), [[UV6]]
+    ; CHECK: [[FCMP3:%[0-9]+]]:_(s1) = G_FCMP floatpred(oeq), [[UV3]](s32), [[UV7]]
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s1>) = G_BUILD_VECTOR [[FCMP]](s1), [[FCMP1]](s1), [[FCMP2]](s1), [[FCMP3]](s1)
+    ; CHECK: S_NOP 0, implicit [[BUILD_VECTOR]](<4 x s1>)
+    %0:_(p1) = G_IMPLICIT_DEF
+    %1:_(<4 x s32>) = G_LOAD %0 :: (volatile load 16)
+    %2:_(<4 x s32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    %3:_(<4 x s1>) = G_FCMP floatpred(oeq) , %1, %2
+    S_NOP 0, implicit %3
 
-    %2:_(s1) = G_FCMP floatpred(uge), %0, %1
 ...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/legalize-fconstant.mir b/test/CodeGen/AMDGPU/GlobalISel/legalize-fconstant.mir
new file mode 100644
index 0000000000000000000000000000000000000000..0d46f5fe0026d48d7bec6a619a74bbe4059b7567
--- /dev/null
+++ b/test/CodeGen/AMDGPU/GlobalISel/legalize-fconstant.mir
@@ -0,0 +1,51 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=tahiti -O0 -run-pass=legalizer %s -o - | FileCheck -check-prefix=SI  %s
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -O0 -run-pass=legalizer %s -o - | FileCheck -check-prefix=VI %s
+
+---
+name: test_fconstant_s32
+body: |
+  bb.0:
+
+    ; SI-LABEL: name: test_fconstant_s32
+    ; SI: [[C:%[0-9]+]]:_(s32) = G_FCONSTANT float 1.000000e+00
+    ; SI: $vgpr0 = COPY [[C]](s32)
+    ; VI-LABEL: name: test_fconstant_s32
+    ; VI: [[C:%[0-9]+]]:_(s32) = G_FCONSTANT float 1.000000e+00
+    ; VI: $vgpr0 = COPY [[C]](s32)
+    %0:_(s32) = G_FCONSTANT float 1.0
+    $vgpr0 = COPY %0
+...
+---
+name: test_fconstant_s64
+body: |
+  bb.0:
+
+    ; SI-LABEL: name: test_fconstant_s64
+    ; SI: [[C:%[0-9]+]]:_(s64) = G_FCONSTANT double 1.000000e+00
+    ; SI: $vgpr0_vgpr1 = COPY [[C]](s64)
+    ; VI-LABEL: name: test_fconstant_s64
+    ; VI: [[C:%[0-9]+]]:_(s64) = G_FCONSTANT double 1.000000e+00
+    ; VI: $vgpr0_vgpr1 = COPY [[C]](s64)
+    %0:_(s64) = G_FCONSTANT double 1.0
+    $vgpr0_vgpr1 = COPY %0
+...
+
+---
+name: test_fconstant_s16
+body: |
+  bb.0:
+
+    ; SI-LABEL: name: test_fconstant_s16
+    ; SI: [[C:%[0-9]+]]:_(s32) = G_FCONSTANT float 1.000000e+00
+    ; SI: [[FPTRUNC:%[0-9]+]]:_(s16) = G_FPTRUNC [[C]](s32)
+    ; SI: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[FPTRUNC]](s16)
+    ; SI: $vgpr0 = COPY [[ANYEXT]](s32)
+    ; VI-LABEL: name: test_fconstant_s16
+    ; VI: [[C:%[0-9]+]]:_(s16) = G_FCONSTANT half 0xH3C00
+    ; VI: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[C]](s16)
+    ; VI: $vgpr0 = COPY [[ANYEXT]](s32)
+    %0:_(s16) = G_FCONSTANT half 1.0
+    %1:_(s32) = G_ANYEXT %0
+    $vgpr0 = COPY %1
+...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/legalize-fexp.mir b/test/CodeGen/AMDGPU/GlobalISel/legalize-fexp.mir
new file mode 100644
index 0000000000000000000000000000000000000000..d780957daa97e17e0244105496bb1cfcaa1a5358
--- /dev/null
+++ b/test/CodeGen/AMDGPU/GlobalISel/legalize-fexp.mir
@@ -0,0 +1,54 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=tahiti -run-pass=legalizer %s -o - | FileCheck %s
+
+---
+name: test_fexp_s32
+body: |
+  bb.0:
+    liveins: $vgpr0
+
+    ; CHECK-LABEL: name: test_fexp_s32
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[FEXP_:%[0-9]+]]:_(s32) = G_FEXP [[COPY]]
+    ; CHECK: $vgpr0 = COPY [[FEXP_]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = G_FEXP %0
+    $vgpr0 = COPY %1
+...
+
+---
+name: test_fexp_v2s32
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; CHECK-LABEL: name: test_fexp_v2s32
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    ; CHECK: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
+    ; CHECK: [[FEXP_:%[0-9]+]]:_(s32) = G_FEXP [[UV]]
+    ; CHECK: [[FEXP_1:%[0-9]+]]:_(s32) = G_FEXP [[UV1]]
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[FEXP_]](s32), [[FEXP_1]](s32)
+    ; CHECK: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
+    %0:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    %1:_(<2 x s32>) = G_FEXP %0
+    $vgpr0_vgpr1 = COPY %1
+...
+
+---
+name: test_fexp_v3s32
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2
+
+    ; CHECK-LABEL: name: test_fexp_v3s32
+    ; CHECK: [[COPY:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr0_vgpr1_vgpr2
+    ; CHECK: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<3 x s32>)
+    ; CHECK: [[FEXP_:%[0-9]+]]:_(s32) = G_FEXP [[UV]]
+    ; CHECK: [[FEXP_1:%[0-9]+]]:_(s32) = G_FEXP [[UV1]]
+    ; CHECK: [[FEXP_2:%[0-9]+]]:_(s32) = G_FEXP [[UV2]]
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[FEXP_]](s32), [[FEXP_1]](s32), [[FEXP_2]](s32)
+    ; CHECK: $vgpr0_vgpr1_vgpr2 = COPY [[BUILD_VECTOR]](<3 x s32>)
+    %0:_(<3 x s32>) = COPY $vgpr0_vgpr1_vgpr2
+    %1:_(<3 x  s32>) = G_FEXP %0
+    $vgpr0_vgpr1_vgpr2 = COPY %1
+...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/legalize-fexp2.mir b/test/CodeGen/AMDGPU/GlobalISel/legalize-fexp2.mir
new file mode 100644
index 0000000000000000000000000000000000000000..36ee554cd54ea4f31241c141e385382ae912e155
--- /dev/null
+++ b/test/CodeGen/AMDGPU/GlobalISel/legalize-fexp2.mir
@@ -0,0 +1,54 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=tahiti -run-pass=legalizer %s -o - | FileCheck %s
+
+---
+name: test_fexp2_s32
+body: |
+  bb.0:
+    liveins: $vgpr0
+
+    ; CHECK-LABEL: name: test_fexp2_s32
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[FEXP2_:%[0-9]+]]:_(s32) = G_FEXP2 [[COPY]]
+    ; CHECK: $vgpr0 = COPY [[FEXP2_]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = G_FEXP2 %0
+    $vgpr0 = COPY %1
+...
+
+---
+name: test_fexp2_v2s32
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; CHECK-LABEL: name: test_fexp2_v2s32
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    ; CHECK: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
+    ; CHECK: [[FEXP2_:%[0-9]+]]:_(s32) = G_FEXP2 [[UV]]
+    ; CHECK: [[FEXP2_1:%[0-9]+]]:_(s32) = G_FEXP2 [[UV1]]
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[FEXP2_]](s32), [[FEXP2_1]](s32)
+    ; CHECK: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
+    %0:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    %1:_(<2 x s32>) = G_FEXP2 %0
+    $vgpr0_vgpr1 = COPY %1
+...
+
+---
+name: test_fexp2_v3s32
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2
+
+    ; CHECK-LABEL: name: test_fexp2_v3s32
+    ; CHECK: [[COPY:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr0_vgpr1_vgpr2
+    ; CHECK: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<3 x s32>)
+    ; CHECK: [[FEXP2_:%[0-9]+]]:_(s32) = G_FEXP2 [[UV]]
+    ; CHECK: [[FEXP2_1:%[0-9]+]]:_(s32) = G_FEXP2 [[UV1]]
+    ; CHECK: [[FEXP2_2:%[0-9]+]]:_(s32) = G_FEXP2 [[UV2]]
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[FEXP2_]](s32), [[FEXP2_1]](s32), [[FEXP2_2]](s32)
+    ; CHECK: $vgpr0_vgpr1_vgpr2 = COPY [[BUILD_VECTOR]](<3 x s32>)
+    %0:_(<3 x s32>) = COPY $vgpr0_vgpr1_vgpr2
+    %1:_(<3 x  s32>) = G_FEXP2 %0
+    $vgpr0_vgpr1_vgpr2 = COPY %1
+...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/legalize-flog.mir b/test/CodeGen/AMDGPU/GlobalISel/legalize-flog.mir
new file mode 100644
index 0000000000000000000000000000000000000000..08dd07b88b4d0a81d8e6c2a98fdf2b4261890a02
--- /dev/null
+++ b/test/CodeGen/AMDGPU/GlobalISel/legalize-flog.mir
@@ -0,0 +1,54 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=tahiti -run-pass=legalizer %s -o - | FileCheck %s
+
+---
+name: test_flog_s32
+body: |
+  bb.0:
+    liveins: $vgpr0
+
+    ; CHECK-LABEL: name: test_flog_s32
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[FLOG:%[0-9]+]]:_(s32) = G_FLOG [[COPY]]
+    ; CHECK: $vgpr0 = COPY [[FLOG]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = G_FLOG %0
+    $vgpr0 = COPY %1
+...
+
+---
+name: test_flog_v2s32
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; CHECK-LABEL: name: test_flog_v2s32
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    ; CHECK: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
+    ; CHECK: [[FLOG:%[0-9]+]]:_(s32) = G_FLOG [[UV]]
+    ; CHECK: [[FLOG1:%[0-9]+]]:_(s32) = G_FLOG [[UV1]]
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[FLOG]](s32), [[FLOG1]](s32)
+    ; CHECK: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
+    %0:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    %1:_(<2 x s32>) = G_FLOG %0
+    $vgpr0_vgpr1 = COPY %1
+...
+
+---
+name: test_flog_v3s32
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2
+
+    ; CHECK-LABEL: name: test_flog_v3s32
+    ; CHECK: [[COPY:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr0_vgpr1_vgpr2
+    ; CHECK: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<3 x s32>)
+    ; CHECK: [[FLOG:%[0-9]+]]:_(s32) = G_FLOG [[UV]]
+    ; CHECK: [[FLOG1:%[0-9]+]]:_(s32) = G_FLOG [[UV1]]
+    ; CHECK: [[FLOG2:%[0-9]+]]:_(s32) = G_FLOG [[UV2]]
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[FLOG]](s32), [[FLOG1]](s32), [[FLOG2]](s32)
+    ; CHECK: $vgpr0_vgpr1_vgpr2 = COPY [[BUILD_VECTOR]](<3 x s32>)
+    %0:_(<3 x s32>) = COPY $vgpr0_vgpr1_vgpr2
+    %1:_(<3 x  s32>) = G_FLOG %0
+    $vgpr0_vgpr1_vgpr2 = COPY %1
+...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/legalize-flog10.mir b/test/CodeGen/AMDGPU/GlobalISel/legalize-flog10.mir
new file mode 100644
index 0000000000000000000000000000000000000000..7bcfa6bff624101090db29c89c95a179035d545a
--- /dev/null
+++ b/test/CodeGen/AMDGPU/GlobalISel/legalize-flog10.mir
@@ -0,0 +1,54 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=tahiti -run-pass=legalizer %s -o - | FileCheck %s
+
+---
+name: test_flog10_s32
+body: |
+  bb.0:
+    liveins: $vgpr0
+
+    ; CHECK-LABEL: name: test_flog10_s32
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[FLOG10_:%[0-9]+]]:_(s32) = G_FLOG10 [[COPY]]
+    ; CHECK: $vgpr0 = COPY [[FLOG10_]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = G_FLOG10 %0
+    $vgpr0 = COPY %1
+...
+
+---
+name: test_flog10_v2s32
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; CHECK-LABEL: name: test_flog10_v2s32
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    ; CHECK: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
+    ; CHECK: [[FLOG10_:%[0-9]+]]:_(s32) = G_FLOG10 [[UV]]
+    ; CHECK: [[FLOG10_1:%[0-9]+]]:_(s32) = G_FLOG10 [[UV1]]
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[FLOG10_]](s32), [[FLOG10_1]](s32)
+    ; CHECK: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
+    %0:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    %1:_(<2 x s32>) = G_FLOG10 %0
+    $vgpr0_vgpr1 = COPY %1
+...
+
+---
+name: test_flog10_v3s32
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2
+
+    ; CHECK-LABEL: name: test_flog10_v3s32
+    ; CHECK: [[COPY:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr0_vgpr1_vgpr2
+    ; CHECK: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<3 x s32>)
+    ; CHECK: [[FLOG10_:%[0-9]+]]:_(s32) = G_FLOG10 [[UV]]
+    ; CHECK: [[FLOG10_1:%[0-9]+]]:_(s32) = G_FLOG10 [[UV1]]
+    ; CHECK: [[FLOG10_2:%[0-9]+]]:_(s32) = G_FLOG10 [[UV2]]
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[FLOG10_]](s32), [[FLOG10_1]](s32), [[FLOG10_2]](s32)
+    ; CHECK: $vgpr0_vgpr1_vgpr2 = COPY [[BUILD_VECTOR]](<3 x s32>)
+    %0:_(<3 x s32>) = COPY $vgpr0_vgpr1_vgpr2
+    %1:_(<3 x  s32>) = G_FLOG10 %0
+    $vgpr0_vgpr1_vgpr2 = COPY %1
+...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/legalize-flog2.mir b/test/CodeGen/AMDGPU/GlobalISel/legalize-flog2.mir
new file mode 100644
index 0000000000000000000000000000000000000000..91f3dc59db36a1ff5a153efd6cf60e166557cd14
--- /dev/null
+++ b/test/CodeGen/AMDGPU/GlobalISel/legalize-flog2.mir
@@ -0,0 +1,54 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=tahiti -run-pass=legalizer %s -o - | FileCheck %s
+
+---
+name: test_flog2_s32
+body: |
+  bb.0:
+    liveins: $vgpr0
+
+    ; CHECK-LABEL: name: test_flog2_s32
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[FLOG2_:%[0-9]+]]:_(s32) = G_FLOG2 [[COPY]]
+    ; CHECK: $vgpr0 = COPY [[FLOG2_]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = G_FLOG2 %0
+    $vgpr0 = COPY %1
+...
+
+---
+name: test_flog2_v2s32
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; CHECK-LABEL: name: test_flog2_v2s32
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    ; CHECK: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
+    ; CHECK: [[FLOG2_:%[0-9]+]]:_(s32) = G_FLOG2 [[UV]]
+    ; CHECK: [[FLOG2_1:%[0-9]+]]:_(s32) = G_FLOG2 [[UV1]]
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[FLOG2_]](s32), [[FLOG2_1]](s32)
+    ; CHECK: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
+    %0:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    %1:_(<2 x s32>) = G_FLOG2 %0
+    $vgpr0_vgpr1 = COPY %1
+...
+
+---
+name: test_flog2_v3s32
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2
+
+    ; CHECK-LABEL: name: test_flog2_v3s32
+    ; CHECK: [[COPY:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr0_vgpr1_vgpr2
+    ; CHECK: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<3 x s32>)
+    ; CHECK: [[FLOG2_:%[0-9]+]]:_(s32) = G_FLOG2 [[UV]]
+    ; CHECK: [[FLOG2_1:%[0-9]+]]:_(s32) = G_FLOG2 [[UV1]]
+    ; CHECK: [[FLOG2_2:%[0-9]+]]:_(s32) = G_FLOG2 [[UV2]]
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[FLOG2_]](s32), [[FLOG2_1]](s32), [[FLOG2_2]](s32)
+    ; CHECK: $vgpr0_vgpr1_vgpr2 = COPY [[BUILD_VECTOR]](<3 x s32>)
+    %0:_(<3 x s32>) = COPY $vgpr0_vgpr1_vgpr2
+    %1:_(<3 x  s32>) = G_FLOG2 %0
+    $vgpr0_vgpr1_vgpr2 = COPY %1
+...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/legalize-fma.mir b/test/CodeGen/AMDGPU/GlobalISel/legalize-fma.mir
index 954859e8cdaf309916a93b0182a8910b9911bdf6..12c0ab485c72e3ece6e8e0306c43affc2ca4d91a 100644
--- a/test/CodeGen/AMDGPU/GlobalISel/legalize-fma.mir
+++ b/test/CodeGen/AMDGPU/GlobalISel/legalize-fma.mir
@@ -1,16 +1,32 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -run-pass=legalizer %s -o - | FileCheck %s
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=tahiti -run-pass=legalizer %s -o - | FileCheck -check-prefix=SI  %s
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -run-pass=legalizer %s -o - | FileCheck -check-prefix=VI %s
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx900 -run-pass=legalizer %s -o - | FileCheck -check-prefix=GFX9  %s
 
 ---
-name: test_fma_f32
+name: test_fma_s32
 body: |
   bb.0:
     liveins: $vgpr0, $vgpr1, $vgpr2
 
-    ; CHECK-LABEL: name: test_fma
-    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
-    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
-    ; CHECK: [[FMA:%[0-9]+]]:_(s32) = G_FMA [[COPY]], [[COPY1]]
+    ; SI-LABEL: name: test_fma_s32
+    ; SI: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; SI: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; SI: [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+    ; SI: [[FMA:%[0-9]+]]:_(s32) = G_FMA [[COPY]], [[COPY1]], [[COPY2]]
+    ; SI: $vgpr0 = COPY [[FMA]](s32)
+    ; VI-LABEL: name: test_fma_s32
+    ; VI: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; VI: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; VI: [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+    ; VI: [[FMA:%[0-9]+]]:_(s32) = G_FMA [[COPY]], [[COPY1]], [[COPY2]]
+    ; VI: $vgpr0 = COPY [[FMA]](s32)
+    ; GFX9-LABEL: name: test_fma_s32
+    ; GFX9: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; GFX9: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; GFX9: [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+    ; GFX9: [[FMA:%[0-9]+]]:_(s32) = G_FMA [[COPY]], [[COPY1]], [[COPY2]]
+    ; GFX9: $vgpr0 = COPY [[FMA]](s32)
     %0:_(s32) = COPY $vgpr0
     %1:_(s32) = COPY $vgpr1
     %2:_(s32) = COPY $vgpr2
@@ -18,18 +34,466 @@ body: |
     $vgpr0 = COPY %3
 ...
 ---
-name: test_fma_f64
+name: test_fma_s64
 body: |
   bb.0:
     liveins: $vgpr0_vgpr1, $vgpr2_vgpr3,  $vgpr4_vgpr5
 
-    ; CHECK-LABEL: name: test_fma
-    ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
-    ; CHECK: [[COPY1:%[0-9]+]]:_(s64) = COPY $vgpr2_vgpr3
-    ; CHECK: [[FMA:%[0-9]+]]:_(s64) = G_FMA [[COPY]], [[COPY1]]
+    ; SI-LABEL: name: test_fma_s64
+    ; SI: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
+    ; SI: [[COPY1:%[0-9]+]]:_(s64) = COPY $vgpr2_vgpr3
+    ; SI: [[COPY2:%[0-9]+]]:_(s64) = COPY $vgpr4_vgpr5
+    ; SI: [[FMA:%[0-9]+]]:_(s64) = G_FMA [[COPY]], [[COPY1]], [[COPY2]]
+    ; SI: $vgpr0_vgpr1 = COPY [[FMA]](s64)
+    ; VI-LABEL: name: test_fma_s64
+    ; VI: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
+    ; VI: [[COPY1:%[0-9]+]]:_(s64) = COPY $vgpr2_vgpr3
+    ; VI: [[COPY2:%[0-9]+]]:_(s64) = COPY $vgpr4_vgpr5
+    ; VI: [[FMA:%[0-9]+]]:_(s64) = G_FMA [[COPY]], [[COPY1]], [[COPY2]]
+    ; VI: $vgpr0_vgpr1 = COPY [[FMA]](s64)
+    ; GFX9-LABEL: name: test_fma_s64
+    ; GFX9: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
+    ; GFX9: [[COPY1:%[0-9]+]]:_(s64) = COPY $vgpr2_vgpr3
+    ; GFX9: [[COPY2:%[0-9]+]]:_(s64) = COPY $vgpr4_vgpr5
+    ; GFX9: [[FMA:%[0-9]+]]:_(s64) = G_FMA [[COPY]], [[COPY1]], [[COPY2]]
+    ; GFX9: $vgpr0_vgpr1 = COPY [[FMA]](s64)
     %0:_(s64) = COPY $vgpr0_vgpr1
     %1:_(s64) = COPY $vgpr2_vgpr3
     %2:_(s64) = COPY $vgpr4_vgpr5
     %3:_(s64) = G_FMA %0, %1, %2
     $vgpr0_vgpr1 = COPY %3
 ...
+
+---
+name: test_fma_s16
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2
+
+    ; SI-LABEL: name: test_fma_s16
+    ; SI: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; SI: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; SI: [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; SI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+    ; SI: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32)
+    ; SI: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[COPY2]](s32)
+    ; SI: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC]](s16)
+    ; SI: [[FPEXT1:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC1]](s16)
+    ; SI: [[FPEXT2:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC2]](s16)
+    ; SI: [[FMA:%[0-9]+]]:_(s32) = G_FMA [[FPEXT]], [[FPEXT1]], [[FPEXT2]]
+    ; SI: [[FPTRUNC:%[0-9]+]]:_(s16) = G_FPTRUNC [[FMA]](s32)
+    ; SI: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[FPTRUNC]](s16)
+    ; SI: $vgpr0 = COPY [[ANYEXT]](s32)
+    ; VI-LABEL: name: test_fma_s16
+    ; VI: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; VI: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; VI: [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; VI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+    ; VI: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32)
+    ; VI: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[COPY2]](s32)
+    ; VI: [[FMA:%[0-9]+]]:_(s16) = G_FMA [[TRUNC]], [[TRUNC1]], [[TRUNC2]]
+    ; VI: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[FMA]](s16)
+    ; VI: $vgpr0 = COPY [[ANYEXT]](s32)
+    ; GFX9-LABEL: name: test_fma_s16
+    ; GFX9: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; GFX9: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; GFX9: [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; GFX9: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+    ; GFX9: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32)
+    ; GFX9: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[COPY2]](s32)
+    ; GFX9: [[FMA:%[0-9]+]]:_(s16) = G_FMA [[TRUNC]], [[TRUNC1]], [[TRUNC2]]
+    ; GFX9: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[FMA]](s16)
+    ; GFX9: $vgpr0 = COPY [[ANYEXT]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s32) = COPY $vgpr1
+    %3:_(s16) = G_TRUNC %0
+    %4:_(s16) = G_TRUNC %1
+    %5:_(s16) = G_TRUNC %2
+
+    %6:_(s16) = G_FMA %3, %4, %5
+    %7:_(s32) = G_ANYEXT %6
+    $vgpr0 = COPY %7
+...
+
+---
+name: test_fma_v2s32
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2_vgpr3, $vgpr4_vgpr5
+
+    ; SI-LABEL: name: test_fma_v2s32
+    ; SI: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    ; SI: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr2_vgpr3
+    ; SI: [[COPY2:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr4_vgpr5
+    ; SI: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
+    ; SI: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<2 x s32>)
+    ; SI: [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY2]](<2 x s32>)
+    ; SI: [[FMA:%[0-9]+]]:_(s32) = G_FMA [[UV]], [[UV2]], [[UV4]]
+    ; SI: [[FMA1:%[0-9]+]]:_(s32) = G_FMA [[UV1]], [[UV3]], [[UV5]]
+    ; SI: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[FMA]](s32), [[FMA1]](s32)
+    ; SI: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
+    ; VI-LABEL: name: test_fma_v2s32
+    ; VI: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    ; VI: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr2_vgpr3
+    ; VI: [[COPY2:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr4_vgpr5
+    ; VI: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
+    ; VI: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<2 x s32>)
+    ; VI: [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY2]](<2 x s32>)
+    ; VI: [[FMA:%[0-9]+]]:_(s32) = G_FMA [[UV]], [[UV2]], [[UV4]]
+    ; VI: [[FMA1:%[0-9]+]]:_(s32) = G_FMA [[UV1]], [[UV3]], [[UV5]]
+    ; VI: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[FMA]](s32), [[FMA1]](s32)
+    ; VI: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
+    ; GFX9-LABEL: name: test_fma_v2s32
+    ; GFX9: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    ; GFX9: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr2_vgpr3
+    ; GFX9: [[COPY2:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr4_vgpr5
+    ; GFX9: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
+    ; GFX9: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<2 x s32>)
+    ; GFX9: [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY2]](<2 x s32>)
+    ; GFX9: [[FMA:%[0-9]+]]:_(s32) = G_FMA [[UV]], [[UV2]], [[UV4]]
+    ; GFX9: [[FMA1:%[0-9]+]]:_(s32) = G_FMA [[UV1]], [[UV3]], [[UV5]]
+    ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[FMA]](s32), [[FMA1]](s32)
+    ; GFX9: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
+    %0:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    %1:_(<2 x s32>) = COPY $vgpr2_vgpr3
+    %2:_(<2 x s32>) = COPY $vgpr4_vgpr5
+    %3:_(<2 x s32>) = G_FMA %0, %1, %2
+    $vgpr0_vgpr1 = COPY %3
+...
+
+---
+name: test_fma_v3s32
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2, $vgpr3_vgpr4_vgpr5, $vgpr6_vgpr7_vgpr8
+
+    ; SI-LABEL: name: test_fma_v3s32
+    ; SI: [[COPY:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr0_vgpr1_vgpr2
+    ; SI: [[COPY1:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr3_vgpr4_vgpr5
+    ; SI: [[COPY2:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr6_vgpr7_vgpr8
+    ; SI: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<3 x s32>)
+    ; SI: [[UV3:%[0-9]+]]:_(s32), [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<3 x s32>)
+    ; SI: [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32), [[UV8:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY2]](<3 x s32>)
+    ; SI: [[FMA:%[0-9]+]]:_(s32) = G_FMA [[UV]], [[UV3]], [[UV6]]
+    ; SI: [[FMA1:%[0-9]+]]:_(s32) = G_FMA [[UV1]], [[UV4]], [[UV7]]
+    ; SI: [[FMA2:%[0-9]+]]:_(s32) = G_FMA [[UV2]], [[UV5]], [[UV8]]
+    ; SI: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[FMA]](s32), [[FMA1]](s32), [[FMA2]](s32)
+    ; SI: $vgpr0_vgpr1_vgpr2 = COPY [[BUILD_VECTOR]](<3 x s32>)
+    ; VI-LABEL: name: test_fma_v3s32
+    ; VI: [[COPY:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr0_vgpr1_vgpr2
+    ; VI: [[COPY1:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr3_vgpr4_vgpr5
+    ; VI: [[COPY2:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr6_vgpr7_vgpr8
+    ; VI: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<3 x s32>)
+    ; VI: [[UV3:%[0-9]+]]:_(s32), [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<3 x s32>)
+    ; VI: [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32), [[UV8:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY2]](<3 x s32>)
+    ; VI: [[FMA:%[0-9]+]]:_(s32) = G_FMA [[UV]], [[UV3]], [[UV6]]
+    ; VI: [[FMA1:%[0-9]+]]:_(s32) = G_FMA [[UV1]], [[UV4]], [[UV7]]
+    ; VI: [[FMA2:%[0-9]+]]:_(s32) = G_FMA [[UV2]], [[UV5]], [[UV8]]
+    ; VI: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[FMA]](s32), [[FMA1]](s32), [[FMA2]](s32)
+    ; VI: $vgpr0_vgpr1_vgpr2 = COPY [[BUILD_VECTOR]](<3 x s32>)
+    ; GFX9-LABEL: name: test_fma_v3s32
+    ; GFX9: [[COPY:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr0_vgpr1_vgpr2
+    ; GFX9: [[COPY1:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr3_vgpr4_vgpr5
+    ; GFX9: [[COPY2:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr6_vgpr7_vgpr8
+    ; GFX9: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<3 x s32>)
+    ; GFX9: [[UV3:%[0-9]+]]:_(s32), [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<3 x s32>)
+    ; GFX9: [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32), [[UV8:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY2]](<3 x s32>)
+    ; GFX9: [[FMA:%[0-9]+]]:_(s32) = G_FMA [[UV]], [[UV3]], [[UV6]]
+    ; GFX9: [[FMA1:%[0-9]+]]:_(s32) = G_FMA [[UV1]], [[UV4]], [[UV7]]
+    ; GFX9: [[FMA2:%[0-9]+]]:_(s32) = G_FMA [[UV2]], [[UV5]], [[UV8]]
+    ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[FMA]](s32), [[FMA1]](s32), [[FMA2]](s32)
+    ; GFX9: $vgpr0_vgpr1_vgpr2 = COPY [[BUILD_VECTOR]](<3 x s32>)
+    %0:_(<3 x s32>) = COPY $vgpr0_vgpr1_vgpr2
+    %1:_(<3 x s32>) = COPY $vgpr3_vgpr4_vgpr5
+    %2:_(<3 x s32>) = COPY $vgpr6_vgpr7_vgpr8
+    %3:_(<3 x s32>) = G_FMA %0, %1, %2
+    $vgpr0_vgpr1_vgpr2 = COPY %3
+...
+
+---
+name: test_fma_v4s32
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11
+
+    ; SI-LABEL: name: test_fma_v4s32
+    ; SI: [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; SI: [[COPY1:%[0-9]+]]:_(<4 x s32>) = COPY $vgpr4_vgpr5_vgpr6_vgpr7
+    ; SI: [[COPY2:%[0-9]+]]:_(<4 x s32>) = COPY $vgpr8_vgpr9_vgpr10_vgpr11
+    ; SI: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<4 x s32>)
+    ; SI: [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32), [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<4 x s32>)
+    ; SI: [[UV8:%[0-9]+]]:_(s32), [[UV9:%[0-9]+]]:_(s32), [[UV10:%[0-9]+]]:_(s32), [[UV11:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY2]](<4 x s32>)
+    ; SI: [[FMA:%[0-9]+]]:_(s32) = G_FMA [[UV]], [[UV4]], [[UV8]]
+    ; SI: [[FMA1:%[0-9]+]]:_(s32) = G_FMA [[UV1]], [[UV5]], [[UV9]]
+    ; SI: [[FMA2:%[0-9]+]]:_(s32) = G_FMA [[UV2]], [[UV6]], [[UV10]]
+    ; SI: [[FMA3:%[0-9]+]]:_(s32) = G_FMA [[UV3]], [[UV7]], [[UV11]]
+    ; SI: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[FMA]](s32), [[FMA1]](s32), [[FMA2]](s32), [[FMA3]](s32)
+    ; SI: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<4 x s32>)
+    ; VI-LABEL: name: test_fma_v4s32
+    ; VI: [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; VI: [[COPY1:%[0-9]+]]:_(<4 x s32>) = COPY $vgpr4_vgpr5_vgpr6_vgpr7
+    ; VI: [[COPY2:%[0-9]+]]:_(<4 x s32>) = COPY $vgpr8_vgpr9_vgpr10_vgpr11
+    ; VI: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<4 x s32>)
+    ; VI: [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32), [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<4 x s32>)
+    ; VI: [[UV8:%[0-9]+]]:_(s32), [[UV9:%[0-9]+]]:_(s32), [[UV10:%[0-9]+]]:_(s32), [[UV11:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY2]](<4 x s32>)
+    ; VI: [[FMA:%[0-9]+]]:_(s32) = G_FMA [[UV]], [[UV4]], [[UV8]]
+    ; VI: [[FMA1:%[0-9]+]]:_(s32) = G_FMA [[UV1]], [[UV5]], [[UV9]]
+    ; VI: [[FMA2:%[0-9]+]]:_(s32) = G_FMA [[UV2]], [[UV6]], [[UV10]]
+    ; VI: [[FMA3:%[0-9]+]]:_(s32) = G_FMA [[UV3]], [[UV7]], [[UV11]]
+    ; VI: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[FMA]](s32), [[FMA1]](s32), [[FMA2]](s32), [[FMA3]](s32)
+    ; VI: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<4 x s32>)
+    ; GFX9-LABEL: name: test_fma_v4s32
+    ; GFX9: [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; GFX9: [[COPY1:%[0-9]+]]:_(<4 x s32>) = COPY $vgpr4_vgpr5_vgpr6_vgpr7
+    ; GFX9: [[COPY2:%[0-9]+]]:_(<4 x s32>) = COPY $vgpr8_vgpr9_vgpr10_vgpr11
+    ; GFX9: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<4 x s32>)
+    ; GFX9: [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32), [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<4 x s32>)
+    ; GFX9: [[UV8:%[0-9]+]]:_(s32), [[UV9:%[0-9]+]]:_(s32), [[UV10:%[0-9]+]]:_(s32), [[UV11:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY2]](<4 x s32>)
+    ; GFX9: [[FMA:%[0-9]+]]:_(s32) = G_FMA [[UV]], [[UV4]], [[UV8]]
+    ; GFX9: [[FMA1:%[0-9]+]]:_(s32) = G_FMA [[UV1]], [[UV5]], [[UV9]]
+    ; GFX9: [[FMA2:%[0-9]+]]:_(s32) = G_FMA [[UV2]], [[UV6]], [[UV10]]
+    ; GFX9: [[FMA3:%[0-9]+]]:_(s32) = G_FMA [[UV3]], [[UV7]], [[UV11]]
+    ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[FMA]](s32), [[FMA1]](s32), [[FMA2]](s32), [[FMA3]](s32)
+    ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<4 x s32>)
+    %0:_(<4 x s32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    %1:_(<4 x s32>) = COPY $vgpr4_vgpr5_vgpr6_vgpr7
+    %2:_(<4 x s32>) = COPY $vgpr8_vgpr9_vgpr10_vgpr11
+    %3:_(<4 x s32>) = G_FMA %0, %1, %2
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %3
+...
+
+---
+name: test_fma_v2s64
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11
+
+    ; SI-LABEL: name: test_fma_v2s64
+    ; SI: [[COPY:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; SI: [[COPY1:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr4_vgpr5_vgpr6_vgpr7
+    ; SI: [[COPY2:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr8_vgpr9_vgpr10_vgpr11
+    ; SI: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](<2 x s64>)
+    ; SI: [[UV2:%[0-9]+]]:_(s64), [[UV3:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY1]](<2 x s64>)
+    ; SI: [[UV4:%[0-9]+]]:_(s64), [[UV5:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY2]](<2 x s64>)
+    ; SI: [[FMA:%[0-9]+]]:_(s64) = G_FMA [[UV]], [[UV2]], [[UV4]]
+    ; SI: [[FMA1:%[0-9]+]]:_(s64) = G_FMA [[UV1]], [[UV3]], [[UV5]]
+    ; SI: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[FMA]](s64), [[FMA1]](s64)
+    ; SI: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>)
+    ; VI-LABEL: name: test_fma_v2s64
+    ; VI: [[COPY:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; VI: [[COPY1:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr4_vgpr5_vgpr6_vgpr7
+    ; VI: [[COPY2:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr8_vgpr9_vgpr10_vgpr11
+    ; VI: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](<2 x s64>)
+    ; VI: [[UV2:%[0-9]+]]:_(s64), [[UV3:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY1]](<2 x s64>)
+    ; VI: [[UV4:%[0-9]+]]:_(s64), [[UV5:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY2]](<2 x s64>)
+    ; VI: [[FMA:%[0-9]+]]:_(s64) = G_FMA [[UV]], [[UV2]], [[UV4]]
+    ; VI: [[FMA1:%[0-9]+]]:_(s64) = G_FMA [[UV1]], [[UV3]], [[UV5]]
+    ; VI: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[FMA]](s64), [[FMA1]](s64)
+    ; VI: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>)
+    ; GFX9-LABEL: name: test_fma_v2s64
+    ; GFX9: [[COPY:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; GFX9: [[COPY1:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr4_vgpr5_vgpr6_vgpr7
+    ; GFX9: [[COPY2:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr8_vgpr9_vgpr10_vgpr11
+    ; GFX9: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](<2 x s64>)
+    ; GFX9: [[UV2:%[0-9]+]]:_(s64), [[UV3:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY1]](<2 x s64>)
+    ; GFX9: [[UV4:%[0-9]+]]:_(s64), [[UV5:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY2]](<2 x s64>)
+    ; GFX9: [[FMA:%[0-9]+]]:_(s64) = G_FMA [[UV]], [[UV2]], [[UV4]]
+    ; GFX9: [[FMA1:%[0-9]+]]:_(s64) = G_FMA [[UV1]], [[UV3]], [[UV5]]
+    ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[FMA]](s64), [[FMA1]](s64)
+    ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>)
+    %0:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    %1:_(<2 x s64>) = COPY $vgpr4_vgpr5_vgpr6_vgpr7
+    %2:_(<2 x s64>) = COPY $vgpr8_vgpr9_vgpr10_vgpr11
+    %3:_(<2 x s64>) = G_FMA %0, %1, %2
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %3
+...
+
+---
+name: test_fma_v2s16
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2
+
+    ; SI-LABEL: name: test_fma_v2s16
+    ; SI: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
+    ; SI: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
+    ; SI: [[COPY2:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr2
+    ; SI: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY]](<2 x s16>)
+    ; SI: [[UV2:%[0-9]+]]:_(s16), [[UV3:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY1]](<2 x s16>)
+    ; SI: [[UV4:%[0-9]+]]:_(s16), [[UV5:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY2]](<2 x s16>)
+    ; SI: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT [[UV]](s16)
+    ; SI: [[FPEXT1:%[0-9]+]]:_(s32) = G_FPEXT [[UV2]](s16)
+    ; SI: [[FPEXT2:%[0-9]+]]:_(s32) = G_FPEXT [[UV4]](s16)
+    ; SI: [[FMA:%[0-9]+]]:_(s32) = G_FMA [[FPEXT]], [[FPEXT1]], [[FPEXT2]]
+    ; SI: [[FPTRUNC:%[0-9]+]]:_(s16) = G_FPTRUNC [[FMA]](s32)
+    ; SI: [[FPEXT3:%[0-9]+]]:_(s32) = G_FPEXT [[UV1]](s16)
+    ; SI: [[FPEXT4:%[0-9]+]]:_(s32) = G_FPEXT [[UV3]](s16)
+    ; SI: [[FPEXT5:%[0-9]+]]:_(s32) = G_FPEXT [[UV5]](s16)
+    ; SI: [[FMA1:%[0-9]+]]:_(s32) = G_FMA [[FPEXT3]], [[FPEXT4]], [[FPEXT5]]
+    ; SI: [[FPTRUNC1:%[0-9]+]]:_(s16) = G_FPTRUNC [[FMA1]](s32)
+    ; SI: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[FPTRUNC]](s16), [[FPTRUNC1]](s16)
+    ; SI: $vgpr0 = COPY [[BUILD_VECTOR]](<2 x s16>)
+    ; VI-LABEL: name: test_fma_v2s16
+    ; VI: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
+    ; VI: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
+    ; VI: [[COPY2:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr2
+    ; VI: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY]](<2 x s16>)
+    ; VI: [[UV2:%[0-9]+]]:_(s16), [[UV3:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY1]](<2 x s16>)
+    ; VI: [[UV4:%[0-9]+]]:_(s16), [[UV5:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY2]](<2 x s16>)
+    ; VI: [[FMA:%[0-9]+]]:_(s16) = G_FMA [[UV]], [[UV2]], [[UV4]]
+    ; VI: [[FMA1:%[0-9]+]]:_(s16) = G_FMA [[UV1]], [[UV3]], [[UV5]]
+    ; VI: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[FMA]](s16), [[FMA1]](s16)
+    ; VI: $vgpr0 = COPY [[BUILD_VECTOR]](<2 x s16>)
+    ; GFX9-LABEL: name: test_fma_v2s16
+    ; GFX9: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
+    ; GFX9: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
+    ; GFX9: [[COPY2:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr2
+    ; GFX9: [[FMA:%[0-9]+]]:_(<2 x s16>) = G_FMA [[COPY]], [[COPY1]], [[COPY2]]
+    ; GFX9: $vgpr0 = COPY [[FMA]](<2 x s16>)
+    %0:_(<2 x s16>) = COPY $vgpr0
+    %1:_(<2 x s16>) = COPY $vgpr1
+    %2:_(<2 x s16>) = COPY $vgpr2
+    %3:_(<2 x s16>) = G_FMA %0, %1, %2
+    $vgpr0 = COPY %3
+...
+
+---
+name: test_fma_v3s16
+body: |
+  bb.0:
+
+    ; SI-LABEL: name: test_fma_v3s16
+    ; SI: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; SI: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
+    ; SI: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; SI: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF1]](<4 x s16>), 0
+    ; SI: [[DEF2:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; SI: [[EXTRACT2:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF2]](<4 x s16>), 0
+    ; SI: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16), [[UV2:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[EXTRACT]](<3 x s16>)
+    ; SI: [[UV3:%[0-9]+]]:_(s16), [[UV4:%[0-9]+]]:_(s16), [[UV5:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[EXTRACT1]](<3 x s16>)
+    ; SI: [[UV6:%[0-9]+]]:_(s16), [[UV7:%[0-9]+]]:_(s16), [[UV8:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[EXTRACT2]](<3 x s16>)
+    ; SI: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT [[UV]](s16)
+    ; SI: [[FPEXT1:%[0-9]+]]:_(s32) = G_FPEXT [[UV3]](s16)
+    ; SI: [[FPEXT2:%[0-9]+]]:_(s32) = G_FPEXT [[UV6]](s16)
+    ; SI: [[FMA:%[0-9]+]]:_(s32) = G_FMA [[FPEXT]], [[FPEXT1]], [[FPEXT2]]
+    ; SI: [[FPTRUNC:%[0-9]+]]:_(s16) = G_FPTRUNC [[FMA]](s32)
+    ; SI: [[FPEXT3:%[0-9]+]]:_(s32) = G_FPEXT [[UV1]](s16)
+    ; SI: [[FPEXT4:%[0-9]+]]:_(s32) = G_FPEXT [[UV4]](s16)
+    ; SI: [[FPEXT5:%[0-9]+]]:_(s32) = G_FPEXT [[UV7]](s16)
+    ; SI: [[FMA1:%[0-9]+]]:_(s32) = G_FMA [[FPEXT3]], [[FPEXT4]], [[FPEXT5]]
+    ; SI: [[FPTRUNC1:%[0-9]+]]:_(s16) = G_FPTRUNC [[FMA1]](s32)
+    ; SI: [[FPEXT6:%[0-9]+]]:_(s32) = G_FPEXT [[UV2]](s16)
+    ; SI: [[FPEXT7:%[0-9]+]]:_(s32) = G_FPEXT [[UV5]](s16)
+    ; SI: [[FPEXT8:%[0-9]+]]:_(s32) = G_FPEXT [[UV8]](s16)
+    ; SI: [[FMA2:%[0-9]+]]:_(s32) = G_FMA [[FPEXT6]], [[FPEXT7]], [[FPEXT8]]
+    ; SI: [[FPTRUNC2:%[0-9]+]]:_(s16) = G_FPTRUNC [[FMA2]](s32)
+    ; SI: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s16>) = G_BUILD_VECTOR [[FPTRUNC]](s16), [[FPTRUNC1]](s16), [[FPTRUNC2]](s16)
+    ; SI: S_NOP 0, implicit [[BUILD_VECTOR]](<3 x s16>)
+    ; VI-LABEL: name: test_fma_v3s16
+    ; VI: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; VI: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
+    ; VI: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; VI: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF1]](<4 x s16>), 0
+    ; VI: [[DEF2:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; VI: [[EXTRACT2:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF2]](<4 x s16>), 0
+    ; VI: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16), [[UV2:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[EXTRACT]](<3 x s16>)
+    ; VI: [[UV3:%[0-9]+]]:_(s16), [[UV4:%[0-9]+]]:_(s16), [[UV5:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[EXTRACT1]](<3 x s16>)
+    ; VI: [[UV6:%[0-9]+]]:_(s16), [[UV7:%[0-9]+]]:_(s16), [[UV8:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[EXTRACT2]](<3 x s16>)
+    ; VI: [[FMA:%[0-9]+]]:_(s16) = G_FMA [[UV]], [[UV3]], [[UV6]]
+    ; VI: [[FMA1:%[0-9]+]]:_(s16) = G_FMA [[UV1]], [[UV4]], [[UV7]]
+    ; VI: [[FMA2:%[0-9]+]]:_(s16) = G_FMA [[UV2]], [[UV5]], [[UV8]]
+    ; VI: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s16>) = G_BUILD_VECTOR [[FMA]](s16), [[FMA1]](s16), [[FMA2]](s16)
+    ; VI: S_NOP 0, implicit [[BUILD_VECTOR]](<3 x s16>)
+    ; GFX9-LABEL: name: test_fma_v3s16
+    ; GFX9: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; GFX9: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
+    ; GFX9: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; GFX9: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF1]](<4 x s16>), 0
+    ; GFX9: [[DEF2:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; GFX9: [[EXTRACT2:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF2]](<4 x s16>), 0
+    ; GFX9: [[DEF3:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; GFX9: [[EXTRACT3:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF3]](<4 x s16>), 0
+    ; GFX9: [[EXTRACT4:%[0-9]+]]:_(<2 x s16>) = G_EXTRACT [[EXTRACT]](<3 x s16>), 0
+    ; GFX9: [[EXTRACT5:%[0-9]+]]:_(<2 x s16>) = G_EXTRACT [[EXTRACT1]](<3 x s16>), 0
+    ; GFX9: [[EXTRACT6:%[0-9]+]]:_(<2 x s16>) = G_EXTRACT [[EXTRACT2]](<3 x s16>), 0
+    ; GFX9: [[FMA:%[0-9]+]]:_(<2 x s16>) = G_FMA [[EXTRACT4]], [[EXTRACT5]], [[EXTRACT6]]
+    ; GFX9: [[INSERT:%[0-9]+]]:_(<3 x s16>) = G_INSERT [[EXTRACT3]], [[FMA]](<2 x s16>), 0
+    ; GFX9: [[EXTRACT7:%[0-9]+]]:_(s16) = G_EXTRACT [[EXTRACT]](<3 x s16>), 32
+    ; GFX9: [[EXTRACT8:%[0-9]+]]:_(s16) = G_EXTRACT [[EXTRACT1]](<3 x s16>), 32
+    ; GFX9: [[EXTRACT9:%[0-9]+]]:_(s16) = G_EXTRACT [[EXTRACT2]](<3 x s16>), 32
+    ; GFX9: [[FMA1:%[0-9]+]]:_(s16) = G_FMA [[EXTRACT7]], [[EXTRACT8]], [[EXTRACT9]]
+    ; GFX9: [[INSERT1:%[0-9]+]]:_(<3 x s16>) = G_INSERT [[INSERT]], [[FMA1]](s16), 32
+    ; GFX9: S_NOP 0, implicit [[INSERT1]](<3 x s16>)
+    %0:_(<3 x s16>) = G_IMPLICIT_DEF
+    %1:_(<3 x s16>) = G_IMPLICIT_DEF
+    %2:_(<3 x s16>) = G_IMPLICIT_DEF
+    %3:_(<3 x s16>) = G_FMA %0, %1, %2
+    S_NOP 0, implicit %3
+...
+
+---
+name: test_fma_v4s16
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2_vgpr3, $vgpr4_vgpr5
+
+    ; SI-LABEL: name: test_fma_v4s16
+    ; SI: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
+    ; SI: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
+    ; SI: [[COPY2:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr4_vgpr5
+    ; SI: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16), [[UV2:%[0-9]+]]:_(s16), [[UV3:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
+    ; SI: [[UV4:%[0-9]+]]:_(s16), [[UV5:%[0-9]+]]:_(s16), [[UV6:%[0-9]+]]:_(s16), [[UV7:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
+    ; SI: [[UV8:%[0-9]+]]:_(s16), [[UV9:%[0-9]+]]:_(s16), [[UV10:%[0-9]+]]:_(s16), [[UV11:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY2]](<4 x s16>)
+    ; SI: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT [[UV]](s16)
+    ; SI: [[FPEXT1:%[0-9]+]]:_(s32) = G_FPEXT [[UV4]](s16)
+    ; SI: [[FPEXT2:%[0-9]+]]:_(s32) = G_FPEXT [[UV8]](s16)
+    ; SI: [[FMA:%[0-9]+]]:_(s32) = G_FMA [[FPEXT]], [[FPEXT1]], [[FPEXT2]]
+    ; SI: [[FPTRUNC:%[0-9]+]]:_(s16) = G_FPTRUNC [[FMA]](s32)
+    ; SI: [[FPEXT3:%[0-9]+]]:_(s32) = G_FPEXT [[UV1]](s16)
+    ; SI: [[FPEXT4:%[0-9]+]]:_(s32) = G_FPEXT [[UV5]](s16)
+    ; SI: [[FPEXT5:%[0-9]+]]:_(s32) = G_FPEXT [[UV9]](s16)
+    ; SI: [[FMA1:%[0-9]+]]:_(s32) = G_FMA [[FPEXT3]], [[FPEXT4]], [[FPEXT5]]
+    ; SI: [[FPTRUNC1:%[0-9]+]]:_(s16) = G_FPTRUNC [[FMA1]](s32)
+    ; SI: [[FPEXT6:%[0-9]+]]:_(s32) = G_FPEXT [[UV2]](s16)
+    ; SI: [[FPEXT7:%[0-9]+]]:_(s32) = G_FPEXT [[UV6]](s16)
+    ; SI: [[FPEXT8:%[0-9]+]]:_(s32) = G_FPEXT [[UV10]](s16)
+    ; SI: [[FMA2:%[0-9]+]]:_(s32) = G_FMA [[FPEXT6]], [[FPEXT7]], [[FPEXT8]]
+    ; SI: [[FPTRUNC2:%[0-9]+]]:_(s16) = G_FPTRUNC [[FMA2]](s32)
+    ; SI: [[FPEXT9:%[0-9]+]]:_(s32) = G_FPEXT [[UV3]](s16)
+    ; SI: [[FPEXT10:%[0-9]+]]:_(s32) = G_FPEXT [[UV7]](s16)
+    ; SI: [[FPEXT11:%[0-9]+]]:_(s32) = G_FPEXT [[UV11]](s16)
+    ; SI: [[FMA3:%[0-9]+]]:_(s32) = G_FMA [[FPEXT9]], [[FPEXT10]], [[FPEXT11]]
+    ; SI: [[FPTRUNC3:%[0-9]+]]:_(s16) = G_FPTRUNC [[FMA3]](s32)
+    ; SI: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s16>) = G_BUILD_VECTOR [[FPTRUNC]](s16), [[FPTRUNC1]](s16), [[FPTRUNC2]](s16), [[FPTRUNC3]](s16)
+    ; SI: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<4 x s16>)
+    ; VI-LABEL: name: test_fma_v4s16
+    ; VI: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
+    ; VI: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
+    ; VI: [[COPY2:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr4_vgpr5
+    ; VI: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16), [[UV2:%[0-9]+]]:_(s16), [[UV3:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
+    ; VI: [[UV4:%[0-9]+]]:_(s16), [[UV5:%[0-9]+]]:_(s16), [[UV6:%[0-9]+]]:_(s16), [[UV7:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
+    ; VI: [[UV8:%[0-9]+]]:_(s16), [[UV9:%[0-9]+]]:_(s16), [[UV10:%[0-9]+]]:_(s16), [[UV11:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY2]](<4 x s16>)
+    ; VI: [[FMA:%[0-9]+]]:_(s16) = G_FMA [[UV]], [[UV4]], [[UV8]]
+    ; VI: [[FMA1:%[0-9]+]]:_(s16) = G_FMA [[UV1]], [[UV5]], [[UV9]]
+    ; VI: [[FMA2:%[0-9]+]]:_(s16) = G_FMA [[UV2]], [[UV6]], [[UV10]]
+    ; VI: [[FMA3:%[0-9]+]]:_(s16) = G_FMA [[UV3]], [[UV7]], [[UV11]]
+    ; VI: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s16>) = G_BUILD_VECTOR [[FMA]](s16), [[FMA1]](s16), [[FMA2]](s16), [[FMA3]](s16)
+    ; VI: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<4 x s16>)
+    ; GFX9-LABEL: name: test_fma_v4s16
+    ; GFX9: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
+    ; GFX9: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
+    ; GFX9: [[COPY2:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr4_vgpr5
+    ; GFX9: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
+    ; GFX9: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
+    ; GFX9: [[UV4:%[0-9]+]]:_(<2 x s16>), [[UV5:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY2]](<4 x s16>)
+    ; GFX9: [[FMA:%[0-9]+]]:_(<2 x s16>) = G_FMA [[UV]], [[UV2]], [[UV4]]
+    ; GFX9: [[FMA1:%[0-9]+]]:_(<2 x s16>) = G_FMA [[UV1]], [[UV3]], [[UV5]]
+    ; GFX9: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[FMA]](<2 x s16>), [[FMA1]](<2 x s16>)
+    ; GFX9: $vgpr0_vgpr1 = COPY [[CONCAT_VECTORS]](<4 x s16>)
+    %0:_(<4 x s16>) = COPY $vgpr0_vgpr1
+    %1:_(<4 x s16>) = COPY $vgpr2_vgpr3
+    %2:_(<4 x s16>) = COPY $vgpr4_vgpr5
+    %3:_(<4 x s16>) = G_FMA %0, %1, %2
+    $vgpr0_vgpr1 = COPY %3
+...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/legalize-fmul.mir b/test/CodeGen/AMDGPU/GlobalISel/legalize-fmul.mir
index 8a0ce1c6e91a5c69b3010b17ddde4e00695977e7..250e3f0d6038baee576142b035c6c13267963623 100644
--- a/test/CodeGen/AMDGPU/GlobalISel/legalize-fmul.mir
+++ b/test/CodeGen/AMDGPU/GlobalISel/legalize-fmul.mir
@@ -1,33 +1,416 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -run-pass=legalizer %s -o - | FileCheck %s
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=tahiti -run-pass=legalizer %s -o - | FileCheck -check-prefix=SI  %s
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -run-pass=legalizer %s -o - | FileCheck -check-prefix=VI %s
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx900 -run-pass=legalizer %s -o - | FileCheck -check-prefix=GFX9  %s
 
 ---
-name: test_fmul_f32
+name: test_fmul_s32
 body: |
   bb.0:
     liveins: $vgpr0, $vgpr1
 
-    ; CHECK-LABEL: name: test_fmul
-    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
-    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
-    ; CHECK: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[COPY]], [[COPY1]]
+    ; SI-LABEL: name: test_fmul_s32
+    ; SI: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; SI: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; SI: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[COPY]], [[COPY1]]
+    ; SI: $vgpr0 = COPY [[FMUL]](s32)
+    ; VI-LABEL: name: test_fmul_s32
+    ; VI: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; VI: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; VI: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[COPY]], [[COPY1]]
+    ; VI: $vgpr0 = COPY [[FMUL]](s32)
+    ; GFX9-LABEL: name: test_fmul_s32
+    ; GFX9: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; GFX9: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; GFX9: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[COPY]], [[COPY1]]
+    ; GFX9: $vgpr0 = COPY [[FMUL]](s32)
     %0:_(s32) = COPY $vgpr0
     %1:_(s32) = COPY $vgpr1
     %2:_(s32) = G_FMUL %0, %1
     $vgpr0 = COPY %2
 ...
 ---
-name: test_fmul_f64
+name: test_fmul_s64
 body: |
   bb.0:
     liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
 
-    ; CHECK-LABEL: name: test_fmul
-    ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
-    ; CHECK: [[COPY1:%[0-9]+]]:_(s64) = COPY $vgpr2_vgpr3
-    ; CHECK: [[FMUL:%[0-9]+]]:_(s64) = G_FMUL [[COPY]], [[COPY1]]
+    ; SI-LABEL: name: test_fmul_s64
+    ; SI: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
+    ; SI: [[COPY1:%[0-9]+]]:_(s64) = COPY $vgpr2_vgpr3
+    ; SI: [[FMUL:%[0-9]+]]:_(s64) = G_FMUL [[COPY]], [[COPY1]]
+    ; SI: $vgpr0_vgpr1 = COPY [[FMUL]](s64)
+    ; VI-LABEL: name: test_fmul_s64
+    ; VI: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
+    ; VI: [[COPY1:%[0-9]+]]:_(s64) = COPY $vgpr2_vgpr3
+    ; VI: [[FMUL:%[0-9]+]]:_(s64) = G_FMUL [[COPY]], [[COPY1]]
+    ; VI: $vgpr0_vgpr1 = COPY [[FMUL]](s64)
+    ; GFX9-LABEL: name: test_fmul_s64
+    ; GFX9: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
+    ; GFX9: [[COPY1:%[0-9]+]]:_(s64) = COPY $vgpr2_vgpr3
+    ; GFX9: [[FMUL:%[0-9]+]]:_(s64) = G_FMUL [[COPY]], [[COPY1]]
+    ; GFX9: $vgpr0_vgpr1 = COPY [[FMUL]](s64)
     %0:_(s64) = COPY $vgpr0_vgpr1
     %1:_(s64) = COPY $vgpr2_vgpr3
     %2:_(s64) = G_FMUL %0, %1
     $vgpr0_vgpr1 = COPY %2
 ...
+
+---
+name: test_fmul_s16
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+
+    ; SI-LABEL: name: test_fmul_s16
+    ; SI: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; SI: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; SI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+    ; SI: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32)
+    ; SI: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC]](s16)
+    ; SI: [[FPEXT1:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC1]](s16)
+    ; SI: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[FPEXT]], [[FPEXT1]]
+    ; SI: [[FPTRUNC:%[0-9]+]]:_(s16) = G_FPTRUNC [[FMUL]](s32)
+    ; SI: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[FPTRUNC]](s16)
+    ; SI: $vgpr0 = COPY [[ANYEXT]](s32)
+    ; VI-LABEL: name: test_fmul_s16
+    ; VI: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; VI: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; VI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+    ; VI: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32)
+    ; VI: [[FMUL:%[0-9]+]]:_(s16) = G_FMUL [[TRUNC]], [[TRUNC1]]
+    ; VI: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[FMUL]](s16)
+    ; VI: $vgpr0 = COPY [[ANYEXT]](s32)
+    ; GFX9-LABEL: name: test_fmul_s16
+    ; GFX9: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; GFX9: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; GFX9: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+    ; GFX9: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32)
+    ; GFX9: [[FMUL:%[0-9]+]]:_(s16) = G_FMUL [[TRUNC]], [[TRUNC1]]
+    ; GFX9: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[FMUL]](s16)
+    ; GFX9: $vgpr0 = COPY [[ANYEXT]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s16) = G_TRUNC %0
+    %3:_(s16) = G_TRUNC %1
+
+    %4:_(s16) = G_FMUL %2, %3
+    %5:_(s32) = G_ANYEXT %4
+    $vgpr0 = COPY %5
+...
+
+---
+name: test_fmul_v2s32
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
+
+    ; SI-LABEL: name: test_fmul_v2s32
+    ; SI: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    ; SI: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr2_vgpr3
+    ; SI: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
+    ; SI: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<2 x s32>)
+    ; SI: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[UV]], [[UV2]]
+    ; SI: [[FMUL1:%[0-9]+]]:_(s32) = G_FMUL [[UV1]], [[UV3]]
+    ; SI: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[FMUL]](s32), [[FMUL1]](s32)
+    ; SI: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
+    ; VI-LABEL: name: test_fmul_v2s32
+    ; VI: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    ; VI: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr2_vgpr3
+    ; VI: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
+    ; VI: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<2 x s32>)
+    ; VI: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[UV]], [[UV2]]
+    ; VI: [[FMUL1:%[0-9]+]]:_(s32) = G_FMUL [[UV1]], [[UV3]]
+    ; VI: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[FMUL]](s32), [[FMUL1]](s32)
+    ; VI: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
+    ; GFX9-LABEL: name: test_fmul_v2s32
+    ; GFX9: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    ; GFX9: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr2_vgpr3
+    ; GFX9: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
+    ; GFX9: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<2 x s32>)
+    ; GFX9: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[UV]], [[UV2]]
+    ; GFX9: [[FMUL1:%[0-9]+]]:_(s32) = G_FMUL [[UV1]], [[UV3]]
+    ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[FMUL]](s32), [[FMUL1]](s32)
+    ; GFX9: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
+    %0:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    %1:_(<2 x s32>) = COPY $vgpr2_vgpr3
+    %2:_(<2 x s32>) = G_FMUL %0, %1
+    $vgpr0_vgpr1 = COPY %2
+...
+
+---
+name: test_fmul_v2s32_flags
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
+
+    ; SI-LABEL: name: test_fmul_v2s32_flags
+    ; SI: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    ; SI: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr2_vgpr3
+    ; SI: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
+    ; SI: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<2 x s32>)
+    ; SI: %7:_(s32) = nnan G_FMUL [[UV]], [[UV2]]
+    ; SI: %8:_(s32) = nnan G_FMUL [[UV1]], [[UV3]]
+    ; SI: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR %7(s32), %8(s32)
+    ; SI: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
+    ; VI-LABEL: name: test_fmul_v2s32_flags
+    ; VI: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    ; VI: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr2_vgpr3
+    ; VI: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
+    ; VI: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<2 x s32>)
+    ; VI: %7:_(s32) = nnan G_FMUL [[UV]], [[UV2]]
+    ; VI: %8:_(s32) = nnan G_FMUL [[UV1]], [[UV3]]
+    ; VI: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR %7(s32), %8(s32)
+    ; VI: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
+    ; GFX9-LABEL: name: test_fmul_v2s32_flags
+    ; GFX9: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    ; GFX9: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr2_vgpr3
+    ; GFX9: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
+    ; GFX9: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<2 x s32>)
+    ; GFX9: %7:_(s32) = nnan G_FMUL [[UV]], [[UV2]]
+    ; GFX9: %8:_(s32) = nnan G_FMUL [[UV1]], [[UV3]]
+    ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR %7(s32), %8(s32)
+    ; GFX9: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
+    %0:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    %1:_(<2 x s32>) = COPY $vgpr2_vgpr3
+    %2:_(<2 x s32>) = nnan G_FMUL %0, %1
+    $vgpr0_vgpr1 = COPY %2
+...
+
+---
+name: test_fmul_v3s32
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2, $vgpr3_vgpr4_vgpr5
+
+    ; SI-LABEL: name: test_fmul_v3s32
+    ; SI: [[COPY:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr0_vgpr1_vgpr2
+    ; SI: [[COPY1:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr3_vgpr4_vgpr5
+    ; SI: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<3 x s32>)
+    ; SI: [[UV3:%[0-9]+]]:_(s32), [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<3 x s32>)
+    ; SI: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[UV]], [[UV3]]
+    ; SI: [[FMUL1:%[0-9]+]]:_(s32) = G_FMUL [[UV1]], [[UV4]]
+    ; SI: [[FMUL2:%[0-9]+]]:_(s32) = G_FMUL [[UV2]], [[UV5]]
+    ; SI: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[FMUL]](s32), [[FMUL1]](s32), [[FMUL2]](s32)
+    ; SI: $vgpr0_vgpr1_vgpr2 = COPY [[BUILD_VECTOR]](<3 x s32>)
+    ; VI-LABEL: name: test_fmul_v3s32
+    ; VI: [[COPY:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr0_vgpr1_vgpr2
+    ; VI: [[COPY1:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr3_vgpr4_vgpr5
+    ; VI: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<3 x s32>)
+    ; VI: [[UV3:%[0-9]+]]:_(s32), [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<3 x s32>)
+    ; VI: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[UV]], [[UV3]]
+    ; VI: [[FMUL1:%[0-9]+]]:_(s32) = G_FMUL [[UV1]], [[UV4]]
+    ; VI: [[FMUL2:%[0-9]+]]:_(s32) = G_FMUL [[UV2]], [[UV5]]
+    ; VI: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[FMUL]](s32), [[FMUL1]](s32), [[FMUL2]](s32)
+    ; VI: $vgpr0_vgpr1_vgpr2 = COPY [[BUILD_VECTOR]](<3 x s32>)
+    ; GFX9-LABEL: name: test_fmul_v3s32
+    ; GFX9: [[COPY:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr0_vgpr1_vgpr2
+    ; GFX9: [[COPY1:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr3_vgpr4_vgpr5
+    ; GFX9: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<3 x s32>)
+    ; GFX9: [[UV3:%[0-9]+]]:_(s32), [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<3 x s32>)
+    ; GFX9: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[UV]], [[UV3]]
+    ; GFX9: [[FMUL1:%[0-9]+]]:_(s32) = G_FMUL [[UV1]], [[UV4]]
+    ; GFX9: [[FMUL2:%[0-9]+]]:_(s32) = G_FMUL [[UV2]], [[UV5]]
+    ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[FMUL]](s32), [[FMUL1]](s32), [[FMUL2]](s32)
+    ; GFX9: $vgpr0_vgpr1_vgpr2 = COPY [[BUILD_VECTOR]](<3 x s32>)
+    %0:_(<3 x s32>) = COPY $vgpr0_vgpr1_vgpr2
+    %1:_(<3 x s32>) = COPY $vgpr3_vgpr4_vgpr5
+    %2:_(<3 x s32>) = G_FMUL %0, %1
+    $vgpr0_vgpr1_vgpr2 = COPY %2
+...
+
+---
+name: test_fmul_v2s64
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7
+
+    ; SI-LABEL: name: test_fmul_v2s64
+    ; SI: [[COPY:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; SI: [[COPY1:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr4_vgpr5_vgpr6_vgpr7
+    ; SI: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](<2 x s64>)
+    ; SI: [[UV2:%[0-9]+]]:_(s64), [[UV3:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY1]](<2 x s64>)
+    ; SI: [[FMUL:%[0-9]+]]:_(s64) = G_FMUL [[UV]], [[UV2]]
+    ; SI: [[FMUL1:%[0-9]+]]:_(s64) = G_FMUL [[UV1]], [[UV3]]
+    ; SI: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[FMUL]](s64), [[FMUL1]](s64)
+    ; SI: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>)
+    ; VI-LABEL: name: test_fmul_v2s64
+    ; VI: [[COPY:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; VI: [[COPY1:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr4_vgpr5_vgpr6_vgpr7
+    ; VI: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](<2 x s64>)
+    ; VI: [[UV2:%[0-9]+]]:_(s64), [[UV3:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY1]](<2 x s64>)
+    ; VI: [[FMUL:%[0-9]+]]:_(s64) = G_FMUL [[UV]], [[UV2]]
+    ; VI: [[FMUL1:%[0-9]+]]:_(s64) = G_FMUL [[UV1]], [[UV3]]
+    ; VI: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[FMUL]](s64), [[FMUL1]](s64)
+    ; VI: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>)
+    ; GFX9-LABEL: name: test_fmul_v2s64
+    ; GFX9: [[COPY:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; GFX9: [[COPY1:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr4_vgpr5_vgpr6_vgpr7
+    ; GFX9: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](<2 x s64>)
+    ; GFX9: [[UV2:%[0-9]+]]:_(s64), [[UV3:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY1]](<2 x s64>)
+    ; GFX9: [[FMUL:%[0-9]+]]:_(s64) = G_FMUL [[UV]], [[UV2]]
+    ; GFX9: [[FMUL1:%[0-9]+]]:_(s64) = G_FMUL [[UV1]], [[UV3]]
+    ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[FMUL]](s64), [[FMUL1]](s64)
+    ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>)
+    %0:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    %1:_(<2 x s64>) = COPY $vgpr4_vgpr5_vgpr6_vgpr7
+    %2:_(<2 x s64>) = G_FMUL %0, %1
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %2
+...
+
+---
+name: test_fmul_v2s16
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+
+    ; SI-LABEL: name: test_fmul_v2s16
+    ; SI: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
+    ; SI: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
+    ; SI: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY]](<2 x s16>)
+    ; SI: [[UV2:%[0-9]+]]:_(s16), [[UV3:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY1]](<2 x s16>)
+    ; SI: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT [[UV]](s16)
+    ; SI: [[FPEXT1:%[0-9]+]]:_(s32) = G_FPEXT [[UV2]](s16)
+    ; SI: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[FPEXT]], [[FPEXT1]]
+    ; SI: [[FPTRUNC:%[0-9]+]]:_(s16) = G_FPTRUNC [[FMUL]](s32)
+    ; SI: [[FPEXT2:%[0-9]+]]:_(s32) = G_FPEXT [[UV1]](s16)
+    ; SI: [[FPEXT3:%[0-9]+]]:_(s32) = G_FPEXT [[UV3]](s16)
+    ; SI: [[FMUL1:%[0-9]+]]:_(s32) = G_FMUL [[FPEXT2]], [[FPEXT3]]
+    ; SI: [[FPTRUNC1:%[0-9]+]]:_(s16) = G_FPTRUNC [[FMUL1]](s32)
+    ; SI: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[FPTRUNC]](s16), [[FPTRUNC1]](s16)
+    ; SI: $vgpr0 = COPY [[BUILD_VECTOR]](<2 x s16>)
+    ; VI-LABEL: name: test_fmul_v2s16
+    ; VI: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
+    ; VI: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
+    ; VI: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY]](<2 x s16>)
+    ; VI: [[UV2:%[0-9]+]]:_(s16), [[UV3:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY1]](<2 x s16>)
+    ; VI: [[FMUL:%[0-9]+]]:_(s16) = G_FMUL [[UV]], [[UV2]]
+    ; VI: [[FMUL1:%[0-9]+]]:_(s16) = G_FMUL [[UV1]], [[UV3]]
+    ; VI: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[FMUL]](s16), [[FMUL1]](s16)
+    ; VI: $vgpr0 = COPY [[BUILD_VECTOR]](<2 x s16>)
+    ; GFX9-LABEL: name: test_fmul_v2s16
+    ; GFX9: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
+    ; GFX9: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
+    ; GFX9: [[FMUL:%[0-9]+]]:_(<2 x s16>) = G_FMUL [[COPY]], [[COPY1]]
+    ; GFX9: $vgpr0 = COPY [[FMUL]](<2 x s16>)
+    %0:_(<2 x s16>) = COPY $vgpr0
+    %1:_(<2 x s16>) = COPY $vgpr1
+    %2:_(<2 x s16>) = G_FMUL %0, %1
+    $vgpr0 = COPY %2
+...
+
+---
+name: test_fmul_v3s16
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+
+    ; SI-LABEL: name: test_fmul_v3s16
+    ; SI: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; SI: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
+    ; SI: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; SI: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF1]](<4 x s16>), 0
+    ; SI: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16), [[UV2:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[EXTRACT]](<3 x s16>)
+    ; SI: [[UV3:%[0-9]+]]:_(s16), [[UV4:%[0-9]+]]:_(s16), [[UV5:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[EXTRACT1]](<3 x s16>)
+    ; SI: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT [[UV]](s16)
+    ; SI: [[FPEXT1:%[0-9]+]]:_(s32) = G_FPEXT [[UV3]](s16)
+    ; SI: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[FPEXT]], [[FPEXT1]]
+    ; SI: [[FPTRUNC:%[0-9]+]]:_(s16) = G_FPTRUNC [[FMUL]](s32)
+    ; SI: [[FPEXT2:%[0-9]+]]:_(s32) = G_FPEXT [[UV1]](s16)
+    ; SI: [[FPEXT3:%[0-9]+]]:_(s32) = G_FPEXT [[UV4]](s16)
+    ; SI: [[FMUL1:%[0-9]+]]:_(s32) = G_FMUL [[FPEXT2]], [[FPEXT3]]
+    ; SI: [[FPTRUNC1:%[0-9]+]]:_(s16) = G_FPTRUNC [[FMUL1]](s32)
+    ; SI: [[FPEXT4:%[0-9]+]]:_(s32) = G_FPEXT [[UV2]](s16)
+    ; SI: [[FPEXT5:%[0-9]+]]:_(s32) = G_FPEXT [[UV5]](s16)
+    ; SI: [[FMUL2:%[0-9]+]]:_(s32) = G_FMUL [[FPEXT4]], [[FPEXT5]]
+    ; SI: [[FPTRUNC2:%[0-9]+]]:_(s16) = G_FPTRUNC [[FMUL2]](s32)
+    ; SI: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s16>) = G_BUILD_VECTOR [[FPTRUNC]](s16), [[FPTRUNC1]](s16), [[FPTRUNC2]](s16)
+    ; SI: S_NOP 0, implicit [[BUILD_VECTOR]](<3 x s16>)
+    ; VI-LABEL: name: test_fmul_v3s16
+    ; VI: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; VI: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
+    ; VI: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; VI: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF1]](<4 x s16>), 0
+    ; VI: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16), [[UV2:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[EXTRACT]](<3 x s16>)
+    ; VI: [[UV3:%[0-9]+]]:_(s16), [[UV4:%[0-9]+]]:_(s16), [[UV5:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[EXTRACT1]](<3 x s16>)
+    ; VI: [[FMUL:%[0-9]+]]:_(s16) = G_FMUL [[UV]], [[UV3]]
+    ; VI: [[FMUL1:%[0-9]+]]:_(s16) = G_FMUL [[UV1]], [[UV4]]
+    ; VI: [[FMUL2:%[0-9]+]]:_(s16) = G_FMUL [[UV2]], [[UV5]]
+    ; VI: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s16>) = G_BUILD_VECTOR [[FMUL]](s16), [[FMUL1]](s16), [[FMUL2]](s16)
+    ; VI: S_NOP 0, implicit [[BUILD_VECTOR]](<3 x s16>)
+    ; GFX9-LABEL: name: test_fmul_v3s16
+    ; GFX9: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; GFX9: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
+    ; GFX9: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; GFX9: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF1]](<4 x s16>), 0
+    ; GFX9: [[DEF2:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; GFX9: [[EXTRACT2:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF2]](<4 x s16>), 0
+    ; GFX9: [[EXTRACT3:%[0-9]+]]:_(<2 x s16>) = G_EXTRACT [[EXTRACT]](<3 x s16>), 0
+    ; GFX9: [[EXTRACT4:%[0-9]+]]:_(<2 x s16>) = G_EXTRACT [[EXTRACT1]](<3 x s16>), 0
+    ; GFX9: [[FMUL:%[0-9]+]]:_(<2 x s16>) = G_FMUL [[EXTRACT3]], [[EXTRACT4]]
+    ; GFX9: [[INSERT:%[0-9]+]]:_(<3 x s16>) = G_INSERT [[EXTRACT2]], [[FMUL]](<2 x s16>), 0
+    ; GFX9: [[EXTRACT5:%[0-9]+]]:_(s16) = G_EXTRACT [[EXTRACT]](<3 x s16>), 32
+    ; GFX9: [[EXTRACT6:%[0-9]+]]:_(s16) = G_EXTRACT [[EXTRACT1]](<3 x s16>), 32
+    ; GFX9: [[FMUL1:%[0-9]+]]:_(s16) = G_FMUL [[EXTRACT5]], [[EXTRACT6]]
+    ; GFX9: [[INSERT1:%[0-9]+]]:_(<3 x s16>) = G_INSERT [[INSERT]], [[FMUL1]](s16), 32
+    ; GFX9: S_NOP 0, implicit [[INSERT1]](<3 x s16>)
+    %0:_(<3 x s16>) = G_IMPLICIT_DEF
+    %1:_(<3 x s16>) = G_IMPLICIT_DEF
+    %2:_(<3 x s16>) = G_FMUL %0, %1
+    S_NOP 0, implicit %2
+...
+
+---
+name: test_fmul_v4s16
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
+
+    ; SI-LABEL: name: test_fmul_v4s16
+    ; SI: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
+    ; SI: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
+    ; SI: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16), [[UV2:%[0-9]+]]:_(s16), [[UV3:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
+    ; SI: [[UV4:%[0-9]+]]:_(s16), [[UV5:%[0-9]+]]:_(s16), [[UV6:%[0-9]+]]:_(s16), [[UV7:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
+    ; SI: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT [[UV]](s16)
+    ; SI: [[FPEXT1:%[0-9]+]]:_(s32) = G_FPEXT [[UV4]](s16)
+    ; SI: [[FMUL:%[0-9]+]]:_(s32) = G_FMUL [[FPEXT]], [[FPEXT1]]
+    ; SI: [[FPTRUNC:%[0-9]+]]:_(s16) = G_FPTRUNC [[FMUL]](s32)
+    ; SI: [[FPEXT2:%[0-9]+]]:_(s32) = G_FPEXT [[UV1]](s16)
+    ; SI: [[FPEXT3:%[0-9]+]]:_(s32) = G_FPEXT [[UV5]](s16)
+    ; SI: [[FMUL1:%[0-9]+]]:_(s32) = G_FMUL [[FPEXT2]], [[FPEXT3]]
+    ; SI: [[FPTRUNC1:%[0-9]+]]:_(s16) = G_FPTRUNC [[FMUL1]](s32)
+    ; SI: [[FPEXT4:%[0-9]+]]:_(s32) = G_FPEXT [[UV2]](s16)
+    ; SI: [[FPEXT5:%[0-9]+]]:_(s32) = G_FPEXT [[UV6]](s16)
+    ; SI: [[FMUL2:%[0-9]+]]:_(s32) = G_FMUL [[FPEXT4]], [[FPEXT5]]
+    ; SI: [[FPTRUNC2:%[0-9]+]]:_(s16) = G_FPTRUNC [[FMUL2]](s32)
+    ; SI: [[FPEXT6:%[0-9]+]]:_(s32) = G_FPEXT [[UV3]](s16)
+    ; SI: [[FPEXT7:%[0-9]+]]:_(s32) = G_FPEXT [[UV7]](s16)
+    ; SI: [[FMUL3:%[0-9]+]]:_(s32) = G_FMUL [[FPEXT6]], [[FPEXT7]]
+    ; SI: [[FPTRUNC3:%[0-9]+]]:_(s16) = G_FPTRUNC [[FMUL3]](s32)
+    ; SI: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s16>) = G_BUILD_VECTOR [[FPTRUNC]](s16), [[FPTRUNC1]](s16), [[FPTRUNC2]](s16), [[FPTRUNC3]](s16)
+    ; SI: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<4 x s16>)
+    ; VI-LABEL: name: test_fmul_v4s16
+    ; VI: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
+    ; VI: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
+    ; VI: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16), [[UV2:%[0-9]+]]:_(s16), [[UV3:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
+    ; VI: [[UV4:%[0-9]+]]:_(s16), [[UV5:%[0-9]+]]:_(s16), [[UV6:%[0-9]+]]:_(s16), [[UV7:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
+    ; VI: [[FMUL:%[0-9]+]]:_(s16) = G_FMUL [[UV]], [[UV4]]
+    ; VI: [[FMUL1:%[0-9]+]]:_(s16) = G_FMUL [[UV1]], [[UV5]]
+    ; VI: [[FMUL2:%[0-9]+]]:_(s16) = G_FMUL [[UV2]], [[UV6]]
+    ; VI: [[FMUL3:%[0-9]+]]:_(s16) = G_FMUL [[UV3]], [[UV7]]
+    ; VI: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s16>) = G_BUILD_VECTOR [[FMUL]](s16), [[FMUL1]](s16), [[FMUL2]](s16), [[FMUL3]](s16)
+    ; VI: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<4 x s16>)
+    ; GFX9-LABEL: name: test_fmul_v4s16
+    ; GFX9: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
+    ; GFX9: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
+    ; GFX9: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
+    ; GFX9: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
+    ; GFX9: [[FMUL:%[0-9]+]]:_(<2 x s16>) = G_FMUL [[UV]], [[UV2]]
+    ; GFX9: [[FMUL1:%[0-9]+]]:_(<2 x s16>) = G_FMUL [[UV1]], [[UV3]]
+    ; GFX9: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[FMUL]](<2 x s16>), [[FMUL1]](<2 x s16>)
+    ; GFX9: $vgpr0_vgpr1 = COPY [[CONCAT_VECTORS]](<4 x s16>)
+    %0:_(<4 x s16>) = COPY $vgpr0_vgpr1
+    %1:_(<4 x s16>) = COPY $vgpr2_vgpr3
+    %2:_(<4 x s16>) = G_FMUL %0, %1
+    $vgpr0_vgpr1 = COPY %2
+...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/legalize-fneg.mir b/test/CodeGen/AMDGPU/GlobalISel/legalize-fneg.mir
index 7d280f7f9c11781490a514e78cb0efbe0717a51c..98af236936bd9f4eb088d514204f9ff2d92952d9 100644
--- a/test/CodeGen/AMDGPU/GlobalISel/legalize-fneg.mir
+++ b/test/CodeGen/AMDGPU/GlobalISel/legalize-fneg.mir
@@ -1,25 +1,303 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=fiji -run-pass=legalizer -global-isel %s -o - | FileCheck  %s
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=tahiti -run-pass=legalizer %s -o - | FileCheck -check-prefix=SI  %s
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -run-pass=legalizer %s -o - | FileCheck -check-prefix=VI %s
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx900 -run-pass=legalizer %s -o - | FileCheck -check-prefix=GFX9  %s
 
 ---
-name: test_fneg_f32
+name: test_fneg_s32
 body: |
   bb.0:
     liveins: $vgpr0
 
-    ; CHECK-LABEL: name: test_fneg_f32
-    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; SI-LABEL: name: test_fneg_s32
+    ; SI: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; SI: [[FNEG:%[0-9]+]]:_(s32) = G_FNEG [[COPY]]
+    ; SI: $vgpr0 = COPY [[FNEG]](s32)
+    ; VI-LABEL: name: test_fneg_s32
+    ; VI: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; VI: [[FNEG:%[0-9]+]]:_(s32) = G_FNEG [[COPY]]
+    ; VI: $vgpr0 = COPY [[FNEG]](s32)
+    ; GFX9-LABEL: name: test_fneg_s32
+    ; GFX9: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; GFX9: [[FNEG:%[0-9]+]]:_(s32) = G_FNEG [[COPY]]
+    ; GFX9: $vgpr0 = COPY [[FNEG]](s32)
     %0:_(s32) = COPY $vgpr0
     %1:_(s32) = G_FNEG %0
+    $vgpr0 = COPY %1
 ...
 ---
-name: test_fneg_f64
+name: test_fneg_s64
 body: |
   bb.0:
     liveins: $vgpr0
 
-    ; CHECK-LABEL: name: test_fneg_f64
-    ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
+    ; SI-LABEL: name: test_fneg_s64
+    ; SI: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
+    ; SI: [[FNEG:%[0-9]+]]:_(s64) = G_FNEG [[COPY]]
+    ; SI: $vgpr0_vgpr1 = COPY [[FNEG]](s64)
+    ; VI-LABEL: name: test_fneg_s64
+    ; VI: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
+    ; VI: [[FNEG:%[0-9]+]]:_(s64) = G_FNEG [[COPY]]
+    ; VI: $vgpr0_vgpr1 = COPY [[FNEG]](s64)
+    ; GFX9-LABEL: name: test_fneg_s64
+    ; GFX9: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
+    ; GFX9: [[FNEG:%[0-9]+]]:_(s64) = G_FNEG [[COPY]]
+    ; GFX9: $vgpr0_vgpr1 = COPY [[FNEG]](s64)
     %0:_(s64) = COPY $vgpr0_vgpr1
     %1:_(s64) = G_FNEG %0
+    $vgpr0_vgpr1 = COPY %1
+...
+---
+name: test_fneg_s16
+body: |
+  bb.0:
+    liveins: $vgpr0
+
+    ; SI-LABEL: name: test_fneg_s16
+    ; SI: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; SI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+    ; SI: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC]](s16)
+    ; SI: [[FNEG:%[0-9]+]]:_(s32) = G_FNEG [[FPEXT]]
+    ; SI: [[FPTRUNC:%[0-9]+]]:_(s16) = G_FPTRUNC [[FNEG]](s32)
+    ; SI: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[FPTRUNC]](s16)
+    ; SI: $vgpr0 = COPY [[ANYEXT]](s32)
+    ; VI-LABEL: name: test_fneg_s16
+    ; VI: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; VI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+    ; VI: [[FNEG:%[0-9]+]]:_(s16) = G_FNEG [[TRUNC]]
+    ; VI: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[FNEG]](s16)
+    ; VI: $vgpr0 = COPY [[ANYEXT]](s32)
+    ; GFX9-LABEL: name: test_fneg_s16
+    ; GFX9: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; GFX9: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+    ; GFX9: [[FNEG:%[0-9]+]]:_(s16) = G_FNEG [[TRUNC]]
+    ; GFX9: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[FNEG]](s16)
+    ; GFX9: $vgpr0 = COPY [[ANYEXT]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s16) = G_TRUNC %0
+    %2:_(s16) = G_FNEG %1
+    %3:_(s32) = G_ANYEXT %2
+    $vgpr0 = COPY %3
+...
+
+---
+name: test_fneg_v2s32
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; SI-LABEL: name: test_fneg_v2s32
+    ; SI: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    ; SI: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
+    ; SI: [[FNEG:%[0-9]+]]:_(s32) = G_FNEG [[UV]]
+    ; SI: [[FNEG1:%[0-9]+]]:_(s32) = G_FNEG [[UV1]]
+    ; SI: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[FNEG]](s32), [[FNEG1]](s32)
+    ; SI: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
+    ; VI-LABEL: name: test_fneg_v2s32
+    ; VI: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    ; VI: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
+    ; VI: [[FNEG:%[0-9]+]]:_(s32) = G_FNEG [[UV]]
+    ; VI: [[FNEG1:%[0-9]+]]:_(s32) = G_FNEG [[UV1]]
+    ; VI: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[FNEG]](s32), [[FNEG1]](s32)
+    ; VI: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
+    ; GFX9-LABEL: name: test_fneg_v2s32
+    ; GFX9: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    ; GFX9: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
+    ; GFX9: [[FNEG:%[0-9]+]]:_(s32) = G_FNEG [[UV]]
+    ; GFX9: [[FNEG1:%[0-9]+]]:_(s32) = G_FNEG [[UV1]]
+    ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[FNEG]](s32), [[FNEG1]](s32)
+    ; GFX9: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
+    %0:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    %1:_(<2 x s32>) = G_FNEG %0
+    $vgpr0_vgpr1 = COPY %1
+...
+
+---
+name: test_fneg_v3s32
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2
+
+    ; SI-LABEL: name: test_fneg_v3s32
+    ; SI: [[COPY:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr0_vgpr1_vgpr2
+    ; SI: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<3 x s32>)
+    ; SI: [[FNEG:%[0-9]+]]:_(s32) = G_FNEG [[UV]]
+    ; SI: [[FNEG1:%[0-9]+]]:_(s32) = G_FNEG [[UV1]]
+    ; SI: [[FNEG2:%[0-9]+]]:_(s32) = G_FNEG [[UV2]]
+    ; SI: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[FNEG]](s32), [[FNEG1]](s32), [[FNEG2]](s32)
+    ; SI: $vgpr0_vgpr1_vgpr2 = COPY [[BUILD_VECTOR]](<3 x s32>)
+    ; VI-LABEL: name: test_fneg_v3s32
+    ; VI: [[COPY:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr0_vgpr1_vgpr2
+    ; VI: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<3 x s32>)
+    ; VI: [[FNEG:%[0-9]+]]:_(s32) = G_FNEG [[UV]]
+    ; VI: [[FNEG1:%[0-9]+]]:_(s32) = G_FNEG [[UV1]]
+    ; VI: [[FNEG2:%[0-9]+]]:_(s32) = G_FNEG [[UV2]]
+    ; VI: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[FNEG]](s32), [[FNEG1]](s32), [[FNEG2]](s32)
+    ; VI: $vgpr0_vgpr1_vgpr2 = COPY [[BUILD_VECTOR]](<3 x s32>)
+    ; GFX9-LABEL: name: test_fneg_v3s32
+    ; GFX9: [[COPY:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr0_vgpr1_vgpr2
+    ; GFX9: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<3 x s32>)
+    ; GFX9: [[FNEG:%[0-9]+]]:_(s32) = G_FNEG [[UV]]
+    ; GFX9: [[FNEG1:%[0-9]+]]:_(s32) = G_FNEG [[UV1]]
+    ; GFX9: [[FNEG2:%[0-9]+]]:_(s32) = G_FNEG [[UV2]]
+    ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[FNEG]](s32), [[FNEG1]](s32), [[FNEG2]](s32)
+    ; GFX9: $vgpr0_vgpr1_vgpr2 = COPY [[BUILD_VECTOR]](<3 x s32>)
+    %0:_(<3 x s32>) = COPY $vgpr0_vgpr1_vgpr2
+    %1:_(<3 x  s32>) = G_FNEG %0
+    $vgpr0_vgpr1_vgpr2 = COPY %1
+...
+
+---
+name: test_fneg_v2s64
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3
+
+    ; SI-LABEL: name: test_fneg_v2s64
+    ; SI: [[COPY:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; SI: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](<2 x s64>)
+    ; SI: [[FNEG:%[0-9]+]]:_(s64) = G_FNEG [[UV]]
+    ; SI: [[FNEG1:%[0-9]+]]:_(s64) = G_FNEG [[UV1]]
+    ; SI: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[FNEG]](s64), [[FNEG1]](s64)
+    ; SI: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>)
+    ; VI-LABEL: name: test_fneg_v2s64
+    ; VI: [[COPY:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; VI: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](<2 x s64>)
+    ; VI: [[FNEG:%[0-9]+]]:_(s64) = G_FNEG [[UV]]
+    ; VI: [[FNEG1:%[0-9]+]]:_(s64) = G_FNEG [[UV1]]
+    ; VI: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[FNEG]](s64), [[FNEG1]](s64)
+    ; VI: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>)
+    ; GFX9-LABEL: name: test_fneg_v2s64
+    ; GFX9: [[COPY:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; GFX9: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](<2 x s64>)
+    ; GFX9: [[FNEG:%[0-9]+]]:_(s64) = G_FNEG [[UV]]
+    ; GFX9: [[FNEG1:%[0-9]+]]:_(s64) = G_FNEG [[UV1]]
+    ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[FNEG]](s64), [[FNEG1]](s64)
+    ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>)
+    %0:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    %1:_(<2 x s64>) = G_FNEG %0
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %1
+...
+
+---
+name: test_fneg_v2s16
+body: |
+  bb.0:
+    liveins: $vgpr0
+
+    ; SI-LABEL: name: test_fneg_v2s16
+    ; SI: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
+    ; SI: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY]](<2 x s16>)
+    ; SI: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT [[UV]](s16)
+    ; SI: [[FNEG:%[0-9]+]]:_(s32) = G_FNEG [[FPEXT]]
+    ; SI: [[FPTRUNC:%[0-9]+]]:_(s16) = G_FPTRUNC [[FNEG]](s32)
+    ; SI: [[FPEXT1:%[0-9]+]]:_(s32) = G_FPEXT [[UV1]](s16)
+    ; SI: [[FNEG1:%[0-9]+]]:_(s32) = G_FNEG [[FPEXT1]]
+    ; SI: [[FPTRUNC1:%[0-9]+]]:_(s16) = G_FPTRUNC [[FNEG1]](s32)
+    ; SI: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[FPTRUNC]](s16), [[FPTRUNC1]](s16)
+    ; SI: $vgpr0 = COPY [[BUILD_VECTOR]](<2 x s16>)
+    ; VI-LABEL: name: test_fneg_v2s16
+    ; VI: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
+    ; VI: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY]](<2 x s16>)
+    ; VI: [[FNEG:%[0-9]+]]:_(s16) = G_FNEG [[UV]]
+    ; VI: [[FNEG1:%[0-9]+]]:_(s16) = G_FNEG [[UV1]]
+    ; VI: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[FNEG]](s16), [[FNEG1]](s16)
+    ; VI: $vgpr0 = COPY [[BUILD_VECTOR]](<2 x s16>)
+    ; GFX9-LABEL: name: test_fneg_v2s16
+    ; GFX9: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
+    ; GFX9: [[FNEG:%[0-9]+]]:_(<2 x s16>) = G_FNEG [[COPY]]
+    ; GFX9: $vgpr0 = COPY [[FNEG]](<2 x s16>)
+    %0:_(<2 x s16>) = COPY $vgpr0
+    %1:_(<2 x s16>) = G_FNEG %0
+    $vgpr0 = COPY %1
+...
+
+---
+name: test_fneg_v3s16
+body: |
+  bb.0:
+
+    ; SI-LABEL: name: test_fneg_v3s16
+    ; SI: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; SI: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
+    ; SI: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16), [[UV2:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[EXTRACT]](<3 x s16>)
+    ; SI: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT [[UV]](s16)
+    ; SI: [[FNEG:%[0-9]+]]:_(s32) = G_FNEG [[FPEXT]]
+    ; SI: [[FPTRUNC:%[0-9]+]]:_(s16) = G_FPTRUNC [[FNEG]](s32)
+    ; SI: [[FPEXT1:%[0-9]+]]:_(s32) = G_FPEXT [[UV1]](s16)
+    ; SI: [[FNEG1:%[0-9]+]]:_(s32) = G_FNEG [[FPEXT1]]
+    ; SI: [[FPTRUNC1:%[0-9]+]]:_(s16) = G_FPTRUNC [[FNEG1]](s32)
+    ; SI: [[FPEXT2:%[0-9]+]]:_(s32) = G_FPEXT [[UV2]](s16)
+    ; SI: [[FNEG2:%[0-9]+]]:_(s32) = G_FNEG [[FPEXT2]]
+    ; SI: [[FPTRUNC2:%[0-9]+]]:_(s16) = G_FPTRUNC [[FNEG2]](s32)
+    ; SI: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s16>) = G_BUILD_VECTOR [[FPTRUNC]](s16), [[FPTRUNC1]](s16), [[FPTRUNC2]](s16)
+    ; SI: S_NOP 0, implicit [[BUILD_VECTOR]](<3 x s16>)
+    ; VI-LABEL: name: test_fneg_v3s16
+    ; VI: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; VI: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
+    ; VI: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16), [[UV2:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[EXTRACT]](<3 x s16>)
+    ; VI: [[FNEG:%[0-9]+]]:_(s16) = G_FNEG [[UV]]
+    ; VI: [[FNEG1:%[0-9]+]]:_(s16) = G_FNEG [[UV1]]
+    ; VI: [[FNEG2:%[0-9]+]]:_(s16) = G_FNEG [[UV2]]
+    ; VI: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s16>) = G_BUILD_VECTOR [[FNEG]](s16), [[FNEG1]](s16), [[FNEG2]](s16)
+    ; VI: S_NOP 0, implicit [[BUILD_VECTOR]](<3 x s16>)
+    ; GFX9-LABEL: name: test_fneg_v3s16
+    ; GFX9: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; GFX9: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
+    ; GFX9: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; GFX9: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF1]](<4 x s16>), 0
+    ; GFX9: [[EXTRACT2:%[0-9]+]]:_(<2 x s16>) = G_EXTRACT [[EXTRACT]](<3 x s16>), 0
+    ; GFX9: [[FNEG:%[0-9]+]]:_(<2 x s16>) = G_FNEG [[EXTRACT2]]
+    ; GFX9: [[INSERT:%[0-9]+]]:_(<3 x s16>) = G_INSERT [[EXTRACT1]], [[FNEG]](<2 x s16>), 0
+    ; GFX9: [[EXTRACT3:%[0-9]+]]:_(s16) = G_EXTRACT [[EXTRACT]](<3 x s16>), 32
+    ; GFX9: [[FNEG1:%[0-9]+]]:_(s16) = G_FNEG [[EXTRACT3]]
+    ; GFX9: [[INSERT1:%[0-9]+]]:_(<3 x s16>) = G_INSERT [[INSERT]], [[FNEG1]](s16), 32
+    ; GFX9: S_NOP 0, implicit [[INSERT1]](<3 x s16>)
+    %0:_(<3 x s16>) = G_IMPLICIT_DEF
+    %1:_(<3 x s16>) = G_FNEG %0
+    S_NOP 0, implicit %1
+...
+
+---
+name: test_fneg_v4s16
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; SI-LABEL: name: test_fneg_v4s16
+    ; SI: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
+    ; SI: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16), [[UV2:%[0-9]+]]:_(s16), [[UV3:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
+    ; SI: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT [[UV]](s16)
+    ; SI: [[FNEG:%[0-9]+]]:_(s32) = G_FNEG [[FPEXT]]
+    ; SI: [[FPTRUNC:%[0-9]+]]:_(s16) = G_FPTRUNC [[FNEG]](s32)
+    ; SI: [[FPEXT1:%[0-9]+]]:_(s32) = G_FPEXT [[UV1]](s16)
+    ; SI: [[FNEG1:%[0-9]+]]:_(s32) = G_FNEG [[FPEXT1]]
+    ; SI: [[FPTRUNC1:%[0-9]+]]:_(s16) = G_FPTRUNC [[FNEG1]](s32)
+    ; SI: [[FPEXT2:%[0-9]+]]:_(s32) = G_FPEXT [[UV2]](s16)
+    ; SI: [[FNEG2:%[0-9]+]]:_(s32) = G_FNEG [[FPEXT2]]
+    ; SI: [[FPTRUNC2:%[0-9]+]]:_(s16) = G_FPTRUNC [[FNEG2]](s32)
+    ; SI: [[FPEXT3:%[0-9]+]]:_(s32) = G_FPEXT [[UV3]](s16)
+    ; SI: [[FNEG3:%[0-9]+]]:_(s32) = G_FNEG [[FPEXT3]]
+    ; SI: [[FPTRUNC3:%[0-9]+]]:_(s16) = G_FPTRUNC [[FNEG3]](s32)
+    ; SI: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s16>) = G_BUILD_VECTOR [[FPTRUNC]](s16), [[FPTRUNC1]](s16), [[FPTRUNC2]](s16), [[FPTRUNC3]](s16)
+    ; SI: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<4 x s16>)
+    ; VI-LABEL: name: test_fneg_v4s16
+    ; VI: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
+    ; VI: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16), [[UV2:%[0-9]+]]:_(s16), [[UV3:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
+    ; VI: [[FNEG:%[0-9]+]]:_(s16) = G_FNEG [[UV]]
+    ; VI: [[FNEG1:%[0-9]+]]:_(s16) = G_FNEG [[UV1]]
+    ; VI: [[FNEG2:%[0-9]+]]:_(s16) = G_FNEG [[UV2]]
+    ; VI: [[FNEG3:%[0-9]+]]:_(s16) = G_FNEG [[UV3]]
+    ; VI: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s16>) = G_BUILD_VECTOR [[FNEG]](s16), [[FNEG1]](s16), [[FNEG2]](s16), [[FNEG3]](s16)
+    ; VI: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<4 x s16>)
+    ; GFX9-LABEL: name: test_fneg_v4s16
+    ; GFX9: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
+    ; GFX9: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
+    ; GFX9: [[FNEG:%[0-9]+]]:_(<2 x s16>) = G_FNEG [[UV]]
+    ; GFX9: [[FNEG1:%[0-9]+]]:_(<2 x s16>) = G_FNEG [[UV1]]
+    ; GFX9: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[FNEG]](<2 x s16>), [[FNEG1]](<2 x s16>)
+    ; GFX9: $vgpr0_vgpr1 = COPY [[CONCAT_VECTORS]](<4 x s16>)
+    %0:_(<4 x s16>) = COPY $vgpr0_vgpr1
+    %1:_(<4 x s16>) = G_FNEG %0
+    $vgpr0_vgpr1 = COPY %1
 ...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/legalize-fpext.mir b/test/CodeGen/AMDGPU/GlobalISel/legalize-fpext.mir
new file mode 100644
index 0000000000000000000000000000000000000000..257385ff9f44d2cfc7b339cf9c8e44fb9a4d50f0
--- /dev/null
+++ b/test/CodeGen/AMDGPU/GlobalISel/legalize-fpext.mir
@@ -0,0 +1,168 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -run-pass=legalizer %s -o - | FileCheck %s
+
+---
+name: test_fpext_f16_to_f32
+body: |
+  bb.0:
+    liveins: $vgpr0
+
+    ; CHECK-LABEL: name: test_fpext_f16_to_f32
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+    ; CHECK: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC]](s16)
+    ; CHECK: $vgpr0 = COPY [[FPEXT]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s16) = G_TRUNC %0
+    %2:_(s32) = G_FPEXT %1
+    $vgpr0 = COPY %2
+...
+
+---
+name: test_fpext_v2f16_to_v2f32
+body: |
+  bb.0:
+    liveins: $vgpr0
+
+    ; CHECK-LABEL: name: test_fpext_v2f16_to_v2f32
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
+    ; CHECK: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY]](<2 x s16>)
+    ; CHECK: %4:_(s32) = nnan G_FPEXT [[UV]](s16)
+    ; CHECK: %5:_(s32) = nnan G_FPEXT [[UV1]](s16)
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR %4(s32), %5(s32)
+    ; CHECK: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
+    %0:_(<2 x s16>) = COPY $vgpr0
+    %1:_(<2 x s32>) = nnan G_FPEXT %0
+    $vgpr0_vgpr1 = COPY %1
+...
+
+---
+name: test_fpext_v2f16_to_v2f32_w_flags
+body: |
+  bb.0:
+    liveins: $vgpr0
+
+    ; CHECK-LABEL: name: test_fpext_v2f16_to_v2f32_w_flags
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
+    ; CHECK: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY]](<2 x s16>)
+    ; CHECK: %4:_(s32) = nnan G_FPEXT [[UV]](s16)
+    ; CHECK: %5:_(s32) = nnan G_FPEXT [[UV1]](s16)
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR %4(s32), %5(s32)
+    ; CHECK: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
+    %0:_(<2 x s16>) = COPY $vgpr0
+    %1:_(<2 x s32>) = nnan G_FPEXT %0
+    $vgpr0_vgpr1 = COPY %1
+...
+
+---
+name: test_fpext_v3f16_to_v3f32
+body: |
+  bb.0:
+    liveins: $vgpr0
+
+    ; CHECK-LABEL: name: test_fpext_v3f16_to_v3f32
+    ; CHECK: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; CHECK: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
+    ; CHECK: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16), [[UV2:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[EXTRACT]](<3 x s16>)
+    ; CHECK: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT [[UV]](s16)
+    ; CHECK: [[FPEXT1:%[0-9]+]]:_(s32) = G_FPEXT [[UV1]](s16)
+    ; CHECK: [[FPEXT2:%[0-9]+]]:_(s32) = G_FPEXT [[UV2]](s16)
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[FPEXT]](s32), [[FPEXT1]](s32), [[FPEXT2]](s32)
+    ; CHECK: $vgpr0_vgpr1_vgpr2 = COPY [[BUILD_VECTOR]](<3 x s32>)
+    %0:_(<3 x s16>) = G_IMPLICIT_DEF
+    %1:_(<3 x s32>) = G_FPEXT %0
+    $vgpr0_vgpr1_vgpr2 = COPY %1
+...
+
+---
+name: test_fpext_v4f16_to_v4f32
+body: |
+  bb.0:
+    liveins: $vgpr0
+
+    ; CHECK-LABEL: name: test_fpext_v4f16_to_v4f32
+    ; CHECK: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; CHECK: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16), [[UV2:%[0-9]+]]:_(s16), [[UV3:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[DEF]](<4 x s16>)
+    ; CHECK: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT [[UV]](s16)
+    ; CHECK: [[FPEXT1:%[0-9]+]]:_(s32) = G_FPEXT [[UV1]](s16)
+    ; CHECK: [[FPEXT2:%[0-9]+]]:_(s32) = G_FPEXT [[UV2]](s16)
+    ; CHECK: [[FPEXT3:%[0-9]+]]:_(s32) = G_FPEXT [[UV3]](s16)
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[FPEXT]](s32), [[FPEXT1]](s32), [[FPEXT2]](s32), [[FPEXT3]](s32)
+    ; CHECK: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<4 x s32>)
+    %0:_(<4 x s16>) = G_IMPLICIT_DEF
+    %1:_(<4 x s32>) = G_FPEXT %0
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %1
+...
+
+---
+name: test_fpext_f32_to_f64
+body: |
+  bb.0:
+    liveins: $vgpr0
+
+    ; CHECK-LABEL: name: test_fpext_f32_to_f64
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[FPEXT:%[0-9]+]]:_(s64) = G_FPEXT [[COPY]](s32)
+    ; CHECK: $vgpr0_vgpr1 = COPY [[FPEXT]](s64)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s64) = G_FPEXT %0
+    $vgpr0_vgpr1 = COPY %1
+...
+
+---
+name: test_fpext_v2f32_to_v2f64
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; CHECK-LABEL: name: test_fpext_v2f32_to_v2f64
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    ; CHECK: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
+    ; CHECK: [[FPEXT:%[0-9]+]]:_(s64) = G_FPEXT [[UV]](s32)
+    ; CHECK: [[FPEXT1:%[0-9]+]]:_(s64) = G_FPEXT [[UV1]](s32)
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[FPEXT]](s64), [[FPEXT1]](s64)
+    ; CHECK: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>)
+    %0:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    %1:_(<2 x s64>) = G_FPEXT %0
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %1
+...
+
+---
+name: test_fpext_v3f32_to_v3f64
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2
+
+    ; CHECK-LABEL: name: test_fpext_v3f32_to_v3f64
+    ; CHECK: [[COPY:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr0_vgpr1_vgpr2
+    ; CHECK: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<3 x s32>)
+    ; CHECK: [[FPEXT:%[0-9]+]]:_(s64) = G_FPEXT [[UV]](s32)
+    ; CHECK: [[FPEXT1:%[0-9]+]]:_(s64) = G_FPEXT [[UV1]](s32)
+    ; CHECK: [[FPEXT2:%[0-9]+]]:_(s64) = G_FPEXT [[UV2]](s32)
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s64>) = G_BUILD_VECTOR [[FPEXT]](s64), [[FPEXT1]](s64), [[FPEXT2]](s64)
+    ; CHECK: S_NOP 0, implicit [[BUILD_VECTOR]](<3 x s64>)
+    %0:_(<3 x s32>) = COPY $vgpr0_vgpr1_vgpr2
+    %1:_(<3 x s64>) = G_FPEXT %0
+    S_NOP 0, implicit %1
+
+...
+
+---
+name: test_fpext_v4f32_to_v4f64
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3
+
+    ; CHECK-LABEL: name: test_fpext_v4f32_to_v4f64
+    ; CHECK: [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; CHECK: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<4 x s32>)
+    ; CHECK: [[FPEXT:%[0-9]+]]:_(s64) = G_FPEXT [[UV]](s32)
+    ; CHECK: [[FPEXT1:%[0-9]+]]:_(s64) = G_FPEXT [[UV1]](s32)
+    ; CHECK: [[FPEXT2:%[0-9]+]]:_(s64) = G_FPEXT [[UV2]](s32)
+    ; CHECK: [[FPEXT3:%[0-9]+]]:_(s64) = G_FPEXT [[UV3]](s32)
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s64>) = G_BUILD_VECTOR [[FPEXT]](s64), [[FPEXT1]](s64), [[FPEXT2]](s64), [[FPEXT3]](s64)
+    ; CHECK: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY [[BUILD_VECTOR]](<4 x s64>)
+    %0:_(<4 x s32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    %1:_(<4 x s64>) = G_FPEXT %0
+    $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY %1
+...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/legalize-fpow.mir b/test/CodeGen/AMDGPU/GlobalISel/legalize-fpow.mir
new file mode 100644
index 0000000000000000000000000000000000000000..a03b6e2259b777e53b1b6b7e6b01167d36f5a86a
--- /dev/null
+++ b/test/CodeGen/AMDGPU/GlobalISel/legalize-fpow.mir
@@ -0,0 +1,62 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=tahiti -run-pass=legalizer %s -o - | FileCheck %s
+
+---
+name: test_fpow_s32
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+
+    ; CHECK-LABEL: name: test_fpow_s32
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; CHECK: [[FPOW:%[0-9]+]]:_(s32) = G_FPOW [[COPY]], [[COPY1]]
+    ; CHECK: $vgpr0 = COPY [[FPOW]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s32) = G_FPOW %0, %1
+    $vgpr0 = COPY %2
+...
+
+---
+name: test_fpow_v2s32
+body: |
+  bb.0.entry:
+    liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
+
+    ; CHECK-LABEL: name: test_fpow_v2s32
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    ; CHECK: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr2_vgpr3
+    ; CHECK: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
+    ; CHECK: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<2 x s32>)
+    ; CHECK: [[FPOW:%[0-9]+]]:_(s32) = G_FPOW [[UV]], [[UV2]]
+    ; CHECK: [[FPOW1:%[0-9]+]]:_(s32) = G_FPOW [[UV1]], [[UV3]]
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[FPOW]](s32), [[FPOW1]](s32)
+    ; CHECK: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
+    %0:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    %1:_(<2 x s32>) = COPY $vgpr2_vgpr3
+    %2:_(<2 x s32>) = G_FPOW %0, %1
+    $vgpr0_vgpr1 = COPY %2
+...
+
+---
+name: test_fpow_v3s32
+body: |
+  bb.0.entry:
+    liveins: $vgpr0_vgpr1_vgpr2, $vgpr3_vgpr4_vgpr5
+
+    ; CHECK-LABEL: name: test_fpow_v3s32
+    ; CHECK: [[COPY:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr0_vgpr1_vgpr2
+    ; CHECK: [[COPY1:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr3_vgpr4_vgpr5
+    ; CHECK: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<3 x s32>)
+    ; CHECK: [[UV3:%[0-9]+]]:_(s32), [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<3 x s32>)
+    ; CHECK: [[FPOW:%[0-9]+]]:_(s32) = G_FPOW [[UV]], [[UV3]]
+    ; CHECK: [[FPOW1:%[0-9]+]]:_(s32) = G_FPOW [[UV1]], [[UV4]]
+    ; CHECK: [[FPOW2:%[0-9]+]]:_(s32) = G_FPOW [[UV2]], [[UV5]]
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[FPOW]](s32), [[FPOW1]](s32), [[FPOW2]](s32)
+    ; CHECK: $vgpr0_vgpr1_vgpr2 = COPY [[BUILD_VECTOR]](<3 x s32>)
+    %0:_(<3 x s32>) = COPY $vgpr0_vgpr1_vgpr2
+    %1:_(<3 x s32>) = COPY $vgpr3_vgpr4_vgpr5
+    %2:_(<3 x s32>) = G_FPOW %0, %1
+    $vgpr0_vgpr1_vgpr2 = COPY %2
+...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/legalize-fptosi.mir b/test/CodeGen/AMDGPU/GlobalISel/legalize-fptosi.mir
index d4414bc62f0e0742b97db9eb396728fa794315e3..2da5ffd1d034e31a61bdea54074c3cbb8676b2cf 100644
--- a/test/CodeGen/AMDGPU/GlobalISel/legalize-fptosi.mir
+++ b/test/CodeGen/AMDGPU/GlobalISel/legalize-fptosi.mir
@@ -2,13 +2,67 @@
 # RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -run-pass=legalizer %s -o - | FileCheck %s
 
 ---
-name: test_fptosi_f32_to_i32
+name: test_fptosi_s32_to_s32
 body: |
   bb.0:
     liveins: $vgpr0
 
-    ; CHECK-LABEL: name: test_fptosi_f32_to_i32
+    ; CHECK-LABEL: name: test_fptosi_s32_to_s32
     ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[FPTOSI:%[0-9]+]]:_(s32) = G_FPTOSI [[COPY]](s32)
+    ; CHECK: $vgpr0 = COPY [[FPTOSI]](s32)
     %0:_(s32) = COPY $vgpr0
     %1:_(s32) = G_FPTOSI %0
+    $vgpr0 = COPY %1
+...
+
+---
+name: test_fptosi_s64_to_s32
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; CHECK-LABEL: name: test_fptosi_s64_to_s32
+    ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
+    ; CHECK: [[FPTOSI:%[0-9]+]]:_(s32) = G_FPTOSI [[COPY]](s64)
+    ; CHECK: $vgpr0 = COPY [[FPTOSI]](s32)
+    %0:_(s64) = COPY $vgpr0_vgpr1
+    %1:_(s32) = G_FPTOSI %0
+    $vgpr0 = COPY %1
+...
+
+---
+name: test_fptosi_v2s32_to_v2s32
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; CHECK-LABEL: name: test_fptosi_v2s32_to_v2s32
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    ; CHECK: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
+    ; CHECK: [[FPTOSI:%[0-9]+]]:_(s32) = G_FPTOSI [[UV]](s32)
+    ; CHECK: [[FPTOSI1:%[0-9]+]]:_(s32) = G_FPTOSI [[UV1]](s32)
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[FPTOSI]](s32), [[FPTOSI1]](s32)
+    ; CHECK: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
+    %0:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    %1:_(<2 x s32>) = G_FPTOSI %0
+    $vgpr0_vgpr1 = COPY %1
+...
+
+---
+name: test_fptosi_v2s64_to_v2s32
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3
+
+    ; CHECK-LABEL: name: test_fptosi_v2s64_to_v2s32
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; CHECK: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](<2 x s64>)
+    ; CHECK: [[FPTOSI:%[0-9]+]]:_(s32) = G_FPTOSI [[UV]](s64)
+    ; CHECK: [[FPTOSI1:%[0-9]+]]:_(s32) = G_FPTOSI [[UV1]](s64)
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[FPTOSI]](s32), [[FPTOSI1]](s32)
+    ; CHECK: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
+    %0:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    %1:_(<2 x s32>) = G_FPTOSI %0
+    $vgpr0_vgpr1 = COPY %1
 ...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/legalize-fptoui.mir b/test/CodeGen/AMDGPU/GlobalISel/legalize-fptoui.mir
index aede6af2603590517f71fe17103590369c327e49..f82c72bd2332eb9eed38404c42e04aa058460e4a 100644
--- a/test/CodeGen/AMDGPU/GlobalISel/legalize-fptoui.mir
+++ b/test/CodeGen/AMDGPU/GlobalISel/legalize-fptoui.mir
@@ -1,14 +1,68 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
 # RUN: llc -mtriple=amdgcn-mesa-mesa3d -run-pass=legalizer %s -o - | FileCheck %s
 
 ---
-name:            test_fptoui
+name: test_fptoui_s32_s32
 body: |
   bb.0:
     liveins: $vgpr0
-    ; CHECK-LABEL: name: test_fptoui
-    ; CHECK: %1:_(s32) = G_FPTOUI %0
 
+    ; CHECK-LABEL: name: test_fptoui_s32_s32
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[COPY]](s32)
+    ; CHECK: $vgpr0 = COPY [[FPTOUI]](s32)
     %0:_(s32) = COPY $vgpr0
     %1:_(s32) = G_FPTOUI %0
     $vgpr0 = COPY %1
 ...
+
+---
+name: test_fptoui_s32_s64
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; CHECK-LABEL: name: test_fptoui_s32_s64
+    ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
+    ; CHECK: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[COPY]](s64)
+    ; CHECK: $vgpr0 = COPY [[FPTOUI]](s32)
+    %0:_(s64) = COPY $vgpr0_vgpr1
+    %1:_(s32) = G_FPTOUI %0
+    $vgpr0 = COPY %1
+...
+
+---
+name: test_fptoui_v2s32_to_v2s32
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; CHECK-LABEL: name: test_fptoui_v2s32_to_v2s32
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    ; CHECK: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
+    ; CHECK: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[UV]](s32)
+    ; CHECK: [[FPTOUI1:%[0-9]+]]:_(s32) = G_FPTOUI [[UV1]](s32)
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[FPTOUI]](s32), [[FPTOUI1]](s32)
+    ; CHECK: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
+    %0:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    %1:_(<2 x s32>) = G_FPTOUI %0
+    $vgpr0_vgpr1 = COPY %1
+...
+
+---
+name: test_fptoui_v2s64_to_v2s32
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3
+
+    ; CHECK-LABEL: name: test_fptoui_v2s64_to_v2s32
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; CHECK: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](<2 x s64>)
+    ; CHECK: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[UV]](s64)
+    ; CHECK: [[FPTOUI1:%[0-9]+]]:_(s32) = G_FPTOUI [[UV1]](s64)
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[FPTOUI]](s32), [[FPTOUI1]](s32)
+    ; CHECK: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
+    %0:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    %1:_(<2 x s32>) = G_FPTOUI %0
+    $vgpr0_vgpr1 = COPY %1
+...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/legalize-fptrunc.mir b/test/CodeGen/AMDGPU/GlobalISel/legalize-fptrunc.mir
new file mode 100644
index 0000000000000000000000000000000000000000..5a66e8a6c2e064d3e97ad91af26fa1987aa7095a
--- /dev/null
+++ b/test/CodeGen/AMDGPU/GlobalISel/legalize-fptrunc.mir
@@ -0,0 +1,73 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -run-pass=legalizer -global-isel %s -o - | FileCheck %s
+
+---
+name: test_fptrunc_s64_to_s32
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; CHECK-LABEL: name: test_fptrunc_s64_to_s32
+    ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
+    ; CHECK: [[FPTRUNC:%[0-9]+]]:_(s32) = G_FPTRUNC [[COPY]](s64)
+    ; CHECK: $vgpr0 = COPY [[FPTRUNC]](s32)
+    %0:_(s64) = COPY $vgpr0_vgpr1
+    %1:_(s32) = G_FPTRUNC %0
+    $vgpr0 = COPY %1
+...
+
+---
+name: test_fptrunc_s32_to_s16
+body: |
+  bb.0:
+    liveins: $vgpr0
+
+    ; CHECK-LABEL: name: test_fptrunc_s32_to_s16
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[FPTRUNC:%[0-9]+]]:_(s16) = G_FPTRUNC [[COPY]](s32)
+    ; CHECK: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[FPTRUNC]](s16)
+    ; CHECK: $vgpr0 = COPY [[ANYEXT]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s16) = G_FPTRUNC %0
+    %2:_(s32) = G_ANYEXT %1
+    $vgpr0 = COPY %2
+...
+
+---
+name: test_fptrunc_v2s64_to_v2s32
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3
+
+    ; CHECK-LABEL: name: test_fptrunc_v2s64_to_v2s32
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; CHECK: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](<2 x s64>)
+    ; CHECK: [[FPTRUNC:%[0-9]+]]:_(s32) = G_FPTRUNC [[UV]](s64)
+    ; CHECK: [[FPTRUNC1:%[0-9]+]]:_(s32) = G_FPTRUNC [[UV1]](s64)
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[FPTRUNC]](s32), [[FPTRUNC1]](s32)
+    ; CHECK: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
+    %0:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    %1:_(<2 x s32>) = G_FPTRUNC %0
+    $vgpr0_vgpr1 = COPY %1
+...
+
+---
+name: test_fptrunc_v2s32_to_v2s16
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; CHECK-LABEL: name: test_fptrunc_v2s32_to_v2s16
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    ; CHECK: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
+    ; CHECK: [[FPTRUNC:%[0-9]+]]:_(s16) = G_FPTRUNC [[UV]](s32)
+    ; CHECK: [[FPTRUNC1:%[0-9]+]]:_(s16) = G_FPTRUNC [[UV1]](s32)
+    ; CHECK: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[FPTRUNC]](s16)
+    ; CHECK: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[FPTRUNC1]](s16)
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[ANYEXT]](s32), [[ANYEXT1]](s32)
+    ; CHECK: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
+    %0:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    %1:_(<2 x s16>) = G_FPTRUNC %0
+    %2:_(<2 x s32>) = G_ANYEXT %1
+    $vgpr0_vgpr1 = COPY %2
+...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/legalize-fsqrt.mir b/test/CodeGen/AMDGPU/GlobalISel/legalize-fsqrt.mir
new file mode 100644
index 0000000000000000000000000000000000000000..446907b991cb47413831d23e2ac8a8fd3617cc9c
--- /dev/null
+++ b/test/CodeGen/AMDGPU/GlobalISel/legalize-fsqrt.mir
@@ -0,0 +1,307 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=tahiti -run-pass=legalizer %s -o - | FileCheck -check-prefix=SI  %s
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -run-pass=legalizer %s -o - | FileCheck -check-prefix=VI %s
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx900 -run-pass=legalizer %s -o - | FileCheck -check-prefix=GFX9  %s
+
+---
+name: test_fsqrt_s32
+body: |
+  bb.0:
+    liveins: $vgpr0
+
+    ; SI-LABEL: name: test_fsqrt_s32
+    ; SI: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; SI: [[FSQRT:%[0-9]+]]:_(s32) = G_FSQRT [[COPY]]
+    ; SI: $vgpr0 = COPY [[FSQRT]](s32)
+    ; VI-LABEL: name: test_fsqrt_s32
+    ; VI: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; VI: [[FSQRT:%[0-9]+]]:_(s32) = G_FSQRT [[COPY]]
+    ; VI: $vgpr0 = COPY [[FSQRT]](s32)
+    ; GFX9-LABEL: name: test_fsqrt_s32
+    ; GFX9: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; GFX9: [[FSQRT:%[0-9]+]]:_(s32) = G_FSQRT [[COPY]]
+    ; GFX9: $vgpr0 = COPY [[FSQRT]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = G_FSQRT %0
+    $vgpr0 = COPY %1
+...
+
+---
+name: test_fsqrt_s64
+body: |
+  bb.0:
+    liveins: $vgpr0
+
+    ; SI-LABEL: name: test_fsqrt_s64
+    ; SI: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
+    ; SI: [[FSQRT:%[0-9]+]]:_(s64) = G_FSQRT [[COPY]]
+    ; SI: $vgpr0_vgpr1 = COPY [[FSQRT]](s64)
+    ; VI-LABEL: name: test_fsqrt_s64
+    ; VI: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
+    ; VI: [[FSQRT:%[0-9]+]]:_(s64) = G_FSQRT [[COPY]]
+    ; VI: $vgpr0_vgpr1 = COPY [[FSQRT]](s64)
+    ; GFX9-LABEL: name: test_fsqrt_s64
+    ; GFX9: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
+    ; GFX9: [[FSQRT:%[0-9]+]]:_(s64) = G_FSQRT [[COPY]]
+    ; GFX9: $vgpr0_vgpr1 = COPY [[FSQRT]](s64)
+    %0:_(s64) = COPY $vgpr0_vgpr1
+    %1:_(s64) = G_FSQRT %0
+    $vgpr0_vgpr1 = COPY %1
+
+...
+---
+name: test_fsqrt_s16
+body: |
+  bb.0:
+    liveins: $vgpr0
+
+    ; SI-LABEL: name: test_fsqrt_s16
+    ; SI: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; SI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+    ; SI: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC]](s16)
+    ; SI: [[FSQRT:%[0-9]+]]:_(s32) = G_FSQRT [[FPEXT]]
+    ; SI: [[FPTRUNC:%[0-9]+]]:_(s16) = G_FPTRUNC [[FSQRT]](s32)
+    ; SI: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[FPTRUNC]](s16)
+    ; SI: $vgpr0 = COPY [[ANYEXT]](s32)
+    ; VI-LABEL: name: test_fsqrt_s16
+    ; VI: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; VI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+    ; VI: [[FSQRT:%[0-9]+]]:_(s16) = G_FSQRT [[TRUNC]]
+    ; VI: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[FSQRT]](s16)
+    ; VI: $vgpr0 = COPY [[ANYEXT]](s32)
+    ; GFX9-LABEL: name: test_fsqrt_s16
+    ; GFX9: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; GFX9: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+    ; GFX9: [[FSQRT:%[0-9]+]]:_(s16) = G_FSQRT [[TRUNC]]
+    ; GFX9: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[FSQRT]](s16)
+    ; GFX9: $vgpr0 = COPY [[ANYEXT]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s16) = G_TRUNC %0
+    %2:_(s16) = G_FSQRT %1
+    %3:_(s32) = G_ANYEXT %2
+    $vgpr0 = COPY %3
+...
+
+---
+name: test_fsqrt_v2s32
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; SI-LABEL: name: test_fsqrt_v2s32
+    ; SI: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    ; SI: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
+    ; SI: [[FSQRT:%[0-9]+]]:_(s32) = G_FSQRT [[UV]]
+    ; SI: [[FSQRT1:%[0-9]+]]:_(s32) = G_FSQRT [[UV1]]
+    ; SI: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[FSQRT]](s32), [[FSQRT1]](s32)
+    ; SI: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
+    ; VI-LABEL: name: test_fsqrt_v2s32
+    ; VI: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    ; VI: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
+    ; VI: [[FSQRT:%[0-9]+]]:_(s32) = G_FSQRT [[UV]]
+    ; VI: [[FSQRT1:%[0-9]+]]:_(s32) = G_FSQRT [[UV1]]
+    ; VI: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[FSQRT]](s32), [[FSQRT1]](s32)
+    ; VI: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
+    ; GFX9-LABEL: name: test_fsqrt_v2s32
+    ; GFX9: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    ; GFX9: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
+    ; GFX9: [[FSQRT:%[0-9]+]]:_(s32) = G_FSQRT [[UV]]
+    ; GFX9: [[FSQRT1:%[0-9]+]]:_(s32) = G_FSQRT [[UV1]]
+    ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[FSQRT]](s32), [[FSQRT1]](s32)
+    ; GFX9: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
+    %0:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    %1:_(<2 x s32>) = G_FSQRT %0
+    $vgpr0_vgpr1 = COPY %1
+...
+
+---
+name: test_fsqrt_v3s32
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2
+
+    ; SI-LABEL: name: test_fsqrt_v3s32
+    ; SI: [[COPY:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr0_vgpr1_vgpr2
+    ; SI: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<3 x s32>)
+    ; SI: [[FSQRT:%[0-9]+]]:_(s32) = G_FSQRT [[UV]]
+    ; SI: [[FSQRT1:%[0-9]+]]:_(s32) = G_FSQRT [[UV1]]
+    ; SI: [[FSQRT2:%[0-9]+]]:_(s32) = G_FSQRT [[UV2]]
+    ; SI: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[FSQRT]](s32), [[FSQRT1]](s32), [[FSQRT2]](s32)
+    ; SI: $vgpr0_vgpr1_vgpr2 = COPY [[BUILD_VECTOR]](<3 x s32>)
+    ; VI-LABEL: name: test_fsqrt_v3s32
+    ; VI: [[COPY:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr0_vgpr1_vgpr2
+    ; VI: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<3 x s32>)
+    ; VI: [[FSQRT:%[0-9]+]]:_(s32) = G_FSQRT [[UV]]
+    ; VI: [[FSQRT1:%[0-9]+]]:_(s32) = G_FSQRT [[UV1]]
+    ; VI: [[FSQRT2:%[0-9]+]]:_(s32) = G_FSQRT [[UV2]]
+    ; VI: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[FSQRT]](s32), [[FSQRT1]](s32), [[FSQRT2]](s32)
+    ; VI: $vgpr0_vgpr1_vgpr2 = COPY [[BUILD_VECTOR]](<3 x s32>)
+    ; GFX9-LABEL: name: test_fsqrt_v3s32
+    ; GFX9: [[COPY:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr0_vgpr1_vgpr2
+    ; GFX9: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<3 x s32>)
+    ; GFX9: [[FSQRT:%[0-9]+]]:_(s32) = G_FSQRT [[UV]]
+    ; GFX9: [[FSQRT1:%[0-9]+]]:_(s32) = G_FSQRT [[UV1]]
+    ; GFX9: [[FSQRT2:%[0-9]+]]:_(s32) = G_FSQRT [[UV2]]
+    ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[FSQRT]](s32), [[FSQRT1]](s32), [[FSQRT2]](s32)
+    ; GFX9: $vgpr0_vgpr1_vgpr2 = COPY [[BUILD_VECTOR]](<3 x s32>)
+    %0:_(<3 x s32>) = COPY $vgpr0_vgpr1_vgpr2
+    %1:_(<3 x  s32>) = G_FSQRT %0
+    $vgpr0_vgpr1_vgpr2 = COPY %1
+...
+
+---
+name: test_fsqrt_v2s64
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3
+
+    ; SI-LABEL: name: test_fsqrt_v2s64
+    ; SI: [[COPY:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; SI: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](<2 x s64>)
+    ; SI: [[FSQRT:%[0-9]+]]:_(s64) = G_FSQRT [[UV]]
+    ; SI: [[FSQRT1:%[0-9]+]]:_(s64) = G_FSQRT [[UV1]]
+    ; SI: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[FSQRT]](s64), [[FSQRT1]](s64)
+    ; SI: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>)
+    ; VI-LABEL: name: test_fsqrt_v2s64
+    ; VI: [[COPY:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; VI: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](<2 x s64>)
+    ; VI: [[FSQRT:%[0-9]+]]:_(s64) = G_FSQRT [[UV]]
+    ; VI: [[FSQRT1:%[0-9]+]]:_(s64) = G_FSQRT [[UV1]]
+    ; VI: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[FSQRT]](s64), [[FSQRT1]](s64)
+    ; VI: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>)
+    ; GFX9-LABEL: name: test_fsqrt_v2s64
+    ; GFX9: [[COPY:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; GFX9: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](<2 x s64>)
+    ; GFX9: [[FSQRT:%[0-9]+]]:_(s64) = G_FSQRT [[UV]]
+    ; GFX9: [[FSQRT1:%[0-9]+]]:_(s64) = G_FSQRT [[UV1]]
+    ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[FSQRT]](s64), [[FSQRT1]](s64)
+    ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>)
+    %0:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    %1:_(<2 x s64>) = G_FSQRT %0
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %1
+...
+
+---
+name: test_fsqrt_v2s16
+body: |
+  bb.0:
+    liveins: $vgpr0
+
+    ; SI-LABEL: name: test_fsqrt_v2s16
+    ; SI: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
+    ; SI: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY]](<2 x s16>)
+    ; SI: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT [[UV]](s16)
+    ; SI: [[FSQRT:%[0-9]+]]:_(s32) = G_FSQRT [[FPEXT]]
+    ; SI: [[FPTRUNC:%[0-9]+]]:_(s16) = G_FPTRUNC [[FSQRT]](s32)
+    ; SI: [[FPEXT1:%[0-9]+]]:_(s32) = G_FPEXT [[UV1]](s16)
+    ; SI: [[FSQRT1:%[0-9]+]]:_(s32) = G_FSQRT [[FPEXT1]]
+    ; SI: [[FPTRUNC1:%[0-9]+]]:_(s16) = G_FPTRUNC [[FSQRT1]](s32)
+    ; SI: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[FPTRUNC]](s16), [[FPTRUNC1]](s16)
+    ; SI: $vgpr0 = COPY [[BUILD_VECTOR]](<2 x s16>)
+    ; VI-LABEL: name: test_fsqrt_v2s16
+    ; VI: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
+    ; VI: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY]](<2 x s16>)
+    ; VI: [[FSQRT:%[0-9]+]]:_(s16) = G_FSQRT [[UV]]
+    ; VI: [[FSQRT1:%[0-9]+]]:_(s16) = G_FSQRT [[UV1]]
+    ; VI: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[FSQRT]](s16), [[FSQRT1]](s16)
+    ; VI: $vgpr0 = COPY [[BUILD_VECTOR]](<2 x s16>)
+    ; GFX9-LABEL: name: test_fsqrt_v2s16
+    ; GFX9: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
+    ; GFX9: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY]](<2 x s16>)
+    ; GFX9: [[FSQRT:%[0-9]+]]:_(s16) = G_FSQRT [[UV]]
+    ; GFX9: [[FSQRT1:%[0-9]+]]:_(s16) = G_FSQRT [[UV1]]
+    ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[FSQRT]](s16), [[FSQRT1]](s16)
+    ; GFX9: $vgpr0 = COPY [[BUILD_VECTOR]](<2 x s16>)
+    %0:_(<2 x s16>) = COPY $vgpr0
+    %1:_(<2 x s16>) = G_FSQRT %0
+    $vgpr0 = COPY %1
+...
+
+---
+name: test_fsqrt_v3s16
+body: |
+  bb.0:
+
+    ; SI-LABEL: name: test_fsqrt_v3s16
+    ; SI: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; SI: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
+    ; SI: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16), [[UV2:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[EXTRACT]](<3 x s16>)
+    ; SI: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT [[UV]](s16)
+    ; SI: [[FSQRT:%[0-9]+]]:_(s32) = G_FSQRT [[FPEXT]]
+    ; SI: [[FPTRUNC:%[0-9]+]]:_(s16) = G_FPTRUNC [[FSQRT]](s32)
+    ; SI: [[FPEXT1:%[0-9]+]]:_(s32) = G_FPEXT [[UV1]](s16)
+    ; SI: [[FSQRT1:%[0-9]+]]:_(s32) = G_FSQRT [[FPEXT1]]
+    ; SI: [[FPTRUNC1:%[0-9]+]]:_(s16) = G_FPTRUNC [[FSQRT1]](s32)
+    ; SI: [[FPEXT2:%[0-9]+]]:_(s32) = G_FPEXT [[UV2]](s16)
+    ; SI: [[FSQRT2:%[0-9]+]]:_(s32) = G_FSQRT [[FPEXT2]]
+    ; SI: [[FPTRUNC2:%[0-9]+]]:_(s16) = G_FPTRUNC [[FSQRT2]](s32)
+    ; SI: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s16>) = G_BUILD_VECTOR [[FPTRUNC]](s16), [[FPTRUNC1]](s16), [[FPTRUNC2]](s16)
+    ; SI: S_NOP 0, implicit [[BUILD_VECTOR]](<3 x s16>)
+    ; VI-LABEL: name: test_fsqrt_v3s16
+    ; VI: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; VI: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
+    ; VI: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16), [[UV2:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[EXTRACT]](<3 x s16>)
+    ; VI: [[FSQRT:%[0-9]+]]:_(s16) = G_FSQRT [[UV]]
+    ; VI: [[FSQRT1:%[0-9]+]]:_(s16) = G_FSQRT [[UV1]]
+    ; VI: [[FSQRT2:%[0-9]+]]:_(s16) = G_FSQRT [[UV2]]
+    ; VI: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s16>) = G_BUILD_VECTOR [[FSQRT]](s16), [[FSQRT1]](s16), [[FSQRT2]](s16)
+    ; VI: S_NOP 0, implicit [[BUILD_VECTOR]](<3 x s16>)
+    ; GFX9-LABEL: name: test_fsqrt_v3s16
+    ; GFX9: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; GFX9: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
+    ; GFX9: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16), [[UV2:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[EXTRACT]](<3 x s16>)
+    ; GFX9: [[FSQRT:%[0-9]+]]:_(s16) = G_FSQRT [[UV]]
+    ; GFX9: [[FSQRT1:%[0-9]+]]:_(s16) = G_FSQRT [[UV1]]
+    ; GFX9: [[FSQRT2:%[0-9]+]]:_(s16) = G_FSQRT [[UV2]]
+    ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s16>) = G_BUILD_VECTOR [[FSQRT]](s16), [[FSQRT1]](s16), [[FSQRT2]](s16)
+    ; GFX9: S_NOP 0, implicit [[BUILD_VECTOR]](<3 x s16>)
+    %0:_(<3 x s16>) = G_IMPLICIT_DEF
+    %1:_(<3 x s16>) = G_FSQRT %0
+    S_NOP 0, implicit %1
+...
+
+---
+name: test_fsqrt_v4s16
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; SI-LABEL: name: test_fsqrt_v4s16
+    ; SI: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
+    ; SI: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16), [[UV2:%[0-9]+]]:_(s16), [[UV3:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
+    ; SI: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT [[UV]](s16)
+    ; SI: [[FSQRT:%[0-9]+]]:_(s32) = G_FSQRT [[FPEXT]]
+    ; SI: [[FPTRUNC:%[0-9]+]]:_(s16) = G_FPTRUNC [[FSQRT]](s32)
+    ; SI: [[FPEXT1:%[0-9]+]]:_(s32) = G_FPEXT [[UV1]](s16)
+    ; SI: [[FSQRT1:%[0-9]+]]:_(s32) = G_FSQRT [[FPEXT1]]
+    ; SI: [[FPTRUNC1:%[0-9]+]]:_(s16) = G_FPTRUNC [[FSQRT1]](s32)
+    ; SI: [[FPEXT2:%[0-9]+]]:_(s32) = G_FPEXT [[UV2]](s16)
+    ; SI: [[FSQRT2:%[0-9]+]]:_(s32) = G_FSQRT [[FPEXT2]]
+    ; SI: [[FPTRUNC2:%[0-9]+]]:_(s16) = G_FPTRUNC [[FSQRT2]](s32)
+    ; SI: [[FPEXT3:%[0-9]+]]:_(s32) = G_FPEXT [[UV3]](s16)
+    ; SI: [[FSQRT3:%[0-9]+]]:_(s32) = G_FSQRT [[FPEXT3]]
+    ; SI: [[FPTRUNC3:%[0-9]+]]:_(s16) = G_FPTRUNC [[FSQRT3]](s32)
+    ; SI: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s16>) = G_BUILD_VECTOR [[FPTRUNC]](s16), [[FPTRUNC1]](s16), [[FPTRUNC2]](s16), [[FPTRUNC3]](s16)
+    ; SI: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<4 x s16>)
+    ; VI-LABEL: name: test_fsqrt_v4s16
+    ; VI: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
+    ; VI: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16), [[UV2:%[0-9]+]]:_(s16), [[UV3:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
+    ; VI: [[FSQRT:%[0-9]+]]:_(s16) = G_FSQRT [[UV]]
+    ; VI: [[FSQRT1:%[0-9]+]]:_(s16) = G_FSQRT [[UV1]]
+    ; VI: [[FSQRT2:%[0-9]+]]:_(s16) = G_FSQRT [[UV2]]
+    ; VI: [[FSQRT3:%[0-9]+]]:_(s16) = G_FSQRT [[UV3]]
+    ; VI: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s16>) = G_BUILD_VECTOR [[FSQRT]](s16), [[FSQRT1]](s16), [[FSQRT2]](s16), [[FSQRT3]](s16)
+    ; VI: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<4 x s16>)
+    ; GFX9-LABEL: name: test_fsqrt_v4s16
+    ; GFX9: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
+    ; GFX9: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16), [[UV2:%[0-9]+]]:_(s16), [[UV3:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
+    ; GFX9: [[FSQRT:%[0-9]+]]:_(s16) = G_FSQRT [[UV]]
+    ; GFX9: [[FSQRT1:%[0-9]+]]:_(s16) = G_FSQRT [[UV1]]
+    ; GFX9: [[FSQRT2:%[0-9]+]]:_(s16) = G_FSQRT [[UV2]]
+    ; GFX9: [[FSQRT3:%[0-9]+]]:_(s16) = G_FSQRT [[UV3]]
+    ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s16>) = G_BUILD_VECTOR [[FSQRT]](s16), [[FSQRT1]](s16), [[FSQRT2]](s16), [[FSQRT3]](s16)
+    ; GFX9: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<4 x s16>)
+    %0:_(<4 x s16>) = COPY $vgpr0_vgpr1
+    %1:_(<4 x s16>) = G_FSQRT %0
+    $vgpr0_vgpr1 = COPY %1
+...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/legalize-fsub.mir b/test/CodeGen/AMDGPU/GlobalISel/legalize-fsub.mir
index 13012f459ed3bfb245fab07f8d2cf1cae3548e8a..bdba3d6926dc800f7a235a209dc41413faf02d26 100644
--- a/test/CodeGen/AMDGPU/GlobalISel/legalize-fsub.mir
+++ b/test/CodeGen/AMDGPU/GlobalISel/legalize-fsub.mir
@@ -1,36 +1,472 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -run-pass=legalizer %s -o - | FileCheck %s
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=tahiti -run-pass=legalizer %s -o - | FileCheck -check-prefix=SI  %s
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -run-pass=legalizer %s -o - | FileCheck -check-prefix=VI %s
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx900 -run-pass=legalizer %s -o - | FileCheck -check-prefix=GFX9  %s
 
 ---
-name: test_fsub_f32
+name: test_fsub_s32
 body: |
   bb.0:
     liveins: $vgpr0, $vgpr1
 
-    ; CHECK-LABEL: name: test_fsub_f32
-    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
-    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
-    ; CHECK: [[FSUB:%[0-9]+]]:_(s32) = G_FSUB [[COPY]], [[COPY1]]
-    ; CHECK: $vgpr0 = COPY [[FSUB]](s32)
+    ; SI-LABEL: name: test_fsub_s32
+    ; SI: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; SI: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; SI: [[FSUB:%[0-9]+]]:_(s32) = G_FSUB [[COPY]], [[COPY1]]
+    ; SI: $vgpr0 = COPY [[FSUB]](s32)
+    ; VI-LABEL: name: test_fsub_s32
+    ; VI: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; VI: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; VI: [[FSUB:%[0-9]+]]:_(s32) = G_FSUB [[COPY]], [[COPY1]]
+    ; VI: $vgpr0 = COPY [[FSUB]](s32)
+    ; GFX9-LABEL: name: test_fsub_s32
+    ; GFX9: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; GFX9: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; GFX9: [[FSUB:%[0-9]+]]:_(s32) = G_FSUB [[COPY]], [[COPY1]]
+    ; GFX9: $vgpr0 = COPY [[FSUB]](s32)
     %0:_(s32) = COPY $vgpr0
     %1:_(s32) = COPY $vgpr1
     %2:_(s32) = G_FSUB %0, %1
     $vgpr0 = COPY %2
 ...
 ---
-name: test_fsub_f64
+name: test_fsub_s64
 body: |
   bb.0:
     liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
 
-    ; CHECK-LABEL: name: test_fsub_f64
-    ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
-    ; CHECK: [[COPY1:%[0-9]+]]:_(s64) = COPY $vgpr2_vgpr3
-    ; CHECK: [[FNEG:%[0-9]+]]:_(s64) = G_FNEG [[COPY1]]
-    ; CHECK: [[FADD:%[0-9]+]]:_(s64) = G_FADD [[COPY]], [[FNEG]]
-    ; CHECK: $vgpr0_vgpr1 = COPY [[FADD]](s64)
+    ; SI-LABEL: name: test_fsub_s64
+    ; SI: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
+    ; SI: [[COPY1:%[0-9]+]]:_(s64) = COPY $vgpr2_vgpr3
+    ; SI: [[FNEG:%[0-9]+]]:_(s64) = G_FNEG [[COPY1]]
+    ; SI: [[FADD:%[0-9]+]]:_(s64) = G_FADD [[COPY]], [[FNEG]]
+    ; SI: $vgpr0_vgpr1 = COPY [[FADD]](s64)
+    ; VI-LABEL: name: test_fsub_s64
+    ; VI: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
+    ; VI: [[COPY1:%[0-9]+]]:_(s64) = COPY $vgpr2_vgpr3
+    ; VI: [[FNEG:%[0-9]+]]:_(s64) = G_FNEG [[COPY1]]
+    ; VI: [[FADD:%[0-9]+]]:_(s64) = G_FADD [[COPY]], [[FNEG]]
+    ; VI: $vgpr0_vgpr1 = COPY [[FADD]](s64)
+    ; GFX9-LABEL: name: test_fsub_s64
+    ; GFX9: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
+    ; GFX9: [[COPY1:%[0-9]+]]:_(s64) = COPY $vgpr2_vgpr3
+    ; GFX9: [[FNEG:%[0-9]+]]:_(s64) = G_FNEG [[COPY1]]
+    ; GFX9: [[FADD:%[0-9]+]]:_(s64) = G_FADD [[COPY]], [[FNEG]]
+    ; GFX9: $vgpr0_vgpr1 = COPY [[FADD]](s64)
     %0:_(s64) = COPY $vgpr0_vgpr1
     %1:_(s64) = COPY $vgpr2_vgpr3
     %2:_(s64) = G_FSUB %0, %1
     $vgpr0_vgpr1 = COPY %2
 ...
+
+---
+name: test_fsub_s16
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+
+    ; SI-LABEL: name: test_fsub_s16
+    ; SI: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; SI: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; SI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+    ; SI: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32)
+    ; SI: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC1]](s16)
+    ; SI: [[FNEG:%[0-9]+]]:_(s32) = G_FNEG [[FPEXT]]
+    ; SI: [[FPTRUNC:%[0-9]+]]:_(s16) = G_FPTRUNC [[FNEG]](s32)
+    ; SI: [[FPEXT1:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC]](s16)
+    ; SI: [[FPEXT2:%[0-9]+]]:_(s32) = G_FPEXT [[FPTRUNC]](s16)
+    ; SI: [[FADD:%[0-9]+]]:_(s32) = G_FADD [[FPEXT1]], [[FPEXT2]]
+    ; SI: [[FPTRUNC1:%[0-9]+]]:_(s16) = G_FPTRUNC [[FADD]](s32)
+    ; SI: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[FPTRUNC1]](s16)
+    ; SI: $vgpr0 = COPY [[ANYEXT]](s32)
+    ; VI-LABEL: name: test_fsub_s16
+    ; VI: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; VI: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; VI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+    ; VI: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32)
+    ; VI: [[FNEG:%[0-9]+]]:_(s16) = G_FNEG [[TRUNC1]]
+    ; VI: [[FADD:%[0-9]+]]:_(s16) = G_FADD [[TRUNC]], [[FNEG]]
+    ; VI: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[FADD]](s16)
+    ; VI: $vgpr0 = COPY [[ANYEXT]](s32)
+    ; GFX9-LABEL: name: test_fsub_s16
+    ; GFX9: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; GFX9: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; GFX9: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+    ; GFX9: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32)
+    ; GFX9: [[FNEG:%[0-9]+]]:_(s16) = G_FNEG [[TRUNC1]]
+    ; GFX9: [[FADD:%[0-9]+]]:_(s16) = G_FADD [[TRUNC]], [[FNEG]]
+    ; GFX9: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[FADD]](s16)
+    ; GFX9: $vgpr0 = COPY [[ANYEXT]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s16) = G_TRUNC %0
+    %3:_(s16) = G_TRUNC %1
+
+    %4:_(s16) = G_FSUB %2, %3
+    %5:_(s32) = G_ANYEXT %4
+    $vgpr0 = COPY %5
+...
+
+---
+name: test_fsub_v2s32
+body: |
+  bb.0.entry:
+    liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
+
+    ; SI-LABEL: name: test_fsub_v2s32
+    ; SI: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    ; SI: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr2_vgpr3
+    ; SI: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
+    ; SI: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<2 x s32>)
+    ; SI: [[FSUB:%[0-9]+]]:_(s32) = G_FSUB [[UV]], [[UV2]]
+    ; SI: [[FSUB1:%[0-9]+]]:_(s32) = G_FSUB [[UV1]], [[UV3]]
+    ; SI: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[FSUB]](s32), [[FSUB1]](s32)
+    ; SI: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
+    ; VI-LABEL: name: test_fsub_v2s32
+    ; VI: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    ; VI: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr2_vgpr3
+    ; VI: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
+    ; VI: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<2 x s32>)
+    ; VI: [[FSUB:%[0-9]+]]:_(s32) = G_FSUB [[UV]], [[UV2]]
+    ; VI: [[FSUB1:%[0-9]+]]:_(s32) = G_FSUB [[UV1]], [[UV3]]
+    ; VI: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[FSUB]](s32), [[FSUB1]](s32)
+    ; VI: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
+    ; GFX9-LABEL: name: test_fsub_v2s32
+    ; GFX9: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    ; GFX9: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr2_vgpr3
+    ; GFX9: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
+    ; GFX9: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<2 x s32>)
+    ; GFX9: [[FSUB:%[0-9]+]]:_(s32) = G_FSUB [[UV]], [[UV2]]
+    ; GFX9: [[FSUB1:%[0-9]+]]:_(s32) = G_FSUB [[UV1]], [[UV3]]
+    ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[FSUB]](s32), [[FSUB1]](s32)
+    ; GFX9: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
+    %0:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    %1:_(<2 x s32>) = COPY $vgpr2_vgpr3
+    %2:_(<2 x s32>) = G_FSUB %0, %1
+    $vgpr0_vgpr1 = COPY %2
+...
+
+---
+name: test_fsub_v2s32_flags
+body: |
+  bb.0.entry:
+    liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
+
+    ; SI-LABEL: name: test_fsub_v2s32_flags
+    ; SI: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    ; SI: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr2_vgpr3
+    ; SI: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
+    ; SI: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<2 x s32>)
+    ; SI: %7:_(s32) = nnan G_FSUB [[UV]], [[UV2]]
+    ; SI: %8:_(s32) = nnan G_FSUB [[UV1]], [[UV3]]
+    ; SI: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR %7(s32), %8(s32)
+    ; SI: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
+    ; VI-LABEL: name: test_fsub_v2s32_flags
+    ; VI: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    ; VI: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr2_vgpr3
+    ; VI: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
+    ; VI: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<2 x s32>)
+    ; VI: %7:_(s32) = nnan G_FSUB [[UV]], [[UV2]]
+    ; VI: %8:_(s32) = nnan G_FSUB [[UV1]], [[UV3]]
+    ; VI: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR %7(s32), %8(s32)
+    ; VI: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
+    ; GFX9-LABEL: name: test_fsub_v2s32_flags
+    ; GFX9: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    ; GFX9: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr2_vgpr3
+    ; GFX9: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
+    ; GFX9: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<2 x s32>)
+    ; GFX9: %7:_(s32) = nnan G_FSUB [[UV]], [[UV2]]
+    ; GFX9: %8:_(s32) = nnan G_FSUB [[UV1]], [[UV3]]
+    ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR %7(s32), %8(s32)
+    ; GFX9: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
+    %0:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    %1:_(<2 x s32>) = COPY $vgpr2_vgpr3
+    %2:_(<2 x s32>) = nnan G_FSUB %0, %1
+    $vgpr0_vgpr1 = COPY %2
+...
+
+---
+name: test_fsub_v3s32
+body: |
+  bb.0.entry:
+    liveins: $vgpr0_vgpr1_vgpr2, $vgpr3_vgpr4_vgpr5
+
+    ; SI-LABEL: name: test_fsub_v3s32
+    ; SI: [[COPY:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr0_vgpr1_vgpr2
+    ; SI: [[COPY1:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr3_vgpr4_vgpr5
+    ; SI: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<3 x s32>)
+    ; SI: [[UV3:%[0-9]+]]:_(s32), [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<3 x s32>)
+    ; SI: [[FSUB:%[0-9]+]]:_(s32) = G_FSUB [[UV]], [[UV3]]
+    ; SI: [[FSUB1:%[0-9]+]]:_(s32) = G_FSUB [[UV1]], [[UV4]]
+    ; SI: [[FSUB2:%[0-9]+]]:_(s32) = G_FSUB [[UV2]], [[UV5]]
+    ; SI: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[FSUB]](s32), [[FSUB1]](s32), [[FSUB2]](s32)
+    ; SI: $vgpr0_vgpr1_vgpr2 = COPY [[BUILD_VECTOR]](<3 x s32>)
+    ; VI-LABEL: name: test_fsub_v3s32
+    ; VI: [[COPY:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr0_vgpr1_vgpr2
+    ; VI: [[COPY1:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr3_vgpr4_vgpr5
+    ; VI: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<3 x s32>)
+    ; VI: [[UV3:%[0-9]+]]:_(s32), [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<3 x s32>)
+    ; VI: [[FSUB:%[0-9]+]]:_(s32) = G_FSUB [[UV]], [[UV3]]
+    ; VI: [[FSUB1:%[0-9]+]]:_(s32) = G_FSUB [[UV1]], [[UV4]]
+    ; VI: [[FSUB2:%[0-9]+]]:_(s32) = G_FSUB [[UV2]], [[UV5]]
+    ; VI: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[FSUB]](s32), [[FSUB1]](s32), [[FSUB2]](s32)
+    ; VI: $vgpr0_vgpr1_vgpr2 = COPY [[BUILD_VECTOR]](<3 x s32>)
+    ; GFX9-LABEL: name: test_fsub_v3s32
+    ; GFX9: [[COPY:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr0_vgpr1_vgpr2
+    ; GFX9: [[COPY1:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr3_vgpr4_vgpr5
+    ; GFX9: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<3 x s32>)
+    ; GFX9: [[UV3:%[0-9]+]]:_(s32), [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<3 x s32>)
+    ; GFX9: [[FSUB:%[0-9]+]]:_(s32) = G_FSUB [[UV]], [[UV3]]
+    ; GFX9: [[FSUB1:%[0-9]+]]:_(s32) = G_FSUB [[UV1]], [[UV4]]
+    ; GFX9: [[FSUB2:%[0-9]+]]:_(s32) = G_FSUB [[UV2]], [[UV5]]
+    ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[FSUB]](s32), [[FSUB1]](s32), [[FSUB2]](s32)
+    ; GFX9: $vgpr0_vgpr1_vgpr2 = COPY [[BUILD_VECTOR]](<3 x s32>)
+    %0:_(<3 x s32>) = COPY $vgpr0_vgpr1_vgpr2
+    %1:_(<3 x s32>) = COPY $vgpr3_vgpr4_vgpr5
+    %2:_(<3 x s32>) = G_FSUB %0, %1
+    $vgpr0_vgpr1_vgpr2 = COPY %2
+...
+
+---
+name: test_fsub_v2s64
+body: |
+  bb.0.entry:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7
+
+    ; SI-LABEL: name: test_fsub_v2s64
+    ; SI: [[COPY:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; SI: [[COPY1:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr4_vgpr5_vgpr6_vgpr7
+    ; SI: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](<2 x s64>)
+    ; SI: [[UV2:%[0-9]+]]:_(s64), [[UV3:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY1]](<2 x s64>)
+    ; SI: [[FNEG:%[0-9]+]]:_(s64) = G_FNEG [[UV2]]
+    ; SI: [[FADD:%[0-9]+]]:_(s64) = G_FADD [[UV]], [[FNEG]]
+    ; SI: [[FNEG1:%[0-9]+]]:_(s64) = G_FNEG [[UV3]]
+    ; SI: [[FADD1:%[0-9]+]]:_(s64) = G_FADD [[UV1]], [[FNEG1]]
+    ; SI: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[FADD]](s64), [[FADD1]](s64)
+    ; SI: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>)
+    ; VI-LABEL: name: test_fsub_v2s64
+    ; VI: [[COPY:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; VI: [[COPY1:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr4_vgpr5_vgpr6_vgpr7
+    ; VI: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](<2 x s64>)
+    ; VI: [[UV2:%[0-9]+]]:_(s64), [[UV3:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY1]](<2 x s64>)
+    ; VI: [[FNEG:%[0-9]+]]:_(s64) = G_FNEG [[UV2]]
+    ; VI: [[FADD:%[0-9]+]]:_(s64) = G_FADD [[UV]], [[FNEG]]
+    ; VI: [[FNEG1:%[0-9]+]]:_(s64) = G_FNEG [[UV3]]
+    ; VI: [[FADD1:%[0-9]+]]:_(s64) = G_FADD [[UV1]], [[FNEG1]]
+    ; VI: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[FADD]](s64), [[FADD1]](s64)
+    ; VI: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>)
+    ; GFX9-LABEL: name: test_fsub_v2s64
+    ; GFX9: [[COPY:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; GFX9: [[COPY1:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr4_vgpr5_vgpr6_vgpr7
+    ; GFX9: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](<2 x s64>)
+    ; GFX9: [[UV2:%[0-9]+]]:_(s64), [[UV3:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY1]](<2 x s64>)
+    ; GFX9: [[FNEG:%[0-9]+]]:_(s64) = G_FNEG [[UV2]]
+    ; GFX9: [[FADD:%[0-9]+]]:_(s64) = G_FADD [[UV]], [[FNEG]]
+    ; GFX9: [[FNEG1:%[0-9]+]]:_(s64) = G_FNEG [[UV3]]
+    ; GFX9: [[FADD1:%[0-9]+]]:_(s64) = G_FADD [[UV1]], [[FNEG1]]
+    ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[FADD]](s64), [[FADD1]](s64)
+    ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>)
+    %0:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    %1:_(<2 x s64>) = COPY $vgpr4_vgpr5_vgpr6_vgpr7
+    %2:_(<2 x s64>) = G_FSUB %0, %1
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %2
+...
+
+---
+name: test_fsub_v2s16
+body: |
+  bb.0.entry:
+    liveins: $vgpr0, $vgpr1
+
+    ; SI-LABEL: name: test_fsub_v2s16
+    ; SI: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
+    ; SI: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
+    ; SI: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY1]](<2 x s16>)
+    ; SI: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT [[UV]](s16)
+    ; SI: [[FNEG:%[0-9]+]]:_(s32) = G_FNEG [[FPEXT]]
+    ; SI: [[FPTRUNC:%[0-9]+]]:_(s16) = G_FPTRUNC [[FNEG]](s32)
+    ; SI: [[FPEXT1:%[0-9]+]]:_(s32) = G_FPEXT [[UV1]](s16)
+    ; SI: [[FNEG1:%[0-9]+]]:_(s32) = G_FNEG [[FPEXT1]]
+    ; SI: [[FPTRUNC1:%[0-9]+]]:_(s16) = G_FPTRUNC [[FNEG1]](s32)
+    ; SI: [[UV2:%[0-9]+]]:_(s16), [[UV3:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY]](<2 x s16>)
+    ; SI: [[FPEXT2:%[0-9]+]]:_(s32) = G_FPEXT [[UV2]](s16)
+    ; SI: [[FPEXT3:%[0-9]+]]:_(s32) = G_FPEXT [[FPTRUNC]](s16)
+    ; SI: [[FADD:%[0-9]+]]:_(s32) = G_FADD [[FPEXT2]], [[FPEXT3]]
+    ; SI: [[FPTRUNC2:%[0-9]+]]:_(s16) = G_FPTRUNC [[FADD]](s32)
+    ; SI: [[FPEXT4:%[0-9]+]]:_(s32) = G_FPEXT [[UV3]](s16)
+    ; SI: [[FPEXT5:%[0-9]+]]:_(s32) = G_FPEXT [[FPTRUNC1]](s16)
+    ; SI: [[FADD1:%[0-9]+]]:_(s32) = G_FADD [[FPEXT4]], [[FPEXT5]]
+    ; SI: [[FPTRUNC3:%[0-9]+]]:_(s16) = G_FPTRUNC [[FADD1]](s32)
+    ; SI: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[FPTRUNC2]](s16), [[FPTRUNC3]](s16)
+    ; SI: $vgpr0 = COPY [[BUILD_VECTOR]](<2 x s16>)
+    ; VI-LABEL: name: test_fsub_v2s16
+    ; VI: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
+    ; VI: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
+    ; VI: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY1]](<2 x s16>)
+    ; VI: [[FNEG:%[0-9]+]]:_(s16) = G_FNEG [[UV]]
+    ; VI: [[FNEG1:%[0-9]+]]:_(s16) = G_FNEG [[UV1]]
+    ; VI: [[UV2:%[0-9]+]]:_(s16), [[UV3:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY]](<2 x s16>)
+    ; VI: [[FADD:%[0-9]+]]:_(s16) = G_FADD [[UV2]], [[FNEG]]
+    ; VI: [[FADD1:%[0-9]+]]:_(s16) = G_FADD [[UV3]], [[FNEG1]]
+    ; VI: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[FADD]](s16), [[FADD1]](s16)
+    ; VI: $vgpr0 = COPY [[BUILD_VECTOR]](<2 x s16>)
+    ; GFX9-LABEL: name: test_fsub_v2s16
+    ; GFX9: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
+    ; GFX9: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
+    ; GFX9: [[FNEG:%[0-9]+]]:_(<2 x s16>) = G_FNEG [[COPY1]]
+    ; GFX9: [[FADD:%[0-9]+]]:_(<2 x s16>) = G_FADD [[COPY]], [[FNEG]]
+    ; GFX9: $vgpr0 = COPY [[FADD]](<2 x s16>)
+    %0:_(<2 x s16>) = COPY $vgpr0
+    %1:_(<2 x s16>) = COPY $vgpr1
+    %2:_(<2 x s16>) = G_FSUB %0, %1
+    $vgpr0 = COPY %2
+...
+
+---
+name: test_fsub_v3s16
+body: |
+  bb.0.entry:
+    liveins: $vgpr0, $vgpr1
+
+    ; SI-LABEL: name: test_fsub_v3s16
+    ; SI: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; SI: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
+    ; SI: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; SI: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF1]](<4 x s16>), 0
+    ; SI: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16), [[UV2:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[EXTRACT]](<3 x s16>)
+    ; SI: [[UV3:%[0-9]+]]:_(s16), [[UV4:%[0-9]+]]:_(s16), [[UV5:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[EXTRACT1]](<3 x s16>)
+    ; SI: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT [[UV3]](s16)
+    ; SI: [[FNEG:%[0-9]+]]:_(s32) = G_FNEG [[FPEXT]]
+    ; SI: [[FPTRUNC:%[0-9]+]]:_(s16) = G_FPTRUNC [[FNEG]](s32)
+    ; SI: [[FPEXT1:%[0-9]+]]:_(s32) = G_FPEXT [[UV]](s16)
+    ; SI: [[FPEXT2:%[0-9]+]]:_(s32) = G_FPEXT [[FPTRUNC]](s16)
+    ; SI: [[FADD:%[0-9]+]]:_(s32) = G_FADD [[FPEXT1]], [[FPEXT2]]
+    ; SI: [[FPTRUNC1:%[0-9]+]]:_(s16) = G_FPTRUNC [[FADD]](s32)
+    ; SI: [[FPEXT3:%[0-9]+]]:_(s32) = G_FPEXT [[UV4]](s16)
+    ; SI: [[FNEG1:%[0-9]+]]:_(s32) = G_FNEG [[FPEXT3]]
+    ; SI: [[FPTRUNC2:%[0-9]+]]:_(s16) = G_FPTRUNC [[FNEG1]](s32)
+    ; SI: [[FPEXT4:%[0-9]+]]:_(s32) = G_FPEXT [[UV1]](s16)
+    ; SI: [[FPEXT5:%[0-9]+]]:_(s32) = G_FPEXT [[FPTRUNC2]](s16)
+    ; SI: [[FADD1:%[0-9]+]]:_(s32) = G_FADD [[FPEXT4]], [[FPEXT5]]
+    ; SI: [[FPTRUNC3:%[0-9]+]]:_(s16) = G_FPTRUNC [[FADD1]](s32)
+    ; SI: [[FPEXT6:%[0-9]+]]:_(s32) = G_FPEXT [[UV5]](s16)
+    ; SI: [[FNEG2:%[0-9]+]]:_(s32) = G_FNEG [[FPEXT6]]
+    ; SI: [[FPTRUNC4:%[0-9]+]]:_(s16) = G_FPTRUNC [[FNEG2]](s32)
+    ; SI: [[FPEXT7:%[0-9]+]]:_(s32) = G_FPEXT [[UV2]](s16)
+    ; SI: [[FPEXT8:%[0-9]+]]:_(s32) = G_FPEXT [[FPTRUNC4]](s16)
+    ; SI: [[FADD2:%[0-9]+]]:_(s32) = G_FADD [[FPEXT7]], [[FPEXT8]]
+    ; SI: [[FPTRUNC5:%[0-9]+]]:_(s16) = G_FPTRUNC [[FADD2]](s32)
+    ; SI: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s16>) = G_BUILD_VECTOR [[FPTRUNC1]](s16), [[FPTRUNC3]](s16), [[FPTRUNC5]](s16)
+    ; SI: S_NOP 0, implicit [[BUILD_VECTOR]](<3 x s16>)
+    ; VI-LABEL: name: test_fsub_v3s16
+    ; VI: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; VI: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
+    ; VI: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; VI: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF1]](<4 x s16>), 0
+    ; VI: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16), [[UV2:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[EXTRACT]](<3 x s16>)
+    ; VI: [[UV3:%[0-9]+]]:_(s16), [[UV4:%[0-9]+]]:_(s16), [[UV5:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[EXTRACT1]](<3 x s16>)
+    ; VI: [[FNEG:%[0-9]+]]:_(s16) = G_FNEG [[UV3]]
+    ; VI: [[FADD:%[0-9]+]]:_(s16) = G_FADD [[UV]], [[FNEG]]
+    ; VI: [[FNEG1:%[0-9]+]]:_(s16) = G_FNEG [[UV4]]
+    ; VI: [[FADD1:%[0-9]+]]:_(s16) = G_FADD [[UV1]], [[FNEG1]]
+    ; VI: [[FNEG2:%[0-9]+]]:_(s16) = G_FNEG [[UV5]]
+    ; VI: [[FADD2:%[0-9]+]]:_(s16) = G_FADD [[UV2]], [[FNEG2]]
+    ; VI: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s16>) = G_BUILD_VECTOR [[FADD]](s16), [[FADD1]](s16), [[FADD2]](s16)
+    ; VI: S_NOP 0, implicit [[BUILD_VECTOR]](<3 x s16>)
+    ; GFX9-LABEL: name: test_fsub_v3s16
+    ; GFX9: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; GFX9: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
+    ; GFX9: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; GFX9: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF1]](<4 x s16>), 0
+    ; GFX9: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16), [[UV2:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[EXTRACT]](<3 x s16>)
+    ; GFX9: [[UV3:%[0-9]+]]:_(s16), [[UV4:%[0-9]+]]:_(s16), [[UV5:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[EXTRACT1]](<3 x s16>)
+    ; GFX9: [[FNEG:%[0-9]+]]:_(s16) = G_FNEG [[UV3]]
+    ; GFX9: [[FADD:%[0-9]+]]:_(s16) = G_FADD [[UV]], [[FNEG]]
+    ; GFX9: [[FNEG1:%[0-9]+]]:_(s16) = G_FNEG [[UV4]]
+    ; GFX9: [[FADD1:%[0-9]+]]:_(s16) = G_FADD [[UV1]], [[FNEG1]]
+    ; GFX9: [[FNEG2:%[0-9]+]]:_(s16) = G_FNEG [[UV5]]
+    ; GFX9: [[FADD2:%[0-9]+]]:_(s16) = G_FADD [[UV2]], [[FNEG2]]
+    ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s16>) = G_BUILD_VECTOR [[FADD]](s16), [[FADD1]](s16), [[FADD2]](s16)
+    ; GFX9: S_NOP 0, implicit [[BUILD_VECTOR]](<3 x s16>)
+    %0:_(<3 x s16>) = G_IMPLICIT_DEF
+    %1:_(<3 x s16>) = G_IMPLICIT_DEF
+    %2:_(<3 x s16>) = G_FSUB %0, %1
+    S_NOP 0, implicit %2
+...
+
+---
+name: test_fsub_v4s16
+body: |
+  bb.0.entry:
+    liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
+
+    ; SI-LABEL: name: test_fsub_v4s16
+    ; SI: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
+    ; SI: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
+    ; SI: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16), [[UV2:%[0-9]+]]:_(s16), [[UV3:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
+    ; SI: [[UV4:%[0-9]+]]:_(s16), [[UV5:%[0-9]+]]:_(s16), [[UV6:%[0-9]+]]:_(s16), [[UV7:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
+    ; SI: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT [[UV4]](s16)
+    ; SI: [[FNEG:%[0-9]+]]:_(s32) = G_FNEG [[FPEXT]]
+    ; SI: [[FPTRUNC:%[0-9]+]]:_(s16) = G_FPTRUNC [[FNEG]](s32)
+    ; SI: [[FPEXT1:%[0-9]+]]:_(s32) = G_FPEXT [[UV]](s16)
+    ; SI: [[FPEXT2:%[0-9]+]]:_(s32) = G_FPEXT [[FPTRUNC]](s16)
+    ; SI: [[FADD:%[0-9]+]]:_(s32) = G_FADD [[FPEXT1]], [[FPEXT2]]
+    ; SI: [[FPTRUNC1:%[0-9]+]]:_(s16) = G_FPTRUNC [[FADD]](s32)
+    ; SI: [[FPEXT3:%[0-9]+]]:_(s32) = G_FPEXT [[UV5]](s16)
+    ; SI: [[FNEG1:%[0-9]+]]:_(s32) = G_FNEG [[FPEXT3]]
+    ; SI: [[FPTRUNC2:%[0-9]+]]:_(s16) = G_FPTRUNC [[FNEG1]](s32)
+    ; SI: [[FPEXT4:%[0-9]+]]:_(s32) = G_FPEXT [[UV1]](s16)
+    ; SI: [[FPEXT5:%[0-9]+]]:_(s32) = G_FPEXT [[FPTRUNC2]](s16)
+    ; SI: [[FADD1:%[0-9]+]]:_(s32) = G_FADD [[FPEXT4]], [[FPEXT5]]
+    ; SI: [[FPTRUNC3:%[0-9]+]]:_(s16) = G_FPTRUNC [[FADD1]](s32)
+    ; SI: [[FPEXT6:%[0-9]+]]:_(s32) = G_FPEXT [[UV6]](s16)
+    ; SI: [[FNEG2:%[0-9]+]]:_(s32) = G_FNEG [[FPEXT6]]
+    ; SI: [[FPTRUNC4:%[0-9]+]]:_(s16) = G_FPTRUNC [[FNEG2]](s32)
+    ; SI: [[FPEXT7:%[0-9]+]]:_(s32) = G_FPEXT [[UV2]](s16)
+    ; SI: [[FPEXT8:%[0-9]+]]:_(s32) = G_FPEXT [[FPTRUNC4]](s16)
+    ; SI: [[FADD2:%[0-9]+]]:_(s32) = G_FADD [[FPEXT7]], [[FPEXT8]]
+    ; SI: [[FPTRUNC5:%[0-9]+]]:_(s16) = G_FPTRUNC [[FADD2]](s32)
+    ; SI: [[FPEXT9:%[0-9]+]]:_(s32) = G_FPEXT [[UV7]](s16)
+    ; SI: [[FNEG3:%[0-9]+]]:_(s32) = G_FNEG [[FPEXT9]]
+    ; SI: [[FPTRUNC6:%[0-9]+]]:_(s16) = G_FPTRUNC [[FNEG3]](s32)
+    ; SI: [[FPEXT10:%[0-9]+]]:_(s32) = G_FPEXT [[UV3]](s16)
+    ; SI: [[FPEXT11:%[0-9]+]]:_(s32) = G_FPEXT [[FPTRUNC6]](s16)
+    ; SI: [[FADD3:%[0-9]+]]:_(s32) = G_FADD [[FPEXT10]], [[FPEXT11]]
+    ; SI: [[FPTRUNC7:%[0-9]+]]:_(s16) = G_FPTRUNC [[FADD3]](s32)
+    ; SI: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s16>) = G_BUILD_VECTOR [[FPTRUNC1]](s16), [[FPTRUNC3]](s16), [[FPTRUNC5]](s16), [[FPTRUNC7]](s16)
+    ; SI: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<4 x s16>)
+    ; VI-LABEL: name: test_fsub_v4s16
+    ; VI: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
+    ; VI: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
+    ; VI: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16), [[UV2:%[0-9]+]]:_(s16), [[UV3:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
+    ; VI: [[UV4:%[0-9]+]]:_(s16), [[UV5:%[0-9]+]]:_(s16), [[UV6:%[0-9]+]]:_(s16), [[UV7:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
+    ; VI: [[FNEG:%[0-9]+]]:_(s16) = G_FNEG [[UV4]]
+    ; VI: [[FADD:%[0-9]+]]:_(s16) = G_FADD [[UV]], [[FNEG]]
+    ; VI: [[FNEG1:%[0-9]+]]:_(s16) = G_FNEG [[UV5]]
+    ; VI: [[FADD1:%[0-9]+]]:_(s16) = G_FADD [[UV1]], [[FNEG1]]
+    ; VI: [[FNEG2:%[0-9]+]]:_(s16) = G_FNEG [[UV6]]
+    ; VI: [[FADD2:%[0-9]+]]:_(s16) = G_FADD [[UV2]], [[FNEG2]]
+    ; VI: [[FNEG3:%[0-9]+]]:_(s16) = G_FNEG [[UV7]]
+    ; VI: [[FADD3:%[0-9]+]]:_(s16) = G_FADD [[UV3]], [[FNEG3]]
+    ; VI: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s16>) = G_BUILD_VECTOR [[FADD]](s16), [[FADD1]](s16), [[FADD2]](s16), [[FADD3]](s16)
+    ; VI: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<4 x s16>)
+    ; GFX9-LABEL: name: test_fsub_v4s16
+    ; GFX9: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
+    ; GFX9: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
+    ; GFX9: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16), [[UV2:%[0-9]+]]:_(s16), [[UV3:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
+    ; GFX9: [[UV4:%[0-9]+]]:_(s16), [[UV5:%[0-9]+]]:_(s16), [[UV6:%[0-9]+]]:_(s16), [[UV7:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
+    ; GFX9: [[FNEG:%[0-9]+]]:_(s16) = G_FNEG [[UV4]]
+    ; GFX9: [[FADD:%[0-9]+]]:_(s16) = G_FADD [[UV]], [[FNEG]]
+    ; GFX9: [[FNEG1:%[0-9]+]]:_(s16) = G_FNEG [[UV5]]
+    ; GFX9: [[FADD1:%[0-9]+]]:_(s16) = G_FADD [[UV1]], [[FNEG1]]
+    ; GFX9: [[FNEG2:%[0-9]+]]:_(s16) = G_FNEG [[UV6]]
+    ; GFX9: [[FADD2:%[0-9]+]]:_(s16) = G_FADD [[UV2]], [[FNEG2]]
+    ; GFX9: [[FNEG3:%[0-9]+]]:_(s16) = G_FNEG [[UV7]]
+    ; GFX9: [[FADD3:%[0-9]+]]:_(s16) = G_FADD [[UV3]], [[FNEG3]]
+    ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s16>) = G_BUILD_VECTOR [[FADD]](s16), [[FADD1]](s16), [[FADD2]](s16), [[FADD3]](s16)
+    ; GFX9: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<4 x s16>)
+    %0:_(<4 x s16>) = COPY $vgpr0_vgpr1
+    %1:_(<4 x s16>) = COPY $vgpr2_vgpr3
+    %2:_(<4 x s16>) = G_FSUB %0, %1
+    $vgpr0_vgpr1 = COPY %2
+...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/legalize-icmp.mir b/test/CodeGen/AMDGPU/GlobalISel/legalize-icmp.mir
index 00c0c96335074a36be17ebf207cfba35f1303e49..ca6bbea9095a870e83b8a49fc08c85eafc21c4e6 100644
--- a/test/CodeGen/AMDGPU/GlobalISel/legalize-icmp.mir
+++ b/test/CodeGen/AMDGPU/GlobalISel/legalize-icmp.mir
@@ -2,11 +2,11 @@
 # RUN: llc -O0 -march=amdgcn -mcpu=fiji  -run-pass=legalizer %s -o - | FileCheck %s
 
 ---
-name:            test_icmp
+name: test_icmp_s32
 body: |
   bb.0.entry:
     liveins: $vgpr0
-    ; CHECK-LABEL: name: test_icmp
+    ; CHECK-LABEL: name: test_icmp_s32
     ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
     ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
     ; CHECK: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[C]](s32), [[COPY]]
@@ -15,6 +15,327 @@ body: |
     %0:_(s32) = G_CONSTANT i32 0
     %1:_(s32) = COPY $vgpr0
     %2:_(s1) = G_ICMP intpred(ne), %0, %1
-    %3:_(s32) = G_SELECT %2(s1), %0(s32), %1(s32)
+    %3:_(s32) = G_SELECT %2, %0, %1
     $vgpr0 = COPY %3
 ...
+
+---
+name: test_icmp_s64
+body: |
+  bb.0.entry:
+    liveins: $vgpr0_vgpr1
+    ; CHECK-LABEL: name: test_icmp_s64
+    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
+    ; CHECK: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[C]](s64), [[COPY]]
+    ; CHECK: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[C]], [[COPY]]
+    ; CHECK: $vgpr0_vgpr1 = COPY [[SELECT]](s64)
+    %0:_(s64) = G_CONSTANT i64 0
+    %1:_(s64) = COPY $vgpr0_vgpr1
+    %2:_(s1) = G_ICMP intpred(ne), %0, %1
+    %3:_(s64) = G_SELECT %2, %0, %1
+    $vgpr0_vgpr1 = COPY %3
+...
+
+---
+name: test_icmp_s16
+body: |
+  bb.0.entry:
+    liveins: $vgpr0
+    ; CHECK-LABEL: name: test_icmp_s16
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; CHECK: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[C]](s32)
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY [[C]](s32)
+    ; CHECK: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C1]]
+    ; CHECK: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; CHECK: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; CHECK: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C2]]
+    ; CHECK: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[AND]](s32), [[AND1]]
+    ; CHECK: [[COPY3:%[0-9]+]]:_(s32) = COPY [[C]](s32)
+    ; CHECK: [[COPY4:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; CHECK: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[COPY3]], [[COPY4]]
+    ; CHECK: [[COPY5:%[0-9]+]]:_(s32) = COPY [[SELECT]](s32)
+    ; CHECK: $vgpr0 = COPY [[COPY5]](s32)
+    %0:_(s16) = G_CONSTANT i16 0
+    %1:_(s32) = COPY $vgpr0
+    %2:_(s16) = G_TRUNC %1
+    %3:_(s1) = G_ICMP intpred(ne), %0, %2
+    %4:_(s16) = G_SELECT %3, %0, %2
+    %5:_(s32) = G_ANYEXT %4
+    $vgpr0 = COPY %5
+...
+
+---
+name: test_icmp_s8
+body: |
+  bb.0.entry:
+    liveins: $vgpr0
+    ; CHECK-LABEL: name: test_icmp_s8
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; CHECK: [[TRUNC:%[0-9]+]]:_(s8) = G_TRUNC [[C]](s32)
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY [[C]](s32)
+    ; CHECK: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C1]]
+    ; CHECK: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; CHECK: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; CHECK: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C2]]
+    ; CHECK: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[AND]](s32), [[AND1]]
+    ; CHECK: [[COPY3:%[0-9]+]]:_(s32) = COPY [[C]](s32)
+    ; CHECK: [[COPY4:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; CHECK: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[COPY3]], [[COPY4]]
+    ; CHECK: [[COPY5:%[0-9]+]]:_(s32) = COPY [[SELECT]](s32)
+    ; CHECK: $vgpr0 = COPY [[COPY5]](s32)
+    %0:_(s8) = G_CONSTANT i8 0
+    %1:_(s32) = COPY $vgpr0
+    %2:_(s8) = G_TRUNC %1
+    %3:_(s1) = G_ICMP intpred(ne), %0, %2
+    %4:_(s8) = G_SELECT %3, %0, %2
+    %5:_(s32) = G_ANYEXT %4
+    $vgpr0 = COPY %5
+...
+
+---
+name: test_icmp_v2s32
+body: |
+  bb.0.entry:
+    liveins: $vgpr0_vgpr1
+    ; CHECK-LABEL: name: test_icmp_v2s32
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    ; CHECK: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
+    ; CHECK: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[C]](s32), [[UV]]
+    ; CHECK: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[C]](s32), [[UV1]]
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s1>) = G_BUILD_VECTOR [[ICMP]](s1), [[ICMP1]](s1)
+    ; CHECK: S_NOP 0, implicit [[BUILD_VECTOR]](<2 x s1>)
+    %0:_(s32) = G_CONSTANT i32 0
+    %1:_(<2 x s32>) = G_BUILD_VECTOR %0, %0
+    %2:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    %3:_(<2 x s1>) = G_ICMP intpred(ne), %1, %2
+    S_NOP 0, implicit %3
+...
+
+---
+name: test_icmp_v3s32
+body: |
+  bb.0.entry:
+    liveins: $vgpr0_vgpr1_vgpr2
+
+    ; CHECK-LABEL: name: test_icmp_v3s32
+    ; CHECK: [[DEF:%[0-9]+]]:_(<3 x s32>) = G_IMPLICIT_DEF
+    ; CHECK: [[COPY:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr0_vgpr1_vgpr2
+    ; CHECK: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[DEF]](<3 x s32>)
+    ; CHECK: [[UV3:%[0-9]+]]:_(s32), [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<3 x s32>)
+    ; CHECK: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[UV]](s32), [[UV3]]
+    ; CHECK: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[UV1]](s32), [[UV4]]
+    ; CHECK: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[UV2]](s32), [[UV5]]
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s1>) = G_BUILD_VECTOR [[ICMP]](s1), [[ICMP1]](s1), [[ICMP2]](s1)
+    ; CHECK: S_NOP 0, implicit [[BUILD_VECTOR]](<3 x s1>)
+    %0:_(<3 x s32>) = G_IMPLICIT_DEF
+    %1:_(<3 x s32>) = COPY $vgpr0_vgpr1_vgpr2
+    %2:_(<3 x s1>) = G_ICMP intpred(ne), %0, %1
+    S_NOP 0, implicit %2
+...
+
+---
+name: test_icmp_v4s32
+body: |
+  bb.0.entry:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3
+
+    ; CHECK-LABEL: name: test_icmp_v4s32
+    ; CHECK: [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+    ; CHECK: [[LOAD:%[0-9]+]]:_(<4 x s32>) = G_LOAD [[DEF]](p1) :: (volatile load 16)
+    ; CHECK: [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; CHECK: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[LOAD]](<4 x s32>)
+    ; CHECK: [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32), [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<4 x s32>)
+    ; CHECK: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[UV]](s32), [[UV4]]
+    ; CHECK: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[UV1]](s32), [[UV5]]
+    ; CHECK: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[UV2]](s32), [[UV6]]
+    ; CHECK: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[UV3]](s32), [[UV7]]
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s1>) = G_BUILD_VECTOR [[ICMP]](s1), [[ICMP1]](s1), [[ICMP2]](s1), [[ICMP3]](s1)
+    ; CHECK: S_NOP 0, implicit [[BUILD_VECTOR]](<4 x s1>)
+    %0:_(p1) = G_IMPLICIT_DEF
+    %1:_(<4 x s32>) = G_LOAD %0 :: (volatile load 16)
+    %2:_(<4 x s32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    %3:_(<4 x s1>) = G_ICMP intpred(ne), %1, %2
+    S_NOP 0, implicit %3
+
+...
+
+---
+name: test_icmp_p0
+body: |
+  bb.0.entry:
+    liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
+    ; CHECK-LABEL: name: test_icmp_p0
+    ; CHECK: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
+    ; CHECK: [[COPY1:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
+    ; CHECK: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[COPY]](p0), [[COPY1]]
+    ; CHECK: [[SEXT:%[0-9]+]]:_(s32) = G_SEXT [[ICMP]](s1)
+    ; CHECK: $vgpr0 = COPY [[SEXT]](s32)
+    %0:_(p0) = COPY $vgpr0_vgpr1
+    %1:_(p0) = COPY $vgpr0_vgpr1
+    %2:_(s1) = G_ICMP intpred(ne), %0, %1
+    %3:_(s32) = G_SEXT %2
+    $vgpr0 = COPY %3
+...
+
+---
+name: test_icmp_p1
+body: |
+  bb.0.entry:
+    liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
+    ; CHECK-LABEL: name: test_icmp_p1
+    ; CHECK: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
+    ; CHECK: [[COPY1:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
+    ; CHECK: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[COPY]](p1), [[COPY1]]
+    ; CHECK: [[SEXT:%[0-9]+]]:_(s32) = G_SEXT [[ICMP]](s1)
+    ; CHECK: $vgpr0 = COPY [[SEXT]](s32)
+    %0:_(p1) = COPY $vgpr0_vgpr1
+    %1:_(p1) = COPY $vgpr0_vgpr1
+    %2:_(s1) = G_ICMP intpred(ne), %0, %1
+    %3:_(s32) = G_SEXT %2
+    $vgpr0 = COPY %3
+...
+
+---
+name: test_icmp_p2
+body: |
+  bb.0.entry:
+    liveins: $vgpr0, $vgpr1
+
+    ; CHECK-LABEL: name: test_icmp_p2
+    ; CHECK: [[COPY:%[0-9]+]]:_(p2) = COPY $vgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(p2) = COPY $vgpr1
+    ; CHECK: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[COPY]](p2), [[COPY1]]
+    ; CHECK: [[SEXT:%[0-9]+]]:_(s32) = G_SEXT [[ICMP]](s1)
+    ; CHECK: $vgpr0 = COPY [[SEXT]](s32)
+    %0:_(p2) = COPY $vgpr0
+    %1:_(p2) = COPY $vgpr1
+    %2:_(s1) = G_ICMP intpred(ne), %0, %1
+    %3:_(s32) = G_SEXT %2
+    $vgpr0 = COPY %3
+...
+
+---
+name: test_icmp_p3
+body: |
+  bb.0.entry:
+    liveins: $vgpr0, $vgpr1
+
+    ; CHECK-LABEL: name: test_icmp_p3
+    ; CHECK: [[COPY:%[0-9]+]]:_(p3) = COPY $vgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(p3) = COPY $vgpr1
+    ; CHECK: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[COPY]](p3), [[COPY1]]
+    ; CHECK: [[SEXT:%[0-9]+]]:_(s32) = G_SEXT [[ICMP]](s1)
+    ; CHECK: $vgpr0 = COPY [[SEXT]](s32)
+    %0:_(p3) = COPY $vgpr0
+    %1:_(p3) = COPY $vgpr1
+    %2:_(s1) = G_ICMP intpred(ne), %0, %1
+    %3:_(s32) = G_SEXT %2
+    $vgpr0 = COPY %3
+...
+
+---
+name: test_icmp_p4
+body: |
+  bb.0.entry:
+    liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
+    ; CHECK-LABEL: name: test_icmp_p4
+    ; CHECK: [[COPY:%[0-9]+]]:_(p4) = COPY $vgpr0_vgpr1
+    ; CHECK: [[COPY1:%[0-9]+]]:_(p4) = COPY $vgpr0_vgpr1
+    ; CHECK: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[COPY]](p4), [[COPY1]]
+    ; CHECK: [[SEXT:%[0-9]+]]:_(s32) = G_SEXT [[ICMP]](s1)
+    ; CHECK: $vgpr0 = COPY [[SEXT]](s32)
+    %0:_(p4) = COPY $vgpr0_vgpr1
+    %1:_(p4) = COPY $vgpr0_vgpr1
+    %2:_(s1) = G_ICMP intpred(ne), %0, %1
+    %3:_(s32) = G_SEXT %2
+    $vgpr0 = COPY %3
+...
+
+---
+name: test_icmp_p5
+body: |
+  bb.0.entry:
+    liveins: $vgpr0, $vgpr1
+
+    ; CHECK-LABEL: name: test_icmp_p5
+    ; CHECK: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(p5) = COPY $vgpr1
+    ; CHECK: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[COPY]](p5), [[COPY1]]
+    ; CHECK: [[SEXT:%[0-9]+]]:_(s32) = G_SEXT [[ICMP]](s1)
+    ; CHECK: $vgpr0 = COPY [[SEXT]](s32)
+    %0:_(p5) = COPY $vgpr0
+    %1:_(p5) = COPY $vgpr1
+    %2:_(s1) = G_ICMP intpred(ne), %0, %1
+    %3:_(s32) = G_SEXT %2
+    $vgpr0 = COPY %3
+...
+
+---
+name: test_icmp_p999
+body: |
+  bb.0.entry:
+    liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
+    ; CHECK-LABEL: name: test_icmp_p999
+    ; CHECK: [[COPY:%[0-9]+]]:_(p999) = COPY $vgpr0_vgpr1
+    ; CHECK: [[COPY1:%[0-9]+]]:_(p999) = COPY $vgpr0_vgpr1
+    ; CHECK: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[COPY]](p999), [[COPY1]]
+    ; CHECK: [[SEXT:%[0-9]+]]:_(s32) = G_SEXT [[ICMP]](s1)
+    ; CHECK: $vgpr0 = COPY [[SEXT]](s32)
+    %0:_(p999) = COPY $vgpr0_vgpr1
+    %1:_(p999) = COPY $vgpr0_vgpr1
+    %2:_(s1) = G_ICMP intpred(ne), %0, %1
+    %3:_(s32) = G_SEXT %2
+    $vgpr0 = COPY %3
+...
+
+---
+name: test_icmp_v2p3
+body: |
+  bb.0.entry:
+    liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
+    ; CHECK-LABEL: name: test_icmp_v2p3
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x p3>) = COPY $vgpr0_vgpr1
+    ; CHECK: [[COPY1:%[0-9]+]]:_(<2 x p3>) = COPY $vgpr0_vgpr1
+    ; CHECK: [[UV:%[0-9]+]]:_(p3), [[UV1:%[0-9]+]]:_(p3) = G_UNMERGE_VALUES [[COPY]](<2 x p3>)
+    ; CHECK: [[UV2:%[0-9]+]]:_(p3), [[UV3:%[0-9]+]]:_(p3) = G_UNMERGE_VALUES [[COPY1]](<2 x p3>)
+    ; CHECK: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[UV]](p3), [[UV2]]
+    ; CHECK: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[UV1]](p3), [[UV3]]
+    ; CHECK: [[SEXT:%[0-9]+]]:_(s32) = G_SEXT [[ICMP]](s1)
+    ; CHECK: [[SEXT1:%[0-9]+]]:_(s32) = G_SEXT [[ICMP1]](s1)
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[SEXT]](s32), [[SEXT1]](s32)
+    ; CHECK: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
+    %0:_(<2 x p3>) = COPY $vgpr0_vgpr1
+    %1:_(<2 x p3>) = COPY $vgpr0_vgpr1
+    %2:_(<2 x s1>) = G_ICMP intpred(ne), %0, %1
+    %3:_(<2 x s32>) = G_SEXT %2
+    $vgpr0_vgpr1 = COPY %3
+...
+
+---
+name: test_icmp_v2p999
+body: |
+  bb.0.entry:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7
+    ; CHECK-LABEL: name: test_icmp_v2p999
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x p999>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; CHECK: [[COPY1:%[0-9]+]]:_(<2 x p999>) = COPY $vgpr4_vgpr5_vgpr6_vgpr7
+    ; CHECK: [[UV:%[0-9]+]]:_(p999), [[UV1:%[0-9]+]]:_(p999) = G_UNMERGE_VALUES [[COPY]](<2 x p999>)
+    ; CHECK: [[UV2:%[0-9]+]]:_(p999), [[UV3:%[0-9]+]]:_(p999) = G_UNMERGE_VALUES [[COPY1]](<2 x p999>)
+    ; CHECK: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[UV]](p999), [[UV2]]
+    ; CHECK: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[UV1]](p999), [[UV3]]
+    ; CHECK: [[SEXT:%[0-9]+]]:_(s32) = G_SEXT [[ICMP]](s1)
+    ; CHECK: [[SEXT1:%[0-9]+]]:_(s32) = G_SEXT [[ICMP1]](s1)
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[SEXT]](s32), [[SEXT1]](s32)
+    ; CHECK: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
+    %0:_(<2 x p999>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    %1:_(<2 x p999>) = COPY $vgpr4_vgpr5_vgpr6_vgpr7
+    %2:_(<2 x s1>) = G_ICMP intpred(ne), %0, %1
+    %3:_(<2 x s32>) = G_SEXT %2
+    $vgpr0_vgpr1 = COPY %3
+...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/legalize-implicit-def.mir b/test/CodeGen/AMDGPU/GlobalISel/legalize-implicit-def.mir
index df7e40ee3601ef843dc185de0ccacbcd8974b883..e6d94ee05bab3a8b8a0206c6e14d0ef90e6fc0e9 100644
--- a/test/CodeGen/AMDGPU/GlobalISel/legalize-implicit-def.mir
+++ b/test/CodeGen/AMDGPU/GlobalISel/legalize-implicit-def.mir
@@ -1,16 +1,144 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn-mesa-mesa3d -run-pass=legalizer %s -o - | FileCheck %s
+# RUN: llc -O0 -mtriple=amdgcn-mesa-mesa3d -mcpu=tahiti -run-pass=legalizer -global-isel-abort=0 %s -o - | FileCheck %s
+# RUN: llc -O0 -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -run-pass=legalizer -global-isel-abort=0 %s -o - | FileCheck %s
 
-# FIXME: Need to add test for IMPLICIT_DEF > 512 once all the operations used
-# to legalize IMPLICIT_DEF are leagl.
+---
+name: test_implicit_def_s1
+body: |
+  bb.0:
+
+    ; CHECK-LABEL: name: test_implicit_def_s1
+    ; CHECK: [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
+    ; CHECK: $vgpr0 = COPY [[DEF]](s32)
+    %0:_(s1) = G_IMPLICIT_DEF
+    %1:_(s32) = G_ANYEXT %0
+    $vgpr0 = COPY %1
+...
+
+---
+name: test_implicit_def_s7
+body: |
+  bb.0:
+
+    ; CHECK-LABEL: name: test_implicit_def_s7
+    ; CHECK: [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY [[DEF]](s32)
+    ; CHECK: $vgpr0 = COPY [[COPY]](s32)
+    %0:_(s7) = G_IMPLICIT_DEF
+    %1:_(s32) = G_ANYEXT %0
+    $vgpr0 = COPY %1
+...
+
+---
+name: test_implicit_def_s8
+body: |
+  bb.0:
+
+    ; CHECK-LABEL: name: test_implicit_def_s8
+    ; CHECK: [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY [[DEF]](s32)
+    ; CHECK: $vgpr0 = COPY [[COPY]](s32)
+    %0:_(s8) = G_IMPLICIT_DEF
+    %1:_(s32) = G_ANYEXT %0
+    $vgpr0 = COPY %1
+...
+
+---
+name: test_implicit_def_s16
+body: |
+  bb.0:
+
+    ; CHECK-LABEL: name: test_implicit_def_s16
+    ; CHECK: [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY [[DEF]](s32)
+    ; CHECK: $vgpr0 = COPY [[COPY]](s32)
+    %0:_(s16) = G_IMPLICIT_DEF
+    %1:_(s32) = G_ANYEXT %0
+    $vgpr0 = COPY %1
+...
 
 ---
-name:            test_implicit_def
+name: test_implicit_def_s32
 body: |
   bb.0:
-    liveins: $vgpr0, $vgpr1
 
-    ; CHECK-LABEL: name: test_implicit_def
+    ; CHECK-LABEL: name: test_implicit_def_s32
+    ; CHECK: [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
+    ; CHECK: $vgpr0 = COPY [[DEF]](s32)
+    %0:_(s32) = G_IMPLICIT_DEF
+    $vgpr0 = COPY %0
+...
+
+---
+name: test_implicit_def_48
+body: |
+  bb.0:
+
+    ; CHECK-LABEL: name: test_implicit_def_48
+    ; CHECK: [[DEF:%[0-9]+]]:_(s64) = G_IMPLICIT_DEF
+    ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY [[DEF]](s64)
+    ; CHECK: $vgpr0_vgpr1 = COPY [[COPY]](s64)
+    %0:_(s48) = G_IMPLICIT_DEF
+    %1:_(s64) = G_ANYEXT %0
+    $vgpr0_vgpr1 = COPY %1
+...
+
+---
+name: test_implicit_def_s64
+body: |
+  bb.0:
+
+    ; CHECK-LABEL: name: test_implicit_def_s64
+    ; CHECK: [[DEF:%[0-9]+]]:_(s64) = G_IMPLICIT_DEF
+    ; CHECK: $vgpr0_vgpr1 = COPY [[DEF]](s64)
+    %0:_(s64) = G_IMPLICIT_DEF
+    $vgpr0_vgpr1 = COPY %0
+...
+
+---
+name: test_implicit_def_s65
+body: |
+  bb.0:
+
+    ; CHECK-LABEL: name: test_implicit_def_s65
+    ; CHECK: [[DEF:%[0-9]+]]:_(s128) = G_IMPLICIT_DEF
+    ; CHECK: [[TRUNC:%[0-9]+]]:_(s96) = G_TRUNC [[DEF]](s128)
+    ; CHECK: $vgpr0_vgpr1_vgpr2 = COPY [[TRUNC]](s96)
+    %0:_(s65) = G_IMPLICIT_DEF
+    %1:_(s96) = G_ANYEXT %0
+    $vgpr0_vgpr1_vgpr2 = COPY %1
+...
+
+---
+name: test_implicit_def_s128
+body: |
+  bb.0:
+
+    ; CHECK-LABEL: name: test_implicit_def_s128
+    ; CHECK: [[DEF:%[0-9]+]]:_(s128) = G_IMPLICIT_DEF
+    ; CHECK: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[DEF]](s128)
+    %0:_(s128) = G_IMPLICIT_DEF
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %0
+...
+
+---
+name: test_implicit_def_256
+body: |
+  bb.0:
+
+    ; CHECK-LABEL: name: test_implicit_def_256
+    ; CHECK: [[DEF:%[0-9]+]]:_(s256) = G_IMPLICIT_DEF
+    ; CHECK: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY [[DEF]](s256)
+    %0:_(s256) = G_IMPLICIT_DEF
+    $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY %0
+...
+
+---
+name: test_implicit_def_s448
+body: |
+  bb.0:
+
+    ; CHECK-LABEL: name: test_implicit_def_s448
     ; CHECK: [[DEF:%[0-9]+]]:_(s448) = G_IMPLICIT_DEF
     ; CHECK: [[EXTRACT:%[0-9]+]]:_(s32) = G_EXTRACT [[DEF]](s448), 0
     ; CHECK: $vgpr0 = COPY [[EXTRACT]](s32)
@@ -18,3 +146,336 @@ body: |
     %1:_(s32) = G_EXTRACT %0, 0
     $vgpr0 = COPY %1
 ...
+
+---
+name: test_implicit_def_s512
+body: |
+  bb.0:
+
+    ; CHECK-LABEL: name: test_implicit_def_s512
+    ; CHECK: [[DEF:%[0-9]+]]:_(s512) = G_IMPLICIT_DEF
+    ; CHECK: [[EXTRACT:%[0-9]+]]:_(s32) = G_EXTRACT [[DEF]](s512), 0
+    ; CHECK: $vgpr0 = COPY [[EXTRACT]](s32)
+    %0:_(s512) = G_IMPLICIT_DEF
+    %1:_(s32) = G_EXTRACT %0, 0
+    $vgpr0 = COPY %1
+...
+
+---
+name: test_implicit_def_s1024
+body: |
+  bb.0:
+
+    ; CHECK-LABEL: name: test_implicit_def_s1024
+    ; CHECK: [[DEF:%[0-9]+]]:_(s512) = G_IMPLICIT_DEF
+    ; CHECK: [[DEF1:%[0-9]+]]:_(s512) = G_IMPLICIT_DEF
+    ; CHECK: [[EXTRACT:%[0-9]+]]:_(s32) = G_EXTRACT [[DEF]](s512), 0
+    ; CHECK: $vgpr0 = COPY [[EXTRACT]](s32)
+    %0:_(s1024) = G_IMPLICIT_DEF
+    %1:_(s32) = G_EXTRACT %0, 0
+    $vgpr0 = COPY %1
+...
+
+---
+name: test_implicit_def_v2s32
+body: |
+  bb.0:
+
+    ; CHECK-LABEL: name: test_implicit_def_v2s32
+    ; CHECK: [[DEF:%[0-9]+]]:_(<2 x s32>) = G_IMPLICIT_DEF
+    ; CHECK: $vgpr0_vgpr1 = COPY [[DEF]](<2 x s32>)
+    %0:_(<2 x s32>) = G_IMPLICIT_DEF
+    $vgpr0_vgpr1 = COPY %0
+...
+
+---
+name: test_implicit_def_v3s32
+body: |
+  bb.0:
+
+    ; CHECK-LABEL: name: test_implicit_def_v3s32
+    ; CHECK: [[DEF:%[0-9]+]]:_(<3 x s32>) = G_IMPLICIT_DEF
+    ; CHECK: $vgpr0_vgpr1_vgpr2 = COPY [[DEF]](<3 x s32>)
+    %0:_(<3 x s32>) = G_IMPLICIT_DEF
+    $vgpr0_vgpr1_vgpr2 = COPY %0
+...
+
+---
+name: test_implicit_def_v4s32
+body: |
+  bb.0:
+
+    ; CHECK-LABEL: name: test_implicit_def_v4s32
+    ; CHECK: [[DEF:%[0-9]+]]:_(<4 x s32>) = G_IMPLICIT_DEF
+    ; CHECK: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[DEF]](<4 x s32>)
+    %0:_(<4 x s32>) = G_IMPLICIT_DEF
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %0
+...
+
+---
+name: test_implicit_def_v2s1
+body: |
+  bb.0:
+
+    ; CHECK-LABEL: name: test_implicit_def_v2s1
+    ; CHECK: [[DEF:%[0-9]+]]:_(<2 x s32>) = G_IMPLICIT_DEF
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY [[DEF]](<2 x s32>)
+    ; CHECK: $vgpr0_vgpr1 = COPY [[COPY]](<2 x s32>)
+    %0:_(<2 x s1>) = G_IMPLICIT_DEF
+    %1:_(<2 x s32>) = G_ANYEXT %0
+    $vgpr0_vgpr1 = COPY %1
+...
+
+---
+name: test_implicit_def_v3s1
+body: |
+  bb.0:
+
+    ; CHECK-LABEL: name: test_implicit_def_v3s1
+    ; CHECK: [[DEF:%[0-9]+]]:_(<4 x s32>) = G_IMPLICIT_DEF
+    ; CHECK: [[TRUNC:%[0-9]+]]:_(<4 x s1>) = G_TRUNC [[DEF]](<4 x s32>)
+    ; CHECK: [[EXTRACT:%[0-9]+]]:_(<3 x s1>) = G_EXTRACT [[TRUNC]](<4 x s1>), 0
+    ; CHECK: [[UV:%[0-9]+]]:_(s1), [[UV1:%[0-9]+]]:_(s1), [[UV2:%[0-9]+]]:_(s1) = G_UNMERGE_VALUES [[EXTRACT]](<3 x s1>)
+    ; CHECK: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[UV]](s1)
+    ; CHECK: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[UV1]](s1)
+    ; CHECK: [[ANYEXT2:%[0-9]+]]:_(s32) = G_ANYEXT [[UV2]](s1)
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[ANYEXT]](s32), [[ANYEXT1]](s32), [[ANYEXT2]](s32)
+    ; CHECK: $vgpr0_vgpr1_vgpr2 = COPY [[BUILD_VECTOR]](<3 x s32>)
+    %0:_(<3 x s1>) = G_IMPLICIT_DEF
+    %1:_(<3 x s32>) = G_ANYEXT %0
+    $vgpr0_vgpr1_vgpr2 = COPY %1
+...
+
+---
+name: test_implicit_def_v2s8
+body: |
+  bb.0:
+
+    ; CHECK-LABEL: name: test_implicit_def_v2s8
+    ; CHECK: [[DEF:%[0-9]+]]:_(<2 x s32>) = G_IMPLICIT_DEF
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY [[DEF]](<2 x s32>)
+    ; CHECK: $vgpr0_vgpr1 = COPY [[COPY]](<2 x s32>)
+    %0:_(<2 x s8>) = G_IMPLICIT_DEF
+    %1:_(<2 x s32>) = G_ANYEXT %0
+    $vgpr0_vgpr1 = COPY %1
+...
+
+---
+name: test_implicit_def_v3s8
+body: |
+  bb.0:
+
+    ; CHECK-LABEL: name: test_implicit_def_v3s8
+    ; CHECK: [[DEF:%[0-9]+]]:_(<4 x s32>) = G_IMPLICIT_DEF
+    ; CHECK: [[TRUNC:%[0-9]+]]:_(<4 x s8>) = G_TRUNC [[DEF]](<4 x s32>)
+    ; CHECK: [[EXTRACT:%[0-9]+]]:_(<3 x s8>) = G_EXTRACT [[TRUNC]](<4 x s8>), 0
+    ; CHECK: [[UV:%[0-9]+]]:_(s8), [[UV1:%[0-9]+]]:_(s8), [[UV2:%[0-9]+]]:_(s8) = G_UNMERGE_VALUES [[EXTRACT]](<3 x s8>)
+    ; CHECK: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[UV]](s8)
+    ; CHECK: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[UV1]](s8)
+    ; CHECK: [[ANYEXT2:%[0-9]+]]:_(s32) = G_ANYEXT [[UV2]](s8)
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[ANYEXT]](s32), [[ANYEXT1]](s32), [[ANYEXT2]](s32)
+    ; CHECK: $vgpr0_vgpr1_vgpr2 = COPY [[BUILD_VECTOR]](<3 x s32>)
+    %0:_(<3 x s8>) = G_IMPLICIT_DEF
+    %1:_(<3 x s32>) = G_ANYEXT %0
+    $vgpr0_vgpr1_vgpr2 = COPY %1
+...
+
+---
+name: test_implicit_def_v2s16
+body: |
+  bb.0:
+
+    ; CHECK-LABEL: name: test_implicit_def_v2s16
+    ; CHECK: [[DEF:%[0-9]+]]:_(<2 x s16>) = G_IMPLICIT_DEF
+    ; CHECK: $vgpr0 = COPY [[DEF]](<2 x s16>)
+    %0:_(<2 x s16>) = G_IMPLICIT_DEF
+    $vgpr0 = COPY %0
+...
+
+---
+name: test_implicit_def_v3s16
+body: |
+  bb.0:
+
+    ; CHECK-LABEL: name: test_implicit_def_v3s16
+    ; CHECK: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; CHECK: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
+    ; CHECK: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; CHECK: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[EXTRACT]](<3 x s16>), 0
+    ; CHECK: $vgpr0_vgpr1 = COPY [[INSERT]](<4 x s16>)
+    %0:_(<3 x s16>) = G_IMPLICIT_DEF
+    %1:_(<4 x s16>) = G_IMPLICIT_DEF
+    %2:_(<4 x s16>) = G_INSERT %1, %0, 0
+    $vgpr0_vgpr1 = COPY %2
+...
+
+---
+name: test_implicit_def_v4s16
+body: |
+  bb.0:
+
+    ; CHECK-LABEL: name: test_implicit_def_v4s16
+    ; CHECK: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; CHECK: $vgpr0_vgpr1 = COPY [[DEF]](<4 x s16>)
+    %0:_(<4 x s16>) = G_IMPLICIT_DEF
+    $vgpr0_vgpr1 = COPY %0
+...
+
+---
+name: test_implicit_def_v5s16
+body: |
+  bb.0:
+
+    ; CHECK-LABEL: name: test_implicit_def_v5s16
+    ; CHECK: [[DEF:%[0-9]+]]:_(<6 x s32>) = G_IMPLICIT_DEF
+    ; CHECK: [[TRUNC:%[0-9]+]]:_(<6 x s16>) = G_TRUNC [[DEF]](<6 x s32>)
+    ; CHECK: [[EXTRACT:%[0-9]+]]:_(<5 x s16>) = G_EXTRACT [[TRUNC]](<6 x s16>), 0
+    ; CHECK: [[DEF1:%[0-9]+]]:_(<8 x s32>) = G_IMPLICIT_DEF
+    ; CHECK: [[TRUNC1:%[0-9]+]]:_(<8 x s16>) = G_TRUNC [[DEF1]](<8 x s32>)
+    ; CHECK: [[INSERT:%[0-9]+]]:_(<8 x s16>) = G_INSERT [[TRUNC1]], [[EXTRACT]](<5 x s16>), 0
+    ; CHECK: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[INSERT]](<8 x s16>)
+    %0:_(<5 x s16>) = G_IMPLICIT_DEF
+    %1:_(<8 x s16>) = G_IMPLICIT_DEF
+    %2:_(<8 x s16>) = G_INSERT %1, %0, 0
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %2
+...
+
+---
+name: test_implicit_def_v6s16
+body: |
+  bb.0:
+
+    ; CHECK-LABEL: name: test_implicit_def_v6s16
+    ; CHECK: [[DEF:%[0-9]+]]:_(<6 x s32>) = G_IMPLICIT_DEF
+    ; CHECK: [[TRUNC:%[0-9]+]]:_(<6 x s16>) = G_TRUNC [[DEF]](<6 x s32>)
+    ; CHECK: [[DEF1:%[0-9]+]]:_(<8 x s32>) = G_IMPLICIT_DEF
+    ; CHECK: [[TRUNC1:%[0-9]+]]:_(<8 x s16>) = G_TRUNC [[DEF1]](<8 x s32>)
+    ; CHECK: [[INSERT:%[0-9]+]]:_(<8 x s16>) = G_INSERT [[TRUNC1]], [[TRUNC]](<6 x s16>), 0
+    ; CHECK: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[INSERT]](<8 x s16>)
+    %0:_(<6 x s16>) = G_IMPLICIT_DEF
+    %1:_(<8 x s16>) = G_IMPLICIT_DEF
+    %2:_(<8 x s16>) = G_INSERT %1, %0, 0
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %2
+...
+
+---
+name: test_implicit_def_v8s16
+body: |
+  bb.0:
+
+    ; CHECK-LABEL: name: test_implicit_def_v8s16
+    ; CHECK: [[DEF:%[0-9]+]]:_(<8 x s32>) = G_IMPLICIT_DEF
+    ; CHECK: [[TRUNC:%[0-9]+]]:_(<8 x s16>) = G_TRUNC [[DEF]](<8 x s32>)
+    ; CHECK: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[TRUNC]](<8 x s16>)
+    %0:_(<8 x s16>) = G_IMPLICIT_DEF
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %0
+...
+
+---
+name: test_implicit_def_v2s64
+body: |
+  bb.0:
+
+    ; CHECK-LABEL: name: test_implicit_def_v2s64
+    ; CHECK: [[DEF:%[0-9]+]]:_(<2 x s64>) = G_IMPLICIT_DEF
+    ; CHECK: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[DEF]](<2 x s64>)
+    %0:_(<2 x s64>) = G_IMPLICIT_DEF
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %0
+...
+
+---
+name: test_implicit_def_v4s8
+body: |
+  bb.0:
+
+    ; CHECK-LABEL: name: test_implicit_def_v4s8
+    ; CHECK: [[DEF:%[0-9]+]]:_(<4 x s32>) = G_IMPLICIT_DEF
+    ; CHECK: [[TRUNC:%[0-9]+]]:_(<4 x s8>) = G_TRUNC [[DEF]](<4 x s32>)
+    ; CHECK: $vgpr0 = COPY [[TRUNC]](<4 x s8>)
+    %0:_(<4 x s8>) = G_IMPLICIT_DEF
+    $vgpr0 = COPY %0
+...
+
+---
+name: test_implicit_def_p0
+body: |
+  bb.0:
+
+    ; CHECK-LABEL: name: test_implicit_def_p0
+    ; CHECK: [[DEF:%[0-9]+]]:_(p0) = G_IMPLICIT_DEF
+    ; CHECK: $vgpr0_vgpr1 = COPY [[DEF]](p0)
+    %0:_(p0) = G_IMPLICIT_DEF
+    $vgpr0_vgpr1 = COPY %0
+...
+
+---
+name: test_implicit_def_p1
+body: |
+  bb.0:
+
+    ; CHECK-LABEL: name: test_implicit_def_p1
+    ; CHECK: [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+    ; CHECK: $vgpr0_vgpr1 = COPY [[DEF]](p1)
+    %0:_(p1) = G_IMPLICIT_DEF
+    $vgpr0_vgpr1 = COPY %0
+...
+
+---
+name: test_implicit_def_p2
+body: |
+  bb.0:
+
+    ; CHECK-LABEL: name: test_implicit_def_p2
+    ; CHECK: [[DEF:%[0-9]+]]:_(p2) = G_IMPLICIT_DEF
+    ; CHECK: $vgpr0 = COPY [[DEF]](p2)
+    %0:_(p2) = G_IMPLICIT_DEF
+    $vgpr0 = COPY %0
+...
+
+---
+name: test_implicit_def_p3
+body: |
+  bb.0:
+
+    ; CHECK-LABEL: name: test_implicit_def_p3
+    ; CHECK: [[DEF:%[0-9]+]]:_(p3) = G_IMPLICIT_DEF
+    ; CHECK: $vgpr0 = COPY [[DEF]](p3)
+    %0:_(p3) = G_IMPLICIT_DEF
+    $vgpr0 = COPY %0
+...
+
+---
+name: test_implicit_def_p4
+body: |
+  bb.0:
+
+    ; CHECK-LABEL: name: test_implicit_def_p4
+    ; CHECK: [[DEF:%[0-9]+]]:_(p4) = G_IMPLICIT_DEF
+    ; CHECK: $vgpr0_vgpr1 = COPY [[DEF]](p4)
+    %0:_(p4) = G_IMPLICIT_DEF
+    $vgpr0_vgpr1 = COPY %0
+...
+
+---
+name: test_implicit_def_p5
+body: |
+  bb.0:
+
+    ; CHECK-LABEL: name: test_implicit_def_p5
+    ; CHECK: [[DEF:%[0-9]+]]:_(p5) = G_IMPLICIT_DEF
+    ; CHECK: $vgpr0 = COPY [[DEF]](p5)
+    %0:_(p5) = G_IMPLICIT_DEF
+    $vgpr0 = COPY %0
+...
+
+---
+name: test_implicit_def_p999
+body: |
+  bb.0:
+
+    ; CHECK-LABEL: name: test_implicit_def_p999
+    ; CHECK: [[DEF:%[0-9]+]]:_(p999) = G_IMPLICIT_DEF
+    ; CHECK: $vgpr0_vgpr1 = COPY [[DEF]](p999)
+    %0:_(p999) = G_IMPLICIT_DEF
+    $vgpr0_vgpr1 = COPY %0
+...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/legalize-insert.mir b/test/CodeGen/AMDGPU/GlobalISel/legalize-insert.mir
index 569f6a3b260c167c5abbc2b10ef1d5332f58b945..0572ea4d8243d86397ba55345ab0ec27be896170 100644
--- a/test/CodeGen/AMDGPU/GlobalISel/legalize-insert.mir
+++ b/test/CodeGen/AMDGPU/GlobalISel/legalize-insert.mir
@@ -1,123 +1,659 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
 # RUN: llc -mtriple=amdgcn-mesa-mesa3d -run-pass=legalizer %s -o - | FileCheck %s
 
 ---
-name: test_insert_lo32_i64
+name: test_insert_s64_s32_offset0
 body: |
-  bb.0.entry:
+  bb.0:
     liveins: $vgpr0_vgpr1, $vgpr2
 
-    ; CHECK-LABEL: name: test_insert_lo32_i64
-    ; CHECK: %2:_(s64) = G_INSERT %0, %1(s32), 0
+    ; CHECK-LABEL: name: test_insert_s64_s32_offset0
+    ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr2
+    ; CHECK: [[INSERT:%[0-9]+]]:_(s64) = G_INSERT [[COPY]], [[COPY1]](s32), 0
+    ; CHECK: $vgpr0_vgpr1 = COPY [[INSERT]](s64)
     %0:_(s64) = COPY $vgpr0_vgpr1
     %1:_(s32) = COPY $vgpr2
     %2:_(s64) = G_INSERT %0, %1, 0
-    S_ENDPGM implicit %2
+    $vgpr0_vgpr1 = COPY %2
 ...
 ---
-name: test_insert_hi32_i64
+name: test_insert_s64_s32_offset32
 body: |
-  bb.0.entry:
+  bb.0:
     liveins: $vgpr0_vgpr1, $vgpr2
 
-    ; CHECK-LABEL: name: test_insert_hi32_i64
-    ; CHECK: %2:_(s64) = G_INSERT %0, %1(s32), 32
+    ; CHECK-LABEL: name: test_insert_s64_s32_offset32
+    ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr2
+    ; CHECK: [[INSERT:%[0-9]+]]:_(s64) = G_INSERT [[COPY]], [[COPY1]](s32), 32
+    ; CHECK: $vgpr0_vgpr1 = COPY [[INSERT]](s64)
     %0:_(s64) = COPY $vgpr0_vgpr1
     %1:_(s32) = COPY $vgpr2
     %2:_(s64) = G_INSERT %0, %1, 32
-    S_ENDPGM implicit %2
+    $vgpr0_vgpr1 = COPY %2
 ...
+---
+name: test_insert_s96_s32_offset0
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2, $vgpr3
+
+    ; CHECK-LABEL: name: test_insert_s96_s32_offset0
+    ; CHECK: [[COPY:%[0-9]+]]:_(s96) = COPY $vgpr0_vgpr1_vgpr2
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr3
+    ; CHECK: [[INSERT:%[0-9]+]]:_(s96) = G_INSERT [[COPY]], [[COPY1]](s32), 0
+    ; CHECK: $vgpr0_vgpr1_vgpr2 = COPY [[INSERT]](s96)
+    %0:_(s96) = COPY $vgpr0_vgpr1_vgpr2
+    %1:_(s32) = COPY $vgpr3
+    %2:_(s96) = G_INSERT %0, %1, 0
+    $vgpr0_vgpr1_vgpr2 = COPY %2
+...
+---
+name: test_insert_s96_s32_offset32
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2, $vgpr3
 
+    ; CHECK-LABEL: name: test_insert_s96_s32_offset32
+    ; CHECK: [[COPY:%[0-9]+]]:_(s96) = COPY $vgpr0_vgpr1_vgpr2
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr3
+    ; CHECK: [[INSERT:%[0-9]+]]:_(s96) = G_INSERT [[COPY]], [[COPY1]](s32), 32
+    ; CHECK: $vgpr0_vgpr1_vgpr2 = COPY [[INSERT]](s96)
+    %0:_(s96) = COPY $vgpr0_vgpr1_vgpr2
+    %1:_(s32) = COPY $vgpr3
+    %2:_(s96) = G_INSERT %0, %1, 32
+    $vgpr0_vgpr1_vgpr2 = COPY %2
+...
+---
+name: test_insert_s96_s32_offset64
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2, $vgpr3
+
+    ; CHECK-LABEL: name: test_insert_s96_s32_offset64
+    ; CHECK: [[COPY:%[0-9]+]]:_(s96) = COPY $vgpr0_vgpr1_vgpr2
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr3
+    ; CHECK: [[INSERT:%[0-9]+]]:_(s96) = G_INSERT [[COPY]], [[COPY1]](s32), 64
+    ; CHECK: $vgpr0_vgpr1_vgpr2 = COPY [[INSERT]](s96)
+    %0:_(s96) = COPY $vgpr0_vgpr1_vgpr2
+    %1:_(s32) = COPY $vgpr3
+    %2:_(s96) = G_INSERT %0, %1, 64
+    $vgpr0_vgpr1_vgpr2 = COPY %2
+...
 ---
-name: test_insert_dword0_i128
+name: test_insert_s128_s32_offset0
 body: |
-  bb.0.entry:
+  bb.0:
     liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4
 
-    ; CHECK-LABEL: name: test_insert_dword0_i128
-    ; CHECK: %2:_(s128) = G_INSERT %0, %1(s32), 0
+    ; CHECK-LABEL: name: test_insert_s128_s32_offset0
+    ; CHECK: [[COPY:%[0-9]+]]:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr4
+    ; CHECK: [[INSERT:%[0-9]+]]:_(s128) = G_INSERT [[COPY]], [[COPY1]](s32), 0
+    ; CHECK: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[INSERT]](s128)
     %0:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
     %1:_(s32) = COPY $vgpr4
     %2:_(s128) = G_INSERT %0, %1, 0
-    S_ENDPGM implicit %2
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %2
 ...
-
 ---
-name: test_insert_dword1_i128
+name: test_insert_s128_s32_offset32
 body: |
-  bb.0.entry:
+  bb.0:
     liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4
 
-    ; CHECK-LABEL: name: test_insert_dword1_i128
-    ; CHECK: %2:_(s128) = G_INSERT %0, %1(s32), 32
+    ; CHECK-LABEL: name: test_insert_s128_s32_offset32
+    ; CHECK: [[COPY:%[0-9]+]]:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr4
+    ; CHECK: [[INSERT:%[0-9]+]]:_(s128) = G_INSERT [[COPY]], [[COPY1]](s32), 32
+    ; CHECK: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[INSERT]](s128)
     %0:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
     %1:_(s32) = COPY $vgpr4
     %2:_(s128) = G_INSERT %0, %1, 32
-    S_ENDPGM implicit %2
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %2
 ...
-
 ---
-name: test_insert_dword2_i128
+name: test_insert_s128_s32_offset64
 body: |
-  bb.0.entry:
+  bb.0:
     liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4
 
-    ; CHECK-LABEL: name: test_insert_dword2_i128
-    ; CHECK: %2:_(s128) = G_INSERT %0, %1(s32), 64
+    ; CHECK-LABEL: name: test_insert_s128_s32_offset64
+    ; CHECK: [[COPY:%[0-9]+]]:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr4
+    ; CHECK: [[INSERT:%[0-9]+]]:_(s128) = G_INSERT [[COPY]], [[COPY1]](s32), 64
+    ; CHECK: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[INSERT]](s128)
     %0:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
     %1:_(s32) = COPY $vgpr4
     %2:_(s128) = G_INSERT %0, %1, 64
-    S_ENDPGM implicit %2
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %2
 ...
-
 ---
-name: test_insert_dword3_i128
+name: test_insert_s128_s32_offset96
 body: |
-  bb.0.entry:
+  bb.0:
     liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4
 
-    ; CHECK-LABEL: name: test_insert_dword3_i128
-    ; CHECK: %2:_(s128) = G_INSERT %0, %1(s32), 96
+    ; CHECK-LABEL: name: test_insert_s128_s32_offset96
+    ; CHECK: [[COPY:%[0-9]+]]:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr4
+    ; CHECK: [[INSERT:%[0-9]+]]:_(s128) = G_INSERT [[COPY]], [[COPY1]](s32), 96
+    ; CHECK: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[INSERT]](s128)
     %0:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
     %1:_(s32) = COPY $vgpr4
     %2:_(s128) = G_INSERT %0, %1, 96
-    S_ENDPGM implicit %2
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %2
+...
+---
+name: test_insert_s128_s64_offset0
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5
+
+    ; CHECK-LABEL: name: test_insert_s128_s64_offset0
+    ; CHECK: [[COPY:%[0-9]+]]:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s64) = COPY $vgpr4_vgpr5
+    ; CHECK: [[INSERT:%[0-9]+]]:_(s128) = G_INSERT [[COPY]], [[COPY1]](s64), 0
+    ; CHECK: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[INSERT]](s128)
+    %0:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    %1:_(s64) = COPY $vgpr4_vgpr5
+    %2:_(s128) = G_INSERT %0, %1, 0
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %2
+...
+---
+name: test_insert_s128_s64_offset32
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5
+
+    ; CHECK-LABEL: name: test_insert_s128_s64_offset32
+    ; CHECK: [[COPY:%[0-9]+]]:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s64) = COPY $vgpr4_vgpr5
+    ; CHECK: [[INSERT:%[0-9]+]]:_(s128) = G_INSERT [[COPY]], [[COPY1]](s64), 32
+    ; CHECK: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[INSERT]](s128)
+    %0:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    %1:_(s64) = COPY $vgpr4_vgpr5
+    %2:_(s128) = G_INSERT %0, %1, 32
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %2
+...
+---
+name: test_insert_s128_s64_offset64
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5
+
+    ; CHECK-LABEL: name: test_insert_s128_s64_offset64
+    ; CHECK: [[COPY:%[0-9]+]]:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s64) = COPY $vgpr4_vgpr5
+    ; CHECK: [[INSERT:%[0-9]+]]:_(s128) = G_INSERT [[COPY]], [[COPY1]](s64), 64
+    ; CHECK: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[INSERT]](s128)
+    %0:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    %1:_(s64) = COPY $vgpr4_vgpr5
+    %2:_(s128) = G_INSERT %0, %1, 64
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %2
+...
+---
+name: test_insert_s128_s96_offset0
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6
+
+    ; CHECK-LABEL: name: test_insert_s128_s96_offset0
+    ; CHECK: [[COPY:%[0-9]+]]:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s96) = COPY $vgpr4_vgpr5_vgpr6
+    ; CHECK: [[INSERT:%[0-9]+]]:_(s128) = G_INSERT [[COPY]], [[COPY1]](s96), 0
+    ; CHECK: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[INSERT]](s128)
+    %0:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    %1:_(s96) = COPY $vgpr4_vgpr5_vgpr6
+    %2:_(s128) = G_INSERT %0, %1, 0
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %2
+...
+---
+name: test_insert_s128_s96_offset32
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6
+
+    ; CHECK-LABEL: name: test_insert_s128_s96_offset32
+    ; CHECK: [[COPY:%[0-9]+]]:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s96) = COPY $vgpr4_vgpr5_vgpr6
+    ; CHECK: [[INSERT:%[0-9]+]]:_(s128) = G_INSERT [[COPY]], [[COPY1]](s96), 32
+    ; CHECK: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[INSERT]](s128)
+    %0:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    %1:_(s96) = COPY $vgpr4_vgpr5_vgpr6
+    %2:_(s128) = G_INSERT %0, %1, 32
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %2
+...
+---
+name: test_insert_p0_s32_offset0
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2
+
+    ; CHECK-LABEL: name: test_insert_p0_s32_offset0
+    ; CHECK: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr2
+    ; CHECK: [[INSERT:%[0-9]+]]:_(p0) = G_INSERT [[COPY]], [[COPY1]](s32), 0
+    ; CHECK: $vgpr0_vgpr1 = COPY [[INSERT]](p0)
+    %0:_(p0) = COPY $vgpr0_vgpr1
+    %1:_(s32) = COPY $vgpr2
+    %2:_(p0) = G_INSERT %0, %1, 0
+    $vgpr0_vgpr1 = COPY %2
+...
+---
+name: test_insert_p0_s32_offset32
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2
+
+    ; CHECK-LABEL: name: test_insert_p0_s32_offset32
+    ; CHECK: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr2
+    ; CHECK: [[INSERT:%[0-9]+]]:_(p0) = G_INSERT [[COPY]], [[COPY1]](s32), 32
+    ; CHECK: $vgpr0_vgpr1 = COPY [[INSERT]](p0)
+    %0:_(p0) = COPY $vgpr0_vgpr1
+    %1:_(s32) = COPY $vgpr2
+    %2:_(p0) = G_INSERT %0, %1, 32
+    $vgpr0_vgpr1 = COPY %2
+...
+---
+name: test_insert_s128_p0_offset0
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5
+
+    ; CHECK-LABEL: name: test_insert_s128_p0_offset0
+    ; CHECK: [[COPY:%[0-9]+]]:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; CHECK: [[COPY1:%[0-9]+]]:_(p0) = COPY $vgpr4_vgpr5
+    ; CHECK: [[INSERT:%[0-9]+]]:_(s128) = G_INSERT [[COPY]], [[COPY1]](p0), 0
+    ; CHECK: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[INSERT]](s128)
+    %0:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    %1:_(p0) = COPY $vgpr4_vgpr5
+    %2:_(s128) = G_INSERT %0, %1, 0
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %2
+...
+---
+name: test_insert_s128_p0_offset32
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5
+
+    ; CHECK-LABEL: name: test_insert_s128_p0_offset32
+    ; CHECK: [[COPY:%[0-9]+]]:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; CHECK: [[COPY1:%[0-9]+]]:_(p0) = COPY $vgpr4_vgpr5
+    ; CHECK: [[INSERT:%[0-9]+]]:_(s128) = G_INSERT [[COPY]], [[COPY1]](p0), 32
+    ; CHECK: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[INSERT]](s128)
+    %0:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    %1:_(p0) = COPY $vgpr4_vgpr5
+    %2:_(s128) = G_INSERT %0, %1, 32
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %2
+...
+---
+name: test_insert_s128_p0_offset64
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5
+
+    ; CHECK-LABEL: name: test_insert_s128_p0_offset64
+    ; CHECK: [[COPY:%[0-9]+]]:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; CHECK: [[COPY1:%[0-9]+]]:_(p0) = COPY $vgpr4_vgpr5
+    ; CHECK: [[INSERT:%[0-9]+]]:_(s128) = G_INSERT [[COPY]], [[COPY1]](p0), 64
+    ; CHECK: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[INSERT]](s128)
+    %0:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    %1:_(p0) = COPY $vgpr4_vgpr5
+    %2:_(s128) = G_INSERT %0, %1, 64
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %2
+...
+---
+name: test_insert_v2s32_s32_offset0
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2
+
+    ; CHECK-LABEL: name: test_insert_v2s32_s32_offset0
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr2
+    ; CHECK: [[INSERT:%[0-9]+]]:_(<2 x s32>) = G_INSERT [[COPY]], [[COPY1]](s32), 0
+    ; CHECK: $vgpr0_vgpr1 = COPY [[INSERT]](<2 x s32>)
+    %0:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    %1:_(s32) = COPY $vgpr2
+    %2:_(<2 x s32>) = G_INSERT %0, %1, 0
+    $vgpr0_vgpr1 = COPY %2
+...
+---
+name: test_insert_v2s32_s32_offset32
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2
+
+    ; CHECK-LABEL: name: test_insert_v2s32_s32_offset32
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr2
+    ; CHECK: [[INSERT:%[0-9]+]]:_(<2 x s32>) = G_INSERT [[COPY]], [[COPY1]](s32), 32
+    ; CHECK: $vgpr0_vgpr1 = COPY [[INSERT]](<2 x s32>)
+    %0:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    %1:_(s32) = COPY $vgpr2
+    %2:_(<2 x s32>) = G_INSERT %0, %1, 32
+    $vgpr0_vgpr1 = COPY %2
 ...
 ---
-name: test_insert_dword0_i96
+name: test_insert_v3s32_s32_offset0
 body: |
-  bb.0.entry:
+  bb.0:
     liveins: $vgpr0_vgpr1_vgpr2, $vgpr3
 
-    ; CHECK-LABEL: name: test_insert_dword0_i96
-    ; CHECK: %2:_(s96) = G_INSERT %0, %1(s32), 0
-    %0:_(s96) = COPY $vgpr0_vgpr1_vgpr2
+    ; CHECK-LABEL: name: test_insert_v3s32_s32_offset0
+    ; CHECK: [[COPY:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr0_vgpr1_vgpr2
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr3
+    ; CHECK: [[INSERT:%[0-9]+]]:_(<3 x s32>) = G_INSERT [[COPY]], [[COPY1]](s32), 0
+    ; CHECK: $vgpr0_vgpr1_vgpr2 = COPY [[INSERT]](<3 x s32>)
+    %0:_(<3 x s32>) = COPY $vgpr0_vgpr1_vgpr2
     %1:_(s32) = COPY $vgpr3
-    %2:_(s96) = G_INSERT %0, %1, 0
-    S_ENDPGM implicit %2
+    %2:_(<3 x s32>) = G_INSERT %0, %1, 0
+    $vgpr0_vgpr1_vgpr2 = COPY %2
 ...
 ---
-name: test_insert_dword1_i96
+name: test_insert_v3s32_s32_offset32
 body: |
-  bb.0.entry:
+  bb.0:
     liveins: $vgpr0_vgpr1_vgpr2, $vgpr3
 
-    ; CHECK-LABEL: name: test_insert_dword1_i96
-    ; CHECK: %2:_(s96) = G_INSERT %0, %1(s32), 32
-    %0:_(s96) = COPY $vgpr0_vgpr1_vgpr2
+    ; CHECK-LABEL: name: test_insert_v3s32_s32_offset32
+    ; CHECK: [[COPY:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr0_vgpr1_vgpr2
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr3
+    ; CHECK: [[INSERT:%[0-9]+]]:_(<3 x s32>) = G_INSERT [[COPY]], [[COPY1]](s32), 32
+    ; CHECK: $vgpr0_vgpr1_vgpr2 = COPY [[INSERT]](<3 x s32>)
+    %0:_(<3 x s32>) = COPY $vgpr0_vgpr1_vgpr2
     %1:_(s32) = COPY $vgpr3
-    %2:_(s96) = G_INSERT %0, %1, 32
-    S_ENDPGM implicit %2
+    %2:_(<3 x s32>) = G_INSERT %0, %1, 32
+    $vgpr0_vgpr1_vgpr2 = COPY %2
 ...
 ---
-name: test_insert_dword2_i96
+name: test_insert_v3s32_s32_offset64
 body: |
-  bb.0.entry:
+  bb.0:
     liveins: $vgpr0_vgpr1_vgpr2, $vgpr3
 
-    ; CHECK-LABEL: name: test_insert_dword2_i96
-    ; CHECK: %2:_(s96) = G_INSERT %0, %1(s32), 64
-    %0:_(s96) = COPY $vgpr0_vgpr1_vgpr2
+    ; CHECK-LABEL: name: test_insert_v3s32_s32_offset64
+    ; CHECK: [[COPY:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr0_vgpr1_vgpr2
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr3
+    ; CHECK: [[INSERT:%[0-9]+]]:_(<3 x s32>) = G_INSERT [[COPY]], [[COPY1]](s32), 64
+    ; CHECK: $vgpr0_vgpr1_vgpr2 = COPY [[INSERT]](<3 x s32>)
+    %0:_(<3 x s32>) = COPY $vgpr0_vgpr1_vgpr2
     %1:_(s32) = COPY $vgpr3
-    %2:_(s96) = G_INSERT %0, %1, 64
-    S_ENDPGM implicit %2
+    %2:_(<3 x s32>) = G_INSERT %0, %1, 64
+    $vgpr0_vgpr1_vgpr2 = COPY %2
+...
+---
+name: test_insert_v4s32_s32_offset0
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4
+
+    ; CHECK-LABEL: name: test_insert_v4s32_s32_offset0
+    ; CHECK: [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr4
+    ; CHECK: [[INSERT:%[0-9]+]]:_(<4 x s32>) = G_INSERT [[COPY]], [[COPY1]](s32), 0
+    ; CHECK: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[INSERT]](<4 x s32>)
+    %0:_(<4 x s32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    %1:_(s32) = COPY $vgpr4
+    %2:_(<4 x s32>) = G_INSERT %0, %1, 0
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %2
+...
+---
+name: test_insert_v4s32_s32_offset32
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4
+
+    ; CHECK-LABEL: name: test_insert_v4s32_s32_offset32
+    ; CHECK: [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr4
+    ; CHECK: [[INSERT:%[0-9]+]]:_(<4 x s32>) = G_INSERT [[COPY]], [[COPY1]](s32), 32
+    ; CHECK: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[INSERT]](<4 x s32>)
+    %0:_(<4 x s32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    %1:_(s32) = COPY $vgpr4
+    %2:_(<4 x s32>) = G_INSERT %0, %1, 32
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %2
+...
+---
+name: test_insert_v4s32_s32_offset64
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4
+
+    ; CHECK-LABEL: name: test_insert_v4s32_s32_offset64
+    ; CHECK: [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr4
+    ; CHECK: [[INSERT:%[0-9]+]]:_(<4 x s32>) = G_INSERT [[COPY]], [[COPY1]](s32), 64
+    ; CHECK: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[INSERT]](<4 x s32>)
+    %0:_(<4 x s32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    %1:_(s32) = COPY $vgpr4
+    %2:_(<4 x s32>) = G_INSERT %0, %1, 64
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %2
+...
+---
+name: test_insert_v4s32_s32_offset96
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4
+
+    ; CHECK-LABEL: name: test_insert_v4s32_s32_offset96
+    ; CHECK: [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr4
+    ; CHECK: [[INSERT:%[0-9]+]]:_(<4 x s32>) = G_INSERT [[COPY]], [[COPY1]](s32), 96
+    ; CHECK: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[INSERT]](<4 x s32>)
+    %0:_(<4 x s32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    %1:_(s32) = COPY $vgpr4
+    %2:_(<4 x s32>) = G_INSERT %0, %1, 96
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %2
+...
+---
+name: test_insert_v4s32_s64_offset0
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5
+
+    ; CHECK-LABEL: name: test_insert_v4s32_s64_offset0
+    ; CHECK: [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s64) = COPY $vgpr4_vgpr5
+    ; CHECK: [[INSERT:%[0-9]+]]:_(<4 x s32>) = G_INSERT [[COPY]], [[COPY1]](s64), 0
+    ; CHECK: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[INSERT]](<4 x s32>)
+    %0:_(<4 x s32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    %1:_(s64) = COPY $vgpr4_vgpr5
+    %2:_(<4 x s32>) = G_INSERT %0, %1, 0
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %2
+...
+---
+name: test_insert_v4s32_s64_offset32
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5
+
+    ; CHECK-LABEL: name: test_insert_v4s32_s64_offset32
+    ; CHECK: [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s64) = COPY $vgpr4_vgpr5
+    ; CHECK: [[INSERT:%[0-9]+]]:_(<4 x s32>) = G_INSERT [[COPY]], [[COPY1]](s64), 32
+    ; CHECK: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[INSERT]](<4 x s32>)
+    %0:_(<4 x s32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    %1:_(s64) = COPY $vgpr4_vgpr5
+    %2:_(<4 x s32>) = G_INSERT %0, %1, 32
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %2
+...
+---
+name: test_insert_v4s32_s64_offset64
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5
+
+    ; CHECK-LABEL: name: test_insert_v4s32_s64_offset64
+    ; CHECK: [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s64) = COPY $vgpr4_vgpr5
+    ; CHECK: [[INSERT:%[0-9]+]]:_(<4 x s32>) = G_INSERT [[COPY]], [[COPY1]](s64), 64
+    ; CHECK: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[INSERT]](<4 x s32>)
+    %0:_(<4 x s32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    %1:_(s64) = COPY $vgpr4_vgpr5
+    %2:_(<4 x s32>) = G_INSERT %0, %1, 64
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %2
+...
+---
+name: test_insert_v4s32_s96_offset0
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6
+
+    ; CHECK-LABEL: name: test_insert_v4s32_s96_offset0
+    ; CHECK: [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s96) = COPY $vgpr4_vgpr5_vgpr6
+    ; CHECK: [[INSERT:%[0-9]+]]:_(<4 x s32>) = G_INSERT [[COPY]], [[COPY1]](s96), 0
+    ; CHECK: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[INSERT]](<4 x s32>)
+    %0:_(<4 x s32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    %1:_(s96) = COPY $vgpr4_vgpr5_vgpr6
+    %2:_(<4 x s32>) = G_INSERT %0, %1, 0
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %2
+...
+---
+name: test_insert_v4s32_s96_offset32
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6
+
+    ; CHECK-LABEL: name: test_insert_v4s32_s96_offset32
+    ; CHECK: [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s96) = COPY $vgpr4_vgpr5_vgpr6
+    ; CHECK: [[INSERT:%[0-9]+]]:_(<4 x s32>) = G_INSERT [[COPY]], [[COPY1]](s96), 32
+    ; CHECK: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[INSERT]](<4 x s32>)
+    %0:_(<4 x s32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    %1:_(s96) = COPY $vgpr4_vgpr5_vgpr6
+    %2:_(<4 x s32>) = G_INSERT %0, %1, 32
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %2
+...
+---
+name: test_insert_v4s32_v2s32_offset0
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5
+
+    ; CHECK-LABEL: name: test_insert_v4s32_v2s32_offset0
+    ; CHECK: [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; CHECK: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr4_vgpr5
+    ; CHECK: [[INSERT:%[0-9]+]]:_(<4 x s32>) = G_INSERT [[COPY]], [[COPY1]](<2 x s32>), 0
+    ; CHECK: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[INSERT]](<4 x s32>)
+    %0:_(<4 x s32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    %1:_(<2 x s32>) = COPY $vgpr4_vgpr5
+    %2:_(<4 x s32>) = G_INSERT %0, %1, 0
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %2
+...
+---
+name: test_insert_v4s32_v2s32_offset32
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5
+
+    ; CHECK-LABEL: name: test_insert_v4s32_v2s32_offset32
+    ; CHECK: [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; CHECK: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr4_vgpr5
+    ; CHECK: [[INSERT:%[0-9]+]]:_(<4 x s32>) = G_INSERT [[COPY]], [[COPY1]](<2 x s32>), 32
+    ; CHECK: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[INSERT]](<4 x s32>)
+    %0:_(<4 x s32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    %1:_(<2 x s32>) = COPY $vgpr4_vgpr5
+    %2:_(<4 x s32>) = G_INSERT %0, %1, 32
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %2
+...
+---
+name: test_insert_v4s32_v2s32_offset64
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5
+
+    ; CHECK-LABEL: name: test_insert_v4s32_v2s32_offset64
+    ; CHECK: [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; CHECK: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr4_vgpr5
+    ; CHECK: [[INSERT:%[0-9]+]]:_(<4 x s32>) = G_INSERT [[COPY]], [[COPY1]](<2 x s32>), 64
+    ; CHECK: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[INSERT]](<4 x s32>)
+    %0:_(<4 x s32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    %1:_(<2 x s32>) = COPY $vgpr4_vgpr5
+    %2:_(<4 x s32>) = G_INSERT %0, %1, 64
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %2
+...
+---
+name: test_insert_v4s32_v3s32_offset0
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6
+
+    ; CHECK-LABEL: name: test_insert_v4s32_v3s32_offset0
+    ; CHECK: [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; CHECK: [[COPY1:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr4_vgpr5_vgpr6
+    ; CHECK: [[INSERT:%[0-9]+]]:_(<4 x s32>) = G_INSERT [[COPY]], [[COPY1]](<3 x s32>), 0
+    ; CHECK: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[INSERT]](<4 x s32>)
+    %0:_(<4 x s32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    %1:_(<3 x s32>) = COPY $vgpr4_vgpr5_vgpr6
+    %2:_(<4 x s32>) = G_INSERT %0, %1, 0
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %2
+...
+---
+name: test_insert_v4s32_v3s32_offset32
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6
+
+    ; CHECK-LABEL: name: test_insert_v4s32_v3s32_offset32
+    ; CHECK: [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; CHECK: [[COPY1:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr4_vgpr5_vgpr6
+    ; CHECK: [[INSERT:%[0-9]+]]:_(<4 x s32>) = G_INSERT [[COPY]], [[COPY1]](<3 x s32>), 32
+    ; CHECK: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[INSERT]](<4 x s32>)
+    %0:_(<4 x s32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    %1:_(<3 x s32>) = COPY $vgpr4_vgpr5_vgpr6
+    %2:_(<4 x s32>) = G_INSERT %0, %1, 32
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %2
+...
+---
+name: test_insert_v4s32_p0_offset0
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5
+
+    ; CHECK-LABEL: name: test_insert_v4s32_p0_offset0
+    ; CHECK: [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; CHECK: [[COPY1:%[0-9]+]]:_(p0) = COPY $vgpr4_vgpr5
+    ; CHECK: [[INSERT:%[0-9]+]]:_(<4 x s32>) = G_INSERT [[COPY]], [[COPY1]](p0), 0
+    ; CHECK: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[INSERT]](<4 x s32>)
+    %0:_(<4 x s32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    %1:_(p0) = COPY $vgpr4_vgpr5
+    %2:_(<4 x s32>) = G_INSERT %0, %1, 0
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %2
+...
+---
+name: test_insert_v4s32_p0_offset32
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5
+
+    ; CHECK-LABEL: name: test_insert_v4s32_p0_offset32
+    ; CHECK: [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; CHECK: [[COPY1:%[0-9]+]]:_(p0) = COPY $vgpr4_vgpr5
+    ; CHECK: [[INSERT:%[0-9]+]]:_(<4 x s32>) = G_INSERT [[COPY]], [[COPY1]](p0), 32
+    ; CHECK: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[INSERT]](<4 x s32>)
+    %0:_(<4 x s32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    %1:_(p0) = COPY $vgpr4_vgpr5
+    %2:_(<4 x s32>) = G_INSERT %0, %1, 32
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %2
+...
+---
+name: test_insert_v4s32_p0_offset64
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5
+
+    ; CHECK-LABEL: name: test_insert_v4s32_p0_offset64
+    ; CHECK: [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; CHECK: [[COPY1:%[0-9]+]]:_(p0) = COPY $vgpr4_vgpr5
+    ; CHECK: [[INSERT:%[0-9]+]]:_(<4 x s32>) = G_INSERT [[COPY]], [[COPY1]](p0), 64
+    ; CHECK: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[INSERT]](<4 x s32>)
+    %0:_(<4 x s32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    %1:_(p0) = COPY $vgpr4_vgpr5
+    %2:_(<4 x s32>) = G_INSERT %0, %1, 64
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %2
 ...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/legalize-intrinsic-round.mir b/test/CodeGen/AMDGPU/GlobalISel/legalize-intrinsic-round.mir
new file mode 100644
index 0000000000000000000000000000000000000000..220ce911addacfb8bb9b334f2bcab1e85a4ac9b2
--- /dev/null
+++ b/test/CodeGen/AMDGPU/GlobalISel/legalize-intrinsic-round.mir
@@ -0,0 +1,67 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -run-pass=legalizer %s -o - | FileCheck %s
+
+---
+name: test_intrinsic_round_s32
+body: |
+  bb.0:
+    liveins: $vgpr0
+
+    ; CHECK-LABEL: name: test_intrinsic_round_s32
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: $vgpr0 = COPY [[COPY]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = G_INTRINSIC_ROUND %0
+    $vgpr0 = COPY %0
+...
+
+---
+name: test_intrinsic_round_s64
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; CHECK-LABEL: name: test_intrinsic_round_s64
+    ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
+    ; CHECK: [[INTRINSIC_ROUND:%[0-9]+]]:_(s64) = G_INTRINSIC_ROUND [[COPY]]
+    ; CHECK: $vgpr0_vgpr1 = COPY [[INTRINSIC_ROUND]](s64)
+    %0:_(s64) = COPY $vgpr0_vgpr1
+    %1:_(s64) = G_INTRINSIC_ROUND %0
+    $vgpr0_vgpr1 = COPY %1
+...
+
+---
+name: test_intrinsic_round_v2s32
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; CHECK-LABEL: name: test_intrinsic_round_v2s32
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    ; CHECK: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
+    ; CHECK: [[INTRINSIC_ROUND:%[0-9]+]]:_(s32) = G_INTRINSIC_ROUND [[UV]]
+    ; CHECK: [[INTRINSIC_ROUND1:%[0-9]+]]:_(s32) = G_INTRINSIC_ROUND [[UV1]]
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[INTRINSIC_ROUND]](s32), [[INTRINSIC_ROUND1]](s32)
+    ; CHECK: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
+    %0:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    %1:_(<2 x s32>) = G_INTRINSIC_ROUND %0
+    $vgpr0_vgpr1 = COPY %1
+...
+
+---
+name: test_intrinsic_round_v2s64
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3
+
+    ; CHECK-LABEL: name: test_intrinsic_round_v2s64
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; CHECK: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](<2 x s64>)
+    ; CHECK: [[INTRINSIC_ROUND:%[0-9]+]]:_(s64) = G_INTRINSIC_ROUND [[UV]]
+    ; CHECK: [[INTRINSIC_ROUND1:%[0-9]+]]:_(s64) = G_INTRINSIC_ROUND [[UV1]]
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[INTRINSIC_ROUND]](s64), [[INTRINSIC_ROUND1]](s64)
+    ; CHECK: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>)
+    %0:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    %1:_(<2 x s64>) = G_INTRINSIC_ROUND %0
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %1
+...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/legalize-intrinsic-trunc.mir b/test/CodeGen/AMDGPU/GlobalISel/legalize-intrinsic-trunc.mir
new file mode 100644
index 0000000000000000000000000000000000000000..12936ec01dd533ed17e5340f360c85cb5bcd351b
--- /dev/null
+++ b/test/CodeGen/AMDGPU/GlobalISel/legalize-intrinsic-trunc.mir
@@ -0,0 +1,67 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -run-pass=legalizer %s -o - | FileCheck %s
+
+---
+name: test_intrinsic_trunc_s32
+body: |
+  bb.0:
+    liveins: $vgpr0
+
+    ; CHECK-LABEL: name: test_intrinsic_trunc_s32
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: $vgpr0 = COPY [[COPY]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = G_INTRINSIC_TRUNC %0
+    $vgpr0 = COPY %0
+...
+
+---
+name: test_intrinsic_trunc_s64
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; CHECK-LABEL: name: test_intrinsic_trunc_s64
+    ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
+    ; CHECK: [[INTRINSIC_TRUNC:%[0-9]+]]:_(s64) = G_INTRINSIC_TRUNC [[COPY]]
+    ; CHECK: $vgpr0_vgpr1 = COPY [[INTRINSIC_TRUNC]](s64)
+    %0:_(s64) = COPY $vgpr0_vgpr1
+    %1:_(s64) = G_INTRINSIC_TRUNC %0
+    $vgpr0_vgpr1 = COPY %1
+...
+
+---
+name: test_intrinsic_trunc_v2s32
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; CHECK-LABEL: name: test_intrinsic_trunc_v2s32
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    ; CHECK: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
+    ; CHECK: [[INTRINSIC_TRUNC:%[0-9]+]]:_(s32) = G_INTRINSIC_TRUNC [[UV]]
+    ; CHECK: [[INTRINSIC_TRUNC1:%[0-9]+]]:_(s32) = G_INTRINSIC_TRUNC [[UV1]]
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[INTRINSIC_TRUNC]](s32), [[INTRINSIC_TRUNC1]](s32)
+    ; CHECK: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
+    %0:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    %1:_(<2 x s32>) = G_INTRINSIC_TRUNC %0
+    $vgpr0_vgpr1 = COPY %1
+...
+
+---
+name: test_intrinsic_trunc_v2s64
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3
+
+    ; CHECK-LABEL: name: test_intrinsic_trunc_v2s64
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; CHECK: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](<2 x s64>)
+    ; CHECK: [[INTRINSIC_TRUNC:%[0-9]+]]:_(s64) = G_INTRINSIC_TRUNC [[UV]]
+    ; CHECK: [[INTRINSIC_TRUNC1:%[0-9]+]]:_(s64) = G_INTRINSIC_TRUNC [[UV1]]
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[INTRINSIC_TRUNC]](s64), [[INTRINSIC_TRUNC1]](s64)
+    ; CHECK: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>)
+    %0:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    %1:_(<2 x s64>) = G_INTRINSIC_TRUNC %0
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %1
+...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/legalize-inttoptr.mir b/test/CodeGen/AMDGPU/GlobalISel/legalize-inttoptr.mir
index a1fd6c60e57473ac06a47ac98ac6e2142a00b112..87c72f1154c0f4e8a6b64c3834c7b893a27eef27 100644
--- a/test/CodeGen/AMDGPU/GlobalISel/legalize-inttoptr.mir
+++ b/test/CodeGen/AMDGPU/GlobalISel/legalize-inttoptr.mir
@@ -1,29 +1,162 @@
-# RUN: llc -march=amdgcn -run-pass=legalizer %s -o - | FileCheck %s
-
----
-name: test_inttoptr
-body: |
-  bb.0:
-    liveins: $sgpr0_sgpr1, $sgpr2, $vgpr0_vgpr1, $vgpr2
-
-    ; CHECK-LABEL: name: test_inttoptr
-    ; CHECK: [[S64:%[0-9]+]]:_(s64) = COPY $sgpr0_sgpr1
-    ; CHECK: [[S32:%[0-9]+]]:_(s32) = COPY $sgpr2
-    ; CHECK: [[V64:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
-    ; CHECK: [[V32:%[0-9]+]]:_(s32) = COPY $vgpr2
-    ; CHECK: (p0) = G_INTTOPTR [[V64]]
-    ; CHECK: (p1) = G_INTTOPTR [[V64]]
-    ; CHECK: (p3) = G_INTTOPTR [[V32]]
-    ; CHECK: (p4) = G_INTTOPTR [[S64]]
-    ; CHECK: (p5) = G_INTTOPTR [[S32]]
-    %0:_(s64) = COPY $sgpr0_sgpr1
-    %1:_(s32) = COPY $sgpr2
-    %2:_(s64) = COPY $vgpr0_vgpr1
-    %3:_(s32) = COPY $vgpr2
-    %4:_(p0) = G_INTTOPTR %2
-    %5:_(p1) = G_INTTOPTR %2
-    %6:_(p3) = G_INTTOPTR %3
-    %7:_(p4) = G_INTTOPTR %0
-    %8:_(p5) = G_INTTOPTR %1
-    S_ENDPGM implicit %4, implicit %5, implicit %6, implicit %7, implicit %8
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -march=amdgcn -O0 -run-pass=legalizer -global-isel-abort=0 -o - %s | FileCheck %s
+
+---
+name: test_inttoptr_s64_to_p0
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; CHECK-LABEL: name: test_inttoptr_s64_to_p0
+    ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
+    ; CHECK: [[INTTOPTR:%[0-9]+]]:_(p0) = G_INTTOPTR [[COPY]](s64)
+    ; CHECK: $vgpr0_vgpr1 = COPY [[INTTOPTR]](p0)
+    %0:_(s64) = COPY $vgpr0_vgpr1
+    %1:_(p0) = G_INTTOPTR %0
+    $vgpr0_vgpr1 = COPY %1
+...
+
+---
+name: test_inttoptr_s64_to_p1
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; CHECK-LABEL: name: test_inttoptr_s64_to_p1
+    ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
+    ; CHECK: [[INTTOPTR:%[0-9]+]]:_(p1) = G_INTTOPTR [[COPY]](s64)
+    ; CHECK: $vgpr0_vgpr1 = COPY [[INTTOPTR]](p1)
+    %0:_(s64) = COPY $vgpr0_vgpr1
+    %1:_(p1) = G_INTTOPTR %0
+    $vgpr0_vgpr1 = COPY %1
+...
+
+---
+name: test_inttoptr_s64_to_p4
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; CHECK-LABEL: name: test_inttoptr_s64_to_p4
+    ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
+    ; CHECK: [[INTTOPTR:%[0-9]+]]:_(p4) = G_INTTOPTR [[COPY]](s64)
+    ; CHECK: $vgpr0_vgpr1 = COPY [[INTTOPTR]](p4)
+    %0:_(s64) = COPY $vgpr0_vgpr1
+    %1:_(p4) = G_INTTOPTR %0
+    $vgpr0_vgpr1 = COPY %1
+...
+
+---
+name: test_inttoptr_s32_to_p3
+body: |
+  bb.0:
+    liveins: $vgpr0
+
+    ; CHECK-LABEL: name: test_inttoptr_s32_to_p3
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[INTTOPTR:%[0-9]+]]:_(p3) = G_INTTOPTR [[COPY]](s32)
+    ; CHECK: $vgpr0 = COPY [[INTTOPTR]](p3)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(p3) = G_INTTOPTR %0
+    $vgpr0 = COPY %1
+...
+
+---
+name: test_inttoptr_s32_to_p5
+body: |
+  bb.0:
+    liveins: $vgpr0
+
+    ; CHECK-LABEL: name: test_inttoptr_s32_to_p5
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[INTTOPTR:%[0-9]+]]:_(p5) = G_INTTOPTR [[COPY]](s32)
+    ; CHECK: $vgpr0 = COPY [[INTTOPTR]](p5)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(p5) = G_INTTOPTR %0
+    $vgpr0 = COPY %1
+...
+
+---
+name: test_inttoptr_s64_to_p999
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; CHECK-LABEL: name: test_inttoptr_s64_to_p999
+    ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
+    ; CHECK: [[INTTOPTR:%[0-9]+]]:_(p999) = G_INTTOPTR [[COPY]](s64)
+    ; CHECK: $vgpr0_vgpr1 = COPY [[INTTOPTR]](p999)
+    %0:_(s64) = COPY $vgpr0_vgpr1
+    %1:_(p999) = G_INTTOPTR %0
+    $vgpr0_vgpr1 = COPY %1
+...
+
+---
+name: test_inttoptr_s32_to_p0
+body: |
+  bb.0:
+    liveins: $vgpr0
+
+    ; CHECK-LABEL: name: test_inttoptr_s32_to_p0
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[COPY]](s32)
+    ; CHECK: [[INTTOPTR:%[0-9]+]]:_(p0) = G_INTTOPTR [[ZEXT]](s64)
+    ; CHECK: $vgpr0_vgpr1 = COPY [[INTTOPTR]](p0)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(p0) = G_INTTOPTR %0
+    $vgpr0_vgpr1 = COPY %1
+...
+
+---
+name: test_inttoptr_s128_to_p0
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3
+
+    ; CHECK-LABEL: name: test_inttoptr_s128_to_p0
+    ; CHECK: [[COPY:%[0-9]+]]:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; CHECK: [[TRUNC:%[0-9]+]]:_(s64) = G_TRUNC [[COPY]](s128)
+    ; CHECK: [[INTTOPTR:%[0-9]+]]:_(p0) = G_INTTOPTR [[TRUNC]](s64)
+    ; CHECK: $vgpr0_vgpr1 = COPY [[INTTOPTR]](p0)
+    %0:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    %1:_(p0) = G_INTTOPTR %0
+    $vgpr0_vgpr1 = COPY %1
+...
+
+---
+name: test_inttoptr_v2s64_to_v2p0
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3
+
+    ; CHECK-LABEL: name: test_inttoptr_v2s64_to_v2p0
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; CHECK: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](<2 x s64>)
+    ; CHECK: [[INTTOPTR:%[0-9]+]]:_(p0) = G_INTTOPTR [[UV]](s64)
+    ; CHECK: [[INTTOPTR1:%[0-9]+]]:_(p0) = G_INTTOPTR [[UV1]](s64)
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x p0>) = G_BUILD_VECTOR [[INTTOPTR]](p0), [[INTTOPTR1]](p0)
+    ; CHECK: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x p0>)
+    %0:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    %1:_(<2 x p0>) = G_INTTOPTR %0
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %1
+...
+
+---
+name: test_inttoptr_v2s32_to_v2p0
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; CHECK-LABEL: name: test_inttoptr_v2s32_to_v2p0
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    ; CHECK: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
+    ; CHECK: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[UV]](s32)
+    ; CHECK: [[INTTOPTR:%[0-9]+]]:_(p0) = G_INTTOPTR [[ZEXT]](s64)
+    ; CHECK: [[ZEXT1:%[0-9]+]]:_(s64) = G_ZEXT [[UV1]](s32)
+    ; CHECK: [[INTTOPTR1:%[0-9]+]]:_(p0) = G_INTTOPTR [[ZEXT1]](s64)
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x p0>) = G_BUILD_VECTOR [[INTTOPTR]](p0), [[INTTOPTR1]](p0)
+    ; CHECK: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x p0>)
+    %0:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    %1:_(<2 x p0>) = G_INTTOPTR %0
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %1
 ...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/legalize-load.mir b/test/CodeGen/AMDGPU/GlobalISel/legalize-load.mir
index cf25192ca6ac974795bf923f1356e028dc71c6a4..bfccdb177570673c38592eab1749f1282b2398f6 100644
--- a/test/CodeGen/AMDGPU/GlobalISel/legalize-load.mir
+++ b/test/CodeGen/AMDGPU/GlobalISel/legalize-load.mir
@@ -1,5 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -O0 -run-pass=legalizer %s -o - | FileCheck %s
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=tahiti -O0 -run-pass=legalizer %s -o - | FileCheck -check-prefix=SI %s
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -O0 -run-pass=legalizer %s -o - | FileCheck -check-prefix=VI %s
 
 ---
 name: test_load_global_i32
@@ -7,13 +8,16 @@ body: |
   bb.0:
     liveins: $vgpr0_vgpr1
 
-    ; CHECK-LABEL: name: test_load_global_i32
-    ; CHECK: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
-    ; CHECK: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p1) :: (load 4, addrspace 1)
-    ; CHECK: $vgpr0 = COPY [[LOAD]](s32)
+    ; SI-LABEL: name: test_load_global_i32
+    ; SI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
+    ; SI: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p1) :: (load 4, addrspace 1)
+    ; SI: $vgpr0 = COPY [[LOAD]](s32)
+    ; VI-LABEL: name: test_load_global_i32
+    ; VI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
+    ; VI: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p1) :: (load 4, addrspace 1)
+    ; VI: $vgpr0 = COPY [[LOAD]](s32)
     %0:_(p1) = COPY $vgpr0_vgpr1
     %1:_(s32) = G_LOAD %0 :: (load 4, addrspace 1)
-
     $vgpr0 = COPY %1
 ...
 
@@ -23,13 +27,16 @@ body: |
   bb.0:
     liveins: $vgpr0_vgpr1
 
-    ; CHECK-LABEL: name: test_load_global_i64
-    ; CHECK: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
-    ; CHECK: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p1) :: (load 4, addrspace 1)
-    ; CHECK: $vgpr0 = COPY [[LOAD]](s32)
+    ; SI-LABEL: name: test_load_global_i64
+    ; SI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
+    ; SI: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p1) :: (load 4, addrspace 1)
+    ; SI: $vgpr0 = COPY [[LOAD]](s32)
+    ; VI-LABEL: name: test_load_global_i64
+    ; VI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
+    ; VI: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p1) :: (load 4, addrspace 1)
+    ; VI: $vgpr0 = COPY [[LOAD]](s32)
     %0:_(p1) = COPY $vgpr0_vgpr1
     %1:_(s32) = G_LOAD %0 :: (load 4, addrspace 1)
-
     $vgpr0 = COPY %1
 ...
 
@@ -39,13 +46,16 @@ body: |
   bb.0:
     liveins: $vgpr0_vgpr1
 
-    ; CHECK-LABEL: name: test_load_global_p1
-    ; CHECK: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
-    ; CHECK: [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[COPY]](p1) :: (load 8, addrspace 1)
-    ; CHECK: $vgpr0_vgpr1 = COPY [[LOAD]](p1)
+    ; SI-LABEL: name: test_load_global_p1
+    ; SI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
+    ; SI: [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[COPY]](p1) :: (load 8, addrspace 1)
+    ; SI: $vgpr0_vgpr1 = COPY [[LOAD]](p1)
+    ; VI-LABEL: name: test_load_global_p1
+    ; VI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
+    ; VI: [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[COPY]](p1) :: (load 8, addrspace 1)
+    ; VI: $vgpr0_vgpr1 = COPY [[LOAD]](p1)
     %0:_(p1) = COPY $vgpr0_vgpr1
     %1:_(p1) = G_LOAD %0 :: (load 8, addrspace 1)
-
     $vgpr0_vgpr1 = COPY %1
 ...
 
@@ -55,30 +65,35 @@ body: |
   bb.0:
     liveins: $vgpr0_vgpr1
 
-    ; CHECK-LABEL: name: test_load_global_p4
-    ; CHECK: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
-    ; CHECK: [[LOAD:%[0-9]+]]:_(p4) = G_LOAD [[COPY]](p1) :: (load 8, addrspace 1)
-    ; CHECK: $vgpr0_vgpr1 = COPY [[LOAD]](p4)
+    ; SI-LABEL: name: test_load_global_p4
+    ; SI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
+    ; SI: [[LOAD:%[0-9]+]]:_(p4) = G_LOAD [[COPY]](p1) :: (load 8, addrspace 1)
+    ; SI: $vgpr0_vgpr1 = COPY [[LOAD]](p4)
+    ; VI-LABEL: name: test_load_global_p4
+    ; VI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
+    ; VI: [[LOAD:%[0-9]+]]:_(p4) = G_LOAD [[COPY]](p1) :: (load 8, addrspace 1)
+    ; VI: $vgpr0_vgpr1 = COPY [[LOAD]](p4)
     %0:_(p1) = COPY $vgpr0_vgpr1
     %1:_(p4) = G_LOAD %0 :: (load 8, addrspace 1)
-
     $vgpr0_vgpr1 = COPY %1
 ...
 
-
 ---
 name: test_load_global_p3
 body: |
   bb.0:
     liveins: $vgpr0_vgpr1
 
-    ; CHECK-LABEL: name: test_load_global_p3
-    ; CHECK: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
-    ; CHECK: [[LOAD:%[0-9]+]]:_(p3) = G_LOAD [[COPY]](p1) :: (load 4, addrspace 1)
-    ; CHECK: $vgpr0 = COPY [[LOAD]](p3)
+    ; SI-LABEL: name: test_load_global_p3
+    ; SI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
+    ; SI: [[LOAD:%[0-9]+]]:_(p3) = G_LOAD [[COPY]](p1) :: (load 4, addrspace 1)
+    ; SI: $vgpr0 = COPY [[LOAD]](p3)
+    ; VI-LABEL: name: test_load_global_p3
+    ; VI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
+    ; VI: [[LOAD:%[0-9]+]]:_(p3) = G_LOAD [[COPY]](p1) :: (load 4, addrspace 1)
+    ; VI: $vgpr0 = COPY [[LOAD]](p3)
     %0:_(p1) = COPY $vgpr0_vgpr1
     %1:_(p3) = G_LOAD %0 :: (load 4, addrspace 1)
-
     $vgpr0 = COPY %1
 ...
 
@@ -88,13 +103,16 @@ body: |
   bb.0:
     liveins: $vgpr0_vgpr1
 
-    ; CHECK-LABEL: name: test_load_global_v2s32
-    ; CHECK: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
-    ; CHECK: [[LOAD:%[0-9]+]]:_(<2 x s32>) = G_LOAD [[COPY]](p1) :: (load 8, addrspace 1)
-    ; CHECK: $vgpr0_vgpr1 = COPY [[LOAD]](<2 x s32>)
+    ; SI-LABEL: name: test_load_global_v2s32
+    ; SI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
+    ; SI: [[LOAD:%[0-9]+]]:_(<2 x s32>) = G_LOAD [[COPY]](p1) :: (load 8, addrspace 1)
+    ; SI: $vgpr0_vgpr1 = COPY [[LOAD]](<2 x s32>)
+    ; VI-LABEL: name: test_load_global_v2s32
+    ; VI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
+    ; VI: [[LOAD:%[0-9]+]]:_(<2 x s32>) = G_LOAD [[COPY]](p1) :: (load 8, addrspace 1)
+    ; VI: $vgpr0_vgpr1 = COPY [[LOAD]](<2 x s32>)
     %0:_(p1) = COPY $vgpr0_vgpr1
     %1:_(<2 x s32>) = G_LOAD %0 :: (load 8, addrspace 1)
-
     $vgpr0_vgpr1 = COPY %1
 ...
 
@@ -105,10 +123,14 @@ body: |
   bb.0:
     liveins: $vgpr0_vgpr1
 
-    ; CHECK-LABEL: name: test_load_global_v2s16
-    ; CHECK: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
-    ; CHECK: [[LOAD:%[0-9]+]]:_(<2 x s16>) = G_LOAD [[COPY]](p1) :: (load 4, addrspace 1)
-    ; CHECK: $vgpr0 = COPY [[LOAD]](<2 x s16>)
+    ; SI-LABEL: name: test_load_global_v2s16
+    ; SI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
+    ; SI: [[LOAD:%[0-9]+]]:_(<2 x s16>) = G_LOAD [[COPY]](p1) :: (load 4, addrspace 1)
+    ; SI: $vgpr0 = COPY [[LOAD]](<2 x s16>)
+    ; VI-LABEL: name: test_load_global_v2s16
+    ; VI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
+    ; VI: [[LOAD:%[0-9]+]]:_(<2 x s16>) = G_LOAD [[COPY]](p1) :: (load 4, addrspace 1)
+    ; VI: $vgpr0 = COPY [[LOAD]](<2 x s16>)
     %0:_(p1) = COPY $vgpr0_vgpr1
     %1:_(<2 x s16>) = G_LOAD %0 :: (load 4, addrspace 1)
     $vgpr0 = COPY %1
@@ -120,12 +142,251 @@ body: |
   bb.0:
     liveins: $vgpr0_vgpr1
 
-    ; CHECK-LABEL: name: test_load_global_v3i32
-    ; CHECK: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
-    ; CHECK: [[LOAD:%[0-9]+]]:_(<3 x s32>) = G_LOAD [[COPY]](p1) :: (load 12, align 4, addrspace 1)
-    ; CHECK: $vgpr0_vgpr1_vgpr2 = COPY [[LOAD]](<3 x s32>)
+    ; SI-LABEL: name: test_load_global_v3i32
+    ; SI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
+    ; SI: [[LOAD:%[0-9]+]]:_(<2 x s32>) = G_LOAD [[COPY]](p1) :: (load 8, align 4, addrspace 1)
+    ; SI: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
+    ; SI: [[GEP:%[0-9]+]]:_(p1) = G_GEP [[COPY]], [[C]](s64)
+    ; SI: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[GEP]](p1) :: (load 4, addrspace 1)
+    ; SI: [[DEF:%[0-9]+]]:_(<3 x s32>) = G_IMPLICIT_DEF
+    ; SI: [[INSERT:%[0-9]+]]:_(<3 x s32>) = G_INSERT [[DEF]], [[LOAD]](<2 x s32>), 0
+    ; SI: [[INSERT1:%[0-9]+]]:_(<3 x s32>) = G_INSERT [[INSERT]], [[LOAD1]](s32), 64
+    ; SI: $vgpr0_vgpr1_vgpr2 = COPY [[INSERT1]](<3 x s32>)
+    ; VI-LABEL: name: test_load_global_v3i32
+    ; VI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
+    ; VI: [[LOAD:%[0-9]+]]:_(<3 x s32>) = G_LOAD [[COPY]](p1) :: (load 12, align 4, addrspace 1)
+    ; VI: $vgpr0_vgpr1_vgpr2 = COPY [[LOAD]](<3 x s32>)
     %0:_(p1) = COPY $vgpr0_vgpr1
     %1:_(<3 x s32>) = G_LOAD %0 :: (load 12, align 4, addrspace 1)
+    $vgpr0_vgpr1_vgpr2 = COPY %1
+...
+
+---
+name: test_ext_load_global_s64_from_1_align1
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; SI-LABEL: name: test_ext_load_global_s64_from_1_align1
+    ; SI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
+    ; SI: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p1) :: (load 1, align 4, addrspace 1)
+    ; SI: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[LOAD]](s32)
+    ; SI: $vgpr0_vgpr1 = COPY [[ANYEXT]](s64)
+    ; VI-LABEL: name: test_ext_load_global_s64_from_1_align1
+    ; VI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
+    ; VI: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p1) :: (load 1, align 4, addrspace 1)
+    ; VI: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[LOAD]](s32)
+    ; VI: $vgpr0_vgpr1 = COPY [[ANYEXT]](s64)
+    %0:_(p1) = COPY $vgpr0_vgpr1
+    %1:_(s64) = G_LOAD %0 :: (load 1, addrspace 1, align 4)
+    $vgpr0_vgpr1 = COPY %1
+...
+
+---
+name: test_ext_load_global_s64_from_2_align2
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; SI-LABEL: name: test_ext_load_global_s64_from_2_align2
+    ; SI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
+    ; SI: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p1) :: (load 2, align 4, addrspace 1)
+    ; SI: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[LOAD]](s32)
+    ; SI: $vgpr0_vgpr1 = COPY [[ANYEXT]](s64)
+    ; VI-LABEL: name: test_ext_load_global_s64_from_2_align2
+    ; VI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
+    ; VI: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p1) :: (load 2, align 4, addrspace 1)
+    ; VI: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[LOAD]](s32)
+    ; VI: $vgpr0_vgpr1 = COPY [[ANYEXT]](s64)
+    %0:_(p1) = COPY $vgpr0_vgpr1
+    %1:_(s64) = G_LOAD %0 :: (load 2, addrspace 1, align 4)
+    $vgpr0_vgpr1 = COPY %1
+...
+
+---
+name: test_ext_load_global_s64_from_4_align4
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; SI-LABEL: name: test_ext_load_global_s64_from_4_align4
+    ; SI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
+    ; SI: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p1) :: (load 4, addrspace 1)
+    ; SI: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[LOAD]](s32)
+    ; SI: $vgpr0_vgpr1 = COPY [[ANYEXT]](s64)
+    ; VI-LABEL: name: test_ext_load_global_s64_from_4_align4
+    ; VI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
+    ; VI: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p1) :: (load 4, addrspace 1)
+    ; VI: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[LOAD]](s32)
+    ; VI: $vgpr0_vgpr1 = COPY [[ANYEXT]](s64)
+    %0:_(p1) = COPY $vgpr0_vgpr1
+    %1:_(s64) = G_LOAD %0 :: (load 4, addrspace 1, align 4)
+    $vgpr0_vgpr1 = COPY %1
+...
 
+---
+name: test_ext_load_global_s128_from_4_align4
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; SI-LABEL: name: test_ext_load_global_s128_from_4_align4
+    ; SI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
+    ; SI: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p1) :: (load 4, addrspace 1)
+    ; SI: [[ANYEXT:%[0-9]+]]:_(s128) = G_ANYEXT [[LOAD]](s32)
+    ; SI: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[ANYEXT]](s128)
+    ; VI-LABEL: name: test_ext_load_global_s128_from_4_align4
+    ; VI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
+    ; VI: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p1) :: (load 4, addrspace 1)
+    ; VI: [[ANYEXT:%[0-9]+]]:_(s128) = G_ANYEXT [[LOAD]](s32)
+    ; VI: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[ANYEXT]](s128)
+    %0:_(p1) = COPY $vgpr0_vgpr1
+    %1:_(s128) = G_LOAD %0 :: (load 4, addrspace 1, align 4)
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %1
+
+...
+
+---
+name: test_load_global_s96_align4
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; SI-LABEL: name: test_load_global_s96_align4
+    ; SI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
+    ; SI: [[LOAD:%[0-9]+]]:_(s64) = G_LOAD [[COPY]](p1) :: (load 8, align 4, addrspace 1)
+    ; SI: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
+    ; SI: [[GEP:%[0-9]+]]:_(p1) = G_GEP [[COPY]], [[C]](s64)
+    ; SI: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[GEP]](p1) :: (load 4, addrspace 1)
+    ; SI: [[DEF:%[0-9]+]]:_(s96) = G_IMPLICIT_DEF
+    ; SI: [[INSERT:%[0-9]+]]:_(s96) = G_INSERT [[DEF]], [[LOAD]](s64), 0
+    ; SI: [[INSERT1:%[0-9]+]]:_(s96) = G_INSERT [[INSERT]], [[LOAD1]](s32), 64
+    ; SI: $vgpr0_vgpr1_vgpr2 = COPY [[INSERT1]](s96)
+    ; VI-LABEL: name: test_load_global_s96_align4
+    ; VI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
+    ; VI: [[LOAD:%[0-9]+]]:_(s96) = G_LOAD [[COPY]](p1) :: (load 12, align 4, addrspace 1)
+    ; VI: $vgpr0_vgpr1_vgpr2 = COPY [[LOAD]](s96)
+    %0:_(p1) = COPY $vgpr0_vgpr1
+    %1:_(s96) = G_LOAD %0 :: (load 12, addrspace 1, align 4)
+    $vgpr0_vgpr1_vgpr2 = COPY %1
+
+...
+
+---
+name: test_load_global_s160_align4
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; SI-LABEL: name: test_load_global_s160_align4
+    ; SI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
+    ; SI: [[LOAD:%[0-9]+]]:_(s64) = G_LOAD [[COPY]](p1) :: (load 8, align 4, addrspace 1)
+    ; SI: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
+    ; SI: [[GEP:%[0-9]+]]:_(p1) = G_GEP [[COPY]], [[C]](s64)
+    ; SI: [[LOAD1:%[0-9]+]]:_(s64) = G_LOAD [[GEP]](p1) :: (load 8, align 4, addrspace 1)
+    ; SI: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
+    ; SI: [[GEP1:%[0-9]+]]:_(p1) = G_GEP [[COPY]], [[C1]](s64)
+    ; SI: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[GEP1]](p1) :: (load 4, addrspace 1)
+    ; SI: [[DEF:%[0-9]+]]:_(s160) = G_IMPLICIT_DEF
+    ; SI: [[INSERT:%[0-9]+]]:_(s160) = G_INSERT [[DEF]], [[LOAD]](s64), 0
+    ; SI: [[INSERT1:%[0-9]+]]:_(s160) = G_INSERT [[INSERT]], [[LOAD1]](s64), 64
+    ; SI: [[INSERT2:%[0-9]+]]:_(s160) = G_INSERT [[INSERT1]], [[LOAD2]](s32), 128
+    ; SI: S_NOP 0, implicit [[INSERT2]](s160)
+    ; VI-LABEL: name: test_load_global_s160_align4
+    ; VI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
+    ; VI: [[LOAD:%[0-9]+]]:_(s64) = G_LOAD [[COPY]](p1) :: (load 8, align 4, addrspace 1)
+    ; VI: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
+    ; VI: [[GEP:%[0-9]+]]:_(p1) = G_GEP [[COPY]], [[C]](s64)
+    ; VI: [[LOAD1:%[0-9]+]]:_(s64) = G_LOAD [[GEP]](p1) :: (load 8, align 4, addrspace 1)
+    ; VI: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
+    ; VI: [[GEP1:%[0-9]+]]:_(p1) = G_GEP [[COPY]], [[C1]](s64)
+    ; VI: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[GEP1]](p1) :: (load 4, addrspace 1)
+    ; VI: [[DEF:%[0-9]+]]:_(s160) = G_IMPLICIT_DEF
+    ; VI: [[INSERT:%[0-9]+]]:_(s160) = G_INSERT [[DEF]], [[LOAD]](s64), 0
+    ; VI: [[INSERT1:%[0-9]+]]:_(s160) = G_INSERT [[INSERT]], [[LOAD1]](s64), 64
+    ; VI: [[INSERT2:%[0-9]+]]:_(s160) = G_INSERT [[INSERT1]], [[LOAD2]](s32), 128
+    ; VI: S_NOP 0, implicit [[INSERT2]](s160)
+    %0:_(p1) = COPY $vgpr0_vgpr1
+    %1:_(s160) = G_LOAD %0 :: (load 20, addrspace 1, align 4)
+    S_NOP 0, implicit %1
+...
+
+---
+name: test_load_global_s224_align4
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; SI-LABEL: name: test_load_global_s224_align4
+    ; SI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
+    ; SI: [[LOAD:%[0-9]+]]:_(s64) = G_LOAD [[COPY]](p1) :: (load 8, align 4, addrspace 1)
+    ; SI: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
+    ; SI: [[GEP:%[0-9]+]]:_(p1) = G_GEP [[COPY]], [[C]](s64)
+    ; SI: [[LOAD1:%[0-9]+]]:_(s64) = G_LOAD [[GEP]](p1) :: (load 8, align 4, addrspace 1)
+    ; SI: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
+    ; SI: [[GEP1:%[0-9]+]]:_(p1) = G_GEP [[COPY]], [[C1]](s64)
+    ; SI: [[LOAD2:%[0-9]+]]:_(s64) = G_LOAD [[GEP1]](p1) :: (load 8, align 4, addrspace 1)
+    ; SI: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 24
+    ; SI: [[GEP2:%[0-9]+]]:_(p1) = G_GEP [[COPY]], [[C2]](s64)
+    ; SI: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[GEP2]](p1) :: (load 4, addrspace 1)
+    ; SI: [[DEF:%[0-9]+]]:_(s224) = G_IMPLICIT_DEF
+    ; SI: [[INSERT:%[0-9]+]]:_(s224) = G_INSERT [[DEF]], [[LOAD]](s64), 0
+    ; SI: [[INSERT1:%[0-9]+]]:_(s224) = G_INSERT [[INSERT]], [[LOAD1]](s64), 64
+    ; SI: [[INSERT2:%[0-9]+]]:_(s224) = G_INSERT [[INSERT1]], [[LOAD2]](s64), 128
+    ; SI: [[INSERT3:%[0-9]+]]:_(s224) = G_INSERT [[INSERT2]], [[LOAD3]](s32), 192
+    ; SI: [[DEF1:%[0-9]+]]:_(s256) = G_IMPLICIT_DEF
+    ; SI: [[INSERT4:%[0-9]+]]:_(s256) = G_INSERT [[DEF1]], [[INSERT3]](s224), 0
+    ; SI: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY [[INSERT4]](s256)
+    ; VI-LABEL: name: test_load_global_s224_align4
+    ; VI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
+    ; VI: [[LOAD:%[0-9]+]]:_(s64) = G_LOAD [[COPY]](p1) :: (load 8, align 4, addrspace 1)
+    ; VI: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
+    ; VI: [[GEP:%[0-9]+]]:_(p1) = G_GEP [[COPY]], [[C]](s64)
+    ; VI: [[LOAD1:%[0-9]+]]:_(s64) = G_LOAD [[GEP]](p1) :: (load 8, align 4, addrspace 1)
+    ; VI: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
+    ; VI: [[GEP1:%[0-9]+]]:_(p1) = G_GEP [[COPY]], [[C1]](s64)
+    ; VI: [[LOAD2:%[0-9]+]]:_(s64) = G_LOAD [[GEP1]](p1) :: (load 8, align 4, addrspace 1)
+    ; VI: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 24
+    ; VI: [[GEP2:%[0-9]+]]:_(p1) = G_GEP [[COPY]], [[C2]](s64)
+    ; VI: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[GEP2]](p1) :: (load 4, addrspace 1)
+    ; VI: [[DEF:%[0-9]+]]:_(s224) = G_IMPLICIT_DEF
+    ; VI: [[INSERT:%[0-9]+]]:_(s224) = G_INSERT [[DEF]], [[LOAD]](s64), 0
+    ; VI: [[INSERT1:%[0-9]+]]:_(s224) = G_INSERT [[INSERT]], [[LOAD1]](s64), 64
+    ; VI: [[INSERT2:%[0-9]+]]:_(s224) = G_INSERT [[INSERT1]], [[LOAD2]](s64), 128
+    ; VI: [[INSERT3:%[0-9]+]]:_(s224) = G_INSERT [[INSERT2]], [[LOAD3]](s32), 192
+    ; VI: [[DEF1:%[0-9]+]]:_(s256) = G_IMPLICIT_DEF
+    ; VI: [[INSERT4:%[0-9]+]]:_(s256) = G_INSERT [[DEF1]], [[INSERT3]](s224), 0
+    ; VI: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY [[INSERT4]](s256)
+    %0:_(p1) = COPY $vgpr0_vgpr1
+    %1:_(s224) = G_LOAD %0 :: (load 28, addrspace 1, align 4)
+
+     %2:_(s256) = G_IMPLICIT_DEF
+     %3:_(s256) = G_INSERT %2, %1, 0
+     $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY %3
+
+...
+
+---
+name: test_load_global_v3s32
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; SI-LABEL: name: test_load_global_v3s32
+    ; SI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
+    ; SI: [[LOAD:%[0-9]+]]:_(<2 x s32>) = G_LOAD [[COPY]](p1) :: (load 8, align 16, addrspace 1)
+    ; SI: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
+    ; SI: [[GEP:%[0-9]+]]:_(p1) = G_GEP [[COPY]], [[C]](s64)
+    ; SI: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[GEP]](p1) :: (load 4, align 8, addrspace 1)
+    ; SI: [[DEF:%[0-9]+]]:_(<3 x s32>) = G_IMPLICIT_DEF
+    ; SI: [[INSERT:%[0-9]+]]:_(<3 x s32>) = G_INSERT [[DEF]], [[LOAD]](<2 x s32>), 0
+    ; SI: [[INSERT1:%[0-9]+]]:_(<3 x s32>) = G_INSERT [[INSERT]], [[LOAD1]](s32), 64
+    ; SI: $vgpr0_vgpr1_vgpr2 = COPY [[INSERT1]](<3 x s32>)
+    ; VI-LABEL: name: test_load_global_v3s32
+    ; VI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
+    ; VI: [[LOAD:%[0-9]+]]:_(<3 x s32>) = G_LOAD [[COPY]](p1) :: (load 12, align 16, addrspace 1)
+    ; VI: $vgpr0_vgpr1_vgpr2 = COPY [[LOAD]](<3 x s32>)
+    %0:_(p1) = COPY $vgpr0_vgpr1
+    %1:_(<3 x s32>) = G_LOAD %0 :: (load 12, addrspace 1, align 16)
     $vgpr0_vgpr1_vgpr2 = COPY %1
 ...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/legalize-lshr.mir b/test/CodeGen/AMDGPU/GlobalISel/legalize-lshr.mir
new file mode 100644
index 0000000000000000000000000000000000000000..0876c6098feffb2992bb25c3655e5f6c245ce0fa
--- /dev/null
+++ b/test/CodeGen/AMDGPU/GlobalISel/legalize-lshr.mir
@@ -0,0 +1,1514 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=tahiti -O0 -run-pass=legalizer -global-isel-abort=0 -o - %s | FileCheck -check-prefix=SI %s
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -O0 -run-pass=legalizer -global-isel-abort=0 -o - %s | FileCheck -check-prefix=VI %s
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx900 -O0 -run-pass=legalizer -global-isel-abort=0 -o - %s | FileCheck -check-prefix=GFX9  %s
+
+---
+name: test_lshr_s32_s32
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+
+    ; SI-LABEL: name: test_lshr_s32_s32
+    ; SI: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; SI: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; SI: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[COPY1]](s32)
+    ; SI: $vgpr0 = COPY [[LSHR]](s32)
+    ; VI-LABEL: name: test_lshr_s32_s32
+    ; VI: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; VI: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; VI: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[COPY1]](s32)
+    ; VI: $vgpr0 = COPY [[LSHR]](s32)
+    ; GFX9-LABEL: name: test_lshr_s32_s32
+    ; GFX9: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; GFX9: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; GFX9: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[COPY1]](s32)
+    ; GFX9: $vgpr0 = COPY [[LSHR]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s32) = G_LSHR %0, %1
+    $vgpr0 = COPY %2
+...
+---
+name: test_lshr_s64_s64
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
+
+    ; SI-LABEL: name: test_lshr_s64_s64
+    ; SI: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
+    ; SI: [[COPY1:%[0-9]+]]:_(s64) = COPY $vgpr2_vgpr3
+    ; SI: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY1]](s64)
+    ; SI: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[COPY]], [[TRUNC]](s32)
+    ; SI: $vgpr0_vgpr1 = COPY [[LSHR]](s64)
+    ; VI-LABEL: name: test_lshr_s64_s64
+    ; VI: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
+    ; VI: [[COPY1:%[0-9]+]]:_(s64) = COPY $vgpr2_vgpr3
+    ; VI: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY1]](s64)
+    ; VI: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[COPY]], [[TRUNC]](s32)
+    ; VI: $vgpr0_vgpr1 = COPY [[LSHR]](s64)
+    ; GFX9-LABEL: name: test_lshr_s64_s64
+    ; GFX9: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
+    ; GFX9: [[COPY1:%[0-9]+]]:_(s64) = COPY $vgpr2_vgpr3
+    ; GFX9: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY1]](s64)
+    ; GFX9: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[COPY]], [[TRUNC]](s32)
+    ; GFX9: $vgpr0_vgpr1 = COPY [[LSHR]](s64)
+    %0:_(s64) = COPY $vgpr0_vgpr1
+    %1:_(s64) = COPY $vgpr2_vgpr3
+    %2:_(s64) = G_LSHR %0, %1
+    $vgpr0_vgpr1 = COPY %2
+...
+---
+name: test_lshr_s64_s32
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2
+
+    ; SI-LABEL: name: test_lshr_s64_s32
+    ; SI: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
+    ; SI: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr2
+    ; SI: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[COPY]], [[COPY1]](s32)
+    ; SI: $vgpr0_vgpr1 = COPY [[LSHR]](s64)
+    ; VI-LABEL: name: test_lshr_s64_s32
+    ; VI: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
+    ; VI: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr2
+    ; VI: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[COPY]], [[COPY1]](s32)
+    ; VI: $vgpr0_vgpr1 = COPY [[LSHR]](s64)
+    ; GFX9-LABEL: name: test_lshr_s64_s32
+    ; GFX9: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
+    ; GFX9: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr2
+    ; GFX9: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[COPY]], [[COPY1]](s32)
+    ; GFX9: $vgpr0_vgpr1 = COPY [[LSHR]](s64)
+    %0:_(s64) = COPY $vgpr0_vgpr1
+    %1:_(s32) = COPY $vgpr2
+    %2:_(s64) = G_LSHR %0, %1
+    $vgpr0_vgpr1 = COPY %2
+...
+---
+name: test_lshr_s64_s16
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2
+
+    ; SI-LABEL: name: test_lshr_s64_s16
+    ; SI: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
+    ; SI: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr2
+    ; SI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; SI: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+    ; SI: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C]]
+    ; SI: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[COPY]], [[AND]](s32)
+    ; SI: $vgpr0_vgpr1 = COPY [[LSHR]](s64)
+    ; VI-LABEL: name: test_lshr_s64_s16
+    ; VI: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
+    ; VI: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr2
+    ; VI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32)
+    ; VI: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[COPY]], [[TRUNC]](s16)
+    ; VI: $vgpr0_vgpr1 = COPY [[LSHR]](s64)
+    ; GFX9-LABEL: name: test_lshr_s64_s16
+    ; GFX9: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
+    ; GFX9: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr2
+    ; GFX9: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32)
+    ; GFX9: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[COPY]], [[TRUNC]](s16)
+    ; GFX9: $vgpr0_vgpr1 = COPY [[LSHR]](s64)
+    %0:_(s64) = COPY $vgpr0_vgpr1
+    %1:_(s32) = COPY $vgpr2
+    %2:_(s16) = G_TRUNC %1
+    %3:_(s64) = G_LSHR %0, %2
+    $vgpr0_vgpr1 = COPY %3
+...
+
+---
+name: test_lshr_s16_s32
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+
+    ; SI-LABEL: name: test_lshr_s16_s32
+    ; SI: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; SI: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; SI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; SI: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; SI: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C]]
+    ; SI: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[AND]], [[COPY1]](s32)
+    ; SI: [[COPY3:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; SI: $vgpr0 = COPY [[COPY3]](s32)
+    ; VI-LABEL: name: test_lshr_s16_s32
+    ; VI: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; VI: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; VI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+    ; VI: [[LSHR:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC]], [[COPY1]](s32)
+    ; VI: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[LSHR]](s16)
+    ; VI: $vgpr0 = COPY [[ANYEXT]](s32)
+    ; GFX9-LABEL: name: test_lshr_s16_s32
+    ; GFX9: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; GFX9: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; GFX9: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+    ; GFX9: [[LSHR:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC]], [[COPY1]](s32)
+    ; GFX9: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[LSHR]](s16)
+    ; GFX9: $vgpr0 = COPY [[ANYEXT]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s16) = G_TRUNC %0
+    %3:_(s16) = G_LSHR %2, %1
+    %4:_(s32) = G_ANYEXT %3
+    $vgpr0 = COPY %4
+...
+
+---
+name: test_lshr_s16_s16
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+
+    ; SI-LABEL: name: test_lshr_s16_s16
+    ; SI: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; SI: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; SI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; SI: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+    ; SI: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C]]
+    ; SI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; SI: [[COPY3:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; SI: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[C1]]
+    ; SI: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[AND1]], [[AND]](s32)
+    ; SI: [[COPY4:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; SI: $vgpr0 = COPY [[COPY4]](s32)
+    ; VI-LABEL: name: test_lshr_s16_s16
+    ; VI: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; VI: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; VI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+    ; VI: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32)
+    ; VI: [[LSHR:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC]], [[TRUNC1]](s16)
+    ; VI: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[LSHR]](s16)
+    ; VI: $vgpr0 = COPY [[ANYEXT]](s32)
+    ; GFX9-LABEL: name: test_lshr_s16_s16
+    ; GFX9: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; GFX9: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; GFX9: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+    ; GFX9: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32)
+    ; GFX9: [[LSHR:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC]], [[TRUNC1]](s16)
+    ; GFX9: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[LSHR]](s16)
+    ; GFX9: $vgpr0 = COPY [[ANYEXT]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s16) = G_TRUNC %0
+    %3:_(s16) = G_TRUNC %1
+    %4:_(s16) = G_LSHR %2, %3
+    %5:_(s32) = G_ANYEXT %4
+    $vgpr0 = COPY %5
+...
+
+---
+name: test_lshr_s16_i8
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+
+    ; SI-LABEL: name: test_lshr_s16_i8
+    ; SI: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; SI: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; SI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; SI: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+    ; SI: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C]]
+    ; SI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; SI: [[COPY3:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; SI: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[C1]]
+    ; SI: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[AND1]], [[AND]](s32)
+    ; SI: [[COPY4:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; SI: $vgpr0 = COPY [[COPY4]](s32)
+    ; VI-LABEL: name: test_lshr_s16_i8
+    ; VI: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; VI: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; VI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+    ; VI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; VI: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+    ; VI: [[COPY3:%[0-9]+]]:_(s32) = COPY [[C]](s32)
+    ; VI: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[COPY3]]
+    ; VI: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[AND]](s32)
+    ; VI: [[LSHR:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC]], [[TRUNC1]](s16)
+    ; VI: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[LSHR]](s16)
+    ; VI: $vgpr0 = COPY [[ANYEXT]](s32)
+    ; GFX9-LABEL: name: test_lshr_s16_i8
+    ; GFX9: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; GFX9: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; GFX9: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; GFX9: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+    ; GFX9: [[COPY3:%[0-9]+]]:_(s32) = COPY [[C]](s32)
+    ; GFX9: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[COPY3]]
+    ; GFX9: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[AND]](s32)
+    ; GFX9: [[LSHR:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC]], [[TRUNC1]](s16)
+    ; GFX9: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[LSHR]](s16)
+    ; GFX9: $vgpr0 = COPY [[ANYEXT]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s16) = G_TRUNC %0
+    %3:_(s8) = G_TRUNC %1
+    %4:_(s16) = G_LSHR %2, %3
+    %5:_(s32) = G_ANYEXT %4
+    $vgpr0 = COPY %5
+...
+
+---
+name: test_lshr_i8_i8
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+
+    ; SI-LABEL: name: test_lshr_i8_i8
+    ; SI: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; SI: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; SI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; SI: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+    ; SI: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C]]
+    ; SI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; SI: [[COPY3:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; SI: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[C1]]
+    ; SI: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[AND1]], [[AND]](s32)
+    ; SI: [[COPY4:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; SI: $vgpr0 = COPY [[COPY4]](s32)
+    ; VI-LABEL: name: test_lshr_i8_i8
+    ; VI: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; VI: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; VI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; VI: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+    ; VI: [[COPY3:%[0-9]+]]:_(s32) = COPY [[C]](s32)
+    ; VI: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[COPY3]]
+    ; VI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[AND]](s32)
+    ; VI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; VI: [[COPY4:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; VI: [[COPY5:%[0-9]+]]:_(s32) = COPY [[C1]](s32)
+    ; VI: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY4]], [[COPY5]]
+    ; VI: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[AND1]](s32)
+    ; VI: [[LSHR:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC1]], [[TRUNC]](s16)
+    ; VI: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[LSHR]](s16)
+    ; VI: $vgpr0 = COPY [[ANYEXT]](s32)
+    ; GFX9-LABEL: name: test_lshr_i8_i8
+    ; GFX9: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; GFX9: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; GFX9: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+    ; GFX9: [[COPY3:%[0-9]+]]:_(s32) = COPY [[C]](s32)
+    ; GFX9: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[COPY3]]
+    ; GFX9: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[AND]](s32)
+    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; GFX9: [[COPY4:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; GFX9: [[COPY5:%[0-9]+]]:_(s32) = COPY [[C1]](s32)
+    ; GFX9: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY4]], [[COPY5]]
+    ; GFX9: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[AND1]](s32)
+    ; GFX9: [[LSHR:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC1]], [[TRUNC]](s16)
+    ; GFX9: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[LSHR]](s16)
+    ; GFX9: $vgpr0 = COPY [[ANYEXT]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s8) = G_TRUNC %0
+    %3:_(s8) = G_TRUNC %1
+    %4:_(s8) = G_LSHR %2, %3
+    %5:_(s32) = G_ANYEXT %4
+    $vgpr0 = COPY %5
+...
+
+---
+name: test_lshr_v2s32_v2s32
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
+
+    ; SI-LABEL: name: test_lshr_v2s32_v2s32
+    ; SI: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    ; SI: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr2_vgpr3
+    ; SI: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
+    ; SI: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<2 x s32>)
+    ; SI: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[UV]], [[UV2]](s32)
+    ; SI: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[UV1]], [[UV3]](s32)
+    ; SI: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[LSHR]](s32), [[LSHR1]](s32)
+    ; SI: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
+    ; VI-LABEL: name: test_lshr_v2s32_v2s32
+    ; VI: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    ; VI: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr2_vgpr3
+    ; VI: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
+    ; VI: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<2 x s32>)
+    ; VI: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[UV]], [[UV2]](s32)
+    ; VI: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[UV1]], [[UV3]](s32)
+    ; VI: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[LSHR]](s32), [[LSHR1]](s32)
+    ; VI: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
+    ; GFX9-LABEL: name: test_lshr_v2s32_v2s32
+    ; GFX9: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    ; GFX9: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr2_vgpr3
+    ; GFX9: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
+    ; GFX9: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<2 x s32>)
+    ; GFX9: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[UV]], [[UV2]](s32)
+    ; GFX9: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[UV1]], [[UV3]](s32)
+    ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[LSHR]](s32), [[LSHR1]](s32)
+    ; GFX9: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
+    %0:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    %1:_(<2 x s32>) = COPY $vgpr2_vgpr3
+    %2:_(<2 x s32>) = G_LSHR %0, %1
+    $vgpr0_vgpr1 = COPY %2
+...
+
+---
+name: test_lshr_v3s32_v3s32
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2, $vgpr3_vgpr4_vgpr5
+
+    ; SI-LABEL: name: test_lshr_v3s32_v3s32
+    ; SI: [[COPY:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr0_vgpr1_vgpr2
+    ; SI: [[COPY1:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr3_vgpr4_vgpr5
+    ; SI: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<3 x s32>)
+    ; SI: [[UV3:%[0-9]+]]:_(s32), [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<3 x s32>)
+    ; SI: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[UV]], [[UV3]](s32)
+    ; SI: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[UV1]], [[UV4]](s32)
+    ; SI: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[UV2]], [[UV5]](s32)
+    ; SI: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[LSHR]](s32), [[LSHR1]](s32), [[LSHR2]](s32)
+    ; SI: $vgpr0_vgpr1_vgpr2 = COPY [[BUILD_VECTOR]](<3 x s32>)
+    ; VI-LABEL: name: test_lshr_v3s32_v3s32
+    ; VI: [[COPY:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr0_vgpr1_vgpr2
+    ; VI: [[COPY1:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr3_vgpr4_vgpr5
+    ; VI: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<3 x s32>)
+    ; VI: [[UV3:%[0-9]+]]:_(s32), [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<3 x s32>)
+    ; VI: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[UV]], [[UV3]](s32)
+    ; VI: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[UV1]], [[UV4]](s32)
+    ; VI: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[UV2]], [[UV5]](s32)
+    ; VI: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[LSHR]](s32), [[LSHR1]](s32), [[LSHR2]](s32)
+    ; VI: $vgpr0_vgpr1_vgpr2 = COPY [[BUILD_VECTOR]](<3 x s32>)
+    ; GFX9-LABEL: name: test_lshr_v3s32_v3s32
+    ; GFX9: [[COPY:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr0_vgpr1_vgpr2
+    ; GFX9: [[COPY1:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr3_vgpr4_vgpr5
+    ; GFX9: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<3 x s32>)
+    ; GFX9: [[UV3:%[0-9]+]]:_(s32), [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<3 x s32>)
+    ; GFX9: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[UV]], [[UV3]](s32)
+    ; GFX9: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[UV1]], [[UV4]](s32)
+    ; GFX9: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[UV2]], [[UV5]](s32)
+    ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[LSHR]](s32), [[LSHR1]](s32), [[LSHR2]](s32)
+    ; GFX9: $vgpr0_vgpr1_vgpr2 = COPY [[BUILD_VECTOR]](<3 x s32>)
+    %0:_(<3 x s32>) = COPY $vgpr0_vgpr1_vgpr2
+    %1:_(<3 x s32>) = COPY $vgpr3_vgpr4_vgpr5
+    %2:_(<3 x s32>) = G_LSHR %0, %1
+    $vgpr0_vgpr1_vgpr2 = COPY %2
+...
+
+---
+name: test_lshr_v2s64_v2s32
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5
+
+    ; SI-LABEL: name: test_lshr_v2s64_v2s32
+    ; SI: [[COPY:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; SI: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr4_vgpr5
+    ; SI: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](<2 x s64>)
+    ; SI: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<2 x s32>)
+    ; SI: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[UV]], [[UV2]](s32)
+    ; SI: [[LSHR1:%[0-9]+]]:_(s64) = G_LSHR [[UV1]], [[UV3]](s32)
+    ; SI: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[LSHR]](s64), [[LSHR1]](s64)
+    ; SI: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>)
+    ; VI-LABEL: name: test_lshr_v2s64_v2s32
+    ; VI: [[COPY:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; VI: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr4_vgpr5
+    ; VI: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](<2 x s64>)
+    ; VI: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<2 x s32>)
+    ; VI: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[UV]], [[UV2]](s32)
+    ; VI: [[LSHR1:%[0-9]+]]:_(s64) = G_LSHR [[UV1]], [[UV3]](s32)
+    ; VI: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[LSHR]](s64), [[LSHR1]](s64)
+    ; VI: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>)
+    ; GFX9-LABEL: name: test_lshr_v2s64_v2s32
+    ; GFX9: [[COPY:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; GFX9: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr4_vgpr5
+    ; GFX9: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](<2 x s64>)
+    ; GFX9: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<2 x s32>)
+    ; GFX9: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[UV]], [[UV2]](s32)
+    ; GFX9: [[LSHR1:%[0-9]+]]:_(s64) = G_LSHR [[UV1]], [[UV3]](s32)
+    ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[LSHR]](s64), [[LSHR1]](s64)
+    ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>)
+    %0:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    %1:_(<2 x s32>) = COPY $vgpr4_vgpr5
+    %2:_(<2 x s64>) = G_LSHR %0, %1
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %2
+...
+
+---
+name: test_lshr_v3s64_v3s32
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10
+
+    ; SI-LABEL: name: test_lshr_v3s64_v3s32
+    ; SI: [[COPY:%[0-9]+]]:_(<4 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+    ; SI: [[EXTRACT:%[0-9]+]]:_(<3 x s64>) = G_EXTRACT [[COPY]](<4 x s64>), 0
+    ; SI: [[COPY1:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr8_vgpr9_vgpr10
+    ; SI: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64), [[UV2:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[EXTRACT]](<3 x s64>)
+    ; SI: [[UV3:%[0-9]+]]:_(s32), [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<3 x s32>)
+    ; SI: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[UV]], [[UV3]](s32)
+    ; SI: [[LSHR1:%[0-9]+]]:_(s64) = G_LSHR [[UV1]], [[UV4]](s32)
+    ; SI: [[LSHR2:%[0-9]+]]:_(s64) = G_LSHR [[UV2]], [[UV5]](s32)
+    ; SI: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s64>) = G_BUILD_VECTOR [[LSHR]](s64), [[LSHR1]](s64), [[LSHR2]](s64)
+    ; SI: [[DEF:%[0-9]+]]:_(<4 x s64>) = G_IMPLICIT_DEF
+    ; SI: [[INSERT:%[0-9]+]]:_(<4 x s64>) = G_INSERT [[DEF]], [[BUILD_VECTOR]](<3 x s64>), 0
+    ; SI: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY [[INSERT]](<4 x s64>)
+    ; VI-LABEL: name: test_lshr_v3s64_v3s32
+    ; VI: [[COPY:%[0-9]+]]:_(<4 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+    ; VI: [[EXTRACT:%[0-9]+]]:_(<3 x s64>) = G_EXTRACT [[COPY]](<4 x s64>), 0
+    ; VI: [[COPY1:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr8_vgpr9_vgpr10
+    ; VI: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64), [[UV2:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[EXTRACT]](<3 x s64>)
+    ; VI: [[UV3:%[0-9]+]]:_(s32), [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<3 x s32>)
+    ; VI: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[UV]], [[UV3]](s32)
+    ; VI: [[LSHR1:%[0-9]+]]:_(s64) = G_LSHR [[UV1]], [[UV4]](s32)
+    ; VI: [[LSHR2:%[0-9]+]]:_(s64) = G_LSHR [[UV2]], [[UV5]](s32)
+    ; VI: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s64>) = G_BUILD_VECTOR [[LSHR]](s64), [[LSHR1]](s64), [[LSHR2]](s64)
+    ; VI: [[DEF:%[0-9]+]]:_(<4 x s64>) = G_IMPLICIT_DEF
+    ; VI: [[INSERT:%[0-9]+]]:_(<4 x s64>) = G_INSERT [[DEF]], [[BUILD_VECTOR]](<3 x s64>), 0
+    ; VI: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY [[INSERT]](<4 x s64>)
+    ; GFX9-LABEL: name: test_lshr_v3s64_v3s32
+    ; GFX9: [[COPY:%[0-9]+]]:_(<4 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+    ; GFX9: [[EXTRACT:%[0-9]+]]:_(<3 x s64>) = G_EXTRACT [[COPY]](<4 x s64>), 0
+    ; GFX9: [[COPY1:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr8_vgpr9_vgpr10
+    ; GFX9: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64), [[UV2:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[EXTRACT]](<3 x s64>)
+    ; GFX9: [[UV3:%[0-9]+]]:_(s32), [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<3 x s32>)
+    ; GFX9: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[UV]], [[UV3]](s32)
+    ; GFX9: [[LSHR1:%[0-9]+]]:_(s64) = G_LSHR [[UV1]], [[UV4]](s32)
+    ; GFX9: [[LSHR2:%[0-9]+]]:_(s64) = G_LSHR [[UV2]], [[UV5]](s32)
+    ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s64>) = G_BUILD_VECTOR [[LSHR]](s64), [[LSHR1]](s64), [[LSHR2]](s64)
+    ; GFX9: [[DEF:%[0-9]+]]:_(<4 x s64>) = G_IMPLICIT_DEF
+    ; GFX9: [[INSERT:%[0-9]+]]:_(<4 x s64>) = G_INSERT [[DEF]], [[BUILD_VECTOR]](<3 x s64>), 0
+    ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY [[INSERT]](<4 x s64>)
+    %0:_(<4 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+    %1:_(<3 x s64>) = G_EXTRACT %0, 0
+    %2:_(<3 x s32>) = COPY $vgpr8_vgpr9_vgpr10
+    %3:_(<3 x s64>) = G_LSHR %1, %2
+    %4:_(<4 x s64>) = G_IMPLICIT_DEF
+    %5:_(<4 x s64>) = G_INSERT %4, %3, 0
+    $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY %5
+...
+
+---
+name: test_lshr_v2s16_v2s16
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+
+    ; SI-LABEL: name: test_lshr_v2s16_v2s16
+    ; SI: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
+    ; SI: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
+    ; SI: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY]](<2 x s16>)
+    ; SI: [[UV2:%[0-9]+]]:_(s16), [[UV3:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY1]](<2 x s16>)
+    ; SI: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[UV2]](s16)
+    ; SI: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[UV]](s16)
+    ; SI: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[ZEXT1]], [[ZEXT]](s32)
+    ; SI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+    ; SI: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[UV3]](s16)
+    ; SI: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[UV1]](s16)
+    ; SI: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[ZEXT3]], [[ZEXT2]](s32)
+    ; SI: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+    ; SI: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
+    ; SI: $vgpr0 = COPY [[BUILD_VECTOR]](<2 x s16>)
+    ; VI-LABEL: name: test_lshr_v2s16_v2s16
+    ; VI: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
+    ; VI: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
+    ; VI: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY]](<2 x s16>)
+    ; VI: [[UV2:%[0-9]+]]:_(s16), [[UV3:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY1]](<2 x s16>)
+    ; VI: [[LSHR:%[0-9]+]]:_(s16) = G_LSHR [[UV]], [[UV2]](s16)
+    ; VI: [[LSHR1:%[0-9]+]]:_(s16) = G_LSHR [[UV1]], [[UV3]](s16)
+    ; VI: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[LSHR]](s16), [[LSHR1]](s16)
+    ; VI: $vgpr0 = COPY [[BUILD_VECTOR]](<2 x s16>)
+    ; GFX9-LABEL: name: test_lshr_v2s16_v2s16
+    ; GFX9: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
+    ; GFX9: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
+    ; GFX9: [[LSHR:%[0-9]+]]:_(<2 x s16>) = G_LSHR [[COPY]], [[COPY1]](<2 x s16>)
+    ; GFX9: $vgpr0 = COPY [[LSHR]](<2 x s16>)
+    %0:_(<2 x s16>) = COPY $vgpr0
+    %1:_(<2 x s16>) = COPY $vgpr1
+    %2:_(<2 x s16>) = G_LSHR %0, %1
+    $vgpr0 = COPY %2
+...
+
+---
+name: test_lshr_v2s16_v2s32
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr0_vgpr1
+
+    ; SI-LABEL: name: test_lshr_v2s16_v2s32
+    ; SI: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
+    ; SI: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    ; SI: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY]](<2 x s16>)
+    ; SI: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<2 x s32>)
+    ; SI: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[UV]](s16)
+    ; SI: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[ZEXT]], [[UV2]](s32)
+    ; SI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+    ; SI: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[UV1]](s16)
+    ; SI: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[ZEXT1]], [[UV3]](s32)
+    ; SI: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+    ; SI: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
+    ; SI: $vgpr0 = COPY [[BUILD_VECTOR]](<2 x s16>)
+    ; VI-LABEL: name: test_lshr_v2s16_v2s32
+    ; VI: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
+    ; VI: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    ; VI: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY]](<2 x s16>)
+    ; VI: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<2 x s32>)
+    ; VI: [[LSHR:%[0-9]+]]:_(s16) = G_LSHR [[UV]], [[UV2]](s32)
+    ; VI: [[LSHR1:%[0-9]+]]:_(s16) = G_LSHR [[UV1]], [[UV3]](s32)
+    ; VI: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[LSHR]](s16), [[LSHR1]](s16)
+    ; VI: $vgpr0 = COPY [[BUILD_VECTOR]](<2 x s16>)
+    ; GFX9-LABEL: name: test_lshr_v2s16_v2s32
+    ; GFX9: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
+    ; GFX9: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    ; GFX9: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY]](<2 x s16>)
+    ; GFX9: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<2 x s32>)
+    ; GFX9: [[LSHR:%[0-9]+]]:_(s16) = G_LSHR [[UV]], [[UV2]](s32)
+    ; GFX9: [[LSHR1:%[0-9]+]]:_(s16) = G_LSHR [[UV1]], [[UV3]](s32)
+    ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[LSHR]](s16), [[LSHR1]](s16)
+    ; GFX9: $vgpr0 = COPY [[BUILD_VECTOR]](<2 x s16>)
+    %0:_(<2 x s16>) = COPY $vgpr0
+    %1:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    %2:_(<2 x s16>) = G_LSHR %0, %1
+    $vgpr0 = COPY %2
+...
+
+---
+name: test_lshr_v3s16_v3s16
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
+
+    ; SI-LABEL: name: test_lshr_v3s16_v3s16
+    ; SI: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
+    ; SI: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
+    ; SI: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[COPY]](<4 x s16>), 0
+    ; SI: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[COPY1]](<4 x s16>), 0
+    ; SI: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16), [[UV2:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[EXTRACT]](<3 x s16>)
+    ; SI: [[UV3:%[0-9]+]]:_(s16), [[UV4:%[0-9]+]]:_(s16), [[UV5:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[EXTRACT1]](<3 x s16>)
+    ; SI: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[UV3]](s16)
+    ; SI: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[UV]](s16)
+    ; SI: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[ZEXT1]], [[ZEXT]](s32)
+    ; SI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+    ; SI: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[UV4]](s16)
+    ; SI: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[UV1]](s16)
+    ; SI: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[ZEXT3]], [[ZEXT2]](s32)
+    ; SI: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+    ; SI: [[ZEXT4:%[0-9]+]]:_(s32) = G_ZEXT [[UV5]](s16)
+    ; SI: [[ZEXT5:%[0-9]+]]:_(s32) = G_ZEXT [[UV2]](s16)
+    ; SI: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[ZEXT5]], [[ZEXT4]](s32)
+    ; SI: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
+    ; SI: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16), [[TRUNC2]](s16)
+    ; SI: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; SI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[BUILD_VECTOR]](<3 x s16>), 0
+    ; SI: $vgpr0_vgpr1 = COPY [[INSERT]](<4 x s16>)
+    ; VI-LABEL: name: test_lshr_v3s16_v3s16
+    ; VI: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
+    ; VI: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
+    ; VI: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[COPY]](<4 x s16>), 0
+    ; VI: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[COPY1]](<4 x s16>), 0
+    ; VI: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16), [[UV2:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[EXTRACT]](<3 x s16>)
+    ; VI: [[UV3:%[0-9]+]]:_(s16), [[UV4:%[0-9]+]]:_(s16), [[UV5:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[EXTRACT1]](<3 x s16>)
+    ; VI: [[LSHR:%[0-9]+]]:_(s16) = G_LSHR [[UV]], [[UV3]](s16)
+    ; VI: [[LSHR1:%[0-9]+]]:_(s16) = G_LSHR [[UV1]], [[UV4]](s16)
+    ; VI: [[LSHR2:%[0-9]+]]:_(s16) = G_LSHR [[UV2]], [[UV5]](s16)
+    ; VI: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s16>) = G_BUILD_VECTOR [[LSHR]](s16), [[LSHR1]](s16), [[LSHR2]](s16)
+    ; VI: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; VI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[BUILD_VECTOR]](<3 x s16>), 0
+    ; VI: $vgpr0_vgpr1 = COPY [[INSERT]](<4 x s16>)
+    ; GFX9-LABEL: name: test_lshr_v3s16_v3s16
+    ; GFX9: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
+    ; GFX9: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
+    ; GFX9: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[COPY]](<4 x s16>), 0
+    ; GFX9: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[COPY1]](<4 x s16>), 0
+    ; GFX9: [[EXTRACT2:%[0-9]+]]:_(<2 x s16>) = G_EXTRACT [[EXTRACT]](<3 x s16>), 0
+    ; GFX9: [[EXTRACT3:%[0-9]+]]:_(s16) = G_EXTRACT [[EXTRACT]](<3 x s16>), 32
+    ; GFX9: [[EXTRACT4:%[0-9]+]]:_(<2 x s16>) = G_EXTRACT [[EXTRACT1]](<3 x s16>), 0
+    ; GFX9: [[EXTRACT5:%[0-9]+]]:_(s16) = G_EXTRACT [[EXTRACT1]](<3 x s16>), 32
+    ; GFX9: [[LSHR:%[0-9]+]]:_(<2 x s16>) = G_LSHR [[EXTRACT2]], [[EXTRACT4]](<2 x s16>)
+    ; GFX9: [[LSHR1:%[0-9]+]]:_(s16) = G_LSHR [[EXTRACT3]], [[EXTRACT5]](s16)
+    ; GFX9: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; GFX9: [[EXTRACT6:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
+    ; GFX9: [[INSERT:%[0-9]+]]:_(<3 x s16>) = G_INSERT [[EXTRACT6]], [[LSHR]](<2 x s16>), 0
+    ; GFX9: [[INSERT1:%[0-9]+]]:_(<3 x s16>) = G_INSERT [[INSERT]], [[LSHR1]](s16), 32
+    ; GFX9: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; GFX9: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[INSERT1]](<3 x s16>), 0
+    ; GFX9: $vgpr0_vgpr1 = COPY [[INSERT2]](<4 x s16>)
+    %0:_(<4 x s16>) = COPY $vgpr0_vgpr1
+    %1:_(<4 x s16>) = COPY $vgpr2_vgpr3
+    %2:_(<3 x s16>) = G_EXTRACT %0, 0
+    %3:_(<3 x s16>) = G_EXTRACT %1, 0
+    %4:_(<3 x s16>) = G_LSHR %2, %3
+    %5:_(<4 x s16>) = G_IMPLICIT_DEF
+    %6:_(<4 x s16>) = G_INSERT %5, %4, 0
+    $vgpr0_vgpr1 = COPY %6
+...
+
+---
+name: test_lshr_v4s16_v4s16
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
+
+    ; SI-LABEL: name: test_lshr_v4s16_v4s16
+    ; SI: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
+    ; SI: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
+    ; SI: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16), [[UV2:%[0-9]+]]:_(s16), [[UV3:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
+    ; SI: [[UV4:%[0-9]+]]:_(s16), [[UV5:%[0-9]+]]:_(s16), [[UV6:%[0-9]+]]:_(s16), [[UV7:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
+    ; SI: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[UV4]](s16)
+    ; SI: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[UV]](s16)
+    ; SI: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[ZEXT1]], [[ZEXT]](s32)
+    ; SI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+    ; SI: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[UV5]](s16)
+    ; SI: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[UV1]](s16)
+    ; SI: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[ZEXT3]], [[ZEXT2]](s32)
+    ; SI: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+    ; SI: [[ZEXT4:%[0-9]+]]:_(s32) = G_ZEXT [[UV6]](s16)
+    ; SI: [[ZEXT5:%[0-9]+]]:_(s32) = G_ZEXT [[UV2]](s16)
+    ; SI: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[ZEXT5]], [[ZEXT4]](s32)
+    ; SI: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
+    ; SI: [[ZEXT6:%[0-9]+]]:_(s32) = G_ZEXT [[UV7]](s16)
+    ; SI: [[ZEXT7:%[0-9]+]]:_(s32) = G_ZEXT [[UV3]](s16)
+    ; SI: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[ZEXT7]], [[ZEXT6]](s32)
+    ; SI: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](s32)
+    ; SI: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16), [[TRUNC2]](s16), [[TRUNC3]](s16)
+    ; SI: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<4 x s16>)
+    ; VI-LABEL: name: test_lshr_v4s16_v4s16
+    ; VI: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
+    ; VI: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
+    ; VI: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16), [[UV2:%[0-9]+]]:_(s16), [[UV3:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
+    ; VI: [[UV4:%[0-9]+]]:_(s16), [[UV5:%[0-9]+]]:_(s16), [[UV6:%[0-9]+]]:_(s16), [[UV7:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
+    ; VI: [[LSHR:%[0-9]+]]:_(s16) = G_LSHR [[UV]], [[UV4]](s16)
+    ; VI: [[LSHR1:%[0-9]+]]:_(s16) = G_LSHR [[UV1]], [[UV5]](s16)
+    ; VI: [[LSHR2:%[0-9]+]]:_(s16) = G_LSHR [[UV2]], [[UV6]](s16)
+    ; VI: [[LSHR3:%[0-9]+]]:_(s16) = G_LSHR [[UV3]], [[UV7]](s16)
+    ; VI: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s16>) = G_BUILD_VECTOR [[LSHR]](s16), [[LSHR1]](s16), [[LSHR2]](s16), [[LSHR3]](s16)
+    ; VI: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<4 x s16>)
+    ; GFX9-LABEL: name: test_lshr_v4s16_v4s16
+    ; GFX9: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
+    ; GFX9: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
+    ; GFX9: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
+    ; GFX9: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
+    ; GFX9: [[LSHR:%[0-9]+]]:_(<2 x s16>) = G_LSHR [[UV]], [[UV2]](<2 x s16>)
+    ; GFX9: [[LSHR1:%[0-9]+]]:_(<2 x s16>) = G_LSHR [[UV1]], [[UV3]](<2 x s16>)
+    ; GFX9: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[LSHR]](<2 x s16>), [[LSHR1]](<2 x s16>)
+    ; GFX9: $vgpr0_vgpr1 = COPY [[CONCAT_VECTORS]](<4 x s16>)
+    %0:_(<4 x s16>) = COPY $vgpr0_vgpr1
+    %1:_(<4 x s16>) = COPY $vgpr2_vgpr3
+    %2:_(<4 x s16>) = G_LSHR %0, %1
+    $vgpr0_vgpr1 = COPY %2
+...
+
+---
+name: test_lshr_s128_s128
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4
+
+    ; SI-LABEL: name: test_lshr_s128_s128
+    ; SI: [[COPY:%[0-9]+]]:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; SI: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr4
+    ; SI: [[ZEXT:%[0-9]+]]:_(s128) = G_ZEXT [[COPY1]](s32)
+    ; SI: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[ZEXT]](s128)
+    ; SI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 64
+    ; SI: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](s128)
+    ; SI: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[TRUNC]], [[C]]
+    ; SI: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[C]], [[TRUNC]]
+    ; SI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; SI: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[TRUNC]](s32), [[C]]
+    ; SI: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[TRUNC]](s32), [[C1]]
+    ; SI: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[UV1]], [[TRUNC]](s32)
+    ; SI: [[LSHR1:%[0-9]+]]:_(s64) = G_LSHR [[UV]], [[TRUNC]](s32)
+    ; SI: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[UV1]], [[SUB1]](s32)
+    ; SI: [[OR:%[0-9]+]]:_(s64) = G_OR [[LSHR1]], [[SHL]]
+    ; SI: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; SI: [[LSHR2:%[0-9]+]]:_(s64) = G_LSHR [[UV1]], [[SUB]](s32)
+    ; SI: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[OR]], [[LSHR2]]
+    ; SI: [[SELECT1:%[0-9]+]]:_(s64) = G_SELECT [[ICMP1]](s1), [[UV]], [[SELECT]]
+    ; SI: [[SELECT2:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[LSHR]], [[C2]]
+    ; SI: [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[SELECT1]](s64), [[SELECT2]](s64)
+    ; SI: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[MV]](s128)
+    ; VI-LABEL: name: test_lshr_s128_s128
+    ; VI: [[COPY:%[0-9]+]]:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; VI: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr4
+    ; VI: [[ZEXT:%[0-9]+]]:_(s128) = G_ZEXT [[COPY1]](s32)
+    ; VI: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[ZEXT]](s128)
+    ; VI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 64
+    ; VI: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](s128)
+    ; VI: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[TRUNC]], [[C]]
+    ; VI: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[C]], [[TRUNC]]
+    ; VI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; VI: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[TRUNC]](s32), [[C]]
+    ; VI: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[TRUNC]](s32), [[C1]]
+    ; VI: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[UV1]], [[TRUNC]](s32)
+    ; VI: [[LSHR1:%[0-9]+]]:_(s64) = G_LSHR [[UV]], [[TRUNC]](s32)
+    ; VI: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[UV1]], [[SUB1]](s32)
+    ; VI: [[OR:%[0-9]+]]:_(s64) = G_OR [[LSHR1]], [[SHL]]
+    ; VI: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; VI: [[LSHR2:%[0-9]+]]:_(s64) = G_LSHR [[UV1]], [[SUB]](s32)
+    ; VI: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[OR]], [[LSHR2]]
+    ; VI: [[SELECT1:%[0-9]+]]:_(s64) = G_SELECT [[ICMP1]](s1), [[UV]], [[SELECT]]
+    ; VI: [[SELECT2:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[LSHR]], [[C2]]
+    ; VI: [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[SELECT1]](s64), [[SELECT2]](s64)
+    ; VI: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[MV]](s128)
+    ; GFX9-LABEL: name: test_lshr_s128_s128
+    ; GFX9: [[COPY:%[0-9]+]]:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; GFX9: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr4
+    ; GFX9: [[ZEXT:%[0-9]+]]:_(s128) = G_ZEXT [[COPY1]](s32)
+    ; GFX9: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[ZEXT]](s128)
+    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 64
+    ; GFX9: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](s128)
+    ; GFX9: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[TRUNC]], [[C]]
+    ; GFX9: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[C]], [[TRUNC]]
+    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; GFX9: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[TRUNC]](s32), [[C]]
+    ; GFX9: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[TRUNC]](s32), [[C1]]
+    ; GFX9: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[UV1]], [[TRUNC]](s32)
+    ; GFX9: [[LSHR1:%[0-9]+]]:_(s64) = G_LSHR [[UV]], [[TRUNC]](s32)
+    ; GFX9: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[UV1]], [[SUB1]](s32)
+    ; GFX9: [[OR:%[0-9]+]]:_(s64) = G_OR [[LSHR1]], [[SHL]]
+    ; GFX9: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; GFX9: [[LSHR2:%[0-9]+]]:_(s64) = G_LSHR [[UV1]], [[SUB]](s32)
+    ; GFX9: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[OR]], [[LSHR2]]
+    ; GFX9: [[SELECT1:%[0-9]+]]:_(s64) = G_SELECT [[ICMP1]](s1), [[UV]], [[SELECT]]
+    ; GFX9: [[SELECT2:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[LSHR]], [[C2]]
+    ; GFX9: [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[SELECT1]](s64), [[SELECT2]](s64)
+    ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[MV]](s128)
+    %0:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    %1:_(s32) = COPY $vgpr4
+    %2:_(s128) = G_ZEXT %1
+    %3:_(s128) = G_LSHR %0, %2
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %3
+...
+
+---
+name: test_lshr_s128_s132
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4
+
+    ; SI-LABEL: name: test_lshr_s128_s132
+    ; SI: [[COPY:%[0-9]+]]:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; SI: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr4
+    ; SI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 64
+    ; SI: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](s128)
+    ; SI: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[COPY1]], [[C]]
+    ; SI: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[C]], [[COPY1]]
+    ; SI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; SI: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[COPY1]](s32), [[C]]
+    ; SI: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[COPY1]](s32), [[C1]]
+    ; SI: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[UV1]], [[COPY1]](s32)
+    ; SI: [[LSHR1:%[0-9]+]]:_(s64) = G_LSHR [[UV]], [[COPY1]](s32)
+    ; SI: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[UV1]], [[SUB1]](s32)
+    ; SI: [[OR:%[0-9]+]]:_(s64) = G_OR [[LSHR1]], [[SHL]]
+    ; SI: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; SI: [[LSHR2:%[0-9]+]]:_(s64) = G_LSHR [[UV1]], [[SUB]](s32)
+    ; SI: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[OR]], [[LSHR2]]
+    ; SI: [[SELECT1:%[0-9]+]]:_(s64) = G_SELECT [[ICMP1]](s1), [[UV]], [[SELECT]]
+    ; SI: [[SELECT2:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[LSHR]], [[C2]]
+    ; SI: [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[SELECT1]](s64), [[SELECT2]](s64)
+    ; SI: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[MV]](s128)
+    ; VI-LABEL: name: test_lshr_s128_s132
+    ; VI: [[COPY:%[0-9]+]]:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; VI: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr4
+    ; VI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 64
+    ; VI: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](s128)
+    ; VI: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[COPY1]], [[C]]
+    ; VI: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[C]], [[COPY1]]
+    ; VI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; VI: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[COPY1]](s32), [[C]]
+    ; VI: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[COPY1]](s32), [[C1]]
+    ; VI: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[UV1]], [[COPY1]](s32)
+    ; VI: [[LSHR1:%[0-9]+]]:_(s64) = G_LSHR [[UV]], [[COPY1]](s32)
+    ; VI: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[UV1]], [[SUB1]](s32)
+    ; VI: [[OR:%[0-9]+]]:_(s64) = G_OR [[LSHR1]], [[SHL]]
+    ; VI: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; VI: [[LSHR2:%[0-9]+]]:_(s64) = G_LSHR [[UV1]], [[SUB]](s32)
+    ; VI: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[OR]], [[LSHR2]]
+    ; VI: [[SELECT1:%[0-9]+]]:_(s64) = G_SELECT [[ICMP1]](s1), [[UV]], [[SELECT]]
+    ; VI: [[SELECT2:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[LSHR]], [[C2]]
+    ; VI: [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[SELECT1]](s64), [[SELECT2]](s64)
+    ; VI: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[MV]](s128)
+    ; GFX9-LABEL: name: test_lshr_s128_s132
+    ; GFX9: [[COPY:%[0-9]+]]:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; GFX9: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr4
+    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 64
+    ; GFX9: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](s128)
+    ; GFX9: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[COPY1]], [[C]]
+    ; GFX9: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[C]], [[COPY1]]
+    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; GFX9: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[COPY1]](s32), [[C]]
+    ; GFX9: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[COPY1]](s32), [[C1]]
+    ; GFX9: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[UV1]], [[COPY1]](s32)
+    ; GFX9: [[LSHR1:%[0-9]+]]:_(s64) = G_LSHR [[UV]], [[COPY1]](s32)
+    ; GFX9: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[UV1]], [[SUB1]](s32)
+    ; GFX9: [[OR:%[0-9]+]]:_(s64) = G_OR [[LSHR1]], [[SHL]]
+    ; GFX9: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; GFX9: [[LSHR2:%[0-9]+]]:_(s64) = G_LSHR [[UV1]], [[SUB]](s32)
+    ; GFX9: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[OR]], [[LSHR2]]
+    ; GFX9: [[SELECT1:%[0-9]+]]:_(s64) = G_SELECT [[ICMP1]](s1), [[UV]], [[SELECT]]
+    ; GFX9: [[SELECT2:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[LSHR]], [[C2]]
+    ; GFX9: [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[SELECT1]](s64), [[SELECT2]](s64)
+    ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[MV]](s128)
+    %0:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    %1:_(s32) = COPY $vgpr4
+    %2:_(s128) = G_LSHR %0, %1
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %2
+...
+
+---
+name: test_lshr_s128_s32_0
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4
+
+    ; SI-LABEL: name: test_lshr_s128_s32_0
+    ; SI: [[COPY:%[0-9]+]]:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; SI: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](s128)
+    ; SI: [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[UV]](s64), [[UV1]](s64)
+    ; SI: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[MV]](s128)
+    ; VI-LABEL: name: test_lshr_s128_s32_0
+    ; VI: [[COPY:%[0-9]+]]:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; VI: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](s128)
+    ; VI: [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[UV]](s64), [[UV1]](s64)
+    ; VI: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[MV]](s128)
+    ; GFX9-LABEL: name: test_lshr_s128_s32_0
+    ; GFX9: [[COPY:%[0-9]+]]:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; GFX9: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](s128)
+    ; GFX9: [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[UV]](s64), [[UV1]](s64)
+    ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[MV]](s128)
+    %0:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    %1:_(s32) = G_CONSTANT i32 0
+    %3:_(s128) = G_LSHR %0, %1
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %3
+...
+
+
+---
+name: test_lshr_s128_s32_23
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4
+
+    ; SI-LABEL: name: test_lshr_s128_s32_23
+    ; SI: [[COPY:%[0-9]+]]:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; SI: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](s128)
+    ; SI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 23
+    ; SI: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[UV]], [[C]](s32)
+    ; SI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 41
+    ; SI: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[UV1]], [[C1]](s32)
+    ; SI: [[OR:%[0-9]+]]:_(s64) = G_OR [[LSHR]], [[SHL]]
+    ; SI: [[LSHR1:%[0-9]+]]:_(s64) = G_LSHR [[UV1]], [[C]](s32)
+    ; SI: [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[OR]](s64), [[LSHR1]](s64)
+    ; SI: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[MV]](s128)
+    ; VI-LABEL: name: test_lshr_s128_s32_23
+    ; VI: [[COPY:%[0-9]+]]:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; VI: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](s128)
+    ; VI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 23
+    ; VI: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[UV]], [[C]](s32)
+    ; VI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 41
+    ; VI: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[UV1]], [[C1]](s32)
+    ; VI: [[OR:%[0-9]+]]:_(s64) = G_OR [[LSHR]], [[SHL]]
+    ; VI: [[LSHR1:%[0-9]+]]:_(s64) = G_LSHR [[UV1]], [[C]](s32)
+    ; VI: [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[OR]](s64), [[LSHR1]](s64)
+    ; VI: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[MV]](s128)
+    ; GFX9-LABEL: name: test_lshr_s128_s32_23
+    ; GFX9: [[COPY:%[0-9]+]]:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; GFX9: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](s128)
+    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 23
+    ; GFX9: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[UV]], [[C]](s32)
+    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 41
+    ; GFX9: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[UV1]], [[C1]](s32)
+    ; GFX9: [[OR:%[0-9]+]]:_(s64) = G_OR [[LSHR]], [[SHL]]
+    ; GFX9: [[LSHR1:%[0-9]+]]:_(s64) = G_LSHR [[UV1]], [[C]](s32)
+    ; GFX9: [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[OR]](s64), [[LSHR1]](s64)
+    ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[MV]](s128)
+    %0:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    %1:_(s32) = G_CONSTANT i32 23
+    %3:_(s128) = G_LSHR %0, %1
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %3
+...
+
+---
+name: test_lshr_s128_s32_31
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4
+
+    ; SI-LABEL: name: test_lshr_s128_s32_31
+    ; SI: [[COPY:%[0-9]+]]:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; SI: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](s128)
+    ; SI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 31
+    ; SI: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[UV]], [[C]](s32)
+    ; SI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 33
+    ; SI: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[UV1]], [[C1]](s32)
+    ; SI: [[OR:%[0-9]+]]:_(s64) = G_OR [[LSHR]], [[SHL]]
+    ; SI: [[LSHR1:%[0-9]+]]:_(s64) = G_LSHR [[UV1]], [[C]](s32)
+    ; SI: [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[OR]](s64), [[LSHR1]](s64)
+    ; SI: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[MV]](s128)
+    ; VI-LABEL: name: test_lshr_s128_s32_31
+    ; VI: [[COPY:%[0-9]+]]:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; VI: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](s128)
+    ; VI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 31
+    ; VI: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[UV]], [[C]](s32)
+    ; VI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 33
+    ; VI: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[UV1]], [[C1]](s32)
+    ; VI: [[OR:%[0-9]+]]:_(s64) = G_OR [[LSHR]], [[SHL]]
+    ; VI: [[LSHR1:%[0-9]+]]:_(s64) = G_LSHR [[UV1]], [[C]](s32)
+    ; VI: [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[OR]](s64), [[LSHR1]](s64)
+    ; VI: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[MV]](s128)
+    ; GFX9-LABEL: name: test_lshr_s128_s32_31
+    ; GFX9: [[COPY:%[0-9]+]]:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; GFX9: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](s128)
+    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 31
+    ; GFX9: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[UV]], [[C]](s32)
+    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 33
+    ; GFX9: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[UV1]], [[C1]](s32)
+    ; GFX9: [[OR:%[0-9]+]]:_(s64) = G_OR [[LSHR]], [[SHL]]
+    ; GFX9: [[LSHR1:%[0-9]+]]:_(s64) = G_LSHR [[UV1]], [[C]](s32)
+    ; GFX9: [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[OR]](s64), [[LSHR1]](s64)
+    ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[MV]](s128)
+    %0:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    %1:_(s32) = G_CONSTANT i32 31
+    %3:_(s128) = G_LSHR %0, %1
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %3
+...
+
+---
+name: test_lshr_s128_s32_32
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4
+
+    ; SI-LABEL: name: test_lshr_s128_s32_32
+    ; SI: [[COPY:%[0-9]+]]:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; SI: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](s128)
+    ; SI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 32
+    ; SI: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[UV]], [[C]](s32)
+    ; SI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 32
+    ; SI: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[UV1]], [[C1]](s32)
+    ; SI: [[OR:%[0-9]+]]:_(s64) = G_OR [[LSHR]], [[SHL]]
+    ; SI: [[LSHR1:%[0-9]+]]:_(s64) = G_LSHR [[UV1]], [[C]](s32)
+    ; SI: [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[OR]](s64), [[LSHR1]](s64)
+    ; SI: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[MV]](s128)
+    ; VI-LABEL: name: test_lshr_s128_s32_32
+    ; VI: [[COPY:%[0-9]+]]:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; VI: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](s128)
+    ; VI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 32
+    ; VI: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[UV]], [[C]](s32)
+    ; VI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 32
+    ; VI: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[UV1]], [[C1]](s32)
+    ; VI: [[OR:%[0-9]+]]:_(s64) = G_OR [[LSHR]], [[SHL]]
+    ; VI: [[LSHR1:%[0-9]+]]:_(s64) = G_LSHR [[UV1]], [[C]](s32)
+    ; VI: [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[OR]](s64), [[LSHR1]](s64)
+    ; VI: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[MV]](s128)
+    ; GFX9-LABEL: name: test_lshr_s128_s32_32
+    ; GFX9: [[COPY:%[0-9]+]]:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; GFX9: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](s128)
+    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 32
+    ; GFX9: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[UV]], [[C]](s32)
+    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 32
+    ; GFX9: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[UV1]], [[C1]](s32)
+    ; GFX9: [[OR:%[0-9]+]]:_(s64) = G_OR [[LSHR]], [[SHL]]
+    ; GFX9: [[LSHR1:%[0-9]+]]:_(s64) = G_LSHR [[UV1]], [[C]](s32)
+    ; GFX9: [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[OR]](s64), [[LSHR1]](s64)
+    ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[MV]](s128)
+    %0:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    %1:_(s32) = G_CONSTANT i32 32
+    %3:_(s128) = G_LSHR %0, %1
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %3
+...
+
+---
+name: test_lshr_s128_s32_33
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4
+
+    ; SI-LABEL: name: test_lshr_s128_s32_33
+    ; SI: [[COPY:%[0-9]+]]:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; SI: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](s128)
+    ; SI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 33
+    ; SI: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[UV]], [[C]](s32)
+    ; SI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 31
+    ; SI: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[UV1]], [[C1]](s32)
+    ; SI: [[OR:%[0-9]+]]:_(s64) = G_OR [[LSHR]], [[SHL]]
+    ; SI: [[LSHR1:%[0-9]+]]:_(s64) = G_LSHR [[UV1]], [[C]](s32)
+    ; SI: [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[OR]](s64), [[LSHR1]](s64)
+    ; SI: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[MV]](s128)
+    ; VI-LABEL: name: test_lshr_s128_s32_33
+    ; VI: [[COPY:%[0-9]+]]:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; VI: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](s128)
+    ; VI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 33
+    ; VI: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[UV]], [[C]](s32)
+    ; VI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 31
+    ; VI: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[UV1]], [[C1]](s32)
+    ; VI: [[OR:%[0-9]+]]:_(s64) = G_OR [[LSHR]], [[SHL]]
+    ; VI: [[LSHR1:%[0-9]+]]:_(s64) = G_LSHR [[UV1]], [[C]](s32)
+    ; VI: [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[OR]](s64), [[LSHR1]](s64)
+    ; VI: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[MV]](s128)
+    ; GFX9-LABEL: name: test_lshr_s128_s32_33
+    ; GFX9: [[COPY:%[0-9]+]]:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; GFX9: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](s128)
+    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 33
+    ; GFX9: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[UV]], [[C]](s32)
+    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 31
+    ; GFX9: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[UV1]], [[C1]](s32)
+    ; GFX9: [[OR:%[0-9]+]]:_(s64) = G_OR [[LSHR]], [[SHL]]
+    ; GFX9: [[LSHR1:%[0-9]+]]:_(s64) = G_LSHR [[UV1]], [[C]](s32)
+    ; GFX9: [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[OR]](s64), [[LSHR1]](s64)
+    ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[MV]](s128)
+    %0:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    %1:_(s32) = G_CONSTANT i32 33
+    %3:_(s128) = G_LSHR %0, %1
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %3
+...
+
+---
+name: test_lshr_s128_s32_127
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4
+
+    ; SI-LABEL: name: test_lshr_s128_s32_127
+    ; SI: [[COPY:%[0-9]+]]:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; SI: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](s128)
+    ; SI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 63
+    ; SI: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[UV1]], [[C]](s32)
+    ; SI: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; SI: [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[LSHR]](s64), [[C1]](s64)
+    ; SI: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[MV]](s128)
+    ; VI-LABEL: name: test_lshr_s128_s32_127
+    ; VI: [[COPY:%[0-9]+]]:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; VI: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](s128)
+    ; VI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 63
+    ; VI: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[UV1]], [[C]](s32)
+    ; VI: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; VI: [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[LSHR]](s64), [[C1]](s64)
+    ; VI: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[MV]](s128)
+    ; GFX9-LABEL: name: test_lshr_s128_s32_127
+    ; GFX9: [[COPY:%[0-9]+]]:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; GFX9: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](s128)
+    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 63
+    ; GFX9: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[UV1]], [[C]](s32)
+    ; GFX9: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; GFX9: [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[LSHR]](s64), [[C1]](s64)
+    ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[MV]](s128)
+    %0:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    %1:_(s32) = G_CONSTANT i32 127
+    %3:_(s128) = G_LSHR %0, %1
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %3
+...
+
+---
+name: test_lshr_s256_s256
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8
+
+    ; SI-LABEL: name: test_lshr_s256_s256
+    ; SI: [[COPY:%[0-9]+]]:_(s256) = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+    ; SI: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr8
+    ; SI: [[ZEXT:%[0-9]+]]:_(s256) = G_ZEXT [[COPY1]](s32)
+    ; SI: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[ZEXT]](s256)
+    ; SI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 128
+    ; SI: [[UV:%[0-9]+]]:_(s128), [[UV1:%[0-9]+]]:_(s128) = G_UNMERGE_VALUES [[COPY]](s256)
+    ; SI: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[TRUNC]], [[C]]
+    ; SI: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[C]], [[TRUNC]]
+    ; SI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; SI: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[TRUNC]](s32), [[C]]
+    ; SI: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[TRUNC]](s32), [[C1]]
+    ; SI: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 64
+    ; SI: [[UV2:%[0-9]+]]:_(s64), [[UV3:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[UV1]](s128)
+    ; SI: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[TRUNC]], [[C2]]
+    ; SI: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[TRUNC]]
+    ; SI: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; SI: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[TRUNC]](s32), [[C2]]
+    ; SI: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[TRUNC]](s32), [[C3]]
+    ; SI: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[UV3]], [[TRUNC]](s32)
+    ; SI: [[LSHR1:%[0-9]+]]:_(s64) = G_LSHR [[UV2]], [[TRUNC]](s32)
+    ; SI: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[UV3]], [[SUB3]](s32)
+    ; SI: [[OR:%[0-9]+]]:_(s64) = G_OR [[LSHR1]], [[SHL]]
+    ; SI: [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; SI: [[LSHR2:%[0-9]+]]:_(s64) = G_LSHR [[UV3]], [[SUB2]](s32)
+    ; SI: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[ICMP2]](s1), [[OR]], [[LSHR2]]
+    ; SI: [[SELECT1:%[0-9]+]]:_(s64) = G_SELECT [[ICMP3]](s1), [[UV2]], [[SELECT]]
+    ; SI: [[SELECT2:%[0-9]+]]:_(s64) = G_SELECT [[ICMP2]](s1), [[LSHR]], [[C4]]
+    ; SI: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 64
+    ; SI: [[UV4:%[0-9]+]]:_(s64), [[UV5:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[UV]](s128)
+    ; SI: [[SUB4:%[0-9]+]]:_(s32) = G_SUB [[TRUNC]], [[C5]]
+    ; SI: [[SUB5:%[0-9]+]]:_(s32) = G_SUB [[C5]], [[TRUNC]]
+    ; SI: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; SI: [[ICMP4:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[TRUNC]](s32), [[C5]]
+    ; SI: [[ICMP5:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[TRUNC]](s32), [[C6]]
+    ; SI: [[LSHR3:%[0-9]+]]:_(s64) = G_LSHR [[UV5]], [[TRUNC]](s32)
+    ; SI: [[LSHR4:%[0-9]+]]:_(s64) = G_LSHR [[UV4]], [[TRUNC]](s32)
+    ; SI: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[UV5]], [[SUB5]](s32)
+    ; SI: [[OR1:%[0-9]+]]:_(s64) = G_OR [[LSHR4]], [[SHL1]]
+    ; SI: [[C7:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; SI: [[LSHR5:%[0-9]+]]:_(s64) = G_LSHR [[UV5]], [[SUB4]](s32)
+    ; SI: [[SELECT3:%[0-9]+]]:_(s64) = G_SELECT [[ICMP4]](s1), [[OR1]], [[LSHR5]]
+    ; SI: [[SELECT4:%[0-9]+]]:_(s64) = G_SELECT [[ICMP5]](s1), [[UV4]], [[SELECT3]]
+    ; SI: [[SELECT5:%[0-9]+]]:_(s64) = G_SELECT [[ICMP4]](s1), [[LSHR3]], [[C7]]
+    ; SI: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 64
+    ; SI: [[UV6:%[0-9]+]]:_(s64), [[UV7:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[UV1]](s128)
+    ; SI: [[SUB6:%[0-9]+]]:_(s32) = G_SUB [[SUB1]], [[C8]]
+    ; SI: [[SUB7:%[0-9]+]]:_(s32) = G_SUB [[C8]], [[SUB1]]
+    ; SI: [[C9:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; SI: [[ICMP6:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[SUB1]](s32), [[C8]]
+    ; SI: [[ICMP7:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[SUB1]](s32), [[C9]]
+    ; SI: [[SHL2:%[0-9]+]]:_(s64) = G_SHL [[UV7]], [[SUB1]](s32)
+    ; SI: [[SHL3:%[0-9]+]]:_(s64) = G_SHL [[UV7]], [[SUB1]](s32)
+    ; SI: [[LSHR6:%[0-9]+]]:_(s64) = G_LSHR [[UV6]], [[SUB7]](s32)
+    ; SI: [[OR2:%[0-9]+]]:_(s64) = G_OR [[SHL3]], [[LSHR6]]
+    ; SI: [[C10:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; SI: [[SHL4:%[0-9]+]]:_(s64) = G_SHL [[UV6]], [[SUB6]](s32)
+    ; SI: [[SELECT6:%[0-9]+]]:_(s64) = G_SELECT [[ICMP6]](s1), [[SHL2]], [[C10]]
+    ; SI: [[SELECT7:%[0-9]+]]:_(s64) = G_SELECT [[ICMP6]](s1), [[OR2]], [[SHL4]]
+    ; SI: [[SELECT8:%[0-9]+]]:_(s64) = G_SELECT [[ICMP7]](s1), [[UV7]], [[SELECT7]]
+    ; SI: [[OR3:%[0-9]+]]:_(s64) = G_OR [[SELECT4]], [[SELECT6]]
+    ; SI: [[OR4:%[0-9]+]]:_(s64) = G_OR [[SELECT5]], [[SELECT8]]
+    ; SI: [[C11:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; SI: [[C12:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; SI: [[C13:%[0-9]+]]:_(s32) = G_CONSTANT i32 64
+    ; SI: [[UV8:%[0-9]+]]:_(s64), [[UV9:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[UV1]](s128)
+    ; SI: [[SUB8:%[0-9]+]]:_(s32) = G_SUB [[SUB]], [[C13]]
+    ; SI: [[SUB9:%[0-9]+]]:_(s32) = G_SUB [[C13]], [[SUB]]
+    ; SI: [[C14:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; SI: [[ICMP8:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[SUB]](s32), [[C13]]
+    ; SI: [[ICMP9:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[SUB]](s32), [[C14]]
+    ; SI: [[LSHR7:%[0-9]+]]:_(s64) = G_LSHR [[UV9]], [[SUB]](s32)
+    ; SI: [[LSHR8:%[0-9]+]]:_(s64) = G_LSHR [[UV8]], [[SUB]](s32)
+    ; SI: [[SHL5:%[0-9]+]]:_(s64) = G_SHL [[UV9]], [[SUB9]](s32)
+    ; SI: [[OR5:%[0-9]+]]:_(s64) = G_OR [[LSHR8]], [[SHL5]]
+    ; SI: [[C15:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; SI: [[LSHR9:%[0-9]+]]:_(s64) = G_LSHR [[UV9]], [[SUB8]](s32)
+    ; SI: [[SELECT9:%[0-9]+]]:_(s64) = G_SELECT [[ICMP8]](s1), [[OR5]], [[LSHR9]]
+    ; SI: [[SELECT10:%[0-9]+]]:_(s64) = G_SELECT [[ICMP9]](s1), [[UV8]], [[SELECT9]]
+    ; SI: [[SELECT11:%[0-9]+]]:_(s64) = G_SELECT [[ICMP8]](s1), [[LSHR7]], [[C15]]
+    ; SI: [[SELECT12:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[OR3]], [[SELECT10]]
+    ; SI: [[SELECT13:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[OR4]], [[SELECT11]]
+    ; SI: [[UV10:%[0-9]+]]:_(s64), [[UV11:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[UV]](s128)
+    ; SI: [[SELECT14:%[0-9]+]]:_(s64) = G_SELECT [[ICMP1]](s1), [[UV10]], [[SELECT12]]
+    ; SI: [[SELECT15:%[0-9]+]]:_(s64) = G_SELECT [[ICMP1]](s1), [[UV11]], [[SELECT13]]
+    ; SI: [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[SELECT14]](s64), [[SELECT15]](s64)
+    ; SI: [[SELECT16:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[SELECT1]], [[C11]]
+    ; SI: [[SELECT17:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[SELECT2]], [[C12]]
+    ; SI: [[MV1:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[SELECT16]](s64), [[SELECT17]](s64)
+    ; SI: [[MV2:%[0-9]+]]:_(s256) = G_MERGE_VALUES [[MV]](s128), [[MV1]](s128)
+    ; SI: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY [[MV2]](s256)
+    ; VI-LABEL: name: test_lshr_s256_s256
+    ; VI: [[COPY:%[0-9]+]]:_(s256) = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+    ; VI: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr8
+    ; VI: [[ZEXT:%[0-9]+]]:_(s256) = G_ZEXT [[COPY1]](s32)
+    ; VI: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[ZEXT]](s256)
+    ; VI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 128
+    ; VI: [[UV:%[0-9]+]]:_(s128), [[UV1:%[0-9]+]]:_(s128) = G_UNMERGE_VALUES [[COPY]](s256)
+    ; VI: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[TRUNC]], [[C]]
+    ; VI: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[C]], [[TRUNC]]
+    ; VI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; VI: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[TRUNC]](s32), [[C]]
+    ; VI: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[TRUNC]](s32), [[C1]]
+    ; VI: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 64
+    ; VI: [[UV2:%[0-9]+]]:_(s64), [[UV3:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[UV1]](s128)
+    ; VI: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[TRUNC]], [[C2]]
+    ; VI: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[TRUNC]]
+    ; VI: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; VI: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[TRUNC]](s32), [[C2]]
+    ; VI: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[TRUNC]](s32), [[C3]]
+    ; VI: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[UV3]], [[TRUNC]](s32)
+    ; VI: [[LSHR1:%[0-9]+]]:_(s64) = G_LSHR [[UV2]], [[TRUNC]](s32)
+    ; VI: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[UV3]], [[SUB3]](s32)
+    ; VI: [[OR:%[0-9]+]]:_(s64) = G_OR [[LSHR1]], [[SHL]]
+    ; VI: [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; VI: [[LSHR2:%[0-9]+]]:_(s64) = G_LSHR [[UV3]], [[SUB2]](s32)
+    ; VI: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[ICMP2]](s1), [[OR]], [[LSHR2]]
+    ; VI: [[SELECT1:%[0-9]+]]:_(s64) = G_SELECT [[ICMP3]](s1), [[UV2]], [[SELECT]]
+    ; VI: [[SELECT2:%[0-9]+]]:_(s64) = G_SELECT [[ICMP2]](s1), [[LSHR]], [[C4]]
+    ; VI: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 64
+    ; VI: [[UV4:%[0-9]+]]:_(s64), [[UV5:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[UV]](s128)
+    ; VI: [[SUB4:%[0-9]+]]:_(s32) = G_SUB [[TRUNC]], [[C5]]
+    ; VI: [[SUB5:%[0-9]+]]:_(s32) = G_SUB [[C5]], [[TRUNC]]
+    ; VI: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; VI: [[ICMP4:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[TRUNC]](s32), [[C5]]
+    ; VI: [[ICMP5:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[TRUNC]](s32), [[C6]]
+    ; VI: [[LSHR3:%[0-9]+]]:_(s64) = G_LSHR [[UV5]], [[TRUNC]](s32)
+    ; VI: [[LSHR4:%[0-9]+]]:_(s64) = G_LSHR [[UV4]], [[TRUNC]](s32)
+    ; VI: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[UV5]], [[SUB5]](s32)
+    ; VI: [[OR1:%[0-9]+]]:_(s64) = G_OR [[LSHR4]], [[SHL1]]
+    ; VI: [[C7:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; VI: [[LSHR5:%[0-9]+]]:_(s64) = G_LSHR [[UV5]], [[SUB4]](s32)
+    ; VI: [[SELECT3:%[0-9]+]]:_(s64) = G_SELECT [[ICMP4]](s1), [[OR1]], [[LSHR5]]
+    ; VI: [[SELECT4:%[0-9]+]]:_(s64) = G_SELECT [[ICMP5]](s1), [[UV4]], [[SELECT3]]
+    ; VI: [[SELECT5:%[0-9]+]]:_(s64) = G_SELECT [[ICMP4]](s1), [[LSHR3]], [[C7]]
+    ; VI: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 64
+    ; VI: [[UV6:%[0-9]+]]:_(s64), [[UV7:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[UV1]](s128)
+    ; VI: [[SUB6:%[0-9]+]]:_(s32) = G_SUB [[SUB1]], [[C8]]
+    ; VI: [[SUB7:%[0-9]+]]:_(s32) = G_SUB [[C8]], [[SUB1]]
+    ; VI: [[C9:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; VI: [[ICMP6:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[SUB1]](s32), [[C8]]
+    ; VI: [[ICMP7:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[SUB1]](s32), [[C9]]
+    ; VI: [[SHL2:%[0-9]+]]:_(s64) = G_SHL [[UV7]], [[SUB1]](s32)
+    ; VI: [[SHL3:%[0-9]+]]:_(s64) = G_SHL [[UV7]], [[SUB1]](s32)
+    ; VI: [[LSHR6:%[0-9]+]]:_(s64) = G_LSHR [[UV6]], [[SUB7]](s32)
+    ; VI: [[OR2:%[0-9]+]]:_(s64) = G_OR [[SHL3]], [[LSHR6]]
+    ; VI: [[C10:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; VI: [[SHL4:%[0-9]+]]:_(s64) = G_SHL [[UV6]], [[SUB6]](s32)
+    ; VI: [[SELECT6:%[0-9]+]]:_(s64) = G_SELECT [[ICMP6]](s1), [[SHL2]], [[C10]]
+    ; VI: [[SELECT7:%[0-9]+]]:_(s64) = G_SELECT [[ICMP6]](s1), [[OR2]], [[SHL4]]
+    ; VI: [[SELECT8:%[0-9]+]]:_(s64) = G_SELECT [[ICMP7]](s1), [[UV7]], [[SELECT7]]
+    ; VI: [[OR3:%[0-9]+]]:_(s64) = G_OR [[SELECT4]], [[SELECT6]]
+    ; VI: [[OR4:%[0-9]+]]:_(s64) = G_OR [[SELECT5]], [[SELECT8]]
+    ; VI: [[C11:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; VI: [[C12:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; VI: [[C13:%[0-9]+]]:_(s32) = G_CONSTANT i32 64
+    ; VI: [[UV8:%[0-9]+]]:_(s64), [[UV9:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[UV1]](s128)
+    ; VI: [[SUB8:%[0-9]+]]:_(s32) = G_SUB [[SUB]], [[C13]]
+    ; VI: [[SUB9:%[0-9]+]]:_(s32) = G_SUB [[C13]], [[SUB]]
+    ; VI: [[C14:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; VI: [[ICMP8:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[SUB]](s32), [[C13]]
+    ; VI: [[ICMP9:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[SUB]](s32), [[C14]]
+    ; VI: [[LSHR7:%[0-9]+]]:_(s64) = G_LSHR [[UV9]], [[SUB]](s32)
+    ; VI: [[LSHR8:%[0-9]+]]:_(s64) = G_LSHR [[UV8]], [[SUB]](s32)
+    ; VI: [[SHL5:%[0-9]+]]:_(s64) = G_SHL [[UV9]], [[SUB9]](s32)
+    ; VI: [[OR5:%[0-9]+]]:_(s64) = G_OR [[LSHR8]], [[SHL5]]
+    ; VI: [[C15:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; VI: [[LSHR9:%[0-9]+]]:_(s64) = G_LSHR [[UV9]], [[SUB8]](s32)
+    ; VI: [[SELECT9:%[0-9]+]]:_(s64) = G_SELECT [[ICMP8]](s1), [[OR5]], [[LSHR9]]
+    ; VI: [[SELECT10:%[0-9]+]]:_(s64) = G_SELECT [[ICMP9]](s1), [[UV8]], [[SELECT9]]
+    ; VI: [[SELECT11:%[0-9]+]]:_(s64) = G_SELECT [[ICMP8]](s1), [[LSHR7]], [[C15]]
+    ; VI: [[SELECT12:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[OR3]], [[SELECT10]]
+    ; VI: [[SELECT13:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[OR4]], [[SELECT11]]
+    ; VI: [[UV10:%[0-9]+]]:_(s64), [[UV11:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[UV]](s128)
+    ; VI: [[SELECT14:%[0-9]+]]:_(s64) = G_SELECT [[ICMP1]](s1), [[UV10]], [[SELECT12]]
+    ; VI: [[SELECT15:%[0-9]+]]:_(s64) = G_SELECT [[ICMP1]](s1), [[UV11]], [[SELECT13]]
+    ; VI: [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[SELECT14]](s64), [[SELECT15]](s64)
+    ; VI: [[SELECT16:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[SELECT1]], [[C11]]
+    ; VI: [[SELECT17:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[SELECT2]], [[C12]]
+    ; VI: [[MV1:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[SELECT16]](s64), [[SELECT17]](s64)
+    ; VI: [[MV2:%[0-9]+]]:_(s256) = G_MERGE_VALUES [[MV]](s128), [[MV1]](s128)
+    ; VI: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY [[MV2]](s256)
+    ; GFX9-LABEL: name: test_lshr_s256_s256
+    ; GFX9: [[COPY:%[0-9]+]]:_(s256) = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+    ; GFX9: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr8
+    ; GFX9: [[ZEXT:%[0-9]+]]:_(s256) = G_ZEXT [[COPY1]](s32)
+    ; GFX9: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[ZEXT]](s256)
+    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 128
+    ; GFX9: [[UV:%[0-9]+]]:_(s128), [[UV1:%[0-9]+]]:_(s128) = G_UNMERGE_VALUES [[COPY]](s256)
+    ; GFX9: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[TRUNC]], [[C]]
+    ; GFX9: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[C]], [[TRUNC]]
+    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; GFX9: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[TRUNC]](s32), [[C]]
+    ; GFX9: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[TRUNC]](s32), [[C1]]
+    ; GFX9: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 64
+    ; GFX9: [[UV2:%[0-9]+]]:_(s64), [[UV3:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[UV1]](s128)
+    ; GFX9: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[TRUNC]], [[C2]]
+    ; GFX9: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[TRUNC]]
+    ; GFX9: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; GFX9: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[TRUNC]](s32), [[C2]]
+    ; GFX9: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[TRUNC]](s32), [[C3]]
+    ; GFX9: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[UV3]], [[TRUNC]](s32)
+    ; GFX9: [[LSHR1:%[0-9]+]]:_(s64) = G_LSHR [[UV2]], [[TRUNC]](s32)
+    ; GFX9: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[UV3]], [[SUB3]](s32)
+    ; GFX9: [[OR:%[0-9]+]]:_(s64) = G_OR [[LSHR1]], [[SHL]]
+    ; GFX9: [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; GFX9: [[LSHR2:%[0-9]+]]:_(s64) = G_LSHR [[UV3]], [[SUB2]](s32)
+    ; GFX9: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[ICMP2]](s1), [[OR]], [[LSHR2]]
+    ; GFX9: [[SELECT1:%[0-9]+]]:_(s64) = G_SELECT [[ICMP3]](s1), [[UV2]], [[SELECT]]
+    ; GFX9: [[SELECT2:%[0-9]+]]:_(s64) = G_SELECT [[ICMP2]](s1), [[LSHR]], [[C4]]
+    ; GFX9: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 64
+    ; GFX9: [[UV4:%[0-9]+]]:_(s64), [[UV5:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[UV]](s128)
+    ; GFX9: [[SUB4:%[0-9]+]]:_(s32) = G_SUB [[TRUNC]], [[C5]]
+    ; GFX9: [[SUB5:%[0-9]+]]:_(s32) = G_SUB [[C5]], [[TRUNC]]
+    ; GFX9: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; GFX9: [[ICMP4:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[TRUNC]](s32), [[C5]]
+    ; GFX9: [[ICMP5:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[TRUNC]](s32), [[C6]]
+    ; GFX9: [[LSHR3:%[0-9]+]]:_(s64) = G_LSHR [[UV5]], [[TRUNC]](s32)
+    ; GFX9: [[LSHR4:%[0-9]+]]:_(s64) = G_LSHR [[UV4]], [[TRUNC]](s32)
+    ; GFX9: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[UV5]], [[SUB5]](s32)
+    ; GFX9: [[OR1:%[0-9]+]]:_(s64) = G_OR [[LSHR4]], [[SHL1]]
+    ; GFX9: [[C7:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; GFX9: [[LSHR5:%[0-9]+]]:_(s64) = G_LSHR [[UV5]], [[SUB4]](s32)
+    ; GFX9: [[SELECT3:%[0-9]+]]:_(s64) = G_SELECT [[ICMP4]](s1), [[OR1]], [[LSHR5]]
+    ; GFX9: [[SELECT4:%[0-9]+]]:_(s64) = G_SELECT [[ICMP5]](s1), [[UV4]], [[SELECT3]]
+    ; GFX9: [[SELECT5:%[0-9]+]]:_(s64) = G_SELECT [[ICMP4]](s1), [[LSHR3]], [[C7]]
+    ; GFX9: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 64
+    ; GFX9: [[UV6:%[0-9]+]]:_(s64), [[UV7:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[UV1]](s128)
+    ; GFX9: [[SUB6:%[0-9]+]]:_(s32) = G_SUB [[SUB1]], [[C8]]
+    ; GFX9: [[SUB7:%[0-9]+]]:_(s32) = G_SUB [[C8]], [[SUB1]]
+    ; GFX9: [[C9:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; GFX9: [[ICMP6:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[SUB1]](s32), [[C8]]
+    ; GFX9: [[ICMP7:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[SUB1]](s32), [[C9]]
+    ; GFX9: [[SHL2:%[0-9]+]]:_(s64) = G_SHL [[UV7]], [[SUB1]](s32)
+    ; GFX9: [[SHL3:%[0-9]+]]:_(s64) = G_SHL [[UV7]], [[SUB1]](s32)
+    ; GFX9: [[LSHR6:%[0-9]+]]:_(s64) = G_LSHR [[UV6]], [[SUB7]](s32)
+    ; GFX9: [[OR2:%[0-9]+]]:_(s64) = G_OR [[SHL3]], [[LSHR6]]
+    ; GFX9: [[C10:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; GFX9: [[SHL4:%[0-9]+]]:_(s64) = G_SHL [[UV6]], [[SUB6]](s32)
+    ; GFX9: [[SELECT6:%[0-9]+]]:_(s64) = G_SELECT [[ICMP6]](s1), [[SHL2]], [[C10]]
+    ; GFX9: [[SELECT7:%[0-9]+]]:_(s64) = G_SELECT [[ICMP6]](s1), [[OR2]], [[SHL4]]
+    ; GFX9: [[SELECT8:%[0-9]+]]:_(s64) = G_SELECT [[ICMP7]](s1), [[UV7]], [[SELECT7]]
+    ; GFX9: [[OR3:%[0-9]+]]:_(s64) = G_OR [[SELECT4]], [[SELECT6]]
+    ; GFX9: [[OR4:%[0-9]+]]:_(s64) = G_OR [[SELECT5]], [[SELECT8]]
+    ; GFX9: [[C11:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; GFX9: [[C12:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; GFX9: [[C13:%[0-9]+]]:_(s32) = G_CONSTANT i32 64
+    ; GFX9: [[UV8:%[0-9]+]]:_(s64), [[UV9:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[UV1]](s128)
+    ; GFX9: [[SUB8:%[0-9]+]]:_(s32) = G_SUB [[SUB]], [[C13]]
+    ; GFX9: [[SUB9:%[0-9]+]]:_(s32) = G_SUB [[C13]], [[SUB]]
+    ; GFX9: [[C14:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; GFX9: [[ICMP8:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[SUB]](s32), [[C13]]
+    ; GFX9: [[ICMP9:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[SUB]](s32), [[C14]]
+    ; GFX9: [[LSHR7:%[0-9]+]]:_(s64) = G_LSHR [[UV9]], [[SUB]](s32)
+    ; GFX9: [[LSHR8:%[0-9]+]]:_(s64) = G_LSHR [[UV8]], [[SUB]](s32)
+    ; GFX9: [[SHL5:%[0-9]+]]:_(s64) = G_SHL [[UV9]], [[SUB9]](s32)
+    ; GFX9: [[OR5:%[0-9]+]]:_(s64) = G_OR [[LSHR8]], [[SHL5]]
+    ; GFX9: [[C15:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; GFX9: [[LSHR9:%[0-9]+]]:_(s64) = G_LSHR [[UV9]], [[SUB8]](s32)
+    ; GFX9: [[SELECT9:%[0-9]+]]:_(s64) = G_SELECT [[ICMP8]](s1), [[OR5]], [[LSHR9]]
+    ; GFX9: [[SELECT10:%[0-9]+]]:_(s64) = G_SELECT [[ICMP9]](s1), [[UV8]], [[SELECT9]]
+    ; GFX9: [[SELECT11:%[0-9]+]]:_(s64) = G_SELECT [[ICMP8]](s1), [[LSHR7]], [[C15]]
+    ; GFX9: [[SELECT12:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[OR3]], [[SELECT10]]
+    ; GFX9: [[SELECT13:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[OR4]], [[SELECT11]]
+    ; GFX9: [[UV10:%[0-9]+]]:_(s64), [[UV11:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[UV]](s128)
+    ; GFX9: [[SELECT14:%[0-9]+]]:_(s64) = G_SELECT [[ICMP1]](s1), [[UV10]], [[SELECT12]]
+    ; GFX9: [[SELECT15:%[0-9]+]]:_(s64) = G_SELECT [[ICMP1]](s1), [[UV11]], [[SELECT13]]
+    ; GFX9: [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[SELECT14]](s64), [[SELECT15]](s64)
+    ; GFX9: [[SELECT16:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[SELECT1]], [[C11]]
+    ; GFX9: [[SELECT17:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[SELECT2]], [[C12]]
+    ; GFX9: [[MV1:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[SELECT16]](s64), [[SELECT17]](s64)
+    ; GFX9: [[MV2:%[0-9]+]]:_(s256) = G_MERGE_VALUES [[MV]](s128), [[MV1]](s128)
+    ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY [[MV2]](s256)
+    %0:_(s256) = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+    %1:_(s32) = COPY $vgpr8
+    %2:_(s256) = G_ZEXT %1
+    %3:_(s256) = G_LSHR %0, %2
+    $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7  = COPY %3
+...
+
+---
+name: test_lshr_v2s128_v2s32
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, $vgpr4_vgpr5
+
+    ; SI-LABEL: name: test_lshr_v2s128_v2s32
+    ; SI: [[COPY:%[0-9]+]]:_(<2 x s128>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+    ; SI: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr4_vgpr5
+    ; SI: [[UV:%[0-9]+]]:_(s128), [[UV1:%[0-9]+]]:_(s128) = G_UNMERGE_VALUES [[COPY]](<2 x s128>)
+    ; SI: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<2 x s32>)
+    ; SI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 64
+    ; SI: [[UV4:%[0-9]+]]:_(s64), [[UV5:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[UV]](s128)
+    ; SI: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[UV2]], [[C]]
+    ; SI: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[C]], [[UV2]]
+    ; SI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; SI: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[UV2]](s32), [[C]]
+    ; SI: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[UV2]](s32), [[C1]]
+    ; SI: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[UV5]], [[UV2]](s32)
+    ; SI: [[LSHR1:%[0-9]+]]:_(s64) = G_LSHR [[UV4]], [[UV2]](s32)
+    ; SI: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[UV5]], [[SUB1]](s32)
+    ; SI: [[OR:%[0-9]+]]:_(s64) = G_OR [[LSHR1]], [[SHL]]
+    ; SI: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; SI: [[LSHR2:%[0-9]+]]:_(s64) = G_LSHR [[UV5]], [[SUB]](s32)
+    ; SI: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[OR]], [[LSHR2]]
+    ; SI: [[SELECT1:%[0-9]+]]:_(s64) = G_SELECT [[ICMP1]](s1), [[UV4]], [[SELECT]]
+    ; SI: [[SELECT2:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[LSHR]], [[C2]]
+    ; SI: [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[SELECT1]](s64), [[SELECT2]](s64)
+    ; SI: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 64
+    ; SI: [[UV6:%[0-9]+]]:_(s64), [[UV7:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[UV1]](s128)
+    ; SI: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[UV3]], [[C3]]
+    ; SI: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[C3]], [[UV3]]
+    ; SI: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; SI: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[UV3]](s32), [[C3]]
+    ; SI: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[UV3]](s32), [[C4]]
+    ; SI: [[LSHR3:%[0-9]+]]:_(s64) = G_LSHR [[UV7]], [[UV3]](s32)
+    ; SI: [[LSHR4:%[0-9]+]]:_(s64) = G_LSHR [[UV6]], [[UV3]](s32)
+    ; SI: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[UV7]], [[SUB3]](s32)
+    ; SI: [[OR1:%[0-9]+]]:_(s64) = G_OR [[LSHR4]], [[SHL1]]
+    ; SI: [[C5:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; SI: [[LSHR5:%[0-9]+]]:_(s64) = G_LSHR [[UV7]], [[SUB2]](s32)
+    ; SI: [[SELECT3:%[0-9]+]]:_(s64) = G_SELECT [[ICMP2]](s1), [[OR1]], [[LSHR5]]
+    ; SI: [[SELECT4:%[0-9]+]]:_(s64) = G_SELECT [[ICMP3]](s1), [[UV6]], [[SELECT3]]
+    ; SI: [[SELECT5:%[0-9]+]]:_(s64) = G_SELECT [[ICMP2]](s1), [[LSHR3]], [[C5]]
+    ; SI: [[MV1:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[SELECT4]](s64), [[SELECT5]](s64)
+    ; SI: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s128>) = G_BUILD_VECTOR [[MV]](s128), [[MV1]](s128)
+    ; SI: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY [[BUILD_VECTOR]](<2 x s128>)
+    ; VI-LABEL: name: test_lshr_v2s128_v2s32
+    ; VI: [[COPY:%[0-9]+]]:_(<2 x s128>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+    ; VI: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr4_vgpr5
+    ; VI: [[UV:%[0-9]+]]:_(s128), [[UV1:%[0-9]+]]:_(s128) = G_UNMERGE_VALUES [[COPY]](<2 x s128>)
+    ; VI: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<2 x s32>)
+    ; VI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 64
+    ; VI: [[UV4:%[0-9]+]]:_(s64), [[UV5:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[UV]](s128)
+    ; VI: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[UV2]], [[C]]
+    ; VI: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[C]], [[UV2]]
+    ; VI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; VI: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[UV2]](s32), [[C]]
+    ; VI: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[UV2]](s32), [[C1]]
+    ; VI: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[UV5]], [[UV2]](s32)
+    ; VI: [[LSHR1:%[0-9]+]]:_(s64) = G_LSHR [[UV4]], [[UV2]](s32)
+    ; VI: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[UV5]], [[SUB1]](s32)
+    ; VI: [[OR:%[0-9]+]]:_(s64) = G_OR [[LSHR1]], [[SHL]]
+    ; VI: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; VI: [[LSHR2:%[0-9]+]]:_(s64) = G_LSHR [[UV5]], [[SUB]](s32)
+    ; VI: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[OR]], [[LSHR2]]
+    ; VI: [[SELECT1:%[0-9]+]]:_(s64) = G_SELECT [[ICMP1]](s1), [[UV4]], [[SELECT]]
+    ; VI: [[SELECT2:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[LSHR]], [[C2]]
+    ; VI: [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[SELECT1]](s64), [[SELECT2]](s64)
+    ; VI: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 64
+    ; VI: [[UV6:%[0-9]+]]:_(s64), [[UV7:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[UV1]](s128)
+    ; VI: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[UV3]], [[C3]]
+    ; VI: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[C3]], [[UV3]]
+    ; VI: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; VI: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[UV3]](s32), [[C3]]
+    ; VI: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[UV3]](s32), [[C4]]
+    ; VI: [[LSHR3:%[0-9]+]]:_(s64) = G_LSHR [[UV7]], [[UV3]](s32)
+    ; VI: [[LSHR4:%[0-9]+]]:_(s64) = G_LSHR [[UV6]], [[UV3]](s32)
+    ; VI: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[UV7]], [[SUB3]](s32)
+    ; VI: [[OR1:%[0-9]+]]:_(s64) = G_OR [[LSHR4]], [[SHL1]]
+    ; VI: [[C5:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; VI: [[LSHR5:%[0-9]+]]:_(s64) = G_LSHR [[UV7]], [[SUB2]](s32)
+    ; VI: [[SELECT3:%[0-9]+]]:_(s64) = G_SELECT [[ICMP2]](s1), [[OR1]], [[LSHR5]]
+    ; VI: [[SELECT4:%[0-9]+]]:_(s64) = G_SELECT [[ICMP3]](s1), [[UV6]], [[SELECT3]]
+    ; VI: [[SELECT5:%[0-9]+]]:_(s64) = G_SELECT [[ICMP2]](s1), [[LSHR3]], [[C5]]
+    ; VI: [[MV1:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[SELECT4]](s64), [[SELECT5]](s64)
+    ; VI: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s128>) = G_BUILD_VECTOR [[MV]](s128), [[MV1]](s128)
+    ; VI: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY [[BUILD_VECTOR]](<2 x s128>)
+    ; GFX9-LABEL: name: test_lshr_v2s128_v2s32
+    ; GFX9: [[COPY:%[0-9]+]]:_(<2 x s128>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+    ; GFX9: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr4_vgpr5
+    ; GFX9: [[UV:%[0-9]+]]:_(s128), [[UV1:%[0-9]+]]:_(s128) = G_UNMERGE_VALUES [[COPY]](<2 x s128>)
+    ; GFX9: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<2 x s32>)
+    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 64
+    ; GFX9: [[UV4:%[0-9]+]]:_(s64), [[UV5:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[UV]](s128)
+    ; GFX9: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[UV2]], [[C]]
+    ; GFX9: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[C]], [[UV2]]
+    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; GFX9: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[UV2]](s32), [[C]]
+    ; GFX9: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[UV2]](s32), [[C1]]
+    ; GFX9: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[UV5]], [[UV2]](s32)
+    ; GFX9: [[LSHR1:%[0-9]+]]:_(s64) = G_LSHR [[UV4]], [[UV2]](s32)
+    ; GFX9: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[UV5]], [[SUB1]](s32)
+    ; GFX9: [[OR:%[0-9]+]]:_(s64) = G_OR [[LSHR1]], [[SHL]]
+    ; GFX9: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; GFX9: [[LSHR2:%[0-9]+]]:_(s64) = G_LSHR [[UV5]], [[SUB]](s32)
+    ; GFX9: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[OR]], [[LSHR2]]
+    ; GFX9: [[SELECT1:%[0-9]+]]:_(s64) = G_SELECT [[ICMP1]](s1), [[UV4]], [[SELECT]]
+    ; GFX9: [[SELECT2:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[LSHR]], [[C2]]
+    ; GFX9: [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[SELECT1]](s64), [[SELECT2]](s64)
+    ; GFX9: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 64
+    ; GFX9: [[UV6:%[0-9]+]]:_(s64), [[UV7:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[UV1]](s128)
+    ; GFX9: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[UV3]], [[C3]]
+    ; GFX9: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[C3]], [[UV3]]
+    ; GFX9: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; GFX9: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[UV3]](s32), [[C3]]
+    ; GFX9: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[UV3]](s32), [[C4]]
+    ; GFX9: [[LSHR3:%[0-9]+]]:_(s64) = G_LSHR [[UV7]], [[UV3]](s32)
+    ; GFX9: [[LSHR4:%[0-9]+]]:_(s64) = G_LSHR [[UV6]], [[UV3]](s32)
+    ; GFX9: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[UV7]], [[SUB3]](s32)
+    ; GFX9: [[OR1:%[0-9]+]]:_(s64) = G_OR [[LSHR4]], [[SHL1]]
+    ; GFX9: [[C5:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; GFX9: [[LSHR5:%[0-9]+]]:_(s64) = G_LSHR [[UV7]], [[SUB2]](s32)
+    ; GFX9: [[SELECT3:%[0-9]+]]:_(s64) = G_SELECT [[ICMP2]](s1), [[OR1]], [[LSHR5]]
+    ; GFX9: [[SELECT4:%[0-9]+]]:_(s64) = G_SELECT [[ICMP3]](s1), [[UV6]], [[SELECT3]]
+    ; GFX9: [[SELECT5:%[0-9]+]]:_(s64) = G_SELECT [[ICMP2]](s1), [[LSHR3]], [[C5]]
+    ; GFX9: [[MV1:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[SELECT4]](s64), [[SELECT5]](s64)
+    ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s128>) = G_BUILD_VECTOR [[MV]](s128), [[MV1]](s128)
+    ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY [[BUILD_VECTOR]](<2 x s128>)
+    %0:_(<2 x s128>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+    %1:_(<2 x s32>) = COPY $vgpr4_vgpr5
+    %2:_(<2 x s128>) = G_LSHR %0, %1
+    $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7  = COPY %2
+...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/legalize-merge-values.mir b/test/CodeGen/AMDGPU/GlobalISel/legalize-merge-values.mir
new file mode 100644
index 0000000000000000000000000000000000000000..4a4d77e63d5718142ee46b821a9593a30a5e3ccd
--- /dev/null
+++ b/test/CodeGen/AMDGPU/GlobalISel/legalize-merge-values.mir
@@ -0,0 +1,156 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=tahiti -O0 -run-pass=legalizer %s -o - | FileCheck %s
+
+---
+name: test_merge_s16_s8_s8
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: test_merge_s16_s8_s8
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; CHECK: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; CHECK: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; CHECK: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY [[C2]](s32)
+    ; CHECK: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C3]]
+    ; CHECK: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY [[C1]](s32)
+    ; CHECK: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C4]]
+    ; CHECK: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[AND]](s32)
+    ; CHECK: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; CHECK: [[COPY2:%[0-9]+]]:_(s32) = COPY [[C]](s32)
+    ; CHECK: [[AND2:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C5]]
+    ; CHECK: [[COPY3:%[0-9]+]]:_(s32) = COPY [[SHL]](s32)
+    ; CHECK: [[OR:%[0-9]+]]:_(s32) = G_OR [[AND2]], [[COPY3]]
+    ; CHECK: [[COPY4:%[0-9]+]]:_(s32) = COPY [[OR]](s32)
+    ; CHECK: $vgpr0 = COPY [[COPY4]](s32)
+    %0:_(s8) = G_CONSTANT i8 0
+    %1:_(s8) = G_CONSTANT i8 1
+    %2:_(s16) = G_MERGE_VALUES %0, %1
+    %3:_(s32) = G_ANYEXT %2
+    $vgpr0 = COPY %3
+...
+
+---
+name: test_merge_s24_s8_s8_s8
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: test_merge_s24_s8_s8_s8
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; CHECK: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; CHECK: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; CHECK: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; CHECK: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 16777215
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY [[C3]](s32)
+    ; CHECK: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C4]]
+    ; CHECK: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY [[C1]](s32)
+    ; CHECK: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C5]]
+    ; CHECK: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[AND]](s32)
+    ; CHECK: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; CHECK: [[COPY2:%[0-9]+]]:_(s32) = COPY [[C]](s32)
+    ; CHECK: [[AND2:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C6]]
+    ; CHECK: [[COPY3:%[0-9]+]]:_(s32) = COPY [[SHL]](s32)
+    ; CHECK: [[OR:%[0-9]+]]:_(s32) = G_OR [[AND2]], [[COPY3]]
+    ; CHECK: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; CHECK: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 16777215
+    ; CHECK: [[COPY4:%[0-9]+]]:_(s32) = COPY [[C7]](s32)
+    ; CHECK: [[AND3:%[0-9]+]]:_(s32) = G_AND [[COPY4]], [[C8]]
+    ; CHECK: [[C9:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; CHECK: [[COPY5:%[0-9]+]]:_(s32) = COPY [[C2]](s32)
+    ; CHECK: [[AND4:%[0-9]+]]:_(s32) = G_AND [[COPY5]], [[C9]]
+    ; CHECK: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND4]], [[AND3]](s32)
+    ; CHECK: [[COPY6:%[0-9]+]]:_(s32) = COPY [[OR]](s32)
+    ; CHECK: [[COPY7:%[0-9]+]]:_(s32) = COPY [[SHL1]](s32)
+    ; CHECK: [[OR1:%[0-9]+]]:_(s32) = G_OR [[COPY6]], [[COPY7]]
+    ; CHECK: [[COPY8:%[0-9]+]]:_(s32) = COPY [[OR1]](s32)
+    ; CHECK: $vgpr0 = COPY [[COPY8]](s32)
+    %0:_(s8) = G_CONSTANT i8 0
+    %1:_(s8) = G_CONSTANT i8 1
+    %2:_(s8) = G_CONSTANT i8 2
+    %3:_(s24) = G_MERGE_VALUES %0, %1, %2
+    %4:_(s32) = G_ANYEXT %3
+    $vgpr0 = COPY %4
+...
+
+---
+name: test_merge_s32_s8_s8_s8_s8
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: test_merge_s32_s8_s8_s8_s8
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; CHECK: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; CHECK: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; CHECK: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 3
+    ; CHECK: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY [[C]](s32)
+    ; CHECK: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C4]]
+    ; CHECK: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY [[C1]](s32)
+    ; CHECK: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C5]]
+    ; CHECK: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; CHECK: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C6]](s32)
+    ; CHECK: [[OR:%[0-9]+]]:_(s32) = G_OR [[AND]], [[SHL]]
+    ; CHECK: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; CHECK: [[COPY2:%[0-9]+]]:_(s32) = COPY [[C2]](s32)
+    ; CHECK: [[AND2:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C7]]
+    ; CHECK: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; CHECK: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND2]], [[C8]](s32)
+    ; CHECK: [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
+    ; CHECK: [[C9:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; CHECK: [[COPY3:%[0-9]+]]:_(s32) = COPY [[C3]](s32)
+    ; CHECK: [[AND3:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[C9]]
+    ; CHECK: [[C10:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; CHECK: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C10]](s32)
+    ; CHECK: [[OR2:%[0-9]+]]:_(s32) = G_OR [[OR1]], [[SHL2]]
+    ; CHECK: $vgpr0 = COPY [[OR2]](s32)
+    %0:_(s8) = G_CONSTANT i8 0
+    %1:_(s8) = G_CONSTANT i8 1
+    %2:_(s8) = G_CONSTANT i8 2
+    %3:_(s8) = G_CONSTANT i8 3
+    %4:_(s32) = G_MERGE_VALUES %0, %1, %2, %3
+    $vgpr0 = COPY %4
+...
+
+---
+name: test_merge_s64_s32_s32
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+    ; CHECK-LABEL: name: test_merge_s64_s32_s32
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; CHECK: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[COPY]](s32), [[COPY1]](s32)
+    ; CHECK: $vgpr1_vgpr2 = COPY [[MV]](s64)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s64) = G_MERGE_VALUES %0, %1
+    $vgpr1_vgpr2 = COPY %2
+...
+
+---
+name: test_merge_s64_s16_s16_s16_s16
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3
+    ; CHECK-LABEL: name: test_merge_s64_s16_s16_s16_s16
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; CHECK: [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+    ; CHECK: [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr3
+    ; CHECK: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+    ; CHECK: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32)
+    ; CHECK: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[COPY2]](s32)
+    ; CHECK: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[COPY3]](s32)
+    ; CHECK: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[TRUNC]](s16), [[TRUNC1]](s16), [[TRUNC2]](s16), [[TRUNC3]](s16)
+    ; CHECK: $vgpr1_vgpr2 = COPY [[MV]](s64)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s32) = COPY $vgpr2
+    %3:_(s32) = COPY $vgpr3
+    %4:_(s16) = G_TRUNC %0
+    %5:_(s16) = G_TRUNC %1
+    %6:_(s16) = G_TRUNC %2
+    %7:_(s16) = G_TRUNC %3
+    %8:_(s64) = G_MERGE_VALUES %4, %5, %6, %7
+    $vgpr1_vgpr2 = COPY %8
+...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/legalize-mul.mir b/test/CodeGen/AMDGPU/GlobalISel/legalize-mul.mir
index e24177234b7dcef7e321076b01c21237b192ff87..e7b79490b779c4abafb2e980886876dc9fcb8c81 100644
--- a/test/CodeGen/AMDGPU/GlobalISel/legalize-mul.mir
+++ b/test/CodeGen/AMDGPU/GlobalISel/legalize-mul.mir
@@ -2,17 +2,65 @@
 # RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -O0 -run-pass=legalizer %s -o - | FileCheck %s
 
 ---
-name: test_mul
+name: test_mul_s32
 body: |
   bb.0:
     liveins: $vgpr0, $vgpr1
 
-    ; CHECK-LABEL: name: test_mul
+    ; CHECK-LABEL: name: test_mul_s32
     ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
     ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
     ; CHECK: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[COPY]], [[COPY1]]
+    ; CHECK: $vgpr0 = COPY [[MUL]](s32)
     %0:_(s32) = COPY $vgpr0
     %1:_(s32) = COPY $vgpr1
     %2:_(s32) = G_MUL %0, %1
     $vgpr0 = COPY %2
 ...
+
+---
+name: test_mul_v2s32
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
+
+    ; CHECK-LABEL: name: test_mul_v2s32
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    ; CHECK: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr2_vgpr3
+    ; CHECK: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
+    ; CHECK: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<2 x s32>)
+    ; CHECK: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[UV]], [[UV2]]
+    ; CHECK: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[UV1]], [[UV3]]
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[MUL]](s32), [[MUL1]](s32)
+    ; CHECK: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
+    %0:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    %1:_(<2 x s32>) = COPY $vgpr2_vgpr3
+    %2:_(<2 x s32>) = G_MUL %0, %1
+    $vgpr0_vgpr1 = COPY %2
+...
+
+---
+name: test_mul_s64
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
+
+    ; CHECK-LABEL: name: test_mul_s64
+    ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s64) = COPY $vgpr2_vgpr3
+    ; CHECK: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](s64)
+    ; CHECK: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](s64)
+    ; CHECK: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[UV]], [[UV2]]
+    ; CHECK: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[UV1]], [[UV3]]
+    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 32
+    ; CHECK: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[UMULH]](s32)
+    ; CHECK: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[C]](s64)
+    ; CHECK: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[ANYEXT]], [[TRUNC]](s32)
+    ; CHECK: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[MUL]](s32)
+    ; CHECK: [[OR:%[0-9]+]]:_(s64) = G_OR [[ZEXT]], [[SHL]]
+    ; CHECK: $vgpr0_vgpr1 = COPY [[OR]](s64)
+    %0:_(s64) = COPY $vgpr0_vgpr1
+    %1:_(s64) = COPY $vgpr2_vgpr3
+    %2:_(s64) = G_MUL %0, %1
+    $vgpr0_vgpr1 = COPY %2
+...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/legalize-or.mir b/test/CodeGen/AMDGPU/GlobalISel/legalize-or.mir
index 6e988dabe172f278923f2c64c9bd694a2df71e8a..0da10d32391d887f1ae8231c4c62db1989cef3cc 100644
--- a/test/CodeGen/AMDGPU/GlobalISel/legalize-or.mir
+++ b/test/CodeGen/AMDGPU/GlobalISel/legalize-or.mir
@@ -2,17 +2,255 @@
 # RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -O0 -run-pass=legalizer %s -o - | FileCheck %s
 
 ---
-name:            test_or
+name: test_or_s32
 body: |
   bb.0:
     liveins: $vgpr0, $vgpr1
 
-    ; CHECK-LABEL: name: test_or
+    ; CHECK-LABEL: name: test_or_s32
     ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
     ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
     ; CHECK: [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY]], [[COPY1]]
+    ; CHECK: $vgpr0 = COPY [[OR]](s32)
     %0:_(s32) = COPY $vgpr0
     %1:_(s32) = COPY $vgpr1
     %2:_(s32) = G_OR %0, %1
     $vgpr0 = COPY %2
 ...
+
+---
+name: test_or_s1
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+
+    ; CHECK-LABEL: name: test_or_s1
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; CHECK: [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY]], [[COPY1]]
+    ; CHECK: S_NOP 0, implicit [[OR]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s32) = G_CONSTANT i32 0
+    %3:_(s1) = G_ICMP intpred(ne), %0, %2
+    %4:_(s1) = G_ICMP intpred(ne), %1, %2
+    %5:_(s32) = G_OR %0, %1
+    S_NOP 0, implicit %5
+...
+
+---
+name: test_or_s64
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
+
+    ; CHECK-LABEL: name: test_or_s64
+    ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s64) = COPY $vgpr2_vgpr3
+    ; CHECK: [[OR:%[0-9]+]]:_(s64) = G_OR [[COPY]], [[COPY1]]
+    ; CHECK: $vgpr0_vgpr1 = COPY [[OR]](s64)
+    %0:_(s64) = COPY $vgpr0_vgpr1
+    %1:_(s64) = COPY $vgpr2_vgpr3
+    %2:_(s64) = G_OR %0, %1
+    $vgpr0_vgpr1 = COPY %2
+...
+
+---
+name: test_or_s7
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+
+    ; CHECK-LABEL: name: test_or_s7
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; CHECK: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; CHECK: [[COPY3:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+    ; CHECK: [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY2]], [[COPY3]]
+    ; CHECK: [[COPY4:%[0-9]+]]:_(s32) = COPY [[OR]](s32)
+    ; CHECK: $vgpr0 = COPY [[COPY4]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s7) = G_TRUNC %0
+    %3:_(s7) = G_TRUNC %1
+    %4:_(s7) = G_OR %2, %3
+    %5:_(s32) = G_ANYEXT %4
+    $vgpr0 = COPY %5
+...
+
+---
+name: test_or_s8
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+
+    ; CHECK-LABEL: name: test_or_s8
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; CHECK: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; CHECK: [[COPY3:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+    ; CHECK: [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY2]], [[COPY3]]
+    ; CHECK: [[COPY4:%[0-9]+]]:_(s32) = COPY [[OR]](s32)
+    ; CHECK: $vgpr0 = COPY [[COPY4]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s8) = G_TRUNC %0
+    %3:_(s8) = G_TRUNC %1
+    %4:_(s8) = G_OR %2, %3
+    %5:_(s32) = G_ANYEXT %4
+    $vgpr0 = COPY %5
+...
+
+---
+name: test_or_s16
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+
+    ; CHECK-LABEL: name: test_or_s16
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; CHECK: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; CHECK: [[COPY3:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+    ; CHECK: [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY2]], [[COPY3]]
+    ; CHECK: [[COPY4:%[0-9]+]]:_(s32) = COPY [[OR]](s32)
+    ; CHECK: $vgpr0 = COPY [[COPY4]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s16) = G_TRUNC %0
+    %3:_(s16) = G_TRUNC %1
+    %4:_(s16) = G_OR %2, %3
+    %5:_(s32) = G_ANYEXT %4
+    $vgpr0 = COPY %5
+...
+
+---
+name: test_or_s24
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+
+    ; CHECK-LABEL: name: test_or_s24
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; CHECK: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; CHECK: [[COPY3:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+    ; CHECK: [[OR:%[0-9]+]]:_(s32) = G_OR [[COPY2]], [[COPY3]]
+    ; CHECK: [[COPY4:%[0-9]+]]:_(s32) = COPY [[OR]](s32)
+    ; CHECK: $vgpr0 = COPY [[COPY4]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s16) = G_TRUNC %0
+    %3:_(s16) = G_TRUNC %1
+    %4:_(s16) = G_OR %2, %3
+    %5:_(s32) = G_ANYEXT %4
+    $vgpr0 = COPY %5
+...
+
+---
+name: test_or_v2s32
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
+
+    ; CHECK-LABEL: name: test_or_v2s32
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    ; CHECK: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr2_vgpr3
+    ; CHECK: [[OR:%[0-9]+]]:_(<2 x s32>) = G_OR [[COPY]], [[COPY1]]
+    ; CHECK: $vgpr0_vgpr1 = COPY [[OR]](<2 x s32>)
+    %0:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    %1:_(<2 x s32>) = COPY $vgpr2_vgpr3
+    %2:_(<2 x s32>) = G_OR %0, %1
+    $vgpr0_vgpr1 = COPY %2
+...
+
+---
+name: test_or_v2s64
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7
+
+    ; CHECK-LABEL: name: test_or_v2s64
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; CHECK: [[COPY1:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr4_vgpr5_vgpr6_vgpr7
+    ; CHECK: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](<2 x s64>)
+    ; CHECK: [[UV2:%[0-9]+]]:_(s64), [[UV3:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY1]](<2 x s64>)
+    ; CHECK: [[OR:%[0-9]+]]:_(s64) = G_OR [[UV]], [[UV2]]
+    ; CHECK: [[OR1:%[0-9]+]]:_(s64) = G_OR [[UV1]], [[UV3]]
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[OR]](s64), [[OR1]](s64)
+    ; CHECK: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>)
+    %0:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    %1:_(<2 x s64>) = COPY $vgpr4_vgpr5_vgpr6_vgpr7
+    %2:_(<2 x s64>) = G_OR %0, %1
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %2
+...
+
+---
+name: test_or_v2s16
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+
+    ; CHECK-LABEL: name: test_or_v2s16
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
+    ; CHECK: [[OR:%[0-9]+]]:_(<2 x s16>) = G_OR [[COPY]], [[COPY1]]
+    ; CHECK: $vgpr0 = COPY [[OR]](<2 x s16>)
+    %0:_(<2 x s16>) = COPY $vgpr0
+    %1:_(<2 x s16>) = COPY $vgpr1
+    %2:_(<2 x s16>) = G_OR %0, %1
+    $vgpr0 = COPY %2
+...
+
+---
+name: test_or_v3s16
+body: |
+  bb.0:
+
+    ; CHECK-LABEL: name: test_or_v3s16
+    ; CHECK: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; CHECK: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
+    ; CHECK: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; CHECK: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF1]](<4 x s16>), 0
+    ; CHECK: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16), [[UV2:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[EXTRACT]](<3 x s16>)
+    ; CHECK: [[UV3:%[0-9]+]]:_(s16), [[UV4:%[0-9]+]]:_(s16), [[UV5:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[EXTRACT1]](<3 x s16>)
+    ; CHECK: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[UV]](s16)
+    ; CHECK: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[UV3]](s16)
+    ; CHECK: [[OR:%[0-9]+]]:_(s32) = G_OR [[ANYEXT]], [[ANYEXT1]]
+    ; CHECK: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[OR]](s32)
+    ; CHECK: [[ANYEXT2:%[0-9]+]]:_(s32) = G_ANYEXT [[UV1]](s16)
+    ; CHECK: [[ANYEXT3:%[0-9]+]]:_(s32) = G_ANYEXT [[UV4]](s16)
+    ; CHECK: [[OR1:%[0-9]+]]:_(s32) = G_OR [[ANYEXT2]], [[ANYEXT3]]
+    ; CHECK: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[OR1]](s32)
+    ; CHECK: [[ANYEXT4:%[0-9]+]]:_(s32) = G_ANYEXT [[UV2]](s16)
+    ; CHECK: [[ANYEXT5:%[0-9]+]]:_(s32) = G_ANYEXT [[UV5]](s16)
+    ; CHECK: [[OR2:%[0-9]+]]:_(s32) = G_OR [[ANYEXT4]], [[ANYEXT5]]
+    ; CHECK: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[OR2]](s32)
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16), [[TRUNC2]](s16)
+    ; CHECK: [[DEF2:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; CHECK: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF2]], [[BUILD_VECTOR]](<3 x s16>), 0
+    ; CHECK: $vgpr0_vgpr1 = COPY [[INSERT]](<4 x s16>)
+    %0:_(<3 x s16>) = G_IMPLICIT_DEF
+    %1:_(<3 x s16>) = G_IMPLICIT_DEF
+    %2:_(<3 x s16>) = G_OR %0, %1
+    %4:_(<4 x s16>) = G_IMPLICIT_DEF
+    %5:_(<4 x s16>) = G_INSERT %4, %2, 0
+    $vgpr0_vgpr1 = COPY %5
+...
+
+---
+name: test_or_v4s16
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
+
+    ; CHECK-LABEL: name: test_or_v4s16
+    ; CHECK: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
+    ; CHECK: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
+    ; CHECK: [[OR:%[0-9]+]]:_(<4 x s16>) = G_OR [[COPY]], [[COPY1]]
+    ; CHECK: $vgpr0_vgpr1 = COPY [[OR]](<4 x s16>)
+    %0:_(<4 x s16>) = COPY $vgpr0_vgpr1
+    %1:_(<4 x s16>) = COPY $vgpr2_vgpr3
+    %2:_(<4 x s16>) = G_OR %0, %1
+    $vgpr0_vgpr1 = COPY %2
+...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/legalize-ptrtoint.mir b/test/CodeGen/AMDGPU/GlobalISel/legalize-ptrtoint.mir
new file mode 100644
index 0000000000000000000000000000000000000000..37ad65b6e2cb1ff5f56d64e2ee8754edd24c1361
--- /dev/null
+++ b/test/CodeGen/AMDGPU/GlobalISel/legalize-ptrtoint.mir
@@ -0,0 +1,162 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -march=amdgcn -O0 -run-pass=legalizer -global-isel-abort=0 -o - %s  | FileCheck %s
+
+---
+name: test_ptrtoint_p0_to_s64
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; CHECK-LABEL: name: test_ptrtoint_p0_to_s64
+    ; CHECK: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
+    ; CHECK: [[PTRTOINT:%[0-9]+]]:_(s64) = G_PTRTOINT [[COPY]](p0)
+    ; CHECK: $vgpr0_vgpr1 = COPY [[PTRTOINT]](s64)
+    %0:_(p0) = COPY $vgpr0_vgpr1
+    %1:_(s64) = G_PTRTOINT %0
+    $vgpr0_vgpr1 = COPY %1
+...
+
+---
+name: test_ptrtoint_p1_to_s64
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; CHECK-LABEL: name: test_ptrtoint_p1_to_s64
+    ; CHECK: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
+    ; CHECK: [[PTRTOINT:%[0-9]+]]:_(s64) = G_PTRTOINT [[COPY]](p1)
+    ; CHECK: $vgpr0_vgpr1 = COPY [[PTRTOINT]](s64)
+    %0:_(p1) = COPY $vgpr0_vgpr1
+    %1:_(s64) = G_PTRTOINT %0
+    $vgpr0_vgpr1 = COPY %1
+...
+
+---
+name: test_ptrtoint_p4_to_s64
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; CHECK-LABEL: name: test_ptrtoint_p4_to_s64
+    ; CHECK: [[COPY:%[0-9]+]]:_(p4) = COPY $vgpr0_vgpr1
+    ; CHECK: [[PTRTOINT:%[0-9]+]]:_(s64) = G_PTRTOINT [[COPY]](p4)
+    ; CHECK: $vgpr0_vgpr1 = COPY [[PTRTOINT]](s64)
+    %0:_(p4) = COPY $vgpr0_vgpr1
+    %1:_(s64) = G_PTRTOINT %0
+    $vgpr0_vgpr1 = COPY %1
+...
+
+---
+name: test_ptrtoint_p3_to_s32
+body: |
+  bb.0:
+    liveins: $vgpr0
+
+    ; CHECK-LABEL: name: test_ptrtoint_p3_to_s32
+    ; CHECK: [[COPY:%[0-9]+]]:_(p3) = COPY $vgpr0
+    ; CHECK: [[PTRTOINT:%[0-9]+]]:_(s32) = G_PTRTOINT [[COPY]](p3)
+    ; CHECK: $vgpr0 = COPY [[PTRTOINT]](s32)
+    %0:_(p3) = COPY $vgpr0
+    %1:_(s32) = G_PTRTOINT %0
+    $vgpr0 = COPY %1
+...
+
+---
+name: test_ptrtoint_p5_to_s32
+body: |
+  bb.0:
+    liveins: $vgpr0
+
+    ; CHECK-LABEL: name: test_ptrtoint_p5_to_s32
+    ; CHECK: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
+    ; CHECK: [[PTRTOINT:%[0-9]+]]:_(s32) = G_PTRTOINT [[COPY]](p5)
+    ; CHECK: $vgpr0 = COPY [[PTRTOINT]](s32)
+    %0:_(p5) = COPY $vgpr0
+    %1:_(s32) = G_PTRTOINT %0
+    $vgpr0 = COPY %1
+...
+
+---
+name: test_ptrtoint_p999_to_s64
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; CHECK-LABEL: name: test_ptrtoint_p999_to_s64
+    ; CHECK: [[COPY:%[0-9]+]]:_(p999) = COPY $vgpr0_vgpr1
+    ; CHECK: [[PTRTOINT:%[0-9]+]]:_(s64) = G_PTRTOINT [[COPY]](p999)
+    ; CHECK: $vgpr0_vgpr1 = COPY [[PTRTOINT]](s64)
+    %0:_(p999) = COPY $vgpr0_vgpr1
+    %1:_(s64) = G_PTRTOINT %0
+    $vgpr0_vgpr1 = COPY %1
+...
+
+---
+name: test_ptrtoint_p0_to_s32
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; CHECK-LABEL: name: test_ptrtoint_p0_to_s32
+    ; CHECK: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
+    ; CHECK: [[PTRTOINT:%[0-9]+]]:_(s64) = G_PTRTOINT [[COPY]](p0)
+    ; CHECK: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[PTRTOINT]](s64)
+    ; CHECK: $vgpr0 = COPY [[TRUNC]](s32)
+    %0:_(p0) = COPY $vgpr0_vgpr1
+    %1:_(s32) = G_PTRTOINT %0
+    $vgpr0 = COPY %1
+...
+
+---
+name: test_ptrtoint_p0_to_s128
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; CHECK-LABEL: name: test_ptrtoint_p0_to_s128
+    ; CHECK: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
+    ; CHECK: [[PTRTOINT:%[0-9]+]]:_(s64) = G_PTRTOINT [[COPY]](p0)
+    ; CHECK: [[ZEXT:%[0-9]+]]:_(s128) = G_ZEXT [[PTRTOINT]](s64)
+    ; CHECK: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[ZEXT]](s128)
+    %0:_(p0) = COPY $vgpr0_vgpr1
+    %1:_(s128) = G_PTRTOINT %0
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %1
+...
+
+---
+name: test_ptrtoint_v2p0_to_v2s64
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3
+
+    ; CHECK-LABEL: name: test_ptrtoint_v2p0_to_v2s64
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x p0>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; CHECK: [[UV:%[0-9]+]]:_(p0), [[UV1:%[0-9]+]]:_(p0) = G_UNMERGE_VALUES [[COPY]](<2 x p0>)
+    ; CHECK: [[PTRTOINT:%[0-9]+]]:_(s64) = G_PTRTOINT [[UV]](p0)
+    ; CHECK: [[PTRTOINT1:%[0-9]+]]:_(s64) = G_PTRTOINT [[UV1]](p0)
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[PTRTOINT]](s64), [[PTRTOINT1]](s64)
+    ; CHECK: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>)
+    %0:_(<2 x p0>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    %1:_(<2 x s64>) = G_PTRTOINT %0
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %1
+...
+
+---
+name: test_ptrtoint_v2s32_to_v2p0
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3
+
+    ; CHECK-LABEL: name: test_ptrtoint_v2s32_to_v2p0
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x p0>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; CHECK: [[UV:%[0-9]+]]:_(p0), [[UV1:%[0-9]+]]:_(p0) = G_UNMERGE_VALUES [[COPY]](<2 x p0>)
+    ; CHECK: [[PTRTOINT:%[0-9]+]]:_(s64) = G_PTRTOINT [[UV]](p0)
+    ; CHECK: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[PTRTOINT]](s64)
+    ; CHECK: [[PTRTOINT1:%[0-9]+]]:_(s64) = G_PTRTOINT [[UV1]](p0)
+    ; CHECK: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[PTRTOINT1]](s64)
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[TRUNC]](s32), [[TRUNC1]](s32)
+    ; CHECK: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
+    %0:_(<2 x p0>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    %1:_(<2 x s32>) = G_PTRTOINT %0
+    $vgpr0_vgpr1 = COPY %1
+...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/legalize-select.mir b/test/CodeGen/AMDGPU/GlobalISel/legalize-select.mir
index 4c04612240e81435c5866b296683611fc1a0edeb..5e8799ee0911be6d099911408f4b10f8a3523052 100644
--- a/test/CodeGen/AMDGPU/GlobalISel/legalize-select.mir
+++ b/test/CodeGen/AMDGPU/GlobalISel/legalize-select.mir
@@ -1,18 +1,19 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -O0 -march=amdgcn -mcpu=fiji  -run-pass=legalizer %s -o - | FileCheck %s
+# RUN: llc -O0 -march=amdgcn -mcpu=tahiti -run-pass=legalizer -global-isel-abort=0 %s -o - | FileCheck %s
 
 ---
-name:            test_select
+name: test_select_s32
 body: |
   bb.0:
     liveins: $vgpr0
-    ; CHECK-LABEL: name: test_select
+    ; CHECK-LABEL: name: test_select_s32
     ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
     ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
     ; CHECK: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[C]](s32), [[COPY]]
     ; CHECK: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
     ; CHECK: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
     ; CHECK: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[C1]], [[C2]]
+    ; CHECK: $vgpr0 = COPY [[SELECT]](s32)
     %0:_(s32) = G_CONSTANT i32 0
     %1:_(s32) = COPY $vgpr0
 
@@ -23,3 +24,1151 @@ body: |
     $vgpr0 = COPY %5
 
 ...
+
+---
+name: test_select_s64
+body: |
+  bb.0:
+    liveins: $vgpr0
+    ; CHECK-LABEL: name: test_select_s64
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[C]](s32), [[COPY]]
+    ; CHECK: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
+    ; CHECK: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 2
+    ; CHECK: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[C1]], [[C2]]
+    ; CHECK: $vgpr0_vgpr1 = COPY [[SELECT]](s64)
+    %0:_(s32) = G_CONSTANT i32 0
+    %1:_(s32) = COPY $vgpr0
+
+    %2:_(s1) = G_ICMP intpred(ne), %0, %1
+    %3:_(s64) = G_CONSTANT i64 1
+    %4:_(s64) = G_CONSTANT i64 2
+    %5:_(s64) = G_SELECT %2, %3, %4
+    $vgpr0_vgpr1 = COPY %5
+
+...
+
+---
+name: test_select_s16
+body: |
+  bb.0:
+    liveins: $vgpr0
+    ; CHECK-LABEL: name: test_select_s16
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[C]](s32), [[COPY]]
+    ; CHECK: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; CHECK: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY [[C1]](s32)
+    ; CHECK: [[COPY2:%[0-9]+]]:_(s32) = COPY [[C2]](s32)
+    ; CHECK: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[COPY1]], [[COPY2]]
+    ; CHECK: [[COPY3:%[0-9]+]]:_(s32) = COPY [[SELECT]](s32)
+    ; CHECK: $vgpr0 = COPY [[COPY3]](s32)
+    %0:_(s32) = G_CONSTANT i32 0
+    %1:_(s32) = COPY $vgpr0
+
+    %2:_(s1) = G_ICMP intpred(ne), %0, %1
+    %3:_(s16) = G_CONSTANT i16 1
+    %4:_(s16) = G_CONSTANT i16 2
+    %5:_(s16) = G_SELECT %2, %3, %4
+    %6:_(s32) = G_ANYEXT %5
+    $vgpr0 = COPY %6
+
+...
+
+---
+name: test_select_s8
+body: |
+  bb.0:
+    liveins: $vgpr0
+    ; CHECK-LABEL: name: test_select_s8
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[C]](s32), [[COPY]]
+    ; CHECK: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; CHECK: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY [[C1]](s32)
+    ; CHECK: [[COPY2:%[0-9]+]]:_(s32) = COPY [[C2]](s32)
+    ; CHECK: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[COPY1]], [[COPY2]]
+    ; CHECK: [[COPY3:%[0-9]+]]:_(s32) = COPY [[SELECT]](s32)
+    ; CHECK: $vgpr0 = COPY [[COPY3]](s32)
+    %0:_(s32) = G_CONSTANT i32 0
+    %1:_(s32) = COPY $vgpr0
+
+    %2:_(s1) = G_ICMP intpred(ne), %0, %1
+    %3:_(s8) = G_CONSTANT i8 1
+    %4:_(s8) = G_CONSTANT i8 2
+    %5:_(s8) = G_SELECT %2, %3, %4
+    %6:_(s32) = G_ANYEXT %5
+    $vgpr0 = COPY %6
+
+...
+
+---
+name: test_select_s7
+body: |
+  bb.0:
+    liveins: $vgpr0
+    ; CHECK-LABEL: name: test_select_s7
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[C]](s32), [[COPY]]
+    ; CHECK: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; CHECK: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY [[C1]](s32)
+    ; CHECK: [[COPY2:%[0-9]+]]:_(s32) = COPY [[C2]](s32)
+    ; CHECK: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[COPY1]], [[COPY2]]
+    ; CHECK: [[COPY3:%[0-9]+]]:_(s32) = COPY [[SELECT]](s32)
+    ; CHECK: $vgpr0 = COPY [[COPY3]](s32)
+    %0:_(s32) = G_CONSTANT i32 0
+    %1:_(s32) = COPY $vgpr0
+
+    %2:_(s1) = G_ICMP intpred(ne), %0, %1
+    %3:_(s7) = G_CONSTANT i7 1
+    %4:_(s7) = G_CONSTANT i7 2
+    %5:_(s7) = G_SELECT %2, %3, %4
+    %6:_(s32) = G_ANYEXT %5
+    $vgpr0 = COPY %6
+
+...
+
+---
+name: test_select_s96
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2, $vgpr3_vgpr4_vgpr5 , $vgpr6
+    ; CHECK-LABEL: name: test_select_s96
+    ; CHECK: [[COPY:%[0-9]+]]:_(s96) = COPY $vgpr0_vgpr1_vgpr2
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s96) = COPY $vgpr3_vgpr4_vgpr5
+    ; CHECK: [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr6
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; CHECK: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[COPY2]](s32), [[C]]
+    ; CHECK: [[EXTRACT:%[0-9]+]]:_(s64) = G_EXTRACT [[COPY]](s96), 0
+    ; CHECK: [[EXTRACT1:%[0-9]+]]:_(s32) = G_EXTRACT [[COPY]](s96), 64
+    ; CHECK: [[EXTRACT2:%[0-9]+]]:_(s64) = G_EXTRACT [[COPY1]](s96), 0
+    ; CHECK: [[EXTRACT3:%[0-9]+]]:_(s32) = G_EXTRACT [[COPY1]](s96), 64
+    ; CHECK: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[EXTRACT]], [[EXTRACT2]]
+    ; CHECK: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[EXTRACT1]], [[EXTRACT3]]
+    ; CHECK: [[DEF:%[0-9]+]]:_(s96) = G_IMPLICIT_DEF
+    ; CHECK: [[INSERT:%[0-9]+]]:_(s96) = G_INSERT [[DEF]], [[SELECT]](s64), 0
+    ; CHECK: [[INSERT1:%[0-9]+]]:_(s96) = G_INSERT [[INSERT]], [[SELECT1]](s32), 64
+    ; CHECK: $vgpr0_vgpr1_vgpr2 = COPY [[INSERT1]](s96)
+    %0:_(s96) = COPY $vgpr0_vgpr1_vgpr2
+    %1:_(s96) = COPY $vgpr3_vgpr4_vgpr5
+    %2:_(s32) = COPY $vgpr6
+    %3:_(s32) = G_CONSTANT i32 0
+
+    %4:_(s1) = G_ICMP intpred(ne), %2, %3
+    %5:_(s96) = G_SELECT %4, %0, %1
+    $vgpr0_vgpr1_vgpr2 = COPY %5
+
+...
+
+---
+name: test_select_s128
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7 , $vgpr8
+
+    ; CHECK-LABEL: name: test_select_s128
+    ; CHECK: [[COPY:%[0-9]+]]:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s128) = COPY $vgpr4_vgpr5_vgpr6_vgpr7
+    ; CHECK: [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr8
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; CHECK: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[COPY2]](s32), [[C]]
+    ; CHECK: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](s128)
+    ; CHECK: [[UV2:%[0-9]+]]:_(s64), [[UV3:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY1]](s128)
+    ; CHECK: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[UV]], [[UV2]]
+    ; CHECK: [[SELECT1:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[UV1]], [[UV3]]
+    ; CHECK: [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[SELECT]](s64), [[SELECT1]](s64)
+    ; CHECK: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[MV]](s128)
+    %0:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    %1:_(s128) = COPY $vgpr4_vgpr5_vgpr6_vgpr7
+    %2:_(s32) = COPY $vgpr8
+    %3:_(s32) = G_CONSTANT i32 0
+    %4:_(s1) = G_ICMP intpred(ne), %2, %3
+    %5:_(s128) = G_SELECT %4, %0, %1
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %5
+
+...
+
+---
+name: test_select_v2s8
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1_vgpr2, $vgpr3_vgpr4
+
+    ; CHECK-LABEL: name: test_select_v2s8
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr1_vgpr2
+    ; CHECK: [[COPY2:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr3_vgpr4
+    ; CHECK: [[TRUNC:%[0-9]+]]:_(<2 x s8>) = G_TRUNC [[COPY1]](<2 x s32>)
+    ; CHECK: [[TRUNC1:%[0-9]+]]:_(<2 x s8>) = G_TRUNC [[COPY2]](<2 x s32>)
+    ; CHECK: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[COPY]](s32), [[C]]
+    ; CHECK: [[UV:%[0-9]+]]:_(s8), [[UV1:%[0-9]+]]:_(s8) = G_UNMERGE_VALUES [[TRUNC]](<2 x s8>)
+    ; CHECK: [[UV2:%[0-9]+]]:_(s8), [[UV3:%[0-9]+]]:_(s8) = G_UNMERGE_VALUES [[TRUNC1]](<2 x s8>)
+    ; CHECK: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[UV]](s8)
+    ; CHECK: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[UV2]](s8)
+    ; CHECK: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[ANYEXT]], [[ANYEXT1]]
+    ; CHECK: [[TRUNC2:%[0-9]+]]:_(s8) = G_TRUNC [[SELECT]](s32)
+    ; CHECK: [[ANYEXT2:%[0-9]+]]:_(s32) = G_ANYEXT [[UV1]](s8)
+    ; CHECK: [[ANYEXT3:%[0-9]+]]:_(s32) = G_ANYEXT [[UV3]](s8)
+    ; CHECK: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[ANYEXT2]], [[ANYEXT3]]
+    ; CHECK: [[TRUNC3:%[0-9]+]]:_(s8) = G_TRUNC [[SELECT1]](s32)
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s8>) = G_BUILD_VECTOR [[TRUNC2]](s8), [[TRUNC3]](s8)
+    ; CHECK: [[ANYEXT4:%[0-9]+]]:_(<2 x s32>) = G_ANYEXT [[BUILD_VECTOR]](<2 x s8>)
+    ; CHECK: $vgpr0_vgpr1 = COPY [[ANYEXT4]](<2 x s32>)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = G_CONSTANT i32 0
+    %2:_(<2 x s32>) = COPY $vgpr1_vgpr2
+    %3:_(<2 x s32>) = COPY $vgpr3_vgpr4
+    %4:_(<2 x s8>) = G_TRUNC %2
+    %5:_(<2 x s8>) = G_TRUNC %3
+
+    %6:_(s1) = G_ICMP intpred(ne), %0, %1
+    %7:_(<2 x s8>) = G_SELECT %6, %4, %5
+    %8:_(<2 x s32>) = G_ANYEXT %7
+    $vgpr0_vgpr1 = COPY %8
+
+...
+
+---
+name: test_select_v3s8
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6
+
+    ; CHECK-LABEL: name: test_select_v3s8
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr1_vgpr2_vgpr3
+    ; CHECK: [[COPY2:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr4_vgpr5_vgpr6
+    ; CHECK: [[TRUNC:%[0-9]+]]:_(<3 x s8>) = G_TRUNC [[COPY1]](<3 x s32>)
+    ; CHECK: [[TRUNC1:%[0-9]+]]:_(<3 x s8>) = G_TRUNC [[COPY2]](<3 x s32>)
+    ; CHECK: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[COPY]](s32), [[C]]
+    ; CHECK: [[UV:%[0-9]+]]:_(s8), [[UV1:%[0-9]+]]:_(s8), [[UV2:%[0-9]+]]:_(s8) = G_UNMERGE_VALUES [[TRUNC]](<3 x s8>)
+    ; CHECK: [[UV3:%[0-9]+]]:_(s8), [[UV4:%[0-9]+]]:_(s8), [[UV5:%[0-9]+]]:_(s8) = G_UNMERGE_VALUES [[TRUNC1]](<3 x s8>)
+    ; CHECK: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[UV]](s8)
+    ; CHECK: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[UV3]](s8)
+    ; CHECK: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[ANYEXT]], [[ANYEXT1]]
+    ; CHECK: [[TRUNC2:%[0-9]+]]:_(s8) = G_TRUNC [[SELECT]](s32)
+    ; CHECK: [[ANYEXT2:%[0-9]+]]:_(s32) = G_ANYEXT [[UV1]](s8)
+    ; CHECK: [[ANYEXT3:%[0-9]+]]:_(s32) = G_ANYEXT [[UV4]](s8)
+    ; CHECK: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[ANYEXT2]], [[ANYEXT3]]
+    ; CHECK: [[TRUNC3:%[0-9]+]]:_(s8) = G_TRUNC [[SELECT1]](s32)
+    ; CHECK: [[ANYEXT4:%[0-9]+]]:_(s32) = G_ANYEXT [[UV2]](s8)
+    ; CHECK: [[ANYEXT5:%[0-9]+]]:_(s32) = G_ANYEXT [[UV5]](s8)
+    ; CHECK: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[ANYEXT4]], [[ANYEXT5]]
+    ; CHECK: [[TRUNC4:%[0-9]+]]:_(s8) = G_TRUNC [[SELECT2]](s32)
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s8>) = G_BUILD_VECTOR [[TRUNC2]](s8), [[TRUNC3]](s8), [[TRUNC4]](s8)
+    ; CHECK: [[ANYEXT6:%[0-9]+]]:_(<3 x s32>) = G_ANYEXT [[BUILD_VECTOR]](<3 x s8>)
+    ; CHECK: $vgpr0_vgpr1_vgpr2 = COPY [[ANYEXT6]](<3 x s32>)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = G_CONSTANT i32 0
+    %2:_(<3 x s32>) = COPY $vgpr1_vgpr2_vgpr3
+    %3:_(<3 x s32>) = COPY $vgpr4_vgpr5_vgpr6
+    %4:_(<3 x s8>) = G_TRUNC %2
+    %5:_(<3 x s8>) = G_TRUNC %3
+
+    %6:_(s1) = G_ICMP intpred(ne), %0, %1
+    %7:_(<3 x s8>) = G_SELECT %6, %4, %5
+    %8:_(<3 x s32>) = G_ANYEXT %7
+    $vgpr0_vgpr1_vgpr2 = COPY %8
+
+...
+
+---
+name: test_select_v4s8
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1_vgpr2_vgpr3_vgpr4, $vgpr5_vgpr6_vgpr7_vgpr8
+
+    ; CHECK-LABEL: name: test_select_v4s8
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(<4 x s32>) = COPY $vgpr1_vgpr2_vgpr3_vgpr4
+    ; CHECK: [[COPY2:%[0-9]+]]:_(<4 x s32>) = COPY $vgpr5_vgpr6_vgpr7_vgpr8
+    ; CHECK: [[TRUNC:%[0-9]+]]:_(<4 x s8>) = G_TRUNC [[COPY1]](<4 x s32>)
+    ; CHECK: [[TRUNC1:%[0-9]+]]:_(<4 x s8>) = G_TRUNC [[COPY2]](<4 x s32>)
+    ; CHECK: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[COPY]](s32), [[C]]
+    ; CHECK: [[UV:%[0-9]+]]:_(s8), [[UV1:%[0-9]+]]:_(s8), [[UV2:%[0-9]+]]:_(s8), [[UV3:%[0-9]+]]:_(s8) = G_UNMERGE_VALUES [[TRUNC]](<4 x s8>)
+    ; CHECK: [[UV4:%[0-9]+]]:_(s8), [[UV5:%[0-9]+]]:_(s8), [[UV6:%[0-9]+]]:_(s8), [[UV7:%[0-9]+]]:_(s8) = G_UNMERGE_VALUES [[TRUNC1]](<4 x s8>)
+    ; CHECK: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[UV]](s8)
+    ; CHECK: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[UV4]](s8)
+    ; CHECK: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[ANYEXT]], [[ANYEXT1]]
+    ; CHECK: [[TRUNC2:%[0-9]+]]:_(s8) = G_TRUNC [[SELECT]](s32)
+    ; CHECK: [[ANYEXT2:%[0-9]+]]:_(s32) = G_ANYEXT [[UV1]](s8)
+    ; CHECK: [[ANYEXT3:%[0-9]+]]:_(s32) = G_ANYEXT [[UV5]](s8)
+    ; CHECK: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[ANYEXT2]], [[ANYEXT3]]
+    ; CHECK: [[TRUNC3:%[0-9]+]]:_(s8) = G_TRUNC [[SELECT1]](s32)
+    ; CHECK: [[ANYEXT4:%[0-9]+]]:_(s32) = G_ANYEXT [[UV2]](s8)
+    ; CHECK: [[ANYEXT5:%[0-9]+]]:_(s32) = G_ANYEXT [[UV6]](s8)
+    ; CHECK: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[ANYEXT4]], [[ANYEXT5]]
+    ; CHECK: [[TRUNC4:%[0-9]+]]:_(s8) = G_TRUNC [[SELECT2]](s32)
+    ; CHECK: [[ANYEXT6:%[0-9]+]]:_(s32) = G_ANYEXT [[UV3]](s8)
+    ; CHECK: [[ANYEXT7:%[0-9]+]]:_(s32) = G_ANYEXT [[UV7]](s8)
+    ; CHECK: [[SELECT3:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[ANYEXT6]], [[ANYEXT7]]
+    ; CHECK: [[TRUNC5:%[0-9]+]]:_(s8) = G_TRUNC [[SELECT3]](s32)
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s8>) = G_BUILD_VECTOR [[TRUNC2]](s8), [[TRUNC3]](s8), [[TRUNC4]](s8), [[TRUNC5]](s8)
+    ; CHECK: [[ANYEXT8:%[0-9]+]]:_(<4 x s32>) = G_ANYEXT [[BUILD_VECTOR]](<4 x s8>)
+    ; CHECK: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[ANYEXT8]](<4 x s32>)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = G_CONSTANT i32 0
+    %2:_(<4 x s32>) = COPY $vgpr1_vgpr2_vgpr3_vgpr4
+    %3:_(<4 x s32>) = COPY $vgpr5_vgpr6_vgpr7_vgpr8
+    %4:_(<4 x s8>) = G_TRUNC %2
+    %5:_(<4 x s8>) = G_TRUNC %3
+
+    %6:_(s1) = G_ICMP intpred(ne), %0, %1
+    %7:_(<4 x s8>) = G_SELECT %6, %4, %5
+    %8:_(<4 x s32>) = G_ANYEXT %7
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %8
+
+...
+
+---
+name: test_select_v2s16
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2
+    ; CHECK-LABEL: name: test_select_v2s16
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
+    ; CHECK: [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; CHECK: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[COPY2]](s32), [[C]]
+    ; CHECK: [[SELECT:%[0-9]+]]:_(<2 x s16>) = G_SELECT [[ICMP]](s1), [[COPY]], [[COPY1]]
+    ; CHECK: $vgpr0 = COPY [[SELECT]](<2 x s16>)
+    %0:_(<2 x s16>) = COPY $vgpr0
+    %1:_(<2 x s16>) = COPY $vgpr1
+    %2:_(s32) = COPY $vgpr2
+    %3:_(s32) = G_CONSTANT i32 0
+
+    %4:_(s1) = G_ICMP intpred(ne), %2, %3
+    %5:_(<2 x s16>) = G_SELECT %4, %0, %1
+    $vgpr0 = COPY %5
+
+...
+
+---
+name: test_select_v3s16
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1_vgpr2, $vgpr3_vgpr4
+    ; CHECK-LABEL: name: test_select_v3s16
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr1_vgpr2
+    ; CHECK: [[COPY2:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr3_vgpr4
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; CHECK: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[COPY]](s32), [[C]]
+    ; CHECK: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[COPY1]](<4 x s16>), 0
+    ; CHECK: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[COPY2]](<4 x s16>), 0
+    ; CHECK: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16), [[UV2:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[EXTRACT]](<3 x s16>)
+    ; CHECK: [[UV3:%[0-9]+]]:_(s16), [[UV4:%[0-9]+]]:_(s16), [[UV5:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[EXTRACT1]](<3 x s16>)
+    ; CHECK: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[UV]](s16)
+    ; CHECK: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[UV3]](s16)
+    ; CHECK: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[ANYEXT]], [[ANYEXT1]]
+    ; CHECK: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[SELECT]](s32)
+    ; CHECK: [[ANYEXT2:%[0-9]+]]:_(s32) = G_ANYEXT [[UV1]](s16)
+    ; CHECK: [[ANYEXT3:%[0-9]+]]:_(s32) = G_ANYEXT [[UV4]](s16)
+    ; CHECK: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[ANYEXT2]], [[ANYEXT3]]
+    ; CHECK: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[SELECT1]](s32)
+    ; CHECK: [[ANYEXT4:%[0-9]+]]:_(s32) = G_ANYEXT [[UV2]](s16)
+    ; CHECK: [[ANYEXT5:%[0-9]+]]:_(s32) = G_ANYEXT [[UV5]](s16)
+    ; CHECK: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[ANYEXT4]], [[ANYEXT5]]
+    ; CHECK: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[SELECT2]](s32)
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16), [[TRUNC2]](s16)
+    ; CHECK: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; CHECK: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[BUILD_VECTOR]](<3 x s16>), 0
+    ; CHECK: $vgpr0_vgpr1 = COPY [[INSERT]](<4 x s16>)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(<4 x s16>) = COPY $vgpr1_vgpr2
+    %2:_(<4 x s16>) = COPY $vgpr3_vgpr4
+    %4:_(s32) = G_CONSTANT i32 0
+    %5:_(s1) = G_ICMP intpred(ne), %0, %4
+
+    %6:_(<3 x s16>) = G_EXTRACT %1, 0
+    %7:_(<3 x s16>) = G_EXTRACT %2, 0
+    %8:_(<3 x s16>) = G_SELECT %5, %6, %7
+    %9:_(<4 x s16>) = G_IMPLICIT_DEF
+    %10:_(<4 x s16>) = G_INSERT %9, %8, 0
+    $vgpr0_vgpr1 = COPY %10
+
+...
+
+---
+name: test_select_v4s16
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1_vgpr2, $vgpr3_vgpr4
+    ; CHECK-LABEL: name: test_select_v4s16
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr1_vgpr2
+    ; CHECK: [[COPY2:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr3_vgpr4
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; CHECK: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[COPY]](s32), [[C]]
+    ; CHECK: [[SELECT:%[0-9]+]]:_(<4 x s16>) = G_SELECT [[ICMP]](s1), [[COPY1]], [[COPY2]]
+    ; CHECK: $vgpr0_vgpr1 = COPY [[SELECT]](<4 x s16>)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(<4 x s16>) = COPY $vgpr1_vgpr2
+    %2:_(<4 x s16>) = COPY $vgpr3_vgpr4
+    %4:_(s32) = G_CONSTANT i32 0
+
+    %5:_(s1) = G_ICMP intpred(ne), %0, %4
+    %6:_(<4 x s16>) = G_SELECT %5, %1, %2
+    $vgpr0_vgpr1 = COPY %6
+
+...
+
+---
+name: test_select_v2s32
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1_vgpr2, $vgpr3_vgpr4
+    ; CHECK-LABEL: name: test_select_v2s32
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr1_vgpr2
+    ; CHECK: [[COPY2:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr3_vgpr4
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; CHECK: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[COPY]](s32), [[C]]
+    ; CHECK: [[SELECT:%[0-9]+]]:_(<2 x s32>) = G_SELECT [[ICMP]](s1), [[COPY1]], [[COPY2]]
+    ; CHECK: $vgpr0_vgpr1 = COPY [[SELECT]](<2 x s32>)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(<2 x s32>) = COPY $vgpr1_vgpr2
+    %2:_(<2 x s32>) = COPY $vgpr3_vgpr4
+    %4:_(s32) = G_CONSTANT i32 0
+
+    %5:_(s1) = G_ICMP intpred(ne), %0, %4
+    %6:_(<2 x s32>) = G_SELECT %5, %1, %2
+    $vgpr0_vgpr1 = COPY %6
+
+...
+
+---
+name: test_select_v3s32
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6
+    ; CHECK-LABEL: name: test_select_v3s32
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr1_vgpr2_vgpr3
+    ; CHECK: [[COPY2:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr4_vgpr5_vgpr6
+    ; CHECK: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[C]](s32), [[COPY]]
+    ; CHECK: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<3 x s32>)
+    ; CHECK: [[UV3:%[0-9]+]]:_(s32), [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY2]](<3 x s32>)
+    ; CHECK: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[UV]], [[UV3]]
+    ; CHECK: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[UV1]], [[UV4]]
+    ; CHECK: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[UV2]], [[UV5]]
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[SELECT]](s32), [[SELECT1]](s32), [[SELECT2]](s32)
+    ; CHECK: $vgpr0_vgpr1_vgpr2 = COPY [[BUILD_VECTOR]](<3 x s32>)
+    %0:_(s32) = G_CONSTANT i32 0
+    %1:_(s32) = COPY $vgpr0
+    %2:_(<3 x s32>) = COPY $vgpr1_vgpr2_vgpr3
+    %3:_(<3 x s32>) = COPY $vgpr4_vgpr5_vgpr6
+
+    %4:_(s1) = G_ICMP intpred(ne), %0, %1
+    %5:_(<3 x s32>) = G_SELECT %4, %2, %3
+    $vgpr0_vgpr1_vgpr2 = COPY %5
+
+...
+
+---
+name: test_select_v4s32
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1_vgpr2_vgpr3_vgpr4, $vgpr5_vgpr6_vgpr7_vgpr8
+    ; CHECK-LABEL: name: test_select_v4s32
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(<4 x s32>) = COPY $vgpr1_vgpr2_vgpr3_vgpr4
+    ; CHECK: [[COPY2:%[0-9]+]]:_(<4 x s32>) = COPY $vgpr5_vgpr6_vgpr7_vgpr8
+    ; CHECK: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[C]](s32), [[COPY]]
+    ; CHECK: [[UV:%[0-9]+]]:_(<2 x s32>), [[UV1:%[0-9]+]]:_(<2 x s32>) = G_UNMERGE_VALUES [[COPY1]](<4 x s32>)
+    ; CHECK: [[UV2:%[0-9]+]]:_(<2 x s32>), [[UV3:%[0-9]+]]:_(<2 x s32>) = G_UNMERGE_VALUES [[COPY2]](<4 x s32>)
+    ; CHECK: [[SELECT:%[0-9]+]]:_(<2 x s32>) = G_SELECT [[ICMP]](s1), [[UV]], [[UV2]]
+    ; CHECK: [[SELECT1:%[0-9]+]]:_(<2 x s32>) = G_SELECT [[ICMP]](s1), [[UV1]], [[UV3]]
+    ; CHECK: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s32>) = G_CONCAT_VECTORS [[SELECT]](<2 x s32>), [[SELECT1]](<2 x s32>)
+    ; CHECK: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[CONCAT_VECTORS]](<4 x s32>)
+    %0:_(s32) = G_CONSTANT i32 0
+    %1:_(s32) = COPY $vgpr0
+    %2:_(<4 x s32>) = COPY $vgpr1_vgpr2_vgpr3_vgpr4
+    %3:_(<4 x s32>) = COPY $vgpr5_vgpr6_vgpr7_vgpr8
+
+    %4:_(s1) = G_ICMP intpred(ne), %0, %1
+    %5:_(<4 x s32>) = G_SELECT %4, %2, %3
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %5
+
+...
+
+---
+name: test_select_v2s64
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1_vgpr2_vgpr3_vgpr4, $vgpr5_vgpr6_vgpr7_vgpr8
+    ; CHECK-LABEL: name: test_select_v2s64
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr1_vgpr2_vgpr3_vgpr4
+    ; CHECK: [[COPY2:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr5_vgpr6_vgpr7_vgpr8
+    ; CHECK: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[C]](s32), [[COPY]]
+    ; CHECK: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY1]](<2 x s64>)
+    ; CHECK: [[UV2:%[0-9]+]]:_(s64), [[UV3:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY2]](<2 x s64>)
+    ; CHECK: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[UV]], [[UV2]]
+    ; CHECK: [[SELECT1:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[UV1]], [[UV3]]
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[SELECT]](s64), [[SELECT1]](s64)
+    ; CHECK: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>)
+    %0:_(s32) = G_CONSTANT i32 0
+    %1:_(s32) = COPY $vgpr0
+    %2:_(<2 x s64>) = COPY $vgpr1_vgpr2_vgpr3_vgpr4
+    %3:_(<2 x s64>) = COPY $vgpr5_vgpr6_vgpr7_vgpr8
+
+    %4:_(s1) = G_ICMP intpred(ne), %0, %1
+    %5:_(<2 x s64>) = G_SELECT %4, %2, %3
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %5
+
+...
+
+---
+name: test_select_p0
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1_vgpr2, $vgpr3_vgpr4
+    ; CHECK-LABEL: name: test_select_p0
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(p0) = COPY $vgpr1_vgpr2
+    ; CHECK: [[COPY2:%[0-9]+]]:_(p0) = COPY $vgpr3_vgpr4
+    ; CHECK: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[C]](s32), [[COPY]]
+    ; CHECK: [[SELECT:%[0-9]+]]:_(p0) = G_SELECT [[ICMP]](s1), [[COPY1]], [[COPY2]]
+    ; CHECK: $vgpr0_vgpr1 = COPY [[SELECT]](p0)
+    %0:_(s32) = G_CONSTANT i32 0
+    %1:_(s32) = COPY $vgpr0
+    %2:_(p0) = COPY $vgpr1_vgpr2
+    %3:_(p0) = COPY $vgpr3_vgpr4
+
+    %4:_(s1) = G_ICMP intpred(ne), %0, %1
+    %5:_(p0) = G_SELECT %4, %2, %3
+    $vgpr0_vgpr1 = COPY %5
+
+...
+
+---
+name: test_select_p1
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1_vgpr2, $vgpr3_vgpr4
+    ; CHECK-LABEL: name: test_select_p1
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(p1) = COPY $vgpr1_vgpr2
+    ; CHECK: [[COPY2:%[0-9]+]]:_(p1) = COPY $vgpr3_vgpr4
+    ; CHECK: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[C]](s32), [[COPY]]
+    ; CHECK: [[SELECT:%[0-9]+]]:_(p1) = G_SELECT [[ICMP]](s1), [[COPY1]], [[COPY2]]
+    ; CHECK: $vgpr0_vgpr1 = COPY [[SELECT]](p1)
+    %0:_(s32) = G_CONSTANT i32 0
+    %1:_(s32) = COPY $vgpr0
+    %2:_(p1) = COPY $vgpr1_vgpr2
+    %3:_(p1) = COPY $vgpr3_vgpr4
+
+    %4:_(s1) = G_ICMP intpred(ne), %0, %1
+    %5:_(p1) = G_SELECT %4, %2, %3
+    $vgpr0_vgpr1 = COPY %5
+
+...
+
+---
+name: test_select_p2
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2
+    ; CHECK-LABEL: name: test_select_p2
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(p2) = COPY $vgpr1
+    ; CHECK: [[COPY2:%[0-9]+]]:_(p2) = COPY $vgpr2
+    ; CHECK: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[C]](s32), [[COPY]]
+    ; CHECK: [[SELECT:%[0-9]+]]:_(p2) = G_SELECT [[ICMP]](s1), [[COPY1]], [[COPY2]]
+    ; CHECK: $vgpr0 = COPY [[SELECT]](p2)
+    %0:_(s32) = G_CONSTANT i32 0
+    %1:_(s32) = COPY $vgpr0
+    %2:_(p2) = COPY $vgpr1
+    %3:_(p2) = COPY $vgpr2
+
+    %4:_(s1) = G_ICMP intpred(ne), %0, %1
+    %5:_(p2) = G_SELECT %4, %2, %3
+    $vgpr0 = COPY %5
+
+...
+
+---
+name: test_select_p3
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2
+    ; CHECK-LABEL: name: test_select_p3
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(p3) = COPY $vgpr1
+    ; CHECK: [[COPY2:%[0-9]+]]:_(p3) = COPY $vgpr2
+    ; CHECK: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[C]](s32), [[COPY]]
+    ; CHECK: [[SELECT:%[0-9]+]]:_(p3) = G_SELECT [[ICMP]](s1), [[COPY1]], [[COPY2]]
+    ; CHECK: $vgpr0 = COPY [[SELECT]](p3)
+    %0:_(s32) = G_CONSTANT i32 0
+    %1:_(s32) = COPY $vgpr0
+    %2:_(p3) = COPY $vgpr1
+    %3:_(p3) = COPY $vgpr2
+
+    %4:_(s1) = G_ICMP intpred(ne), %0, %1
+    %5:_(p3) = G_SELECT %4, %2, %3
+    $vgpr0 = COPY %5
+
+...
+
+---
+name: test_select_p4
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1_vgpr2, $vgpr3_vgpr4
+    ; CHECK-LABEL: name: test_select_p4
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(p4) = COPY $vgpr1_vgpr2
+    ; CHECK: [[COPY2:%[0-9]+]]:_(p4) = COPY $vgpr3_vgpr4
+    ; CHECK: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[C]](s32), [[COPY]]
+    ; CHECK: [[SELECT:%[0-9]+]]:_(p4) = G_SELECT [[ICMP]](s1), [[COPY1]], [[COPY2]]
+    ; CHECK: $vgpr0_vgpr1 = COPY [[SELECT]](p4)
+    %0:_(s32) = G_CONSTANT i32 0
+    %1:_(s32) = COPY $vgpr0
+    %2:_(p4) = COPY $vgpr1_vgpr2
+    %3:_(p4) = COPY $vgpr3_vgpr4
+
+    %4:_(s1) = G_ICMP intpred(ne), %0, %1
+    %5:_(p4) = G_SELECT %4, %2, %3
+    $vgpr0_vgpr1 = COPY %5
+
+...
+
+---
+name: test_select_p5
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2
+
+    ; CHECK-LABEL: name: test_select_p5
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(p5) = COPY $vgpr1
+    ; CHECK: [[COPY2:%[0-9]+]]:_(p5) = COPY $vgpr2
+    ; CHECK: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[C]](s32), [[COPY]]
+    ; CHECK: [[SELECT:%[0-9]+]]:_(p5) = G_SELECT [[ICMP]](s1), [[COPY1]], [[COPY2]]
+    ; CHECK: $vgpr0 = COPY [[SELECT]](p5)
+    %0:_(s32) = G_CONSTANT i32 0
+    %1:_(s32) = COPY $vgpr0
+    %2:_(p5) = COPY $vgpr1
+    %3:_(p5) = COPY $vgpr2
+    %4:_(s1) = G_ICMP intpred(ne), %0, %1
+    %5:_(p5) = G_SELECT %4, %2, %3
+    $vgpr0 = COPY %5
+
+...
+
+---
+name: test_select_p999
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1_vgpr2, $vgpr3_vgpr4
+
+    ; CHECK-LABEL: name: test_select_p999
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(p999) = COPY $vgpr1_vgpr2
+    ; CHECK: [[COPY2:%[0-9]+]]:_(p999) = COPY $vgpr3_vgpr4
+    ; CHECK: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[C]](s32), [[COPY]]
+    ; CHECK: [[SELECT:%[0-9]+]]:_(p999) = G_SELECT [[ICMP]](s1), [[COPY1]], [[COPY2]]
+    ; CHECK: $vgpr0_vgpr1 = COPY [[SELECT]](p999)
+    %0:_(s32) = G_CONSTANT i32 0
+    %1:_(s32) = COPY $vgpr0
+    %2:_(p999) = COPY $vgpr1_vgpr2
+    %3:_(p999) = COPY $vgpr3_vgpr4
+    %4:_(s1) = G_ICMP intpred(ne), %0, %1
+    %5:_(p999) = G_SELECT %4, %2, %3
+    $vgpr0_vgpr1 = COPY %5
+
+...
+
+---
+
+name: test_select_v2p3
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1_vgpr2, $vgpr3_vgpr4
+
+    ; CHECK-LABEL: name: test_select_v2p3
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(<2 x p3>) = COPY $vgpr1_vgpr2
+    ; CHECK: [[COPY2:%[0-9]+]]:_(<2 x p3>) = COPY $vgpr3_vgpr4
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; CHECK: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[COPY]](s32), [[C]]
+    ; CHECK: [[SELECT:%[0-9]+]]:_(<2 x p3>) = G_SELECT [[ICMP]](s1), [[COPY1]], [[COPY2]]
+    ; CHECK: $vgpr0_vgpr1 = COPY [[SELECT]](<2 x p3>)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(<2 x p3>) = COPY $vgpr1_vgpr2
+    %2:_(<2 x p3>) = COPY $vgpr3_vgpr4
+    %4:_(s32) = G_CONSTANT i32 0
+    %5:_(s1) = G_ICMP intpred(ne), %0, %4
+    %6:_(<2 x p3>) = G_SELECT %5, %1, %2
+    $vgpr0_vgpr1 = COPY %6
+
+...
+
+---
+name: test_select_v3p3
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6
+
+    ; CHECK-LABEL: name: test_select_v3p3
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(<3 x p3>) = COPY $vgpr1_vgpr2_vgpr3
+    ; CHECK: [[COPY2:%[0-9]+]]:_(<3 x p3>) = COPY $vgpr4_vgpr5_vgpr6
+    ; CHECK: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[C]](s32), [[COPY]]
+    ; CHECK: [[UV:%[0-9]+]]:_(p3), [[UV1:%[0-9]+]]:_(p3), [[UV2:%[0-9]+]]:_(p3) = G_UNMERGE_VALUES [[COPY1]](<3 x p3>)
+    ; CHECK: [[UV3:%[0-9]+]]:_(p3), [[UV4:%[0-9]+]]:_(p3), [[UV5:%[0-9]+]]:_(p3) = G_UNMERGE_VALUES [[COPY2]](<3 x p3>)
+    ; CHECK: [[SELECT:%[0-9]+]]:_(p3) = G_SELECT [[ICMP]](s1), [[UV]], [[UV3]]
+    ; CHECK: [[SELECT1:%[0-9]+]]:_(p3) = G_SELECT [[ICMP]](s1), [[UV1]], [[UV4]]
+    ; CHECK: [[SELECT2:%[0-9]+]]:_(p3) = G_SELECT [[ICMP]](s1), [[UV2]], [[UV5]]
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x p3>) = G_BUILD_VECTOR [[SELECT]](p3), [[SELECT1]](p3), [[SELECT2]](p3)
+    ; CHECK: $vgpr0_vgpr1_vgpr2 = COPY [[BUILD_VECTOR]](<3 x p3>)
+    %0:_(s32) = G_CONSTANT i32 0
+    %1:_(s32) = COPY $vgpr0
+    %2:_(<3 x p3>) = COPY $vgpr1_vgpr2_vgpr3
+    %3:_(<3 x p3>) = COPY $vgpr4_vgpr5_vgpr6
+
+    %4:_(s1) = G_ICMP intpred(ne), %0, %1
+    %5:_(<3 x p3>) = G_SELECT %4, %2, %3
+    $vgpr0_vgpr1_vgpr2 = COPY %5
+
+...
+
+---
+name: test_select_v4p3
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1_vgpr2_vgpr3_vgpr4, $vgpr5_vgpr6_vgpr7_vgpr8
+
+    ; CHECK-LABEL: name: test_select_v4p3
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(<4 x p3>) = COPY $vgpr1_vgpr2_vgpr3_vgpr4
+    ; CHECK: [[COPY2:%[0-9]+]]:_(<4 x p3>) = COPY $vgpr5_vgpr6_vgpr7_vgpr8
+    ; CHECK: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[C]](s32), [[COPY]]
+    ; CHECK: [[UV:%[0-9]+]]:_(<2 x p3>), [[UV1:%[0-9]+]]:_(<2 x p3>) = G_UNMERGE_VALUES [[COPY1]](<4 x p3>)
+    ; CHECK: [[UV2:%[0-9]+]]:_(<2 x p3>), [[UV3:%[0-9]+]]:_(<2 x p3>) = G_UNMERGE_VALUES [[COPY2]](<4 x p3>)
+    ; CHECK: [[SELECT:%[0-9]+]]:_(<2 x p3>) = G_SELECT [[ICMP]](s1), [[UV]], [[UV2]]
+    ; CHECK: [[SELECT1:%[0-9]+]]:_(<2 x p3>) = G_SELECT [[ICMP]](s1), [[UV1]], [[UV3]]
+    ; CHECK: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x p3>) = G_CONCAT_VECTORS [[SELECT]](<2 x p3>), [[SELECT1]](<2 x p3>)
+    ; CHECK: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[CONCAT_VECTORS]](<4 x p3>)
+    %0:_(s32) = G_CONSTANT i32 0
+    %1:_(s32) = COPY $vgpr0
+    %2:_(<4 x p3>) = COPY $vgpr1_vgpr2_vgpr3_vgpr4
+    %3:_(<4 x p3>) = COPY $vgpr5_vgpr6_vgpr7_vgpr8
+
+    %4:_(s1) = G_ICMP intpred(ne), %0, %1
+    %5:_(<4 x p3>) = G_SELECT %4, %2, %3
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %5
+
+...
+
+---
+name: test_select_v4p5
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1_vgpr2_vgpr3_vgpr4, $vgpr5_vgpr6_vgpr7_vgpr8
+
+    ; CHECK-LABEL: name: test_select_v4p5
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(<4 x p5>) = COPY $vgpr1_vgpr2_vgpr3_vgpr4
+    ; CHECK: [[COPY2:%[0-9]+]]:_(<4 x p5>) = COPY $vgpr5_vgpr6_vgpr7_vgpr8
+    ; CHECK: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[C]](s32), [[COPY]]
+    ; CHECK: [[UV:%[0-9]+]]:_(<2 x p5>), [[UV1:%[0-9]+]]:_(<2 x p5>) = G_UNMERGE_VALUES [[COPY1]](<4 x p5>)
+    ; CHECK: [[UV2:%[0-9]+]]:_(<2 x p5>), [[UV3:%[0-9]+]]:_(<2 x p5>) = G_UNMERGE_VALUES [[COPY2]](<4 x p5>)
+    ; CHECK: [[SELECT:%[0-9]+]]:_(<2 x p5>) = G_SELECT [[ICMP]](s1), [[UV]], [[UV2]]
+    ; CHECK: [[SELECT1:%[0-9]+]]:_(<2 x p5>) = G_SELECT [[ICMP]](s1), [[UV1]], [[UV3]]
+    ; CHECK: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x p5>) = G_CONCAT_VECTORS [[SELECT]](<2 x p5>), [[SELECT1]](<2 x p5>)
+    ; CHECK: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[CONCAT_VECTORS]](<4 x p5>)
+    %0:_(s32) = G_CONSTANT i32 0
+    %1:_(s32) = COPY $vgpr0
+    %2:_(<4 x p5>) = COPY $vgpr1_vgpr2_vgpr3_vgpr4
+    %3:_(<4 x p5>) = COPY $vgpr5_vgpr6_vgpr7_vgpr8
+
+    %4:_(s1) = G_ICMP intpred(ne), %0, %1
+    %5:_(<4 x p5>) = G_SELECT %4, %2, %3
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %5
+
+...
+
+---
+name: test_select_v2p0
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1_vgpr2_vgpr3_vgpr4, $vgpr5_vgpr6_vgpr7_vgpr8
+
+    ; CHECK-LABEL: name: test_select_v2p0
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(<2 x p0>) = COPY $vgpr1_vgpr2_vgpr3_vgpr4
+    ; CHECK: [[COPY2:%[0-9]+]]:_(<2 x p0>) = COPY $vgpr5_vgpr6_vgpr7_vgpr8
+    ; CHECK: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[C]](s32), [[COPY]]
+    ; CHECK: [[UV:%[0-9]+]]:_(p0), [[UV1:%[0-9]+]]:_(p0) = G_UNMERGE_VALUES [[COPY1]](<2 x p0>)
+    ; CHECK: [[UV2:%[0-9]+]]:_(p0), [[UV3:%[0-9]+]]:_(p0) = G_UNMERGE_VALUES [[COPY2]](<2 x p0>)
+    ; CHECK: [[SELECT:%[0-9]+]]:_(p0) = G_SELECT [[ICMP]](s1), [[UV]], [[UV2]]
+    ; CHECK: [[SELECT1:%[0-9]+]]:_(p0) = G_SELECT [[ICMP]](s1), [[UV1]], [[UV3]]
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x p0>) = G_BUILD_VECTOR [[SELECT]](p0), [[SELECT1]](p0)
+    ; CHECK: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x p0>)
+    %0:_(s32) = G_CONSTANT i32 0
+    %1:_(s32) = COPY $vgpr0
+    %2:_(<2 x p0>) = COPY $vgpr1_vgpr2_vgpr3_vgpr4
+    %3:_(<2 x p0>) = COPY $vgpr5_vgpr6_vgpr7_vgpr8
+
+    %4:_(s1) = G_ICMP intpred(ne), %0, %1
+    %5:_(<2 x p0>) = G_SELECT %4, %2, %3
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %5
+
+...
+
+---
+name: test_select_v4p0
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8, $vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16
+
+    ; CHECK-LABEL: name: test_select_v4p0
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(<4 x p0>) = COPY $vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8
+    ; CHECK: [[COPY2:%[0-9]+]]:_(<4 x p0>) = COPY $vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16
+    ; CHECK: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[C]](s32), [[COPY]]
+    ; CHECK: [[UV:%[0-9]+]]:_(p0), [[UV1:%[0-9]+]]:_(p0), [[UV2:%[0-9]+]]:_(p0), [[UV3:%[0-9]+]]:_(p0) = G_UNMERGE_VALUES [[COPY1]](<4 x p0>)
+    ; CHECK: [[UV4:%[0-9]+]]:_(p0), [[UV5:%[0-9]+]]:_(p0), [[UV6:%[0-9]+]]:_(p0), [[UV7:%[0-9]+]]:_(p0) = G_UNMERGE_VALUES [[COPY2]](<4 x p0>)
+    ; CHECK: [[SELECT:%[0-9]+]]:_(p0) = G_SELECT [[ICMP]](s1), [[UV]], [[UV4]]
+    ; CHECK: [[SELECT1:%[0-9]+]]:_(p0) = G_SELECT [[ICMP]](s1), [[UV1]], [[UV5]]
+    ; CHECK: [[SELECT2:%[0-9]+]]:_(p0) = G_SELECT [[ICMP]](s1), [[UV2]], [[UV6]]
+    ; CHECK: [[SELECT3:%[0-9]+]]:_(p0) = G_SELECT [[ICMP]](s1), [[UV3]], [[UV7]]
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x p0>) = G_BUILD_VECTOR [[SELECT]](p0), [[SELECT1]](p0), [[SELECT2]](p0), [[SELECT3]](p0)
+    ; CHECK: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY [[BUILD_VECTOR]](<4 x p0>)
+    %0:_(s32) = G_CONSTANT i32 0
+    %1:_(s32) = COPY $vgpr0
+    %2:_(<4 x p0>) = COPY $vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8
+    %3:_(<4 x p0>) = COPY $vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16
+
+    %4:_(s1) = G_ICMP intpred(ne), %0, %1
+    %5:_(<4 x p0>) = G_SELECT %4, %2, %3
+    $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY %5
+
+...
+
+---
+name: test_select_v2s96
+body: |
+  bb.0:
+    liveins: $vgpr0
+
+    ; CHECK-LABEL: name: test_select_v2s96
+    ; CHECK: [[DEF:%[0-9]+]]:_(<2 x s96>) = G_IMPLICIT_DEF
+    ; CHECK: [[DEF1:%[0-9]+]]:_(<2 x s96>) = G_IMPLICIT_DEF
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; CHECK: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[COPY]](s32), [[C]]
+    ; CHECK: [[UV:%[0-9]+]]:_(s96), [[UV1:%[0-9]+]]:_(s96) = G_UNMERGE_VALUES [[DEF]](<2 x s96>)
+    ; CHECK: [[UV2:%[0-9]+]]:_(s96), [[UV3:%[0-9]+]]:_(s96) = G_UNMERGE_VALUES [[DEF1]](<2 x s96>)
+    ; CHECK: [[EXTRACT:%[0-9]+]]:_(s64) = G_EXTRACT [[UV]](s96), 0
+    ; CHECK: [[EXTRACT1:%[0-9]+]]:_(s32) = G_EXTRACT [[UV]](s96), 64
+    ; CHECK: [[EXTRACT2:%[0-9]+]]:_(s64) = G_EXTRACT [[UV2]](s96), 0
+    ; CHECK: [[EXTRACT3:%[0-9]+]]:_(s32) = G_EXTRACT [[UV2]](s96), 64
+    ; CHECK: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[EXTRACT]], [[EXTRACT2]]
+    ; CHECK: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[EXTRACT1]], [[EXTRACT3]]
+    ; CHECK: [[DEF2:%[0-9]+]]:_(s96) = G_IMPLICIT_DEF
+    ; CHECK: [[INSERT:%[0-9]+]]:_(s96) = G_INSERT [[DEF2]], [[SELECT]](s64), 0
+    ; CHECK: [[INSERT1:%[0-9]+]]:_(s96) = G_INSERT [[INSERT]], [[SELECT1]](s32), 64
+    ; CHECK: [[EXTRACT4:%[0-9]+]]:_(s64) = G_EXTRACT [[UV1]](s96), 0
+    ; CHECK: [[EXTRACT5:%[0-9]+]]:_(s32) = G_EXTRACT [[UV1]](s96), 64
+    ; CHECK: [[EXTRACT6:%[0-9]+]]:_(s64) = G_EXTRACT [[UV3]](s96), 0
+    ; CHECK: [[EXTRACT7:%[0-9]+]]:_(s32) = G_EXTRACT [[UV3]](s96), 64
+    ; CHECK: [[SELECT2:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[EXTRACT4]], [[EXTRACT6]]
+    ; CHECK: [[SELECT3:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[EXTRACT5]], [[EXTRACT7]]
+    ; CHECK: [[DEF3:%[0-9]+]]:_(s96) = G_IMPLICIT_DEF
+    ; CHECK: [[INSERT2:%[0-9]+]]:_(s96) = G_INSERT [[DEF3]], [[SELECT2]](s64), 0
+    ; CHECK: [[INSERT3:%[0-9]+]]:_(s96) = G_INSERT [[INSERT2]], [[SELECT3]](s32), 64
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s96>) = G_BUILD_VECTOR [[INSERT1]](s96), [[INSERT3]](s96)
+    ; CHECK: S_NOP 0, implicit [[BUILD_VECTOR]](<2 x s96>)
+    %0:_(<2 x s96>) = G_IMPLICIT_DEF
+    %1:_(<2 x s96>) = G_IMPLICIT_DEF
+    %2:_(s32) = COPY $vgpr0
+    %3:_(s32) = G_CONSTANT i32 0
+    %4:_(s1) = G_ICMP intpred(ne), %2, %3
+    %5:_(<2 x s96>) = G_SELECT %4, %0, %1
+    S_NOP 0, implicit %5
+
+...
+
+---
+
+name: test_select_v8p0
+body: |
+  bb.0:
+    liveins: $vgpr0
+
+    ; CHECK-LABEL: name: test_select_v8p0
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[DEF:%[0-9]+]]:_(<8 x p0>) = G_IMPLICIT_DEF
+    ; CHECK: [[DEF1:%[0-9]+]]:_(<8 x p0>) = G_IMPLICIT_DEF
+    ; CHECK: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[C]](s32), [[COPY]]
+    ; CHECK: [[UV:%[0-9]+]]:_(p0), [[UV1:%[0-9]+]]:_(p0), [[UV2:%[0-9]+]]:_(p0), [[UV3:%[0-9]+]]:_(p0), [[UV4:%[0-9]+]]:_(p0), [[UV5:%[0-9]+]]:_(p0), [[UV6:%[0-9]+]]:_(p0), [[UV7:%[0-9]+]]:_(p0) = G_UNMERGE_VALUES [[DEF]](<8 x p0>)
+    ; CHECK: [[UV8:%[0-9]+]]:_(p0), [[UV9:%[0-9]+]]:_(p0), [[UV10:%[0-9]+]]:_(p0), [[UV11:%[0-9]+]]:_(p0), [[UV12:%[0-9]+]]:_(p0), [[UV13:%[0-9]+]]:_(p0), [[UV14:%[0-9]+]]:_(p0), [[UV15:%[0-9]+]]:_(p0) = G_UNMERGE_VALUES [[DEF1]](<8 x p0>)
+    ; CHECK: [[SELECT:%[0-9]+]]:_(p0) = G_SELECT [[ICMP]](s1), [[UV]], [[UV8]]
+    ; CHECK: [[SELECT1:%[0-9]+]]:_(p0) = G_SELECT [[ICMP]](s1), [[UV1]], [[UV9]]
+    ; CHECK: [[SELECT2:%[0-9]+]]:_(p0) = G_SELECT [[ICMP]](s1), [[UV2]], [[UV10]]
+    ; CHECK: [[SELECT3:%[0-9]+]]:_(p0) = G_SELECT [[ICMP]](s1), [[UV3]], [[UV11]]
+    ; CHECK: [[SELECT4:%[0-9]+]]:_(p0) = G_SELECT [[ICMP]](s1), [[UV4]], [[UV12]]
+    ; CHECK: [[SELECT5:%[0-9]+]]:_(p0) = G_SELECT [[ICMP]](s1), [[UV5]], [[UV13]]
+    ; CHECK: [[SELECT6:%[0-9]+]]:_(p0) = G_SELECT [[ICMP]](s1), [[UV6]], [[UV14]]
+    ; CHECK: [[SELECT7:%[0-9]+]]:_(p0) = G_SELECT [[ICMP]](s1), [[UV7]], [[UV15]]
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x p0>) = G_BUILD_VECTOR [[SELECT]](p0), [[SELECT1]](p0), [[SELECT2]](p0), [[SELECT3]](p0), [[SELECT4]](p0), [[SELECT5]](p0), [[SELECT6]](p0), [[SELECT7]](p0)
+    ; CHECK: S_NOP 0, implicit [[BUILD_VECTOR]](<8 x p0>)
+    %0:_(s32) = G_CONSTANT i32 0
+    %1:_(s32) = COPY $vgpr0
+    %2:_(<8 x p0>) = G_IMPLICIT_DEF
+    %3:_(<8 x p0>) = G_IMPLICIT_DEF
+
+    %4:_(s1) = G_ICMP intpred(ne), %0, %1
+    %5:_(<8 x p0>) = G_SELECT %4, %2, %3
+    S_NOP 0, implicit %5
+...
+---
+name: test_select_v2s128
+body: |
+  bb.0:
+    liveins:  $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15, $vgpr16
+
+    ; CHECK-LABEL: name: test_select_v2s128
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s128>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+    ; CHECK: [[COPY1:%[0-9]+]]:_(<2 x s128>) = COPY $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+    ; CHECK: [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr16
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; CHECK: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[COPY2]](s32), [[C]]
+    ; CHECK: [[UV:%[0-9]+]]:_(s128), [[UV1:%[0-9]+]]:_(s128) = G_UNMERGE_VALUES [[COPY]](<2 x s128>)
+    ; CHECK: [[UV2:%[0-9]+]]:_(s128), [[UV3:%[0-9]+]]:_(s128) = G_UNMERGE_VALUES [[COPY1]](<2 x s128>)
+    ; CHECK: [[UV4:%[0-9]+]]:_(s64), [[UV5:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[UV]](s128)
+    ; CHECK: [[UV6:%[0-9]+]]:_(s64), [[UV7:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[UV2]](s128)
+    ; CHECK: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[UV4]], [[UV6]]
+    ; CHECK: [[SELECT1:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[UV5]], [[UV7]]
+    ; CHECK: [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[SELECT]](s64), [[SELECT1]](s64)
+    ; CHECK: [[UV8:%[0-9]+]]:_(s64), [[UV9:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[UV1]](s128)
+    ; CHECK: [[UV10:%[0-9]+]]:_(s64), [[UV11:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[UV3]](s128)
+    ; CHECK: [[SELECT2:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[UV8]], [[UV10]]
+    ; CHECK: [[SELECT3:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[UV9]], [[UV11]]
+    ; CHECK: [[MV1:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[SELECT2]](s64), [[SELECT3]](s64)
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s128>) = G_BUILD_VECTOR [[MV]](s128), [[MV1]](s128)
+    ; CHECK: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY [[BUILD_VECTOR]](<2 x s128>)
+    %0:_(<2 x s128>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 ,
+    %1:_(<2 x s128>) = COPY $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+    %2:_(s32) = COPY $vgpr16
+    %3:_(s32) = G_CONSTANT i32 0
+    %4:_(s1) = G_ICMP intpred(ne), %2, %3
+    %5:_(<2 x s128>) = G_SELECT %4, %0, %1
+    $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7   = COPY %5
+
+...
+
+---
+name: test_vselect_v2s32
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2_vgpr3, $vgpr4_vgpr5, $vgpr6_vgpr7
+    ; CHECK-LABEL: name: test_vselect_v2s32
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    ; CHECK: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr2_vgpr3
+    ; CHECK: [[COPY2:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr4_vgpr5
+    ; CHECK: [[COPY3:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr6_vgpr7
+    ; CHECK: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
+    ; CHECK: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<2 x s32>)
+    ; CHECK: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[UV]](s32), [[UV2]]
+    ; CHECK: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[UV1]](s32), [[UV3]]
+    ; CHECK: [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY2]](<2 x s32>)
+    ; CHECK: [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY3]](<2 x s32>)
+    ; CHECK: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[UV4]], [[UV6]]
+    ; CHECK: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP1]](s1), [[UV5]], [[UV7]]
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[SELECT]](s32), [[SELECT1]](s32)
+    ; CHECK: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
+    %0:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    %1:_(<2 x s32>) = COPY $vgpr2_vgpr3
+    %2:_(<2 x s32>) = COPY $vgpr4_vgpr5
+    %3:_(<2 x s32>) = COPY $vgpr6_vgpr7
+
+    %4:_(<2 x s1>) = G_ICMP intpred(ne), %0, %1
+    %5:_(<2 x s32>) = G_SELECT %4, %2, %3
+    $vgpr0_vgpr1 = COPY %5
+
+...
+
+---
+name: test_vselect_v3s32
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2, $vgpr3_vgpr4_vgpr5, $vgpr6_vgpr7_vgpr8
+    ; CHECK-LABEL: name: test_vselect_v3s32
+    ; CHECK: [[COPY:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr0_vgpr1_vgpr2
+    ; CHECK: [[COPY1:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr3_vgpr4_vgpr5
+    ; CHECK: [[COPY2:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr6_vgpr7_vgpr8
+    ; CHECK: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<3 x s32>)
+    ; CHECK: [[UV3:%[0-9]+]]:_(s32), [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<3 x s32>)
+    ; CHECK: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[UV]](s32), [[UV3]]
+    ; CHECK: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[UV1]](s32), [[UV4]]
+    ; CHECK: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[UV2]](s32), [[UV5]]
+    ; CHECK: [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32), [[UV8:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<3 x s32>)
+    ; CHECK: [[UV9:%[0-9]+]]:_(s32), [[UV10:%[0-9]+]]:_(s32), [[UV11:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY2]](<3 x s32>)
+    ; CHECK: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[UV6]], [[UV9]]
+    ; CHECK: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP1]](s1), [[UV7]], [[UV10]]
+    ; CHECK: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[UV8]], [[UV11]]
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[SELECT]](s32), [[SELECT1]](s32), [[SELECT2]](s32)
+    ; CHECK: $vgpr0_vgpr1_vgpr2 = COPY [[BUILD_VECTOR]](<3 x s32>)
+    %0:_(<3 x s32>) = COPY $vgpr0_vgpr1_vgpr2
+    %1:_(<3 x s32>) = COPY $vgpr3_vgpr4_vgpr5
+    %2:_(<3 x s32>) = COPY $vgpr6_vgpr7_vgpr8
+
+    %3:_(<3 x s1>) = G_ICMP intpred(ne), %0, %1
+    %4:_(<3 x s32>) = G_SELECT %3, %1, %2
+    $vgpr0_vgpr1_vgpr2 = COPY %4
+
+...
+
+---
+name: test_vselect_v4s32
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11
+    ; CHECK-LABEL: name: test_vselect_v4s32
+    ; CHECK: [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; CHECK: [[COPY1:%[0-9]+]]:_(<4 x s32>) = COPY $vgpr4_vgpr5_vgpr6_vgpr7
+    ; CHECK: [[COPY2:%[0-9]+]]:_(<4 x s32>) = COPY $vgpr8_vgpr9_vgpr10_vgpr11
+    ; CHECK: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<4 x s32>)
+    ; CHECK: [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32), [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<4 x s32>)
+    ; CHECK: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[UV]](s32), [[UV4]]
+    ; CHECK: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[UV1]](s32), [[UV5]]
+    ; CHECK: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[UV2]](s32), [[UV6]]
+    ; CHECK: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[UV3]](s32), [[UV7]]
+    ; CHECK: [[UV8:%[0-9]+]]:_(s32), [[UV9:%[0-9]+]]:_(s32), [[UV10:%[0-9]+]]:_(s32), [[UV11:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<4 x s32>)
+    ; CHECK: [[UV12:%[0-9]+]]:_(s32), [[UV13:%[0-9]+]]:_(s32), [[UV14:%[0-9]+]]:_(s32), [[UV15:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY2]](<4 x s32>)
+    ; CHECK: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[ICMP]](s1), [[UV8]], [[UV12]]
+    ; CHECK: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP1]](s1), [[UV9]], [[UV13]]
+    ; CHECK: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[UV10]], [[UV14]]
+    ; CHECK: [[SELECT3:%[0-9]+]]:_(s32) = G_SELECT [[ICMP3]](s1), [[UV11]], [[UV15]]
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[SELECT]](s32), [[SELECT1]](s32), [[SELECT2]](s32), [[SELECT3]](s32)
+    ; CHECK: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<4 x s32>)
+    %0:_(<4 x s32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    %1:_(<4 x s32>) = COPY $vgpr4_vgpr5_vgpr6_vgpr7
+    %2:_(<4 x s32>) = COPY $vgpr8_vgpr9_vgpr10_vgpr11
+
+    %3:_(<4 x s1>) = G_ICMP intpred(ne), %0, %1
+    %4:_(<4 x s32>) = G_SELECT %3, %1, %2
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %4
+
+...
+
+---
+name: test_vselect_v2s64
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11
+    ; CHECK-LABEL: name: test_vselect_v2s64
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; CHECK: [[COPY1:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr4_vgpr5_vgpr6_vgpr7
+    ; CHECK: [[COPY2:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr8_vgpr9_vgpr10_vgpr11
+    ; CHECK: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](<2 x s64>)
+    ; CHECK: [[UV2:%[0-9]+]]:_(s64), [[UV3:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY1]](<2 x s64>)
+    ; CHECK: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[UV]](s64), [[UV2]]
+    ; CHECK: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[UV1]](s64), [[UV3]]
+    ; CHECK: [[UV4:%[0-9]+]]:_(s64), [[UV5:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY1]](<2 x s64>)
+    ; CHECK: [[UV6:%[0-9]+]]:_(s64), [[UV7:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY2]](<2 x s64>)
+    ; CHECK: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[UV4]], [[UV6]]
+    ; CHECK: [[SELECT1:%[0-9]+]]:_(s64) = G_SELECT [[ICMP1]](s1), [[UV5]], [[UV7]]
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[SELECT]](s64), [[SELECT1]](s64)
+    ; CHECK: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>)
+    %0:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    %1:_(<2 x s64>) = COPY $vgpr4_vgpr5_vgpr6_vgpr7
+    %2:_(<2 x s64>) = COPY $vgpr8_vgpr9_vgpr10_vgpr11
+
+    %3:_(<2 x s1>) = G_ICMP intpred(ne), %0, %1
+    %4:_(<2 x s64>) = G_SELECT %3, %1, %2
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %4
+
+...
+
+---
+name: test_vselect_v2p3
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2_vgpr3, $vgpr4_vgpr5, $vgpr6_vgpr7
+    ; CHECK-LABEL: name: test_vselect_v2p3
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    ; CHECK: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr2_vgpr3
+    ; CHECK: [[COPY2:%[0-9]+]]:_(<2 x p3>) = COPY $vgpr4_vgpr5
+    ; CHECK: [[COPY3:%[0-9]+]]:_(<2 x p3>) = COPY $vgpr6_vgpr7
+    ; CHECK: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
+    ; CHECK: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<2 x s32>)
+    ; CHECK: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[UV]](s32), [[UV2]]
+    ; CHECK: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[UV1]](s32), [[UV3]]
+    ; CHECK: [[UV4:%[0-9]+]]:_(p3), [[UV5:%[0-9]+]]:_(p3) = G_UNMERGE_VALUES [[COPY2]](<2 x p3>)
+    ; CHECK: [[UV6:%[0-9]+]]:_(p3), [[UV7:%[0-9]+]]:_(p3) = G_UNMERGE_VALUES [[COPY3]](<2 x p3>)
+    ; CHECK: [[SELECT:%[0-9]+]]:_(p3) = G_SELECT [[ICMP]](s1), [[UV4]], [[UV6]]
+    ; CHECK: [[SELECT1:%[0-9]+]]:_(p3) = G_SELECT [[ICMP1]](s1), [[UV5]], [[UV7]]
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x p3>) = G_BUILD_VECTOR [[SELECT]](p3), [[SELECT1]](p3)
+    ; CHECK: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x p3>)
+    %0:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    %1:_(<2 x s32>) = COPY $vgpr2_vgpr3
+    %2:_(<2 x p3>) = COPY $vgpr4_vgpr5
+    %3:_(<2 x p3>) = COPY $vgpr6_vgpr7
+
+    %4:_(<2 x s1>) = G_ICMP intpred(ne), %0, %1
+    %5:_(<2 x p3>) = G_SELECT %4, %2, %3
+    $vgpr0_vgpr1 = COPY %5
+
+...
+
+---
+name: test_vselect_v2p0
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10_vgpr11, $vgpr12_vgpr13_vgpr14_vgpr15
+
+    ; CHECK-LABEL: name: test_vselect_v2p0
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; CHECK: [[COPY1:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr4_vgpr5_vgpr6_vgpr7
+    ; CHECK: [[COPY2:%[0-9]+]]:_(<2 x p0>) = COPY $vgpr8_vgpr9_vgpr10_vgpr11
+    ; CHECK: [[COPY3:%[0-9]+]]:_(<2 x p0>) = COPY $vgpr12_vgpr13_vgpr14_vgpr15
+    ; CHECK: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](<2 x s64>)
+    ; CHECK: [[UV2:%[0-9]+]]:_(s64), [[UV3:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY1]](<2 x s64>)
+    ; CHECK: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[UV]](s64), [[UV2]]
+    ; CHECK: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[UV1]](s64), [[UV3]]
+    ; CHECK: [[UV4:%[0-9]+]]:_(p0), [[UV5:%[0-9]+]]:_(p0) = G_UNMERGE_VALUES [[COPY2]](<2 x p0>)
+    ; CHECK: [[UV6:%[0-9]+]]:_(p0), [[UV7:%[0-9]+]]:_(p0) = G_UNMERGE_VALUES [[COPY3]](<2 x p0>)
+    ; CHECK: [[SELECT:%[0-9]+]]:_(p0) = G_SELECT [[ICMP]](s1), [[UV4]], [[UV6]]
+    ; CHECK: [[SELECT1:%[0-9]+]]:_(p0) = G_SELECT [[ICMP1]](s1), [[UV5]], [[UV7]]
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x p0>) = G_BUILD_VECTOR [[SELECT]](p0), [[SELECT1]](p0)
+    ; CHECK: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x p0>)
+    %0:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    %1:_(<2 x s64>) = COPY $vgpr4_vgpr5_vgpr6_vgpr7
+    %2:_(<2 x p0>) = COPY $vgpr8_vgpr9_vgpr10_vgpr11
+    %3:_(<2 x p0>) = COPY $vgpr12_vgpr13_vgpr14_vgpr15
+
+    %4:_(<2 x s1>) = G_ICMP intpred(ne), %0, %1
+    %5:_(<2 x p0>) = G_SELECT %4, %2, %3
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %5
+
+...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/legalize-sext.mir b/test/CodeGen/AMDGPU/GlobalISel/legalize-sext.mir
new file mode 100644
index 0000000000000000000000000000000000000000..20dc8713888993baec213fb5bfacc9b1b413e903
--- /dev/null
+++ b/test/CodeGen/AMDGPU/GlobalISel/legalize-sext.mir
@@ -0,0 +1,193 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -O0 -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -run-pass=legalizer %s -o - | FileCheck %s
+
+---
+name: test_sext_s32_to_s64
+body: |
+  bb.0:
+    liveins: $vgpr0
+
+    ; CHECK-LABEL: name: test_sext_s32_to_s64
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[SEXT:%[0-9]+]]:_(s64) = G_SEXT [[COPY]](s32)
+    ; CHECK: $vgpr0_vgpr1 = COPY [[SEXT]](s64)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s64) = G_SEXT %0
+    $vgpr0_vgpr1 = COPY %1
+...
+
+---
+name: test_sext_s16_to_s64
+body: |
+  bb.0:
+    liveins: $vgpr0
+
+    ; CHECK-LABEL: name: test_sext_s16_to_s64
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 48
+    ; CHECK: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[COPY]](s32)
+    ; CHECK: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[C]](s64)
+    ; CHECK: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[ANYEXT]], [[TRUNC]](s32)
+    ; CHECK: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[C]](s64)
+    ; CHECK: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[SHL]], [[TRUNC1]](s32)
+    ; CHECK: $vgpr0_vgpr1 = COPY [[ASHR]](s64)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s16) = G_TRUNC %0
+    %2:_(s64) = G_SEXT %1
+    $vgpr0_vgpr1 = COPY %2
+...
+
+---
+name: test_sext_s16_to_s32
+body: |
+  bb.0:
+    liveins: $vgpr0
+
+    ; CHECK-LABEL: name: test_sext_s16_to_s32
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; CHECK: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY1]], [[C]](s32)
+    ; CHECK: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C]](s32)
+    ; CHECK: $vgpr0 = COPY [[ASHR]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s16) = G_TRUNC %0
+    %2:_(s32) = G_SEXT %1
+    $vgpr0 = COPY %2
+...
+
+---
+name: test_sext_i1_to_s32
+body: |
+  bb.0:
+    liveins: $vgpr0
+
+    ; CHECK-LABEL: name: test_sext_i1_to_s32
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 31
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; CHECK: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY1]], [[C]](s32)
+    ; CHECK: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C]](s32)
+    ; CHECK: $vgpr0 = COPY [[ASHR]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s1) = G_TRUNC %0
+    %2:_(s32) = G_SEXT %1
+    $vgpr0 = COPY %2
+...
+
+---
+name: test_sext_v2s16_to_v2s32
+body: |
+  bb.0:
+    liveins: $vgpr0
+
+    ; CHECK-LABEL: name: test_sext_v2s16_to_v2s32
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
+    ; CHECK: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY]](<2 x s16>)
+    ; CHECK: [[SEXT:%[0-9]+]]:_(s32) = G_SEXT [[UV]](s16)
+    ; CHECK: [[SEXT1:%[0-9]+]]:_(s32) = G_SEXT [[UV1]](s16)
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[SEXT]](s32), [[SEXT1]](s32)
+    ; CHECK: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
+    %0:_(<2 x s16>) = COPY $vgpr0
+    %1:_(<2 x s32>) = G_SEXT %0
+    $vgpr0_vgpr1 = COPY %1
+...
+
+---
+name: test_sext_v3s16_to_v3s32
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; CHECK-LABEL: name: test_sext_v3s16_to_v3s32
+    ; CHECK: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
+    ; CHECK: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[COPY]](<4 x s16>), 0
+    ; CHECK: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16), [[UV2:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[EXTRACT]](<3 x s16>)
+    ; CHECK: [[SEXT:%[0-9]+]]:_(s32) = G_SEXT [[UV]](s16)
+    ; CHECK: [[SEXT1:%[0-9]+]]:_(s32) = G_SEXT [[UV1]](s16)
+    ; CHECK: [[SEXT2:%[0-9]+]]:_(s32) = G_SEXT [[UV2]](s16)
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[SEXT]](s32), [[SEXT1]](s32), [[SEXT2]](s32)
+    ; CHECK: $vgpr0_vgpr1_vgpr2 = COPY [[BUILD_VECTOR]](<3 x s32>)
+    %0:_(<4 x s16>) = COPY $vgpr0_vgpr1
+    %1:_(<3 x s16>) = G_EXTRACT %0, 0
+    %2:_(<3 x s32>) = G_SEXT %1
+    $vgpr0_vgpr1_vgpr2 = COPY %2
+...
+
+---
+name: test_sext_v4s16_to_v4s32
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; CHECK-LABEL: name: test_sext_v4s16_to_v4s32
+    ; CHECK: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
+    ; CHECK: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16), [[UV2:%[0-9]+]]:_(s16), [[UV3:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
+    ; CHECK: [[SEXT:%[0-9]+]]:_(s32) = G_SEXT [[UV]](s16)
+    ; CHECK: [[SEXT1:%[0-9]+]]:_(s32) = G_SEXT [[UV1]](s16)
+    ; CHECK: [[SEXT2:%[0-9]+]]:_(s32) = G_SEXT [[UV2]](s16)
+    ; CHECK: [[SEXT3:%[0-9]+]]:_(s32) = G_SEXT [[UV3]](s16)
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[SEXT]](s32), [[SEXT1]](s32), [[SEXT2]](s32), [[SEXT3]](s32)
+    ; CHECK: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<4 x s32>)
+    %0:_(<4 x s16>) = COPY $vgpr0_vgpr1
+    %1:_(<4 x s32>) = G_SEXT %0
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %1
+...
+
+---
+name: test_sext_v2s32_to_v2s64
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; CHECK-LABEL: name: test_sext_v2s32_to_v2s64
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    ; CHECK: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
+    ; CHECK: [[SEXT:%[0-9]+]]:_(s64) = G_SEXT [[UV]](s32)
+    ; CHECK: [[SEXT1:%[0-9]+]]:_(s64) = G_SEXT [[UV1]](s32)
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[SEXT]](s64), [[SEXT1]](s64)
+    ; CHECK: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>)
+    %0:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    %1:_(<2 x s64>) = G_SEXT %0
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %1
+...
+
+---
+name: test_sext_v3s32_to_v3s64
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2
+
+    ; CHECK-LABEL: name: test_sext_v3s32_to_v3s64
+    ; CHECK: [[COPY:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr0_vgpr1_vgpr2
+    ; CHECK: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<3 x s32>)
+    ; CHECK: [[SEXT:%[0-9]+]]:_(s64) = G_SEXT [[UV]](s32)
+    ; CHECK: [[SEXT1:%[0-9]+]]:_(s64) = G_SEXT [[UV1]](s32)
+    ; CHECK: [[SEXT2:%[0-9]+]]:_(s64) = G_SEXT [[UV2]](s32)
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s64>) = G_BUILD_VECTOR [[SEXT]](s64), [[SEXT1]](s64), [[SEXT2]](s64)
+    ; CHECK: S_NOP 0, implicit [[BUILD_VECTOR]](<3 x s64>)
+    %0:_(<3 x s32>) = COPY $vgpr0_vgpr1_vgpr2
+    %1:_(<3 x s64>) = G_SEXT %0
+    S_NOP 0, implicit %1
+
+...
+
+---
+name: test_sext_v4s32_to_v4s64
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3
+
+    ; CHECK-LABEL: name: test_sext_v4s32_to_v4s64
+    ; CHECK: [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; CHECK: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<4 x s32>)
+    ; CHECK: [[SEXT:%[0-9]+]]:_(s64) = G_SEXT [[UV]](s32)
+    ; CHECK: [[SEXT1:%[0-9]+]]:_(s64) = G_SEXT [[UV1]](s32)
+    ; CHECK: [[SEXT2:%[0-9]+]]:_(s64) = G_SEXT [[UV2]](s32)
+    ; CHECK: [[SEXT3:%[0-9]+]]:_(s64) = G_SEXT [[UV3]](s32)
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s64>) = G_BUILD_VECTOR [[SEXT]](s64), [[SEXT1]](s64), [[SEXT2]](s64), [[SEXT3]](s64)
+    ; CHECK: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY [[BUILD_VECTOR]](<4 x s64>)
+    %0:_(<4 x s32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    %1:_(<4 x s64>) = G_SEXT %0
+    $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY %1
+...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/legalize-sextload-flat.mir b/test/CodeGen/AMDGPU/GlobalISel/legalize-sextload-flat.mir
new file mode 100644
index 0000000000000000000000000000000000000000..98b5c45eced8128469d2720453c84866970be63a
--- /dev/null
+++ b/test/CodeGen/AMDGPU/GlobalISel/legalize-sextload-flat.mir
@@ -0,0 +1,140 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -run-pass=legalizer -o - %s | FileCheck %s -check-prefixes=GCN,SI
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=tahiti -run-pass=legalizer -o - %s | FileCheck %s -check-prefixes=GCN,VI
+---
+name: test_sextload_flat_i32_i8
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; SI-LABEL: name: test_sextload_flat_i32_i8
+    ; SI: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
+    ; SI: [[SEXTLOAD:%[0-9]+]]:_(s32) = G_SEXTLOAD [[COPY]](p0) :: (load 1)
+    ; SI: $vgpr0 = COPY [[SEXTLOAD]](s32)
+    ; VI-LABEL: name: test_sextload_flat_i32_i8
+    ; VI: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
+    ; VI: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p0) :: (load 1)
+    ; VI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; VI: [[COPY1:%[0-9]+]]:_(s32) = COPY [[LOAD]](s32)
+    ; VI: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY1]], [[C]](s32)
+    ; VI: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C]](s32)
+    ; VI: $vgpr0 = COPY [[ASHR]](s32)
+    %0:_(p0) = COPY $vgpr0_vgpr1
+    %1:_(s32) = G_SEXTLOAD %0 :: (load 1, addrspace 0)
+    $vgpr0 = COPY %1
+...
+---
+name: test_sextload_flat_i32_i16
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; SI-LABEL: name: test_sextload_flat_i32_i16
+    ; SI: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
+    ; SI: [[SEXTLOAD:%[0-9]+]]:_(s32) = G_SEXTLOAD [[COPY]](p0) :: (load 2)
+    ; SI: $vgpr0 = COPY [[SEXTLOAD]](s32)
+    ; VI-LABEL: name: test_sextload_flat_i32_i16
+    ; VI: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
+    ; VI: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p0) :: (load 2)
+    ; VI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; VI: [[COPY1:%[0-9]+]]:_(s32) = COPY [[LOAD]](s32)
+    ; VI: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY1]], [[C]](s32)
+    ; VI: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C]](s32)
+    ; VI: $vgpr0 = COPY [[ASHR]](s32)
+     %0:_(p0) = COPY $vgpr0_vgpr1
+    %1:_(s32) = G_SEXTLOAD %0 :: (load 2, addrspace 0)
+    $vgpr0 = COPY %1
+...
+---
+name: test_sextload_flat_i31_i8
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; SI-LABEL: name: test_sextload_flat_i31_i8
+    ; SI: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
+    ; SI: [[SEXTLOAD:%[0-9]+]]:_(s32) = G_SEXTLOAD [[COPY]](p0) :: (load 1)
+    ; SI: [[COPY1:%[0-9]+]]:_(s32) = COPY [[SEXTLOAD]](s32)
+    ; SI: $vgpr0 = COPY [[COPY1]](s32)
+    ; VI-LABEL: name: test_sextload_flat_i31_i8
+    ; VI: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
+    ; VI: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p0) :: (load 1)
+    ; VI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; VI: [[COPY1:%[0-9]+]]:_(s32) = COPY [[LOAD]](s32)
+    ; VI: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY1]], [[C]](s32)
+    ; VI: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C]](s32)
+    ; VI: [[COPY2:%[0-9]+]]:_(s32) = COPY [[ASHR]](s32)
+    ; VI: $vgpr0 = COPY [[COPY2]](s32)
+    %0:_(p0) = COPY $vgpr0_vgpr1
+    %1:_(s31) = G_SEXTLOAD %0 :: (load 1, addrspace 0)
+    %2:_(s32) = G_ANYEXT %1
+    $vgpr0 = COPY %2
+...
+---
+name: test_sextload_flat_i64_i8
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; SI-LABEL: name: test_sextload_flat_i64_i8
+    ; SI: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
+    ; SI: [[SEXTLOAD:%[0-9]+]]:_(s32) = G_SEXTLOAD [[COPY]](p0) :: (load 1)
+    ; SI: [[SEXT:%[0-9]+]]:_(s64) = G_SEXT [[SEXTLOAD]](s32)
+    ; SI: $vgpr0_vgpr1 = COPY [[SEXT]](s64)
+    ; VI-LABEL: name: test_sextload_flat_i64_i8
+    ; VI: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
+    ; VI: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p0) :: (load 1)
+    ; VI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; VI: [[COPY1:%[0-9]+]]:_(s32) = COPY [[LOAD]](s32)
+    ; VI: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY1]], [[C]](s32)
+    ; VI: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C]](s32)
+    ; VI: [[SEXT:%[0-9]+]]:_(s64) = G_SEXT [[ASHR]](s32)
+    ; VI: $vgpr0_vgpr1 = COPY [[SEXT]](s64)
+    %0:_(p0) = COPY $vgpr0_vgpr1
+    %1:_(s64) = G_SEXTLOAD %0 :: (load 1, addrspace 0)
+    $vgpr0_vgpr1 = COPY %1
+...
+---
+name: test_sextload_flat_i64_i16
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; SI-LABEL: name: test_sextload_flat_i64_i16
+    ; SI: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
+    ; SI: [[SEXTLOAD:%[0-9]+]]:_(s32) = G_SEXTLOAD [[COPY]](p0) :: (load 2)
+    ; SI: [[SEXT:%[0-9]+]]:_(s64) = G_SEXT [[SEXTLOAD]](s32)
+    ; SI: $vgpr0_vgpr1 = COPY [[SEXT]](s64)
+    ; VI-LABEL: name: test_sextload_flat_i64_i16
+    ; VI: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
+    ; VI: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p0) :: (load 2)
+    ; VI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; VI: [[COPY1:%[0-9]+]]:_(s32) = COPY [[LOAD]](s32)
+    ; VI: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY1]], [[C]](s32)
+    ; VI: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C]](s32)
+    ; VI: [[SEXT:%[0-9]+]]:_(s64) = G_SEXT [[ASHR]](s32)
+    ; VI: $vgpr0_vgpr1 = COPY [[SEXT]](s64)
+    %0:_(p0) = COPY $vgpr0_vgpr1
+    %1:_(s64) = G_SEXTLOAD %0 :: (load 2, addrspace 0)
+    $vgpr0_vgpr1 = COPY %1
+...
+---
+name: test_sextload_flat_i64_i32
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; SI-LABEL: name: test_sextload_flat_i64_i32
+    ; SI: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
+    ; SI: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p0) :: (load 4)
+    ; SI: [[SEXT:%[0-9]+]]:_(s64) = G_SEXT [[LOAD]](s32)
+    ; SI: $vgpr0_vgpr1 = COPY [[SEXT]](s64)
+    ; VI-LABEL: name: test_sextload_flat_i64_i32
+    ; VI: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
+    ; VI: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p0) :: (load 4)
+    ; VI: [[SEXT:%[0-9]+]]:_(s64) = G_SEXT [[LOAD]](s32)
+    ; VI: $vgpr0_vgpr1 = COPY [[SEXT]](s64)
+    %0:_(p0) = COPY $vgpr0_vgpr1
+    %1:_(s64) = G_SEXTLOAD %0 :: (load 4, addrspace 0)
+    $vgpr0_vgpr1 = COPY %1
+...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/legalize-sextload-global.mir b/test/CodeGen/AMDGPU/GlobalISel/legalize-sextload-global.mir
new file mode 100644
index 0000000000000000000000000000000000000000..40105bdb150b05c3ffc57aed49cd41ff087acb12
--- /dev/null
+++ b/test/CodeGen/AMDGPU/GlobalISel/legalize-sextload-global.mir
@@ -0,0 +1,93 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -run-pass=legalizer -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=tahiti -run-pass=legalizer -o - %s | FileCheck %s
+
+---
+name: test_sextload_global_i32_i8
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; CHECK-LABEL: name: test_sextload_global_i32_i8
+    ; CHECK: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
+    ; CHECK: [[SEXTLOAD:%[0-9]+]]:_(s32) = G_SEXTLOAD [[COPY]](p1) :: (load 1, addrspace 1)
+    ; CHECK: $vgpr0 = COPY [[SEXTLOAD]](s32)
+    %0:_(p1) = COPY $vgpr0_vgpr1
+    %1:_(s32) = G_SEXTLOAD %0 :: (load 1, addrspace 1)
+    $vgpr0 = COPY %1
+...
+---
+name: test_sextload_global_i32_i16
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; CHECK-LABEL: name: test_sextload_global_i32_i16
+    ; CHECK: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
+    ; CHECK: [[SEXTLOAD:%[0-9]+]]:_(s32) = G_SEXTLOAD [[COPY]](p1) :: (load 2, addrspace 1)
+    ; CHECK: $vgpr0 = COPY [[SEXTLOAD]](s32)
+    %0:_(p1) = COPY $vgpr0_vgpr1
+    %1:_(s32) = G_SEXTLOAD %0 :: (load 2, addrspace 1)
+    $vgpr0 = COPY %1
+...
+---
+name: test_sextload_global_i31_i8
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; CHECK-LABEL: name: test_sextload_global_i31_i8
+    ; CHECK: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
+    ; CHECK: [[SEXTLOAD:%[0-9]+]]:_(s32) = G_SEXTLOAD [[COPY]](p1) :: (load 1, addrspace 1)
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY [[SEXTLOAD]](s32)
+    ; CHECK: $vgpr0 = COPY [[COPY1]](s32)
+    %0:_(p1) = COPY $vgpr0_vgpr1
+    %1:_(s31) = G_SEXTLOAD %0 :: (load 1, addrspace 1)
+    %2:_(s32) = G_ANYEXT %1
+    $vgpr0 = COPY %2
+...
+---
+name: test_sextload_global_i64_i8
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; CHECK-LABEL: name: test_sextload_global_i64_i8
+    ; CHECK: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
+    ; CHECK: [[SEXTLOAD:%[0-9]+]]:_(s32) = G_SEXTLOAD [[COPY]](p1) :: (load 1, addrspace 1)
+    ; CHECK: [[SEXT:%[0-9]+]]:_(s64) = G_SEXT [[SEXTLOAD]](s32)
+    ; CHECK: $vgpr0_vgpr1 = COPY [[SEXT]](s64)
+    %0:_(p1) = COPY $vgpr0_vgpr1
+    %1:_(s64) = G_SEXTLOAD %0 :: (load 1, addrspace 1)
+    $vgpr0_vgpr1 = COPY %1
+...
+---
+name: test_sextload_global_i64_i16
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; CHECK-LABEL: name: test_sextload_global_i64_i16
+    ; CHECK: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
+    ; CHECK: [[SEXTLOAD:%[0-9]+]]:_(s32) = G_SEXTLOAD [[COPY]](p1) :: (load 2, addrspace 1)
+    ; CHECK: [[SEXT:%[0-9]+]]:_(s64) = G_SEXT [[SEXTLOAD]](s32)
+    ; CHECK: $vgpr0_vgpr1 = COPY [[SEXT]](s64)
+    %0:_(p1) = COPY $vgpr0_vgpr1
+    %1:_(s64) = G_SEXTLOAD %0 :: (load 2, addrspace 1)
+    $vgpr0_vgpr1 = COPY %1
+...
+---
+name: test_sextload_global_i64_i32
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; CHECK-LABEL: name: test_sextload_global_i64_i32
+    ; CHECK: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
+    ; CHECK: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p1) :: (load 4, addrspace 1)
+    ; CHECK: [[SEXT:%[0-9]+]]:_(s64) = G_SEXT [[LOAD]](s32)
+    ; CHECK: $vgpr0_vgpr1 = COPY [[SEXT]](s64)
+    %0:_(p1) = COPY $vgpr0_vgpr1
+    %1:_(s64) = G_SEXTLOAD %0 :: (load 4, addrspace 1)
+    $vgpr0_vgpr1 = COPY %1
+...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/legalize-sextload-local.mir b/test/CodeGen/AMDGPU/GlobalISel/legalize-sextload-local.mir
new file mode 100644
index 0000000000000000000000000000000000000000..6363200890837e93fb543cf8b286cf73b25ce911
--- /dev/null
+++ b/test/CodeGen/AMDGPU/GlobalISel/legalize-sextload-local.mir
@@ -0,0 +1,92 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -run-pass=legalizer -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=tahiti -run-pass=legalizer -o - %s | FileCheck %s
+---
+name: test_sextload_local_i32_i8
+body: |
+  bb.0:
+    liveins: $vgpr0
+
+    ; CHECK-LABEL: name: test_sextload_local_i32_i8
+    ; CHECK: [[COPY:%[0-9]+]]:_(p3) = COPY $vgpr0
+    ; CHECK: [[SEXTLOAD:%[0-9]+]]:_(s32) = G_SEXTLOAD [[COPY]](p3) :: (load 1, addrspace 3)
+    ; CHECK: $vgpr0 = COPY [[SEXTLOAD]](s32)
+    %0:_(p3) = COPY $vgpr0
+    %1:_(s32) = G_SEXTLOAD %0 :: (load 1, addrspace 3)
+    $vgpr0 = COPY %1
+...
+---
+name: test_sextload_local_i32_i16
+body: |
+  bb.0:
+    liveins: $vgpr0
+
+    ; CHECK-LABEL: name: test_sextload_local_i32_i16
+    ; CHECK: [[COPY:%[0-9]+]]:_(p3) = COPY $vgpr0
+    ; CHECK: [[SEXTLOAD:%[0-9]+]]:_(s32) = G_SEXTLOAD [[COPY]](p3) :: (load 2, addrspace 3)
+    ; CHECK: $vgpr0 = COPY [[SEXTLOAD]](s32)
+    %0:_(p3) = COPY $vgpr0
+    %1:_(s32) = G_SEXTLOAD %0 :: (load 2, addrspace 3)
+    $vgpr0 = COPY %1
+...
+---
+name: test_sextload_local_i31_i8
+body: |
+  bb.0:
+    liveins: $vgpr0
+
+    ; CHECK-LABEL: name: test_sextload_local_i31_i8
+    ; CHECK: [[COPY:%[0-9]+]]:_(p3) = COPY $vgpr0
+    ; CHECK: [[SEXTLOAD:%[0-9]+]]:_(s32) = G_SEXTLOAD [[COPY]](p3) :: (load 1, addrspace 3)
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY [[SEXTLOAD]](s32)
+    ; CHECK: $vgpr0 = COPY [[COPY1]](s32)
+    %0:_(p3) = COPY $vgpr0
+    %1:_(s31) = G_SEXTLOAD %0 :: (load 1, addrspace 3)
+    %2:_(s32) = G_ANYEXT %1
+    $vgpr0 = COPY %2
+...
+---
+name: test_sextload_local_i64_i8
+body: |
+  bb.0:
+    liveins: $vgpr0
+
+    ; CHECK-LABEL: name: test_sextload_local_i64_i8
+    ; CHECK: [[COPY:%[0-9]+]]:_(p3) = COPY $vgpr0
+    ; CHECK: [[SEXTLOAD:%[0-9]+]]:_(s32) = G_SEXTLOAD [[COPY]](p3) :: (load 1, addrspace 3)
+    ; CHECK: [[SEXT:%[0-9]+]]:_(s64) = G_SEXT [[SEXTLOAD]](s32)
+    ; CHECK: $vgpr0_vgpr1 = COPY [[SEXT]](s64)
+    %0:_(p3) = COPY $vgpr0
+    %1:_(s64) = G_SEXTLOAD %0 :: (load 1, addrspace 3)
+    $vgpr0_vgpr1 = COPY %1
+...
+---
+name: test_sextload_local_i64_i16
+body: |
+  bb.0:
+    liveins: $vgpr0
+
+    ; CHECK-LABEL: name: test_sextload_local_i64_i16
+    ; CHECK: [[COPY:%[0-9]+]]:_(p3) = COPY $vgpr0
+    ; CHECK: [[SEXTLOAD:%[0-9]+]]:_(s32) = G_SEXTLOAD [[COPY]](p3) :: (load 2, addrspace 3)
+    ; CHECK: [[SEXT:%[0-9]+]]:_(s64) = G_SEXT [[SEXTLOAD]](s32)
+    ; CHECK: $vgpr0_vgpr1 = COPY [[SEXT]](s64)
+    %0:_(p3) = COPY $vgpr0
+    %1:_(s64) = G_SEXTLOAD %0 :: (load 2, addrspace 3)
+    $vgpr0_vgpr1 = COPY %1
+...
+---
+name: test_sextload_local_i64_i32
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; CHECK-LABEL: name: test_sextload_local_i64_i32
+    ; CHECK: [[COPY:%[0-9]+]]:_(p3) = COPY $vgpr0
+    ; CHECK: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p3) :: (load 4, addrspace 3)
+    ; CHECK: [[SEXT:%[0-9]+]]:_(s64) = G_SEXT [[LOAD]](s32)
+    ; CHECK: $vgpr0_vgpr1 = COPY [[SEXT]](s64)
+    %0:_(p3) = COPY $vgpr0
+    %1:_(s64) = G_SEXTLOAD %0 :: (load 4, addrspace 3)
+    $vgpr0_vgpr1 = COPY %1
+...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/legalize-sextload-private.mir b/test/CodeGen/AMDGPU/GlobalISel/legalize-sextload-private.mir
new file mode 100644
index 0000000000000000000000000000000000000000..53d4d0d48da18a8f2015c12f2ee607ef7804fff4
--- /dev/null
+++ b/test/CodeGen/AMDGPU/GlobalISel/legalize-sextload-private.mir
@@ -0,0 +1,94 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -run-pass=legalizer -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=tahiti -run-pass=legalizer -o - %s | FileCheck %s
+
+---
+name: test_sextload_private_i32_i8
+body: |
+  bb.0:
+    liveins: $vgpr0
+
+    ; CHECK-LABEL: name: test_sextload_private_i32_i8
+    ; CHECK: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
+    ; CHECK: [[SEXTLOAD:%[0-9]+]]:_(s32) = G_SEXTLOAD [[COPY]](p5) :: (load 1, addrspace 5)
+    ; CHECK: $vgpr0 = COPY [[SEXTLOAD]](s32)
+    %0:_(p5) = COPY $vgpr0
+    %1:_(s32) = G_SEXTLOAD %0 :: (load 1, addrspace 5)
+
+    $vgpr0 = COPY %1
+...
+---
+name: test_sextload_private_i32_i16
+body: |
+  bb.0:
+    liveins: $vgpr0
+
+    ; CHECK-LABEL: name: test_sextload_private_i32_i16
+    ; CHECK: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
+    ; CHECK: [[SEXTLOAD:%[0-9]+]]:_(s32) = G_SEXTLOAD [[COPY]](p5) :: (load 2, addrspace 5)
+    ; CHECK: $vgpr0 = COPY [[SEXTLOAD]](s32)
+    %0:_(p5) = COPY $vgpr0
+    %1:_(s32) = G_SEXTLOAD %0 :: (load 2, addrspace 5)
+    $vgpr0 = COPY %1
+...
+---
+name: test_sextload_private_i31_i8
+body: |
+  bb.0:
+    liveins: $vgpr0
+
+    ; CHECK-LABEL: name: test_sextload_private_i31_i8
+    ; CHECK: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
+    ; CHECK: [[SEXTLOAD:%[0-9]+]]:_(s32) = G_SEXTLOAD [[COPY]](p5) :: (load 1, addrspace 5)
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY [[SEXTLOAD]](s32)
+    ; CHECK: $vgpr0 = COPY [[COPY1]](s32)
+    %0:_(p5) = COPY $vgpr0
+    %1:_(s31) = G_SEXTLOAD %0 :: (load 1, addrspace 5)
+    %2:_(s32) = G_ANYEXT %1
+    $vgpr0 = COPY %2
+...
+---
+name: test_sextload_private_i64_i8
+body: |
+  bb.0:
+    liveins: $vgpr0
+
+    ; CHECK-LABEL: name: test_sextload_private_i64_i8
+    ; CHECK: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
+    ; CHECK: [[SEXTLOAD:%[0-9]+]]:_(s32) = G_SEXTLOAD [[COPY]](p5) :: (load 1, addrspace 5)
+    ; CHECK: [[SEXT:%[0-9]+]]:_(s64) = G_SEXT [[SEXTLOAD]](s32)
+    ; CHECK: $vgpr0_vgpr1 = COPY [[SEXT]](s64)
+    %0:_(p5) = COPY $vgpr0
+    %1:_(s64) = G_SEXTLOAD %0 :: (load 1, addrspace 5)
+    $vgpr0_vgpr1 = COPY %1
+...
+---
+name: test_sextload_private_i64_i16
+body: |
+  bb.0:
+    liveins: $vgpr0
+
+    ; CHECK-LABEL: name: test_sextload_private_i64_i16
+    ; CHECK: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
+    ; CHECK: [[SEXTLOAD:%[0-9]+]]:_(s32) = G_SEXTLOAD [[COPY]](p5) :: (load 2, addrspace 5)
+    ; CHECK: [[SEXT:%[0-9]+]]:_(s64) = G_SEXT [[SEXTLOAD]](s32)
+    ; CHECK: $vgpr0_vgpr1 = COPY [[SEXT]](s64)
+    %0:_(p5) = COPY $vgpr0
+    %1:_(s64) = G_SEXTLOAD %0 :: (load 2, addrspace 5)
+    $vgpr0_vgpr1 = COPY %1
+...
+---
+name: test_sextload_private_i64_i32
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; CHECK-LABEL: name: test_sextload_private_i64_i32
+    ; CHECK: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
+    ; CHECK: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load 4, addrspace 5)
+    ; CHECK: [[SEXT:%[0-9]+]]:_(s64) = G_SEXT [[LOAD]](s32)
+    ; CHECK: $vgpr0_vgpr1 = COPY [[SEXT]](s64)
+    %0:_(p5) = COPY $vgpr0
+    %1:_(s64) = G_SEXTLOAD %0 :: (load 4, addrspace 5)
+    $vgpr0_vgpr1 = COPY %1
+...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/legalize-shl.mir b/test/CodeGen/AMDGPU/GlobalISel/legalize-shl.mir
index cd24d48a0dbe4f51b668a3d81f51ecd3020fd8f6..159fa8e3bd8ac5158566e698a0bd8179967f2106 100644
--- a/test/CodeGen/AMDGPU/GlobalISel/legalize-shl.mir
+++ b/test/CodeGen/AMDGPU/GlobalISel/legalize-shl.mir
@@ -1,18 +1,1595 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -O0 -run-pass=legalizer %s -o - | FileCheck %s
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=tahiti -O0 -run-pass=legalizer -global-isel-abort=0 %s -o - | FileCheck -check-prefix=SI %s
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -O0 -run-pass=legalizer  -global-isel-abort=0 %s -o - | FileCheck -check-prefix=VI %s
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx900 -O0 -run-pass=legalizer  -global-isel-abort=0 %s -o - | FileCheck -check-prefix=GFX9  %s
 
 ---
-name:            test_shl
+name: test_shl_s32_s32
 body: |
-  bb.0.entry:
+  bb.0:
     liveins: $vgpr0, $vgpr1
 
-    ; CHECK-LABEL: name: test_shl
-    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
-    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
-    ; CHECK: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY]], [[COPY1]]
+    ; SI-LABEL: name: test_shl_s32_s32
+    ; SI: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; SI: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; SI: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY]], [[COPY1]](s32)
+    ; SI: $vgpr0 = COPY [[SHL]](s32)
+    ; VI-LABEL: name: test_shl_s32_s32
+    ; VI: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; VI: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; VI: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY]], [[COPY1]](s32)
+    ; VI: $vgpr0 = COPY [[SHL]](s32)
+    ; GFX9-LABEL: name: test_shl_s32_s32
+    ; GFX9: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; GFX9: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; GFX9: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY]], [[COPY1]](s32)
+    ; GFX9: $vgpr0 = COPY [[SHL]](s32)
     %0:_(s32) = COPY $vgpr0
     %1:_(s32) = COPY $vgpr1
     %2:_(s32) = G_SHL %0, %1
     $vgpr0 = COPY %2
 ...
+---
+name: test_shl_s64_s64
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
+
+    ; SI-LABEL: name: test_shl_s64_s64
+    ; SI: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
+    ; SI: [[COPY1:%[0-9]+]]:_(s64) = COPY $vgpr2_vgpr3
+    ; SI: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY1]](s64)
+    ; SI: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[COPY]], [[TRUNC]](s32)
+    ; SI: $vgpr0_vgpr1 = COPY [[SHL]](s64)
+    ; VI-LABEL: name: test_shl_s64_s64
+    ; VI: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
+    ; VI: [[COPY1:%[0-9]+]]:_(s64) = COPY $vgpr2_vgpr3
+    ; VI: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY1]](s64)
+    ; VI: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[COPY]], [[TRUNC]](s32)
+    ; VI: $vgpr0_vgpr1 = COPY [[SHL]](s64)
+    ; GFX9-LABEL: name: test_shl_s64_s64
+    ; GFX9: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
+    ; GFX9: [[COPY1:%[0-9]+]]:_(s64) = COPY $vgpr2_vgpr3
+    ; GFX9: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY1]](s64)
+    ; GFX9: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[COPY]], [[TRUNC]](s32)
+    ; GFX9: $vgpr0_vgpr1 = COPY [[SHL]](s64)
+    %0:_(s64) = COPY $vgpr0_vgpr1
+    %1:_(s64) = COPY $vgpr2_vgpr3
+    %2:_(s64) = G_SHL %0, %1
+    $vgpr0_vgpr1 = COPY %2
+...
+---
+name: test_shl_s64_s32
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2
+
+    ; SI-LABEL: name: test_shl_s64_s32
+    ; SI: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
+    ; SI: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr2
+    ; SI: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[COPY]], [[COPY1]](s32)
+    ; SI: $vgpr0_vgpr1 = COPY [[SHL]](s64)
+    ; VI-LABEL: name: test_shl_s64_s32
+    ; VI: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
+    ; VI: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr2
+    ; VI: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[COPY]], [[COPY1]](s32)
+    ; VI: $vgpr0_vgpr1 = COPY [[SHL]](s64)
+    ; GFX9-LABEL: name: test_shl_s64_s32
+    ; GFX9: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
+    ; GFX9: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr2
+    ; GFX9: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[COPY]], [[COPY1]](s32)
+    ; GFX9: $vgpr0_vgpr1 = COPY [[SHL]](s64)
+    %0:_(s64) = COPY $vgpr0_vgpr1
+    %1:_(s32) = COPY $vgpr2
+    %2:_(s64) = G_SHL %0, %1
+    $vgpr0_vgpr1 = COPY %2
+...
+---
+name: test_shl_s64_s16
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2
+
+    ; SI-LABEL: name: test_shl_s64_s16
+    ; SI: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
+    ; SI: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr2
+    ; SI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; SI: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+    ; SI: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C]]
+    ; SI: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[COPY]], [[AND]](s32)
+    ; SI: $vgpr0_vgpr1 = COPY [[SHL]](s64)
+    ; VI-LABEL: name: test_shl_s64_s16
+    ; VI: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
+    ; VI: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr2
+    ; VI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32)
+    ; VI: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[COPY]], [[TRUNC]](s16)
+    ; VI: $vgpr0_vgpr1 = COPY [[SHL]](s64)
+    ; GFX9-LABEL: name: test_shl_s64_s16
+    ; GFX9: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
+    ; GFX9: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr2
+    ; GFX9: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32)
+    ; GFX9: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[COPY]], [[TRUNC]](s16)
+    ; GFX9: $vgpr0_vgpr1 = COPY [[SHL]](s64)
+    %0:_(s64) = COPY $vgpr0_vgpr1
+    %1:_(s32) = COPY $vgpr2
+    %2:_(s16) = G_TRUNC %1
+    %3:_(s64) = G_SHL %0, %2
+    $vgpr0_vgpr1 = COPY %3
+...
+
+---
+name: test_shl_s16_s32
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+
+    ; SI-LABEL: name: test_shl_s16_s32
+    ; SI: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; SI: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; SI: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; SI: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY2]], [[COPY1]](s32)
+    ; SI: [[COPY3:%[0-9]+]]:_(s32) = COPY [[SHL]](s32)
+    ; SI: $vgpr0 = COPY [[COPY3]](s32)
+    ; VI-LABEL: name: test_shl_s16_s32
+    ; VI: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; VI: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; VI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+    ; VI: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[TRUNC]], [[COPY1]](s32)
+    ; VI: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[SHL]](s16)
+    ; VI: $vgpr0 = COPY [[ANYEXT]](s32)
+    ; GFX9-LABEL: name: test_shl_s16_s32
+    ; GFX9: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; GFX9: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; GFX9: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+    ; GFX9: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[TRUNC]], [[COPY1]](s32)
+    ; GFX9: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[SHL]](s16)
+    ; GFX9: $vgpr0 = COPY [[ANYEXT]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s16) = G_TRUNC %0
+    %3:_(s16) = G_SHL %2, %1
+    %4:_(s32) = G_ANYEXT %3
+    $vgpr0 = COPY %4
+...
+
+---
+name: test_shl_s16_s16
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+
+    ; SI-LABEL: name: test_shl_s16_s16
+    ; SI: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; SI: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; SI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; SI: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+    ; SI: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C]]
+    ; SI: [[COPY3:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; SI: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY3]], [[AND]](s32)
+    ; SI: [[COPY4:%[0-9]+]]:_(s32) = COPY [[SHL]](s32)
+    ; SI: $vgpr0 = COPY [[COPY4]](s32)
+    ; VI-LABEL: name: test_shl_s16_s16
+    ; VI: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; VI: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; VI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+    ; VI: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32)
+    ; VI: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[TRUNC]], [[TRUNC1]](s16)
+    ; VI: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[SHL]](s16)
+    ; VI: $vgpr0 = COPY [[ANYEXT]](s32)
+    ; GFX9-LABEL: name: test_shl_s16_s16
+    ; GFX9: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; GFX9: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; GFX9: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+    ; GFX9: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY1]](s32)
+    ; GFX9: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[TRUNC]], [[TRUNC1]](s16)
+    ; GFX9: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[SHL]](s16)
+    ; GFX9: $vgpr0 = COPY [[ANYEXT]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s16) = G_TRUNC %0
+    %3:_(s16) = G_TRUNC %1
+    %4:_(s16) = G_SHL %2, %3
+    %5:_(s32) = G_ANYEXT %4
+    $vgpr0 = COPY %5
+...
+
+---
+name: test_shl_s16_i8
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+
+    ; SI-LABEL: name: test_shl_s16_i8
+    ; SI: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; SI: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; SI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; SI: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+    ; SI: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C]]
+    ; SI: [[COPY3:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; SI: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY3]], [[AND]](s32)
+    ; SI: [[COPY4:%[0-9]+]]:_(s32) = COPY [[SHL]](s32)
+    ; SI: $vgpr0 = COPY [[COPY4]](s32)
+    ; VI-LABEL: name: test_shl_s16_i8
+    ; VI: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; VI: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; VI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+    ; VI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; VI: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+    ; VI: [[COPY3:%[0-9]+]]:_(s32) = COPY [[C]](s32)
+    ; VI: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[COPY3]]
+    ; VI: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[AND]](s32)
+    ; VI: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[TRUNC]], [[TRUNC1]](s16)
+    ; VI: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[SHL]](s16)
+    ; VI: $vgpr0 = COPY [[ANYEXT]](s32)
+    ; GFX9-LABEL: name: test_shl_s16_i8
+    ; GFX9: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; GFX9: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; GFX9: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; GFX9: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+    ; GFX9: [[COPY3:%[0-9]+]]:_(s32) = COPY [[C]](s32)
+    ; GFX9: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[COPY3]]
+    ; GFX9: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[AND]](s32)
+    ; GFX9: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[TRUNC]], [[TRUNC1]](s16)
+    ; GFX9: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[SHL]](s16)
+    ; GFX9: $vgpr0 = COPY [[ANYEXT]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s16) = G_TRUNC %0
+    %3:_(s8) = G_TRUNC %1
+    %4:_(s16) = G_SHL %2, %3
+    %5:_(s32) = G_ANYEXT %4
+    $vgpr0 = COPY %5
+...
+
+---
+name: test_shl_i8_i8
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+
+    ; SI-LABEL: name: test_shl_i8_i8
+    ; SI: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; SI: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; SI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; SI: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+    ; SI: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C]]
+    ; SI: [[COPY3:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; SI: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY3]], [[AND]](s32)
+    ; SI: [[COPY4:%[0-9]+]]:_(s32) = COPY [[SHL]](s32)
+    ; SI: $vgpr0 = COPY [[COPY4]](s32)
+    ; VI-LABEL: name: test_shl_i8_i8
+    ; VI: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; VI: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; VI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; VI: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+    ; VI: [[COPY3:%[0-9]+]]:_(s32) = COPY [[C]](s32)
+    ; VI: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[COPY3]]
+    ; VI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[AND]](s32)
+    ; VI: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+    ; VI: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[TRUNC1]], [[TRUNC]](s16)
+    ; VI: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[SHL]](s16)
+    ; VI: $vgpr0 = COPY [[ANYEXT]](s32)
+    ; GFX9-LABEL: name: test_shl_i8_i8
+    ; GFX9: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; GFX9: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; GFX9: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+    ; GFX9: [[COPY3:%[0-9]+]]:_(s32) = COPY [[C]](s32)
+    ; GFX9: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[COPY3]]
+    ; GFX9: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[AND]](s32)
+    ; GFX9: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+    ; GFX9: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[TRUNC1]], [[TRUNC]](s16)
+    ; GFX9: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[SHL]](s16)
+    ; GFX9: $vgpr0 = COPY [[ANYEXT]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s8) = G_TRUNC %0
+    %3:_(s8) = G_TRUNC %1
+    %4:_(s8) = G_SHL %2, %3
+    %5:_(s32) = G_ANYEXT %4
+    $vgpr0 = COPY %5
+...
+
+---
+
+name: test_shl_v2s32_v2s32
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
+
+    ; SI-LABEL: name: test_shl_v2s32_v2s32
+    ; SI: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    ; SI: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr2_vgpr3
+    ; SI: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
+    ; SI: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<2 x s32>)
+    ; SI: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[UV]], [[UV2]](s32)
+    ; SI: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[UV1]], [[UV3]](s32)
+    ; SI: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[SHL]](s32), [[SHL1]](s32)
+    ; SI: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
+    ; VI-LABEL: name: test_shl_v2s32_v2s32
+    ; VI: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    ; VI: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr2_vgpr3
+    ; VI: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
+    ; VI: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<2 x s32>)
+    ; VI: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[UV]], [[UV2]](s32)
+    ; VI: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[UV1]], [[UV3]](s32)
+    ; VI: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[SHL]](s32), [[SHL1]](s32)
+    ; VI: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
+    ; GFX9-LABEL: name: test_shl_v2s32_v2s32
+    ; GFX9: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    ; GFX9: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr2_vgpr3
+    ; GFX9: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
+    ; GFX9: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<2 x s32>)
+    ; GFX9: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[UV]], [[UV2]](s32)
+    ; GFX9: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[UV1]], [[UV3]](s32)
+    ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[SHL]](s32), [[SHL1]](s32)
+    ; GFX9: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
+    %0:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    %1:_(<2 x s32>) = COPY $vgpr2_vgpr3
+    %2:_(<2 x s32>) = G_SHL %0, %1
+    $vgpr0_vgpr1 = COPY %2
+...
+
+---
+name: test_shl_v3s32_v3s32
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2, $vgpr3_vgpr4_vgpr5
+
+    ; SI-LABEL: name: test_shl_v3s32_v3s32
+    ; SI: [[COPY:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr0_vgpr1_vgpr2
+    ; SI: [[COPY1:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr3_vgpr4_vgpr5
+    ; SI: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<3 x s32>)
+    ; SI: [[UV3:%[0-9]+]]:_(s32), [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<3 x s32>)
+    ; SI: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[UV]], [[UV3]](s32)
+    ; SI: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[UV1]], [[UV4]](s32)
+    ; SI: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[UV2]], [[UV5]](s32)
+    ; SI: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[SHL]](s32), [[SHL1]](s32), [[SHL2]](s32)
+    ; SI: $vgpr0_vgpr1_vgpr2 = COPY [[BUILD_VECTOR]](<3 x s32>)
+    ; VI-LABEL: name: test_shl_v3s32_v3s32
+    ; VI: [[COPY:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr0_vgpr1_vgpr2
+    ; VI: [[COPY1:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr3_vgpr4_vgpr5
+    ; VI: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<3 x s32>)
+    ; VI: [[UV3:%[0-9]+]]:_(s32), [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<3 x s32>)
+    ; VI: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[UV]], [[UV3]](s32)
+    ; VI: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[UV1]], [[UV4]](s32)
+    ; VI: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[UV2]], [[UV5]](s32)
+    ; VI: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[SHL]](s32), [[SHL1]](s32), [[SHL2]](s32)
+    ; VI: $vgpr0_vgpr1_vgpr2 = COPY [[BUILD_VECTOR]](<3 x s32>)
+    ; GFX9-LABEL: name: test_shl_v3s32_v3s32
+    ; GFX9: [[COPY:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr0_vgpr1_vgpr2
+    ; GFX9: [[COPY1:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr3_vgpr4_vgpr5
+    ; GFX9: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<3 x s32>)
+    ; GFX9: [[UV3:%[0-9]+]]:_(s32), [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<3 x s32>)
+    ; GFX9: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[UV]], [[UV3]](s32)
+    ; GFX9: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[UV1]], [[UV4]](s32)
+    ; GFX9: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[UV2]], [[UV5]](s32)
+    ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[SHL]](s32), [[SHL1]](s32), [[SHL2]](s32)
+    ; GFX9: $vgpr0_vgpr1_vgpr2 = COPY [[BUILD_VECTOR]](<3 x s32>)
+    %0:_(<3 x s32>) = COPY $vgpr0_vgpr1_vgpr2
+    %1:_(<3 x s32>) = COPY $vgpr3_vgpr4_vgpr5
+    %2:_(<3 x s32>) = G_SHL %0, %1
+    $vgpr0_vgpr1_vgpr2 = COPY %2
+...
+
+---
+name: test_shl_v2s64_v2s32
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5
+
+    ; SI-LABEL: name: test_shl_v2s64_v2s32
+    ; SI: [[COPY:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; SI: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr4_vgpr5
+    ; SI: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](<2 x s64>)
+    ; SI: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<2 x s32>)
+    ; SI: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[UV]], [[UV2]](s32)
+    ; SI: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[UV1]], [[UV3]](s32)
+    ; SI: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[SHL]](s64), [[SHL1]](s64)
+    ; SI: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>)
+    ; VI-LABEL: name: test_shl_v2s64_v2s32
+    ; VI: [[COPY:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; VI: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr4_vgpr5
+    ; VI: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](<2 x s64>)
+    ; VI: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<2 x s32>)
+    ; VI: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[UV]], [[UV2]](s32)
+    ; VI: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[UV1]], [[UV3]](s32)
+    ; VI: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[SHL]](s64), [[SHL1]](s64)
+    ; VI: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>)
+    ; GFX9-LABEL: name: test_shl_v2s64_v2s32
+    ; GFX9: [[COPY:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; GFX9: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr4_vgpr5
+    ; GFX9: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](<2 x s64>)
+    ; GFX9: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<2 x s32>)
+    ; GFX9: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[UV]], [[UV2]](s32)
+    ; GFX9: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[UV1]], [[UV3]](s32)
+    ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[SHL]](s64), [[SHL1]](s64)
+    ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>)
+    %0:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    %1:_(<2 x s32>) = COPY $vgpr4_vgpr5
+    %2:_(<2 x s64>) = G_SHL %0, %1
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %2
+...
+
+---
+name: test_shl_v3s64_v3s32
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8_vgpr9_vgpr10
+
+    ; SI-LABEL: name: test_shl_v3s64_v3s32
+    ; SI: [[COPY:%[0-9]+]]:_(<4 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+    ; SI: [[EXTRACT:%[0-9]+]]:_(<3 x s64>) = G_EXTRACT [[COPY]](<4 x s64>), 0
+    ; SI: [[COPY1:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr8_vgpr9_vgpr10
+    ; SI: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64), [[UV2:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[EXTRACT]](<3 x s64>)
+    ; SI: [[UV3:%[0-9]+]]:_(s32), [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<3 x s32>)
+    ; SI: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[UV]], [[UV3]](s32)
+    ; SI: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[UV1]], [[UV4]](s32)
+    ; SI: [[SHL2:%[0-9]+]]:_(s64) = G_SHL [[UV2]], [[UV5]](s32)
+    ; SI: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s64>) = G_BUILD_VECTOR [[SHL]](s64), [[SHL1]](s64), [[SHL2]](s64)
+    ; SI: [[DEF:%[0-9]+]]:_(<4 x s64>) = G_IMPLICIT_DEF
+    ; SI: [[INSERT:%[0-9]+]]:_(<4 x s64>) = G_INSERT [[DEF]], [[BUILD_VECTOR]](<3 x s64>), 0
+    ; SI: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY [[INSERT]](<4 x s64>)
+    ; VI-LABEL: name: test_shl_v3s64_v3s32
+    ; VI: [[COPY:%[0-9]+]]:_(<4 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+    ; VI: [[EXTRACT:%[0-9]+]]:_(<3 x s64>) = G_EXTRACT [[COPY]](<4 x s64>), 0
+    ; VI: [[COPY1:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr8_vgpr9_vgpr10
+    ; VI: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64), [[UV2:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[EXTRACT]](<3 x s64>)
+    ; VI: [[UV3:%[0-9]+]]:_(s32), [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<3 x s32>)
+    ; VI: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[UV]], [[UV3]](s32)
+    ; VI: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[UV1]], [[UV4]](s32)
+    ; VI: [[SHL2:%[0-9]+]]:_(s64) = G_SHL [[UV2]], [[UV5]](s32)
+    ; VI: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s64>) = G_BUILD_VECTOR [[SHL]](s64), [[SHL1]](s64), [[SHL2]](s64)
+    ; VI: [[DEF:%[0-9]+]]:_(<4 x s64>) = G_IMPLICIT_DEF
+    ; VI: [[INSERT:%[0-9]+]]:_(<4 x s64>) = G_INSERT [[DEF]], [[BUILD_VECTOR]](<3 x s64>), 0
+    ; VI: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY [[INSERT]](<4 x s64>)
+    ; GFX9-LABEL: name: test_shl_v3s64_v3s32
+    ; GFX9: [[COPY:%[0-9]+]]:_(<4 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+    ; GFX9: [[EXTRACT:%[0-9]+]]:_(<3 x s64>) = G_EXTRACT [[COPY]](<4 x s64>), 0
+    ; GFX9: [[COPY1:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr8_vgpr9_vgpr10
+    ; GFX9: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64), [[UV2:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[EXTRACT]](<3 x s64>)
+    ; GFX9: [[UV3:%[0-9]+]]:_(s32), [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<3 x s32>)
+    ; GFX9: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[UV]], [[UV3]](s32)
+    ; GFX9: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[UV1]], [[UV4]](s32)
+    ; GFX9: [[SHL2:%[0-9]+]]:_(s64) = G_SHL [[UV2]], [[UV5]](s32)
+    ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s64>) = G_BUILD_VECTOR [[SHL]](s64), [[SHL1]](s64), [[SHL2]](s64)
+    ; GFX9: [[DEF:%[0-9]+]]:_(<4 x s64>) = G_IMPLICIT_DEF
+    ; GFX9: [[INSERT:%[0-9]+]]:_(<4 x s64>) = G_INSERT [[DEF]], [[BUILD_VECTOR]](<3 x s64>), 0
+    ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY [[INSERT]](<4 x s64>)
+    %0:_(<4 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+    %1:_(<3 x s64>) = G_EXTRACT %0, 0
+    %2:_(<3 x s32>) = COPY $vgpr8_vgpr9_vgpr10
+    %3:_(<3 x s64>) = G_SHL %1, %2
+    %4:_(<4 x s64>) = G_IMPLICIT_DEF
+    %5:_(<4 x s64>) = G_INSERT %4, %3, 0
+    $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY %5
+...
+
+---
+name: test_shl_v2s16_v2s16
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+
+    ; SI-LABEL: name: test_shl_v2s16_v2s16
+    ; SI: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
+    ; SI: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
+    ; SI: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY]](<2 x s16>)
+    ; SI: [[UV2:%[0-9]+]]:_(s16), [[UV3:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY1]](<2 x s16>)
+    ; SI: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[UV2]](s16)
+    ; SI: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[UV]](s16)
+    ; SI: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ANYEXT]], [[ZEXT]](s32)
+    ; SI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[SHL]](s32)
+    ; SI: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[UV3]](s16)
+    ; SI: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[UV1]](s16)
+    ; SI: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ANYEXT1]], [[ZEXT1]](s32)
+    ; SI: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[SHL1]](s32)
+    ; SI: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
+    ; SI: $vgpr0 = COPY [[BUILD_VECTOR]](<2 x s16>)
+    ; VI-LABEL: name: test_shl_v2s16_v2s16
+    ; VI: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
+    ; VI: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
+    ; VI: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY]](<2 x s16>)
+    ; VI: [[UV2:%[0-9]+]]:_(s16), [[UV3:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY1]](<2 x s16>)
+    ; VI: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[UV]], [[UV2]](s16)
+    ; VI: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[UV1]], [[UV3]](s16)
+    ; VI: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[SHL]](s16), [[SHL1]](s16)
+    ; VI: $vgpr0 = COPY [[BUILD_VECTOR]](<2 x s16>)
+    ; GFX9-LABEL: name: test_shl_v2s16_v2s16
+    ; GFX9: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
+    ; GFX9: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
+    ; GFX9: [[SHL:%[0-9]+]]:_(<2 x s16>) = G_SHL [[COPY]], [[COPY1]](<2 x s16>)
+    ; GFX9: $vgpr0 = COPY [[SHL]](<2 x s16>)
+    %0:_(<2 x s16>) = COPY $vgpr0
+    %1:_(<2 x s16>) = COPY $vgpr1
+    %2:_(<2 x s16>) = G_SHL %0, %1
+    $vgpr0 = COPY %2
+...
+
+---
+name: test_shl_v2s16_v2s32
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr0_vgpr1
+
+    ; SI-LABEL: name: test_shl_v2s16_v2s32
+    ; SI: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
+    ; SI: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    ; SI: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY]](<2 x s16>)
+    ; SI: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<2 x s32>)
+    ; SI: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[UV]](s16)
+    ; SI: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ANYEXT]], [[UV2]](s32)
+    ; SI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[SHL]](s32)
+    ; SI: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[UV1]](s16)
+    ; SI: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ANYEXT1]], [[UV3]](s32)
+    ; SI: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[SHL1]](s32)
+    ; SI: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
+    ; SI: $vgpr0 = COPY [[BUILD_VECTOR]](<2 x s16>)
+    ; VI-LABEL: name: test_shl_v2s16_v2s32
+    ; VI: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
+    ; VI: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    ; VI: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY]](<2 x s16>)
+    ; VI: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<2 x s32>)
+    ; VI: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[UV]], [[UV2]](s32)
+    ; VI: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[UV1]], [[UV3]](s32)
+    ; VI: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[SHL]](s16), [[SHL1]](s16)
+    ; VI: $vgpr0 = COPY [[BUILD_VECTOR]](<2 x s16>)
+    ; GFX9-LABEL: name: test_shl_v2s16_v2s32
+    ; GFX9: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
+    ; GFX9: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    ; GFX9: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY]](<2 x s16>)
+    ; GFX9: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<2 x s32>)
+    ; GFX9: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[UV]], [[UV2]](s32)
+    ; GFX9: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[UV1]], [[UV3]](s32)
+    ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[SHL]](s16), [[SHL1]](s16)
+    ; GFX9: $vgpr0 = COPY [[BUILD_VECTOR]](<2 x s16>)
+    %0:_(<2 x s16>) = COPY $vgpr0
+    %1:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    %2:_(<2 x s16>) = G_SHL %0, %1
+    $vgpr0 = COPY %2
+...
+
+---
+name: test_shl_v3s16_v3s16
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
+
+    ; SI-LABEL: name: test_shl_v3s16_v3s16
+    ; SI: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
+    ; SI: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
+    ; SI: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[COPY]](<4 x s16>), 0
+    ; SI: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[COPY1]](<4 x s16>), 0
+    ; SI: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16), [[UV2:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[EXTRACT]](<3 x s16>)
+    ; SI: [[UV3:%[0-9]+]]:_(s16), [[UV4:%[0-9]+]]:_(s16), [[UV5:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[EXTRACT1]](<3 x s16>)
+    ; SI: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[UV3]](s16)
+    ; SI: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[UV]](s16)
+    ; SI: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ANYEXT]], [[ZEXT]](s32)
+    ; SI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[SHL]](s32)
+    ; SI: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[UV4]](s16)
+    ; SI: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[UV1]](s16)
+    ; SI: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ANYEXT1]], [[ZEXT1]](s32)
+    ; SI: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[SHL1]](s32)
+    ; SI: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[UV5]](s16)
+    ; SI: [[ANYEXT2:%[0-9]+]]:_(s32) = G_ANYEXT [[UV2]](s16)
+    ; SI: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[ANYEXT2]], [[ZEXT2]](s32)
+    ; SI: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[SHL2]](s32)
+    ; SI: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16), [[TRUNC2]](s16)
+    ; SI: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; SI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[BUILD_VECTOR]](<3 x s16>), 0
+    ; SI: $vgpr0_vgpr1 = COPY [[INSERT]](<4 x s16>)
+    ; VI-LABEL: name: test_shl_v3s16_v3s16
+    ; VI: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
+    ; VI: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
+    ; VI: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[COPY]](<4 x s16>), 0
+    ; VI: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[COPY1]](<4 x s16>), 0
+    ; VI: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16), [[UV2:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[EXTRACT]](<3 x s16>)
+    ; VI: [[UV3:%[0-9]+]]:_(s16), [[UV4:%[0-9]+]]:_(s16), [[UV5:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[EXTRACT1]](<3 x s16>)
+    ; VI: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[UV]], [[UV3]](s16)
+    ; VI: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[UV1]], [[UV4]](s16)
+    ; VI: [[SHL2:%[0-9]+]]:_(s16) = G_SHL [[UV2]], [[UV5]](s16)
+    ; VI: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s16>) = G_BUILD_VECTOR [[SHL]](s16), [[SHL1]](s16), [[SHL2]](s16)
+    ; VI: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; VI: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF]], [[BUILD_VECTOR]](<3 x s16>), 0
+    ; VI: $vgpr0_vgpr1 = COPY [[INSERT]](<4 x s16>)
+    ; GFX9-LABEL: name: test_shl_v3s16_v3s16
+    ; GFX9: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
+    ; GFX9: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
+    ; GFX9: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[COPY]](<4 x s16>), 0
+    ; GFX9: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[COPY1]](<4 x s16>), 0
+    ; GFX9: [[EXTRACT2:%[0-9]+]]:_(<2 x s16>) = G_EXTRACT [[EXTRACT]](<3 x s16>), 0
+    ; GFX9: [[EXTRACT3:%[0-9]+]]:_(s16) = G_EXTRACT [[EXTRACT]](<3 x s16>), 32
+    ; GFX9: [[EXTRACT4:%[0-9]+]]:_(<2 x s16>) = G_EXTRACT [[EXTRACT1]](<3 x s16>), 0
+    ; GFX9: [[EXTRACT5:%[0-9]+]]:_(s16) = G_EXTRACT [[EXTRACT1]](<3 x s16>), 32
+    ; GFX9: [[SHL:%[0-9]+]]:_(<2 x s16>) = G_SHL [[EXTRACT2]], [[EXTRACT4]](<2 x s16>)
+    ; GFX9: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[EXTRACT3]], [[EXTRACT5]](s16)
+    ; GFX9: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; GFX9: [[EXTRACT6:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
+    ; GFX9: [[INSERT:%[0-9]+]]:_(<3 x s16>) = G_INSERT [[EXTRACT6]], [[SHL]](<2 x s16>), 0
+    ; GFX9: [[INSERT1:%[0-9]+]]:_(<3 x s16>) = G_INSERT [[INSERT]], [[SHL1]](s16), 32
+    ; GFX9: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; GFX9: [[INSERT2:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF1]], [[INSERT1]](<3 x s16>), 0
+    ; GFX9: $vgpr0_vgpr1 = COPY [[INSERT2]](<4 x s16>)
+    %0:_(<4 x s16>) = COPY $vgpr0_vgpr1
+    %1:_(<4 x s16>) = COPY $vgpr2_vgpr3
+    %2:_(<3 x s16>) = G_EXTRACT %0, 0
+    %3:_(<3 x s16>) = G_EXTRACT %1, 0
+    %4:_(<3 x s16>) = G_SHL %2, %3
+    %5:_(<4 x s16>) = G_IMPLICIT_DEF
+    %6:_(<4 x s16>) = G_INSERT %5, %4, 0
+    $vgpr0_vgpr1 = COPY %6
+...
+
+---
+name: test_shl_v4s16_v4s16
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
+
+    ; SI-LABEL: name: test_shl_v4s16_v4s16
+    ; SI: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
+    ; SI: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
+    ; SI: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16), [[UV2:%[0-9]+]]:_(s16), [[UV3:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
+    ; SI: [[UV4:%[0-9]+]]:_(s16), [[UV5:%[0-9]+]]:_(s16), [[UV6:%[0-9]+]]:_(s16), [[UV7:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
+    ; SI: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[UV4]](s16)
+    ; SI: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[UV]](s16)
+    ; SI: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ANYEXT]], [[ZEXT]](s32)
+    ; SI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[SHL]](s32)
+    ; SI: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[UV5]](s16)
+    ; SI: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[UV1]](s16)
+    ; SI: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ANYEXT1]], [[ZEXT1]](s32)
+    ; SI: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[SHL1]](s32)
+    ; SI: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[UV6]](s16)
+    ; SI: [[ANYEXT2:%[0-9]+]]:_(s32) = G_ANYEXT [[UV2]](s16)
+    ; SI: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[ANYEXT2]], [[ZEXT2]](s32)
+    ; SI: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[SHL2]](s32)
+    ; SI: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[UV7]](s16)
+    ; SI: [[ANYEXT3:%[0-9]+]]:_(s32) = G_ANYEXT [[UV3]](s16)
+    ; SI: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ANYEXT3]], [[ZEXT3]](s32)
+    ; SI: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[SHL3]](s32)
+    ; SI: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16), [[TRUNC2]](s16), [[TRUNC3]](s16)
+    ; SI: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<4 x s16>)
+    ; VI-LABEL: name: test_shl_v4s16_v4s16
+    ; VI: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
+    ; VI: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
+    ; VI: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16), [[UV2:%[0-9]+]]:_(s16), [[UV3:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
+    ; VI: [[UV4:%[0-9]+]]:_(s16), [[UV5:%[0-9]+]]:_(s16), [[UV6:%[0-9]+]]:_(s16), [[UV7:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
+    ; VI: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[UV]], [[UV4]](s16)
+    ; VI: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[UV1]], [[UV5]](s16)
+    ; VI: [[SHL2:%[0-9]+]]:_(s16) = G_SHL [[UV2]], [[UV6]](s16)
+    ; VI: [[SHL3:%[0-9]+]]:_(s16) = G_SHL [[UV3]], [[UV7]](s16)
+    ; VI: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s16>) = G_BUILD_VECTOR [[SHL]](s16), [[SHL1]](s16), [[SHL2]](s16), [[SHL3]](s16)
+    ; VI: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<4 x s16>)
+    ; GFX9-LABEL: name: test_shl_v4s16_v4s16
+    ; GFX9: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
+    ; GFX9: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
+    ; GFX9: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
+    ; GFX9: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
+    ; GFX9: [[SHL:%[0-9]+]]:_(<2 x s16>) = G_SHL [[UV]], [[UV2]](<2 x s16>)
+    ; GFX9: [[SHL1:%[0-9]+]]:_(<2 x s16>) = G_SHL [[UV1]], [[UV3]](<2 x s16>)
+    ; GFX9: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[SHL]](<2 x s16>), [[SHL1]](<2 x s16>)
+    ; GFX9: $vgpr0_vgpr1 = COPY [[CONCAT_VECTORS]](<4 x s16>)
+    %0:_(<4 x s16>) = COPY $vgpr0_vgpr1
+    %1:_(<4 x s16>) = COPY $vgpr2_vgpr3
+    %2:_(<4 x s16>) = G_SHL %0, %1
+    $vgpr0_vgpr1 = COPY %2
+...
+
+---
+name: test_shl_s7_s7
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+
+    ; SI-LABEL: name: test_shl_s7_s7
+    ; SI: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; SI: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; SI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 127
+    ; SI: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+    ; SI: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C]]
+    ; SI: [[COPY3:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; SI: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY3]], [[AND]](s32)
+    ; SI: [[COPY4:%[0-9]+]]:_(s32) = COPY [[SHL]](s32)
+    ; SI: $vgpr0 = COPY [[COPY4]](s32)
+    ; VI-LABEL: name: test_shl_s7_s7
+    ; VI: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; VI: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; VI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 127
+    ; VI: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+    ; VI: [[COPY3:%[0-9]+]]:_(s32) = COPY [[C]](s32)
+    ; VI: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[COPY3]]
+    ; VI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[AND]](s32)
+    ; VI: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+    ; VI: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[TRUNC1]], [[TRUNC]](s16)
+    ; VI: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[SHL]](s16)
+    ; VI: $vgpr0 = COPY [[ANYEXT]](s32)
+    ; GFX9-LABEL: name: test_shl_s7_s7
+    ; GFX9: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; GFX9: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 127
+    ; GFX9: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+    ; GFX9: [[COPY3:%[0-9]+]]:_(s32) = COPY [[C]](s32)
+    ; GFX9: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[COPY3]]
+    ; GFX9: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[AND]](s32)
+    ; GFX9: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+    ; GFX9: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[TRUNC1]], [[TRUNC]](s16)
+    ; GFX9: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[SHL]](s16)
+    ; GFX9: $vgpr0 = COPY [[ANYEXT]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s7) = G_TRUNC %0
+    %3:_(s7) = G_TRUNC %1
+    %4:_(s7) = G_SHL %2, %3
+    %5:_(s32) = G_ANYEXT %4
+    $vgpr0 = COPY %5
+...
+
+---
+name: test_shl_i24_i32
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+
+    ; SI-LABEL: name: test_shl_i24_i32
+    ; SI: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; SI: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; SI: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; SI: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY2]], [[COPY1]](s32)
+    ; SI: [[COPY3:%[0-9]+]]:_(s32) = COPY [[SHL]](s32)
+    ; SI: $vgpr0 = COPY [[COPY3]](s32)
+    ; VI-LABEL: name: test_shl_i24_i32
+    ; VI: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; VI: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; VI: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; VI: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY2]], [[COPY1]](s32)
+    ; VI: [[COPY3:%[0-9]+]]:_(s32) = COPY [[SHL]](s32)
+    ; VI: $vgpr0 = COPY [[COPY3]](s32)
+    ; GFX9-LABEL: name: test_shl_i24_i32
+    ; GFX9: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; GFX9: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; GFX9: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; GFX9: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY2]], [[COPY1]](s32)
+    ; GFX9: [[COPY3:%[0-9]+]]:_(s32) = COPY [[SHL]](s32)
+    ; GFX9: $vgpr0 = COPY [[COPY3]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s24) = G_TRUNC %0
+    %3:_(s24) = G_SHL %2, %1
+    %4:_(s32) = G_ANYEXT %3
+    $vgpr0 = COPY %4
+...
+
+---
+name: test_shl_s128_s128
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4
+
+    ; SI-LABEL: name: test_shl_s128_s128
+    ; SI: [[COPY:%[0-9]+]]:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; SI: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr4
+    ; SI: [[ZEXT:%[0-9]+]]:_(s128) = G_ZEXT [[COPY1]](s32)
+    ; SI: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[ZEXT]](s128)
+    ; SI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 64
+    ; SI: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](s128)
+    ; SI: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[TRUNC]], [[C]]
+    ; SI: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[C]], [[TRUNC]]
+    ; SI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; SI: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[TRUNC]](s32), [[C]]
+    ; SI: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[TRUNC]](s32), [[C1]]
+    ; SI: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[UV1]], [[TRUNC]](s32)
+    ; SI: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[UV1]], [[TRUNC]](s32)
+    ; SI: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[UV]], [[SUB1]](s32)
+    ; SI: [[OR:%[0-9]+]]:_(s64) = G_OR [[SHL1]], [[LSHR]]
+    ; SI: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; SI: [[SHL2:%[0-9]+]]:_(s64) = G_SHL [[UV]], [[SUB]](s32)
+    ; SI: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[SHL]], [[C2]]
+    ; SI: [[SELECT1:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[OR]], [[SHL2]]
+    ; SI: [[SELECT2:%[0-9]+]]:_(s64) = G_SELECT [[ICMP1]](s1), [[UV1]], [[SELECT1]]
+    ; SI: [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[SELECT]](s64), [[SELECT2]](s64)
+    ; SI: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[MV]](s128)
+    ; VI-LABEL: name: test_shl_s128_s128
+    ; VI: [[COPY:%[0-9]+]]:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; VI: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr4
+    ; VI: [[ZEXT:%[0-9]+]]:_(s128) = G_ZEXT [[COPY1]](s32)
+    ; VI: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[ZEXT]](s128)
+    ; VI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 64
+    ; VI: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](s128)
+    ; VI: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[TRUNC]], [[C]]
+    ; VI: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[C]], [[TRUNC]]
+    ; VI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; VI: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[TRUNC]](s32), [[C]]
+    ; VI: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[TRUNC]](s32), [[C1]]
+    ; VI: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[UV1]], [[TRUNC]](s32)
+    ; VI: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[UV1]], [[TRUNC]](s32)
+    ; VI: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[UV]], [[SUB1]](s32)
+    ; VI: [[OR:%[0-9]+]]:_(s64) = G_OR [[SHL1]], [[LSHR]]
+    ; VI: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; VI: [[SHL2:%[0-9]+]]:_(s64) = G_SHL [[UV]], [[SUB]](s32)
+    ; VI: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[SHL]], [[C2]]
+    ; VI: [[SELECT1:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[OR]], [[SHL2]]
+    ; VI: [[SELECT2:%[0-9]+]]:_(s64) = G_SELECT [[ICMP1]](s1), [[UV1]], [[SELECT1]]
+    ; VI: [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[SELECT]](s64), [[SELECT2]](s64)
+    ; VI: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[MV]](s128)
+    ; GFX9-LABEL: name: test_shl_s128_s128
+    ; GFX9: [[COPY:%[0-9]+]]:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; GFX9: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr4
+    ; GFX9: [[ZEXT:%[0-9]+]]:_(s128) = G_ZEXT [[COPY1]](s32)
+    ; GFX9: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[ZEXT]](s128)
+    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 64
+    ; GFX9: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](s128)
+    ; GFX9: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[TRUNC]], [[C]]
+    ; GFX9: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[C]], [[TRUNC]]
+    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; GFX9: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[TRUNC]](s32), [[C]]
+    ; GFX9: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[TRUNC]](s32), [[C1]]
+    ; GFX9: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[UV1]], [[TRUNC]](s32)
+    ; GFX9: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[UV1]], [[TRUNC]](s32)
+    ; GFX9: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[UV]], [[SUB1]](s32)
+    ; GFX9: [[OR:%[0-9]+]]:_(s64) = G_OR [[SHL1]], [[LSHR]]
+    ; GFX9: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; GFX9: [[SHL2:%[0-9]+]]:_(s64) = G_SHL [[UV]], [[SUB]](s32)
+    ; GFX9: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[SHL]], [[C2]]
+    ; GFX9: [[SELECT1:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[OR]], [[SHL2]]
+    ; GFX9: [[SELECT2:%[0-9]+]]:_(s64) = G_SELECT [[ICMP1]](s1), [[UV1]], [[SELECT1]]
+    ; GFX9: [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[SELECT]](s64), [[SELECT2]](s64)
+    ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[MV]](s128)
+    %0:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    %1:_(s32) = COPY $vgpr4
+    %2:_(s128) = G_ZEXT %1
+    %3:_(s128) = G_SHL %0, %2
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %3
+...
+
+---
+name: test_shl_s128_s132
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4
+
+    ; SI-LABEL: name: test_shl_s128_s132
+    ; SI: [[COPY:%[0-9]+]]:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; SI: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr4
+    ; SI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 64
+    ; SI: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](s128)
+    ; SI: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[COPY1]], [[C]]
+    ; SI: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[C]], [[COPY1]]
+    ; SI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; SI: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[COPY1]](s32), [[C]]
+    ; SI: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[COPY1]](s32), [[C1]]
+    ; SI: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[UV1]], [[COPY1]](s32)
+    ; SI: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[UV1]], [[COPY1]](s32)
+    ; SI: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[UV]], [[SUB1]](s32)
+    ; SI: [[OR:%[0-9]+]]:_(s64) = G_OR [[SHL1]], [[LSHR]]
+    ; SI: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; SI: [[SHL2:%[0-9]+]]:_(s64) = G_SHL [[UV]], [[SUB]](s32)
+    ; SI: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[SHL]], [[C2]]
+    ; SI: [[SELECT1:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[OR]], [[SHL2]]
+    ; SI: [[SELECT2:%[0-9]+]]:_(s64) = G_SELECT [[ICMP1]](s1), [[UV1]], [[SELECT1]]
+    ; SI: [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[SELECT]](s64), [[SELECT2]](s64)
+    ; SI: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[MV]](s128)
+    ; VI-LABEL: name: test_shl_s128_s132
+    ; VI: [[COPY:%[0-9]+]]:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; VI: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr4
+    ; VI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 64
+    ; VI: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](s128)
+    ; VI: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[COPY1]], [[C]]
+    ; VI: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[C]], [[COPY1]]
+    ; VI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; VI: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[COPY1]](s32), [[C]]
+    ; VI: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[COPY1]](s32), [[C1]]
+    ; VI: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[UV1]], [[COPY1]](s32)
+    ; VI: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[UV1]], [[COPY1]](s32)
+    ; VI: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[UV]], [[SUB1]](s32)
+    ; VI: [[OR:%[0-9]+]]:_(s64) = G_OR [[SHL1]], [[LSHR]]
+    ; VI: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; VI: [[SHL2:%[0-9]+]]:_(s64) = G_SHL [[UV]], [[SUB]](s32)
+    ; VI: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[SHL]], [[C2]]
+    ; VI: [[SELECT1:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[OR]], [[SHL2]]
+    ; VI: [[SELECT2:%[0-9]+]]:_(s64) = G_SELECT [[ICMP1]](s1), [[UV1]], [[SELECT1]]
+    ; VI: [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[SELECT]](s64), [[SELECT2]](s64)
+    ; VI: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[MV]](s128)
+    ; GFX9-LABEL: name: test_shl_s128_s132
+    ; GFX9: [[COPY:%[0-9]+]]:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; GFX9: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr4
+    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 64
+    ; GFX9: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](s128)
+    ; GFX9: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[COPY1]], [[C]]
+    ; GFX9: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[C]], [[COPY1]]
+    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; GFX9: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[COPY1]](s32), [[C]]
+    ; GFX9: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[COPY1]](s32), [[C1]]
+    ; GFX9: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[UV1]], [[COPY1]](s32)
+    ; GFX9: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[UV1]], [[COPY1]](s32)
+    ; GFX9: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[UV]], [[SUB1]](s32)
+    ; GFX9: [[OR:%[0-9]+]]:_(s64) = G_OR [[SHL1]], [[LSHR]]
+    ; GFX9: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; GFX9: [[SHL2:%[0-9]+]]:_(s64) = G_SHL [[UV]], [[SUB]](s32)
+    ; GFX9: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[SHL]], [[C2]]
+    ; GFX9: [[SELECT1:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[OR]], [[SHL2]]
+    ; GFX9: [[SELECT2:%[0-9]+]]:_(s64) = G_SELECT [[ICMP1]](s1), [[UV1]], [[SELECT1]]
+    ; GFX9: [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[SELECT]](s64), [[SELECT2]](s64)
+    ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[MV]](s128)
+    %0:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    %1:_(s32) = COPY $vgpr4
+    %2:_(s128) = G_SHL %0, %1
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %2
+...
+
+---
+name: test_shl_s128_s32_0
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4
+
+    ; SI-LABEL: name: test_shl_s128_s32_0
+    ; SI: [[COPY:%[0-9]+]]:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; SI: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](s128)
+    ; SI: [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[UV]](s64), [[UV1]](s64)
+    ; SI: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[MV]](s128)
+    ; VI-LABEL: name: test_shl_s128_s32_0
+    ; VI: [[COPY:%[0-9]+]]:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; VI: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](s128)
+    ; VI: [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[UV]](s64), [[UV1]](s64)
+    ; VI: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[MV]](s128)
+    ; GFX9-LABEL: name: test_shl_s128_s32_0
+    ; GFX9: [[COPY:%[0-9]+]]:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; GFX9: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](s128)
+    ; GFX9: [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[UV]](s64), [[UV1]](s64)
+    ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[MV]](s128)
+    %0:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    %1:_(s32) = G_CONSTANT i32 0
+    %3:_(s128) = G_SHL %0, %1
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %3
+...
+
+
+---
+name: test_shl_s128_s32_23
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4
+
+    ; SI-LABEL: name: test_shl_s128_s32_23
+    ; SI: [[COPY:%[0-9]+]]:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; SI: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](s128)
+    ; SI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 23
+    ; SI: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[UV]], [[C]](s32)
+    ; SI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 23
+    ; SI: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[UV1]], [[C1]](s32)
+    ; SI: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 41
+    ; SI: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[UV]], [[C2]](s32)
+    ; SI: [[OR:%[0-9]+]]:_(s64) = G_OR [[SHL1]], [[LSHR]]
+    ; SI: [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[SHL]](s64), [[OR]](s64)
+    ; SI: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[MV]](s128)
+    ; VI-LABEL: name: test_shl_s128_s32_23
+    ; VI: [[COPY:%[0-9]+]]:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; VI: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](s128)
+    ; VI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 23
+    ; VI: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[UV]], [[C]](s32)
+    ; VI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 23
+    ; VI: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[UV1]], [[C1]](s32)
+    ; VI: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 41
+    ; VI: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[UV]], [[C2]](s32)
+    ; VI: [[OR:%[0-9]+]]:_(s64) = G_OR [[SHL1]], [[LSHR]]
+    ; VI: [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[SHL]](s64), [[OR]](s64)
+    ; VI: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[MV]](s128)
+    ; GFX9-LABEL: name: test_shl_s128_s32_23
+    ; GFX9: [[COPY:%[0-9]+]]:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; GFX9: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](s128)
+    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 23
+    ; GFX9: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[UV]], [[C]](s32)
+    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 23
+    ; GFX9: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[UV1]], [[C1]](s32)
+    ; GFX9: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 41
+    ; GFX9: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[UV]], [[C2]](s32)
+    ; GFX9: [[OR:%[0-9]+]]:_(s64) = G_OR [[SHL1]], [[LSHR]]
+    ; GFX9: [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[SHL]](s64), [[OR]](s64)
+    ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[MV]](s128)
+    %0:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    %1:_(s32) = G_CONSTANT i32 23
+    %3:_(s128) = G_SHL %0, %1
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %3
+...
+
+---
+name: test_shl_s128_s32_31
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4
+
+    ; SI-LABEL: name: test_shl_s128_s32_31
+    ; SI: [[COPY:%[0-9]+]]:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; SI: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](s128)
+    ; SI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 31
+    ; SI: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[UV]], [[C]](s32)
+    ; SI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 31
+    ; SI: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[UV1]], [[C1]](s32)
+    ; SI: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 33
+    ; SI: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[UV]], [[C2]](s32)
+    ; SI: [[OR:%[0-9]+]]:_(s64) = G_OR [[SHL1]], [[LSHR]]
+    ; SI: [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[SHL]](s64), [[OR]](s64)
+    ; SI: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[MV]](s128)
+    ; VI-LABEL: name: test_shl_s128_s32_31
+    ; VI: [[COPY:%[0-9]+]]:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; VI: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](s128)
+    ; VI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 31
+    ; VI: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[UV]], [[C]](s32)
+    ; VI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 31
+    ; VI: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[UV1]], [[C1]](s32)
+    ; VI: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 33
+    ; VI: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[UV]], [[C2]](s32)
+    ; VI: [[OR:%[0-9]+]]:_(s64) = G_OR [[SHL1]], [[LSHR]]
+    ; VI: [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[SHL]](s64), [[OR]](s64)
+    ; VI: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[MV]](s128)
+    ; GFX9-LABEL: name: test_shl_s128_s32_31
+    ; GFX9: [[COPY:%[0-9]+]]:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; GFX9: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](s128)
+    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 31
+    ; GFX9: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[UV]], [[C]](s32)
+    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 31
+    ; GFX9: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[UV1]], [[C1]](s32)
+    ; GFX9: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 33
+    ; GFX9: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[UV]], [[C2]](s32)
+    ; GFX9: [[OR:%[0-9]+]]:_(s64) = G_OR [[SHL1]], [[LSHR]]
+    ; GFX9: [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[SHL]](s64), [[OR]](s64)
+    ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[MV]](s128)
+    %0:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    %1:_(s32) = G_CONSTANT i32 31
+    %3:_(s128) = G_SHL %0, %1
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %3
+...
+
+---
+name: test_shl_s128_s32_32
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4
+
+    ; SI-LABEL: name: test_shl_s128_s32_32
+    ; SI: [[COPY:%[0-9]+]]:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; SI: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](s128)
+    ; SI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 32
+    ; SI: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[UV]], [[C]](s32)
+    ; SI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 32
+    ; SI: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[UV1]], [[C1]](s32)
+    ; SI: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 32
+    ; SI: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[UV]], [[C2]](s32)
+    ; SI: [[OR:%[0-9]+]]:_(s64) = G_OR [[SHL1]], [[LSHR]]
+    ; SI: [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[SHL]](s64), [[OR]](s64)
+    ; SI: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[MV]](s128)
+    ; VI-LABEL: name: test_shl_s128_s32_32
+    ; VI: [[COPY:%[0-9]+]]:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; VI: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](s128)
+    ; VI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 32
+    ; VI: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[UV]], [[C]](s32)
+    ; VI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 32
+    ; VI: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[UV1]], [[C1]](s32)
+    ; VI: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 32
+    ; VI: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[UV]], [[C2]](s32)
+    ; VI: [[OR:%[0-9]+]]:_(s64) = G_OR [[SHL1]], [[LSHR]]
+    ; VI: [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[SHL]](s64), [[OR]](s64)
+    ; VI: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[MV]](s128)
+    ; GFX9-LABEL: name: test_shl_s128_s32_32
+    ; GFX9: [[COPY:%[0-9]+]]:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; GFX9: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](s128)
+    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 32
+    ; GFX9: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[UV]], [[C]](s32)
+    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 32
+    ; GFX9: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[UV1]], [[C1]](s32)
+    ; GFX9: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 32
+    ; GFX9: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[UV]], [[C2]](s32)
+    ; GFX9: [[OR:%[0-9]+]]:_(s64) = G_OR [[SHL1]], [[LSHR]]
+    ; GFX9: [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[SHL]](s64), [[OR]](s64)
+    ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[MV]](s128)
+    %0:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    %1:_(s32) = G_CONSTANT i32 32
+    %3:_(s128) = G_SHL %0, %1
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %3
+...
+
+---
+name: test_shl_s128_s32_33
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4
+
+    ; SI-LABEL: name: test_shl_s128_s32_33
+    ; SI: [[COPY:%[0-9]+]]:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; SI: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](s128)
+    ; SI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 33
+    ; SI: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[UV]], [[C]](s32)
+    ; SI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 33
+    ; SI: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[UV1]], [[C1]](s32)
+    ; SI: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 31
+    ; SI: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[UV]], [[C2]](s32)
+    ; SI: [[OR:%[0-9]+]]:_(s64) = G_OR [[SHL1]], [[LSHR]]
+    ; SI: [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[SHL]](s64), [[OR]](s64)
+    ; SI: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[MV]](s128)
+    ; VI-LABEL: name: test_shl_s128_s32_33
+    ; VI: [[COPY:%[0-9]+]]:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; VI: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](s128)
+    ; VI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 33
+    ; VI: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[UV]], [[C]](s32)
+    ; VI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 33
+    ; VI: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[UV1]], [[C1]](s32)
+    ; VI: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 31
+    ; VI: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[UV]], [[C2]](s32)
+    ; VI: [[OR:%[0-9]+]]:_(s64) = G_OR [[SHL1]], [[LSHR]]
+    ; VI: [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[SHL]](s64), [[OR]](s64)
+    ; VI: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[MV]](s128)
+    ; GFX9-LABEL: name: test_shl_s128_s32_33
+    ; GFX9: [[COPY:%[0-9]+]]:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; GFX9: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](s128)
+    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 33
+    ; GFX9: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[UV]], [[C]](s32)
+    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 33
+    ; GFX9: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[UV1]], [[C1]](s32)
+    ; GFX9: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 31
+    ; GFX9: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[UV]], [[C2]](s32)
+    ; GFX9: [[OR:%[0-9]+]]:_(s64) = G_OR [[SHL1]], [[LSHR]]
+    ; GFX9: [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[SHL]](s64), [[OR]](s64)
+    ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[MV]](s128)
+    %0:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    %1:_(s32) = G_CONSTANT i32 33
+    %3:_(s128) = G_SHL %0, %1
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %3
+...
+
+---
+name: test_shl_s128_s32_127
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4
+
+    ; SI-LABEL: name: test_shl_s128_s32_127
+    ; SI: [[COPY:%[0-9]+]]:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; SI: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](s128)
+    ; SI: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; SI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 63
+    ; SI: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[UV]], [[C1]](s32)
+    ; SI: [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[C]](s64), [[SHL]](s64)
+    ; SI: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[MV]](s128)
+    ; VI-LABEL: name: test_shl_s128_s32_127
+    ; VI: [[COPY:%[0-9]+]]:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; VI: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](s128)
+    ; VI: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; VI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 63
+    ; VI: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[UV]], [[C1]](s32)
+    ; VI: [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[C]](s64), [[SHL]](s64)
+    ; VI: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[MV]](s128)
+    ; GFX9-LABEL: name: test_shl_s128_s32_127
+    ; GFX9: [[COPY:%[0-9]+]]:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; GFX9: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](s128)
+    ; GFX9: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 63
+    ; GFX9: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[UV]], [[C1]](s32)
+    ; GFX9: [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[C]](s64), [[SHL]](s64)
+    ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[MV]](s128)
+    %0:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    %1:_(s32) = G_CONSTANT i32 127
+    %3:_(s128) = G_SHL %0, %1
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %3
+...
+
+---
+name: test_shl_s256_s256
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, $vgpr8
+
+    ; SI-LABEL: name: test_shl_s256_s256
+    ; SI: [[COPY:%[0-9]+]]:_(s256) = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+    ; SI: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr8
+    ; SI: [[ZEXT:%[0-9]+]]:_(s256) = G_ZEXT [[COPY1]](s32)
+    ; SI: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[ZEXT]](s256)
+    ; SI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 128
+    ; SI: [[UV:%[0-9]+]]:_(s128), [[UV1:%[0-9]+]]:_(s128) = G_UNMERGE_VALUES [[COPY]](s256)
+    ; SI: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[TRUNC]], [[C]]
+    ; SI: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[C]], [[TRUNC]]
+    ; SI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; SI: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[TRUNC]](s32), [[C]]
+    ; SI: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[TRUNC]](s32), [[C1]]
+    ; SI: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 64
+    ; SI: [[UV2:%[0-9]+]]:_(s64), [[UV3:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[UV1]](s128)
+    ; SI: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[TRUNC]], [[C2]]
+    ; SI: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[TRUNC]]
+    ; SI: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; SI: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[TRUNC]](s32), [[C2]]
+    ; SI: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[TRUNC]](s32), [[C3]]
+    ; SI: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[UV3]], [[TRUNC]](s32)
+    ; SI: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[UV3]], [[TRUNC]](s32)
+    ; SI: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[UV2]], [[SUB3]](s32)
+    ; SI: [[OR:%[0-9]+]]:_(s64) = G_OR [[SHL1]], [[LSHR]]
+    ; SI: [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; SI: [[SHL2:%[0-9]+]]:_(s64) = G_SHL [[UV2]], [[SUB2]](s32)
+    ; SI: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[ICMP2]](s1), [[SHL]], [[C4]]
+    ; SI: [[SELECT1:%[0-9]+]]:_(s64) = G_SELECT [[ICMP2]](s1), [[OR]], [[SHL2]]
+    ; SI: [[SELECT2:%[0-9]+]]:_(s64) = G_SELECT [[ICMP3]](s1), [[UV3]], [[SELECT1]]
+    ; SI: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 64
+    ; SI: [[UV4:%[0-9]+]]:_(s64), [[UV5:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[UV1]](s128)
+    ; SI: [[SUB4:%[0-9]+]]:_(s32) = G_SUB [[TRUNC]], [[C5]]
+    ; SI: [[SUB5:%[0-9]+]]:_(s32) = G_SUB [[C5]], [[TRUNC]]
+    ; SI: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; SI: [[ICMP4:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[TRUNC]](s32), [[C5]]
+    ; SI: [[ICMP5:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[TRUNC]](s32), [[C6]]
+    ; SI: [[SHL3:%[0-9]+]]:_(s64) = G_SHL [[UV5]], [[TRUNC]](s32)
+    ; SI: [[SHL4:%[0-9]+]]:_(s64) = G_SHL [[UV5]], [[TRUNC]](s32)
+    ; SI: [[LSHR1:%[0-9]+]]:_(s64) = G_LSHR [[UV4]], [[SUB5]](s32)
+    ; SI: [[OR1:%[0-9]+]]:_(s64) = G_OR [[SHL4]], [[LSHR1]]
+    ; SI: [[C7:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; SI: [[SHL5:%[0-9]+]]:_(s64) = G_SHL [[UV4]], [[SUB4]](s32)
+    ; SI: [[SELECT3:%[0-9]+]]:_(s64) = G_SELECT [[ICMP4]](s1), [[SHL3]], [[C7]]
+    ; SI: [[SELECT4:%[0-9]+]]:_(s64) = G_SELECT [[ICMP4]](s1), [[OR1]], [[SHL5]]
+    ; SI: [[SELECT5:%[0-9]+]]:_(s64) = G_SELECT [[ICMP5]](s1), [[UV5]], [[SELECT4]]
+    ; SI: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 64
+    ; SI: [[UV6:%[0-9]+]]:_(s64), [[UV7:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[UV]](s128)
+    ; SI: [[SUB6:%[0-9]+]]:_(s32) = G_SUB [[SUB1]], [[C8]]
+    ; SI: [[SUB7:%[0-9]+]]:_(s32) = G_SUB [[C8]], [[SUB1]]
+    ; SI: [[C9:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; SI: [[ICMP6:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[SUB1]](s32), [[C8]]
+    ; SI: [[ICMP7:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[SUB1]](s32), [[C9]]
+    ; SI: [[LSHR2:%[0-9]+]]:_(s64) = G_LSHR [[UV7]], [[SUB1]](s32)
+    ; SI: [[LSHR3:%[0-9]+]]:_(s64) = G_LSHR [[UV6]], [[SUB1]](s32)
+    ; SI: [[SHL6:%[0-9]+]]:_(s64) = G_SHL [[UV7]], [[SUB7]](s32)
+    ; SI: [[OR2:%[0-9]+]]:_(s64) = G_OR [[LSHR3]], [[SHL6]]
+    ; SI: [[C10:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; SI: [[LSHR4:%[0-9]+]]:_(s64) = G_LSHR [[UV7]], [[SUB6]](s32)
+    ; SI: [[SELECT6:%[0-9]+]]:_(s64) = G_SELECT [[ICMP6]](s1), [[OR2]], [[LSHR4]]
+    ; SI: [[SELECT7:%[0-9]+]]:_(s64) = G_SELECT [[ICMP7]](s1), [[UV6]], [[SELECT6]]
+    ; SI: [[SELECT8:%[0-9]+]]:_(s64) = G_SELECT [[ICMP6]](s1), [[LSHR2]], [[C10]]
+    ; SI: [[OR3:%[0-9]+]]:_(s64) = G_OR [[SELECT3]], [[SELECT7]]
+    ; SI: [[OR4:%[0-9]+]]:_(s64) = G_OR [[SELECT5]], [[SELECT8]]
+    ; SI: [[C11:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; SI: [[C12:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; SI: [[C13:%[0-9]+]]:_(s32) = G_CONSTANT i32 64
+    ; SI: [[UV8:%[0-9]+]]:_(s64), [[UV9:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[UV]](s128)
+    ; SI: [[SUB8:%[0-9]+]]:_(s32) = G_SUB [[SUB]], [[C13]]
+    ; SI: [[SUB9:%[0-9]+]]:_(s32) = G_SUB [[C13]], [[SUB]]
+    ; SI: [[C14:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; SI: [[ICMP8:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[SUB]](s32), [[C13]]
+    ; SI: [[ICMP9:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[SUB]](s32), [[C14]]
+    ; SI: [[SHL7:%[0-9]+]]:_(s64) = G_SHL [[UV9]], [[SUB]](s32)
+    ; SI: [[SHL8:%[0-9]+]]:_(s64) = G_SHL [[UV9]], [[SUB]](s32)
+    ; SI: [[LSHR5:%[0-9]+]]:_(s64) = G_LSHR [[UV8]], [[SUB9]](s32)
+    ; SI: [[OR5:%[0-9]+]]:_(s64) = G_OR [[SHL8]], [[LSHR5]]
+    ; SI: [[C15:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; SI: [[SHL9:%[0-9]+]]:_(s64) = G_SHL [[UV8]], [[SUB8]](s32)
+    ; SI: [[SELECT9:%[0-9]+]]:_(s64) = G_SELECT [[ICMP8]](s1), [[SHL7]], [[C15]]
+    ; SI: [[SELECT10:%[0-9]+]]:_(s64) = G_SELECT [[ICMP8]](s1), [[OR5]], [[SHL9]]
+    ; SI: [[SELECT11:%[0-9]+]]:_(s64) = G_SELECT [[ICMP9]](s1), [[UV9]], [[SELECT10]]
+    ; SI: [[SELECT12:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[SELECT]], [[C11]]
+    ; SI: [[SELECT13:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[SELECT2]], [[C12]]
+    ; SI: [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[SELECT12]](s64), [[SELECT13]](s64)
+    ; SI: [[SELECT14:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[OR3]], [[SELECT9]]
+    ; SI: [[SELECT15:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[OR4]], [[SELECT11]]
+    ; SI: [[UV10:%[0-9]+]]:_(s64), [[UV11:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[UV1]](s128)
+    ; SI: [[SELECT16:%[0-9]+]]:_(s64) = G_SELECT [[ICMP1]](s1), [[UV10]], [[SELECT14]]
+    ; SI: [[SELECT17:%[0-9]+]]:_(s64) = G_SELECT [[ICMP1]](s1), [[UV11]], [[SELECT15]]
+    ; SI: [[MV1:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[SELECT16]](s64), [[SELECT17]](s64)
+    ; SI: [[MV2:%[0-9]+]]:_(s256) = G_MERGE_VALUES [[MV]](s128), [[MV1]](s128)
+    ; SI: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY [[MV2]](s256)
+    ; VI-LABEL: name: test_shl_s256_s256
+    ; VI: [[COPY:%[0-9]+]]:_(s256) = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+    ; VI: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr8
+    ; VI: [[ZEXT:%[0-9]+]]:_(s256) = G_ZEXT [[COPY1]](s32)
+    ; VI: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[ZEXT]](s256)
+    ; VI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 128
+    ; VI: [[UV:%[0-9]+]]:_(s128), [[UV1:%[0-9]+]]:_(s128) = G_UNMERGE_VALUES [[COPY]](s256)
+    ; VI: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[TRUNC]], [[C]]
+    ; VI: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[C]], [[TRUNC]]
+    ; VI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; VI: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[TRUNC]](s32), [[C]]
+    ; VI: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[TRUNC]](s32), [[C1]]
+    ; VI: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 64
+    ; VI: [[UV2:%[0-9]+]]:_(s64), [[UV3:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[UV1]](s128)
+    ; VI: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[TRUNC]], [[C2]]
+    ; VI: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[TRUNC]]
+    ; VI: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; VI: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[TRUNC]](s32), [[C2]]
+    ; VI: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[TRUNC]](s32), [[C3]]
+    ; VI: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[UV3]], [[TRUNC]](s32)
+    ; VI: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[UV3]], [[TRUNC]](s32)
+    ; VI: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[UV2]], [[SUB3]](s32)
+    ; VI: [[OR:%[0-9]+]]:_(s64) = G_OR [[SHL1]], [[LSHR]]
+    ; VI: [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; VI: [[SHL2:%[0-9]+]]:_(s64) = G_SHL [[UV2]], [[SUB2]](s32)
+    ; VI: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[ICMP2]](s1), [[SHL]], [[C4]]
+    ; VI: [[SELECT1:%[0-9]+]]:_(s64) = G_SELECT [[ICMP2]](s1), [[OR]], [[SHL2]]
+    ; VI: [[SELECT2:%[0-9]+]]:_(s64) = G_SELECT [[ICMP3]](s1), [[UV3]], [[SELECT1]]
+    ; VI: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 64
+    ; VI: [[UV4:%[0-9]+]]:_(s64), [[UV5:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[UV1]](s128)
+    ; VI: [[SUB4:%[0-9]+]]:_(s32) = G_SUB [[TRUNC]], [[C5]]
+    ; VI: [[SUB5:%[0-9]+]]:_(s32) = G_SUB [[C5]], [[TRUNC]]
+    ; VI: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; VI: [[ICMP4:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[TRUNC]](s32), [[C5]]
+    ; VI: [[ICMP5:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[TRUNC]](s32), [[C6]]
+    ; VI: [[SHL3:%[0-9]+]]:_(s64) = G_SHL [[UV5]], [[TRUNC]](s32)
+    ; VI: [[SHL4:%[0-9]+]]:_(s64) = G_SHL [[UV5]], [[TRUNC]](s32)
+    ; VI: [[LSHR1:%[0-9]+]]:_(s64) = G_LSHR [[UV4]], [[SUB5]](s32)
+    ; VI: [[OR1:%[0-9]+]]:_(s64) = G_OR [[SHL4]], [[LSHR1]]
+    ; VI: [[C7:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; VI: [[SHL5:%[0-9]+]]:_(s64) = G_SHL [[UV4]], [[SUB4]](s32)
+    ; VI: [[SELECT3:%[0-9]+]]:_(s64) = G_SELECT [[ICMP4]](s1), [[SHL3]], [[C7]]
+    ; VI: [[SELECT4:%[0-9]+]]:_(s64) = G_SELECT [[ICMP4]](s1), [[OR1]], [[SHL5]]
+    ; VI: [[SELECT5:%[0-9]+]]:_(s64) = G_SELECT [[ICMP5]](s1), [[UV5]], [[SELECT4]]
+    ; VI: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 64
+    ; VI: [[UV6:%[0-9]+]]:_(s64), [[UV7:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[UV]](s128)
+    ; VI: [[SUB6:%[0-9]+]]:_(s32) = G_SUB [[SUB1]], [[C8]]
+    ; VI: [[SUB7:%[0-9]+]]:_(s32) = G_SUB [[C8]], [[SUB1]]
+    ; VI: [[C9:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; VI: [[ICMP6:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[SUB1]](s32), [[C8]]
+    ; VI: [[ICMP7:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[SUB1]](s32), [[C9]]
+    ; VI: [[LSHR2:%[0-9]+]]:_(s64) = G_LSHR [[UV7]], [[SUB1]](s32)
+    ; VI: [[LSHR3:%[0-9]+]]:_(s64) = G_LSHR [[UV6]], [[SUB1]](s32)
+    ; VI: [[SHL6:%[0-9]+]]:_(s64) = G_SHL [[UV7]], [[SUB7]](s32)
+    ; VI: [[OR2:%[0-9]+]]:_(s64) = G_OR [[LSHR3]], [[SHL6]]
+    ; VI: [[C10:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; VI: [[LSHR4:%[0-9]+]]:_(s64) = G_LSHR [[UV7]], [[SUB6]](s32)
+    ; VI: [[SELECT6:%[0-9]+]]:_(s64) = G_SELECT [[ICMP6]](s1), [[OR2]], [[LSHR4]]
+    ; VI: [[SELECT7:%[0-9]+]]:_(s64) = G_SELECT [[ICMP7]](s1), [[UV6]], [[SELECT6]]
+    ; VI: [[SELECT8:%[0-9]+]]:_(s64) = G_SELECT [[ICMP6]](s1), [[LSHR2]], [[C10]]
+    ; VI: [[OR3:%[0-9]+]]:_(s64) = G_OR [[SELECT3]], [[SELECT7]]
+    ; VI: [[OR4:%[0-9]+]]:_(s64) = G_OR [[SELECT5]], [[SELECT8]]
+    ; VI: [[C11:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; VI: [[C12:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; VI: [[C13:%[0-9]+]]:_(s32) = G_CONSTANT i32 64
+    ; VI: [[UV8:%[0-9]+]]:_(s64), [[UV9:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[UV]](s128)
+    ; VI: [[SUB8:%[0-9]+]]:_(s32) = G_SUB [[SUB]], [[C13]]
+    ; VI: [[SUB9:%[0-9]+]]:_(s32) = G_SUB [[C13]], [[SUB]]
+    ; VI: [[C14:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; VI: [[ICMP8:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[SUB]](s32), [[C13]]
+    ; VI: [[ICMP9:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[SUB]](s32), [[C14]]
+    ; VI: [[SHL7:%[0-9]+]]:_(s64) = G_SHL [[UV9]], [[SUB]](s32)
+    ; VI: [[SHL8:%[0-9]+]]:_(s64) = G_SHL [[UV9]], [[SUB]](s32)
+    ; VI: [[LSHR5:%[0-9]+]]:_(s64) = G_LSHR [[UV8]], [[SUB9]](s32)
+    ; VI: [[OR5:%[0-9]+]]:_(s64) = G_OR [[SHL8]], [[LSHR5]]
+    ; VI: [[C15:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; VI: [[SHL9:%[0-9]+]]:_(s64) = G_SHL [[UV8]], [[SUB8]](s32)
+    ; VI: [[SELECT9:%[0-9]+]]:_(s64) = G_SELECT [[ICMP8]](s1), [[SHL7]], [[C15]]
+    ; VI: [[SELECT10:%[0-9]+]]:_(s64) = G_SELECT [[ICMP8]](s1), [[OR5]], [[SHL9]]
+    ; VI: [[SELECT11:%[0-9]+]]:_(s64) = G_SELECT [[ICMP9]](s1), [[UV9]], [[SELECT10]]
+    ; VI: [[SELECT12:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[SELECT]], [[C11]]
+    ; VI: [[SELECT13:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[SELECT2]], [[C12]]
+    ; VI: [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[SELECT12]](s64), [[SELECT13]](s64)
+    ; VI: [[SELECT14:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[OR3]], [[SELECT9]]
+    ; VI: [[SELECT15:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[OR4]], [[SELECT11]]
+    ; VI: [[UV10:%[0-9]+]]:_(s64), [[UV11:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[UV1]](s128)
+    ; VI: [[SELECT16:%[0-9]+]]:_(s64) = G_SELECT [[ICMP1]](s1), [[UV10]], [[SELECT14]]
+    ; VI: [[SELECT17:%[0-9]+]]:_(s64) = G_SELECT [[ICMP1]](s1), [[UV11]], [[SELECT15]]
+    ; VI: [[MV1:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[SELECT16]](s64), [[SELECT17]](s64)
+    ; VI: [[MV2:%[0-9]+]]:_(s256) = G_MERGE_VALUES [[MV]](s128), [[MV1]](s128)
+    ; VI: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY [[MV2]](s256)
+    ; GFX9-LABEL: name: test_shl_s256_s256
+    ; GFX9: [[COPY:%[0-9]+]]:_(s256) = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+    ; GFX9: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr8
+    ; GFX9: [[ZEXT:%[0-9]+]]:_(s256) = G_ZEXT [[COPY1]](s32)
+    ; GFX9: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[ZEXT]](s256)
+    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 128
+    ; GFX9: [[UV:%[0-9]+]]:_(s128), [[UV1:%[0-9]+]]:_(s128) = G_UNMERGE_VALUES [[COPY]](s256)
+    ; GFX9: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[TRUNC]], [[C]]
+    ; GFX9: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[C]], [[TRUNC]]
+    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; GFX9: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[TRUNC]](s32), [[C]]
+    ; GFX9: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[TRUNC]](s32), [[C1]]
+    ; GFX9: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 64
+    ; GFX9: [[UV2:%[0-9]+]]:_(s64), [[UV3:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[UV1]](s128)
+    ; GFX9: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[TRUNC]], [[C2]]
+    ; GFX9: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[C2]], [[TRUNC]]
+    ; GFX9: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; GFX9: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[TRUNC]](s32), [[C2]]
+    ; GFX9: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[TRUNC]](s32), [[C3]]
+    ; GFX9: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[UV3]], [[TRUNC]](s32)
+    ; GFX9: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[UV3]], [[TRUNC]](s32)
+    ; GFX9: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[UV2]], [[SUB3]](s32)
+    ; GFX9: [[OR:%[0-9]+]]:_(s64) = G_OR [[SHL1]], [[LSHR]]
+    ; GFX9: [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; GFX9: [[SHL2:%[0-9]+]]:_(s64) = G_SHL [[UV2]], [[SUB2]](s32)
+    ; GFX9: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[ICMP2]](s1), [[SHL]], [[C4]]
+    ; GFX9: [[SELECT1:%[0-9]+]]:_(s64) = G_SELECT [[ICMP2]](s1), [[OR]], [[SHL2]]
+    ; GFX9: [[SELECT2:%[0-9]+]]:_(s64) = G_SELECT [[ICMP3]](s1), [[UV3]], [[SELECT1]]
+    ; GFX9: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 64
+    ; GFX9: [[UV4:%[0-9]+]]:_(s64), [[UV5:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[UV1]](s128)
+    ; GFX9: [[SUB4:%[0-9]+]]:_(s32) = G_SUB [[TRUNC]], [[C5]]
+    ; GFX9: [[SUB5:%[0-9]+]]:_(s32) = G_SUB [[C5]], [[TRUNC]]
+    ; GFX9: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; GFX9: [[ICMP4:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[TRUNC]](s32), [[C5]]
+    ; GFX9: [[ICMP5:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[TRUNC]](s32), [[C6]]
+    ; GFX9: [[SHL3:%[0-9]+]]:_(s64) = G_SHL [[UV5]], [[TRUNC]](s32)
+    ; GFX9: [[SHL4:%[0-9]+]]:_(s64) = G_SHL [[UV5]], [[TRUNC]](s32)
+    ; GFX9: [[LSHR1:%[0-9]+]]:_(s64) = G_LSHR [[UV4]], [[SUB5]](s32)
+    ; GFX9: [[OR1:%[0-9]+]]:_(s64) = G_OR [[SHL4]], [[LSHR1]]
+    ; GFX9: [[C7:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; GFX9: [[SHL5:%[0-9]+]]:_(s64) = G_SHL [[UV4]], [[SUB4]](s32)
+    ; GFX9: [[SELECT3:%[0-9]+]]:_(s64) = G_SELECT [[ICMP4]](s1), [[SHL3]], [[C7]]
+    ; GFX9: [[SELECT4:%[0-9]+]]:_(s64) = G_SELECT [[ICMP4]](s1), [[OR1]], [[SHL5]]
+    ; GFX9: [[SELECT5:%[0-9]+]]:_(s64) = G_SELECT [[ICMP5]](s1), [[UV5]], [[SELECT4]]
+    ; GFX9: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 64
+    ; GFX9: [[UV6:%[0-9]+]]:_(s64), [[UV7:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[UV]](s128)
+    ; GFX9: [[SUB6:%[0-9]+]]:_(s32) = G_SUB [[SUB1]], [[C8]]
+    ; GFX9: [[SUB7:%[0-9]+]]:_(s32) = G_SUB [[C8]], [[SUB1]]
+    ; GFX9: [[C9:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; GFX9: [[ICMP6:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[SUB1]](s32), [[C8]]
+    ; GFX9: [[ICMP7:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[SUB1]](s32), [[C9]]
+    ; GFX9: [[LSHR2:%[0-9]+]]:_(s64) = G_LSHR [[UV7]], [[SUB1]](s32)
+    ; GFX9: [[LSHR3:%[0-9]+]]:_(s64) = G_LSHR [[UV6]], [[SUB1]](s32)
+    ; GFX9: [[SHL6:%[0-9]+]]:_(s64) = G_SHL [[UV7]], [[SUB7]](s32)
+    ; GFX9: [[OR2:%[0-9]+]]:_(s64) = G_OR [[LSHR3]], [[SHL6]]
+    ; GFX9: [[C10:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; GFX9: [[LSHR4:%[0-9]+]]:_(s64) = G_LSHR [[UV7]], [[SUB6]](s32)
+    ; GFX9: [[SELECT6:%[0-9]+]]:_(s64) = G_SELECT [[ICMP6]](s1), [[OR2]], [[LSHR4]]
+    ; GFX9: [[SELECT7:%[0-9]+]]:_(s64) = G_SELECT [[ICMP7]](s1), [[UV6]], [[SELECT6]]
+    ; GFX9: [[SELECT8:%[0-9]+]]:_(s64) = G_SELECT [[ICMP6]](s1), [[LSHR2]], [[C10]]
+    ; GFX9: [[OR3:%[0-9]+]]:_(s64) = G_OR [[SELECT3]], [[SELECT7]]
+    ; GFX9: [[OR4:%[0-9]+]]:_(s64) = G_OR [[SELECT5]], [[SELECT8]]
+    ; GFX9: [[C11:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; GFX9: [[C12:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; GFX9: [[C13:%[0-9]+]]:_(s32) = G_CONSTANT i32 64
+    ; GFX9: [[UV8:%[0-9]+]]:_(s64), [[UV9:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[UV]](s128)
+    ; GFX9: [[SUB8:%[0-9]+]]:_(s32) = G_SUB [[SUB]], [[C13]]
+    ; GFX9: [[SUB9:%[0-9]+]]:_(s32) = G_SUB [[C13]], [[SUB]]
+    ; GFX9: [[C14:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; GFX9: [[ICMP8:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[SUB]](s32), [[C13]]
+    ; GFX9: [[ICMP9:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[SUB]](s32), [[C14]]
+    ; GFX9: [[SHL7:%[0-9]+]]:_(s64) = G_SHL [[UV9]], [[SUB]](s32)
+    ; GFX9: [[SHL8:%[0-9]+]]:_(s64) = G_SHL [[UV9]], [[SUB]](s32)
+    ; GFX9: [[LSHR5:%[0-9]+]]:_(s64) = G_LSHR [[UV8]], [[SUB9]](s32)
+    ; GFX9: [[OR5:%[0-9]+]]:_(s64) = G_OR [[SHL8]], [[LSHR5]]
+    ; GFX9: [[C15:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; GFX9: [[SHL9:%[0-9]+]]:_(s64) = G_SHL [[UV8]], [[SUB8]](s32)
+    ; GFX9: [[SELECT9:%[0-9]+]]:_(s64) = G_SELECT [[ICMP8]](s1), [[SHL7]], [[C15]]
+    ; GFX9: [[SELECT10:%[0-9]+]]:_(s64) = G_SELECT [[ICMP8]](s1), [[OR5]], [[SHL9]]
+    ; GFX9: [[SELECT11:%[0-9]+]]:_(s64) = G_SELECT [[ICMP9]](s1), [[UV9]], [[SELECT10]]
+    ; GFX9: [[SELECT12:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[SELECT]], [[C11]]
+    ; GFX9: [[SELECT13:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[SELECT2]], [[C12]]
+    ; GFX9: [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[SELECT12]](s64), [[SELECT13]](s64)
+    ; GFX9: [[SELECT14:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[OR3]], [[SELECT9]]
+    ; GFX9: [[SELECT15:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[OR4]], [[SELECT11]]
+    ; GFX9: [[UV10:%[0-9]+]]:_(s64), [[UV11:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[UV1]](s128)
+    ; GFX9: [[SELECT16:%[0-9]+]]:_(s64) = G_SELECT [[ICMP1]](s1), [[UV10]], [[SELECT14]]
+    ; GFX9: [[SELECT17:%[0-9]+]]:_(s64) = G_SELECT [[ICMP1]](s1), [[UV11]], [[SELECT15]]
+    ; GFX9: [[MV1:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[SELECT16]](s64), [[SELECT17]](s64)
+    ; GFX9: [[MV2:%[0-9]+]]:_(s256) = G_MERGE_VALUES [[MV]](s128), [[MV1]](s128)
+    ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY [[MV2]](s256)
+    %0:_(s256) = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+    %1:_(s32) = COPY $vgpr8
+    %2:_(s256) = G_ZEXT %1
+    %3:_(s256) = G_SHL %0, %2
+    $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7  = COPY %3
+...
+
+---
+name: test_shl_v2s128_v2s32
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7, $vgpr4_vgpr5
+
+    ; SI-LABEL: name: test_shl_v2s128_v2s32
+    ; SI: [[COPY:%[0-9]+]]:_(<2 x s128>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+    ; SI: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr4_vgpr5
+    ; SI: [[UV:%[0-9]+]]:_(s128), [[UV1:%[0-9]+]]:_(s128) = G_UNMERGE_VALUES [[COPY]](<2 x s128>)
+    ; SI: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<2 x s32>)
+    ; SI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 64
+    ; SI: [[UV4:%[0-9]+]]:_(s64), [[UV5:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[UV]](s128)
+    ; SI: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[UV2]], [[C]]
+    ; SI: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[C]], [[UV2]]
+    ; SI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; SI: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[UV2]](s32), [[C]]
+    ; SI: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[UV2]](s32), [[C1]]
+    ; SI: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[UV5]], [[UV2]](s32)
+    ; SI: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[UV5]], [[UV2]](s32)
+    ; SI: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[UV4]], [[SUB1]](s32)
+    ; SI: [[OR:%[0-9]+]]:_(s64) = G_OR [[SHL1]], [[LSHR]]
+    ; SI: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; SI: [[SHL2:%[0-9]+]]:_(s64) = G_SHL [[UV4]], [[SUB]](s32)
+    ; SI: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[SHL]], [[C2]]
+    ; SI: [[SELECT1:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[OR]], [[SHL2]]
+    ; SI: [[SELECT2:%[0-9]+]]:_(s64) = G_SELECT [[ICMP1]](s1), [[UV5]], [[SELECT1]]
+    ; SI: [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[SELECT]](s64), [[SELECT2]](s64)
+    ; SI: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 64
+    ; SI: [[UV6:%[0-9]+]]:_(s64), [[UV7:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[UV1]](s128)
+    ; SI: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[UV3]], [[C3]]
+    ; SI: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[C3]], [[UV3]]
+    ; SI: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; SI: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[UV3]](s32), [[C3]]
+    ; SI: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[UV3]](s32), [[C4]]
+    ; SI: [[SHL3:%[0-9]+]]:_(s64) = G_SHL [[UV7]], [[UV3]](s32)
+    ; SI: [[SHL4:%[0-9]+]]:_(s64) = G_SHL [[UV7]], [[UV3]](s32)
+    ; SI: [[LSHR1:%[0-9]+]]:_(s64) = G_LSHR [[UV6]], [[SUB3]](s32)
+    ; SI: [[OR1:%[0-9]+]]:_(s64) = G_OR [[SHL4]], [[LSHR1]]
+    ; SI: [[C5:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; SI: [[SHL5:%[0-9]+]]:_(s64) = G_SHL [[UV6]], [[SUB2]](s32)
+    ; SI: [[SELECT3:%[0-9]+]]:_(s64) = G_SELECT [[ICMP2]](s1), [[SHL3]], [[C5]]
+    ; SI: [[SELECT4:%[0-9]+]]:_(s64) = G_SELECT [[ICMP2]](s1), [[OR1]], [[SHL5]]
+    ; SI: [[SELECT5:%[0-9]+]]:_(s64) = G_SELECT [[ICMP3]](s1), [[UV7]], [[SELECT4]]
+    ; SI: [[MV1:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[SELECT3]](s64), [[SELECT5]](s64)
+    ; SI: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s128>) = G_BUILD_VECTOR [[MV]](s128), [[MV1]](s128)
+    ; SI: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY [[BUILD_VECTOR]](<2 x s128>)
+    ; VI-LABEL: name: test_shl_v2s128_v2s32
+    ; VI: [[COPY:%[0-9]+]]:_(<2 x s128>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+    ; VI: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr4_vgpr5
+    ; VI: [[UV:%[0-9]+]]:_(s128), [[UV1:%[0-9]+]]:_(s128) = G_UNMERGE_VALUES [[COPY]](<2 x s128>)
+    ; VI: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<2 x s32>)
+    ; VI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 64
+    ; VI: [[UV4:%[0-9]+]]:_(s64), [[UV5:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[UV]](s128)
+    ; VI: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[UV2]], [[C]]
+    ; VI: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[C]], [[UV2]]
+    ; VI: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; VI: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[UV2]](s32), [[C]]
+    ; VI: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[UV2]](s32), [[C1]]
+    ; VI: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[UV5]], [[UV2]](s32)
+    ; VI: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[UV5]], [[UV2]](s32)
+    ; VI: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[UV4]], [[SUB1]](s32)
+    ; VI: [[OR:%[0-9]+]]:_(s64) = G_OR [[SHL1]], [[LSHR]]
+    ; VI: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; VI: [[SHL2:%[0-9]+]]:_(s64) = G_SHL [[UV4]], [[SUB]](s32)
+    ; VI: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[SHL]], [[C2]]
+    ; VI: [[SELECT1:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[OR]], [[SHL2]]
+    ; VI: [[SELECT2:%[0-9]+]]:_(s64) = G_SELECT [[ICMP1]](s1), [[UV5]], [[SELECT1]]
+    ; VI: [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[SELECT]](s64), [[SELECT2]](s64)
+    ; VI: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 64
+    ; VI: [[UV6:%[0-9]+]]:_(s64), [[UV7:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[UV1]](s128)
+    ; VI: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[UV3]], [[C3]]
+    ; VI: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[C3]], [[UV3]]
+    ; VI: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; VI: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[UV3]](s32), [[C3]]
+    ; VI: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[UV3]](s32), [[C4]]
+    ; VI: [[SHL3:%[0-9]+]]:_(s64) = G_SHL [[UV7]], [[UV3]](s32)
+    ; VI: [[SHL4:%[0-9]+]]:_(s64) = G_SHL [[UV7]], [[UV3]](s32)
+    ; VI: [[LSHR1:%[0-9]+]]:_(s64) = G_LSHR [[UV6]], [[SUB3]](s32)
+    ; VI: [[OR1:%[0-9]+]]:_(s64) = G_OR [[SHL4]], [[LSHR1]]
+    ; VI: [[C5:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; VI: [[SHL5:%[0-9]+]]:_(s64) = G_SHL [[UV6]], [[SUB2]](s32)
+    ; VI: [[SELECT3:%[0-9]+]]:_(s64) = G_SELECT [[ICMP2]](s1), [[SHL3]], [[C5]]
+    ; VI: [[SELECT4:%[0-9]+]]:_(s64) = G_SELECT [[ICMP2]](s1), [[OR1]], [[SHL5]]
+    ; VI: [[SELECT5:%[0-9]+]]:_(s64) = G_SELECT [[ICMP3]](s1), [[UV7]], [[SELECT4]]
+    ; VI: [[MV1:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[SELECT3]](s64), [[SELECT5]](s64)
+    ; VI: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s128>) = G_BUILD_VECTOR [[MV]](s128), [[MV1]](s128)
+    ; VI: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY [[BUILD_VECTOR]](<2 x s128>)
+    ; GFX9-LABEL: name: test_shl_v2s128_v2s32
+    ; GFX9: [[COPY:%[0-9]+]]:_(<2 x s128>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+    ; GFX9: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr4_vgpr5
+    ; GFX9: [[UV:%[0-9]+]]:_(s128), [[UV1:%[0-9]+]]:_(s128) = G_UNMERGE_VALUES [[COPY]](<2 x s128>)
+    ; GFX9: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<2 x s32>)
+    ; GFX9: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 64
+    ; GFX9: [[UV4:%[0-9]+]]:_(s64), [[UV5:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[UV]](s128)
+    ; GFX9: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[UV2]], [[C]]
+    ; GFX9: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[C]], [[UV2]]
+    ; GFX9: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; GFX9: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[UV2]](s32), [[C]]
+    ; GFX9: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[UV2]](s32), [[C1]]
+    ; GFX9: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[UV5]], [[UV2]](s32)
+    ; GFX9: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[UV5]], [[UV2]](s32)
+    ; GFX9: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[UV4]], [[SUB1]](s32)
+    ; GFX9: [[OR:%[0-9]+]]:_(s64) = G_OR [[SHL1]], [[LSHR]]
+    ; GFX9: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; GFX9: [[SHL2:%[0-9]+]]:_(s64) = G_SHL [[UV4]], [[SUB]](s32)
+    ; GFX9: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[SHL]], [[C2]]
+    ; GFX9: [[SELECT1:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[OR]], [[SHL2]]
+    ; GFX9: [[SELECT2:%[0-9]+]]:_(s64) = G_SELECT [[ICMP1]](s1), [[UV5]], [[SELECT1]]
+    ; GFX9: [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[SELECT]](s64), [[SELECT2]](s64)
+    ; GFX9: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 64
+    ; GFX9: [[UV6:%[0-9]+]]:_(s64), [[UV7:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[UV1]](s128)
+    ; GFX9: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[UV3]], [[C3]]
+    ; GFX9: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[C3]], [[UV3]]
+    ; GFX9: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; GFX9: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[UV3]](s32), [[C3]]
+    ; GFX9: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[UV3]](s32), [[C4]]
+    ; GFX9: [[SHL3:%[0-9]+]]:_(s64) = G_SHL [[UV7]], [[UV3]](s32)
+    ; GFX9: [[SHL4:%[0-9]+]]:_(s64) = G_SHL [[UV7]], [[UV3]](s32)
+    ; GFX9: [[LSHR1:%[0-9]+]]:_(s64) = G_LSHR [[UV6]], [[SUB3]](s32)
+    ; GFX9: [[OR1:%[0-9]+]]:_(s64) = G_OR [[SHL4]], [[LSHR1]]
+    ; GFX9: [[C5:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; GFX9: [[SHL5:%[0-9]+]]:_(s64) = G_SHL [[UV6]], [[SUB2]](s32)
+    ; GFX9: [[SELECT3:%[0-9]+]]:_(s64) = G_SELECT [[ICMP2]](s1), [[SHL3]], [[C5]]
+    ; GFX9: [[SELECT4:%[0-9]+]]:_(s64) = G_SELECT [[ICMP2]](s1), [[OR1]], [[SHL5]]
+    ; GFX9: [[SELECT5:%[0-9]+]]:_(s64) = G_SELECT [[ICMP3]](s1), [[UV7]], [[SELECT4]]
+    ; GFX9: [[MV1:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[SELECT3]](s64), [[SELECT5]](s64)
+    ; GFX9: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s128>) = G_BUILD_VECTOR [[MV]](s128), [[MV1]](s128)
+    ; GFX9: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY [[BUILD_VECTOR]](<2 x s128>)
+    %0:_(<2 x s128>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+    %1:_(<2 x s32>) = COPY $vgpr4_vgpr5
+    %2:_(<2 x s128>) = G_SHL %0, %1
+    $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7  = COPY %2
+...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/legalize-sitofp.mir b/test/CodeGen/AMDGPU/GlobalISel/legalize-sitofp.mir
index 649f8903cd1c291830ef47682d14aefa4749d4a8..6a45e73a18c34fa6130b4fc12e6af4adfbe4ce25 100644
--- a/test/CodeGen/AMDGPU/GlobalISel/legalize-sitofp.mir
+++ b/test/CodeGen/AMDGPU/GlobalISel/legalize-sitofp.mir
@@ -2,13 +2,67 @@
 # RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -run-pass=legalizer -global-isel %s -o - | FileCheck %s
 
 ---
-name: test_sitofp_f32_to_i32
+name: test_sitofp_s32_to_s32
 body: |
   bb.0:
     liveins: $vgpr0
 
-    ; CHECK-LABEL: name: test_sitofp_f32_to_i32
+    ; CHECK-LABEL: name: test_sitofp_s32_to_s32
     ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[SITOFP:%[0-9]+]]:_(s32) = G_SITOFP [[COPY]](s32)
+    ; CHECK: $vgpr0 = COPY [[SITOFP]](s32)
     %0:_(s32) = COPY $vgpr0
     %1:_(s32) = G_SITOFP %0
+    $vgpr0 = COPY %1
+...
+
+---
+name: test_sitofp_s32_to_s64
+body: |
+  bb.0:
+    liveins: $vgpr0
+
+    ; CHECK-LABEL: name: test_sitofp_s32_to_s64
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[SITOFP:%[0-9]+]]:_(s64) = G_SITOFP [[COPY]](s32)
+    ; CHECK: $vgpr0_vgpr1 = COPY [[SITOFP]](s64)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s64) = G_SITOFP %0
+    $vgpr0_vgpr1 = COPY %1
+...
+
+---
+name: test_sitofp_v2s32_to_v2s32
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; CHECK-LABEL: name: test_sitofp_v2s32_to_v2s32
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    ; CHECK: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
+    ; CHECK: [[SITOFP:%[0-9]+]]:_(s32) = G_SITOFP [[UV]](s32)
+    ; CHECK: [[SITOFP1:%[0-9]+]]:_(s32) = G_SITOFP [[UV1]](s32)
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[SITOFP]](s32), [[SITOFP1]](s32)
+    ; CHECK: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
+    %0:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    %1:_(<2 x s32>) = G_SITOFP %0
+    $vgpr0_vgpr1 = COPY %1
+...
+
+---
+name: test_sitofp_v2s32_to_v2s64
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; CHECK-LABEL: name: test_sitofp_v2s32_to_v2s64
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    ; CHECK: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
+    ; CHECK: [[SITOFP:%[0-9]+]]:_(s64) = G_SITOFP [[UV]](s32)
+    ; CHECK: [[SITOFP1:%[0-9]+]]:_(s64) = G_SITOFP [[UV1]](s32)
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[SITOFP]](s64), [[SITOFP1]](s64)
+    ; CHECK: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>)
+    %0:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    %1:_(<2 x s64>) = G_SITOFP %0
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %1
 ...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/legalize-smulh.mir b/test/CodeGen/AMDGPU/GlobalISel/legalize-smulh.mir
new file mode 100644
index 0000000000000000000000000000000000000000..bc08d2e422df03e9a0c2993e87f9fa2eefdd2be8
--- /dev/null
+++ b/test/CodeGen/AMDGPU/GlobalISel/legalize-smulh.mir
@@ -0,0 +1,40 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -O0 -run-pass=legalizer %s -o - | FileCheck %s
+
+---
+name: test_smulh_s32
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+
+    ; CHECK-LABEL: name: test_smulh_s32
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; CHECK: [[SMULH:%[0-9]+]]:_(s32) = G_SMULH [[COPY]], [[COPY1]]
+    ; CHECK: $vgpr0 = COPY [[SMULH]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s32) = G_SMULH %0, %1
+    $vgpr0 = COPY %2
+...
+
+---
+name: test_smulh_v2s32
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
+
+    ; CHECK-LABEL: name: test_smulh_v2s32
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    ; CHECK: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr2_vgpr3
+    ; CHECK: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
+    ; CHECK: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<2 x s32>)
+    ; CHECK: [[SMULH:%[0-9]+]]:_(s32) = G_SMULH [[UV]], [[UV2]]
+    ; CHECK: [[SMULH1:%[0-9]+]]:_(s32) = G_SMULH [[UV1]], [[UV3]]
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[SMULH]](s32), [[SMULH1]](s32)
+    ; CHECK: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
+    %0:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    %1:_(<2 x s32>) = COPY $vgpr2_vgpr3
+    %2:_(<2 x s32>) = G_SMULH %0, %1
+    $vgpr0_vgpr1 = COPY %2
+...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/legalize-store.mir b/test/CodeGen/AMDGPU/GlobalISel/legalize-store.mir
index cb4f83296d2805b71102e3cd0a4fb50ee75976a8..d5fabac98559d9d64bab1468d65609cebc6a8879 100644
--- a/test/CodeGen/AMDGPU/GlobalISel/legalize-store.mir
+++ b/test/CodeGen/AMDGPU/GlobalISel/legalize-store.mir
@@ -1,16 +1,20 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -O0 -run-pass=legalizer %s -o - | FileCheck %s
-
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=tahiti -O0 -run-pass=legalizer %s -o - | FileCheck -check-prefix=SI %s
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -O0 -run-pass=legalizer %s -o - | FileCheck -check-prefix=VI %s
 ---
 name: test_store_global_i32
 body: |
   bb.0:
     liveins: $vgpr0_vgpr1, $vgpr2
 
-    ; CHECK-LABEL: name: test_store_global_i32
-    ; CHECK: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
-    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr2
-    ; CHECK: G_STORE [[COPY1]](s32), [[COPY]](p1) :: (store 4, addrspace 1)
+    ; SI-LABEL: name: test_store_global_i32
+    ; SI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
+    ; SI: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr2
+    ; SI: G_STORE [[COPY1]](s32), [[COPY]](p1) :: (store 4, addrspace 1)
+    ; VI-LABEL: name: test_store_global_i32
+    ; VI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
+    ; VI: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr2
+    ; VI: G_STORE [[COPY1]](s32), [[COPY]](p1) :: (store 4, addrspace 1)
     %0:_(p1) = COPY $vgpr0_vgpr1
     %1:_(s32) = COPY $vgpr2
     G_STORE %1, %0 :: (store 4, addrspace 1)
@@ -22,10 +26,14 @@ body: |
   bb.0:
     liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
 
-    ; CHECK-LABEL: name: test_store_global_i64
-    ; CHECK: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
-    ; CHECK: [[COPY1:%[0-9]+]]:_(s64) = COPY $vgpr2_vgpr3
-    ; CHECK: G_STORE [[COPY1]](s64), [[COPY]](p1) :: (store 8, addrspace 1)
+    ; SI-LABEL: name: test_store_global_i64
+    ; SI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
+    ; SI: [[COPY1:%[0-9]+]]:_(s64) = COPY $vgpr2_vgpr3
+    ; SI: G_STORE [[COPY1]](s64), [[COPY]](p1) :: (store 8, addrspace 1)
+    ; VI-LABEL: name: test_store_global_i64
+    ; VI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
+    ; VI: [[COPY1:%[0-9]+]]:_(s64) = COPY $vgpr2_vgpr3
+    ; VI: G_STORE [[COPY1]](s64), [[COPY]](p1) :: (store 8, addrspace 1)
     %0:_(p1) = COPY $vgpr0_vgpr1
     %1:_(s64) = COPY $vgpr2_vgpr3
     G_STORE %1, %0 :: (store 8, addrspace 1)
@@ -37,10 +45,14 @@ body: |
   bb.0:
     liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
 
-    ; CHECK-LABEL: name: test_store_global_p1
-    ; CHECK: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
-    ; CHECK: [[COPY1:%[0-9]+]]:_(p1) = COPY $vgpr2_vgpr3
-    ; CHECK: G_STORE [[COPY1]](p1), [[COPY]](p1) :: (store 8, addrspace 1)
+    ; SI-LABEL: name: test_store_global_p1
+    ; SI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
+    ; SI: [[COPY1:%[0-9]+]]:_(p1) = COPY $vgpr2_vgpr3
+    ; SI: G_STORE [[COPY1]](p1), [[COPY]](p1) :: (store 8, addrspace 1)
+    ; VI-LABEL: name: test_store_global_p1
+    ; VI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
+    ; VI: [[COPY1:%[0-9]+]]:_(p1) = COPY $vgpr2_vgpr3
+    ; VI: G_STORE [[COPY1]](p1), [[COPY]](p1) :: (store 8, addrspace 1)
     %0:_(p1) = COPY $vgpr0_vgpr1
     %1:_(p1) = COPY $vgpr2_vgpr3
     G_STORE %1, %0 :: (store 8, addrspace 1)
@@ -52,10 +64,14 @@ body: |
   bb.0:
     liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
 
-    ; CHECK-LABEL: name: test_store_global_p4
-    ; CHECK: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
-    ; CHECK: [[COPY1:%[0-9]+]]:_(p4) = COPY $vgpr2_vgpr3
-    ; CHECK: G_STORE [[COPY1]](p4), [[COPY]](p1) :: (store 8, addrspace 1)
+    ; SI-LABEL: name: test_store_global_p4
+    ; SI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
+    ; SI: [[COPY1:%[0-9]+]]:_(p4) = COPY $vgpr2_vgpr3
+    ; SI: G_STORE [[COPY1]](p4), [[COPY]](p1) :: (store 8, addrspace 1)
+    ; VI-LABEL: name: test_store_global_p4
+    ; VI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
+    ; VI: [[COPY1:%[0-9]+]]:_(p4) = COPY $vgpr2_vgpr3
+    ; VI: G_STORE [[COPY1]](p4), [[COPY]](p1) :: (store 8, addrspace 1)
     %0:_(p1) = COPY $vgpr0_vgpr1
     %1:_(p4) = COPY $vgpr2_vgpr3
     G_STORE %1, %0 :: (store 8, addrspace 1)
@@ -67,10 +83,14 @@ body: |
   bb.0:
     liveins: $vgpr0_vgpr1, $vgpr2
 
-    ; CHECK-LABEL: name: test_store_global_p3
-    ; CHECK: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
-    ; CHECK: [[COPY1:%[0-9]+]]:_(p3) = COPY $vgpr2
-    ; CHECK: G_STORE [[COPY1]](p3), [[COPY]](p1) :: (store 4, addrspace 1)
+    ; SI-LABEL: name: test_store_global_p3
+    ; SI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
+    ; SI: [[COPY1:%[0-9]+]]:_(p3) = COPY $vgpr2
+    ; SI: G_STORE [[COPY1]](p3), [[COPY]](p1) :: (store 4, addrspace 1)
+    ; VI-LABEL: name: test_store_global_p3
+    ; VI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
+    ; VI: [[COPY1:%[0-9]+]]:_(p3) = COPY $vgpr2
+    ; VI: G_STORE [[COPY1]](p3), [[COPY]](p1) :: (store 4, addrspace 1)
     %0:_(p1) = COPY $vgpr0_vgpr1
     %1:_(p3) = COPY $vgpr2
     G_STORE %1, %0 :: (store 4, addrspace 1)
@@ -82,10 +102,14 @@ body: |
   bb.0:
     liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
 
-    ; CHECK-LABEL: name: test_store_global_v2s32
-    ; CHECK: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
-    ; CHECK: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr2_vgpr3
-    ; CHECK: G_STORE [[COPY1]](<2 x s32>), [[COPY]](p1) :: (store 8, addrspace 1)
+    ; SI-LABEL: name: test_store_global_v2s32
+    ; SI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
+    ; SI: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr2_vgpr3
+    ; SI: G_STORE [[COPY1]](<2 x s32>), [[COPY]](p1) :: (store 8, addrspace 1)
+    ; VI-LABEL: name: test_store_global_v2s32
+    ; VI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
+    ; VI: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr2_vgpr3
+    ; VI: G_STORE [[COPY1]](<2 x s32>), [[COPY]](p1) :: (store 8, addrspace 1)
     %0:_(p1) = COPY $vgpr0_vgpr1
     %1:_(<2 x s32>) = COPY $vgpr2_vgpr3
     G_STORE %1, %0 :: (store 8, addrspace 1)
@@ -97,10 +121,14 @@ body: |
   bb.0:
     liveins: $vgpr0_vgpr1, $vgpr2
 
-    ; CHECK-LABEL: name: test_store_global_v2s16
-    ; CHECK: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
-    ; CHECK: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr2
-    ; CHECK: G_STORE [[COPY1]](<2 x s16>), [[COPY]](p1) :: (store 4, addrspace 1)
+    ; SI-LABEL: name: test_store_global_v2s16
+    ; SI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
+    ; SI: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr2
+    ; SI: G_STORE [[COPY1]](<2 x s16>), [[COPY]](p1) :: (store 4, addrspace 1)
+    ; VI-LABEL: name: test_store_global_v2s16
+    ; VI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
+    ; VI: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr2
+    ; VI: G_STORE [[COPY1]](<2 x s16>), [[COPY]](p1) :: (store 4, addrspace 1)
     %0:_(p1) = COPY $vgpr0_vgpr1
     %1:_(<2 x s16>) = COPY $vgpr2
     G_STORE %1, %0 :: (store 4, addrspace 1)
@@ -112,11 +140,234 @@ body: |
   bb.0:
     liveins: $vgpr0_vgpr1, $vgpr2_vgpr3_vgpr4
 
-    ; CHECK-LABEL: name: test_store_global_v3s32
-    ; CHECK: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
-    ; CHECK: [[COPY1:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr2_vgpr3_vgpr4
-    ; CHECK: G_STORE [[COPY1]](<3 x s32>), [[COPY]](p1) :: (store 12, align 4, addrspace 1)
+    ; SI-LABEL: name: test_store_global_v3s32
+    ; SI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
+    ; SI: [[COPY1:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr2_vgpr3_vgpr4
+    ; SI: [[EXTRACT:%[0-9]+]]:_(<2 x s32>) = G_EXTRACT [[COPY1]](<3 x s32>), 0
+    ; SI: [[EXTRACT1:%[0-9]+]]:_(s32) = G_EXTRACT [[COPY1]](<3 x s32>), 64
+    ; SI: G_STORE [[EXTRACT]](<2 x s32>), [[COPY]](p1) :: (store 8, align 4, addrspace 1)
+    ; SI: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
+    ; SI: [[GEP:%[0-9]+]]:_(p1) = G_GEP [[COPY]], [[C]](s64)
+    ; SI: G_STORE [[EXTRACT1]](s32), [[GEP]](p1) :: (store 4, addrspace 1)
+    ; VI-LABEL: name: test_store_global_v3s32
+    ; VI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
+    ; VI: [[COPY1:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr2_vgpr3_vgpr4
+    ; VI: G_STORE [[COPY1]](<3 x s32>), [[COPY]](p1) :: (store 12, align 4, addrspace 1)
     %0:_(p1) = COPY $vgpr0_vgpr1
     %1:_(<3 x s32>) = COPY $vgpr2_vgpr3_vgpr4
     G_STORE %1, %0 :: (store 12, align 4, addrspace 1)
 ...
+
+---
+name: test_truncstore_global_s64_to_s8
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
+
+    ; SI-LABEL: name: test_truncstore_global_s64_to_s8
+    ; SI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
+    ; SI: [[COPY1:%[0-9]+]]:_(s64) = COPY $vgpr2_vgpr3
+    ; SI: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY1]](s64)
+    ; SI: G_STORE [[TRUNC]](s32), [[COPY]](p1) :: (store 1, addrspace 1)
+    ; VI-LABEL: name: test_truncstore_global_s64_to_s8
+    ; VI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
+    ; VI: [[COPY1:%[0-9]+]]:_(s64) = COPY $vgpr2_vgpr3
+    ; VI: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY1]](s64)
+    ; VI: G_STORE [[TRUNC]](s32), [[COPY]](p1) :: (store 1, addrspace 1)
+    %0:_(p1) = COPY $vgpr0_vgpr1
+    %1:_(s64) = COPY $vgpr2_vgpr3
+    G_STORE %1, %0 :: (store 1, addrspace 1)
+...
+
+---
+name: test_truncstore_global_s64_to_s16
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
+
+    ; SI-LABEL: name: test_truncstore_global_s64_to_s16
+    ; SI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
+    ; SI: [[COPY1:%[0-9]+]]:_(s64) = COPY $vgpr2_vgpr3
+    ; SI: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY1]](s64)
+    ; SI: G_STORE [[TRUNC]](s32), [[COPY]](p1) :: (store 1, addrspace 1)
+    ; VI-LABEL: name: test_truncstore_global_s64_to_s16
+    ; VI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
+    ; VI: [[COPY1:%[0-9]+]]:_(s64) = COPY $vgpr2_vgpr3
+    ; VI: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY1]](s64)
+    ; VI: G_STORE [[TRUNC]](s32), [[COPY]](p1) :: (store 1, addrspace 1)
+    %0:_(p1) = COPY $vgpr0_vgpr1
+    %1:_(s64) = COPY $vgpr2_vgpr3
+    G_STORE %1, %0 :: (store 1, addrspace 1)
+...
+
+---
+name: test_truncstore_global_s64_to_s32
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
+
+    ; SI-LABEL: name: test_truncstore_global_s64_to_s32
+    ; SI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
+    ; SI: [[COPY1:%[0-9]+]]:_(s64) = COPY $vgpr2_vgpr3
+    ; SI: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY1]](s64)
+    ; SI: G_STORE [[TRUNC]](s32), [[COPY]](p1) :: (store 4, addrspace 1)
+    ; VI-LABEL: name: test_truncstore_global_s64_to_s32
+    ; VI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
+    ; VI: [[COPY1:%[0-9]+]]:_(s64) = COPY $vgpr2_vgpr3
+    ; VI: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY1]](s64)
+    ; VI: G_STORE [[TRUNC]](s32), [[COPY]](p1) :: (store 4, addrspace 1)
+    %0:_(p1) = COPY $vgpr0_vgpr1
+    %1:_(s64) = COPY $vgpr2_vgpr3
+    G_STORE %1, %0 :: (store 4, addrspace 1)
+...
+
+---
+name: test_truncstore_global_s128_to_s16
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2_vgpr3_vgpr4_vgpr5
+
+    %0:_(p1) = COPY $vgpr0_vgpr1
+    %1:_(s128) = COPY $vgpr2_vgpr3_vgpr4_vgpr5
+    G_STORE %1, %0 :: (store 2, addrspace 1)
+...
+
+---
+name: test_truncstore_global_s128_to_s8
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2_vgpr3_vgpr4_vgpr5
+
+    ; SI-LABEL: name: test_truncstore_global_s128_to_s16
+    ; SI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
+    ; SI: [[COPY1:%[0-9]+]]:_(s128) = COPY $vgpr2_vgpr3_vgpr4_vgpr5
+    ; SI: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY1]](s128)
+    ; SI: G_STORE [[TRUNC]](s32), [[COPY]](p1) :: (store 1, addrspace 1)
+    ; VI-LABEL: name: test_truncstore_global_s128_to_s16
+    ; VI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
+    ; VI: [[COPY1:%[0-9]+]]:_(s128) = COPY $vgpr2_vgpr3_vgpr4_vgpr5
+    ; VI: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY1]](s128)
+    ; VI: G_STORE [[TRUNC]](s32), [[COPY]](p1) :: (store 1, addrspace 1)
+    %0:_(p1) = COPY $vgpr0_vgpr1
+    %1:_(s128) = COPY $vgpr2_vgpr3_vgpr4_vgpr5
+    G_STORE %1, %0 :: (store 1, addrspace 1)
+...
+
+name: test_store_global_i1
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2
+
+    %0:_(p1) = COPY $vgpr0_vgpr1
+    %1:_(s32) = COPY $vgpr2
+    %2:_(s1) = G_TRUNC %1
+    G_STORE %2, %0 :: (store 1, addrspace 1)
+...
+
+---
+name: test_store_global_i8
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2
+
+    ; SI-LABEL: name: test_store_global_i8
+    ; SI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
+    ; SI: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr2
+    ; SI: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+    ; SI: G_STORE [[COPY2]](s32), [[COPY]](p1) :: (store 1, addrspace 1)
+    ; VI-LABEL: name: test_store_global_i8
+    ; VI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
+    ; VI: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr2
+    ; VI: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+    ; VI: G_STORE [[COPY2]](s32), [[COPY]](p1) :: (store 1, addrspace 1)
+    %0:_(p1) = COPY $vgpr0_vgpr1
+    %1:_(s32) = COPY $vgpr2
+    %2:_(s8) = G_TRUNC %1
+    G_STORE %2, %0 :: (store 1, addrspace 1)
+...
+
+---
+name: test_store_global_i16
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2
+
+    ; SI-LABEL: name: test_store_global_i16
+    ; SI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
+    ; SI: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr2
+    ; SI: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+    ; SI: G_STORE [[COPY2]](s32), [[COPY]](p1) :: (store 2, addrspace 1)
+    ; VI-LABEL: name: test_store_global_i16
+    ; VI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
+    ; VI: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr2
+    ; VI: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+    ; VI: G_STORE [[COPY2]](s32), [[COPY]](p1) :: (store 2, addrspace 1)
+    %0:_(p1) = COPY $vgpr0_vgpr1
+    %1:_(s32) = COPY $vgpr2
+    %2:_(s16) = G_TRUNC %1
+    G_STORE %2, %0 :: (store 2, addrspace 1)
+...
+
+---
+name: test_store_global_96
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2, $vgpr3_vgpr4
+
+    ; SI-LABEL: name: test_store_global_96
+    ; SI: [[COPY:%[0-9]+]]:_(s96) = COPY $vgpr0_vgpr1_vgpr2
+    ; SI: [[COPY1:%[0-9]+]]:_(p1) = COPY $vgpr3_vgpr4
+    ; SI: [[EXTRACT:%[0-9]+]]:_(s64) = G_EXTRACT [[COPY]](s96), 0
+    ; SI: [[EXTRACT1:%[0-9]+]]:_(s32) = G_EXTRACT [[COPY]](s96), 64
+    ; SI: G_STORE [[EXTRACT]](s64), [[COPY1]](p1) :: (store 8, align 16, addrspace 1)
+    ; SI: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
+    ; SI: [[GEP:%[0-9]+]]:_(p1) = G_GEP [[COPY1]], [[C]](s64)
+    ; SI: G_STORE [[EXTRACT1]](s32), [[GEP]](p1) :: (store 4, align 8, addrspace 1)
+    ; VI-LABEL: name: test_store_global_96
+    ; VI: [[COPY:%[0-9]+]]:_(s96) = COPY $vgpr0_vgpr1_vgpr2
+    ; VI: [[COPY1:%[0-9]+]]:_(p1) = COPY $vgpr3_vgpr4
+    ; VI: G_STORE [[COPY]](s96), [[COPY1]](p1) :: (store 12, align 16, addrspace 1)
+    %0:_(s96) = COPY $vgpr0_vgpr1_vgpr2
+    %1:_(p1) = COPY $vgpr3_vgpr4
+
+    G_STORE %0, %1 :: (store 12, addrspace 1, align 16)
+...
+
+---
+name: test_store_global_i128
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2_vgpr3_vgpr4_vgpr5
+
+    ; SI-LABEL: name: test_store_global_i128
+    ; SI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
+    ; SI: [[COPY1:%[0-9]+]]:_(s128) = COPY $vgpr2_vgpr3_vgpr4_vgpr5
+    ; SI: G_STORE [[COPY1]](s128), [[COPY]](p1) :: (store 16, addrspace 1)
+    ; VI-LABEL: name: test_store_global_i128
+    ; VI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
+    ; VI: [[COPY1:%[0-9]+]]:_(s128) = COPY $vgpr2_vgpr3_vgpr4_vgpr5
+    ; VI: G_STORE [[COPY1]](s128), [[COPY]](p1) :: (store 16, addrspace 1)
+    %0:_(p1) = COPY $vgpr0_vgpr1
+    %1:_(s128) = COPY $vgpr2_vgpr3_vgpr4_vgpr5
+    G_STORE %1, %0 :: (store 16, addrspace 1)
+...
+
+---
+name: test_store_global_v2s64
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2_vgpr3_vgpr4_vgpr5
+
+    ; SI-LABEL: name: test_store_global_v2s64
+    ; SI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
+    ; SI: [[COPY1:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr2_vgpr3_vgpr4_vgpr5
+    ; SI: G_STORE [[COPY1]](<2 x s64>), [[COPY]](p1) :: (store 16, addrspace 1)
+    ; VI-LABEL: name: test_store_global_v2s64
+    ; VI: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
+    ; VI: [[COPY1:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr2_vgpr3_vgpr4_vgpr5
+    ; VI: G_STORE [[COPY1]](<2 x s64>), [[COPY]](p1) :: (store 16, addrspace 1)
+    %0:_(p1) = COPY $vgpr0_vgpr1
+    %1:_(<2 x s64>) = COPY $vgpr2_vgpr3_vgpr4_vgpr5
+    G_STORE %1, %0 :: (store 16, addrspace 1)
+
+...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/legalize-sub.mir b/test/CodeGen/AMDGPU/GlobalISel/legalize-sub.mir
new file mode 100644
index 0000000000000000000000000000000000000000..4f6fd64bc60aa30e7f0fd33f5d77d669e1100fc3
--- /dev/null
+++ b/test/CodeGen/AMDGPU/GlobalISel/legalize-sub.mir
@@ -0,0 +1,40 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -O0 -run-pass=legalizer %s -o - | FileCheck %s
+
+---
+name: test_sub_s32
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+
+    ; CHECK-LABEL: name: test_sub_s32
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; CHECK: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[COPY]], [[COPY1]]
+    ; CHECK: $vgpr0 = COPY [[SUB]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s32) = G_SUB %0, %1
+    $vgpr0 = COPY %2
+...
+
+---
+name: test_sub_v2s32
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
+
+    ; CHECK-LABEL: name: test_sub_v2s32
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    ; CHECK: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr2_vgpr3
+    ; CHECK: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
+    ; CHECK: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<2 x s32>)
+    ; CHECK: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[UV]], [[UV2]]
+    ; CHECK: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[UV1]], [[UV3]]
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[SUB]](s32), [[SUB1]](s32)
+    ; CHECK: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
+    %0:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    %1:_(<2 x s32>) = COPY $vgpr2_vgpr3
+    %2:_(<2 x s32>) = G_SUB %0, %1
+    $vgpr0_vgpr1 = COPY %2
+...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/legalize-uaddo.mir b/test/CodeGen/AMDGPU/GlobalISel/legalize-uaddo.mir
new file mode 100644
index 0000000000000000000000000000000000000000..a4159c322e751df2756a13d2350dcc7580e5cb52
--- /dev/null
+++ b/test/CodeGen/AMDGPU/GlobalISel/legalize-uaddo.mir
@@ -0,0 +1,97 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -O0 -run-pass=legalizer %s -o - | FileCheck %s
+
+---
+name: test_uaddo_s32
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+
+    ; CHECK-LABEL: name: test_uaddo_s32
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; CHECK: [[UADDO:%[0-9]+]]:_(s32), [[UADDO1:%[0-9]+]]:_(s1) = G_UADDO [[COPY]], [[COPY1]]
+    ; CHECK: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[UADDO1]](s1)
+    ; CHECK: $vgpr0 = COPY [[UADDO]](s32)
+    ; CHECK: $vgpr1 = COPY [[ZEXT]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s32), %3:_(s1) = G_UADDO %0, %1
+    %4:_(s32) = G_ZEXT %3
+    $vgpr0 = COPY %2
+    $vgpr1 = COPY %4
+...
+
+---
+name: test_uaddo_s7
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+
+    ; CHECK-LABEL: name: test_uaddo_s7
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 127
+    ; CHECK: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; CHECK: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C]]
+    ; CHECK: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 127
+    ; CHECK: [[COPY3:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+    ; CHECK: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[C1]]
+    ; CHECK: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[AND]], [[AND1]]
+    ; CHECK: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 127
+    ; CHECK: [[AND2:%[0-9]+]]:_(s32) = G_AND [[ADD]], [[C2]]
+    ; CHECK: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[ADD]](s32), [[AND2]]
+    ; CHECK: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 127
+    ; CHECK: [[COPY4:%[0-9]+]]:_(s32) = COPY [[ADD]](s32)
+    ; CHECK: [[AND3:%[0-9]+]]:_(s32) = G_AND [[COPY4]], [[C3]]
+    ; CHECK: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[ICMP]](s1)
+    ; CHECK: $vgpr0 = COPY [[AND3]](s32)
+    ; CHECK: $vgpr1 = COPY [[ZEXT]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s7) = G_TRUNC %0
+    %3:_(s7) = G_TRUNC %1
+
+    %4:_(s7), %5:_(s1) = G_UADDO %2, %3
+    %6:_(s32) = G_ZEXT %4
+    %7:_(s32) = G_ZEXT %5
+    $vgpr0 = COPY %6
+    $vgpr1 = COPY %7
+...
+
+---
+name: test_uaddo_s16
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+
+    ; CHECK-LABEL: name: test_uaddo_s16
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; CHECK: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; CHECK: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C]]
+    ; CHECK: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; CHECK: [[COPY3:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+    ; CHECK: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[C1]]
+    ; CHECK: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[AND]], [[AND1]]
+    ; CHECK: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; CHECK: [[AND2:%[0-9]+]]:_(s32) = G_AND [[ADD]], [[C2]]
+    ; CHECK: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[ADD]](s32), [[AND2]]
+    ; CHECK: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; CHECK: [[COPY4:%[0-9]+]]:_(s32) = COPY [[ADD]](s32)
+    ; CHECK: [[AND3:%[0-9]+]]:_(s32) = G_AND [[COPY4]], [[C3]]
+    ; CHECK: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[ICMP]](s1)
+    ; CHECK: $vgpr0 = COPY [[AND3]](s32)
+    ; CHECK: $vgpr1 = COPY [[ZEXT]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s16) = G_TRUNC %0
+    %3:_(s16) = G_TRUNC %1
+
+    %4:_(s16), %5:_(s1) = G_UADDO %2, %3
+    %6:_(s32) = G_ZEXT %4
+    %7:_(s32) = G_ZEXT %5
+    $vgpr0 = COPY %6
+    $vgpr1 = COPY %7
+...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/legalize-uitofp.mir b/test/CodeGen/AMDGPU/GlobalISel/legalize-uitofp.mir
new file mode 100644
index 0000000000000000000000000000000000000000..f376054e6462ebfa759422a643aa994dfe7bd0b1
--- /dev/null
+++ b/test/CodeGen/AMDGPU/GlobalISel/legalize-uitofp.mir
@@ -0,0 +1,33 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -run-pass=legalizer -global-isel %s -o - | FileCheck %s
+
+---
+name: test_uitofp_s32_to_s32
+body: |
+  bb.0:
+    liveins: $vgpr0
+
+    ; CHECK-LABEL: name: test_uitofp_s32_to_s32
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[COPY]](s32)
+    ; CHECK: $vgpr0 = COPY [[UITOFP]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = G_UITOFP %0
+    $vgpr0 = COPY %1
+...
+
+---
+name: test_uitofp_s32_to_s64
+body: |
+  bb.0:
+    liveins: $vgpr0
+
+    ; CHECK-LABEL: name: test_uitofp_s32_to_s64
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[UITOFP:%[0-9]+]]:_(s64) = G_UITOFP [[COPY]](s32)
+    ; CHECK: $vgpr0_vgpr1 = COPY [[UITOFP]](s64)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s64) = G_UITOFP %0
+    $vgpr0_vgpr1 = COPY %1
+...
+
diff --git a/test/CodeGen/AMDGPU/GlobalISel/legalize-umulh.mir b/test/CodeGen/AMDGPU/GlobalISel/legalize-umulh.mir
new file mode 100644
index 0000000000000000000000000000000000000000..957bbb8c7695e141445c4787d54e44cd5bd172e4
--- /dev/null
+++ b/test/CodeGen/AMDGPU/GlobalISel/legalize-umulh.mir
@@ -0,0 +1,40 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -O0 -run-pass=legalizer %s -o - | FileCheck %s
+
+---
+name: test_umulh_s32
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+
+    ; CHECK-LABEL: name: test_umulh_s32
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; CHECK: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[COPY]], [[COPY1]]
+    ; CHECK: $vgpr0 = COPY [[UMULH]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s32) = G_UMULH %0, %1
+    $vgpr0 = COPY %2
+...
+
+---
+name: test_umulh_v2s32
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
+
+    ; CHECK-LABEL: name: test_umulh_v2s32
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    ; CHECK: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr2_vgpr3
+    ; CHECK: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
+    ; CHECK: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<2 x s32>)
+    ; CHECK: [[UMULH:%[0-9]+]]:_(s32) = G_UMULH [[UV]], [[UV2]]
+    ; CHECK: [[UMULH1:%[0-9]+]]:_(s32) = G_UMULH [[UV1]], [[UV3]]
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[UMULH]](s32), [[UMULH1]](s32)
+    ; CHECK: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
+    %0:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    %1:_(<2 x s32>) = COPY $vgpr2_vgpr3
+    %2:_(<2 x s32>) = G_UMULH %0, %1
+    $vgpr0_vgpr1 = COPY %2
+...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/legalize-unmerge-values-xfail.mir b/test/CodeGen/AMDGPU/GlobalISel/legalize-unmerge-values-xfail.mir
new file mode 100644
index 0000000000000000000000000000000000000000..5ffbeab9872d91f736d64148cc592d22c5e6b793
--- /dev/null
+++ b/test/CodeGen/AMDGPU/GlobalISel/legalize-unmerge-values-xfail.mir
@@ -0,0 +1,14 @@
+# RUN: not llc -mtriple=amdgcn-- -O0 -run-pass=legalizer -o - %s 2>&1 | FileCheck %s
+
+# CHECK: LLVM ERROR: unable to legalize instruction: %1:_(s1), %2:_(s1) = G_UNMERGE_VALUES %0:_(<2 x s1>) (in function: test_unmerge_v2s1)
+
+---
+name: test_unmerge_v2s1
+body: |
+  bb.0:
+    %0:_(<2 x s1>) = G_IMPLICIT_DEF
+    %1:_(s1), %2:_(s1) = G_UNMERGE_VALUES %0
+    S_NOP 0, implicit %1
+    S_NOP 0, implicit %2
+...
+
diff --git a/test/CodeGen/AMDGPU/GlobalISel/legalize-unmerge-values.mir b/test/CodeGen/AMDGPU/GlobalISel/legalize-unmerge-values.mir
index 1e004ac478b3e21e0f4bfb58098eb006eebaa044..fbd97179a656a6e680746ab54ab82cf3de98fbd2 100644
--- a/test/CodeGen/AMDGPU/GlobalISel/legalize-unmerge-values.mir
+++ b/test/CodeGen/AMDGPU/GlobalISel/legalize-unmerge-values.mir
@@ -1,18 +1,431 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn-- -O0 -run-pass=legalizer -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn-- -O0 -run-pass=legalizer -global-isel-abort=0 -o - %s | FileCheck %s
 
 ---
-name: test_unmerge_s64_s32
+name: test_unmerge_s32_s64
 body: |
   bb.0:
-    ; CHECK-LABEL: name: test_unmerge_s64_s32
+    ; CHECK-LABEL: name: test_unmerge_s32_s64
     ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
     ; CHECK: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[C]](s64)
     ; CHECK: $vgpr0 = COPY [[UV]](s32)
-    ; CHECK: $vgpr2 = COPY [[UV1]](s32)
+    ; CHECK: $vgpr1 = COPY [[UV1]](s32)
     %0:_(s64) = G_CONSTANT i64 0
     %1:_(s32), %2:_(s32) = G_UNMERGE_VALUES %0:_(s64)
     $vgpr0 = COPY %1(s32)
-    $vgpr2 = COPY %2(s32)
+    $vgpr1 = COPY %2(s32)
 ...
 
+---
+name: test_unmerge_s32_v2s32
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+    ; CHECK-LABEL: name: test_unmerge_s32_v2s32
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    ; CHECK: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
+    ; CHECK: $vgpr0 = COPY [[UV]](s32)
+    ; CHECK: $vgpr21 = COPY [[UV1]](s32)
+    %0:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    %1:_(s32), %2:_(s32) = G_UNMERGE_VALUES %0
+    $vgpr0 = COPY %1
+    $vgpr21= COPY %2
+...
+
+---
+name: test_unmerge_s16_v2s16
+body: |
+  bb.0:
+    liveins: $vgpr0
+    ; CHECK-LABEL: name: test_unmerge_s16_v2s16
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
+    ; CHECK: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY]](<2 x s16>)
+    ; CHECK: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[UV]](s16)
+    ; CHECK: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[UV1]](s16)
+    ; CHECK: $vgpr0 = COPY [[ANYEXT]](s32)
+    ; CHECK: $vgpr1 = COPY [[ANYEXT1]](s32)
+    %0:_(<2 x s16>) = COPY $vgpr0
+    %1:_(s16), %2:_(s16) = G_UNMERGE_VALUES %0
+    %3:_(s32) = G_ANYEXT %1
+    %4:_(s32) = G_ANYEXT %2
+    $vgpr0 = COPY %3
+    $vgpr1 = COPY %4
+...
+
+---
+name: test_unmerge_s16_v3s16
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: test_unmerge_s16_v3s16
+    ; CHECK: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; CHECK: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
+    ; CHECK: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16), [[UV2:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[EXTRACT]](<3 x s16>)
+    ; CHECK: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[UV]](s16)
+    ; CHECK: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[UV1]](s16)
+    ; CHECK: [[ANYEXT2:%[0-9]+]]:_(s32) = G_ANYEXT [[UV2]](s16)
+    ; CHECK: $vgpr0 = COPY [[ANYEXT]](s32)
+    ; CHECK: $vgpr1 = COPY [[ANYEXT1]](s32)
+    ; CHECK: $vgpr2 = COPY [[ANYEXT2]](s32)
+    %0:_(<3 x s16>) = G_IMPLICIT_DEF
+    %1:_(s16), %2:_(s16),  %3:_(s16) = G_UNMERGE_VALUES %0
+    %4:_(s32) = G_ANYEXT %1
+    %5:_(s32) = G_ANYEXT %2
+    %6:_(s32) = G_ANYEXT %3
+    $vgpr0 = COPY %4
+    $vgpr1 = COPY %5
+    $vgpr2 = COPY %6
+...
+
+---
+name: test_unmerge_s8_s16
+body: |
+  bb.0:
+    liveins: $vgpr0
+    ; CHECK-LABEL: name: test_unmerge_s8_s16
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; CHECK: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
+    ; CHECK: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+    ; CHECK: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND]], [[C1]](s32)
+    ; CHECK: [[OR:%[0-9]+]]:_(s32) = G_OR [[AND]], [[SHL]]
+    ; CHECK: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[OR]](s32)
+    ; CHECK: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[UV]](s16)
+    ; CHECK: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[UV1]](s16)
+    ; CHECK: $vgpr0 = COPY [[ANYEXT]](s32)
+    ; CHECK: $vgpr1 = COPY [[ANYEXT1]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s16) = G_TRUNC %0
+    %2:_(s8), %3:_(s8) = G_UNMERGE_VALUES %1
+    %4:_(s32) = G_ANYEXT %2
+    %5:_(s32) = G_ANYEXT %3
+    $vgpr0 = COPY %4
+    $vgpr1 = COPY %5
+...
+
+---
+name: test_unmerge_s8_s32
+body: |
+  bb.0:
+    liveins: $vgpr0
+    ; CHECK-LABEL: name: test_unmerge_s8_s32
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[COPY]](s32)
+    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
+    ; CHECK: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[C]](s64)
+    ; CHECK: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[ZEXT]], [[TRUNC]](s32)
+    ; CHECK: [[OR:%[0-9]+]]:_(s64) = G_OR [[ZEXT]], [[SHL]]
+    ; CHECK: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
+    ; CHECK: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[C1]](s64)
+    ; CHECK: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[OR]], [[TRUNC1]](s32)
+    ; CHECK: [[OR1:%[0-9]+]]:_(s64) = G_OR [[OR]], [[SHL1]]
+    ; CHECK: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 24
+    ; CHECK: [[TRUNC2:%[0-9]+]]:_(s32) = G_TRUNC [[C2]](s64)
+    ; CHECK: [[SHL2:%[0-9]+]]:_(s64) = G_SHL [[OR1]], [[TRUNC2]](s32)
+    ; CHECK: [[OR2:%[0-9]+]]:_(s64) = G_OR [[OR1]], [[SHL2]]
+    ; CHECK: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16), [[UV2:%[0-9]+]]:_(s16), [[UV3:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[OR2]](s64)
+    ; CHECK: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[UV]](s16)
+    ; CHECK: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[UV1]](s16)
+    ; CHECK: [[ANYEXT2:%[0-9]+]]:_(s32) = G_ANYEXT [[UV2]](s16)
+    ; CHECK: [[ANYEXT3:%[0-9]+]]:_(s32) = G_ANYEXT [[UV3]](s16)
+    ; CHECK: $vgpr0 = COPY [[ANYEXT]](s32)
+    ; CHECK: $vgpr1 = COPY [[ANYEXT1]](s32)
+    ; CHECK: $vgpr2 = COPY [[ANYEXT2]](s32)
+    ; CHECK: $vgpr3 = COPY [[ANYEXT3]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s8), %2:_(s8), %3:_(s8), %4:_(s8) = G_UNMERGE_VALUES %0
+    %5:_(s32) = G_ANYEXT %1
+    %6:_(s32) = G_ANYEXT %2
+    %7:_(s32) = G_ANYEXT %3
+    %8:_(s32) = G_ANYEXT %4
+    $vgpr0 = COPY %5
+    $vgpr1 = COPY %6
+    $vgpr2 = COPY %7
+    $vgpr3 = COPY %8
+...
+
+---
+name: test_unmerge_s16_s32
+body: |
+  bb.0:
+    liveins: $vgpr0
+    ; CHECK-LABEL: name: test_unmerge_s16_s32
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY]](s32)
+    ; CHECK: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[UV]](s16)
+    ; CHECK: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[UV1]](s16)
+    ; CHECK: $vgpr0 = COPY [[ANYEXT]](s32)
+    ; CHECK: $vgpr1 = COPY [[ANYEXT1]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s16), %2:_(s16) = G_UNMERGE_VALUES %0
+    %3:_(s32) = G_ANYEXT %1
+    %4:_(s32) = G_ANYEXT %2
+    $vgpr0 = COPY %3
+    $vgpr1 = COPY %4
+...
+
+---
+name: test_unmerge_s16_s64
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+    ; CHECK-LABEL: name: test_unmerge_s16_s64
+    ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
+    ; CHECK: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16), [[UV2:%[0-9]+]]:_(s16), [[UV3:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY]](s64)
+    ; CHECK: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[UV]](s16)
+    ; CHECK: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[UV1]](s16)
+    ; CHECK: [[ANYEXT2:%[0-9]+]]:_(s32) = G_ANYEXT [[UV2]](s16)
+    ; CHECK: [[ANYEXT3:%[0-9]+]]:_(s32) = G_ANYEXT [[UV3]](s16)
+    ; CHECK: $vgpr0 = COPY [[ANYEXT]](s32)
+    ; CHECK: $vgpr1 = COPY [[ANYEXT1]](s32)
+    ; CHECK: $vgpr2 = COPY [[ANYEXT2]](s32)
+    ; CHECK: $vgpr3 = COPY [[ANYEXT3]](s32)
+    %0:_(s64) = COPY $vgpr0_vgpr1
+    %1:_(s16), %2:_(s16), %3:_(s16), %4:_(s16) = G_UNMERGE_VALUES %0
+    %5:_(s32) = G_ANYEXT %1
+    %6:_(s32) = G_ANYEXT %2
+    %7:_(s32) = G_ANYEXT %3
+    %8:_(s32) = G_ANYEXT %4
+    $vgpr0 = COPY %5
+    $vgpr1 = COPY %6
+    $vgpr2 = COPY %7
+    $vgpr3 = COPY %8
+...
+
+---
+name: test_unmerge_s1_s3
+body: |
+  bb.0:
+    liveins: $vgpr0
+    ; CHECK-LABEL: name: test_unmerge_s1_s3
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[TRUNC:%[0-9]+]]:_(s3) = G_TRUNC [[COPY]](s32)
+    ; CHECK: [[ZEXT:%[0-9]+]]:_(s48) = G_ZEXT [[TRUNC]](s3)
+    ; CHECK: [[C:%[0-9]+]]:_(s48) = G_CONSTANT i48 15
+    ; CHECK: [[SHL:%[0-9]+]]:_(s48) = G_SHL [[ZEXT]], [[C]](s48)
+    ; CHECK: [[OR:%[0-9]+]]:_(s48) = G_OR [[ZEXT]], [[SHL]]
+    ; CHECK: [[C1:%[0-9]+]]:_(s48) = G_CONSTANT i48 30
+    ; CHECK: [[SHL1:%[0-9]+]]:_(s48) = G_SHL [[OR]], [[C1]](s48)
+    ; CHECK: [[OR1:%[0-9]+]]:_(s48) = G_OR [[OR]], [[SHL1]]
+    ; CHECK: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16), [[UV2:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[OR1]](s48)
+    ; CHECK: [[TRUNC1:%[0-9]+]]:_(s1) = G_TRUNC [[UV]](s16)
+    ; CHECK: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[TRUNC1]](s1)
+    ; CHECK: [[TRUNC2:%[0-9]+]]:_(s1) = G_TRUNC [[UV1]](s16)
+    ; CHECK: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[TRUNC2]](s1)
+    ; CHECK: [[TRUNC3:%[0-9]+]]:_(s1) = G_TRUNC [[UV2]](s16)
+    ; CHECK: [[ANYEXT2:%[0-9]+]]:_(s32) = G_ANYEXT [[TRUNC3]](s1)
+    ; CHECK: $vgpr0 = COPY [[ANYEXT]](s32)
+    ; CHECK: $vgpr1 = COPY [[ANYEXT1]](s32)
+    ; CHECK: $vgpr2 = COPY [[ANYEXT2]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s3) = G_TRUNC %0
+    %2:_(s1), %3:_(s1), %4:_(s1) = G_UNMERGE_VALUES %1
+    %5:_(s32) = G_ANYEXT %2
+    %6:_(s32) = G_ANYEXT %3
+    %7:_(s32) = G_ANYEXT %4
+    $vgpr0 = COPY %5
+    $vgpr1 = COPY %6
+    $vgpr2 = COPY %7
+...
+
+---
+name: test_unmerge_s1_s8
+body: |
+  bb.0:
+    liveins: $vgpr0
+    ; CHECK-LABEL: name: test_unmerge_s1_s8
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 255
+    ; CHECK: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; CHECK: [[ANYEXT:%[0-9]+]]:_(s128) = G_ANYEXT [[COPY]](s32)
+    ; CHECK: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[ANYEXT]](s128)
+    ; CHECK: [[AND:%[0-9]+]]:_(s64) = G_AND [[UV]], [[C]]
+    ; CHECK: [[AND1:%[0-9]+]]:_(s64) = G_AND [[UV1]], [[C1]]
+    ; CHECK: [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[AND]](s64), [[AND1]](s64)
+    ; CHECK: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 15
+    ; CHECK: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; CHECK: [[MV1:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[C2]](s64), [[C3]](s64)
+    ; CHECK: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[MV1]](s128)
+    ; CHECK: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 64
+    ; CHECK: [[UV2:%[0-9]+]]:_(s64), [[UV3:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[MV]](s128)
+    ; CHECK: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[TRUNC]], [[C4]]
+    ; CHECK: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[C4]], [[TRUNC]]
+    ; CHECK: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; CHECK: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[TRUNC]](s32), [[C4]]
+    ; CHECK: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[TRUNC]](s32), [[C5]]
+    ; CHECK: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[UV3]], [[TRUNC]](s32)
+    ; CHECK: [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[UV3]], [[TRUNC]](s32)
+    ; CHECK: [[LSHR:%[0-9]+]]:_(s64) = G_LSHR [[UV2]], [[SUB1]](s32)
+    ; CHECK: [[OR:%[0-9]+]]:_(s64) = G_OR [[SHL1]], [[LSHR]]
+    ; CHECK: [[C6:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; CHECK: [[SHL2:%[0-9]+]]:_(s64) = G_SHL [[UV2]], [[SUB]](s32)
+    ; CHECK: [[SELECT:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[SHL]], [[C6]]
+    ; CHECK: [[SELECT1:%[0-9]+]]:_(s64) = G_SELECT [[ICMP]](s1), [[OR]], [[SHL2]]
+    ; CHECK: [[SELECT2:%[0-9]+]]:_(s64) = G_SELECT [[ICMP1]](s1), [[UV3]], [[SELECT1]]
+    ; CHECK: [[UV4:%[0-9]+]]:_(s64), [[UV5:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[MV]](s128)
+    ; CHECK: [[OR1:%[0-9]+]]:_(s64) = G_OR [[UV4]], [[SELECT]]
+    ; CHECK: [[OR2:%[0-9]+]]:_(s64) = G_OR [[UV5]], [[SELECT2]]
+    ; CHECK: [[C7:%[0-9]+]]:_(s64) = G_CONSTANT i64 30
+    ; CHECK: [[C8:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; CHECK: [[MV2:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[C7]](s64), [[C8]](s64)
+    ; CHECK: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[MV2]](s128)
+    ; CHECK: [[C9:%[0-9]+]]:_(s32) = G_CONSTANT i32 64
+    ; CHECK: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[TRUNC1]], [[C9]]
+    ; CHECK: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[C9]], [[TRUNC1]]
+    ; CHECK: [[C10:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; CHECK: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[TRUNC1]](s32), [[C9]]
+    ; CHECK: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[TRUNC1]](s32), [[C10]]
+    ; CHECK: [[SHL3:%[0-9]+]]:_(s64) = G_SHL [[OR2]], [[TRUNC1]](s32)
+    ; CHECK: [[SHL4:%[0-9]+]]:_(s64) = G_SHL [[OR2]], [[TRUNC1]](s32)
+    ; CHECK: [[LSHR1:%[0-9]+]]:_(s64) = G_LSHR [[OR1]], [[SUB3]](s32)
+    ; CHECK: [[OR3:%[0-9]+]]:_(s64) = G_OR [[SHL4]], [[LSHR1]]
+    ; CHECK: [[C11:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; CHECK: [[SHL5:%[0-9]+]]:_(s64) = G_SHL [[OR1]], [[SUB2]](s32)
+    ; CHECK: [[SELECT3:%[0-9]+]]:_(s64) = G_SELECT [[ICMP2]](s1), [[SHL3]], [[C11]]
+    ; CHECK: [[SELECT4:%[0-9]+]]:_(s64) = G_SELECT [[ICMP2]](s1), [[OR3]], [[SHL5]]
+    ; CHECK: [[SELECT5:%[0-9]+]]:_(s64) = G_SELECT [[ICMP3]](s1), [[OR2]], [[SELECT4]]
+    ; CHECK: [[OR4:%[0-9]+]]:_(s64) = G_OR [[OR1]], [[SELECT3]]
+    ; CHECK: [[OR5:%[0-9]+]]:_(s64) = G_OR [[OR2]], [[SELECT5]]
+    ; CHECK: [[C12:%[0-9]+]]:_(s64) = G_CONSTANT i64 45
+    ; CHECK: [[C13:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; CHECK: [[MV3:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[C12]](s64), [[C13]](s64)
+    ; CHECK: [[TRUNC2:%[0-9]+]]:_(s32) = G_TRUNC [[MV3]](s128)
+    ; CHECK: [[C14:%[0-9]+]]:_(s32) = G_CONSTANT i32 64
+    ; CHECK: [[SUB4:%[0-9]+]]:_(s32) = G_SUB [[TRUNC2]], [[C14]]
+    ; CHECK: [[SUB5:%[0-9]+]]:_(s32) = G_SUB [[C14]], [[TRUNC2]]
+    ; CHECK: [[C15:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; CHECK: [[ICMP4:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[TRUNC2]](s32), [[C14]]
+    ; CHECK: [[ICMP5:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[TRUNC2]](s32), [[C15]]
+    ; CHECK: [[SHL6:%[0-9]+]]:_(s64) = G_SHL [[OR5]], [[TRUNC2]](s32)
+    ; CHECK: [[SHL7:%[0-9]+]]:_(s64) = G_SHL [[OR5]], [[TRUNC2]](s32)
+    ; CHECK: [[LSHR2:%[0-9]+]]:_(s64) = G_LSHR [[OR4]], [[SUB5]](s32)
+    ; CHECK: [[OR6:%[0-9]+]]:_(s64) = G_OR [[SHL7]], [[LSHR2]]
+    ; CHECK: [[C16:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; CHECK: [[SHL8:%[0-9]+]]:_(s64) = G_SHL [[OR4]], [[SUB4]](s32)
+    ; CHECK: [[SELECT6:%[0-9]+]]:_(s64) = G_SELECT [[ICMP4]](s1), [[SHL6]], [[C16]]
+    ; CHECK: [[SELECT7:%[0-9]+]]:_(s64) = G_SELECT [[ICMP4]](s1), [[OR6]], [[SHL8]]
+    ; CHECK: [[SELECT8:%[0-9]+]]:_(s64) = G_SELECT [[ICMP5]](s1), [[OR5]], [[SELECT7]]
+    ; CHECK: [[OR7:%[0-9]+]]:_(s64) = G_OR [[OR4]], [[SELECT6]]
+    ; CHECK: [[OR8:%[0-9]+]]:_(s64) = G_OR [[OR5]], [[SELECT8]]
+    ; CHECK: [[C17:%[0-9]+]]:_(s64) = G_CONSTANT i64 60
+    ; CHECK: [[C18:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; CHECK: [[MV4:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[C17]](s64), [[C18]](s64)
+    ; CHECK: [[TRUNC3:%[0-9]+]]:_(s32) = G_TRUNC [[MV4]](s128)
+    ; CHECK: [[C19:%[0-9]+]]:_(s32) = G_CONSTANT i32 64
+    ; CHECK: [[SUB6:%[0-9]+]]:_(s32) = G_SUB [[TRUNC3]], [[C19]]
+    ; CHECK: [[SUB7:%[0-9]+]]:_(s32) = G_SUB [[C19]], [[TRUNC3]]
+    ; CHECK: [[C20:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; CHECK: [[ICMP6:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[TRUNC3]](s32), [[C19]]
+    ; CHECK: [[ICMP7:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[TRUNC3]](s32), [[C20]]
+    ; CHECK: [[SHL9:%[0-9]+]]:_(s64) = G_SHL [[OR8]], [[TRUNC3]](s32)
+    ; CHECK: [[SHL10:%[0-9]+]]:_(s64) = G_SHL [[OR8]], [[TRUNC3]](s32)
+    ; CHECK: [[LSHR3:%[0-9]+]]:_(s64) = G_LSHR [[OR7]], [[SUB7]](s32)
+    ; CHECK: [[OR9:%[0-9]+]]:_(s64) = G_OR [[SHL10]], [[LSHR3]]
+    ; CHECK: [[C21:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; CHECK: [[SHL11:%[0-9]+]]:_(s64) = G_SHL [[OR7]], [[SUB6]](s32)
+    ; CHECK: [[SELECT9:%[0-9]+]]:_(s64) = G_SELECT [[ICMP6]](s1), [[SHL9]], [[C21]]
+    ; CHECK: [[SELECT10:%[0-9]+]]:_(s64) = G_SELECT [[ICMP6]](s1), [[OR9]], [[SHL11]]
+    ; CHECK: [[SELECT11:%[0-9]+]]:_(s64) = G_SELECT [[ICMP7]](s1), [[OR8]], [[SELECT10]]
+    ; CHECK: [[OR10:%[0-9]+]]:_(s64) = G_OR [[OR7]], [[SELECT9]]
+    ; CHECK: [[OR11:%[0-9]+]]:_(s64) = G_OR [[OR8]], [[SELECT11]]
+    ; CHECK: [[C22:%[0-9]+]]:_(s64) = G_CONSTANT i64 75
+    ; CHECK: [[C23:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; CHECK: [[MV5:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[C22]](s64), [[C23]](s64)
+    ; CHECK: [[TRUNC4:%[0-9]+]]:_(s32) = G_TRUNC [[MV5]](s128)
+    ; CHECK: [[C24:%[0-9]+]]:_(s32) = G_CONSTANT i32 64
+    ; CHECK: [[SUB8:%[0-9]+]]:_(s32) = G_SUB [[TRUNC4]], [[C24]]
+    ; CHECK: [[SUB9:%[0-9]+]]:_(s32) = G_SUB [[C24]], [[TRUNC4]]
+    ; CHECK: [[C25:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; CHECK: [[ICMP8:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[TRUNC4]](s32), [[C24]]
+    ; CHECK: [[ICMP9:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[TRUNC4]](s32), [[C25]]
+    ; CHECK: [[SHL12:%[0-9]+]]:_(s64) = G_SHL [[OR11]], [[TRUNC4]](s32)
+    ; CHECK: [[SHL13:%[0-9]+]]:_(s64) = G_SHL [[OR11]], [[TRUNC4]](s32)
+    ; CHECK: [[LSHR4:%[0-9]+]]:_(s64) = G_LSHR [[OR10]], [[SUB9]](s32)
+    ; CHECK: [[OR12:%[0-9]+]]:_(s64) = G_OR [[SHL13]], [[LSHR4]]
+    ; CHECK: [[C26:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; CHECK: [[SHL14:%[0-9]+]]:_(s64) = G_SHL [[OR10]], [[SUB8]](s32)
+    ; CHECK: [[SELECT12:%[0-9]+]]:_(s64) = G_SELECT [[ICMP8]](s1), [[SHL12]], [[C26]]
+    ; CHECK: [[SELECT13:%[0-9]+]]:_(s64) = G_SELECT [[ICMP8]](s1), [[OR12]], [[SHL14]]
+    ; CHECK: [[SELECT14:%[0-9]+]]:_(s64) = G_SELECT [[ICMP9]](s1), [[OR11]], [[SELECT13]]
+    ; CHECK: [[OR13:%[0-9]+]]:_(s64) = G_OR [[OR10]], [[SELECT12]]
+    ; CHECK: [[OR14:%[0-9]+]]:_(s64) = G_OR [[OR11]], [[SELECT14]]
+    ; CHECK: [[C27:%[0-9]+]]:_(s64) = G_CONSTANT i64 90
+    ; CHECK: [[C28:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; CHECK: [[MV6:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[C27]](s64), [[C28]](s64)
+    ; CHECK: [[TRUNC5:%[0-9]+]]:_(s32) = G_TRUNC [[MV6]](s128)
+    ; CHECK: [[C29:%[0-9]+]]:_(s32) = G_CONSTANT i32 64
+    ; CHECK: [[SUB10:%[0-9]+]]:_(s32) = G_SUB [[TRUNC5]], [[C29]]
+    ; CHECK: [[SUB11:%[0-9]+]]:_(s32) = G_SUB [[C29]], [[TRUNC5]]
+    ; CHECK: [[C30:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; CHECK: [[ICMP10:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[TRUNC5]](s32), [[C29]]
+    ; CHECK: [[ICMP11:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[TRUNC5]](s32), [[C30]]
+    ; CHECK: [[SHL15:%[0-9]+]]:_(s64) = G_SHL [[OR14]], [[TRUNC5]](s32)
+    ; CHECK: [[SHL16:%[0-9]+]]:_(s64) = G_SHL [[OR14]], [[TRUNC5]](s32)
+    ; CHECK: [[LSHR5:%[0-9]+]]:_(s64) = G_LSHR [[OR13]], [[SUB11]](s32)
+    ; CHECK: [[OR15:%[0-9]+]]:_(s64) = G_OR [[SHL16]], [[LSHR5]]
+    ; CHECK: [[C31:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; CHECK: [[SHL17:%[0-9]+]]:_(s64) = G_SHL [[OR13]], [[SUB10]](s32)
+    ; CHECK: [[SELECT15:%[0-9]+]]:_(s64) = G_SELECT [[ICMP10]](s1), [[SHL15]], [[C31]]
+    ; CHECK: [[SELECT16:%[0-9]+]]:_(s64) = G_SELECT [[ICMP10]](s1), [[OR15]], [[SHL17]]
+    ; CHECK: [[SELECT17:%[0-9]+]]:_(s64) = G_SELECT [[ICMP11]](s1), [[OR14]], [[SELECT16]]
+    ; CHECK: [[OR16:%[0-9]+]]:_(s64) = G_OR [[OR13]], [[SELECT15]]
+    ; CHECK: [[OR17:%[0-9]+]]:_(s64) = G_OR [[OR14]], [[SELECT17]]
+    ; CHECK: [[C32:%[0-9]+]]:_(s64) = G_CONSTANT i64 105
+    ; CHECK: [[C33:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; CHECK: [[MV7:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[C32]](s64), [[C33]](s64)
+    ; CHECK: [[TRUNC6:%[0-9]+]]:_(s32) = G_TRUNC [[MV7]](s128)
+    ; CHECK: [[C34:%[0-9]+]]:_(s32) = G_CONSTANT i32 64
+    ; CHECK: [[SUB12:%[0-9]+]]:_(s32) = G_SUB [[TRUNC6]], [[C34]]
+    ; CHECK: [[SUB13:%[0-9]+]]:_(s32) = G_SUB [[C34]], [[TRUNC6]]
+    ; CHECK: [[C35:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; CHECK: [[ICMP12:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[TRUNC6]](s32), [[C34]]
+    ; CHECK: [[ICMP13:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[TRUNC6]](s32), [[C35]]
+    ; CHECK: [[SHL18:%[0-9]+]]:_(s64) = G_SHL [[OR17]], [[TRUNC6]](s32)
+    ; CHECK: [[SHL19:%[0-9]+]]:_(s64) = G_SHL [[OR17]], [[TRUNC6]](s32)
+    ; CHECK: [[LSHR6:%[0-9]+]]:_(s64) = G_LSHR [[OR16]], [[SUB13]](s32)
+    ; CHECK: [[OR18:%[0-9]+]]:_(s64) = G_OR [[SHL19]], [[LSHR6]]
+    ; CHECK: [[C36:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; CHECK: [[SHL20:%[0-9]+]]:_(s64) = G_SHL [[OR16]], [[SUB12]](s32)
+    ; CHECK: [[SELECT18:%[0-9]+]]:_(s64) = G_SELECT [[ICMP12]](s1), [[SHL18]], [[C36]]
+    ; CHECK: [[SELECT19:%[0-9]+]]:_(s64) = G_SELECT [[ICMP12]](s1), [[OR18]], [[SHL20]]
+    ; CHECK: [[SELECT20:%[0-9]+]]:_(s64) = G_SELECT [[ICMP13]](s1), [[OR17]], [[SELECT19]]
+    ; CHECK: [[OR19:%[0-9]+]]:_(s64) = G_OR [[OR16]], [[SELECT18]]
+    ; CHECK: [[OR20:%[0-9]+]]:_(s64) = G_OR [[OR17]], [[SELECT20]]
+    ; CHECK: [[UV6:%[0-9]+]]:_(s16), [[UV7:%[0-9]+]]:_(s16), [[UV8:%[0-9]+]]:_(s16), [[UV9:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[OR19]](s64)
+    ; CHECK: [[UV10:%[0-9]+]]:_(s16), [[UV11:%[0-9]+]]:_(s16), [[UV12:%[0-9]+]]:_(s16), [[UV13:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[OR20]](s64)
+    ; CHECK: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[UV6]](s16)
+    ; CHECK: [[ANYEXT2:%[0-9]+]]:_(s32) = G_ANYEXT [[UV7]](s16)
+    ; CHECK: [[ANYEXT3:%[0-9]+]]:_(s32) = G_ANYEXT [[UV8]](s16)
+    ; CHECK: [[ANYEXT4:%[0-9]+]]:_(s32) = G_ANYEXT [[UV9]](s16)
+    ; CHECK: [[ANYEXT5:%[0-9]+]]:_(s32) = G_ANYEXT [[UV10]](s16)
+    ; CHECK: [[ANYEXT6:%[0-9]+]]:_(s32) = G_ANYEXT [[UV11]](s16)
+    ; CHECK: [[ANYEXT7:%[0-9]+]]:_(s32) = G_ANYEXT [[UV12]](s16)
+    ; CHECK: [[ANYEXT8:%[0-9]+]]:_(s32) = G_ANYEXT [[UV13]](s16)
+    ; CHECK: $vgpr0 = COPY [[ANYEXT1]](s32)
+    ; CHECK: $vgpr1 = COPY [[ANYEXT2]](s32)
+    ; CHECK: $vgpr2 = COPY [[ANYEXT3]](s32)
+    ; CHECK: $vgpr3 = COPY [[ANYEXT4]](s32)
+    ; CHECK: $vgpr4 = COPY [[ANYEXT5]](s32)
+    ; CHECK: $vgpr5 = COPY [[ANYEXT6]](s32)
+    ; CHECK: $vgpr6 = COPY [[ANYEXT7]](s32)
+    ; CHECK: $vgpr7 = COPY [[ANYEXT8]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s8) = G_TRUNC %0
+    %2:_(s1), %3:_(s1), %4:_(s1), %5:_(s1), %6:_(s1), %7:_(s1), %8:_(s1), %9:_(s1) = G_UNMERGE_VALUES %1
+    %10:_(s32) = G_ANYEXT %2
+    %11:_(s32) = G_ANYEXT %3
+    %12:_(s32) = G_ANYEXT %4
+    %13:_(s32) = G_ANYEXT %5
+    %14:_(s32) = G_ANYEXT %6
+    %15:_(s32) = G_ANYEXT %7
+    %16:_(s32) = G_ANYEXT %8
+    %17:_(s32) = G_ANYEXT %9
+    $vgpr0 = COPY %10
+    $vgpr1 = COPY %11
+    $vgpr2 = COPY %12
+    $vgpr3 = COPY %13
+    $vgpr4 = COPY %14
+    $vgpr5 = COPY %15
+    $vgpr6 = COPY %16
+    $vgpr7 = COPY %17
+...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/legalize-usubo.mir b/test/CodeGen/AMDGPU/GlobalISel/legalize-usubo.mir
new file mode 100644
index 0000000000000000000000000000000000000000..9764231bcf56868a4891610c5d8217a6f1414b71
--- /dev/null
+++ b/test/CodeGen/AMDGPU/GlobalISel/legalize-usubo.mir
@@ -0,0 +1,97 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -O0 -run-pass=legalizer %s -o - | FileCheck %s
+
+---
+name: test_usubo_s32
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+
+    ; CHECK-LABEL: name: test_usubo_s32
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; CHECK: [[USUBO:%[0-9]+]]:_(s32), [[USUBO1:%[0-9]+]]:_(s1) = G_USUBO [[COPY]], [[COPY1]]
+    ; CHECK: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[USUBO1]](s1)
+    ; CHECK: $vgpr0 = COPY [[USUBO]](s32)
+    ; CHECK: $vgpr1 = COPY [[ZEXT]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s32), %3:_(s1) = G_USUBO %0, %1
+    %4:_(s32) = G_ZEXT %3
+    $vgpr0 = COPY %2
+    $vgpr1 = COPY %4
+...
+
+---
+name: test_usubo_s7
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+
+    ; CHECK-LABEL: name: test_usubo_s7
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 127
+    ; CHECK: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; CHECK: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C]]
+    ; CHECK: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 127
+    ; CHECK: [[COPY3:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+    ; CHECK: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[C1]]
+    ; CHECK: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[AND]], [[AND1]]
+    ; CHECK: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 127
+    ; CHECK: [[AND2:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C2]]
+    ; CHECK: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SUB]](s32), [[AND2]]
+    ; CHECK: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 127
+    ; CHECK: [[COPY4:%[0-9]+]]:_(s32) = COPY [[SUB]](s32)
+    ; CHECK: [[AND3:%[0-9]+]]:_(s32) = G_AND [[COPY4]], [[C3]]
+    ; CHECK: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[ICMP]](s1)
+    ; CHECK: $vgpr0 = COPY [[AND3]](s32)
+    ; CHECK: $vgpr1 = COPY [[ZEXT]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s7) = G_TRUNC %0
+    %3:_(s7) = G_TRUNC %1
+
+    %4:_(s7), %5:_(s1) = G_USUBO %2, %3
+    %6:_(s32) = G_ZEXT %4
+    %7:_(s32) = G_ZEXT %5
+    $vgpr0 = COPY %6
+    $vgpr1 = COPY %7
+...
+
+---
+name: test_usubo_s16
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+
+    ; CHECK-LABEL: name: test_usubo_s16
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; CHECK: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; CHECK: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C]]
+    ; CHECK: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; CHECK: [[COPY3:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+    ; CHECK: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[C1]]
+    ; CHECK: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[AND]], [[AND1]]
+    ; CHECK: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; CHECK: [[AND2:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C2]]
+    ; CHECK: [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SUB]](s32), [[AND2]]
+    ; CHECK: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; CHECK: [[COPY4:%[0-9]+]]:_(s32) = COPY [[SUB]](s32)
+    ; CHECK: [[AND3:%[0-9]+]]:_(s32) = G_AND [[COPY4]], [[C3]]
+    ; CHECK: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[ICMP]](s1)
+    ; CHECK: $vgpr0 = COPY [[AND3]](s32)
+    ; CHECK: $vgpr1 = COPY [[ZEXT]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s16) = G_TRUNC %0
+    %3:_(s16) = G_TRUNC %1
+
+    %4:_(s16), %5:_(s1) = G_USUBO %2, %3
+    %6:_(s32) = G_ZEXT %4
+    %7:_(s32) = G_ZEXT %5
+    $vgpr0 = COPY %6
+    $vgpr1 = COPY %7
+...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/legalize-xor.mir b/test/CodeGen/AMDGPU/GlobalISel/legalize-xor.mir
index e5f44004b648a4349c47fe6eaa1502c31eb656a3..96501b6133bb0427524591359b6afc07b38fca32 100644
--- a/test/CodeGen/AMDGPU/GlobalISel/legalize-xor.mir
+++ b/test/CodeGen/AMDGPU/GlobalISel/legalize-xor.mir
@@ -2,17 +2,255 @@
 # RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -O0 -run-pass=legalizer %s -o - | FileCheck %s
 
 ---
-name: test_xor
+name: test_xor_s32
 body: |
   bb.0:
     liveins: $vgpr0, $vgpr1
 
-    ; CHECK-LABEL: name: test_xor
+    ; CHECK-LABEL: name: test_xor_s32
     ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
     ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
     ; CHECK: [[XOR:%[0-9]+]]:_(s32) = G_XOR [[COPY]], [[COPY1]]
+    ; CHECK: $vgpr0 = COPY [[XOR]](s32)
     %0:_(s32) = COPY $vgpr0
     %1:_(s32) = COPY $vgpr1
     %2:_(s32) = G_XOR %0, %1
     $vgpr0 = COPY %2
 ...
+
+---
+name: test_xor_s1
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+
+    ; CHECK-LABEL: name: test_xor_s1
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; CHECK: [[XOR:%[0-9]+]]:_(s32) = G_XOR [[COPY]], [[COPY1]]
+    ; CHECK: S_NOP 0, implicit [[XOR]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s32) = G_CONSTANT i32 0
+    %3:_(s1) = G_ICMP intpred(ne), %0, %2
+    %4:_(s1) = G_ICMP intpred(ne), %1, %2
+    %5:_(s32) = G_XOR %0, %1
+    S_NOP 0, implicit %5
+...
+
+---
+name: test_xor_s64
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
+
+    ; CHECK-LABEL: name: test_xor_s64
+    ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s64) = COPY $vgpr2_vgpr3
+    ; CHECK: [[XOR:%[0-9]+]]:_(s64) = G_XOR [[COPY]], [[COPY1]]
+    ; CHECK: $vgpr0_vgpr1 = COPY [[XOR]](s64)
+    %0:_(s64) = COPY $vgpr0_vgpr1
+    %1:_(s64) = COPY $vgpr2_vgpr3
+    %2:_(s64) = G_XOR %0, %1
+    $vgpr0_vgpr1 = COPY %2
+...
+
+---
+name: test_xor_s7
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+
+    ; CHECK-LABEL: name: test_xor_s7
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; CHECK: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; CHECK: [[COPY3:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+    ; CHECK: [[XOR:%[0-9]+]]:_(s32) = G_XOR [[COPY2]], [[COPY3]]
+    ; CHECK: [[COPY4:%[0-9]+]]:_(s32) = COPY [[XOR]](s32)
+    ; CHECK: $vgpr0 = COPY [[COPY4]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s7) = G_TRUNC %0
+    %3:_(s7) = G_TRUNC %1
+    %4:_(s7) = G_XOR %2, %3
+    %5:_(s32) = G_ANYEXT %4
+    $vgpr0 = COPY %5
+...
+
+---
+name: test_xor_s8
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+
+    ; CHECK-LABEL: name: test_xor_s8
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; CHECK: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; CHECK: [[COPY3:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+    ; CHECK: [[XOR:%[0-9]+]]:_(s32) = G_XOR [[COPY2]], [[COPY3]]
+    ; CHECK: [[COPY4:%[0-9]+]]:_(s32) = COPY [[XOR]](s32)
+    ; CHECK: $vgpr0 = COPY [[COPY4]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s8) = G_TRUNC %0
+    %3:_(s8) = G_TRUNC %1
+    %4:_(s8) = G_XOR %2, %3
+    %5:_(s32) = G_ANYEXT %4
+    $vgpr0 = COPY %5
+...
+
+---
+name: test_xor_s16
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+
+    ; CHECK-LABEL: name: test_xor_s16
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; CHECK: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; CHECK: [[COPY3:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+    ; CHECK: [[XOR:%[0-9]+]]:_(s32) = G_XOR [[COPY2]], [[COPY3]]
+    ; CHECK: [[COPY4:%[0-9]+]]:_(s32) = COPY [[XOR]](s32)
+    ; CHECK: $vgpr0 = COPY [[COPY4]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s16) = G_TRUNC %0
+    %3:_(s16) = G_TRUNC %1
+    %4:_(s16) = G_XOR %2, %3
+    %5:_(s32) = G_ANYEXT %4
+    $vgpr0 = COPY %5
+...
+
+---
+name: test_xor_s24
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+
+    ; CHECK-LABEL: name: test_xor_s24
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; CHECK: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; CHECK: [[COPY3:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+    ; CHECK: [[XOR:%[0-9]+]]:_(s32) = G_XOR [[COPY2]], [[COPY3]]
+    ; CHECK: [[COPY4:%[0-9]+]]:_(s32) = COPY [[XOR]](s32)
+    ; CHECK: $vgpr0 = COPY [[COPY4]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s16) = G_TRUNC %0
+    %3:_(s16) = G_TRUNC %1
+    %4:_(s16) = G_XOR %2, %3
+    %5:_(s32) = G_ANYEXT %4
+    $vgpr0 = COPY %5
+...
+
+---
+name: test_xor_v2s32
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
+
+    ; CHECK-LABEL: name: test_xor_v2s32
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    ; CHECK: [[COPY1:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr2_vgpr3
+    ; CHECK: [[XOR:%[0-9]+]]:_(<2 x s32>) = G_XOR [[COPY]], [[COPY1]]
+    ; CHECK: $vgpr0_vgpr1 = COPY [[XOR]](<2 x s32>)
+    %0:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    %1:_(<2 x s32>) = COPY $vgpr2_vgpr3
+    %2:_(<2 x s32>) = G_XOR %0, %1
+    $vgpr0_vgpr1 = COPY %2
+...
+
+---
+name: test_xor_v2s64
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4_vgpr5_vgpr6_vgpr7
+
+    ; CHECK-LABEL: name: test_xor_v2s64
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; CHECK: [[COPY1:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr4_vgpr5_vgpr6_vgpr7
+    ; CHECK: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](<2 x s64>)
+    ; CHECK: [[UV2:%[0-9]+]]:_(s64), [[UV3:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY1]](<2 x s64>)
+    ; CHECK: [[XOR:%[0-9]+]]:_(s64) = G_XOR [[UV]], [[UV2]]
+    ; CHECK: [[XOR1:%[0-9]+]]:_(s64) = G_XOR [[UV1]], [[UV3]]
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[XOR]](s64), [[XOR1]](s64)
+    ; CHECK: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>)
+    %0:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    %1:_(<2 x s64>) = COPY $vgpr4_vgpr5_vgpr6_vgpr7
+    %2:_(<2 x s64>) = G_XOR %0, %1
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %2
+...
+
+---
+name: test_xor_v2s16
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+
+    ; CHECK-LABEL: name: test_xor_v2s16
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
+    ; CHECK: [[XOR:%[0-9]+]]:_(<2 x s16>) = G_XOR [[COPY]], [[COPY1]]
+    ; CHECK: $vgpr0 = COPY [[XOR]](<2 x s16>)
+    %0:_(<2 x s16>) = COPY $vgpr0
+    %1:_(<2 x s16>) = COPY $vgpr1
+    %2:_(<2 x s16>) = G_XOR %0, %1
+    $vgpr0 = COPY %2
+...
+
+---
+name: test_xor_v3s16
+body: |
+  bb.0:
+
+    ; CHECK-LABEL: name: test_xor_v3s16
+    ; CHECK: [[DEF:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; CHECK: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF]](<4 x s16>), 0
+    ; CHECK: [[DEF1:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; CHECK: [[EXTRACT1:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[DEF1]](<4 x s16>), 0
+    ; CHECK: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16), [[UV2:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[EXTRACT]](<3 x s16>)
+    ; CHECK: [[UV3:%[0-9]+]]:_(s16), [[UV4:%[0-9]+]]:_(s16), [[UV5:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[EXTRACT1]](<3 x s16>)
+    ; CHECK: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[UV]](s16)
+    ; CHECK: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[UV3]](s16)
+    ; CHECK: [[XOR:%[0-9]+]]:_(s32) = G_XOR [[ANYEXT]], [[ANYEXT1]]
+    ; CHECK: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[XOR]](s32)
+    ; CHECK: [[ANYEXT2:%[0-9]+]]:_(s32) = G_ANYEXT [[UV1]](s16)
+    ; CHECK: [[ANYEXT3:%[0-9]+]]:_(s32) = G_ANYEXT [[UV4]](s16)
+    ; CHECK: [[XOR1:%[0-9]+]]:_(s32) = G_XOR [[ANYEXT2]], [[ANYEXT3]]
+    ; CHECK: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[XOR1]](s32)
+    ; CHECK: [[ANYEXT4:%[0-9]+]]:_(s32) = G_ANYEXT [[UV2]](s16)
+    ; CHECK: [[ANYEXT5:%[0-9]+]]:_(s32) = G_ANYEXT [[UV5]](s16)
+    ; CHECK: [[XOR2:%[0-9]+]]:_(s32) = G_XOR [[ANYEXT4]], [[ANYEXT5]]
+    ; CHECK: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[XOR2]](s32)
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16), [[TRUNC2]](s16)
+    ; CHECK: [[DEF2:%[0-9]+]]:_(<4 x s16>) = G_IMPLICIT_DEF
+    ; CHECK: [[INSERT:%[0-9]+]]:_(<4 x s16>) = G_INSERT [[DEF2]], [[BUILD_VECTOR]](<3 x s16>), 0
+    ; CHECK: $vgpr0_vgpr1 = COPY [[INSERT]](<4 x s16>)
+    %0:_(<3 x s16>) = G_IMPLICIT_DEF
+    %1:_(<3 x s16>) = G_IMPLICIT_DEF
+    %2:_(<3 x s16>) = G_XOR %0, %1
+    %4:_(<4 x s16>) = G_IMPLICIT_DEF
+    %5:_(<4 x s16>) = G_INSERT %4, %2, 0
+    $vgpr0_vgpr1 = COPY %5
+...
+
+---
+name: test_xor_v4s16
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
+
+    ; CHECK-LABEL: name: test_xor_v4s16
+    ; CHECK: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
+    ; CHECK: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
+    ; CHECK: [[XOR:%[0-9]+]]:_(<4 x s16>) = G_XOR [[COPY]], [[COPY1]]
+    ; CHECK: $vgpr0_vgpr1 = COPY [[XOR]](<4 x s16>)
+    %0:_(<4 x s16>) = COPY $vgpr0_vgpr1
+    %1:_(<4 x s16>) = COPY $vgpr2_vgpr3
+    %2:_(<4 x s16>) = G_XOR %0, %1
+    $vgpr0_vgpr1 = COPY %2
+...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/legalize-zext.mir b/test/CodeGen/AMDGPU/GlobalISel/legalize-zext.mir
index a8567fb52bc6a3636081724d14227900d1db7cfd..f4f2bf3eaab4af0d09746b10434343390cb12026 100644
--- a/test/CodeGen/AMDGPU/GlobalISel/legalize-zext.mir
+++ b/test/CodeGen/AMDGPU/GlobalISel/legalize-zext.mir
@@ -2,13 +2,197 @@
 # RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -run-pass=legalizer %s -o - | FileCheck %s
 
 ---
-name: test_zext_i32_to_i64
+name: test_zext_s32_to_s64
 body: |
-  bb.0.entry:
+  bb.0:
     liveins: $vgpr0
 
-    ; CHECK-LABEL: name: test_zext_i32_to_i64
+    ; CHECK-LABEL: name: test_zext_s32_to_s64
     ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[COPY]](s32)
+    ; CHECK: $vgpr0_vgpr1 = COPY [[ZEXT]](s64)
     %0:_(s32) = COPY $vgpr0
     %1:_(s64) = G_ZEXT %0
+    $vgpr0_vgpr1 = COPY %1
+...
+
+---
+name: test_zext_s16_to_s64
+body: |
+  bb.0:
+    liveins: $vgpr0
+
+    ; CHECK-LABEL: name: test_zext_s16_to_s64
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 65535
+    ; CHECK: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[COPY]](s32)
+    ; CHECK: [[AND:%[0-9]+]]:_(s64) = G_AND [[ANYEXT]], [[C]]
+    ; CHECK: $vgpr0_vgpr1 = COPY [[AND]](s64)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s16) = G_TRUNC %0
+    %2:_(s64) = G_ZEXT %1
+    $vgpr0_vgpr1 = COPY %2
+...
+
+---
+name: test_zext_s16_to_s32
+body: |
+  bb.0:
+    liveins: $vgpr0
+
+    ; CHECK-LABEL: name: test_zext_s16_to_s32
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; CHECK: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
+    ; CHECK: $vgpr0 = COPY [[AND]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s16) = G_TRUNC %0
+    %2:_(s32) = G_ZEXT %1
+    $vgpr0 = COPY %2
+...
+
+---
+name: test_zext_i1_to_s32
+body: |
+  bb.0:
+
+    ; CHECK-LABEL: name: test_zext_i1_to_s32
+    ; CHECK: [[C:%[0-9]+]]:_(s1) = G_CONSTANT i1 false
+    ; CHECK: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[C]](s1)
+    ; CHECK: $vgpr0 = COPY [[ZEXT]](s32)
+    %0:_(s1) = G_CONSTANT i1 0
+    %1:_(s32) = G_ZEXT %0
+    $vgpr0 = COPY %1
+...
+
+---
+name: test_zext_i1_to_i64
+body: |
+  bb.0:
+
+    ; CHECK-LABEL: name: test_zext_i1_to_i64
+    ; CHECK: [[C:%[0-9]+]]:_(s1) = G_CONSTANT i1 false
+    ; CHECK: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[C]](s1)
+    ; CHECK: $vgpr0_vgpr1 = COPY [[ZEXT]](s64)
+    %0:_(s1) = G_CONSTANT i1 0
+    %1:_(s64) = G_ZEXT %0
+    $vgpr0_vgpr1 = COPY %1
+...
+
+---
+name: test_zext_v2s16_to_v2s32
+body: |
+  bb.0:
+    liveins: $vgpr0
+
+    ; CHECK-LABEL: name: test_zext_v2s16_to_v2s32
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
+    ; CHECK: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY]](<2 x s16>)
+    ; CHECK: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[UV]](s16)
+    ; CHECK: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[UV1]](s16)
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[ZEXT]](s32), [[ZEXT1]](s32)
+    ; CHECK: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
+    %0:_(<2 x s16>) = COPY $vgpr0
+    %1:_(<2 x s32>) = G_ZEXT %0
+    $vgpr0_vgpr1 = COPY %1
+...
+
+---
+name: test_zext_v3s16_to_v3s32
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; CHECK-LABEL: name: test_zext_v3s16_to_v3s32
+    ; CHECK: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
+    ; CHECK: [[EXTRACT:%[0-9]+]]:_(<3 x s16>) = G_EXTRACT [[COPY]](<4 x s16>), 0
+    ; CHECK: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16), [[UV2:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[EXTRACT]](<3 x s16>)
+    ; CHECK: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[UV]](s16)
+    ; CHECK: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[UV1]](s16)
+    ; CHECK: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[UV2]](s16)
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[ZEXT]](s32), [[ZEXT1]](s32), [[ZEXT2]](s32)
+    ; CHECK: $vgpr0_vgpr1_vgpr2 = COPY [[BUILD_VECTOR]](<3 x s32>)
+    %0:_(<4 x s16>) = COPY $vgpr0_vgpr1
+    %1:_(<3 x s16>) = G_EXTRACT %0, 0
+    %2:_(<3 x s32>) = G_ZEXT %1
+    $vgpr0_vgpr1_vgpr2 = COPY %2
+...
+
+---
+name: test_zext_v4s16_to_v4s32
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; CHECK-LABEL: name: test_zext_v4s16_to_v4s32
+    ; CHECK: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
+    ; CHECK: [[UV:%[0-9]+]]:_(s16), [[UV1:%[0-9]+]]:_(s16), [[UV2:%[0-9]+]]:_(s16), [[UV3:%[0-9]+]]:_(s16) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
+    ; CHECK: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[UV]](s16)
+    ; CHECK: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[UV1]](s16)
+    ; CHECK: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[UV2]](s16)
+    ; CHECK: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[UV3]](s16)
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[ZEXT]](s32), [[ZEXT1]](s32), [[ZEXT2]](s32), [[ZEXT3]](s32)
+    ; CHECK: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<4 x s32>)
+    %0:_(<4 x s16>) = COPY $vgpr0_vgpr1
+    %1:_(<4 x s32>) = G_ZEXT %0
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %1
+...
+
+---
+name: test_zext_v2s32_to_v2s64
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; CHECK-LABEL: name: test_zext_v2s32_to_v2s64
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    ; CHECK: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
+    ; CHECK: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[UV]](s32)
+    ; CHECK: [[ZEXT1:%[0-9]+]]:_(s64) = G_ZEXT [[UV1]](s32)
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[ZEXT]](s64), [[ZEXT1]](s64)
+    ; CHECK: $vgpr0_vgpr1_vgpr2_vgpr3 = COPY [[BUILD_VECTOR]](<2 x s64>)
+    %0:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    %1:_(<2 x s64>) = G_ZEXT %0
+    $vgpr0_vgpr1_vgpr2_vgpr3 = COPY %1
+...
+
+---
+name: test_zext_v3s32_to_v3s64
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2
+
+    ; CHECK-LABEL: name: test_zext_v3s32_to_v3s64
+    ; CHECK: [[COPY:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr0_vgpr1_vgpr2
+    ; CHECK: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<3 x s32>)
+    ; CHECK: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[UV]](s32)
+    ; CHECK: [[ZEXT1:%[0-9]+]]:_(s64) = G_ZEXT [[UV1]](s32)
+    ; CHECK: [[ZEXT2:%[0-9]+]]:_(s64) = G_ZEXT [[UV2]](s32)
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s64>) = G_BUILD_VECTOR [[ZEXT]](s64), [[ZEXT1]](s64), [[ZEXT2]](s64)
+    ; CHECK: S_NOP 0, implicit [[BUILD_VECTOR]](<3 x s64>)
+    %0:_(<3 x s32>) = COPY $vgpr0_vgpr1_vgpr2
+    %1:_(<3 x s64>) = G_ZEXT %0
+    S_NOP 0, implicit %1
+
+...
+
+---
+name: test_zext_v4s32_to_v4s64
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3
+
+    ; CHECK-LABEL: name: test_zext_v4s32_to_v4s64
+    ; CHECK: [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; CHECK: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<4 x s32>)
+    ; CHECK: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[UV]](s32)
+    ; CHECK: [[ZEXT1:%[0-9]+]]:_(s64) = G_ZEXT [[UV1]](s32)
+    ; CHECK: [[ZEXT2:%[0-9]+]]:_(s64) = G_ZEXT [[UV2]](s32)
+    ; CHECK: [[ZEXT3:%[0-9]+]]:_(s64) = G_ZEXT [[UV3]](s32)
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s64>) = G_BUILD_VECTOR [[ZEXT]](s64), [[ZEXT1]](s64), [[ZEXT2]](s64), [[ZEXT3]](s64)
+    ; CHECK: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY [[BUILD_VECTOR]](<4 x s64>)
+    %0:_(<4 x s32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    %1:_(<4 x s64>) = G_ZEXT %0
+    $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY %1
 ...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/legalize-zextload-flat.mir b/test/CodeGen/AMDGPU/GlobalISel/legalize-zextload-flat.mir
new file mode 100644
index 0000000000000000000000000000000000000000..9ac58da0612eef321e575e579cfd2c0767ab9fd6
--- /dev/null
+++ b/test/CodeGen/AMDGPU/GlobalISel/legalize-zextload-flat.mir
@@ -0,0 +1,135 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -run-pass=legalizer -o - %s | FileCheck %s -check-prefixes=GCN,SI
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=tahiti -run-pass=legalizer -o - %s | FileCheck %s -check-prefixes=GCN,VI
+---
+name: test_zextload_flat_i32_i8
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; SI-LABEL: name: test_zextload_flat_i32_i8
+    ; SI: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
+    ; SI: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p0) :: (load 1)
+    ; SI: $vgpr0 = COPY [[ZEXTLOAD]](s32)
+    ; VI-LABEL: name: test_zextload_flat_i32_i8
+    ; VI: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
+    ; VI: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p0) :: (load 1)
+    ; VI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; VI: [[COPY1:%[0-9]+]]:_(s32) = COPY [[LOAD]](s32)
+    ; VI: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
+    ; VI: $vgpr0 = COPY [[AND]](s32)
+    %0:_(p0) = COPY $vgpr0_vgpr1
+    %1:_(s32) = G_ZEXTLOAD %0 :: (load 1, addrspace 0)
+    $vgpr0 = COPY %1
+...
+---
+name: test_zextload_flat_i32_i16
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; SI-LABEL: name: test_zextload_flat_i32_i16
+    ; SI: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
+    ; SI: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p0) :: (load 2)
+    ; SI: $vgpr0 = COPY [[ZEXTLOAD]](s32)
+    ; VI-LABEL: name: test_zextload_flat_i32_i16
+    ; VI: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
+    ; VI: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p0) :: (load 2)
+    ; VI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; VI: [[COPY1:%[0-9]+]]:_(s32) = COPY [[LOAD]](s32)
+    ; VI: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
+    ; VI: $vgpr0 = COPY [[AND]](s32)
+     %0:_(p0) = COPY $vgpr0_vgpr1
+    %1:_(s32) = G_ZEXTLOAD %0 :: (load 2, addrspace 0)
+    $vgpr0 = COPY %1
+...
+---
+name: test_zextload_flat_i31_i8
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; SI-LABEL: name: test_zextload_flat_i31_i8
+    ; SI: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
+    ; SI: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p0) :: (load 1)
+    ; SI: [[COPY1:%[0-9]+]]:_(s32) = COPY [[ZEXTLOAD]](s32)
+    ; SI: $vgpr0 = COPY [[COPY1]](s32)
+    ; VI-LABEL: name: test_zextload_flat_i31_i8
+    ; VI: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
+    ; VI: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p0) :: (load 1)
+    ; VI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; VI: [[COPY1:%[0-9]+]]:_(s32) = COPY [[LOAD]](s32)
+    ; VI: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
+    ; VI: [[COPY2:%[0-9]+]]:_(s32) = COPY [[AND]](s32)
+    ; VI: $vgpr0 = COPY [[COPY2]](s32)
+    %0:_(p0) = COPY $vgpr0_vgpr1
+    %1:_(s31) = G_ZEXTLOAD %0 :: (load 1, addrspace 0)
+    %2:_(s32) = G_ANYEXT %1
+    $vgpr0 = COPY %2
+...
+---
+name: test_zextload_flat_i64_i8
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; SI-LABEL: name: test_zextload_flat_i64_i8
+    ; SI: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
+    ; SI: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p0) :: (load 1)
+    ; SI: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[ZEXTLOAD]](s32)
+    ; SI: $vgpr0_vgpr1 = COPY [[ZEXT]](s64)
+    ; VI-LABEL: name: test_zextload_flat_i64_i8
+    ; VI: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
+    ; VI: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p0) :: (load 1)
+    ; VI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; VI: [[COPY1:%[0-9]+]]:_(s32) = COPY [[LOAD]](s32)
+    ; VI: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
+    ; VI: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[AND]](s32)
+    ; VI: $vgpr0_vgpr1 = COPY [[ZEXT]](s64)
+    %0:_(p0) = COPY $vgpr0_vgpr1
+    %1:_(s64) = G_ZEXTLOAD %0 :: (load 1, addrspace 0)
+    $vgpr0_vgpr1 = COPY %1
+...
+---
+name: test_zextload_flat_i64_i16
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; SI-LABEL: name: test_zextload_flat_i64_i16
+    ; SI: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
+    ; SI: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p0) :: (load 2)
+    ; SI: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[ZEXTLOAD]](s32)
+    ; SI: $vgpr0_vgpr1 = COPY [[ZEXT]](s64)
+    ; VI-LABEL: name: test_zextload_flat_i64_i16
+    ; VI: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
+    ; VI: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p0) :: (load 2)
+    ; VI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; VI: [[COPY1:%[0-9]+]]:_(s32) = COPY [[LOAD]](s32)
+    ; VI: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
+    ; VI: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[AND]](s32)
+    ; VI: $vgpr0_vgpr1 = COPY [[ZEXT]](s64)
+    %0:_(p0) = COPY $vgpr0_vgpr1
+    %1:_(s64) = G_ZEXTLOAD %0 :: (load 2, addrspace 0)
+    $vgpr0_vgpr1 = COPY %1
+...
+---
+name: test_zextload_flat_i64_i32
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; SI-LABEL: name: test_zextload_flat_i64_i32
+    ; SI: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
+    ; SI: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p0) :: (load 4)
+    ; SI: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[LOAD]](s32)
+    ; SI: $vgpr0_vgpr1 = COPY [[ZEXT]](s64)
+    ; VI-LABEL: name: test_zextload_flat_i64_i32
+    ; VI: [[COPY:%[0-9]+]]:_(p0) = COPY $vgpr0_vgpr1
+    ; VI: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p0) :: (load 4)
+    ; VI: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[LOAD]](s32)
+    ; VI: $vgpr0_vgpr1 = COPY [[ZEXT]](s64)
+    %0:_(p0) = COPY $vgpr0_vgpr1
+    %1:_(s64) = G_ZEXTLOAD %0 :: (load 4, addrspace 0)
+    $vgpr0_vgpr1 = COPY %1
+...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/legalize-zextload-global.mir b/test/CodeGen/AMDGPU/GlobalISel/legalize-zextload-global.mir
new file mode 100644
index 0000000000000000000000000000000000000000..3552d0e62cb57015dbb463be2618a5dd870459ed
--- /dev/null
+++ b/test/CodeGen/AMDGPU/GlobalISel/legalize-zextload-global.mir
@@ -0,0 +1,93 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -run-pass=legalizer -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=tahiti -run-pass=legalizer -o - %s | FileCheck %s
+
+---
+name: test_zextload_global_i32_i8
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; CHECK-LABEL: name: test_zextload_global_i32_i8
+    ; CHECK: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
+    ; CHECK: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p1) :: (load 1, addrspace 1)
+    ; CHECK: $vgpr0 = COPY [[ZEXTLOAD]](s32)
+    %0:_(p1) = COPY $vgpr0_vgpr1
+    %1:_(s32) = G_ZEXTLOAD %0 :: (load 1, addrspace 1)
+    $vgpr0 = COPY %1
+...
+---
+name: test_zextload_global_i32_i16
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; CHECK-LABEL: name: test_zextload_global_i32_i16
+    ; CHECK: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
+    ; CHECK: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p1) :: (load 2, addrspace 1)
+    ; CHECK: $vgpr0 = COPY [[ZEXTLOAD]](s32)
+    %0:_(p1) = COPY $vgpr0_vgpr1
+    %1:_(s32) = G_ZEXTLOAD %0 :: (load 2, addrspace 1)
+    $vgpr0 = COPY %1
+...
+---
+name: test_zextload_global_i31_i8
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; CHECK-LABEL: name: test_zextload_global_i31_i8
+    ; CHECK: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
+    ; CHECK: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p1) :: (load 1, addrspace 1)
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY [[ZEXTLOAD]](s32)
+    ; CHECK: $vgpr0 = COPY [[COPY1]](s32)
+    %0:_(p1) = COPY $vgpr0_vgpr1
+    %1:_(s31) = G_ZEXTLOAD %0 :: (load 1, addrspace 1)
+    %2:_(s32) = G_ANYEXT %1
+    $vgpr0 = COPY %2
+...
+---
+name: test_zextload_global_i64_i8
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; CHECK-LABEL: name: test_zextload_global_i64_i8
+    ; CHECK: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
+    ; CHECK: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p1) :: (load 1, addrspace 1)
+    ; CHECK: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[ZEXTLOAD]](s32)
+    ; CHECK: $vgpr0_vgpr1 = COPY [[ZEXT]](s64)
+    %0:_(p1) = COPY $vgpr0_vgpr1
+    %1:_(s64) = G_ZEXTLOAD %0 :: (load 1, addrspace 1)
+    $vgpr0_vgpr1 = COPY %1
+...
+---
+name: test_zextload_global_i64_i16
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; CHECK-LABEL: name: test_zextload_global_i64_i16
+    ; CHECK: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
+    ; CHECK: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p1) :: (load 2, addrspace 1)
+    ; CHECK: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[ZEXTLOAD]](s32)
+    ; CHECK: $vgpr0_vgpr1 = COPY [[ZEXT]](s64)
+    %0:_(p1) = COPY $vgpr0_vgpr1
+    %1:_(s64) = G_ZEXTLOAD %0 :: (load 2, addrspace 1)
+    $vgpr0_vgpr1 = COPY %1
+...
+---
+name: test_zextload_global_i64_i32
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; CHECK-LABEL: name: test_zextload_global_i64_i32
+    ; CHECK: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
+    ; CHECK: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p1) :: (load 4, addrspace 1)
+    ; CHECK: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[LOAD]](s32)
+    ; CHECK: $vgpr0_vgpr1 = COPY [[ZEXT]](s64)
+    %0:_(p1) = COPY $vgpr0_vgpr1
+    %1:_(s64) = G_ZEXTLOAD %0 :: (load 4, addrspace 1)
+    $vgpr0_vgpr1 = COPY %1
+...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/legalize-zextload-local.mir b/test/CodeGen/AMDGPU/GlobalISel/legalize-zextload-local.mir
new file mode 100644
index 0000000000000000000000000000000000000000..648b9697bdcbdd764b3f33025494c3c8a48c332c
--- /dev/null
+++ b/test/CodeGen/AMDGPU/GlobalISel/legalize-zextload-local.mir
@@ -0,0 +1,92 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -run-pass=legalizer -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=tahiti -run-pass=legalizer -o - %s | FileCheck %s
+---
+name: test_zextload_local_i32_i8
+body: |
+  bb.0:
+    liveins: $vgpr0
+
+    ; CHECK-LABEL: name: test_zextload_local_i32_i8
+    ; CHECK: [[COPY:%[0-9]+]]:_(p3) = COPY $vgpr0
+    ; CHECK: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p3) :: (load 1, addrspace 3)
+    ; CHECK: $vgpr0 = COPY [[ZEXTLOAD]](s32)
+    %0:_(p3) = COPY $vgpr0
+    %1:_(s32) = G_ZEXTLOAD %0 :: (load 1, addrspace 3)
+    $vgpr0 = COPY %1
+...
+---
+name: test_zextload_local_i32_i16
+body: |
+  bb.0:
+    liveins: $vgpr0
+
+    ; CHECK-LABEL: name: test_zextload_local_i32_i16
+    ; CHECK: [[COPY:%[0-9]+]]:_(p3) = COPY $vgpr0
+    ; CHECK: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p3) :: (load 2, addrspace 3)
+    ; CHECK: $vgpr0 = COPY [[ZEXTLOAD]](s32)
+    %0:_(p3) = COPY $vgpr0
+    %1:_(s32) = G_ZEXTLOAD %0 :: (load 2, addrspace 3)
+    $vgpr0 = COPY %1
+...
+---
+name: test_zextload_local_i31_i8
+body: |
+  bb.0:
+    liveins: $vgpr0
+
+    ; CHECK-LABEL: name: test_zextload_local_i31_i8
+    ; CHECK: [[COPY:%[0-9]+]]:_(p3) = COPY $vgpr0
+    ; CHECK: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p3) :: (load 1, addrspace 3)
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY [[ZEXTLOAD]](s32)
+    ; CHECK: $vgpr0 = COPY [[COPY1]](s32)
+    %0:_(p3) = COPY $vgpr0
+    %1:_(s31) = G_ZEXTLOAD %0 :: (load 1, addrspace 3)
+    %2:_(s32) = G_ANYEXT %1
+    $vgpr0 = COPY %2
+...
+---
+name: test_zextload_local_i64_i8
+body: |
+  bb.0:
+    liveins: $vgpr0
+
+    ; CHECK-LABEL: name: test_zextload_local_i64_i8
+    ; CHECK: [[COPY:%[0-9]+]]:_(p3) = COPY $vgpr0
+    ; CHECK: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p3) :: (load 1, addrspace 3)
+    ; CHECK: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[ZEXTLOAD]](s32)
+    ; CHECK: $vgpr0_vgpr1 = COPY [[ZEXT]](s64)
+    %0:_(p3) = COPY $vgpr0
+    %1:_(s64) = G_ZEXTLOAD %0 :: (load 1, addrspace 3)
+    $vgpr0_vgpr1 = COPY %1
+...
+---
+name: test_zextload_local_i64_i16
+body: |
+  bb.0:
+    liveins: $vgpr0
+
+    ; CHECK-LABEL: name: test_zextload_local_i64_i16
+    ; CHECK: [[COPY:%[0-9]+]]:_(p3) = COPY $vgpr0
+    ; CHECK: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p3) :: (load 2, addrspace 3)
+    ; CHECK: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[ZEXTLOAD]](s32)
+    ; CHECK: $vgpr0_vgpr1 = COPY [[ZEXT]](s64)
+    %0:_(p3) = COPY $vgpr0
+    %1:_(s64) = G_ZEXTLOAD %0 :: (load 2, addrspace 3)
+    $vgpr0_vgpr1 = COPY %1
+...
+---
+name: test_zextload_local_i64_i32
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; CHECK-LABEL: name: test_zextload_local_i64_i32
+    ; CHECK: [[COPY:%[0-9]+]]:_(p3) = COPY $vgpr0
+    ; CHECK: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p3) :: (load 4, addrspace 3)
+    ; CHECK: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[LOAD]](s32)
+    ; CHECK: $vgpr0_vgpr1 = COPY [[ZEXT]](s64)
+    %0:_(p3) = COPY $vgpr0
+    %1:_(s64) = G_ZEXTLOAD %0 :: (load 4, addrspace 3)
+    $vgpr0_vgpr1 = COPY %1
+...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/legalize-zextload-private.mir b/test/CodeGen/AMDGPU/GlobalISel/legalize-zextload-private.mir
new file mode 100644
index 0000000000000000000000000000000000000000..67619983436cdc1069871cf4ba4324d6202593f1
--- /dev/null
+++ b/test/CodeGen/AMDGPU/GlobalISel/legalize-zextload-private.mir
@@ -0,0 +1,93 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -run-pass=legalizer -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=tahiti -run-pass=legalizer -o - %s | FileCheck %s
+
+---
+name: test_zextload_private_i32_i8
+body: |
+  bb.0:
+    liveins: $vgpr0
+
+    ; CHECK-LABEL: name: test_zextload_private_i32_i8
+    ; CHECK: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
+    ; CHECK: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load 1, addrspace 5)
+    ; CHECK: $vgpr0 = COPY [[ZEXTLOAD]](s32)
+    %0:_(p5) = COPY $vgpr0
+    %1:_(s32) = G_ZEXTLOAD %0 :: (load 1, addrspace 5)
+    $vgpr0 = COPY %1
+...
+---
+name: test_zextload_private_i32_i16
+body: |
+  bb.0:
+    liveins: $vgpr0
+
+    ; CHECK-LABEL: name: test_zextload_private_i32_i16
+    ; CHECK: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
+    ; CHECK: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load 2, addrspace 5)
+    ; CHECK: $vgpr0 = COPY [[ZEXTLOAD]](s32)
+    %0:_(p5) = COPY $vgpr0
+    %1:_(s32) = G_ZEXTLOAD %0 :: (load 2, addrspace 5)
+    $vgpr0 = COPY %1
+...
+---
+name: test_zextload_private_i31_i8
+body: |
+  bb.0:
+    liveins: $vgpr0
+
+    ; CHECK-LABEL: name: test_zextload_private_i31_i8
+    ; CHECK: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
+    ; CHECK: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load 1, addrspace 5)
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY [[ZEXTLOAD]](s32)
+    ; CHECK: $vgpr0 = COPY [[COPY1]](s32)
+    %0:_(p5) = COPY $vgpr0
+    %1:_(s31) = G_ZEXTLOAD %0 :: (load 1, addrspace 5)
+    %2:_(s32) = G_ANYEXT %1
+    $vgpr0 = COPY %2
+...
+---
+name: test_zextload_private_i64_i8
+body: |
+  bb.0:
+    liveins: $vgpr0
+
+    ; CHECK-LABEL: name: test_zextload_private_i64_i8
+    ; CHECK: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
+    ; CHECK: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load 1, addrspace 5)
+    ; CHECK: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[ZEXTLOAD]](s32)
+    ; CHECK: $vgpr0_vgpr1 = COPY [[ZEXT]](s64)
+    %0:_(p5) = COPY $vgpr0
+    %1:_(s64) = G_ZEXTLOAD %0 :: (load 1, addrspace 5)
+    $vgpr0_vgpr1 = COPY %1
+...
+---
+name: test_zextload_private_i64_i16
+body: |
+  bb.0:
+    liveins: $vgpr0
+
+    ; CHECK-LABEL: name: test_zextload_private_i64_i16
+    ; CHECK: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
+    ; CHECK: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p5) :: (load 2, addrspace 5)
+    ; CHECK: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[ZEXTLOAD]](s32)
+    ; CHECK: $vgpr0_vgpr1 = COPY [[ZEXT]](s64)
+    %0:_(p5) = COPY $vgpr0
+    %1:_(s64) = G_ZEXTLOAD %0 :: (load 2, addrspace 5)
+    $vgpr0_vgpr1 = COPY %1
+...
+---
+name: test_zextload_private_i64_i32
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; CHECK-LABEL: name: test_zextload_private_i64_i32
+    ; CHECK: [[COPY:%[0-9]+]]:_(p5) = COPY $vgpr0
+    ; CHECK: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p5) :: (load 4, addrspace 5)
+    ; CHECK: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[LOAD]](s32)
+    ; CHECK: $vgpr0_vgpr1 = COPY [[ZEXT]](s64)
+    %0:_(p5) = COPY $vgpr0
+    %1:_(s64) = G_ZEXTLOAD %0 :: (load 4, addrspace 5)
+    $vgpr0_vgpr1 = COPY %1
+...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/no-legalize-atomic.mir b/test/CodeGen/AMDGPU/GlobalISel/no-legalize-atomic.mir
new file mode 100644
index 0000000000000000000000000000000000000000..a54a0957e818e3670a014d6b5bcb700e30c587e8
--- /dev/null
+++ b/test/CodeGen/AMDGPU/GlobalISel/no-legalize-atomic.mir
@@ -0,0 +1,46 @@
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=tahiti -O0 -run-pass=legalizer -global-isel-abort=0 -o - %s | FileCheck %s
+
+# CHECK: %1:_(<8 x s32>) = G_LOAD %0(p1) :: (load monotonic 32, addrspace 1)
+# CHECK:  G_STORE %1(<8 x s32>), %0(p1) :: (store monotonic 32, addrspace 1)
+# CHECK: %1:_(s256) = G_LOAD %0(p1) :: (load monotonic 32, addrspace 1)
+# CHECK:  G_STORE %1(s256), %0(p1) :: (store monotonic 32, addrspace 1)
+
+---
+name: test_atomic_load_global_v8s32
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+    %0:_(p1) = COPY $vgpr0_vgpr1
+    %1:_(<8 x s32>) = G_LOAD %0 :: (load monotonic  32, addrspace 1, align 32)
+    $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY %1
+...
+
+---
+name: test_atomic_store_global_v8s32
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9
+    %0:_(p1) = COPY $vgpr0_vgpr1
+    %1:_(<8 x s32>) = COPY $vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9
+    G_STORE %1, %0 :: (store monotonic  32, addrspace 1, align 32)
+...
+
+---
+name: test_atomic_load_global_s256
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+    %0:_(p1) = COPY $vgpr0_vgpr1
+    %1:_(s256) = G_LOAD %0 :: (load monotonic  32, addrspace 1, align 32)
+    $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7 = COPY %1
+...
+
+---
+name: test_atomic_store_global_s256
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9
+    %0:_(p1) = COPY $vgpr0_vgpr1
+    %1:_(s256) = COPY $vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9
+    G_STORE %1, %0 :: (store monotonic  32, addrspace 1, align 32)
+...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn-wqm-vote.mir b/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn-wqm-vote.mir
new file mode 100644
index 0000000000000000000000000000000000000000..a3a59941b49469d41157d4c9a2e567d83e896bf1
--- /dev/null
+++ b/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn-wqm-vote.mir
@@ -0,0 +1,57 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -march=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
+# RUN: llc -march=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
+
+---
+name: wqm_vote_scc
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0, $sgpr1
+    ; CHECK-LABEL: name: wqm_vote_scc
+    ; CHECK: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
+    ; CHECK: [[ICMP:%[0-9]+]]:scc(s1) = G_ICMP intpred(ne), [[COPY]](s32), [[COPY1]]
+    ; CHECK: [[COPY2:%[0-9]+]]:sgpr(s1) = COPY [[ICMP]](s1)
+    ; CHECK: [[INT:%[0-9]+]]:sgpr(s1) = G_INTRINSIC intrinsic(@llvm.amdgcn.wqm.vote), [[COPY2]](s1)
+    %0:_(s32) = COPY $sgpr0
+    %1:_(s32) = COPY $sgpr1
+    %2:_(s1) = G_ICMP intpred(ne), %0, %1
+    %3:_(s1) = G_INTRINSIC intrinsic(@llvm.amdgcn.wqm.vote), %2
+...
+
+---
+name: wqm_vote_vcc
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+    ; CHECK-LABEL: name: wqm_vote_vcc
+    ; CHECK: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+    ; CHECK: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(ne), [[COPY]](s32), [[COPY1]]
+    ; CHECK: [[COPY2:%[0-9]+]]:sgpr(s1) = COPY [[ICMP]](s1)
+    ; CHECK: [[INT:%[0-9]+]]:sgpr(s1) = G_INTRINSIC intrinsic(@llvm.amdgcn.wqm.vote), [[COPY2]](s1)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s1) = G_ICMP intpred(ne), %0, %1
+    %3:_(s1) = G_INTRINSIC intrinsic(@llvm.amdgcn.wqm.vote), %2
+...
+
+---
+name: wqm_vote_sgpr
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0, $sgpr1
+    ; CHECK-LABEL: name: wqm_vote_sgpr
+    ; CHECK: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; CHECK: [[TRUNC:%[0-9]+]]:sgpr(s1) = G_TRUNC [[COPY]](s32)
+    ; CHECK: [[INT:%[0-9]+]]:sgpr(s1) = G_INTRINSIC intrinsic(@llvm.amdgcn.wqm.vote), [[TRUNC]](s1)
+    %0:_(s32) = COPY $sgpr0
+    %1:_(s1) = G_TRUNC %0
+    %2:_(s1) = G_INTRINSIC intrinsic(@llvm.amdgcn.wqm.vote), %1
+...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/regbankselect-and.mir b/test/CodeGen/AMDGPU/GlobalISel/regbankselect-and.mir
index e1864d849e6afcb9f643d027fa6e9cec4dcfc893..9e2d66269852ee27419eef6536060aea2b4fdedd 100644
--- a/test/CodeGen/AMDGPU/GlobalISel/regbankselect-and.mir
+++ b/test/CodeGen/AMDGPU/GlobalISel/regbankselect-and.mir
@@ -3,13 +3,13 @@
 # RUN: llc -march=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
 
 ---
-name: and_ss
+name: and_i32_ss
 legalized: true
 
 body: |
   bb.0:
     liveins: $sgpr0, $sgpr1
-    ; CHECK-LABEL: name: and_ss
+    ; CHECK-LABEL: name: and_i32_ss
     ; CHECK: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
     ; CHECK: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
     ; CHECK: [[AND:%[0-9]+]]:sgpr(s32) = G_AND [[COPY]], [[COPY1]]
@@ -19,13 +19,13 @@ body: |
 ...
 
 ---
-name: and_sv
+name: and_i32_sv
 legalized: true
 
 body: |
   bb.0:
     liveins: $sgpr0, $vgpr0
-    ; CHECK-LABEL: name: and_sv
+    ; CHECK-LABEL: name: and_i32_sv
     ; CHECK: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
     ; CHECK: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
     ; CHECK: [[AND:%[0-9]+]]:vgpr(s32) = G_AND [[COPY]], [[COPY1]]
@@ -35,13 +35,13 @@ body: |
 ...
 
 ---
-name: and_vs
+name: and_i32_vs
 legalized: true
 
 body: |
   bb.0:
     liveins: $sgpr0, $vgpr0
-    ; CHECK-LABEL: name: and_vs
+    ; CHECK-LABEL: name: and_i32_vs
     ; CHECK: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
     ; CHECK: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
     ; CHECK: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32)
@@ -52,13 +52,13 @@ body: |
 ...
 
 ---
-name: and_vv
+name: and_i32_vv
 legalized: true
 
 body: |
   bb.0:
     liveins: $vgpr0, $vgpr1
-    ; CHECK-LABEL: name: and_vv
+    ; CHECK-LABEL: name: and_i32_vv
     ; CHECK: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
     ; CHECK: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
     ; CHECK: [[AND:%[0-9]+]]:vgpr(s32) = G_AND [[COPY]], [[COPY1]]
@@ -66,3 +66,533 @@ body: |
     %1:_(s32) = COPY $vgpr1
     %2:_(s32) = G_AND %0, %1
 ...
+
+---
+name: and_i1_scc_scc
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0, $sgpr1
+    ; CHECK-LABEL: name: and_i1_scc_scc
+    ; CHECK: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
+    ; CHECK: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
+    ; CHECK: [[ICMP:%[0-9]+]]:scc(s1) = G_ICMP intpred(ne), [[COPY]](s32), [[C]]
+    ; CHECK: [[ICMP1:%[0-9]+]]:scc(s1) = G_ICMP intpred(ne), [[COPY1]](s32), [[C]]
+    ; CHECK: [[COPY2:%[0-9]+]]:sgpr(s1) = COPY [[ICMP]](s1)
+    ; CHECK: [[COPY3:%[0-9]+]]:sgpr(s1) = COPY [[ICMP1]](s1)
+    ; CHECK: [[AND:%[0-9]+]]:sgpr(s1) = G_AND [[COPY2]], [[COPY3]]
+    ; CHECK: S_NOP 0, implicit [[AND]](s1)
+      %0:_(s32) = COPY $sgpr0
+      %1:_(s32) = COPY $sgpr1
+      %2:_(s32) = G_CONSTANT i32 0
+      %4:_(s1) = G_ICMP intpred(ne), %0, %2
+      %5:_(s1) = G_ICMP intpred(ne), %1, %2
+      %6:_(s1) = G_AND %4, %5
+      S_NOP 0, implicit %6
+...
+
+---
+name: and_i1_vcc_vcc
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+    ; CHECK-LABEL: name: and_i1_vcc_vcc
+    ; CHECK: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+    ; CHECK: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
+    ; CHECK: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(ne), [[COPY]](s32), [[C]]
+    ; CHECK: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(ne), [[COPY1]](s32), [[C]]
+    ; CHECK: [[COPY2:%[0-9]+]]:sgpr(s1) = COPY [[ICMP]](s1)
+    ; CHECK: [[COPY3:%[0-9]+]]:sgpr(s1) = COPY [[ICMP1]](s1)
+    ; CHECK: [[AND:%[0-9]+]]:sgpr(s1) = G_AND [[COPY2]], [[COPY3]]
+    ; CHECK: S_NOP 0, implicit [[AND]](s1)
+      %0:_(s32) = COPY $vgpr0
+      %1:_(s32) = COPY $vgpr1
+      %2:_(s32) = G_CONSTANT i32 0
+      %4:_(s1) = G_ICMP intpred(ne), %0, %2
+      %5:_(s1) = G_ICMP intpred(ne), %1, %2
+      %6:_(s1) = G_AND %4, %5
+      S_NOP 0, implicit %6
+...
+
+---
+name: and_i1_scc_vcc
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0, $vgpr0
+    ; CHECK-LABEL: name: and_i1_scc_vcc
+    ; CHECK: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; CHECK: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
+    ; CHECK: [[ICMP:%[0-9]+]]:scc(s1) = G_ICMP intpred(ne), [[COPY]](s32), [[C]]
+    ; CHECK: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(ne), [[COPY1]](s32), [[C]]
+    ; CHECK: [[COPY2:%[0-9]+]]:sgpr(s1) = COPY [[ICMP]](s1)
+    ; CHECK: [[COPY3:%[0-9]+]]:sgpr(s1) = COPY [[ICMP1]](s1)
+    ; CHECK: [[AND:%[0-9]+]]:sgpr(s1) = G_AND [[COPY2]], [[COPY3]]
+    ; CHECK: S_NOP 0, implicit [[AND]](s1)
+      %0:_(s32) = COPY $sgpr0
+      %1:_(s32) = COPY $vgpr0
+      %2:_(s32) = G_CONSTANT i32 0
+      %4:_(s1) = G_ICMP intpred(ne), %0, %2
+      %5:_(s1) = G_ICMP intpred(ne), %1, %2
+      %6:_(s1) = G_AND %4, %5
+      S_NOP 0, implicit %6
+...
+
+---
+name: and_i1_sgpr_trunc_sgpr_trunc
+legalized: true
+body:             |
+  bb.0.entry:
+    liveins: $sgpr0, $sgpr1
+    ; CHECK-LABEL: name: and_i1_sgpr_trunc_sgpr_trunc
+    ; CHECK: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
+    ; CHECK: [[TRUNC:%[0-9]+]]:sgpr(s1) = G_TRUNC [[COPY]](s32)
+    ; CHECK: [[TRUNC1:%[0-9]+]]:sgpr(s1) = G_TRUNC [[COPY1]](s32)
+    ; CHECK: [[AND:%[0-9]+]]:sgpr(s1) = G_AND [[TRUNC]], [[TRUNC1]]
+    ; CHECK: S_NOP 0, implicit [[AND]](s1)
+    %0:_(s32) = COPY $sgpr0
+    %1:_(s32) = COPY $sgpr1
+    %2:_(s1) = G_TRUNC %0
+    %3:_(s1) = G_TRUNC %1
+    %4:_(s1) = G_AND %2, %3
+    S_NOP 0, implicit %4
+
+...
+
+---
+name: and_i1_trunc_scc
+legalized: true
+body:             |
+  bb.0.entry:
+    liveins: $sgpr0, $sgpr1
+    ; CHECK-LABEL: name: and_i1_trunc_scc
+    ; CHECK: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
+    ; CHECK: [[TRUNC:%[0-9]+]]:sgpr(s1) = G_TRUNC [[COPY]](s32)
+    ; CHECK: [[ICMP:%[0-9]+]]:scc(s1) = G_ICMP intpred(ne), [[COPY]](s32), [[COPY1]]
+    ; CHECK: [[COPY2:%[0-9]+]]:sgpr(s1) = COPY [[ICMP]](s1)
+    ; CHECK: [[AND:%[0-9]+]]:sgpr(s1) = G_AND [[TRUNC]], [[COPY2]]
+    ; CHECK: S_NOP 0, implicit [[AND]](s1)
+    %0:_(s32) = COPY $sgpr0
+    %1:_(s32) = COPY $sgpr1
+    %2:_(s1) = G_TRUNC %0
+    %3:_(s1) = G_ICMP intpred(ne), %0, %1
+    %4:_(s1) = G_AND %2, %3
+    S_NOP 0, implicit %4
+...
+
+---
+name: and_i1_s_trunc_vcc
+legalized: true
+body:             |
+  bb.0.entry:
+    liveins: $sgpr0, $vgpr0
+    ; CHECK-LABEL: name: and_i1_s_trunc_vcc
+    ; CHECK: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; CHECK: [[TRUNC:%[0-9]+]]:sgpr(s1) = G_TRUNC [[COPY]](s32)
+    ; CHECK: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(ne), [[COPY]](s32), [[COPY1]]
+    ; CHECK: [[COPY2:%[0-9]+]]:sgpr(s1) = COPY [[ICMP]](s1)
+    ; CHECK: [[AND:%[0-9]+]]:sgpr(s1) = G_AND [[TRUNC]], [[COPY2]]
+    ; CHECK: S_NOP 0, implicit [[AND]](s1)
+    %0:_(s32) = COPY $sgpr0
+    %1:_(s32) = COPY $vgpr0
+    %2:_(s1) = G_TRUNC %0
+    %3:_(s1) = G_ICMP intpred(ne), %0, %1
+    %4:_(s1) = G_AND %2, %3
+    S_NOP 0, implicit %4
+...
+
+---
+name: and_i64_ss
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0_sgpr1, $sgpr2_sgpr3
+    ; CHECK-LABEL: name: and_i64_ss
+    ; CHECK: [[COPY:%[0-9]+]]:sgpr(s64) = COPY $sgpr0_sgpr1
+    ; CHECK: [[COPY1:%[0-9]+]]:sgpr(s64) = COPY $sgpr2_sgpr3
+    ; CHECK: [[AND:%[0-9]+]]:sgpr(s64) = G_AND [[COPY]], [[COPY1]]
+    %0:_(s64) = COPY $sgpr0_sgpr1
+    %1:_(s64) = COPY $sgpr2_sgpr3
+    %2:_(s64) = G_AND %0, %1
+...
+
+---
+name: and_i64_sv
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0_sgpr1, $vgpr0_vgpr1
+    ; CHECK-LABEL: name: and_i64_sv
+    ; CHECK: [[COPY:%[0-9]+]]:sgpr(s64) = COPY $sgpr0_sgpr1
+    ; CHECK: [[COPY1:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
+    ; CHECK: [[UV:%[0-9]+]]:sgpr(s32), [[UV1:%[0-9]+]]:sgpr(s32) = G_UNMERGE_VALUES [[COPY]](s64)
+    ; CHECK: [[UV2:%[0-9]+]]:vgpr(s32), [[UV3:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[COPY1]](s64)
+    ; CHECK: [[AND:%[0-9]+]]:vgpr(s32) = G_AND [[UV]], [[UV2]]
+    ; CHECK: [[AND1:%[0-9]+]]:vgpr(s32) = G_AND [[UV1]], [[UV3]]
+    ; CHECK: [[MV:%[0-9]+]]:vgpr(s64) = G_MERGE_VALUES [[AND]](s32), [[AND1]](s32)
+    %0:_(s64) = COPY $sgpr0_sgpr1
+    %1:_(s64) = COPY $vgpr0_vgpr1
+    %2:_(s64) = G_AND %0, %1
+...
+
+---
+name: and_i64_vs
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0_sgpr1, $vgpr0_vgpr1
+    ; CHECK-LABEL: name: and_i64_vs
+    ; CHECK: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
+    ; CHECK: [[COPY1:%[0-9]+]]:sgpr(s64) = COPY $sgpr0_sgpr1
+    ; CHECK: [[UV:%[0-9]+]]:vgpr(s32), [[UV1:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[COPY]](s64)
+    ; CHECK: [[UV2:%[0-9]+]]:sgpr(s32), [[UV3:%[0-9]+]]:sgpr(s32) = G_UNMERGE_VALUES [[COPY1]](s64)
+    ; CHECK: [[AND:%[0-9]+]]:vgpr(s32) = G_AND [[UV]], [[UV2]]
+    ; CHECK: [[AND1:%[0-9]+]]:vgpr(s32) = G_AND [[UV1]], [[UV3]]
+    ; CHECK: [[MV:%[0-9]+]]:vgpr(s64) = G_MERGE_VALUES [[AND]](s32), [[AND1]](s32)
+    %0:_(s64) = COPY $vgpr0_vgpr1
+    %1:_(s64) = COPY $sgpr0_sgpr1
+    %2:_(s64) = G_AND %0, %1
+...
+
+---
+name: and_i64_vv
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
+    ; CHECK-LABEL: name: and_i64_vv
+    ; CHECK: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
+    ; CHECK: [[COPY1:%[0-9]+]]:vgpr(s64) = COPY $vgpr2_vgpr3
+    ; CHECK: [[UV:%[0-9]+]]:vgpr(s32), [[UV1:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[COPY]](s64)
+    ; CHECK: [[UV2:%[0-9]+]]:vgpr(s32), [[UV3:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[COPY1]](s64)
+    ; CHECK: [[AND:%[0-9]+]]:vgpr(s32) = G_AND [[UV]], [[UV2]]
+    ; CHECK: [[AND1:%[0-9]+]]:vgpr(s32) = G_AND [[UV1]], [[UV3]]
+    ; CHECK: [[MV:%[0-9]+]]:vgpr(s64) = G_MERGE_VALUES [[AND]](s32), [[AND1]](s32)
+    %0:_(s64) = COPY $vgpr0_vgpr1
+    %1:_(s64) = COPY $vgpr2_vgpr3
+    %2:_(s64) = G_AND %0, %1
+...
+
+---
+name: and_i64_vv_user
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
+    ; CHECK-LABEL: name: and_i64_vv_user
+    ; CHECK: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
+    ; CHECK: [[COPY1:%[0-9]+]]:vgpr(s64) = COPY $vgpr2_vgpr3
+    ; CHECK: [[UV:%[0-9]+]]:vgpr(s32), [[UV1:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[COPY]](s64)
+    ; CHECK: [[UV2:%[0-9]+]]:vgpr(s32), [[UV3:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[COPY1]](s64)
+    ; CHECK: [[AND:%[0-9]+]]:vgpr(s32) = G_AND [[UV]], [[UV2]]
+    ; CHECK: [[AND1:%[0-9]+]]:vgpr(s32) = G_AND [[UV1]], [[UV3]]
+    ; CHECK: [[MV:%[0-9]+]]:vgpr(s64) = G_MERGE_VALUES [[AND]](s32), [[AND1]](s32)
+    ; CHECK: S_NOP 0, implicit [[MV]](s64)
+    %0:_(s64) = COPY $vgpr0_vgpr1
+    %1:_(s64) = COPY $vgpr2_vgpr3
+    %2:_(s64) = G_AND %0, %1
+    S_NOP 0, implicit %2
+...
+---
+name: and_i64_ss_ss_merge
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3
+    ; CHECK-LABEL: name: and_i64_ss_ss_merge
+    ; CHECK: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
+    ; CHECK: [[COPY2:%[0-9]+]]:sgpr(s32) = COPY $sgpr2
+    ; CHECK: [[COPY3:%[0-9]+]]:sgpr(s32) = COPY $sgpr3
+    ; CHECK: [[MV:%[0-9]+]]:sgpr(s64) = G_MERGE_VALUES [[COPY]](s32), [[COPY1]](s32)
+    ; CHECK: [[MV1:%[0-9]+]]:sgpr(s64) = G_MERGE_VALUES [[COPY2]](s32), [[COPY3]](s32)
+    ; CHECK: [[AND:%[0-9]+]]:sgpr(s64) = G_AND [[MV]], [[MV1]]
+    ; CHECK: S_NOP 0, implicit [[AND]](s64)
+    %0:_(s32) = COPY $sgpr0
+    %1:_(s32) = COPY $sgpr1
+    %2:_(s32) = COPY $sgpr2
+    %3:_(s32) = COPY $sgpr3
+    %4:_(s64) = G_MERGE_VALUES %0, %1
+    %5:_(s64) = G_MERGE_VALUES %2, %3
+    %6:_(s64) = G_AND %4, %5
+    S_NOP 0, implicit %6
+...
+
+---
+name: and_i64_vv_vv_merge
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3
+    ; CHECK-LABEL: name: and_i64_vv_vv_merge
+    ; CHECK: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+    ; CHECK: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY $vgpr2
+    ; CHECK: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY $vgpr3
+    ; CHECK: [[MV:%[0-9]+]]:vgpr(s64) = G_MERGE_VALUES [[COPY]](s32), [[COPY1]](s32)
+    ; CHECK: [[MV1:%[0-9]+]]:vgpr(s64) = G_MERGE_VALUES [[COPY2]](s32), [[COPY3]](s32)
+    ; CHECK: [[UV:%[0-9]+]]:vgpr(s32), [[UV1:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[MV]](s64)
+    ; CHECK: [[UV2:%[0-9]+]]:vgpr(s32), [[UV3:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[MV1]](s64)
+    ; CHECK: [[AND:%[0-9]+]]:vgpr(s32) = G_AND [[UV]], [[UV2]]
+    ; CHECK: [[AND1:%[0-9]+]]:vgpr(s32) = G_AND [[UV1]], [[UV3]]
+    ; CHECK: [[MV2:%[0-9]+]]:vgpr(s64) = G_MERGE_VALUES [[AND]](s32), [[AND1]](s32)
+    ; CHECK: S_NOP 0, implicit [[MV2]](s64)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s32) = COPY $vgpr2
+    %3:_(s32) = COPY $vgpr3
+    %4:_(s64) = G_MERGE_VALUES %0, %1
+    %5:_(s64) = G_MERGE_VALUES %2, %3
+    %6:_(s64) = G_AND %4, %5
+    S_NOP 0, implicit %6
+...
+
+---
+name: and_i64_s_sv_merge
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0_sgpr1, $sgpr2, $vgpr0
+    ; CHECK-LABEL: name: and_i64_s_sv_merge
+    ; CHECK: [[COPY:%[0-9]+]]:sgpr(s64) = COPY $sgpr0_sgpr1
+    ; CHECK: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr2
+    ; CHECK: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; CHECK: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32)
+    ; CHECK: [[MV:%[0-9]+]]:vgpr(s64) = G_MERGE_VALUES [[COPY3]](s32), [[COPY2]](s32)
+    ; CHECK: [[UV:%[0-9]+]]:sgpr(s32), [[UV1:%[0-9]+]]:sgpr(s32) = G_UNMERGE_VALUES [[COPY]](s64)
+    ; CHECK: [[UV2:%[0-9]+]]:vgpr(s32), [[UV3:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[MV]](s64)
+    ; CHECK: [[AND:%[0-9]+]]:vgpr(s32) = G_AND [[UV]], [[UV2]]
+    ; CHECK: [[AND1:%[0-9]+]]:vgpr(s32) = G_AND [[UV1]], [[UV3]]
+    ; CHECK: [[MV1:%[0-9]+]]:vgpr(s64) = G_MERGE_VALUES [[AND]](s32), [[AND1]](s32)
+    ; CHECK: S_NOP 0, implicit [[MV1]](s64)
+    %0:_(s64) = COPY $sgpr0_sgpr1
+    %1:_(s32) = COPY $sgpr2
+    %2:_(s32) = COPY $vgpr0
+    %3:_(s64) = G_MERGE_VALUES %1, %2
+    %4:_(s64) = G_AND %0, %3
+    S_NOP 0, implicit %4
+...
+
+---
+name: and_i64_s_vs_merge
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0_sgpr1, $sgpr2, $vgpr0
+    ; CHECK-LABEL: name: and_i64_s_vs_merge
+    ; CHECK: [[COPY:%[0-9]+]]:sgpr(s64) = COPY $sgpr0_sgpr1
+    ; CHECK: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr2
+    ; CHECK: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; CHECK: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32)
+    ; CHECK: [[MV:%[0-9]+]]:vgpr(s64) = G_MERGE_VALUES [[COPY2]](s32), [[COPY3]](s32)
+    ; CHECK: [[UV:%[0-9]+]]:sgpr(s32), [[UV1:%[0-9]+]]:sgpr(s32) = G_UNMERGE_VALUES [[COPY]](s64)
+    ; CHECK: [[UV2:%[0-9]+]]:vgpr(s32), [[UV3:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[MV]](s64)
+    ; CHECK: [[AND:%[0-9]+]]:vgpr(s32) = G_AND [[UV]], [[UV2]]
+    ; CHECK: [[AND1:%[0-9]+]]:vgpr(s32) = G_AND [[UV1]], [[UV3]]
+    ; CHECK: [[MV1:%[0-9]+]]:vgpr(s64) = G_MERGE_VALUES [[AND]](s32), [[AND1]](s32)
+    ; CHECK: S_NOP 0, implicit [[MV1]](s64)
+    %0:_(s64) = COPY $sgpr0_sgpr1
+    %1:_(s32) = COPY $sgpr2
+    %2:_(s32) = COPY $vgpr0
+    %3:_(s64) = G_MERGE_VALUES %2, %1
+    %4:_(s64) = G_AND %0, %3
+    S_NOP 0, implicit %4
+...
+
+---
+name: and_i64_sv_sv_merge
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0, $sgpr1, $vgpr0, $vgpr1
+    ; CHECK-LABEL: name: and_i64_sv_sv_merge
+    ; CHECK: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
+    ; CHECK: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; CHECK: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+    ; CHECK: [[COPY4:%[0-9]+]]:vgpr(s32) = COPY [[COPY]](s32)
+    ; CHECK: [[MV:%[0-9]+]]:vgpr(s64) = G_MERGE_VALUES [[COPY4]](s32), [[COPY2]](s32)
+    ; CHECK: [[COPY5:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32)
+    ; CHECK: [[MV1:%[0-9]+]]:vgpr(s64) = G_MERGE_VALUES [[COPY5]](s32), [[COPY3]](s32)
+    ; CHECK: [[UV:%[0-9]+]]:vgpr(s32), [[UV1:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[MV]](s64)
+    ; CHECK: [[UV2:%[0-9]+]]:vgpr(s32), [[UV3:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[MV1]](s64)
+    ; CHECK: [[AND:%[0-9]+]]:vgpr(s32) = G_AND [[UV]], [[UV2]]
+    ; CHECK: [[AND1:%[0-9]+]]:vgpr(s32) = G_AND [[UV1]], [[UV3]]
+    ; CHECK: [[MV2:%[0-9]+]]:vgpr(s64) = G_MERGE_VALUES [[AND]](s32), [[AND1]](s32)
+    ; CHECK: S_NOP 0, implicit [[MV2]](s64)
+    %0:_(s32) = COPY $sgpr0
+    %1:_(s32) = COPY $sgpr1
+    %2:_(s32) = COPY $vgpr0
+    %3:_(s32) = COPY $vgpr1
+    %4:_(s64) = G_MERGE_VALUES %0, %2
+    %5:_(s64) = G_MERGE_VALUES %1, %3
+    %6:_(s64) = G_AND %4, %5
+    S_NOP 0, implicit %6
+...
+
+---
+name: and_i64_sv_vs_merge
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0, $sgpr1, $vgpr0, $vgpr1
+    ; CHECK-LABEL: name: and_i64_sv_vs_merge
+    ; CHECK: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
+    ; CHECK: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; CHECK: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+    ; CHECK: [[COPY4:%[0-9]+]]:vgpr(s32) = COPY [[COPY]](s32)
+    ; CHECK: [[MV:%[0-9]+]]:vgpr(s64) = G_MERGE_VALUES [[COPY4]](s32), [[COPY2]](s32)
+    ; CHECK: [[COPY5:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32)
+    ; CHECK: [[MV1:%[0-9]+]]:vgpr(s64) = G_MERGE_VALUES [[COPY3]](s32), [[COPY5]](s32)
+    ; CHECK: [[UV:%[0-9]+]]:vgpr(s32), [[UV1:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[MV]](s64)
+    ; CHECK: [[UV2:%[0-9]+]]:vgpr(s32), [[UV3:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[MV1]](s64)
+    ; CHECK: [[AND:%[0-9]+]]:vgpr(s32) = G_AND [[UV]], [[UV2]]
+    ; CHECK: [[AND1:%[0-9]+]]:vgpr(s32) = G_AND [[UV1]], [[UV3]]
+    ; CHECK: [[MV2:%[0-9]+]]:vgpr(s64) = G_MERGE_VALUES [[AND]](s32), [[AND1]](s32)
+    ; CHECK: S_NOP 0, implicit [[MV2]](s64)
+    %0:_(s32) = COPY $sgpr0
+    %1:_(s32) = COPY $sgpr1
+    %2:_(s32) = COPY $vgpr0
+    %3:_(s32) = COPY $vgpr1
+    %4:_(s64) = G_MERGE_VALUES %0, %2
+    %5:_(s64) = G_MERGE_VALUES %3, %1
+    %6:_(s64) = G_AND %4, %5
+    S_NOP 0, implicit %6
+...
+
+---
+name: and_chain_i64_sv
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0_sgpr1, $sgpr2_sgpr3, $vgpr0_vgpr1
+    ; CHECK-LABEL: name: and_chain_i64_sv
+    ; CHECK: [[COPY:%[0-9]+]]:sgpr(s64) = COPY $sgpr0_sgpr1
+    ; CHECK: [[COPY1:%[0-9]+]]:sgpr(s64) = COPY $sgpr2_sgpr3
+    ; CHECK: [[COPY2:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
+    ; CHECK: [[UV:%[0-9]+]]:sgpr(s32), [[UV1:%[0-9]+]]:sgpr(s32) = G_UNMERGE_VALUES [[COPY]](s64)
+    ; CHECK: [[UV2:%[0-9]+]]:vgpr(s32), [[UV3:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[COPY2]](s64)
+    ; CHECK: [[AND:%[0-9]+]]:vgpr(s32) = G_AND [[UV]], [[UV2]]
+    ; CHECK: [[AND1:%[0-9]+]]:vgpr(s32) = G_AND [[UV1]], [[UV3]]
+    ; CHECK: [[MV:%[0-9]+]]:vgpr(s64) = G_MERGE_VALUES [[AND]](s32), [[AND1]](s32)
+    ; CHECK: [[UV4:%[0-9]+]]:sgpr(s32), [[UV5:%[0-9]+]]:sgpr(s32) = G_UNMERGE_VALUES [[COPY1]](s64)
+    ; CHECK: [[UV6:%[0-9]+]]:vgpr(s32), [[UV7:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[MV]](s64)
+    ; CHECK: [[AND2:%[0-9]+]]:vgpr(s32) = G_AND [[UV4]], [[UV6]]
+    ; CHECK: [[AND3:%[0-9]+]]:vgpr(s32) = G_AND [[UV5]], [[UV7]]
+    ; CHECK: [[MV1:%[0-9]+]]:vgpr(s64) = G_MERGE_VALUES [[AND2]](s32), [[AND3]](s32)
+    ; CHECK: S_NOP 0, implicit [[MV1]](s64)
+    %0:_(s64) = COPY $sgpr0_sgpr1
+    %1:_(s64) = COPY $sgpr2_sgpr3
+    %2:_(s64) = COPY $vgpr0_vgpr1
+    %3:_(s64) = G_AND %0, %2
+    %4:_(s64) = G_AND %1, %3
+    S_NOP 0, implicit %4
+...
+
+---
+name: and_v2i32_ss
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0_sgpr1, $sgpr2_sgpr3
+    ; CHECK-LABEL: name: and_v2i32_ss
+    ; CHECK: [[COPY:%[0-9]+]]:sgpr(<2 x s32>) = COPY $sgpr0_sgpr1
+    ; CHECK: [[COPY1:%[0-9]+]]:sgpr(<2 x s32>) = COPY $sgpr2_sgpr3
+    ; CHECK: [[AND:%[0-9]+]]:sgpr(<2 x s32>) = G_AND [[COPY]], [[COPY1]]
+    ; CHECK: S_NOP 0, implicit [[AND]](<2 x s32>)
+    %0:_(<2 x s32>) = COPY $sgpr0_sgpr1
+    %1:_(<2 x s32>) = COPY $sgpr2_sgpr3
+    %2:_(<2 x s32>) = G_AND %0, %1
+    S_NOP 0, implicit %2
+...
+
+---
+name: and_v2i32_sv
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0_sgpr1, $vgpr0_vgpr1
+    ; CHECK-LABEL: name: and_v2i32_sv
+    ; CHECK: [[COPY:%[0-9]+]]:sgpr(<2 x s32>) = COPY $sgpr0_sgpr1
+    ; CHECK: [[COPY1:%[0-9]+]]:vgpr(<2 x s32>) = COPY $vgpr0_vgpr1
+    ; CHECK: [[UV:%[0-9]+]]:sgpr(s32), [[UV1:%[0-9]+]]:sgpr(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
+    ; CHECK: [[UV2:%[0-9]+]]:vgpr(s32), [[UV3:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[COPY1]](<2 x s32>)
+    ; CHECK: [[AND:%[0-9]+]]:vgpr(s32) = G_AND [[UV]], [[UV2]]
+    ; CHECK: [[AND1:%[0-9]+]]:vgpr(s32) = G_AND [[UV1]], [[UV3]]
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:vgpr(<2 x s32>) = G_BUILD_VECTOR [[AND]](s32), [[AND1]](s32)
+    ; CHECK: S_NOP 0, implicit [[BUILD_VECTOR]](<2 x s32>)
+    %0:_(<2 x s32>) = COPY $sgpr0_sgpr1
+    %1:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    %2:_(<2 x s32>) = G_AND %0, %1
+    S_NOP 0, implicit %2
+...
+
+---
+name: and_v2i32_vs
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0_sgpr1, $vgpr0_vgpr1
+
+    ; CHECK-LABEL: name: and_v2i32_vs
+    ; CHECK: [[COPY:%[0-9]+]]:vgpr(<2 x s32>) = COPY $vgpr0_vgpr1
+    ; CHECK: [[COPY1:%[0-9]+]]:sgpr(<2 x s32>) = COPY $sgpr0_sgpr1
+    ; CHECK: [[UV:%[0-9]+]]:vgpr(s32), [[UV1:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
+    ; CHECK: [[UV2:%[0-9]+]]:sgpr(s32), [[UV3:%[0-9]+]]:sgpr(s32) = G_UNMERGE_VALUES [[COPY1]](<2 x s32>)
+    ; CHECK: [[AND:%[0-9]+]]:vgpr(s32) = G_AND [[UV]], [[UV2]]
+    ; CHECK: [[AND1:%[0-9]+]]:vgpr(s32) = G_AND [[UV1]], [[UV3]]
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:vgpr(<2 x s32>) = G_BUILD_VECTOR [[AND]](s32), [[AND1]](s32)
+    ; CHECK: S_NOP 0, implicit [[BUILD_VECTOR]](<2 x s32>)
+    %0:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    %1:_(<2 x s32>) = COPY $sgpr0_sgpr1
+    %2:_(<2 x s32>) = G_AND %0, %1
+    S_NOP 0, implicit %2
+...
+
+---
+name: and_v2i32_vv
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
+    ; CHECK-LABEL: name: and_v2i32_vv
+    ; CHECK: [[COPY:%[0-9]+]]:vgpr(<2 x s32>) = COPY $vgpr0_vgpr1
+    ; CHECK: [[COPY1:%[0-9]+]]:vgpr(<2 x s32>) = COPY $vgpr2_vgpr3
+    ; CHECK: [[UV:%[0-9]+]]:vgpr(s32), [[UV1:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
+    ; CHECK: [[UV2:%[0-9]+]]:vgpr(s32), [[UV3:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[COPY1]](<2 x s32>)
+    ; CHECK: [[AND:%[0-9]+]]:vgpr(s32) = G_AND [[UV]], [[UV2]]
+    ; CHECK: [[AND1:%[0-9]+]]:vgpr(s32) = G_AND [[UV1]], [[UV3]]
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:vgpr(<2 x s32>) = G_BUILD_VECTOR [[AND]](s32), [[AND1]](s32)
+    ; CHECK: S_NOP 0, implicit [[BUILD_VECTOR]](<2 x s32>)
+    %0:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    %1:_(<2 x s32>) = COPY $vgpr2_vgpr3
+    %2:_(<2 x s32>) = G_AND %0, %1
+    S_NOP 0, implicit %2
+...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/regbankselect-ashr.mir b/test/CodeGen/AMDGPU/GlobalISel/regbankselect-ashr.mir
new file mode 100644
index 0000000000000000000000000000000000000000..f66b6f95ab005608d791397cab3ff93385e481f4
--- /dev/null
+++ b/test/CodeGen/AMDGPU/GlobalISel/regbankselect-ashr.mir
@@ -0,0 +1,68 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -march=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
+# RUN: llc -march=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
+
+---
+name: ashr_ss
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0, $sgpr1
+    ; CHECK-LABEL: name: ashr_ss
+    ; CHECK: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
+    ; CHECK: [[ASHR:%[0-9]+]]:sgpr(s32) = G_ASHR [[COPY]], [[COPY1]]
+    %0:_(s32) = COPY $sgpr0
+    %1:_(s32) = COPY $sgpr1
+    %2:_(s32) = G_ASHR %0, %1
+...
+
+---
+name: ashr_sv
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0, $vgpr0
+    ; CHECK-LABEL: name: ashr_sv
+    ; CHECK: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; CHECK: [[ASHR:%[0-9]+]]:vgpr(s32) = G_ASHR [[COPY]], [[COPY1]]
+    %0:_(s32) = COPY $sgpr0
+    %1:_(s32) = COPY $vgpr0
+    %2:_(s32) = G_ASHR %0, %1
+...
+
+---
+name: ashr_vs
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0, $vgpr0
+    ; CHECK-LABEL: name: ashr_vs
+    ; CHECK: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; CHECK: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32)
+    ; CHECK: [[ASHR:%[0-9]+]]:vgpr(s32) = G_ASHR [[COPY]], [[COPY2]]
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $sgpr0
+    %2:_(s32) = G_ASHR %0, %1
+...
+
+---
+name: ashr_vv
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+    ; CHECK-LABEL: name: ashr_vv
+    ; CHECK: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+    ; CHECK: [[ASHR:%[0-9]+]]:vgpr(s32) = G_ASHR [[COPY]], [[COPY1]]
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s32) = G_ASHR %0, %1
+...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/regbankselect-atomic-cmpxchg.mir b/test/CodeGen/AMDGPU/GlobalISel/regbankselect-atomic-cmpxchg.mir
new file mode 100644
index 0000000000000000000000000000000000000000..4681284163f2fce532213ce42aae5026b2681442
--- /dev/null
+++ b/test/CodeGen/AMDGPU/GlobalISel/regbankselect-atomic-cmpxchg.mir
@@ -0,0 +1,66 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -march=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
+# RUN: llc -march=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
+
+---
+name: atomic_cmpxchg_global_i32_sss
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0_sgpr1, $sgpr2, $sgpr3
+    ; CHECK-LABEL: name: atomic_cmpxchg_global_i32_sss
+    ; CHECK: [[COPY:%[0-9]+]]:sgpr(p1) = COPY $sgpr0_sgpr1
+    ; CHECK: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr2
+    ; CHECK: [[COPY2:%[0-9]+]]:sgpr(s32) = COPY $sgpr3
+    ; CHECK: [[COPY3:%[0-9]+]]:vgpr(p1) = COPY [[COPY]](p1)
+    ; CHECK: [[COPY4:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32)
+    ; CHECK: [[COPY5:%[0-9]+]]:vgpr(s32) = COPY [[COPY2]](s32)
+    ; CHECK: [[ATOMIC_CMPXCHG:%[0-9]+]]:vgpr(s32) = G_ATOMIC_CMPXCHG [[COPY3]](p1), [[COPY4]], [[COPY5]] :: (load store seq_cst 4, addrspace 1)
+    %0:_(p1) = COPY $sgpr0_sgpr1
+    %1:_(s32) = COPY $sgpr2
+    %2:_(s32) = COPY $sgpr3
+    %3:_(s32) = G_ATOMIC_CMPXCHG %0, %1, %2 :: (load store seq_cst 4, addrspace 1)
+...
+
+---
+name: atomic_cmpxchg_flat_i32_sss
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0_sgpr1, $sgpr2, $sgpr3
+    ; CHECK-LABEL: name: atomic_cmpxchg_flat_i32_sss
+    ; CHECK: [[COPY:%[0-9]+]]:sgpr(p0) = COPY $sgpr0_sgpr1
+    ; CHECK: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr2
+    ; CHECK: [[COPY2:%[0-9]+]]:sgpr(s32) = COPY $sgpr3
+    ; CHECK: [[COPY3:%[0-9]+]]:vgpr(p0) = COPY [[COPY]](p0)
+    ; CHECK: [[COPY4:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32)
+    ; CHECK: [[COPY5:%[0-9]+]]:vgpr(s32) = COPY [[COPY2]](s32)
+    ; CHECK: [[ATOMIC_CMPXCHG:%[0-9]+]]:vgpr(s32) = G_ATOMIC_CMPXCHG [[COPY3]](p0), [[COPY4]], [[COPY5]] :: (load store seq_cst 4)
+    %0:_(p0) = COPY $sgpr0_sgpr1
+    %1:_(s32) = COPY $sgpr2
+    %2:_(s32) = COPY $sgpr3
+    %3:_(s32) = G_ATOMIC_CMPXCHG %0, %1, %2 :: (load store seq_cst 4, addrspace 0)
+...
+
+---
+name: atomic_cmpxchg_local_i32_sss
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0, $sgpr1, $sgpr2
+    ; CHECK-LABEL: name: atomic_cmpxchg_local_i32_sss
+    ; CHECK: [[COPY:%[0-9]+]]:sgpr(p3) = COPY $sgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
+    ; CHECK: [[COPY2:%[0-9]+]]:sgpr(s32) = COPY $sgpr2
+    ; CHECK: [[COPY3:%[0-9]+]]:vgpr(p3) = COPY [[COPY]](p3)
+    ; CHECK: [[COPY4:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32)
+    ; CHECK: [[COPY5:%[0-9]+]]:vgpr(s32) = COPY [[COPY2]](s32)
+    ; CHECK: [[ATOMIC_CMPXCHG:%[0-9]+]]:vgpr(s32) = G_ATOMIC_CMPXCHG [[COPY3]](p3), [[COPY4]], [[COPY5]] :: (load store seq_cst 4, addrspace 3)
+    %0:_(p3) = COPY $sgpr0
+    %1:_(s32) = COPY $sgpr1
+    %2:_(s32) = COPY $sgpr2
+    %3:_(s32) = G_ATOMIC_CMPXCHG %0, %1, %2 :: (load store seq_cst 4, addrspace 3)
+...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/regbankselect-atomicrmw-add.mir b/test/CodeGen/AMDGPU/GlobalISel/regbankselect-atomicrmw-add.mir
new file mode 100644
index 0000000000000000000000000000000000000000..fdcb8ffbc7ca3cba57e40df0290a1439dd6d4c65
--- /dev/null
+++ b/test/CodeGen/AMDGPU/GlobalISel/regbankselect-atomicrmw-add.mir
@@ -0,0 +1,57 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -march=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
+# RUN: llc -march=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
+
+---
+name: atomicrmw_add_global_i32_ss
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0_sgpr1, $sgpr2
+    ; CHECK-LABEL: name: atomicrmw_add_global_i32_ss
+    ; CHECK: [[COPY:%[0-9]+]]:sgpr(p1) = COPY $sgpr0_sgpr1
+    ; CHECK: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr2
+    ; CHECK: [[COPY2:%[0-9]+]]:vgpr(p1) = COPY [[COPY]](p1)
+    ; CHECK: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32)
+    ; CHECK: [[ATOMICRMW_ADD:%[0-9]+]]:vgpr(s32) = G_ATOMICRMW_ADD [[COPY2]](p1), [[COPY3]] :: (load store seq_cst 4, addrspace 1)
+    %0:_(p1) = COPY $sgpr0_sgpr1
+    %1:_(s32) = COPY $sgpr2
+    %2:_(s32) = G_ATOMICRMW_ADD %0, %1 :: (load store seq_cst 4, addrspace 1)
+...
+
+---
+name: atomicrmw_add_flat_i32_ss
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0_sgpr1, $sgpr2
+    ; CHECK-LABEL: name: atomicrmw_add_flat_i32_ss
+    ; CHECK: [[COPY:%[0-9]+]]:sgpr(p0) = COPY $sgpr0_sgpr1
+    ; CHECK: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr2
+    ; CHECK: [[COPY2:%[0-9]+]]:vgpr(p0) = COPY [[COPY]](p0)
+    ; CHECK: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32)
+    ; CHECK: [[ATOMICRMW_ADD:%[0-9]+]]:vgpr(s32) = G_ATOMICRMW_ADD [[COPY2]](p0), [[COPY3]] :: (load store seq_cst 4)
+    %0:_(p0) = COPY $sgpr0_sgpr1
+    %1:_(s32) = COPY $sgpr2
+    %2:_(s32) = G_ATOMICRMW_ADD %0, %1 :: (load store seq_cst 4, addrspace 0)
+...
+
+---
+name: atomicrmw_add_local_i32_ss
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0, $sgpr1
+    ; CHECK-LABEL: name: atomicrmw_add_local_i32_ss
+    ; CHECK: [[COPY:%[0-9]+]]:sgpr(p3) = COPY $sgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
+    ; CHECK: [[COPY2:%[0-9]+]]:vgpr(p3) = COPY [[COPY]](p3)
+    ; CHECK: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32)
+    ; CHECK: [[ATOMICRMW_ADD:%[0-9]+]]:vgpr(s32) = G_ATOMICRMW_ADD [[COPY2]](p3), [[COPY3]] :: (load store seq_cst 4, addrspace 3)
+    %0:_(p3) = COPY $sgpr0
+    %1:_(s32) = COPY $sgpr1
+    %2:_(s32) = G_ATOMICRMW_ADD %0, %1 :: (load store seq_cst 4, addrspace 3)
+...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/regbankselect-atomicrmw-and.mir b/test/CodeGen/AMDGPU/GlobalISel/regbankselect-atomicrmw-and.mir
new file mode 100644
index 0000000000000000000000000000000000000000..c075f7bbe0ccef94d92736037c218041b417a4ee
--- /dev/null
+++ b/test/CodeGen/AMDGPU/GlobalISel/regbankselect-atomicrmw-and.mir
@@ -0,0 +1,57 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -march=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
+# RUN: llc -march=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
+
+---
+name: atomicrmw_and_global_i32_ss
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0_sgpr1, $sgpr2
+    ; CHECK-LABEL: name: atomicrmw_and_global_i32_ss
+    ; CHECK: [[COPY:%[0-9]+]]:sgpr(p1) = COPY $sgpr0_sgpr1
+    ; CHECK: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr2
+    ; CHECK: [[COPY2:%[0-9]+]]:vgpr(p1) = COPY [[COPY]](p1)
+    ; CHECK: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32)
+    ; CHECK: [[ATOMICRMW_ADD:%[0-9]+]]:vgpr(s32) = G_ATOMICRMW_ADD [[COPY2]](p1), [[COPY3]] :: (load store seq_cst 4, addrspace 1)
+    %0:_(p1) = COPY $sgpr0_sgpr1
+    %1:_(s32) = COPY $sgpr2
+    %2:_(s32) = G_ATOMICRMW_ADD %0, %1 :: (load store seq_cst 4, addrspace 1)
+...
+
+---
+name: atomicrmw_and_flat_i32_ss
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0_sgpr1, $sgpr2
+    ; CHECK-LABEL: name: atomicrmw_and_flat_i32_ss
+    ; CHECK: [[COPY:%[0-9]+]]:sgpr(p0) = COPY $sgpr0_sgpr1
+    ; CHECK: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr2
+    ; CHECK: [[COPY2:%[0-9]+]]:vgpr(p0) = COPY [[COPY]](p0)
+    ; CHECK: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32)
+    ; CHECK: [[ATOMICRMW_ADD:%[0-9]+]]:vgpr(s32) = G_ATOMICRMW_ADD [[COPY2]](p0), [[COPY3]] :: (load store seq_cst 4)
+    %0:_(p0) = COPY $sgpr0_sgpr1
+    %1:_(s32) = COPY $sgpr2
+    %2:_(s32) = G_ATOMICRMW_ADD %0, %1 :: (load store seq_cst 4, addrspace 0)
+...
+
+---
+name: atomicrmw_and_local_i32_ss
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0, $sgpr1
+    ; CHECK-LABEL: name: atomicrmw_and_local_i32_ss
+    ; CHECK: [[COPY:%[0-9]+]]:sgpr(p3) = COPY $sgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
+    ; CHECK: [[COPY2:%[0-9]+]]:vgpr(p3) = COPY [[COPY]](p3)
+    ; CHECK: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32)
+    ; CHECK: [[ATOMICRMW_ADD:%[0-9]+]]:vgpr(s32) = G_ATOMICRMW_ADD [[COPY2]](p3), [[COPY3]] :: (load store seq_cst 4, addrspace 3)
+    %0:_(p3) = COPY $sgpr0
+    %1:_(s32) = COPY $sgpr1
+    %2:_(s32) = G_ATOMICRMW_ADD %0, %1 :: (load store seq_cst 4, addrspace 3)
+...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/regbankselect-atomicrmw-max.mir b/test/CodeGen/AMDGPU/GlobalISel/regbankselect-atomicrmw-max.mir
new file mode 100644
index 0000000000000000000000000000000000000000..67e932d94b8e41f5d99c1b3177fe1fd6bbae2233
--- /dev/null
+++ b/test/CodeGen/AMDGPU/GlobalISel/regbankselect-atomicrmw-max.mir
@@ -0,0 +1,57 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -march=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
+# RUN: llc -march=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
+
+---
+name: atomicrmw_max_global_i32_ss
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0_sgpr1, $sgpr2
+    ; CHECK-LABEL: name: atomicrmw_max_global_i32_ss
+    ; CHECK: [[COPY:%[0-9]+]]:sgpr(p1) = COPY $sgpr0_sgpr1
+    ; CHECK: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr2
+    ; CHECK: [[COPY2:%[0-9]+]]:vgpr(p1) = COPY [[COPY]](p1)
+    ; CHECK: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32)
+    ; CHECK: [[ATOMICRMW_ADD:%[0-9]+]]:vgpr(s32) = G_ATOMICRMW_ADD [[COPY2]](p1), [[COPY3]] :: (load store seq_cst 4, addrspace 1)
+    %0:_(p1) = COPY $sgpr0_sgpr1
+    %1:_(s32) = COPY $sgpr2
+    %2:_(s32) = G_ATOMICRMW_ADD %0, %1 :: (load store seq_cst 4, addrspace 1)
+...
+
+---
+name: atomicrmw_max_flat_i32_ss
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0_sgpr1, $sgpr2
+    ; CHECK-LABEL: name: atomicrmw_max_flat_i32_ss
+    ; CHECK: [[COPY:%[0-9]+]]:sgpr(p0) = COPY $sgpr0_sgpr1
+    ; CHECK: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr2
+    ; CHECK: [[COPY2:%[0-9]+]]:vgpr(p0) = COPY [[COPY]](p0)
+    ; CHECK: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32)
+    ; CHECK: [[ATOMICRMW_ADD:%[0-9]+]]:vgpr(s32) = G_ATOMICRMW_ADD [[COPY2]](p0), [[COPY3]] :: (load store seq_cst 4)
+    %0:_(p0) = COPY $sgpr0_sgpr1
+    %1:_(s32) = COPY $sgpr2
+    %2:_(s32) = G_ATOMICRMW_ADD %0, %1 :: (load store seq_cst 4, addrspace 0)
+...
+
+---
+name: atomicrmw_max_local_i32_ss
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0, $sgpr1
+    ; CHECK-LABEL: name: atomicrmw_max_local_i32_ss
+    ; CHECK: [[COPY:%[0-9]+]]:sgpr(p3) = COPY $sgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
+    ; CHECK: [[COPY2:%[0-9]+]]:vgpr(p3) = COPY [[COPY]](p3)
+    ; CHECK: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32)
+    ; CHECK: [[ATOMICRMW_ADD:%[0-9]+]]:vgpr(s32) = G_ATOMICRMW_ADD [[COPY2]](p3), [[COPY3]] :: (load store seq_cst 4, addrspace 3)
+    %0:_(p3) = COPY $sgpr0
+    %1:_(s32) = COPY $sgpr1
+    %2:_(s32) = G_ATOMICRMW_ADD %0, %1 :: (load store seq_cst 4, addrspace 3)
+...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/regbankselect-atomicrmw-min.mir b/test/CodeGen/AMDGPU/GlobalISel/regbankselect-atomicrmw-min.mir
new file mode 100644
index 0000000000000000000000000000000000000000..749f773fb1073f2feaf4c04475660b9c062d75a1
--- /dev/null
+++ b/test/CodeGen/AMDGPU/GlobalISel/regbankselect-atomicrmw-min.mir
@@ -0,0 +1,57 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -march=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
+# RUN: llc -march=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
+
+---
+name: atomicrmw_min_global_i32_ss
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0_sgpr1, $sgpr2
+    ; CHECK-LABEL: name: atomicrmw_min_global_i32_ss
+    ; CHECK: [[COPY:%[0-9]+]]:sgpr(p1) = COPY $sgpr0_sgpr1
+    ; CHECK: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr2
+    ; CHECK: [[COPY2:%[0-9]+]]:vgpr(p1) = COPY [[COPY]](p1)
+    ; CHECK: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32)
+    ; CHECK: [[ATOMICRMW_ADD:%[0-9]+]]:vgpr(s32) = G_ATOMICRMW_ADD [[COPY2]](p1), [[COPY3]] :: (load store seq_cst 4, addrspace 1)
+    %0:_(p1) = COPY $sgpr0_sgpr1
+    %1:_(s32) = COPY $sgpr2
+    %2:_(s32) = G_ATOMICRMW_ADD %0, %1 :: (load store seq_cst 4, addrspace 1)
+...
+
+---
+name: atomicrmw_min_flat_i32_ss
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0_sgpr1, $sgpr2
+    ; CHECK-LABEL: name: atomicrmw_min_flat_i32_ss
+    ; CHECK: [[COPY:%[0-9]+]]:sgpr(p0) = COPY $sgpr0_sgpr1
+    ; CHECK: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr2
+    ; CHECK: [[COPY2:%[0-9]+]]:vgpr(p0) = COPY [[COPY]](p0)
+    ; CHECK: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32)
+    ; CHECK: [[ATOMICRMW_ADD:%[0-9]+]]:vgpr(s32) = G_ATOMICRMW_ADD [[COPY2]](p0), [[COPY3]] :: (load store seq_cst 4)
+    %0:_(p0) = COPY $sgpr0_sgpr1
+    %1:_(s32) = COPY $sgpr2
+    %2:_(s32) = G_ATOMICRMW_ADD %0, %1 :: (load store seq_cst 4, addrspace 0)
+...
+
+---
+name: atomicrmw_min_local_i32_ss
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0, $sgpr1
+    ; CHECK-LABEL: name: atomicrmw_min_local_i32_ss
+    ; CHECK: [[COPY:%[0-9]+]]:sgpr(p3) = COPY $sgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
+    ; CHECK: [[COPY2:%[0-9]+]]:vgpr(p3) = COPY [[COPY]](p3)
+    ; CHECK: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32)
+    ; CHECK: [[ATOMICRMW_ADD:%[0-9]+]]:vgpr(s32) = G_ATOMICRMW_ADD [[COPY2]](p3), [[COPY3]] :: (load store seq_cst 4, addrspace 3)
+    %0:_(p3) = COPY $sgpr0
+    %1:_(s32) = COPY $sgpr1
+    %2:_(s32) = G_ATOMICRMW_ADD %0, %1 :: (load store seq_cst 4, addrspace 3)
+...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/regbankselect-atomicrmw-or.mir b/test/CodeGen/AMDGPU/GlobalISel/regbankselect-atomicrmw-or.mir
new file mode 100644
index 0000000000000000000000000000000000000000..5fbd24e8b5c10f6b513bdcbea5a88c732dec4724
--- /dev/null
+++ b/test/CodeGen/AMDGPU/GlobalISel/regbankselect-atomicrmw-or.mir
@@ -0,0 +1,57 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -march=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
+# RUN: llc -march=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
+
+---
+name: atomicrmw_or_global_i32_ss
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0_sgpr1, $sgpr2
+    ; CHECK-LABEL: name: atomicrmw_or_global_i32_ss
+    ; CHECK: [[COPY:%[0-9]+]]:sgpr(p1) = COPY $sgpr0_sgpr1
+    ; CHECK: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr2
+    ; CHECK: [[COPY2:%[0-9]+]]:vgpr(p1) = COPY [[COPY]](p1)
+    ; CHECK: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32)
+    ; CHECK: [[ATOMICRMW_ADD:%[0-9]+]]:vgpr(s32) = G_ATOMICRMW_ADD [[COPY2]](p1), [[COPY3]] :: (load store seq_cst 4, addrspace 1)
+    %0:_(p1) = COPY $sgpr0_sgpr1
+    %1:_(s32) = COPY $sgpr2
+    %2:_(s32) = G_ATOMICRMW_ADD %0, %1 :: (load store seq_cst 4, addrspace 1)
+...
+
+---
+name: atomicrmw_or_flat_i32_ss
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0_sgpr1, $sgpr2
+    ; CHECK-LABEL: name: atomicrmw_or_flat_i32_ss
+    ; CHECK: [[COPY:%[0-9]+]]:sgpr(p0) = COPY $sgpr0_sgpr1
+    ; CHECK: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr2
+    ; CHECK: [[COPY2:%[0-9]+]]:vgpr(p0) = COPY [[COPY]](p0)
+    ; CHECK: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32)
+    ; CHECK: [[ATOMICRMW_ADD:%[0-9]+]]:vgpr(s32) = G_ATOMICRMW_ADD [[COPY2]](p0), [[COPY3]] :: (load store seq_cst 4)
+    %0:_(p0) = COPY $sgpr0_sgpr1
+    %1:_(s32) = COPY $sgpr2
+    %2:_(s32) = G_ATOMICRMW_ADD %0, %1 :: (load store seq_cst 4, addrspace 0)
+...
+
+---
+name: atomicrmw_or_local_i32_ss
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0, $sgpr1
+    ; CHECK-LABEL: name: atomicrmw_or_local_i32_ss
+    ; CHECK: [[COPY:%[0-9]+]]:sgpr(p3) = COPY $sgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
+    ; CHECK: [[COPY2:%[0-9]+]]:vgpr(p3) = COPY [[COPY]](p3)
+    ; CHECK: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32)
+    ; CHECK: [[ATOMICRMW_ADD:%[0-9]+]]:vgpr(s32) = G_ATOMICRMW_ADD [[COPY2]](p3), [[COPY3]] :: (load store seq_cst 4, addrspace 3)
+    %0:_(p3) = COPY $sgpr0
+    %1:_(s32) = COPY $sgpr1
+    %2:_(s32) = G_ATOMICRMW_ADD %0, %1 :: (load store seq_cst 4, addrspace 3)
+...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/regbankselect-atomicrmw-sub.mir b/test/CodeGen/AMDGPU/GlobalISel/regbankselect-atomicrmw-sub.mir
new file mode 100644
index 0000000000000000000000000000000000000000..0fa3243cb5f78578dc2647a306f5754e4f9fed34
--- /dev/null
+++ b/test/CodeGen/AMDGPU/GlobalISel/regbankselect-atomicrmw-sub.mir
@@ -0,0 +1,57 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -march=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
+# RUN: llc -march=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
+
+---
+name: atomicrmw_sub_global_i32_ss
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0_sgpr1, $sgpr2
+    ; CHECK-LABEL: name: atomicrmw_sub_global_i32_ss
+    ; CHECK: [[COPY:%[0-9]+]]:sgpr(p1) = COPY $sgpr0_sgpr1
+    ; CHECK: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr2
+    ; CHECK: [[COPY2:%[0-9]+]]:vgpr(p1) = COPY [[COPY]](p1)
+    ; CHECK: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32)
+    ; CHECK: [[ATOMICRMW_ADD:%[0-9]+]]:vgpr(s32) = G_ATOMICRMW_ADD [[COPY2]](p1), [[COPY3]] :: (load store seq_cst 4, addrspace 1)
+    %0:_(p1) = COPY $sgpr0_sgpr1
+    %1:_(s32) = COPY $sgpr2
+    %2:_(s32) = G_ATOMICRMW_ADD %0, %1 :: (load store seq_cst 4, addrspace 1)
+...
+
+---
+name: atomicrmw_sub_flat_i32_ss
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0_sgpr1, $sgpr2
+    ; CHECK-LABEL: name: atomicrmw_sub_flat_i32_ss
+    ; CHECK: [[COPY:%[0-9]+]]:sgpr(p0) = COPY $sgpr0_sgpr1
+    ; CHECK: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr2
+    ; CHECK: [[COPY2:%[0-9]+]]:vgpr(p0) = COPY [[COPY]](p0)
+    ; CHECK: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32)
+    ; CHECK: [[ATOMICRMW_ADD:%[0-9]+]]:vgpr(s32) = G_ATOMICRMW_ADD [[COPY2]](p0), [[COPY3]] :: (load store seq_cst 4)
+    %0:_(p0) = COPY $sgpr0_sgpr1
+    %1:_(s32) = COPY $sgpr2
+    %2:_(s32) = G_ATOMICRMW_ADD %0, %1 :: (load store seq_cst 4, addrspace 0)
+...
+
+---
+name: atomicrmw_sub_local_i32_ss
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0, $sgpr1
+    ; CHECK-LABEL: name: atomicrmw_sub_local_i32_ss
+    ; CHECK: [[COPY:%[0-9]+]]:sgpr(p3) = COPY $sgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
+    ; CHECK: [[COPY2:%[0-9]+]]:vgpr(p3) = COPY [[COPY]](p3)
+    ; CHECK: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32)
+    ; CHECK: [[ATOMICRMW_ADD:%[0-9]+]]:vgpr(s32) = G_ATOMICRMW_ADD [[COPY2]](p3), [[COPY3]] :: (load store seq_cst 4, addrspace 3)
+    %0:_(p3) = COPY $sgpr0
+    %1:_(s32) = COPY $sgpr1
+    %2:_(s32) = G_ATOMICRMW_ADD %0, %1 :: (load store seq_cst 4, addrspace 3)
+...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/regbankselect-atomicrmw-umax.mir b/test/CodeGen/AMDGPU/GlobalISel/regbankselect-atomicrmw-umax.mir
new file mode 100644
index 0000000000000000000000000000000000000000..866c1e09340a966b3d8cd08809b5912cdf7b446b
--- /dev/null
+++ b/test/CodeGen/AMDGPU/GlobalISel/regbankselect-atomicrmw-umax.mir
@@ -0,0 +1,57 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -march=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
+# RUN: llc -march=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
+
+---
+name: atomicrmw_umax_global_i32_ss
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0_sgpr1, $sgpr2
+    ; CHECK-LABEL: name: atomicrmw_umax_global_i32_ss
+    ; CHECK: [[COPY:%[0-9]+]]:sgpr(p1) = COPY $sgpr0_sgpr1
+    ; CHECK: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr2
+    ; CHECK: [[COPY2:%[0-9]+]]:vgpr(p1) = COPY [[COPY]](p1)
+    ; CHECK: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32)
+    ; CHECK: [[ATOMICRMW_ADD:%[0-9]+]]:vgpr(s32) = G_ATOMICRMW_ADD [[COPY2]](p1), [[COPY3]] :: (load store seq_cst 4, addrspace 1)
+    %0:_(p1) = COPY $sgpr0_sgpr1
+    %1:_(s32) = COPY $sgpr2
+    %2:_(s32) = G_ATOMICRMW_ADD %0, %1 :: (load store seq_cst 4, addrspace 1)
+...
+
+---
+name: atomicrmw_umax_flat_i32_ss
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0_sgpr1, $sgpr2
+    ; CHECK-LABEL: name: atomicrmw_umax_flat_i32_ss
+    ; CHECK: [[COPY:%[0-9]+]]:sgpr(p0) = COPY $sgpr0_sgpr1
+    ; CHECK: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr2
+    ; CHECK: [[COPY2:%[0-9]+]]:vgpr(p0) = COPY [[COPY]](p0)
+    ; CHECK: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32)
+    ; CHECK: [[ATOMICRMW_ADD:%[0-9]+]]:vgpr(s32) = G_ATOMICRMW_ADD [[COPY2]](p0), [[COPY3]] :: (load store seq_cst 4)
+    %0:_(p0) = COPY $sgpr0_sgpr1
+    %1:_(s32) = COPY $sgpr2
+    %2:_(s32) = G_ATOMICRMW_ADD %0, %1 :: (load store seq_cst 4, addrspace 0)
+...
+
+---
+name: atomicrmw_umax_local_i32_ss
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0, $sgpr1
+    ; CHECK-LABEL: name: atomicrmw_umax_local_i32_ss
+    ; CHECK: [[COPY:%[0-9]+]]:sgpr(p3) = COPY $sgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
+    ; CHECK: [[COPY2:%[0-9]+]]:vgpr(p3) = COPY [[COPY]](p3)
+    ; CHECK: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32)
+    ; CHECK: [[ATOMICRMW_ADD:%[0-9]+]]:vgpr(s32) = G_ATOMICRMW_ADD [[COPY2]](p3), [[COPY3]] :: (load store seq_cst 4, addrspace 3)
+    %0:_(p3) = COPY $sgpr0
+    %1:_(s32) = COPY $sgpr1
+    %2:_(s32) = G_ATOMICRMW_ADD %0, %1 :: (load store seq_cst 4, addrspace 3)
+...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/regbankselect-atomicrmw-umin.mir b/test/CodeGen/AMDGPU/GlobalISel/regbankselect-atomicrmw-umin.mir
new file mode 100644
index 0000000000000000000000000000000000000000..0de2e8989b6d98ba657b69ce24f28bac8d9e6add
--- /dev/null
+++ b/test/CodeGen/AMDGPU/GlobalISel/regbankselect-atomicrmw-umin.mir
@@ -0,0 +1,57 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -march=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
+# RUN: llc -march=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
+
+---
+name: atomicrmw_umin_global_i32_ss
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0_sgpr1, $sgpr2
+    ; CHECK-LABEL: name: atomicrmw_umin_global_i32_ss
+    ; CHECK: [[COPY:%[0-9]+]]:sgpr(p1) = COPY $sgpr0_sgpr1
+    ; CHECK: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr2
+    ; CHECK: [[COPY2:%[0-9]+]]:vgpr(p1) = COPY [[COPY]](p1)
+    ; CHECK: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32)
+    ; CHECK: [[ATOMICRMW_ADD:%[0-9]+]]:vgpr(s32) = G_ATOMICRMW_ADD [[COPY2]](p1), [[COPY3]] :: (load store seq_cst 4, addrspace 1)
+    %0:_(p1) = COPY $sgpr0_sgpr1
+    %1:_(s32) = COPY $sgpr2
+    %2:_(s32) = G_ATOMICRMW_ADD %0, %1 :: (load store seq_cst 4, addrspace 1)
+...
+
+---
+name: atomicrmw_umin_flat_i32_ss
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0_sgpr1, $sgpr2
+    ; CHECK-LABEL: name: atomicrmw_umin_flat_i32_ss
+    ; CHECK: [[COPY:%[0-9]+]]:sgpr(p0) = COPY $sgpr0_sgpr1
+    ; CHECK: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr2
+    ; CHECK: [[COPY2:%[0-9]+]]:vgpr(p0) = COPY [[COPY]](p0)
+    ; CHECK: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32)
+    ; CHECK: [[ATOMICRMW_ADD:%[0-9]+]]:vgpr(s32) = G_ATOMICRMW_ADD [[COPY2]](p0), [[COPY3]] :: (load store seq_cst 4)
+    %0:_(p0) = COPY $sgpr0_sgpr1
+    %1:_(s32) = COPY $sgpr2
+    %2:_(s32) = G_ATOMICRMW_ADD %0, %1 :: (load store seq_cst 4, addrspace 0)
+...
+
+---
+name: atomicrmw_umin_local_i32_ss
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0, $sgpr1
+    ; CHECK-LABEL: name: atomicrmw_umin_local_i32_ss
+    ; CHECK: [[COPY:%[0-9]+]]:sgpr(p3) = COPY $sgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
+    ; CHECK: [[COPY2:%[0-9]+]]:vgpr(p3) = COPY [[COPY]](p3)
+    ; CHECK: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32)
+    ; CHECK: [[ATOMICRMW_ADD:%[0-9]+]]:vgpr(s32) = G_ATOMICRMW_ADD [[COPY2]](p3), [[COPY3]] :: (load store seq_cst 4, addrspace 3)
+    %0:_(p3) = COPY $sgpr0
+    %1:_(s32) = COPY $sgpr1
+    %2:_(s32) = G_ATOMICRMW_ADD %0, %1 :: (load store seq_cst 4, addrspace 3)
+...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/regbankselect-atomicrmw-xchg.mir b/test/CodeGen/AMDGPU/GlobalISel/regbankselect-atomicrmw-xchg.mir
new file mode 100644
index 0000000000000000000000000000000000000000..d7d16afed0b5289abf4dbab7b928dcbc12c340d6
--- /dev/null
+++ b/test/CodeGen/AMDGPU/GlobalISel/regbankselect-atomicrmw-xchg.mir
@@ -0,0 +1,57 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -march=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
+# RUN: llc -march=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
+
+---
+name: atomicrmw_xchg_global_i32_ss
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0_sgpr1, $sgpr2
+    ; CHECK-LABEL: name: atomicrmw_xchg_global_i32_ss
+    ; CHECK: [[COPY:%[0-9]+]]:sgpr(p1) = COPY $sgpr0_sgpr1
+    ; CHECK: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr2
+    ; CHECK: [[COPY2:%[0-9]+]]:vgpr(p1) = COPY [[COPY]](p1)
+    ; CHECK: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32)
+    ; CHECK: [[ATOMICRMW_ADD:%[0-9]+]]:vgpr(s32) = G_ATOMICRMW_ADD [[COPY2]](p1), [[COPY3]] :: (load store seq_cst 4, addrspace 1)
+    %0:_(p1) = COPY $sgpr0_sgpr1
+    %1:_(s32) = COPY $sgpr2
+    %2:_(s32) = G_ATOMICRMW_ADD %0, %1 :: (load store seq_cst 4, addrspace 1)
+...
+
+---
+name: atomicrmw_xchg_flat_i32_ss
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0_sgpr1, $sgpr2
+    ; CHECK-LABEL: name: atomicrmw_xchg_flat_i32_ss
+    ; CHECK: [[COPY:%[0-9]+]]:sgpr(p0) = COPY $sgpr0_sgpr1
+    ; CHECK: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr2
+    ; CHECK: [[COPY2:%[0-9]+]]:vgpr(p0) = COPY [[COPY]](p0)
+    ; CHECK: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32)
+    ; CHECK: [[ATOMICRMW_ADD:%[0-9]+]]:vgpr(s32) = G_ATOMICRMW_ADD [[COPY2]](p0), [[COPY3]] :: (load store seq_cst 4)
+    %0:_(p0) = COPY $sgpr0_sgpr1
+    %1:_(s32) = COPY $sgpr2
+    %2:_(s32) = G_ATOMICRMW_ADD %0, %1 :: (load store seq_cst 4, addrspace 0)
+...
+
+---
+name: atomicrmw_xchg_local_i32_ss
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0, $sgpr1
+    ; CHECK-LABEL: name: atomicrmw_xchg_local_i32_ss
+    ; CHECK: [[COPY:%[0-9]+]]:sgpr(p3) = COPY $sgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
+    ; CHECK: [[COPY2:%[0-9]+]]:vgpr(p3) = COPY [[COPY]](p3)
+    ; CHECK: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32)
+    ; CHECK: [[ATOMICRMW_ADD:%[0-9]+]]:vgpr(s32) = G_ATOMICRMW_ADD [[COPY2]](p3), [[COPY3]] :: (load store seq_cst 4, addrspace 3)
+    %0:_(p3) = COPY $sgpr0
+    %1:_(s32) = COPY $sgpr1
+    %2:_(s32) = G_ATOMICRMW_ADD %0, %1 :: (load store seq_cst 4, addrspace 3)
+...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/regbankselect-atomicrmw-xor.mir b/test/CodeGen/AMDGPU/GlobalISel/regbankselect-atomicrmw-xor.mir
new file mode 100644
index 0000000000000000000000000000000000000000..c0b7206cd5bc0b153cc5aaac37912f0058ceac23
--- /dev/null
+++ b/test/CodeGen/AMDGPU/GlobalISel/regbankselect-atomicrmw-xor.mir
@@ -0,0 +1,57 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -march=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
+# RUN: llc -march=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
+
+---
+name: atomicrmw_xor_global_i32_ss
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0_sgpr1, $sgpr2
+    ; CHECK-LABEL: name: atomicrmw_xor_global_i32_ss
+    ; CHECK: [[COPY:%[0-9]+]]:sgpr(p1) = COPY $sgpr0_sgpr1
+    ; CHECK: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr2
+    ; CHECK: [[COPY2:%[0-9]+]]:vgpr(p1) = COPY [[COPY]](p1)
+    ; CHECK: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32)
+    ; CHECK: [[ATOMICRMW_ADD:%[0-9]+]]:vgpr(s32) = G_ATOMICRMW_ADD [[COPY2]](p1), [[COPY3]] :: (load store seq_cst 4, addrspace 1)
+    %0:_(p1) = COPY $sgpr0_sgpr1
+    %1:_(s32) = COPY $sgpr2
+    %2:_(s32) = G_ATOMICRMW_ADD %0, %1 :: (load store seq_cst 4, addrspace 1)
+...
+
+---
+name: atomicrmw_xor_flat_i32_ss
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0_sgpr1, $sgpr2
+    ; CHECK-LABEL: name: atomicrmw_xor_flat_i32_ss
+    ; CHECK: [[COPY:%[0-9]+]]:sgpr(p0) = COPY $sgpr0_sgpr1
+    ; CHECK: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr2
+    ; CHECK: [[COPY2:%[0-9]+]]:vgpr(p0) = COPY [[COPY]](p0)
+    ; CHECK: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32)
+    ; CHECK: [[ATOMICRMW_ADD:%[0-9]+]]:vgpr(s32) = G_ATOMICRMW_ADD [[COPY2]](p0), [[COPY3]] :: (load store seq_cst 4)
+    %0:_(p0) = COPY $sgpr0_sgpr1
+    %1:_(s32) = COPY $sgpr2
+    %2:_(s32) = G_ATOMICRMW_ADD %0, %1 :: (load store seq_cst 4, addrspace 0)
+...
+
+---
+name: atomicrmw_xor_local_i32_ss
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0, $sgpr1
+    ; CHECK-LABEL: name: atomicrmw_xor_local_i32_ss
+    ; CHECK: [[COPY:%[0-9]+]]:sgpr(p3) = COPY $sgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
+    ; CHECK: [[COPY2:%[0-9]+]]:vgpr(p3) = COPY [[COPY]](p3)
+    ; CHECK: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32)
+    ; CHECK: [[ATOMICRMW_ADD:%[0-9]+]]:vgpr(s32) = G_ATOMICRMW_ADD [[COPY2]](p3), [[COPY3]] :: (load store seq_cst 4, addrspace 3)
+    %0:_(p3) = COPY $sgpr0
+    %1:_(s32) = COPY $sgpr1
+    %2:_(s32) = G_ATOMICRMW_ADD %0, %1 :: (load store seq_cst 4, addrspace 3)
+...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/regbankselect-brcond.mir b/test/CodeGen/AMDGPU/GlobalISel/regbankselect-brcond.mir
new file mode 100644
index 0000000000000000000000000000000000000000..8b2ef915cd20ee973c447b8078150db7ce86ac62
--- /dev/null
+++ b/test/CodeGen/AMDGPU/GlobalISel/regbankselect-brcond.mir
@@ -0,0 +1,180 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -march=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
+# RUN: llc -march=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
+
+---
+name: brcond_vcc_cond
+legalized: true
+body:             |
+  ; CHECK-LABEL: name: brcond_vcc_cond
+  ; CHECK: bb.0.entry:
+  ; CHECK:   successors: %bb.1(0x80000000)
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+  ; CHECK:   [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(ne), [[COPY]](s32), [[COPY1]]
+  ; CHECK:   G_BRCOND [[ICMP]](s1), %bb.1
+  ; CHECK: bb.1:
+  bb.0.entry:
+    successors: %bb.1
+    liveins: $vgpr0, $vgpr1
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s1) = G_ICMP intpred(ne), %0, %1
+    G_BRCOND %2, %bb.1
+
+  bb.1:
+...
+
+---
+name: brcond_scc_cond
+legalized: true
+body:             |
+  ; CHECK-LABEL: name: brcond_scc_cond
+  ; CHECK: bb.0.entry:
+  ; CHECK:   successors: %bb.1(0x80000000)
+  ; CHECK:   [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
+  ; CHECK:   [[ICMP:%[0-9]+]]:scc(s1) = G_ICMP intpred(ne), [[COPY]](s32), [[COPY1]]
+  ; CHECK:   G_BRCOND [[ICMP]](s1), %bb.1
+  ; CHECK: bb.1:
+  bb.0.entry:
+    successors: %bb.1
+    liveins: $sgpr0, $sgpr1
+    %0:_(s32) = COPY $sgpr0
+    %1:_(s32) = COPY $sgpr1
+    %2:_(s1) = G_ICMP intpred(ne), %0, %1
+    G_BRCOND %2, %bb.1
+
+  bb.1:
+...
+
+---
+name: brcond_sgpr_cond
+legalized: true
+body:             |
+  ; CHECK-LABEL: name: brcond_sgpr_cond
+  ; CHECK: bb.0.entry:
+  ; CHECK:   successors: %bb.1(0x80000000)
+  ; CHECK:   [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+  ; CHECK:   [[TRUNC:%[0-9]+]]:sgpr(s1) = G_TRUNC [[COPY]](s32)
+  ; CHECK:   [[COPY1:%[0-9]+]]:vcc(s1) = COPY [[TRUNC]](s1)
+  ; CHECK:   G_BRCOND [[COPY1]](s1), %bb.1
+  ; CHECK: bb.1:
+  bb.0.entry:
+    successors: %bb.1
+    liveins: $sgpr0
+    %0:_(s32) = COPY $sgpr0
+    %1:_(s1) = G_TRUNC %0
+    G_BRCOND %1, %bb.1
+
+  bb.1:
+...
+
+---
+name: brcond_vgpr_cond
+legalized: true
+body:             |
+  ; CHECK-LABEL: name: brcond_vgpr_cond
+  ; CHECK: bb.0.entry:
+  ; CHECK:   successors: %bb.1(0x80000000)
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+  ; CHECK:   [[TRUNC:%[0-9]+]]:vgpr(s1) = G_TRUNC [[COPY]](s32)
+  ; CHECK:   [[COPY1:%[0-9]+]]:vcc(s1) = COPY [[TRUNC]](s1)
+  ; CHECK:   G_BRCOND [[COPY1]](s1), %bb.1
+  ; CHECK: bb.1:
+  bb.0.entry:
+    successors: %bb.1
+    liveins: $vgpr0
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s1) = G_TRUNC %0
+    G_BRCOND %1, %bb.1
+
+  bb.1:
+...
+
+
+# The terminator that needs handling is the only instruction in the
+# block.
+
+---
+name: empty_block_vgpr_brcond
+legalized: true
+body:             |
+  ; CHECK-LABEL: name: empty_block_vgpr_brcond
+  ; CHECK: bb.0.entry:
+  ; CHECK:   successors: %bb.1(0x80000000)
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+  ; CHECK:   [[TRUNC:%[0-9]+]]:vgpr(s1) = G_TRUNC [[COPY]](s32)
+  ; CHECK: bb.1:
+  ; CHECK:   successors: %bb.1(0x40000000), %bb.2(0x40000000)
+  ; CHECK:   [[COPY1:%[0-9]+]]:vcc(s1) = COPY [[TRUNC]](s1)
+  ; CHECK:   G_BRCOND [[COPY1]](s1), %bb.1
+  ; CHECK: bb.2:
+  bb.0.entry:
+    successors: %bb.1
+    liveins: $vgpr0
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s1) = G_TRUNC %0
+
+  bb.1:
+    G_BRCOND %1, %bb.1
+
+  bb.2:
+...
+
+
+# Make sure the first instruction in the block isn't skipped.
+---
+name: copy_first_inst_brcond
+legalized: true
+body:             |
+  ; CHECK-LABEL: name: copy_first_inst_brcond
+  ; CHECK: bb.0.entry:
+  ; CHECK:   successors: %bb.1(0x80000000)
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+  ; CHECK: bb.1:
+  ; CHECK:   successors: %bb.1(0x40000000), %bb.2(0x40000000)
+  ; CHECK:   [[TRUNC:%[0-9]+]]:vgpr(s1) = G_TRUNC [[COPY]](s32)
+  ; CHECK:   [[COPY1:%[0-9]+]]:vcc(s1) = COPY [[TRUNC]](s1)
+  ; CHECK:   G_BRCOND [[COPY1]](s1), %bb.1
+  ; CHECK: bb.2:
+  bb.0.entry:
+    successors: %bb.1
+    liveins: $vgpr0
+    %0:_(s32) = COPY $vgpr0
+
+  bb.1:
+    %1:_(s1) = G_TRUNC %0
+    G_BRCOND %1, %bb.1
+
+  bb.2:
+...
+
+# Extra instruction separates brcond from the condition def
+---
+name: copy_middle_inst_brcond
+legalized: true
+body:             |
+  ; CHECK-LABEL: name: copy_middle_inst_brcond
+  ; CHECK: bb.0.entry:
+  ; CHECK:   successors: %bb.1(0x80000000)
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+  ; CHECK: bb.1:
+  ; CHECK:   successors: %bb.1(0x40000000), %bb.2(0x40000000)
+  ; CHECK:   [[TRUNC:%[0-9]+]]:vgpr(s1) = G_TRUNC [[COPY]](s32)
+  ; CHECK:   S_NOP 0
+  ; CHECK:   [[COPY1:%[0-9]+]]:vcc(s1) = COPY [[TRUNC]](s1)
+  ; CHECK:   G_BRCOND [[COPY1]](s1), %bb.1
+  ; CHECK: bb.2:
+  bb.0.entry:
+    successors: %bb.1
+    liveins: $vgpr0
+    %0:_(s32) = COPY $vgpr0
+
+  bb.1:
+    %1:_(s1) = G_TRUNC %0
+    S_NOP 0
+    G_BRCOND %1, %bb.1
+
+  bb.2:
+...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/regbankselect-default.mir b/test/CodeGen/AMDGPU/GlobalISel/regbankselect-default.mir
index 2370f5b96a5799643c6cda2f218e7f2736cebaa7..dad757b41f17599059da00ea6c93ed61e4ac101c 100644
--- a/test/CodeGen/AMDGPU/GlobalISel/regbankselect-default.mir
+++ b/test/CodeGen/AMDGPU/GlobalISel/regbankselect-default.mir
@@ -27,8 +27,9 @@ legalized:       true
 body: |
   bb.0:
     ; CHECK-LABEL: name: test_fconstant_f16_1
-    ; CHECK: [[C:%[0-9]+]]:sgpr(s32) = G_FCONSTANT half 0xH3C00
-    %0:_(s32) = G_FCONSTANT half 1.0
+    ; CHECK: [[C:%[0-9]+]]:sgpr(s16) = G_FCONSTANT half 0xH3C00
+    %0:_(s16) = G_FCONSTANT half 1.0
+    %1:_(s32) = G_ANYEXT %0
 ...
 
 ---
diff --git a/test/CodeGen/AMDGPU/GlobalISel/regbankselect-fcmp.mir b/test/CodeGen/AMDGPU/GlobalISel/regbankselect-fcmp.mir
index b866ab499b3049d6b16fef59d6ab8edbcc7406ed..d07c9f34e10293b00562fb76607b61e996e9c401 100644
--- a/test/CodeGen/AMDGPU/GlobalISel/regbankselect-fcmp.mir
+++ b/test/CodeGen/AMDGPU/GlobalISel/regbankselect-fcmp.mir
@@ -13,7 +13,7 @@ body: |
     ; CHECK: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
     ; CHECK: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
     ; CHECK: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32)
-    ; CHECK: [[FCMP:%[0-9]+]]:sgpr(s1) = G_FCMP floatpred(uge), [[COPY]](s32), [[COPY2]]
+    ; CHECK: [[FCMP:%[0-9]+]]:vcc(s1) = G_FCMP floatpred(uge), [[COPY]](s32), [[COPY2]]
     %0:_(s32) = COPY $sgpr0
     %1:_(s32) = COPY $sgpr1
     %2:_(s1) = G_FCMP floatpred(uge), %0(s32), %1
@@ -29,7 +29,7 @@ body: |
     ; CHECK-LABEL: name: fcmp_sv
     ; CHECK: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
     ; CHECK: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
-    ; CHECK: [[FCMP:%[0-9]+]]:sgpr(s1) = G_FCMP floatpred(uge), [[COPY]](s32), [[COPY1]]
+    ; CHECK: [[FCMP:%[0-9]+]]:vcc(s1) = G_FCMP floatpred(uge), [[COPY]](s32), [[COPY1]]
     %0:_(s32) = COPY $sgpr0
     %1:_(s32) = COPY $vgpr0
     %2:_(s1) = G_FCMP floatpred(uge), %0, %1
@@ -46,7 +46,7 @@ body: |
     ; CHECK: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
     ; CHECK: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
     ; CHECK: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY [[COPY]](s32)
-    ; CHECK: [[FCMP:%[0-9]+]]:sgpr(s1) = G_FCMP floatpred(uge), [[COPY1]](s32), [[COPY2]]
+    ; CHECK: [[FCMP:%[0-9]+]]:vcc(s1) = G_FCMP floatpred(uge), [[COPY1]](s32), [[COPY2]]
     %0:_(s32) = COPY $sgpr0
     %1:_(s32) = COPY $vgpr0
     %2:_(s1) = G_FCMP floatpred(uge), %1, %0
@@ -62,7 +62,7 @@ body: |
     ; CHECK-LABEL: name: fcmp_vv
     ; CHECK: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
     ; CHECK: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
-    ; CHECK: [[ICMP:%[0-9]+]]:sgpr(s1) = G_ICMP floatpred(uge), [[COPY]](s32), [[COPY1]]
+    ; CHECK: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP floatpred(uge), [[COPY]](s32), [[COPY1]]
     %0:_(s32) = COPY $vgpr0
     %1:_(s32) = COPY $vgpr1
     %2:_(s1) = G_ICMP floatpred(uge), %0, %1
diff --git a/test/CodeGen/AMDGPU/GlobalISel/regbankselect-fexp2.mir b/test/CodeGen/AMDGPU/GlobalISel/regbankselect-fexp2.mir
new file mode 100644
index 0000000000000000000000000000000000000000..1b56ca3025c4ad713f352f754895c0a9a1f8cff7
--- /dev/null
+++ b/test/CodeGen/AMDGPU/GlobalISel/regbankselect-fexp2.mir
@@ -0,0 +1,31 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -march=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
+# RUN: llc -march=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
+
+---
+name: fexp2_s
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0
+    ; CHECK-LABEL: name: fexp2_s
+    ; CHECK: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; CHECK: [[FEXP2_:%[0-9]+]]:vgpr(s32) = G_FEXP2 [[COPY]]
+    %0:_(s32) = COPY $sgpr0
+    %1:_(s32) = G_FEXP2 %0
+...
+
+---
+name: fexp2_v
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $vgpr0
+    ; CHECK-LABEL: name: fexp2_v
+    ; CHECK: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; CHECK: [[FEXP2_:%[0-9]+]]:vgpr(s32) = G_FEXP2 [[COPY]]
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = G_FEXP2 %0
+...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/regbankselect-flog2.mir b/test/CodeGen/AMDGPU/GlobalISel/regbankselect-flog2.mir
new file mode 100644
index 0000000000000000000000000000000000000000..2915137a929c6ec73e30b096a2f470acbd86ce7b
--- /dev/null
+++ b/test/CodeGen/AMDGPU/GlobalISel/regbankselect-flog2.mir
@@ -0,0 +1,31 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -march=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
+# RUN: llc -march=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
+
+---
+name: flog2_s
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0
+    ; CHECK-LABEL: name: flog2_s
+    ; CHECK: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; CHECK: [[FLOG2_:%[0-9]+]]:vgpr(s32) = G_FLOG2 [[COPY]]
+    %0:_(s32) = COPY $sgpr0
+    %1:_(s32) = G_FLOG2 %0
+...
+
+---
+name: flog2_v
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $vgpr0
+    ; CHECK-LABEL: name: flog2_v
+    ; CHECK: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; CHECK: [[FLOG2_:%[0-9]+]]:vgpr(s32) = G_FLOG2 [[COPY]]
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = G_FLOG2 %0
+...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/regbankselect-fpext.mir b/test/CodeGen/AMDGPU/GlobalISel/regbankselect-fpext.mir
new file mode 100644
index 0000000000000000000000000000000000000000..123b3558da64adf93a49356d542623780fe3a28a
--- /dev/null
+++ b/test/CodeGen/AMDGPU/GlobalISel/regbankselect-fpext.mir
@@ -0,0 +1,31 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -march=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
+# RUN: llc -march=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
+
+---
+name: fpext_s
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0
+    ; CHECK-LABEL: name: fpext_s
+    ; CHECK: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; CHECK: [[FPEXT:%[0-9]+]]:vgpr(s64) = G_FPEXT [[COPY]](s32)
+    %0:_(s32) = COPY $sgpr0
+    %1:_(s64) = G_FPEXT %0
+...
+
+---
+name: fpext_v
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $vgpr0
+    ; CHECK-LABEL: name: fpext_v
+    ; CHECK: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; CHECK: [[FPEXT:%[0-9]+]]:vgpr(s64) = G_FPEXT [[COPY]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s64) = G_FPEXT %0
+...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/regbankselect-fptrunc.mir b/test/CodeGen/AMDGPU/GlobalISel/regbankselect-fptrunc.mir
new file mode 100644
index 0000000000000000000000000000000000000000..02cb49a4b8f49964b37205afc6c757cb1f9d3625
--- /dev/null
+++ b/test/CodeGen/AMDGPU/GlobalISel/regbankselect-fptrunc.mir
@@ -0,0 +1,31 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -march=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
+# RUN: llc -march=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
+
+---
+name: fptrunc_s
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0_sgpr1
+    ; CHECK-LABEL: name: fptrunc_s
+    ; CHECK: [[COPY:%[0-9]+]]:sgpr(s64) = COPY $sgpr0_sgpr1
+    ; CHECK: [[FPTRUNC:%[0-9]+]]:vgpr(s32) = G_FPTRUNC [[COPY]](s64)
+    %0:_(s64) = COPY $sgpr0_sgpr1
+    %1:_(s32) = G_FPTRUNC %0
+...
+
+---
+name: fptrunc_v
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+    ; CHECK-LABEL: name: fptrunc_v
+    ; CHECK: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
+    ; CHECK: [[FPTRUNC:%[0-9]+]]:vgpr(s32) = G_FPTRUNC [[COPY]](s64)
+    %0:_(s64) = COPY $vgpr0_vgpr1
+    %1:_(s32) = G_FPTRUNC %0
+...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/regbankselect-fsqrt.mir b/test/CodeGen/AMDGPU/GlobalISel/regbankselect-fsqrt.mir
new file mode 100644
index 0000000000000000000000000000000000000000..a2e3a4320597c378a3b933adc64083b391c88d73
--- /dev/null
+++ b/test/CodeGen/AMDGPU/GlobalISel/regbankselect-fsqrt.mir
@@ -0,0 +1,35 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -march=amdgcn -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
+# RUN: llc -march=amdgcn -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
+
+---
+name: fsqrt_s
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0_sgpr1
+    ; CHECK-LABEL: name: fsqrt_s
+    ; CHECK: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; CHECK: [[FSQRT:%[0-9]+]]:vgpr(s32) = G_FSQRT [[COPY]]
+    ; CHECK: $vgpr0 = COPY [[FSQRT]](s32)
+    %0:_(s32) = COPY $sgpr0
+    %1:_(s32) = G_FSQRT %0
+    $vgpr0 = COPY %1
+...
+
+---
+name: fsqrt_v
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+    ; CHECK-LABEL: name: fsqrt_v
+    ; CHECK: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; CHECK: [[FSQRT:%[0-9]+]]:vgpr(s32) = G_FSQRT [[COPY]]
+    ; CHECK: $vgpr0 = COPY [[FSQRT]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = G_FSQRT %0
+    $vgpr0 = COPY %1
+...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/regbankselect-icmp.mir b/test/CodeGen/AMDGPU/GlobalISel/regbankselect-icmp.mir
index dba76e8156d93d07df31426c4b289c3a15a7013f..15aa98ba077aa2e8de0ed0814b4f80f42778ee43 100644
--- a/test/CodeGen/AMDGPU/GlobalISel/regbankselect-icmp.mir
+++ b/test/CodeGen/AMDGPU/GlobalISel/regbankselect-icmp.mir
@@ -28,7 +28,7 @@ body: |
     ; CHECK-LABEL: name: icmp_sv
     ; CHECK: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
     ; CHECK: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
-    ; CHECK: [[ICMP:%[0-9]+]]:sgpr(s1) = G_ICMP intpred(ne), [[COPY]](s32), [[COPY1]]
+    ; CHECK: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(ne), [[COPY]](s32), [[COPY1]]
     %0:_(s32) = COPY $sgpr0
     %1:_(s32) = COPY $vgpr0
     %2:_(s1) = G_ICMP intpred(ne), %0, %1
@@ -44,7 +44,7 @@ body: |
     ; CHECK-LABEL: name: icmp_vs
     ; CHECK: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
     ; CHECK: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
-    ; CHECK: [[ICMP:%[0-9]+]]:sgpr(s1) = G_ICMP intpred(ne), [[COPY1]](s32), [[COPY]]
+    ; CHECK: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(ne), [[COPY1]](s32), [[COPY]]
     %0:_(s32) = COPY $sgpr0
     %1:_(s32) = COPY $vgpr0
     %2:_(s1) = G_ICMP intpred(ne), %1, %0
@@ -60,7 +60,7 @@ body: |
     ; CHECK-LABEL: name: icmp_vv
     ; CHECK: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
     ; CHECK: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
-    ; CHECK: [[ICMP:%[0-9]+]]:sgpr(s1) = G_ICMP intpred(ne), [[COPY]](s32), [[COPY1]]
+    ; CHECK: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(ne), [[COPY]](s32), [[COPY1]]
     %0:_(s32) = COPY $vgpr0
     %1:_(s32) = COPY $vgpr1
     %2:_(s1) = G_ICMP intpred(ne), %0, %1
diff --git a/test/CodeGen/AMDGPU/GlobalISel/regbankselect-intrinsic-round.mir b/test/CodeGen/AMDGPU/GlobalISel/regbankselect-intrinsic-round.mir
new file mode 100644
index 0000000000000000000000000000000000000000..456baf4aff18c8d8086eb7eb17bcb16fafe9e8d8
--- /dev/null
+++ b/test/CodeGen/AMDGPU/GlobalISel/regbankselect-intrinsic-round.mir
@@ -0,0 +1,31 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -march=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
+# RUN: llc -march=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
+
+---
+name: intrinsic_round_s
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0
+    ; CHECK-LABEL: name: intrinsic_round_s
+    ; CHECK: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; CHECK: [[INTRINSIC_ROUND:%[0-9]+]]:vgpr(s32) = G_INTRINSIC_ROUND [[COPY]]
+    %0:_(s32) = COPY $sgpr0
+    %1:_(s32) = G_INTRINSIC_ROUND %0
+...
+
+---
+name: intrinsic_round_v
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $vgpr0
+    ; CHECK-LABEL: name: intrinsic_round_v
+    ; CHECK: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; CHECK: [[INTRINSIC_ROUND:%[0-9]+]]:vgpr(s32) = G_INTRINSIC_ROUND [[COPY]]
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = G_INTRINSIC_ROUND %0
+...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/regbankselect-intrinsic-trunc.mir b/test/CodeGen/AMDGPU/GlobalISel/regbankselect-intrinsic-trunc.mir
new file mode 100644
index 0000000000000000000000000000000000000000..fef4d49211752e75c31e36133f93c73b84d9c9c3
--- /dev/null
+++ b/test/CodeGen/AMDGPU/GlobalISel/regbankselect-intrinsic-trunc.mir
@@ -0,0 +1,31 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -march=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
+# RUN: llc -march=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
+
+---
+name: intrinsic_trunc_s
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0
+    ; CHECK-LABEL: name: intrinsic_trunc_s
+    ; CHECK: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; CHECK: [[INTRINSIC_TRUNC:%[0-9]+]]:vgpr(s32) = G_INTRINSIC_TRUNC [[COPY]]
+    %0:_(s32) = COPY $sgpr0
+    %1:_(s32) = G_INTRINSIC_TRUNC %0
+...
+
+---
+name: intrinsic_trunc_v
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $vgpr0
+    ; CHECK-LABEL: name: intrinsic_trunc_v
+    ; CHECK: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; CHECK: [[INTRINSIC_TRUNC:%[0-9]+]]:vgpr(s32) = G_INTRINSIC_TRUNC [[COPY]]
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = G_INTRINSIC_TRUNC %0
+...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/regbankselect-lshr.mir b/test/CodeGen/AMDGPU/GlobalISel/regbankselect-lshr.mir
new file mode 100644
index 0000000000000000000000000000000000000000..a4e20fc5e51c365b09d451c5fbd7129993d17a9f
--- /dev/null
+++ b/test/CodeGen/AMDGPU/GlobalISel/regbankselect-lshr.mir
@@ -0,0 +1,65 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -march=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
+# RUN: llc -march=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
+
+---
+name: lshr_ss
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0, $sgpr1
+    ; CHECK-LABEL: name: lshr_ss
+    ; CHECK: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
+    ; CHECK: [[LSHR:%[0-9]+]]:sgpr(s32) = G_LSHR [[COPY]], [[COPY1]]
+    %0:_(s32) = COPY $sgpr0
+    %1:_(s32) = COPY $sgpr1
+    %2:_(s32) = G_LSHR %0, %1
+...
+---
+name: lshr_sv
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0, $vgpr0
+    ; CHECK-LABEL: name: lshr_sv
+    ; CHECK: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; CHECK: [[LSHR:%[0-9]+]]:vgpr(s32) = G_LSHR [[COPY]], [[COPY1]]
+    %0:_(s32) = COPY $sgpr0
+    %1:_(s32) = COPY $vgpr0
+    %2:_(s32) = G_LSHR %0, %1
+...
+---
+name: lshr_vs
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0, $vgpr0
+    ; CHECK-LABEL: name: lshr_vs
+    ; CHECK: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; CHECK: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32)
+    ; CHECK: [[LSHR:%[0-9]+]]:vgpr(s32) = G_LSHR [[COPY]], [[COPY2]]
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $sgpr0
+    %2:_(s32) = G_LSHR %0, %1
+...
+---
+name: lshr_vv
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+    ; CHECK-LABEL: name: lshr_vv
+    ; CHECK: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+    ; CHECK: [[LSHR:%[0-9]+]]:vgpr(s32) = G_LSHR [[COPY]], [[COPY1]]
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s32) = G_LSHR %0, %1
+...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/regbankselect-or.mir b/test/CodeGen/AMDGPU/GlobalISel/regbankselect-or.mir
index 7381a5f92ee0a0ee4d6ae0846d18f520f6e2019f..83d244bbfb3d9aeaf10c82c4d882778e7fce20b3 100644
--- a/test/CodeGen/AMDGPU/GlobalISel/regbankselect-or.mir
+++ b/test/CodeGen/AMDGPU/GlobalISel/regbankselect-or.mir
@@ -3,13 +3,13 @@
 # RUN: llc -march=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
 
 ---
-name: or_ss
+name: or_i32_ss
 legalized: true
 
 body: |
   bb.0:
     liveins: $sgpr0, $sgpr1
-    ; CHECK-LABEL: name: or_ss
+    ; CHECK-LABEL: name: or_i32_ss
     ; CHECK: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
     ; CHECK: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
     ; CHECK: [[OR:%[0-9]+]]:sgpr(s32) = G_OR [[COPY]], [[COPY1]]
@@ -19,13 +19,13 @@ body: |
 ...
 
 ---
-name: or_sv
+name: or_i32_sv
 legalized: true
 
 body: |
   bb.0:
     liveins: $sgpr0, $vgpr0
-    ; CHECK-LABEL: name: or_sv
+    ; CHECK-LABEL: name: or_i32_sv
     ; CHECK: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
     ; CHECK: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
     ; CHECK: [[OR:%[0-9]+]]:vgpr(s32) = G_OR [[COPY]], [[COPY1]]
@@ -35,13 +35,13 @@ body: |
 ...
 
 ---
-name: or_vs
+name: or_i32_vs
 legalized: true
 
 body: |
   bb.0:
     liveins: $sgpr0, $vgpr0
-    ; CHECK-LABEL: name: or_vs
+    ; CHECK-LABEL: name: or_i32_vs
     ; CHECK: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
     ; CHECK: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
     ; CHECK: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32)
@@ -52,13 +52,13 @@ body: |
 ...
 
 ---
-name: or_vv
+name: or_i32_vv
 legalized: true
 
 body: |
   bb.0:
     liveins: $vgpr0, $vgpr1
-    ; CHECK-LABEL: name: or_vv
+    ; CHECK-LABEL: name: or_i32_vv
     ; CHECK: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
     ; CHECK: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
     ; CHECK: [[OR:%[0-9]+]]:vgpr(s32) = G_OR [[COPY]], [[COPY1]]
diff --git a/test/CodeGen/AMDGPU/GlobalISel/regbankselect-ptrtoint.mir b/test/CodeGen/AMDGPU/GlobalISel/regbankselect-ptrtoint.mir
index 83d8994d0fe310eaf6cd8d135160d76427e39e36..686190d8871438a11a2e2c276c333bbd627e224e 100644
--- a/test/CodeGen/AMDGPU/GlobalISel/regbankselect-ptrtoint.mir
+++ b/test/CodeGen/AMDGPU/GlobalISel/regbankselect-ptrtoint.mir
@@ -10,10 +10,10 @@ body: |
   bb.0:
     liveins: $sgpr0_sgpr1
     ; CHECK-LABEL: name: ptrtoint_s
-    ; CHECK: [[COPY:%[0-9]+]]:sgpr(s64) = COPY $sgpr0_sgpr1
-    ; CHECK: [[PTRTOINT:%[0-9]+]]:sgpr(p4) = G_PTRTOINT [[COPY]](s64)
-    %0:_(s64) = COPY $sgpr0_sgpr1
-    %1:_(p4) = G_PTRTOINT %0
+    ; CHECK: [[COPY:%[0-9]+]]:sgpr(p1) = COPY $sgpr0_sgpr1
+    ; CHECK: [[PTRTOINT:%[0-9]+]]:sgpr(s64) = G_PTRTOINT [[COPY]](p1)
+    %0:_(p1) = COPY $sgpr0_sgpr1
+    %1:_(s64) = G_PTRTOINT %0
 ...
 
 ---
@@ -24,8 +24,8 @@ body: |
   bb.0:
     liveins: $vgpr0_vgpr1
     ; CHECK-LABEL: name: ptrtoint_v
-    ; CHECK: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
-    ; CHECK: [[PTRTOINT:%[0-9]+]]:vgpr(p0) = G_PTRTOINT [[COPY]](s64)
-    %0:_(s64) = COPY $vgpr0_vgpr1
-    %1:_(p0) = G_PTRTOINT %0
+    ; CHECK: [[COPY:%[0-9]+]]:vgpr(p1) = COPY $vgpr0_vgpr1
+    ; CHECK: [[PTRTOINT:%[0-9]+]]:vgpr(s64) = G_PTRTOINT [[COPY]](p1)
+    %0:_(p1) = COPY $vgpr0_vgpr1
+    %1:_(s64) = G_PTRTOINT %0
 ...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/regbankselect-sadde.mir b/test/CodeGen/AMDGPU/GlobalISel/regbankselect-sadde.mir
new file mode 100644
index 0000000000000000000000000000000000000000..929882ac679c472ddc11928b0c2d01d09134e6ed
--- /dev/null
+++ b/test/CodeGen/AMDGPU/GlobalISel/regbankselect-sadde.mir
@@ -0,0 +1,154 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -march=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck -check-prefix=FAST %s
+# RUN: llc -march=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck -check-prefix=GREEDY %s
+
+---
+name: sadde_s32_sss
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0, $sgpr1, $sgpr2
+    ; FAST-LABEL: name: sadde_s32_sss
+    ; FAST: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; FAST: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
+    ; FAST: [[COPY2:%[0-9]+]]:sgpr(s32) = COPY $sgpr2
+    ; FAST: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
+    ; FAST: [[ICMP:%[0-9]+]]:scc(s1) = G_ICMP intpred(eq), [[COPY2]](s32), [[C]]
+    ; FAST: [[SADDE:%[0-9]+]]:sgpr(s32), [[SADDE1:%[0-9]+]]:scc(s1) = G_SADDE [[COPY]], [[COPY1]], [[ICMP]]
+    ; GREEDY-LABEL: name: sadde_s32_sss
+    ; GREEDY: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; GREEDY: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
+    ; GREEDY: [[COPY2:%[0-9]+]]:sgpr(s32) = COPY $sgpr2
+    ; GREEDY: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
+    ; GREEDY: [[ICMP:%[0-9]+]]:scc(s1) = G_ICMP intpred(eq), [[COPY2]](s32), [[C]]
+    ; GREEDY: [[SADDE:%[0-9]+]]:sgpr(s32), [[SADDE1:%[0-9]+]]:scc(s1) = G_SADDE [[COPY]], [[COPY1]], [[ICMP]]
+    %0:_(s32) = COPY $sgpr0
+    %1:_(s32) = COPY $sgpr1
+    %2:_(s32) = COPY $sgpr2
+    %3:_(s32) = G_CONSTANT i32 0
+    %4:_(s1) = G_ICMP intpred(eq), %2, %3
+    %5:_(s32), %6:_(s1) = G_SADDE %0, %1, %4
+...
+
+---
+name: sadde_s32_vss
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $vgpr0, $sgpr0, $sgpr1
+    ; FAST-LABEL: name: sadde_s32_vss
+    ; FAST: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; FAST: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; FAST: [[COPY2:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
+    ; FAST: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
+    ; FAST: [[ICMP:%[0-9]+]]:scc(s1) = G_ICMP intpred(eq), [[COPY2]](s32), [[C]]
+    ; FAST: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32)
+    ; FAST: [[COPY4:%[0-9]+]]:vcc(s1) = COPY [[ICMP]](s1)
+    ; FAST: [[SADDE:%[0-9]+]]:vgpr(s32), [[SADDE1:%[0-9]+]]:vcc(s1) = G_SADDE [[COPY]], [[COPY3]], [[COPY4]]
+    ; GREEDY-LABEL: name: sadde_s32_vss
+    ; GREEDY: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; GREEDY: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; GREEDY: [[COPY2:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
+    ; GREEDY: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
+    ; GREEDY: [[ICMP:%[0-9]+]]:scc(s1) = G_ICMP intpred(eq), [[COPY2]](s32), [[C]]
+    ; GREEDY: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32)
+    ; GREEDY: [[COPY4:%[0-9]+]]:vcc(s1) = COPY [[ICMP]](s1)
+    ; GREEDY: [[SADDE:%[0-9]+]]:vgpr(s32), [[SADDE1:%[0-9]+]]:vcc(s1) = G_SADDE [[COPY]], [[COPY3]], [[COPY4]]
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $sgpr0
+    %2:_(s32) = COPY $sgpr1
+    %3:_(s32) = G_CONSTANT i32 0
+    %4:_(s1) = G_ICMP intpred(eq), %2, %3
+    %5:_(s32), %6:_(s1) = G_SADDE %0, %1, %4
+...
+---
+name: sadde_s32_ssv
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0, $sgpr1, $vgpr0
+    ; FAST-LABEL: name: sadde_s32_ssv
+    ; FAST: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; FAST: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
+    ; FAST: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; FAST: [[TRUNC:%[0-9]+]]:vgpr(s1) = G_TRUNC [[COPY2]](s32)
+    ; FAST: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY [[COPY]](s32)
+    ; FAST: [[COPY4:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32)
+    ; FAST: [[COPY5:%[0-9]+]]:vcc(s1) = COPY [[TRUNC]](s1)
+    ; FAST: [[SADDE:%[0-9]+]]:vgpr(s32), [[SADDE1:%[0-9]+]]:vcc(s1) = G_SADDE [[COPY3]], [[COPY4]], [[COPY5]]
+    ; GREEDY-LABEL: name: sadde_s32_ssv
+    ; GREEDY: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; GREEDY: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
+    ; GREEDY: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; GREEDY: [[TRUNC:%[0-9]+]]:vgpr(s1) = G_TRUNC [[COPY2]](s32)
+    ; GREEDY: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY [[COPY]](s32)
+    ; GREEDY: [[COPY4:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32)
+    ; GREEDY: [[COPY5:%[0-9]+]]:vcc(s1) = COPY [[TRUNC]](s1)
+    ; GREEDY: [[SADDE:%[0-9]+]]:vgpr(s32), [[SADDE1:%[0-9]+]]:vcc(s1) = G_SADDE [[COPY3]], [[COPY4]], [[COPY5]]
+    %0:_(s32) = COPY $sgpr0
+    %1:_(s32) = COPY $sgpr1
+    %2:_(s32) = COPY $vgpr0
+    %3:_(s1) = G_TRUNC %2
+    %4:_(s32), %5:_(s1) = G_SADDE %0, %1, %3
+...
+
+---
+name: sadde_s32_vvs
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $sgpr0
+    ; FAST-LABEL: name: sadde_s32_vvs
+    ; FAST: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; FAST: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+    ; FAST: [[COPY2:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; FAST: [[TRUNC:%[0-9]+]]:sgpr(s1) = G_TRUNC [[COPY2]](s32)
+    ; FAST: [[COPY3:%[0-9]+]]:vcc(s1) = COPY [[TRUNC]](s1)
+    ; FAST: [[SADDE:%[0-9]+]]:vgpr(s32), [[SADDE1:%[0-9]+]]:vcc(s1) = G_SADDE [[COPY]], [[COPY1]], [[COPY3]]
+    ; GREEDY-LABEL: name: sadde_s32_vvs
+    ; GREEDY: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; GREEDY: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+    ; GREEDY: [[COPY2:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; GREEDY: [[TRUNC:%[0-9]+]]:sgpr(s1) = G_TRUNC [[COPY2]](s32)
+    ; GREEDY: [[COPY3:%[0-9]+]]:vcc(s1) = COPY [[TRUNC]](s1)
+    ; GREEDY: [[SADDE:%[0-9]+]]:vgpr(s32), [[SADDE1:%[0-9]+]]:vcc(s1) = G_SADDE [[COPY]], [[COPY1]], [[COPY3]]
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s32) = COPY $sgpr0
+    %3:_(s1) = G_TRUNC %2
+    %4:_(s32), %5:_(s1) = G_SADDE %0, %1, %3
+...
+
+---
+name: sadde_s32_sss_noscc
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0, $sgpr1, $sgpr2
+    ; FAST-LABEL: name: sadde_s32_sss_noscc
+    ; FAST: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; FAST: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
+    ; FAST: [[COPY2:%[0-9]+]]:sgpr(s32) = COPY $sgpr2
+    ; FAST: [[TRUNC:%[0-9]+]]:sgpr(s1) = G_TRUNC [[COPY2]](s32)
+    ; FAST: [[COPY3:%[0-9]+]]:scc(s1) = COPY [[TRUNC]](s1)
+    ; FAST: [[SADDE:%[0-9]+]]:sgpr(s32), [[SADDE1:%[0-9]+]]:scc(s1) = G_SADDE [[COPY]], [[COPY1]], [[COPY3]]
+    ; GREEDY-LABEL: name: sadde_s32_sss_noscc
+    ; GREEDY: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; GREEDY: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
+    ; GREEDY: [[COPY2:%[0-9]+]]:sgpr(s32) = COPY $sgpr2
+    ; GREEDY: [[TRUNC:%[0-9]+]]:sgpr(s1) = G_TRUNC [[COPY2]](s32)
+    ; GREEDY: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY [[COPY]](s32)
+    ; GREEDY: [[COPY4:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32)
+    ; GREEDY: [[COPY5:%[0-9]+]]:vcc(s1) = COPY [[TRUNC]](s1)
+    ; GREEDY: [[SADDE:%[0-9]+]]:vgpr(s32), [[SADDE1:%[0-9]+]]:vcc(s1) = G_SADDE [[COPY3]], [[COPY4]], [[COPY5]]
+    %0:_(s32) = COPY $sgpr0
+    %1:_(s32) = COPY $sgpr1
+    %2:_(s32) = COPY $sgpr2
+    %3:_(s1) = G_TRUNC %2
+    %4:_(s32), %5:_(s1) = G_SADDE %0, %1, %3
+...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/regbankselect-saddo.mir b/test/CodeGen/AMDGPU/GlobalISel/regbankselect-saddo.mir
new file mode 100644
index 0000000000000000000000000000000000000000..2f60834479a732871c2306284d530ad4c3b70f36
--- /dev/null
+++ b/test/CodeGen/AMDGPU/GlobalISel/regbankselect-saddo.mir
@@ -0,0 +1,68 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -march=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
+# RUN: llc -march=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
+---
+name: saddo_s32_ss
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0, $sgpr1
+    ; CHECK-LABEL: name: saddo_s32_ss
+    ; CHECK: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
+    ; CHECK: [[SADDO:%[0-9]+]]:sgpr(s32), [[SADDO1:%[0-9]+]]:scc(s1) = G_SADDO [[COPY]], [[COPY1]]
+    %0:_(s32) = COPY $sgpr0
+    %1:_(s32) = COPY $sgpr1
+    %2:_(s32), %3:_(s1) = G_SADDO %0, %1
+...
+
+---
+name: saddo_s32_sv
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0, $vgpr0
+    ; CHECK-LABEL: name: saddo_s32_sv
+    ; CHECK: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; CHECK: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY [[COPY]](s32)
+    ; CHECK: [[SADDO:%[0-9]+]]:vgpr(s32), [[SADDO1:%[0-9]+]]:vcc(s1) = G_SADDO [[COPY2]], [[COPY1]]
+    %0:_(s32) = COPY $sgpr0
+    %1:_(s32) = COPY $vgpr0
+    %2:_(s32), %3:_(s1) = G_SADDO %0, %1
+...
+
+---
+name: saddo_s32_vs
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0, $vgpr0
+    ; CHECK-LABEL: name: saddo_s32_vs
+    ; CHECK: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; CHECK: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32)
+    ; CHECK: [[SADDO:%[0-9]+]]:vgpr(s32), [[SADDO1:%[0-9]+]]:vcc(s1) = G_SADDO [[COPY]], [[COPY2]]
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $sgpr0
+    %2:_(s32), %3:_(s1) = G_SADDO %0, %1
+...
+
+---
+name: saddo_s32_vv
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+    ; CHECK-LABEL: name: saddo_s32_vv
+    ; CHECK: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+    ; CHECK: [[SADDO:%[0-9]+]]:vgpr(s32), [[SADDO1:%[0-9]+]]:vcc(s1) = G_SADDO [[COPY]], [[COPY1]]
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s32), %3:_(s1) = G_SADDO %0, %1
+...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/regbankselect-select.mir b/test/CodeGen/AMDGPU/GlobalISel/regbankselect-select.mir
index 331de8072070ff12f00e8aa1858e9328886ac92e..29a0194382616bfbb1988d1ebb8970a7de4ac050 100644
--- a/test/CodeGen/AMDGPU/GlobalISel/regbankselect-select.mir
+++ b/test/CodeGen/AMDGPU/GlobalISel/regbankselect-select.mir
@@ -1,34 +1,20 @@
-# RUN: llc -march=amdgcn -mcpu=fiji -run-pass=regbankselect -global-isel %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck --check-prefixes=GCN,FAST %s
-# RUN: llc -march=amdgcn -mcpu=fiji -run-pass=regbankselect -global-isel %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck --check-prefixes=GCN,GREEDY %s
-
-# REQUIRES: global-isel
-
---- |
-  define void @select_sss() { ret void }
-  define void @select_ssv() { ret void }
-  define void @select_svs() { ret void }
-  define void @select_svv() { ret void }
-  define void @select_vss() { ret void }
-  define void @select_vsv() { ret void }
-  define void @select_vvs() { ret void }
-  define void @select_vvv() { ret void }
-...
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -march=amdgcn -mcpu=fiji -run-pass=regbankselect -global-isel %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
+# RUN: llc -march=amdgcn -mcpu=fiji -run-pass=regbankselect -global-isel %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
 
 ---
-name: select_sss
+name: select_s32_sss
 legalized: true
-
-# GCN-LABEL: name: select_sss
-# GCN: [[SGPR0:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
-# GCN: [[SGPR1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
-# GCN: [[SGPR2:%[0-9]+]]:sgpr(s32) = COPY $sgpr2
-# GCN: [[SGPR3:%[0-9]+]]:sgpr(s32) = COPY $sgpr3
-# GCN: [[SCC:%[0-9]+]]:scc(s1) = G_ICMP intpred(ne), [[SGPR0]](s32), [[SGPR1]]
-# GCN: {{%[0-9]+}}:sgpr(s32) = G_SELECT [[SCC]](s1), [[SGPR2]], [[SGPR3]]
-
 body: |
   bb.0:
     liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3
+    ; CHECK-LABEL: name: select_s32_sss
+    ; CHECK: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
+    ; CHECK: [[COPY2:%[0-9]+]]:sgpr(s32) = COPY $sgpr2
+    ; CHECK: [[COPY3:%[0-9]+]]:sgpr(s32) = COPY $sgpr3
+    ; CHECK: [[ICMP:%[0-9]+]]:scc(s1) = G_ICMP intpred(ne), [[COPY]](s32), [[COPY1]]
+    ; CHECK: [[SELECT:%[0-9]+]]:sgpr(s32) = G_SELECT [[ICMP]](s1), [[COPY2]], [[COPY3]]
     %0:_(s32) = COPY $sgpr0
     %1:_(s32) = COPY $sgpr1
     %2:_(s32) = COPY $sgpr2
@@ -38,26 +24,23 @@ body: |
 ...
 
 ---
-name: select_ssv
+name: select_s32_ssv
 legalized: true
-
-# GCN-LABEL: name: select_ssv
-# GCN: [[SGPR0:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
-# GCN: [[SGPR1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
-# GCN: [[SGPR2:%[0-9]+]]:sgpr(s32) = COPY $sgpr2
-# GCN: [[VGPR0:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
-# GCN: [[SCC:%[0-9]+]]:scc(s1) = G_ICMP intpred(ne), [[SGPR0]](s32), [[SGPR1]]
-# GCN: [[SCC_S:%[0-9]+]]:sgpr(s1) = COPY [[SCC]]
-# FAST:   [[SGPR2_V:%[0-9]+]]:vgpr(s32) = COPY [[SGPR2]]
-# FAST:   {{%[0-9]+}}:vgpr(s32) = G_SELECT [[SCC_S]](s1), [[SGPR2_V]], [[VGPR0]]
-# GREEDY: {{%[0-9]+}}:vgpr(s32) = G_SELECT [[SCC_S]](s1), [[SGPR2]], [[VGPR0]]
-
 body: |
   bb.0:
     liveins: $sgpr0, $sgpr1, $sgpr2, $vgpr0
+    ; CHECK-LABEL: name: select_s32_ssv
+    ; CHECK: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
+    ; CHECK: [[COPY2:%[0-9]+]]:sgpr(s32) = COPY $sgpr2
+    ; CHECK: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; CHECK: [[ICMP:%[0-9]+]]:scc(s1) = G_ICMP intpred(ne), [[COPY]](s32), [[COPY1]]
+    ; CHECK: [[COPY4:%[0-9]+]]:vcc(s1) = COPY [[ICMP]](s1)
+    ; CHECK: [[COPY5:%[0-9]+]]:vgpr(s32) = COPY [[COPY2]](s32)
+    ; CHECK: [[SELECT:%[0-9]+]]:vgpr(s32) = G_SELECT [[COPY4]](s1), [[COPY5]], [[COPY3]]
     %0:_(s32) = COPY $sgpr0
     %1:_(s32) = COPY $sgpr1
-    %2:_(s32) = COPY $sgpr2 
+    %2:_(s32) = COPY $sgpr2
     %3:_(s32) = COPY $vgpr0
     %4:_(s1) = G_ICMP intpred(ne), %0, %1
     %5:_(s32) = G_SELECT %4, %2, %3
@@ -65,23 +48,20 @@ body: |
 ...
 
 ---
-name: select_svs
+name: select_s32_svs
 legalized: true
-
-# GCN-LABEL: name: select_svs
-# GCN: [[SGPR0:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
-# GCN: [[SGPR1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
-# GCN: [[SGPR2:%[0-9]+]]:sgpr(s32) = COPY $sgpr2
-# GCN: [[VGPR0:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
-# GCN: [[SCC:%[0-9]+]]:scc(s1) = G_ICMP intpred(ne), [[SGPR0]](s32), [[SGPR1]]
-# GCN: [[SCC_S:%[0-9]+]]:sgpr(s1) = COPY [[SCC]]
-# FAST:   [[SGPR2_V:%[0-9]+]]:vgpr(s32) = COPY [[SGPR2]]
-# FAST:   {{%[0-9]+}}:vgpr(s32) = G_SELECT [[SCC_S]](s1), [[VGPR0]], [[SGPR2_V]]
-# GREEDY: {{%[0-9]+}}:vgpr(s32) = G_SELECT [[SCC_S]](s1), [[VGPR0]], [[SGPR2]]
-
 body: |
   bb.0:
     liveins: $sgpr0, $sgpr1, $sgpr2, $vgpr0
+    ; CHECK-LABEL: name: select_s32_svs
+    ; CHECK: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
+    ; CHECK: [[COPY2:%[0-9]+]]:sgpr(s32) = COPY $sgpr2
+    ; CHECK: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; CHECK: [[ICMP:%[0-9]+]]:scc(s1) = G_ICMP intpred(ne), [[COPY]](s32), [[COPY1]]
+    ; CHECK: [[COPY4:%[0-9]+]]:vcc(s1) = COPY [[ICMP]](s1)
+    ; CHECK: [[COPY5:%[0-9]+]]:vgpr(s32) = COPY [[COPY2]](s32)
+    ; CHECK: [[SELECT:%[0-9]+]]:vgpr(s32) = G_SELECT [[COPY4]](s1), [[COPY3]], [[COPY5]]
     %0:_(s32) = COPY $sgpr0
     %1:_(s32) = COPY $sgpr1
     %2:_(s32) = COPY $sgpr2
@@ -91,21 +71,19 @@ body: |
 ...
 
 ---
-name: select_svv
+name: select_s32_svv
 legalized: true
-
-# GCN-LABEL: name: select_svv
-# GCN: [[SGPR0:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
-# GCN: [[SGPR1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
-# GCN: [[VGPR0:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
-# GCN: [[VGPR1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
-# GCN: [[SCC:%[0-9]+]]:scc(s1) = G_ICMP intpred(ne), [[SGPR0]](s32), [[SGPR1]]
-# GCN: [[SCC_S:%[0-9]+]]:sgpr(s1) = COPY [[SCC]]
-# GCN: {{%[0-9]+}}:vgpr(s32) = G_SELECT [[SCC_S]](s1), [[VGPR0]], [[VGPR1]]
-
 body: |
   bb.0:
     liveins: $sgpr0, $sgpr1, $vgpr0, $vgpr1
+    ; CHECK-LABEL: name: select_s32_svv
+    ; CHECK: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
+    ; CHECK: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; CHECK: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+    ; CHECK: [[ICMP:%[0-9]+]]:scc(s1) = G_ICMP intpred(ne), [[COPY]](s32), [[COPY1]]
+    ; CHECK: [[COPY4:%[0-9]+]]:vcc(s1) = COPY [[ICMP]](s1)
+    ; CHECK: [[SELECT:%[0-9]+]]:vgpr(s32) = G_SELECT [[COPY4]](s1), [[COPY2]], [[COPY3]]
     %0:_(s32) = COPY $sgpr0
     %1:_(s32) = COPY $sgpr1
     %2:_(s32) = COPY $vgpr0
@@ -115,23 +93,20 @@ body: |
 ...
 
 ---
-name: select_vss
+name: select_s32_vss
 legalized: true
-
-# GCN-LABEL: name: select_vss
-# GCN:  [[SGPR0:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
-# GCN:  [[SGPR1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
-# GCN:  [[VGPR0:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
-# GCN:  [[VGPR1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
-# GCN:  [[VCC:%[0-9]+]]:sgpr(s1) = G_ICMP intpred(ne), [[VGPR0]](s32), [[VGPR1]]
-# FAST:  [[SGPR0_V:%[0-9]+]]:vgpr(s32) = COPY [[SGPR0]]
-# GCN:  [[SGPR1_V:%[0-9]+]]:vgpr(s32) = COPY [[SGPR1]]
-# FAST:  {{%[0-9]+}}:vgpr(s32) = G_SELECT [[VCC]](s1), [[SGPR0_V]], [[SGPR1_V]]
-# GREDY: {{%[0-9]+}}:vgpr(s32) = G_SELECT [[VCC]](s1), [[SGPR0]], [[SGPR1_V]]
-
 body: |
   bb.0:
     liveins: $sgpr0, $sgpr1, $vgpr0, $vgpr1
+    ; CHECK-LABEL: name: select_s32_vss
+    ; CHECK: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
+    ; CHECK: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; CHECK: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+    ; CHECK: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(ne), [[COPY2]](s32), [[COPY3]]
+    ; CHECK: [[COPY4:%[0-9]+]]:vgpr(s32) = COPY [[COPY]](s32)
+    ; CHECK: [[COPY5:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32)
+    ; CHECK: [[SELECT:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP]](s1), [[COPY4]], [[COPY5]]
     %0:_(s32) = COPY $sgpr0
     %1:_(s32) = COPY $sgpr1
     %2:_(s32) = COPY $vgpr0
@@ -141,22 +116,19 @@ body: |
 ...
 
 ---
-name: select_vsv
+name: select_s32_vsv
 legalized: true
-
-# GCN-LABEL: name: select_vsv
-# GCN:  [[SGPR0:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
-# GCN:  [[VGPR0:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
-# GCN:  [[VGPR1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
-# GCN:  [[VGPR2:%[0-9]+]]:vgpr(s32) = COPY $vgpr2
-# GCN:  [[VCC:%[0-9]+]]:sgpr(s1) = G_ICMP intpred(ne), [[VGPR0]](s32), [[VGPR1]]
-# FAST:   [[SGPR0_V:%[0-9]+]]:vgpr(s32) = COPY [[SGPR0]]
-# FAST:   {{%[0-9]+}}:vgpr(s32) = G_SELECT [[VCC]](s1), [[SGPR0_V]], [[VGPR2]]
-# GREEDY: {{%[0-9]+}}:vgpr(s32) = G_SELECT [[VCC]](s1), [[SGPR0]], [[VGPR2]]
-
 body: |
   bb.0:
     liveins: $sgpr0, $vgpr0, $vgpr1, $vgpr2
+    ; CHECK-LABEL: name: select_s32_vsv
+    ; CHECK: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; CHECK: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+    ; CHECK: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY $vgpr2
+    ; CHECK: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(ne), [[COPY1]](s32), [[COPY2]]
+    ; CHECK: [[COPY4:%[0-9]+]]:vgpr(s32) = COPY [[COPY]](s32)
+    ; CHECK: [[SELECT:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP]](s1), [[COPY4]], [[COPY3]]
     %0:_(s32) = COPY $sgpr0
     %1:_(s32) = COPY $vgpr0
     %2:_(s32) = COPY $vgpr1
@@ -166,22 +138,19 @@ body: |
 ...
 
 ---
-name: select_vvs
+name: select_s32_vvs
 legalized: true
-
-# GCN-LABEL: name: select_vvs
-# GCN:  [[SGPR0:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
-# GCN:  [[VGPR0:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
-# GCN:  [[VGPR1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
-# GCN:  [[VGPR2:%[0-9]+]]:vgpr(s32) = COPY $vgpr2
-# GCN:  [[VCC:%[0-9]+]]:sgpr(s1) = G_ICMP intpred(ne), [[VGPR0]](s32), [[VGPR1]]
-# FAST:   [[SGPR0_V:%[0-9]+]]:vgpr(s32) = COPY [[SGPR0]]
-# FAST:   {{%[0-9]+}}:vgpr(s32) = G_SELECT [[VCC]](s1), [[VGPR2]], [[SGPR0_V]]
-# GREEDY: {{%[0-9]+}}:vgpr(s32) = G_SELECT [[VCC]](s1), [[VGPR2]], [[SGPR0]]
-
 body: |
   bb.0:
     liveins: $sgpr0, $vgpr0, $vgpr1, $vgpr2
+    ; CHECK-LABEL: name: select_s32_vvs
+    ; CHECK: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; CHECK: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+    ; CHECK: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY $vgpr2
+    ; CHECK: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(ne), [[COPY1]](s32), [[COPY2]]
+    ; CHECK: [[COPY4:%[0-9]+]]:vgpr(s32) = COPY [[COPY]](s32)
+    ; CHECK: [[SELECT:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP]](s1), [[COPY3]], [[COPY4]]
     %0:_(s32) = COPY $sgpr0
     %1:_(s32) = COPY $vgpr0
     %2:_(s32) = COPY $vgpr1
@@ -191,20 +160,18 @@ body: |
 ...
 
 ---
-name: select_vvv
+name: select_s32_vvv
 legalized: true
-
-# GCN-LABEL: name: select_vvv
-# GCN: [[VGPR0:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
-# GCN: [[VGPR1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
-# GCN: [[VGPR2:%[0-9]+]]:vgpr(s32) = COPY $vgpr2
-# GCN: [[VGPR3:%[0-9]+]]:vgpr(s32) = COPY $vgpr3
-# GCN: [[VCC:%[0-9]+]]:sgpr(s1) = G_ICMP intpred(ne), [[VGPR0]](s32), [[VGPR1]]
-# GCN: {{%[0-9]+}}:vgpr(s32) = G_SELECT [[VCC]](s1), [[VGPR2]], [[VGPR3]]
-
 body: |
   bb.0:
     liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3
+    ; CHECK-LABEL: name: select_s32_vvv
+    ; CHECK: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+    ; CHECK: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY $vgpr2
+    ; CHECK: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY $vgpr3
+    ; CHECK: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(ne), [[COPY]](s32), [[COPY1]]
+    ; CHECK: [[SELECT:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP]](s1), [[COPY2]], [[COPY3]]
     %0:_(s32) = COPY $vgpr0
     %1:_(s32) = COPY $vgpr1
     %2:_(s32) = COPY $vgpr2
diff --git a/test/CodeGen/AMDGPU/GlobalISel/regbankselect-sitofp.mir b/test/CodeGen/AMDGPU/GlobalISel/regbankselect-sitofp.mir
new file mode 100644
index 0000000000000000000000000000000000000000..e48292127d54718e5225eb76fefc9853dec3db14
--- /dev/null
+++ b/test/CodeGen/AMDGPU/GlobalISel/regbankselect-sitofp.mir
@@ -0,0 +1,31 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -march=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
+# RUN: llc -march=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
+
+---
+name: sitofp_s
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0
+    ; CHECK-LABEL: name: sitofp_s
+    ; CHECK: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; CHECK: [[SITOFP:%[0-9]+]]:vgpr(s32) = G_SITOFP [[COPY]](s32)
+    %0:_(s32) = COPY $sgpr0
+    %1:_(s32) = G_SITOFP %0
+...
+
+---
+name: sitofp_v
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $vgpr0
+    ; CHECK-LABEL: name: sitofp_v
+    ; CHECK: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; CHECK: [[SITOFP:%[0-9]+]]:vgpr(s32) = G_SITOFP [[COPY]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = G_SITOFP %0
+...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/regbankselect-smulh.mir b/test/CodeGen/AMDGPU/GlobalISel/regbankselect-smulh.mir
new file mode 100644
index 0000000000000000000000000000000000000000..4eb3cae9ef90b1541b2d0f8e1b8d554135cc9118
--- /dev/null
+++ b/test/CodeGen/AMDGPU/GlobalISel/regbankselect-smulh.mir
@@ -0,0 +1,67 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -march=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
+
+---
+name: smulh_s32_ss
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0, $sgpr1
+    ; CHECK-LABEL: name: smulh_s32_ss
+    ; CHECK: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
+    ; CHECK: [[SMULH:%[0-9]+]]:sgpr(s32) = G_SMULH [[COPY]], [[COPY1]]
+    %0:_(s32) = COPY $sgpr0
+    %1:_(s32) = COPY $sgpr1
+    %2:_(s32) = G_SMULH %0, %1
+...
+
+---
+name: smulh_s32_sv
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0, $vgpr0
+    ; CHECK-LABEL: name: smulh_s32_sv
+    ; CHECK: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; CHECK: [[SMULH:%[0-9]+]]:vgpr(s32) = G_SMULH [[COPY]], [[COPY1]]
+    %0:_(s32) = COPY $sgpr0
+    %1:_(s32) = COPY $vgpr0
+    %2:_(s32) = G_SMULH %0, %1
+...
+
+---
+name: smulh_s32_vs
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0, $vgpr0
+    ; CHECK-LABEL: name: smulh_s32_vs
+    ; CHECK: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; CHECK: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32)
+    ; CHECK: [[SMULH:%[0-9]+]]:vgpr(s32) = G_SMULH [[COPY]], [[COPY2]]
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $sgpr0
+    %2:_(s32) = G_SMULH %0, %1
+...
+
+---
+name: smulh_s32_vv
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+    ; CHECK-LABEL: name: smulh_s32_vv
+    ; CHECK: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+    ; CHECK: [[SMULH:%[0-9]+]]:vgpr(s32) = G_SMULH [[COPY]], [[COPY1]]
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s32) = G_SMULH %0, %1
+...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/regbankselect-ssube.mir b/test/CodeGen/AMDGPU/GlobalISel/regbankselect-ssube.mir
new file mode 100644
index 0000000000000000000000000000000000000000..ca360cdd3760530d1a814d3aefd38f07b919cd40
--- /dev/null
+++ b/test/CodeGen/AMDGPU/GlobalISel/regbankselect-ssube.mir
@@ -0,0 +1,154 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -march=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck -check-prefix=FAST %s
+# RUN: llc -march=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck -check-prefix=GREEDY %s
+
+---
+name: ssube_s32_sss
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0, $sgpr1, $sgpr2
+    ; FAST-LABEL: name: ssube_s32_sss
+    ; FAST: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; FAST: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
+    ; FAST: [[COPY2:%[0-9]+]]:sgpr(s32) = COPY $sgpr2
+    ; FAST: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
+    ; FAST: [[ICMP:%[0-9]+]]:scc(s1) = G_ICMP intpred(eq), [[COPY2]](s32), [[C]]
+    ; FAST: [[SSUBE:%[0-9]+]]:sgpr(s32), [[SSUBE1:%[0-9]+]]:scc(s1) = G_SSUBE [[COPY]], [[COPY1]], [[ICMP]]
+    ; GREEDY-LABEL: name: ssube_s32_sss
+    ; GREEDY: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; GREEDY: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
+    ; GREEDY: [[COPY2:%[0-9]+]]:sgpr(s32) = COPY $sgpr2
+    ; GREEDY: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
+    ; GREEDY: [[ICMP:%[0-9]+]]:scc(s1) = G_ICMP intpred(eq), [[COPY2]](s32), [[C]]
+    ; GREEDY: [[SSUBE:%[0-9]+]]:sgpr(s32), [[SSUBE1:%[0-9]+]]:scc(s1) = G_SSUBE [[COPY]], [[COPY1]], [[ICMP]]
+    %0:_(s32) = COPY $sgpr0
+    %1:_(s32) = COPY $sgpr1
+    %2:_(s32) = COPY $sgpr2
+    %3:_(s32) = G_CONSTANT i32 0
+    %4:_(s1) = G_ICMP intpred(eq), %2, %3
+    %5:_(s32), %6:_(s1) = G_SSUBE %0, %1, %4
+...
+
+---
+name: ssube_s32_vss
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $vgpr0, $sgpr0, $sgpr1
+    ; FAST-LABEL: name: ssube_s32_vss
+    ; FAST: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; FAST: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; FAST: [[COPY2:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
+    ; FAST: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
+    ; FAST: [[ICMP:%[0-9]+]]:scc(s1) = G_ICMP intpred(eq), [[COPY2]](s32), [[C]]
+    ; FAST: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32)
+    ; FAST: [[COPY4:%[0-9]+]]:vcc(s1) = COPY [[ICMP]](s1)
+    ; FAST: [[SSUBE:%[0-9]+]]:vgpr(s32), [[SSUBE1:%[0-9]+]]:vcc(s1) = G_SSUBE [[COPY]], [[COPY3]], [[COPY4]]
+    ; GREEDY-LABEL: name: ssube_s32_vss
+    ; GREEDY: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; GREEDY: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; GREEDY: [[COPY2:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
+    ; GREEDY: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
+    ; GREEDY: [[ICMP:%[0-9]+]]:scc(s1) = G_ICMP intpred(eq), [[COPY2]](s32), [[C]]
+    ; GREEDY: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32)
+    ; GREEDY: [[COPY4:%[0-9]+]]:vcc(s1) = COPY [[ICMP]](s1)
+    ; GREEDY: [[SSUBE:%[0-9]+]]:vgpr(s32), [[SSUBE1:%[0-9]+]]:vcc(s1) = G_SSUBE [[COPY]], [[COPY3]], [[COPY4]]
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $sgpr0
+    %2:_(s32) = COPY $sgpr1
+    %3:_(s32) = G_CONSTANT i32 0
+    %4:_(s1) = G_ICMP intpred(eq), %2, %3
+    %5:_(s32), %6:_(s1) = G_SSUBE %0, %1, %4
+...
+---
+name: ssube_s32_ssv
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0, $sgpr1, $vgpr0
+    ; FAST-LABEL: name: ssube_s32_ssv
+    ; FAST: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; FAST: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
+    ; FAST: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; FAST: [[TRUNC:%[0-9]+]]:vgpr(s1) = G_TRUNC [[COPY2]](s32)
+    ; FAST: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY [[COPY]](s32)
+    ; FAST: [[COPY4:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32)
+    ; FAST: [[COPY5:%[0-9]+]]:vcc(s1) = COPY [[TRUNC]](s1)
+    ; FAST: [[SSUBE:%[0-9]+]]:vgpr(s32), [[SSUBE1:%[0-9]+]]:vcc(s1) = G_SSUBE [[COPY3]], [[COPY4]], [[COPY5]]
+    ; GREEDY-LABEL: name: ssube_s32_ssv
+    ; GREEDY: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; GREEDY: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
+    ; GREEDY: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; GREEDY: [[TRUNC:%[0-9]+]]:vgpr(s1) = G_TRUNC [[COPY2]](s32)
+    ; GREEDY: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY [[COPY]](s32)
+    ; GREEDY: [[COPY4:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32)
+    ; GREEDY: [[COPY5:%[0-9]+]]:vcc(s1) = COPY [[TRUNC]](s1)
+    ; GREEDY: [[SSUBE:%[0-9]+]]:vgpr(s32), [[SSUBE1:%[0-9]+]]:vcc(s1) = G_SSUBE [[COPY3]], [[COPY4]], [[COPY5]]
+    %0:_(s32) = COPY $sgpr0
+    %1:_(s32) = COPY $sgpr1
+    %2:_(s32) = COPY $vgpr0
+    %3:_(s1) = G_TRUNC %2
+    %4:_(s32), %5:_(s1) = G_SSUBE %0, %1, %3
+...
+
+---
+name: ssube_s32_vvs
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $sgpr0
+    ; FAST-LABEL: name: ssube_s32_vvs
+    ; FAST: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; FAST: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+    ; FAST: [[COPY2:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; FAST: [[TRUNC:%[0-9]+]]:sgpr(s1) = G_TRUNC [[COPY2]](s32)
+    ; FAST: [[COPY3:%[0-9]+]]:vcc(s1) = COPY [[TRUNC]](s1)
+    ; FAST: [[SSUBE:%[0-9]+]]:vgpr(s32), [[SSUBE1:%[0-9]+]]:vcc(s1) = G_SSUBE [[COPY]], [[COPY1]], [[COPY3]]
+    ; GREEDY-LABEL: name: ssube_s32_vvs
+    ; GREEDY: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; GREEDY: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+    ; GREEDY: [[COPY2:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; GREEDY: [[TRUNC:%[0-9]+]]:sgpr(s1) = G_TRUNC [[COPY2]](s32)
+    ; GREEDY: [[COPY3:%[0-9]+]]:vcc(s1) = COPY [[TRUNC]](s1)
+    ; GREEDY: [[SSUBE:%[0-9]+]]:vgpr(s32), [[SSUBE1:%[0-9]+]]:vcc(s1) = G_SSUBE [[COPY]], [[COPY1]], [[COPY3]]
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s32) = COPY $sgpr0
+    %3:_(s1) = G_TRUNC %2
+    %4:_(s32), %5:_(s1) = G_SSUBE %0, %1, %3
+...
+
+---
+name: ssubee_s32_sss_noscc
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0, $sgpr1, $sgpr2
+    ; FAST-LABEL: name: ssubee_s32_sss_noscc
+    ; FAST: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; FAST: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
+    ; FAST: [[COPY2:%[0-9]+]]:sgpr(s32) = COPY $sgpr2
+    ; FAST: [[TRUNC:%[0-9]+]]:sgpr(s1) = G_TRUNC [[COPY2]](s32)
+    ; FAST: [[COPY3:%[0-9]+]]:scc(s1) = COPY [[TRUNC]](s1)
+    ; FAST: [[SSUBE:%[0-9]+]]:sgpr(s32), [[SSUBE1:%[0-9]+]]:scc(s1) = G_SSUBE [[COPY]], [[COPY1]], [[COPY3]]
+    ; GREEDY-LABEL: name: ssubee_s32_sss_noscc
+    ; GREEDY: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; GREEDY: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
+    ; GREEDY: [[COPY2:%[0-9]+]]:sgpr(s32) = COPY $sgpr2
+    ; GREEDY: [[TRUNC:%[0-9]+]]:sgpr(s1) = G_TRUNC [[COPY2]](s32)
+    ; GREEDY: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY [[COPY]](s32)
+    ; GREEDY: [[COPY4:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32)
+    ; GREEDY: [[COPY5:%[0-9]+]]:vcc(s1) = COPY [[TRUNC]](s1)
+    ; GREEDY: [[SSUBE:%[0-9]+]]:vgpr(s32), [[SSUBE1:%[0-9]+]]:vcc(s1) = G_SSUBE [[COPY3]], [[COPY4]], [[COPY5]]
+    %0:_(s32) = COPY $sgpr0
+    %1:_(s32) = COPY $sgpr1
+    %2:_(s32) = COPY $sgpr2
+    %3:_(s1) = G_TRUNC %2
+    %4:_(s32), %5:_(s1) = G_SSUBE %0, %1, %3
+...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/regbankselect-ssubo.mir b/test/CodeGen/AMDGPU/GlobalISel/regbankselect-ssubo.mir
new file mode 100644
index 0000000000000000000000000000000000000000..63834e4e7464b7d085f4025487985e6bf02e418a
--- /dev/null
+++ b/test/CodeGen/AMDGPU/GlobalISel/regbankselect-ssubo.mir
@@ -0,0 +1,69 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -march=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
+# RUN: llc -march=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
+
+---
+name: ssubo_s32_ss
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0, $sgpr1
+    ; CHECK-LABEL: name: ssubo_s32_ss
+    ; CHECK: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
+    ; CHECK: [[SSUBO:%[0-9]+]]:sgpr(s32), [[SSUBO1:%[0-9]+]]:scc(s1) = G_SSUBO [[COPY]], [[COPY1]]
+    %0:_(s32) = COPY $sgpr0
+    %1:_(s32) = COPY $sgpr1
+    %2:_(s32), %3:_(s1) = G_SSUBO %0, %1
+...
+
+---
+name: ssubo_s32_sv
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0, $vgpr0
+    ; CHECK-LABEL: name: ssubo_s32_sv
+    ; CHECK: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; CHECK: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY [[COPY]](s32)
+    ; CHECK: [[SSUBO:%[0-9]+]]:vgpr(s32), [[SSUBO1:%[0-9]+]]:vcc(s1) = G_SSUBO [[COPY2]], [[COPY1]]
+    %0:_(s32) = COPY $sgpr0
+    %1:_(s32) = COPY $vgpr0
+    %2:_(s32), %3:_(s1) = G_SSUBO %0, %1
+...
+
+---
+name: ssubo_s32_vs
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0, $vgpr0
+    ; CHECK-LABEL: name: ssubo_s32_vs
+    ; CHECK: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; CHECK: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32)
+    ; CHECK: [[SSUBO:%[0-9]+]]:vgpr(s32), [[SSUBO1:%[0-9]+]]:vcc(s1) = G_SSUBO [[COPY]], [[COPY2]]
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $sgpr0
+    %2:_(s32), %3:_(s1) = G_SSUBO %0, %1
+...
+
+---
+name: ssubo_s32_vv
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+    ; CHECK-LABEL: name: ssubo_s32_vv
+    ; CHECK: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+    ; CHECK: [[SSUBO:%[0-9]+]]:vgpr(s32), [[SSUBO1:%[0-9]+]]:vcc(s1) = G_SSUBO [[COPY]], [[COPY1]]
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s32), %3:_(s1) = G_SSUBO %0, %1
+...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/regbankselect-trunc.mir b/test/CodeGen/AMDGPU/GlobalISel/regbankselect-trunc.mir
index 14de218554f45591f1575fd30ce106dbb820d45a..d171d9f9f9bf95c377604880acc6db08c69423e7 100644
--- a/test/CodeGen/AMDGPU/GlobalISel/regbankselect-trunc.mir
+++ b/test/CodeGen/AMDGPU/GlobalISel/regbankselect-trunc.mir
@@ -10,7 +10,7 @@ body: |
   bb.0:
     liveins: $sgpr0_sgpr1
     ; CHECK-LABEL: name: trunc_i64_to_i32_s
-    ; CHECK: [[COPY:%[0-9]+]]:sgpr(s64) = COPY $sgpr0
+    ; CHECK: [[COPY:%[0-9]+]]:sgpr(s64) = COPY $sgpr0_sgpr1
     ; CHECK: [[TRUNC:%[0-9]+]]:sgpr(s32) = G_TRUNC [[COPY]](s64)
     %0:_(s64) = COPY $sgpr0_sgpr1
     %1:_(s32) = G_TRUNC %0
@@ -24,8 +24,63 @@ body: |
   bb.0:
     liveins: $vgpr0_vgpr1
     ; CHECK-LABEL: name: trunc_i64_to_i32_v
-    ; CHECK: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0
+    ; CHECK: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
     ; CHECK: [[TRUNC:%[0-9]+]]:vgpr(s32) = G_TRUNC [[COPY]](s64)
     %0:_(s64) = COPY $vgpr0_vgpr1
     %1:_(s32) = G_TRUNC %0
 ...
+---
+name: trunc_i64_to_i1_s
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0_sgpr1
+    ; CHECK-LABEL: name: trunc_i64_to_i1_s
+    ; CHECK: [[COPY:%[0-9]+]]:sgpr(s64) = COPY $sgpr0_sgpr1
+    ; CHECK: [[TRUNC:%[0-9]+]]:sgpr(s1) = G_TRUNC [[COPY]](s64)
+    %0:_(s64) = COPY $sgpr0_sgpr1
+    %1:_(s1) = G_TRUNC %0
+...
+
+---
+name: trunc_i64_to_i1_v
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+    ; CHECK-LABEL: name: trunc_i64_to_i1_v
+    ; CHECK: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
+    ; CHECK: [[TRUNC:%[0-9]+]]:vgpr(s1) = G_TRUNC [[COPY]](s64)
+    %0:_(s64) = COPY $vgpr0_vgpr1
+    %1:_(s1) = G_TRUNC %0
+...
+
+---
+name: trunc_i32_to_i1_s
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0
+    ; CHECK-LABEL: name: trunc_i32_to_i1_s
+    ; CHECK: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; CHECK: [[TRUNC:%[0-9]+]]:sgpr(s1) = G_TRUNC [[COPY]](s32)
+    %0:_(s32) = COPY $sgpr0
+    %1:_(s1) = G_TRUNC %0
+...
+
+---
+name: trunc_i32_to_i1_v
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $vgpr0
+    ; CHECK-LABEL: name: trunc_i32_to_i1_v
+    ; CHECK: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; CHECK: [[TRUNC:%[0-9]+]]:vgpr(s1) = G_TRUNC [[COPY]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s1) = G_TRUNC %0
+...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/regbankselect-uadde.mir b/test/CodeGen/AMDGPU/GlobalISel/regbankselect-uadde.mir
new file mode 100644
index 0000000000000000000000000000000000000000..2477ea0760726994fed2e83e6dc49149678084e9
--- /dev/null
+++ b/test/CodeGen/AMDGPU/GlobalISel/regbankselect-uadde.mir
@@ -0,0 +1,153 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -march=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck -check-prefix=FAST %s
+# RUN: llc -march=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck -check-prefix=GREEDY %s
+---
+name: uadde_s32_sss
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0, $sgpr1, $sgpr2
+    ; FAST-LABEL: name: uadde_s32_sss
+    ; FAST: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; FAST: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
+    ; FAST: [[COPY2:%[0-9]+]]:sgpr(s32) = COPY $sgpr2
+    ; FAST: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
+    ; FAST: [[ICMP:%[0-9]+]]:scc(s1) = G_ICMP intpred(eq), [[COPY2]](s32), [[C]]
+    ; FAST: [[UADDE:%[0-9]+]]:sgpr(s32), [[UADDE1:%[0-9]+]]:scc(s1) = G_UADDE [[COPY]], [[COPY1]], [[ICMP]]
+    ; GREEDY-LABEL: name: uadde_s32_sss
+    ; GREEDY: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; GREEDY: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
+    ; GREEDY: [[COPY2:%[0-9]+]]:sgpr(s32) = COPY $sgpr2
+    ; GREEDY: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
+    ; GREEDY: [[ICMP:%[0-9]+]]:scc(s1) = G_ICMP intpred(eq), [[COPY2]](s32), [[C]]
+    ; GREEDY: [[UADDE:%[0-9]+]]:sgpr(s32), [[UADDE1:%[0-9]+]]:scc(s1) = G_UADDE [[COPY]], [[COPY1]], [[ICMP]]
+    %0:_(s32) = COPY $sgpr0
+    %1:_(s32) = COPY $sgpr1
+    %2:_(s32) = COPY $sgpr2
+    %3:_(s32) = G_CONSTANT i32 0
+    %4:_(s1) = G_ICMP intpred(eq), %2, %3
+    %5:_(s32), %6:_(s1) = G_UADDE %0, %1, %4
+...
+
+---
+name: uadde_s32_vss
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $vgpr0, $sgpr0, $sgpr1
+    ; FAST-LABEL: name: uadde_s32_vss
+    ; FAST: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; FAST: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; FAST: [[COPY2:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
+    ; FAST: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
+    ; FAST: [[ICMP:%[0-9]+]]:scc(s1) = G_ICMP intpred(eq), [[COPY2]](s32), [[C]]
+    ; FAST: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32)
+    ; FAST: [[COPY4:%[0-9]+]]:vcc(s1) = COPY [[ICMP]](s1)
+    ; FAST: [[UADDE:%[0-9]+]]:vgpr(s32), [[UADDE1:%[0-9]+]]:vcc(s1) = G_UADDE [[COPY]], [[COPY3]], [[COPY4]]
+    ; GREEDY-LABEL: name: uadde_s32_vss
+    ; GREEDY: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; GREEDY: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; GREEDY: [[COPY2:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
+    ; GREEDY: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
+    ; GREEDY: [[ICMP:%[0-9]+]]:scc(s1) = G_ICMP intpred(eq), [[COPY2]](s32), [[C]]
+    ; GREEDY: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32)
+    ; GREEDY: [[COPY4:%[0-9]+]]:vcc(s1) = COPY [[ICMP]](s1)
+    ; GREEDY: [[UADDE:%[0-9]+]]:vgpr(s32), [[UADDE1:%[0-9]+]]:vcc(s1) = G_UADDE [[COPY]], [[COPY3]], [[COPY4]]
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $sgpr0
+    %2:_(s32) = COPY $sgpr1
+    %3:_(s32) = G_CONSTANT i32 0
+    %4:_(s1) = G_ICMP intpred(eq), %2, %3
+    %5:_(s32), %6:_(s1) = G_UADDE %0, %1, %4
+...
+---
+name: uadde_s32_ssv
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0, $sgpr1, $vgpr0
+    ; FAST-LABEL: name: uadde_s32_ssv
+    ; FAST: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; FAST: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
+    ; FAST: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; FAST: [[TRUNC:%[0-9]+]]:vgpr(s1) = G_TRUNC [[COPY2]](s32)
+    ; FAST: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY [[COPY]](s32)
+    ; FAST: [[COPY4:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32)
+    ; FAST: [[COPY5:%[0-9]+]]:vcc(s1) = COPY [[TRUNC]](s1)
+    ; FAST: [[UADDE:%[0-9]+]]:vgpr(s32), [[UADDE1:%[0-9]+]]:vcc(s1) = G_UADDE [[COPY3]], [[COPY4]], [[COPY5]]
+    ; GREEDY-LABEL: name: uadde_s32_ssv
+    ; GREEDY: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; GREEDY: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
+    ; GREEDY: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; GREEDY: [[TRUNC:%[0-9]+]]:vgpr(s1) = G_TRUNC [[COPY2]](s32)
+    ; GREEDY: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY [[COPY]](s32)
+    ; GREEDY: [[COPY4:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32)
+    ; GREEDY: [[COPY5:%[0-9]+]]:vcc(s1) = COPY [[TRUNC]](s1)
+    ; GREEDY: [[UADDE:%[0-9]+]]:vgpr(s32), [[UADDE1:%[0-9]+]]:vcc(s1) = G_UADDE [[COPY3]], [[COPY4]], [[COPY5]]
+    %0:_(s32) = COPY $sgpr0
+    %1:_(s32) = COPY $sgpr1
+    %2:_(s32) = COPY $vgpr0
+    %3:_(s1) = G_TRUNC %2
+    %4:_(s32), %5:_(s1) = G_UADDE %0, %1, %3
+...
+
+---
+name: uadde_s32_vvs
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $sgpr0
+    ; FAST-LABEL: name: uadde_s32_vvs
+    ; FAST: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; FAST: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+    ; FAST: [[COPY2:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; FAST: [[TRUNC:%[0-9]+]]:sgpr(s1) = G_TRUNC [[COPY2]](s32)
+    ; FAST: [[COPY3:%[0-9]+]]:vcc(s1) = COPY [[TRUNC]](s1)
+    ; FAST: [[UADDE:%[0-9]+]]:vgpr(s32), [[UADDE1:%[0-9]+]]:vcc(s1) = G_UADDE [[COPY]], [[COPY1]], [[COPY3]]
+    ; GREEDY-LABEL: name: uadde_s32_vvs
+    ; GREEDY: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; GREEDY: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+    ; GREEDY: [[COPY2:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; GREEDY: [[TRUNC:%[0-9]+]]:sgpr(s1) = G_TRUNC [[COPY2]](s32)
+    ; GREEDY: [[COPY3:%[0-9]+]]:vcc(s1) = COPY [[TRUNC]](s1)
+    ; GREEDY: [[UADDE:%[0-9]+]]:vgpr(s32), [[UADDE1:%[0-9]+]]:vcc(s1) = G_UADDE [[COPY]], [[COPY1]], [[COPY3]]
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s32) = COPY $sgpr0
+    %3:_(s1) = G_TRUNC %2
+    %4:_(s32), %5:_(s1) = G_UADDE %0, %1, %3
+...
+
+---
+name: uadde_s32_sss_noscc
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0, $sgpr1, $sgpr2
+    ; FAST-LABEL: name: uadde_s32_sss_noscc
+    ; FAST: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; FAST: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
+    ; FAST: [[COPY2:%[0-9]+]]:sgpr(s32) = COPY $sgpr2
+    ; FAST: [[TRUNC:%[0-9]+]]:sgpr(s1) = G_TRUNC [[COPY2]](s32)
+    ; FAST: [[COPY3:%[0-9]+]]:scc(s1) = COPY [[TRUNC]](s1)
+    ; FAST: [[UADDE:%[0-9]+]]:sgpr(s32), [[UADDE1:%[0-9]+]]:scc(s1) = G_UADDE [[COPY]], [[COPY1]], [[COPY3]]
+    ; GREEDY-LABEL: name: uadde_s32_sss_noscc
+    ; GREEDY: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; GREEDY: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
+    ; GREEDY: [[COPY2:%[0-9]+]]:sgpr(s32) = COPY $sgpr2
+    ; GREEDY: [[TRUNC:%[0-9]+]]:sgpr(s1) = G_TRUNC [[COPY2]](s32)
+    ; GREEDY: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY [[COPY]](s32)
+    ; GREEDY: [[COPY4:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32)
+    ; GREEDY: [[COPY5:%[0-9]+]]:vcc(s1) = COPY [[TRUNC]](s1)
+    ; GREEDY: [[UADDE:%[0-9]+]]:vgpr(s32), [[UADDE1:%[0-9]+]]:vcc(s1) = G_UADDE [[COPY3]], [[COPY4]], [[COPY5]]
+    %0:_(s32) = COPY $sgpr0
+    %1:_(s32) = COPY $sgpr1
+    %2:_(s32) = COPY $sgpr2
+    %3:_(s1) = G_TRUNC %2
+    %4:_(s32), %5:_(s1) = G_UADDE %0, %1, %3
+...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/regbankselect-uaddo.mir b/test/CodeGen/AMDGPU/GlobalISel/regbankselect-uaddo.mir
new file mode 100644
index 0000000000000000000000000000000000000000..c6234fdf409426188c1d062f8b40d018bcebfded
--- /dev/null
+++ b/test/CodeGen/AMDGPU/GlobalISel/regbankselect-uaddo.mir
@@ -0,0 +1,69 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -march=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
+# RUN: llc -march=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
+
+---
+name: uaddo_s32_ss
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0, $sgpr1
+    ; CHECK-LABEL: name: uaddo_s32_ss
+    ; CHECK: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
+    ; CHECK: [[UADDO:%[0-9]+]]:sgpr(s32), [[UADDO1:%[0-9]+]]:scc(s1) = G_UADDO [[COPY]], [[COPY1]]
+    %0:_(s32) = COPY $sgpr0
+    %1:_(s32) = COPY $sgpr1
+    %2:_(s32), %3:_(s1) = G_UADDO %0, %1
+...
+
+---
+name: uaddo_s32_sv
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0, $vgpr0
+    ; CHECK-LABEL: name: uaddo_s32_sv
+    ; CHECK: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; CHECK: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY [[COPY]](s32)
+    ; CHECK: [[UADDO:%[0-9]+]]:vgpr(s32), [[UADDO1:%[0-9]+]]:vcc(s1) = G_UADDO [[COPY2]], [[COPY1]]
+    %0:_(s32) = COPY $sgpr0
+    %1:_(s32) = COPY $vgpr0
+    %2:_(s32),  %3:_(s1) = G_UADDO %0, %1
+...
+
+---
+name: uaddo_s32_vs
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0, $vgpr0
+    ; CHECK-LABEL: name: uaddo_s32_vs
+    ; CHECK: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; CHECK: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32)
+    ; CHECK: [[UADDO:%[0-9]+]]:vgpr(s32), [[UADDO1:%[0-9]+]]:vcc(s1) = G_UADDO [[COPY]], [[COPY2]]
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $sgpr0
+    %2:_(s32), %3:_(s1) = G_UADDO %0, %1
+...
+
+---
+name: uaddo_s32_vv
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+    ; CHECK-LABEL: name: uaddo_s32_vv
+    ; CHECK: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+    ; CHECK: [[UADDO:%[0-9]+]]:vgpr(s32), [[UADDO1:%[0-9]+]]:vcc(s1) = G_UADDO [[COPY]], [[COPY1]]
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s32), %3:_(s1) = G_UADDO %0, %1
+...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/regbankselect-uitofp.mir b/test/CodeGen/AMDGPU/GlobalISel/regbankselect-uitofp.mir
new file mode 100644
index 0000000000000000000000000000000000000000..07f47c559a77c1eb46408f396cbc30f12ffef881
--- /dev/null
+++ b/test/CodeGen/AMDGPU/GlobalISel/regbankselect-uitofp.mir
@@ -0,0 +1,31 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -march=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
+# RUN: llc -march=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
+
+---
+name: uitofp_s
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0
+    ; CHECK-LABEL: name: uitofp_s
+    ; CHECK: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; CHECK: [[UITOFP:%[0-9]+]]:vgpr(s32) = G_UITOFP [[COPY]](s32)
+    %0:_(s32) = COPY $sgpr0
+    %1:_(s32) = G_UITOFP %0
+...
+
+---
+name: uitofp_v
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $vgpr0
+    ; CHECK-LABEL: name: uitofp_v
+    ; CHECK: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; CHECK: [[UITOFP:%[0-9]+]]:vgpr(s32) = G_UITOFP [[COPY]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = G_UITOFP %0
+...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/regbankselect-umulh.mir b/test/CodeGen/AMDGPU/GlobalISel/regbankselect-umulh.mir
new file mode 100644
index 0000000000000000000000000000000000000000..d801149e7a16aa52592c8f52f14d8c5af0879d0e
--- /dev/null
+++ b/test/CodeGen/AMDGPU/GlobalISel/regbankselect-umulh.mir
@@ -0,0 +1,67 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -march=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
+
+---
+name: umulh_s32_ss
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0, $sgpr1
+    ; CHECK-LABEL: name: umulh_s32_ss
+    ; CHECK: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
+    ; CHECK: [[UMULH:%[0-9]+]]:sgpr(s32) = G_UMULH [[COPY]], [[COPY1]]
+    %0:_(s32) = COPY $sgpr0
+    %1:_(s32) = COPY $sgpr1
+    %2:_(s32) = G_UMULH %0, %1
+...
+
+---
+name: umulh_s32_sv
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0, $vgpr0
+    ; CHECK-LABEL: name: umulh_s32_sv
+    ; CHECK: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; CHECK: [[UMULH:%[0-9]+]]:vgpr(s32) = G_UMULH [[COPY]], [[COPY1]]
+    %0:_(s32) = COPY $sgpr0
+    %1:_(s32) = COPY $vgpr0
+    %2:_(s32) = G_UMULH %0, %1
+...
+
+---
+name: umulh_s32_vs
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0, $vgpr0
+    ; CHECK-LABEL: name: umulh_s32_vs
+    ; CHECK: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; CHECK: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32)
+    ; CHECK: [[UMULH:%[0-9]+]]:vgpr(s32) = G_UMULH [[COPY]], [[COPY2]]
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $sgpr0
+    %2:_(s32) = G_UMULH %0, %1
+...
+
+---
+name: umulh_s32_vv
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+    ; CHECK-LABEL: name: umulh_s32_vv
+    ; CHECK: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+    ; CHECK: [[UMULH:%[0-9]+]]:vgpr(s32) = G_UMULH [[COPY]], [[COPY1]]
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s32) = G_UMULH %0, %1
+...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/regbankselect-unmerge-values.mir b/test/CodeGen/AMDGPU/GlobalISel/regbankselect-unmerge-values.mir
new file mode 100644
index 0000000000000000000000000000000000000000..5a3057224cf3b5754db5cd265061fe2709c857ab
--- /dev/null
+++ b/test/CodeGen/AMDGPU/GlobalISel/regbankselect-unmerge-values.mir
@@ -0,0 +1,38 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -march=amdgcn -mcpu=fiji -run-pass=regbankselect -global-isel %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
+
+---
+name: test_unmerge_s64_s32_s
+legalized: true
+
+body: |
+  bb.0:
+   liveins: $sgpr0_sgpr1
+    ; CHECK-LABEL: name: test_unmerge_s64_s32_s
+    ; CHECK: [[COPY:%[0-9]+]]:sgpr(s64) = COPY $sgpr0_sgpr1
+    ; CHECK: [[UV:%[0-9]+]]:sgpr(s32), [[UV1:%[0-9]+]]:sgpr(s32) = G_UNMERGE_VALUES [[COPY]](s64)
+    ; CHECK: $vgpr0 = COPY [[UV]](s32)
+    ; CHECK: $vgpr2 = COPY [[UV]](s32)
+    %0:_(s64) = COPY $sgpr0_sgpr1
+    %1:_(s32), %2:_(s32) = G_UNMERGE_VALUES %0:_(s64)
+    $vgpr0 = COPY %1(s32)
+    $vgpr2 = COPY %1(s32)
+...
+
+---
+name: test_unmerge_s64_s32_v
+legalized: true
+
+body: |
+  bb.0:
+   liveins: $vgpr0_vgpr1
+    ; CHECK-LABEL: name: test_unmerge_s64_s32_v
+    ; CHECK: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
+    ; CHECK: [[UV:%[0-9]+]]:vgpr(s32), [[UV1:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[COPY]](s64)
+    ; CHECK: $vgpr0 = COPY [[UV]](s32)
+    ; CHECK: $vgpr2 = COPY [[UV]](s32)
+    %0:_(s64) = COPY $vgpr0_vgpr1
+    %1:_(s32), %2:_(s32) = G_UNMERGE_VALUES %0:_(s64)
+    $vgpr0 = COPY %1(s32)
+    $vgpr2 = COPY %1(s32)
+...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/regbankselect-usube.mir b/test/CodeGen/AMDGPU/GlobalISel/regbankselect-usube.mir
new file mode 100644
index 0000000000000000000000000000000000000000..f59d706473fee5dea5ececc0e0b3f7dd1efca049
--- /dev/null
+++ b/test/CodeGen/AMDGPU/GlobalISel/regbankselect-usube.mir
@@ -0,0 +1,154 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -march=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck -check-prefix=FAST %s
+# RUN: llc -march=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck -check-prefix=GREEDY %s
+
+---
+name: usube_s32_sss
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0, $sgpr1, $sgpr2
+    ; FAST-LABEL: name: usube_s32_sss
+    ; FAST: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; FAST: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
+    ; FAST: [[COPY2:%[0-9]+]]:sgpr(s32) = COPY $sgpr2
+    ; FAST: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
+    ; FAST: [[ICMP:%[0-9]+]]:scc(s1) = G_ICMP intpred(eq), [[COPY2]](s32), [[C]]
+    ; FAST: [[USUBE:%[0-9]+]]:sgpr(s32), [[USUBE1:%[0-9]+]]:scc(s1) = G_USUBE [[COPY]], [[COPY1]], [[ICMP]]
+    ; GREEDY-LABEL: name: usube_s32_sss
+    ; GREEDY: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; GREEDY: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
+    ; GREEDY: [[COPY2:%[0-9]+]]:sgpr(s32) = COPY $sgpr2
+    ; GREEDY: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
+    ; GREEDY: [[ICMP:%[0-9]+]]:scc(s1) = G_ICMP intpred(eq), [[COPY2]](s32), [[C]]
+    ; GREEDY: [[USUBE:%[0-9]+]]:sgpr(s32), [[USUBE1:%[0-9]+]]:scc(s1) = G_USUBE [[COPY]], [[COPY1]], [[ICMP]]
+    %0:_(s32) = COPY $sgpr0
+    %1:_(s32) = COPY $sgpr1
+    %2:_(s32) = COPY $sgpr2
+    %3:_(s32) = G_CONSTANT i32 0
+    %4:_(s1) = G_ICMP intpred(eq), %2, %3
+    %5:_(s32), %6:_(s1) = G_USUBE %0, %1, %4
+...
+
+---
+name: usube_s32_vss
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $vgpr0, $sgpr0, $sgpr1
+    ; FAST-LABEL: name: usube_s32_vss
+    ; FAST: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; FAST: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; FAST: [[COPY2:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
+    ; FAST: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
+    ; FAST: [[ICMP:%[0-9]+]]:scc(s1) = G_ICMP intpred(eq), [[COPY2]](s32), [[C]]
+    ; FAST: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32)
+    ; FAST: [[COPY4:%[0-9]+]]:vcc(s1) = COPY [[ICMP]](s1)
+    ; FAST: [[USUBE:%[0-9]+]]:vgpr(s32), [[USUBE1:%[0-9]+]]:vcc(s1) = G_USUBE [[COPY]], [[COPY3]], [[COPY4]]
+    ; GREEDY-LABEL: name: usube_s32_vss
+    ; GREEDY: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; GREEDY: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; GREEDY: [[COPY2:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
+    ; GREEDY: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
+    ; GREEDY: [[ICMP:%[0-9]+]]:scc(s1) = G_ICMP intpred(eq), [[COPY2]](s32), [[C]]
+    ; GREEDY: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32)
+    ; GREEDY: [[COPY4:%[0-9]+]]:vcc(s1) = COPY [[ICMP]](s1)
+    ; GREEDY: [[USUBE:%[0-9]+]]:vgpr(s32), [[USUBE1:%[0-9]+]]:vcc(s1) = G_USUBE [[COPY]], [[COPY3]], [[COPY4]]
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $sgpr0
+    %2:_(s32) = COPY $sgpr1
+    %3:_(s32) = G_CONSTANT i32 0
+    %4:_(s1) = G_ICMP intpred(eq), %2, %3
+    %5:_(s32), %6:_(s1) = G_USUBE %0, %1, %4
+...
+---
+name: usube_s32_ssv
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0, $sgpr1, $vgpr0
+    ; FAST-LABEL: name: usube_s32_ssv
+    ; FAST: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; FAST: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
+    ; FAST: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; FAST: [[TRUNC:%[0-9]+]]:vgpr(s1) = G_TRUNC [[COPY2]](s32)
+    ; FAST: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY [[COPY]](s32)
+    ; FAST: [[COPY4:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32)
+    ; FAST: [[COPY5:%[0-9]+]]:vcc(s1) = COPY [[TRUNC]](s1)
+    ; FAST: [[USUBE:%[0-9]+]]:vgpr(s32), [[USUBE1:%[0-9]+]]:vcc(s1) = G_USUBE [[COPY3]], [[COPY4]], [[COPY5]]
+    ; GREEDY-LABEL: name: usube_s32_ssv
+    ; GREEDY: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; GREEDY: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
+    ; GREEDY: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; GREEDY: [[TRUNC:%[0-9]+]]:vgpr(s1) = G_TRUNC [[COPY2]](s32)
+    ; GREEDY: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY [[COPY]](s32)
+    ; GREEDY: [[COPY4:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32)
+    ; GREEDY: [[COPY5:%[0-9]+]]:vcc(s1) = COPY [[TRUNC]](s1)
+    ; GREEDY: [[USUBE:%[0-9]+]]:vgpr(s32), [[USUBE1:%[0-9]+]]:vcc(s1) = G_USUBE [[COPY3]], [[COPY4]], [[COPY5]]
+    %0:_(s32) = COPY $sgpr0
+    %1:_(s32) = COPY $sgpr1
+    %2:_(s32) = COPY $vgpr0
+    %3:_(s1) = G_TRUNC %2
+    %4:_(s32), %5:_(s1) = G_USUBE %0, %1, %3
+...
+
+---
+name: usube_s32_vvs
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $sgpr0
+    ; FAST-LABEL: name: usube_s32_vvs
+    ; FAST: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; FAST: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+    ; FAST: [[COPY2:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; FAST: [[TRUNC:%[0-9]+]]:sgpr(s1) = G_TRUNC [[COPY2]](s32)
+    ; FAST: [[COPY3:%[0-9]+]]:vcc(s1) = COPY [[TRUNC]](s1)
+    ; FAST: [[USUBE:%[0-9]+]]:vgpr(s32), [[USUBE1:%[0-9]+]]:vcc(s1) = G_USUBE [[COPY]], [[COPY1]], [[COPY3]]
+    ; GREEDY-LABEL: name: usube_s32_vvs
+    ; GREEDY: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; GREEDY: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+    ; GREEDY: [[COPY2:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; GREEDY: [[TRUNC:%[0-9]+]]:sgpr(s1) = G_TRUNC [[COPY2]](s32)
+    ; GREEDY: [[COPY3:%[0-9]+]]:vcc(s1) = COPY [[TRUNC]](s1)
+    ; GREEDY: [[USUBE:%[0-9]+]]:vgpr(s32), [[USUBE1:%[0-9]+]]:vcc(s1) = G_USUBE [[COPY]], [[COPY1]], [[COPY3]]
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s32) = COPY $sgpr0
+    %3:_(s1) = G_TRUNC %2
+    %4:_(s32), %5:_(s1) = G_USUBE %0, %1, %3
+...
+
+---
+name: usube_s32_sss_noscc
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0, $sgpr1, $sgpr2
+    ; FAST-LABEL: name: usube_s32_sss_noscc
+    ; FAST: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; FAST: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
+    ; FAST: [[COPY2:%[0-9]+]]:sgpr(s32) = COPY $sgpr2
+    ; FAST: [[TRUNC:%[0-9]+]]:sgpr(s1) = G_TRUNC [[COPY2]](s32)
+    ; FAST: [[COPY3:%[0-9]+]]:scc(s1) = COPY [[TRUNC]](s1)
+    ; FAST: [[USUBE:%[0-9]+]]:sgpr(s32), [[USUBE1:%[0-9]+]]:scc(s1) = G_USUBE [[COPY]], [[COPY1]], [[COPY3]]
+    ; GREEDY-LABEL: name: usube_s32_sss_noscc
+    ; GREEDY: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; GREEDY: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
+    ; GREEDY: [[COPY2:%[0-9]+]]:sgpr(s32) = COPY $sgpr2
+    ; GREEDY: [[TRUNC:%[0-9]+]]:sgpr(s1) = G_TRUNC [[COPY2]](s32)
+    ; GREEDY: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY [[COPY]](s32)
+    ; GREEDY: [[COPY4:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32)
+    ; GREEDY: [[COPY5:%[0-9]+]]:vcc(s1) = COPY [[TRUNC]](s1)
+    ; GREEDY: [[USUBE:%[0-9]+]]:vgpr(s32), [[USUBE1:%[0-9]+]]:vcc(s1) = G_USUBE [[COPY3]], [[COPY4]], [[COPY5]]
+    %0:_(s32) = COPY $sgpr0
+    %1:_(s32) = COPY $sgpr1
+    %2:_(s32) = COPY $sgpr2
+    %3:_(s1) = G_TRUNC %2
+    %4:_(s32), %5:_(s1) = G_USUBE %0, %1, %3
+...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/regbankselect-usubo.mir b/test/CodeGen/AMDGPU/GlobalISel/regbankselect-usubo.mir
new file mode 100644
index 0000000000000000000000000000000000000000..64ed8fcdbe58cfab0747f84c22c42e5b5b3fbd81
--- /dev/null
+++ b/test/CodeGen/AMDGPU/GlobalISel/regbankselect-usubo.mir
@@ -0,0 +1,69 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -march=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
+# RUN: llc -march=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
+
+---
+name: usubo_s32_ss
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0, $sgpr1
+    ; CHECK-LABEL: name: usubo_s32_ss
+    ; CHECK: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
+    ; CHECK: [[USUBO:%[0-9]+]]:sgpr(s32), [[USUBO1:%[0-9]+]]:scc(s1) = G_USUBO [[COPY]], [[COPY1]]
+    %0:_(s32) = COPY $sgpr0
+    %1:_(s32) = COPY $sgpr1
+    %2:_(s32), %3:_(s1) = G_USUBO %0, %1
+...
+
+---
+name: usubo_s32_sv
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0, $vgpr0
+    ; CHECK-LABEL: name: usubo_s32_sv
+    ; CHECK: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; CHECK: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY [[COPY]](s32)
+    ; CHECK: [[USUBO:%[0-9]+]]:vgpr(s32), [[USUBO1:%[0-9]+]]:vcc(s1) = G_USUBO [[COPY2]], [[COPY1]]
+    %0:_(s32) = COPY $sgpr0
+    %1:_(s32) = COPY $vgpr0
+    %2:_(s32),  %3:_(s1) = G_USUBO %0, %1
+...
+
+---
+name: usubo_s32_vs
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0, $vgpr0
+    ; CHECK-LABEL: name: usubo_s32_vs
+    ; CHECK: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; CHECK: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32)
+    ; CHECK: [[USUBO:%[0-9]+]]:vgpr(s32), [[USUBO1:%[0-9]+]]:vcc(s1) = G_USUBO [[COPY]], [[COPY2]]
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $sgpr0
+    %2:_(s32), %3:_(s1) = G_USUBO %0, %1
+...
+
+---
+name: usubo_s32_vv
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+    ; CHECK-LABEL: name: usubo_s32_vv
+    ; CHECK: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+    ; CHECK: [[USUBO:%[0-9]+]]:vgpr(s32), [[USUBO1:%[0-9]+]]:vcc(s1) = G_USUBO [[COPY]], [[COPY1]]
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s32), %3:_(s1) = G_USUBO %0, %1
+...
diff --git a/test/CodeGen/AMDGPU/GlobalISel/regbankselect-xor.mir b/test/CodeGen/AMDGPU/GlobalISel/regbankselect-xor.mir
index 848126a85b5cec6d648026dad302e16c47cc3038..fab5ae8f9eaba3a3b37483d52a4b034b2d59c1ed 100644
--- a/test/CodeGen/AMDGPU/GlobalISel/regbankselect-xor.mir
+++ b/test/CodeGen/AMDGPU/GlobalISel/regbankselect-xor.mir
@@ -3,13 +3,13 @@
 # RUN: llc -march=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
 
 ---
-name: xor_ss
+name: xor_i32_ss
 legalized: true
 
 body: |
   bb.0:
     liveins: $sgpr0, $sgpr1
-    ; CHECK-LABEL: name: xor_ss
+    ; CHECK-LABEL: name: xor_i32_ss
     ; CHECK: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
     ; CHECK: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
     ; CHECK: [[XOR:%[0-9]+]]:sgpr(s32) = G_XOR [[COPY]], [[COPY1]]
@@ -19,13 +19,13 @@ body: |
 ...
 
 ---
-name: xor_sv
+name: xor_i32_sv
 legalized: true
 
 body: |
   bb.0:
     liveins: $sgpr0, $vgpr0
-    ; CHECK-LABEL: name: xor_sv
+    ; CHECK-LABEL: name: xor_i32_sv
     ; CHECK: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
     ; CHECK: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
     ; CHECK: [[XOR:%[0-9]+]]:vgpr(s32) = G_XOR [[COPY]], [[COPY1]]
@@ -35,13 +35,13 @@ body: |
 ...
 
 ---
-name: xor_vs
+name: xor_i32_vs
 legalized: true
 
 body: |
   bb.0:
     liveins: $sgpr0, $vgpr0
-    ; CHECK-LABEL: name: xor_vs
+    ; CHECK-LABEL: name: xor_i32_vs
     ; CHECK: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
     ; CHECK: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
     ; CHECK: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32)
@@ -52,13 +52,13 @@ body: |
 ...
 
 ---
-name: xor_vv
+name: xor_i32_vv
 legalized: true
 
 body: |
   bb.0:
     liveins: $vgpr0, $vgpr1
-    ; CHECK-LABEL: name: xor_vv
+    ; CHECK-LABEL: name: xor_i32_vv
     ; CHECK: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
     ; CHECK: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
     ; CHECK: [[XOR:%[0-9]+]]:vgpr(s32) = G_XOR [[COPY]], [[COPY1]]
diff --git a/test/CodeGen/AMDGPU/add3.ll b/test/CodeGen/AMDGPU/add3.ll
index 35055190b348e628033c1489105a425b31a20afb..e49f57ca448151d4ca8c76bb8fbaf7969efac3a0 100644
--- a/test/CodeGen/AMDGPU/add3.ll
+++ b/test/CodeGen/AMDGPU/add3.ll
@@ -23,6 +23,32 @@ define amdgpu_ps float @add3(i32 %a, i32 %b, i32 %c) {
   ret float %bc
 }
 
+; V_MAD_U32_U24 is given higher priority.
+define amdgpu_ps float @mad_no_add3(i32 %a, i32 %b, i32 %c, i32 %d, i32 %e) {
+; GFX9-LABEL: mad_no_add3:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    v_mad_u32_u24 v0, v0, v1, v4
+; GFX9-NEXT:    v_mad_u32_u24 v0, v2, v3, v0
+; GFX9-NEXT:    ; return to shader part epilog
+  %a0 = shl i32 %a, 8
+  %a1 = lshr i32 %a0, 8
+  %b0 = shl i32 %b, 8
+  %b1 = lshr i32 %b0, 8
+  %mul1 = mul i32 %a1, %b1
+
+  %c0 = shl i32 %c, 8
+  %c1 = lshr i32 %c0, 8
+  %d0 = shl i32 %d, 8
+  %d1 = lshr i32 %d0, 8
+  %mul2 = mul i32 %c1, %d1
+
+  %add0 = add i32 %e, %mul1
+  %add1 = add i32 %mul2, %add0
+
+  %bc = bitcast i32 %add1 to float
+  ret float %bc
+}
+
 ; ThreeOp instruction variant not used due to Constant Bus Limitations
 ; TODO: with reassociation it is possible to replace a v_add_u32_e32 with a s_add_i32
 define amdgpu_ps float @add3_vgpr_b(i32 inreg %a, i32 %b, i32 inreg %c) {
diff --git a/test/CodeGen/AMDGPU/amdpal.ll b/test/CodeGen/AMDGPU/amdpal.ll
index 8d4c09cdeaf556eb7642b4af035248e30b5c0e35..c6d082ded0487cd6dd0024b234fd3cfee6b15a4a 100644
--- a/test/CodeGen/AMDGPU/amdpal.ll
+++ b/test/CodeGen/AMDGPU/amdpal.ll
@@ -1,4 +1,5 @@
-; RUN: llc < %s -mtriple=amdgcn--amdpal -mcpu=tahiti | FileCheck --check-prefix=PAL --enable-var-scope %s
+; RUN: llc < %s -mtriple=amdgcn--amdpal -mcpu=tahiti | FileCheck --check-prefixes=PAL,CI --enable-var-scope %s
+; RUN: llc < %s -mtriple=amdgcn--amdpal -mcpu=tonga | FileCheck --check-prefixes=PAL,VI --enable-var-scope %s
 
 ; PAL-NOT: .AMDGPU.config
 ; PAL-LABEL: {{^}}simple:
@@ -55,11 +56,13 @@ entry:
 ; Check code sequence for amdpal use of scratch for alloca in a compute shader.
 ; The scratch descriptor is loaded from offset 0x10 of the GIT, rather than offset
 ; 0 in a graphics shader.
+; Prior to GCN3 s_load_dword offsets are dwords, so the offset will be 0x4.
 
 ; PAL-LABEL: {{^}}scratch2_cs:
 ; PAL: s_movk_i32 s{{[0-9]+}}, 0x1234
 ; PAL: s_mov_b32 s[[GITPTR:[0-9]+]], s0
-; PAL: s_load_dwordx4 s{{\[}}[[SCRATCHDESC:[0-9]+]]:{{[0-9]+]}}, s{{\[}}[[GITPTR]]:{{[0-9]+\]}}, 0x10
+; CI: s_load_dwordx4 s{{\[}}[[SCRATCHDESC:[0-9]+]]:{{[0-9]+]}}, s{{\[}}[[GITPTR]]:{{[0-9]+\]}}, 0x4
+; VI: s_load_dwordx4 s{{\[}}[[SCRATCHDESC:[0-9]+]]:{{[0-9]+]}}, s{{\[}}[[GITPTR]]:{{[0-9]+\]}}, 0x10
 ; PAL: buffer_store{{.*}}, s{{\[}}[[SCRATCHDESC]]:
 
 define amdgpu_cs void @scratch2_cs(i32 inreg, i32 inreg, i32 inreg, <3 x i32> inreg, i32 inreg, <3 x i32> %coord, <2 x i32> %in, i32 %extra, i32 %idx) #0 {
diff --git a/test/CodeGen/AMDGPU/atomic_optimizations_buffer.ll b/test/CodeGen/AMDGPU/atomic_optimizations_buffer.ll
index c4b814041c85d40f880bca8778d0dca3c6b9fe54..88824aa64eec4b1bb05998f2c3e1459cc73d65ee 100644
--- a/test/CodeGen/AMDGPU/atomic_optimizations_buffer.ll
+++ b/test/CodeGen/AMDGPU/atomic_optimizations_buffer.ll
@@ -1,6 +1,6 @@
 ; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -amdgpu-atomic-optimizations=true -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX7LESS %s
-; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=tonga -mattr=-flat-for-global -amdgpu-atomic-optimizations=true -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX8MORE %s
-; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=gfx900 -mattr=-flat-for-global -amdgpu-atomic-optimizations=true -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX8MORE %s
+; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=tonga -mattr=-flat-for-global -amdgpu-atomic-optimizations=true -amdgpu-dpp-combine=false -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX8MORE %s
+; RUN: llc -march=amdgcn -mtriple=amdgcn---amdgiz -mcpu=gfx900 -mattr=-flat-for-global -amdgpu-atomic-optimizations=true -amdgpu-dpp-combine=false -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX8MORE %s
 
 declare i32 @llvm.amdgcn.workitem.id.x()
 declare i32 @llvm.amdgcn.buffer.atomic.add(i32, <4 x i32>, i32, i32, i1)
@@ -9,7 +9,7 @@ declare i32 @llvm.amdgcn.buffer.atomic.sub(i32, <4 x i32>, i32, i32, i1)
 ; Show that what the atomic optimization pass will do for raw buffers.
 
 ; GCN-LABEL: add_i32_constant:
-; GCN: s_mov_b64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, exec
+; GCN: v_cmp_ne_u32_e64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, 1, 0
 ; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_lo:[0-9]+]], s[[exec_lo]], 0
 ; GCN: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_hi:[0-9]+]], s[[exec_hi]], v[[mbcnt_lo]]
 ; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc, 0, v[[mbcnt_hi]]
@@ -24,7 +24,7 @@ entry:
 }
 
 ; GCN-LABEL: add_i32_uniform:
-; GCN: s_mov_b64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, exec
+; GCN: v_cmp_ne_u32_e64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, 1, 0
 ; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_lo:[0-9]+]], s[[exec_lo]], 0
 ; GCN: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_hi:[0-9]+]], s[[exec_hi]], v[[mbcnt_lo]]
 ; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc, 0, v[[mbcnt_hi]]
@@ -44,6 +44,14 @@ entry:
 ; GFX7LESS-NOT: v_mbcnt_hi_u32_b32
 ; GFX7LESS-NOT: s_bcnt1_i32_b64
 ; GFX7LESS: buffer_atomic_add v{{[0-9]+}}
+; GFX8MORE: v_mov_b32_dpp v[[wave_shr1:[0-9]+]], v{{[0-9]+}} wave_shr:1 row_mask:0xf bank_mask:0xf
+; GFX8MORE: v_mov_b32_dpp v{{[0-9]+}}, v[[wave_shr1]] row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX8MORE: v_mov_b32_dpp v{{[0-9]+}}, v[[wave_shr1]] row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX8MORE: v_mov_b32_dpp v{{[0-9]+}}, v[[wave_shr1]] row_shr:3 row_mask:0xf bank_mask:0xf
+; GFX8MORE: v_mov_b32_dpp v{{[0-9]+}}, v{{[0-9]+}} row_shr:4 row_mask:0xf bank_mask:0xe
+; GFX8MORE: v_mov_b32_dpp v{{[0-9]+}}, v{{[0-9]+}} row_shr:8 row_mask:0xf bank_mask:0xc
+; GFX8MORE: v_mov_b32_dpp v{{[0-9]+}}, v{{[0-9]+}} row_bcast:15 row_mask:0xa bank_mask:0xf
+; GFX8MORE: v_mov_b32_dpp v{{[0-9]+}}, v{{[0-9]+}} row_bcast:31 row_mask:0xc bank_mask:0xf
 ; GFX8MORE: v_readlane_b32 s[[scalar_value:[0-9]+]], v{{[0-9]+}}, 63
 ; GFX8MORE: v_mov_b32{{(_e[0-9]+)?}} v[[value:[0-9]+]], s[[scalar_value]]
 ; GFX8MORE: buffer_atomic_add v[[value]]
@@ -69,7 +77,7 @@ entry:
 }
 
 ; GCN-LABEL: sub_i32_constant:
-; GCN: s_mov_b64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, exec
+; GCN: v_cmp_ne_u32_e64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, 1, 0
 ; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_lo:[0-9]+]], s[[exec_lo]], 0
 ; GCN: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_hi:[0-9]+]], s[[exec_hi]], v[[mbcnt_lo]]
 ; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc, 0, v[[mbcnt_hi]]
@@ -84,7 +92,7 @@ entry:
 }
 
 ; GCN-LABEL: sub_i32_uniform:
-; GCN: s_mov_b64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, exec
+; GCN: v_cmp_ne_u32_e64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, 1, 0
 ; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_lo:[0-9]+]], s[[exec_lo]], 0
 ; GCN: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_hi:[0-9]+]], s[[exec_hi]], v[[mbcnt_lo]]
 ; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc, 0, v[[mbcnt_hi]]
@@ -104,6 +112,14 @@ entry:
 ; GFX7LESS-NOT: v_mbcnt_hi_u32_b32
 ; GFX7LESS-NOT: s_bcnt1_i32_b64
 ; GFX7LESS: buffer_atomic_sub v{{[0-9]+}}
+; GFX8MORE: v_mov_b32_dpp v[[wave_shr1:[0-9]+]], v{{[0-9]+}} wave_shr:1 row_mask:0xf bank_mask:0xf
+; GFX8MORE: v_mov_b32_dpp v{{[0-9]+}}, v[[wave_shr1]] row_shr:1 row_mask:0xf bank_mask:0xf
+; GFX8MORE: v_mov_b32_dpp v{{[0-9]+}}, v[[wave_shr1]] row_shr:2 row_mask:0xf bank_mask:0xf
+; GFX8MORE: v_mov_b32_dpp v{{[0-9]+}}, v[[wave_shr1]] row_shr:3 row_mask:0xf bank_mask:0xf
+; GFX8MORE: v_mov_b32_dpp v{{[0-9]+}}, v{{[0-9]+}} row_shr:4 row_mask:0xf bank_mask:0xe
+; GFX8MORE: v_mov_b32_dpp v{{[0-9]+}}, v{{[0-9]+}} row_shr:8 row_mask:0xf bank_mask:0xc
+; GFX8MORE: v_mov_b32_dpp v{{[0-9]+}}, v{{[0-9]+}} row_bcast:15 row_mask:0xa bank_mask:0xf
+; GFX8MORE: v_mov_b32_dpp v{{[0-9]+}}, v{{[0-9]+}} row_bcast:31 row_mask:0xc bank_mask:0xf
 ; GFX8MORE: v_readlane_b32 s[[scalar_value:[0-9]+]], v{{[0-9]+}}, 63
 ; GFX8MORE: v_mov_b32{{(_e[0-9]+)?}} v[[value:[0-9]+]], s[[scalar_value]]
 ; GFX8MORE: buffer_atomic_sub v[[value]]
diff --git a/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll b/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll
index 2e51aba5f69c731a468de55d81af6165f8d9ac31..644814f4fa024200c597218fe634670bbfb70b9b 100644
--- a/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll
+++ b/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll
@@ -7,7 +7,7 @@ declare i32 @llvm.amdgcn.workitem.id.x()
 ; Show that what the atomic optimization pass will do for global pointers.
 
 ; GCN-LABEL: add_i32_constant:
-; GCN: s_mov_b64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, exec
+; GCN: v_cmp_ne_u32_e64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, 1, 0
 ; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_lo:[0-9]+]], s[[exec_lo]], 0
 ; GCN: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_hi:[0-9]+]], s[[exec_hi]], v[[mbcnt_lo]]
 ; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc, 0, v[[mbcnt_hi]]
@@ -22,7 +22,7 @@ entry:
 }
 
 ; GCN-LABEL: add_i32_uniform:
-; GCN: s_mov_b64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, exec
+; GCN: v_cmp_ne_u32_e64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, 1, 0
 ; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_lo:[0-9]+]], s[[exec_lo]], 0
 ; GCN: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_hi:[0-9]+]], s[[exec_hi]], v[[mbcnt_lo]]
 ; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc, 0, v[[mbcnt_hi]]
@@ -54,7 +54,7 @@ entry:
 }
 
 ; GCN-LABEL: add_i64_constant:
-; GCN: s_mov_b64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, exec
+; GCN: v_cmp_ne_u32_e64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, 1, 0
 ; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_lo:[0-9]+]], s[[exec_lo]], 0
 ; GCN: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_hi:[0-9]+]], s[[exec_hi]], v[[mbcnt_lo]]
 ; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc, 0, v[[mbcnt_hi]]
@@ -70,7 +70,7 @@ entry:
 }
 
 ; GCN-LABEL: add_i64_uniform:
-; GCN: s_mov_b64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, exec
+; GCN: v_cmp_ne_u32_e64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, 1, 0
 ; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_lo:[0-9]+]], s[[exec_lo]], 0
 ; GCN: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_hi:[0-9]+]], s[[exec_hi]], v[[mbcnt_lo]]
 ; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc, 0, v[[mbcnt_hi]]
@@ -98,7 +98,7 @@ entry:
 }
 
 ; GCN-LABEL: sub_i32_constant:
-; GCN: s_mov_b64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, exec
+; GCN: v_cmp_ne_u32_e64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, 1, 0
 ; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_lo:[0-9]+]], s[[exec_lo]], 0
 ; GCN: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_hi:[0-9]+]], s[[exec_hi]], v[[mbcnt_lo]]
 ; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc, 0, v[[mbcnt_hi]]
@@ -113,7 +113,7 @@ entry:
 }
 
 ; GCN-LABEL: sub_i32_uniform:
-; GCN: s_mov_b64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, exec
+; GCN: v_cmp_ne_u32_e64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, 1, 0
 ; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_lo:[0-9]+]], s[[exec_lo]], 0
 ; GCN: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_hi:[0-9]+]], s[[exec_hi]], v[[mbcnt_lo]]
 ; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc, 0, v[[mbcnt_hi]]
@@ -145,7 +145,7 @@ entry:
 }
 
 ; GCN-LABEL: sub_i64_constant:
-; GCN: s_mov_b64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, exec
+; GCN: v_cmp_ne_u32_e64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, 1, 0
 ; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_lo:[0-9]+]], s[[exec_lo]], 0
 ; GCN: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_hi:[0-9]+]], s[[exec_hi]], v[[mbcnt_lo]]
 ; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc, 0, v[[mbcnt_hi]]
@@ -161,7 +161,7 @@ entry:
 }
 
 ; GCN-LABEL: sub_i64_uniform:
-; GCN: s_mov_b64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, exec
+; GCN: v_cmp_ne_u32_e64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, 1, 0
 ; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_lo:[0-9]+]], s[[exec_lo]], 0
 ; GCN: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_hi:[0-9]+]], s[[exec_hi]], v[[mbcnt_lo]]
 ; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc, 0, v[[mbcnt_hi]]
diff --git a/test/CodeGen/AMDGPU/atomic_optimizations_local_pointer.ll b/test/CodeGen/AMDGPU/atomic_optimizations_local_pointer.ll
index 849e2b61803d5280dec20a79b909b49dd5180bb9..3ce91e83cf3341cfe38a4e0472f7f7146766573f 100644
--- a/test/CodeGen/AMDGPU/atomic_optimizations_local_pointer.ll
+++ b/test/CodeGen/AMDGPU/atomic_optimizations_local_pointer.ll
@@ -10,7 +10,7 @@ declare i32 @llvm.amdgcn.workitem.id.x()
 ; Show that what the atomic optimization pass will do for local pointers.
 
 ; GCN-LABEL: add_i32_constant:
-; GCN: s_mov_b64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, exec
+; GCN: v_cmp_ne_u32_e64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, 1, 0
 ; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_lo:[0-9]+]], s[[exec_lo]], 0
 ; GCN: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_hi:[0-9]+]], s[[exec_hi]], v[[mbcnt_lo]]
 ; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc, 0, v[[mbcnt_hi]]
@@ -25,7 +25,7 @@ entry:
 }
 
 ; GCN-LABEL: add_i32_uniform:
-; GCN: s_mov_b64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, exec
+; GCN: v_cmp_ne_u32_e64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, 1, 0
 ; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_lo:[0-9]+]], s[[exec_lo]], 0
 ; GCN: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_hi:[0-9]+]], s[[exec_hi]], v[[mbcnt_lo]]
 ; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc, 0, v[[mbcnt_hi]]
@@ -57,7 +57,7 @@ entry:
 }
 
 ; GCN-LABEL: add_i64_constant:
-; GCN: s_mov_b64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, exec
+; GCN: v_cmp_ne_u32_e64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, 1, 0
 ; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_lo:[0-9]+]], s[[exec_lo]], 0
 ; GCN: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_hi:[0-9]+]], s[[exec_hi]], v[[mbcnt_lo]]
 ; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc, 0, v[[mbcnt_hi]]
@@ -73,7 +73,7 @@ entry:
 }
 
 ; GCN-LABEL: add_i64_uniform:
-; GCN: s_mov_b64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, exec
+; GCN: v_cmp_ne_u32_e64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, 1, 0
 ; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_lo:[0-9]+]], s[[exec_lo]], 0
 ; GCN: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_hi:[0-9]+]], s[[exec_hi]], v[[mbcnt_lo]]
 ; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc, 0, v[[mbcnt_hi]]
@@ -101,7 +101,7 @@ entry:
 }
 
 ; GCN-LABEL: sub_i32_constant:
-; GCN: s_mov_b64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, exec
+; GCN: v_cmp_ne_u32_e64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, 1, 0
 ; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_lo:[0-9]+]], s[[exec_lo]], 0
 ; GCN: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_hi:[0-9]+]], s[[exec_hi]], v[[mbcnt_lo]]
 ; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc, 0, v[[mbcnt_hi]]
@@ -116,7 +116,7 @@ entry:
 }
 
 ; GCN-LABEL: sub_i32_uniform:
-; GCN: s_mov_b64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, exec
+; GCN: v_cmp_ne_u32_e64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, 1, 0
 ; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_lo:[0-9]+]], s[[exec_lo]], 0
 ; GCN: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_hi:[0-9]+]], s[[exec_hi]], v[[mbcnt_lo]]
 ; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc, 0, v[[mbcnt_hi]]
@@ -148,7 +148,7 @@ entry:
 }
 
 ; GCN-LABEL: sub_i64_constant:
-; GCN: s_mov_b64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, exec
+; GCN: v_cmp_ne_u32_e64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, 1, 0
 ; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_lo:[0-9]+]], s[[exec_lo]], 0
 ; GCN: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_hi:[0-9]+]], s[[exec_hi]], v[[mbcnt_lo]]
 ; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc, 0, v[[mbcnt_hi]]
@@ -164,7 +164,7 @@ entry:
 }
 
 ; GCN-LABEL: sub_i64_uniform:
-; GCN: s_mov_b64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, exec
+; GCN: v_cmp_ne_u32_e64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, 1, 0
 ; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_lo:[0-9]+]], s[[exec_lo]], 0
 ; GCN: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_hi:[0-9]+]], s[[exec_hi]], v[[mbcnt_lo]]
 ; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc, 0, v[[mbcnt_hi]]
diff --git a/test/CodeGen/AMDGPU/atomic_optimizations_pixelshader.ll b/test/CodeGen/AMDGPU/atomic_optimizations_pixelshader.ll
index 4b4b268df1faa61638a9a5f81a301b0744cb930c..c1ce93242eeeab1081e8a5f127cee973ae77902e 100644
--- a/test/CodeGen/AMDGPU/atomic_optimizations_pixelshader.ll
+++ b/test/CodeGen/AMDGPU/atomic_optimizations_pixelshader.ll
@@ -10,7 +10,7 @@ declare void @llvm.amdgcn.buffer.store.f32(float, <4 x i32>, i32, i32, i1, i1)
 
 ; GCN-LABEL: add_i32_constant:
 ; GCN-LABEL: BB0_1:
-; GCN: s_mov_b64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, exec
+; GCN: v_cmp_ne_u32_e64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, 1, 0
 ; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_lo:[0-9]+]], s[[exec_lo]], 0
 ; GCN: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_hi:[0-9]+]], s[[exec_hi]], v[[mbcnt_lo]]
 ; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc, 0, v[[mbcnt_hi]]
@@ -36,8 +36,9 @@ else:
 ; GCN-LABEL: add_i32_varying:
 ; GFX7LESS-NOT: v_mbcnt_lo_u32_b32
 ; GFX7LESS-NOT: v_mbcnt_hi_u32_b32
-; GFX8MORE: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_lo:[0-9]+]], exec_lo, 0
-; GFX8MORE: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_hi:[0-9]+]], exec_hi, v[[mbcnt_lo]]
+; GFX8MORE: v_cmp_ne_u32_e64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, 1, 0
+; GFX8MORE: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_lo:[0-9]+]], s[[exec_lo]], 0
+; GFX8MORE: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_hi:[0-9]+]], s[[exec_hi]], v[[mbcnt_lo]]
 ; GFX8MORE: v_readlane_b32 s[[scalar_value:[0-9]+]], v{{[0-9]+}}, 63
 ; GFX8MORE: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc, 0, v[[mbcnt_hi]]
 ; GFX8MORE: v_mov_b32{{(_e[0-9]+)?}} v[[value:[0-9]+]], s[[scalar_value]]
diff --git a/test/CodeGen/AMDGPU/atomic_optimizations_raw_buffer.ll b/test/CodeGen/AMDGPU/atomic_optimizations_raw_buffer.ll
index 3e71e02ebfad4ba37ace20e5aa621f7886f73a9c..c2db5547201f966b23667c621c68db49071df95d 100644
--- a/test/CodeGen/AMDGPU/atomic_optimizations_raw_buffer.ll
+++ b/test/CodeGen/AMDGPU/atomic_optimizations_raw_buffer.ll
@@ -9,7 +9,7 @@ declare i32 @llvm.amdgcn.raw.buffer.atomic.sub(i32, <4 x i32>, i32, i32, i32)
 ; Show that what the atomic optimization pass will do for raw buffers.
 
 ; GCN-LABEL: add_i32_constant:
-; GCN: s_mov_b64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, exec
+; GCN: v_cmp_ne_u32_e64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, 1, 0
 ; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_lo:[0-9]+]], s[[exec_lo]], 0
 ; GCN: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_hi:[0-9]+]], s[[exec_hi]], v[[mbcnt_lo]]
 ; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc, 0, v[[mbcnt_hi]]
@@ -24,7 +24,7 @@ entry:
 }
 
 ; GCN-LABEL: add_i32_uniform:
-; GCN: s_mov_b64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, exec
+; GCN: v_cmp_ne_u32_e64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, 1, 0
 ; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_lo:[0-9]+]], s[[exec_lo]], 0
 ; GCN: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_hi:[0-9]+]], s[[exec_hi]], v[[mbcnt_lo]]
 ; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc, 0, v[[mbcnt_hi]]
@@ -69,7 +69,7 @@ entry:
 }
 
 ; GCN-LABEL: sub_i32_constant:
-; GCN: s_mov_b64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, exec
+; GCN: v_cmp_ne_u32_e64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, 1, 0
 ; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_lo:[0-9]+]], s[[exec_lo]], 0
 ; GCN: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_hi:[0-9]+]], s[[exec_hi]], v[[mbcnt_lo]]
 ; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc, 0, v[[mbcnt_hi]]
@@ -84,7 +84,7 @@ entry:
 }
 
 ; GCN-LABEL: sub_i32_uniform:
-; GCN: s_mov_b64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, exec
+; GCN: v_cmp_ne_u32_e64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, 1, 0
 ; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_lo:[0-9]+]], s[[exec_lo]], 0
 ; GCN: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_hi:[0-9]+]], s[[exec_hi]], v[[mbcnt_lo]]
 ; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc, 0, v[[mbcnt_hi]]
diff --git a/test/CodeGen/AMDGPU/atomic_optimizations_struct_buffer.ll b/test/CodeGen/AMDGPU/atomic_optimizations_struct_buffer.ll
index dab6cbbe8a9b310b0c5f8441de994dfb28c3abf7..eb3f0ab17ac83588b7fc947faf3693b28b236b6b 100644
--- a/test/CodeGen/AMDGPU/atomic_optimizations_struct_buffer.ll
+++ b/test/CodeGen/AMDGPU/atomic_optimizations_struct_buffer.ll
@@ -9,7 +9,7 @@ declare i32 @llvm.amdgcn.struct.buffer.atomic.sub(i32, <4 x i32>, i32, i32, i32,
 ; Show that what the atomic optimization pass will do for struct buffers.
 
 ; GCN-LABEL: add_i32_constant:
-; GCN: s_mov_b64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, exec
+; GCN: v_cmp_ne_u32_e64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, 1, 0
 ; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_lo:[0-9]+]], s[[exec_lo]], 0
 ; GCN: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_hi:[0-9]+]], s[[exec_hi]], v[[mbcnt_lo]]
 ; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc, 0, v[[mbcnt_hi]]
@@ -24,7 +24,7 @@ entry:
 }
 
 ; GCN-LABEL: add_i32_uniform:
-; GCN: s_mov_b64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, exec
+; GCN: v_cmp_ne_u32_e64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, 1, 0
 ; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_lo:[0-9]+]], s[[exec_lo]], 0
 ; GCN: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_hi:[0-9]+]], s[[exec_hi]], v[[mbcnt_lo]]
 ; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc, 0, v[[mbcnt_hi]]
@@ -82,7 +82,7 @@ entry:
 }
 
 ; GCN-LABEL: sub_i32_constant:
-; GCN: s_mov_b64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, exec
+; GCN: v_cmp_ne_u32_e64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, 1, 0
 ; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_lo:[0-9]+]], s[[exec_lo]], 0
 ; GCN: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_hi:[0-9]+]], s[[exec_hi]], v[[mbcnt_lo]]
 ; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc, 0, v[[mbcnt_hi]]
@@ -97,7 +97,7 @@ entry:
 }
 
 ; GCN-LABEL: sub_i32_uniform:
-; GCN: s_mov_b64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, exec
+; GCN: v_cmp_ne_u32_e64 s{{\[}}[[exec_lo:[0-9]+]]:[[exec_hi:[0-9]+]]{{\]}}, 1, 0
 ; GCN: v_mbcnt_lo_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_lo:[0-9]+]], s[[exec_lo]], 0
 ; GCN: v_mbcnt_hi_u32_b32{{(_e[0-9]+)?}} v[[mbcnt_hi:[0-9]+]], s[[exec_hi]], v[[mbcnt_lo]]
 ; GCN: v_cmp_eq_u32{{(_e[0-9]+)?}} vcc, 0, v[[mbcnt_hi]]
diff --git a/test/CodeGen/AMDGPU/bitcast-v4f16-v4i16.ll b/test/CodeGen/AMDGPU/bitcast-v4f16-v4i16.ll
new file mode 100644
index 0000000000000000000000000000000000000000..24cc1c4e9ec380db89a45690d780c8831ea48af2
--- /dev/null
+++ b/test/CodeGen/AMDGPU/bitcast-v4f16-v4i16.ll
@@ -0,0 +1,35 @@
+; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -enable-var-scope %s
+
+; creating v4i16->v4f16 and v4f16->v4i16 bitcasts in the selection DAG is rather
+; difficult, so this test has to throw in some llvm.amdgcn.wqm to get them
+
+; LABEL: {{^}}test_to_i16:
+; CHECK: s_endpgm
+define amdgpu_ps void @test_to_i16(<4 x i32> inreg, <4 x half> inreg) #0 {
+  %a_tmp = call <4 x half> @llvm.amdgcn.wqm.v4f16(<4 x half> %1)
+  %a_i16_tmp = bitcast <4 x half> %a_tmp to <4 x i16>
+  %a_i16 = call <4 x i16> @llvm.amdgcn.wqm.v4i16(<4 x i16> %a_i16_tmp)
+
+  %a_i32 = bitcast <4 x i16> %a_i16 to <2 x i32>
+  call void @llvm.amdgcn.raw.buffer.store.v2i32(<2 x i32> %a_i32, <4 x i32> %0, i32 0, i32 0, i32 0)
+  ret void
+}
+
+; LABEL: {{^}}test_to_half:
+; CHECK: s_endpgm
+define amdgpu_ps void @test_to_half(<4 x i32> inreg, <4 x i16> inreg) #0 {
+  %a_tmp = call <4 x i16> @llvm.amdgcn.wqm.v4i16(<4 x i16> %1)
+  %a_half_tmp = bitcast <4 x i16> %a_tmp to <4 x half>
+  %a_half = call <4 x half> @llvm.amdgcn.wqm.v4f16(<4 x half> %a_half_tmp)
+
+  %a_i32 = bitcast <4 x half> %a_half to <2 x i32>
+  call void @llvm.amdgcn.raw.buffer.store.v2i32(<2 x i32> %a_i32, <4 x i32> %0, i32 0, i32 0, i32 0)
+  ret void
+}
+
+declare <4 x half> @llvm.amdgcn.wqm.v4f16(<4 x half>) #1
+declare <4 x i16> @llvm.amdgcn.wqm.v4i16(<4 x i16>) #1
+declare void @llvm.amdgcn.raw.buffer.store.v2i32(<2 x i32>, <4 x i32>, i32, i32, i32) #0
+
+attributes #0 = { nounwind }
+attributes #1 = { nounwind readonly }
diff --git a/test/CodeGen/AMDGPU/bug-vopc-commute.ll b/test/CodeGen/AMDGPU/bug-vopc-commute.ll
index e951b5e089279388f14e30ea8581cd909d9e1107..31788a9b6ce8c8b09b13d4531278a093ba99be6a 100644
--- a/test/CodeGen/AMDGPU/bug-vopc-commute.ll
+++ b/test/CodeGen/AMDGPU/bug-vopc-commute.ll
@@ -8,8 +8,8 @@
 ; of which were in SGPRs.
 define amdgpu_vs float @main(i32 %v) {
 main_body:
-  %d1 = call float @llvm.SI.load.const.v4i32(<4 x i32> undef, i32 960)
-  %d2 = call float @llvm.SI.load.const.v4i32(<4 x i32> undef, i32 976)
+  %d1 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> undef, i32 960, i32 0)
+  %d2 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> undef, i32 976, i32 0)
   br i1 undef, label %ENDIF56, label %IF57
 
 IF57:                                             ; preds = %ENDIF
@@ -41,7 +41,7 @@ ENDIF62:                                          ; preds = %ENDIF59
 }
 
 ; Function Attrs: nounwind readnone
-declare float @llvm.SI.load.const.v4i32(<4 x i32>, i32) #0
+declare float @llvm.amdgcn.s.buffer.load.f32(<4 x i32>, i32, i32) #0
 
 attributes #0 = { nounwind readnone }
 attributes #1 = { readnone }
diff --git a/test/CodeGen/AMDGPU/byval-frame-setup.ll b/test/CodeGen/AMDGPU/byval-frame-setup.ll
index 3a48ffdda9ed22deb0b0adc6cc268903ae866e3b..b606de17bb6c8b3369aca602b085fbcae2fc4ea2 100644
--- a/test/CodeGen/AMDGPU/byval-frame-setup.ll
+++ b/test/CodeGen/AMDGPU/byval-frame-setup.ll
@@ -14,7 +14,7 @@
 ; GCN-NOT: s32
 ; GCN: buffer_store_dword [[LOAD1]], off, s[0:3], s5 offset:20{{$}}
 ; GCN-NOT: s32
-define void @void_func_byval_struct(%struct.ByValStruct addrspace(5)* byval noalias nocapture align 4 %arg0, %struct.ByValStruct addrspace(5)* byval noalias nocapture align 4 %arg1) #1 {
+define hidden void @void_func_byval_struct(%struct.ByValStruct addrspace(5)* byval noalias nocapture align 4 %arg0, %struct.ByValStruct addrspace(5)* byval noalias nocapture align 4 %arg1) #1 {
 entry:
   %arrayidx = getelementptr inbounds %struct.ByValStruct, %struct.ByValStruct addrspace(5)* %arg0, i32 0, i32 0, i32 0
   %tmp = load volatile i32, i32 addrspace(5)* %arrayidx, align 4
@@ -193,7 +193,7 @@ entry:
 ; GCN-NOT: s32
 ; GCN: buffer_store_dword [[LOAD1]], off, s[0:3], s5 offset:24{{$}}
 ; GCN-NOT: s32
-define void @void_func_byval_struct_align8(%struct.ByValStruct addrspace(5)* byval noalias nocapture align 8 %arg0, %struct.ByValStruct addrspace(5)* byval noalias nocapture align 8 %arg1) #1 {
+define hidden void @void_func_byval_struct_align8(%struct.ByValStruct addrspace(5)* byval noalias nocapture align 8 %arg0, %struct.ByValStruct addrspace(5)* byval noalias nocapture align 8 %arg1) #1 {
 entry:
   %arrayidx = getelementptr inbounds %struct.ByValStruct, %struct.ByValStruct addrspace(5)* %arg0, i32 0, i32 0, i32 0
   %tmp = load volatile i32, i32 addrspace(5)* %arrayidx, align 8
diff --git a/test/CodeGen/AMDGPU/call-argument-types.ll b/test/CodeGen/AMDGPU/call-argument-types.ll
index 67614adc9152d6426895dca0eb6be40062182629..78ccc8cf267883e74face59e8cf8bca23b6d014b 100644
--- a/test/CodeGen/AMDGPU/call-argument-types.ll
+++ b/test/CodeGen/AMDGPU/call-argument-types.ll
@@ -3,56 +3,56 @@
 ; RUN: llc -march=amdgcn -mcpu=gfx900 -mattr=-flat-for-global -amdgpu-scalarize-global-loads=0 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX9,VI,MESA %s
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=fiji -mattr=-flat-for-global -amdgpu-scalarize-global-loads=0 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,VI,HSA %s
 
-declare void @external_void_func_i1(i1) #0
-declare void @external_void_func_i1_signext(i1 signext) #0
-declare void @external_void_func_i1_zeroext(i1 zeroext) #0
-
-declare void @external_void_func_i8(i8) #0
-declare void @external_void_func_i8_signext(i8 signext) #0
-declare void @external_void_func_i8_zeroext(i8 zeroext) #0
-
-declare void @external_void_func_i16(i16) #0
-declare void @external_void_func_i16_signext(i16 signext) #0
-declare void @external_void_func_i16_zeroext(i16 zeroext) #0
-
-declare void @external_void_func_i32(i32) #0
-declare void @external_void_func_i64(i64) #0
-declare void @external_void_func_v2i64(<2 x i64>) #0
-declare void @external_void_func_v3i64(<3 x i64>) #0
-declare void @external_void_func_v4i64(<4 x i64>) #0
-
-declare void @external_void_func_f16(half) #0
-declare void @external_void_func_f32(float) #0
-declare void @external_void_func_f64(double) #0
-declare void @external_void_func_v2f32(<2 x float>) #0
-declare void @external_void_func_v2f64(<2 x double>) #0
-declare void @external_void_func_v3f64(<3 x double>) #0
-
-declare void @external_void_func_v2i16(<2 x i16>) #0
-declare void @external_void_func_v2f16(<2 x half>) #0
-declare void @external_void_func_v3i16(<3 x i16>) #0
-declare void @external_void_func_v3f16(<3 x half>) #0
-declare void @external_void_func_v4i16(<4 x i16>) #0
-declare void @external_void_func_v4f16(<4 x half>) #0
-
-declare void @external_void_func_v2i32(<2 x i32>) #0
-declare void @external_void_func_v3i32(<3 x i32>) #0
-declare void @external_void_func_v3i32_i32(<3 x i32>, i32) #0
-declare void @external_void_func_v4i32(<4 x i32>) #0
-declare void @external_void_func_v8i32(<8 x i32>) #0
-declare void @external_void_func_v16i32(<16 x i32>) #0
-declare void @external_void_func_v32i32(<32 x i32>) #0
-declare void @external_void_func_v32i32_i32(<32 x i32>, i32) #0
+declare hidden void @external_void_func_i1(i1) #0
+declare hidden void @external_void_func_i1_signext(i1 signext) #0
+declare hidden void @external_void_func_i1_zeroext(i1 zeroext) #0
+
+declare hidden void @external_void_func_i8(i8) #0
+declare hidden void @external_void_func_i8_signext(i8 signext) #0
+declare hidden void @external_void_func_i8_zeroext(i8 zeroext) #0
+
+declare hidden void @external_void_func_i16(i16) #0
+declare hidden void @external_void_func_i16_signext(i16 signext) #0
+declare hidden void @external_void_func_i16_zeroext(i16 zeroext) #0
+
+declare hidden void @external_void_func_i32(i32) #0
+declare hidden void @external_void_func_i64(i64) #0
+declare hidden void @external_void_func_v2i64(<2 x i64>) #0
+declare hidden void @external_void_func_v3i64(<3 x i64>) #0
+declare hidden void @external_void_func_v4i64(<4 x i64>) #0
+
+declare hidden void @external_void_func_f16(half) #0
+declare hidden void @external_void_func_f32(float) #0
+declare hidden void @external_void_func_f64(double) #0
+declare hidden void @external_void_func_v2f32(<2 x float>) #0
+declare hidden void @external_void_func_v2f64(<2 x double>) #0
+declare hidden void @external_void_func_v3f64(<3 x double>) #0
+
+declare hidden void @external_void_func_v2i16(<2 x i16>) #0
+declare hidden void @external_void_func_v2f16(<2 x half>) #0
+declare hidden void @external_void_func_v3i16(<3 x i16>) #0
+declare hidden void @external_void_func_v3f16(<3 x half>) #0
+declare hidden void @external_void_func_v4i16(<4 x i16>) #0
+declare hidden void @external_void_func_v4f16(<4 x half>) #0
+
+declare hidden void @external_void_func_v2i32(<2 x i32>) #0
+declare hidden void @external_void_func_v3i32(<3 x i32>) #0
+declare hidden void @external_void_func_v3i32_i32(<3 x i32>, i32) #0
+declare hidden void @external_void_func_v4i32(<4 x i32>) #0
+declare hidden void @external_void_func_v8i32(<8 x i32>) #0
+declare hidden void @external_void_func_v16i32(<16 x i32>) #0
+declare hidden void @external_void_func_v32i32(<32 x i32>) #0
+declare hidden void @external_void_func_v32i32_i32(<32 x i32>, i32) #0
 
 ; return value and argument
-declare i32 @external_i32_func_i32(i32) #0
+declare hidden i32 @external_i32_func_i32(i32) #0
 
 ; Structs
-declare void @external_void_func_struct_i8_i32({ i8, i32 }) #0
-declare void @external_void_func_byval_struct_i8_i32({ i8, i32 } addrspace(5)* byval) #0
-declare void @external_void_func_sret_struct_i8_i32_byval_struct_i8_i32({ i8, i32 } addrspace(5)* sret, { i8, i32 } addrspace(5)* byval) #0
+declare hidden void @external_void_func_struct_i8_i32({ i8, i32 }) #0
+declare hidden void @external_void_func_byval_struct_i8_i32({ i8, i32 } addrspace(5)* byval) #0
+declare hidden void @external_void_func_sret_struct_i8_i32_byval_struct_i8_i32({ i8, i32 } addrspace(5)* sret, { i8, i32 } addrspace(5)* byval) #0
 
-declare void @external_void_func_v16i8(<16 x i8>) #0
+declare hidden void @external_void_func_v16i8(<16 x i8>) #0
 
 
 ; FIXME: Should be passing -1
@@ -764,8 +764,8 @@ entry:
   ret void
 }
 
-declare void @byval_align16_f64_arg(<32 x i32>, double addrspace(5)* byval align 16) #0
-declare void @stack_passed_f64_arg(<32 x i32>, double) #0
+declare hidden void @byval_align16_f64_arg(<32 x i32>, double addrspace(5)* byval align 16) #0
+declare hidden void @stack_passed_f64_arg(<32 x i32>, double) #0
 
 attributes #0 = { nounwind }
 attributes #1 = { nounwind readnone }
diff --git a/test/CodeGen/AMDGPU/call-constexpr.ll b/test/CodeGen/AMDGPU/call-constexpr.ll
index e0a39680bdf08af0c002ef20c4000b30da0d60db..0ebf11dc72f5c18eb948b957dac19660c8b1a91e 100644
--- a/test/CodeGen/AMDGPU/call-constexpr.ll
+++ b/test/CodeGen/AMDGPU/call-constexpr.ll
@@ -67,7 +67,7 @@ define amdgpu_kernel void @test_bitcast_argument_and_return_types() #0 {
 ; GCN: s_waitcnt
 ; GCN-NEXT: v_add_i32_e32 v0, vcc, v1, v0
 ; GCN-NEXT: s_setpc_b64
-define i32 @use_workitem_id_x(i32 %arg0) #0 {
+define hidden i32 @use_workitem_id_x(i32 %arg0) #0 {
   %id = call i32 @llvm.amdgcn.workitem.id.x()
   %op = add i32 %id, %arg0
   ret i32 %op
@@ -121,15 +121,15 @@ continue:
 ; Callees appears last in source file to test that we still lower their
 ; arguments before we lower any calls to them.
 
-define i32 @ret_i32_noinline() #0 {
+define hidden i32 @ret_i32_noinline() #0 {
   ret i32 4
 }
 
-define i32 @ret_i32_alwaysinline() #1 {
+define hidden i32 @ret_i32_alwaysinline() #1 {
   ret i32 4
 }
 
-define i32 @ident_i32(i32 %i) #0 {
+define hidden i32 @ident_i32(i32 %i) #0 {
   ret i32 %i
 }
 
diff --git a/test/CodeGen/AMDGPU/call-preserved-registers.ll b/test/CodeGen/AMDGPU/call-preserved-registers.ll
index 57bc6171d7a86e6d47c293bfcdab8f5882d8d258..ee18d322914bdd22a40d356f0ce75dd18ef8e63d 100644
--- a/test/CodeGen/AMDGPU/call-preserved-registers.ll
+++ b/test/CodeGen/AMDGPU/call-preserved-registers.ll
@@ -2,7 +2,7 @@
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=hawaii -enable-ipra=0 -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -enable-ipra=0 -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
 
-declare void @external_void_func_void() #0
+declare hidden void @external_void_func_void() #0
 
 ; GCN-LABEL: {{^}}test_kernel_call_external_void_func_void_clobber_s30_s31_call_external_void_func_void:
 ; GCN: s_mov_b32 s33, s7
@@ -84,7 +84,7 @@ define void @void_func_void_clobber_s30_s31() #2 {
 ; GCN-NEXT: ;;#ASMSTART
 ; GCN-NEXT: ;;#ASMEND
 ; GCN-NEXT: s_setpc_b64 s[30:31]
-define void @void_func_void_clobber_vcc() #2 {
+define hidden void @void_func_void_clobber_vcc() #2 {
   call void asm sideeffect "", "~{VCC}"() #0
   ret void
 }
@@ -176,7 +176,7 @@ define amdgpu_kernel void @test_call_void_func_void_preserves_v32(i32 addrspace(
 ; GCN-NEXT: #ASMEND
 ; GCN-NEXT:	v_readlane_b32 s33, v0, 0
 ; GCN-NEXT: s_setpc_b64
-define void @void_func_void_clobber_s33() #2 {
+define hidden void @void_func_void_clobber_s33() #2 {
   call void asm sideeffect "; clobber", "~{s33}"() #0
   ret void
 }
diff --git a/test/CodeGen/AMDGPU/chain-hi-to-lo.ll b/test/CodeGen/AMDGPU/chain-hi-to-lo.ll
new file mode 100644
index 0000000000000000000000000000000000000000..320d008c220c9fe745cf8c466c05626f9ae4fcef
--- /dev/null
+++ b/test/CodeGen/AMDGPU/chain-hi-to-lo.ll
@@ -0,0 +1,141 @@
+; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
+
+; GCN-LABEL: {{^}}chain_hi_to_lo_private:
+; GCN: buffer_load_ushort [[DST:v[0-9]+]], off, [[RSRC:s\[[0-9]+:[0-9]+\]]], [[SOFF:s[0-9]+]] offset:2
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: buffer_load_short_d16_hi [[DST]], off, [[RSRC]], [[SOFF]]
+define <2 x half> @chain_hi_to_lo_private() {
+bb:
+  %gep_lo = getelementptr inbounds half, half addrspace(5)* null, i64 1
+  %load_lo = load half, half addrspace(5)* %gep_lo
+  %gep_hi = getelementptr inbounds half, half addrspace(5)* null, i64 0
+  %load_hi = load half, half addrspace(5)* %gep_hi
+
+  %temp = insertelement <2 x half> undef, half %load_lo, i32 0
+  %result = insertelement <2 x half> %temp, half %load_hi, i32 1
+
+  ret <2 x half> %result
+}
+
+; GCN-LABEL: {{^}}chain_hi_to_lo_private_different_bases:
+; GCN: buffer_load_ushort [[DST:v[0-9]+]], v{{[0-9]+}}, [[RSRC:s\[[0-9]+:[0-9]+\]]], [[SOFF:s[0-9]+]] offen
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: buffer_load_short_d16_hi [[DST]], v{{[0-9]+}}, [[RSRC]], [[SOFF]] offen
+define <2 x half> @chain_hi_to_lo_private_different_bases(half addrspace(5)* %base_lo, half addrspace(5)* %base_hi) {
+bb:
+  %load_lo = load half, half addrspace(5)* %base_lo
+  %load_hi = load half, half addrspace(5)* %base_hi
+
+  %temp = insertelement <2 x half> undef, half %load_lo, i32 0
+  %result = insertelement <2 x half> %temp, half %load_hi, i32 1
+
+  ret <2 x half> %result
+}
+
+; GCN-LABEL: {{^}}chain_hi_to_lo_arithmatic:
+; GCN: v_add_f16_e32 [[DST:v[0-9]+]], 1.0, v{{[0-9]+}}
+; GCN-NEXT: buffer_load_short_d16_hi [[DST]], v{{[0-9]+}}, s{{\[[0-9]+:[0-9]+\]}}, s{{[0-9]+}} offen
+define <2 x half> @chain_hi_to_lo_arithmatic(half addrspace(5)* %base, half %in) {
+bb:
+  %arith_lo = fadd half %in, 1.0
+  %load_hi = load half, half addrspace(5)* %base
+
+  %temp = insertelement <2 x half> undef, half %arith_lo, i32 0
+  %result = insertelement <2 x half> %temp, half %load_hi, i32 1
+
+  ret <2 x half> %result
+}
+
+; GCN-LABEL: {{^}}chain_hi_to_lo_group:
+; GCN: ds_read_u16 [[DST:v[0-9]+]], [[ADDR:v[0-9]+]] offset:2
+; GCN-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NEXT: ds_read_u16_d16_hi [[DST]], [[ADDR]]
+define <2 x half> @chain_hi_to_lo_group() {
+bb:
+  %gep_lo = getelementptr inbounds half, half addrspace(3)* null, i64 1
+  %load_lo = load half, half addrspace(3)* %gep_lo
+  %gep_hi = getelementptr inbounds half, half addrspace(3)* null, i64 0
+  %load_hi = load half, half addrspace(3)* %gep_hi
+
+  %temp = insertelement <2 x half> undef, half %load_lo, i32 0
+  %result = insertelement <2 x half> %temp, half %load_hi, i32 1
+
+  ret <2 x half> %result
+}
+
+; GCN-LABEL: {{^}}chain_hi_to_lo_group_different_bases:
+; GCN: ds_read_u16 [[DST:v[0-9]+]], v{{[0-9]+}}
+; GCN-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NEXT: ds_read_u16_d16_hi [[DST]], v{{[0-9]+}}
+define <2 x half> @chain_hi_to_lo_group_different_bases(half addrspace(3)* %base_lo, half addrspace(3)* %base_hi) {
+bb:
+  %load_lo = load half, half addrspace(3)* %base_lo
+  %load_hi = load half, half addrspace(3)* %base_hi
+
+  %temp = insertelement <2 x half> undef, half %load_lo, i32 0
+  %result = insertelement <2 x half> %temp, half %load_hi, i32 1
+
+  ret <2 x half> %result
+}
+
+; GCN-LABEL: {{^}}chain_hi_to_lo_global:
+; GCN: global_load_ushort [[DST:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, off
+; GCN: global_load_short_d16_hi [[DST]], v{{\[[0-9]+:[0-9]+\]}}, off
+define <2 x half> @chain_hi_to_lo_global() {
+bb:
+  %gep_lo = getelementptr inbounds half, half addrspace(1)* null, i64 1
+  %load_lo = load half, half addrspace(1)* %gep_lo
+  %gep_hi = getelementptr inbounds half, half addrspace(1)* null, i64 0
+  %load_hi = load half, half addrspace(1)* %gep_hi
+
+  %temp = insertelement <2 x half> undef, half %load_lo, i32 0
+  %result = insertelement <2 x half> %temp, half %load_hi, i32 1
+
+  ret <2 x half> %result
+}
+
+; GCN-LABEL: {{^}}chain_hi_to_lo_global_different_bases:
+; GCN: global_load_ushort [[DST:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}, off
+; GCN-NEXT: s_waitcnt vmcnt(0)
+; GCN-NEXT: global_load_short_d16_hi [[DST]], v{{\[[0-9]+:[0-9]+\]}}, off
+define <2 x half> @chain_hi_to_lo_global_different_bases(half addrspace(1)* %base_lo, half addrspace(1)* %base_hi) {
+bb:
+  %load_lo = load half, half addrspace(1)* %base_lo
+  %load_hi = load half, half addrspace(1)* %base_hi
+
+  %temp = insertelement <2 x half> undef, half %load_lo, i32 0
+  %result = insertelement <2 x half> %temp, half %load_hi, i32 1
+
+  ret <2 x half> %result
+}
+
+; GCN-LABEL: {{^}}chain_hi_to_lo_flat:
+; GCN: flat_load_ushort [[DST:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}
+; GCN: flat_load_short_d16_hi [[DST]], v{{\[[0-9]+:[0-9]+\]}}
+define <2 x half> @chain_hi_to_lo_flat() {
+bb:
+  %gep_lo = getelementptr inbounds half, half* null, i64 1
+  %load_lo = load half, half* %gep_lo
+  %gep_hi = getelementptr inbounds half, half* null, i64 0
+  %load_hi = load half, half* %gep_hi
+
+  %temp = insertelement <2 x half> undef, half %load_lo, i32 0
+  %result = insertelement <2 x half> %temp, half %load_hi, i32 1
+
+  ret <2 x half> %result
+}
+
+; GCN-LABEL: {{^}}chain_hi_to_lo_flat_different_bases:
+; GCN: flat_load_ushort [[DST:v[0-9]+]], v{{\[[0-9]+:[0-9]+\]}}
+; GCN-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GCN-NEXT: flat_load_short_d16_hi [[DST]], v{{\[[0-9]+:[0-9]+\]}}
+define <2 x half> @chain_hi_to_lo_flat_different_bases(half* %base_lo, half* %base_hi) {
+bb:
+  %load_lo = load half, half* %base_lo
+  %load_hi = load half, half* %base_hi
+
+  %temp = insertelement <2 x half> undef, half %load_lo, i32 0
+  %result = insertelement <2 x half> %temp, half %load_hi, i32 1
+
+  ret <2 x half> %result
+}
diff --git a/test/CodeGen/AMDGPU/cross-block-use-is-not-abi-copy.ll b/test/CodeGen/AMDGPU/cross-block-use-is-not-abi-copy.ll
index 805b2f5ad2bf8e4cd6b06cccf1d01205b363d616..10573461b3795bba779cd3b9cd95b6e9baa342aa 100644
--- a/test/CodeGen/AMDGPU/cross-block-use-is-not-abi-copy.ll
+++ b/test/CodeGen/AMDGPU/cross-block-use-is-not-abi-copy.ll
@@ -166,11 +166,11 @@ bb1:
 }
 
 
-declare <2 x float> @func_v2f32() #0
-declare <3 x float> @func_v3f32() #0
-declare <4 x float> @func_v4f32() #0
-declare <4 x half> @func_v4f16() #0
+declare hidden <2 x float> @func_v2f32() #0
+declare hidden <3 x float> @func_v3f32() #0
+declare hidden <4 x float> @func_v4f32() #0
+declare hidden <4 x half> @func_v4f16() #0
 
-declare { <4 x i32>, <4 x half> } @func_struct() #0
+declare hidden { <4 x i32>, <4 x half> } @func_struct() #0
 
 attributes #0 = { nounwind}
diff --git a/test/CodeGen/AMDGPU/cvt_f32_ubyte.ll b/test/CodeGen/AMDGPU/cvt_f32_ubyte.ll
index d8126fa635af8367d45aa9d06ec7b25906e40094..4c41565dca9598a319236c6f85c2e43e8eea1c5c 100644
--- a/test/CodeGen/AMDGPU/cvt_f32_ubyte.ll
+++ b/test/CodeGen/AMDGPU/cvt_f32_ubyte.ll
@@ -37,9 +37,10 @@ define amdgpu_kernel void @load_v2i8_to_v2f32(<2 x float> addrspace(1)* noalias
 ; GCN: {{buffer|flat}}_load_dword [[VAL:v[0-9]+]]
 ; GCN-NOT: v_cvt_f32_ubyte3_e32
 ; GCN-DAG: v_cvt_f32_ubyte2_e32 v[[HIRESULT:[0-9]+]], [[VAL]]
-; GCN-DAG: v_cvt_f32_ubyte1_e32 v{{[0-9]+}}, [[VAL]]
+; GCN-DAG: v_cvt_f32_ubyte1_e32 v[[MDRESULT:[0-9]+]], [[VAL]]
 ; GCN-DAG: v_cvt_f32_ubyte0_e32 v[[LORESULT:[0-9]+]], [[VAL]]
-; GCN: buffer_store_dwordx3 v{{\[}}[[LORESULT]]:[[HIRESULT]]{{\]}},
+; SI: buffer_store_dwordx2 v{{\[}}[[LORESULT]]:[[MDRESULT]]{{\]}},
+; VI: buffer_store_dwordx3 v{{\[}}[[LORESULT]]:[[HIRESULT]]{{\]}},
 define amdgpu_kernel void @load_v3i8_to_v3f32(<3 x float> addrspace(1)* noalias %out, <3 x i8> addrspace(1)* noalias %in) nounwind {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %gep = getelementptr <3 x i8>, <3 x i8> addrspace(1)* %in, i32 %tid
@@ -281,3 +282,23 @@ define amdgpu_kernel void @extract_byte3_to_f32(float addrspace(1)* noalias %out
   store float %cvt, float addrspace(1)* %out
   ret void
 }
+
+; GCN-LABEL: {{^}}cvt_ubyte0_or_multiuse:
+; GCN:     {{buffer|flat}}_load_dword [[LOADREG:v[0-9]+]],
+; GCN-DAG: v_or_b32_e32 [[OR:v[0-9]+]], 0x80000001, [[LOADREG]]
+; GCN-DAG: v_cvt_f32_ubyte0_e32 [[CONV:v[0-9]+]], [[OR]]
+; GCN:     v_add_f32_e32 [[RES:v[0-9]+]], [[OR]], [[CONV]]
+; GCN:     buffer_store_dword [[RES]],
+define amdgpu_kernel void @cvt_ubyte0_or_multiuse(i32 addrspace(1)* %in, float addrspace(1)* %out) {
+bb:
+  %lid = tail call i32 @llvm.amdgcn.workitem.id.x()
+  %gep = getelementptr inbounds i32, i32 addrspace(1)* %in, i32 %lid
+  %load = load i32, i32 addrspace(1)* %gep
+  %or = or i32 %load, -2147483647
+  %and = and i32 %or, 255
+  %uitofp = uitofp i32 %and to float
+  %cast = bitcast i32 %or to float
+  %add = fadd float %cast, %uitofp
+  store float %add, float addrspace(1)* %out
+  ret void
+}
diff --git a/test/CodeGen/AMDGPU/debug-value2.ll b/test/CodeGen/AMDGPU/debug-value2.ll
index 5200315ed74c50ff3394ba8c614a182befc2f0c8..3a13a33484892643a0adaf7dd9b87642a4d52a06 100644
--- a/test/CodeGen/AMDGPU/debug-value2.ll
+++ b/test/CodeGen/AMDGPU/debug-value2.ll
@@ -278,7 +278,7 @@ declare void @llvm.dbg.value(metadata, metadata, metadata)
 !opencl.ocl.version = !{!107}
 !llvm.ident = !{!108, !109}
 
-!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 7.0.0 (https://github.com/llvm-mirror/clang.git 3edc9a6d1f98fec61a944167cb5c36c40104918a) (https://github.com/llvm-mirror/llvm.git 90eddc791688f226397e600c287c043d9b0e35fa)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, retainedTypes: !74)
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 7.0.0", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, retainedTypes: !74)
 !1 = !DIFile(filename: "tmp.cl", directory: "/home/yaxunl/h/git/llvm/assert")
 !2 = !{!3, !27, !37, !42, !46, !51, !55, !68}
 !3 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "BrdfType", file: !4, line: 1334, size: 32, elements: !5)
@@ -386,7 +386,7 @@ declare void @llvm.dbg.value(metadata, metadata, metadata)
 !105 = !{i32 2, !"Debug Info Version", i32 3}
 !106 = !{i32 1, !"wchar_size", i32 4}
 !107 = !{i32 2, i32 0}
-!108 = !{!"clang version 7.0.0 (https://github.com/llvm-mirror/clang.git 3edc9a6d1f98fec61a944167cb5c36c40104918a) (https://github.com/llvm-mirror/llvm.git 90eddc791688f226397e600c287c043d9b0e35fa)"}
+!108 = !{!"clang version 7.0.0"}
 !109 = !{!"clang version 4.0 "}
 !110 = distinct !DISubprogram(name: "Scene_transformT", scope: !4, file: !4, line: 2182, type: !111, isLocal: false, isDefinition: true, scopeLine: 2183, flags: DIFlagPrototyped, isOptimized: true, unit: !0, retainedNodes: !116)
 !111 = !DISubroutineType(types: !112)
diff --git a/test/CodeGen/AMDGPU/divergent-branch-uniform-condition.ll b/test/CodeGen/AMDGPU/divergent-branch-uniform-condition.ll
new file mode 100644
index 0000000000000000000000000000000000000000..8d21050ebee01a6b98cb056b02df4ea43aeea9e4
--- /dev/null
+++ b/test/CodeGen/AMDGPU/divergent-branch-uniform-condition.ll
@@ -0,0 +1,97 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck %s
+
+; This module creates a divergent branch. The branch is marked as divergent by
+; the divergence analysis but the condition is not. This test ensures that the
+; divergence of the branch is tested, not its condition, so that branch is
+; correctly emitted as divergent.
+
+target triple = "amdgcn-mesa-mesa3d"
+
+define amdgpu_ps void @main(i32, float) {
+; CHECK-LABEL: main:
+; CHECK:       ; %bb.0: ; %start
+; CHECK-NEXT:    v_readfirstlane_b32 s0, v0
+; CHECK-NEXT:    s_mov_b32 m0, s0
+; CHECK-NEXT:    s_mov_b64 s[4:5], 0
+; CHECK-NEXT:    v_interp_p1_f32_e32 v0, v1, attr0.x
+; CHECK-NEXT:    v_cmp_nlt_f32_e64 s[0:1], 0, v0
+; CHECK-NEXT:    v_mov_b32_e32 v1, 0
+; CHECK-NEXT:    ; implicit-def: $sgpr2_sgpr3
+; CHECK-NEXT:    ; implicit-def: $sgpr6_sgpr7
+; CHECK-NEXT:  BB0_1: ; %loop
+; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    v_cmp_gt_u32_e32 vcc, 32, v1
+; CHECK-NEXT:    s_and_b64 vcc, exec, vcc
+; CHECK-NEXT:    s_or_b64 s[6:7], s[6:7], exec
+; CHECK-NEXT:    s_or_b64 s[2:3], s[2:3], exec
+; CHECK-NEXT:    s_cbranch_vccz BB0_5
+; CHECK-NEXT:  ; %bb.2: ; %endif1
+; CHECK-NEXT:    ; in Loop: Header=BB0_1 Depth=1
+; CHECK-NEXT:    s_mov_b64 s[6:7], -1
+; CHECK-NEXT:    s_and_saveexec_b64 s[8:9], s[0:1]
+; CHECK-NEXT:    s_xor_b64 s[8:9], exec, s[8:9]
+; CHECK-NEXT:    ; mask branch BB0_4
+; CHECK-NEXT:  BB0_3: ; %endif2
+; CHECK-NEXT:    ; in Loop: Header=BB0_1 Depth=1
+; CHECK-NEXT:    v_add_u32_e32 v1, 1, v1
+; CHECK-NEXT:    s_xor_b64 s[6:7], exec, -1
+; CHECK-NEXT:  BB0_4: ; %Flow1
+; CHECK-NEXT:    ; in Loop: Header=BB0_1 Depth=1
+; CHECK-NEXT:    s_or_b64 exec, exec, s[8:9]
+; CHECK-NEXT:    s_andn2_b64 s[2:3], s[2:3], exec
+; CHECK-NEXT:    s_branch BB0_6
+; CHECK-NEXT:  BB0_5: ; in Loop: Header=BB0_1 Depth=1
+; CHECK-NEXT:    ; implicit-def: $vgpr1
+; CHECK-NEXT:  BB0_6: ; %Flow
+; CHECK-NEXT:    ; in Loop: Header=BB0_1 Depth=1
+; CHECK-NEXT:    s_and_b64 s[8:9], exec, s[6:7]
+; CHECK-NEXT:    s_or_b64 s[8:9], s[8:9], s[4:5]
+; CHECK-NEXT:    s_mov_b64 s[4:5], s[8:9]
+; CHECK-NEXT:    s_andn2_b64 exec, exec, s[8:9]
+; CHECK-NEXT:    s_cbranch_execnz BB0_1
+; CHECK-NEXT:  ; %bb.7: ; %Flow2
+; CHECK-NEXT:    s_or_b64 exec, exec, s[8:9]
+; CHECK-NEXT:    v_mov_b32_e32 v1, 0
+; this is the divergent branch with the condition not marked as divergent
+; CHECK-NEXT:    s_and_saveexec_b64 s[0:1], s[2:3]
+; CHECK-NEXT:    ; mask branch BB0_9
+; CHECK-NEXT:  BB0_8: ; %if1
+; CHECK-NEXT:    v_sqrt_f32_e32 v1, v0
+; CHECK-NEXT:  BB0_9: ; %endloop
+; CHECK-NEXT:    s_or_b64 exec, exec, s[0:1]
+; CHECK-NEXT:    exp mrt0 v1, v1, v1, v1 done vm
+; CHECK-NEXT:    s_endpgm
+start:
+  %v0 = call float @llvm.amdgcn.interp.p1(float %1, i32 0, i32 0, i32 %0)
+  br label %loop
+
+loop:
+  %v1 = phi i32 [ 0, %start ], [ %v5, %endif2 ]
+  %v2 = icmp ugt i32 %v1, 31
+  br i1 %v2, label %if1, label %endif1
+
+if1:
+  %v3 = call float @llvm.sqrt.f32(float %v0)
+  br label %endloop
+
+endif1:
+  %v4 = fcmp ogt float %v0, 0.000000e+00
+  br i1 %v4, label %endloop, label %endif2
+
+endif2:
+  %v5 = add i32 %v1, 1
+  br label %loop
+
+endloop:
+  %v6 = phi float [ %v3, %if1 ], [ 0.0, %endif1 ]
+  call void @llvm.amdgcn.exp.v4f32(i32 0, i32 15, float %v6, float %v6, float %v6, float %v6, i1 true, i1 true)
+  ret void
+}
+
+declare float @llvm.sqrt.f32(float) #1
+declare float @llvm.amdgcn.interp.p1(float, i32, i32, i32) #1
+declare void @llvm.amdgcn.exp.v4f32(i32, i32, float, float, float, float, i1, i1) #0
+
+attributes #0 = { nounwind }
+attributes #1 = { nounwind readnone }
diff --git a/test/CodeGen/AMDGPU/dpp_combine.ll b/test/CodeGen/AMDGPU/dpp_combine.ll
deleted file mode 100644
index 36356a72a772705fed2eb772e75141b282010dcc..0000000000000000000000000000000000000000
--- a/test/CodeGen/AMDGPU/dpp_combine.ll
+++ /dev/null
@@ -1,185 +0,0 @@
-; RUN: llc -march=amdgcn -mcpu=tonga -amdgpu-dpp-combine -verify-machineinstrs < %s | FileCheck %s
-
-; VOP2 with literal cannot be combined
-; CHECK-LABEL: {{^}}dpp_combine_i32_literal:
-; CHECK: v_mov_b32_dpp [[OLD:v[0-9]+]], {{v[0-9]+}} quad_perm:[1,0,0,0] row_mask:0x2 bank_mask:0x1 bound_ctrl:0
-; CHECK: v_add_u32_e32 {{v[0-9]+}}, vcc, 42, [[OLD]]
-define amdgpu_kernel void @dpp_combine_i32_literal(i32 addrspace(1)* %out, i32 %in) {
-  %dpp = call i32 @llvm.amdgcn.update.dpp.i32(i32 undef, i32 %in, i32 1, i32 2, i32 1, i1 1) #0
-  %res = add nsw i32 %dpp, 42
-  store i32 %res, i32 addrspace(1)* %out
-  ret void
-}
-
-; CHECK-LABEL: {{^}}dpp_combine_i32_bz:
-; CHECK: v_add_u32_dpp {{v[0-9]+}}, vcc, {{v[0-9]+}}, v0  quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1 bound_ctrl:0
-define amdgpu_kernel void @dpp_combine_i32_bz(i32 addrspace(1)* %out, i32 %in) {
-  %x = tail call i32 @llvm.amdgcn.workitem.id.x()
-  %dpp = call i32 @llvm.amdgcn.update.dpp.i32(i32 undef, i32 %in, i32 1, i32 1, i32 1, i1 1) #0
-  %res = add nsw i32 %dpp, %x
-  store i32 %res, i32 addrspace(1)* %out
-  ret void
-}
-
-; CHECK-LABEL: {{^}}dpp_combine_i32_boff_undef:
-; CHECK: v_add_u32_dpp {{v[0-9]+}}, vcc, {{v[0-9]+}}, v0  quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1
-define amdgpu_kernel void @dpp_combine_i32_boff_undef(i32 addrspace(1)* %out, i32 %in) {
-  %x = tail call i32 @llvm.amdgcn.workitem.id.x()
-  %dpp = call i32 @llvm.amdgcn.update.dpp.i32(i32 undef, i32 %in, i32 1, i32 1, i32 1, i1 0) #0
-  %res = add nsw i32 %dpp, %x
-  store i32 %res, i32 addrspace(1)* %out
-  ret void
-}
-
-; CHECK-LABEL: {{^}}dpp_combine_i32_boff_0:
-; CHECK: v_add_u32_dpp {{v[0-9]+}}, vcc, {{v[0-9]+}}, v0  quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1 bound_ctrl:0
-define amdgpu_kernel void @dpp_combine_i32_boff_0(i32 addrspace(1)* %out, i32 %in) {
-  %x = tail call i32 @llvm.amdgcn.workitem.id.x()
-  %dpp = call i32 @llvm.amdgcn.update.dpp.i32(i32 0, i32 %in, i32 1, i32 1, i32 1, i1 0) #0
-  %res = add nsw i32 %dpp, %x
-  store i32 %res, i32 addrspace(1)* %out
-  ret void
-}
-
-; CHECK-LABEL: {{^}}dpp_combine_i32_boff_max:
-; CHECK: v_bfrev_b32_e32 [[OLD:v[0-9]+]], -2
-; CHECK: v_max_i32_dpp [[OLD]], {{v[0-9]+}}, v0  quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1
-define amdgpu_kernel void @dpp_combine_i32_boff_max(i32 addrspace(1)* %out, i32 %in) {
-  %x = tail call i32 @llvm.amdgcn.workitem.id.x()
-  %dpp = call i32 @llvm.amdgcn.update.dpp.i32(i32 2147483647, i32 %in, i32 1, i32 1, i32 1, i1 0) #0
-  %cmp = icmp sge i32 %dpp, %x
-  %res = select i1 %cmp, i32 %dpp, i32 %x
-  store i32 %res, i32 addrspace(1)* %out
-  ret void
-}
-
-; CHECK-LABEL: {{^}}dpp_combine_i32_boff_min:
-; CHECK: v_bfrev_b32_e32 [[OLD:v[0-9]+]], 1
-; CHECK: v_min_i32_dpp [[OLD]], {{v[0-9]+}}, v0  quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1
-define amdgpu_kernel void @dpp_combine_i32_boff_min(i32 addrspace(1)* %out, i32 %in) {
-  %x = tail call i32 @llvm.amdgcn.workitem.id.x()
-  %dpp = call i32 @llvm.amdgcn.update.dpp.i32(i32 -2147483648, i32 %in, i32 1, i32 1, i32 1, i1 0) #0
-  %cmp = icmp sle i32 %dpp, %x
-  %res = select i1 %cmp, i32 %dpp, i32 %x
-  store i32 %res, i32 addrspace(1)* %out
-  ret void
-}
-
-; CHECK-LABEL: {{^}}dpp_combine_i32_boff_mul:
-; CHECK: v_mul_i32_i24_dpp v0, v3, v0  quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1
-define amdgpu_kernel void @dpp_combine_i32_boff_mul(i32 addrspace(1)* %out, i32 %in) {
-  %x = tail call i32 @llvm.amdgcn.workitem.id.x()
-  %dpp = call i32 @llvm.amdgcn.update.dpp.i32(i32 1, i32 %in, i32 1, i32 1, i32 1, i1 0) #0
-
-  %dpp.shl = shl i32 %dpp, 8
-  %dpp.24 = ashr i32 %dpp.shl, 8
-  %x.shl = shl i32 %x, 8
-  %x.24 = ashr i32 %x.shl, 8
-  %res = mul i32 %dpp.24, %x.24
-  store i32 %res, i32 addrspace(1)* %out
-  ret void
-}
-
-; CHECK-LABEL: {{^}}dpp_combine_i32_commute:
-; CHECK: v_subrev_u32_dpp {{v[0-9]+}}, vcc, {{v[0-9]+}}, v0  quad_perm:[2,0,0,0] row_mask:0x1 bank_mask:0x1 bound_ctrl:0
-define amdgpu_kernel void @dpp_combine_i32_commute(i32 addrspace(1)* %out, i32 %in) {
-  %x = tail call i32 @llvm.amdgcn.workitem.id.x()
-  %dpp = call i32 @llvm.amdgcn.update.dpp.i32(i32 undef, i32 %in, i32 2, i32 1, i32 1, i1 1) #0
-  %res = sub nsw i32 %x, %dpp
-  store i32 %res, i32 addrspace(1)* %out
-  ret void
-}
-
-; CHECK-LABEL: {{^}}dpp_combine_f32:
-; CHECK: v_add_f32_dpp {{v[0-9]+}}, {{v[0-9]+}}, v0  quad_perm:[3,0,0,0] row_mask:0x1 bank_mask:0x1 bound_ctrl:0
-define amdgpu_kernel void @dpp_combine_f32(i32 addrspace(1)* %out, i32 %in) {
-  %x = tail call i32 @llvm.amdgcn.workitem.id.x()
-
-  %dpp = call i32 @llvm.amdgcn.update.dpp.i32(i32 undef, i32 %in, i32 3, i32 1, i32 1, i1 1) #0
-  %dpp.f32 = bitcast i32 %dpp to float
-  %x.f32 = bitcast i32 %x to float
-  %res.f32 = fadd float %x.f32, %dpp.f32
-  %res = bitcast float %res.f32 to i32
-  store i32 %res, i32 addrspace(1)* %out
-  ret void
-}
-
-; CHECK-LABEL: {{^}}dpp_combine_test_f32_mods:
-; CHECK: v_mul_f32_dpp {{v[0-9]+}}, |{{v[0-9]+}}|, -v0  quad_perm:[0,1,0,0] row_mask:0x1 bank_mask:0x1 bound_ctrl:0
-define amdgpu_kernel void @dpp_combine_test_f32_mods(i32 addrspace(1)* %out, i32 %in) {
-  %x = tail call i32 @llvm.amdgcn.workitem.id.x()
-
-  %dpp = call i32 @llvm.amdgcn.update.dpp.i32(i32 undef, i32 %in, i32 4, i32 1, i32 1, i1 1) #0
-
-  %x.f32 = bitcast i32 %x to float
-  %x.f32.neg = fsub float -0.000000e+00, %x.f32
-
-  %dpp.f32 = bitcast i32 %dpp to float
-  %dpp.f32.cmp = fcmp fast olt float %dpp.f32, 0.000000e+00
-  %dpp.f32.sign = select i1 %dpp.f32.cmp, float -1.000000e+00, float 1.000000e+00
-  %dpp.f32.abs = fmul fast float %dpp.f32, %dpp.f32.sign
-
-  %res.f32 = fmul float %x.f32.neg, %dpp.f32.abs
-  %res = bitcast float %res.f32 to i32
-  store i32 %res, i32 addrspace(1)* %out
-  ret void
-}
-
-; CHECK-LABEL: {{^}}dpp_combine_mac:
-; CHECK: v_mac_f32_dpp v0, {{v[0-9]+}}, v1  quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1 bound_ctrl:0
-define amdgpu_kernel void @dpp_combine_mac(float addrspace(1)* %out, i32 %in) {
-  %x = tail call i32 @llvm.amdgcn.workitem.id.x()
-  %y = tail call i32 @llvm.amdgcn.workitem.id.y()
-  %dpp = call i32 @llvm.amdgcn.update.dpp.i32(i32 undef, i32 %in, i32 1, i32 1, i32 1, i1 1) #0
-  %dpp.f32 = bitcast i32 %dpp to float
-  %x.f32 = bitcast i32 %x to float
-  %y.f32 = bitcast i32 %y to float
-
-  %mult = fmul float %dpp.f32, %y.f32
-  %res = fadd float %mult, %x.f32
-  store float %res, float addrspace(1)* %out
-  ret void
-}
-
-; CHECK-LABEL: {{^}}dpp_combine_sequence:
-define amdgpu_kernel void @dpp_combine_sequence(i32 addrspace(1)* %out, i32 %in, i1 %cmp) {
-  %x = tail call i32 @llvm.amdgcn.workitem.id.x()
-  %dpp = call i32 @llvm.amdgcn.update.dpp.i32(i32 undef, i32 %in, i32 1, i32 1, i32 1, i1 1) #0
-  br i1 %cmp, label %bb1, label %bb2
-bb1:
-; CHECK: v_add_u32_dpp {{v[0-9]+}}, vcc, {{v[0-9]+}}, v0  quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1 bound_ctrl:0
-  %resadd = add nsw i32 %dpp, %x
-  br label %bb3
-bb2:
-; CHECK: v_subrev_u32_dpp {{v[0-9]+}}, vcc, {{v[0-9]+}}, v0  quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1 bound_ctrl:0
-  %ressub = sub nsw i32 %x, %dpp
-  br label %bb3
-bb3:
-  %res = phi i32 [%resadd, %bb1], [%ressub, %bb2]
-  store i32 %res, i32 addrspace(1)* %out
-  ret void
-}
-
-; CHECK-LABEL: {{^}}dpp_combine_sequence_negative:
-; CHECK: v_mov_b32_dpp v1, v1  quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1 bound_ctrl:0
-define amdgpu_kernel void @dpp_combine_sequence_negative(i32 addrspace(1)* %out, i32 %in, i1 %cmp) {
-  %x = tail call i32 @llvm.amdgcn.workitem.id.x()
-  %dpp = call i32 @llvm.amdgcn.update.dpp.i32(i32 undef, i32 %in, i32 1, i32 1, i32 1, i1 1) #0
-  br i1 %cmp, label %bb1, label %bb2
-bb1:
-  %resadd = add nsw i32 %dpp, %x
-  br label %bb3
-bb2:
-  %ressub = sub nsw i32 2, %dpp ; break seq
-  br label %bb3
-bb3:
-  %res = phi i32 [%resadd, %bb1], [%ressub, %bb2]
-  store i32 %res, i32 addrspace(1)* %out
-  ret void
-}
-
-declare i32 @llvm.amdgcn.workitem.id.x()
-declare i32 @llvm.amdgcn.workitem.id.y()
-declare i32 @llvm.amdgcn.update.dpp.i32(i32, i32, i32, i32, i32, i1) #0
-
-attributes #0 = { nounwind readnone convergent }
diff --git a/test/CodeGen/AMDGPU/dpp_combine.mir b/test/CodeGen/AMDGPU/dpp_combine.mir
new file mode 100644
index 0000000000000000000000000000000000000000..fbeadcf2e0081701216781c3ef1e981a164c2ae3
--- /dev/null
+++ b/test/CodeGen/AMDGPU/dpp_combine.mir
@@ -0,0 +1,472 @@
+# RUN: llc -march=amdgcn -mcpu=tonga  -run-pass=gcn-dpp-combine  -o - %s | FileCheck %s
+
+---
+# old is undefined: only combine when masks are fully enabled and
+# bound_ctrl:0 is set, otherwise the result of DPP VALU op can be undefined.
+# CHECK-LABEL: name: old_is_undef
+# CHECK: %2:vgpr_32 = IMPLICIT_DEF
+# VOP2:
+# CHECK: %4:vgpr_32 = V_ADD_U32_dpp %2, %0, %1, 1, 15, 15, 1, implicit $exec
+# CHECK: %6:vgpr_32 = V_ADD_U32_e32 %5, %1, implicit $exec
+# CHECK: %8:vgpr_32 = V_ADD_U32_e32 %7, %1, implicit $exec
+# CHECK: %10:vgpr_32 = V_ADD_U32_e32 %9, %1, implicit $exec
+# VOP1:
+# CHECK: %12:vgpr_32 = V_NOT_B32_dpp %2, %0, 1, 15, 15, 1, implicit $exec
+# CHECK: %14:vgpr_32 = V_NOT_B32_e32 %13, implicit $exec
+# CHECK: %16:vgpr_32 = V_NOT_B32_e32 %15, implicit $exec
+# CHECK: %18:vgpr_32 = V_NOT_B32_e32 %17, implicit $exec
+name: old_is_undef
+tracksRegLiveness: true
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+    %0:vgpr_32 = COPY $vgpr0
+    %1:vgpr_32 = COPY $vgpr1
+    %2:vgpr_32 = IMPLICIT_DEF
+
+    ; VOP2
+    %3:vgpr_32 = V_MOV_B32_dpp %2, %0, 1, 15, 15, 1, implicit $exec
+    %4:vgpr_32 = V_ADD_U32_e32 %3, %1, implicit $exec
+
+    %5:vgpr_32 = V_MOV_B32_dpp %2, %0, 1, 15, 15, 0, implicit $exec
+    %6:vgpr_32 = V_ADD_U32_e32 %5, %1, implicit $exec
+
+    %7:vgpr_32 = V_MOV_B32_dpp %2, %0, 1, 14, 15, 1, implicit $exec
+    %8:vgpr_32 = V_ADD_U32_e32 %7, %1, implicit $exec
+
+    %9:vgpr_32 = V_MOV_B32_dpp %2, %0, 1, 14, 15, 0, implicit $exec
+    %10:vgpr_32 = V_ADD_U32_e32 %9, %1, implicit $exec
+
+    ; VOP1
+    %11:vgpr_32 = V_MOV_B32_dpp %2, %0, 1, 15, 15, 1, implicit $exec
+    %12:vgpr_32 = V_NOT_B32_e32 %11, implicit $exec
+
+    %13:vgpr_32 = V_MOV_B32_dpp %2, %0, 1, 15, 15, 0, implicit $exec
+    %14:vgpr_32 = V_NOT_B32_e32 %13, implicit $exec
+
+    %15:vgpr_32 = V_MOV_B32_dpp %2, %0, 1, 14, 15, 1, implicit $exec
+    %16:vgpr_32 = V_NOT_B32_e32 %15, implicit $exec
+
+    %17:vgpr_32 = V_MOV_B32_dpp %2, %0, 1, 14, 15, 0, implicit $exec
+    %18:vgpr_32 = V_NOT_B32_e32 %17, implicit $exec
+...
+
+# old is zero cases:
+
+# CHECK-LABEL: name: old_is_0
+
+# VOP2:
+# case 1: old is zero, masks are fully enabled, bound_ctrl:0 is on:
+# the DPP mov result would be either zero ({src lane disabled}|{src lane is
+# out of range}) or active src lane result - can combine with old = undef.
+# undef is preffered as it makes life easier for the regalloc.
+# CHECK: [[U1:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+# CHECK: %4:vgpr_32 = V_ADD_U32_dpp [[U1]], %0, %1, 1, 15, 15, 1, implicit $exec
+
+# case 2: old is zero, masks are fully enabled, bound_ctrl:0 is off:
+# as the DPP mov old is zero this case is no different from case 1 - combine it
+# setting bound_ctrl0 on for the combined DPP VALU op to make old undefined
+# CHECK: [[U2:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+# CHECK: %6:vgpr_32 = V_ADD_U32_dpp [[U2]], %0, %1, 1, 15, 15, 1, implicit $exec
+
+# case 3: masks are partialy disabled, bound_ctrl:0 is on:
+# the DPP mov result would be either zero ({src lane disabled}|{src lane is
+# out of range} or {the DPP mov's dest VGPR write is disabled by masks}) or
+# active src lane result - can combine with old = src1 of the VALU op.
+# The VALU op should have the same masks as DPP mov as they select lanes
+# with identity value.
+# Special case: the bound_ctrl for the combined DPP VALU op isn't important
+# here but let's make it off to keep the combiner's logic simpler.
+# CHECK: %8:vgpr_32 = V_ADD_U32_dpp %1, %0, %1, 1, 14, 15, 0, implicit $exec
+
+# case 4: masks are partialy disabled, bound_ctrl:0 is off:
+# the DPP mov result would be either zero ({src lane disabled}|{src lane is
+# out of range} or {the DPP mov's dest VGPR write is disabled by masks}) or
+# active src lane result - can combine with old = src1 of the VALU op.
+# The VALU op should have the same masks as DPP mov as they select
+# lanes with identity value
+# CHECK: %10:vgpr_32 = V_ADD_U32_dpp %1, %0, %1, 1, 14, 15, 0, implicit $exec
+
+# VOP1:
+# see case 1
+# CHECK: [[U3:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+# CHECK: %12:vgpr_32 = V_NOT_B32_dpp [[U3]], %0, 1, 15, 15, 1, implicit $exec
+# see case 2
+# CHECK: [[U4:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+# CHECK: %14:vgpr_32 = V_NOT_B32_dpp [[U4]], %0, 1, 15, 15, 1, implicit $exec
+# case 3 and 4 not appliable as there is no way to specify unchanged result
+# for the unary VALU op
+# CHECK: %16:vgpr_32 = V_NOT_B32_e32 %15, implicit $exec
+# CHECK: %18:vgpr_32 = V_NOT_B32_e32 %17, implicit $exec
+
+name: old_is_0
+tracksRegLiveness: true
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+    %0:vgpr_32 = COPY $vgpr0
+    %1:vgpr_32 = COPY $vgpr1
+    %2:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+
+    ; VOP2
+    %3:vgpr_32 = V_MOV_B32_dpp %2, %0, 1, 15, 15, 1, implicit $exec
+    %4:vgpr_32 = V_ADD_U32_e32 %3, %1, implicit $exec
+
+    %5:vgpr_32 = V_MOV_B32_dpp %2, %0, 1, 15, 15, 0, implicit $exec
+    %6:vgpr_32 = V_ADD_U32_e32 %5, %1, implicit $exec
+
+    %7:vgpr_32 = V_MOV_B32_dpp %2, %0, 1, 14, 15, 1, implicit $exec
+    %8:vgpr_32 = V_ADD_U32_e32 %7, %1, implicit $exec
+
+    %9:vgpr_32 = V_MOV_B32_dpp %2, %0, 1, 14, 15, 0, implicit $exec
+    %10:vgpr_32 = V_ADD_U32_e32 %9, %1, implicit $exec
+
+    ; VOP1
+    %11:vgpr_32 = V_MOV_B32_dpp %2, %0, 1, 15, 15, 1, implicit $exec
+    %12:vgpr_32 = V_NOT_B32_e32 %11, implicit $exec
+
+    %13:vgpr_32 = V_MOV_B32_dpp %2, %0, 1, 15, 15, 0, implicit $exec
+    %14:vgpr_32 = V_NOT_B32_e32 %13, implicit $exec
+
+    %15:vgpr_32 = V_MOV_B32_dpp %2, %0, 1, 14, 15, 1, implicit $exec
+    %16:vgpr_32 = V_NOT_B32_e32 %15, implicit $exec
+
+    %17:vgpr_32 = V_MOV_B32_dpp %2, %0, 1, 14, 15, 0, implicit $exec
+    %18:vgpr_32 = V_NOT_B32_e32 %17, implicit $exec
+...
+
+# old is nonzero identity cases:
+
+# old is nonzero identity, masks are fully enabled, bound_ctrl:0 is off:
+# the DPP mov result would be either identity ({src lane disabled}|{out of
+# range}) or src lane result - can combine with old = src1 of the VALU op
+# The DPP VALU op should have the same masks (and bctrl) as DPP mov as they
+# select lanes with identity value
+
+# CHECK-LABEL: name: nonzero_old_is_identity_masks_enabled_bctl_off
+# CHECK: %4:vgpr_32 = V_MUL_U32_U24_dpp %1, %0, %1, 1, 15, 15, 0, implicit $exec
+# CHECK: %7:vgpr_32 = V_AND_B32_dpp %1, %0, %1, 1, 15, 15, 0, implicit $exec
+# CHECK: %10:vgpr_32 = V_MAX_I32_dpp %1, %0, %1, 1, 15, 15, 0, implicit $exec
+# CHECK: %13:vgpr_32 = V_MIN_I32_dpp %1, %0, %1, 1, 15, 15, 0, implicit $exec
+
+name: nonzero_old_is_identity_masks_enabled_bctl_off
+tracksRegLiveness: true
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+    %0:vgpr_32 = COPY $vgpr0
+    %1:vgpr_32 = COPY $vgpr1
+
+    %2:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
+    %3:vgpr_32 = V_MOV_B32_dpp %2, %0, 1, 15, 15, 0, implicit $exec
+    %4:vgpr_32 = V_MUL_U32_U24_e32 %3, %1, implicit $exec
+
+    %5:vgpr_32 = V_MOV_B32_e32 4294967295, implicit $exec
+    %6:vgpr_32 = V_MOV_B32_dpp %5, %0, 1, 15, 15, 0, implicit $exec
+    %7:vgpr_32 = V_AND_B32_e32 %6, %1, implicit $exec
+
+    %8:vgpr_32 = V_MOV_B32_e32 -2147483648, implicit $exec
+    %9:vgpr_32 = V_MOV_B32_dpp %8, %0, 1, 15, 15, 0, implicit $exec
+    %10:vgpr_32 = V_MAX_I32_e32 %9, %1, implicit $exec
+
+    %11:vgpr_32 = V_MOV_B32_e32 2147483647, implicit $exec
+    %12:vgpr_32 = V_MOV_B32_dpp %11, %0, 1, 15, 15, 0, implicit $exec
+    %13:vgpr_32 = V_MIN_I32_e32 %12, %1, implicit $exec
+...
+
+# old is nonzero identity, masks are partially enabled, bound_ctrl:0 is off:
+# the DPP mov result would be either identity ({src lane disabled}|{src lane is
+# out of range} or {the DPP mov's dest VGPR write is disabled by masks}) or
+# active src lane result - can combine with old = src1 of the VALU op.
+# The DPP VALU op should have the same masks (and bctrl) as DPP mov as they
+# select lanes with identity value
+
+# CHECK-LABEL: name: nonzero_old_is_identity_masks_partially_disabled_bctl_off
+# CHECK: %4:vgpr_32 = V_MUL_U32_U24_dpp %1, %0, %1, 1, 14, 15, 0, implicit $exec
+# CHECK: %7:vgpr_32 = V_AND_B32_dpp %1, %0, %1, 1, 15, 14, 0, implicit $exec
+# CHECK: %10:vgpr_32 = V_MAX_I32_dpp %1, %0, %1, 1, 14, 15, 0, implicit $exec
+# CHECK: %13:vgpr_32 = V_MIN_I32_dpp %1, %0, %1, 1, 15, 14, 0, implicit $exec
+
+name: nonzero_old_is_identity_masks_partially_disabled_bctl_off
+tracksRegLiveness: true
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+    %0:vgpr_32 = COPY $vgpr0
+    %1:vgpr_32 = COPY $vgpr1
+
+    %2:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
+    %3:vgpr_32 = V_MOV_B32_dpp %2, %0, 1, 14, 15, 0, implicit $exec
+    %4:vgpr_32 = V_MUL_U32_U24_e32 %3, %1, implicit $exec
+
+    %5:vgpr_32 = V_MOV_B32_e32 4294967295, implicit $exec
+    %6:vgpr_32 = V_MOV_B32_dpp %5, %0, 1, 15, 14, 0, implicit $exec
+    %7:vgpr_32 = V_AND_B32_e32 %6, %1, implicit $exec
+
+    %8:vgpr_32 = V_MOV_B32_e32 -2147483648, implicit $exec
+    %9:vgpr_32 = V_MOV_B32_dpp %8, %0, 1, 14, 15, 0, implicit $exec
+    %10:vgpr_32 = V_MAX_I32_e32 %9, %1, implicit $exec
+
+    %11:vgpr_32 = V_MOV_B32_e32 2147483647, implicit $exec
+    %12:vgpr_32 = V_MOV_B32_dpp %11, %0, 1, 15, 14, 0, implicit $exec
+    %13:vgpr_32 = V_MIN_I32_e32 %12, %1, implicit $exec
+...
+
+# old is nonzero identity, masks are partially enabled, bound_ctrl:0 is on:
+# the DPP mov result may have 3 different values:
+#   1. the active src lane result
+#   2. 0 if the src lane is disabled|out of range
+#   3. DPP mov's old value if the mov's dest VGPR write is disabled by masks
+# can't combine
+
+# CHECK-LABEL: name: nonzero_old_is_identity_masks_partially_disabled_bctl0
+# CHECK: %4:vgpr_32 = V_MUL_U32_U24_e32 %3, %1, implicit $exec
+# CHECK: %7:vgpr_32 = V_AND_B32_e32 %6, %1, implicit $exec
+# CHECK: %10:vgpr_32 = V_MAX_I32_e32 %9, %1, implicit $exec
+# CHECK: %13:vgpr_32 = V_MIN_I32_e32 %12, %1, implicit $exec
+
+name: nonzero_old_is_identity_masks_partially_disabled_bctl0
+tracksRegLiveness: true
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+    %0:vgpr_32 = COPY $vgpr0
+    %1:vgpr_32 = COPY $vgpr1
+
+    %2:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
+    %3:vgpr_32 = V_MOV_B32_dpp %2, %0, 1, 14, 15, 1, implicit $exec
+    %4:vgpr_32 = V_MUL_U32_U24_e32 %3, %1, implicit $exec
+
+    %5:vgpr_32 = V_MOV_B32_e32 4294967295, implicit $exec
+    %6:vgpr_32 = V_MOV_B32_dpp %5, %0, 1, 15, 14, 1, implicit $exec
+    %7:vgpr_32 = V_AND_B32_e32 %6, %1, implicit $exec
+
+    %8:vgpr_32 = V_MOV_B32_e32 -2147483648, implicit $exec
+    %9:vgpr_32 = V_MOV_B32_dpp %8, %0, 1, 14, 15, 1, implicit $exec
+    %10:vgpr_32 = V_MAX_I32_e32 %9, %1, implicit $exec
+
+    %11:vgpr_32 = V_MOV_B32_e32 2147483647, implicit $exec
+    %12:vgpr_32 = V_MOV_B32_dpp %11, %0, 1, 15, 14, 1, implicit $exec
+    %13:vgpr_32 = V_MIN_I32_e32 %12, %1, implicit $exec
+...
+
+# when the DPP source isn't a src0 operand the operation should be commuted if possible
+# CHECK-LABEL: name: dpp_commute
+# CHECK: %4:vgpr_32 = V_MUL_U32_U24_dpp %1, %0, %1, 1, 14, 15, 0, implicit $exec
+# CHECK: %7:vgpr_32 = V_AND_B32_dpp %1, %0, %1, 1, 15, 14, 0, implicit $exec
+# CHECK: %10:vgpr_32 = V_MAX_I32_dpp %1, %0, %1, 1, 14, 15, 0, implicit $exec
+# CHECK: %13:vgpr_32 = V_MIN_I32_dpp %1, %0, %1, 1, 15, 14, 0, implicit $exec
+# CHECK: %16:vgpr_32 = V_SUBREV_I32_dpp %1, %0, %1, 1, 14, 15, 0, implicit-def $vcc, implicit $exec
+# CHECK: %19:vgpr_32 = V_ADD_I32_e32 5, %18, implicit-def $vcc, implicit $exec
+name: dpp_commute
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+
+    %0:vgpr_32 = COPY $vgpr0
+    %1:vgpr_32 = COPY $vgpr1
+
+    %2:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
+    %3:vgpr_32 = V_MOV_B32_dpp %2, %0, 1, 14, 15, 0, implicit $exec
+    %4:vgpr_32 = V_MUL_U32_U24_e32 %1, %3, implicit $exec
+
+    %5:vgpr_32 = V_MOV_B32_e32 4294967295, implicit $exec
+    %6:vgpr_32 = V_MOV_B32_dpp %5, %0, 1, 15, 14, 0, implicit $exec
+    %7:vgpr_32 = V_AND_B32_e32 %1, %6, implicit $exec
+
+    %8:vgpr_32 = V_MOV_B32_e32 -2147483648, implicit $exec
+    %9:vgpr_32 = V_MOV_B32_dpp %8, %0, 1, 14, 15, 0, implicit $exec
+    %10:vgpr_32 = V_MAX_I32_e32 %1, %9, implicit $exec
+
+    %11:vgpr_32 = V_MOV_B32_e32 2147483647, implicit $exec
+    %12:vgpr_32 = V_MOV_B32_dpp %11, %0, 1, 15, 14, 0, implicit $exec
+    %13:vgpr_32 = V_MIN_I32_e32 %1, %12, implicit $exec
+
+    %14:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    %15:vgpr_32 = V_MOV_B32_dpp %14, %0, 1, 14, 15, 0, implicit $exec
+    %16:vgpr_32 = V_SUB_I32_e32 %1, %15, implicit-def $vcc, implicit $exec
+
+    ; this cannot be combined because immediate as src0 isn't commutable
+    %17:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    %18:vgpr_32 = V_MOV_B32_dpp %17, %0, 1, 14, 15, 0, implicit $exec
+    %19:vgpr_32 = V_ADD_I32_e32 5, %18, implicit-def $vcc, implicit $exec
+...
+
+# check for floating point modifiers
+# CHECK-LABEL: name: add_f32_e64
+# CHECK: %3:vgpr_32 = V_MOV_B32_dpp undef %2, %1, 1, 15, 15, 1, implicit $exec
+# CHECK: %4:vgpr_32 = V_ADD_F32_e64 0, %3, 0, %0, 0, 1, implicit $exec
+# CHECK: %6:vgpr_32 = V_ADD_F32_dpp %2, 0, %1, 0, %0, 1, 15, 15, 1, implicit $exec
+# CHECK: %8:vgpr_32 = V_ADD_F32_dpp %2, 1, %1, 2, %0, 1, 15, 15, 1, implicit $exec
+# CHECK: %10:vgpr_32 = V_ADD_F32_e64 4, %9, 8, %0, 0, 0, implicit $exec
+
+name: add_f32_e64
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+
+    %0:vgpr_32 = COPY $vgpr0
+    %1:vgpr_32 = COPY $vgpr1
+    %2:vgpr_32 = IMPLICIT_DEF
+
+    ; this shouldn't be combined as omod is set
+    %3:vgpr_32 = V_MOV_B32_dpp undef %2, %1, 1, 15, 15, 1, implicit $exec
+    %4:vgpr_32 = V_ADD_F32_e64 0, %3, 0, %0, 0, 1, implicit $exec
+
+    ; this should be combined as all modifiers are default
+    %5:vgpr_32 = V_MOV_B32_dpp undef %2, %1, 1, 15, 15, 1, implicit $exec
+    %6:vgpr_32 = V_ADD_F32_e64 0, %5, 0, %0, 0, 0, implicit $exec
+
+    ; this should be combined as modifiers other than abs|neg are default
+    %7:vgpr_32 = V_MOV_B32_dpp undef %2, %1, 1, 15, 15, 1, implicit $exec
+    %8:vgpr_32 = V_ADD_F32_e64 1, %7, 2, %0, 0, 0, implicit $exec
+
+    ; this shouldn't be combined as modifiers aren't abs|neg
+    %9:vgpr_32 = V_MOV_B32_dpp undef %2, %1, 1, 15, 15, 1, implicit $exec
+    %10:vgpr_32 = V_ADD_F32_e64 4, %9, 8, %0, 0, 0, implicit $exec
+...
+
+# tests on sequences of dpp consumers
+# CHECK-LABEL: name: dpp_seq
+# CHECK: %4:vgpr_32 = V_ADD_I32_dpp %1, %0, %1, 1, 14, 15, 0, implicit-def $vcc, implicit $exec
+# CHECK: %5:vgpr_32 = V_SUBREV_I32_dpp %1, %0, %1, 1, 14, 15, 0, implicit-def $vcc, implicit $exec
+# CHECK: %6:vgpr_32 = V_OR_B32_dpp %1, %0, %1, 1, 14, 15, 0, implicit $exec
+# broken sequence:
+# CHECK: %7:vgpr_32 = V_MOV_B32_dpp %2, %0, 1, 14, 15, 0, implicit $exec
+
+name: dpp_seq
+tracksRegLiveness: true
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+    %0:vgpr_32 = COPY $vgpr0
+    %1:vgpr_32 = COPY $vgpr1
+    %2:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+
+    %3:vgpr_32 = V_MOV_B32_dpp %2, %0, 1, 14, 15, 0, implicit $exec
+    %4:vgpr_32 = V_ADD_I32_e32 %3, %1, implicit-def $vcc, implicit $exec
+    %5:vgpr_32 = V_SUB_I32_e32 %1, %3, implicit-def $vcc, implicit $exec
+    %6:vgpr_32 = V_OR_B32_e32 %3, %1, implicit $exec
+
+    %7:vgpr_32 = V_MOV_B32_dpp %2, %0, 1, 14, 15, 0, implicit $exec
+    %8:vgpr_32 = V_ADD_I32_e32 %7, %1, implicit-def $vcc, implicit $exec
+    ; this breaks the sequence
+    %9:vgpr_32 = V_SUB_I32_e32 5, %7, implicit-def $vcc, implicit $exec
+...
+
+# old reg def is in diff BB - cannot combine
+# CHECK-LABEL: name: old_in_diff_bb
+# CHECK: %3:vgpr_32 = V_MOV_B32_dpp %2, %1, 1, 1, 1, 0, implicit $exec
+
+name: old_in_diff_bb
+tracksRegLiveness: true
+body: |
+  bb.0:
+    successors: %bb.1
+    liveins: $vgpr0, $vgpr1
+
+    %0:vgpr_32 = COPY $vgpr0
+    %1:vgpr_32 = COPY $vgpr1
+    %2:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    S_BRANCH %bb.1
+
+  bb.1:
+    %3:vgpr_32 = V_MOV_B32_dpp %2, %1, 1, 1, 1, 0, implicit $exec
+    %4:vgpr_32 = V_ADD_U32_e32 %3, %0, implicit $exec
+...
+
+# old reg def is in diff BB but bound_ctrl:0 - can combine
+# CHECK-LABEL: name: old_in_diff_bb_bctrl_zero
+# CHECK: %4:vgpr_32 = V_ADD_U32_dpp {{%[0-9]}}, %0, %1, 1, 15, 15, 1, implicit $exec
+
+name: old_in_diff_bb_bctrl_zero
+tracksRegLiveness: true
+body: |
+  bb.0:
+    successors: %bb.1
+    liveins: $vgpr0, $vgpr1
+
+    %0:vgpr_32 = COPY $vgpr0
+    %1:vgpr_32 = COPY $vgpr1
+    %2:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    S_BRANCH %bb.1
+
+  bb.1:
+    %3:vgpr_32 = V_MOV_B32_dpp %2, %0, 1, 15, 15, 1, implicit $exec
+    %4:vgpr_32 = V_ADD_U32_e32 %3, %1, implicit $exec
+...
+
+# EXEC mask changed between def and use - cannot combine
+# CHECK-LABEL: name: exec_changed
+# CHECK: %3:vgpr_32 = V_MOV_B32_dpp %2, %0, 1, 15, 15, 1, implicit $exec
+
+name: exec_changed
+tracksRegLiveness: true
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+
+    %0:vgpr_32 = COPY $vgpr0
+    %1:vgpr_32 = COPY $vgpr1
+    %2:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    %3:vgpr_32 = V_MOV_B32_dpp %2, %0, 1, 15, 15, 1, implicit $exec
+    %4:vgpr_32 = V_ADD_U32_e32 %3, %1, implicit $exec
+    %5:sreg_64 = COPY $exec, implicit-def $exec
+    %6:vgpr_32 = V_ADD_U32_e32 %3, %1, implicit $exec
+...
+
+# test if $old definition is correctly tracked through subreg manipulation pseudos
+
+# CHECK-LABEL: name: mul_old_subreg
+# CHECK: %7:vgpr_32 = V_MUL_I32_I24_dpp %0.sub1, %1, %0.sub1, 1, 1, 1, 0, implicit $exec
+
+name:            mul_old_subreg
+tracksRegLiveness: true
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+
+    %0:vreg_64 = COPY $vgpr0
+    %1:vgpr_32 = COPY $vgpr1
+    %2:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
+    %3:vgpr_32 = V_MOV_B32_e32 42, implicit $exec
+    %4:vreg_64 = REG_SEQUENCE %2, %subreg.sub0, %3, %subreg.sub1
+    %5:vreg_64 = INSERT_SUBREG %4, %1, %subreg.sub1 ; %5.sub0 is taken from %4
+    %6:vgpr_32 = V_MOV_B32_dpp %5.sub0, %1, 1, 1, 1, 0, implicit $exec
+    %7:vgpr_32 = V_MUL_I32_I24_e32 %6, %0.sub1, implicit $exec
+...
+
+# CHECK-LABEL: name: add_old_subreg
+# CHECK: %5:vgpr_32 = V_ADD_U32_dpp %0.sub1, %1, %0.sub1, 1, 1, 1, 0, implicit $exec
+
+name:            add_old_subreg
+tracksRegLiveness: true
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+
+    %0:vreg_64 = COPY $vgpr0
+    %1:vgpr_32 = COPY $vgpr1
+    %2:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    %3:vreg_64 = INSERT_SUBREG %0, %2, %subreg.sub1 ; %3.sub1 is inserted
+    %4:vgpr_32 = V_MOV_B32_dpp %3.sub1, %1, 1, 1, 1, 0, implicit $exec
+    %5:vgpr_32 = V_ADD_U32_e32 %4, %0.sub1, implicit $exec
+...
+
+# CHECK-LABEL: name: add_old_subreg_undef
+# CHECK: %5:vgpr_32 = V_ADD_U32_dpp %3.sub1, %1, %0.sub1, 1, 15, 15, 1, implicit $exec
+
+name:            add_old_subreg_undef
+tracksRegLiveness: true
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+
+    %0:vreg_64 = COPY $vgpr0
+    %1:vgpr_32 = COPY $vgpr1
+    %2:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    %3:vreg_64 = REG_SEQUENCE %2, %subreg.sub0 ; %3.sub1 is undef
+    %4:vgpr_32 = V_MOV_B32_dpp %3.sub1, %1, 1, 15, 15, 1, implicit $exec
+    %5:vgpr_32 = V_ADD_U32_e32 %4, %0.sub1, implicit $exec
diff --git a/test/CodeGen/AMDGPU/dpp_combine_subregs.mir b/test/CodeGen/AMDGPU/dpp_combine_subregs.mir
deleted file mode 100644
index 83f992f492a7bb72b6e1a7a8e0fe69680ee1bb26..0000000000000000000000000000000000000000
--- a/test/CodeGen/AMDGPU/dpp_combine_subregs.mir
+++ /dev/null
@@ -1,143 +0,0 @@
-# RUN: llc -march=amdgcn -mcpu=tonga  -run-pass=gcn-dpp-combine  -o - %s | FileCheck %s
-
-# test if $old definition is correctly tracked through subreg manipulation pseudos
-
----
-# CHECK-LABEL: name: mul_old_subreg
-# CHECK: %7:vgpr_32 = V_MUL_I32_I24_dpp %0.sub1, %1, %0.sub1, 1, 1, 1, 0, implicit $exec
-
-name:            mul_old_subreg
-tracksRegLiveness: true
-registers:
-  - { id: 0, class: vreg_64 }
-  - { id: 1, class: vgpr_32 }
-  - { id: 2, class: vgpr_32 }
-  - { id: 3, class: vgpr_32 }
-  - { id: 4, class: vreg_64 }
-  - { id: 5, class: vreg_64 }
-  - { id: 6, class: vgpr_32 }
-  - { id: 7, class: vgpr_32 }
-
-liveins:
-  - { reg: '$vgpr0', virtual-reg: '%0' }
-  - { reg: '$vgpr1', virtual-reg: '%1' }
-body:             |
-  bb.0:
-    liveins: $vgpr0, $vgpr1
-
-    %0:vreg_64 = COPY $vgpr0
-    %1:vgpr_32 = COPY $vgpr1
-    %2:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
-    %3:vgpr_32 = V_MOV_B32_e32 42, implicit $exec
-    %4 = REG_SEQUENCE %2, %subreg.sub0, %3, %subreg.sub1
-    %5 = INSERT_SUBREG %4, %1, %subreg.sub1 ; %5.sub0 is taken from %4
-    %6:vgpr_32 = V_MOV_B32_dpp %5.sub0, %1, 1, 1, 1, 0, implicit $exec
-    %7:vgpr_32 = V_MUL_I32_I24_e32 %6, %0.sub1, implicit $exec
-...
-
-# CHECK-LABEL: name: add_old_subreg
-# CHECK: [[OLD:\%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
-# CHECK: %5:vgpr_32 = V_ADD_U32_dpp [[OLD]], %1, %0.sub1, 1, 1, 1, 1, implicit $exec
-
-name:            add_old_subreg
-tracksRegLiveness: true
-registers:
-  - { id: 0, class: vreg_64 }
-  - { id: 1, class: vgpr_32 }
-  - { id: 2, class: vgpr_32 }
-  - { id: 3, class: vreg_64 }
-  - { id: 4, class: vgpr_32 }
-  - { id: 5, class: vgpr_32 }
-
-liveins:
-  - { reg: '$vgpr0', virtual-reg: '%0' }
-  - { reg: '$vgpr1', virtual-reg: '%1' }
-body:             |
-  bb.0:
-    liveins: $vgpr0, $vgpr1
-
-    %0:vreg_64 = COPY $vgpr0
-    %1:vgpr_32 = COPY $vgpr1
-    %2:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
-    %3:vreg_64 = INSERT_SUBREG %0, %2, %subreg.sub1 ; %3.sub1 is inserted
-    %4:vgpr_32 = V_MOV_B32_dpp %3.sub1, %1, 1, 1, 1, 0, implicit $exec
-    %5:vgpr_32 = V_ADD_U32_e32 %4, %0.sub1, implicit $exec
-...
-
-# CHECK-LABEL: name: add_old_subreg_undef
-# CHECK: %5:vgpr_32 = V_ADD_U32_dpp %3.sub1, %1, %0.sub1, 1, 1, 1, 0, implicit $exec
-
-name:            add_old_subreg_undef
-tracksRegLiveness: true
-registers:
-  - { id: 0, class: vreg_64 }
-  - { id: 1, class: vgpr_32 }
-  - { id: 2, class: vgpr_32 }
-  - { id: 3, class: vreg_64 }
-  - { id: 4, class: vgpr_32 }
-  - { id: 5, class: vgpr_32 }
-
-liveins:
-  - { reg: '$vgpr0', virtual-reg: '%0' }
-  - { reg: '$vgpr1', virtual-reg: '%1' }
-body:             |
-  bb.0:
-    liveins: $vgpr0, $vgpr1
-
-    %0:vreg_64 = COPY $vgpr0
-    %1:vgpr_32 = COPY $vgpr1
-    %2:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
-    %3:vreg_64 = REG_SEQUENCE %2, %subreg.sub0 ; %3.sub1 is undef
-    %4:vgpr_32 = V_MOV_B32_dpp %3.sub1, %1, 1, 1, 1, 0, implicit $exec
-    %5:vgpr_32 = V_ADD_U32_e32 %4, %0.sub1, implicit $exec
-...
-
-# CHECK-LABEL: name: add_f32_e64
-# CHECK: %3:vgpr_32 = V_MOV_B32_dpp undef %2, %1, 1, 1, 1, 1, implicit $exec
-# CHECK: %4:vgpr_32 = V_ADD_F32_e64 0, %3, 0, %0, 0, 1, implicit $exec
-# CHECK: %6:vgpr_32 = V_ADD_F32_dpp %2, 0, %1, 0, %0, 1, 1, 1, 1, implicit $exec
-# CHECK: %7:vgpr_32 = V_ADD_F32_dpp %2, 1, %1, 2, %0, 1, 1, 1, 1, implicit $exec
-# CHECK: %9:vgpr_32 = V_ADD_F32_e64 4, %8, 8, %0, 0, 0, implicit $exec
-
-name:            add_f32_e64
-tracksRegLiveness: true
-registers:
-  - { id: 0, class: vgpr_32 }
-  - { id: 1, class: vgpr_32 }
-  - { id: 2, class: vgpr_32 }
-  - { id: 3, class: vgpr_32 }
-  - { id: 4, class: vgpr_32 }
-  - { id: 5, class: vgpr_32 }
-  - { id: 6, class: vgpr_32 }
-  - { id: 7, class: vgpr_32 }
-  - { id: 8, class: vgpr_32 }
-  - { id: 9, class: vgpr_32 }
-
-liveins:
-  - { reg: '$vgpr0', virtual-reg: '%0' }
-  - { reg: '$vgpr1', virtual-reg: '%1' }
-body:             |
-  bb.0:
-    liveins: $vgpr0, $vgpr1
-
-    %0:vgpr_32 = COPY $vgpr0
-    %1:vgpr_32 = COPY $vgpr1
-    %2:vgpr_32 = IMPLICIT_DEF
-    %3:vgpr_32 = V_MOV_B32_dpp undef %2, %1, 1, 1, 1, 1, implicit $exec
-
-    ; this shouldn't be combined as omod is set
-    %4:vgpr_32 = V_ADD_F32_e64 0, %3, 0, %0, 0, 1, implicit $exec
-
-    %5:vgpr_32 = V_MOV_B32_dpp undef %2, %1, 1, 1, 1, 1, implicit $exec
-
-    ; this should be combined as all modifiers are default
-    %6:vgpr_32 = V_ADD_F32_e64 0, %5, 0, %0, 0, 0, implicit $exec
-
-    ; this should be combined as modifiers other than abs|neg are default
-    %7:vgpr_32 = V_ADD_F32_e64 1, %5, 2, %0, 0, 0, implicit $exec
-
-    %8:vgpr_32 = V_MOV_B32_dpp undef %2, %1, 1, 1, 1, 1, implicit $exec
-
-    ; this shouldn't be combined as modifiers aren't abs|neg
-    %9:vgpr_32 = V_ADD_F32_e64 4, %8, 8, %0, 0, 0, implicit $exec
-...
diff --git a/test/CodeGen/AMDGPU/early-if-convert-cost.ll b/test/CodeGen/AMDGPU/early-if-convert-cost.ll
index b8bb10632acb61d6e81db8b39b275ffaa06a7d03..626a6e2c5b82854815e57ab6bf6e31300616c660 100644
--- a/test/CodeGen/AMDGPU/early-if-convert-cost.ll
+++ b/test/CodeGen/AMDGPU/early-if-convert-cost.ll
@@ -60,7 +60,8 @@ endif:
 ; GCN: v_cndmask_b32_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}, vcc
 ; GCN: v_cndmask_b32_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}, vcc
 
-; GCN-DAG: buffer_store_dwordx3
+; GCN-DAG: buffer_store_dword v
+; GCN-DAG: buffer_store_dwordx2
 define amdgpu_kernel void @test_vccnz_ifcvt_triangle96(<3 x i32> addrspace(1)* %out, <3 x i32> addrspace(1)* %in, float %cnd) #0 {
 entry:
   %v = load <3 x i32>, <3 x i32> addrspace(1)* %in
diff --git a/test/CodeGen/AMDGPU/elf.metadata.ll b/test/CodeGen/AMDGPU/elf.metadata.ll
index 097310a167392b43a3b2dbf395d9d37285091bd5..9214ca2e68e9c139de33b52a3f31ed4fa5f42a49 100644
--- a/test/CodeGen/AMDGPU/elf.metadata.ll
+++ b/test/CodeGen/AMDGPU/elf.metadata.ll
@@ -1,7 +1,7 @@
 ; RUN: llc < %s -march=amdgcn -mcpu=fiji -filetype=obj | llvm-readobj -symbols -s -sd - | FileCheck %s
 
 ; CHECK: Section {
-; CHECK: Name: .AMDGPU.metadata.info_1
+; CHECK: Name: .AMDGPU.comment.info_1
 ; CHECK: Type: SHT_PROGBITS (0x1)
 ; CHECK: Flags [ (0x0)
 ; CHECK: Size: 16
@@ -11,7 +11,7 @@
 ; CHECK: }
 
 ; CHECK: Section {
-; CHECK: Name: .AMDGPU.metadata.info_2
+; CHECK: Name: .AMDGPU.comment.info_2
 ; CHECK: Type: SHT_PROGBITS (0x1)
 ; CHECK: Flags [ (0x0)
 ; CHECK: Size: 16
@@ -21,7 +21,7 @@
 ; CHECK: }
 
 ; CHECK: Section {
-; CHECK: Name: .AMDGPU.metadata.info_3
+; CHECK: Name: .AMDGPU.comment.info_3
 ; CHECK: Type: SHT_PROGBITS (0x1)
 ; CHECK: Flags [ (0x0)
 ; CHECK: Size: 16
@@ -31,26 +31,26 @@
 ; CHECK: }
 
 ; CHECK: Symbol {
-; CHECK: Name: metadata_info_var_1
+; CHECK: Name: comment_info_var_1
 ; CHECK: Size: 16
 ; CHECK: Binding: Local
-; CHECK: Section: .AMDGPU.metadata.info_1
+; CHECK: Section: .AMDGPU.comment.info_1
 ; CHECK: }
 
 ; CHECK: Symbol {
-; CHECK: Name: metadata_info_var_2
+; CHECK: Name: comment_info_var_2
 ; CHECK: Size: 16
 ; CHECK: Binding: Global
-; CHECK: Section: .AMDGPU.metadata.info_2
+; CHECK: Section: .AMDGPU.comment.info_2
 ; CHECK: }
 
 ; CHECK: Symbol {
-; CHECK: Name: metadata_info_var_3
+; CHECK: Name: comment_info_var_3
 ; CHECK: Size: 16
 ; CHECK: Binding: Global
-; CHECK: Section: .AMDGPU.metadata.info_3
+; CHECK: Section: .AMDGPU.comment.info_3
 ; CHECK: }
 
-@metadata_info_var_1 = internal global [4 x i32][i32 826559809, i32 826559809, i32 826559809, i32 826559809], align 1, section ".AMDGPU.metadata.info_1"
-@metadata_info_var_2 = constant [4 x i32][i32 843337025, i32 843337025, i32 843337025, i32 843337025], align 1, section ".AMDGPU.metadata.info_2"
-@metadata_info_var_3 = global [4 x i32][i32 860114241, i32 860114241, i32 860114241, i32 860114241], align 1, section ".AMDGPU.metadata.info_3"
+@comment_info_var_1 = internal global [4 x i32][i32 826559809, i32 826559809, i32 826559809, i32 826559809], align 1, section ".AMDGPU.comment.info_1"
+@comment_info_var_2 = constant [4 x i32][i32 843337025, i32 843337025, i32 843337025, i32 843337025], align 1, section ".AMDGPU.comment.info_2"
+@comment_info_var_3 = global [4 x i32][i32 860114241, i32 860114241, i32 860114241, i32 860114241], align 1, section ".AMDGPU.comment.info_3"
diff --git a/test/CodeGen/AMDGPU/fcanonicalize-elimination.ll b/test/CodeGen/AMDGPU/fcanonicalize-elimination.ll
index e2741c25382eda73541d65efcb0542bd9e637efb..cee091af6435ecc1bb66506caada74434428314d 100644
--- a/test/CodeGen/AMDGPU/fcanonicalize-elimination.ll
+++ b/test/CodeGen/AMDGPU/fcanonicalize-elimination.ll
@@ -814,8 +814,8 @@ define half @v_test_canonicalize_extract_element_v2f16(<2 x half> %vec) {
 }
 
 ; GCN-LABEL: {{^}}v_test_canonicalize_insertelement_v2f16:
-; GFX9: v_pk_mul_f16
 ; GFX9: v_mul_f16_e32
+; GFX9: v_pk_mul_f16
 ; GFX9-NOT: v_max
 ; GFX9-NOT: v_pk_max
 define <2 x half> @v_test_canonicalize_insertelement_v2f16(<2 x half> %vec, half %val, i32 %idx) {
diff --git a/test/CodeGen/AMDGPU/fdiv32-to-rcp-folding.ll b/test/CodeGen/AMDGPU/fdiv32-to-rcp-folding.ll
index 3746aa8784bff03d33bc988f6111c79fe9ea35c1..a3f176b3ef025301f69d9537cdfd741af0f5193c 100644
--- a/test/CodeGen/AMDGPU/fdiv32-to-rcp-folding.ll
+++ b/test/CodeGen/AMDGPU/fdiv32-to-rcp-folding.ll
@@ -131,10 +131,10 @@ define amdgpu_kernel void @div_v4_1_by_x_25ulp(<4 x float> addrspace(1)* %arg) {
 ; GCN-DENORM-DAG: v_cndmask_b32_e32 v{{[0-9]+}}, 1.0, [[S]], vcc
 ; GCN-DENORM-DAG: v_cmp_gt_f32_e64 vcc, |v{{[0-9]+}}|, [[L]]
 ; GCN-DENORM-DAG: v_cndmask_b32_e32 v{{[0-9]+}}, 1.0, [[S]], vcc
-; GCN-DENORM-DAG: v_mul_f32_e64 v{{[0-9]+}}, v{{[0-9]+}}, -v{{[0-9]+}}
-; GCN-DENORM-DAG: v_mul_f32_e64 v{{[0-9]+}}, v{{[0-9]+}}, -v{{[0-9]+}}
-; GCN-DENORM-DAG: v_mul_f32_e64 v{{[0-9]+}}, v{{[0-9]+}}, -v{{[0-9]+}}
-; GCN-DENORM-DAG: v_mul_f32_e64 v{{[0-9]+}}, v{{[0-9]+}}, -v{{[0-9]+}}
+; GCN-DENORM-DAG: v_mul_f32_e64 v{{[0-9]+}}, s{{[0-9]+}}, -v{{[0-9]+}}
+; GCN-DENORM-DAG: v_mul_f32_e64 v{{[0-9]+}}, s{{[0-9]+}}, -v{{[0-9]+}}
+; GCN-DENORM-DAG: v_mul_f32_e64 v{{[0-9]+}}, s{{[0-9]+}}, -v{{[0-9]+}}
+; GCN-DENORM-DAG: v_mul_f32_e64 v{{[0-9]+}}, s{{[0-9]+}}, -v{{[0-9]+}}
 ; GCN-DENORM-DAG: v_rcp_f32_e32
 ; GCN-DENORM-DAG: v_rcp_f32_e32
 ; GCN-DENORM-DAG: v_rcp_f32_e32
@@ -166,10 +166,10 @@ define amdgpu_kernel void @div_v4_minus_1_by_x_25ulp(<4 x float> addrspace(1)* %
 ; GCN-DENORM-DAG: v_cndmask_b32_e32 v{{[0-9]+}}, 1.0, [[S]], vcc
 ; GCN-DENORM-DAG: v_cmp_gt_f32_e64 vcc, |v{{[0-9]+}}|, [[L]]
 ; GCN-DENORM-DAG: v_cndmask_b32_e32 v{{[0-9]+}}, 1.0, [[S]], vcc
-; GCN-DENORM-DAG: v_mul_f32_e64 v{{[0-9]+}}, -v{{[0-9]+}}, v{{[0-9]+}}
-; GCN-DENORM-DAG: v_mul_f32_e64 v{{[0-9]+}}, -v{{[0-9]+}}, v{{[0-9]+}}
-; GCN-DENORM-DAG: v_mul_f32_e64 v{{[0-9]+}}, -v{{[0-9]+}}, v{{[0-9]+}}
-; GCN-DENORM-DAG: v_mul_f32_e64 v{{[0-9]+}}, -v{{[0-9]+}}, v{{[0-9]+}}
+; GCN-DENORM-DAG: v_mul_f32_e64 v{{[0-9]+}}, -s{{[0-9]+}}, v{{[0-9]+}}
+; GCN-DENORM-DAG: v_mul_f32_e64 v{{[0-9]+}}, -s{{[0-9]+}}, v{{[0-9]+}}
+; GCN-DENORM-DAG: v_mul_f32_e64 v{{[0-9]+}}, -s{{[0-9]+}}, v{{[0-9]+}}
+; GCN-DENORM-DAG: v_mul_f32_e64 v{{[0-9]+}}, -s{{[0-9]+}}, v{{[0-9]+}}
 ; GCN-DENORM-DAG: v_rcp_f32_e32
 ; GCN-DENORM-DAG: v_rcp_f32_e32
 ; GCN-DENORM-DAG: v_rcp_f32_e32
@@ -246,7 +246,7 @@ define amdgpu_kernel void @div_v4_minus_1_by_minus_x_25ulp(<4 x float> addrspace
 ; GCN-DAG:        v_cndmask_b32_e32 v{{[0-9]+}}, 1.0, [[S]], vcc
 
 ; GCN-DENORM-DAG: v_mul_f32_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
-; GCN-DENORM-DAG: v_mul_f32_e64 v{{[0-9]+}}, v{{[0-9]+}}, -v{{[0-9]+}}
+; GCN-DENORM-DAG: v_mul_f32_e64 v{{[0-9]+}}, s{{[0-9]+}}, -v{{[0-9]+}}
 ; GCN-DENORM-DAG: v_rcp_f32_e32 [[RCP1:v[0-9]+]], v{{[0-9]+}}
 ; GCN-DENORM-DAG: v_mul_f32_e32 v{{[0-9]+}}, v{{[0-9]+}}, [[RCP1]]
 ; GCN-DENORM-DAG: v_rcp_f32_e32 [[RCP2:v[0-9]+]], v{{[0-9]+}}
@@ -288,7 +288,7 @@ define amdgpu_kernel void @div_v4_c_by_x_25ulp(<4 x float> addrspace(1)* %arg) {
 ; GCN-DAG:        v_cndmask_b32_e32 v{{[0-9]+}}, 1.0, [[S]], vcc
 
 ; GCN-DENORM-DAG: v_mul_f32_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
-; GCN-DENORM-DAG: v_mul_f32_e64 v{{[0-9]+}}, -v{{[0-9]+}}, v{{[0-9]+}}
+; GCN-DENORM-DAG: v_mul_f32_e64 v{{[0-9]+}}, -s{{[0-9]+}}, v{{[0-9]+}}
 ; GCN-DENORM-DAG: v_rcp_f32_e32 [[RCP1:v[0-9]+]], v{{[0-9]+}}
 ; GCN-DENORM-DAG: v_mul_f32_e32 v{{[0-9]+}}, v{{[0-9]+}}, [[RCP1]]
 ; GCN-DENORM-DAG: v_rcp_f32_e32 [[RCP2:v[0-9]+]], v{{[0-9]+}}
diff --git a/test/CodeGen/AMDGPU/fix-wwm-vgpr-copy.ll b/test/CodeGen/AMDGPU/fix-wwm-vgpr-copy.ll
new file mode 100644
index 0000000000000000000000000000000000000000..3375efa14dd9b26ab80f902691d1ce88f3caca9f
--- /dev/null
+++ b/test/CodeGen/AMDGPU/fix-wwm-vgpr-copy.ll
@@ -0,0 +1,47 @@
+; RUN: llc -mtriple=amdgcn--amdpal -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
+
+declare i64 @llvm.amdgcn.icmp.i32(i32, i32, i32) #0
+declare i32 @llvm.amdgcn.set.inactive.i32(i32, i32) #0
+declare i32 @llvm.amdgcn.wwm.i32(i32) #1
+declare void @llvm.amdgcn.tbuffer.store.f32(float, <4 x i32>, i32, i32, i32, i32, i32, i32, i1, i1) #2
+declare void @llvm.amdgcn.exp.f32(i32, i32, float, float, float, float, i1, i1) #2
+
+define amdgpu_hs void @foo(i32 inreg %arg, <4 x i32> inreg %buffer) {
+entry:
+  br label %work
+
+bb42:
+  br label %bb602
+
+bb602:
+  %tmp603 = phi i32 [ 0, %bb42 ], [ 1, %work ]
+  %tmp607 = icmp eq i32 %tmp603, %tmp1196
+  br i1 %tmp607, label %bb49, label %bb54
+
+bb49:
+  tail call void @llvm.amdgcn.tbuffer.store.f32(float 1.000000e+00, <4 x i32> %buffer, i32 0, i32 1, i32 1, i32 4, i32 4, i32 7, i1 true, i1 false) #7
+  ret void
+
+bb54:
+  ret void
+
+work:
+; GCN: s_not_b64 exec, exec
+; GCN: v_mov_b32_e32 v[[tmp1189:[0-9]+]], 1
+; GCN: s_not_b64 exec, exec
+  %tmp1189 = tail call i32 @llvm.amdgcn.set.inactive.i32(i32 4, i32 1)
+
+; GCN: s_or_saveexec_b64 s{{\[}}[[LO:[0-9]+]]:[[HI:[0-9]+]]{{\]}}, -1
+; GCN: v_lshlrev_b32_e32 v[[tmp1191:[0-9]+]], 2, v[[tmp1189]]
+  %tmp1191 = mul i32 %tmp1189, 4
+
+; GCN: s_mov_b64 exec, s{{\[}}[[LO]]:[[HI]]{{\]}}
+  %tmp1196 = tail call i32 @llvm.amdgcn.wwm.i32(i32 %tmp1191)
+
+  %tmp34 = icmp eq i32 %arg, 0
+  br i1 %tmp34, label %bb602, label %bb42
+}
+
+attributes #0 = { convergent nounwind readnone }
+attributes #1 = { nounwind readnone speculatable }
+attributes #2 = { nounwind writeonly }
diff --git a/test/CodeGen/AMDGPU/flat_atomics.ll b/test/CodeGen/AMDGPU/flat_atomics.ll
index 1edb48600f7fc4947f2eed8dbea68dfdd48334a8..d20894979ed20fc25270074bbc26421bc2ddcd09 100644
--- a/test/CodeGen/AMDGPU/flat_atomics.ll
+++ b/test/CodeGen/AMDGPU/flat_atomics.ll
@@ -703,6 +703,16 @@ entry:
   ret void
 }
 
+; GCN-LABEL: {{^}}atomic_xchg_f32_offset:
+; CIVI: flat_atomic_swap v[{{[0-9]+:[0-9]+}}], v{{[0-9]+$}}
+; GFX9: flat_atomic_swap v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}} offset:16{{$}}
+define amdgpu_kernel void @atomic_xchg_f32_offset(float* %out, float %in) {
+entry:
+  %gep = getelementptr float, float* %out, i32 4
+  %val = atomicrmw volatile xchg float* %gep, float %in seq_cst
+  ret void
+}
+
 ; GCN-LABEL: {{^}}atomic_xchg_i32_ret_offset:
 ; CIVI: flat_atomic_swap [[RET:v[0-9]+]], v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}} glc{{$}}
 ; GFX9: flat_atomic_swap [[RET:v[0-9]+]], v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}} offset:16 glc{{$}}
diff --git a/test/CodeGen/AMDGPU/flat_atomics_i64.ll b/test/CodeGen/AMDGPU/flat_atomics_i64.ll
index ca1364e6753a723c34569756c1fb28024788713d..0cf28275b87b717d2797ece63f1c037af631b2d5 100644
--- a/test/CodeGen/AMDGPU/flat_atomics_i64.ll
+++ b/test/CodeGen/AMDGPU/flat_atomics_i64.ll
@@ -650,6 +650,15 @@ entry:
   ret void
 }
 
+; GCN-LABEL: {{^}}atomic_xchg_f64_offset:
+; GCN: flat_atomic_swap_x2 v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]$}}
+define amdgpu_kernel void @atomic_xchg_f64_offset(double* %out, double %in) {
+entry:
+  %gep = getelementptr double, double* %out, i64 4
+  %tmp0 = atomicrmw volatile xchg double* %gep, double %in seq_cst
+  ret void
+}
+
 ; GCN-LABEL: {{^}}atomic_xchg_i64_ret_offset:
 ; GCN: flat_atomic_swap_x2 [[RET:v\[[0-9]+:[0-9]+\]]], v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}} glc{{$}}
 ; GCN: flat_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, [[RET]]
diff --git a/test/CodeGen/AMDGPU/function-call-relocs.ll b/test/CodeGen/AMDGPU/function-call-relocs.ll
new file mode 100644
index 0000000000000000000000000000000000000000..c38179f9d434ff880ba7e32def00019036ca8b39
--- /dev/null
+++ b/test/CodeGen/AMDGPU/function-call-relocs.ll
@@ -0,0 +1,51 @@
+; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=fiji < %s | FileCheck %s
+
+declare void @func(i32 addrspace(1)* %out)
+
+declare protected void @protected_func(i32 addrspace(1)* %out)
+
+declare hidden void @hidden_func(i32 addrspace(1)* %out)
+
+; CHECK-LABEL: call_func:
+; CHECK: s_getpc_b64 s{{\[}}[[PC_LO:[0-9]+]]:[[PC_HI:[0-9]+]]{{\]}}
+; CHECK: s_add_u32 s[[GOT_ADDR_LO:[0-9]+]], s[[PC_LO]], func@gotpcrel32@lo+4
+; CHECK: s_addc_u32 s[[GOT_ADDR_HI:[0-9]+]], s[[PC_HI]], func@gotpcrel32@hi+4
+; CHECK: s_load_dwordx2 s{{\[}}[[ADDR_LO:[0-9]+]]:[[ADDR_HI:[0-9]+]]{{\]}}, s{{\[}}[[GOT_ADDR_LO]]:[[GOT_ADDR_HI]]{{\]}}, 0x0
+; CHECK: s_swappc_b64 s{{\[}}{{[0-9]+:[0-9]+}}{{\]}}, s{{\[}}[[ADDR_LO]]:[[ADDR_HI]]{{\]}}
+define amdgpu_kernel void @call_func(i32 addrspace(1)* %out) {
+  call void @func(i32 addrspace(1)* %out)
+  ret void
+}
+
+; CHECK-LABEL: call_protected_func:
+; CHECK: s_getpc_b64 s{{\[}}[[PC_LO:[0-9]+]]:[[PC_HI:[0-9]+]]{{\]}}
+; CHECK: s_add_u32 s[[ADDR_LO:[0-9]+]], s[[PC_LO]], protected_func@rel32@lo+4
+; CHECK: s_addc_u32 s[[ADDR_HI:[0-9]+]], s[[PC_HI]], protected_func@rel32@hi+4
+; CHECK: s_swappc_b64 s{{\[}}{{[0-9]+:[0-9]+}}{{\]}}, s{{\[}}[[ADDR_LO]]:[[ADDR_HI]]{{\]}}
+define amdgpu_kernel void @call_protected_func(i32 addrspace(1)* %out) {
+  call void @protected_func(i32 addrspace(1)* %out)
+  ret void
+}
+
+; CHECK-LABEL: call_hidden_func:
+; CHECK: s_getpc_b64 s{{\[}}[[PC_LO:[0-9]+]]:[[PC_HI:[0-9]+]]{{\]}}
+; CHECK: s_add_u32 s[[ADDR_LO:[0-9]+]], s[[PC_LO]], hidden_func@rel32@lo+4
+; CHECK: s_addc_u32 s[[ADDR_HI:[0-9]+]], s[[PC_HI]], hidden_func@rel32@hi+4
+; CHECK: s_swappc_b64 s{{\[}}{{[0-9]+:[0-9]+}}{{\]}}, s{{\[}}[[ADDR_LO]]:[[ADDR_HI]]{{\]}}
+define amdgpu_kernel void @call_hidden_func(i32 addrspace(1)* %out) {
+  call void @hidden_func(i32 addrspace(1)* %out)
+  ret void
+}
+
+declare i64 @funci()
+
+; CHECK-LABEL: tail_call_func:
+; CHECK: s_getpc_b64 s{{\[}}[[PC_LO:[0-9]+]]:[[PC_HI:[0-9]+]]{{\]}}
+; CHECK: s_add_u32 s[[GOT_ADDR_LO:[0-9]+]], s[[PC_LO]], funci@gotpcrel32@lo+4
+; CHECK: s_addc_u32 s[[GOT_ADDR_HI:[0-9]+]], s[[PC_HI]], funci@gotpcrel32@hi+4
+; CHECK: s_load_dwordx2 s{{\[}}[[ADDR_LO:[0-9]+]]:[[ADDR_HI:[0-9]+]]{{\]}}, s{{\[}}[[GOT_ADDR_LO]]:[[GOT_ADDR_HI]]{{\]}}, 0x0
+; CHECK: s_setpc_b64 s{{\[}}[[ADDR_LO]]:[[ADDR_HI]]{{\]}}
+define i64 @tail_call_func() {
+  %ret = tail call i64 @funci()
+  ret i64 %ret
+}
diff --git a/test/CodeGen/AMDGPU/global_atomics.ll b/test/CodeGen/AMDGPU/global_atomics.ll
index 031f17f1385504af34012496cdce611670d4fb8c..72f748578b34271a46232b4aaf023867549e0508 100644
--- a/test/CodeGen/AMDGPU/global_atomics.ll
+++ b/test/CodeGen/AMDGPU/global_atomics.ll
@@ -839,6 +839,17 @@ entry:
   ret void
 }
 
+; GCN-LABEL: {{^}}atomic_xchg_f32_offset:
+; SIVI: buffer_atomic_swap v{{[0-9]+}}, off, s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:16{{$}}
+
+; GFX9: global_atomic_swap v[{{[0-9]+:[0-9]+}}], v{{[0-9]+}}, off offset:16{{$}}
+define amdgpu_kernel void @atomic_xchg_f32_offset(float addrspace(1)* %out, float %in) {
+entry:
+  %gep = getelementptr float, float addrspace(1)* %out, i64 4
+  %val = atomicrmw volatile xchg float addrspace(1)* %gep, float %in seq_cst
+  ret void
+}
+
 ; GCN-LABEL: {{^}}atomic_xchg_i32_ret_offset:
 ; SIVI: buffer_atomic_swap [[RET:v[0-9]+]], off, s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:16 glc{{$}}
 ; SIVI: buffer_store_dword [[RET]]
diff --git a/test/CodeGen/AMDGPU/global_atomics_i64.ll b/test/CodeGen/AMDGPU/global_atomics_i64.ll
index 3a26b84c2230c0b4d6878719d2874c718758023c..0ef58fc1fb82d0eadc02af72b7b2e9fcb30a856e 100644
--- a/test/CodeGen/AMDGPU/global_atomics_i64.ll
+++ b/test/CodeGen/AMDGPU/global_atomics_i64.ll
@@ -783,6 +783,17 @@ entry:
   ret void
 }
 
+; GCN-LABEL: {{^}}atomic_xchg_f64_offset:
+; CIVI: buffer_atomic_swap_x2 v{{\[[0-9]+:[0-9]+\]}}, off, s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:32{{$}}
+
+; GFX9: global_atomic_swap_x2 v[{{[0-9]+:[0-9]+}}], v{{\[[0-9]+:[0-9]+\]}}, off offset:32{{$}}
+define amdgpu_kernel void @atomic_xchg_f64_offset(double addrspace(1)* %out, double %in) {
+entry:
+  %gep = getelementptr double, double addrspace(1)* %out, i64 4
+  %tmp0 = atomicrmw volatile xchg double addrspace(1)* %gep, double %in seq_cst
+  ret void
+}
+
 ; GCN-LABEL: {{^}}atomic_xchg_i64_ret_offset:
 ; CIVI: buffer_atomic_swap_x2 [[RET:v\[[0-9]+:[0-9]+\]]], off, s[{{[0-9]+}}:{{[0-9]+}}], 0 offset:32 glc{{$}}
 ; CIVI: buffer_store_dwordx2 [[RET]]
diff --git a/test/CodeGen/AMDGPU/idot4s.ll b/test/CodeGen/AMDGPU/idot4s.ll
new file mode 100644
index 0000000000000000000000000000000000000000..2c5873ea17a80c4b85821e8fc71e6ccabffde45d
--- /dev/null
+++ b/test/CodeGen/AMDGPU/idot4s.ll
@@ -0,0 +1,1006 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=amdgcn -mcpu=gfx700 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX7 %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx803 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX8 %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX9-NODL %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx906 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX9-DL %s
+
+define amdgpu_kernel void @idot4_acc32(<4 x i8> addrspace(1)* %src1,
+; GFX7-LABEL: idot4_acc32:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
+; GFX7-NEXT:    s_mov_b32 s3, 0xf000
+; GFX7-NEXT:    s_mov_b32 s2, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s5, s[6:7], 0x0
+; GFX7-NEXT:    s_load_dword s6, s[0:1], 0x0
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_sext_i32_i8 s7, s4
+; GFX7-NEXT:    s_sext_i32_i8 s8, s5
+; GFX7-NEXT:    s_bfe_i32 s10, s5, 0x80008
+; GFX7-NEXT:    v_mov_b32_e32 v0, s8
+; GFX7-NEXT:    v_mov_b32_e32 v1, s6
+; GFX7-NEXT:    s_bfe_i32 s12, s5, 0x80010
+; GFX7-NEXT:    v_mad_i32_i24 v0, s7, v0, v1
+; GFX7-NEXT:    s_bfe_i32 s9, s4, 0x80008
+; GFX7-NEXT:    v_mov_b32_e32 v1, s10
+; GFX7-NEXT:    s_bfe_i32 s11, s4, 0x80010
+; GFX7-NEXT:    v_mad_i32_i24 v0, s9, v1, v0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s12
+; GFX7-NEXT:    s_ashr_i32 s5, s5, 24
+; GFX7-NEXT:    v_mad_i32_i24 v0, s11, v1, v0
+; GFX7-NEXT:    s_ashr_i32 s4, s4, 24
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mad_i32_i24 v0, s4, v1, v0
+; GFX7-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX8-LABEL: idot4_acc32:
+; GFX8:       ; %bb.0: ; %entry
+; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX8-NEXT:    s_load_dword s3, s[6:7], 0x0
+; GFX8-NEXT:    s_load_dword s4, s[0:1], 0x0
+; GFX8-NEXT:    v_mov_b32_e32 v0, s0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    s_sext_i32_i8 s0, s2
+; GFX8-NEXT:    s_sext_i32_i8 s1, s3
+; GFX8-NEXT:    s_bfe_i32 s6, s3, 0x80008
+; GFX8-NEXT:    v_mov_b32_e32 v2, s1
+; GFX8-NEXT:    v_mov_b32_e32 v3, s4
+; GFX8-NEXT:    s_bfe_i32 s8, s3, 0x80010
+; GFX8-NEXT:    v_mad_i32_i24 v2, s0, v2, v3
+; GFX8-NEXT:    s_bfe_i32 s5, s2, 0x80008
+; GFX8-NEXT:    v_mov_b32_e32 v3, s6
+; GFX8-NEXT:    s_bfe_i32 s7, s2, 0x80010
+; GFX8-NEXT:    v_mad_i32_i24 v2, s5, v3, v2
+; GFX8-NEXT:    v_mov_b32_e32 v3, s8
+; GFX8-NEXT:    s_ashr_i32 s3, s3, 24
+; GFX8-NEXT:    v_mad_i32_i24 v2, s7, v3, v2
+; GFX8-NEXT:    s_ashr_i32 s2, s2, 24
+; GFX8-NEXT:    v_mov_b32_e32 v3, s3
+; GFX8-NEXT:    v_mad_i32_i24 v2, s2, v3, v2
+; GFX8-NEXT:    flat_store_dword v[0:1], v2
+; GFX8-NEXT:    s_endpgm
+;
+; GFX9-NODL-LABEL: idot4_acc32:
+; GFX9-NODL:       ; %bb.0: ; %entry
+; GFX9-NODL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX9-NODL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NODL-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX9-NODL-NEXT:    s_load_dword s3, s[6:7], 0x0
+; GFX9-NODL-NEXT:    s_load_dword s4, s[0:1], 0x0
+; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NODL-NEXT:    s_sext_i32_i8 s0, s2
+; GFX9-NODL-NEXT:    s_sext_i32_i8 s1, s3
+; GFX9-NODL-NEXT:    s_bfe_i32 s6, s3, 0x80008
+; GFX9-NODL-NEXT:    v_mov_b32_e32 v2, s1
+; GFX9-NODL-NEXT:    v_mov_b32_e32 v3, s4
+; GFX9-NODL-NEXT:    s_bfe_i32 s8, s3, 0x80010
+; GFX9-NODL-NEXT:    v_mad_i32_i24 v2, s0, v2, v3
+; GFX9-NODL-NEXT:    s_bfe_i32 s5, s2, 0x80008
+; GFX9-NODL-NEXT:    v_mov_b32_e32 v3, s6
+; GFX9-NODL-NEXT:    s_bfe_i32 s7, s2, 0x80010
+; GFX9-NODL-NEXT:    v_mad_i32_i24 v2, s5, v3, v2
+; GFX9-NODL-NEXT:    v_mov_b32_e32 v3, s8
+; GFX9-NODL-NEXT:    s_ashr_i32 s3, s3, 24
+; GFX9-NODL-NEXT:    v_mad_i32_i24 v2, s7, v3, v2
+; GFX9-NODL-NEXT:    s_ashr_i32 s2, s2, 24
+; GFX9-NODL-NEXT:    v_mov_b32_e32 v3, s3
+; GFX9-NODL-NEXT:    v_mad_i32_i24 v2, s2, v3, v2
+; GFX9-NODL-NEXT:    global_store_dword v[0:1], v2, off
+; GFX9-NODL-NEXT:    s_endpgm
+;
+; GFX9-DL-LABEL: idot4_acc32:
+; GFX9-DL:       ; %bb.0: ; %entry
+; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX9-DL-NEXT:    s_load_dword s3, s[6:7], 0x0
+; GFX9-DL-NEXT:    s_load_dword s4, s[0:1], 0x0
+; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s3
+; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s4
+; GFX9-DL-NEXT:    v_dot4_i32_i8 v2, s2, v2, v3
+; GFX9-DL-NEXT:    global_store_dword v[0:1], v2, off
+; GFX9-DL-NEXT:    s_endpgm
+                                       <4 x i8> addrspace(1)* %src2,
+                                       i32 addrspace(1)* nocapture %dst) {
+entry:
+  %vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %src1
+  %vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %src2
+
+  %v1e0 = extractelement <4 x i8> %vec1, i64 0
+  %cv1e0 = sext i8 %v1e0 to i32
+  %v2e0 = extractelement <4 x i8> %vec2, i64 0
+  %cv2e0 = sext i8 %v2e0 to i32
+  %mul1 = mul nuw nsw i32 %cv1e0, %cv2e0
+
+  %v1e1 = extractelement <4 x i8> %vec1, i64 1
+  %cv1e1 = sext i8 %v1e1 to i32
+  %v2e1 = extractelement <4 x i8> %vec2, i64 1
+  %cv2e1 = sext i8 %v2e1 to i32
+  %mul2 = mul nuw nsw i32 %cv1e1, %cv2e1
+
+  %v1e2 = extractelement <4 x i8> %vec1, i64 2
+  %cv1e2 = sext i8 %v1e2 to i32
+  %v2e2 = extractelement <4 x i8> %vec2, i64 2
+  %cv2e2 = sext i8 %v2e2 to i32
+  %mul3 = mul nuw nsw i32 %cv1e2, %cv2e2
+
+  %v1e3 = extractelement <4 x i8> %vec1, i64 3
+  %cv1e3 = sext i8 %v1e3 to i32
+  %v2e3 = extractelement <4 x i8> %vec2, i64 3
+  %cv2e3 = sext i8 %v2e3 to i32
+  %mul4 = mul nuw nsw i32 %cv1e3, %cv2e3
+
+  %acc = load i32, i32 addrspace(1)* %dst, align 4
+  %add1 = add i32 %mul1, %acc
+  %add2 = add i32 %add1, %mul2
+  %add3 = add i32 %add2, %mul3
+  %add4 = add i32 %add3, %mul4
+  store i32 %add4, i32 addrspace(1)* %dst, align 4
+  ret void
+}
+
+; TODO: Currently, vector elements{0 and 3} get zero_extended from i16 to i32 which should
+; be sign_extended directly to i32; prevents the pattern recognizer to recognize this pattern.
+define amdgpu_kernel void @idot4_acc16(<4 x i8> addrspace(1)* %src1,
+; GFX7-LABEL: idot4_acc16:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
+; GFX7-NEXT:    s_mov_b32 s3, 0xf000
+; GFX7-NEXT:    s_mov_b32 s2, -1
+; GFX7-NEXT:    s_mov_b32 s8, 0xffff
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
+; GFX7-NEXT:    buffer_load_ushort v0, off, s[0:3], 0
+; GFX7-NEXT:    s_load_dword s5, s[6:7], 0x0
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_sext_i32_i8 s6, s4
+; GFX7-NEXT:    s_bfe_i32 s9, s4, 0x80008
+; GFX7-NEXT:    s_sext_i32_i8 s7, s5
+; GFX7-NEXT:    s_bfe_i32 s10, s5, 0x80008
+; GFX7-NEXT:    s_and_b32 s7, s7, s8
+; GFX7-NEXT:    s_bfe_i32 s12, s5, 0x80010
+; GFX7-NEXT:    s_and_b32 s10, s10, s8
+; GFX7-NEXT:    s_and_b32 s6, s6, s8
+; GFX7-NEXT:    v_mov_b32_e32 v1, s7
+; GFX7-NEXT:    s_bfe_i32 s11, s4, 0x80010
+; GFX7-NEXT:    s_ashr_i32 s5, s5, 24
+; GFX7-NEXT:    s_and_b32 s12, s12, s8
+; GFX7-NEXT:    s_and_b32 s9, s9, s8
+; GFX7-NEXT:    v_mov_b32_e32 v2, s10
+; GFX7-NEXT:    s_ashr_i32 s4, s4, 24
+; GFX7-NEXT:    s_and_b32 s11, s11, s8
+; GFX7-NEXT:    s_and_b32 s5, s5, s8
+; GFX7-NEXT:    v_mov_b32_e32 v3, s12
+; GFX7-NEXT:    s_and_b32 s4, s4, s8
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    v_mad_u32_u24 v0, s6, v1, v0
+; GFX7-NEXT:    v_mad_u32_u24 v0, s9, v2, v0
+; GFX7-NEXT:    v_mad_u32_u24 v0, s11, v3, v0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mad_u32_u24 v0, s4, v1, v0
+; GFX7-NEXT:    buffer_store_short v0, off, s[0:3], 0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX8-LABEL: idot4_acc16:
+; GFX8:       ; %bb.0: ; %entry
+; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v0, s0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-NEXT:    flat_load_ushort v2, v[0:1]
+; GFX8-NEXT:    s_load_dword s1, s[4:5], 0x0
+; GFX8-NEXT:    s_load_dword s2, s[6:7], 0x0
+; GFX8-NEXT:    s_mov_b32 s0, 0xffff
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    s_lshr_b32 s5, s1, 8
+; GFX8-NEXT:    s_lshr_b32 s6, s2, 8
+; GFX8-NEXT:    s_sext_i32_i8 s4, s2
+; GFX8-NEXT:    s_bfe_i32 s5, s5, 0x80000
+; GFX8-NEXT:    s_bfe_i32 s6, s6, 0x80000
+; GFX8-NEXT:    s_bfe_i32 s8, s2, 0x80010
+; GFX8-NEXT:    s_lshr_b32 s2, s2, 24
+; GFX8-NEXT:    v_mov_b32_e32 v3, s4
+; GFX8-NEXT:    s_sext_i32_i8 s3, s1
+; GFX8-NEXT:    s_bfe_i32 s7, s1, 0x80010
+; GFX8-NEXT:    s_lshr_b32 s1, s1, 24
+; GFX8-NEXT:    s_and_b32 s4, s0, s5
+; GFX8-NEXT:    s_and_b32 s5, s0, s6
+; GFX8-NEXT:    s_bfe_i32 s1, s1, 0x80000
+; GFX8-NEXT:    s_bfe_i32 s2, s2, 0x80000
+; GFX8-NEXT:    v_mov_b32_e32 v5, s5
+; GFX8-NEXT:    s_and_b32 s1, s0, s1
+; GFX8-NEXT:    v_mov_b32_e32 v4, s8
+; GFX8-NEXT:    s_and_b32 s0, s0, s2
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    v_mad_i32_i24 v2, s3, v3, v2
+; GFX8-NEXT:    v_mad_u32_u24 v2, s4, v5, v2
+; GFX8-NEXT:    v_mad_i32_i24 v2, s7, v4, v2
+; GFX8-NEXT:    v_mov_b32_e32 v3, s0
+; GFX8-NEXT:    v_mad_u32_u24 v2, s1, v3, v2
+; GFX8-NEXT:    flat_store_short v[0:1], v2
+; GFX8-NEXT:    s_endpgm
+;
+; GFX9-NODL-LABEL: idot4_acc16:
+; GFX9-NODL:       ; %bb.0: ; %entry
+; GFX9-NODL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX9-NODL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NODL-NEXT:    global_load_ushort v2, v[0:1], off
+; GFX9-NODL-NEXT:    s_load_dword s1, s[4:5], 0x0
+; GFX9-NODL-NEXT:    s_load_dword s2, s[6:7], 0x0
+; GFX9-NODL-NEXT:    s_mov_b32 s0, 0xffff
+; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NODL-NEXT:    s_lshr_b32 s5, s1, 8
+; GFX9-NODL-NEXT:    s_lshr_b32 s6, s2, 8
+; GFX9-NODL-NEXT:    s_sext_i32_i8 s4, s2
+; GFX9-NODL-NEXT:    s_bfe_i32 s5, s5, 0x80000
+; GFX9-NODL-NEXT:    s_bfe_i32 s6, s6, 0x80000
+; GFX9-NODL-NEXT:    s_bfe_i32 s8, s2, 0x80010
+; GFX9-NODL-NEXT:    s_lshr_b32 s2, s2, 24
+; GFX9-NODL-NEXT:    v_mov_b32_e32 v3, s4
+; GFX9-NODL-NEXT:    s_sext_i32_i8 s3, s1
+; GFX9-NODL-NEXT:    s_bfe_i32 s7, s1, 0x80010
+; GFX9-NODL-NEXT:    s_lshr_b32 s1, s1, 24
+; GFX9-NODL-NEXT:    s_and_b32 s4, s0, s5
+; GFX9-NODL-NEXT:    s_and_b32 s5, s0, s6
+; GFX9-NODL-NEXT:    s_bfe_i32 s1, s1, 0x80000
+; GFX9-NODL-NEXT:    s_bfe_i32 s2, s2, 0x80000
+; GFX9-NODL-NEXT:    v_mov_b32_e32 v5, s5
+; GFX9-NODL-NEXT:    s_and_b32 s1, s0, s1
+; GFX9-NODL-NEXT:    v_mov_b32_e32 v4, s8
+; GFX9-NODL-NEXT:    s_and_b32 s0, s0, s2
+; GFX9-NODL-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NODL-NEXT:    v_mad_i32_i24 v2, s3, v3, v2
+; GFX9-NODL-NEXT:    v_mad_u32_u24 v2, s4, v5, v2
+; GFX9-NODL-NEXT:    v_mad_i32_i24 v2, s7, v4, v2
+; GFX9-NODL-NEXT:    v_mov_b32_e32 v3, s0
+; GFX9-NODL-NEXT:    v_mad_u32_u24 v2, s1, v3, v2
+; GFX9-NODL-NEXT:    global_store_short v[0:1], v2, off
+; GFX9-NODL-NEXT:    s_endpgm
+;
+; GFX9-DL-LABEL: idot4_acc16:
+; GFX9-DL:       ; %bb.0: ; %entry
+; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-DL-NEXT:    global_load_ushort v2, v[0:1], off
+; GFX9-DL-NEXT:    s_load_dword s1, s[4:5], 0x0
+; GFX9-DL-NEXT:    s_load_dword s2, s[6:7], 0x0
+; GFX9-DL-NEXT:    s_mov_b32 s0, 0xffff
+; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-DL-NEXT:    s_lshr_b32 s5, s1, 8
+; GFX9-DL-NEXT:    s_lshr_b32 s6, s2, 8
+; GFX9-DL-NEXT:    s_sext_i32_i8 s4, s2
+; GFX9-DL-NEXT:    s_bfe_i32 s5, s5, 0x80000
+; GFX9-DL-NEXT:    s_bfe_i32 s6, s6, 0x80000
+; GFX9-DL-NEXT:    s_bfe_i32 s8, s2, 0x80010
+; GFX9-DL-NEXT:    s_lshr_b32 s2, s2, 24
+; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s4
+; GFX9-DL-NEXT:    s_sext_i32_i8 s3, s1
+; GFX9-DL-NEXT:    s_bfe_i32 s7, s1, 0x80010
+; GFX9-DL-NEXT:    s_lshr_b32 s1, s1, 24
+; GFX9-DL-NEXT:    s_and_b32 s4, s0, s5
+; GFX9-DL-NEXT:    s_and_b32 s5, s0, s6
+; GFX9-DL-NEXT:    s_bfe_i32 s1, s1, 0x80000
+; GFX9-DL-NEXT:    s_bfe_i32 s2, s2, 0x80000
+; GFX9-DL-NEXT:    v_mov_b32_e32 v5, s5
+; GFX9-DL-NEXT:    s_and_b32 s1, s0, s1
+; GFX9-DL-NEXT:    v_mov_b32_e32 v4, s8
+; GFX9-DL-NEXT:    s_and_b32 s0, s0, s2
+; GFX9-DL-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s3, v3, v2
+; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s4, v5, v2
+; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s7, v4, v2
+; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s0
+; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s1, v3, v2
+; GFX9-DL-NEXT:    global_store_short v[0:1], v2, off
+; GFX9-DL-NEXT:    s_endpgm
+                                       <4 x i8> addrspace(1)* %src2,
+                                       i16 addrspace(1)* nocapture %dst) {
+entry:
+  %vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %src1
+  %vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %src2
+
+  %v1e0 = extractelement <4 x i8> %vec1, i64 0
+  %cv1e0 = sext i8 %v1e0 to i16
+  %v2e0 = extractelement <4 x i8> %vec2, i64 0
+  %cv2e0 = sext i8 %v2e0 to i16
+  %mul1 = mul nsw i16 %cv1e0, %cv2e0
+
+  %v1e1 = extractelement <4 x i8> %vec1, i64 1
+  %cv1e1 = sext i8 %v1e1 to i16
+  %v2e1 = extractelement <4 x i8> %vec2, i64 1
+  %cv2e1 = sext i8 %v2e1 to i16
+  %mul2 = mul nsw i16 %cv1e1, %cv2e1
+
+  %v1e2 = extractelement <4 x i8> %vec1, i64 2
+  %cv1e2 = sext i8 %v1e2 to i16
+  %v2e2 = extractelement <4 x i8> %vec2, i64 2
+  %cv2e2 = sext i8 %v2e2 to i16
+  %mul3 = mul nsw i16 %cv1e2, %cv2e2
+
+  %v1e3 = extractelement <4 x i8> %vec1, i64 3
+  %cv1e3 = sext i8 %v1e3 to i16
+  %v2e3 = extractelement <4 x i8> %vec2, i64 3
+  %cv2e3 = sext i8 %v2e3 to i16
+  %mul4 = mul nsw i16 %cv1e3, %cv2e3
+
+  %acc = load i16, i16 addrspace(1)* %dst, align 2
+  %add1 = add i16 %mul1, %acc
+  %add2 = add i16 %add1, %mul2
+  %add3 = add i16 %add2, %mul3
+  %add4 = add i16 %add3, %mul4
+  store i16 %add4, i16 addrspace(1)* %dst, align 2
+  ret void
+}
+
+define amdgpu_kernel void @idot4_acc8(<4 x i8> addrspace(1)* %src1,
+; GFX7-LABEL: idot4_acc8:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
+; GFX7-NEXT:    s_mov_b32 s3, 0xf000
+; GFX7-NEXT:    s_mov_b32 s2, -1
+; GFX7-NEXT:    s_movk_i32 s8, 0xff
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
+; GFX7-NEXT:    buffer_load_ubyte v0, off, s[0:3], 0
+; GFX7-NEXT:    s_load_dword s5, s[6:7], 0x0
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_and_b32 s7, s4, s8
+; GFX7-NEXT:    s_bfe_u32 s9, s4, 0x80008
+; GFX7-NEXT:    s_and_b32 s6, s5, s8
+; GFX7-NEXT:    s_bfe_u32 s8, s5, 0x80008
+; GFX7-NEXT:    v_mov_b32_e32 v1, s6
+; GFX7-NEXT:    s_bfe_u32 s10, s5, 0x80010
+; GFX7-NEXT:    v_mov_b32_e32 v2, s8
+; GFX7-NEXT:    s_bfe_u32 s11, s4, 0x80010
+; GFX7-NEXT:    s_lshr_b32 s5, s5, 24
+; GFX7-NEXT:    v_mov_b32_e32 v3, s10
+; GFX7-NEXT:    s_lshr_b32 s4, s4, 24
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    v_mad_u32_u24 v0, s7, v1, v0
+; GFX7-NEXT:    v_mad_u32_u24 v0, s9, v2, v0
+; GFX7-NEXT:    v_mad_u32_u24 v0, s11, v3, v0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mad_u32_u24 v0, s4, v1, v0
+; GFX7-NEXT:    buffer_store_byte v0, off, s[0:3], 0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX8-LABEL: idot4_acc8:
+; GFX8:       ; %bb.0: ; %entry
+; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX8-NEXT:    s_movk_i32 s2, 0xff
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v0, s0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-NEXT:    flat_load_ubyte v2, v[0:1]
+; GFX8-NEXT:    s_load_dword s0, s[4:5], 0x0
+; GFX8-NEXT:    s_load_dword s1, s[6:7], 0x0
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    s_bfe_u32 s4, s0, 0x80008
+; GFX8-NEXT:    s_and_b32 s3, s1, s2
+; GFX8-NEXT:    s_bfe_u32 s5, s1, 0x80008
+; GFX8-NEXT:    s_and_b32 s2, s0, s2
+; GFX8-NEXT:    v_mov_b32_e32 v3, s3
+; GFX8-NEXT:    s_bfe_u32 s6, s1, 0x80010
+; GFX8-NEXT:    v_mov_b32_e32 v4, s5
+; GFX8-NEXT:    s_bfe_u32 s7, s0, 0x80010
+; GFX8-NEXT:    s_lshr_b32 s1, s1, 24
+; GFX8-NEXT:    v_mov_b32_e32 v5, s6
+; GFX8-NEXT:    s_lshr_b32 s0, s0, 24
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    v_mad_u32_u24 v2, s2, v3, v2
+; GFX8-NEXT:    v_mad_u32_u24 v2, s4, v4, v2
+; GFX8-NEXT:    v_mad_u32_u24 v2, s7, v5, v2
+; GFX8-NEXT:    v_mov_b32_e32 v3, s1
+; GFX8-NEXT:    v_mad_u32_u24 v2, s0, v3, v2
+; GFX8-NEXT:    flat_store_byte v[0:1], v2
+; GFX8-NEXT:    s_endpgm
+;
+; GFX9-NODL-LABEL: idot4_acc8:
+; GFX9-NODL:       ; %bb.0: ; %entry
+; GFX9-NODL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX9-NODL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX9-NODL-NEXT:    s_movk_i32 s2, 0xff
+; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NODL-NEXT:    global_load_ubyte v2, v[0:1], off
+; GFX9-NODL-NEXT:    s_load_dword s0, s[4:5], 0x0
+; GFX9-NODL-NEXT:    s_load_dword s1, s[6:7], 0x0
+; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NODL-NEXT:    s_bfe_u32 s4, s0, 0x80008
+; GFX9-NODL-NEXT:    s_and_b32 s3, s1, s2
+; GFX9-NODL-NEXT:    s_bfe_u32 s5, s1, 0x80008
+; GFX9-NODL-NEXT:    s_and_b32 s2, s0, s2
+; GFX9-NODL-NEXT:    v_mov_b32_e32 v3, s3
+; GFX9-NODL-NEXT:    s_bfe_u32 s6, s1, 0x80010
+; GFX9-NODL-NEXT:    v_mov_b32_e32 v4, s5
+; GFX9-NODL-NEXT:    s_bfe_u32 s7, s0, 0x80010
+; GFX9-NODL-NEXT:    s_lshr_b32 s1, s1, 24
+; GFX9-NODL-NEXT:    v_mov_b32_e32 v5, s6
+; GFX9-NODL-NEXT:    s_lshr_b32 s0, s0, 24
+; GFX9-NODL-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NODL-NEXT:    v_mad_u32_u24 v2, s2, v3, v2
+; GFX9-NODL-NEXT:    v_mad_u32_u24 v2, s4, v4, v2
+; GFX9-NODL-NEXT:    v_mad_u32_u24 v2, s7, v5, v2
+; GFX9-NODL-NEXT:    v_mov_b32_e32 v3, s1
+; GFX9-NODL-NEXT:    v_mad_u32_u24 v2, s0, v3, v2
+; GFX9-NODL-NEXT:    global_store_byte v[0:1], v2, off
+; GFX9-NODL-NEXT:    s_endpgm
+;
+; GFX9-DL-LABEL: idot4_acc8:
+; GFX9-DL:       ; %bb.0: ; %entry
+; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX9-DL-NEXT:    s_load_dword s3, s[6:7], 0x0
+; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-DL-NEXT:    global_load_ubyte v2, v[0:1], off
+; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s3
+; GFX9-DL-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-DL-NEXT:    v_dot4_u32_u8 v2, s2, v3, v2
+; GFX9-DL-NEXT:    global_store_byte v[0:1], v2, off
+; GFX9-DL-NEXT:    s_endpgm
+                                      <4 x i8> addrspace(1)* %src2,
+                                      i8 addrspace(1)* nocapture %dst) {
+entry:
+  %vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %src1
+  %vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %src2
+
+  %v1e0 = extractelement <4 x i8> %vec1, i64 0
+  %v2e0 = extractelement <4 x i8> %vec2, i64 0
+  %mul1 = mul i8 %v1e0, %v2e0
+
+  %v1e1 = extractelement <4 x i8> %vec1, i64 1
+  %v2e1 = extractelement <4 x i8> %vec2, i64 1
+  %mul2 = mul i8 %v1e1, %v2e1
+
+  %v1e2 = extractelement <4 x i8> %vec1, i64 2
+  %v2e2 = extractelement <4 x i8> %vec2, i64 2
+  %mul3 = mul i8 %v1e2, %v2e2
+
+  %v1e3 = extractelement <4 x i8> %vec1, i64 3
+  %v2e3 = extractelement <4 x i8> %vec2, i64 3
+  %mul4 = mul i8 %v1e3, %v2e3
+
+  %acc = load i8, i8 addrspace(1)* %dst, align 2
+  %add1 = add i8 %mul1, %acc
+  %add2 = add i8 %add1, %mul2
+  %add3 = add i8 %add2, %mul3
+  %add4 = add nsw i8 %add3, %mul4
+  store i8 %add4, i8 addrspace(1)* %dst, align 2
+  ret void
+}
+
+define amdgpu_kernel void @idot4_multiuse_mul1(<4 x i8> addrspace(1)* %src1,
+; GFX7-LABEL: idot4_multiuse_mul1:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
+; GFX7-NEXT:    s_mov_b32 s3, 0xf000
+; GFX7-NEXT:    s_mov_b32 s2, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s5, s[6:7], 0x0
+; GFX7-NEXT:    s_load_dword s6, s[0:1], 0x0
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_sext_i32_i8 s7, s4
+; GFX7-NEXT:    s_sext_i32_i8 s8, s5
+; GFX7-NEXT:    s_bfe_i32 s10, s5, 0x80008
+; GFX7-NEXT:    v_mov_b32_e32 v0, s8
+; GFX7-NEXT:    v_mov_b32_e32 v1, s6
+; GFX7-NEXT:    s_bfe_i32 s9, s4, 0x80008
+; GFX7-NEXT:    v_mad_i32_i24 v1, s7, v0, v1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s10
+; GFX7-NEXT:    s_bfe_i32 s12, s5, 0x80010
+; GFX7-NEXT:    v_mad_i32_i24 v1, s9, v2, v1
+; GFX7-NEXT:    s_bfe_i32 s11, s4, 0x80010
+; GFX7-NEXT:    v_mad_i32_i24 v0, s7, v0, v1
+; GFX7-NEXT:    v_mov_b32_e32 v1, s12
+; GFX7-NEXT:    s_ashr_i32 s5, s5, 24
+; GFX7-NEXT:    v_mad_i32_i24 v0, s11, v1, v0
+; GFX7-NEXT:    s_ashr_i32 s4, s4, 24
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mad_i32_i24 v0, s4, v1, v0
+; GFX7-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX8-LABEL: idot4_multiuse_mul1:
+; GFX8:       ; %bb.0: ; %entry
+; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX8-NEXT:    s_load_dword s3, s[6:7], 0x0
+; GFX8-NEXT:    s_load_dword s4, s[0:1], 0x0
+; GFX8-NEXT:    v_mov_b32_e32 v0, s0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    s_sext_i32_i8 s0, s2
+; GFX8-NEXT:    s_sext_i32_i8 s1, s3
+; GFX8-NEXT:    s_bfe_i32 s6, s3, 0x80008
+; GFX8-NEXT:    v_mov_b32_e32 v2, s1
+; GFX8-NEXT:    v_mov_b32_e32 v3, s4
+; GFX8-NEXT:    s_bfe_i32 s5, s2, 0x80008
+; GFX8-NEXT:    v_mad_i32_i24 v3, s0, v2, v3
+; GFX8-NEXT:    v_mov_b32_e32 v4, s6
+; GFX8-NEXT:    s_bfe_i32 s8, s3, 0x80010
+; GFX8-NEXT:    v_mad_i32_i24 v3, s5, v4, v3
+; GFX8-NEXT:    s_bfe_i32 s7, s2, 0x80010
+; GFX8-NEXT:    v_mad_i32_i24 v2, s0, v2, v3
+; GFX8-NEXT:    v_mov_b32_e32 v3, s8
+; GFX8-NEXT:    s_ashr_i32 s3, s3, 24
+; GFX8-NEXT:    v_mad_i32_i24 v2, s7, v3, v2
+; GFX8-NEXT:    s_ashr_i32 s2, s2, 24
+; GFX8-NEXT:    v_mov_b32_e32 v3, s3
+; GFX8-NEXT:    v_mad_i32_i24 v2, s2, v3, v2
+; GFX8-NEXT:    flat_store_dword v[0:1], v2
+; GFX8-NEXT:    s_endpgm
+;
+; GFX9-NODL-LABEL: idot4_multiuse_mul1:
+; GFX9-NODL:       ; %bb.0: ; %entry
+; GFX9-NODL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX9-NODL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NODL-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX9-NODL-NEXT:    s_load_dword s3, s[6:7], 0x0
+; GFX9-NODL-NEXT:    s_load_dword s4, s[0:1], 0x0
+; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NODL-NEXT:    s_sext_i32_i8 s0, s2
+; GFX9-NODL-NEXT:    s_sext_i32_i8 s1, s3
+; GFX9-NODL-NEXT:    s_bfe_i32 s6, s3, 0x80008
+; GFX9-NODL-NEXT:    v_mov_b32_e32 v2, s1
+; GFX9-NODL-NEXT:    v_mov_b32_e32 v3, s4
+; GFX9-NODL-NEXT:    s_bfe_i32 s5, s2, 0x80008
+; GFX9-NODL-NEXT:    v_mad_i32_i24 v3, s0, v2, v3
+; GFX9-NODL-NEXT:    v_mov_b32_e32 v4, s6
+; GFX9-NODL-NEXT:    s_bfe_i32 s8, s3, 0x80010
+; GFX9-NODL-NEXT:    v_mad_i32_i24 v3, s5, v4, v3
+; GFX9-NODL-NEXT:    s_bfe_i32 s7, s2, 0x80010
+; GFX9-NODL-NEXT:    v_mad_i32_i24 v2, s0, v2, v3
+; GFX9-NODL-NEXT:    v_mov_b32_e32 v3, s8
+; GFX9-NODL-NEXT:    s_ashr_i32 s3, s3, 24
+; GFX9-NODL-NEXT:    v_mad_i32_i24 v2, s7, v3, v2
+; GFX9-NODL-NEXT:    s_ashr_i32 s2, s2, 24
+; GFX9-NODL-NEXT:    v_mov_b32_e32 v3, s3
+; GFX9-NODL-NEXT:    v_mad_i32_i24 v2, s2, v3, v2
+; GFX9-NODL-NEXT:    global_store_dword v[0:1], v2, off
+; GFX9-NODL-NEXT:    s_endpgm
+;
+; GFX9-DL-LABEL: idot4_multiuse_mul1:
+; GFX9-DL:       ; %bb.0: ; %entry
+; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX9-DL-NEXT:    s_load_dword s3, s[6:7], 0x0
+; GFX9-DL-NEXT:    s_load_dword s4, s[0:1], 0x0
+; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-DL-NEXT:    s_sext_i32_i8 s0, s2
+; GFX9-DL-NEXT:    s_sext_i32_i8 s1, s3
+; GFX9-DL-NEXT:    s_bfe_i32 s6, s3, 0x80008
+; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s1
+; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s4
+; GFX9-DL-NEXT:    s_bfe_i32 s5, s2, 0x80008
+; GFX9-DL-NEXT:    v_mad_i32_i24 v3, s0, v2, v3
+; GFX9-DL-NEXT:    v_mov_b32_e32 v4, s6
+; GFX9-DL-NEXT:    s_bfe_i32 s8, s3, 0x80010
+; GFX9-DL-NEXT:    v_mad_i32_i24 v3, s5, v4, v3
+; GFX9-DL-NEXT:    s_bfe_i32 s7, s2, 0x80010
+; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s0, v2, v3
+; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s8
+; GFX9-DL-NEXT:    s_ashr_i32 s3, s3, 24
+; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s7, v3, v2
+; GFX9-DL-NEXT:    s_ashr_i32 s2, s2, 24
+; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s3
+; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s2, v3, v2
+; GFX9-DL-NEXT:    global_store_dword v[0:1], v2, off
+; GFX9-DL-NEXT:    s_endpgm
+                                               <4 x i8> addrspace(1)* %src2,
+                                               i32 addrspace(1)* nocapture %dst) {
+entry:
+  %vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %src1
+  %vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %src2
+
+  %v1e0 = extractelement <4 x i8> %vec1, i64 0
+  %cv1e0 = sext i8 %v1e0 to i32
+  %v2e0 = extractelement <4 x i8> %vec2, i64 0
+  %cv2e0 = sext i8 %v2e0 to i32
+  %mul1 = mul nuw nsw i32 %cv1e0, %cv2e0
+
+  %v1e1 = extractelement <4 x i8> %vec1, i64 1
+  %cv1e1 = sext i8 %v1e1 to i32
+  %v2e1 = extractelement <4 x i8> %vec2, i64 1
+  %cv2e1 = sext i8 %v2e1 to i32
+  %mul2 = mul nuw nsw i32 %cv1e1, %cv2e1
+
+  %v1e2 = extractelement <4 x i8> %vec1, i64 2
+  %cv1e2 = sext i8 %v1e2 to i32
+  %v2e2 = extractelement <4 x i8> %vec2, i64 2
+  %cv2e2 = sext i8 %v2e2 to i32
+  %mul3 = mul nuw nsw i32 %cv1e2, %cv2e2
+
+  %v1e3 = extractelement <4 x i8> %vec1, i64 3
+  %cv1e3 = sext i8 %v1e3 to i32
+  %v2e3 = extractelement <4 x i8> %vec2, i64 3
+  %cv2e3 = sext i8 %v2e3 to i32
+  %mul4 = mul nuw nsw i32 %cv1e3, %cv2e3
+
+  %acc = load i32, i32 addrspace(1)* %dst, align 4
+  %add = add i32 %mul1, %acc
+  %add1 = add i32 %mul2, %add
+  %add2 = add i32 %add1, %mul1
+  %add3 = add i32 %add2, %mul3
+  %add4 = add i32 %add3, %mul4
+
+  store i32 %add4, i32 addrspace(1)* %dst, align 4
+  ret void
+}
+
+; TODO: Support this pattern.
+define amdgpu_kernel void @idot4_acc32_vecMul(<4 x i8> addrspace(1)* %src1,
+; GFX7-LABEL: idot4_acc32_vecMul:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
+; GFX7-NEXT:    s_mov_b32 s3, 0xf000
+; GFX7-NEXT:    s_mov_b32 s2, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
+; GFX7-NEXT:    s_load_dword s5, s[6:7], 0x0
+; GFX7-NEXT:    s_load_dword s6, s[0:1], 0x0
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_ashr_i32 s7, s4, 24
+; GFX7-NEXT:    s_ashr_i32 s10, s5, 24
+; GFX7-NEXT:    s_bfe_i32 s11, s5, 0x80010
+; GFX7-NEXT:    s_bfe_i32 s12, s5, 0x80008
+; GFX7-NEXT:    s_sext_i32_i8 s5, s5
+; GFX7-NEXT:    s_bfe_i32 s8, s4, 0x80010
+; GFX7-NEXT:    s_bfe_i32 s9, s4, 0x80008
+; GFX7-NEXT:    s_sext_i32_i8 s4, s4
+; GFX7-NEXT:    v_mov_b32_e32 v0, s5
+; GFX7-NEXT:    v_mov_b32_e32 v1, s6
+; GFX7-NEXT:    v_mad_i32_i24 v0, s4, v0, v1
+; GFX7-NEXT:    v_mov_b32_e32 v1, s12
+; GFX7-NEXT:    v_mad_i32_i24 v0, s9, v1, v0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s11
+; GFX7-NEXT:    v_mad_i32_i24 v0, s8, v1, v0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s10
+; GFX7-NEXT:    v_mad_i32_i24 v0, s7, v1, v0
+; GFX7-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX8-LABEL: idot4_acc32_vecMul:
+; GFX8:       ; %bb.0: ; %entry
+; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX8-NEXT:    s_load_dword s3, s[6:7], 0x0
+; GFX8-NEXT:    s_load_dword s4, s[0:1], 0x0
+; GFX8-NEXT:    v_mov_b32_e32 v0, s0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    v_lshrrev_b16_e64 v2, 8, s2
+; GFX8-NEXT:    v_lshrrev_b16_e64 v3, 8, s3
+; GFX8-NEXT:    s_ashr_i32 s5, s3, 24
+; GFX8-NEXT:    s_bfe_i32 s6, s3, 0x80010
+; GFX8-NEXT:    s_sext_i32_i8 s3, s3
+; GFX8-NEXT:    s_ashr_i32 s0, s2, 24
+; GFX8-NEXT:    s_bfe_i32 s1, s2, 0x80010
+; GFX8-NEXT:    s_sext_i32_i8 s2, s2
+; GFX8-NEXT:    v_mov_b32_e32 v4, s3
+; GFX8-NEXT:    v_mov_b32_e32 v5, s4
+; GFX8-NEXT:    v_bfe_i32 v2, v2, 0, 8
+; GFX8-NEXT:    v_bfe_i32 v3, v3, 0, 8
+; GFX8-NEXT:    v_mad_i32_i24 v4, s2, v4, v5
+; GFX8-NEXT:    v_mad_i32_i24 v2, v2, v3, v4
+; GFX8-NEXT:    v_mov_b32_e32 v3, s6
+; GFX8-NEXT:    v_mad_i32_i24 v2, s1, v3, v2
+; GFX8-NEXT:    v_mov_b32_e32 v3, s5
+; GFX8-NEXT:    v_mad_i32_i24 v2, s0, v3, v2
+; GFX8-NEXT:    flat_store_dword v[0:1], v2
+; GFX8-NEXT:    s_endpgm
+;
+; GFX9-NODL-LABEL: idot4_acc32_vecMul:
+; GFX9-NODL:       ; %bb.0: ; %entry
+; GFX9-NODL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX9-NODL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NODL-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX9-NODL-NEXT:    s_load_dword s3, s[6:7], 0x0
+; GFX9-NODL-NEXT:    s_load_dword s4, s[0:1], 0x0
+; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NODL-NEXT:    v_lshrrev_b16_e64 v2, 8, s2
+; GFX9-NODL-NEXT:    v_lshrrev_b16_e64 v3, 8, s3
+; GFX9-NODL-NEXT:    s_ashr_i32 s5, s3, 24
+; GFX9-NODL-NEXT:    s_bfe_i32 s6, s3, 0x80010
+; GFX9-NODL-NEXT:    s_sext_i32_i8 s3, s3
+; GFX9-NODL-NEXT:    s_ashr_i32 s0, s2, 24
+; GFX9-NODL-NEXT:    s_bfe_i32 s1, s2, 0x80010
+; GFX9-NODL-NEXT:    s_sext_i32_i8 s2, s2
+; GFX9-NODL-NEXT:    v_mov_b32_e32 v4, s3
+; GFX9-NODL-NEXT:    v_mov_b32_e32 v5, s4
+; GFX9-NODL-NEXT:    v_bfe_i32 v2, v2, 0, 8
+; GFX9-NODL-NEXT:    v_bfe_i32 v3, v3, 0, 8
+; GFX9-NODL-NEXT:    v_mad_i32_i24 v4, s2, v4, v5
+; GFX9-NODL-NEXT:    v_mad_i32_i24 v2, v2, v3, v4
+; GFX9-NODL-NEXT:    v_mov_b32_e32 v3, s6
+; GFX9-NODL-NEXT:    v_mad_i32_i24 v2, s1, v3, v2
+; GFX9-NODL-NEXT:    v_mov_b32_e32 v3, s5
+; GFX9-NODL-NEXT:    v_mad_i32_i24 v2, s0, v3, v2
+; GFX9-NODL-NEXT:    global_store_dword v[0:1], v2, off
+; GFX9-NODL-NEXT:    s_endpgm
+;
+; GFX9-DL-LABEL: idot4_acc32_vecMul:
+; GFX9-DL:       ; %bb.0: ; %entry
+; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX9-DL-NEXT:    s_load_dword s3, s[6:7], 0x0
+; GFX9-DL-NEXT:    s_load_dword s4, s[0:1], 0x0
+; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-DL-NEXT:    v_lshrrev_b16_e64 v2, 8, s2
+; GFX9-DL-NEXT:    v_lshrrev_b16_e64 v3, 8, s3
+; GFX9-DL-NEXT:    s_ashr_i32 s5, s3, 24
+; GFX9-DL-NEXT:    s_bfe_i32 s6, s3, 0x80010
+; GFX9-DL-NEXT:    s_sext_i32_i8 s3, s3
+; GFX9-DL-NEXT:    s_ashr_i32 s0, s2, 24
+; GFX9-DL-NEXT:    s_bfe_i32 s1, s2, 0x80010
+; GFX9-DL-NEXT:    s_sext_i32_i8 s2, s2
+; GFX9-DL-NEXT:    v_mov_b32_e32 v4, s3
+; GFX9-DL-NEXT:    v_mov_b32_e32 v5, s4
+; GFX9-DL-NEXT:    v_bfe_i32 v2, v2, 0, 8
+; GFX9-DL-NEXT:    v_bfe_i32 v3, v3, 0, 8
+; GFX9-DL-NEXT:    v_mad_i32_i24 v4, s2, v4, v5
+; GFX9-DL-NEXT:    v_mad_i32_i24 v2, v2, v3, v4
+; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s6
+; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s1, v3, v2
+; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s5
+; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s0, v3, v2
+; GFX9-DL-NEXT:    global_store_dword v[0:1], v2, off
+; GFX9-DL-NEXT:    s_endpgm
+                                              <4 x i8> addrspace(1)* %src2,
+                                              i32 addrspace(1)* nocapture %dst) {
+entry:
+  %vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %src1
+  %vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %src2
+
+  %cvec1 = sext <4 x i8> %vec1 to <4 x i32>
+  %cvec2 = sext <4 x i8> %vec2 to <4 x i32>
+
+  %mul = mul <4 x i32> %cvec1, %cvec2
+  %mul0 = extractelement <4 x i32> %mul, i64 0
+  %mul1 = extractelement <4 x i32> %mul, i64 1
+  %mul2 = extractelement <4 x i32> %mul, i64 2
+  %mul3 = extractelement <4 x i32> %mul, i64 3
+
+  %acc = load i32, i32 addrspace(1)* %dst, align 4
+  %add1 = add i32 %mul0, %acc
+  %add2 = add i32 %add1, %mul1
+  %add3 = add i32 %add2, %mul2
+  %add4 = add i32 %add3, %mul3
+
+  store i32 %add4, i32 addrspace(1)* %dst, align 4
+  ret void
+}
+
+define amdgpu_kernel void @idot4_acc16_vecMul(<4 x i8> addrspace(1)* %src1,
+; GFX7-LABEL: idot4_acc16_vecMul:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
+; GFX7-NEXT:    s_mov_b32 s3, 0xf000
+; GFX7-NEXT:    s_mov_b32 s2, -1
+; GFX7-NEXT:    s_mov_b32 s8, 0xffff
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
+; GFX7-NEXT:    buffer_load_ushort v0, off, s[0:3], 0
+; GFX7-NEXT:    s_load_dword s5, s[6:7], 0x0
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_sext_i32_i8 s6, s4
+; GFX7-NEXT:    s_bfe_i32 s7, s4, 0x80008
+; GFX7-NEXT:    s_sext_i32_i8 s10, s5
+; GFX7-NEXT:    s_bfe_i32 s11, s5, 0x80008
+; GFX7-NEXT:    s_bfe_i32 s12, s5, 0x80010
+; GFX7-NEXT:    s_ashr_i32 s5, s5, 24
+; GFX7-NEXT:    v_mov_b32_e32 v3, s11
+; GFX7-NEXT:    v_mov_b32_e32 v4, s10
+; GFX7-NEXT:    s_bfe_i32 s9, s4, 0x80010
+; GFX7-NEXT:    v_mov_b32_e32 v2, s12
+; GFX7-NEXT:    s_ashr_i32 s4, s4, 24
+; GFX7-NEXT:    v_mov_b32_e32 v1, s5
+; GFX7-NEXT:    v_mul_i32_i24_e32 v1, s4, v1
+; GFX7-NEXT:    v_mul_i32_i24_e32 v2, s9, v2
+; GFX7-NEXT:    v_mul_i32_i24_e32 v3, s7, v3
+; GFX7-NEXT:    v_mul_i32_i24_e32 v4, s6, v4
+; GFX7-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX7-NEXT:    v_and_b32_e32 v2, s8, v2
+; GFX7-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GFX7-NEXT:    v_and_b32_e32 v4, s8, v4
+; GFX7-NEXT:    v_or_b32_e32 v1, v2, v1
+; GFX7-NEXT:    v_or_b32_e32 v2, v4, v3
+; GFX7-NEXT:    v_lshrrev_b32_e32 v3, 16, v2
+; GFX7-NEXT:    v_lshrrev_b32_e32 v4, 16, v1
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
+; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v3, v0
+; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
+; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v4, v0
+; GFX7-NEXT:    buffer_store_short v0, off, s[0:3], 0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX8-LABEL: idot4_acc16_vecMul:
+; GFX8:       ; %bb.0: ; %entry
+; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v0, s0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-NEXT:    flat_load_ushort v2, v[0:1]
+; GFX8-NEXT:    s_load_dword s1, s[4:5], 0x0
+; GFX8-NEXT:    s_load_dword s2, s[6:7], 0x0
+; GFX8-NEXT:    s_mov_b32 s0, 0xffff
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    s_lshr_b32 s3, s1, 16
+; GFX8-NEXT:    s_bfe_i32 s6, s2, 0x80000
+; GFX8-NEXT:    s_lshr_b32 s4, s2, 16
+; GFX8-NEXT:    s_bfe_i32 s5, s1, 0x80000
+; GFX8-NEXT:    v_ashrrev_i16_e64 v4, 8, s1
+; GFX8-NEXT:    s_bfe_i32 s1, s3, 0x80000
+; GFX8-NEXT:    v_ashrrev_i16_e64 v6, 8, s3
+; GFX8-NEXT:    s_and_b32 s3, s0, s6
+; GFX8-NEXT:    v_ashrrev_i16_e64 v3, 8, s2
+; GFX8-NEXT:    s_bfe_i32 s2, s4, 0x80000
+; GFX8-NEXT:    v_ashrrev_i16_e64 v5, 8, s4
+; GFX8-NEXT:    s_and_b32 s4, s0, s5
+; GFX8-NEXT:    v_mov_b32_e32 v7, s3
+; GFX8-NEXT:    s_and_b32 s2, s0, s2
+; GFX8-NEXT:    s_and_b32 s0, s0, s1
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    v_mad_u32_u24 v2, s4, v7, v2
+; GFX8-NEXT:    v_mad_u32_u24 v2, v4, v3, v2
+; GFX8-NEXT:    v_mov_b32_e32 v3, s2
+; GFX8-NEXT:    v_mad_u32_u24 v2, s0, v3, v2
+; GFX8-NEXT:    v_mad_u32_u24 v2, v6, v5, v2
+; GFX8-NEXT:    flat_store_short v[0:1], v2
+; GFX8-NEXT:    s_endpgm
+;
+; GFX9-NODL-LABEL: idot4_acc16_vecMul:
+; GFX9-NODL:       ; %bb.0: ; %entry
+; GFX9-NODL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX9-NODL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX9-NODL-NEXT:    v_mov_b32_e32 v4, 0xffff
+; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NODL-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX9-NODL-NEXT:    s_load_dword s3, s[6:7], 0x0
+; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NODL-NEXT:    s_lshr_b32 s4, s2, 16
+; GFX9-NODL-NEXT:    s_lshr_b32 s5, s3, 16
+; GFX9-NODL-NEXT:    v_ashrrev_i16_e64 v3, 8, s5
+; GFX9-NODL-NEXT:    s_bfe_i32 s5, s5, 0x80000
+; GFX9-NODL-NEXT:    v_ashrrev_i16_e64 v2, 8, s4
+; GFX9-NODL-NEXT:    v_and_b32_e32 v5, s5, v4
+; GFX9-NODL-NEXT:    s_bfe_i32 s4, s4, 0x80000
+; GFX9-NODL-NEXT:    v_lshl_or_b32 v3, v3, 16, v5
+; GFX9-NODL-NEXT:    v_and_b32_e32 v5, s4, v4
+; GFX9-NODL-NEXT:    v_lshl_or_b32 v2, v2, 16, v5
+; GFX9-NODL-NEXT:    v_ashrrev_i16_e64 v1, 8, s3
+; GFX9-NODL-NEXT:    s_bfe_i32 s3, s3, 0x80000
+; GFX9-NODL-NEXT:    v_ashrrev_i16_e64 v0, 8, s2
+; GFX9-NODL-NEXT:    v_pk_mul_lo_u16 v2, v2, v3
+; GFX9-NODL-NEXT:    v_and_b32_e32 v3, s3, v4
+; GFX9-NODL-NEXT:    s_bfe_i32 s2, s2, 0x80000
+; GFX9-NODL-NEXT:    v_lshl_or_b32 v1, v1, 16, v3
+; GFX9-NODL-NEXT:    v_and_b32_e32 v3, s2, v4
+; GFX9-NODL-NEXT:    v_lshl_or_b32 v0, v0, 16, v3
+; GFX9-NODL-NEXT:    v_pk_mul_lo_u16 v3, v0, v1
+; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NODL-NEXT:    global_load_ushort v4, v[0:1], off
+; GFX9-NODL-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NODL-NEXT:    v_add_u32_e32 v4, v3, v4
+; GFX9-NODL-NEXT:    v_add_u32_sdwa v3, v4, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NODL-NEXT:    v_add_u32_e32 v3, v3, v2
+; GFX9-NODL-NEXT:    v_add_u32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NODL-NEXT:    global_store_short v[0:1], v2, off
+; GFX9-NODL-NEXT:    s_endpgm
+;
+; GFX9-DL-LABEL: idot4_acc16_vecMul:
+; GFX9-DL:       ; %bb.0: ; %entry
+; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX9-DL-NEXT:    v_mov_b32_e32 v4, 0xffff
+; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX9-DL-NEXT:    s_load_dword s3, s[6:7], 0x0
+; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-DL-NEXT:    s_lshr_b32 s4, s2, 16
+; GFX9-DL-NEXT:    s_lshr_b32 s5, s3, 16
+; GFX9-DL-NEXT:    v_ashrrev_i16_e64 v3, 8, s5
+; GFX9-DL-NEXT:    s_bfe_i32 s5, s5, 0x80000
+; GFX9-DL-NEXT:    v_ashrrev_i16_e64 v2, 8, s4
+; GFX9-DL-NEXT:    v_and_b32_e32 v5, s5, v4
+; GFX9-DL-NEXT:    s_bfe_i32 s4, s4, 0x80000
+; GFX9-DL-NEXT:    v_lshl_or_b32 v3, v3, 16, v5
+; GFX9-DL-NEXT:    v_and_b32_e32 v5, s4, v4
+; GFX9-DL-NEXT:    v_lshl_or_b32 v2, v2, 16, v5
+; GFX9-DL-NEXT:    v_ashrrev_i16_e64 v1, 8, s3
+; GFX9-DL-NEXT:    s_bfe_i32 s3, s3, 0x80000
+; GFX9-DL-NEXT:    v_ashrrev_i16_e64 v0, 8, s2
+; GFX9-DL-NEXT:    v_pk_mul_lo_u16 v2, v2, v3
+; GFX9-DL-NEXT:    v_and_b32_e32 v3, s3, v4
+; GFX9-DL-NEXT:    s_bfe_i32 s2, s2, 0x80000
+; GFX9-DL-NEXT:    v_lshl_or_b32 v1, v1, 16, v3
+; GFX9-DL-NEXT:    v_and_b32_e32 v3, s2, v4
+; GFX9-DL-NEXT:    v_lshl_or_b32 v0, v0, 16, v3
+; GFX9-DL-NEXT:    v_pk_mul_lo_u16 v3, v0, v1
+; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-DL-NEXT:    global_load_ushort v4, v[0:1], off
+; GFX9-DL-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-DL-NEXT:    v_add_u32_e32 v4, v3, v4
+; GFX9-DL-NEXT:    v_add_u32_sdwa v3, v4, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-DL-NEXT:    v_add_u32_e32 v3, v3, v2
+; GFX9-DL-NEXT:    v_add_u32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-DL-NEXT:    global_store_short v[0:1], v2, off
+; GFX9-DL-NEXT:    s_endpgm
+                                              <4 x i8> addrspace(1)* %src2,
+                                              i16 addrspace(1)* nocapture %dst) {
+entry:
+  %vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %src1
+  %vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %src2
+
+  %cvec1 = sext <4 x i8> %vec1 to <4 x i16>
+  %cvec2 = sext <4 x i8> %vec2 to <4 x i16>
+
+  %mul = mul <4 x i16> %cvec1, %cvec2
+  %mul0 = extractelement <4 x i16> %mul, i64 0
+  %mul1 = extractelement <4 x i16> %mul, i64 1
+  %mul2 = extractelement <4 x i16> %mul, i64 2
+  %mul3 = extractelement <4 x i16> %mul, i64 3
+
+  %acc = load i16, i16 addrspace(1)* %dst, align 4
+  %add1 = add i16 %mul0, %acc
+  %add2 = add i16 %add1, %mul1
+  %add3 = add i16 %add2, %mul2
+  %add4 = add i16 %add3, %mul3
+
+  store i16 %add4, i16 addrspace(1)* %dst, align 4
+  ret void
+}
diff --git a/test/CodeGen/AMDGPU/idot4.ll b/test/CodeGen/AMDGPU/idot4u.ll
similarity index 66%
rename from test/CodeGen/AMDGPU/idot4.ll
rename to test/CodeGen/AMDGPU/idot4u.ll
index 103245553a084815de580a06a6fced7b2d2eff34..bac1a9e3f411b4ac70d5abca98e393e978aada33 100644
--- a/test/CodeGen/AMDGPU/idot4.ll
+++ b/test/CodeGen/AMDGPU/idot4u.ll
@@ -1405,669 +1405,6 @@ entry:
   ret void
 }
 
-define amdgpu_kernel void @idot4_acc32(<4 x i8> addrspace(1)* %src1,
-; GFX7-LABEL: idot4_acc32:
-; GFX7:       ; %bb.0: ; %entry
-; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
-; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
-; GFX7-NEXT:    s_mov_b32 s3, 0xf000
-; GFX7-NEXT:    s_mov_b32 s2, -1
-; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
-; GFX7-NEXT:    s_load_dword s5, s[6:7], 0x0
-; GFX7-NEXT:    s_load_dword s6, s[0:1], 0x0
-; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    s_sext_i32_i8 s7, s4
-; GFX7-NEXT:    s_sext_i32_i8 s8, s5
-; GFX7-NEXT:    s_bfe_i32 s10, s5, 0x80008
-; GFX7-NEXT:    v_mov_b32_e32 v0, s8
-; GFX7-NEXT:    v_mov_b32_e32 v1, s6
-; GFX7-NEXT:    s_bfe_i32 s12, s5, 0x80010
-; GFX7-NEXT:    v_mad_i32_i24 v0, s7, v0, v1
-; GFX7-NEXT:    s_bfe_i32 s9, s4, 0x80008
-; GFX7-NEXT:    v_mov_b32_e32 v1, s10
-; GFX7-NEXT:    s_bfe_i32 s11, s4, 0x80010
-; GFX7-NEXT:    v_mad_i32_i24 v0, s9, v1, v0
-; GFX7-NEXT:    v_mov_b32_e32 v1, s12
-; GFX7-NEXT:    s_ashr_i32 s5, s5, 24
-; GFX7-NEXT:    v_mad_i32_i24 v0, s11, v1, v0
-; GFX7-NEXT:    s_ashr_i32 s4, s4, 24
-; GFX7-NEXT:    v_mov_b32_e32 v1, s5
-; GFX7-NEXT:    v_mad_i32_i24 v0, s4, v1, v0
-; GFX7-NEXT:    buffer_store_dword v0, off, s[0:3], 0
-; GFX7-NEXT:    s_endpgm
-;
-; GFX8-LABEL: idot4_acc32:
-; GFX8:       ; %bb.0: ; %entry
-; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
-; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
-; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    s_load_dword s2, s[4:5], 0x0
-; GFX8-NEXT:    s_load_dword s3, s[6:7], 0x0
-; GFX8-NEXT:    s_load_dword s4, s[0:1], 0x0
-; GFX8-NEXT:    v_mov_b32_e32 v0, s0
-; GFX8-NEXT:    v_mov_b32_e32 v1, s1
-; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    s_sext_i32_i8 s0, s2
-; GFX8-NEXT:    s_sext_i32_i8 s1, s3
-; GFX8-NEXT:    s_bfe_i32 s6, s3, 0x80008
-; GFX8-NEXT:    v_mov_b32_e32 v2, s1
-; GFX8-NEXT:    v_mov_b32_e32 v3, s4
-; GFX8-NEXT:    s_bfe_i32 s8, s3, 0x80010
-; GFX8-NEXT:    v_mad_i32_i24 v2, s0, v2, v3
-; GFX8-NEXT:    s_bfe_i32 s5, s2, 0x80008
-; GFX8-NEXT:    v_mov_b32_e32 v3, s6
-; GFX8-NEXT:    s_bfe_i32 s7, s2, 0x80010
-; GFX8-NEXT:    v_mad_i32_i24 v2, s5, v3, v2
-; GFX8-NEXT:    v_mov_b32_e32 v3, s8
-; GFX8-NEXT:    s_ashr_i32 s3, s3, 24
-; GFX8-NEXT:    v_mad_i32_i24 v2, s7, v3, v2
-; GFX8-NEXT:    s_ashr_i32 s2, s2, 24
-; GFX8-NEXT:    v_mov_b32_e32 v3, s3
-; GFX8-NEXT:    v_mad_i32_i24 v2, s2, v3, v2
-; GFX8-NEXT:    flat_store_dword v[0:1], v2
-; GFX8-NEXT:    s_endpgm
-;
-; GFX9-NODL-LABEL: idot4_acc32:
-; GFX9-NODL:       ; %bb.0: ; %entry
-; GFX9-NODL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
-; GFX9-NODL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
-; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NODL-NEXT:    s_load_dword s2, s[4:5], 0x0
-; GFX9-NODL-NEXT:    s_load_dword s3, s[6:7], 0x0
-; GFX9-NODL-NEXT:    s_load_dword s4, s[0:1], 0x0
-; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
-; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
-; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NODL-NEXT:    s_sext_i32_i8 s0, s2
-; GFX9-NODL-NEXT:    s_sext_i32_i8 s1, s3
-; GFX9-NODL-NEXT:    s_bfe_i32 s6, s3, 0x80008
-; GFX9-NODL-NEXT:    v_mov_b32_e32 v2, s1
-; GFX9-NODL-NEXT:    v_mov_b32_e32 v3, s4
-; GFX9-NODL-NEXT:    s_bfe_i32 s8, s3, 0x80010
-; GFX9-NODL-NEXT:    v_mad_i32_i24 v2, s0, v2, v3
-; GFX9-NODL-NEXT:    s_bfe_i32 s5, s2, 0x80008
-; GFX9-NODL-NEXT:    v_mov_b32_e32 v3, s6
-; GFX9-NODL-NEXT:    s_bfe_i32 s7, s2, 0x80010
-; GFX9-NODL-NEXT:    v_mad_i32_i24 v2, s5, v3, v2
-; GFX9-NODL-NEXT:    v_mov_b32_e32 v3, s8
-; GFX9-NODL-NEXT:    s_ashr_i32 s3, s3, 24
-; GFX9-NODL-NEXT:    v_mad_i32_i24 v2, s7, v3, v2
-; GFX9-NODL-NEXT:    s_ashr_i32 s2, s2, 24
-; GFX9-NODL-NEXT:    v_mov_b32_e32 v3, s3
-; GFX9-NODL-NEXT:    v_mad_i32_i24 v2, s2, v3, v2
-; GFX9-NODL-NEXT:    global_store_dword v[0:1], v2, off
-; GFX9-NODL-NEXT:    s_endpgm
-;
-; GFX9-DL-LABEL: idot4_acc32:
-; GFX9-DL:       ; %bb.0: ; %entry
-; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
-; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
-; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
-; GFX9-DL-NEXT:    s_load_dword s3, s[6:7], 0x0
-; GFX9-DL-NEXT:    s_load_dword s4, s[0:1], 0x0
-; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
-; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
-; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s3
-; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s4
-; GFX9-DL-NEXT:    v_dot4_i32_i8 v2, s2, v2, v3
-; GFX9-DL-NEXT:    global_store_dword v[0:1], v2, off
-; GFX9-DL-NEXT:    s_endpgm
-                                       <4 x i8> addrspace(1)* %src2,
-                                       i32 addrspace(1)* nocapture %dst) {
-entry:
-  %vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %src1
-  %vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %src2
-
-  %v1e0 = extractelement <4 x i8> %vec1, i64 0
-  %cv1e0 = sext i8 %v1e0 to i32
-  %v2e0 = extractelement <4 x i8> %vec2, i64 0
-  %cv2e0 = sext i8 %v2e0 to i32
-  %mul1 = mul nuw nsw i32 %cv1e0, %cv2e0
-
-  %v1e1 = extractelement <4 x i8> %vec1, i64 1
-  %cv1e1 = sext i8 %v1e1 to i32
-  %v2e1 = extractelement <4 x i8> %vec2, i64 1
-  %cv2e1 = sext i8 %v2e1 to i32
-  %mul2 = mul nuw nsw i32 %cv1e1, %cv2e1
-
-  %v1e2 = extractelement <4 x i8> %vec1, i64 2
-  %cv1e2 = sext i8 %v1e2 to i32
-  %v2e2 = extractelement <4 x i8> %vec2, i64 2
-  %cv2e2 = sext i8 %v2e2 to i32
-  %mul3 = mul nuw nsw i32 %cv1e2, %cv2e2
-
-  %v1e3 = extractelement <4 x i8> %vec1, i64 3
-  %cv1e3 = sext i8 %v1e3 to i32
-  %v2e3 = extractelement <4 x i8> %vec2, i64 3
-  %cv2e3 = sext i8 %v2e3 to i32
-  %mul4 = mul nuw nsw i32 %cv1e3, %cv2e3
-
-  %acc = load i32, i32 addrspace(1)* %dst, align 4
-  %add1 = add i32 %mul1, %acc
-  %add2 = add i32 %add1, %mul2
-  %add3 = add i32 %add2, %mul3
-  %add4 = add i32 %add3, %mul4
-  store i32 %add4, i32 addrspace(1)* %dst, align 4
-  ret void
-}
-
-; TODO: Currently, vector elements{0 and 3} get zero_extended from i16 to i32 which should
-; be sign_extended directly to i32; prevents the pattern recognizer to recognize this pattern.
-define amdgpu_kernel void @idot4_acc16(<4 x i8> addrspace(1)* %src1,
-; GFX7-LABEL: idot4_acc16:
-; GFX7:       ; %bb.0: ; %entry
-; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
-; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
-; GFX7-NEXT:    s_mov_b32 s3, 0xf000
-; GFX7-NEXT:    s_mov_b32 s2, -1
-; GFX7-NEXT:    s_mov_b32 s8, 0xffff
-; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
-; GFX7-NEXT:    buffer_load_ushort v0, off, s[0:3], 0
-; GFX7-NEXT:    s_load_dword s5, s[6:7], 0x0
-; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    s_sext_i32_i8 s6, s4
-; GFX7-NEXT:    s_bfe_i32 s9, s4, 0x80008
-; GFX7-NEXT:    s_sext_i32_i8 s7, s5
-; GFX7-NEXT:    s_bfe_i32 s10, s5, 0x80008
-; GFX7-NEXT:    s_and_b32 s7, s7, s8
-; GFX7-NEXT:    s_bfe_i32 s12, s5, 0x80010
-; GFX7-NEXT:    s_and_b32 s10, s10, s8
-; GFX7-NEXT:    s_and_b32 s6, s6, s8
-; GFX7-NEXT:    v_mov_b32_e32 v1, s7
-; GFX7-NEXT:    s_bfe_i32 s11, s4, 0x80010
-; GFX7-NEXT:    s_ashr_i32 s5, s5, 24
-; GFX7-NEXT:    s_and_b32 s12, s12, s8
-; GFX7-NEXT:    s_and_b32 s9, s9, s8
-; GFX7-NEXT:    v_mov_b32_e32 v2, s10
-; GFX7-NEXT:    s_ashr_i32 s4, s4, 24
-; GFX7-NEXT:    s_and_b32 s11, s11, s8
-; GFX7-NEXT:    s_and_b32 s5, s5, s8
-; GFX7-NEXT:    v_mov_b32_e32 v3, s12
-; GFX7-NEXT:    s_and_b32 s4, s4, s8
-; GFX7-NEXT:    s_waitcnt vmcnt(0)
-; GFX7-NEXT:    v_mad_u32_u24 v0, s6, v1, v0
-; GFX7-NEXT:    v_mad_u32_u24 v0, s9, v2, v0
-; GFX7-NEXT:    v_mad_u32_u24 v0, s11, v3, v0
-; GFX7-NEXT:    v_mov_b32_e32 v1, s5
-; GFX7-NEXT:    v_mad_u32_u24 v0, s4, v1, v0
-; GFX7-NEXT:    buffer_store_short v0, off, s[0:3], 0
-; GFX7-NEXT:    s_endpgm
-;
-; GFX8-LABEL: idot4_acc16:
-; GFX8:       ; %bb.0: ; %entry
-; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
-; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
-; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    v_mov_b32_e32 v0, s0
-; GFX8-NEXT:    v_mov_b32_e32 v1, s1
-; GFX8-NEXT:    flat_load_ushort v2, v[0:1]
-; GFX8-NEXT:    s_load_dword s1, s[4:5], 0x0
-; GFX8-NEXT:    s_load_dword s2, s[6:7], 0x0
-; GFX8-NEXT:    s_mov_b32 s0, 0xffff
-; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    s_lshr_b32 s5, s1, 8
-; GFX8-NEXT:    s_lshr_b32 s6, s2, 8
-; GFX8-NEXT:    s_sext_i32_i8 s4, s2
-; GFX8-NEXT:    s_bfe_i32 s5, s5, 0x80000
-; GFX8-NEXT:    s_bfe_i32 s6, s6, 0x80000
-; GFX8-NEXT:    s_bfe_i32 s8, s2, 0x80010
-; GFX8-NEXT:    s_lshr_b32 s2, s2, 24
-; GFX8-NEXT:    v_mov_b32_e32 v3, s4
-; GFX8-NEXT:    s_sext_i32_i8 s3, s1
-; GFX8-NEXT:    s_bfe_i32 s7, s1, 0x80010
-; GFX8-NEXT:    s_lshr_b32 s1, s1, 24
-; GFX8-NEXT:    s_and_b32 s4, s0, s5
-; GFX8-NEXT:    s_and_b32 s5, s0, s6
-; GFX8-NEXT:    s_bfe_i32 s1, s1, 0x80000
-; GFX8-NEXT:    s_bfe_i32 s2, s2, 0x80000
-; GFX8-NEXT:    v_mov_b32_e32 v5, s5
-; GFX8-NEXT:    s_and_b32 s1, s0, s1
-; GFX8-NEXT:    v_mov_b32_e32 v4, s8
-; GFX8-NEXT:    s_and_b32 s0, s0, s2
-; GFX8-NEXT:    s_waitcnt vmcnt(0)
-; GFX8-NEXT:    v_mad_i32_i24 v2, s3, v3, v2
-; GFX8-NEXT:    v_mad_u32_u24 v2, s4, v5, v2
-; GFX8-NEXT:    v_mad_i32_i24 v2, s7, v4, v2
-; GFX8-NEXT:    v_mov_b32_e32 v3, s0
-; GFX8-NEXT:    v_mad_u32_u24 v2, s1, v3, v2
-; GFX8-NEXT:    flat_store_short v[0:1], v2
-; GFX8-NEXT:    s_endpgm
-;
-; GFX9-NODL-LABEL: idot4_acc16:
-; GFX9-NODL:       ; %bb.0: ; %entry
-; GFX9-NODL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
-; GFX9-NODL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
-; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
-; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
-; GFX9-NODL-NEXT:    global_load_ushort v2, v[0:1], off
-; GFX9-NODL-NEXT:    s_load_dword s1, s[4:5], 0x0
-; GFX9-NODL-NEXT:    s_load_dword s2, s[6:7], 0x0
-; GFX9-NODL-NEXT:    s_mov_b32 s0, 0xffff
-; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NODL-NEXT:    s_lshr_b32 s5, s1, 8
-; GFX9-NODL-NEXT:    s_lshr_b32 s6, s2, 8
-; GFX9-NODL-NEXT:    s_sext_i32_i8 s4, s2
-; GFX9-NODL-NEXT:    s_bfe_i32 s5, s5, 0x80000
-; GFX9-NODL-NEXT:    s_bfe_i32 s6, s6, 0x80000
-; GFX9-NODL-NEXT:    s_bfe_i32 s8, s2, 0x80010
-; GFX9-NODL-NEXT:    s_lshr_b32 s2, s2, 24
-; GFX9-NODL-NEXT:    v_mov_b32_e32 v3, s4
-; GFX9-NODL-NEXT:    s_sext_i32_i8 s3, s1
-; GFX9-NODL-NEXT:    s_bfe_i32 s7, s1, 0x80010
-; GFX9-NODL-NEXT:    s_lshr_b32 s1, s1, 24
-; GFX9-NODL-NEXT:    s_and_b32 s4, s0, s5
-; GFX9-NODL-NEXT:    s_and_b32 s5, s0, s6
-; GFX9-NODL-NEXT:    s_bfe_i32 s1, s1, 0x80000
-; GFX9-NODL-NEXT:    s_bfe_i32 s2, s2, 0x80000
-; GFX9-NODL-NEXT:    v_mov_b32_e32 v5, s5
-; GFX9-NODL-NEXT:    s_and_b32 s1, s0, s1
-; GFX9-NODL-NEXT:    v_mov_b32_e32 v4, s8
-; GFX9-NODL-NEXT:    s_and_b32 s0, s0, s2
-; GFX9-NODL-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NODL-NEXT:    v_mad_i32_i24 v2, s3, v3, v2
-; GFX9-NODL-NEXT:    v_mad_u32_u24 v2, s4, v5, v2
-; GFX9-NODL-NEXT:    v_mad_i32_i24 v2, s7, v4, v2
-; GFX9-NODL-NEXT:    v_mov_b32_e32 v3, s0
-; GFX9-NODL-NEXT:    v_mad_u32_u24 v2, s1, v3, v2
-; GFX9-NODL-NEXT:    global_store_short v[0:1], v2, off
-; GFX9-NODL-NEXT:    s_endpgm
-;
-; GFX9-DL-LABEL: idot4_acc16:
-; GFX9-DL:       ; %bb.0: ; %entry
-; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
-; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
-; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
-; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
-; GFX9-DL-NEXT:    global_load_ushort v2, v[0:1], off
-; GFX9-DL-NEXT:    s_load_dword s1, s[4:5], 0x0
-; GFX9-DL-NEXT:    s_load_dword s2, s[6:7], 0x0
-; GFX9-DL-NEXT:    s_mov_b32 s0, 0xffff
-; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT:    s_lshr_b32 s5, s1, 8
-; GFX9-DL-NEXT:    s_lshr_b32 s6, s2, 8
-; GFX9-DL-NEXT:    s_sext_i32_i8 s4, s2
-; GFX9-DL-NEXT:    s_bfe_i32 s5, s5, 0x80000
-; GFX9-DL-NEXT:    s_bfe_i32 s6, s6, 0x80000
-; GFX9-DL-NEXT:    s_bfe_i32 s8, s2, 0x80010
-; GFX9-DL-NEXT:    s_lshr_b32 s2, s2, 24
-; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s4
-; GFX9-DL-NEXT:    s_sext_i32_i8 s3, s1
-; GFX9-DL-NEXT:    s_bfe_i32 s7, s1, 0x80010
-; GFX9-DL-NEXT:    s_lshr_b32 s1, s1, 24
-; GFX9-DL-NEXT:    s_and_b32 s4, s0, s5
-; GFX9-DL-NEXT:    s_and_b32 s5, s0, s6
-; GFX9-DL-NEXT:    s_bfe_i32 s1, s1, 0x80000
-; GFX9-DL-NEXT:    s_bfe_i32 s2, s2, 0x80000
-; GFX9-DL-NEXT:    v_mov_b32_e32 v5, s5
-; GFX9-DL-NEXT:    s_and_b32 s1, s0, s1
-; GFX9-DL-NEXT:    v_mov_b32_e32 v4, s8
-; GFX9-DL-NEXT:    s_and_b32 s0, s0, s2
-; GFX9-DL-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s3, v3, v2
-; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s4, v5, v2
-; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s7, v4, v2
-; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s0
-; GFX9-DL-NEXT:    v_mad_u32_u24 v2, s1, v3, v2
-; GFX9-DL-NEXT:    global_store_short v[0:1], v2, off
-; GFX9-DL-NEXT:    s_endpgm
-                                       <4 x i8> addrspace(1)* %src2,
-                                       i16 addrspace(1)* nocapture %dst) {
-entry:
-  %vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %src1
-  %vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %src2
-
-  %v1e0 = extractelement <4 x i8> %vec1, i64 0
-  %cv1e0 = sext i8 %v1e0 to i16
-  %v2e0 = extractelement <4 x i8> %vec2, i64 0
-  %cv2e0 = sext i8 %v2e0 to i16
-  %mul1 = mul nsw i16 %cv1e0, %cv2e0
-
-  %v1e1 = extractelement <4 x i8> %vec1, i64 1
-  %cv1e1 = sext i8 %v1e1 to i16
-  %v2e1 = extractelement <4 x i8> %vec2, i64 1
-  %cv2e1 = sext i8 %v2e1 to i16
-  %mul2 = mul nsw i16 %cv1e1, %cv2e1
-
-  %v1e2 = extractelement <4 x i8> %vec1, i64 2
-  %cv1e2 = sext i8 %v1e2 to i16
-  %v2e2 = extractelement <4 x i8> %vec2, i64 2
-  %cv2e2 = sext i8 %v2e2 to i16
-  %mul3 = mul nsw i16 %cv1e2, %cv2e2
-
-  %v1e3 = extractelement <4 x i8> %vec1, i64 3
-  %cv1e3 = sext i8 %v1e3 to i16
-  %v2e3 = extractelement <4 x i8> %vec2, i64 3
-  %cv2e3 = sext i8 %v2e3 to i16
-  %mul4 = mul nsw i16 %cv1e3, %cv2e3
-
-  %acc = load i16, i16 addrspace(1)* %dst, align 2
-  %add1 = add i16 %mul1, %acc
-  %add2 = add i16 %add1, %mul2
-  %add3 = add i16 %add2, %mul3
-  %add4 = add i16 %add3, %mul4
-  store i16 %add4, i16 addrspace(1)* %dst, align 2
-  ret void
-}
-
-define amdgpu_kernel void @idot4_acc8(<4 x i8> addrspace(1)* %src1,
-; GFX7-LABEL: idot4_acc8:
-; GFX7:       ; %bb.0: ; %entry
-; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
-; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
-; GFX7-NEXT:    s_mov_b32 s3, 0xf000
-; GFX7-NEXT:    s_mov_b32 s2, -1
-; GFX7-NEXT:    s_movk_i32 s8, 0xff
-; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
-; GFX7-NEXT:    buffer_load_ubyte v0, off, s[0:3], 0
-; GFX7-NEXT:    s_load_dword s5, s[6:7], 0x0
-; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    s_and_b32 s7, s4, s8
-; GFX7-NEXT:    s_bfe_u32 s9, s4, 0x80008
-; GFX7-NEXT:    s_and_b32 s6, s5, s8
-; GFX7-NEXT:    s_bfe_u32 s8, s5, 0x80008
-; GFX7-NEXT:    v_mov_b32_e32 v1, s6
-; GFX7-NEXT:    s_bfe_u32 s10, s5, 0x80010
-; GFX7-NEXT:    v_mov_b32_e32 v2, s8
-; GFX7-NEXT:    s_bfe_u32 s11, s4, 0x80010
-; GFX7-NEXT:    s_lshr_b32 s5, s5, 24
-; GFX7-NEXT:    v_mov_b32_e32 v3, s10
-; GFX7-NEXT:    s_lshr_b32 s4, s4, 24
-; GFX7-NEXT:    s_waitcnt vmcnt(0)
-; GFX7-NEXT:    v_mad_u32_u24 v0, s7, v1, v0
-; GFX7-NEXT:    v_mad_u32_u24 v0, s9, v2, v0
-; GFX7-NEXT:    v_mad_u32_u24 v0, s11, v3, v0
-; GFX7-NEXT:    v_mov_b32_e32 v1, s5
-; GFX7-NEXT:    v_mad_u32_u24 v0, s4, v1, v0
-; GFX7-NEXT:    buffer_store_byte v0, off, s[0:3], 0
-; GFX7-NEXT:    s_endpgm
-;
-; GFX8-LABEL: idot4_acc8:
-; GFX8:       ; %bb.0: ; %entry
-; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
-; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
-; GFX8-NEXT:    s_movk_i32 s2, 0xff
-; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    v_mov_b32_e32 v0, s0
-; GFX8-NEXT:    v_mov_b32_e32 v1, s1
-; GFX8-NEXT:    flat_load_ubyte v2, v[0:1]
-; GFX8-NEXT:    s_load_dword s0, s[4:5], 0x0
-; GFX8-NEXT:    s_load_dword s1, s[6:7], 0x0
-; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    s_bfe_u32 s4, s0, 0x80008
-; GFX8-NEXT:    s_and_b32 s3, s1, s2
-; GFX8-NEXT:    s_bfe_u32 s5, s1, 0x80008
-; GFX8-NEXT:    s_and_b32 s2, s0, s2
-; GFX8-NEXT:    v_mov_b32_e32 v3, s3
-; GFX8-NEXT:    s_bfe_u32 s6, s1, 0x80010
-; GFX8-NEXT:    v_mov_b32_e32 v4, s5
-; GFX8-NEXT:    s_bfe_u32 s7, s0, 0x80010
-; GFX8-NEXT:    s_lshr_b32 s1, s1, 24
-; GFX8-NEXT:    v_mov_b32_e32 v5, s6
-; GFX8-NEXT:    s_lshr_b32 s0, s0, 24
-; GFX8-NEXT:    s_waitcnt vmcnt(0)
-; GFX8-NEXT:    v_mad_u32_u24 v2, s2, v3, v2
-; GFX8-NEXT:    v_mad_u32_u24 v2, s4, v4, v2
-; GFX8-NEXT:    v_mad_u32_u24 v2, s7, v5, v2
-; GFX8-NEXT:    v_mov_b32_e32 v3, s1
-; GFX8-NEXT:    v_mad_u32_u24 v2, s0, v3, v2
-; GFX8-NEXT:    flat_store_byte v[0:1], v2
-; GFX8-NEXT:    s_endpgm
-;
-; GFX9-NODL-LABEL: idot4_acc8:
-; GFX9-NODL:       ; %bb.0: ; %entry
-; GFX9-NODL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
-; GFX9-NODL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
-; GFX9-NODL-NEXT:    s_movk_i32 s2, 0xff
-; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
-; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
-; GFX9-NODL-NEXT:    global_load_ubyte v2, v[0:1], off
-; GFX9-NODL-NEXT:    s_load_dword s0, s[4:5], 0x0
-; GFX9-NODL-NEXT:    s_load_dword s1, s[6:7], 0x0
-; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NODL-NEXT:    s_bfe_u32 s4, s0, 0x80008
-; GFX9-NODL-NEXT:    s_and_b32 s3, s1, s2
-; GFX9-NODL-NEXT:    s_bfe_u32 s5, s1, 0x80008
-; GFX9-NODL-NEXT:    s_and_b32 s2, s0, s2
-; GFX9-NODL-NEXT:    v_mov_b32_e32 v3, s3
-; GFX9-NODL-NEXT:    s_bfe_u32 s6, s1, 0x80010
-; GFX9-NODL-NEXT:    v_mov_b32_e32 v4, s5
-; GFX9-NODL-NEXT:    s_bfe_u32 s7, s0, 0x80010
-; GFX9-NODL-NEXT:    s_lshr_b32 s1, s1, 24
-; GFX9-NODL-NEXT:    v_mov_b32_e32 v5, s6
-; GFX9-NODL-NEXT:    s_lshr_b32 s0, s0, 24
-; GFX9-NODL-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NODL-NEXT:    v_mad_u32_u24 v2, s2, v3, v2
-; GFX9-NODL-NEXT:    v_mad_u32_u24 v2, s4, v4, v2
-; GFX9-NODL-NEXT:    v_mad_u32_u24 v2, s7, v5, v2
-; GFX9-NODL-NEXT:    v_mov_b32_e32 v3, s1
-; GFX9-NODL-NEXT:    v_mad_u32_u24 v2, s0, v3, v2
-; GFX9-NODL-NEXT:    global_store_byte v[0:1], v2, off
-; GFX9-NODL-NEXT:    s_endpgm
-;
-; GFX9-DL-LABEL: idot4_acc8:
-; GFX9-DL:       ; %bb.0: ; %entry
-; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
-; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
-; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
-; GFX9-DL-NEXT:    s_load_dword s3, s[6:7], 0x0
-; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
-; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
-; GFX9-DL-NEXT:    global_load_ubyte v2, v[0:1], off
-; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s3
-; GFX9-DL-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-DL-NEXT:    v_dot4_u32_u8 v2, s2, v3, v2
-; GFX9-DL-NEXT:    global_store_byte v[0:1], v2, off
-; GFX9-DL-NEXT:    s_endpgm
-                                      <4 x i8> addrspace(1)* %src2,
-                                      i8 addrspace(1)* nocapture %dst) {
-entry:
-  %vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %src1
-  %vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %src2
-
-  %v1e0 = extractelement <4 x i8> %vec1, i64 0
-  %v2e0 = extractelement <4 x i8> %vec2, i64 0
-  %mul1 = mul i8 %v1e0, %v2e0
-
-  %v1e1 = extractelement <4 x i8> %vec1, i64 1
-  %v2e1 = extractelement <4 x i8> %vec2, i64 1
-  %mul2 = mul i8 %v1e1, %v2e1
-
-  %v1e2 = extractelement <4 x i8> %vec1, i64 2
-  %v2e2 = extractelement <4 x i8> %vec2, i64 2
-  %mul3 = mul i8 %v1e2, %v2e2
-
-  %v1e3 = extractelement <4 x i8> %vec1, i64 3
-  %v2e3 = extractelement <4 x i8> %vec2, i64 3
-  %mul4 = mul i8 %v1e3, %v2e3
-
-  %acc = load i8, i8 addrspace(1)* %dst, align 2
-  %add1 = add i8 %mul1, %acc
-  %add2 = add i8 %add1, %mul2
-  %add3 = add i8 %add2, %mul3
-  %add4 = add nsw i8 %add3, %mul4
-  store i8 %add4, i8 addrspace(1)* %dst, align 2
-  ret void
-}
-
-define amdgpu_kernel void @idot4_multiuse_mul1(<4 x i8> addrspace(1)* %src1,
-; GFX7-LABEL: idot4_multiuse_mul1:
-; GFX7:       ; %bb.0: ; %entry
-; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
-; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
-; GFX7-NEXT:    s_mov_b32 s3, 0xf000
-; GFX7-NEXT:    s_mov_b32 s2, -1
-; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
-; GFX7-NEXT:    s_load_dword s5, s[6:7], 0x0
-; GFX7-NEXT:    s_load_dword s6, s[0:1], 0x0
-; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    s_sext_i32_i8 s7, s4
-; GFX7-NEXT:    s_sext_i32_i8 s8, s5
-; GFX7-NEXT:    s_bfe_i32 s10, s5, 0x80008
-; GFX7-NEXT:    v_mov_b32_e32 v0, s8
-; GFX7-NEXT:    v_mov_b32_e32 v1, s6
-; GFX7-NEXT:    s_bfe_i32 s9, s4, 0x80008
-; GFX7-NEXT:    v_mad_i32_i24 v1, s7, v0, v1
-; GFX7-NEXT:    v_mov_b32_e32 v2, s10
-; GFX7-NEXT:    s_bfe_i32 s12, s5, 0x80010
-; GFX7-NEXT:    v_mad_i32_i24 v1, s9, v2, v1
-; GFX7-NEXT:    s_bfe_i32 s11, s4, 0x80010
-; GFX7-NEXT:    v_mad_i32_i24 v0, s7, v0, v1
-; GFX7-NEXT:    v_mov_b32_e32 v1, s12
-; GFX7-NEXT:    s_ashr_i32 s5, s5, 24
-; GFX7-NEXT:    v_mad_i32_i24 v0, s11, v1, v0
-; GFX7-NEXT:    s_ashr_i32 s4, s4, 24
-; GFX7-NEXT:    v_mov_b32_e32 v1, s5
-; GFX7-NEXT:    v_mad_i32_i24 v0, s4, v1, v0
-; GFX7-NEXT:    buffer_store_dword v0, off, s[0:3], 0
-; GFX7-NEXT:    s_endpgm
-;
-; GFX8-LABEL: idot4_multiuse_mul1:
-; GFX8:       ; %bb.0: ; %entry
-; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
-; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
-; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    s_load_dword s2, s[4:5], 0x0
-; GFX8-NEXT:    s_load_dword s3, s[6:7], 0x0
-; GFX8-NEXT:    s_load_dword s4, s[0:1], 0x0
-; GFX8-NEXT:    v_mov_b32_e32 v0, s0
-; GFX8-NEXT:    v_mov_b32_e32 v1, s1
-; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    s_sext_i32_i8 s0, s2
-; GFX8-NEXT:    s_sext_i32_i8 s1, s3
-; GFX8-NEXT:    s_bfe_i32 s6, s3, 0x80008
-; GFX8-NEXT:    v_mov_b32_e32 v2, s1
-; GFX8-NEXT:    v_mov_b32_e32 v3, s4
-; GFX8-NEXT:    s_bfe_i32 s5, s2, 0x80008
-; GFX8-NEXT:    v_mad_i32_i24 v3, s0, v2, v3
-; GFX8-NEXT:    v_mov_b32_e32 v4, s6
-; GFX8-NEXT:    s_bfe_i32 s8, s3, 0x80010
-; GFX8-NEXT:    v_mad_i32_i24 v3, s5, v4, v3
-; GFX8-NEXT:    s_bfe_i32 s7, s2, 0x80010
-; GFX8-NEXT:    v_mad_i32_i24 v2, s0, v2, v3
-; GFX8-NEXT:    v_mov_b32_e32 v3, s8
-; GFX8-NEXT:    s_ashr_i32 s3, s3, 24
-; GFX8-NEXT:    v_mad_i32_i24 v2, s7, v3, v2
-; GFX8-NEXT:    s_ashr_i32 s2, s2, 24
-; GFX8-NEXT:    v_mov_b32_e32 v3, s3
-; GFX8-NEXT:    v_mad_i32_i24 v2, s2, v3, v2
-; GFX8-NEXT:    flat_store_dword v[0:1], v2
-; GFX8-NEXT:    s_endpgm
-;
-; GFX9-NODL-LABEL: idot4_multiuse_mul1:
-; GFX9-NODL:       ; %bb.0: ; %entry
-; GFX9-NODL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
-; GFX9-NODL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
-; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NODL-NEXT:    s_load_dword s2, s[4:5], 0x0
-; GFX9-NODL-NEXT:    s_load_dword s3, s[6:7], 0x0
-; GFX9-NODL-NEXT:    s_load_dword s4, s[0:1], 0x0
-; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
-; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
-; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NODL-NEXT:    s_sext_i32_i8 s0, s2
-; GFX9-NODL-NEXT:    s_sext_i32_i8 s1, s3
-; GFX9-NODL-NEXT:    s_bfe_i32 s6, s3, 0x80008
-; GFX9-NODL-NEXT:    v_mov_b32_e32 v2, s1
-; GFX9-NODL-NEXT:    v_mov_b32_e32 v3, s4
-; GFX9-NODL-NEXT:    s_bfe_i32 s5, s2, 0x80008
-; GFX9-NODL-NEXT:    v_mad_i32_i24 v3, s0, v2, v3
-; GFX9-NODL-NEXT:    v_mov_b32_e32 v4, s6
-; GFX9-NODL-NEXT:    s_bfe_i32 s8, s3, 0x80010
-; GFX9-NODL-NEXT:    v_mad_i32_i24 v3, s5, v4, v3
-; GFX9-NODL-NEXT:    s_bfe_i32 s7, s2, 0x80010
-; GFX9-NODL-NEXT:    v_mad_i32_i24 v2, s0, v2, v3
-; GFX9-NODL-NEXT:    v_mov_b32_e32 v3, s8
-; GFX9-NODL-NEXT:    s_ashr_i32 s3, s3, 24
-; GFX9-NODL-NEXT:    v_mad_i32_i24 v2, s7, v3, v2
-; GFX9-NODL-NEXT:    s_ashr_i32 s2, s2, 24
-; GFX9-NODL-NEXT:    v_mov_b32_e32 v3, s3
-; GFX9-NODL-NEXT:    v_mad_i32_i24 v2, s2, v3, v2
-; GFX9-NODL-NEXT:    global_store_dword v[0:1], v2, off
-; GFX9-NODL-NEXT:    s_endpgm
-;
-; GFX9-DL-LABEL: idot4_multiuse_mul1:
-; GFX9-DL:       ; %bb.0: ; %entry
-; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
-; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
-; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
-; GFX9-DL-NEXT:    s_load_dword s3, s[6:7], 0x0
-; GFX9-DL-NEXT:    s_load_dword s4, s[0:1], 0x0
-; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
-; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
-; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT:    s_sext_i32_i8 s0, s2
-; GFX9-DL-NEXT:    s_sext_i32_i8 s1, s3
-; GFX9-DL-NEXT:    s_bfe_i32 s6, s3, 0x80008
-; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s1
-; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s4
-; GFX9-DL-NEXT:    s_bfe_i32 s5, s2, 0x80008
-; GFX9-DL-NEXT:    v_mad_i32_i24 v3, s0, v2, v3
-; GFX9-DL-NEXT:    v_mov_b32_e32 v4, s6
-; GFX9-DL-NEXT:    s_bfe_i32 s8, s3, 0x80010
-; GFX9-DL-NEXT:    v_mad_i32_i24 v3, s5, v4, v3
-; GFX9-DL-NEXT:    s_bfe_i32 s7, s2, 0x80010
-; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s0, v2, v3
-; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s8
-; GFX9-DL-NEXT:    s_ashr_i32 s3, s3, 24
-; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s7, v3, v2
-; GFX9-DL-NEXT:    s_ashr_i32 s2, s2, 24
-; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s3
-; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s2, v3, v2
-; GFX9-DL-NEXT:    global_store_dword v[0:1], v2, off
-; GFX9-DL-NEXT:    s_endpgm
-                                               <4 x i8> addrspace(1)* %src2,
-                                               i32 addrspace(1)* nocapture %dst) {
-entry:
-  %vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %src1
-  %vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %src2
-
-  %v1e0 = extractelement <4 x i8> %vec1, i64 0
-  %cv1e0 = sext i8 %v1e0 to i32
-  %v2e0 = extractelement <4 x i8> %vec2, i64 0
-  %cv2e0 = sext i8 %v2e0 to i32
-  %mul1 = mul nuw nsw i32 %cv1e0, %cv2e0
-
-  %v1e1 = extractelement <4 x i8> %vec1, i64 1
-  %cv1e1 = sext i8 %v1e1 to i32
-  %v2e1 = extractelement <4 x i8> %vec2, i64 1
-  %cv2e1 = sext i8 %v2e1 to i32
-  %mul2 = mul nuw nsw i32 %cv1e1, %cv2e1
-
-  %v1e2 = extractelement <4 x i8> %vec1, i64 2
-  %cv1e2 = sext i8 %v1e2 to i32
-  %v2e2 = extractelement <4 x i8> %vec2, i64 2
-  %cv2e2 = sext i8 %v2e2 to i32
-  %mul3 = mul nuw nsw i32 %cv1e2, %cv2e2
-
-  %v1e3 = extractelement <4 x i8> %vec1, i64 3
-  %cv1e3 = sext i8 %v1e3 to i32
-  %v2e3 = extractelement <4 x i8> %vec2, i64 3
-  %cv2e3 = sext i8 %v2e3 to i32
-  %mul4 = mul nuw nsw i32 %cv1e3, %cv2e3
-
-  %acc = load i32, i32 addrspace(1)* %dst, align 4
-  %add = add i32 %mul1, %acc
-  %add1 = add i32 %mul2, %add
-  %add2 = add i32 %add1, %mul1
-  %add3 = add i32 %add2, %mul3
-  %add4 = add i32 %add3, %mul4
-
-  store i32 %add4, i32 addrspace(1)* %dst, align 4
-  ret void
-}
-
 ; TODO: cleanup s_lshr_b32 and support this pattern.
 define amdgpu_kernel void @udot4_acc32_vecMul(<4 x i8> addrspace(1)* %src1,
 ; GFX7-LABEL: udot4_acc32_vecMul:
@@ -2219,159 +1556,6 @@ entry:
   ret void
 }
 
-; TODO: Support this pattern.
-define amdgpu_kernel void @idot4_acc32_vecMul(<4 x i8> addrspace(1)* %src1,
-; GFX7-LABEL: idot4_acc32_vecMul:
-; GFX7:       ; %bb.0: ; %entry
-; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
-; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
-; GFX7-NEXT:    s_mov_b32 s3, 0xf000
-; GFX7-NEXT:    s_mov_b32 s2, -1
-; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
-; GFX7-NEXT:    s_load_dword s5, s[6:7], 0x0
-; GFX7-NEXT:    s_load_dword s6, s[0:1], 0x0
-; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    s_ashr_i32 s7, s4, 24
-; GFX7-NEXT:    s_ashr_i32 s10, s5, 24
-; GFX7-NEXT:    s_bfe_i32 s11, s5, 0x80010
-; GFX7-NEXT:    s_bfe_i32 s12, s5, 0x80008
-; GFX7-NEXT:    s_sext_i32_i8 s5, s5
-; GFX7-NEXT:    s_bfe_i32 s8, s4, 0x80010
-; GFX7-NEXT:    s_bfe_i32 s9, s4, 0x80008
-; GFX7-NEXT:    s_sext_i32_i8 s4, s4
-; GFX7-NEXT:    v_mov_b32_e32 v0, s5
-; GFX7-NEXT:    v_mov_b32_e32 v1, s6
-; GFX7-NEXT:    v_mad_i32_i24 v0, s4, v0, v1
-; GFX7-NEXT:    v_mov_b32_e32 v1, s12
-; GFX7-NEXT:    v_mad_i32_i24 v0, s9, v1, v0
-; GFX7-NEXT:    v_mov_b32_e32 v1, s11
-; GFX7-NEXT:    v_mad_i32_i24 v0, s8, v1, v0
-; GFX7-NEXT:    v_mov_b32_e32 v1, s10
-; GFX7-NEXT:    v_mad_i32_i24 v0, s7, v1, v0
-; GFX7-NEXT:    buffer_store_dword v0, off, s[0:3], 0
-; GFX7-NEXT:    s_endpgm
-;
-; GFX8-LABEL: idot4_acc32_vecMul:
-; GFX8:       ; %bb.0: ; %entry
-; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
-; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
-; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    s_load_dword s2, s[4:5], 0x0
-; GFX8-NEXT:    s_load_dword s3, s[6:7], 0x0
-; GFX8-NEXT:    s_load_dword s4, s[0:1], 0x0
-; GFX8-NEXT:    v_mov_b32_e32 v0, s0
-; GFX8-NEXT:    v_mov_b32_e32 v1, s1
-; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    v_lshrrev_b16_e64 v2, 8, s2
-; GFX8-NEXT:    v_lshrrev_b16_e64 v3, 8, s3
-; GFX8-NEXT:    s_ashr_i32 s5, s3, 24
-; GFX8-NEXT:    s_bfe_i32 s6, s3, 0x80010
-; GFX8-NEXT:    s_sext_i32_i8 s3, s3
-; GFX8-NEXT:    s_ashr_i32 s0, s2, 24
-; GFX8-NEXT:    s_bfe_i32 s1, s2, 0x80010
-; GFX8-NEXT:    s_sext_i32_i8 s2, s2
-; GFX8-NEXT:    v_mov_b32_e32 v4, s3
-; GFX8-NEXT:    v_mov_b32_e32 v5, s4
-; GFX8-NEXT:    v_bfe_i32 v2, v2, 0, 8
-; GFX8-NEXT:    v_bfe_i32 v3, v3, 0, 8
-; GFX8-NEXT:    v_mad_i32_i24 v4, s2, v4, v5
-; GFX8-NEXT:    v_mad_i32_i24 v2, v2, v3, v4
-; GFX8-NEXT:    v_mov_b32_e32 v3, s6
-; GFX8-NEXT:    v_mad_i32_i24 v2, s1, v3, v2
-; GFX8-NEXT:    v_mov_b32_e32 v3, s5
-; GFX8-NEXT:    v_mad_i32_i24 v2, s0, v3, v2
-; GFX8-NEXT:    flat_store_dword v[0:1], v2
-; GFX8-NEXT:    s_endpgm
-;
-; GFX9-NODL-LABEL: idot4_acc32_vecMul:
-; GFX9-NODL:       ; %bb.0: ; %entry
-; GFX9-NODL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
-; GFX9-NODL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
-; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NODL-NEXT:    s_load_dword s2, s[4:5], 0x0
-; GFX9-NODL-NEXT:    s_load_dword s3, s[6:7], 0x0
-; GFX9-NODL-NEXT:    s_load_dword s4, s[0:1], 0x0
-; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
-; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
-; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NODL-NEXT:    v_lshrrev_b16_e64 v2, 8, s2
-; GFX9-NODL-NEXT:    v_lshrrev_b16_e64 v3, 8, s3
-; GFX9-NODL-NEXT:    s_ashr_i32 s5, s3, 24
-; GFX9-NODL-NEXT:    s_bfe_i32 s6, s3, 0x80010
-; GFX9-NODL-NEXT:    s_sext_i32_i8 s3, s3
-; GFX9-NODL-NEXT:    s_ashr_i32 s0, s2, 24
-; GFX9-NODL-NEXT:    s_bfe_i32 s1, s2, 0x80010
-; GFX9-NODL-NEXT:    s_sext_i32_i8 s2, s2
-; GFX9-NODL-NEXT:    v_mov_b32_e32 v4, s3
-; GFX9-NODL-NEXT:    v_mov_b32_e32 v5, s4
-; GFX9-NODL-NEXT:    v_bfe_i32 v2, v2, 0, 8
-; GFX9-NODL-NEXT:    v_bfe_i32 v3, v3, 0, 8
-; GFX9-NODL-NEXT:    v_mad_i32_i24 v4, s2, v4, v5
-; GFX9-NODL-NEXT:    v_mad_i32_i24 v2, v2, v3, v4
-; GFX9-NODL-NEXT:    v_mov_b32_e32 v3, s6
-; GFX9-NODL-NEXT:    v_mad_i32_i24 v2, s1, v3, v2
-; GFX9-NODL-NEXT:    v_mov_b32_e32 v3, s5
-; GFX9-NODL-NEXT:    v_mad_i32_i24 v2, s0, v3, v2
-; GFX9-NODL-NEXT:    global_store_dword v[0:1], v2, off
-; GFX9-NODL-NEXT:    s_endpgm
-;
-; GFX9-DL-LABEL: idot4_acc32_vecMul:
-; GFX9-DL:       ; %bb.0: ; %entry
-; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
-; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
-; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
-; GFX9-DL-NEXT:    s_load_dword s3, s[6:7], 0x0
-; GFX9-DL-NEXT:    s_load_dword s4, s[0:1], 0x0
-; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
-; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
-; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT:    v_lshrrev_b16_e64 v2, 8, s2
-; GFX9-DL-NEXT:    v_lshrrev_b16_e64 v3, 8, s3
-; GFX9-DL-NEXT:    s_ashr_i32 s5, s3, 24
-; GFX9-DL-NEXT:    s_bfe_i32 s6, s3, 0x80010
-; GFX9-DL-NEXT:    s_sext_i32_i8 s3, s3
-; GFX9-DL-NEXT:    s_ashr_i32 s0, s2, 24
-; GFX9-DL-NEXT:    s_bfe_i32 s1, s2, 0x80010
-; GFX9-DL-NEXT:    s_sext_i32_i8 s2, s2
-; GFX9-DL-NEXT:    v_mov_b32_e32 v4, s3
-; GFX9-DL-NEXT:    v_mov_b32_e32 v5, s4
-; GFX9-DL-NEXT:    v_bfe_i32 v2, v2, 0, 8
-; GFX9-DL-NEXT:    v_bfe_i32 v3, v3, 0, 8
-; GFX9-DL-NEXT:    v_mad_i32_i24 v4, s2, v4, v5
-; GFX9-DL-NEXT:    v_mad_i32_i24 v2, v2, v3, v4
-; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s6
-; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s1, v3, v2
-; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s5
-; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s0, v3, v2
-; GFX9-DL-NEXT:    global_store_dword v[0:1], v2, off
-; GFX9-DL-NEXT:    s_endpgm
-                                              <4 x i8> addrspace(1)* %src2,
-                                              i32 addrspace(1)* nocapture %dst) {
-entry:
-  %vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %src1
-  %vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %src2
-
-  %cvec1 = sext <4 x i8> %vec1 to <4 x i32>
-  %cvec2 = sext <4 x i8> %vec2 to <4 x i32>
-
-  %mul = mul <4 x i32> %cvec1, %cvec2
-  %mul0 = extractelement <4 x i32> %mul, i64 0
-  %mul1 = extractelement <4 x i32> %mul, i64 1
-  %mul2 = extractelement <4 x i32> %mul, i64 2
-  %mul3 = extractelement <4 x i32> %mul, i64 3
-
-  %acc = load i32, i32 addrspace(1)* %dst, align 4
-  %add1 = add i32 %mul0, %acc
-  %add2 = add i32 %add1, %mul1
-  %add3 = add i32 %add2, %mul2
-  %add4 = add i32 %add3, %mul3
-
-  store i32 %add4, i32 addrspace(1)* %dst, align 4
-  ret void
-}
-
 ; TODO: This pattern should be recognized.
 define amdgpu_kernel void @udot4_acc16_vecMul(<4 x i8> addrspace(1)* %src1,
 ; GFX7-LABEL: udot4_acc16_vecMul:
@@ -2543,191 +1727,6 @@ entry:
   ret void
 }
 
-define amdgpu_kernel void @idot4_acc16_vecMul(<4 x i8> addrspace(1)* %src1,
-; GFX7-LABEL: idot4_acc16_vecMul:
-; GFX7:       ; %bb.0: ; %entry
-; GFX7-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
-; GFX7-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
-; GFX7-NEXT:    s_mov_b32 s3, 0xf000
-; GFX7-NEXT:    s_mov_b32 s2, -1
-; GFX7-NEXT:    s_mov_b32 s8, 0xffff
-; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    s_load_dword s4, s[4:5], 0x0
-; GFX7-NEXT:    buffer_load_ushort v0, off, s[0:3], 0
-; GFX7-NEXT:    s_load_dword s5, s[6:7], 0x0
-; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    s_sext_i32_i8 s6, s4
-; GFX7-NEXT:    s_bfe_i32 s7, s4, 0x80008
-; GFX7-NEXT:    s_sext_i32_i8 s10, s5
-; GFX7-NEXT:    s_bfe_i32 s11, s5, 0x80008
-; GFX7-NEXT:    s_bfe_i32 s12, s5, 0x80010
-; GFX7-NEXT:    s_ashr_i32 s5, s5, 24
-; GFX7-NEXT:    v_mov_b32_e32 v3, s11
-; GFX7-NEXT:    v_mov_b32_e32 v4, s10
-; GFX7-NEXT:    s_bfe_i32 s9, s4, 0x80010
-; GFX7-NEXT:    v_mov_b32_e32 v2, s12
-; GFX7-NEXT:    s_ashr_i32 s4, s4, 24
-; GFX7-NEXT:    v_mov_b32_e32 v1, s5
-; GFX7-NEXT:    v_mul_i32_i24_e32 v1, s4, v1
-; GFX7-NEXT:    v_mul_i32_i24_e32 v2, s9, v2
-; GFX7-NEXT:    v_mul_i32_i24_e32 v3, s7, v3
-; GFX7-NEXT:    v_mul_i32_i24_e32 v4, s6, v4
-; GFX7-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; GFX7-NEXT:    v_and_b32_e32 v2, s8, v2
-; GFX7-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; GFX7-NEXT:    v_and_b32_e32 v4, s8, v4
-; GFX7-NEXT:    v_or_b32_e32 v1, v2, v1
-; GFX7-NEXT:    v_or_b32_e32 v2, v4, v3
-; GFX7-NEXT:    v_lshrrev_b32_e32 v3, 16, v2
-; GFX7-NEXT:    v_lshrrev_b32_e32 v4, 16, v1
-; GFX7-NEXT:    s_waitcnt vmcnt(0)
-; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
-; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v3, v0
-; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
-; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v4, v0
-; GFX7-NEXT:    buffer_store_short v0, off, s[0:3], 0
-; GFX7-NEXT:    s_endpgm
-;
-; GFX8-LABEL: idot4_acc16_vecMul:
-; GFX8:       ; %bb.0: ; %entry
-; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
-; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
-; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    v_mov_b32_e32 v0, s0
-; GFX8-NEXT:    v_mov_b32_e32 v1, s1
-; GFX8-NEXT:    flat_load_ushort v2, v[0:1]
-; GFX8-NEXT:    s_load_dword s1, s[4:5], 0x0
-; GFX8-NEXT:    s_load_dword s2, s[6:7], 0x0
-; GFX8-NEXT:    s_mov_b32 s0, 0xffff
-; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    s_lshr_b32 s3, s1, 16
-; GFX8-NEXT:    s_bfe_i32 s6, s2, 0x80000
-; GFX8-NEXT:    s_lshr_b32 s4, s2, 16
-; GFX8-NEXT:    s_bfe_i32 s5, s1, 0x80000
-; GFX8-NEXT:    v_ashrrev_i16_e64 v4, 8, s1
-; GFX8-NEXT:    s_bfe_i32 s1, s3, 0x80000
-; GFX8-NEXT:    v_ashrrev_i16_e64 v6, 8, s3
-; GFX8-NEXT:    s_and_b32 s3, s0, s6
-; GFX8-NEXT:    v_ashrrev_i16_e64 v3, 8, s2
-; GFX8-NEXT:    s_bfe_i32 s2, s4, 0x80000
-; GFX8-NEXT:    v_ashrrev_i16_e64 v5, 8, s4
-; GFX8-NEXT:    s_and_b32 s4, s0, s5
-; GFX8-NEXT:    v_mov_b32_e32 v7, s3
-; GFX8-NEXT:    s_and_b32 s2, s0, s2
-; GFX8-NEXT:    s_and_b32 s0, s0, s1
-; GFX8-NEXT:    s_waitcnt vmcnt(0)
-; GFX8-NEXT:    v_mad_u32_u24 v2, s4, v7, v2
-; GFX8-NEXT:    v_mad_u32_u24 v2, v4, v3, v2
-; GFX8-NEXT:    v_mov_b32_e32 v3, s2
-; GFX8-NEXT:    v_mad_u32_u24 v2, s0, v3, v2
-; GFX8-NEXT:    v_mad_u32_u24 v2, v6, v5, v2
-; GFX8-NEXT:    flat_store_short v[0:1], v2
-; GFX8-NEXT:    s_endpgm
-;
-; GFX9-NODL-LABEL: idot4_acc16_vecMul:
-; GFX9-NODL:       ; %bb.0: ; %entry
-; GFX9-NODL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
-; GFX9-NODL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
-; GFX9-NODL-NEXT:    v_mov_b32_e32 v4, 0xffff
-; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NODL-NEXT:    s_load_dword s2, s[4:5], 0x0
-; GFX9-NODL-NEXT:    s_load_dword s3, s[6:7], 0x0
-; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NODL-NEXT:    s_lshr_b32 s4, s2, 16
-; GFX9-NODL-NEXT:    s_lshr_b32 s5, s3, 16
-; GFX9-NODL-NEXT:    v_ashrrev_i16_e64 v3, 8, s5
-; GFX9-NODL-NEXT:    s_bfe_i32 s5, s5, 0x80000
-; GFX9-NODL-NEXT:    v_ashrrev_i16_e64 v2, 8, s4
-; GFX9-NODL-NEXT:    v_and_b32_e32 v5, s5, v4
-; GFX9-NODL-NEXT:    s_bfe_i32 s4, s4, 0x80000
-; GFX9-NODL-NEXT:    v_lshl_or_b32 v3, v3, 16, v5
-; GFX9-NODL-NEXT:    v_and_b32_e32 v5, s4, v4
-; GFX9-NODL-NEXT:    v_lshl_or_b32 v2, v2, 16, v5
-; GFX9-NODL-NEXT:    v_ashrrev_i16_e64 v1, 8, s3
-; GFX9-NODL-NEXT:    s_bfe_i32 s3, s3, 0x80000
-; GFX9-NODL-NEXT:    v_ashrrev_i16_e64 v0, 8, s2
-; GFX9-NODL-NEXT:    v_pk_mul_lo_u16 v2, v2, v3
-; GFX9-NODL-NEXT:    v_and_b32_e32 v3, s3, v4
-; GFX9-NODL-NEXT:    s_bfe_i32 s2, s2, 0x80000
-; GFX9-NODL-NEXT:    v_lshl_or_b32 v1, v1, 16, v3
-; GFX9-NODL-NEXT:    v_and_b32_e32 v3, s2, v4
-; GFX9-NODL-NEXT:    v_lshl_or_b32 v0, v0, 16, v3
-; GFX9-NODL-NEXT:    v_pk_mul_lo_u16 v3, v0, v1
-; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, s0
-; GFX9-NODL-NEXT:    v_mov_b32_e32 v1, s1
-; GFX9-NODL-NEXT:    global_load_ushort v4, v[0:1], off
-; GFX9-NODL-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NODL-NEXT:    v_add_u32_e32 v4, v3, v4
-; GFX9-NODL-NEXT:    v_add_u32_sdwa v3, v4, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NODL-NEXT:    v_add_u32_e32 v3, v3, v2
-; GFX9-NODL-NEXT:    v_add_u32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NODL-NEXT:    global_store_short v[0:1], v2, off
-; GFX9-NODL-NEXT:    s_endpgm
-;
-; GFX9-DL-LABEL: idot4_acc16_vecMul:
-; GFX9-DL:       ; %bb.0: ; %entry
-; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
-; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
-; GFX9-DL-NEXT:    v_mov_b32_e32 v4, 0xffff
-; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
-; GFX9-DL-NEXT:    s_load_dword s3, s[6:7], 0x0
-; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT:    s_lshr_b32 s4, s2, 16
-; GFX9-DL-NEXT:    s_lshr_b32 s5, s3, 16
-; GFX9-DL-NEXT:    v_ashrrev_i16_e64 v3, 8, s5
-; GFX9-DL-NEXT:    s_bfe_i32 s5, s5, 0x80000
-; GFX9-DL-NEXT:    v_ashrrev_i16_e64 v2, 8, s4
-; GFX9-DL-NEXT:    v_and_b32_e32 v5, s5, v4
-; GFX9-DL-NEXT:    s_bfe_i32 s4, s4, 0x80000
-; GFX9-DL-NEXT:    v_lshl_or_b32 v3, v3, 16, v5
-; GFX9-DL-NEXT:    v_and_b32_e32 v5, s4, v4
-; GFX9-DL-NEXT:    v_lshl_or_b32 v2, v2, 16, v5
-; GFX9-DL-NEXT:    v_ashrrev_i16_e64 v1, 8, s3
-; GFX9-DL-NEXT:    s_bfe_i32 s3, s3, 0x80000
-; GFX9-DL-NEXT:    v_ashrrev_i16_e64 v0, 8, s2
-; GFX9-DL-NEXT:    v_pk_mul_lo_u16 v2, v2, v3
-; GFX9-DL-NEXT:    v_and_b32_e32 v3, s3, v4
-; GFX9-DL-NEXT:    s_bfe_i32 s2, s2, 0x80000
-; GFX9-DL-NEXT:    v_lshl_or_b32 v1, v1, 16, v3
-; GFX9-DL-NEXT:    v_and_b32_e32 v3, s2, v4
-; GFX9-DL-NEXT:    v_lshl_or_b32 v0, v0, 16, v3
-; GFX9-DL-NEXT:    v_pk_mul_lo_u16 v3, v0, v1
-; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
-; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
-; GFX9-DL-NEXT:    global_load_ushort v4, v[0:1], off
-; GFX9-DL-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-DL-NEXT:    v_add_u32_e32 v4, v3, v4
-; GFX9-DL-NEXT:    v_add_u32_sdwa v3, v4, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-DL-NEXT:    v_add_u32_e32 v3, v3, v2
-; GFX9-DL-NEXT:    v_add_u32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-DL-NEXT:    global_store_short v[0:1], v2, off
-; GFX9-DL-NEXT:    s_endpgm
-                                              <4 x i8> addrspace(1)* %src2,
-                                              i16 addrspace(1)* nocapture %dst) {
-entry:
-  %vec1 = load <4 x i8>, <4 x i8> addrspace(1)* %src1
-  %vec2 = load <4 x i8>, <4 x i8> addrspace(1)* %src2
-
-  %cvec1 = sext <4 x i8> %vec1 to <4 x i16>
-  %cvec2 = sext <4 x i8> %vec2 to <4 x i16>
-
-  %mul = mul <4 x i16> %cvec1, %cvec2
-  %mul0 = extractelement <4 x i16> %mul, i64 0
-  %mul1 = extractelement <4 x i16> %mul, i64 1
-  %mul2 = extractelement <4 x i16> %mul, i64 2
-  %mul3 = extractelement <4 x i16> %mul, i64 3
-
-  %acc = load i16, i16 addrspace(1)* %dst, align 4
-  %add1 = add i16 %mul0, %acc
-  %add2 = add i16 %add1, %mul1
-  %add3 = add i16 %add2, %mul2
-  %add4 = add i16 %add3, %mul3
-
-  store i16 %add4, i16 addrspace(1)* %dst, align 4
-  ret void
-}
-
 ; TODO: Support this pattern.
 define amdgpu_kernel void @udot4_acc8_vecMul(<4 x i8> addrspace(1)* %src1,
 ; GFX7-LABEL: udot4_acc8_vecMul:
diff --git a/test/CodeGen/AMDGPU/idot8s.ll b/test/CodeGen/AMDGPU/idot8s.ll
new file mode 100644
index 0000000000000000000000000000000000000000..a805bb7098e4805f79be63f0a5ddb288bf61171e
--- /dev/null
+++ b/test/CodeGen/AMDGPU/idot8s.ll
@@ -0,0 +1,2111 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=amdgcn -mcpu=gfx700 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX7 %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx803 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX8 %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX9 %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx906 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX9-DL %s
+
+define amdgpu_kernel void @idot8_acc32(<8 x i4> addrspace(1)* %src1,
+; GFX7-LABEL: idot8_acc32:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[8:11], s[0:1], 0x9
+; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0xd
+; GFX7-NEXT:    s_mov_b32 s7, 0xf000
+; GFX7-NEXT:    s_mov_b32 s6, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_load_dword s0, s[8:9], 0x0
+; GFX7-NEXT:    s_load_dword s1, s[10:11], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_bfe_i32 s8, s0, 0x40000
+; GFX7-NEXT:    s_bfe_i32 s9, s1, 0x40000
+; GFX7-NEXT:    s_bfe_i32 s11, s1, 0x40004
+; GFX7-NEXT:    v_mov_b32_e32 v0, s9
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mad_i32_i24 v0, s8, v0, v1
+; GFX7-NEXT:    s_bfe_i32 s10, s0, 0x40004
+; GFX7-NEXT:    v_mov_b32_e32 v1, s11
+; GFX7-NEXT:    s_bfe_i32 s13, s1, 0x40008
+; GFX7-NEXT:    v_mad_i32_i24 v0, s10, v1, v0
+; GFX7-NEXT:    s_bfe_i32 s12, s0, 0x40008
+; GFX7-NEXT:    v_mov_b32_e32 v1, s13
+; GFX7-NEXT:    s_bfe_i32 s15, s1, 0x4000c
+; GFX7-NEXT:    v_mad_i32_i24 v0, s12, v1, v0
+; GFX7-NEXT:    s_bfe_i32 s14, s0, 0x4000c
+; GFX7-NEXT:    v_mov_b32_e32 v1, s15
+; GFX7-NEXT:    s_bfe_i32 s17, s1, 0x40010
+; GFX7-NEXT:    v_mad_i32_i24 v0, s14, v1, v0
+; GFX7-NEXT:    s_bfe_i32 s16, s0, 0x40010
+; GFX7-NEXT:    v_mov_b32_e32 v1, s17
+; GFX7-NEXT:    s_bfe_i32 s19, s1, 0x40014
+; GFX7-NEXT:    s_bfe_i32 s21, s1, 0x40018
+; GFX7-NEXT:    v_mad_i32_i24 v0, s16, v1, v0
+; GFX7-NEXT:    s_bfe_i32 s18, s0, 0x40014
+; GFX7-NEXT:    v_mov_b32_e32 v1, s19
+; GFX7-NEXT:    s_bfe_i32 s20, s0, 0x40018
+; GFX7-NEXT:    v_mad_i32_i24 v0, s18, v1, v0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s21
+; GFX7-NEXT:    s_ashr_i32 s1, s1, 28
+; GFX7-NEXT:    v_mad_i32_i24 v0, s20, v1, v0
+; GFX7-NEXT:    s_ashr_i32 s0, s0, 28
+; GFX7-NEXT:    v_mov_b32_e32 v1, s1
+; GFX7-NEXT:    v_mad_i32_i24 v0, s0, v1, v0
+; GFX7-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX8-LABEL: idot8_acc32:
+; GFX8:       ; %bb.0: ; %entry
+; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX8-NEXT:    s_load_dword s4, s[6:7], 0x0
+; GFX8-NEXT:    s_load_dword s5, s[0:1], 0x0
+; GFX8-NEXT:    v_mov_b32_e32 v0, s0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    s_bfe_i32 s0, s2, 0x40000
+; GFX8-NEXT:    s_bfe_i32 s1, s4, 0x40000
+; GFX8-NEXT:    s_bfe_i32 s7, s4, 0x40004
+; GFX8-NEXT:    v_mov_b32_e32 v2, s1
+; GFX8-NEXT:    v_mov_b32_e32 v3, s5
+; GFX8-NEXT:    v_mad_i32_i24 v2, s0, v2, v3
+; GFX8-NEXT:    s_bfe_i32 s6, s2, 0x40004
+; GFX8-NEXT:    v_mov_b32_e32 v3, s7
+; GFX8-NEXT:    s_bfe_i32 s9, s4, 0x40008
+; GFX8-NEXT:    v_mad_i32_i24 v2, s6, v3, v2
+; GFX8-NEXT:    s_bfe_i32 s8, s2, 0x40008
+; GFX8-NEXT:    v_mov_b32_e32 v3, s9
+; GFX8-NEXT:    s_bfe_i32 s11, s4, 0x4000c
+; GFX8-NEXT:    v_mad_i32_i24 v2, s8, v3, v2
+; GFX8-NEXT:    s_bfe_i32 s10, s2, 0x4000c
+; GFX8-NEXT:    v_mov_b32_e32 v3, s11
+; GFX8-NEXT:    s_bfe_i32 s13, s4, 0x40010
+; GFX8-NEXT:    v_mad_i32_i24 v2, s10, v3, v2
+; GFX8-NEXT:    s_bfe_i32 s12, s2, 0x40010
+; GFX8-NEXT:    v_mov_b32_e32 v3, s13
+; GFX8-NEXT:    s_bfe_i32 s15, s4, 0x40014
+; GFX8-NEXT:    s_bfe_i32 s17, s4, 0x40018
+; GFX8-NEXT:    v_mad_i32_i24 v2, s12, v3, v2
+; GFX8-NEXT:    s_bfe_i32 s14, s2, 0x40014
+; GFX8-NEXT:    v_mov_b32_e32 v3, s15
+; GFX8-NEXT:    s_bfe_i32 s16, s2, 0x40018
+; GFX8-NEXT:    v_mad_i32_i24 v2, s14, v3, v2
+; GFX8-NEXT:    v_mov_b32_e32 v3, s17
+; GFX8-NEXT:    s_ashr_i32 s4, s4, 28
+; GFX8-NEXT:    v_mad_i32_i24 v2, s16, v3, v2
+; GFX8-NEXT:    s_ashr_i32 s2, s2, 28
+; GFX8-NEXT:    v_mov_b32_e32 v3, s4
+; GFX8-NEXT:    v_mad_i32_i24 v2, s2, v3, v2
+; GFX8-NEXT:    flat_store_dword v[0:1], v2
+; GFX8-NEXT:    s_endpgm
+;
+; GFX9-LABEL: idot8_acc32:
+; GFX9:       ; %bb.0: ; %entry
+; GFX9-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX9-NEXT:    s_load_dword s4, s[6:7], 0x0
+; GFX9-NEXT:    s_load_dword s5, s[0:1], 0x0
+; GFX9-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    s_bfe_i32 s0, s2, 0x40000
+; GFX9-NEXT:    s_bfe_i32 s1, s4, 0x40000
+; GFX9-NEXT:    s_bfe_i32 s7, s4, 0x40004
+; GFX9-NEXT:    v_mov_b32_e32 v2, s1
+; GFX9-NEXT:    v_mov_b32_e32 v3, s5
+; GFX9-NEXT:    v_mad_i32_i24 v2, s0, v2, v3
+; GFX9-NEXT:    s_bfe_i32 s6, s2, 0x40004
+; GFX9-NEXT:    v_mov_b32_e32 v3, s7
+; GFX9-NEXT:    s_bfe_i32 s9, s4, 0x40008
+; GFX9-NEXT:    v_mad_i32_i24 v2, s6, v3, v2
+; GFX9-NEXT:    s_bfe_i32 s8, s2, 0x40008
+; GFX9-NEXT:    v_mov_b32_e32 v3, s9
+; GFX9-NEXT:    s_bfe_i32 s11, s4, 0x4000c
+; GFX9-NEXT:    v_mad_i32_i24 v2, s8, v3, v2
+; GFX9-NEXT:    s_bfe_i32 s10, s2, 0x4000c
+; GFX9-NEXT:    v_mov_b32_e32 v3, s11
+; GFX9-NEXT:    s_bfe_i32 s13, s4, 0x40010
+; GFX9-NEXT:    v_mad_i32_i24 v2, s10, v3, v2
+; GFX9-NEXT:    s_bfe_i32 s12, s2, 0x40010
+; GFX9-NEXT:    v_mov_b32_e32 v3, s13
+; GFX9-NEXT:    s_bfe_i32 s15, s4, 0x40014
+; GFX9-NEXT:    s_bfe_i32 s17, s4, 0x40018
+; GFX9-NEXT:    v_mad_i32_i24 v2, s12, v3, v2
+; GFX9-NEXT:    s_bfe_i32 s14, s2, 0x40014
+; GFX9-NEXT:    v_mov_b32_e32 v3, s15
+; GFX9-NEXT:    s_bfe_i32 s16, s2, 0x40018
+; GFX9-NEXT:    v_mad_i32_i24 v2, s14, v3, v2
+; GFX9-NEXT:    v_mov_b32_e32 v3, s17
+; GFX9-NEXT:    s_ashr_i32 s4, s4, 28
+; GFX9-NEXT:    v_mad_i32_i24 v2, s16, v3, v2
+; GFX9-NEXT:    s_ashr_i32 s2, s2, 28
+; GFX9-NEXT:    v_mov_b32_e32 v3, s4
+; GFX9-NEXT:    v_mad_i32_i24 v2, s2, v3, v2
+; GFX9-NEXT:    global_store_dword v[0:1], v2, off
+; GFX9-NEXT:    s_endpgm
+;
+; GFX9-DL-LABEL: idot8_acc32:
+; GFX9-DL:       ; %bb.0: ; %entry
+; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX9-DL-NEXT:    s_load_dword s4, s[6:7], 0x0
+; GFX9-DL-NEXT:    s_load_dword s5, s[0:1], 0x0
+; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s4
+; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s5
+; GFX9-DL-NEXT:    v_dot8_i32_i4 v2, s2, v2, v3
+; GFX9-DL-NEXT:    global_store_dword v[0:1], v2, off
+; GFX9-DL-NEXT:    s_endpgm
+                                       <8 x i4> addrspace(1)* %src2,
+                                       i32 addrspace(1)* nocapture %dst) {
+entry:
+  %vec1 = load <8 x i4>, <8 x i4> addrspace(1)* %src1
+  %vec2 = load <8 x i4>, <8 x i4> addrspace(1)* %src2
+
+  %v1e0 = extractelement <8 x i4> %vec1, i64 0
+  %cv1e0 = sext i4 %v1e0 to i32
+  %v2e0 = extractelement <8 x i4> %vec2, i64 0
+  %cv2e0 = sext i4 %v2e0 to i32
+  %mul0 = mul nuw nsw i32 %cv1e0, %cv2e0
+
+  %v1e1 = extractelement <8 x i4> %vec1, i64 1
+  %cv1e1 = sext i4 %v1e1 to i32
+  %v2e1 = extractelement <8 x i4> %vec2, i64 1
+  %cv2e1 = sext i4 %v2e1 to i32
+  %mul1 = mul nuw nsw i32 %cv1e1, %cv2e1
+
+  %v1e2 = extractelement <8 x i4> %vec1, i64 2
+  %cv1e2 = sext i4 %v1e2 to i32
+  %v2e2 = extractelement <8 x i4> %vec2, i64 2
+  %cv2e2 = sext i4 %v2e2 to i32
+  %mul2 = mul nuw nsw i32 %cv1e2, %cv2e2
+
+  %v1e3 = extractelement <8 x i4> %vec1, i64 3
+  %cv1e3 = sext i4 %v1e3 to i32
+  %v2e3 = extractelement <8 x i4> %vec2, i64 3
+  %cv2e3 = sext i4 %v2e3 to i32
+  %mul3 = mul nuw nsw i32 %cv1e3, %cv2e3
+
+  %v1e4 = extractelement <8 x i4> %vec1, i64 4
+  %cv1e4 = sext i4 %v1e4 to i32
+  %v2e4 = extractelement <8 x i4> %vec2, i64 4
+  %cv2e4 = sext i4 %v2e4 to i32
+  %mul4 = mul nuw nsw i32 %cv1e4, %cv2e4
+
+  %v1e5 = extractelement <8 x i4> %vec1, i64 5
+  %cv1e5 = sext i4 %v1e5 to i32
+  %v2e5 = extractelement <8 x i4> %vec2, i64 5
+  %cv2e5 = sext i4 %v2e5 to i32
+  %mul5 = mul nuw nsw i32 %cv1e5, %cv2e5
+
+  %v1e6 = extractelement <8 x i4> %vec1, i64 6
+  %cv1e6 = sext i4 %v1e6 to i32
+  %v2e6 = extractelement <8 x i4> %vec2, i64 6
+  %cv2e6 = sext i4 %v2e6 to i32
+  %mul6 = mul nuw nsw i32 %cv1e6, %cv2e6
+
+  %v1e7 = extractelement <8 x i4> %vec1, i64 7
+  %cv1e7 = sext i4 %v1e7 to i32
+  %v2e7 = extractelement <8 x i4> %vec2, i64 7
+  %cv2e7 = sext i4 %v2e7 to i32
+  %mul7 = mul nuw nsw i32 %cv1e7, %cv2e7
+
+  %acc = load i32, i32 addrspace(1)* %dst, align 4
+  %add1 = add i32 %mul0, %acc
+  %add2 = add i32 %add1, %mul1
+  %add3 = add i32 %add2, %mul2
+  %add4 = add i32 %add3, %mul3
+  %add5 = add i32 %add4, %mul4
+  %add6 = add i32 %add5, %mul5
+  %add7 = add i32 %add6, %mul6
+  %add8 = add i32 %add7, %mul7
+
+  store i32 %add8, i32 addrspace(1)* %dst, align 4
+  ret void
+}
+
+; TODO: Once the unnecessary zero extentions of the elements are removed;
+; pattern recognizer will kick in.
+define amdgpu_kernel void @idot8_acc16(<8 x i4> addrspace(1)* %src1,
+; GFX7-LABEL: idot8_acc16:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[8:11], s[0:1], 0x9
+; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0xd
+; GFX7-NEXT:    s_mov_b32 s7, 0xf000
+; GFX7-NEXT:    s_mov_b32 s6, -1
+; GFX7-NEXT:    s_mov_b32 s0, 0xffff
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_load_dword s1, s[8:9], 0x0
+; GFX7-NEXT:    buffer_load_ushort v0, off, s[4:7], 0
+; GFX7-NEXT:    s_load_dword s2, s[10:11], 0x0
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_bfe_i32 s8, s1, 0x40000
+; GFX7-NEXT:    s_bfe_i32 s10, s1, 0x40004
+; GFX7-NEXT:    s_bfe_i32 s9, s2, 0x40000
+; GFX7-NEXT:    s_bfe_i32 s11, s2, 0x40004
+; GFX7-NEXT:    s_and_b32 s9, s9, s0
+; GFX7-NEXT:    s_bfe_i32 s13, s2, 0x40008
+; GFX7-NEXT:    s_and_b32 s11, s11, s0
+; GFX7-NEXT:    s_and_b32 s8, s8, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s9
+; GFX7-NEXT:    s_bfe_i32 s12, s1, 0x40008
+; GFX7-NEXT:    s_bfe_i32 s15, s2, 0x4000c
+; GFX7-NEXT:    s_and_b32 s13, s13, s0
+; GFX7-NEXT:    s_and_b32 s10, s10, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s11
+; GFX7-NEXT:    s_bfe_i32 s14, s1, 0x4000c
+; GFX7-NEXT:    s_bfe_i32 s17, s2, 0x40010
+; GFX7-NEXT:    s_and_b32 s15, s15, s0
+; GFX7-NEXT:    s_and_b32 s12, s12, s0
+; GFX7-NEXT:    v_mov_b32_e32 v3, s13
+; GFX7-NEXT:    s_bfe_i32 s16, s1, 0x40010
+; GFX7-NEXT:    s_bfe_i32 s19, s2, 0x40014
+; GFX7-NEXT:    s_and_b32 s17, s17, s0
+; GFX7-NEXT:    s_and_b32 s14, s14, s0
+; GFX7-NEXT:    v_mov_b32_e32 v4, s15
+; GFX7-NEXT:    s_bfe_i32 s21, s2, 0x40018
+; GFX7-NEXT:    s_bfe_i32 s18, s1, 0x40014
+; GFX7-NEXT:    s_and_b32 s19, s19, s0
+; GFX7-NEXT:    s_and_b32 s16, s16, s0
+; GFX7-NEXT:    v_mov_b32_e32 v5, s17
+; GFX7-NEXT:    s_bfe_i32 s20, s1, 0x40018
+; GFX7-NEXT:    s_ashr_i32 s2, s2, 28
+; GFX7-NEXT:    s_and_b32 s21, s21, s0
+; GFX7-NEXT:    s_and_b32 s18, s18, s0
+; GFX7-NEXT:    v_mov_b32_e32 v6, s19
+; GFX7-NEXT:    s_ashr_i32 s1, s1, 28
+; GFX7-NEXT:    s_and_b32 s20, s20, s0
+; GFX7-NEXT:    s_and_b32 s2, s2, s0
+; GFX7-NEXT:    v_mov_b32_e32 v7, s21
+; GFX7-NEXT:    s_and_b32 s0, s1, s0
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    v_mad_u32_u24 v0, s8, v1, v0
+; GFX7-NEXT:    v_mad_u32_u24 v0, s10, v2, v0
+; GFX7-NEXT:    v_mad_u32_u24 v0, s12, v3, v0
+; GFX7-NEXT:    v_mad_u32_u24 v0, s14, v4, v0
+; GFX7-NEXT:    v_mad_u32_u24 v0, s16, v5, v0
+; GFX7-NEXT:    v_mad_u32_u24 v0, s18, v6, v0
+; GFX7-NEXT:    v_mad_u32_u24 v0, s20, v7, v0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mad_u32_u24 v0, s0, v1, v0
+; GFX7-NEXT:    buffer_store_short v0, off, s[4:7], 0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX8-LABEL: idot8_acc16:
+; GFX8:       ; %bb.0: ; %entry
+; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX8-NEXT:    s_load_dword s4, s[6:7], 0x0
+; GFX8-NEXT:    v_mov_b32_e32 v0, s0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-NEXT:    flat_load_ushort v2, v[0:1]
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    s_lshr_b32 s0, s2, 4
+; GFX8-NEXT:    s_lshr_b32 s1, s4, 4
+; GFX8-NEXT:    s_bfe_i32 s5, s4, 0x40000
+; GFX8-NEXT:    v_lshlrev_b16_e64 v3, 12, s0
+; GFX8-NEXT:    v_lshlrev_b16_e64 v4, 12, s1
+; GFX8-NEXT:    s_bfe_i32 s0, s4, 0x40008
+; GFX8-NEXT:    v_mov_b32_e32 v5, s5
+; GFX8-NEXT:    s_bfe_i32 s6, s2, 0x40000
+; GFX8-NEXT:    s_lshr_b32 s1, s2, 12
+; GFX8-NEXT:    s_lshr_b32 s5, s4, 12
+; GFX8-NEXT:    v_mov_b32_e32 v6, s0
+; GFX8-NEXT:    s_bfe_i32 s7, s2, 0x40008
+; GFX8-NEXT:    v_ashrrev_i16_e32 v3, 12, v3
+; GFX8-NEXT:    v_ashrrev_i16_e32 v4, 12, v4
+; GFX8-NEXT:    v_lshlrev_b16_e64 v7, 12, s1
+; GFX8-NEXT:    v_lshlrev_b16_e64 v8, 12, s5
+; GFX8-NEXT:    v_mul_i32_i24_e32 v6, s7, v6
+; GFX8-NEXT:    s_lshr_b32 s0, s2, 20
+; GFX8-NEXT:    s_lshr_b32 s1, s4, 20
+; GFX8-NEXT:    s_bfe_i32 s5, s4, 0x40010
+; GFX8-NEXT:    v_ashrrev_i16_e32 v7, 12, v7
+; GFX8-NEXT:    v_ashrrev_i16_e32 v8, 12, v8
+; GFX8-NEXT:    v_lshlrev_b16_e64 v9, 12, s0
+; GFX8-NEXT:    v_lshlrev_b16_e64 v10, 12, s1
+; GFX8-NEXT:    s_bfe_i32 s8, s2, 0x40010
+; GFX8-NEXT:    v_mov_b32_e32 v13, s5
+; GFX8-NEXT:    s_lshr_b32 s0, s2, 28
+; GFX8-NEXT:    s_lshr_b32 s9, s4, 28
+; GFX8-NEXT:    s_bfe_i32 s4, s4, 0x40018
+; GFX8-NEXT:    v_ashrrev_i16_e32 v9, 12, v9
+; GFX8-NEXT:    v_ashrrev_i16_e32 v10, 12, v10
+; GFX8-NEXT:    v_lshlrev_b16_e64 v11, 12, s0
+; GFX8-NEXT:    v_lshlrev_b16_e64 v12, 12, s9
+; GFX8-NEXT:    s_bfe_i32 s2, s2, 0x40018
+; GFX8-NEXT:    v_ashrrev_i16_e32 v11, 12, v11
+; GFX8-NEXT:    v_ashrrev_i16_e32 v12, 12, v12
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    v_mad_i32_i24 v2, s6, v5, v2
+; GFX8-NEXT:    v_mad_u32_u24 v2, v3, v4, v2
+; GFX8-NEXT:    v_add_u32_sdwa v2, vcc, v6, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
+; GFX8-NEXT:    v_mad_u32_u24 v2, v7, v8, v2
+; GFX8-NEXT:    v_mad_i32_i24 v2, s8, v13, v2
+; GFX8-NEXT:    v_mad_u32_u24 v2, v9, v10, v2
+; GFX8-NEXT:    v_mov_b32_e32 v3, s4
+; GFX8-NEXT:    v_mad_i32_i24 v2, s2, v3, v2
+; GFX8-NEXT:    v_mad_u32_u24 v2, v11, v12, v2
+; GFX8-NEXT:    flat_store_short v[0:1], v2
+; GFX8-NEXT:    s_endpgm
+;
+; GFX9-LABEL: idot8_acc16:
+; GFX9:       ; %bb.0: ; %entry
+; GFX9-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX9-NEXT:    s_load_dword s4, s[6:7], 0x0
+; GFX9-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    global_load_ushort v2, v[0:1], off
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    s_lshr_b32 s0, s2, 4
+; GFX9-NEXT:    s_lshr_b32 s1, s4, 4
+; GFX9-NEXT:    s_bfe_i32 s5, s4, 0x40000
+; GFX9-NEXT:    v_lshlrev_b16_e64 v3, 12, s0
+; GFX9-NEXT:    v_lshlrev_b16_e64 v4, 12, s1
+; GFX9-NEXT:    s_bfe_i32 s0, s4, 0x40008
+; GFX9-NEXT:    v_mov_b32_e32 v5, s5
+; GFX9-NEXT:    s_bfe_i32 s6, s2, 0x40000
+; GFX9-NEXT:    s_lshr_b32 s1, s2, 12
+; GFX9-NEXT:    s_lshr_b32 s5, s4, 12
+; GFX9-NEXT:    v_mov_b32_e32 v6, s0
+; GFX9-NEXT:    s_bfe_i32 s7, s2, 0x40008
+; GFX9-NEXT:    v_ashrrev_i16_e32 v3, 12, v3
+; GFX9-NEXT:    v_ashrrev_i16_e32 v4, 12, v4
+; GFX9-NEXT:    v_lshlrev_b16_e64 v7, 12, s1
+; GFX9-NEXT:    v_lshlrev_b16_e64 v8, 12, s5
+; GFX9-NEXT:    v_mul_i32_i24_e32 v6, s7, v6
+; GFX9-NEXT:    s_lshr_b32 s0, s2, 20
+; GFX9-NEXT:    s_lshr_b32 s1, s4, 20
+; GFX9-NEXT:    s_bfe_i32 s5, s4, 0x40010
+; GFX9-NEXT:    v_ashrrev_i16_e32 v7, 12, v7
+; GFX9-NEXT:    v_ashrrev_i16_e32 v8, 12, v8
+; GFX9-NEXT:    v_lshlrev_b16_e64 v9, 12, s0
+; GFX9-NEXT:    v_lshlrev_b16_e64 v10, 12, s1
+; GFX9-NEXT:    s_bfe_i32 s8, s2, 0x40010
+; GFX9-NEXT:    v_mov_b32_e32 v13, s5
+; GFX9-NEXT:    s_lshr_b32 s0, s2, 28
+; GFX9-NEXT:    s_lshr_b32 s9, s4, 28
+; GFX9-NEXT:    s_bfe_i32 s4, s4, 0x40018
+; GFX9-NEXT:    v_ashrrev_i16_e32 v9, 12, v9
+; GFX9-NEXT:    v_ashrrev_i16_e32 v10, 12, v10
+; GFX9-NEXT:    v_lshlrev_b16_e64 v11, 12, s0
+; GFX9-NEXT:    v_lshlrev_b16_e64 v12, 12, s9
+; GFX9-NEXT:    s_bfe_i32 s2, s2, 0x40018
+; GFX9-NEXT:    v_ashrrev_i16_e32 v11, 12, v11
+; GFX9-NEXT:    v_ashrrev_i16_e32 v12, 12, v12
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_mad_i32_i24 v2, s6, v5, v2
+; GFX9-NEXT:    v_mad_u32_u24 v2, v3, v4, v2
+; GFX9-NEXT:    v_add_u32_sdwa v2, v2, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
+; GFX9-NEXT:    v_mad_u32_u24 v2, v7, v8, v2
+; GFX9-NEXT:    v_mad_i32_i24 v2, s8, v13, v2
+; GFX9-NEXT:    v_mad_u32_u24 v2, v9, v10, v2
+; GFX9-NEXT:    v_mov_b32_e32 v3, s4
+; GFX9-NEXT:    v_mad_i32_i24 v2, s2, v3, v2
+; GFX9-NEXT:    v_mad_u32_u24 v2, v11, v12, v2
+; GFX9-NEXT:    global_store_short v[0:1], v2, off
+; GFX9-NEXT:    s_endpgm
+;
+; GFX9-DL-LABEL: idot8_acc16:
+; GFX9-DL:       ; %bb.0: ; %entry
+; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX9-DL-NEXT:    s_load_dword s4, s[6:7], 0x0
+; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-DL-NEXT:    global_load_ushort v2, v[0:1], off
+; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-DL-NEXT:    s_lshr_b32 s0, s2, 4
+; GFX9-DL-NEXT:    s_lshr_b32 s1, s4, 4
+; GFX9-DL-NEXT:    s_bfe_i32 s5, s4, 0x40000
+; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v3, 12, s0
+; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v4, 12, s1
+; GFX9-DL-NEXT:    s_bfe_i32 s0, s4, 0x40008
+; GFX9-DL-NEXT:    v_mov_b32_e32 v5, s5
+; GFX9-DL-NEXT:    s_bfe_i32 s6, s2, 0x40000
+; GFX9-DL-NEXT:    s_lshr_b32 s1, s2, 12
+; GFX9-DL-NEXT:    s_lshr_b32 s5, s4, 12
+; GFX9-DL-NEXT:    v_mov_b32_e32 v6, s0
+; GFX9-DL-NEXT:    s_bfe_i32 s7, s2, 0x40008
+; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v3, 12, v3
+; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v4, 12, v4
+; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v7, 12, s1
+; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v8, 12, s5
+; GFX9-DL-NEXT:    v_mul_i32_i24_e32 v6, s7, v6
+; GFX9-DL-NEXT:    s_lshr_b32 s0, s2, 20
+; GFX9-DL-NEXT:    s_lshr_b32 s1, s4, 20
+; GFX9-DL-NEXT:    s_bfe_i32 s5, s4, 0x40010
+; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v7, 12, v7
+; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v8, 12, v8
+; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v9, 12, s0
+; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v10, 12, s1
+; GFX9-DL-NEXT:    s_bfe_i32 s8, s2, 0x40010
+; GFX9-DL-NEXT:    v_mov_b32_e32 v13, s5
+; GFX9-DL-NEXT:    s_lshr_b32 s0, s2, 28
+; GFX9-DL-NEXT:    s_lshr_b32 s9, s4, 28
+; GFX9-DL-NEXT:    s_bfe_i32 s4, s4, 0x40018
+; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v9, 12, v9
+; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v10, 12, v10
+; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v11, 12, s0
+; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v12, 12, s9
+; GFX9-DL-NEXT:    s_bfe_i32 s2, s2, 0x40018
+; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v11, 12, v11
+; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v12, 12, v12
+; GFX9-DL-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s6, v5, v2
+; GFX9-DL-NEXT:    v_mad_u32_u24 v2, v3, v4, v2
+; GFX9-DL-NEXT:    v_add_u32_sdwa v2, v2, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
+; GFX9-DL-NEXT:    v_mad_u32_u24 v2, v7, v8, v2
+; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s8, v13, v2
+; GFX9-DL-NEXT:    v_mad_u32_u24 v2, v9, v10, v2
+; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s4
+; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s2, v3, v2
+; GFX9-DL-NEXT:    v_mad_u32_u24 v2, v11, v12, v2
+; GFX9-DL-NEXT:    global_store_short v[0:1], v2, off
+; GFX9-DL-NEXT:    s_endpgm
+                                       <8 x i4> addrspace(1)* %src2,
+                                       i16 addrspace(1)* nocapture %dst) {
+entry:
+  %vec1 = load <8 x i4>, <8 x i4> addrspace(1)* %src1
+  %vec2 = load <8 x i4>, <8 x i4> addrspace(1)* %src2
+
+  %v1e0 = extractelement <8 x i4> %vec1, i64 0
+  %cv1e0 = sext i4 %v1e0 to i16
+  %v2e0 = extractelement <8 x i4> %vec2, i64 0
+  %cv2e0 = sext i4 %v2e0 to i16
+  %mul0 = mul nuw nsw i16 %cv1e0, %cv2e0
+
+  %v1e1 = extractelement <8 x i4> %vec1, i64 1
+  %cv1e1 = sext i4 %v1e1 to i16
+  %v2e1 = extractelement <8 x i4> %vec2, i64 1
+  %cv2e1 = sext i4 %v2e1 to i16
+  %mul1 = mul nuw nsw i16 %cv1e1, %cv2e1
+
+  %v1e2 = extractelement <8 x i4> %vec1, i64 2
+  %cv1e2 = sext i4 %v1e2 to i16
+  %v2e2 = extractelement <8 x i4> %vec2, i64 2
+  %cv2e2 = sext i4 %v2e2 to i16
+  %mul2 = mul nuw nsw i16 %cv1e2, %cv2e2
+
+  %v1e3 = extractelement <8 x i4> %vec1, i64 3
+  %cv1e3 = sext i4 %v1e3 to i16
+  %v2e3 = extractelement <8 x i4> %vec2, i64 3
+  %cv2e3 = sext i4 %v2e3 to i16
+  %mul3 = mul nuw nsw i16 %cv1e3, %cv2e3
+
+  %v1e4 = extractelement <8 x i4> %vec1, i64 4
+  %cv1e4 = sext i4 %v1e4 to i16
+  %v2e4 = extractelement <8 x i4> %vec2, i64 4
+  %cv2e4 = sext i4 %v2e4 to i16
+  %mul4 = mul nuw nsw i16 %cv1e4, %cv2e4
+
+  %v1e5 = extractelement <8 x i4> %vec1, i64 5
+  %cv1e5 = sext i4 %v1e5 to i16
+  %v2e5 = extractelement <8 x i4> %vec2, i64 5
+  %cv2e5 = sext i4 %v2e5 to i16
+  %mul5 = mul nuw nsw i16 %cv1e5, %cv2e5
+
+  %v1e6 = extractelement <8 x i4> %vec1, i64 6
+  %cv1e6 = sext i4 %v1e6 to i16
+  %v2e6 = extractelement <8 x i4> %vec2, i64 6
+  %cv2e6 = sext i4 %v2e6 to i16
+  %mul6 = mul nuw nsw i16 %cv1e6, %cv2e6
+
+  %v1e7 = extractelement <8 x i4> %vec1, i64 7
+  %cv1e7 = sext i4 %v1e7 to i16
+  %v2e7 = extractelement <8 x i4> %vec2, i64 7
+  %cv2e7 = sext i4 %v2e7 to i16
+  %mul7 = mul nuw nsw i16 %cv1e7, %cv2e7
+
+  %acc = load i16, i16 addrspace(1)* %dst, align 4
+  %add1 = add i16 %mul0, %acc
+  %add2 = add i16 %add1, %mul1
+  %add3 = add i16 %add2, %mul2
+  %add4 = add i16 %add3, %mul3
+  %add5 = add i16 %add4, %mul4
+  %add6 = add i16 %add5, %mul5
+  %add7 = add i16 %add6, %mul6
+  %add8 = add i16 %add7, %mul7
+
+  store i16 %add8, i16 addrspace(1)* %dst, align 4
+  ret void
+}
+
+; TODO: Support this pattern.
+define amdgpu_kernel void @idot8_acc8(<8 x i4> addrspace(1)* %src1,
+; GFX7-LABEL: idot8_acc8:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[8:11], s[0:1], 0x9
+; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0xd
+; GFX7-NEXT:    s_mov_b32 s7, 0xf000
+; GFX7-NEXT:    s_mov_b32 s6, -1
+; GFX7-NEXT:    s_movk_i32 s0, 0xff
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_load_dword s1, s[8:9], 0x0
+; GFX7-NEXT:    buffer_load_ubyte v0, off, s[4:7], 0
+; GFX7-NEXT:    s_load_dword s2, s[10:11], 0x0
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_bfe_i32 s8, s1, 0x40000
+; GFX7-NEXT:    s_bfe_i32 s10, s1, 0x40004
+; GFX7-NEXT:    s_bfe_i32 s9, s2, 0x40000
+; GFX7-NEXT:    s_bfe_i32 s11, s2, 0x40004
+; GFX7-NEXT:    s_and_b32 s9, s9, s0
+; GFX7-NEXT:    s_bfe_i32 s13, s2, 0x40008
+; GFX7-NEXT:    s_and_b32 s11, s11, s0
+; GFX7-NEXT:    s_and_b32 s8, s8, s0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s9
+; GFX7-NEXT:    s_bfe_i32 s12, s1, 0x40008
+; GFX7-NEXT:    s_bfe_i32 s15, s2, 0x4000c
+; GFX7-NEXT:    s_and_b32 s13, s13, s0
+; GFX7-NEXT:    s_and_b32 s10, s10, s0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s11
+; GFX7-NEXT:    s_bfe_i32 s14, s1, 0x4000c
+; GFX7-NEXT:    s_bfe_i32 s17, s2, 0x40010
+; GFX7-NEXT:    s_and_b32 s15, s15, s0
+; GFX7-NEXT:    s_and_b32 s12, s12, s0
+; GFX7-NEXT:    v_mov_b32_e32 v3, s13
+; GFX7-NEXT:    s_bfe_i32 s16, s1, 0x40010
+; GFX7-NEXT:    s_bfe_i32 s19, s2, 0x40014
+; GFX7-NEXT:    s_and_b32 s17, s17, s0
+; GFX7-NEXT:    s_and_b32 s14, s14, s0
+; GFX7-NEXT:    v_mov_b32_e32 v4, s15
+; GFX7-NEXT:    s_bfe_i32 s21, s2, 0x40018
+; GFX7-NEXT:    s_bfe_i32 s18, s1, 0x40014
+; GFX7-NEXT:    s_and_b32 s19, s19, s0
+; GFX7-NEXT:    s_and_b32 s16, s16, s0
+; GFX7-NEXT:    v_mov_b32_e32 v5, s17
+; GFX7-NEXT:    s_bfe_i32 s20, s1, 0x40018
+; GFX7-NEXT:    s_ashr_i32 s2, s2, 28
+; GFX7-NEXT:    s_and_b32 s21, s21, s0
+; GFX7-NEXT:    s_and_b32 s18, s18, s0
+; GFX7-NEXT:    v_mov_b32_e32 v6, s19
+; GFX7-NEXT:    s_ashr_i32 s1, s1, 28
+; GFX7-NEXT:    s_and_b32 s20, s20, s0
+; GFX7-NEXT:    s_and_b32 s2, s2, s0
+; GFX7-NEXT:    v_mov_b32_e32 v7, s21
+; GFX7-NEXT:    s_and_b32 s0, s1, s0
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    v_mad_u32_u24 v0, s8, v1, v0
+; GFX7-NEXT:    v_mad_u32_u24 v0, s10, v2, v0
+; GFX7-NEXT:    v_mad_u32_u24 v0, s12, v3, v0
+; GFX7-NEXT:    v_mad_u32_u24 v0, s14, v4, v0
+; GFX7-NEXT:    v_mad_u32_u24 v0, s16, v5, v0
+; GFX7-NEXT:    v_mad_u32_u24 v0, s18, v6, v0
+; GFX7-NEXT:    v_mad_u32_u24 v0, s20, v7, v0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mad_u32_u24 v0, s0, v1, v0
+; GFX7-NEXT:    buffer_store_byte v0, off, s[4:7], 0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX8-LABEL: idot8_acc8:
+; GFX8:       ; %bb.0: ; %entry
+; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX8-NEXT:    s_movk_i32 s2, 0xff
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    v_mov_b32_e32 v0, s0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-NEXT:    flat_load_ubyte v2, v[0:1]
+; GFX8-NEXT:    s_load_dword s0, s[4:5], 0x0
+; GFX8-NEXT:    s_load_dword s1, s[6:7], 0x0
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    s_lshr_b32 s7, s0, 4
+; GFX8-NEXT:    s_lshr_b32 s11, s1, 4
+; GFX8-NEXT:    v_lshlrev_b16_e64 v3, 12, s7
+; GFX8-NEXT:    v_lshlrev_b16_e64 v4, 12, s11
+; GFX8-NEXT:    s_bfe_i32 s13, s1, 0x40000
+; GFX8-NEXT:    v_ashrrev_i16_e32 v3, 12, v3
+; GFX8-NEXT:    v_ashrrev_i16_e32 v4, 12, v4
+; GFX8-NEXT:    s_lshr_b32 s6, s0, 12
+; GFX8-NEXT:    s_lshr_b32 s10, s1, 12
+; GFX8-NEXT:    s_bfe_i32 s15, s1, 0x40008
+; GFX8-NEXT:    s_bfe_i32 s12, s0, 0x40000
+; GFX8-NEXT:    v_mov_b32_e32 v12, s13
+; GFX8-NEXT:    v_lshlrev_b16_e64 v6, 12, s6
+; GFX8-NEXT:    v_lshlrev_b16_e64 v7, 12, s10
+; GFX8-NEXT:    s_bfe_i32 s14, s0, 0x40008
+; GFX8-NEXT:    v_mov_b32_e32 v5, s15
+; GFX8-NEXT:    v_and_b32_e32 v3, s2, v3
+; GFX8-NEXT:    v_and_b32_e32 v4, s2, v4
+; GFX8-NEXT:    v_ashrrev_i16_e32 v6, 12, v6
+; GFX8-NEXT:    v_ashrrev_i16_e32 v7, 12, v7
+; GFX8-NEXT:    s_lshr_b32 s5, s0, 20
+; GFX8-NEXT:    s_lshr_b32 s9, s1, 20
+; GFX8-NEXT:    v_mul_i32_i24_e32 v5, s14, v5
+; GFX8-NEXT:    v_lshlrev_b16_e64 v8, 12, s5
+; GFX8-NEXT:    v_lshlrev_b16_e64 v9, 12, s9
+; GFX8-NEXT:    s_bfe_i32 s17, s1, 0x40010
+; GFX8-NEXT:    v_and_b32_e32 v6, s2, v6
+; GFX8-NEXT:    v_and_b32_e32 v7, s2, v7
+; GFX8-NEXT:    s_lshr_b32 s8, s1, 28
+; GFX8-NEXT:    v_ashrrev_i16_e32 v8, 12, v8
+; GFX8-NEXT:    v_ashrrev_i16_e32 v9, 12, v9
+; GFX8-NEXT:    s_lshr_b32 s4, s0, 28
+; GFX8-NEXT:    s_bfe_i32 s16, s0, 0x40010
+; GFX8-NEXT:    v_mov_b32_e32 v13, s17
+; GFX8-NEXT:    v_lshlrev_b16_e64 v10, 12, s4
+; GFX8-NEXT:    v_lshlrev_b16_e64 v11, 12, s8
+; GFX8-NEXT:    s_bfe_i32 s1, s1, 0x40018
+; GFX8-NEXT:    v_and_b32_e32 v8, s2, v8
+; GFX8-NEXT:    v_and_b32_e32 v9, s2, v9
+; GFX8-NEXT:    v_ashrrev_i16_e32 v10, 12, v10
+; GFX8-NEXT:    v_ashrrev_i16_e32 v11, 12, v11
+; GFX8-NEXT:    s_bfe_i32 s0, s0, 0x40018
+; GFX8-NEXT:    v_and_b32_e32 v10, s2, v10
+; GFX8-NEXT:    v_and_b32_e32 v11, s2, v11
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    v_mad_i32_i24 v2, s12, v12, v2
+; GFX8-NEXT:    v_mad_u32_u24 v2, v3, v4, v2
+; GFX8-NEXT:    v_add_u32_sdwa v2, vcc, v5, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX8-NEXT:    v_mad_u32_u24 v2, v6, v7, v2
+; GFX8-NEXT:    v_mad_i32_i24 v2, s16, v13, v2
+; GFX8-NEXT:    v_mad_u32_u24 v2, v8, v9, v2
+; GFX8-NEXT:    v_mov_b32_e32 v3, s1
+; GFX8-NEXT:    v_mad_i32_i24 v2, s0, v3, v2
+; GFX8-NEXT:    v_mad_u32_u24 v2, v10, v11, v2
+; GFX8-NEXT:    flat_store_byte v[0:1], v2
+; GFX8-NEXT:    s_endpgm
+;
+; GFX9-LABEL: idot8_acc8:
+; GFX9:       ; %bb.0: ; %entry
+; GFX9-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX9-NEXT:    s_movk_i32 s2, 0xff
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    global_load_ubyte v2, v[0:1], off
+; GFX9-NEXT:    s_load_dword s0, s[4:5], 0x0
+; GFX9-NEXT:    s_load_dword s1, s[6:7], 0x0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    s_lshr_b32 s7, s0, 4
+; GFX9-NEXT:    s_lshr_b32 s11, s1, 4
+; GFX9-NEXT:    v_lshlrev_b16_e64 v3, 12, s7
+; GFX9-NEXT:    v_lshlrev_b16_e64 v4, 12, s11
+; GFX9-NEXT:    s_bfe_i32 s13, s1, 0x40000
+; GFX9-NEXT:    v_ashrrev_i16_e32 v3, 12, v3
+; GFX9-NEXT:    v_ashrrev_i16_e32 v4, 12, v4
+; GFX9-NEXT:    s_lshr_b32 s6, s0, 12
+; GFX9-NEXT:    s_lshr_b32 s10, s1, 12
+; GFX9-NEXT:    s_bfe_i32 s15, s1, 0x40008
+; GFX9-NEXT:    s_bfe_i32 s12, s0, 0x40000
+; GFX9-NEXT:    v_mov_b32_e32 v12, s13
+; GFX9-NEXT:    v_lshlrev_b16_e64 v6, 12, s6
+; GFX9-NEXT:    v_lshlrev_b16_e64 v7, 12, s10
+; GFX9-NEXT:    s_bfe_i32 s14, s0, 0x40008
+; GFX9-NEXT:    v_mov_b32_e32 v5, s15
+; GFX9-NEXT:    v_and_b32_e32 v3, s2, v3
+; GFX9-NEXT:    v_and_b32_e32 v4, s2, v4
+; GFX9-NEXT:    v_ashrrev_i16_e32 v6, 12, v6
+; GFX9-NEXT:    v_ashrrev_i16_e32 v7, 12, v7
+; GFX9-NEXT:    s_lshr_b32 s5, s0, 20
+; GFX9-NEXT:    s_lshr_b32 s9, s1, 20
+; GFX9-NEXT:    v_mul_i32_i24_e32 v5, s14, v5
+; GFX9-NEXT:    v_lshlrev_b16_e64 v8, 12, s5
+; GFX9-NEXT:    v_lshlrev_b16_e64 v9, 12, s9
+; GFX9-NEXT:    s_bfe_i32 s17, s1, 0x40010
+; GFX9-NEXT:    v_and_b32_e32 v6, s2, v6
+; GFX9-NEXT:    v_and_b32_e32 v7, s2, v7
+; GFX9-NEXT:    s_lshr_b32 s8, s1, 28
+; GFX9-NEXT:    v_ashrrev_i16_e32 v8, 12, v8
+; GFX9-NEXT:    v_ashrrev_i16_e32 v9, 12, v9
+; GFX9-NEXT:    s_lshr_b32 s4, s0, 28
+; GFX9-NEXT:    s_bfe_i32 s16, s0, 0x40010
+; GFX9-NEXT:    v_mov_b32_e32 v13, s17
+; GFX9-NEXT:    v_lshlrev_b16_e64 v10, 12, s4
+; GFX9-NEXT:    v_lshlrev_b16_e64 v11, 12, s8
+; GFX9-NEXT:    s_bfe_i32 s1, s1, 0x40018
+; GFX9-NEXT:    v_and_b32_e32 v8, s2, v8
+; GFX9-NEXT:    v_and_b32_e32 v9, s2, v9
+; GFX9-NEXT:    v_ashrrev_i16_e32 v10, 12, v10
+; GFX9-NEXT:    v_ashrrev_i16_e32 v11, 12, v11
+; GFX9-NEXT:    s_bfe_i32 s0, s0, 0x40018
+; GFX9-NEXT:    v_and_b32_e32 v10, s2, v10
+; GFX9-NEXT:    v_and_b32_e32 v11, s2, v11
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_mad_i32_i24 v2, s12, v12, v2
+; GFX9-NEXT:    v_mad_u32_u24 v2, v3, v4, v2
+; GFX9-NEXT:    v_add_u32_sdwa v2, v2, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX9-NEXT:    v_mad_u32_u24 v2, v6, v7, v2
+; GFX9-NEXT:    v_mad_i32_i24 v2, s16, v13, v2
+; GFX9-NEXT:    v_mad_u32_u24 v2, v8, v9, v2
+; GFX9-NEXT:    v_mov_b32_e32 v3, s1
+; GFX9-NEXT:    v_mad_i32_i24 v2, s0, v3, v2
+; GFX9-NEXT:    v_mad_u32_u24 v2, v10, v11, v2
+; GFX9-NEXT:    global_store_byte v[0:1], v2, off
+; GFX9-NEXT:    s_endpgm
+;
+; GFX9-DL-LABEL: idot8_acc8:
+; GFX9-DL:       ; %bb.0: ; %entry
+; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX9-DL-NEXT:    s_movk_i32 s2, 0xff
+; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-DL-NEXT:    global_load_ubyte v2, v[0:1], off
+; GFX9-DL-NEXT:    s_load_dword s0, s[4:5], 0x0
+; GFX9-DL-NEXT:    s_load_dword s1, s[6:7], 0x0
+; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-DL-NEXT:    s_lshr_b32 s7, s0, 4
+; GFX9-DL-NEXT:    s_lshr_b32 s11, s1, 4
+; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v3, 12, s7
+; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v4, 12, s11
+; GFX9-DL-NEXT:    s_bfe_i32 s13, s1, 0x40000
+; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v3, 12, v3
+; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v4, 12, v4
+; GFX9-DL-NEXT:    s_lshr_b32 s6, s0, 12
+; GFX9-DL-NEXT:    s_lshr_b32 s10, s1, 12
+; GFX9-DL-NEXT:    s_bfe_i32 s15, s1, 0x40008
+; GFX9-DL-NEXT:    s_bfe_i32 s12, s0, 0x40000
+; GFX9-DL-NEXT:    v_mov_b32_e32 v12, s13
+; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v6, 12, s6
+; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v7, 12, s10
+; GFX9-DL-NEXT:    s_bfe_i32 s14, s0, 0x40008
+; GFX9-DL-NEXT:    v_mov_b32_e32 v5, s15
+; GFX9-DL-NEXT:    v_and_b32_e32 v3, s2, v3
+; GFX9-DL-NEXT:    v_and_b32_e32 v4, s2, v4
+; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v6, 12, v6
+; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v7, 12, v7
+; GFX9-DL-NEXT:    s_lshr_b32 s5, s0, 20
+; GFX9-DL-NEXT:    s_lshr_b32 s9, s1, 20
+; GFX9-DL-NEXT:    v_mul_i32_i24_e32 v5, s14, v5
+; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v8, 12, s5
+; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v9, 12, s9
+; GFX9-DL-NEXT:    s_bfe_i32 s17, s1, 0x40010
+; GFX9-DL-NEXT:    v_and_b32_e32 v6, s2, v6
+; GFX9-DL-NEXT:    v_and_b32_e32 v7, s2, v7
+; GFX9-DL-NEXT:    s_lshr_b32 s8, s1, 28
+; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v8, 12, v8
+; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v9, 12, v9
+; GFX9-DL-NEXT:    s_lshr_b32 s4, s0, 28
+; GFX9-DL-NEXT:    s_bfe_i32 s16, s0, 0x40010
+; GFX9-DL-NEXT:    v_mov_b32_e32 v13, s17
+; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v10, 12, s4
+; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v11, 12, s8
+; GFX9-DL-NEXT:    s_bfe_i32 s1, s1, 0x40018
+; GFX9-DL-NEXT:    v_and_b32_e32 v8, s2, v8
+; GFX9-DL-NEXT:    v_and_b32_e32 v9, s2, v9
+; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v10, 12, v10
+; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v11, 12, v11
+; GFX9-DL-NEXT:    s_bfe_i32 s0, s0, 0x40018
+; GFX9-DL-NEXT:    v_and_b32_e32 v10, s2, v10
+; GFX9-DL-NEXT:    v_and_b32_e32 v11, s2, v11
+; GFX9-DL-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s12, v12, v2
+; GFX9-DL-NEXT:    v_mad_u32_u24 v2, v3, v4, v2
+; GFX9-DL-NEXT:    v_add_u32_sdwa v2, v2, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX9-DL-NEXT:    v_mad_u32_u24 v2, v6, v7, v2
+; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s16, v13, v2
+; GFX9-DL-NEXT:    v_mad_u32_u24 v2, v8, v9, v2
+; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s1
+; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s0, v3, v2
+; GFX9-DL-NEXT:    v_mad_u32_u24 v2, v10, v11, v2
+; GFX9-DL-NEXT:    global_store_byte v[0:1], v2, off
+; GFX9-DL-NEXT:    s_endpgm
+                                       <8 x i4> addrspace(1)* %src2,
+                                       i8 addrspace(1)* nocapture %dst) {
+entry:
+  %vec1 = load <8 x i4>, <8 x i4> addrspace(1)* %src1
+  %vec2 = load <8 x i4>, <8 x i4> addrspace(1)* %src2
+
+  %v1e0 = extractelement <8 x i4> %vec1, i64 0
+  %cv1e0 = sext i4 %v1e0 to i8
+  %v2e0 = extractelement <8 x i4> %vec2, i64 0
+  %cv2e0 = sext i4 %v2e0 to i8
+  %mul0 = mul nuw nsw i8 %cv1e0, %cv2e0
+
+  %v1e1 = extractelement <8 x i4> %vec1, i64 1
+  %cv1e1 = sext i4 %v1e1 to i8
+  %v2e1 = extractelement <8 x i4> %vec2, i64 1
+  %cv2e1 = sext i4 %v2e1 to i8
+  %mul1 = mul nuw nsw i8 %cv1e1, %cv2e1
+
+  %v1e2 = extractelement <8 x i4> %vec1, i64 2
+  %cv1e2 = sext i4 %v1e2 to i8
+  %v2e2 = extractelement <8 x i4> %vec2, i64 2
+  %cv2e2 = sext i4 %v2e2 to i8
+  %mul2 = mul nuw nsw i8 %cv1e2, %cv2e2
+
+  %v1e3 = extractelement <8 x i4> %vec1, i64 3
+  %cv1e3 = sext i4 %v1e3 to i8
+  %v2e3 = extractelement <8 x i4> %vec2, i64 3
+  %cv2e3 = sext i4 %v2e3 to i8
+  %mul3 = mul nuw nsw i8 %cv1e3, %cv2e3
+
+  %v1e4 = extractelement <8 x i4> %vec1, i64 4
+  %cv1e4 = sext i4 %v1e4 to i8
+  %v2e4 = extractelement <8 x i4> %vec2, i64 4
+  %cv2e4 = sext i4 %v2e4 to i8
+  %mul4 = mul nuw nsw i8 %cv1e4, %cv2e4
+
+  %v1e5 = extractelement <8 x i4> %vec1, i64 5
+  %cv1e5 = sext i4 %v1e5 to i8
+  %v2e5 = extractelement <8 x i4> %vec2, i64 5
+  %cv2e5 = sext i4 %v2e5 to i8
+  %mul5 = mul nuw nsw i8 %cv1e5, %cv2e5
+
+  %v1e6 = extractelement <8 x i4> %vec1, i64 6
+  %cv1e6 = sext i4 %v1e6 to i8
+  %v2e6 = extractelement <8 x i4> %vec2, i64 6
+  %cv2e6 = sext i4 %v2e6 to i8
+  %mul6 = mul nuw nsw i8 %cv1e6, %cv2e6
+
+  %v1e7 = extractelement <8 x i4> %vec1, i64 7
+  %cv1e7 = sext i4 %v1e7 to i8
+  %v2e7 = extractelement <8 x i4> %vec2, i64 7
+  %cv2e7 = sext i4 %v2e7 to i8
+  %mul7 = mul nuw nsw i8 %cv1e7, %cv2e7
+
+  %acc = load i8, i8 addrspace(1)* %dst, align 4
+  %add1 = add i8 %mul0, %acc
+  %add2 = add i8 %add1, %mul1
+  %add3 = add i8 %add2, %mul2
+  %add4 = add i8 %add3, %mul3
+  %add5 = add i8 %add4, %mul4
+  %add6 = add i8 %add5, %mul5
+  %add7 = add i8 %add6, %mul6
+  %add8 = add i8 %add7, %mul7
+
+  store i8 %add8, i8 addrspace(1)* %dst, align 4
+  ret void
+}
+
+; Make sure the pattern is not recognized if there are multiple uses of the
+; intermediate multiplications.
+define amdgpu_kernel void @idot8_multiuses_mul1(<8 x i4> addrspace(1)* %src1,
+; GFX7-LABEL: idot8_multiuses_mul1:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[8:11], s[0:1], 0x9
+; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0xd
+; GFX7-NEXT:    s_mov_b32 s7, 0xf000
+; GFX7-NEXT:    s_mov_b32 s6, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_load_dword s0, s[8:9], 0x0
+; GFX7-NEXT:    s_load_dword s1, s[10:11], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_bfe_i32 s8, s0, 0x40000
+; GFX7-NEXT:    s_bfe_i32 s9, s1, 0x40000
+; GFX7-NEXT:    v_mov_b32_e32 v0, s9
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mad_i32_i24 v1, s8, v0, v1
+; GFX7-NEXT:    s_bfe_i32 s11, s1, 0x40004
+; GFX7-NEXT:    s_bfe_i32 s10, s0, 0x40004
+; GFX7-NEXT:    s_bfe_i32 s13, s1, 0x40008
+; GFX7-NEXT:    v_mad_i32_i24 v0, s8, v0, v1
+; GFX7-NEXT:    v_mov_b32_e32 v2, s11
+; GFX7-NEXT:    v_mad_i32_i24 v0, s10, v2, v0
+; GFX7-NEXT:    s_bfe_i32 s12, s0, 0x40008
+; GFX7-NEXT:    v_mov_b32_e32 v2, s13
+; GFX7-NEXT:    s_bfe_i32 s15, s1, 0x4000c
+; GFX7-NEXT:    v_mad_i32_i24 v0, s12, v2, v0
+; GFX7-NEXT:    s_bfe_i32 s14, s0, 0x4000c
+; GFX7-NEXT:    v_mov_b32_e32 v2, s15
+; GFX7-NEXT:    s_bfe_i32 s17, s1, 0x40010
+; GFX7-NEXT:    v_mad_i32_i24 v0, s14, v2, v0
+; GFX7-NEXT:    s_bfe_i32 s16, s0, 0x40010
+; GFX7-NEXT:    v_mov_b32_e32 v2, s17
+; GFX7-NEXT:    s_bfe_i32 s19, s1, 0x40014
+; GFX7-NEXT:    s_bfe_i32 s21, s1, 0x40018
+; GFX7-NEXT:    v_mad_i32_i24 v0, s16, v2, v0
+; GFX7-NEXT:    s_bfe_i32 s18, s0, 0x40014
+; GFX7-NEXT:    v_mov_b32_e32 v2, s19
+; GFX7-NEXT:    s_bfe_i32 s20, s0, 0x40018
+; GFX7-NEXT:    v_mad_i32_i24 v0, s18, v2, v0
+; GFX7-NEXT:    v_mov_b32_e32 v2, s21
+; GFX7-NEXT:    s_ashr_i32 s1, s1, 28
+; GFX7-NEXT:    v_mad_i32_i24 v0, s20, v2, v0
+; GFX7-NEXT:    s_ashr_i32 s0, s0, 28
+; GFX7-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-NEXT:    v_mad_i32_i24 v0, s0, v2, v0
+; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
+; GFX7-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX8-LABEL: idot8_multiuses_mul1:
+; GFX8:       ; %bb.0: ; %entry
+; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX8-NEXT:    s_load_dword s4, s[6:7], 0x0
+; GFX8-NEXT:    s_load_dword s5, s[0:1], 0x0
+; GFX8-NEXT:    v_mov_b32_e32 v0, s0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    s_bfe_i32 s0, s2, 0x40000
+; GFX8-NEXT:    s_bfe_i32 s1, s4, 0x40000
+; GFX8-NEXT:    v_mov_b32_e32 v2, s1
+; GFX8-NEXT:    v_mov_b32_e32 v3, s5
+; GFX8-NEXT:    v_mad_i32_i24 v3, s0, v2, v3
+; GFX8-NEXT:    s_bfe_i32 s7, s4, 0x40004
+; GFX8-NEXT:    s_bfe_i32 s6, s2, 0x40004
+; GFX8-NEXT:    s_bfe_i32 s9, s4, 0x40008
+; GFX8-NEXT:    v_mad_i32_i24 v2, s0, v2, v3
+; GFX8-NEXT:    v_mov_b32_e32 v4, s7
+; GFX8-NEXT:    v_mad_i32_i24 v2, s6, v4, v2
+; GFX8-NEXT:    s_bfe_i32 s8, s2, 0x40008
+; GFX8-NEXT:    v_mov_b32_e32 v4, s9
+; GFX8-NEXT:    s_bfe_i32 s11, s4, 0x4000c
+; GFX8-NEXT:    v_mad_i32_i24 v2, s8, v4, v2
+; GFX8-NEXT:    s_bfe_i32 s10, s2, 0x4000c
+; GFX8-NEXT:    v_mov_b32_e32 v4, s11
+; GFX8-NEXT:    s_bfe_i32 s13, s4, 0x40010
+; GFX8-NEXT:    v_mad_i32_i24 v2, s10, v4, v2
+; GFX8-NEXT:    s_bfe_i32 s12, s2, 0x40010
+; GFX8-NEXT:    v_mov_b32_e32 v4, s13
+; GFX8-NEXT:    s_bfe_i32 s15, s4, 0x40014
+; GFX8-NEXT:    s_bfe_i32 s17, s4, 0x40018
+; GFX8-NEXT:    v_mad_i32_i24 v2, s12, v4, v2
+; GFX8-NEXT:    s_bfe_i32 s14, s2, 0x40014
+; GFX8-NEXT:    v_mov_b32_e32 v4, s15
+; GFX8-NEXT:    s_bfe_i32 s16, s2, 0x40018
+; GFX8-NEXT:    v_mad_i32_i24 v2, s14, v4, v2
+; GFX8-NEXT:    v_mov_b32_e32 v4, s17
+; GFX8-NEXT:    s_ashr_i32 s4, s4, 28
+; GFX8-NEXT:    v_mad_i32_i24 v2, s16, v4, v2
+; GFX8-NEXT:    s_ashr_i32 s2, s2, 28
+; GFX8-NEXT:    v_mov_b32_e32 v4, s4
+; GFX8-NEXT:    v_mad_i32_i24 v2, s2, v4, v2
+; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v2, v3
+; GFX8-NEXT:    flat_store_dword v[0:1], v2
+; GFX8-NEXT:    s_endpgm
+;
+; GFX9-LABEL: idot8_multiuses_mul1:
+; GFX9:       ; %bb.0: ; %entry
+; GFX9-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX9-NEXT:    s_load_dword s4, s[6:7], 0x0
+; GFX9-NEXT:    s_load_dword s5, s[0:1], 0x0
+; GFX9-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    s_bfe_i32 s0, s2, 0x40000
+; GFX9-NEXT:    s_bfe_i32 s1, s4, 0x40000
+; GFX9-NEXT:    v_mov_b32_e32 v2, s1
+; GFX9-NEXT:    v_mov_b32_e32 v3, s5
+; GFX9-NEXT:    v_mad_i32_i24 v3, s0, v2, v3
+; GFX9-NEXT:    s_bfe_i32 s7, s4, 0x40004
+; GFX9-NEXT:    s_bfe_i32 s6, s2, 0x40004
+; GFX9-NEXT:    s_bfe_i32 s9, s4, 0x40008
+; GFX9-NEXT:    v_mad_i32_i24 v2, s0, v2, v3
+; GFX9-NEXT:    v_mov_b32_e32 v4, s7
+; GFX9-NEXT:    v_mad_i32_i24 v2, s6, v4, v2
+; GFX9-NEXT:    s_bfe_i32 s8, s2, 0x40008
+; GFX9-NEXT:    v_mov_b32_e32 v4, s9
+; GFX9-NEXT:    s_bfe_i32 s11, s4, 0x4000c
+; GFX9-NEXT:    v_mad_i32_i24 v2, s8, v4, v2
+; GFX9-NEXT:    s_bfe_i32 s10, s2, 0x4000c
+; GFX9-NEXT:    v_mov_b32_e32 v4, s11
+; GFX9-NEXT:    s_bfe_i32 s13, s4, 0x40010
+; GFX9-NEXT:    v_mad_i32_i24 v2, s10, v4, v2
+; GFX9-NEXT:    s_bfe_i32 s12, s2, 0x40010
+; GFX9-NEXT:    v_mov_b32_e32 v4, s13
+; GFX9-NEXT:    s_bfe_i32 s15, s4, 0x40014
+; GFX9-NEXT:    s_bfe_i32 s17, s4, 0x40018
+; GFX9-NEXT:    v_mad_i32_i24 v2, s12, v4, v2
+; GFX9-NEXT:    s_bfe_i32 s14, s2, 0x40014
+; GFX9-NEXT:    v_mov_b32_e32 v4, s15
+; GFX9-NEXT:    s_bfe_i32 s16, s2, 0x40018
+; GFX9-NEXT:    v_mad_i32_i24 v2, s14, v4, v2
+; GFX9-NEXT:    v_mov_b32_e32 v4, s17
+; GFX9-NEXT:    s_ashr_i32 s4, s4, 28
+; GFX9-NEXT:    v_mad_i32_i24 v2, s16, v4, v2
+; GFX9-NEXT:    s_ashr_i32 s2, s2, 28
+; GFX9-NEXT:    v_mov_b32_e32 v4, s4
+; GFX9-NEXT:    v_mad_i32_i24 v2, s2, v4, v2
+; GFX9-NEXT:    v_add_u32_e32 v2, v3, v2
+; GFX9-NEXT:    global_store_dword v[0:1], v2, off
+; GFX9-NEXT:    s_endpgm
+;
+; GFX9-DL-LABEL: idot8_multiuses_mul1:
+; GFX9-DL:       ; %bb.0: ; %entry
+; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX9-DL-NEXT:    s_load_dword s4, s[6:7], 0x0
+; GFX9-DL-NEXT:    s_load_dword s5, s[0:1], 0x0
+; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-DL-NEXT:    s_bfe_i32 s0, s2, 0x40000
+; GFX9-DL-NEXT:    s_bfe_i32 s1, s4, 0x40000
+; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s1
+; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s5
+; GFX9-DL-NEXT:    v_mad_i32_i24 v3, s0, v2, v3
+; GFX9-DL-NEXT:    s_bfe_i32 s7, s4, 0x40004
+; GFX9-DL-NEXT:    s_bfe_i32 s6, s2, 0x40004
+; GFX9-DL-NEXT:    s_bfe_i32 s9, s4, 0x40008
+; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s0, v2, v3
+; GFX9-DL-NEXT:    v_mov_b32_e32 v4, s7
+; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s6, v4, v2
+; GFX9-DL-NEXT:    s_bfe_i32 s8, s2, 0x40008
+; GFX9-DL-NEXT:    v_mov_b32_e32 v4, s9
+; GFX9-DL-NEXT:    s_bfe_i32 s11, s4, 0x4000c
+; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s8, v4, v2
+; GFX9-DL-NEXT:    s_bfe_i32 s10, s2, 0x4000c
+; GFX9-DL-NEXT:    v_mov_b32_e32 v4, s11
+; GFX9-DL-NEXT:    s_bfe_i32 s13, s4, 0x40010
+; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s10, v4, v2
+; GFX9-DL-NEXT:    s_bfe_i32 s12, s2, 0x40010
+; GFX9-DL-NEXT:    v_mov_b32_e32 v4, s13
+; GFX9-DL-NEXT:    s_bfe_i32 s15, s4, 0x40014
+; GFX9-DL-NEXT:    s_bfe_i32 s17, s4, 0x40018
+; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s12, v4, v2
+; GFX9-DL-NEXT:    s_bfe_i32 s14, s2, 0x40014
+; GFX9-DL-NEXT:    v_mov_b32_e32 v4, s15
+; GFX9-DL-NEXT:    s_bfe_i32 s16, s2, 0x40018
+; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s14, v4, v2
+; GFX9-DL-NEXT:    v_mov_b32_e32 v4, s17
+; GFX9-DL-NEXT:    s_ashr_i32 s4, s4, 28
+; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s16, v4, v2
+; GFX9-DL-NEXT:    s_ashr_i32 s2, s2, 28
+; GFX9-DL-NEXT:    v_mov_b32_e32 v4, s4
+; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s2, v4, v2
+; GFX9-DL-NEXT:    v_add_u32_e32 v2, v3, v2
+; GFX9-DL-NEXT:    global_store_dword v[0:1], v2, off
+; GFX9-DL-NEXT:    s_endpgm
+                                                <8 x i4> addrspace(1)* %src2,
+                                                i32 addrspace(1)* nocapture %dst) {
+entry:
+  %vec1 = load <8 x i4>, <8 x i4> addrspace(1)* %src1
+  %vec2 = load <8 x i4>, <8 x i4> addrspace(1)* %src2
+
+  %v1e0 = extractelement <8 x i4> %vec1, i64 0
+  %cv1e0 = sext i4 %v1e0 to i32
+  %v2e0 = extractelement <8 x i4> %vec2, i64 0
+  %cv2e0 = sext i4 %v2e0 to i32
+  %mul0 = mul nuw nsw i32 %cv1e0, %cv2e0
+
+  %v1e1 = extractelement <8 x i4> %vec1, i64 1
+  %cv1e1 = sext i4 %v1e1 to i32
+  %v2e1 = extractelement <8 x i4> %vec2, i64 1
+  %cv2e1 = sext i4 %v2e1 to i32
+  %mul1 = mul nuw nsw i32 %cv1e1, %cv2e1
+
+  %v1e2 = extractelement <8 x i4> %vec1, i64 2
+  %cv1e2 = sext i4 %v1e2 to i32
+  %v2e2 = extractelement <8 x i4> %vec2, i64 2
+  %cv2e2 = sext i4 %v2e2 to i32
+  %mul2 = mul nuw nsw i32 %cv1e2, %cv2e2
+
+  %v1e3 = extractelement <8 x i4> %vec1, i64 3
+  %cv1e3 = sext i4 %v1e3 to i32
+  %v2e3 = extractelement <8 x i4> %vec2, i64 3
+  %cv2e3 = sext i4 %v2e3 to i32
+  %mul3 = mul nuw nsw i32 %cv1e3, %cv2e3
+
+  %v1e4 = extractelement <8 x i4> %vec1, i64 4
+  %cv1e4 = sext i4 %v1e4 to i32
+  %v2e4 = extractelement <8 x i4> %vec2, i64 4
+  %cv2e4 = sext i4 %v2e4 to i32
+  %mul4 = mul nuw nsw i32 %cv1e4, %cv2e4
+
+  %v1e5 = extractelement <8 x i4> %vec1, i64 5
+  %cv1e5 = sext i4 %v1e5 to i32
+  %v2e5 = extractelement <8 x i4> %vec2, i64 5
+  %cv2e5 = sext i4 %v2e5 to i32
+  %mul5 = mul nuw nsw i32 %cv1e5, %cv2e5
+
+  %v1e6 = extractelement <8 x i4> %vec1, i64 6
+  %cv1e6 = sext i4 %v1e6 to i32
+  %v2e6 = extractelement <8 x i4> %vec2, i64 6
+  %cv2e6 = sext i4 %v2e6 to i32
+  %mul6 = mul nuw nsw i32 %cv1e6, %cv2e6
+
+  %v1e7 = extractelement <8 x i4> %vec1, i64 7
+  %cv1e7 = sext i4 %v1e7 to i32
+  %v2e7 = extractelement <8 x i4> %vec2, i64 7
+  %cv2e7 = sext i4 %v2e7 to i32
+  %mul7 = mul nuw nsw i32 %cv1e7, %cv2e7
+
+  %acc = load i32, i32 addrspace(1)* %dst, align 4
+  %add =  add i32  %mul0, %acc
+  %add1 = add i32 %mul0, %add
+  %add2 = add i32 %add1, %mul1
+  %add3 = add i32 %add2, %mul2
+  %add4 = add i32 %add3, %mul3
+  %add5 = add i32 %add4, %mul4
+  %add6 = add i32 %add5, %mul5
+  %add7 = add i32 %add6, %mul6
+  %add8 = add i32 %add7, %mul7
+
+  %res = add i32 %add, %add8
+  store i32 %res, i32 addrspace(1)* %dst, align 4
+  ret void
+}
+
+; TODO: Support this pattern.
+define amdgpu_kernel void @idot8_acc32_vecMul(<8 x i4> addrspace(1)* %src1,
+; GFX7-LABEL: idot8_acc32_vecMul:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[8:11], s[0:1], 0x9
+; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0xd
+; GFX7-NEXT:    s_mov_b32 s7, 0xf000
+; GFX7-NEXT:    s_mov_b32 s6, -1
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_load_dword s1, s[8:9], 0x0
+; GFX7-NEXT:    s_load_dword s9, s[10:11], 0x0
+; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_ashr_i64 s[10:11], s[0:1], 60
+; GFX7-NEXT:    s_lshl_b32 s11, s1, 4
+; GFX7-NEXT:    s_ashr_i64 s[14:15], s[10:11], 60
+; GFX7-NEXT:    s_lshl_b32 s11, s1, 12
+; GFX7-NEXT:    s_ashr_i64 s[16:17], s[10:11], 60
+; GFX7-NEXT:    s_lshl_b32 s11, s1, 16
+; GFX7-NEXT:    s_ashr_i64 s[18:19], s[10:11], 60
+; GFX7-NEXT:    s_lshl_b32 s11, s1, 20
+; GFX7-NEXT:    s_lshl_b32 s13, s1, 8
+; GFX7-NEXT:    s_ashr_i64 s[20:21], s[10:11], 60
+; GFX7-NEXT:    s_lshl_b32 s11, s1, 24
+; GFX7-NEXT:    s_lshl_b32 s1, s1, 28
+; GFX7-NEXT:    s_ashr_i64 s[0:1], s[0:1], 60
+; GFX7-NEXT:    s_lshl_b32 s1, s9, 4
+; GFX7-NEXT:    s_ashr_i64 s[26:27], s[0:1], 60
+; GFX7-NEXT:    s_lshl_b32 s1, s9, 8
+; GFX7-NEXT:    s_ashr_i64 s[28:29], s[0:1], 60
+; GFX7-NEXT:    s_lshl_b32 s1, s9, 12
+; GFX7-NEXT:    s_ashr_i64 s[30:31], s[0:1], 60
+; GFX7-NEXT:    s_lshl_b32 s1, s9, 16
+; GFX7-NEXT:    s_ashr_i64 s[32:33], s[0:1], 60
+; GFX7-NEXT:    s_lshl_b32 s1, s9, 20
+; GFX7-NEXT:    s_ashr_i64 s[34:35], s[0:1], 60
+; GFX7-NEXT:    s_lshl_b32 s1, s9, 24
+; GFX7-NEXT:    s_ashr_i64 s[36:37], s[0:1], 60
+; GFX7-NEXT:    s_lshl_b32 s1, s9, 28
+; GFX7-NEXT:    s_ashr_i64 s[24:25], s[8:9], 60
+; GFX7-NEXT:    s_ashr_i64 s[8:9], s[0:1], 60
+; GFX7-NEXT:    v_mov_b32_e32 v0, s8
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mad_i32_i24 v0, s0, v0, v1
+; GFX7-NEXT:    s_ashr_i64 s[22:23], s[10:11], 60
+; GFX7-NEXT:    v_mov_b32_e32 v1, s36
+; GFX7-NEXT:    v_mad_i32_i24 v0, s22, v1, v0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s34
+; GFX7-NEXT:    v_mad_i32_i24 v0, s20, v1, v0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s32
+; GFX7-NEXT:    v_mad_i32_i24 v0, s18, v1, v0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s30
+; GFX7-NEXT:    v_mad_i32_i24 v0, s16, v1, v0
+; GFX7-NEXT:    s_ashr_i64 s[12:13], s[12:13], 60
+; GFX7-NEXT:    v_mov_b32_e32 v1, s28
+; GFX7-NEXT:    v_mad_i32_i24 v0, s12, v1, v0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s26
+; GFX7-NEXT:    v_mad_i32_i24 v0, s14, v1, v0
+; GFX7-NEXT:    v_mov_b32_e32 v1, s24
+; GFX7-NEXT:    v_mad_i32_i24 v0, s10, v1, v0
+; GFX7-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX8-LABEL: idot8_acc32_vecMul:
+; GFX8:       ; %bb.0: ; %entry
+; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    s_load_dword s5, s[4:5], 0x0
+; GFX8-NEXT:    s_load_dword s7, s[6:7], 0x0
+; GFX8-NEXT:    s_load_dword s2, s[0:1], 0x0
+; GFX8-NEXT:    v_mov_b32_e32 v0, s0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    s_ashr_i64 s[0:1], s[4:5], 60
+; GFX8-NEXT:    s_lshl_b32 s1, s5, 4
+; GFX8-NEXT:    s_ashr_i64 s[12:13], s[0:1], 60
+; GFX8-NEXT:    s_lshl_b32 s1, s5, 16
+; GFX8-NEXT:    s_ashr_i64 s[14:15], s[0:1], 60
+; GFX8-NEXT:    s_lshl_b32 s1, s5, 20
+; GFX8-NEXT:    s_ashr_i64 s[16:17], s[0:1], 60
+; GFX8-NEXT:    s_lshl_b32 s1, s5, 24
+; GFX8-NEXT:    s_ashr_i64 s[18:19], s[0:1], 60
+; GFX8-NEXT:    s_lshl_b32 s1, s5, 28
+; GFX8-NEXT:    s_lshl_b32 s9, s5, 8
+; GFX8-NEXT:    s_lshl_b32 s11, s5, 12
+; GFX8-NEXT:    s_ashr_i64 s[4:5], s[0:1], 60
+; GFX8-NEXT:    s_lshl_b32 s1, s7, 4
+; GFX8-NEXT:    s_ashr_i64 s[22:23], s[0:1], 60
+; GFX8-NEXT:    s_lshl_b32 s1, s7, 8
+; GFX8-NEXT:    s_ashr_i64 s[24:25], s[0:1], 60
+; GFX8-NEXT:    s_lshl_b32 s1, s7, 12
+; GFX8-NEXT:    s_ashr_i64 s[26:27], s[0:1], 60
+; GFX8-NEXT:    s_lshl_b32 s1, s7, 16
+; GFX8-NEXT:    s_ashr_i64 s[28:29], s[0:1], 60
+; GFX8-NEXT:    s_lshl_b32 s1, s7, 20
+; GFX8-NEXT:    s_ashr_i64 s[30:31], s[0:1], 60
+; GFX8-NEXT:    s_lshl_b32 s1, s7, 24
+; GFX8-NEXT:    s_ashr_i64 s[32:33], s[0:1], 60
+; GFX8-NEXT:    s_lshl_b32 s1, s7, 28
+; GFX8-NEXT:    s_ashr_i64 s[20:21], s[6:7], 60
+; GFX8-NEXT:    s_ashr_i64 s[6:7], s[0:1], 60
+; GFX8-NEXT:    v_mov_b32_e32 v2, s6
+; GFX8-NEXT:    v_mov_b32_e32 v3, s2
+; GFX8-NEXT:    v_mad_i32_i24 v2, s4, v2, v3
+; GFX8-NEXT:    v_mov_b32_e32 v3, s32
+; GFX8-NEXT:    v_mad_i32_i24 v2, s18, v3, v2
+; GFX8-NEXT:    v_mov_b32_e32 v3, s30
+; GFX8-NEXT:    v_mad_i32_i24 v2, s16, v3, v2
+; GFX8-NEXT:    v_mov_b32_e32 v3, s28
+; GFX8-NEXT:    v_mad_i32_i24 v2, s14, v3, v2
+; GFX8-NEXT:    s_ashr_i64 s[10:11], s[10:11], 60
+; GFX8-NEXT:    v_mov_b32_e32 v3, s26
+; GFX8-NEXT:    v_mad_i32_i24 v2, s10, v3, v2
+; GFX8-NEXT:    s_ashr_i64 s[8:9], s[8:9], 60
+; GFX8-NEXT:    v_mov_b32_e32 v3, s24
+; GFX8-NEXT:    v_mad_i32_i24 v2, s8, v3, v2
+; GFX8-NEXT:    v_mov_b32_e32 v3, s22
+; GFX8-NEXT:    v_mad_i32_i24 v2, s12, v3, v2
+; GFX8-NEXT:    v_mov_b32_e32 v3, s20
+; GFX8-NEXT:    v_mad_i32_i24 v2, s0, v3, v2
+; GFX8-NEXT:    flat_store_dword v[0:1], v2
+; GFX8-NEXT:    s_endpgm
+;
+; GFX9-LABEL: idot8_acc32_vecMul:
+; GFX9:       ; %bb.0: ; %entry
+; GFX9-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    s_load_dword s5, s[4:5], 0x0
+; GFX9-NEXT:    s_load_dword s7, s[6:7], 0x0
+; GFX9-NEXT:    s_load_dword s2, s[0:1], 0x0
+; GFX9-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    s_ashr_i64 s[0:1], s[4:5], 60
+; GFX9-NEXT:    s_lshl_b32 s1, s5, 4
+; GFX9-NEXT:    s_ashr_i64 s[12:13], s[0:1], 60
+; GFX9-NEXT:    s_lshl_b32 s1, s5, 16
+; GFX9-NEXT:    s_ashr_i64 s[14:15], s[0:1], 60
+; GFX9-NEXT:    s_lshl_b32 s1, s5, 20
+; GFX9-NEXT:    s_ashr_i64 s[16:17], s[0:1], 60
+; GFX9-NEXT:    s_lshl_b32 s1, s5, 24
+; GFX9-NEXT:    s_ashr_i64 s[18:19], s[0:1], 60
+; GFX9-NEXT:    s_lshl_b32 s1, s5, 28
+; GFX9-NEXT:    s_lshl_b32 s9, s5, 8
+; GFX9-NEXT:    s_lshl_b32 s11, s5, 12
+; GFX9-NEXT:    s_ashr_i64 s[4:5], s[0:1], 60
+; GFX9-NEXT:    s_lshl_b32 s1, s7, 4
+; GFX9-NEXT:    s_ashr_i64 s[22:23], s[0:1], 60
+; GFX9-NEXT:    s_lshl_b32 s1, s7, 8
+; GFX9-NEXT:    s_ashr_i64 s[24:25], s[0:1], 60
+; GFX9-NEXT:    s_lshl_b32 s1, s7, 12
+; GFX9-NEXT:    s_ashr_i64 s[26:27], s[0:1], 60
+; GFX9-NEXT:    s_lshl_b32 s1, s7, 16
+; GFX9-NEXT:    s_ashr_i64 s[28:29], s[0:1], 60
+; GFX9-NEXT:    s_lshl_b32 s1, s7, 20
+; GFX9-NEXT:    s_ashr_i64 s[30:31], s[0:1], 60
+; GFX9-NEXT:    s_lshl_b32 s1, s7, 24
+; GFX9-NEXT:    s_ashr_i64 s[32:33], s[0:1], 60
+; GFX9-NEXT:    s_lshl_b32 s1, s7, 28
+; GFX9-NEXT:    s_ashr_i64 s[20:21], s[6:7], 60
+; GFX9-NEXT:    s_ashr_i64 s[6:7], s[0:1], 60
+; GFX9-NEXT:    v_mov_b32_e32 v2, s6
+; GFX9-NEXT:    v_mov_b32_e32 v3, s2
+; GFX9-NEXT:    v_mad_i32_i24 v2, s4, v2, v3
+; GFX9-NEXT:    v_mov_b32_e32 v3, s32
+; GFX9-NEXT:    v_mad_i32_i24 v2, s18, v3, v2
+; GFX9-NEXT:    v_mov_b32_e32 v3, s30
+; GFX9-NEXT:    v_mad_i32_i24 v2, s16, v3, v2
+; GFX9-NEXT:    v_mov_b32_e32 v3, s28
+; GFX9-NEXT:    v_mad_i32_i24 v2, s14, v3, v2
+; GFX9-NEXT:    s_ashr_i64 s[10:11], s[10:11], 60
+; GFX9-NEXT:    v_mov_b32_e32 v3, s26
+; GFX9-NEXT:    v_mad_i32_i24 v2, s10, v3, v2
+; GFX9-NEXT:    s_ashr_i64 s[8:9], s[8:9], 60
+; GFX9-NEXT:    v_mov_b32_e32 v3, s24
+; GFX9-NEXT:    v_mad_i32_i24 v2, s8, v3, v2
+; GFX9-NEXT:    v_mov_b32_e32 v3, s22
+; GFX9-NEXT:    v_mad_i32_i24 v2, s12, v3, v2
+; GFX9-NEXT:    v_mov_b32_e32 v3, s20
+; GFX9-NEXT:    v_mad_i32_i24 v2, s0, v3, v2
+; GFX9-NEXT:    global_store_dword v[0:1], v2, off
+; GFX9-NEXT:    s_endpgm
+;
+; GFX9-DL-LABEL: idot8_acc32_vecMul:
+; GFX9-DL:       ; %bb.0: ; %entry
+; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-DL-NEXT:    s_load_dword s5, s[4:5], 0x0
+; GFX9-DL-NEXT:    s_load_dword s7, s[6:7], 0x0
+; GFX9-DL-NEXT:    s_load_dword s2, s[0:1], 0x0
+; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-DL-NEXT:    s_ashr_i64 s[0:1], s[4:5], 60
+; GFX9-DL-NEXT:    s_lshl_b32 s1, s5, 4
+; GFX9-DL-NEXT:    s_ashr_i64 s[12:13], s[0:1], 60
+; GFX9-DL-NEXT:    s_lshl_b32 s1, s5, 16
+; GFX9-DL-NEXT:    s_ashr_i64 s[14:15], s[0:1], 60
+; GFX9-DL-NEXT:    s_lshl_b32 s1, s5, 20
+; GFX9-DL-NEXT:    s_ashr_i64 s[16:17], s[0:1], 60
+; GFX9-DL-NEXT:    s_lshl_b32 s1, s5, 24
+; GFX9-DL-NEXT:    s_ashr_i64 s[18:19], s[0:1], 60
+; GFX9-DL-NEXT:    s_lshl_b32 s1, s5, 28
+; GFX9-DL-NEXT:    s_lshl_b32 s9, s5, 8
+; GFX9-DL-NEXT:    s_lshl_b32 s11, s5, 12
+; GFX9-DL-NEXT:    s_ashr_i64 s[4:5], s[0:1], 60
+; GFX9-DL-NEXT:    s_lshl_b32 s1, s7, 4
+; GFX9-DL-NEXT:    s_ashr_i64 s[22:23], s[0:1], 60
+; GFX9-DL-NEXT:    s_lshl_b32 s1, s7, 8
+; GFX9-DL-NEXT:    s_ashr_i64 s[24:25], s[0:1], 60
+; GFX9-DL-NEXT:    s_lshl_b32 s1, s7, 12
+; GFX9-DL-NEXT:    s_ashr_i64 s[26:27], s[0:1], 60
+; GFX9-DL-NEXT:    s_lshl_b32 s1, s7, 16
+; GFX9-DL-NEXT:    s_ashr_i64 s[28:29], s[0:1], 60
+; GFX9-DL-NEXT:    s_lshl_b32 s1, s7, 20
+; GFX9-DL-NEXT:    s_ashr_i64 s[30:31], s[0:1], 60
+; GFX9-DL-NEXT:    s_lshl_b32 s1, s7, 24
+; GFX9-DL-NEXT:    s_ashr_i64 s[32:33], s[0:1], 60
+; GFX9-DL-NEXT:    s_lshl_b32 s1, s7, 28
+; GFX9-DL-NEXT:    s_ashr_i64 s[20:21], s[6:7], 60
+; GFX9-DL-NEXT:    s_ashr_i64 s[6:7], s[0:1], 60
+; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s6
+; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s2
+; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s4, v2, v3
+; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s32
+; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s18, v3, v2
+; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s30
+; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s16, v3, v2
+; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s28
+; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s14, v3, v2
+; GFX9-DL-NEXT:    s_ashr_i64 s[10:11], s[10:11], 60
+; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s26
+; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s10, v3, v2
+; GFX9-DL-NEXT:    s_ashr_i64 s[8:9], s[8:9], 60
+; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s24
+; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s8, v3, v2
+; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s22
+; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s12, v3, v2
+; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s20
+; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s0, v3, v2
+; GFX9-DL-NEXT:    global_store_dword v[0:1], v2, off
+; GFX9-DL-NEXT:    s_endpgm
+                                              <8 x i4> addrspace(1)* %src2,
+                                              i32 addrspace(1)* nocapture %dst) {
+entry:
+  %vec1 = load <8 x i4>, <8 x i4> addrspace(1)* %src1
+  %vec2 = load <8 x i4>, <8 x i4> addrspace(1)* %src2
+
+  %cvec1 = sext <8 x i4> %vec1 to <8 x i32>
+  %cvec2 = sext <8 x i4> %vec2 to <8 x i32>
+
+  %mul = mul <8 x i32> %cvec1, %cvec2
+  %mul0 = extractelement <8 x i32> %mul, i64 0
+  %mul1 = extractelement <8 x i32> %mul, i64 1
+  %mul2 = extractelement <8 x i32> %mul, i64 2
+  %mul3 = extractelement <8 x i32> %mul, i64 3
+  %mul4 = extractelement <8 x i32> %mul, i64 4
+  %mul5 = extractelement <8 x i32> %mul, i64 5
+  %mul6 = extractelement <8 x i32> %mul, i64 6
+  %mul7 = extractelement <8 x i32> %mul, i64 7
+
+  %acc = load i32, i32 addrspace(1)* %dst, align 4
+  %add1 = add i32 %mul0, %acc
+  %add2 = add i32 %add1, %mul1
+  %add3 = add i32 %add2, %mul2
+  %add4 = add i32 %add3, %mul3
+  %add5 = add i32 %add4, %mul4
+  %add6 = add i32 %add5, %mul5
+  %add7 = add i32 %add6, %mul6
+  %add8 = add i32 %add7, %mul7
+
+  store i32 %add8, i32 addrspace(1)* %dst, align 4
+  ret void
+}
+
+; TODO: Support this pattern.
+define amdgpu_kernel void @idot8_acc16_vecMul(<8 x i4> addrspace(1)* %src1,
+; GFX7-LABEL: idot8_acc16_vecMul:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[8:11], s[0:1], 0x9
+; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0xd
+; GFX7-NEXT:    s_mov_b32 s7, 0xf000
+; GFX7-NEXT:    s_mov_b32 s6, -1
+; GFX7-NEXT:    s_mov_b32 s0, 0xffff
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_load_dword s1, s[8:9], 0x0
+; GFX7-NEXT:    buffer_load_ushort v0, off, s[4:7], 0
+; GFX7-NEXT:    s_load_dword s2, s[10:11], 0x0
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_bfe_i32 s8, s1, 0x40010
+; GFX7-NEXT:    s_bfe_i32 s9, s1, 0x40014
+; GFX7-NEXT:    s_bfe_i32 s15, s2, 0x40010
+; GFX7-NEXT:    s_bfe_i32 s16, s2, 0x40014
+; GFX7-NEXT:    s_bfe_i32 s17, s2, 0x40018
+; GFX7-NEXT:    s_ashr_i32 s18, s2, 28
+; GFX7-NEXT:    s_bfe_i32 s19, s2, 0x40000
+; GFX7-NEXT:    s_bfe_i32 s20, s2, 0x40004
+; GFX7-NEXT:    s_bfe_i32 s21, s2, 0x40008
+; GFX7-NEXT:    s_bfe_i32 s2, s2, 0x4000c
+; GFX7-NEXT:    s_bfe_i32 s10, s1, 0x40018
+; GFX7-NEXT:    s_ashr_i32 s11, s1, 28
+; GFX7-NEXT:    s_bfe_i32 s12, s1, 0x40000
+; GFX7-NEXT:    v_mov_b32_e32 v4, s19
+; GFX7-NEXT:    s_bfe_i32 s13, s1, 0x40004
+; GFX7-NEXT:    v_mov_b32_e32 v3, s20
+; GFX7-NEXT:    s_bfe_i32 s14, s1, 0x40008
+; GFX7-NEXT:    v_mov_b32_e32 v2, s21
+; GFX7-NEXT:    s_bfe_i32 s1, s1, 0x4000c
+; GFX7-NEXT:    v_mov_b32_e32 v1, s2
+; GFX7-NEXT:    v_mov_b32_e32 v5, s18
+; GFX7-NEXT:    v_mov_b32_e32 v6, s17
+; GFX7-NEXT:    v_mul_i32_i24_e32 v1, s1, v1
+; GFX7-NEXT:    v_mul_i32_i24_e32 v2, s14, v2
+; GFX7-NEXT:    v_mul_i32_i24_e32 v3, s13, v3
+; GFX7-NEXT:    v_mul_i32_i24_e32 v4, s12, v4
+; GFX7-NEXT:    v_mul_i32_i24_e32 v5, s11, v5
+; GFX7-NEXT:    v_mul_i32_i24_e32 v6, s10, v6
+; GFX7-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX7-NEXT:    v_and_b32_e32 v2, s0, v2
+; GFX7-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GFX7-NEXT:    v_and_b32_e32 v4, s0, v4
+; GFX7-NEXT:    v_or_b32_e32 v1, v2, v1
+; GFX7-NEXT:    v_or_b32_e32 v2, v4, v3
+; GFX7-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
+; GFX7-NEXT:    v_and_b32_e32 v6, s0, v6
+; GFX7-NEXT:    v_mov_b32_e32 v7, s16
+; GFX7-NEXT:    v_mov_b32_e32 v8, s15
+; GFX7-NEXT:    v_or_b32_e32 v3, v6, v5
+; GFX7-NEXT:    v_alignbit_b32 v5, v1, v2, 16
+; GFX7-NEXT:    v_mul_i32_i24_e32 v7, s9, v7
+; GFX7-NEXT:    v_mul_i32_i24_e32 v8, s8, v8
+; GFX7-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
+; GFX7-NEXT:    v_and_b32_e32 v8, s0, v8
+; GFX7-NEXT:    v_lshrrev_b32_e32 v6, 16, v1
+; GFX7-NEXT:    v_or_b32_e32 v4, v8, v7
+; GFX7-NEXT:    v_lshrrev_b32_e32 v7, 16, v4
+; GFX7-NEXT:    v_lshrrev_b32_e32 v8, 16, v3
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
+; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v5, v0
+; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
+; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v6, v0
+; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v4, v0
+; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v7, v0
+; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v3, v0
+; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v8, v0
+; GFX7-NEXT:    buffer_store_short v0, off, s[4:7], 0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX8-LABEL: idot8_acc16_vecMul:
+; GFX8:       ; %bb.0: ; %entry
+; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX8-NEXT:    s_load_dword s4, s[6:7], 0x0
+; GFX8-NEXT:    v_mov_b32_e32 v0, s0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-NEXT:    flat_load_ushort v2, v[0:1]
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    v_lshlrev_b16_e64 v3, 12, s2
+; GFX8-NEXT:    v_lshlrev_b16_e64 v4, 12, s4
+; GFX8-NEXT:    s_lshr_b32 s0, s2, 4
+; GFX8-NEXT:    s_lshr_b32 s1, s2, 8
+; GFX8-NEXT:    s_lshr_b32 s5, s4, 4
+; GFX8-NEXT:    s_lshr_b32 s6, s4, 8
+; GFX8-NEXT:    v_lshlrev_b16_e64 v5, 12, s1
+; GFX8-NEXT:    v_lshlrev_b16_e64 v6, 12, s0
+; GFX8-NEXT:    v_lshlrev_b16_e64 v7, 12, s6
+; GFX8-NEXT:    v_lshlrev_b16_e64 v8, 12, s5
+; GFX8-NEXT:    v_ashrrev_i16_e32 v3, 12, v3
+; GFX8-NEXT:    v_ashrrev_i16_e32 v4, 12, v4
+; GFX8-NEXT:    s_lshr_b32 s0, s2, 12
+; GFX8-NEXT:    s_lshr_b32 s1, s4, 12
+; GFX8-NEXT:    v_ashrrev_i16_e32 v6, 12, v6
+; GFX8-NEXT:    v_ashrrev_i16_e32 v5, 12, v5
+; GFX8-NEXT:    v_ashrrev_i16_e32 v7, 12, v7
+; GFX8-NEXT:    v_ashrrev_i16_e32 v8, 12, v8
+; GFX8-NEXT:    v_lshlrev_b16_e64 v9, 12, s0
+; GFX8-NEXT:    v_lshlrev_b16_e64 v10, 12, s1
+; GFX8-NEXT:    s_lshr_b32 s5, s2, 16
+; GFX8-NEXT:    s_lshr_b32 s6, s4, 16
+; GFX8-NEXT:    v_mul_u32_u24_e32 v5, v5, v7
+; GFX8-NEXT:    v_lshlrev_b16_e64 v11, 12, s5
+; GFX8-NEXT:    v_lshlrev_b16_e64 v12, 12, s6
+; GFX8-NEXT:    s_lshr_b32 s0, s2, 20
+; GFX8-NEXT:    s_lshr_b32 s1, s4, 20
+; GFX8-NEXT:    v_ashrrev_i16_e32 v9, 12, v9
+; GFX8-NEXT:    v_ashrrev_i16_e32 v10, 12, v10
+; GFX8-NEXT:    v_lshlrev_b16_e64 v13, 12, s0
+; GFX8-NEXT:    v_lshlrev_b16_e64 v14, 12, s1
+; GFX8-NEXT:    s_lshr_b32 s5, s2, 24
+; GFX8-NEXT:    s_lshr_b32 s6, s4, 24
+; GFX8-NEXT:    v_ashrrev_i16_e32 v11, 12, v11
+; GFX8-NEXT:    v_ashrrev_i16_e32 v12, 12, v12
+; GFX8-NEXT:    v_lshlrev_b16_e64 v15, 12, s5
+; GFX8-NEXT:    v_lshlrev_b16_e64 v17, 12, s6
+; GFX8-NEXT:    s_lshr_b32 s0, s2, 28
+; GFX8-NEXT:    s_lshr_b32 s1, s4, 28
+; GFX8-NEXT:    v_ashrrev_i16_e32 v13, 12, v13
+; GFX8-NEXT:    v_ashrrev_i16_e32 v14, 12, v14
+; GFX8-NEXT:    v_lshlrev_b16_e64 v16, 12, s0
+; GFX8-NEXT:    v_lshlrev_b16_e64 v18, 12, s1
+; GFX8-NEXT:    v_ashrrev_i16_e32 v15, 12, v15
+; GFX8-NEXT:    v_ashrrev_i16_e32 v17, 12, v17
+; GFX8-NEXT:    v_ashrrev_i16_e32 v16, 12, v16
+; GFX8-NEXT:    v_ashrrev_i16_e32 v18, 12, v18
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    v_mad_u32_u24 v2, v3, v4, v2
+; GFX8-NEXT:    v_mad_u32_u24 v2, v6, v8, v2
+; GFX8-NEXT:    v_add_u32_sdwa v2, vcc, v5, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
+; GFX8-NEXT:    v_mad_u32_u24 v2, v9, v10, v2
+; GFX8-NEXT:    v_mad_u32_u24 v2, v11, v12, v2
+; GFX8-NEXT:    v_mad_u32_u24 v2, v13, v14, v2
+; GFX8-NEXT:    v_mad_u32_u24 v2, v15, v17, v2
+; GFX8-NEXT:    v_mad_u32_u24 v2, v16, v18, v2
+; GFX8-NEXT:    flat_store_short v[0:1], v2
+; GFX8-NEXT:    s_endpgm
+;
+; GFX9-LABEL: idot8_acc16_vecMul:
+; GFX9:       ; %bb.0: ; %entry
+; GFX9-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX9-NEXT:    s_load_dword s4, s[6:7], 0x0
+; GFX9-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    global_load_ushort v2, v[0:1], off
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    s_and_b32 s0, s2, 15
+; GFX9-NEXT:    s_bfe_u32 s1, s2, 0x40004
+; GFX9-NEXT:    s_and_b32 s5, s4, 15
+; GFX9-NEXT:    s_bfe_u32 s6, s4, 0x40004
+; GFX9-NEXT:    s_pack_ll_b32_b16 s0, s0, s1
+; GFX9-NEXT:    s_pack_ll_b32_b16 s1, s5, s6
+; GFX9-NEXT:    s_bfe_u32 s5, s2, 0x40008
+; GFX9-NEXT:    s_bfe_u32 s6, s2, 0x4000c
+; GFX9-NEXT:    v_pk_lshlrev_b16 v3, 12, s0 op_sel_hi:[0,1]
+; GFX9-NEXT:    s_pack_ll_b32_b16 s0, s5, s6
+; GFX9-NEXT:    s_bfe_u32 s9, s2, 0x40010
+; GFX9-NEXT:    s_bfe_u32 s10, s2, 0x40014
+; GFX9-NEXT:    v_pk_lshlrev_b16 v4, 12, s0 op_sel_hi:[0,1]
+; GFX9-NEXT:    s_bfe_u32 s13, s2, 0x40018
+; GFX9-NEXT:    s_pack_ll_b32_b16 s0, s9, s10
+; GFX9-NEXT:    s_lshr_b32 s2, s2, 28
+; GFX9-NEXT:    v_pk_lshlrev_b16 v5, 12, s0 op_sel_hi:[0,1]
+; GFX9-NEXT:    v_pk_lshlrev_b16 v7, 12, s1 op_sel_hi:[0,1]
+; GFX9-NEXT:    s_pack_ll_b32_b16 s0, s13, s2
+; GFX9-NEXT:    s_bfe_u32 s7, s4, 0x40008
+; GFX9-NEXT:    s_bfe_u32 s8, s4, 0x4000c
+; GFX9-NEXT:    v_pk_lshlrev_b16 v6, 12, s0 op_sel_hi:[0,1]
+; GFX9-NEXT:    s_pack_ll_b32_b16 s0, s7, s8
+; GFX9-NEXT:    v_pk_ashrrev_i16 v3, 12, v3 op_sel_hi:[0,1]
+; GFX9-NEXT:    v_pk_ashrrev_i16 v7, 12, v7 op_sel_hi:[0,1]
+; GFX9-NEXT:    v_pk_lshlrev_b16 v8, 12, s0 op_sel_hi:[0,1]
+; GFX9-NEXT:    v_pk_mul_lo_u16 v3, v3, v7
+; GFX9-NEXT:    s_bfe_u32 s11, s4, 0x40010
+; GFX9-NEXT:    s_bfe_u32 s12, s4, 0x40014
+; GFX9-NEXT:    s_pack_ll_b32_b16 s0, s11, s12
+; GFX9-NEXT:    v_pk_ashrrev_i16 v4, 12, v4 op_sel_hi:[0,1]
+; GFX9-NEXT:    v_pk_ashrrev_i16 v8, 12, v8 op_sel_hi:[0,1]
+; GFX9-NEXT:    v_pk_lshlrev_b16 v9, 12, s0 op_sel_hi:[0,1]
+; GFX9-NEXT:    s_bfe_u32 s14, s4, 0x40018
+; GFX9-NEXT:    s_lshr_b32 s4, s4, 28
+; GFX9-NEXT:    v_pk_mul_lo_u16 v4, v4, v8
+; GFX9-NEXT:    s_pack_ll_b32_b16 s0, s14, s4
+; GFX9-NEXT:    v_pk_ashrrev_i16 v5, 12, v5 op_sel_hi:[0,1]
+; GFX9-NEXT:    v_pk_ashrrev_i16 v9, 12, v9 op_sel_hi:[0,1]
+; GFX9-NEXT:    v_pk_lshlrev_b16 v10, 12, s0 op_sel_hi:[0,1]
+; GFX9-NEXT:    v_pk_mul_lo_u16 v5, v5, v9
+; GFX9-NEXT:    v_pk_ashrrev_i16 v6, 12, v6 op_sel_hi:[0,1]
+; GFX9-NEXT:    v_pk_ashrrev_i16 v10, 12, v10 op_sel_hi:[0,1]
+; GFX9-NEXT:    v_pk_mul_lo_u16 v6, v6, v10
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_add_u32_e32 v2, v3, v2
+; GFX9-NEXT:    v_add_u32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT:    v_add_u32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
+; GFX9-NEXT:    v_add_u32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT:    v_add_u32_e32 v2, v2, v5
+; GFX9-NEXT:    v_add_u32_sdwa v2, v2, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT:    v_add_u32_e32 v2, v2, v6
+; GFX9-NEXT:    v_add_u32_sdwa v2, v2, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT:    global_store_short v[0:1], v2, off
+; GFX9-NEXT:    s_endpgm
+;
+; GFX9-DL-LABEL: idot8_acc16_vecMul:
+; GFX9-DL:       ; %bb.0: ; %entry
+; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX9-DL-NEXT:    s_load_dword s4, s[6:7], 0x0
+; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-DL-NEXT:    global_load_ushort v2, v[0:1], off
+; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-DL-NEXT:    s_and_b32 s0, s2, 15
+; GFX9-DL-NEXT:    s_bfe_u32 s1, s2, 0x40004
+; GFX9-DL-NEXT:    s_and_b32 s5, s4, 15
+; GFX9-DL-NEXT:    s_bfe_u32 s6, s4, 0x40004
+; GFX9-DL-NEXT:    s_pack_ll_b32_b16 s0, s0, s1
+; GFX9-DL-NEXT:    s_pack_ll_b32_b16 s1, s5, s6
+; GFX9-DL-NEXT:    s_bfe_u32 s5, s2, 0x40008
+; GFX9-DL-NEXT:    s_bfe_u32 s6, s2, 0x4000c
+; GFX9-DL-NEXT:    v_pk_lshlrev_b16 v3, 12, s0 op_sel_hi:[0,1]
+; GFX9-DL-NEXT:    s_pack_ll_b32_b16 s0, s5, s6
+; GFX9-DL-NEXT:    s_bfe_u32 s9, s2, 0x40010
+; GFX9-DL-NEXT:    s_bfe_u32 s10, s2, 0x40014
+; GFX9-DL-NEXT:    v_pk_lshlrev_b16 v4, 12, s0 op_sel_hi:[0,1]
+; GFX9-DL-NEXT:    s_bfe_u32 s13, s2, 0x40018
+; GFX9-DL-NEXT:    s_pack_ll_b32_b16 s0, s9, s10
+; GFX9-DL-NEXT:    s_lshr_b32 s2, s2, 28
+; GFX9-DL-NEXT:    v_pk_lshlrev_b16 v5, 12, s0 op_sel_hi:[0,1]
+; GFX9-DL-NEXT:    v_pk_lshlrev_b16 v7, 12, s1 op_sel_hi:[0,1]
+; GFX9-DL-NEXT:    s_pack_ll_b32_b16 s0, s13, s2
+; GFX9-DL-NEXT:    s_bfe_u32 s7, s4, 0x40008
+; GFX9-DL-NEXT:    s_bfe_u32 s8, s4, 0x4000c
+; GFX9-DL-NEXT:    v_pk_lshlrev_b16 v6, 12, s0 op_sel_hi:[0,1]
+; GFX9-DL-NEXT:    s_pack_ll_b32_b16 s0, s7, s8
+; GFX9-DL-NEXT:    v_pk_ashrrev_i16 v3, 12, v3 op_sel_hi:[0,1]
+; GFX9-DL-NEXT:    v_pk_ashrrev_i16 v7, 12, v7 op_sel_hi:[0,1]
+; GFX9-DL-NEXT:    v_pk_lshlrev_b16 v8, 12, s0 op_sel_hi:[0,1]
+; GFX9-DL-NEXT:    v_pk_mul_lo_u16 v3, v3, v7
+; GFX9-DL-NEXT:    s_bfe_u32 s11, s4, 0x40010
+; GFX9-DL-NEXT:    s_bfe_u32 s12, s4, 0x40014
+; GFX9-DL-NEXT:    s_pack_ll_b32_b16 s0, s11, s12
+; GFX9-DL-NEXT:    v_pk_ashrrev_i16 v4, 12, v4 op_sel_hi:[0,1]
+; GFX9-DL-NEXT:    v_pk_ashrrev_i16 v8, 12, v8 op_sel_hi:[0,1]
+; GFX9-DL-NEXT:    v_pk_lshlrev_b16 v9, 12, s0 op_sel_hi:[0,1]
+; GFX9-DL-NEXT:    s_bfe_u32 s14, s4, 0x40018
+; GFX9-DL-NEXT:    s_lshr_b32 s4, s4, 28
+; GFX9-DL-NEXT:    v_pk_mul_lo_u16 v4, v4, v8
+; GFX9-DL-NEXT:    s_pack_ll_b32_b16 s0, s14, s4
+; GFX9-DL-NEXT:    v_pk_ashrrev_i16 v5, 12, v5 op_sel_hi:[0,1]
+; GFX9-DL-NEXT:    v_pk_ashrrev_i16 v9, 12, v9 op_sel_hi:[0,1]
+; GFX9-DL-NEXT:    v_pk_lshlrev_b16 v10, 12, s0 op_sel_hi:[0,1]
+; GFX9-DL-NEXT:    v_pk_mul_lo_u16 v5, v5, v9
+; GFX9-DL-NEXT:    v_pk_ashrrev_i16 v6, 12, v6 op_sel_hi:[0,1]
+; GFX9-DL-NEXT:    v_pk_ashrrev_i16 v10, 12, v10 op_sel_hi:[0,1]
+; GFX9-DL-NEXT:    v_pk_mul_lo_u16 v6, v6, v10
+; GFX9-DL-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-DL-NEXT:    v_add_u32_e32 v2, v3, v2
+; GFX9-DL-NEXT:    v_add_u32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-DL-NEXT:    v_add_u32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
+; GFX9-DL-NEXT:    v_add_u32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-DL-NEXT:    v_add_u32_e32 v2, v2, v5
+; GFX9-DL-NEXT:    v_add_u32_sdwa v2, v2, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-DL-NEXT:    v_add_u32_e32 v2, v2, v6
+; GFX9-DL-NEXT:    v_add_u32_sdwa v2, v2, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-DL-NEXT:    global_store_short v[0:1], v2, off
+; GFX9-DL-NEXT:    s_endpgm
+                                              <8 x i4> addrspace(1)* %src2,
+                                              i16 addrspace(1)* nocapture %dst) {
+entry:
+  %vec1 = load <8 x i4>, <8 x i4> addrspace(1)* %src1
+  %vec2 = load <8 x i4>, <8 x i4> addrspace(1)* %src2
+
+  %cvec1 = sext <8 x i4> %vec1 to <8 x i16>
+  %cvec2 = sext <8 x i4> %vec2 to <8 x i16>
+
+  %mul = mul <8 x i16> %cvec1, %cvec2
+  %mul0 = extractelement <8 x i16> %mul, i64 0
+  %mul1 = extractelement <8 x i16> %mul, i64 1
+  %mul2 = extractelement <8 x i16> %mul, i64 2
+  %mul3 = extractelement <8 x i16> %mul, i64 3
+  %mul4 = extractelement <8 x i16> %mul, i64 4
+  %mul5 = extractelement <8 x i16> %mul, i64 5
+  %mul6 = extractelement <8 x i16> %mul, i64 6
+  %mul7 = extractelement <8 x i16> %mul, i64 7
+
+  %acc = load i16, i16 addrspace(1)* %dst, align 4
+  %add1 = add i16 %mul0, %acc
+  %add2 = add i16 %add1, %mul1
+  %add3 = add i16 %add2, %mul2
+  %add4 = add i16 %add3, %mul3
+  %add5 = add i16 %add4, %mul4
+  %add6 = add i16 %add5, %mul5
+  %add7 = add i16 %add6, %mul6
+  %add8 = add i16 %add7, %mul7
+
+  store i16 %add8, i16 addrspace(1)* %dst, align 4
+  ret void
+}
+
+; TODO: Support this pattern.
+define amdgpu_kernel void @idot8_acc8_vecMul(<8 x i4> addrspace(1)* %src1,
+; GFX7-LABEL: idot8_acc8_vecMul:
+; GFX7:       ; %bb.0: ; %entry
+; GFX7-NEXT:    s_load_dwordx4 s[8:11], s[0:1], 0x9
+; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0xd
+; GFX7-NEXT:    s_mov_b32 s7, 0xf000
+; GFX7-NEXT:    s_mov_b32 s6, -1
+; GFX7-NEXT:    s_movk_i32 s0, 0xff
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_load_dword s2, s[8:9], 0x0
+; GFX7-NEXT:    buffer_load_ubyte v0, off, s[4:7], 0
+; GFX7-NEXT:    s_load_dword s8, s[10:11], 0x0
+; GFX7-NEXT:    s_mov_b32 s1, 0xffff
+; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX7-NEXT:    s_bfe_i32 s9, s2, 0x40000
+; GFX7-NEXT:    s_bfe_i32 s10, s2, 0x40004
+; GFX7-NEXT:    s_bfe_i32 s16, s8, 0x40000
+; GFX7-NEXT:    s_bfe_i32 s17, s8, 0x40004
+; GFX7-NEXT:    s_bfe_i32 s18, s8, 0x40008
+; GFX7-NEXT:    s_bfe_i32 s19, s8, 0x4000c
+; GFX7-NEXT:    s_bfe_i32 s20, s8, 0x40010
+; GFX7-NEXT:    s_bfe_i32 s21, s8, 0x40014
+; GFX7-NEXT:    s_bfe_i32 s22, s8, 0x40018
+; GFX7-NEXT:    s_ashr_i32 s8, s8, 28
+; GFX7-NEXT:    v_mov_b32_e32 v7, s17
+; GFX7-NEXT:    v_mov_b32_e32 v8, s16
+; GFX7-NEXT:    s_bfe_i32 s11, s2, 0x40008
+; GFX7-NEXT:    v_mov_b32_e32 v6, s18
+; GFX7-NEXT:    s_bfe_i32 s12, s2, 0x4000c
+; GFX7-NEXT:    v_mov_b32_e32 v5, s19
+; GFX7-NEXT:    s_bfe_i32 s13, s2, 0x40010
+; GFX7-NEXT:    v_mov_b32_e32 v4, s20
+; GFX7-NEXT:    s_bfe_i32 s14, s2, 0x40014
+; GFX7-NEXT:    v_mov_b32_e32 v3, s21
+; GFX7-NEXT:    s_bfe_i32 s15, s2, 0x40018
+; GFX7-NEXT:    v_mov_b32_e32 v2, s22
+; GFX7-NEXT:    s_ashr_i32 s2, s2, 28
+; GFX7-NEXT:    v_mov_b32_e32 v1, s8
+; GFX7-NEXT:    v_mul_i32_i24_e32 v1, s2, v1
+; GFX7-NEXT:    v_mul_i32_i24_e32 v2, s15, v2
+; GFX7-NEXT:    v_mul_i32_i24_e32 v3, s14, v3
+; GFX7-NEXT:    v_mul_i32_i24_e32 v4, s13, v4
+; GFX7-NEXT:    v_mul_i32_i24_e32 v5, s12, v5
+; GFX7-NEXT:    v_mul_i32_i24_e32 v6, s11, v6
+; GFX7-NEXT:    v_mul_i32_i24_e32 v7, s10, v7
+; GFX7-NEXT:    v_mul_i32_i24_e32 v8, s9, v8
+; GFX7-NEXT:    v_lshlrev_b32_e32 v1, 8, v1
+; GFX7-NEXT:    v_and_b32_e32 v2, s0, v2
+; GFX7-NEXT:    v_lshlrev_b32_e32 v3, 8, v3
+; GFX7-NEXT:    v_and_b32_e32 v4, s0, v4
+; GFX7-NEXT:    v_lshlrev_b32_e32 v5, 8, v5
+; GFX7-NEXT:    v_and_b32_e32 v6, s0, v6
+; GFX7-NEXT:    v_lshlrev_b32_e32 v7, 8, v7
+; GFX7-NEXT:    v_and_b32_e32 v8, s0, v8
+; GFX7-NEXT:    v_or_b32_e32 v1, v2, v1
+; GFX7-NEXT:    v_or_b32_e32 v2, v4, v3
+; GFX7-NEXT:    v_or_b32_e32 v3, v6, v5
+; GFX7-NEXT:    v_or_b32_e32 v4, v8, v7
+; GFX7-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX7-NEXT:    v_and_b32_e32 v2, s1, v2
+; GFX7-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
+; GFX7-NEXT:    v_and_b32_e32 v4, s1, v4
+; GFX7-NEXT:    v_or_b32_e32 v1, v2, v1
+; GFX7-NEXT:    v_or_b32_e32 v2, v4, v3
+; GFX7-NEXT:    v_alignbit_b32 v3, v1, v2, 8
+; GFX7-NEXT:    v_alignbit_b32 v4, v1, v2, 16
+; GFX7-NEXT:    v_lshrrev_b32_e32 v5, 24, v2
+; GFX7-NEXT:    v_lshrrev_b32_e32 v6, 8, v1
+; GFX7-NEXT:    v_lshrrev_b32_e32 v7, 16, v1
+; GFX7-NEXT:    v_lshrrev_b32_e32 v8, 24, v1
+; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
+; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v3, v0
+; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v4, v0
+; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v5, v0
+; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
+; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v6, v0
+; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v7, v0
+; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v8, v0
+; GFX7-NEXT:    buffer_store_byte v0, off, s[4:7], 0
+; GFX7-NEXT:    s_endpgm
+;
+; GFX8-LABEL: idot8_acc8_vecMul:
+; GFX8:       ; %bb.0: ; %entry
+; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX8-NEXT:    s_load_dword s4, s[6:7], 0x0
+; GFX8-NEXT:    v_mov_b32_e32 v0, s0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-NEXT:    flat_load_ubyte v2, v[0:1]
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    s_lshr_b32 s0, s2, 4
+; GFX8-NEXT:    s_lshr_b32 s1, s2, 12
+; GFX8-NEXT:    s_lshr_b32 s5, s2, 8
+; GFX8-NEXT:    s_lshr_b32 s6, s4, 4
+; GFX8-NEXT:    s_lshr_b32 s7, s4, 12
+; GFX8-NEXT:    s_lshr_b32 s8, s4, 8
+; GFX8-NEXT:    v_lshlrev_b16_e64 v3, 12, s5
+; GFX8-NEXT:    v_lshlrev_b16_e64 v4, 12, s1
+; GFX8-NEXT:    v_lshlrev_b16_e64 v5, 12, s0
+; GFX8-NEXT:    v_lshlrev_b16_e64 v7, 12, s8
+; GFX8-NEXT:    v_lshlrev_b16_e64 v8, 12, s7
+; GFX8-NEXT:    v_lshlrev_b16_e64 v9, 12, s6
+; GFX8-NEXT:    v_lshlrev_b16_e64 v6, 12, s2
+; GFX8-NEXT:    v_lshlrev_b16_e64 v10, 12, s4
+; GFX8-NEXT:    v_ashrrev_i16_e32 v3, 12, v3
+; GFX8-NEXT:    v_ashrrev_i16_e32 v7, 12, v7
+; GFX8-NEXT:    v_ashrrev_i16_e32 v4, 12, v4
+; GFX8-NEXT:    v_ashrrev_i16_e32 v8, 12, v8
+; GFX8-NEXT:    v_ashrrev_i16_e32 v5, 12, v5
+; GFX8-NEXT:    v_ashrrev_i16_e32 v9, 12, v9
+; GFX8-NEXT:    v_ashrrev_i16_e32 v6, 12, v6
+; GFX8-NEXT:    v_ashrrev_i16_e32 v10, 12, v10
+; GFX8-NEXT:    v_mul_u32_u24_sdwa v3, v3, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX8-NEXT:    v_mul_u32_u24_sdwa v4, v4, v8 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX8-NEXT:    v_mul_u32_u24_sdwa v5, v5, v9 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX8-NEXT:    v_mul_u32_u24_sdwa v6, v6, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX8-NEXT:    s_lshr_b32 s0, s2, 20
+; GFX8-NEXT:    s_lshr_b32 s1, s2, 16
+; GFX8-NEXT:    s_lshr_b32 s5, s2, 28
+; GFX8-NEXT:    s_lshr_b32 s2, s2, 24
+; GFX8-NEXT:    s_lshr_b32 s6, s4, 20
+; GFX8-NEXT:    s_lshr_b32 s7, s4, 16
+; GFX8-NEXT:    s_lshr_b32 s8, s4, 28
+; GFX8-NEXT:    s_lshr_b32 s4, s4, 24
+; GFX8-NEXT:    v_or_b32_sdwa v5, v6, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_sdwa v3, v3, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_sdwa v3, v5, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT:    v_lshlrev_b16_e64 v7, 12, s2
+; GFX8-NEXT:    v_lshlrev_b16_e64 v8, 12, s5
+; GFX8-NEXT:    v_lshlrev_b16_e64 v9, 12, s1
+; GFX8-NEXT:    v_lshlrev_b16_e64 v10, 12, s0
+; GFX8-NEXT:    v_lshlrev_b16_e64 v11, 12, s4
+; GFX8-NEXT:    v_lshlrev_b16_e64 v12, 12, s8
+; GFX8-NEXT:    v_lshlrev_b16_e64 v13, 12, s7
+; GFX8-NEXT:    v_lshlrev_b16_e64 v14, 12, s6
+; GFX8-NEXT:    v_ashrrev_i16_e32 v7, 12, v7
+; GFX8-NEXT:    v_ashrrev_i16_e32 v11, 12, v11
+; GFX8-NEXT:    v_ashrrev_i16_e32 v8, 12, v8
+; GFX8-NEXT:    v_ashrrev_i16_e32 v12, 12, v12
+; GFX8-NEXT:    v_ashrrev_i16_e32 v9, 12, v9
+; GFX8-NEXT:    v_ashrrev_i16_e32 v13, 12, v13
+; GFX8-NEXT:    v_ashrrev_i16_e32 v10, 12, v10
+; GFX8-NEXT:    v_ashrrev_i16_e32 v14, 12, v14
+; GFX8-NEXT:    v_lshrrev_b32_e32 v5, 8, v3
+; GFX8-NEXT:    v_mul_u32_u24_sdwa v7, v7, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX8-NEXT:    v_mul_u32_u24_sdwa v8, v8, v12 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX8-NEXT:    v_mul_u32_u24_sdwa v9, v9, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX8-NEXT:    v_mul_u32_u24_sdwa v10, v10, v14 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX8-NEXT:    v_or_b32_sdwa v9, v9, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_sdwa v7, v7, v8 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX8-NEXT:    v_or_b32_sdwa v4, v9, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT:    v_lshrrev_b32_e32 v6, 8, v4
+; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v2, v3
+; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v5, v2
+; GFX8-NEXT:    v_add_u32_sdwa v2, vcc, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_2
+; GFX8-NEXT:    v_add_u32_sdwa v2, vcc, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
+; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v2, v4
+; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v2, v6
+; GFX8-NEXT:    v_add_u32_sdwa v2, vcc, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT:    v_add_u32_sdwa v2, vcc, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
+; GFX8-NEXT:    flat_store_byte v[0:1], v2
+; GFX8-NEXT:    s_endpgm
+;
+; GFX9-LABEL: idot8_acc8_vecMul:
+; GFX9:       ; %bb.0: ; %entry
+; GFX9-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX9-NEXT:    s_load_dword s4, s[6:7], 0x0
+; GFX9-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    global_load_ubyte v2, v[0:1], off
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    s_lshr_b32 s0, s2, 4
+; GFX9-NEXT:    s_lshr_b32 s1, s2, 12
+; GFX9-NEXT:    s_lshr_b32 s5, s2, 8
+; GFX9-NEXT:    s_lshr_b32 s6, s4, 4
+; GFX9-NEXT:    s_lshr_b32 s7, s4, 12
+; GFX9-NEXT:    s_lshr_b32 s8, s4, 8
+; GFX9-NEXT:    v_lshlrev_b16_e64 v3, 12, s5
+; GFX9-NEXT:    v_lshlrev_b16_e64 v4, 12, s1
+; GFX9-NEXT:    v_lshlrev_b16_e64 v5, 12, s0
+; GFX9-NEXT:    v_lshlrev_b16_e64 v7, 12, s8
+; GFX9-NEXT:    v_lshlrev_b16_e64 v8, 12, s7
+; GFX9-NEXT:    v_lshlrev_b16_e64 v9, 12, s6
+; GFX9-NEXT:    v_lshlrev_b16_e64 v6, 12, s2
+; GFX9-NEXT:    v_lshlrev_b16_e64 v10, 12, s4
+; GFX9-NEXT:    v_ashrrev_i16_e32 v3, 12, v3
+; GFX9-NEXT:    v_ashrrev_i16_e32 v7, 12, v7
+; GFX9-NEXT:    v_ashrrev_i16_e32 v4, 12, v4
+; GFX9-NEXT:    v_ashrrev_i16_e32 v8, 12, v8
+; GFX9-NEXT:    v_ashrrev_i16_e32 v5, 12, v5
+; GFX9-NEXT:    v_ashrrev_i16_e32 v9, 12, v9
+; GFX9-NEXT:    v_ashrrev_i16_e32 v6, 12, v6
+; GFX9-NEXT:    v_ashrrev_i16_e32 v10, 12, v10
+; GFX9-NEXT:    v_mul_lo_u16_e32 v6, v6, v10
+; GFX9-NEXT:    v_mul_lo_u16_sdwa v5, v5, v9 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT:    v_mul_lo_u16_sdwa v4, v4, v8 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT:    v_mul_lo_u16_e32 v3, v3, v7
+; GFX9-NEXT:    s_lshr_b32 s0, s2, 20
+; GFX9-NEXT:    s_lshr_b32 s1, s2, 16
+; GFX9-NEXT:    s_lshr_b32 s5, s2, 28
+; GFX9-NEXT:    s_lshr_b32 s2, s2, 24
+; GFX9-NEXT:    s_lshr_b32 s6, s4, 20
+; GFX9-NEXT:    s_lshr_b32 s7, s4, 16
+; GFX9-NEXT:    s_lshr_b32 s8, s4, 28
+; GFX9-NEXT:    s_lshr_b32 s4, s4, 24
+; GFX9-NEXT:    v_or_b32_sdwa v5, v6, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v3, v3, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v3, v5, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX9-NEXT:    v_lshlrev_b16_e64 v9, 12, s2
+; GFX9-NEXT:    v_lshlrev_b16_e64 v10, 12, s5
+; GFX9-NEXT:    v_lshlrev_b16_e64 v11, 12, s1
+; GFX9-NEXT:    v_lshlrev_b16_e64 v12, 12, s0
+; GFX9-NEXT:    v_lshlrev_b16_e64 v13, 12, s4
+; GFX9-NEXT:    v_lshlrev_b16_e64 v14, 12, s8
+; GFX9-NEXT:    v_lshlrev_b16_e64 v15, 12, s7
+; GFX9-NEXT:    v_lshlrev_b16_e64 v16, 12, s6
+; GFX9-NEXT:    v_ashrrev_i16_e32 v9, 12, v9
+; GFX9-NEXT:    v_ashrrev_i16_e32 v13, 12, v13
+; GFX9-NEXT:    v_ashrrev_i16_e32 v10, 12, v10
+; GFX9-NEXT:    v_ashrrev_i16_e32 v14, 12, v14
+; GFX9-NEXT:    v_ashrrev_i16_e32 v11, 12, v11
+; GFX9-NEXT:    v_ashrrev_i16_e32 v15, 12, v15
+; GFX9-NEXT:    v_ashrrev_i16_e32 v12, 12, v12
+; GFX9-NEXT:    v_ashrrev_i16_e32 v16, 12, v16
+; GFX9-NEXT:    v_lshrrev_b32_e32 v5, 8, v3
+; GFX9-NEXT:    v_mul_lo_u16_sdwa v12, v12, v16 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT:    v_mul_lo_u16_e32 v11, v11, v15
+; GFX9-NEXT:    v_mul_lo_u16_sdwa v10, v10, v14 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT:    v_mul_lo_u16_e32 v9, v9, v13
+; GFX9-NEXT:    v_or_b32_sdwa v7, v11, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v8, v9, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v4, v7, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_add_u32_e32 v2, v3, v2
+; GFX9-NEXT:    v_add_u32_e32 v2, v2, v5
+; GFX9-NEXT:    v_add_u32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_2
+; GFX9-NEXT:    v_add_u32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
+; GFX9-NEXT:    v_add_u32_e32 v2, v2, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v3, 8, v4
+; GFX9-NEXT:    v_add_u32_e32 v2, v2, v3
+; GFX9-NEXT:    v_add_u32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT:    v_add_u32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
+; GFX9-NEXT:    global_store_byte v[0:1], v2, off
+; GFX9-NEXT:    s_endpgm
+;
+; GFX9-DL-LABEL: idot8_acc8_vecMul:
+; GFX9-DL:       ; %bb.0: ; %entry
+; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
+; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
+; GFX9-DL-NEXT:    s_load_dword s4, s[6:7], 0x0
+; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-DL-NEXT:    global_load_ubyte v2, v[0:1], off
+; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-DL-NEXT:    s_lshr_b32 s0, s2, 4
+; GFX9-DL-NEXT:    s_lshr_b32 s1, s2, 12
+; GFX9-DL-NEXT:    s_lshr_b32 s5, s2, 8
+; GFX9-DL-NEXT:    s_lshr_b32 s6, s4, 4
+; GFX9-DL-NEXT:    s_lshr_b32 s7, s4, 12
+; GFX9-DL-NEXT:    s_lshr_b32 s8, s4, 8
+; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v3, 12, s5
+; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v4, 12, s1
+; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v5, 12, s0
+; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v7, 12, s8
+; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v8, 12, s7
+; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v9, 12, s6
+; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v6, 12, s2
+; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v10, 12, s4
+; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v3, 12, v3
+; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v7, 12, v7
+; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v4, 12, v4
+; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v8, 12, v8
+; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v5, 12, v5
+; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v9, 12, v9
+; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v6, 12, v6
+; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v10, 12, v10
+; GFX9-DL-NEXT:    v_mul_lo_u16_e32 v6, v6, v10
+; GFX9-DL-NEXT:    v_mul_lo_u16_sdwa v5, v5, v9 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-DL-NEXT:    v_mul_lo_u16_sdwa v4, v4, v8 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-DL-NEXT:    v_mul_lo_u16_e32 v3, v3, v7
+; GFX9-DL-NEXT:    s_lshr_b32 s0, s2, 20
+; GFX9-DL-NEXT:    s_lshr_b32 s1, s2, 16
+; GFX9-DL-NEXT:    s_lshr_b32 s5, s2, 28
+; GFX9-DL-NEXT:    s_lshr_b32 s2, s2, 24
+; GFX9-DL-NEXT:    s_lshr_b32 s6, s4, 20
+; GFX9-DL-NEXT:    s_lshr_b32 s7, s4, 16
+; GFX9-DL-NEXT:    s_lshr_b32 s8, s4, 28
+; GFX9-DL-NEXT:    s_lshr_b32 s4, s4, 24
+; GFX9-DL-NEXT:    v_or_b32_sdwa v5, v6, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-DL-NEXT:    v_or_b32_sdwa v3, v3, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-DL-NEXT:    v_or_b32_sdwa v3, v5, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v9, 12, s2
+; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v10, 12, s5
+; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v11, 12, s1
+; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v12, 12, s0
+; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v13, 12, s4
+; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v14, 12, s8
+; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v15, 12, s7
+; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v16, 12, s6
+; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v9, 12, v9
+; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v13, 12, v13
+; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v10, 12, v10
+; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v14, 12, v14
+; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v11, 12, v11
+; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v15, 12, v15
+; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v12, 12, v12
+; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v16, 12, v16
+; GFX9-DL-NEXT:    v_lshrrev_b32_e32 v5, 8, v3
+; GFX9-DL-NEXT:    v_mul_lo_u16_sdwa v12, v12, v16 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-DL-NEXT:    v_mul_lo_u16_e32 v11, v11, v15
+; GFX9-DL-NEXT:    v_mul_lo_u16_sdwa v10, v10, v14 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-DL-NEXT:    v_mul_lo_u16_e32 v9, v9, v13
+; GFX9-DL-NEXT:    v_or_b32_sdwa v7, v11, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-DL-NEXT:    v_or_b32_sdwa v8, v9, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-DL-NEXT:    v_or_b32_sdwa v4, v7, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX9-DL-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-DL-NEXT:    v_add_u32_e32 v2, v3, v2
+; GFX9-DL-NEXT:    v_add_u32_e32 v2, v2, v5
+; GFX9-DL-NEXT:    v_add_u32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_2
+; GFX9-DL-NEXT:    v_add_u32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
+; GFX9-DL-NEXT:    v_add_u32_e32 v2, v2, v4
+; GFX9-DL-NEXT:    v_lshrrev_b32_e32 v3, 8, v4
+; GFX9-DL-NEXT:    v_add_u32_e32 v2, v2, v3
+; GFX9-DL-NEXT:    v_add_u32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-DL-NEXT:    v_add_u32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
+; GFX9-DL-NEXT:    global_store_byte v[0:1], v2, off
+; GFX9-DL-NEXT:    s_endpgm
+                                             <8 x i4> addrspace(1)* %src2,
+                                             i8 addrspace(1)* nocapture %dst) {
+entry:
+  %vec1 = load <8 x i4>, <8 x i4> addrspace(1)* %src1
+  %vec2 = load <8 x i4>, <8 x i4> addrspace(1)* %src2
+
+  %cvec1 = sext <8 x i4> %vec1 to <8 x i8>
+  %cvec2 = sext <8 x i4> %vec2 to <8 x i8>
+
+  %mul = mul <8 x i8> %cvec1, %cvec2
+  %mul0 = extractelement <8 x i8> %mul, i64 0
+  %mul1 = extractelement <8 x i8> %mul, i64 1
+  %mul2 = extractelement <8 x i8> %mul, i64 2
+  %mul3 = extractelement <8 x i8> %mul, i64 3
+  %mul4 = extractelement <8 x i8> %mul, i64 4
+  %mul5 = extractelement <8 x i8> %mul, i64 5
+  %mul6 = extractelement <8 x i8> %mul, i64 6
+  %mul7 = extractelement <8 x i8> %mul, i64 7
+
+  %acc = load i8, i8 addrspace(1)* %dst, align 4
+  %add1 = add i8 %mul0, %acc
+  %add2 = add i8 %add1, %mul1
+  %add3 = add i8 %add2, %mul2
+  %add4 = add i8 %add3, %mul3
+  %add5 = add i8 %add4, %mul4
+  %add6 = add i8 %add5, %mul5
+  %add7 = add i8 %add6, %mul6
+  %add8 = add i8 %add7, %mul7
+
+  store i8 %add8, i8 addrspace(1)* %dst, align 4
+  ret void
+}
diff --git a/test/CodeGen/AMDGPU/idot8.ll b/test/CodeGen/AMDGPU/idot8u.ll
similarity index 55%
rename from test/CodeGen/AMDGPU/idot8.ll
rename to test/CodeGen/AMDGPU/idot8u.ll
index edb88f1912efc322ff9b6ae66c8eb6c2e47990f3..b480ac22ea9976b489972ad6ca0d8e9782acbf73 100644
--- a/test/CodeGen/AMDGPU/idot8.ll
+++ b/test/CodeGen/AMDGPU/idot8u.ll
@@ -2502,2112 +2502,6 @@ entry:
   ret void
 }
 
-define amdgpu_kernel void @idot8_acc32(<8 x i4> addrspace(1)* %src1,
-; GFX7-LABEL: idot8_acc32:
-; GFX7:       ; %bb.0: ; %entry
-; GFX7-NEXT:    s_load_dwordx4 s[8:11], s[0:1], 0x9
-; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0xd
-; GFX7-NEXT:    s_mov_b32 s7, 0xf000
-; GFX7-NEXT:    s_mov_b32 s6, -1
-; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    s_load_dword s0, s[8:9], 0x0
-; GFX7-NEXT:    s_load_dword s1, s[10:11], 0x0
-; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
-; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    s_bfe_i32 s8, s0, 0x40000
-; GFX7-NEXT:    s_bfe_i32 s9, s1, 0x40000
-; GFX7-NEXT:    s_bfe_i32 s11, s1, 0x40004
-; GFX7-NEXT:    v_mov_b32_e32 v0, s9
-; GFX7-NEXT:    v_mov_b32_e32 v1, s2
-; GFX7-NEXT:    v_mad_i32_i24 v0, s8, v0, v1
-; GFX7-NEXT:    s_bfe_i32 s10, s0, 0x40004
-; GFX7-NEXT:    v_mov_b32_e32 v1, s11
-; GFX7-NEXT:    s_bfe_i32 s13, s1, 0x40008
-; GFX7-NEXT:    v_mad_i32_i24 v0, s10, v1, v0
-; GFX7-NEXT:    s_bfe_i32 s12, s0, 0x40008
-; GFX7-NEXT:    v_mov_b32_e32 v1, s13
-; GFX7-NEXT:    s_bfe_i32 s15, s1, 0x4000c
-; GFX7-NEXT:    v_mad_i32_i24 v0, s12, v1, v0
-; GFX7-NEXT:    s_bfe_i32 s14, s0, 0x4000c
-; GFX7-NEXT:    v_mov_b32_e32 v1, s15
-; GFX7-NEXT:    s_bfe_i32 s17, s1, 0x40010
-; GFX7-NEXT:    v_mad_i32_i24 v0, s14, v1, v0
-; GFX7-NEXT:    s_bfe_i32 s16, s0, 0x40010
-; GFX7-NEXT:    v_mov_b32_e32 v1, s17
-; GFX7-NEXT:    s_bfe_i32 s19, s1, 0x40014
-; GFX7-NEXT:    s_bfe_i32 s21, s1, 0x40018
-; GFX7-NEXT:    v_mad_i32_i24 v0, s16, v1, v0
-; GFX7-NEXT:    s_bfe_i32 s18, s0, 0x40014
-; GFX7-NEXT:    v_mov_b32_e32 v1, s19
-; GFX7-NEXT:    s_bfe_i32 s20, s0, 0x40018
-; GFX7-NEXT:    v_mad_i32_i24 v0, s18, v1, v0
-; GFX7-NEXT:    v_mov_b32_e32 v1, s21
-; GFX7-NEXT:    s_ashr_i32 s1, s1, 28
-; GFX7-NEXT:    v_mad_i32_i24 v0, s20, v1, v0
-; GFX7-NEXT:    s_ashr_i32 s0, s0, 28
-; GFX7-NEXT:    v_mov_b32_e32 v1, s1
-; GFX7-NEXT:    v_mad_i32_i24 v0, s0, v1, v0
-; GFX7-NEXT:    buffer_store_dword v0, off, s[4:7], 0
-; GFX7-NEXT:    s_endpgm
-;
-; GFX8-LABEL: idot8_acc32:
-; GFX8:       ; %bb.0: ; %entry
-; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
-; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
-; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    s_load_dword s2, s[4:5], 0x0
-; GFX8-NEXT:    s_load_dword s4, s[6:7], 0x0
-; GFX8-NEXT:    s_load_dword s5, s[0:1], 0x0
-; GFX8-NEXT:    v_mov_b32_e32 v0, s0
-; GFX8-NEXT:    v_mov_b32_e32 v1, s1
-; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    s_bfe_i32 s0, s2, 0x40000
-; GFX8-NEXT:    s_bfe_i32 s1, s4, 0x40000
-; GFX8-NEXT:    s_bfe_i32 s7, s4, 0x40004
-; GFX8-NEXT:    v_mov_b32_e32 v2, s1
-; GFX8-NEXT:    v_mov_b32_e32 v3, s5
-; GFX8-NEXT:    v_mad_i32_i24 v2, s0, v2, v3
-; GFX8-NEXT:    s_bfe_i32 s6, s2, 0x40004
-; GFX8-NEXT:    v_mov_b32_e32 v3, s7
-; GFX8-NEXT:    s_bfe_i32 s9, s4, 0x40008
-; GFX8-NEXT:    v_mad_i32_i24 v2, s6, v3, v2
-; GFX8-NEXT:    s_bfe_i32 s8, s2, 0x40008
-; GFX8-NEXT:    v_mov_b32_e32 v3, s9
-; GFX8-NEXT:    s_bfe_i32 s11, s4, 0x4000c
-; GFX8-NEXT:    v_mad_i32_i24 v2, s8, v3, v2
-; GFX8-NEXT:    s_bfe_i32 s10, s2, 0x4000c
-; GFX8-NEXT:    v_mov_b32_e32 v3, s11
-; GFX8-NEXT:    s_bfe_i32 s13, s4, 0x40010
-; GFX8-NEXT:    v_mad_i32_i24 v2, s10, v3, v2
-; GFX8-NEXT:    s_bfe_i32 s12, s2, 0x40010
-; GFX8-NEXT:    v_mov_b32_e32 v3, s13
-; GFX8-NEXT:    s_bfe_i32 s15, s4, 0x40014
-; GFX8-NEXT:    s_bfe_i32 s17, s4, 0x40018
-; GFX8-NEXT:    v_mad_i32_i24 v2, s12, v3, v2
-; GFX8-NEXT:    s_bfe_i32 s14, s2, 0x40014
-; GFX8-NEXT:    v_mov_b32_e32 v3, s15
-; GFX8-NEXT:    s_bfe_i32 s16, s2, 0x40018
-; GFX8-NEXT:    v_mad_i32_i24 v2, s14, v3, v2
-; GFX8-NEXT:    v_mov_b32_e32 v3, s17
-; GFX8-NEXT:    s_ashr_i32 s4, s4, 28
-; GFX8-NEXT:    v_mad_i32_i24 v2, s16, v3, v2
-; GFX8-NEXT:    s_ashr_i32 s2, s2, 28
-; GFX8-NEXT:    v_mov_b32_e32 v3, s4
-; GFX8-NEXT:    v_mad_i32_i24 v2, s2, v3, v2
-; GFX8-NEXT:    flat_store_dword v[0:1], v2
-; GFX8-NEXT:    s_endpgm
-;
-; GFX9-LABEL: idot8_acc32:
-; GFX9:       ; %bb.0: ; %entry
-; GFX9-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
-; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
-; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_load_dword s2, s[4:5], 0x0
-; GFX9-NEXT:    s_load_dword s4, s[6:7], 0x0
-; GFX9-NEXT:    s_load_dword s5, s[0:1], 0x0
-; GFX9-NEXT:    v_mov_b32_e32 v0, s0
-; GFX9-NEXT:    v_mov_b32_e32 v1, s1
-; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_bfe_i32 s0, s2, 0x40000
-; GFX9-NEXT:    s_bfe_i32 s1, s4, 0x40000
-; GFX9-NEXT:    s_bfe_i32 s7, s4, 0x40004
-; GFX9-NEXT:    v_mov_b32_e32 v2, s1
-; GFX9-NEXT:    v_mov_b32_e32 v3, s5
-; GFX9-NEXT:    v_mad_i32_i24 v2, s0, v2, v3
-; GFX9-NEXT:    s_bfe_i32 s6, s2, 0x40004
-; GFX9-NEXT:    v_mov_b32_e32 v3, s7
-; GFX9-NEXT:    s_bfe_i32 s9, s4, 0x40008
-; GFX9-NEXT:    v_mad_i32_i24 v2, s6, v3, v2
-; GFX9-NEXT:    s_bfe_i32 s8, s2, 0x40008
-; GFX9-NEXT:    v_mov_b32_e32 v3, s9
-; GFX9-NEXT:    s_bfe_i32 s11, s4, 0x4000c
-; GFX9-NEXT:    v_mad_i32_i24 v2, s8, v3, v2
-; GFX9-NEXT:    s_bfe_i32 s10, s2, 0x4000c
-; GFX9-NEXT:    v_mov_b32_e32 v3, s11
-; GFX9-NEXT:    s_bfe_i32 s13, s4, 0x40010
-; GFX9-NEXT:    v_mad_i32_i24 v2, s10, v3, v2
-; GFX9-NEXT:    s_bfe_i32 s12, s2, 0x40010
-; GFX9-NEXT:    v_mov_b32_e32 v3, s13
-; GFX9-NEXT:    s_bfe_i32 s15, s4, 0x40014
-; GFX9-NEXT:    s_bfe_i32 s17, s4, 0x40018
-; GFX9-NEXT:    v_mad_i32_i24 v2, s12, v3, v2
-; GFX9-NEXT:    s_bfe_i32 s14, s2, 0x40014
-; GFX9-NEXT:    v_mov_b32_e32 v3, s15
-; GFX9-NEXT:    s_bfe_i32 s16, s2, 0x40018
-; GFX9-NEXT:    v_mad_i32_i24 v2, s14, v3, v2
-; GFX9-NEXT:    v_mov_b32_e32 v3, s17
-; GFX9-NEXT:    s_ashr_i32 s4, s4, 28
-; GFX9-NEXT:    v_mad_i32_i24 v2, s16, v3, v2
-; GFX9-NEXT:    s_ashr_i32 s2, s2, 28
-; GFX9-NEXT:    v_mov_b32_e32 v3, s4
-; GFX9-NEXT:    v_mad_i32_i24 v2, s2, v3, v2
-; GFX9-NEXT:    global_store_dword v[0:1], v2, off
-; GFX9-NEXT:    s_endpgm
-;
-; GFX9-DL-LABEL: idot8_acc32:
-; GFX9-DL:       ; %bb.0: ; %entry
-; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
-; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
-; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
-; GFX9-DL-NEXT:    s_load_dword s4, s[6:7], 0x0
-; GFX9-DL-NEXT:    s_load_dword s5, s[0:1], 0x0
-; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
-; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
-; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s4
-; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s5
-; GFX9-DL-NEXT:    v_dot8_i32_i4 v2, s2, v2, v3
-; GFX9-DL-NEXT:    global_store_dword v[0:1], v2, off
-; GFX9-DL-NEXT:    s_endpgm
-                                       <8 x i4> addrspace(1)* %src2,
-                                       i32 addrspace(1)* nocapture %dst) {
-entry:
-  %vec1 = load <8 x i4>, <8 x i4> addrspace(1)* %src1
-  %vec2 = load <8 x i4>, <8 x i4> addrspace(1)* %src2
-
-  %v1e0 = extractelement <8 x i4> %vec1, i64 0
-  %cv1e0 = sext i4 %v1e0 to i32
-  %v2e0 = extractelement <8 x i4> %vec2, i64 0
-  %cv2e0 = sext i4 %v2e0 to i32
-  %mul0 = mul nuw nsw i32 %cv1e0, %cv2e0
-
-  %v1e1 = extractelement <8 x i4> %vec1, i64 1
-  %cv1e1 = sext i4 %v1e1 to i32
-  %v2e1 = extractelement <8 x i4> %vec2, i64 1
-  %cv2e1 = sext i4 %v2e1 to i32
-  %mul1 = mul nuw nsw i32 %cv1e1, %cv2e1
-
-  %v1e2 = extractelement <8 x i4> %vec1, i64 2
-  %cv1e2 = sext i4 %v1e2 to i32
-  %v2e2 = extractelement <8 x i4> %vec2, i64 2
-  %cv2e2 = sext i4 %v2e2 to i32
-  %mul2 = mul nuw nsw i32 %cv1e2, %cv2e2
-
-  %v1e3 = extractelement <8 x i4> %vec1, i64 3
-  %cv1e3 = sext i4 %v1e3 to i32
-  %v2e3 = extractelement <8 x i4> %vec2, i64 3
-  %cv2e3 = sext i4 %v2e3 to i32
-  %mul3 = mul nuw nsw i32 %cv1e3, %cv2e3
-
-  %v1e4 = extractelement <8 x i4> %vec1, i64 4
-  %cv1e4 = sext i4 %v1e4 to i32
-  %v2e4 = extractelement <8 x i4> %vec2, i64 4
-  %cv2e4 = sext i4 %v2e4 to i32
-  %mul4 = mul nuw nsw i32 %cv1e4, %cv2e4
-
-  %v1e5 = extractelement <8 x i4> %vec1, i64 5
-  %cv1e5 = sext i4 %v1e5 to i32
-  %v2e5 = extractelement <8 x i4> %vec2, i64 5
-  %cv2e5 = sext i4 %v2e5 to i32
-  %mul5 = mul nuw nsw i32 %cv1e5, %cv2e5
-
-  %v1e6 = extractelement <8 x i4> %vec1, i64 6
-  %cv1e6 = sext i4 %v1e6 to i32
-  %v2e6 = extractelement <8 x i4> %vec2, i64 6
-  %cv2e6 = sext i4 %v2e6 to i32
-  %mul6 = mul nuw nsw i32 %cv1e6, %cv2e6
-
-  %v1e7 = extractelement <8 x i4> %vec1, i64 7
-  %cv1e7 = sext i4 %v1e7 to i32
-  %v2e7 = extractelement <8 x i4> %vec2, i64 7
-  %cv2e7 = sext i4 %v2e7 to i32
-  %mul7 = mul nuw nsw i32 %cv1e7, %cv2e7
-
-  %acc = load i32, i32 addrspace(1)* %dst, align 4
-  %add1 = add i32 %mul0, %acc
-  %add2 = add i32 %add1, %mul1
-  %add3 = add i32 %add2, %mul2
-  %add4 = add i32 %add3, %mul3
-  %add5 = add i32 %add4, %mul4
-  %add6 = add i32 %add5, %mul5
-  %add7 = add i32 %add6, %mul6
-  %add8 = add i32 %add7, %mul7
-
-  store i32 %add8, i32 addrspace(1)* %dst, align 4
-  ret void
-}
-
-; TODO: Once the unnecessary zero extentions of the elements are removed;
-; pattern recognizer will kick in.
-define amdgpu_kernel void @idot8_acc16(<8 x i4> addrspace(1)* %src1,
-; GFX7-LABEL: idot8_acc16:
-; GFX7:       ; %bb.0: ; %entry
-; GFX7-NEXT:    s_load_dwordx4 s[8:11], s[0:1], 0x9
-; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0xd
-; GFX7-NEXT:    s_mov_b32 s7, 0xf000
-; GFX7-NEXT:    s_mov_b32 s6, -1
-; GFX7-NEXT:    s_mov_b32 s0, 0xffff
-; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    s_load_dword s1, s[8:9], 0x0
-; GFX7-NEXT:    buffer_load_ushort v0, off, s[4:7], 0
-; GFX7-NEXT:    s_load_dword s2, s[10:11], 0x0
-; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    s_bfe_i32 s8, s1, 0x40000
-; GFX7-NEXT:    s_bfe_i32 s10, s1, 0x40004
-; GFX7-NEXT:    s_bfe_i32 s9, s2, 0x40000
-; GFX7-NEXT:    s_bfe_i32 s11, s2, 0x40004
-; GFX7-NEXT:    s_and_b32 s9, s9, s0
-; GFX7-NEXT:    s_bfe_i32 s13, s2, 0x40008
-; GFX7-NEXT:    s_and_b32 s11, s11, s0
-; GFX7-NEXT:    s_and_b32 s8, s8, s0
-; GFX7-NEXT:    v_mov_b32_e32 v1, s9
-; GFX7-NEXT:    s_bfe_i32 s12, s1, 0x40008
-; GFX7-NEXT:    s_bfe_i32 s15, s2, 0x4000c
-; GFX7-NEXT:    s_and_b32 s13, s13, s0
-; GFX7-NEXT:    s_and_b32 s10, s10, s0
-; GFX7-NEXT:    v_mov_b32_e32 v2, s11
-; GFX7-NEXT:    s_bfe_i32 s14, s1, 0x4000c
-; GFX7-NEXT:    s_bfe_i32 s17, s2, 0x40010
-; GFX7-NEXT:    s_and_b32 s15, s15, s0
-; GFX7-NEXT:    s_and_b32 s12, s12, s0
-; GFX7-NEXT:    v_mov_b32_e32 v3, s13
-; GFX7-NEXT:    s_bfe_i32 s16, s1, 0x40010
-; GFX7-NEXT:    s_bfe_i32 s19, s2, 0x40014
-; GFX7-NEXT:    s_and_b32 s17, s17, s0
-; GFX7-NEXT:    s_and_b32 s14, s14, s0
-; GFX7-NEXT:    v_mov_b32_e32 v4, s15
-; GFX7-NEXT:    s_bfe_i32 s21, s2, 0x40018
-; GFX7-NEXT:    s_bfe_i32 s18, s1, 0x40014
-; GFX7-NEXT:    s_and_b32 s19, s19, s0
-; GFX7-NEXT:    s_and_b32 s16, s16, s0
-; GFX7-NEXT:    v_mov_b32_e32 v5, s17
-; GFX7-NEXT:    s_bfe_i32 s20, s1, 0x40018
-; GFX7-NEXT:    s_ashr_i32 s2, s2, 28
-; GFX7-NEXT:    s_and_b32 s21, s21, s0
-; GFX7-NEXT:    s_and_b32 s18, s18, s0
-; GFX7-NEXT:    v_mov_b32_e32 v6, s19
-; GFX7-NEXT:    s_ashr_i32 s1, s1, 28
-; GFX7-NEXT:    s_and_b32 s20, s20, s0
-; GFX7-NEXT:    s_and_b32 s2, s2, s0
-; GFX7-NEXT:    v_mov_b32_e32 v7, s21
-; GFX7-NEXT:    s_and_b32 s0, s1, s0
-; GFX7-NEXT:    s_waitcnt vmcnt(0)
-; GFX7-NEXT:    v_mad_u32_u24 v0, s8, v1, v0
-; GFX7-NEXT:    v_mad_u32_u24 v0, s10, v2, v0
-; GFX7-NEXT:    v_mad_u32_u24 v0, s12, v3, v0
-; GFX7-NEXT:    v_mad_u32_u24 v0, s14, v4, v0
-; GFX7-NEXT:    v_mad_u32_u24 v0, s16, v5, v0
-; GFX7-NEXT:    v_mad_u32_u24 v0, s18, v6, v0
-; GFX7-NEXT:    v_mad_u32_u24 v0, s20, v7, v0
-; GFX7-NEXT:    v_mov_b32_e32 v1, s2
-; GFX7-NEXT:    v_mad_u32_u24 v0, s0, v1, v0
-; GFX7-NEXT:    buffer_store_short v0, off, s[4:7], 0
-; GFX7-NEXT:    s_endpgm
-;
-; GFX8-LABEL: idot8_acc16:
-; GFX8:       ; %bb.0: ; %entry
-; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
-; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
-; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    s_load_dword s2, s[4:5], 0x0
-; GFX8-NEXT:    s_load_dword s4, s[6:7], 0x0
-; GFX8-NEXT:    v_mov_b32_e32 v0, s0
-; GFX8-NEXT:    v_mov_b32_e32 v1, s1
-; GFX8-NEXT:    flat_load_ushort v2, v[0:1]
-; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    s_lshr_b32 s0, s2, 4
-; GFX8-NEXT:    s_lshr_b32 s1, s4, 4
-; GFX8-NEXT:    s_bfe_i32 s5, s4, 0x40000
-; GFX8-NEXT:    v_lshlrev_b16_e64 v3, 12, s0
-; GFX8-NEXT:    v_lshlrev_b16_e64 v4, 12, s1
-; GFX8-NEXT:    s_bfe_i32 s0, s4, 0x40008
-; GFX8-NEXT:    v_mov_b32_e32 v5, s5
-; GFX8-NEXT:    s_bfe_i32 s6, s2, 0x40000
-; GFX8-NEXT:    s_lshr_b32 s1, s2, 12
-; GFX8-NEXT:    s_lshr_b32 s5, s4, 12
-; GFX8-NEXT:    v_mov_b32_e32 v6, s0
-; GFX8-NEXT:    s_bfe_i32 s7, s2, 0x40008
-; GFX8-NEXT:    v_ashrrev_i16_e32 v3, 12, v3
-; GFX8-NEXT:    v_ashrrev_i16_e32 v4, 12, v4
-; GFX8-NEXT:    v_lshlrev_b16_e64 v7, 12, s1
-; GFX8-NEXT:    v_lshlrev_b16_e64 v8, 12, s5
-; GFX8-NEXT:    v_mul_i32_i24_e32 v6, s7, v6
-; GFX8-NEXT:    s_lshr_b32 s0, s2, 20
-; GFX8-NEXT:    s_lshr_b32 s1, s4, 20
-; GFX8-NEXT:    s_bfe_i32 s5, s4, 0x40010
-; GFX8-NEXT:    v_ashrrev_i16_e32 v7, 12, v7
-; GFX8-NEXT:    v_ashrrev_i16_e32 v8, 12, v8
-; GFX8-NEXT:    v_lshlrev_b16_e64 v9, 12, s0
-; GFX8-NEXT:    v_lshlrev_b16_e64 v10, 12, s1
-; GFX8-NEXT:    s_bfe_i32 s8, s2, 0x40010
-; GFX8-NEXT:    v_mov_b32_e32 v13, s5
-; GFX8-NEXT:    s_lshr_b32 s0, s2, 28
-; GFX8-NEXT:    s_lshr_b32 s9, s4, 28
-; GFX8-NEXT:    s_bfe_i32 s4, s4, 0x40018
-; GFX8-NEXT:    v_ashrrev_i16_e32 v9, 12, v9
-; GFX8-NEXT:    v_ashrrev_i16_e32 v10, 12, v10
-; GFX8-NEXT:    v_lshlrev_b16_e64 v11, 12, s0
-; GFX8-NEXT:    v_lshlrev_b16_e64 v12, 12, s9
-; GFX8-NEXT:    s_bfe_i32 s2, s2, 0x40018
-; GFX8-NEXT:    v_ashrrev_i16_e32 v11, 12, v11
-; GFX8-NEXT:    v_ashrrev_i16_e32 v12, 12, v12
-; GFX8-NEXT:    s_waitcnt vmcnt(0)
-; GFX8-NEXT:    v_mad_i32_i24 v2, s6, v5, v2
-; GFX8-NEXT:    v_mad_u32_u24 v2, v3, v4, v2
-; GFX8-NEXT:    v_add_u32_sdwa v2, vcc, v6, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
-; GFX8-NEXT:    v_mad_u32_u24 v2, v7, v8, v2
-; GFX8-NEXT:    v_mad_i32_i24 v2, s8, v13, v2
-; GFX8-NEXT:    v_mad_u32_u24 v2, v9, v10, v2
-; GFX8-NEXT:    v_mov_b32_e32 v3, s4
-; GFX8-NEXT:    v_mad_i32_i24 v2, s2, v3, v2
-; GFX8-NEXT:    v_mad_u32_u24 v2, v11, v12, v2
-; GFX8-NEXT:    flat_store_short v[0:1], v2
-; GFX8-NEXT:    s_endpgm
-;
-; GFX9-LABEL: idot8_acc16:
-; GFX9:       ; %bb.0: ; %entry
-; GFX9-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
-; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
-; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_load_dword s2, s[4:5], 0x0
-; GFX9-NEXT:    s_load_dword s4, s[6:7], 0x0
-; GFX9-NEXT:    v_mov_b32_e32 v0, s0
-; GFX9-NEXT:    v_mov_b32_e32 v1, s1
-; GFX9-NEXT:    global_load_ushort v2, v[0:1], off
-; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_lshr_b32 s0, s2, 4
-; GFX9-NEXT:    s_lshr_b32 s1, s4, 4
-; GFX9-NEXT:    s_bfe_i32 s5, s4, 0x40000
-; GFX9-NEXT:    v_lshlrev_b16_e64 v3, 12, s0
-; GFX9-NEXT:    v_lshlrev_b16_e64 v4, 12, s1
-; GFX9-NEXT:    s_bfe_i32 s0, s4, 0x40008
-; GFX9-NEXT:    v_mov_b32_e32 v5, s5
-; GFX9-NEXT:    s_bfe_i32 s6, s2, 0x40000
-; GFX9-NEXT:    s_lshr_b32 s1, s2, 12
-; GFX9-NEXT:    s_lshr_b32 s5, s4, 12
-; GFX9-NEXT:    v_mov_b32_e32 v6, s0
-; GFX9-NEXT:    s_bfe_i32 s7, s2, 0x40008
-; GFX9-NEXT:    v_ashrrev_i16_e32 v3, 12, v3
-; GFX9-NEXT:    v_ashrrev_i16_e32 v4, 12, v4
-; GFX9-NEXT:    v_lshlrev_b16_e64 v7, 12, s1
-; GFX9-NEXT:    v_lshlrev_b16_e64 v8, 12, s5
-; GFX9-NEXT:    v_mul_i32_i24_e32 v6, s7, v6
-; GFX9-NEXT:    s_lshr_b32 s0, s2, 20
-; GFX9-NEXT:    s_lshr_b32 s1, s4, 20
-; GFX9-NEXT:    s_bfe_i32 s5, s4, 0x40010
-; GFX9-NEXT:    v_ashrrev_i16_e32 v7, 12, v7
-; GFX9-NEXT:    v_ashrrev_i16_e32 v8, 12, v8
-; GFX9-NEXT:    v_lshlrev_b16_e64 v9, 12, s0
-; GFX9-NEXT:    v_lshlrev_b16_e64 v10, 12, s1
-; GFX9-NEXT:    s_bfe_i32 s8, s2, 0x40010
-; GFX9-NEXT:    v_mov_b32_e32 v13, s5
-; GFX9-NEXT:    s_lshr_b32 s0, s2, 28
-; GFX9-NEXT:    s_lshr_b32 s9, s4, 28
-; GFX9-NEXT:    s_bfe_i32 s4, s4, 0x40018
-; GFX9-NEXT:    v_ashrrev_i16_e32 v9, 12, v9
-; GFX9-NEXT:    v_ashrrev_i16_e32 v10, 12, v10
-; GFX9-NEXT:    v_lshlrev_b16_e64 v11, 12, s0
-; GFX9-NEXT:    v_lshlrev_b16_e64 v12, 12, s9
-; GFX9-NEXT:    s_bfe_i32 s2, s2, 0x40018
-; GFX9-NEXT:    v_ashrrev_i16_e32 v11, 12, v11
-; GFX9-NEXT:    v_ashrrev_i16_e32 v12, 12, v12
-; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_mad_i32_i24 v2, s6, v5, v2
-; GFX9-NEXT:    v_mad_u32_u24 v2, v3, v4, v2
-; GFX9-NEXT:    v_add_u32_sdwa v2, v2, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
-; GFX9-NEXT:    v_mad_u32_u24 v2, v7, v8, v2
-; GFX9-NEXT:    v_mad_i32_i24 v2, s8, v13, v2
-; GFX9-NEXT:    v_mad_u32_u24 v2, v9, v10, v2
-; GFX9-NEXT:    v_mov_b32_e32 v3, s4
-; GFX9-NEXT:    v_mad_i32_i24 v2, s2, v3, v2
-; GFX9-NEXT:    v_mad_u32_u24 v2, v11, v12, v2
-; GFX9-NEXT:    global_store_short v[0:1], v2, off
-; GFX9-NEXT:    s_endpgm
-;
-; GFX9-DL-LABEL: idot8_acc16:
-; GFX9-DL:       ; %bb.0: ; %entry
-; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
-; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
-; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
-; GFX9-DL-NEXT:    s_load_dword s4, s[6:7], 0x0
-; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
-; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
-; GFX9-DL-NEXT:    global_load_ushort v2, v[0:1], off
-; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT:    s_lshr_b32 s0, s2, 4
-; GFX9-DL-NEXT:    s_lshr_b32 s1, s4, 4
-; GFX9-DL-NEXT:    s_bfe_i32 s5, s4, 0x40000
-; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v3, 12, s0
-; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v4, 12, s1
-; GFX9-DL-NEXT:    s_bfe_i32 s0, s4, 0x40008
-; GFX9-DL-NEXT:    v_mov_b32_e32 v5, s5
-; GFX9-DL-NEXT:    s_bfe_i32 s6, s2, 0x40000
-; GFX9-DL-NEXT:    s_lshr_b32 s1, s2, 12
-; GFX9-DL-NEXT:    s_lshr_b32 s5, s4, 12
-; GFX9-DL-NEXT:    v_mov_b32_e32 v6, s0
-; GFX9-DL-NEXT:    s_bfe_i32 s7, s2, 0x40008
-; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v3, 12, v3
-; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v4, 12, v4
-; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v7, 12, s1
-; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v8, 12, s5
-; GFX9-DL-NEXT:    v_mul_i32_i24_e32 v6, s7, v6
-; GFX9-DL-NEXT:    s_lshr_b32 s0, s2, 20
-; GFX9-DL-NEXT:    s_lshr_b32 s1, s4, 20
-; GFX9-DL-NEXT:    s_bfe_i32 s5, s4, 0x40010
-; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v7, 12, v7
-; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v8, 12, v8
-; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v9, 12, s0
-; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v10, 12, s1
-; GFX9-DL-NEXT:    s_bfe_i32 s8, s2, 0x40010
-; GFX9-DL-NEXT:    v_mov_b32_e32 v13, s5
-; GFX9-DL-NEXT:    s_lshr_b32 s0, s2, 28
-; GFX9-DL-NEXT:    s_lshr_b32 s9, s4, 28
-; GFX9-DL-NEXT:    s_bfe_i32 s4, s4, 0x40018
-; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v9, 12, v9
-; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v10, 12, v10
-; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v11, 12, s0
-; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v12, 12, s9
-; GFX9-DL-NEXT:    s_bfe_i32 s2, s2, 0x40018
-; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v11, 12, v11
-; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v12, 12, v12
-; GFX9-DL-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s6, v5, v2
-; GFX9-DL-NEXT:    v_mad_u32_u24 v2, v3, v4, v2
-; GFX9-DL-NEXT:    v_add_u32_sdwa v2, v2, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
-; GFX9-DL-NEXT:    v_mad_u32_u24 v2, v7, v8, v2
-; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s8, v13, v2
-; GFX9-DL-NEXT:    v_mad_u32_u24 v2, v9, v10, v2
-; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s4
-; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s2, v3, v2
-; GFX9-DL-NEXT:    v_mad_u32_u24 v2, v11, v12, v2
-; GFX9-DL-NEXT:    global_store_short v[0:1], v2, off
-; GFX9-DL-NEXT:    s_endpgm
-                                       <8 x i4> addrspace(1)* %src2,
-                                       i16 addrspace(1)* nocapture %dst) {
-entry:
-  %vec1 = load <8 x i4>, <8 x i4> addrspace(1)* %src1
-  %vec2 = load <8 x i4>, <8 x i4> addrspace(1)* %src2
-
-  %v1e0 = extractelement <8 x i4> %vec1, i64 0
-  %cv1e0 = sext i4 %v1e0 to i16
-  %v2e0 = extractelement <8 x i4> %vec2, i64 0
-  %cv2e0 = sext i4 %v2e0 to i16
-  %mul0 = mul nuw nsw i16 %cv1e0, %cv2e0
-
-  %v1e1 = extractelement <8 x i4> %vec1, i64 1
-  %cv1e1 = sext i4 %v1e1 to i16
-  %v2e1 = extractelement <8 x i4> %vec2, i64 1
-  %cv2e1 = sext i4 %v2e1 to i16
-  %mul1 = mul nuw nsw i16 %cv1e1, %cv2e1
-
-  %v1e2 = extractelement <8 x i4> %vec1, i64 2
-  %cv1e2 = sext i4 %v1e2 to i16
-  %v2e2 = extractelement <8 x i4> %vec2, i64 2
-  %cv2e2 = sext i4 %v2e2 to i16
-  %mul2 = mul nuw nsw i16 %cv1e2, %cv2e2
-
-  %v1e3 = extractelement <8 x i4> %vec1, i64 3
-  %cv1e3 = sext i4 %v1e3 to i16
-  %v2e3 = extractelement <8 x i4> %vec2, i64 3
-  %cv2e3 = sext i4 %v2e3 to i16
-  %mul3 = mul nuw nsw i16 %cv1e3, %cv2e3
-
-  %v1e4 = extractelement <8 x i4> %vec1, i64 4
-  %cv1e4 = sext i4 %v1e4 to i16
-  %v2e4 = extractelement <8 x i4> %vec2, i64 4
-  %cv2e4 = sext i4 %v2e4 to i16
-  %mul4 = mul nuw nsw i16 %cv1e4, %cv2e4
-
-  %v1e5 = extractelement <8 x i4> %vec1, i64 5
-  %cv1e5 = sext i4 %v1e5 to i16
-  %v2e5 = extractelement <8 x i4> %vec2, i64 5
-  %cv2e5 = sext i4 %v2e5 to i16
-  %mul5 = mul nuw nsw i16 %cv1e5, %cv2e5
-
-  %v1e6 = extractelement <8 x i4> %vec1, i64 6
-  %cv1e6 = sext i4 %v1e6 to i16
-  %v2e6 = extractelement <8 x i4> %vec2, i64 6
-  %cv2e6 = sext i4 %v2e6 to i16
-  %mul6 = mul nuw nsw i16 %cv1e6, %cv2e6
-
-  %v1e7 = extractelement <8 x i4> %vec1, i64 7
-  %cv1e7 = sext i4 %v1e7 to i16
-  %v2e7 = extractelement <8 x i4> %vec2, i64 7
-  %cv2e7 = sext i4 %v2e7 to i16
-  %mul7 = mul nuw nsw i16 %cv1e7, %cv2e7
-
-  %acc = load i16, i16 addrspace(1)* %dst, align 4
-  %add1 = add i16 %mul0, %acc
-  %add2 = add i16 %add1, %mul1
-  %add3 = add i16 %add2, %mul2
-  %add4 = add i16 %add3, %mul3
-  %add5 = add i16 %add4, %mul4
-  %add6 = add i16 %add5, %mul5
-  %add7 = add i16 %add6, %mul6
-  %add8 = add i16 %add7, %mul7
-
-  store i16 %add8, i16 addrspace(1)* %dst, align 4
-  ret void
-}
-
-; TODO: Support this pattern.
-define amdgpu_kernel void @idot8_acc8(<8 x i4> addrspace(1)* %src1,
-; GFX7-LABEL: idot8_acc8:
-; GFX7:       ; %bb.0: ; %entry
-; GFX7-NEXT:    s_load_dwordx4 s[8:11], s[0:1], 0x9
-; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0xd
-; GFX7-NEXT:    s_mov_b32 s7, 0xf000
-; GFX7-NEXT:    s_mov_b32 s6, -1
-; GFX7-NEXT:    s_movk_i32 s0, 0xff
-; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    s_load_dword s1, s[8:9], 0x0
-; GFX7-NEXT:    buffer_load_ubyte v0, off, s[4:7], 0
-; GFX7-NEXT:    s_load_dword s2, s[10:11], 0x0
-; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    s_bfe_i32 s8, s1, 0x40000
-; GFX7-NEXT:    s_bfe_i32 s10, s1, 0x40004
-; GFX7-NEXT:    s_bfe_i32 s9, s2, 0x40000
-; GFX7-NEXT:    s_bfe_i32 s11, s2, 0x40004
-; GFX7-NEXT:    s_and_b32 s9, s9, s0
-; GFX7-NEXT:    s_bfe_i32 s13, s2, 0x40008
-; GFX7-NEXT:    s_and_b32 s11, s11, s0
-; GFX7-NEXT:    s_and_b32 s8, s8, s0
-; GFX7-NEXT:    v_mov_b32_e32 v1, s9
-; GFX7-NEXT:    s_bfe_i32 s12, s1, 0x40008
-; GFX7-NEXT:    s_bfe_i32 s15, s2, 0x4000c
-; GFX7-NEXT:    s_and_b32 s13, s13, s0
-; GFX7-NEXT:    s_and_b32 s10, s10, s0
-; GFX7-NEXT:    v_mov_b32_e32 v2, s11
-; GFX7-NEXT:    s_bfe_i32 s14, s1, 0x4000c
-; GFX7-NEXT:    s_bfe_i32 s17, s2, 0x40010
-; GFX7-NEXT:    s_and_b32 s15, s15, s0
-; GFX7-NEXT:    s_and_b32 s12, s12, s0
-; GFX7-NEXT:    v_mov_b32_e32 v3, s13
-; GFX7-NEXT:    s_bfe_i32 s16, s1, 0x40010
-; GFX7-NEXT:    s_bfe_i32 s19, s2, 0x40014
-; GFX7-NEXT:    s_and_b32 s17, s17, s0
-; GFX7-NEXT:    s_and_b32 s14, s14, s0
-; GFX7-NEXT:    v_mov_b32_e32 v4, s15
-; GFX7-NEXT:    s_bfe_i32 s21, s2, 0x40018
-; GFX7-NEXT:    s_bfe_i32 s18, s1, 0x40014
-; GFX7-NEXT:    s_and_b32 s19, s19, s0
-; GFX7-NEXT:    s_and_b32 s16, s16, s0
-; GFX7-NEXT:    v_mov_b32_e32 v5, s17
-; GFX7-NEXT:    s_bfe_i32 s20, s1, 0x40018
-; GFX7-NEXT:    s_ashr_i32 s2, s2, 28
-; GFX7-NEXT:    s_and_b32 s21, s21, s0
-; GFX7-NEXT:    s_and_b32 s18, s18, s0
-; GFX7-NEXT:    v_mov_b32_e32 v6, s19
-; GFX7-NEXT:    s_ashr_i32 s1, s1, 28
-; GFX7-NEXT:    s_and_b32 s20, s20, s0
-; GFX7-NEXT:    s_and_b32 s2, s2, s0
-; GFX7-NEXT:    v_mov_b32_e32 v7, s21
-; GFX7-NEXT:    s_and_b32 s0, s1, s0
-; GFX7-NEXT:    s_waitcnt vmcnt(0)
-; GFX7-NEXT:    v_mad_u32_u24 v0, s8, v1, v0
-; GFX7-NEXT:    v_mad_u32_u24 v0, s10, v2, v0
-; GFX7-NEXT:    v_mad_u32_u24 v0, s12, v3, v0
-; GFX7-NEXT:    v_mad_u32_u24 v0, s14, v4, v0
-; GFX7-NEXT:    v_mad_u32_u24 v0, s16, v5, v0
-; GFX7-NEXT:    v_mad_u32_u24 v0, s18, v6, v0
-; GFX7-NEXT:    v_mad_u32_u24 v0, s20, v7, v0
-; GFX7-NEXT:    v_mov_b32_e32 v1, s2
-; GFX7-NEXT:    v_mad_u32_u24 v0, s0, v1, v0
-; GFX7-NEXT:    buffer_store_byte v0, off, s[4:7], 0
-; GFX7-NEXT:    s_endpgm
-;
-; GFX8-LABEL: idot8_acc8:
-; GFX8:       ; %bb.0: ; %entry
-; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
-; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
-; GFX8-NEXT:    s_movk_i32 s2, 0xff
-; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    v_mov_b32_e32 v0, s0
-; GFX8-NEXT:    v_mov_b32_e32 v1, s1
-; GFX8-NEXT:    flat_load_ubyte v2, v[0:1]
-; GFX8-NEXT:    s_load_dword s0, s[4:5], 0x0
-; GFX8-NEXT:    s_load_dword s1, s[6:7], 0x0
-; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    s_lshr_b32 s7, s0, 4
-; GFX8-NEXT:    s_lshr_b32 s11, s1, 4
-; GFX8-NEXT:    v_lshlrev_b16_e64 v3, 12, s7
-; GFX8-NEXT:    v_lshlrev_b16_e64 v4, 12, s11
-; GFX8-NEXT:    s_bfe_i32 s13, s1, 0x40000
-; GFX8-NEXT:    v_ashrrev_i16_e32 v3, 12, v3
-; GFX8-NEXT:    v_ashrrev_i16_e32 v4, 12, v4
-; GFX8-NEXT:    s_lshr_b32 s6, s0, 12
-; GFX8-NEXT:    s_lshr_b32 s10, s1, 12
-; GFX8-NEXT:    s_bfe_i32 s15, s1, 0x40008
-; GFX8-NEXT:    s_bfe_i32 s12, s0, 0x40000
-; GFX8-NEXT:    v_mov_b32_e32 v12, s13
-; GFX8-NEXT:    v_lshlrev_b16_e64 v6, 12, s6
-; GFX8-NEXT:    v_lshlrev_b16_e64 v7, 12, s10
-; GFX8-NEXT:    s_bfe_i32 s14, s0, 0x40008
-; GFX8-NEXT:    v_mov_b32_e32 v5, s15
-; GFX8-NEXT:    v_and_b32_e32 v3, s2, v3
-; GFX8-NEXT:    v_and_b32_e32 v4, s2, v4
-; GFX8-NEXT:    v_ashrrev_i16_e32 v6, 12, v6
-; GFX8-NEXT:    v_ashrrev_i16_e32 v7, 12, v7
-; GFX8-NEXT:    s_lshr_b32 s5, s0, 20
-; GFX8-NEXT:    s_lshr_b32 s9, s1, 20
-; GFX8-NEXT:    v_mul_i32_i24_e32 v5, s14, v5
-; GFX8-NEXT:    v_lshlrev_b16_e64 v8, 12, s5
-; GFX8-NEXT:    v_lshlrev_b16_e64 v9, 12, s9
-; GFX8-NEXT:    s_bfe_i32 s17, s1, 0x40010
-; GFX8-NEXT:    v_and_b32_e32 v6, s2, v6
-; GFX8-NEXT:    v_and_b32_e32 v7, s2, v7
-; GFX8-NEXT:    s_lshr_b32 s8, s1, 28
-; GFX8-NEXT:    v_ashrrev_i16_e32 v8, 12, v8
-; GFX8-NEXT:    v_ashrrev_i16_e32 v9, 12, v9
-; GFX8-NEXT:    s_lshr_b32 s4, s0, 28
-; GFX8-NEXT:    s_bfe_i32 s16, s0, 0x40010
-; GFX8-NEXT:    v_mov_b32_e32 v13, s17
-; GFX8-NEXT:    v_lshlrev_b16_e64 v10, 12, s4
-; GFX8-NEXT:    v_lshlrev_b16_e64 v11, 12, s8
-; GFX8-NEXT:    s_bfe_i32 s1, s1, 0x40018
-; GFX8-NEXT:    v_and_b32_e32 v8, s2, v8
-; GFX8-NEXT:    v_and_b32_e32 v9, s2, v9
-; GFX8-NEXT:    v_ashrrev_i16_e32 v10, 12, v10
-; GFX8-NEXT:    v_ashrrev_i16_e32 v11, 12, v11
-; GFX8-NEXT:    s_bfe_i32 s0, s0, 0x40018
-; GFX8-NEXT:    v_and_b32_e32 v10, s2, v10
-; GFX8-NEXT:    v_and_b32_e32 v11, s2, v11
-; GFX8-NEXT:    s_waitcnt vmcnt(0)
-; GFX8-NEXT:    v_mad_i32_i24 v2, s12, v12, v2
-; GFX8-NEXT:    v_mad_u32_u24 v2, v3, v4, v2
-; GFX8-NEXT:    v_add_u32_sdwa v2, vcc, v5, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX8-NEXT:    v_mad_u32_u24 v2, v6, v7, v2
-; GFX8-NEXT:    v_mad_i32_i24 v2, s16, v13, v2
-; GFX8-NEXT:    v_mad_u32_u24 v2, v8, v9, v2
-; GFX8-NEXT:    v_mov_b32_e32 v3, s1
-; GFX8-NEXT:    v_mad_i32_i24 v2, s0, v3, v2
-; GFX8-NEXT:    v_mad_u32_u24 v2, v10, v11, v2
-; GFX8-NEXT:    flat_store_byte v[0:1], v2
-; GFX8-NEXT:    s_endpgm
-;
-; GFX9-LABEL: idot8_acc8:
-; GFX9:       ; %bb.0: ; %entry
-; GFX9-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
-; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
-; GFX9-NEXT:    s_movk_i32 s2, 0xff
-; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v0, s0
-; GFX9-NEXT:    v_mov_b32_e32 v1, s1
-; GFX9-NEXT:    global_load_ubyte v2, v[0:1], off
-; GFX9-NEXT:    s_load_dword s0, s[4:5], 0x0
-; GFX9-NEXT:    s_load_dword s1, s[6:7], 0x0
-; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_lshr_b32 s7, s0, 4
-; GFX9-NEXT:    s_lshr_b32 s11, s1, 4
-; GFX9-NEXT:    v_lshlrev_b16_e64 v3, 12, s7
-; GFX9-NEXT:    v_lshlrev_b16_e64 v4, 12, s11
-; GFX9-NEXT:    s_bfe_i32 s13, s1, 0x40000
-; GFX9-NEXT:    v_ashrrev_i16_e32 v3, 12, v3
-; GFX9-NEXT:    v_ashrrev_i16_e32 v4, 12, v4
-; GFX9-NEXT:    s_lshr_b32 s6, s0, 12
-; GFX9-NEXT:    s_lshr_b32 s10, s1, 12
-; GFX9-NEXT:    s_bfe_i32 s15, s1, 0x40008
-; GFX9-NEXT:    s_bfe_i32 s12, s0, 0x40000
-; GFX9-NEXT:    v_mov_b32_e32 v12, s13
-; GFX9-NEXT:    v_lshlrev_b16_e64 v6, 12, s6
-; GFX9-NEXT:    v_lshlrev_b16_e64 v7, 12, s10
-; GFX9-NEXT:    s_bfe_i32 s14, s0, 0x40008
-; GFX9-NEXT:    v_mov_b32_e32 v5, s15
-; GFX9-NEXT:    v_and_b32_e32 v3, s2, v3
-; GFX9-NEXT:    v_and_b32_e32 v4, s2, v4
-; GFX9-NEXT:    v_ashrrev_i16_e32 v6, 12, v6
-; GFX9-NEXT:    v_ashrrev_i16_e32 v7, 12, v7
-; GFX9-NEXT:    s_lshr_b32 s5, s0, 20
-; GFX9-NEXT:    s_lshr_b32 s9, s1, 20
-; GFX9-NEXT:    v_mul_i32_i24_e32 v5, s14, v5
-; GFX9-NEXT:    v_lshlrev_b16_e64 v8, 12, s5
-; GFX9-NEXT:    v_lshlrev_b16_e64 v9, 12, s9
-; GFX9-NEXT:    s_bfe_i32 s17, s1, 0x40010
-; GFX9-NEXT:    v_and_b32_e32 v6, s2, v6
-; GFX9-NEXT:    v_and_b32_e32 v7, s2, v7
-; GFX9-NEXT:    s_lshr_b32 s8, s1, 28
-; GFX9-NEXT:    v_ashrrev_i16_e32 v8, 12, v8
-; GFX9-NEXT:    v_ashrrev_i16_e32 v9, 12, v9
-; GFX9-NEXT:    s_lshr_b32 s4, s0, 28
-; GFX9-NEXT:    s_bfe_i32 s16, s0, 0x40010
-; GFX9-NEXT:    v_mov_b32_e32 v13, s17
-; GFX9-NEXT:    v_lshlrev_b16_e64 v10, 12, s4
-; GFX9-NEXT:    v_lshlrev_b16_e64 v11, 12, s8
-; GFX9-NEXT:    s_bfe_i32 s1, s1, 0x40018
-; GFX9-NEXT:    v_and_b32_e32 v8, s2, v8
-; GFX9-NEXT:    v_and_b32_e32 v9, s2, v9
-; GFX9-NEXT:    v_ashrrev_i16_e32 v10, 12, v10
-; GFX9-NEXT:    v_ashrrev_i16_e32 v11, 12, v11
-; GFX9-NEXT:    s_bfe_i32 s0, s0, 0x40018
-; GFX9-NEXT:    v_and_b32_e32 v10, s2, v10
-; GFX9-NEXT:    v_and_b32_e32 v11, s2, v11
-; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_mad_i32_i24 v2, s12, v12, v2
-; GFX9-NEXT:    v_mad_u32_u24 v2, v3, v4, v2
-; GFX9-NEXT:    v_add_u32_sdwa v2, v2, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX9-NEXT:    v_mad_u32_u24 v2, v6, v7, v2
-; GFX9-NEXT:    v_mad_i32_i24 v2, s16, v13, v2
-; GFX9-NEXT:    v_mad_u32_u24 v2, v8, v9, v2
-; GFX9-NEXT:    v_mov_b32_e32 v3, s1
-; GFX9-NEXT:    v_mad_i32_i24 v2, s0, v3, v2
-; GFX9-NEXT:    v_mad_u32_u24 v2, v10, v11, v2
-; GFX9-NEXT:    global_store_byte v[0:1], v2, off
-; GFX9-NEXT:    s_endpgm
-;
-; GFX9-DL-LABEL: idot8_acc8:
-; GFX9-DL:       ; %bb.0: ; %entry
-; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
-; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
-; GFX9-DL-NEXT:    s_movk_i32 s2, 0xff
-; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
-; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
-; GFX9-DL-NEXT:    global_load_ubyte v2, v[0:1], off
-; GFX9-DL-NEXT:    s_load_dword s0, s[4:5], 0x0
-; GFX9-DL-NEXT:    s_load_dword s1, s[6:7], 0x0
-; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT:    s_lshr_b32 s7, s0, 4
-; GFX9-DL-NEXT:    s_lshr_b32 s11, s1, 4
-; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v3, 12, s7
-; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v4, 12, s11
-; GFX9-DL-NEXT:    s_bfe_i32 s13, s1, 0x40000
-; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v3, 12, v3
-; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v4, 12, v4
-; GFX9-DL-NEXT:    s_lshr_b32 s6, s0, 12
-; GFX9-DL-NEXT:    s_lshr_b32 s10, s1, 12
-; GFX9-DL-NEXT:    s_bfe_i32 s15, s1, 0x40008
-; GFX9-DL-NEXT:    s_bfe_i32 s12, s0, 0x40000
-; GFX9-DL-NEXT:    v_mov_b32_e32 v12, s13
-; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v6, 12, s6
-; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v7, 12, s10
-; GFX9-DL-NEXT:    s_bfe_i32 s14, s0, 0x40008
-; GFX9-DL-NEXT:    v_mov_b32_e32 v5, s15
-; GFX9-DL-NEXT:    v_and_b32_e32 v3, s2, v3
-; GFX9-DL-NEXT:    v_and_b32_e32 v4, s2, v4
-; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v6, 12, v6
-; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v7, 12, v7
-; GFX9-DL-NEXT:    s_lshr_b32 s5, s0, 20
-; GFX9-DL-NEXT:    s_lshr_b32 s9, s1, 20
-; GFX9-DL-NEXT:    v_mul_i32_i24_e32 v5, s14, v5
-; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v8, 12, s5
-; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v9, 12, s9
-; GFX9-DL-NEXT:    s_bfe_i32 s17, s1, 0x40010
-; GFX9-DL-NEXT:    v_and_b32_e32 v6, s2, v6
-; GFX9-DL-NEXT:    v_and_b32_e32 v7, s2, v7
-; GFX9-DL-NEXT:    s_lshr_b32 s8, s1, 28
-; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v8, 12, v8
-; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v9, 12, v9
-; GFX9-DL-NEXT:    s_lshr_b32 s4, s0, 28
-; GFX9-DL-NEXT:    s_bfe_i32 s16, s0, 0x40010
-; GFX9-DL-NEXT:    v_mov_b32_e32 v13, s17
-; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v10, 12, s4
-; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v11, 12, s8
-; GFX9-DL-NEXT:    s_bfe_i32 s1, s1, 0x40018
-; GFX9-DL-NEXT:    v_and_b32_e32 v8, s2, v8
-; GFX9-DL-NEXT:    v_and_b32_e32 v9, s2, v9
-; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v10, 12, v10
-; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v11, 12, v11
-; GFX9-DL-NEXT:    s_bfe_i32 s0, s0, 0x40018
-; GFX9-DL-NEXT:    v_and_b32_e32 v10, s2, v10
-; GFX9-DL-NEXT:    v_and_b32_e32 v11, s2, v11
-; GFX9-DL-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s12, v12, v2
-; GFX9-DL-NEXT:    v_mad_u32_u24 v2, v3, v4, v2
-; GFX9-DL-NEXT:    v_add_u32_sdwa v2, v2, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX9-DL-NEXT:    v_mad_u32_u24 v2, v6, v7, v2
-; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s16, v13, v2
-; GFX9-DL-NEXT:    v_mad_u32_u24 v2, v8, v9, v2
-; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s1
-; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s0, v3, v2
-; GFX9-DL-NEXT:    v_mad_u32_u24 v2, v10, v11, v2
-; GFX9-DL-NEXT:    global_store_byte v[0:1], v2, off
-; GFX9-DL-NEXT:    s_endpgm
-                                       <8 x i4> addrspace(1)* %src2,
-                                       i8 addrspace(1)* nocapture %dst) {
-entry:
-  %vec1 = load <8 x i4>, <8 x i4> addrspace(1)* %src1
-  %vec2 = load <8 x i4>, <8 x i4> addrspace(1)* %src2
-
-  %v1e0 = extractelement <8 x i4> %vec1, i64 0
-  %cv1e0 = sext i4 %v1e0 to i8
-  %v2e0 = extractelement <8 x i4> %vec2, i64 0
-  %cv2e0 = sext i4 %v2e0 to i8
-  %mul0 = mul nuw nsw i8 %cv1e0, %cv2e0
-
-  %v1e1 = extractelement <8 x i4> %vec1, i64 1
-  %cv1e1 = sext i4 %v1e1 to i8
-  %v2e1 = extractelement <8 x i4> %vec2, i64 1
-  %cv2e1 = sext i4 %v2e1 to i8
-  %mul1 = mul nuw nsw i8 %cv1e1, %cv2e1
-
-  %v1e2 = extractelement <8 x i4> %vec1, i64 2
-  %cv1e2 = sext i4 %v1e2 to i8
-  %v2e2 = extractelement <8 x i4> %vec2, i64 2
-  %cv2e2 = sext i4 %v2e2 to i8
-  %mul2 = mul nuw nsw i8 %cv1e2, %cv2e2
-
-  %v1e3 = extractelement <8 x i4> %vec1, i64 3
-  %cv1e3 = sext i4 %v1e3 to i8
-  %v2e3 = extractelement <8 x i4> %vec2, i64 3
-  %cv2e3 = sext i4 %v2e3 to i8
-  %mul3 = mul nuw nsw i8 %cv1e3, %cv2e3
-
-  %v1e4 = extractelement <8 x i4> %vec1, i64 4
-  %cv1e4 = sext i4 %v1e4 to i8
-  %v2e4 = extractelement <8 x i4> %vec2, i64 4
-  %cv2e4 = sext i4 %v2e4 to i8
-  %mul4 = mul nuw nsw i8 %cv1e4, %cv2e4
-
-  %v1e5 = extractelement <8 x i4> %vec1, i64 5
-  %cv1e5 = sext i4 %v1e5 to i8
-  %v2e5 = extractelement <8 x i4> %vec2, i64 5
-  %cv2e5 = sext i4 %v2e5 to i8
-  %mul5 = mul nuw nsw i8 %cv1e5, %cv2e5
-
-  %v1e6 = extractelement <8 x i4> %vec1, i64 6
-  %cv1e6 = sext i4 %v1e6 to i8
-  %v2e6 = extractelement <8 x i4> %vec2, i64 6
-  %cv2e6 = sext i4 %v2e6 to i8
-  %mul6 = mul nuw nsw i8 %cv1e6, %cv2e6
-
-  %v1e7 = extractelement <8 x i4> %vec1, i64 7
-  %cv1e7 = sext i4 %v1e7 to i8
-  %v2e7 = extractelement <8 x i4> %vec2, i64 7
-  %cv2e7 = sext i4 %v2e7 to i8
-  %mul7 = mul nuw nsw i8 %cv1e7, %cv2e7
-
-  %acc = load i8, i8 addrspace(1)* %dst, align 4
-  %add1 = add i8 %mul0, %acc
-  %add2 = add i8 %add1, %mul1
-  %add3 = add i8 %add2, %mul2
-  %add4 = add i8 %add3, %mul3
-  %add5 = add i8 %add4, %mul4
-  %add6 = add i8 %add5, %mul5
-  %add7 = add i8 %add6, %mul6
-  %add8 = add i8 %add7, %mul7
-
-  store i8 %add8, i8 addrspace(1)* %dst, align 4
-  ret void
-}
-
-; Make sure the pattern is not recognized if there are multiple uses of the
-; intermediate multiplications.
-define amdgpu_kernel void @idot8_multiuses_mul1(<8 x i4> addrspace(1)* %src1,
-; GFX7-LABEL: idot8_multiuses_mul1:
-; GFX7:       ; %bb.0: ; %entry
-; GFX7-NEXT:    s_load_dwordx4 s[8:11], s[0:1], 0x9
-; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0xd
-; GFX7-NEXT:    s_mov_b32 s7, 0xf000
-; GFX7-NEXT:    s_mov_b32 s6, -1
-; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    s_load_dword s0, s[8:9], 0x0
-; GFX7-NEXT:    s_load_dword s1, s[10:11], 0x0
-; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
-; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    s_bfe_i32 s8, s0, 0x40000
-; GFX7-NEXT:    s_bfe_i32 s9, s1, 0x40000
-; GFX7-NEXT:    v_mov_b32_e32 v0, s9
-; GFX7-NEXT:    v_mov_b32_e32 v1, s2
-; GFX7-NEXT:    v_mad_i32_i24 v1, s8, v0, v1
-; GFX7-NEXT:    s_bfe_i32 s11, s1, 0x40004
-; GFX7-NEXT:    s_bfe_i32 s10, s0, 0x40004
-; GFX7-NEXT:    s_bfe_i32 s13, s1, 0x40008
-; GFX7-NEXT:    v_mad_i32_i24 v0, s8, v0, v1
-; GFX7-NEXT:    v_mov_b32_e32 v2, s11
-; GFX7-NEXT:    v_mad_i32_i24 v0, s10, v2, v0
-; GFX7-NEXT:    s_bfe_i32 s12, s0, 0x40008
-; GFX7-NEXT:    v_mov_b32_e32 v2, s13
-; GFX7-NEXT:    s_bfe_i32 s15, s1, 0x4000c
-; GFX7-NEXT:    v_mad_i32_i24 v0, s12, v2, v0
-; GFX7-NEXT:    s_bfe_i32 s14, s0, 0x4000c
-; GFX7-NEXT:    v_mov_b32_e32 v2, s15
-; GFX7-NEXT:    s_bfe_i32 s17, s1, 0x40010
-; GFX7-NEXT:    v_mad_i32_i24 v0, s14, v2, v0
-; GFX7-NEXT:    s_bfe_i32 s16, s0, 0x40010
-; GFX7-NEXT:    v_mov_b32_e32 v2, s17
-; GFX7-NEXT:    s_bfe_i32 s19, s1, 0x40014
-; GFX7-NEXT:    s_bfe_i32 s21, s1, 0x40018
-; GFX7-NEXT:    v_mad_i32_i24 v0, s16, v2, v0
-; GFX7-NEXT:    s_bfe_i32 s18, s0, 0x40014
-; GFX7-NEXT:    v_mov_b32_e32 v2, s19
-; GFX7-NEXT:    s_bfe_i32 s20, s0, 0x40018
-; GFX7-NEXT:    v_mad_i32_i24 v0, s18, v2, v0
-; GFX7-NEXT:    v_mov_b32_e32 v2, s21
-; GFX7-NEXT:    s_ashr_i32 s1, s1, 28
-; GFX7-NEXT:    v_mad_i32_i24 v0, s20, v2, v0
-; GFX7-NEXT:    s_ashr_i32 s0, s0, 28
-; GFX7-NEXT:    v_mov_b32_e32 v2, s1
-; GFX7-NEXT:    v_mad_i32_i24 v0, s0, v2, v0
-; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
-; GFX7-NEXT:    buffer_store_dword v0, off, s[4:7], 0
-; GFX7-NEXT:    s_endpgm
-;
-; GFX8-LABEL: idot8_multiuses_mul1:
-; GFX8:       ; %bb.0: ; %entry
-; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
-; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
-; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    s_load_dword s2, s[4:5], 0x0
-; GFX8-NEXT:    s_load_dword s4, s[6:7], 0x0
-; GFX8-NEXT:    s_load_dword s5, s[0:1], 0x0
-; GFX8-NEXT:    v_mov_b32_e32 v0, s0
-; GFX8-NEXT:    v_mov_b32_e32 v1, s1
-; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    s_bfe_i32 s0, s2, 0x40000
-; GFX8-NEXT:    s_bfe_i32 s1, s4, 0x40000
-; GFX8-NEXT:    v_mov_b32_e32 v2, s1
-; GFX8-NEXT:    v_mov_b32_e32 v3, s5
-; GFX8-NEXT:    v_mad_i32_i24 v3, s0, v2, v3
-; GFX8-NEXT:    s_bfe_i32 s7, s4, 0x40004
-; GFX8-NEXT:    s_bfe_i32 s6, s2, 0x40004
-; GFX8-NEXT:    s_bfe_i32 s9, s4, 0x40008
-; GFX8-NEXT:    v_mad_i32_i24 v2, s0, v2, v3
-; GFX8-NEXT:    v_mov_b32_e32 v4, s7
-; GFX8-NEXT:    v_mad_i32_i24 v2, s6, v4, v2
-; GFX8-NEXT:    s_bfe_i32 s8, s2, 0x40008
-; GFX8-NEXT:    v_mov_b32_e32 v4, s9
-; GFX8-NEXT:    s_bfe_i32 s11, s4, 0x4000c
-; GFX8-NEXT:    v_mad_i32_i24 v2, s8, v4, v2
-; GFX8-NEXT:    s_bfe_i32 s10, s2, 0x4000c
-; GFX8-NEXT:    v_mov_b32_e32 v4, s11
-; GFX8-NEXT:    s_bfe_i32 s13, s4, 0x40010
-; GFX8-NEXT:    v_mad_i32_i24 v2, s10, v4, v2
-; GFX8-NEXT:    s_bfe_i32 s12, s2, 0x40010
-; GFX8-NEXT:    v_mov_b32_e32 v4, s13
-; GFX8-NEXT:    s_bfe_i32 s15, s4, 0x40014
-; GFX8-NEXT:    s_bfe_i32 s17, s4, 0x40018
-; GFX8-NEXT:    v_mad_i32_i24 v2, s12, v4, v2
-; GFX8-NEXT:    s_bfe_i32 s14, s2, 0x40014
-; GFX8-NEXT:    v_mov_b32_e32 v4, s15
-; GFX8-NEXT:    s_bfe_i32 s16, s2, 0x40018
-; GFX8-NEXT:    v_mad_i32_i24 v2, s14, v4, v2
-; GFX8-NEXT:    v_mov_b32_e32 v4, s17
-; GFX8-NEXT:    s_ashr_i32 s4, s4, 28
-; GFX8-NEXT:    v_mad_i32_i24 v2, s16, v4, v2
-; GFX8-NEXT:    s_ashr_i32 s2, s2, 28
-; GFX8-NEXT:    v_mov_b32_e32 v4, s4
-; GFX8-NEXT:    v_mad_i32_i24 v2, s2, v4, v2
-; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v2, v3
-; GFX8-NEXT:    flat_store_dword v[0:1], v2
-; GFX8-NEXT:    s_endpgm
-;
-; GFX9-LABEL: idot8_multiuses_mul1:
-; GFX9:       ; %bb.0: ; %entry
-; GFX9-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
-; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
-; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_load_dword s2, s[4:5], 0x0
-; GFX9-NEXT:    s_load_dword s4, s[6:7], 0x0
-; GFX9-NEXT:    s_load_dword s5, s[0:1], 0x0
-; GFX9-NEXT:    v_mov_b32_e32 v0, s0
-; GFX9-NEXT:    v_mov_b32_e32 v1, s1
-; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_bfe_i32 s0, s2, 0x40000
-; GFX9-NEXT:    s_bfe_i32 s1, s4, 0x40000
-; GFX9-NEXT:    v_mov_b32_e32 v2, s1
-; GFX9-NEXT:    v_mov_b32_e32 v3, s5
-; GFX9-NEXT:    v_mad_i32_i24 v3, s0, v2, v3
-; GFX9-NEXT:    s_bfe_i32 s7, s4, 0x40004
-; GFX9-NEXT:    s_bfe_i32 s6, s2, 0x40004
-; GFX9-NEXT:    s_bfe_i32 s9, s4, 0x40008
-; GFX9-NEXT:    v_mad_i32_i24 v2, s0, v2, v3
-; GFX9-NEXT:    v_mov_b32_e32 v4, s7
-; GFX9-NEXT:    v_mad_i32_i24 v2, s6, v4, v2
-; GFX9-NEXT:    s_bfe_i32 s8, s2, 0x40008
-; GFX9-NEXT:    v_mov_b32_e32 v4, s9
-; GFX9-NEXT:    s_bfe_i32 s11, s4, 0x4000c
-; GFX9-NEXT:    v_mad_i32_i24 v2, s8, v4, v2
-; GFX9-NEXT:    s_bfe_i32 s10, s2, 0x4000c
-; GFX9-NEXT:    v_mov_b32_e32 v4, s11
-; GFX9-NEXT:    s_bfe_i32 s13, s4, 0x40010
-; GFX9-NEXT:    v_mad_i32_i24 v2, s10, v4, v2
-; GFX9-NEXT:    s_bfe_i32 s12, s2, 0x40010
-; GFX9-NEXT:    v_mov_b32_e32 v4, s13
-; GFX9-NEXT:    s_bfe_i32 s15, s4, 0x40014
-; GFX9-NEXT:    s_bfe_i32 s17, s4, 0x40018
-; GFX9-NEXT:    v_mad_i32_i24 v2, s12, v4, v2
-; GFX9-NEXT:    s_bfe_i32 s14, s2, 0x40014
-; GFX9-NEXT:    v_mov_b32_e32 v4, s15
-; GFX9-NEXT:    s_bfe_i32 s16, s2, 0x40018
-; GFX9-NEXT:    v_mad_i32_i24 v2, s14, v4, v2
-; GFX9-NEXT:    v_mov_b32_e32 v4, s17
-; GFX9-NEXT:    s_ashr_i32 s4, s4, 28
-; GFX9-NEXT:    v_mad_i32_i24 v2, s16, v4, v2
-; GFX9-NEXT:    s_ashr_i32 s2, s2, 28
-; GFX9-NEXT:    v_mov_b32_e32 v4, s4
-; GFX9-NEXT:    v_mad_i32_i24 v2, s2, v4, v2
-; GFX9-NEXT:    v_add_u32_e32 v2, v3, v2
-; GFX9-NEXT:    global_store_dword v[0:1], v2, off
-; GFX9-NEXT:    s_endpgm
-;
-; GFX9-DL-LABEL: idot8_multiuses_mul1:
-; GFX9-DL:       ; %bb.0: ; %entry
-; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
-; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
-; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
-; GFX9-DL-NEXT:    s_load_dword s4, s[6:7], 0x0
-; GFX9-DL-NEXT:    s_load_dword s5, s[0:1], 0x0
-; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
-; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
-; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT:    s_bfe_i32 s0, s2, 0x40000
-; GFX9-DL-NEXT:    s_bfe_i32 s1, s4, 0x40000
-; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s1
-; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s5
-; GFX9-DL-NEXT:    v_mad_i32_i24 v3, s0, v2, v3
-; GFX9-DL-NEXT:    s_bfe_i32 s7, s4, 0x40004
-; GFX9-DL-NEXT:    s_bfe_i32 s6, s2, 0x40004
-; GFX9-DL-NEXT:    s_bfe_i32 s9, s4, 0x40008
-; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s0, v2, v3
-; GFX9-DL-NEXT:    v_mov_b32_e32 v4, s7
-; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s6, v4, v2
-; GFX9-DL-NEXT:    s_bfe_i32 s8, s2, 0x40008
-; GFX9-DL-NEXT:    v_mov_b32_e32 v4, s9
-; GFX9-DL-NEXT:    s_bfe_i32 s11, s4, 0x4000c
-; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s8, v4, v2
-; GFX9-DL-NEXT:    s_bfe_i32 s10, s2, 0x4000c
-; GFX9-DL-NEXT:    v_mov_b32_e32 v4, s11
-; GFX9-DL-NEXT:    s_bfe_i32 s13, s4, 0x40010
-; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s10, v4, v2
-; GFX9-DL-NEXT:    s_bfe_i32 s12, s2, 0x40010
-; GFX9-DL-NEXT:    v_mov_b32_e32 v4, s13
-; GFX9-DL-NEXT:    s_bfe_i32 s15, s4, 0x40014
-; GFX9-DL-NEXT:    s_bfe_i32 s17, s4, 0x40018
-; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s12, v4, v2
-; GFX9-DL-NEXT:    s_bfe_i32 s14, s2, 0x40014
-; GFX9-DL-NEXT:    v_mov_b32_e32 v4, s15
-; GFX9-DL-NEXT:    s_bfe_i32 s16, s2, 0x40018
-; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s14, v4, v2
-; GFX9-DL-NEXT:    v_mov_b32_e32 v4, s17
-; GFX9-DL-NEXT:    s_ashr_i32 s4, s4, 28
-; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s16, v4, v2
-; GFX9-DL-NEXT:    s_ashr_i32 s2, s2, 28
-; GFX9-DL-NEXT:    v_mov_b32_e32 v4, s4
-; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s2, v4, v2
-; GFX9-DL-NEXT:    v_add_u32_e32 v2, v3, v2
-; GFX9-DL-NEXT:    global_store_dword v[0:1], v2, off
-; GFX9-DL-NEXT:    s_endpgm
-                                                <8 x i4> addrspace(1)* %src2,
-                                                i32 addrspace(1)* nocapture %dst) {
-entry:
-  %vec1 = load <8 x i4>, <8 x i4> addrspace(1)* %src1
-  %vec2 = load <8 x i4>, <8 x i4> addrspace(1)* %src2
-
-  %v1e0 = extractelement <8 x i4> %vec1, i64 0
-  %cv1e0 = sext i4 %v1e0 to i32
-  %v2e0 = extractelement <8 x i4> %vec2, i64 0
-  %cv2e0 = sext i4 %v2e0 to i32
-  %mul0 = mul nuw nsw i32 %cv1e0, %cv2e0
-
-  %v1e1 = extractelement <8 x i4> %vec1, i64 1
-  %cv1e1 = sext i4 %v1e1 to i32
-  %v2e1 = extractelement <8 x i4> %vec2, i64 1
-  %cv2e1 = sext i4 %v2e1 to i32
-  %mul1 = mul nuw nsw i32 %cv1e1, %cv2e1
-
-  %v1e2 = extractelement <8 x i4> %vec1, i64 2
-  %cv1e2 = sext i4 %v1e2 to i32
-  %v2e2 = extractelement <8 x i4> %vec2, i64 2
-  %cv2e2 = sext i4 %v2e2 to i32
-  %mul2 = mul nuw nsw i32 %cv1e2, %cv2e2
-
-  %v1e3 = extractelement <8 x i4> %vec1, i64 3
-  %cv1e3 = sext i4 %v1e3 to i32
-  %v2e3 = extractelement <8 x i4> %vec2, i64 3
-  %cv2e3 = sext i4 %v2e3 to i32
-  %mul3 = mul nuw nsw i32 %cv1e3, %cv2e3
-
-  %v1e4 = extractelement <8 x i4> %vec1, i64 4
-  %cv1e4 = sext i4 %v1e4 to i32
-  %v2e4 = extractelement <8 x i4> %vec2, i64 4
-  %cv2e4 = sext i4 %v2e4 to i32
-  %mul4 = mul nuw nsw i32 %cv1e4, %cv2e4
-
-  %v1e5 = extractelement <8 x i4> %vec1, i64 5
-  %cv1e5 = sext i4 %v1e5 to i32
-  %v2e5 = extractelement <8 x i4> %vec2, i64 5
-  %cv2e5 = sext i4 %v2e5 to i32
-  %mul5 = mul nuw nsw i32 %cv1e5, %cv2e5
-
-  %v1e6 = extractelement <8 x i4> %vec1, i64 6
-  %cv1e6 = sext i4 %v1e6 to i32
-  %v2e6 = extractelement <8 x i4> %vec2, i64 6
-  %cv2e6 = sext i4 %v2e6 to i32
-  %mul6 = mul nuw nsw i32 %cv1e6, %cv2e6
-
-  %v1e7 = extractelement <8 x i4> %vec1, i64 7
-  %cv1e7 = sext i4 %v1e7 to i32
-  %v2e7 = extractelement <8 x i4> %vec2, i64 7
-  %cv2e7 = sext i4 %v2e7 to i32
-  %mul7 = mul nuw nsw i32 %cv1e7, %cv2e7
-
-  %acc = load i32, i32 addrspace(1)* %dst, align 4
-  %add =  add i32  %mul0, %acc
-  %add1 = add i32 %mul0, %add
-  %add2 = add i32 %add1, %mul1
-  %add3 = add i32 %add2, %mul2
-  %add4 = add i32 %add3, %mul3
-  %add5 = add i32 %add4, %mul4
-  %add6 = add i32 %add5, %mul5
-  %add7 = add i32 %add6, %mul6
-  %add8 = add i32 %add7, %mul7
-
-  %res = add i32 %add, %add8
-  store i32 %res, i32 addrspace(1)* %dst, align 4
-  ret void
-}
-
-; TODO: Support this pattern.
-define amdgpu_kernel void @idot8_acc32_vecMul(<8 x i4> addrspace(1)* %src1,
-; GFX7-LABEL: idot8_acc32_vecMul:
-; GFX7:       ; %bb.0: ; %entry
-; GFX7-NEXT:    s_load_dwordx4 s[8:11], s[0:1], 0x9
-; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0xd
-; GFX7-NEXT:    s_mov_b32 s7, 0xf000
-; GFX7-NEXT:    s_mov_b32 s6, -1
-; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    s_load_dword s1, s[8:9], 0x0
-; GFX7-NEXT:    s_load_dword s9, s[10:11], 0x0
-; GFX7-NEXT:    s_load_dword s2, s[4:5], 0x0
-; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    s_ashr_i64 s[10:11], s[0:1], 60
-; GFX7-NEXT:    s_lshl_b32 s11, s1, 4
-; GFX7-NEXT:    s_ashr_i64 s[14:15], s[10:11], 60
-; GFX7-NEXT:    s_lshl_b32 s11, s1, 12
-; GFX7-NEXT:    s_ashr_i64 s[16:17], s[10:11], 60
-; GFX7-NEXT:    s_lshl_b32 s11, s1, 16
-; GFX7-NEXT:    s_ashr_i64 s[18:19], s[10:11], 60
-; GFX7-NEXT:    s_lshl_b32 s11, s1, 20
-; GFX7-NEXT:    s_lshl_b32 s13, s1, 8
-; GFX7-NEXT:    s_ashr_i64 s[20:21], s[10:11], 60
-; GFX7-NEXT:    s_lshl_b32 s11, s1, 24
-; GFX7-NEXT:    s_lshl_b32 s1, s1, 28
-; GFX7-NEXT:    s_ashr_i64 s[0:1], s[0:1], 60
-; GFX7-NEXT:    s_lshl_b32 s1, s9, 4
-; GFX7-NEXT:    s_ashr_i64 s[26:27], s[0:1], 60
-; GFX7-NEXT:    s_lshl_b32 s1, s9, 8
-; GFX7-NEXT:    s_ashr_i64 s[28:29], s[0:1], 60
-; GFX7-NEXT:    s_lshl_b32 s1, s9, 12
-; GFX7-NEXT:    s_ashr_i64 s[30:31], s[0:1], 60
-; GFX7-NEXT:    s_lshl_b32 s1, s9, 16
-; GFX7-NEXT:    s_ashr_i64 s[32:33], s[0:1], 60
-; GFX7-NEXT:    s_lshl_b32 s1, s9, 20
-; GFX7-NEXT:    s_ashr_i64 s[34:35], s[0:1], 60
-; GFX7-NEXT:    s_lshl_b32 s1, s9, 24
-; GFX7-NEXT:    s_ashr_i64 s[36:37], s[0:1], 60
-; GFX7-NEXT:    s_lshl_b32 s1, s9, 28
-; GFX7-NEXT:    s_ashr_i64 s[24:25], s[8:9], 60
-; GFX7-NEXT:    s_ashr_i64 s[8:9], s[0:1], 60
-; GFX7-NEXT:    v_mov_b32_e32 v0, s8
-; GFX7-NEXT:    v_mov_b32_e32 v1, s2
-; GFX7-NEXT:    v_mad_i32_i24 v0, s0, v0, v1
-; GFX7-NEXT:    s_ashr_i64 s[22:23], s[10:11], 60
-; GFX7-NEXT:    v_mov_b32_e32 v1, s36
-; GFX7-NEXT:    v_mad_i32_i24 v0, s22, v1, v0
-; GFX7-NEXT:    v_mov_b32_e32 v1, s34
-; GFX7-NEXT:    v_mad_i32_i24 v0, s20, v1, v0
-; GFX7-NEXT:    v_mov_b32_e32 v1, s32
-; GFX7-NEXT:    v_mad_i32_i24 v0, s18, v1, v0
-; GFX7-NEXT:    v_mov_b32_e32 v1, s30
-; GFX7-NEXT:    v_mad_i32_i24 v0, s16, v1, v0
-; GFX7-NEXT:    s_ashr_i64 s[12:13], s[12:13], 60
-; GFX7-NEXT:    v_mov_b32_e32 v1, s28
-; GFX7-NEXT:    v_mad_i32_i24 v0, s12, v1, v0
-; GFX7-NEXT:    v_mov_b32_e32 v1, s26
-; GFX7-NEXT:    v_mad_i32_i24 v0, s14, v1, v0
-; GFX7-NEXT:    v_mov_b32_e32 v1, s24
-; GFX7-NEXT:    v_mad_i32_i24 v0, s10, v1, v0
-; GFX7-NEXT:    buffer_store_dword v0, off, s[4:7], 0
-; GFX7-NEXT:    s_endpgm
-;
-; GFX8-LABEL: idot8_acc32_vecMul:
-; GFX8:       ; %bb.0: ; %entry
-; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
-; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
-; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    s_load_dword s5, s[4:5], 0x0
-; GFX8-NEXT:    s_load_dword s7, s[6:7], 0x0
-; GFX8-NEXT:    s_load_dword s2, s[0:1], 0x0
-; GFX8-NEXT:    v_mov_b32_e32 v0, s0
-; GFX8-NEXT:    v_mov_b32_e32 v1, s1
-; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    s_ashr_i64 s[0:1], s[4:5], 60
-; GFX8-NEXT:    s_lshl_b32 s1, s5, 4
-; GFX8-NEXT:    s_ashr_i64 s[12:13], s[0:1], 60
-; GFX8-NEXT:    s_lshl_b32 s1, s5, 16
-; GFX8-NEXT:    s_ashr_i64 s[14:15], s[0:1], 60
-; GFX8-NEXT:    s_lshl_b32 s1, s5, 20
-; GFX8-NEXT:    s_ashr_i64 s[16:17], s[0:1], 60
-; GFX8-NEXT:    s_lshl_b32 s1, s5, 24
-; GFX8-NEXT:    s_ashr_i64 s[18:19], s[0:1], 60
-; GFX8-NEXT:    s_lshl_b32 s1, s5, 28
-; GFX8-NEXT:    s_lshl_b32 s9, s5, 8
-; GFX8-NEXT:    s_lshl_b32 s11, s5, 12
-; GFX8-NEXT:    s_ashr_i64 s[4:5], s[0:1], 60
-; GFX8-NEXT:    s_lshl_b32 s1, s7, 4
-; GFX8-NEXT:    s_ashr_i64 s[22:23], s[0:1], 60
-; GFX8-NEXT:    s_lshl_b32 s1, s7, 8
-; GFX8-NEXT:    s_ashr_i64 s[24:25], s[0:1], 60
-; GFX8-NEXT:    s_lshl_b32 s1, s7, 12
-; GFX8-NEXT:    s_ashr_i64 s[26:27], s[0:1], 60
-; GFX8-NEXT:    s_lshl_b32 s1, s7, 16
-; GFX8-NEXT:    s_ashr_i64 s[28:29], s[0:1], 60
-; GFX8-NEXT:    s_lshl_b32 s1, s7, 20
-; GFX8-NEXT:    s_ashr_i64 s[30:31], s[0:1], 60
-; GFX8-NEXT:    s_lshl_b32 s1, s7, 24
-; GFX8-NEXT:    s_ashr_i64 s[32:33], s[0:1], 60
-; GFX8-NEXT:    s_lshl_b32 s1, s7, 28
-; GFX8-NEXT:    s_ashr_i64 s[20:21], s[6:7], 60
-; GFX8-NEXT:    s_ashr_i64 s[6:7], s[0:1], 60
-; GFX8-NEXT:    v_mov_b32_e32 v2, s6
-; GFX8-NEXT:    v_mov_b32_e32 v3, s2
-; GFX8-NEXT:    v_mad_i32_i24 v2, s4, v2, v3
-; GFX8-NEXT:    v_mov_b32_e32 v3, s32
-; GFX8-NEXT:    v_mad_i32_i24 v2, s18, v3, v2
-; GFX8-NEXT:    v_mov_b32_e32 v3, s30
-; GFX8-NEXT:    v_mad_i32_i24 v2, s16, v3, v2
-; GFX8-NEXT:    v_mov_b32_e32 v3, s28
-; GFX8-NEXT:    v_mad_i32_i24 v2, s14, v3, v2
-; GFX8-NEXT:    s_ashr_i64 s[10:11], s[10:11], 60
-; GFX8-NEXT:    v_mov_b32_e32 v3, s26
-; GFX8-NEXT:    v_mad_i32_i24 v2, s10, v3, v2
-; GFX8-NEXT:    s_ashr_i64 s[8:9], s[8:9], 60
-; GFX8-NEXT:    v_mov_b32_e32 v3, s24
-; GFX8-NEXT:    v_mad_i32_i24 v2, s8, v3, v2
-; GFX8-NEXT:    v_mov_b32_e32 v3, s22
-; GFX8-NEXT:    v_mad_i32_i24 v2, s12, v3, v2
-; GFX8-NEXT:    v_mov_b32_e32 v3, s20
-; GFX8-NEXT:    v_mad_i32_i24 v2, s0, v3, v2
-; GFX8-NEXT:    flat_store_dword v[0:1], v2
-; GFX8-NEXT:    s_endpgm
-;
-; GFX9-LABEL: idot8_acc32_vecMul:
-; GFX9:       ; %bb.0: ; %entry
-; GFX9-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
-; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
-; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_load_dword s5, s[4:5], 0x0
-; GFX9-NEXT:    s_load_dword s7, s[6:7], 0x0
-; GFX9-NEXT:    s_load_dword s2, s[0:1], 0x0
-; GFX9-NEXT:    v_mov_b32_e32 v0, s0
-; GFX9-NEXT:    v_mov_b32_e32 v1, s1
-; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_ashr_i64 s[0:1], s[4:5], 60
-; GFX9-NEXT:    s_lshl_b32 s1, s5, 4
-; GFX9-NEXT:    s_ashr_i64 s[12:13], s[0:1], 60
-; GFX9-NEXT:    s_lshl_b32 s1, s5, 16
-; GFX9-NEXT:    s_ashr_i64 s[14:15], s[0:1], 60
-; GFX9-NEXT:    s_lshl_b32 s1, s5, 20
-; GFX9-NEXT:    s_ashr_i64 s[16:17], s[0:1], 60
-; GFX9-NEXT:    s_lshl_b32 s1, s5, 24
-; GFX9-NEXT:    s_ashr_i64 s[18:19], s[0:1], 60
-; GFX9-NEXT:    s_lshl_b32 s1, s5, 28
-; GFX9-NEXT:    s_lshl_b32 s9, s5, 8
-; GFX9-NEXT:    s_lshl_b32 s11, s5, 12
-; GFX9-NEXT:    s_ashr_i64 s[4:5], s[0:1], 60
-; GFX9-NEXT:    s_lshl_b32 s1, s7, 4
-; GFX9-NEXT:    s_ashr_i64 s[22:23], s[0:1], 60
-; GFX9-NEXT:    s_lshl_b32 s1, s7, 8
-; GFX9-NEXT:    s_ashr_i64 s[24:25], s[0:1], 60
-; GFX9-NEXT:    s_lshl_b32 s1, s7, 12
-; GFX9-NEXT:    s_ashr_i64 s[26:27], s[0:1], 60
-; GFX9-NEXT:    s_lshl_b32 s1, s7, 16
-; GFX9-NEXT:    s_ashr_i64 s[28:29], s[0:1], 60
-; GFX9-NEXT:    s_lshl_b32 s1, s7, 20
-; GFX9-NEXT:    s_ashr_i64 s[30:31], s[0:1], 60
-; GFX9-NEXT:    s_lshl_b32 s1, s7, 24
-; GFX9-NEXT:    s_ashr_i64 s[32:33], s[0:1], 60
-; GFX9-NEXT:    s_lshl_b32 s1, s7, 28
-; GFX9-NEXT:    s_ashr_i64 s[20:21], s[6:7], 60
-; GFX9-NEXT:    s_ashr_i64 s[6:7], s[0:1], 60
-; GFX9-NEXT:    v_mov_b32_e32 v2, s6
-; GFX9-NEXT:    v_mov_b32_e32 v3, s2
-; GFX9-NEXT:    v_mad_i32_i24 v2, s4, v2, v3
-; GFX9-NEXT:    v_mov_b32_e32 v3, s32
-; GFX9-NEXT:    v_mad_i32_i24 v2, s18, v3, v2
-; GFX9-NEXT:    v_mov_b32_e32 v3, s30
-; GFX9-NEXT:    v_mad_i32_i24 v2, s16, v3, v2
-; GFX9-NEXT:    v_mov_b32_e32 v3, s28
-; GFX9-NEXT:    v_mad_i32_i24 v2, s14, v3, v2
-; GFX9-NEXT:    s_ashr_i64 s[10:11], s[10:11], 60
-; GFX9-NEXT:    v_mov_b32_e32 v3, s26
-; GFX9-NEXT:    v_mad_i32_i24 v2, s10, v3, v2
-; GFX9-NEXT:    s_ashr_i64 s[8:9], s[8:9], 60
-; GFX9-NEXT:    v_mov_b32_e32 v3, s24
-; GFX9-NEXT:    v_mad_i32_i24 v2, s8, v3, v2
-; GFX9-NEXT:    v_mov_b32_e32 v3, s22
-; GFX9-NEXT:    v_mad_i32_i24 v2, s12, v3, v2
-; GFX9-NEXT:    v_mov_b32_e32 v3, s20
-; GFX9-NEXT:    v_mad_i32_i24 v2, s0, v3, v2
-; GFX9-NEXT:    global_store_dword v[0:1], v2, off
-; GFX9-NEXT:    s_endpgm
-;
-; GFX9-DL-LABEL: idot8_acc32_vecMul:
-; GFX9-DL:       ; %bb.0: ; %entry
-; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
-; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
-; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT:    s_load_dword s5, s[4:5], 0x0
-; GFX9-DL-NEXT:    s_load_dword s7, s[6:7], 0x0
-; GFX9-DL-NEXT:    s_load_dword s2, s[0:1], 0x0
-; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
-; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
-; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT:    s_ashr_i64 s[0:1], s[4:5], 60
-; GFX9-DL-NEXT:    s_lshl_b32 s1, s5, 4
-; GFX9-DL-NEXT:    s_ashr_i64 s[12:13], s[0:1], 60
-; GFX9-DL-NEXT:    s_lshl_b32 s1, s5, 16
-; GFX9-DL-NEXT:    s_ashr_i64 s[14:15], s[0:1], 60
-; GFX9-DL-NEXT:    s_lshl_b32 s1, s5, 20
-; GFX9-DL-NEXT:    s_ashr_i64 s[16:17], s[0:1], 60
-; GFX9-DL-NEXT:    s_lshl_b32 s1, s5, 24
-; GFX9-DL-NEXT:    s_ashr_i64 s[18:19], s[0:1], 60
-; GFX9-DL-NEXT:    s_lshl_b32 s1, s5, 28
-; GFX9-DL-NEXT:    s_lshl_b32 s9, s5, 8
-; GFX9-DL-NEXT:    s_lshl_b32 s11, s5, 12
-; GFX9-DL-NEXT:    s_ashr_i64 s[4:5], s[0:1], 60
-; GFX9-DL-NEXT:    s_lshl_b32 s1, s7, 4
-; GFX9-DL-NEXT:    s_ashr_i64 s[22:23], s[0:1], 60
-; GFX9-DL-NEXT:    s_lshl_b32 s1, s7, 8
-; GFX9-DL-NEXT:    s_ashr_i64 s[24:25], s[0:1], 60
-; GFX9-DL-NEXT:    s_lshl_b32 s1, s7, 12
-; GFX9-DL-NEXT:    s_ashr_i64 s[26:27], s[0:1], 60
-; GFX9-DL-NEXT:    s_lshl_b32 s1, s7, 16
-; GFX9-DL-NEXT:    s_ashr_i64 s[28:29], s[0:1], 60
-; GFX9-DL-NEXT:    s_lshl_b32 s1, s7, 20
-; GFX9-DL-NEXT:    s_ashr_i64 s[30:31], s[0:1], 60
-; GFX9-DL-NEXT:    s_lshl_b32 s1, s7, 24
-; GFX9-DL-NEXT:    s_ashr_i64 s[32:33], s[0:1], 60
-; GFX9-DL-NEXT:    s_lshl_b32 s1, s7, 28
-; GFX9-DL-NEXT:    s_ashr_i64 s[20:21], s[6:7], 60
-; GFX9-DL-NEXT:    s_ashr_i64 s[6:7], s[0:1], 60
-; GFX9-DL-NEXT:    v_mov_b32_e32 v2, s6
-; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s2
-; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s4, v2, v3
-; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s32
-; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s18, v3, v2
-; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s30
-; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s16, v3, v2
-; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s28
-; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s14, v3, v2
-; GFX9-DL-NEXT:    s_ashr_i64 s[10:11], s[10:11], 60
-; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s26
-; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s10, v3, v2
-; GFX9-DL-NEXT:    s_ashr_i64 s[8:9], s[8:9], 60
-; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s24
-; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s8, v3, v2
-; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s22
-; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s12, v3, v2
-; GFX9-DL-NEXT:    v_mov_b32_e32 v3, s20
-; GFX9-DL-NEXT:    v_mad_i32_i24 v2, s0, v3, v2
-; GFX9-DL-NEXT:    global_store_dword v[0:1], v2, off
-; GFX9-DL-NEXT:    s_endpgm
-                                              <8 x i4> addrspace(1)* %src2,
-                                              i32 addrspace(1)* nocapture %dst) {
-entry:
-  %vec1 = load <8 x i4>, <8 x i4> addrspace(1)* %src1
-  %vec2 = load <8 x i4>, <8 x i4> addrspace(1)* %src2
-
-  %cvec1 = sext <8 x i4> %vec1 to <8 x i32>
-  %cvec2 = sext <8 x i4> %vec2 to <8 x i32>
-
-  %mul = mul <8 x i32> %cvec1, %cvec2
-  %mul0 = extractelement <8 x i32> %mul, i64 0
-  %mul1 = extractelement <8 x i32> %mul, i64 1
-  %mul2 = extractelement <8 x i32> %mul, i64 2
-  %mul3 = extractelement <8 x i32> %mul, i64 3
-  %mul4 = extractelement <8 x i32> %mul, i64 4
-  %mul5 = extractelement <8 x i32> %mul, i64 5
-  %mul6 = extractelement <8 x i32> %mul, i64 6
-  %mul7 = extractelement <8 x i32> %mul, i64 7
-
-  %acc = load i32, i32 addrspace(1)* %dst, align 4
-  %add1 = add i32 %mul0, %acc
-  %add2 = add i32 %add1, %mul1
-  %add3 = add i32 %add2, %mul2
-  %add4 = add i32 %add3, %mul3
-  %add5 = add i32 %add4, %mul4
-  %add6 = add i32 %add5, %mul5
-  %add7 = add i32 %add6, %mul6
-  %add8 = add i32 %add7, %mul7
-
-  store i32 %add8, i32 addrspace(1)* %dst, align 4
-  ret void
-}
-
-; TODO: Support this pattern.
-define amdgpu_kernel void @idot8_acc16_vecMul(<8 x i4> addrspace(1)* %src1,
-; GFX7-LABEL: idot8_acc16_vecMul:
-; GFX7:       ; %bb.0: ; %entry
-; GFX7-NEXT:    s_load_dwordx4 s[8:11], s[0:1], 0x9
-; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0xd
-; GFX7-NEXT:    s_mov_b32 s7, 0xf000
-; GFX7-NEXT:    s_mov_b32 s6, -1
-; GFX7-NEXT:    s_mov_b32 s0, 0xffff
-; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    s_load_dword s1, s[8:9], 0x0
-; GFX7-NEXT:    buffer_load_ushort v0, off, s[4:7], 0
-; GFX7-NEXT:    s_load_dword s2, s[10:11], 0x0
-; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    s_bfe_i32 s8, s1, 0x40010
-; GFX7-NEXT:    s_bfe_i32 s9, s1, 0x40014
-; GFX7-NEXT:    s_bfe_i32 s15, s2, 0x40010
-; GFX7-NEXT:    s_bfe_i32 s16, s2, 0x40014
-; GFX7-NEXT:    s_bfe_i32 s17, s2, 0x40018
-; GFX7-NEXT:    s_ashr_i32 s18, s2, 28
-; GFX7-NEXT:    s_bfe_i32 s19, s2, 0x40000
-; GFX7-NEXT:    s_bfe_i32 s20, s2, 0x40004
-; GFX7-NEXT:    s_bfe_i32 s21, s2, 0x40008
-; GFX7-NEXT:    s_bfe_i32 s2, s2, 0x4000c
-; GFX7-NEXT:    s_bfe_i32 s10, s1, 0x40018
-; GFX7-NEXT:    s_ashr_i32 s11, s1, 28
-; GFX7-NEXT:    s_bfe_i32 s12, s1, 0x40000
-; GFX7-NEXT:    v_mov_b32_e32 v4, s19
-; GFX7-NEXT:    s_bfe_i32 s13, s1, 0x40004
-; GFX7-NEXT:    v_mov_b32_e32 v3, s20
-; GFX7-NEXT:    s_bfe_i32 s14, s1, 0x40008
-; GFX7-NEXT:    v_mov_b32_e32 v2, s21
-; GFX7-NEXT:    s_bfe_i32 s1, s1, 0x4000c
-; GFX7-NEXT:    v_mov_b32_e32 v1, s2
-; GFX7-NEXT:    v_mov_b32_e32 v5, s18
-; GFX7-NEXT:    v_mov_b32_e32 v6, s17
-; GFX7-NEXT:    v_mul_i32_i24_e32 v1, s1, v1
-; GFX7-NEXT:    v_mul_i32_i24_e32 v2, s14, v2
-; GFX7-NEXT:    v_mul_i32_i24_e32 v3, s13, v3
-; GFX7-NEXT:    v_mul_i32_i24_e32 v4, s12, v4
-; GFX7-NEXT:    v_mul_i32_i24_e32 v5, s11, v5
-; GFX7-NEXT:    v_mul_i32_i24_e32 v6, s10, v6
-; GFX7-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; GFX7-NEXT:    v_and_b32_e32 v2, s0, v2
-; GFX7-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; GFX7-NEXT:    v_and_b32_e32 v4, s0, v4
-; GFX7-NEXT:    v_or_b32_e32 v1, v2, v1
-; GFX7-NEXT:    v_or_b32_e32 v2, v4, v3
-; GFX7-NEXT:    v_lshlrev_b32_e32 v5, 16, v5
-; GFX7-NEXT:    v_and_b32_e32 v6, s0, v6
-; GFX7-NEXT:    v_mov_b32_e32 v7, s16
-; GFX7-NEXT:    v_mov_b32_e32 v8, s15
-; GFX7-NEXT:    v_or_b32_e32 v3, v6, v5
-; GFX7-NEXT:    v_alignbit_b32 v5, v1, v2, 16
-; GFX7-NEXT:    v_mul_i32_i24_e32 v7, s9, v7
-; GFX7-NEXT:    v_mul_i32_i24_e32 v8, s8, v8
-; GFX7-NEXT:    v_lshlrev_b32_e32 v7, 16, v7
-; GFX7-NEXT:    v_and_b32_e32 v8, s0, v8
-; GFX7-NEXT:    v_lshrrev_b32_e32 v6, 16, v1
-; GFX7-NEXT:    v_or_b32_e32 v4, v8, v7
-; GFX7-NEXT:    v_lshrrev_b32_e32 v7, 16, v4
-; GFX7-NEXT:    v_lshrrev_b32_e32 v8, 16, v3
-; GFX7-NEXT:    s_waitcnt vmcnt(0)
-; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
-; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v5, v0
-; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
-; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v6, v0
-; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v4, v0
-; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v7, v0
-; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v3, v0
-; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v8, v0
-; GFX7-NEXT:    buffer_store_short v0, off, s[4:7], 0
-; GFX7-NEXT:    s_endpgm
-;
-; GFX8-LABEL: idot8_acc16_vecMul:
-; GFX8:       ; %bb.0: ; %entry
-; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
-; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
-; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    s_load_dword s2, s[4:5], 0x0
-; GFX8-NEXT:    s_load_dword s4, s[6:7], 0x0
-; GFX8-NEXT:    v_mov_b32_e32 v0, s0
-; GFX8-NEXT:    v_mov_b32_e32 v1, s1
-; GFX8-NEXT:    flat_load_ushort v2, v[0:1]
-; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    v_lshlrev_b16_e64 v3, 12, s2
-; GFX8-NEXT:    v_lshlrev_b16_e64 v4, 12, s4
-; GFX8-NEXT:    s_lshr_b32 s0, s2, 4
-; GFX8-NEXT:    s_lshr_b32 s1, s2, 8
-; GFX8-NEXT:    s_lshr_b32 s5, s4, 4
-; GFX8-NEXT:    s_lshr_b32 s6, s4, 8
-; GFX8-NEXT:    v_lshlrev_b16_e64 v5, 12, s1
-; GFX8-NEXT:    v_lshlrev_b16_e64 v6, 12, s0
-; GFX8-NEXT:    v_lshlrev_b16_e64 v7, 12, s6
-; GFX8-NEXT:    v_lshlrev_b16_e64 v8, 12, s5
-; GFX8-NEXT:    v_ashrrev_i16_e32 v3, 12, v3
-; GFX8-NEXT:    v_ashrrev_i16_e32 v4, 12, v4
-; GFX8-NEXT:    s_lshr_b32 s0, s2, 12
-; GFX8-NEXT:    s_lshr_b32 s1, s4, 12
-; GFX8-NEXT:    v_ashrrev_i16_e32 v6, 12, v6
-; GFX8-NEXT:    v_ashrrev_i16_e32 v5, 12, v5
-; GFX8-NEXT:    v_ashrrev_i16_e32 v7, 12, v7
-; GFX8-NEXT:    v_ashrrev_i16_e32 v8, 12, v8
-; GFX8-NEXT:    v_lshlrev_b16_e64 v9, 12, s0
-; GFX8-NEXT:    v_lshlrev_b16_e64 v10, 12, s1
-; GFX8-NEXT:    s_lshr_b32 s5, s2, 16
-; GFX8-NEXT:    s_lshr_b32 s6, s4, 16
-; GFX8-NEXT:    v_mul_u32_u24_e32 v5, v5, v7
-; GFX8-NEXT:    v_lshlrev_b16_e64 v11, 12, s5
-; GFX8-NEXT:    v_lshlrev_b16_e64 v12, 12, s6
-; GFX8-NEXT:    s_lshr_b32 s0, s2, 20
-; GFX8-NEXT:    s_lshr_b32 s1, s4, 20
-; GFX8-NEXT:    v_ashrrev_i16_e32 v9, 12, v9
-; GFX8-NEXT:    v_ashrrev_i16_e32 v10, 12, v10
-; GFX8-NEXT:    v_lshlrev_b16_e64 v13, 12, s0
-; GFX8-NEXT:    v_lshlrev_b16_e64 v14, 12, s1
-; GFX8-NEXT:    s_lshr_b32 s5, s2, 24
-; GFX8-NEXT:    s_lshr_b32 s6, s4, 24
-; GFX8-NEXT:    v_ashrrev_i16_e32 v11, 12, v11
-; GFX8-NEXT:    v_ashrrev_i16_e32 v12, 12, v12
-; GFX8-NEXT:    v_lshlrev_b16_e64 v15, 12, s5
-; GFX8-NEXT:    v_lshlrev_b16_e64 v17, 12, s6
-; GFX8-NEXT:    s_lshr_b32 s0, s2, 28
-; GFX8-NEXT:    s_lshr_b32 s1, s4, 28
-; GFX8-NEXT:    v_ashrrev_i16_e32 v13, 12, v13
-; GFX8-NEXT:    v_ashrrev_i16_e32 v14, 12, v14
-; GFX8-NEXT:    v_lshlrev_b16_e64 v16, 12, s0
-; GFX8-NEXT:    v_lshlrev_b16_e64 v18, 12, s1
-; GFX8-NEXT:    v_ashrrev_i16_e32 v15, 12, v15
-; GFX8-NEXT:    v_ashrrev_i16_e32 v17, 12, v17
-; GFX8-NEXT:    v_ashrrev_i16_e32 v16, 12, v16
-; GFX8-NEXT:    v_ashrrev_i16_e32 v18, 12, v18
-; GFX8-NEXT:    s_waitcnt vmcnt(0)
-; GFX8-NEXT:    v_mad_u32_u24 v2, v3, v4, v2
-; GFX8-NEXT:    v_mad_u32_u24 v2, v6, v8, v2
-; GFX8-NEXT:    v_add_u32_sdwa v2, vcc, v5, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
-; GFX8-NEXT:    v_mad_u32_u24 v2, v9, v10, v2
-; GFX8-NEXT:    v_mad_u32_u24 v2, v11, v12, v2
-; GFX8-NEXT:    v_mad_u32_u24 v2, v13, v14, v2
-; GFX8-NEXT:    v_mad_u32_u24 v2, v15, v17, v2
-; GFX8-NEXT:    v_mad_u32_u24 v2, v16, v18, v2
-; GFX8-NEXT:    flat_store_short v[0:1], v2
-; GFX8-NEXT:    s_endpgm
-;
-; GFX9-LABEL: idot8_acc16_vecMul:
-; GFX9:       ; %bb.0: ; %entry
-; GFX9-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
-; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
-; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_load_dword s2, s[4:5], 0x0
-; GFX9-NEXT:    s_load_dword s4, s[6:7], 0x0
-; GFX9-NEXT:    v_mov_b32_e32 v0, s0
-; GFX9-NEXT:    v_mov_b32_e32 v1, s1
-; GFX9-NEXT:    global_load_ushort v2, v[0:1], off
-; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_and_b32 s0, s2, 15
-; GFX9-NEXT:    s_bfe_u32 s1, s2, 0x40004
-; GFX9-NEXT:    s_and_b32 s5, s4, 15
-; GFX9-NEXT:    s_bfe_u32 s6, s4, 0x40004
-; GFX9-NEXT:    s_pack_ll_b32_b16 s0, s0, s1
-; GFX9-NEXT:    s_pack_ll_b32_b16 s1, s5, s6
-; GFX9-NEXT:    s_bfe_u32 s5, s2, 0x40008
-; GFX9-NEXT:    s_bfe_u32 s6, s2, 0x4000c
-; GFX9-NEXT:    v_pk_lshlrev_b16 v3, 12, s0 op_sel_hi:[0,1]
-; GFX9-NEXT:    s_pack_ll_b32_b16 s0, s5, s6
-; GFX9-NEXT:    s_bfe_u32 s9, s2, 0x40010
-; GFX9-NEXT:    s_bfe_u32 s10, s2, 0x40014
-; GFX9-NEXT:    v_pk_lshlrev_b16 v4, 12, s0 op_sel_hi:[0,1]
-; GFX9-NEXT:    s_bfe_u32 s13, s2, 0x40018
-; GFX9-NEXT:    s_pack_ll_b32_b16 s0, s9, s10
-; GFX9-NEXT:    s_lshr_b32 s2, s2, 28
-; GFX9-NEXT:    v_pk_lshlrev_b16 v5, 12, s0 op_sel_hi:[0,1]
-; GFX9-NEXT:    v_pk_lshlrev_b16 v7, 12, s1 op_sel_hi:[0,1]
-; GFX9-NEXT:    s_pack_ll_b32_b16 s0, s13, s2
-; GFX9-NEXT:    s_bfe_u32 s7, s4, 0x40008
-; GFX9-NEXT:    s_bfe_u32 s8, s4, 0x4000c
-; GFX9-NEXT:    v_pk_lshlrev_b16 v6, 12, s0 op_sel_hi:[0,1]
-; GFX9-NEXT:    s_pack_ll_b32_b16 s0, s7, s8
-; GFX9-NEXT:    v_pk_ashrrev_i16 v3, 12, v3 op_sel_hi:[0,1]
-; GFX9-NEXT:    v_pk_ashrrev_i16 v7, 12, v7 op_sel_hi:[0,1]
-; GFX9-NEXT:    v_pk_lshlrev_b16 v8, 12, s0 op_sel_hi:[0,1]
-; GFX9-NEXT:    v_pk_mul_lo_u16 v3, v3, v7
-; GFX9-NEXT:    s_bfe_u32 s11, s4, 0x40010
-; GFX9-NEXT:    s_bfe_u32 s12, s4, 0x40014
-; GFX9-NEXT:    s_pack_ll_b32_b16 s0, s11, s12
-; GFX9-NEXT:    v_pk_ashrrev_i16 v4, 12, v4 op_sel_hi:[0,1]
-; GFX9-NEXT:    v_pk_ashrrev_i16 v8, 12, v8 op_sel_hi:[0,1]
-; GFX9-NEXT:    v_pk_lshlrev_b16 v9, 12, s0 op_sel_hi:[0,1]
-; GFX9-NEXT:    s_bfe_u32 s14, s4, 0x40018
-; GFX9-NEXT:    s_lshr_b32 s4, s4, 28
-; GFX9-NEXT:    v_pk_mul_lo_u16 v4, v4, v8
-; GFX9-NEXT:    s_pack_ll_b32_b16 s0, s14, s4
-; GFX9-NEXT:    v_pk_ashrrev_i16 v5, 12, v5 op_sel_hi:[0,1]
-; GFX9-NEXT:    v_pk_ashrrev_i16 v9, 12, v9 op_sel_hi:[0,1]
-; GFX9-NEXT:    v_pk_lshlrev_b16 v10, 12, s0 op_sel_hi:[0,1]
-; GFX9-NEXT:    v_pk_mul_lo_u16 v5, v5, v9
-; GFX9-NEXT:    v_pk_ashrrev_i16 v6, 12, v6 op_sel_hi:[0,1]
-; GFX9-NEXT:    v_pk_ashrrev_i16 v10, 12, v10 op_sel_hi:[0,1]
-; GFX9-NEXT:    v_pk_mul_lo_u16 v6, v6, v10
-; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_add_u32_e32 v2, v3, v2
-; GFX9-NEXT:    v_add_u32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT:    v_add_u32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
-; GFX9-NEXT:    v_add_u32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT:    v_add_u32_e32 v2, v2, v5
-; GFX9-NEXT:    v_add_u32_sdwa v2, v2, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT:    v_add_u32_e32 v2, v2, v6
-; GFX9-NEXT:    v_add_u32_sdwa v2, v2, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT:    global_store_short v[0:1], v2, off
-; GFX9-NEXT:    s_endpgm
-;
-; GFX9-DL-LABEL: idot8_acc16_vecMul:
-; GFX9-DL:       ; %bb.0: ; %entry
-; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
-; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
-; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
-; GFX9-DL-NEXT:    s_load_dword s4, s[6:7], 0x0
-; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
-; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
-; GFX9-DL-NEXT:    global_load_ushort v2, v[0:1], off
-; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT:    s_and_b32 s0, s2, 15
-; GFX9-DL-NEXT:    s_bfe_u32 s1, s2, 0x40004
-; GFX9-DL-NEXT:    s_and_b32 s5, s4, 15
-; GFX9-DL-NEXT:    s_bfe_u32 s6, s4, 0x40004
-; GFX9-DL-NEXT:    s_pack_ll_b32_b16 s0, s0, s1
-; GFX9-DL-NEXT:    s_pack_ll_b32_b16 s1, s5, s6
-; GFX9-DL-NEXT:    s_bfe_u32 s5, s2, 0x40008
-; GFX9-DL-NEXT:    s_bfe_u32 s6, s2, 0x4000c
-; GFX9-DL-NEXT:    v_pk_lshlrev_b16 v3, 12, s0 op_sel_hi:[0,1]
-; GFX9-DL-NEXT:    s_pack_ll_b32_b16 s0, s5, s6
-; GFX9-DL-NEXT:    s_bfe_u32 s9, s2, 0x40010
-; GFX9-DL-NEXT:    s_bfe_u32 s10, s2, 0x40014
-; GFX9-DL-NEXT:    v_pk_lshlrev_b16 v4, 12, s0 op_sel_hi:[0,1]
-; GFX9-DL-NEXT:    s_bfe_u32 s13, s2, 0x40018
-; GFX9-DL-NEXT:    s_pack_ll_b32_b16 s0, s9, s10
-; GFX9-DL-NEXT:    s_lshr_b32 s2, s2, 28
-; GFX9-DL-NEXT:    v_pk_lshlrev_b16 v5, 12, s0 op_sel_hi:[0,1]
-; GFX9-DL-NEXT:    v_pk_lshlrev_b16 v7, 12, s1 op_sel_hi:[0,1]
-; GFX9-DL-NEXT:    s_pack_ll_b32_b16 s0, s13, s2
-; GFX9-DL-NEXT:    s_bfe_u32 s7, s4, 0x40008
-; GFX9-DL-NEXT:    s_bfe_u32 s8, s4, 0x4000c
-; GFX9-DL-NEXT:    v_pk_lshlrev_b16 v6, 12, s0 op_sel_hi:[0,1]
-; GFX9-DL-NEXT:    s_pack_ll_b32_b16 s0, s7, s8
-; GFX9-DL-NEXT:    v_pk_ashrrev_i16 v3, 12, v3 op_sel_hi:[0,1]
-; GFX9-DL-NEXT:    v_pk_ashrrev_i16 v7, 12, v7 op_sel_hi:[0,1]
-; GFX9-DL-NEXT:    v_pk_lshlrev_b16 v8, 12, s0 op_sel_hi:[0,1]
-; GFX9-DL-NEXT:    v_pk_mul_lo_u16 v3, v3, v7
-; GFX9-DL-NEXT:    s_bfe_u32 s11, s4, 0x40010
-; GFX9-DL-NEXT:    s_bfe_u32 s12, s4, 0x40014
-; GFX9-DL-NEXT:    s_pack_ll_b32_b16 s0, s11, s12
-; GFX9-DL-NEXT:    v_pk_ashrrev_i16 v4, 12, v4 op_sel_hi:[0,1]
-; GFX9-DL-NEXT:    v_pk_ashrrev_i16 v8, 12, v8 op_sel_hi:[0,1]
-; GFX9-DL-NEXT:    v_pk_lshlrev_b16 v9, 12, s0 op_sel_hi:[0,1]
-; GFX9-DL-NEXT:    s_bfe_u32 s14, s4, 0x40018
-; GFX9-DL-NEXT:    s_lshr_b32 s4, s4, 28
-; GFX9-DL-NEXT:    v_pk_mul_lo_u16 v4, v4, v8
-; GFX9-DL-NEXT:    s_pack_ll_b32_b16 s0, s14, s4
-; GFX9-DL-NEXT:    v_pk_ashrrev_i16 v5, 12, v5 op_sel_hi:[0,1]
-; GFX9-DL-NEXT:    v_pk_ashrrev_i16 v9, 12, v9 op_sel_hi:[0,1]
-; GFX9-DL-NEXT:    v_pk_lshlrev_b16 v10, 12, s0 op_sel_hi:[0,1]
-; GFX9-DL-NEXT:    v_pk_mul_lo_u16 v5, v5, v9
-; GFX9-DL-NEXT:    v_pk_ashrrev_i16 v6, 12, v6 op_sel_hi:[0,1]
-; GFX9-DL-NEXT:    v_pk_ashrrev_i16 v10, 12, v10 op_sel_hi:[0,1]
-; GFX9-DL-NEXT:    v_pk_mul_lo_u16 v6, v6, v10
-; GFX9-DL-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-DL-NEXT:    v_add_u32_e32 v2, v3, v2
-; GFX9-DL-NEXT:    v_add_u32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-DL-NEXT:    v_add_u32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0
-; GFX9-DL-NEXT:    v_add_u32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-DL-NEXT:    v_add_u32_e32 v2, v2, v5
-; GFX9-DL-NEXT:    v_add_u32_sdwa v2, v2, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-DL-NEXT:    v_add_u32_e32 v2, v2, v6
-; GFX9-DL-NEXT:    v_add_u32_sdwa v2, v2, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-DL-NEXT:    global_store_short v[0:1], v2, off
-; GFX9-DL-NEXT:    s_endpgm
-                                              <8 x i4> addrspace(1)* %src2,
-                                              i16 addrspace(1)* nocapture %dst) {
-entry:
-  %vec1 = load <8 x i4>, <8 x i4> addrspace(1)* %src1
-  %vec2 = load <8 x i4>, <8 x i4> addrspace(1)* %src2
-
-  %cvec1 = sext <8 x i4> %vec1 to <8 x i16>
-  %cvec2 = sext <8 x i4> %vec2 to <8 x i16>
-
-  %mul = mul <8 x i16> %cvec1, %cvec2
-  %mul0 = extractelement <8 x i16> %mul, i64 0
-  %mul1 = extractelement <8 x i16> %mul, i64 1
-  %mul2 = extractelement <8 x i16> %mul, i64 2
-  %mul3 = extractelement <8 x i16> %mul, i64 3
-  %mul4 = extractelement <8 x i16> %mul, i64 4
-  %mul5 = extractelement <8 x i16> %mul, i64 5
-  %mul6 = extractelement <8 x i16> %mul, i64 6
-  %mul7 = extractelement <8 x i16> %mul, i64 7
-
-  %acc = load i16, i16 addrspace(1)* %dst, align 4
-  %add1 = add i16 %mul0, %acc
-  %add2 = add i16 %add1, %mul1
-  %add3 = add i16 %add2, %mul2
-  %add4 = add i16 %add3, %mul3
-  %add5 = add i16 %add4, %mul4
-  %add6 = add i16 %add5, %mul5
-  %add7 = add i16 %add6, %mul6
-  %add8 = add i16 %add7, %mul7
-
-  store i16 %add8, i16 addrspace(1)* %dst, align 4
-  ret void
-}
-
-; TODO: Support this pattern.
-define amdgpu_kernel void @idot8_acc8_vecMul(<8 x i4> addrspace(1)* %src1,
-; GFX7-LABEL: idot8_acc8_vecMul:
-; GFX7:       ; %bb.0: ; %entry
-; GFX7-NEXT:    s_load_dwordx4 s[8:11], s[0:1], 0x9
-; GFX7-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0xd
-; GFX7-NEXT:    s_mov_b32 s7, 0xf000
-; GFX7-NEXT:    s_mov_b32 s6, -1
-; GFX7-NEXT:    s_movk_i32 s0, 0xff
-; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    s_load_dword s2, s[8:9], 0x0
-; GFX7-NEXT:    buffer_load_ubyte v0, off, s[4:7], 0
-; GFX7-NEXT:    s_load_dword s8, s[10:11], 0x0
-; GFX7-NEXT:    s_mov_b32 s1, 0xffff
-; GFX7-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-NEXT:    s_bfe_i32 s9, s2, 0x40000
-; GFX7-NEXT:    s_bfe_i32 s10, s2, 0x40004
-; GFX7-NEXT:    s_bfe_i32 s16, s8, 0x40000
-; GFX7-NEXT:    s_bfe_i32 s17, s8, 0x40004
-; GFX7-NEXT:    s_bfe_i32 s18, s8, 0x40008
-; GFX7-NEXT:    s_bfe_i32 s19, s8, 0x4000c
-; GFX7-NEXT:    s_bfe_i32 s20, s8, 0x40010
-; GFX7-NEXT:    s_bfe_i32 s21, s8, 0x40014
-; GFX7-NEXT:    s_bfe_i32 s22, s8, 0x40018
-; GFX7-NEXT:    s_ashr_i32 s8, s8, 28
-; GFX7-NEXT:    v_mov_b32_e32 v7, s17
-; GFX7-NEXT:    v_mov_b32_e32 v8, s16
-; GFX7-NEXT:    s_bfe_i32 s11, s2, 0x40008
-; GFX7-NEXT:    v_mov_b32_e32 v6, s18
-; GFX7-NEXT:    s_bfe_i32 s12, s2, 0x4000c
-; GFX7-NEXT:    v_mov_b32_e32 v5, s19
-; GFX7-NEXT:    s_bfe_i32 s13, s2, 0x40010
-; GFX7-NEXT:    v_mov_b32_e32 v4, s20
-; GFX7-NEXT:    s_bfe_i32 s14, s2, 0x40014
-; GFX7-NEXT:    v_mov_b32_e32 v3, s21
-; GFX7-NEXT:    s_bfe_i32 s15, s2, 0x40018
-; GFX7-NEXT:    v_mov_b32_e32 v2, s22
-; GFX7-NEXT:    s_ashr_i32 s2, s2, 28
-; GFX7-NEXT:    v_mov_b32_e32 v1, s8
-; GFX7-NEXT:    v_mul_i32_i24_e32 v1, s2, v1
-; GFX7-NEXT:    v_mul_i32_i24_e32 v2, s15, v2
-; GFX7-NEXT:    v_mul_i32_i24_e32 v3, s14, v3
-; GFX7-NEXT:    v_mul_i32_i24_e32 v4, s13, v4
-; GFX7-NEXT:    v_mul_i32_i24_e32 v5, s12, v5
-; GFX7-NEXT:    v_mul_i32_i24_e32 v6, s11, v6
-; GFX7-NEXT:    v_mul_i32_i24_e32 v7, s10, v7
-; GFX7-NEXT:    v_mul_i32_i24_e32 v8, s9, v8
-; GFX7-NEXT:    v_lshlrev_b32_e32 v1, 8, v1
-; GFX7-NEXT:    v_and_b32_e32 v2, s0, v2
-; GFX7-NEXT:    v_lshlrev_b32_e32 v3, 8, v3
-; GFX7-NEXT:    v_and_b32_e32 v4, s0, v4
-; GFX7-NEXT:    v_lshlrev_b32_e32 v5, 8, v5
-; GFX7-NEXT:    v_and_b32_e32 v6, s0, v6
-; GFX7-NEXT:    v_lshlrev_b32_e32 v7, 8, v7
-; GFX7-NEXT:    v_and_b32_e32 v8, s0, v8
-; GFX7-NEXT:    v_or_b32_e32 v1, v2, v1
-; GFX7-NEXT:    v_or_b32_e32 v2, v4, v3
-; GFX7-NEXT:    v_or_b32_e32 v3, v6, v5
-; GFX7-NEXT:    v_or_b32_e32 v4, v8, v7
-; GFX7-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; GFX7-NEXT:    v_and_b32_e32 v2, s1, v2
-; GFX7-NEXT:    v_lshlrev_b32_e32 v3, 16, v3
-; GFX7-NEXT:    v_and_b32_e32 v4, s1, v4
-; GFX7-NEXT:    v_or_b32_e32 v1, v2, v1
-; GFX7-NEXT:    v_or_b32_e32 v2, v4, v3
-; GFX7-NEXT:    v_alignbit_b32 v3, v1, v2, 8
-; GFX7-NEXT:    v_alignbit_b32 v4, v1, v2, 16
-; GFX7-NEXT:    v_lshrrev_b32_e32 v5, 24, v2
-; GFX7-NEXT:    v_lshrrev_b32_e32 v6, 8, v1
-; GFX7-NEXT:    v_lshrrev_b32_e32 v7, 16, v1
-; GFX7-NEXT:    v_lshrrev_b32_e32 v8, 24, v1
-; GFX7-NEXT:    s_waitcnt vmcnt(0)
-; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
-; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v3, v0
-; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v4, v0
-; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v5, v0
-; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v0, v1
-; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v6, v0
-; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v7, v0
-; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v8, v0
-; GFX7-NEXT:    buffer_store_byte v0, off, s[4:7], 0
-; GFX7-NEXT:    s_endpgm
-;
-; GFX8-LABEL: idot8_acc8_vecMul:
-; GFX8:       ; %bb.0: ; %entry
-; GFX8-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
-; GFX8-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
-; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    s_load_dword s2, s[4:5], 0x0
-; GFX8-NEXT:    s_load_dword s4, s[6:7], 0x0
-; GFX8-NEXT:    v_mov_b32_e32 v0, s0
-; GFX8-NEXT:    v_mov_b32_e32 v1, s1
-; GFX8-NEXT:    flat_load_ubyte v2, v[0:1]
-; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    s_lshr_b32 s0, s2, 4
-; GFX8-NEXT:    s_lshr_b32 s1, s2, 12
-; GFX8-NEXT:    s_lshr_b32 s5, s2, 8
-; GFX8-NEXT:    s_lshr_b32 s6, s4, 4
-; GFX8-NEXT:    s_lshr_b32 s7, s4, 12
-; GFX8-NEXT:    s_lshr_b32 s8, s4, 8
-; GFX8-NEXT:    v_lshlrev_b16_e64 v3, 12, s5
-; GFX8-NEXT:    v_lshlrev_b16_e64 v4, 12, s1
-; GFX8-NEXT:    v_lshlrev_b16_e64 v5, 12, s0
-; GFX8-NEXT:    v_lshlrev_b16_e64 v7, 12, s8
-; GFX8-NEXT:    v_lshlrev_b16_e64 v8, 12, s7
-; GFX8-NEXT:    v_lshlrev_b16_e64 v9, 12, s6
-; GFX8-NEXT:    v_lshlrev_b16_e64 v6, 12, s2
-; GFX8-NEXT:    v_lshlrev_b16_e64 v10, 12, s4
-; GFX8-NEXT:    v_ashrrev_i16_e32 v3, 12, v3
-; GFX8-NEXT:    v_ashrrev_i16_e32 v7, 12, v7
-; GFX8-NEXT:    v_ashrrev_i16_e32 v4, 12, v4
-; GFX8-NEXT:    v_ashrrev_i16_e32 v8, 12, v8
-; GFX8-NEXT:    v_ashrrev_i16_e32 v5, 12, v5
-; GFX8-NEXT:    v_ashrrev_i16_e32 v9, 12, v9
-; GFX8-NEXT:    v_ashrrev_i16_e32 v6, 12, v6
-; GFX8-NEXT:    v_ashrrev_i16_e32 v10, 12, v10
-; GFX8-NEXT:    v_mul_u32_u24_sdwa v3, v3, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX8-NEXT:    v_mul_u32_u24_sdwa v4, v4, v8 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX8-NEXT:    v_mul_u32_u24_sdwa v5, v5, v9 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX8-NEXT:    v_mul_u32_u24_sdwa v6, v6, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX8-NEXT:    s_lshr_b32 s0, s2, 20
-; GFX8-NEXT:    s_lshr_b32 s1, s2, 16
-; GFX8-NEXT:    s_lshr_b32 s5, s2, 28
-; GFX8-NEXT:    s_lshr_b32 s2, s2, 24
-; GFX8-NEXT:    s_lshr_b32 s6, s4, 20
-; GFX8-NEXT:    s_lshr_b32 s7, s4, 16
-; GFX8-NEXT:    s_lshr_b32 s8, s4, 28
-; GFX8-NEXT:    s_lshr_b32 s4, s4, 24
-; GFX8-NEXT:    v_or_b32_sdwa v5, v6, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX8-NEXT:    v_or_b32_sdwa v3, v3, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX8-NEXT:    v_or_b32_sdwa v3, v5, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT:    v_lshlrev_b16_e64 v7, 12, s2
-; GFX8-NEXT:    v_lshlrev_b16_e64 v8, 12, s5
-; GFX8-NEXT:    v_lshlrev_b16_e64 v9, 12, s1
-; GFX8-NEXT:    v_lshlrev_b16_e64 v10, 12, s0
-; GFX8-NEXT:    v_lshlrev_b16_e64 v11, 12, s4
-; GFX8-NEXT:    v_lshlrev_b16_e64 v12, 12, s8
-; GFX8-NEXT:    v_lshlrev_b16_e64 v13, 12, s7
-; GFX8-NEXT:    v_lshlrev_b16_e64 v14, 12, s6
-; GFX8-NEXT:    v_ashrrev_i16_e32 v7, 12, v7
-; GFX8-NEXT:    v_ashrrev_i16_e32 v11, 12, v11
-; GFX8-NEXT:    v_ashrrev_i16_e32 v8, 12, v8
-; GFX8-NEXT:    v_ashrrev_i16_e32 v12, 12, v12
-; GFX8-NEXT:    v_ashrrev_i16_e32 v9, 12, v9
-; GFX8-NEXT:    v_ashrrev_i16_e32 v13, 12, v13
-; GFX8-NEXT:    v_ashrrev_i16_e32 v10, 12, v10
-; GFX8-NEXT:    v_ashrrev_i16_e32 v14, 12, v14
-; GFX8-NEXT:    v_lshrrev_b32_e32 v5, 8, v3
-; GFX8-NEXT:    v_mul_u32_u24_sdwa v7, v7, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX8-NEXT:    v_mul_u32_u24_sdwa v8, v8, v12 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX8-NEXT:    v_mul_u32_u24_sdwa v9, v9, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX8-NEXT:    v_mul_u32_u24_sdwa v10, v10, v14 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX8-NEXT:    v_or_b32_sdwa v9, v9, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX8-NEXT:    v_or_b32_sdwa v7, v7, v8 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX8-NEXT:    v_or_b32_sdwa v4, v9, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT:    v_lshrrev_b32_e32 v6, 8, v4
-; GFX8-NEXT:    s_waitcnt vmcnt(0)
-; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v2, v3
-; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v5, v2
-; GFX8-NEXT:    v_add_u32_sdwa v2, vcc, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_2
-; GFX8-NEXT:    v_add_u32_sdwa v2, vcc, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
-; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v2, v4
-; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v2, v6
-; GFX8-NEXT:    v_add_u32_sdwa v2, vcc, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX8-NEXT:    v_add_u32_sdwa v2, vcc, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
-; GFX8-NEXT:    flat_store_byte v[0:1], v2
-; GFX8-NEXT:    s_endpgm
-;
-; GFX9-LABEL: idot8_acc8_vecMul:
-; GFX9:       ; %bb.0: ; %entry
-; GFX9-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
-; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
-; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_load_dword s2, s[4:5], 0x0
-; GFX9-NEXT:    s_load_dword s4, s[6:7], 0x0
-; GFX9-NEXT:    v_mov_b32_e32 v0, s0
-; GFX9-NEXT:    v_mov_b32_e32 v1, s1
-; GFX9-NEXT:    global_load_ubyte v2, v[0:1], off
-; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_lshr_b32 s0, s2, 4
-; GFX9-NEXT:    s_lshr_b32 s1, s2, 12
-; GFX9-NEXT:    s_lshr_b32 s5, s2, 8
-; GFX9-NEXT:    s_lshr_b32 s6, s4, 4
-; GFX9-NEXT:    s_lshr_b32 s7, s4, 12
-; GFX9-NEXT:    s_lshr_b32 s8, s4, 8
-; GFX9-NEXT:    v_lshlrev_b16_e64 v3, 12, s5
-; GFX9-NEXT:    v_lshlrev_b16_e64 v4, 12, s1
-; GFX9-NEXT:    v_lshlrev_b16_e64 v5, 12, s0
-; GFX9-NEXT:    v_lshlrev_b16_e64 v7, 12, s8
-; GFX9-NEXT:    v_lshlrev_b16_e64 v8, 12, s7
-; GFX9-NEXT:    v_lshlrev_b16_e64 v9, 12, s6
-; GFX9-NEXT:    v_lshlrev_b16_e64 v6, 12, s2
-; GFX9-NEXT:    v_lshlrev_b16_e64 v10, 12, s4
-; GFX9-NEXT:    v_ashrrev_i16_e32 v3, 12, v3
-; GFX9-NEXT:    v_ashrrev_i16_e32 v7, 12, v7
-; GFX9-NEXT:    v_ashrrev_i16_e32 v4, 12, v4
-; GFX9-NEXT:    v_ashrrev_i16_e32 v8, 12, v8
-; GFX9-NEXT:    v_ashrrev_i16_e32 v5, 12, v5
-; GFX9-NEXT:    v_ashrrev_i16_e32 v9, 12, v9
-; GFX9-NEXT:    v_ashrrev_i16_e32 v6, 12, v6
-; GFX9-NEXT:    v_ashrrev_i16_e32 v10, 12, v10
-; GFX9-NEXT:    v_mul_lo_u16_e32 v6, v6, v10
-; GFX9-NEXT:    v_mul_lo_u16_sdwa v5, v5, v9 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT:    v_mul_lo_u16_sdwa v4, v4, v8 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT:    v_mul_lo_u16_e32 v3, v3, v7
-; GFX9-NEXT:    s_lshr_b32 s0, s2, 20
-; GFX9-NEXT:    s_lshr_b32 s1, s2, 16
-; GFX9-NEXT:    s_lshr_b32 s5, s2, 28
-; GFX9-NEXT:    s_lshr_b32 s2, s2, 24
-; GFX9-NEXT:    s_lshr_b32 s6, s4, 20
-; GFX9-NEXT:    s_lshr_b32 s7, s4, 16
-; GFX9-NEXT:    s_lshr_b32 s8, s4, 28
-; GFX9-NEXT:    s_lshr_b32 s4, s4, 24
-; GFX9-NEXT:    v_or_b32_sdwa v5, v6, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v3, v3, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v3, v5, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT:    v_lshlrev_b16_e64 v9, 12, s2
-; GFX9-NEXT:    v_lshlrev_b16_e64 v10, 12, s5
-; GFX9-NEXT:    v_lshlrev_b16_e64 v11, 12, s1
-; GFX9-NEXT:    v_lshlrev_b16_e64 v12, 12, s0
-; GFX9-NEXT:    v_lshlrev_b16_e64 v13, 12, s4
-; GFX9-NEXT:    v_lshlrev_b16_e64 v14, 12, s8
-; GFX9-NEXT:    v_lshlrev_b16_e64 v15, 12, s7
-; GFX9-NEXT:    v_lshlrev_b16_e64 v16, 12, s6
-; GFX9-NEXT:    v_ashrrev_i16_e32 v9, 12, v9
-; GFX9-NEXT:    v_ashrrev_i16_e32 v13, 12, v13
-; GFX9-NEXT:    v_ashrrev_i16_e32 v10, 12, v10
-; GFX9-NEXT:    v_ashrrev_i16_e32 v14, 12, v14
-; GFX9-NEXT:    v_ashrrev_i16_e32 v11, 12, v11
-; GFX9-NEXT:    v_ashrrev_i16_e32 v15, 12, v15
-; GFX9-NEXT:    v_ashrrev_i16_e32 v12, 12, v12
-; GFX9-NEXT:    v_ashrrev_i16_e32 v16, 12, v16
-; GFX9-NEXT:    v_lshrrev_b32_e32 v5, 8, v3
-; GFX9-NEXT:    v_mul_lo_u16_sdwa v12, v12, v16 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT:    v_mul_lo_u16_e32 v11, v11, v15
-; GFX9-NEXT:    v_mul_lo_u16_sdwa v10, v10, v14 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT:    v_mul_lo_u16_e32 v9, v9, v13
-; GFX9-NEXT:    v_or_b32_sdwa v7, v11, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v8, v9, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT:    v_or_b32_sdwa v4, v7, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_add_u32_e32 v2, v3, v2
-; GFX9-NEXT:    v_add_u32_e32 v2, v2, v5
-; GFX9-NEXT:    v_add_u32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_2
-; GFX9-NEXT:    v_add_u32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
-; GFX9-NEXT:    v_add_u32_e32 v2, v2, v4
-; GFX9-NEXT:    v_lshrrev_b32_e32 v3, 8, v4
-; GFX9-NEXT:    v_add_u32_e32 v2, v2, v3
-; GFX9-NEXT:    v_add_u32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT:    v_add_u32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
-; GFX9-NEXT:    global_store_byte v[0:1], v2, off
-; GFX9-NEXT:    s_endpgm
-;
-; GFX9-DL-LABEL: idot8_acc8_vecMul:
-; GFX9-DL:       ; %bb.0: ; %entry
-; GFX9-DL-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
-; GFX9-DL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
-; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT:    s_load_dword s2, s[4:5], 0x0
-; GFX9-DL-NEXT:    s_load_dword s4, s[6:7], 0x0
-; GFX9-DL-NEXT:    v_mov_b32_e32 v0, s0
-; GFX9-DL-NEXT:    v_mov_b32_e32 v1, s1
-; GFX9-DL-NEXT:    global_load_ubyte v2, v[0:1], off
-; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-DL-NEXT:    s_lshr_b32 s0, s2, 4
-; GFX9-DL-NEXT:    s_lshr_b32 s1, s2, 12
-; GFX9-DL-NEXT:    s_lshr_b32 s5, s2, 8
-; GFX9-DL-NEXT:    s_lshr_b32 s6, s4, 4
-; GFX9-DL-NEXT:    s_lshr_b32 s7, s4, 12
-; GFX9-DL-NEXT:    s_lshr_b32 s8, s4, 8
-; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v3, 12, s5
-; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v4, 12, s1
-; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v5, 12, s0
-; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v7, 12, s8
-; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v8, 12, s7
-; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v9, 12, s6
-; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v6, 12, s2
-; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v10, 12, s4
-; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v3, 12, v3
-; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v7, 12, v7
-; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v4, 12, v4
-; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v8, 12, v8
-; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v5, 12, v5
-; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v9, 12, v9
-; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v6, 12, v6
-; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v10, 12, v10
-; GFX9-DL-NEXT:    v_mul_lo_u16_e32 v6, v6, v10
-; GFX9-DL-NEXT:    v_mul_lo_u16_sdwa v5, v5, v9 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-DL-NEXT:    v_mul_lo_u16_sdwa v4, v4, v8 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-DL-NEXT:    v_mul_lo_u16_e32 v3, v3, v7
-; GFX9-DL-NEXT:    s_lshr_b32 s0, s2, 20
-; GFX9-DL-NEXT:    s_lshr_b32 s1, s2, 16
-; GFX9-DL-NEXT:    s_lshr_b32 s5, s2, 28
-; GFX9-DL-NEXT:    s_lshr_b32 s2, s2, 24
-; GFX9-DL-NEXT:    s_lshr_b32 s6, s4, 20
-; GFX9-DL-NEXT:    s_lshr_b32 s7, s4, 16
-; GFX9-DL-NEXT:    s_lshr_b32 s8, s4, 28
-; GFX9-DL-NEXT:    s_lshr_b32 s4, s4, 24
-; GFX9-DL-NEXT:    v_or_b32_sdwa v5, v6, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-DL-NEXT:    v_or_b32_sdwa v3, v3, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-DL-NEXT:    v_or_b32_sdwa v3, v5, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v9, 12, s2
-; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v10, 12, s5
-; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v11, 12, s1
-; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v12, 12, s0
-; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v13, 12, s4
-; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v14, 12, s8
-; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v15, 12, s7
-; GFX9-DL-NEXT:    v_lshlrev_b16_e64 v16, 12, s6
-; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v9, 12, v9
-; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v13, 12, v13
-; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v10, 12, v10
-; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v14, 12, v14
-; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v11, 12, v11
-; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v15, 12, v15
-; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v12, 12, v12
-; GFX9-DL-NEXT:    v_ashrrev_i16_e32 v16, 12, v16
-; GFX9-DL-NEXT:    v_lshrrev_b32_e32 v5, 8, v3
-; GFX9-DL-NEXT:    v_mul_lo_u16_sdwa v12, v12, v16 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-DL-NEXT:    v_mul_lo_u16_e32 v11, v11, v15
-; GFX9-DL-NEXT:    v_mul_lo_u16_sdwa v10, v10, v14 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-DL-NEXT:    v_mul_lo_u16_e32 v9, v9, v13
-; GFX9-DL-NEXT:    v_or_b32_sdwa v7, v11, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-DL-NEXT:    v_or_b32_sdwa v8, v9, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-DL-NEXT:    v_or_b32_sdwa v4, v7, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX9-DL-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-DL-NEXT:    v_add_u32_e32 v2, v3, v2
-; GFX9-DL-NEXT:    v_add_u32_e32 v2, v2, v5
-; GFX9-DL-NEXT:    v_add_u32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_2
-; GFX9-DL-NEXT:    v_add_u32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
-; GFX9-DL-NEXT:    v_add_u32_e32 v2, v2, v4
-; GFX9-DL-NEXT:    v_lshrrev_b32_e32 v3, 8, v4
-; GFX9-DL-NEXT:    v_add_u32_e32 v2, v2, v3
-; GFX9-DL-NEXT:    v_add_u32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-DL-NEXT:    v_add_u32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
-; GFX9-DL-NEXT:    global_store_byte v[0:1], v2, off
-; GFX9-DL-NEXT:    s_endpgm
-                                             <8 x i4> addrspace(1)* %src2,
-                                             i8 addrspace(1)* nocapture %dst) {
-entry:
-  %vec1 = load <8 x i4>, <8 x i4> addrspace(1)* %src1
-  %vec2 = load <8 x i4>, <8 x i4> addrspace(1)* %src2
-
-  %cvec1 = sext <8 x i4> %vec1 to <8 x i8>
-  %cvec2 = sext <8 x i4> %vec2 to <8 x i8>
-
-  %mul = mul <8 x i8> %cvec1, %cvec2
-  %mul0 = extractelement <8 x i8> %mul, i64 0
-  %mul1 = extractelement <8 x i8> %mul, i64 1
-  %mul2 = extractelement <8 x i8> %mul, i64 2
-  %mul3 = extractelement <8 x i8> %mul, i64 3
-  %mul4 = extractelement <8 x i8> %mul, i64 4
-  %mul5 = extractelement <8 x i8> %mul, i64 5
-  %mul6 = extractelement <8 x i8> %mul, i64 6
-  %mul7 = extractelement <8 x i8> %mul, i64 7
-
-  %acc = load i8, i8 addrspace(1)* %dst, align 4
-  %add1 = add i8 %mul0, %acc
-  %add2 = add i8 %add1, %mul1
-  %add3 = add i8 %add2, %mul2
-  %add4 = add i8 %add3, %mul3
-  %add5 = add i8 %add4, %mul4
-  %add6 = add i8 %add5, %mul5
-  %add7 = add i8 %add6, %mul6
-  %add8 = add i8 %add7, %mul7
-
-  store i8 %add8, i8 addrspace(1)* %dst, align 4
-  ret void
-}
-
 define amdgpu_kernel void @udot8_variant1(i32 addrspace(1)* %v1addr,
 ; GFX7-LABEL: udot8_variant1:
 ; GFX7:       ; %bb.0: ; %entry
diff --git a/test/CodeGen/AMDGPU/imm.ll b/test/CodeGen/AMDGPU/imm.ll
index bb9da71d1a9f8759a5715e2ce124f614758cc1c5..2ef4c744841633cc3856d3cebe2fdb28a5038eed 100644
--- a/test/CodeGen/AMDGPU/imm.ll
+++ b/test/CodeGen/AMDGPU/imm.ll
@@ -449,7 +449,7 @@ define amdgpu_kernel void @add_inline_imm_neg_4.0_f64(double addrspace(1)* %out,
 ; SI: v_add_f64 v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, v{{\[}}[[LO_VREG]]:[[HI_VREG]]{{\]}}
 
 ; VI: s_load_dwordx2 [[VAL:s\[[0-9]+:[0-9]+\]]], {{s\[[0-9]+:[0-9]+\]}}, 0x4c
-; VI: v_add_f64 [[REG:v\[[0-9]+:[0-9]+\]]], [[VAL]], 0.15915494{{$}}
+; VI: v_add_f64 [[REG:v\[[0-9]+:[0-9]+\]]], [[VAL]], 0.15915494309189532{{$}}
 ; VI: buffer_store_dwordx2 [[REG]]
 define amdgpu_kernel void @add_inline_imm_inv_2pi_f64(double addrspace(1)* %out, [8 x i32], double %x) {
   %y = fadd double %x, 0x3fc45f306dc9c882
diff --git a/test/CodeGen/AMDGPU/imm16.ll b/test/CodeGen/AMDGPU/imm16.ll
index dcf3b36be2bc4652c3f9a1aaed30c5018774c11c..29b80da593535c4723139f072e9a9d9982583477 100644
--- a/test/CodeGen/AMDGPU/imm16.ll
+++ b/test/CodeGen/AMDGPU/imm16.ll
@@ -266,7 +266,7 @@ define amdgpu_kernel void @add_inline_imm_16_f16(half addrspace(1)* %out, half %
 }
 
 ; GCN-LABEL: {{^}}add_inline_imm_neg_1_f16:
-; VI: v_add_u32_e32 [[REG:v[0-9]+]], vcc, -1
+; VI: v_add_u16_e32 [[REG:v[0-9]+]], -1, [[REG:v[0-9]+]]
 ; VI: buffer_store_short [[REG]]
 define amdgpu_kernel void @add_inline_imm_neg_1_f16(half addrspace(1)* %out, i16 addrspace(1)* %in) {
   %x = load i16, i16 addrspace(1)* %in
@@ -277,7 +277,7 @@ define amdgpu_kernel void @add_inline_imm_neg_1_f16(half addrspace(1)* %out, i16
 }
 
 ; GCN-LABEL: {{^}}add_inline_imm_neg_2_f16:
-; VI: v_add_u32_e32 [[REG:v[0-9]+]], vcc, 0xfffe
+; VI: v_add_u16_e32 [[REG:v[0-9]+]], -2, [[REG:v[0-9]+]]
 ; VI: buffer_store_short [[REG]]
 define amdgpu_kernel void @add_inline_imm_neg_2_f16(half addrspace(1)* %out, i16 addrspace(1)* %in) {
   %x = load i16, i16 addrspace(1)* %in
@@ -288,7 +288,7 @@ define amdgpu_kernel void @add_inline_imm_neg_2_f16(half addrspace(1)* %out, i16
 }
 
 ; GCN-LABEL: {{^}}add_inline_imm_neg_16_f16:
-; VI: v_add_u32_e32 [[REG:v[0-9]+]], vcc, 0xfff0
+; VI: v_add_u16_e32 [[REG:v[0-9]+]], -16, [[REG:v[0-9]+]]
 ; VI: buffer_store_short [[REG]]
 define amdgpu_kernel void @add_inline_imm_neg_16_f16(half addrspace(1)* %out, i16 addrspace(1)* %in) {
   %x = load i16, i16 addrspace(1)* %in
diff --git a/test/CodeGen/AMDGPU/indirect-addressing-si-gfx9.ll b/test/CodeGen/AMDGPU/indirect-addressing-si-gfx9.ll
index c8f75518518be3124bc27e67875a5dc1a7110e98..b266dd1f8c1fa355ae6d8c38ad2c24a43d5fc115 100644
--- a/test/CodeGen/AMDGPU/indirect-addressing-si-gfx9.ll
+++ b/test/CodeGen/AMDGPU/indirect-addressing-si-gfx9.ll
@@ -14,8 +14,9 @@
 ; GCN-DAG: v_mov_b32_e32 v[[VEC_ELT15:[0-9]+]], s[[S_ELT15]]
 ; GCN-DAG: v_mov_b32_e32 v[[VEC_ELT0:[0-9]+]], s[[S_ELT0]]
 
+; GCN-DAG: v_add_u32_e32 [[IDX1:v[0-9]+]], 1, [[IDX0]]
+
 ; GCN: [[LOOP0:BB[0-9]+_[0-9]+]]:
-; GCN-NEXT: s_waitcnt vmcnt(0)
 ; GCN-NEXT: v_readfirstlane_b32 [[READLANE:s[0-9]+]], [[IDX0]]
 ; GCN: v_cmp_eq_u32_e32 vcc, [[READLANE]], [[IDX0]]
 ; GCN: s_and_saveexec_b64 vcc, vcc
@@ -36,8 +37,8 @@
 ; GCN: s_mov_b64 [[MASK]], exec
 
 ; GCN: [[LOOP1:BB[0-9]+_[0-9]+]]:
-; GCN-NEXT: v_readfirstlane_b32 [[READLANE:s[0-9]+]], [[IDX0]]
-; GCN: v_cmp_eq_u32_e32 vcc, [[READLANE]], [[IDX0]]
+; GCN-NEXT: v_readfirstlane_b32 [[READLANE:s[0-9]+]], [[IDX1]]
+; GCN: v_cmp_eq_u32_e32 vcc, [[READLANE]], [[IDX1]]
 ; GCN: s_and_saveexec_b64 vcc, vcc
 
 ; MOVREL: s_mov_b32 m0, [[READLANE]]
diff --git a/test/CodeGen/AMDGPU/indirect-addressing-si-pregfx9.ll b/test/CodeGen/AMDGPU/indirect-addressing-si-pregfx9.ll
index 544ab990fee8457de8356d44d8a8f42434eee699..e9b640c25917b5649d980af8f3cc070b8575fa04 100644
--- a/test/CodeGen/AMDGPU/indirect-addressing-si-pregfx9.ll
+++ b/test/CodeGen/AMDGPU/indirect-addressing-si-pregfx9.ll
@@ -17,8 +17,9 @@
 ; GCN-DAG: v_mov_b32_e32 v[[VEC_ELT15:[0-9]+]], s[[S_ELT15]]
 ; GCN-DAG: v_mov_b32_e32 v[[VEC_ELT0:[0-9]+]], s[[S_ELT0]]
 
+; GCN-DAG: v_add_{{i32|u32}}_e32 [[IDX1:v[0-9]+]], vcc, 1, [[IDX0]]
+
 ; GCN: [[LOOP0:BB[0-9]+_[0-9]+]]:
-; GCN-NEXT: s_waitcnt vmcnt(0)
 ; GCN-NEXT: v_readfirstlane_b32 [[READLANE:s[0-9]+]], [[IDX0]]
 ; GCN: v_cmp_eq_u32_e32 vcc, [[READLANE]], [[IDX0]]
 ; GCN: s_and_saveexec_b64 vcc, vcc
@@ -39,8 +40,8 @@
 ; GCN: s_mov_b64 [[MASK]], exec
 
 ; GCN: [[LOOP1:BB[0-9]+_[0-9]+]]:
-; GCN-NEXT: v_readfirstlane_b32 [[READLANE:s[0-9]+]], [[IDX0]]
-; GCN: v_cmp_eq_u32_e32 vcc, [[READLANE]], [[IDX0]]
+; GCN-NEXT: v_readfirstlane_b32 [[READLANE:s[0-9]+]], [[IDX1]]
+; GCN: v_cmp_eq_u32_e32 vcc, [[READLANE]], [[IDX1]]
 ; GCN: s_and_saveexec_b64 vcc, vcc
 
 ; MOVREL: s_mov_b32 m0, [[READLANE]]
diff --git a/test/CodeGen/AMDGPU/indirect-addressing-si.ll b/test/CodeGen/AMDGPU/indirect-addressing-si.ll
index 081f623ac13d5f43bb79297ddedc80fb0772a9a0..3412ef50d164aea8ff6f3a7b279b67fabd9cec0b 100644
--- a/test/CodeGen/AMDGPU/indirect-addressing-si.ll
+++ b/test/CodeGen/AMDGPU/indirect-addressing-si.ll
@@ -7,11 +7,12 @@
 ; indexing of vectors.
 
 ; GCN-LABEL: {{^}}extract_w_offset:
-; GCN-DAG: s_load_dword [[IN:s[0-9]+]]
+; GCN-DAG: s_load_dword [[IN0:s[0-9]+]]
 ; GCN-DAG: v_mov_b32_e32 v{{[0-9]+}}, 4.0
 ; GCN-DAG: v_mov_b32_e32 v{{[0-9]+}}, 0x40400000
-; GCN-DAG: v_mov_b32_e32 [[BASEREG:v[0-9]+]], 2.0
-; GCN-DAG: v_mov_b32_e32 v{{[0-9]+}}, 1.0
+; GCN-DAG: v_mov_b32_e32 v{{[0-9]+}}, 2.0
+; GCN-DAG: v_mov_b32_e32 [[BASEREG:v[0-9]+]], 1.0
+; GCN-DAG: s_add_i32 [[IN:s[0-9]+]], [[IN0]], 1
 
 ; MOVREL-DAG: s_mov_b32 m0, [[IN]]
 ; MOVREL: v_movrels_b32_e32 v{{[0-9]+}}, [[BASEREG]]
@@ -29,16 +30,17 @@ entry:
 
 ; XXX: Could do v_or_b32 directly
 ; GCN-LABEL: {{^}}extract_w_offset_salu_use_vector:
-; MOVREL: s_mov_b32 m0
 ; GCN-DAG: s_or_b32
 ; GCN-DAG: s_or_b32
 ; GCN-DAG: s_or_b32
 ; GCN-DAG: s_or_b32
+; MOVREL: s_mov_b32 m0
 ; GCN-DAG: v_mov_b32_e32 v{{[0-9]+}}, s{{[0-9]+}}
 ; GCN-DAG: v_mov_b32_e32 v{{[0-9]+}}, s{{[0-9]+}}
 ; GCN-DAG: v_mov_b32_e32 v{{[0-9]+}}, s{{[0-9]+}}
 ; GCN-DAG: v_mov_b32_e32 v{{[0-9]+}}, s{{[0-9]+}}
 
+
 ; MOVREL: v_movrels_b32_e32
 
 ; IDXMODE: s_set_gpr_idx_on s{{[0-9]+}}, src0{{$}}
@@ -176,7 +178,8 @@ entry:
 }
 
 ; GCN-LABEL: {{^}}insert_w_offset:
-; GCN-DAG: s_load_dword [[IN:s[0-9]+]]
+; GCN-DAG: s_load_dword [[IN0:s[0-9]+]]
+; MOVREL-DAG: s_add_i32 [[IN:s[0-9]+]], [[IN0]], 1
 ; MOVREL-DAG: s_mov_b32 m0, [[IN]]
 ; GCN-DAG: v_mov_b32_e32 v[[ELT0:[0-9]+]], 1.0
 ; GCN-DAG: v_mov_b32_e32 v[[ELT1:[0-9]+]], 2.0
@@ -185,7 +188,7 @@ entry:
 ; GCN-DAG: v_mov_b32_e32 v[[ELT15:[0-9]+]], 0x41800000
 ; GCN-DAG: v_mov_b32_e32 v[[INS:[0-9]+]], 0x41880000
 
-; MOVREL: v_movreld_b32_e32 v[[ELT1]], v[[INS]]
+; MOVREL: v_movreld_b32_e32 v[[ELT0]], v[[INS]]
 ; MOVREL: buffer_store_dwordx4 v{{\[}}[[ELT0]]:[[ELT3]]{{\]}}
 define amdgpu_kernel void @insert_w_offset(<16 x float> addrspace(1)* %out, i32 %in) {
 entry:
@@ -195,6 +198,51 @@ entry:
   ret void
 }
 
+; GCN-LABEL: {{^}}insert_unsigned_base_plus_offset:
+; GCN-DAG: s_load_dword [[IN:s[0-9]+]]
+; GCN-DAG: v_mov_b32_e32 [[ELT0:v[0-9]+]], 1.0
+; GCN-DAG: v_mov_b32_e32 [[ELT1:v[0-9]+]], 2.0
+; GCN-DAG: s_and_b32 [[BASE:s[0-9]+]], [[IN]], 0xffff
+
+; MOVREL: s_mov_b32 m0, [[BASE]]
+; MOVREL: v_movreld_b32_e32 [[ELT1]], v{{[0-9]+}}
+
+; IDXMODE: s_set_gpr_idx_on [[BASE]], dst
+; IDXMODE-NEXT: v_mov_b32_e32 [[ELT1]], v{{[0-9]+}}
+; IDXMODE-NEXT: s_set_gpr_idx_off
+define amdgpu_kernel void @insert_unsigned_base_plus_offset(<16 x float> addrspace(1)* %out, i16 %in) {
+entry:
+  %base = zext i16 %in to i32
+  %add = add i32 %base, 1
+  %ins = insertelement <16 x float> <float 1.0, float 2.0, float 3.0, float 4.0, float 5.0, float 6.0, float 7.0, float 8.0, float 9.0, float 10.0, float 11.0, float 12.0, float 13.0, float 14.0, float 15.0, float 16.0>, float 17.0, i32 %add
+  store <16 x float> %ins, <16 x float> addrspace(1)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}insert_signed_base_plus_offset:
+; GCN-DAG: s_load_dword [[IN:s[0-9]+]]
+; GCN-DAG: v_mov_b32_e32 [[ELT0:v[0-9]+]], 1.0
+; GCN-DAG: v_mov_b32_e32 [[ELT1:v[0-9]+]], 2.0
+
+; GCN-DAG: s_sext_i32_i16 [[BASE:s[0-9]+]], [[IN]]
+; GCN-DAG: s_add_i32 [[BASE_PLUS_OFFSET:s[0-9]+]], [[BASE]], 1
+
+; MOVREL: s_mov_b32 m0, [[BASE_PLUS_OFFSET]]
+; MOVREL: v_movreld_b32_e32 [[ELT0]], v{{[0-9]+}}
+
+; IDXMODE: s_set_gpr_idx_on [[BASE_PLUS_OFFSET]], dst
+; IDXMODE-NEXT: v_mov_b32_e32 [[ELT0]], v{{[0-9]+}}
+; IDXMODE-NEXT: s_set_gpr_idx_off
+define amdgpu_kernel void @insert_signed_base_plus_offset(<16 x float> addrspace(1)* %out, i16 %in) {
+entry:
+  %base = sext i16 %in to i32
+  %add = add i32 %base, 1
+  %ins = insertelement <16 x float> <float 1.0, float 2.0, float 3.0, float 4.0, float 5.0, float 6.0, float 7.0, float 8.0, float 9.0, float 10.0, float 11.0, float 12.0, float 13.0, float 14.0, float 15.0, float 16.0>, float 17.0, i32 %add
+  store <16 x float> %ins, <16 x float> addrspace(1)* %out
+  ret void
+}
+
+
 ; GCN-LABEL: {{^}}insert_wo_offset:
 ; GCN: s_load_dword [[IN:s[0-9]+]]
 
@@ -354,8 +402,12 @@ entry:
 
 ; GCN: s_mov_b64 [[MASK:s\[[0-9]+:[0-9]+\]]], exec
 
+; GCN: s_waitcnt vmcnt(0)
+; PREGFX9: v_add_{{i32|u32}}_e32 [[IDX1:v[0-9]+]], vcc, 1, [[IDX0]]
+; GFX9: v_add_{{i32|u32}}_e32 [[IDX1:v[0-9]+]], 1, [[IDX0]]
+
+
 ; GCN: [[LOOP0:BB[0-9]+_[0-9]+]]:
-; GCN-NEXT: s_waitcnt vmcnt(0)
 ; GCN-NEXT: v_readfirstlane_b32 [[READLANE:s[0-9]+]], [[IDX0]]
 ; GCN: v_cmp_eq_u32_e32 vcc, [[READLANE]], [[IDX0]]
 ; GCN: s_and_saveexec_b64 vcc, vcc
@@ -373,20 +425,20 @@ entry:
 ; FIXME: Redundant copy
 ; GCN: s_mov_b64 exec, [[MASK]]
 
-; GCN: v_mov_b32_e32 [[VEC_ELT1_2:v[0-9]+]], [[S_ELT1]]
+; GCN: v_mov_b32_e32 [[VEC_ELT0_2:v[0-9]+]], [[S_ELT0]]
 
 ; GCN: s_mov_b64 [[MASK2:s\[[0-9]+:[0-9]+\]]], exec
 
 ; GCN: [[LOOP1:BB[0-9]+_[0-9]+]]:
-; GCN-NEXT: v_readfirstlane_b32 [[READLANE:s[0-9]+]], [[IDX0]]
-; GCN: v_cmp_eq_u32_e32 vcc, [[READLANE]], [[IDX0]]
+; GCN-NEXT: v_readfirstlane_b32 [[READLANE:s[0-9]+]], [[IDX1]]
+; GCN: v_cmp_eq_u32_e32 vcc, [[READLANE]], [[IDX1]]
 ; GCN: s_and_saveexec_b64 vcc, vcc
 
 ; MOVREL: s_mov_b32 m0, [[READLANE]]
-; MOVREL-NEXT: v_movrels_b32_e32 [[MOVREL1:v[0-9]+]], [[VEC_ELT1_2]]
+; MOVREL-NEXT: v_movrels_b32_e32 [[MOVREL1:v[0-9]+]], [[VEC_ELT0_2]]
 
 ; IDXMODE: s_set_gpr_idx_on [[READLANE]], src0
-; IDXMODE-NEXT: v_mov_b32_e32 [[MOVREL1:v[0-9]+]], [[VEC_ELT1_2]]
+; IDXMODE-NEXT: v_mov_b32_e32 [[MOVREL1:v[0-9]+]], [[VEC_ELT0_2]]
 ; IDXMODE: s_set_gpr_idx_off
 
 ; GCN-NEXT: s_xor_b64 exec, exec, vcc
@@ -492,13 +544,15 @@ bb:
 
 ; offset puts outside of superegister bounaries, so clamp to 1st element.
 ; GCN-LABEL: {{^}}extract_largest_inbounds_offset:
-; GCN-DAG: buffer_load_dwordx4 v{{\[}}[[LO_ELT:[0-9]+]]:[[HI_ELT:[0-9]+]]{{\].* offset:48}}
-; GCN-DAG: s_load_dword [[IDX:s[0-9]+]]
+; GCN-DAG: buffer_load_dwordx4 v{{\[}}[[LO_ELT:[0-9]+]]:[[HI_ELT:[0-9]+]]
+; GCN-DAG: s_load_dword [[IDX0:s[0-9]+]]
+; GCN-DAG: s_add_i32 [[IDX:s[0-9]+]], [[IDX0]], 15
+
 ; MOVREL: s_mov_b32 m0, [[IDX]]
-; MOVREL: v_movrels_b32_e32 [[EXTRACT:v[0-9]+]], v[[HI_ELT]]
+; MOVREL: v_movrels_b32_e32 [[EXTRACT:v[0-9]+]], v[[LO_ELT]]
 
 ; IDXMODE: s_set_gpr_idx_on [[IDX]], src0
-; IDXMODE: v_mov_b32_e32 [[EXTRACT:v[0-9]+]], v[[HI_ELT]]
+; IDXMODE: v_mov_b32_e32 [[EXTRACT:v[0-9]+]], v[[LO_ELT]]
 ; IDXMODE: s_set_gpr_idx_off
 
 ; GCN: buffer_store_dword [[EXTRACT]]
@@ -514,10 +568,11 @@ entry:
 ; GCN-LABEL: {{^}}extract_out_of_bounds_offset:
 ; GCN-DAG: buffer_load_dwordx4 v{{\[}}[[LO_ELT:[0-9]+]]:[[HI_ELT:[0-9]+]]{{\]}}
 ; GCN-DAG: s_load_dword [[IDX:s[0-9]+]]
-; MOVREL: s_add_i32 m0, [[IDX]], 16
+; GCN: s_add_i32 [[ADD_IDX:s[0-9]+]], [[IDX]], 16
+
+; MOVREL: s_mov_b32 m0, [[ADD_IDX]]
 ; MOVREL: v_movrels_b32_e32 [[EXTRACT:v[0-9]+]], v[[LO_ELT]]
 
-; IDXMODE: s_add_i32 [[ADD_IDX:s[0-9]+]], [[IDX]], 16
 ; IDXMODE: s_set_gpr_idx_on [[ADD_IDX]], src0
 ; IDXMODE: v_mov_b32_e32 [[EXTRACT:v[0-9]+]], v[[LO_ELT]]
 ; IDXMODE: s_set_gpr_idx_off
@@ -532,18 +587,15 @@ entry:
   ret void
 }
 
-; Test that the or is folded into the base address register instead of
-; added to m0
-
 ; GCN-LABEL: {{^}}extractelement_v16i32_or_index:
 ; GCN: s_load_dword [[IDX_IN:s[0-9]+]]
 ; GCN: s_lshl_b32 [[IDX_SHL:s[0-9]+]], [[IDX_IN]]
-; GCN-NOT: [[IDX_SHL]]
+; GCN: s_or_b32 [[IDX_FIN:s[0-9]+]], [[IDX_SHL]], 1
 
-; MOVREL: s_mov_b32 m0, [[IDX_SHL]]
+; MOVREL: s_mov_b32 m0, [[IDX_FIN]]
 ; MOVREL: v_movrels_b32_e32 v{{[0-9]+}}, v{{[0-9]+}}
 
-; IDXMODE: s_set_gpr_idx_on [[IDX_SHL]], src0
+; IDXMODE: s_set_gpr_idx_on [[IDX_FIN]], src0
 ; IDXMODE: v_mov_b32_e32 v{{[0-9]+}}, v{{[0-9]+}}
 ; IDXMODE: s_set_gpr_idx_off
 define amdgpu_kernel void @extractelement_v16i32_or_index(i32 addrspace(1)* %out, <16 x i32> addrspace(1)* %in, i32 %idx.in) {
@@ -559,12 +611,12 @@ entry:
 ; GCN-LABEL: {{^}}insertelement_v16f32_or_index:
 ; GCN: s_load_dword [[IDX_IN:s[0-9]+]]
 ; GCN: s_lshl_b32 [[IDX_SHL:s[0-9]+]], [[IDX_IN]]
-; GCN-NOT: [[IDX_SHL]]
+; GCN: s_or_b32 [[IDX_FIN:s[0-9]+]], [[IDX_SHL]], 1
 
-; MOVREL: s_mov_b32 m0, [[IDX_SHL]]
+; MOVREL: s_mov_b32 m0, [[IDX_FIN]]
 ; MOVREL: v_movreld_b32_e32 v{{[0-9]+}}, v{{[0-9]+}}
 
-; IDXMODE: s_set_gpr_idx_on [[IDX_SHL]], dst
+; IDXMODE: s_set_gpr_idx_on [[IDX_FIN]], dst
 ; IDXMODE: v_mov_b32_e32 v{{[0-9]+}}, v{{[0-9]+}}
 ; IDXMODE: s_set_gpr_idx_off
 define amdgpu_kernel void @insertelement_v16f32_or_index(<16 x float> addrspace(1)* %out, <16 x float> %a, i32 %idx.in) nounwind {
diff --git a/test/CodeGen/AMDGPU/indirect-addressing-term.ll b/test/CodeGen/AMDGPU/indirect-addressing-term.ll
new file mode 100644
index 0000000000000000000000000000000000000000..358aa5f38ec6919b1ff12adc298d1b21633a8171
--- /dev/null
+++ b/test/CodeGen/AMDGPU/indirect-addressing-term.ll
@@ -0,0 +1,115 @@
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+; RUN: llc -O0 -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=gfx900 -mattr=-flat-for-global -verify-machineinstrs -stop-after=regallocfast < %s | FileCheck -check-prefixes=GCN %s
+
+; Verify that we consider the xor at the end of the waterfall loop emitted for
+; divergent indirect addressing as a terminator.
+
+declare i32 @llvm.amdgcn.workitem.id.x() #1
+
+; There should be no spill code inserted between the xor and the real terminator
+define amdgpu_kernel void @extract_w_offset_vgpr(i32 addrspace(1)* %out) {
+  ; GCN-LABEL: name: extract_w_offset_vgpr
+  ; GCN: bb.0.entry:
+  ; GCN:   successors: %bb.1(0x80000000)
+  ; GCN:   liveins: $vgpr0, $sgpr0_sgpr1
+  ; GCN:   renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed renamable $sgpr0_sgpr1, 36, 0 :: (dereferenceable invariant load 8 from %ir.out.kernarg.offset.cast, align 4, addrspace 4)
+  ; GCN:   renamable $sgpr2 = COPY renamable $sgpr1
+  ; GCN:   renamable $sgpr4 = COPY renamable $sgpr0, implicit killed $sgpr0_sgpr1
+  ; GCN:   renamable $sgpr5 = S_MOV_B32 61440
+  ; GCN:   renamable $sgpr6 = S_MOV_B32 -1
+  ; GCN:   undef renamable $sgpr8 = COPY killed renamable $sgpr4, implicit-def $sgpr8_sgpr9_sgpr10_sgpr11
+  ; GCN:   renamable $sgpr9 = COPY killed renamable $sgpr2
+  ; GCN:   renamable $sgpr10 = COPY killed renamable $sgpr6
+  ; GCN:   renamable $sgpr11 = COPY killed renamable $sgpr5
+  ; GCN:   renamable $sgpr2 = S_MOV_B32 16
+  ; GCN:   renamable $sgpr4 = S_MOV_B32 15
+  ; GCN:   renamable $sgpr5 = S_MOV_B32 14
+  ; GCN:   renamable $sgpr6 = S_MOV_B32 13
+  ; GCN:   renamable $sgpr7 = S_MOV_B32 12
+  ; GCN:   renamable $sgpr12 = S_MOV_B32 11
+  ; GCN:   renamable $sgpr13 = S_MOV_B32 10
+  ; GCN:   renamable $sgpr14 = S_MOV_B32 9
+  ; GCN:   renamable $sgpr15 = S_MOV_B32 8
+  ; GCN:   renamable $sgpr16 = S_MOV_B32 7
+  ; GCN:   renamable $sgpr17 = S_MOV_B32 6
+  ; GCN:   renamable $sgpr18 = S_MOV_B32 5
+  ; GCN:   renamable $sgpr19 = S_MOV_B32 3
+  ; GCN:   renamable $sgpr20 = S_MOV_B32 2
+  ; GCN:   renamable $sgpr21 = S_MOV_B32 1
+  ; GCN:   renamable $sgpr22 = S_MOV_B32 0
+  ; GCN:   renamable $vgpr1 = COPY killed renamable $sgpr22
+  ; GCN:   renamable $vgpr2 = COPY killed renamable $sgpr21
+  ; GCN:   renamable $vgpr3 = COPY killed renamable $sgpr20
+  ; GCN:   renamable $vgpr4 = COPY killed renamable $sgpr19
+  ; GCN:   renamable $vgpr5 = COPY killed renamable $sgpr18
+  ; GCN:   renamable $vgpr6 = COPY killed renamable $sgpr17
+  ; GCN:   renamable $vgpr7 = COPY killed renamable $sgpr16
+  ; GCN:   renamable $vgpr8 = COPY killed renamable $sgpr15
+  ; GCN:   renamable $vgpr9 = COPY killed renamable $sgpr14
+  ; GCN:   renamable $vgpr10 = COPY killed renamable $sgpr13
+  ; GCN:   renamable $vgpr11 = COPY killed renamable $sgpr12
+  ; GCN:   renamable $vgpr12 = COPY killed renamable $sgpr7
+  ; GCN:   renamable $vgpr13 = COPY killed renamable $sgpr6
+  ; GCN:   renamable $vgpr14 = COPY killed renamable $sgpr5
+  ; GCN:   renamable $vgpr15 = COPY killed renamable $sgpr4
+  ; GCN:   renamable $vgpr16 = COPY killed renamable $sgpr2
+  ; GCN:   undef renamable $vgpr17 = COPY killed renamable $vgpr1, implicit-def $vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32
+  ; GCN:   renamable $vgpr18 = COPY killed renamable $vgpr2
+  ; GCN:   renamable $vgpr19 = COPY killed renamable $vgpr3
+  ; GCN:   renamable $vgpr20 = COPY killed renamable $vgpr4
+  ; GCN:   renamable $vgpr21 = COPY killed renamable $vgpr5
+  ; GCN:   renamable $vgpr22 = COPY killed renamable $vgpr6
+  ; GCN:   renamable $vgpr23 = COPY killed renamable $vgpr7
+  ; GCN:   renamable $vgpr24 = COPY killed renamable $vgpr8
+  ; GCN:   renamable $vgpr25 = COPY killed renamable $vgpr9
+  ; GCN:   renamable $vgpr26 = COPY killed renamable $vgpr10
+  ; GCN:   renamable $vgpr27 = COPY killed renamable $vgpr11
+  ; GCN:   renamable $vgpr28 = COPY killed renamable $vgpr12
+  ; GCN:   renamable $vgpr29 = COPY killed renamable $vgpr13
+  ; GCN:   renamable $vgpr30 = COPY killed renamable $vgpr14
+  ; GCN:   renamable $vgpr31 = COPY killed renamable $vgpr15
+  ; GCN:   renamable $vgpr32 = COPY killed renamable $vgpr16
+  ; GCN:   renamable $sgpr0_sgpr1 = S_MOV_B64 $exec
+  ; GCN:   renamable $vgpr1 = IMPLICIT_DEF
+  ; GCN:   renamable $sgpr24_sgpr25 = IMPLICIT_DEF
+  ; GCN:   SI_SPILL_V32_SAVE killed $vgpr0, %stack.0, $sgpr96_sgpr97_sgpr98_sgpr99, $sgpr3, 0, implicit $exec :: (store 4 into %stack.0, addrspace 5)
+  ; GCN:   SI_SPILL_S128_SAVE killed $sgpr8_sgpr9_sgpr10_sgpr11, %stack.1, implicit $exec, implicit $sgpr96_sgpr97_sgpr98_sgpr99, implicit $sgpr3, implicit-def dead $m0 :: (store 16 into %stack.1, align 4, addrspace 5)
+  ; GCN:   SI_SPILL_V512_SAVE killed $vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31_vgpr32, %stack.2, $sgpr96_sgpr97_sgpr98_sgpr99, $sgpr3, 0, implicit $exec :: (store 64 into %stack.2, align 4, addrspace 5)
+  ; GCN:   SI_SPILL_S64_SAVE killed $sgpr0_sgpr1, %stack.3, implicit $exec, implicit $sgpr96_sgpr97_sgpr98_sgpr99, implicit $sgpr3, implicit-def dead $m0 :: (store 8 into %stack.3, align 4, addrspace 5)
+  ; GCN:   SI_SPILL_V32_SAVE killed $vgpr1, %stack.4, $sgpr96_sgpr97_sgpr98_sgpr99, $sgpr3, 0, implicit $exec :: (store 4 into %stack.4, addrspace 5)
+  ; GCN:   SI_SPILL_S64_SAVE killed $sgpr24_sgpr25, %stack.5, implicit $exec, implicit $sgpr96_sgpr97_sgpr98_sgpr99, implicit $sgpr3, implicit-def dead $m0 :: (store 8 into %stack.5, align 4, addrspace 5)
+  ; GCN: bb.1:
+  ; GCN:   successors: %bb.1(0x40000000), %bb.2(0x40000000)
+  ; GCN:   $sgpr0_sgpr1 = SI_SPILL_S64_RESTORE %stack.5, implicit $exec, implicit $sgpr96_sgpr97_sgpr98_sgpr99, implicit $sgpr3, implicit-def dead $m0 :: (load 8 from %stack.5, align 4, addrspace 5)
+  ; GCN:   $vgpr0 = SI_SPILL_V32_RESTORE %stack.4, $sgpr96_sgpr97_sgpr98_sgpr99, $sgpr3, 0, implicit $exec :: (load 4 from %stack.4, addrspace 5)
+  ; GCN:   $vgpr1 = SI_SPILL_V32_RESTORE %stack.0, $sgpr96_sgpr97_sgpr98_sgpr99, $sgpr3, 0, implicit $exec :: (load 4 from %stack.0, addrspace 5)
+  ; GCN:   renamable $sgpr2 = V_READFIRSTLANE_B32 $vgpr1, implicit $exec
+  ; GCN:   renamable $sgpr4_sgpr5 = V_CMP_EQ_U32_e64 $sgpr2, killed $vgpr1, implicit $exec
+  ; GCN:   renamable $sgpr4_sgpr5 = S_AND_SAVEEXEC_B64 killed renamable $sgpr4_sgpr5, implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GCN:   S_SET_GPR_IDX_ON killed renamable $sgpr2, 1, implicit-def $m0, implicit undef $m0
+  ; GCN:   $vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17 = SI_SPILL_V512_RESTORE %stack.2, $sgpr96_sgpr97_sgpr98_sgpr99, $sgpr3, 0, implicit $exec :: (load 64 from %stack.2, align 4, addrspace 5)
+  ; GCN:   renamable $vgpr18 = V_MOV_B32_e32 undef $vgpr3, implicit $exec, implicit killed $vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17, implicit $m0
+  ; GCN:   S_SET_GPR_IDX_OFF
+  ; GCN:   renamable $vgpr19 = COPY renamable $vgpr18
+  ; GCN:   renamable $sgpr6_sgpr7 = COPY renamable $sgpr4_sgpr5
+  ; GCN:   SI_SPILL_S64_SAVE killed $sgpr6_sgpr7, %stack.5, implicit $exec, implicit $sgpr96_sgpr97_sgpr98_sgpr99, implicit $sgpr3, implicit-def dead $m0 :: (store 8 into %stack.5, align 4, addrspace 5)
+  ; GCN:   SI_SPILL_S64_SAVE killed $sgpr0_sgpr1, %stack.6, implicit $exec, implicit $sgpr96_sgpr97_sgpr98_sgpr99, implicit $sgpr3, implicit-def dead $m0 :: (store 8 into %stack.6, align 4, addrspace 5)
+  ; GCN:   SI_SPILL_V32_SAVE killed $vgpr19, %stack.4, $sgpr96_sgpr97_sgpr98_sgpr99, $sgpr3, 0, implicit $exec :: (store 4 into %stack.4, addrspace 5)
+  ; GCN:   SI_SPILL_V32_SAVE killed $vgpr0, %stack.7, $sgpr96_sgpr97_sgpr98_sgpr99, $sgpr3, 0, implicit $exec :: (store 4 into %stack.7, addrspace 5)
+  ; GCN:   SI_SPILL_V32_SAVE killed $vgpr18, %stack.8, $sgpr96_sgpr97_sgpr98_sgpr99, $sgpr3, 0, implicit $exec :: (store 4 into %stack.8, addrspace 5)
+  ; GCN:   $exec = S_XOR_B64_term $exec, killed renamable $sgpr4_sgpr5, implicit-def $scc
+  ; GCN:   S_CBRANCH_EXECNZ %bb.1, implicit $exec
+  ; GCN: bb.2:
+  ; GCN:   $sgpr0_sgpr1 = SI_SPILL_S64_RESTORE %stack.3, implicit $exec, implicit $sgpr96_sgpr97_sgpr98_sgpr99, implicit $sgpr3, implicit-def dead $m0 :: (load 8 from %stack.3, align 4, addrspace 5)
+  ; GCN:   $exec = S_MOV_B64 killed renamable $sgpr0_sgpr1
+  ; GCN:   $vgpr0 = SI_SPILL_V32_RESTORE %stack.8, $sgpr96_sgpr97_sgpr98_sgpr99, $sgpr3, 0, implicit $exec :: (load 4 from %stack.8, addrspace 5)
+  ; GCN:   $sgpr4_sgpr5_sgpr6_sgpr7 = SI_SPILL_S128_RESTORE %stack.1, implicit $exec, implicit $sgpr96_sgpr97_sgpr98_sgpr99, implicit $sgpr3, implicit-def dead $m0 :: (load 16 from %stack.1, align 4, addrspace 5)
+  ; GCN:   BUFFER_STORE_DWORD_OFFSET killed renamable $vgpr0, killed renamable $sgpr4_sgpr5_sgpr6_sgpr7, 0, 0, 0, 0, 0, implicit $exec :: (store 4 into %ir.out.load, addrspace 1)
+  ; GCN:   S_ENDPGM
+entry:
+  %id = call i32 @llvm.amdgcn.workitem.id.x() #1
+  %index = add i32 %id, 1
+  %value = extractelement <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16>, i32 %index
+  store i32 %value, i32 addrspace(1)* %out
+  ret void
+}
diff --git a/test/CodeGen/AMDGPU/insert_vector_dynelt.ll b/test/CodeGen/AMDGPU/insert_vector_dynelt.ll
index 80309b40e17335da2f5f0294b3ed1670c3d9b5d0..b311b6aa29d7e9386c82800faa1f1fd6a80d414b 100644
--- a/test/CodeGen/AMDGPU/insert_vector_dynelt.ll
+++ b/test/CodeGen/AMDGPU/insert_vector_dynelt.ll
@@ -112,7 +112,10 @@ entry:
 ; GCN-NOT: buffer_
 ; GCN:     s_lshl_b32 [[SEL:s[0-9]+]], s{{[0-9]+}}, 4
 ; GCN:     s_lshl_b64 s[{{[0-9:]+}}], s[{{[0-9:]+}}], [[SEL]]
-; GCN:     s_and_b32 s{{[0-9]+}}, s{{[0-9]+}}, 0x3c00
+; GCN:     s_mov_b32 [[K:s[0-9]+]], 0x3c003c00
+; GCN:     v_mov_b32_e32 [[V:v[0-9]+]], [[K]]
+; GCN:     v_bfi_b32 v{{[0-9]+}}, s{{[0-9]+}}, [[V]], v{{[0-9]+}}
+; GCN:     v_bfi_b32 v{{[0-9]+}}, s{{[0-9]+}}, [[V]], v{{[0-9]+}}
 define amdgpu_kernel void @half4_inselt(<4 x half> addrspace(1)* %out, <4 x half> %vec, i32 %sel) {
 entry:
   %v = insertelement <4 x half> %vec, half 1.000000e+00, i32 %sel
@@ -168,9 +171,10 @@ entry:
 ; GCN-NOT: v_cndmask_b32
 ; GCN-NOT: v_movrel
 ; GCN-NOT: buffer_
+; GCN:     v_mov_b32_e32 [[K:v[0-9]+]], 0x10001
 ; GCN:     s_lshl_b32 [[SEL:s[0-9]+]], s{{[0-9]+}}, 4
 ; GCN:     s_lshl_b32 [[V:s[0-9]+]], 0xffff, [[SEL]]
-; GCN:     v_bfi_b32 v{{[0-9]+}}, [[V]], 1, v{{[0-9]+}}
+; GCN:     v_bfi_b32 v{{[0-9]+}}, [[V]], [[K]], v{{[0-9]+}}
 define amdgpu_kernel void @short2_inselt(<2 x i16> addrspace(1)* %out, <2 x i16> %vec, i32 %sel) {
 entry:
   %v = insertelement <2 x i16> %vec, i16 1, i32 %sel
@@ -184,7 +188,10 @@ entry:
 ; GCN-NOT: buffer_
 ; GCN:     s_lshl_b32 [[SEL:s[0-9]+]], s{{[0-9]+}}, 4
 ; GCN:     s_lshl_b64 s[{{[0-9:]+}}], s[{{[0-9:]+}}], [[SEL]]
-; GCN:     s_and_b32 s{{[0-9]+}}, s{{[0-9]+}}, 1
+; GCN:     s_mov_b32 [[K:s[0-9]+]], 0x10001
+; GCN:     v_mov_b32_e32 [[V:v[0-9]+]], [[K]]
+; GCN:     v_bfi_b32 v{{[0-9]+}}, s{{[0-9]+}}, [[V]], v{{[0-9]+}}
+; GCN:     v_bfi_b32 v{{[0-9]+}}, s{{[0-9]+}}, [[V]], v{{[0-9]+}}
 define amdgpu_kernel void @short4_inselt(<4 x i16> addrspace(1)* %out, <4 x i16> %vec, i32 %sel) {
 entry:
   %v = insertelement <4 x i16> %vec, i16 1, i32 %sel
@@ -197,7 +204,11 @@ entry:
 ; GCN-NOT: buffer_
 ; GCN:     s_lshl_b32 [[SEL:s[0-9]+]], s{{[0-9]+}}, 3
 ; GCN:     s_lshl_b64 s[{{[0-9:]+}}], s[{{[0-9:]+}}], [[SEL]]
-; GCN:     s_and_b32 s{{[0-9]+}}, s{{[0-9]+}}, 1
+; GCN:     s_mov_b32 [[K:s[0-9]+]], 0x1010101
+; GCN:     s_and_b32 s3, s1, [[K]]
+; GCN:     s_and_b32 s{{[0-9]+}}, s{{[0-9]+}}, [[K]]
+; GCN:     s_andn2_b64 s[{{[0-9:]+}}], s[{{[0-9:]+}}], s[{{[0-9:]+}}]
+; GCN:     s_or_b64 s[{{[0-9:]+}}], s[{{[0-9:]+}}], s[{{[0-9:]+}}]
 define amdgpu_kernel void @byte8_inselt(<8 x i8> addrspace(1)* %out, <8 x i8> %vec, i32 %sel) {
 entry:
   %v = insertelement <8 x i8> %vec, i8 1, i32 %sel
diff --git a/test/CodeGen/AMDGPU/insert_vector_elt.ll b/test/CodeGen/AMDGPU/insert_vector_elt.ll
index 93ee16ea85d85369e49a75a8fdf3f81794e5c890..47e080a94baa49f080676aeba31b0942c44385e7 100644
--- a/test/CodeGen/AMDGPU/insert_vector_elt.ll
+++ b/test/CodeGen/AMDGPU/insert_vector_elt.ll
@@ -242,7 +242,7 @@ define amdgpu_kernel void @dynamic_insertelement_v3i16(<3 x i16> addrspace(1)* %
 ; VI-NOT: _load
 ; VI: s_lshl_b32 [[SCALED_IDX:s[0-9]+]], [[IDX]], 3
 ; VI: v_lshlrev_b16_e64 [[MASK:v[0-9]+]], [[SCALED_IDX]], -1
-; VI: v_and_b32_e32 [[INSERT:v[0-9]+]], 5, [[MASK]]
+; VI: v_and_b32_e32 [[INSERT:v[0-9]+]], 0x505, [[MASK]]
 ; VI: v_xor_b32_e32 [[NOT_MASK:v[0-9]+]], -1, [[MASK]]
 ; VI: v_and_b32_e32 [[AND_NOT_MASK:v[0-9]+]], [[LOAD]], [[NOT_MASK]]
 ; VI: v_or_b32_e32 [[OR:v[0-9]+]], [[INSERT]], [[AND_NOT_MASK]]
@@ -261,15 +261,14 @@ define amdgpu_kernel void @dynamic_insertelement_v2i8(<2 x i8> addrspace(1)* %ou
 ; VI-NEXT: s_load_dword [[IDX:s[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, 0x4c
 ; VI-NOT: _load
 
+; VI: v_mov_b32_e32 [[VAL:v[0-9]+]], 0x5050505
 ; VI: v_mov_b32_e32 [[V_LOAD:v[0-9]+]], [[LOAD]]
 ; VI: s_lshl_b32 [[SCALED_IDX:s[0-9]+]], [[IDX]], 3
 ; VI: s_lshl_b32 [[SHIFTED_MASK:s[0-9]+]], 0xffff, [[SCALED_IDX]]
-; VI: s_andn2_b32 [[AND_NOT_MASK:s[0-9]+]], [[LOAD]],  [[SHIFTED_MASK]]
-; VI: v_bfi_b32 [[BFI:v[0-9]+]], [[SHIFTED_MASK]], 5, [[V_LOAD]]
-; VI: s_lshr_b32 [[HI2:s[0-9]+]], [[AND_NOT_MASK]], 16
+; VI: v_bfi_b32 [[BFI:v[0-9]+]], [[SHIFTED_MASK]], [[VAL]], [[V_LOAD]]
+; VI: v_lshrrev_b32_e32 [[V_HI2:v[0-9]+]], 16, [[BFI]]
 
-; VI-DAG: buffer_store_short [[BFI]]
-; VI-DAG: v_mov_b32_e32 [[V_HI2:v[0-9]+]], [[HI2]]
+; VI: buffer_store_short [[BFI]]
 ; VI: buffer_store_byte [[V_HI2]]
 define amdgpu_kernel void @dynamic_insertelement_v3i8(<3 x i8> addrspace(1)* %out, [8 x i32], <3 x i8> %a, [8 x i32], i32 %b) nounwind {
   %vecins = insertelement <3 x i8> %a, i8 5, i32 %b
@@ -282,10 +281,11 @@ define amdgpu_kernel void @dynamic_insertelement_v3i8(<3 x i8> addrspace(1)* %ou
 ; VI-NEXT: s_load_dword [[IDX:s[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, 0x4c
 ; VI-NOT: _load
 
+; VI: v_mov_b32_e32 [[VAL:v[0-9]+]], 0x5050505
 ; VI: v_mov_b32_e32 [[V_LOAD:v[0-9]+]], [[LOAD]]
 ; VI-DAG: s_lshl_b32 [[SCALED_IDX:s[0-9]+]], [[IDX]], 3
 ; VI: s_lshl_b32 [[SHIFTED_MASK:s[0-9]+]], 0xffff, [[SCALED_IDX]]
-; VI: v_bfi_b32 [[BFI:v[0-9]+]], [[SHIFTED_MASK]], 5, [[V_LOAD]]
+; VI: v_bfi_b32 [[BFI:v[0-9]+]], [[SHIFTED_MASK]], [[VAL]], [[V_LOAD]]
 ; VI: buffer_store_dword [[BFI]]
 define amdgpu_kernel void @dynamic_insertelement_v4i8(<4 x i8> addrspace(1)* %out, [8 x i32], <4 x i8> %a, [8 x i32], i32 %b) nounwind {
   %vecins = insertelement <4 x i8> %a, i8 5, i32 %b
@@ -303,9 +303,11 @@ define amdgpu_kernel void @dynamic_insertelement_v4i8(<4 x i8> addrspace(1)* %ou
 ; VI-DAG: s_lshl_b32 [[SCALED_IDX:s[0-9]+]], [[IDX]], 3
 ; VI-DAG: s_mov_b32 s[[MASK_LO:[0-9]+]], 0xffff
 ; VI: s_lshl_b64 s{{\[}}[[MASK_SHIFT_LO:[0-9]+]]:[[MASK_SHIFT_HI:[0-9]+]]{{\]}}, s{{\[}}[[MASK_LO]]:[[MASK_HI]]{{\]}}, [[SCALED_IDX]]
+; VI: s_mov_b32 [[VAL:s[0-9]+]], 0x5050505
+; VI: s_and_b32 s[[INS_HI:[0-9]+]], s[[MASK_SHIFT_HI]], [[VAL]]
+; VI: s_and_b32 s[[INS_LO:[0-9]+]], s[[MASK_SHIFT_LO]], [[VAL]]
 ; VI: s_andn2_b64 [[AND:s\[[0-9]+:[0-9]+\]]], [[VEC]], s{{\[}}[[MASK_SHIFT_LO]]:[[MASK_SHIFT_HI]]{{\]}}
-; VI: s_and_b32 s[[INS:[0-9]+]], s[[MASK_SHIFT_LO]], 5
-; VI: s_or_b64 s{{\[}}[[RESULT0:[0-9]+]]:[[RESULT1:[0-9]+]]{{\]}}, s{{\[}}[[INS]]:[[MASK_HI]]{{\]}}, [[AND]]
+; VI: s_or_b64 s{{\[}}[[RESULT0:[0-9]+]]:[[RESULT1:[0-9]+]]{{\]}}, s{{\[}}[[INS_LO]]:[[INS_HI]]{{\]}}, [[AND]]
 ; VI: v_mov_b32_e32 v[[V_RESULT0:[0-9]+]], s[[RESULT0]]
 ; VI: v_mov_b32_e32 v[[V_RESULT1:[0-9]+]], s[[RESULT1]]
 ; VI: buffer_store_dwordx2 v{{\[}}[[V_RESULT0]]:[[V_RESULT1]]{{\]}}
diff --git a/test/CodeGen/AMDGPU/insert_vector_elt.v2i16.ll b/test/CodeGen/AMDGPU/insert_vector_elt.v2i16.ll
index b4fb59983cbbe934cb8083274cef57a991ab60ae..a8a298045ff0833d5810df5fe94671f5fb29c011 100644
--- a/test/CodeGen/AMDGPU/insert_vector_elt.v2i16.ll
+++ b/test/CodeGen/AMDGPU/insert_vector_elt.v2i16.ll
@@ -446,7 +446,7 @@ define amdgpu_kernel void @v_insertelement_v2i16_dynamic_sgpr(<2 x i16> addrspac
 
 ; GCN-LABEL: {{^}}v_insertelement_v2f16_dynamic_vgpr:
 ; GFX89-DAG: s_mov_b32 [[MASKK:s[0-9]+]], 0xffff{{$}}
-; GCN-DAG: s_movk_i32 [[K:s[0-9]+]], 0x1234
+; GCN-DAG: s_mov_b32 [[K:s[0-9]+]], 0x12341234
 
 ; GCN-DAG: {{flat|global}}_load_dword [[IDX:v[0-9]+]]
 ; GCN-DAG: {{flat|global}}_load_dword [[VEC:v[0-9]+]]
@@ -611,25 +611,20 @@ define amdgpu_kernel void @v_insertelement_v4i16_2(<4 x i16> addrspace(1)* %out,
 ; GCN-DAG: s_load_dword [[VAL:s[0-9]+]]
 ; GCN-DAG: {{flat|global}}_load_dwordx2 v{{\[}}[[LO:[0-9]+]]:[[HI:[0-9]+]]{{\]}}
 
-; GCN-DAG: v_lshlrev_b32_e32 [[SCALED_IDX:v[0-9]+]], 4, [[IDX]]
+; GCN-DAG: s_mov_b32 s[[MASK_LO:[0-9]+]], 0xffff
 ; GCN-DAG: s_mov_b32 s[[MASK_HI:[0-9]+]], 0
-; GCN-DAG: s_mov_b32 s[[MASK_LO:[0-9]+]], 0xffff{{$}}
-
-; GFX89: v_lshlrev_b64 v{{\[}}[[SHIFT_LO:[0-9]+]]:[[SHIFT_HI:[0-9]+]]{{\]}}, [[SCALED_IDX]], s{{\[}}[[MASK_LO]]:[[MASK_HI]]{{\]}}
-; GFX89-DAG: v_not_b32_e32 v[[NOT_SHIFT_LO:[0-9+]]], v[[SHIFT_LO]]
-; GFX89-DAG: v_not_b32_e32 v[[NOT_SHIFT_HI:[0-9+]]], v[[SHIFT_HI]]
-; GFX89-DAG: v_and_b32_e32 v[[MASK:[0-9]+]], [[VAL]], v[[SHIFT_LO]]
-
-; GFX89-DAG: v_and_b32_e32 v[[AND0:[0-9]+]], v[[NOT_SHIFT_LO]], v[[LO]]
-; GFX89-DAG: v_and_b32_e32 v[[AND1:[0-9]+]], v[[NOT_SHIFT_HI]], v[[HI]]
-; GFX89: v_or_b32_sdwa v[[OR_SDWA:[0-9]+]], v[[MASK]], v[[AND0]] dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-
-
-; CI: v_lshl_b64 v{{\[}}[[SHIFT_LO:[0-9]+]]:[[SHIFT_HI:[0-9]+]]{{\]}}, s{{\[}}[[MASK_LO]]:[[MASK_HI]]{{\]}}, [[SCALED_IDX]]
-; CI-DAG: v_bfi_b32 v[[OR_SDWA:[0-9]+]], v[[SHIFT_LO]],
-; CI-DAG: v_bfi_b32 v[[AND1:[0-9]+]], v[[SHIFT_HI]], 0,
+; CIVI-DAG: s_and_b32 [[MASKED_VAL:s[0-9]+]], [[VAL]], s[[MASK_LO]]
+; VI-DAG: s_lshl_b32 [[SHIFTED_VAL:s[0-9]+]], [[MASKED_VAL]], 16
+; CI-DAG: s_lshl_b32 [[SHIFTED_VAL:s[0-9]+]], [[VAL]], 16
+; CIVI: s_or_b32 [[DUP_VAL:s[0-9]+]], [[MASKED_VAL]], [[SHIFTED_VAL]]
+; GCN-DAG: v_lshlrev_b32_e32 [[SCALED_IDX:v[0-9]+]], 4, [[IDX]]
+; GFX9-DAG: s_pack_ll_b32_b16 [[DUP_VAL:s[0-9]+]], [[VAL]], [[VAL]]
+; GFX89: v_lshlrev_b64 v[{{[0-9:]+}}], [[SCALED_IDX]], s{{\[}}[[MASK_LO]]:[[MASK_HI]]{{\]}}
+; CI: v_lshl_b64 v[{{[0-9:]+}}], s[{{[0-9:]+}}], [[SCALED_IDX]]
+; GCN: v_bfi_b32 v{{[0-9]+}}, v{{[0-9]+}}, [[DUP_VAL]], v{{[0-9]+}}
+; GCN: v_bfi_b32 v{{[0-9]+}}, v{{[0-9]+}}, [[DUP_VAL]], v{{[0-9]+}}
 
-; GCN: {{flat|global}}_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[}}[[OR_SDWA]]:[[AND1]]{{\]}}
+; GCN: {{flat|global}}_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}}
 define amdgpu_kernel void @v_insertelement_v4i16_dynamic_vgpr(<4 x i16> addrspace(1)* %out, <4 x i16> addrspace(1)* %in, i32 %val) #0 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x() #1
   %tid.ext = sext i32 %tid to i64
diff --git a/test/CodeGen/AMDGPU/insert_vector_elt.v2i16.subtest-nosaddr.ll b/test/CodeGen/AMDGPU/insert_vector_elt.v2i16.subtest-nosaddr.ll
index e35cf7a349ac572da1bd3985d7b81259205c0718..34e1d201c9cb6252b4aefd9ca8fdca5f0c322212 100644
--- a/test/CodeGen/AMDGPU/insert_vector_elt.v2i16.subtest-nosaddr.ll
+++ b/test/CodeGen/AMDGPU/insert_vector_elt.v2i16.subtest-nosaddr.ll
@@ -3,7 +3,7 @@
 
 ; GCN-LABEL: {{^}}v_insertelement_v2i16_dynamic_vgpr:
 ; GFX89-DAG: s_mov_b32 [[MASKK:s[0-9]+]], 0xffff{{$}}
-; GCN-DAG: s_movk_i32 [[K:s[0-9]+]], 0x3e7
+; GCN-DAG: s_mov_b32 [[K:s[0-9]+]], 0x3e703e7
 
 ; GCN: {{flat|global}}_load_dword [[IDX:v[0-9]+]]
 ; GCN: {{flat|global}}_load_dword [[VEC:v[0-9]+]]
diff --git a/test/CodeGen/AMDGPU/insert_vector_elt.v2i16.subtest-saddr.ll b/test/CodeGen/AMDGPU/insert_vector_elt.v2i16.subtest-saddr.ll
index 07ee65526c9610c2badeda24bb719fb8dab72c23..95e38c36e62a0bf04832e29b9afb78120cfdd87f 100644
--- a/test/CodeGen/AMDGPU/insert_vector_elt.v2i16.subtest-saddr.ll
+++ b/test/CodeGen/AMDGPU/insert_vector_elt.v2i16.subtest-saddr.ll
@@ -6,7 +6,7 @@
 ; GCN: {{flat|global}}_load_dword [[VEC:v[0-9]+]]
 
 ; GFX89-DAG: s_mov_b32 [[MASKK:s[0-9]+]], 0xffff{{$}}
-; GCN-DAG: s_movk_i32 [[K:s[0-9]+]], 0x3e7
+; GCN-DAG: s_mov_b32 [[K:s[0-9]+]], 0x3e7
 
 ; GFX89-DAG: v_lshlrev_b32_e32 [[SCALED_IDX:v[0-9]+]], 4, [[IDX]]
 ; GFX89-DAG: v_lshlrev_b32_e64 [[MASK:v[0-9]+]], [[SCALED_IDX]], [[MASKK]]
diff --git a/test/CodeGen/AMDGPU/ipra.ll b/test/CodeGen/AMDGPU/ipra.ll
index 276de14612d7ed3dab2cca0a5c59eb7f12696128..8027d6739daf550d378b24a0670f2f87fb2e0ec2 100644
--- a/test/CodeGen/AMDGPU/ipra.ll
+++ b/test/CodeGen/AMDGPU/ipra.ll
@@ -12,7 +12,7 @@ entry:
 
 ; GCN-LABEL: {{^}}func:
 ; GCN: ; NumVgprs: 8
-define void @func() #1 {
+define hidden void @func() #1 {
   call void asm sideeffect "", "~{v0},~{v1},~{v2},~{v3},~{v4},~{v5},~{v6},~{v7}"() #0
   ret void
 }
diff --git a/test/CodeGen/AMDGPU/llvm.amdgcn.buffer.load.ll b/test/CodeGen/AMDGPU/llvm.amdgcn.buffer.load.ll
index ba9f206f1512ca0110c0bce7a746c65ca526933e..bcde25ade0fca55ecb59aaf51d5c841548861ee8 100644
--- a/test/CodeGen/AMDGPU/llvm.amdgcn.buffer.load.ll
+++ b/test/CodeGen/AMDGPU/llvm.amdgcn.buffer.load.ll
@@ -195,7 +195,7 @@ main_body:
 
 ;CHECK-LABEL: {{^}}buffer_load_x3_offen_merged:
 ;CHECK-NEXT: %bb.
-;CHECK-NEXT: buffer_load_dwordx3 v[{{[0-9]}}:{{[0-9]}}], v0, s[0:3], 0 offen offset:4
+;VI-NEXT: buffer_load_dwordx3 v[{{[0-9]}}:{{[0-9]}}], v0, s[0:3], 0 offen offset:4
 ;CHECK: s_waitcnt
 define amdgpu_ps void @buffer_load_x3_offen_merged(<4 x i32> inreg %rsrc, i32 %a) {
 main_body:
@@ -245,7 +245,7 @@ main_body:
 
 ;CHECK-LABEL: {{^}}buffer_load_x3_offset_merged:
 ;CHECK-NEXT: %bb.
-;CHECK-NEXT: buffer_load_dwordx3 v[{{[0-9]}}:{{[0-9]}}], off, s[0:3], 0 offset:4
+;VI-NEXT: buffer_load_dwordx3 v[{{[0-9]}}:{{[0-9]}}], off, s[0:3], 0 offset:4
 ;CHECK: s_waitcnt
 define amdgpu_ps void @buffer_load_x3_offset_merged(<4 x i32> inreg %rsrc) {
 main_body:
diff --git a/test/CodeGen/AMDGPU/llvm.amdgcn.ds.append.ll b/test/CodeGen/AMDGPU/llvm.amdgcn.ds.append.ll
new file mode 100644
index 0000000000000000000000000000000000000000..132e3f717c54e53b064e00ed1ceaab1c66eb1d1c
--- /dev/null
+++ b/test/CodeGen/AMDGPU/llvm.amdgcn.ds.append.ll
@@ -0,0 +1,125 @@
+; RUN: llc -march=amdgcn -mcpu=tahiti -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SI,NOTGFX9 %s
+; RUN: llc -march=amdgcn -mcpu=bonaire -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,CIPLUS,NOTGFX9 %s
+; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,CIPLUS,NOTGFX9 %s
+; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,CIPLUS,GFX9 %s
+
+; GCN-LABEL: {{^}}ds_append_lds:
+; GCN: s_load_dword [[PTR:s[0-9]+]]
+; GCN: s_mov_b32 m0, [[PTR]]
+; GCN: ds_append [[RESULT:v[0-9]+]]{{$}}
+; GCN: {{.*}}store{{.*}} [[RESULT]]
+define amdgpu_kernel void @ds_append_lds(i32 addrspace(3)* %lds, i32 addrspace(1)* %out) #0 {
+  %val = call i32 @llvm.amdgcn.ds.append.p3i32(i32 addrspace(3)* %lds, i1 false)
+  store i32 %val, i32 addrspace(1)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}ds_append_lds_max_offset:
+; GCN: s_load_dword [[PTR:s[0-9]+]]
+; GCN: s_mov_b32 m0, [[PTR]]
+; GCN: ds_append [[RESULT:v[0-9]+]] offset:65532{{$}}
+; GCN: {{.*}}store{{.*}} [[RESULT]]
+define amdgpu_kernel void @ds_append_lds_max_offset(i32 addrspace(3)* %lds, i32 addrspace(1)* %out) #0 {
+  %gep = getelementptr inbounds i32, i32 addrspace(3)* %lds, i32 16383
+  %val = call i32 @llvm.amdgcn.ds.append.p3i32(i32 addrspace(3)* %gep, i1 false)
+  store i32 %val, i32 addrspace(1)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}ds_append_no_fold_offset_si:
+; GCN: s_load_dword [[PTR:s[0-9]+]]
+
+; SI: s_add_i32 [[PTR]], [[PTR]], 16
+; SI: s_mov_b32 m0, [[PTR]]
+; SI: ds_append [[RESULT:v[0-9]+]]{{$}}
+
+; CIPLUS: s_mov_b32 m0, [[PTR]]
+; CIPLUS: ds_append [[RESULT:v[0-9]+]] offset:16{{$}}
+
+; GCN: {{.*}}store{{.*}} [[RESULT]]
+define amdgpu_kernel void @ds_append_no_fold_offset_si(i32 addrspace(3)* addrspace(4)* %lds.ptr, i32 addrspace(1)* %out) #0 {
+  %lds = load i32 addrspace(3)*, i32 addrspace(3)* addrspace(4)* %lds.ptr, align 4
+  %gep = getelementptr inbounds i32, i32 addrspace(3)* %lds, i32 4
+  %val = call i32 @llvm.amdgcn.ds.append.p3i32(i32 addrspace(3)* %gep, i1 false)
+  store i32 %val, i32 addrspace(1)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}ds_append_lds_over_max_offset:
+; GCN: s_load_dword [[PTR:s[0-9]+]]
+
+; SI: s_bitset1_b32 [[PTR]], 16
+; CIPLUS: s_add_i32 [[PTR]], [[PTR]], 0x10000
+
+; GCN: s_mov_b32 m0, [[PTR]]
+; GCN: ds_append [[RESULT:v[0-9]+]]{{$}}
+; GCN: {{.*}}store{{.*}} [[RESULT]]
+define amdgpu_kernel void @ds_append_lds_over_max_offset(i32 addrspace(3)* %lds, i32 addrspace(1)* %out) #0 {
+  %gep = getelementptr inbounds i32, i32 addrspace(3)* %lds, i32 16384
+  %val = call i32 @llvm.amdgcn.ds.append.p3i32(i32 addrspace(3)* %gep, i1 false)
+  store i32 %val, i32 addrspace(1)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}ds_append_lds_vgpr_addr:
+; GCN: v_readfirstlane_b32 [[READLANE:s[0-9]+]], v0
+; GCN: s_mov_b32 m0, [[READLANE]]
+; GCN: ds_append [[RESULT:v[0-9]+]]{{$}}
+; GCN: {{.*}}store{{.*}} [[RESULT]]
+define void @ds_append_lds_vgpr_addr(i32 addrspace(3)* %lds, i32 addrspace(1)* %out) #0 {
+  %val = call i32 @llvm.amdgcn.ds.append.p3i32(i32 addrspace(3)* %lds, i1 false)
+  store i32 %val, i32 addrspace(1)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}ds_append_gds:
+; GCN: s_load_dword [[PTR:s[0-9]+]]
+; GCN: s_mov_b32 m0, [[PTR]]
+; GCN: ds_append [[RESULT:v[0-9]+]] gds{{$}}
+; GCN: {{.*}}store{{.*}} [[RESULT]]
+define amdgpu_kernel void @ds_append_gds(i32 addrspace(2)* %gds, i32 addrspace(1)* %out) #0 {
+  %val = call i32 @llvm.amdgcn.ds.append.p2i32(i32 addrspace(2)* %gds, i1 false)
+  store i32 %val, i32 addrspace(1)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}ds_append_gds_max_offset:
+; GCN: s_load_dword [[PTR:s[0-9]+]]
+; GCN: s_mov_b32 m0, [[PTR]]
+; GCN: ds_append [[RESULT:v[0-9]+]] offset:65532 gds{{$}}
+; GCN: {{.*}}store{{.*}} [[RESULT]]
+define amdgpu_kernel void @ds_append_gds_max_offset(i32 addrspace(2)* %gds, i32 addrspace(1)* %out) #0 {
+  %gep = getelementptr inbounds i32, i32 addrspace(2)* %gds, i32 16383
+  %val = call i32 @llvm.amdgcn.ds.append.p2i32(i32 addrspace(2)* %gep, i1 false)
+  store i32 %val, i32 addrspace(1)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}ds_append_gds_over_max_offset:
+define amdgpu_kernel void @ds_append_gds_over_max_offset(i32 addrspace(2)* %gds, i32 addrspace(1)* %out) #0 {
+  %gep = getelementptr inbounds i32, i32 addrspace(2)* %gds, i32 16384
+  %val = call i32 @llvm.amdgcn.ds.append.p2i32(i32 addrspace(2)* %gep, i1 false)
+  store i32 %val, i32 addrspace(1)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}ds_append_lds_m0_restore:
+; GCN: s_load_dword [[PTR:s[0-9]+]]
+; GCN: s_mov_b32 m0, [[PTR]]
+; GCN: ds_append [[RESULT:v[0-9]+]]{{$}}
+; NOTGFX9: s_mov_b32 m0, -1
+; GFX9-NOT: m0
+; GCN: _store_dword
+; GCN: ds_read_b32
+define amdgpu_kernel void @ds_append_lds_m0_restore(i32 addrspace(3)* %lds, i32 addrspace(1)* %out) #0 {
+  %val0 = call i32 @llvm.amdgcn.ds.append.p3i32(i32 addrspace(3)* %lds, i1 false)
+  store i32 %val0, i32 addrspace(1)* %out
+  %val1 = load volatile i32, i32 addrspace(3)* %lds
+  ret void
+}
+
+declare i32 @llvm.amdgcn.ds.append.p3i32(i32 addrspace(3)* nocapture, i1) #1
+declare i32 @llvm.amdgcn.ds.append.p2i32(i32 addrspace(2)* nocapture, i1) #1
+
+attributes #0 = { nounwind }
+attributes #1 = { argmemonly convergent nounwind }
diff --git a/test/CodeGen/AMDGPU/llvm.amdgcn.ds.consume.ll b/test/CodeGen/AMDGPU/llvm.amdgcn.ds.consume.ll
new file mode 100644
index 0000000000000000000000000000000000000000..e6cac3a32892e8d0e665f4e36f2e4230a66f6a7d
--- /dev/null
+++ b/test/CodeGen/AMDGPU/llvm.amdgcn.ds.consume.ll
@@ -0,0 +1,125 @@
+; RUN: llc -march=amdgcn -mcpu=tahiti -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SI,NOTGFX9 %s
+; RUN: llc -march=amdgcn -mcpu=bonaire -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,CIPLUS,NOTGFX9 %s
+; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,CIPLUS,NOTGFX9 %s
+; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,CIPLUS,GFX9 %s
+
+; GCN-LABEL: {{^}}ds_consume_lds:
+; GCN: s_load_dword [[PTR:s[0-9]+]]
+; GCN: s_mov_b32 m0, [[PTR]]
+; GCN: ds_consume [[RESULT:v[0-9]+]]{{$}}
+; GCN: {{.*}}store{{.*}} [[RESULT]]
+define amdgpu_kernel void @ds_consume_lds(i32 addrspace(3)* %lds, i32 addrspace(1)* %out) #0 {
+  %val = call i32 @llvm.amdgcn.ds.consume.p3i32(i32 addrspace(3)* %lds, i1 false)
+  store i32 %val, i32 addrspace(1)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}ds_consume_lds_max_offset:
+; GCN: s_load_dword [[PTR:s[0-9]+]]
+; GCN: s_mov_b32 m0, [[PTR]]
+; GCN: ds_consume [[RESULT:v[0-9]+]] offset:65532{{$}}
+; GCN: {{.*}}store{{.*}} [[RESULT]]
+define amdgpu_kernel void @ds_consume_lds_max_offset(i32 addrspace(3)* %lds, i32 addrspace(1)* %out) #0 {
+  %gep = getelementptr inbounds i32, i32 addrspace(3)* %lds, i32 16383
+  %val = call i32 @llvm.amdgcn.ds.consume.p3i32(i32 addrspace(3)* %gep, i1 false)
+  store i32 %val, i32 addrspace(1)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}ds_consume_no_fold_offset_si:
+; GCN: s_load_dword [[PTR:s[0-9]+]]
+
+; SI: s_add_i32 [[PTR]], [[PTR]], 16
+; SI: s_mov_b32 m0, [[PTR]]
+; SI: ds_consume [[RESULT:v[0-9]+]]{{$}}
+
+; CIPLUS: s_mov_b32 m0, [[PTR]]
+; CIPLUS: ds_consume [[RESULT:v[0-9]+]] offset:16{{$}}
+
+; GCN: {{.*}}store{{.*}} [[RESULT]]
+define amdgpu_kernel void @ds_consume_no_fold_offset_si(i32 addrspace(3)* addrspace(4)* %lds.ptr, i32 addrspace(1)* %out) #0 {
+  %lds = load i32 addrspace(3)*, i32 addrspace(3)* addrspace(4)* %lds.ptr, align 4
+  %gep = getelementptr inbounds i32, i32 addrspace(3)* %lds, i32 4
+  %val = call i32 @llvm.amdgcn.ds.consume.p3i32(i32 addrspace(3)* %gep, i1 false)
+  store i32 %val, i32 addrspace(1)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}ds_consume_lds_over_max_offset:
+; GCN: s_load_dword [[PTR:s[0-9]+]]
+
+; SI: s_bitset1_b32 [[PTR]], 16
+; CIPLUS: s_add_i32 [[PTR]], [[PTR]], 0x10000
+
+; GCN: s_mov_b32 m0, [[PTR]]
+; GCN: ds_consume [[RESULT:v[0-9]+]]{{$}}
+; GCN: {{.*}}store{{.*}} [[RESULT]]
+define amdgpu_kernel void @ds_consume_lds_over_max_offset(i32 addrspace(3)* %lds, i32 addrspace(1)* %out) #0 {
+  %gep = getelementptr inbounds i32, i32 addrspace(3)* %lds, i32 16384
+  %val = call i32 @llvm.amdgcn.ds.consume.p3i32(i32 addrspace(3)* %gep, i1 false)
+  store i32 %val, i32 addrspace(1)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}ds_consume_lds_vgpr_addr:
+; GCN: v_readfirstlane_b32 [[READLANE:s[0-9]+]], v0
+; GCN: s_mov_b32 m0, [[READLANE]]
+; GCN: ds_consume [[RESULT:v[0-9]+]]{{$}}
+; GCN: {{.*}}store{{.*}} [[RESULT]]
+define void @ds_consume_lds_vgpr_addr(i32 addrspace(3)* %lds, i32 addrspace(1)* %out) #0 {
+  %val = call i32 @llvm.amdgcn.ds.consume.p3i32(i32 addrspace(3)* %lds, i1 false)
+  store i32 %val, i32 addrspace(1)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}ds_consume_gds:
+; GCN: s_load_dword [[PTR:s[0-9]+]]
+; GCN: s_mov_b32 m0, [[PTR]]
+; GCN: ds_consume [[RESULT:v[0-9]+]] gds{{$}}
+; GCN: {{.*}}store{{.*}} [[RESULT]]
+define amdgpu_kernel void @ds_consume_gds(i32 addrspace(2)* %gds, i32 addrspace(1)* %out) #0 {
+  %val = call i32 @llvm.amdgcn.ds.consume.p2i32(i32 addrspace(2)* %gds, i1 false)
+  store i32 %val, i32 addrspace(1)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}ds_consume_gds_max_offset:
+; GCN: s_load_dword [[PTR:s[0-9]+]]
+; GCN: s_mov_b32 m0, [[PTR]]
+; GCN: ds_consume [[RESULT:v[0-9]+]] offset:65532 gds{{$}}
+; GCN: {{.*}}store{{.*}} [[RESULT]]
+define amdgpu_kernel void @ds_consume_gds_max_offset(i32 addrspace(2)* %gds, i32 addrspace(1)* %out) #0 {
+  %gep = getelementptr inbounds i32, i32 addrspace(2)* %gds, i32 16383
+  %val = call i32 @llvm.amdgcn.ds.consume.p2i32(i32 addrspace(2)* %gep, i1 false)
+  store i32 %val, i32 addrspace(1)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}ds_consume_gds_over_max_offset:
+define amdgpu_kernel void @ds_consume_gds_over_max_offset(i32 addrspace(2)* %gds, i32 addrspace(1)* %out) #0 {
+  %gep = getelementptr inbounds i32, i32 addrspace(2)* %gds, i32 16384
+  %val = call i32 @llvm.amdgcn.ds.consume.p2i32(i32 addrspace(2)* %gep, i1 false)
+  store i32 %val, i32 addrspace(1)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}ds_consume_lds_m0_restore:
+; GCN: s_load_dword [[PTR:s[0-9]+]]
+; GCN: s_mov_b32 m0, [[PTR]]
+; GCN: ds_consume [[RESULT:v[0-9]+]]{{$}}
+; NOTGFX9: s_mov_b32 m0, -1
+; GFX9-NOT: m0
+; GCN: _store_dword
+; GCN: ds_read_b32
+define amdgpu_kernel void @ds_consume_lds_m0_restore(i32 addrspace(3)* %lds, i32 addrspace(1)* %out) #0 {
+  %val0 = call i32 @llvm.amdgcn.ds.consume.p3i32(i32 addrspace(3)* %lds, i1 false)
+  store i32 %val0, i32 addrspace(1)* %out
+  %val1 = load volatile i32, i32 addrspace(3)* %lds
+  ret void
+}
+
+declare i32 @llvm.amdgcn.ds.consume.p3i32(i32 addrspace(3)* nocapture, i1) #1
+declare i32 @llvm.amdgcn.ds.consume.p2i32(i32 addrspace(2)* nocapture, i1) #1
+
+attributes #0 = { nounwind }
+attributes #1 = { argmemonly convergent nounwind }
diff --git a/test/CodeGen/AMDGPU/llvm.amdgcn.ds.ordered.add.ll b/test/CodeGen/AMDGPU/llvm.amdgcn.ds.ordered.add.ll
new file mode 100644
index 0000000000000000000000000000000000000000..ad489debc46cb14fdc1a1ee36d65eb46d9b1cd70
--- /dev/null
+++ b/test/CodeGen/AMDGPU/llvm.amdgcn.ds.ordered.add.ll
@@ -0,0 +1,96 @@
+; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,FUNC %s
+; RUN: llc -march=amdgcn -mcpu=bonaire -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,FUNC %s
+; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,VIGFX9,FUNC %s
+; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,VIGFX9,FUNC %s
+
+; FUNC-LABEL: {{^}}ds_ordered_add:
+; GCN-DAG: v_mov_b32_e32 v[[INCR:[0-9]+]], 31
+; GCN-DAG: s_mov_b32 m0,
+; GCN: ds_ordered_count v{{[0-9]+}}, v[[INCR]] offset:772 gds
+define amdgpu_kernel void @ds_ordered_add(i32 addrspace(2)* inreg %gds, i32 addrspace(1)* %out) {
+  %val = call i32@llvm.amdgcn.ds.ordered.add(i32 addrspace(2)* %gds, i32 31, i32 0, i32 0, i1 false, i32 1, i1 true, i1 true)
+  store i32 %val, i32 addrspace(1)* %out
+  ret void
+}
+
+; Below are various modifications of input operands and shader types.
+
+; FUNC-LABEL: {{^}}ds_ordered_add_counter2:
+; GCN-DAG: v_mov_b32_e32 v[[INCR:[0-9]+]], 31
+; GCN-DAG: s_mov_b32 m0,
+; GCN: ds_ordered_count v{{[0-9]+}}, v[[INCR]] offset:776 gds
+define amdgpu_kernel void @ds_ordered_add_counter2(i32 addrspace(2)* inreg %gds, i32 addrspace(1)* %out) {
+  %val = call i32@llvm.amdgcn.ds.ordered.add(i32 addrspace(2)* %gds, i32 31, i32 0, i32 0, i1 false, i32 2, i1 true, i1 true)
+  store i32 %val, i32 addrspace(1)* %out
+  ret void
+}
+
+; FUNC-LABEL: {{^}}ds_ordered_add_nodone:
+; GCN-DAG: v_mov_b32_e32 v[[INCR:[0-9]+]], 31
+; GCN-DAG: s_mov_b32 m0,
+; GCN: ds_ordered_count v{{[0-9]+}}, v[[INCR]] offset:260 gds
+define amdgpu_kernel void @ds_ordered_add_nodone(i32 addrspace(2)* inreg %gds, i32 addrspace(1)* %out) {
+  %val = call i32@llvm.amdgcn.ds.ordered.add(i32 addrspace(2)* %gds, i32 31, i32 0, i32 0, i1 false, i32 1, i1 true, i1 false)
+  store i32 %val, i32 addrspace(1)* %out
+  ret void
+}
+
+; FUNC-LABEL: {{^}}ds_ordered_add_norelease:
+; GCN-DAG: v_mov_b32_e32 v[[INCR:[0-9]+]], 31
+; GCN-DAG: s_mov_b32 m0,
+; GCN: ds_ordered_count v{{[0-9]+}}, v[[INCR]] offset:4 gds
+define amdgpu_kernel void @ds_ordered_add_norelease(i32 addrspace(2)* inreg %gds, i32 addrspace(1)* %out) {
+  %val = call i32@llvm.amdgcn.ds.ordered.add(i32 addrspace(2)* %gds, i32 31, i32 0, i32 0, i1 false, i32 1, i1 false, i1 false)
+  store i32 %val, i32 addrspace(1)* %out
+  ret void
+}
+
+; FUNC-LABEL: {{^}}ds_ordered_add_cs:
+; GCN: v_mov_b32_e32 v[[INCR:[0-9]+]], 31
+; GCN: s_mov_b32 m0, s0
+; VIGFX9-NEXT: s_nop 0
+; GCN-NEXT: ds_ordered_count v{{[0-9]+}}, v[[INCR]] offset:772 gds
+; GCN-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+define amdgpu_cs float @ds_ordered_add_cs(i32 addrspace(2)* inreg %gds) {
+  %val = call i32@llvm.amdgcn.ds.ordered.add(i32 addrspace(2)* %gds, i32 31, i32 0, i32 0, i1 false, i32 1, i1 true, i1 true)
+  %r = bitcast i32 %val to float
+  ret float %r
+}
+
+; FUNC-LABEL: {{^}}ds_ordered_add_ps:
+; GCN: v_mov_b32_e32 v[[INCR:[0-9]+]], 31
+; GCN: s_mov_b32 m0, s0
+; VIGFX9-NEXT: s_nop 0
+; GCN-NEXT: ds_ordered_count v{{[0-9]+}}, v[[INCR]] offset:1796 gds
+; GCN-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+define amdgpu_ps float @ds_ordered_add_ps(i32 addrspace(2)* inreg %gds) {
+  %val = call i32@llvm.amdgcn.ds.ordered.add(i32 addrspace(2)* %gds, i32 31, i32 0, i32 0, i1 false, i32 1, i1 true, i1 true)
+  %r = bitcast i32 %val to float
+  ret float %r
+}
+
+; FUNC-LABEL: {{^}}ds_ordered_add_vs:
+; GCN: v_mov_b32_e32 v[[INCR:[0-9]+]], 31
+; GCN: s_mov_b32 m0, s0
+; VIGFX9-NEXT: s_nop 0
+; GCN-NEXT: ds_ordered_count v{{[0-9]+}}, v[[INCR]] offset:2820 gds
+; GCN-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+define amdgpu_vs float @ds_ordered_add_vs(i32 addrspace(2)* inreg %gds) {
+  %val = call i32@llvm.amdgcn.ds.ordered.add(i32 addrspace(2)* %gds, i32 31, i32 0, i32 0, i1 false, i32 1, i1 true, i1 true)
+  %r = bitcast i32 %val to float
+  ret float %r
+}
+
+; FUNC-LABEL: {{^}}ds_ordered_add_gs:
+; GCN: v_mov_b32_e32 v[[INCR:[0-9]+]], 31
+; GCN: s_mov_b32 m0, s0
+; VIGFX9-NEXT: s_nop 0
+; GCN-NEXT: ds_ordered_count v{{[0-9]+}}, v[[INCR]] offset:3844 gds
+; GCN-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+define amdgpu_gs float @ds_ordered_add_gs(i32 addrspace(2)* inreg %gds) {
+  %val = call i32@llvm.amdgcn.ds.ordered.add(i32 addrspace(2)* %gds, i32 31, i32 0, i32 0, i1 false, i32 1, i1 true, i1 true)
+  %r = bitcast i32 %val to float
+  ret float %r
+}
+
+declare i32 @llvm.amdgcn.ds.ordered.add(i32 addrspace(2)* nocapture, i32, i32, i32, i1, i32, i1, i1)
diff --git a/test/CodeGen/AMDGPU/llvm.amdgcn.ds.ordered.swap.ll b/test/CodeGen/AMDGPU/llvm.amdgcn.ds.ordered.swap.ll
new file mode 100644
index 0000000000000000000000000000000000000000..acb1133c6a0b8bedb19b847bd135838886eff477
--- /dev/null
+++ b/test/CodeGen/AMDGPU/llvm.amdgcn.ds.ordered.swap.ll
@@ -0,0 +1,45 @@
+; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,FUNC %s
+; RUN: llc -march=amdgcn -mcpu=bonaire -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,FUNC %s
+; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,VIGFX9,FUNC %s
+; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,VIGFX9,FUNC %s
+
+; FUNC-LABEL: {{^}}ds_ordered_swap:
+; GCN: s_mov_b32 m0, s0
+; VIGFX9-NEXT: s_nop 0
+; GCN-NEXT: ds_ordered_count v{{[0-9]+}}, v0 offset:4868 gds
+; GCN-NEXT: s_waitcnt expcnt(0) lgkmcnt(0)
+define amdgpu_cs float @ds_ordered_swap(i32 addrspace(2)* inreg %gds, i32 %value) {
+  %val = call i32@llvm.amdgcn.ds.ordered.swap(i32 addrspace(2)* %gds, i32 %value, i32 0, i32 0, i1 false, i32 1, i1 true, i1 true)
+  %r = bitcast i32 %val to float
+  ret float %r
+}
+
+; FUNC-LABEL: {{^}}ds_ordered_swap_conditional:
+; GCN: v_cmp_ne_u32_e32 vcc, 0, v0
+; GCN: s_and_saveexec_b64 s[[SAVED:\[[0-9]+:[0-9]+\]]], vcc
+; // We have to use s_cbranch, because ds_ordered_count has side effects with EXEC=0
+; GCN: s_cbranch_execz [[BB:BB._.]]
+; GCN: s_mov_b32 m0, s0
+; VIGFX9-NEXT: s_nop 0
+; GCN-NEXT: ds_ordered_count v{{[0-9]+}}, v0 offset:4868 gds
+; GCN-NEXT: [[BB]]:
+; // Wait for expcnt(0) before modifying EXEC
+; GCN-NEXT: s_waitcnt expcnt(0)
+; GCN-NEXT: s_or_b64 exec, exec, s[[SAVED]]
+; GCN-NEXT: s_waitcnt lgkmcnt(0)
+define amdgpu_cs float @ds_ordered_swap_conditional(i32 addrspace(2)* inreg %gds, i32 %value) {
+entry:
+  %c = icmp ne i32 %value, 0
+  br i1 %c, label %if-true, label %endif
+
+if-true:
+  %val = call i32@llvm.amdgcn.ds.ordered.swap(i32 addrspace(2)* %gds, i32 %value, i32 0, i32 0, i1 false, i32 1, i1 true, i1 true)
+  br label %endif
+
+endif:
+  %v = phi i32 [ %val, %if-true ], [ undef, %entry ]
+  %r = bitcast i32 %v to float
+  ret float %r
+}
+
+declare i32 @llvm.amdgcn.ds.ordered.swap(i32 addrspace(2)* nocapture, i32, i32, i32, i1, i32, i1, i1)
diff --git a/test/CodeGen/AMDGPU/llvm.amdgcn.icmp.ll b/test/CodeGen/AMDGPU/llvm.amdgcn.icmp.ll
index 584fb43fd8e1bae340ead9fca939090ec6ea4186..06ac7da1288ae8ff496d79fc0033ab08bf9e2d93 100644
--- a/test/CodeGen/AMDGPU/llvm.amdgcn.icmp.ll
+++ b/test/CodeGen/AMDGPU/llvm.amdgcn.icmp.ll
@@ -4,6 +4,7 @@
 declare i64 @llvm.amdgcn.icmp.i32(i32, i32, i32) #0
 declare i64 @llvm.amdgcn.icmp.i64(i64, i64, i32) #0
 declare i64 @llvm.amdgcn.icmp.i16(i16, i16, i32) #0
+declare i64 @llvm.amdgcn.icmp.i1(i1, i1, i32) #0
 
 ; No crash on invalid input
 ; GCN-LABEL: {{^}}v_icmp_i32_dynamic_cc:
@@ -314,4 +315,21 @@ define amdgpu_kernel void @v_icmp_i16_sle(i64 addrspace(1)* %out, i16 %src) {
   ret void
 }
 
+; GCN-LABEL: {{^}}v_icmp_i1_ne0:
+; GCN: v_cmp_gt_u32_e64 s[[C0:\[[0-9]+:[0-9]+\]]],
+; GCN: v_cmp_gt_u32_e64 s[[C1:\[[0-9]+:[0-9]+\]]],
+; GCN: s_and_b64 s[[SRC:\[[0-9]+:[0-9]+\]]], s[[C0]], s[[C1]]
+; SI-NEXT: s_mov_b32 s{{[0-9]+}}, -1
+; GCN-NEXT: v_mov_b32_e32
+; GCN-NEXT: v_mov_b32_e32
+; GCN-NEXT: {{global|flat|buffer}}_store_dwordx2
+define amdgpu_kernel void @v_icmp_i1_ne0(i64 addrspace(1)* %out, i32 %a, i32 %b) {
+  %c0 = icmp ugt i32 %a, 1
+  %c1 = icmp ugt i32 %b, 2
+  %src = and i1 %c0, %c1
+  %result = call i64 @llvm.amdgcn.icmp.i1(i1 %src, i1 false, i32 33)
+  store i64 %result, i64 addrspace(1)* %out
+  ret void
+}
+
 attributes #0 = { nounwind readnone convergent }
diff --git a/test/CodeGen/AMDGPU/llvm.amdgcn.image.dim.ll b/test/CodeGen/AMDGPU/llvm.amdgcn.image.dim.ll
index bf93ffa937acbd715f2ceb67bca318a7074f9340..b297acab36c6fb1cf15334c058eb8769a1ab00a7 100644
--- a/test/CodeGen/AMDGPU/llvm.amdgcn.image.dim.ll
+++ b/test/CodeGen/AMDGPU/llvm.amdgcn.image.dim.ll
@@ -1,6 +1,7 @@
-; RUN: llc -march=amdgcn -mcpu=verde -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SI %s
-; RUN: llc -march=amdgcn -mcpu=fiji -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,VI %s
-; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN %s
+; RUN: llc -march=amdgcn -mcpu=verde -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SI,SIVI,PRT %s
+; RUN: llc -march=amdgcn -mcpu=fiji  -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,VI,SIVI,PRT %s
+; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX900,PRT %s
+; RUN: llc -march=amdgcn -mcpu=gfx900 -mattr=-enable-prt-strict-null -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX900,NOPRT %s
 
 ; GCN-LABEL: {{^}}load_1d:
 ; GCN: image_load v[0:3], v0, s[0:7] dmask:0xf unorm{{$}}
@@ -10,6 +11,52 @@ main_body:
   ret <4 x float> %v
 }
 
+; GCN-LABEL: {{^}}load_1d_tfe:
+; PRT: v_mov_b32_e32 v0, 0
+; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0
+; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0
+; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0
+; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0
+; NOPRT: v_mov_b32_e32 v4, 0
+; NOPRT-NOT: v_mov_b32_e32 v0
+; NOPRT-NOT: v_mov_b32_e32 v1
+; NOPRT-NOT: v_mov_b32_e32 v2
+; NOPRT-NOT: v_mov_b32_e32 v3
+; GCN: image_load v[0:7], v{{[0-9]+}}, s[0:7] dmask:0xf unorm tfe{{$}}
+; SIVI: buffer_store_dword v4, off, s[8:11], 0
+; GFX900: global_store_dword v[{{[0-9]+:[0-9]+}}], v4
+define amdgpu_ps <4 x float> @load_1d_tfe(<8 x i32> inreg %rsrc, i32 addrspace(1)* inreg %out, i32 %s) {
+main_body:
+  %v = call {<4 x float>,i32} @llvm.amdgcn.image.load.1d.v4f32i32.i32(i32 15, i32 %s, <8 x i32> %rsrc, i32 1, i32 0)
+  %v.vec = extractvalue {<4 x float>, i32} %v, 0
+  %v.err = extractvalue {<4 x float>, i32} %v, 1
+  store i32 %v.err, i32 addrspace(1)* %out, align 4
+  ret <4 x float> %v.vec
+}
+
+; GCN-LABEL: {{^}}load_1d_lwe:
+; PRT: v_mov_b32_e32 v0, 0
+; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0
+; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0
+; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0
+; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0
+; NOPRT: v_mov_b32_e32 v4, 0
+; NOPRT-NOT: v_mov_b32_e32 v0
+; NOPRT-NOT: v_mov_b32_e32 v1
+; NOPRT-NOT: v_mov_b32_e32 v2
+; NOPRT-NOT: v_mov_b32_e32 v3
+; GCN: image_load v[0:7], v{{[0-9]+}}, s[0:7] dmask:0xf unorm lwe{{$}}
+; SIVI: buffer_store_dword v4, off, s[8:11], 0
+; GFX900: global_store_dword v[{{[0-9]+:[0-9]+}}], v4
+define amdgpu_ps <4 x float> @load_1d_lwe(<8 x i32> inreg %rsrc, i32 addrspace(1)* inreg %out, i32 %s) {
+main_body:
+  %v = call {<4 x float>, i32} @llvm.amdgcn.image.load.1d.v4f32i32.i32(i32 15, i32 %s, <8 x i32> %rsrc, i32 2, i32 0)
+  %v.vec = extractvalue {<4 x float>, i32} %v, 0
+  %v.err = extractvalue {<4 x float>, i32} %v, 1
+  store i32 %v.err, i32 addrspace(1)* %out, align 4
+  ret <4 x float> %v.vec
+}
+
 ; GCN-LABEL: {{^}}load_2d:
 ; GCN: image_load v[0:3], v[0:1], s[0:7] dmask:0xf unorm{{$}}
 define amdgpu_ps <4 x float> @load_2d(<8 x i32> inreg %rsrc, i32 %s, i32 %t) {
@@ -18,6 +65,29 @@ main_body:
   ret <4 x float> %v
 }
 
+; GCN-LABEL: {{^}}load_2d_tfe:
+; PRT: v_mov_b32_e32 v0, 0
+; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0
+; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0
+; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0
+; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0
+; NOPRT: v_mov_b32_e32 v4, 0
+; NOPRT-NOT: v_mov_b32_e32 v0
+; NOPRT-NOT: v_mov_b32_e32 v1
+; NOPRT-NOT: v_mov_b32_e32 v2
+; NOPRT-NOT: v_mov_b32_e32 v3
+; GCN: image_load v[0:7], v[{{[0-9]+:[0-9]+}}], s[0:7] dmask:0xf unorm tfe{{$}}
+; SIVI: buffer_store_dword v4, off, s[8:11], 0
+; GFX900: global_store_dword v[{{[0-9]+:[0-9]+}}], v4
+define amdgpu_ps <4 x float> @load_2d_tfe(<8 x i32> inreg %rsrc, i32 addrspace(1)* inreg %out, i32 %s, i32 %t) {
+main_body:
+  %v = call {<4 x float>,i32} @llvm.amdgcn.image.load.2d.v4f32i32.i32(i32 15, i32 %s, i32 %t, <8 x i32> %rsrc, i32 1, i32 0)
+  %v.vec = extractvalue {<4 x float>, i32} %v, 0
+  %v.err = extractvalue {<4 x float>, i32} %v, 1
+  store i32 %v.err, i32 addrspace(1)* %out, align 4
+  ret <4 x float> %v.vec
+}
+
 ; GCN-LABEL: {{^}}load_3d:
 ; GCN: image_load v[0:3], v[0:3], s[0:7] dmask:0xf unorm{{$}}
 define amdgpu_ps <4 x float> @load_3d(<8 x i32> inreg %rsrc, i32 %s, i32 %t, i32 %r) {
@@ -26,6 +96,29 @@ main_body:
   ret <4 x float> %v
 }
 
+; GCN-LABEL: {{^}}load_3d_tfe_lwe:
+; PRT: v_mov_b32_e32 v0, 0
+; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0
+; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0
+; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0
+; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0
+; NOPRT: v_mov_b32_e32 v4, 0
+; NOPRT-NOT: v_mov_b32_e32 v0
+; NOPRT-NOT: v_mov_b32_e32 v1
+; NOPRT-NOT: v_mov_b32_e32 v2
+; NOPRT-NOT: v_mov_b32_e32 v3
+; GCN: image_load v[0:7], v[{{[0-9]+:[0-9]+}}], s[0:7] dmask:0xf unorm tfe lwe{{$}}
+; SIVI: buffer_store_dword v4, off, s[8:11], 0
+; GFX900: global_store_dword v[{{[0-9]+:[0-9]+}}], v4
+define amdgpu_ps <4 x float> @load_3d_tfe_lwe(<8 x i32> inreg %rsrc, i32 addrspace(1)* inreg %out, i32 %s, i32 %t, i32 %r) {
+main_body:
+  %v = call {<4 x float>,i32} @llvm.amdgcn.image.load.3d.v4f32i32.i32(i32 15, i32 %s, i32 %t, i32 %r, <8 x i32> %rsrc, i32 3, i32 0)
+  %v.vec = extractvalue {<4 x float>, i32} %v, 0
+  %v.err = extractvalue {<4 x float>, i32} %v, 1
+  store i32 %v.err, i32 addrspace(1)* %out, align 4
+  ret <4 x float> %v.vec
+}
+
 ; GCN-LABEL: {{^}}load_cube:
 ; GCN: image_load v[0:3], v[0:3], s[0:7] dmask:0xf unorm da{{$}}
 define amdgpu_ps <4 x float> @load_cube(<8 x i32> inreg %rsrc, i32 %s, i32 %t, i32 %slice) {
@@ -34,6 +127,29 @@ main_body:
   ret <4 x float> %v
 }
 
+; GCN-LABEL: {{^}}load_cube_lwe:
+; PRT: v_mov_b32_e32 v0, 0
+; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0
+; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0
+; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0
+; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0
+; NOPRT: v_mov_b32_e32 v4, 0
+; NOPRT-NOT: v_mov_b32_e32 v0
+; NOPRT-NOT: v_mov_b32_e32 v1
+; NOPRT-NOT: v_mov_b32_e32 v2
+; NOPRT-NOT: v_mov_b32_e32 v3
+; GCN: image_load v[0:7], v[{{[0-9]+:[0-9]+}}], s[0:7] dmask:0xf unorm lwe da{{$}}
+; SIVI: buffer_store_dword v4, off, s[8:11], 0
+; GFX900: global_store_dword v[{{[0-9]+:[0-9]+}}], v4
+define amdgpu_ps <4 x float> @load_cube_lwe(<8 x i32> inreg %rsrc, i32 addrspace(1)* inreg %out, i32 %s, i32 %t, i32 %slice) {
+main_body:
+  %v = call {<4 x float>,i32} @llvm.amdgcn.image.load.cube.v4f32i32.i32(i32 15, i32 %s, i32 %t, i32 %slice, <8 x i32> %rsrc, i32 2, i32 0)
+  %v.vec = extractvalue {<4 x float>, i32} %v, 0
+  %v.err = extractvalue {<4 x float>, i32} %v, 1
+  store i32 %v.err, i32 addrspace(1)* %out, align 4
+  ret <4 x float> %v.vec
+}
+
 ; GCN-LABEL: {{^}}load_1darray:
 ; GCN: image_load v[0:3], v[0:1], s[0:7] dmask:0xf unorm da{{$}}
 define amdgpu_ps <4 x float> @load_1darray(<8 x i32> inreg %rsrc, i32 %s, i32 %slice) {
@@ -42,6 +158,29 @@ main_body:
   ret <4 x float> %v
 }
 
+; GCN-LABEL: {{^}}load_1darray_tfe:
+; PRT: v_mov_b32_e32 v0, 0
+; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0
+; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0
+; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0
+; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0
+; NOPRT: v_mov_b32_e32 v4, 0
+; NOPRT-NOT: v_mov_b32_e32 v0
+; NOPRT-NOT: v_mov_b32_e32 v1
+; NOPRT-NOT: v_mov_b32_e32 v2
+; NOPRT-NOT: v_mov_b32_e32 v3
+; GCN: image_load v[0:7], v[{{[0-9]+:[0-9]+}}], s[0:7] dmask:0xf unorm tfe da{{$}}
+; SIVI: buffer_store_dword v4, off, s[8:11], 0
+; GFX900: global_store_dword v[{{[0-9]+:[0-9]+}}], v4
+define amdgpu_ps <4 x float> @load_1darray_tfe(<8 x i32> inreg %rsrc, i32 addrspace(1)* inreg %out, i32 %s, i32 %slice) {
+main_body:
+  %v = call {<4 x float>,i32} @llvm.amdgcn.image.load.1darray.v4f32i32.i32(i32 15, i32 %s, i32 %slice, <8 x i32> %rsrc, i32 1, i32 0)
+  %v.vec = extractvalue {<4 x float>, i32} %v, 0
+  %v.err = extractvalue {<4 x float>, i32} %v, 1
+  store i32 %v.err, i32 addrspace(1)* %out, align 4
+  ret <4 x float> %v.vec
+}
+
 ; GCN-LABEL: {{^}}load_2darray:
 ; GCN: image_load v[0:3], v[0:3], s[0:7] dmask:0xf unorm da{{$}}
 define amdgpu_ps <4 x float> @load_2darray(<8 x i32> inreg %rsrc, i32 %s, i32 %t, i32 %slice) {
@@ -50,6 +189,29 @@ main_body:
   ret <4 x float> %v
 }
 
+; GCN-LABEL: {{^}}load_2darray_lwe:
+; PRT: v_mov_b32_e32 v0, 0
+; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0
+; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0
+; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0
+; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0
+; NOPRT: v_mov_b32_e32 v4, 0
+; NOPRT-NOT: v_mov_b32_e32 v0
+; NOPRT-NOT: v_mov_b32_e32 v1
+; NOPRT-NOT: v_mov_b32_e32 v2
+; NOPRT-NOT: v_mov_b32_e32 v3
+; GCN: image_load v[0:7], v[{{[0-9]+:[0-9]+}}], s[0:7] dmask:0xf unorm lwe da{{$}}
+; SIVI: buffer_store_dword v4, off, s[8:11], 0
+; GFX900: global_store_dword v[{{[0-9]+:[0-9]+}}], v4
+define amdgpu_ps <4 x float> @load_2darray_lwe(<8 x i32> inreg %rsrc, i32 addrspace(1)* inreg %out, i32 %s, i32 %t, i32 %slice) {
+main_body:
+  %v = call {<4 x float>,i32} @llvm.amdgcn.image.load.2darray.v4f32i32.i32(i32 15, i32 %s, i32 %t, i32 %slice, <8 x i32> %rsrc, i32 2, i32 0)
+  %v.vec = extractvalue {<4 x float>, i32} %v, 0
+  %v.err = extractvalue {<4 x float>, i32} %v, 1
+  store i32 %v.err, i32 addrspace(1)* %out, align 4
+  ret <4 x float> %v.vec
+}
+
 ; GCN-LABEL: {{^}}load_2dmsaa:
 ; GCN: image_load v[0:3], v[0:3], s[0:7] dmask:0xf unorm{{$}}
 define amdgpu_ps <4 x float> @load_2dmsaa(<8 x i32> inreg %rsrc, i32 %s, i32 %t, i32 %fragid) {
@@ -58,6 +220,29 @@ main_body:
   ret <4 x float> %v
 }
 
+; GCN-LABEL: {{^}}load_2dmsaa_both:
+; PRT: v_mov_b32_e32 v0, 0
+; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0
+; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0
+; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0
+; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0
+; NOPRT: v_mov_b32_e32 v4, 0
+; NOPRT-NOT: v_mov_b32_e32 v0
+; NOPRT-NOT: v_mov_b32_e32 v1
+; NOPRT-NOT: v_mov_b32_e32 v2
+; NOPRT-NOT: v_mov_b32_e32 v3
+; GCN: image_load v[0:7], v[{{[0-9]+:[0-9]+}}], s[0:7] dmask:0xf unorm tfe lwe{{$}}
+; SIVI: buffer_store_dword v4, off, s[8:11], 0
+; GFX900: global_store_dword v[{{[0-9]+:[0-9]+}}], v4
+define amdgpu_ps <4 x float> @load_2dmsaa_both(<8 x i32> inreg %rsrc, i32 addrspace(1)* inreg %out, i32 %s, i32 %t, i32 %fragid) {
+main_body:
+  %v = call {<4 x float>,i32} @llvm.amdgcn.image.load.2dmsaa.v4f32i32.i32(i32 15, i32 %s, i32 %t, i32 %fragid, <8 x i32> %rsrc, i32 3, i32 0)
+  %v.vec = extractvalue {<4 x float>, i32} %v, 0
+  %v.err = extractvalue {<4 x float>, i32} %v, 1
+  store i32 %v.err, i32 addrspace(1)* %out, align 4
+  ret <4 x float> %v.vec
+}
+
 ; GCN-LABEL: {{^}}load_2darraymsaa:
 ; GCN: image_load v[0:3], v[0:3], s[0:7] dmask:0xf unorm da{{$}}
 define amdgpu_ps <4 x float> @load_2darraymsaa(<8 x i32> inreg %rsrc, i32 %s, i32 %t, i32 %slice, i32 %fragid) {
@@ -66,6 +251,29 @@ main_body:
   ret <4 x float> %v
 }
 
+; GCN-LABEL: {{^}}load_2darraymsaa_tfe:
+; PRT: v_mov_b32_e32 v0, 0
+; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0
+; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0
+; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0
+; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0
+; NOPRT: v_mov_b32_e32 v4, 0
+; NOPRT-NOT: v_mov_b32_e32 v0
+; NOPRT-NOT: v_mov_b32_e32 v1
+; NOPRT-NOT: v_mov_b32_e32 v2
+; NOPRT-NOT: v_mov_b32_e32 v3
+; GCN: image_load v[0:7], v[{{[0-9]+:[0-9]+}}], s[0:7] dmask:0xf unorm tfe da{{$}}
+; SIVI: buffer_store_dword v4, off, s[8:11], 0
+; GFX900: global_store_dword v[{{[0-9]+:[0-9]+}}], v4
+define amdgpu_ps <4 x float> @load_2darraymsaa_tfe(<8 x i32> inreg %rsrc, i32 addrspace(1)* inreg %out, i32 %s, i32 %t, i32 %slice, i32 %fragid) {
+main_body:
+  %v = call {<4 x float>,i32} @llvm.amdgcn.image.load.2darraymsaa.v4f32i32.i32(i32 15, i32 %s, i32 %t, i32 %slice, i32 %fragid, <8 x i32> %rsrc, i32 1, i32 0)
+  %v.vec = extractvalue {<4 x float>, i32} %v, 0
+  %v.err = extractvalue {<4 x float>, i32} %v, 1
+  store i32 %v.err, i32 addrspace(1)* %out, align 4
+  ret <4 x float> %v.vec
+}
+
 ; GCN-LABEL: {{^}}load_mip_1d:
 ; GCN: image_load_mip v[0:3], v[0:1], s[0:7] dmask:0xf unorm{{$}}
 define amdgpu_ps <4 x float> @load_mip_1d(<8 x i32> inreg %rsrc, i32 %s, i32 %mip) {
@@ -74,6 +282,29 @@ main_body:
   ret <4 x float> %v
 }
 
+; GCN-LABEL: {{^}}load_mip_1d_lwe:
+; PRT: v_mov_b32_e32 v0, 0
+; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0
+; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0
+; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0
+; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0
+; NOPRT: v_mov_b32_e32 v4, 0
+; NOPRT-NOT: v_mov_b32_e32 v0
+; NOPRT-NOT: v_mov_b32_e32 v1
+; NOPRT-NOT: v_mov_b32_e32 v2
+; NOPRT-NOT: v_mov_b32_e32 v3
+; GCN: image_load_mip v[0:7], v[{{[0-9]+:[0-9]+}}], s[0:7] dmask:0xf unorm lwe{{$}}
+; SIVI: buffer_store_dword v4, off, s[8:11], 0
+; GFX900: global_store_dword v[{{[0-9]+:[0-9]+}}], v4
+define amdgpu_ps <4 x float> @load_mip_1d_lwe(<8 x i32> inreg %rsrc, i32 addrspace(1)* inreg %out, i32 %s, i32 %mip) {
+main_body:
+  %v = call {<4 x float>,i32} @llvm.amdgcn.image.load.mip.1d.v4f32i32.i32(i32 15, i32 %s, i32 %mip, <8 x i32> %rsrc, i32 2, i32 0)
+  %v.vec = extractvalue {<4 x float>, i32} %v, 0
+  %v.err = extractvalue {<4 x float>, i32} %v, 1
+  store i32 %v.err, i32 addrspace(1)* %out, align 4
+  ret <4 x float> %v.vec
+}
+
 ; GCN-LABEL: {{^}}load_mip_2d:
 ; GCN: image_load_mip v[0:3], v[0:3], s[0:7] dmask:0xf unorm{{$}}
 define amdgpu_ps <4 x float> @load_mip_2d(<8 x i32> inreg %rsrc, i32 %s, i32 %t, i32 %mip) {
@@ -82,6 +313,191 @@ main_body:
   ret <4 x float> %v
 }
 
+; GCN-LABEL: {{^}}load_mip_2d_tfe:
+; PRT: v_mov_b32_e32 v0, 0
+; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0
+; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0
+; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0
+; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0
+; NOPRT: v_mov_b32_e32 v4, 0
+; NOPRT-NOT: v_mov_b32_e32 v0
+; NOPRT-NOT: v_mov_b32_e32 v1
+; NOPRT-NOT: v_mov_b32_e32 v2
+; NOPRT-NOT: v_mov_b32_e32 v3
+; GCN: image_load_mip v[0:7], v[{{[0-9]+:[0-9]+}}], s[0:7] dmask:0xf unorm tfe{{$}}
+; SIVI: buffer_store_dword v4, off, s[8:11], 0
+; GFX900: global_store_dword v[{{[0-9]+:[0-9]+}}], v4
+define amdgpu_ps <4 x float> @load_mip_2d_tfe(<8 x i32> inreg %rsrc, i32 addrspace(1)* inreg %out, i32 %s, i32 %t, i32 %mip) {
+main_body:
+  %v = call {<4 x float>,i32} @llvm.amdgcn.image.load.mip.2d.v4f32i32.i32(i32 15, i32 %s, i32 %t, i32 %mip, <8 x i32> %rsrc, i32 1, i32 0)
+  %v.vec = extractvalue {<4 x float>, i32} %v, 0
+  %v.err = extractvalue {<4 x float>, i32} %v, 1
+  store i32 %v.err, i32 addrspace(1)* %out, align 4
+  ret <4 x float> %v.vec
+}
+
+; Make sure that error flag is returned even with dmask 0
+; GCN-LABEL: {{^}}load_1d_V2_tfe_dmask0:
+; GCN: v_mov_b32_e32 v1, 0
+; PRT-DAG: v_mov_b32_e32 v2, v1
+; PRT: image_load v[1:2], v0, s[0:7] dmask:0x1 unorm tfe{{$}}
+; NOPRT-NOT: v_mov_b32_e32 v1
+; NOPRT: image_load v[0:1], v0, s[0:7] dmask:0x1 unorm tfe{{$}}
+define amdgpu_ps float @load_1d_V2_tfe_dmask0(<8 x i32> inreg %rsrc, i32 %s) {
+main_body:
+  %v = call {<2 x float>,i32} @llvm.amdgcn.image.load.1d.v2f32i32.i32(i32 0, i32 %s, <8 x i32> %rsrc, i32 1, i32 0)
+  %v.err = extractvalue {<2 x float>, i32} %v, 1
+  %vv = bitcast i32 %v.err to float
+  ret float %vv
+}
+
+; GCN-LABEL: {{^}}load_1d_V1_tfe_dmask0:
+; GCN: v_mov_b32_e32 v1, 0
+; PRT-DAG: v_mov_b32_e32 v2, v1
+; PRT: image_load v[1:2], v0, s[0:7] dmask:0x1 unorm tfe{{$}}
+; NOPRT-NOT: v_mov_b32_e32 v1
+; NOPRT: image_load v[0:1], v0, s[0:7] dmask:0x1 unorm tfe{{$}}
+define amdgpu_ps float @load_1d_V1_tfe_dmask0(<8 x i32> inreg %rsrc, i32 %s) {
+main_body:
+  %v = call {float,i32} @llvm.amdgcn.image.load.1d.f32i32.i32(i32 0, i32 %s, <8 x i32> %rsrc, i32 1, i32 0)
+  %v.err = extractvalue {float, i32} %v, 1
+  %vv = bitcast i32 %v.err to float
+  ret float %vv
+}
+
+; GCN-LABEL: {{^}}load_mip_2d_tfe_dmask0:
+; GCN: v_mov_b32_e32 v3, 0
+; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v3
+; PRT: image_load_mip v[3:4], v[{{[0-9]+:[0-9]+}}], s[0:7] dmask:0x1 unorm tfe{{$}}
+; NOPRT-NOT: v_mov_b32_e32 v2
+; NOPRT: image_load_mip v[2:3], v[{{[0-9]+:[0-9]+}}], s[0:7] dmask:0x1 unorm tfe{{$}}
+define amdgpu_ps float @load_mip_2d_tfe_dmask0(<8 x i32> inreg %rsrc, i32 %s, i32 %t, i32 %mip) {
+main_body:
+  %v = call {<4 x float>,i32} @llvm.amdgcn.image.load.mip.2d.v4f32i32.i32(i32 0, i32 %s, i32 %t, i32 %mip, <8 x i32> %rsrc, i32 1, i32 0)
+  %v.err = extractvalue {<4 x float>, i32} %v, 1
+  %vv = bitcast i32 %v.err to float
+  ret float %vv
+}
+
+; Do not make dmask 0 even if no result (other than tfe) is used.
+; GCN-LABEL: {{^}}load_mip_2d_tfe_nouse:
+; GCN: v_mov_b32_e32 v3, 0
+; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v3
+; PRT: image_load_mip v[3:4], v[{{[0-9]+:[0-9]+}}], s[0:7] dmask:0x1 unorm tfe{{$}}
+; NOPRT-NOT: v_mov_b32_e32 v2
+; NOPRT: image_load_mip v[2:3], v[{{[0-9]+:[0-9]+}}], s[0:7] dmask:0x1 unorm tfe{{$}}
+define amdgpu_ps float @load_mip_2d_tfe_nouse(<8 x i32> inreg %rsrc, i32 %s, i32 %t, i32 %mip) {
+main_body:
+  %v = call {<4 x float>,i32} @llvm.amdgcn.image.load.mip.2d.v4f32i32.i32(i32 15, i32 %s, i32 %t, i32 %mip, <8 x i32> %rsrc, i32 1, i32 0)
+  %v.err = extractvalue {<4 x float>, i32} %v, 1
+  %vv = bitcast i32 %v.err to float
+  ret float %vv
+}
+
+; GCN-LABEL: {{^}}load_mip_2d_tfe_nouse_V2:
+; GCN: v_mov_b32_e32 v3, 0
+; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v3
+; PRT: image_load_mip v[3:4], v[{{[0-9]+:[0-9]+}}], s[0:7] dmask:0x1 unorm tfe{{$}}
+; NOPRT-NOT: v_mov_b32_e32 v2
+; NOPRT: image_load_mip v[2:3], v[{{[0-9]+:[0-9]+}}], s[0:7] dmask:0x1 unorm tfe{{$}}
+define amdgpu_ps float @load_mip_2d_tfe_nouse_V2(<8 x i32> inreg %rsrc, i32 %s, i32 %t, i32 %mip) {
+main_body:
+  %v = call {<2 x float>,i32} @llvm.amdgcn.image.load.mip.2d.v2f32i32.i32(i32 6, i32 %s, i32 %t, i32 %mip, <8 x i32> %rsrc, i32 1, i32 0)
+  %v.err = extractvalue {<2 x float>, i32} %v, 1
+  %vv = bitcast i32 %v.err to float
+  ret float %vv
+}
+
+; GCN-LABEL: {{^}}load_mip_2d_tfe_nouse_V1:
+; GCN: v_mov_b32_e32 v3, 0
+; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v3
+; PRT: image_load_mip v[3:4], v[{{[0-9]+:[0-9]+}}], s[0:7] dmask:0x2 unorm tfe{{$}}
+; NOPRT-NOT: v_mov_b32_e32 v2
+; NOPRT: image_load_mip v[2:3], v[{{[0-9]+:[0-9]+}}], s[0:7] dmask:0x2 unorm tfe{{$}}
+define amdgpu_ps float @load_mip_2d_tfe_nouse_V1(<8 x i32> inreg %rsrc, i32 %s, i32 %t, i32 %mip) {
+main_body:
+  %v = call {float, i32} @llvm.amdgcn.image.load.mip.2d.f32i32.i32(i32 2, i32 %s, i32 %t, i32 %mip, <8 x i32> %rsrc, i32 1, i32 0)
+  %v.err = extractvalue {float, i32} %v, 1
+  %vv = bitcast i32 %v.err to float
+  ret float %vv
+}
+
+; Check for dmask being materially smaller than return type
+; GCN-LABEL: {{^}}load_1d_tfe_V4_dmask3:
+; PRT: v_mov_b32_e32 v0, 0
+; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0
+; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0
+; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0
+; NOPRT: v_mov_b32_e32 v3, 0
+; NOPRT-NOT: v_mov_b32_e32 v0
+; NOPRT-NOT: v_mov_b32_e32 v1
+; NOPRT-NOT: v_mov_b32_e32 v2
+; GCN: image_load v[0:3], v{{[0-9]+}}, s[0:7] dmask:0x7 unorm tfe{{$}}
+; SIVI: buffer_store_dword v3, off, s[8:11], 0
+; GFX900: global_store_dword v[{{[0-9]+:[0-9]+}}], v3
+define amdgpu_ps <4 x float> @load_1d_tfe_V4_dmask3(<8 x i32> inreg %rsrc, i32 addrspace(1)* inreg %out, i32 %s) {
+main_body:
+  %v = call {<4 x float>,i32} @llvm.amdgcn.image.load.1d.v4f32i32.i32(i32 7, i32 %s, <8 x i32> %rsrc, i32 1, i32 0)
+  %v.vec = extractvalue {<4 x float>, i32} %v, 0
+  %v.err = extractvalue {<4 x float>, i32} %v, 1
+  store i32 %v.err, i32 addrspace(1)* %out, align 4
+  ret <4 x float> %v.vec
+}
+
+; GCN-LABEL: {{^}}load_1d_tfe_V4_dmask2:
+; PRT: v_mov_b32_e32 v0, 0
+; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0
+; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0
+; NOPRT: v_mov_b32_e32 v2, 0
+; NOPRT-NOT: v_mov_b32_e32 v0
+; NOPRT-NOT: v_mov_b32_e32 v1
+; GCN: image_load v[0:3], v{{[0-9]+}}, s[0:7] dmask:0x6 unorm tfe{{$}}
+; SIVI: buffer_store_dword v2, off, s[8:11], 0
+; GFX900: global_store_dword v[{{[0-9]+:[0-9]+}}], v2
+define amdgpu_ps <4 x float> @load_1d_tfe_V4_dmask2(<8 x i32> inreg %rsrc, i32 addrspace(1)* inreg %out, i32 %s) {
+main_body:
+  %v = call {<4 x float>,i32} @llvm.amdgcn.image.load.1d.v4f32i32.i32(i32 6, i32 %s, <8 x i32> %rsrc, i32 1, i32 0)
+  %v.vec = extractvalue {<4 x float>, i32} %v, 0
+  %v.err = extractvalue {<4 x float>, i32} %v, 1
+  store i32 %v.err, i32 addrspace(1)* %out, align 4
+  ret <4 x float> %v.vec
+}
+
+; GCN-LABEL: {{^}}load_1d_tfe_V4_dmask1:
+; PRT: v_mov_b32_e32 v0, 0
+; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0
+; NOPRT: v_mov_b32_e32 v1, 0
+; NOPRT-NOT: v_mov_b32_e32 v0
+; GCN: image_load v[0:1], v{{[0-9]+}}, s[0:7] dmask:0x8 unorm tfe{{$}}
+; SIVI: buffer_store_dword v1, off, s[8:11], 0
+; GFX900: global_store_dword v[{{[0-9]+:[0-9]+}}], v1
+define amdgpu_ps <4 x float> @load_1d_tfe_V4_dmask1(<8 x i32> inreg %rsrc, i32 addrspace(1)* inreg %out, i32 %s) {
+main_body:
+  %v = call {<4 x float>,i32} @llvm.amdgcn.image.load.1d.v4f32i32.i32(i32 8, i32 %s, <8 x i32> %rsrc, i32 1, i32 0)
+  %v.vec = extractvalue {<4 x float>, i32} %v, 0
+  %v.err = extractvalue {<4 x float>, i32} %v, 1
+  store i32 %v.err, i32 addrspace(1)* %out, align 4
+  ret <4 x float> %v.vec
+}
+
+; GCN-LABEL: {{^}}load_1d_tfe_V2_dmask1:
+; PRT: v_mov_b32_e32 v0, 0
+; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0
+; NOPRT: v_mov_b32_e32 v1, 0
+; NOPRT-NOT: v_mov_b32_e32 v0
+; GCN: image_load v[0:1], v{{[0-9]+}}, s[0:7] dmask:0x8 unorm tfe{{$}}
+; SIVI: buffer_store_dword v1, off, s[8:11], 0
+; GFX900: global_store_dword v[{{[0-9]+:[0-9]+}}], v1
+define amdgpu_ps <2 x float> @load_1d_tfe_V2_dmask1(<8 x i32> inreg %rsrc, i32 addrspace(1)* inreg %out, i32 %s) {
+main_body:
+  %v = call {<2 x float>,i32} @llvm.amdgcn.image.load.1d.v2f32i32.i32(i32 8, i32 %s, <8 x i32> %rsrc, i32 1, i32 0)
+  %v.vec = extractvalue {<2 x float>, i32} %v, 0
+  %v.err = extractvalue {<2 x float>, i32} %v, 1
+  store i32 %v.err, i32 addrspace(1)* %out, align 4
+  ret <2 x float> %v.vec
+}
+
+
 ; GCN-LABEL: {{^}}load_mip_3d:
 ; GCN: image_load_mip v[0:3], v[0:3], s[0:7] dmask:0xf unorm{{$}}
 define amdgpu_ps <4 x float> @load_mip_3d(<8 x i32> inreg %rsrc, i32 %s, i32 %t, i32 %r, i32 %mip) {
@@ -404,23 +820,37 @@ define amdgpu_ps float @image_load_mmo(<8 x i32> inreg %rsrc, float addrspace(3)
   store float 0.000000e+00, float addrspace(3)* %lds
   %c0 = extractelement <2 x i32> %c, i32 0
   %c1 = extractelement <2 x i32> %c, i32 1
-  %tex = call float @llvm.amdgcn.image.load.2d.f32.i32(i32 15, i32 %c0, i32 %c1, <8 x i32> %rsrc, i32 0, i32 0)
+  %tex = call float @llvm.amdgcn.image.load.2d.f32.i32(i32 1, i32 %c0, i32 %c1, <8 x i32> %rsrc, i32 0, i32 0)
   %tmp2 = getelementptr float, float addrspace(3)* %lds, i32 4
   store float 0.000000e+00, float addrspace(3)* %tmp2
   ret float %tex
 }
 
 declare <4 x float> @llvm.amdgcn.image.load.1d.v4f32.i32(i32, i32, <8 x i32>, i32, i32) #1
+declare {float,i32} @llvm.amdgcn.image.load.1d.f32i32.i32(i32, i32, <8 x i32>, i32, i32) #1
+declare {<2 x float>,i32} @llvm.amdgcn.image.load.1d.v2f32i32.i32(i32, i32, <8 x i32>, i32, i32) #1
+declare {<4 x float>,i32} @llvm.amdgcn.image.load.1d.v4f32i32.i32(i32, i32, <8 x i32>, i32, i32) #1
 declare <4 x float> @llvm.amdgcn.image.load.2d.v4f32.i32(i32, i32, i32, <8 x i32>, i32, i32) #1
+declare {<4 x float>,i32} @llvm.amdgcn.image.load.2d.v4f32i32.i32(i32, i32, i32, <8 x i32>, i32, i32) #1
 declare <4 x float> @llvm.amdgcn.image.load.3d.v4f32.i32(i32, i32, i32, i32, <8 x i32>, i32, i32) #1
+declare {<4 x float>,i32} @llvm.amdgcn.image.load.3d.v4f32i32.i32(i32, i32, i32, i32, <8 x i32>, i32, i32) #1
 declare <4 x float> @llvm.amdgcn.image.load.cube.v4f32.i32(i32, i32, i32, i32, <8 x i32>, i32, i32) #1
+declare {<4 x float>,i32} @llvm.amdgcn.image.load.cube.v4f32i32.i32(i32, i32, i32, i32, <8 x i32>, i32, i32) #1
 declare <4 x float> @llvm.amdgcn.image.load.1darray.v4f32.i32(i32, i32, i32, <8 x i32>, i32, i32) #1
+declare {<4 x float>,i32} @llvm.amdgcn.image.load.1darray.v4f32i32.i32(i32, i32, i32, <8 x i32>, i32, i32) #1
 declare <4 x float> @llvm.amdgcn.image.load.2darray.v4f32.i32(i32, i32, i32, i32, <8 x i32>, i32, i32) #1
+declare {<4 x float>,i32} @llvm.amdgcn.image.load.2darray.v4f32i32.i32(i32, i32, i32, i32, <8 x i32>, i32, i32) #1
 declare <4 x float> @llvm.amdgcn.image.load.2dmsaa.v4f32.i32(i32, i32, i32, i32, <8 x i32>, i32, i32) #1
+declare {<4 x float>,i32} @llvm.amdgcn.image.load.2dmsaa.v4f32i32.i32(i32, i32, i32, i32, <8 x i32>, i32, i32) #1
 declare <4 x float> @llvm.amdgcn.image.load.2darraymsaa.v4f32.i32(i32, i32, i32, i32, i32, <8 x i32>, i32, i32) #1
+declare {<4 x float>,i32} @llvm.amdgcn.image.load.2darraymsaa.v4f32i32.i32(i32, i32, i32, i32, i32, <8 x i32>, i32, i32) #1
 
 declare <4 x float> @llvm.amdgcn.image.load.mip.1d.v4f32.i32(i32, i32, i32, <8 x i32>, i32, i32) #1
 declare <4 x float> @llvm.amdgcn.image.load.mip.2d.v4f32.i32(i32, i32, i32, i32, <8 x i32>, i32, i32) #1
+declare {<4 x float>,i32} @llvm.amdgcn.image.load.mip.1d.v4f32i32.i32(i32, i32, i32, <8 x i32>, i32, i32) #1
+declare {<4 x float>,i32} @llvm.amdgcn.image.load.mip.2d.v4f32i32.i32(i32, i32, i32, i32, <8 x i32>, i32, i32) #1
+declare {<2 x float>,i32} @llvm.amdgcn.image.load.mip.2d.v2f32i32.i32(i32, i32, i32, i32, <8 x i32>, i32, i32) #1
+declare {float,i32} @llvm.amdgcn.image.load.mip.2d.f32i32.i32(i32, i32, i32, i32, <8 x i32>, i32, i32) #1
 declare <4 x float> @llvm.amdgcn.image.load.mip.3d.v4f32.i32(i32, i32, i32, i32, i32, <8 x i32>, i32, i32) #1
 declare <4 x float> @llvm.amdgcn.image.load.mip.cube.v4f32.i32(i32, i32, i32, i32, i32, <8 x i32>, i32, i32) #1
 declare <4 x float> @llvm.amdgcn.image.load.mip.1darray.v4f32.i32(i32, i32, i32, i32, <8 x i32>, i32, i32) #1
diff --git a/test/CodeGen/AMDGPU/llvm.amdgcn.image.load.a16.d16.ll b/test/CodeGen/AMDGPU/llvm.amdgcn.image.load.a16.d16.ll
index 1fbfccb0e39a3acd3e90818e66810ac8550045b6..fd2c6e796c8111d6029705c63044e6c6895b112f 100644
--- a/test/CodeGen/AMDGPU/llvm.amdgcn.image.load.a16.d16.ll
+++ b/test/CodeGen/AMDGPU/llvm.amdgcn.image.load.a16.d16.ll
@@ -1,7 +1,7 @@
 ; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN %s
 
 ; GCN-LABEL: {{^}}load.f16.1d:
-; GCN: image_load v[0:1], v0, s[0:7] dmask:0x1 unorm a16 d16
+; GCN: image_load v0, v0, s[0:7] dmask:0x1 unorm a16 d16
 define amdgpu_ps <4 x half> @load.f16.1d(<8 x i32> inreg %rsrc, <2 x i16> %coords) {
 main_body:
   %x = extractelement <2 x i16> %coords, i32 0
@@ -10,7 +10,7 @@ main_body:
 }
 
 ; GCN-LABEL: {{^}}load.v2f16.1d:
-; GCN: image_load v[0:1], v0, s[0:7] dmask:0x3 unorm a16 d16
+; GCN: image_load v0, v0, s[0:7] dmask:0x3 unorm a16 d16
 define amdgpu_ps <4 x half> @load.v2f16.1d(<8 x i32> inreg %rsrc, <2 x i16> %coords) {
 main_body:
   %x = extractelement <2 x i16> %coords, i32 0
@@ -37,7 +37,7 @@ main_body:
 }
 
 ; GCN-LABEL: {{^}}load.f16.2d:
-; GCN: image_load v[0:1], v0, s[0:7] dmask:0x1 unorm a16 d16
+; GCN: image_load v0, v0, s[0:7] dmask:0x1 unorm a16 d16
 define amdgpu_ps <4 x half> @load.f16.2d(<8 x i32> inreg %rsrc, <2 x i16> %coords) {
 main_body:
   %x = extractelement <2 x i16> %coords, i32 0
@@ -47,7 +47,7 @@ main_body:
 }
 
 ; GCN-LABEL: {{^}}load.v2f16.2d:
-; GCN: image_load v[0:1], v0, s[0:7] dmask:0x3 unorm a16 d16
+; GCN: image_load v0, v0, s[0:7] dmask:0x3 unorm a16 d16
 define amdgpu_ps <4 x half> @load.v2f16.2d(<8 x i32> inreg %rsrc, <2 x i16> %coords) {
 main_body:
   %x = extractelement <2 x i16> %coords, i32 0
@@ -77,7 +77,7 @@ main_body:
 }
 
 ; GCN-LABEL: {{^}}load.f16.3d:
-; GCN: image_load v[0:1], v[0:1], s[0:7] dmask:0x1 unorm a16 d16
+; GCN: image_load v0, v[0:1], s[0:7] dmask:0x1 unorm a16 d16
 define amdgpu_ps <4 x half> @load.f16.3d(<8 x i32> inreg %rsrc, <2 x i16> %coords_lo, <2 x i16> %coords_hi) {
 main_body:
   %x = extractelement <2 x i16> %coords_lo, i32 0
@@ -88,7 +88,7 @@ main_body:
 }
 
 ; GCN-LABEL: {{^}}load.v2f16.3d:
-; GCN: image_load v[0:1], v[0:1], s[0:7] dmask:0x3 unorm a16 d16
+; GCN: image_load v0, v[0:1], s[0:7] dmask:0x3 unorm a16 d16
 define amdgpu_ps <4 x half> @load.v2f16.3d(<8 x i32> inreg %rsrc, <2 x i16> %coords_lo, <2 x i16> %coords_hi) {
 main_body:
   %x = extractelement <2 x i16> %coords_lo, i32 0
diff --git a/test/CodeGen/AMDGPU/llvm.amdgcn.image.load.a16.ll b/test/CodeGen/AMDGPU/llvm.amdgcn.image.load.a16.ll
index d857ae115a7c3e1699133a9744c04b1ced8d1fd4..be579b84eb4a1c6dabf268173b60b5c8a0e6edea 100644
--- a/test/CodeGen/AMDGPU/llvm.amdgcn.image.load.a16.ll
+++ b/test/CodeGen/AMDGPU/llvm.amdgcn.image.load.a16.ll
@@ -1,7 +1,7 @@
 ; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN %s
 
 ; GCN-LABEL: {{^}}load.f32.1d:
-; GCN: image_load v[0:3], v0, s[0:7] dmask:0x1 unorm a16
+; GCN: image_load v0, v0, s[0:7] dmask:0x1 unorm a16
 define amdgpu_ps <4 x float> @load.f32.1d(<8 x i32> inreg %rsrc, <2 x i16> %coords) {
 main_body:
   %x = extractelement <2 x i16> %coords, i32 0
@@ -10,7 +10,7 @@ main_body:
 }
 
 ; GCN-LABEL: {{^}}load.v2f32.1d:
-; GCN: image_load v[0:3], v0, s[0:7] dmask:0x3 unorm a16
+; GCN: image_load v[0:1], v0, s[0:7] dmask:0x3 unorm a16
 define amdgpu_ps <4 x float> @load.v2f32.1d(<8 x i32> inreg %rsrc, <2 x i16> %coords) {
 main_body:
   %x = extractelement <2 x i16> %coords, i32 0
@@ -37,7 +37,7 @@ main_body:
 }
 
 ; GCN-LABEL: {{^}}load.f32.2d:
-; GCN: image_load v[0:3], v0, s[0:7] dmask:0x1 unorm a16
+; GCN: image_load v0, v0, s[0:7] dmask:0x1 unorm a16
 define amdgpu_ps <4 x float> @load.f32.2d(<8 x i32> inreg %rsrc, <2 x i16> %coords) {
 main_body:
   %x = extractelement <2 x i16> %coords, i32 0
@@ -47,7 +47,7 @@ main_body:
 }
 
 ; GCN-LABEL: {{^}}load.v2f32.2d:
-; GCN: image_load v[0:3], v0, s[0:7] dmask:0x3 unorm a16
+; GCN: image_load v[0:1], v0, s[0:7] dmask:0x3 unorm a16
 define amdgpu_ps <4 x float> @load.v2f32.2d(<8 x i32> inreg %rsrc, <2 x i16> %coords) {
 main_body:
   %x = extractelement <2 x i16> %coords, i32 0
@@ -77,7 +77,7 @@ main_body:
 }
 
 ; GCN-LABEL: {{^}}load.f32.3d:
-; GCN: image_load v[0:3], v[0:1], s[0:7] dmask:0x1 unorm a16
+; GCN: image_load v0, v[0:1], s[0:7] dmask:0x1 unorm a16
 define amdgpu_ps <4 x float> @load.f32.3d(<8 x i32> inreg %rsrc, <2 x i16> %coords_lo, <2 x i16> %coords_hi) {
 main_body:
   %x = extractelement <2 x i16> %coords_lo, i32 0
@@ -88,7 +88,7 @@ main_body:
 }
 
 ; GCN-LABEL: {{^}}load.v2f32.3d:
-; GCN: image_load v[0:3], v[0:1], s[0:7] dmask:0x3 unorm a16
+; GCN: image_load v[0:1], v[0:1], s[0:7] dmask:0x3 unorm a16
 define amdgpu_ps <4 x float> @load.v2f32.3d(<8 x i32> inreg %rsrc, <2 x i16> %coords_lo, <2 x i16> %coords_hi) {
 main_body:
   %x = extractelement <2 x i16> %coords_lo, i32 0
diff --git a/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.d16.dim.ll b/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.d16.dim.ll
index 9619304e1aa920386869a54b9c293978e6c44f8b..b6260f4af83efe11caa3fcf9cd1ed0a00ef80ace 100644
--- a/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.d16.dim.ll
+++ b/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.d16.dim.ll
@@ -10,6 +10,19 @@ main_body:
   ret half %tex
 }
 
+; GCN-LABEL: {{^}}image_sample_2d_f16_tfe:
+; GCN: v_mov_b32_e32 v{{[0-9]+}}, 0
+; PACKED: image_sample v[2:3], v[0:1], s[0:7], s[8:11] dmask:0x1 tfe d16{{$}}
+; UNPACKED: image_sample v[2:3], v[0:1], s[0:7], s[8:11] dmask:0x1 tfe d16{{$}}
+define amdgpu_ps half @image_sample_2d_f16_tfe(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %s, float %t, i32 addrspace(1)* inreg %out) {
+main_body:
+  %tex = call {half,i32} @llvm.amdgcn.image.sample.2d.f16i32.f32(i32 1, float %s, float %t, <8 x i32> %rsrc, <4 x i32> %samp, i1 false, i32 1, i32 0)
+  %tex.vec = extractvalue {half, i32} %tex, 0
+  %tex.err = extractvalue {half, i32} %tex, 1
+  store i32 %tex.err, i32 addrspace(1)* %out, align 4
+  ret half %tex.vec
+}
+
 ; GCN-LABEL: {{^}}image_sample_c_d_1d_v2f16:
 ; UNPACKED: image_sample_c_d v[0:1], v[0:3], s[0:7], s[8:11] dmask:0x3 d16{{$}}
 ; PACKED: image_sample_c_d v0, v[0:3], s[0:7], s[8:11] dmask:0x3 d16{{$}}
@@ -20,6 +33,22 @@ main_body:
   ret float %r
 }
 
+; GCN-LABEL: {{^}}image_sample_c_d_1d_v2f16_tfe:
+; GCN: v_mov_b32_e32 v{{[0-9]+}}, 0
+; UNPACKED: image_sample_c_d v[{{[0-9]+:[0-9]+}}], v[0:3], s[0:7], s[8:11] dmask:0x3 tfe d16{{$}}
+; PACKED: image_sample_c_d v[{{[0-9]+:[0-9]+}}], v[0:3], s[0:7], s[8:11] dmask:0x3 tfe d16{{$}}
+define amdgpu_ps <2 x float> @image_sample_c_d_1d_v2f16_tfe(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %zcompare, float %dsdh, float %dsdv, float %s) {
+main_body:
+  %tex = call {<2 x half>,i32} @llvm.amdgcn.image.sample.c.d.1d.v2f16i32.f32.f32(i32 3, float %zcompare, float %dsdh, float %dsdv, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 false, i32 1, i32 0)
+  %tex.vec = extractvalue {<2 x half>, i32} %tex, 0
+  %tex.err = extractvalue {<2 x half>, i32} %tex, 1
+  %tex.vecf = bitcast <2 x half> %tex.vec to float
+  %r.0 = insertelement <2 x float> undef, float %tex.vecf, i32 0
+  %tex.errf = bitcast i32 %tex.err to float
+  %r = insertelement <2 x float> %r.0, float %tex.errf, i32 1
+  ret <2 x float> %r
+}
+
 ; GCN-LABEL: {{^}}image_sample_b_2d_v4f16:
 ; UNPACKED: image_sample_b v[0:3], v[0:3], s[0:7], s[8:11] dmask:0xf d16{{$}}
 ; PACKED: image_sample_b v[0:1], v[0:3], s[0:7], s[8:11] dmask:0xf d16{{$}}
@@ -30,9 +59,33 @@ main_body:
   ret <2 x float> %r
 }
 
+; GCN-LABEL: {{^}}image_sample_b_2d_v4f16_tfe:
+; GCN: v_mov_b32_e32 v{{[0-9]+}}, 0
+; UNPACKED: image_sample_b v[{{[0-9]+:[0-9]+}}], v[0:3], s[0:7], s[8:11] dmask:0xf tfe d16{{$}}
+; PACKED: image_sample_b v[{{[0-9]+:[0-9]+}}], v[0:3], s[0:7], s[8:11] dmask:0xf tfe d16{{$}}
+define amdgpu_ps <4 x float> @image_sample_b_2d_v4f16_tfe(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %bias, float %s, float %t) {
+main_body:
+  %tex = call {<4 x half>,i32} @llvm.amdgcn.image.sample.b.2d.v4f16i32.f32.f32(i32 15, float %bias, float %s, float %t, <8 x i32> %rsrc, <4 x i32> %samp, i1 false, i32 1, i32 0)
+  %tex.vec = extractvalue {<4 x half>, i32} %tex, 0
+  %tex.err = extractvalue {<4 x half>, i32} %tex, 1
+  %tex.vecf = bitcast <4 x half> %tex.vec to <2 x float>
+  %tex.vecf.0 = extractelement <2 x float> %tex.vecf, i32 0
+  %tex.vecf.1 = extractelement <2 x float> %tex.vecf, i32 1
+  %r.0 = insertelement <4 x float> undef, float %tex.vecf.0, i32 0
+  %r.1 = insertelement <4 x float> %r.0, float %tex.vecf.1, i32 1
+  %tex.errf = bitcast i32 %tex.err to float
+  %r = insertelement <4 x float> %r.1, float %tex.errf, i32 2
+  ret <4 x float> %r
+}
+
 declare half @llvm.amdgcn.image.sample.2d.f16.f32(i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare {half,i32} @llvm.amdgcn.image.sample.2d.f16i32.f32(i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare <4 x half> @llvm.amdgcn.image.sample.2d.v4f16.f32(i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare {<2 x half>,i32} @llvm.amdgcn.image.sample.2d.v2f16i32.f32(i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 declare <2 x half> @llvm.amdgcn.image.sample.c.d.1d.v2f16.f32.f32(i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare {<2 x half>,i32} @llvm.amdgcn.image.sample.c.d.1d.v2f16i32.f32.f32(i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 declare <4 x half> @llvm.amdgcn.image.sample.b.2d.v4f16.f32.f32(i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare {<4 x half>,i32} @llvm.amdgcn.image.sample.b.2d.v4f16i32.f32.f32(i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 attributes #0 = { nounwind }
 attributes #1 = { nounwind readonly }
diff --git a/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.dim.ll b/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.dim.ll
index 65f4b46d0ae48828a3c8c29a378af27d1abdbb35..2ee69ac6e8b8bf248080a8524314f4f233a50496 100644
--- a/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.dim.ll
+++ b/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.dim.ll
@@ -9,6 +9,162 @@ main_body:
   ret <4 x float> %v
 }
 
+; GCN-LABEL: {{^}}sample_1d_tfe:
+; GCN: v_mov_b32_e32 v0, 0
+; GCN: v_mov_b32_e32 v1, v0
+; GCN: v_mov_b32_e32 v2, v0
+; GCN: v_mov_b32_e32 v3, v0
+; GCN: v_mov_b32_e32 v4, v0
+; GCN: image_sample v[0:7], v5, s[0:7], s[8:11] dmask:0xf tfe{{$}}
+define amdgpu_ps <4 x float> @sample_1d_tfe(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, i32 addrspace(1)* inreg %out, float %s) {
+main_body:
+  %v = call {<4 x float>,i32} @llvm.amdgcn.image.sample.1d.v4f32i32.f32(i32 15, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 1, i32 0)
+  %v.vec = extractvalue {<4 x float>, i32} %v, 0
+  %v.err = extractvalue {<4 x float>, i32} %v, 1
+  store i32 %v.err, i32 addrspace(1)* %out, align 4
+  ret <4 x float> %v.vec
+}
+
+; GCN-LABEL: {{^}}sample_1d_tfe_adjust_writemask_1:
+; GCN: v_mov_b32_e32 v0, 0
+; GCN: v_mov_b32_e32 v1, v0
+; GCN: image_sample v[0:1], v2, s[0:7], s[8:11] dmask:0x1 tfe{{$}}
+define amdgpu_ps <2 x float> @sample_1d_tfe_adjust_writemask_1(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, i32 addrspace(1)* inreg %out, float %s) {
+main_body:
+  %v = call {<4 x float>,i32} @llvm.amdgcn.image.sample.1d.v4f32i32.f32(i32 15, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 1, i32 0)
+  %res.vec = extractvalue {<4 x float>,i32} %v, 0
+  %res.f = extractelement <4 x float> %res.vec, i32 0
+  %res.err = extractvalue {<4 x float>,i32} %v, 1
+  %res.errf = bitcast i32 %res.err to float
+  %res.tmp = insertelement <2 x float> undef, float %res.f, i32 0
+  %res = insertelement <2 x float> %res.tmp, float %res.errf, i32 1
+  ret <2 x float> %res
+}
+
+; GCN-LABEL: {{^}}sample_1d_tfe_adjust_writemask_2:
+; GCN: v_mov_b32_e32 v0, 0
+; GCN: v_mov_b32_e32 v1, v0
+; GCN: image_sample v[0:1], v2, s[0:7], s[8:11] dmask:0x2 tfe{{$}}
+define amdgpu_ps <2 x float> @sample_1d_tfe_adjust_writemask_2(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %s) {
+main_body:
+  %v = call {<4 x float>,i32} @llvm.amdgcn.image.sample.1d.v4f32i32.f32(i32 15, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 1, i32 0)
+  %res.vec = extractvalue {<4 x float>,i32} %v, 0
+  %res.f = extractelement <4 x float> %res.vec, i32 1
+  %res.err = extractvalue {<4 x float>,i32} %v, 1
+  %res.errf = bitcast i32 %res.err to float
+  %res.tmp = insertelement <2 x float> undef, float %res.f, i32 0
+  %res = insertelement <2 x float> %res.tmp, float %res.errf, i32 1
+  ret <2 x float> %res
+}
+
+; GCN-LABEL: {{^}}sample_1d_tfe_adjust_writemask_3:
+; GCN: v_mov_b32_e32 v0, 0
+; GCN: v_mov_b32_e32 v1, v0
+; GCN: image_sample v[0:1], v2, s[0:7], s[8:11] dmask:0x4 tfe{{$}}
+define amdgpu_ps <2 x float> @sample_1d_tfe_adjust_writemask_3(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %s) {
+main_body:
+  %v = call {<4 x float>,i32} @llvm.amdgcn.image.sample.1d.v4f32i32.f32(i32 15, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 1, i32 0)
+  %res.vec = extractvalue {<4 x float>,i32} %v, 0
+  %res.f = extractelement <4 x float> %res.vec, i32 2
+  %res.err = extractvalue {<4 x float>,i32} %v, 1
+  %res.errf = bitcast i32 %res.err to float
+  %res.tmp = insertelement <2 x float> undef, float %res.f, i32 0
+  %res = insertelement <2 x float> %res.tmp, float %res.errf, i32 1
+  ret <2 x float> %res
+}
+
+; GCN-LABEL: {{^}}sample_1d_tfe_adjust_writemask_4:
+; GCN: v_mov_b32_e32 v0, 0
+; GCN: v_mov_b32_e32 v1, v0
+; GCN: image_sample v[0:1], v2, s[0:7], s[8:11] dmask:0x8 tfe{{$}}
+define amdgpu_ps <2 x float> @sample_1d_tfe_adjust_writemask_4(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %s) {
+main_body:
+  %v = call {<4 x float>,i32} @llvm.amdgcn.image.sample.1d.v4f32i32.f32(i32 15, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 1, i32 0)
+  %res.vec = extractvalue {<4 x float>,i32} %v, 0
+  %res.f = extractelement <4 x float> %res.vec, i32 3
+  %res.err = extractvalue {<4 x float>,i32} %v, 1
+  %res.errf = bitcast i32 %res.err to float
+  %res.tmp = insertelement <2 x float> undef, float %res.f, i32 0
+  %res = insertelement <2 x float> %res.tmp, float %res.errf, i32 1
+  ret <2 x float> %res
+}
+
+; GCN-LABEL: {{^}}sample_1d_tfe_adjust_writemask_12:
+; GCN: v_mov_b32_e32 v0, 0
+; GCN: v_mov_b32_e32 v1, v0
+; GCN: v_mov_b32_e32 v2, v0
+; GCN: image_sample v[0:2], v3, s[0:7], s[8:11] dmask:0x3 tfe{{$}}
+define amdgpu_ps <4 x float> @sample_1d_tfe_adjust_writemask_12(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %s) {
+main_body:
+  %v = call {<4 x float>,i32} @llvm.amdgcn.image.sample.1d.v4f32i32.f32(i32 15, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 1, i32 0)
+  %res.vec = extractvalue {<4 x float>,i32} %v, 0
+  %res.f1 = extractelement <4 x float> %res.vec, i32 0
+  %res.f2 = extractelement <4 x float> %res.vec, i32 1
+  %res.err = extractvalue {<4 x float>,i32} %v, 1
+  %res.errf = bitcast i32 %res.err to float
+  %res.tmp1 = insertelement <4 x float> undef, float %res.f1, i32 0
+  %res.tmp2 = insertelement <4 x float> %res.tmp1, float %res.f2, i32 1
+  %res = insertelement <4 x float> %res.tmp2, float %res.errf, i32 2
+  ret <4 x float> %res
+}
+
+; GCN-LABEL: {{^}}sample_1d_tfe_adjust_writemask_24:
+; GCN: v_mov_b32_e32 v0, 0
+; GCN: v_mov_b32_e32 v1, v0
+; GCN: v_mov_b32_e32 v2, v0
+; GCN: image_sample v[0:2], v3, s[0:7], s[8:11] dmask:0xa tfe{{$}}
+define amdgpu_ps <4 x float> @sample_1d_tfe_adjust_writemask_24(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %s) {
+main_body:
+  %v = call {<4 x float>,i32} @llvm.amdgcn.image.sample.1d.v4f32i32.f32(i32 15, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 1, i32 0)
+  %res.vec = extractvalue {<4 x float>,i32} %v, 0
+  %res.f1 = extractelement <4 x float> %res.vec, i32 1
+  %res.f2 = extractelement <4 x float> %res.vec, i32 3
+  %res.err = extractvalue {<4 x float>,i32} %v, 1
+  %res.errf = bitcast i32 %res.err to float
+  %res.tmp1 = insertelement <4 x float> undef, float %res.f1, i32 0
+  %res.tmp2 = insertelement <4 x float> %res.tmp1, float %res.f2, i32 1
+  %res = insertelement <4 x float> %res.tmp2, float %res.errf, i32 2
+  ret <4 x float> %res
+}
+
+; GCN-LABEL: {{^}}sample_1d_tfe_adjust_writemask_134:
+; GCN: v_mov_b32_e32 v0, 0
+; GCN: v_mov_b32_e32 v1, v0
+; GCN: v_mov_b32_e32 v2, v0
+; GCN: v_mov_b32_e32 v3, v0
+; GCN: image_sample v[0:3], v4, s[0:7], s[8:11] dmask:0xd tfe{{$}}
+define amdgpu_ps <4 x float> @sample_1d_tfe_adjust_writemask_134(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %s) {
+main_body:
+  %v = call {<4 x float>,i32} @llvm.amdgcn.image.sample.1d.v4f32i32.f32(i32 15, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 1, i32 0)
+  %res.vec = extractvalue {<4 x float>,i32} %v, 0
+  %res.f1 = extractelement <4 x float> %res.vec, i32 0
+  %res.f2 = extractelement <4 x float> %res.vec, i32 2
+  %res.f3 = extractelement <4 x float> %res.vec, i32 3
+  %res.err = extractvalue {<4 x float>,i32} %v, 1
+  %res.errf = bitcast i32 %res.err to float
+  %res.tmp1 = insertelement <4 x float> undef, float %res.f1, i32 0
+  %res.tmp2 = insertelement <4 x float> %res.tmp1, float %res.f2, i32 1
+  %res.tmp3 = insertelement <4 x float> %res.tmp2, float %res.f3, i32 2
+  %res = insertelement <4 x float> %res.tmp3, float %res.errf, i32 3
+  ret <4 x float> %res
+}
+
+; GCN-LABEL: {{^}}sample_1d_lwe:
+; GCN: v_mov_b32_e32 v0, 0
+; GCN: v_mov_b32_e32 v1, v0
+; GCN: v_mov_b32_e32 v2, v0
+; GCN: v_mov_b32_e32 v3, v0
+; GCN: v_mov_b32_e32 v4, v0
+; GCN: image_sample v[0:7], v5, s[0:7], s[8:11] dmask:0xf lwe{{$}}
+define amdgpu_ps <4 x float> @sample_1d_lwe(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, i32 addrspace(1)* inreg %out, float %s) {
+main_body:
+  %v = call {<4 x float>,i32} @llvm.amdgcn.image.sample.1d.v4f32i32.f32(i32 15, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 2, i32 0)
+  %v.vec = extractvalue {<4 x float>, i32} %v, 0
+  %v.err = extractvalue {<4 x float>, i32} %v, 1
+  store i32 %v.err, i32 addrspace(1)* %out, align 4
+  ret <4 x float> %v.vec
+}
+
 ; GCN-LABEL: {{^}}sample_2d:
 ; GCN: image_sample v[0:3], v[0:1], s[0:7], s[8:11] dmask:0xf{{$}}
 define amdgpu_ps <4 x float> @sample_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %s, float %t) {
@@ -361,6 +517,17 @@ main_body:
   ret float %v
 }
 
+; GCN-LABEL: {{^}}sample_c_d_o_2darray_V1_tfe:
+; GCN: image_sample_c_d_o v[9:10], v[0:15], s[0:7], s[8:11] dmask:0x4 tfe da{{$}}
+define amdgpu_ps float @sample_c_d_o_2darray_V1_tfe(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, i32 %offset, float %zcompare, float %dsdh, float %dtdh, float %dsdv, float %dtdv, float %s, float %t, float %slice, i32 addrspace(1)* inreg %out) {
+main_body:
+  %v = call {float,i32} @llvm.amdgcn.image.sample.c.d.o.2darray.f32i32.f32.f32(i32 4, i32 %offset, float %zcompare, float %dsdh, float %dtdh, float %dsdv, float %dtdv, float %s, float %t, float %slice, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 1, i32 0)
+  %v.vec = extractvalue {float, i32} %v, 0
+  %v.err = extractvalue {float, i32} %v, 1
+  store i32 %v.err, i32 addrspace(1)* %out, align 4
+  ret float %v.vec
+}
+
 ; GCN-LABEL: {{^}}sample_c_d_o_2darray_V2:
 ; GCN: image_sample_c_d_o v[0:1], v[0:15], s[0:7], s[8:11] dmask:0x6 da{{$}}
 define amdgpu_ps <2 x float> @sample_c_d_o_2darray_V2(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, i32 %offset, float %zcompare, float %dsdh, float %dtdh, float %dsdv, float %dtdv, float %s, float %t, float %slice) {
@@ -369,6 +536,22 @@ main_body:
   ret <2 x float> %v
 }
 
+; GCN-LABEL: {{^}}sample_c_d_o_2darray_V2_tfe:
+; GCN: image_sample_c_d_o v[9:12], v[0:15], s[0:7], s[8:11] dmask:0x6 tfe da{{$}}
+define amdgpu_ps <4 x float> @sample_c_d_o_2darray_V2_tfe(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, i32 %offset, float %zcompare, float %dsdh, float %dtdh, float %dsdv, float %dtdv, float %s, float %t, float %slice) {
+main_body:
+  %v = call {<2 x float>, i32} @llvm.amdgcn.image.sample.c.d.o.2darray.v2f32i32.f32.f32(i32 6, i32 %offset, float %zcompare, float %dsdh, float %dtdh, float %dsdv, float %dtdv, float %s, float %t, float %slice, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 1, i32 0)
+  %v.vec = extractvalue {<2 x float>, i32} %v, 0
+  %v.f1 = extractelement <2 x float> %v.vec, i32 0
+  %v.f2 = extractelement <2 x float> %v.vec, i32 1
+  %v.err = extractvalue {<2 x float>, i32} %v, 1
+  %v.errf = bitcast i32 %v.err to float
+  %res.0 = insertelement <4 x float> undef, float %v.f1, i32 0
+  %res.1 = insertelement <4 x float> %res.0, float %v.f2, i32 1
+  %res.2 = insertelement <4 x float> %res.1, float %v.errf, i32 2
+  ret <4 x float> %res.2
+}
+
 ; GCN-LABEL: {{^}}sample_1d_unorm:
 ; GCN: image_sample v[0:3], v0, s[0:7], s[8:11] dmask:0xf unorm{{$}}
 define amdgpu_ps <4 x float> @sample_1d_unorm(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %s) {
@@ -491,6 +674,7 @@ main_body:
 }
 
 declare <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare {<4 x float>,i32} @llvm.amdgcn.image.sample.1d.v4f32i32.f32(i32, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 declare <4 x float> @llvm.amdgcn.image.sample.2d.v4f32.f32(i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 declare <4 x float> @llvm.amdgcn.image.sample.3d.v4f32.f32(i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 declare <4 x float> @llvm.amdgcn.image.sample.cube.v4f32.f32(i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
@@ -542,7 +726,9 @@ declare <4 x float> @llvm.amdgcn.image.sample.c.lz.1d.v4f32.f32(i32, float, floa
 declare <4 x float> @llvm.amdgcn.image.sample.c.lz.2d.v4f32.f32(i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 declare float @llvm.amdgcn.image.sample.c.d.o.2darray.f32.f32.f32(i32, i32, float, float, float, float, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare {float, i32} @llvm.amdgcn.image.sample.c.d.o.2darray.f32i32.f32.f32(i32, i32, float, float, float, float, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 declare <2 x float> @llvm.amdgcn.image.sample.c.d.o.2darray.v2f32.f32.f32(i32, i32, float, float, float, float, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare {<2 x float>, i32} @llvm.amdgcn.image.sample.c.d.o.2darray.v2f32i32.f32.f32(i32, i32, float, float, float, float, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 
 attributes #0 = { nounwind }
 attributes #1 = { nounwind readonly }
diff --git a/test/CodeGen/AMDGPU/llvm.amdgcn.interp.f16.ll b/test/CodeGen/AMDGPU/llvm.amdgcn.interp.f16.ll
new file mode 100644
index 0000000000000000000000000000000000000000..fb08e6d8890671456c060e28b06b084c7df48ca5
--- /dev/null
+++ b/test/CodeGen/AMDGPU/llvm.amdgcn.interp.f16.ll
@@ -0,0 +1,187 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX9-32BANK %s
+; RUN: llc -mtriple=amdgcn -mcpu=fiji -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX8-32BANK %s
+; RUN: llc -mtriple=amdgcn -mcpu=gfx810 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX8-16BANK %s
+
+define amdgpu_ps half @interp_f16(float inreg %i, float inreg %j, i32 inreg %m0) #0 {
+; GFX9-32BANK-LABEL: interp_f16:
+; GFX9-32BANK:       ; %bb.0: ; %main_body
+; GFX9-32BANK-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-32BANK-NEXT:    s_mov_b32 m0, s2
+; GFX9-32BANK-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 3
+; GFX9-32BANK-NEXT:    v_interp_p1ll_f16 v1, v0, attr2.y
+; GFX9-32BANK-NEXT:    v_mov_b32_e32 v2, s1
+; GFX9-32BANK-NEXT:    v_interp_p1ll_f16 v0, v0, attr2.y high
+; GFX9-32BANK-NEXT:    v_interp_p2_legacy_f16 v1, v2, attr2.y, v1
+; GFX9-32BANK-NEXT:    v_interp_p2_legacy_f16 v0, v2, attr2.y, v0 high
+; GFX9-32BANK-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 0
+; GFX9-32BANK-NEXT:    v_add_f16_e32 v0, v1, v0
+; GFX9-32BANK-NEXT:    ; return to shader part epilog
+;
+; GFX8-32BANK-LABEL: interp_f16:
+; GFX8-32BANK:       ; %bb.0: ; %main_body
+; GFX8-32BANK-NEXT:    v_mov_b32_e32 v0, s0
+; GFX8-32BANK-NEXT:    s_mov_b32 m0, s2
+; GFX8-32BANK-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 3
+; GFX8-32BANK-NEXT:    v_interp_p1ll_f16 v1, v0, attr2.y
+; GFX8-32BANK-NEXT:    v_mov_b32_e32 v2, s1
+; GFX8-32BANK-NEXT:    v_interp_p1ll_f16 v0, v0, attr2.y high
+; GFX8-32BANK-NEXT:    v_interp_p2_f16 v1, v2, attr2.y, v1
+; GFX8-32BANK-NEXT:    v_interp_p2_f16 v0, v2, attr2.y, v0 high
+; GFX8-32BANK-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 0
+; GFX8-32BANK-NEXT:    v_add_f16_e32 v0, v1, v0
+; GFX8-32BANK-NEXT:    ; return to shader part epilog
+;
+; GFX8-16BANK-LABEL: interp_f16:
+; GFX8-16BANK:       ; %bb.0: ; %main_body
+; GFX8-16BANK-NEXT:    s_mov_b32 m0, s2
+; GFX8-16BANK-NEXT:    v_interp_mov_f32_e32 v0, p0, attr2.y
+; GFX8-16BANK-NEXT:    v_mov_b32_e32 v1, s0
+; GFX8-16BANK-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 3
+; GFX8-16BANK-NEXT:    v_interp_p1lv_f16 v2, v1, attr2.y, v0
+; GFX8-16BANK-NEXT:    v_mov_b32_e32 v3, s1
+; GFX8-16BANK-NEXT:    v_interp_p1lv_f16 v0, v1, attr2.y, v0 high
+; GFX8-16BANK-NEXT:    v_interp_p2_f16 v2, v3, attr2.y, v2
+; GFX8-16BANK-NEXT:    v_interp_p2_f16 v0, v3, attr2.y, v0 high
+; GFX8-16BANK-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 0
+; GFX8-16BANK-NEXT:    v_add_f16_e32 v0, v2, v0
+; GFX8-16BANK-NEXT:    ; return to shader part epilog
+main_body:
+  %p1_0 = call float @llvm.amdgcn.interp.p1.f16(float %i, i32 1, i32 2, i1 0, i32 %m0)
+  %p2_0 = call half @llvm.amdgcn.interp.p2.f16(float %p1_0, float %j, i32 1, i32 2, i1 0, i32 %m0)
+  %p1_1 = call float @llvm.amdgcn.interp.p1.f16(float %i, i32 1, i32 2, i1 1, i32 %m0)
+  %p2_1 = call half @llvm.amdgcn.interp.p2.f16(float %p1_1, float %j, i32 1, i32 2, i1 1, i32 %m0)
+  %res = fadd half %p2_0, %p2_1
+  ret half %res
+}
+
+; check that m0 is setup correctly before the interp p1 instruction
+define amdgpu_ps half @interp_p1_m0_setup(float inreg %i, float inreg %j, i32 inreg %m0) #0 {
+; GFX9-32BANK-LABEL: interp_p1_m0_setup:
+; GFX9-32BANK:       ; %bb.0: ; %main_body
+; GFX9-32BANK-NEXT:    ;;#ASMSTART
+; GFX9-32BANK-NEXT:    s_mov_b32 m0, 0
+; GFX9-32BANK-NEXT:    ;;#ASMEND
+; GFX9-32BANK-NEXT:    s_mov_b32 s3, m0
+; GFX9-32BANK-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-32BANK-NEXT:    s_mov_b32 m0, s2
+; GFX9-32BANK-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 3
+; GFX9-32BANK-NEXT:    v_interp_p1ll_f16 v0, v0, attr2.y
+; GFX9-32BANK-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-32BANK-NEXT:    v_interp_p2_legacy_f16 v0, v1, attr2.y, v0
+; GFX9-32BANK-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 0
+; GFX9-32BANK-NEXT:    v_add_f16_e32 v0, s3, v0
+; GFX9-32BANK-NEXT:    ; return to shader part epilog
+;
+; GFX8-32BANK-LABEL: interp_p1_m0_setup:
+; GFX8-32BANK:       ; %bb.0: ; %main_body
+; GFX8-32BANK-NEXT:    ;;#ASMSTART
+; GFX8-32BANK-NEXT:    s_mov_b32 m0, 0
+; GFX8-32BANK-NEXT:    ;;#ASMEND
+; GFX8-32BANK-NEXT:    s_mov_b32 s3, m0
+; GFX8-32BANK-NEXT:    v_mov_b32_e32 v0, s0
+; GFX8-32BANK-NEXT:    s_mov_b32 m0, s2
+; GFX8-32BANK-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 3
+; GFX8-32BANK-NEXT:    v_interp_p1ll_f16 v0, v0, attr2.y
+; GFX8-32BANK-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-32BANK-NEXT:    v_interp_p2_f16 v0, v1, attr2.y, v0
+; GFX8-32BANK-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 0
+; GFX8-32BANK-NEXT:    v_add_f16_e32 v0, s3, v0
+; GFX8-32BANK-NEXT:    ; return to shader part epilog
+;
+; GFX8-16BANK-LABEL: interp_p1_m0_setup:
+; GFX8-16BANK:       ; %bb.0: ; %main_body
+; GFX8-16BANK-NEXT:    ;;#ASMSTART
+; GFX8-16BANK-NEXT:    s_mov_b32 m0, 0
+; GFX8-16BANK-NEXT:    ;;#ASMEND
+; GFX8-16BANK-NEXT:    s_mov_b32 s3, m0
+; GFX8-16BANK-NEXT:    s_mov_b32 m0, s2
+; GFX8-16BANK-NEXT:    v_interp_mov_f32_e32 v0, p0, attr2.y
+; GFX8-16BANK-NEXT:    v_mov_b32_e32 v1, s0
+; GFX8-16BANK-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 3
+; GFX8-16BANK-NEXT:    v_interp_p1lv_f16 v0, v1, attr2.y, v0
+; GFX8-16BANK-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-16BANK-NEXT:    v_interp_p2_f16 v0, v1, attr2.y, v0
+; GFX8-16BANK-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 0
+; GFX8-16BANK-NEXT:    v_add_f16_e32 v0, s3, v0
+; GFX8-16BANK-NEXT:    ; return to shader part epilog
+main_body:
+  %mx = call i32 asm sideeffect "s_mov_b32 m0, 0", "={M0}"() #0
+  %p1_0 = call float @llvm.amdgcn.interp.p1.f16(float %i, i32 1, i32 2, i1 0, i32 %m0)
+  %p2_0 = call half @llvm.amdgcn.interp.p2.f16(float %p1_0, float %j, i32 1, i32 2, i1 0, i32 %m0)
+  %my = trunc i32 %mx to i16
+  %mh = bitcast i16 %my to half
+  %res = fadd half %p2_0, %mh
+  ret half %res
+}
+
+; check that m0 is setup correctly before the interp p2 instruction
+define amdgpu_ps half @interp_p2_m0_setup(float inreg %i, float inreg %j, i32 inreg %m0) #0 {
+; GFX9-32BANK-LABEL: interp_p2_m0_setup:
+; GFX9-32BANK:       ; %bb.0: ; %main_body
+; GFX9-32BANK-NEXT:    v_mov_b32_e32 v0, s0
+; GFX9-32BANK-NEXT:    s_mov_b32 m0, s2
+; GFX9-32BANK-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 3
+; GFX9-32BANK-NEXT:    v_interp_p1ll_f16 v0, v0, attr2.y
+; GFX9-32BANK-NEXT:    ;;#ASMSTART
+; GFX9-32BANK-NEXT:    s_mov_b32 m0, 0
+; GFX9-32BANK-NEXT:    ;;#ASMEND
+; GFX9-32BANK-NEXT:    s_mov_b32 s0, m0
+; GFX9-32BANK-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-32BANK-NEXT:    s_mov_b32 m0, s2
+; GFX9-32BANK-NEXT:    v_interp_p2_legacy_f16 v0, v1, attr2.y, v0
+; GFX9-32BANK-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 0
+; GFX9-32BANK-NEXT:    v_add_f16_e32 v0, s0, v0
+; GFX9-32BANK-NEXT:    ; return to shader part epilog
+;
+; GFX8-32BANK-LABEL: interp_p2_m0_setup:
+; GFX8-32BANK:       ; %bb.0: ; %main_body
+; GFX8-32BANK-NEXT:    v_mov_b32_e32 v0, s0
+; GFX8-32BANK-NEXT:    s_mov_b32 m0, s2
+; GFX8-32BANK-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 3
+; GFX8-32BANK-NEXT:    v_interp_p1ll_f16 v0, v0, attr2.y
+; GFX8-32BANK-NEXT:    ;;#ASMSTART
+; GFX8-32BANK-NEXT:    s_mov_b32 m0, 0
+; GFX8-32BANK-NEXT:    ;;#ASMEND
+; GFX8-32BANK-NEXT:    s_mov_b32 s0, m0
+; GFX8-32BANK-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-32BANK-NEXT:    s_mov_b32 m0, s2
+; GFX8-32BANK-NEXT:    v_interp_p2_f16 v0, v1, attr2.y, v0
+; GFX8-32BANK-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 0
+; GFX8-32BANK-NEXT:    v_add_f16_e32 v0, s0, v0
+; GFX8-32BANK-NEXT:    ; return to shader part epilog
+;
+; GFX8-16BANK-LABEL: interp_p2_m0_setup:
+; GFX8-16BANK:       ; %bb.0: ; %main_body
+; GFX8-16BANK-NEXT:    s_mov_b32 m0, s2
+; GFX8-16BANK-NEXT:    v_interp_mov_f32_e32 v0, p0, attr2.y
+; GFX8-16BANK-NEXT:    v_mov_b32_e32 v1, s0
+; GFX8-16BANK-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 3
+; GFX8-16BANK-NEXT:    v_interp_p1lv_f16 v0, v1, attr2.y, v0
+; GFX8-16BANK-NEXT:    ;;#ASMSTART
+; GFX8-16BANK-NEXT:    s_mov_b32 m0, 0
+; GFX8-16BANK-NEXT:    ;;#ASMEND
+; GFX8-16BANK-NEXT:    s_mov_b32 s0, m0
+; GFX8-16BANK-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-16BANK-NEXT:    s_mov_b32 m0, s2
+; GFX8-16BANK-NEXT:    v_interp_p2_f16 v0, v1, attr2.y, v0
+; GFX8-16BANK-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 0
+; GFX8-16BANK-NEXT:    v_add_f16_e32 v0, s0, v0
+; GFX8-16BANK-NEXT:    ; return to shader part epilog
+main_body:
+  %p1_0 = call float @llvm.amdgcn.interp.p1.f16(float %i, i32 1, i32 2, i1 0, i32 %m0)
+  %mx = call i32 asm sideeffect "s_mov_b32 m0, 0", "={M0}"() #0
+  %p2_0 = call half @llvm.amdgcn.interp.p2.f16(float %p1_0, float %j, i32 1, i32 2, i1 0, i32 %m0)
+  %my = trunc i32 %mx to i16
+  %mh = bitcast i16 %my to half
+  %res = fadd half %p2_0, %mh
+  ret half %res
+}
+
+; float @llvm.amdgcn.interp.p1.f16(i, attrchan, attr, high, m0)
+declare float @llvm.amdgcn.interp.p1.f16(float, i32, i32, i1, i32) #0
+; half @llvm.amdgcn.interp.p1.f16(p1, j, attrchan, attr, high, m0)
+declare half @llvm.amdgcn.interp.p2.f16(float, float, i32, i32, i1, i32) #0
+declare float @llvm.amdgcn.interp.mov(i32, i32, i32, i32) #0
+
+attributes #0 = { nounwind readnone }
diff --git a/test/CodeGen/AMDGPU/llvm.amdgcn.raw.buffer.load.ll b/test/CodeGen/AMDGPU/llvm.amdgcn.raw.buffer.load.ll
index dcae08d55ab43ab999600c31d4bc17685f435bb2..33b2967d34a348c3fdc3e76da38be2a8c4c161a2 100644
--- a/test/CodeGen/AMDGPU/llvm.amdgcn.raw.buffer.load.ll
+++ b/test/CodeGen/AMDGPU/llvm.amdgcn.raw.buffer.load.ll
@@ -97,12 +97,12 @@ entry:
   ret float %val
 }
 
-;CHECK-LABEL: {{^}}buffer_load_x1_offen_merged:
+;CHECK-LABEL: {{^}}buffer_load_x1_offen_merged_and:
 ;CHECK-NEXT: %bb.
 ;CHECK-NEXT: buffer_load_dwordx4 v[{{[0-9]}}:{{[0-9]}}], v0, s[0:3], 0 offen offset:4
 ;CHECK-NEXT: buffer_load_dwordx2 v[{{[0-9]}}:{{[0-9]}}], v0, s[0:3], 0 offen offset:28
 ;CHECK: s_waitcnt
-define amdgpu_ps void @buffer_load_x1_offen_merged(<4 x i32> inreg %rsrc, i32 %a) {
+define amdgpu_ps void @buffer_load_x1_offen_merged_and(<4 x i32> inreg %rsrc, i32 %a) {
 main_body:
   %a1 = add i32 %a, 4
   %a2 = add i32 %a, 8
@@ -121,6 +121,32 @@ main_body:
   ret void
 }
 
+;CHECK-LABEL: {{^}}buffer_load_x1_offen_merged_or:
+;CHECK-NEXT: %bb.
+;CHECK-NEXT: v_lshlrev_b32_e32 v{{[0-9]}}, 6, v0
+;CHECK-NEXT: buffer_load_dwordx4 v[{{[0-9]}}:{{[0-9]}}], v{{[0-9]}}, s[0:3], 0 offen offset:4
+;CHECK-NEXT: buffer_load_dwordx2 v[{{[0-9]}}:{{[0-9]}}], v{{[0-9]}}, s[0:3], 0 offen offset:28
+;CHECK: s_waitcnt
+define amdgpu_ps void @buffer_load_x1_offen_merged_or(<4 x i32> inreg %rsrc, i32 %inp) {
+main_body:
+  %a = shl i32 %inp, 6
+  %a1 = or i32 %a, 4
+  %a2 = or i32 %a, 8
+  %a3 = or i32 %a, 12
+  %a4 = or i32 %a, 16
+  %a5 = or i32 %a, 28
+  %a6 = or i32 %a, 32
+  %r1 = call float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32> %rsrc, i32 %a1, i32 0, i32 0)
+  %r2 = call float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32> %rsrc, i32 %a2, i32 0, i32 0)
+  %r3 = call float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32> %rsrc, i32 %a3, i32 0, i32 0)
+  %r4 = call float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32> %rsrc, i32 %a4, i32 0, i32 0)
+  %r5 = call float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32> %rsrc, i32 %a5, i32 0, i32 0)
+  %r6 = call float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32> %rsrc, i32 %a6, i32 0, i32 0)
+  call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %r1, float %r2, float %r3, float %r4, i1 true, i1 true)
+  call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %r5, float %r6, float undef, float undef, i1 true, i1 true)
+  ret void
+}
+
 ;CHECK-LABEL: {{^}}buffer_load_x1_offen_merged_glc_slc:
 ;CHECK-NEXT: %bb.
 ;CHECK-NEXT: buffer_load_dwordx2 v[{{[0-9]}}:{{[0-9]}}], v0, s[0:3], 0 offen offset:4{{$}}
@@ -146,12 +172,32 @@ main_body:
   ret void
 }
 
-;CHECK-LABEL: {{^}}buffer_load_x2_offen_merged:
+;CHECK-LABEL: {{^}}buffer_load_x2_offen_merged_and:
 ;CHECK-NEXT: %bb.
 ;CHECK-NEXT: buffer_load_dwordx4 v[{{[0-9]}}:{{[0-9]}}], v0, s[0:3], 0 offen offset:4
 ;CHECK: s_waitcnt
-define amdgpu_ps void @buffer_load_x2_offen_merged(<4 x i32> inreg %rsrc, i32 %a) {
+define amdgpu_ps void @buffer_load_x2_offen_merged_and(<4 x i32> inreg %rsrc, i32 %a) {
+main_body:
+  %a1 = add i32 %a, 4
+  %a2 = add i32 %a, 12
+  %vr1 = call <2 x float> @llvm.amdgcn.raw.buffer.load.v2f32(<4 x i32> %rsrc, i32 %a1, i32 0, i32 0)
+  %vr2 = call <2 x float> @llvm.amdgcn.raw.buffer.load.v2f32(<4 x i32> %rsrc, i32 %a2, i32 0, i32 0)
+  %r1 = extractelement <2 x float> %vr1, i32 0
+  %r2 = extractelement <2 x float> %vr1, i32 1
+  %r3 = extractelement <2 x float> %vr2, i32 0
+  %r4 = extractelement <2 x float> %vr2, i32 1
+  call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %r1, float %r2, float %r3, float %r4, i1 true, i1 true)
+  ret void
+}
+
+;CHECK-LABEL: {{^}}buffer_load_x2_offen_merged_or:
+;CHECK-NEXT: %bb.
+;CHECK-NEXT: v_lshlrev_b32_e32 v{{[0-9]}}, 4, v0
+;CHECK-NEXT: buffer_load_dwordx4 v[{{[0-9]}}:{{[0-9]}}], v{{[0-9]}}, s[0:3], 0 offen offset:4
+;CHECK: s_waitcnt
+define amdgpu_ps void @buffer_load_x2_offen_merged_or(<4 x i32> inreg %rsrc, i32 %inp) {
 main_body:
+  %a = shl i32 %inp, 4
   %a1 = add i32 %a, 4
   %a2 = add i32 %a, 12
   %vr1 = call <2 x float> @llvm.amdgcn.raw.buffer.load.v2f32(<4 x i32> %rsrc, i32 %a1, i32 0, i32 0)
diff --git a/test/CodeGen/AMDGPU/llvm.amdgcn.raw.buffer.store.ll b/test/CodeGen/AMDGPU/llvm.amdgcn.raw.buffer.store.ll
index afb2ef803f014455bef8ad99bb638b47df18b711..4f39867e46aba7b3bff95e13f69029612d709c33 100644
--- a/test/CodeGen/AMDGPU/llvm.amdgcn.raw.buffer.store.ll
+++ b/test/CodeGen/AMDGPU/llvm.amdgcn.raw.buffer.store.ll
@@ -67,11 +67,11 @@ main_body:
   ret void
 }
 
-;CHECK-LABEL: {{^}}buffer_store_x1_offen_merged:
+;CHECK-LABEL: {{^}}buffer_store_x1_offen_merged_and:
 ;CHECK-NOT: s_waitcnt
 ;CHECK-DAG: buffer_store_dwordx4 v[{{[0-9]}}:{{[0-9]}}], v0, s[0:3], 0 offen offset:4
 ;CHECK-DAG: buffer_store_dwordx2 v[{{[0-9]}}:{{[0-9]}}], v0, s[0:3], 0 offen offset:28
-define amdgpu_ps void @buffer_store_x1_offen_merged(<4 x i32> inreg %rsrc, i32 %a, float %v1, float %v2, float %v3, float %v4, float %v5, float %v6) {
+define amdgpu_ps void @buffer_store_x1_offen_merged_and(<4 x i32> inreg %rsrc, i32 %a, float %v1, float %v2, float %v3, float %v4, float %v5, float %v6) {
   %a1 = add i32 %a, 4
   %a2 = add i32 %a, 8
   %a3 = add i32 %a, 12
@@ -87,6 +87,28 @@ define amdgpu_ps void @buffer_store_x1_offen_merged(<4 x i32> inreg %rsrc, i32 %
   ret void
 }
 
+;CHECK-LABEL: {{^}}buffer_store_x1_offen_merged_or:
+;CHECK-NOT: s_waitcnt
+;CHECK-DAG: buffer_store_dwordx4 v[{{[0-9]}}:{{[0-9]}}], v{{[0-9]}}, s[0:3], 0 offen offset:4
+;CHECK-DAG: buffer_store_dwordx2 v[{{[0-9]}}:{{[0-9]}}], v{{[0-9]}}, s[0:3], 0 offen offset:28
+define amdgpu_ps void @buffer_store_x1_offen_merged_or(<4 x i32> inreg %rsrc, i32 %inp, float %v1, float %v2, float %v3, float %v4, float %v5, float %v6) {
+  %a = shl i32 %inp, 6
+  %a1 = add i32 %a, 4
+  %a2 = add i32 %a, 8
+  %a3 = add i32 %a, 12
+  %a4 = add i32 %a, 16
+  %a5 = add i32 %a, 28
+  %a6 = add i32 %a, 32
+  call void @llvm.amdgcn.raw.buffer.store.f32(float %v1, <4 x i32> %rsrc, i32 %a1, i32 0, i32 0)
+  call void @llvm.amdgcn.raw.buffer.store.f32(float %v2, <4 x i32> %rsrc, i32 %a2, i32 0, i32 0)
+  call void @llvm.amdgcn.raw.buffer.store.f32(float %v3, <4 x i32> %rsrc, i32 %a3, i32 0, i32 0)
+  call void @llvm.amdgcn.raw.buffer.store.f32(float %v4, <4 x i32> %rsrc, i32 %a4, i32 0, i32 0)
+  call void @llvm.amdgcn.raw.buffer.store.f32(float %v5, <4 x i32> %rsrc, i32 %a5, i32 0, i32 0)
+  call void @llvm.amdgcn.raw.buffer.store.f32(float %v6, <4 x i32> %rsrc, i32 %a6, i32 0, i32 0)
+  ret void
+}
+
+
 ;CHECK-LABEL: {{^}}buffer_store_x1_offen_merged_glc_slc:
 ;CHECK-NOT: s_waitcnt
 ;CHECK-DAG: buffer_store_dwordx2 v[{{[0-9]}}:{{[0-9]}}], v0, s[0:3], 0 offen offset:4{{$}}
@@ -108,10 +130,22 @@ define amdgpu_ps void @buffer_store_x1_offen_merged_glc_slc(<4 x i32> inreg %rsr
   ret void
 }
 
-;CHECK-LABEL: {{^}}buffer_store_x2_offen_merged:
+;CHECK-LABEL: {{^}}buffer_store_x2_offen_merged_and:
 ;CHECK-NOT: s_waitcnt
 ;CHECK: buffer_store_dwordx4 v[{{[0-9]}}:{{[0-9]}}], v0, s[0:3], 0 offen offset:4
-define amdgpu_ps void @buffer_store_x2_offen_merged(<4 x i32> inreg %rsrc, i32 %a, <2 x float> %v1, <2 x float> %v2) {
+define amdgpu_ps void @buffer_store_x2_offen_merged_and(<4 x i32> inreg %rsrc, i32 %a, <2 x float> %v1, <2 x float> %v2) {
+  %a1 = add i32 %a, 4
+  %a2 = add i32 %a, 12
+  call void @llvm.amdgcn.raw.buffer.store.v2f32(<2 x float> %v1, <4 x i32> %rsrc, i32 %a1, i32 0, i32 0)
+  call void @llvm.amdgcn.raw.buffer.store.v2f32(<2 x float> %v2, <4 x i32> %rsrc, i32 %a2, i32 0, i32 0)
+  ret void
+}
+
+;CHECK-LABEL: {{^}}buffer_store_x2_offen_merged_or:
+;CHECK-NOT: s_waitcnt
+;CHECK: buffer_store_dwordx4 v[{{[0-9]}}:{{[0-9]}}], v{{[0-9]}}, s[0:3], 0 offen offset:4
+define amdgpu_ps void @buffer_store_x2_offen_merged_or(<4 x i32> inreg %rsrc, i32 %inp, <2 x float> %v1, <2 x float> %v2) {
+  %a = shl i32 %inp, 4
   %a1 = add i32 %a, 4
   %a2 = add i32 %a, 12
   call void @llvm.amdgcn.raw.buffer.store.v2f32(<2 x float> %v1, <4 x i32> %rsrc, i32 %a1, i32 0, i32 0)
diff --git a/test/CodeGen/AMDGPU/llvm.dbg.value.ll b/test/CodeGen/AMDGPU/llvm.dbg.value.ll
index be1920f3ca4cc84c9d73affe0f81e3fe164df3fb..697fdfb3695ec99db21bb56ab5928a7893d667fa 100644
--- a/test/CodeGen/AMDGPU/llvm.dbg.value.ll
+++ b/test/CodeGen/AMDGPU/llvm.dbg.value.ll
@@ -4,6 +4,7 @@
 ; GCN-LABEL: {{^}}test_debug_value:
 ; NOOPT: .loc	1 1 42 prologue_end     ; /tmp/test_debug_value.cl:1:42
 ; NOOPT-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x0
+; NOOPT-NEXT: .Ltmp
 ; NOOPT-NEXT: ;DEBUG_VALUE: test_debug_value:globalptr_arg <- $sgpr4_sgpr5
 
 ; GCN: flat_store_dword
diff --git a/test/CodeGen/AMDGPU/local-atomics-fp.ll b/test/CodeGen/AMDGPU/local-atomics-fp.ll
new file mode 100644
index 0000000000000000000000000000000000000000..17c45c1a2f4433c6e5d851c456f6ba5fef6c1195
--- /dev/null
+++ b/test/CodeGen/AMDGPU/local-atomics-fp.ll
@@ -0,0 +1,109 @@
+; RUN: llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,VI,GFX678,HAS-ATOMICS %s
+; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX9,HAS-ATOMICS %s
+; RUN: llc -march=amdgcn -mcpu=tahiti -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX678,NO-ATOMICS %s
+; RUN: llc -march=amdgcn -mcpu=hawaii -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX678,NO-ATOMICS %s
+
+; GCN-LABEL: {{^}}lds_atomic_fadd_ret_f32:
+; GFX678-DAG: s_mov_b32 m0
+; GFX9-NOT: m0
+; HAS-ATOMICS-DAG: v_mov_b32_e32 [[K:v[0-9]+]], 4.0
+; HAS-ATOMICS: ds_add_rtn_f32 v0, v0, [[K]]
+
+; NO-ATOMICS: ds_read_b32
+; NO-ATOMICS: v_add_f32
+; NO-ATOMICS: ds_cmpst_rtn_b32
+; NO-ATOMICS: s_cbranch_execnz
+define float @lds_atomic_fadd_ret_f32(float addrspace(3)* %ptr) nounwind {
+  %result = atomicrmw fadd float addrspace(3)* %ptr, float 4.0 seq_cst
+  ret float %result
+}
+
+; GCN-LABEL: {{^}}lds_atomic_fadd_noret_f32:
+; GFX678-DAG: s_mov_b32 m0
+; GFX9-NOT: m0
+; HAS-ATOMICS-DAG: v_mov_b32_e32 [[K:v[0-9]+]], 4.0
+; HAS-ATOMICS: ds_add_f32 v0, [[K]]
+define void @lds_atomic_fadd_noret_f32(float addrspace(3)* %ptr) nounwind {
+  %result = atomicrmw fadd float addrspace(3)* %ptr, float 4.0 seq_cst
+  ret void
+}
+
+; GCN-LABEL: {{^}}lds_ds_fadd:
+; VI-DAG: s_mov_b32 m0
+; GFX9-NOT: m0
+; HAS-ATOMICS-DAG: v_mov_b32_e32 [[V0:v[0-9]+]], 0x42280000
+; HAS-ATOMICS: ds_add_rtn_f32 [[V2:v[0-9]+]], [[V1:v[0-9]+]], [[V0]] offset:32
+; HAS-ATOMICS: ds_add_f32 [[V3:v[0-9]+]], [[V0]] offset:64
+; HAS-ATOMICS: s_waitcnt lgkmcnt(1)
+; HAS-ATOMICS: ds_add_rtn_f32 {{v[0-9]+}}, {{v[0-9]+}}, [[V2]]
+define amdgpu_kernel void @lds_ds_fadd(float addrspace(1)* %out, float addrspace(3)* %ptrf, i32 %idx) {
+  %idx.add = add nuw i32 %idx, 4
+  %shl0 = shl i32 %idx.add, 3
+  %shl1 = shl i32 %idx.add, 4
+  %ptr0 = inttoptr i32 %shl0 to float addrspace(3)*
+  %ptr1 = inttoptr i32 %shl1 to float addrspace(3)*
+  %a1 = atomicrmw fadd float addrspace(3)* %ptr0, float 4.2e+1 seq_cst
+  %a2 = atomicrmw fadd float addrspace(3)* %ptr1, float 4.2e+1 seq_cst
+  %a3 = atomicrmw fadd float addrspace(3)* %ptrf, float %a1 seq_cst
+  store float %a3, float addrspace(1)* %out
+  ret void
+}
+
+; GCN-LABEL: {{^}}lds_atomic_fadd_ret_f64:
+; GCN: ds_read_b64
+; GCN: v_add_f64
+; GCN: ds_cmpst_rtn_b64
+; GCN: s_cbranch_execnz
+define double @lds_atomic_fadd_ret_f64(double addrspace(3)* %ptr) nounwind {
+  %result = atomicrmw fadd double addrspace(3)* %ptr, double 4.0 seq_cst
+  ret double %result
+}
+
+; GCN-LABEL: {{^}}lds_atomic_fadd_noret_f64:
+; GCN: ds_read_b64
+; GCN: v_add_f64
+; GCN: ds_cmpst_rtn_b64
+; GCN: s_cbranch_execnz
+define void @lds_atomic_fadd_noret_f64(double addrspace(3)* %ptr) nounwind {
+  %result = atomicrmw fadd double addrspace(3)* %ptr, double 4.0 seq_cst
+  ret void
+}
+
+; GCN-LABEL: {{^}}lds_atomic_fsub_ret_f32:
+; GCN: ds_read_b32
+; GCN: v_sub_f32
+; GCN: ds_cmpst_rtn_b32
+; GCN: s_cbranch_execnz
+define float @lds_atomic_fsub_ret_f32(float addrspace(3)* %ptr, float %val) nounwind {
+  %result = atomicrmw fsub float addrspace(3)* %ptr, float %val seq_cst
+  ret float %result
+}
+
+; GCN-LABEL: {{^}}lds_atomic_fsub_noret_f32:
+; GCN: ds_read_b32
+; GCN: v_sub_f32
+; GCN: ds_cmpst_rtn_b32
+define void @lds_atomic_fsub_noret_f32(float addrspace(3)* %ptr, float %val) nounwind {
+  %result = atomicrmw fsub float addrspace(3)* %ptr, float %val seq_cst
+  ret void
+}
+
+; GCN-LABEL: {{^}}lds_atomic_fsub_ret_f64:
+; GCN: ds_read_b64
+; GCN: v_add_f64 v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}}, -v{{\[[0-9]+:[0-9]+\]}}
+; GCN: ds_cmpst_rtn_b64
+
+define double @lds_atomic_fsub_ret_f64(double addrspace(3)* %ptr, double %val) nounwind {
+  %result = atomicrmw fsub double addrspace(3)* %ptr, double %val seq_cst
+  ret double %result
+}
+
+; GCN-LABEL: {{^}}lds_atomic_fsub_noret_f64:
+; GCN: ds_read_b64
+; GCN: v_add_f64 v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}}, -v{{\[[0-9]+:[0-9]+\]}}
+; GCN: ds_cmpst_rtn_b64
+; GCN: s_cbranch_execnz
+define void @lds_atomic_fsub_noret_f64(double addrspace(3)* %ptr, double %val) nounwind {
+  %result = atomicrmw fsub double addrspace(3)* %ptr, double %val seq_cst
+  ret void
+}
diff --git a/test/CodeGen/AMDGPU/local-atomics.ll b/test/CodeGen/AMDGPU/local-atomics.ll
index 9cf4e6a131efcf4461a0865338e01088d438683a..fcbe38e40070b4072a2208127d72b5d1d9bd2207 100644
--- a/test/CodeGen/AMDGPU/local-atomics.ll
+++ b/test/CodeGen/AMDGPU/local-atomics.ll
@@ -36,6 +36,20 @@ define amdgpu_kernel void @lds_atomic_xchg_ret_i32_offset(i32 addrspace(1)* %out
   ret void
 }
 
+; FUNC-LABEL: {{^}}lds_atomic_xchg_ret_f32_offset:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
+; EG: LDS_WRXCHG_RET *
+; GCN: ds_wrxchg_rtn_b32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} offset:16
+; GCN: s_endpgm
+define amdgpu_kernel void @lds_atomic_xchg_ret_f32_offset(float addrspace(1)* %out, float addrspace(3)* %ptr) nounwind {
+  %gep = getelementptr float, float addrspace(3)* %ptr, i32 4
+  %result = atomicrmw xchg float addrspace(3)* %gep, float 4.0 seq_cst
+  store float %result, float addrspace(1)* %out, align 4
+  ret void
+}
+
 ; XXX - Is it really necessary to load 4 into VGPR?
 ; FUNC-LABEL: {{^}}lds_atomic_add_ret_i32:
 ; EG: LDS_ADD_RET *
diff --git a/test/CodeGen/AMDGPU/local-atomics64.ll b/test/CodeGen/AMDGPU/local-atomics64.ll
index af64f2b1d578ee16eec77410793042a3c6e58a74..7a527f4f5dfb5ec5a5a6212e51771910dd330311 100644
--- a/test/CodeGen/AMDGPU/local-atomics64.ll
+++ b/test/CodeGen/AMDGPU/local-atomics64.ll
@@ -27,6 +27,19 @@ define amdgpu_kernel void @lds_atomic_xchg_ret_i64_offset(i64 addrspace(1)* %out
   ret void
 }
 
+; GCN-LABEL: {{^}}lds_atomic_xchg_ret_f64_offset:
+; SICIVI: s_mov_b32 m0
+; GFX9-NOT: m0
+
+; GCN: ds_wrxchg_rtn_b64 {{.*}} offset:32
+; GCN: s_endpgm
+define amdgpu_kernel void @lds_atomic_xchg_ret_f64_offset(double addrspace(1)* %out, double addrspace(3)* %ptr) nounwind {
+  %gep = getelementptr double, double addrspace(3)* %ptr, i32 4
+  %result = atomicrmw xchg double addrspace(3)* %gep, double 4.0 seq_cst
+  store double %result, double addrspace(1)* %out, align 8
+  ret void
+}
+
 ; GCN-LABEL: {{^}}lds_atomic_add_ret_i64:
 ; SICIVI: s_mov_b32 m0
 ; GFX9-NOT: m0
diff --git a/test/CodeGen/AMDGPU/lower-kernargs.ll b/test/CodeGen/AMDGPU/lower-kernargs.ll
index 630aa4a96bfb386cebf71d1cf78689f1a45ca061..3a489d58b48b8da66ccd2525d8ad4ecf452e4ccc 100644
--- a/test/CodeGen/AMDGPU/lower-kernargs.ll
+++ b/test/CodeGen/AMDGPU/lower-kernargs.ll
@@ -306,20 +306,18 @@ define amdgpu_kernel void @kern_v3i32(<3 x i32> %arg0) {
 ; HSA-LABEL: @kern_v3i32(
 ; HSA-NEXT:    [[KERN_V3I32_KERNARG_SEGMENT:%.*]] = call nonnull align 16 dereferenceable(16) i8 addrspace(4)* @llvm.amdgcn.kernarg.segment.ptr()
 ; HSA-NEXT:    [[ARG0_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, i8 addrspace(4)* [[KERN_V3I32_KERNARG_SEGMENT]], i64 0
-; HSA-NEXT:    [[ARG0_KERNARG_OFFSET_CAST:%.*]] = bitcast i8 addrspace(4)* [[ARG0_KERNARG_OFFSET]] to <3 x i32> addrspace(4)*
-; HSA-NEXT:    [[TMP1:%.*]] = bitcast <3 x i32> addrspace(4)* [[ARG0_KERNARG_OFFSET_CAST]] to <4 x i32> addrspace(4)*
-; HSA-NEXT:    [[TMP2:%.*]] = load <4 x i32>, <4 x i32> addrspace(4)* [[TMP1]], align 16, !invariant.load !0
-; HSA-NEXT:    [[ARG0_LOAD:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> undef, <3 x i32> <i32 0, i32 1, i32 2>
+; HSA-NEXT:    [[ARG0_KERNARG_OFFSET_CAST:%.*]] = bitcast i8 addrspace(4)* [[ARG0_KERNARG_OFFSET]] to <4 x i32> addrspace(4)*
+; HSA-NEXT:    [[TMP:%.*]] = load <4 x i32>, <4 x i32> addrspace(4)* [[ARG0_KERNARG_OFFSET_CAST]], align 16, !invariant.load !0
+; HSA-NEXT:    [[ARG0_LOAD:%.*]] = shufflevector <4 x i32> [[TMP]], <4 x i32> undef, <3 x i32> <i32 0, i32 1, i32 2>
 ; HSA-NEXT:    store <3 x i32> [[ARG0_LOAD]], <3 x i32> addrspace(1)* undef, align 4
 ; HSA-NEXT:    ret void
 ;
 ; MESA-LABEL: @kern_v3i32(
 ; MESA-NEXT:    [[KERN_V3I32_KERNARG_SEGMENT:%.*]] = call nonnull align 16 dereferenceable(52) i8 addrspace(4)* @llvm.amdgcn.kernarg.segment.ptr()
 ; MESA-NEXT:    [[ARG0_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, i8 addrspace(4)* [[KERN_V3I32_KERNARG_SEGMENT]], i64 36
-; MESA-NEXT:    [[ARG0_KERNARG_OFFSET_CAST:%.*]] = bitcast i8 addrspace(4)* [[ARG0_KERNARG_OFFSET]] to <3 x i32> addrspace(4)*
-; MESA-NEXT:    [[TMP1:%.*]] = bitcast <3 x i32> addrspace(4)* [[ARG0_KERNARG_OFFSET_CAST]] to <4 x i32> addrspace(4)*
-; MESA-NEXT:    [[TMP2:%.*]] = load <4 x i32>, <4 x i32> addrspace(4)* [[TMP1]], align 4, !invariant.load !0
-; MESA-NEXT:    [[ARG0_LOAD:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> undef, <3 x i32> <i32 0, i32 1, i32 2>
+; MESA-NEXT:    [[ARG0_KERNARG_OFFSET_CAST:%.*]] = bitcast i8 addrspace(4)* [[ARG0_KERNARG_OFFSET]] to <4 x i32> addrspace(4)*
+; MESA-NEXT:    [[TMP:%.*]] = load <4 x i32>, <4 x i32> addrspace(4)* [[ARG0_KERNARG_OFFSET_CAST]], align 4, !invariant.load !0
+; MESA-NEXT:    [[ARG0_LOAD:%.*]] = shufflevector <4 x i32> [[TMP]], <4 x i32> undef, <3 x i32> <i32 0, i32 1, i32 2>
 ; MESA-NEXT:    store <3 x i32> [[ARG0_LOAD]], <3 x i32> addrspace(1)* undef, align 4
 ; MESA-NEXT:    ret void
 ;
@@ -397,10 +395,9 @@ define amdgpu_kernel void @kern_i32_v3i32(i32 %arg0, <3 x i32> %arg1) {
 ; HSA-NEXT:    [[ARG0_KERNARG_OFFSET_CAST:%.*]] = bitcast i8 addrspace(4)* [[ARG0_KERNARG_OFFSET]] to i32 addrspace(4)*
 ; HSA-NEXT:    [[ARG0_LOAD:%.*]] = load i32, i32 addrspace(4)* [[ARG0_KERNARG_OFFSET_CAST]], align 16, !invariant.load !0
 ; HSA-NEXT:    [[ARG1_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, i8 addrspace(4)* [[KERN_I32_V3I32_KERNARG_SEGMENT]], i64 16
-; HSA-NEXT:    [[ARG1_KERNARG_OFFSET_CAST:%.*]] = bitcast i8 addrspace(4)* [[ARG1_KERNARG_OFFSET]] to <3 x i32> addrspace(4)*
-; HSA-NEXT:    [[TMP1:%.*]] = bitcast <3 x i32> addrspace(4)* [[ARG1_KERNARG_OFFSET_CAST]] to <4 x i32> addrspace(4)*
-; HSA-NEXT:    [[TMP2:%.*]] = load <4 x i32>, <4 x i32> addrspace(4)* [[TMP1]], align 16, !invariant.load !0
-; HSA-NEXT:    [[ARG1_LOAD:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> undef, <3 x i32> <i32 0, i32 1, i32 2>
+; HSA-NEXT:    [[ARG1_KERNARG_OFFSET_CAST:%.*]] = bitcast i8 addrspace(4)* [[ARG1_KERNARG_OFFSET]] to <4 x i32> addrspace(4)*
+; HSA-NEXT:    [[TMP:%.*]] = load <4 x i32>, <4 x i32> addrspace(4)* [[ARG1_KERNARG_OFFSET_CAST]], align 16, !invariant.load !0
+; HSA-NEXT:    [[ARG1_LOAD:%.*]] = shufflevector <4 x i32> [[TMP]], <4 x i32> undef, <3 x i32> <i32 0, i32 1, i32 2>
 ; HSA-NEXT:    store i32 [[ARG0_LOAD]], i32 addrspace(1)* undef
 ; HSA-NEXT:    store <3 x i32> [[ARG1_LOAD]], <3 x i32> addrspace(1)* undef, align 4
 ; HSA-NEXT:    ret void
@@ -411,10 +408,9 @@ define amdgpu_kernel void @kern_i32_v3i32(i32 %arg0, <3 x i32> %arg1) {
 ; MESA-NEXT:    [[ARG0_KERNARG_OFFSET_CAST:%.*]] = bitcast i8 addrspace(4)* [[ARG0_KERNARG_OFFSET]] to i32 addrspace(4)*
 ; MESA-NEXT:    [[ARG0_LOAD:%.*]] = load i32, i32 addrspace(4)* [[ARG0_KERNARG_OFFSET_CAST]], align 4, !invariant.load !0
 ; MESA-NEXT:    [[ARG1_KERNARG_OFFSET:%.*]] = getelementptr inbounds i8, i8 addrspace(4)* [[KERN_I32_V3I32_KERNARG_SEGMENT]], i64 52
-; MESA-NEXT:    [[ARG1_KERNARG_OFFSET_CAST:%.*]] = bitcast i8 addrspace(4)* [[ARG1_KERNARG_OFFSET]] to <3 x i32> addrspace(4)*
-; MESA-NEXT:    [[TMP1:%.*]] = bitcast <3 x i32> addrspace(4)* [[ARG1_KERNARG_OFFSET_CAST]] to <4 x i32> addrspace(4)*
-; MESA-NEXT:    [[TMP2:%.*]] = load <4 x i32>, <4 x i32> addrspace(4)* [[TMP1]], align 4, !invariant.load !0
-; MESA-NEXT:    [[ARG1_LOAD:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> undef, <3 x i32> <i32 0, i32 1, i32 2>
+; MESA-NEXT:    [[ARG1_KERNARG_OFFSET_CAST:%.*]] = bitcast i8 addrspace(4)* [[ARG1_KERNARG_OFFSET]] to <4 x i32> addrspace(4)*
+; MESA-NEXT:    [[TMP:%.*]] = load <4 x i32>, <4 x i32> addrspace(4)* [[ARG1_KERNARG_OFFSET_CAST]], align 4, !invariant.load !0
+; MESA-NEXT:    [[ARG1_LOAD:%.*]] = shufflevector <4 x i32> [[TMP]], <4 x i32> undef, <3 x i32> <i32 0, i32 1, i32 2>
 ; MESA-NEXT:    store i32 [[ARG0_LOAD]], i32 addrspace(1)* undef
 ; MESA-NEXT:    store <3 x i32> [[ARG1_LOAD]], <3 x i32> addrspace(1)* undef, align 4
 ; MESA-NEXT:    ret void
diff --git a/test/CodeGen/AMDGPU/lshl64-to-32.ll b/test/CodeGen/AMDGPU/lshl64-to-32.ll
index b7e5eab275c13df813c7410d464455612efa0b11..711979591135443e9dc2f388dcb081bb6cb7d862 100644
--- a/test/CodeGen/AMDGPU/lshl64-to-32.ll
+++ b/test/CodeGen/AMDGPU/lshl64-to-32.ll
@@ -1,9 +1,19 @@
-; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=amdgcn-- -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
 
-; GCN-LABEL: {{^}}zext_shl64_to_32:
-; GCN: s_lshl_b32
-; GCN-NOT: s_lshl_b64
 define amdgpu_kernel void @zext_shl64_to_32(i64 addrspace(1)* nocapture %out, i32 %x) {
+; GCN-LABEL: zext_shl64_to_32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dword s4, s[0:1], 0xb
+; GCN-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; GCN-NEXT:    s_mov_b32 s3, 0xf000
+; GCN-NEXT:    s_mov_b32 s2, -1
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_lshl_b32 s4, s4, 2
+; GCN-NEXT:    v_mov_b32_e32 v1, 0
+; GCN-NEXT:    v_mov_b32_e32 v0, s4
+; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GCN-NEXT:    s_endpgm
   %and = and i32 %x, 1073741823
   %ext = zext i32 %and to i64
   %shl = shl i64 %ext, 2
@@ -11,10 +21,20 @@ define amdgpu_kernel void @zext_shl64_to_32(i64 addrspace(1)* nocapture %out, i3
   ret void
 }
 
-; GCN-LABEL: {{^}}sext_shl64_to_32:
-; GCN: s_lshl_b32
-; GCN-NOT: s_lshl_b64
 define amdgpu_kernel void @sext_shl64_to_32(i64 addrspace(1)* nocapture %out, i32 %x) {
+; GCN-LABEL: sext_shl64_to_32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dword s4, s[0:1], 0xb
+; GCN-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; GCN-NEXT:    s_mov_b32 s3, 0xf000
+; GCN-NEXT:    s_mov_b32 s2, -1
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_and_b32 s4, s4, 0x1fffffff
+; GCN-NEXT:    s_lshl_b32 s4, s4, 2
+; GCN-NEXT:    v_mov_b32_e32 v1, 0
+; GCN-NEXT:    v_mov_b32_e32 v0, s4
+; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GCN-NEXT:    s_endpgm
   %and = and i32 %x, 536870911
   %ext = sext i32 %and to i64
   %shl = shl i64 %ext, 2
@@ -22,10 +42,21 @@ define amdgpu_kernel void @sext_shl64_to_32(i64 addrspace(1)* nocapture %out, i3
   ret void
 }
 
-; GCN-LABEL: {{^}}zext_shl64_overflow:
-; GCN: s_lshl_b64
-; GCN-NOT: s_lshl_b32
 define amdgpu_kernel void @zext_shl64_overflow(i64 addrspace(1)* nocapture %out, i32 %x) {
+; GCN-LABEL: zext_shl64_overflow:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dword s2, s[0:1], 0xb
+; GCN-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; GCN-NEXT:    s_mov_b32 s3, 0xf000
+; GCN-NEXT:    s_mov_b32 s5, 0
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_and_b32 s4, s2, 0x7fffffff
+; GCN-NEXT:    s_lshl_b64 s[4:5], s[4:5], 2
+; GCN-NEXT:    s_mov_b32 s2, -1
+; GCN-NEXT:    v_mov_b32_e32 v0, s4
+; GCN-NEXT:    v_mov_b32_e32 v1, s5
+; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GCN-NEXT:    s_endpgm
   %and = and i32 %x, 2147483647
   %ext = zext i32 %and to i64
   %shl = shl i64 %ext, 2
@@ -33,10 +64,21 @@ define amdgpu_kernel void @zext_shl64_overflow(i64 addrspace(1)* nocapture %out,
   ret void
 }
 
-; GCN-LABEL: {{^}}sext_shl64_overflow:
-; GCN: s_lshl_b64
-; GCN-NOT: s_lshl_b32
 define amdgpu_kernel void @sext_shl64_overflow(i64 addrspace(1)* nocapture %out, i32 %x) {
+; GCN-LABEL: sext_shl64_overflow:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_load_dword s2, s[0:1], 0xb
+; GCN-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; GCN-NEXT:    s_mov_b32 s3, 0xf000
+; GCN-NEXT:    s_mov_b32 s5, 0
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_and_b32 s4, s2, 0x7fffffff
+; GCN-NEXT:    s_lshl_b64 s[4:5], s[4:5], 2
+; GCN-NEXT:    s_mov_b32 s2, -1
+; GCN-NEXT:    v_mov_b32_e32 v0, s4
+; GCN-NEXT:    v_mov_b32_e32 v1, s5
+; GCN-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GCN-NEXT:    s_endpgm
   %and = and i32 %x, 2147483647
   %ext = sext i32 %and to i64
   %shl = shl i64 %ext, 2
@@ -44,10 +86,19 @@ define amdgpu_kernel void @sext_shl64_overflow(i64 addrspace(1)* nocapture %out,
   ret void
 }
 
-; GCN-LABEL: {{^}}mulu24_shl64:
-; GCN: v_mul_u32_u24_e32 [[M:v[0-9]+]], 7, v{{[0-9]+}}
-; GCN: v_lshlrev_b32_e32 v{{[0-9]+}}, 2, [[M]]
 define amdgpu_kernel void @mulu24_shl64(i32 addrspace(1)* nocapture %arg) {
+; GCN-LABEL: mulu24_shl64:
+; GCN:       ; %bb.0: ; %bb
+; GCN-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; GCN-NEXT:    v_and_b32_e32 v0, 6, v0
+; GCN-NEXT:    s_mov_b32 s3, 0xf000
+; GCN-NEXT:    v_mul_u32_u24_e32 v0, 7, v0
+; GCN-NEXT:    s_mov_b32 s2, 0
+; GCN-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GCN-NEXT:    v_mov_b32_e32 v1, 0
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v1, v[0:1], s[0:3], 0 addr64
+; GCN-NEXT:    s_endpgm
 bb:
   %tmp = tail call i32 @llvm.amdgcn.workitem.id.x()
   %tmp1 = and i32 %tmp, 6
@@ -58,10 +109,26 @@ bb:
   ret void
 }
 
-; GCN-LABEL: {{^}}muli24_shl64:
-; GCN: v_mul_i32_i24_e32 [[M:v[0-9]+]], -7, v{{[0-9]+}}
-; GCN: v_lshlrev_b32_e32 v{{[0-9]+}}, 3, [[M]]
 define amdgpu_kernel void @muli24_shl64(i64 addrspace(1)* nocapture %arg, i32 addrspace(1)* nocapture readonly %arg1) {
+; GCN-LABEL: muli24_shl64:
+; GCN:       ; %bb.0: ; %bb
+; GCN-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
+; GCN-NEXT:    s_mov_b32 s7, 0xf000
+; GCN-NEXT:    s_mov_b32 s6, 0
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 2, v0
+; GCN-NEXT:    v_mov_b32_e32 v2, 0
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; GCN-NEXT:    buffer_load_dword v1, v[1:2], s[4:7], 0 addr64
+; GCN-NEXT:    s_mov_b64 s[2:3], s[6:7]
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_or_b32_e32 v1, 0x800000, v1
+; GCN-NEXT:    v_mul_i32_i24_e32 v1, 0xfffff9, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 3, v1
+; GCN-NEXT:    v_lshlrev_b32_e32 v3, 3, v0
+; GCN-NEXT:    v_mov_b32_e32 v4, v2
+; GCN-NEXT:    buffer_store_dwordx2 v[1:2], v[3:4], s[0:3], 0 addr64
+; GCN-NEXT:    s_endpgm
 bb:
   %tmp = tail call i32 @llvm.amdgcn.workitem.id.x()
   %tmp2 = sext i32 %tmp to i64
diff --git a/test/CodeGen/AMDGPU/mem-builtins.ll b/test/CodeGen/AMDGPU/mem-builtins.ll
index c7607a32f1b4f144ebcaed024e1db1ea4be77bf2..37bfac346044d28ad3efda5ba10558a724d90054 100644
--- a/test/CodeGen/AMDGPU/mem-builtins.ll
+++ b/test/CodeGen/AMDGPU/mem-builtins.ll
@@ -1,12 +1,12 @@
 ; RUN: not llc -march=r600 < %s 2>&1 | FileCheck -check-prefix=ERROR %s
 ; RUN: llc -march=amdgcn < %s | FileCheck -check-prefix=GCN %s
 
-declare i32 @memcmp(i8 addrspace(1)* readonly nocapture, i8 addrspace(1)* readonly nocapture, i64) #0
-declare i8 addrspace(1)* @memchr(i8 addrspace(1)* readonly nocapture, i32, i64) #1
-declare i8* @strcpy(i8* nocapture, i8* readonly nocapture) #0
-declare i32 @strlen(i8* nocapture) #1
-declare i32 @strnlen(i8* nocapture, i32) #1
-declare i32 @strcmp(i8* nocapture, i8* nocapture) #1
+declare hidden i32 @memcmp(i8 addrspace(1)* readonly nocapture, i8 addrspace(1)* readonly nocapture, i64) #0
+declare hidden i8 addrspace(1)* @memchr(i8 addrspace(1)* readonly nocapture, i32, i64) #1
+declare hidden i8* @strcpy(i8* nocapture, i8* readonly nocapture) #0
+declare hidden i32 @strlen(i8* nocapture) #1
+declare hidden i32 @strnlen(i8* nocapture, i32) #1
+declare hidden i32 @strcmp(i8* nocapture, i8* nocapture) #1
 
 
 ; ERROR: error: <unknown>:0:0: in function test_memcmp void (i8 addrspace(1)*, i8 addrspace(1)*, i32*): unsupported call to function memcmp
diff --git a/test/CodeGen/AMDGPU/memory-legalizer-mesa3d.ll b/test/CodeGen/AMDGPU/memory-legalizer-mesa3d.ll
index 609dc0400c490034e20fe58c51a86dafa8631e05..44b4d19dbb1f1315058b0d4912b42d3a272fdb33 100644
--- a/test/CodeGen/AMDGPU/memory-legalizer-mesa3d.ll
+++ b/test/CodeGen/AMDGPU/memory-legalizer-mesa3d.ll
@@ -1,6 +1,6 @@
 ; RUN: llc -mtriple=amdgcn--mesa3d -mcpu=gfx600 -verify-machineinstrs < %s | FileCheck -check-prefixes=FUNC,GCN %s
 ; RUN: llc -mtriple=amdgcn--mesa3d -mcpu=gfx700 -verify-machineinstrs < %s | FileCheck -check-prefixes=FUNC,GCN %s
-; RUN: llc -mtriple=amdgcn--mesa3d -mcpu=gfx800 -verify-machineinstrs < %s | FileCheck -check-prefixes=FUNC,GCN %s
+; RUN: llc -mtriple=amdgcn--mesa3d -mcpu=gfx803 -verify-machineinstrs < %s | FileCheck -check-prefixes=FUNC,GCN %s
 ; RUN: llc -mtriple=amdgcn--mesa3d -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=FUNC,GCN %s
 
 ; FUNC-LABEL: {{^}}system_acquire:
diff --git a/test/CodeGen/AMDGPU/merge-stores.ll b/test/CodeGen/AMDGPU/merge-stores.ll
index 7d0c0db22b7e3750630fe230265dcd7619bf1111..7fb0e354df2c14e5aa6f3cc0531306562c423413 100644
--- a/test/CodeGen/AMDGPU/merge-stores.ll
+++ b/test/CodeGen/AMDGPU/merge-stores.ll
@@ -1,5 +1,5 @@
 ; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=tahiti -verify-machineinstrs -amdgpu-load-store-vectorizer=0 < %s | FileCheck -check-prefix=SI -check-prefix=GCN -check-prefix=GCN-AA %s
-; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=bonaire -verify-machineinstrs -amdgpu-load-store-vectorizer=0 < %s | FileCheck -check-prefix=SI -check-prefix=GCN -check-prefix=GCN-AA %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=bonaire -verify-machineinstrs -amdgpu-load-store-vectorizer=0 < %s | FileCheck -check-prefix=CI -check-prefix=GCN -check-prefix=GCN-AA %s
 
 ; This test is mostly to test DAG store merging, so disable the vectorizer.
 ; Run with devices with different unaligned load restrictions.
@@ -65,8 +65,8 @@ define amdgpu_kernel void @merge_global_store_2_constants_i16_natural_align(i16
 }
 
 ; GCN-LABEL: {{^}}merge_global_store_2_constants_i32:
-; SI-DAG: v_mov_b32_e32 v[[LO:[0-9]+]], 0x1c8
-; SI-DAG: v_mov_b32_e32 v[[HI:[0-9]+]], 0x7b
+; GCN-DAG: v_mov_b32_e32 v[[LO:[0-9]+]], 0x1c8
+; GCN-DAG: v_mov_b32_e32 v[[HI:[0-9]+]], 0x7b
 ; GCN: buffer_store_dwordx2 v{{\[}}[[LO]]:[[HI]]{{\]}}
 define amdgpu_kernel void @merge_global_store_2_constants_i32(i32 addrspace(1)* %out) #0 {
   %out.gep.1 = getelementptr i32, i32 addrspace(1)* %out, i32 1
@@ -87,8 +87,8 @@ define amdgpu_kernel void @merge_global_store_2_constants_i32_f32(i32 addrspace(
 }
 
 ; GCN-LABEL: {{^}}merge_global_store_2_constants_f32_i32:
-; SI-DAG: v_mov_b32_e32 v[[VLO:[0-9]+]], 4.0
-; SI-DAG: v_mov_b32_e32 v[[VHI:[0-9]+]], 0x7b
+; GCN-DAG: v_mov_b32_e32 v[[VLO:[0-9]+]], 4.0
+; GCN-DAG: v_mov_b32_e32 v[[VHI:[0-9]+]], 0x7b
 ; GCN: buffer_store_dwordx2 v{{\[}}[[VLO]]:[[VHI]]{{\]}}
 define amdgpu_kernel void @merge_global_store_2_constants_f32_i32(float addrspace(1)* %out) #0 {
   %out.gep.1 = getelementptr float, float addrspace(1)* %out, i32 1
@@ -164,9 +164,10 @@ define amdgpu_kernel void @merge_global_store_4_constants_mixed_i32_f32(float ad
 }
 
 ; GCN-LABEL: {{^}}merge_global_store_3_constants_i32:
-; SI-DAG: buffer_store_dwordx3
-; SI-NOT: buffer_store_dwordx2
-; SI-NOT: buffer_store_dword
+; SI-DAG: buffer_store_dwordx2
+; SI-DAG: buffer_store_dword v
+; CI-DAG: buffer_store_dwordx3
+; GCN-NOT: buffer_store_dword
 ; GCN: s_endpgm
 define amdgpu_kernel void @merge_global_store_3_constants_i32(i32 addrspace(1)* %out) #0 {
   %out.gep.1 = getelementptr i32, i32 addrspace(1)* %out, i32 1
@@ -274,9 +275,13 @@ define amdgpu_kernel void @merge_global_store_4_adjacent_loads_i32(i32 addrspace
 }
 
 ; GCN-LABEL: {{^}}merge_global_store_3_adjacent_loads_i32:
-; SI-DAG: buffer_load_dwordx3
+; SI-DAG: buffer_load_dwordx2
+; SI-DAG: buffer_load_dword v
+; CI-DAG: buffer_load_dwordx3
 ; GCN: s_waitcnt
-; SI-DAG: buffer_store_dwordx3 v
+; SI-DAG: buffer_store_dwordx2
+; SI-DAG: buffer_store_dword v
+; CI-DAG: buffer_store_dwordx3
 ; GCN: s_endpgm
 define amdgpu_kernel void @merge_global_store_3_adjacent_loads_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %in) #0 {
   %out.gep.1 = getelementptr i32, i32 addrspace(1)* %out, i32 1
@@ -561,7 +566,9 @@ define amdgpu_kernel void @merge_global_store_6_constants_i32(i32 addrspace(1)*
 
 ; GCN-LABEL: {{^}}merge_global_store_7_constants_i32:
 ; GCN: buffer_store_dwordx4
-; GCN: buffer_store_dwordx3
+; SI: buffer_store_dwordx2
+; SI: buffer_store_dword v
+; CI: buffer_store_dwordx3
 define amdgpu_kernel void @merge_global_store_7_constants_i32(i32 addrspace(1)* %out) {
   store i32 34, i32 addrspace(1)* %out, align 4
   %idx1 = getelementptr inbounds i32, i32 addrspace(1)* %out, i64 1
@@ -608,11 +615,15 @@ define amdgpu_kernel void @merge_global_store_8_constants_i32(i32 addrspace(1)*
 
 ; GCN-LABEL: {{^}}copy_v3i32_align4:
 ; GCN-NOT: SCRATCH_RSRC_DWORD
-; GCN-DAG: buffer_load_dwordx3 v{{\[[0-9]+:[0-9]+\]}}, off, s{{\[[0-9]+:[0-9]+\]}}, 0{{$}}
+; SI-DAG: buffer_load_dword v{{[0-9]+}}, off, s{{\[[0-9]+:[0-9]+\]}}, 0 offset:8
+; SI-DAG: buffer_load_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, off, s{{\[[0-9]+:[0-9]+\]}}, 0{{$}}
+; CI-DAG: buffer_load_dwordx3 v{{\[[0-9]+:[0-9]+\]}}, off, s{{\[[0-9]+:[0-9]+\]}}, 0{{$}}
 ; GCN-NOT: offen
 ; GCN: s_waitcnt vmcnt
 ; GCN-NOT: offen
-; GCN-DAG: buffer_store_dwordx3 v{{\[[0-9]+:[0-9]+\]}}, off, s{{\[[0-9]+:[0-9]+\]}}, 0{{$}}
+; SI-DAG: buffer_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, off, s{{\[[0-9]+:[0-9]+\]}}, 0{{$}}
+; SI-DAG: buffer_store_dword v{{[0-9]+}}, off, s{{\[[0-9]+:[0-9]+\]}}, 0 offset:8
+; CI-DAG: buffer_store_dwordx3 v{{\[[0-9]+:[0-9]+\]}}, off, s{{\[[0-9]+:[0-9]+\]}}, 0{{$}}
 
 ; GCN: ScratchSize: 0{{$}}
 define amdgpu_kernel void @copy_v3i32_align4(<3 x i32> addrspace(1)* noalias %out, <3 x i32> addrspace(1)* noalias %in) #0 {
@@ -639,11 +650,15 @@ define amdgpu_kernel void @copy_v3i64_align4(<3 x i64> addrspace(1)* noalias %ou
 
 ; GCN-LABEL: {{^}}copy_v3f32_align4:
 ; GCN-NOT: SCRATCH_RSRC_DWORD
-; GCN-DAG: buffer_load_dwordx3 v{{\[[0-9]+:[0-9]+\]}}, off, s{{\[[0-9]+:[0-9]+\]}}, 0{{$}}
+; SI-DAG: buffer_load_dword v{{[0-9]+}}, off, s{{\[[0-9]+:[0-9]+\]}}, 0 offset:8
+; SI-DAG: buffer_load_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, off, s{{\[[0-9]+:[0-9]+\]}}, 0{{$}}
+; CI-DAG: buffer_load_dwordx3 v{{\[[0-9]+:[0-9]+\]}}, off, s{{\[[0-9]+:[0-9]+\]}}, 0{{$}}
 ; GCN-NOT: offen
 ; GCN: s_waitcnt vmcnt
 ; GCN-NOT: offen
-; GCN-DAG: buffer_store_dwordx3 v{{\[[0-9]+:[0-9]+\]}}, off, s{{\[[0-9]+:[0-9]+\]}}, 0{{$}}
+; SI-DAG: buffer_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, off, s{{\[[0-9]+:[0-9]+\]}}, 0{{$}}
+; SI-DAG: buffer_store_dword v{{[0-9]+}}, off, s{{\[[0-9]+:[0-9]+\]}}, 0 offset:8
+; CI-DAG: buffer_store_dwordx3 v{{\[[0-9]+:[0-9]+\]}}, off, s{{\[[0-9]+:[0-9]+\]}}, 0{{$}}
 ; GCN: ScratchSize: 0{{$}}
 define amdgpu_kernel void @copy_v3f32_align4(<3 x float> addrspace(1)* noalias %out, <3 x float> addrspace(1)* noalias %in) #0 {
   %vec = load <3 x float>, <3 x float> addrspace(1)* %in, align 4
diff --git a/test/CodeGen/AMDGPU/no-remat-indirect-mov.mir b/test/CodeGen/AMDGPU/no-remat-indirect-mov.mir
new file mode 100644
index 0000000000000000000000000000000000000000..8d7a93ec01b62cd8a3094b0766a6caaf2412a549
--- /dev/null
+++ b/test/CodeGen/AMDGPU/no-remat-indirect-mov.mir
@@ -0,0 +1,112 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900  -start-after=phi-node-elimination -stop-before=greedy -o - %s | FileCheck -check-prefix=GFX9 %s
+
+# Make sure that the V_MOV_B32 isn't rematerialized out of the loop. This was also breaking RenameIndependentSubregisters which missed the use of all subregisters.
+
+---
+name: index_vgpr_waterfall_loop
+tracksRegLiveness: true
+liveins:
+  - { reg: '$vgpr0', virtual-reg: '%0' }
+  - { reg: '$vgpr1', virtual-reg: '%1' }
+  - { reg: '$vgpr2', virtual-reg: '%2' }
+  - { reg: '$vgpr3', virtual-reg: '%3' }
+  - { reg: '$vgpr4', virtual-reg: '%4' }
+  - { reg: '$vgpr5', virtual-reg: '%5' }
+  - { reg: '$vgpr6', virtual-reg: '%6' }
+  - { reg: '$vgpr7', virtual-reg: '%7' }
+  - { reg: '$vgpr8', virtual-reg: '%8' }
+  - { reg: '$vgpr9', virtual-reg: '%9' }
+  - { reg: '$vgpr10', virtual-reg: '%10' }
+  - { reg: '$vgpr11', virtual-reg: '%11' }
+  - { reg: '$vgpr12', virtual-reg: '%12' }
+  - { reg: '$vgpr13', virtual-reg: '%13' }
+  - { reg: '$vgpr14', virtual-reg: '%14' }
+  - { reg: '$vgpr15', virtual-reg: '%15' }
+  - { reg: '$vgpr16', virtual-reg: '%16' }
+  - { reg: '$sgpr30_sgpr31', virtual-reg: '%17' }
+body:             |
+  ; GFX9-LABEL: name: index_vgpr_waterfall_loop
+  ; GFX9: bb.0:
+  ; GFX9:   successors: %bb.1(0x80000000)
+  ; GFX9:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8, $vgpr9, $vgpr10, $vgpr11, $vgpr12, $vgpr13, $vgpr14, $vgpr15, $vgpr16, $sgpr30_sgpr31
+  ; GFX9:   [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr30_sgpr31
+  ; GFX9:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr16
+  ; GFX9:   undef %18.sub15:vreg_512 = COPY $vgpr15
+  ; GFX9:   %18.sub14:vreg_512 = COPY $vgpr14
+  ; GFX9:   %18.sub13:vreg_512 = COPY $vgpr13
+  ; GFX9:   %18.sub12:vreg_512 = COPY $vgpr12
+  ; GFX9:   %18.sub11:vreg_512 = COPY $vgpr11
+  ; GFX9:   %18.sub10:vreg_512 = COPY $vgpr10
+  ; GFX9:   %18.sub9:vreg_512 = COPY $vgpr9
+  ; GFX9:   %18.sub8:vreg_512 = COPY $vgpr8
+  ; GFX9:   %18.sub7:vreg_512 = COPY $vgpr7
+  ; GFX9:   %18.sub6:vreg_512 = COPY $vgpr6
+  ; GFX9:   %18.sub5:vreg_512 = COPY $vgpr5
+  ; GFX9:   %18.sub4:vreg_512 = COPY $vgpr4
+  ; GFX9:   %18.sub3:vreg_512 = COPY $vgpr3
+  ; GFX9:   %18.sub2:vreg_512 = COPY $vgpr2
+  ; GFX9:   %18.sub1:vreg_512 = COPY $vgpr1
+  ; GFX9:   %18.sub0:vreg_512 = COPY $vgpr0
+  ; GFX9:   [[V_AND_B32_e32_:%[0-9]+]]:vgpr_32 = V_AND_B32_e32 15, [[COPY1]], implicit $exec
+  ; GFX9:   [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
+  ; GFX9: bb.1:
+  ; GFX9:   successors: %bb.1(0x40000000), %bb.2(0x40000000)
+  ; GFX9:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sgpr_32 = V_READFIRSTLANE_B32 [[V_AND_B32_e32_]], implicit $exec
+  ; GFX9:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64 = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_]], [[V_AND_B32_e32_]], implicit $exec
+  ; GFX9:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64 = S_AND_SAVEEXEC_B64 [[V_CMP_EQ_U32_e64_]], implicit-def $exec, implicit-def dead $scc, implicit $exec
+  ; GFX9:   S_SET_GPR_IDX_ON [[V_READFIRSTLANE_B32_]], 1, implicit-def $m0, implicit undef $m0
+  ; GFX9:   [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 undef %18.sub0, implicit $exec, implicit %18, implicit $m0
+  ; GFX9:   S_SET_GPR_IDX_OFF
+  ; GFX9:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def dead $scc
+  ; GFX9:   S_CBRANCH_EXECNZ %bb.1, implicit $exec
+  ; GFX9: bb.2:
+  ; GFX9:   $exec = S_MOV_B64 [[S_MOV_B64_]]
+  ; GFX9:   $sgpr30_sgpr31 = COPY [[COPY]]
+  ; GFX9:   $vgpr0 = COPY [[V_MOV_B32_e32_]]
+  ; GFX9:   S_SETPC_B64_return $sgpr30_sgpr31, implicit $vgpr0, implicit undef $vgpr1, implicit undef $vgpr2, implicit undef $vgpr3
+  bb.0:
+    successors: %bb.1
+    liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8, $vgpr9, $vgpr10, $vgpr11, $vgpr12, $vgpr13, $vgpr14, $vgpr15, $vgpr16, $sgpr30_sgpr31
+
+    %17:sreg_64 = COPY killed $sgpr30_sgpr31
+    %16:vgpr_32 = COPY killed $vgpr16
+    %15:vgpr_32 = COPY killed $vgpr15
+    %14:vgpr_32 = COPY killed $vgpr14
+    %13:vgpr_32 = COPY killed $vgpr13
+    %12:vgpr_32 = COPY killed $vgpr12
+    %11:vgpr_32 = COPY killed $vgpr11
+    %10:vgpr_32 = COPY killed $vgpr10
+    %9:vgpr_32 = COPY killed $vgpr9
+    %8:vgpr_32 = COPY killed $vgpr8
+    %7:vgpr_32 = COPY killed $vgpr7
+    %6:vgpr_32 = COPY killed $vgpr6
+    %5:vgpr_32 = COPY killed $vgpr5
+    %4:vgpr_32 = COPY killed $vgpr4
+    %3:vgpr_32 = COPY killed $vgpr3
+    %2:vgpr_32 = COPY killed $vgpr2
+    %1:vgpr_32 = COPY killed $vgpr1
+    %0:vgpr_32 = COPY killed $vgpr0
+    %18:vreg_512 = REG_SEQUENCE killed %0, %subreg.sub0, killed %1, %subreg.sub1, killed %2, %subreg.sub2, killed %3, %subreg.sub3, killed %4, %subreg.sub4, killed %5, %subreg.sub5, killed %6, %subreg.sub6, killed %7, %subreg.sub7, killed %8, %subreg.sub8, killed %9, %subreg.sub9, killed %10, %subreg.sub10, killed %11, %subreg.sub11, killed %12, %subreg.sub12, killed %13, %subreg.sub13, killed %14, %subreg.sub14, killed %15, %subreg.sub15
+    %19:vgpr_32 = V_AND_B32_e32 15, killed %16, implicit $exec
+    %20:sreg_64_xexec = S_MOV_B64 $exec
+
+  bb.1:
+    successors: %bb.1, %bb.2
+
+    %21:sgpr_32 = V_READFIRSTLANE_B32 %19, implicit $exec
+    %22:sreg_64 = V_CMP_EQ_U32_e64 %21, %19, implicit $exec
+    %23:sreg_64 = S_AND_SAVEEXEC_B64 killed %22, implicit-def $exec, implicit-def dead $scc, implicit $exec
+    S_SET_GPR_IDX_ON killed %21, 1, implicit-def $m0, implicit undef $m0
+    %24:vgpr_32 = V_MOV_B32_e32 undef %18.sub0, implicit $exec, implicit %18, implicit $m0
+    S_SET_GPR_IDX_OFF
+    $exec = S_XOR_B64_term $exec, killed %23, implicit-def dead $scc
+    S_CBRANCH_EXECNZ %bb.1, implicit $exec
+
+  bb.2:
+    $exec = S_MOV_B64 killed %20
+    $sgpr30_sgpr31 = COPY killed %17
+    $vgpr0 = COPY killed %24
+    S_SETPC_B64_return killed $sgpr30_sgpr31, implicit killed $vgpr0, implicit undef $vgpr1, implicit undef $vgpr2, implicit undef $vgpr3
+
+...
diff --git a/test/CodeGen/AMDGPU/promote-alloca-mem-intrinsics.ll b/test/CodeGen/AMDGPU/promote-alloca-mem-intrinsics.ll
index fcf64ce8016db67da6471712cb8837d32767cb91..4515447b74972e838e1b63e7e9815cc564eb0780 100644
--- a/test/CodeGen/AMDGPU/promote-alloca-mem-intrinsics.ll
+++ b/test/CodeGen/AMDGPU/promote-alloca-mem-intrinsics.ll
@@ -8,7 +8,7 @@ declare void @llvm.memmove.p1i8.p0i8.i32(i8 addrspace(1)* nocapture, i8* nocaptu
 
 declare void @llvm.memset.p0i8.i32(i8* nocapture, i8, i32, i1) #0
 
-declare i32 @llvm.objectsize.i32.p0i8(i8*, i1, i1) #1
+declare i32 @llvm.objectsize.i32.p0i8(i8*, i1, i1, i1) #1
 
 ; CHECK-LABEL: @promote_with_memcpy(
 ; CHECK: getelementptr inbounds [64 x [17 x i32]], [64 x [17 x i32]] addrspace(3)* @promote_with_memcpy.alloca, i32 0, i32 %{{[0-9]+}}
@@ -52,11 +52,11 @@ define amdgpu_kernel void @promote_with_memset(i32 addrspace(1)* %out, i32 addrs
 
 ; CHECK-LABEL: @promote_with_objectsize(
 ; CHECK: [[PTR:%[0-9]+]] = getelementptr inbounds [64 x [17 x i32]], [64 x [17 x i32]] addrspace(3)* @promote_with_objectsize.alloca, i32 0, i32 %{{[0-9]+}}
-; CHECK: call i32 @llvm.objectsize.i32.p3i8(i8 addrspace(3)* %alloca.bc, i1 false, i1 false)
+; CHECK: call i32 @llvm.objectsize.i32.p3i8(i8 addrspace(3)* %alloca.bc, i1 false, i1 false, i1 false)
 define amdgpu_kernel void @promote_with_objectsize(i32 addrspace(1)* %out) #0 {
   %alloca = alloca [17 x i32], align 4
   %alloca.bc = bitcast [17 x i32]* %alloca to i8*
-  %size = call i32 @llvm.objectsize.i32.p0i8(i8* %alloca.bc, i1 false, i1 false)
+  %size = call i32 @llvm.objectsize.i32.p0i8(i8* %alloca.bc, i1 false, i1 false, i1 false)
   store i32 %size, i32 addrspace(1)* %out
   ret void
 }
diff --git a/test/CodeGen/AMDGPU/regcoalesce-cannot-join-failures.mir b/test/CodeGen/AMDGPU/regcoalesce-cannot-join-failures.mir
new file mode 100644
index 0000000000000000000000000000000000000000..846607be790ebf31090ac78d882acb30f933ee0f
--- /dev/null
+++ b/test/CodeGen/AMDGPU/regcoalesce-cannot-join-failures.mir
@@ -0,0 +1,118 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -verify-coalescing -run-pass=simple-register-coalescing -verify-machineinstrs -o - %s | FileCheck %s
+
+---
+name: couldnt_join_subrange_implicit_def_pred_block
+tracksRegLiveness: true
+body:             |
+  ; CHECK-LABEL: name: couldnt_join_subrange_implicit_def_pred_block
+  ; CHECK: bb.0:
+  ; CHECK:   successors: %bb.1(0x80000000)
+  ; CHECK:   undef %0.sub0:sreg_64_xexec = IMPLICIT_DEF
+  ; CHECK: bb.1:
+  ; CHECK:   successors: %bb.2(0x80000000)
+  ; CHECK:   %0.sub1:sreg_64_xexec = COPY %0.sub0
+  ; CHECK:   S_BRANCH %bb.2
+  ; CHECK: bb.2:
+  ; CHECK:   S_ENDPGM implicit %0
+  bb.0:
+    successors: %bb.1
+
+    undef %0.sub0:sreg_64_xexec = IMPLICIT_DEF
+
+  bb.1:
+    successors: %bb.2
+
+    %1:sreg_64 = COPY %0:sreg_64_xexec
+    %0.sub1:sreg_64_xexec = COPY %0.sub0:sreg_64_xexec
+    S_BRANCH %bb.2
+
+  bb.2:
+    dead %2:sreg_32_xm0 = COPY %0.sub0:sreg_64_xexec
+    S_ENDPGM implicit killed %1
+
+...
+---
+name: couldnt_join_subrange_no_implicit_def_inst
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    ; CHECK-LABEL: name: couldnt_join_subrange_no_implicit_def_inst
+    ; CHECK: undef %0.sub0:sreg_64 = S_MOV_B32 0
+    ; CHECK: %0.sub1:sreg_64 = COPY %0.sub0
+    ; CHECK: S_ENDPGM implicit %0.sub1
+    undef %0.sub0:sreg_64 = S_MOV_B32 0
+    %1:sreg_64 = COPY %0:sreg_64
+    %0.sub1:sreg_64 = COPY %0.sub0:sreg_64
+    S_ENDPGM implicit %1.sub1:sreg_64
+
+...
+---
+name: couldnt_join_subrange0
+tracksRegLiveness: true
+body:             |
+  ; CHECK-LABEL: name: couldnt_join_subrange0
+  ; CHECK: bb.0:
+  ; CHECK:   successors: %bb.1(0x80000000)
+  ; CHECK:   undef %0.sub1:sreg_64 = S_MOV_B32 -1
+  ; CHECK: bb.1:
+  ; CHECK:   %0.sub0:sreg_64 = S_MOV_B32 0
+  ; CHECK:   [[COPY:%[0-9]+]]:sreg_64 = COPY %0
+  ; CHECK:   dead %0.sub1:sreg_64 = COPY %0.sub0
+  ; CHECK:   S_ENDPGM implicit [[COPY]].sub1
+  bb.0:
+    successors: %bb.1
+    undef %0.sub1:sreg_64 = S_MOV_B32 -1
+
+  bb.1:
+    %0.sub0:sreg_64 = S_MOV_B32 0
+    %1:sreg_64 = COPY %0:sreg_64
+    dead %0.sub1:sreg_64 = COPY %0.sub0:sreg_64
+    S_ENDPGM implicit %1.sub1:sreg_64
+
+...
+---
+name: lanes_not_tracked_subreg_join_couldnt_join_subrange
+tracksRegLiveness: true
+body:             |
+  bb.0:
+
+    ; CHECK-LABEL: name: lanes_not_tracked_subreg_join_couldnt_join_subrange
+    ; CHECK: undef %0.sub0:sreg_64_xexec = S_MOV_B32 0
+    ; CHECK: %0.sub1:sreg_64_xexec = S_MOV_B32 0
+    ; CHECK: S_NOP 0, implicit %0.sub1
+    ; CHECK: S_NOP 0, implicit %0
+    ; CHECK: S_ENDPGM
+    undef %0.sub0:sreg_64_xexec = S_MOV_B32 0
+    %1:sreg_64 = COPY %0
+    %0.sub1:sreg_64_xexec = S_MOV_B32 0
+    S_NOP 0, implicit %0.sub1
+    S_NOP 0, implicit %1
+    S_ENDPGM
+
+...
+---
+name: couldnt_join_subrange1
+tracksRegLiveness: true
+body:             |
+  ; CHECK-LABEL: name: couldnt_join_subrange1
+  ; CHECK: bb.0:
+  ; CHECK:   successors: %bb.1(0x80000000)
+  ; CHECK:   undef %0.sub0:sreg_64_xexec = S_MOV_B32 0
+  ; CHECK:   %0.sub1:sreg_64_xexec = COPY %0.sub0
+  ; CHECK: bb.1:
+  ; CHECK:   S_NOP 0, implicit %0.sub1
+  ; CHECK:   S_ENDPGM implicit %0
+  bb.0:
+    successors: %bb.1
+
+    undef %0.sub0:sreg_64_xexec = S_MOV_B32 0
+    %1:sreg_64 = COPY %0
+    %0.sub1:sreg_64_xexec = COPY %0.sub0
+
+  bb.1:
+
+    S_NOP 0, implicit %0.sub1
+    S_ENDPGM implicit %1
+
+...
diff --git a/test/CodeGen/AMDGPU/regcoalesce-keep-valid-lanes-implicit-def-bug39602.mir b/test/CodeGen/AMDGPU/regcoalesce-keep-valid-lanes-implicit-def-bug39602.mir
new file mode 100644
index 0000000000000000000000000000000000000000..f7a9915a19888f6861b98ddee1ff973aa74997e9
--- /dev/null
+++ b/test/CodeGen/AMDGPU/regcoalesce-keep-valid-lanes-implicit-def-bug39602.mir
@@ -0,0 +1,57 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -verify-coalescing -run-pass=simple-register-coalescing -verify-machineinstrs -o - %s | FileCheck %s
+
+# Bug 39602: Avoid "Couldn't join subrange" error when clearing valid
+# lanes on an implicit_def that later cannot be erased.
+
+---
+name: lost_valid_lanes_maybe_erasable_implicit_def
+tracksRegLiveness: true
+body:             |
+  ; CHECK-LABEL: name: lost_valid_lanes_maybe_erasable_implicit_def
+  ; CHECK: bb.0:
+  ; CHECK:   successors: %bb.1(0x80000000)
+  ; CHECK:   undef %0.sub1:sreg_64 = IMPLICIT_DEF
+  ; CHECK: bb.1:
+  ; CHECK:   %0.sub0:sreg_64 = S_MOV_B32 0
+  ; CHECK:   [[COPY:%[0-9]+]]:sreg_64 = COPY %0
+  ; CHECK:   dead %0.sub1:sreg_64 = COPY %0.sub0
+  ; CHECK:   S_ENDPGM implicit [[COPY]].sub1
+  bb.0:
+    successors: %bb.1
+    undef %0.sub1:sreg_64 = IMPLICIT_DEF
+
+  bb.1:
+    %0.sub0:sreg_64 = S_MOV_B32 0
+    %1:sreg_64 = COPY %0:sreg_64
+    dead %0.sub1:sreg_64 = COPY %0.sub0:sreg_64
+    S_ENDPGM implicit %1.sub1:sreg_64
+
+...
+---
+# Same as previous, except with a real value instead of
+# IMPLICIT_DEF. These should both be handled the same way.
+
+name:  lost_valid_lanes_real_value
+tracksRegLiveness: true
+body:             |
+  ; CHECK-LABEL: name: lost_valid_lanes_real_value
+  ; CHECK: bb.0:
+  ; CHECK:   successors: %bb.1(0x80000000)
+  ; CHECK:   undef %0.sub1:sreg_64 = S_MOV_B32 -1
+  ; CHECK: bb.1:
+  ; CHECK:   %0.sub0:sreg_64 = S_MOV_B32 0
+  ; CHECK:   [[COPY:%[0-9]+]]:sreg_64 = COPY %0
+  ; CHECK:   dead %0.sub1:sreg_64 = COPY %0.sub0
+  ; CHECK:   S_ENDPGM implicit [[COPY]].sub1
+  bb.0:
+    successors: %bb.1
+    undef %0.sub1:sreg_64 = S_MOV_B32 -1
+
+  bb.1:
+    %0.sub0:sreg_64 = S_MOV_B32 0
+    %1:sreg_64 = COPY %0:sreg_64
+    dead %0.sub1:sreg_64 = COPY %0.sub0:sreg_64
+    S_ENDPGM implicit %1.sub1:sreg_64
+
+...
diff --git a/test/CodeGen/AMDGPU/saddo.ll b/test/CodeGen/AMDGPU/saddo.ll
index 0b52821f72cc94f3e9e351fd90a9bb9026384ed1..a5d7592faaffdcec20693a12434a29250a4913cb 100644
--- a/test/CodeGen/AMDGPU/saddo.ll
+++ b/test/CodeGen/AMDGPU/saddo.ll
@@ -1,11 +1,14 @@
 ; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=tahiti -verify-machineinstrs< %s | FileCheck -check-prefixes=GCN,SICIVI,FUNC %s
 ; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=tonga -verify-machineinstrs< %s | FileCheck -check-prefixes=GCN,SICIVI,FUNC %s
 ; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=gfx900 -verify-machineinstrs< %s | FileCheck -check-prefixes=GCN,GFX9,FUNC %s
-; RUN: llc -amdgpu-scalarize-global-loads=false -march=r600 -mcpu=cypress -verify-machineinstrs< %s
+
 
 declare { i32, i1 } @llvm.sadd.with.overflow.i32(i32, i32) nounwind readnone
 declare { i64, i1 } @llvm.sadd.with.overflow.i64(i64, i64) nounwind readnone
 
+
+declare { <2 x i32>, <2 x i1> } @llvm.sadd.with.overflow.v2i32(<2 x i32>, <2 x i32>) nounwind readnone
+
 ; FUNC-LABEL: {{^}}saddo_i64_zext:
 define amdgpu_kernel void @saddo_i64_zext(i64 addrspace(1)* %out, i64 %a, i64 %b) nounwind {
   %sadd = call { i64, i1 } @llvm.sadd.with.overflow.i64(i64 %a, i64 %b) nounwind
@@ -65,3 +68,22 @@ define amdgpu_kernel void @v_saddo_i64(i64 addrspace(1)* %out, i1 addrspace(1)*
   store i1 %carry, i1 addrspace(1)* %carryout
   ret void
 }
+
+; FUNC-LABEL: {{^}}v_saddo_v2i32:
+; SICIVI: v_cmp_lt_i32
+; SICIVI: v_cmp_lt_i32
+; SICIVI: v_add_{{[iu]}}32
+; SICIVI: v_cmp_lt_i32
+; SICIVI: v_cmp_lt_i32
+; SICIVI: v_add_{{[iu]}}32
+define amdgpu_kernel void @v_saddo_v2i32(<2 x i32> addrspace(1)* %out, <2 x i32> addrspace(1)* %carryout, <2 x i32> addrspace(1)* %aptr, <2 x i32> addrspace(1)* %bptr) nounwind {
+  %a = load <2 x i32>, <2 x i32> addrspace(1)* %aptr, align 4
+  %b = load <2 x i32>, <2 x i32> addrspace(1)* %bptr, align 4
+  %sadd = call { <2 x i32>, <2 x i1> } @llvm.sadd.with.overflow.v2i32(<2 x i32> %a, <2 x i32> %b) nounwind
+  %val = extractvalue { <2 x i32>, <2 x i1> } %sadd, 0
+  %carry = extractvalue { <2 x i32>, <2 x i1> } %sadd, 1
+  store <2 x i32> %val, <2 x i32> addrspace(1)* %out, align 4
+  %carry.ext = zext <2 x i1> %carry to <2 x i32>
+  store <2 x i32> %carry.ext, <2 x i32> addrspace(1)* %carryout
+  ret void
+}
diff --git a/test/CodeGen/AMDGPU/scheduler-subrange-crash.ll b/test/CodeGen/AMDGPU/scheduler-subrange-crash.ll
index d93cde6a885f76fc2f9b60f8f4a45898963d9b8f..be76371c42df33bb9bc3f358b7cfb6a74b709990 100644
--- a/test/CodeGen/AMDGPU/scheduler-subrange-crash.ll
+++ b/test/CodeGen/AMDGPU/scheduler-subrange-crash.ll
@@ -15,9 +15,9 @@ target triple = "amdgcn--"
 
 define amdgpu_gs void @main(i32 inreg %arg) #0 {
 main_body:
-  %tmp = call float @llvm.SI.load.const.v4i32(<4 x i32> undef, i32 20)
-  %tmp1 = call float @llvm.SI.load.const.v4i32(<4 x i32> undef, i32 24)
-  %tmp2 = call float @llvm.SI.load.const.v4i32(<4 x i32> undef, i32 48)
+  %tmp = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> undef, i32 20, i32 0)
+  %tmp1 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> undef, i32 24, i32 0)
+  %tmp2 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> undef, i32 48, i32 0)
   %array_vector3 = insertelement <4 x float> zeroinitializer, float %tmp2, i32 3
   %array_vector5 = insertelement <4 x float> <float 0.000000e+00, float undef, float undef, float undef>, float %tmp, i32 1
   %array_vector6 = insertelement <4 x float> %array_vector5, float undef, i32 2
@@ -45,7 +45,7 @@ main_body:
   ret void
 }
 
-declare float @llvm.SI.load.const.v4i32(<4 x i32>, i32) #1
+declare float @llvm.amdgcn.s.buffer.load.f32(<4 x i32>, i32, i32) #1
 declare i32 @llvm.amdgcn.raw.buffer.load.i32(<4 x i32>, i32, i32, i32) #2
 declare void @llvm.amdgcn.tbuffer.store.i32(i32, <4 x i32>, i32, i32, i32, i32, i32, i32, i1, i1) #3
 
diff --git a/test/CodeGen/AMDGPU/scratch-simple.ll b/test/CodeGen/AMDGPU/scratch-simple.ll
index b2781a77811b853240c0c741639efa024bab0f02..0ddd7b907b1ecb0f575635e90774131c0099e42f 100644
--- a/test/CodeGen/AMDGPU/scratch-simple.ll
+++ b/test/CodeGen/AMDGPU/scratch-simple.ll
@@ -1,6 +1,10 @@
 ; RUN: llc -march=amdgcn -mtriple=amdgcn-- -mcpu=verde -verify-machineinstrs < %s | FileCheck --check-prefix=GCN --check-prefix=SI %s
 ; RUN: llc -march=amdgcn -mtriple=amdgcn-- -mcpu=gfx803 -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck --check-prefix=GCN --check-prefix=SI %s
 ; RUN: llc -march=amdgcn -mtriple=amdgcn-- -mcpu=gfx900 -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck --check-prefix=GCN --check-prefix=GFX9 %s
+; RUN: llc -march=amdgcn -mtriple=amdgcn-- -mcpu=gfx900 -filetype=obj < %s | llvm-readobj -relocations | FileCheck --check-prefix=RELS %s
+
+; RELS: R_AMDGPU_ABS32_LO SCRATCH_RSRC_DWORD0 0x0
+; RELS: R_AMDGPU_ABS32_LO SCRATCH_RSRC_DWORD1 0x0
 
 ; This used to fail due to a v_add_i32 instruction with an illegal immediate
 ; operand that was created during Local Stack Slot Allocation. Test case derived
diff --git a/test/CodeGen/AMDGPU/sgpr-copy.ll b/test/CodeGen/AMDGPU/sgpr-copy.ll
index be4e9fbdb95872dd6804cc7550a6e77dfbad59a6..eabda11e39d11ae0922541223dc6c8db029ed2d7 100644
--- a/test/CodeGen/AMDGPU/sgpr-copy.ll
+++ b/test/CodeGen/AMDGPU/sgpr-copy.ll
@@ -8,9 +8,9 @@ define amdgpu_ps void @phi1(<4 x i32> addrspace(4)* inreg %arg, <4 x i32> addrsp
 main_body:
   %tmp = getelementptr <4 x i32>, <4 x i32> addrspace(4)* %arg, i32 0
   %tmp20 = load <4 x i32>, <4 x i32> addrspace(4)* %tmp, !tbaa !0
-  %tmp21 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp20, i32 0)
-  %tmp22 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp20, i32 16)
-  %tmp23 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp20, i32 32)
+  %tmp21 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp20, i32 0, i32 0)
+  %tmp22 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp20, i32 16, i32 0)
+  %tmp23 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp20, i32 32, i32 0)
   %tmp24 = fptosi float %tmp22 to i32
   %tmp25 = icmp ne i32 %tmp24, 0
   br i1 %tmp25, label %ENDIF, label %ELSE
@@ -32,21 +32,21 @@ define amdgpu_ps void @phi2(<4 x i32> addrspace(4)* inreg %arg, <4 x i32> addrsp
 main_body:
   %tmp = getelementptr <4 x i32>, <4 x i32> addrspace(4)* %arg, i32 0
   %tmp20 = load <4 x i32>, <4 x i32> addrspace(4)* %tmp, !tbaa !0
-  %tmp21 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp20, i32 16)
-  %tmp22 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp20, i32 32)
-  %tmp23 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp20, i32 36)
-  %tmp24 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp20, i32 40)
-  %tmp25 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp20, i32 48)
-  %tmp26 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp20, i32 52)
-  %tmp27 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp20, i32 56)
-  %tmp28 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp20, i32 64)
-  %tmp29 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp20, i32 68)
-  %tmp30 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp20, i32 72)
-  %tmp31 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp20, i32 76)
-  %tmp32 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp20, i32 80)
-  %tmp33 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp20, i32 84)
-  %tmp34 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp20, i32 88)
-  %tmp35 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp20, i32 92)
+  %tmp21 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp20, i32 16, i32 0)
+  %tmp22 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp20, i32 32, i32 0)
+  %tmp23 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp20, i32 36, i32 0)
+  %tmp24 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp20, i32 40, i32 0)
+  %tmp25 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp20, i32 48, i32 0)
+  %tmp26 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp20, i32 52, i32 0)
+  %tmp27 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp20, i32 56, i32 0)
+  %tmp28 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp20, i32 64, i32 0)
+  %tmp29 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp20, i32 68, i32 0)
+  %tmp30 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp20, i32 72, i32 0)
+  %tmp31 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp20, i32 76, i32 0)
+  %tmp32 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp20, i32 80, i32 0)
+  %tmp33 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp20, i32 84, i32 0)
+  %tmp34 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp20, i32 88, i32 0)
+  %tmp35 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp20, i32 92, i32 0)
   %tmp36 = getelementptr <8 x i32>, <8 x i32> addrspace(4)* %arg2, i32 0
   %tmp37 = load <8 x i32>, <8 x i32> addrspace(4)* %tmp36, !tbaa !0
   %tmp38 = getelementptr <4 x i32>, <4 x i32> addrspace(4)* %arg1, i32 0
@@ -172,10 +172,10 @@ define amdgpu_ps void @loop(<4 x i32> addrspace(4)* inreg %arg, <4 x i32> addrsp
 main_body:
   %tmp = getelementptr <4 x i32>, <4 x i32> addrspace(4)* %arg, i32 0
   %tmp20 = load <4 x i32>, <4 x i32> addrspace(4)* %tmp, !tbaa !0
-  %tmp21 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp20, i32 0)
-  %tmp22 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp20, i32 4)
-  %tmp23 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp20, i32 8)
-  %tmp24 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp20, i32 12)
+  %tmp21 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp20, i32 0, i32 0)
+  %tmp22 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp20, i32 4, i32 0)
+  %tmp23 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp20, i32 8, i32 0)
+  %tmp24 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp20, i32 12, i32 0)
   %tmp25 = fptosi float %tmp24 to i32
   %tmp26 = bitcast i32 %tmp25 to float
   %tmp27 = bitcast float %tmp26 to i32
@@ -225,7 +225,7 @@ define amdgpu_ps void @sample_v3([17 x <4 x i32>] addrspace(4)* byval %arg, [32
 entry:
   %tmp = getelementptr [17 x <4 x i32>], [17 x <4 x i32>] addrspace(4)* %arg, i64 0, i32 0
   %tmp21 = load <4 x i32>, <4 x i32> addrspace(4)* %tmp, !tbaa !0
-  %tmp22 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 16)
+  %tmp22 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 16, i32 0)
   %tmp23 = getelementptr [16 x <8 x i32>], [16 x <8 x i32>] addrspace(4)* %arg2, i64 0, i32 0
   %tmp24 = load <8 x i32>, <8 x i32> addrspace(4)* %tmp23, !tbaa !0
   %tmp25 = getelementptr [32 x <4 x i32>], [32 x <4 x i32>] addrspace(4)* %arg1, i64 0, i32 0
@@ -325,7 +325,7 @@ define amdgpu_ps void @sample_rsrc([6 x <4 x i32>] addrspace(4)* byval %arg, [17
 bb:
   %tmp = getelementptr [17 x <4 x i32>], [17 x <4 x i32>] addrspace(4)* %arg1, i32 0, i32 0
   %tmp22 = load <4 x i32>, <4 x i32> addrspace(4)* %tmp, !tbaa !3
-  %tmp23 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp22, i32 16)
+  %tmp23 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp22, i32 16, i32 0)
   %tmp25 = getelementptr [32 x <8 x i32>], [32 x <8 x i32>] addrspace(4)* %arg3, i32 0, i32 0
   %tmp26 = load <8 x i32>, <8 x i32> addrspace(4)* %tmp25, !tbaa !3
   %tmp27 = getelementptr [16 x <4 x i32>], [16 x <4 x i32>] addrspace(4)* %arg2, i32 0, i32 0
@@ -409,7 +409,7 @@ declare void @llvm.amdgcn.exp.f32(i32, i32, float, float, float, float, i1, i1)
 declare void @llvm.amdgcn.exp.compr.v2f16(i32, i32, <2 x half>, <2 x half>, i1, i1) #0
 declare <2 x half> @llvm.amdgcn.cvt.pkrtz(float, float) #1
 declare <4 x float> @llvm.amdgcn.image.sample.2d.v4f32.f32(i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #2
-declare float @llvm.SI.load.const.v4i32(<4 x i32>, i32) #1
+declare float @llvm.amdgcn.s.buffer.load.f32(<4 x i32>, i32, i32) #1
 
 attributes #0 = { nounwind }
 attributes #1 = { nounwind readnone }
diff --git a/test/CodeGen/AMDGPU/si-annotate-cf.ll b/test/CodeGen/AMDGPU/si-annotate-cf.ll
index 6215a486a3683d350cb9cf04dd256b59a99cda7a..ad68d300de05937096232775322c4c444ed74641 100644
--- a/test/CodeGen/AMDGPU/si-annotate-cf.ll
+++ b/test/CodeGen/AMDGPU/si-annotate-cf.ll
@@ -96,20 +96,20 @@ declare float @llvm.fabs.f32(float) nounwind readnone
 ; FUNC-LABEL: {{^}}loop_land_info_assert:
 ; SI:      v_cmp_lt_i32_e64 [[CMP4:s\[[0-9:]+\]]], s{{[0-9]+}}, 4{{$}}
 ; SI:      s_and_b64        [[CMP4M:s\[[0-9]+:[0-9]+\]]], exec, [[CMP4]]
-; SI:      s_mov_b64        vcc, [[CMP4M]]
-; SI-NEXT: s_cbranch_vccnz  [[CONVEX_EXIT:BB[0-9_]+]]
-; SI-NEXT: s_branch         [[FOR_COND_PREHDR:BB[0-9_]+]]
+; SI:      s_branch         [[INFLOOP:BB[0-9]+_[0-9]+]]
+
+; SI:      [[CONVEX_EXIT:BB[0-9_]+]]
+; SI:      s_mov_b64        vcc,
+; SI-NEXT: s_cbranch_vccnz  [[ENDPGM:BB[0-9]+_[0-9]+]]
+; SI:      s_cbranch_vccnz  [[INFLOOP]]
 
 ; SI: ; %if.else
 ; SI:      buffer_store_dword
 
-; SI:      [[INFLOOP:BB[0-9]+_[0-9]+]]:
+; SI:      [[INFLOOP]]:
+; SI:      s_cbranch_vccnz [[CONVEX_EXIT]]
 
-; SI:      [[CONVEX_EXIT]]:
-; SI:      s_mov_b64        vcc,
-; SI-NEXT: s_cbranch_vccnz  [[ENDPGM:BB[0-9]+_[0-9]+]]
-; SI:      s_branch [[INFLOOP]]
-; SI-NEXT: [[FOR_COND_PREHDR]]:
+; SI: ; %for.cond.preheader
 ; SI:      s_cbranch_vccz [[ENDPGM]]
 
 ; SI:      [[ENDPGM]]:
diff --git a/test/CodeGen/AMDGPU/si-sgpr-spill.ll b/test/CodeGen/AMDGPU/si-sgpr-spill.ll
index c4964e68e28009386d25c0e22d01b6067afdf8ff..9a731c6bbbc2ae8b7ea0a2c3e087abc6dc18e504 100644
--- a/test/CodeGen/AMDGPU/si-sgpr-spill.ll
+++ b/test/CodeGen/AMDGPU/si-sgpr-spill.ll
@@ -28,44 +28,44 @@ define amdgpu_ps void @main([17 x <4 x i32>] addrspace(4)* byval %arg, [32 x <4
 main_body:
   %tmp = getelementptr [17 x <4 x i32>], [17 x <4 x i32>] addrspace(4)* %arg, i64 0, i32 0
   %tmp21 = load <4 x i32>, <4 x i32> addrspace(4)* %tmp, !tbaa !0
-  %tmp22 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 96)
-  %tmp23 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 100)
-  %tmp24 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 104)
-  %tmp25 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 112)
-  %tmp26 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 116)
-  %tmp27 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 120)
-  %tmp28 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 128)
-  %tmp29 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 132)
-  %tmp30 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 140)
-  %tmp31 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 144)
-  %tmp32 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 160)
-  %tmp33 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 176)
-  %tmp34 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 180)
-  %tmp35 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 184)
-  %tmp36 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 192)
-  %tmp37 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 196)
-  %tmp38 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 200)
-  %tmp39 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 208)
-  %tmp40 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 212)
-  %tmp41 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 216)
-  %tmp42 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 224)
-  %tmp43 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 240)
-  %tmp44 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 244)
-  %tmp45 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 248)
-  %tmp46 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 256)
-  %tmp47 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 272)
-  %tmp48 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 276)
-  %tmp49 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 280)
-  %tmp50 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 288)
-  %tmp51 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 292)
-  %tmp52 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 296)
-  %tmp53 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 304)
-  %tmp54 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 308)
-  %tmp55 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 312)
-  %tmp56 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 368)
-  %tmp57 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 372)
-  %tmp58 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 376)
-  %tmp59 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 384)
+  %tmp22 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 96, i32 0)
+  %tmp23 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 100, i32 0)
+  %tmp24 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 104, i32 0)
+  %tmp25 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 112, i32 0)
+  %tmp26 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 116, i32 0)
+  %tmp27 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 120, i32 0)
+  %tmp28 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 128, i32 0)
+  %tmp29 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 132, i32 0)
+  %tmp30 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 140, i32 0)
+  %tmp31 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 144, i32 0)
+  %tmp32 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 160, i32 0)
+  %tmp33 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 176, i32 0)
+  %tmp34 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 180, i32 0)
+  %tmp35 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 184, i32 0)
+  %tmp36 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 192, i32 0)
+  %tmp37 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 196, i32 0)
+  %tmp38 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 200, i32 0)
+  %tmp39 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 208, i32 0)
+  %tmp40 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 212, i32 0)
+  %tmp41 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 216, i32 0)
+  %tmp42 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 224, i32 0)
+  %tmp43 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 240, i32 0)
+  %tmp44 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 244, i32 0)
+  %tmp45 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 248, i32 0)
+  %tmp46 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 256, i32 0)
+  %tmp47 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 272, i32 0)
+  %tmp48 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 276, i32 0)
+  %tmp49 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 280, i32 0)
+  %tmp50 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 288, i32 0)
+  %tmp51 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 292, i32 0)
+  %tmp52 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 296, i32 0)
+  %tmp53 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 304, i32 0)
+  %tmp54 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 308, i32 0)
+  %tmp55 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 312, i32 0)
+  %tmp56 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 368, i32 0)
+  %tmp57 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 372, i32 0)
+  %tmp58 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 376, i32 0)
+  %tmp59 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 384, i32 0)
   %tmp60 = getelementptr [16 x <8 x i32>], [16 x <8 x i32>] addrspace(4)* %arg2, i64 0, i32 0
   %tmp61 = load <8 x i32>, <8 x i32> addrspace(4)* %tmp60, !tbaa !0
   %tmp62 = getelementptr [32 x <4 x i32>], [32 x <4 x i32>] addrspace(4)* %arg1, i64 0, i32 0
@@ -647,109 +647,109 @@ define amdgpu_ps void @main1([17 x <4 x i32>] addrspace(4)* byval %arg, [32 x <4
 main_body:
   %tmp = getelementptr [17 x <4 x i32>], [17 x <4 x i32>] addrspace(4)* %arg, i64 0, i32 0
   %tmp21 = load <4 x i32>, <4 x i32> addrspace(4)* %tmp, !tbaa !0
-  %tmp22 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 0)
-  %tmp23 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 4)
-  %tmp24 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 8)
-  %tmp25 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 12)
-  %tmp26 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 28)
-  %tmp27 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 48)
-  %tmp28 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 52)
-  %tmp29 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 56)
-  %tmp30 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 64)
-  %tmp31 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 68)
-  %tmp32 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 72)
-  %tmp33 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 76)
-  %tmp34 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 128)
-  %tmp35 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 132)
-  %tmp36 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 144)
-  %tmp37 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 148)
-  %tmp38 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 152)
-  %tmp39 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 160)
-  %tmp40 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 164)
-  %tmp41 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 168)
-  %tmp42 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 172)
-  %tmp43 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 176)
-  %tmp44 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 180)
-  %tmp45 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 184)
-  %tmp46 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 192)
-  %tmp47 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 196)
-  %tmp48 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 200)
-  %tmp49 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 208)
-  %tmp50 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 212)
-  %tmp51 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 216)
-  %tmp52 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 220)
-  %tmp53 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 236)
-  %tmp54 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 240)
-  %tmp55 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 244)
-  %tmp56 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 248)
-  %tmp57 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 252)
-  %tmp58 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 256)
-  %tmp59 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 260)
-  %tmp60 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 264)
-  %tmp61 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 268)
-  %tmp62 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 272)
-  %tmp63 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 276)
-  %tmp64 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 280)
-  %tmp65 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 284)
-  %tmp66 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 288)
-  %tmp67 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 292)
-  %tmp68 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 464)
-  %tmp69 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 468)
-  %tmp70 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 472)
-  %tmp71 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 496)
-  %tmp72 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 500)
-  %tmp73 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 504)
-  %tmp74 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 512)
-  %tmp75 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 516)
-  %tmp76 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 524)
-  %tmp77 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 532)
-  %tmp78 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 536)
-  %tmp79 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 540)
-  %tmp80 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 544)
-  %tmp81 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 548)
-  %tmp82 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 552)
-  %tmp83 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 556)
-  %tmp84 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 560)
-  %tmp85 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 564)
-  %tmp86 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 568)
-  %tmp87 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 572)
-  %tmp88 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 576)
-  %tmp89 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 580)
-  %tmp90 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 584)
-  %tmp91 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 588)
-  %tmp92 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 592)
-  %tmp93 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 596)
-  %tmp94 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 600)
-  %tmp95 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 604)
-  %tmp96 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 608)
-  %tmp97 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 612)
-  %tmp98 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 616)
-  %tmp99 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 624)
-  %tmp100 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 628)
-  %tmp101 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 632)
-  %tmp102 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 636)
-  %tmp103 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 640)
-  %tmp104 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 644)
-  %tmp105 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 648)
-  %tmp106 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 652)
-  %tmp107 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 656)
-  %tmp108 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 660)
-  %tmp109 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 664)
-  %tmp110 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 668)
-  %tmp111 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 672)
-  %tmp112 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 676)
-  %tmp113 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 680)
-  %tmp114 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 684)
-  %tmp115 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 688)
-  %tmp116 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 692)
-  %tmp117 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 696)
-  %tmp118 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 700)
-  %tmp119 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 704)
-  %tmp120 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 708)
-  %tmp121 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 712)
-  %tmp122 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 716)
-  %tmp123 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 864)
-  %tmp124 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp21, i32 868)
+  %tmp22 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 0, i32 0)
+  %tmp23 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 4, i32 0)
+  %tmp24 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 8, i32 0)
+  %tmp25 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 12, i32 0)
+  %tmp26 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 28, i32 0)
+  %tmp27 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 48, i32 0)
+  %tmp28 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 52, i32 0)
+  %tmp29 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 56, i32 0)
+  %tmp30 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 64, i32 0)
+  %tmp31 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 68, i32 0)
+  %tmp32 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 72, i32 0)
+  %tmp33 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 76, i32 0)
+  %tmp34 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 128, i32 0)
+  %tmp35 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 132, i32 0)
+  %tmp36 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 144, i32 0)
+  %tmp37 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 148, i32 0)
+  %tmp38 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 152, i32 0)
+  %tmp39 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 160, i32 0)
+  %tmp40 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 164, i32 0)
+  %tmp41 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 168, i32 0)
+  %tmp42 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 172, i32 0)
+  %tmp43 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 176, i32 0)
+  %tmp44 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 180, i32 0)
+  %tmp45 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 184, i32 0)
+  %tmp46 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 192, i32 0)
+  %tmp47 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 196, i32 0)
+  %tmp48 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 200, i32 0)
+  %tmp49 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 208, i32 0)
+  %tmp50 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 212, i32 0)
+  %tmp51 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 216, i32 0)
+  %tmp52 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 220, i32 0)
+  %tmp53 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 236, i32 0)
+  %tmp54 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 240, i32 0)
+  %tmp55 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 244, i32 0)
+  %tmp56 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 248, i32 0)
+  %tmp57 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 252, i32 0)
+  %tmp58 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 256, i32 0)
+  %tmp59 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 260, i32 0)
+  %tmp60 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 264, i32 0)
+  %tmp61 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 268, i32 0)
+  %tmp62 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 272, i32 0)
+  %tmp63 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 276, i32 0)
+  %tmp64 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 280, i32 0)
+  %tmp65 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 284, i32 0)
+  %tmp66 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 288, i32 0)
+  %tmp67 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 292, i32 0)
+  %tmp68 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 464, i32 0)
+  %tmp69 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 468, i32 0)
+  %tmp70 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 472, i32 0)
+  %tmp71 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 496, i32 0)
+  %tmp72 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 500, i32 0)
+  %tmp73 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 504, i32 0)
+  %tmp74 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 512, i32 0)
+  %tmp75 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 516, i32 0)
+  %tmp76 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 524, i32 0)
+  %tmp77 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 532, i32 0)
+  %tmp78 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 536, i32 0)
+  %tmp79 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 540, i32 0)
+  %tmp80 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 544, i32 0)
+  %tmp81 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 548, i32 0)
+  %tmp82 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 552, i32 0)
+  %tmp83 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 556, i32 0)
+  %tmp84 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 560, i32 0)
+  %tmp85 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 564, i32 0)
+  %tmp86 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 568, i32 0)
+  %tmp87 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 572, i32 0)
+  %tmp88 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 576, i32 0)
+  %tmp89 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 580, i32 0)
+  %tmp90 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 584, i32 0)
+  %tmp91 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 588, i32 0)
+  %tmp92 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 592, i32 0)
+  %tmp93 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 596, i32 0)
+  %tmp94 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 600, i32 0)
+  %tmp95 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 604, i32 0)
+  %tmp96 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 608, i32 0)
+  %tmp97 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 612, i32 0)
+  %tmp98 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 616, i32 0)
+  %tmp99 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 624, i32 0)
+  %tmp100 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 628, i32 0)
+  %tmp101 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 632, i32 0)
+  %tmp102 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 636, i32 0)
+  %tmp103 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 640, i32 0)
+  %tmp104 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 644, i32 0)
+  %tmp105 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 648, i32 0)
+  %tmp106 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 652, i32 0)
+  %tmp107 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 656, i32 0)
+  %tmp108 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 660, i32 0)
+  %tmp109 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 664, i32 0)
+  %tmp110 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 668, i32 0)
+  %tmp111 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 672, i32 0)
+  %tmp112 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 676, i32 0)
+  %tmp113 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 680, i32 0)
+  %tmp114 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 684, i32 0)
+  %tmp115 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 688, i32 0)
+  %tmp116 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 692, i32 0)
+  %tmp117 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 696, i32 0)
+  %tmp118 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 700, i32 0)
+  %tmp119 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 704, i32 0)
+  %tmp120 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 708, i32 0)
+  %tmp121 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 712, i32 0)
+  %tmp122 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 716, i32 0)
+  %tmp123 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 864, i32 0)
+  %tmp124 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp21, i32 868, i32 0)
   %tmp125 = getelementptr [16 x <8 x i32>], [16 x <8 x i32>] addrspace(4)* %arg2, i64 0, i32 0
   %tmp126 = load <8 x i32>, <8 x i32> addrspace(4)* %tmp125, !tbaa !0
   %tmp127 = getelementptr [32 x <4 x i32>], [32 x <4 x i32>] addrspace(4)* %arg1, i64 0, i32 0
@@ -1683,7 +1683,7 @@ declare <4 x float> @llvm.amdgcn.image.sample.2d.v4f32.f32(i32, float, float, <8
 declare <4 x float> @llvm.amdgcn.image.sample.cube.v4f32.f32(i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #2
 declare <4 x float> @llvm.amdgcn.image.sample.d.2d.v4f32.f32.f32(i32, float, float, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #2
 declare <4 x float> @llvm.amdgcn.image.sample.l.2d.v4f32.f32(i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #2
-declare float @llvm.SI.load.const.v4i32(<4 x i32>, i32) #1
+declare float @llvm.amdgcn.s.buffer.load.f32(<4 x i32>, i32, i32) #1
 
 attributes #0 = { nounwind }
 attributes #1 = { nounwind readnone }
diff --git a/test/CodeGen/AMDGPU/si-spill-cf.ll b/test/CodeGen/AMDGPU/si-spill-cf.ll
index 2a8ced59ddef6e933b02fc26bb5023fb751e2435..153766716790cb77feca79afe2dc6bbb0986233b 100644
--- a/test/CodeGen/AMDGPU/si-spill-cf.ll
+++ b/test/CodeGen/AMDGPU/si-spill-cf.ll
@@ -9,73 +9,73 @@
 
 define amdgpu_ps void @main() #0 {
 main_body:
-  %tmp = call float @llvm.SI.load.const.v4i32(<4 x i32> undef, i32 16)
-  %tmp1 = call float @llvm.SI.load.const.v4i32(<4 x i32> undef, i32 32)
-  %tmp2 = call float @llvm.SI.load.const.v4i32(<4 x i32> undef, i32 80)
-  %tmp3 = call float @llvm.SI.load.const.v4i32(<4 x i32> undef, i32 84)
-  %tmp4 = call float @llvm.SI.load.const.v4i32(<4 x i32> undef, i32 88)
-  %tmp5 = call float @llvm.SI.load.const.v4i32(<4 x i32> undef, i32 96)
-  %tmp6 = call float @llvm.SI.load.const.v4i32(<4 x i32> undef, i32 100)
-  %tmp7 = call float @llvm.SI.load.const.v4i32(<4 x i32> undef, i32 104)
-  %tmp8 = call float @llvm.SI.load.const.v4i32(<4 x i32> undef, i32 112)
-  %tmp9 = call float @llvm.SI.load.const.v4i32(<4 x i32> undef, i32 116)
-  %tmp10 = call float @llvm.SI.load.const.v4i32(<4 x i32> undef, i32 120)
-  %tmp11 = call float @llvm.SI.load.const.v4i32(<4 x i32> undef, i32 128)
-  %tmp12 = call float @llvm.SI.load.const.v4i32(<4 x i32> undef, i32 132)
-  %tmp13 = call float @llvm.SI.load.const.v4i32(<4 x i32> undef, i32 136)
-  %tmp14 = call float @llvm.SI.load.const.v4i32(<4 x i32> undef, i32 144)
-  %tmp15 = call float @llvm.SI.load.const.v4i32(<4 x i32> undef, i32 148)
-  %tmp16 = call float @llvm.SI.load.const.v4i32(<4 x i32> undef, i32 152)
-  %tmp17 = call float @llvm.SI.load.const.v4i32(<4 x i32> undef, i32 160)
-  %tmp18 = call float @llvm.SI.load.const.v4i32(<4 x i32> undef, i32 164)
-  %tmp19 = call float @llvm.SI.load.const.v4i32(<4 x i32> undef, i32 168)
-  %tmp20 = call float @llvm.SI.load.const.v4i32(<4 x i32> undef, i32 176)
-  %tmp21 = call float @llvm.SI.load.const.v4i32(<4 x i32> undef, i32 180)
-  %tmp22 = call float @llvm.SI.load.const.v4i32(<4 x i32> undef, i32 184)
-  %tmp23 = call float @llvm.SI.load.const.v4i32(<4 x i32> undef, i32 192)
-  %tmp24 = call float @llvm.SI.load.const.v4i32(<4 x i32> undef, i32 196)
-  %tmp25 = call float @llvm.SI.load.const.v4i32(<4 x i32> undef, i32 200)
-  %tmp26 = call float @llvm.SI.load.const.v4i32(<4 x i32> undef, i32 208)
-  %tmp27 = call float @llvm.SI.load.const.v4i32(<4 x i32> undef, i32 212)
-  %tmp28 = call float @llvm.SI.load.const.v4i32(<4 x i32> undef, i32 216)
-  %tmp29 = call float @llvm.SI.load.const.v4i32(<4 x i32> undef, i32 224)
-  %tmp30 = call float @llvm.SI.load.const.v4i32(<4 x i32> undef, i32 228)
-  %tmp31 = call float @llvm.SI.load.const.v4i32(<4 x i32> undef, i32 232)
-  %tmp32 = call float @llvm.SI.load.const.v4i32(<4 x i32> undef, i32 240)
-  %tmp33 = call float @llvm.SI.load.const.v4i32(<4 x i32> undef, i32 244)
-  %tmp34 = call float @llvm.SI.load.const.v4i32(<4 x i32> undef, i32 248)
-  %tmp35 = call float @llvm.SI.load.const.v4i32(<4 x i32> undef, i32 256)
-  %tmp36 = call float @llvm.SI.load.const.v4i32(<4 x i32> undef, i32 260)
-  %tmp37 = call float @llvm.SI.load.const.v4i32(<4 x i32> undef, i32 264)
-  %tmp38 = call float @llvm.SI.load.const.v4i32(<4 x i32> undef, i32 272)
-  %tmp39 = call float @llvm.SI.load.const.v4i32(<4 x i32> undef, i32 276)
-  %tmp40 = call float @llvm.SI.load.const.v4i32(<4 x i32> undef, i32 280)
-  %tmp41 = call float @llvm.SI.load.const.v4i32(<4 x i32> undef, i32 288)
-  %tmp42 = call float @llvm.SI.load.const.v4i32(<4 x i32> undef, i32 292)
-  %tmp43 = call float @llvm.SI.load.const.v4i32(<4 x i32> undef, i32 296)
-  %tmp44 = call float @llvm.SI.load.const.v4i32(<4 x i32> undef, i32 304)
-  %tmp45 = call float @llvm.SI.load.const.v4i32(<4 x i32> undef, i32 308)
-  %tmp46 = call float @llvm.SI.load.const.v4i32(<4 x i32> undef, i32 312)
-  %tmp47 = call float @llvm.SI.load.const.v4i32(<4 x i32> undef, i32 320)
-  %tmp48 = call float @llvm.SI.load.const.v4i32(<4 x i32> undef, i32 324)
-  %tmp49 = call float @llvm.SI.load.const.v4i32(<4 x i32> undef, i32 328)
-  %tmp50 = call float @llvm.SI.load.const.v4i32(<4 x i32> undef, i32 336)
-  %tmp51 = call float @llvm.SI.load.const.v4i32(<4 x i32> undef, i32 340)
-  %tmp52 = call float @llvm.SI.load.const.v4i32(<4 x i32> undef, i32 344)
-  %tmp53 = call float @llvm.SI.load.const.v4i32(<4 x i32> undef, i32 352)
-  %tmp54 = call float @llvm.SI.load.const.v4i32(<4 x i32> undef, i32 356)
-  %tmp55 = call float @llvm.SI.load.const.v4i32(<4 x i32> undef, i32 360)
-  %tmp56 = call float @llvm.SI.load.const.v4i32(<4 x i32> undef, i32 368)
-  %tmp57 = call float @llvm.SI.load.const.v4i32(<4 x i32> undef, i32 372)
-  %tmp58 = call float @llvm.SI.load.const.v4i32(<4 x i32> undef, i32 376)
-  %tmp59 = call float @llvm.SI.load.const.v4i32(<4 x i32> undef, i32 384)
-  %tmp60 = call float @llvm.SI.load.const.v4i32(<4 x i32> undef, i32 388)
-  %tmp61 = call float @llvm.SI.load.const.v4i32(<4 x i32> undef, i32 392)
-  %tmp62 = call float @llvm.SI.load.const.v4i32(<4 x i32> undef, i32 400)
-  %tmp63 = call float @llvm.SI.load.const.v4i32(<4 x i32> undef, i32 404)
-  %tmp64 = call float @llvm.SI.load.const.v4i32(<4 x i32> undef, i32 408)
-  %tmp65 = call float @llvm.SI.load.const.v4i32(<4 x i32> undef, i32 416)
-  %tmp66 = call float @llvm.SI.load.const.v4i32(<4 x i32> undef, i32 420)
+  %tmp = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> undef, i32 16, i32 0)
+  %tmp1 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> undef, i32 32, i32 0)
+  %tmp2 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> undef, i32 80, i32 0)
+  %tmp3 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> undef, i32 84, i32 0)
+  %tmp4 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> undef, i32 88, i32 0)
+  %tmp5 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> undef, i32 96, i32 0)
+  %tmp6 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> undef, i32 100, i32 0)
+  %tmp7 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> undef, i32 104, i32 0)
+  %tmp8 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> undef, i32 112, i32 0)
+  %tmp9 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> undef, i32 116, i32 0)
+  %tmp10 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> undef, i32 120, i32 0)
+  %tmp11 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> undef, i32 128, i32 0)
+  %tmp12 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> undef, i32 132, i32 0)
+  %tmp13 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> undef, i32 136, i32 0)
+  %tmp14 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> undef, i32 144, i32 0)
+  %tmp15 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> undef, i32 148, i32 0)
+  %tmp16 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> undef, i32 152, i32 0)
+  %tmp17 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> undef, i32 160, i32 0)
+  %tmp18 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> undef, i32 164, i32 0)
+  %tmp19 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> undef, i32 168, i32 0)
+  %tmp20 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> undef, i32 176, i32 0)
+  %tmp21 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> undef, i32 180, i32 0)
+  %tmp22 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> undef, i32 184, i32 0)
+  %tmp23 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> undef, i32 192, i32 0)
+  %tmp24 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> undef, i32 196, i32 0)
+  %tmp25 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> undef, i32 200, i32 0)
+  %tmp26 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> undef, i32 208, i32 0)
+  %tmp27 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> undef, i32 212, i32 0)
+  %tmp28 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> undef, i32 216, i32 0)
+  %tmp29 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> undef, i32 224, i32 0)
+  %tmp30 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> undef, i32 228, i32 0)
+  %tmp31 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> undef, i32 232, i32 0)
+  %tmp32 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> undef, i32 240, i32 0)
+  %tmp33 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> undef, i32 244, i32 0)
+  %tmp34 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> undef, i32 248, i32 0)
+  %tmp35 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> undef, i32 256, i32 0)
+  %tmp36 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> undef, i32 260, i32 0)
+  %tmp37 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> undef, i32 264, i32 0)
+  %tmp38 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> undef, i32 272, i32 0)
+  %tmp39 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> undef, i32 276, i32 0)
+  %tmp40 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> undef, i32 280, i32 0)
+  %tmp41 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> undef, i32 288, i32 0)
+  %tmp42 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> undef, i32 292, i32 0)
+  %tmp43 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> undef, i32 296, i32 0)
+  %tmp44 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> undef, i32 304, i32 0)
+  %tmp45 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> undef, i32 308, i32 0)
+  %tmp46 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> undef, i32 312, i32 0)
+  %tmp47 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> undef, i32 320, i32 0)
+  %tmp48 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> undef, i32 324, i32 0)
+  %tmp49 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> undef, i32 328, i32 0)
+  %tmp50 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> undef, i32 336, i32 0)
+  %tmp51 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> undef, i32 340, i32 0)
+  %tmp52 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> undef, i32 344, i32 0)
+  %tmp53 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> undef, i32 352, i32 0)
+  %tmp54 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> undef, i32 356, i32 0)
+  %tmp55 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> undef, i32 360, i32 0)
+  %tmp56 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> undef, i32 368, i32 0)
+  %tmp57 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> undef, i32 372, i32 0)
+  %tmp58 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> undef, i32 376, i32 0)
+  %tmp59 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> undef, i32 384, i32 0)
+  %tmp60 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> undef, i32 388, i32 0)
+  %tmp61 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> undef, i32 392, i32 0)
+  %tmp62 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> undef, i32 400, i32 0)
+  %tmp63 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> undef, i32 404, i32 0)
+  %tmp64 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> undef, i32 408, i32 0)
+  %tmp65 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> undef, i32 416, i32 0)
+  %tmp66 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> undef, i32 420, i32 0)
   br label %LOOP
 
 LOOP:                                             ; preds = %ENDIF2795, %main_body
@@ -497,7 +497,7 @@ declare float @llvm.minnum.f32(float, float) #1
 declare float @llvm.maxnum.f32(float, float) #1
 declare i32 @llvm.amdgcn.mbcnt.lo(i32, i32) #1
 declare void @llvm.amdgcn.exp.f32(i32, i32, float, float, float, float, i1, i1) #0
-declare float @llvm.SI.load.const.v4i32(<4 x i32>, i32) #1
+declare float @llvm.amdgcn.s.buffer.load.f32(<4 x i32>, i32, i32) #1
 
 attributes #0 = { nounwind }
 attributes #1 = { nounwind readnone }
diff --git a/test/CodeGen/AMDGPU/sibling-call.ll b/test/CodeGen/AMDGPU/sibling-call.ll
index 260953b184c3e23e103d973e2b97c357595c2d87..1881b526dcda8087799c640b622174bbbca8605b 100644
--- a/test/CodeGen/AMDGPU/sibling-call.ll
+++ b/test/CodeGen/AMDGPU/sibling-call.ll
@@ -32,7 +32,7 @@ define fastcc i32 @i32_fastcc_i32_i32_stack_object(i32 %arg0, i32 %arg1) #1 {
 }
 
 ; GCN-LABEL: {{^}}sibling_call_i32_fastcc_i32_i32:
-define fastcc i32 @sibling_call_i32_fastcc_i32_i32(i32 %a, i32 %b, i32 %c) #1 {
+define hidden fastcc i32 @sibling_call_i32_fastcc_i32_i32(i32 %a, i32 %b, i32 %c) #1 {
 entry:
   %ret = tail call fastcc i32 @i32_fastcc_i32_i32(i32 %a, i32 %b)
   ret i32 %ret
@@ -92,7 +92,7 @@ entry:
 ; GFX9-NEXT: v_add_u32_e32 v0, v0, v1
 
 ; GCN-NEXT: s_setpc_b64 s[30:31]
-define fastcc i32 @i32_fastcc_i32_byval_i32(i32 %arg0, i32 addrspace(5)* byval align 4 %arg1) #1 {
+define hidden fastcc i32 @i32_fastcc_i32_byval_i32(i32 %arg0, i32 addrspace(5)* byval align 4 %arg1) #1 {
   %arg1.load = load i32, i32 addrspace(5)* %arg1, align 4
   %add0 = add i32 %arg0, %arg1.load
   ret i32 %add0
diff --git a/test/CodeGen/AMDGPU/smem-no-clause-coalesced.mir b/test/CodeGen/AMDGPU/smem-no-clause-coalesced.mir
new file mode 100644
index 0000000000000000000000000000000000000000..2906259fb46cf176b30f01f2b2418eb538c82555
--- /dev/null
+++ b/test/CodeGen/AMDGPU/smem-no-clause-coalesced.mir
@@ -0,0 +1,48 @@
+# RUN: llc -march=amdgcn -mcpu=gfx902 -o - %s -run-pass si-form-memory-clauses -verify-machineinstrs | FileCheck -check-prefix=XNACK %s
+
+# The SIFormMemoryClauses pass must not form a clause (indicated by BUNDLE)
+# from the two adjacent smem instructions, because the first one has its
+# result coalesced with an operand.
+
+# XNACK-LABEL: body:
+# XNACK-NOT: BUNDLE
+
+---
+name:            _amdgpu_cs_main
+alignment:       0
+exposesReturnsTwice: false
+legalized:       false
+regBankSelected: false
+selected:        false
+failedISel:      false
+tracksRegLiveness: true
+hasWinCFI:       false
+fixedStack:      []
+stack:           []
+constants:       []
+body:             |
+  bb.0:
+    liveins: $sgpr2, $sgpr3, $sgpr12, $sgpr13, $sgpr14, $vgpr0, $vgpr1
+
+    %0:vgpr_32 = COPY $vgpr1
+    %1:sgpr_32 = COPY $sgpr12
+    %2:sgpr_32 = COPY $sgpr3
+    undef %3.sub0:sgpr_128 = COPY $sgpr2
+    %4:vgpr_32 = COPY $vgpr0
+    %5:sgpr_32 = COPY $sgpr14
+    %6:sgpr_32 = COPY $sgpr13
+    %7:sreg_64_xexec = S_GETPC_B64
+    %7.sub0:sreg_64_xexec = COPY %1
+    %3.sub1:sgpr_128 = S_AND_B32 %2, 65535, implicit-def dead $scc
+    %3.sub3:sgpr_128 = S_MOV_B32 151468
+    %3.sub2:sgpr_128 = S_MOV_B32 -1
+    %7.sub0:sreg_64_xexec = S_LOAD_DWORD_IMM %7, 48, 0 :: (load 4 from `i8 addrspace(4)* undef`, addrspace 4)
+    %8:sreg_64_xexec = S_BUFFER_LOAD_DWORDX2_IMM %3, 640, 0 :: (dereferenceable invariant load 8)
+    undef %9.sub0:vreg_128 = V_LSHL_ADD_U32 %6, 4, %4, implicit $exec
+    %9.sub1:vreg_128 = V_LSHL_ADD_U32 %5, 4, %0, implicit $exec
+    S_ENDPGM
+
+...
+
+
+
diff --git a/test/CodeGen/AMDGPU/smrd.ll b/test/CodeGen/AMDGPU/smrd.ll
index 22ee62ef427606039e07d2bc6a026cccd485b881..c83eb378a1e1fd07e6b5c30abcb339d50cb0e701 100644
--- a/test/CodeGen/AMDGPU/smrd.ll
+++ b/test/CodeGen/AMDGPU/smrd.ll
@@ -98,7 +98,7 @@ main_body:
   %d1 = insertelement <4 x i32> %d0, i32 1, i32 1
   %d2 = insertelement <4 x i32> %d1, i32 2, i32 2
   %d3 = insertelement <4 x i32> %d2, i32 3, i32 3
-  %r = call float @llvm.SI.load.const.v4i32(<4 x i32> %d3, i32 0)
+  %r = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %d3, i32 0, i32 0)
   ret float %r
 }
 
@@ -110,7 +110,7 @@ define amdgpu_ps void @smrd_load_const0(<4 x i32> addrspace(4)* inreg %arg, <4 x
 main_body:
   %tmp = getelementptr <4 x i32>, <4 x i32> addrspace(4)* %arg, i32 0
   %tmp20 = load <4 x i32>, <4 x i32> addrspace(4)* %tmp
-  %tmp21 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp20, i32 16)
+  %tmp21 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp20, i32 16, i32 0)
   call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %tmp21, float %tmp21, float %tmp21, float %tmp21, i1 true, i1 true) #0
   ret void
 }
@@ -126,7 +126,7 @@ define amdgpu_ps void @smrd_load_const1(<4 x i32> addrspace(4)* inreg %arg, <4 x
 main_body:
   %tmp = getelementptr <4 x i32>, <4 x i32> addrspace(4)* %arg, i32 0
   %tmp20 = load <4 x i32>, <4 x i32> addrspace(4)* %tmp
-  %tmp21 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp20, i32 1020)
+  %tmp21 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp20, i32 1020, i32 0)
   %tmp22 = load <4 x i32>, <4 x i32> addrspace(4)* %in
   %s.buffer = call i32 @llvm.amdgcn.s.buffer.load.i32(<4 x i32> %tmp22, i32 1020, i32 1)
   %s.buffer.float = bitcast i32 %s.buffer to float
@@ -149,7 +149,7 @@ define amdgpu_ps void @smrd_load_const2(<4 x i32> addrspace(4)* inreg %arg, <4 x
 main_body:
   %tmp = getelementptr <4 x i32>, <4 x i32> addrspace(4)* %arg, i32 0
   %tmp20 = load <4 x i32>, <4 x i32> addrspace(4)* %tmp
-  %tmp21 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp20, i32 1024)
+  %tmp21 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp20, i32 1024, i32 0)
   %tmp22 = load <4 x i32>, <4 x i32> addrspace(4)* %in
   %s.buffer = call i32 @llvm.amdgcn.s.buffer.load.i32(<4 x i32> %tmp22, i32 1024, i32 0)
   %s.buffer.float = bitcast i32 %s.buffer to float
@@ -170,7 +170,7 @@ define amdgpu_ps void @smrd_load_const3(<4 x i32> addrspace(4)* inreg %arg, <4 x
 main_body:
   %tmp = getelementptr <4 x i32>, <4 x i32> addrspace(4)* %arg, i32 0
   %tmp20 = load <4 x i32>, <4 x i32> addrspace(4)* %tmp
-  %tmp21 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp20, i32 1048572)
+  %tmp21 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp20, i32 1048572, i32 0)
   %tmp22 = load <4 x i32>, <4 x i32> addrspace(4)* %in
   %s.buffer = call i32 @llvm.amdgcn.s.buffer.load.i32(<4 x i32> %tmp22, i32 1048572, i32 0)
   %s.buffer.float = bitcast i32 %s.buffer to float
@@ -190,7 +190,7 @@ define amdgpu_ps void @smrd_load_const4(<4 x i32> addrspace(4)* inreg %arg, <4 x
 main_body:
   %tmp = getelementptr <4 x i32>, <4 x i32> addrspace(4)* %arg, i32 0
   %tmp20 = load <4 x i32>, <4 x i32> addrspace(4)* %tmp
-  %tmp21 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp20, i32 1048576)
+  %tmp21 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp20, i32 1048576, i32 0)
   %tmp22 = load <4 x i32>, <4 x i32> addrspace(4)* %in
   %s.buffer = call i32 @llvm.amdgcn.s.buffer.load.i32(<4 x i32> %tmp22, i32 1048576, i32 0)
   %s.buffer.float = bitcast i32 %s.buffer to float
@@ -278,7 +278,7 @@ main_body:
 ; GCN: s_buffer_load_dword s{{[0-9]}}, s[0:3], s4
 define amdgpu_ps float @smrd_sgpr_offset(<4 x i32> inreg %desc, i32 inreg %offset) #0 {
 main_body:
-  %r = call float @llvm.SI.load.const.v4i32(<4 x i32> %desc, i32 %offset)
+  %r = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %desc, i32 %offset, i32 0)
   ret float %r
 }
 
@@ -286,7 +286,7 @@ main_body:
 ; GCN: buffer_load_dword v{{[0-9]}}, v0, s[0:3], 0 offen ;
 define amdgpu_ps float @smrd_vgpr_offset(<4 x i32> inreg %desc, i32 %offset) #0 {
 main_body:
-  %r = call float @llvm.SI.load.const.v4i32(<4 x i32> %desc, i32 %offset)
+  %r = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %desc, i32 %offset, i32 0)
   ret float %r
 }
 
@@ -296,7 +296,7 @@ main_body:
 define amdgpu_ps float @smrd_vgpr_offset_imm(<4 x i32> inreg %desc, i32 %offset) #0 {
 main_body:
   %off = add i32 %offset, 4092
-  %r = call float @llvm.SI.load.const.v4i32(<4 x i32> %desc, i32 %off)
+  %r = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %desc, i32 %off, i32 0)
   ret float %r
 }
 
@@ -308,7 +308,7 @@ main_body:
 define amdgpu_ps float @smrd_vgpr_offset_imm_too_large(<4 x i32> inreg %desc, i32 %offset) #0 {
 main_body:
   %off = add i32 %offset, 4096
-  %r = call float @llvm.SI.load.const.v4i32(<4 x i32> %desc, i32 %off)
+  %r = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %desc, i32 %off, i32 0)
   ret float %r
 }
 
@@ -320,12 +320,12 @@ main_body:
 ; VIGFX9-NEXT: s_buffer_load_dwordx2 s[{{[0-9]}}:{{[0-9]}}], s[0:3], 0x1c
 define amdgpu_ps void @smrd_imm_merged(<4 x i32> inreg %desc) #0 {
 main_body:
-  %r1 = call float @llvm.SI.load.const.v4i32(<4 x i32> %desc, i32 4)
-  %r2 = call float @llvm.SI.load.const.v4i32(<4 x i32> %desc, i32 8)
-  %r3 = call float @llvm.SI.load.const.v4i32(<4 x i32> %desc, i32 12)
-  %r4 = call float @llvm.SI.load.const.v4i32(<4 x i32> %desc, i32 16)
-  %r5 = call float @llvm.SI.load.const.v4i32(<4 x i32> %desc, i32 28)
-  %r6 = call float @llvm.SI.load.const.v4i32(<4 x i32> %desc, i32 32)
+  %r1 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %desc, i32 4, i32 0)
+  %r2 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %desc, i32 8, i32 0)
+  %r3 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %desc, i32 12, i32 0)
+  %r4 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %desc, i32 16, i32 0)
+  %r5 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %desc, i32 28, i32 0)
+  %r6 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %desc, i32 32, i32 0)
   call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %r1, float %r2, float %r3, float %r4, i1 true, i1 true) #0
   call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %r5, float %r6, float undef, float undef, i1 true, i1 true) #0
   ret void
@@ -352,7 +352,7 @@ main_body:
 ;
 define amdgpu_ps float @smrd_imm_merge_m0(<4 x i32> inreg %desc, i32 inreg %prim, float %u, float %v) #0 {
 main_body:
-  %idx1.f = call float @llvm.SI.load.const.v4i32(<4 x i32> %desc, i32 0)
+  %idx1.f = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %desc, i32 0, i32 0)
   %idx1 = bitcast float %idx1.f to i32
 
   %v0.x1 = call nsz float @llvm.amdgcn.interp.p1(float %u, i32 0, i32 0, i32 %prim)
@@ -377,7 +377,7 @@ main_body:
   %v1 = insertelement <3 x float> %v0.tmp1, float %v0.z, i32 2
 
   %b = extractelement <3 x float> %v1, i32 %idx1
-  %c = call float @llvm.SI.load.const.v4i32(<4 x i32> %desc, i32 4)
+  %c = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %desc, i32 4, i32 0)
 
   %res.tmp = fadd float %a, %b
   %res = fadd float %res.tmp, %c
@@ -396,12 +396,12 @@ main_body:
   %a4 = add i32 %a, 16
   %a5 = add i32 %a, 28
   %a6 = add i32 %a, 32
-  %r1 = call float @llvm.SI.load.const.v4i32(<4 x i32> %desc, i32 %a1)
-  %r2 = call float @llvm.SI.load.const.v4i32(<4 x i32> %desc, i32 %a2)
-  %r3 = call float @llvm.SI.load.const.v4i32(<4 x i32> %desc, i32 %a3)
-  %r4 = call float @llvm.SI.load.const.v4i32(<4 x i32> %desc, i32 %a4)
-  %r5 = call float @llvm.SI.load.const.v4i32(<4 x i32> %desc, i32 %a5)
-  %r6 = call float @llvm.SI.load.const.v4i32(<4 x i32> %desc, i32 %a6)
+  %r1 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %desc, i32 %a1, i32 0)
+  %r2 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %desc, i32 %a2, i32 0)
+  %r3 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %desc, i32 %a3, i32 0)
+  %r4 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %desc, i32 %a4, i32 0)
+  %r5 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %desc, i32 %a5, i32 0)
+  %r6 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %desc, i32 %a6, i32 0)
   call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %r1, float %r2, float %r3, float %r4, i1 true, i1 true) #0
   call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %r5, float %r6, float undef, float undef, i1 true, i1 true) #0
   ret void
@@ -428,14 +428,14 @@ ret_block:                                       ; preds = %.outer, %.label22, %
 
 .inner_loop_body:
   %descriptor = load <4 x i32>, <4 x i32> addrspace(4)* %descptr, align 16, !invariant.load !0
-  %load1result = call float @llvm.SI.load.const.v4i32(<4 x i32> %descriptor, i32 0)
+  %load1result = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %descriptor, i32 0, i32 0)
   store float %load1result, float addrspace(1)* undef
   %inner_br2 = icmp uge i32 %1, 10
   br i1 %inner_br2, label %.inner_loop_header, label %.outer_loop_body
 
 .outer_loop_body:
   %offset = shl i32 %loopctr.2, 6
-  %load2result = call float @llvm.SI.load.const.v4i32(<4 x i32> %descriptor, i32 %offset)
+  %load2result = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %descriptor, i32 %offset, i32 0)
   %outer_br = fcmp ueq float %load2result, 0x0
   br i1 %outer_br, label %.outer_loop_header, label %ret_block
 }
@@ -451,7 +451,7 @@ define amdgpu_ps void @smrd_load_nonconst0(<4 x i32> addrspace(4)* inreg %arg, <
 main_body:
   %tmp = getelementptr <4 x i32>, <4 x i32> addrspace(4)* %arg, i32 0
   %tmp20 = load <4 x i32>, <4 x i32> addrspace(4)* %tmp
-  %tmp21 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp20, i32 %ncoff)
+  %tmp21 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp20, i32 %ncoff, i32 0)
   %tmp22 = load <4 x i32>, <4 x i32> addrspace(4)* %in
   %s.buffer = call i32 @llvm.amdgcn.s.buffer.load.i32(<4 x i32> %tmp22, i32 %ncoff, i32 0)
   %s.buffer.float = bitcast i32 %s.buffer to float
@@ -470,7 +470,7 @@ define amdgpu_ps void @smrd_load_nonconst1(<4 x i32> addrspace(4)* inreg %arg, <
 main_body:
   %tmp = getelementptr <4 x i32>, <4 x i32> addrspace(4)* %arg, i32 0
   %tmp20 = load <4 x i32>, <4 x i32> addrspace(4)* %tmp
-  %tmp21 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp20, i32 %ncoff)
+  %tmp21 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp20, i32 %ncoff, i32 0)
   %tmp22 = load <4 x i32>, <4 x i32> addrspace(4)* %in
   %s.buffer = call i32 @llvm.amdgcn.s.buffer.load.i32(<4 x i32> %tmp22, i32 %ncoff, i32 0)
   %s.buffer.float = bitcast i32 %s.buffer to float
@@ -489,7 +489,7 @@ define amdgpu_ps void @smrd_load_nonconst2(<4 x i32> addrspace(4)* inreg %arg, <
 main_body:
   %tmp = getelementptr <4 x i32>, <4 x i32> addrspace(4)* %arg, i32 0
   %tmp20 = load <4 x i32>, <4 x i32> addrspace(4)* %tmp
-  %tmp21 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp20, i32 %ncoff)
+  %tmp21 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp20, i32 %ncoff, i32 0)
   %tmp22 = load <4 x i32>, <4 x i32> addrspace(4)* %in
   %s.buffer = call <8 x i32> @llvm.amdgcn.s.buffer.load.v8i32(<4 x i32> %tmp22, i32 %ncoff, i32 0)
   %s.buffer.elt = extractelement <8 x i32> %s.buffer, i32 1
@@ -581,7 +581,7 @@ loop:
   %counter = phi i32 [ 0, %main_body ], [ %counter.next, %loop ]
   %sum = phi float [ 0.0, %main_body ], [ %sum.next, %loop ]
   %offset = shl i32 %counter, 2
-  %v = call float @llvm.SI.load.const.v4i32(<4 x i32> %desc, i32 %offset)
+  %v = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %desc, i32 %offset, i32 0)
   %sum.next = fadd float %sum, %v
   %counter.next = add i32 %counter, 1
   %cc = icmp uge i32 %counter.next, %bound
@@ -607,7 +607,7 @@ loop:
   %counter = phi i32 [ 0, %main_body ], [ %counter.next, %loop.a ], [ %counter.next, %loop.b ]
   %sum = phi float [ 0.0, %main_body ], [ %sum.next, %loop.a ], [ %sum.next.b, %loop.b ]
   %offset = shl i32 %counter, 2
-  %v = call float @llvm.SI.load.const.v4i32(<4 x i32> %desc, i32 %offset)
+  %v = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %desc, i32 %offset, i32 0)
   %sum.next = fadd float %sum, %v
   %counter.next = add i32 %counter, 1
   %cc = icmp uge i32 %counter.next, %bound
@@ -644,7 +644,7 @@ if1:                                              ; preds = %main_body
 
 endif1:                                           ; preds = %if1, %main_body
   %tmp13 = extractelement <3 x i32> %arg4, i32 0
-  %tmp97 = call float @llvm.SI.load.const.v4i32(<4 x i32> undef, i32 %tmp13)
+  %tmp97 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> undef, i32 %tmp13, i32 0)
   ret float %tmp97
 }
 
@@ -689,10 +689,10 @@ define amdgpu_ps void @s_buffer_load_v16f32(<4 x i32> inreg %rsrc, i32 inreg %of
 }
 
 declare void @llvm.amdgcn.exp.f32(i32, i32, float, float, float, float, i1, i1) #0
-declare float @llvm.SI.load.const.v4i32(<4 x i32>, i32) #1
 declare float @llvm.amdgcn.interp.p1(float, i32, i32, i32) #2
 declare float @llvm.amdgcn.interp.p2(float, float, i32, i32, i32) #2
-declare i32 @llvm.amdgcn.s.buffer.load.i32(<4 x i32>, i32, i32)
+
+declare i32 @llvm.amdgcn.s.buffer.load.i32(<4 x i32>, i32, i32) #1
 declare <2 x i32> @llvm.amdgcn.s.buffer.load.v2i32(<4 x i32>, i32, i32)
 declare <4 x i32> @llvm.amdgcn.s.buffer.load.v4i32(<4 x i32>, i32, i32)
 declare <8 x i32> @llvm.amdgcn.s.buffer.load.v8i32(<4 x i32>, i32, i32)
diff --git a/test/CodeGen/AMDGPU/split-smrd.ll b/test/CodeGen/AMDGPU/split-smrd.ll
index 46355b762b41aa584bef5c73ef039aa87fd910d9..8fa272762ce03490aae1fd7ff6e49382459820d3 100644
--- a/test/CodeGen/AMDGPU/split-smrd.ll
+++ b/test/CodeGen/AMDGPU/split-smrd.ll
@@ -8,7 +8,7 @@
 ; GCN: image_sample v{{[0-9]+}}, v[{{[0-9]+:[0-9]+}}], s[{{[0-9]+:[0-9]+}}], s[{{[0-9]+:[0-9]+}}] dmask:0x1
 define amdgpu_ps void @split_smrd_add_worklist([34 x <8 x i32>] addrspace(4)* byval %arg) #0 {
 bb:
-  %tmp = call float @llvm.SI.load.const.v4i32(<4 x i32> undef, i32 96)
+  %tmp = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> undef, i32 96, i32 0)
   %tmp1 = bitcast float %tmp to i32
   br i1 undef, label %bb2, label %bb3
 
@@ -31,7 +31,7 @@ bb3:                                              ; preds = %bb
 declare <2 x half> @llvm.amdgcn.cvt.pkrtz(float, float) #1
 declare void @llvm.amdgcn.exp.compr.v2f16(i32, i32, <2 x half>, <2 x half>, i1, i1) #0
 declare <4 x float> @llvm.amdgcn.image.sample.2d.v4f32.f32(i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #2
-declare float @llvm.SI.load.const.v4i32(<4 x i32>, i32) #1
+declare float @llvm.amdgcn.s.buffer.load.f32(<4 x i32>, i32, i32) #1
 
 attributes #0 = { nounwind }
 attributes #1 = { nounwind readnone }
diff --git a/test/CodeGen/AMDGPU/ssubo.ll b/test/CodeGen/AMDGPU/ssubo.ll
index fee14b48b44adc8ef91b920f620483b4491e8008..4f4bab84150de9b1cc19ea6644c85f546dc37ef9 100644
--- a/test/CodeGen/AMDGPU/ssubo.ll
+++ b/test/CodeGen/AMDGPU/ssubo.ll
@@ -1,10 +1,11 @@
 ; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=tahiti -verify-machineinstrs< %s | FileCheck -check-prefixes=GCN,SI,FUNC %s
 ; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=tonga -verify-machineinstrs< %s | FileCheck -check-prefixes=GCN,VI,FUNC %s
 ; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=gfx900 -verify-machineinstrs< %s | FileCheck -check-prefixes=GCN,GFX9,FUNC %s
-; RUN: llc -amdgpu-scalarize-global-loads=false -march=r600 -mcpu=cypress -verify-machineinstrs< %s
+
 
 declare { i32, i1 } @llvm.ssub.with.overflow.i32(i32, i32) nounwind readnone
 declare { i64, i1 } @llvm.ssub.with.overflow.i64(i64, i64) nounwind readnone
+declare { <2 x i32>, <2 x i1> } @llvm.ssub.with.overflow.v2i32(<2 x i32>, <2 x i32>) nounwind readnone
 
 ; FUNC-LABEL: {{^}}ssubo_i64_zext:
 define amdgpu_kernel void @ssubo_i64_zext(i64 addrspace(1)* %out, i64 %a, i64 %b) nounwind {
@@ -70,3 +71,22 @@ define amdgpu_kernel void @v_ssubo_i64(i64 addrspace(1)* %out, i1 addrspace(1)*
   store i1 %carry, i1 addrspace(1)* %carryout
   ret void
 }
+
+; FUNC-LABEL: {{^}}v_ssubo_v2i32:
+; SICIVI: v_cmp_lt_i32
+; SICIVI: v_cmp_lt_i32
+; SICIVI: v_sub_{{[iu]}}32
+; SICIVI: v_cmp_lt_i32
+; SICIVI: v_cmp_lt_i32
+; SICIVI: v_sub_{{[iu]}}32
+define amdgpu_kernel void @v_ssubo_v2i32(<2 x i32> addrspace(1)* %out, <2 x i32> addrspace(1)* %carryout, <2 x i32> addrspace(1)* %aptr, <2 x i32> addrspace(1)* %bptr) nounwind {
+  %a = load <2 x i32>, <2 x i32> addrspace(1)* %aptr, align 4
+  %b = load <2 x i32>, <2 x i32> addrspace(1)* %bptr, align 4
+  %sadd = call { <2 x i32>, <2 x i1> } @llvm.ssub.with.overflow.v2i32(<2 x i32> %a, <2 x i32> %b) nounwind
+  %val = extractvalue { <2 x i32>, <2 x i1> } %sadd, 0
+  %carry = extractvalue { <2 x i32>, <2 x i1> } %sadd, 1
+  store <2 x i32> %val, <2 x i32> addrspace(1)* %out, align 4
+  %carry.ext = zext <2 x i1> %carry to <2 x i32>
+  store <2 x i32> %carry.ext, <2 x i32> addrspace(1)* %carryout
+  ret void
+}
diff --git a/test/CodeGen/AMDGPU/store-global.ll b/test/CodeGen/AMDGPU/store-global.ll
index 50f6b81d7ac71cadbc229b1fd9b9fea9b15cf5a8..c0b9f6cdc333b744d8af7ec4dea044e5801c271c 100644
--- a/test/CodeGen/AMDGPU/store-global.ll
+++ b/test/CodeGen/AMDGPU/store-global.ll
@@ -1,5 +1,5 @@
-; RUN: llc -march=amdgcn -mtriple=amdgcn-- -mcpu=verde -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SIVI -check-prefix=FUNC %s
-; RUN: llc -march=amdgcn -mtriple=amdgcn-- -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SIVI -check-prefix=FUNC %s
+; RUN: llc -march=amdgcn -mtriple=amdgcn-- -mcpu=verde -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SIVI -check-prefix=SI -check-prefix=FUNC %s
+; RUN: llc -march=amdgcn -mtriple=amdgcn-- -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=SIVI -check-prefix=VI -check-prefix=FUNC %s
 ; RUN: llc -march=amdgcn -mtriple=amdgcn-- -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=GFX9 -check-prefix=FUNC %s
 ; RUN: llc -march=r600 -mtriple=r600-- -mcpu=redwood -verify-machineinstrs < %s | FileCheck -check-prefix=EG -check-prefix=FUNC %s
 ; RUN: llc -march=r600 -mtriple=r600-- -mcpu=cayman -verify-machineinstrs < %s | FileCheck -check-prefix=CM -check-prefix=FUNC %s
@@ -273,7 +273,10 @@ entry:
 }
 
 ; FUNC-LABEL: {{^}}store_v3i32:
-; SIVI-DAG: buffer_store_dwordx3
+; SI-DAG: buffer_store_dwordx2
+; SI-DAG: buffer_store_dword v
+
+; VI-DAG: buffer_store_dwordx3
 
 ; GFX9-DAG: global_store_dwordx2
 ; GFX9-DAG: global_store_dword v
diff --git a/test/CodeGen/AMDGPU/store-v3i64.ll b/test/CodeGen/AMDGPU/store-v3i64.ll
index 534347f3fae96b0674e938d22151ce9044c5884d..7af1736b3207dc3b42b47e6d6eb091bc0927236d 100644
--- a/test/CodeGen/AMDGPU/store-v3i64.ll
+++ b/test/CodeGen/AMDGPU/store-v3i64.ll
@@ -89,7 +89,9 @@ define amdgpu_kernel void @local_store_v3i64_unaligned(<3 x i64> addrspace(3)* %
 }
 
 ; GCN-LABEL: {{^}}global_truncstore_v3i64_to_v3i32:
-; GCN-DAG: buffer_store_dwordx3
+; SI-DAG: buffer_store_dwordx2
+; SI-DAG: buffer_store_dword v
+; VI-DAG: buffer_store_dwordx3
 define amdgpu_kernel void @global_truncstore_v3i64_to_v3i32(<3 x i32> addrspace(1)* %out, <3 x i64> %x) {
   %trunc = trunc <3 x i64> %x to <3 x i32>
   store <3 x i32> %trunc, <3 x i32> addrspace(1)* %out
diff --git a/test/CodeGen/AMDGPU/trunc-combine.ll b/test/CodeGen/AMDGPU/trunc-combine.ll
index 2325a3d41484f6524f9cc7dcb9989587c587746b..8b7791905ddb0dfcfb7775f325ae121eaf491d9f 100644
--- a/test/CodeGen/AMDGPU/trunc-combine.ll
+++ b/test/CodeGen/AMDGPU/trunc-combine.ll
@@ -1,3 +1,4 @@
+; RUN: llc -march=amdgcn -mcpu=tahiti -verify-machineinstrs< %s | FileCheck -enable-var-scope -check-prefixes=GCN,SI %s
 ; RUN: llc -march=amdgcn -mcpu=fiji -verify-machineinstrs< %s | FileCheck -enable-var-scope -check-prefixes=GCN,VI %s
 
 ; Make sure high constant 0 isn't pointlessly materialized
@@ -25,7 +26,7 @@ define i32 @trunc_bitcast_i64_lshr_32_i32(i64 %bar) {
 ; GCN: _load_dword
 ; GCN-NOT: _load_dword
 ; GCN-NOT: v_mov_b32
-; GCN: v_add_u32_e32 v0, vcc, 4, v0
+; VI: v_add_u16_e32 v0, 4, v0
 define i16 @trunc_bitcast_v2i32_to_i16(<2 x i32> %bar) {
   %load0 = load i32, i32 addrspace(1)* undef
   %load1 = load i32, i32 addrspace(1)* null
@@ -42,7 +43,7 @@ define i16 @trunc_bitcast_v2i32_to_i16(<2 x i32> %bar) {
 ; GCN: _load_dword
 ; GCN-NOT: _load_dword
 ; GCN-NOT: v_mov_b32
-; GCN: v_add_u32_e32 v0, vcc, 4, v0
+; VI: v_add_u16_e32 v0, 4, v0
 define i16 @trunc_bitcast_v2f32_to_i16(<2 x float> %bar) {
   %load0 = load float, float addrspace(1)* undef
   %load1 = load float, float addrspace(1)* null
@@ -80,3 +81,18 @@ bb:
   store <2 x i16> %tmp14, <2 x i16> addrspace(1)* %tmp15, align 4
   ret void
 }
+
+; GCN-LABEL: {{^}}trunc_v2i64_arg_to_v2i16:
+; GCN: v_lshlrev_b32_e32 v1, 16, v2
+
+; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-NEXT: v_or_b32_e32 v0, v0, v1
+; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+
+; VI-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+
+; GCN-NEXT: s_setpc_b64
+define <2 x i16> @trunc_v2i64_arg_to_v2i16(<2 x i64> %arg0) #0 {
+  %trunc = trunc <2 x i64> %arg0 to <2 x i16>
+  ret <2 x i16> %trunc
+}
diff --git a/test/CodeGen/AMDGPU/uaddo.ll b/test/CodeGen/AMDGPU/uaddo.ll
index 0cb2487dd4ac65042bdb3b01044d29ff5c77ac15..cd9ea4d343786548a8dea1269669d1332c858670 100644
--- a/test/CodeGen/AMDGPU/uaddo.ll
+++ b/test/CodeGen/AMDGPU/uaddo.ll
@@ -1,7 +1,6 @@
 ; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SI,FUNC %s
 ; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,VI,FUNC %s
 ; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX9,FUNC %s
-; RUN: llc -amdgpu-scalarize-global-loads=false -march=r600 -mcpu=cypress -verify-machineinstrs < %s | FileCheck -check-prefixes=EG,FUNC %s
 
 ; FUNC-LABEL: {{^}}s_uaddo_i64_zext:
 ; GCN: s_add_u32
@@ -152,10 +151,32 @@ define amdgpu_kernel void @v_uaddo_i16(i16 addrspace(1)* %out, i1 addrspace(1)*
   ret void
 }
 
+; FUNC-LABEL: {{^}}v_uaddo_v2i32:
+; SICIVI: v_cmp_lt_i32
+; SICIVI: v_cmp_lt_i32
+; SICIVI: v_add_{{[iu]}}32
+; SICIVI: v_cmp_lt_i32
+; SICIVI: v_cmp_lt_i32
+; SICIVI: v_add_{{[iu]}}32
+define amdgpu_kernel void @v_uaddo_v2i32(<2 x i32> addrspace(1)* %out, <2 x i32> addrspace(1)* %carryout, <2 x i32> addrspace(1)* %aptr, <2 x i32> addrspace(1)* %bptr) nounwind {
+  %a = load <2 x i32>, <2 x i32> addrspace(1)* %aptr, align 4
+  %b = load <2 x i32>, <2 x i32> addrspace(1)* %bptr, align 4
+  %sadd = call { <2 x i32>, <2 x i1> } @llvm.uadd.with.overflow.v2i32(<2 x i32> %a, <2 x i32> %b) nounwind
+  %val = extractvalue { <2 x i32>, <2 x i1> } %sadd, 0
+  %carry = extractvalue { <2 x i32>, <2 x i1> } %sadd, 1
+  store <2 x i32> %val, <2 x i32> addrspace(1)* %out, align 4
+  %carry.ext = zext <2 x i1> %carry to <2 x i32>
+  store <2 x i32> %carry.ext, <2 x i32> addrspace(1)* %carryout
+  ret void
+}
+
+
 declare i32 @llvm.amdgcn.workitem.id.x() #1
 declare { i16, i1 } @llvm.uadd.with.overflow.i16(i16, i16) #1
 declare { i32, i1 } @llvm.uadd.with.overflow.i32(i32, i32) #1
 declare { i64, i1 } @llvm.uadd.with.overflow.i64(i64, i64) #1
+declare { <2 x i32>, <2 x i1> } @llvm.uadd.with.overflow.v2i32(<2 x i32>, <2 x i32>) nounwind readnone
+
 
 attributes #0 = { nounwind }
 attributes #1 = { nounwind readnone }
diff --git a/test/CodeGen/AMDGPU/usubo.ll b/test/CodeGen/AMDGPU/usubo.ll
index eeb19f86f38495dd69bdba2fabd2a14623728da9..f4e1a9afd6fe8f395d3d1bb156dc6fab15a806c9 100644
--- a/test/CodeGen/AMDGPU/usubo.ll
+++ b/test/CodeGen/AMDGPU/usubo.ll
@@ -1,7 +1,7 @@
 ; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SI,SICIVI,FUNC %s
 ; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,VI,SICIVI,FUNC %s
 ; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX9,FUNC %s
-; RUN: llc -amdgpu-scalarize-global-loads=false -march=r600 -mcpu=cypress -verify-machineinstrs < %s | FileCheck -check-prefixes=EG,FUNC %s
+
 
 ; FUNC-LABEL: {{^}}s_usubo_i64_zext:
 ; GCN: s_sub_u32
@@ -159,10 +159,28 @@ define amdgpu_kernel void @v_usubo_i16(i16 addrspace(1)* %out, i1 addrspace(1)*
   ret void
 }
 
+; FUNC-LABEL: {{^}}v_usubo_v2i32:
+; SICIVI: v_sub_{{[iu]}}32
+; SICIVI: v_cndmask_b32
+; SICIVI: v_sub_{{[iu]}}32
+; SICIVI: v_cndmask_b32
+define amdgpu_kernel void @v_usubo_v2i32(<2 x i32> addrspace(1)* %out, <2 x i32> addrspace(1)* %carryout, <2 x i32> addrspace(1)* %aptr, <2 x i32> addrspace(1)* %bptr) nounwind {
+  %a = load <2 x i32>, <2 x i32> addrspace(1)* %aptr, align 4
+  %b = load <2 x i32>, <2 x i32> addrspace(1)* %bptr, align 4
+  %sadd = call { <2 x i32>, <2 x i1> } @llvm.usub.with.overflow.v2i32(<2 x i32> %a, <2 x i32> %b) nounwind
+  %val = extractvalue { <2 x i32>, <2 x i1> } %sadd, 0
+  %carry = extractvalue { <2 x i32>, <2 x i1> } %sadd, 1
+  store <2 x i32> %val, <2 x i32> addrspace(1)* %out, align 4
+  %carry.ext = zext <2 x i1> %carry to <2 x i32>
+  store <2 x i32> %carry.ext, <2 x i32> addrspace(1)* %carryout
+  ret void
+}
+
 declare i32 @llvm.amdgcn.workitem.id.x() #1
 declare { i16, i1 } @llvm.usub.with.overflow.i16(i16, i16) #1
 declare { i32, i1 } @llvm.usub.with.overflow.i32(i32, i32) #1
 declare { i64, i1 } @llvm.usub.with.overflow.i64(i64, i64) #1
+declare { <2 x i32>, <2 x i1> } @llvm.usub.with.overflow.v2i32(<2 x i32>, <2 x i32>) nounwind readnone
 
 attributes #0 = { nounwind }
 attributes #1 = { nounwind readnone }
diff --git a/test/CodeGen/AMDGPU/vgpr-spill-emergency-stack-slot.ll b/test/CodeGen/AMDGPU/vgpr-spill-emergency-stack-slot.ll
index e803bd40684ccdb312dc4830734818a7161ff50b..6fb3ed416813cc5f2372228aadeda1283ec2e659 100644
--- a/test/CodeGen/AMDGPU/vgpr-spill-emergency-stack-slot.ll
+++ b/test/CodeGen/AMDGPU/vgpr-spill-emergency-stack-slot.ll
@@ -31,9 +31,9 @@ define amdgpu_vs void @main([9 x <4 x i32>] addrspace(4)* byval %arg, [17 x <4 x
 bb:
   %tmp = getelementptr [17 x <4 x i32>], [17 x <4 x i32>] addrspace(4)* %arg1, i64 0, i64 0
   %tmp11 = load <4 x i32>, <4 x i32> addrspace(4)* %tmp, align 16, !tbaa !0
-  %tmp12 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp11, i32 0)
-  %tmp13 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp11, i32 16)
-  %tmp14 = call float @llvm.SI.load.const.v4i32(<4 x i32> %tmp11, i32 32)
+  %tmp12 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp11, i32 0, i32 0)
+  %tmp13 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp11, i32 16, i32 0)
+  %tmp14 = call float @llvm.amdgcn.s.buffer.load.f32(<4 x i32> %tmp11, i32 32, i32 0)
   %tmp15 = getelementptr [16 x <4 x i32>], [16 x <4 x i32>] addrspace(4)* %arg4, i64 0, i64 0
   %tmp16 = load <4 x i32>, <4 x i32> addrspace(4)* %tmp15, align 16, !tbaa !0
   %tmp17 = add i32 %arg5, %arg7
@@ -488,7 +488,7 @@ bb157:                                            ; preds = %bb24
 declare i32 @llvm.amdgcn.mbcnt.lo(i32, i32) #1
 declare void @llvm.amdgcn.exp.f32(i32, i32, float, float, float, float, i1, i1) #0
 
-declare float @llvm.SI.load.const.v4i32(<4 x i32>, i32) #1
+declare float @llvm.amdgcn.s.buffer.load.f32(<4 x i32>, i32, i32) #1
 declare <4 x float> @llvm.amdgcn.buffer.load.format.v4f32(<4 x i32>, i32, i32, i1, i1) #2
 
 attributes #0 = { nounwind }
diff --git a/test/CodeGen/AMDGPU/vmem-vcc-hazard.mir b/test/CodeGen/AMDGPU/vmem-vcc-hazard.mir
new file mode 100644
index 0000000000000000000000000000000000000000..5ccdf14031c900ddd6ed14dfe7d688f2934e1ff9
--- /dev/null
+++ b/test/CodeGen/AMDGPU/vmem-vcc-hazard.mir
@@ -0,0 +1,230 @@
+# RUN: llc -march=amdgcn -mcpu=fiji -verify-machineinstrs -run-pass post-RA-hazard-rec -o - %s | FileCheck -check-prefix=GCN %s
+
+# GCN-LABEL: name: vmem_vcc_fallthrough
+# GCN:      bb.1:
+# GCN-NEXT: S_NOP
+# GCN-NEXT: S_NOP
+# GCN-NEXT: S_NOP
+# GCN-NEXT: S_NOP
+# GCN-NEXT: S_NOP
+# GCN-NEXT: BUFFER_LOAD_DWORD_OFFEN
+---
+name:            vmem_vcc_fallthrough
+body:             |
+  bb.0:
+    successors: %bb.1
+
+    $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+    $vgpr0 = IMPLICIT_DEF
+    $vgpr1 = V_ADDC_U32_e32 $vgpr0, $vgpr0, implicit-def $vcc, implicit $vcc, implicit $exec
+
+  bb.1:
+    $vgpr1 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $vcc_lo, 0, 0, 0, 0, implicit $exec
+...
+# GCN-LABEL: name: vmem_vcc_branch_to_next
+# GCN:      bb.1:
+# GCN-NEXT: S_NOP
+# GCN-NEXT: S_NOP
+# GCN-NEXT: S_NOP
+# GCN-NEXT: S_NOP
+# GCN-NEXT: BUFFER_LOAD_DWORD_OFFEN
+---
+name:            vmem_vcc_branch_to_next
+body:             |
+  bb.0:
+    successors: %bb.1
+
+    $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+    $vgpr0 = IMPLICIT_DEF
+    $vgpr1 = V_ADDC_U32_e32 $vgpr0, $vgpr0, implicit-def $vcc, implicit $vcc, implicit $exec
+    S_BRANCH %bb.1
+
+  bb.1:
+    $vgpr1 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $vcc_lo, 0, 0, 0, 0, implicit $exec
+...
+# GCN-LABEL: name: vmem_vcc_fallthrough_no_hazard_too_far
+# GCN:      bb.1:
+# GCN-NEXT: BUFFER_LOAD_DWORD_OFFEN
+---
+name:            vmem_vcc_fallthrough_no_hazard_too_far
+body:             |
+  bb.0:
+    successors: %bb.1
+
+    $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+    $vgpr0 = IMPLICIT_DEF
+    $vgpr1 = V_ADDC_U32_e32 $vgpr0, $vgpr0, implicit-def $vcc, implicit $vcc, implicit $exec
+    $sgpr0 = S_MOV_B32 0
+    $sgpr0 = S_MOV_B32 0
+    $sgpr0 = S_MOV_B32 0
+    $sgpr0 = S_MOV_B32 0
+    $sgpr0 = S_MOV_B32 0
+
+  bb.1:
+    $vgpr1 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $vcc_lo, 0, 0, 0, 0, implicit $exec
+...
+# GCN-LABEL: name: vmem_vcc_fallthrough_no_hazard_nops
+# GCN:      bb.1:
+# GCN-NEXT: BUFFER_LOAD_DWORD_OFFEN
+---
+name:            vmem_vcc_fallthrough_no_hazard_nops
+body:             |
+  bb.0:
+    successors: %bb.1
+
+    $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+    $vgpr0 = IMPLICIT_DEF
+    $vgpr1 = V_ADDC_U32_e32 $vgpr0, $vgpr0, implicit-def $vcc, implicit $vcc, implicit $exec
+    S_NOP 4
+
+  bb.1:
+    $vgpr1 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $vcc_lo, 0, 0, 0, 0, implicit $exec
+...
+# GCN-LABEL: name: vmem_vcc_branch_around
+# GCN:      bb.2:
+# GCN-NEXT: S_NOP
+# GCN-NEXT: S_NOP
+# GCN-NEXT: S_NOP
+# GCN-NEXT: S_NOP
+# GCN-NEXT: BUFFER_LOAD_DWORD_OFFEN
+---
+name:            vmem_vcc_branch_around
+body:             |
+  bb.0:
+    successors: %bb.2
+
+    $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+    $vgpr0 = IMPLICIT_DEF
+    $vgpr1 = V_ADDC_U32_e32 $vgpr0, $vgpr0, implicit-def $vcc, implicit $vcc, implicit $exec
+    S_BRANCH %bb.2
+
+  bb.1:
+    successors: %bb.2
+
+    S_NOP 0
+    S_NOP 0
+    S_NOP 0
+    S_NOP 0
+
+  bb.2:
+    $vgpr1 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $vcc_lo, 0, 0, 0, 0, implicit $exec
+...
+# GCN-LABEL: name: vmem_vcc_branch_backedge
+# GCN:      S_NOP
+# GCN-NEXT: S_NOP
+# GCN-NEXT: S_NOP
+# GCN-NEXT: S_NOP
+# GCN-NEXT: BUFFER_LOAD_DWORD_OFFEN
+---
+name:            vmem_vcc_branch_backedge
+body:             |
+  bb.0:
+    successors: %bb.1
+
+    $vgpr0 = IMPLICIT_DEF
+    $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+    $vgpr1 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $vcc_lo, 0, 0, 0, 0, implicit $exec
+
+  bb.1:
+    $vgpr0 = IMPLICIT_DEF
+    $vgpr1 = V_ADDC_U32_e32 $vgpr0, $vgpr0, implicit-def $vcc, implicit $vcc, implicit $exec
+    S_BRANCH %bb.0
+...
+# GCN-LABEL: name: vmem_vcc_min_of_two
+# GCN:      bb.2:
+# GCN-NEXT: S_NOP
+# GCN-NEXT: S_NOP
+# GCN-NEXT: S_NOP
+# GCN-NEXT: S_NOP
+# GCN-NEXT: S_NOP
+# GCN-NEXT: BUFFER_LOAD_DWORD_OFFEN
+---
+name:            vmem_vcc_min_of_two
+body:             |
+  bb.0:
+    successors: %bb.2
+
+    $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+    $vgpr0 = IMPLICIT_DEF
+    $vgpr1 = V_ADDC_U32_e32 $vgpr0, $vgpr0, implicit-def $vcc, implicit $vcc, implicit $exec
+    S_NOP 0
+    S_BRANCH %bb.2
+
+  bb.1:
+    successors: %bb.2
+
+    $vgpr1 = V_ADDC_U32_e32 $vgpr0, $vgpr0, implicit-def $vcc, implicit $vcc, implicit $exec
+
+  bb.2:
+    $vgpr1 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $vcc_lo, 0, 0, 0, 0, implicit $exec
+...
+# GCN-LABEL: name: vmem_vcc_self_loop
+# GCN:      S_NOP
+# GCN-NEXT: S_NOP
+# GCN-NEXT: S_NOP
+# GCN-NEXT: S_NOP
+# GCN-NEXT: BUFFER_LOAD_DWORD_OFFEN
+---
+name:            vmem_vcc_self_loop
+body:             |
+  bb.0:
+    successors: %bb.0
+
+    $vgpr0 = IMPLICIT_DEF
+    $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+    $vgpr1 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $vcc_lo, 0, 0, 0, 0, implicit $exec
+    $vgpr1 = V_ADDC_U32_e32 $vgpr0, $vgpr0, implicit-def $vcc, implicit $vcc, implicit $exec
+    S_BRANCH %bb.0
+...
+# GCN-LABEL: name: vmem_vcc_min_of_two_self_loop1
+# GCN:      bb.1:
+# GCN:      $sgpr0 = S_MOV_B32 0
+# GCN-NEXT: S_NOP
+# GCN-NEXT: S_NOP
+# GCN-NEXT: S_NOP
+# GCN-NEXT: S_NOP
+# GCN-NEXT: BUFFER_LOAD_DWORD_OFFEN
+---
+name:            vmem_vcc_min_of_two_self_loop1
+body:             |
+  bb.0:
+    successors: %bb.1
+
+    $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+    $vgpr0 = IMPLICIT_DEF
+    $vgpr1 = V_ADDC_U32_e32 $vgpr0, $vgpr0, implicit-def $vcc, implicit $vcc, implicit $exec
+
+  bb.1:
+    successors: %bb.1
+
+    $sgpr0 = S_MOV_B32 0
+    $vgpr1 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $vcc_lo, 0, 0, 0, 0, implicit $exec
+    $vgpr1 = V_ADDC_U32_e32 $vgpr1, $vgpr1, implicit-def $vcc, implicit $vcc, implicit $exec
+    S_BRANCH %bb.1
+...
+# GCN-LABEL: name: vmem_vcc_min_of_two_self_loop2
+# GCN:      bb.1:
+# GCN:      $sgpr0 = S_MOV_B32 0
+# GCN-NEXT: S_NOP
+# GCN-NEXT: S_NOP
+# GCN-NEXT: S_NOP
+# GCN-NEXT: BUFFER_LOAD_DWORD_OFFEN
+---
+name:            vmem_vcc_min_of_two_self_loop2
+body:             |
+  bb.0:
+    successors: %bb.1
+
+    $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
+    $vgpr0 = IMPLICIT_DEF
+    $vgpr1 = V_ADDC_U32_e32 $vgpr0, $vgpr0, implicit-def $vcc, implicit $vcc, implicit $exec
+    S_NOP 0
+
+  bb.1:
+    successors: %bb.1
+
+    $sgpr0 = S_MOV_B32 0
+    $vgpr1 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $vcc_lo, 0, 0, 0, 0, implicit $exec
+    $vgpr1 = V_ADDC_U32_e32 $vgpr1, $vgpr1, implicit-def $vcc, implicit $vcc, implicit $exec
+    S_BRANCH %bb.1
+...
diff --git a/test/CodeGen/AMDGPU/waitcnt-loop-irreducible.mir b/test/CodeGen/AMDGPU/waitcnt-loop-irreducible.mir
index b0ca67febbf8ac384d012b1251cd14f098bb9e87..7afb2ba89c191a315f563f5953498f8d26ac8e0e 100644
--- a/test/CodeGen/AMDGPU/waitcnt-loop-irreducible.mir
+++ b/test/CodeGen/AMDGPU/waitcnt-loop-irreducible.mir
@@ -1,20 +1,26 @@
-# RUN: llc -march=amdgcn -verify-machineinstrs -run-pass si-insert-waitcnts -o - %s | FileCheck -check-prefixes=GCN %s
+# RUN: llc -march=amdgcn -mcpu=gfx803 -verify-machineinstrs -run-pass si-insert-waitcnts -o - %s | FileCheck -check-prefixes=GCN,GFX8 %s
+# RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs -run-pass si-insert-waitcnts -o - %s | FileCheck -check-prefixes=GCN,GFX9 %s
 
-# GCN-LABEL: name: irreducible_loop{{$}}
-# GCN: S_LOAD_DWORDX4_IMM
-# GCN: S_WAITCNT 127{{$}}
-# GCN: S_BUFFER_LOAD_DWORD_IMM
-# GCN: S_WAITCNT 127{{$}}
-# GCN: S_CMP_GE_I32
 --- |
 
   define amdgpu_ps void @irreducible_loop() {
-  main:
+    ret void
+  }
+  define amdgpu_ps void @irreducible_loop_extended() {
     ret void
   }
 
 ...
 ---
+
+# GCN-LABEL: name: irreducible_loop{{$}}
+# GCN: S_LOAD_DWORDX4_IMM
+# GFX8: S_WAITCNT 127{{$}}
+# GFX9: S_WAITCNT 49279{{$}}
+# GCN: S_BUFFER_LOAD_DWORD_IMM
+# GFX8: S_WAITCNT 127{{$}}
+# GFX9: S_WAITCNT 49279{{$}}
+# GCN: S_CMP_GE_I32
 name:            irreducible_loop
 body:             |
   bb.0:
@@ -45,3 +51,53 @@ body:             |
     S_ENDPGM
 
 ...
+
+# GCN-LABEL: name: irreducible_loop_extended
+
+# GCN: S_LOAD_DWORDX4_IMM
+# GFX8: S_WAITCNT 127{{$}}
+# GFX9: S_WAITCNT 49279{{$}}
+# GCN: BUFFER_STORE_DWORD_OFFEN_exact
+# GFX8: S_WAITCNT 127{{$}}
+# GFX9: S_WAITCNT 49279{{$}}
+# GCN: BUFFER_STORE_DWORD_OFFEN_exact
+# GCN: S_LOAD_DWORDX4_IMM
+# GFX8: S_WAITCNT 127{{$}}
+# GFX9: S_WAITCNT 49279{{$}}
+# GCN: BUFFER_ATOMIC_ADD_OFFSET_RTN
+# GCN: S_WAITCNT 3952
+# GCN: FLAT_STORE_DWORD
+# GCN: S_ENDPGM
+name: irreducible_loop_extended
+
+body: |
+  bb.0:
+    successors: %bb.1, %bb.2
+    $sgpr4_sgpr5_sgpr6_sgpr7 = S_LOAD_DWORDX4_IMM renamable $sgpr2_sgpr3, 0, 0
+    S_CBRANCH_VCCZ %bb.2, implicit $vcc
+
+  bb.1:
+    successors: %bb.2
+    BUFFER_STORE_DWORD_OFFEN_exact killed renamable $vgpr3, renamable $vgpr2, renamable $sgpr4_sgpr5_sgpr6_sgpr7, 0, 0, 0, 0, 0, implicit $exec
+
+  bb.2:
+    successors: %bb.3, %bb.6
+    S_CBRANCH_VCCNZ %bb.6, implicit $vcc
+
+  bb.3:
+    successors: %bb.4, %bb.5
+    BUFFER_STORE_DWORD_OFFEN_exact killed renamable $vgpr3, killed renamable $vgpr2, killed renamable $sgpr4_sgpr5_sgpr6_sgpr7, 0, 0, 0, 0, 0, implicit $exec
+    S_CBRANCH_VCCNZ %bb.5, implicit $vcc
+
+  bb.4:
+    successors: %bb.5
+    renamable $sgpr12_sgpr13_sgpr14_sgpr15 = S_LOAD_DWORDX4_IMM killed renamable $sgpr2_sgpr3, 64, 0
+    renamable $vgpr2 = BUFFER_ATOMIC_ADD_OFFSET_RTN killed renamable $vgpr2, killed renamable $sgpr12_sgpr13_sgpr14_sgpr15, 0, 0, 0, implicit $exec
+
+  bb.5:
+    successors: %bb.6
+
+  bb.6:
+    FLAT_STORE_DWORD $vgpr3_vgpr4, $vgpr2, 0, 0, 0, implicit $exec, implicit $flat_scr
+    S_ENDPGM
+...
diff --git a/test/CodeGen/AMDGPU/widen-smrd-loads.ll b/test/CodeGen/AMDGPU/widen-smrd-loads.ll
index 9a2e4280ffb302ac5a0a3a9203b21ec4f54f03cc..c950e2d7cd37870ba4844e7444354adfc3c73e16 100644
--- a/test/CodeGen/AMDGPU/widen-smrd-loads.ll
+++ b/test/CodeGen/AMDGPU/widen-smrd-loads.ll
@@ -1,11 +1,38 @@
-; RUN: llc -amdgpu-codegenprepare-widen-constant-loads=0 -march=amdgcn -mcpu=tahiti -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,SI %s
-; RUN: llc -amdgpu-codegenprepare-widen-constant-loads=0 -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,VI %s
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -amdgpu-codegenprepare-widen-constant-loads=0 -mtriple=amdgcn -mcpu=tahiti -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,SI %s
+; RUN: llc -amdgpu-codegenprepare-widen-constant-loads=0 -mtriple=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,VI %s
 
-; GCN-LABEL: {{^}}widen_i16_constant_load:
-; GCN: s_load_dword [[VAL:s[0-9]+]]
-; GCN: s_addk_i32 [[VAL]], 0x3e7
-; GCN: s_or_b32 [[OR:s[0-9]+]], [[VAL]], 4
 define amdgpu_kernel void @widen_i16_constant_load(i16 addrspace(4)* %arg) {
+; SI-LABEL: widen_i16_constant_load:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SI-NEXT:    s_mov_b32 s5, 0
+; SI-NEXT:    s_mov_b32 s4, 0
+; SI-NEXT:    s_mov_b32 s7, 0xf000
+; SI-NEXT:    s_mov_b32 s6, -1
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_load_dword s0, s[0:1], 0x0
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_addk_i32 s0, 0x3e7
+; SI-NEXT:    s_or_b32 s0, s0, 4
+; SI-NEXT:    v_mov_b32_e32 v0, s0
+; SI-NEXT:    buffer_store_short v0, off, s[4:7], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: widen_i16_constant_load:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; VI-NEXT:    v_mov_b32_e32 v0, 0
+; VI-NEXT:    v_mov_b32_e32 v1, 0
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_load_dword s0, s[0:1], 0x0
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_and_b32 s0, s0, 0xffff
+; VI-NEXT:    s_addk_i32 s0, 0x3e7
+; VI-NEXT:    s_or_b32 s0, s0, 4
+; VI-NEXT:    v_mov_b32_e32 v2, s0
+; VI-NEXT:    flat_store_short v[0:1], v2
+; VI-NEXT:    s_endpgm
   %load = load i16, i16 addrspace(4)* %arg, align 4
   %add = add i16 %load, 999
   %or = or i16 %add, 4
@@ -13,12 +40,38 @@ define amdgpu_kernel void @widen_i16_constant_load(i16 addrspace(4)* %arg) {
   ret void
 }
 
-; GCN-LABEL: {{^}}widen_i16_constant_load_zext_i32:
-; GCN: s_load_dword [[VAL:s[0-9]+]]
-; GCN: s_and_b32 [[TRUNC:s[0-9]+]], [[VAL]], 0xffff{{$}}
-; GCN: s_addk_i32 [[TRUNC]], 0x3e7
-; GCN: s_or_b32 [[OR:s[0-9]+]], [[TRUNC]], 4
 define amdgpu_kernel void @widen_i16_constant_load_zext_i32(i16 addrspace(4)* %arg) {
+; SI-LABEL: widen_i16_constant_load_zext_i32:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SI-NEXT:    s_mov_b32 s5, 0
+; SI-NEXT:    s_mov_b32 s4, 0
+; SI-NEXT:    s_mov_b32 s7, 0xf000
+; SI-NEXT:    s_mov_b32 s6, -1
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_load_dword s0, s[0:1], 0x0
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_and_b32 s0, s0, 0xffff
+; SI-NEXT:    s_addk_i32 s0, 0x3e7
+; SI-NEXT:    s_or_b32 s0, s0, 4
+; SI-NEXT:    v_mov_b32_e32 v0, s0
+; SI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: widen_i16_constant_load_zext_i32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; VI-NEXT:    v_mov_b32_e32 v0, 0
+; VI-NEXT:    v_mov_b32_e32 v1, 0
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_load_dword s0, s[0:1], 0x0
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_and_b32 s0, s0, 0xffff
+; VI-NEXT:    s_addk_i32 s0, 0x3e7
+; VI-NEXT:    s_or_b32 s0, s0, 4
+; VI-NEXT:    v_mov_b32_e32 v2, s0
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
   %load = load i16, i16 addrspace(4)* %arg, align 4
   %ext = zext i16 %load to i32
   %add = add i32 %ext, 999
@@ -27,12 +80,38 @@ define amdgpu_kernel void @widen_i16_constant_load_zext_i32(i16 addrspace(4)* %a
   ret void
 }
 
-; GCN-LABEL: {{^}}widen_i16_constant_load_sext_i32:
-; GCN: s_load_dword [[VAL:s[0-9]+]]
-; GCN: s_sext_i32_i16 [[EXT:s[0-9]+]], [[VAL]]
-; GCN: s_addk_i32 [[EXT]], 0x3e7
-; GCN: s_or_b32 [[OR:s[0-9]+]], [[EXT]], 4
 define amdgpu_kernel void @widen_i16_constant_load_sext_i32(i16 addrspace(4)* %arg) {
+; SI-LABEL: widen_i16_constant_load_sext_i32:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SI-NEXT:    s_mov_b32 s5, 0
+; SI-NEXT:    s_mov_b32 s4, 0
+; SI-NEXT:    s_mov_b32 s7, 0xf000
+; SI-NEXT:    s_mov_b32 s6, -1
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_load_dword s0, s[0:1], 0x0
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_sext_i32_i16 s0, s0
+; SI-NEXT:    s_addk_i32 s0, 0x3e7
+; SI-NEXT:    s_or_b32 s0, s0, 4
+; SI-NEXT:    v_mov_b32_e32 v0, s0
+; SI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: widen_i16_constant_load_sext_i32:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; VI-NEXT:    v_mov_b32_e32 v0, 0
+; VI-NEXT:    v_mov_b32_e32 v1, 0
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_load_dword s0, s[0:1], 0x0
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_sext_i32_i16 s0, s0
+; VI-NEXT:    s_addk_i32 s0, 0x3e7
+; VI-NEXT:    s_or_b32 s0, s0, 4
+; VI-NEXT:    v_mov_b32_e32 v2, s0
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
   %load = load i16, i16 addrspace(4)* %arg, align 4
   %ext = sext i16 %load to i32
   %add = add i32 %ext, 999
@@ -41,12 +120,46 @@ define amdgpu_kernel void @widen_i16_constant_load_sext_i32(i16 addrspace(4)* %a
   ret void
 }
 
-; GCN-LABEL: {{^}}widen_i17_constant_load:
-; GCN: s_load_dword [[VAL:s[0-9]+]]
-; GCN: s_add_i32 [[ADD:s[0-9]+]], [[VAL]], 34
-; GCN: s_or_b32 [[OR:s[0-9]+]], [[ADD]], 4
-; GCN: s_bfe_u32 s{{[0-9]+}}, [[OR]], 0x10010
 define amdgpu_kernel void @widen_i17_constant_load(i17 addrspace(4)* %arg) {
+; SI-LABEL: widen_i17_constant_load:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SI-NEXT:    s_mov_b32 s5, 0
+; SI-NEXT:    s_mov_b32 s4, 0
+; SI-NEXT:    s_mov_b32 s7, 0xf000
+; SI-NEXT:    s_mov_b32 s6, -1
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_load_dword s0, s[0:1], 0x0
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_add_i32 s0, s0, 34
+; SI-NEXT:    s_or_b32 s0, s0, 4
+; SI-NEXT:    v_mov_b32_e32 v0, s0
+; SI-NEXT:    s_bfe_u32 s0, s0, 0x10010
+; SI-NEXT:    buffer_store_short v0, off, s[4:7], 0
+; SI-NEXT:    s_mov_b32 s4, 2
+; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_mov_b32_e32 v0, s0
+; SI-NEXT:    buffer_store_byte v0, off, s[4:7], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: widen_i17_constant_load:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; VI-NEXT:    v_mov_b32_e32 v0, 0
+; VI-NEXT:    v_mov_b32_e32 v2, 2
+; VI-NEXT:    v_mov_b32_e32 v1, 0
+; VI-NEXT:    v_mov_b32_e32 v3, 0
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_load_dword s0, s[0:1], 0x0
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_add_i32 s0, s0, 34
+; VI-NEXT:    s_or_b32 s0, s0, 4
+; VI-NEXT:    v_mov_b32_e32 v4, s0
+; VI-NEXT:    s_bfe_u32 s0, s0, 0x10010
+; VI-NEXT:    v_mov_b32_e32 v5, s0
+; VI-NEXT:    flat_store_short v[0:1], v4
+; VI-NEXT:    flat_store_byte v[2:3], v5
+; VI-NEXT:    s_endpgm
   %load = load i17, i17 addrspace(4)* %arg, align 4
   %add = add i17 %load, 34
   %or = or i17 %add, 4
@@ -54,13 +167,34 @@ define amdgpu_kernel void @widen_i17_constant_load(i17 addrspace(4)* %arg) {
   ret void
 }
 
-; GCN-LABEL: {{^}}widen_f16_constant_load:
-; GCN: s_load_dword [[VAL:s[0-9]+]]
-; SI: v_cvt_f32_f16_e32 [[CVT:v[0-9]+]], [[VAL]]
-; SI: v_add_f32_e32 [[ADD:v[0-9]+]], 4.0, [[CVT]]
-
-; VI: v_add_f16_e64 [[ADD:v[0-9]+]], [[VAL]], 4.0
 define amdgpu_kernel void @widen_f16_constant_load(half addrspace(4)* %arg) {
+; SI-LABEL: widen_f16_constant_load:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SI-NEXT:    s_mov_b32 s3, 0xf000
+; SI-NEXT:    s_mov_b32 s2, -1
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_load_dword s0, s[0:1], 0x0
+; SI-NEXT:    s_mov_b32 s1, 0
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    v_cvt_f32_f16_e32 v0, s0
+; SI-NEXT:    s_mov_b32 s0, 0
+; SI-NEXT:    v_add_f32_e32 v0, 4.0, v0
+; SI-NEXT:    v_cvt_f16_f32_e32 v0, v0
+; SI-NEXT:    buffer_store_short v0, off, s[0:3], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: widen_f16_constant_load:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; VI-NEXT:    v_mov_b32_e32 v0, 0
+; VI-NEXT:    v_mov_b32_e32 v1, 0
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_load_dword s0, s[0:1], 0x0
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_add_f16_e64 v2, s0, 4.0
+; VI-NEXT:    flat_store_short v[0:1], v2
+; VI-NEXT:    s_endpgm
   %load = load half, half addrspace(4)* %arg, align 4
   %add = fadd half %load, 4.0
   store half %add, half addrspace(1)* null
@@ -68,21 +202,49 @@ define amdgpu_kernel void @widen_f16_constant_load(half addrspace(4)* %arg) {
 }
 
 ; FIXME: valu usage on VI
-; GCN-LABEL: {{^}}widen_v2i8_constant_load:
-; GCN: s_load_dword [[VAL:s[0-9]+]]
-
-; SI: s_add_i32
-; SI: s_or_b32
-; SI: s_addk_i32
-; SI: s_and_b32
-; SI: s_or_b32
-; SI: s_or_b32
-
-; VI: s_add_i32
-; VI: v_add_u32_sdwa
-; VI: v_or_b32_sdwa
-; VI: v_or_b32_e32
 define amdgpu_kernel void @widen_v2i8_constant_load(<2 x i8> addrspace(4)* %arg) {
+; SI-LABEL: widen_v2i8_constant_load:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SI-NEXT:    s_mov_b32 s5, 0
+; SI-NEXT:    s_mov_b32 s4, 0
+; SI-NEXT:    s_mov_b32 s7, 0xf000
+; SI-NEXT:    s_mov_b32 s6, -1
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_load_dword s0, s[0:1], 0x0
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_and_b32 s1, s0, 0xff00
+; SI-NEXT:    s_and_b32 s0, s0, 0xffff
+; SI-NEXT:    s_add_i32 s0, s0, 12
+; SI-NEXT:    s_or_b32 s0, s0, 4
+; SI-NEXT:    s_addk_i32 s1, 0x2c00
+; SI-NEXT:    s_and_b32 s0, s0, 0xff
+; SI-NEXT:    s_or_b32 s0, s0, s1
+; SI-NEXT:    s_or_b32 s0, s0, 0x300
+; SI-NEXT:    v_mov_b32_e32 v0, s0
+; SI-NEXT:    buffer_store_short v0, off, s[4:7], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: widen_v2i8_constant_load:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; VI-NEXT:    v_mov_b32_e32 v0, 44
+; VI-NEXT:    v_mov_b32_e32 v1, 3
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_load_dword s0, s[0:1], 0x0
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_and_b32 s1, s0, 0xffff
+; VI-NEXT:    v_mov_b32_e32 v2, s0
+; VI-NEXT:    s_add_i32 s1, s1, 12
+; VI-NEXT:    v_add_u32_sdwa v0, vcc, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_1
+; VI-NEXT:    s_or_b32 s0, s1, 4
+; VI-NEXT:    v_or_b32_sdwa v0, v0, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; VI-NEXT:    s_and_b32 s0, s0, 0xff
+; VI-NEXT:    v_or_b32_e32 v2, s0, v0
+; VI-NEXT:    v_mov_b32_e32 v0, 0
+; VI-NEXT:    v_mov_b32_e32 v1, 0
+; VI-NEXT:    flat_store_short v[0:1], v2
+; VI-NEXT:    s_endpgm
   %load = load <2 x i8>, <2 x i8> addrspace(4)* %arg, align 4
   %add = add <2 x i8> %load, <i8 12, i8 44>
   %or = or <2 x i8> %add, <i8 4, i8 3>
@@ -90,9 +252,41 @@ define amdgpu_kernel void @widen_v2i8_constant_load(<2 x i8> addrspace(4)* %arg)
   ret void
 }
 
-; GCN-LABEL: {{^}}no_widen_i16_constant_divergent_load:
-; GCN: {{buffer|flat}}_load_ushort
 define amdgpu_kernel void @no_widen_i16_constant_divergent_load(i16 addrspace(4)* %arg) {
+; SI-LABEL: no_widen_i16_constant_divergent_load:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SI-NEXT:    s_mov_b32 s2, 0
+; SI-NEXT:    s_mov_b32 s3, 0xf000
+; SI-NEXT:    v_lshlrev_b32_e32 v0, 1, v0
+; SI-NEXT:    v_mov_b32_e32 v1, 0
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    buffer_load_ushort v0, v[0:1], s[0:3], 0 addr64
+; SI-NEXT:    s_mov_b32 s1, 0
+; SI-NEXT:    s_mov_b32 s0, 0
+; SI-NEXT:    s_mov_b32 s2, -1
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_add_i32_e32 v0, vcc, 0x3e7, v0
+; SI-NEXT:    v_or_b32_e32 v0, 4, v0
+; SI-NEXT:    buffer_store_short v0, off, s[0:3], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: no_widen_i16_constant_divergent_load:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; VI-NEXT:    v_lshlrev_b32_e32 v2, 1, v0
+; VI-NEXT:    v_mov_b32_e32 v0, 0
+; VI-NEXT:    v_mov_b32_e32 v1, 0
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v3, s1
+; VI-NEXT:    v_add_u32_e32 v2, vcc, s0, v2
+; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; VI-NEXT:    flat_load_ushort v2, v[2:3]
+; VI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; VI-NEXT:    v_add_u16_e32 v2, 0x3e7, v2
+; VI-NEXT:    v_or_b32_e32 v2, 4, v2
+; VI-NEXT:    flat_store_short v[0:1], v2
+; VI-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %tid.ext = zext i32 %tid to i64
   %gep.arg = getelementptr inbounds i16, i16 addrspace(4)* %arg, i64 %tid.ext
@@ -103,22 +297,72 @@ define amdgpu_kernel void @no_widen_i16_constant_divergent_load(i16 addrspace(4)
   ret void
 }
 
-; GCN-LABEL: {{^}}widen_i1_constant_load:
-; GCN: s_load_dword [[VAL:s[0-9]+]]
-; GCN: s_and_b32 {{s[0-9]+}}, [[VAL]], 1{{$}}
 define amdgpu_kernel void @widen_i1_constant_load(i1 addrspace(4)* %arg) {
+; SI-LABEL: widen_i1_constant_load:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SI-NEXT:    s_mov_b32 s5, 0
+; SI-NEXT:    s_mov_b32 s4, 0
+; SI-NEXT:    s_mov_b32 s7, 0xf000
+; SI-NEXT:    s_mov_b32 s6, -1
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_load_dword s0, s[0:1], 0x0
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_and_b32 s0, s0, 1
+; SI-NEXT:    v_mov_b32_e32 v0, s0
+; SI-NEXT:    buffer_store_byte v0, off, s[4:7], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: widen_i1_constant_load:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; VI-NEXT:    v_mov_b32_e32 v0, 0
+; VI-NEXT:    v_mov_b32_e32 v1, 0
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_load_dword s0, s[0:1], 0x0
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_and_b32 s0, s0, 1
+; VI-NEXT:    v_mov_b32_e32 v2, s0
+; VI-NEXT:    flat_store_byte v[0:1], v2
+; VI-NEXT:    s_endpgm
   %load = load i1, i1 addrspace(4)* %arg, align 4
   %and = and i1 %load, true
   store i1 %and, i1 addrspace(1)* null
   ret void
 }
 
-; GCN-LABEL: {{^}}widen_i16_zextload_i64_constant_load:
-; GCN: s_load_dword [[VAL:s[0-9]+]]
-; GCN: s_and_b32 [[TRUNC:s[0-9]+]], [[VAL]], 0xffff{{$}}
-; GCN: s_addk_i32 [[TRUNC]], 0x3e7
-; GCN: s_or_b32 [[OR:s[0-9]+]], [[TRUNC]], 4
 define amdgpu_kernel void @widen_i16_zextload_i64_constant_load(i16 addrspace(4)* %arg) {
+; SI-LABEL: widen_i16_zextload_i64_constant_load:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SI-NEXT:    s_mov_b32 s5, 0
+; SI-NEXT:    s_mov_b32 s4, 0
+; SI-NEXT:    s_mov_b32 s7, 0xf000
+; SI-NEXT:    s_mov_b32 s6, -1
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_load_dword s0, s[0:1], 0x0
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_and_b32 s0, s0, 0xffff
+; SI-NEXT:    s_addk_i32 s0, 0x3e7
+; SI-NEXT:    s_or_b32 s0, s0, 4
+; SI-NEXT:    v_mov_b32_e32 v0, s0
+; SI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: widen_i16_zextload_i64_constant_load:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; VI-NEXT:    v_mov_b32_e32 v0, 0
+; VI-NEXT:    v_mov_b32_e32 v1, 0
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_load_dword s0, s[0:1], 0x0
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_and_b32 s0, s0, 0xffff
+; VI-NEXT:    s_addk_i32 s0, 0x3e7
+; VI-NEXT:    s_or_b32 s0, s0, 4
+; VI-NEXT:    v_mov_b32_e32 v2, s0
+; VI-NEXT:    flat_store_dword v[0:1], v2
+; VI-NEXT:    s_endpgm
   %load = load i16, i16 addrspace(4)* %arg, align 4
   %zext = zext i16 %load to i32
   %add = add i32 %zext, 999
@@ -127,12 +371,40 @@ define amdgpu_kernel void @widen_i16_zextload_i64_constant_load(i16 addrspace(4)
   ret void
 }
 
-; GCN-LABEL: {{^}}widen_i1_zext_to_i64_constant_load:
-; GCN: s_load_dword [[VAL:s[0-9]+]]
-; GCN: s_and_b32 [[AND:s[0-9]+]], [[VAL]], 1
-; GCN: s_add_u32 [[ADD:s[0-9]+]], [[AND]], 0x3e7
-; GCN: s_addc_u32 s{{[0-9]+}}, 0, 0
 define amdgpu_kernel void @widen_i1_zext_to_i64_constant_load(i1 addrspace(4)* %arg) {
+; SI-LABEL: widen_i1_zext_to_i64_constant_load:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SI-NEXT:    s_mov_b32 s5, 0
+; SI-NEXT:    s_mov_b32 s4, 0
+; SI-NEXT:    s_mov_b32 s7, 0xf000
+; SI-NEXT:    s_mov_b32 s6, -1
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_load_dword s0, s[0:1], 0x0
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_and_b32 s0, s0, 1
+; SI-NEXT:    s_add_u32 s0, s0, 0x3e7
+; SI-NEXT:    s_addc_u32 s1, 0, 0
+; SI-NEXT:    v_mov_b32_e32 v0, s0
+; SI-NEXT:    v_mov_b32_e32 v1, s1
+; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: widen_i1_zext_to_i64_constant_load:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; VI-NEXT:    v_mov_b32_e32 v0, 0
+; VI-NEXT:    v_mov_b32_e32 v1, 0
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_load_dword s0, s[0:1], 0x0
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_and_b32 s0, s0, 1
+; VI-NEXT:    s_add_u32 s0, s0, 0x3e7
+; VI-NEXT:    s_addc_u32 s1, 0, 0
+; VI-NEXT:    v_mov_b32_e32 v3, s1
+; VI-NEXT:    v_mov_b32_e32 v2, s0
+; VI-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
+; VI-NEXT:    s_endpgm
   %load = load i1, i1 addrspace(4)* %arg, align 4
   %zext = zext i1 %load to i64
   %add = add i64 %zext, 999
@@ -140,11 +412,39 @@ define amdgpu_kernel void @widen_i1_zext_to_i64_constant_load(i1 addrspace(4)* %
   ret void
 }
 
-; GCN-LABEL: {{^}}widen_i16_constant32_load:
-; GCN: s_load_dword [[VAL:s[0-9]+]]
-; GCN: s_addk_i32 [[VAL]], 0x3e7
-; GCN: s_or_b32 [[OR:s[0-9]+]], [[VAL]], 4
 define amdgpu_kernel void @widen_i16_constant32_load(i16 addrspace(6)* %arg) {
+; SI-LABEL: widen_i16_constant32_load:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dword s0, s[0:1], 0x9
+; SI-NEXT:    s_mov_b32 s1, 0
+; SI-NEXT:    s_mov_b32 s5, 0
+; SI-NEXT:    s_mov_b32 s4, 0
+; SI-NEXT:    s_mov_b32 s7, 0xf000
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_load_dword s0, s[0:1], 0x0
+; SI-NEXT:    s_mov_b32 s6, -1
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_addk_i32 s0, 0x3e7
+; SI-NEXT:    s_or_b32 s0, s0, 4
+; SI-NEXT:    v_mov_b32_e32 v0, s0
+; SI-NEXT:    buffer_store_short v0, off, s[4:7], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: widen_i16_constant32_load:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dword s0, s[0:1], 0x24
+; VI-NEXT:    s_mov_b32 s1, 0
+; VI-NEXT:    v_mov_b32_e32 v0, 0
+; VI-NEXT:    v_mov_b32_e32 v1, 0
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_load_dword s0, s[0:1], 0x0
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_and_b32 s0, s0, 0xffff
+; VI-NEXT:    s_addk_i32 s0, 0x3e7
+; VI-NEXT:    s_or_b32 s0, s0, 4
+; VI-NEXT:    v_mov_b32_e32 v2, s0
+; VI-NEXT:    flat_store_short v[0:1], v2
+; VI-NEXT:    s_endpgm
   %load = load i16, i16 addrspace(6)* %arg, align 4
   %add = add i16 %load, 999
   %or = or i16 %add, 4
@@ -152,11 +452,37 @@ define amdgpu_kernel void @widen_i16_constant32_load(i16 addrspace(6)* %arg) {
   ret void
 }
 
-; GCN-LABEL: {{^}}widen_i16_global_invariant_load:
-; GCN: s_load_dword [[VAL:s[0-9]+]]
-; GCN: s_addk_i32 [[VAL]], 0x3e7
-; GCN: s_or_b32 [[OR:s[0-9]+]], [[VAL]], 1
 define amdgpu_kernel void @widen_i16_global_invariant_load(i16 addrspace(1)* %arg) {
+; SI-LABEL: widen_i16_global_invariant_load:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SI-NEXT:    s_mov_b32 s5, 0
+; SI-NEXT:    s_mov_b32 s4, 0
+; SI-NEXT:    s_mov_b32 s7, 0xf000
+; SI-NEXT:    s_mov_b32 s6, -1
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_load_dword s0, s[0:1], 0x0
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_addk_i32 s0, 0x3e7
+; SI-NEXT:    s_or_b32 s0, s0, 1
+; SI-NEXT:    v_mov_b32_e32 v0, s0
+; SI-NEXT:    buffer_store_short v0, off, s[4:7], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: widen_i16_global_invariant_load:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; VI-NEXT:    v_mov_b32_e32 v0, 0
+; VI-NEXT:    v_mov_b32_e32 v1, 0
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_load_dword s0, s[0:1], 0x0
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    s_and_b32 s0, s0, 0xffff
+; VI-NEXT:    s_addk_i32 s0, 0x3e7
+; VI-NEXT:    s_or_b32 s0, s0, 1
+; VI-NEXT:    v_mov_b32_e32 v2, s0
+; VI-NEXT:    flat_store_short v[0:1], v2
+; VI-NEXT:    s_endpgm
   %load = load i16, i16 addrspace(1)* %arg, align 4, !invariant.load !0
   %add = add i16 %load, 999
   %or = or i16 %add, 1
diff --git a/test/CodeGen/ARM/2009-08-21-PostRAKill2.ll b/test/CodeGen/ARM/2009-08-21-PostRAKill2.ll
index 489d4e45236caa08bbec3a20d188a9f438b13c1b..d5570df717f59382581ea1f64de71352dd8bb908 100644
--- a/test/CodeGen/ARM/2009-08-21-PostRAKill2.ll
+++ b/test/CodeGen/ARM/2009-08-21-PostRAKill2.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -asm-verbose=false -O3 -relocation-model=pic -disable-fp-elim -mtriple=thumbv7-apple-darwin -mcpu=cortex-a8 -post-RA-scheduler
+; RUN: llc < %s -asm-verbose=false -O3 -relocation-model=pic -frame-pointer=all -mtriple=thumbv7-apple-darwin -mcpu=cortex-a8 -post-RA-scheduler
 
 ; ModuleID = '<stdin>'
 target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:32-f32:32:32-f64:32:32-v64:64:64-v128:128:128-a0:0:64"
diff --git a/test/CodeGen/ARM/2009-08-21-PostRAKill3.ll b/test/CodeGen/ARM/2009-08-21-PostRAKill3.ll
index 133fc0588a919dd7a92c1d80b357112bd83e6559..b7a252beefbda3ad568fbd0e2e74c9978ddd5096 100644
--- a/test/CodeGen/ARM/2009-08-21-PostRAKill3.ll
+++ b/test/CodeGen/ARM/2009-08-21-PostRAKill3.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -asm-verbose=false -O3 -relocation-model=pic -disable-fp-elim -mtriple=thumbv7-apple-darwin -mcpu=cortex-a8 -post-RA-scheduler
+; RUN: llc < %s -asm-verbose=false -O3 -relocation-model=pic -frame-pointer=all -mtriple=thumbv7-apple-darwin -mcpu=cortex-a8 -post-RA-scheduler
 
 ; ModuleID = '<stdin>'
 target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:32-f32:32:32-f64:32:32-v64:64:64-v128:128:128-a0:0:64"
diff --git a/test/CodeGen/ARM/2011-03-10-DAGCombineCrash.ll b/test/CodeGen/ARM/2011-03-10-DAGCombineCrash.ll
index 30a388bb587726a5d228a9310c0bc3d048b0a070..b33b333b299d09c52cdc38999afd4d6bcca6da08 100644
--- a/test/CodeGen/ARM/2011-03-10-DAGCombineCrash.ll
+++ b/test/CodeGen/ARM/2011-03-10-DAGCombineCrash.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -mtriple=thumbv7-apple-darwin10 -relocation-model=pic -disable-fp-elim -mcpu=cortex-a8
+; RUN: llc < %s -mtriple=thumbv7-apple-darwin10 -relocation-model=pic -frame-pointer=all -mcpu=cortex-a8
 
 ; rdar://9117613
 
diff --git a/test/CodeGen/ARM/2011-03-15-LdStMultipleBug.ll b/test/CodeGen/ARM/2011-03-15-LdStMultipleBug.ll
index 92bdd19a7b3d7608fca36d9f0dffdd274c74e936..b526b8c3075d53dd2116ec1a1c64d01413b0e726 100644
--- a/test/CodeGen/ARM/2011-03-15-LdStMultipleBug.ll
+++ b/test/CodeGen/ARM/2011-03-15-LdStMultipleBug.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -mtriple=thumbv7-apple-darwin10 -relocation-model=pic -disable-fp-elim -mcpu=cortex-a8 | FileCheck %s
+; RUN: llc < %s -mtriple=thumbv7-apple-darwin10 -relocation-model=pic -frame-pointer=all -mcpu=cortex-a8 | FileCheck %s
 
 ; Do not form Thumb2 ldrd / strd if the offset is not multiple of 4.
 ; rdar://9133587
diff --git a/test/CodeGen/ARM/2011-03-23-PeepholeBug.ll b/test/CodeGen/ARM/2011-03-23-PeepholeBug.ll
index 83c7676e57ef67ec31fee8028cf89466d5e657d6..4567b7f5fe2145df302d1d70ed49e9217e84be8f 100644
--- a/test/CodeGen/ARM/2011-03-23-PeepholeBug.ll
+++ b/test/CodeGen/ARM/2011-03-23-PeepholeBug.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -mtriple=thumbv7-apple-darwin10 -relocation-model=pic -disable-fp-elim -mcpu=cortex-a8 | FileCheck %s
+; RUN: llc < %s -mtriple=thumbv7-apple-darwin10 -relocation-model=pic -frame-pointer=all -mcpu=cortex-a8 | FileCheck %s
 
 ; subs r4, #1
 ; cmp r4, 0
diff --git a/test/CodeGen/ARM/2012-04-24-SplitEHCriticalEdge.ll b/test/CodeGen/ARM/2012-04-24-SplitEHCriticalEdge.ll
index ef33b2f50184cd366a7d5c6303cfa7d4f8847314..4ddc7284f58e575b137bcfb55d910c7a81c28aab 100644
--- a/test/CodeGen/ARM/2012-04-24-SplitEHCriticalEdge.ll
+++ b/test/CodeGen/ARM/2012-04-24-SplitEHCriticalEdge.ll
@@ -1,4 +1,4 @@
-; RUN: llc -mtriple=thumbv7-apple-ios -relocation-model=pic -disable-fp-elim -mcpu=cortex-a8 < %s
+; RUN: llc -mtriple=thumbv7-apple-ios -relocation-model=pic -frame-pointer=all -mcpu=cortex-a8 < %s
 
 ; CodeGen SplitCriticalEdge() shouldn't try to break edge to a landing pad.
 ; rdar://11300144
diff --git a/test/CodeGen/ARM/CGP/arm-cgp-casts.ll b/test/CodeGen/ARM/CGP/arm-cgp-casts.ll
index 273fea370a1f647dbf05e639b660bdb9228d1f24..3e5e2fa339ccebef76a3725327409d831f6edcb6 100644
--- a/test/CodeGen/ARM/CGP/arm-cgp-casts.ll
+++ b/test/CodeGen/ARM/CGP/arm-cgp-casts.ll
@@ -588,3 +588,47 @@ cond.end:
   store i8 %cond, i8* @a, align 1
   ret void
 }
+
+; CHECK-LABEL: dont_replace_trunc_1
+; CHECK: cmp
+; CHECK: uxtb
+define void @dont_replace_trunc_1(i8* %a, i16* %b, i16* %c, i32* %d, i8* %e, i32* %f) {
+entry:
+  %0 = load i16, i16* %c, align 2
+  %1 = load i16, i16* %b, align 2
+  %conv = sext i16 %1 to i32
+  store i32 %conv, i32* %f, align 4
+  %2 = trunc i16 %1 to i8
+  %conv1 = and i8 %2, 1
+  store i8 %conv1, i8* %e, align 1
+  %3 = load i8, i8* %a, align 1
+  %narrow = mul nuw i8 %3, %conv1
+  %mul = zext i8 %narrow to i32
+  store i32 %mul, i32* %d, align 4
+  %4 = zext i8 %narrow to i16
+  %conv5 = or i16 %0, %4
+  %tobool = icmp eq i16 %conv5, 0
+  br i1 %tobool, label %if.end, label %for.cond
+
+for.cond:                                         ; preds = %entry, %for.cond
+  br label %for.cond
+
+if.end:                                           ; preds = %entry
+  ret void
+}
+
+; CHECK-LABEL: dont_replace_trunc_2
+; CHECK: cmp
+; CHECK: uxtb
+define i32 @dont_replace_trunc_2(i16* %a, i8* %b) {
+entry:
+  %0 = load i16, i16* %a, align 2
+  %cmp = icmp ugt i16 %0, 8
+  %narrow = select i1 %cmp, i16 %0, i16 0
+  %cond = trunc i16 %narrow to i8
+  %1 = load i8, i8* %b, align 1
+  %or = or i8 %1, %cond
+  store i8 %or, i8* %b, align 1
+  %conv5 = zext i8 %or to i32
+  ret i32 %conv5
+}
diff --git a/test/CodeGen/ARM/GlobalISel/arm-instruction-select-combos.mir b/test/CodeGen/ARM/GlobalISel/arm-instruction-select-combos.mir
index d3e9796b72f5282270359acbc532cc2584fee714..c2561ade8de137287cbe42219933bacd0ff09200 100644
--- a/test/CodeGen/ARM/GlobalISel/arm-instruction-select-combos.mir
+++ b/test/CodeGen/ARM/GlobalISel/arm-instruction-select-combos.mir
@@ -7,11 +7,6 @@
   define void @test_mls() #2 { ret void }
   define void @test_no_mls() { ret void }
 
-  define void @test_shifts_to_revsh() #0 { ret void }
-  define void @test_shifts_to_revsh_commutative() #0 { ret void }
-  define void @test_shifts_no_revsh_features() #1 { ret void }
-  define void @test_shifts_no_revsh_constants() #0 { ret void }
-
   define void @test_bicrr() { ret void }
   define void @test_bicrr_commutative() { ret void }
 
@@ -20,14 +15,6 @@
   define void @test_bicri_commutative_and() { ret void }
   define void @test_bicri_commutative_both() { ret void }
 
-  define void @test_pkhbt() #0 { ret void }
-  define void @test_pkhbt_commutative() #0 { ret void }
-  define void @test_pkhbt_imm16_31() #0 { ret void }
-  define void @test_pkhbt_unshifted() #0 { ret void }
-
-  define void @test_pkhtb_imm16() #0 { ret void }
-  define void @test_pkhtb_imm1_15() #0 { ret void }
-
   define void @test_movti16_0xffff() #2 { ret void }
 
   define void @test_vnmuls() #3 { ret void }
@@ -223,186 +210,6 @@ body:             |
     ; CHECK: BX_RET 14, $noreg, implicit $r0
 ...
 ---
-name:            test_shifts_to_revsh
-# CHECK-LABEL: name: test_shifts_to_revsh
-legalized:       true
-regBankSelected: true
-selected:        false
-# CHECK: selected: true
-registers:
-  - { id: 0, class: gprb }
-  - { id: 1, class: gprb }
-  - { id: 2, class: gprb }
-  - { id: 3, class: gprb }
-  - { id: 4, class: gprb }
-  - { id: 5, class: gprb }
-  - { id: 6, class: gprb }
-  - { id: 7, class: gprb }
-  - { id: 8, class: gprb }
-  - { id: 9, class: gprb }
-body:             |
-  bb.0:
-    liveins: $r0
-
-    %0(s32) = COPY $r0
-    ; CHECK: [[VREGX:%[0-9]+]]:gpr = COPY $r0
-
-    %1(s32) = G_CONSTANT i32 24
-    %2(s32) = G_SHL %0(s32), %1(s32)
-
-    %3(s32) = G_CONSTANT i32 16
-    %4(s32) = G_ASHR %2(s32), %3(s32)
-
-    %5(s32) = G_CONSTANT i32 8
-    %6(s32) = G_LSHR %0(s32), %5(s32)
-
-    %7(s32) = G_CONSTANT 255
-    %8(s32) = G_AND %6(s32), %7(s32)
-
-    %9(s32) = G_OR %4(s32), %8(s32)
-    ; CHECK: [[VREGR:%[0-9]+]]:gpr = REVSH [[VREGX]]
-
-    $r0 = COPY %9(s32)
-    ; CHECK: $r0 = COPY [[VREGR]]
-
-    BX_RET 14, $noreg, implicit $r0
-    ; CHECK: BX_RET 14, $noreg, implicit $r0
-...
----
-name:            test_shifts_to_revsh_commutative
-# CHECK-LABEL: name: test_shifts_to_revsh_commutative
-legalized:       true
-regBankSelected: true
-selected:        false
-# CHECK: selected: true
-registers:
-  - { id: 0, class: gprb }
-  - { id: 1, class: gprb }
-  - { id: 2, class: gprb }
-  - { id: 3, class: gprb }
-  - { id: 4, class: gprb }
-  - { id: 5, class: gprb }
-  - { id: 6, class: gprb }
-  - { id: 7, class: gprb }
-  - { id: 8, class: gprb }
-  - { id: 9, class: gprb }
-body:             |
-  bb.0:
-    liveins: $r0
-
-    %0(s32) = COPY $r0
-    ; CHECK: [[VREGX:%[0-9]+]]:gpr = COPY $r0
-
-    %1(s32) = G_CONSTANT i32 24
-    %2(s32) = G_SHL %0(s32), %1(s32)
-
-    %3(s32) = G_CONSTANT i32 16
-    %4(s32) = G_ASHR %2(s32), %3(s32)
-
-    %5(s32) = G_CONSTANT i32 8
-    %6(s32) = G_LSHR %0(s32), %5(s32)
-
-    %7(s32) = G_CONSTANT 255
-    %8(s32) = G_AND %6(s32), %7(s32)
-
-    %9(s32) = G_OR %8(s32), %4(s32)
-    ; CHECK: [[VREGR:%[0-9]+]]:gpr = REVSH [[VREGX]]
-
-    $r0 = COPY %9(s32)
-    ; CHECK: $r0 = COPY [[VREGR]]
-
-    BX_RET 14, $noreg, implicit $r0
-    ; CHECK: BX_RET 14, $noreg, implicit $r0
-...
----
-name:            test_shifts_no_revsh_features
-# CHECK-LABEL: name: test_shifts_no_revsh
-legalized:       true
-regBankSelected: true
-selected:        false
-# CHECK: selected: true
-registers:
-  - { id: 0, class: gprb }
-  - { id: 1, class: gprb }
-  - { id: 2, class: gprb }
-  - { id: 3, class: gprb }
-  - { id: 4, class: gprb }
-  - { id: 5, class: gprb }
-  - { id: 6, class: gprb }
-  - { id: 7, class: gprb }
-  - { id: 8, class: gprb }
-  - { id: 9, class: gprb }
-body:             |
-  bb.0:
-    liveins: $r0
-
-    %0(s32) = COPY $r0
-
-    %1(s32) = G_CONSTANT i32 24
-    %2(s32) = G_SHL %0(s32), %1(s32)
-
-    %3(s32) = G_CONSTANT i32 16
-    %4(s32) = G_ASHR %2(s32), %3(s32)
-
-    %5(s32) = G_CONSTANT i32 8
-    %6(s32) = G_LSHR %0(s32), %5(s32)
-
-    %7(s32) = G_CONSTANT 255
-    %8(s32) = G_AND %6(s32), %7(s32)
-
-    %9(s32) = G_OR %4(s32), %8(s32)
-    ; We don't really care how this is folded as long as it's not into a REVSH.
-    ; CHECK-NOT: REVSH
-
-    $r0 = COPY %9(s32)
-
-    BX_RET 14, $noreg, implicit $r0
-...
----
-name:            test_shifts_no_revsh_constants
-# CHECK-LABEL: name: test_shifts_no_revsh_constants
-legalized:       true
-regBankSelected: true
-selected:        false
-# CHECK: selected: true
-registers:
-  - { id: 0, class: gprb }
-  - { id: 1, class: gprb }
-  - { id: 2, class: gprb }
-  - { id: 3, class: gprb }
-  - { id: 4, class: gprb }
-  - { id: 5, class: gprb }
-  - { id: 6, class: gprb }
-  - { id: 7, class: gprb }
-  - { id: 8, class: gprb }
-  - { id: 9, class: gprb }
-body:             |
-  bb.0:
-    liveins: $r0
-
-    %0(s32) = COPY $r0
-
-    %1(s32) = G_CONSTANT i32 16 ; REVSH needs 24 here
-    %2(s32) = G_SHL %0(s32), %1(s32)
-
-    %3(s32) = G_CONSTANT i32 24 ; REVSH needs 16 here
-    %4(s32) = G_ASHR %2(s32), %3(s32)
-
-    %5(s32) = G_CONSTANT i32 8
-    %6(s32) = G_LSHR %0(s32), %5(s32)
-
-    %7(s32) = G_CONSTANT 255
-    %8(s32) = G_AND %6(s32), %7(s32)
-
-    %9(s32) = G_OR %4(s32), %8(s32)
-    ; We don't really care how this is folded as long as it's not into a REVSH.
-    ; CHECK-NOT: REVSH
-
-    $r0 = COPY %9(s32)
-
-    BX_RET 14, $noreg, implicit $r0
-...
----
 name:            test_bicrr
 # CHECK-LABEL: name: test_bicrr
 legalized:       true
@@ -606,252 +413,6 @@ body:             |
     ; CHECK: BX_RET 14, $noreg, implicit $r0
 ...
 ---
-name:            test_pkhbt
-# CHECK-LABEL: name: test_pkhbt
-legalized:       true
-regBankSelected: true
-selected:        false
-# CHECK: selected: true
-registers:
-  - { id: 0, class: gprb }
-  - { id: 1, class: gprb }
-  - { id: 2, class: gprb }
-  - { id: 3, class: gprb }
-  - { id: 4, class: gprb }
-  - { id: 5, class: gprb }
-  - { id: 6, class: gprb }
-  - { id: 7, class: gprb }
-  - { id: 8, class: gprb }
-body:             |
-  bb.0:
-    liveins: $r0, $r1
-
-    %0(s32) = COPY $r0
-    %1(s32) = COPY $r1
-    ; CHECK-DAG: [[VREGX:%[0-9]+]]:gprnopc = COPY $r0
-    ; CHECK-DAG: [[VREGY:%[0-9]+]]:gprnopc = COPY $r1
-
-    %2(s32) = G_CONSTANT i32 65535 ; 0xFFFF
-    %3(s32) = G_AND %0, %2
-
-    %4(s32) = G_CONSTANT i32 7
-    %5(s32) = G_SHL %1, %4
-    %6(s32) = G_CONSTANT i32 4294901760 ; 0xFFFF0000
-    %7(s32) = G_AND %5, %6
-
-    %8(s32) = G_OR %3, %7
-    ; CHECK: [[VREGR:%[0-9]+]]:gprnopc = PKHBT [[VREGX]], [[VREGY]], 7, 14, $noreg
-
-    $r0 = COPY %8(s32)
-    ; CHECK: $r0 = COPY [[VREGR]]
-
-    BX_RET 14, $noreg, implicit $r0
-    ; CHECK: BX_RET 14, $noreg, implicit $r0
-...
----
-name:            test_pkhbt_commutative
-# CHECK-LABEL: name: test_pkhbt_commutative
-legalized:       true
-regBankSelected: true
-selected:        false
-# CHECK: selected: true
-registers:
-  - { id: 0, class: gprb }
-  - { id: 1, class: gprb }
-  - { id: 2, class: gprb }
-  - { id: 3, class: gprb }
-  - { id: 4, class: gprb }
-  - { id: 5, class: gprb }
-  - { id: 6, class: gprb }
-  - { id: 7, class: gprb }
-  - { id: 8, class: gprb }
-body:             |
-  bb.0:
-    liveins: $r0, $r1
-
-    %0(s32) = COPY $r0
-    %1(s32) = COPY $r1
-    ; CHECK-DAG: [[VREGX:%[0-9]+]]:gprnopc = COPY $r0
-    ; CHECK-DAG: [[VREGY:%[0-9]+]]:gprnopc = COPY $r1
-
-    %2(s32) = G_CONSTANT i32 65535 ; 0xFFFF
-    %3(s32) = G_AND %0, %2
-
-    %4(s32) = G_CONSTANT i32 7
-    %5(s32) = G_SHL %1, %4
-    %6(s32) = G_CONSTANT i32 4294901760 ; 0xFFFF0000
-    %7(s32) = G_AND %5, %6
-
-    %8(s32) = G_OR %7, %3
-    ; CHECK: [[VREGR:%[0-9]+]]:gprnopc = PKHBT [[VREGX]], [[VREGY]], 7, 14, $noreg
-
-    $r0 = COPY %8(s32)
-    ; CHECK: $r0 = COPY [[VREGR]]
-
-    BX_RET 14, $noreg, implicit $r0
-    ; CHECK: BX_RET 14, $noreg, implicit $r0
-...
----
-name:            test_pkhbt_imm16_31
-# CHECK-LABEL: name: test_pkhbt_imm16_31
-legalized:       true
-regBankSelected: true
-selected:        false
-# CHECK: selected: true
-registers:
-  - { id: 0, class: gprb }
-  - { id: 1, class: gprb }
-  - { id: 2, class: gprb }
-  - { id: 3, class: gprb }
-  - { id: 4, class: gprb }
-  - { id: 5, class: gprb }
-  - { id: 6, class: gprb }
-body:             |
-  bb.0:
-    liveins: $r0, $r1
-
-    %0(s32) = COPY $r0
-    %1(s32) = COPY $r1
-    ; CHECK-DAG: [[VREGX:%[0-9]+]]:gprnopc = COPY $r0
-    ; CHECK-DAG: [[VREGY:%[0-9]+]]:gprnopc = COPY $r1
-
-    %2(s32) = G_CONSTANT i32 65535 ; 0xFFFF
-    %3(s32) = G_AND %0, %2
-
-    %4(s32) = G_CONSTANT i32 17
-    %5(s32) = G_SHL %1, %4
-
-    %6(s32) = G_OR %3, %5
-    ; CHECK: [[VREGR:%[0-9]+]]:gprnopc = PKHBT [[VREGX]], [[VREGY]], 17, 14, $noreg
-
-    $r0 = COPY %6(s32)
-    ; CHECK: $r0 = COPY [[VREGR]]
-
-    BX_RET 14, $noreg, implicit $r0
-    ; CHECK: BX_RET 14, $noreg, implicit $r0
-...
----
-name:            test_pkhbt_unshifted
-# CHECK-LABEL: name: test_pkhbt_unshifted
-legalized:       true
-regBankSelected: true
-selected:        false
-# CHECK: selected: true
-registers:
-  - { id: 0, class: gprb }
-  - { id: 1, class: gprb }
-  - { id: 2, class: gprb }
-  - { id: 3, class: gprb }
-  - { id: 4, class: gprb }
-  - { id: 5, class: gprb }
-  - { id: 6, class: gprb }
-body:             |
-  bb.0:
-    liveins: $r0, $r1
-
-    %0(s32) = COPY $r0
-    %1(s32) = COPY $r1
-    ; CHECK-DAG: [[VREGX:%[0-9]+]]:gprnopc = COPY $r0
-    ; CHECK-DAG: [[VREGY:%[0-9]+]]:gprnopc = COPY $r1
-
-    %2(s32) = G_CONSTANT i32 65535 ; 0xFFFF
-    %3(s32) = G_AND %0, %2
-
-    %4(s32) = G_CONSTANT i32 4294901760 ; 0xFFFF0000
-    %5(s32) = G_AND %1, %4
-
-    %6(s32) = G_OR %3, %5
-    ; CHECK: [[VREGR:%[0-9]+]]:gprnopc = PKHBT [[VREGX]], [[VREGY]], 0, 14, $noreg
-
-    $r0 = COPY %6(s32)
-    ; CHECK: $r0 = COPY [[VREGR]]
-
-    BX_RET 14, $noreg, implicit $r0
-    ; CHECK: BX_RET 14, $noreg, implicit $r0
-...
----
-name:            test_pkhtb_imm16
-# CHECK-LABEL: name: test_pkhtb_imm16
-legalized:       true
-regBankSelected: true
-selected:        false
-# CHECK: selected: true
-registers:
-  - { id: 0, class: gprb }
-  - { id: 1, class: gprb }
-  - { id: 2, class: gprb }
-  - { id: 3, class: gprb }
-  - { id: 4, class: gprb }
-  - { id: 5, class: gprb }
-  - { id: 6, class: gprb }
-body:             |
-  bb.0:
-    liveins: $r0, $r1
-
-    %0(s32) = COPY $r0
-    %1(s32) = COPY $r1
-    ; CHECK-DAG: [[VREGX:%[0-9]+]]:gprnopc = COPY $r0
-    ; CHECK-DAG: [[VREGY:%[0-9]+]]:gprnopc = COPY $r1
-
-    %2(s32) = G_CONSTANT i32 4294901760 ; 0xFFFF0000
-    %3(s32) = G_AND %0, %2
-
-    %4(s32) = G_CONSTANT i32 16
-    %5(s32) = G_LSHR %1, %4
-
-    %6(s32) = G_OR %3, %5
-    ; CHECK: [[VREGR:%[0-9]+]]:gprnopc = PKHTB [[VREGX]], [[VREGY]], 16, 14, $noreg
-
-    $r0 = COPY %6(s32)
-    ; CHECK: $r0 = COPY [[VREGR]]
-
-    BX_RET 14, $noreg, implicit $r0
-    ; CHECK: BX_RET 14, $noreg, implicit $r0
-...
----
-name:            test_pkhtb_imm1_15
-# CHECK-LABEL: name: test_pkhtb_imm1_15
-legalized:       true
-regBankSelected: true
-selected:        false
-# CHECK: selected: true
-registers:
-  - { id: 0, class: gprb }
-  - { id: 1, class: gprb }
-  - { id: 2, class: gprb }
-  - { id: 3, class: gprb }
-  - { id: 4, class: gprb }
-  - { id: 5, class: gprb }
-  - { id: 6, class: gprb }
-  - { id: 7, class: gprb }
-  - { id: 8, class: gprb }
-body:             |
-  bb.0:
-    liveins: $r0, $r1
-
-    %0(s32) = COPY $r0
-    %1(s32) = COPY $r1
-    ; CHECK-DAG: [[VREGX:%[0-9]+]]:gprnopc = COPY $r0
-    ; CHECK-DAG: [[VREGY:%[0-9]+]]:gprnopc = COPY $r1
-
-    %2(s32) = G_CONSTANT i32 4294901760 ; 0xFFFF0000
-    %3(s32) = G_AND %0, %2
-
-    %4(s32) = G_CONSTANT i32 7
-    %5(s32) = G_LSHR %1, %4
-    %6(s32) = G_CONSTANT i32 65535 ; 0xFFFF
-    %7(s32) = G_AND %5, %6
-
-    %8(s32) = G_OR %3, %7
-    ; CHECK: [[VREGR:%[0-9]+]]:gprnopc = PKHTB [[VREGX]], [[VREGY]], 7, 14, $noreg
-
-    $r0 = COPY %8(s32)
-    ; CHECK: $r0 = COPY [[VREGR]]
-
-    BX_RET 14, $noreg, implicit $r0
-    ; CHECK: BX_RET 14, $noreg, implicit $r0
-...
----
 name:            test_movti16_0xffff
 # CHECK-LABEL: name: test_movti16_0xffff
 legalized:       true
diff --git a/test/CodeGen/ARM/GlobalISel/arm-instruction-select.mir b/test/CodeGen/ARM/GlobalISel/arm-instruction-select.mir
index 4b9812c50d15b5c98c22de06a6d6b32f0fe40f14..82d5069dbf8e624976dd1d40144d0e14ec2b6897 100644
--- a/test/CodeGen/ARM/GlobalISel/arm-instruction-select.mir
+++ b/test/CodeGen/ARM/GlobalISel/arm-instruction-select.mir
@@ -64,8 +64,12 @@
   define void @test_stores() #0 { ret void }
 
   define void @test_gep() { ret void }
+
+  define void @test_MOVi32imm() #3 { ret void }
+
   define void @test_constant_imm() { ret void }
   define void @test_constant_cimm() { ret void }
+
   define void @test_pointer_constant_unconstrained() { ret void }
   define void @test_pointer_constant_constrained() { ret void }
 
@@ -1481,6 +1485,23 @@ body:             |
     BX_RET 14, $noreg, implicit $r0
 ...
 ---
+name:            test_MOVi32imm
+# CHECK-LABEL: name: test_MOVi32imm
+legalized:       true
+regBankSelected: true
+selected:        false
+# CHECK: selected: true
+registers:
+  - { id: 0, class: gprb }
+body:             |
+  bb.0:
+    %0(s32) = G_CONSTANT i32 65537
+    ; CHECK: %[[C:[0-9]+]]:gpr = MOVi32imm 65537
+
+    $r0 = COPY %0(s32)
+    BX_RET 14, $noreg, implicit $r0
+...
+---
 name:            test_constant_imm
 # CHECK-LABEL: name: test_constant_imm
 legalized:       true
@@ -1491,7 +1512,7 @@ registers:
   - { id: 0, class: gprb }
 body:             |
   bb.0:
-    %0(s32) = G_CONSTANT 42
+    %0(s32) = G_CONSTANT i32 42
     ; CHECK: %[[C:[0-9]+]]:gpr = MOVi 42, 14, $noreg, $noreg
 
     $r0 = COPY %0(s32)
diff --git a/test/CodeGen/ARM/GlobalISel/arm-legalize-binops.mir b/test/CodeGen/ARM/GlobalISel/arm-legalize-binops.mir
index 51783caee3c66948147adc9e079568995e240b19..b95fe1f9aa53bcbdacb4f9142b741a5c28bbc055 100644
--- a/test/CodeGen/ARM/GlobalISel/arm-legalize-binops.mir
+++ b/test/CodeGen/ARM/GlobalISel/arm-legalize-binops.mir
@@ -24,6 +24,10 @@
   define void @test_xor_s8() { ret void }
   define void @test_xor_s16() { ret void }
   define void @test_xor_s32() { ret void }
+
+  define void @test_lshr_s32() { ret void }
+  define void @test_ashr_s32() { ret void }
+  define void @test_shl_s32() { ret void }
 ...
 ---
 name:            test_add_s8
@@ -559,3 +563,78 @@ body:             |
     BX_RET 14, $noreg, implicit $r0
 
 ...
+---
+name:            test_lshr_s32
+# CHECK-LABEL: name: test_lshr_s32
+legalized:       false
+# CHECK: legalized: true
+regBankSelected: false
+selected:        false
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+  - { id: 2, class: _ }
+body:             |
+  bb.0:
+    liveins: $r0, $r1
+
+    %0(s32) = COPY $r0
+    %1(s32) = COPY $r1
+    %2(s32) = G_LSHR %0, %1
+    ; G_LSHR with s32 is legal, so we should find it unchanged in the output
+    ; CHECK: {{%[0-9]+}}:_(s32) = G_LSHR {{%[0-9]+, %[0-9]+}}
+    $r0 = COPY %2(s32)
+    BX_RET 14, $noreg, implicit $r0
+
+...
+---
+name:            test_ashr_s32
+# CHECK-LABEL: name: test_ashr_s32
+legalized:       false
+# CHECK: legalized: true
+regBankSelected: false
+selected:        false
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+  - { id: 2, class: _ }
+body:             |
+  bb.0:
+    liveins: $r0, $r1
+
+    %0(s32) = COPY $r0
+    %1(s32) = COPY $r1
+    %2(s32) = G_ASHR %0, %1
+    ; G_ASHR with s32 is legal, so we should find it unchanged in the output
+    ; CHECK: {{%[0-9]+}}:_(s32) = G_ASHR {{%[0-9]+, %[0-9]+}}
+    $r0 = COPY %2(s32)
+    BX_RET 14, $noreg, implicit $r0
+
+...
+---
+name:            test_shl_s32
+# CHECK-LABEL: name: test_shl_s32
+legalized:       false
+# CHECK: legalized: true
+regBankSelected: false
+selected:        false
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+  - { id: 2, class: _ }
+body:             |
+  bb.0:
+    liveins: $r0, $r1
+
+    %0(s32) = COPY $r0
+    %1(s32) = COPY $r1
+    %2(s32) = G_SHL %0, %1
+    ; G_SHL with s32 is legal, so we should find it unchanged in the output
+    ; CHECK: {{%[0-9]+}}:_(s32) = G_SHL {{%[0-9]+, %[0-9]+}}
+    $r0 = COPY %2(s32)
+    BX_RET 14, $noreg, implicit $r0
+
+...
diff --git a/test/CodeGen/ARM/GlobalISel/arm-legalize-bitcounts.mir b/test/CodeGen/ARM/GlobalISel/arm-legalize-bitcounts.mir
index c9323eedced43e8ab5d91f7c3761296933cb8a6a..962dd844b28426b191bb8a4e82239bdd2ac6cb97 100644
--- a/test/CodeGen/ARM/GlobalISel/arm-legalize-bitcounts.mir
+++ b/test/CodeGen/ARM/GlobalISel/arm-legalize-bitcounts.mir
@@ -1,5 +1,5 @@
-# RUN: llc -mtriple arm-linux-gnueabi -mattr=+v5t -run-pass=legalizer %s -o - | FileCheck %s -check-prefixes=CHECK,CLZ
-# RUN: llc -mtriple arm-linux-gnueabi -mattr=-v5t -run-pass=legalizer %s -o - | FileCheck %s -check-prefixes=CHECK,LIBCALLS
+# RUN: llc -O0 -mtriple arm-linux-gnueabi -mattr=+v5t -run-pass=legalizer %s -o - | FileCheck %s -check-prefixes=CHECK,CLZ
+# RUN: llc -O0 -mtriple arm-linux-gnueabi -mattr=-v5t -run-pass=legalizer %s -o - | FileCheck %s -check-prefixes=CHECK,LIBCALLS
 --- |
   define void @test_ctlz_s32() { ret void }
   define void @test_ctlz_zero_undef_s32() { ret void }
diff --git a/test/CodeGen/ARM/GlobalISel/arm-legalize-cmp.mir b/test/CodeGen/ARM/GlobalISel/arm-legalize-cmp.mir
new file mode 100644
index 0000000000000000000000000000000000000000..dd06933603c2c290f53469430ba9605c7fc4f9b4
--- /dev/null
+++ b/test/CodeGen/ARM/GlobalISel/arm-legalize-cmp.mir
@@ -0,0 +1,123 @@
+# RUN: llc -mtriple arm-- -run-pass=legalizer %s -o - | FileCheck %s
+# RUN: llc -mtriple thumb-- -mattr=+v6t2 -run-pass=legalizer %s -o - | FileCheck %s
+--- |
+  define void @test_icmp_s8() { ret void }
+  define void @test_icmp_s16() { ret void }
+  define void @test_icmp_s32() { ret void }
+
+  define void @test_icmp_p0() { ret void }
+...
+---
+name:            test_icmp_s8
+# CHECK-LABEL: name: test_icmp_s8
+legalized:       false
+# CHECK: legalized: true
+regBankSelected: false
+selected:        false
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+  - { id: 2, class: _ }
+  - { id: 3, class: _ }
+  - { id: 4, class: _ }
+  - { id: 5, class: _ }
+body:             |
+  bb.0:
+    liveins: $r0, $r1
+
+    %0(p0) = COPY $r0
+    %1(s8) = G_LOAD %0 :: (load 1)
+    %2(p0) = COPY $r1
+    %3(s8) = G_LOAD %2 :: (load 1)
+    %4(s1) = G_ICMP intpred(ne), %1(s8), %3
+    ; G_ICMP with s8 should widen
+    ; CHECK: {{%[0-9]+}}:_(s1) = G_ICMP intpred(ne), {{%[0-9]+}}(s32), {{%[0-9]+}}
+    ; CHECK-NOT: {{%[0-9]+}}:_(s1) = G_ICMP intpred(ne), {{%[0-9]+}}(s8), {{%[0-9]+}}
+    %5(s32) = G_ZEXT %4(s1)
+    $r0 = COPY %5(s32)
+    BX_RET 14, $noreg, implicit $r0
+...
+---
+name:            test_icmp_s16
+# CHECK-LABEL: name: test_icmp_s16
+legalized:       false
+# CHECK: legalized: true
+regBankSelected: false
+selected:        false
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+  - { id: 2, class: _ }
+  - { id: 3, class: _ }
+  - { id: 4, class: _ }
+  - { id: 5, class: _ }
+body:             |
+  bb.0:
+    liveins: $r0, $r1
+
+    %0(p0) = COPY $r0
+    %1(s16) = G_LOAD %0 :: (load 2)
+    %2(p0) = COPY $r1
+    %3(s16) = G_LOAD %2 :: (load 2)
+    %4(s1) = G_ICMP intpred(slt), %1(s16), %3
+    ; G_ICMP with s16 should widen
+    ; CHECK: {{%[0-9]+}}:_(s1) = G_ICMP intpred(slt), {{%[0-9]+}}(s32), {{%[0-9]+}}
+    ; CHECK-NOT: {{%[0-9]+}}:_(s1) = G_ICMP intpred(slt), {{%[0-9]+}}(s16), {{%[0-9]+}}
+    %5(s32) = G_ZEXT %4(s1)
+    $r0 = COPY %5(s32)
+    BX_RET 14, $noreg, implicit $r0
+...
+---
+name:            test_icmp_s32
+# CHECK-LABEL: name: test_icmp_s32
+legalized:       false
+# CHECK: legalized: true
+regBankSelected: false
+selected:        false
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+  - { id: 2, class: _ }
+  - { id: 3, class: _ }
+body:             |
+  bb.0:
+    liveins: $r0, $r1
+
+    %0(s32) = COPY $r0
+    %1(s32) = COPY $r1
+    %2(s1) = G_ICMP intpred(eq), %0(s32), %1
+    ; G_ICMP with s32 is legal, so we should find it unchanged in the output
+    ; CHECK: {{%[0-9]+}}:_(s1) = G_ICMP intpred(eq), {{%[0-9]+}}(s32), {{%[0-9]+}}
+    %3(s32) = G_ZEXT %2(s1)
+    $r0 = COPY %3(s32)
+    BX_RET 14, $noreg, implicit $r0
+...
+---
+name:            test_icmp_p0
+# CHECK-LABEL: name: test_icmp_p0
+legalized:       false
+# CHECK: legalized: true
+regBankSelected: false
+selected:        false
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+  - { id: 2, class: _ }
+  - { id: 3, class: _ }
+body:             |
+  bb.0:
+    liveins: $r0, $r1
+
+    %0(p0) = COPY $r0
+    %1(p0) = COPY $r1
+    %2(s1) = G_ICMP intpred(eq), %0(p0), %1
+    ; G_ICMP with p0 is legal, so we should find it unchanged in the output
+    ; CHECK: {{%[0-9]+}}:_(s1) = G_ICMP intpred(eq), {{%[0-9]+}}(p0), {{%[0-9]+}}
+    %3(s32) = G_ZEXT %2(s1)
+    $r0 = COPY %3(s32)
+    BX_RET 14, $noreg, implicit $r0
+...
diff --git a/test/CodeGen/ARM/GlobalISel/arm-legalize-consts.mir b/test/CodeGen/ARM/GlobalISel/arm-legalize-consts.mir
index d3248fe63a024864bb4f33ccc9cc650b7b651218..de5545594bf39af4752fc5f66a5af4d6c4889195 100644
--- a/test/CodeGen/ARM/GlobalISel/arm-legalize-consts.mir
+++ b/test/CodeGen/ARM/GlobalISel/arm-legalize-consts.mir
@@ -24,8 +24,8 @@ body:             |
 
     %4(p0) = COPY $r0
 
-    %0(s32) = G_CONSTANT 42
-    ; CHECK: {{%[0-9]+}}:_(s32) = G_CONSTANT 42
+    %0(s32) = G_CONSTANT i32 42
+    ; CHECK: {{%[0-9]+}}:_(s32) = G_CONSTANT i32 42
 
     %1(s16) = G_CONSTANT i16 21
     G_STORE %1(s16), %4(p0) :: (store 2)
@@ -48,9 +48,9 @@ body:             |
     ; CHECK: {{%[0-9]+}}:_(s1) = G_TRUNC [[EXT]](s32)
     ; CHECK-NOT: G_CONSTANT i1
 
-    %5(p0) = G_CONSTANT 0
+    %5(p0) = G_CONSTANT i32 0
     G_STORE %5(p0), %4(p0) :: (store 4)
-    ; CHECK: {{%[0-9]+}}:_(p0) = G_CONSTANT 0
+    ; CHECK: {{%[0-9]+}}:_(p0) = G_CONSTANT i32 0
 
     $r0 = COPY %0(s32)
     BX_RET 14, $noreg, implicit $r0
diff --git a/test/CodeGen/ARM/GlobalISel/arm-legalize-divmod.mir b/test/CodeGen/ARM/GlobalISel/arm-legalize-divmod.mir
index 89b056865c88c9ca5026d54ac410f056099018cc..462351ef19f18e245c67b5f9a07e32b18f1efe46 100644
--- a/test/CodeGen/ARM/GlobalISel/arm-legalize-divmod.mir
+++ b/test/CodeGen/ARM/GlobalISel/arm-legalize-divmod.mir
@@ -1,7 +1,11 @@
-# RUN: llc -mtriple arm-linux-gnueabi -mattr=+hwdiv-arm -run-pass=legalizer %s -o - | FileCheck %s -check-prefixes=CHECK,HWDIV
-# RUN: llc -mtriple arm-linux-gnueabi -mattr=-hwdiv-arm -run-pass=legalizer %s -o - | FileCheck %s -check-prefixes=CHECK,SOFT,SOFT-AEABI
-# RUN: llc -mtriple arm-linux-gnu -mattr=+hwdiv-arm -run-pass=legalizer %s -o - | FileCheck %s  -check-prefixes=CHECK,HWDIV
-# RUN: llc -mtriple arm-linux-gnu -mattr=-hwdiv-arm -run-pass=legalizer %s -o - | FileCheck %s  -check-prefixes=CHECK,SOFT,SOFT-DEFAULT
+# RUN: llc -O0 -mtriple arm-linux-gnueabi -mattr=+hwdiv-arm -run-pass=legalizer %s -o - | FileCheck %s -check-prefixes=CHECK,HWDIV
+# RUN: llc -O0 -mtriple arm-linux-gnueabi -mattr=-hwdiv-arm -run-pass=legalizer %s -o - | FileCheck %s -check-prefixes=CHECK,SOFT,SOFT-AEABI,ARM-AEABI
+# RUN: llc -O0 -mtriple arm-linux-gnu -mattr=+hwdiv-arm -run-pass=legalizer %s -o - | FileCheck %s  -check-prefixes=CHECK,HWDIV
+# RUN: llc -O0 -mtriple arm-linux-gnu -mattr=-hwdiv-arm -run-pass=legalizer %s -o - | FileCheck %s  -check-prefixes=CHECK,SOFT,SOFT-DEFAULT,ARM-DEFAULT
+# RUN: llc -O0 -mtriple thumb-linux-gnueabi -mattr=+v6t2,+hwdiv -run-pass=legalizer %s -o - | FileCheck %s -check-prefixes=CHECK,HWDIV
+# RUN: llc -O0 -mtriple thumb-linux-gnueabi -mattr=+v6t2,-hwdiv -run-pass=legalizer %s -o - | FileCheck %s -check-prefixes=CHECK,SOFT,SOFT-AEABI,THUMB-AEABI
+# RUN: llc -O0 -mtriple thumb-linux-gnu -mattr=+v6t2,+hwdiv -run-pass=legalizer %s -o - | FileCheck %s  -check-prefixes=CHECK,HWDIV
+# RUN: llc -O0 -mtriple thumb-linux-gnu -mattr=+v6t2,-hwdiv -run-pass=legalizer %s -o - | FileCheck %s  -check-prefixes=CHECK,SOFT,SOFT-DEFAULT,THUMB-DEFAULT
 --- |
   define void @test_sdiv_i32() { ret void }
   define void @test_udiv_i32() { ret void }
@@ -46,10 +50,11 @@ body:             |
     ; SOFT: ADJCALLSTACKDOWN
     ; SOFT-DAG: $r0 = COPY [[X]]
     ; SOFT-DAG: $r1 = COPY [[Y]]
-    ; SOFT-AEABI: BL &__aeabi_idiv, {{.*}}, implicit $r0, implicit $r1, implicit-def $r0
-    ; SOFT-AEABI: [[R:%[0-9]+]]:_(s32) = COPY $r0
-    ; SOFT-DEFAULT: BL &__divsi3, {{.*}}, implicit $r0, implicit $r1, implicit-def $r0
-    ; SOFT-DEFAULT: [[R:%[0-9]+]]:_(s32) = COPY $r0
+    ; ARM-AEABI: BL &__aeabi_idiv, {{.*}}, implicit $r0, implicit $r1, implicit-def $r0
+    ; THUMB-AEABI: tBL 14, $noreg, &__aeabi_idiv, {{.*}}, implicit $r0, implicit $r1, implicit-def $r0
+    ; ARM-DEFAULT: BL &__divsi3, {{.*}}, implicit $r0, implicit $r1, implicit-def $r0
+    ; THUMB-DEFAULT: tBL 14, $noreg, &__divsi3, {{.*}}, implicit $r0, implicit $r1, implicit-def $r0
+    ; SOFT: [[R:%[0-9]+]]:_(s32) = COPY $r0
     ; SOFT: ADJCALLSTACKUP
     ; SOFT-NOT: G_SDIV
     %2(s32) = G_SDIV %0, %1
@@ -82,10 +87,11 @@ body:             |
     ; SOFT: ADJCALLSTACKDOWN
     ; SOFT-DAG: $r0 = COPY [[X]]
     ; SOFT-DAG: $r1 = COPY [[Y]]
-    ; SOFT-AEABI: BL &__aeabi_uidiv, {{.*}}, implicit $r0, implicit $r1, implicit-def $r0
-    ; SOFT-AEABI: [[R:%[0-9]+]]:_(s32) = COPY $r0
-    ; SOFT-DEFAULT: BL &__udivsi3, {{.*}}, implicit $r0, implicit $r1, implicit-def $r0
-    ; SOFT-DEFAULT: [[R:%[0-9]+]]:_(s32) = COPY $r0
+    ; ARM-AEABI: BL &__aeabi_uidiv, {{.*}}, implicit $r0, implicit $r1, implicit-def $r0
+    ; THUMB-AEABI: tBL 14, $noreg, &__aeabi_uidiv, {{.*}}, implicit $r0, implicit $r1, implicit-def $r0
+    ; ARM-DEFAULT: BL &__udivsi3, {{.*}}, implicit $r0, implicit $r1, implicit-def $r0
+    ; THUMB-DEFAULT: tBL 14, $noreg, &__udivsi3, {{.*}}, implicit $r0, implicit $r1, implicit-def $r0
+    ; SOFT: [[R:%[0-9]+]]:_(s32) = COPY $r0
     ; SOFT: ADJCALLSTACKUP
     ; SOFT-NOT: G_UDIV
     %2(s32) = G_UDIV %0, %1
@@ -133,10 +139,11 @@ body:             |
     ; SOFT: ADJCALLSTACKDOWN
     ; SOFT-DAG: $r0 = COPY [[X32]]
     ; SOFT-DAG: $r1 = COPY [[Y32]]
-    ; SOFT-AEABI: BL &__aeabi_idiv, {{.*}}, implicit $r0, implicit $r1, implicit-def $r0
-    ; SOFT-AEABI: [[R32:%[0-9]+]]:_(s32) = COPY $r0
-    ; SOFT-DEFAULT: BL &__divsi3, {{.*}}, implicit $r0, implicit $r1, implicit-def $r0
-    ; SOFT-DEFAULT: [[R32:%[0-9]+]]:_(s32) = COPY $r0
+    ; ARM-AEABI: BL &__aeabi_idiv, {{.*}}, implicit $r0, implicit $r1, implicit-def $r0
+    ; THUMB-AEABI: tBL 14, $noreg, &__aeabi_idiv, {{.*}}, implicit $r0, implicit $r1, implicit-def $r0
+    ; ARM-DEFAULT: BL &__divsi3, {{.*}}, implicit $r0, implicit $r1, implicit-def $r0
+    ; THUMB-DEFAULT: tBL 14, $noreg, &__divsi3, {{.*}}, implicit $r0, implicit $r1, implicit-def $r0
+    ; SOFT: [[R32:%[0-9]+]]:_(s32) = COPY $r0
     ; SOFT: ADJCALLSTACKUP
     ; SOFT-NOT: G_SDIV
     ; CHECK: [[R:%[0-9]+]]:_(s32) = G_ASHR
@@ -185,10 +192,11 @@ body:             |
     ; SOFT: ADJCALLSTACKDOWN
     ; SOFT-DAG: $r0 = COPY [[X32]]
     ; SOFT-DAG: $r1 = COPY [[Y32]]
-    ; SOFT-AEABI: BL &__aeabi_uidiv, {{.*}}, implicit $r0, implicit $r1, implicit-def $r0
-    ; SOFT-AEABI: [[R32:%[0-9]+]]:_(s32) = COPY $r0
-    ; SOFT-DEFAULT: BL &__udivsi3, {{.*}}, implicit $r0, implicit $r1, implicit-def $r0
-    ; SOFT-DEFAULT: [[R32:%[0-9]+]]:_(s32) = COPY $r0
+    ; ARM-AEABI: BL &__aeabi_uidiv, {{.*}}, implicit $r0, implicit $r1, implicit-def $r0
+    ; THUMB-AEABI: tBL 14, $noreg, &__aeabi_uidiv, {{.*}}, implicit $r0, implicit $r1, implicit-def $r0
+    ; ARM-DEFAULT: BL &__udivsi3, {{.*}}, implicit $r0, implicit $r1, implicit-def $r0
+    ; THUMB-DEFAULT: tBL 14, $noreg, &__udivsi3, {{.*}}, implicit $r0, implicit $r1, implicit-def $r0
+    ; SOFT: [[R32:%[0-9]+]]:_(s32) = COPY $r0
     ; SOFT: ADJCALLSTACKUP
     ; SOFT-NOT: G_UDIV
     ; CHECK: [[R:%[0-9]+]]:_(s32) = G_AND
@@ -239,10 +247,11 @@ body:             |
     ; SOFT: ADJCALLSTACKDOWN
     ; SOFT-DAG: $r0 = COPY [[X32]]
     ; SOFT-DAG: $r1 = COPY [[Y32]]
-    ; SOFT-AEABI: BL &__aeabi_idiv, {{.*}}, implicit $r0, implicit $r1, implicit-def $r0
-    ; SOFT-AEABI: [[R32:%[0-9]+]]:_(s32) = COPY $r0
-    ; SOFT-DEFAULT: BL &__divsi3, {{.*}}, implicit $r0, implicit $r1, implicit-def $r0
-    ; SOFT-DEFAULT: [[R32:%[0-9]+]]:_(s32) = COPY $r0
+    ; ARM-AEABI: BL &__aeabi_idiv, {{.*}}, implicit $r0, implicit $r1, implicit-def $r0
+    ; THUMB-AEABI: tBL 14, $noreg, &__aeabi_idiv, {{.*}}, implicit $r0, implicit $r1, implicit-def $r0
+    ; ARM-DEFAULT: BL &__divsi3, {{.*}}, implicit $r0, implicit $r1, implicit-def $r0
+    ; THUMB-DEFAULT: tBL 14, $noreg, &__divsi3, {{.*}}, implicit $r0, implicit $r1, implicit-def $r0
+    ; SOFT: [[R32:%[0-9]+]]:_(s32) = COPY $r0
     ; SOFT: ADJCALLSTACKUP
     ; SOFT-NOT: G_SDIV
     ; CHECK: [[R:%[0-9]+]]:_(s32) = G_ASHR
@@ -291,10 +300,11 @@ body:             |
     ; SOFT: ADJCALLSTACKDOWN
     ; SOFT-DAG: $r0 = COPY [[X32]]
     ; SOFT-DAG: $r1 = COPY [[Y32]]
-    ; SOFT-AEABI: BL &__aeabi_uidiv, {{.*}}, implicit $r0, implicit $r1, implicit-def $r0
-    ; SOFT-AEABI: [[R32:%[0-9]+]]:_(s32) = COPY $r0
-    ; SOFT-DEFAULT: BL &__udivsi3, {{.*}}, implicit $r0, implicit $r1, implicit-def $r0
-    ; SOFT-DEFAULT: [[R32:%[0-9]+]]:_(s32) = COPY $r0
+    ; ARM-AEABI: BL &__aeabi_uidiv, {{.*}}, implicit $r0, implicit $r1, implicit-def $r0
+    ; THUMB-AEABI: tBL 14, $noreg, &__aeabi_uidiv, {{.*}}, implicit $r0, implicit $r1, implicit-def $r0
+    ; ARM-DEFAULT: BL &__udivsi3, {{.*}}, implicit $r0, implicit $r1, implicit-def $r0
+    ; THUMB-DEFAULT: tBL 14, $noreg, &__udivsi3, {{.*}}, implicit $r0, implicit $r1, implicit-def $r0
+    ; SOFT: [[R32:%[0-9]+]]:_(s32) = COPY $r0
     ; SOFT: ADJCALLSTACKUP
     ; SOFT-NOT: G_UDIV
     ; CHECK: [[R:%[0-9]+]]:_(s32) = G_AND
@@ -332,9 +342,11 @@ body:             |
     ; SOFT: ADJCALLSTACKDOWN
     ; SOFT-DAG: $r0 = COPY [[X]]
     ; SOFT-DAG: $r1 = COPY [[Y]]
-    ; SOFT-AEABI: BL &__aeabi_idivmod, {{.*}}, implicit $r0, implicit $r1, implicit-def $r0, implicit-def $r1
+    ; ARM-AEABI: BL &__aeabi_idivmod, {{.*}}, implicit $r0, implicit $r1, implicit-def $r0, implicit-def $r1
+    ; THUMB-AEABI: tBL 14, $noreg, &__aeabi_idivmod, {{.*}}, implicit $r0, implicit $r1, implicit-def $r0, implicit-def $r1
     ; SOFT-AEABI: [[R:%[0-9]+]]:_(s32) = COPY $r1
-    ; SOFT-DEFAULT: BL &__modsi3, {{.*}}, implicit $r0, implicit $r1, implicit-def $r0
+    ; ARM-DEFAULT: BL &__modsi3, {{.*}}, implicit $r0, implicit $r1, implicit-def $r0
+    ; THUMB-DEFAULT: tBL 14, $noreg, &__modsi3, {{.*}}, implicit $r0, implicit $r1, implicit-def $r0
     ; SOFT-DEFAULT: [[R:%[0-9]+]]:_(s32) = COPY $r0
     ; SOFT: ADJCALLSTACKUP
     ; SOFT-NOT: G_SREM
@@ -370,9 +382,11 @@ body:             |
     ; SOFT: ADJCALLSTACKDOWN
     ; SOFT-DAG: $r0 = COPY [[X]]
     ; SOFT-DAG: $r1 = COPY [[Y]]
-    ; SOFT-AEABI: BL &__aeabi_uidivmod, {{.*}}, implicit $r0, implicit $r1, implicit-def $r0, implicit-def $r1
+    ; ARM-AEABI: BL &__aeabi_uidivmod, {{.*}}, implicit $r0, implicit $r1, implicit-def $r0, implicit-def $r1
+    ; THUMB-AEABI: tBL 14, $noreg, &__aeabi_uidivmod, {{.*}}, implicit $r0, implicit $r1, implicit-def $r0, implicit-def $r1
     ; SOFT-AEABI: [[R:%[0-9]+]]:_(s32) = COPY $r1
-    ; SOFT-DEFAULT: BL &__umodsi3, {{.*}}, implicit $r0, implicit $r1, implicit-def $r0
+    ; ARM-DEFAULT: BL &__umodsi3, {{.*}}, implicit $r0, implicit $r1, implicit-def $r0
+    ; THUMB-DEFAULT: tBL 14, $noreg, &__umodsi3, {{.*}}, implicit $r0, implicit $r1, implicit-def $r0
     ; SOFT-DEFAULT: [[R:%[0-9]+]]:_(s32) = COPY $r0
     ; SOFT: ADJCALLSTACKUP
     ; SOFT-NOT: G_UREM
@@ -423,9 +437,11 @@ body:             |
     ; SOFT: ADJCALLSTACKDOWN
     ; SOFT-DAG: $r0 = COPY [[X32]]
     ; SOFT-DAG: $r1 = COPY [[Y32]]
-    ; SOFT-AEABI: BL &__aeabi_idivmod, {{.*}}, implicit $r0, implicit $r1, implicit-def $r0
+    ; ARM-AEABI: BL &__aeabi_idivmod, {{.*}}, implicit $r0, implicit $r1, implicit-def $r0
+    ; THUMB-AEABI: tBL 14, $noreg, &__aeabi_idivmod, {{.*}}, implicit $r0, implicit $r1, implicit-def $r0
     ; SOFT-AEABI: [[R32:%[0-9]+]]:_(s32) = COPY $r1
-    ; SOFT-DEFAULT: BL &__modsi3, {{.*}}, implicit $r0, implicit $r1, implicit-def $r0
+    ; ARM-DEFAULT: BL &__modsi3, {{.*}}, implicit $r0, implicit $r1, implicit-def $r0
+    ; THUMB-DEFAULT: tBL 14, $noreg, &__modsi3, {{.*}}, implicit $r0, implicit $r1, implicit-def $r0
     ; SOFT-DEFAULT: [[R32:%[0-9]+]]:_(s32) = COPY $r0
     ; SOFT: ADJCALLSTACKUP
     ; SOFT-NOT: G_SREM
@@ -477,9 +493,11 @@ body:             |
     ; SOFT: ADJCALLSTACKDOWN
     ; SOFT-DAG: $r0 = COPY [[X32]]
     ; SOFT-DAG: $r1 = COPY [[Y32]]
-    ; SOFT-AEABI: BL &__aeabi_uidivmod, {{.*}}, implicit $r0, implicit $r1, implicit-def $r0
+    ; ARM-AEABI: BL &__aeabi_uidivmod, {{.*}}, implicit $r0, implicit $r1, implicit-def $r0
+    ; THUMB-AEABI: tBL 14, $noreg, &__aeabi_uidivmod, {{.*}}, implicit $r0, implicit $r1, implicit-def $r0
     ; SOFT-AEABI: [[R32:%[0-9]+]]:_(s32) = COPY $r1
-    ; SOFT-DEFAULT: BL &__umodsi3, {{.*}}, implicit $r0, implicit $r1, implicit-def $r0
+    ; ARM-DEFAULT: BL &__umodsi3, {{.*}}, implicit $r0, implicit $r1, implicit-def $r0
+    ; THUMB-DEFAULT: tBL 14, $noreg, &__umodsi3, {{.*}}, implicit $r0, implicit $r1, implicit-def $r0
     ; SOFT-DEFAULT: [[R32:%[0-9]+]]:_(s32) = COPY $r0
     ; SOFT: ADJCALLSTACKUP
     ; SOFT-NOT: G_UREM
@@ -533,9 +551,11 @@ body:             |
     ; SOFT: ADJCALLSTACKDOWN
     ; SOFT-DAG: $r0 = COPY [[X32]]
     ; SOFT-DAG: $r1 = COPY [[Y32]]
-    ; SOFT-AEABI: BL &__aeabi_idivmod, {{.*}}, implicit $r0, implicit $r1, implicit-def $r0
+    ; ARM-AEABI: BL &__aeabi_idivmod, {{.*}}, implicit $r0, implicit $r1, implicit-def $r0
+    ; THUMB-AEABI: tBL 14, $noreg, &__aeabi_idivmod, {{.*}}, implicit $r0, implicit $r1, implicit-def $r0
     ; SOFT-AEABI: [[R32:%[0-9]+]]:_(s32) = COPY $r1
-    ; SOFT-DEFAULT: BL &__modsi3, {{.*}}, implicit $r0, implicit $r1, implicit-def $r0
+    ; ARM-DEFAULT: BL &__modsi3, {{.*}}, implicit $r0, implicit $r1, implicit-def $r0
+    ; THUMB-DEFAULT: tBL 14, $noreg, &__modsi3, {{.*}}, implicit $r0, implicit $r1, implicit-def $r0
     ; SOFT-DEFAULT: [[R32:%[0-9]+]]:_(s32) = COPY $r0
     ; SOFT: ADJCALLSTACKUP
     ; SOFT-NOT: G_SREM
@@ -587,9 +607,11 @@ body:             |
     ; SOFT: ADJCALLSTACKDOWN
     ; SOFT-DAG: $r0 = COPY [[X32]]
     ; SOFT-DAG: $r1 = COPY [[Y32]]
-    ; SOFT-AEABI: BL &__aeabi_uidivmod, {{.*}}, implicit $r0, implicit $r1, implicit-def $r0
+    ; ARM-AEABI: BL &__aeabi_uidivmod, {{.*}}, implicit $r0, implicit $r1, implicit-def $r0
+    ; THUMB-AEABI: tBL 14, $noreg, &__aeabi_uidivmod, {{.*}}, implicit $r0, implicit $r1, implicit-def $r0
     ; SOFT-AEABI: [[R32:%[0-9]+]]:_(s32) = COPY $r1
-    ; SOFT-DEFAULT: BL &__umodsi3, {{.*}}, implicit $r0, implicit $r1, implicit-def $r0
+    ; ARM-DEFAULT: BL &__umodsi3, {{.*}}, implicit $r0, implicit $r1, implicit-def $r0
+    ; THUMB-DEFAULT: tBL 14, $noreg, &__umodsi3, {{.*}}, implicit $r0, implicit $r1, implicit-def $r0
     ; SOFT-DEFAULT: [[R32:%[0-9]+]]:_(s32) = COPY $r0
     ; SOFT: ADJCALLSTACKUP
     ; SOFT-NOT: G_UREM
diff --git a/test/CodeGen/ARM/GlobalISel/arm-legalize-fp.mir b/test/CodeGen/ARM/GlobalISel/arm-legalize-fp.mir
index 250cab78f5d1dbc87b10c39805e8d8f0fef483e9..8704790f4ccc4553480eddb43fb6e4560f04f710 100644
--- a/test/CodeGen/ARM/GlobalISel/arm-legalize-fp.mir
+++ b/test/CodeGen/ARM/GlobalISel/arm-legalize-fp.mir
@@ -1,6 +1,6 @@
-# RUN: llc -mtriple arm-linux-gnueabihf -mattr=+vfp2 -float-abi=hard -run-pass=legalizer %s -o - | FileCheck %s -check-prefix CHECK -check-prefix HARD
-# RUN: llc -mtriple arm-linux-gnueabi -mattr=+vfp2,+soft-float -float-abi=soft -run-pass=legalizer %s -o - | FileCheck %s -check-prefix CHECK -check-prefix SOFT -check-prefix SOFT-AEABI
-# RUN: llc -mtriple arm-linux-gnu -mattr=+soft-float -float-abi=soft -run-pass=legalizer %s -o - | FileCheck %s  -check-prefix CHECK -check-prefix SOFT -check-prefix SOFT-DEFAULT
+# RUN: llc -O0 -mtriple arm-linux-gnueabihf -mattr=+vfp2 -float-abi=hard -run-pass=legalizer %s -o - | FileCheck %s -check-prefix CHECK -check-prefix HARD
+# RUN: llc -O0 -mtriple arm-linux-gnueabi -mattr=+vfp2,+soft-float -float-abi=soft -run-pass=legalizer %s -o - | FileCheck %s -check-prefix CHECK -check-prefix SOFT -check-prefix SOFT-AEABI
+# RUN: llc -O0 -mtriple arm-linux-gnu -mattr=+soft-float -float-abi=soft -run-pass=legalizer %s -o - | FileCheck %s  -check-prefix CHECK -check-prefix SOFT -check-prefix SOFT-DEFAULT
 --- |
   define void @test_frem_float() { ret void }
   define void @test_frem_double() { ret void }
diff --git a/test/CodeGen/ARM/GlobalISel/arm-legalize-load-store.mir b/test/CodeGen/ARM/GlobalISel/arm-legalize-load-store.mir
index a955af9a97ffbe27fc3f54c88c7578d3d0e70f33..a3b4fece523b71bdc4aa4d3b0716422a38a98175 100644
--- a/test/CodeGen/ARM/GlobalISel/arm-legalize-load-store.mir
+++ b/test/CodeGen/ARM/GlobalISel/arm-legalize-load-store.mir
@@ -2,6 +2,8 @@
 # RUN: llc -mtriple thumb-- -mattr=+v6t2 -run-pass=legalizer %s -o - | FileCheck %s
 --- |
   define void @test_legal_loads_stores() { ret void }
+
+  define void @test_gep() { ret void }
 ...
 ---
 name:            test_legal_loads_stores
@@ -47,3 +49,28 @@ body:             |
     G_STORE %6(p0), %0(p0) :: (store 4)
     BX_RET 14, $noreg
 ...
+---
+name:            test_gep
+# CHECK-LABEL: name: test_gep
+legalized:       false
+# CHECK: legalized: true
+regBankSelected: false
+selected:        false
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+  - { id: 2, class: _ }
+body:             |
+  bb.0:
+    liveins: $r0, $r1
+
+    %0(p0) = COPY $r0
+    %1(s32) = COPY $r1
+
+    ; CHECK: {{%[0-9]+}}:_(p0) = G_GEP {{%[0-9]+}}, {{%[0-9]+}}(s32)
+    %2(p0) = G_GEP %0, %1(s32)
+
+    $r0 = COPY %2(p0)
+    BX_RET 14, $noreg, implicit $r0
+...
diff --git a/test/CodeGen/ARM/GlobalISel/arm-legalize-select.mir b/test/CodeGen/ARM/GlobalISel/arm-legalize-select.mir
new file mode 100644
index 0000000000000000000000000000000000000000..0431f92d0997622db559a6d7361b80fe474a9d53
--- /dev/null
+++ b/test/CodeGen/ARM/GlobalISel/arm-legalize-select.mir
@@ -0,0 +1,58 @@
+# RUN: llc -mtriple arm-- -run-pass=legalizer %s -o - | FileCheck %s
+# RUN: llc -mtriple thumb-- -mattr=+v6t2 -run-pass=legalizer %s -o - | FileCheck %s
+--- |
+  define void @test_select_s32() { ret void }
+  define void @test_select_ptr() { ret void }
+...
+---
+name:            test_select_s32
+# CHECK-LABEL: name: test_select_s32
+legalized:       false
+# CHECK: legalized: true
+regBankSelected: false
+selected:        false
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+  - { id: 2, class: _ }
+  - { id: 3, class: _ }
+body:             |
+  bb.0:
+    liveins: $r0, $r1, $r2
+
+    %0(s32) = COPY $r0
+    %1(s32) = COPY $r1
+    %2(s1) = G_CONSTANT i1 1
+    %3(s32) = G_SELECT %2(s1), %0, %1
+    ; G_SELECT with s32 is legal, so we should find it unchanged in the output
+    ; CHECK: {{%[0-9]+}}:_(s32) = G_SELECT {{%[0-9]+}}(s1), {{%[0-9]+}}, {{%[0-9]+}}
+    $r0 = COPY %3(s32)
+    BX_RET 14, $noreg, implicit $r0
+...
+---
+name:            test_select_ptr
+# CHECK-LABEL: name: test_select_ptr
+legalized:       false
+# CHECK: legalized: true
+regBankSelected: false
+selected:        false
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+  - { id: 2, class: _ }
+  - { id: 3, class: _ }
+body:             |
+  bb.0:
+    liveins: $r0, $r1, $r2
+
+    %0(p0) = COPY $r0
+    %1(p0) = COPY $r1
+    %2(s1) = G_CONSTANT i1 0
+    %3(p0) = G_SELECT %2(s1), %0, %1
+    ; G_SELECT with p0 is legal, so we should find it unchanged in the output
+    ; CHECK: {{%[0-9]+}}:_(p0) = G_SELECT {{%[0-9]+}}(s1), {{%[0-9]+}}, {{%[0-9]+}}
+    $r0 = COPY %3(p0)
+    BX_RET 14, $noreg, implicit $r0
+...
diff --git a/test/CodeGen/ARM/GlobalISel/arm-legalize-vfp4.mir b/test/CodeGen/ARM/GlobalISel/arm-legalize-vfp4.mir
index b9f409525f4dd76904c1f7cb09327a002d595645..5ecf9b795399940d286d6ef71fd11f1a4c25da09 100644
--- a/test/CodeGen/ARM/GlobalISel/arm-legalize-vfp4.mir
+++ b/test/CodeGen/ARM/GlobalISel/arm-legalize-vfp4.mir
@@ -96,11 +96,11 @@ body:             |
     ; SOFT-ABI: [[OFF1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
     ; SOFT-ABI: [[FI1:%[0-9]+]]:_(p0) = G_GEP [[SP1]], [[OFF1]](s32)
 
-    ; FIXME: This ought to be align 8 but ARM's call lowering hardcodes it to 0
-    ; SOFT-ABI: G_STORE [[Y0]](s32), [[FI1]](p0){{.*}}store 4 into stack, align 0)
+    ; FIXME: This ought to be align 8 but ARM's call lowering hardcodes it to 1
+    ; SOFT-ABI: G_STORE [[Y0]](s32), [[FI1]](p0){{.*}}store 4 into stack, align 1)
     ; SOFT-ABI: [[OFF2:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
     ; SOFT-ABI: [[FI2:%[0-9]+]]:_(p0) = G_GEP [[FI1]], [[OFF2]](s32)
-    ; SOFT-ABI: G_STORE [[Y1]](s32), [[FI2]](p0){{.*}}store 4 into stack + 4)
+    ; SOFT-ABI: G_STORE [[Y1]](s32), [[FI2]](p0){{.*}}store 4 into stack + 4, align 1)
     ; SOFT-ABI: BL &fma, {{.*}}, implicit $r0, implicit $r1, implicit $r2, implicit $r3, implicit-def $r0, implicit-def $r1
     ; SOFT-ABI-DAG: [[R0:%[0-9]+]]:_(s32) = COPY $r0
     ; SOFT-ABI-DAG: [[R1:%[0-9]+]]:_(s32) = COPY $r1
diff --git a/test/CodeGen/ARM/GlobalISel/arm-legalizer.mir b/test/CodeGen/ARM/GlobalISel/arm-legalizer.mir
index bee8b7bbbabd2b801c2ae01c8357f4fec3ab56f0..e74366dcbe5a8afab61171a46ee516e4e1158b4b 100644
--- a/test/CodeGen/ARM/GlobalISel/arm-legalizer.mir
+++ b/test/CodeGen/ARM/GlobalISel/arm-legalizer.mir
@@ -1,23 +1,10 @@
 # RUN: llc -mtriple arm-- -run-pass=legalizer %s -o - | FileCheck %s
 --- |
-  define void @test_lshr_s32() { ret void }
-  define void @test_ashr_s32() { ret void }
-  define void @test_shl_s32() { ret void }
-
   define void @test_load_from_stack() { ret void }
   define void @test_load_store_64() #0 { ret void }
 
-  define void @test_gep() { ret void }
-
   define void @test_constants_s64() { ret void }
 
-  define void @test_icmp_s8() { ret void }
-  define void @test_icmp_s16() { ret void }
-  define void @test_icmp_s32() { ret void }
-
-  define void @test_select_s32() { ret void }
-  define void @test_select_ptr() { ret void }
-
   define void @test_brcond() { ret void }
 
   define void @test_phi_s32() { ret void }
@@ -31,81 +18,6 @@
   attributes #0 = { "target-features"="+vfp2" }
 ...
 ---
-name:            test_lshr_s32
-# CHECK-LABEL: name: test_lshr_s32
-legalized:       false
-# CHECK: legalized: true
-regBankSelected: false
-selected:        false
-tracksRegLiveness: true
-registers:
-  - { id: 0, class: _ }
-  - { id: 1, class: _ }
-  - { id: 2, class: _ }
-body:             |
-  bb.0:
-    liveins: $r0, $r1
-
-    %0(s32) = COPY $r0
-    %1(s32) = COPY $r1
-    %2(s32) = G_LSHR %0, %1
-    ; G_LSHR with s32 is legal, so we should find it unchanged in the output
-    ; CHECK: {{%[0-9]+}}:_(s32) = G_LSHR {{%[0-9]+, %[0-9]+}}
-    $r0 = COPY %2(s32)
-    BX_RET 14, $noreg, implicit $r0
-
-...
----
-name:            test_ashr_s32
-# CHECK-LABEL: name: test_ashr_s32
-legalized:       false
-# CHECK: legalized: true
-regBankSelected: false
-selected:        false
-tracksRegLiveness: true
-registers:
-  - { id: 0, class: _ }
-  - { id: 1, class: _ }
-  - { id: 2, class: _ }
-body:             |
-  bb.0:
-    liveins: $r0, $r1
-
-    %0(s32) = COPY $r0
-    %1(s32) = COPY $r1
-    %2(s32) = G_ASHR %0, %1
-    ; G_ASHR with s32 is legal, so we should find it unchanged in the output
-    ; CHECK: {{%[0-9]+}}:_(s32) = G_ASHR {{%[0-9]+, %[0-9]+}}
-    $r0 = COPY %2(s32)
-    BX_RET 14, $noreg, implicit $r0
-
-...
----
-name:            test_shl_s32
-# CHECK-LABEL: name: test_shl_s32
-legalized:       false
-# CHECK: legalized: true
-regBankSelected: false
-selected:        false
-tracksRegLiveness: true
-registers:
-  - { id: 0, class: _ }
-  - { id: 1, class: _ }
-  - { id: 2, class: _ }
-body:             |
-  bb.0:
-    liveins: $r0, $r1
-
-    %0(s32) = COPY $r0
-    %1(s32) = COPY $r1
-    %2(s32) = G_SHL %0, %1
-    ; G_SHL with s32 is legal, so we should find it unchanged in the output
-    ; CHECK: {{%[0-9]+}}:_(s32) = G_SHL {{%[0-9]+, %[0-9]+}}
-    $r0 = COPY %2(s32)
-    BX_RET 14, $noreg, implicit $r0
-
-...
----
 name:            test_load_from_stack
 # CHECK-LABEL: name: test_load_from_stack
 legalized:       false
@@ -164,31 +76,6 @@ body:             |
     BX_RET 14, $noreg
 ...
 ---
-name:            test_gep
-# CHECK-LABEL: name: test_gep
-legalized:       false
-# CHECK: legalized: true
-regBankSelected: false
-selected:        false
-tracksRegLiveness: true
-registers:
-  - { id: 0, class: _ }
-  - { id: 1, class: _ }
-  - { id: 2, class: _ }
-body:             |
-  bb.0:
-    liveins: $r0, $r1
-
-    %0(p0) = COPY $r0
-    %1(s32) = COPY $r1
-
-    ; CHECK: {{%[0-9]+}}:_(p0) = G_GEP {{%[0-9]+}}, {{%[0-9]+}}(s32)
-    %2(p0) = G_GEP %0, %1(s32)
-
-    $r0 = COPY %2(p0)
-    BX_RET 14, $noreg, implicit $r0
-...
----
 name:            test_constants_s64
 # CHECK-LABEL: name: test_constants_s64
 legalized:       false
@@ -218,146 +105,6 @@ body:             |
     BX_RET 14, $noreg
 ...
 ---
-name:            test_icmp_s8
-# CHECK-LABEL: name: test_icmp_s8
-legalized:       false
-# CHECK: legalized: true
-regBankSelected: false
-selected:        false
-tracksRegLiveness: true
-registers:
-  - { id: 0, class: _ }
-  - { id: 1, class: _ }
-  - { id: 2, class: _ }
-  - { id: 3, class: _ }
-  - { id: 4, class: _ }
-  - { id: 5, class: _ }
-body:             |
-  bb.0:
-    liveins: $r0, $r1
-
-    %0(p0) = COPY $r0
-    %1(s8) = G_LOAD %0 :: (load 1)
-    %2(p0) = COPY $r1
-    %3(s8) = G_LOAD %2 :: (load 1)
-    %4(s1) = G_ICMP intpred(ne), %1(s8), %3
-    ; G_ICMP with s8 should widen
-    ; CHECK: {{%[0-9]+}}:_(s1) = G_ICMP intpred(ne), {{%[0-9]+}}(s32), {{%[0-9]+}}
-    ; CHECK-NOT: {{%[0-9]+}}:_(s1) = G_ICMP intpred(ne), {{%[0-9]+}}(s8), {{%[0-9]+}}
-    %5(s32) = G_ZEXT %4(s1)
-    $r0 = COPY %5(s32)
-    BX_RET 14, $noreg, implicit $r0
-...
----
-name:            test_icmp_s16
-# CHECK-LABEL: name: test_icmp_s16
-legalized:       false
-# CHECK: legalized: true
-regBankSelected: false
-selected:        false
-tracksRegLiveness: true
-registers:
-  - { id: 0, class: _ }
-  - { id: 1, class: _ }
-  - { id: 2, class: _ }
-  - { id: 3, class: _ }
-  - { id: 4, class: _ }
-  - { id: 5, class: _ }
-body:             |
-  bb.0:
-    liveins: $r0, $r1
-
-    %0(p0) = COPY $r0
-    %1(s16) = G_LOAD %0 :: (load 2)
-    %2(p0) = COPY $r1
-    %3(s16) = G_LOAD %2 :: (load 2)
-    %4(s1) = G_ICMP intpred(slt), %1(s16), %3
-    ; G_ICMP with s16 should widen
-    ; CHECK: {{%[0-9]+}}:_(s1) = G_ICMP intpred(slt), {{%[0-9]+}}(s32), {{%[0-9]+}}
-    ; CHECK-NOT: {{%[0-9]+}}:_(s1) = G_ICMP intpred(slt), {{%[0-9]+}}(s16), {{%[0-9]+}}
-    %5(s32) = G_ZEXT %4(s1)
-    $r0 = COPY %5(s32)
-    BX_RET 14, $noreg, implicit $r0
-...
----
-name:            test_icmp_s32
-# CHECK-LABEL: name: test_icmp_s32
-legalized:       false
-# CHECK: legalized: true
-regBankSelected: false
-selected:        false
-tracksRegLiveness: true
-registers:
-  - { id: 0, class: _ }
-  - { id: 1, class: _ }
-  - { id: 2, class: _ }
-  - { id: 3, class: _ }
-body:             |
-  bb.0:
-    liveins: $r0, $r1
-
-    %0(s32) = COPY $r0
-    %1(s32) = COPY $r1
-    %2(s1) = G_ICMP intpred(eq), %0(s32), %1
-    ; G_ICMP with s32 is legal, so we should find it unchanged in the output
-    ; CHECK: {{%[0-9]+}}:_(s1) = G_ICMP intpred(eq), {{%[0-9]+}}(s32), {{%[0-9]+}}
-    %3(s32) = G_ZEXT %2(s1)
-    $r0 = COPY %3(s32)
-    BX_RET 14, $noreg, implicit $r0
-...
----
-name:            test_select_s32
-# CHECK-LABEL: name: test_select_s32
-legalized:       false
-# CHECK: legalized: true
-regBankSelected: false
-selected:        false
-tracksRegLiveness: true
-registers:
-  - { id: 0, class: _ }
-  - { id: 1, class: _ }
-  - { id: 2, class: _ }
-  - { id: 3, class: _ }
-body:             |
-  bb.0:
-    liveins: $r0, $r1, $r2
-
-    %0(s32) = COPY $r0
-    %1(s32) = COPY $r1
-    %2(s1) = G_CONSTANT i1 1
-    %3(s32) = G_SELECT %2(s1), %0, %1
-    ; G_SELECT with s32 is legal, so we should find it unchanged in the output
-    ; CHECK: {{%[0-9]+}}:_(s32) = G_SELECT {{%[0-9]+}}(s1), {{%[0-9]+}}, {{%[0-9]+}}
-    $r0 = COPY %3(s32)
-    BX_RET 14, $noreg, implicit $r0
-...
----
-name:            test_select_ptr
-# CHECK-LABEL: name: test_select_ptr
-legalized:       false
-# CHECK: legalized: true
-regBankSelected: false
-selected:        false
-tracksRegLiveness: true
-registers:
-  - { id: 0, class: _ }
-  - { id: 1, class: _ }
-  - { id: 2, class: _ }
-  - { id: 3, class: _ }
-body:             |
-  bb.0:
-    liveins: $r0, $r1, $r2
-
-    %0(p0) = COPY $r0
-    %1(p0) = COPY $r1
-    %2(s1) = G_CONSTANT i1 0
-    %3(p0) = G_SELECT %2(s1), %0, %1
-    ; G_SELECT with p0 is legal, so we should find it unchanged in the output
-    ; CHECK: {{%[0-9]+}}:_(p0) = G_SELECT {{%[0-9]+}}(s1), {{%[0-9]+}}, {{%[0-9]+}}
-    $r0 = COPY %3(p0)
-    BX_RET 14, $noreg, implicit $r0
-...
----
 name:            test_brcond
 # CHECK-LABEL: name: test_brcond
 legalized:       false
diff --git a/test/CodeGen/ARM/GlobalISel/arm-param-lowering.ll b/test/CodeGen/ARM/GlobalISel/arm-param-lowering.ll
index 3c3da0edd53e7884dace11c30146034d1850b8b7..cff38c0339c49bf5698e7f1ae2e45009a8961185 100644
--- a/test/CodeGen/ARM/GlobalISel/arm-param-lowering.ll
+++ b/test/CodeGen/ARM/GlobalISel/arm-param-lowering.ll
@@ -1,7 +1,7 @@
-; RUN: llc -mtriple arm-unknown -mattr=+vfp2,+v4t -global-isel -stop-after=irtranslator -verify-machineinstrs %s -o - | FileCheck %s -check-prefix=CHECK -check-prefix=ARM -check-prefix=LITTLE
-; RUN: llc -mtriple armeb-unknown -mattr=+vfp2,+v4t -global-isel -global-isel-abort=0 -stop-after=irtranslator -verify-machineinstrs %s -o - | FileCheck %s -check-prefix=CHECK -check-prefix=ARM -check-prefix=BIG
+; RUN: llc -O0 -mtriple arm-unknown -mattr=+vfp2,+v4t -global-isel -stop-after=irtranslator -verify-machineinstrs %s -o - | FileCheck %s -check-prefix=CHECK -check-prefix=ARM -check-prefix=LITTLE
+; RUN: llc -O0 -mtriple armeb-unknown -mattr=+vfp2,+v4t -global-isel -global-isel-abort=0 -stop-after=irtranslator -verify-machineinstrs %s -o - | FileCheck %s -check-prefix=CHECK -check-prefix=ARM -check-prefix=BIG
 ; XFAIL: armeb
-; RUN: llc -mtriple thumb-unknown -mattr=+vfp2,+v6t2 -global-isel -stop-after=irtranslator -verify-machineinstrs %s -o - | FileCheck %s -check-prefix=CHECK -check-prefix=LITTLE -check-prefix=THUMB
+; RUN: llc -O0 -mtriple thumb-unknown -mattr=+vfp2,+v6t2 -global-isel -stop-after=irtranslator -verify-machineinstrs %s -o - | FileCheck %s -check-prefix=CHECK -check-prefix=LITTLE -check-prefix=THUMB
 
 declare arm_aapcscc i32* @simple_reg_params_target(i32, i32*)
 
diff --git a/test/CodeGen/ARM/GlobalISel/arm-regbankselect.mir b/test/CodeGen/ARM/GlobalISel/arm-regbankselect.mir
index 281218192a18826c0b1322fbe3394089693c24ac..c218df33ab61e90e4304243298eba64fa1c3b2f3 100644
--- a/test/CodeGen/ARM/GlobalISel/arm-regbankselect.mir
+++ b/test/CodeGen/ARM/GlobalISel/arm-regbankselect.mir
@@ -474,7 +474,7 @@ fixedStack:
 body:             |
   bb.0:
     %0(p0) = G_FRAME_INDEX %fixed-stack.0
-    %1(s32) = G_LOAD %0(p0) :: (load 4 from %fixed-stack.0, align 0)
+    %1(s32) = G_LOAD %0(p0) :: (load 4 from %fixed-stack.0, align 4)
 
     %2(p0) = COPY $sp
     %3(s32) = G_CONSTANT i32 8
@@ -521,7 +521,7 @@ registers:
   - { id: 0, class: _ }
 body:             |
   bb.0:
-    %0(s32) = G_CONSTANT 42
+    %0(s32) = G_CONSTANT i32 42
     $r0 = COPY %0(s32)
     BX_RET 14, $noreg, implicit $r0
 ...
diff --git a/test/CodeGen/ARM/GlobalISel/arm-unsupported.ll b/test/CodeGen/ARM/GlobalISel/arm-unsupported.ll
index 501153343baba97e1468864c60bc6bd3011efd76..e7df31210372ab565ab8531a32f60005b4837f50 100644
--- a/test/CodeGen/ARM/GlobalISel/arm-unsupported.ll
+++ b/test/CodeGen/ARM/GlobalISel/arm-unsupported.ll
@@ -83,13 +83,6 @@ define void @test_vararg_definition(i32 %a, ...) {
   ret void
 }
 
-define void @test_vararg_call(i32 %a) {
-; CHECK: remark: {{.*}} unable to translate instruction: call
-; CHECK-LABEL: warning: Instruction selection used fallback path for test_vararg_call
-  call void(i32, ...) @test_vararg_definition(i32 %a, i32 %a, i32 %a)
-  ret void
-}
-
 define i32 @test_thumb(i32 %a) #0 {
 ; CHECK: remark: {{.*}} unable to lower arguments: i32 (i32)*
 ; CHECK-LABEL: warning: Instruction selection used fallback path for test_thumb
diff --git a/test/CodeGen/ARM/GlobalISel/irtranslator-varargs-lowering.ll b/test/CodeGen/ARM/GlobalISel/irtranslator-varargs-lowering.ll
new file mode 100644
index 0000000000000000000000000000000000000000..351594d1fc270ec3dbd878297a84969a895bf004
--- /dev/null
+++ b/test/CodeGen/ARM/GlobalISel/irtranslator-varargs-lowering.ll
@@ -0,0 +1,86 @@
+; RUN: llc -mtriple arm-unknown -mattr=+vfp2,+v6t2 -global-isel -stop-after=irtranslator -verify-machineinstrs %s -o - | FileCheck %s -check-prefix=CHECK -check-prefix=ARM
+; RUN: llc -mtriple thumb-unknown -mattr=+vfp2,+v6t2 -global-isel -stop-after=irtranslator -verify-machineinstrs %s -o - | FileCheck %s -check-prefix=CHECK -check-prefix=THUMB
+
+declare arm_aapcscc i32 @int_varargs_target(i32, ...)
+
+define arm_aapcscc i32 @test_call_to_varargs_with_ints(i32 *%a, i32 %b) {
+; CHECK-LABEL: name: test_call_to_varargs_with_ints
+; CHECK-DAG: [[AVREG:%[0-9]+]]:_(p0) = COPY $r0
+; CHECK-DAG: [[BVREG:%[0-9]+]]:_(s32) = COPY $r1
+; CHECK: ADJCALLSTACKDOWN 8, 0, 14, $noreg, implicit-def $sp, implicit $sp
+; CHECK-DAG: $r0 = COPY [[BVREG]]
+; CHECK-DAG: $r1 = COPY [[AVREG]]
+; CHECK-DAG: $r2 = COPY [[BVREG]]
+; CHECK-DAG: $r3 = COPY [[AVREG]]
+; CHECK: [[SP1:%[0-9]+]]:_(p0) = COPY $sp
+; CHECK: [[OFF1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+; CHECK: [[FI1:%[0-9]+]]:_(p0) = G_GEP [[SP1]], [[OFF1]](s32)
+; CHECK: G_STORE [[BVREG]](s32), [[FI1]](p0){{.*}}store 4
+; CHECK: [[SP2:%[0-9]+]]:_(p0) = COPY $sp
+; CHECK: [[OFF2:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+; CHECK: [[FI2:%[0-9]+]]:_(p0) = G_GEP [[SP2]], [[OFF2]](s32)
+; CHECK: G_STORE [[AVREG]](p0), [[FI2]](p0){{.*}}store 4
+; ARM: BL @int_varargs_target, csr_aapcs, implicit-def $lr, implicit $sp, implicit $r0, implicit $r1, implicit $r2, implicit $r3, implicit-def $r0
+; THUMB: tBL 14, $noreg, @int_varargs_target, csr_aapcs, implicit-def $lr, implicit $sp, implicit $r0, implicit $r1, implicit $r2, implicit $r3, implicit-def $r0
+; CHECK: [[RVREG:%[0-9]+]]:_(s32) = COPY $r0
+; CHECK: ADJCALLSTACKUP 8, 0, 14, $noreg, implicit-def $sp, implicit $sp
+; CHECK: $r0 = COPY [[RVREG]]
+; ARM: BX_RET 14, $noreg, implicit $r0
+; THUMB: tBX_RET 14, $noreg, implicit $r0
+entry:
+  %r = notail call arm_aapcscc i32(i32, ...) @int_varargs_target(i32 %b, i32 *%a, i32 %b, i32 *%a, i32 %b, i32 *%a)
+  ret i32 %r
+}
+
+declare arm_aapcs_vfpcc float @float_varargs_target(float, double, ...)
+
+define arm_aapcs_vfpcc float @test_call_to_varargs_with_floats(float %a, double %b) {
+; CHECK-LABEL: name: test_call_to_varargs_with_floats
+; CHECK-DAG: [[AVREG:%[0-9]+]]:_(s32) = COPY $s0
+; CHECK-DAG: [[BVREG:%[0-9]+]]:_(s64) = COPY $d1
+; CHECK: ADJCALLSTACKDOWN 8, 0, 14, $noreg, implicit-def $sp, implicit $sp
+; CHECK-DAG: $r0 = COPY [[AVREG]]
+; CHECK-DAG: [[B1:%[0-9]+]]:_(s32), [[B2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[BVREG]](s64)
+; CHECK-DAG: $r2 = COPY [[B1]]
+; CHECK-DAG: $r3 = COPY [[B2]]
+; CHECK: [[SP1:%[0-9]+]]:_(p0) = COPY $sp
+; CHECK: [[OFF1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+; CHECK: [[FI1:%[0-9]+]]:_(p0) = G_GEP [[SP1]], [[OFF1]](s32)
+; CHECK: G_STORE [[BVREG]](s64), [[FI1]](p0){{.*}}store 8
+; ARM: BL @float_varargs_target, csr_aapcs, implicit-def $lr, implicit $sp, implicit $r0, implicit $r2, implicit $r3, implicit-def $r0
+; THUMB: tBL 14, $noreg, @float_varargs_target, csr_aapcs, implicit-def $lr, implicit $sp, implicit $r0, implicit $r2, implicit $r3, implicit-def $r0
+; CHECK: [[RVREG:%[0-9]+]]:_(s32) = COPY $r0
+; CHECK: ADJCALLSTACKUP 8, 0, 14, $noreg, implicit-def $sp, implicit $sp
+; CHECK: $s0 = COPY [[RVREG]]
+; ARM: BX_RET 14, $noreg, implicit $s0
+; THUMB: tBX_RET 14, $noreg, implicit $s0
+entry:
+  %r = notail call arm_aapcs_vfpcc float(float, double, ...) @float_varargs_target(float %a, double %b, double %b)
+  ret float %r
+}
+
+define arm_aapcs_vfpcc float @test_indirect_call_to_varargs(float (float, double, ...) *%fptr, float %a, double %b) {
+; CHECK-LABEL: name: test_indirect_call_to_varargs
+; CHECK-DAG: [[FPTRVREG:%[0-9]+]]:gpr(p0) = COPY $r0
+; CHECK-DAG: [[AVREG:%[0-9]+]]:_(s32) = COPY $s0
+; CHECK-DAG: [[BVREG:%[0-9]+]]:_(s64) = COPY $d1
+; CHECK: ADJCALLSTACKDOWN 8, 0, 14, $noreg, implicit-def $sp, implicit $sp
+; CHECK-DAG: $r0 = COPY [[AVREG]]
+; CHECK-DAG: [[B1:%[0-9]+]]:_(s32), [[B2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[BVREG]](s64)
+; CHECK-DAG: $r2 = COPY [[B1]]
+; CHECK-DAG: $r3 = COPY [[B2]]
+; CHECK: [[SP1:%[0-9]+]]:_(p0) = COPY $sp
+; CHECK: [[OFF1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+; CHECK: [[FI1:%[0-9]+]]:_(p0) = G_GEP [[SP1]], [[OFF1]](s32)
+; CHECK: G_STORE [[BVREG]](s64), [[FI1]](p0){{.*}}store 8
+; ARM: BLX [[FPTRVREG]](p0), csr_aapcs, implicit-def $lr, implicit $sp, implicit $r0, implicit $r2, implicit $r3, implicit-def $r0
+; THUMB: tBLXr 14, $noreg, [[FPTRVREG]](p0), csr_aapcs, implicit-def $lr, implicit $sp, implicit $r0, implicit $r2, implicit $r3, implicit-def $r0
+; CHECK: [[RVREG:%[0-9]+]]:_(s32) = COPY $r0
+; CHECK: ADJCALLSTACKUP 8, 0, 14, $noreg, implicit-def $sp, implicit $sp
+; CHECK: $s0 = COPY [[RVREG]]
+; ARM: BX_RET 14, $noreg, implicit $s0
+; THUMB: tBX_RET 14, $noreg, implicit $s0
+entry:
+  %r = notail call arm_aapcs_vfpcc float(float, double, ...) %fptr(float %a, double %b, double %b)
+  ret float %r
+}
diff --git a/test/CodeGen/ARM/GlobalISel/select-pkhbt.mir b/test/CodeGen/ARM/GlobalISel/select-pkhbt.mir
new file mode 100644
index 0000000000000000000000000000000000000000..6abee0956b97490ec8632ff455fff8cb84f932db
--- /dev/null
+++ b/test/CodeGen/ARM/GlobalISel/select-pkhbt.mir
@@ -0,0 +1,275 @@
+# RUN: llc -O0 -mtriple arm-- -mattr=+v6 -run-pass=instruction-select -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,ARM
+# RUN: llc -O0 -mtriple thumb-- -mattr=+v6t2,+dsp -run-pass=instruction-select -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,THUMB
+--- |
+  define void @test_pkhbt() { ret void }
+  define void @test_pkhbt_commutative() { ret void }
+  define void @test_pkhbt_imm16_31() { ret void }
+  define void @test_pkhbt_unshifted() { ret void }
+
+  define void @test_pkhtb_imm16() { ret void }
+  define void @test_pkhtb_imm1_15() { ret void }
+...
+---
+name:            test_pkhbt
+# CHECK-LABEL: name: test_pkhbt
+legalized:       true
+regBankSelected: true
+selected:        false
+# CHECK: selected: true
+registers:
+  - { id: 0, class: gprb }
+  - { id: 1, class: gprb }
+  - { id: 2, class: gprb }
+  - { id: 3, class: gprb }
+  - { id: 4, class: gprb }
+  - { id: 5, class: gprb }
+  - { id: 6, class: gprb }
+  - { id: 7, class: gprb }
+  - { id: 8, class: gprb }
+body:             |
+  bb.0:
+    liveins: $r0, $r1
+
+    %0(s32) = COPY $r0
+    %1(s32) = COPY $r1
+    ; ARM-DAG: [[VREGX:%[0-9]+]]:gprnopc = COPY $r0
+    ; ARM-DAG: [[VREGY:%[0-9]+]]:gprnopc = COPY $r1
+    ; THUMB-DAG: [[VREGX:%[0-9]+]]:rgpr = COPY $r0
+    ; THUMB-DAG: [[VREGY:%[0-9]+]]:rgpr = COPY $r1
+
+    %2(s32) = G_CONSTANT i32 65535 ; 0xFFFF
+    %3(s32) = G_AND %0, %2
+
+    %4(s32) = G_CONSTANT i32 7
+    %5(s32) = G_SHL %1, %4
+    %6(s32) = G_CONSTANT i32 4294901760 ; 0xFFFF0000
+    %7(s32) = G_AND %5, %6
+
+    %8(s32) = G_OR %3, %7
+    ; ARM: [[VREGR:%[0-9]+]]:gprnopc = PKHBT [[VREGX]], [[VREGY]], 7, 14, $noreg
+    ; THUMB: [[VREGR:%[0-9]+]]:rgpr = t2PKHBT [[VREGX]], [[VREGY]], 7, 14, $noreg
+
+    $r0 = COPY %8(s32)
+    ; CHECK: $r0 = COPY [[VREGR]]
+
+    BX_RET 14, $noreg, implicit $r0
+    ; CHECK: BX_RET 14, $noreg, implicit $r0
+...
+---
+name:            test_pkhbt_commutative
+# CHECK-LABEL: name: test_pkhbt_commutative
+legalized:       true
+regBankSelected: true
+selected:        false
+# CHECK: selected: true
+registers:
+  - { id: 0, class: gprb }
+  - { id: 1, class: gprb }
+  - { id: 2, class: gprb }
+  - { id: 3, class: gprb }
+  - { id: 4, class: gprb }
+  - { id: 5, class: gprb }
+  - { id: 6, class: gprb }
+  - { id: 7, class: gprb }
+  - { id: 8, class: gprb }
+body:             |
+  bb.0:
+    liveins: $r0, $r1
+
+    %0(s32) = COPY $r0
+    %1(s32) = COPY $r1
+    ; ARM-DAG: [[VREGX:%[0-9]+]]:gprnopc = COPY $r0
+    ; ARM-DAG: [[VREGY:%[0-9]+]]:gprnopc = COPY $r1
+    ; THUMB-DAG: [[VREGX:%[0-9]+]]:rgpr = COPY $r0
+    ; THUMB-DAG: [[VREGY:%[0-9]+]]:rgpr = COPY $r1
+
+    %2(s32) = G_CONSTANT i32 65535 ; 0xFFFF
+    %3(s32) = G_AND %0, %2
+
+    %4(s32) = G_CONSTANT i32 7
+    %5(s32) = G_SHL %1, %4
+    %6(s32) = G_CONSTANT i32 4294901760 ; 0xFFFF0000
+    %7(s32) = G_AND %5, %6
+
+    %8(s32) = G_OR %7, %3
+    ; ARM: [[VREGR:%[0-9]+]]:gprnopc = PKHBT [[VREGX]], [[VREGY]], 7, 14, $noreg
+    ; THUMB: [[VREGR:%[0-9]+]]:rgpr = t2PKHBT [[VREGX]], [[VREGY]], 7, 14, $noreg
+
+    $r0 = COPY %8(s32)
+    ; CHECK: $r0 = COPY [[VREGR]]
+
+    BX_RET 14, $noreg, implicit $r0
+    ; CHECK: BX_RET 14, $noreg, implicit $r0
+...
+---
+name:            test_pkhbt_imm16_31
+# CHECK-LABEL: name: test_pkhbt_imm16_31
+legalized:       true
+regBankSelected: true
+selected:        false
+# CHECK: selected: true
+registers:
+  - { id: 0, class: gprb }
+  - { id: 1, class: gprb }
+  - { id: 2, class: gprb }
+  - { id: 3, class: gprb }
+  - { id: 4, class: gprb }
+  - { id: 5, class: gprb }
+  - { id: 6, class: gprb }
+body:             |
+  bb.0:
+    liveins: $r0, $r1
+
+    %0(s32) = COPY $r0
+    %1(s32) = COPY $r1
+    ; ARM-DAG: [[VREGX:%[0-9]+]]:gprnopc = COPY $r0
+    ; ARM-DAG: [[VREGY:%[0-9]+]]:gprnopc = COPY $r1
+    ; THUMB-DAG: [[VREGX:%[0-9]+]]:rgpr = COPY $r0
+    ; THUMB-DAG: [[VREGY:%[0-9]+]]:rgpr = COPY $r1
+
+    %2(s32) = G_CONSTANT i32 65535 ; 0xFFFF
+    %3(s32) = G_AND %0, %2
+
+    %4(s32) = G_CONSTANT i32 17
+    %5(s32) = G_SHL %1, %4
+
+    %6(s32) = G_OR %3, %5
+    ; ARM: [[VREGR:%[0-9]+]]:gprnopc = PKHBT [[VREGX]], [[VREGY]], 17, 14, $noreg
+    ; THUMB: [[VREGR:%[0-9]+]]:rgpr = t2PKHBT [[VREGX]], [[VREGY]], 17, 14, $noreg
+
+    $r0 = COPY %6(s32)
+    ; CHECK: $r0 = COPY [[VREGR]]
+
+    BX_RET 14, $noreg, implicit $r0
+    ; CHECK: BX_RET 14, $noreg, implicit $r0
+...
+---
+name:            test_pkhbt_unshifted
+# CHECK-LABEL: name: test_pkhbt_unshifted
+legalized:       true
+regBankSelected: true
+selected:        false
+# CHECK: selected: true
+registers:
+  - { id: 0, class: gprb }
+  - { id: 1, class: gprb }
+  - { id: 2, class: gprb }
+  - { id: 3, class: gprb }
+  - { id: 4, class: gprb }
+  - { id: 5, class: gprb }
+  - { id: 6, class: gprb }
+body:             |
+  bb.0:
+    liveins: $r0, $r1
+
+    %0(s32) = COPY $r0
+    %1(s32) = COPY $r1
+    ; ARM-DAG: [[VREGX:%[0-9]+]]:gprnopc = COPY $r0
+    ; ARM-DAG: [[VREGY:%[0-9]+]]:gprnopc = COPY $r1
+    ; THUMB-DAG: [[VREGX:%[0-9]+]]:rgpr = COPY $r0
+    ; THUMB-DAG: [[VREGY:%[0-9]+]]:rgpr = COPY $r1
+
+    %2(s32) = G_CONSTANT i32 65535 ; 0xFFFF
+    %3(s32) = G_AND %0, %2
+
+    %4(s32) = G_CONSTANT i32 4294901760 ; 0xFFFF0000
+    %5(s32) = G_AND %1, %4
+
+    %6(s32) = G_OR %3, %5
+    ; ARM: [[VREGR:%[0-9]+]]:gprnopc = PKHBT [[VREGX]], [[VREGY]], 0, 14, $noreg
+    ; THUMB: [[VREGR:%[0-9]+]]:rgpr = t2PKHBT [[VREGX]], [[VREGY]], 0, 14, $noreg
+
+    $r0 = COPY %6(s32)
+    ; CHECK: $r0 = COPY [[VREGR]]
+
+    BX_RET 14, $noreg, implicit $r0
+    ; CHECK: BX_RET 14, $noreg, implicit $r0
+...
+---
+name:            test_pkhtb_imm16
+# CHECK-LABEL: name: test_pkhtb_imm16
+legalized:       true
+regBankSelected: true
+selected:        false
+# CHECK: selected: true
+registers:
+  - { id: 0, class: gprb }
+  - { id: 1, class: gprb }
+  - { id: 2, class: gprb }
+  - { id: 3, class: gprb }
+  - { id: 4, class: gprb }
+  - { id: 5, class: gprb }
+  - { id: 6, class: gprb }
+body:             |
+  bb.0:
+    liveins: $r0, $r1
+
+    %0(s32) = COPY $r0
+    %1(s32) = COPY $r1
+    ; ARM-DAG: [[VREGX:%[0-9]+]]:gprnopc = COPY $r0
+    ; ARM-DAG: [[VREGY:%[0-9]+]]:gprnopc = COPY $r1
+    ; THUMB-DAG: [[VREGX:%[0-9]+]]:rgpr = COPY $r0
+    ; THUMB-DAG: [[VREGY:%[0-9]+]]:rgpr = COPY $r1
+
+    %2(s32) = G_CONSTANT i32 4294901760 ; 0xFFFF0000
+    %3(s32) = G_AND %0, %2
+
+    %4(s32) = G_CONSTANT i32 16
+    %5(s32) = G_LSHR %1, %4
+
+    %6(s32) = G_OR %3, %5
+    ; ARM: [[VREGR:%[0-9]+]]:gprnopc = PKHTB [[VREGX]], [[VREGY]], 16, 14, $noreg
+    ; THUMB: [[VREGR:%[0-9]+]]:rgpr = t2PKHTB [[VREGX]], [[VREGY]], 16, 14, $noreg
+
+    $r0 = COPY %6(s32)
+    ; CHECK: $r0 = COPY [[VREGR]]
+
+    BX_RET 14, $noreg, implicit $r0
+    ; CHECK: BX_RET 14, $noreg, implicit $r0
+...
+---
+name:            test_pkhtb_imm1_15
+# CHECK-LABEL: name: test_pkhtb_imm1_15
+legalized:       true
+regBankSelected: true
+selected:        false
+# CHECK: selected: true
+registers:
+  - { id: 0, class: gprb }
+  - { id: 1, class: gprb }
+  - { id: 2, class: gprb }
+  - { id: 3, class: gprb }
+  - { id: 4, class: gprb }
+  - { id: 5, class: gprb }
+  - { id: 6, class: gprb }
+  - { id: 7, class: gprb }
+  - { id: 8, class: gprb }
+body:             |
+  bb.0:
+    liveins: $r0, $r1
+
+    %0(s32) = COPY $r0
+    %1(s32) = COPY $r1
+    ; ARM-DAG: [[VREGX:%[0-9]+]]:gprnopc = COPY $r0
+    ; ARM-DAG: [[VREGY:%[0-9]+]]:gprnopc = COPY $r1
+    ; THUMB-DAG: [[VREGX:%[0-9]+]]:rgpr = COPY $r0
+    ; THUMB-DAG: [[VREGY:%[0-9]+]]:rgpr = COPY $r1
+
+    %2(s32) = G_CONSTANT i32 4294901760 ; 0xFFFF0000
+    %3(s32) = G_AND %0, %2
+
+    %4(s32) = G_CONSTANT i32 7
+    %5(s32) = G_LSHR %1, %4
+    %6(s32) = G_CONSTANT i32 65535 ; 0xFFFF
+    %7(s32) = G_AND %5, %6
+
+    %8(s32) = G_OR %3, %7
+    ; ARM: [[VREGR:%[0-9]+]]:gprnopc = PKHTB [[VREGX]], [[VREGY]], 7, 14, $noreg
+    ; THUMB: [[VREGR:%[0-9]+]]:rgpr = t2PKHTB [[VREGX]], [[VREGY]], 7, 14, $noreg
+
+    $r0 = COPY %8(s32)
+    ; CHECK: $r0 = COPY [[VREGR]]
+
+    BX_RET 14, $noreg, implicit $r0
+    ; CHECK: BX_RET 14, $noreg, implicit $r0
+...
diff --git a/test/CodeGen/ARM/GlobalISel/select-revsh.mir b/test/CodeGen/ARM/GlobalISel/select-revsh.mir
new file mode 100644
index 0000000000000000000000000000000000000000..1193fa0c1759bb360bb4f298d915adaa101f8910
--- /dev/null
+++ b/test/CodeGen/ARM/GlobalISel/select-revsh.mir
@@ -0,0 +1,147 @@
+# RUN: llc -O0 -mtriple arm-- -mattr=+v6 -run-pass=instruction-select -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,ARM
+# RUN: llc -O0 -mtriple thumb-- -mattr=+v6t2 -run-pass=instruction-select -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,THUMB
+--- |
+  define void @test_shifts_to_revsh() { ret void }
+  define void @test_shifts_to_revsh_commutative() { ret void }
+  define void @test_shifts_no_revsh_constants() { ret void }
+...
+---
+name:            test_shifts_to_revsh
+# CHECK-LABEL: name: test_shifts_to_revsh
+legalized:       true
+regBankSelected: true
+selected:        false
+# CHECK: selected: true
+registers:
+  - { id: 0, class: gprb }
+  - { id: 1, class: gprb }
+  - { id: 2, class: gprb }
+  - { id: 3, class: gprb }
+  - { id: 4, class: gprb }
+  - { id: 5, class: gprb }
+  - { id: 6, class: gprb }
+  - { id: 7, class: gprb }
+  - { id: 8, class: gprb }
+  - { id: 9, class: gprb }
+body:             |
+  bb.0:
+    liveins: $r0
+
+    %0(s32) = COPY $r0
+    ; ARM: [[VREGX:%[0-9]+]]:gpr = COPY $r0
+    ; THUMB: [[VREGX:%[0-9]+]]:rgpr = COPY $r0
+
+    %1(s32) = G_CONSTANT i32 24
+    %2(s32) = G_SHL %0(s32), %1(s32)
+
+    %3(s32) = G_CONSTANT i32 16
+    %4(s32) = G_ASHR %2(s32), %3(s32)
+
+    %5(s32) = G_CONSTANT i32 8
+    %6(s32) = G_LSHR %0(s32), %5(s32)
+
+    %7(s32) = G_CONSTANT i32 255
+    %8(s32) = G_AND %6(s32), %7(s32)
+
+    %9(s32) = G_OR %4(s32), %8(s32)
+    ; ARM: [[VREGR:%[0-9]+]]:gpr = REVSH [[VREGX]]
+    ; THUMB: [[VREGR:%[0-9]+]]:rgpr = t2REVSH [[VREGX]]
+
+    $r0 = COPY %9(s32)
+    ; CHECK: $r0 = COPY [[VREGR]]
+
+    BX_RET 14, $noreg, implicit $r0
+    ; CHECK: BX_RET 14, $noreg, implicit $r0
+...
+---
+name:            test_shifts_to_revsh_commutative
+# CHECK-LABEL: name: test_shifts_to_revsh_commutative
+legalized:       true
+regBankSelected: true
+selected:        false
+# CHECK: selected: true
+registers:
+  - { id: 0, class: gprb }
+  - { id: 1, class: gprb }
+  - { id: 2, class: gprb }
+  - { id: 3, class: gprb }
+  - { id: 4, class: gprb }
+  - { id: 5, class: gprb }
+  - { id: 6, class: gprb }
+  - { id: 7, class: gprb }
+  - { id: 8, class: gprb }
+  - { id: 9, class: gprb }
+body:             |
+  bb.0:
+    liveins: $r0
+
+    %0(s32) = COPY $r0
+    ; ARM: [[VREGX:%[0-9]+]]:gpr = COPY $r0
+    ; THUMB: [[VREGX:%[0-9]+]]:rgpr = COPY $r0
+
+    %1(s32) = G_CONSTANT i32 24
+    %2(s32) = G_SHL %0(s32), %1(s32)
+
+    %3(s32) = G_CONSTANT i32 16
+    %4(s32) = G_ASHR %2(s32), %3(s32)
+
+    %5(s32) = G_CONSTANT i32 8
+    %6(s32) = G_LSHR %0(s32), %5(s32)
+
+    %7(s32) = G_CONSTANT i32 255
+    %8(s32) = G_AND %6(s32), %7(s32)
+
+    %9(s32) = G_OR %8(s32), %4(s32)
+    ; ARM: [[VREGR:%[0-9]+]]:gpr = REVSH [[VREGX]]
+    ; THUMB: [[VREGR:%[0-9]+]]:rgpr = t2REVSH [[VREGX]]
+
+    $r0 = COPY %9(s32)
+    ; CHECK: $r0 = COPY [[VREGR]]
+
+    BX_RET 14, $noreg, implicit $r0
+    ; CHECK: BX_RET 14, $noreg, implicit $r0
+...
+---
+name:            test_shifts_no_revsh_constants
+# CHECK-LABEL: name: test_shifts_no_revsh_constants
+legalized:       true
+regBankSelected: true
+selected:        false
+# CHECK: selected: true
+registers:
+  - { id: 0, class: gprb }
+  - { id: 1, class: gprb }
+  - { id: 2, class: gprb }
+  - { id: 3, class: gprb }
+  - { id: 4, class: gprb }
+  - { id: 5, class: gprb }
+  - { id: 6, class: gprb }
+  - { id: 7, class: gprb }
+  - { id: 8, class: gprb }
+  - { id: 9, class: gprb }
+body:             |
+  bb.0:
+    liveins: $r0
+
+    %0(s32) = COPY $r0
+
+    %1(s32) = G_CONSTANT i32 16 ; REVSH needs 24 here
+    %2(s32) = G_SHL %0(s32), %1(s32)
+
+    %3(s32) = G_CONSTANT i32 24 ; REVSH needs 16 here
+    %4(s32) = G_ASHR %2(s32), %3(s32)
+
+    %5(s32) = G_CONSTANT i32 8
+    %6(s32) = G_LSHR %0(s32), %5(s32)
+
+    %7(s32) = G_CONSTANT i32 255
+    %8(s32) = G_AND %6(s32), %7(s32)
+
+    %9(s32) = G_OR %4(s32), %8(s32)
+    ; We don't really care how this is folded as long as it's not into a REVSH.
+    ; CHECK-NOT: REVSH
+
+    $r0 = COPY %9(s32)
+
+    BX_RET 14, $noreg, implicit $r0
+...
diff --git a/test/CodeGen/ARM/GlobalISel/thumb-instruction-select-cmp.mir b/test/CodeGen/ARM/GlobalISel/thumb-instruction-select-cmp.mir
new file mode 100644
index 0000000000000000000000000000000000000000..7da7dc55bc78cf3466b0abc6a0cfccedc3293c76
--- /dev/null
+++ b/test/CodeGen/ARM/GlobalISel/thumb-instruction-select-cmp.mir
@@ -0,0 +1,313 @@
+# RUN: llc -O0 -mtriple thumb-- -mattr=+v6t2 -run-pass=instruction-select -verify-machineinstrs %s -o - | FileCheck %s
+--- |
+  define void @test_icmp_eq_s32() { ret void }
+  define void @test_icmp_ne_s32() { ret void }
+  define void @test_icmp_ugt_s32() { ret void }
+  define void @test_icmp_uge_s32() { ret void }
+  define void @test_icmp_ult_s32() { ret void }
+  define void @test_icmp_ule_s32() { ret void }
+  define void @test_icmp_sgt_s32() { ret void }
+  define void @test_icmp_sge_s32() { ret void }
+  define void @test_icmp_slt_s32() { ret void }
+  define void @test_icmp_sle_s32() { ret void }
+...
+---
+name:            test_icmp_eq_s32
+legalized:       true
+regBankSelected: true
+selected:        false
+registers:
+  - { id: 0, class: gprb }
+  - { id: 1, class: gprb }
+  - { id: 2, class: gprb }
+  - { id: 3, class: gprb }
+body:             |
+  bb.0:
+    liveins: $r0, $r1
+
+    ; CHECK-LABEL: name: test_icmp_eq_s32
+    ; CHECK: [[COPY:%[0-9]+]]:gprnopc = COPY $r0
+    ; CHECK: [[COPY1:%[0-9]+]]:rgpr = COPY $r1
+    ; CHECK: [[MOVi:%[0-9]+]]:rgpr = t2MOVi 0, 14, $noreg, $noreg
+    ; CHECK: t2CMPrr [[COPY]], [[COPY1]], 14, $noreg, implicit-def $cpsr
+    ; CHECK: [[MOVCCi:%[0-9]+]]:rgpr = t2MOVCCi [[MOVi]], 1, 0, $cpsr
+    ; CHECK: [[ANDri:%[0-9]+]]:rgpr = t2ANDri [[MOVCCi]], 1, 14, $noreg, $noreg
+    ; CHECK: $r0 = COPY [[ANDri]]
+    ; CHECK: BX_RET 14, $noreg, implicit $r0
+    %0(s32) = COPY $r0
+    %1(s32) = COPY $r1
+    %2(s1) = G_ICMP intpred(eq),  %0(s32), %1
+    %3(s32) = G_ZEXT %2(s1)
+    $r0 = COPY %3(s32)
+    BX_RET 14, $noreg, implicit $r0
+...
+---
+name:            test_icmp_ne_s32
+legalized:       true
+regBankSelected: true
+selected:        false
+registers:
+  - { id: 0, class: gprb }
+  - { id: 1, class: gprb }
+  - { id: 2, class: gprb }
+  - { id: 3, class: gprb }
+body:             |
+  bb.0:
+    liveins: $r0, $r1
+
+    ; CHECK-LABEL: name: test_icmp_ne_s32
+    ; CHECK: [[COPY:%[0-9]+]]:gprnopc = COPY $r0
+    ; CHECK: [[COPY1:%[0-9]+]]:rgpr = COPY $r1
+    ; CHECK: [[MOVi:%[0-9]+]]:rgpr = t2MOVi 0, 14, $noreg, $noreg
+    ; CHECK: t2CMPrr [[COPY]], [[COPY1]], 14, $noreg, implicit-def $cpsr
+    ; CHECK: [[MOVCCi:%[0-9]+]]:rgpr = t2MOVCCi [[MOVi]], 1, 1, $cpsr
+    ; CHECK: [[ANDri:%[0-9]+]]:rgpr = t2ANDri [[MOVCCi]], 1, 14, $noreg, $noreg
+    ; CHECK: $r0 = COPY [[ANDri]]
+    ; CHECK: BX_RET 14, $noreg, implicit $r0
+    %0(s32) = COPY $r0
+    %1(s32) = COPY $r1
+    %2(s1) = G_ICMP intpred(ne),  %0(s32), %1
+    %3(s32) = G_ZEXT %2(s1)
+    $r0 = COPY %3(s32)
+    BX_RET 14, $noreg, implicit $r0
+...
+---
+name:            test_icmp_ugt_s32
+legalized:       true
+regBankSelected: true
+selected:        false
+registers:
+  - { id: 0, class: gprb }
+  - { id: 1, class: gprb }
+  - { id: 2, class: gprb }
+  - { id: 3, class: gprb }
+body:             |
+  bb.0:
+    liveins: $r0, $r1
+
+    ; CHECK-LABEL: name: test_icmp_ugt_s32
+    ; CHECK: [[COPY:%[0-9]+]]:gprnopc = COPY $r0
+    ; CHECK: [[COPY1:%[0-9]+]]:rgpr = COPY $r1
+    ; CHECK: [[MOVi:%[0-9]+]]:rgpr = t2MOVi 0, 14, $noreg, $noreg
+    ; CHECK: t2CMPrr [[COPY]], [[COPY1]], 14, $noreg, implicit-def $cpsr
+    ; CHECK: [[MOVCCi:%[0-9]+]]:rgpr = t2MOVCCi [[MOVi]], 1, 8, $cpsr
+    ; CHECK: [[ANDri:%[0-9]+]]:rgpr = t2ANDri [[MOVCCi]], 1, 14, $noreg, $noreg
+    ; CHECK: $r0 = COPY [[ANDri]]
+    ; CHECK: BX_RET 14, $noreg, implicit $r0
+    %0(s32) = COPY $r0
+    %1(s32) = COPY $r1
+    %2(s1) = G_ICMP intpred(ugt),  %0(s32), %1
+    %3(s32) = G_ZEXT %2(s1)
+    $r0 = COPY %3(s32)
+    BX_RET 14, $noreg, implicit $r0
+...
+---
+name:            test_icmp_uge_s32
+legalized:       true
+regBankSelected: true
+selected:        false
+registers:
+  - { id: 0, class: gprb }
+  - { id: 1, class: gprb }
+  - { id: 2, class: gprb }
+  - { id: 3, class: gprb }
+body:             |
+  bb.0:
+    liveins: $r0, $r1
+
+    ; CHECK-LABEL: name: test_icmp_uge_s32
+    ; CHECK: [[COPY:%[0-9]+]]:gprnopc = COPY $r0
+    ; CHECK: [[COPY1:%[0-9]+]]:rgpr = COPY $r1
+    ; CHECK: [[MOVi:%[0-9]+]]:rgpr = t2MOVi 0, 14, $noreg, $noreg
+    ; CHECK: t2CMPrr [[COPY]], [[COPY1]], 14, $noreg, implicit-def $cpsr
+    ; CHECK: [[MOVCCi:%[0-9]+]]:rgpr = t2MOVCCi [[MOVi]], 1, 2, $cpsr
+    ; CHECK: [[ANDri:%[0-9]+]]:rgpr = t2ANDri [[MOVCCi]], 1, 14, $noreg, $noreg
+    ; CHECK: $r0 = COPY [[ANDri]]
+    ; CHECK: BX_RET 14, $noreg, implicit $r0
+    %0(s32) = COPY $r0
+    %1(s32) = COPY $r1
+    %2(s1) = G_ICMP intpred(uge),  %0(s32), %1
+    %3(s32) = G_ZEXT %2(s1)
+    $r0 = COPY %3(s32)
+    BX_RET 14, $noreg, implicit $r0
+...
+---
+name:            test_icmp_ult_s32
+legalized:       true
+regBankSelected: true
+selected:        false
+registers:
+  - { id: 0, class: gprb }
+  - { id: 1, class: gprb }
+  - { id: 2, class: gprb }
+  - { id: 3, class: gprb }
+body:             |
+  bb.0:
+    liveins: $r0, $r1
+
+    ; CHECK-LABEL: name: test_icmp_ult_s32
+    ; CHECK: [[COPY:%[0-9]+]]:gprnopc = COPY $r0
+    ; CHECK: [[COPY1:%[0-9]+]]:rgpr = COPY $r1
+    ; CHECK: [[MOVi:%[0-9]+]]:rgpr = t2MOVi 0, 14, $noreg, $noreg
+    ; CHECK: t2CMPrr [[COPY]], [[COPY1]], 14, $noreg, implicit-def $cpsr
+    ; CHECK: [[MOVCCi:%[0-9]+]]:rgpr = t2MOVCCi [[MOVi]], 1, 3, $cpsr
+    ; CHECK: [[ANDri:%[0-9]+]]:rgpr = t2ANDri [[MOVCCi]], 1, 14, $noreg, $noreg
+    ; CHECK: $r0 = COPY [[ANDri]]
+    ; CHECK: BX_RET 14, $noreg, implicit $r0
+    %0(s32) = COPY $r0
+    %1(s32) = COPY $r1
+    %2(s1) = G_ICMP intpred(ult),  %0(s32), %1
+    %3(s32) = G_ZEXT %2(s1)
+    $r0 = COPY %3(s32)
+    BX_RET 14, $noreg, implicit $r0
+...
+---
+name:            test_icmp_ule_s32
+legalized:       true
+regBankSelected: true
+selected:        false
+registers:
+  - { id: 0, class: gprb }
+  - { id: 1, class: gprb }
+  - { id: 2, class: gprb }
+  - { id: 3, class: gprb }
+body:             |
+  bb.0:
+    liveins: $r0, $r1
+
+    ; CHECK-LABEL: name: test_icmp_ule_s32
+    ; CHECK: [[COPY:%[0-9]+]]:gprnopc = COPY $r0
+    ; CHECK: [[COPY1:%[0-9]+]]:rgpr = COPY $r1
+    ; CHECK: [[MOVi:%[0-9]+]]:rgpr = t2MOVi 0, 14, $noreg, $noreg
+    ; CHECK: t2CMPrr [[COPY]], [[COPY1]], 14, $noreg, implicit-def $cpsr
+    ; CHECK: [[MOVCCi:%[0-9]+]]:rgpr = t2MOVCCi [[MOVi]], 1, 9, $cpsr
+    ; CHECK: [[ANDri:%[0-9]+]]:rgpr = t2ANDri [[MOVCCi]], 1, 14, $noreg, $noreg
+    ; CHECK: $r0 = COPY [[ANDri]]
+    ; CHECK: BX_RET 14, $noreg, implicit $r0
+    %0(s32) = COPY $r0
+    %1(s32) = COPY $r1
+    %2(s1) = G_ICMP intpred(ule),  %0(s32), %1
+    %3(s32) = G_ZEXT %2(s1)
+    $r0 = COPY %3(s32)
+    BX_RET 14, $noreg, implicit $r0
+...
+---
+name:            test_icmp_sgt_s32
+legalized:       true
+regBankSelected: true
+selected:        false
+registers:
+  - { id: 0, class: gprb }
+  - { id: 1, class: gprb }
+  - { id: 2, class: gprb }
+  - { id: 3, class: gprb }
+body:             |
+  bb.0:
+    liveins: $r0, $r1
+
+    ; CHECK-LABEL: name: test_icmp_sgt_s32
+    ; CHECK: [[COPY:%[0-9]+]]:gprnopc = COPY $r0
+    ; CHECK: [[COPY1:%[0-9]+]]:rgpr = COPY $r1
+    ; CHECK: [[MOVi:%[0-9]+]]:rgpr = t2MOVi 0, 14, $noreg, $noreg
+    ; CHECK: t2CMPrr [[COPY]], [[COPY1]], 14, $noreg, implicit-def $cpsr
+    ; CHECK: [[MOVCCi:%[0-9]+]]:rgpr = t2MOVCCi [[MOVi]], 1, 12, $cpsr
+    ; CHECK: [[ANDri:%[0-9]+]]:rgpr = t2ANDri [[MOVCCi]], 1, 14, $noreg, $noreg
+    ; CHECK: $r0 = COPY [[ANDri]]
+    ; CHECK: BX_RET 14, $noreg, implicit $r0
+    %0(s32) = COPY $r0
+    %1(s32) = COPY $r1
+    %2(s1) = G_ICMP intpred(sgt),  %0(s32), %1
+    %3(s32) = G_ZEXT %2(s1)
+    $r0 = COPY %3(s32)
+    BX_RET 14, $noreg, implicit $r0
+...
+---
+name:            test_icmp_sge_s32
+legalized:       true
+regBankSelected: true
+selected:        false
+registers:
+  - { id: 0, class: gprb }
+  - { id: 1, class: gprb }
+  - { id: 2, class: gprb }
+  - { id: 3, class: gprb }
+body:             |
+  bb.0:
+    liveins: $r0, $r1
+
+    ; CHECK-LABEL: name: test_icmp_sge_s32
+    ; CHECK: [[COPY:%[0-9]+]]:gprnopc = COPY $r0
+    ; CHECK: [[COPY1:%[0-9]+]]:rgpr = COPY $r1
+    ; CHECK: [[MOVi:%[0-9]+]]:rgpr = t2MOVi 0, 14, $noreg, $noreg
+    ; CHECK: t2CMPrr [[COPY]], [[COPY1]], 14, $noreg, implicit-def $cpsr
+    ; CHECK: [[MOVCCi:%[0-9]+]]:rgpr = t2MOVCCi [[MOVi]], 1, 10, $cpsr
+    ; CHECK: [[ANDri:%[0-9]+]]:rgpr = t2ANDri [[MOVCCi]], 1, 14, $noreg, $noreg
+    ; CHECK: $r0 = COPY [[ANDri]]
+    ; CHECK: BX_RET 14, $noreg, implicit $r0
+    %0(s32) = COPY $r0
+    %1(s32) = COPY $r1
+    %2(s1) = G_ICMP intpred(sge),  %0(s32), %1
+    %3(s32) = G_ZEXT %2(s1)
+    $r0 = COPY %3(s32)
+    BX_RET 14, $noreg, implicit $r0
+...
+---
+name:            test_icmp_slt_s32
+legalized:       true
+regBankSelected: true
+selected:        false
+registers:
+  - { id: 0, class: gprb }
+  - { id: 1, class: gprb }
+  - { id: 2, class: gprb }
+  - { id: 3, class: gprb }
+body:             |
+  bb.0:
+    liveins: $r0, $r1
+
+    ; CHECK-LABEL: name: test_icmp_slt_s32
+    ; CHECK: [[COPY:%[0-9]+]]:gprnopc = COPY $r0
+    ; CHECK: [[COPY1:%[0-9]+]]:rgpr = COPY $r1
+    ; CHECK: [[MOVi:%[0-9]+]]:rgpr = t2MOVi 0, 14, $noreg, $noreg
+    ; CHECK: t2CMPrr [[COPY]], [[COPY1]], 14, $noreg, implicit-def $cpsr
+    ; CHECK: [[MOVCCi:%[0-9]+]]:rgpr = t2MOVCCi [[MOVi]], 1, 11, $cpsr
+    ; CHECK: [[ANDri:%[0-9]+]]:rgpr = t2ANDri [[MOVCCi]], 1, 14, $noreg, $noreg
+    ; CHECK: $r0 = COPY [[ANDri]]
+    ; CHECK: BX_RET 14, $noreg, implicit $r0
+    %0(s32) = COPY $r0
+    %1(s32) = COPY $r1
+    %2(s1) = G_ICMP intpred(slt),  %0(s32), %1
+    %3(s32) = G_ZEXT %2(s1)
+    $r0 = COPY %3(s32)
+    BX_RET 14, $noreg, implicit $r0
+...
+---
+name:            test_icmp_sle_s32
+legalized:       true
+regBankSelected: true
+selected:        false
+registers:
+  - { id: 0, class: gprb }
+  - { id: 1, class: gprb }
+  - { id: 2, class: gprb }
+  - { id: 3, class: gprb }
+body:             |
+  bb.0:
+    liveins: $r0, $r1
+
+    ; CHECK-LABEL: name: test_icmp_sle_s32
+    ; CHECK: [[COPY:%[0-9]+]]:gprnopc = COPY $r0
+    ; CHECK: [[COPY1:%[0-9]+]]:rgpr = COPY $r1
+    ; CHECK: [[MOVi:%[0-9]+]]:rgpr = t2MOVi 0, 14, $noreg, $noreg
+    ; CHECK: t2CMPrr [[COPY]], [[COPY1]], 14, $noreg, implicit-def $cpsr
+    ; CHECK: [[MOVCCi:%[0-9]+]]:rgpr = t2MOVCCi [[MOVi]], 1, 13, $cpsr
+    ; CHECK: [[ANDri:%[0-9]+]]:rgpr = t2ANDri [[MOVCCi]], 1, 14, $noreg, $noreg
+    ; CHECK: $r0 = COPY [[ANDri]]
+    ; CHECK: BX_RET 14, $noreg, implicit $r0
+    %0(s32) = COPY $r0
+    %1(s32) = COPY $r1
+    %2(s1) = G_ICMP intpred(sle),  %0(s32), %1
+    %3(s32) = G_ZEXT %2(s1)
+    $r0 = COPY %3(s32)
+    BX_RET 14, $noreg, implicit $r0
+...
diff --git a/test/CodeGen/ARM/GlobalISel/thumb-select-arithmetic-ops.mir b/test/CodeGen/ARM/GlobalISel/thumb-select-arithmetic-ops.mir
index cf42a815cd4b8b4668da0f1324b21865a09d1e4b..f760af5e0c78cb33f8b5dc855ad4b7f934595fbf 100644
--- a/test/CodeGen/ARM/GlobalISel/thumb-select-arithmetic-ops.mir
+++ b/test/CodeGen/ARM/GlobalISel/thumb-select-arithmetic-ops.mir
@@ -1,4 +1,4 @@
-# RUN: llc -O0 -mtriple thumb-- -mattr=+v6t2 -run-pass=instruction-select -verify-machineinstrs %s -o - | FileCheck %s
+# RUN: llc -O0 -mtriple thumb-- -mattr=+v6t2,+hwdiv -run-pass=instruction-select -verify-machineinstrs %s -o - | FileCheck %s
 --- |
   define void @test_add_regs() { ret void }
   define void @test_add_fold_imm() { ret void }
@@ -10,6 +10,9 @@
 
   define void @test_mul() { ret void }
   define void @test_mla() { ret void }
+
+  define void @test_sdiv() { ret void }
+  define void @test_udiv() { ret void }
 ...
 ---
 name:            test_add_regs
@@ -249,3 +252,63 @@ body:             |
     BX_RET 14, $noreg, implicit $r0
     ; CHECK: BX_RET 14, $noreg, implicit $r0
 ...
+---
+name:            test_sdiv
+# CHECK-LABEL: name: test_sdiv
+legalized:       true
+regBankSelected: true
+selected:        false
+# CHECK: selected: true
+registers:
+  - { id: 0, class: gprb }
+  - { id: 1, class: gprb }
+  - { id: 2, class: gprb }
+body:             |
+  bb.0:
+    liveins: $r0, $r1
+
+    %0(s32) = COPY $r0
+    ; CHECK: [[VREGX:%[0-9]+]]:rgpr = COPY $r0
+
+    %1(s32) = COPY $r1
+    ; CHECK: [[VREGY:%[0-9]+]]:rgpr = COPY $r1
+
+    %2(s32) = G_SDIV %0, %1
+    ; CHECK: [[VREGRES:%[0-9]+]]:rgpr = t2SDIV [[VREGX]], [[VREGY]], 14, $noreg
+
+    $r0 = COPY %2(s32)
+    ; CHECK: $r0 = COPY [[VREGRES]]
+
+    BX_RET 14, $noreg, implicit $r0
+    ; CHECK: BX_RET 14, $noreg, implicit $r0
+...
+---
+name:            test_udiv
+# CHECK-LABEL: name: test_udiv
+legalized:       true
+regBankSelected: true
+selected:        false
+# CHECK: selected: true
+registers:
+  - { id: 0, class: gprb }
+  - { id: 1, class: gprb }
+  - { id: 2, class: gprb }
+body:             |
+  bb.0:
+    liveins: $r0, $r1
+
+    %0(s32) = COPY $r0
+    ; CHECK: [[VREGX:%[0-9]+]]:rgpr = COPY $r0
+
+    %1(s32) = COPY $r1
+    ; CHECK: [[VREGY:%[0-9]+]]:rgpr = COPY $r1
+
+    %2(s32) = G_UDIV %0, %1
+    ; CHECK: [[VREGRES:%[0-9]+]]:rgpr = t2UDIV [[VREGX]], [[VREGY]], 14, $noreg
+
+    $r0 = COPY %2(s32)
+    ; CHECK: $r0 = COPY [[VREGRES]]
+
+    BX_RET 14, $noreg, implicit $r0
+    ; CHECK: BX_RET 14, $noreg, implicit $r0
+...
diff --git a/test/CodeGen/ARM/GlobalISel/thumb-select-load-store.mir b/test/CodeGen/ARM/GlobalISel/thumb-select-load-store.mir
index 170c26e996cb512731422e4a85431360416a807b..ec64c8a2c1ed08bd35099c7f23884604c70b57ea 100644
--- a/test/CodeGen/ARM/GlobalISel/thumb-select-load-store.mir
+++ b/test/CodeGen/ARM/GlobalISel/thumb-select-load-store.mir
@@ -3,6 +3,8 @@
   define void @test_s8() { ret void }
   define void @test_s16() { ret void }
   define void @test_s32() { ret void }
+
+  define void @test_gep() { ret void }
 ...
 ---
 name:            test_s8
@@ -82,3 +84,33 @@ body:             |
     BX_RET 14, $noreg
     ; CHECK: BX_RET 14, $noreg
 ...
+---
+name:            test_gep
+# CHECK-LABEL: name: test_gep
+legalized:       true
+regBankSelected: true
+selected:        false
+# CHECK: selected: true
+registers:
+  - { id: 0, class: gprb }
+  - { id: 1, class: gprb }
+  - { id: 2, class: gprb }
+body:             |
+  bb.0:
+    liveins: $r0, $r1
+
+    %0(p0) = COPY $r0
+    ; CHECK: [[PTR:%[0-9]+]]:gprnopc = COPY $r0
+
+    %1(s32) = COPY $r1
+    ; CHECK: [[OFF:%[0-9]+]]:rgpr = COPY $r1
+
+    %2(p0) = G_GEP %0, %1(s32)
+    ; CHECK: [[GEP:%[0-9]+]]:gprnopc = t2ADDrr [[PTR]], [[OFF]], 14, $noreg, $noreg
+
+    $r0 = COPY %2(p0)
+    ; CHECK: $r0 = COPY [[GEP]]
+
+    BX_RET 14, $noreg, implicit $r0
+    ; CHECK: BX_RET 14, $noreg, implicit $r0
+...
diff --git a/test/CodeGen/ARM/GlobalISel/thumb-select-select.mir b/test/CodeGen/ARM/GlobalISel/thumb-select-select.mir
new file mode 100644
index 0000000000000000000000000000000000000000..d1a482b5514c50f0d43974f699e3684d17f25db3
--- /dev/null
+++ b/test/CodeGen/ARM/GlobalISel/thumb-select-select.mir
@@ -0,0 +1,79 @@
+# RUN: llc -O0 -mtriple thumb-- -mattr=+v6t2 -run-pass=instruction-select -verify-machineinstrs %s -o - | FileCheck %s
+--- |
+  define void @test_select_s32() { ret void }
+  define void @test_select_ptr() { ret void }
+...
+---
+name:            test_select_s32
+# CHECK-LABEL: name: test_select_s32
+legalized:       true
+regBankSelected: true
+selected:        false
+# CHECK: selected: true
+registers:
+  - { id: 0, class: gprb }
+  - { id: 1, class: gprb }
+  - { id: 2, class: gprb }
+  - { id: 3, class: gprb }
+body:             |
+  bb.0:
+    liveins: $r0, $r1
+
+    %0(s32) = COPY $r0
+    ; CHECK: [[VREGX:%[0-9]+]]:rgpr = COPY $r0
+
+    %1(s32) = COPY $r1
+    ; CHECK: [[VREGY:%[0-9]+]]:rgpr = COPY $r1
+
+    %2(s1) = G_TRUNC %1(s32)
+    ; CHECK: [[VREGC:%[0-9]+]]:gprnopc = COPY [[VREGY]]
+
+    %3(s32) = G_SELECT %2(s1),  %0, %1
+    ; CHECK: t2CMPri [[VREGC]], 0, 14, $noreg, implicit-def $cpsr
+    ; CHECK: [[RES:%[0-9]+]]:rgpr = t2MOVCCr [[VREGX]], [[VREGY]], 0, $cpsr
+
+    $r0 = COPY %3(s32)
+    ; CHECK: $r0 = COPY [[RES]]
+
+    BX_RET 14, $noreg, implicit $r0
+    ; CHECK: BX_RET 14, $noreg, implicit $r0
+...
+---
+name:            test_select_ptr
+# CHECK-LABEL: name: test_select_ptr
+legalized:       true
+regBankSelected: true
+selected:        false
+# CHECK: selected: true
+registers:
+  - { id: 0, class: gprb }
+  - { id: 1, class: gprb }
+  - { id: 2, class: gprb }
+  - { id: 3, class: gprb }
+  - { id: 4, class: gprb }
+body:             |
+  bb.0:
+    liveins: $r0, $r1, $r2
+
+    %0(p0) = COPY $r0
+    ; CHECK: [[VREGX:%[0-9]+]]:rgpr = COPY $r0
+
+    %1(p0) = COPY $r1
+    ; CHECK: [[VREGY:%[0-9]+]]:rgpr = COPY $r1
+
+    %2(s32) = COPY $r2
+    ; CHECK: [[VREGC32:%[0-9]+]]:gpr = COPY $r2
+
+    %3(s1) = G_TRUNC %2(s32)
+    ; CHECK: [[VREGC:%[0-9]+]]:gprnopc = COPY [[VREGC32]]
+
+    %4(p0) = G_SELECT %3(s1),  %0, %1
+    ; CHECK: t2CMPri [[VREGC]], 0, 14, $noreg, implicit-def $cpsr
+    ; CHECK: [[RES:%[0-9]+]]:rgpr = t2MOVCCr [[VREGX]], [[VREGY]], 0, $cpsr
+
+    $r0 = COPY %4(p0)
+    ; CHECK: $r0 = COPY [[RES]]
+
+    BX_RET 14, $noreg, implicit $r0
+    ; CHECK: BX_RET 14, $noreg, implicit $r0
+...
diff --git a/test/CodeGen/ARM/GlobalISel/thumb-select-shifts.mir b/test/CodeGen/ARM/GlobalISel/thumb-select-shifts.mir
new file mode 100644
index 0000000000000000000000000000000000000000..81c7cc22dd05594ee18f4348a3bc83d761260ea1
--- /dev/null
+++ b/test/CodeGen/ARM/GlobalISel/thumb-select-shifts.mir
@@ -0,0 +1,95 @@
+# RUN: llc -O0 -mtriple thumb-- -mattr=+v6t2 -run-pass=instruction-select -verify-machineinstrs %s -o - | FileCheck %s
+--- |
+  define void @test_ashr_rr() { ret void }
+
+  define void @test_shl_ri() { ret void }
+  define void @test_shl_ri_bad_imm() { ret void }
+...
+---
+name:            test_ashr_rr
+# CHECK-LABEL: name: test_ashr_rr
+legalized:       true
+regBankSelected: true
+selected:        false
+# CHECK: selected: true
+registers:
+  - { id: 0, class: gprb }
+  - { id: 1, class: gprb }
+  - { id: 2, class: gprb }
+body:             |
+  bb.0:
+    liveins: $r0, $r1
+
+    %0(s32) = COPY $r0
+    ; CHECK: [[VREGX:%[0-9]+]]:rgpr = COPY $r0
+
+    %1(s32) = COPY $r1
+    ; CHECK: [[VREGY:%[0-9]+]]:rgpr = COPY $r1
+
+    %2(s32) = G_ASHR %0, %1
+    ; CHECK: [[VREGRES:%[0-9]+]]:rgpr = t2ASRrr [[VREGX]], [[VREGY]], 14, $noreg, $noreg
+
+    $r0 = COPY %2(s32)
+    ; CHECK: $r0 = COPY [[VREGRES]]
+
+    BX_RET 14, $noreg, implicit $r0
+    ; CHECK: BX_RET 14, $noreg, implicit $r0
+...
+---
+name:            test_shl_ri
+# CHECK-LABEL: name: test_shl_ri
+legalized:       true
+regBankSelected: true
+selected:        false
+# CHECK: selected: true
+registers:
+  - { id: 0, class: gprb }
+  - { id: 1, class: gprb }
+  - { id: 2, class: gprb }
+body:             |
+  bb.0:
+    liveins: $r0
+
+    %0(s32) = COPY $r0
+    ; CHECK: [[VREGX:%[0-9]+]]:rgpr = COPY $r0
+
+    %1(s32) = G_CONSTANT i32 31
+    %2(s32) = G_SHL %0, %1
+    ; CHECK: [[VREGRES:%[0-9]+]]:rgpr = t2LSLri [[VREGX]], 31, 14, $noreg, $noreg
+
+    $r0 = COPY %2(s32)
+    ; CHECK: $r0 = COPY [[VREGRES]]
+
+    BX_RET 14, $noreg, implicit $r0
+    ; CHECK: BX_RET 14, $noreg, implicit $r0
+...
+---
+name:            test_shl_ri_bad_imm
+# CHECK-LABEL: name: test_shl_ri_bad_imm
+legalized:       true
+regBankSelected: true
+selected:        false
+# CHECK: selected: true
+registers:
+  - { id: 0, class: gprb }
+  - { id: 1, class: gprb }
+  - { id: 2, class: gprb }
+body:             |
+  bb.0:
+    liveins: $r0
+
+    %0(s32) = COPY $r0
+    ; CHECK: [[VREGX:%[0-9]+]]:rgpr = COPY $r0
+
+    %1(s32) = G_CONSTANT i32 32
+    ; CHECK: [[VREGY:%[0-9]+]]:rgpr = t2MOVi 32, 14, $noreg, $noreg
+
+    %2(s32) = G_SHL %0, %1
+    ; CHECK: [[VREGRES:%[0-9]+]]:rgpr = t2LSLrr [[VREGX]], [[VREGY]], 14, $noreg, $noreg
+
+    $r0 = COPY %2(s32)
+    ; CHECK: $r0 = COPY [[VREGRES]]
+
+    BX_RET 14, $noreg, implicit $r0
+    ; CHECK: BX_RET 14, $noreg, implicit $r0
+...
diff --git a/test/CodeGen/ARM/Windows/frame-register.ll b/test/CodeGen/ARM/Windows/frame-register.ll
index 7ecfc1a71714af82e8c01bd91da668b88c483cca..6605ffc60f4aa3384f8d5772c1eac301105f3c89 100644
--- a/test/CodeGen/ARM/Windows/frame-register.ll
+++ b/test/CodeGen/ARM/Windows/frame-register.ll
@@ -1,4 +1,4 @@
-; RUN: llc -mtriple thumbv7-windows -disable-fp-elim -filetype asm -o - %s \
+; RUN: llc -mtriple thumbv7-windows -frame-pointer=all -filetype asm -o - %s \
 ; RUN:     | FileCheck %s
 
 declare void @callee(i32)
diff --git a/test/CodeGen/ARM/and-load-combine.ll b/test/CodeGen/ARM/and-load-combine.ll
index 8f08909c816d1e584140934ed12560cbc5dd7fe6..ef6a2ad7603f0685bb19147e0336838f0487117c 100644
--- a/test/CodeGen/ARM/and-load-combine.ll
+++ b/test/CodeGen/ARM/and-load-combine.ll
@@ -1549,3 +1549,39 @@ define arm_aapcscc i64 @test26(i64* nocapture %p) {
   %and = and i64 %1, -281474976710656
   ret i64 %and
 }
+
+; ARM-LABEL: test27:
+; ARM:       @ %bb.0:
+; ARM-NEXT:    ldrb r1, [r0, #1]
+; ARM-NEXT:    lsl r1, r1, #16
+; ARM-NEXT:    str r1, [r0]
+; ARM-NEXT:    bx lr
+;
+; ARMEB-LABEL: test27:
+; ARMEB:     @ %bb.0:
+; ARMEB-NEXT:  ldrb r1, [r0, #2]
+; ARMEB-NEXT:  lsl r1, r1, #16
+; ARMEB-NEXT:  str r1, [r0]
+; ARMEB-NEXT:  bx lr
+;
+; THUMB1-LABEL: test27:
+; THUMB1:     @ %bb.0:
+; THUMB1-NEXT:  ldrb r1, [r0, #1]
+; THUMB1-NEXT:  lsls r1, r1, #16
+; THUMB1-NEXT:  str r1, [r0]
+; THUMB1-NEXT:  bx lr
+;
+; THUMB2-LABEL: test27:
+; THUMB2:       @ %bb.0:
+; THUMB2-NEXT:    ldrb r1, [r0, #1]
+; THUMB2-NEXT:    lsls r1, r1, #16
+; THUMB2-NEXT:    str r1, [r0]
+; THUMB2-NEXT:    bx lr
+define void @test27(i32* nocapture %ptr) {
+entry:
+  %0 = load i32, i32* %ptr, align 4
+  %and = and i32 %0, 65280
+  %shl = shl i32 %and, 8
+  store i32 %shl, i32* %ptr, align 4
+  ret void
+}
diff --git a/test/CodeGen/ARM/build-attributes.ll b/test/CodeGen/ARM/build-attributes.ll
index 3d0c9419e8fef9e66e740d5515b835d0117a4fba..fefe0c8fd231bd0645a687c999188cdbd5885cbd 100644
--- a/test/CodeGen/ARM/build-attributes.ll
+++ b/test/CodeGen/ARM/build-attributes.ll
@@ -3,23 +3,23 @@
 
 ; RUN: llc < %s -mtriple=thumbv5-linux-gnueabi -mcpu=xscale -mattr=+strict-align | FileCheck %s --check-prefix=XSCALE
 ; RUN: llc < %s -mtriple=armv6-linux-gnueabi -mattr=+strict-align | FileCheck %s --check-prefix=V6
-; RUN: llc < %s -mtriple=armv6-linux-gnueabi -mattr=+strict-align  -enable-unsafe-fp-math -disable-fp-elim -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=V6-FAST
+; RUN: llc < %s -mtriple=armv6-linux-gnueabi -mattr=+strict-align  -enable-unsafe-fp-math -frame-pointer=all -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=V6-FAST
 ; RUN: llc < %s -mtriple=armv6-linux-gnueabi -mattr=+strict-align -enable-sign-dependent-rounding-fp-math | FileCheck %s --check-prefix=DYN-ROUNDING
 ; RUN: llc < %s -mtriple=thumbv6m-linux-gnueabi -mattr=+strict-align | FileCheck %s --check-prefix=V6M
-; RUN: llc < %s -mtriple=thumbv6m-linux-gnueabi -mattr=+strict-align  -enable-unsafe-fp-math -disable-fp-elim -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=V6M-FAST
+; RUN: llc < %s -mtriple=thumbv6m-linux-gnueabi -mattr=+strict-align  -enable-unsafe-fp-math -frame-pointer=all -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=V6M-FAST
 ; RUN: llc < %s -mtriple=thumbv6sm-linux-gnueabi -mattr=+strict-align | FileCheck %s --check-prefix=V6M
-; RUN: llc < %s -mtriple=thumbv6sm-linux-gnueabi -mattr=+strict-align -enable-unsafe-fp-math -disable-fp-elim -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=V6M-FAST
+; RUN: llc < %s -mtriple=thumbv6sm-linux-gnueabi -mattr=+strict-align -enable-unsafe-fp-math -frame-pointer=all -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=V6M-FAST
 ; RUN: llc < %s -mtriple=armv6-linux-gnueabi -mcpu=arm1156t2f-s -mattr=+strict-align | FileCheck %s --check-prefix=ARM1156T2F-S
-; RUN: llc < %s -mtriple=armv6-linux-gnueabi -mcpu=arm1156t2f-s -mattr=+strict-align  -enable-unsafe-fp-math -disable-fp-elim -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast  | FileCheck %s --check-prefix=ARM1156T2F-S-FAST
+; RUN: llc < %s -mtriple=armv6-linux-gnueabi -mcpu=arm1156t2f-s -mattr=+strict-align  -enable-unsafe-fp-math -frame-pointer=all -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast  | FileCheck %s --check-prefix=ARM1156T2F-S-FAST
 ; RUN: llc < %s -mtriple=armv6-linux-gnueabi -mcpu=arm1156t2f-s -mattr=+strict-align -enable-sign-dependent-rounding-fp-math | FileCheck %s --check-prefix=DYN-ROUNDING
 ; RUN: llc < %s -mtriple=thumbv7m-linux-gnueabi | FileCheck %s --check-prefix=V7M
-; RUN: llc < %s -mtriple=thumbv7m-linux-gnueabi  -enable-unsafe-fp-math -disable-fp-elim -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=V7M-FAST
+; RUN: llc < %s -mtriple=thumbv7m-linux-gnueabi  -enable-unsafe-fp-math -frame-pointer=all -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=V7M-FAST
 ; RUN: llc < %s -mtriple=thumbv7m-linux-gnueabi -enable-sign-dependent-rounding-fp-math | FileCheck %s --check-prefix=DYN-ROUNDING
 ; RUN: llc < %s -mtriple=armv7-linux-gnueabi | FileCheck %s --check-prefix=V7
 ; RUN: llc < %s -mtriple=armv7-linux-gnueabi -enable-sign-dependent-rounding-fp-math | FileCheck %s --check-prefix=DYN-ROUNDING
-; RUN: llc < %s -mtriple=armv7-linux-gnueabi  -enable-unsafe-fp-math -disable-fp-elim -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=V7-FAST
+; RUN: llc < %s -mtriple=armv7-linux-gnueabi  -enable-unsafe-fp-math -frame-pointer=all -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=V7-FAST
 ; RUN: llc < %s -mtriple=armv8-linux-gnueabi | FileCheck %s --check-prefix=V8
-; RUN: llc < %s -mtriple=armv8-linux-gnueabi  -enable-unsafe-fp-math -disable-fp-elim -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=V8-FAST
+; RUN: llc < %s -mtriple=armv8-linux-gnueabi  -enable-unsafe-fp-math -frame-pointer=all -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=V8-FAST
 ; RUN: llc < %s -mtriple=armv8-linux-gnueabi -enable-sign-dependent-rounding-fp-math | FileCheck %s --check-prefix=DYN-ROUNDING
 ; RUN: llc < %s -mtriple=thumbv8-linux-gnueabi | FileCheck %s --check-prefix=Vt8
 ; RUN: llc < %s -mtriple=thumbv8-linux-gnueabi -enable-sign-dependent-rounding-fp-math | FileCheck %s --check-prefix=DYN-ROUNDING
@@ -31,35 +31,35 @@
 ; RUN: llc < %s -mtriple=thumbv8m.main-linux-gnueabi | FileCheck %s --check-prefix=V8MMAINLINE
 ; RUN: llc < %s -mtriple=thumbv8m.main-linux-gnueabi -mattr=+dsp | FileCheck %s --check-prefix=V8MMAINLINE_DSP
 ; RUN: llc < %s -mtriple=armv7-linux-gnueabi -mcpu=cortex-a5 | FileCheck %s --check-prefix=CORTEX-A5-DEFAULT
-; RUN: llc < %s -mtriple=armv7-linux-gnueabi -mcpu=cortex-a5  -enable-unsafe-fp-math -disable-fp-elim -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=CORTEX-A5-DEFAULT-FAST
+; RUN: llc < %s -mtriple=armv7-linux-gnueabi -mcpu=cortex-a5  -enable-unsafe-fp-math -frame-pointer=all -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=CORTEX-A5-DEFAULT-FAST
 ; RUN: llc < %s -mtriple=armv7-linux-gnueabi -mcpu=cortex-a5 -enable-sign-dependent-rounding-fp-math | FileCheck %s --check-prefix=DYN-ROUNDING
 ; RUN: llc < %s -mtriple=armv7-linux-gnueabi -mcpu=cortex-a5 -mattr=-neon,+d16 | FileCheck %s --check-prefix=CORTEX-A5-NONEON
 ; RUN: llc < %s -mtriple=armv7-linux-gnueabi -mcpu=cortex-a5 -mattr=-vfp2 | FileCheck %s --check-prefix=CORTEX-A5-NOFPU
-; RUN: llc < %s -mtriple=armv7-linux-gnueabi -mcpu=cortex-a5 -mattr=-vfp2  -enable-unsafe-fp-math -disable-fp-elim -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=CORTEX-A5-NOFPU-FAST
+; RUN: llc < %s -mtriple=armv7-linux-gnueabi -mcpu=cortex-a5 -mattr=-vfp2  -enable-unsafe-fp-math -frame-pointer=all -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=CORTEX-A5-NOFPU-FAST
 ; RUN: llc < %s -mtriple=armv7-linux-gnueabi -mcpu=cortex-a8 -float-abi=soft | FileCheck %s --check-prefix=CORTEX-A8-SOFT
-; RUN: llc < %s -mtriple=armv7-linux-gnueabi -mcpu=cortex-a8 -float-abi=soft  -enable-unsafe-fp-math -disable-fp-elim -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=CORTEX-A8-SOFT-FAST
+; RUN: llc < %s -mtriple=armv7-linux-gnueabi -mcpu=cortex-a8 -float-abi=soft  -enable-unsafe-fp-math -frame-pointer=all -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=CORTEX-A8-SOFT-FAST
 ; RUN: llc < %s -mtriple=armv7-linux-gnueabi -mcpu=cortex-a8 -float-abi=hard | FileCheck %s --check-prefix=CORTEX-A8-HARD
-; RUN: llc < %s -mtriple=armv7-linux-gnueabi -mcpu=cortex-a8 -float-abi=hard  -enable-unsafe-fp-math -disable-fp-elim -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=CORTEX-A8-HARD-FAST
+; RUN: llc < %s -mtriple=armv7-linux-gnueabi -mcpu=cortex-a8 -float-abi=hard  -enable-unsafe-fp-math -frame-pointer=all -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=CORTEX-A8-HARD-FAST
 ; RUN: llc < %s -mtriple=armv7-linux-gnueabi -mcpu=cortex-a8 -enable-sign-dependent-rounding-fp-math | FileCheck %s --check-prefix=DYN-ROUNDING
 ; RUN: llc < %s -mtriple=armv7-linux-gnueabi -mcpu=cortex-a8 -float-abi=soft | FileCheck %s --check-prefix=CORTEX-A8-SOFT
 ; RUN: llc < %s -mtriple=armv7-linux-gnueabi -mcpu=cortex-a9 -float-abi=soft | FileCheck %s --check-prefix=CORTEX-A9-SOFT
-; RUN: llc < %s -mtriple=armv7-linux-gnueabi -mcpu=cortex-a9 -float-abi=soft  -enable-unsafe-fp-math -disable-fp-elim -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=CORTEX-A9-SOFT-FAST
+; RUN: llc < %s -mtriple=armv7-linux-gnueabi -mcpu=cortex-a9 -float-abi=soft  -enable-unsafe-fp-math -frame-pointer=all -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=CORTEX-A9-SOFT-FAST
 ; RUN: llc < %s -mtriple=armv7-linux-gnueabi -mcpu=cortex-a9 -float-abi=hard | FileCheck %s --check-prefix=CORTEX-A9-HARD
-; RUN: llc < %s -mtriple=armv7-linux-gnueabi -mcpu=cortex-a9 -float-abi=hard  -enable-unsafe-fp-math -disable-fp-elim -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=CORTEX-A9-HARD-FAST
+; RUN: llc < %s -mtriple=armv7-linux-gnueabi -mcpu=cortex-a9 -float-abi=hard  -enable-unsafe-fp-math -frame-pointer=all -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=CORTEX-A9-HARD-FAST
 ; RUN: llc < %s -mtriple=armv7-linux-gnueabi -mcpu=cortex-a9 -enable-sign-dependent-rounding-fp-math | FileCheck %s --check-prefix=DYN-ROUNDING
 ; RUN: llc < %s -mtriple=armv7-linux-gnueabi -mcpu=cortex-a12 | FileCheck %s --check-prefix=CORTEX-A12-DEFAULT
 ; RUN: llc < %s -mtriple=armv7-linux-gnueabi -mcpu=cortex-a9 -float-abi=soft | FileCheck %s --check-prefix=CORTEX-A9-SOFT
-; RUN: llc < %s -mtriple=armv7-linux-gnueabi -mcpu=cortex-a12  -enable-unsafe-fp-math -disable-fp-elim -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=CORTEX-A12-DEFAULT-FAST
+; RUN: llc < %s -mtriple=armv7-linux-gnueabi -mcpu=cortex-a12  -enable-unsafe-fp-math -frame-pointer=all -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=CORTEX-A12-DEFAULT-FAST
 ; RUN: llc < %s -mtriple=armv7-linux-gnueabi -mcpu=cortex-a12 -mattr=-vfp2 | FileCheck %s --check-prefix=CORTEX-A12-NOFPU
-; RUN: llc < %s -mtriple=armv7-linux-gnueabi -mcpu=cortex-a12 -mattr=-vfp2  -enable-unsafe-fp-math -disable-fp-elim -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=CORTEX-A12-NOFPU-FAST
+; RUN: llc < %s -mtriple=armv7-linux-gnueabi -mcpu=cortex-a12 -mattr=-vfp2  -enable-unsafe-fp-math -frame-pointer=all -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=CORTEX-A12-NOFPU-FAST
 ; RUN: llc < %s -mtriple=armv7-linux-gnueabi -mcpu=cortex-a12 -enable-sign-dependent-rounding-fp-math | FileCheck %s --check-prefix=DYN-ROUNDING
 ; RUN: llc < %s -mtriple=armv7-linux-gnueabi -mcpu=cortex-a15 | FileCheck %s --check-prefix=CORTEX-A15
-; RUN: llc < %s -mtriple=armv7-linux-gnueabi -mcpu=cortex-a15  -enable-unsafe-fp-math -disable-fp-elim -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=CORTEX-A15-FAST
+; RUN: llc < %s -mtriple=armv7-linux-gnueabi -mcpu=cortex-a15  -enable-unsafe-fp-math -frame-pointer=all -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=CORTEX-A15-FAST
 ; RUN: llc < %s -mtriple=armv7-linux-gnueabi -mcpu=cortex-a15 -enable-sign-dependent-rounding-fp-math | FileCheck %s --check-prefix=DYN-ROUNDING
 ; RUN: llc < %s -mtriple=armv7-linux-gnueabi -mcpu=cortex-a17 | FileCheck %s --check-prefix=CORTEX-A17-DEFAULT
-; RUN: llc < %s -mtriple=armv7-linux-gnueabi -mcpu=cortex-a17  -enable-unsafe-fp-math -disable-fp-elim -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=CORTEX-A17-FAST
+; RUN: llc < %s -mtriple=armv7-linux-gnueabi -mcpu=cortex-a17  -enable-unsafe-fp-math -frame-pointer=all -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=CORTEX-A17-FAST
 ; RUN: llc < %s -mtriple=armv7-linux-gnueabi -mcpu=cortex-a17 -mattr=-vfp2 | FileCheck %s --check-prefix=CORTEX-A17-NOFPU
-; RUN: llc < %s -mtriple=armv7-linux-gnueabi -mcpu=cortex-a17 -mattr=-vfp2  -enable-unsafe-fp-math -disable-fp-elim -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=CORTEX-A17-NOFPU-FAST
+; RUN: llc < %s -mtriple=armv7-linux-gnueabi -mcpu=cortex-a17 -mattr=-vfp2  -enable-unsafe-fp-math -frame-pointer=all -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=CORTEX-A17-NOFPU-FAST
 
 ; RUN: llc < %s -mtriple=armv7-linux-gnueabi -mcpu=cortex-a15 -enable-no-trapping-fp-math | FileCheck %s --check-prefix=NO-TRAPPING-MATH
 ; RUN: llc < %s -mtriple=armv7-linux-gnueabi -mcpu=cortex-a15 -denormal-fp-math=ieee | FileCheck %s --check-prefix=DENORMAL-IEEE
@@ -74,87 +74,87 @@
 
 ; RUN: llc < %s -mtriple=armv7-linux-gnueabi -mcpu=cortex-a17 -enable-sign-dependent-rounding-fp-math | FileCheck %s --check-prefix=DYN-ROUNDING
 ; RUN: llc < %s -mtriple=thumbv6m-linux-gnueabi -mcpu=cortex-m0 | FileCheck %s --check-prefix=CORTEX-M0
-; RUN: llc < %s -mtriple=thumbv6m-linux-gnueabi -mcpu=cortex-m0 -enable-unsafe-fp-math -disable-fp-elim -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=CORTEX-M0-FAST
+; RUN: llc < %s -mtriple=thumbv6m-linux-gnueabi -mcpu=cortex-m0 -enable-unsafe-fp-math -frame-pointer=all -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=CORTEX-M0-FAST
 ; RUN: llc < %s -mtriple=thumbv6m-linux-gnueabi -mcpu=cortex-m0 -enable-sign-dependent-rounding-fp-math | FileCheck %s --check-prefix=DYN-ROUNDING
 ; RUN: llc < %s -mtriple=thumbv6m-linux-gnueabi -mcpu=cortex-m0plus | FileCheck %s --check-prefix=CORTEX-M0PLUS
-; RUN: llc < %s -mtriple=thumbv6m-linux-gnueabi -mcpu=cortex-m0plus -enable-unsafe-fp-math -disable-fp-elim -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=CORTEX-M0PLUS-FAST
+; RUN: llc < %s -mtriple=thumbv6m-linux-gnueabi -mcpu=cortex-m0plus -enable-unsafe-fp-math -frame-pointer=all -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=CORTEX-M0PLUS-FAST
 ; RUN: llc < %s -mtriple=thumbv6m-linux-gnueabi -mcpu=cortex-m0plus -enable-sign-dependent-rounding-fp-math | FileCheck %s --check-prefix=DYN-ROUNDING
 ; RUN: llc < %s -mtriple=thumbv6m-linux-gnueabi -mcpu=cortex-m1 | FileCheck %s --check-prefix=CORTEX-M1
-; RUN: llc < %s -mtriple=thumbv6m-linux-gnueabi -mcpu=cortex-m1 -mattr=+strict-align  -enable-unsafe-fp-math -disable-fp-elim -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=CORTEX-M1-FAST
+; RUN: llc < %s -mtriple=thumbv6m-linux-gnueabi -mcpu=cortex-m1 -mattr=+strict-align  -enable-unsafe-fp-math -frame-pointer=all -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=CORTEX-M1-FAST
 ; RUN: llc < %s -mtriple=thumbv6m-linux-gnueabi -mcpu=cortex-m1 -mattr=+strict-align -enable-sign-dependent-rounding-fp-math | FileCheck %s --check-prefix=DYN-ROUNDING
 ; RUN: llc < %s -mtriple=thumbv6m-linux-gnueabi -mcpu=sc000 -mattr=+strict-align | FileCheck %s --check-prefix=SC000
-; RUN: llc < %s -mtriple=thumbv6m-linux-gnueabi -mcpu=sc000 -mattr=+strict-align  -enable-unsafe-fp-math -disable-fp-elim -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=SC000-FAST
+; RUN: llc < %s -mtriple=thumbv6m-linux-gnueabi -mcpu=sc000 -mattr=+strict-align  -enable-unsafe-fp-math -frame-pointer=all -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=SC000-FAST
 ; RUN: llc < %s -mtriple=thumbv6m-linux-gnueabi -mcpu=sc000 -mattr=+strict-align -enable-sign-dependent-rounding-fp-math | FileCheck %s --check-prefix=DYN-ROUNDING
 ; RUN: llc < %s -mtriple=thumbv7m-linux-gnueabi -mcpu=cortex-m3 | FileCheck %s --check-prefix=CORTEX-M3
-; RUN: llc < %s -mtriple=thumbv7m-linux-gnueabi -mcpu=cortex-m3  -enable-unsafe-fp-math -disable-fp-elim -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=CORTEX-M3-FAST
+; RUN: llc < %s -mtriple=thumbv7m-linux-gnueabi -mcpu=cortex-m3  -enable-unsafe-fp-math -frame-pointer=all -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=CORTEX-M3-FAST
 ; RUN: llc < %s -mtriple=thumbv7m-linux-gnueabi -mcpu=cortex-m3 -enable-sign-dependent-rounding-fp-math | FileCheck %s --check-prefix=DYN-ROUNDING
 ; RUN: llc < %s -mtriple=thumbv7m-linux-gnueabi -mcpu=sc300 | FileCheck %s --check-prefix=SC300
-; RUN: llc < %s -mtriple=thumbv7m-linux-gnueabi -mcpu=sc300  -enable-unsafe-fp-math -disable-fp-elim -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=SC300-FAST
+; RUN: llc < %s -mtriple=thumbv7m-linux-gnueabi -mcpu=sc300  -enable-unsafe-fp-math -frame-pointer=all -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=SC300-FAST
 ; RUN: llc < %s -mtriple=thumbv7m-linux-gnueabi -mcpu=sc300 -enable-sign-dependent-rounding-fp-math | FileCheck %s --check-prefix=DYN-ROUNDING
 ; RUN: llc < %s -mtriple=thumbv7m-linux-gnueabi -mcpu=cortex-m4 -float-abi=soft | FileCheck %s --check-prefix=CORTEX-M4-SOFT
-; RUN: llc < %s -mtriple=thumbv7m-linux-gnueabi -mcpu=cortex-m4 -float-abi=soft  -enable-unsafe-fp-math -disable-fp-elim -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=CORTEX-M4-SOFT-FAST
+; RUN: llc < %s -mtriple=thumbv7m-linux-gnueabi -mcpu=cortex-m4 -float-abi=soft  -enable-unsafe-fp-math -frame-pointer=all -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=CORTEX-M4-SOFT-FAST
 ; RUN: llc < %s -mtriple=thumbv7m-linux-gnueabi -mcpu=cortex-m4 -float-abi=hard | FileCheck %s --check-prefix=CORTEX-M4-HARD
-; RUN: llc < %s -mtriple=thumbv7m-linux-gnueabi -mcpu=cortex-m4 -float-abi=hard  -enable-unsafe-fp-math -disable-fp-elim -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=CORTEX-M4-HARD-FAST
+; RUN: llc < %s -mtriple=thumbv7m-linux-gnueabi -mcpu=cortex-m4 -float-abi=hard  -enable-unsafe-fp-math -frame-pointer=all -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=CORTEX-M4-HARD-FAST
 ; RUN: llc < %s -mtriple=thumbv7m-linux-gnueabi -mcpu=cortex-m4 -enable-sign-dependent-rounding-fp-math | FileCheck %s --check-prefix=DYN-ROUNDING
 ; RUN: llc < %s -mtriple=thumbv7em-linux-gnueabi -mcpu=cortex-m7 -mattr=-vfp2 | FileCheck %s --check-prefix=CORTEX-M7 --check-prefix=CORTEX-M7-SOFT
-; RUN: llc < %s -mtriple=thumbv7em-linux-gnueabi -mcpu=cortex-m7 -mattr=-vfp2  -enable-unsafe-fp-math -disable-fp-elim -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=CORTEX-M7-NOFPU-FAST
+; RUN: llc < %s -mtriple=thumbv7em-linux-gnueabi -mcpu=cortex-m7 -mattr=-vfp2  -enable-unsafe-fp-math -frame-pointer=all -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=CORTEX-M7-NOFPU-FAST
 ; RUN: llc < %s -mtriple=thumbv7em-linux-gnueabi -mcpu=cortex-m7 -mattr=+fp-only-sp | FileCheck %s --check-prefix=CORTEX-M7 --check-prefix=CORTEX-M7-SINGLE
-; RUN: llc < %s -mtriple=thumbv7em-linux-gnueabi -mcpu=cortex-m7 -mattr=+fp-only-sp  -enable-unsafe-fp-math -disable-fp-elim -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=CORTEX-M7-FAST
+; RUN: llc < %s -mtriple=thumbv7em-linux-gnueabi -mcpu=cortex-m7 -mattr=+fp-only-sp  -enable-unsafe-fp-math -frame-pointer=all -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=CORTEX-M7-FAST
 ; RUN: llc < %s -mtriple=thumbv7em-linux-gnueabi -mcpu=cortex-m7 | FileCheck %s --check-prefix=CORTEX-M7-DOUBLE
 ; RUN: llc < %s -mtriple=thumbv7em-linux-gnueabi -mcpu=cortex-m7 -enable-sign-dependent-rounding-fp-math | FileCheck %s --check-prefix=DYN-ROUNDING
 ; RUN: llc < %s -mtriple=thumbv8-linux-gnueabi -mcpu=cortex-m23 | FileCheck %s --check-prefix=CORTEX-M23
 ; RUN: llc < %s -mtriple=thumbv8-linux-gnueabi -mcpu=cortex-m33 | FileCheck %s --check-prefix=CORTEX-M33
-; RUN: llc < %s -mtriple=thumbv8-linux-gnueabi -mcpu=cortex-m33 -enable-unsafe-fp-math -disable-fp-elim -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=CORTEX-M33-FAST
+; RUN: llc < %s -mtriple=thumbv8-linux-gnueabi -mcpu=cortex-m33 -enable-unsafe-fp-math -frame-pointer=all -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=CORTEX-M33-FAST
 ; RUN: llc < %s -mtriple=thumbv8-linux-gnueabi -mcpu=cortex-m33 -enable-sign-dependent-rounding-fp-math | FileCheck %s --check-prefix=DYN-ROUNDING
 ; RUN: llc < %s -mtriple=armv7r-linux-gnueabi -mcpu=cortex-r4 | FileCheck %s --check-prefix=CORTEX-R4
 ; RUN: llc < %s -mtriple=armv7r-linux-gnueabi -mcpu=cortex-r4f | FileCheck %s --check-prefix=CORTEX-R4F
 ; RUN: llc < %s -mtriple=armv7r-linux-gnueabi -mcpu=cortex-r5 | FileCheck %s --check-prefix=CORTEX-R5
-; RUN: llc < %s -mtriple=armv7r-linux-gnueabi -mcpu=cortex-r5  -enable-unsafe-fp-math -disable-fp-elim -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=CORTEX-R5-FAST
+; RUN: llc < %s -mtriple=armv7r-linux-gnueabi -mcpu=cortex-r5  -enable-unsafe-fp-math -frame-pointer=all -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=CORTEX-R5-FAST
 ; RUN: llc < %s -mtriple=armv7r-linux-gnueabi -mcpu=cortex-r5 -enable-sign-dependent-rounding-fp-math | FileCheck %s --check-prefix=DYN-ROUNDING
 ; RUN: llc < %s -mtriple=armv7r-linux-gnueabi -mcpu=cortex-r7 | FileCheck %s --check-prefix=CORTEX-R7
-; RUN: llc < %s -mtriple=armv7r-linux-gnueabi -mcpu=cortex-r7  -enable-unsafe-fp-math -disable-fp-elim -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=CORTEX-R7-FAST
+; RUN: llc < %s -mtriple=armv7r-linux-gnueabi -mcpu=cortex-r7  -enable-unsafe-fp-math -frame-pointer=all -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=CORTEX-R7-FAST
 ; RUN: llc < %s -mtriple=armv7r-linux-gnueabi -mcpu=cortex-r7 -enable-sign-dependent-rounding-fp-math | FileCheck %s --check-prefix=DYN-ROUNDING
 ; RUN: llc < %s -mtriple=armv7r-linux-gnueabi -mcpu=cortex-r8 | FileCheck %s --check-prefix=CORTEX-R8
-; RUN: llc < %s -mtriple=armv7r-linux-gnueabi -mcpu=cortex-r8  -enable-unsafe-fp-math -disable-fp-elim -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=CORTEX-R8-FAST
+; RUN: llc < %s -mtriple=armv7r-linux-gnueabi -mcpu=cortex-r8  -enable-unsafe-fp-math -frame-pointer=all -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=CORTEX-R8-FAST
 ; RUN: llc < %s -mtriple=armv7r-linux-gnueabi -mcpu=cortex-r8 -enable-sign-dependent-rounding-fp-math | FileCheck %s --check-prefix=DYN-ROUNDING
 ; RUN: llc < %s -mtriple=armv8-linux-gnueabi -mcpu=cortex-a32 | FileCheck %s --check-prefix=CORTEX-A32
-; RUN: llc < %s -mtriple=armv8-linux-gnueabi -mcpu=cortex-a32  -enable-unsafe-fp-math -disable-fp-elim -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=CORTEX-A32-FAST
+; RUN: llc < %s -mtriple=armv8-linux-gnueabi -mcpu=cortex-a32  -enable-unsafe-fp-math -frame-pointer=all -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=CORTEX-A32-FAST
 ; RUN: llc < %s -mtriple=armv8-linux-gnueabi -mcpu=cortex-a32 -enable-sign-dependent-rounding-fp-math | FileCheck %s --check-prefix=DYN-ROUNDING
 ; RUN: llc < %s -mtriple=armv8-linux-gnueabi -mcpu=cortex-a35 | FileCheck %s --check-prefix=CORTEX-A35
-; RUN: llc < %s -mtriple=armv8-linux-gnueabi -mcpu=cortex-a35  -enable-unsafe-fp-math -disable-fp-elim -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=CORTEX-A35-FAST
+; RUN: llc < %s -mtriple=armv8-linux-gnueabi -mcpu=cortex-a35  -enable-unsafe-fp-math -frame-pointer=all -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=CORTEX-A35-FAST
 ; RUN: llc < %s -mtriple=armv8-linux-gnueabi -mcpu=cortex-a35 -enable-sign-dependent-rounding-fp-math | FileCheck %s --check-prefix=DYN-ROUNDING
 ; RUN: llc < %s -mtriple=armv8-linux-gnueabi -mcpu=cortex-a53 | FileCheck %s --check-prefix=CORTEX-A53
-; RUN: llc < %s -mtriple=armv8-linux-gnueabi -mcpu=cortex-a53  -enable-unsafe-fp-math -disable-fp-elim -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=CORTEX-A53-FAST
+; RUN: llc < %s -mtriple=armv8-linux-gnueabi -mcpu=cortex-a53  -enable-unsafe-fp-math -frame-pointer=all -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=CORTEX-A53-FAST
 ; RUN: llc < %s -mtriple=armv8-linux-gnueabi -mcpu=cortex-a53 -enable-sign-dependent-rounding-fp-math | FileCheck %s --check-prefix=DYN-ROUNDING
 ; RUN: llc < %s -mtriple=armv8-linux-gnueabi -mcpu=cortex-a57 | FileCheck %s --check-prefix=CORTEX-A57
-; RUN: llc < %s -mtriple=armv8-linux-gnueabi -mcpu=cortex-a57  -enable-unsafe-fp-math -disable-fp-elim -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=CORTEX-A57-FAST
+; RUN: llc < %s -mtriple=armv8-linux-gnueabi -mcpu=cortex-a57  -enable-unsafe-fp-math -frame-pointer=all -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=CORTEX-A57-FAST
 ; RUN: llc < %s -mtriple=armv8-linux-gnueabi -mcpu=cortex-a57 -enable-sign-dependent-rounding-fp-math | FileCheck %s --check-prefix=DYN-ROUNDING
 ; RUN: llc < %s -mtriple=armv8-linux-gnueabi -mcpu=cortex-a72 | FileCheck %s --check-prefix=CORTEX-A72
-; RUN: llc < %s -mtriple=armv8-linux-gnueabi -mcpu=cortex-a72  -enable-unsafe-fp-math -disable-fp-elim -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=CORTEX-A72-FAST
+; RUN: llc < %s -mtriple=armv8-linux-gnueabi -mcpu=cortex-a72  -enable-unsafe-fp-math -frame-pointer=all -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=CORTEX-A72-FAST
 ; RUN: llc < %s -mtriple=armv8-linux-gnueabi -mcpu=cortex-a72 -enable-sign-dependent-rounding-fp-math | FileCheck %s --check-prefix=DYN-ROUNDING
 ; RUN: llc < %s -mtriple=armv8-linux-gnueabi -mcpu=cortex-a73 | FileCheck %s --check-prefix=CORTEX-A73
 ; RUN: llc < %s -mtriple=armv8.1a-linux-gnueabi | FileCheck %s --check-prefix=GENERIC-ARMV8_1-A
 ; RUN: llc < %s -mtriple=armv8-linux-gnueabi -mcpu=exynos-m1 | FileCheck %s --check-prefix=EXYNOS-M1
-; RUN: llc < %s -mtriple=armv8-linux-gnueabi -mcpu=exynos-m1  -enable-unsafe-fp-math -disable-fp-elim -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=EXYNOS-M1-FAST
+; RUN: llc < %s -mtriple=armv8-linux-gnueabi -mcpu=exynos-m1  -enable-unsafe-fp-math -frame-pointer=all -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=EXYNOS-M1-FAST
 ; RUN: llc < %s -mtriple=armv8-linux-gnueabi -mcpu=exynos-m1 -enable-sign-dependent-rounding-fp-math | FileCheck %s --check-prefix=DYN-ROUNDING
 ; RUN: llc < %s -mtriple=armv8-linux-gnueabi -mcpu=exynos-m2 | FileCheck %s --check-prefix=EXYNOS-M2
-; RUN: llc < %s -mtriple=armv8-linux-gnueabi -mcpu=exynos-m2  -enable-unsafe-fp-math -disable-fp-elim -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=EXYNOS-M1-FAST
+; RUN: llc < %s -mtriple=armv8-linux-gnueabi -mcpu=exynos-m2  -enable-unsafe-fp-math -frame-pointer=all -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=EXYNOS-M1-FAST
 ; RUN: llc < %s -mtriple=armv8-linux-gnueabi -mcpu=exynos-m2 -enable-sign-dependent-rounding-fp-math | FileCheck %s --check-prefix=DYN-ROUNDING
 ; RUN: llc < %s -mtriple=armv8-linux-gnueabi -mcpu=exynos-m3 | FileCheck %s --check-prefix=EXYNOS-M3
-; RUN: llc < %s -mtriple=armv8-linux-gnueabi -mcpu=exynos-m3  -enable-unsafe-fp-math -disable-fp-elim -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=EXYNOS-M1-FAST
+; RUN: llc < %s -mtriple=armv8-linux-gnueabi -mcpu=exynos-m3  -enable-unsafe-fp-math -frame-pointer=all -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=EXYNOS-M1-FAST
 ; RUN: llc < %s -mtriple=armv8-linux-gnueabi -mcpu=exynos-m3 -enable-sign-dependent-rounding-fp-math | FileCheck %s --check-prefix=DYN-ROUNDING
 ; RUN: llc < %s -mtriple=armv8-linux-gnueabi -mcpu=exynos-m4 | FileCheck %s --check-prefix=EXYNOS-M4
-; RUN: llc < %s -mtriple=armv8-linux-gnueabi -mcpu=exynos-m4  -enable-unsafe-fp-math -disable-fp-elim -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=EXYNOS-M1-FAST
+; RUN: llc < %s -mtriple=armv8-linux-gnueabi -mcpu=exynos-m4  -enable-unsafe-fp-math -frame-pointer=all -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=EXYNOS-M1-FAST
 ; RUN: llc < %s -mtriple=armv8-linux-gnueabi -mcpu=exynos-m4 -enable-sign-dependent-rounding-fp-math | FileCheck %s --check-prefix=DYN-ROUNDING
-; RUN: llc < %s -mtriple=armv8.1a-linux-gnueabi  -enable-unsafe-fp-math -disable-fp-elim -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=GENERIC-ARMV8_1-A-FAST
+; RUN: llc < %s -mtriple=armv8.1a-linux-gnueabi  -enable-unsafe-fp-math -frame-pointer=all -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=GENERIC-ARMV8_1-A-FAST
 ; RUN: llc < %s -mtriple=armv8.1a-linux-gnueabi -enable-sign-dependent-rounding-fp-math | FileCheck %s --check-prefix=DYN-ROUNDING
 ; RUN: llc < %s -mtriple=armv7-none-linux-gnueabi -mcpu=cortex-a7 | FileCheck %s  --check-prefix=CORTEX-A7-CHECK
-; RUN: llc < %s -mtriple=armv7-none-linux-gnueabi -mcpu=cortex-a7  -enable-unsafe-fp-math -disable-fp-elim -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s  --check-prefix=CORTEX-A7-CHECK-FAST
+; RUN: llc < %s -mtriple=armv7-none-linux-gnueabi -mcpu=cortex-a7  -enable-unsafe-fp-math -frame-pointer=all -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s  --check-prefix=CORTEX-A7-CHECK-FAST
 ; RUN: llc < %s -mtriple=armv7-none-linux-gnueabi -mcpu=cortex-a7 -mattr=-vfp2,-vfp3,-vfp4,-neon,-fp16 | FileCheck %s --check-prefix=CORTEX-A7-NOFPU
-; RUN: llc < %s -mtriple=armv7-none-linux-gnueabi -mcpu=cortex-a7 -mattr=-vfp2,-vfp3,-vfp4,-neon,-fp16  -enable-unsafe-fp-math -disable-fp-elim -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=CORTEX-A7-NOFPU-FAST
+; RUN: llc < %s -mtriple=armv7-none-linux-gnueabi -mcpu=cortex-a7 -mattr=-vfp2,-vfp3,-vfp4,-neon,-fp16  -enable-unsafe-fp-math -frame-pointer=all -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=CORTEX-A7-NOFPU-FAST
 ; RUN: llc < %s -mtriple=armv7-none-linux-gnueabi -mcpu=cortex-a7 -mattr=+vfp4,-neon | FileCheck %s --check-prefix=CORTEX-A7-FPUV4
 ; RUN: llc < %s -mtriple=armv7-none-linux-gnueabi -mcpu=cortex-a7 -enable-sign-dependent-rounding-fp-math | FileCheck %s --check-prefix=DYN-ROUNDING
-; RUN: llc < %s -mtriple=armv7-none-linux-gnueabi -mcpu=cortex-a7 -mattr=+vfp4,-neon  -enable-unsafe-fp-math -disable-fp-elim -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=CORTEX-A7-FPUV4-FAST
+; RUN: llc < %s -mtriple=armv7-none-linux-gnueabi -mcpu=cortex-a7 -mattr=+vfp4,-neon  -enable-unsafe-fp-math -frame-pointer=all -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=CORTEX-A7-FPUV4-FAST
 ; RUN: llc < %s -mtriple=armv7-none-linux-gnueabi -mcpu=cortex-a7 -mattr=+vfp4,,+d16,-neon | FileCheck %s --check-prefix=CORTEX-A7-FPUV4
 ; RUN: llc < %s -mtriple=arm-none-linux-gnueabi -mattr=+strict-align -relocation-model=pic | FileCheck %s --check-prefix=RELOC-PIC
 ; RUN: llc < %s -mtriple=arm-none-linux-gnueabi -mattr=+strict-align -relocation-model=static | FileCheck %s --check-prefix=RELOC-OTHER
@@ -1623,7 +1623,7 @@
 ; EXYNOS-M4:  .eabi_attribute 8, 1
 ; EXYNOS-M4:  .eabi_attribute 9, 2
 ; EXYNOS-M4:  .fpu crypto-neon-fp-armv8
-; EXYNOS-M4:  .eabi_attribute 12, 3
+; EXYNOS-M4:  .eabi_attribute 12, 4
 ; EXYNOS-M4-NOT:  .eabi_attribute 27
 ; EXYNOS-M4:  .eabi_attribute 36, 1
 ; EXYNOS-M4:  .eabi_attribute 42, 1
diff --git a/test/CodeGen/ARM/constantpool-promote-dbg.ll b/test/CodeGen/ARM/constantpool-promote-dbg.ll
index 4a6a7fe44aee31edd2902f7f949afdb7a26fb1f3..261131032d1828471e123658f464ebb371111130 100644
--- a/test/CodeGen/ARM/constantpool-promote-dbg.ll
+++ b/test/CodeGen/ARM/constantpool-promote-dbg.ll
@@ -25,14 +25,14 @@ attributes #0 = { minsize norecurse nounwind optsize readnone "disable-tail-call
 !llvm.module.flags = !{!3, !4, !5, !6}
 !llvm.ident = !{!7}
 
-!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 3.9.0 (http://llvm.org/git/clang.git 075a2bc2570dfcbb6d6aed6c836e4c62b37afea6)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2)
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 3.9.0", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2)
 !1 = !DIFile(filename: "/Users/jammol01/Code/test.c", directory: "/Users/jammol01/Code/llvm-git/build")
 !2 = !{}
 !3 = !{i32 2, !"Dwarf Version", i32 4}
 !4 = !{i32 2, !"Debug Info Version", i32 3}
 !5 = !{i32 1, !"wchar_size", i32 4}
 !6 = !{i32 1, !"min_enum_size", i32 4}
-!7 = !{!"clang version 3.9.0 (http://llvm.org/git/clang.git 075a2bc2570dfcbb6d6aed6c836e4c62b37afea6)"}
+!7 = !{!"clang version 3.9.0"}
 !8 = distinct !DISubprogram(name: "fn1", scope: !1, file: !1, line: 1, type: !9, isLocal: false, isDefinition: true, scopeLine: 1, isOptimized: true, unit: !0, retainedNodes: !2)
 !9 = !DISubroutineType(types: !10)
 !10 = !{!11}
diff --git a/test/CodeGen/ARM/copy-by-struct-i32.ll b/test/CodeGen/ARM/copy-by-struct-i32.ll
new file mode 100644
index 0000000000000000000000000000000000000000..a57506872f9ef4549d2b2cf9d2e7944e2d830c3c
--- /dev/null
+++ b/test/CodeGen/ARM/copy-by-struct-i32.ll
@@ -0,0 +1,61 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=armv7-unknown-linux < %s -stop-before=expand-isel-pseudos | FileCheck --check-prefix=BEFORE-EXPAND %s
+; RUN: llc -mtriple=armv7-unknown-linux < %s | FileCheck --check-prefix=ASSEMBLY %s
+
+; Check COPY_STRUCT_BYVAL_I32 has CPSR as operand.
+; BEFORE-EXPAND: COPY_STRUCT_BYVAL_I32 {{.*}} implicit-def dead $cpsr
+; BEFORE-EXPAND: COPY_STRUCT_BYVAL_I32 {{.*}} implicit-def dead $cpsr
+
+%struct.anon = type { i32, i32, i32, i32, i32, i32, i32, %struct.f, i32, i64, i32 }
+%struct.f = type { i32, i32, i32, i32, i32 }
+
+define arm_aapcscc void @s(i64* %q, %struct.anon* %p) {
+; ASSEMBLY-LABEL: s:
+; ASSEMBLY:       @ %bb.0: @ %entry
+; ASSEMBLY-NEXT:    push {r4, r5, r11, lr}
+; ASSEMBLY-NEXT:    sub sp, sp, #136
+; ASSEMBLY-NEXT:    ldrd r4, r5, [r0]
+; ASSEMBLY-NEXT:    add lr, sp, #56
+; ASSEMBLY-NEXT:    ldm r1, {r0, r12}
+; ASSEMBLY-NEXT:    subs r4, r4, #1
+; ASSEMBLY-NEXT:    sbc r5, r5, #0
+; ASSEMBLY-NEXT:    ldr r2, [r1, #8]
+; ASSEMBLY-NEXT:    ldr r3, [r1, #12]
+; ASSEMBLY-NEXT:    str r5, [sp, #132]
+; ASSEMBLY-NEXT:    add r5, r1, #16
+; ASSEMBLY-NEXT:    str r4, [sp, #128]
+; ASSEMBLY-NEXT:    mov r4, sp
+; ASSEMBLY-NEXT:    vld1.32 {d16}, [r5]!
+; ASSEMBLY-NEXT:    vst1.32 {d16}, [r4]!
+; ASSEMBLY-NEXT:    vld1.32 {d16}, [r5]!
+; ASSEMBLY-NEXT:    vst1.32 {d16}, [r4]!
+; ASSEMBLY-NEXT:    vld1.32 {d16}, [r5]!
+; ASSEMBLY-NEXT:    vst1.32 {d16}, [r4]!
+; ASSEMBLY-NEXT:    vld1.32 {d16}, [r5]!
+; ASSEMBLY-NEXT:    vst1.32 {d16}, [r4]!
+; ASSEMBLY-NEXT:    vld1.32 {d16}, [r5]!
+; ASSEMBLY-NEXT:    vst1.32 {d16}, [r4]!
+; ASSEMBLY-NEXT:    vld1.32 {d16}, [r5]!
+; ASSEMBLY-NEXT:    vst1.32 {d16}, [r4]!
+; ASSEMBLY-NEXT:    vld1.32 {d16}, [r5]!
+; ASSEMBLY-NEXT:    vst1.32 {d16}, [r4]!
+; ASSEMBLY-NEXT:    movw r4, #72
+; ASSEMBLY-NEXT:  .LBB0_1: @ %entry
+; ASSEMBLY-NEXT:    @ =>This Inner Loop Header: Depth=1
+; ASSEMBLY-NEXT:    vld1.32 {d16}, [r1]!
+; ASSEMBLY-NEXT:    subs r4, r4, #8
+; ASSEMBLY-NEXT:    vst1.32 {d16}, [lr]!
+; ASSEMBLY-NEXT:    bne .LBB0_1
+; ASSEMBLY-NEXT:  @ %bb.2: @ %entry
+; ASSEMBLY-NEXT:    mov r1, r12
+; ASSEMBLY-NEXT:    bl r
+; ASSEMBLY-NEXT:    add sp, sp, #136
+; ASSEMBLY-NEXT:    pop {r4, r5, r11, pc}
+entry:
+  %0 = load i64, i64* %q, align 8
+  %sub = add nsw i64 %0, -1
+  tail call arm_aapcscc void bitcast (void (...)* @r to void (%struct.anon*, %struct.anon*, i64)*)(%struct.anon* byval nonnull align 8 %p, %struct.anon* byval nonnull align 8 %p, i64 %sub)
+  ret void
+}
+
+declare arm_aapcscc void @r(...)
diff --git a/test/CodeGen/ARM/crash-O0.ll b/test/CodeGen/ARM/crash-O0.ll
index bfbab8a99336041cd6369647f335aecf03235bf6..29110fcf3bd27a8e9eecfbb686c3621056139d8b 100644
--- a/test/CodeGen/ARM/crash-O0.ll
+++ b/test/CodeGen/ARM/crash-O0.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -O0 -relocation-model=pic -disable-fp-elim -no-integrated-as
+; RUN: llc < %s -O0 -relocation-model=pic -frame-pointer=all -no-integrated-as
 target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:32-f32:32:32-f64:32:32-v64:64:64-v128:128:128-a0:0:64-n32"
 target triple = "armv6-apple-darwin10"
 
diff --git a/test/CodeGen/ARM/crash-greedy-v6.ll b/test/CodeGen/ARM/crash-greedy-v6.ll
index 287c081ac5ec6b30a5cbb3d122fd0d3abed4ca1f..d3c5057e38214adf58ade3eff6414545f29d40ab 100644
--- a/test/CodeGen/ARM/crash-greedy-v6.ll
+++ b/test/CodeGen/ARM/crash-greedy-v6.ll
@@ -1,5 +1,5 @@
-; RUN: llc -disable-fp-elim -relocation-model=pic < %s
-; RUN: llc -disable-fp-elim -relocation-model=pic -O0 -pre-RA-sched=source < %s | FileCheck %s --check-prefix=SOURCE-SCHED
+; RUN: llc -frame-pointer=all -relocation-model=pic < %s
+; RUN: llc -frame-pointer=all -relocation-model=pic -O0 -pre-RA-sched=source < %s | FileCheck %s --check-prefix=SOURCE-SCHED
 target triple = "armv6-apple-ios"
 
 ; Reduced from 177.mesa. This test causes a live range split before an LDR_POST instruction.
diff --git a/test/CodeGen/ARM/crash-greedy.ll b/test/CodeGen/ARM/crash-greedy.ll
index 5320a163c0b8dace2a394896d6b23947cc37c352..444505f8786a85c3021e5a08e6992626d4afa9c8 100644
--- a/test/CodeGen/ARM/crash-greedy.ll
+++ b/test/CodeGen/ARM/crash-greedy.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -regalloc=greedy -mcpu=cortex-a8 -relocation-model=pic -disable-fp-elim -verify-machineinstrs | FileCheck %s
+; RUN: llc < %s -regalloc=greedy -mcpu=cortex-a8 -relocation-model=pic -frame-pointer=all -verify-machineinstrs | FileCheck %s
 ;
 ; ARM tests that crash or fail with the greedy register allocator.
 
diff --git a/test/CodeGen/ARM/dagcombine-anyexttozeroext.ll b/test/CodeGen/ARM/dagcombine-anyexttozeroext.ll
index 86291aaeaa647abd80a03fe6d6c79a4e2aa53081..de24681470d12a73d0a72bc295098bee11ebc716 100644
--- a/test/CodeGen/ARM/dagcombine-anyexttozeroext.ll
+++ b/test/CodeGen/ARM/dagcombine-anyexttozeroext.ll
@@ -1,50 +1,52 @@
 ; RUN: llc -mtriple armv7 %s -o - | FileCheck %s
 
-; CHECK-LABEL: f:
 define float @f(<4 x i16>* nocapture %in) {
-  ; CHECK: vld1
-  ; CHECK: vmovl.u16
-  ; CHECK-NOT: vand
+; CHECK-LABEL: f:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vld1.16 {d16}, [r0:64]
+; CHECK-NEXT:    vmovl.u16 q8, d16
+; CHECK-NEXT:    vcvt.f32.u32 q0, q8
+; CHECK-NEXT:    vadd.f32 s4, s0, s1
+; CHECK-NEXT:    vadd.f32 s0, s4, s2
+; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    bx lr
   %1 = load <4 x i16>, <4 x i16>* %in
-  ; CHECK: vcvt.f32.u32
   %2 = uitofp <4 x i16> %1 to <4 x float>
   %3 = extractelement <4 x float> %2, i32 0
   %4 = extractelement <4 x float> %2, i32 1
   %5 = extractelement <4 x float> %2, i32 2
 
-  ; CHECK: vadd.f32
   %6 = fadd float %3, %4
   %7 = fadd float %6, %5
 
   ret float %7
 }
 
-; CHECK-LABEL: g:
 define float @g(<4 x i16>* nocapture %in) {
-  ; CHECK: vldr
+; CHECK-LABEL: g:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vldr d16, [r0]
+; CHECK-NEXT:    vmov.u16 r0, d16[0]
+; CHECK-NEXT:    vmov s0, r0
+; CHECK-NEXT:    vcvt.f32.u32 s0, s0
+; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    bx lr
   %1 = load <4 x i16>, <4 x i16>* %in
-
-  ; For now we're generating a vmov.16 and a uxth instruction.
-  ; The uxth is redundant, and we should be able to extend without
-  ; having to generate cross-domain copies. Once we can do this
-  ; we should modify the checks below.
-
-  ; CHECK: uxth
   %2 = extractelement <4 x i16> %1, i32 0
-  ; CHECK: vcvt.f32.u32
   %3 = uitofp i16 %2 to float
   ret float %3
 }
 
 ; Make sure we generate zext from <4 x i8> to <4 x 32>.
-
-; CHECK-LABEL: h:
-; CHECK: vld1.32
-; CHECK: vmovl.u8 q8, d16
-; CHECK: vmovl.u16 q8, d16
-; CHECK: vmov r0, r1, d16
-; CHECK: vmov r2, r3, d17
 define <4 x i32> @h(<4 x i8> *%in) {
+; CHECK-LABEL: h:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vld1.32 {d16[0]}, [r0:32]
+; CHECK-NEXT:    vmovl.u8 q8, d16
+; CHECK-NEXT:    vmovl.u16 q8, d16
+; CHECK-NEXT:    vmov r0, r1, d16
+; CHECK-NEXT:    vmov r2, r3, d17
+; CHECK-NEXT:    bx lr
   %1 = load <4 x i8>, <4 x i8>* %in, align 4
   %2 = extractelement <4 x i8> %1, i32 0
   %3 = zext i8 %2 to i32
@@ -60,3 +62,79 @@ define <4 x i32> @h(<4 x i8> *%in) {
   %13 = insertelement <4 x i32> %10, i32 %12, i32 3
   ret <4 x i32> %13
 }
+
+define float @i(<4 x i16>* nocapture %in) {
+  ; FIXME: The vmov.u + sxt can convert to a vmov.s
+; CHECK-LABEL: i:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vldr d16, [r0]
+; CHECK-NEXT:    vmov.u16 r0, d16[0]
+; CHECK-NEXT:    sxth r0, r0
+; CHECK-NEXT:    vmov s0, r0
+; CHECK-NEXT:    vcvt.f32.s32 s0, s0
+; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    bx lr
+  %1 = load <4 x i16>, <4 x i16>* %in
+  %2 = extractelement <4 x i16> %1, i32 0
+  %3 = sitofp i16 %2 to float
+  ret float %3
+}
+
+define float @j(<8 x i8>* nocapture %in) {
+; CHECK-LABEL: j:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vldr d16, [r0]
+; CHECK-NEXT:    vmov.u8 r0, d16[7]
+; CHECK-NEXT:    vmov s0, r0
+; CHECK-NEXT:    vcvt.f32.u32 s0, s0
+; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    bx lr
+  %1 = load <8 x i8>, <8 x i8>* %in
+  %2 = extractelement <8 x i8> %1, i32 7
+  %3 = uitofp i8 %2 to float
+  ret float %3
+}
+
+define float @k(<8 x i8>* nocapture %in) {
+; FIXME: The vmov.u + sxt can convert to a vmov.s
+; CHECK-LABEL: k:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vldr d16, [r0]
+; CHECK-NEXT:    vmov.u8 r0, d16[7]
+; CHECK-NEXT:    sxtb r0, r0
+; CHECK-NEXT:    vmov s0, r0
+; CHECK-NEXT:    vcvt.f32.s32 s0, s0
+; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    bx lr
+  %1 = load <8 x i8>, <8 x i8>* %in
+  %2 = extractelement <8 x i8> %1, i32 7
+  %3 = sitofp i8 %2 to float
+  ret float %3
+}
+
+define float @KnownUpperZero(<4 x i16> %v) {
+; FIXME: uxtb are not required
+; CHECK-LABEL: KnownUpperZero:
+; CHECK:       @ %bb.0:
+; CHECK-NEXT:    vmov.i16 d16, #0x3
+; CHECK-NEXT:    vmov d17, r0, r1
+; CHECK-NEXT:    vand d16, d17, d16
+; CHECK-NEXT:    vmov.u16 r0, d16[0]
+; CHECK-NEXT:    vmov.u16 r1, d16[3]
+; CHECK-NEXT:    uxtb r0, r0
+; CHECK-NEXT:    vmov s0, r0
+; CHECK-NEXT:    uxtb r0, r1
+; CHECK-NEXT:    vmov s2, r0
+; CHECK-NEXT:    vcvt.f32.s32 s0, s0
+; CHECK-NEXT:    vcvt.f32.s32 s2, s2
+; CHECK-NEXT:    vadd.f32 s0, s2, s0
+; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    bx lr
+  %1 = and <4 x i16> %v, <i16 3,i16 3,i16 3,i16 3>
+  %2 = extractelement <4 x i16> %1, i32 3
+  %3 = extractelement <4 x i16> %1, i32 0
+  %sinf1 = sitofp i16 %2 to float
+  %sinf2 = sitofp i16 %3 to float
+  %sum =   fadd float %sinf1, %sinf2
+  ret float %sum
+}
diff --git a/test/CodeGen/ARM/dbg-range-extension.mir b/test/CodeGen/ARM/dbg-range-extension.mir
index f2b174a8ac0bdb33b3a13cf9cc693f7d5e780eb6..0a48ba83c09fdad65405e20fd9222c76f638f778 100644
--- a/test/CodeGen/ARM/dbg-range-extension.mir
+++ b/test/CodeGen/ARM/dbg-range-extension.mir
@@ -122,7 +122,7 @@
   !4 = !{i32 2, !"Debug Info Version", i32 3}
   !5 = !{i32 1, !"wchar_size", i32 4}
   !6 = !{i32 1, !"min_enum_size", i32 4}
-  !7 = !{!"clang version 4.0.0 (http://llvm.org/git/clang.git b8f10df3679b36f51e1de7c4351b82d297825089) (http://llvm.org/git/llvm.git c2a5d16d1e3b8c49f5bbb1ff87a76ac4f88edb89)"}
+  !7 = !{!"clang version 4.0.0"}
   !8 = distinct !DISubprogram(name: "func", scope: !1, file: !1, line: 2, type: !9, isLocal: false, isDefinition: true, scopeLine: 2, flags: DIFlagPrototyped, isOptimized: true, unit: !0, retainedNodes: !12)
   !9 = !DISubroutineType(types: !10)
   !10 = !{null, !11}
diff --git a/test/CodeGen/ARM/debug-frame-large-stack.ll b/test/CodeGen/ARM/debug-frame-large-stack.ll
index b81672553d8322c7c835e5ed00dd5195f6d3daa1..e5d24253b9888ce6c9cc33273f4d220ed3f07fbc 100644
--- a/test/CodeGen/ARM/debug-frame-large-stack.ll
+++ b/test/CodeGen/ARM/debug-frame-large-stack.ll
@@ -1,4 +1,4 @@
-; RUN: llc -filetype=asm -o - < %s -mtriple arm-arm-netbsd-eabi -disable-fp-elim| FileCheck %s --check-prefix=CHECK-ARM
+; RUN: llc -filetype=asm -o - < %s -mtriple arm-arm-netbsd-eabi -frame-pointer=all| FileCheck %s --check-prefix=CHECK-ARM
 ; RUN: llc -filetype=asm -o - < %s -mtriple arm-arm-netbsd-eabi | FileCheck %s --check-prefix=CHECK-ARM-FP-ELIM
 
 define void @test1() {
diff --git a/test/CodeGen/ARM/debug-frame-no-debug.ll b/test/CodeGen/ARM/debug-frame-no-debug.ll
index 8a07f261f41b0928db1071c57471e6625117ac28..8f3965a32e6201c59b25695f0819c01c1b0a4935 100644
--- a/test/CodeGen/ARM/debug-frame-no-debug.ll
+++ b/test/CodeGen/ARM/debug-frame-no-debug.ll
@@ -8,7 +8,7 @@
 ; RUN:   | FileCheck %s --check-prefix=CHECK-FP-ELIM
 
 ; RUN: llc -mtriple thumb-unknown-linux-gnueabi \
-; RUN:     -disable-fp-elim -filetype=asm -o - %s \
+; RUN:     -frame-pointer=all -filetype=asm -o - %s \
 ; RUN:   | FileCheck %s --check-prefix=CHECK-THUMB-FP
 
 ;-------------------------------------------------------------------------------
diff --git a/test/CodeGen/ARM/debug-frame-vararg.ll b/test/CodeGen/ARM/debug-frame-vararg.ll
index e675647e26ceb1f65747ad76f4f5dc4fb85061ab..c9dcc0b468d92de3c7199b7f601f6c418de8d38b 100644
--- a/test/CodeGen/ARM/debug-frame-vararg.ll
+++ b/test/CodeGen/ARM/debug-frame-vararg.ll
@@ -1,7 +1,7 @@
 ; RUN: llc -mtriple arm-unknown-linux-gnueabi -filetype asm -o - %s | FileCheck %s --check-prefix=CHECK-FP
-; RUN: llc -mtriple arm-unknown-linux-gnueabi -filetype asm -o - %s -disable-fp-elim | FileCheck %s --check-prefix=CHECK-FP-ELIM
+; RUN: llc -mtriple arm-unknown-linux-gnueabi -filetype asm -o - %s -frame-pointer=all | FileCheck %s --check-prefix=CHECK-FP-ELIM
 ; RUN: llc -mtriple thumb-unknown-linux-gnueabi -filetype asm -o - %s | FileCheck %s --check-prefix=CHECK-THUMB-FP
-; RUN: llc -mtriple thumb-unknown-linux-gnueabi -filetype asm -o - %s -disable-fp-elim | FileCheck %s --check-prefix=CHECK-THUMB-FP-ELIM
+; RUN: llc -mtriple thumb-unknown-linux-gnueabi -filetype asm -o - %s -frame-pointer=all | FileCheck %s --check-prefix=CHECK-THUMB-FP-ELIM
 
 ; Tests that the initial space allocated to the varargs on the stack is
 ; taken into account in the .cfi_ directives.
diff --git a/test/CodeGen/ARM/debug-frame.ll b/test/CodeGen/ARM/debug-frame.ll
index f0333634cb5e7a3430dc8c2c9fe188d5fd6fedc8..6efe58afb38ad4e77fdeb184f0e016839fa0ec18 100644
--- a/test/CodeGen/ARM/debug-frame.ll
+++ b/test/CodeGen/ARM/debug-frame.ll
@@ -4,18 +4,18 @@
 ; are properly generated or not.
 
 ; We have to check several cases:
-; (1) arm with -disable-fp-elim
-; (2) arm without -disable-fp-elim
-; (3) armv7 with -disable-fp-elim
-; (4) armv7 without -disable-fp-elim
-; (5) thumb with -disable-fp-elim
-; (6) thumb without -disable-fp-elim
-; (7) thumbv7 with -disable-fp-elim
-; (8) thumbv7 without -disable-fp-elim
+; (1) arm with -frame-pointer=all
+; (2) arm without -frame-pointer=all
+; (3) armv7 with -frame-pointer=all
+; (4) armv7 without -frame-pointer=all
+; (5) thumb with -frame-pointer=all
+; (6) thumb without -frame-pointer=all
+; (7) thumbv7 with -frame-pointer=all
+; (8) thumbv7 without -frame-pointer=all
 ; (9) thumbv7 with -no-integrated-as
 
 ; RUN: llc -mtriple arm-unknown-linux-gnueabi \
-; RUN:     -disable-fp-elim -filetype=asm -o - %s \
+; RUN:     -frame-pointer=all -filetype=asm -o - %s \
 ; RUN:   | FileCheck %s --check-prefix=CHECK-FP
 
 ; RUN: llc -mtriple arm-unknown-linux-gnueabi \
@@ -23,7 +23,7 @@
 ; RUN:   | FileCheck %s --check-prefix=CHECK-FP-ELIM
 
 ; RUN: llc -mtriple armv7-unknown-linux-gnueabi \
-; RUN:     -disable-fp-elim -filetype=asm -o - %s \
+; RUN:     -frame-pointer=all -filetype=asm -o - %s \
 ; RUN:   | FileCheck %s --check-prefix=CHECK-V7-FP
 
 ; RUN: llc -mtriple armv7-unknown-linux-gnueabi \
@@ -31,7 +31,7 @@
 ; RUN:   | FileCheck %s --check-prefix=CHECK-V7-FP-ELIM
 
 ; RUN: llc -mtriple thumbv5-unknown-linux-gnueabi \
-; RUN:     -disable-fp-elim -filetype=asm -o - %s \
+; RUN:     -frame-pointer=all -filetype=asm -o - %s \
 ; RUN:   | FileCheck %s --check-prefix=CHECK-THUMB-FP
 
 ; RUN: llc -mtriple thumbv5-unknown-linux-gnueabi \
@@ -39,7 +39,7 @@
 ; RUN:   | FileCheck %s --check-prefix=CHECK-THUMB-FP-ELIM
 
 ; RUN: llc -mtriple thumbv7-unknown-linux-gnueabi \
-; RUN:     -disable-fp-elim -filetype=asm -o - %s \
+; RUN:     -frame-pointer=all -filetype=asm -o - %s \
 ; RUN:   | FileCheck %s --check-prefix=CHECK-THUMB-V7-FP
 
 ; RUN: llc -mtriple thumbv7-unknown-linux-gnueabi \
@@ -47,7 +47,7 @@
 ; RUN:   | FileCheck %s --check-prefix=CHECK-THUMB-V7-FP-ELIM
 
 ; RUN: llc -mtriple thumbv7-unknown-linux-gnueabi \
-; RUN:     -disable-fp-elim -no-integrated-as -filetype=asm -o - %s \
+; RUN:     -frame-pointer=all -no-integrated-as -filetype=asm -o - %s \
 ; RUN:   | FileCheck %s --check-prefix=CHECK-THUMB-V7-FP-NOIAS
 
 ;-------------------------------------------------------------------------------
diff --git a/test/CodeGen/ARM/disable-fp-elim.ll b/test/CodeGen/ARM/disable-fp-elim.ll
index dafeda2ac76215b86c4805710f52400bd4345771..ddbe36597e17fe8bfc40d3b6a556ed556bbf67a9 100644
--- a/test/CodeGen/ARM/disable-fp-elim.ll
+++ b/test/CodeGen/ARM/disable-fp-elim.ll
@@ -1,9 +1,9 @@
 ; RUN: llc < %s -mtriple armv7-none-linux-gnueabi -O1 | FileCheck %s --check-prefix=DISABLE-FP-ELIM
-; RUN: llc < %s -mtriple armv7-none-linux-gnueabi -disable-fp-elim -O1 | FileCheck %s --check-prefix=DISABLE-FP-ELIM
-; RUN: llc < %s -mtriple armv7-none-linux-gnueabi -disable-fp-elim=false -O1 | FileCheck %s --check-prefix=ENABLE-FP-ELIM
-; RUN: llc < %s -mtriple armv7-none-linux-gnueabi -disable-fp-elim=false -O0 | FileCheck %s --check-prefix=DISABLE-FP-ELIM
+; RUN: llc < %s -mtriple armv7-none-linux-gnueabi -frame-pointer=all -O1 | FileCheck %s --check-prefix=DISABLE-FP-ELIM
+; RUN: llc < %s -mtriple armv7-none-linux-gnueabi -frame-pointer=none -O1 | FileCheck %s --check-prefix=ENABLE-FP-ELIM
+; RUN: llc < %s -mtriple armv7-none-linux-gnueabi -frame-pointer=none -O0 | FileCheck %s --check-prefix=DISABLE-FP-ELIM
 
-; Check that command line option "-disable-fp-elim" overrides function attribute
+; Check that command line option "-frame-pointer=all" overrides function attribute
 ; "no-frame-pointer-elim". Also, check frame pointer elimination is disabled
 ; when fast-isel is used.
 
diff --git a/test/CodeGen/ARM/dsp-loop-indexing.ll b/test/CodeGen/ARM/dsp-loop-indexing.ll
new file mode 100644
index 0000000000000000000000000000000000000000..7b80b400af40922fce18af27329e512b21031584
--- /dev/null
+++ b/test/CodeGen/ARM/dsp-loop-indexing.ll
@@ -0,0 +1,310 @@
+; RUN: llc -mtriple=thumbv7em -mattr=+fp-armv8 %s -o - | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-DEFAULT
+; RUN: llc -mtriple=thumbv8m.main -mattr=+fp-armv8,+dsp %s -o - | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-DEFAULT
+; RUN: llc -mtriple=thumbv8m.main -mattr=+fp-armv8,+dsp -lsr-backedge-indexing=false %s -o - | FileCheck %s --check-prefix=CHECK --check-prefix=DISABLED
+; RUN: llc -mtriple=thumbv8 %s -o - | FileCheck %s --check-prefix=CHECK --check-prefix=DISABLED
+; RUN: llc -mtriple=thumbv8m.main -mattr=+fp-armv8,+dsp -lsr-complexity-limit=2147483647 %s -o - | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-COMPLEX
+
+; CHECK-LABEL: test_qadd_2
+; CHECK: @ %loop
+
+; CHECK-DEFAULT: ldr{{.*}}, #4]
+; CHECK-DEFAULT: ldr{{.*}}, #4]
+; CHECK-DEFAULT: str{{.*}}, #4]
+; CHECK-DEFAULT: ldr{{.*}}, #8]!
+; CHECK-DEAFULT: ldr{{.*}}, #8]!
+; CHECK-DEFAULT: str{{.*}}, #8]!
+
+; CHECK-COMPLEX: ldr{{.*}}, #8]!
+; CHECK-COMPLEX: ldr{{.*}}, #8]!
+; CHECK-COMPLEX: str{{.*}}, #8]!
+; CHECK-COMPLEX: ldr{{.*}}, #4]
+; CHECK-COMPLEX: ldr{{.*}}, #4]
+; CHECK-COMPLEX: str{{.*}}, #4]
+
+; DISABLED-NOT: ldr{{.*}}]!
+; DISABLED-NOT: str{{.*}}]!
+
+define void @test_qadd_2(i32* %a.array, i32* %b.array, i32* %out.array, i32 %N) {
+entry:
+  br label %loop
+
+loop:
+  %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+  %idx.1 = phi i32 [ 0, %entry ], [ %idx.next, %loop ]
+  %gep.a.1 = getelementptr inbounds i32, i32* %a.array, i32 %idx.1
+  %a.1 = load i32, i32* %gep.a.1
+  %gep.b.1 = getelementptr inbounds i32, i32* %b.array, i32 %idx.1
+  %b.1 = load i32, i32* %gep.b.1
+  %qadd.1 = call i32 @llvm.arm.qadd(i32 %a.1, i32 %b.1)
+  %addr.1 = getelementptr inbounds i32, i32* %out.array, i32 %idx.1
+  store i32 %qadd.1, i32* %addr.1
+  %idx.2 = or i32 %idx.1, 1
+  %gep.a.2 = getelementptr inbounds i32, i32* %a.array, i32 %idx.2
+  %a.2 = load i32, i32* %gep.a.2
+  %gep.b.2 = getelementptr inbounds i32, i32* %b.array, i32 %idx.2
+  %b.2 = load i32, i32* %gep.b.2
+  %qadd.2 = call i32 @llvm.arm.qadd(i32 %a.2, i32 %b.2)
+  %addr.2 = getelementptr inbounds i32, i32* %out.array, i32 %idx.2
+  store i32 %qadd.2, i32* %addr.2
+  %i.next = add nsw nuw i32 %i, -2
+  %idx.next = add nsw nuw i32 %idx.1, 2
+  %cmp = icmp ult i32 %i.next, %N
+  br i1 %cmp, label %loop, label %exit
+
+exit:
+  ret void
+}
+
+; CHECK-LABEL: test_qadd_2_backwards
+; TODO: Indexes should be generated.
+
+; CHECK: @ %loop
+
+; CHECK-DEFAULT: ldr{{.*}},
+; CHECK-DEFAULT: ldr{{.*}},
+; CHECK-DEFAULT: str{{.*}},
+; CHECK-DEFAULT: ldr{{.*}}, #-4]
+; CHECK-DEFAULT: ldr{{.*}}, #-4]
+; CHECK-DEFAULT: sub{{.*}}, #8
+; CHECK-DEFAULT: str{{.*}}, #-4]
+; CHECK-DEFAULT: sub{{.*}}, #8
+
+; CHECK-COMPLEX: ldr{{.*}} lsl #2]
+; CHECK-COMPLEX: ldr{{.*}} lsl #2]
+; CHECK-COMPLEX: str{{.*}} lsl #2]
+; CHECK-COMPLEX: ldr{{.*}} lsl #2]
+; CHECK-COMPLEX: ldr{{.*}} lsl #2]
+; CHECK-COMPLEX: str{{.*}} lsl #2]
+
+; DISABLED-NOT: ldr{{.*}}]!
+; DISABLED-NOT: str{{.*}}]!
+
+define void @test_qadd_2_backwards(i32* %a.array, i32* %b.array, i32* %out.array, i32 %N) {
+entry:
+  br label %loop
+
+loop:
+  %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+  %idx.1 = phi i32 [ %N, %entry ], [ %idx.next, %loop ]
+  %gep.a.1 = getelementptr inbounds i32, i32* %a.array, i32 %idx.1
+  %a.1 = load i32, i32* %gep.a.1
+  %gep.b.1 = getelementptr inbounds i32, i32* %b.array, i32 %idx.1
+  %b.1 = load i32, i32* %gep.b.1
+  %qadd.1 = call i32 @llvm.arm.qadd(i32 %a.1, i32 %b.1)
+  %addr.1 = getelementptr inbounds i32, i32* %out.array, i32 %idx.1
+  store i32 %qadd.1, i32* %addr.1
+  %idx.2 = sub nsw nuw i32 %idx.1, 1
+  %gep.a.2 = getelementptr inbounds i32, i32* %a.array, i32 %idx.2
+  %a.2 = load i32, i32* %gep.a.2
+  %gep.b.2 = getelementptr inbounds i32, i32* %b.array, i32 %idx.2
+  %b.2 = load i32, i32* %gep.b.2
+  %qadd.2 = call i32 @llvm.arm.qadd(i32 %a.2, i32 %b.2)
+  %addr.2 = getelementptr inbounds i32, i32* %out.array, i32 %idx.2
+  store i32 %qadd.2, i32* %addr.2
+  %i.next = add nsw nuw i32 %i, -2
+  %idx.next = sub nsw nuw i32 %idx.1, 2
+  %cmp = icmp ult i32 %i.next, %N
+  br i1 %cmp, label %loop, label %exit
+
+exit:
+  ret void
+}
+
+; CHECK-LABEL: test_qadd_3
+; CHECK: @ %loop
+
+; CHECK-DEFAULT: ldr{{.*}}, #8]
+; CHECK-DEFAULT: ldr{{.*}}, #8]
+; CHECK-DEFAULT: str{{.*}}, #8]
+; CHECK-DEFAULT: ldr{{.*}}, #12]!
+; CHECK-DEFAULT: ldr{{.*}}, #12]!
+; CHECK-DEFAULT: str{{.*}}, #12]!
+
+; CHECK-COMPLEX: ldr{{.*}}, #12]!
+; CHECK-COMPLEX: ldr{{.*}}, #12]!
+; CHECK-COMPLEX: str{{.*}}, #12]!
+; CHECK-COMPLEX: ldr{{.*}}, #4]
+; CHECK-COMPLEX: ldr{{.*}}, #4]
+; CHECK-COMPLEX: str{{.*}}, #4]
+; CHECK-COMPLEX: ldr{{.*}}, #8]
+; CHECK-COMPLEX: ldr{{.*}}, #8]
+; CHECK-COMPLEX: str{{.*}}, #8]
+
+; DISABLED-NOT: ldr{{.*}}]!
+; DISABLED-NOT: str{{.*}}]!
+
+define void @test_qadd_3(i32* %a.array, i32* %b.array, i32* %out.array, i32 %N) {
+entry:
+  br label %loop
+
+loop:
+  %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+  %idx.1 = phi i32 [ 0, %entry ], [ %idx.next, %loop ]
+  %gep.a.1 = getelementptr inbounds i32, i32* %a.array, i32 %idx.1
+  %a.1 = load i32, i32* %gep.a.1
+  %gep.b.1 = getelementptr inbounds i32, i32* %b.array, i32 %idx.1
+  %b.1 = load i32, i32* %gep.b.1
+  %qadd.1 = call i32 @llvm.arm.qadd(i32 %a.1, i32 %b.1)
+  %addr.1 = getelementptr inbounds i32, i32* %out.array, i32 %idx.1
+  store i32 %qadd.1, i32* %addr.1
+  %idx.2 = add nuw nsw i32 %idx.1, 1
+  %gep.a.2 = getelementptr inbounds i32, i32* %a.array, i32 %idx.2
+  %a.2 = load i32, i32* %gep.a.2
+  %gep.b.2 = getelementptr inbounds i32, i32* %b.array, i32 %idx.2
+  %b.2 = load i32, i32* %gep.b.2
+  %qadd.2 = call i32 @llvm.arm.qadd(i32 %a.2, i32 %b.2)
+  %addr.2 = getelementptr inbounds i32, i32* %out.array, i32 %idx.2
+  store i32 %qadd.2, i32* %addr.2
+  %idx.3 = add nuw nsw i32 %idx.1, 2
+  %gep.a.3 = getelementptr inbounds i32, i32* %a.array, i32 %idx.3
+  %a.3 = load i32, i32* %gep.a.3
+  %gep.b.3 = getelementptr inbounds i32, i32* %b.array, i32 %idx.3
+  %b.3 = load i32, i32* %gep.b.3
+  %qadd.3 = call i32 @llvm.arm.qadd(i32 %a.3, i32 %b.3)
+  %addr.3 = getelementptr inbounds i32, i32* %out.array, i32 %idx.3
+  store i32 %qadd.3, i32* %addr.3
+  %i.next = add nsw nuw i32 %i, -3
+  %idx.next = add nsw nuw i32 %idx.1, 3
+  %cmp = icmp ult i32 %i.next, %N
+  br i1 %cmp, label %loop, label %exit
+
+exit:
+  ret void
+}
+
+; CHECK-LABEL: test_qadd_4
+; CHECK: @ %loop
+
+; TODO: pre-inc store
+
+; CHECK-DEFAULT: ldr{{.*}}, #4]
+; CHECK-DEFAULT: ldr{{.*}}, #4]
+; CHECK-DEFAULT: str{{.*}}, #4]
+; CHECK-DEFAULT: ldr{{.*}}, #8]
+; CHECK-DEFAULT: ldr{{.*}}, #8]
+; CHECK-DEFAULT: str{{.*}}, #8]
+; CHECK-DEFAULT: ldr{{.*}}, #12]
+; CHECK-DEFAULT: ldr{{.*}}, #12]
+; CHECK-DEFAULT: str{{.*}}, #12]
+
+; CHECK-COMPLEX: ldr{{.*}}, #16]!
+; CHECK-COMPLEX: ldr{{.*}}, #16]!
+; CHECK-COMPLEX: str{{.*}}, #16]!
+; CHECK-COMPLEX: ldr{{.*}}, #4]
+; CHECK-COMPLEX: ldr{{.*}}, #4]
+; CHECK-COMPLEX: str{{.*}}, #4]
+; CHECK-COMPLEX: ldr{{.*}}, #8]
+; CHECK-COMPLEX: ldr{{.*}}, #8]
+; CHECK-COMPLEX: str{{.*}}, #8]
+; CHECK-COMPLEX: ldr{{.*}}, #12]
+; CHECK-COMPLEX: ldr{{.*}}, #12]
+; CHECK-COMPLEX: str{{.*}}, #12]
+
+; DISABLED-NOT: ldr{{.*}}]!
+; DISABLED-NOT: str{{.*}}]!
+
+define void @test_qadd_4(i32* %a.array, i32* %b.array, i32* %out.array, i32 %N) {
+entry:
+  br label %loop
+
+loop:
+  %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+  %idx.1 = phi i32 [ 0, %entry ], [ %idx.next, %loop ]
+  %gep.a.1 = getelementptr inbounds i32, i32* %a.array, i32 %idx.1
+  %a.1 = load i32, i32* %gep.a.1
+  %gep.b.1 = getelementptr inbounds i32, i32* %b.array, i32 %idx.1
+  %b.1 = load i32, i32* %gep.b.1
+  %qadd.1 = call i32 @llvm.arm.qadd(i32 %a.1, i32 %b.1)
+  %addr.1 = getelementptr inbounds i32, i32* %out.array, i32 %idx.1
+  store i32 %qadd.1, i32* %addr.1
+  %idx.2 = or i32 %idx.1, 1
+  %gep.a.2 = getelementptr inbounds i32, i32* %a.array, i32 %idx.2
+  %a.2 = load i32, i32* %gep.a.2
+  %gep.b.2 = getelementptr inbounds i32, i32* %b.array, i32 %idx.2
+  %b.2 = load i32, i32* %gep.b.2
+  %qadd.2 = call i32 @llvm.arm.qadd(i32 %a.2, i32 %b.2)
+  %addr.2 = getelementptr inbounds i32, i32* %out.array, i32 %idx.2
+  store i32 %qadd.2, i32* %addr.2
+  %idx.3 = or i32 %idx.1, 2
+  %gep.a.3 = getelementptr inbounds i32, i32* %a.array, i32 %idx.3
+  %a.3 = load i32, i32* %gep.a.3
+  %gep.b.3 = getelementptr inbounds i32, i32* %b.array, i32 %idx.3
+  %b.3 = load i32, i32* %gep.b.3
+  %qadd.3 = call i32 @llvm.arm.qadd(i32 %a.3, i32 %b.3)
+  %addr.3 = getelementptr inbounds i32, i32* %out.array, i32 %idx.3
+  store i32 %qadd.3, i32* %addr.3
+  %idx.4 = or i32 %idx.1, 3
+  %gep.a.4 = getelementptr inbounds i32, i32* %a.array, i32 %idx.4
+  %a.4 = load i32, i32* %gep.a.4
+  %gep.b.4 = getelementptr inbounds i32, i32* %b.array, i32 %idx.4
+  %b.4 = load i32, i32* %gep.b.4
+  %qadd.4 = call i32 @llvm.arm.qadd(i32 %a.4, i32 %b.4)
+  %addr.4 = getelementptr inbounds i32, i32* %out.array, i32 %idx.4
+  store i32 %qadd.4, i32* %addr.4
+  %i.next = add nsw nuw i32 %i, -4
+  %idx.next = add nsw nuw i32 %idx.1, 4
+  %cmp = icmp ult i32 %i.next, %N
+  br i1 %cmp, label %loop, label %exit
+
+exit:
+  ret void
+}
+
+; CHECK-LABEL: test_qadd16_2
+; CHECK: @ %loop
+; TODO: pre-inc store.
+
+; CHECK-DEFAULT: ldr{{.*}}, #4]
+; CHECK-DEFAULT: ldr{{.*}}, #4]
+; CHECK-DEFAULT: str{{.*}}, #8]
+; CHECK-DEFAULT: ldr{{.*}}, #8]!
+; CHECK-DEFAULT: ldr{{.*}}, #8]!
+; CHECK-DEFAULT: str{{.*}}, #16]!
+
+; CHECK-COMPLEX: ldr{{.*}}, #8]!
+; CHECK-COMPLEX: ldr{{.*}}, #8]!
+; CHECK-COMPLEX: str{{.*}}, #16]!
+; CHECK-COMPLEX: ldr{{.*}}, #4]
+; CHECK-COMPLEX: ldr{{.*}}, #4]
+; CHECK-COMPLEX: str{{.*}}, #8]
+
+; DISABLED-NOT: ldr{{.*}}]!
+; DISABLED-NOT: str{{.*}}]!
+
+define void @test_qadd16_2(i16* %a.array, i16* %b.array, i32* %out.array, i32 %N) {
+entry:
+  br label %loop
+
+loop:
+  %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+  %idx.1 = phi i32 [ 0, %entry ], [ %idx.next, %loop ]
+  %gep.a.1 = getelementptr inbounds i16, i16* %a.array, i32 %idx.1
+  %cast.a.1 = bitcast i16* %gep.a.1 to i32*
+  %a.1 = load i32, i32* %cast.a.1
+  %gep.b.1 = getelementptr inbounds i16, i16* %b.array, i32 %idx.1
+  %cast.b.1 = bitcast i16* %gep.b.1 to i32*
+  %b.1 = load i32, i32* %cast.b.1
+  %qadd.1 = call i32 @llvm.arm.qadd16(i32 %a.1, i32 %b.1)
+  %addr.1 = getelementptr inbounds i32, i32* %out.array, i32 %idx.1
+  store i32 %qadd.1, i32* %addr.1
+  %idx.2 = add nsw nuw i32 %idx.1, 2
+  %gep.a.2 = getelementptr inbounds i16, i16* %a.array, i32 %idx.2
+  %cast.a.2 = bitcast i16* %gep.a.2 to i32*
+  %a.2 = load i32, i32* %cast.a.2
+  %gep.b.2 = getelementptr inbounds i16, i16* %b.array, i32 %idx.2
+  %cast.b.2 = bitcast i16* %gep.b.2 to i32*
+  %b.2 = load i32, i32* %cast.b.2
+  %qadd.2 = call i32 @llvm.arm.qadd16(i32 %a.2, i32 %b.2)
+  %addr.2 = getelementptr inbounds i32, i32* %out.array, i32 %idx.2
+  store i32 %qadd.2, i32* %addr.2
+  %i.next = add nsw nuw i32 %i, -2
+  %idx.next = add nsw nuw i32 %idx.1, 4
+  %cmp = icmp ult i32 %i.next, %N
+  br i1 %cmp, label %loop, label %exit
+
+exit:
+  ret void
+}
+
+declare i32 @llvm.arm.qadd(i32, i32)
+declare i32 @llvm.arm.qadd16(i32, i32)
diff --git a/test/CodeGen/ARM/ehabi-unwind.ll b/test/CodeGen/ARM/ehabi-unwind.ll
index a86f340d74e621911728c3ee81c2906af3e709be..57d3eda921c177b80428a1de192341cfbd2931b5 100644
--- a/test/CodeGen/ARM/ehabi-unwind.ll
+++ b/test/CodeGen/ARM/ehabi-unwind.ll
@@ -1,6 +1,6 @@
 ; Test that the EHABI unwind instruction generator does not encounter any
 ; unfamiliar instructions.
-; RUN: llc < %s -mtriple=thumbv7 -disable-fp-elim
+; RUN: llc < %s -mtriple=thumbv7 -frame-pointer=all
 ; RUN: llc < %s -mtriple=thumbv7
 
 define void @_Z1fv() nounwind {
diff --git a/test/CodeGen/ARM/ehabi.ll b/test/CodeGen/ARM/ehabi.ll
index f5a433b14bf1b4a95cc76755343e708810cea80b..5c4a2b620a1f019f817acf7a1e10e059dd828d70 100644
--- a/test/CodeGen/ARM/ehabi.ll
+++ b/test/CodeGen/ARM/ehabi.ll
@@ -13,13 +13,13 @@
 ;     nounwind function attribute.
 
 ; We have to check several cases:
-; (1) arm with -disable-fp-elim
-; (2) arm without -disable-fp-elim
-; (3) armv7 with -disable-fp-elim
-; (4) armv7 without -disable-fp-elim
+; (1) arm with -frame-pointer=all
+; (2) arm without -frame-pointer=all
+; (3) armv7 with -frame-pointer=all
+; (4) armv7 without -frame-pointer=all
 
 ; RUN: llc -mtriple arm-unknown-linux-gnueabi \
-; RUN:     -disable-fp-elim -filetype=asm -o - %s \
+; RUN:     -frame-pointer=all -filetype=asm -o - %s \
 ; RUN:   | FileCheck %s --check-prefix=CHECK-FP
 
 ; RUN: llc -mtriple arm-unknown-linux-gnueabi \
@@ -27,7 +27,7 @@
 ; RUN:   | FileCheck %s --check-prefix=CHECK-FP-ELIM
 
 ; RUN: llc -mtriple armv7-unknown-linux-gnueabi \
-; RUN:     -disable-fp-elim -filetype=asm -o - %s \
+; RUN:     -frame-pointer=all -filetype=asm -o - %s \
 ; RUN:   | FileCheck %s --check-prefix=CHECK-V7-FP
 
 ; RUN: llc -mtriple armv7-unknown-linux-gnueabi \
@@ -35,7 +35,7 @@
 ; RUN:   | FileCheck %s --check-prefix=CHECK-V7-FP-ELIM
 
 ; RUN: llc -mtriple arm-unknown-linux-musleabi \
-; RUN:     -disable-fp-elim -filetype=asm -o - %s \
+; RUN:     -frame-pointer=all -filetype=asm -o - %s \
 ; RUN:   | FileCheck %s --check-prefix=CHECK-FP
 
 ; RUN: llc -mtriple arm-unknown-linux-musleabi \
@@ -43,7 +43,7 @@
 ; RUN:   | FileCheck %s --check-prefix=CHECK-FP-ELIM
 
 ; RUN: llc -mtriple armv7-unknown-linux-musleabi \
-; RUN:     -disable-fp-elim -filetype=asm -o - %s \
+; RUN:     -frame-pointer=all -filetype=asm -o - %s \
 ; RUN:   | FileCheck %s --check-prefix=CHECK-V7-FP
 
 ; RUN: llc -mtriple armv7-unknown-linux-musleabi \
@@ -51,7 +51,7 @@
 ; RUN:   | FileCheck %s --check-prefix=CHECK-V7-FP-ELIM
 
 ; RUN: llc -mtriple arm-unknown-linux-androideabi \
-; RUN:     -disable-fp-elim -filetype=asm -o - %s \
+; RUN:     -frame-pointer=all -filetype=asm -o - %s \
 ; RUN:   | FileCheck %s --check-prefix=CHECK-FP
 
 ; RUN: llc -mtriple arm-unknown-linux-androideabi \
@@ -59,7 +59,7 @@
 ; RUN:   | FileCheck %s --check-prefix=CHECK-FP-ELIM
 
 ; RUN: llc -mtriple armv7-unknown-linux-androideabi \
-; RUN:     -disable-fp-elim -filetype=asm -o - %s \
+; RUN:     -frame-pointer=all -filetype=asm -o - %s \
 ; RUN:   | FileCheck %s --check-prefix=CHECK-V7-FP
 
 ; RUN: llc -mtriple armv7-unknown-linux-androideabi \
@@ -67,7 +67,7 @@
 ; RUN:   | FileCheck %s --check-prefix=CHECK-V7-FP-ELIM
 
 ; RUN: llc -mtriple arm-unknown-netbsd-eabi \
-; RUN:     -disable-fp-elim -filetype=asm -o - %s \
+; RUN:     -frame-pointer=all -filetype=asm -o - %s \
 ; RUN:   | FileCheck %s --check-prefix=DWARF-FP
 
 ; RUN: llc -mtriple arm-unknown-netbsd-eabi \
@@ -75,7 +75,7 @@
 ; RUN:   | FileCheck %s --check-prefix=DWARF-FP-ELIM
 
 ; RUN: llc -mtriple armv7-unknown-netbsd-eabi \
-; RUN:     -disable-fp-elim -filetype=asm -o - %s \
+; RUN:     -frame-pointer=all -filetype=asm -o - %s \
 ; RUN:   | FileCheck %s --check-prefix=DWARF-V7-FP
 
 ; RUN: llc -mtriple armv7-unknown-netbsd-eabi \
diff --git a/test/CodeGen/ARM/fold-stack-adjust.ll b/test/CodeGen/ARM/fold-stack-adjust.ll
index eb32ee54c09594c5059f1853adc64b880bbf8122..6b86c6a9d23044bf38a26d314fa1d4dae04b9686 100644
--- a/test/CodeGen/ARM/fold-stack-adjust.ll
+++ b/test/CodeGen/ARM/fold-stack-adjust.ll
@@ -1,9 +1,9 @@
 ; Disable shrink-wrapping on the first test otherwise we wouldn't
 ; exerce the path for PR18136.
 ; RUN: llc -mtriple=thumbv7-apple-none-macho < %s -enable-shrink-wrap=false | FileCheck %s
-; RUN: llc -mtriple=thumbv6m-apple-none-macho -disable-fp-elim < %s | FileCheck %s --check-prefix=CHECK-T1
-; RUN: llc -mtriple=thumbv7-apple-darwin-ios -disable-fp-elim < %s | FileCheck %s --check-prefix=CHECK-IOS
-; RUN: llc -mtriple=thumbv7--linux-gnueabi -disable-fp-elim < %s | FileCheck %s --check-prefix=CHECK-LINUX
+; RUN: llc -mtriple=thumbv6m-apple-none-macho -frame-pointer=all < %s | FileCheck %s --check-prefix=CHECK-T1
+; RUN: llc -mtriple=thumbv7-apple-darwin-ios -frame-pointer=all < %s | FileCheck %s --check-prefix=CHECK-IOS
+; RUN: llc -mtriple=thumbv7--linux-gnueabi -frame-pointer=all < %s | FileCheck %s --check-prefix=CHECK-LINUX
 
 
 declare void @bar(i8*)
diff --git a/test/CodeGen/ARM/fp16-litpool3-arm.mir b/test/CodeGen/ARM/fp16-litpool3-arm.mir
index 6d27d9619c13f00bf4d635dd42da0fb9bf18ddee..049a06dfb070b388921ce57723fb5695b9db3a32 100644
--- a/test/CodeGen/ARM/fp16-litpool3-arm.mir
+++ b/test/CodeGen/ARM/fp16-litpool3-arm.mir
@@ -82,11 +82,11 @@ constants:
 #CHECK:    CONSTPOOL_ENTRY 1, %const{{.*}}, 2
 #
 # We don't want to decrease alignment if the block already has been
-# aligned; this can e.g. be an existing CPE that has been carefully 
-# aligned. Here BB.1.LA has already an 8-byte alignment, and we are 
+# aligned; this can e.g. be an existing CPE that has been carefully
+# aligned. Here BB.1.LA has already an 8-byte alignment, and we are
 # checking we don't set it to 4:
 #
-#CHECK:  bb.{{.*}}.LA (align 3):
+#CHECK:  bb.{{.*}}.LA (align 8):
 
 body:             |
   bb.0.entry:
@@ -100,7 +100,7 @@ body:             |
     FMSTAT 14, $noreg, implicit-def $cpsr, implicit killed $fpscr_nzcv
     Bcc %bb.2, 0, killed $cpsr
 
-  bb.1.LA (align 3):
+  bb.1.LA (align 8):
     successors: %bb.2(0x80000000)
 
     dead renamable $r0 = SPACE 1000, undef renamable $r0
diff --git a/test/CodeGen/ARM/frame-register.ll b/test/CodeGen/ARM/frame-register.ll
index 0cc5005ec488d7c4537569fe4053e176275af92b..c008b21a2909ce0401a187cea6dafd0a6d83ff49 100644
--- a/test/CodeGen/ARM/frame-register.ll
+++ b/test/CodeGen/ARM/frame-register.ll
@@ -1,13 +1,13 @@
-; RUN: llc -mtriple arm-eabi -disable-fp-elim -filetype asm -o - %s \
+; RUN: llc -mtriple arm-eabi -frame-pointer=all -filetype asm -o - %s \
 ; RUN:     | FileCheck -check-prefix CHECK-ARM %s
 
-; RUN: llc -mtriple thumb-eabi -disable-fp-elim -filetype asm -o - %s \
+; RUN: llc -mtriple thumb-eabi -frame-pointer=all -filetype asm -o - %s \
 ; RUN:     | FileCheck -check-prefix CHECK-THUMB %s
 
-; RUN: llc -mtriple arm-darwin -disable-fp-elim -filetype asm -o - %s \
+; RUN: llc -mtriple arm-darwin -frame-pointer=all -filetype asm -o - %s \
 ; RUN:     | FileCheck -check-prefix CHECK-DARWIN-ARM %s
 
-; RUN: llc -mtriple thumb-darwin -disable-fp-elim -filetype asm -o - %s \
+; RUN: llc -mtriple thumb-darwin -frame-pointer=all -filetype asm -o - %s \
 ; RUN:     | FileCheck -check-prefix CHECK-DARWIN-THUMB %s
 
 declare void @callee(i32)
diff --git a/test/CodeGen/ARM/hello.ll b/test/CodeGen/ARM/hello.ll
index 264105994935940c4625ed12e6e07c93d299aa87..bdeb41decc84e4b2c77cf48473a65c8285cc46f8 100644
--- a/test/CodeGen/ARM/hello.ll
+++ b/test/CodeGen/ARM/hello.ll
@@ -1,7 +1,7 @@
 ; RUN: llc -mtriple=arm-eabi %s -o /dev/null
 ; RUN: llc -mtriple=armv6-linux-gnueabi %s -o - | FileCheck %s
 
-; RUN: llc -mtriple=armv6-linux-gnu --disable-fp-elim %s -o - \
+; RUN: llc -mtriple=armv6-linux-gnu --frame-pointer=all %s -o - \
 ; RUN:  | FileCheck %s -check-prefix CHECK-FP-ELIM
 
 ; RUN: llc -mtriple=armv6-apple-ios %s -o - \
diff --git a/test/CodeGen/ARM/indexed-mem.ll b/test/CodeGen/ARM/indexed-mem.ll
new file mode 100644
index 0000000000000000000000000000000000000000..b318c7642566013e0c7a2f04c383b0a94559e9d5
--- /dev/null
+++ b/test/CodeGen/ARM/indexed-mem.ll
@@ -0,0 +1,186 @@
+; RUN: llc -mtriple=thumbv8m.main %s -o - | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-V8M
+; RUN: llc -mtriple=armv8 %s -o - | FileCheck %s --check-prefix=CHECK-V8
+
+; CHECK-LABEL: pre_inc_ldr
+; CHECK: ldr{{.*}}, [r0, #4]!
+define i32* @pre_inc_ldr(i32* %base, i32 %a) {
+  %addr = getelementptr i32, i32* %base, i32 1
+  %ld = load i32, i32* %addr
+  %addr.1 = getelementptr i32, i32* %base, i32 2
+  %res = add i32 %ld, %a
+  store i32 %res, i32* %addr.1
+  ret i32* %addr
+}
+
+; CHECK-LABEL: pre_dec_ldr
+; CHECK: ldr{{.*}}, [r0, #-4]!
+define i32* @pre_dec_ldr(i32* %base, i32 %a) {
+  %addr = getelementptr i32, i32* %base, i32 -1
+  %ld = load i32, i32* %addr
+  %addr.1 = getelementptr i32, i32* %base, i32 2
+  %res = add i32 %ld, %a
+  store i32 %res, i32* %addr.1
+  ret i32* %addr
+}
+
+; CHECK-LABEL: post_inc_ldr
+; CHECK: ldr{{.*}}, [r0], #4
+define i32* @post_inc_ldr(i32* %base, i32* %addr.2, i32 %a) {
+  %addr = getelementptr i32, i32* %base, i32 0
+  %ld = load i32, i32* %addr
+  %addr.1 = getelementptr i32, i32* %base, i32 1
+  %res = add i32 %ld, %a
+  store i32 %res, i32* %addr.2
+  ret i32* %addr.1
+}
+
+; CHECK-LABEL: post_dec_ldr
+; CHECK: ldr{{.*}}, [r0], #-4
+define i32* @post_dec_ldr(i32* %base, i32* %addr.2, i32 %a) {
+  %addr = getelementptr i32, i32* %base, i32 0
+  %ld = load i32, i32* %addr
+  %addr.1 = getelementptr i32, i32* %base, i32 -1
+  %res = add i32 %ld, %a
+  store i32 %res, i32* %addr.2
+  ret i32* %addr.1
+}
+
+; CHECK-LABEL: pre_inc_str
+; CHECK: str{{.*}}, [r0, #4]!
+define i32* @pre_inc_str(i32* %base, i32 %a, i32 %b) {
+  %addr.1 = getelementptr i32, i32* %base, i32 1
+  %res = add i32 %a, %b
+  store i32 %res, i32* %addr.1
+  ret i32* %addr.1
+}
+
+; CHECK-LABEL: pre_dec_str
+; CHECK: str{{.*}}, [r0, #-4]!
+define i32* @pre_dec_str(i32* %base, i32 %a, i32 %b) {
+  %res = add i32 %a, %b
+  %addr.1 = getelementptr i32, i32* %base, i32 -1
+  store i32 %res, i32* %addr.1
+  ret i32* %addr.1
+}
+
+; CHECK-LABEL: post_inc_str
+; CHECK: str{{.*}}, [r0], #4
+define i32* @post_inc_str(i32* %base, i32 %a, i32 %b) {
+  %addr.1 = getelementptr i32, i32* %base, i32 1
+  %res = add i32 %a, %b
+  store i32 %res, i32* %base
+  ret i32* %addr.1
+}
+
+; CHECK-LABEL: post_dec_str
+; CHECK: str{{.*}}, [r0], #-4
+define i32* @post_dec_str(i32* %base, i32 %a, i32 %b) {
+  %addr.1 = getelementptr i32, i32* %base, i32 -1
+  %res = add i32 %a, %b
+  store i32 %res, i32* %base
+  ret i32* %addr.1
+}
+
+; TODO: Generate ldrd
+; CHECK-LABEL: pre_inc_ldrd
+; CHECK: ldr{{.*}}, #4]!
+define i32* @pre_inc_ldrd(i32* %base) {
+  %addr = getelementptr i32, i32* %base, i32 1
+  %addr.1 = getelementptr i32, i32* %base, i32 2
+  %addr.2 = getelementptr i32, i32* %base, i32 3
+  %ld = load i32, i32* %addr
+  %ld.1 = load i32, i32* %addr.1
+  %res = add i32 %ld, %ld.1
+  store i32 %res, i32* %addr.2
+  ret i32* %addr
+}
+
+; TODO: Generate ldrd
+; CHECK-LABEL: pre_dec_ldrd
+; CHECK: ldr{{.*}}, #-4]!
+define i32* @pre_dec_ldrd(i32* %base) {
+  %addr = getelementptr i32, i32* %base, i32 -1
+  %addr.1 = getelementptr i32, i32* %base, i32 -2
+  %addr.2 = getelementptr i32, i32* %base, i32 -3
+  %ld = load i32, i32* %addr
+  %ld.1 = load i32, i32* %addr.1
+  %res = add i32 %ld, %ld.1
+  store i32 %res, i32* %addr.2
+  ret i32* %addr
+}
+
+; TODO: Generate post inc
+; CHECK-LABEL: post_inc_ldrd
+; CHECK-V8M: ldrd{{.*}}, [r0]
+; CHECK-V8: ldm
+; CHECK: add{{.*}}, #8
+define i32* @post_inc_ldrd(i32* %base, i32* %addr.3) {
+  %addr = getelementptr i32, i32* %base, i32 0
+  %ld = load i32, i32* %addr
+  %addr.1 = getelementptr i32, i32* %base, i32 1
+  %ld.1 = load i32, i32* %addr.1
+  %addr.2 = getelementptr i32, i32* %base, i32 2
+  %res = add i32 %ld, %ld.1
+  store i32 %res, i32* %addr.3
+  ret i32* %addr.2
+}
+
+; CHECK-LABEL: pre_inc_str_multi
+; CHECK: str{{.*}}, #8]!
+define i32* @pre_inc_str_multi(i32* %base) {
+  %addr = getelementptr i32, i32* %base, i32 0
+  %addr.1 = getelementptr i32, i32* %base, i32 1
+  %ld = load i32, i32* %addr
+  %ld.1 = load i32, i32* %addr.1
+  %res = add i32 %ld, %ld.1
+  %addr.2 = getelementptr i32, i32* %base, i32 2
+  store i32 %res, i32* %addr.2
+  ret i32* %addr.2
+}
+
+; CHECK-LABEL: pre_dec_str_multi
+; CHECK: str{{.*}}, #-4]!
+define i32* @pre_dec_str_multi(i32* %base) {
+  %addr = getelementptr i32, i32* %base, i32 0
+  %addr.1 = getelementptr i32, i32* %base, i32 1
+  %ld = load i32, i32* %addr
+  %ld.1 = load i32, i32* %addr.1
+  %res = add i32 %ld, %ld.1
+  %addr.2 = getelementptr i32, i32* %base, i32 -1
+  store i32 %res, i32* %addr.2
+  ret i32* %addr.2
+}
+
+; CHECK-LABEL: illegal_pre_inc_store_1
+; CHECK-NOT: str{{.*}} ]!
+define i32* @illegal_pre_inc_store_1(i32* %base) {
+entry:
+  %ptr.to.use = getelementptr i32, i32* %base, i32 2
+  %ptr.to.store = ptrtoint i32* %base to i32
+  store i32 %ptr.to.store, i32* %ptr.to.use, align 4
+  ret i32* %ptr.to.use
+}
+
+; TODO: The mov should be unecessary
+; CHECK-LABEL: legal_pre_inc_store_needs_copy_1
+; CHECK: add{{.*}}, #8
+; CHECK-NOT: str{{.*}}]!
+; CHECK: mov
+define i32* @legal_pre_inc_store_needs_copy_1(i32* %base) {
+entry:
+  %ptr.to.use = getelementptr i32, i32* %base, i32 2
+  %ptr.to.store = ptrtoint i32* %ptr.to.use to i32
+  store i32 %ptr.to.store, i32* %ptr.to.use, align 4
+  ret i32* %ptr.to.use
+}
+
+; CHECK-LABEL: legal_pre_inc_store_needs_copy_2
+; CHECK-NOT: mov
+; CHECK-NOT: str{{.*}}]!
+define i32* @legal_pre_inc_store_needs_copy_2(i32 %base) {
+entry:
+  %ptr = inttoptr i32 %base to i32*
+  %ptr.to.use = getelementptr i32, i32* %ptr, i32 2
+  store i32 %base, i32* %ptr.to.use, align 4
+  ret i32* %ptr.to.use
+}
diff --git a/test/CodeGen/ARM/inline-asm-clobber.ll b/test/CodeGen/ARM/inline-asm-clobber.ll
index 458949a5c944a8e63fa814f8cfffe4ceb686bab0..cb2069c20bf6ff7f560940cc28b4c8be2566fa71 100644
--- a/test/CodeGen/ARM/inline-asm-clobber.ll
+++ b/test/CodeGen/ARM/inline-asm-clobber.ll
@@ -3,7 +3,7 @@
 ; RUN: llc <%s -mtriple=arm-none-eabi -relocation-model=rwpi 2>&1 \
 ; RUN:   | FileCheck %s -check-prefix=RWPI
 
-; RUN: llc <%s -mtriple=arm-none-eabi --disable-fp-elim 2>&1 \
+; RUN: llc <%s -mtriple=arm-none-eabi --frame-pointer=all 2>&1 \
 ; RUN:   | FileCheck %s -check-prefix=NO_FP_ELIM
 
 ; CHECK: warning: inline asm clobber list contains reserved registers: SP, PC
diff --git a/test/CodeGen/ARM/legalize-fneg.ll b/test/CodeGen/ARM/legalize-fneg.ll
new file mode 100644
index 0000000000000000000000000000000000000000..f7578ae98fa2d1a17001aff5a2bf9ed61c1b7eee
--- /dev/null
+++ b/test/CodeGen/ARM/legalize-fneg.ll
@@ -0,0 +1,61 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=arm-eabi -float-abi=soft  -verify-machineinstrs < %s \
+; RUN:   | FileCheck --check-prefixes=ARM %s
+; RUN: llc -mtriple=arm-eabi -float-abi=soft  -verify-machineinstrs < %s \
+; RUN:   | FileCheck --check-prefixes=NOLIB %s
+
+; Check Y = FNEG(X) -> Y = X ^ sign mask and no lib call is generated.
+define void @test1(float* %a, float* %b) {
+; ARM-LABEL: test1:
+; ARM:       @ %bb.0: @ %entry
+; ARM-NEXT:    ldr r1, [r1]
+; ARM-NEXT:    eor r1, r1, #-2147483648
+; ARM-NEXT:    str r1, [r0]
+; ARM-NEXT:    mov pc, lr
+; NOLIB-LABEL:  test1:
+; NOLIB:       eor
+; NOLIB-NOT:   bl __aeabi_fsub
+entry:
+  %0 = load float, float* %b
+  %neg = fneg float %0
+  store float %neg, float* %a
+  ret void
+}
+
+define void @test2(double* %a, double* %b) {
+; ARM-LABEL: test2:
+; ARM:       @ %bb.0: @ %entry
+; ARM-NEXT:    ldr r2, [r1]
+; ARM-NEXT:    ldr r1, [r1, #4]
+; ARM-NEXT:    str r2, [r0]
+; ARM-NEXT:    eor r1, r1, #-2147483648
+; ARM-NEXT:    str r1, [r0, #4]
+; ARM-NEXT:    mov pc, lr
+; NOLIB-LABEL:  test2:
+; NOLIB:       eor
+; NOLIB-NOT:   bl __aeabi_dsub
+entry:
+  %0 = load double, double* %b
+  %neg = fneg double %0
+  store double %neg, double* %a
+  ret void
+}
+
+define void @test3(fp128* %a, fp128* %b) {
+; ARM-LABEL: test3:
+; ARM:       @ %bb.0: @ %entry
+; ARM-NEXT:    ldm r1, {r2, r3, r12}
+; ARM-NEXT:    ldr r1, [r1, #12]
+; ARM-NEXT:    stm r0, {r2, r3, r12}
+; ARM-NEXT:    eor r1, r1, #-2147483648
+; ARM-NEXT:    str r1, [r0, #12]
+; ARM-NEXT:    mov pc, lr
+; NOLIB-LABEL: test3:
+; NOLIB:       eor
+; NOLIB-NOT:   bl __subtf3
+entry:
+  %0 = load fp128, fp128* %b
+  %neg = fneg fp128 %0
+  store fp128 %neg, fp128* %a
+  ret void
+}
diff --git a/test/CodeGen/ARM/load.ll b/test/CodeGen/ARM/load.ll
index b8f3003755a05c493707b939ede757ea9b8a6560..f33294267f518b3658de716d1d6fb516140116e6 100644
--- a/test/CodeGen/ARM/load.ll
+++ b/test/CodeGen/ARM/load.ll
@@ -562,3 +562,473 @@ entry:
   store i32 %x, i32* %0, align 4
   ret void
 }
+
+
+; Negative offset
+
+define i32 @ldrsb_ri_negative(i8* %p) {
+; CHECK-T1-LABEL: ldrsb_ri_negative:
+; CHECK-T1:       @ %bb.0: @ %entry
+; CHECK-T1-NEXT:    movs r1, #0
+; CHECK-T1-NEXT:    mvns r1, r1
+; CHECK-T1-NEXT:    ldrsb r0, [r0, r1]
+; CHECK-T1-NEXT:    bx lr
+;
+; CHECK-T2-LABEL: ldrsb_ri_negative:
+; CHECK-T2:       @ %bb.0: @ %entry
+; CHECK-T2-NEXT:    ldrsb r0, [r0, #-1]
+; CHECK-T2-NEXT:    bx lr
+entry:
+  %add.ptr = getelementptr inbounds i8, i8* %p, i32 -1
+  %0 = load i8, i8* %add.ptr, align 1
+  %conv = sext i8 %0 to i32
+  ret i32 %conv
+}
+
+define i32 @ldrsh_ri_negative(i8* %p) {
+; CHECK-T1-LABEL: ldrsh_ri_negative:
+; CHECK-T1:       @ %bb.0: @ %entry
+; CHECK-T1-NEXT:    movs r1, #0
+; CHECK-T1-NEXT:    mvns r1, r1
+; CHECK-T1-NEXT:    ldrsh r0, [r0, r1]
+; CHECK-T1-NEXT:    bx lr
+;
+; CHECK-T2-LABEL: ldrsh_ri_negative:
+; CHECK-T2:       @ %bb.0: @ %entry
+; CHECK-T2-NEXT:    ldrsh r0, [r0, #-1]
+; CHECK-T2-NEXT:    bx lr
+entry:
+  %add.ptr = getelementptr inbounds i8, i8* %p, i32 -1
+  %0 = bitcast i8* %add.ptr to i16*
+  %1 = load i16, i16* %0, align 2
+  %conv = sext i16 %1 to i32
+  ret i32 %conv
+}
+
+define i32 @ldrb_ri_negative(i8* %p) {
+; CHECK-T1-LABEL: ldrb_ri_negative:
+; CHECK-T1:       @ %bb.0: @ %entry
+; CHECK-T1-NEXT:    subs r0, r0, #1
+; CHECK-T1-NEXT:    ldrb r0, [r0]
+; CHECK-T1-NEXT:    bx lr
+;
+; CHECK-T2-LABEL: ldrb_ri_negative:
+; CHECK-T2:       @ %bb.0: @ %entry
+; CHECK-T2-NEXT:    ldrb r0, [r0, #-1]
+; CHECK-T2-NEXT:    bx lr
+entry:
+  %add.ptr = getelementptr inbounds i8, i8* %p, i32 -1
+  %0 = load i8, i8* %add.ptr, align 1
+  %conv = zext i8 %0 to i32
+  ret i32 %conv
+}
+
+define i32 @ldrh_ri_negative(i8* %p) {
+; CHECK-T1-LABEL: ldrh_ri_negative:
+; CHECK-T1:       @ %bb.0: @ %entry
+; CHECK-T1-NEXT:    subs r0, r0, #1
+; CHECK-T1-NEXT:    ldrh r0, [r0]
+; CHECK-T1-NEXT:    bx lr
+;
+; CHECK-T2-LABEL: ldrh_ri_negative:
+; CHECK-T2:       @ %bb.0: @ %entry
+; CHECK-T2-NEXT:    ldrh r0, [r0, #-1]
+; CHECK-T2-NEXT:    bx lr
+entry:
+  %add.ptr = getelementptr inbounds i8, i8* %p, i32 -1
+  %0 = bitcast i8* %add.ptr to i16*
+  %1 = load i16, i16* %0, align 2
+  %conv = zext i16 %1 to i32
+  ret i32 %conv
+}
+
+define i32 @ldr_ri_negative(i8* %p) {
+; CHECK-T1-LABEL: ldr_ri_negative:
+; CHECK-T1:       @ %bb.0: @ %entry
+; CHECK-T1-NEXT:    subs r0, r0, #1
+; CHECK-T1-NEXT:    ldr r0, [r0]
+; CHECK-T1-NEXT:    bx lr
+;
+; CHECK-T2-LABEL: ldr_ri_negative:
+; CHECK-T2:       @ %bb.0: @ %entry
+; CHECK-T2-NEXT:    ldr r0, [r0, #-1]
+; CHECK-T2-NEXT:    bx lr
+entry:
+  %add.ptr = getelementptr inbounds i8, i8* %p, i32 -1
+  %0 = bitcast i8* %add.ptr to i32*
+  %1 = load i32, i32* %0, align 4
+  ret i32 %1
+}
+
+define void @strb_ri_negative(i8* %p, i32 %x) {
+; CHECK-T1-LABEL: strb_ri_negative:
+; CHECK-T1:       @ %bb.0: @ %entry
+; CHECK-T1-NEXT:    subs r0, r0, #1
+; CHECK-T1-NEXT:    strb r1, [r0]
+; CHECK-T1-NEXT:    bx lr
+;
+; CHECK-T2-LABEL: strb_ri_negative:
+; CHECK-T2:       @ %bb.0: @ %entry
+; CHECK-T2-NEXT:    strb r1, [r0, #-1]
+; CHECK-T2-NEXT:    bx lr
+entry:
+  %conv = trunc i32 %x to i8
+  %add.ptr = getelementptr inbounds i8, i8* %p, i32 -1
+  store i8 %conv, i8* %add.ptr, align 1
+  ret void
+}
+
+define void @strh_ri_negative(i8* %p, i32 %x) {
+; CHECK-T1-LABEL: strh_ri_negative:
+; CHECK-T1:       @ %bb.0: @ %entry
+; CHECK-T1-NEXT:    subs r0, r0, #1
+; CHECK-T1-NEXT:    strh r1, [r0]
+; CHECK-T1-NEXT:    bx lr
+;
+; CHECK-T2-LABEL: strh_ri_negative:
+; CHECK-T2:       @ %bb.0: @ %entry
+; CHECK-T2-NEXT:    strh r1, [r0, #-1]
+; CHECK-T2-NEXT:    bx lr
+entry:
+  %conv = trunc i32 %x to i16
+  %add.ptr = getelementptr inbounds i8, i8* %p, i32 -1
+  %0 = bitcast i8* %add.ptr to i16*
+  store i16 %conv, i16* %0, align 2
+  ret void
+}
+
+define void @str_ri_negative(i8* %p, i32 %x) {
+; CHECK-T1-LABEL: str_ri_negative:
+; CHECK-T1:       @ %bb.0: @ %entry
+; CHECK-T1-NEXT:    subs r0, r0, #1
+; CHECK-T1-NEXT:    str r1, [r0]
+; CHECK-T1-NEXT:    bx lr
+;
+; CHECK-T2-LABEL: str_ri_negative:
+; CHECK-T2:       @ %bb.0: @ %entry
+; CHECK-T2-NEXT:    str r1, [r0, #-1]
+; CHECK-T2-NEXT:    bx lr
+entry:
+  %add.ptr = getelementptr inbounds i8, i8* %p, i32 -1
+  %0 = bitcast i8* %add.ptr to i32*
+  store i32 %x, i32* %0, align 4
+  ret void
+}
+
+
+; Negative 255 offset
+
+define i32 @ldrsb_ri_negative255(i8* %p) {
+; CHECK-T1-LABEL: ldrsb_ri_negative255:
+; CHECK-T1:       @ %bb.0: @ %entry
+; CHECK-T1-NEXT:    movs r1, #254
+; CHECK-T1-NEXT:    mvns r1, r1
+; CHECK-T1-NEXT:    ldrsb r0, [r0, r1]
+; CHECK-T1-NEXT:    bx lr
+;
+; CHECK-T2-LABEL: ldrsb_ri_negative255:
+; CHECK-T2:       @ %bb.0: @ %entry
+; CHECK-T2-NEXT:    ldrsb r0, [r0, #-255]
+; CHECK-T2-NEXT:    bx lr
+entry:
+  %add.ptr = getelementptr inbounds i8, i8* %p, i32 -255
+  %0 = load i8, i8* %add.ptr, align 1
+  %conv = sext i8 %0 to i32
+  ret i32 %conv
+}
+
+define i32 @ldrsh_ri_negative255(i8* %p) {
+; CHECK-T1-LABEL: ldrsh_ri_negative255:
+; CHECK-T1:       @ %bb.0: @ %entry
+; CHECK-T1-NEXT:    movs r1, #254
+; CHECK-T1-NEXT:    mvns r1, r1
+; CHECK-T1-NEXT:    ldrsh r0, [r0, r1]
+; CHECK-T1-NEXT:    bx lr
+;
+; CHECK-T2-LABEL: ldrsh_ri_negative255:
+; CHECK-T2:       @ %bb.0: @ %entry
+; CHECK-T2-NEXT:    ldrsh r0, [r0, #-255]
+; CHECK-T2-NEXT:    bx lr
+entry:
+  %add.ptr = getelementptr inbounds i8, i8* %p, i32 -255
+  %0 = bitcast i8* %add.ptr to i16*
+  %1 = load i16, i16* %0, align 2
+  %conv = sext i16 %1 to i32
+  ret i32 %conv
+}
+
+define i32 @ldrb_ri_negative255(i8* %p) {
+; CHECK-T1-LABEL: ldrb_ri_negative255:
+; CHECK-T1:       @ %bb.0: @ %entry
+; CHECK-T1-NEXT:    subs r0, #255
+; CHECK-T1-NEXT:    ldrb r0, [r0]
+; CHECK-T1-NEXT:    bx lr
+;
+; CHECK-T2-LABEL: ldrb_ri_negative255:
+; CHECK-T2:       @ %bb.0: @ %entry
+; CHECK-T2-NEXT:    ldrb r0, [r0, #-255]
+; CHECK-T2-NEXT:    bx lr
+entry:
+  %add.ptr = getelementptr inbounds i8, i8* %p, i32 -255
+  %0 = load i8, i8* %add.ptr, align 1
+  %conv = zext i8 %0 to i32
+  ret i32 %conv
+}
+
+define i32 @ldrh_ri_negative255(i8* %p) {
+; CHECK-T1-LABEL: ldrh_ri_negative255:
+; CHECK-T1:       @ %bb.0: @ %entry
+; CHECK-T1-NEXT:    subs r0, #255
+; CHECK-T1-NEXT:    ldrh r0, [r0]
+; CHECK-T1-NEXT:    bx lr
+;
+; CHECK-T2-LABEL: ldrh_ri_negative255:
+; CHECK-T2:       @ %bb.0: @ %entry
+; CHECK-T2-NEXT:    ldrh r0, [r0, #-255]
+; CHECK-T2-NEXT:    bx lr
+entry:
+  %add.ptr = getelementptr inbounds i8, i8* %p, i32 -255
+  %0 = bitcast i8* %add.ptr to i16*
+  %1 = load i16, i16* %0, align 2
+  %conv = zext i16 %1 to i32
+  ret i32 %conv
+}
+
+define i32 @ldr_ri_negative255(i8* %p) {
+; CHECK-T1-LABEL: ldr_ri_negative255:
+; CHECK-T1:       @ %bb.0: @ %entry
+; CHECK-T1-NEXT:    subs r0, #255
+; CHECK-T1-NEXT:    ldr r0, [r0]
+; CHECK-T1-NEXT:    bx lr
+;
+; CHECK-T2-LABEL: ldr_ri_negative255:
+; CHECK-T2:       @ %bb.0: @ %entry
+; CHECK-T2-NEXT:    ldr r0, [r0, #-255]
+; CHECK-T2-NEXT:    bx lr
+entry:
+  %add.ptr = getelementptr inbounds i8, i8* %p, i32 -255
+  %0 = bitcast i8* %add.ptr to i32*
+  %1 = load i32, i32* %0, align 4
+  ret i32 %1
+}
+
+define void @strb_ri_negative255(i8* %p, i32 %x) {
+; CHECK-T1-LABEL: strb_ri_negative255:
+; CHECK-T1:       @ %bb.0: @ %entry
+; CHECK-T1-NEXT:    subs r0, #255
+; CHECK-T1-NEXT:    strb r1, [r0]
+; CHECK-T1-NEXT:    bx lr
+;
+; CHECK-T2-LABEL: strb_ri_negative255:
+; CHECK-T2:       @ %bb.0: @ %entry
+; CHECK-T2-NEXT:    strb r1, [r0, #-255]
+; CHECK-T2-NEXT:    bx lr
+entry:
+  %conv = trunc i32 %x to i8
+  %add.ptr = getelementptr inbounds i8, i8* %p, i32 -255
+  store i8 %conv, i8* %add.ptr, align 1
+  ret void
+}
+
+define void @strh_ri_negative255(i8* %p, i32 %x) {
+; CHECK-T1-LABEL: strh_ri_negative255:
+; CHECK-T1:       @ %bb.0: @ %entry
+; CHECK-T1-NEXT:    subs r0, #255
+; CHECK-T1-NEXT:    strh r1, [r0]
+; CHECK-T1-NEXT:    bx lr
+;
+; CHECK-T2-LABEL: strh_ri_negative255:
+; CHECK-T2:       @ %bb.0: @ %entry
+; CHECK-T2-NEXT:    strh r1, [r0, #-255]
+; CHECK-T2-NEXT:    bx lr
+entry:
+  %conv = trunc i32 %x to i16
+  %add.ptr = getelementptr inbounds i8, i8* %p, i32 -255
+  %0 = bitcast i8* %add.ptr to i16*
+  store i16 %conv, i16* %0, align 2
+  ret void
+}
+
+define void @str_ri_negative255(i8* %p, i32 %x) {
+; CHECK-T1-LABEL: str_ri_negative255:
+; CHECK-T1:       @ %bb.0: @ %entry
+; CHECK-T1-NEXT:    subs r0, #255
+; CHECK-T1-NEXT:    str r1, [r0]
+; CHECK-T1-NEXT:    bx lr
+;
+; CHECK-T2-LABEL: str_ri_negative255:
+; CHECK-T2:       @ %bb.0: @ %entry
+; CHECK-T2-NEXT:    str r1, [r0, #-255]
+; CHECK-T2-NEXT:    bx lr
+entry:
+  %add.ptr = getelementptr inbounds i8, i8* %p, i32 -255
+  %0 = bitcast i8* %add.ptr to i32*
+  store i32 %x, i32* %0, align 4
+  ret void
+}
+
+
+; Negative 256 offset
+
+define i32 @ldrsb_ri_negative256(i8* %p) {
+; CHECK-T1-LABEL: ldrsb_ri_negative256:
+; CHECK-T1:       @ %bb.0: @ %entry
+; CHECK-T1-NEXT:    movs r1, #255
+; CHECK-T1-NEXT:    mvns r1, r1
+; CHECK-T1-NEXT:    ldrsb r0, [r0, r1]
+; CHECK-T1-NEXT:    bx lr
+;
+; CHECK-T2-LABEL: ldrsb_ri_negative256:
+; CHECK-T2:       @ %bb.0: @ %entry
+; CHECK-T2-NEXT:    mvn r1, #255
+; CHECK-T2-NEXT:    ldrsb r0, [r0, r1]
+; CHECK-T2-NEXT:    bx lr
+entry:
+  %add.ptr = getelementptr inbounds i8, i8* %p, i32 -256
+  %0 = load i8, i8* %add.ptr, align 1
+  %conv = sext i8 %0 to i32
+  ret i32 %conv
+}
+
+define i32 @ldrsh_ri_negative256(i8* %p) {
+; CHECK-T1-LABEL: ldrsh_ri_negative256:
+; CHECK-T1:       @ %bb.0: @ %entry
+; CHECK-T1-NEXT:    movs r1, #255
+; CHECK-T1-NEXT:    mvns r1, r1
+; CHECK-T1-NEXT:    ldrsh r0, [r0, r1]
+; CHECK-T1-NEXT:    bx lr
+;
+; CHECK-T2-LABEL: ldrsh_ri_negative256:
+; CHECK-T2:       @ %bb.0: @ %entry
+; CHECK-T2-NEXT:    mvn r1, #255
+; CHECK-T2-NEXT:    ldrsh r0, [r0, r1]
+; CHECK-T2-NEXT:    bx lr
+entry:
+  %add.ptr = getelementptr inbounds i8, i8* %p, i32 -256
+  %0 = bitcast i8* %add.ptr to i16*
+  %1 = load i16, i16* %0, align 2
+  %conv = sext i16 %1 to i32
+  ret i32 %conv
+}
+
+define i32 @ldrb_ri_negative256(i8* %p) {
+; CHECK-T1-LABEL: ldrb_ri_negative256:
+; CHECK-T1:       @ %bb.0: @ %entry
+; CHECK-T1-NEXT:    movs r1, #255
+; CHECK-T1-NEXT:    mvns r1, r1
+; CHECK-T1-NEXT:    ldrb r0, [r0, r1]
+; CHECK-T1-NEXT:    bx lr
+;
+; CHECK-T2-LABEL: ldrb_ri_negative256:
+; CHECK-T2:       @ %bb.0: @ %entry
+; CHECK-T2-NEXT:    mvn r1, #255
+; CHECK-T2-NEXT:    ldrb r0, [r0, r1]
+; CHECK-T2-NEXT:    bx lr
+entry:
+  %add.ptr = getelementptr inbounds i8, i8* %p, i32 -256
+  %0 = load i8, i8* %add.ptr, align 1
+  %conv = zext i8 %0 to i32
+  ret i32 %conv
+}
+
+define i32 @ldrh_ri_negative256(i8* %p) {
+; CHECK-T1-LABEL: ldrh_ri_negative256:
+; CHECK-T1:       @ %bb.0: @ %entry
+; CHECK-T1-NEXT:    movs r1, #255
+; CHECK-T1-NEXT:    mvns r1, r1
+; CHECK-T1-NEXT:    ldrh r0, [r0, r1]
+; CHECK-T1-NEXT:    bx lr
+;
+; CHECK-T2-LABEL: ldrh_ri_negative256:
+; CHECK-T2:       @ %bb.0: @ %entry
+; CHECK-T2-NEXT:    mvn r1, #255
+; CHECK-T2-NEXT:    ldrh r0, [r0, r1]
+; CHECK-T2-NEXT:    bx lr
+entry:
+  %add.ptr = getelementptr inbounds i8, i8* %p, i32 -256
+  %0 = bitcast i8* %add.ptr to i16*
+  %1 = load i16, i16* %0, align 2
+  %conv = zext i16 %1 to i32
+  ret i32 %conv
+}
+
+define i32 @ldr_ri_negative256(i8* %p) {
+; CHECK-T1-LABEL: ldr_ri_negative256:
+; CHECK-T1:       @ %bb.0: @ %entry
+; CHECK-T1-NEXT:    movs r1, #255
+; CHECK-T1-NEXT:    mvns r1, r1
+; CHECK-T1-NEXT:    ldr r0, [r0, r1]
+; CHECK-T1-NEXT:    bx lr
+;
+; CHECK-T2-LABEL: ldr_ri_negative256:
+; CHECK-T2:       @ %bb.0: @ %entry
+; CHECK-T2-NEXT:    mvn r1, #255
+; CHECK-T2-NEXT:    ldr r0, [r0, r1]
+; CHECK-T2-NEXT:    bx lr
+entry:
+  %add.ptr = getelementptr inbounds i8, i8* %p, i32 -256
+  %0 = bitcast i8* %add.ptr to i32*
+  %1 = load i32, i32* %0, align 4
+  ret i32 %1
+}
+
+define void @strb_ri_negative256(i8* %p, i32 %x) {
+; CHECK-T1-LABEL: strb_ri_negative256:
+; CHECK-T1:       @ %bb.0: @ %entry
+; CHECK-T1-NEXT:    movs r2, #255
+; CHECK-T1-NEXT:    mvns r2, r2
+; CHECK-T1-NEXT:    strb r1, [r0, r2]
+; CHECK-T1-NEXT:    bx lr
+;
+; CHECK-T2-LABEL: strb_ri_negative256:
+; CHECK-T2:       @ %bb.0: @ %entry
+; CHECK-T2-NEXT:    mvn r2, #255
+; CHECK-T2-NEXT:    strb r1, [r0, r2]
+; CHECK-T2-NEXT:    bx lr
+entry:
+  %conv = trunc i32 %x to i8
+  %add.ptr = getelementptr inbounds i8, i8* %p, i32 -256
+  store i8 %conv, i8* %add.ptr, align 1
+  ret void
+}
+
+define void @strh_ri_negative256(i8* %p, i32 %x) {
+; CHECK-T1-LABEL: strh_ri_negative256:
+; CHECK-T1:       @ %bb.0: @ %entry
+; CHECK-T1-NEXT:    movs r2, #255
+; CHECK-T1-NEXT:    mvns r2, r2
+; CHECK-T1-NEXT:    strh r1, [r0, r2]
+; CHECK-T1-NEXT:    bx lr
+;
+; CHECK-T2-LABEL: strh_ri_negative256:
+; CHECK-T2:       @ %bb.0: @ %entry
+; CHECK-T2-NEXT:    mvn r2, #255
+; CHECK-T2-NEXT:    strh r1, [r0, r2]
+; CHECK-T2-NEXT:    bx lr
+entry:
+  %conv = trunc i32 %x to i16
+  %add.ptr = getelementptr inbounds i8, i8* %p, i32 -256
+  %0 = bitcast i8* %add.ptr to i16*
+  store i16 %conv, i16* %0, align 2
+  ret void
+}
+
+define void @str_ri_negative256(i8* %p, i32 %x) {
+; CHECK-T1-LABEL: str_ri_negative256:
+; CHECK-T1:       @ %bb.0: @ %entry
+; CHECK-T1-NEXT:    movs r2, #255
+; CHECK-T1-NEXT:    mvns r2, r2
+; CHECK-T1-NEXT:    str r1, [r0, r2]
+; CHECK-T1-NEXT:    bx lr
+;
+; CHECK-T2-LABEL: str_ri_negative256:
+; CHECK-T2:       @ %bb.0: @ %entry
+; CHECK-T2-NEXT:    mvn r2, #255
+; CHECK-T2-NEXT:    str r1, [r0, r2]
+; CHECK-T2-NEXT:    bx lr
+entry:
+  %add.ptr = getelementptr inbounds i8, i8* %p, i32 -256
+  %0 = bitcast i8* %add.ptr to i32*
+  store i32 %x, i32* %0, align 4
+  ret void
+}
diff --git a/test/CodeGen/ARM/loop-align-cortex-m.ll b/test/CodeGen/ARM/loop-align-cortex-m.ll
index 1b41c1b6c3f1836d58cdf4680ce520fb98085ce0..61ba1a6ca2df36466dc0f56498820cf4112c3120 100644
--- a/test/CodeGen/ARM/loop-align-cortex-m.ll
+++ b/test/CodeGen/ARM/loop-align-cortex-m.ll
@@ -1,10 +1,10 @@
 ; RUN: llc -mtriple=thumbv7m-none-eabi %s -mcpu=cortex-m3 -o - | FileCheck %s
 ; RUN: llc -mtriple=thumbv7m-none-eabi %s -mcpu=cortex-m4 -o - | FileCheck %s
-; RUN: llc -mtriple=thumbv7m-none-eabi %s -mcpu=cortex-m33 -o - | FileCheck %s
+; RUN: llc -mtriple=thumbv8m-none-eabi %s -mcpu=cortex-m33 -o - | FileCheck %s
 
 define void @test_loop_alignment(i32* %in, i32*  %out) optsize {
 ; CHECK-LABEL: test_loop_alignment:
-; CHECK: movs {{r[0-9]+}}, #0
+; CHECK: mov{{.*}}, #0
 ; CHECK: .p2align 2
 
 entry:
diff --git a/test/CodeGen/ARM/loop-indexing.ll b/test/CodeGen/ARM/loop-indexing.ll
new file mode 100644
index 0000000000000000000000000000000000000000..d8a8c3d1d7962ceff4600f96907843cac0b2d326
--- /dev/null
+++ b/test/CodeGen/ARM/loop-indexing.ll
@@ -0,0 +1,1117 @@
+; RUN: llc -mtriple=thumbv7em -mattr=+fp-armv8 %s -o - | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-DEFAULT --check-prefix=CHECK-T2
+; RUN: llc -mtriple=thumbv8m.main -mattr=+fp-armv8,+dsp %s -o - | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-DEFAULT --check-prefix=CHECK-T2
+; RUN: llc -mtriple=thumbv8m.main -mattr=+fp-armv8,+dsp -lsr-backedge-indexing=false %s -o - | FileCheck %s --check-prefix=CHECK --check-prefix=DISABLED
+; RUN: llc -mtriple=thumbv8m.base %s -o - | FileCheck %s --check-prefix=CHECK --check-prefix=DISABLED
+; RUN: llc -mtriple=thumbv8 %s -o - | FileCheck %s --check-prefix=CHECK --check-prefix=DISABLED
+; RUN: llc -mtriple=thumbv8m.main -mattr=+fp-armv8,+dsp -lsr-complexity-limit=2147483647 %s -o - | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-COMPLEX --check-prefix=CHECK-T2
+
+; Tests to check that post increment addressing modes are used instead of
+; updating base pointers with add instructions.
+
+; TODO: I think we should be able to use post inc addressing with VLDM
+; instructions.
+; CHECK-LABEL: test_fma
+; CHECK: @ %loop
+
+; CHECK-DEFAULT: vldr s{{.*}}, #8]
+; CHECK-DEFAULT: vldr s{{.*}}, #8]
+; CHECK-DEFAULT: vldr s{{.*}}, #12]
+; CHECK-DEFAULT: vldr s{{.*}}, #12]
+
+; CHECK-COMPLEX: vldr s{{.*}}, #8]
+; CHECK-COMPLEX: vldr s{{.*}}, #8]
+; CHECK-COMPLEX: vldr s{{.*}}, #12]
+; CHECK-COMPLEX: vldr s{{.*}}, #12]
+
+define float @test_fma(float* %a, float* %b, i32 %N) {
+entry:
+  br label %loop
+
+loop:
+  %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
+  %idx.1 = phi i32 [ 0, %entry ], [ %idx.next, %loop ]
+  %res = phi float [ 0.0, %entry ], [ %fma.2, %loop ]
+  %gep.a.1 = getelementptr inbounds float, float* %a, i32 %idx.1
+  %a.1 = load float, float* %gep.a.1
+  %gep.b.1 = getelementptr inbounds float, float* %b, i32 %idx.1
+  %b.1 = load float, float* %gep.b.1
+  %fmul.1 = fmul float %a.1, %b.1
+  %fma.1 = fadd float %fmul.1, %res
+  %idx.2 = or i32 %idx.1, 1
+  %gep.a.2 = getelementptr inbounds float, float* %a, i32 %idx.2
+  %a.2 = load float, float* %gep.a.2
+  %gep.b.2 = getelementptr inbounds float, float* %b, i32 %idx.2
+  %b.2 = load float, float* %gep.b.2
+  %fmul.2 = fmul float %a.2, %b.2
+  %fma.2 = fadd float %fmul.2, %fma.1
+  %i.next = add nsw nuw i32 %i, -2
+  %idx.next = add nsw nuw i32 %idx.1, 2
+  %cmp = icmp ult i32 %i.next, %N
+  br i1 %cmp, label %loop, label %exit
+
+exit:
+  ret float %fma.2
+}
+
+; CHECK-LABEL: convolve_16bit
+; TODO: Both arrays should use indexing
+; CHECK-DEFAULT: ldr{{.*}}, #8]!
+; CHECK-DEFAULT-NOT: ldr{{.*}}]!
+
+; CHECK-COMPLEX: ldr{{.*}}, #8]!
+; CHECK-COMPLEX-NOT: ldr{{.*}}]!
+
+; DISABLED-NOT: ldr{{.*}}]!
+; DISABLED-NOT: str{{.*}}]!
+
+define void @convolve_16bit(i16** nocapture readonly %input_image, i16** nocapture readonly %filter,
+                            i32 %filter_dim, i32 %out_width, i32 %out_height,
+                            i32** nocapture readonly %convolved) {
+entry:
+  %cmp92 = icmp eq i32 %out_height, 0
+  br i1 %cmp92, label %for.cond.cleanup, label %for.cond1.preheader.lr.ph
+
+for.cond1.preheader.lr.ph:                        ; preds = %entry
+  %xtraiter = and i32 %filter_dim, 3
+  %unroll_iter = sub i32 %filter_dim, %xtraiter
+  br label %for.cond1.preheader
+
+for.cond1.preheader:                              ; preds = %for.cond.cleanup3, %for.cond1.preheader.lr.ph
+  %res_y.093 = phi i32 [ 0, %for.cond1.preheader.lr.ph ], [ %add28, %for.cond.cleanup3 ]
+  %arrayidx22 = getelementptr inbounds i32*, i32** %convolved, i32 %res_y.093
+  %tmp3 = load i32*, i32** %arrayidx22, align 4
+  br label %for.cond9.preheader.us.us.preheader
+
+for.cond9.preheader.us.us.preheader:              ; preds = %for.cond5.for.cond.cleanup7_crit_edge.us, %for.cond5.preheader.lr.ph
+  %res_x.060.us = phi i32 [ %add25.us, %for.cond5.for.cond.cleanup7_crit_edge.us ], [ 0, %for.cond1.preheader ]
+  br label %for.cond9.preheader.us.us
+
+for.cond9.preheader.us.us:                        ; preds = %for.cond9.for.cond.cleanup11_crit_edge.us.us, %for.cond9.preheader.us.us.preheader
+  %filter_y.056.us.us = phi i32 [ %inc20.us.us, %for.cond9.for.cond.cleanup11_crit_edge.us.us.unr-lcssa ], [ 0, %for.cond9.preheader.us.us.preheader ]
+  %result_element.055.us.us = phi i32 [ %add18.us.us.3, %for.cond9.for.cond.cleanup11_crit_edge.us.us.unr-lcssa ], [ 0, %for.cond9.preheader.us.us.preheader ]
+  %add.us.us = add i32 %filter_y.056.us.us, %res_y.093
+  %arrayidx.us.us = getelementptr inbounds i16*, i16** %filter, i32 %filter_y.056.us.us
+  %tmp5 = load i16*, i16** %arrayidx.us.us, align 4
+  %arrayidx15.us.us = getelementptr inbounds i16*, i16** %input_image, i32 %add.us.us
+  %tmp6 = load i16*, i16** %arrayidx15.us.us, align 4
+  br label %for.body12.us.us
+
+for.body12.us.us:                                 ; preds = %for.body12.us.us, %for.cond9.preheader.us.us
+  %filter_x.053.us.us = phi i32 [ %inc.us.us.3, %for.body12.us.us ], [ 0, %for.cond9.preheader.us.us ]
+  %result_element.152.us.us = phi i32 [ %add18.us.us.3, %for.body12.us.us ], [ %result_element.055.us.us, %for.cond9.preheader.us.us ]
+  %niter = phi i32 [ %niter.nsub.3, %for.body12.us.us ], [ %unroll_iter, %for.cond9.preheader.us.us ]
+  %add13.us.us = add i32 %filter_x.053.us.us, %res_x.060.us
+  %arrayidx14.us.us = getelementptr inbounds i16, i16* %tmp5, i32 %filter_x.053.us.us
+  %tmp9 = load i16, i16* %arrayidx14.us.us, align 2
+  %conv.us.us = sext i16 %tmp9 to i32
+  %arrayidx16.us.us = getelementptr inbounds i16, i16* %tmp6, i32 %add13.us.us
+  %tmp10 = load i16, i16* %arrayidx16.us.us, align 2
+  %conv17.us.us = sext i16 %tmp10 to i32
+  %mul.us.us = mul nsw i32 %conv17.us.us, %conv.us.us
+  %add18.us.us = add nsw i32 %mul.us.us, %result_element.152.us.us
+  %inc.us.us = or i32 %filter_x.053.us.us, 1
+  %add13.us.us.1 = add i32 %inc.us.us, %res_x.060.us
+  %arrayidx14.us.us.1 = getelementptr inbounds i16, i16* %tmp5, i32 %inc.us.us
+  %tmp11 = load i16, i16* %arrayidx14.us.us.1, align 2
+  %conv.us.us.1 = sext i16 %tmp11 to i32
+  %arrayidx16.us.us.1 = getelementptr inbounds i16, i16* %tmp6, i32 %add13.us.us.1
+  %tmp12 = load i16, i16* %arrayidx16.us.us.1, align 2
+  %conv17.us.us.1 = sext i16 %tmp12 to i32
+  %mul.us.us.1 = mul nsw i32 %conv17.us.us.1, %conv.us.us.1
+  %add18.us.us.1 = add nsw i32 %mul.us.us.1, %add18.us.us
+  %inc.us.us.1 = or i32 %filter_x.053.us.us, 2
+  %add13.us.us.2 = add i32 %inc.us.us.1, %res_x.060.us
+  %arrayidx14.us.us.2 = getelementptr inbounds i16, i16* %tmp5, i32 %inc.us.us.1
+  %tmp13 = load i16, i16* %arrayidx14.us.us.2, align 2
+  %conv.us.us.2 = sext i16 %tmp13 to i32
+  %arrayidx16.us.us.2 = getelementptr inbounds i16, i16* %tmp6, i32 %add13.us.us.2
+  %tmp14 = load i16, i16* %arrayidx16.us.us.2, align 2
+  %conv17.us.us.2 = sext i16 %tmp14 to i32
+  %mul.us.us.2 = mul nsw i32 %conv17.us.us.2, %conv.us.us.2
+  %add18.us.us.2 = add nsw i32 %mul.us.us.2, %add18.us.us.1
+  %inc.us.us.2 = or i32 %filter_x.053.us.us, 3
+  %add13.us.us.3 = add i32 %inc.us.us.2, %res_x.060.us
+  %arrayidx14.us.us.3 = getelementptr inbounds i16, i16* %tmp5, i32 %inc.us.us.2
+  %tmp15 = load i16, i16* %arrayidx14.us.us.3, align 2
+  %conv.us.us.3 = sext i16 %tmp15 to i32
+  %arrayidx16.us.us.3 = getelementptr inbounds i16, i16* %tmp6, i32 %add13.us.us.3
+  %tmp16 = load i16, i16* %arrayidx16.us.us.3, align 2
+  %conv17.us.us.3 = sext i16 %tmp16 to i32
+  %mul.us.us.3 = mul nsw i32 %conv17.us.us.3, %conv.us.us.3
+  %add18.us.us.3 = add nsw i32 %mul.us.us.3, %add18.us.us.2
+  %inc.us.us.3 = add i32 %filter_x.053.us.us, 4
+  %niter.nsub.3 = add i32 %niter, -4
+  %niter.ncmp.3 = icmp eq i32 %niter.nsub.3, 0
+  br i1 %niter.ncmp.3, label %for.cond9.for.cond.cleanup11_crit_edge.us.us.unr-lcssa, label %for.body12.us.us
+
+for.cond9.for.cond.cleanup11_crit_edge.us.us.unr-lcssa: ; preds = %for.body12.us.us, %for.cond9.preheader.us.us
+  %inc20.us.us = add nuw i32 %filter_y.056.us.us, 1
+  %exitcond98 = icmp eq i32 %inc20.us.us, %filter_dim
+  br i1 %exitcond98, label %for.cond5.for.cond.cleanup7_crit_edge.us, label %for.cond9.preheader.us.us
+
+for.cond5.for.cond.cleanup7_crit_edge.us:         ; preds = %for.cond9.for.cond.cleanup11_crit_edge.us.us
+  %arrayidx23.us = getelementptr inbounds i32, i32* %tmp3, i32 %res_x.060.us
+  store i32 %add18.us.us.3, i32* %arrayidx23.us, align 4
+  %add25.us = add nuw i32 %res_x.060.us, 1
+  %exitcond99 = icmp eq i32 %add25.us, %out_width
+  br i1 %exitcond99, label %for.cond.cleanup3, label %for.cond9.preheader.us.us.preheader
+
+for.cond.cleanup3:                                ; preds = %for.cond5.for.cond.cleanup7_crit_edge.us, %for.cond5.preheader.preheader, %for.cond1.preheader
+  %add28 = add nuw i32 %res_y.093, 1
+  %exitcond100 = icmp eq i32 %add28, %out_height
+  br i1 %exitcond100, label %for.cond.cleanup, label %for.cond1.preheader
+
+for.cond.cleanup:                                 ; preds = %for.cond.cleanup3, %entry
+  ret void
+}
+
+; CHECK-LABEL: mul_8x8
+; CHECK: @ %for.body
+
+; CHECK-DEFAULT: str{{.*}}, #16]!
+; CHECK-DEFAULT: ldrb{{.*}}, #4]!
+; CHECK-DEFAULT: ldrb{{.*}}, #4]!
+
+; CHECK-COMPLEX: str{{.*}}, #16]!
+; CHECK-COMPLEX: ldrb{{.*}}, #4]!
+; CHECK-COMPLEX: ldrb{{.*}}, #4]!
+
+; DISABLED-NOT: ldr{{.*}}]!
+; DISABLED-NOT: str{{.*}}]!
+
+; CHECK-T2: @ %for.body.epil
+; CHECK-T2: ldrb{{.*}}, #1]!
+; CHECK-T2: ldrb{{.*}}, #1]!
+; CHECK-T2: str{{.*}}, #4]!
+
+define void @mul_8x8(i8* nocapture readonly %A, i8* nocapture readonly %B, i32* nocapture %C, i32 %N) {
+entry:
+  %cmp9 = icmp eq i32 %N, 0
+  br i1 %cmp9, label %for.cond.cleanup, label %for.body.preheader
+
+for.body.preheader:                               ; preds = %entry
+  %tmp = add i32 %N, -1
+  %xtraiter = and i32 %N, 3
+  %tmp1 = icmp ult i32 %tmp, 3
+  br i1 %tmp1, label %for.cond.cleanup.loopexit.unr-lcssa, label %for.body.preheader.new
+
+for.body.preheader.new:                           ; preds = %for.body.preheader
+  %unroll_iter = sub i32 %N, %xtraiter
+  br label %for.body
+
+for.cond.cleanup.loopexit.unr-lcssa:              ; preds = %for.body, %for.body.preheader
+  %i.010.unr = phi i32 [ 0, %for.body.preheader ], [ %inc.3, %for.body ]
+  %lcmp.mod = icmp eq i32 %xtraiter, 0
+  br i1 %lcmp.mod, label %for.cond.cleanup, label %for.body.epil
+
+for.body.epil:                                    ; preds = %for.body.epil, %for.cond.cleanup.loopexit.unr-lcssa
+  %i.010.epil = phi i32 [ %inc.epil, %for.body.epil ], [ %i.010.unr, %for.cond.cleanup.loopexit.unr-lcssa ]
+  %epil.iter = phi i32 [ %epil.iter.sub, %for.body.epil ], [ %xtraiter, %for.cond.cleanup.loopexit.unr-lcssa ]
+  %arrayidx.epil = getelementptr inbounds i8, i8* %A, i32 %i.010.epil
+  %tmp2 = load i8, i8* %arrayidx.epil, align 1
+  %conv.epil = zext i8 %tmp2 to i32
+  %arrayidx1.epil = getelementptr inbounds i8, i8* %B, i32 %i.010.epil
+  %tmp3 = load i8, i8* %arrayidx1.epil, align 1
+  %conv2.epil = zext i8 %tmp3 to i32
+  %mul.epil = mul nuw nsw i32 %conv2.epil, %conv.epil
+  %arrayidx3.epil = getelementptr inbounds i32, i32* %C, i32 %i.010.epil
+  store i32 %mul.epil, i32* %arrayidx3.epil, align 4
+  %inc.epil = add nuw i32 %i.010.epil, 1
+  %epil.iter.sub = add i32 %epil.iter, -1
+  %epil.iter.cmp = icmp eq i32 %epil.iter.sub, 0
+  br i1 %epil.iter.cmp, label %for.cond.cleanup, label %for.body.epil
+
+for.cond.cleanup:                                 ; preds = %for.body.epil, %for.cond.cleanup.loopexit.unr-lcssa, %entry
+  ret void
+
+for.body:                                         ; preds = %for.body, %for.body.preheader.new
+  %i.010 = phi i32 [ 0, %for.body.preheader.new ], [ %inc.3, %for.body ]
+  %niter = phi i32 [ %unroll_iter, %for.body.preheader.new ], [ %niter.nsub.3, %for.body ]
+  %arrayidx = getelementptr inbounds i8, i8* %A, i32 %i.010
+  %tmp4 = load i8, i8* %arrayidx, align 1
+  %conv = zext i8 %tmp4 to i32
+  %arrayidx1 = getelementptr inbounds i8, i8* %B, i32 %i.010
+  %tmp5 = load i8, i8* %arrayidx1, align 1
+  %conv2 = zext i8 %tmp5 to i32
+  %mul = mul nuw nsw i32 %conv2, %conv
+  %arrayidx3 = getelementptr inbounds i32, i32* %C, i32 %i.010
+  store i32 %mul, i32* %arrayidx3, align 4
+  %inc = or i32 %i.010, 1
+  %arrayidx.1 = getelementptr inbounds i8, i8* %A, i32 %inc
+  %tmp6 = load i8, i8* %arrayidx.1, align 1
+  %conv.1 = zext i8 %tmp6 to i32
+  %arrayidx1.1 = getelementptr inbounds i8, i8* %B, i32 %inc
+  %tmp7 = load i8, i8* %arrayidx1.1, align 1
+  %conv2.1 = zext i8 %tmp7 to i32
+  %mul.1 = mul nuw nsw i32 %conv2.1, %conv.1
+  %arrayidx3.1 = getelementptr inbounds i32, i32* %C, i32 %inc
+  store i32 %mul.1, i32* %arrayidx3.1, align 4
+  %inc.1 = or i32 %i.010, 2
+  %arrayidx.2 = getelementptr inbounds i8, i8* %A, i32 %inc.1
+  %tmp8 = load i8, i8* %arrayidx.2, align 1
+  %conv.2 = zext i8 %tmp8 to i32
+  %arrayidx1.2 = getelementptr inbounds i8, i8* %B, i32 %inc.1
+  %tmp9 = load i8, i8* %arrayidx1.2, align 1
+  %conv2.2 = zext i8 %tmp9 to i32
+  %mul.2 = mul nuw nsw i32 %conv2.2, %conv.2
+  %arrayidx3.2 = getelementptr inbounds i32, i32* %C, i32 %inc.1
+  store i32 %mul.2, i32* %arrayidx3.2, align 4
+  %inc.2 = or i32 %i.010, 3
+  %arrayidx.3 = getelementptr inbounds i8, i8* %A, i32 %inc.2
+  %tmp10 = load i8, i8* %arrayidx.3, align 1
+  %conv.3 = zext i8 %tmp10 to i32
+  %arrayidx1.3 = getelementptr inbounds i8, i8* %B, i32 %inc.2
+  %tmp11 = load i8, i8* %arrayidx1.3, align 1
+  %conv2.3 = zext i8 %tmp11 to i32
+  %mul.3 = mul nuw nsw i32 %conv2.3, %conv.3
+  %arrayidx3.3 = getelementptr inbounds i32, i32* %C, i32 %inc.2
+  store i32 %mul.3, i32* %arrayidx3.3, align 4
+  %inc.3 = add i32 %i.010, 4
+  %niter.nsub.3 = add i32 %niter, -4
+  %niter.ncmp.3 = icmp eq i32 %niter.nsub.3, 0
+  br i1 %niter.ncmp.3, label %for.cond.cleanup.loopexit.unr-lcssa, label %for.body
+}
+
+; CHECK-LABEL: mul_16x8
+; CHECK: @ %for.body 
+
+; CHECK-DEFAULT: str{{.*}}, #16]!
+; CHECK-DEFAULT: ldrsh{{.*}}, #8]!
+
+; CHECK-COMPLEX: ldrsh{{.*}}, #8]!
+; CHECK-COMPLEX: str{{.*}}, #16]!
+; CHECK-COMPLEX: ldrb{{.*}}, #4]!
+
+; DISABLED-NOT: ldr{{.*}}]!
+; DISABLED-NOT: str{{.*}}]!
+
+; CHECK-T2: @ %for.body.epil
+; CHECK-T2: ldrsh{{.*}}, #2]!
+; CHECK-T2: ldrb{{.*}}, #1]!
+; CHECK-T2: str{{.*}}, #4]!
+
+define void @mul_16x8(i16* nocapture readonly %A, i8* nocapture readonly %B, i32* nocapture %C, i32 %N) {
+entry:
+  %cmp9 = icmp eq i32 %N, 0
+  br i1 %cmp9, label %for.cond.cleanup, label %for.body.preheader
+
+for.body.preheader:                               ; preds = %entry
+  %tmp = add i32 %N, -1
+  %xtraiter = and i32 %N, 3
+  %tmp1 = icmp ult i32 %tmp, 3
+  br i1 %tmp1, label %for.cond.cleanup.loopexit.unr-lcssa, label %for.body.preheader.new
+
+for.body.preheader.new:                           ; preds = %for.body.preheader
+  %unroll_iter = sub i32 %N, %xtraiter
+  br label %for.body
+
+for.cond.cleanup.loopexit.unr-lcssa:              ; preds = %for.body, %for.body.preheader
+  %i.010.unr = phi i32 [ 0, %for.body.preheader ], [ %inc.3, %for.body ]
+  %lcmp.mod = icmp eq i32 %xtraiter, 0
+  br i1 %lcmp.mod, label %for.cond.cleanup, label %for.body.epil
+
+for.body.epil:                                    ; preds = %for.body.epil, %for.cond.cleanup.loopexit.unr-lcssa
+  %i.010.epil = phi i32 [ %inc.epil, %for.body.epil ], [ %i.010.unr, %for.cond.cleanup.loopexit.unr-lcssa ]
+  %epil.iter = phi i32 [ %epil.iter.sub, %for.body.epil ], [ %xtraiter, %for.cond.cleanup.loopexit.unr-lcssa ]
+  %arrayidx.epil = getelementptr inbounds i16, i16* %A, i32 %i.010.epil
+  %tmp2 = load i16, i16* %arrayidx.epil, align 2
+  %conv.epil = sext i16 %tmp2 to i32
+  %arrayidx1.epil = getelementptr inbounds i8, i8* %B, i32 %i.010.epil
+  %tmp3 = load i8, i8* %arrayidx1.epil, align 1
+  %conv2.epil = zext i8 %tmp3 to i32
+  %mul.epil = mul nsw i32 %conv2.epil, %conv.epil
+  %arrayidx3.epil = getelementptr inbounds i32, i32* %C, i32 %i.010.epil
+  store i32 %mul.epil, i32* %arrayidx3.epil, align 4
+  %inc.epil = add nuw i32 %i.010.epil, 1
+  %epil.iter.sub = add i32 %epil.iter, -1
+  %epil.iter.cmp = icmp eq i32 %epil.iter.sub, 0
+  br i1 %epil.iter.cmp, label %for.cond.cleanup, label %for.body.epil
+
+for.cond.cleanup:                                 ; preds = %for.body.epil, %for.cond.cleanup.loopexit.unr-lcssa, %entry
+  ret void
+
+for.body:                                         ; preds = %for.body, %for.body.preheader.new
+  %i.010 = phi i32 [ 0, %for.body.preheader.new ], [ %inc.3, %for.body ]
+  %niter = phi i32 [ %unroll_iter, %for.body.preheader.new ], [ %niter.nsub.3, %for.body ]
+  %arrayidx = getelementptr inbounds i16, i16* %A, i32 %i.010
+  %tmp4 = load i16, i16* %arrayidx, align 2
+  %conv = sext i16 %tmp4 to i32
+  %arrayidx1 = getelementptr inbounds i8, i8* %B, i32 %i.010
+  %tmp5 = load i8, i8* %arrayidx1, align 1
+  %conv2 = zext i8 %tmp5 to i32
+  %mul = mul nsw i32 %conv2, %conv
+  %arrayidx3 = getelementptr inbounds i32, i32* %C, i32 %i.010
+  store i32 %mul, i32* %arrayidx3, align 4
+  %inc = or i32 %i.010, 1
+  %arrayidx.1 = getelementptr inbounds i16, i16* %A, i32 %inc
+  %tmp6 = load i16, i16* %arrayidx.1, align 2
+  %conv.1 = sext i16 %tmp6 to i32
+  %arrayidx1.1 = getelementptr inbounds i8, i8* %B, i32 %inc
+  %tmp7 = load i8, i8* %arrayidx1.1, align 1
+  %conv2.1 = zext i8 %tmp7 to i32
+  %mul.1 = mul nsw i32 %conv2.1, %conv.1
+  %arrayidx3.1 = getelementptr inbounds i32, i32* %C, i32 %inc
+  store i32 %mul.1, i32* %arrayidx3.1, align 4
+  %inc.1 = or i32 %i.010, 2
+  %arrayidx.2 = getelementptr inbounds i16, i16* %A, i32 %inc.1
+  %tmp8 = load i16, i16* %arrayidx.2, align 2
+  %conv.2 = sext i16 %tmp8 to i32
+  %arrayidx1.2 = getelementptr inbounds i8, i8* %B, i32 %inc.1
+  %tmp9 = load i8, i8* %arrayidx1.2, align 1
+  %conv2.2 = zext i8 %tmp9 to i32
+  %mul.2 = mul nsw i32 %conv2.2, %conv.2
+  %arrayidx3.2 = getelementptr inbounds i32, i32* %C, i32 %inc.1
+  store i32 %mul.2, i32* %arrayidx3.2, align 4
+  %inc.2 = or i32 %i.010, 3
+  %arrayidx.3 = getelementptr inbounds i16, i16* %A, i32 %inc.2
+  %tmp10 = load i16, i16* %arrayidx.3, align 2
+  %conv.3 = sext i16 %tmp10 to i32
+  %arrayidx1.3 = getelementptr inbounds i8, i8* %B, i32 %inc.2
+  %tmp11 = load i8, i8* %arrayidx1.3, align 1
+  %conv2.3 = zext i8 %tmp11 to i32
+  %mul.3 = mul nsw i32 %conv2.3, %conv.3
+  %arrayidx3.3 = getelementptr inbounds i32, i32* %C, i32 %inc.2
+  store i32 %mul.3, i32* %arrayidx3.3, align 4
+  %inc.3 = add i32 %i.010, 4
+  %niter.nsub.3 = add i32 %niter, -4
+  %niter.ncmp.3 = icmp eq i32 %niter.nsub.3, 0
+  br i1 %niter.ncmp.3, label %for.cond.cleanup.loopexit.unr-lcssa, label %for.body
+}
+
+; CHECK-LABEL: mul_16x16
+; CHECK: @ %for.body
+
+; TODO: pre-indexed loads
+; CHECK-DEFAULT-NOT: ldrsh{{.*}}]!
+; CHECK-DEFAULT: str{{.*}}, #16]!
+; CHECK-DEFAULT-NOT: ldrsh{{.*}}]!
+
+; CHECK-COMPLEX: ldrsh{{.*}}]!
+; CHECK-COMPLEX: ldrsh{{.*}}]!
+; CHECK-COMPLEX: str{{.*}}]!
+
+; DISABLED-NOT: ldr{{.*}}]!
+; DISABLED-NOT: str{{.*}}]!
+
+; CHECK-T2: @ %for.body.epil
+; CHECK-T2: ldrsh{{.*}}, #2]!
+; CHECK-T2: ldrsh{{.*}}, #2]!
+; CHECK-T2: str{{.*}}, #4]!
+
+define void @mul_16x16(i16* nocapture readonly %A, i16* nocapture readonly %B, i32* nocapture %C, i32 %N) {
+entry:
+  %cmp9 = icmp eq i32 %N, 0
+  br i1 %cmp9, label %for.cond.cleanup, label %for.body.preheader
+
+for.body.preheader:                               ; preds = %entry
+  %tmp = add i32 %N, -1
+  %xtraiter = and i32 %N, 3
+  %tmp1 = icmp ult i32 %tmp, 3
+  br i1 %tmp1, label %for.cond.cleanup.loopexit.unr-lcssa, label %for.body.preheader.new
+
+for.body.preheader.new:                           ; preds = %for.body.preheader
+  %unroll_iter = sub i32 %N, %xtraiter
+  br label %for.body
+
+for.cond.cleanup.loopexit.unr-lcssa:              ; preds = %for.body, %for.body.preheader
+  %i.010.unr = phi i32 [ 0, %for.body.preheader ], [ %inc.3, %for.body ]
+  %lcmp.mod = icmp eq i32 %xtraiter, 0
+  br i1 %lcmp.mod, label %for.cond.cleanup, label %for.body.epil
+
+for.body.epil:                                    ; preds = %for.body.epil, %for.cond.cleanup.loopexit.unr-lcssa
+  %i.010.epil = phi i32 [ %inc.epil, %for.body.epil ], [ %i.010.unr, %for.cond.cleanup.loopexit.unr-lcssa ]
+  %epil.iter = phi i32 [ %epil.iter.sub, %for.body.epil ], [ %xtraiter, %for.cond.cleanup.loopexit.unr-lcssa ]
+  %arrayidx.epil = getelementptr inbounds i16, i16* %A, i32 %i.010.epil
+  %tmp2 = load i16, i16* %arrayidx.epil, align 2
+  %conv.epil = sext i16 %tmp2 to i32
+  %arrayidx1.epil = getelementptr inbounds i16, i16* %B, i32 %i.010.epil
+  %tmp3 = load i16, i16* %arrayidx1.epil, align 2
+  %conv2.epil = sext i16 %tmp3 to i32
+  %mul.epil = mul nsw i32 %conv2.epil, %conv.epil
+  %arrayidx3.epil = getelementptr inbounds i32, i32* %C, i32 %i.010.epil
+  store i32 %mul.epil, i32* %arrayidx3.epil, align 4
+  %inc.epil = add nuw i32 %i.010.epil, 1
+  %epil.iter.sub = add i32 %epil.iter, -1
+  %epil.iter.cmp = icmp eq i32 %epil.iter.sub, 0
+  br i1 %epil.iter.cmp, label %for.cond.cleanup, label %for.body.epil
+
+for.cond.cleanup:                                 ; preds = %for.body.epil, %for.cond.cleanup.loopexit.unr-lcssa, %entry
+  ret void
+
+for.body:                                         ; preds = %for.body, %for.body.preheader.new
+  %i.010 = phi i32 [ 0, %for.body.preheader.new ], [ %inc.3, %for.body ]
+  %niter = phi i32 [ %unroll_iter, %for.body.preheader.new ], [ %niter.nsub.3, %for.body ]
+  %arrayidx = getelementptr inbounds i16, i16* %A, i32 %i.010
+  %tmp4 = load i16, i16* %arrayidx, align 2
+  %conv = sext i16 %tmp4 to i32
+  %arrayidx1 = getelementptr inbounds i16, i16* %B, i32 %i.010
+  %tmp5 = load i16, i16* %arrayidx1, align 2
+  %conv2 = sext i16 %tmp5 to i32
+  %mul = mul nsw i32 %conv2, %conv
+  %arrayidx3 = getelementptr inbounds i32, i32* %C, i32 %i.010
+  store i32 %mul, i32* %arrayidx3, align 4
+  %inc = or i32 %i.010, 1
+  %arrayidx.1 = getelementptr inbounds i16, i16* %A, i32 %inc
+  %tmp6 = load i16, i16* %arrayidx.1, align 2
+  %conv.1 = sext i16 %tmp6 to i32
+  %arrayidx1.1 = getelementptr inbounds i16, i16* %B, i32 %inc
+  %tmp7 = load i16, i16* %arrayidx1.1, align 2
+  %conv2.1 = sext i16 %tmp7 to i32
+  %mul.1 = mul nsw i32 %conv2.1, %conv.1
+  %arrayidx3.1 = getelementptr inbounds i32, i32* %C, i32 %inc
+  store i32 %mul.1, i32* %arrayidx3.1, align 4
+  %inc.1 = or i32 %i.010, 2
+  %arrayidx.2 = getelementptr inbounds i16, i16* %A, i32 %inc.1
+  %tmp8 = load i16, i16* %arrayidx.2, align 2
+  %conv.2 = sext i16 %tmp8 to i32
+  %arrayidx1.2 = getelementptr inbounds i16, i16* %B, i32 %inc.1
+  %tmp9 = load i16, i16* %arrayidx1.2, align 2
+  %conv2.2 = sext i16 %tmp9 to i32
+  %mul.2 = mul nsw i32 %conv2.2, %conv.2
+  %arrayidx3.2 = getelementptr inbounds i32, i32* %C, i32 %inc.1
+  store i32 %mul.2, i32* %arrayidx3.2, align 4
+  %inc.2 = or i32 %i.010, 3
+  %arrayidx.3 = getelementptr inbounds i16, i16* %A, i32 %inc.2
+  %tmp10 = load i16, i16* %arrayidx.3, align 2
+  %conv.3 = sext i16 %tmp10 to i32
+  %arrayidx1.3 = getelementptr inbounds i16, i16* %B, i32 %inc.2
+  %tmp11 = load i16, i16* %arrayidx1.3, align 2
+  %conv2.3 = sext i16 %tmp11 to i32
+  %mul.3 = mul nsw i32 %conv2.3, %conv.3
+  %arrayidx3.3 = getelementptr inbounds i32, i32* %C, i32 %inc.2
+  store i32 %mul.3, i32* %arrayidx3.3, align 4
+  %inc.3 = add i32 %i.010, 4
+  %niter.nsub.3 = add i32 %niter, -4
+  %niter.ncmp.3 = icmp eq i32 %niter.nsub.3, 0
+  br i1 %niter.ncmp.3, label %for.cond.cleanup.loopexit.unr-lcssa, label %for.body
+}
+
+; CHECK-LABEL: mul_8x8_2d
+; CHECK: @ %for.body4.us
+
+; CHECK-DEFAULT: ldr{{.*}}, #16]!
+; CHECK-DEFAULT: ldrb{{.*}}, #4]!
+
+; DISABLED-NOT: ldr{{.*}}]!
+; DISABLED-NOT: str{{.*}}]!
+
+; CHECK-T2: @ %for.body4.us.epil
+; CHECK-T2: ldrb{{.*}}, #1]!
+; CHECK-T2: ldr{{.*}}, #4]!
+
+define void @mul_8x8_2d(i8* nocapture readonly %A, i8** nocapture readonly %B, i32** nocapture readonly %C, i32 %N, i32 %M) {
+entry:
+  %cmp24 = icmp eq i32 %N, 0
+  %cmp222 = icmp eq i32 %M, 0
+  %or.cond = or i1 %cmp24, %cmp222
+  br i1 %or.cond, label %for.cond.cleanup, label %for.cond1.preheader.us.preheader
+
+for.cond1.preheader.us.preheader:                 ; preds = %entry
+  %tmp = add i32 %M, -1
+  %xtraiter = and i32 %M, 3
+  %tmp1 = icmp ult i32 %tmp, 3
+  %unroll_iter = sub i32 %M, %xtraiter
+  %lcmp.mod = icmp eq i32 %xtraiter, 0
+  br label %for.cond1.preheader.us
+
+for.cond1.preheader.us:                           ; preds = %for.cond1.for.cond.cleanup3_crit_edge.us, %for.cond1.preheader.us.preheader
+  %i.025.us = phi i32 [ %inc11.us, %for.cond1.for.cond.cleanup3_crit_edge.us ], [ 0, %for.cond1.preheader.us.preheader ]
+  %arrayidx.us = getelementptr inbounds i8, i8* %A, i32 %i.025.us
+  %arrayidx5.us = getelementptr inbounds i8*, i8** %B, i32 %i.025.us
+  %arrayidx8.us = getelementptr inbounds i32*, i32** %C, i32 %i.025.us
+  %.pre = load i8*, i8** %arrayidx5.us, align 4
+  %.pre30 = load i32*, i32** %arrayidx8.us, align 4
+  br i1 %tmp1, label %for.cond1.for.cond.cleanup3_crit_edge.us.unr-lcssa, label %for.body4.us
+
+for.body4.us:                                     ; preds = %for.body4.us, %for.cond1.preheader.us
+  %j.023.us = phi i32 [ %inc.us.3, %for.body4.us ], [ 0, %for.cond1.preheader.us ]
+  %niter = phi i32 [ %niter.nsub.3, %for.body4.us ], [ %unroll_iter, %for.cond1.preheader.us ]
+  %tmp2 = load i8, i8* %arrayidx.us, align 1
+  %conv.us = zext i8 %tmp2 to i32
+  %arrayidx6.us = getelementptr inbounds i8, i8* %.pre, i32 %j.023.us
+  %tmp3 = load i8, i8* %arrayidx6.us, align 1
+  %conv7.us = zext i8 %tmp3 to i32
+  %mul.us = mul nuw nsw i32 %conv7.us, %conv.us
+  %arrayidx9.us = getelementptr inbounds i32, i32* %.pre30, i32 %j.023.us
+  %tmp4 = load i32, i32* %arrayidx9.us, align 4
+  %add.us = add nsw i32 %tmp4, %mul.us
+  store i32 %add.us, i32* %arrayidx9.us, align 4
+  %inc.us = or i32 %j.023.us, 1
+  %tmp5 = load i8, i8* %arrayidx.us, align 1
+  %conv.us.1 = zext i8 %tmp5 to i32
+  %arrayidx6.us.1 = getelementptr inbounds i8, i8* %.pre, i32 %inc.us
+  %tmp6 = load i8, i8* %arrayidx6.us.1, align 1
+  %conv7.us.1 = zext i8 %tmp6 to i32
+  %mul.us.1 = mul nuw nsw i32 %conv7.us.1, %conv.us.1
+  %arrayidx9.us.1 = getelementptr inbounds i32, i32* %.pre30, i32 %inc.us
+  %tmp7 = load i32, i32* %arrayidx9.us.1, align 4
+  %add.us.1 = add nsw i32 %tmp7, %mul.us.1
+  store i32 %add.us.1, i32* %arrayidx9.us.1, align 4
+  %inc.us.1 = or i32 %j.023.us, 2
+  %tmp8 = load i8, i8* %arrayidx.us, align 1
+  %conv.us.2 = zext i8 %tmp8 to i32
+  %arrayidx6.us.2 = getelementptr inbounds i8, i8* %.pre, i32 %inc.us.1
+  %tmp9 = load i8, i8* %arrayidx6.us.2, align 1
+  %conv7.us.2 = zext i8 %tmp9 to i32
+  %mul.us.2 = mul nuw nsw i32 %conv7.us.2, %conv.us.2
+  %arrayidx9.us.2 = getelementptr inbounds i32, i32* %.pre30, i32 %inc.us.1
+  %tmp10 = load i32, i32* %arrayidx9.us.2, align 4
+  %add.us.2 = add nsw i32 %tmp10, %mul.us.2
+  store i32 %add.us.2, i32* %arrayidx9.us.2, align 4
+  %inc.us.2 = or i32 %j.023.us, 3
+  %tmp11 = load i8, i8* %arrayidx.us, align 1
+  %conv.us.3 = zext i8 %tmp11 to i32
+  %arrayidx6.us.3 = getelementptr inbounds i8, i8* %.pre, i32 %inc.us.2
+  %tmp12 = load i8, i8* %arrayidx6.us.3, align 1
+  %conv7.us.3 = zext i8 %tmp12 to i32
+  %mul.us.3 = mul nuw nsw i32 %conv7.us.3, %conv.us.3
+  %arrayidx9.us.3 = getelementptr inbounds i32, i32* %.pre30, i32 %inc.us.2
+  %tmp13 = load i32, i32* %arrayidx9.us.3, align 4
+  %add.us.3 = add nsw i32 %tmp13, %mul.us.3
+  store i32 %add.us.3, i32* %arrayidx9.us.3, align 4
+  %inc.us.3 = add i32 %j.023.us, 4
+  %niter.nsub.3 = add i32 %niter, -4
+  %niter.ncmp.3 = icmp eq i32 %niter.nsub.3, 0
+  br i1 %niter.ncmp.3, label %for.cond1.for.cond.cleanup3_crit_edge.us.unr-lcssa, label %for.body4.us
+
+for.cond1.for.cond.cleanup3_crit_edge.us.unr-lcssa: ; preds = %for.body4.us, %for.cond1.preheader.us
+  %j.023.us.unr = phi i32 [ 0, %for.cond1.preheader.us ], [ %inc.us.3, %for.body4.us ]
+  br i1 %lcmp.mod, label %for.cond1.for.cond.cleanup3_crit_edge.us, label %for.body4.us.epil
+
+for.body4.us.epil:                                ; preds = %for.body4.us.epil, %for.cond1.for.cond.cleanup3_crit_edge.us.unr-lcssa
+  %j.023.us.epil = phi i32 [ %inc.us.epil, %for.body4.us.epil ], [ %j.023.us.unr, %for.cond1.for.cond.cleanup3_crit_edge.us.unr-lcssa ]
+  %epil.iter = phi i32 [ %epil.iter.sub, %for.body4.us.epil ], [ %xtraiter, %for.cond1.for.cond.cleanup3_crit_edge.us.unr-lcssa ]
+  %tmp14 = load i8, i8* %arrayidx.us, align 1
+  %conv.us.epil = zext i8 %tmp14 to i32
+  %arrayidx6.us.epil = getelementptr inbounds i8, i8* %.pre, i32 %j.023.us.epil
+  %tmp15 = load i8, i8* %arrayidx6.us.epil, align 1
+  %conv7.us.epil = zext i8 %tmp15 to i32
+  %mul.us.epil = mul nuw nsw i32 %conv7.us.epil, %conv.us.epil
+  %arrayidx9.us.epil = getelementptr inbounds i32, i32* %.pre30, i32 %j.023.us.epil
+  %tmp16 = load i32, i32* %arrayidx9.us.epil, align 4
+  %add.us.epil = add nsw i32 %tmp16, %mul.us.epil
+  store i32 %add.us.epil, i32* %arrayidx9.us.epil, align 4
+  %inc.us.epil = add nuw i32 %j.023.us.epil, 1
+  %epil.iter.sub = add i32 %epil.iter, -1
+  %epil.iter.cmp = icmp eq i32 %epil.iter.sub, 0
+  br i1 %epil.iter.cmp, label %for.cond1.for.cond.cleanup3_crit_edge.us, label %for.body4.us.epil
+
+for.cond1.for.cond.cleanup3_crit_edge.us:         ; preds = %for.body4.us.epil, %for.cond1.for.cond.cleanup3_crit_edge.us.unr-lcssa
+  %inc11.us = add nuw i32 %i.025.us, 1
+  %exitcond28 = icmp eq i32 %inc11.us, %N
+  br i1 %exitcond28, label %for.cond.cleanup, label %for.cond1.preheader.us
+
+for.cond.cleanup:                                 ; preds = %for.cond1.for.cond.cleanup3_crit_edge.us, %entry
+  ret void
+}
+
+; CHECK-LABEL: mul_16x16_2d
+; CHECK: @ %for.body4.us
+
+; CHECK-DEFAULT: ldr{{.*}}, #16]!
+; CHECK-DEFAULT: ldrsh{{.*}}, #8]!
+
+; DISABLED-NOT: ldr{{.*}}]!
+; DISABLED-NOT: str{{.*}}]!
+
+; CHECK-T2: @ %for.body4.us.epil
+; CHECK-T2: ldrsh{{.*}}, #2]!
+; CHECK-T2: ldr{{.*}}, #4]!
+
+define void @mul_16x16_2d(i16* nocapture readonly %A, i16** nocapture readonly %B, i32** nocapture readonly %C, i32 %N, i32 %M) {
+entry:
+  %cmp24 = icmp eq i32 %N, 0
+  %cmp222 = icmp eq i32 %M, 0
+  %or.cond = or i1 %cmp24, %cmp222
+  br i1 %or.cond, label %for.cond.cleanup, label %for.cond1.preheader.us.preheader
+
+for.cond1.preheader.us.preheader:                 ; preds = %entry
+  %tmp = add i32 %M, -1
+  %xtraiter = and i32 %M, 3
+  %tmp1 = icmp ult i32 %tmp, 3
+  %unroll_iter = sub i32 %M, %xtraiter
+  %lcmp.mod = icmp eq i32 %xtraiter, 0
+  br label %for.cond1.preheader.us
+
+for.cond1.preheader.us:                           ; preds = %for.cond1.for.cond.cleanup3_crit_edge.us, %for.cond1.preheader.us.preheader
+  %i.025.us = phi i32 [ %inc11.us, %for.cond1.for.cond.cleanup3_crit_edge.us ], [ 0, %for.cond1.preheader.us.preheader ]
+  %arrayidx.us = getelementptr inbounds i16, i16* %A, i32 %i.025.us
+  %tmp2 = load i16, i16* %arrayidx.us, align 2
+  %conv.us = sext i16 %tmp2 to i32
+  %arrayidx5.us = getelementptr inbounds i16*, i16** %B, i32 %i.025.us
+  %tmp3 = load i16*, i16** %arrayidx5.us, align 4
+  %arrayidx8.us = getelementptr inbounds i32*, i32** %C, i32 %i.025.us
+  %tmp4 = load i32*, i32** %arrayidx8.us, align 4
+  br i1 %tmp1, label %for.cond1.for.cond.cleanup3_crit_edge.us.unr-lcssa, label %for.body4.us
+
+for.body4.us:                                     ; preds = %for.body4.us, %for.cond1.preheader.us
+  %j.023.us = phi i32 [ %inc.us.3, %for.body4.us ], [ 0, %for.cond1.preheader.us ]
+  %niter = phi i32 [ %niter.nsub.3, %for.body4.us ], [ %unroll_iter, %for.cond1.preheader.us ]
+  %arrayidx6.us = getelementptr inbounds i16, i16* %tmp3, i32 %j.023.us
+  %tmp5 = load i16, i16* %arrayidx6.us, align 2
+  %conv7.us = sext i16 %tmp5 to i32
+  %mul.us = mul nsw i32 %conv7.us, %conv.us
+  %arrayidx9.us = getelementptr inbounds i32, i32* %tmp4, i32 %j.023.us
+  %tmp6 = load i32, i32* %arrayidx9.us, align 4
+  %add.us = add nsw i32 %tmp6, %mul.us
+  store i32 %add.us, i32* %arrayidx9.us, align 4
+  %inc.us = or i32 %j.023.us, 1
+  %arrayidx6.us.1 = getelementptr inbounds i16, i16* %tmp3, i32 %inc.us
+  %tmp7 = load i16, i16* %arrayidx6.us.1, align 2
+  %conv7.us.1 = sext i16 %tmp7 to i32
+  %mul.us.1 = mul nsw i32 %conv7.us.1, %conv.us
+  %arrayidx9.us.1 = getelementptr inbounds i32, i32* %tmp4, i32 %inc.us
+  %tmp8 = load i32, i32* %arrayidx9.us.1, align 4
+  %add.us.1 = add nsw i32 %tmp8, %mul.us.1
+  store i32 %add.us.1, i32* %arrayidx9.us.1, align 4
+  %inc.us.1 = or i32 %j.023.us, 2
+  %arrayidx6.us.2 = getelementptr inbounds i16, i16* %tmp3, i32 %inc.us.1
+  %tmp9 = load i16, i16* %arrayidx6.us.2, align 2
+  %conv7.us.2 = sext i16 %tmp9 to i32
+  %mul.us.2 = mul nsw i32 %conv7.us.2, %conv.us
+  %arrayidx9.us.2 = getelementptr inbounds i32, i32* %tmp4, i32 %inc.us.1
+  %tmp10 = load i32, i32* %arrayidx9.us.2, align 4
+  %add.us.2 = add nsw i32 %tmp10, %mul.us.2
+  store i32 %add.us.2, i32* %arrayidx9.us.2, align 4
+  %inc.us.2 = or i32 %j.023.us, 3
+  %arrayidx6.us.3 = getelementptr inbounds i16, i16* %tmp3, i32 %inc.us.2
+  %tmp11 = load i16, i16* %arrayidx6.us.3, align 2
+  %conv7.us.3 = sext i16 %tmp11 to i32
+  %mul.us.3 = mul nsw i32 %conv7.us.3, %conv.us
+  %arrayidx9.us.3 = getelementptr inbounds i32, i32* %tmp4, i32 %inc.us.2
+  %tmp12 = load i32, i32* %arrayidx9.us.3, align 4
+  %add.us.3 = add nsw i32 %tmp12, %mul.us.3
+  store i32 %add.us.3, i32* %arrayidx9.us.3, align 4
+  %inc.us.3 = add i32 %j.023.us, 4
+  %niter.nsub.3 = add i32 %niter, -4
+  %niter.ncmp.3 = icmp eq i32 %niter.nsub.3, 0
+  br i1 %niter.ncmp.3, label %for.cond1.for.cond.cleanup3_crit_edge.us.unr-lcssa, label %for.body4.us
+
+for.cond1.for.cond.cleanup3_crit_edge.us.unr-lcssa: ; preds = %for.body4.us, %for.cond1.preheader.us
+  %j.023.us.unr = phi i32 [ 0, %for.cond1.preheader.us ], [ %inc.us.3, %for.body4.us ]
+  br i1 %lcmp.mod, label %for.cond1.for.cond.cleanup3_crit_edge.us, label %for.body4.us.epil
+
+for.body4.us.epil:                                ; preds = %for.body4.us.epil, %for.cond1.for.cond.cleanup3_crit_edge.us.unr-lcssa
+  %j.023.us.epil = phi i32 [ %inc.us.epil, %for.body4.us.epil ], [ %j.023.us.unr, %for.cond1.for.cond.cleanup3_crit_edge.us.unr-lcssa ]
+  %epil.iter = phi i32 [ %epil.iter.sub, %for.body4.us.epil ], [ %xtraiter, %for.cond1.for.cond.cleanup3_crit_edge.us.unr-lcssa ]
+  %arrayidx6.us.epil = getelementptr inbounds i16, i16* %tmp3, i32 %j.023.us.epil
+  %tmp13 = load i16, i16* %arrayidx6.us.epil, align 2
+  %conv7.us.epil = sext i16 %tmp13 to i32
+  %mul.us.epil = mul nsw i32 %conv7.us.epil, %conv.us
+  %arrayidx9.us.epil = getelementptr inbounds i32, i32* %tmp4, i32 %j.023.us.epil
+  %tmp14 = load i32, i32* %arrayidx9.us.epil, align 4
+  %add.us.epil = add nsw i32 %tmp14, %mul.us.epil
+  store i32 %add.us.epil, i32* %arrayidx9.us.epil, align 4
+  %inc.us.epil = add nuw i32 %j.023.us.epil, 1
+  %epil.iter.sub = add i32 %epil.iter, -1
+  %epil.iter.cmp = icmp eq i32 %epil.iter.sub, 0
+  br i1 %epil.iter.cmp, label %for.cond1.for.cond.cleanup3_crit_edge.us, label %for.body4.us.epil
+
+for.cond1.for.cond.cleanup3_crit_edge.us:         ; preds = %for.body4.us.epil, %for.cond1.for.cond.cleanup3_crit_edge.us.unr-lcssa
+  %inc11.us = add nuw i32 %i.025.us, 1
+  %exitcond28 = icmp eq i32 %inc11.us, %N
+  br i1 %exitcond28, label %for.cond.cleanup, label %for.cond1.preheader.us
+
+for.cond.cleanup:                                 ; preds = %for.cond1.for.cond.cleanup3_crit_edge.us, %entry
+  ret void
+}
+
+; CHECK-LABEL: mac_8x8_2d
+; CHECK: @ %for.body4.us
+
+; TODO: Both input arrays could use pre-indexed loads.
+; TODO: pre-indexed stores.
+; CHECK-DEFAULT: ldrb{{.*}}, #4]!
+; CHECK-DEFAULT-NOT: ldr{{.*}}]!
+; CHECK-DEFAULT-NOT: str{{.*}}]!
+
+; TODO: Increased complexity shouldn't prevent indexed accesses.
+; CHECK-COMPLEX-NOT: ldr{{.*}}]!
+; CHECK-COMPLEX-NOT: str{{.*}}]!
+
+; DISABLED-NOT: ldr{{.*}}]!
+; DISABLED-NOT: str{{.*}}]!
+
+; CHECK-T2: @ %for.body4.us.epil
+; CHECK-T2: ldrb{{.*}}, #1]!
+
+define void @mac_8x8_2d(i8* nocapture readonly %A, i8** nocapture readonly %B, i32* nocapture %C, i32 %N, i32 %M) {
+entry:
+  %cmp22 = icmp eq i32 %N, 0
+  %cmp220 = icmp eq i32 %M, 0
+  %or.cond = or i1 %cmp22, %cmp220
+  br i1 %or.cond, label %for.cond.cleanup, label %for.cond1.preheader.us.preheader
+
+for.cond1.preheader.us.preheader:                 ; preds = %entry
+  %tmp = add i32 %M, -1
+  %xtraiter = and i32 %M, 3
+  %tmp1 = icmp ult i32 %tmp, 3
+  %unroll_iter = sub i32 %M, %xtraiter
+  %lcmp.mod = icmp eq i32 %xtraiter, 0
+  br label %for.cond1.preheader.us
+
+for.cond1.preheader.us:                           ; preds = %for.cond1.for.cond.cleanup3_crit_edge.us, %for.cond1.preheader.us.preheader
+  %i.023.us = phi i32 [ %inc10.us, %for.cond1.for.cond.cleanup3_crit_edge.us ], [ 0, %for.cond1.preheader.us.preheader ]
+  %arrayidx.us = getelementptr inbounds i8, i8* %A, i32 %i.023.us
+  %arrayidx5.us = getelementptr inbounds i8*, i8** %B, i32 %i.023.us
+  %arrayidx8.us = getelementptr inbounds i32, i32* %C, i32 %i.023.us
+  %.pre = load i8*, i8** %arrayidx5.us, align 4
+  %.pre28 = load i32, i32* %arrayidx8.us, align 4
+  br i1 %tmp1, label %for.cond1.for.cond.cleanup3_crit_edge.us.unr-lcssa, label %for.body4.us
+
+for.body4.us:                                     ; preds = %for.body4.us, %for.cond1.preheader.us
+  %tmp2 = phi i32 [ %add.us.3, %for.body4.us ], [ %.pre28, %for.cond1.preheader.us ]
+  %j.021.us = phi i32 [ %inc.us.3, %for.body4.us ], [ 0, %for.cond1.preheader.us ]
+  %niter = phi i32 [ %niter.nsub.3, %for.body4.us ], [ %unroll_iter, %for.cond1.preheader.us ]
+  %tmp3 = load i8, i8* %arrayidx.us, align 1
+  %conv.us = zext i8 %tmp3 to i32
+  %arrayidx6.us = getelementptr inbounds i8, i8* %.pre, i32 %j.021.us
+  %tmp4 = load i8, i8* %arrayidx6.us, align 1
+  %conv7.us = zext i8 %tmp4 to i32
+  %mul.us = mul nuw nsw i32 %conv7.us, %conv.us
+  %add.us = add nsw i32 %mul.us, %tmp2
+  store i32 %add.us, i32* %arrayidx8.us, align 4
+  %inc.us = or i32 %j.021.us, 1
+  %tmp5 = load i8, i8* %arrayidx.us, align 1
+  %conv.us.1 = zext i8 %tmp5 to i32
+  %arrayidx6.us.1 = getelementptr inbounds i8, i8* %.pre, i32 %inc.us
+  %tmp6 = load i8, i8* %arrayidx6.us.1, align 1
+  %conv7.us.1 = zext i8 %tmp6 to i32
+  %mul.us.1 = mul nuw nsw i32 %conv7.us.1, %conv.us.1
+  %add.us.1 = add nsw i32 %mul.us.1, %add.us
+  store i32 %add.us.1, i32* %arrayidx8.us, align 4
+  %inc.us.1 = or i32 %j.021.us, 2
+  %tmp7 = load i8, i8* %arrayidx.us, align 1
+  %conv.us.2 = zext i8 %tmp7 to i32
+  %arrayidx6.us.2 = getelementptr inbounds i8, i8* %.pre, i32 %inc.us.1
+  %tmp8 = load i8, i8* %arrayidx6.us.2, align 1
+  %conv7.us.2 = zext i8 %tmp8 to i32
+  %mul.us.2 = mul nuw nsw i32 %conv7.us.2, %conv.us.2
+  %add.us.2 = add nsw i32 %mul.us.2, %add.us.1
+  store i32 %add.us.2, i32* %arrayidx8.us, align 4
+  %inc.us.2 = or i32 %j.021.us, 3
+  %tmp9 = load i8, i8* %arrayidx.us, align 1
+  %conv.us.3 = zext i8 %tmp9 to i32
+  %arrayidx6.us.3 = getelementptr inbounds i8, i8* %.pre, i32 %inc.us.2
+  %tmp10 = load i8, i8* %arrayidx6.us.3, align 1
+  %conv7.us.3 = zext i8 %tmp10 to i32
+  %mul.us.3 = mul nuw nsw i32 %conv7.us.3, %conv.us.3
+  %add.us.3 = add nsw i32 %mul.us.3, %add.us.2
+  store i32 %add.us.3, i32* %arrayidx8.us, align 4
+  %inc.us.3 = add i32 %j.021.us, 4
+  %niter.nsub.3 = add i32 %niter, -4
+  %niter.ncmp.3 = icmp eq i32 %niter.nsub.3, 0
+  br i1 %niter.ncmp.3, label %for.cond1.for.cond.cleanup3_crit_edge.us.unr-lcssa, label %for.body4.us
+
+for.cond1.for.cond.cleanup3_crit_edge.us.unr-lcssa: ; preds = %for.body4.us, %for.cond1.preheader.us
+  %.unr = phi i32 [ %.pre28, %for.cond1.preheader.us ], [ %add.us.3, %for.body4.us ]
+  %j.021.us.unr = phi i32 [ 0, %for.cond1.preheader.us ], [ %inc.us.3, %for.body4.us ]
+  br i1 %lcmp.mod, label %for.cond1.for.cond.cleanup3_crit_edge.us, label %for.body4.us.epil
+
+for.body4.us.epil:                                ; preds = %for.body4.us.epil, %for.cond1.for.cond.cleanup3_crit_edge.us.unr-lcssa
+  %tmp11 = phi i32 [ %add.us.epil, %for.body4.us.epil ], [ %.unr, %for.cond1.for.cond.cleanup3_crit_edge.us.unr-lcssa ]
+  %j.021.us.epil = phi i32 [ %inc.us.epil, %for.body4.us.epil ], [ %j.021.us.unr, %for.cond1.for.cond.cleanup3_crit_edge.us.unr-lcssa ]
+  %epil.iter = phi i32 [ %epil.iter.sub, %for.body4.us.epil ], [ %xtraiter, %for.cond1.for.cond.cleanup3_crit_edge.us.unr-lcssa ]
+  %tmp12 = load i8, i8* %arrayidx.us, align 1
+  %conv.us.epil = zext i8 %tmp12 to i32
+  %arrayidx6.us.epil = getelementptr inbounds i8, i8* %.pre, i32 %j.021.us.epil
+  %tmp13 = load i8, i8* %arrayidx6.us.epil, align 1
+  %conv7.us.epil = zext i8 %tmp13 to i32
+  %mul.us.epil = mul nuw nsw i32 %conv7.us.epil, %conv.us.epil
+  %add.us.epil = add nsw i32 %mul.us.epil, %tmp11
+  store i32 %add.us.epil, i32* %arrayidx8.us, align 4
+  %inc.us.epil = add nuw i32 %j.021.us.epil, 1
+  %epil.iter.sub = add i32 %epil.iter, -1
+  %epil.iter.cmp = icmp eq i32 %epil.iter.sub, 0
+  br i1 %epil.iter.cmp, label %for.cond1.for.cond.cleanup3_crit_edge.us, label %for.body4.us.epil
+
+for.cond1.for.cond.cleanup3_crit_edge.us:         ; preds = %for.body4.us.epil, %for.cond1.for.cond.cleanup3_crit_edge.us.unr-lcssa
+  %inc10.us = add nuw i32 %i.023.us, 1
+  %exitcond26 = icmp eq i32 %inc10.us, %N
+  br i1 %exitcond26, label %for.cond.cleanup, label %for.cond1.preheader.us
+
+for.cond.cleanup:                                 ; preds = %for.cond1.for.cond.cleanup3_crit_edge.us, %entry
+  ret void
+}
+
+; CHECK-LABEL: mac_16x16_2d
+; CHECK: @ %for.body4.us
+
+; TODO: pre-indexed loads for both input arrays.
+; CHECK-DEFAULT: ldrsh{{.*}}, #8]!
+; CHECK-DEFAULT-NOT: ldr{{.*}}]!
+
+; TODO: increased complexity should lead to better codegen.
+; CHECK-COMPLEX-NOT: ldr{{.*}}]!
+
+; DISABLED-NOT: ldr{{.*}}]!
+
+; CHECK-T2: @ %for.body4.us.epil
+; CHECK-T2: ldrsh{{.*}}, #2]!
+
+define void @mac_16x16_2d(i16* nocapture readonly %A, i16** nocapture readonly %B, i32* nocapture %C, i32 %N, i32 %M) {
+entry:
+  %cmp23 = icmp eq i32 %N, 0
+  %cmp220 = icmp eq i32 %M, 0
+  %or.cond = or i1 %cmp23, %cmp220
+  br i1 %or.cond, label %for.cond.cleanup, label %for.cond1.preheader.us.preheader
+
+for.cond1.preheader.us.preheader:                 ; preds = %entry
+  %tmp = add i32 %M, -1
+  %xtraiter = and i32 %M, 3
+  %tmp1 = icmp ult i32 %tmp, 3
+  %unroll_iter = sub i32 %M, %xtraiter
+  %lcmp.mod = icmp eq i32 %xtraiter, 0
+  br label %for.cond1.preheader.us
+
+for.cond1.preheader.us:                           ; preds = %for.cond1.for.cond.cleanup3_crit_edge.us, %for.cond1.preheader.us.preheader
+  %i.024.us = phi i32 [ %inc10.us, %for.cond1.for.cond.cleanup3_crit_edge.us ], [ 0, %for.cond1.preheader.us.preheader ]
+  %arrayidx.us = getelementptr inbounds i16, i16* %A, i32 %i.024.us
+  %tmp2 = load i16, i16* %arrayidx.us, align 2
+  %conv.us = sext i16 %tmp2 to i32
+  %arrayidx5.us = getelementptr inbounds i16*, i16** %B, i32 %i.024.us
+  %tmp3 = load i16*, i16** %arrayidx5.us, align 4
+  %arrayidx8.us = getelementptr inbounds i32, i32* %C, i32 %i.024.us
+  %arrayidx8.promoted.us = load i32, i32* %arrayidx8.us, align 4
+  br i1 %tmp1, label %for.cond1.for.cond.cleanup3_crit_edge.us.unr-lcssa, label %for.body4.us
+
+for.body4.us:                                     ; preds = %for.body4.us, %for.cond1.preheader.us
+  %add22.us = phi i32 [ %add.us.3, %for.body4.us ], [ %arrayidx8.promoted.us, %for.cond1.preheader.us ]
+  %j.021.us = phi i32 [ %inc.us.3, %for.body4.us ], [ 0, %for.cond1.preheader.us ]
+  %niter = phi i32 [ %niter.nsub.3, %for.body4.us ], [ %unroll_iter, %for.cond1.preheader.us ]
+  %arrayidx6.us = getelementptr inbounds i16, i16* %tmp3, i32 %j.021.us
+  %tmp4 = load i16, i16* %arrayidx6.us, align 2
+  %conv7.us = sext i16 %tmp4 to i32
+  %mul.us = mul nsw i32 %conv7.us, %conv.us
+  %add.us = add nsw i32 %mul.us, %add22.us
+  %inc.us = or i32 %j.021.us, 1
+  %arrayidx6.us.1 = getelementptr inbounds i16, i16* %tmp3, i32 %inc.us
+  %tmp5 = load i16, i16* %arrayidx6.us.1, align 2
+  %conv7.us.1 = sext i16 %tmp5 to i32
+  %mul.us.1 = mul nsw i32 %conv7.us.1, %conv.us
+  %add.us.1 = add nsw i32 %mul.us.1, %add.us
+  %inc.us.1 = or i32 %j.021.us, 2
+  %arrayidx6.us.2 = getelementptr inbounds i16, i16* %tmp3, i32 %inc.us.1
+  %tmp6 = load i16, i16* %arrayidx6.us.2, align 2
+  %conv7.us.2 = sext i16 %tmp6 to i32
+  %mul.us.2 = mul nsw i32 %conv7.us.2, %conv.us
+  %add.us.2 = add nsw i32 %mul.us.2, %add.us.1
+  %inc.us.2 = or i32 %j.021.us, 3
+  %arrayidx6.us.3 = getelementptr inbounds i16, i16* %tmp3, i32 %inc.us.2
+  %tmp7 = load i16, i16* %arrayidx6.us.3, align 2
+  %conv7.us.3 = sext i16 %tmp7 to i32
+  %mul.us.3 = mul nsw i32 %conv7.us.3, %conv.us
+  %add.us.3 = add nsw i32 %mul.us.3, %add.us.2
+  %inc.us.3 = add i32 %j.021.us, 4
+  %niter.nsub.3 = add i32 %niter, -4
+  %niter.ncmp.3 = icmp eq i32 %niter.nsub.3, 0
+  br i1 %niter.ncmp.3, label %for.cond1.for.cond.cleanup3_crit_edge.us.unr-lcssa, label %for.body4.us
+
+for.cond1.for.cond.cleanup3_crit_edge.us.unr-lcssa: ; preds = %for.body4.us, %for.cond1.preheader.us
+  %add.us.lcssa.ph = phi i32 [ undef, %for.cond1.preheader.us ], [ %add.us.3, %for.body4.us ]
+  %add22.us.unr = phi i32 [ %arrayidx8.promoted.us, %for.cond1.preheader.us ], [ %add.us.3, %for.body4.us ]
+  %j.021.us.unr = phi i32 [ 0, %for.cond1.preheader.us ], [ %inc.us.3, %for.body4.us ]
+  br i1 %lcmp.mod, label %for.cond1.for.cond.cleanup3_crit_edge.us, label %for.body4.us.epil
+
+for.body4.us.epil:                                ; preds = %for.body4.us.epil, %for.cond1.for.cond.cleanup3_crit_edge.us.unr-lcssa
+  %add22.us.epil = phi i32 [ %add.us.epil, %for.body4.us.epil ], [ %add22.us.unr, %for.cond1.for.cond.cleanup3_crit_edge.us.unr-lcssa ]
+  %j.021.us.epil = phi i32 [ %inc.us.epil, %for.body4.us.epil ], [ %j.021.us.unr, %for.cond1.for.cond.cleanup3_crit_edge.us.unr-lcssa ]
+  %epil.iter = phi i32 [ %epil.iter.sub, %for.body4.us.epil ], [ %xtraiter, %for.cond1.for.cond.cleanup3_crit_edge.us.unr-lcssa ]
+  %arrayidx6.us.epil = getelementptr inbounds i16, i16* %tmp3, i32 %j.021.us.epil
+  %tmp8 = load i16, i16* %arrayidx6.us.epil, align 2
+  %conv7.us.epil = sext i16 %tmp8 to i32
+  %mul.us.epil = mul nsw i32 %conv7.us.epil, %conv.us
+  %add.us.epil = add nsw i32 %mul.us.epil, %add22.us.epil
+  %inc.us.epil = add nuw i32 %j.021.us.epil, 1
+  %epil.iter.sub = add i32 %epil.iter, -1
+  %epil.iter.cmp = icmp eq i32 %epil.iter.sub, 0
+  br i1 %epil.iter.cmp, label %for.cond1.for.cond.cleanup3_crit_edge.us, label %for.body4.us.epil
+
+for.cond1.for.cond.cleanup3_crit_edge.us:         ; preds = %for.body4.us.epil, %for.cond1.for.cond.cleanup3_crit_edge.us.unr-lcssa
+  %add.us.lcssa = phi i32 [ %add.us.lcssa.ph, %for.cond1.for.cond.cleanup3_crit_edge.us.unr-lcssa ], [ %add.us.epil, %for.body4.us.epil ]
+  store i32 %add.us.lcssa, i32* %arrayidx8.us, align 4
+  %inc10.us = add nuw i32 %i.024.us, 1
+  %exitcond27 = icmp eq i32 %inc10.us, %N
+  br i1 %exitcond27, label %for.cond.cleanup, label %for.cond1.preheader.us
+
+for.cond.cleanup:                                 ; preds = %for.cond1.for.cond.cleanup3_crit_edge.us, %entry
+  ret void
+}
+
+; CHECK-LABEL: mul32x32_backwards
+; CHECK: @ %for.body
+
+; TODO: post increments for decreasing addresses
+; CHECK-DEFAULT-NOT: ldr{{.*}}]!
+; CHECK-DEFAULT-NOT: str{{.*}}]!
+
+; CHECK-COMPLEX-NOT: ldr{{.*}}]!
+; CHECK-COMPLEX-NOT: str{{.*}}]!
+
+define void @mul32x32_backwards(i32* nocapture %a, i32* nocapture readonly %b, i32* nocapture readonly %c, i32 %N) {
+entry:
+  %i.08 = add i32 %N, -1
+  %cmp9 = icmp sgt i32 %i.08, -1
+  br i1 %cmp9, label %for.body.preheader, label %for.cond.cleanup
+
+for.body.preheader:                               ; preds = %entry
+  %xtraiter = and i32 %N, 3
+  %lcmp.mod = icmp eq i32 %xtraiter, 0
+  br i1 %lcmp.mod, label %for.body.prol.loopexit, label %for.body.prol
+
+for.body.prol:                                    ; preds = %for.body.prol, %for.body.preheader
+  %i.010.prol = phi i32 [ %i.0.prol, %for.body.prol ], [ %i.08, %for.body.preheader ]
+  %prol.iter = phi i32 [ %prol.iter.sub, %for.body.prol ], [ %xtraiter, %for.body.preheader ]
+  %arrayidx.prol = getelementptr inbounds i32, i32* %b, i32 %i.010.prol
+  %tmp = load i32, i32* %arrayidx.prol, align 4
+  %arrayidx1.prol = getelementptr inbounds i32, i32* %c, i32 %i.010.prol
+  %tmp1 = load i32, i32* %arrayidx1.prol, align 4
+  %mul.prol = mul nsw i32 %tmp1, %tmp
+  %arrayidx2.prol = getelementptr inbounds i32, i32* %a, i32 %i.010.prol
+  store i32 %mul.prol, i32* %arrayidx2.prol, align 4
+  %i.0.prol = add i32 %i.010.prol, -1
+  %prol.iter.sub = add i32 %prol.iter, -1
+  %prol.iter.cmp = icmp eq i32 %prol.iter.sub, 0
+  br i1 %prol.iter.cmp, label %for.body.prol.loopexit, label %for.body.prol
+
+for.body.prol.loopexit:                           ; preds = %for.body.prol, %for.body.preheader
+  %i.010.unr = phi i32 [ %i.08, %for.body.preheader ], [ %i.0.prol, %for.body.prol ]
+  %tmp2 = icmp ult i32 %i.08, 3
+  br i1 %tmp2, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:                                 ; preds = %for.body, %for.body.prol.loopexit, %entry
+  ret void
+
+for.body:                                         ; preds = %for.body, %for.body.prol.loopexit
+  %i.010 = phi i32 [ %i.0.3, %for.body ], [ %i.010.unr, %for.body.prol.loopexit ]
+  %arrayidx = getelementptr inbounds i32, i32* %b, i32 %i.010
+  %tmp3 = load i32, i32* %arrayidx, align 4
+  %arrayidx1 = getelementptr inbounds i32, i32* %c, i32 %i.010
+  %tmp4 = load i32, i32* %arrayidx1, align 4
+  %mul = mul nsw i32 %tmp4, %tmp3
+  %arrayidx2 = getelementptr inbounds i32, i32* %a, i32 %i.010
+  store i32 %mul, i32* %arrayidx2, align 4
+  %i.0 = add i32 %i.010, -1
+  %arrayidx.1 = getelementptr inbounds i32, i32* %b, i32 %i.0
+  %tmp5 = load i32, i32* %arrayidx.1, align 4
+  %arrayidx1.1 = getelementptr inbounds i32, i32* %c, i32 %i.0
+  %tmp6 = load i32, i32* %arrayidx1.1, align 4
+  %mul.1 = mul nsw i32 %tmp6, %tmp5
+  %arrayidx2.1 = getelementptr inbounds i32, i32* %a, i32 %i.0
+  store i32 %mul.1, i32* %arrayidx2.1, align 4
+  %i.0.1 = add i32 %i.010, -2
+  %arrayidx.2 = getelementptr inbounds i32, i32* %b, i32 %i.0.1
+  %tmp7 = load i32, i32* %arrayidx.2, align 4
+  %arrayidx1.2 = getelementptr inbounds i32, i32* %c, i32 %i.0.1
+  %tmp8 = load i32, i32* %arrayidx1.2, align 4
+  %mul.2 = mul nsw i32 %tmp8, %tmp7
+  %arrayidx2.2 = getelementptr inbounds i32, i32* %a, i32 %i.0.1
+  store i32 %mul.2, i32* %arrayidx2.2, align 4
+  %i.0.2 = add i32 %i.010, -3
+  %arrayidx.3 = getelementptr inbounds i32, i32* %b, i32 %i.0.2
+  %tmp9 = load i32, i32* %arrayidx.3, align 4
+  %arrayidx1.3 = getelementptr inbounds i32, i32* %c, i32 %i.0.2
+  %tmp10 = load i32, i32* %arrayidx1.3, align 4
+  %mul.3 = mul nsw i32 %tmp10, %tmp9
+  %arrayidx2.3 = getelementptr inbounds i32, i32* %a, i32 %i.0.2
+  store i32 %mul.3, i32* %arrayidx2.3, align 4
+  %i.0.3 = add i32 %i.010, -4
+  %cmp.3 = icmp sgt i32 %i.0.3, -1
+  br i1 %cmp.3, label %for.body, label %for.cond.cleanup
+}
+
+; CHECK-LABEL: mul32x32_forwards
+; CHECK: @ %for.body
+
+; TODO: Would be good for the complexity limit didn't have to be increased to
+; enable the pre-indexed accesses.
+
+; CHECK-DEFAULT-NOT: ldr{{.*}}]!
+; CHECK-DEFAULT-NOT: str{{.*}}]!
+
+; CHECK-COMPLEX: ldr{{.*}}, #16]!
+; CHECK-COMPLEX: ldr{{.*}}, #16]!
+; CHECK-COMPLEX: str{{.*}}, #16]!
+
+; CHECK-T2: @ %for.body.epil
+; CHECK-T2: ldr{{.*}}, #4]!
+; CHECK-T2: ldr{{.*}}, #4]!
+; CHECK-T2: str{{.*}}, #4]!
+
+define void @mul32x32_forwards(i32* nocapture %a, i32* nocapture readonly %b, i32* nocapture readonly %c, i32 %N) {
+entry:
+  %cmp8 = icmp eq i32 %N, 0
+  br i1 %cmp8, label %for.cond.cleanup, label %for.body.preheader
+
+for.body.preheader:                               ; preds = %entry
+  %tmp = add i32 %N, -1
+  %xtraiter = and i32 %N, 3
+  %tmp1 = icmp ult i32 %tmp, 3
+  br i1 %tmp1, label %for.cond.cleanup.loopexit.unr-lcssa, label %for.body.preheader.new
+
+for.body.preheader.new:                           ; preds = %for.body.preheader
+  %unroll_iter = sub i32 %N, %xtraiter
+  br label %for.body
+
+for.cond.cleanup.loopexit.unr-lcssa:              ; preds = %for.body, %for.body.preheader
+  %i.09.unr = phi i32 [ 0, %for.body.preheader ], [ %inc.3, %for.body ]
+  %lcmp.mod = icmp eq i32 %xtraiter, 0
+  br i1 %lcmp.mod, label %for.cond.cleanup, label %for.body.epil
+
+for.body.epil:                                    ; preds = %for.body.epil, %for.cond.cleanup.loopexit.unr-lcssa
+  %i.09.epil = phi i32 [ %inc.epil, %for.body.epil ], [ %i.09.unr, %for.cond.cleanup.loopexit.unr-lcssa ]
+  %epil.iter = phi i32 [ %epil.iter.sub, %for.body.epil ], [ %xtraiter, %for.cond.cleanup.loopexit.unr-lcssa ]
+  %arrayidx.epil = getelementptr inbounds i32, i32* %b, i32 %i.09.epil
+  %tmp2 = load i32, i32* %arrayidx.epil, align 4
+  %arrayidx1.epil = getelementptr inbounds i32, i32* %c, i32 %i.09.epil
+  %tmp3 = load i32, i32* %arrayidx1.epil, align 4
+  %mul.epil = mul nsw i32 %tmp3, %tmp2
+  %arrayidx2.epil = getelementptr inbounds i32, i32* %a, i32 %i.09.epil
+  store i32 %mul.epil, i32* %arrayidx2.epil, align 4
+  %inc.epil = add nuw nsw i32 %i.09.epil, 1
+  %epil.iter.sub = add i32 %epil.iter, -1
+  %epil.iter.cmp = icmp eq i32 %epil.iter.sub, 0
+  br i1 %epil.iter.cmp, label %for.cond.cleanup, label %for.body.epil
+
+for.cond.cleanup:                                 ; preds = %for.body.epil, %for.cond.cleanup.loopexit.unr-lcssa, %entry
+  ret void
+
+for.body:                                         ; preds = %for.body, %for.body.preheader.new
+  %i.09 = phi i32 [ 0, %for.body.preheader.new ], [ %inc.3, %for.body ]
+  %niter = phi i32 [ %unroll_iter, %for.body.preheader.new ], [ %niter.nsub.3, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32* %b, i32 %i.09
+  %tmp4 = load i32, i32* %arrayidx, align 4
+  %arrayidx1 = getelementptr inbounds i32, i32* %c, i32 %i.09
+  %tmp5 = load i32, i32* %arrayidx1, align 4
+  %mul = mul nsw i32 %tmp5, %tmp4
+  %arrayidx2 = getelementptr inbounds i32, i32* %a, i32 %i.09
+  store i32 %mul, i32* %arrayidx2, align 4
+  %inc = or i32 %i.09, 1
+  %arrayidx.1 = getelementptr inbounds i32, i32* %b, i32 %inc
+  %tmp6 = load i32, i32* %arrayidx.1, align 4
+  %arrayidx1.1 = getelementptr inbounds i32, i32* %c, i32 %inc
+  %tmp7 = load i32, i32* %arrayidx1.1, align 4
+  %mul.1 = mul nsw i32 %tmp7, %tmp6
+  %arrayidx2.1 = getelementptr inbounds i32, i32* %a, i32 %inc
+  store i32 %mul.1, i32* %arrayidx2.1, align 4
+  %inc.1 = or i32 %i.09, 2
+  %arrayidx.2 = getelementptr inbounds i32, i32* %b, i32 %inc.1
+  %tmp8 = load i32, i32* %arrayidx.2, align 4
+  %arrayidx1.2 = getelementptr inbounds i32, i32* %c, i32 %inc.1
+  %tmp9 = load i32, i32* %arrayidx1.2, align 4
+  %mul.2 = mul nsw i32 %tmp9, %tmp8
+  %arrayidx2.2 = getelementptr inbounds i32, i32* %a, i32 %inc.1
+  store i32 %mul.2, i32* %arrayidx2.2, align 4
+  %inc.2 = or i32 %i.09, 3
+  %arrayidx.3 = getelementptr inbounds i32, i32* %b, i32 %inc.2
+  %tmp10 = load i32, i32* %arrayidx.3, align 4
+  %arrayidx1.3 = getelementptr inbounds i32, i32* %c, i32 %inc.2
+  %tmp11 = load i32, i32* %arrayidx1.3, align 4
+  %mul.3 = mul nsw i32 %tmp11, %tmp10
+  %arrayidx2.3 = getelementptr inbounds i32, i32* %a, i32 %inc.2
+  store i32 %mul.3, i32* %arrayidx2.3, align 4
+  %inc.3 = add nuw nsw i32 %i.09, 4
+  %niter.nsub.3 = add i32 %niter, -4
+  %niter.ncmp.3 = icmp eq i32 %niter.nsub.3, 0
+  br i1 %niter.ncmp.3, label %for.cond.cleanup.loopexit.unr-lcssa, label %for.body
+}
diff --git a/test/CodeGen/ARM/lowerMUL-newload.ll b/test/CodeGen/ARM/lowerMUL-newload.ll
index 93d765cba1168cef45bb940647dafbfdb09ab846..1d483c96f7ec0d61820ef9fe61ec9425b07e884c 100644
--- a/test/CodeGen/ARM/lowerMUL-newload.ll
+++ b/test/CodeGen/ARM/lowerMUL-newload.ll
@@ -1,25 +1,41 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=arm-eabi -mcpu=krait | FileCheck %s
 
 define void @func1(i16* %a, i16* %b, i16* %c) {
+; CHECK-LABEL: func1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    add r3, r1, #16
+; CHECK-NEXT:    vldr d18, [r2, #16]
+; CHECK-NEXT:    vld1.16 {d16}, [r3:64]
+; CHECK-NEXT:    vmovl.u16 q8, d16
+; CHECK-NEXT:    vaddw.s16 q10, q8, d18
+; CHECK-NEXT:    vmovn.i32 d19, q10
+; CHECK-NEXT:    vldr d20, [r0, #16]
+; CHECK-NEXT:    vstr d19, [r0, #16]
+; CHECK-NEXT:    vldr d19, [r2, #16]
+; CHECK-NEXT:    vmull.s16 q11, d18, d19
+; CHECK-NEXT:    vmovl.s16 q9, d19
+; CHECK-NEXT:    vmla.i32 q11, q8, q9
+; CHECK-NEXT:    vmovn.i32 d16, q11
+; CHECK-NEXT:    vstr d16, [r1, #16]
+; CHECK-NEXT:    vldr d16, [r2, #16]
+; CHECK-NEXT:    vmlal.s16 q11, d16, d20
+; CHECK-NEXT:    vmovn.i32 d16, q11
+; CHECK-NEXT:    vstr d16, [r0, #16]
+; CHECK-NEXT:    bx lr
 entry:
 ; The test case trying to vectorize the pseudo code below.
 ; a[i] = b[i] + c[i];
 ; b[i] = a[i] * c[i];
 ; a[i] = b[i] + a[i] * c[i];
-;
 ; Checking that vector load a[i] for "a[i] = b[i] + a[i] * c[i]" is
 ; scheduled before the first vector store to "a[i] = b[i] + c[i]".
 ; Checking that there is no vector load a[i] scheduled between the vector
 ; stores to a[i], otherwise the load of a[i] will be polluted by the first
 ; vector store to a[i].
-;
 ; This test case check that the chain information is updated during
 ; lowerMUL for the new created Load SDNode.
 
-; CHECK: vldr {{.*}} [r0, #16]
-; CHECK: vstr {{.*}} [r0, #16]
-; CHECK-NOT: vldr {{.*}} [r0, #16]
-; CHECK: vstr {{.*}} [r0, #16]
 
   %scevgep0 = getelementptr i16, i16* %a, i32 8
   %vector_ptr0 = bitcast i16* %scevgep0 to <4 x i16>*
@@ -57,26 +73,41 @@ entry:
 }
 
 define void @func2(i16* %a, i16* %b, i16* %c) {
+; CHECK-LABEL: func2:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    add r3, r1, #16
+; CHECK-NEXT:    vldr d18, [r2, #16]
+; CHECK-NEXT:    vld1.16 {d16}, [r3:64]
+; CHECK-NEXT:    vmovl.u16 q8, d16
+; CHECK-NEXT:    vaddw.s16 q10, q8, d18
+; CHECK-NEXT:    vmovn.i32 d19, q10
+; CHECK-NEXT:    vldr d20, [r0, #16]
+; CHECK-NEXT:    vstr d19, [r0, #16]
+; CHECK-NEXT:    vldr d19, [r2, #16]
+; CHECK-NEXT:    vmull.s16 q11, d18, d19
+; CHECK-NEXT:    vmovl.s16 q9, d19
+; CHECK-NEXT:    vmla.i32 q11, q8, q9
+; CHECK-NEXT:    vmovn.i32 d16, q11
+; CHECK-NEXT:    vstr d16, [r1, #16]
+; CHECK-NEXT:    vldr d16, [r2, #16]
+; CHECK-NEXT:    vmlal.s16 q11, d16, d20
+; CHECK-NEXT:    vaddw.s16 q8, q11, d20
+; CHECK-NEXT:    vmovn.i32 d16, q8
+; CHECK-NEXT:    vstr d16, [r0, #16]
+; CHECK-NEXT:    bx lr
 entry:
 ; The test case trying to vectorize the pseudo code below.
 ; a[i] = b[i] + c[i];
 ; b[i] = a[i] * c[i];
 ; a[i] = b[i] + a[i] * c[i] + a[i];
-;
 ; Checking that vector load a[i] for "a[i] = b[i] + a[i] * c[i] + a[i]"
 ; is scheduled before the first vector store to "a[i] = b[i] + c[i]".
 ; Checking that there is no vector load a[i] scheduled between the first
 ; vector store to a[i] and the vector add of a[i], otherwise the load of
 ; a[i] will be polluted by the first vector store to a[i].
-;
 ; This test case check that both the chain and value of the new created
 ; Load SDNode are updated during lowerMUL.
 
-; CHECK: vldr {{.*}} [r0, #16]
-; CHECK: vstr {{.*}} [r0, #16]
-; CHECK-NOT: vldr {{.*}} [r0, #16]
-; CHECK: vaddw.s16
-; CHECK: vstr {{.*}} [r0, #16]
 
   %scevgep0 = getelementptr i16, i16* %a, i32 8
   %vector_ptr0 = bitcast i16* %scevgep0 to <4 x i16>*
diff --git a/test/CodeGen/ARM/machine-licm.ll b/test/CodeGen/ARM/machine-licm.ll
index 9ed1a57616c9bf395941fa8edfada14684df2b29..1cf291be6621d75df1e2031f022e14cc7f4e512e 100644
--- a/test/CodeGen/ARM/machine-licm.ll
+++ b/test/CodeGen/ARM/machine-licm.ll
@@ -1,6 +1,6 @@
-; RUN: llc < %s -mtriple=thumb-apple-darwin -relocation-model=pic -disable-fp-elim | FileCheck %s -check-prefix=THUMB
-; RUN: llc < %s -mtriple=arm-apple-darwin -relocation-model=pic -disable-fp-elim   | FileCheck %s -check-prefix=ARM
-; RUN: llc < %s -mtriple=arm-apple-darwin -relocation-model=pic -disable-fp-elim -mattr=+v6t2 | FileCheck %s -check-prefix=MOVT
+; RUN: llc < %s -mtriple=thumb-apple-darwin -relocation-model=pic -frame-pointer=all | FileCheck %s -check-prefix=THUMB
+; RUN: llc < %s -mtriple=arm-apple-darwin -relocation-model=pic -frame-pointer=all   | FileCheck %s -check-prefix=ARM
+; RUN: llc < %s -mtriple=arm-apple-darwin -relocation-model=pic -frame-pointer=all -mattr=+v6t2 | FileCheck %s -check-prefix=MOVT
 ; rdar://7353541
 ; rdar://7354376
 ; rdar://8887598
diff --git a/test/CodeGen/ARM/macho-frame-offset.ll b/test/CodeGen/ARM/macho-frame-offset.ll
index f3dacf66b6c39279cc35e2d5648fc5943709a0c4..b61a7d706f0588d40f1a9bbe217657e1b359607d 100644
--- a/test/CodeGen/ARM/macho-frame-offset.ll
+++ b/test/CodeGen/ARM/macho-frame-offset.ll
@@ -1,4 +1,4 @@
-; RUN: llc -mtriple thumbv7m-apple-macho -disable-fp-elim -o - %s | FileCheck %s
+; RUN: llc -mtriple thumbv7m-apple-macho -frame-pointer=all -o - %s | FileCheck %s
 
 define void @func() {
 ; CHECK-LABEL: func:
diff --git a/test/CodeGen/ARM/none-macho.ll b/test/CodeGen/ARM/none-macho.ll
index fee459f4f5e1b9b2f5d975df24c620c3012001fa..057da94961452f40a41b9038b1c1c24c453dee5d 100644
--- a/test/CodeGen/ARM/none-macho.ll
+++ b/test/CodeGen/ARM/none-macho.ll
@@ -1,5 +1,5 @@
-; RUN: llc -mtriple=thumbv7m-none-macho %s -o - -relocation-model=pic -disable-fp-elim | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-NON-FAST
-; RUN: llc -mtriple=thumbv7m-none-macho -O0 %s -o - -relocation-model=pic -disable-fp-elim | FileCheck %s
+; RUN: llc -mtriple=thumbv7m-none-macho %s -o - -relocation-model=pic -frame-pointer=all | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-NON-FAST
+; RUN: llc -mtriple=thumbv7m-none-macho -O0 %s -o - -relocation-model=pic -frame-pointer=all | FileCheck %s
 ; RUN: llc -mtriple=thumbv7m-none-macho -filetype=obj %s -o /dev/null
 
 @var = external global i32
diff --git a/test/CodeGen/ARM/pow.75.ll b/test/CodeGen/ARM/pow.75.ll
new file mode 100644
index 0000000000000000000000000000000000000000..32719f164fb1a2a3d3f3a508adcd34cfcabf432e
--- /dev/null
+++ b/test/CodeGen/ARM/pow.75.ll
@@ -0,0 +1,70 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=thumbv8-linux-gnueabihf -mattr=neon -debug 2>&1 | FileCheck %s
+; REQUIRES: asserts
+
+declare float @llvm.pow.f32(float, float)
+declare <4 x float> @llvm.pow.v4f32(<4 x float>, <4 x float>)
+declare double @llvm.pow.f64(double, double)
+declare <2 x double> @llvm.pow.v2f64(<2 x double>, <2 x double>)
+
+define float @pow_f32_three_fourth_fmf(float %x) nounwind {
+; CHECK: Combining: {{.*}}: f32 = fpow ninf nsz afn [[X:t[0-9]+]], ConstantFP:f32<7.500000e-01>
+; CHECK-NEXT: Creating new node: [[SQRT:t[0-9]+]]: f32 = fsqrt ninf nsz afn [[X]]
+; CHECK-NEXT: Creating new node: [[SQRTSQRT:t[0-9]+]]: f32 = fsqrt ninf nsz afn [[SQRT]]
+; CHECK-NEXT: Creating new node: [[R:t[0-9]+]]: f32 = fmul ninf nsz afn [[SQRT]], [[SQRTSQRT]]
+; CHECK-NEXT:  ... into: [[R]]: f32 = fmul ninf nsz afn [[SQRT]], [[SQRTSQRT]]
+  %r = call nsz ninf afn float @llvm.pow.f32(float %x, float 7.5e-01)
+  ret float %r
+}
+
+define double @pow_f64_three_fourth_fmf(double %x) nounwind {
+; CHECK: Combining: {{.*}}: f64 = fpow ninf nsz afn t2, ConstantFP:f64<7.500000e-01>
+; CHECK-NEXT: Creating new node: [[SQRT:t[0-9]+]]: f64 = fsqrt ninf nsz afn [[X]]
+; CHECK-NEXT: Creating new node: [[SQRTSQRT:t[0-9]+]]: f64 = fsqrt ninf nsz afn [[SQRT]]
+; CHECK-NEXT: Creating new node: [[R:t[0-9]+]]: f64 = fmul ninf nsz afn [[SQRT]], [[SQRTSQRT]]
+; CHECK-NEXT:  ... into: [[R]]: f64 = fmul ninf nsz afn [[SQRT]], [[SQRTSQRT]]
+  %r = call nsz ninf afn double @llvm.pow.f64(double %x, double 7.5e-01)
+  ret double %r
+}
+
+define <4 x float> @pow_v4f32_three_fourth_fmf(<4 x float> %x) nounwind {
+; CHECK: Combining: {{.*}}: v4f32 = BUILD_VECTOR [[FORTH:t[0-9]+]], [[THIRD:t[0-9]+]], [[SECOND:t[0-9]+]], [[FIRST:t[0-9]+]]
+; CHECK: Combining: [[FIRST]]: f32 = fpow nnan ninf nsz arcp contract afn reassoc [[X:t[0-9]+]], ConstantFP:f32<7.500000e-01>
+; CHECK-NEXT: Creating new node: [[SQRT:t[0-9]+]]: f32 = fsqrt nnan ninf nsz arcp contract afn reassoc [[X]]
+; CHECK-NEXT: Creating new node: [[SQRTSQRT:t[0-9]+]]: f32 = fsqrt nnan ninf nsz arcp contract afn reassoc [[SQRT]]
+; CHECK-NEXT: Creating new node: [[R:t[0-9]+]]: f32 = fmul nnan ninf nsz arcp contract afn reassoc [[SQRT]], [[SQRTSQRT]]
+; CHECK-NEXT:  ... into: [[R]]: f32 = fmul nnan ninf nsz arcp contract afn reassoc [[SQRT]], [[SQRTSQRT]]
+; CHECK: Combining: [[SECOND]]: f32 = fpow nnan ninf nsz arcp contract afn reassoc [[X:t[0-9]+]], ConstantFP:f32<7.500000e-01>
+; CHECK-NEXT: Creating new node: [[SQRT:t[0-9]+]]: f32 = fsqrt nnan ninf nsz arcp contract afn reassoc [[X]]
+; CHECK-NEXT: Creating new node: [[SQRTSQRT:t[0-9]+]]: f32 = fsqrt nnan ninf nsz arcp contract afn reassoc [[SQRT]]
+; CHECK-NEXT: Creating new node: [[R:t[0-9]+]]: f32 = fmul nnan ninf nsz arcp contract afn reassoc [[SQRT]], [[SQRTSQRT]]
+; CHECK-NEXT:  ... into: [[R]]: f32 = fmul nnan ninf nsz arcp contract afn reassoc [[SQRT]], [[SQRTSQRT]]
+; CHECK: Combining: [[THIRD]]: f32 = fpow nnan ninf nsz arcp contract afn reassoc [[X:t[0-9]+]], ConstantFP:f32<7.500000e-01>
+; CHECK-NEXT: Creating new node: [[SQRT:t[0-9]+]]: f32 = fsqrt nnan ninf nsz arcp contract afn reassoc [[X]]
+; CHECK-NEXT: Creating new node: [[SQRTSQRT:t[0-9]+]]: f32 = fsqrt nnan ninf nsz arcp contract afn reassoc [[SQRT]]
+; CHECK-NEXT: Creating new node: [[R:t[0-9]+]]: f32 = fmul nnan ninf nsz arcp contract afn reassoc [[SQRT]], [[SQRTSQRT]]
+; CHECK-NEXT:  ... into: [[R]]: f32 = fmul nnan ninf nsz arcp contract afn reassoc [[SQRT]], [[SQRTSQRT]]
+; CHECK: Combining: [[FORTH]]: f32 = fpow nnan ninf nsz arcp contract afn reassoc [[X:t[0-9]+]], ConstantFP:f32<7.500000e-01>
+; CHECK-NEXT: Creating new node: [[SQRT:t[0-9]+]]: f32 = fsqrt nnan ninf nsz arcp contract afn reassoc [[X]]
+; CHECK-NEXT: Creating new node: [[SQRTSQRT:t[0-9]+]]: f32 = fsqrt nnan ninf nsz arcp contract afn reassoc [[SQRT]]
+; CHECK-NEXT: Creating new node: [[R:t[0-9]+]]: f32 = fmul nnan ninf nsz arcp contract afn reassoc [[SQRT]], [[SQRTSQRT]]
+; CHECK-NEXT:  ... into: [[R]]: f32 = fmul nnan ninf nsz arcp contract afn reassoc [[SQRT]], [[SQRTSQRT]]
+  %r = call fast <4 x float> @llvm.pow.v4f32(<4 x float> %x, <4 x float> <float 7.5e-1, float 7.5e-1, float 7.5e-01, float 7.5e-01>)
+  ret <4 x float> %r
+}
+
+define <2 x double> @pow_v2f64_three_fourth_fmf(<2 x double> %x) nounwind {
+; CHECK: Combining: {{.*}}: v2f64 = BUILD_VECTOR [[SECOND:t[0-9]+]], [[FIRST:t[0-9]+]]
+; CHECK: Combining: [[FIRST]]: f64 = fpow nnan ninf nsz arcp contract afn reassoc [[X:t[0-9]+]], ConstantFP:f64<7.500000e-01>
+; CHECK-NEXT: Creating new node: [[SQRT:t[0-9]+]]: f64 = fsqrt nnan ninf nsz arcp contract afn reassoc [[X]]
+; CHECK-NEXT: Creating new node: [[SQRTSQRT:t[0-9]+]]: f64 = fsqrt nnan ninf nsz arcp contract afn reassoc [[SQRT]]
+; CHECK-NEXT: Creating new node: [[R:t[0-9]+]]: f64 = fmul nnan ninf nsz arcp contract afn reassoc [[SQRT]], [[SQRTSQRT]]
+; CHECK-NEXT: ... into: [[R]]: f64 = fmul nnan ninf nsz arcp contract afn reassoc t16, t17
+; CHECK: Combining: [[SECOND]]: f64 = fpow nnan ninf nsz arcp contract afn reassoc [[X:t[0-9]+]], ConstantFP:f64<7.500000e-01>
+; CHECK-NEXT: Creating new node: [[SQRT:t[0-9]+]]: f64 = fsqrt nnan ninf nsz arcp contract afn reassoc [[X]]
+; CHECK-NEXT: Creating new node: [[SQRTSQRT:t[0-9]+]]: f64 = fsqrt nnan ninf nsz arcp contract afn reassoc [[SQRT]]
+; CHECK-NEXT: Creating new node: [[R:t[0-9]+]]: f64 = fmul nnan ninf nsz arcp contract afn reassoc [[SQRT]], [[SQRTSQRT]]
+; CHECK-NEXT:  ... into: [[R]]: f64 = fmul nnan ninf nsz arcp contract afn reassoc t19, t20
+  %r = call fast <2 x double> @llvm.pow.v2f64(<2 x double> %x, <2 x double> <double 7.5e-1, double 7.5e-1>)
+  ret <2 x double> %r
+}
diff --git a/test/CodeGen/ARM/prera-ldst-insertpt.mir b/test/CodeGen/ARM/prera-ldst-insertpt.mir
index f05bae45f4002daee2301f86be336a31e1ae2e18..c5c463446adade96affd3cbefdbb49282c878f69 100644
--- a/test/CodeGen/ARM/prera-ldst-insertpt.mir
+++ b/test/CodeGen/ARM/prera-ldst-insertpt.mir
@@ -1,4 +1,6 @@
 # RUN: llc -run-pass arm-prera-ldst-opt %s -o - | FileCheck %s
+# RUN: llc -run-pass arm-prera-ldst-opt -arm-prera-ldst-opt-reorder-limit=3 %s -o - | FileCheck %s
+# RUN: llc -run-pass arm-prera-ldst-opt -arm-prera-ldst-opt-reorder-limit=2 %s -o - | FileCheck %s --check-prefix=CHECK-LIMIT
 --- |
   target triple = "thumbv7---eabi"
 
@@ -79,12 +81,18 @@ body:             |
 
     ; Make sure we move the paired stores next to each other, and
     ; insert them in an appropriate location.
-    ; CHECK: t2STRi12 {{.*}}, 0
+    ; CHECK:      t2STRi12 {{.*}}, 0
     ; CHECK-NEXT: t2STRi12 {{.*}}, 4
     ; CHECK-NEXT: t2STRi12 {{.*}}, 8
     ; CHECK-NEXT: t2MUL
     ; CHECK-NEXT: t2MOVi32imm
 
+    ; CHECK-LIMIT-LABEL: name: b
+    ; CHECK-LIMIT:      t2STRi12 {{.*}}, 0
+    ; CHECK-LIMIT-NEXT: t2STRi12 {{.*}}, 4
+    ; CHECK-LIMIT-NEXT: t2MUL
+    ; CHECK-LIMIT-NEXT: t2STRi12 {{.*}}, 8
+
     %4 : rgpr = t2MUL %1, %1, 14, $noreg
     %5 : rgpr = t2MOVi32imm -858993459
     %6 : rgpr, %7 : rgpr  = t2UMULL killed %3, %5, 14, $noreg
diff --git a/test/CodeGen/ARM/register-scavenger-exceptions.mir b/test/CodeGen/ARM/register-scavenger-exceptions.mir
new file mode 100644
index 0000000000000000000000000000000000000000..58f6cd77557c0d50666ebc55ce82910fc80b3e2e
--- /dev/null
+++ b/test/CodeGen/ARM/register-scavenger-exceptions.mir
@@ -0,0 +1,47 @@
+# RUN: llc -o - %s -run-pass=prologepilog | FileCheck %s
+
+--- |
+  target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64"
+  target triple = "armv7--none-eabi"
+
+  define hidden void @_Z3foov(i32 %P0, ...) {
+  entry:
+    %V1 = alloca [4075 x i8], align 8
+    %tmp3 = alloca i8, i32 undef, align 8
+    unreachable
+  }
+
+  declare dso_local void @_Z3barv() noreturn
+
+...
+---
+# Check that the register scavenger does pick r5 (not preserved in prolog) for
+# materialising a stack frame address when the function ends in throwing an
+# exception.
+# CHECK: $sp = frame-setup STMDB_UPD $sp, 14, $noreg, killed $r4, killed $r10, killed $r11, killed $lr
+# CHECK-NOT: $r5
+name:            _Z3foov
+stack:
+  - { id: 0, name: V1, type: default, offset: 0, size: 5000, alignment: 8,
+      stack-id: 0, callee-saved-register: '', callee-saved-restored: true,
+      local-offset: -4080, debug-info-variable: '', debug-info-expression: '',
+      debug-info-location: '' }
+  - { id: 1, name: tmp3, type: variable-sized, offset: 0, alignment: 1,
+      stack-id: 0, callee-saved-register: '', callee-saved-restored: true,
+      local-offset: -4112, debug-info-variable: '', debug-info-expression: '',
+      debug-info-location: '' }
+  - { id: 2, name: '', type: spill-slot, offset: 0, size: 4, alignment: 4,
+      stack-id: 0, callee-saved-register: '', callee-saved-restored: true,
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+body:             |
+  bb.0.entry:
+
+    $r0 = MOVi 0, 14, $noreg, $noreg
+    $r1 = MOVi 0, 14, $noreg, $noreg
+    $r2 = MOVi 0, 14, $noreg, $noreg
+    $r3 = MOVi 0, 14, $noreg, $noreg
+    $r4 = MOVi 0, 14, $noreg, $noreg
+    STRi12 killed $lr, %stack.2, 0, 14, $noreg :: (store 4 into %stack.2)
+    BL @_Z3barv, csr_aapcs, implicit-def dead $lr, implicit $sp, implicit killed $r0, implicit killed $r1, implicit killed $r2, implicit killed $r3, implicit killed $r4, implicit-def $sp
+
+...
diff --git a/test/CodeGen/ARM/shift_minsize.ll b/test/CodeGen/ARM/shift_minsize.ll
new file mode 100644
index 0000000000000000000000000000000000000000..4d10c64392dab6dcf636569b0379eab409d49fe6
--- /dev/null
+++ b/test/CodeGen/ARM/shift_minsize.ll
@@ -0,0 +1,32 @@
+; RUN: llc -mtriple=arm-eabi %s -o - | FileCheck %s
+
+define i64 @f0(i64 %val, i64 %amt) minsize optsize {
+; CHECK-LABEL:   f0:
+; CHECK:         bl  __aeabi_llsl
+  %res = shl i64 %val, %amt
+  ret i64 %res
+}
+
+define i32 @f1(i64 %x, i64 %y) minsize optsize {
+; CHECK-LABEL:   f1:
+; CHECK:         bl  __aeabi_llsl
+	%a = shl i64 %x, %y
+	%b = trunc i64 %a to i32
+	ret i32 %b
+}
+
+define i32 @f2(i64 %x, i64 %y) minsize optsize {
+; CHECK-LABEL:   f2:
+; CHECK:         bl  __aeabi_lasr
+	%a = ashr i64 %x, %y
+	%b = trunc i64 %a to i32
+	ret i32 %b
+}
+
+define i32 @f3(i64 %x, i64 %y) minsize optsize {
+; CHECK-LABEL:   f3:
+; CHECK:         bl  __aeabi_llsr
+	%a = lshr i64 %x, %y
+	%b = trunc i64 %a to i32
+	ret i32 %b
+}
diff --git a/test/CodeGen/ARM/shuffle.ll b/test/CodeGen/ARM/shuffle.ll
index 7d6be4f5e6c332138703643df46787b1c62fe750..17ec7c51daf44caa869d545183bd0e2ecf42ca55 100644
--- a/test/CodeGen/ARM/shuffle.ll
+++ b/test/CodeGen/ARM/shuffle.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -mtriple=thumbv7-apple-darwin -relocation-model=pic -disable-fp-elim | FileCheck %s
+; RUN: llc < %s -mtriple=thumbv7-apple-darwin -relocation-model=pic -frame-pointer=all | FileCheck %s
 
 target datalayout = "e-p:32:32:32-i1:8:32-i8:8:32-i16:16:32-i32:32:32-i64:32:32-f32:32:32-f64:32:32-v64:64:64-v128:128:128-a0:0:32-n32"
 target triple = "thumbv7-apple-darwin"
diff --git a/test/CodeGen/ARM/smul.ll b/test/CodeGen/ARM/smul.ll
index 2b7be41ddb24e6df0ce7ebd3e40258fc456f731d..7091f8d191483e5ca546c153ef63b01c57ebae7f 100644
--- a/test/CodeGen/ARM/smul.ll
+++ b/test/CodeGen/ARM/smul.ll
@@ -1,14 +1,14 @@
-; RUN: llc -mtriple=arm-eabi -mcpu=generic %s -o /dev/null
+; RUN: llc -mtriple=arm-eabi -mcpu=generic %s -o - | FileCheck %s --check-prefix=DISABLED
 ; RUN: llc -mtriple=arm-eabi -mcpu=cortex-a8 %s -o - | FileCheck %s
 ; RUN: llc -mtriple=thumb--none-eabi -mcpu=cortex-a8 %s -o - | FileCheck %s
 ; RUN: llc -mtriple=thumbv6t2-none-eabi %s -o - | FileCheck %s
-; RUN: llc -mtriple=thumbv6-none-eabi %s -o - | FileCheck %s -check-prefix=CHECK-THUMBV6
+; RUN: llc -mtriple=thumbv6-none-eabi %s -o - | FileCheck %s -check-prefix=DISABLED
 
 define i32 @f1(i16 %x, i32 %y) {
 ; CHECK-LABEL: f1:
 ; CHECK-NOT: sxth
 ; CHECK: {{smulbt r0, r0, r1|smultb r0, r1, r0}}
-; CHECK-THUMBV6-NOT: {{smulbt|smultb}}
+; DISABLED-NOT: {{smulbt|smultb}}
         %tmp1 = sext i16 %x to i32
         %tmp2 = ashr i32 %y, 16
         %tmp3 = mul i32 %tmp2, %tmp1
@@ -18,7 +18,7 @@ define i32 @f1(i16 %x, i32 %y) {
 define i32 @f2(i32 %x, i32 %y) {
 ; CHECK-LABEL: f2:
 ; CHECK: smultt
-; CHECK-THUMBV6-NOT: smultt
+; DISABLED-NOT: smultt
         %tmp1 = ashr i32 %x, 16
         %tmp3 = ashr i32 %y, 16
         %tmp4 = mul i32 %tmp3, %tmp1
@@ -29,7 +29,7 @@ define i32 @f3(i32 %a, i16 %x, i32 %y) {
 ; CHECK-LABEL: f3:
 ; CHECK-NOT: sxth
 ; CHECK: {{smlabt r0, r1, r2, r0|smlatb r0, r2, r1, r0}}
-; CHECK-THUMBV6-NOT: {{smlabt|smlatb}}
+; DISABLED-NOT: {{smlabt|smlatb}}
         %tmp = sext i16 %x to i32
         %tmp2 = ashr i32 %y, 16
         %tmp3 = mul i32 %tmp2, %tmp
@@ -40,7 +40,7 @@ define i32 @f3(i32 %a, i16 %x, i32 %y) {
 define i32 @f4(i32 %a, i32 %x, i32 %y) {
 ; CHECK-LABEL: f4:
 ; CHECK: smlatt
-; CHECK-THUMBV6-NOT: smlatt
+; DISABLED-NOT: smlatt
         %tmp1 = ashr i32 %x, 16
         %tmp3 = ashr i32 %y, 16
         %tmp4 = mul i32 %tmp3, %tmp1
@@ -52,7 +52,7 @@ define i32 @f5(i32 %a, i16 %x, i16 %y) {
 ; CHECK-LABEL: f5:
 ; CHECK-NOT: sxth
 ; CHECK: smlabb
-; CHECK-THUMBV6-NOT: smlabb
+; DISABLED-NOT: smlabb
         %tmp1 = sext i16 %x to i32
         %tmp3 = sext i16 %y to i32
         %tmp4 = mul i32 %tmp3, %tmp1
@@ -64,7 +64,7 @@ define i32 @f6(i32 %a, i32 %x, i16 %y) {
 ; CHECK-LABEL: f6:
 ; CHECK-NOT: sxth
 ; CHECK: {{smlatb r0, r1, r2, r0|smlabt r0, r2, r1, r0}}
-; CHECK-THUMBV6-NOT: {{smlatb|smlabt}}
+; DISABLED-NOT: {{smlatb|smlabt}}
         %tmp1 = sext i16 %y to i32
         %tmp2 = ashr i32 %x, 16
         %tmp3 = mul i32 %tmp2, %tmp1
@@ -75,7 +75,7 @@ define i32 @f6(i32 %a, i32 %x, i16 %y) {
 define i32 @f7(i32 %a, i32 %b, i32 %c) {
 ; CHECK-LABEL: f7:
 ; CHECK: smlawb r0, r0, r1, r2
-; CHECK-THUMBV6-NOT: smlawb
+; DISABLED-NOT: smlawb
         %shl = shl i32 %b, 16
         %shr = ashr exact i32 %shl, 16
         %conv = sext i32 %a to i64
@@ -91,7 +91,7 @@ define i32 @f8(i32 %a, i16 signext %b, i32 %c) {
 ; CHECK-LABEL: f8:
 ; CHECK-NOT: sxth
 ; CHECK: smlawb r0, r0, r1, r2
-; CHECK-THUMBV6-NOT: smlawb
+; DISABLED-NOT: smlawb
         %conv = sext i32 %a to i64
         %conv1 = sext i16 %b to i64
         %mul = mul nsw i64 %conv1, %conv
@@ -104,7 +104,7 @@ define i32 @f8(i32 %a, i16 signext %b, i32 %c) {
 define i32 @f9(i32 %a, i32 %b, i32 %c) {
 ; CHECK-LABEL: f9:
 ; CHECK: smlawt r0, r0, r1, r2
-; CHECK-THUMBV6-NOT: smlawt
+; DISABLED-NOT: smlawt
         %conv = sext i32 %a to i64
         %shr = ashr i32 %b, 16
         %conv1 = sext i32 %shr to i64
@@ -118,7 +118,7 @@ define i32 @f9(i32 %a, i32 %b, i32 %c) {
 define i32 @f10(i32 %a, i32 %b) {
 ; CHECK-LABEL: f10:
 ; CHECK: smulwb r0, r0, r1
-; CHECK-THUMBV6-NOT: smulwb
+; DISABLED-NOT: smulwb
         %shl = shl i32 %b, 16
         %shr = ashr exact i32 %shl, 16
         %conv = sext i32 %a to i64
@@ -133,7 +133,7 @@ define i32 @f11(i32 %a, i16 signext %b) {
 ; CHECK-LABEL: f11:
 ; CHECK-NOT: sxth
 ; CHECK: smulwb r0, r0, r1
-; CHECK-THUMBV6-NOT: smulwb
+; DISABLED-NOT: smulwb
         %conv = sext i32 %a to i64
         %conv1 = sext i16 %b to i64
         %mul = mul nsw i64 %conv1, %conv
@@ -145,7 +145,7 @@ define i32 @f11(i32 %a, i16 signext %b) {
 define i32 @f12(i32 %a, i32 %b) {
 ; CHECK-LABEL: f12:
 ; CHECK: smulwt r0, r0, r1
-; CHECK-THUMBV6-NOT: smulwt
+; DISABLED-NOT: smulwt
         %conv = sext i32 %a to i64
         %shr = ashr i32 %b, 16
         %conv1 = sext i32 %shr to i64
@@ -159,7 +159,7 @@ define i32 @f13(i32 %x, i16 %y) {
 ; CHECK-LABEL: f13:
 ; CHECK-NOT: sxth
 ; CHECK: {{smultb r0, r0, r1|smulbt r0, r1, r0}}
-; CHECK-THUMBV6-NOT: {{smultb|smulbt}}
+; DISABLED-NOT: {{smultb|smulbt}}
         %tmp1 = sext i16 %y to i32
         %tmp2 = ashr i32 %x, 16
         %tmp3 = mul i32 %tmp2, %tmp1
@@ -169,11 +169,11 @@ define i32 @f13(i32 %x, i16 %y) {
 define i32 @f14(i32 %x, i32 %y) {
 ; CHECK-LABEL: f14:
 ; CHECK-NOT: sxth
-; CHECK: {{smultb r0, r0, r1|smulbt r0, r1, r0}}
-; CHECK-THUMBV6-NOT: {{smultb|smulbt}}
-        %tmp1 = shl i32 %y, 16
+; CHECK: {{smultb r0, r1, r0|smulbt r0, r0, r1}}
+; DISABLED-NOT: {{smultb|smulbt}}
+        %tmp1 = shl i32 %x, 16
         %tmp2 = ashr i32 %tmp1, 16
-        %tmp3 = ashr i32 %x, 16
+        %tmp3 = ashr i32 %y, 16
         %tmp4 = mul i32 %tmp3, %tmp2
         ret i32 %tmp4
 }
@@ -182,7 +182,7 @@ define i32 @f15(i32 %x, i32 %y) {
 ; CHECK-LABEL: f15:
 ; CHECK-NOT: sxth
 ; CHECK: {{smulbt r0, r0, r1|smultb r0, r1, r0}}
-; CHECK-THUMBV6-NOT: {{smulbt|smultb}}
+; DISABLED-NOT: {{smulbt|smultb}}
         %tmp1 = shl i32 %x, 16
         %tmp2 = ashr i32 %tmp1, 16
         %tmp3 = ashr i32 %y, 16
@@ -194,7 +194,7 @@ define i32 @f16(i16 %x, i16 %y) {
 ; CHECK-LABEL: f16:
 ; CHECK-NOT: sxth
 ; CHECK: smulbb
-; CHECK-THUMBV6-NOT: smulbb
+; DISABLED-NOT: smulbb
         %tmp1 = sext i16 %x to i32
         %tmp2 = sext i16 %x to i32
         %tmp3 = mul i32 %tmp1, %tmp2
@@ -203,8 +203,9 @@ define i32 @f16(i16 %x, i16 %y) {
 
 define i32 @f17(i32 %x, i32 %y) {
 ; CHECK-LABEL: f17:
+; CHECK-NOT: sxth
 ; CHECK: smulbb
-; CHECK-THUMBV6-NOT: smulbb
+; DISABLED-NOT: smulbb
         %tmp1 = shl i32 %x, 16
         %tmp2 = shl i32 %y, 16
         %tmp3 = ashr i32 %tmp1, 16
@@ -215,8 +216,9 @@ define i32 @f17(i32 %x, i32 %y) {
 
 define i32 @f18(i32 %a, i32 %x, i32 %y) {
 ; CHECK-LABEL: f18:
+; CHECK-NOT: sxth
 ; CHECK: {{smlabt r0, r1, r2, r0|smlatb r0, r2, r1, r0}}
-; CHECK-THUMBV6-NOT: {{smlabt|smlatb}}
+; DISABLED-NOT: {{smlabt|smlatb}}
         %tmp0 = shl i32 %x, 16
         %tmp1 = ashr i32 %tmp0, 16
         %tmp2 = ashr i32 %y, 16
@@ -227,20 +229,21 @@ define i32 @f18(i32 %a, i32 %x, i32 %y) {
 
 define i32 @f19(i32 %a, i32 %x, i32 %y) {
 ; CHECK-LABEL: f19:
-; CHECK: {{smlatb r0, r1, r2, r0|smlabt r0, r2, r1, r0}}
-; CHECK-THUMBV6-NOT: {{smlatb|smlabt}}
-        %tmp0 = shl i32 %y, 16
+; CHECK: {{smlatb r0, r2, r1, r0|smlabt r0, r1, r2, r0}}
+; DISABLED-NOT: {{smlatb|smlabt}}
+        %tmp0 = shl i32 %x, 16
         %tmp1 = ashr i32 %tmp0, 16
-        %tmp2 = ashr i32 %x, 16
-        %tmp3 = mul i32 %tmp2, %tmp1
+        %tmp2 = ashr i32 %y, 16
+        %tmp3 = mul i32 %tmp1, %tmp2
         %tmp5 = add i32 %tmp3, %a
         ret i32 %tmp5
 }
 
 define i32 @f20(i32 %a, i32 %x, i32 %y) {
 ; CHECK-LABEL: f20:
+; CHECK-NOT: sxth
 ; CHECK: smlabb
-; CHECK-THUMBV6-NOT: smlabb
+; DISABLED-NOT: smlabb
         %tmp1 = shl i32 %x, 16
         %tmp2 = ashr i32 %tmp1, 16
         %tmp3 = shl i32 %y, 16
@@ -254,7 +257,7 @@ define i32 @f21(i32 %a, i32 %x, i16 %y) {
 ; CHECK-LABEL: f21
 ; CHECK-NOT: sxth
 ; CHECK: smlabb
-; CHECK-THUMBV6-NOT: smlabb
+; DISABLED-NOT: smlabb
         %tmp1 = shl i32 %x, 16
         %tmp2 = ashr i32 %tmp1, 16
         %tmp3 = sext i16 %y to i32
@@ -263,12 +266,26 @@ define i32 @f21(i32 %a, i32 %x, i16 %y) {
         ret i32 %tmp5
 }
 
+define i32 @f21_b(i32 %a, i32 %x, i16 %y) {
+; CHECK-LABEL: f21_b
+; CHECK-NOT: sxth
+; CHECK: smlabb
+; DISABLED-NOT: smlabb
+        %tmp1 = shl i32 %x, 16
+        %tmp2 = ashr i32 %tmp1, 16
+        %tmp3 = sext i16 %y to i32
+        %tmp4 = mul i32 %tmp3, %tmp2
+        %tmp5 = add i32 %a, %tmp4
+        ret i32 %tmp5
+}
+
 @global_b = external global i16, align 2
 
 define i32 @f22(i32 %a) {
 ; CHECK-LABEL: f22:
+; CHECK-NOT: sxth
 ; CHECK: smulwb r0, r0, r1
-; CHECK-THUMBV6-NOT: smulwb
+; DISABLED-NOT: smulwb
         %b = load i16, i16* @global_b, align 2
         %sext = sext i16 %b to i64
         %conv = sext i32 %a to i64
@@ -280,8 +297,9 @@ define i32 @f22(i32 %a) {
 
 define i32 @f23(i32 %a, i32 %c) {
 ; CHECK-LABEL: f23:
+; CHECK-NOT: sxth
 ; CHECK: smlawb r0, r0, r2, r1
-; CHECK-THUMBV6-NOT: smlawb
+; DISABLED-NOT: smlawb
         %b = load i16, i16* @global_b, align 2
         %sext = sext i16 %b to i64
         %conv = sext i32 %a to i64
@@ -291,3 +309,102 @@ define i32 @f23(i32 %a, i32 %c) {
         %add = add nsw i32 %conv5, %c
         ret i32 %add
 }
+
+; CHECK-LABEL: f24
+; CHECK-NOT: sxth
+; CHECK: smulbb
+define i32 @f24(i16* %a, i32* %b, i32* %c) {
+  %ld.0 = load i16, i16* %a, align 2
+  %ld.1 = load i32, i32* %b, align 4
+  %conv.0 = sext i16 %ld.0 to i32
+  %shift = shl i32 %ld.1, 16
+  %conv.1 = ashr i32 %shift, 16
+  %mul.0 = mul i32 %conv.0, %conv.1
+  store i32 %ld.1, i32* %c
+  ret i32 %mul.0
+}
+
+; CHECK-LABEL: f25
+; CHECK-NOT: sxth
+; CHECK: smulbb
+define i32 @f25(i16* %a, i32 %b, i32* %c) {
+  %ld.0 = load i16, i16* %a, align 2
+  %conv.0 = sext i16 %ld.0 to i32
+  %shift = shl i32 %b, 16
+  %conv.1 = ashr i32 %shift, 16
+  %mul.0 = mul i32 %conv.0, %conv.1
+  store i32 %b, i32* %c
+  ret i32 %mul.0
+}
+
+; CHECK-LABEL: f25_b
+; CHECK-NOT: sxth
+; CHECK: smulbb
+define i32 @f25_b(i16* %a, i32 %b, i32* %c) {
+  %ld.0 = load i16, i16* %a, align 2
+  %conv.0 = sext i16 %ld.0 to i32
+  %shift = shl i32 %b, 16
+  %conv.1 = ashr i32 %shift, 16
+  %mul.0 = mul i32 %conv.1, %conv.0
+  store i32 %b, i32* %c
+  ret i32 %mul.0
+}
+
+; CHECK-LABEL: f26
+; CHECK-NOT: sxth
+; CHECK: {{smulbt | smultb}}
+define i32 @f26(i16* %a, i32 %b, i32* %c) {
+  %ld.0 = load i16, i16* %a, align 2
+  %conv.0 = sext i16 %ld.0 to i32
+  %conv.1 = ashr i32 %b, 16
+  %mul.0 = mul i32 %conv.0, %conv.1
+  store i32 %b, i32* %c
+  ret i32 %mul.0
+}
+
+; CHECK-LABEL: f26_b
+; CHECK-NOT: sxth
+; CHECK: {{smulbt | smultb}}
+define i32 @f26_b(i16* %a, i32 %b, i32* %c) {
+  %ld.0 = load i16, i16* %a, align 2
+  %conv.0 = sext i16 %ld.0 to i32
+  %conv.1 = ashr i32 %b, 16
+  %mul.0 = mul i32 %conv.1, %conv.0
+  store i32 %b, i32* %c
+  ret i32 %mul.0
+}
+
+; CHECK-LABEL: f27
+; CHECK-NOT: sxth
+; CHECK: smulbb
+; CHECK: {{smlabt | smlatb}}
+define i32 @f27(i16* %a, i32* %b) {
+  %ld.0 = load i16, i16* %a, align 2
+  %ld.1 = load i32, i32* %b, align 4
+  %conv.0 = sext i16 %ld.0 to i32
+  %shift = shl i32 %ld.1, 16
+  %conv.1 = ashr i32 %shift, 16
+  %conv.2 = ashr i32 %ld.1, 16
+  %mul.0 = mul i32 %conv.0, %conv.1
+  %mul.1 = mul i32 %conv.0, %conv.2
+  %add = add i32 %mul.0, %mul.1
+  ret i32 %add
+}
+
+; CHECK-LABEL: f27_b
+; CHECK-NOT: sxth
+; CHECK: smulbb
+; CHECK: {{smlabt | smlatb}}
+define i32 @f27_b(i16* %a, i32* %b) {
+  %ld.0 = load i16, i16* %a, align 2
+  %ld.1 = load i32, i32* %b, align 4
+  %conv.0 = sext i16 %ld.0 to i32
+  %shift = shl i32 %ld.1, 16
+  %conv.1 = ashr i32 %shift, 16
+  %conv.2 = ashr i32 %ld.1, 16
+  %mul.0 = mul i32 %conv.0, %conv.1
+  %mul.1 = mul i32 %conv.2, %conv.0
+  %add = add i32 %mul.0, %mul.1
+  ret i32 %add
+}
+
diff --git a/test/CodeGen/ARM/ssp-data-layout.ll b/test/CodeGen/ARM/ssp-data-layout.ll
index b087fa9489f48d1f21ddfd12d9543d15ddeceac7..feb0189be9ed27920903f78e2a52e82dd409a9a1 100644
--- a/test/CodeGen/ARM/ssp-data-layout.ll
+++ b/test/CodeGen/ARM/ssp-data-layout.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -disable-fp-elim -mcpu=cortex-a8 -mtriple arm-linux-gnu -target-abi=apcs -o - | FileCheck %s
+; RUN: llc < %s -frame-pointer=all -mcpu=cortex-a8 -mtriple arm-linux-gnu -target-abi=apcs -o - | FileCheck %s
 ;  This test is fairly fragile.  The goal is to ensure that "large" stack
 ;  objects are allocated closest to the stack protector (i.e., farthest away 
 ;  from the Stack Pointer.)  In standard SSP mode this means that large (>=
diff --git a/test/CodeGen/ARM/sub.ll b/test/CodeGen/ARM/sub.ll
index 9ac314dd60819b8f5708b85d7a2227e3776d3d1b..249f42d9b6911008c2633d7b0bb5bfcc0e1e42fa 100644
--- a/test/CodeGen/ARM/sub.ll
+++ b/test/CodeGen/ARM/sub.ll
@@ -1,10 +1,13 @@
-; RUN: llc -mtriple=arm-eabi -mcpu=cortex-a8 %s -o - | FileCheck %s -check-prefix=CHECK -check-prefix=CHECK-LE
+; RUN: llc -mtriple=arm-eabi -mcpu=cortex-a8 %s -show-mc-encoding -o - | FileCheck %s -check-prefix=CHECK -check-prefix=CHECK-LE
 ; RUN: llc -mtriple=armeb-eabi -mcpu=cortex-a8 %s -o - | FileCheck %s -check-prefix=CHECK -check-prefix=CHECK-BE
+; RUN: llc -mtriple=thumbv6m %s -o - | FileCheck %s --check-prefix=CHECK-V6M
+; RUN: llc -mtriple=thumbv8m.base -show-mc-encoding %s -o - | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-V8M
+; RUN: llc -mtriple=thumbv8m.main -show-mc-encoding %s -o - | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-V8M
 
 ; 171 = 0x000000ab
 define i64 @f1(i64 %a) {
-; CHECK: f1
-; CHECK-LE: subs r0, r0, #171
+; CHECK-LABEL: f1
+; CHECK-LE: subs{{.*}} r0, #171
 ; CHECK-LE: sbc r1, r1, #0
 ; CHECK-BE: subs r1, r1, #171
 ; CHECK-BE: sbc r0, r0, #0
@@ -14,8 +17,8 @@ define i64 @f1(i64 %a) {
 
 ; 66846720 = 0x03fc0000
 define i64 @f2(i64 %a) {
-; CHECK: f2
-; CHECK-LE: subs r0, r0, #66846720
+; CHECK-LABEL: f2
+; CHECK-LE: subs{{.*}} r0, r0, #66846720
 ; CHECK-LE: sbc r1, r1, #0
 ; CHECK-BE: subs r1, r1, #66846720
 ; CHECK-BE: sbc r0, r0, #0
@@ -25,8 +28,8 @@ define i64 @f2(i64 %a) {
 
 ; 734439407618 = 0x000000ab00000002
 define i64 @f3(i64 %a) {
-; CHECK: f3
-; CHECK-LE: subs r0, r0, #2
+; CHECK-LABEL: f3
+; CHECK-LE: subs{{.*}} r0, #2
 ; CHECK-LE: sbc r1, r1, #171
 ; CHECK-BE: subs r1, r1, #2
 ; CHECK-BE: sbc r0, r0, #171
@@ -36,22 +39,47 @@ define i64 @f3(i64 %a) {
 
 define i32 @f4(i32 %x) {
 entry:
-; CHECK: f4
-; CHECK: rsbs
+; CHECK-LABEL: f4
+; CHECK-LE: rsbs
+; CHECK-BE: rsbs
   %sub = sub i32 1, %x
   %cmp = icmp ugt i32 %sub, 0
   %sel = select i1 %cmp, i32 1, i32 %sub
   ret i32 %sel
 }
 
-; rdar://11726136
 define i32 @f5(i32 %x) {
 entry:
-; CHECK: f5
-; CHECK: movw r1, #65535
+; CHECK-LABEL: f5:
+; CHECK-LE:  movw r1, #65535 @ encoding: [0xff,0x1f,0x0f,0xe3]
+; CHECK-V8M: movw r1, #65535 @ encoding: [0x4f,0xf6,0xff,0x71]
 ; CHECK-NOT: movt
 ; CHECK-NOT: add
-; CHECK: sub r0, r0, r1
+; CHECK: sub{{.*}} r0, r0, r1
+
+; CHECK-V6M-LABEL: f5
+; CHECK-V6M: ldr [[NEG:r[0-1]+]], [[CONST:.[A-Z0-9_]+]]
+; CHECK-V6M: add{{.*}} r0, [[NEG]]
+; CHECK-V6M: [[CONST]]
+; CHECK-V6M: .long   4294901761
   %sub = add i32 %x, -65535
   ret i32 %sub
 }
+
+define i32 @f6(i32 %x) {
+entry:
+; CHECK-LABEL: f6:
+; CHECK-LE:  movw r1, #65535 @ encoding: [0xff,0x1f,0x0f,0xe3]
+; CHECK-V8M: movw r1, #65535 @ encoding: [0x4f,0xf6,0xff,0x71]
+; CHECK-NOT: movt
+; CHECK-NOT: sub
+; CHECK: add{{.*}} r0, r1
+
+; CHECK-V6M-LABEL: f6
+; CHECK-V6M: ldr [[NEG:r[0-1]+]], [[CONST:.[A-Z0-9_]+]]
+; CHECK-V6M: add{{.*}} r0, [[NEG]]
+; CHECK-V6M: [[CONST]]
+; CHECK-V6M: .long 65535
+  %sub = sub i32 %x, -65535
+  ret i32 %sub
+}
diff --git a/test/CodeGen/ARM/subreg-remat.ll b/test/CodeGen/ARM/subreg-remat.ll
index 1b406103d1183c9f3be1dd266697255c0d0bddd0..6166a947fad8f67e1150129d5d7fe9e539a6fb19 100644
--- a/test/CodeGen/ARM/subreg-remat.ll
+++ b/test/CodeGen/ARM/subreg-remat.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -relocation-model=pic -disable-fp-elim -mcpu=cortex-a8 -pre-RA-sched=source -no-integrated-as | FileCheck %s
+; RUN: llc < %s -relocation-model=pic -frame-pointer=all -mcpu=cortex-a8 -pre-RA-sched=source -no-integrated-as | FileCheck %s
 target triple = "thumbv7-apple-ios"
 ; <rdar://problem/10032939>
 ;
diff --git a/test/CodeGen/ARM/subtarget-no-movt.ll b/test/CodeGen/ARM/subtarget-no-movt.ll
index fb0d5adc3f48bf388f1581273e846aedf457c26b..3fb84f00524f5c408821e17f68d5c3001d708ff9 100644
--- a/test/CodeGen/ARM/subtarget-no-movt.ll
+++ b/test/CodeGen/ARM/subtarget-no-movt.ll
@@ -1,99 +1,97 @@
-; RUN: llc -mcpu=cortex-a8 -relocation-model=static %s -o - | FileCheck -check-prefix=NO-OPTION %s
-; RUN: llc -mcpu=cortex-a8 -relocation-model=static %s -o - -mattr=-no-movt | FileCheck -check-prefix=USE-MOVT %s
-; RUN: llc -mcpu=cortex-a8 -relocation-model=static %s -o - -mattr=+no-movt | FileCheck -check-prefix=NO-USE-MOVT %s
-; RUN: llc -mcpu=cortex-a8 -relocation-model=static %s -o - -O0 | FileCheck -check-prefix=NO-OPTION-O0 %s
-; RUN: llc -mcpu=cortex-a8 -relocation-model=static %s -o - -O0 -mattr=-no-movt | FileCheck -check-prefix=USE-MOVT-O0 %s
-; RUN: llc -mcpu=cortex-a8 -relocation-model=static %s -o - -O0 -mattr=+no-movt | FileCheck -check-prefix=NO-USE-MOVT-O0 %s
+; RUN: llc -mcpu=cortex-a8 -relocation-model=static %s -o - | \
+; RUN:   FileCheck -check-prefixes=CHECK,NO-OPTION,NO-OPTION-COMMON %s
+; RUN: llc -mcpu=cortex-a8 -relocation-model=static %s -o - -mattr=-no-movt | \
+; RUN:   FileCheck -check-prefixes=CHECK,USE-MOVT,USE-MOVT-COMMON %s
+; RUN: llc -mcpu=cortex-a8 -relocation-model=static %s -o - -mattr=+no-movt | \
+; RUN:   FileCheck -check-prefixes=CHECK,NO-USE-MOVT,NO-USE-MOVT-COMMON %s
+; RUN: llc -mcpu=cortex-a8 -relocation-model=static %s -o - -O0 | \
+; RUN:   FileCheck -check-prefixes=CHECK,NO-OPTION-O0,NO-OPTION-COMMON %s
+; RUN: llc -mcpu=cortex-a8 -relocation-model=static %s -o - -O0 -mattr=-no-movt | \
+; RUN:  FileCheck -check-prefixes=CHECK,USE-MOVT-O0,USE-MOVT-COMMON %s
+; RUN: llc -mcpu=cortex-a8 -relocation-model=static %s -o - -O0 -mattr=+no-movt | \
+; RUN:   FileCheck -check-prefixes=CHECK,NO-USE-MOVT-O0,NO-USE-MOVT-COMMON %s
 
 target triple = "thumb-apple-darwin"
 
-; NO-OPTION-LABEL: {{_?}}foo0
-; NO-OPTION: ldr [[R0:r[0-9]+]], [[L0:.*]]
-; NO-OPTION: [[L0]]:
-; NO-OPTION: .long 2296237089
+; NO-OPTION-COMMON-LABEL: {{_?}}foo0
+; NO-OPTION-COMMON: ldr [[R0:r[0-9]+]], [[L0:.*]]
+; NO-OPTION-COMMON: [[L0]]:
+; NO-OPTION-COMMON: .long 2296237089
 
-; NO-OPTION-O0-LABEL: {{_?}}foo0
-; NO-OPTION-O0: ldr [[R0:r[0-9]+]], [[L0:.*]]
-; NO-OPTION-O0: [[L0]]:
-; NO-OPTION-O0: .long 2296237089
+; USE-MOVT-COMMON-LABEL: {{_?}}foo0
+; USE-MOVT-COMMON: movw [[R0:r[0-9]+]], #52257
+; USE-MOVT-COMMON: movt [[R0]], #35037
 
-; USE-MOVT-LABEL: {{_?}}foo0
-; USE-MOVT: movw [[R0:r[0-9]+]], #52257
-; USE-MOVT: movt [[R0]], #35037
-
-; USE-MOVT-O0-LABEL: {{_?}}foo0
-; USE-MOVT-O0: movw [[R0:r[0-9]+]], #52257
-; USE-MOVT-O0: movt [[R0]], #35037
-
-; NO-USE-MOVT-LABEL: {{_?}}foo0
-; NO-USE-MOVT: ldr [[R0:r[0-9]+]], [[L0:.*]]
-; NO-USE-MOVT: [[L0]]:
-; NO-USE-MOVT: .long 2296237089
-
-; NO-USE-MOVT-O0-LABEL: {{_?}}foo0
-; NO-USE-MOVT-O0: ldr [[R0:r[0-9]+]], [[L0:.*]]
-; NO-USE-MOVT-O0: [[L0]]:
-; NO-USE-MOVT-O0: .long 2296237089
+; NO-USE-MOVT-COMMON-LABEL: {{_?}}foo0
+; NO-USE-MOVT-COMMON: ldr [[R0:r[0-9]+]], [[L0:.*]]
+; NO-USE-MOVT-COMMON: [[L0]]:
+; NO-USE-MOVT-COMMON: .long 2296237089
 
 define i32 @foo0(i32 %a) #0 {
   %1 = xor i32 -1998730207, %a
   ret i32 %1
 }
 
-; NO-OPTION-LABEL: {{_?}}foo1
-; NO-OPTION: movw [[R0:r[0-9]+]], #52257
-; NO-OPTION: movt [[R0]], #35037
-
-; NO-OPTION-O0-LABEL: {{_?}}foo1
-; NO-OPTION-O0: movw [[R0:r[0-9]+]], #52257
-; NO-OPTION-O0: movt [[R0]], #35037
+; NO-OPTION-COMMON-LABEL: {{_?}}foo1
+; NO-OPTION-COMMON: movw [[R0:r[0-9]+]], #52257
+; NO-OPTION-COMMON: movt [[R0]], #35037
 
-; USE-MOVT-LABEL: {{_?}}foo1
-; USE-MOVT: movw [[R0:r[0-9]+]], #52257
-; USE-MOVT: movt [[R0]], #35037
+; USE-MOVT-COMMON-LABEL: {{_?}}foo1
+; USE-MOVT-COMMON: movw [[R0:r[0-9]+]], #52257
+; USE-MOVT-COMMON: movt [[R0]], #35037
 
-; USE-MOVT-O0-LABEL: {{_?}}foo1
-; USE-MOVT-O0: movw [[R0:r[0-9]+]], #52257
-; USE-MOVT-O0: movt [[R0]], #35037
-
-; NO-USE-MOVT-LABEL: {{_?}}foo1
-; NO-USE-MOVT: ldr [[R0:r[0-9]+]], [[L0:.*]]
-; NO-USE-MOVT: [[L0]]:
-; NO-USE-MOVT: .long 2296237089
-
-; NO-USE-MOVT-O0-LABEL: {{_?}}foo1
-; NO-USE-MOVT-O0: ldr [[R0:r[0-9]+]], [[L0:.*]]
-; NO-USE-MOVT-O0: [[L0]]:
-; NO-USE-MOVT-O0: .long 2296237089
+; NO-USE-MOVT-COMMON-LABEL: {{_?}}foo1
+; NO-USE-MOVT-COMMON: ldr [[R0:r[0-9]+]], [[L0:.*]]
+; NO-USE-MOVT-COMMON: [[L0]]:
+; NO-USE-MOVT-COMMON: .long 2296237089
 
 define i32 @foo1(i32 %a) {
   %1 = xor i32 -1998730207, %a
   ret i32 %1
 }
 
-; NO-OPTION-LABEL: {{_?}}foo2
+; NO-OPTION-COMMON-LABEL: {{_?}}foo2
 ; NO-OPTION:   mov.w	[[R0:r[0-9]+]], #-536813568
-
-; USE-MOVT-LABEL: {{_?}}foo2
-; USE-MOVT:    mov.w	[[R0:r[0-9]+]], #-536813568
-
-; NO-USE-MOVT-LABEL: {{_?}}foo2
-; NO-USE-MOVT: mov.w	[[R0:r[0-9]+]], #-536813568
-
-; NO-OPTION-O0-LABEL: {{_?}}foo2
 ; NO-OPTION-O0: movw	[[R0:r[0-9]+]], #57344
 ; NO-OPTION-O0: movt	[[R0]], #57344
 
-; USE-MOVT-O0-LABEL: {{_?}}foo2
-; USE-MOVT-O0:  movw	[[R0:r[0-9]+]], #57344
-; USE-MOVTT-O0: movt	[[R0]], #57344
+; USE-MOVT-COMMON-LABEL: {{_?}}foo2
+; USE-MOVT:     mov.w [[R0:r[0-9]+]], #-536813568
+; USE-MOVT-O0:  movw  [[R0:r[0-9]+]], #57344
+; USE-MOVT-O0:  movt  [[R0]], #57344
 
-; NO-USE-MOVT-O0-LABEL: {{_?}}foo2
+; NO-USE-MOVT-COMMON-LABEL: {{_?}}foo2
+; NO-USE-MOVT: mov.w	[[R0:r[0-9]+]], #-536813568
 ; NO-USE-MOVT-O0: ldr [[R0:r[0-9]+]], [[L0:.*]]
 ; NO-USE-MOVT-O0: [[L0]]:
 ; NO-USE-MOVT-O0: .long 3758153728     @ 0xe000e000
+
 define i32 @foo2() {
   %1 = load i32, i32* inttoptr (i32 -536813568 to i32*) ; load from 0xe000e000
   ret i32 %1
 }
 attributes #0 = { "target-features"="+no-movt" }
+
+define hidden i32 @no_litpool() minsize optsize {
+; CHECK-LABEL:  no_litpool:
+; CHECK:        mov.w r{{.}}, #65536
+; CHECK:        mov.w r{{.}}, #-134217728
+; CHECK:        mvn r{{.}}, #-134217728
+entry:
+  %call0 = tail call i32 @eat_const(i32 65536)
+  %call1 = tail call i32 @eat_const(i32 -134217728)
+  %call2 = tail call i32 @eat_const(i32 134217727)
+  ret i32 %call2
+}
+
+define hidden i32 @litpool() minsize optsize {
+; CHECK-LABEL:  litpool:
+; CHECK:        ldr r0, {{.*}}LCPI{{.*}}
+; CHECK-NEXT:   b.w {{.*}}eat_const
+entry:
+  %call1 = tail call i32 @eat_const(i32 8388601)
+  ret i32 %call1
+}
+
+declare dso_local i32 @eat_const(i32)
+
diff --git a/test/CodeGen/AVR/PR37143.ll b/test/CodeGen/AVR/PR37143.ll
new file mode 100644
index 0000000000000000000000000000000000000000..db157edc22f08723abc86de030a1f5df2f3e6ec5
--- /dev/null
+++ b/test/CodeGen/AVR/PR37143.ll
@@ -0,0 +1,13 @@
+; RUN: llc -mattr=avr6,sram < %s -march=avr | FileCheck %s
+
+; CHECK: ld {{r[0-9]+}}, [[PTR:[YZ]]]
+; CHECK: ldd {{r[0-9]+}}, [[PTR]]+1
+; CHECK: st [[PTR]], {{r[0-9]+}}
+; CHECK: std [[PTR]]+1, {{r[0-9]+}}
+define void @load_store_16(i16* nocapture %ptr) local_unnamed_addr #1 {
+entry:
+  %0 = load i16, i16* %ptr, align 2
+  %add = add i16 %0, 5
+  store i16 %add, i16* %ptr, align 2
+  ret void
+}
diff --git a/test/CodeGen/AVR/atomics/load16.ll b/test/CodeGen/AVR/atomics/load16.ll
index 2b51afe45f4fa34c0a32af28fd60bcb14232781b..6b96edbdab8e7cf5369b70b21713bd481c3937b2 100644
--- a/test/CodeGen/AVR/atomics/load16.ll
+++ b/test/CodeGen/AVR/atomics/load16.ll
@@ -3,8 +3,8 @@
 ; CHECK-LABEL: atomic_load16
 ; CHECK:      in r0, 63
 ; CHECK-NEXT: cli
-; CHECK-NEXT: ld [[RR:r[0-9]+]], [[RD:(X|Y|Z)]]+
-; CHECK-NEXT: ld [[RR:r[0-9]+]], [[RD:(X|Y|Z)]]
+; CHECK-NEXT: ld  [[RR:r[0-9]+]], [[RD:(X|Y|Z)]]
+; CHECK-NEXT: ldd [[RR:r[0-9]+]], [[RD]]+1
 ; CHECK-NEXT: out 63, r0
 define i16 @atomic_load16(i16* %foo) {
   %val = load atomic i16, i16* %foo unordered, align 2
@@ -29,12 +29,12 @@ define i16 @atomic_load_cmp_swap16(i16* %foo) {
 ; CHECK-LABEL: atomic_load_add16
 ; CHECK:      in r0, 63
 ; CHECK-NEXT: cli
-; CHECK-NEXT: ld [[RR1:r[0-9]+]], [[RD1:(X|Y|Z)]]+
-; CHECK-NEXT: ld [[RR2:r[0-9]+]], [[RD2:(X|Y|Z)]]
+; CHECK-NEXT: ld [[RR1:r[0-9]+]], [[RD:(X|Y|Z)]]
+; CHECK-NEXT: ldd [[RR2:r[0-9]+]], [[RD]]+1
 ; CHECK-NEXT: add [[RR1]], [[TMP:r[0-9]+]]
 ; CHECK-NEXT: adc [[RR2]], [[TMP:r[0-9]+]]
-; CHECK-NEXT: st [[RD1]], [[RR1]]
-; CHECK-NEXT: std [[RD1]]+1, [[A:r[0-9]+]]
+; CHECK-NEXT: st [[RD]], [[RR1]]
+; CHECK-NEXT: std [[RD]]+1, [[A:r[0-9]+]]
 ; CHECK-NEXT: out 63, r0
 define i16 @atomic_load_add16(i16* %foo) {
   %val = atomicrmw add i16* %foo, i16 13 seq_cst
@@ -44,12 +44,12 @@ define i16 @atomic_load_add16(i16* %foo) {
 ; CHECK-LABEL: atomic_load_sub16
 ; CHECK:      in r0, 63
 ; CHECK-NEXT: cli
-; CHECK-NEXT: ld [[RR1:r[0-9]+]], [[RD1:(X|Y|Z)]]+
-; CHECK-NEXT: ld [[RR2:r[0-9]+]], [[RD2:(X|Y|Z)]]
+; CHECK-NEXT: ld [[RR1:r[0-9]+]], [[RD:(X|Y|Z)]]
+; CHECK-NEXT: ldd [[RR2:r[0-9]+]], [[RD]]+1
 ; CHECK-NEXT: sub [[RR1]], [[TMP:r[0-9]+]]
 ; CHECK-NEXT: sbc [[RR2]], [[TMP:r[0-9]+]]
-; CHECK-NEXT: st [[RD1]], [[RR1]]
-; CHECK-NEXT: std [[RD1]]+1, [[A:r[0-9]+]]
+; CHECK-NEXT: st [[RD]], [[RR1]]
+; CHECK-NEXT: std [[RD]]+1, [[A:r[0-9]+]]
 ; CHECK-NEXT: out 63, r0
 define i16 @atomic_load_sub16(i16* %foo) {
   %val = atomicrmw sub i16* %foo, i16 13 seq_cst
@@ -59,12 +59,12 @@ define i16 @atomic_load_sub16(i16* %foo) {
 ; CHECK-LABEL: atomic_load_and16
 ; CHECK:      in r0, 63
 ; CHECK-NEXT: cli
-; CHECK-NEXT: ld [[RR1:r[0-9]+]], [[RD1:(X|Y|Z)]]+
-; CHECK-NEXT: ld [[RR2:r[0-9]+]], [[RD2:(X|Y|Z)]]
+; CHECK-NEXT: ld [[RR1:r[0-9]+]], [[RD:(X|Y|Z)]]
+; CHECK-NEXT: ldd [[RR2:r[0-9]+]], [[RD]]+1
 ; CHECK-NEXT: and [[RR1]], [[TMP:r[0-9]+]]
 ; CHECK-NEXT: and [[RR2]], [[TMP:r[0-9]+]]
-; CHECK-NEXT: st [[RD1]], [[RR1]]
-; CHECK-NEXT: std [[RD1]]+1, [[A:r[0-9]+]]
+; CHECK-NEXT: st [[RD]], [[RR1]]
+; CHECK-NEXT: std [[RD]]+1, [[A:r[0-9]+]]
 ; CHECK-NEXT: out 63, r0
 define i16 @atomic_load_and16(i16* %foo) {
   %val = atomicrmw and i16* %foo, i16 13 seq_cst
@@ -74,12 +74,12 @@ define i16 @atomic_load_and16(i16* %foo) {
 ; CHECK-LABEL: atomic_load_or16
 ; CHECK:      in r0, 63
 ; CHECK-NEXT: cli
-; CHECK-NEXT: ld [[RR1:r[0-9]+]], [[RD1:(X|Y|Z)]]+
-; CHECK-NEXT: ld [[RR2:r[0-9]+]], [[RD2:(X|Y|Z)]]
+; CHECK-NEXT: ld [[RR1:r[0-9]+]], [[RD:(X|Y|Z)]]
+; CHECK-NEXT: ldd [[RR2:r[0-9]+]], [[RD]]+1
 ; CHECK-NEXT: or [[RR1]], [[TMP:r[0-9]+]]
 ; CHECK-NEXT: or [[RR2]], [[TMP:r[0-9]+]]
-; CHECK-NEXT: st [[RD1]], [[RR1]]
-; CHECK-NEXT: std [[RD1]]+1, [[A:r[0-9]+]]
+; CHECK-NEXT: st [[RD]], [[RR1]]
+; CHECK-NEXT: std [[RD]]+1, [[A:r[0-9]+]]
 ; CHECK-NEXT: out 63, r0
 define i16 @atomic_load_or16(i16* %foo) {
   %val = atomicrmw or i16* %foo, i16 13 seq_cst
@@ -89,12 +89,12 @@ define i16 @atomic_load_or16(i16* %foo) {
 ; CHECK-LABEL: atomic_load_xor16
 ; CHECK:      in r0, 63
 ; CHECK-NEXT: cli
-; CHECK-NEXT: ld [[RR1:r[0-9]+]], [[RD1:(X|Y|Z)]]+
-; CHECK-NEXT: ld [[RR2:r[0-9]+]], [[RD2:(X|Y|Z)]]
+; CHECK-NEXT: ld [[RR1:r[0-9]+]], [[RD:(X|Y|Z)]]
+; CHECK-NEXT: ldd [[RR2:r[0-9]+]], [[RD]]+1
 ; CHECK-NEXT: eor [[RR1]], [[TMP:r[0-9]+]]
 ; CHECK-NEXT: eor [[RR2]], [[TMP:r[0-9]+]]
-; CHECK-NEXT: st [[RD1]], [[RR1]]
-; CHECK-NEXT: std [[RD1]]+1, [[A:r[0-9]+]]
+; CHECK-NEXT: st [[RD]], [[RR1]]
+; CHECK-NEXT: std [[RD]]+1, [[A:r[0-9]+]]
 ; CHECK-NEXT: out 63, r0
 define i16 @atomic_load_xor16(i16* %foo) {
   %val = atomicrmw xor i16* %foo, i16 13 seq_cst
diff --git a/test/CodeGen/AVR/avr-rust-issue-123.ll b/test/CodeGen/AVR/avr-rust-issue-123.ll
new file mode 100644
index 0000000000000000000000000000000000000000..b2e4bf0e1448f6ae8faeb06b51d98301defe457f
--- /dev/null
+++ b/test/CodeGen/AVR/avr-rust-issue-123.ll
@@ -0,0 +1,74 @@
+; RUN: llc -O1 < %s -march=avr | FileCheck %s
+
+; This test ensures that the Select8/Select16 expansion
+; pass inserts an unconditional branch to the previous adjacent
+; basic block when inserting new basic blocks when the
+; prior block has a fallthrough.
+;
+; Before this bug was fixed, Select8/Select16 expansion
+; would leave a dangling fallthrough to an undefined block.
+;
+; The BranchFolding pass would later rearrange the basic
+; blocks based on predecessor/successor list assumptions
+; which were made incorrect due to the invalid Select
+; expansion.
+
+; More information in
+; https://github.com/avr-rust/rust/issues/123.
+
+%UInt8 = type <{ i8 }>
+%UInt32 = type <{ i32 }>
+%Sb = type <{ i1 }>
+
+@delayFactor = hidden global %UInt8 zeroinitializer, align 1
+@delay = hidden global %UInt32 zeroinitializer, align 4
+@flag = hidden global %Sb zeroinitializer, align 1
+
+declare void @eeprom_write(i16, i8)
+
+; CHECK-LABEL: update_register
+define hidden void @update_register(i8 %arg, i8 %arg1) {
+entry:
+  ; CHECK: push [[PRELUDER:r[0-9]+]]
+  ; CHECK: cpi  r24, 7
+  switch i8 %arg, label %bb7 [
+    i8 6, label %bb
+    i8 7, label %bb6
+  ]
+
+; CHECK-NOT: ret
+bb:                                               ; preds = %entry
+  %tmp = icmp ugt i8 %arg1, 90
+  %tmp2 = icmp ult i8 %arg1, 5
+  %. = select i1 %tmp2, i8 5, i8 %arg1
+  %tmp3 = select i1 %tmp, i8 90, i8 %.
+  ; CHECK: sts delayFactor, r{{[0-9]+}}
+  store i8 %tmp3, i8* getelementptr inbounds (%UInt8, %UInt8* @delayFactor, i64 0, i32 0), align 1
+  %tmp4 = zext i8 %tmp3 to i32
+  %tmp5 = mul nuw nsw i32 %tmp4, 100
+  ; CHECK:      sts  delay+3, r{{[0-9]+}}
+  ; CHECK-NEXT: sts  delay+2, r{{[0-9]+}}
+  ; CHECK-NEXT: sts  delay+1, r{{[0-9]+}}
+  ; CHECK-NEXT: sts  delay, r{{[0-9]+}}
+  store i32 %tmp5, i32* getelementptr inbounds (%UInt32, %UInt32* @delay, i64 0, i32 0), align 4
+  tail call void @eeprom_write(i16 34, i8 %tmp3)
+  br label %bb7
+
+bb6:                                              ; preds = %entry
+  %not. = icmp ne i8 %arg1, 0
+  %.2 = zext i1 %not. to i8
+  store i1 %not., i1* getelementptr inbounds (%Sb, %Sb* @flag, i64 0, i32 0), align 1
+
+  ; CHECK: call eeprom_write
+  tail call void @eeprom_write(i16 35, i8 %.2)
+  br label %bb7
+
+  ; CHECK: LBB0_{{[0-9]+}}
+  ; CHECK: pop [[PRELUDER]]
+  ; CHECK-NEXT: ret
+bb7:                                              ; preds = %bb6, %bb, %entry
+  ret void
+}
+; CHECK-NOT: LBB0_{{[0-9]+}}:
+; CHECK-LABEL: .Lfunc_end0
+; CHECK: .size  update_register, .Lfunc_end0-update_register
diff --git a/test/CodeGen/AVR/mul.ll b/test/CodeGen/AVR/hardware-mul.ll
similarity index 90%
rename from test/CodeGen/AVR/mul.ll
rename to test/CodeGen/AVR/hardware-mul.ll
index 2f169347c46e0fdf0a282368d7e40305b5769e72..650697857b76b151ac8bb29c4ba556cd11d98d23 100644
--- a/test/CodeGen/AVR/mul.ll
+++ b/test/CodeGen/AVR/hardware-mul.ll
@@ -1,5 +1,7 @@
 ; RUN: llc -mattr=mul,movw < %s -march=avr | FileCheck %s
 
+; Tests lowering of multiplication to hardware instructions.
+
 define i8 @mult8(i8 %a, i8 %b) {
 ; CHECK-LABEL: mult8:
 ; CHECK: muls r22, r24
diff --git a/test/CodeGen/AVR/integration/blink.ll b/test/CodeGen/AVR/integration/blink.ll
index ef88e3ea24b1870feb157c244a20112825eaf8a4..29c3b7f48ff19d416245d64a61ad3e9fcb05f23c 100644
--- a/test/CodeGen/AVR/integration/blink.ll
+++ b/test/CodeGen/AVR/integration/blink.ll
@@ -34,15 +34,8 @@
 define void @setup_ddr() {
 entry:
 
-  ; This should load the value of DDRB, OR it with the bit number and store
-  ; the result back to DDRB.
-
-  ; CHECK:      in      [[TMPREG:r[0-9]+]], 4
-  ; CHECK-NEXT: ori     [[TMPREG]], 32
-
-  ; CHECK-NOT: ori     {{r[0-9]+}}, 0
-
-  ; CHECK-NEXT: out     4, [[TMPREG]]
+  ; This should set the 5th bit of DDRB.
+  ; CHECK:      sbi	4, 5
   ; CHECK-NEXT: ret
 
   %0 = load volatile i8, i8* inttoptr (i16 36 to i8*), align 1
@@ -58,15 +51,8 @@ entry:
 define void @turn_on() {
 entry:
 
-  ; This should load the value of PORTB, OR it with the bit number and store
-  ; the result back to DDRB.
-
-  ; CHECK:      in      [[TMPREG:r[0-9]+]], 5
-  ; CHECK-NEXT: ori     [[TMPREG]], 32
-
-  ; CHECK-NOT: ori     {{r[0-9]+}}, 0
-
-  ; CHECK-NEXT: out     5, [[TMPREG]]
+  ; This should set the 5th bit of PORTB
+  ; CHECK:      sbi	5, 5
   ; CHECK-NEXT: ret
 
   %0 = load volatile i8, i8* inttoptr (i16 37 to i8*), align 1
@@ -82,14 +68,8 @@ entry:
 define void @turn_off() {
 entry:
 
-  ; This should load the value of PORTB, OR it with the bit number and store
-  ; the result back to DDRB.
-
-
-  ; CHECK:      in      [[TMPREG:r[0-9]+]], 5
-  ; CHECK-NEXT: andi    [[TMPREG]], 223
-  ; CHECK-NEXT: andi    {{r[0-9]+}}, 0
-  ; CHECK-NEXT: out     5, [[TMPREG]]
+  ; This should clear the 5th bit of PORTB
+  ; CHECK:      cbi     5, 5
   ; CHECK-NEXT: ret
 
   %0 = load volatile i8, i8* inttoptr (i16 37 to i8*), align 1
diff --git a/test/CodeGen/AVR/load.ll b/test/CodeGen/AVR/load.ll
index 73568b54096566cb884a25a1d01e75b61e4d9112..f58edeb425a72189f07ba42249a43ea8dd3d85e6 100644
--- a/test/CodeGen/AVR/load.ll
+++ b/test/CodeGen/AVR/load.ll
@@ -9,8 +9,8 @@ define i8 @load8(i8* %x) {
 
 define i16 @load16(i16* %x) {
 ; CHECK-LABEL: load16:
-; CHECK: ld r24, {{[XYZ]}}+
-; CHECK: ld r25, {{[XYZ]}}
+; CHECK: ld r24,  [[PTR:[XYZ]]]
+; CHECK: ldd r25, [[PTR]]+1
   %1 = load i16, i16* %x
   ret i16 %1
 }
@@ -36,8 +36,8 @@ define i8 @load8nodisp(i8* %x) {
 
 define i16 @load16disp(i16* %x) {
 ; CHECK-LABEL: load16disp:
-; CHECK: ldd r24, {{[YZ]}}+62
-; CHECK: ldd r25, {{[YZ]}}+63
+; CHECK: ldd r24, [[PTR:[YZ]]]+62
+; CHECK: ldd r25, [[PTR]]+63
   %1 = getelementptr inbounds i16, i16* %x, i64 31
   %2 = load i16, i16* %1
   ret i16 %2
@@ -48,8 +48,8 @@ define i16 @load16nodisp(i16* %x) {
 ; CHECK: movw r26, r24
 ; CHECK: subi r26, 192
 ; CHECK: sbci r27, 255
-; CHECK: ld r24, {{[XYZ]}}+
-; CHECK: ld r25, {{[XYZ]}}
+; CHECK: ld r24,  [[PTR:[XYZ]]]
+; CHECK: ldd r25, [[PTR]]+1
   %1 = getelementptr inbounds i16, i16* %x, i64 32
   %2 = load i16, i16* %1
   ret i16 %2
diff --git a/test/CodeGen/AVR/pre-schedule.ll b/test/CodeGen/AVR/pre-schedule.ll
new file mode 100644
index 0000000000000000000000000000000000000000..c724eea5d365520015ef8496aa3df18f0066bd1e
--- /dev/null
+++ b/test/CodeGen/AVR/pre-schedule.ll
@@ -0,0 +1,32 @@
+; RUN: llc < %s -march=avr | FileCheck %s
+target triple = "avr-unknown-unknown"
+
+; The case illustrate DAG schedular may pre-schedule the node has
+; ADJCALLSTACKDOWN parent, so ADJCALLSTACKUP may hold CallResource too long
+; and make other calls can't be scheduled. If there's no other available node
+; to schedule, the scheduler will try to rename the register by creating the
+; copy to avoid the conflict which will fail because CallResource is not a real
+; physical register.
+;
+; The issue is found by Tim on https://github.com/avr-rust/rust/issues/111 and
+; discuss in http://lists.llvm.org/pipermail/llvm-dev/2018-October/127083.html.
+
+define void @"main"() addrspace(1) {
+start:
+  %0 = or i64 undef, undef
+   br i1 undef, label %mul_and_call, label %fail
+
+  mul_and_call:
+  %1 = mul i64 %0, %0
+  call addrspace(1) void @"three_ints"(i64 undef, i64 %1, i64 %0)
+; The CHECK line only want to make sure the following assertion message
+; won't trigger due to create copy of artificial CallResource register.
+; llc: llvm/lib/CodeGen/TargetRegisterInfo.cpp:203: const llvm::TargetRegisterClass* llvm::TargetRegisterInfo::getMinimalPhysRegClass(unsigned int, llvm::MVT) const: Assertion `BestRC && "Couldn't find the register class"' failed.
+; CHECK: call    __muldi3
+  ret void
+
+  fail:
+    ret void
+}
+
+declare void @"three_ints"(i64, i64, i64) addrspace(1)
diff --git a/test/CodeGen/AVR/pseudo/LDWRdPtr-same-src-dst.mir b/test/CodeGen/AVR/pseudo/LDWRdPtr-same-src-dst.mir
index 4fc97f63d95c47cf9ea370197f5fafe4000ebc28..3c3a7219ee3e403d08dcb12d548e4e93ce23e082 100644
--- a/test/CodeGen/AVR/pseudo/LDWRdPtr-same-src-dst.mir
+++ b/test/CodeGen/AVR/pseudo/LDWRdPtr-same-src-dst.mir
@@ -18,9 +18,9 @@ body: |
 
     ; CHECK-LABEL: test_ldwrdptr
 
-    ; CHECK:      ld [[SCRATCH:r[0-9]+]], Z+
+    ; CHECK:      ld [[SCRATCH:r[0-9]+]], Z
     ; CHECK-NEXT: push [[SCRATCH]]
-    ; CHECK-NEXT: ld [[SCRATCH]], Z
+    ; CHECK-NEXT: ldd [[SCRATCH]], Z+1
     ; CHECK-NEXT: mov r31, [[SCRATCH]]
     ; CHECK-NEXT: pop r30
 
diff --git a/test/CodeGen/AVR/pseudo/LDWRdPtr.mir b/test/CodeGen/AVR/pseudo/LDWRdPtr.mir
index 5398a8b1a982ac37ed9b2219be59a492f624b89a..5bd4bf2d431c8b517a4e95c755132575368d7ed1 100644
--- a/test/CodeGen/AVR/pseudo/LDWRdPtr.mir
+++ b/test/CodeGen/AVR/pseudo/LDWRdPtr.mir
@@ -17,8 +17,8 @@ body: |
 
     ; CHECK-LABEL: test_ldwrdptr
 
-    ; CHECK:      $r0, $r31r30 = LDRdPtrPi $r31r30
-    ; CHECK-NEXT:          $r1 = LDRdPtr $r31r30
+    ; CHECK:      $r0, $r31r30 = LDRdPtr
+    ; CHECK-NEXT:          $r1 = LDDRdPtrQ $r31r30, 1
 
     $r1r0 = LDWRdPtr $r31r30
 ...
diff --git a/test/CodeGen/AVR/smul-with-overflow.ll b/test/CodeGen/AVR/smul-with-overflow.ll
index 745e93005cc2f22400f94409393ad1311addf28c..9eb2c7411dee85b93b7fe6355ae5630058b72301 100644
--- a/test/CodeGen/AVR/smul-with-overflow.ll
+++ b/test/CodeGen/AVR/smul-with-overflow.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -march=avr | FileCheck %s
+; RUN: llc -mattr=avr6 < %s -march=avr | FileCheck %s
 
 define i1 @signed_multiplication_did_overflow(i8, i8) unnamed_addr {
 ; CHECK-LABEL: signed_multiplication_did_overflow:
diff --git a/test/CodeGen/AVR/software-mul.ll b/test/CodeGen/AVR/software-mul.ll
new file mode 100644
index 0000000000000000000000000000000000000000..9a4d28127eb876a3247b3e410ec187f1709149b0
--- /dev/null
+++ b/test/CodeGen/AVR/software-mul.ll
@@ -0,0 +1,28 @@
+; RUN: llc -mattr=avr6,-mul < %s -march=avr | FileCheck %s
+; RUN: llc -mcpu=attiny85 < %s -march=avr | FileCheck %s
+; RUN: llc -mcpu=ata5272 < %s -march=avr | FileCheck %s
+; RUN: llc -mcpu=attiny861a < %s -march=avr | FileCheck %s
+; RUN: llc -mcpu=at90usb82 < %s -march=avr | FileCheck %s
+
+; Tests lowering of multiplication to compiler support routines.
+
+; CHECK-LABEL: mul8:
+define i8 @mul8(i8 %a, i8 %b) {
+; CHECK: mov  r25, r24
+; CHECK: mov  r24, r22
+; CHECK: mov  r22, r25
+; CHECK: call __mulqi3
+  %mul = mul i8 %b, %a
+  ret i8 %mul
+}
+
+; CHECK-LABEL: mul16:
+define i16 @mul16(i16 %a, i16 %b) {
+; CHECK: movw  r18, r24
+; CHECK: movw  r24, r22
+; CHECK: movw  r22, r18
+; CHECK: call  __mulhi3
+  %mul = mul nsw i16 %b, %a
+  ret i16 %mul
+}
+
diff --git a/test/CodeGen/AVR/umul-with-overflow.ll b/test/CodeGen/AVR/umul-with-overflow.ll
index aa8b10a313d380e063fb5e161d03ff023bb18bee..c6457552dea88f90d28aa92474b2869f4b4ad1c4 100644
--- a/test/CodeGen/AVR/umul-with-overflow.ll
+++ b/test/CodeGen/AVR/umul-with-overflow.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -march=avr | FileCheck %s
+; RUN: llc -mattr=avr6 < %s -march=avr | FileCheck %s
 
 define i1 @unsigned_multiplication_did_overflow(i8, i8) unnamed_addr {
 ; CHECK-LABEL: unsigned_multiplication_did_overflow:
diff --git a/test/CodeGen/BPF/BTF/array-1d-char.ll b/test/CodeGen/BPF/BTF/array-1d-char.ll
new file mode 100644
index 0000000000000000000000000000000000000000..5ef3e9209a091fbad7313fdc0759c77ab23c3faa
--- /dev/null
+++ b/test/CodeGen/BPF/BTF/array-1d-char.ll
@@ -0,0 +1,68 @@
+; RUN: llc -march=bpfel -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -march=bpfeb -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+
+; Source code:
+;   char a[10];
+; Compilation flag:
+;   clang -target bpf -O2 -g -S -emit-llvm t.c
+
+@a = common dso_local local_unnamed_addr global [10 x i8] zeroinitializer, align 1, !dbg !0
+
+!llvm.dbg.cu = !{!2}
+!llvm.module.flags = !{!10, !11, !12}
+!llvm.ident = !{!13}
+
+; CHECK:             .section        .BTF,"",@progbits
+; CHECK-NEXT:        .short  60319                   # 0xeb9f
+; CHECK-NEXT:        .byte   1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .long   24
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   56
+; CHECK-NEXT:        .long   56
+; CHECK-NEXT:        .long   26
+; CHECK-NEXT:        .long   0                       # BTF_KIND_ARRAY(id = 1)
+; CHECK-NEXT:        .long   50331648                # 0x3000000
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   3
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   10
+; CHECK-NEXT:        .long   1                       # BTF_KIND_INT(id = 2)
+; CHECK-NEXT:        .long   16777216                # 0x1000000
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   32                      # 0x20
+; CHECK-NEXT:        .long   21                      # BTF_KIND_INT(id = 3)
+; CHECK-NEXT:        .long   16777216                # 0x1000000
+; CHECK-NEXT:        .long   1
+; CHECK-NEXT:        .long   16777224                # 0x1000008
+; CHECK-NEXT:        .byte   0                       # string offset=0
+; CHECK-NEXT:        .ascii  "__ARRAY_SIZE_TYPE__"   # string offset=1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .ascii  "char"                  # string offset=21
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .section        .BTF.ext,"",@progbits
+; CHECK-NEXT:        .short  60319                   # 0xeb9f
+; CHECK-NEXT:        .byte   1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .long   24
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   8                       # FuncInfo
+; CHECK-NEXT:        .long   16                      # LineInfo
+
+!0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression())
+!1 = distinct !DIGlobalVariable(name: "a", scope: !2, file: !3, line: 1, type: !6, isLocal: false, isDefinition: true)
+!2 = distinct !DICompileUnit(language: DW_LANG_C99, file: !3, producer: "clang version 8.0.0 (trunk 345296) (llvm/trunk 345297)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, globals: !5, nameTableKind: None)
+!3 = !DIFile(filename: "t.c", directory: "/home/yhs/tmp")
+!4 = !{}
+!5 = !{!0}
+!6 = !DICompositeType(tag: DW_TAG_array_type, baseType: !7, size: 80, elements: !8)
+!7 = !DIBasicType(name: "char", size: 8, encoding: DW_ATE_signed_char)
+!8 = !{!9}
+!9 = !DISubrange(count: 10)
+!10 = !{i32 2, !"Dwarf Version", i32 4}
+!11 = !{i32 2, !"Debug Info Version", i32 3}
+!12 = !{i32 1, !"wchar_size", i32 4}
+!13 = !{!"clang version 8.0.0 (trunk 345296) (llvm/trunk 345297)"}
diff --git a/test/CodeGen/BPF/BTF/array-1d-int.ll b/test/CodeGen/BPF/BTF/array-1d-int.ll
new file mode 100644
index 0000000000000000000000000000000000000000..8d81dbe1017749e74739eecfa18b736d8083db3a
--- /dev/null
+++ b/test/CodeGen/BPF/BTF/array-1d-int.ll
@@ -0,0 +1,68 @@
+; RUN: llc -march=bpfel -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -march=bpfeb -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+
+; Source code:
+;   int a[10];
+; Compilation flag:
+;   clang -target bpf -O2 -g -S -emit-llvm t.c
+
+@a = common dso_local local_unnamed_addr global [10 x i32] zeroinitializer, align 4, !dbg !0
+
+!llvm.dbg.cu = !{!2}
+!llvm.module.flags = !{!10, !11, !12}
+!llvm.ident = !{!13}
+
+; CHECK:             .section        .BTF,"",@progbits
+; CHECK-NEXT:        .short  60319                   # 0xeb9f
+; CHECK-NEXT:        .byte   1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .long   24
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   56
+; CHECK-NEXT:        .long   56
+; CHECK-NEXT:        .long   25
+; CHECK-NEXT:        .long   0                       # BTF_KIND_ARRAY(id = 1)
+; CHECK-NEXT:        .long   50331648                # 0x3000000
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   3
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   10
+; CHECK-NEXT:        .long   1                       # BTF_KIND_INT(id = 2)
+; CHECK-NEXT:        .long   16777216                # 0x1000000
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   32                      # 0x20
+; CHECK-NEXT:        .long   21                      # BTF_KIND_INT(id = 3)
+; CHECK-NEXT:        .long   16777216                # 0x1000000
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   16777248                # 0x1000020
+; CHECK-NEXT:        .byte   0                       # string offset=0
+; CHECK-NEXT:        .ascii  "__ARRAY_SIZE_TYPE__"   # string offset=1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .ascii  "int"                   # string offset=21
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .section        .BTF.ext,"",@progbits
+; CHECK-NEXT:        .short  60319                   # 0xeb9f
+; CHECK-NEXT:        .byte   1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .long   24
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   8                       # FuncInfo
+; CHECK-NEXT:        .long   16                      # LineInfo
+
+!0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression())
+!1 = distinct !DIGlobalVariable(name: "a", scope: !2, file: !3, line: 1, type: !6, isLocal: false, isDefinition: true)
+!2 = distinct !DICompileUnit(language: DW_LANG_C99, file: !3, producer: "clang version 8.0.0 (trunk 345296) (llvm/trunk 345297)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, globals: !5, nameTableKind: None)
+!3 = !DIFile(filename: "t.c", directory: "/home/yhs/tmp")
+!4 = !{}
+!5 = !{!0}
+!6 = !DICompositeType(tag: DW_TAG_array_type, baseType: !7, size: 320, elements: !8)
+!7 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!8 = !{!9}
+!9 = !DISubrange(count: 10)
+!10 = !{i32 2, !"Dwarf Version", i32 4}
+!11 = !{i32 2, !"Debug Info Version", i32 3}
+!12 = !{i32 1, !"wchar_size", i32 4}
+!13 = !{!"clang version 8.0.0 (trunk 345296) (llvm/trunk 345297)"}
diff --git a/test/CodeGen/BPF/BTF/array-2d-int.ll b/test/CodeGen/BPF/BTF/array-2d-int.ll
new file mode 100644
index 0000000000000000000000000000000000000000..00db42e4959573819cadc920ce4af41544c636ef
--- /dev/null
+++ b/test/CodeGen/BPF/BTF/array-2d-int.ll
@@ -0,0 +1,68 @@
+; RUN: llc -march=bpfel -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -march=bpfeb -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+
+; Source code:
+;   int a[10][10];
+; Compilation flag:
+;   clang -target bpf -O2 -g -S -emit-llvm t.c
+
+@a = common dso_local local_unnamed_addr global [10 x [10 x i32]] zeroinitializer, align 4, !dbg !0
+
+!llvm.dbg.cu = !{!2}
+!llvm.module.flags = !{!10, !11, !12}
+!llvm.ident = !{!13}
+
+; CHECK:             .section        .BTF,"",@progbits
+; CHECK-NEXT:        .short  60319                   # 0xeb9f
+; CHECK-NEXT:        .byte   1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .long   24
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   56
+; CHECK-NEXT:        .long   56
+; CHECK-NEXT:        .long   25
+; CHECK-NEXT:        .long   0                       # BTF_KIND_ARRAY(id = 1)
+; CHECK-NEXT:        .long   50331648                # 0x3000000
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   3
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   100
+; CHECK-NEXT:        .long   1                       # BTF_KIND_INT(id = 2)
+; CHECK-NEXT:        .long   16777216                # 0x1000000
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   32                      # 0x20
+; CHECK-NEXT:        .long   21                      # BTF_KIND_INT(id = 3)
+; CHECK-NEXT:        .long   16777216                # 0x1000000
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   16777248                # 0x1000020
+; CHECK-NEXT:        .byte   0                       # string offset=0
+; CHECK-NEXT:        .ascii  "__ARRAY_SIZE_TYPE__"   # string offset=1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .ascii  "int"                   # string offset=21
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .section        .BTF.ext,"",@progbits
+; CHECK-NEXT:        .short  60319                   # 0xeb9f
+; CHECK-NEXT:        .byte   1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .long   24
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   8                       # FuncInfo
+; CHECK-NEXT:        .long   16                      # LineInfo
+
+!0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression())
+!1 = distinct !DIGlobalVariable(name: "a", scope: !2, file: !3, line: 1, type: !6, isLocal: false, isDefinition: true)
+!2 = distinct !DICompileUnit(language: DW_LANG_C99, file: !3, producer: "clang version 8.0.0 (trunk 345296) (llvm/trunk 345297)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, globals: !5, nameTableKind: None)
+!3 = !DIFile(filename: "t.c", directory: "/home/yhs/tmp")
+!4 = !{}
+!5 = !{!0}
+!6 = !DICompositeType(tag: DW_TAG_array_type, baseType: !7, size: 3200, elements: !8)
+!7 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!8 = !{!9, !9}
+!9 = !DISubrange(count: 10)
+!10 = !{i32 2, !"Dwarf Version", i32 4}
+!11 = !{i32 2, !"Debug Info Version", i32 3}
+!12 = !{i32 1, !"wchar_size", i32 4}
+!13 = !{!"clang version 8.0.0 (trunk 345296) (llvm/trunk 345297)"}
diff --git a/test/CodeGen/BPF/BTF/array-size-0.ll b/test/CodeGen/BPF/BTF/array-size-0.ll
new file mode 100644
index 0000000000000000000000000000000000000000..11e694a26ec46074b375d0147e40b03b8abcb923
--- /dev/null
+++ b/test/CodeGen/BPF/BTF/array-size-0.ll
@@ -0,0 +1,70 @@
+; RUN: llc -march=bpfel -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -march=bpfeb -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+
+; Source code:
+;   struct t {};
+;   struct t a[10];
+; Compilation flag:
+;   clang -target bpf -O2 -g -S -emit-llvm t.c
+
+%struct.t = type {}
+
+@a = common dso_local local_unnamed_addr global [10 x %struct.t] zeroinitializer, align 1, !dbg !0
+
+!llvm.dbg.cu = !{!2}
+!llvm.module.flags = !{!10, !11, !12}
+!llvm.ident = !{!13}
+
+; CHECK:             .section        .BTF,"",@progbits
+; CHECK-NEXT:        .short  60319                   # 0xeb9f
+; CHECK-NEXT:        .byte   1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .long   24
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   52
+; CHECK-NEXT:        .long   52
+; CHECK-NEXT:        .long   23
+; CHECK-NEXT:        .long   0                       # BTF_KIND_ARRAY(id = 1)
+; CHECK-NEXT:        .long   50331648                # 0x3000000
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   3
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   1                       # BTF_KIND_INT(id = 2)
+; CHECK-NEXT:        .long   16777216                # 0x1000000
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   32                      # 0x20
+; CHECK-NEXT:        .long   21                      # BTF_KIND_STRUCT(id = 3)
+; CHECK-NEXT:        .long   67108864                # 0x4000000
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .byte   0                       # string offset=0
+; CHECK-NEXT:        .ascii  "__ARRAY_SIZE_TYPE__"   # string offset=1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .byte   116                     # string offset=21
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .section        .BTF.ext,"",@progbits
+; CHECK-NEXT:        .short  60319                   # 0xeb9f
+; CHECK-NEXT:        .byte   1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .long   24
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   8                       # FuncInfo
+; CHECK-NEXT:        .long   16                      # LineInfo
+
+!0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression())
+!1 = distinct !DIGlobalVariable(name: "a", scope: !2, file: !3, line: 2, type: !6, isLocal: false, isDefinition: true)
+!2 = distinct !DICompileUnit(language: DW_LANG_C99, file: !3, producer: "clang version 8.0.0 (trunk 345296) (llvm/trunk 345297)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, globals: !5, nameTableKind: None)
+!3 = !DIFile(filename: "t.c", directory: "/home/yhs/tmp")
+!4 = !{}
+!5 = !{!0}
+!6 = !DICompositeType(tag: DW_TAG_array_type, baseType: !7, elements: !8)
+!7 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "t", file: !3, line: 1, elements: !4)
+!8 = !{!9}
+!9 = !DISubrange(count: 10)
+!10 = !{i32 2, !"Dwarf Version", i32 4}
+!11 = !{i32 2, !"Debug Info Version", i32 3}
+!12 = !{i32 1, !"wchar_size", i32 4}
+!13 = !{!"clang version 8.0.0 (trunk 345296) (llvm/trunk 345297)"}
diff --git a/test/CodeGen/BPF/BTF/array-typedef.ll b/test/CodeGen/BPF/BTF/array-typedef.ll
new file mode 100644
index 0000000000000000000000000000000000000000..9fe57c9cb79222194ffcbca4e77fddaf94b05414
--- /dev/null
+++ b/test/CodeGen/BPF/BTF/array-typedef.ll
@@ -0,0 +1,82 @@
+; RUN: llc -march=bpfel -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -march=bpfeb -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+
+; Source code:
+;   typedef unsigned _int;
+;   typedef _int __int;
+;   __int a[10];
+; Compilation flag:
+;   clang -target bpf -O2 -g -S -emit-llvm t.c
+
+@a = common dso_local local_unnamed_addr global [10 x i32] zeroinitializer, align 4, !dbg !0
+
+!llvm.dbg.cu = !{!2}
+!llvm.module.flags = !{!12, !13, !14}
+!llvm.ident = !{!15}
+
+; CHECK:             .section        .BTF,"",@progbits
+; CHECK-NEXT:        .short  60319                   # 0xeb9f
+; CHECK-NEXT:        .byte   1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .long   24
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   80
+; CHECK-NEXT:        .long   80
+; CHECK-NEXT:        .long   45
+; CHECK-NEXT:        .long   0                       # BTF_KIND_ARRAY(id = 1)
+; CHECK-NEXT:        .long   50331648                # 0x3000000
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   3
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   10
+; CHECK-NEXT:        .long   1                       # BTF_KIND_INT(id = 2)
+; CHECK-NEXT:        .long   16777216                # 0x1000000
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   32                      # 0x20
+; CHECK-NEXT:        .long   21                      # BTF_KIND_TYPEDEF(id = 3)
+; CHECK-NEXT:        .long   134217728               # 0x8000000
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   27                      # BTF_KIND_TYPEDEF(id = 4)
+; CHECK-NEXT:        .long   134217728               # 0x8000000
+; CHECK-NEXT:        .long   5
+; CHECK-NEXT:        .long   32                      # BTF_KIND_INT(id = 5)
+; CHECK-NEXT:        .long   16777216                # 0x1000000
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   32                      # 0x20
+; CHECK-NEXT:        .byte   0                       # string offset=0
+; CHECK-NEXT:        .ascii  "__ARRAY_SIZE_TYPE__"   # string offset=1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .ascii  "__int"                 # string offset=21
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .ascii  "_int"                  # string offset=27
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .ascii  "unsigned int"          # string offset=32
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .section        .BTF.ext,"",@progbits
+; CHECK-NEXT:        .short  60319                   # 0xeb9f
+; CHECK-NEXT:        .byte   1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .long   24
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   8                       # FuncInfo
+; CHECK-NEXT:        .long   16                      # LineInfo
+
+!0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression())
+!1 = distinct !DIGlobalVariable(name: "a", scope: !2, file: !3, line: 3, type: !6, isLocal: false, isDefinition: true)
+!2 = distinct !DICompileUnit(language: DW_LANG_C99, file: !3, producer: "clang version 8.0.0 (trunk 345296) (llvm/trunk 345297)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, globals: !5, nameTableKind: None)
+!3 = !DIFile(filename: "t.c", directory: "/home/yhs/tmp")
+!4 = !{}
+!5 = !{!0}
+!6 = !DICompositeType(tag: DW_TAG_array_type, baseType: !7, size: 320, elements: !10)
+!7 = !DIDerivedType(tag: DW_TAG_typedef, name: "__int", file: !3, line: 2, baseType: !8)
+!8 = !DIDerivedType(tag: DW_TAG_typedef, name: "_int", file: !3, line: 1, baseType: !9)
+!9 = !DIBasicType(name: "unsigned int", size: 32, encoding: DW_ATE_unsigned)
+!10 = !{!11}
+!11 = !DISubrange(count: 10)
+!12 = !{i32 2, !"Dwarf Version", i32 4}
+!13 = !{i32 2, !"Debug Info Version", i32 3}
+!14 = !{i32 1, !"wchar_size", i32 4}
+!15 = !{!"clang version 8.0.0 (trunk 345296) (llvm/trunk 345297)"}
diff --git a/test/CodeGen/BPF/BTF/binary-format.ll b/test/CodeGen/BPF/BTF/binary-format.ll
new file mode 100644
index 0000000000000000000000000000000000000000..39d699b2848e6df6502b6576f5c7bf84bc5ed07c
--- /dev/null
+++ b/test/CodeGen/BPF/BTF/binary-format.ll
@@ -0,0 +1,68 @@
+; RUN: llc -march=bpfel -filetype=obj -o - %s | llvm-readelf -x ".BTF" -x ".BTF.ext" | FileCheck -check-prefixes=CHECK,CHECK-EL %s
+; RUN: llc -march=bpfeb -filetype=obj -o - %s | llvm-readelf -x ".BTF" -x ".BTF.ext" | FileCheck -check-prefixes=CHECK,CHECK-EB %s
+
+; Source code:
+;   int f(int a) { return a; }
+; Compilation flag:
+;   clang -target bpf -O2 -g -gdwarf-5 -gembed-source -S -emit-llvm t.c
+
+; Function Attrs: nounwind readnone
+define dso_local i32 @f(i32 returned %a) local_unnamed_addr #0 !dbg !7 {
+entry:
+  call void @llvm.dbg.value(metadata i32 %a, metadata !12, metadata !DIExpression()), !dbg !13
+  ret i32 %a, !dbg !14
+}
+
+; CHECK:    '.BTF'
+; CHECK-EL: 0x00000000 9feb0100 18000000 00000000 30000000
+; CHECK-EL: 0x00000010 30000000 33000000 2b000000 00000001
+; CHECK-EL: 0x00000020 04000000 20000001 00000000 0100000d
+; CHECK-EL: 0x00000030 01000000 2f000000 01000000 31000000
+; CHECK-EL: 0x00000040 0000000c 02000000 002e7465 7874002f
+; CHECK-EB: 0x00000000 eb9f0100 00000018 00000000 00000030
+; CHECK-EB: 0x00000010 00000030 00000033 0000002b 01000000
+; CHECK-EB: 0x00000020 00000004 01000020 00000000 0d000001
+; CHECK-EB: 0x00000030 00000001 0000002f 00000001 00000031
+; CHECK-EB: 0x00000040 0c000000 00000002 002e7465 7874002f
+; CHECK:    0x00000050 746d702f 742e6300 696e7420 6628696e
+; CHECK:    0x00000060 74206129 207b2072 65747572 6e20613b
+; CHECK:    0x00000070 207d0069 6e740061 006600
+; CHECK:    '.BTF.ext'
+; CHECK-EL: 0x00000000 9feb0100 18000000 00000000 14000000
+; CHECK-EL: 0x00000010 14000000 2c000000 08000000 01000000
+; CHECK-EL: 0x00000020 01000000 00000000 03000000 10000000
+; CHECK-EL: 0x00000030 01000000 02000000 00000000 07000000
+; CHECK-EL: 0x00000040 10000000 00040000 08000000 07000000
+; CHECK-EL: 0x00000050 10000000 10040000
+; CHECK-EB: 0x00000000 eb9f0100 00000018 00000000 00000014
+; CHECK-EB: 0x00000010 00000014 0000002c 00000008 00000001
+; CHECK-EB: 0x00000020 00000001 00000000 00000003 00000010
+; CHECK-EB: 0x00000030 00000001 00000002 00000000 00000007
+; CHECK-EB: 0x00000040 00000010 00000400 00000008 00000007
+; CHECK-EB: 0x00000050 00000010 00000410
+
+; Function Attrs: nounwind readnone speculatable
+declare void @llvm.dbg.value(metadata, metadata, metadata) #1
+
+attributes #0 = { nounwind readnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind readnone speculatable }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4, !5}
+!llvm.ident = !{!6}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 8.0.0 (trunk 344789) (llvm/trunk 344782)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, nameTableKind: None)
+!1 = !DIFile(filename: "t.c", directory: "/tmp", checksumkind: CSK_MD5, checksum: "1924f0d78deb326ceb76cd8e9f450775", source: "int f(int a) { return a; }\0A")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 5}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{i32 1, !"wchar_size", i32 4}
+!6 = !{!"clang version 8.0.0 (trunk 344789) (llvm/trunk 344782)"}
+!7 = distinct !DISubprogram(name: "f", scope: !1, file: !1, line: 1, type: !8, isLocal: false, isDefinition: true, scopeLine: 1, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, isOptimized: true, unit: !0, retainedNodes: !11)
+!8 = !DISubroutineType(types: !9)
+!9 = !{!10, !10}
+!10 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!11 = !{!12}
+!12 = !DILocalVariable(name: "a", arg: 1, scope: !7, file: !1, line: 1, type: !10)
+!13 = !DILocation(line: 1, column: 11, scope: !7)
+!14 = !DILocation(line: 1, column: 16, scope: !7)
diff --git a/test/CodeGen/BPF/BTF/char.ll b/test/CodeGen/BPF/BTF/char.ll
new file mode 100644
index 0000000000000000000000000000000000000000..122f698b322b8223149c73ac1dde5af64b2fb436
--- /dev/null
+++ b/test/CodeGen/BPF/BTF/char.ll
@@ -0,0 +1,53 @@
+; RUN: llc -march=bpfel -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -march=bpfeb -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+
+; Source code:
+;   char a;
+; Compilation flag:
+;   clang -target bpf -O2 -g -S -emit-llvm t.c
+
+@a = common dso_local local_unnamed_addr global i8 0, align 1, !dbg !0
+
+!llvm.dbg.cu = !{!2}
+!llvm.module.flags = !{!7, !8, !9}
+!llvm.ident = !{!10}
+
+; CHECK:             .section        .BTF,"",@progbits
+; CHECK-NEXT:        .short  60319                   # 0xeb9f
+; CHECK-NEXT:        .byte   1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .long   24
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   16
+; CHECK-NEXT:        .long   16
+; CHECK-NEXT:        .long   6
+; CHECK-NEXT:        .long   1                       # BTF_KIND_INT(id = 1)
+; CHECK-NEXT:        .long   16777216                # 0x1000000
+; CHECK-NEXT:        .long   1
+; CHECK-NEXT:        .long   16777224                # 0x1000008
+; CHECK-NEXT:        .byte   0                       # string offset=0
+; CHECK-NEXT:        .ascii  "char"                  # string offset=1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .section        .BTF.ext,"",@progbits
+; CHECK-NEXT:        .short  60319                   # 0xeb9f
+; CHECK-NEXT:        .byte   1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .long   24
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   8                       # FuncInfo
+; CHECK-NEXT:        .long   16                      # LineInfo
+
+!0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression())
+!1 = distinct !DIGlobalVariable(name: "a", scope: !2, file: !3, line: 1, type: !6, isLocal: false, isDefinition: true)
+!2 = distinct !DICompileUnit(language: DW_LANG_C99, file: !3, producer: "clang version 8.0.0 (trunk 344789) (llvm/trunk 344782)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, globals: !5, nameTableKind: None)
+!3 = !DIFile(filename: "t.c", directory: "/home/yhs/tmp")
+!4 = !{}
+!5 = !{!0}
+!6 = !DIBasicType(name: "char", size: 8, encoding: DW_ATE_signed_char)
+!7 = !{i32 2, !"Dwarf Version", i32 4}
+!8 = !{i32 2, !"Debug Info Version", i32 3}
+!9 = !{i32 1, !"wchar_size", i32 4}
+!10 = !{!"clang version 8.0.0 (trunk 344789) (llvm/trunk 344782)"}
diff --git a/test/CodeGen/BPF/BTF/enum-basic.ll b/test/CodeGen/BPF/BTF/enum-basic.ll
new file mode 100644
index 0000000000000000000000000000000000000000..5b710309d95581c8c7e76403709934c62c9fa5a2
--- /dev/null
+++ b/test/CodeGen/BPF/BTF/enum-basic.ll
@@ -0,0 +1,62 @@
+; RUN: llc -march=bpfel -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -march=bpfeb -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+
+; Source code:
+;   enum { A = -1, B = 2 } a;
+; Compilation flag:
+;   clang -target bpf -O2 -g -S -emit-llvm t.c
+
+@a = common dso_local local_unnamed_addr global i32 0, align 4, !dbg !0
+
+!llvm.dbg.cu = !{!2}
+!llvm.module.flags = !{!11, !12, !13}
+!llvm.ident = !{!14}
+
+; CHECK:             .section        .BTF,"",@progbits
+; CHECK-NEXT:        .short  60319                   # 0xeb9f
+; CHECK-NEXT:        .byte   1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .long   24
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   28
+; CHECK-NEXT:        .long   28
+; CHECK-NEXT:        .long   5
+; CHECK-NEXT:        .long   0                       # BTF_KIND_ENUM(id = 1)
+; CHECK-NEXT:        .long   100663298               # 0x6000002
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   1
+; CHECK-NEXT:        .long   -1
+; CHECK-NEXT:        .long   3
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .byte   0                       # string offset=0
+; CHECK-NEXT:        .byte   65                      # string offset=1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .byte   66                      # string offset=3
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .section        .BTF.ext,"",@progbits
+; CHECK-NEXT:        .short  60319                   # 0xeb9f
+; CHECK-NEXT:        .byte   1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .long   24
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   8                       # FuncInfo
+; CHECK-NEXT:        .long   16                      # LineInfo
+
+!0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression())
+!1 = distinct !DIGlobalVariable(name: "a", scope: !2, file: !3, line: 1, type: !5, isLocal: false, isDefinition: true)
+!2 = distinct !DICompileUnit(language: DW_LANG_C99, file: !3, producer: "clang version 8.0.0 (trunk 344789) (llvm/trunk 344782)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, globals: !10, nameTableKind: None)
+!3 = !DIFile(filename: "t.c", directory: "/home/yhs/tmp")
+!4 = !{!5}
+!5 = !DICompositeType(tag: DW_TAG_enumeration_type, file: !3, line: 1, baseType: !6, size: 32, elements: !7)
+!6 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!7 = !{!8, !9}
+!8 = !DIEnumerator(name: "A", value: -1)
+!9 = !DIEnumerator(name: "B", value: 2)
+!10 = !{!0}
+!11 = !{i32 2, !"Dwarf Version", i32 4}
+!12 = !{i32 2, !"Debug Info Version", i32 3}
+!13 = !{i32 1, !"wchar_size", i32 4}
+!14 = !{!"clang version 8.0.0 (trunk 344789) (llvm/trunk 344782)"}
diff --git a/test/CodeGen/BPF/BTF/filename.ll b/test/CodeGen/BPF/BTF/filename.ll
new file mode 100644
index 0000000000000000000000000000000000000000..d0ada61e5a15b2ff72c6a35954d8b2efb7cb2dfc
--- /dev/null
+++ b/test/CodeGen/BPF/BTF/filename.ll
@@ -0,0 +1,81 @@
+; RUN: llc -march=bpfel -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -march=bpfeb -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+
+; Source code:
+;   int test() { return 0; }
+; Compilation flag:
+;   clang -target bpf -O2 -g -S -emit-llvm t.c
+
+; Function Attrs: norecurse nounwind readnone uwtable
+define dso_local i32 @test() local_unnamed_addr #0 !dbg !7 {
+  ret i32 0, !dbg !11
+}
+
+; CHECK:             .section        .BTF,"",@progbits
+; CHECK-NEXT:        .short  60319                   # 0xeb9f
+; CHECK-NEXT:        .byte   1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .long   24
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   40
+; CHECK-NEXT:        .long   40
+; CHECK-NEXT:        .long   35
+; CHECK-NEXT:        .long   0                       # BTF_KIND_FUNC_PROTO(id = 1)
+; CHECK-NEXT:        .long   218103808               # 0xd000000
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   26                      # BTF_KIND_INT(id = 2)
+; CHECK-NEXT:        .long   16777216                # 0x1000000
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   16777248                # 0x1000020
+; CHECK-NEXT:        .long   30                      # BTF_KIND_FUNC(id = 3)
+; CHECK-NEXT:        .long   201326592               # 0xc000000
+; CHECK-NEXT:        .long   1
+; CHECK-NEXT:        .byte   0                       # string offset=0
+; CHECK-NEXT:        .ascii  ".text"                 # string offset=1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .ascii  "/home/yhs/ttmp/t.c"    # string offset=7
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .ascii  "int"                   # string offset=26
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .ascii  "test"                  # string offset=30
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .section        .BTF.ext,"",@progbits
+; CHECK-NEXT:        .short  60319                   # 0xeb9f
+; CHECK-NEXT:        .byte   1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .long   24
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   20
+; CHECK-NEXT:        .long   20
+; CHECK-NEXT:        .long   28
+; CHECK-NEXT:        .long   8                       # FuncInfo
+; CHECK-NEXT:        .long   1                       # FuncInfo section string offset=1
+; CHECK-NEXT:        .long   1
+; CHECK-NEXT:        .long   .Lfunc_begin{{[0-9]+}}
+; CHECK-NEXT:        .long   3
+; CHECK-NEXT:        .long   16                      # LineInfo
+; CHECK-NEXT:        .long   1                       # LineInfo section string offset=1
+; CHECK-NEXT:        .long   1
+; CHECK-NEXT:        .long   .Ltmp{{[0-9]+}}
+; CHECK-NEXT:        .long   7
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   1038                    # Line 1 Col 14
+
+attributes #0 = { norecurse nounwind readnone uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4, !5}
+!llvm.ident = !{!6}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 8.0.20181009 ", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, nameTableKind: None)
+!1 = !DIFile(filename: "/home/yhs/ttmp/t.c", directory: "/home/yhs/ttmp")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{i32 1, !"wchar_size", i32 4}
+!6 = !{!"clang version 8.0.20181009 "}
+!7 = distinct !DISubprogram(name: "test", scope: !1, file: !1, line: 1, type: !8, isLocal: false, isDefinition: true, scopeLine: 1, isOptimized: true, unit: !0, retainedNodes: !2)
+!8 = !DISubroutineType(types: !9)
+!9 = !{!10}
+!10 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!11 = !DILocation(line: 1, column: 14, scope: !7)
diff --git a/test/CodeGen/BPF/BTF/func-func-ptr.ll b/test/CodeGen/BPF/BTF/func-func-ptr.ll
new file mode 100644
index 0000000000000000000000000000000000000000..df564f5eb8a9c745201d0c646d2aec356363b9e1
--- /dev/null
+++ b/test/CodeGen/BPF/BTF/func-func-ptr.ll
@@ -0,0 +1,128 @@
+; RUN: llc -march=bpfel -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -march=bpfeb -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+
+; Source code:
+;   void (*a1)(int p1);
+;   struct t1 { void (*a1)(int p1); } b1;
+;   void f1(int p2) { }
+; Compilation flag:
+;   clang -target bpf -O2 -g -S -emit-llvm t.c
+
+%struct.t1 = type { void (i32)* }
+
+@a1 = common dso_local local_unnamed_addr global void (i32)* null, align 8, !dbg !0
+@b1 = common dso_local local_unnamed_addr global %struct.t1 zeroinitializer, align 8, !dbg !6
+
+; Function Attrs: nounwind readnone
+define dso_local void @f1(i32 %p2) local_unnamed_addr #0 !dbg !19 {
+entry:
+  call void @llvm.dbg.value(metadata i32 %p2, metadata !21, metadata !DIExpression()), !dbg !22
+  ret void, !dbg !23
+}
+
+; CHECK:             .section        .BTF,"",@progbits
+; CHECK-NEXT:        .short  60319                   # 0xeb9f
+; CHECK-NEXT:        .byte   1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .long   24
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   104
+; CHECK-NEXT:        .long   104
+; CHECK-NEXT:        .long   32
+; CHECK-NEXT:        .long   16                      # BTF_KIND_INT(id = 1)
+; CHECK-NEXT:        .long   16777216                # 0x1000000
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   16777248                # 0x1000020
+; CHECK-NEXT:        .long   0                       # BTF_KIND_FUNC_PROTO(id = 2)
+; CHECK-NEXT:        .long   218103809               # 0xd000001
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   20
+; CHECK-NEXT:        .long   1
+; CHECK-NEXT:        .long   23                      # BTF_KIND_FUNC(id = 3)
+; CHECK-NEXT:        .long   201326592               # 0xc000000
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   0                       # BTF_KIND_PTR(id = 4)
+; CHECK-NEXT:        .long   33554432                # 0x2000000
+; CHECK-NEXT:        .long   5
+; CHECK-NEXT:        .long   0                       # BTF_KIND_FUNC_PROTO(id = 5)
+; CHECK-NEXT:        .long   218103809               # 0xd000001
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   1
+; CHECK-NEXT:        .long   26                      # BTF_KIND_STRUCT(id = 6)
+; CHECK-NEXT:        .long   67108865                # 0x4000001
+; CHECK-NEXT:        .long   8
+; CHECK-NEXT:        .long   29
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .byte   0                       # string offset=0
+; CHECK-NEXT:        .ascii  ".text"                 # string offset=1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .ascii  "/tmp/t.c"              # string offset=7
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .ascii  "int"                   # string offset=16
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .ascii  "p2"                    # string offset=20
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .ascii  "f1"                    # string offset=23
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .ascii  "t1"                    # string offset=26
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .ascii  "a1"                    # string offset=29
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .section        .BTF.ext,"",@progbits
+; CHECK-NEXT:        .short  60319                   # 0xeb9f
+; CHECK-NEXT:        .byte   1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .long   24
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   20
+; CHECK-NEXT:        .long   20
+; CHECK-NEXT:        .long   28
+; CHECK-NEXT:        .long   8                       # FuncInfo
+; CHECK-NEXT:        .long   1                       # FuncInfo section string offset=1
+; CHECK-NEXT:        .long   1
+; CHECK-NEXT:        .long   .Lfunc_begin0
+; CHECK-NEXT:        .long   3
+; CHECK-NEXT:        .long   16                      # LineInfo
+; CHECK-NEXT:        .long   1                       # LineInfo section string offset=1
+; CHECK-NEXT:        .long   1
+; CHECK-NEXT:        .long   .Ltmp{{[0-9]+}}
+; CHECK-NEXT:        .long   7
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   3091                    # Line 3 Col 19
+
+; Function Attrs: nounwind readnone speculatable
+declare void @llvm.dbg.value(metadata, metadata, metadata) #1
+
+attributes #0 = { nounwind readnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind readnone speculatable }
+
+!llvm.dbg.cu = !{!2}
+!llvm.module.flags = !{!15, !16, !17}
+!llvm.ident = !{!18}
+
+!0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression())
+!1 = distinct !DIGlobalVariable(name: "a1", scope: !2, file: !3, line: 1, type: !11, isLocal: false, isDefinition: true)
+!2 = distinct !DICompileUnit(language: DW_LANG_C99, file: !3, producer: "clang version 8.0.0 (trunk 345296) (llvm/trunk 345297)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, globals: !5, nameTableKind: None)
+!3 = !DIFile(filename: "t.c", directory: "/tmp")
+!4 = !{}
+!5 = !{!0, !6}
+!6 = !DIGlobalVariableExpression(var: !7, expr: !DIExpression())
+!7 = distinct !DIGlobalVariable(name: "b1", scope: !2, file: !3, line: 2, type: !8, isLocal: false, isDefinition: true)
+!8 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "t1", file: !3, line: 2, size: 64, elements: !9)
+!9 = !{!10}
+!10 = !DIDerivedType(tag: DW_TAG_member, name: "a1", scope: !8, file: !3, line: 2, baseType: !11, size: 64)
+!11 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !12, size: 64)
+!12 = !DISubroutineType(types: !13)
+!13 = !{null, !14}
+!14 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!15 = !{i32 2, !"Dwarf Version", i32 4}
+!16 = !{i32 2, !"Debug Info Version", i32 3}
+!17 = !{i32 1, !"wchar_size", i32 4}
+!18 = !{!"clang version 8.0.0 (trunk 345296) (llvm/trunk 345297)"}
+!19 = distinct !DISubprogram(name: "f1", scope: !3, file: !3, line: 3, type: !12, isLocal: false, isDefinition: true, scopeLine: 3, flags: DIFlagPrototyped, isOptimized: true, unit: !2, retainedNodes: !20)
+!20 = !{!21}
+!21 = !DILocalVariable(name: "p2", arg: 1, scope: !19, file: !3, line: 3, type: !14)
+!22 = !DILocation(line: 3, column: 13, scope: !19)
+!23 = !DILocation(line: 3, column: 19, scope: !19)
diff --git a/test/CodeGen/BPF/BTF/func-non-void.ll b/test/CodeGen/BPF/BTF/func-non-void.ll
new file mode 100644
index 0000000000000000000000000000000000000000..e5361b98e7d77d19d61d94d8b4afc64c14768584
--- /dev/null
+++ b/test/CodeGen/BPF/BTF/func-non-void.ll
@@ -0,0 +1,97 @@
+; RUN: llc -march=bpfel -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -march=bpfeb -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+
+; Source code:
+;   int f1(int a1) { return a1; }
+; Compilation flag:
+;   clang -target bpf -O2 -g -S -emit-llvm t.c
+
+; Function Attrs: nounwind readnone
+define dso_local i32 @f1(i32 returned) local_unnamed_addr #0 !dbg !7 {
+  call void @llvm.dbg.value(metadata i32 %0, metadata !12, metadata !DIExpression()), !dbg !13
+  ret i32 %0, !dbg !14
+}
+
+; CHECK:             .section        .BTF,"",@progbits
+; CHECK-NEXT:        .short  60319                   # 0xeb9f
+; CHECK-NEXT:        .byte   1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .long   24
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   48
+; CHECK-NEXT:        .long   48
+; CHECK-NEXT:        .long   26
+; CHECK-NEXT:        .long   16                      # BTF_KIND_INT(id = 1)
+; CHECK-NEXT:        .long   16777216                # 0x1000000
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   16777248                # 0x1000020
+; CHECK-NEXT:        .long   0                       # BTF_KIND_FUNC_PROTO(id = 2)
+; CHECK-NEXT:        .long   218103809               # 0xd000001
+; CHECK-NEXT:        .long   1
+; CHECK-NEXT:        .long   20
+; CHECK-NEXT:        .long   1
+; CHECK-NEXT:        .long   23                      # BTF_KIND_FUNC(id = 3)
+; CHECK-NEXT:        .long   201326592               # 0xc000000
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .byte   0                       # string offset=0
+; CHECK-NEXT:        .ascii  ".text"                 # string offset=1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .ascii  "/tmp/t.c"              # string offset=7
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .ascii  "int"                   # string offset=16
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .ascii  "a1"                    # string offset=20
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .ascii  "f1"                    # string offset=23
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .section        .BTF.ext,"",@progbits
+; CHECK-NEXT:        .short  60319                   # 0xeb9f
+; CHECK-NEXT:        .byte   1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .long   24
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   20
+; CHECK-NEXT:        .long   20
+; CHECK-NEXT:        .long   44
+; CHECK-NEXT:        .long   8                       # FuncInfo
+; CHECK-NEXT:        .long   1                       # FuncInfo section string offset=1
+; CHECK-NEXT:        .long   1
+; CHECK-NEXT:        .long   .Lfunc_begin0
+; CHECK-NEXT:        .long   3
+; CHECK-NEXT:        .long   16                      # LineInfo
+; CHECK-NEXT:        .long   1                       # LineInfo section string offset=1
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   .Lfunc_begin0
+; CHECK-NEXT:        .long   7
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   1024                    # Line 1 Col 0
+; CHECK-NEXT:        .long   .Ltmp{{[0-9]+}}
+; CHECK-NEXT:        .long   7
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   1042                    # Line 1 Col 18
+
+; Function Attrs: nounwind readnone speculatable
+declare void @llvm.dbg.value(metadata, metadata, metadata) #1
+
+attributes #0 = { nounwind readnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind readnone speculatable }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4, !5}
+!llvm.ident = !{!6}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 8.0.0 (trunk 345562) (llvm/trunk 345560)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, nameTableKind: None)
+!1 = !DIFile(filename: "t.c", directory: "/tmp")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{i32 1, !"wchar_size", i32 4}
+!6 = !{!"clang version 8.0.0 (trunk 345562) (llvm/trunk 345560)"}
+!7 = distinct !DISubprogram(name: "f1", scope: !1, file: !1, line: 1, type: !8, isLocal: false, isDefinition: true, scopeLine: 1, flags: DIFlagPrototyped, isOptimized: true, unit: !0, retainedNodes: !11)
+!8 = !DISubroutineType(types: !9)
+!9 = !{!10, !10}
+!10 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!11 = !{!12}
+!12 = !DILocalVariable(name: "a1", arg: 1, scope: !7, file: !1, line: 1, type: !10)
+!13 = !DILocation(line: 1, column: 12, scope: !7)
+!14 = !DILocation(line: 1, column: 18, scope: !7)
diff --git a/test/CodeGen/BPF/BTF/func-source.ll b/test/CodeGen/BPF/BTF/func-source.ll
new file mode 100644
index 0000000000000000000000000000000000000000..6e1bed332d756f0df3137e4ba7cba056dbd3ab19
--- /dev/null
+++ b/test/CodeGen/BPF/BTF/func-source.ll
@@ -0,0 +1,80 @@
+; RUN: llc -march=bpfel -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -march=bpfeb -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+
+; Source code:
+;   void f(void) { }
+; Compilation flag:
+;   clang -target bpf -O2 -g -gdwarf-5 -gembed-source -S -emit-llvm t.c
+;
+; This test embeds the source code in the IR, so the line info should have
+; correct reference to the lines in the string table.
+
+; Function Attrs: norecurse nounwind readnone
+define dso_local void @f() local_unnamed_addr #0 !dbg !7 {
+entry:
+  ret void, !dbg !10
+}
+
+; CHECK:             .section        .BTF,"",@progbits
+; CHECK-NEXT:        .short  60319                   # 0xeb9f
+; CHECK-NEXT:        .byte   1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .long   24
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   24
+; CHECK-NEXT:        .long   24
+; CHECK-NEXT:        .long   35
+; CHECK-NEXT:        .long   0                       # BTF_KIND_FUNC_PROTO(id = 1)
+; CHECK-NEXT:        .long   218103808               # 0xd000000
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   33                      # BTF_KIND_FUNC(id = 2)
+; CHECK-NEXT:        .long   201326592               # 0xc000000
+; CHECK-NEXT:        .long   1
+; CHECK-NEXT:        .byte   0                       # string offset=0
+; CHECK-NEXT:        .ascii  ".text"                 # string offset=1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .ascii  "/tmp/t.c"              # string offset=7
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .ascii  "void f(void) { }"      # string offset=16
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .byte   102                     # string offset=33
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .section        .BTF.ext,"",@progbits
+; CHECK-NEXT:        .short  60319                   # 0xeb9f
+; CHECK-NEXT:        .byte   1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .long   24
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   20
+; CHECK-NEXT:        .long   20
+; CHECK-NEXT:        .long   28
+; CHECK-NEXT:        .long   8                       # FuncInfo
+; CHECK-NEXT:        .long   1                       # FuncInfo section string offset=1
+; CHECK-NEXT:        .long   1
+; CHECK-NEXT:        .long   .Lfunc_begin0
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   16                      # LineInfo
+; CHECK-NEXT:        .long   1                       # LineInfo section string offset=1
+; CHECK-NEXT:        .long   1
+; CHECK-NEXT:        .long   .Ltmp{{[0-9]+}}
+; CHECK-NEXT:        .long   7
+; CHECK-NEXT:        .long   16
+; CHECK-NEXT:        .long   1040                    # Line 1 Col 16
+
+attributes #0 = { norecurse nounwind readnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4, !5}
+!llvm.ident = !{!6}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 8.0.0 (trunk 344789) (llvm/trunk 344782)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, nameTableKind: None)
+!1 = !DIFile(filename: "t.c", directory: "/tmp", checksumkind: CSK_MD5, checksum: "978599fafe3a080b456e3d95a3710359", source: "void f(void) { }\0A")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 5}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{i32 1, !"wchar_size", i32 4}
+!6 = !{!"clang version 8.0.0 (trunk 344789) (llvm/trunk 344782)"}
+!7 = distinct !DISubprogram(name: "f", scope: !1, file: !1, line: 1, type: !8, isLocal: false, isDefinition: true, scopeLine: 1, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, isOptimized: true, unit: !0, retainedNodes: !2)
+!8 = !DISubroutineType(types: !9)
+!9 = !{null}
+!10 = !DILocation(line: 1, column: 16, scope: !7)
diff --git a/test/CodeGen/BPF/BTF/func-typedef.ll b/test/CodeGen/BPF/BTF/func-typedef.ll
new file mode 100644
index 0000000000000000000000000000000000000000..8deac0f9441b6980e762ab894fd7082c78a7d4e1
--- /dev/null
+++ b/test/CodeGen/BPF/BTF/func-typedef.ll
@@ -0,0 +1,112 @@
+; RUN: llc -march=bpfel -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -march=bpfeb -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+
+; Source code:
+;   typedef int _int;
+;   typedef _int __int;
+;   __int f(__int a) { return a; }
+; Compilation flag:
+;   clang -target bpf -O2 -g -S -emit-llvm t.c
+
+; Function Attrs: nounwind readnone
+define dso_local i32 @f(i32 returned %a) local_unnamed_addr #0 !dbg !7 {
+entry:
+  call void @llvm.dbg.value(metadata i32 %a, metadata !14, metadata !DIExpression()), !dbg !15
+  ret i32 %a, !dbg !16
+}
+
+; CHECK:             .section        .BTF,"",@progbits
+; CHECK-NEXT:        .short  60319                   # 0xeb9f
+; CHECK-NEXT:        .byte   1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .long   24
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   72
+; CHECK-NEXT:        .long   72
+; CHECK-NEXT:        .long   35
+; CHECK-NEXT:        .long   16                      # BTF_KIND_TYPEDEF(id = 1)
+; CHECK-NEXT:        .long   134217728               # 0x8000000
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   22                      # BTF_KIND_TYPEDEF(id = 2)
+; CHECK-NEXT:        .long   134217728               # 0x8000000
+; CHECK-NEXT:        .long   3
+; CHECK-NEXT:        .long   27                      # BTF_KIND_INT(id = 3)
+; CHECK-NEXT:        .long   16777216                # 0x1000000
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   16777248                # 0x1000020
+; CHECK-NEXT:        .long   0                       # BTF_KIND_FUNC_PROTO(id = 4)
+; CHECK-NEXT:        .long   218103809               # 0xd000001
+; CHECK-NEXT:        .long   1
+; CHECK-NEXT:        .long   31
+; CHECK-NEXT:        .long   1
+; CHECK-NEXT:        .long   33                      # BTF_KIND_FUNC(id = 5)
+; CHECK-NEXT:        .long   201326592               # 0xc000000
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .byte   0                       # string offset=0
+; CHECK-NEXT:        .ascii  ".text"                 # string offset=1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .ascii  "/tmp/t.c"              # string offset=7
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .ascii  "__int"                 # string offset=16
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .ascii  "_int"                  # string offset=22
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .ascii  "int"                   # string offset=27
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .byte   97                      # string offset=31
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .byte   102                     # string offset=33
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .section        .BTF.ext,"",@progbits
+; CHECK-NEXT:        .short  60319                   # 0xeb9f
+; CHECK-NEXT:        .byte   1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .long   24
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   20
+; CHECK-NEXT:        .long   20
+; CHECK-NEXT:        .long   44
+; CHECK-NEXT:        .long   8                       # FuncInfo
+; CHECK-NEXT:        .long   1                       # FuncInfo section string offset=1
+; CHECK-NEXT:        .long   1
+; CHECK-NEXT:        .long   .Lfunc_begin0
+; CHECK-NEXT:        .long   5
+; CHECK-NEXT:        .long   16                      # LineInfo
+; CHECK-NEXT:        .long   1                       # LineInfo section string offset=1
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   .Lfunc_begin0
+; CHECK-NEXT:        .long   7
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   3072                    # Line 3 Col 0
+; CHECK-NEXT:        .long   .Ltmp{{[0-9]+}}
+; CHECK-NEXT:        .long   7
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   3092                    # Line 3 Col 20
+
+; Function Attrs: nounwind readnone speculatable
+declare void @llvm.dbg.value(metadata, metadata, metadata) #1
+
+attributes #0 = { nounwind readnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind readnone speculatable }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4, !5}
+!llvm.ident = !{!6}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 8.0.0 (trunk 344789) (llvm/trunk 344782)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, nameTableKind: None)
+!1 = !DIFile(filename: "t.c", directory: "/tmp")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{i32 1, !"wchar_size", i32 4}
+!6 = !{!"clang version 8.0.0 (trunk 344789) (llvm/trunk 344782)"}
+!7 = distinct !DISubprogram(name: "f", scope: !1, file: !1, line: 3, type: !8, isLocal: false, isDefinition: true, scopeLine: 3, flags: DIFlagPrototyped, isOptimized: true, unit: !0, retainedNodes: !13)
+!8 = !DISubroutineType(types: !9)
+!9 = !{!10, !10}
+!10 = !DIDerivedType(tag: DW_TAG_typedef, name: "__int", file: !1, line: 2, baseType: !11)
+!11 = !DIDerivedType(tag: DW_TAG_typedef, name: "_int", file: !1, line: 1, baseType: !12)
+!12 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!13 = !{!14}
+!14 = !DILocalVariable(name: "a", arg: 1, scope: !7, file: !1, line: 3, type: !10)
+!15 = !DILocation(line: 3, column: 15, scope: !7)
+!16 = !DILocation(line: 3, column: 20, scope: !7)
diff --git a/test/CodeGen/BPF/BTF/func-unused-arg.ll b/test/CodeGen/BPF/BTF/func-unused-arg.ll
new file mode 100644
index 0000000000000000000000000000000000000000..62e9d40db9adceeaa1aa295f6338055ec3897427
--- /dev/null
+++ b/test/CodeGen/BPF/BTF/func-unused-arg.ll
@@ -0,0 +1,93 @@
+; RUN: llc -march=bpfel -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -march=bpfeb -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+
+; Source code:
+;   int f1(int a1) { return 0; }
+; Compilation flag:
+;   clang -target bpf -O2 -g -S -emit-llvm t.c
+
+; Function Attrs: nounwind readnone
+define dso_local i32 @f1(i32) local_unnamed_addr #0 !dbg !7 {
+  call void @llvm.dbg.value(metadata i32 %0, metadata !12, metadata !DIExpression()), !dbg !13
+  ret i32 0, !dbg !14
+}
+
+; CHECK:             .section        .BTF,"",@progbits
+; CHECK-NEXT:        .short  60319                   # 0xeb9f
+; CHECK-NEXT:        .byte   1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .long   24
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   48
+; CHECK-NEXT:        .long   48
+; CHECK-NEXT:        .long   26
+; CHECK-NEXT:        .long   16                      # BTF_KIND_INT(id = 1)
+; CHECK-NEXT:        .long   16777216                # 0x1000000
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   16777248                # 0x1000020
+; CHECK-NEXT:        .long   0                       # BTF_KIND_FUNC_PROTO(id = 2)
+; CHECK-NEXT:        .long   218103809               # 0xd000001
+; CHECK-NEXT:        .long   1
+; CHECK-NEXT:        .long   20
+; CHECK-NEXT:        .long   1
+; CHECK-NEXT:        .long   23                      # BTF_KIND_FUNC(id = 3)
+; CHECK-NEXT:        .long   201326592               # 0xc000000
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .byte   0                       # string offset=0
+; CHECK-NEXT:        .ascii  ".text"                 # string offset=1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .ascii  "/tmp/t.c"              # string offset=7
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .ascii  "int"                   # string offset=16
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .ascii  "a1"                    # string offset=20
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .ascii  "f1"                    # string offset=23
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .section        .BTF.ext,"",@progbits
+; CHECK-NEXT:        .short  60319                   # 0xeb9f
+; CHECK-NEXT:        .byte   1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .long   24
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   20
+; CHECK-NEXT:        .long   20
+; CHECK-NEXT:        .long   28
+; CHECK-NEXT:        .long   8                       # FuncInfo
+; CHECK-NEXT:        .long   1                       # FuncInfo section string offset=1
+; CHECK-NEXT:        .long   1
+; CHECK-NEXT:        .long   .Lfunc_begin0
+; CHECK-NEXT:        .long   3
+; CHECK-NEXT:        .long   16                      # LineInfo
+; CHECK-NEXT:        .long   1                       # LineInfo section string offset=1
+; CHECK-NEXT:        .long   1
+; CHECK-NEXT:        .long   .Ltmp{{[0-9]+}}
+; CHECK-NEXT:        .long   7
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   1042                    # Line 1 Col 18
+
+; Function Attrs: nounwind readnone speculatable
+declare void @llvm.dbg.value(metadata, metadata, metadata) #1
+
+attributes #0 = { nounwind readnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind readnone speculatable }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4, !5}
+!llvm.ident = !{!6}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 8.0.0 (trunk 345562) (llvm/trunk 345560)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, nameTableKind: None)
+!1 = !DIFile(filename: "t.c", directory: "/tmp")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{i32 1, !"wchar_size", i32 4}
+!6 = !{!"clang version 8.0.0 (trunk 345562) (llvm/trunk 345560)"}
+!7 = distinct !DISubprogram(name: "f1", scope: !1, file: !1, line: 1, type: !8, isLocal: false, isDefinition: true, scopeLine: 1, flags: DIFlagPrototyped, isOptimized: true, unit: !0, retainedNodes: !11)
+!8 = !DISubroutineType(types: !9)
+!9 = !{!10, !10}
+!10 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!11 = !{!12}
+!12 = !DILocalVariable(name: "a1", arg: 1, scope: !7, file: !1, line: 1, type: !10)
+!13 = !DILocation(line: 1, column: 12, scope: !7)
+!14 = !DILocation(line: 1, column: 18, scope: !7)
diff --git a/test/CodeGen/BPF/BTF/func-void.ll b/test/CodeGen/BPF/BTF/func-void.ll
new file mode 100644
index 0000000000000000000000000000000000000000..f86643b3de451a4dac5e8286e4d7e9ad7828f6d2
--- /dev/null
+++ b/test/CodeGen/BPF/BTF/func-void.ll
@@ -0,0 +1,74 @@
+; RUN: llc -march=bpfel -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -march=bpfeb -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+
+; Source code:
+;   void f1(void) {}
+; Compilation flag:
+;   clang -target bpf -O2 -g -S -emit-llvm t.c
+
+; Function Attrs: norecurse nounwind readnone
+define dso_local void @f1() local_unnamed_addr #0 !dbg !7 {
+  ret void, !dbg !10
+}
+
+; CHECK:             .section        .BTF,"",@progbits
+; CHECK-NEXT:        .short  60319                   # 0xeb9f
+; CHECK-NEXT:        .byte   1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .long   24
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   24
+; CHECK-NEXT:        .long   24
+; CHECK-NEXT:        .long   19
+; CHECK-NEXT:        .long   0                       # BTF_KIND_FUNC_PROTO(id = 1)
+; CHECK-NEXT:        .long   218103808               # 0xd000000
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   16                      # BTF_KIND_FUNC(id = 2)
+; CHECK-NEXT:        .long   201326592               # 0xc000000
+; CHECK-NEXT:        .long   1
+; CHECK-NEXT:        .byte   0                       # string offset=0
+; CHECK-NEXT:        .ascii  ".text"                 # string offset=1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .ascii  "/tmp/t.c"              # string offset=7
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .ascii  "f1"                    # string offset=16
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .section        .BTF.ext,"",@progbits
+; CHECK-NEXT:        .short  60319                   # 0xeb9f
+; CHECK-NEXT:        .byte   1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .long   24
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   20
+; CHECK-NEXT:        .long   20
+; CHECK-NEXT:        .long   28
+; CHECK-NEXT:        .long   8                       # FuncInfo
+; CHECK-NEXT:        .long   1                       # FuncInfo section string offset=1
+; CHECK-NEXT:        .long   1
+; CHECK-NEXT:        .long   .Lfunc_begin0
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   16                      # LineInfo
+; CHECK-NEXT:        .long   1                       # LineInfo section string offset=1
+; CHECK-NEXT:        .long   1
+; CHECK-NEXT:        .long   .Ltmp{{[0-9]+}}
+; CHECK-NEXT:        .long   7
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   1040                    # Line 1 Col 16
+
+attributes #0 = { norecurse nounwind readnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4, !5}
+!llvm.ident = !{!6}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 8.0.0 (trunk 345562) (llvm/trunk 345560)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, nameTableKind: None)
+!1 = !DIFile(filename: "t.c", directory: "/tmp")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{i32 1, !"wchar_size", i32 4}
+!6 = !{!"clang version 8.0.0 (trunk 345562) (llvm/trunk 345560)"}
+!7 = distinct !DISubprogram(name: "f1", scope: !1, file: !1, line: 1, type: !8, isLocal: false, isDefinition: true, scopeLine: 1, flags: DIFlagPrototyped, isOptimized: true, unit: !0, retainedNodes: !2)
+!8 = !DISubroutineType(types: !9)
+!9 = !{null}
+!10 = !DILocation(line: 1, column: 16, scope: !7)
diff --git a/test/CodeGen/BPF/BTF/fwd-no-define.ll b/test/CodeGen/BPF/BTF/fwd-no-define.ll
new file mode 100644
index 0000000000000000000000000000000000000000..c3d082ee53c27199442d3c4c1107402409531742
--- /dev/null
+++ b/test/CodeGen/BPF/BTF/fwd-no-define.ll
@@ -0,0 +1,73 @@
+; RUN: llc -march=bpfel -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -march=bpfeb -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+
+; Source code:
+;   struct t1;
+;   struct t2 {struct t1 *p;} a;
+; Compilation flag:
+;   clang -target bpf -O2 -g -S -emit-llvm t.c
+
+%struct.t2 = type { %struct.t1* }
+%struct.t1 = type opaque
+
+@a = common dso_local local_unnamed_addr global %struct.t2 zeroinitializer, align 8, !dbg !0
+
+!llvm.dbg.cu = !{!2}
+!llvm.module.flags = !{!11, !12, !13}
+!llvm.ident = !{!14}
+
+; CHECK:             .section        .BTF,"",@progbits
+; CHECK-NEXT:        .short  60319                   # 0xeb9f
+; CHECK-NEXT:        .byte   1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .long   24
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   48
+; CHECK-NEXT:        .long   48
+; CHECK-NEXT:        .long   9
+; CHECK-NEXT:        .long   1                       # BTF_KIND_STRUCT(id = 1)
+; CHECK-NEXT:        .long   67108865                # 0x4000001
+; CHECK-NEXT:        .long   8
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   0                       # BTF_KIND_PTR(id = 2)
+; CHECK-NEXT:        .long   33554432                # 0x2000000
+; CHECK-NEXT:        .long   3
+; CHECK-NEXT:        .long   6                       # BTF_KIND_FWD(id = 3)
+; CHECK-NEXT:        .long   117440512               # 0x7000000
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .byte   0                       # string offset=0
+; CHECK-NEXT:        .ascii  "t2"                    # string offset=1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .byte   112                     # string offset=4
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .ascii  "t1"                    # string offset=6
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .section        .BTF.ext,"",@progbits
+; CHECK-NEXT:        .short  60319                   # 0xeb9f
+; CHECK-NEXT:        .byte   1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .long   24
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   8                       # FuncInfo
+; CHECK-NEXT:        .long   16                      # LineInfo
+
+!0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression())
+!1 = distinct !DIGlobalVariable(name: "a", scope: !2, file: !3, line: 2, type: !6, isLocal: false, isDefinition: true)
+!2 = distinct !DICompileUnit(language: DW_LANG_C99, file: !3, producer: "clang version 8.0.0 (trunk 344789) (llvm/trunk 344782)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, globals: !5, nameTableKind: None)
+!3 = !DIFile(filename: "t.c", directory: "/home/yhs/tmp")
+!4 = !{}
+!5 = !{!0}
+!6 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "t2", file: !3, line: 2, size: 64, elements: !7)
+!7 = !{!8}
+!8 = !DIDerivedType(tag: DW_TAG_member, name: "p", scope: !6, file: !3, line: 2, baseType: !9, size: 64)
+!9 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !10, size: 64)
+!10 = !DICompositeType(tag: DW_TAG_structure_type, name: "t1", file: !3, line: 1, flags: DIFlagFwdDecl)
+!11 = !{i32 2, !"Dwarf Version", i32 4}
+!12 = !{i32 2, !"Debug Info Version", i32 3}
+!13 = !{i32 1, !"wchar_size", i32 4}
+!14 = !{!"clang version 8.0.0 (trunk 344789) (llvm/trunk 344782)"}
diff --git a/test/CodeGen/BPF/BTF/fwd-with-define.ll b/test/CodeGen/BPF/BTF/fwd-with-define.ll
new file mode 100644
index 0000000000000000000000000000000000000000..8a88c8956f40dad2efee6b551ac26e1378c315a0
--- /dev/null
+++ b/test/CodeGen/BPF/BTF/fwd-with-define.ll
@@ -0,0 +1,66 @@
+; RUN: llc -march=bpfel -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -march=bpfeb -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+
+; Source code:
+;   struct t1;
+;   struct t1 {struct t1 *p;} a;
+; Compilation flag:
+;   clang -target bpf -O2 -g -S -emit-llvm t.c
+
+%struct.t1 = type { %struct.t1* }
+
+@a = common dso_local local_unnamed_addr global %struct.t1 zeroinitializer, align 8, !dbg !0
+
+!llvm.dbg.cu = !{!2}
+!llvm.module.flags = !{!10, !11, !12}
+!llvm.ident = !{!13}
+
+; CHECK:             .section        .BTF,"",@progbits
+; CHECK-NEXT:        .short  60319                   # 0xeb9f
+; CHECK-NEXT:        .byte   1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .long   24
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   36
+; CHECK-NEXT:        .long   36
+; CHECK-NEXT:        .long   6
+; CHECK-NEXT:        .long   1                       # BTF_KIND_STRUCT(id = 1)
+; CHECK-NEXT:        .long   67108865                # 0x4000001
+; CHECK-NEXT:        .long   8
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   0                       # BTF_KIND_PTR(id = 2)
+; CHECK-NEXT:        .long   33554432                # 0x2000000
+; CHECK-NEXT:        .long   1
+; CHECK-NEXT:        .byte   0                       # string offset=0
+; CHECK-NEXT:        .ascii  "t1"                    # string offset=1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .byte   112                     # string offset=4
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .section        .BTF.ext,"",@progbits
+; CHECK-NEXT:        .short  60319                   # 0xeb9f
+; CHECK-NEXT:        .byte   1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .long   24
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   8                       # FuncInfo
+; CHECK-NEXT:        .long   16                      # LineInfo
+
+!0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression())
+!1 = distinct !DIGlobalVariable(name: "a", scope: !2, file: !3, line: 2, type: !6, isLocal: false, isDefinition: true)
+!2 = distinct !DICompileUnit(language: DW_LANG_C99, file: !3, producer: "clang version 8.0.0 (trunk 344789) (llvm/trunk 344782)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, globals: !5, nameTableKind: None)
+!3 = !DIFile(filename: "t.c", directory: "/home/yhs/tmp")
+!4 = !{}
+!5 = !{!0}
+!6 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "t1", file: !3, line: 2, size: 64, elements: !7)
+!7 = !{!8}
+!8 = !DIDerivedType(tag: DW_TAG_member, name: "p", scope: !6, file: !3, line: 2, baseType: !9, size: 64)
+!9 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !6, size: 64)
+!10 = !{i32 2, !"Dwarf Version", i32 4}
+!11 = !{i32 2, !"Debug Info Version", i32 3}
+!12 = !{i32 1, !"wchar_size", i32 4}
+!13 = !{!"clang version 8.0.0 (trunk 344789) (llvm/trunk 344782)"}
diff --git a/test/CodeGen/BPF/BTF/int.ll b/test/CodeGen/BPF/BTF/int.ll
new file mode 100644
index 0000000000000000000000000000000000000000..fd12cd0238a2cbee0381c95169a6af26b9704aa7
--- /dev/null
+++ b/test/CodeGen/BPF/BTF/int.ll
@@ -0,0 +1,53 @@
+; RUN: llc -march=bpfel -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -march=bpfeb -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+
+; Source code:
+;   int a;
+; Compilation flag:
+;   clang -target bpf -O2 -g -S -emit-llvm t.c
+
+@a = common dso_local local_unnamed_addr global i32 0, align 4, !dbg !0
+
+!llvm.dbg.cu = !{!2}
+!llvm.module.flags = !{!7, !8, !9}
+!llvm.ident = !{!10}
+
+; CHECK:             .section        .BTF,"",@progbits
+; CHECK-NEXT:        .short  60319                   # 0xeb9f
+; CHECK-NEXT:        .byte   1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .long   24
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   16
+; CHECK-NEXT:        .long   16
+; CHECK-NEXT:        .long   5
+; CHECK-NEXT:        .long   1                       # BTF_KIND_INT(id = 1)
+; CHECK-NEXT:        .long   16777216                # 0x1000000
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   16777248                # 0x1000020
+; CHECK-NEXT:        .byte   0                       # string offset=0
+; CHECK-NEXT:        .ascii  "int"                   # string offset=1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .section        .BTF.ext,"",@progbits
+; CHECK-NEXT:        .short  60319                   # 0xeb9f
+; CHECK-NEXT:        .byte   1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .long   24
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   8                       # FuncInfo
+; CHECK-NEXT:        .long   16                      # LineInfo
+
+!0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression())
+!1 = distinct !DIGlobalVariable(name: "a", scope: !2, file: !3, line: 1, type: !6, isLocal: false, isDefinition: true)
+!2 = distinct !DICompileUnit(language: DW_LANG_C99, file: !3, producer: "clang version 8.0.0 (trunk 344789) (llvm/trunk 344782)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, globals: !5, nameTableKind: None)
+!3 = !DIFile(filename: "t.c", directory: "/home/yhs/tmp")
+!4 = !{}
+!5 = !{!0}
+!6 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!7 = !{i32 2, !"Dwarf Version", i32 4}
+!8 = !{i32 2, !"Debug Info Version", i32 3}
+!9 = !{i32 1, !"wchar_size", i32 4}
+!10 = !{!"clang version 8.0.0 (trunk 344789) (llvm/trunk 344782)"}
diff --git a/test/CodeGen/BPF/BTF/longlong.ll b/test/CodeGen/BPF/BTF/longlong.ll
new file mode 100644
index 0000000000000000000000000000000000000000..f936e1eeafb7fdca18c97954beaf309e6793499c
--- /dev/null
+++ b/test/CodeGen/BPF/BTF/longlong.ll
@@ -0,0 +1,53 @@
+; RUN: llc -march=bpfel -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -march=bpfeb -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+
+; Source code:
+;   long long a;
+; Compilation flag:
+;   clang -target bpf -O2 -g -S -emit-llvm t.c
+
+@a = common dso_local local_unnamed_addr global i64 0, align 8, !dbg !0
+
+!llvm.dbg.cu = !{!2}
+!llvm.module.flags = !{!7, !8, !9}
+!llvm.ident = !{!10}
+
+; CHECK:             .section        .BTF,"",@progbits
+; CHECK-NEXT:        .short  60319                   # 0xeb9f
+; CHECK-NEXT:        .byte   1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .long   24
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   16
+; CHECK-NEXT:        .long   16
+; CHECK-NEXT:        .long   15
+; CHECK-NEXT:        .long   1                       # BTF_KIND_INT(id = 1)
+; CHECK-NEXT:        .long   16777216                # 0x1000000
+; CHECK-NEXT:        .long   8
+; CHECK-NEXT:        .long   16777280                # 0x1000040
+; CHECK-NEXT:        .byte   0                       # string offset=0
+; CHECK-NEXT:        .ascii  "long long int"         # string offset=1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .section        .BTF.ext,"",@progbits
+; CHECK-NEXT:        .short  60319                   # 0xeb9f
+; CHECK-NEXT:        .byte   1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .long   24
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   8                       # FuncInfo
+; CHECK-NEXT:        .long   16                      # LineInfo
+
+!0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression())
+!1 = distinct !DIGlobalVariable(name: "a", scope: !2, file: !3, line: 1, type: !6, isLocal: false, isDefinition: true)
+!2 = distinct !DICompileUnit(language: DW_LANG_C99, file: !3, producer: "clang version 8.0.0 (trunk 344789) (llvm/trunk 344782)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, globals: !5, nameTableKind: None)
+!3 = !DIFile(filename: "t.c", directory: "/home/yhs/tmp")
+!4 = !{}
+!5 = !{!0}
+!6 = !DIBasicType(name: "long long int", size: 64, encoding: DW_ATE_signed)
+!7 = !{i32 2, !"Dwarf Version", i32 4}
+!8 = !{i32 2, !"Debug Info Version", i32 3}
+!9 = !{i32 1, !"wchar_size", i32 4}
+!10 = !{!"clang version 8.0.0 (trunk 344789) (llvm/trunk 344782)"}
diff --git a/test/CodeGen/BPF/BTF/ptr-const-void.ll b/test/CodeGen/BPF/BTF/ptr-const-void.ll
new file mode 100644
index 0000000000000000000000000000000000000000..ccc7b8dc30241c6f6048dae298ea7d7b6003704c
--- /dev/null
+++ b/test/CodeGen/BPF/BTF/ptr-const-void.ll
@@ -0,0 +1,54 @@
+; RUN: llc -march=bpfel -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -march=bpfeb -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+
+; Source code:
+;   const void *a;
+; Compilation flag:
+;   clang -target bpf -O2 -g -S -emit-llvm t.c
+
+@p = common dso_local local_unnamed_addr global i8* null, align 8, !dbg !0
+
+!llvm.dbg.cu = !{!2}
+!llvm.module.flags = !{!8, !9, !10}
+!llvm.ident = !{!11}
+
+; CHECK:             .section        .BTF,"",@progbits
+; CHECK-NEXT:        .short  60319                   # 0xeb9f
+; CHECK-NEXT:        .byte   1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .long   24
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   24
+; CHECK-NEXT:        .long   24
+; CHECK-NEXT:        .long   1
+; CHECK-NEXT:        .long   0                       # BTF_KIND_PTR(id = 1)
+; CHECK-NEXT:        .long   33554432                # 0x2000000
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   0                       # BTF_KIND_CONST(id = 2)
+; CHECK-NEXT:        .long   167772160               # 0xa000000
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .byte   0                       # string offset=0
+; CHECK-NEXT:        .section        .BTF.ext,"",@progbits
+; CHECK-NEXT:        .short  60319                   # 0xeb9f
+; CHECK-NEXT:        .byte   1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .long   24
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   8                       # FuncInfo
+; CHECK-NEXT:        .long   16                      # LineInfo
+
+!0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression())
+!1 = distinct !DIGlobalVariable(name: "p", scope: !2, file: !3, line: 1, type: !6, isLocal: false, isDefinition: true)
+!2 = distinct !DICompileUnit(language: DW_LANG_C99, file: !3, producer: "clang version 8.0.0 (trunk 344789) (llvm/trunk 344782)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, globals: !5, nameTableKind: None)
+!3 = !DIFile(filename: "t.c", directory: "/home/yhs/tmp")
+!4 = !{}
+!5 = !{!0}
+!6 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !7, size: 64)
+!7 = !DIDerivedType(tag: DW_TAG_const_type, baseType: null)
+!8 = !{i32 2, !"Dwarf Version", i32 4}
+!9 = !{i32 2, !"Debug Info Version", i32 3}
+!10 = !{i32 1, !"wchar_size", i32 4}
+!11 = !{!"clang version 8.0.0 (trunk 344789) (llvm/trunk 344782)"}
diff --git a/test/CodeGen/BPF/BTF/ptr-func-1.ll b/test/CodeGen/BPF/BTF/ptr-func-1.ll
new file mode 100644
index 0000000000000000000000000000000000000000..e80158b242a03f6c277d7f45446d4d5c4d778895
--- /dev/null
+++ b/test/CodeGen/BPF/BTF/ptr-func-1.ll
@@ -0,0 +1,55 @@
+; RUN: llc -march=bpfel -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -march=bpfeb -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+
+; Source code:
+;   void (*a)(void);
+; Compilation flag:
+;   clang -target bpf -O2 -g -S -emit-llvm t.c
+
+@a = common dso_local local_unnamed_addr global void ()* null, align 8, !dbg !0
+
+!llvm.dbg.cu = !{!2}
+!llvm.module.flags = !{!9, !10, !11}
+!llvm.ident = !{!12}
+
+; CHECK:             .section        .BTF,"",@progbits
+; CHECK-NEXT:        .short  60319                   # 0xeb9f
+; CHECK-NEXT:        .byte   1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .long   24
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   24
+; CHECK-NEXT:        .long   24
+; CHECK-NEXT:        .long   1
+; CHECK-NEXT:        .long   0                       # BTF_KIND_PTR(id = 1)
+; CHECK-NEXT:        .long   33554432                # 0x2000000
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   0                       # BTF_KIND_FUNC_PROTO(id = 2)
+; CHECK-NEXT:        .long   218103808               # 0xd000000
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .byte   0                       # string offset=0
+; CHECK-NEXT:        .section        .BTF.ext,"",@progbits
+; CHECK-NEXT:        .short  60319                   # 0xeb9f
+; CHECK-NEXT:        .byte   1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .long   24
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   8                       # FuncInfo
+; CHECK-NEXT:        .long   16                      # LineInfo
+
+!0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression())
+!1 = distinct !DIGlobalVariable(name: "a", scope: !2, file: !3, line: 1, type: !6, isLocal: false, isDefinition: true)
+!2 = distinct !DICompileUnit(language: DW_LANG_C99, file: !3, producer: "clang version 8.0.0 (trunk 344789) (llvm/trunk 344782)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, globals: !5, nameTableKind: None)
+!3 = !DIFile(filename: "t.c", directory: "/home/yhs/tmp")
+!4 = !{}
+!5 = !{!0}
+!6 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !7, size: 64)
+!7 = !DISubroutineType(types: !8)
+!8 = !{null}
+!9 = !{i32 2, !"Dwarf Version", i32 4}
+!10 = !{i32 2, !"Debug Info Version", i32 3}
+!11 = !{i32 1, !"wchar_size", i32 4}
+!12 = !{!"clang version 8.0.0 (trunk 344789) (llvm/trunk 344782)"}
diff --git a/test/CodeGen/BPF/BTF/ptr-func-2.ll b/test/CodeGen/BPF/BTF/ptr-func-2.ll
new file mode 100644
index 0000000000000000000000000000000000000000..297212109052a7ac68125f277e0af4fb3ff4448a
--- /dev/null
+++ b/test/CodeGen/BPF/BTF/ptr-func-2.ll
@@ -0,0 +1,73 @@
+; RUN: llc -march=bpfel -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -march=bpfeb -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+
+; Source code:
+;   int (*a)(int a, char b);
+; Compilation flag:
+;   clang -target bpf -O2 -g -S -emit-llvm t.c
+
+@a = common dso_local local_unnamed_addr global i32 (i32, i8)* null, align 8, !dbg !0
+
+!llvm.dbg.cu = !{!2}
+!llvm.module.flags = !{!11, !12, !13}
+!llvm.ident = !{!14}
+
+; CHECK:             .section        .BTF,"",@progbits
+; CHECK-NEXT:        .short  60319                   # 0xeb9f
+; CHECK-NEXT:        .byte   1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .long   24
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   72
+; CHECK-NEXT:        .long   72
+; CHECK-NEXT:        .long   10
+; CHECK-NEXT:        .long   0                       # BTF_KIND_PTR(id = 1)
+; CHECK-NEXT:        .long   33554432                # 0x2000000
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   0                       # BTF_KIND_FUNC_PROTO(id = 2)
+; CHECK-NEXT:        .long   218103810               # 0xd000002
+; CHECK-NEXT:        .long   3
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   3
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   1                       # BTF_KIND_INT(id = 3)
+; CHECK-NEXT:        .long   16777216                # 0x1000000
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   16777248                # 0x1000020
+; CHECK-NEXT:        .long   5                       # BTF_KIND_INT(id = 4)
+; CHECK-NEXT:        .long   16777216                # 0x1000000
+; CHECK-NEXT:        .long   1
+; CHECK-NEXT:        .long   16777224                # 0x1000008
+; CHECK-NEXT:        .byte   0                       # string offset=0
+; CHECK-NEXT:        .ascii  "int"                   # string offset=1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .ascii  "char"                  # string offset=5
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .section        .BTF.ext,"",@progbits
+; CHECK-NEXT:        .short  60319                   # 0xeb9f
+; CHECK-NEXT:        .byte   1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .long   24
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   8                       # FuncInfo
+; CHECK-NEXT:        .long   16                      # LineInfo
+
+!0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression())
+!1 = distinct !DIGlobalVariable(name: "a", scope: !2, file: !3, line: 1, type: !6, isLocal: false, isDefinition: true)
+!2 = distinct !DICompileUnit(language: DW_LANG_C99, file: !3, producer: "clang version 8.0.0 (trunk 344789) (llvm/trunk 344782)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, globals: !5, nameTableKind: None)
+!3 = !DIFile(filename: "t.c", directory: "/home/yhs/tmp")
+!4 = !{}
+!5 = !{!0}
+!6 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !7, size: 64)
+!7 = !DISubroutineType(types: !8)
+!8 = !{!9, !9, !10}
+!9 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!10 = !DIBasicType(name: "char", size: 8, encoding: DW_ATE_signed_char)
+!11 = !{i32 2, !"Dwarf Version", i32 4}
+!12 = !{i32 2, !"Debug Info Version", i32 3}
+!13 = !{i32 1, !"wchar_size", i32 4}
+!14 = !{!"clang version 8.0.0 (trunk 344789) (llvm/trunk 344782)"}
diff --git a/test/CodeGen/BPF/BTF/ptr-func-3.ll b/test/CodeGen/BPF/BTF/ptr-func-3.ll
new file mode 100644
index 0000000000000000000000000000000000000000..85a15b3dbf273aab8bb3e7917b761bac65a99f1e
--- /dev/null
+++ b/test/CodeGen/BPF/BTF/ptr-func-3.ll
@@ -0,0 +1,73 @@
+; RUN: llc -march=bpfel -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -march=bpfeb -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+
+; Source code:
+;   typedef int __int;
+;   __int (*a)(__int a, __int b);
+; Compilation flag:
+;   clang -target bpf -O2 -g -S -emit-llvm t.c
+
+@a = common dso_local local_unnamed_addr global i32 (i32, i32)* null, align 8, !dbg !0
+
+!llvm.dbg.cu = !{!2}
+!llvm.module.flags = !{!11, !12, !13}
+!llvm.ident = !{!14}
+
+; CHECK:             .section        .BTF,"",@progbits
+; CHECK-NEXT:        .short  60319                   # 0xeb9f
+; CHECK-NEXT:        .byte   1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .long   24
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   68
+; CHECK-NEXT:        .long   68
+; CHECK-NEXT:        .long   11
+; CHECK-NEXT:        .long   0                       # BTF_KIND_PTR(id = 1)
+; CHECK-NEXT:        .long   33554432                # 0x2000000
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   0                       # BTF_KIND_FUNC_PROTO(id = 2)
+; CHECK-NEXT:        .long   218103810               # 0xd000002
+; CHECK-NEXT:        .long   3
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   3
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   3
+; CHECK-NEXT:        .long   1                       # BTF_KIND_TYPEDEF(id = 3)
+; CHECK-NEXT:        .long   134217728               # 0x8000000
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   7                       # BTF_KIND_INT(id = 4)
+; CHECK-NEXT:        .long   16777216                # 0x1000000
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   16777248                # 0x1000020
+; CHECK-NEXT:        .byte   0                       # string offset=0
+; CHECK-NEXT:        .ascii  "__int"                 # string offset=1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .ascii  "int"                   # string offset=7
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .section        .BTF.ext,"",@progbits
+; CHECK-NEXT:        .short  60319                   # 0xeb9f
+; CHECK-NEXT:        .byte   1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .long   24
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   8                       # FuncInfo
+; CHECK-NEXT:        .long   16                      # LineInfo
+
+!0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression())
+!1 = distinct !DIGlobalVariable(name: "a", scope: !2, file: !3, line: 2, type: !6, isLocal: false, isDefinition: true)
+!2 = distinct !DICompileUnit(language: DW_LANG_C99, file: !3, producer: "clang version 8.0.0 (trunk 344789) (llvm/trunk 344782)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, globals: !5, nameTableKind: None)
+!3 = !DIFile(filename: "t.c", directory: "/home/yhs/tmp")
+!4 = !{}
+!5 = !{!0}
+!6 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !7, size: 64)
+!7 = !DISubroutineType(types: !8)
+!8 = !{!9, !9, !9}
+!9 = !DIDerivedType(tag: DW_TAG_typedef, name: "__int", file: !3, line: 1, baseType: !10)
+!10 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!11 = !{i32 2, !"Dwarf Version", i32 4}
+!12 = !{i32 2, !"Debug Info Version", i32 3}
+!13 = !{i32 1, !"wchar_size", i32 4}
+!14 = !{!"clang version 8.0.0 (trunk 344789) (llvm/trunk 344782)"}
diff --git a/test/CodeGen/BPF/BTF/ptr-int.ll b/test/CodeGen/BPF/BTF/ptr-int.ll
new file mode 100644
index 0000000000000000000000000000000000000000..1203f79879e5cddfdc08727a7e7a2474e976ec21
--- /dev/null
+++ b/test/CodeGen/BPF/BTF/ptr-int.ll
@@ -0,0 +1,58 @@
+; RUN: llc -march=bpfel -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -march=bpfeb -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+
+; Source code:
+;   int *a;
+; Compilation flag:
+;   clang -target bpf -O2 -g -S -emit-llvm t.c
+
+@a = common dso_local local_unnamed_addr global i32* null, align 8, !dbg !0
+
+!llvm.dbg.cu = !{!2}
+!llvm.module.flags = !{!8, !9, !10}
+!llvm.ident = !{!11}
+
+; CHECK:             .section        .BTF,"",@progbits
+; CHECK-NEXT:        .short  60319                   # 0xeb9f
+; CHECK-NEXT:        .byte   1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .long   24
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   28
+; CHECK-NEXT:        .long   28
+; CHECK-NEXT:        .long   5
+; CHECK-NEXT:        .long   0                       # BTF_KIND_PTR(id = 1)
+; CHECK-NEXT:        .long   33554432                # 0x2000000
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   1                       # BTF_KIND_INT(id = 2)
+; CHECK-NEXT:        .long   16777216                # 0x1000000
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   16777248                # 0x1000020
+; CHECK-NEXT:        .byte   0                       # string offset=0
+; CHECK-NEXT:        .ascii  "int"                   # string offset=1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .section        .BTF.ext,"",@progbits
+; CHECK-NEXT:        .short  60319                   # 0xeb9f
+; CHECK-NEXT:        .byte   1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .long   24
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   8                       # FuncInfo
+; CHECK-NEXT:        .long   16                      # LineInfo
+
+
+!0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression())
+!1 = distinct !DIGlobalVariable(name: "a", scope: !2, file: !3, line: 1, type: !6, isLocal: false, isDefinition: true)
+!2 = distinct !DICompileUnit(language: DW_LANG_C99, file: !3, producer: "clang version 8.0.0 (trunk 344789) (llvm/trunk 344782)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, globals: !5, nameTableKind: None)
+!3 = !DIFile(filename: "t.c", directory: "/home/yhs/tmp")
+!4 = !{}
+!5 = !{!0}
+!6 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !7, size: 64)
+!7 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!8 = !{i32 2, !"Dwarf Version", i32 4}
+!9 = !{i32 2, !"Debug Info Version", i32 3}
+!10 = !{i32 1, !"wchar_size", i32 4}
+!11 = !{!"clang version 8.0.0 (trunk 344789) (llvm/trunk 344782)"}
diff --git a/test/CodeGen/BPF/BTF/ptr-void.ll b/test/CodeGen/BPF/BTF/ptr-void.ll
new file mode 100644
index 0000000000000000000000000000000000000000..e83e13073db281dfb05f94ee5c54ac5c95ce6e4d
--- /dev/null
+++ b/test/CodeGen/BPF/BTF/ptr-void.ll
@@ -0,0 +1,50 @@
+; RUN: llc -march=bpfel -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -march=bpfeb -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+
+; Source code:
+;   void *a;
+; Compilation flag:
+;   clang -target bpf -O2 -g -S -emit-llvm t.c
+
+@a = common dso_local local_unnamed_addr global i8* null, align 8, !dbg !0
+
+!llvm.dbg.cu = !{!2}
+!llvm.module.flags = !{!7, !8, !9}
+!llvm.ident = !{!10}
+
+; CHECK:             .section        .BTF,"",@progbits
+; CHECK-NEXT:        .short  60319                   # 0xeb9f
+; CHECK-NEXT:        .byte   1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .long   24
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   12
+; CHECK-NEXT:        .long   12
+; CHECK-NEXT:        .long   1
+; CHECK-NEXT:        .long   0                       # BTF_KIND_PTR(id = 1)
+; CHECK-NEXT:        .long   33554432                # 0x2000000
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .byte   0                       # string offset=0
+; CHECK-NEXT:        .section        .BTF.ext,"",@progbits
+; CHECK-NEXT:        .short  60319                   # 0xeb9f
+; CHECK-NEXT:        .byte   1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .long   24
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   8                       # FuncInfo
+; CHECK-NEXT:        .long   16                      # LineInfo
+
+!0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression())
+!1 = distinct !DIGlobalVariable(name: "a", scope: !2, file: !3, line: 1, type: !6, isLocal: false, isDefinition: true)
+!2 = distinct !DICompileUnit(language: DW_LANG_C99, file: !3, producer: "clang version 8.0.0 (trunk 345296) (llvm/trunk 345297)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, globals: !5, nameTableKind: None)
+!3 = !DIFile(filename: "t.c", directory: "/home/yhs/tmp")
+!4 = !{}
+!5 = !{!0}
+!6 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: null, size: 64)
+!7 = !{i32 2, !"Dwarf Version", i32 4}
+!8 = !{i32 2, !"Debug Info Version", i32 3}
+!9 = !{i32 1, !"wchar_size", i32 4}
+!10 = !{!"clang version 8.0.0 (trunk 345296) (llvm/trunk 345297)"}
diff --git a/test/CodeGen/BPF/BTF/ptr-volatile-const-void.ll b/test/CodeGen/BPF/BTF/ptr-volatile-const-void.ll
new file mode 100644
index 0000000000000000000000000000000000000000..3d5a2273079bad636979f6a906692e410f914f71
--- /dev/null
+++ b/test/CodeGen/BPF/BTF/ptr-volatile-const-void.ll
@@ -0,0 +1,58 @@
+; RUN: llc -march=bpfel -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -march=bpfeb -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+
+; Source code:
+;   volatile const void *p;
+; Compilation flag:
+;   clang -target bpf -O2 -g -S -emit-llvm t.c
+
+@p = common dso_local local_unnamed_addr global i8* null, align 8, !dbg !0
+
+!llvm.dbg.cu = !{!2}
+!llvm.module.flags = !{!9, !10, !11}
+!llvm.ident = !{!12}
+
+; CHECK:             .section        .BTF,"",@progbits
+; CHECK-NEXT:        .short  60319                   # 0xeb9f
+; CHECK-NEXT:        .byte   1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .long   24
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   36
+; CHECK-NEXT:        .long   36
+; CHECK-NEXT:        .long   1
+; CHECK-NEXT:        .long   0                       # BTF_KIND_PTR(id = 1)
+; CHECK-NEXT:        .long   33554432                # 0x2000000
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   0                       # BTF_KIND_CONST(id = 2)
+; CHECK-NEXT:        .long   167772160               # 0xa000000
+; CHECK-NEXT:        .long   3
+; CHECK-NEXT:        .long   0                       # BTF_KIND_VOLATILE(id = 3)
+; CHECK-NEXT:        .long   150994944               # 0x9000000
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .byte   0                       # string offset=0
+; CHECK-NEXT:        .section        .BTF.ext,"",@progbits
+; CHECK-NEXT:        .short  60319                   # 0xeb9f
+; CHECK-NEXT:        .byte   1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .long   24
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   8                       # FuncInfo
+; CHECK-NEXT:        .long   16                      # LineInfo
+
+!0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression())
+!1 = distinct !DIGlobalVariable(name: "p", scope: !2, file: !3, line: 1, type: !6, isLocal: false, isDefinition: true)
+!2 = distinct !DICompileUnit(language: DW_LANG_C99, file: !3, producer: "clang version 8.0.0 (trunk 344789) (llvm/trunk 344782)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, globals: !5, nameTableKind: None)
+!3 = !DIFile(filename: "t.c", directory: "/home/yhs/tmp")
+!4 = !{}
+!5 = !{!0}
+!6 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !7, size: 64)
+!7 = !DIDerivedType(tag: DW_TAG_const_type, baseType: !8)
+!8 = !DIDerivedType(tag: DW_TAG_volatile_type, baseType: null)
+!9 = !{i32 2, !"Dwarf Version", i32 4}
+!10 = !{i32 2, !"Debug Info Version", i32 3}
+!11 = !{i32 1, !"wchar_size", i32 4}
+!12 = !{!"clang version 8.0.0 (trunk 344789) (llvm/trunk 344782)"}
diff --git a/test/CodeGen/BPF/BTF/ptr-volatile-void.ll b/test/CodeGen/BPF/BTF/ptr-volatile-void.ll
new file mode 100644
index 0000000000000000000000000000000000000000..185eee043478c595c33175b5abad14539afa0804
--- /dev/null
+++ b/test/CodeGen/BPF/BTF/ptr-volatile-void.ll
@@ -0,0 +1,54 @@
+; RUN: llc -march=bpfel -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -march=bpfeb -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+
+; Source code:
+;   volatile void *a;
+; Compilation flag:
+;   clang -target bpf -O2 -g -S -emit-llvm t.c
+
+@p = common dso_local local_unnamed_addr global i8* null, align 8, !dbg !0
+
+!llvm.dbg.cu = !{!2}
+!llvm.module.flags = !{!8, !9, !10}
+!llvm.ident = !{!11}
+
+; CHECK:             .section        .BTF,"",@progbits
+; CHECK-NEXT:        .short  60319                   # 0xeb9f
+; CHECK-NEXT:        .byte   1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .long   24
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   24
+; CHECK-NEXT:        .long   24
+; CHECK-NEXT:        .long   1
+; CHECK-NEXT:        .long   0                       # BTF_KIND_PTR(id = 1)
+; CHECK-NEXT:        .long   33554432                # 0x2000000
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   0                       # BTF_KIND_VOLATILE(id = 2)
+; CHECK-NEXT:        .long   150994944               # 0x9000000
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .byte   0                       # string offset=0
+; CHECK-NEXT:        .section        .BTF.ext,"",@progbits
+; CHECK-NEXT:        .short  60319                   # 0xeb9f
+; CHECK-NEXT:        .byte   1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .long   24
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   8                       # FuncInfo
+; CHECK-NEXT:        .long   16                      # LineInfo
+
+!0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression())
+!1 = distinct !DIGlobalVariable(name: "p", scope: !2, file: !3, line: 1, type: !6, isLocal: false, isDefinition: true)
+!2 = distinct !DICompileUnit(language: DW_LANG_C99, file: !3, producer: "clang version 8.0.0 (trunk 344789) (llvm/trunk 344782)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, globals: !5, nameTableKind: None)
+!3 = !DIFile(filename: "t.c", directory: "/home/yhs/tmp")
+!4 = !{}
+!5 = !{!0}
+!6 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !7, size: 64)
+!7 = !DIDerivedType(tag: DW_TAG_volatile_type, baseType: null)
+!8 = !{i32 2, !"Dwarf Version", i32 4}
+!9 = !{i32 2, !"Debug Info Version", i32 3}
+!10 = !{i32 1, !"wchar_size", i32 4}
+!11 = !{!"clang version 8.0.0 (trunk 344789) (llvm/trunk 344782)"}
diff --git a/test/CodeGen/BPF/BTF/restrict-ptr.ll b/test/CodeGen/BPF/BTF/restrict-ptr.ll
new file mode 100644
index 0000000000000000000000000000000000000000..97cbbe4ddc062d3baca87f55be36932080936eff
--- /dev/null
+++ b/test/CodeGen/BPF/BTF/restrict-ptr.ll
@@ -0,0 +1,61 @@
+; RUN: llc -march=bpfel -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -march=bpfeb -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+
+; Source code:
+;   int * restrict p;
+; Compilation flag:
+;   clang -target bpf -O2 -g -S -emit-llvm t.c
+
+@p = common dso_local local_unnamed_addr global i32* null, align 8, !dbg !0
+
+!llvm.dbg.cu = !{!2}
+!llvm.module.flags = !{!9, !10, !11}
+!llvm.ident = !{!12}
+
+; CHECK:             .section        .BTF,"",@progbits
+; CHECK-NEXT:        .short  60319                   # 0xeb9f
+; CHECK-NEXT:        .byte   1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .long   24
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   40
+; CHECK-NEXT:        .long   40
+; CHECK-NEXT:        .long   5
+; CHECK-NEXT:        .long   0                       # BTF_KIND_RESTRICT(id = 1)
+; CHECK-NEXT:        .long   184549376               # 0xb000000
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   0                       # BTF_KIND_PTR(id = 2)
+; CHECK-NEXT:        .long   33554432                # 0x2000000
+; CHECK-NEXT:        .long   3
+; CHECK-NEXT:        .long   1                       # BTF_KIND_INT(id = 3)
+; CHECK-NEXT:        .long   16777216                # 0x1000000
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   16777248                # 0x1000020
+; CHECK-NEXT:        .byte   0                       # string offset=0
+; CHECK-NEXT:        .ascii  "int"                   # string offset=1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .section        .BTF.ext,"",@progbits
+; CHECK-NEXT:        .short  60319                   # 0xeb9f
+; CHECK-NEXT:        .byte   1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .long   24
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   8                       # FuncInfo
+; CHECK-NEXT:        .long   16                      # LineInfo
+
+!0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression())
+!1 = distinct !DIGlobalVariable(name: "p", scope: !2, file: !3, line: 1, type: !6, isLocal: false, isDefinition: true)
+!2 = distinct !DICompileUnit(language: DW_LANG_C99, file: !3, producer: "clang version 8.0.0 (trunk 344789) (llvm/trunk 344782)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, globals: !5, nameTableKind: None)
+!3 = !DIFile(filename: "t.c", directory: "/home/yhs/tmp")
+!4 = !{}
+!5 = !{!0}
+!6 = !DIDerivedType(tag: DW_TAG_restrict_type, baseType: !7)
+!7 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !8, size: 64)
+!8 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!9 = !{i32 2, !"Dwarf Version", i32 4}
+!10 = !{i32 2, !"Debug Info Version", i32 3}
+!11 = !{i32 1, !"wchar_size", i32 4}
+!12 = !{!"clang version 8.0.0 (trunk 344789) (llvm/trunk 344782)"}
diff --git a/test/CodeGen/BPF/BTF/short.ll b/test/CodeGen/BPF/BTF/short.ll
new file mode 100644
index 0000000000000000000000000000000000000000..56b779c890a1ea00d99dc12cb2b212410bd4316c
--- /dev/null
+++ b/test/CodeGen/BPF/BTF/short.ll
@@ -0,0 +1,54 @@
+; RUN: llc -march=bpfel -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -march=bpfeb -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+
+; Source code:
+;   short a;
+; Compilation flag:
+;   clang -target bpf -O2 -g -S -emit-llvm t.c
+
+
+@a = common dso_local local_unnamed_addr global i16 0, align 2, !dbg !0
+
+!llvm.dbg.cu = !{!2}
+!llvm.module.flags = !{!7, !8, !9}
+!llvm.ident = !{!10}
+
+; CHECK:             .section        .BTF,"",@progbits
+; CHECK-NEXT:        .short  60319                   # 0xeb9f
+; CHECK-NEXT:        .byte   1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .long   24
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   16
+; CHECK-NEXT:        .long   16
+; CHECK-NEXT:        .long   7
+; CHECK-NEXT:        .long   1                       # BTF_KIND_INT(id = 1)
+; CHECK-NEXT:        .long   16777216                # 0x1000000
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   16777232                # 0x1000010
+; CHECK-NEXT:        .byte   0                       # string offset=0
+; CHECK-NEXT:        .ascii  "short"                 # string offset=1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .section        .BTF.ext,"",@progbits
+; CHECK-NEXT:        .short  60319                   # 0xeb9f
+; CHECK-NEXT:        .byte   1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .long   24
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   8                       # FuncInfo
+; CHECK-NEXT:        .long   16                      # LineInfo
+
+!0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression())
+!1 = distinct !DIGlobalVariable(name: "a", scope: !2, file: !3, line: 1, type: !6, isLocal: false, isDefinition: true)
+!2 = distinct !DICompileUnit(language: DW_LANG_C99, file: !3, producer: "clang version 8.0.0 (trunk 344789) (llvm/trunk 344782)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, globals: !5, nameTableKind: None)
+!3 = !DIFile(filename: "t.c", directory: "/home/yhs/tmp")
+!4 = !{}
+!5 = !{!0}
+!6 = !DIBasicType(name: "short", size: 16, encoding: DW_ATE_signed)
+!7 = !{i32 2, !"Dwarf Version", i32 4}
+!8 = !{i32 2, !"Debug Info Version", i32 3}
+!9 = !{i32 1, !"wchar_size", i32 4}
+!10 = !{!"clang version 8.0.0 (trunk 344789) (llvm/trunk 344782)"}
diff --git a/test/CodeGen/BPF/BTF/struct-anon.ll b/test/CodeGen/BPF/BTF/struct-anon.ll
new file mode 100644
index 0000000000000000000000000000000000000000..3be1752cc379d9dee855dc10cbe07fae1a38f2f7
--- /dev/null
+++ b/test/CodeGen/BPF/BTF/struct-anon.ll
@@ -0,0 +1,76 @@
+; RUN: llc -march=bpfel -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -march=bpfeb -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+
+; Source code:
+;   struct { struct {int m;}; } a;
+; Compilation flag:
+;   clang -target bpf -O2 -g -S -emit-llvm t.c
+
+%struct.anon = type { %struct.anon.0 }
+%struct.anon.0 = type { i32 }
+
+@a = common dso_local local_unnamed_addr global %struct.anon zeroinitializer, align 4, !dbg !0
+
+!llvm.dbg.cu = !{!2}
+!llvm.module.flags = !{!13, !14, !15}
+!llvm.ident = !{!16}
+
+; CHECK:             .section        .BTF,"",@progbits
+; CHECK-NEXT:        .short  60319                   # 0xeb9f
+; CHECK-NEXT:        .byte   1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .long   24
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   64
+; CHECK-NEXT:        .long   64
+; CHECK-NEXT:        .long   7
+; CHECK-NEXT:        .long   0                       # BTF_KIND_STRUCT(id = 1)
+; CHECK-NEXT:        .long   67108865                # 0x4000001
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   0                       # BTF_KIND_STRUCT(id = 2)
+; CHECK-NEXT:        .long   67108865                # 0x4000001
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   1
+; CHECK-NEXT:        .long   3
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   3                       # BTF_KIND_INT(id = 3)
+; CHECK-NEXT:        .long   16777216                # 0x1000000
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   16777248                # 0x1000020
+; CHECK-NEXT:        .byte   0                       # string offset=0
+; CHECK-NEXT:        .byte   109                     # string offset=1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .ascii  "int"                   # string offset=3
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .section        .BTF.ext,"",@progbits
+; CHECK-NEXT:        .short  60319                   # 0xeb9f
+; CHECK-NEXT:        .byte   1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .long   24
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   8                       # FuncInfo
+; CHECK-NEXT:        .long   16                      # LineInfo
+
+!0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression())
+!1 = distinct !DIGlobalVariable(name: "a", scope: !2, file: !3, line: 1, type: !6, isLocal: false, isDefinition: true)
+!2 = distinct !DICompileUnit(language: DW_LANG_C99, file: !3, producer: "clang version 8.0.0 (trunk 344789) (llvm/trunk 344782)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, globals: !5, nameTableKind: None)
+!3 = !DIFile(filename: "t.c", directory: "/home/yhs/tmp")
+!4 = !{}
+!5 = !{!0}
+!6 = distinct !DICompositeType(tag: DW_TAG_structure_type, file: !3, line: 1, size: 32, elements: !7)
+!7 = !{!8}
+!8 = !DIDerivedType(tag: DW_TAG_member, scope: !6, file: !3, line: 1, baseType: !9, size: 32)
+!9 = distinct !DICompositeType(tag: DW_TAG_structure_type, scope: !6, file: !3, line: 1, size: 32, elements: !10)
+!10 = !{!11}
+!11 = !DIDerivedType(tag: DW_TAG_member, name: "m", scope: !9, file: !3, line: 1, baseType: !12, size: 32)
+!12 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!13 = !{i32 2, !"Dwarf Version", i32 4}
+!14 = !{i32 2, !"Debug Info Version", i32 3}
+!15 = !{i32 1, !"wchar_size", i32 4}
+!16 = !{!"clang version 8.0.0 (trunk 344789) (llvm/trunk 344782)"}
diff --git a/test/CodeGen/BPF/BTF/struct-basic.ll b/test/CodeGen/BPF/BTF/struct-basic.ll
new file mode 100644
index 0000000000000000000000000000000000000000..69dc0ca35020c88aafb2b33186f629ae4309dfd9
--- /dev/null
+++ b/test/CodeGen/BPF/BTF/struct-basic.ll
@@ -0,0 +1,81 @@
+; RUN: llc -march=bpfel -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -march=bpfeb -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+
+; Source code:
+;   struct t1 {char m1; int n1;} a;
+; Compilation flag:
+;   clang -target bpf -O2 -g -S -emit-llvm t.c
+
+%struct.t1 = type { i8, i32 }
+
+@a = common dso_local local_unnamed_addr global %struct.t1 zeroinitializer, align 4, !dbg !0
+
+!llvm.dbg.cu = !{!2}
+!llvm.module.flags = !{!12, !13, !14}
+!llvm.ident = !{!15}
+
+; CHECK:             .section        .BTF,"",@progbits
+; CHECK-NEXT:        .short  60319                   # 0xeb9f
+; CHECK-NEXT:        .byte   1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .long   24
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   68
+; CHECK-NEXT:        .long   68
+; CHECK-NEXT:        .long   19
+; CHECK-NEXT:        .long   1                       # BTF_KIND_STRUCT(id = 1)
+; CHECK-NEXT:        .long   67108866                # 0x4000002
+; CHECK-NEXT:        .long   8
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   7
+; CHECK-NEXT:        .long   3
+; CHECK-NEXT:        .long   32
+; CHECK-NEXT:        .long   10                      # BTF_KIND_INT(id = 2)
+; CHECK-NEXT:        .long   16777216                # 0x1000000
+; CHECK-NEXT:        .long   1
+; CHECK-NEXT:        .long   16777224                # 0x1000008
+; CHECK-NEXT:        .long   15                      # BTF_KIND_INT(id = 3)
+; CHECK-NEXT:        .long   16777216                # 0x1000000
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   16777248                # 0x1000020
+; CHECK-NEXT:        .byte   0                       # string offset=0
+; CHECK-NEXT:        .ascii  "t1"                    # string offset=1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .ascii  "m1"                    # string offset=4
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .ascii  "n1"                    # string offset=7
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .ascii  "char"                  # string offset=10
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .ascii  "int"                   # string offset=15
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .section        .BTF.ext,"",@progbits
+; CHECK-NEXT:        .short  60319                   # 0xeb9f
+; CHECK-NEXT:        .byte   1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .long   24
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   8                       # FuncInfo
+; CHECK-NEXT:        .long   16                      # LineInfo
+
+!0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression())
+!1 = distinct !DIGlobalVariable(name: "a", scope: !2, file: !3, line: 1, type: !6, isLocal: false, isDefinition: true)
+!2 = distinct !DICompileUnit(language: DW_LANG_C99, file: !3, producer: "clang version 8.0.0 (trunk 345296) (llvm/trunk 345297)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, globals: !5, nameTableKind: None)
+!3 = !DIFile(filename: "t.c", directory: "/home/yhs/tmp")
+!4 = !{}
+!5 = !{!0}
+!6 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "t1", file: !3, line: 1, size: 64, elements: !7)
+!7 = !{!8, !10}
+!8 = !DIDerivedType(tag: DW_TAG_member, name: "m1", scope: !6, file: !3, line: 1, baseType: !9, size: 8)
+!9 = !DIBasicType(name: "char", size: 8, encoding: DW_ATE_signed_char)
+!10 = !DIDerivedType(tag: DW_TAG_member, name: "n1", scope: !6, file: !3, line: 1, baseType: !11, size: 32, offset: 32)
+!11 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!12 = !{i32 2, !"Dwarf Version", i32 4}
+!13 = !{i32 2, !"Debug Info Version", i32 3}
+!14 = !{i32 1, !"wchar_size", i32 4}
+!15 = !{!"clang version 8.0.0 (trunk 345296) (llvm/trunk 345297)"}
diff --git a/test/CodeGen/BPF/BTF/struct-bitfield-typedef.ll b/test/CodeGen/BPF/BTF/struct-bitfield-typedef.ll
new file mode 100644
index 0000000000000000000000000000000000000000..502efb0e9845ddedbe8712cc9a0689a60a89b12e
--- /dev/null
+++ b/test/CodeGen/BPF/BTF/struct-bitfield-typedef.ll
@@ -0,0 +1,99 @@
+; RUN: llc -march=bpfel -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -march=bpfeb -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+
+; Source code:
+;   typedef int _int;
+;   typedef _int __int;
+;   struct {char m:2; __int n:3; char p;} a;
+; Compilation flag:
+;   clang -target bpf -O2 -g -S -emit-llvm t.c
+
+%struct.anon = type { i8, i8, [2 x i8] }
+
+@a = common dso_local local_unnamed_addr global %struct.anon zeroinitializer, align 4, !dbg !0
+
+!llvm.dbg.cu = !{!2}
+!llvm.module.flags = !{!15, !16, !17}
+!llvm.ident = !{!18}
+
+; CHECK:             .section        .BTF,"",@progbits
+; CHECK-NEXT:        .short  60319                   # 0xeb9f
+; CHECK-NEXT:        .byte   1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .long   24
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   104
+; CHECK-NEXT:        .long   104
+; CHECK-NEXT:        .long   27
+; CHECK-NEXT:        .long   0                       # BTF_KIND_STRUCT(id = 1)
+; CHECK-NEXT:        .long   2214592515              # 0x84000003
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   1
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   33554432                # 0x2000000
+; CHECK-NEXT:        .long   3
+; CHECK-NEXT:        .long   3
+; CHECK-NEXT:        .long   50331650                # 0x3000002
+; CHECK-NEXT:        .long   5
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   8                       # 0x8
+; CHECK-NEXT:        .long   7                       # BTF_KIND_INT(id = 2)
+; CHECK-NEXT:        .long   16777216                # 0x1000000
+; CHECK-NEXT:        .long   1
+; CHECK-NEXT:        .long   16777224                # 0x1000008
+; CHECK-NEXT:        .long   12                      # BTF_KIND_TYPEDEF(id = 3)
+; CHECK-NEXT:        .long   134217728               # 0x8000000
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   18                      # BTF_KIND_TYPEDEF(id = 4)
+; CHECK-NEXT:        .long   134217728               # 0x8000000
+; CHECK-NEXT:        .long   5
+; CHECK-NEXT:        .long   23                      # BTF_KIND_INT(id = 5)
+; CHECK-NEXT:        .long   16777216                # 0x1000000
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   16777248                # 0x1000020
+; CHECK-NEXT:        .byte   0                       # string offset=0
+; CHECK-NEXT:        .byte   109                     # string offset=1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .byte   110                     # string offset=3
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .byte   112                     # string offset=5
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .ascii  "char"                  # string offset=7
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .ascii  "__int"                 # string offset=12
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .ascii  "_int"                  # string offset=18
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .ascii  "int"                   # string offset=23
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .section        .BTF.ext,"",@progbits
+; CHECK-NEXT:        .short  60319                   # 0xeb9f
+; CHECK-NEXT:        .byte   1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .long   24
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   8                       # FuncInfo
+; CHECK-NEXT:        .long   16                      # LineInfo
+
+!0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression())
+!1 = distinct !DIGlobalVariable(name: "a", scope: !2, file: !3, line: 3, type: !6, isLocal: false, isDefinition: true)
+!2 = distinct !DICompileUnit(language: DW_LANG_C99, file: !3, producer: "clang version 8.0.0 (trunk 345296) (llvm/trunk 345297)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, globals: !5, nameTableKind: None)
+!3 = !DIFile(filename: "t.c", directory: "/home/yhs/tmp")
+!4 = !{}
+!5 = !{!0}
+!6 = distinct !DICompositeType(tag: DW_TAG_structure_type, file: !3, line: 3, size: 32, elements: !7)
+!7 = !{!8, !10, !14}
+!8 = !DIDerivedType(tag: DW_TAG_member, name: "m", scope: !6, file: !3, line: 3, baseType: !9, size: 2, flags: DIFlagBitField, extraData: i64 0)
+!9 = !DIBasicType(name: "char", size: 8, encoding: DW_ATE_signed_char)
+!10 = !DIDerivedType(tag: DW_TAG_member, name: "n", scope: !6, file: !3, line: 3, baseType: !11, size: 3, offset: 2, flags: DIFlagBitField, extraData: i64 0)
+!11 = !DIDerivedType(tag: DW_TAG_typedef, name: "__int", file: !3, line: 2, baseType: !12)
+!12 = !DIDerivedType(tag: DW_TAG_typedef, name: "_int", file: !3, line: 1, baseType: !13)
+!13 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!14 = !DIDerivedType(tag: DW_TAG_member, name: "p", scope: !6, file: !3, line: 3, baseType: !9, size: 8, offset: 8)
+!15 = !{i32 2, !"Dwarf Version", i32 4}
+!16 = !{i32 2, !"Debug Info Version", i32 3}
+!17 = !{i32 1, !"wchar_size", i32 4}
+!18 = !{!"clang version 8.0.0 (trunk 345296) (llvm/trunk 345297)"}
diff --git a/test/CodeGen/BPF/BTF/struct-enum.ll b/test/CodeGen/BPF/BTF/struct-enum.ll
new file mode 100644
index 0000000000000000000000000000000000000000..c5d924308d340db5bb30308160af2c071c9f1daa
--- /dev/null
+++ b/test/CodeGen/BPF/BTF/struct-enum.ll
@@ -0,0 +1,86 @@
+; RUN: llc -march=bpfel -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -march=bpfeb -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+
+; Source code:
+;   enum t1 { A , B };
+;   struct t2 { enum t1 m:2; enum t1 n; } a;
+; Compilation flag:
+;   clang -target bpf -O2 -g -S -emit-llvm t.c
+
+%struct.t2 = type { i8, i32 }
+
+@a = common dso_local local_unnamed_addr global %struct.t2 zeroinitializer, align 4, !dbg !0
+
+!llvm.dbg.cu = !{!2}
+!llvm.module.flags = !{!15, !16, !17}
+!llvm.ident = !{!18}
+
+; CHECK:             .section        .BTF,"",@progbits
+; CHECK-NEXT:        .short  60319                   # 0xeb9f
+; CHECK-NEXT:        .byte   1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .long   24
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   64
+; CHECK-NEXT:        .long   64
+; CHECK-NEXT:        .long   15
+; CHECK-NEXT:        .long   1                       # BTF_KIND_STRUCT(id = 1)
+; CHECK-NEXT:        .long   2214592514              # 0x84000002
+; CHECK-NEXT:        .long   8
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   33554432                # 0x2000000
+; CHECK-NEXT:        .long   6
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   32                      # 0x20
+; CHECK-NEXT:        .long   8                       # BTF_KIND_ENUM(id = 2)
+; CHECK-NEXT:        .long   100663298               # 0x6000002
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   11
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   13
+; CHECK-NEXT:        .long   1
+; CHECK-NEXT:        .byte   0                       # string offset=0
+; CHECK-NEXT:        .ascii  "t2"                    # string offset=1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .byte   109                     # string offset=4
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .byte   110                     # string offset=6
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .ascii  "t1"                    # string offset=8
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .byte   65                      # string offset=11
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .byte   66                      # string offset=13
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .section        .BTF.ext,"",@progbits
+; CHECK-NEXT:        .short  60319                   # 0xeb9f
+; CHECK-NEXT:        .byte   1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .long   24
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   8                       # FuncInfo
+; CHECK-NEXT:        .long   16                      # LineInfo
+
+!0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression())
+!1 = distinct !DIGlobalVariable(name: "a", scope: !2, file: !3, line: 2, type: !11, isLocal: false, isDefinition: true)
+!2 = distinct !DICompileUnit(language: DW_LANG_C99, file: !3, producer: "clang version 8.0.0 (trunk 344789) (llvm/trunk 344782)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, globals: !10, nameTableKind: None)
+!3 = !DIFile(filename: "t.c", directory: "/home/yhs/tmp")
+!4 = !{!5}
+!5 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "t1", file: !3, line: 1, baseType: !6, size: 32, elements: !7)
+!6 = !DIBasicType(name: "unsigned int", size: 32, encoding: DW_ATE_unsigned)
+!7 = !{!8, !9}
+!8 = !DIEnumerator(name: "A", value: 0, isUnsigned: true)
+!9 = !DIEnumerator(name: "B", value: 1, isUnsigned: true)
+!10 = !{!0}
+!11 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "t2", file: !3, line: 2, size: 64, elements: !12)
+!12 = !{!13, !14}
+!13 = !DIDerivedType(tag: DW_TAG_member, name: "m", scope: !11, file: !3, line: 2, baseType: !5, size: 2, flags: DIFlagBitField, extraData: i64 0)
+!14 = !DIDerivedType(tag: DW_TAG_member, name: "n", scope: !11, file: !3, line: 2, baseType: !5, size: 32, offset: 32)
+!15 = !{i32 2, !"Dwarf Version", i32 4}
+!16 = !{i32 2, !"Debug Info Version", i32 3}
+!17 = !{i32 1, !"wchar_size", i32 4}
+!18 = !{!"clang version 8.0.0 (trunk 344789) (llvm/trunk 344782)"}
diff --git a/test/CodeGen/BPF/BTF/uchar.ll b/test/CodeGen/BPF/BTF/uchar.ll
new file mode 100644
index 0000000000000000000000000000000000000000..19dad38ee86e5e9ec4dabfb0f57cd85700ddea89
--- /dev/null
+++ b/test/CodeGen/BPF/BTF/uchar.ll
@@ -0,0 +1,53 @@
+; RUN: llc -march=bpfel -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -march=bpfeb -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+
+; Source code:
+;   unsigned char a;
+; Compilation flag:
+;   clang -target bpf -O2 -g -S -emit-llvm t.c
+
+@a = common dso_local local_unnamed_addr global i8 0, align 1, !dbg !0
+
+!llvm.dbg.cu = !{!2}
+!llvm.module.flags = !{!7, !8, !9}
+!llvm.ident = !{!10}
+
+; CHECK:             .section        .BTF,"",@progbits
+; CHECK-NEXT:        .short  60319                   # 0xeb9f
+; CHECK-NEXT:        .byte   1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .long   24
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   16
+; CHECK-NEXT:        .long   16
+; CHECK-NEXT:        .long   15
+; CHECK-NEXT:        .long   1                       # BTF_KIND_INT(id = 1)
+; CHECK-NEXT:        .long   16777216                # 0x1000000
+; CHECK-NEXT:        .long   1
+; CHECK-NEXT:        .long   8                       # 0x8
+; CHECK-NEXT:        .byte   0                       # string offset=0
+; CHECK-NEXT:        .ascii  "unsigned char"         # string offset=1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .section        .BTF.ext,"",@progbits
+; CHECK-NEXT:        .short  60319                   # 0xeb9f
+; CHECK-NEXT:        .byte   1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .long   24
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   8                       # FuncInfo
+; CHECK-NEXT:        .long   16                      # LineInfo
+
+!0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression())
+!1 = distinct !DIGlobalVariable(name: "a", scope: !2, file: !3, line: 1, type: !6, isLocal: false, isDefinition: true)
+!2 = distinct !DICompileUnit(language: DW_LANG_C99, file: !3, producer: "clang version 8.0.0 (trunk 345296) (llvm/trunk 345297)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, globals: !5, nameTableKind: None)
+!3 = !DIFile(filename: "t.c", directory: "/home/yhs/tmp")
+!4 = !{}
+!5 = !{!0}
+!6 = !DIBasicType(name: "unsigned char", size: 8, encoding: DW_ATE_unsigned_char)
+!7 = !{i32 2, !"Dwarf Version", i32 4}
+!8 = !{i32 2, !"Debug Info Version", i32 3}
+!9 = !{i32 1, !"wchar_size", i32 4}
+!10 = !{!"clang version 8.0.0 (trunk 345296) (llvm/trunk 345297)"}
diff --git a/test/CodeGen/BPF/BTF/uint.ll b/test/CodeGen/BPF/BTF/uint.ll
new file mode 100644
index 0000000000000000000000000000000000000000..b87b2ab49167144926e331129520c59beda02b10
--- /dev/null
+++ b/test/CodeGen/BPF/BTF/uint.ll
@@ -0,0 +1,53 @@
+; RUN: llc -march=bpfel -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -march=bpfeb -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+
+; Source code:
+;   unsigned a;
+; Compilation flag:
+;   clang -target bpf -O2 -g -S -emit-llvm t.c
+
+@a = common dso_local local_unnamed_addr global i32 0, align 4, !dbg !0
+
+!llvm.dbg.cu = !{!2}
+!llvm.module.flags = !{!7, !8, !9}
+!llvm.ident = !{!10}
+
+; CHECK:             .section        .BTF,"",@progbits
+; CHECK-NEXT:        .short  60319                   # 0xeb9f
+; CHECK-NEXT:        .byte   1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .long   24
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   16
+; CHECK-NEXT:        .long   16
+; CHECK-NEXT:        .long   14
+; CHECK-NEXT:        .long   1                       # BTF_KIND_INT(id = 1)
+; CHECK-NEXT:        .long   16777216                # 0x1000000
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   32                      # 0x20
+; CHECK-NEXT:        .byte   0                       # string offset=0
+; CHECK-NEXT:        .ascii  "unsigned int"          # string offset=1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .section        .BTF.ext,"",@progbits
+; CHECK-NEXT:        .short  60319                   # 0xeb9f
+; CHECK-NEXT:        .byte   1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .long   24
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   8                       # FuncInfo
+; CHECK-NEXT:        .long   16                      # LineInfo
+
+!0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression())
+!1 = distinct !DIGlobalVariable(name: "a", scope: !2, file: !3, line: 1, type: !6, isLocal: false, isDefinition: true)
+!2 = distinct !DICompileUnit(language: DW_LANG_C99, file: !3, producer: "clang version 8.0.0 (trunk 344789) (llvm/trunk 344782)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, globals: !5, nameTableKind: None)
+!3 = !DIFile(filename: "t.c", directory: "/home/yhs/tmp")
+!4 = !{}
+!5 = !{!0}
+!6 = !DIBasicType(name: "unsigned int", size: 32, encoding: DW_ATE_unsigned)
+!7 = !{i32 2, !"Dwarf Version", i32 4}
+!8 = !{i32 2, !"Debug Info Version", i32 3}
+!9 = !{i32 1, !"wchar_size", i32 4}
+!10 = !{!"clang version 8.0.0 (trunk 344789) (llvm/trunk 344782)"}
diff --git a/test/CodeGen/BPF/BTF/ulonglong.ll b/test/CodeGen/BPF/BTF/ulonglong.ll
new file mode 100644
index 0000000000000000000000000000000000000000..b8ff1441d2f66fc307801052f461a03d26aebc3d
--- /dev/null
+++ b/test/CodeGen/BPF/BTF/ulonglong.ll
@@ -0,0 +1,53 @@
+; RUN: llc -march=bpfel -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -march=bpfeb -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+
+; Source code:
+;   unsigned long long a;
+; Compilation flag:
+;   clang -target bpf -O2 -g -S -emit-llvm t.c
+
+@a = common dso_local local_unnamed_addr global i64 0, align 8, !dbg !0
+
+!llvm.dbg.cu = !{!2}
+!llvm.module.flags = !{!7, !8, !9}
+!llvm.ident = !{!10}
+
+; CHECK:             .section        .BTF,"",@progbits
+; CHECK-NEXT:        .short  60319                   # 0xeb9f
+; CHECK-NEXT:        .byte   1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .long   24
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   16
+; CHECK-NEXT:        .long   16
+; CHECK-NEXT:        .long   24
+; CHECK-NEXT:        .long   1                       # BTF_KIND_INT(id = 1)
+; CHECK-NEXT:        .long   16777216                # 0x1000000
+; CHECK-NEXT:        .long   8
+; CHECK-NEXT:        .long   64                      # 0x40
+; CHECK-NEXT:        .byte   0                       # string offset=0
+; CHECK-NEXT:        .ascii  "long long unsigned int" # string offset=1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .section        .BTF.ext,"",@progbits
+; CHECK-NEXT:        .short  60319                   # 0xeb9f
+; CHECK-NEXT:        .byte   1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .long   24
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   8                       # FuncInfo
+; CHECK-NEXT:        .long   16                      # LineInfo
+
+!0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression())
+!1 = distinct !DIGlobalVariable(name: "a", scope: !2, file: !3, line: 1, type: !6, isLocal: false, isDefinition: true)
+!2 = distinct !DICompileUnit(language: DW_LANG_C99, file: !3, producer: "clang version 8.0.0 (trunk 344789) (llvm/trunk 344782)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, globals: !5, nameTableKind: None)
+!3 = !DIFile(filename: "t.c", directory: "/home/yhs/tmp")
+!4 = !{}
+!5 = !{!0}
+!6 = !DIBasicType(name: "long long unsigned int", size: 64, encoding: DW_ATE_unsigned)
+!7 = !{i32 2, !"Dwarf Version", i32 4}
+!8 = !{i32 2, !"Debug Info Version", i32 3}
+!9 = !{i32 1, !"wchar_size", i32 4}
+!10 = !{!"clang version 8.0.0 (trunk 344789) (llvm/trunk 344782)"}
diff --git a/test/CodeGen/BPF/BTF/union-array-typedef.ll b/test/CodeGen/BPF/BTF/union-array-typedef.ll
new file mode 100644
index 0000000000000000000000000000000000000000..53a4c6013e99c5062aab95e298c42d008290a933
--- /dev/null
+++ b/test/CodeGen/BPF/BTF/union-array-typedef.ll
@@ -0,0 +1,103 @@
+; RUN: llc -march=bpfel -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -march=bpfeb -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+
+; Source code:
+;   typedef int _int;
+;   union t {char m[4]; _int n;} a;
+; Compilation flag:
+;   clang -target bpf -O2 -g -S -emit-llvm t.c
+
+%union.t = type { i32 }
+
+@a = common dso_local local_unnamed_addr global %union.t zeroinitializer, align 4, !dbg !0
+
+!llvm.dbg.cu = !{!2}
+!llvm.module.flags = !{!16, !17, !18}
+!llvm.ident = !{!19}
+
+; CHECK:             .section        .BTF,"",@progbits
+; CHECK-NEXT:        .short  60319                   # 0xeb9f
+; CHECK-NEXT:        .byte   1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .long   24
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   120
+; CHECK-NEXT:        .long   120
+; CHECK-NEXT:        .long   41
+; CHECK-NEXT:        .long   1                       # BTF_KIND_UNION(id = 1)
+; CHECK-NEXT:        .long   83886082                # 0x5000002
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   3
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   5
+; CHECK-NEXT:        .long   5
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   0                       # BTF_KIND_ARRAY(id = 2)
+; CHECK-NEXT:        .long   50331648                # 0x3000000
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   3
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   7                       # BTF_KIND_INT(id = 3)
+; CHECK-NEXT:        .long   16777216                # 0x1000000
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   32                      # 0x20
+; CHECK-NEXT:        .long   27                      # BTF_KIND_INT(id = 4)
+; CHECK-NEXT:        .long   16777216                # 0x1000000
+; CHECK-NEXT:        .long   1
+; CHECK-NEXT:        .long   16777224                # 0x1000008
+; CHECK-NEXT:        .long   32                      # BTF_KIND_TYPEDEF(id = 5)
+; CHECK-NEXT:        .long   134217728               # 0x8000000
+; CHECK-NEXT:        .long   6
+; CHECK-NEXT:        .long   37                      # BTF_KIND_INT(id = 6)
+; CHECK-NEXT:        .long   16777216                # 0x1000000
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   16777248                # 0x1000020
+; CHECK-NEXT:        .byte   0                       # string offset=0
+; CHECK-NEXT:        .byte   116                     # string offset=1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .byte   109                     # string offset=3
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .byte   110                     # string offset=5
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .ascii  "__ARRAY_SIZE_TYPE__"   # string offset=7
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .ascii  "char"                  # string offset=27
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .ascii  "_int"                  # string offset=32
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .ascii  "int"                   # string offset=37
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .section        .BTF.ext,"",@progbits
+; CHECK-NEXT:        .short  60319                   # 0xeb9f
+; CHECK-NEXT:        .byte   1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .long   24
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   8                       # FuncInfo
+; CHECK-NEXT:        .long   16                      # LineInfo
+
+!0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression())
+!1 = distinct !DIGlobalVariable(name: "a", scope: !2, file: !3, line: 2, type: !6, isLocal: false, isDefinition: true)
+!2 = distinct !DICompileUnit(language: DW_LANG_C99, file: !3, producer: "clang version 8.0.0 (trunk 345296) (llvm/trunk 345297)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, globals: !5, nameTableKind: None)
+!3 = !DIFile(filename: "t.c", directory: "/home/yhs/tmp")
+!4 = !{}
+!5 = !{!0}
+!6 = distinct !DICompositeType(tag: DW_TAG_union_type, name: "t", file: !3, line: 2, size: 32, elements: !7)
+!7 = !{!8, !13}
+!8 = !DIDerivedType(tag: DW_TAG_member, name: "m", scope: !6, file: !3, line: 2, baseType: !9, size: 32)
+!9 = !DICompositeType(tag: DW_TAG_array_type, baseType: !10, size: 32, elements: !11)
+!10 = !DIBasicType(name: "char", size: 8, encoding: DW_ATE_signed_char)
+!11 = !{!12}
+!12 = !DISubrange(count: 4)
+!13 = !DIDerivedType(tag: DW_TAG_member, name: "n", scope: !6, file: !3, line: 2, baseType: !14, size: 32)
+!14 = !DIDerivedType(tag: DW_TAG_typedef, name: "_int", file: !3, line: 1, baseType: !15)
+!15 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!16 = !{i32 2, !"Dwarf Version", i32 4}
+!17 = !{i32 2, !"Debug Info Version", i32 3}
+!18 = !{i32 1, !"wchar_size", i32 4}
+!19 = !{!"clang version 8.0.0 (trunk 345296) (llvm/trunk 345297)"}
diff --git a/test/CodeGen/BPF/BTF/ushort.ll b/test/CodeGen/BPF/BTF/ushort.ll
new file mode 100644
index 0000000000000000000000000000000000000000..2eb28a31a0bc97bd626b362bbeb23e822feff7fd
--- /dev/null
+++ b/test/CodeGen/BPF/BTF/ushort.ll
@@ -0,0 +1,53 @@
+; RUN: llc -march=bpfel -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -march=bpfeb -filetype=asm -o - %s | FileCheck -check-prefixes=CHECK %s
+
+; Source code:
+;   unsigned short a;
+; Compilation flag:
+;   clang -target bpf -O2 -g -S -emit-llvm t.c
+
+@a = common dso_local local_unnamed_addr global i16 0, align 2, !dbg !0
+
+!llvm.dbg.cu = !{!2}
+!llvm.module.flags = !{!7, !8, !9}
+!llvm.ident = !{!10}
+
+; CHECK:             .section        .BTF,"",@progbits
+; CHECK-NEXT:        .short  60319                   # 0xeb9f
+; CHECK-NEXT:        .byte   1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .long   24
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   16
+; CHECK-NEXT:        .long   16
+; CHECK-NEXT:        .long   16
+; CHECK-NEXT:        .long   1                       # BTF_KIND_INT(id = 1)
+; CHECK-NEXT:        .long   16777216                # 0x1000000
+; CHECK-NEXT:        .long   2
+; CHECK-NEXT:        .long   16                      # 0x10
+; CHECK-NEXT:        .byte   0                       # string offset=0
+; CHECK-NEXT:        .ascii  "unsigned short"        # string offset=1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .section        .BTF.ext,"",@progbits
+; CHECK-NEXT:        .short  60319                   # 0xeb9f
+; CHECK-NEXT:        .byte   1
+; CHECK-NEXT:        .byte   0
+; CHECK-NEXT:        .long   24
+; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   4
+; CHECK-NEXT:        .long   8                       # FuncInfo
+; CHECK-NEXT:        .long   16                      # LineInfo
+
+!0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression())
+!1 = distinct !DIGlobalVariable(name: "a", scope: !2, file: !3, line: 1, type: !6, isLocal: false, isDefinition: true)
+!2 = distinct !DICompileUnit(language: DW_LANG_C99, file: !3, producer: "clang version 8.0.0 (trunk 344789) (llvm/trunk 344782)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, globals: !5, nameTableKind: None)
+!3 = !DIFile(filename: "t.c", directory: "/home/yhs/tmp")
+!4 = !{}
+!5 = !{!0}
+!6 = !DIBasicType(name: "unsigned short", size: 16, encoding: DW_ATE_unsigned)
+!7 = !{i32 2, !"Dwarf Version", i32 4}
+!8 = !{i32 2, !"Debug Info Version", i32 3}
+!9 = !{i32 1, !"wchar_size", i32 4}
+!10 = !{!"clang version 8.0.0 (trunk 344789) (llvm/trunk 344782)"}
diff --git a/test/CodeGen/BPF/reloc-btf.ll b/test/CodeGen/BPF/reloc-btf.ll
new file mode 100644
index 0000000000000000000000000000000000000000..4d6e3f5b1f04c11ffea7526755395db18c8df07d
--- /dev/null
+++ b/test/CodeGen/BPF/reloc-btf.ll
@@ -0,0 +1,33 @@
+; RUN: llc -march=bpfel -filetype=obj < %s | llvm-objdump -r - | FileCheck --check-prefix=CHECK-RELOC %s
+
+; Function Attrs: norecurse nounwind readnone
+define dso_local i32 @test() local_unnamed_addr #0 !dbg !7 {
+entry:
+  ret i32 0, !dbg !11
+}
+
+; CHECK-RELOC: file format ELF64-BPF
+; CHECK-RELOC: RELOCATION RECORDS FOR [.rel.debug_info]:
+; CHECK-RELOC: R_BPF_64_32 .debug_abbrev
+; CHECK-RELOC: R_BPF_64_64 .text
+; CHECK-RELOC: RELOCATION RECORDS FOR [.rel.BTF.ext]:
+; CHECK-RELOC: R_BPF_NONE .text
+
+attributes #0 = { norecurse nounwind readnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4, !5}
+!llvm.ident = !{!6}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 8.0.0 (trunk 350573) (llvm/trunk 350569)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, nameTableKind: None)
+!1 = !DIFile(filename: "tt.c", directory: "/home/yhs/tmp")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{i32 1, !"wchar_size", i32 4}
+!6 = !{!"clang version 8.0.0 (trunk 350573) (llvm/trunk 350569)"}
+!7 = distinct !DISubprogram(name: "test", scope: !1, file: !1, line: 1, type: !8, scopeLine: 1, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !2)
+!8 = !DISubroutineType(types: !9)
+!9 = !{!10}
+!10 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!11 = !DILocation(line: 1, column: 14, scope: !7)
diff --git a/test/CodeGen/Generic/MachineBranchProb.ll b/test/CodeGen/Generic/MachineBranchProb.ll
index 85fafba227ae5cf6f02d78ea0222858b110779b9..b80d8b2e73c32b07c6f47a82fbf7ec6ab2740aaa 100644
--- a/test/CodeGen/Generic/MachineBranchProb.ll
+++ b/test/CodeGen/Generic/MachineBranchProb.ll
@@ -4,9 +4,6 @@
 ; to fail.
 ; XFAIL: hexagon
 
-; Bug: PR31899
-; XFAIL: avr
-
 declare void @foo()
 
 ; Make sure we have the correct weight attached to each successor.
diff --git a/test/CodeGen/Generic/live-debug-label.ll b/test/CodeGen/Generic/live-debug-label.ll
new file mode 100644
index 0000000000000000000000000000000000000000..6f2f06a69addd6541edd6f9339375491279e0c05
--- /dev/null
+++ b/test/CodeGen/Generic/live-debug-label.ll
@@ -0,0 +1,141 @@
+; RUN: llc < %s -stop-after=virtregrewriter -o - | FileCheck %s
+;
+; Generated with "clang++ -g -O1 -S -emit-llvm"
+;
+; inline bool bar(char c) {
+;   return c >= '0' && c <= '9';
+; }
+;
+; unsigned foo(const char* data,
+;              int length,
+;              int* parsing_result) {
+;   unsigned value = 0;
+;   int result = 1;
+;   bool overflow = 0;
+;
+;   while (bar(*data)) {
+;     if (value > 1) {
+;       result = 2;
+;       overflow = 1;
+;     }
+;
+;     if (!overflow)
+;       value = value + 1;
+;   }
+;
+;   if (length == 0 || value) {
+;     if (!overflow)
+;       result = 0;
+;   } else {
+;     result = 1;
+;   }
+; bye:
+;   *parsing_result = result;
+;   return result == 0 ? value : 0;
+; }
+;
+; CHECK: {{^body:}}
+; CHECK: bye.thread21:
+; CHECK: DBG_LABEL !14
+; CHECK: if.then5:
+; CHECK: DBG_LABEL !14
+; CHECK-NOT: DBG_LABEL !14
+
+$_Z3barc = comdat any
+
+; Function Attrs: nounwind uwtable
+define dso_local i32 @_Z3fooPKciPi(i8* nocapture readonly %data, i32 %length, i32* nocapture %parsing_result) local_unnamed_addr !dbg !4 {
+entry:
+  %0 = load i8, i8* %data, align 1
+  %call23 = tail call zeroext i1 @_Z3barc(i8 signext %0), !dbg !15
+  br i1 %call23, label %while.body, label %while.end
+
+while.body:                                       ; preds = %entry, %while.body
+  %overflow.026 = phi i8 [ %spec.select18, %while.body ], [ 0, %entry ]
+  %result.025 = phi i32 [ %spec.select, %while.body ], [ 1, %entry ]
+  %value.024 = phi i32 [ %value.1, %while.body ], [ 0, %entry ]
+  %cmp = icmp ugt i32 %value.024, 1
+  %spec.select = select i1 %cmp, i32 2, i32 %result.025
+  %spec.select18 = select i1 %cmp, i8 1, i8 %overflow.026
+  %1 = and i8 %spec.select18, 1
+  %2 = xor i8 %1, 1
+  %3 = zext i8 %2 to i32
+  %value.1 = add i32 %value.024, %3
+  %4 = load i8, i8* %data, align 1
+  %call = tail call zeroext i1 @_Z3barc(i8 signext %4), !dbg !15
+  br i1 %call, label %while.body, label %while.end.loopexit
+
+while.end.loopexit:                               ; preds = %while.body
+  %phitmp = and i8 %spec.select18, 1
+  br label %while.end
+
+while.end:                                        ; preds = %while.end.loopexit, %entry
+  %value.0.lcssa = phi i32 [ 0, %entry ], [ %value.1, %while.end.loopexit ]
+  %result.0.lcssa = phi i32 [ 1, %entry ], [ %spec.select, %while.end.loopexit ]
+  %overflow.0.lcssa = phi i8 [ 0, %entry ], [ %phitmp, %while.end.loopexit ]
+  %cmp3 = icmp eq i32 %length, 0
+  %tobool4 = icmp ne i32 %value.0.lcssa, 0
+  %or.cond = or i1 %cmp3, %tobool4
+  br i1 %or.cond, label %if.then5, label %bye.thread21
+
+bye.thread21:                                     ; preds = %while.end
+  call void @llvm.dbg.label(metadata !14), !dbg !16
+  store i32 1, i32* %parsing_result, align 4
+  br label %6
+
+if.then5:                                         ; preds = %while.end
+  %tobool6 = icmp eq i8 %overflow.0.lcssa, 0
+  call void @llvm.dbg.label(metadata !14), !dbg !16
+  call void @llvm.dbg.label(metadata !14), !dbg !16
+  br i1 %tobool6, label %bye.thread, label %bye
+
+bye.thread:                                       ; preds = %if.then5
+  store i32 0, i32* %parsing_result, align 4
+  br label %5
+
+bye:                                              ; preds = %if.then5
+  store i32 %result.0.lcssa, i32* %parsing_result, align 4
+  %cmp10 = icmp eq i32 %result.0.lcssa, 0
+  br i1 %cmp10, label %5, label %6
+
+; <label>:5:                                      ; preds = %bye.thread, %bye
+  br label %6
+
+; <label>:6:                                      ; preds = %bye.thread21, %bye, %5
+  %7 = phi i32 [ %value.0.lcssa, %5 ], [ 0, %bye ], [ 0, %bye.thread21 ]
+  ret i32 %7
+}
+
+; Function Attrs: inlinehint nounwind uwtable
+define linkonce_odr dso_local zeroext i1 @_Z3barc(i8 signext %c) local_unnamed_addr comdat {
+entry:
+  %c.off = add i8 %c, -48
+  %0 = icmp ult i8 %c.off, 10
+  ret i1 %0
+}
+
+; Function Attrs: nounwind readnone speculatable
+declare void @llvm.dbg.label(metadata) #0
+
+attributes #0 = { nounwind readnone speculatable }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !1, isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, nameTableKind: None)
+!1 = !DIFile(filename: "live-debug-label.cc", directory: ".")
+!2 = !{}
+!3 = !{i32 2, !"Debug Info Version", i32 3}
+!4 = distinct !DISubprogram(name: "foo", linkageName: "_Z3fooPKciPi", scope: !1, file: !1, line: 5, type: !5, isLocal: false, isDefinition: true, scopeLine: 7, flags: DIFlagPrototyped, isOptimized: true, unit: !0, retainedNodes: !13)
+!5 = !DISubroutineType(types: !6)
+!6 = !{!7, !8, !11, !12}
+!7 = !DIBasicType(name: "unsigned int", size: 32, encoding: DW_ATE_unsigned)
+!8 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !9, size: 64)
+!9 = !DIDerivedType(tag: DW_TAG_const_type, baseType: !10)
+!10 = !DIBasicType(name: "char", size: 8, encoding: DW_ATE_signed_char)
+!11 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!12 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !11, size: 64)
+!13 = !{!14}
+!14 = !DILabel(scope: !4, name: "bye", file: !1, line: 28)
+!15 = !DILocation(line: 12, column: 10, scope: !4)
+!16 = !DILocation(line: 28, column: 1, scope: !4)
diff --git a/test/CodeGen/Generic/print-mul-exp.ll b/test/CodeGen/Generic/print-mul-exp.ll
index 1426fb59f6695ccc24d526bc31b976de44984866..91c8147aaad91cd1430cabf827a8cc570548a695 100644
--- a/test/CodeGen/Generic/print-mul-exp.ll
+++ b/test/CodeGen/Generic/print-mul-exp.ll
@@ -1,5 +1,4 @@
 ; RUN: llc < %s
-; XFAIL: avr
 
 @a_str = internal constant [8 x i8] c"a = %d\0A\00"		; <[8 x i8]*> [#uses=1]
 @a_mul_str = internal constant [13 x i8] c"a * %d = %d\0A\00"		; <[13 x i8]*> [#uses=1]
diff --git a/test/CodeGen/Generic/print-mul.ll b/test/CodeGen/Generic/print-mul.ll
index 20fb1be6edef100ce76d0d6bd0709e487baf8548..4b60d759278ac93c0a17caa3fad8415b1d235807 100644
--- a/test/CodeGen/Generic/print-mul.ll
+++ b/test/CodeGen/Generic/print-mul.ll
@@ -1,5 +1,4 @@
 ; RUN: llc < %s
-; XFAIL: avr
 
 @a_str = internal constant [8 x i8] c"a = %d\0A\00"		; <[8 x i8]*> [#uses=1]
 @b_str = internal constant [8 x i8] c"b = %d\0A\00"		; <[8 x i8]*> [#uses=1]
diff --git a/test/CodeGen/Generic/print-shift.ll b/test/CodeGen/Generic/print-shift.ll
index 1fda55420b5958000d67f393db3cae617f0b8db8..56b3ec1df760d8dd8da9618508171e06bf628719 100644
--- a/test/CodeGen/Generic/print-shift.ll
+++ b/test/CodeGen/Generic/print-shift.ll
@@ -1,5 +1,4 @@
 ; RUN: llc < %s
-; XFAIL: avr
 
 @a_str = internal constant [8 x i8] c"a = %d\0A\00"             ; <[8 x i8]*> [#uses=1]
 @b_str = internal constant [8 x i8] c"b = %d\0A\00"             ; <[8 x i8]*> [#uses=1]
diff --git a/test/CodeGen/Generic/select-cc.ll b/test/CodeGen/Generic/select-cc.ll
index c67f57fde18b0881b3bcf54c0839a148d0fc996e..2b39fc17a55d17938cdd3d0f56f1cea982b94508 100644
--- a/test/CodeGen/Generic/select-cc.ll
+++ b/test/CodeGen/Generic/select-cc.ll
@@ -1,8 +1,5 @@
 ; RUN: llc < %s
 
-; PR31338
-; XFAIL: avr
-
 define <2 x double> @vector_select(<2 x double> %x, <2 x double> %y) nounwind  {
 	%x.lo = extractelement <2 x double> %x, i32 0		; <double> [#uses=1]
 	%x.lo.ge = fcmp oge double %x.lo, 0.000000e+00		; <i1> [#uses=1]
diff --git a/test/CodeGen/Hexagon/cfi-late.ll b/test/CodeGen/Hexagon/cfi-late.ll
index 460b645b4a4e6d31125d303aa3f5a59a5cb2d058..67485421fe8a1f24f138d3b02e894a31cbd1c616 100644
--- a/test/CodeGen/Hexagon/cfi-late.ll
+++ b/test/CodeGen/Hexagon/cfi-late.ll
@@ -41,7 +41,7 @@ attributes #3 = { nounwind }
 !llvm.module.flags = !{!11, !12}
 !llvm.ident = !{!13}
 
-!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 3.8.0 (http://llvm.org/git/clang.git 15506a21305e212c406f980ed9b6b1bac785df56)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2)
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 3.8.0", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2)
 !1 = !DIFile(filename: "cfi-late.c", directory: "/test")
 !2 = !{}
 !4 = distinct !DISubprogram(name: "foo", scope: !1, file: !1, line: 3, type: !5, isLocal: false, isDefinition: true, scopeLine: 3, flags: DIFlagPrototyped, isOptimized: true, unit: !0, retainedNodes: !8)
@@ -53,7 +53,7 @@ attributes #3 = { nounwind }
 !10 = !DILocalVariable(name: "y", arg: 2, scope: !4, file: !1, line: 3, type: !7)
 !11 = !{i32 2, !"Dwarf Version", i32 4}
 !12 = !{i32 2, !"Debug Info Version", i32 3}
-!13 = !{!"clang version 3.8.0 (http://llvm.org/git/clang.git 15506a21305e212c406f980ed9b6b1bac785df56)"}
+!13 = !{!"clang version 3.8.0"}
 !14 = !DIExpression()
 !15 = !DILocation(line: 3, column: 13, scope: !4)
 !16 = !DILocation(line: 3, column: 20, scope: !4)
diff --git a/test/CodeGen/Hexagon/copy-to-combine-dbg.ll b/test/CodeGen/Hexagon/copy-to-combine-dbg.ll
index 9a4d4322b69f48b266bf2a31c56ebb9b7494ed0a..837e97d79651f02ff29c9be90b3e08c3b80453fd 100644
--- a/test/CodeGen/Hexagon/copy-to-combine-dbg.ll
+++ b/test/CodeGen/Hexagon/copy-to-combine-dbg.ll
@@ -36,7 +36,7 @@ attributes #1 = { nounwind readnone speculatable }
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!3, !4}
 
-!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 4.0.0 (http://llvm.org/git/clang.git 37afcb099ac2b001f4c826da7ca1d077b67a508c) (http://llvm.org/git/llvm.git 5887f1c75b3ba216850c834b186efdd3e54b7d4f)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2)
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 4.0.0", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2)
 !1 = !DIFile(filename: "file.c", directory: "/")
 !2 = !{}
 !3 = !{i32 2, !"Dwarf Version", i32 4}
diff --git a/test/CodeGen/Hexagon/funnel-shift.ll b/test/CodeGen/Hexagon/funnel-shift.ll
new file mode 100644
index 0000000000000000000000000000000000000000..fcf623f1cec301ac9ef62c0a7837027666299314
--- /dev/null
+++ b/test/CodeGen/Hexagon/funnel-shift.ll
@@ -0,0 +1,265 @@
+; RUN: llc -march=hexagon < %s | FileCheck %s
+
+; CHECK-LABEL: f0:
+; CHECK: r[[R00:[0-9]+]]:[[R01:[0-9]+]] = combine(r0,r1)
+; CHECK: r[[R02:[0-9]+]]:[[R03:[0-9]+]] = asl(r[[R00]]:[[R01]],#17)
+define i32 @f0(i32 %a0, i32 %a1) #1 {
+b0:
+  %v0 = tail call i32 @llvm.fshl.i32(i32 %a0, i32 %a1, i32 17)
+  ret i32 %v0
+}
+
+; CHECK-LABEL: f1:
+; CHECK: r[[R10:[0-9]+]]:[[R11:[0-9]+]] = combine(r0,r1)
+; CHECK: r[[R12:[0-9]+]]:[[R13:[0-9]+]] = asl(r[[R10]]:[[R11]],r2)
+define i32 @f1(i32 %a0, i32 %a1, i32 %a2) #1 {
+b0:
+  %v0 = tail call i32 @llvm.fshl.i32(i32 %a0, i32 %a1, i32 %a2)
+  ret i32 %v0
+}
+
+; CHECK-LABEL: f2:
+; CHECK: r[[R20:[0-9]+]]:[[R21:[0-9]+]] = asl(r3:2,#17)
+; CHECK: r[[R20]]:[[R21]] |= lsr(r1:0,#47)
+define i64 @f2(i64 %a0, i64 %a1) #1 {
+b0:
+  %v0 = tail call i64 @llvm.fshl.i64(i64 %a0, i64 %a1, i64 17)
+  ret i64 %v0
+}
+
+; CHECK-LABEL: f3:
+; CHECK: r[[R30:[0-9]+]]:[[R31:[0-9]+]] = asl(r3:2,r4)
+; CHECK: r[[R32:[0-9]+]] = sub(#64,r4)
+; CHECK: r[[R30]]:[[R31]] |= lsr(r1:0,r[[R32]])
+define i64 @f3(i64 %a0, i64 %a1, i64 %a2) #1 {
+b0:
+  %v0 = tail call i64 @llvm.fshl.i64(i64 %a0, i64 %a1, i64 %a2)
+  ret i64 %v0
+}
+
+; CHECK-LABEL: f4:
+; CHECK: r[[R40:[0-9]+]]:[[R41:[0-9]+]] = combine(r0,r1)
+; CHECK: r[[R42:[0-9]+]]:[[R43:[0-9]+]] = lsr(r[[R40]]:[[R41]],#17)
+define i32 @f4(i32 %a0, i32 %a1) #1 {
+b0:
+  %v0 = tail call i32 @llvm.fshr.i32(i32 %a0, i32 %a1, i32 17)
+  ret i32 %v0
+}
+
+; CHECK-LABEL: f5:
+; CHECK: r[[R50:[0-9]+]]:[[R51:[0-9]+]] = combine(r0,r1)
+; CHECK: r[[R52:[0-9]+]]:[[R53:[0-9]+]] = lsr(r[[R50]]:[[R51]],r2)
+define i32 @f5(i32 %a0, i32 %a1, i32 %a2) #1 {
+b0:
+  %v0 = tail call i32 @llvm.fshr.i32(i32 %a0, i32 %a1, i32 %a2)
+  ret i32 %v0
+}
+
+; CHECK-LABEL: f6:
+; CHECK: r[[R60:[0-9]+]]:[[R61:[0-9]+]] = lsr(r3:2,#17)
+; CHECK: r[[R60]]:[[R61]] |= asl(r1:0,#47)
+define i64 @f6(i64 %a0, i64 %a1) #1 {
+b0:
+  %v0 = tail call i64 @llvm.fshr.i64(i64 %a0, i64 %a1, i64 17)
+  ret i64 %v0
+}
+
+; CHECK-LABEL: f7:
+; CHECK: r[[R70:[0-9]+]]:[[R71:[0-9]+]] = lsr(r3:2,r4)
+; CHECK: r[[R72:[0-9]+]] = sub(#64,r4)
+; CHECK: r[[R70]]:[[R71]] |= asl(r1:0,r6)
+define i64 @f7(i64 %a0, i64 %a1, i64 %a2) #1 {
+b0:
+  %v0 = tail call i64 @llvm.fshr.i64(i64 %a0, i64 %a1, i64 %a2)
+  ret i64 %v0
+}
+
+; CHECK-LABEL: f8:
+; CHECK: r[[R80:[0-9]+]] = rol(r0,#17)
+define i32 @f8(i32 %a0) #1 {
+b0:
+  %v0 = tail call i32 @llvm.fshl.i32(i32 %a0, i32 %a0, i32 17)
+  ret i32 %v0
+}
+
+; CHECK-LABEL: f9:
+; CHECK: r[[R90:[0-9]+]]:[[R91:[0-9]+]] = combine(r0,r0)
+; CHECK: r[[R92:[0-9]+]]:[[R93:[0-9]+]] = asl(r[[R90]]:[[R91]],r1)
+define i32 @f9(i32 %a0, i32 %a1) #1 {
+b0:
+  %v0 = tail call i32 @llvm.fshl.i32(i32 %a0, i32 %a0, i32 %a1)
+  ret i32 %v0
+}
+
+; CHECK-LABEL: f10:
+; CHECK: r[[RA0:[0-9]+]]:[[RA1:[0-9]+]] = rol(r1:0,#17)
+define i64 @f10(i64 %a0, i64 %a1) #1 {
+b0:
+  %v0 = tail call i64 @llvm.fshl.i64(i64 %a0, i64 %a0, i64 17)
+  ret i64 %v0
+}
+
+; CHECK-LABEL: f11:
+; CHECK: r[[RB0:[0-9]+]]:[[RB1:[0-9]+]] = asl(r1:0,r2)
+; CHECK: r[[RB2:[0-9]+]] = sub(#64,r2)
+; CHECK: r[[RB0]]:[[RB1]] |= lsr(r1:0,r[[RB2]])
+define i64 @f11(i64 %a0, i64 %a1) #1 {
+b0:
+  %v0 = tail call i64 @llvm.fshl.i64(i64 %a0, i64 %a0, i64 %a1)
+  ret i64 %v0
+}
+
+; CHECK-LABEL: f12:
+; CHECK: r[[RC0:[0-9]+]] = rol(r0,#15)
+define i32 @f12(i32 %a0, i32 %a1) #1 {
+b0:
+  %v0 = tail call i32 @llvm.fshr.i32(i32 %a0, i32 %a0, i32 17)
+  ret i32 %v0
+}
+
+; CHECK-LABEL: f13:
+; CHECK: r[[RD0:[0-9]+]]:[[RD1:[0-9]+]] = combine(r0,r0)
+; CHECK: r[[RD2:[0-9]+]]:[[RD3:[0-9]+]] = lsr(r[[RD0]]:[[RD1]],r1)
+define i32 @f13(i32 %a0, i32 %a1) #1 {
+b0:
+  %v0 = tail call i32 @llvm.fshr.i32(i32 %a0, i32 %a0, i32 %a1)
+  ret i32 %v0
+}
+
+; CHECK-LABEL: f14:
+; CHECK: r[[RE0:[0-9]+]]:[[RE1:[0-9]+]] = rol(r1:0,#47)
+define i64 @f14(i64 %a0, i64 %a1) #1 {
+b0:
+  %v0 = tail call i64 @llvm.fshr.i64(i64 %a0, i64 %a0, i64 17)
+  ret i64 %v0
+}
+
+; CHECK-LABEL: f15:
+; CHECK: r[[RF0:[0-9]+]]:[[RF1:[0-9]+]] = lsr(r1:0,r2)
+; CHECK: r[[RF2:[0-9]+]] = sub(#64,r2)
+; CHECK: r[[RF0]]:[[RF1]] |= asl(r1:0,r[[RF2]])
+define i64 @f15(i64 %a0, i64 %a1) #1 {
+b0:
+  %v0 = tail call i64 @llvm.fshr.i64(i64 %a0, i64 %a0, i64 %a1)
+  ret i64 %v0
+}
+
+; CHECK-LABEL: f16:
+; CHECK: r[[RG0:[0-9]+]]:[[RG1:[0-9]+]] = valignb(r1:0,r3:2,#7)
+define i64 @f16(i64 %a0, i64 %a1) #1 {
+b0:
+  %v0 = tail call i64 @llvm.fshl.i64(i64 %a0, i64 %a1, i64 8)
+  ret i64 %v0
+}
+
+; CHECK-LABEL: f17:
+; CHECK: r[[RH0:[0-9]+]]:[[RH1:[0-9]+]] = valignb(r1:0,r3:2,#6)
+define i64 @f17(i64 %a0, i64 %a1) #1 {
+b0:
+  %v0 = tail call i64 @llvm.fshl.i64(i64 %a0, i64 %a1, i64 16)
+  ret i64 %v0
+}
+
+; CHECK-LABEL: f18:
+; CHECK: r[[RI0:[0-9]+]]:[[RI1:[0-9]+]] = valignb(r1:0,r3:2,#5)
+define i64 @f18(i64 %a0, i64 %a1) #1 {
+b0:
+  %v0 = tail call i64 @llvm.fshl.i64(i64 %a0, i64 %a1, i64 24)
+  ret i64 %v0
+}
+
+; CHECK-LABEL: f19:
+; CHECK: r[[RJ0:[0-9]+]]:[[RJ1:[0-9]+]] = valignb(r1:0,r3:2,#4)
+define i64 @f19(i64 %a0, i64 %a1) #1 {
+b0:
+  %v0 = tail call i64 @llvm.fshl.i64(i64 %a0, i64 %a1, i64 32)
+  ret i64 %v0
+}
+
+; CHECK-LABEL: f20:
+; CHECK: r[[RK0:[0-9]+]]:[[RK1:[0-9]+]] = valignb(r1:0,r3:2,#3)
+define i64 @f20(i64 %a0, i64 %a1) #1 {
+b0:
+  %v0 = tail call i64 @llvm.fshl.i64(i64 %a0, i64 %a1, i64 40)
+  ret i64 %v0
+}
+
+; CHECK-LABEL: f21:
+; CHECK: r[[RL0:[0-9]+]]:[[RL1:[0-9]+]] = valignb(r1:0,r3:2,#2)
+define i64 @f21(i64 %a0, i64 %a1) #1 {
+b0:
+  %v0 = tail call i64 @llvm.fshl.i64(i64 %a0, i64 %a1, i64 48)
+  ret i64 %v0
+}
+
+; CHECK-LABEL: f22:
+; CHECK: r[[RM0:[0-9]+]]:[[RM1:[0-9]+]] = valignb(r1:0,r3:2,#1)
+define i64 @f22(i64 %a0, i64 %a1) #1 {
+b0:
+  %v0 = tail call i64 @llvm.fshl.i64(i64 %a0, i64 %a1, i64 56)
+  ret i64 %v0
+}
+
+; CHECK-LABEL: f23:
+; CHECK: r[[RN0:[0-9]+]]:[[RN1:[0-9]+]] = valignb(r1:0,r3:2,#1)
+define i64 @f23(i64 %a0, i64 %a1) #1 {
+b0:
+  %v0 = tail call i64 @llvm.fshr.i64(i64 %a0, i64 %a1, i64 8)
+  ret i64 %v0
+}
+
+; CHECK-LABEL: f24:
+; CHECK: r[[RO0:[0-9]+]]:[[RO1:[0-9]+]] = valignb(r1:0,r3:2,#2)
+define i64 @f24(i64 %a0, i64 %a1) #1 {
+b0:
+  %v0 = tail call i64 @llvm.fshr.i64(i64 %a0, i64 %a1, i64 16)
+  ret i64 %v0
+}
+
+; CHECK-LABEL: f25:
+; CHECK: r[[RP0:[0-9]+]]:[[RP1:[0-9]+]] = valignb(r1:0,r3:2,#3)
+define i64 @f25(i64 %a0, i64 %a1) #1 {
+b0:
+  %v0 = tail call i64 @llvm.fshr.i64(i64 %a0, i64 %a1, i64 24)
+  ret i64 %v0
+}
+
+; CHECK-LABEL: f26:
+; CHECK: r[[RQ0:[0-9]+]]:[[RQ1:[0-9]+]] = valignb(r1:0,r3:2,#4)
+define i64 @f26(i64 %a0, i64 %a1) #1 {
+b0:
+  %v0 = tail call i64 @llvm.fshr.i64(i64 %a0, i64 %a1, i64 32)
+  ret i64 %v0
+}
+
+; CHECK-LABEL: f27:
+; CHECK: r[[RR0:[0-9]+]]:[[RR1:[0-9]+]] = valignb(r1:0,r3:2,#5)
+define i64 @f27(i64 %a0, i64 %a1) #1 {
+b0:
+  %v0 = tail call i64 @llvm.fshr.i64(i64 %a0, i64 %a1, i64 40)
+  ret i64 %v0
+}
+
+; CHECK-LABEL: f28:
+; CHECK: r[[RS0:[0-9]+]]:[[RS1:[0-9]+]] = valignb(r1:0,r3:2,#6)
+define i64 @f28(i64 %a0, i64 %a1) #1 {
+b0:
+  %v0 = tail call i64 @llvm.fshr.i64(i64 %a0, i64 %a1, i64 48)
+  ret i64 %v0
+}
+
+; CHECK-LABEL: f29:
+; CHECK: r[[RT0:[0-9]+]]:[[RT1:[0-9]+]] = valignb(r1:0,r3:2,#7)
+define i64 @f29(i64 %a0, i64 %a1) #1 {
+b0:
+  %v0 = tail call i64 @llvm.fshr.i64(i64 %a0, i64 %a1, i64 56)
+  ret i64 %v0
+}
+
+declare i32 @llvm.fshl.i32(i32, i32, i32) #0
+declare i32 @llvm.fshr.i32(i32, i32, i32) #0
+declare i64 @llvm.fshl.i64(i64, i64, i64) #0
+declare i64 @llvm.fshr.i64(i64, i64, i64) #0
+
+attributes #0 = { nounwind readnone speculatable }
+attributes #1 = { nounwind readnone "target-cpu"="hexagonv60" "target-features"="-packets" }
diff --git a/test/CodeGen/Hexagon/misaligned-const-load.ll b/test/CodeGen/Hexagon/misaligned-const-load.ll
index 0a437964cc7e078016e1433a9c614f1305b98a92..37d1155ca275515b30bb619adb100483f7b994cb 100644
--- a/test/CodeGen/Hexagon/misaligned-const-load.ll
+++ b/test/CodeGen/Hexagon/misaligned-const-load.ll
@@ -17,7 +17,7 @@ attributes #0 = { norecurse nounwind readonly "target-cpu"="hexagonv60" }
 !llvm.module.flags = !{!6, !7, !8}
 !llvm.ident = !{!9}
 
-!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 8.0.0 (http://llvm.org/git/clang.git 3fb90d137ea16e5c3a4580b9db5fd18d93df1a90) (http://llvm.org/git/llvm.git daf385e5698c00fdd693fac736acc96b95ccccd3)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, retainedTypes: !3)
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 8.0.0", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, retainedTypes: !3)
 !1 = !DIFile(filename: "misaligned-const-load.c", directory: "/test")
 !2 = !{}
 !3 = !{!4}
@@ -26,7 +26,7 @@ attributes #0 = { norecurse nounwind readonly "target-cpu"="hexagonv60" }
 !6 = !{i32 2, !"Dwarf Version", i32 4}
 !7 = !{i32 2, !"Debug Info Version", i32 3}
 !8 = !{i32 1, !"wchar_size", i32 4}
-!9 = !{!"clang version 8.0.0 (http://llvm.org/git/clang.git 3fb90d137ea16e5c3a4580b9db5fd18d93df1a90) (http://llvm.org/git/llvm.git daf385e5698c00fdd693fac736acc96b95ccccd3)"}
+!9 = !{!"clang version 8.0.0"}
 !10 = distinct !DISubprogram(name: "bad_load", scope: !1, file: !1, line: 1, type: !11, isLocal: false, isDefinition: true, scopeLine: 1, isOptimized: true, unit: !0, retainedNodes: !2)
 !11 = !DISubroutineType(types: !12)
 !12 = !{!5}
diff --git a/test/CodeGen/Hexagon/misaligned-const-store.ll b/test/CodeGen/Hexagon/misaligned-const-store.ll
index b24cd778449b80beaa2b0a7d7855dea880e863a3..311a56e13a75978005b673283fb72665d73fd87f 100644
--- a/test/CodeGen/Hexagon/misaligned-const-store.ll
+++ b/test/CodeGen/Hexagon/misaligned-const-store.ll
@@ -17,7 +17,7 @@ attributes #0 = { norecurse nounwind readonly "target-cpu"="hexagonv60" }
 !llvm.module.flags = !{!6, !7, !8}
 !llvm.ident = !{!9}
 
-!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 8.0.0 (http://llvm.org/git/clang.git 3fb90d137ea16e5c3a4580b9db5fd18d93df1a90) (http://llvm.org/git/llvm.git daf385e5698c00fdd693fac736acc96b95ccccd3)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, retainedTypes: !3)
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 8.0.0", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, retainedTypes: !3)
 !1 = !DIFile(filename: "misaligned-const-store.c", directory: "/test")
 !2 = !{}
 !3 = !{!4}
@@ -26,7 +26,7 @@ attributes #0 = { norecurse nounwind readonly "target-cpu"="hexagonv60" }
 !6 = !{i32 2, !"Dwarf Version", i32 4}
 !7 = !{i32 2, !"Debug Info Version", i32 3}
 !8 = !{i32 1, !"wchar_size", i32 4}
-!9 = !{!"clang version 8.0.0 (http://llvm.org/git/clang.git 3fb90d137ea16e5c3a4580b9db5fd18d93df1a90) (http://llvm.org/git/llvm.git daf385e5698c00fdd693fac736acc96b95ccccd3)"}
+!9 = !{!"clang version 8.0.0"}
 !10 = distinct !DISubprogram(name: "bad_store", scope: !1, file: !1, line: 1, type: !11, isLocal: false, isDefinition: true, scopeLine: 1, isOptimized: true, unit: !0, retainedNodes: !2)
 !11 = !DISubroutineType(types: !12)
 !12 = !{!5}
diff --git a/test/CodeGen/Hexagon/misched-top-rptracker-sync.ll b/test/CodeGen/Hexagon/misched-top-rptracker-sync.ll
index 9bd33bc1bc7e842361945513750bfa6cfe9a2061..c83389ee21089726440cb5c3fd7f66674937317a 100644
--- a/test/CodeGen/Hexagon/misched-top-rptracker-sync.ll
+++ b/test/CodeGen/Hexagon/misched-top-rptracker-sync.ll
@@ -79,12 +79,12 @@ attributes #1 = { nounwind readnone }
 !llvm.module.flags = !{!3, !4}
 !llvm.ident = !{!5}
 
-!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 3.9.0 (http://llvm.org/git/clang.git 4b380bc1db8b0c72bdbdaf0e4697b1a84100a369) (http://llvm.org/git/llvm.git 6217a62bc009d55e160dbb694f2e94a22c80809f)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2)
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 3.9.0", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2)
 !1 = !DIFile(filename: "bug.c", directory: "/")
 !2 = !{}
 !3 = !{i32 2, !"Dwarf Version", i32 4}
 !4 = !{i32 2, !"Debug Info Version", i32 3}
-!5 = !{!"clang version 3.9.0 (http://llvm.org/git/clang.git 4b380bc1db8b0c72bdbdaf0e4697b1a84100a369) (http://llvm.org/git/llvm.git 6217a62bc009d55e160dbb694f2e94a22c80809f)"}
+!5 = !{!"clang version 3.9.0"}
 !6 = distinct !DISubprogram(name: "fred", scope: !1, file: !1, line: 138, type: !7, isLocal: false, isDefinition: true, scopeLine: 139, flags: DIFlagPrototyped, isOptimized: true, unit: !0, retainedNodes: !25)
 !7 = !DISubroutineType(types: !8)
 !8 = !{null, !9, !15}
diff --git a/test/CodeGen/Hexagon/rotate.ll b/test/CodeGen/Hexagon/rotate.ll
index bcc978fdce60cfe074639327dffa5de26f8ff953..3015a5a0b83e942819bd928a051635386a85c85e 100644
--- a/test/CodeGen/Hexagon/rotate.ll
+++ b/test/CodeGen/Hexagon/rotate.ll
@@ -13,13 +13,10 @@ b0:
 }
 
 ; CHECK-LABEL: f1
-; No variable-shift rotates. Check for the default expansion code.
-; This is a rotate left by %a1(r1).
-; CHECK: r[[R10:[0-9]+]] = sub(#32,r1)
-; CHECK: r[[R11:[0-9]+]] = and(r1,#31)
-; CHECK: r[[R12:[0-9]+]] = and(r[[R10]],#31)
-; CHECK: r[[R13:[0-9]+]] = asl(r0,r[[R11]])
-; CHECK: r[[R13]] |= lsr(r0,r[[R12]])
+; This is a rotate left by %a1(r1). Use register-pair shift to implement it.
+; CHECK: r[[R10:[0-9]+]]:[[R11:[0-9]+]] = combine(r0,r0)
+; CHECK: r[[R12:[0-9]+]]:[[R13:[0-9]+]] = asl(r[[R10]]:[[R11]],r1)
+; CHECK: r0 = r[[R12]]
 define i32 @f1(i32 %a0, i32 %a1) #0 {
 b0:
   %v0 = shl i32 %a0, %a1
@@ -40,13 +37,9 @@ b0:
 }
 
 ; CHECK-LABEL: f3
-; No variable-shift rotates. Check for the default expansion code.
-; This is a rotate right by %a1(r1) that became a rotate left by 32-%a1.
-; CHECK: r[[R30:[0-9]+]] = sub(#32,r1)
-; CHECK: r[[R31:[0-9]+]] = and(r1,#31)
-; CHECK: r[[R32:[0-9]+]] = and(r[[R30]],#31)
-; CHECK: r[[R33:[0-9]+]] = asl(r0,r[[R32]])
-; CHECK: r[[R33]] |= lsr(r0,r[[R31]])
+; This is a rotate right by %a1(r1). Use register-pair shift to implement it.
+; CHECK: r[[R30:[0-9]+]]:[[R31:[0-9]+]] = combine(r0,r0)
+; CHECK: r[[R32:[0-9]+]]:[[R33:[0-9]+]] = lsr(r[[R30]]:[[R31]],r1)
 define i32 @f3(i32 %a0, i32 %a1) #0 {
 b0:
   %v0 = lshr i32 %a0, %a1
@@ -67,13 +60,10 @@ b0:
 }
 
 ; CHECK-LABEL: f5
-; No variable-shift rotates. Check for the default expansion code.
 ; This is a rotate left by %a1(r2).
-; CHECK: r[[R50:[0-9]+]] = sub(#64,r2)
-; CHECK: r[[R51:[0-9]+]] = and(r2,#63)
-; CHECK: r[[R52:[0-9]+]] = and(r[[R50]],#63)
-; CHECK: r[[R53:[0-9]+]]:[[R54:[0-9]+]] = asl(r1:0,r[[R51]])
-; CHECK: r[[R53]]:[[R54]] |= lsr(r1:0,r[[R52]])
+; CHECK: r[[R50:[0-9]+]]:[[R51:[0-9]+]] = asl(r1:0,r2)
+; CHECK: r[[R52:[0-9]+]] = sub(#64,r2)
+; CHECK: r[[R50]]:[[R51]] |= lsr(r1:0,r[[R52]])
 define i64 @f5(i64 %a0, i32 %a1) #0 {
 b0:
   %v0 = zext i32 %a1 to i64
@@ -96,13 +86,10 @@ b0:
 }
 
 ; CHECK-LABEL: f7
-; No variable-shift rotates. Check for the default expansion code.
-; This is a rotate right by %a1(r2) that became a rotate left by 64-%a1.
-; CHECK: r[[R70:[0-9]+]] = sub(#64,r2)
-; CHECK: r[[R71:[0-9]+]] = and(r2,#63)
-; CHECK: r[[R72:[0-9]+]] = and(r[[R70]],#63)
-; CHECK: r[[R73:[0-9]+]]:[[R75:[0-9]+]] = asl(r1:0,r[[R72]])
-; CHECK: r[[R73]]:[[R75]] |= lsr(r1:0,r[[R71]])
+; This is a rotate right by %a1(r2).
+; CHECK: r[[R70:[0-9]+]]:[[R71:[0-9]+]] = lsr(r1:0,r2)
+; CHECK: r[[R72:[0-9]+]] = sub(#64,r2)
+; CHECK: r[[R70]]:[[R71]] |= asl(r1:0,r[[R72]])
 define i64 @f7(i64 %a0, i32 %a1) #0 {
 b0:
   %v0 = zext i32 %a1 to i64
diff --git a/test/CodeGen/Hexagon/swp-epilog-phi5.ll b/test/CodeGen/Hexagon/swp-epilog-phi5.ll
index 2222deed08e1c28055c2c75284d11a05c9935c86..3dcecad54565afed91fd233daf804b3e9de137df 100644
--- a/test/CodeGen/Hexagon/swp-epilog-phi5.ll
+++ b/test/CodeGen/Hexagon/swp-epilog-phi5.ll
@@ -9,8 +9,8 @@
 
 ; CHECK: loop0
 ; CHECK: [[REG0:r([0-9]+)]] += mpyi
-; CHECK-NOT: r{{[0-9]+}} += add([[REG0]],#8)
-; CHECK: endloop1
+; CHECK: [[REG2:r([0-9]+)]] = add([[REG1:r([0-9]+)]],add([[REG0]],#8
+; CHECK: endloop0
 
 %s.0 = type { %s.1*, %s.4*, %s.7*, i8*, i8, i32, %s.8*, i32, i32, i32, i8, i8, i32, i32, double, i8, i8, i8, i8, i8, i8, i8, i8, i32, i8, i8, i8, i32, i32, i32, i32, i32, i32, i8**, i32, i32, i32, i32, i32, [64 x i32]*, [4 x %s.9*], [4 x %s.10*], [4 x %s.10*], i32, %s.23*, i8, i8, [16 x i8], [16 x i8], [16 x i8], i32, i8, i8, i8, i8, i16, i16, i8, i8, i8, %s.11*, i32, i32, i32, i32, i8*, i32, [4 x %s.23*], i32, i32, i32, [10 x i32], i32, i32, i32, i32, i32, %s.12*, %s.13*, %s.14*, %s.15*, %s.16*, %s.17*, %s.18*, %s.19*, %s.20*, %s.21*, %s.22* }
 %s.1 = type { void (%s.2*)*, void (%s.2*, i32)*, void (%s.2*)*, void (%s.2*, i8*)*, void (%s.2*)*, i32, %s.3, i32, i32, i8**, i32, i8**, i32, i32 }
diff --git a/test/CodeGen/Hexagon/swp-pragma-disable.ii b/test/CodeGen/Hexagon/swp-pragma-disable.ii
new file mode 100644
index 0000000000000000000000000000000000000000..80494f5ac10deb9e29d541952bd966547e4fac9d
--- /dev/null
+++ b/test/CodeGen/Hexagon/swp-pragma-disable.ii
@@ -0,0 +1,49 @@
+; RUN: llc -disable-lsr -march=hexagon -enable-pipeliner  \
+; RUN:     -debug-only=pipeliner < %s 2>&1 > /dev/null | FileCheck %s
+; REQUIRES: asserts
+;
+; Test that checks if pipeliner disabled by pragma 
+
+; CHECK: Can not pipeline loop 
+
+; Function Attrs: nounwind
+define void @f0(i32* nocapture %a0, i32 %a1) #0 {
+b0:
+  %v0 = icmp sgt i32 %a1, 1
+  br i1 %v0, label %b1, label %b4
+
+b1:                                               ; preds = %b0
+  %v1 = load i32, i32* %a0, align 4
+  %v2 = add i32 %v1, 10
+  %v3 = getelementptr i32, i32* %a0, i32 1
+  %v4 = add i32 %a1, -1
+  br label %b2
+
+b2:                                               ; preds = %b2, %b1
+  %v5 = phi i32 [ %v12, %b2 ], [ %v4, %b1 ]
+  %v6 = phi i32* [ %v11, %b2 ], [ %v3, %b1 ]
+  %v7 = phi i32 [ %v10, %b2 ], [ %v2, %b1 ]
+  store i32 %v7, i32* %v6, align 4
+  %v8 = add i32 %v7, 10
+  %v9 = getelementptr i32, i32* %v6, i32 -1
+  store i32 %v8, i32* %v9, align 4
+  %v10 = add i32 %v7, 10
+  %v11 = getelementptr i32, i32* %v6, i32 1
+  %v12 = add i32 %v5, -1
+  %v13 = icmp eq i32 %v12, 0
+  br i1 %v13, label %b3, label %b2
+
+b3:                                               ; preds = %b2
+  br label %b4 , !llvm.loop !2
+
+b4:                                               ; preds = %b3, %b0
+  ret void
+}
+
+attributes #0 = { nounwind }
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{!2, !2, i64 0}
+!2 = distinct !{!2, !3}
+!3 = !{!"llvm.loop.pipeline.disable", i1 true}
+
diff --git a/test/CodeGen/Hexagon/swp-pragma-initiation-interval.ii b/test/CodeGen/Hexagon/swp-pragma-initiation-interval.ii
new file mode 100644
index 0000000000000000000000000000000000000000..6a4ba7eaccd313bfca6bf68ab1ba1172b65b1d9b
--- /dev/null
+++ b/test/CodeGen/Hexagon/swp-pragma-initiation-interval.ii
@@ -0,0 +1,49 @@
+; RUN: llc -disable-lsr -march=hexagon -enable-pipeliner  \
+; RUN:     -debug-only=pipeliner < %s 2>&1 > /dev/null | FileCheck %s
+; REQUIRES: asserts
+;
+; Test that checks if the II set by pragma was taken by pipeliner.
+
+; CHECK: MII = 2 MAX_II = 2 
+
+; Function Attrs: nounwind
+define void @f0(i32* nocapture %a0, i32 %a1) #0 {
+b0:
+  %v0 = icmp sgt i32 %a1, 1
+  br i1 %v0, label %b1, label %b4
+
+b1:                                               ; preds = %b0
+  %v1 = load i32, i32* %a0, align 4
+  %v2 = add i32 %v1, 10
+  %v3 = getelementptr i32, i32* %a0, i32 1
+  %v4 = add i32 %a1, -1
+  br label %b2
+
+b2:                                               ; preds = %b2, %b1
+  %v5 = phi i32 [ %v12, %b2 ], [ %v4, %b1 ]
+  %v6 = phi i32* [ %v11, %b2 ], [ %v3, %b1 ]
+  %v7 = phi i32 [ %v10, %b2 ], [ %v2, %b1 ]
+  store i32 %v7, i32* %v6, align 4
+  %v8 = add i32 %v7, 10
+  %v9 = getelementptr i32, i32* %v6, i32 -1
+  store i32 %v8, i32* %v9, align 4
+  %v10 = add i32 %v7, 10
+  %v11 = getelementptr i32, i32* %v6, i32 1
+  %v12 = add i32 %v5, -1
+  %v13 = icmp eq i32 %v12, 0
+  br i1 %v13, label %b3, label %b2
+
+b3:                                               ; preds = %b2
+  br label %b4 , !llvm.loop !2
+
+b4:                                               ; preds = %b3, %b0
+  ret void
+}
+
+attributes #0 = { nounwind }
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{!2, !2, i64 0}
+!2 = distinct !{!2, !3}
+!3 = !{!"llvm.loop.pipeline.initiationinterval", i32 2}
+
diff --git a/test/CodeGen/Hexagon/swp-resmii-1.ll b/test/CodeGen/Hexagon/swp-resmii-1.ll
index ca9bcff6954f0a1c80100cbfd1b66e4c6e2ab42e..1a9734860d5cb8452bd034228f76662dbd050504 100644
--- a/test/CodeGen/Hexagon/swp-resmii-1.ll
+++ b/test/CodeGen/Hexagon/swp-resmii-1.ll
@@ -3,7 +3,7 @@
 
 ; Test that checks that we compute the correct ResMII for haar.
 
-; CHECK: MII = 4 (rec=1, res=4)
+; CHECK: MII = 4 MAX_II = 14 (rec=1, res=4)
 
 ; Function Attrs: nounwind
 define void @f0(i16* noalias nocapture readonly %a0, i32 %a1, i32 %a2, i32 %a3, i8* noalias nocapture %a4, i32 %a5) #0 {
diff --git a/test/CodeGen/Hexagon/swp-resmii.ll b/test/CodeGen/Hexagon/swp-resmii.ll
index 9cf3b1e34ab0fd64ca3a5b2950694cb6383ac9c3..851d82ea50f091ff409d18fac79a6be9992500d9 100644
--- a/test/CodeGen/Hexagon/swp-resmii.ll
+++ b/test/CodeGen/Hexagon/swp-resmii.ll
@@ -4,7 +4,7 @@
 ;
 ; Test that checks if the ResMII is 1.
 
-; CHECK: MII = 1 (rec=1, res=1)
+; CHECK: MII = 1 MAX_II = 11 (rec=1, res=1)
 
 ; Function Attrs: nounwind
 define void @f0(i32* nocapture %a0, i32 %a1) #0 {
diff --git a/test/CodeGen/Hexagon/vect/vect-vaslw.ll b/test/CodeGen/Hexagon/vect/vect-vaslw.ll
index c662b0bd3de29812f41ded429f8a0733944492bb..23c167608a10fbc832f2708002cedf5e6aaeb0ff 100644
--- a/test/CodeGen/Hexagon/vect/vect-vaslw.ll
+++ b/test/CodeGen/Hexagon/vect/vect-vaslw.ll
@@ -1,5 +1,5 @@
 ; RUN: llc -march=hexagon < %s | FileCheck %s
-; CHECK: vaslw
+; CHECK: vaslh
 
 target datalayout = "e-p:32:32:32-i64:64:64-i32:32:32-i16:16:16-i1:32:32-f64:64:64-f32:32:32-v64:64:64-v32:32:32-a0:0-n16:32"
 target triple = "hexagon-unknown-linux-gnu"
diff --git a/test/CodeGen/MIR/AArch64/parse-low-level-type-invalid10.mir b/test/CodeGen/MIR/AArch64/parse-low-level-type-invalid10.mir
new file mode 100644
index 0000000000000000000000000000000000000000..e8a102ef2cdc7d60d548023293038a91c4c40f46
--- /dev/null
+++ b/test/CodeGen/MIR/AArch64/parse-low-level-type-invalid10.mir
@@ -0,0 +1,12 @@
+# RUN: not llc -mtriple=aarch64-- -run-pass none -o /dev/null %s 2>&1 | FileCheck %s
+
+# When a low-level type pointer has an address space greater than supported, and make sure an implicit truncate to 32-bits doesn't happen.
+
+---
+name: test_address_space_number_too_big64
+body: |
+  bb.0:
+    liveins: $x0
+    ; CHECK: [[@LINE+1]]:10: invalid address space number
+    %0:_(p17179869185) = G_IMPLICIT_DEF
+...
diff --git a/test/CodeGen/MIR/AArch64/parse-low-level-type-invalid4.mir b/test/CodeGen/MIR/AArch64/parse-low-level-type-invalid4.mir
new file mode 100644
index 0000000000000000000000000000000000000000..d66dd1044869225ae2a2e336ecfcd4772005b393
--- /dev/null
+++ b/test/CodeGen/MIR/AArch64/parse-low-level-type-invalid4.mir
@@ -0,0 +1,10 @@
+# RUN: not llc -mtriple=aarch64-- -run-pass none -o /dev/null %s 2>&1 | FileCheck %s
+# When a low-level type is 0 bits
+---
+name: test_scalar_size_0
+body: |
+  bb.0:
+    liveins: $x0
+    ; CHECK: [[@LINE+1]]:10: invalid size for scalar type
+    %0:_(s0) = G_IMPLICIT_DEF
+...
diff --git a/test/CodeGen/MIR/AArch64/parse-low-level-type-invalid5.mir b/test/CodeGen/MIR/AArch64/parse-low-level-type-invalid5.mir
new file mode 100644
index 0000000000000000000000000000000000000000..79f0d554c125102a040057037a4d2fe4dbb9348d
--- /dev/null
+++ b/test/CodeGen/MIR/AArch64/parse-low-level-type-invalid5.mir
@@ -0,0 +1,10 @@
+# RUN: not llc -mtriple=aarch64-- -run-pass none -o /dev/null %s 2>&1 | FileCheck %s
+# When a low-level type is larger than supported
+---
+name: test_scalar_size_65536
+body: |
+  bb.0:
+    liveins: $x0
+    ; CHECK: [[@LINE+1]]:10: invalid size for scalar type
+    %0:_(s65536) = G_IMPLICIT_DEF
+...
diff --git a/test/CodeGen/MIR/AArch64/parse-low-level-type-invalid6.mir b/test/CodeGen/MIR/AArch64/parse-low-level-type-invalid6.mir
new file mode 100644
index 0000000000000000000000000000000000000000..698568701fb1a47210a74d86b9d168a879141a9a
--- /dev/null
+++ b/test/CodeGen/MIR/AArch64/parse-low-level-type-invalid6.mir
@@ -0,0 +1,10 @@
+# RUN: not llc -mtriple=aarch64-- -run-pass none -o /dev/null %s 2>&1 | FileCheck %s
+# When a low-level type vector has a 0-bit element
+---
+name: test_vector_element_size_0
+body: |
+  bb.0:
+    liveins: $x0
+    ; CHECK: [[@LINE+1]]:15: invalid size for scalar type
+    %0:_(<2 x s0>) = G_IMPLICIT_DEF
+...
diff --git a/test/CodeGen/MIR/AArch64/parse-low-level-type-invalid7.mir b/test/CodeGen/MIR/AArch64/parse-low-level-type-invalid7.mir
new file mode 100644
index 0000000000000000000000000000000000000000..9d074648b7ea00c0141c4e6d5f6cae8e1f71fa20
--- /dev/null
+++ b/test/CodeGen/MIR/AArch64/parse-low-level-type-invalid7.mir
@@ -0,0 +1,10 @@
+# RUN: not llc -mtriple=aarch64-- -run-pass none -o /dev/null %s 2>&1 | FileCheck %s
+# When a low-level type vector has 0 elements
+---
+name: test_vector_0_elements
+body: |
+  bb.0:
+    liveins: $x0
+    ; CHECK: [[@LINE+1]]:11: invalid number of vector elements
+    %0:_(<0 x s1>) = G_IMPLICIT_DEF
+...
diff --git a/test/CodeGen/MIR/AArch64/parse-low-level-type-invalid8.mir b/test/CodeGen/MIR/AArch64/parse-low-level-type-invalid8.mir
new file mode 100644
index 0000000000000000000000000000000000000000..1b93834404003ed545d6d8fce526e8db66d0bc83
--- /dev/null
+++ b/test/CodeGen/MIR/AArch64/parse-low-level-type-invalid8.mir
@@ -0,0 +1,10 @@
+# RUN: not llc -mtriple=aarch64-- -run-pass none -o /dev/null %s 2>&1 | FileCheck %s
+# When a low-level type vector has more elements than supported
+---
+name: test_vector_too_many_elements
+body: |
+  bb.0:
+    liveins: $x0
+    ; CHECK: [[@LINE+1]]:11: invalid number of vector elements
+    %0:_(<65536 x s1>) = G_IMPLICIT_DEF
+...
diff --git a/test/CodeGen/MIR/AArch64/parse-low-level-type-invalid9.mir b/test/CodeGen/MIR/AArch64/parse-low-level-type-invalid9.mir
new file mode 100644
index 0000000000000000000000000000000000000000..f42123828012093d6ffcbdd46899f96f0f6fc386
--- /dev/null
+++ b/test/CodeGen/MIR/AArch64/parse-low-level-type-invalid9.mir
@@ -0,0 +1,10 @@
+# RUN: not llc -mtriple=aarch64-- -run-pass none -o /dev/null %s 2>&1 | FileCheck %s
+# When a low-level type pointer has an address space greater than supported.
+---
+name: test_address_space_number_too_big
+body: |
+  bb.0:
+    liveins: $x0
+    ; CHECK: [[@LINE+1]]:10: invalid address space number
+    %0:_(p16777216) = G_IMPLICIT_DEF
+...
diff --git a/test/CodeGen/MIR/AArch64/return-address-signing.mir b/test/CodeGen/MIR/AArch64/return-address-signing.mir
new file mode 100644
index 0000000000000000000000000000000000000000..1489c6775677a37b1b96b7fe196d27e42aea0994
--- /dev/null
+++ b/test/CodeGen/MIR/AArch64/return-address-signing.mir
@@ -0,0 +1,48 @@
+# RUN: llc -mtriple=aarch64-arm-none-eabi -run-pass=prologepilog -o - %s 2>&1 | FileCheck %s
+--- |
+  target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+  target triple = "aarch64-arm-none-eabi"
+
+  define dso_local i32 @foo() "sign-return-address"="all" "sign-return-address-key"="a_key" {
+  entry:
+    ret i32 2
+  }
+
+  define dso_local i32 @bar() "sign-return-address"="all" "sign-return-address-key"="b_key" {
+  entry:
+    ret i32 2
+  }
+...
+---
+#CHECK: foo
+name:            foo
+alignment:       2
+tracksRegLiveness: true
+frameInfo:
+  maxCallFrameSize: 0
+#CHECK:    frame-setup PACIASP implicit-def $lr, implicit $lr, implicit $sp
+#CHECK:    frame-setup CFI_INSTRUCTION negate_ra_sign_state
+#CHECK:    frame-destroy AUTIASP implicit-def $lr, implicit $lr, implicit $sp
+body:             |
+  bb.0.entry:
+    $w0 = MOVi32imm 2
+    RET_ReallyLR implicit killed $w0
+
+...
+---
+#CHECK: bar
+name:            bar
+alignment:       2
+tracksRegLiveness: true
+frameInfo:
+  maxCallFrameSize: 0
+#CHECK:    frame-setup EMITBKEY
+#CHECK:    frame-setup PACIBSP implicit-def $lr, implicit $lr, implicit $sp
+#CHECK:    frame-setup CFI_INSTRUCTION negate_ra_sign_state
+#CHECK:    frame-destroy AUTIBSP implicit-def $lr, implicit $lr, implicit $sp
+body:             |
+  bb.0.entry:
+    $w0 = MOVi32imm 2
+    RET_ReallyLR implicit killed $w0
+
+...
diff --git a/test/CodeGen/MIR/X86/expected-power-of-2-after-align.mir b/test/CodeGen/MIR/X86/expected-power-of-2-after-align.mir
new file mode 100644
index 0000000000000000000000000000000000000000..0842d96cac2bd4a736b49fc48cbe5600fbd8406e
--- /dev/null
+++ b/test/CodeGen/MIR/X86/expected-power-of-2-after-align.mir
@@ -0,0 +1,12 @@
+# RUN: not llc -march=x86-64 -run-pass none -o /dev/null %s 2>&1 | FileCheck %s
+
+---
+name: align_0
+body: |
+  bb.0:
+
+    %0:_(p0) = IMPLICIT_DEF
+    ; CHECK: [[@LINE+1]]:50: expected a power-of-2 literal after 'align'
+    %1:_(s64) = G_LOAD %0(p0) :: (load 8, align 0)
+...
+
diff --git a/test/CodeGen/MIR/X86/memory-operands.mir b/test/CodeGen/MIR/X86/memory-operands.mir
index 2ac7bea2fc9b9143fcf384354c623c69897efc1e..9efc69c0683a35e1395c6f7543e2883aec85f143 100644
--- a/test/CodeGen/MIR/X86/memory-operands.mir
+++ b/test/CodeGen/MIR/X86/memory-operands.mir
@@ -357,10 +357,10 @@ body: |
   bb.0.entry:
     $rsp = frame-setup SUB64ri8 $rsp, 24, implicit-def dead $eflags
     CFI_INSTRUCTION def_cfa_offset 32
-    LD_F80m $rsp, 1, $noreg, 32, $noreg, implicit-def dead $fpsw
+    LD_F80m $rsp, 1, $noreg, 32, $noreg, implicit-def dead $fpsw, implicit $fpcw
   ; CHECK: name: stack_psv
-  ; CHECK: ST_FP80m $rsp, 1, $noreg, 0, $noreg, implicit-def dead $fpsw :: (store 10 into stack, align 16)
-    ST_FP80m $rsp, 1, _, 0, _, implicit-def dead $fpsw :: (store 10 into stack, align 16)
+  ; CHECK: ST_FP80m $rsp, 1, $noreg, 0, $noreg, implicit-def dead $fpsw, implicit $fpcw :: (store 10 into stack, align 16)
+    ST_FP80m $rsp, 1, _, 0, _, implicit-def dead $fpsw, implicit $fpcw :: (store 10 into stack, align 16)
     CALL64pcrel32 &cosl, csr_64, implicit $rsp, implicit-def $rsp, implicit-def $fp0
     $rsp = ADD64ri8 $rsp, 24, implicit-def dead $eflags
     RETQ
diff --git a/test/CodeGen/MSP430/2009-12-21-FrameAddr.ll b/test/CodeGen/MSP430/2009-12-21-FrameAddr.ll
index c3d69c7c0db5e25cf96fd26c8284cd3fce160847..be82e98dffe37028d97e610e4bbd6e9dfa297cb0 100644
--- a/test/CodeGen/MSP430/2009-12-21-FrameAddr.ll
+++ b/test/CodeGen/MSP430/2009-12-21-FrameAddr.ll
@@ -3,7 +3,7 @@
 target datalayout = "e-p:16:8:8-i8:8:8-i16:8:8-i32:8:8"
 target triple = "msp430-unknown-linux-gnu"
 
-define msp430_intrcc void @foo() nounwind {
+define msp430_intrcc void @foo() nounwind #0 {
 entry:
 	%fa = call i8* @llvm.frameaddress(i32 0)
 	store i8 0, i8* %fa
@@ -11,3 +11,5 @@ entry:
 }
 
 declare i8* @llvm.frameaddress(i32)
+
+attributes #0 = { noinline nounwind optnone "interrupt"="2" }
diff --git a/test/CodeGen/MSP430/Inst16mm.ll b/test/CodeGen/MSP430/Inst16mm.ll
index 21fab42fd591fc8d8ae69eb37dab005d3395e5d7..af00a18acf550844e4aafa9b801fa876b6f4cb33 100644
--- a/test/CodeGen/MSP430/Inst16mm.ll
+++ b/test/CodeGen/MSP430/Inst16mm.ll
@@ -67,3 +67,22 @@ entry:
 ; CHECK-DAG:	mov	2(r1), 6(r1)
 ; CHECK-DAG:	mov	0(r1), 4(r1)
 }
+
+define void @cmp(i16* %g, i16* %i) {
+entry:
+; CHECK-LABEL: cmp:
+; CHECK: cmp 8(r12), 4(r13)
+  %add.ptr = getelementptr inbounds i16, i16* %g, i16 4
+  %0 = load i16, i16* %add.ptr, align 2
+  %add.ptr1 = getelementptr inbounds i16, i16* %i, i16 2
+  %1 = load i16, i16* %add.ptr1, align 2
+  %cmp = icmp sgt i16 %0, %1
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  store i16 0, i16* %g, align 2
+  br label %if.end
+
+if.end:                                           ; preds = %if.then, %entry
+  ret void
+}
diff --git a/test/CodeGen/MSP430/Inst8mm.ll b/test/CodeGen/MSP430/Inst8mm.ll
index b9848dc123037e68ea933f8ff4c6ada7609e30fb..5709728642b4c89641c6a8d7d5f67be7e2864b0f 100644
--- a/test/CodeGen/MSP430/Inst8mm.ll
+++ b/test/CodeGen/MSP430/Inst8mm.ll
@@ -53,3 +53,21 @@ define void @xor() nounwind {
 	ret void
 }
 
+define void @cmp(i8* %g, i8* %i) {
+entry:
+; CHECK-LABEL: cmp:
+; CHECK: cmp.b 4(r12), 2(r13)
+  %add.ptr = getelementptr inbounds i8, i8* %g, i16 4
+  %0 = load i8, i8* %add.ptr, align 1
+  %add.ptr1 = getelementptr inbounds i8, i8* %i, i16 2
+  %1 = load i8, i8* %add.ptr1, align 1
+  %cmp = icmp sgt i8 %0, %1
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  store i8 0, i8* %g, align 2
+  br label %if.end
+
+if.end:                                           ; preds = %if.then, %entry
+  ret void
+}
diff --git a/test/CodeGen/MSP430/InstII.ll b/test/CodeGen/MSP430/InstII.ll
new file mode 100644
index 0000000000000000000000000000000000000000..596d5b045beb52f7867a4d4cdafa8fb352103ced
--- /dev/null
+++ b/test/CodeGen/MSP430/InstII.ll
@@ -0,0 +1,68 @@
+; RUN: llc -march=msp430 < %s | FileCheck %s
+target datalayout = "e-p:16:8:8-i8:8:8-i16:8:8-i32:8:8"
+target triple = "msp430-generic-generic"
+
+define void @rra8m(i8* %i) {
+entry:
+; CHECK-LABEL: rra8m:
+; CHECK: rra.b 2(r12)
+  %0 = getelementptr inbounds i8, i8* %i, i16 2
+  %1 = load i8, i8* %0, align 1
+  %shr = ashr i8 %1, 1
+  store i8 %shr, i8* %0, align 1
+  ret void
+}
+
+define void @rra16m(i16* %i) {
+entry:
+; CHECK-LABEL: rra16m:
+; CHECK: rra 4(r12)
+  %0 = getelementptr inbounds i16, i16* %i, i16 2
+  %1 = load i16, i16* %0, align 2
+  %shr = ashr i16 %1, 1
+  store i16 %shr, i16* %0, align 2
+  ret void
+}
+
+; TODO: `clrc; rrc.b 2(r12)` is expected
+define void @rrc8m(i8* %g) {
+entry:
+; CHECK-LABEL: rrc8m:
+; CHECK: mov.b 2(r12), r13
+; CHECK: clrc
+; CHECK: rrc.b r13
+; CHECK: mov.b r13, 2(r12)
+  %add.ptr = getelementptr inbounds i8, i8* %g, i16 2
+  %0 = load i8, i8* %add.ptr, align 1
+  %1 = lshr i8 %0, 1
+  store i8 %1, i8* %add.ptr, align 1
+  ret void
+}
+
+; TODO: `clrc; rrc 4(r12)` is expected
+define void @rrc16m(i16* %g) {
+entry:
+; CHECK-LABEL: rrc16m:
+; CHECK: mov 4(r12), r13
+; CHECK: clrc
+; CHECK: rrc r13
+; CHECK: mov r13, 4(r12)
+  %add.ptr = getelementptr inbounds i16, i16* %g, i16 2
+  %0 = load i16, i16* %add.ptr, align 2
+  %shr = lshr i16 %0, 1
+  store i16 %shr, i16* %add.ptr, align 2
+  ret void
+}
+
+define void @sxt16m(i16* %x) {
+entry:
+; CHECK-LABEL: sxt16m:
+; CHECK: sxt 4(r12)
+  %add.ptr = getelementptr inbounds i16, i16* %x, i16 2
+  %0 = bitcast i16* %add.ptr to i8*
+  %1 = load i8, i8* %0, align 1
+  %conv = sext i8 %1 to i16
+  store i16 %conv, i16* %add.ptr, align 2
+  ret void
+}
+
diff --git a/test/CodeGen/MSP430/fp.ll b/test/CodeGen/MSP430/fp.ll
index 87c4055829c8801c269e3538775e5eeb6ea28643..bf603704a91b688bb21731295a0ded3fd9824d12 100644
--- a/test/CodeGen/MSP430/fp.ll
+++ b/test/CodeGen/MSP430/fp.ll
@@ -1,4 +1,4 @@
-; RUN: llc -O0 -disable-fp-elim < %s | FileCheck %s
+; RUN: llc -O0 -frame-pointer=all < %s | FileCheck %s
 
 target datalayout = "e-p:16:16:16-i8:8:8-i16:16:16-i32:16:32-n8:16"
 target triple = "msp430---elf"
@@ -27,3 +27,5 @@ define msp430_intrcc void @fpb_alloced() #0 {
   call void asm sideeffect "nop", "r"(i8 0)
   ret void
 }
+
+attributes #0 = { noinline nounwind optnone "interrupt"="2" }
diff --git a/test/CodeGen/MSP430/inline-asm-absolute-addressing.ll b/test/CodeGen/MSP430/inline-asm-absolute-addressing.ll
new file mode 100644
index 0000000000000000000000000000000000000000..91505dca48ddfbc53946a2101208d54288b06cb7
--- /dev/null
+++ b/test/CodeGen/MSP430/inline-asm-absolute-addressing.ll
@@ -0,0 +1,15 @@
+; RUN: llc < %s | FileCheck %s
+
+; Check that absolute addressing mode is represented in a way
+; defined in MSP430 EABI and not as indexed addressing mode form.
+; See PR39993 for details.
+
+target datalayout = "e-p:16:8:8-i8:8:8-i16:8:8-i32:8:8-n8:16"
+target triple = "msp430-elf"
+
+define void @f() {
+entry:
+; CHECK: mov r1, &256
+  call void asm sideeffect "mov r1, $0", "*m"(i8* inttoptr (i16 256 to i8*))
+  ret void
+}
diff --git a/test/CodeGen/MSP430/interrupt.ll b/test/CodeGen/MSP430/interrupt.ll
index 5fa0c849c2602a6e4e01392e663a855739cb415a..94fb3bc457a356408ae997f28cb9cded3d01065f 100644
--- a/test/CodeGen/MSP430/interrupt.ll
+++ b/test/CodeGen/MSP430/interrupt.ll
@@ -13,6 +13,9 @@ target triple = "msp430-generic-generic"
 ; instruction RETI, which restores the SR register and branches to the PC where
 ; the interrupt occurred.
 
+; CHECK:      .section	__interrupt_vector_2,"ax",@progbits
+; CHECK-NEXT:	.short	ISR
+
 @g = global float 0.0
 
 define msp430_intrcc void @ISR() #0 {
@@ -47,3 +50,4 @@ entry:
   ret void
 }
 
+attributes #0 = { noinline nounwind optnone "interrupt"="2" }
diff --git a/test/CodeGen/MSP430/mult-alt-generic-msp430.ll b/test/CodeGen/MSP430/mult-alt-generic-msp430.ll
index 8cf83879b090f9eb196680775d05c842f4dbb86c..f8ae49e197b7b1cf4abae38affb3a7273bb868ca 100644
--- a/test/CodeGen/MSP430/mult-alt-generic-msp430.ll
+++ b/test/CodeGen/MSP430/mult-alt-generic-msp430.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -march=msp430
+; RUN: llc < %s -march=msp430 -no-integrated-as
 ; ModuleID = 'mult-alt-generic.c'
 target datalayout = "e-p:16:16:16-i8:8:8-i16:16:16-i32:16:32-n8:16"
 target triple = "msp430"
diff --git a/test/CodeGen/MSP430/shifts.ll b/test/CodeGen/MSP430/shifts.ll
index 073251943d01b0074246a2b151b8b9d90bd606c6..5a1fc9c324bb8df07c0b55fe7458ac19df1ec2c1 100644
--- a/test/CodeGen/MSP430/shifts.ll
+++ b/test/CodeGen/MSP430/shifts.ll
@@ -74,3 +74,14 @@ entry:
   %shr = lshr i16 %a, 10
   ret i16 %shr
 }
+
+define i16 @lshl10_i16(i16 %a) #0 {
+entry:
+; CHECK-LABEL: lshl10_i16:
+; CHECK:      mov.b r12, r12
+; CHECK-NEXT: swpb r12
+; CHECK-NEXT: add r12, r12
+; CHECK-NEXT: add r12, r12
+  %shl = shl i16 %a, 10
+  ret i16 %shl
+}
diff --git a/test/CodeGen/MSP430/stacksave_restore.ll b/test/CodeGen/MSP430/stacksave_restore.ll
new file mode 100644
index 0000000000000000000000000000000000000000..47c4553929d4b545eeadf21deed8d1ed5254cc51
--- /dev/null
+++ b/test/CodeGen/MSP430/stacksave_restore.ll
@@ -0,0 +1,13 @@
+; RUN: llc < %s -march=msp430
+
+target triple = "msp430"
+
+define void @foo() {
+entry:
+  %0 = tail call i8* @llvm.stacksave()
+  tail call void @llvm.stackrestore(i8* %0)
+  ret void
+}
+
+declare i8* @llvm.stacksave()
+declare void @llvm.stackrestore(i8*)
diff --git a/test/CodeGen/Mips/Fast-ISel/stackloadstore.ll b/test/CodeGen/Mips/Fast-ISel/stackloadstore.ll
index 421f8ff2dba0609816f724b72edfebdee05b8d53..982284b64f6630c4232d87d7618e6a70a5a1372d 100644
--- a/test/CodeGen/Mips/Fast-ISel/stackloadstore.ll
+++ b/test/CodeGen/Mips/Fast-ISel/stackloadstore.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -march=mipsel -mcpu=mips32 -fast-isel -disable-fp-elim -relocation-model=pic < %s
+; RUN: llc < %s -march=mipsel -mcpu=mips32 -fast-isel -frame-pointer=all -relocation-model=pic < %s
 
 ; Test that negative array access don't crash constant synthesis when fast isel
 ; generates negative offsets.
diff --git a/test/CodeGen/Mips/GlobalISel/instruction-select/mul.mir b/test/CodeGen/Mips/GlobalISel/instruction-select/mul.mir
new file mode 100644
index 0000000000000000000000000000000000000000..507c33520a80ba8526853ff97ba3324e1dd1f54c
--- /dev/null
+++ b/test/CodeGen/Mips/GlobalISel/instruction-select/mul.mir
@@ -0,0 +1,31 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -O0 -mtriple=mipsel-linux-gnu -run-pass=instruction-select -verify-machineinstrs %s -o - | FileCheck %s -check-prefixes=MIPS32
+--- |
+
+  define void @mul_i32(i32 %x, i32 %y) {entry: ret void}
+
+...
+---
+name:            mul_i32
+alignment:       2
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $a0, $a1
+
+    ; MIPS32-LABEL: name: mul_i32
+    ; MIPS32: liveins: $a0, $a1
+    ; MIPS32: [[COPY:%[0-9]+]]:gpr32 = COPY $a0
+    ; MIPS32: [[COPY1:%[0-9]+]]:gpr32 = COPY $a1
+    ; MIPS32: [[MUL:%[0-9]+]]:gpr32 = MUL [[COPY]], [[COPY1]], implicit-def $hi0, implicit-def $lo0
+    ; MIPS32: $v0 = COPY [[MUL]]
+    ; MIPS32: RetRA implicit $v0
+    %0:gprb(s32) = COPY $a0
+    %1:gprb(s32) = COPY $a1
+    %2:gprb(s32) = G_MUL %0, %1
+    $v0 = COPY %2(s32)
+    RetRA implicit $v0
+
+...
diff --git a/test/CodeGen/Mips/GlobalISel/instruction-select/pointers.mir b/test/CodeGen/Mips/GlobalISel/instruction-select/pointers.mir
index 9f469ea8913aaf1867444c3adbdf137ef514101e..b5dce811994766e8900b96e47a6a9fadc47b231d 100644
--- a/test/CodeGen/Mips/GlobalISel/instruction-select/pointers.mir
+++ b/test/CodeGen/Mips/GlobalISel/instruction-select/pointers.mir
@@ -44,7 +44,7 @@ body:             |
     ; MIPS32-LABEL: name: ptr_arg_on_stack
     ; MIPS32: liveins: $a0, $a1, $a2, $a3
     ; MIPS32: [[ADDiu:%[0-9]+]]:gpr32 = ADDiu %fixed-stack.0, 0
-    ; MIPS32: [[LW:%[0-9]+]]:gpr32 = LW [[ADDiu]], 0 :: (load 4 from %fixed-stack.0, align 0)
+    ; MIPS32: [[LW:%[0-9]+]]:gpr32 = LW [[ADDiu]], 0 :: (load 4 from %fixed-stack.0, align 8)
     ; MIPS32: [[LW1:%[0-9]+]]:gpr32 = LW [[LW]], 0 :: (load 4 from %ir.p)
     ; MIPS32: $v0 = COPY [[LW1]]
     ; MIPS32: RetRA implicit $v0
@@ -53,7 +53,7 @@ body:             |
     %2:gprb(s32) = COPY $a2
     %3:gprb(s32) = COPY $a3
     %5:gprb(p0) = G_FRAME_INDEX %fixed-stack.0
-    %4:gprb(p0) = G_LOAD %5(p0) :: (load 4 from %fixed-stack.0, align 0)
+    %4:gprb(p0) = G_LOAD %5(p0) :: (load 4 from %fixed-stack.0, align 8)
     %6:gprb(s32) = G_LOAD %4(p0) :: (load 4 from %ir.p)
     $v0 = COPY %6(s32)
     RetRA implicit $v0
diff --git a/test/CodeGen/Mips/GlobalISel/instruction-select/select.mir b/test/CodeGen/Mips/GlobalISel/instruction-select/select.mir
new file mode 100644
index 0000000000000000000000000000000000000000..38f90ae2d8a4177061aec5fe2226be086e26febe
--- /dev/null
+++ b/test/CodeGen/Mips/GlobalISel/instruction-select/select.mir
@@ -0,0 +1,72 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -O0 -mtriple=mipsel-linux-gnu -run-pass=instruction-select -verify-machineinstrs %s -o - | FileCheck %s -check-prefixes=MIPS32
+--- |
+
+  define void @select_i32() {entry: ret void}
+  define void @select_ptr() {entry: ret void}
+
+...
+---
+name:            select_i32
+alignment:       2
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.1.entry:
+    liveins: $a0, $a1, $a2
+
+    ; MIPS32-LABEL: name: select_i32
+    ; MIPS32: liveins: $a0, $a1, $a2
+    ; MIPS32: [[COPY:%[0-9]+]]:gpr32 = COPY $a0
+    ; MIPS32: [[COPY1:%[0-9]+]]:gpr32 = COPY $a1
+    ; MIPS32: [[COPY2:%[0-9]+]]:gpr32 = COPY $a2
+    ; MIPS32: [[LUi:%[0-9]+]]:gpr32 = LUi 0
+    ; MIPS32: [[ORi:%[0-9]+]]:gpr32 = ORi [[LUi]], 1
+    ; MIPS32: [[AND:%[0-9]+]]:gpr32 = AND [[COPY]], [[ORi]]
+    ; MIPS32: [[MOVN_I_I:%[0-9]+]]:gpr32 = MOVN_I_I [[COPY1]], [[AND]], [[COPY2]]
+    ; MIPS32: $v0 = COPY [[MOVN_I_I]]
+    ; MIPS32: RetRA implicit $v0
+    %3:gprb(s32) = COPY $a0
+    %1:gprb(s32) = COPY $a1
+    %2:gprb(s32) = COPY $a2
+    %6:gprb(s32) = G_CONSTANT i32 1
+    %7:gprb(s32) = COPY %3(s32)
+    %5:gprb(s32) = G_AND %7, %6
+    %4:gprb(s32) = G_SELECT %5(s32), %1, %2
+    $v0 = COPY %4(s32)
+    RetRA implicit $v0
+
+...
+---
+name:            select_ptr
+alignment:       2
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.1.entry:
+    liveins: $a0, $a1, $a2
+
+    ; MIPS32-LABEL: name: select_ptr
+    ; MIPS32: liveins: $a0, $a1, $a2
+    ; MIPS32: [[COPY:%[0-9]+]]:gpr32 = COPY $a0
+    ; MIPS32: [[COPY1:%[0-9]+]]:gpr32 = COPY $a1
+    ; MIPS32: [[COPY2:%[0-9]+]]:gpr32 = COPY $a2
+    ; MIPS32: [[LUi:%[0-9]+]]:gpr32 = LUi 0
+    ; MIPS32: [[ORi:%[0-9]+]]:gpr32 = ORi [[LUi]], 1
+    ; MIPS32: [[AND:%[0-9]+]]:gpr32 = AND [[COPY]], [[ORi]]
+    ; MIPS32: [[MOVN_I_I:%[0-9]+]]:gpr32 = MOVN_I_I [[COPY1]], [[AND]], [[COPY2]]
+    ; MIPS32: $v0 = COPY [[MOVN_I_I]]
+    ; MIPS32: RetRA implicit $v0
+    %3:gprb(s32) = COPY $a0
+    %1:gprb(p0) = COPY $a1
+    %2:gprb(p0) = COPY $a2
+    %6:gprb(s32) = G_CONSTANT i32 1
+    %7:gprb(s32) = COPY %3(s32)
+    %5:gprb(s32) = G_AND %7, %6
+    %4:gprb(p0) = G_SELECT %5(s32), %1, %2
+    $v0 = COPY %4(p0)
+    RetRA implicit $v0
+
+...
diff --git a/test/CodeGen/Mips/GlobalISel/instruction-select/stack_args.mir b/test/CodeGen/Mips/GlobalISel/instruction-select/stack_args.mir
index f93be8db70307e5a6984e7be83db9a4951067c43..114432a8aa912f887699e6aae947c9c62e25f2e4 100644
--- a/test/CodeGen/Mips/GlobalISel/instruction-select/stack_args.mir
+++ b/test/CodeGen/Mips/GlobalISel/instruction-select/stack_args.mir
@@ -25,7 +25,7 @@ body:             |
     ; MIPS32: [[COPY2:%[0-9]+]]:gpr32 = COPY $a2
     ; MIPS32: [[COPY3:%[0-9]+]]:gpr32 = COPY $a3
     ; MIPS32: [[ADDiu:%[0-9]+]]:gpr32 = ADDiu %fixed-stack.0, 0
-    ; MIPS32: [[LW:%[0-9]+]]:gpr32 = LW [[ADDiu]], 0 :: (load 4 from %fixed-stack.0, align 0)
+    ; MIPS32: [[LW:%[0-9]+]]:gpr32 = LW [[ADDiu]], 0 :: (load 4 from %fixed-stack.0, align 8)
     ; MIPS32: ADJCALLSTACKDOWN 24, 0, implicit-def $sp, implicit $sp
     ; MIPS32: $a0 = COPY [[COPY]]
     ; MIPS32: $a1 = COPY [[COPY1]]
@@ -35,7 +35,7 @@ body:             |
     ; MIPS32: [[LUi:%[0-9]+]]:gpr32 = LUi 0
     ; MIPS32: [[ORi:%[0-9]+]]:gpr32 = ORi [[LUi]], 16
     ; MIPS32: [[ADDu:%[0-9]+]]:gpr32 = ADDu [[COPY4]], [[ORi]]
-    ; MIPS32: SW [[LW]], [[ADDu]], 0 :: (store 4 into stack + 16, align 0)
+    ; MIPS32: SW [[LW]], [[ADDu]], 0 :: (store 4 into stack + 16)
     ; MIPS32: JAL @f, csr_o32, implicit-def $ra, implicit-def $sp, implicit $a0, implicit $a1, implicit $a2, implicit $a3, implicit-def $v0
     ; MIPS32: [[COPY5:%[0-9]+]]:gpr32 = COPY $v0
     ; MIPS32: ADJCALLSTACKUP 24, 0, implicit-def $sp, implicit $sp
@@ -46,7 +46,7 @@ body:             |
     %2:gprb(s32) = COPY $a2
     %3:gprb(s32) = COPY $a3
     %5:gprb(p0) = G_FRAME_INDEX %fixed-stack.0
-    %4:gprb(s32) = G_LOAD %5(p0) :: (load 4 from %fixed-stack.0, align 0)
+    %4:gprb(s32) = G_LOAD %5(p0) :: (load 4 from %fixed-stack.0, align 8)
     ADJCALLSTACKDOWN 24, 0, implicit-def $sp, implicit $sp
     $a0 = COPY %0(s32)
     $a1 = COPY %1(s32)
@@ -55,7 +55,7 @@ body:             |
     %7:gprb(p0) = COPY $sp
     %8:gprb(s32) = G_CONSTANT i32 16
     %9:gprb(p0) = G_GEP %7, %8(s32)
-    G_STORE %4(s32), %9(p0) :: (store 4 into stack + 16, align 0)
+    G_STORE %4(s32), %9(p0) :: (store 4 into stack + 16, align 4)
     JAL @f, csr_o32, implicit-def $ra, implicit-def $sp, implicit $a0, implicit $a1, implicit $a2, implicit $a3, implicit-def $v0
     %6:gprb(s32) = COPY $v0
     ADJCALLSTACKUP 24, 0, implicit-def $sp, implicit $sp
diff --git a/test/CodeGen/Mips/GlobalISel/instruction-select/sub.mir b/test/CodeGen/Mips/GlobalISel/instruction-select/sub.mir
new file mode 100644
index 0000000000000000000000000000000000000000..911073a7f8d4fd7297eda0ddb0bfdc41d8503e82
--- /dev/null
+++ b/test/CodeGen/Mips/GlobalISel/instruction-select/sub.mir
@@ -0,0 +1,31 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -O0 -mtriple=mipsel-linux-gnu -run-pass=instruction-select -verify-machineinstrs %s -o - | FileCheck %s -check-prefixes=MIPS32
+--- |
+
+  define void @sub_i32(i32 %x, i32 %y) {entry: ret void}
+
+...
+---
+name:            sub_i32
+alignment:       2
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $a0, $a1
+
+    ; MIPS32-LABEL: name: sub_i32
+    ; MIPS32: liveins: $a0, $a1
+    ; MIPS32: [[COPY:%[0-9]+]]:gpr32 = COPY $a0
+    ; MIPS32: [[COPY1:%[0-9]+]]:gpr32 = COPY $a1
+    ; MIPS32: [[SUBu:%[0-9]+]]:gpr32 = SUBu [[COPY]], [[COPY1]]
+    ; MIPS32: $v0 = COPY [[SUBu]]
+    ; MIPS32: RetRA implicit $v0
+    %0:gprb(s32) = COPY $a0
+    %1:gprb(s32) = COPY $a1
+    %2:gprb(s32) = G_SUB %0, %1
+    $v0 = COPY %2(s32)
+    RetRA implicit $v0
+
+...
diff --git a/test/CodeGen/Mips/GlobalISel/instruction-select/truncStore_and_aExtLoad.mir b/test/CodeGen/Mips/GlobalISel/instruction-select/truncStore_and_aExtLoad.mir
new file mode 100644
index 0000000000000000000000000000000000000000..4e700bb3f1a31ef00b12c482f951b2b03ed23a19
--- /dev/null
+++ b/test/CodeGen/Mips/GlobalISel/instruction-select/truncStore_and_aExtLoad.mir
@@ -0,0 +1,83 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -O0 -mtriple=mipsel-linux-gnu -run-pass=instruction-select -verify-machineinstrs %s -o - | FileCheck %s -check-prefixes=MIPS32
+--- |
+
+  define void @load_store_i8(i8* %px, i8* %py) {entry: ret void}
+  define void @load_store_i16(i16* %px, i16* %py) {entry: ret void}
+  define void @load_store_i32(i32* %px, i32* %py) {entry: ret void}
+
+...
+---
+name:            load_store_i8
+alignment:       2
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.1.entry:
+    liveins: $a0, $a1
+
+    ; MIPS32-LABEL: name: load_store_i8
+    ; MIPS32: liveins: $a0, $a1
+    ; MIPS32: [[COPY:%[0-9]+]]:gpr32 = COPY $a0
+    ; MIPS32: [[COPY1:%[0-9]+]]:gpr32 = COPY $a1
+    ; MIPS32: [[LBu:%[0-9]+]]:gpr32 = LBu [[COPY1]], 0 :: (load 1 from %ir.py)
+    ; MIPS32: SB [[LBu]], [[COPY]], 0 :: (store 1 into %ir.px)
+    ; MIPS32: RetRA
+    %0:gprb(p0) = COPY $a0
+    %1:gprb(p0) = COPY $a1
+    %4:gprb(s32) = G_LOAD %1(p0) :: (load 1 from %ir.py)
+    %3:gprb(s32) = COPY %4(s32)
+    G_STORE %3(s32), %0(p0) :: (store 1 into %ir.px)
+    RetRA
+
+...
+---
+name:            load_store_i16
+alignment:       2
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.1.entry:
+    liveins: $a0, $a1
+
+    ; MIPS32-LABEL: name: load_store_i16
+    ; MIPS32: liveins: $a0, $a1
+    ; MIPS32: [[COPY:%[0-9]+]]:gpr32 = COPY $a0
+    ; MIPS32: [[COPY1:%[0-9]+]]:gpr32 = COPY $a1
+    ; MIPS32: [[LHu:%[0-9]+]]:gpr32 = LHu [[COPY1]], 0 :: (load 2 from %ir.py)
+    ; MIPS32: SH [[LHu]], [[COPY]], 0 :: (store 2 into %ir.px)
+    ; MIPS32: RetRA
+    %0:gprb(p0) = COPY $a0
+    %1:gprb(p0) = COPY $a1
+    %4:gprb(s32) = G_LOAD %1(p0) :: (load 2 from %ir.py)
+    %3:gprb(s32) = COPY %4(s32)
+    G_STORE %3(s32), %0(p0) :: (store 2 into %ir.px)
+    RetRA
+
+...
+---
+name:            load_store_i32
+alignment:       2
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.1.entry:
+    liveins: $a0, $a1
+
+    ; MIPS32-LABEL: name: load_store_i32
+    ; MIPS32: liveins: $a0, $a1
+    ; MIPS32: [[COPY:%[0-9]+]]:gpr32 = COPY $a0
+    ; MIPS32: [[COPY1:%[0-9]+]]:gpr32 = COPY $a1
+    ; MIPS32: [[LW:%[0-9]+]]:gpr32 = LW [[COPY1]], 0 :: (load 4 from %ir.py)
+    ; MIPS32: SW [[LW]], [[COPY]], 0 :: (store 4 into %ir.px)
+    ; MIPS32: RetRA
+    %0:gprb(p0) = COPY $a0
+    %1:gprb(p0) = COPY $a1
+    %2:gprb(s32) = G_LOAD %1(p0) :: (load 4 from %ir.py)
+    G_STORE %2(s32), %0(p0) :: (store 4 into %ir.px)
+    RetRA
+
+...
diff --git a/test/CodeGen/Mips/GlobalISel/instruction-select/zextLoad_and_sextLoad.mir b/test/CodeGen/Mips/GlobalISel/instruction-select/zextLoad_and_sextLoad.mir
new file mode 100644
index 0000000000000000000000000000000000000000..168bc53309dc1cd481bd7085e4697177c3a46266
--- /dev/null
+++ b/test/CodeGen/Mips/GlobalISel/instruction-select/zextLoad_and_sextLoad.mir
@@ -0,0 +1,98 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -O0 -mtriple=mipsel-linux-gnu -run-pass=instruction-select -verify-machineinstrs %s -o - | FileCheck %s -check-prefixes=MIPS32
+--- |
+
+  define void @load1_s8_to_zextLoad1_s32(i8* %px) {entry: ret void}
+  define void @load2_s16_to_zextLoad2_s32(i16* %px) {entry: ret void}
+  define void @load1_s8_to_sextLoad1_s32(i8* %px) {entry: ret void}
+  define void @load2_s16_to_sextLoad2_s32(i16* %px) {entry: ret void}
+
+...
+---
+name:            load1_s8_to_zextLoad1_s32
+alignment:       2
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.1.entry:
+    liveins: $a0
+
+    ; MIPS32-LABEL: name: load1_s8_to_zextLoad1_s32
+    ; MIPS32: liveins: $a0
+    ; MIPS32: [[COPY:%[0-9]+]]:gpr32 = COPY $a0
+    ; MIPS32: [[LBu:%[0-9]+]]:gpr32 = LBu [[COPY]], 0 :: (load 1 from %ir.px)
+    ; MIPS32: $v0 = COPY [[LBu]]
+    ; MIPS32: RetRA implicit $v0
+    %0:gprb(p0) = COPY $a0
+    %2:gprb(s32) = G_ZEXTLOAD %0(p0) :: (load 1 from %ir.px)
+    $v0 = COPY %2(s32)
+    RetRA implicit $v0
+
+...
+---
+name:            load2_s16_to_zextLoad2_s32
+alignment:       2
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.1.entry:
+    liveins: $a0
+
+    ; MIPS32-LABEL: name: load2_s16_to_zextLoad2_s32
+    ; MIPS32: liveins: $a0
+    ; MIPS32: [[COPY:%[0-9]+]]:gpr32 = COPY $a0
+    ; MIPS32: [[LHu:%[0-9]+]]:gpr32 = LHu [[COPY]], 0 :: (load 2 from %ir.px)
+    ; MIPS32: $v0 = COPY [[LHu]]
+    ; MIPS32: RetRA implicit $v0
+    %0:gprb(p0) = COPY $a0
+    %2:gprb(s32) = G_ZEXTLOAD %0(p0) :: (load 2 from %ir.px)
+    $v0 = COPY %2(s32)
+    RetRA implicit $v0
+
+...
+---
+name:            load1_s8_to_sextLoad1_s32
+alignment:       2
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.1.entry:
+    liveins: $a0
+
+    ; MIPS32-LABEL: name: load1_s8_to_sextLoad1_s32
+    ; MIPS32: liveins: $a0
+    ; MIPS32: [[COPY:%[0-9]+]]:gpr32 = COPY $a0
+    ; MIPS32: [[LB:%[0-9]+]]:gpr32 = LB [[COPY]], 0 :: (load 1 from %ir.px)
+    ; MIPS32: $v0 = COPY [[LB]]
+    ; MIPS32: RetRA implicit $v0
+    %0:gprb(p0) = COPY $a0
+    %2:gprb(s32) = G_SEXTLOAD %0(p0) :: (load 1 from %ir.px)
+    $v0 = COPY %2(s32)
+    RetRA implicit $v0
+
+...
+---
+name:            load2_s16_to_sextLoad2_s32
+alignment:       2
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+body:             |
+  bb.1.entry:
+    liveins: $a0
+
+    ; MIPS32-LABEL: name: load2_s16_to_sextLoad2_s32
+    ; MIPS32: liveins: $a0
+    ; MIPS32: [[COPY:%[0-9]+]]:gpr32 = COPY $a0
+    ; MIPS32: [[LH:%[0-9]+]]:gpr32 = LH [[COPY]], 0 :: (load 2 from %ir.px)
+    ; MIPS32: $v0 = COPY [[LH]]
+    ; MIPS32: RetRA implicit $v0
+    %0:gprb(p0) = COPY $a0
+    %2:gprb(s32) = G_SEXTLOAD %0(p0) :: (load 2 from %ir.px)
+    $v0 = COPY %2(s32)
+    RetRA implicit $v0
+
+...
diff --git a/test/CodeGen/Mips/GlobalISel/irtranslator/extend_args.ll b/test/CodeGen/Mips/GlobalISel/irtranslator/extend_args.ll
index f8cf2c212393714d7ab4f10b7cd082552706a766..a1743c4c7c865b856c04aebd3f24b1edad82fc8c 100644
--- a/test/CodeGen/Mips/GlobalISel/irtranslator/extend_args.ll
+++ b/test/CodeGen/Mips/GlobalISel/irtranslator/extend_args.ll
@@ -54,7 +54,7 @@ define signext i8 @call_sext_stack_arg_i8(i32 %x1, i32 %x2, i32 %x3, i32 %x4, i8
   ; MIPS32:   [[COPY2:%[0-9]+]]:_(s32) = COPY $a2
   ; MIPS32:   [[COPY3:%[0-9]+]]:_(s32) = COPY $a3
   ; MIPS32:   [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.0
-  ; MIPS32:   [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX]](p0) :: (load 4 from %fixed-stack.0, align 0)
+  ; MIPS32:   [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX]](p0) :: (load 4 from %fixed-stack.0, align 8)
   ; MIPS32:   [[TRUNC:%[0-9]+]]:_(s8) = G_TRUNC [[LOAD]](s32)
   ; MIPS32:   ADJCALLSTACKDOWN 24, 0, implicit-def $sp, implicit $sp
   ; MIPS32:   $a0 = COPY [[COPY]](s32)
@@ -65,7 +65,7 @@ define signext i8 @call_sext_stack_arg_i8(i32 %x1, i32 %x2, i32 %x3, i32 %x4, i8
   ; MIPS32:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
   ; MIPS32:   [[GEP:%[0-9]+]]:_(p0) = G_GEP [[COPY4]], [[C]](s32)
   ; MIPS32:   [[SEXT:%[0-9]+]]:_(s32) = G_SEXT [[TRUNC]](s8)
-  ; MIPS32:   G_STORE [[SEXT]](s32), [[GEP]](p0) :: (store 4 into stack + 16, align 0)
+  ; MIPS32:   G_STORE [[SEXT]](s32), [[GEP]](p0) :: (store 4 into stack + 16, align 8)
   ; MIPS32:   JAL @sext_stack_arg_i8, csr_o32, implicit-def $ra, implicit-def $sp, implicit $a0, implicit $a1, implicit $a2, implicit $a3, implicit-def $v0
   ; MIPS32:   [[COPY5:%[0-9]+]]:_(s32) = COPY $v0
   ; MIPS32:   [[TRUNC1:%[0-9]+]]:_(s8) = G_TRUNC [[COPY5]](s32)
@@ -87,7 +87,7 @@ define zeroext i8 @call_zext_stack_arg_i8(i32 %x1, i32 %x2, i32 %x3, i32 %x4, i8
   ; MIPS32:   [[COPY2:%[0-9]+]]:_(s32) = COPY $a2
   ; MIPS32:   [[COPY3:%[0-9]+]]:_(s32) = COPY $a3
   ; MIPS32:   [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.0
-  ; MIPS32:   [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX]](p0) :: (load 4 from %fixed-stack.0, align 0)
+  ; MIPS32:   [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX]](p0) :: (load 4 from %fixed-stack.0, align 8)
   ; MIPS32:   [[TRUNC:%[0-9]+]]:_(s8) = G_TRUNC [[LOAD]](s32)
   ; MIPS32:   ADJCALLSTACKDOWN 24, 0, implicit-def $sp, implicit $sp
   ; MIPS32:   $a0 = COPY [[COPY]](s32)
@@ -98,7 +98,7 @@ define zeroext i8 @call_zext_stack_arg_i8(i32 %x1, i32 %x2, i32 %x3, i32 %x4, i8
   ; MIPS32:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
   ; MIPS32:   [[GEP:%[0-9]+]]:_(p0) = G_GEP [[COPY4]], [[C]](s32)
   ; MIPS32:   [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[TRUNC]](s8)
-  ; MIPS32:   G_STORE [[ZEXT]](s32), [[GEP]](p0) :: (store 4 into stack + 16, align 0)
+  ; MIPS32:   G_STORE [[ZEXT]](s32), [[GEP]](p0) :: (store 4 into stack + 16, align 8)
   ; MIPS32:   JAL @zext_stack_arg_i8, csr_o32, implicit-def $ra, implicit-def $sp, implicit $a0, implicit $a1, implicit $a2, implicit $a3, implicit-def $v0
   ; MIPS32:   [[COPY5:%[0-9]+]]:_(s32) = COPY $v0
   ; MIPS32:   [[TRUNC1:%[0-9]+]]:_(s8) = G_TRUNC [[COPY5]](s32)
@@ -120,7 +120,7 @@ define i8 @call_aext_stack_arg_i8(i32 %x1, i32 %x2, i32 %x3, i32 %x4, i8 %a) {
   ; MIPS32:   [[COPY2:%[0-9]+]]:_(s32) = COPY $a2
   ; MIPS32:   [[COPY3:%[0-9]+]]:_(s32) = COPY $a3
   ; MIPS32:   [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.0
-  ; MIPS32:   [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX]](p0) :: (load 4 from %fixed-stack.0, align 0)
+  ; MIPS32:   [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX]](p0) :: (load 4 from %fixed-stack.0, align 8)
   ; MIPS32:   [[TRUNC:%[0-9]+]]:_(s8) = G_TRUNC [[LOAD]](s32)
   ; MIPS32:   ADJCALLSTACKDOWN 24, 0, implicit-def $sp, implicit $sp
   ; MIPS32:   $a0 = COPY [[COPY]](s32)
@@ -131,7 +131,7 @@ define i8 @call_aext_stack_arg_i8(i32 %x1, i32 %x2, i32 %x3, i32 %x4, i8 %a) {
   ; MIPS32:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
   ; MIPS32:   [[GEP:%[0-9]+]]:_(p0) = G_GEP [[COPY4]], [[C]](s32)
   ; MIPS32:   [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[TRUNC]](s8)
-  ; MIPS32:   G_STORE [[ANYEXT]](s32), [[GEP]](p0) :: (store 4 into stack + 16, align 0)
+  ; MIPS32:   G_STORE [[ANYEXT]](s32), [[GEP]](p0) :: (store 4 into stack + 16, align 8)
   ; MIPS32:   JAL @aext_stack_arg_i8, csr_o32, implicit-def $ra, implicit-def $sp, implicit $a0, implicit $a1, implicit $a2, implicit $a3, implicit-def $v0
   ; MIPS32:   [[COPY5:%[0-9]+]]:_(s32) = COPY $v0
   ; MIPS32:   [[TRUNC1:%[0-9]+]]:_(s8) = G_TRUNC [[COPY5]](s32)
@@ -197,7 +197,7 @@ define signext i16 @call_sext_stack_arg_i16(i32 %x1, i32 %x2, i32 %x3, i32 %x4,
   ; MIPS32:   [[COPY2:%[0-9]+]]:_(s32) = COPY $a2
   ; MIPS32:   [[COPY3:%[0-9]+]]:_(s32) = COPY $a3
   ; MIPS32:   [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.0
-  ; MIPS32:   [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX]](p0) :: (load 4 from %fixed-stack.0, align 0)
+  ; MIPS32:   [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX]](p0) :: (load 4 from %fixed-stack.0, align 8)
   ; MIPS32:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD]](s32)
   ; MIPS32:   ADJCALLSTACKDOWN 24, 0, implicit-def $sp, implicit $sp
   ; MIPS32:   $a0 = COPY [[COPY]](s32)
@@ -208,7 +208,7 @@ define signext i16 @call_sext_stack_arg_i16(i32 %x1, i32 %x2, i32 %x3, i32 %x4,
   ; MIPS32:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
   ; MIPS32:   [[GEP:%[0-9]+]]:_(p0) = G_GEP [[COPY4]], [[C]](s32)
   ; MIPS32:   [[SEXT:%[0-9]+]]:_(s32) = G_SEXT [[TRUNC]](s16)
-  ; MIPS32:   G_STORE [[SEXT]](s32), [[GEP]](p0) :: (store 4 into stack + 16, align 0)
+  ; MIPS32:   G_STORE [[SEXT]](s32), [[GEP]](p0) :: (store 4 into stack + 16, align 8)
   ; MIPS32:   JAL @sext_stack_arg_i16, csr_o32, implicit-def $ra, implicit-def $sp, implicit $a0, implicit $a1, implicit $a2, implicit $a3, implicit-def $v0
   ; MIPS32:   [[COPY5:%[0-9]+]]:_(s32) = COPY $v0
   ; MIPS32:   [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY5]](s32)
@@ -230,7 +230,7 @@ define zeroext i16 @call_zext_stack_arg_i16(i32 %x1, i32 %x2, i32 %x3, i32 %x4,
   ; MIPS32:   [[COPY2:%[0-9]+]]:_(s32) = COPY $a2
   ; MIPS32:   [[COPY3:%[0-9]+]]:_(s32) = COPY $a3
   ; MIPS32:   [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.0
-  ; MIPS32:   [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX]](p0) :: (load 4 from %fixed-stack.0, align 0)
+  ; MIPS32:   [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX]](p0) :: (load 4 from %fixed-stack.0, align 8)
   ; MIPS32:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD]](s32)
   ; MIPS32:   ADJCALLSTACKDOWN 24, 0, implicit-def $sp, implicit $sp
   ; MIPS32:   $a0 = COPY [[COPY]](s32)
@@ -241,7 +241,7 @@ define zeroext i16 @call_zext_stack_arg_i16(i32 %x1, i32 %x2, i32 %x3, i32 %x4,
   ; MIPS32:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
   ; MIPS32:   [[GEP:%[0-9]+]]:_(p0) = G_GEP [[COPY4]], [[C]](s32)
   ; MIPS32:   [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[TRUNC]](s16)
-  ; MIPS32:   G_STORE [[ZEXT]](s32), [[GEP]](p0) :: (store 4 into stack + 16, align 0)
+  ; MIPS32:   G_STORE [[ZEXT]](s32), [[GEP]](p0) :: (store 4 into stack + 16, align 8)
   ; MIPS32:   JAL @zext_stack_arg_i16, csr_o32, implicit-def $ra, implicit-def $sp, implicit $a0, implicit $a1, implicit $a2, implicit $a3, implicit-def $v0
   ; MIPS32:   [[COPY5:%[0-9]+]]:_(s32) = COPY $v0
   ; MIPS32:   [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY5]](s32)
@@ -263,7 +263,7 @@ define i16 @call_aext_stack_arg_i16(i32 %x1, i32 %x2, i32 %x3, i32 %x4, i16 %a)
   ; MIPS32:   [[COPY2:%[0-9]+]]:_(s32) = COPY $a2
   ; MIPS32:   [[COPY3:%[0-9]+]]:_(s32) = COPY $a3
   ; MIPS32:   [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.0
-  ; MIPS32:   [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX]](p0) :: (load 4 from %fixed-stack.0, align 0)
+  ; MIPS32:   [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX]](p0) :: (load 4 from %fixed-stack.0, align 8)
   ; MIPS32:   [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LOAD]](s32)
   ; MIPS32:   ADJCALLSTACKDOWN 24, 0, implicit-def $sp, implicit $sp
   ; MIPS32:   $a0 = COPY [[COPY]](s32)
@@ -274,7 +274,7 @@ define i16 @call_aext_stack_arg_i16(i32 %x1, i32 %x2, i32 %x3, i32 %x4, i16 %a)
   ; MIPS32:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
   ; MIPS32:   [[GEP:%[0-9]+]]:_(p0) = G_GEP [[COPY4]], [[C]](s32)
   ; MIPS32:   [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[TRUNC]](s16)
-  ; MIPS32:   G_STORE [[ANYEXT]](s32), [[GEP]](p0) :: (store 4 into stack + 16, align 0)
+  ; MIPS32:   G_STORE [[ANYEXT]](s32), [[GEP]](p0) :: (store 4 into stack + 16, align 8)
   ; MIPS32:   JAL @aext_stack_arg_i16, csr_o32, implicit-def $ra, implicit-def $sp, implicit $a0, implicit $a1, implicit $a2, implicit $a3, implicit-def $v0
   ; MIPS32:   [[COPY5:%[0-9]+]]:_(s32) = COPY $v0
   ; MIPS32:   [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[COPY5]](s32)
diff --git a/test/CodeGen/Mips/GlobalISel/irtranslator/pointers.ll b/test/CodeGen/Mips/GlobalISel/irtranslator/pointers.ll
index 41fe492e84e3a2c3c2896d4c512176937e557985..0163182589924e541e31e871f02fd1162fa60124 100644
--- a/test/CodeGen/Mips/GlobalISel/irtranslator/pointers.ll
+++ b/test/CodeGen/Mips/GlobalISel/irtranslator/pointers.ll
@@ -24,7 +24,7 @@ define i32 @ptr_arg_on_stack(i32 %x1, i32 %x2, i32 %x3, i32 %x4, i32* %p) {
   ; MIPS32:   [[COPY2:%[0-9]+]]:_(s32) = COPY $a2
   ; MIPS32:   [[COPY3:%[0-9]+]]:_(s32) = COPY $a3
   ; MIPS32:   [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.0
-  ; MIPS32:   [[LOAD:%[0-9]+]]:_(p0) = G_LOAD [[FRAME_INDEX]](p0) :: (load 4 from %fixed-stack.0, align 0)
+  ; MIPS32:   [[LOAD:%[0-9]+]]:_(p0) = G_LOAD [[FRAME_INDEX]](p0) :: (load 4 from %fixed-stack.0, align 8)
   ; MIPS32:   [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[LOAD]](p0) :: (load 4 from %ir.p)
   ; MIPS32:   $v0 = COPY [[LOAD1]](s32)
   ; MIPS32:   RetRA implicit $v0
diff --git a/test/CodeGen/Mips/GlobalISel/irtranslator/split_args.ll b/test/CodeGen/Mips/GlobalISel/irtranslator/split_args.ll
index 13ffd24bcb96e4442f43ef507381d523c0a4d7ec..803de0087a602018fe3630ecaa2e2de997629c13 100644
--- a/test/CodeGen/Mips/GlobalISel/irtranslator/split_args.ll
+++ b/test/CodeGen/Mips/GlobalISel/irtranslator/split_args.ll
@@ -27,9 +27,9 @@ define i64 @i64_stack(i32 %a0, i32 %a1, i32 %a2, i32 %a3, i64 %a) {
   ; MIPS32:   [[COPY2:%[0-9]+]]:_(s32) = COPY $a2
   ; MIPS32:   [[COPY3:%[0-9]+]]:_(s32) = COPY $a3
   ; MIPS32:   [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.1
-  ; MIPS32:   [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX]](p0) :: (load 4 from %fixed-stack.[[STACK1]], align 0)
+  ; MIPS32:   [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX]](p0) :: (load 4 from %fixed-stack.[[STACK1]], align 8)
   ; MIPS32:   [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.0
-  ; MIPS32:   [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX1]](p0) :: (load 4 from %fixed-stack.[[STACK0]], align 0)
+  ; MIPS32:   [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX1]](p0) :: (load 4 from %fixed-stack.[[STACK0]])
   ; MIPS32:   [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD]](s32), [[LOAD1]](s32)
   ; MIPS32:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[MV]](s64)
   ; MIPS32:   $v0 = COPY [[UV]](s32)
@@ -68,11 +68,11 @@ define i64 @i64_stack_allign(i32 %a0, i32 %a1, i32 %a2, i32 %a3, i32 %s16, i64 %
   ; MIPS32:   [[COPY2:%[0-9]+]]:_(s32) = COPY $a2
   ; MIPS32:   [[COPY3:%[0-9]+]]:_(s32) = COPY $a3
   ; MIPS32:   [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.2
-  ; MIPS32:   [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX]](p0) :: (load 4 from %fixed-stack.[[STACK2]], align 0)
+  ; MIPS32:   [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX]](p0) :: (load 4 from %fixed-stack.[[STACK2]], align 8)
   ; MIPS32:   [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.1
-  ; MIPS32:   [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX1]](p0) :: (load 4 from %fixed-stack.[[STACK1]], align 0)
+  ; MIPS32:   [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX1]](p0) :: (load 4 from %fixed-stack.[[STACK1]], align 8)
   ; MIPS32:   [[FRAME_INDEX2:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.0
-  ; MIPS32:   [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX2]](p0) :: (load 4 from %fixed-stack.[[STACK0]], align 0)
+  ; MIPS32:   [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX2]](p0) :: (load 4 from %fixed-stack.[[STACK0]])
   ; MIPS32:   [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD1]](s32), [[LOAD2]](s32)
   ; MIPS32:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[MV]](s64)
   ; MIPS32:   $v0 = COPY [[UV]](s32)
@@ -93,9 +93,9 @@ define i64 @i64_reg_stack(i32 %a0, i32 %a1, i32 %a2, i64 %a) {
   ; MIPS32:   [[COPY1:%[0-9]+]]:_(s32) = COPY $a1
   ; MIPS32:   [[COPY2:%[0-9]+]]:_(s32) = COPY $a2
   ; MIPS32:   [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.1
-  ; MIPS32:   [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX]](p0) :: (load 4 from %fixed-stack.[[STACK1]], align 0)
+  ; MIPS32:   [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX]](p0) :: (load 4 from %fixed-stack.[[STACK1]], align 8)
   ; MIPS32:   [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.0
-  ; MIPS32:   [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX1]](p0) :: (load 4 from %fixed-stack.[[STACK0]], align 0)
+  ; MIPS32:   [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX1]](p0) :: (load 4 from %fixed-stack.[[STACK0]])
   ; MIPS32:   [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD]](s32), [[LOAD1]](s32)
   ; MIPS32:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[MV]](s64)
   ; MIPS32:   $v0 = COPY [[UV]](s32)
diff --git a/test/CodeGen/Mips/GlobalISel/irtranslator/stack_args.ll b/test/CodeGen/Mips/GlobalISel/irtranslator/stack_args.ll
index ae89af4b6ef5aaa80aa2c79cdcde6b42939c4003..37d75bc7f7dd937b814407eed22e8b8ae2f1db0e 100644
--- a/test/CodeGen/Mips/GlobalISel/irtranslator/stack_args.ll
+++ b/test/CodeGen/Mips/GlobalISel/irtranslator/stack_args.ll
@@ -12,7 +12,7 @@ define  i32 @g(i32  %x1, i32 %x2, i32 %x3, i32 %x4, i32 %x5){
   ; MIPS32:   [[COPY2:%[0-9]+]]:_(s32) = COPY $a2
   ; MIPS32:   [[COPY3:%[0-9]+]]:_(s32) = COPY $a3
   ; MIPS32:   [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.0
-  ; MIPS32:   [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX]](p0) :: (load 4 from %fixed-stack.0, align 0)
+  ; MIPS32:   [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX]](p0) :: (load 4 from %fixed-stack.0, align 8)
   ; MIPS32:   ADJCALLSTACKDOWN 24, 0, implicit-def $sp, implicit $sp
   ; MIPS32:   $a0 = COPY [[COPY]](s32)
   ; MIPS32:   $a1 = COPY [[COPY1]](s32)
@@ -21,7 +21,7 @@ define  i32 @g(i32  %x1, i32 %x2, i32 %x3, i32 %x4, i32 %x5){
   ; MIPS32:   [[COPY4:%[0-9]+]]:_(p0) = COPY $sp
   ; MIPS32:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
   ; MIPS32:   [[GEP:%[0-9]+]]:_(p0) = G_GEP [[COPY4]], [[C]](s32)
-  ; MIPS32:   G_STORE [[LOAD]](s32), [[GEP]](p0) :: (store 4 into stack + 16, align 0)
+  ; MIPS32:   G_STORE [[LOAD]](s32), [[GEP]](p0) :: (store 4 into stack + 16, align 8)
   ; MIPS32:   JAL @f, csr_o32, implicit-def $ra, implicit-def $sp, implicit $a0, implicit $a1, implicit $a2, implicit $a3, implicit-def $v0
   ; MIPS32:   [[COPY5:%[0-9]+]]:_(s32) = COPY $v0
   ; MIPS32:   ADJCALLSTACKUP 24, 0, implicit-def $sp, implicit $sp
diff --git a/test/CodeGen/Mips/GlobalISel/legalizer/add.mir b/test/CodeGen/Mips/GlobalISel/legalizer/add.mir
index c90b8e6d86c6ab7a506581c55edb62bb31c4736a..72be67734fdeb419f75ccf3f19d3246e0b51f584 100644
--- a/test/CodeGen/Mips/GlobalISel/legalizer/add.mir
+++ b/test/CodeGen/Mips/GlobalISel/legalizer/add.mir
@@ -52,8 +52,8 @@ body:             |
     ; MIPS32: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[COPY2]], [[COPY3]]
     ; MIPS32: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
     ; MIPS32: [[COPY4:%[0-9]+]]:_(s32) = COPY [[ADD]](s32)
-    ; MIPS32: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY4]], [[C]]
-    ; MIPS32: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C]]
+    ; MIPS32: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY4]], [[C]](s32)
+    ; MIPS32: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C]](s32)
     ; MIPS32: $v0 = COPY [[ASHR]](s32)
     ; MIPS32: RetRA implicit $v0
     %2:_(s32) = COPY $a0
@@ -141,8 +141,8 @@ body:             |
     ; MIPS32: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[COPY2]], [[COPY3]]
     ; MIPS32: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
     ; MIPS32: [[COPY4:%[0-9]+]]:_(s32) = COPY [[ADD]](s32)
-    ; MIPS32: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY4]], [[C]]
-    ; MIPS32: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C]]
+    ; MIPS32: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY4]], [[C]](s32)
+    ; MIPS32: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C]](s32)
     ; MIPS32: $v0 = COPY [[ASHR]](s32)
     ; MIPS32: RetRA implicit $v0
     %2:_(s32) = COPY $a0
@@ -275,13 +275,13 @@ body:             |
     ; MIPS32: [[COPY2:%[0-9]+]]:_(s32) = COPY $a2
     ; MIPS32: [[COPY3:%[0-9]+]]:_(s32) = COPY $a3
     ; MIPS32: [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.0
-    ; MIPS32: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX]](p0) :: (load 4 from %fixed-stack.0, align 0)
+    ; MIPS32: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX]](p0) :: (load 4 from %fixed-stack.0)
     ; MIPS32: [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.1
-    ; MIPS32: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX1]](p0) :: (load 4 from %fixed-stack.1, align 0)
+    ; MIPS32: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX1]](p0) :: (load 4 from %fixed-stack.1)
     ; MIPS32: [[FRAME_INDEX2:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.2
-    ; MIPS32: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX2]](p0) :: (load 4 from %fixed-stack.2, align 0)
+    ; MIPS32: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX2]](p0) :: (load 4 from %fixed-stack.2)
     ; MIPS32: [[FRAME_INDEX3:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.3
-    ; MIPS32: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX3]](p0) :: (load 4 from %fixed-stack.3, align 0)
+    ; MIPS32: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX3]](p0) :: (load 4 from %fixed-stack.3)
     ; MIPS32: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
     ; MIPS32: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[LOAD]], [[COPY]]
     ; MIPS32: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
@@ -317,13 +317,13 @@ body:             |
     %5:_(s32) = COPY $a3
     %0:_(s128) = G_MERGE_VALUES %2(s32), %3(s32), %4(s32), %5(s32)
     %10:_(p0) = G_FRAME_INDEX %fixed-stack.3
-    %6:_(s32) = G_LOAD %10(p0) :: (load 4 from %fixed-stack.3, align 0)
+    %6:_(s32) = G_LOAD %10(p0) :: (load 4 from %fixed-stack.3, align 4)
     %11:_(p0) = G_FRAME_INDEX %fixed-stack.2
-    %7:_(s32) = G_LOAD %11(p0) :: (load 4 from %fixed-stack.2, align 0)
+    %7:_(s32) = G_LOAD %11(p0) :: (load 4 from %fixed-stack.2, align 4)
     %12:_(p0) = G_FRAME_INDEX %fixed-stack.1
-    %8:_(s32) = G_LOAD %12(p0) :: (load 4 from %fixed-stack.1, align 0)
+    %8:_(s32) = G_LOAD %12(p0) :: (load 4 from %fixed-stack.1, align 4)
     %13:_(p0) = G_FRAME_INDEX %fixed-stack.0
-    %9:_(s32) = G_LOAD %13(p0) :: (load 4 from %fixed-stack.0, align 0)
+    %9:_(s32) = G_LOAD %13(p0) :: (load 4 from %fixed-stack.0, align 4)
     %1:_(s128) = G_MERGE_VALUES %6(s32), %7(s32), %8(s32), %9(s32)
     %14:_(s128) = G_ADD %1, %0
     %15:_(s32), %16:_(s32), %17:_(s32), %18:_(s32) = G_UNMERGE_VALUES %14(s128)
diff --git a/test/CodeGen/Mips/GlobalISel/legalizer/mul.mir b/test/CodeGen/Mips/GlobalISel/legalizer/mul.mir
new file mode 100644
index 0000000000000000000000000000000000000000..481f8fce71268903dcf7dc3ebe49e8d5be38fd17
--- /dev/null
+++ b/test/CodeGen/Mips/GlobalISel/legalizer/mul.mir
@@ -0,0 +1,213 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -O0 -mtriple=mipsel-linux-gnu -run-pass=legalizer -verify-machineinstrs %s -o - | FileCheck %s -check-prefixes=MIPS32
+--- |
+
+  define void @mul_i32() {entry: ret void}
+  define void @mul_i8_sext() {entry: ret void}
+  define void @mul_i8_zext() {entry: ret void}
+  define void @mul_i8_aext() {entry: ret void}
+  define void @mul_i16_sext() {entry: ret void}
+  define void @mul_i16_zext() {entry: ret void}
+  define void @mul_i16_aext() {entry: ret void}
+
+...
+---
+name:            mul_i32
+alignment:       2
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $a0, $a1
+
+    ; MIPS32-LABEL: name: mul_i32
+    ; MIPS32: liveins: $a0, $a1
+    ; MIPS32: [[COPY:%[0-9]+]]:_(s32) = COPY $a0
+    ; MIPS32: [[COPY1:%[0-9]+]]:_(s32) = COPY $a1
+    ; MIPS32: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[COPY]], [[COPY1]]
+    ; MIPS32: $v0 = COPY [[MUL]](s32)
+    ; MIPS32: RetRA implicit $v0
+    %0:_(s32) = COPY $a0
+    %1:_(s32) = COPY $a1
+    %2:_(s32) = G_MUL %0, %1
+    $v0 = COPY %2(s32)
+    RetRA implicit $v0
+
+...
+---
+name:            mul_i8_sext
+alignment:       2
+tracksRegLiveness: true
+body:             |
+  bb.1.entry:
+    liveins: $a0, $a1
+
+    ; MIPS32-LABEL: name: mul_i8_sext
+    ; MIPS32: liveins: $a0, $a1
+    ; MIPS32: [[COPY:%[0-9]+]]:_(s32) = COPY $a0
+    ; MIPS32: [[COPY1:%[0-9]+]]:_(s32) = COPY $a1
+    ; MIPS32: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+    ; MIPS32: [[COPY3:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; MIPS32: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[COPY2]], [[COPY3]]
+    ; MIPS32: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; MIPS32: [[COPY4:%[0-9]+]]:_(s32) = COPY [[MUL]](s32)
+    ; MIPS32: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY4]], [[C]](s32)
+    ; MIPS32: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C]](s32)
+    ; MIPS32: $v0 = COPY [[ASHR]](s32)
+    ; MIPS32: RetRA implicit $v0
+    %2:_(s32) = COPY $a0
+    %0:_(s8) = G_TRUNC %2(s32)
+    %3:_(s32) = COPY $a1
+    %1:_(s8) = G_TRUNC %3(s32)
+    %4:_(s8) = G_MUL %1, %0
+    %5:_(s32) = G_SEXT %4(s8)
+    $v0 = COPY %5(s32)
+    RetRA implicit $v0
+
+...
+---
+name:            mul_i8_zext
+alignment:       2
+tracksRegLiveness: true
+body:             |
+  bb.1.entry:
+    liveins: $a0, $a1
+
+    ; MIPS32-LABEL: name: mul_i8_zext
+    ; MIPS32: liveins: $a0, $a1
+    ; MIPS32: [[COPY:%[0-9]+]]:_(s32) = COPY $a0
+    ; MIPS32: [[COPY1:%[0-9]+]]:_(s32) = COPY $a1
+    ; MIPS32: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+    ; MIPS32: [[COPY3:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; MIPS32: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[COPY2]], [[COPY3]]
+    ; MIPS32: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; MIPS32: [[COPY4:%[0-9]+]]:_(s32) = COPY [[MUL]](s32)
+    ; MIPS32: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY4]], [[C]]
+    ; MIPS32: $v0 = COPY [[AND]](s32)
+    ; MIPS32: RetRA implicit $v0
+    %2:_(s32) = COPY $a0
+    %0:_(s8) = G_TRUNC %2(s32)
+    %3:_(s32) = COPY $a1
+    %1:_(s8) = G_TRUNC %3(s32)
+    %4:_(s8) = G_MUL %1, %0
+    %5:_(s32) = G_ZEXT %4(s8)
+    $v0 = COPY %5(s32)
+    RetRA implicit $v0
+
+...
+---
+name:            mul_i8_aext
+alignment:       2
+tracksRegLiveness: true
+body:             |
+  bb.1.entry:
+    liveins: $a0, $a1
+
+    ; MIPS32-LABEL: name: mul_i8_aext
+    ; MIPS32: liveins: $a0, $a1
+    ; MIPS32: [[COPY:%[0-9]+]]:_(s32) = COPY $a0
+    ; MIPS32: [[COPY1:%[0-9]+]]:_(s32) = COPY $a1
+    ; MIPS32: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+    ; MIPS32: [[COPY3:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; MIPS32: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[COPY2]], [[COPY3]]
+    ; MIPS32: [[COPY4:%[0-9]+]]:_(s32) = COPY [[MUL]](s32)
+    ; MIPS32: $v0 = COPY [[COPY4]](s32)
+    ; MIPS32: RetRA implicit $v0
+    %2:_(s32) = COPY $a0
+    %0:_(s8) = G_TRUNC %2(s32)
+    %3:_(s32) = COPY $a1
+    %1:_(s8) = G_TRUNC %3(s32)
+    %4:_(s8) = G_MUL %1, %0
+    %5:_(s32) = G_ANYEXT %4(s8)
+    $v0 = COPY %5(s32)
+    RetRA implicit $v0
+
+...
+---
+name:            mul_i16_sext
+alignment:       2
+tracksRegLiveness: true
+body:             |
+  bb.1.entry:
+    liveins: $a0, $a1
+
+    ; MIPS32-LABEL: name: mul_i16_sext
+    ; MIPS32: liveins: $a0, $a1
+    ; MIPS32: [[COPY:%[0-9]+]]:_(s32) = COPY $a0
+    ; MIPS32: [[COPY1:%[0-9]+]]:_(s32) = COPY $a1
+    ; MIPS32: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+    ; MIPS32: [[COPY3:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; MIPS32: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[COPY2]], [[COPY3]]
+    ; MIPS32: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; MIPS32: [[COPY4:%[0-9]+]]:_(s32) = COPY [[MUL]](s32)
+    ; MIPS32: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY4]], [[C]](s32)
+    ; MIPS32: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C]](s32)
+    ; MIPS32: $v0 = COPY [[ASHR]](s32)
+    ; MIPS32: RetRA implicit $v0
+    %2:_(s32) = COPY $a0
+    %0:_(s16) = G_TRUNC %2(s32)
+    %3:_(s32) = COPY $a1
+    %1:_(s16) = G_TRUNC %3(s32)
+    %4:_(s16) = G_MUL %1, %0
+    %5:_(s32) = G_SEXT %4(s16)
+    $v0 = COPY %5(s32)
+    RetRA implicit $v0
+
+...
+---
+name:            mul_i16_zext
+alignment:       2
+tracksRegLiveness: true
+body:             |
+  bb.1.entry:
+    liveins: $a0, $a1
+
+    ; MIPS32-LABEL: name: mul_i16_zext
+    ; MIPS32: liveins: $a0, $a1
+    ; MIPS32: [[COPY:%[0-9]+]]:_(s32) = COPY $a0
+    ; MIPS32: [[COPY1:%[0-9]+]]:_(s32) = COPY $a1
+    ; MIPS32: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+    ; MIPS32: [[COPY3:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; MIPS32: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[COPY2]], [[COPY3]]
+    ; MIPS32: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; MIPS32: [[COPY4:%[0-9]+]]:_(s32) = COPY [[MUL]](s32)
+    ; MIPS32: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY4]], [[C]]
+    ; MIPS32: $v0 = COPY [[AND]](s32)
+    ; MIPS32: RetRA implicit $v0
+    %2:_(s32) = COPY $a0
+    %0:_(s16) = G_TRUNC %2(s32)
+    %3:_(s32) = COPY $a1
+    %1:_(s16) = G_TRUNC %3(s32)
+    %4:_(s16) = G_MUL %1, %0
+    %5:_(s32) = G_ZEXT %4(s16)
+    $v0 = COPY %5(s32)
+    RetRA implicit $v0
+
+...
+---
+name:            mul_i16_aext
+alignment:       2
+tracksRegLiveness: true
+body:             |
+  bb.1.entry:
+    liveins: $a0, $a1
+
+    ; MIPS32-LABEL: name: mul_i16_aext
+    ; MIPS32: liveins: $a0, $a1
+    ; MIPS32: [[COPY:%[0-9]+]]:_(s32) = COPY $a0
+    ; MIPS32: [[COPY1:%[0-9]+]]:_(s32) = COPY $a1
+    ; MIPS32: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+    ; MIPS32: [[COPY3:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; MIPS32: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[COPY2]], [[COPY3]]
+    ; MIPS32: [[COPY4:%[0-9]+]]:_(s32) = COPY [[MUL]](s32)
+    ; MIPS32: $v0 = COPY [[COPY4]](s32)
+    ; MIPS32: RetRA implicit $v0
+    %2:_(s32) = COPY $a0
+    %0:_(s16) = G_TRUNC %2(s32)
+    %3:_(s32) = COPY $a1
+    %1:_(s16) = G_TRUNC %3(s32)
+    %4:_(s16) = G_MUL %1, %0
+    %5:_(s32) = G_ANYEXT %4(s16)
+    $v0 = COPY %5(s32)
+    RetRA implicit $v0
+
+...
diff --git a/test/CodeGen/Mips/GlobalISel/legalizer/pointers.mir b/test/CodeGen/Mips/GlobalISel/legalizer/pointers.mir
index 0dbeb55108a26ad16480f1e5a44ff7c97f1703be..bef661493444b2ff1280348c32d5108172372cf9 100644
--- a/test/CodeGen/Mips/GlobalISel/legalizer/pointers.mir
+++ b/test/CodeGen/Mips/GlobalISel/legalizer/pointers.mir
@@ -44,7 +44,7 @@ body:             |
     ; MIPS32: [[COPY2:%[0-9]+]]:_(s32) = COPY $a2
     ; MIPS32: [[COPY3:%[0-9]+]]:_(s32) = COPY $a3
     ; MIPS32: [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.0
-    ; MIPS32: [[LOAD:%[0-9]+]]:_(p0) = G_LOAD [[FRAME_INDEX]](p0) :: (load 4 from %fixed-stack.0, align 0)
+    ; MIPS32: [[LOAD:%[0-9]+]]:_(p0) = G_LOAD [[FRAME_INDEX]](p0) :: (load 4 from %fixed-stack.0)
     ; MIPS32: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[LOAD]](p0) :: (load 4 from %ir.p)
     ; MIPS32: $v0 = COPY [[LOAD1]](s32)
     ; MIPS32: RetRA implicit $v0
@@ -53,7 +53,7 @@ body:             |
     %2:_(s32) = COPY $a2
     %3:_(s32) = COPY $a3
     %5:_(p0) = G_FRAME_INDEX %fixed-stack.0
-    %4:_(p0) = G_LOAD %5(p0) :: (load 4 from %fixed-stack.0, align 0)
+    %4:_(p0) = G_LOAD %5(p0) :: (load 4 from %fixed-stack.0, align 4)
     %6:_(s32) = G_LOAD %4(p0) :: (load 4 from %ir.p)
     $v0 = COPY %6(s32)
     RetRA implicit $v0
diff --git a/test/CodeGen/Mips/GlobalISel/legalizer/select.mir b/test/CodeGen/Mips/GlobalISel/legalizer/select.mir
new file mode 100644
index 0000000000000000000000000000000000000000..249c4d3a7f23e4609c01af4c4389dd4338eb236a
--- /dev/null
+++ b/test/CodeGen/Mips/GlobalISel/legalizer/select.mir
@@ -0,0 +1,172 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -O0 -mtriple=mipsel-linux-gnu -run-pass=legalizer -verify-machineinstrs %s -o - | FileCheck %s -check-prefixes=MIPS32
+--- |
+
+  define void @select_i8() {entry: ret void}
+  define void @select_i16() {entry: ret void}
+  define void @select_i32() {entry: ret void}
+  define void @select_ptr() {entry: ret void}
+  define void @select_with_negation() {entry: ret void}
+
+...
+---
+name:            select_i8
+alignment:       2
+tracksRegLiveness: true
+body:             |
+  bb.1.entry:
+    liveins: $a0, $a1, $a2
+
+    ; MIPS32-LABEL: name: select_i8
+    ; MIPS32: liveins: $a0, $a1, $a2
+    ; MIPS32: [[COPY:%[0-9]+]]:_(s32) = COPY $a0
+    ; MIPS32: [[COPY1:%[0-9]+]]:_(s32) = COPY $a1
+    ; MIPS32: [[COPY2:%[0-9]+]]:_(s32) = COPY $a2
+    ; MIPS32: [[COPY3:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+    ; MIPS32: [[COPY4:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+    ; MIPS32: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; MIPS32: [[COPY5:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; MIPS32: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY5]], [[C]]
+    ; MIPS32: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[AND]](s32), [[COPY3]], [[COPY4]]
+    ; MIPS32: [[COPY6:%[0-9]+]]:_(s32) = COPY [[SELECT]](s32)
+    ; MIPS32: $v0 = COPY [[COPY6]](s32)
+    ; MIPS32: RetRA implicit $v0
+    %3:_(s32) = COPY $a0
+    %0:_(s1) = G_TRUNC %3(s32)
+    %4:_(s32) = COPY $a1
+    %1:_(s8) = G_TRUNC %4(s32)
+    %5:_(s32) = COPY $a2
+    %2:_(s8) = G_TRUNC %5(s32)
+    %6:_(s8) = G_SELECT %0(s1), %1, %2
+    %7:_(s32) = G_ANYEXT %6(s8)
+    $v0 = COPY %7(s32)
+    RetRA implicit $v0
+
+...
+---
+name:            select_i16
+alignment:       2
+tracksRegLiveness: true
+body:             |
+  bb.1.entry:
+    liveins: $a0, $a1, $a2
+
+    ; MIPS32-LABEL: name: select_i16
+    ; MIPS32: liveins: $a0, $a1, $a2
+    ; MIPS32: [[COPY:%[0-9]+]]:_(s32) = COPY $a0
+    ; MIPS32: [[COPY1:%[0-9]+]]:_(s32) = COPY $a1
+    ; MIPS32: [[COPY2:%[0-9]+]]:_(s32) = COPY $a2
+    ; MIPS32: [[COPY3:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+    ; MIPS32: [[COPY4:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
+    ; MIPS32: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; MIPS32: [[COPY5:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; MIPS32: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY5]], [[C]]
+    ; MIPS32: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[AND]](s32), [[COPY3]], [[COPY4]]
+    ; MIPS32: [[COPY6:%[0-9]+]]:_(s32) = COPY [[SELECT]](s32)
+    ; MIPS32: $v0 = COPY [[COPY6]](s32)
+    ; MIPS32: RetRA implicit $v0
+    %3:_(s32) = COPY $a0
+    %0:_(s1) = G_TRUNC %3(s32)
+    %4:_(s32) = COPY $a1
+    %1:_(s16) = G_TRUNC %4(s32)
+    %5:_(s32) = COPY $a2
+    %2:_(s16) = G_TRUNC %5(s32)
+    %6:_(s16) = G_SELECT %0(s1), %1, %2
+    %7:_(s32) = G_ANYEXT %6(s16)
+    $v0 = COPY %7(s32)
+    RetRA implicit $v0
+
+...
+---
+name:            select_i32
+alignment:       2
+tracksRegLiveness: true
+body:             |
+  bb.1.entry:
+    liveins: $a0, $a1, $a2
+
+    ; MIPS32-LABEL: name: select_i32
+    ; MIPS32: liveins: $a0, $a1, $a2
+    ; MIPS32: [[COPY:%[0-9]+]]:_(s32) = COPY $a0
+    ; MIPS32: [[COPY1:%[0-9]+]]:_(s32) = COPY $a1
+    ; MIPS32: [[COPY2:%[0-9]+]]:_(s32) = COPY $a2
+    ; MIPS32: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; MIPS32: [[COPY3:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; MIPS32: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[C]]
+    ; MIPS32: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[AND]](s32), [[COPY1]], [[COPY2]]
+    ; MIPS32: $v0 = COPY [[SELECT]](s32)
+    ; MIPS32: RetRA implicit $v0
+    %3:_(s32) = COPY $a0
+    %0:_(s1) = G_TRUNC %3(s32)
+    %1:_(s32) = COPY $a1
+    %2:_(s32) = COPY $a2
+    %4:_(s32) = G_SELECT %0(s1), %1, %2
+    $v0 = COPY %4(s32)
+    RetRA implicit $v0
+
+...
+---
+name:            select_ptr
+alignment:       2
+tracksRegLiveness: true
+body:             |
+  bb.1.entry:
+    liveins: $a0, $a1, $a2
+
+    ; MIPS32-LABEL: name: select_ptr
+    ; MIPS32: liveins: $a0, $a1, $a2
+    ; MIPS32: [[COPY:%[0-9]+]]:_(s32) = COPY $a0
+    ; MIPS32: [[COPY1:%[0-9]+]]:_(p0) = COPY $a1
+    ; MIPS32: [[COPY2:%[0-9]+]]:_(p0) = COPY $a2
+    ; MIPS32: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; MIPS32: [[COPY3:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; MIPS32: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[C]]
+    ; MIPS32: [[SELECT:%[0-9]+]]:_(p0) = G_SELECT [[AND]](s32), [[COPY1]], [[COPY2]]
+    ; MIPS32: $v0 = COPY [[SELECT]](p0)
+    ; MIPS32: RetRA implicit $v0
+    %3:_(s32) = COPY $a0
+    %0:_(s1) = G_TRUNC %3(s32)
+    %1:_(p0) = COPY $a1
+    %2:_(p0) = COPY $a2
+    %4:_(p0) = G_SELECT %0(s1), %1, %2
+    $v0 = COPY %4(p0)
+    RetRA implicit $v0
+
+...
+---
+name:            select_with_negation
+alignment:       2
+tracksRegLiveness: true
+body:             |
+  bb.1.entry:
+    liveins: $a0, $a1, $a2, $a3
+
+    ; MIPS32-LABEL: name: select_with_negation
+    ; MIPS32: liveins: $a0, $a1, $a2, $a3
+    ; MIPS32: [[COPY:%[0-9]+]]:_(s32) = COPY $a0
+    ; MIPS32: [[COPY1:%[0-9]+]]:_(s32) = COPY $a1
+    ; MIPS32: [[COPY2:%[0-9]+]]:_(s32) = COPY $a2
+    ; MIPS32: [[COPY3:%[0-9]+]]:_(s32) = COPY $a3
+    ; MIPS32: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
+    ; MIPS32: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(slt), [[COPY]](s32), [[COPY1]]
+    ; MIPS32: [[COPY4:%[0-9]+]]:_(s32) = COPY [[ICMP]](s32)
+    ; MIPS32: [[COPY5:%[0-9]+]]:_(s32) = COPY [[C]](s32)
+    ; MIPS32: [[XOR:%[0-9]+]]:_(s32) = G_XOR [[COPY4]], [[COPY5]]
+    ; MIPS32: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; MIPS32: [[COPY6:%[0-9]+]]:_(s32) = COPY [[XOR]](s32)
+    ; MIPS32: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY6]], [[C1]]
+    ; MIPS32: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[AND]](s32), [[COPY2]], [[COPY3]]
+    ; MIPS32: $v0 = COPY [[SELECT]](s32)
+    ; MIPS32: RetRA implicit $v0
+    %0:_(s32) = COPY $a0
+    %1:_(s32) = COPY $a1
+    %2:_(s32) = COPY $a2
+    %3:_(s32) = COPY $a3
+    %5:_(s1) = G_CONSTANT i1 true
+    %4:_(s1) = G_ICMP intpred(slt), %0(s32), %1
+    %6:_(s1) = G_XOR %4, %5
+    %7:_(s32) = G_SELECT %6(s1), %2, %3
+    $v0 = COPY %7(s32)
+    RetRA implicit $v0
+
+...
diff --git a/test/CodeGen/Mips/GlobalISel/legalizer/stack_args.mir b/test/CodeGen/Mips/GlobalISel/legalizer/stack_args.mir
index f9d61297358b2ccb34426c712bc82abd50998991..2ba722d1ea38654687b2738bf537cea7af41c41c 100644
--- a/test/CodeGen/Mips/GlobalISel/legalizer/stack_args.mir
+++ b/test/CodeGen/Mips/GlobalISel/legalizer/stack_args.mir
@@ -23,7 +23,7 @@ body:             |
     ; MIPS32: [[COPY2:%[0-9]+]]:_(s32) = COPY $a2
     ; MIPS32: [[COPY3:%[0-9]+]]:_(s32) = COPY $a3
     ; MIPS32: [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.0
-    ; MIPS32: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX]](p0) :: (load 4 from %fixed-stack.0, align 0)
+    ; MIPS32: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX]](p0) :: (load 4 from %fixed-stack.0, align 8)
     ; MIPS32: ADJCALLSTACKDOWN 24, 0, implicit-def $sp, implicit $sp
     ; MIPS32: $a0 = COPY [[COPY]](s32)
     ; MIPS32: $a1 = COPY [[COPY1]](s32)
@@ -32,7 +32,7 @@ body:             |
     ; MIPS32: [[COPY4:%[0-9]+]]:_(p0) = COPY $sp
     ; MIPS32: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
     ; MIPS32: [[GEP:%[0-9]+]]:_(p0) = G_GEP [[COPY4]], [[C]](s32)
-    ; MIPS32: G_STORE [[LOAD]](s32), [[GEP]](p0) :: (store 4 into stack + 16, align 0)
+    ; MIPS32: G_STORE [[LOAD]](s32), [[GEP]](p0) :: (store 4 into stack + 16)
     ; MIPS32: JAL @f, csr_o32, implicit-def $ra, implicit-def $sp, implicit $a0, implicit $a1, implicit $a2, implicit $a3, implicit-def $v0
     ; MIPS32: [[COPY5:%[0-9]+]]:_(s32) = COPY $v0
     ; MIPS32: ADJCALLSTACKUP 24, 0, implicit-def $sp, implicit $sp
@@ -43,7 +43,7 @@ body:             |
     %2:_(s32) = COPY $a2
     %3:_(s32) = COPY $a3
     %5:_(p0) = G_FRAME_INDEX %fixed-stack.0
-    %4:_(s32) = G_LOAD %5(p0) :: (load 4 from %fixed-stack.0, align 0)
+    %4:_(s32) = G_LOAD %5(p0) :: (load 4 from %fixed-stack.0, align 8)
     ADJCALLSTACKDOWN 24, 0, implicit-def $sp, implicit $sp
     $a0 = COPY %0(s32)
     $a1 = COPY %1(s32)
@@ -52,7 +52,7 @@ body:             |
     %7:_(p0) = COPY $sp
     %8:_(s32) = G_CONSTANT i32 16
     %9:_(p0) = G_GEP %7, %8(s32)
-    G_STORE %4(s32), %9(p0) :: (store 4 into stack + 16, align 0)
+    G_STORE %4(s32), %9(p0) :: (store 4 into stack + 16, align 4)
     JAL @f, csr_o32, implicit-def $ra, implicit-def $sp, implicit $a0, implicit $a1, implicit $a2, implicit $a3, implicit-def $v0
     %6:_(s32) = COPY $v0
     ADJCALLSTACKUP 24, 0, implicit-def $sp, implicit $sp
diff --git a/test/CodeGen/Mips/GlobalISel/legalizer/sub.mir b/test/CodeGen/Mips/GlobalISel/legalizer/sub.mir
new file mode 100644
index 0000000000000000000000000000000000000000..9f84e4d05015b66b5af7c73bbe66d468dfc5d8c0
--- /dev/null
+++ b/test/CodeGen/Mips/GlobalISel/legalizer/sub.mir
@@ -0,0 +1,342 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -O0 -mtriple=mipsel-linux-gnu -run-pass=legalizer -verify-machineinstrs %s -o - | FileCheck %s -check-prefixes=MIPS32
+--- |
+
+  define void @sub_i32() {entry: ret void}
+  define void @sub_i8_sext() {entry: ret void}
+  define void @sub_i8_zext() {entry: ret void}
+  define void @sub_i8_aext() {entry: ret void}
+  define void @sub_i16_sext() {entry: ret void}
+  define void @sub_i16_zext() {entry: ret void}
+  define void @sub_i16_aext() {entry: ret void}
+  define void @sub_i64() {entry: ret void}
+  define void @sub_i128() {entry: ret void}
+
+...
+---
+name:            sub_i32
+alignment:       2
+tracksRegLiveness: true
+body:             |
+  bb.1.entry:
+    liveins: $a0, $a1
+
+    ; MIPS32-LABEL: name: sub_i32
+    ; MIPS32: liveins: $a0, $a1
+    ; MIPS32: [[COPY:%[0-9]+]]:_(s32) = COPY $a0
+    ; MIPS32: [[COPY1:%[0-9]+]]:_(s32) = COPY $a1
+    ; MIPS32: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[COPY]], [[COPY1]]
+    ; MIPS32: $v0 = COPY [[SUB]](s32)
+    ; MIPS32: RetRA implicit $v0
+    %0:_(s32) = COPY $a0
+    %1:_(s32) = COPY $a1
+    %2:_(s32) = G_SUB %0, %1
+    $v0 = COPY %2(s32)
+    RetRA implicit $v0
+
+...
+---
+name:            sub_i8_sext
+alignment:       2
+tracksRegLiveness: true
+body:             |
+  bb.1.entry:
+    liveins: $a0, $a1
+
+    ; MIPS32-LABEL: name: sub_i8_sext
+    ; MIPS32: liveins: $a0, $a1
+    ; MIPS32: [[COPY:%[0-9]+]]:_(s32) = COPY $a0
+    ; MIPS32: [[COPY1:%[0-9]+]]:_(s32) = COPY $a1
+    ; MIPS32: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+    ; MIPS32: [[COPY3:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; MIPS32: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[COPY2]], [[COPY3]]
+    ; MIPS32: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
+    ; MIPS32: [[COPY4:%[0-9]+]]:_(s32) = COPY [[SUB]](s32)
+    ; MIPS32: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY4]], [[C]](s32)
+    ; MIPS32: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C]](s32)
+    ; MIPS32: $v0 = COPY [[ASHR]](s32)
+    ; MIPS32: RetRA implicit $v0
+    %2:_(s32) = COPY $a0
+    %0:_(s8) = G_TRUNC %2(s32)
+    %3:_(s32) = COPY $a1
+    %1:_(s8) = G_TRUNC %3(s32)
+    %4:_(s8) = G_SUB %1, %0
+    %5:_(s32) = G_SEXT %4(s8)
+    $v0 = COPY %5(s32)
+    RetRA implicit $v0
+
+...
+---
+name:            sub_i8_zext
+alignment:       2
+tracksRegLiveness: true
+body:             |
+  bb.1.entry:
+    liveins: $a0, $a1
+
+    ; MIPS32-LABEL: name: sub_i8_zext
+    ; MIPS32: liveins: $a0, $a1
+    ; MIPS32: [[COPY:%[0-9]+]]:_(s32) = COPY $a0
+    ; MIPS32: [[COPY1:%[0-9]+]]:_(s32) = COPY $a1
+    ; MIPS32: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+    ; MIPS32: [[COPY3:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; MIPS32: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[COPY2]], [[COPY3]]
+    ; MIPS32: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
+    ; MIPS32: [[COPY4:%[0-9]+]]:_(s32) = COPY [[SUB]](s32)
+    ; MIPS32: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY4]], [[C]]
+    ; MIPS32: $v0 = COPY [[AND]](s32)
+    ; MIPS32: RetRA implicit $v0
+    %2:_(s32) = COPY $a0
+    %0:_(s8) = G_TRUNC %2(s32)
+    %3:_(s32) = COPY $a1
+    %1:_(s8) = G_TRUNC %3(s32)
+    %4:_(s8) = G_SUB %1, %0
+    %5:_(s32) = G_ZEXT %4(s8)
+    $v0 = COPY %5(s32)
+    RetRA implicit $v0
+
+...
+---
+name:            sub_i8_aext
+alignment:       2
+tracksRegLiveness: true
+body:             |
+  bb.1.entry:
+    liveins: $a0, $a1
+
+    ; MIPS32-LABEL: name: sub_i8_aext
+    ; MIPS32: liveins: $a0, $a1
+    ; MIPS32: [[COPY:%[0-9]+]]:_(s32) = COPY $a0
+    ; MIPS32: [[COPY1:%[0-9]+]]:_(s32) = COPY $a1
+    ; MIPS32: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+    ; MIPS32: [[COPY3:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; MIPS32: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[COPY2]], [[COPY3]]
+    ; MIPS32: [[COPY4:%[0-9]+]]:_(s32) = COPY [[SUB]](s32)
+    ; MIPS32: $v0 = COPY [[COPY4]](s32)
+    ; MIPS32: RetRA implicit $v0
+    %2:_(s32) = COPY $a0
+    %0:_(s8) = G_TRUNC %2(s32)
+    %3:_(s32) = COPY $a1
+    %1:_(s8) = G_TRUNC %3(s32)
+    %4:_(s8) = G_SUB %1, %0
+    %5:_(s32) = G_ANYEXT %4(s8)
+    $v0 = COPY %5(s32)
+    RetRA implicit $v0
+
+...
+---
+name:            sub_i16_sext
+alignment:       2
+tracksRegLiveness: true
+body:             |
+  bb.1.entry:
+    liveins: $a0, $a1
+
+    ; MIPS32-LABEL: name: sub_i16_sext
+    ; MIPS32: liveins: $a0, $a1
+    ; MIPS32: [[COPY:%[0-9]+]]:_(s32) = COPY $a0
+    ; MIPS32: [[COPY1:%[0-9]+]]:_(s32) = COPY $a1
+    ; MIPS32: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+    ; MIPS32: [[COPY3:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; MIPS32: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[COPY2]], [[COPY3]]
+    ; MIPS32: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
+    ; MIPS32: [[COPY4:%[0-9]+]]:_(s32) = COPY [[SUB]](s32)
+    ; MIPS32: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY4]], [[C]](s32)
+    ; MIPS32: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C]](s32)
+    ; MIPS32: $v0 = COPY [[ASHR]](s32)
+    ; MIPS32: RetRA implicit $v0
+    %2:_(s32) = COPY $a0
+    %0:_(s16) = G_TRUNC %2(s32)
+    %3:_(s32) = COPY $a1
+    %1:_(s16) = G_TRUNC %3(s32)
+    %4:_(s16) = G_SUB %1, %0
+    %5:_(s32) = G_SEXT %4(s16)
+    $v0 = COPY %5(s32)
+    RetRA implicit $v0
+
+...
+---
+name:            sub_i16_zext
+alignment:       2
+tracksRegLiveness: true
+body:             |
+  bb.1.entry:
+    liveins: $a0, $a1
+
+    ; MIPS32-LABEL: name: sub_i16_zext
+    ; MIPS32: liveins: $a0, $a1
+    ; MIPS32: [[COPY:%[0-9]+]]:_(s32) = COPY $a0
+    ; MIPS32: [[COPY1:%[0-9]+]]:_(s32) = COPY $a1
+    ; MIPS32: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+    ; MIPS32: [[COPY3:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; MIPS32: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[COPY2]], [[COPY3]]
+    ; MIPS32: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+    ; MIPS32: [[COPY4:%[0-9]+]]:_(s32) = COPY [[SUB]](s32)
+    ; MIPS32: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY4]], [[C]]
+    ; MIPS32: $v0 = COPY [[AND]](s32)
+    ; MIPS32: RetRA implicit $v0
+    %2:_(s32) = COPY $a0
+    %0:_(s16) = G_TRUNC %2(s32)
+    %3:_(s32) = COPY $a1
+    %1:_(s16) = G_TRUNC %3(s32)
+    %4:_(s16) = G_SUB %1, %0
+    %5:_(s32) = G_ZEXT %4(s16)
+    $v0 = COPY %5(s32)
+    RetRA implicit $v0
+
+...
+---
+name:            sub_i16_aext
+alignment:       2
+tracksRegLiveness: true
+body:             |
+  bb.1.entry:
+    liveins: $a0, $a1
+
+    ; MIPS32-LABEL: name: sub_i16_aext
+    ; MIPS32: liveins: $a0, $a1
+    ; MIPS32: [[COPY:%[0-9]+]]:_(s32) = COPY $a0
+    ; MIPS32: [[COPY1:%[0-9]+]]:_(s32) = COPY $a1
+    ; MIPS32: [[COPY2:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+    ; MIPS32: [[COPY3:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; MIPS32: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[COPY2]], [[COPY3]]
+    ; MIPS32: [[COPY4:%[0-9]+]]:_(s32) = COPY [[SUB]](s32)
+    ; MIPS32: $v0 = COPY [[COPY4]](s32)
+    ; MIPS32: RetRA implicit $v0
+    %2:_(s32) = COPY $a0
+    %0:_(s16) = G_TRUNC %2(s32)
+    %3:_(s32) = COPY $a1
+    %1:_(s16) = G_TRUNC %3(s32)
+    %4:_(s16) = G_SUB %1, %0
+    %5:_(s32) = G_ANYEXT %4(s16)
+    $v0 = COPY %5(s32)
+    RetRA implicit $v0
+
+...
+---
+name:            sub_i64
+alignment:       2
+tracksRegLiveness: true
+body:             |
+  bb.1.entry:
+    liveins: $a0, $a1, $a2, $a3
+
+    ; MIPS32-LABEL: name: sub_i64
+    ; MIPS32: liveins: $a0, $a1, $a2, $a3
+    ; MIPS32: [[COPY:%[0-9]+]]:_(s32) = COPY $a0
+    ; MIPS32: [[COPY1:%[0-9]+]]:_(s32) = COPY $a1
+    ; MIPS32: [[COPY2:%[0-9]+]]:_(s32) = COPY $a2
+    ; MIPS32: [[COPY3:%[0-9]+]]:_(s32) = COPY $a3
+    ; MIPS32: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[COPY3]], [[COPY1]]
+    ; MIPS32: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), [[COPY3]](s32), [[COPY1]]
+    ; MIPS32: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[COPY2]], [[COPY]]
+    ; MIPS32: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; MIPS32: [[COPY4:%[0-9]+]]:_(s32) = COPY [[ICMP]](s32)
+    ; MIPS32: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY4]], [[C]]
+    ; MIPS32: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[SUB1]], [[AND]]
+    ; MIPS32: $v0 = COPY [[SUB2]](s32)
+    ; MIPS32: $v1 = COPY [[SUB]](s32)
+    ; MIPS32: RetRA implicit $v0, implicit $v1
+    %2:_(s32) = COPY $a0
+    %3:_(s32) = COPY $a1
+    %0:_(s64) = G_MERGE_VALUES %3(s32), %2(s32)
+    %4:_(s32) = COPY $a2
+    %5:_(s32) = COPY $a3
+    %1:_(s64) = G_MERGE_VALUES %5(s32), %4(s32)
+    %6:_(s64) = G_SUB %1, %0
+    %7:_(s32), %8:_(s32) = G_UNMERGE_VALUES %6(s64)
+    $v0 = COPY %8(s32)
+    $v1 = COPY %7(s32)
+    RetRA implicit $v0, implicit $v1
+
+...
+---
+name:            sub_i128
+alignment:       2
+tracksRegLiveness: true
+fixedStack:
+  - { id: 0, offset: 28, size: 4, alignment: 4, stack-id: 0, isImmutable: true }
+  - { id: 1, offset: 24, size: 4, alignment: 8, stack-id: 0, isImmutable: true }
+  - { id: 2, offset: 20, size: 4, alignment: 4, stack-id: 0, isImmutable: true }
+  - { id: 3, offset: 16, size: 4, alignment: 8, stack-id: 0, isImmutable: true }
+body:             |
+  bb.1.entry:
+    liveins: $a0, $a1, $a2, $a3
+
+    ; MIPS32-LABEL: name: sub_i128
+    ; MIPS32: liveins: $a0, $a1, $a2, $a3
+    ; MIPS32: [[COPY:%[0-9]+]]:_(s32) = COPY $a0
+    ; MIPS32: [[COPY1:%[0-9]+]]:_(s32) = COPY $a1
+    ; MIPS32: [[COPY2:%[0-9]+]]:_(s32) = COPY $a2
+    ; MIPS32: [[COPY3:%[0-9]+]]:_(s32) = COPY $a3
+    ; MIPS32: [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.0
+    ; MIPS32: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX]](p0) :: (load 4 from %fixed-stack.0, align 8)
+    ; MIPS32: [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.1
+    ; MIPS32: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX1]](p0) :: (load 4 from %fixed-stack.1)
+    ; MIPS32: [[FRAME_INDEX2:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.2
+    ; MIPS32: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX2]](p0) :: (load 4 from %fixed-stack.2, align 8)
+    ; MIPS32: [[FRAME_INDEX3:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.3
+    ; MIPS32: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX3]](p0) :: (load 4 from %fixed-stack.3)
+    ; MIPS32: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[LOAD]], [[COPY]]
+    ; MIPS32: [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), [[LOAD]](s32), [[COPY]]
+    ; MIPS32: [[TRUNC:%[0-9]+]]:_(s1) = G_TRUNC [[ICMP]](s32)
+    ; MIPS32: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[LOAD1]], [[COPY1]]
+    ; MIPS32: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; MIPS32: [[COPY4:%[0-9]+]]:_(s32) = COPY [[ICMP]](s32)
+    ; MIPS32: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY4]], [[C]]
+    ; MIPS32: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[SUB1]], [[AND]]
+    ; MIPS32: [[ICMP1:%[0-9]+]]:_(s32) = G_ICMP intpred(eq), [[LOAD1]](s32), [[COPY1]]
+    ; MIPS32: [[ICMP2:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), [[LOAD1]](s32), [[COPY1]]
+    ; MIPS32: [[COPY5:%[0-9]+]]:_(s32) = COPY [[ICMP]](s32)
+    ; MIPS32: [[COPY6:%[0-9]+]]:_(s32) = COPY [[ICMP2]](s32)
+    ; MIPS32: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; MIPS32: [[COPY7:%[0-9]+]]:_(s32) = COPY [[ICMP1]](s32)
+    ; MIPS32: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY7]], [[C1]]
+    ; MIPS32: [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[AND1]](s32), [[COPY5]], [[COPY6]]
+    ; MIPS32: [[TRUNC1:%[0-9]+]]:_(s1) = G_TRUNC [[SELECT]](s32)
+    ; MIPS32: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[LOAD2]], [[COPY2]]
+    ; MIPS32: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; MIPS32: [[COPY8:%[0-9]+]]:_(s32) = COPY [[SELECT]](s32)
+    ; MIPS32: [[AND2:%[0-9]+]]:_(s32) = G_AND [[COPY8]], [[C2]]
+    ; MIPS32: [[SUB4:%[0-9]+]]:_(s32) = G_SUB [[SUB3]], [[AND2]]
+    ; MIPS32: [[ICMP3:%[0-9]+]]:_(s32) = G_ICMP intpred(eq), [[LOAD2]](s32), [[COPY2]]
+    ; MIPS32: [[ICMP4:%[0-9]+]]:_(s32) = G_ICMP intpred(ult), [[LOAD2]](s32), [[COPY2]]
+    ; MIPS32: [[COPY9:%[0-9]+]]:_(s32) = COPY [[SELECT]](s32)
+    ; MIPS32: [[COPY10:%[0-9]+]]:_(s32) = COPY [[ICMP4]](s32)
+    ; MIPS32: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; MIPS32: [[COPY11:%[0-9]+]]:_(s32) = COPY [[ICMP3]](s32)
+    ; MIPS32: [[AND3:%[0-9]+]]:_(s32) = G_AND [[COPY11]], [[C3]]
+    ; MIPS32: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[AND3]](s32), [[COPY9]], [[COPY10]]
+    ; MIPS32: [[SUB5:%[0-9]+]]:_(s32) = G_SUB [[LOAD3]], [[COPY3]]
+    ; MIPS32: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; MIPS32: [[COPY12:%[0-9]+]]:_(s32) = COPY [[SELECT1]](s32)
+    ; MIPS32: [[AND4:%[0-9]+]]:_(s32) = G_AND [[COPY12]], [[C4]]
+    ; MIPS32: [[SUB6:%[0-9]+]]:_(s32) = G_SUB [[SUB5]], [[AND4]]
+    ; MIPS32: $v0 = COPY [[SUB]](s32)
+    ; MIPS32: $v1 = COPY [[SUB2]](s32)
+    ; MIPS32: $a0 = COPY [[SUB4]](s32)
+    ; MIPS32: $a1 = COPY [[SUB6]](s32)
+    ; MIPS32: RetRA implicit $v0, implicit $v1, implicit $a0, implicit $a1
+    %2:_(s32) = COPY $a0
+    %3:_(s32) = COPY $a1
+    %4:_(s32) = COPY $a2
+    %5:_(s32) = COPY $a3
+    %0:_(s128) = G_MERGE_VALUES %2(s32), %3(s32), %4(s32), %5(s32)
+    %10:_(p0) = G_FRAME_INDEX %fixed-stack.3
+    %6:_(s32) = G_LOAD %10(p0) :: (load 4 from %fixed-stack.3, align 8)
+    %11:_(p0) = G_FRAME_INDEX %fixed-stack.2
+    %7:_(s32) = G_LOAD %11(p0) :: (load 4 from %fixed-stack.2, align 4)
+    %12:_(p0) = G_FRAME_INDEX %fixed-stack.1
+    %8:_(s32) = G_LOAD %12(p0) :: (load 4 from %fixed-stack.1, align 8)
+    %13:_(p0) = G_FRAME_INDEX %fixed-stack.0
+    %9:_(s32) = G_LOAD %13(p0) :: (load 4 from %fixed-stack.0, align 4)
+    %1:_(s128) = G_MERGE_VALUES %6(s32), %7(s32), %8(s32), %9(s32)
+    %14:_(s128) = G_SUB %1, %0
+    %15:_(s32), %16:_(s32), %17:_(s32), %18:_(s32) = G_UNMERGE_VALUES %14(s128)
+    $v0 = COPY %15(s32)
+    $v1 = COPY %16(s32)
+    $a0 = COPY %17(s32)
+    $a1 = COPY %18(s32)
+    RetRA implicit $v0, implicit $v1, implicit $a0, implicit $a1
+
+...
diff --git a/test/CodeGen/Mips/GlobalISel/legalizer/truncStore_and_aExtLoad.mir b/test/CodeGen/Mips/GlobalISel/legalizer/truncStore_and_aExtLoad.mir
new file mode 100644
index 0000000000000000000000000000000000000000..a65d25b67995ffbf37b96a15251af962f83db504
--- /dev/null
+++ b/test/CodeGen/Mips/GlobalISel/legalizer/truncStore_and_aExtLoad.mir
@@ -0,0 +1,145 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -O0 -mtriple=mipsel-linux-gnu -run-pass=legalizer -verify-machineinstrs %s -o - | FileCheck %s -check-prefixes=MIPS32
+--- |
+
+  define void @load1_s8_to_load1_s32(i8* %px) {entry: ret void}
+  define void @load2_s16_to_load2_s32(i16* %px) {entry: ret void}
+  define void @load_store_i1(i1* %px, i1* %py) {entry: ret void}
+  define void @load_store_i8(i8* %px, i8* %py) {entry: ret void}
+  define void @load_store_i16(i16* %px, i16* %py) {entry: ret void}
+  define void @load_store_i32(i32* %px, i32* %py) {entry: ret void}
+
+...
+---
+name:            load1_s8_to_load1_s32
+alignment:       2
+tracksRegLiveness: true
+body:             |
+  bb.1.entry:
+    liveins: $a0
+
+    ; MIPS32-LABEL: name: load1_s8_to_load1_s32
+    ; MIPS32: liveins: $a0
+    ; MIPS32: [[COPY:%[0-9]+]]:_(p0) = COPY $a0
+    ; MIPS32: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p0) :: (load 1 from %ir.px)
+    ; MIPS32: $v0 = COPY [[LOAD]](s32)
+    ; MIPS32: RetRA implicit $v0
+    %0:_(p0) = COPY $a0
+    %2:_(s32) = G_LOAD %0(p0) :: (load 1 from %ir.px)
+    $v0 = COPY %2(s32)
+    RetRA implicit $v0
+
+...
+---
+name:            load2_s16_to_load2_s32
+alignment:       2
+tracksRegLiveness: true
+body:             |
+  bb.1.entry:
+    liveins: $a0
+
+    ; MIPS32-LABEL: name: load2_s16_to_load2_s32
+    ; MIPS32: liveins: $a0
+    ; MIPS32: [[COPY:%[0-9]+]]:_(p0) = COPY $a0
+    ; MIPS32: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p0) :: (load 2 from %ir.px)
+    ; MIPS32: $v0 = COPY [[LOAD]](s32)
+    ; MIPS32: RetRA implicit $v0
+    %0:_(p0) = COPY $a0
+    %2:_(s32) = G_LOAD %0(p0) :: (load 2 from %ir.px)
+    $v0 = COPY %2(s32)
+    RetRA implicit $v0
+
+...
+---
+name:            load_store_i1
+alignment:       2
+tracksRegLiveness: true
+body:             |
+  bb.1.entry:
+    liveins: $a0, $a1
+
+    ; MIPS32-LABEL: name: load_store_i1
+    ; MIPS32: liveins: $a0, $a1
+    ; MIPS32: [[COPY:%[0-9]+]]:_(p0) = COPY $a0
+    ; MIPS32: [[COPY1:%[0-9]+]]:_(p0) = COPY $a1
+    ; MIPS32: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY1]](p0) :: (load 1 from %ir.py)
+    ; MIPS32: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+    ; MIPS32: [[COPY2:%[0-9]+]]:_(s32) = COPY [[LOAD]](s32)
+    ; MIPS32: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C]]
+    ; MIPS32: G_STORE [[AND]](s32), [[COPY]](p0) :: (store 1 into %ir.px)
+    ; MIPS32: RetRA
+    %0:_(p0) = COPY $a0
+    %1:_(p0) = COPY $a1
+    %2:_(s1) = G_LOAD %1(p0) :: (load 1 from %ir.py)
+    G_STORE %2(s1), %0(p0) :: (store 1 into %ir.px)
+    RetRA
+
+...
+---
+name:            load_store_i8
+alignment:       2
+tracksRegLiveness: true
+body:             |
+  bb.1.entry:
+    liveins: $a0, $a1
+
+    ; MIPS32-LABEL: name: load_store_i8
+    ; MIPS32: liveins: $a0, $a1
+    ; MIPS32: [[COPY:%[0-9]+]]:_(p0) = COPY $a0
+    ; MIPS32: [[COPY1:%[0-9]+]]:_(p0) = COPY $a1
+    ; MIPS32: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY1]](p0) :: (load 1 from %ir.py)
+    ; MIPS32: [[COPY2:%[0-9]+]]:_(s32) = COPY [[LOAD]](s32)
+    ; MIPS32: G_STORE [[COPY2]](s32), [[COPY]](p0) :: (store 1 into %ir.px)
+    ; MIPS32: RetRA
+    %0:_(p0) = COPY $a0
+    %1:_(p0) = COPY $a1
+    %2:_(s8) = G_LOAD %1(p0) :: (load 1 from %ir.py)
+    G_STORE %2(s8), %0(p0) :: (store 1 into %ir.px)
+    RetRA
+
+...
+---
+name:            load_store_i16
+alignment:       2
+tracksRegLiveness: true
+body:             |
+  bb.1.entry:
+    liveins: $a0, $a1
+
+    ; MIPS32-LABEL: name: load_store_i16
+    ; MIPS32: liveins: $a0, $a1
+    ; MIPS32: [[COPY:%[0-9]+]]:_(p0) = COPY $a0
+    ; MIPS32: [[COPY1:%[0-9]+]]:_(p0) = COPY $a1
+    ; MIPS32: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY1]](p0) :: (load 2 from %ir.py)
+    ; MIPS32: [[COPY2:%[0-9]+]]:_(s32) = COPY [[LOAD]](s32)
+    ; MIPS32: G_STORE [[COPY2]](s32), [[COPY]](p0) :: (store 2 into %ir.px)
+    ; MIPS32: RetRA
+    %0:_(p0) = COPY $a0
+    %1:_(p0) = COPY $a1
+    %2:_(s16) = G_LOAD %1(p0) :: (load 2 from %ir.py)
+    G_STORE %2(s16), %0(p0) :: (store 2 into %ir.px)
+    RetRA
+
+...
+---
+name:            load_store_i32
+alignment:       2
+tracksRegLiveness: true
+body:             |
+  bb.1.entry:
+    liveins: $a0, $a1
+
+    ; MIPS32-LABEL: name: load_store_i32
+    ; MIPS32: liveins: $a0, $a1
+    ; MIPS32: [[COPY:%[0-9]+]]:_(p0) = COPY $a0
+    ; MIPS32: [[COPY1:%[0-9]+]]:_(p0) = COPY $a1
+    ; MIPS32: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY1]](p0) :: (load 4 from %ir.py)
+    ; MIPS32: G_STORE [[LOAD]](s32), [[COPY]](p0) :: (store 4 into %ir.px)
+    ; MIPS32: RetRA
+    %0:_(p0) = COPY $a0
+    %1:_(p0) = COPY $a1
+    %2:_(s32) = G_LOAD %1(p0) :: (load 4 from %ir.py)
+    G_STORE %2(s32), %0(p0) :: (store 4 into %ir.px)
+    RetRA
+
+...
diff --git a/test/CodeGen/Mips/GlobalISel/legalizer/zextLoad_and_sextLoad.mir b/test/CodeGen/Mips/GlobalISel/legalizer/zextLoad_and_sextLoad.mir
new file mode 100644
index 0000000000000000000000000000000000000000..51a0e6cc5afe1394e4ba76430f03ab19f885d707
--- /dev/null
+++ b/test/CodeGen/Mips/GlobalISel/legalizer/zextLoad_and_sextLoad.mir
@@ -0,0 +1,178 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -O0 -mtriple=mipsel-linux-gnu -run-pass=legalizer -verify-machineinstrs %s -o - | FileCheck %s -check-prefixes=MIPS32
+--- |
+
+  define void @load1_s8_to_zextLoad1_s32(i8* %px) {entry: ret void}
+  define void @load2_s16_to_zextLoad2_s32(i16* %px) {entry: ret void}
+  define void @load1_s8_to_zextLoad1_s16(i8* %px) {entry: ret void}
+  define void @load1_s8_to_zextLoad1_s16_to_zextLoad1_s32(i8* %px) {entry: ret void}
+  define void @load1_s8_to_sextLoad1_s32(i8* %px) {entry: ret void}
+  define void @load2_s16_to_sextLoad2_s32(i16* %px) {entry: ret void}
+  define void @load1_s8_to_sextLoad1_s16(i8* %px) {entry: ret void}
+  define void @load1_s8_to_sextLoad1_s16_to_sextLoad1_s32(i8* %px) {entry: ret void}
+
+...
+---
+name:            load1_s8_to_zextLoad1_s32
+alignment:       2
+tracksRegLiveness: true
+body:             |
+  bb.1.entry:
+    liveins: $a0
+
+    ; MIPS32-LABEL: name: load1_s8_to_zextLoad1_s32
+    ; MIPS32: liveins: $a0
+    ; MIPS32: [[COPY:%[0-9]+]]:_(p0) = COPY $a0
+    ; MIPS32: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p0) :: (load 1 from %ir.px)
+    ; MIPS32: $v0 = COPY [[ZEXTLOAD]](s32)
+    ; MIPS32: RetRA implicit $v0
+    %0:_(p0) = COPY $a0
+    %2:_(s32) = G_ZEXTLOAD %0(p0) :: (load 1 from %ir.px)
+    $v0 = COPY %2(s32)
+    RetRA implicit $v0
+
+...
+---
+name:            load2_s16_to_zextLoad2_s32
+alignment:       2
+tracksRegLiveness: true
+body:             |
+  bb.1.entry:
+    liveins: $a0
+
+    ; MIPS32-LABEL: name: load2_s16_to_zextLoad2_s32
+    ; MIPS32: liveins: $a0
+    ; MIPS32: [[COPY:%[0-9]+]]:_(p0) = COPY $a0
+    ; MIPS32: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p0) :: (load 2 from %ir.px)
+    ; MIPS32: $v0 = COPY [[ZEXTLOAD]](s32)
+    ; MIPS32: RetRA implicit $v0
+    %0:_(p0) = COPY $a0
+    %2:_(s32) = G_ZEXTLOAD %0(p0) :: (load 2 from %ir.px)
+    $v0 = COPY %2(s32)
+    RetRA implicit $v0
+
+...
+---
+name:            load1_s8_to_zextLoad1_s16
+alignment:       2
+tracksRegLiveness: true
+body:             |
+  bb.1.entry:
+    liveins: $a0
+
+    ; MIPS32-LABEL: name: load1_s8_to_zextLoad1_s16
+    ; MIPS32: liveins: $a0
+    ; MIPS32: [[COPY:%[0-9]+]]:_(p0) = COPY $a0
+    ; MIPS32: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p0) :: (load 1 from %ir.px)
+    ; MIPS32: [[COPY1:%[0-9]+]]:_(s32) = COPY [[ZEXTLOAD]](s32)
+    ; MIPS32: $v0 = COPY [[COPY1]](s32)
+    ; MIPS32: RetRA implicit $v0
+    %0:_(p0) = COPY $a0
+    %2:_(s16) = G_ZEXTLOAD %0(p0) :: (load 1 from %ir.px)
+    %3:_(s32) = G_ANYEXT %2(s16)
+    $v0 = COPY %3(s32)
+    RetRA implicit $v0
+
+...
+---
+name:            load1_s8_to_zextLoad1_s16_to_zextLoad1_s32
+alignment:       2
+tracksRegLiveness: true
+body:             |
+  bb.1.entry:
+    liveins: $a0
+
+    ; MIPS32-LABEL: name: load1_s8_to_zextLoad1_s16_to_zextLoad1_s32
+    ; MIPS32: liveins: $a0
+    ; MIPS32: [[COPY:%[0-9]+]]:_(p0) = COPY $a0
+    ; MIPS32: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p0) :: (load 1 from %ir.px)
+    ; MIPS32: $v0 = COPY [[ZEXTLOAD]](s32)
+    ; MIPS32: RetRA implicit $v0
+    %0:_(p0) = COPY $a0
+    %3:_(s32) = G_ZEXTLOAD %0(p0) :: (load 1 from %ir.px)
+    $v0 = COPY %3(s32)
+    RetRA implicit $v0
+
+...
+---
+name:            load1_s8_to_sextLoad1_s32
+alignment:       2
+tracksRegLiveness: true
+body:             |
+  bb.1.entry:
+    liveins: $a0
+
+    ; MIPS32-LABEL: name: load1_s8_to_sextLoad1_s32
+    ; MIPS32: liveins: $a0
+    ; MIPS32: [[COPY:%[0-9]+]]:_(p0) = COPY $a0
+    ; MIPS32: [[SEXTLOAD:%[0-9]+]]:_(s32) = G_SEXTLOAD [[COPY]](p0) :: (load 1 from %ir.px)
+    ; MIPS32: $v0 = COPY [[SEXTLOAD]](s32)
+    ; MIPS32: RetRA implicit $v0
+    %0:_(p0) = COPY $a0
+    %2:_(s32) = G_SEXTLOAD %0(p0) :: (load 1 from %ir.px)
+    $v0 = COPY %2(s32)
+    RetRA implicit $v0
+
+...
+---
+name:            load2_s16_to_sextLoad2_s32
+alignment:       2
+tracksRegLiveness: true
+body:             |
+  bb.1.entry:
+    liveins: $a0
+
+    ; MIPS32-LABEL: name: load2_s16_to_sextLoad2_s32
+    ; MIPS32: liveins: $a0
+    ; MIPS32: [[COPY:%[0-9]+]]:_(p0) = COPY $a0
+    ; MIPS32: [[SEXTLOAD:%[0-9]+]]:_(s32) = G_SEXTLOAD [[COPY]](p0) :: (load 2 from %ir.px)
+    ; MIPS32: $v0 = COPY [[SEXTLOAD]](s32)
+    ; MIPS32: RetRA implicit $v0
+    %0:_(p0) = COPY $a0
+    %2:_(s32) = G_SEXTLOAD %0(p0) :: (load 2 from %ir.px)
+    $v0 = COPY %2(s32)
+    RetRA implicit $v0
+
+...
+---
+name:            load1_s8_to_sextLoad1_s16
+alignment:       2
+tracksRegLiveness: true
+body:             |
+  bb.1.entry:
+    liveins: $a0
+
+    ; MIPS32-LABEL: name: load1_s8_to_sextLoad1_s16
+    ; MIPS32: liveins: $a0
+    ; MIPS32: [[COPY:%[0-9]+]]:_(p0) = COPY $a0
+    ; MIPS32: [[SEXTLOAD:%[0-9]+]]:_(s32) = G_SEXTLOAD [[COPY]](p0) :: (load 1 from %ir.px)
+    ; MIPS32: [[COPY1:%[0-9]+]]:_(s32) = COPY [[SEXTLOAD]](s32)
+    ; MIPS32: $v0 = COPY [[COPY1]](s32)
+    ; MIPS32: RetRA implicit $v0
+    %0:_(p0) = COPY $a0
+    %2:_(s16) = G_SEXTLOAD %0(p0) :: (load 1 from %ir.px)
+    %3:_(s32) = G_ANYEXT %2(s16)
+    $v0 = COPY %3(s32)
+    RetRA implicit $v0
+
+...
+---
+name:            load1_s8_to_sextLoad1_s16_to_sextLoad1_s32
+alignment:       2
+tracksRegLiveness: true
+body:             |
+  bb.1.entry:
+    liveins: $a0
+
+    ; MIPS32-LABEL: name: load1_s8_to_sextLoad1_s16_to_sextLoad1_s32
+    ; MIPS32: liveins: $a0
+    ; MIPS32: [[COPY:%[0-9]+]]:_(p0) = COPY $a0
+    ; MIPS32: [[SEXTLOAD:%[0-9]+]]:_(s32) = G_SEXTLOAD [[COPY]](p0) :: (load 1 from %ir.px)
+    ; MIPS32: $v0 = COPY [[SEXTLOAD]](s32)
+    ; MIPS32: RetRA implicit $v0
+    %0:_(p0) = COPY $a0
+    %3:_(s32) = G_SEXTLOAD %0(p0) :: (load 1 from %ir.px)
+    $v0 = COPY %3(s32)
+    RetRA implicit $v0
+
+...
diff --git a/test/CodeGen/Mips/GlobalISel/llvm-ir/mul.ll b/test/CodeGen/Mips/GlobalISel/llvm-ir/mul.ll
new file mode 100644
index 0000000000000000000000000000000000000000..11e5426ad771ae136e7fda3059fa0afec4c4e2db
--- /dev/null
+++ b/test/CodeGen/Mips/GlobalISel/llvm-ir/mul.ll
@@ -0,0 +1,88 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc  -O0 -mtriple=mipsel-linux-gnu -global-isel  -verify-machineinstrs %s -o -| FileCheck %s -check-prefixes=MIPS32
+define i32 @mul_i32(i32 %x, i32 %y) {
+; MIPS32-LABEL: mul_i32:
+; MIPS32:       # %bb.0: # %entry
+; MIPS32-NEXT:    mul $2, $4, $5
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+entry:
+  %z = mul i32 %x, %y
+  ret i32 %z
+}
+
+define signext i8 @mul_i8_sext(i8 signext %a, i8 signext %b) {
+; MIPS32-LABEL: mul_i8_sext:
+; MIPS32:       # %bb.0: # %entry
+; MIPS32-NEXT:    mul $4, $5, $4
+; MIPS32-NEXT:    sll $4, $4, 24
+; MIPS32-NEXT:    sra $2, $4, 24
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+entry:
+  %mul = mul i8 %b, %a
+  ret i8 %mul
+}
+
+define zeroext i8 @mul_i8_zext(i8 zeroext %a, i8 zeroext %b) {
+; MIPS32-LABEL: mul_i8_zext:
+; MIPS32:       # %bb.0: # %entry
+; MIPS32-NEXT:    mul $4, $5, $4
+; MIPS32-NEXT:    lui $5, 0
+; MIPS32-NEXT:    ori $5, $5, 255
+; MIPS32-NEXT:    and $2, $4, $5
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+entry:
+  %mul = mul i8 %b, %a
+  ret i8 %mul
+}
+
+define i8 @mul_i8_aext(i8 %a, i8 %b) {
+; MIPS32-LABEL: mul_i8_aext:
+; MIPS32:       # %bb.0: # %entry
+; MIPS32-NEXT:    mul $2, $5, $4
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+entry:
+  %mul = mul i8 %b, %a
+  ret i8 %mul
+}
+
+define signext i16 @mul_i16_sext(i16 signext %a, i16 signext %b) {
+; MIPS32-LABEL: mul_i16_sext:
+; MIPS32:       # %bb.0: # %entry
+; MIPS32-NEXT:    mul $4, $5, $4
+; MIPS32-NEXT:    sll $4, $4, 16
+; MIPS32-NEXT:    sra $2, $4, 16
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+entry:
+  %mul = mul i16 %b, %a
+  ret i16 %mul
+}
+
+define zeroext i16 @mul_i16_zext(i16 zeroext %a, i16 zeroext %b) {
+; MIPS32-LABEL: mul_i16_zext:
+; MIPS32:       # %bb.0: # %entry
+; MIPS32-NEXT:    mul $4, $5, $4
+; MIPS32-NEXT:    lui $5, 0
+; MIPS32-NEXT:    ori $5, $5, 65535
+; MIPS32-NEXT:    and $2, $4, $5
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+entry:
+  %mul = mul i16 %b, %a
+  ret i16 %mul
+}
+
+define i16 @mul_i16_aext(i16 %a, i16 %b) {
+; MIPS32-LABEL: mul_i16_aext:
+; MIPS32:       # %bb.0: # %entry
+; MIPS32-NEXT:    mul $2, $5, $4
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+entry:
+  %mul = mul i16 %b, %a
+  ret i16 %mul
+}
diff --git a/test/CodeGen/Mips/GlobalISel/llvm-ir/select.ll b/test/CodeGen/Mips/GlobalISel/llvm-ir/select.ll
new file mode 100644
index 0000000000000000000000000000000000000000..f15977cf6b78bdfea3565e354b7af0f353029db0
--- /dev/null
+++ b/test/CodeGen/Mips/GlobalISel/llvm-ir/select.ll
@@ -0,0 +1,83 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc  -O0 -mtriple=mipsel-linux-gnu -global-isel  -verify-machineinstrs %s -o -| FileCheck %s -check-prefixes=MIPS32
+
+define i8 @select_i8(i1 %test, i8 %a, i8 %b) {
+; MIPS32-LABEL: select_i8:
+; MIPS32:       # %bb.0: # %entry
+; MIPS32-NEXT:    lui $1, 0
+; MIPS32-NEXT:    ori $1, $1, 1
+; MIPS32-NEXT:    and $1, $4, $1
+; MIPS32-NEXT:    movn $6, $5, $1
+; MIPS32-NEXT:    move $2, $6
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+entry:
+  %cond = select i1 %test, i8 %a, i8 %b
+  ret i8 %cond
+}
+
+define i16 @select_i16(i1 %test, i16 %a, i16 %b) {
+; MIPS32-LABEL: select_i16:
+; MIPS32:       # %bb.0: # %entry
+; MIPS32-NEXT:    lui $1, 0
+; MIPS32-NEXT:    ori $1, $1, 1
+; MIPS32-NEXT:    and $1, $4, $1
+; MIPS32-NEXT:    movn $6, $5, $1
+; MIPS32-NEXT:    move $2, $6
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+entry:
+  %cond = select i1 %test, i16 %a, i16 %b
+  ret i16 %cond
+}
+
+define i32 @select_i32(i1 %test, i32 %a, i32 %b) {
+; MIPS32-LABEL: select_i32:
+; MIPS32:       # %bb.0: # %entry
+; MIPS32-NEXT:    lui $1, 0
+; MIPS32-NEXT:    ori $1, $1, 1
+; MIPS32-NEXT:    and $1, $4, $1
+; MIPS32-NEXT:    movn $6, $5, $1
+; MIPS32-NEXT:    move $2, $6
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+entry:
+  %cond = select i1 %test, i32 %a, i32 %b
+  ret i32 %cond
+}
+
+define i32* @select_ptr(i1 %test, i32* %a, i32* %b) {
+; MIPS32-LABEL: select_ptr:
+; MIPS32:       # %bb.0: # %entry
+; MIPS32-NEXT:    lui $1, 0
+; MIPS32-NEXT:    ori $1, $1, 1
+; MIPS32-NEXT:    and $1, $4, $1
+; MIPS32-NEXT:    movn $6, $5, $1
+; MIPS32-NEXT:    move $2, $6
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+entry:
+  %cond = select i1 %test, i32* %a, i32* %b
+  ret i32* %cond
+}
+
+define i32 @select_with_negation(i32 %a, i32 %b, i32 %x, i32 %y) {
+; MIPS32-LABEL: select_with_negation:
+; MIPS32:       # %bb.0: # %entry
+; MIPS32-NEXT:    lui $1, 65535
+; MIPS32-NEXT:    ori $1, $1, 65535
+; MIPS32-NEXT:    slt $4, $4, $5
+; MIPS32-NEXT:    xor $1, $4, $1
+; MIPS32-NEXT:    lui $4, 0
+; MIPS32-NEXT:    ori $4, $4, 1
+; MIPS32-NEXT:    and $1, $1, $4
+; MIPS32-NEXT:    movn $7, $6, $1
+; MIPS32-NEXT:    move $2, $7
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+entry:
+  %cmp = icmp slt i32 %a, %b
+  %lneg = xor i1 %cmp, true
+  %cond = select i1 %lneg, i32 %x, i32 %y
+  ret i32 %cond
+}
diff --git a/test/CodeGen/Mips/GlobalISel/llvm-ir/sub.ll b/test/CodeGen/Mips/GlobalISel/llvm-ir/sub.ll
new file mode 100644
index 0000000000000000000000000000000000000000..e97fed213b0949bf25496c823b60820e5d904764
--- /dev/null
+++ b/test/CodeGen/Mips/GlobalISel/llvm-ir/sub.ll
@@ -0,0 +1,158 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc  -O0 -mtriple=mipsel-linux-gnu -global-isel  -verify-machineinstrs %s -o -| FileCheck %s -check-prefixes=MIPS32
+
+define i32 @sub_i32(i32 %x, i32 %y) {
+; MIPS32-LABEL: sub_i32:
+; MIPS32:       # %bb.0: # %entry
+; MIPS32-NEXT:    subu $2, $4, $5
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+entry:
+  %z = sub i32 %x, %y
+  ret i32 %z
+}
+
+define signext i8 @sub_i8_sext(i8 signext %a, i8 signext %b) {
+; MIPS32-LABEL: sub_i8_sext:
+; MIPS32:       # %bb.0: # %entry
+; MIPS32-NEXT:    subu $4, $5, $4
+; MIPS32-NEXT:    sll $4, $4, 24
+; MIPS32-NEXT:    sra $2, $4, 24
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+entry:
+  %sub = sub i8 %b, %a
+  ret i8 %sub
+}
+
+define zeroext i8 @sub_i8_zext(i8 zeroext %a, i8 zeroext %b) {
+; MIPS32-LABEL: sub_i8_zext:
+; MIPS32:       # %bb.0: # %entry
+; MIPS32-NEXT:    subu $4, $5, $4
+; MIPS32-NEXT:    lui $5, 0
+; MIPS32-NEXT:    ori $5, $5, 255
+; MIPS32-NEXT:    and $2, $4, $5
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+entry:
+  %sub = sub i8 %b, %a
+  ret i8 %sub
+}
+
+define i8 @sub_i8_aext(i8 %a, i8 %b) {
+; MIPS32-LABEL: sub_i8_aext:
+; MIPS32:       # %bb.0: # %entry
+; MIPS32-NEXT:    subu $2, $5, $4
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+entry:
+  %sub = sub i8 %b, %a
+  ret i8 %sub
+}
+
+define signext i16 @sub_i16_sext(i16 signext %a, i16 signext %b) {
+; MIPS32-LABEL: sub_i16_sext:
+; MIPS32:       # %bb.0: # %entry
+; MIPS32-NEXT:    subu $4, $5, $4
+; MIPS32-NEXT:    sll $4, $4, 16
+; MIPS32-NEXT:    sra $2, $4, 16
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+entry:
+  %sub = sub i16 %b, %a
+  ret i16 %sub
+}
+
+define zeroext i16 @sub_i16_zext(i16 zeroext %a, i16 zeroext %b) {
+; MIPS32-LABEL: sub_i16_zext:
+; MIPS32:       # %bb.0: # %entry
+; MIPS32-NEXT:    subu $4, $5, $4
+; MIPS32-NEXT:    lui $5, 0
+; MIPS32-NEXT:    ori $5, $5, 65535
+; MIPS32-NEXT:    and $2, $4, $5
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+entry:
+  %sub = sub i16 %b, %a
+  ret i16 %sub
+}
+
+define i16 @sub_i16_aext(i16 %a, i16 %b) {
+; MIPS32-LABEL: sub_i16_aext:
+; MIPS32:       # %bb.0: # %entry
+; MIPS32-NEXT:    subu $2, $5, $4
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+entry:
+  %sub = sub i16 %b, %a
+  ret i16 %sub
+}
+
+define i64 @sub_i64(i64 %a, i64 %b) {
+; MIPS32-LABEL: sub_i64:
+; MIPS32:       # %bb.0: # %entry
+; MIPS32-NEXT:    subu $2, $6, $4
+; MIPS32-NEXT:    sltu $4, $6, $4
+; MIPS32-NEXT:    subu $5, $7, $5
+; MIPS32-NEXT:    lui $6, 0
+; MIPS32-NEXT:    ori $6, $6, 1
+; MIPS32-NEXT:    and $4, $4, $6
+; MIPS32-NEXT:    subu $3, $5, $4
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+entry:
+  %sub = sub i64 %b, %a
+  ret i64 %sub
+}
+
+define i128 @sub_i128(i128 %a, i128 %b) {
+; MIPS32-LABEL: sub_i128:
+; MIPS32:       # %bb.0: # %entry
+; MIPS32-NEXT:    addiu $1, $sp, 16
+; MIPS32-NEXT:    lw $1, 0($1)
+; MIPS32-NEXT:    addiu $2, $sp, 20
+; MIPS32-NEXT:    lw $2, 0($2)
+; MIPS32-NEXT:    addiu $3, $sp, 24
+; MIPS32-NEXT:    lw $3, 0($3)
+; MIPS32-NEXT:    addiu $8, $sp, 28
+; MIPS32-NEXT:    lw $8, 0($8)
+; MIPS32-NEXT:    subu $9, $1, $4
+; MIPS32-NEXT:    sltu $1, $1, $4
+; MIPS32-NEXT:    subu $4, $2, $5
+; MIPS32-NEXT:    lui $10, 0
+; MIPS32-NEXT:    ori $10, $10, 1
+; MIPS32-NEXT:    and $10, $1, $10
+; MIPS32-NEXT:    subu $4, $4, $10
+; MIPS32-NEXT:    xor $10, $2, $5
+; MIPS32-NEXT:    sltiu $10, $10, 1
+; MIPS32-NEXT:    sltu $2, $2, $5
+; MIPS32-NEXT:    lui $5, 0
+; MIPS32-NEXT:    ori $5, $5, 1
+; MIPS32-NEXT:    and $5, $10, $5
+; MIPS32-NEXT:    movn $2, $1, $5
+; MIPS32-NEXT:    subu $1, $3, $6
+; MIPS32-NEXT:    lui $5, 0
+; MIPS32-NEXT:    ori $5, $5, 1
+; MIPS32-NEXT:    and $5, $2, $5
+; MIPS32-NEXT:    subu $1, $1, $5
+; MIPS32-NEXT:    xor $5, $3, $6
+; MIPS32-NEXT:    sltiu $5, $5, 1
+; MIPS32-NEXT:    sltu $3, $3, $6
+; MIPS32-NEXT:    lui $6, 0
+; MIPS32-NEXT:    ori $6, $6, 1
+; MIPS32-NEXT:    and $5, $5, $6
+; MIPS32-NEXT:    movn $3, $2, $5
+; MIPS32-NEXT:    subu $2, $8, $7
+; MIPS32-NEXT:    lui $5, 0
+; MIPS32-NEXT:    ori $5, $5, 1
+; MIPS32-NEXT:    and $3, $3, $5
+; MIPS32-NEXT:    subu $5, $2, $3
+; MIPS32-NEXT:    move $2, $9
+; MIPS32-NEXT:    move $3, $4
+; MIPS32-NEXT:    move $4, $1
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+entry:
+  %sub = sub i128 %b, %a
+  ret i128 %sub
+}
diff --git a/test/CodeGen/Mips/GlobalISel/llvm-ir/truncStore_and_aExtLoad.ll b/test/CodeGen/Mips/GlobalISel/llvm-ir/truncStore_and_aExtLoad.ll
new file mode 100644
index 0000000000000000000000000000000000000000..1f4811ad84e40b9c6cca222a3cb426ff7f7b4fa5
--- /dev/null
+++ b/test/CodeGen/Mips/GlobalISel/llvm-ir/truncStore_and_aExtLoad.ll
@@ -0,0 +1,79 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc  -O0 -mtriple=mipsel-linux-gnu -global-isel  -verify-machineinstrs %s -o -| FileCheck %s -check-prefixes=MIPS32
+
+define i8 @load1_s8_to_load1_s32(i8* %px) {
+; MIPS32-LABEL: load1_s8_to_load1_s32:
+; MIPS32:       # %bb.0: # %entry
+; MIPS32-NEXT:    lbu $2, 0($4)
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+entry:
+  %0 = load i8, i8* %px
+  ret i8 %0
+}
+
+define i16 @load2_s16_to_load2_s32(i16* %px) {
+; MIPS32-LABEL: load2_s16_to_load2_s32:
+; MIPS32:       # %bb.0: # %entry
+; MIPS32-NEXT:    lhu $2, 0($4)
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+entry:
+  %0 = load i16, i16* %px
+  ret i16 %0
+}
+
+define void @load_store_i1(i1* %px, i1* %py) {
+; MIPS32-LABEL: load_store_i1:
+; MIPS32:       # %bb.0: # %entry
+; MIPS32-NEXT:    lbu $5, 0($5)
+; MIPS32-NEXT:    lui $1, 0
+; MIPS32-NEXT:    ori $1, $1, 1
+; MIPS32-NEXT:    and $1, $5, $1
+; MIPS32-NEXT:    sb $1, 0($4)
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+entry:
+  %0 = load i1, i1* %py
+  store i1 %0, i1* %px
+  ret void
+}
+
+define void @load_store_i8(i8* %px, i8* %py) {
+; MIPS32-LABEL: load_store_i8:
+; MIPS32:       # %bb.0: # %entry
+; MIPS32-NEXT:    lbu $5, 0($5)
+; MIPS32-NEXT:    sb $5, 0($4)
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+entry:
+  %0 = load i8, i8* %py
+  store i8 %0, i8* %px
+  ret void
+}
+
+define void @load_store_i16(i16* %px, i16* %py) {
+; MIPS32-LABEL: load_store_i16:
+; MIPS32:       # %bb.0: # %entry
+; MIPS32-NEXT:    lhu $5, 0($5)
+; MIPS32-NEXT:    sh $5, 0($4)
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+entry:
+  %0 = load i16, i16* %py
+  store i16 %0, i16* %px
+  ret void
+}
+
+define void @load_store_i32(i32* %px, i32* %py) {
+; MIPS32-LABEL: load_store_i32:
+; MIPS32:       # %bb.0: # %entry
+; MIPS32-NEXT:    lw $5, 0($5)
+; MIPS32-NEXT:    sw $5, 0($4)
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+entry:
+  %0 = load i32, i32* %py
+  store i32 %0, i32* %px
+  ret void
+}
diff --git a/test/CodeGen/Mips/GlobalISel/llvm-ir/zextLoad_and_sextLoad.ll b/test/CodeGen/Mips/GlobalISel/llvm-ir/zextLoad_and_sextLoad.ll
new file mode 100644
index 0000000000000000000000000000000000000000..40bfc1dec0e9f4b8e0b42f3b778d1daed2fe1ed8
--- /dev/null
+++ b/test/CodeGen/Mips/GlobalISel/llvm-ir/zextLoad_and_sextLoad.ll
@@ -0,0 +1,98 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc  -O0 -mtriple=mipsel-linux-gnu -global-isel  -verify-machineinstrs %s -o -| FileCheck %s -check-prefixes=MIPS32
+
+define i32 @load1_s8_to_zextLoad1_s32(i8* %px) {
+; MIPS32-LABEL: load1_s8_to_zextLoad1_s32:
+; MIPS32:       # %bb.0: # %entry
+; MIPS32-NEXT:    lbu $2, 0($4)
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+entry:
+  %0 = load i8, i8* %px
+  %conv = zext i8 %0 to i32
+  ret i32 %conv
+}
+
+define i32 @load2_s16_to_zextLoad2_s32(i16* %px) {
+; MIPS32-LABEL: load2_s16_to_zextLoad2_s32:
+; MIPS32:       # %bb.0: # %entry
+; MIPS32-NEXT:    lhu $2, 0($4)
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+entry:
+  %0 = load i16, i16* %px
+  %conv = zext i16 %0 to i32
+  ret i32 %conv
+}
+
+define i16 @load1_s8_to_zextLoad1_s16(i8* %px) {
+; MIPS32-LABEL: load1_s8_to_zextLoad1_s16:
+; MIPS32:       # %bb.0: # %entry
+; MIPS32-NEXT:    lbu $2, 0($4)
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+entry:
+  %0 = load i8, i8* %px
+  %conv = zext i8 %0 to i16
+  ret i16 %conv
+}
+
+define zeroext i16 @load1_s8_to_zextLoad1_s16_to_zextLoad1_s32(i8* %px) {
+; MIPS32-LABEL: load1_s8_to_zextLoad1_s16_to_zextLoad1_s32:
+; MIPS32:       # %bb.0: # %entry
+; MIPS32-NEXT:    lbu $2, 0($4)
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+entry:
+  %0 = load i8, i8* %px
+  %conv = zext i8 %0 to i16
+  ret i16 %conv
+}
+
+define i32 @load1_s8_to_sextLoad1_s32(i8* %px) {
+; MIPS32-LABEL: load1_s8_to_sextLoad1_s32:
+; MIPS32:       # %bb.0: # %entry
+; MIPS32-NEXT:    lb $2, 0($4)
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+entry:
+  %0 = load i8, i8* %px
+  %conv = sext i8 %0 to i32
+  ret i32 %conv
+}
+
+define i32 @load2_s16_to_sextLoad2_s32(i16* %px) {
+; MIPS32-LABEL: load2_s16_to_sextLoad2_s32:
+; MIPS32:       # %bb.0: # %entry
+; MIPS32-NEXT:    lh $2, 0($4)
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+entry:
+  %0 = load i16, i16* %px
+  %conv = sext i16 %0 to i32
+  ret i32 %conv
+}
+
+define i16 @load1_s8_to_sextLoad1_s16(i8* %px) {
+; MIPS32-LABEL: load1_s8_to_sextLoad1_s16:
+; MIPS32:       # %bb.0: # %entry
+; MIPS32-NEXT:    lb $2, 0($4)
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+entry:
+  %0 = load i8, i8* %px
+  %conv = sext i8 %0 to i16
+  ret i16 %conv
+}
+
+define signext i16 @load1_s8_to_sextLoad1_s16_to_sextLoad1_s32(i8* %px) {
+; MIPS32-LABEL: load1_s8_to_sextLoad1_s16_to_sextLoad1_s32:
+; MIPS32:       # %bb.0: # %entry
+; MIPS32-NEXT:    lb $2, 0($4)
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+entry:
+  %0 = load i8, i8* %px
+  %conv = sext i8 %0 to i16
+  ret i16 %conv
+}
diff --git a/test/CodeGen/Mips/GlobalISel/mips-prelegalizer-combiner/truncStore_and_aExtLoad.mir b/test/CodeGen/Mips/GlobalISel/mips-prelegalizer-combiner/truncStore_and_aExtLoad.mir
new file mode 100644
index 0000000000000000000000000000000000000000..63b9267325ec80c4163679ab5de779d2b4ac8e21
--- /dev/null
+++ b/test/CodeGen/Mips/GlobalISel/mips-prelegalizer-combiner/truncStore_and_aExtLoad.mir
@@ -0,0 +1,50 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -O0 -mtriple=mipsel-linux-gnu -run-pass=mips-prelegalizer-combiner -verify-machineinstrs %s -o - | FileCheck %s -check-prefixes=MIPS32
+--- |
+
+  define void @load1_s8_to_load1_s32(i8* %px) {entry: ret void}
+  define void @load2_s16_to_load2_s32(i16* %px) {entry: ret void}
+
+...
+---
+name:            load1_s8_to_load1_s32
+alignment:       2
+tracksRegLiveness: true
+body:             |
+  bb.1.entry:
+    liveins: $a0
+
+    ; MIPS32-LABEL: name: load1_s8_to_load1_s32
+    ; MIPS32: liveins: $a0
+    ; MIPS32: [[COPY:%[0-9]+]]:_(p0) = COPY $a0
+    ; MIPS32: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p0) :: (load 1 from %ir.px)
+    ; MIPS32: $v0 = COPY [[LOAD]](s32)
+    ; MIPS32: RetRA implicit $v0
+    %0:_(p0) = COPY $a0
+    %1:_(s8) = G_LOAD %0(p0) :: (load 1 from %ir.px)
+    %2:_(s32) = G_ANYEXT %1(s8)
+    $v0 = COPY %2(s32)
+    RetRA implicit $v0
+
+...
+---
+name:            load2_s16_to_load2_s32
+alignment:       2
+tracksRegLiveness: true
+body:             |
+  bb.1.entry:
+    liveins: $a0
+
+    ; MIPS32-LABEL: name: load2_s16_to_load2_s32
+    ; MIPS32: liveins: $a0
+    ; MIPS32: [[COPY:%[0-9]+]]:_(p0) = COPY $a0
+    ; MIPS32: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY]](p0) :: (load 2 from %ir.px)
+    ; MIPS32: $v0 = COPY [[LOAD]](s32)
+    ; MIPS32: RetRA implicit $v0
+    %0:_(p0) = COPY $a0
+    %1:_(s16) = G_LOAD %0(p0) :: (load 2 from %ir.px)
+    %2:_(s32) = G_ANYEXT %1(s16)
+    $v0 = COPY %2(s32)
+    RetRA implicit $v0
+
+...
diff --git a/test/CodeGen/Mips/GlobalISel/mips-prelegalizer-combiner/tryCombine.mir b/test/CodeGen/Mips/GlobalISel/mips-prelegalizer-combiner/tryCombine.mir
new file mode 100644
index 0000000000000000000000000000000000000000..a8b2db19e4181e07b6ad0dfa93ad7e7a722367a8
--- /dev/null
+++ b/test/CodeGen/Mips/GlobalISel/mips-prelegalizer-combiner/tryCombine.mir
@@ -0,0 +1,38 @@
+# RUN: llc -O0 -mtriple=mipsel-linux-gnu -run-pass=mips-prelegalizer-combiner -verify-machineinstrs -debug %s -o - 2>&1 | FileCheck %s -check-prefixes=MIPS32
+# REQUIRES: asserts
+--- |
+
+  define void @f() {entry: ret void}
+
+...
+---
+# Check that we report attempts to combine each instruction from the input
+# since none of them gets changed in this test.
+
+# MIPS32-LABEL: Generic MI Combiner for: f
+# MIPS32: Try combining %0:_(s32) = COPY $a0
+# MIPS32: Try combining %1:_(s32) = COPY $a1
+# MIPS32: Try combining %2:_(s32) = G_ADD %1:_, %0:_
+# MIPS32: Try combining $v0 = COPY %2:_(s32)
+# MIPS32: Try combining RetRA implicit $v0
+name:            f
+alignment:       2
+tracksRegLiveness: true
+body:             |
+  bb.1.entry:
+    liveins: $a0, $a1
+
+    ; MIPS32-LABEL: name: f
+    ; MIPS32: liveins: $a0, $a1
+    ; MIPS32: [[COPY:%[0-9]+]]:_(s32) = COPY $a0
+    ; MIPS32: [[COPY1:%[0-9]+]]:_(s32) = COPY $a1
+    ; MIPS32: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[COPY1]], [[COPY]]
+    ; MIPS32: $v0 = COPY [[ADD]](s32)
+    ; MIPS32: RetRA implicit $v0
+    %0:_(s32) = COPY $a0
+    %1:_(s32) = COPY $a1
+    %2:_(s32) = G_ADD %1, %0
+    $v0 = COPY %2(s32)
+    RetRA implicit $v0
+
+...
diff --git a/test/CodeGen/Mips/GlobalISel/mips-prelegalizer-combiner/zextLoad_and_sextLoad.mir b/test/CodeGen/Mips/GlobalISel/mips-prelegalizer-combiner/zextLoad_and_sextLoad.mir
new file mode 100644
index 0000000000000000000000000000000000000000..c1c6974bf992b5c43934923fafde92cd85fdcb5f
--- /dev/null
+++ b/test/CodeGen/Mips/GlobalISel/mips-prelegalizer-combiner/zextLoad_and_sextLoad.mir
@@ -0,0 +1,188 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -O0 -mtriple=mipsel-linux-gnu -run-pass=mips-prelegalizer-combiner -verify-machineinstrs %s -o - | FileCheck %s -check-prefixes=MIPS32
+--- |
+
+  define void @load1_s8_to_zextLoad1_s32(i8* %px) {entry: ret void}
+  define void @load2_s16_to_zextLoad2_s32(i16* %px) {entry: ret void}
+  define void @load1_s8_to_zextLoad1_s16(i8* %px) {entry: ret void}
+  define void @load1_s8_to_zextLoad1_s16_to_zextLoad1_s32(i8* %px) {entry: ret void}
+  define void @load1_s8_to_sextLoad1_s32(i8* %px) {entry: ret void}
+  define void @load2_s16_to_sextLoad2_s32(i16* %px) {entry: ret void}
+  define void @load1_s8_to_sextLoad1_s16(i8* %px) {entry: ret void}
+  define void @load1_s8_to_sextLoad1_s16_to_sextLoad1_s32(i8* %px) {entry: ret void}
+
+...
+---
+name:            load1_s8_to_zextLoad1_s32
+alignment:       2
+tracksRegLiveness: true
+body:             |
+  bb.1.entry:
+    liveins: $a0
+
+    ; MIPS32-LABEL: name: load1_s8_to_zextLoad1_s32
+    ; MIPS32: liveins: $a0
+    ; MIPS32: [[COPY:%[0-9]+]]:_(p0) = COPY $a0
+    ; MIPS32: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p0) :: (load 1 from %ir.px)
+    ; MIPS32: $v0 = COPY [[ZEXTLOAD]](s32)
+    ; MIPS32: RetRA implicit $v0
+    %0:_(p0) = COPY $a0
+    %1:_(s8) = G_LOAD %0(p0) :: (load 1 from %ir.px)
+    %2:_(s32) = G_ZEXT %1(s8)
+    $v0 = COPY %2(s32)
+    RetRA implicit $v0
+
+...
+---
+name:            load2_s16_to_zextLoad2_s32
+alignment:       2
+tracksRegLiveness: true
+body:             |
+  bb.1.entry:
+    liveins: $a0
+
+    ; MIPS32-LABEL: name: load2_s16_to_zextLoad2_s32
+    ; MIPS32: liveins: $a0
+    ; MIPS32: [[COPY:%[0-9]+]]:_(p0) = COPY $a0
+    ; MIPS32: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p0) :: (load 2 from %ir.px)
+    ; MIPS32: $v0 = COPY [[ZEXTLOAD]](s32)
+    ; MIPS32: RetRA implicit $v0
+    %0:_(p0) = COPY $a0
+    %1:_(s16) = G_LOAD %0(p0) :: (load 2 from %ir.px)
+    %2:_(s32) = G_ZEXT %1(s16)
+    $v0 = COPY %2(s32)
+    RetRA implicit $v0
+
+...
+---
+name:            load1_s8_to_zextLoad1_s16
+alignment:       2
+tracksRegLiveness: true
+body:             |
+  bb.1.entry:
+    liveins: $a0
+
+    ; MIPS32-LABEL: name: load1_s8_to_zextLoad1_s16
+    ; MIPS32: liveins: $a0
+    ; MIPS32: [[COPY:%[0-9]+]]:_(p0) = COPY $a0
+    ; MIPS32: [[ZEXTLOAD:%[0-9]+]]:_(s16) = G_ZEXTLOAD [[COPY]](p0) :: (load 1 from %ir.px)
+    ; MIPS32: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[ZEXTLOAD]](s16)
+    ; MIPS32: $v0 = COPY [[ANYEXT]](s32)
+    ; MIPS32: RetRA implicit $v0
+    %0:_(p0) = COPY $a0
+    %1:_(s8) = G_LOAD %0(p0) :: (load 1 from %ir.px)
+    %2:_(s16) = G_ZEXT %1(s8)
+    %3:_(s32) = G_ANYEXT %2(s16)
+    $v0 = COPY %3(s32)
+    RetRA implicit $v0
+
+...
+---
+name:            load1_s8_to_zextLoad1_s16_to_zextLoad1_s32
+alignment:       2
+tracksRegLiveness: true
+body:             |
+  bb.1.entry:
+    liveins: $a0
+
+    ; MIPS32-LABEL: name: load1_s8_to_zextLoad1_s16_to_zextLoad1_s32
+    ; MIPS32: liveins: $a0
+    ; MIPS32: [[COPY:%[0-9]+]]:_(p0) = COPY $a0
+    ; MIPS32: [[ZEXTLOAD:%[0-9]+]]:_(s32) = G_ZEXTLOAD [[COPY]](p0) :: (load 1 from %ir.px)
+    ; MIPS32: $v0 = COPY [[ZEXTLOAD]](s32)
+    ; MIPS32: RetRA implicit $v0
+    %0:_(p0) = COPY $a0
+    %1:_(s8) = G_LOAD %0(p0) :: (load 1 from %ir.px)
+    %2:_(s16) = G_ZEXT %1(s8)
+    %3:_(s32) = G_ZEXT %2(s16)
+    $v0 = COPY %3(s32)
+    RetRA implicit $v0
+
+...
+---
+name:            load1_s8_to_sextLoad1_s32
+alignment:       2
+tracksRegLiveness: true
+body:             |
+  bb.1.entry:
+    liveins: $a0
+
+    ; MIPS32-LABEL: name: load1_s8_to_sextLoad1_s32
+    ; MIPS32: liveins: $a0
+    ; MIPS32: [[COPY:%[0-9]+]]:_(p0) = COPY $a0
+    ; MIPS32: [[SEXTLOAD:%[0-9]+]]:_(s32) = G_SEXTLOAD [[COPY]](p0) :: (load 1 from %ir.px)
+    ; MIPS32: $v0 = COPY [[SEXTLOAD]](s32)
+    ; MIPS32: RetRA implicit $v0
+    %0:_(p0) = COPY $a0
+    %1:_(s8) = G_LOAD %0(p0) :: (load 1 from %ir.px)
+    %2:_(s32) = G_SEXT %1(s8)
+    $v0 = COPY %2(s32)
+    RetRA implicit $v0
+
+...
+---
+name:            load2_s16_to_sextLoad2_s32
+alignment:       2
+tracksRegLiveness: true
+body:             |
+  bb.1.entry:
+    liveins: $a0
+
+    ; MIPS32-LABEL: name: load2_s16_to_sextLoad2_s32
+    ; MIPS32: liveins: $a0
+    ; MIPS32: [[COPY:%[0-9]+]]:_(p0) = COPY $a0
+    ; MIPS32: [[SEXTLOAD:%[0-9]+]]:_(s32) = G_SEXTLOAD [[COPY]](p0) :: (load 2 from %ir.px)
+    ; MIPS32: $v0 = COPY [[SEXTLOAD]](s32)
+    ; MIPS32: RetRA implicit $v0
+    %0:_(p0) = COPY $a0
+    %1:_(s16) = G_LOAD %0(p0) :: (load 2 from %ir.px)
+    %2:_(s32) = G_SEXT %1(s16)
+    $v0 = COPY %2(s32)
+    RetRA implicit $v0
+
+...
+---
+name:            load1_s8_to_sextLoad1_s16
+alignment:       2
+tracksRegLiveness: true
+body:             |
+  bb.1.entry:
+    liveins: $a0
+
+    ; MIPS32-LABEL: name: load1_s8_to_sextLoad1_s16
+    ; MIPS32: liveins: $a0
+    ; MIPS32: [[COPY:%[0-9]+]]:_(p0) = COPY $a0
+    ; MIPS32: [[SEXTLOAD:%[0-9]+]]:_(s16) = G_SEXTLOAD [[COPY]](p0) :: (load 1 from %ir.px)
+    ; MIPS32: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[SEXTLOAD]](s16)
+    ; MIPS32: $v0 = COPY [[ANYEXT]](s32)
+    ; MIPS32: RetRA implicit $v0
+    %0:_(p0) = COPY $a0
+    %1:_(s8) = G_LOAD %0(p0) :: (load 1 from %ir.px)
+    %2:_(s16) = G_SEXT %1(s8)
+    %3:_(s32) = G_ANYEXT %2(s16)
+    $v0 = COPY %3(s32)
+    RetRA implicit $v0
+
+...
+---
+name:            load1_s8_to_sextLoad1_s16_to_sextLoad1_s32
+alignment:       2
+tracksRegLiveness: true
+body:             |
+  bb.1.entry:
+    liveins: $a0
+
+    ; MIPS32-LABEL: name: load1_s8_to_sextLoad1_s16_to_sextLoad1_s32
+    ; MIPS32: liveins: $a0
+    ; MIPS32: [[COPY:%[0-9]+]]:_(p0) = COPY $a0
+    ; MIPS32: [[SEXTLOAD:%[0-9]+]]:_(s32) = G_SEXTLOAD [[COPY]](p0) :: (load 1 from %ir.px)
+    ; MIPS32: $v0 = COPY [[SEXTLOAD]](s32)
+    ; MIPS32: RetRA implicit $v0
+    %0:_(p0) = COPY $a0
+    %1:_(s8) = G_LOAD %0(p0) :: (load 1 from %ir.px)
+    %2:_(s16) = G_SEXT %1(s8)
+    %3:_(s32) = G_SEXT %2(s16)
+    $v0 = COPY %3(s32)
+    RetRA implicit $v0
+
+...
diff --git a/test/CodeGen/Mips/GlobalISel/regbankselect/mul.mir b/test/CodeGen/Mips/GlobalISel/regbankselect/mul.mir
new file mode 100644
index 0000000000000000000000000000000000000000..18ed66020dd94444451e37f0d7b665734d843880
--- /dev/null
+++ b/test/CodeGen/Mips/GlobalISel/regbankselect/mul.mir
@@ -0,0 +1,30 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -O0 -mtriple=mipsel-linux-gnu -run-pass=regbankselect -verify-machineinstrs %s -o - | FileCheck %s -check-prefixes=MIPS32
+--- |
+
+  define void @mul_i32(i32 %x, i32 %y) {entry: ret void}
+
+...
+---
+name:            mul_i32
+alignment:       2
+legalized:       true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $a0, $a1
+
+    ; MIPS32-LABEL: name: mul_i32
+    ; MIPS32: liveins: $a0, $a1
+    ; MIPS32: [[COPY:%[0-9]+]]:gprb(s32) = COPY $a0
+    ; MIPS32: [[COPY1:%[0-9]+]]:gprb(s32) = COPY $a1
+    ; MIPS32: [[MUL:%[0-9]+]]:gprb(s32) = G_MUL [[COPY]], [[COPY1]]
+    ; MIPS32: $v0 = COPY [[MUL]](s32)
+    ; MIPS32: RetRA implicit $v0
+    %0:_(s32) = COPY $a0
+    %1:_(s32) = COPY $a1
+    %2:_(s32) = G_MUL %0, %1
+    $v0 = COPY %2(s32)
+    RetRA implicit $v0
+
+...
diff --git a/test/CodeGen/Mips/GlobalISel/regbankselect/pointers.mir b/test/CodeGen/Mips/GlobalISel/regbankselect/pointers.mir
index 7c8dc0a194924a00bf40308be0dc55af33fe019f..52a67aea7e291566937912eed44a697731078d15 100644
--- a/test/CodeGen/Mips/GlobalISel/regbankselect/pointers.mir
+++ b/test/CodeGen/Mips/GlobalISel/regbankselect/pointers.mir
@@ -46,7 +46,7 @@ body:             |
     ; MIPS32: [[COPY2:%[0-9]+]]:gprb(s32) = COPY $a2
     ; MIPS32: [[COPY3:%[0-9]+]]:gprb(s32) = COPY $a3
     ; MIPS32: [[FRAME_INDEX:%[0-9]+]]:gprb(p0) = G_FRAME_INDEX %fixed-stack.0
-    ; MIPS32: [[LOAD:%[0-9]+]]:gprb(p0) = G_LOAD [[FRAME_INDEX]](p0) :: (load 4 from %fixed-stack.0, align 0)
+    ; MIPS32: [[LOAD:%[0-9]+]]:gprb(p0) = G_LOAD [[FRAME_INDEX]](p0) :: (load 4 from %fixed-stack.0, align 8)
     ; MIPS32: [[LOAD1:%[0-9]+]]:gprb(s32) = G_LOAD [[LOAD]](p0) :: (load 4 from %ir.p)
     ; MIPS32: $v0 = COPY [[LOAD1]](s32)
     ; MIPS32: RetRA implicit $v0
@@ -55,7 +55,7 @@ body:             |
     %2:_(s32) = COPY $a2
     %3:_(s32) = COPY $a3
     %5:_(p0) = G_FRAME_INDEX %fixed-stack.0
-    %4:_(p0) = G_LOAD %5(p0) :: (load 4 from %fixed-stack.0, align 0)
+    %4:_(p0) = G_LOAD %5(p0) :: (load 4 from %fixed-stack.0, align 8)
     %6:_(s32) = G_LOAD %4(p0) :: (load 4 from %ir.p)
     $v0 = COPY %6(s32)
     RetRA implicit $v0
diff --git a/test/CodeGen/Mips/GlobalISel/regbankselect/select.mir b/test/CodeGen/Mips/GlobalISel/regbankselect/select.mir
new file mode 100644
index 0000000000000000000000000000000000000000..98aae8a7b9d49dea183886167c9e4313fb666d7f
--- /dev/null
+++ b/test/CodeGen/Mips/GlobalISel/regbankselect/select.mir
@@ -0,0 +1,70 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -O0 -mtriple=mipsel-linux-gnu -run-pass=regbankselect -verify-machineinstrs %s -o - | FileCheck %s -check-prefixes=MIPS32
+--- |
+
+  define void @select_i32(i32, i32) {entry: ret void}
+  define void @select_ptr(i32, i32) {entry: ret void}
+
+...
+---
+name:            select_i32
+alignment:       2
+legalized:       true
+tracksRegLiveness: true
+body:             |
+  bb.1.entry:
+    liveins: $a0, $a1, $a2
+
+    ; MIPS32-LABEL: name: select_i32
+    ; MIPS32: liveins: $a0, $a1, $a2
+    ; MIPS32: [[COPY:%[0-9]+]]:gprb(s32) = COPY $a0
+    ; MIPS32: [[COPY1:%[0-9]+]]:gprb(s32) = COPY $a1
+    ; MIPS32: [[COPY2:%[0-9]+]]:gprb(s32) = COPY $a2
+    ; MIPS32: [[C:%[0-9]+]]:gprb(s32) = G_CONSTANT i32 1
+    ; MIPS32: [[COPY3:%[0-9]+]]:gprb(s32) = COPY [[COPY]](s32)
+    ; MIPS32: [[AND:%[0-9]+]]:gprb(s32) = G_AND [[COPY3]], [[C]]
+    ; MIPS32: [[SELECT:%[0-9]+]]:gprb(s32) = G_SELECT [[AND]](s32), [[COPY1]], [[COPY2]]
+    ; MIPS32: $v0 = COPY [[SELECT]](s32)
+    ; MIPS32: RetRA implicit $v0
+    %3:_(s32) = COPY $a0
+    %1:_(s32) = COPY $a1
+    %2:_(s32) = COPY $a2
+    %6:_(s32) = G_CONSTANT i32 1
+    %7:_(s32) = COPY %3(s32)
+    %5:_(s32) = G_AND %7, %6
+    %4:_(s32) = G_SELECT %5(s32), %1, %2
+    $v0 = COPY %4(s32)
+    RetRA implicit $v0
+
+...
+---
+name:            select_ptr
+alignment:       2
+legalized:       true
+tracksRegLiveness: true
+body:             |
+  bb.1.entry:
+    liveins: $a0, $a1, $a2
+
+    ; MIPS32-LABEL: name: select_ptr
+    ; MIPS32: liveins: $a0, $a1, $a2
+    ; MIPS32: [[COPY:%[0-9]+]]:gprb(s32) = COPY $a0
+    ; MIPS32: [[COPY1:%[0-9]+]]:gprb(p0) = COPY $a1
+    ; MIPS32: [[COPY2:%[0-9]+]]:gprb(p0) = COPY $a2
+    ; MIPS32: [[C:%[0-9]+]]:gprb(s32) = G_CONSTANT i32 1
+    ; MIPS32: [[COPY3:%[0-9]+]]:gprb(s32) = COPY [[COPY]](s32)
+    ; MIPS32: [[AND:%[0-9]+]]:gprb(s32) = G_AND [[COPY3]], [[C]]
+    ; MIPS32: [[SELECT:%[0-9]+]]:gprb(p0) = G_SELECT [[AND]](s32), [[COPY1]], [[COPY2]]
+    ; MIPS32: $v0 = COPY [[SELECT]](p0)
+    ; MIPS32: RetRA implicit $v0
+    %3:_(s32) = COPY $a0
+    %1:_(p0) = COPY $a1
+    %2:_(p0) = COPY $a2
+    %6:_(s32) = G_CONSTANT i32 1
+    %7:_(s32) = COPY %3(s32)
+    %5:_(s32) = G_AND %7, %6
+    %4:_(p0) = G_SELECT %5(s32), %1, %2
+    $v0 = COPY %4(p0)
+    RetRA implicit $v0
+
+...
diff --git a/test/CodeGen/Mips/GlobalISel/regbankselect/stack_args.mir b/test/CodeGen/Mips/GlobalISel/regbankselect/stack_args.mir
index c3de383ff99875494de2f97c7a3f3a76f65e8fbe..43caa5e5bd7538f146ee3b7dec85f0020ecf0409 100644
--- a/test/CodeGen/Mips/GlobalISel/regbankselect/stack_args.mir
+++ b/test/CodeGen/Mips/GlobalISel/regbankselect/stack_args.mir
@@ -24,7 +24,7 @@ body:             |
     ; MIPS32: [[COPY2:%[0-9]+]]:gprb(s32) = COPY $a2
     ; MIPS32: [[COPY3:%[0-9]+]]:gprb(s32) = COPY $a3
     ; MIPS32: [[FRAME_INDEX:%[0-9]+]]:gprb(p0) = G_FRAME_INDEX %fixed-stack.0
-    ; MIPS32: [[LOAD:%[0-9]+]]:gprb(s32) = G_LOAD [[FRAME_INDEX]](p0) :: (load 4 from %fixed-stack.0, align 0)
+    ; MIPS32: [[LOAD:%[0-9]+]]:gprb(s32) = G_LOAD [[FRAME_INDEX]](p0) :: (load 4 from %fixed-stack.0, align 8)
     ; MIPS32: ADJCALLSTACKDOWN 24, 0, implicit-def $sp, implicit $sp
     ; MIPS32: $a0 = COPY [[COPY]](s32)
     ; MIPS32: $a1 = COPY [[COPY1]](s32)
@@ -33,7 +33,7 @@ body:             |
     ; MIPS32: [[COPY4:%[0-9]+]]:gprb(p0) = COPY $sp
     ; MIPS32: [[C:%[0-9]+]]:gprb(s32) = G_CONSTANT i32 16
     ; MIPS32: [[GEP:%[0-9]+]]:gprb(p0) = G_GEP [[COPY4]], [[C]](s32)
-    ; MIPS32: G_STORE [[LOAD]](s32), [[GEP]](p0) :: (store 4 into stack + 16, align 0)
+    ; MIPS32: G_STORE [[LOAD]](s32), [[GEP]](p0) :: (store 4 into stack + 16)
     ; MIPS32: JAL @f, csr_o32, implicit-def $ra, implicit-def $sp, implicit $a0, implicit $a1, implicit $a2, implicit $a3, implicit-def $v0
     ; MIPS32: [[COPY5:%[0-9]+]]:gprb(s32) = COPY $v0
     ; MIPS32: ADJCALLSTACKUP 24, 0, implicit-def $sp, implicit $sp
@@ -44,7 +44,7 @@ body:             |
     %2:_(s32) = COPY $a2
     %3:_(s32) = COPY $a3
     %5:_(p0) = G_FRAME_INDEX %fixed-stack.0
-    %4:_(s32) = G_LOAD %5(p0) :: (load 4 from %fixed-stack.0, align 0)
+    %4:_(s32) = G_LOAD %5(p0) :: (load 4 from %fixed-stack.0, align 8)
     ADJCALLSTACKDOWN 24, 0, implicit-def $sp, implicit $sp
     $a0 = COPY %0(s32)
     $a1 = COPY %1(s32)
@@ -53,7 +53,7 @@ body:             |
     %7:_(p0) = COPY $sp
     %8:_(s32) = G_CONSTANT i32 16
     %9:_(p0) = G_GEP %7, %8(s32)
-    G_STORE %4(s32), %9(p0) :: (store 4 into stack + 16, align 0)
+    G_STORE %4(s32), %9(p0) :: (store 4 into stack + 16, align 4)
     JAL @f, csr_o32, implicit-def $ra, implicit-def $sp, implicit $a0, implicit $a1, implicit $a2, implicit $a3, implicit-def $v0
     %6:_(s32) = COPY $v0
     ADJCALLSTACKUP 24, 0, implicit-def $sp, implicit $sp
diff --git a/test/CodeGen/Mips/GlobalISel/regbankselect/sub.mir b/test/CodeGen/Mips/GlobalISel/regbankselect/sub.mir
new file mode 100644
index 0000000000000000000000000000000000000000..487024783e6050ec81f7df829e07201137318bde
--- /dev/null
+++ b/test/CodeGen/Mips/GlobalISel/regbankselect/sub.mir
@@ -0,0 +1,30 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -O0 -mtriple=mipsel-linux-gnu -run-pass=regbankselect -verify-machineinstrs %s -o - | FileCheck %s -check-prefixes=MIPS32
+--- |
+
+  define void @sub_i32(i32 %x, i32 %y) {entry: ret void}
+
+...
+---
+name:            sub_i32
+alignment:       2
+legalized:       true
+tracksRegLiveness: true
+body:             |
+  bb.0.entry:
+    liveins: $a0, $a1
+
+    ; MIPS32-LABEL: name: sub_i32
+    ; MIPS32: liveins: $a0, $a1
+    ; MIPS32: [[COPY:%[0-9]+]]:gprb(s32) = COPY $a0
+    ; MIPS32: [[COPY1:%[0-9]+]]:gprb(s32) = COPY $a1
+    ; MIPS32: [[SUB:%[0-9]+]]:gprb(s32) = G_SUB [[COPY]], [[COPY1]]
+    ; MIPS32: $v0 = COPY [[SUB]](s32)
+    ; MIPS32: RetRA implicit $v0
+    %0:_(s32) = COPY $a0
+    %1:_(s32) = COPY $a1
+    %2:_(s32) = G_SUB %0, %1
+    $v0 = COPY %2(s32)
+    RetRA implicit $v0
+
+...
diff --git a/test/CodeGen/Mips/GlobalISel/regbankselect/truncStore_and_aExtLoad.mir b/test/CodeGen/Mips/GlobalISel/regbankselect/truncStore_and_aExtLoad.mir
new file mode 100644
index 0000000000000000000000000000000000000000..9b4d903434ecd1927539eff5c418e48feda6fdea
--- /dev/null
+++ b/test/CodeGen/Mips/GlobalISel/regbankselect/truncStore_and_aExtLoad.mir
@@ -0,0 +1,82 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -O0 -mtriple=mipsel-linux-gnu -run-pass=regbankselect -verify-machineinstrs %s -o - | FileCheck %s -check-prefixes=MIPS32
+--- |
+
+  define void @load_store_i8(i8* %px, i8* %py) {entry: ret void}
+  define void @load_store_i16(i16* %px, i16* %py) {entry: ret void}
+  define void @load_store_i32(i32* %px, i32* %py) {entry: ret void}
+
+...
+---
+name:            load_store_i8
+alignment:       2
+legalized:       true
+tracksRegLiveness: true
+body:             |
+  bb.1.entry:
+    liveins: $a0, $a1
+
+    ; MIPS32-LABEL: name: load_store_i8
+    ; MIPS32: liveins: $a0, $a1
+    ; MIPS32: [[COPY:%[0-9]+]]:gprb(p0) = COPY $a0
+    ; MIPS32: [[COPY1:%[0-9]+]]:gprb(p0) = COPY $a1
+    ; MIPS32: [[LOAD:%[0-9]+]]:gprb(s32) = G_LOAD [[COPY1]](p0) :: (load 1 from %ir.py)
+    ; MIPS32: [[COPY2:%[0-9]+]]:gprb(s32) = COPY [[LOAD]](s32)
+    ; MIPS32: G_STORE [[COPY2]](s32), [[COPY]](p0) :: (store 1 into %ir.px)
+    ; MIPS32: RetRA
+    %0:_(p0) = COPY $a0
+    %1:_(p0) = COPY $a1
+    %4:_(s32) = G_LOAD %1(p0) :: (load 1 from %ir.py)
+    %3:_(s32) = COPY %4(s32)
+    G_STORE %3(s32), %0(p0) :: (store 1 into %ir.px)
+    RetRA
+
+...
+---
+name:            load_store_i16
+alignment:       2
+legalized:       true
+tracksRegLiveness: true
+body:             |
+  bb.1.entry:
+    liveins: $a0, $a1
+
+    ; MIPS32-LABEL: name: load_store_i16
+    ; MIPS32: liveins: $a0, $a1
+    ; MIPS32: [[COPY:%[0-9]+]]:gprb(p0) = COPY $a0
+    ; MIPS32: [[COPY1:%[0-9]+]]:gprb(p0) = COPY $a1
+    ; MIPS32: [[LOAD:%[0-9]+]]:gprb(s32) = G_LOAD [[COPY1]](p0) :: (load 2 from %ir.py)
+    ; MIPS32: [[COPY2:%[0-9]+]]:gprb(s32) = COPY [[LOAD]](s32)
+    ; MIPS32: G_STORE [[COPY2]](s32), [[COPY]](p0) :: (store 2 into %ir.px)
+    ; MIPS32: RetRA
+    %0:_(p0) = COPY $a0
+    %1:_(p0) = COPY $a1
+    %4:_(s32) = G_LOAD %1(p0) :: (load 2 from %ir.py)
+    %3:_(s32) = COPY %4(s32)
+    G_STORE %3(s32), %0(p0) :: (store 2 into %ir.px)
+    RetRA
+
+...
+---
+name:            load_store_i32
+alignment:       2
+legalized:       true
+tracksRegLiveness: true
+body:             |
+  bb.1.entry:
+    liveins: $a0, $a1
+
+    ; MIPS32-LABEL: name: load_store_i32
+    ; MIPS32: liveins: $a0, $a1
+    ; MIPS32: [[COPY:%[0-9]+]]:gprb(p0) = COPY $a0
+    ; MIPS32: [[COPY1:%[0-9]+]]:gprb(p0) = COPY $a1
+    ; MIPS32: [[LOAD:%[0-9]+]]:gprb(s32) = G_LOAD [[COPY1]](p0) :: (load 4 from %ir.py)
+    ; MIPS32: G_STORE [[LOAD]](s32), [[COPY]](p0) :: (store 4 into %ir.px)
+    ; MIPS32: RetRA
+    %0:_(p0) = COPY $a0
+    %1:_(p0) = COPY $a1
+    %2:_(s32) = G_LOAD %1(p0) :: (load 4 from %ir.py)
+    G_STORE %2(s32), %0(p0) :: (store 4 into %ir.px)
+    RetRA
+
+...
diff --git a/test/CodeGen/Mips/GlobalISel/regbankselect/zextLoad_and_sextLoad.mir b/test/CodeGen/Mips/GlobalISel/regbankselect/zextLoad_and_sextLoad.mir
new file mode 100644
index 0000000000000000000000000000000000000000..eb3d174c2625c76ae93c7f22e15aff761820241f
--- /dev/null
+++ b/test/CodeGen/Mips/GlobalISel/regbankselect/zextLoad_and_sextLoad.mir
@@ -0,0 +1,94 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -O0 -mtriple=mipsel-linux-gnu -run-pass=regbankselect -verify-machineinstrs %s -o - | FileCheck %s -check-prefixes=MIPS32
+--- |
+
+  define void @load1_s8_to_zextLoad1_s32(i8* %px) {entry: ret void}
+  define void @load2_s16_to_zextLoad2_s32(i16* %px) {entry: ret void}
+  define void @load1_s8_to_sextLoad1_s32(i8* %px) {entry: ret void}
+  define void @load2_s16_to_sextLoad2_s32(i16* %px) {entry: ret void}
+
+...
+---
+name:            load1_s8_to_zextLoad1_s32
+alignment:       2
+legalized:       true
+tracksRegLiveness: true
+body:             |
+  bb.1.entry:
+    liveins: $a0
+
+    ; MIPS32-LABEL: name: load1_s8_to_zextLoad1_s32
+    ; MIPS32: liveins: $a0
+    ; MIPS32: [[COPY:%[0-9]+]]:gprb(p0) = COPY $a0
+    ; MIPS32: [[ZEXTLOAD:%[0-9]+]]:gprb(s32) = G_ZEXTLOAD [[COPY]](p0) :: (load 1 from %ir.px)
+    ; MIPS32: $v0 = COPY [[ZEXTLOAD]](s32)
+    ; MIPS32: RetRA implicit $v0
+    %0:_(p0) = COPY $a0
+    %2:_(s32) = G_ZEXTLOAD %0(p0) :: (load 1 from %ir.px)
+    $v0 = COPY %2(s32)
+    RetRA implicit $v0
+
+...
+---
+name:            load2_s16_to_zextLoad2_s32
+alignment:       2
+legalized:       true
+tracksRegLiveness: true
+body:             |
+  bb.1.entry:
+    liveins: $a0
+
+    ; MIPS32-LABEL: name: load2_s16_to_zextLoad2_s32
+    ; MIPS32: liveins: $a0
+    ; MIPS32: [[COPY:%[0-9]+]]:gprb(p0) = COPY $a0
+    ; MIPS32: [[ZEXTLOAD:%[0-9]+]]:gprb(s32) = G_ZEXTLOAD [[COPY]](p0) :: (load 2 from %ir.px)
+    ; MIPS32: $v0 = COPY [[ZEXTLOAD]](s32)
+    ; MIPS32: RetRA implicit $v0
+    %0:_(p0) = COPY $a0
+    %2:_(s32) = G_ZEXTLOAD %0(p0) :: (load 2 from %ir.px)
+    $v0 = COPY %2(s32)
+    RetRA implicit $v0
+
+...
+---
+name:            load1_s8_to_sextLoad1_s32
+alignment:       2
+legalized:       true
+tracksRegLiveness: true
+body:             |
+  bb.1.entry:
+    liveins: $a0
+
+    ; MIPS32-LABEL: name: load1_s8_to_sextLoad1_s32
+    ; MIPS32: liveins: $a0
+    ; MIPS32: [[COPY:%[0-9]+]]:gprb(p0) = COPY $a0
+    ; MIPS32: [[SEXTLOAD:%[0-9]+]]:gprb(s32) = G_SEXTLOAD [[COPY]](p0) :: (load 1 from %ir.px)
+    ; MIPS32: $v0 = COPY [[SEXTLOAD]](s32)
+    ; MIPS32: RetRA implicit $v0
+    %0:_(p0) = COPY $a0
+    %2:_(s32) = G_SEXTLOAD %0(p0) :: (load 1 from %ir.px)
+    $v0 = COPY %2(s32)
+    RetRA implicit $v0
+
+...
+---
+name:            load2_s16_to_sextLoad2_s32
+alignment:       2
+legalized:       true
+tracksRegLiveness: true
+body:             |
+  bb.1.entry:
+    liveins: $a0
+
+    ; MIPS32-LABEL: name: load2_s16_to_sextLoad2_s32
+    ; MIPS32: liveins: $a0
+    ; MIPS32: [[COPY:%[0-9]+]]:gprb(p0) = COPY $a0
+    ; MIPS32: [[SEXTLOAD:%[0-9]+]]:gprb(s32) = G_SEXTLOAD [[COPY]](p0) :: (load 2 from %ir.px)
+    ; MIPS32: $v0 = COPY [[SEXTLOAD]](s32)
+    ; MIPS32: RetRA implicit $v0
+    %0:_(p0) = COPY $a0
+    %2:_(s32) = G_SEXTLOAD %0(p0) :: (load 2 from %ir.px)
+    $v0 = COPY %2(s32)
+    RetRA implicit $v0
+
+...
diff --git a/test/CodeGen/Mips/cconv/vector.ll b/test/CodeGen/Mips/cconv/vector.ll
index 6a07c4f34564deb54d111dd191fb0a2091ecdddf..5c7c3f424c380fa381a8e0f99af0b5ee1b0c7ea1 100644
--- a/test/CodeGen/Mips/cconv/vector.ll
+++ b/test/CodeGen/Mips/cconv/vector.ll
@@ -1,12 +1,12 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=mips-unknown-linux-gnu -mcpu=mips32 -disable-mips-delay-filler | FileCheck %s --check-prefixes=ALL,MIPS32,MIPS32EB
-; RUN: llc < %s -mtriple=mips64-unknown-linux-gnu -relocation-model=pic -mcpu=mips64 -disable-mips-delay-filler | FileCheck %s --check-prefixes=ALL,MIPS64,MIPS64EB
+; RUN: llc < %s -mtriple=mips64-unknown-linux-gnu -relocation-model=pic -mcpu=mips64 -disable-mips-delay-filler -mips-jalr-reloc=false | FileCheck %s --check-prefixes=ALL,MIPS64,MIPS64EB
 ; RUN: llc < %s -mtriple=mips-unknown-linux-gnu -mcpu=mips32r5 -mattr=+fp64,+msa -disable-mips-delay-filler | FileCheck %s --check-prefixes=ALL,MIPS32R5,MIPS32R5EB
-; RUN: llc < %s -mtriple=mips64-unknown-linux-gnu -relocation-model=pic -mcpu=mips64r5 -mattr=+fp64,+msa -disable-mips-delay-filler | FileCheck %s --check-prefixes=ALL,MIPS64R5,MIPS64R5EB
+; RUN: llc < %s -mtriple=mips64-unknown-linux-gnu -relocation-model=pic -mcpu=mips64r5 -mattr=+fp64,+msa -disable-mips-delay-filler -mips-jalr-reloc=false | FileCheck %s --check-prefixes=ALL,MIPS64R5,MIPS64R5EB
 ; RUN: llc < %s -mtriple=mipsel-unknown-linux-gnu -mcpu=mips32 -disable-mips-delay-filler | FileCheck %s --check-prefixes=ALL,MIPS32,MIPS32EL
-; RUN: llc < %s -mtriple=mips64el-unknown-linux-gnu -relocation-model=pic -mcpu=mips64 -disable-mips-delay-filler | FileCheck %s --check-prefixes=ALL,MIPS64,MIPS64EL
+; RUN: llc < %s -mtriple=mips64el-unknown-linux-gnu -relocation-model=pic -mcpu=mips64 -disable-mips-delay-filler -mips-jalr-reloc=false | FileCheck %s --check-prefixes=ALL,MIPS64,MIPS64EL
 ; RUN: llc < %s -mtriple=mipsel-unknown-linux-gnu -mcpu=mips32r5 -mattr=+fp64,+msa -disable-mips-delay-filler | FileCheck %s --check-prefixes=ALL,MIPS32R5,MIPS32R5EL
-; RUN: llc < %s -mtriple=mips64el-unknown-linux-gnu -relocation-model=pic -mcpu=mips64r5 -mattr=+fp64,+msa -disable-mips-delay-filler | FileCheck %s --check-prefixes=ALL,MIPS64R5,MIPS64R5EL
+; RUN: llc < %s -mtriple=mips64el-unknown-linux-gnu -relocation-model=pic -mcpu=mips64r5 -mattr=+fp64,+msa -disable-mips-delay-filler -mips-jalr-reloc=false | FileCheck %s --check-prefixes=ALL,MIPS64R5,MIPS64R5EL
 
 ; Test that vector types are passed through the integer register set whether or
 ; not MSA is enabled. This is a ABI requirement for MIPS. For GCC compatibility
diff --git a/test/CodeGen/Mips/fabs.ll b/test/CodeGen/Mips/fabs.ll
index ce1a9a60e7c23e8503cdf0b60b305facfcd2684a..75aa7d8295f94edb7a6f57734f81bd79f23533b9 100644
--- a/test/CodeGen/Mips/fabs.ll
+++ b/test/CodeGen/Mips/fabs.ll
@@ -1,23 +1,84 @@
-; Check that abs.[ds] is selected and does not depend on -enable-no-nans-fp-math
-; They obey the Has2008 and ABS2008 configuration bits which govern the
-; conformance to IEEE 754 (1985) and IEEE 754 (2008). When these bits are not
-; present, they confirm to 1985.
+; Check that abs.[ds] is only selected for mips32r6 or mips64r6 when no
+; additional options are passed. For revisions prior mips32r6 and mips64r6,
+; abs.[ds] does not generate the correct result when working with NaNs, and
+; should be explicitly enabled with -enable-no-nans-fp-math or +abs2008 options.
+
 ; In 1985 mode, abs.[ds] are arithmetic (i.e. they raise invalid operation
 ; exceptions when given NaN's). In 2008 mode, they are non-arithmetic (i.e.
 ; they are copies and don't raise any exceptions).
 
-; RUN: llc  < %s -mtriple=mipsel-linux-gnu -mcpu=mips32 | FileCheck %s
-; RUN: llc  < %s -mtriple=mipsel-linux-gnu -mcpu=mips32r2 | FileCheck %s
-; RUN: llc  < %s -mtriple=mipsel-linux-gnu -mcpu=mips32 -enable-no-nans-fp-math | FileCheck %s
+; Testing default values
+; RUN: llc  < %s -mtriple=mipsel-linux-gnu -mcpu=mips32 | FileCheck %s \
+; RUN:    -check-prefix=CHECK-ABSLEGACY
+; RUN: llc  < %s -mtriple=mips64el-linux-gnu -mcpu=mips64 | FileCheck %s \
+; RUN:    -check-prefix=CHECK-ABSLEGACY
+; RUN: llc  < %s -mtriple=mipsel-linux-gnu -mcpu=mips32r2 | FileCheck %s \
+; RUN:    -check-prefix=CHECK-ABSLEGACY
+; RUN: llc  < %s -mtriple=mips64el-linux-gnu -mcpu=mips64r2 | FileCheck %s \
+; RUN:    -check-prefix=CHECK-ABSLEGACY
+; RUN: llc  < %s -mtriple=mipsel-linux-gnu -mcpu=mips32r6 | FileCheck %s \
+; RUN:    -check-prefix=CHECK-ABS2008
+; RUN: llc  < %s -mtriple=mips64el-linux-gnu -mcpu=mips64r6 | FileCheck %s \
+; RUN:    -check-prefix=CHECK-ABS2008
+; RUN: llc  < %s -mtriple=mips-linux-gnu -mcpu=mips32 | FileCheck %s \
+; RUN:    -check-prefix=CHECK-ABSLEGACY
+; RUN: llc  < %s -mtriple=mips64-linux-gnu -mcpu=mips64 | FileCheck %s \
+; RUN:    -check-prefix=CHECK-ABSLEGACY
+; RUN: llc  < %s -mtriple=mips-linux-gnu -mcpu=mips32r2 | FileCheck %s \
+; RUN:    -check-prefix=CHECK-ABSLEGACY
+; RUN: llc  < %s -mtriple=mips64-linux-gnu -mcpu=mips64r2 | FileCheck %s \
+; RUN:    -check-prefix=CHECK-ABSLEGACY
+; RUN: llc  < %s -mtriple=mips-linux-gnu -mcpu=mips32r6 | FileCheck %s \
+; RUN:    -check-prefix=CHECK-ABS2008
+; RUN: llc  < %s -mtriple=mips64-linux-gnu -mcpu=mips64r6 | FileCheck %s \
+; RUN:    -check-prefix=CHECK-ABS2008
+; Testing non-default values
+; RUN: llc  < %s -mtriple=mipsel-linux-gnu -mcpu=mips32r2 -mattr=+abs2008 \
+; RUN:    | FileCheck %s -check-prefix=CHECK-ABS2008
+; RUN: llc  < %s -mtriple=mips64el-linux-gnu -mcpu=mips64r2 -mattr=+abs2008 \
+; RUN:    | FileCheck %s -check-prefix=CHECK-ABS2008
+; RUN: llc  < %s -mtriple=mips-linux-gnu -mcpu=mips32r2 -mattr=+abs2008 \
+; RUN:    | FileCheck %s -check-prefix=CHECK-ABS2008
+; RUN: llc  < %s -mtriple=mips64-linux-gnu -mcpu=mips64r2 -mattr=+abs2008 \
+; RUN:    | FileCheck %s -check-prefix=CHECK-ABS2008
+; Testing -enable-no-nans-fp-math
+; RUN: llc  < %s -mtriple=mipsel-linux-gnu -mcpu=mips32 \
+; RUN:    -enable-no-nans-fp-math | FileCheck %s -check-prefix=CHECK-ABS2008
+; RUN: llc  < %s -mtriple=mips64el-linux-gnu -mcpu=mips64 \
+; RUN:    -enable-no-nans-fp-math | FileCheck %s -check-prefix=CHECK-ABS2008
+; RUN: llc  < %s -mtriple=mips-linux-gnu -mcpu=mips32 \
+; RUN:    -enable-no-nans-fp-math | FileCheck %s -check-prefix=CHECK-ABS2008
+; RUN: llc  < %s -mtriple=mips64-linux-gnu -mcpu=mips64 \
+; RUN:    -enable-no-nans-fp-math | FileCheck %s -check-prefix=CHECK-ABS2008
 
-; RUN: llc  < %s -mtriple=mips64el-linux-gnu -mcpu=mips64 | FileCheck %s
-; RUN: llc  < %s -mtriple=mips64el-linux-gnu -mcpu=mips64 -enable-no-nans-fp-math | FileCheck %s
+; microMIPS
+; Testing default values
+; RUN: llc  < %s -mtriple=mipsel-linux-gnu -mcpu=mips32 -mattr=+micromips \
+; RUN:    | FileCheck %s -check-prefix=CHECK-ABSLEGACY
+; RUN: llc  < %s -mtriple=mipsel-linux-gnu -mcpu=mips32r2 -mattr=+micromips \
+; RUN:    | FileCheck %s -check-prefix=CHECK-ABSLEGACY
+; RUN: llc  < %s -mtriple=mips-linux-gnu -mcpu=mips32 -mattr=+micromips \
+; RUN:    | FileCheck %s -check-prefix=CHECK-ABSLEGACY
+; RUN: llc  < %s -mtriple=mips-linux-gnu -mcpu=mips32r2 -mattr=+micromips \
+; RUN:    | FileCheck %s -check-prefix=CHECK-ABSLEGACY
+; Testing non-default values
+; RUN: llc  < %s -mtriple=mipsel-linux-gnu -mcpu=mips32r2 \
+; RUN:    -mattr=+abs2008,+micromips | FileCheck %s -check-prefix=CHECK-ABS2008
+; RUN: llc  < %s -mtriple=mips-linux-gnu -mcpu=mips32r2 \
+; RUN:    -mattr=+abs2008,+micromips | FileCheck %s -check-prefix=CHECK-ABS2008
+; Testing -enable-no-nans-fp-math
+; RUN: llc  < %s -mtriple=mipsel-linux-gnu -mcpu=mips32 -mattr=+micromips \
+; RUN:    -enable-no-nans-fp-math | FileCheck %s -check-prefix=CHECK-ABS2008
+; RUN: llc  < %s -mtriple=mips-linux-gnu -mcpu=mips32 -mattr=+micromips \
+; RUN:    -enable-no-nans-fp-math | FileCheck %s -check-prefix=CHECK-ABS2008
 
 define float @foo0(float %a) nounwind readnone {
 entry:
 
 ; CHECK-LABEL: foo0
-; CHECK: abs.s
+; CHECK-ABS2008: abs.s
+; CHECK-ABSLEGACY: {{(ori|ins)}}
+; CHECK-ABSLEGACY-NOT: abs.s
 
   %call = tail call float @fabsf(float %a) nounwind readnone
   ret float %call
@@ -29,7 +90,9 @@ define double @foo1(double %a) nounwind readnone {
 entry:
 
 ; CHECK-LABEL: foo1:
-; CHECK: abs.d
+; CHECK-ABS2008: abs.d
+; CHECK-ABSLEGACY: {{(ori|ins|dsll)}}
+; CHECK-ABSLEGACY-NOT: abs.d
 
   %call = tail call double @fabs(double %a) nounwind readnone
   ret double %call
diff --git a/test/CodeGen/Mips/gprestore.ll b/test/CodeGen/Mips/gprestore.ll
index 88ac047b66095401c7f36dd66369df1d967309bf..a1e696b0ac08b801d5b9d1aecfef20a8a57e230a 100644
--- a/test/CodeGen/Mips/gprestore.ll
+++ b/test/CodeGen/Mips/gprestore.ll
@@ -1,10 +1,10 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=mips-mti-linux-gnu < %s -relocation-model=pic | FileCheck %s --check-prefix=O32
-; RUN: llc -mtriple=mips64-mti-linux-gnu < %s -relocation-model=pic | FileCheck %s --check-prefix=N64
-; RUN: llc -mtriple=mips64-mti-linux-gnu < %s -relocation-model=pic -target-abi n32 | FileCheck %s --check-prefix=N32
-; RUN: llc -mtriple=mips-mti-linux-gnu < %s -relocation-model=pic -O3 | FileCheck %s --check-prefix=O3O32
-; RUN: llc -mtriple=mips64-mti-linux-gnu < %s -relocation-model=pic -O3 | FileCheck %s --check-prefix=O3N64
-; RUN: llc -mtriple=mips64-mti-linux-gnu < %s -relocation-model=pic -target-abi n32 -O3 | FileCheck %s --check-prefix=O3N32
+; RUN: llc -mtriple=mips-mti-linux-gnu < %s -relocation-model=pic -mips-jalr-reloc=false | FileCheck %s --check-prefix=O32
+; RUN: llc -mtriple=mips64-mti-linux-gnu < %s -relocation-model=pic -mips-jalr-reloc=false | FileCheck %s --check-prefix=N64
+; RUN: llc -mtriple=mips64-mti-linux-gnu < %s -relocation-model=pic -target-abi n32 -mips-jalr-reloc=false | FileCheck %s --check-prefix=N32
+; RUN: llc -mtriple=mips-mti-linux-gnu < %s -relocation-model=pic -O3 -mips-jalr-reloc=false | FileCheck %s --check-prefix=O3O32
+; RUN: llc -mtriple=mips64-mti-linux-gnu < %s -relocation-model=pic -O3 -mips-jalr-reloc=false | FileCheck %s --check-prefix=O3N64
+; RUN: llc -mtriple=mips64-mti-linux-gnu < %s -relocation-model=pic -target-abi n32 -O3 -mips-jalr-reloc=false | FileCheck %s --check-prefix=O3N32
 
 ; Test that PIC calls use the $25 register. This is an ABI requirement.
 
diff --git a/test/CodeGen/Mips/llvm-ir/abs.ll b/test/CodeGen/Mips/llvm-ir/abs.ll
index 3ae8525ec18d050dcf70666ce66256e894e69b63..c0812977e3a11b2fe6fae7b9581ff4ff2deb15a2 100644
--- a/test/CodeGen/Mips/llvm-ir/abs.ll
+++ b/test/CodeGen/Mips/llvm-ir/abs.ll
@@ -1,11 +1,12 @@
-; RUN: llc -march=mips -mcpu=mips32                           -asm-show-inst < %s | FileCheck %s --check-prefix=MIPS32
-; RUN: llc -march=mips -mcpu=mips32r2 -mattr=+fp64            -asm-show-inst < %s | FileCheck %s --check-prefix=MIPS32FP64
-; RUN: llc -march=mips -mcpu=mips32r3 -mattr=+micromips       -asm-show-inst < %s | FileCheck %s --check-prefix=MM
-; RUN: llc -march=mips -mcpu=mips32r3 -mattr=+micromips,+fp64 -asm-show-inst < %s | FileCheck %s --check-prefix=MMFP64
-; RUN: llc -march=mips -mcpu=mips32r6 -mattr=+micromips       -asm-show-inst < %s | FileCheck %s --check-prefix=MMR6
+; RUN: llc -march=mips -mcpu=mips32                                    -asm-show-inst < %s | FileCheck %s --check-prefix=MIPS32
+; RUN: llc -march=mips -mcpu=mips32r2 -mattr=+abs2008,+fp64            -asm-show-inst < %s | FileCheck %s --check-prefix=MIPS32FP64
+; RUN: llc -march=mips -mcpu=mips32r3 -mattr=+abs2008,+micromips       -asm-show-inst < %s | FileCheck %s --check-prefix=MM
+; RUN: llc -march=mips -mcpu=mips32r3 -mattr=+abs2008,+micromips,+fp64 -asm-show-inst < %s | FileCheck %s --check-prefix=MMFP64
+; RUN: llc -march=mips -mcpu=mips32r6 -mattr=+micromips                -asm-show-inst < %s | FileCheck %s --check-prefix=MMR6
 
 define float @abs_s(float %a) {
-; MIPS32:     abs.s  {{.*}}               # <MCInst #{{[0-9]+}} FABS_S
+; MIPS32: {{(ori|ins)}}
+; MIPS32-NOT: abs.s
 ; MIPS32FP64: abs.s  {{.*}}               # <MCInst #{{[0-9]+}} FABS_S
 ; MM:         abs.s  {{.*}}               # <MCInst #{{[0-9]+}} FABS_S_MM
 ; MMFP64:     abs.s  {{.*}}               # <MCInst #{{[0-9]+}} FABS_S_MM
@@ -15,7 +16,8 @@ define float @abs_s(float %a) {
 }
 
 define double @abs_d(double %a) {
-; MIPS32:     abs.d  {{.*}}               # <MCInst #{{[0-9]+}} FABS_D32
+; MIPS32: {{(ori|ins|dsll)}}
+; MIPS32-NOT: abs.d
 ; MIPS32FP64: abs.d  {{.*}}               # <MCInst #{{[0-9]+}} FABS_D64
 ; MM:         abs.d  {{.*}}               # <MCInst #{{[0-9]+}} FABS_D32_MM
 ; MMFP64:     abs.d  {{.*}}               # <MCInst #{{[0-9]+}} FABS_D64_MM
diff --git a/test/CodeGen/Mips/llvm-ir/ashr.ll b/test/CodeGen/Mips/llvm-ir/ashr.ll
index dc19c9e81a9b0f349b0dc7eac88d7a713915b045..3aa384e53084d3515b711bb6cc4d47742e918dc7 100644
--- a/test/CodeGen/Mips/llvm-ir/ashr.ll
+++ b/test/CodeGen/Mips/llvm-ir/ashr.ll
@@ -274,26 +274,21 @@ entry:
 
 define signext i64 @ashr_i64(i64 signext %a, i64 signext %b) {
 ; MIPS-LABEL: ashr_i64:
-; MIPS:       # %bb.0: # %entry
-; MIPS-NEXT:    srav $2, $4, $7
-; MIPS-NEXT:    andi $6, $7, 32
-; MIPS-NEXT:    beqz $6, $BB4_3
-; MIPS-NEXT:    move $3, $2
-; MIPS-NEXT:  # %bb.1: # %entry
-; MIPS-NEXT:    bnez $6, $BB4_4
-; MIPS-NEXT:    nop
-; MIPS-NEXT:  $BB4_2: # %entry
-; MIPS-NEXT:    jr $ra
-; MIPS-NEXT:    nop
-; MIPS-NEXT:  $BB4_3: # %entry
-; MIPS-NEXT:    srlv $1, $5, $7
-; MIPS-NEXT:    not $3, $7
-; MIPS-NEXT:    sll $5, $4, 1
-; MIPS-NEXT:    sllv $3, $5, $3
-; MIPS-NEXT:    beqz $6, $BB4_2
-; MIPS-NEXT:    or $3, $3, $1
-; MIPS-NEXT:  $BB4_4:
-; MIPS-NEXT:    jr $ra
+; MIPS:       # %bb.0:
+; MIPS-NEXT:    andi  $1, $7, 32
+; MIPS-NEXT:    bnez  $1, $BB4_2
+; MIPS-NEXT:    srav  $3, $4, $7
+; MIPS-NEXT:  # %bb.1:
+; MIPS-NEXT:    srlv  $1, $5, $7
+; MIPS-NEXT:    not $2, $7
+; MIPS-NEXT:    sll $4, $4, 1
+; MIPS-NEXT:    sllv  $2, $4, $2
+; MIPS-NEXT:    or  $1, $2, $1
+; MIPS-NEXT:    move  $2, $3
+; MIPS-NEXT:    jr  $ra
+; MIPS-NEXT:    move  $3, $1
+; MIPS-NEXT:  $BB4_2:
+; MIPS-NEXT:    jr  $ra
 ; MIPS-NEXT:    sra $2, $4, 31
 ;
 ; MIPS32-LABEL: ashr_i64:
@@ -400,133 +395,114 @@ entry:
 
 define signext i128 @ashr_i128(i128 signext %a, i128 signext %b) {
 ; MIPS-LABEL: ashr_i128:
-; MIPS:       # %bb.0: # %entry
-; MIPS-NEXT:    addiu $sp, $sp, -8
-; MIPS-NEXT:    .cfi_def_cfa_offset 8
-; MIPS-NEXT:    sw $17, 4($sp) # 4-byte Folded Spill
-; MIPS-NEXT:    sw $16, 0($sp) # 4-byte Folded Spill
-; MIPS-NEXT:    .cfi_offset 17, -4
-; MIPS-NEXT:    .cfi_offset 16, -8
-; MIPS-NEXT:    lw $25, 36($sp)
+; MIPS:       # %bb.0:
+; MIPS-NEXT:    lw  $2, 28($sp)
 ; MIPS-NEXT:    addiu $1, $zero, 64
-; MIPS-NEXT:    subu $11, $1, $25
-; MIPS-NEXT:    sllv $9, $5, $11
-; MIPS-NEXT:    andi $13, $11, 32
-; MIPS-NEXT:    addiu $2, $zero, 0
-; MIPS-NEXT:    bnez $13, $BB5_2
-; MIPS-NEXT:    addiu $3, $zero, 0
-; MIPS-NEXT:  # %bb.1: # %entry
-; MIPS-NEXT:    move $3, $9
-; MIPS-NEXT:  $BB5_2: # %entry
-; MIPS-NEXT:    not $gp, $25
-; MIPS-NEXT:    srlv $12, $6, $25
-; MIPS-NEXT:    andi $8, $25, 32
-; MIPS-NEXT:    bnez $8, $BB5_4
-; MIPS-NEXT:    move $15, $12
-; MIPS-NEXT:  # %bb.3: # %entry
-; MIPS-NEXT:    srlv $1, $7, $25
-; MIPS-NEXT:    sll $10, $6, 1
-; MIPS-NEXT:    sllv $10, $10, $gp
-; MIPS-NEXT:    or $15, $10, $1
-; MIPS-NEXT:  $BB5_4: # %entry
-; MIPS-NEXT:    addiu $10, $25, -64
-; MIPS-NEXT:    sll $17, $4, 1
-; MIPS-NEXT:    srav $14, $4, $10
-; MIPS-NEXT:    andi $24, $10, 32
-; MIPS-NEXT:    bnez $24, $BB5_6
-; MIPS-NEXT:    move $16, $14
-; MIPS-NEXT:  # %bb.5: # %entry
-; MIPS-NEXT:    srlv $1, $5, $10
-; MIPS-NEXT:    not $10, $10
-; MIPS-NEXT:    sllv $10, $17, $10
-; MIPS-NEXT:    or $16, $10, $1
-; MIPS-NEXT:  $BB5_6: # %entry
-; MIPS-NEXT:    sltiu $10, $25, 64
-; MIPS-NEXT:    beqz $10, $BB5_8
+; MIPS-NEXT:    subu  $9, $1, $2
+; MIPS-NEXT:    sllv  $10, $5, $9
+; MIPS-NEXT:    andi  $13, $9, 32
+; MIPS-NEXT:    andi  $3, $2, 32
+; MIPS-NEXT:    addiu $11, $zero, 0
+; MIPS-NEXT:    bnez  $13, $BB5_2
+; MIPS-NEXT:    addiu $12, $zero, 0
+; MIPS-NEXT:  # %bb.1:
+; MIPS-NEXT:    move  $12, $10
+; MIPS-NEXT:  $BB5_2:
+; MIPS-NEXT:    not $8, $2
+; MIPS-NEXT:    bnez  $3, $BB5_5
+; MIPS-NEXT:    srlv  $14, $6, $2
+; MIPS-NEXT:  # %bb.3:
+; MIPS-NEXT:    sll $1, $6, 1
+; MIPS-NEXT:    srlv  $11, $7, $2
+; MIPS-NEXT:    sllv  $1, $1, $8
+; MIPS-NEXT:    or  $15, $1, $11
+; MIPS-NEXT:    bnez  $13, $BB5_7
+; MIPS-NEXT:    move  $11, $14
+; MIPS-NEXT:  # %bb.4:
+; MIPS-NEXT:    b $BB5_6
 ; MIPS-NEXT:    nop
-; MIPS-NEXT:  # %bb.7:
-; MIPS-NEXT:    or $16, $15, $3
-; MIPS-NEXT:  $BB5_8: # %entry
-; MIPS-NEXT:    srav $15, $4, $25
-; MIPS-NEXT:    beqz $8, $BB5_20
-; MIPS-NEXT:    move $3, $15
-; MIPS-NEXT:  # %bb.9: # %entry
-; MIPS-NEXT:    sltiu $gp, $25, 1
-; MIPS-NEXT:    beqz $gp, $BB5_21
+; MIPS-NEXT:  $BB5_5:
+; MIPS-NEXT:    bnez  $13, $BB5_7
+; MIPS-NEXT:    move  $15, $14
+; MIPS-NEXT:  $BB5_6:
+; MIPS-NEXT:    sllv  $1, $4, $9
+; MIPS-NEXT:    not $9, $9
+; MIPS-NEXT:    srl $10, $5, 1
+; MIPS-NEXT:    srlv  $9, $10, $9
+; MIPS-NEXT:    or  $10, $1, $9
+; MIPS-NEXT:  $BB5_7:
+; MIPS-NEXT:    addiu $24, $2, -64
+; MIPS-NEXT:    sll $13, $4, 1
+; MIPS-NEXT:    srav  $14, $4, $24
+; MIPS-NEXT:    andi  $1, $24, 32
+; MIPS-NEXT:    bnez  $1, $BB5_10
+; MIPS-NEXT:    sra $9, $4, 31
+; MIPS-NEXT:  # %bb.8:
+; MIPS-NEXT:    srlv  $1, $5, $24
+; MIPS-NEXT:    not $24, $24
+; MIPS-NEXT:    sllv  $24, $13, $24
+; MIPS-NEXT:    or  $25, $24, $1
+; MIPS-NEXT:    move  $24, $14
+; MIPS-NEXT:    sltiu $14, $2, 64
+; MIPS-NEXT:    beqz  $14, $BB5_12
 ; MIPS-NEXT:    nop
-; MIPS-NEXT:  $BB5_10: # %entry
-; MIPS-NEXT:    beqz $10, $BB5_22
-; MIPS-NEXT:    sra $25, $4, 31
-; MIPS-NEXT:  $BB5_11: # %entry
-; MIPS-NEXT:    beqz $13, $BB5_23
+; MIPS-NEXT:  # %bb.9:
+; MIPS-NEXT:    b $BB5_11
 ; MIPS-NEXT:    nop
-; MIPS-NEXT:  $BB5_12: # %entry
-; MIPS-NEXT:    beqz $8, $BB5_24
+; MIPS-NEXT:  $BB5_10:
+; MIPS-NEXT:    move  $25, $14
+; MIPS-NEXT:    sltiu $14, $2, 64
+; MIPS-NEXT:    beqz  $14, $BB5_12
+; MIPS-NEXT:    move  $24, $9
+; MIPS-NEXT:  $BB5_11:
+; MIPS-NEXT:    or  $25, $15, $12
+; MIPS-NEXT:  $BB5_12:
+; MIPS-NEXT:    sltiu $12, $2, 1
+; MIPS-NEXT:    beqz  $12, $BB5_18
 ; MIPS-NEXT:    nop
-; MIPS-NEXT:  $BB5_13: # %entry
-; MIPS-NEXT:    beqz $24, $BB5_25
-; MIPS-NEXT:    move $4, $25
-; MIPS-NEXT:  $BB5_14: # %entry
-; MIPS-NEXT:    bnez $10, $BB5_26
+; MIPS-NEXT:  # %bb.13:
+; MIPS-NEXT:    bnez  $14, $BB5_19
 ; MIPS-NEXT:    nop
-; MIPS-NEXT:  $BB5_15: # %entry
-; MIPS-NEXT:    beqz $gp, $BB5_27
+; MIPS-NEXT:  $BB5_14:
+; MIPS-NEXT:    beqz  $12, $BB5_20
 ; MIPS-NEXT:    nop
-; MIPS-NEXT:  $BB5_16: # %entry
-; MIPS-NEXT:    beqz $8, $BB5_28
-; MIPS-NEXT:    move $2, $25
-; MIPS-NEXT:  $BB5_17: # %entry
-; MIPS-NEXT:    bnez $10, $BB5_19
+; MIPS-NEXT:  $BB5_15:
+; MIPS-NEXT:    bnez  $3, $BB5_21
+; MIPS-NEXT:    srav  $4, $4, $2
+; MIPS-NEXT:  $BB5_16:
+; MIPS-NEXT:    srlv  $1, $5, $2
+; MIPS-NEXT:    sllv  $2, $13, $8
+; MIPS-NEXT:    or  $3, $2, $1
+; MIPS-NEXT:    bnez  $14, $BB5_23
+; MIPS-NEXT:    move  $2, $4
+; MIPS-NEXT:  # %bb.17:
+; MIPS-NEXT:    b $BB5_22
 ; MIPS-NEXT:    nop
-; MIPS-NEXT:  $BB5_18: # %entry
-; MIPS-NEXT:    move $2, $25
-; MIPS-NEXT:  $BB5_19: # %entry
-; MIPS-NEXT:    move $4, $6
-; MIPS-NEXT:    move $5, $7
-; MIPS-NEXT:    lw $16, 0($sp) # 4-byte Folded Reload
-; MIPS-NEXT:    lw $17, 4($sp) # 4-byte Folded Reload
-; MIPS-NEXT:    jr $ra
-; MIPS-NEXT:    addiu $sp, $sp, 8
-; MIPS-NEXT:  $BB5_20: # %entry
-; MIPS-NEXT:    srlv $1, $5, $25
-; MIPS-NEXT:    sllv $3, $17, $gp
-; MIPS-NEXT:    sltiu $gp, $25, 1
-; MIPS-NEXT:    bnez $gp, $BB5_10
-; MIPS-NEXT:    or $3, $3, $1
-; MIPS-NEXT:  $BB5_21: # %entry
-; MIPS-NEXT:    move $7, $16
-; MIPS-NEXT:    bnez $10, $BB5_11
-; MIPS-NEXT:    sra $25, $4, 31
-; MIPS-NEXT:  $BB5_22: # %entry
-; MIPS-NEXT:    bnez $13, $BB5_12
-; MIPS-NEXT:    move $3, $25
-; MIPS-NEXT:  $BB5_23: # %entry
-; MIPS-NEXT:    not $1, $11
-; MIPS-NEXT:    srl $5, $5, 1
-; MIPS-NEXT:    sllv $4, $4, $11
-; MIPS-NEXT:    srlv $1, $5, $1
-; MIPS-NEXT:    bnez $8, $BB5_13
-; MIPS-NEXT:    or $9, $4, $1
-; MIPS-NEXT:  $BB5_24: # %entry
-; MIPS-NEXT:    move $2, $12
-; MIPS-NEXT:    bnez $24, $BB5_14
-; MIPS-NEXT:    move $4, $25
-; MIPS-NEXT:  $BB5_25: # %entry
-; MIPS-NEXT:    beqz $10, $BB5_15
-; MIPS-NEXT:    move $4, $14
-; MIPS-NEXT:  $BB5_26:
-; MIPS-NEXT:    bnez $gp, $BB5_16
-; MIPS-NEXT:    or $4, $2, $9
-; MIPS-NEXT:  $BB5_27: # %entry
-; MIPS-NEXT:    move $6, $4
-; MIPS-NEXT:    bnez $8, $BB5_17
-; MIPS-NEXT:    move $2, $25
-; MIPS-NEXT:  $BB5_28: # %entry
-; MIPS-NEXT:    bnez $10, $BB5_19
-; MIPS-NEXT:    move $2, $15
-; MIPS-NEXT:  # %bb.29: # %entry
-; MIPS-NEXT:    b $BB5_18
+; MIPS-NEXT:  $BB5_18:
+; MIPS-NEXT:    beqz  $14, $BB5_14
+; MIPS-NEXT:    move  $7, $25
+; MIPS-NEXT:  $BB5_19:
+; MIPS-NEXT:    bnez  $12, $BB5_15
+; MIPS-NEXT:    or  $24, $11, $10
+; MIPS-NEXT:  $BB5_20:
+; MIPS-NEXT:    move  $6, $24
+; MIPS-NEXT:    beqz  $3, $BB5_16
+; MIPS-NEXT:    srav  $4, $4, $2
+; MIPS-NEXT:  $BB5_21:
+; MIPS-NEXT:    move  $2, $9
+; MIPS-NEXT:    bnez  $14, $BB5_23
+; MIPS-NEXT:    move  $3, $4
+; MIPS-NEXT:  $BB5_22:
+; MIPS-NEXT:    move  $2, $9
+; MIPS-NEXT:  $BB5_23:
+; MIPS-NEXT:    bnez  $14, $BB5_25
 ; MIPS-NEXT:    nop
+; MIPS-NEXT:  # %bb.24:
+; MIPS-NEXT:    move  $3, $9
+; MIPS-NEXT:  $BB5_25:
+; MIPS-NEXT:    move  $4, $6
+; MIPS-NEXT:    jr  $ra
+; MIPS-NEXT:    move  $5, $7
 ;
 ; MIPS32-LABEL: ashr_i128:
 ; MIPS32:       # %bb.0: # %entry
@@ -715,27 +691,23 @@ define signext i128 @ashr_i128(i128 signext %a, i128 signext %b) {
 ;
 ; MIPS3-LABEL: ashr_i128:
 ; MIPS3:       # %bb.0: # %entry
-; MIPS3-NEXT:    sll $8, $7, 0
-; MIPS3-NEXT:    dsrav $2, $4, $7
-; MIPS3-NEXT:    andi $6, $8, 64
-; MIPS3-NEXT:    beqz $6, .LBB5_3
-; MIPS3-NEXT:    move $3, $2
-; MIPS3-NEXT:  # %bb.1: # %entry
-; MIPS3-NEXT:    bnez $6, .LBB5_4
-; MIPS3-NEXT:    nop
-; MIPS3-NEXT:  .LBB5_2: # %entry
-; MIPS3-NEXT:    jr $ra
-; MIPS3-NEXT:    nop
-; MIPS3-NEXT:  .LBB5_3: # %entry
+; MIPS3-NEXT:    sll $2, $7, 0
+; MIPS3-NEXT:    andi  $1, $2, 64
+; MIPS3-NEXT:    bnez  $1, .LBB5_2
+; MIPS3-NEXT:    dsrav $3, $4, $7
+; MIPS3-NEXT:  # %bb.1:
 ; MIPS3-NEXT:    dsrlv $1, $5, $7
-; MIPS3-NEXT:    dsll $3, $4, 1
-; MIPS3-NEXT:    not $5, $8
-; MIPS3-NEXT:    dsllv $3, $3, $5
-; MIPS3-NEXT:    beqz $6, .LBB5_2
-; MIPS3-NEXT:    or $3, $3, $1
-; MIPS3-NEXT:  .LBB5_4:
-; MIPS3-NEXT:    jr $ra
-; MIPS3-NEXT:    dsra $2, $4, 63
+; MIPS3-NEXT:    dsll  $4, $4, 1
+; MIPS3-NEXT:    not $2, $2
+; MIPS3-NEXT:    dsllv $2, $4, $2
+; MIPS3-NEXT:    or  $1, $2, $1
+; MIPS3-NEXT:    move  $2, $3
+; MIPS3-NEXT:    jr  $ra
+; MIPS3-NEXT:    move  $3, $1
+; MIPS3-NEXT:  .LBB5_2:
+; MIPS3-NEXT:    jr  $ra
+; MIPS3-NEXT:    dsra  $2, $4, 63
+
 ;
 ; MIPS64-LABEL: ashr_i128:
 ; MIPS64:       # %bb.0: # %entry
diff --git a/test/CodeGen/Mips/llvm-ir/lshr.ll b/test/CodeGen/Mips/llvm-ir/lshr.ll
index 80682409a68458569dc025f250360e99cf043598..57b1b819cca19045f4f4b984801b133d620486a3 100644
--- a/test/CodeGen/Mips/llvm-ir/lshr.ll
+++ b/test/CodeGen/Mips/llvm-ir/lshr.ll
@@ -298,28 +298,22 @@ entry:
 
 define signext i64 @lshr_i64(i64 signext %a, i64 signext %b) {
 ; MIPS2-LABEL: lshr_i64:
-; MIPS2:       # %bb.0: # %entry
-; MIPS2-NEXT:    srlv $6, $4, $7
-; MIPS2-NEXT:    andi $8, $7, 32
-; MIPS2-NEXT:    beqz $8, $BB4_3
-; MIPS2-NEXT:    move $3, $6
-; MIPS2-NEXT:  # %bb.1: # %entry
-; MIPS2-NEXT:    beqz $8, $BB4_4
+; MIPS2:       # %bb.0:
+; MIPS2-NEXT:    srlv  $6, $4, $7
+; MIPS2-NEXT:    andi  $1, $7, 32
+; MIPS2-NEXT:    bnez  $1, $BB4_2
 ; MIPS2-NEXT:    addiu $2, $zero, 0
-; MIPS2-NEXT:  $BB4_2: # %entry
-; MIPS2-NEXT:    jr $ra
-; MIPS2-NEXT:    nop
-; MIPS2-NEXT:  $BB4_3: # %entry
-; MIPS2-NEXT:    srlv $1, $5, $7
+; MIPS2-NEXT:  # %bb.1:
+; MIPS2-NEXT:    srlv  $1, $5, $7
 ; MIPS2-NEXT:    not $2, $7
 ; MIPS2-NEXT:    sll $3, $4, 1
-; MIPS2-NEXT:    sllv $2, $3, $2
-; MIPS2-NEXT:    or $3, $2, $1
-; MIPS2-NEXT:    bnez $8, $BB4_2
-; MIPS2-NEXT:    addiu $2, $zero, 0
-; MIPS2-NEXT:  $BB4_4: # %entry
-; MIPS2-NEXT:    jr $ra
-; MIPS2-NEXT:    move $2, $6
+; MIPS2-NEXT:    sllv  $2, $3, $2
+; MIPS2-NEXT:    or  $3, $2, $1
+; MIPS2-NEXT:    jr  $ra
+; MIPS2-NEXT:    move  $2, $6
+; MIPS2-NEXT:  $BB4_2:
+; MIPS2-NEXT:    jr  $ra
+; MIPS2-NEXT:    move  $3, $6
 ;
 ; MIPS32-LABEL: lshr_i64:
 ; MIPS32:       # %bb.0: # %entry
@@ -423,131 +417,119 @@ entry:
 
 define signext i128 @lshr_i128(i128 signext %a, i128 signext %b) {
 ; MIPS2-LABEL: lshr_i128:
-; MIPS2:       # %bb.0: # %entry
-; MIPS2-NEXT:    addiu $sp, $sp, -8
-; MIPS2-NEXT:    .cfi_def_cfa_offset 8
-; MIPS2-NEXT:    sw $17, 4($sp) # 4-byte Folded Spill
-; MIPS2-NEXT:    sw $16, 0($sp) # 4-byte Folded Spill
-; MIPS2-NEXT:    .cfi_offset 17, -4
-; MIPS2-NEXT:    .cfi_offset 16, -8
-; MIPS2-NEXT:    lw $2, 36($sp)
+; MIPS2:       # %bb.0:
+; MIPS2-NEXT:    lw  $2, 28($sp)
 ; MIPS2-NEXT:    addiu $1, $zero, 64
-; MIPS2-NEXT:    subu $10, $1, $2
-; MIPS2-NEXT:    sllv $9, $5, $10
-; MIPS2-NEXT:    andi $13, $10, 32
-; MIPS2-NEXT:    addiu $8, $zero, 0
-; MIPS2-NEXT:    bnez $13, $BB5_2
-; MIPS2-NEXT:    addiu $25, $zero, 0
-; MIPS2-NEXT:  # %bb.1: # %entry
-; MIPS2-NEXT:    move $25, $9
-; MIPS2-NEXT:  $BB5_2: # %entry
-; MIPS2-NEXT:    not $3, $2
-; MIPS2-NEXT:    srlv $11, $6, $2
-; MIPS2-NEXT:    andi $12, $2, 32
-; MIPS2-NEXT:    bnez $12, $BB5_4
-; MIPS2-NEXT:    move $16, $11
-; MIPS2-NEXT:  # %bb.3: # %entry
-; MIPS2-NEXT:    srlv $1, $7, $2
-; MIPS2-NEXT:    sll $14, $6, 1
-; MIPS2-NEXT:    sllv $14, $14, $3
-; MIPS2-NEXT:    or $16, $14, $1
-; MIPS2-NEXT:  $BB5_4: # %entry
-; MIPS2-NEXT:    addiu $24, $2, -64
-; MIPS2-NEXT:    sll $17, $4, 1
-; MIPS2-NEXT:    srlv $14, $4, $24
-; MIPS2-NEXT:    andi $15, $24, 32
-; MIPS2-NEXT:    bnez $15, $BB5_6
-; MIPS2-NEXT:    move $gp, $14
-; MIPS2-NEXT:  # %bb.5: # %entry
-; MIPS2-NEXT:    srlv $1, $5, $24
-; MIPS2-NEXT:    not $24, $24
-; MIPS2-NEXT:    sllv $24, $17, $24
-; MIPS2-NEXT:    or $gp, $24, $1
-; MIPS2-NEXT:  $BB5_6: # %entry
-; MIPS2-NEXT:    sltiu $24, $2, 64
-; MIPS2-NEXT:    beqz $24, $BB5_8
-; MIPS2-NEXT:    nop
-; MIPS2-NEXT:  # %bb.7:
-; MIPS2-NEXT:    or $gp, $16, $25
-; MIPS2-NEXT:  $BB5_8: # %entry
-; MIPS2-NEXT:    srlv $25, $4, $2
-; MIPS2-NEXT:    bnez $12, $BB5_10
-; MIPS2-NEXT:    move $16, $25
-; MIPS2-NEXT:  # %bb.9: # %entry
-; MIPS2-NEXT:    srlv $1, $5, $2
-; MIPS2-NEXT:    sllv $3, $17, $3
-; MIPS2-NEXT:    or $16, $3, $1
-; MIPS2-NEXT:  $BB5_10: # %entry
-; MIPS2-NEXT:    bnez $12, $BB5_12
-; MIPS2-NEXT:    addiu $3, $zero, 0
-; MIPS2-NEXT:  # %bb.11: # %entry
-; MIPS2-NEXT:    move $3, $25
-; MIPS2-NEXT:  $BB5_12: # %entry
-; MIPS2-NEXT:    addiu $1, $zero, 63
-; MIPS2-NEXT:    sltiu $25, $2, 1
-; MIPS2-NEXT:    beqz $25, $BB5_22
-; MIPS2-NEXT:    sltu $17, $1, $2
-; MIPS2-NEXT:  # %bb.13: # %entry
-; MIPS2-NEXT:    beqz $17, $BB5_23
-; MIPS2-NEXT:    addiu $2, $zero, 0
-; MIPS2-NEXT:  $BB5_14: # %entry
-; MIPS2-NEXT:    beqz $17, $BB5_24
+; MIPS2-NEXT:    subu  $12, $1, $2
+; MIPS2-NEXT:    sllv  $10, $5, $12
+; MIPS2-NEXT:    andi  $15, $12, 32
+; MIPS2-NEXT:    andi  $8, $2, 32
 ; MIPS2-NEXT:    addiu $3, $zero, 0
-; MIPS2-NEXT:  $BB5_15: # %entry
-; MIPS2-NEXT:    beqz $13, $BB5_25
+; MIPS2-NEXT:    bnez  $15, $BB5_2
+; MIPS2-NEXT:    addiu $13, $zero, 0
+; MIPS2-NEXT:  # %bb.1:
+; MIPS2-NEXT:    move  $13, $10
+; MIPS2-NEXT:  $BB5_2:
+; MIPS2-NEXT:    not $9, $2
+; MIPS2-NEXT:    bnez  $8, $BB5_5
+; MIPS2-NEXT:    srlv  $24, $6, $2
+; MIPS2-NEXT:  # %bb.3:
+; MIPS2-NEXT:    sll $1, $6, 1
+; MIPS2-NEXT:    srlv  $11, $7, $2
+; MIPS2-NEXT:    sllv  $1, $1, $9
+; MIPS2-NEXT:    or  $14, $1, $11
+; MIPS2-NEXT:    bnez  $15, $BB5_7
+; MIPS2-NEXT:    move  $11, $24
+; MIPS2-NEXT:  # %bb.4:
+; MIPS2-NEXT:    b $BB5_6
 ; MIPS2-NEXT:    nop
-; MIPS2-NEXT:  $BB5_16: # %entry
-; MIPS2-NEXT:    beqz $12, $BB5_26
-; MIPS2-NEXT:    addiu $4, $zero, 0
-; MIPS2-NEXT:  $BB5_17: # %entry
-; MIPS2-NEXT:    beqz $15, $BB5_27
+; MIPS2-NEXT:  $BB5_5:
+; MIPS2-NEXT:    addiu $11, $zero, 0
+; MIPS2-NEXT:    bnez  $15, $BB5_7
+; MIPS2-NEXT:    move  $14, $24
+; MIPS2-NEXT:  $BB5_6:
+; MIPS2-NEXT:    sllv  $1, $4, $12
+; MIPS2-NEXT:    not $10, $12
+; MIPS2-NEXT:    srl $12, $5, 1
+; MIPS2-NEXT:    srlv  $10, $12, $10
+; MIPS2-NEXT:    or  $10, $1, $10
+; MIPS2-NEXT:  $BB5_7:
+; MIPS2-NEXT:    addiu $15, $2, -64
+; MIPS2-NEXT:    sll $12, $4, 1
+; MIPS2-NEXT:    andi  $1, $15, 32
+; MIPS2-NEXT:    bnez  $1, $BB5_10
+; MIPS2-NEXT:    srlv  $25, $4, $15
+; MIPS2-NEXT:  # %bb.8:
+; MIPS2-NEXT:    srlv  $1, $5, $15
+; MIPS2-NEXT:    not $15, $15
+; MIPS2-NEXT:    sllv  $15, $12, $15
+; MIPS2-NEXT:    or  $24, $15, $1
+; MIPS2-NEXT:    move  $15, $25
+; MIPS2-NEXT:    sltiu $25, $2, 64
+; MIPS2-NEXT:    beqz  $25, $BB5_12
 ; MIPS2-NEXT:    nop
-; MIPS2-NEXT:  $BB5_18: # %entry
-; MIPS2-NEXT:    bnez $24, $BB5_28
+; MIPS2-NEXT:  # %bb.9:
+; MIPS2-NEXT:    b $BB5_11
 ; MIPS2-NEXT:    nop
-; MIPS2-NEXT:  $BB5_19: # %entry
-; MIPS2-NEXT:    bnez $25, $BB5_21
+; MIPS2-NEXT:  $BB5_10:
+; MIPS2-NEXT:    move  $24, $25
+; MIPS2-NEXT:    sltiu $25, $2, 64
+; MIPS2-NEXT:    beqz  $25, $BB5_12
+; MIPS2-NEXT:    addiu $15, $zero, 0
+; MIPS2-NEXT:  $BB5_11:
+; MIPS2-NEXT:    or  $24, $14, $13
+; MIPS2-NEXT:  $BB5_12:
+; MIPS2-NEXT:    sltiu $13, $2, 1
+; MIPS2-NEXT:    beqz  $13, $BB5_19
 ; MIPS2-NEXT:    nop
-; MIPS2-NEXT:  $BB5_20: # %entry
-; MIPS2-NEXT:    move $6, $8
-; MIPS2-NEXT:  $BB5_21: # %entry
-; MIPS2-NEXT:    move $4, $6
-; MIPS2-NEXT:    move $5, $7
-; MIPS2-NEXT:    lw $16, 0($sp) # 4-byte Folded Reload
-; MIPS2-NEXT:    lw $17, 4($sp) # 4-byte Folded Reload
-; MIPS2-NEXT:    jr $ra
-; MIPS2-NEXT:    addiu $sp, $sp, 8
-; MIPS2-NEXT:  $BB5_22: # %entry
-; MIPS2-NEXT:    move $7, $gp
-; MIPS2-NEXT:    bnez $17, $BB5_14
+; MIPS2-NEXT:  # %bb.13:
+; MIPS2-NEXT:    bnez  $25, $BB5_20
+; MIPS2-NEXT:    nop
+; MIPS2-NEXT:  $BB5_14:
+; MIPS2-NEXT:    bnez  $13, $BB5_16
+; MIPS2-NEXT:    addiu $10, $zero, 63
+; MIPS2-NEXT:  $BB5_15:
+; MIPS2-NEXT:    move  $6, $15
+; MIPS2-NEXT:  $BB5_16:
+; MIPS2-NEXT:    sltu  $10, $10, $2
+; MIPS2-NEXT:    bnez  $8, $BB5_22
+; MIPS2-NEXT:    srlv  $11, $4, $2
+; MIPS2-NEXT:  # %bb.17:
+; MIPS2-NEXT:    srlv  $1, $5, $2
+; MIPS2-NEXT:    sllv  $2, $12, $9
+; MIPS2-NEXT:    or  $4, $2, $1
+; MIPS2-NEXT:    move  $5, $11
+; MIPS2-NEXT:    bnez  $10, $BB5_24
 ; MIPS2-NEXT:    addiu $2, $zero, 0
-; MIPS2-NEXT:  $BB5_23: # %entry
-; MIPS2-NEXT:    move $2, $3
-; MIPS2-NEXT:    bnez $17, $BB5_15
-; MIPS2-NEXT:    addiu $3, $zero, 0
-; MIPS2-NEXT:  $BB5_24: # %entry
-; MIPS2-NEXT:    bnez $13, $BB5_16
-; MIPS2-NEXT:    move $3, $16
-; MIPS2-NEXT:  $BB5_25: # %entry
-; MIPS2-NEXT:    not $1, $10
-; MIPS2-NEXT:    srl $5, $5, 1
-; MIPS2-NEXT:    sllv $4, $4, $10
-; MIPS2-NEXT:    srlv $1, $5, $1
-; MIPS2-NEXT:    or $9, $4, $1
-; MIPS2-NEXT:    bnez $12, $BB5_17
-; MIPS2-NEXT:    addiu $4, $zero, 0
-; MIPS2-NEXT:  $BB5_26: # %entry
-; MIPS2-NEXT:    bnez $15, $BB5_18
-; MIPS2-NEXT:    move $4, $11
-; MIPS2-NEXT:  $BB5_27: # %entry
-; MIPS2-NEXT:    beqz $24, $BB5_19
-; MIPS2-NEXT:    move $8, $14
-; MIPS2-NEXT:  $BB5_28:
-; MIPS2-NEXT:    bnez $25, $BB5_21
-; MIPS2-NEXT:    or $8, $4, $9
-; MIPS2-NEXT:  # %bb.29:
-; MIPS2-NEXT:    b $BB5_20
+; MIPS2-NEXT:  # %bb.18:
+; MIPS2-NEXT:    b $BB5_23
 ; MIPS2-NEXT:    nop
+; MIPS2-NEXT:  $BB5_19:
+; MIPS2-NEXT:    beqz  $25, $BB5_14
+; MIPS2-NEXT:    move  $7, $24
+; MIPS2-NEXT:  $BB5_20:
+; MIPS2-NEXT:    or  $15, $11, $10
+; MIPS2-NEXT:    bnez  $13, $BB5_16
+; MIPS2-NEXT:    addiu $10, $zero, 63
+; MIPS2-NEXT:  # %bb.21:
+; MIPS2-NEXT:    b $BB5_15
+; MIPS2-NEXT:    nop
+; MIPS2-NEXT:  $BB5_22:
+; MIPS2-NEXT:    addiu $5, $zero, 0
+; MIPS2-NEXT:    move  $4, $11
+; MIPS2-NEXT:    bnez  $10, $BB5_24
+; MIPS2-NEXT:    addiu $2, $zero, 0
+; MIPS2-NEXT:  $BB5_23:
+; MIPS2-NEXT:    move  $2, $5
+; MIPS2-NEXT:  $BB5_24:
+; MIPS2-NEXT:    bnez  $10, $BB5_26
+; MIPS2-NEXT:    nop
+; MIPS2-NEXT:  # %bb.25:
+; MIPS2-NEXT:    move  $3, $4
+; MIPS2-NEXT:  $BB5_26:
+; MIPS2-NEXT:    move  $4, $6
+; MIPS2-NEXT:    jr  $ra
+; MIPS2-NEXT:    move  $5, $7
 ;
 ; MIPS32-LABEL: lshr_i128:
 ; MIPS32:       # %bb.0: # %entry
@@ -731,29 +713,23 @@ define signext i128 @lshr_i128(i128 signext %a, i128 signext %b) {
 ; MIPS32R6-NEXT:    addiu $sp, $sp, 8
 ;
 ; MIPS3-LABEL: lshr_i128:
-; MIPS3:       # %bb.0: # %entry
-; MIPS3-NEXT:    sll $2, $7, 0
+; MIPS3:       # %bb.0:
+; MIPS3-NEXT:    sll $3, $7, 0
 ; MIPS3-NEXT:    dsrlv $6, $4, $7
-; MIPS3-NEXT:    andi $8, $2, 64
-; MIPS3-NEXT:    beqz $8, .LBB5_3
-; MIPS3-NEXT:    move $3, $6
-; MIPS3-NEXT:  # %bb.1: # %entry
-; MIPS3-NEXT:    beqz $8, .LBB5_4
-; MIPS3-NEXT:    daddiu $2, $zero, 0
-; MIPS3-NEXT:  .LBB5_2: # %entry
-; MIPS3-NEXT:    jr $ra
-; MIPS3-NEXT:    nop
-; MIPS3-NEXT:  .LBB5_3: # %entry
+; MIPS3-NEXT:    andi  $1, $3, 64
+; MIPS3-NEXT:    bnez  $1, .LBB5_2
+; MIPS3-NEXT:    daddiu  $2, $zero, 0
+; MIPS3-NEXT:  # %bb.1:
 ; MIPS3-NEXT:    dsrlv $1, $5, $7
-; MIPS3-NEXT:    dsll $3, $4, 1
-; MIPS3-NEXT:    not $2, $2
-; MIPS3-NEXT:    dsllv $2, $3, $2
-; MIPS3-NEXT:    or $3, $2, $1
-; MIPS3-NEXT:    bnez $8, .LBB5_2
-; MIPS3-NEXT:    daddiu $2, $zero, 0
-; MIPS3-NEXT:  .LBB5_4: # %entry
-; MIPS3-NEXT:    jr $ra
-; MIPS3-NEXT:    move $2, $6
+; MIPS3-NEXT:    dsll  $2, $4, 1
+; MIPS3-NEXT:    not $3, $3
+; MIPS3-NEXT:    dsllv $2, $2, $3
+; MIPS3-NEXT:    or  $3, $2, $1
+; MIPS3-NEXT:    jr  $ra
+; MIPS3-NEXT:    move  $2, $6
+; MIPS3-NEXT:  .LBB5_2:
+; MIPS3-NEXT:    jr  $ra
+; MIPS3-NEXT:    move  $3, $6
 ;
 ; MIPS4-LABEL: lshr_i128:
 ; MIPS4:       # %bb.0: # %entry
diff --git a/test/CodeGen/Mips/llvm-ir/sdiv.ll b/test/CodeGen/Mips/llvm-ir/sdiv.ll
index e54eaa63222a05484bcff47656a6b3203650432a..af3d4f50f3fe4f5bf6ea6705fcd614f2529deb1b 100644
--- a/test/CodeGen/Mips/llvm-ir/sdiv.ll
+++ b/test/CodeGen/Mips/llvm-ir/sdiv.ll
@@ -1,36 +1,38 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=mips -mcpu=mips2 -relocation-model=pic \
-; RUN:   | FileCheck %s -check-prefixes=GP32,GP32R0R2
+; RUN:   -mips-jalr-reloc=false | FileCheck %s -check-prefixes=GP32,GP32R0R2
 ; RUN: llc < %s -mtriple=mips -mcpu=mips32 -relocation-model=pic \
-; RUN:   | FileCheck %s -check-prefixes=GP32,GP32R0R2
+; RUN:   -mips-jalr-reloc=false | FileCheck %s -check-prefixes=GP32,GP32R0R2
 ; RUN: llc < %s -mtriple=mips -mcpu=mips32r2 -relocation-model=pic \
-; RUN:   | FileCheck %s -check-prefixes=GP32,GP32R2R5
+; RUN:   -mips-jalr-reloc=false | FileCheck %s -check-prefixes=GP32,GP32R2R5
 ; RUN: llc < %s -mtriple=mips -mcpu=mips32r3 -relocation-model=pic \
-; RUN:   | FileCheck %s -check-prefixes=GP32,GP32R2R5
+; RUN:   -mips-jalr-reloc=false | FileCheck %s -check-prefixes=GP32,GP32R2R5
 ; RUN: llc < %s -mtriple=mips -mcpu=mips32r5 -relocation-model=pic \
-; RUN:   | FileCheck %s -check-prefixes=GP32,GP32R2R5
+; RUN:   -mips-jalr-reloc=false | FileCheck %s -check-prefixes=GP32,GP32R2R5
 ; RUN: llc < %s -mtriple=mips -mcpu=mips32r6 -relocation-model=pic \
-; RUN:   | FileCheck %s -check-prefix=GP32R6
+; RUN:   -mips-jalr-reloc=false | FileCheck %s -check-prefix=GP32R6
 
 ; RUN: llc < %s -mtriple=mips64 -mcpu=mips3 -relocation-model=pic \
-; RUN:   | FileCheck %s -check-prefixes=GP64,GP64R0R1
+; RUN:   -mips-jalr-reloc=false | FileCheck %s -check-prefixes=GP64,GP64R0R1
 ; RUN: llc < %s -mtriple=mips64 -mcpu=mips4 -relocation-model=pic \
-; RUN:   | FileCheck %s -check-prefixes=GP64,GP64R0R1
+; RUN:   -mips-jalr-reloc=false | FileCheck %s -check-prefixes=GP64,GP64R0R1
 ; RUN: llc < %s -mtriple=mips64 -mcpu=mips64 -relocation-model=pic \
-; RUN:   | FileCheck %s -check-prefixes=GP64,GP64R0R1
+; RUN:   -mips-jalr-reloc=false | FileCheck %s -check-prefixes=GP64,GP64R0R1
 ; RUN: llc < %s -mtriple=mips64 -mcpu=mips64r2 -relocation-model=pic \
-; RUN:   | FileCheck %s -check-prefixes=GP64,GP64R2R5
+; RUN:   -mips-jalr-reloc=false | FileCheck %s -check-prefixes=GP64,GP64R2R5
 ; RUN: llc < %s -mtriple=mips64 -mcpu=mips64r3 -relocation-model=pic \
-; RUN:   | FileCheck %s -check-prefixes=GP64,GP64R2R5
+; RUN:   -mips-jalr-reloc=false | FileCheck %s -check-prefixes=GP64,GP64R2R5
 ; RUN: llc < %s -mtriple=mips64 -mcpu=mips64r5 -relocation-model=pic \
-; RUN:   | FileCheck %s -check-prefixes=GP64,GP64R2R5
+; RUN:   -mips-jalr-reloc=false | FileCheck %s -check-prefixes=GP64,GP64R2R5
 ; RUN: llc < %s -mtriple=mips64 -mcpu=mips64r6 -relocation-model=pic \
-; RUN:   | FileCheck %s -check-prefix=GP64R6
+; RUN:   -mips-jalr-reloc=false | FileCheck %s -check-prefix=GP64R6
 
-; RUN: llc < %s -mtriple=mips -mcpu=mips32r3 -mattr=+micromips -relocation-model=pic \
-; RUN:   | FileCheck %s -check-prefix=MMR3
-; RUN: llc < %s -mtriple=mips -mcpu=mips32r6 -mattr=+micromips -relocation-model=pic \
-; RUN:   | FileCheck %s -check-prefix=MMR6
+; RUN: llc < %s -mtriple=mips -mcpu=mips32r3 -mattr=+micromips \
+; RUN:   -relocation-model=pic -mips-jalr-reloc=false | \
+; RUN:   FileCheck %s -check-prefix=MMR3
+; RUN: llc < %s -mtriple=mips -mcpu=mips32r6 -mattr=+micromips \
+; RUN:   -relocation-model=pic -mips-jalr-reloc=false | \
+; RUN:   FileCheck %s -check-prefix=MMR6
 
 define signext i1 @sdiv_i1(i1 signext %a, i1 signext %b) {
 ; GP32-LABEL: sdiv_i1:
diff --git a/test/CodeGen/Mips/llvm-ir/shl.ll b/test/CodeGen/Mips/llvm-ir/shl.ll
index 6c34f63cf06331f2fde43a40c354da375817f5f4..bd69a3ff0207d3e62a8c839fba5a839ee6f6ad3d 100644
--- a/test/CodeGen/Mips/llvm-ir/shl.ll
+++ b/test/CodeGen/Mips/llvm-ir/shl.ll
@@ -330,28 +330,28 @@ entry:
 
 define signext i64 @shl_i64(i64 signext %a, i64 signext %b) {
 ; MIPS2-LABEL: shl_i64:
-; MIPS2:       # %bb.0: # %entry
-; MIPS2-NEXT:    sllv $6, $5, $7
-; MIPS2-NEXT:    andi $8, $7, 32
-; MIPS2-NEXT:    beqz $8, $BB4_3
-; MIPS2-NEXT:    move $2, $6
-; MIPS2-NEXT:  # %bb.1: # %entry
-; MIPS2-NEXT:    beqz $8, $BB4_4
+; MIPS2:       # %bb.0:
+; MIPS2-NEXT:    sllv  $6, $5, $7
+; MIPS2-NEXT:    andi  $8, $7, 32
+; MIPS2-NEXT:    beqz  $8, $BB4_3
+; MIPS2-NEXT:    move  $2, $6
+; MIPS2-NEXT:  # %bb.1:
+; MIPS2-NEXT:    beqz  $8, $BB4_4
 ; MIPS2-NEXT:    addiu $3, $zero, 0
-; MIPS2-NEXT:  $BB4_2: # %entry
-; MIPS2-NEXT:    jr $ra
+; MIPS2-NEXT:  $BB4_2:
+; MIPS2-NEXT:    jr  $ra
 ; MIPS2-NEXT:    nop
-; MIPS2-NEXT:  $BB4_3: # %entry
-; MIPS2-NEXT:    sllv $1, $4, $7
+; MIPS2-NEXT:  $BB4_3:
+; MIPS2-NEXT:    sllv  $1, $4, $7
 ; MIPS2-NEXT:    not $2, $7
 ; MIPS2-NEXT:    srl $3, $5, 1
-; MIPS2-NEXT:    srlv $2, $3, $2
-; MIPS2-NEXT:    or $2, $1, $2
-; MIPS2-NEXT:    bnez $8, $BB4_2
+; MIPS2-NEXT:    srlv  $2, $3, $2
+; MIPS2-NEXT:    or  $2, $1, $2
+; MIPS2-NEXT:    bnez  $8, $BB4_2
 ; MIPS2-NEXT:    addiu $3, $zero, 0
-; MIPS2-NEXT:  $BB4_4: # %entry
-; MIPS2-NEXT:    jr $ra
-; MIPS2-NEXT:    move $3, $6
+; MIPS2-NEXT:  $BB4_4:
+; MIPS2-NEXT:    jr  $ra
+; MIPS2-NEXT:    move  $3, $6
 ;
 ; MIPS32-LABEL: shl_i64:
 ; MIPS32:       # %bb.0: # %entry
@@ -455,132 +455,131 @@ entry:
 
 define signext i128 @shl_i128(i128 signext %a, i128 signext %b) {
 ; MIPS2-LABEL: shl_i128:
-; MIPS2:       # %bb.0: # %entry
+; MIPS2:       # %bb.0:
 ; MIPS2-NEXT:    addiu $sp, $sp, -8
 ; MIPS2-NEXT:    .cfi_def_cfa_offset 8
-; MIPS2-NEXT:    sw $17, 4($sp) # 4-byte Folded Spill
-; MIPS2-NEXT:    sw $16, 0($sp) # 4-byte Folded Spill
+; MIPS2-NEXT:    sw  $17, 4($sp)
+; MIPS2-NEXT:    sw  $16, 0($sp)
 ; MIPS2-NEXT:    .cfi_offset 17, -4
 ; MIPS2-NEXT:    .cfi_offset 16, -8
-; MIPS2-NEXT:    lw $8, 36($sp)
+; MIPS2-NEXT:    lw  $8, 36($sp)
 ; MIPS2-NEXT:    addiu $1, $zero, 64
-; MIPS2-NEXT:    subu $10, $1, $8
-; MIPS2-NEXT:    srlv $3, $6, $10
-; MIPS2-NEXT:    andi $13, $10, 32
+; MIPS2-NEXT:    subu  $3, $1, $8
+; MIPS2-NEXT:    srlv  $9, $6, $3
+; MIPS2-NEXT:    andi  $1, $3, 32
+; MIPS2-NEXT:    bnez  $1, $BB5_2
 ; MIPS2-NEXT:    addiu $2, $zero, 0
-; MIPS2-NEXT:    bnez $13, $BB5_2
-; MIPS2-NEXT:    addiu $25, $zero, 0
-; MIPS2-NEXT:  # %bb.1: # %entry
-; MIPS2-NEXT:    move $25, $3
-; MIPS2-NEXT:  $BB5_2: # %entry
-; MIPS2-NEXT:    not $9, $8
-; MIPS2-NEXT:    sllv $11, $5, $8
-; MIPS2-NEXT:    andi $12, $8, 32
-; MIPS2-NEXT:    bnez $12, $BB5_4
-; MIPS2-NEXT:    move $16, $11
-; MIPS2-NEXT:  # %bb.3: # %entry
-; MIPS2-NEXT:    sllv $1, $4, $8
-; MIPS2-NEXT:    srl $14, $5, 1
-; MIPS2-NEXT:    srlv $14, $14, $9
-; MIPS2-NEXT:    or $16, $1, $14
-; MIPS2-NEXT:  $BB5_4: # %entry
-; MIPS2-NEXT:    addiu $24, $8, -64
-; MIPS2-NEXT:    srl $17, $7, 1
-; MIPS2-NEXT:    sllv $14, $7, $24
-; MIPS2-NEXT:    andi $15, $24, 32
-; MIPS2-NEXT:    bnez $15, $BB5_6
-; MIPS2-NEXT:    move $gp, $14
-; MIPS2-NEXT:  # %bb.5: # %entry
-; MIPS2-NEXT:    sllv $1, $6, $24
-; MIPS2-NEXT:    not $24, $24
-; MIPS2-NEXT:    srlv $24, $17, $24
-; MIPS2-NEXT:    or $gp, $1, $24
-; MIPS2-NEXT:  $BB5_6: # %entry
-; MIPS2-NEXT:    sltiu $24, $8, 64
-; MIPS2-NEXT:    beqz $24, $BB5_8
+; MIPS2-NEXT:  # %bb.1:
+; MIPS2-NEXT:    srlv  $1, $7, $3
+; MIPS2-NEXT:    not $3, $3
+; MIPS2-NEXT:    sll $10, $6, 1
+; MIPS2-NEXT:    sllv  $3, $10, $3
+; MIPS2-NEXT:    or  $3, $3, $1
+; MIPS2-NEXT:    b $BB5_3
+; MIPS2-NEXT:    move  $15, $9
+; MIPS2-NEXT:  $BB5_2:
+; MIPS2-NEXT:    addiu $15, $zero, 0
+; MIPS2-NEXT:    move  $3, $9
+; MIPS2-NEXT:  $BB5_3:
+; MIPS2-NEXT:    not $13, $8
+; MIPS2-NEXT:    sllv  $9, $5, $8
+; MIPS2-NEXT:    andi  $10, $8, 32
+; MIPS2-NEXT:    bnez  $10, $BB5_5
+; MIPS2-NEXT:    move  $25, $9
+; MIPS2-NEXT:  # %bb.4:
+; MIPS2-NEXT:    sllv  $1, $4, $8
+; MIPS2-NEXT:    srl $11, $5, 1
+; MIPS2-NEXT:    srlv  $11, $11, $13
+; MIPS2-NEXT:    or  $25, $1, $11
+; MIPS2-NEXT:  $BB5_5:
+; MIPS2-NEXT:    addiu $14, $8, -64
+; MIPS2-NEXT:    srl $24, $7, 1
+; MIPS2-NEXT:    sllv  $11, $7, $14
+; MIPS2-NEXT:    andi  $12, $14, 32
+; MIPS2-NEXT:    bnez  $12, $BB5_7
+; MIPS2-NEXT:    move  $gp, $11
+; MIPS2-NEXT:  # %bb.6:
+; MIPS2-NEXT:    sllv  $1, $6, $14
+; MIPS2-NEXT:    not $14, $14
+; MIPS2-NEXT:    srlv  $14, $24, $14
+; MIPS2-NEXT:    or  $gp, $1, $14
+; MIPS2-NEXT:  $BB5_7:
+; MIPS2-NEXT:    sltiu $14, $8, 64
+; MIPS2-NEXT:    beqz  $14, $BB5_9
 ; MIPS2-NEXT:    nop
-; MIPS2-NEXT:  # %bb.7:
-; MIPS2-NEXT:    or $gp, $16, $25
-; MIPS2-NEXT:  $BB5_8: # %entry
-; MIPS2-NEXT:    sllv $25, $7, $8
-; MIPS2-NEXT:    bnez $12, $BB5_10
-; MIPS2-NEXT:    move $16, $25
-; MIPS2-NEXT:  # %bb.9: # %entry
-; MIPS2-NEXT:    sllv $1, $6, $8
-; MIPS2-NEXT:    srlv $9, $17, $9
-; MIPS2-NEXT:    or $16, $1, $9
-; MIPS2-NEXT:  $BB5_10: # %entry
-; MIPS2-NEXT:    bnez $12, $BB5_12
-; MIPS2-NEXT:    addiu $9, $zero, 0
-; MIPS2-NEXT:  # %bb.11: # %entry
-; MIPS2-NEXT:    move $9, $25
-; MIPS2-NEXT:  $BB5_12: # %entry
+; MIPS2-NEXT:  # %bb.8:
+; MIPS2-NEXT:    or  $gp, $25, $15
+; MIPS2-NEXT:  $BB5_9:
+; MIPS2-NEXT:    sllv  $25, $7, $8
+; MIPS2-NEXT:    bnez  $10, $BB5_11
+; MIPS2-NEXT:    addiu $17, $zero, 0
+; MIPS2-NEXT:  # %bb.10:
+; MIPS2-NEXT:    move  $17, $25
+; MIPS2-NEXT:  $BB5_11:
 ; MIPS2-NEXT:    addiu $1, $zero, 63
-; MIPS2-NEXT:    sltiu $25, $8, 1
-; MIPS2-NEXT:    beqz $25, $BB5_22
-; MIPS2-NEXT:    sltu $17, $1, $8
-; MIPS2-NEXT:  # %bb.13: # %entry
-; MIPS2-NEXT:    beqz $17, $BB5_23
-; MIPS2-NEXT:    addiu $8, $zero, 0
-; MIPS2-NEXT:  $BB5_14: # %entry
-; MIPS2-NEXT:    beqz $17, $BB5_24
-; MIPS2-NEXT:    addiu $9, $zero, 0
-; MIPS2-NEXT:  $BB5_15: # %entry
-; MIPS2-NEXT:    beqz $13, $BB5_25
+; MIPS2-NEXT:    sltiu $15, $8, 1
+; MIPS2-NEXT:    beqz  $15, $BB5_21
+; MIPS2-NEXT:    sltu  $16, $1, $8
+; MIPS2-NEXT:  # %bb.12:
+; MIPS2-NEXT:    beqz  $16, $BB5_22
+; MIPS2-NEXT:    addiu $7, $zero, 0
+; MIPS2-NEXT:  $BB5_13:
+; MIPS2-NEXT:    beqz  $10, $BB5_23
 ; MIPS2-NEXT:    nop
-; MIPS2-NEXT:  $BB5_16: # %entry
-; MIPS2-NEXT:    beqz $12, $BB5_26
+; MIPS2-NEXT:  $BB5_14:
+; MIPS2-NEXT:    beqz  $16, $BB5_24
 ; MIPS2-NEXT:    addiu $6, $zero, 0
-; MIPS2-NEXT:  $BB5_17: # %entry
-; MIPS2-NEXT:    beqz $15, $BB5_27
+; MIPS2-NEXT:  $BB5_15:
+; MIPS2-NEXT:    beqz  $10, $BB5_25
+; MIPS2-NEXT:    addiu $8, $zero, 0
+; MIPS2-NEXT:  $BB5_16:
+; MIPS2-NEXT:    beqz  $12, $BB5_26
 ; MIPS2-NEXT:    nop
-; MIPS2-NEXT:  $BB5_18: # %entry
-; MIPS2-NEXT:    bnez $24, $BB5_28
+; MIPS2-NEXT:  $BB5_17:
+; MIPS2-NEXT:    bnez  $14, $BB5_27
 ; MIPS2-NEXT:    nop
-; MIPS2-NEXT:  $BB5_19: # %entry
-; MIPS2-NEXT:    bnez $25, $BB5_21
+; MIPS2-NEXT:  $BB5_18:
+; MIPS2-NEXT:    bnez  $15, $BB5_20
 ; MIPS2-NEXT:    nop
-; MIPS2-NEXT:  $BB5_20: # %entry
-; MIPS2-NEXT:    move $5, $2
-; MIPS2-NEXT:  $BB5_21: # %entry
-; MIPS2-NEXT:    move $2, $4
-; MIPS2-NEXT:    move $3, $5
-; MIPS2-NEXT:    move $4, $9
-; MIPS2-NEXT:    move $5, $8
-; MIPS2-NEXT:    lw $16, 0($sp) # 4-byte Folded Reload
-; MIPS2-NEXT:    lw $17, 4($sp) # 4-byte Folded Reload
-; MIPS2-NEXT:    jr $ra
+; MIPS2-NEXT:  $BB5_19:
+; MIPS2-NEXT:    move  $5, $2
+; MIPS2-NEXT:  $BB5_20:
+; MIPS2-NEXT:    move  $2, $4
+; MIPS2-NEXT:    move  $3, $5
+; MIPS2-NEXT:    move  $4, $6
+; MIPS2-NEXT:    move  $5, $7
+; MIPS2-NEXT:    lw  $16, 0($sp)
+; MIPS2-NEXT:    lw  $17, 4($sp)
+; MIPS2-NEXT:    jr  $ra
 ; MIPS2-NEXT:    addiu $sp, $sp, 8
-; MIPS2-NEXT:  $BB5_22: # %entry
-; MIPS2-NEXT:    move $4, $gp
-; MIPS2-NEXT:    bnez $17, $BB5_14
-; MIPS2-NEXT:    addiu $8, $zero, 0
-; MIPS2-NEXT:  $BB5_23: # %entry
-; MIPS2-NEXT:    move $8, $9
-; MIPS2-NEXT:    bnez $17, $BB5_15
-; MIPS2-NEXT:    addiu $9, $zero, 0
-; MIPS2-NEXT:  $BB5_24: # %entry
-; MIPS2-NEXT:    bnez $13, $BB5_16
-; MIPS2-NEXT:    move $9, $16
-; MIPS2-NEXT:  $BB5_25: # %entry
-; MIPS2-NEXT:    not $1, $10
-; MIPS2-NEXT:    sll $3, $6, 1
-; MIPS2-NEXT:    srlv $6, $7, $10
-; MIPS2-NEXT:    sllv $1, $3, $1
-; MIPS2-NEXT:    or $3, $1, $6
-; MIPS2-NEXT:    bnez $12, $BB5_17
+; MIPS2-NEXT:  $BB5_21:
+; MIPS2-NEXT:    move  $4, $gp
+; MIPS2-NEXT:    bnez  $16, $BB5_13
+; MIPS2-NEXT:    addiu $7, $zero, 0
+; MIPS2-NEXT:  $BB5_22:
+; MIPS2-NEXT:    bnez  $10, $BB5_14
+; MIPS2-NEXT:    move  $7, $17
+; MIPS2-NEXT:  $BB5_23:
+; MIPS2-NEXT:    sllv  $1, $6, $8
+; MIPS2-NEXT:    srlv  $6, $24, $13
+; MIPS2-NEXT:    or  $25, $1, $6
+; MIPS2-NEXT:    bnez  $16, $BB5_15
 ; MIPS2-NEXT:    addiu $6, $zero, 0
-; MIPS2-NEXT:  $BB5_26: # %entry
-; MIPS2-NEXT:    bnez $15, $BB5_18
-; MIPS2-NEXT:    move $6, $11
-; MIPS2-NEXT:  $BB5_27: # %entry
-; MIPS2-NEXT:    beqz $24, $BB5_19
-; MIPS2-NEXT:    move $2, $14
-; MIPS2-NEXT:  $BB5_28:
-; MIPS2-NEXT:    bnez $25, $BB5_21
-; MIPS2-NEXT:    or $2, $6, $3
-; MIPS2-NEXT:  # %bb.29:
-; MIPS2-NEXT:    b $BB5_20
+; MIPS2-NEXT:  $BB5_24:
+; MIPS2-NEXT:    move  $6, $25
+; MIPS2-NEXT:    bnez  $10, $BB5_16
+; MIPS2-NEXT:    addiu $8, $zero, 0
+; MIPS2-NEXT:  $BB5_25:
+; MIPS2-NEXT:    bnez  $12, $BB5_17
+; MIPS2-NEXT:    move  $8, $9
+; MIPS2-NEXT:  $BB5_26:
+; MIPS2-NEXT:    beqz  $14, $BB5_18
+; MIPS2-NEXT:    move  $2, $11
+; MIPS2-NEXT:  $BB5_27:
+; MIPS2-NEXT:    bnez  $15, $BB5_20
+; MIPS2-NEXT:    or  $2, $8, $3
+; MIPS2-NEXT:  # %bb.28:
+; MIPS2-NEXT:    b $BB5_19
 ; MIPS2-NEXT:    nop
 ;
 ; MIPS32-LABEL: shl_i128:
@@ -760,29 +759,29 @@ define signext i128 @shl_i128(i128 signext %a, i128 signext %b) {
 ; MIPS32R6-NEXT:    move $5, $1
 ;
 ; MIPS3-LABEL: shl_i128:
-; MIPS3:       # %bb.0: # %entry
+; MIPS3:       # %bb.0:
 ; MIPS3-NEXT:    sll $3, $7, 0
 ; MIPS3-NEXT:    dsllv $6, $5, $7
-; MIPS3-NEXT:    andi $8, $3, 64
-; MIPS3-NEXT:    beqz $8, .LBB5_3
-; MIPS3-NEXT:    move $2, $6
-; MIPS3-NEXT:  # %bb.1: # %entry
-; MIPS3-NEXT:    beqz $8, .LBB5_4
-; MIPS3-NEXT:    daddiu $3, $zero, 0
-; MIPS3-NEXT:  .LBB5_2: # %entry
-; MIPS3-NEXT:    jr $ra
+; MIPS3-NEXT:    andi  $8, $3, 64
+; MIPS3-NEXT:    beqz  $8, .LBB5_3
+; MIPS3-NEXT:    move  $2, $6
+; MIPS3-NEXT:  # %bb.1:
+; MIPS3-NEXT:    beqz  $8, .LBB5_4
+; MIPS3-NEXT:    daddiu  $3, $zero, 0
+; MIPS3-NEXT:  .LBB5_2:
+; MIPS3-NEXT:    jr  $ra
 ; MIPS3-NEXT:    nop
-; MIPS3-NEXT:  .LBB5_3: # %entry
+; MIPS3-NEXT:  .LBB5_3:
 ; MIPS3-NEXT:    dsllv $1, $4, $7
-; MIPS3-NEXT:    dsrl $2, $5, 1
+; MIPS3-NEXT:    dsrl  $2, $5, 1
 ; MIPS3-NEXT:    not $3, $3
 ; MIPS3-NEXT:    dsrlv $2, $2, $3
-; MIPS3-NEXT:    or $2, $1, $2
-; MIPS3-NEXT:    bnez $8, .LBB5_2
-; MIPS3-NEXT:    daddiu $3, $zero, 0
-; MIPS3-NEXT:  .LBB5_4: # %entry
-; MIPS3-NEXT:    jr $ra
-; MIPS3-NEXT:    move $3, $6
+; MIPS3-NEXT:    or  $2, $1, $2
+; MIPS3-NEXT:    bnez  $8, .LBB5_2
+; MIPS3-NEXT:    daddiu  $3, $zero, 0
+; MIPS3-NEXT:  .LBB5_4:
+; MIPS3-NEXT:    jr  $ra
+; MIPS3-NEXT:    move  $3, $6
 ;
 ; MIPS4-LABEL: shl_i128:
 ; MIPS4:       # %bb.0: # %entry
diff --git a/test/CodeGen/Mips/llvm-ir/srem.ll b/test/CodeGen/Mips/llvm-ir/srem.ll
index ef0502c85d59bb017262fece51229fe6278548da..487a5b9b6cbc5ebf5698ceeaafe26b0d4bafefe5 100644
--- a/test/CodeGen/Mips/llvm-ir/srem.ll
+++ b/test/CodeGen/Mips/llvm-ir/srem.ll
@@ -1,36 +1,38 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=mips -mcpu=mips2 -relocation-model=pic \
-; RUN:   | FileCheck %s -check-prefixes=GP32,GP32R0R2
+; RUN:   -mips-jalr-reloc=false | FileCheck %s -check-prefixes=GP32,GP32R0R2
 ; RUN: llc < %s -mtriple=mips -mcpu=mips32 -relocation-model=pic \
-; RUN:   | FileCheck %s -check-prefixes=GP32,GP32R0R2
+; RUN:   -mips-jalr-reloc=false | FileCheck %s -check-prefixes=GP32,GP32R0R2
 ; RUN: llc < %s -mtriple=mips -mcpu=mips32r2 -relocation-model=pic \
-; RUN:   | FileCheck %s -check-prefixes=GP32,GP32R2R5
+; RUN:   -mips-jalr-reloc=false | FileCheck %s -check-prefixes=GP32,GP32R2R5
 ; RUN: llc < %s -mtriple=mips -mcpu=mips32r3 -relocation-model=pic \
-; RUN:   | FileCheck %s -check-prefixes=GP32,GP32R2R5
+; RUN:   -mips-jalr-reloc=false | FileCheck %s -check-prefixes=GP32,GP32R2R5
 ; RUN: llc < %s -mtriple=mips -mcpu=mips32r5 -relocation-model=pic \
-; RUN:   | FileCheck %s -check-prefixes=GP32,GP32R2R5
+; RUN:   -mips-jalr-reloc=false | FileCheck %s -check-prefixes=GP32,GP32R2R5
 ; RUN: llc < %s -mtriple=mips -mcpu=mips32r6 -relocation-model=pic \
-; RUN:   | FileCheck %s -check-prefix=GP32R6
+; RUN:   -mips-jalr-reloc=false | FileCheck %s -check-prefix=GP32R6
 
 ; RUN: llc < %s -mtriple=mips64 -mcpu=mips3 -relocation-model=pic \
-; RUN:   | FileCheck %s -check-prefixes=GP64,GP64R0R1
+; RUN:   -mips-jalr-reloc=false | FileCheck %s -check-prefixes=GP64,GP64R0R1
 ; RUN: llc < %s -mtriple=mips64 -mcpu=mips4 -relocation-model=pic \
-; RUN:   | FileCheck %s -check-prefixes=GP64,GP64R0R1
+; RUN:   -mips-jalr-reloc=false | FileCheck %s -check-prefixes=GP64,GP64R0R1
 ; RUN: llc < %s -mtriple=mips64 -mcpu=mips64 -relocation-model=pic \
-; RUN:   | FileCheck %s -check-prefixes=GP64,GP64R0R1
+; RUN:   -mips-jalr-reloc=false | FileCheck %s -check-prefixes=GP64,GP64R0R1
 ; RUN: llc < %s -mtriple=mips64 -mcpu=mips64r2 -relocation-model=pic \
-; RUN:   | FileCheck %s -check-prefixes=GP64,GP64R2R5
+; RUN:   -mips-jalr-reloc=false | FileCheck %s -check-prefixes=GP64,GP64R2R5
 ; RUN: llc < %s -mtriple=mips64 -mcpu=mips64r3 -relocation-model=pic \
-; RUN:   | FileCheck %s -check-prefixes=GP64,GP64R2R5
+; RUN:   -mips-jalr-reloc=false | FileCheck %s -check-prefixes=GP64,GP64R2R5
 ; RUN: llc < %s -mtriple=mips64 -mcpu=mips64r5 -relocation-model=pic \
-; RUN:   | FileCheck %s -check-prefixes=GP64,GP64R2R5
+; RUN:   -mips-jalr-reloc=false | FileCheck %s -check-prefixes=GP64,GP64R2R5
 ; RUN: llc < %s -mtriple=mips64 -mcpu=mips64r6 -relocation-model=pic \
-; RUN:   | FileCheck %s -check-prefix=GP64R6
+; RUN:   -mips-jalr-reloc=false | FileCheck %s -check-prefix=GP64R6
 
-; RUN: llc < %s -mtriple=mips -mcpu=mips32r3 -mattr=+micromips -relocation-model=pic \
-; RUN:   | FileCheck %s -check-prefix=MMR3
-; RUN: llc < %s -mtriple=mips -mcpu=mips32r6 -mattr=+micromips -relocation-model=pic \
-; RUN:   | FileCheck %s -check-prefix=MMR6
+; RUN: llc < %s -mtriple=mips -mcpu=mips32r3 -mattr=+micromips \
+; RUN:   -relocation-model=pic -mips-jalr-reloc=false | \
+; RUN:   FileCheck %s -check-prefix=MMR3
+; RUN: llc < %s -mtriple=mips -mcpu=mips32r6 -mattr=+micromips \
+; RUN:   -relocation-model=pic -mips-jalr-reloc=false | \
+; RUN:   FileCheck %s -check-prefix=MMR6
 
 define signext i1 @srem_i1(i1 signext %a, i1 signext %b) {
 ; GP32-LABEL: srem_i1:
diff --git a/test/CodeGen/Mips/llvm-ir/udiv.ll b/test/CodeGen/Mips/llvm-ir/udiv.ll
index 8694a9f92b65ab204ada07b2a5ed1ee445ef96a9..3b7243712024b6b5be33d4d6fa571fe101d2ae74 100644
--- a/test/CodeGen/Mips/llvm-ir/udiv.ll
+++ b/test/CodeGen/Mips/llvm-ir/udiv.ll
@@ -1,36 +1,38 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=mips -mcpu=mips2 -relocation-model=pic \
-; RUN:   | FileCheck %s -check-prefixes=GP32,GP32R0R1
+; RUN:   -mips-jalr-reloc=false | FileCheck %s -check-prefixes=GP32,GP32R0R1
 ; RUN: llc < %s -mtriple=mips -mcpu=mips32 -relocation-model=pic \
-; RUN:   | FileCheck %s -check-prefixes=GP32,GP32R0R1
+; RUN:   -mips-jalr-reloc=false | FileCheck %s -check-prefixes=GP32,GP32R0R1
 ; RUN: llc < %s -mtriple=mips -mcpu=mips32r2 -relocation-model=pic \
-; RUN:   | FileCheck %s -check-prefixes=GP32,GP32R2R5
+; RUN:   -mips-jalr-reloc=false | FileCheck %s -check-prefixes=GP32,GP32R2R5
 ; RUN: llc < %s -mtriple=mips -mcpu=mips32r3 -relocation-model=pic \
-; RUN:   | FileCheck %s -check-prefixes=GP32,GP32R2R5
+; RUN:   -mips-jalr-reloc=false | FileCheck %s -check-prefixes=GP32,GP32R2R5
 ; RUN: llc < %s -mtriple=mips -mcpu=mips32r5 -relocation-model=pic \
-; RUN:   | FileCheck %s -check-prefixes=GP32,GP32R2R5
+; RUN:   -mips-jalr-reloc=false | FileCheck %s -check-prefixes=GP32,GP32R2R5
 ; RUN: llc < %s -mtriple=mips -mcpu=mips32r6 -relocation-model=pic \
-; RUN:   | FileCheck %s -check-prefix=GP32R6
+; RUN:   -mips-jalr-reloc=false | FileCheck %s -check-prefix=GP32R6
 
 ; RUN: llc < %s -mtriple=mips64 -mcpu=mips3 -relocation-model=pic \
-; RUN:   | FileCheck %s -check-prefixes=GP64,GP64R0R1
+; RUN:   -mips-jalr-reloc=false | FileCheck %s -check-prefixes=GP64,GP64R0R1
 ; RUN: llc < %s -mtriple=mips64 -mcpu=mips4 -relocation-model=pic \
-; RUN:   | FileCheck %s -check-prefixes=GP64,GP64R0R1
+; RUN:   -mips-jalr-reloc=false | FileCheck %s -check-prefixes=GP64,GP64R0R1
 ; RUN: llc < %s -mtriple=mips64 -mcpu=mips64 -relocation-model=pic \
-; RUN:   | FileCheck %s -check-prefixes=GP64,GP64R0R2
+; RUN:   -mips-jalr-reloc=false | FileCheck %s -check-prefixes=GP64,GP64R0R2
 ; RUN: llc < %s -mtriple=mips64 -mcpu=mips64r2 -relocation-model=pic \
-; RUN:   | FileCheck %s -check-prefixes=GP64,GP64R2R5
+; RUN:   -mips-jalr-reloc=false | FileCheck %s -check-prefixes=GP64,GP64R2R5
 ; RUN: llc < %s -mtriple=mips64 -mcpu=mips64r3 -relocation-model=pic \
-; RUN:   | FileCheck %s -check-prefixes=GP64,GP64R2R5
+; RUN:   -mips-jalr-reloc=false | FileCheck %s -check-prefixes=GP64,GP64R2R5
 ; RUN: llc < %s -mtriple=mips64 -mcpu=mips64r5 -relocation-model=pic \
-; RUN:   | FileCheck %s -check-prefixes=GP64,GP64R2R5
+; RUN:   -mips-jalr-reloc=false | FileCheck %s -check-prefixes=GP64,GP64R2R5
 ; RUN: llc < %s -mtriple=mips64 -mcpu=mips64r6 -relocation-model=pic \
-; RUN:   | FileCheck %s -check-prefix=GP64R6
+; RUN:   -mips-jalr-reloc=false | FileCheck %s -check-prefix=GP64R6
 
-; RUN: llc < %s -mtriple=mips -mcpu=mips32r3 -mattr=+micromips -relocation-model=pic \
-; RUN:   | FileCheck %s -check-prefix=MMR3
-; RUN: llc < %s -mtriple=mips -mcpu=mips32r6 -mattr=+micromips -relocation-model=pic \
-; RUN:   | FileCheck %s -check-prefix=MMR6
+; RUN: llc < %s -mtriple=mips -mcpu=mips32r3 -mattr=+micromips \
+; RUN:   -relocation-model=pic -mips-jalr-reloc=false | \
+; RUN:   FileCheck %s -check-prefix=MMR3
+; RUN: llc < %s -mtriple=mips -mcpu=mips32r6 -mattr=+micromips \
+; RUN:   -relocation-model=pic -mips-jalr-reloc=false | \
+; RUN:   FileCheck %s -check-prefix=MMR6
 
 define zeroext i1 @udiv_i1(i1 zeroext %a, i1 zeroext %b) {
 ; GP32-LABEL: udiv_i1:
diff --git a/test/CodeGen/Mips/llvm-ir/urem.ll b/test/CodeGen/Mips/llvm-ir/urem.ll
index b744f706cbf9ceb86ae2edc0b106cce8785ae6e9..4105d67da6f1acceebe69edb6b1283671a6dd522 100644
--- a/test/CodeGen/Mips/llvm-ir/urem.ll
+++ b/test/CodeGen/Mips/llvm-ir/urem.ll
@@ -1,36 +1,38 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=mips -mcpu=mips2 -relocation-model=pic \
-; RUN:   | FileCheck %s -check-prefixes=GP32,GP32R0R2
+; RUN:   -mips-jalr-reloc=false | FileCheck %s -check-prefixes=GP32,GP32R0R2
 ; RUN: llc < %s -mtriple=mips -mcpu=mips32 -relocation-model=pic \
-; RUN:   | FileCheck %s -check-prefixes=GP32,GP32R0R2
+; RUN:   -mips-jalr-reloc=false | FileCheck %s -check-prefixes=GP32,GP32R0R2
 ; RUN: llc < %s -mtriple=mips -mcpu=mips32r2 -relocation-model=pic \
-; RUN:   | FileCheck %s -check-prefixes=GP32,GP32R2R5
+; RUN:   -mips-jalr-reloc=false | FileCheck %s -check-prefixes=GP32,GP32R2R5
 ; RUN: llc < %s -mtriple=mips -mcpu=mips32r3 -relocation-model=pic \
-; RUN:   | FileCheck %s -check-prefixes=GP32,GP32R2R5
+; RUN:   -mips-jalr-reloc=false | FileCheck %s -check-prefixes=GP32,GP32R2R5
 ; RUN: llc < %s -mtriple=mips -mcpu=mips32r5 -relocation-model=pic \
-; RUN:   | FileCheck %s -check-prefixes=GP32,GP32R2R5
+; RUN:   -mips-jalr-reloc=false | FileCheck %s -check-prefixes=GP32,GP32R2R5
 ; RUN: llc < %s -mtriple=mips -mcpu=mips32r6 -relocation-model=pic \
-; RUN:   | FileCheck %s -check-prefix=GP32R6
+; RUN:   -mips-jalr-reloc=false | FileCheck %s -check-prefix=GP32R6
 
 ; RUN: llc < %s -mtriple=mips64 -mcpu=mips3 -relocation-model=pic \
-; RUN:   | FileCheck %s -check-prefixes=GP64,GP64R0R1
+; RUN:   -mips-jalr-reloc=false | FileCheck %s -check-prefixes=GP64,GP64R0R1
 ; RUN: llc < %s -mtriple=mips64 -mcpu=mips4 -relocation-model=pic \
-; RUN:   | FileCheck %s -check-prefixes=GP64,GP64R0R1
+; RUN:   -mips-jalr-reloc=false | FileCheck %s -check-prefixes=GP64,GP64R0R1
 ; RUN: llc < %s -mtriple=mips64 -mcpu=mips64 -relocation-model=pic \
-; RUN:   | FileCheck %s -check-prefixes=GP64,GP64R0R1
+; RUN:   -mips-jalr-reloc=false | FileCheck %s -check-prefixes=GP64,GP64R0R1
 ; RUN: llc < %s -mtriple=mips64 -mcpu=mips64r2 -relocation-model=pic \
-; RUN:   | FileCheck %s -check-prefixes=GP64,GP64R2R5
+; RUN:   -mips-jalr-reloc=false | FileCheck %s -check-prefixes=GP64,GP64R2R5
 ; RUN: llc < %s -mtriple=mips64 -mcpu=mips64r3 -relocation-model=pic \
-; RUN:   | FileCheck %s -check-prefixes=GP64,GP64R2R5
+; RUN:   -mips-jalr-reloc=false | FileCheck %s -check-prefixes=GP64,GP64R2R5
 ; RUN: llc < %s -mtriple=mips64 -mcpu=mips64r5 -relocation-model=pic \
-; RUN:   | FileCheck %s -check-prefixes=GP64,GP64R2R5
+; RUN:   -mips-jalr-reloc=false | FileCheck %s -check-prefixes=GP64,GP64R2R5
 ; RUN: llc < %s -mtriple=mips64 -mcpu=mips64r6 -relocation-model=pic \
-; RUN:   | FileCheck %s -check-prefix=GP64R6
+; RUN:   -mips-jalr-reloc=false | FileCheck %s -check-prefix=GP64R6
 
-; RUN: llc < %s -mtriple=mips -mcpu=mips32r3 -mattr=+micromips -relocation-model=pic \
-; RUN:   | FileCheck %s -check-prefix=MMR3
-; RUN: llc < %s -mtriple=mips -mcpu=mips32r6 -mattr=+micromips -relocation-model=pic \
-; RUN:   | FileCheck %s -check-prefix=MMR6
+; RUN: llc < %s -mtriple=mips -mcpu=mips32r3 -mattr=+micromips \
+; RUN:   -relocation-model=pic -mips-jalr-reloc=false | \
+; RUN:   FileCheck %s -check-prefix=MMR3
+; RUN: llc < %s -mtriple=mips -mcpu=mips32r6 -mattr=+micromips \
+; RUN:   -relocation-model=pic -mips-jalr-reloc=false | \
+; RUN:   FileCheck %s -check-prefix=MMR6
 
 define signext i1 @urem_i1(i1 signext %a, i1 signext %b) {
 ; GP32-LABEL: urem_i1:
diff --git a/test/CodeGen/Mips/long-call-attr.ll b/test/CodeGen/Mips/long-call-attr.ll
index 5b6ba94aaa3567616d7d82a3cd69ff62e55a7f94..beda290a9725b45eec3644b32b705055e0f125f5 100644
--- a/test/CodeGen/Mips/long-call-attr.ll
+++ b/test/CodeGen/Mips/long-call-attr.ll
@@ -1,11 +1,11 @@
 ; RUN: llc -march=mips -target-abi o32 --mattr=+long-calls,+noabicalls < %s \
-; RUN:   | FileCheck -check-prefix=O32 %s
+; RUN:   -mips-jalr-reloc=false | FileCheck -check-prefix=O32 %s
 ; RUN: llc -march=mips -target-abi o32 --mattr=-long-calls,+noabicalls < %s \
-; RUN:   | FileCheck -check-prefix=O32 %s
+; RUN:   -mips-jalr-reloc=false | FileCheck -check-prefix=O32 %s
 ; RUN: llc -march=mips64 -target-abi n64 --mattr=+long-calls,+noabicalls < %s \
-; RUN:   | FileCheck -check-prefix=N64 %s
+; RUN:   -mips-jalr-reloc=false | FileCheck -check-prefix=N64 %s
 ; RUN: llc -march=mips64 -target-abi n64 --mattr=-long-calls,+noabicalls < %s \
-; RUN:   | FileCheck -check-prefix=N64 %s
+; RUN:   -mips-jalr-reloc=false | FileCheck -check-prefix=N64 %s
 
 declare void @far() #0
 
diff --git a/test/CodeGen/Mips/long-call-mcount.ll b/test/CodeGen/Mips/long-call-mcount.ll
index 70a4410d060ba32c23f2683b7a2b220757f9776b..580f452526f7369f3e9f17d09c2a6e88b71b5574 100644
--- a/test/CodeGen/Mips/long-call-mcount.ll
+++ b/test/CodeGen/Mips/long-call-mcount.ll
@@ -1,8 +1,8 @@
 ; Check call to mcount in case of long/short call options.
 ; RUN: llc -march=mips -target-abi o32 --mattr=+long-calls,+noabicalls < %s \
-; RUN:   | FileCheck -check-prefixes=CHECK,LONG %s
+; RUN:   -mips-jalr-reloc=false | FileCheck -check-prefixes=CHECK,LONG %s
 ; RUN: llc -march=mips -target-abi o32 --mattr=-long-calls,+noabicalls < %s \
-; RUN:   | FileCheck -check-prefixes=CHECK,SHORT %s
+; RUN:   -mips-jalr-reloc=false | FileCheck -check-prefixes=CHECK,SHORT %s
 
 ; Function Attrs: noinline nounwind optnone
 define void @foo() #0 {
diff --git a/test/CodeGen/Mips/micromips-b-range.ll b/test/CodeGen/Mips/micromips-b-range.ll
index f761d1c31d320996ba8212f1d5e2f35ccc7305de..27a0db545f743fd9f6a0060122905cb1b7ea5a49 100644
--- a/test/CodeGen/Mips/micromips-b-range.ll
+++ b/test/CodeGen/Mips/micromips-b-range.ll
@@ -13,7 +13,7 @@
 ; CHECK-NEXT:    1e:	fb fd 00 00 	sw	$ra, 0($sp)
 ; CHECK-NEXT:    22:	41 a1 00 01 	lui	$1, 1
 ; CHECK-NEXT:    26:	40 60 00 02 	bal	8 <foo+0x2e>
-; CHECK-NEXT:    2a:	30 21 04 68 	addiu	$1, $1, 1128
+; CHECK-NEXT:    2a:	30 21 04 69 	addiu	$1, $1, 1129
 ; CHECK-NEXT:    2e:	00 3f 09 50 	addu	$1, $ra, $1
 ; CHECK-NEXT:    32:	ff fd 00 00 	lw	$ra, 0($sp)
 ; CHECK-NEXT:    36:	00 01 0f 3c 	jr	$1
@@ -27,7 +27,7 @@
 ; CHECK-NEXT:    56:	fb fd 00 00 	sw	$ra, 0($sp)
 ; CHECK-NEXT:    5a:	41 a1 00 01 	lui	$1, 1
 ; CHECK-NEXT:    5e:	40 60 00 02 	bal	8 <foo+0x66>
-; CHECK-NEXT:    62:	30 21 04 5c 	addiu	$1, $1, 1116
+; CHECK-NEXT:    62:	30 21 04 5d 	addiu	$1, $1, 1117
 ; CHECK-NEXT:    66:	00 3f 09 50 	addu	$1, $ra, $1
 ; CHECK-NEXT:    6a:	ff fd 00 00 	lw	$ra, 0($sp)
 ; CHECK-NEXT:    6e:	00 01 0f 3c 	jr	$1
@@ -39,20 +39,19 @@
 ; CHECK-NEXT:    86:	fb fd 00 00 	sw	$ra, 0($sp)
 ; CHECK-NEXT:    8a:	41 a1 00 01 	lui	$1, 1
 ; CHECK-NEXT:    8e:	40 60 00 02 	bal	8 <foo+0x96>
-; CHECK-NEXT:    92:	30 21 04 2c 	addiu	$1, $1, 1068
+; CHECK-NEXT:    92:	30 21 04 2d 	addiu	$1, $1, 1069
 ; CHECK-NEXT:    96:	00 3f 09 50 	addu	$1, $ra, $1
 ; CHECK-NEXT:    9a:	ff fd 00 00 	lw	$ra, 0($sp)
 ; CHECK-NEXT:    9e:	00 01 0f 3c 	jr	$1
 ; CHECK-NEXT:    a2:	33 bd 00 08 	addiu	$sp, $sp, 8
-
-; CHECK:      10466:	00 00 00 00 	nop
+; CHECK:                ...
 ; CHECK-NEXT: 1046a:	94 00 00 02 	b	8 <foo+0x10472>
 ; CHECK-NEXT: 1046e:	00 00 00 00 	nop
 ; CHECK-NEXT: 10472:	33 bd ff f8 	addiu	$sp, $sp, -8
 ; CHECK-NEXT: 10476:	fb fd 00 00 	sw	$ra, 0($sp)
 ; CHECK-NEXT: 1047a:	41 a1 00 01 	lui	$1, 1
 ; CHECK-NEXT: 1047e:	40 60 00 02 	bal	8 <foo+0x10486>
-; CHECK-NEXT: 10482:	30 21 04 00 	addiu	$1, $1, 1024
+; CHECK-NEXT: 10482:	30 21 04 01 	addiu	$1, $1, 1025
 ; CHECK-NEXT: 10486:	00 3f 09 50 	addu	$1, $ra, $1
 ; CHECK-NEXT: 1048a:	ff fd 00 00 	lw	$ra, 0($sp)
 ; CHECK-NEXT: 1048e:	00 01 0f 3c 	jr	$1
diff --git a/test/CodeGen/Mips/msa/f16-llvm-ir.ll b/test/CodeGen/Mips/msa/f16-llvm-ir.ll
index 9105e9249d4f021619229b719460500451e37800..3c1aa8b4792bab9b5cf454f39c643fa6e8758bf7 100644
--- a/test/CodeGen/Mips/msa/f16-llvm-ir.ll
+++ b/test/CodeGen/Mips/msa/f16-llvm-ir.ll
@@ -1,22 +1,22 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -relocation-model=pic -mtriple=mipsel-- -mcpu=mips32r5 \
-; RUN:     -mattr=+fp64,+msa -verify-machineinstrs < %s | FileCheck %s \
+; RUN: llc -relocation-model=pic -mtriple=mipsel-- -mcpu=mips32r5 -mattr=+abs2008 \
+; RUN:     -mattr=+fp64,+msa -verify-machineinstrs -mips-jalr-reloc=false < %s | FileCheck %s \
 ; RUN:     --check-prefixes=ALL,MIPS32,MIPSR5,MIPS32-O32,MIPS32R5-O32
-; RUN: llc -relocation-model=pic -mtriple=mips64el-- -mcpu=mips64r5 \
-; RUN:     -mattr=+fp64,+msa -verify-machineinstrs -target-abi n32 < %s | FileCheck %s \
+; RUN: llc -relocation-model=pic -mtriple=mips64el-- -mcpu=mips64r5 -mattr=+abs2008 \
+; RUN:     -mattr=+fp64,+msa -verify-machineinstrs -target-abi n32 -mips-jalr-reloc=false < %s | FileCheck %s \
 ; RUN:     --check-prefixes=ALL,MIPS64,MIPSR5,MIPS64-N32,MIPS64R5-N32
-; RUN: llc -relocation-model=pic -mtriple=mips64el-- -mcpu=mips64r5 \
-; RUN:     -mattr=+fp64,+msa -verify-machineinstrs -target-abi n64 < %s | FileCheck %s \
+; RUN: llc -relocation-model=pic -mtriple=mips64el-- -mcpu=mips64r5 -mattr=+abs2008 \
+; RUN:     -mattr=+fp64,+msa -verify-machineinstrs -target-abi n64 -mips-jalr-reloc=false < %s | FileCheck %s \
 ; RUN:     --check-prefixes=ALL,MIPS64,MIPSR5,MIPS64-N64,MIPS64R5-N64
 
 ; RUN: llc -relocation-model=pic -mtriple=mipsel-- -mcpu=mips32r6 \
-; RUN:     -mattr=+fp64,+msa -verify-machineinstrs < %s | FileCheck %s \
+; RUN:     -mattr=+fp64,+msa -verify-machineinstrs -mips-jalr-reloc=false < %s | FileCheck %s \
 ; RUN:     --check-prefixes=ALL,MIPS32,MIPSR6,MIPSR6-O32
 ; RUN: llc -relocation-model=pic -mtriple=mips64el-- -mcpu=mips64r6 \
-; RUN:     -mattr=+fp64,+msa -verify-machineinstrs -target-abi n32 < %s | FileCheck %s \
+; RUN:     -mattr=+fp64,+msa -verify-machineinstrs -target-abi n32 -mips-jalr-reloc=false < %s | FileCheck %s \
 ; RUN:     --check-prefixes=ALL,MIPS64,MIPSR6,MIPS64-N32,MIPSR6-N32
 ; RUN: llc -relocation-model=pic -mtriple=mips64el-- -mcpu=mips64r6 \
-; RUN:     -mattr=+fp64,+msa -verify-machineinstrs -target-abi n64 < %s | FileCheck %s \
+; RUN:     -mattr=+fp64,+msa -verify-machineinstrs -target-abi n64 -mips-jalr-reloc=false < %s | FileCheck %s \
 ; RUN:     --check-prefixes=ALL,MIPS64,MIPSR6,MIPS64-N64,MIPSR6-N64
 
 
diff --git a/test/CodeGen/Mips/o32_cc_byval.ll b/test/CodeGen/Mips/o32_cc_byval.ll
index 19eb80b79bafe88b6da98002d8e9ac0b9b2bb6eb..d9951ebeaf3a9925c7f07f182064417c61d953b8 100644
--- a/test/CodeGen/Mips/o32_cc_byval.ll
+++ b/test/CodeGen/Mips/o32_cc_byval.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=mipsel-unknown-linux-gnu -relocation-model=pic < %s | FileCheck %s
+; RUN: llc -mtriple=mipsel-unknown-linux-gnu -relocation-model=pic \
+; RUN:     -mips-jalr-reloc=false < %s | FileCheck %s
 
 %0 = type { i8, i16, i32, i64, double, i32, [4 x i8] }
 %struct.S1 = type { i8, i16, i32, i64, double, i32 }
diff --git a/test/CodeGen/Mips/reloc-jalr.ll b/test/CodeGen/Mips/reloc-jalr.ll
new file mode 100644
index 0000000000000000000000000000000000000000..f8fd903110045d8a9c6b9b9b718edefd64ab1bd9
--- /dev/null
+++ b/test/CodeGen/Mips/reloc-jalr.ll
@@ -0,0 +1,154 @@
+; RUN: llc -mtriple=mips-linux-gnu -relocation-model=pic -mips-tail-calls=1 \
+; RUN:     -O2 < %s | \
+; RUN:     FileCheck %s -check-prefixes=ALL,JALR-32R2,TAILCALL-32R2
+
+; RUN: llc -mtriple=mips64-linux-gnu -relocation-model=pic -mips-tail-calls=1 \
+; RUN:     -O2 < %s | \
+; RUN:     FileCheck %s -check-prefixes=ALL,JALR-64R2,TAILCALL-64R2
+
+; RUN: llc -mtriple=mips-linux-gnu -relocation-model=pic -mips-tail-calls=1 \
+; RUN:     -O2 -mcpu=mips32r6 -mips-compact-branches=always < %s | \
+; RUN:     FileCheck %s -check-prefixes=ALL,JALR-32R6,TAILCALL-32R6
+
+; RUN: llc -mtriple=mips64-linux-gnu -relocation-model=pic -mips-tail-calls=1 \
+; RUN:     -O2 -mcpu=mips64r6 -mips-compact-branches=always < %s | \
+; RUN:     FileCheck %s -check-prefixes=ALL,JALR-64R6,TAILCALL-64R6
+
+; RUN: llc -mtriple=mips-linux-gnu -relocation-model=pic -mips-tail-calls=1 \
+; RUN:     -O2 -mcpu=mips32r6 -mips-compact-branches=never < %s | \
+; RUN:     FileCheck %s -check-prefixes=ALL,JALR-32R2,TAILCALL-32R2
+
+; RUN: llc -mtriple=mips64-linux-gnu -relocation-model=pic -mips-tail-calls=1 \
+; RUN:     -O2 -mcpu=mips64r6 -mips-compact-branches=never < %s | \
+; RUN:     FileCheck %s -check-prefixes=ALL,JALR-64R2,TAILCALL-64R2
+
+; RUN: llc -mtriple=mips-linux-gnu -relocation-model=pic -mips-tail-calls=1 \
+; RUN:     -O2 -mattr=+micromips -mcpu=mips32r2 < %s | \
+; RUN:     FileCheck %s -check-prefixes=ALL,JALR-MM,TAILCALL-MM
+
+; RUN: llc -mtriple=mips-linux-gnu -relocation-model=pic -mips-tail-calls=1 \
+; RUN:     -O2 -mattr=+micromips -mcpu=mips32r6 < %s | \
+; RUN:     FileCheck %s -check-prefixes=ALL,JALR-MM
+
+; RUN: llc -mtriple=mips-linux-gnu -relocation-model=pic \
+; RUN:     -O0 < %s | FileCheck %s -check-prefixes=ALL,JALR-32R2
+
+; RUN: llc -mtriple=mips64-linux-gnu -relocation-model=pic \
+; RUN:     -O0 < %s | FileCheck %s -check-prefixes=ALL,JALR-64R2
+
+; RUN: llc -mtriple=mips-linux-gnu -relocation-model=pic \
+; RUN:     -O0 -mcpu=mips32r6 -mips-compact-branches=always < %s | \
+; RUN:     FileCheck %s -check-prefixes=ALL,JALR-32R6
+
+; RUN: llc -mtriple=mips64-linux-gnu -relocation-model=pic \
+; RUN:     -O0 -mcpu=mips64r6 -mips-compact-branches=always < %s | \
+; RUN:     FileCheck %s -check-prefixes=ALL,JALR-64R6
+
+; RUN: llc -mtriple=mips-linux-gnu -relocation-model=pic \
+; RUN:     -O0 -mcpu=mips32r6 -mips-compact-branches=never < %s | \
+; RUN:     FileCheck %s -check-prefixes=ALL,JALR-32R2
+
+; RUN: llc -mtriple=mips64-linux-gnu -relocation-model=pic \
+; RUN:     -O0 -mcpu=mips64r6 -mips-compact-branches=never < %s | \
+; RUN:     FileCheck %s -check-prefixes=ALL,JALR-64R2
+
+; RUN: llc -mtriple=mips-linux-gnu -relocation-model=pic \
+; RUN:     -O0 -mattr=+micromips -mcpu=mips32r2 < %s | \
+; RUN:     FileCheck %s -check-prefixes=ALL,JALR-MM
+
+; RUN: llc -mtriple=mips-linux-gnu -relocation-model=pic \
+; RUN:     -O0 -mattr=+micromips -mcpu=mips32r6 < %s | \
+; RUN:     FileCheck %s -check-prefixes=ALL,JALR-MM
+
+; RUN: llc -mtriple=mips-linux-gnu -relocation-model=pic -mips-tail-calls=1 \
+; RUN:     -O2 -mips-jalr-reloc=false < %s | \
+; RUN:     FileCheck %s -check-prefixes=ALL,NORELOC
+
+; RUN: llc -mtriple=mips-linux-gnu -relocation-model=static -mips-tail-calls=1 \
+; RUN:     -O2 < %s | \
+; RUN:     FileCheck %s -check-prefixes=ALL,NORELOC
+
+; RUN: llc -mtriple=mips-linux-gnu -relocation-model=pic -mips-tail-calls=1 \
+; RUN:     -O0 -mips-jalr-reloc=false < %s | \
+; RUN:     FileCheck %s -check-prefixes=ALL,NORELOC
+
+; RUN: llc -mtriple=mips-linux-gnu -relocation-model=static -mips-tail-calls=1 \
+; RUN:     -O0 < %s | \
+; RUN:     FileCheck %s -check-prefixes=ALL,NORELOC
+
+; RUN: llc -mtriple=mips64-linux-gnu -relocation-model=pic -mips-tail-calls=1 \
+; RUN:     -O2 -mips-jalr-reloc=false < %s | \
+; RUN:     FileCheck %s -check-prefixes=ALL,NORELOC
+
+; RUN: llc -mtriple=mips64-linux-gnu -mips-tail-calls=1 \
+; RUN:     -O2 -relocation-model=static < %s | \
+; RUN:     FileCheck %s -check-prefixes=ALL,NORELOC
+
+; RUN: llc -mtriple=mips64-linux-gnu -relocation-model=pic \
+; RUN:     -O0 -mips-jalr-reloc=false < %s | \
+; RUN:     FileCheck %s -check-prefixes=ALL,NORELOC
+
+; RUN: llc -mtriple=mips64-linux-gnu -relocation-model=static \
+; RUN:     -O0 < %s | \
+; RUN:     FileCheck %s -check-prefixes=ALL,NORELOC
+
+define internal void @foo() noinline {
+entry:
+  ret void
+}
+
+define void @checkCall() {
+entry:
+; ALL-LABEL: checkCall:
+  call void @foo()
+;	JALR-32R2: 	.reloc ([[TMPLABEL:.*]]), R_MIPS_JALR, foo
+; JALR-32R2-NEXT: [[TMPLABEL]]:
+;	JALR-32R2-NEXT: 	jalr	$25
+
+;	JALR-64R2: 	.reloc [[TMPLABEL:.*]], R_MIPS_JALR, foo
+; JALR-64R2-NEXT: [[TMPLABEL]]:
+;	JALR-64R2-NEXT: 	jalr	$25
+
+;	JALR-MM: 	.reloc ([[TMPLABEL:.*]]), R_MICROMIPS_JALR, foo
+; JALR-MM-NEXT: [[TMPLABEL]]:
+;	JALR-MM-NEXT: 	jalr	$25
+
+;	JALR-32R6: 	.reloc ([[TMPLABEL:.*]]), R_MIPS_JALR, foo
+; JALR-32R6-NEXT: [[TMPLABEL]]:
+;	JALR-32R6-NEXT: 	jalrc	$25
+
+;	JALR-64R6: 	.reloc [[TMPLABEL:.*]], R_MIPS_JALR, foo
+; JALR-64R6-NEXT: [[TMPLABEL]]:
+;	JALR-64R6-NEXT: 	jalrc	$25
+
+; NORELOC-NOT: R_MIPS_JALR
+ ret void
+}
+
+define void @checkTailCall() {
+entry:
+; ALL-LABEL: checkTailCall:
+  tail call void @foo()
+;	TAILCALL-32R2: 	.reloc ([[TMPLABEL:.*]]), R_MIPS_JALR, foo
+; TAILCALL-32R2-NEXT: [[TMPLABEL]]:
+;	TAILCALL-32R2-NEXT: 	jr	$25
+
+;	TAILCALL-64R2: 	.reloc [[TMPLABEL:.*]], R_MIPS_JALR, foo
+; TAILCALL-64R2-NEXT: [[TMPLABEL]]:
+;	TAILCALL-64R2-NEXT: 	jr	$25
+
+;	TAILCALL-MM: 	.reloc ([[TMPLABEL:.*]]), R_MICROMIPS_JALR, foo
+; TAILCALL-MM-NEXT: [[TMPLABEL]]:
+;	TAILCALL-MM-NEXT: 	jrc	$25
+
+;	TAILCALL-32R6: 	.reloc ([[TMPLABEL:.*]]), R_MIPS_JALR, foo
+; TAILCALL-32R6-NEXT: [[TMPLABEL]]:
+;	TAILCALL-32R6-NEXT: 	jrc	$25
+
+;	TAILCALL-64R6: 	.reloc [[TMPLABEL:.*]], R_MIPS_JALR, foo
+; TAILCALL-64R6-NEXT: [[TMPLABEL]]:
+;	TAILCALL-64R6-NEXT: 	jrc	$25
+
+; NORELOC-NOT: R_MIPS_JALR
+  ret void
+}
diff --git a/test/CodeGen/Mips/shrink-wrapping.ll b/test/CodeGen/Mips/shrink-wrapping.ll
index 54ae8699d1c19c32084c958718772b816da17b47..b08d2f1b64678f517c071ba669aa1f3137450c50 100644
--- a/test/CodeGen/Mips/shrink-wrapping.ll
+++ b/test/CodeGen/Mips/shrink-wrapping.ll
@@ -9,11 +9,11 @@
 ; RUN:   FileCheck %s -check-prefix=NO-SHRINK-WRAP-STATIC
 
 ; RUN: llc -mtriple=mips-unknown-linux-gnu -enable-shrink-wrap=true \
-; RUN:   -relocation-model=pic < %s | \
+; RUN:   -relocation-model=pic -mips-jalr-reloc=false < %s | \
 ; RUN:   FileCheck %s -check-prefix=SHRINK-WRAP-PIC
 
 ; RUN: llc -mtriple=mips-unknown-linux-gnu -enable-shrink-wrap=false \
-; RUN:   -relocation-model=pic < %s | \
+; RUN:   -relocation-model=pic -mips-jalr-reloc=false < %s | \
 ; RUN:   FileCheck %s -check-prefix=NO-SHRINK-WRAP-PIC
 
 ; RUN: llc -mtriple=mips64-unknown-linux-gnu -enable-shrink-wrap=true \
@@ -25,11 +25,11 @@
 ; RUN:   FileCheck %s -check-prefix=NO-SHRINK-WRAP-64-STATIC
 
 ; RUN: llc -mtriple=mips64-unknown-linux-gnu -enable-shrink-wrap=true \
-; RUN:   -relocation-model=pic < %s | \
+; RUN:   -relocation-model=pic -mips-jalr-reloc=false < %s | \
 ; RUN:   FileCheck %s -check-prefix=SHRINK-WRAP-64-PIC
 
 ; RUN: llc -mtriple=mips64-unknown-linux-gnu -enable-shrink-wrap=false \
-; RUN:   -relocation-model=pic < %s | \
+; RUN:   -relocation-model=pic -mips-jalr-reloc=false < %s | \
 ; RUN:   FileCheck %s -check-prefix=NO-SHRINK-WRAP-64-PIC
 
 declare void @f(i32 signext)
diff --git a/test/CodeGen/NVPTX/calls-with-phi.ll b/test/CodeGen/NVPTX/calls-with-phi.ll
new file mode 100644
index 0000000000000000000000000000000000000000..6e010ea9adc34c820f1ff5d7315b29c3f3ab55e1
--- /dev/null
+++ b/test/CodeGen/NVPTX/calls-with-phi.ll
@@ -0,0 +1,22 @@
+; RUN: llc < %s -march=nvptx 2>&1 | FileCheck %s
+; Make sure the example doesn't crash with segfault
+
+; CHECK: .visible .func ({{.*}}) loop
+define i32 @loop(i32, i32) {
+entry:
+  br label %loop
+
+loop:
+  %i = phi i32 [ %0, %entry ], [ %res, %loop ]
+  %res = call i32 @div(i32 %i, i32 %1)
+
+  %exitcond = icmp eq i32 %res, %0
+  br i1 %exitcond, label %exit, label %loop
+
+exit:
+  ret i32 %res
+}
+
+define i32 @div(i32, i32) {
+  ret i32 0
+}
diff --git a/test/CodeGen/NVPTX/f16-instructions.ll b/test/CodeGen/NVPTX/f16-instructions.ll
index 4548d5583be55ce90ca712c56dc8ad4664a394b4..7788adc8698934d87f5d9c590d9680f9c2910857 100644
--- a/test/CodeGen/NVPTX/f16-instructions.ll
+++ b/test/CodeGen/NVPTX/f16-instructions.ll
@@ -1,20 +1,20 @@
 ; ## Full FP16 support enabled by default.
 ; RUN: llc < %s -mtriple=nvptx64-nvidia-cuda -mcpu=sm_53 -asm-verbose=false \
-; RUN:          -O0 -disable-post-ra -disable-fp-elim -verify-machineinstrs \
+; RUN:          -O0 -disable-post-ra -frame-pointer=all -verify-machineinstrs \
 ; RUN: | FileCheck -check-prefixes CHECK,CHECK-NOFTZ,CHECK-F16,CHECK-F16-NOFTZ %s
 ; ## Full FP16 with FTZ
 ; RUN: llc < %s -mtriple=nvptx64-nvidia-cuda -mcpu=sm_53 -asm-verbose=false \
-; RUN:          -O0 -disable-post-ra -disable-fp-elim -verify-machineinstrs \
+; RUN:          -O0 -disable-post-ra -frame-pointer=all -verify-machineinstrs \
 ; RUN:          -nvptx-f32ftz \
 ; RUN: | FileCheck -check-prefixes CHECK,CHECK-F16,CHECK-F16-FTZ %s
 ; ## FP16 support explicitly disabled.
 ; RUN: llc < %s -mtriple=nvptx64-nvidia-cuda -mcpu=sm_53 -asm-verbose=false \
-; RUN:          -O0 -disable-post-ra -disable-fp-elim --nvptx-no-f16-math \
+; RUN:          -O0 -disable-post-ra -frame-pointer=all --nvptx-no-f16-math \
 ; RUN:           -verify-machineinstrs \
 ; RUN: | FileCheck -check-prefixes CHECK,CHECK-NOFTZ,CHECK-NOF16 %s
 ; ## FP16 is not supported by hardware.
 ; RUN: llc < %s -O0 -mtriple=nvptx64-nvidia-cuda -mcpu=sm_52 -asm-verbose=false \
-; RUN:          -disable-post-ra -disable-fp-elim -verify-machineinstrs \
+; RUN:          -disable-post-ra -frame-pointer=all -verify-machineinstrs \
 ; RUN: | FileCheck -check-prefixes CHECK,CHECK-NOFTZ,CHECK-NOF16 %s
 
 target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
diff --git a/test/CodeGen/NVPTX/f16x2-instructions.ll b/test/CodeGen/NVPTX/f16x2-instructions.ll
index 77ce45a9dc3a5a798622343e0f0b81ebdbfc6870..a8996815af4134ebc3aa76859b1634d09c269a6e 100644
--- a/test/CodeGen/NVPTX/f16x2-instructions.ll
+++ b/test/CodeGen/NVPTX/f16x2-instructions.ll
@@ -1,15 +1,15 @@
 ; ## Full FP16 support enabled by default.
 ; RUN: llc < %s -mtriple=nvptx64-nvidia-cuda -mcpu=sm_53 -asm-verbose=false \
-; RUN:          -O0 -disable-post-ra -disable-fp-elim -verify-machineinstrs \
+; RUN:          -O0 -disable-post-ra -frame-pointer=all -verify-machineinstrs \
 ; RUN: | FileCheck -allow-deprecated-dag-overlap -check-prefixes CHECK,CHECK-F16 %s
 ; ## FP16 support explicitly disabled.
 ; RUN: llc < %s -mtriple=nvptx64-nvidia-cuda -mcpu=sm_53 -asm-verbose=false \
-; RUN:          -O0 -disable-post-ra -disable-fp-elim --nvptx-no-f16-math \
+; RUN:          -O0 -disable-post-ra -frame-pointer=all --nvptx-no-f16-math \
 ; RUN:           -verify-machineinstrs \
 ; RUN: | FileCheck -allow-deprecated-dag-overlap -check-prefixes CHECK,CHECK-NOF16 %s
 ; ## FP16 is not supported by hardware.
 ; RUN: llc < %s -O0 -mtriple=nvptx64-nvidia-cuda -mcpu=sm_52 -asm-verbose=false \
-; RUN:          -disable-post-ra -disable-fp-elim -verify-machineinstrs \
+; RUN:          -disable-post-ra -frame-pointer=all -verify-machineinstrs \
 ; RUN: | FileCheck -allow-deprecated-dag-overlap -check-prefixes CHECK,CHECK-NOF16 %s
 
 target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
diff --git a/test/CodeGen/NVPTX/intrinsics.ll b/test/CodeGen/NVPTX/intrinsics.ll
index 668de8a994bcc8a39c2fd979dfd38c8863a581c9..4abbfcbd5183de80af2e105bd0ef82f3ae2e7ae1 100644
--- a/test/CodeGen/NVPTX/intrinsics.ll
+++ b/test/CodeGen/NVPTX/intrinsics.ll
@@ -94,6 +94,43 @@ define i32 @test_popc16_to_32(i16 %a) {
   ret i32 %zext
 }
 
+; Most of nvvm.read.ptx.sreg.* intrinsics always return the same value and may
+; be CSE'd.
+; CHECK-LABEL: test_tid
+define i32 @test_tid() {
+; CHECK: mov.u32         %r{{.*}}, %tid.x;
+  %a = tail call i32 @llvm.nvvm.read.ptx.sreg.tid.x()
+; CHECK-NOT: mov.u32         %r{{.*}}, %tid.x;
+  %b = tail call i32 @llvm.nvvm.read.ptx.sreg.tid.x()
+  %ret = add i32 %a, %b
+; CHECK: ret
+  ret i32 %ret
+}
+
+; reading clock() or clock64() should not be CSE'd as each read may return
+; different value.
+; CHECK-LABEL: test_clock
+define i32 @test_clock() {
+; CHECK: mov.u32         %r{{.*}}, %clock;
+  %a = tail call i32 @llvm.nvvm.read.ptx.sreg.clock()
+; CHECK: mov.u32         %r{{.*}}, %clock;
+  %b = tail call i32 @llvm.nvvm.read.ptx.sreg.clock()
+  %ret = add i32 %a, %b
+; CHECK: ret
+  ret i32 %ret
+}
+
+; CHECK-LABEL: test_clock64
+define i64 @test_clock64() {
+; CHECK: mov.u64         %r{{.*}}, %clock64;
+  %a = tail call i64 @llvm.nvvm.read.ptx.sreg.clock64()
+; CHECK: mov.u64         %r{{.*}}, %clock64;
+  %b = tail call i64 @llvm.nvvm.read.ptx.sreg.clock64()
+  %ret = add i64 %a, %b
+; CHECK: ret
+  ret i64 %ret
+}
+
 declare float @llvm.fabs.f32(float)
 declare double @llvm.fabs.f64(double)
 declare float @llvm.nvvm.sqrt.f(float)
@@ -103,3 +140,7 @@ declare i64 @llvm.bitreverse.i64(i64)
 declare i16 @llvm.ctpop.i16(i16)
 declare i32 @llvm.ctpop.i32(i32)
 declare i64 @llvm.ctpop.i64(i64)
+
+declare i32 @llvm.nvvm.read.ptx.sreg.tid.x()
+declare i32 @llvm.nvvm.read.ptx.sreg.clock()
+declare i64 @llvm.nvvm.read.ptx.sreg.clock64()
diff --git a/test/CodeGen/NVPTX/ld-st-addrrspace.py b/test/CodeGen/NVPTX/ld-st-addrrspace.py
index c9440666d99b50591a392118ce2b55745fb7b161..7c28620cec46d9cf66f03a869902b68cd693f25d 100644
--- a/test/CodeGen/NVPTX/ld-st-addrrspace.py
+++ b/test/CodeGen/NVPTX/ld-st-addrrspace.py
@@ -5,6 +5,8 @@
 # RUN: llc < %t.ll -march=nvptx64 -mcpu=sm_30 | FileCheck -check-prefixes=CHECK,CHECK_P64 %t.ll
 # RUN: llc < %t.ll -march=nvptx -mcpu=sm_30 | FileCheck -check-prefixes=CHECK,CHECK_P32 %t.ll
 
+from __future__ import print_function
+
 from itertools import product
 from string import Template
 
diff --git a/test/CodeGen/NVPTX/libcall-fulfilled.ll b/test/CodeGen/NVPTX/libcall-fulfilled.ll
new file mode 100644
index 0000000000000000000000000000000000000000..9d6777d59c3e5a708ae31a26de3944ef6a0413ee
--- /dev/null
+++ b/test/CodeGen/NVPTX/libcall-fulfilled.ll
@@ -0,0 +1,31 @@
+; RUN: llc < %s -march=nvptx 2>&1 | FileCheck %s
+; Allow to make libcalls that are defined in the current module
+
+; Underlying libcall declaration
+; CHECK: .visible .func  (.param .align 16 .b8 func_retval0[16]) __umodti3
+
+define i128 @remainder(i128, i128) {
+bb0:
+  ; CHECK:      { // callseq 0, 0
+  ; CHECK:      call.uni (retval0),
+  ; CHECK-NEXT: __umodti3,
+  ; CHECK-NEXT: (
+  ; CHECK-NEXT: param0,
+  ; CHECK-NEXT: param1
+  ; CHECK-NEXT: );
+  ; CHECK-NEXT: ld.param.v2.b64 {%[[REG0:rd[0-9]+]], %[[REG1:rd[0-9]+]]}, [retval0+0];
+  ; CHECK-NEXT: } // callseq 0
+  %a = urem i128 %0, %1
+  br label %bb1
+
+bb1:
+  ; CHECK-NEXT: st.param.v2.b64 [func_retval0+0], {%[[REG0]], %[[REG1]]};
+  ; CHECK-NEXT: ret;
+  ret i128 %a
+}
+
+; Underlying libcall definition
+; CHECK: .visible .func  (.param .align 16 .b8 func_retval0[16]) __umodti3(
+define i128 @__umodti3(i128, i128) {
+  ret i128 0
+}
diff --git a/test/CodeGen/NVPTX/libcall-instruction.ll b/test/CodeGen/NVPTX/libcall-instruction.ll
index 0c2cab7eaa5a1b435b900f91fe1fff03d7e892d5..a40a504c94cb542b245123c23ac8d7142f37a0dc 100644
--- a/test/CodeGen/NVPTX/libcall-instruction.ll
+++ b/test/CodeGen/NVPTX/libcall-instruction.ll
@@ -1,7 +1,7 @@
 ; RUN: not llc < %s -march=nvptx 2>&1 | FileCheck %s
-; used to panic on failed assetion and now fails with a "Cannot select"
+; used to panic on failed assertion and now fails with an "Undefined external symbol"
 
-; CHECK: LLVM ERROR: Cannot select: {{t28|0x[0-9a-f]+}}: i32 = ExternalSymbol'__umodti3'
+; CHECK: LLVM ERROR: Undefined external symbol "__umodti3"
 define hidden i128 @remainder(i128, i128) {
   %3 = urem i128 %0, %1
   ret i128 %3
diff --git a/test/CodeGen/NVPTX/libcall-intrinsic.ll b/test/CodeGen/NVPTX/libcall-intrinsic.ll
new file mode 100644
index 0000000000000000000000000000000000000000..0b5e0224399d17dfb0f0b54d0ae8c971a3572580
--- /dev/null
+++ b/test/CodeGen/NVPTX/libcall-intrinsic.ll
@@ -0,0 +1,10 @@
+; RUN: not llc < %s -march=nvptx 2>&1 | FileCheck %s
+; used to seqfault and now fails with an "Undefined external symbol"
+
+; CHECK: LLVM ERROR: Undefined external symbol "__powidf2"
+define double @powi(double, i32) {
+  %a = call double @llvm.powi.f64(double %0, i32 %1)
+  ret double %a
+}
+
+declare double @llvm.powi.f64(double, i32) nounwind readnone
diff --git a/test/CodeGen/NVPTX/proxy-reg-erasure-mir.ll b/test/CodeGen/NVPTX/proxy-reg-erasure-mir.ll
new file mode 100644
index 0000000000000000000000000000000000000000..6bfbe2aea8196ce7701eccc013fb62a5a8c50b29
--- /dev/null
+++ b/test/CodeGen/NVPTX/proxy-reg-erasure-mir.ll
@@ -0,0 +1,25 @@
+; RUN: llc -march=nvptx64 -stop-before=nvptx-proxyreg-erasure < %s 2>&1 \
+; RUN:   | FileCheck %s --check-prefix=MIR --check-prefix=MIR-BEFORE
+
+; RUN: llc -march=nvptx64 -stop-after=nvptx-proxyreg-erasure < %s 2>&1 \
+; RUN:   | FileCheck %s --check-prefix=MIR --check-prefix=MIR-AFTER
+
+; Check ProxyRegErasure pass MIR manipulation.
+
+declare <4 x i32> @callee_vec_i32()
+define  <4 x i32> @check_vec_i32() {
+  ; MIR: body:
+  ; MIR-DAG: Callseq_Start {{[0-9]+}}, {{[0-9]+}}
+  ; MIR-DAG: %0:int32regs, %1:int32regs, %2:int32regs, %3:int32regs = LoadParamMemV4I32 0
+  ; MIR-DAG: Callseq_End {{[0-9]+}}
+
+  ; MIR-BEFORE-DAG: %4:int32regs = ProxyRegI32 killed %0
+  ; MIR-BEFORE-DAG: %5:int32regs = ProxyRegI32 killed %1
+  ; MIR-BEFORE-DAG: %6:int32regs = ProxyRegI32 killed %2
+  ; MIR-BEFORE-DAG: %7:int32regs = ProxyRegI32 killed %3
+  ; MIR-BEFORE-DAG: StoreRetvalV4I32 killed %4, killed %5, killed %6, killed %7, 0
+  ; MIR-AFTER-DAG:  StoreRetvalV4I32 killed %0, killed %1, killed %2, killed %3, 0
+
+  %ret = call <4 x i32> @callee_vec_i32()
+  ret <4 x i32> %ret
+}
diff --git a/test/CodeGen/NVPTX/proxy-reg-erasure-ptx.ll b/test/CodeGen/NVPTX/proxy-reg-erasure-ptx.ll
new file mode 100644
index 0000000000000000000000000000000000000000..7a3b29b20c4e973352b1727e0c7d66e8ce76c082
--- /dev/null
+++ b/test/CodeGen/NVPTX/proxy-reg-erasure-ptx.ll
@@ -0,0 +1,183 @@
+; RUN: llc -march=nvptx64 -stop-before=nvptx-proxyreg-erasure < %s 2>&1 \
+; RUN:   | llc -x mir -march=nvptx64 -start-before=nvptx-proxyreg-erasure 2>&1 \
+; RUN:   | FileCheck %s --check-prefix=PTX --check-prefix=PTX-WITH
+
+; RUN: llc -march=nvptx64 -stop-before=nvptx-proxyreg-erasure < %s 2>&1 \
+; RUN:   | llc -x mir -march=nvptx64 -start-after=nvptx-proxyreg-erasure 2>&1 \
+; RUN:   | FileCheck %s --check-prefix=PTX --check-prefix=PTX-WITHOUT
+
+; Thorough testing of ProxyRegErasure: PTX assembly with and without the pass.
+
+declare i1 @callee_i1()
+define i1 @check_i1() {
+  ; PTX-LABEL: check_i1
+  ; PTX-DAG: { // callseq {{[0-9]+}}, {{[0-9]+}}
+  ; PTX-DAG: ld.param.b32 [[LD:%r[0-9]+]], [retval0+0];
+  ; PTX-DAG: } // callseq {{[0-9]+}}
+
+  ; PTX-WITHOUT-DAG: mov.b32 [[PROXY:%r[0-9]+]], [[LD]];
+  ; PTX-WITHOUT-DAG: and.b32 [[RES:%r[0-9]+]], [[PROXY]], 1;
+  ; PTX-WITH-DAG:    and.b32 [[RES:%r[0-9]+]], [[LD]], 1;
+
+  ; PTX-DAG: st.param.b32 [func_retval0+0], [[RES]];
+
+  %ret = call i1 @callee_i1()
+  ret i1 %ret
+}
+
+declare i16 @callee_i16()
+define  i16 @check_i16() {
+  ; PTX-LABEL: check_i16
+  ; PTX-DAG: { // callseq {{[0-9]+}}, {{[0-9]+}}
+  ; PTX-DAG: ld.param.b32 [[LD:%r[0-9]+]], [retval0+0];
+  ; PTX-DAG: } // callseq {{[0-9]+}}
+
+  ; PTX-WITHOUT-DAG: mov.b32 [[PROXY:%r[0-9]+]], [[LD]];
+  ; PTX-WITHOUT-DAG: and.b32 [[RES:%r[0-9]+]], [[PROXY]], 65535;
+  ; PTX-WITH-DAG:    and.b32 [[RES:%r[0-9]+]], [[LD]], 65535;
+
+  ; PTX-DAG: st.param.b32 [func_retval0+0], [[RES]];
+
+  %ret = call i16 @callee_i16()
+  ret i16 %ret
+}
+
+declare i32 @callee_i32()
+define  i32 @check_i32() {
+  ; PTX-LABEL: check_i32
+  ; PTX-DAG: { // callseq {{[0-9]+}}, {{[0-9]+}}
+  ; PTX-DAG: ld.param.b32 [[LD:%r[0-9]+]], [retval0+0];
+  ; PTX-DAG: } // callseq {{[0-9]+}}
+
+  ; PTX-WITHOUT-DAG: mov.b32 [[PROXY:%r[0-9]+]], [[LD]];
+  ; PTX-WITHOUT-DAG: st.param.b32 [func_retval0+0], [[PROXY]];
+  ; PTX-WITH-DAG:    st.param.b32 [func_retval0+0], [[LD]];
+
+  %ret = call i32 @callee_i32()
+  ret i32 %ret
+}
+
+declare i64 @callee_i64()
+define  i64 @check_i64() {
+  ; PTX-LABEL: check_i64
+  ; PTX-DAG: { // callseq {{[0-9]+}}, {{[0-9]+}}
+  ; PTX-DAG: ld.param.b64 [[LD:%rd[0-9]+]], [retval0+0];
+  ; PTX-DAG: } // callseq {{[0-9]+}}
+
+  ; PTX-WITHOUT-DAG: mov.b64 [[PROXY:%rd[0-9]+]], [[LD]];
+  ; PTX-WITHOUT-DAG: st.param.b64 [func_retval0+0], [[PROXY]];
+  ; PTX-WITH-DAG:    st.param.b64 [func_retval0+0], [[LD]];
+
+  %ret = call i64 @callee_i64()
+  ret i64 %ret
+}
+
+declare i128 @callee_i128()
+define  i128 @check_i128() {
+  ; PTX-LABEL: check_i128
+  ; PTX-DAG: { // callseq {{[0-9]+}}, {{[0-9]+}}
+  ; PTX-DAG: ld.param.v2.b64 {[[LD0:%rd[0-9]+]], [[LD1:%rd[0-9]+]]}, [retval0+0];
+  ; PTX-DAG: } // callseq {{[0-9]+}}
+
+  ; PTX-WITHOUT-DAG: mov.b64 [[PROXY0:%rd[0-9]+]], [[LD0]];
+  ; PTX-WITHOUT-DAG: mov.b64 [[PROXY1:%rd[0-9]+]], [[LD1]];
+  ; PTX-WITHOUT-DAG: st.param.v2.b64 [func_retval0+0], {[[PROXY0]], [[PROXY1]]};
+  ; PTX-WITH-DAG:    st.param.v2.b64 [func_retval0+0], {[[LD0]], [[LD1]]};
+
+  %ret = call i128 @callee_i128()
+  ret i128 %ret
+}
+
+declare half @callee_f16()
+define  half @check_f16() {
+  ; PTX-LABEL: check_f16
+  ; PTX-DAG: { // callseq {{[0-9]+}}, {{[0-9]+}}
+  ; PTX-DAG: ld.param.b16 [[LD:%h[0-9]+]], [retval0+0];
+  ; PTX-DAG: } // callseq {{[0-9]+}}
+
+  ; PTX-WITHOUT-DAG: mov.b16 [[PROXY:%h[0-9]+]], [[LD]];
+  ; PTX-WITHOUT-DAG: st.param.b16 [func_retval0+0], [[PROXY]];
+  ; PTX-WITH-DAG:    st.param.b16 [func_retval0+0], [[LD]];
+
+  %ret = call half @callee_f16()
+  ret half %ret
+}
+
+declare float @callee_f32()
+define  float @check_f32() {
+  ; PTX-LABEL: check_f32
+  ; PTX-DAG: { // callseq {{[0-9]+}}, {{[0-9]+}}
+  ; PTX-DAG: ld.param.f32 [[LD:%f[0-9]+]], [retval0+0];
+  ; PTX-DAG: } // callseq {{[0-9]+}}
+
+  ; PTX-WITHOUT-DAG: mov.f32 [[PROXY:%f[0-9]+]], [[LD]];
+  ; PTX-WITHOUT-DAG: st.param.f32 [func_retval0+0], [[PROXY]];
+  ; PTX-WITH-DAG:    st.param.f32 [func_retval0+0], [[LD]];
+
+  %ret = call float @callee_f32()
+  ret float %ret
+}
+
+declare double @callee_f64()
+define  double @check_f64() {
+  ; PTX-LABEL: check_f64
+  ; PTX-DAG: { // callseq {{[0-9]+}}, {{[0-9]+}}
+  ; PTX-DAG: ld.param.f64 [[LD:%fd[0-9]+]], [retval0+0];
+  ; PTX-DAG: } // callseq {{[0-9]+}}
+
+  ; PTX-WITHOUT-DAG: mov.f64 [[PROXY:%fd[0-9]+]], [[LD]];
+  ; PTX-WITHOUT-DAG: st.param.f64 [func_retval0+0], [[PROXY]];
+  ; PTX-WITH-DAG:    st.param.f64 [func_retval0+0], [[LD]];
+
+  %ret = call double @callee_f64()
+  ret double %ret
+}
+
+declare <4 x i32> @callee_vec_i32()
+define  <4 x i32> @check_vec_i32() {
+  ; PTX-LABEL: check_vec_i32
+  ; PTX-DAG: { // callseq {{[0-9]+}}, {{[0-9]+}}
+  ; PTX-DAG: ld.param.v4.b32 {[[LD0:%r[0-9]+]], [[LD1:%r[0-9]+]], [[LD2:%r[0-9]+]], [[LD3:%r[0-9]+]]}, [retval0+0];
+  ; PTX-DAG: } // callseq {{[0-9]+}}
+
+  ; PTX-WITHOUT-DAG: mov.b32 [[PROXY0:%r[0-9]+]], [[LD0]];
+  ; PTX-WITHOUT-DAG: mov.b32 [[PROXY1:%r[0-9]+]], [[LD1]];
+  ; PTX-WITHOUT-DAG: mov.b32 [[PROXY2:%r[0-9]+]], [[LD2]];
+  ; PTX-WITHOUT-DAG: mov.b32 [[PROXY3:%r[0-9]+]], [[LD3]];
+  ; PTX-WITHOUT-DAG: st.param.v4.b32 [func_retval0+0], {[[PROXY0]], [[PROXY1]], [[PROXY2]], [[PROXY3]]};
+  ; PTX-WITH-DAG:    st.param.v4.b32 [func_retval0+0], {[[LD0]], [[LD1]], [[LD2]], [[LD3]]};
+
+  %ret = call <4 x i32> @callee_vec_i32()
+  ret <4 x i32> %ret
+}
+
+declare <2 x half> @callee_vec_f16()
+define  <2 x half> @check_vec_f16() {
+  ; PTX-LABEL: check_vec_f16
+  ; PTX-DAG: { // callseq {{[0-9]+}}, {{[0-9]+}}
+  ; PTX-DAG: ld.param.b32 [[LD:%hh[0-9]+]], [retval0+0];
+  ; PTX-DAG: } // callseq {{[0-9]+}}
+
+  ; PTX-WITHOUT-DAG: mov.b32 [[PROXY:%hh[0-9]+]], [[LD]];
+  ; PTX-WITHOUT-DAG: st.param.b32 [func_retval0+0], [[PROXY]];
+  ; PTX-WITH-DAG:    st.param.b32 [func_retval0+0], [[LD]];
+
+  %ret = call <2 x half> @callee_vec_f16()
+  ret <2 x half> %ret
+}
+
+declare <2 x double> @callee_vec_f64()
+define  <2 x double> @check_vec_f64() {
+  ; PTX-LABEL: check_vec_f64
+  ; PTX-DAG: { // callseq {{[0-9]+}}, {{[0-9]+}}
+  ; PTX-DAG: ld.param.v2.f64 {[[LD0:%fd[0-9]+]], [[LD1:%fd[0-9]+]]}, [retval0+0];
+  ; PTX-DAG: } // callseq {{[0-9]+}}
+
+  ; PTX-WITHOUT-DAG: mov.f64 [[PROXY0:%fd[0-9]+]], [[LD0]];
+  ; PTX-WITHOUT-DAG: mov.f64 [[PROXY1:%fd[0-9]+]], [[LD1]];
+  ; PTX-WITHOUT-DAG: st.param.v2.f64 [func_retval0+0], {[[PROXY0]], [[PROXY1]]};
+  ; PTX-WITH-DAG:    st.param.v2.f64 [func_retval0+0], {[[LD0]], [[LD1]]};
+
+  %ret = call <2 x double> @callee_vec_f64()
+  ret <2 x double> %ret
+}
diff --git a/test/CodeGen/NVPTX/wmma.py b/test/CodeGen/NVPTX/wmma.py
index bec5a274e02344d4ea427d5b8aeff6af3514419e..14bbfd7df094ab038b430f2e7612284bfee391b7 100644
--- a/test/CodeGen/NVPTX/wmma.py
+++ b/test/CodeGen/NVPTX/wmma.py
@@ -4,6 +4,8 @@
 # RUN: python %s > %t.ll
 # RUN: llc < %t.ll -march=nvptx64 -mcpu=sm_70 -mattr=+ptx61 | FileCheck %t.ll
 
+from __future__ import print_function
+
 from itertools import product
 from string import Template
 
diff --git a/test/CodeGen/NVPTX/zero-cs.ll b/test/CodeGen/NVPTX/zero-cs.ll
deleted file mode 100644
index 7a7a99030d48fc6b65559ae557d1c7e2ab380bc0..0000000000000000000000000000000000000000
--- a/test/CodeGen/NVPTX/zero-cs.ll
+++ /dev/null
@@ -1,10 +0,0 @@
-; RUN: not llc < %s -march=nvptx 2>&1 | FileCheck %s
-; used to seqfault and now fails with a "Cannot select"
-
-; CHECK: LLVM ERROR: Cannot select: {{t7|0x[0-9a-f]+}}: i32 = ExternalSymbol'__powidf2'
-define double @powi() {
-  %1 = call double @llvm.powi.f64(double 1.000000e+00, i32 undef)
-  ret double %1
-}
-
-declare double @llvm.powi.f64(double, i32) nounwind readnone
diff --git a/test/CodeGen/Nios2/add-sub.ll b/test/CodeGen/Nios2/add-sub.ll
deleted file mode 100644
index 7c9a2896ed9faab45b21c7ec95d9ef77fff644d4..0000000000000000000000000000000000000000
--- a/test/CodeGen/Nios2/add-sub.ll
+++ /dev/null
@@ -1,19 +0,0 @@
-; RUN: llc < %s -march=nios2 2>&1 | FileCheck %s
-; RUN: llc < %s -march=nios2 -target-abi=nios2r2 2>&1 | FileCheck %s
-
-define i32 @add_reg(i32 %a, i32 %b) nounwind {
-entry:
-; CHECK: add_reg:
-; CHECK:   add r2, r4, r5
-  %c = add i32 %a, %b
-  ret i32 %c
-}
-
-define i32 @sub_reg(i32 %a, i32 %b) nounwind {
-entry:
-; CHECK: sub_reg:
-; CHECK:   sub r2, r4, r5
-  %c = sub i32 %a, %b
-  ret i32 %c
-}
-
diff --git a/test/CodeGen/Nios2/lit.local.cfg b/test/CodeGen/Nios2/lit.local.cfg
deleted file mode 100644
index 84c8b039391bccdc45c19e3670859511b8a5ca6b..0000000000000000000000000000000000000000
--- a/test/CodeGen/Nios2/lit.local.cfg
+++ /dev/null
@@ -1,3 +0,0 @@
-if not 'Nios2' in config.root.targets:
-    config.unsupported = True
-
diff --git a/test/CodeGen/Nios2/mul-div.ll b/test/CodeGen/Nios2/mul-div.ll
deleted file mode 100644
index 8327823cf141822f2327596dc6040dcf8fa73c45..0000000000000000000000000000000000000000
--- a/test/CodeGen/Nios2/mul-div.ll
+++ /dev/null
@@ -1,27 +0,0 @@
-; RUN: llc < %s -march=nios2 2>&1 | FileCheck %s
-; RUN: llc < %s -march=nios2 -target-abi=nios2r2 2>&1 | FileCheck %s
-
-define i32 @mul_reg(i32 %a, i32 %b) nounwind {
-entry:
-; CHECK: mul_reg:
-; CHECK:   mul r2, r4, r5
-  %c = mul i32 %a, %b
-  ret i32 %c
-}
-
-define i32 @div_signed(i32 %a, i32 %b) nounwind {
-entry:
-; CHECK: div_signed:
-; CHECK:   div r2, r4, r5
-  %c = sdiv i32 %a, %b
-  ret i32 %c
-}
-
-define i32 @div_unsigned(i32 %a, i32 %b) nounwind {
-entry:
-; CHECK: div_unsigned:
-; CHECK:   divu r2, r4, r5
-  %c = udiv i32 %a, %b
-  ret i32 %c
-}
-
diff --git a/test/CodeGen/Nios2/proc_support.ll b/test/CodeGen/Nios2/proc_support.ll
deleted file mode 100644
index c83c105724a49aa7b4fdd722f7edacb37c3b1683..0000000000000000000000000000000000000000
--- a/test/CodeGen/Nios2/proc_support.ll
+++ /dev/null
@@ -1,10 +0,0 @@
-; This tests that llc accepts Nios2 processors.
-
-; RUN: not not llc < %s -asm-verbose=false -march=nios2 -mcpu=nios2r1 2>&1 | FileCheck %s --check-prefix=ARCH
-; RUN: not not llc < %s -asm-verbose=false -march=nios2 -mcpu=nios2r2 2>&1 | FileCheck %s --check-prefix=ARCH
-
-; ARCH-NOT: is not a recognized processor
-
-define i32 @f(i32 %i) {
-  ret i32 %i
-}
diff --git a/test/CodeGen/Nios2/ret_generated.ll b/test/CodeGen/Nios2/ret_generated.ll
deleted file mode 100644
index 986c65751df9b2aac5aad91c3c966307491fc76f..0000000000000000000000000000000000000000
--- a/test/CodeGen/Nios2/ret_generated.ll
+++ /dev/null
@@ -1,9 +0,0 @@
-; This tests that llc generates 'ret' instruction in assembly output.
-
-; RUN: llc < %s -march=nios2 2>&1 | FileCheck %s --check-prefix=ARCH
-
-; ARCH: ret
-
-define i32 @f(i32 %i) {
-  ret i32 %i
-}
diff --git a/test/CodeGen/Nios2/shift-rotate.ll b/test/CodeGen/Nios2/shift-rotate.ll
deleted file mode 100644
index d3084b5fb597c29167b0881cbeef678cd98bf96a..0000000000000000000000000000000000000000
--- a/test/CodeGen/Nios2/shift-rotate.ll
+++ /dev/null
@@ -1,26 +0,0 @@
-; RUN: llc < %s -march=nios2 2>&1 | FileCheck %s
-; RUN: llc < %s -march=nios2 -target-abi=nios2r2 2>&1 | FileCheck %s
-
-define i32 @sll_reg(i32 %a, i32 %b) nounwind {
-entry:
-; CHECK: sll_reg:
-; CHECK:   sll r2, r4, r5
-  %c = shl i32 %a, %b
-  ret i32 %c
-}
-
-define i32 @srl_reg(i32 %a, i32 %b) nounwind {
-entry:
-; CHECK: srl_reg:
-; CHECK:   srl r2, r4, r5
-  %c = lshr i32 %a, %b
-  ret i32 %c
-}
-
-define i32 @sra_reg(i32 %a, i32 %b) nounwind {
-entry:
-; CHECK: sra_reg:
-; CHECK:   sra r2, r4, r5
-  %c = ashr i32 %a, %b
-  ret i32 %c
-}
diff --git a/test/CodeGen/Nios2/target_support.ll b/test/CodeGen/Nios2/target_support.ll
deleted file mode 100644
index 90e7020b2fcc6fe2706ff891a568ee12a179085f..0000000000000000000000000000000000000000
--- a/test/CodeGen/Nios2/target_support.ll
+++ /dev/null
@@ -1,11 +0,0 @@
-; This tests that llc accepts Nios2 target.
-
-; RUN: not not llc < %s -asm-verbose=false -march=nios2 2>&1 | FileCheck %s --check-prefix=ARCH
-; RUN: not not llc < %s -asm-verbose=false -mtriple=nios2 2>&1 | FileCheck %s --check-prefix=TRIPLE
-
-; ARCH-NOT: invalid target
-; TRIPLE-NOT: unable to get target
-
-define i32 @f(i32 %i) {
-  ret i32 %i
-}
diff --git a/test/CodeGen/PowerPC/2010-02-04-EmptyGlobal.ll b/test/CodeGen/PowerPC/2010-02-04-EmptyGlobal.ll
index 66f0566b0e6104ea40f4de64a757f311c0d0194a..a9a2e46959a75a26653f954387dbc77d4bcfb26b 100644
--- a/test/CodeGen/PowerPC/2010-02-04-EmptyGlobal.ll
+++ b/test/CodeGen/PowerPC/2010-02-04-EmptyGlobal.ll
@@ -1,4 +1,4 @@
-; RUN: llc -verify-machineinstrs < %s -mtriple=powerpc-unknown-linux-gnu -relocation-model=pic -disable-fp-elim | FileCheck %s
+; RUN: llc -verify-machineinstrs < %s -mtriple=powerpc-unknown-linux-gnu -relocation-model=pic -frame-pointer=all | FileCheck %s
 ; <rdar://problem/7604010>
 
 %cmd.type = type { }
diff --git a/test/CodeGen/PowerPC/2010-12-18-PPCStackRefs.ll b/test/CodeGen/PowerPC/2010-12-18-PPCStackRefs.ll
index c8588234af6190b59aaac726e45bb942a704b83e..f6577e91ff2f84e8fca8bd391ed219192424cf8d 100644
--- a/test/CodeGen/PowerPC/2010-12-18-PPCStackRefs.ll
+++ b/test/CodeGen/PowerPC/2010-12-18-PPCStackRefs.ll
@@ -1,4 +1,4 @@
-; RUN: llc -verify-machineinstrs -disable-fp-elim < %s | FileCheck %s
+; RUN: llc -verify-machineinstrs -frame-pointer=all < %s | FileCheck %s
 ; PR8749
 target datalayout = "E-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f128:64:128-n32"
 target triple = "powerpc-unknown-linux-gnu.8"
diff --git a/test/CodeGen/PowerPC/Frames-alloca.ll b/test/CodeGen/PowerPC/Frames-alloca.ll
index 2f48f3fd3dd62bbf47ede18f5596063304e1bd80..034231ec3adb628e3f7af4beb71c18d909e43c58 100644
--- a/test/CodeGen/PowerPC/Frames-alloca.ll
+++ b/test/CodeGen/PowerPC/Frames-alloca.ll
@@ -1,7 +1,7 @@
 ; RUN: llc < %s -mtriple=powerpc-unknown-linux-gnu | FileCheck %s -check-prefix=CHECK-PPC32
 ; RUN: llc < %s -mtriple=powerpc64-unknown-linux-gnu | FileCheck %s -check-prefix=CHECK-PPC64
-; RUN: llc < %s -mtriple=powerpc-unknown-linux-gnu -disable-fp-elim | FileCheck %s -check-prefix=CHECK-PPC32-NOFP
-; RUN: llc < %s -mtriple=powerpc64-unknown-linux-gnu -disable-fp-elim | FileCheck %s -check-prefix=CHECK-PPC64-NOFP
+; RUN: llc < %s -mtriple=powerpc-unknown-linux-gnu -frame-pointer=all | FileCheck %s -check-prefix=CHECK-PPC32-NOFP
+; RUN: llc < %s -mtriple=powerpc64-unknown-linux-gnu -frame-pointer=all | FileCheck %s -check-prefix=CHECK-PPC64-NOFP
 ; RUN: llc < %s -mtriple=powerpc-unknown-linux-gnu | FileCheck %s -check-prefix=CHECK-PPC32
 
 ; CHECK-PPC32: stwu 1, -32(1)
diff --git a/test/CodeGen/PowerPC/Frames-large.ll b/test/CodeGen/PowerPC/Frames-large.ll
index a2e60c175e67f421a4afdb17cf398955c7f4aa05..1c49de5bfeabc47de05ba1c299e943218cb71bcd 100644
--- a/test/CodeGen/PowerPC/Frames-large.ll
+++ b/test/CodeGen/PowerPC/Frames-large.ll
@@ -1,8 +1,8 @@
 ; RUN: llc -verify-machineinstrs < %s -mtriple=powerpc-unknown-linux-gnu | FileCheck %s -check-prefix=PPC32-NOFP
-; RUN: llc -verify-machineinstrs < %s -mtriple=powerpc-unknown-linux-gnu -disable-fp-elim | FileCheck %s -check-prefix=PPC32-FP
+; RUN: llc -verify-machineinstrs < %s -mtriple=powerpc-unknown-linux-gnu -frame-pointer=all | FileCheck %s -check-prefix=PPC32-FP
 
 ; RUN: llc -verify-machineinstrs < %s -mtriple=powerpc64-unknown-linux-gnu | FileCheck %s -check-prefix=PPC64-NOFP
-; RUN: llc -verify-machineinstrs < %s -mtriple=powerpc64-unknown-linux-gnu -disable-fp-elim | FileCheck %s -check-prefix=PPC64-FP
+; RUN: llc -verify-machineinstrs < %s -mtriple=powerpc64-unknown-linux-gnu -frame-pointer=all | FileCheck %s -check-prefix=PPC64-FP
 
 define i32* @f1() nounwind {
         %tmp = alloca i32, i32 8191             ; <i32*> [#uses=1]
diff --git a/test/CodeGen/PowerPC/Frames-leaf.ll b/test/CodeGen/PowerPC/Frames-leaf.ll
index 6dce8e20bebe6f17982767753398d9f828c9e90e..d06364ad9d00f430570643ccc6499b5ec560733b 100644
--- a/test/CodeGen/PowerPC/Frames-leaf.ll
+++ b/test/CodeGen/PowerPC/Frames-leaf.ll
@@ -6,13 +6,13 @@
 ; RUN:   not grep "addi r1, r1, "
 ; RUN: llc -verify-machineinstrs < %s -mtriple=ppc32-- | \
 ; RUN:   not grep "lwz r31, 20(r1)"
-; RUN: llc -verify-machineinstrs < %s -mtriple=ppc32-- -disable-fp-elim | \
+; RUN: llc -verify-machineinstrs < %s -mtriple=ppc32-- -frame-pointer=all | \
 ; RUN:   not grep "stw r31, 20(r1)"
-; RUN: llc -verify-machineinstrs < %s -mtriple=ppc32-- -disable-fp-elim | \
+; RUN: llc -verify-machineinstrs < %s -mtriple=ppc32-- -frame-pointer=all | \
 ; RUN:   not grep "stwu r1, -.*(r1)"
-; RUN: llc -verify-machineinstrs < %s -mtriple=ppc32-- -disable-fp-elim | \
+; RUN: llc -verify-machineinstrs < %s -mtriple=ppc32-- -frame-pointer=all | \
 ; RUN:   not grep "addi r1, r1, "
-; RUN: llc -verify-machineinstrs < %s -mtriple=ppc32-- -disable-fp-elim | \
+; RUN: llc -verify-machineinstrs < %s -mtriple=ppc32-- -frame-pointer=all | \
 ; RUN:   not grep "lwz r31, 20(r1)"
 ; RUN: llc -verify-machineinstrs < %s -mtriple=ppc64-- | \
 ; RUN:   not grep "std r31, 40(r1)"
@@ -22,13 +22,13 @@
 ; RUN:   not grep "addi r1, r1, "
 ; RUN: llc -verify-machineinstrs < %s -mtriple=ppc64-- | \
 ; RUN:   not grep "ld r31, 40(r1)"
-; RUN: llc -verify-machineinstrs < %s -mtriple=ppc64-- -disable-fp-elim | \
+; RUN: llc -verify-machineinstrs < %s -mtriple=ppc64-- -frame-pointer=all | \
 ; RUN:   not grep "stw r31, 40(r1)"
-; RUN: llc -verify-machineinstrs < %s -mtriple=ppc64-- -disable-fp-elim | \
+; RUN: llc -verify-machineinstrs < %s -mtriple=ppc64-- -frame-pointer=all | \
 ; RUN:   not grep "stdu r1, -.*(r1)"
-; RUN: llc -verify-machineinstrs < %s -mtriple=ppc64-- -disable-fp-elim | \
+; RUN: llc -verify-machineinstrs < %s -mtriple=ppc64-- -frame-pointer=all | \
 ; RUN:   not grep "addi r1, r1, "
-; RUN: llc -verify-machineinstrs < %s -mtriple=ppc64-- -disable-fp-elim | \
+; RUN: llc -verify-machineinstrs < %s -mtriple=ppc64-- -frame-pointer=all | \
 ; RUN:   not grep "ld r31, 40(r1)"
 
 define i32* @f1() {
diff --git a/test/CodeGen/PowerPC/Frames-small.ll b/test/CodeGen/PowerPC/Frames-small.ll
index 57dbe3f37bcdc8763a0dda26f1aa88853db987e7..1fa9908ad58a585f13f4a9dd9bf93225ff48fabd 100644
--- a/test/CodeGen/PowerPC/Frames-small.ll
+++ b/test/CodeGen/PowerPC/Frames-small.ll
@@ -1,7 +1,7 @@
 ; RUN: llc -verify-machineinstrs < %s -mtriple=powerpc-unknown-linux-gnu | FileCheck %s -check-prefix=PPC32-FP
-; RUN: llc -verify-machineinstrs < %s -mtriple=powerpc-unknown-linux-gnu -disable-fp-elim | FileCheck %s -check-prefix=PPC32-NOFP
+; RUN: llc -verify-machineinstrs < %s -mtriple=powerpc-unknown-linux-gnu -frame-pointer=all | FileCheck %s -check-prefix=PPC32-NOFP
 ; RUN: llc -verify-machineinstrs < %s -mtriple=powerpc64-unknown-linux-gnu | FileCheck %s -check-prefix=PPC64-FP
-; RUN: llc -verify-machineinstrs < %s -mtriple=powerpc64-unknown-linux-gnu -disable-fp-elim | FileCheck %s -check-prefix=PPC64-NOFP
+; RUN: llc -verify-machineinstrs < %s -mtriple=powerpc64-unknown-linux-gnu -frame-pointer=all | FileCheck %s -check-prefix=PPC64-NOFP
 
 ;PPC32-FP: f1:
 ;PPC32-FP: stwu 1, -16400(1)
diff --git a/test/CodeGen/PowerPC/NoCRFieldRedefWhenSpillingCRBIT.mir b/test/CodeGen/PowerPC/NoCRFieldRedefWhenSpillingCRBIT.mir
new file mode 100644
index 0000000000000000000000000000000000000000..be28ac3011d2d10ecce62dd8f632e45bb57eb375
--- /dev/null
+++ b/test/CodeGen/PowerPC/NoCRFieldRedefWhenSpillingCRBIT.mir
@@ -0,0 +1,121 @@
+# RUN: llc -mcpu=pwr8 -mtriple=powerpc64le-unknown-linux-gnu -start-after \
+# RUN:   virtregrewriter -ppc-asm-full-reg-names -verify-machineinstrs %s \
+# RUN:   -o - | FileCheck %s
+
+--- |
+  ; ModuleID = 'a.ll'
+  source_filename = "a.c"
+  target datalayout = "e-m:e-i64:64-n32:64"
+  target triple = "powerpc64le-unknown-linux-gnu"
+  
+  ; Function Attrs: nounwind
+  define void @test(i32 signext %a6, i32 signext %a7, i32 signext %a17) local_unnamed_addr #0 {
+  entry:
+    %cmp27 = icmp slt i32 %a6, %a7
+    %cmp29 = icmp sgt i32 %a6, %a17
+    %or.cond781 = or i1 %cmp27, %cmp29
+    tail call void asm sideeffect "# nothing", "~{cr0},~{cr1},~{cr2},~{cr3},~{cr4},~{cr5},~{cr6},~{cr7},~{memory}"() #1, !srcloc !1
+    br label %if.end326
+  
+  if.end326:                                        ; preds = %entry
+    br i1 %or.cond781, label %if.then330, label %if.end331
+  
+  if.then330:                                       ; preds = %if.end326
+    unreachable
+  
+  if.end331:                                        ; preds = %if.end326
+    ret void
+  }
+  
+  ; Function Attrs: nounwind
+  declare void @llvm.stackprotector(i8*, i8**) #1
+  
+  attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="ppc64le" "target-features"="+altivec,+bpermd,+crypto,+direct-move,+extdiv,+htm,+power8-vector,+vsx,-power9-vector,-qpx" "unsafe-fp-math"="false" "use-soft-float"="false" }
+  attributes #1 = { nounwind }
+  
+  !llvm.ident = !{!0}
+  
+  !0 = !{!"clang version 8.0.0 (trunk 349357)"}
+  !1 = !{i32 3373}
+
+...
+---
+name:            test
+alignment:       4
+exposesReturnsTwice: false
+legalized:       false
+regBankSelected: false
+selected:        false
+failedISel:      false
+tracksRegLiveness: true
+hasWinCFI:       false
+registers:       []
+liveins:         
+  - { reg: '$x3', virtual-reg: '' }
+  - { reg: '$x4', virtual-reg: '' }
+  - { reg: '$x5', virtual-reg: '' }
+frameInfo:       
+  isFrameAddressTaken: false
+  isReturnAddressTaken: false
+  hasStackMap:     false
+  hasPatchPoint:   false
+  stackSize:       0
+  offsetAdjustment: 0
+  maxAlignment:    4
+  adjustsStack:    false
+  hasCalls:        false
+  stackProtector:  ''
+  maxCallFrameSize: 4294967295
+  cvBytesOfCalleeSavedRegisters: 0
+  hasOpaqueSPAdjustment: false
+  hasVAStart:      false
+  hasMustTailInVarArgFunc: false
+  localFrameSize:  0
+  savePoint:       ''
+  restorePoint:    ''
+fixedStack:      []
+stack:           
+  - { id: 0, name: '', type: spill-slot, offset: 0, size: 4, alignment: 4, 
+      stack-id: 0, callee-saved-register: '', callee-saved-restored: true, 
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+  - { id: 1, name: '', type: spill-slot, offset: 0, size: 4, alignment: 4, 
+      stack-id: 0, callee-saved-register: '', callee-saved-restored: true, 
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+  - { id: 2, name: '', type: spill-slot, offset: 0, size: 4, alignment: 4, 
+      stack-id: 0, callee-saved-register: '', callee-saved-restored: true, 
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+constants:       []
+body:             |
+  bb.0.entry:
+    liveins: $x3, $x4, $x5
+  
+    renamable $cr0 = CMPW renamable $r3, renamable $r4, implicit $x4
+    renamable $cr1 = CMPW renamable $r3, renamable $r5, implicit $x5, implicit killed $x3
+    renamable $cr5lt = CRNOR renamable $cr0lt, renamable $cr1gt, implicit killed $cr0
+    renamable $cr5gt = COPY renamable $cr1gt, implicit $cr1
+    ; CHECK: crnor 4*cr5+lt, lt, 4*cr1+gt
+    ; CHECK: cror 4*cr5+gt, 4*cr1+gt, 4*cr1+gt
+    SPILL_CRBIT killed renamable $cr5lt, 0, %stack.0 :: (store 4 into %stack.0)
+    renamable $cr1 = CMPW renamable $r4, renamable $r5, implicit killed $x5, implicit killed $x4
+    SPILL_CRBIT killed renamable $cr5gt, 0, %stack.1 :: (store 4 into %stack.1)
+    SPILL_CRBIT killed renamable $cr1gt, 0, %stack.2 :: (store 4 into %stack.2)
+    INLINEASM &"# nothing", 25, 12, implicit-def dead early-clobber $cr0, 12, implicit-def dead early-clobber $cr1, 12, implicit-def dead early-clobber $cr2, 12, implicit-def dead early-clobber $cr3, 12, implicit-def dead early-clobber $cr4, 12, implicit-def dead early-clobber $cr5, 12, implicit-def dead early-clobber $cr6, 12, implicit-def dead early-clobber $cr7, !1
+    BLR8 implicit $lr8, implicit $rm
+  
+  bb.1.if.end326:
+    successors: %bb.2(0x00000001), %bb.3(0x7fffffff)
+  
+    renamable $cr5lt = RESTORE_CRBIT 0, %stack.0 :: (load 4 from %stack.0)
+    renamable $cr5gt = RESTORE_CRBIT 0, %stack.1 :: (load 4 from %stack.1)
+    renamable $cr5lt = CROR killed renamable $cr5lt, killed renamable $cr5gt
+    BCn killed renamable $cr5lt, %bb.3
+    B %bb.2
+  
+  bb.2.if.then330:
+    successors: 
+  
+  
+  bb.3.if.end331:
+    BLR8 implicit $lr8, implicit $rm
+
+...
diff --git a/test/CodeGen/PowerPC/PR33671.ll b/test/CodeGen/PowerPC/PR33671.ll
index 0edd2e8daff4adc7f34bc96134c0f94a24733075..a613387f3c93b2df5011d9e819bf51f09b63c28d 100644
--- a/test/CodeGen/PowerPC/PR33671.ll
+++ b/test/CodeGen/PowerPC/PR33671.ll
@@ -27,6 +27,6 @@ entry:
 ; CHECK-LABEL: test2
 ; CHECK: addi 3, 3, 8
 ; CHECK: lxvx [[LD:[0-9]+]], 0, 3
-; CHECK: addi 3, 4, 4
-; CHECK: stxvx [[LD]], 0, 3
+; CHECK: addi [[REG:[0-9]+]], 4, 4
+; CHECK: stxvx [[LD]], 0, [[REG]] 
 }
diff --git a/test/CodeGen/PowerPC/PR35812-neg-cmpxchg.ll b/test/CodeGen/PowerPC/PR35812-neg-cmpxchg.ll
index f249dd868c956c77259bf27ddde171a2001a1a4b..aa0e1fc1f3951b4fd8833954897ddfc711a4b0f7 100644
--- a/test/CodeGen/PowerPC/PR35812-neg-cmpxchg.ll
+++ b/test/CodeGen/PowerPC/PR35812-neg-cmpxchg.ll
@@ -2,7 +2,7 @@
 ; Make sure that a negative value for the compare-and-swap is zero extended
 ; from i8/i16 to i32 since it will be compared for equality.
 ; RUN: llc -mtriple=powerpc64le-linux-gnu -verify-machineinstrs < %s | FileCheck %s
-; RUN: llc -mtriple=powerpc64le-linux-gnu -mcpu=pwr7 < %s | FileCheck %s --check-prefix=CHECK-P7
+; RUN: llc -mtriple=powerpc64le-linux-gnu -mcpu=pwr7 -verify-machineinstrs < %s | FileCheck %s --check-prefix=CHECK-P7
 
 @str = private unnamed_addr constant [46 x i8] c"FAILED: __atomic_compare_exchange_n() failed.\00"
 @str.1 = private unnamed_addr constant [59 x i8] c"FAILED: __atomic_compare_exchange_n() set the wrong value.\00"
@@ -50,7 +50,7 @@ define signext i32 @main() {
 ; CHECK-P7:  .LBB0_1: # %L.entry
 ; CHECK-P7:    lwarx 9, 0, 4
 ; CHECK-P7:    and 6, 9, 5
-; CHECK-P7:    cmpw 0, 6, 8
+; CHECK-P7:    cmpw 6, 8
 ; CHECK-P7:    bne 0, .LBB0_3
 ; CHECK-P7:    andc 9, 9, 5
 ; CHECK-P7:    or 9, 9, 7
diff --git a/test/CodeGen/PowerPC/VSX-XForm-Scalars.ll b/test/CodeGen/PowerPC/VSX-XForm-Scalars.ll
index 643ec904ea38f5eccbf3072a4a2ae8ec8322136c..937264f989baf72ce2df33cabfcc663a2dabd62c 100644
--- a/test/CodeGen/PowerPC/VSX-XForm-Scalars.ll
+++ b/test/CodeGen/PowerPC/VSX-XForm-Scalars.ll
@@ -33,12 +33,12 @@ define void @testExpandPostRAPseudo(i32* nocapture readonly %ptr) {
 ; CHECK-P9:    xxpermdi vs0, f0, f0, 2
 ; CHECK-P9:    xxspltw vs0, vs0, 3
 ; CHECK-P9:    stxvx vs0, 0, r4
-; CHECK-P9:    lis r4, 1024
 ; CHECK-P9:    lfiwax f0, 0, r3
 ; CHECK-P9:    addis r3, r2, .LC1@toc@ha
 ; CHECK-P9:    ld r3, .LC1@toc@l(r3)
 ; CHECK-P9:    xscvsxdsp f0, f0
 ; CHECK-P9:    ld r3, 0(r3)
+; CHECK-P9:    lis r4, 1024
 ; CHECK-P9:    stfsx f0, r3, r4
 ; CHECK-P9:    blr
 entry:
diff --git a/test/CodeGen/PowerPC/adde_return_type.ll b/test/CodeGen/PowerPC/adde_return_type.ll
new file mode 100644
index 0000000000000000000000000000000000000000..7ce11079a6267631db79ddd82bcc3c2a282079e3
--- /dev/null
+++ b/test/CodeGen/PowerPC/adde_return_type.ll
@@ -0,0 +1,11 @@
+; REQUIRES: asserts
+; RUN: llc -mtriple=powerpc64le-unknown-unknown -debug-only=legalize-types \
+; RUN:   < %s -o /dev/null 2>&1 | FileCheck %s
+
+define i64 @testAddeReturnType(i64 %X, i64 %Z) {
+; CHECK: Legally typed node: {{.*}}: i64,glue = adde {{.*}} 
+  %cmp = icmp ne i64 %Z, 0
+  %conv1 = zext i1 %cmp to i64
+  %add = add nsw i64 %conv1, %X
+  ret i64 %add
+}
diff --git a/test/CodeGen/PowerPC/atomic-2.ll b/test/CodeGen/PowerPC/atomic-2.ll
index c1fbb43502075b16a7e8434382ca1111d1f0cb34..29d16ef82d2fa91e86e6d027bea184500c2ccda7 100644
--- a/test/CodeGen/PowerPC/atomic-2.ll
+++ b/test/CodeGen/PowerPC/atomic-2.ll
@@ -1,7 +1,7 @@
-; RUN: llc < %s -ppc-asm-full-reg-names -mtriple=ppc64-- | FileCheck %s -check-prefix=CHECK -check-prefix=CHECK-BE
-; RUN: llc < %s -ppc-asm-full-reg-names -mtriple=powerpc64le-unknown-linux-gnu | FileCheck %s -check-prefix=CHECK -check-prefix=CHECK-LE
-; RUN: llc < %s -ppc-asm-full-reg-names -mtriple=ppc64-- -mcpu=pwr7 | FileCheck %s
-; RUN: llc < %s -ppc-asm-full-reg-names -mtriple=ppc64-- -mcpu=pwr8 | FileCheck %s -check-prefix=CHECK-P8U
+; RUN: llc -verify-machineinstrs < %s -ppc-asm-full-reg-names -mtriple=ppc64-- | FileCheck %s -check-prefix=CHECK -check-prefix=CHECK-BE
+; RUN: llc -verify-machineinstrs < %s -ppc-asm-full-reg-names -mtriple=powerpc64le-unknown-linux-gnu | FileCheck %s -check-prefix=CHECK -check-prefix=CHECK-LE
+; RUN: llc -verify-machineinstrs < %s -ppc-asm-full-reg-names -mtriple=ppc64-- -mcpu=pwr7 | FileCheck %s
+; RUN: llc -verify-machineinstrs < %s -ppc-asm-full-reg-names -mtriple=ppc64-- -mcpu=pwr8 | FileCheck %s -check-prefix=CHECK-P8U
 
 define i64 @exchange_and_add(i64* %mem, i64 %val) nounwind {
 ; CHECK-LABEL: exchange_and_add:
diff --git a/test/CodeGen/PowerPC/atomic-minmax.ll b/test/CodeGen/PowerPC/atomic-minmax.ll
index 5b9a15331897d8846d7792890261fb25a52c9729..f7369f30a384f06887b5dd36ef94a2c8d805eb1f 100644
--- a/test/CodeGen/PowerPC/atomic-minmax.ll
+++ b/test/CodeGen/PowerPC/atomic-minmax.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s | FileCheck %s
+; RUN: llc -verify-machineinstrs < %s | FileCheck %s
 target datalayout = "E-m:e-i64:64-n32:64"
 target triple = "powerpc64-unknown-linux-gnu"
 
@@ -244,7 +244,7 @@ entry:
 ; CHECK: and [[MOLDV:[0-9]+]], [[OLDV]], [[M]]
 ; CHECK: srw [[SMOLDV:[0-9]+]], [[MOLDV]], [[SA]]
 ; CHECK: extsh [[SESMOLDV:[0-9]+]], [[SMOLDV]]
-; CHECK: cmpw 0, 4, [[SESMOLDV]]
+; CHECK: cmpw 4, [[SESMOLDV]]
 ; CHECK: bgelr 0
 ; CHECK: andc [[NOLDV:[0-9]+]], [[OLDV]], [[M]]
 ; CHECK: or [[NEWV:[0-9]+]], [[SMV]], [[NOLDV]]
@@ -271,7 +271,7 @@ entry:
 ; CHECK: and [[MOLDV:[0-9]+]], [[OLDV]], [[M]]
 ; CHECK: srw [[SMOLDV:[0-9]+]], [[MOLDV]], [[SA]]
 ; CHECK: extsh [[SESMOLDV:[0-9]+]], [[SMOLDV]]
-; CHECK: cmpw 0, 4, [[SESMOLDV]]
+; CHECK: cmpw 4, [[SESMOLDV]]
 ; CHECK: blelr 0
 ; CHECK: andc [[NOLDV:[0-9]+]], [[OLDV]], [[M]]
 ; CHECK: or [[NEWV:[0-9]+]], [[SMV]], [[NOLDV]]
@@ -296,7 +296,7 @@ entry:
 ; CHECK-DAG: and [[SMV:[0-9]+]], [[SV]], [[M]]
 ; CHECK: lwarx [[OLDV:[0-9]+]], 0, 3
 ; CHECK: and [[MOLDV:[0-9]+]], [[OLDV]], [[M]]
-; CHECK: cmplw 0, 4, [[MOLDV]]
+; CHECK: cmplw 4, [[MOLDV]]
 ; CHECK: bgelr 0
 ; CHECK: andc [[NOLDV:[0-9]+]], [[OLDV]], [[M]]
 ; CHECK: or [[NEWV:[0-9]+]], [[SMV]], [[NOLDV]]
@@ -321,7 +321,7 @@ entry:
 ; CHECK-DAG: and [[SMV:[0-9]+]], [[SV]], [[M]]
 ; CHECK: lwarx [[OLDV:[0-9]+]], 0, 3
 ; CHECK: and [[MOLDV:[0-9]+]], [[OLDV]], [[M]]
-; CHECK: cmplw 0, 4, [[MOLDV]]
+; CHECK: cmplw 4, [[MOLDV]]
 ; CHECK: blelr 0
 ; CHECK: andc [[NOLDV:[0-9]+]], [[OLDV]], [[M]]
 ; CHECK: or [[NEWV:[0-9]+]], [[SMV]], [[NOLDV]]
@@ -347,7 +347,7 @@ entry:
 ; CHECK: and [[MOLDV:[0-9]+]], [[OLDV]], [[M]]
 ; CHECK: srw [[SMOLDV:[0-9]+]], [[MOLDV]], [[SA]]
 ; CHECK: extsb [[SESMOLDV:[0-9]+]], [[SMOLDV]]
-; CHECK: cmpw 0, 4, [[SESMOLDV]]
+; CHECK: cmpw 4, [[SESMOLDV]]
 ; CHECK: bgelr 0
 ; CHECK: andc [[NOLDV:[0-9]+]], [[OLDV]], [[M]]
 ; CHECK: or [[NEWV:[0-9]+]], [[SMV]], [[NOLDV]]
@@ -373,7 +373,7 @@ entry:
 ; CHECK: and [[MOLDV:[0-9]+]], [[OLDV]], [[M]]
 ; CHECK: srw [[SMOLDV:[0-9]+]], [[MOLDV]], [[SA]]
 ; CHECK: extsb [[SESMOLDV:[0-9]+]], [[SMOLDV]]
-; CHECK: cmpw 0, 4, [[SESMOLDV]]
+; CHECK: cmpw 4, [[SESMOLDV]]
 ; CHECK: blelr 0
 ; CHECK: andc [[NOLDV:[0-9]+]], [[OLDV]], [[M]]
 ; CHECK: or [[NEWV:[0-9]+]], [[SMV]], [[NOLDV]]
@@ -397,7 +397,7 @@ entry:
 ; CHECK-DAG: and [[SMV:[0-9]+]], [[SV]], [[M]]
 ; CHECK: lwarx [[OLDV:[0-9]+]], 0, 3
 ; CHECK: and [[MOLDV:[0-9]+]], [[OLDV]], [[M]]
-; CHECK: cmplw 0, 4, [[MOLDV]]
+; CHECK: cmplw 4, [[MOLDV]]
 ; CHECK: bgelr 0
 ; CHECK: andc [[NOLDV:[0-9]+]], [[OLDV]], [[M]]
 ; CHECK: or [[NEWV:[0-9]+]], [[SMV]], [[NOLDV]]
@@ -421,7 +421,7 @@ entry:
 ; CHECK-DAG: and [[SMV:[0-9]+]], [[SV]], [[M]]
 ; CHECK: lwarx [[OLDV:[0-9]+]], 0, 3
 ; CHECK: and [[MOLDV:[0-9]+]], [[OLDV]], [[M]]
-; CHECK: cmplw 0, 4, [[MOLDV]]
+; CHECK: cmplw 4, [[MOLDV]]
 ; CHECK: blelr 0
 ; CHECK: andc [[NOLDV:[0-9]+]], [[OLDV]], [[M]]
 ; CHECK: or [[NEWV:[0-9]+]], [[SMV]], [[NOLDV]]
diff --git a/test/CodeGen/PowerPC/atomics.ll b/test/CodeGen/PowerPC/atomics.ll
index 1abc9dc48c9ae377e4fae53facf46068cc0f52d1..c964218cb60bf6a36fefd2b67762530eed4a26c2 100644
--- a/test/CodeGen/PowerPC/atomics.ll
+++ b/test/CodeGen/PowerPC/atomics.ll
@@ -1,7 +1,6 @@
-; RUN: llc < %s -mtriple=powerpc-unknown-linux-gnu -verify-machineinstrs  -ppc-asm-full-reg-names | FileCheck %s --check-prefix=CHECK --check-prefix=PPC32
-; FIXME: -verify-machineinstrs currently fail on ppc64 (mismatched register/instruction).
+; RUN: llc -verify-machineinstrs < %s -mtriple=powerpc-unknown-linux-gnu -verify-machineinstrs  -ppc-asm-full-reg-names | FileCheck %s --check-prefix=CHECK --check-prefix=PPC32
 ; This is already checked for in Atomics-64.ll
-; RUN: llc < %s -mtriple=powerpc64-unknown-linux-gnu  -ppc-asm-full-reg-names | FileCheck %s --check-prefix=CHECK --check-prefix=PPC64
+; RUN: llc -verify-machineinstrs < %s -mtriple=powerpc64-unknown-linux-gnu  -ppc-asm-full-reg-names | FileCheck %s --check-prefix=CHECK --check-prefix=PPC64
 
 ; FIXME: we don't currently check for the operations themselves with CHECK-NEXT,
 ;   because they are implemented in a very messy way with lwarx/stwcx.
diff --git a/test/CodeGen/PowerPC/bitfieldinsert.ll b/test/CodeGen/PowerPC/bitfieldinsert.ll
index 76a648b6f13f4e354446577e4b277505fd6bec42..97d86f20b123889ff68a20086f14d3696d630aa0 100644
--- a/test/CodeGen/PowerPC/bitfieldinsert.ll
+++ b/test/CodeGen/PowerPC/bitfieldinsert.ll
@@ -60,3 +60,61 @@ entry:
   ret void
 }
 
+; test cases which include ISD::TRUNCATE
+; equivalent C code
+;   struct s64b {
+;     int a:4;
+;     int b:16;
+;     int c:24;
+;   };
+;   void bitfieldinsert64b(struct s64b *p, unsigned char v) {
+;     p->b = v;
+;   }
+
+%struct.s64b = type { i24, i24 }
+
+define void @bitfieldinsert64b(%struct.s64b* nocapture %p, i8 zeroext %v) {
+; CHECK-LABEL: @bitfieldinsert64b
+; CHECK: lwz [[REG1:[0-9]+]], 0(3)
+; CHECK-NEXT: rlwimi [[REG1]], 4, 4, 12, 27
+; CHECK-NEXT: stw [[REG1]], 0(3)
+; CHECK-NEXT: blr
+entry:
+  %conv = zext i8 %v to i32
+  %0 = bitcast %struct.s64b* %p to i32*
+  %bf.load = load i32, i32* %0, align 4
+  %bf.shl = shl nuw nsw i32 %conv, 4
+  %bf.clear = and i32 %bf.load, -1048561
+  %bf.set = or i32 %bf.clear, %bf.shl
+  store i32 %bf.set, i32* %0, align 4
+  ret void
+}
+
+; equivalent C code
+;   struct s64c {
+;     int a:5;
+;     int b:16;
+;     long c:10;
+;   };
+;   void bitfieldinsert64c(struct s64c *p, unsigned short v) {
+;     p->b = v;
+;   }
+
+%struct.s64c = type { i32, [4 x i8] }
+
+define void @bitfieldinsert64c(%struct.s64c* nocapture %p, i16 zeroext %v) {
+; CHECK-LABEL: @bitfieldinsert64c
+; CHECK: lwz [[REG1:[0-9]+]], 0(3)
+; CHECK-NEXT: rlwimi [[REG1]], 4, 5, 11, 26
+; CHECK-NEXT: stw [[REG1]], 0(3)
+; CHECK-NEXT: blr
+entry:
+  %conv = zext i16 %v to i32
+  %0 = getelementptr inbounds %struct.s64c, %struct.s64c* %p, i64 0, i32 0
+  %bf.load = load i32, i32* %0, align 8
+  %bf.shl = shl nuw nsw i32 %conv, 5
+  %bf.clear = and i32 %bf.load, -2097121
+  %bf.set = or i32 %bf.clear, %bf.shl
+  store i32 %bf.set, i32* %0, align 8
+  ret void
+}
diff --git a/test/CodeGen/PowerPC/bool-math.ll b/test/CodeGen/PowerPC/bool-math.ll
index 6404e6b2b2ac21739e3bbe9ac515d648c8c4af8a..970ccd0469f68dbbdf007685f7edae30db62adfd 100644
--- a/test/CodeGen/PowerPC/bool-math.ll
+++ b/test/CodeGen/PowerPC/bool-math.ll
@@ -83,9 +83,8 @@ define i8 @add_zext_cmp_mask_narrower_result(i32 %x) {
 define i32 @low_bit_select_constants_bigger_false_same_size_result(i32 %x) {
 ; CHECK-LABEL: low_bit_select_constants_bigger_false_same_size_result:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    not 3, 3
 ; CHECK-NEXT:    clrldi 3, 3, 63
-; CHECK-NEXT:    subfic 3, 3, 43
+; CHECK-NEXT:    ori 3, 3, 42
 ; CHECK-NEXT:    blr
   %a = and i32 %x, 1
   %c = icmp eq i32 %a, 0
@@ -96,9 +95,8 @@ define i32 @low_bit_select_constants_bigger_false_same_size_result(i32 %x) {
 define i64 @low_bit_select_constants_bigger_false_wider_result(i32 %x) {
 ; CHECK-LABEL: low_bit_select_constants_bigger_false_wider_result:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    not 3, 3
 ; CHECK-NEXT:    clrldi 3, 3, 63
-; CHECK-NEXT:    subfic 3, 3, 27
+; CHECK-NEXT:    ori 3, 3, 26
 ; CHECK-NEXT:    blr
   %a = and i32 %x, 1
   %c = icmp eq i32 %a, 0
@@ -109,9 +107,8 @@ define i64 @low_bit_select_constants_bigger_false_wider_result(i32 %x) {
 define i16 @low_bit_select_constants_bigger_false_narrower_result(i32 %x) {
 ; CHECK-LABEL: low_bit_select_constants_bigger_false_narrower_result:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    nor 3, 3, 3
-; CHECK-NEXT:    clrlwi 3, 3, 31
-; CHECK-NEXT:    subfic 3, 3, 37
+; CHECK-NEXT:    clrldi 3, 3, 63
+; CHECK-NEXT:    ori 3, 3, 36
 ; CHECK-NEXT:    blr
   %a = and i32 %x, 1
   %c = icmp eq i32 %a, 0
diff --git a/test/CodeGen/PowerPC/build-vector-tests.ll b/test/CodeGen/PowerPC/build-vector-tests.ll
index 6f65b189b75459458ee40490a354740554b27ed2..3fc5ffe2aceff3d094b42d2419caa3759a3750eb 100644
--- a/test/CodeGen/PowerPC/build-vector-tests.ll
+++ b/test/CodeGen/PowerPC/build-vector-tests.ll
@@ -1244,15 +1244,15 @@ entry:
 ; P9LE-LABEL: fromRegsConvftoi
 ; P8BE-LABEL: fromRegsConvftoi
 ; P8LE-LABEL: fromRegsConvftoi
-; P9BE-DAG: xxmrghd {{[vs]+}}[[REG1:[0-9]+]], vs1, vs3
-; P9BE-DAG: xxmrghd {{[vs]+}}[[REG2:[0-9]+]], vs2, vs4
-; P9BE-DAG: xvcvdpsxws [[REG3:v[0-9]+]], {{[vs]+}}[[REG1]]
-; P9BE-DAG: xvcvdpsxws [[REG4:v[0-9]+]], {{[vs]+}}[[REG2]]
+; P9BE: xxmrghd {{[vs]+}}[[REG2:[0-9]+]], vs2, vs4
+; P9BE: xvcvdpsxws [[REG4:v[0-9]+]], {{[vs]+}}[[REG2]]
+; P9BE: xxmrghd {{[vs]+}}[[REG1:[0-9]+]], vs1, vs3
+; P9BE: xvcvdpsxws [[REG3:v[0-9]+]], {{[vs]+}}[[REG1]]
 ; P9BE: vmrgew v2, [[REG3]], [[REG4]]
-; P9LE-DAG: xxmrghd {{[vs]+}}[[REG1:[0-9]+]], vs3, vs1
-; P9LE-DAG: xxmrghd {{[vs]+}}[[REG2:[0-9]+]], vs4, vs2
-; P9LE-DAG: xvcvdpsxws [[REG3:v[0-9]+]], {{[vs]+}}[[REG1]]
-; P9LE-DAG: xvcvdpsxws [[REG4:v[0-9]+]], {{[vs]+}}[[REG2]]
+; P9LE: xxmrghd {{[vs]+}}[[REG1:[0-9]+]], vs3, vs1
+; P9LE: xvcvdpsxws [[REG3:v[0-9]+]], {{[vs]+}}[[REG1]]
+; P9LE: xxmrghd {{[vs]+}}[[REG2:[0-9]+]], vs4, vs2
+; P9LE: xvcvdpsxws [[REG4:v[0-9]+]], {{[vs]+}}[[REG2]]
 ; P9LE: vmrgew v2, [[REG4]], [[REG3]]
 ; P8BE-DAG: xxmrghd {{[vs]+}}[[REG1:[0-9]+]], vs1, vs3
 ; P8BE-DAG: xxmrghd {{[vs]+}}[[REG2:[0-9]+]], vs2, vs4
@@ -1516,15 +1516,15 @@ entry:
 ; P9LE-LABEL: fromRegsConvdtoi
 ; P8BE-LABEL: fromRegsConvdtoi
 ; P8LE-LABEL: fromRegsConvdtoi
-; P9BE-DAG: xxmrghd {{[vs]+}}[[REG1:[0-9]+]], vs1, vs3
-; P9BE-DAG: xxmrghd {{[vs]+}}[[REG2:[0-9]+]], vs2, vs4
-; P9BE-DAG: xvcvdpsxws [[REG3:v[0-9]+]], {{[vs]+}}[[REG1]]
-; P9BE-DAG: xvcvdpsxws [[REG4:v[0-9]+]], {{[vs]+}}[[REG2]]
+; P9BE: xxmrghd {{[vs]+}}[[REG2:[0-9]+]], vs2, vs4
+; P9BE: xvcvdpsxws [[REG4:v[0-9]+]], {{[vs]+}}[[REG2]]
+; P9BE: xxmrghd {{[vs]+}}[[REG1:[0-9]+]], vs1, vs3
+; P9BE: xvcvdpsxws [[REG3:v[0-9]+]], {{[vs]+}}[[REG1]]
 ; P9BE: vmrgew v2, [[REG3]], [[REG4]]
-; P9LE-DAG: xxmrghd {{[vs]+}}[[REG1:[0-9]+]], vs3, vs1
-; P9LE-DAG: xxmrghd {{[vs]+}}[[REG2:[0-9]+]], vs4, vs2
-; P9LE-DAG: xvcvdpsxws [[REG3:v[0-9]+]], {{[vs]+}}[[REG1]]
-; P9LE-DAG: xvcvdpsxws [[REG4:v[0-9]+]], {{[vs]+}}[[REG2]]
+; P9LE: xxmrghd {{[vs]+}}[[REG1:[0-9]+]], vs3, vs1
+; P9LE: xvcvdpsxws [[REG3:v[0-9]+]], {{[vs]+}}[[REG1]]
+; P9LE: xxmrghd {{[vs]+}}[[REG2:[0-9]+]], vs4, vs2
+; P9LE: xvcvdpsxws [[REG4:v[0-9]+]], {{[vs]+}}[[REG2]]
 ; P9LE: vmrgew v2, [[REG4]], [[REG3]]
 ; P8BE-DAG: xxmrghd {{[vs]+}}[[REG1:[0-9]+]], vs1, vs3
 ; P8BE-DAG: xxmrghd {{[vs]+}}[[REG2:[0-9]+]], vs2, vs4
@@ -1642,8 +1642,8 @@ entry:
 ; P9LE: lfd
 ; P9LE: lfd
 ; P9LE: xxmrghd
-; P9LE: xxmrghd
 ; P9LE: xvcvdpsxws
+; P9LE: xxmrghd
 ; P9LE: xvcvdpsxws
 ; P9LE: vmrgew v2
 ; P8BE: lfdx
@@ -1711,8 +1711,8 @@ entry:
 ; P9LE: lfd
 ; P9LE: lfd
 ; P9LE: xxmrghd
-; P9LE: xxmrghd
 ; P9LE: xvcvdpsxws
+; P9LE: xxmrghd
 ; P9LE: xvcvdpsxws
 ; P9LE: vmrgew v2
 ; P8BE: lfdux
@@ -1780,8 +1780,8 @@ entry:
 ; P9LE: lfd
 ; P9LE: lfd
 ; P9LE: xxmrghd
-; P9LE: xxmrghd
 ; P9LE: xvcvdpsxws
+; P9LE: xxmrghd
 ; P9LE: xvcvdpsxws
 ; P9LE: vmrgew v2
 ; P8BE: lfdux
@@ -2376,15 +2376,15 @@ entry:
 ; P9LE-LABEL: fromRegsConvftoui
 ; P8BE-LABEL: fromRegsConvftoui
 ; P8LE-LABEL: fromRegsConvftoui
-; P9BE-DAG: xxmrghd {{[vs]+}}[[REG1:[0-9]+]], vs1, vs3
-; P9BE-DAG: xxmrghd {{[vs]+}}[[REG2:[0-9]+]], vs2, vs4
-; P9BE-DAG: xvcvdpuxws [[REG3:v[0-9]+]], {{[vs]+}}[[REG1]]
-; P9BE-DAG: xvcvdpuxws [[REG4:v[0-9]+]], {{[vs]+}}[[REG2]]
+; P9BE: xxmrghd {{[vs]+}}[[REG2:[0-9]+]], vs2, vs4
+; P9BE: xvcvdpuxws [[REG4:v[0-9]+]], {{[vs]+}}[[REG2]]
+; P9BE: xxmrghd {{[vs]+}}[[REG1:[0-9]+]], vs1, vs3
+; P9BE: xvcvdpuxws [[REG3:v[0-9]+]], {{[vs]+}}[[REG1]]
 ; P9BE: vmrgew v2, [[REG3]], [[REG4]]
-; P9LE-DAG: xxmrghd {{[vs]+}}[[REG1:[0-9]+]], vs3, vs1
-; P9LE-DAG: xxmrghd {{[vs]+}}[[REG2:[0-9]+]], vs4, vs2
-; P9LE-DAG: xvcvdpuxws [[REG3:v[0-9]+]], {{[vs]+}}[[REG1]]
-; P9LE-DAG: xvcvdpuxws [[REG4:v[0-9]+]], {{[vs]+}}[[REG2]]
+; P9LE: xxmrghd {{[vs]+}}[[REG1:[0-9]+]], vs3, vs1
+; P9LE: xvcvdpuxws [[REG3:v[0-9]+]], {{[vs]+}}[[REG1]]
+; P9LE: xxmrghd {{[vs]+}}[[REG2:[0-9]+]], vs4, vs2
+; P9LE: xvcvdpuxws [[REG4:v[0-9]+]], {{[vs]+}}[[REG2]]
 ; P9LE: vmrgew v2, [[REG4]], [[REG3]]
 ; P8BE-DAG: xxmrghd {{[vs]+}}[[REG1:[0-9]+]], vs1, vs3
 ; P8BE-DAG: xxmrghd {{[vs]+}}[[REG2:[0-9]+]], vs2, vs4
@@ -2648,15 +2648,15 @@ entry:
 ; P9LE-LABEL: fromRegsConvdtoui
 ; P8BE-LABEL: fromRegsConvdtoui
 ; P8LE-LABEL: fromRegsConvdtoui
-; P9BE-DAG: xxmrghd {{[vs]+}}[[REG1:[0-9]+]], vs1, vs3
-; P9BE-DAG: xxmrghd {{[vs]+}}[[REG2:[0-9]+]], vs2, vs4
-; P9BE-DAG: xvcvdpuxws [[REG3:v[0-9]+]], {{[vs]+}}[[REG1]]
-; P9BE-DAG: xvcvdpuxws [[REG4:v[0-9]+]], {{[vs]+}}[[REG2]]
+; P9BE: xxmrghd {{[vs]+}}[[REG2:[0-9]+]], vs2, vs4
+; P9BE: xvcvdpuxws [[REG4:v[0-9]+]], {{[vs]+}}[[REG2]]
+; P9BE: xxmrghd {{[vs]+}}[[REG1:[0-9]+]], vs1, vs3
+; P9BE: xvcvdpuxws [[REG3:v[0-9]+]], {{[vs]+}}[[REG1]]
 ; P9BE: vmrgew v2, [[REG3]], [[REG4]]
-; P9LE-DAG: xxmrghd {{[vs]+}}[[REG1:[0-9]+]], vs3, vs1
-; P9LE-DAG: xxmrghd {{[vs]+}}[[REG2:[0-9]+]], vs4, vs2
-; P9LE-DAG: xvcvdpuxws [[REG3:v[0-9]+]], {{[vs]+}}[[REG1]]
-; P9LE-DAG: xvcvdpuxws [[REG4:v[0-9]+]], {{[vs]+}}[[REG2]]
+; P9LE: xxmrghd {{[vs]+}}[[REG1:[0-9]+]], vs3, vs1
+; P9LE: xvcvdpuxws [[REG3:v[0-9]+]], {{[vs]+}}[[REG1]]
+; P9LE: xxmrghd {{[vs]+}}[[REG2:[0-9]+]], vs4, vs2
+; P9LE: xvcvdpuxws [[REG4:v[0-9]+]], {{[vs]+}}[[REG2]]
 ; P9LE: vmrgew v2, [[REG4]], [[REG3]]
 ; P8BE-DAG: xxmrghd {{[vs]+}}[[REG1:[0-9]+]], vs1, vs3
 ; P8BE-DAG: xxmrghd {{[vs]+}}[[REG2:[0-9]+]], vs2, vs4
@@ -2774,8 +2774,8 @@ entry:
 ; P9LE: lfd
 ; P9LE: lfd
 ; P9LE: xxmrghd
-; P9LE: xxmrghd
 ; P9LE: xvcvdpuxws
+; P9LE: xxmrghd
 ; P9LE: xvcvdpuxws
 ; P9LE: vmrgew v2
 ; P8BE: lfdx
@@ -2843,8 +2843,8 @@ entry:
 ; P9LE: lfd
 ; P9LE: lfd
 ; P9LE: xxmrghd
-; P9LE: xxmrghd
 ; P9LE: xvcvdpuxws
+; P9LE: xxmrghd
 ; P9LE: xvcvdpuxws
 ; P9LE: vmrgew v2
 ; P8BE: lfdux
@@ -2912,8 +2912,8 @@ entry:
 ; P9LE: lfd
 ; P9LE: lfd
 ; P9LE: xxmrghd
-; P9LE: xxmrghd
 ; P9LE: xvcvdpuxws
+; P9LE: xxmrghd
 ; P9LE: xvcvdpuxws
 ; P9LE: vmrgew v2
 ; P8BE: lfdux
diff --git a/test/CodeGen/PowerPC/builtins-ppc-p9-f128.ll b/test/CodeGen/PowerPC/builtins-ppc-p9-f128.ll
index dbfd817dbbc0b6b3e786295c58b781040ebd59b4..e3ffc0f7e4dcd44d53185f207bac0f48881620d2 100644
--- a/test/CodeGen/PowerPC/builtins-ppc-p9-f128.ll
+++ b/test/CodeGen/PowerPC/builtins-ppc-p9-f128.ll
@@ -112,8 +112,8 @@ entry:
   %2 = call fp128 @llvm.ppc.scalar.insert.exp.qp(fp128 %0, i64 %1)
   ret fp128 %2
 ; CHECK-LABEL: insert_exp_qp
-; CHECK: mtvsrd [[FPREG:f[0-9]+]], r3
-; CHECK: lxvx [[VECREG:v[0-9]+]]
+; CHECK-DAG: mtvsrd [[FPREG:f[0-9]+]], r3
+; CHECK-DAG: lxvx [[VECREG:v[0-9]+]]
 ; CHECK: xsiexpqp v2, [[VECREG]], [[FPREG]]
 ; CHECK: blr
 }
diff --git a/test/CodeGen/PowerPC/convert-rr-to-ri-instr-add.mir b/test/CodeGen/PowerPC/convert-rr-to-ri-instr-add.mir
new file mode 100644
index 0000000000000000000000000000000000000000..b92884ae9b4b4b12b9f25c50ae5023f31b609656
--- /dev/null
+++ b/test/CodeGen/PowerPC/convert-rr-to-ri-instr-add.mir
@@ -0,0 +1,35 @@
+# RUN: llc -mtriple=powerpc64le--linux-gnu -stop-after ppc-pre-emit-peephole %s -o - -verify-machineinstrs | FileCheck %s
+
+---
+# ADDI8 + STFSX can be converted to ADDI8 + STFS even ADDI8 can not be erased.
+name: testFwdOperandKilledAfter
+# CHECK: name: testFwdOperandKilledAfter
+tracksRegLiveness: true
+body: |
+  bb.0.entry:
+    liveins: $x3, $f1, $x5
+    $x3 = ADDI8 $x5, 100
+    STFSX killed $f1, $zero8, $x3
+    ; CHECK: STFS killed $f1, 100, $x5
+    STD killed $x3, killed $x5, 100
+    ; CHECK: STD killed $x3, killed $x5, 100
+    BLR8 implicit $lr8, implicit $rm
+...
+---
+# No workaround needed for 64-bit register when calling readsRegister()
+name: testReadsSubRegADDI
+# CHECK: name: testReadsSubRegADDI
+tracksRegLiveness: true
+body: |
+  bb.0.entry:
+    liveins: $x3, $f1, $x5
+    $x3 = ADDI8 $x5, 100
+    ; Following instruction $r3 also reads $x3, ADDI8 can not be erased
+    ; CHECK: $x3 = ADDI8 $x5, 100, implicit-def $r3
+    STW $r3, $x5, 100
+    ; CHECK: STW $r3, $x5, 100
+    STFSX killed $f1, $zero8, $x3
+    ; CHECK: STFS killed $f1, 100, $x5
+    STD $x5, $x5, 100
+    BLR8 implicit $lr8, implicit $rm
+...
diff --git a/test/CodeGen/PowerPC/convert-rr-to-ri-instrs.mir b/test/CodeGen/PowerPC/convert-rr-to-ri-instrs.mir
index e210ec5c523eb0ecc4e9b09e8f48a33286f66fde..eb058bc9b7d8454d6525cdc47dea7ddaab3ca72d 100644
--- a/test/CodeGen/PowerPC/convert-rr-to-ri-instrs.mir
+++ b/test/CodeGen/PowerPC/convert-rr-to-ri-instrs.mir
@@ -1,5 +1,5 @@
-# RUN: llc -run-pass ppc-mi-peepholes -ppc-convert-rr-to-ri %s -o - | FileCheck %s
-# RUN: llc -start-after ppc-mi-peepholes -ppc-late-peephole %s -o - | FileCheck %s --check-prefix=CHECK-LATE
+# RUN: llc -run-pass ppc-mi-peepholes -ppc-convert-rr-to-ri %s -o - -verify-machineinstrs | FileCheck %s
+# RUN: llc -start-after ppc-mi-peepholes -ppc-late-peephole %s -o - -verify-machineinstrs | FileCheck %s --check-prefix=CHECK-LATE
 
 --- |
   ; ModuleID = 'convert-rr-to-ri-instrs.ll'
diff --git a/test/CodeGen/PowerPC/crsave.ll b/test/CodeGen/PowerPC/crsave.ll
index d7be2fd18c9c3818a43cd1b3a8bf5abf56ff9c03..980749a6d4ebd99ee200d3ec8758ebddcc5b0ddf 100644
--- a/test/CodeGen/PowerPC/crsave.ll
+++ b/test/CodeGen/PowerPC/crsave.ll
@@ -1,4 +1,4 @@
-; RUN: llc -O0 -disable-fp-elim -mtriple=powerpc-unknown-linux-gnu -mcpu=g5 < %s | FileCheck %s -check-prefix=PPC32
+; RUN: llc -O0 -frame-pointer=all -mtriple=powerpc-unknown-linux-gnu -mcpu=g5 < %s | FileCheck %s -check-prefix=PPC32
 ; RUN: llc -O0 -mtriple=powerpc64-unknown-linux-gnu -mcpu=g5 < %s | FileCheck %s -check-prefix=PPC64
 ; RUN: llc -O0 -mtriple=powerpc64le-unknown-linux-gnu -verify-machineinstrs < %s | FileCheck %s -check-prefix=PPC64-ELFv2
 
diff --git a/test/CodeGen/PowerPC/ctrloop-shortLoops.ll b/test/CodeGen/PowerPC/ctrloop-shortLoops.ll
index 481ec54e79a4b1556cbb000ca502cc521b29509f..c1ba9214fdaf7eea98ba73f29df2843b2fefd447 100644
--- a/test/CodeGen/PowerPC/ctrloop-shortLoops.ll
+++ b/test/CodeGen/PowerPC/ctrloop-shortLoops.ll
@@ -82,9 +82,8 @@ for.body:                                         ; preds = %entry, %for.body
 
 ; Function Attrs: norecurse nounwind
 define signext i32 @testTripCount2NonSmallLoop() {
-
 ; CHECK-LABEL: testTripCount2NonSmallLoop:
-; CHECK: mtctr
+; CHECK: bge
 ; CHECK: blr
 
 entry:
diff --git a/test/CodeGen/PowerPC/direct-move-profit.ll b/test/CodeGen/PowerPC/direct-move-profit.ll
index 7205d11a105123be445a4522b3f7e6e3a55665c7..2e2da7ce8ae706a9f17683bb838089cc4bc67b48 100644
--- a/test/CodeGen/PowerPC/direct-move-profit.ll
+++ b/test/CodeGen/PowerPC/direct-move-profit.ll
@@ -74,7 +74,7 @@ entry:
 
 }
 
-!0 = !{!"clang version 3.9.0 (http://llvm.org/git/clang.git b88a395e7ba26c0fb96cd99a2a004d76f4f41d0c) (http://llvm.org/git/llvm.git 1ac3fbac0f5b037c17c0b0f9d271c32c4d7ca1b5)"}
+!0 = !{!"clang version 3.9.0"}
 !1 = !{!2, !2, i64 0}
 !2 = !{!"int", !3, i64 0}
 !3 = !{!"omnipotent char", !4, i64 0}
diff --git a/test/CodeGen/PowerPC/empty-functions.ll b/test/CodeGen/PowerPC/empty-functions.ll
index 4f36c49e3bb4f445ec451335d97ed3bb066847f9..f3238bae560e6daa6ae71660404875fe1eab00a4 100644
--- a/test/CodeGen/PowerPC/empty-functions.ll
+++ b/test/CodeGen/PowerPC/empty-functions.ll
@@ -1,5 +1,5 @@
 ; RUN: llc -verify-machineinstrs < %s -mtriple=powerpc-linux-gnu | FileCheck -check-prefix=LINUX-NO-FP %s
-; RUN: llc -verify-machineinstrs < %s -mtriple=powerpc-linux-gnu -disable-fp-elim | FileCheck -check-prefix=LINUX-FP %s
+; RUN: llc -verify-machineinstrs < %s -mtriple=powerpc-linux-gnu -frame-pointer=all | FileCheck -check-prefix=LINUX-FP %s
 
 define void @func() {
 entry:
diff --git a/test/CodeGen/PowerPC/extract-and-store.ll b/test/CodeGen/PowerPC/extract-and-store.ll
index 6426db7923cc7d2301c0726003a49086abd00158..474ba86395ada2ede67c66213c6163c864d805c7 100644
--- a/test/CodeGen/PowerPC/extract-and-store.ll
+++ b/test/CodeGen/PowerPC/extract-and-store.ll
@@ -7,26 +7,66 @@
 ; RUN: llc -mcpu=pwr9 -mtriple=powerpc64le-unkknown-unknown \
 ; RUN:   -ppc-asm-full-reg-names -verify-machineinstrs -O2 < %s | FileCheck %s \
 ; RUN:   --check-prefix=CHECK-P9
-; Function Attrs: norecurse nounwind writeonly
+; RUN: llc -mcpu=pwr9 -mtriple=powerpc64-unkknown-unknown \
+; RUN:   -ppc-asm-full-reg-names -verify-machineinstrs -O2 < %s | FileCheck %s \
+; RUN:   --check-prefix=CHECK-P9-BE
+
+define <2 x i64> @testllv(<2 x i64> returned %a, <2 x i64> %b, i64* nocapture %ap, i64 %Idx) local_unnamed_addr #0 {
+; CHECK-LABEL: testllv:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxswapd vs0, vs34
+; CHECK-NEXT:    sldi r3, r8, 3
+; CHECK-NEXT:    stfdx f0, r7, r3
+; CHECK-NEXT:    blr
+;
+; CHECK-BE-LABEL: testllv:
+; CHECK-BE:       # %bb.0: # %entry
+; CHECK-BE-NEXT:    sldi r3, r8, 3
+; CHECK-BE-NEXT:    stxsdx vs34, r7, r3
+; CHECK-BE-NEXT:    blr
+;
+; CHECK-P9-LABEL: testllv:
+; CHECK-P9:       # %bb.0: # %entry
+; CHECK-P9-NEXT:    xxswapd vs0, vs34
+; CHECK-P9-NEXT:    sldi r3, r8, 3
+; CHECK-P9-NEXT:    stfdx f0, r7, r3
+; CHECK-P9-NEXT:    blr
+;
+; CHECK-P9-BE-LABEL: testllv:
+; CHECK-P9-BE:       # %bb.0: # %entry
+; CHECK-P9-BE-NEXT:    sldi r3, r8, 3
+; CHECK-P9-BE-NEXT:    stxsdx vs34, r7, r3
+; CHECK-P9-BE-NEXT:    blr
+entry:
+  %vecext = extractelement <2 x i64> %a, i32 0
+  %arrayidx = getelementptr inbounds i64, i64* %ap, i64 %Idx
+  store i64 %vecext, i64* %arrayidx, align 8
+  ret <2 x i64> %a
+}
+
 define <2 x i64> @testll0(<2 x i64> returned %a, <2 x i64> %b, i64* nocapture %ap) local_unnamed_addr #0 {
 ; CHECK-LABEL: testll0:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    xxswapd vs0, vs34
-; CHECK-NEXT:    mfvsrd r3, f0
-; CHECK-NEXT:    std r3, 24(r7)
+; CHECK-NEXT:    stfd f0, 24(r7)
 ; CHECK-NEXT:    blr
 ;
 ; CHECK-BE-LABEL: testll0:
 ; CHECK-BE:       # %bb.0: # %entry
-; CHECK-BE-NEXT:    mfvsrd r3, vs34
-; CHECK-BE-NEXT:    std r3, 24(r7)
+; CHECK-BE-NEXT:    addi r3, r7, 24
+; CHECK-BE-NEXT:    stxsdx vs34, 0, r3
 ; CHECK-BE-NEXT:    blr
 ;
 ; CHECK-P9-LABEL: testll0:
 ; CHECK-P9:       # %bb.0: # %entry
-; CHECK-P9-NEXT:    mfvsrld r3, vs34
-; CHECK-P9-NEXT:    std r3, 24(r7)
+; CHECK-P9-NEXT:    xxswapd vs0, vs34
+; CHECK-P9-NEXT:    stfd f0, 24(r7)
 ; CHECK-P9-NEXT:    blr
+;
+; CHECK-P9-BE-LABEL: testll0:
+; CHECK-P9-BE:       # %bb.0: # %entry
+; CHECK-P9-BE-NEXT:    stxsd v2, 24(r7)
+; CHECK-P9-BE-NEXT:    blr
 entry:
   %vecext = extractelement <2 x i64> %a, i32 0
   %arrayidx = getelementptr inbounds i64, i64* %ap, i64 3
@@ -38,22 +78,26 @@ entry:
 define <2 x i64> @testll1(<2 x i64> returned %a, i64 %b, i64* nocapture %ap) local_unnamed_addr #0 {
 ; CHECK-LABEL: testll1:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    mfvsrd r3, vs34
-; CHECK-NEXT:    std r3, 24(r6)
+; CHECK-NEXT:    addi r3, r6, 24
+; CHECK-NEXT:    stxsdx vs34, 0, r3
 ; CHECK-NEXT:    blr
 ;
 ; CHECK-BE-LABEL: testll1:
 ; CHECK-BE:       # %bb.0: # %entry
 ; CHECK-BE-NEXT:    xxswapd vs0, vs34
-; CHECK-BE-NEXT:    mfvsrd r3, f0
-; CHECK-BE-NEXT:    std r3, 24(r6)
+; CHECK-BE-NEXT:    stfd f0, 24(r6)
 ; CHECK-BE-NEXT:    blr
 ;
 ; CHECK-P9-LABEL: testll1:
 ; CHECK-P9:       # %bb.0: # %entry
-; CHECK-P9-NEXT:    mfvsrd r3, vs34
-; CHECK-P9-NEXT:    std r3, 24(r6)
+; CHECK-P9-NEXT:    stxsd v2, 24(r6)
 ; CHECK-P9-NEXT:    blr
+;
+; CHECK-P9-BE-LABEL: testll1:
+; CHECK-P9-BE:       # %bb.0: # %entry
+; CHECK-P9-BE-NEXT:    xxswapd vs0, vs34
+; CHECK-P9-BE-NEXT:    stfd f0, 24(r6)
+; CHECK-P9-BE-NEXT:    blr
 entry:
   %vecext = extractelement <2 x i64> %a, i32 1
   %arrayidx = getelementptr inbounds i64, i64* %ap, i64 3
@@ -61,7 +105,39 @@ entry:
   ret <2 x i64> %a
 }
 
-; Function Attrs: norecurse nounwind writeonly
+define <2 x double> @testdv(<2 x double> returned %a, <2 x double> %b, double* nocapture %ap, i64 %Idx) local_unnamed_addr #0 {
+; CHECK-LABEL: testdv:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xxswapd vs0, vs34
+; CHECK-NEXT:    sldi r3, r8, 3
+; CHECK-NEXT:    stfdx f0, r7, r3
+; CHECK-NEXT:    blr
+;
+; CHECK-BE-LABEL: testdv:
+; CHECK-BE:       # %bb.0: # %entry
+; CHECK-BE-NEXT:    sldi r3, r8, 3
+; CHECK-BE-NEXT:    stxsdx vs34, r7, r3
+; CHECK-BE-NEXT:    blr
+;
+; CHECK-P9-LABEL: testdv:
+; CHECK-P9:       # %bb.0: # %entry
+; CHECK-P9-NEXT:    xxswapd vs0, vs34
+; CHECK-P9-NEXT:    sldi r3, r8, 3
+; CHECK-P9-NEXT:    stfdx f0, r7, r3
+; CHECK-P9-NEXT:    blr
+;
+; CHECK-P9-BE-LABEL: testdv:
+; CHECK-P9-BE:       # %bb.0: # %entry
+; CHECK-P9-BE-NEXT:    sldi r3, r8, 3
+; CHECK-P9-BE-NEXT:    stxsdx vs34, r7, r3
+; CHECK-P9-BE-NEXT:    blr
+entry:
+  %vecext = extractelement <2 x double> %a, i32 0
+  %arrayidx = getelementptr inbounds double, double* %ap, i64 %Idx
+  store double %vecext, double* %arrayidx, align 8
+  ret <2 x double> %a
+}
+
 define <2 x double> @testd0(<2 x double> returned %a, <2 x double> %b, double* nocapture %ap) local_unnamed_addr #0 {
 ; CHECK-LABEL: testd0:
 ; CHECK:       # %bb.0: # %entry
@@ -80,6 +156,11 @@ define <2 x double> @testd0(<2 x double> returned %a, <2 x double> %b, double* n
 ; CHECK-P9-NEXT:    xxswapd vs0, vs34
 ; CHECK-P9-NEXT:    stfd f0, 24(r7)
 ; CHECK-P9-NEXT:    blr
+;
+; CHECK-P9-BE-LABEL: testd0:
+; CHECK-P9-BE:       # %bb.0: # %entry
+; CHECK-P9-BE-NEXT:    stxsd v2, 24(r7)
+; CHECK-P9-BE-NEXT:    blr
 entry:
   %vecext = extractelement <2 x double> %a, i32 0
   %arrayidx = getelementptr inbounds double, double* %ap, i64 3
@@ -105,6 +186,12 @@ define <2 x double> @testd1(<2 x double> returned %a, <2 x double> %b, double* n
 ; CHECK-P9:       # %bb.0: # %entry
 ; CHECK-P9-NEXT:    stxsd v2, 24(r7)
 ; CHECK-P9-NEXT:    blr
+;
+; CHECK-P9-BE-LABEL: testd1:
+; CHECK-P9-BE:       # %bb.0: # %entry
+; CHECK-P9-BE-NEXT:    xxswapd vs0, vs34
+; CHECK-P9-BE-NEXT:    stfd f0, 24(r7)
+; CHECK-P9-BE-NEXT:    blr
 entry:
   %vecext = extractelement <2 x double> %a, i32 1
   %arrayidx = getelementptr inbounds double, double* %ap, i64 3
@@ -116,23 +203,31 @@ entry:
 define <4 x float> @testf0(<4 x float> returned %a, <4 x float> %b, float* nocapture %ap) local_unnamed_addr #0 {
 ; CHECK-LABEL: testf0:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    xxsldwi vs0, vs34, vs34, 3
-; CHECK-NEXT:    xscvspdpn f0, vs0
-; CHECK-NEXT:    stfs f0, 12(r7)
+; CHECK-NEXT:    xxsldwi vs0, vs34, vs34, 2
+; CHECK-NEXT:    addi r3, r7, 12
+; CHECK-NEXT:    stfiwx f0, 0, r3
 ; CHECK-NEXT:    blr
 ;
 ; CHECK-BE-LABEL: testf0:
 ; CHECK-BE:       # %bb.0: # %entry
-; CHECK-BE-NEXT:    xscvspdpn f0, vs34
-; CHECK-BE-NEXT:    stfs f0, 12(r7)
+; CHECK-BE-NEXT:    xxsldwi vs0, vs34, vs34, 3
+; CHECK-BE-NEXT:    addi r3, r7, 12
+; CHECK-BE-NEXT:    stfiwx f0, 0, r3
 ; CHECK-BE-NEXT:    blr
 ;
 ; CHECK-P9-LABEL: testf0:
 ; CHECK-P9:       # %bb.0: # %entry
-; CHECK-P9-NEXT:    xxsldwi vs0, vs34, vs34, 3
-; CHECK-P9-NEXT:    xscvspdpn f0, vs0
-; CHECK-P9-NEXT:    stfs f0, 12(r7)
+; CHECK-P9-NEXT:    xxsldwi vs0, vs34, vs34, 2
+; CHECK-P9-NEXT:    addi r3, r7, 12
+; CHECK-P9-NEXT:    stfiwx f0, 0, r3
 ; CHECK-P9-NEXT:    blr
+;
+; CHECK-P9-BE-LABEL: testf0:
+; CHECK-P9-BE:       # %bb.0: # %entry
+; CHECK-P9-BE-NEXT:    xxsldwi vs0, vs34, vs34, 3
+; CHECK-P9-BE-NEXT:    addi r3, r7, 12
+; CHECK-P9-BE-NEXT:    stfiwx f0, 0, r3
+; CHECK-P9-BE-NEXT:    blr
 entry:
   %vecext = extractelement <4 x float> %a, i32 0
   %arrayidx = getelementptr inbounds float, float* %ap, i64 3
@@ -144,24 +239,29 @@ entry:
 define <4 x float> @testf1(<4 x float> returned %a, <4 x float> %b, float* nocapture %ap) local_unnamed_addr #0 {
 ; CHECK-LABEL: testf1:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    xxswapd vs0, vs34
-; CHECK-NEXT:    xscvspdpn f0, vs0
-; CHECK-NEXT:    stfs f0, 12(r7)
+; CHECK-NEXT:    xxsldwi vs0, vs34, vs34, 1
+; CHECK-NEXT:    addi r3, r7, 12
+; CHECK-NEXT:    stfiwx f0, 0, r3
 ; CHECK-NEXT:    blr
 ;
 ; CHECK-BE-LABEL: testf1:
 ; CHECK-BE:       # %bb.0: # %entry
-; CHECK-BE-NEXT:    xxsldwi vs0, vs34, vs34, 1
-; CHECK-BE-NEXT:    xscvspdpn f0, vs0
-; CHECK-BE-NEXT:    stfs f0, 12(r7)
+; CHECK-BE-NEXT:    addi r3, r7, 12
+; CHECK-BE-NEXT:    stxsiwx vs34, 0, r3
 ; CHECK-BE-NEXT:    blr
 ;
 ; CHECK-P9-LABEL: testf1:
 ; CHECK-P9:       # %bb.0: # %entry
-; CHECK-P9-NEXT:    xxswapd vs0, vs34
-; CHECK-P9-NEXT:    xscvspdpn f0, vs0
-; CHECK-P9-NEXT:    stfs f0, 12(r7)
+; CHECK-P9-NEXT:    xxsldwi vs0, vs34, vs34, 1
+; CHECK-P9-NEXT:    addi r3, r7, 12
+; CHECK-P9-NEXT:    stfiwx f0, 0, r3
 ; CHECK-P9-NEXT:    blr
+;
+; CHECK-P9-BE-LABEL: testf1:
+; CHECK-P9-BE:       # %bb.0: # %entry
+; CHECK-P9-BE-NEXT:    addi r3, r7, 12
+; CHECK-P9-BE-NEXT:    stxsiwx vs34, 0, r3
+; CHECK-P9-BE-NEXT:    blr
 entry:
   %vecext = extractelement <4 x float> %a, i32 1
   %arrayidx = getelementptr inbounds float, float* %ap, i64 3
@@ -173,24 +273,29 @@ entry:
 define <4 x float> @testf2(<4 x float> returned %a, <4 x float> %b, float* nocapture %ap) local_unnamed_addr #0 {
 ; CHECK-LABEL: testf2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    xxsldwi vs0, vs34, vs34, 1
-; CHECK-NEXT:    xscvspdpn f0, vs0
-; CHECK-NEXT:    stfs f0, 12(r7)
+; CHECK-NEXT:    addi r3, r7, 12
+; CHECK-NEXT:    stxsiwx vs34, 0, r3
 ; CHECK-NEXT:    blr
 ;
 ; CHECK-BE-LABEL: testf2:
 ; CHECK-BE:       # %bb.0: # %entry
-; CHECK-BE-NEXT:    xxswapd vs0, vs34
-; CHECK-BE-NEXT:    xscvspdpn f0, vs0
-; CHECK-BE-NEXT:    stfs f0, 12(r7)
+; CHECK-BE-NEXT:    xxsldwi vs0, vs34, vs34, 1
+; CHECK-BE-NEXT:    addi r3, r7, 12
+; CHECK-BE-NEXT:    stfiwx f0, 0, r3
 ; CHECK-BE-NEXT:    blr
 ;
 ; CHECK-P9-LABEL: testf2:
 ; CHECK-P9:       # %bb.0: # %entry
-; CHECK-P9-NEXT:    xxsldwi vs0, vs34, vs34, 1
-; CHECK-P9-NEXT:    xscvspdpn f0, vs0
-; CHECK-P9-NEXT:    stfs f0, 12(r7)
+; CHECK-P9-NEXT:    addi r3, r7, 12
+; CHECK-P9-NEXT:    stxsiwx vs34, 0, r3
 ; CHECK-P9-NEXT:    blr
+;
+; CHECK-P9-BE-LABEL: testf2:
+; CHECK-P9-BE:       # %bb.0: # %entry
+; CHECK-P9-BE-NEXT:    xxsldwi vs0, vs34, vs34, 1
+; CHECK-P9-BE-NEXT:    addi r3, r7, 12
+; CHECK-P9-BE-NEXT:    stfiwx f0, 0, r3
+; CHECK-P9-BE-NEXT:    blr
 entry:
   %vecext = extractelement <4 x float> %a, i32 2
   %arrayidx = getelementptr inbounds float, float* %ap, i64 3
@@ -202,22 +307,31 @@ entry:
 define <4 x float> @testf3(<4 x float> returned %a, <4 x float> %b, float* nocapture %ap) local_unnamed_addr #0 {
 ; CHECK-LABEL: testf3:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    xscvspdpn f0, vs34
-; CHECK-NEXT:    stfs f0, 12(r7)
+; CHECK-NEXT:    xxsldwi vs0, vs34, vs34, 3
+; CHECK-NEXT:    addi r3, r7, 12
+; CHECK-NEXT:    stfiwx f0, 0, r3
 ; CHECK-NEXT:    blr
 ;
 ; CHECK-BE-LABEL: testf3:
 ; CHECK-BE:       # %bb.0: # %entry
-; CHECK-BE-NEXT:    xxsldwi vs0, vs34, vs34, 3
-; CHECK-BE-NEXT:    xscvspdpn f0, vs0
-; CHECK-BE-NEXT:    stfs f0, 12(r7)
+; CHECK-BE-NEXT:    xxsldwi vs0, vs34, vs34, 2
+; CHECK-BE-NEXT:    addi r3, r7, 12
+; CHECK-BE-NEXT:    stfiwx f0, 0, r3
 ; CHECK-BE-NEXT:    blr
 ;
 ; CHECK-P9-LABEL: testf3:
 ; CHECK-P9:       # %bb.0: # %entry
-; CHECK-P9-NEXT:    xscvspdpn f0, vs34
-; CHECK-P9-NEXT:    stfs f0, 12(r7)
+; CHECK-P9-NEXT:    xxsldwi vs0, vs34, vs34, 3
+; CHECK-P9-NEXT:    addi r3, r7, 12
+; CHECK-P9-NEXT:    stfiwx f0, 0, r3
 ; CHECK-P9-NEXT:    blr
+;
+; CHECK-P9-BE-LABEL: testf3:
+; CHECK-P9-BE:       # %bb.0: # %entry
+; CHECK-P9-BE-NEXT:    xxsldwi vs0, vs34, vs34, 2
+; CHECK-P9-BE-NEXT:    addi r3, r7, 12
+; CHECK-P9-BE-NEXT:    stfiwx f0, 0, r3
+; CHECK-P9-BE-NEXT:    blr
 entry:
   %vecext = extractelement <4 x float> %a, i32 3
   %arrayidx = getelementptr inbounds float, float* %ap, i64 3
@@ -229,24 +343,31 @@ entry:
 define <4 x i32> @testi0(<4 x i32> returned %a, <4 x i32> %b, i32* nocapture %ap) local_unnamed_addr #0 {
 ; CHECK-LABEL: testi0:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    xxswapd vs0, vs34
-; CHECK-NEXT:    mfvsrwz r3, f0
-; CHECK-NEXT:    stw r3, 12(r7)
+; CHECK-NEXT:    xxsldwi vs0, vs34, vs34, 2
+; CHECK-NEXT:    addi r3, r7, 12
+; CHECK-NEXT:    stfiwx f0, 0, r3
 ; CHECK-NEXT:    blr
 ;
 ; CHECK-BE-LABEL: testi0:
 ; CHECK-BE:       # %bb.0: # %entry
 ; CHECK-BE-NEXT:    xxsldwi vs0, vs34, vs34, 3
-; CHECK-BE-NEXT:    mfvsrwz r3, f0
-; CHECK-BE-NEXT:    stw r3, 12(r7)
+; CHECK-BE-NEXT:    addi r3, r7, 12
+; CHECK-BE-NEXT:    stfiwx f0, 0, r3
 ; CHECK-BE-NEXT:    blr
 ;
 ; CHECK-P9-LABEL: testi0:
 ; CHECK-P9:       # %bb.0: # %entry
-; CHECK-P9-NEXT:    li r3, 0
-; CHECK-P9-NEXT:    vextuwrx r3, r3, v2
-; CHECK-P9-NEXT:    stw r3, 12(r7)
+; CHECK-P9-NEXT:    xxsldwi vs0, vs34, vs34, 2
+; CHECK-P9-NEXT:    addi r3, r7, 12
+; CHECK-P9-NEXT:    stfiwx f0, 0, r3
 ; CHECK-P9-NEXT:    blr
+;
+; CHECK-P9-BE-LABEL: testi0:
+; CHECK-P9-BE:       # %bb.0: # %entry
+; CHECK-P9-BE-NEXT:    xxsldwi vs0, vs34, vs34, 3
+; CHECK-P9-BE-NEXT:    addi r3, r7, 12
+; CHECK-P9-BE-NEXT:    stfiwx f0, 0, r3
+; CHECK-P9-BE-NEXT:    blr
 entry:
   %vecext = extractelement <4 x i32> %a, i32 0
   %arrayidx = getelementptr inbounds i32, i32* %ap, i64 3
@@ -259,22 +380,28 @@ define <4 x i32> @testi1(<4 x i32> returned %a, <4 x i32> %b, i32* nocapture %ap
 ; CHECK-LABEL: testi1:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    xxsldwi vs0, vs34, vs34, 1
-; CHECK-NEXT:    mfvsrwz r3, f0
-; CHECK-NEXT:    stw r3, 12(r7)
+; CHECK-NEXT:    addi r3, r7, 12
+; CHECK-NEXT:    stfiwx f0, 0, r3
 ; CHECK-NEXT:    blr
 ;
 ; CHECK-BE-LABEL: testi1:
 ; CHECK-BE:       # %bb.0: # %entry
-; CHECK-BE-NEXT:    mfvsrwz r3, vs34
-; CHECK-BE-NEXT:    stw r3, 12(r7)
+; CHECK-BE-NEXT:    addi r3, r7, 12
+; CHECK-BE-NEXT:    stxsiwx vs34, 0, r3
 ; CHECK-BE-NEXT:    blr
 ;
 ; CHECK-P9-LABEL: testi1:
 ; CHECK-P9:       # %bb.0: # %entry
-; CHECK-P9-NEXT:    li r3, 4
-; CHECK-P9-NEXT:    vextuwrx r3, r3, v2
-; CHECK-P9-NEXT:    stw r3, 12(r7)
+; CHECK-P9-NEXT:    xxsldwi vs0, vs34, vs34, 1
+; CHECK-P9-NEXT:    addi r3, r7, 12
+; CHECK-P9-NEXT:    stfiwx f0, 0, r3
 ; CHECK-P9-NEXT:    blr
+;
+; CHECK-P9-BE-LABEL: testi1:
+; CHECK-P9-BE:       # %bb.0: # %entry
+; CHECK-P9-BE-NEXT:    addi r3, r7, 12
+; CHECK-P9-BE-NEXT:    stxsiwx vs34, 0, r3
+; CHECK-P9-BE-NEXT:    blr
 entry:
   %vecext = extractelement <4 x i32> %a, i32 1
   %arrayidx = getelementptr inbounds i32, i32* %ap, i64 3
@@ -286,22 +413,29 @@ entry:
 define <4 x i32> @testi2(<4 x i32> returned %a, <4 x i32> %b, i32* nocapture %ap) local_unnamed_addr #0 {
 ; CHECK-LABEL: testi2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    mfvsrwz r3, vs34
-; CHECK-NEXT:    stw r3, 12(r7)
+; CHECK-NEXT:    addi r3, r7, 12
+; CHECK-NEXT:    stxsiwx vs34, 0, r3
 ; CHECK-NEXT:    blr
 ;
 ; CHECK-BE-LABEL: testi2:
 ; CHECK-BE:       # %bb.0: # %entry
 ; CHECK-BE-NEXT:    xxsldwi vs0, vs34, vs34, 1
-; CHECK-BE-NEXT:    mfvsrwz r3, f0
-; CHECK-BE-NEXT:    stw r3, 12(r7)
+; CHECK-BE-NEXT:    addi r3, r7, 12
+; CHECK-BE-NEXT:    stfiwx f0, 0, r3
 ; CHECK-BE-NEXT:    blr
 ;
 ; CHECK-P9-LABEL: testi2:
 ; CHECK-P9:       # %bb.0: # %entry
-; CHECK-P9-NEXT:    mfvsrwz r3, vs34
-; CHECK-P9-NEXT:    stw r3, 12(r7)
+; CHECK-P9-NEXT:    addi r3, r7, 12
+; CHECK-P9-NEXT:    stxsiwx vs34, 0, r3
 ; CHECK-P9-NEXT:    blr
+;
+; CHECK-P9-BE-LABEL: testi2:
+; CHECK-P9-BE:       # %bb.0: # %entry
+; CHECK-P9-BE-NEXT:    xxsldwi vs0, vs34, vs34, 1
+; CHECK-P9-BE-NEXT:    addi r3, r7, 12
+; CHECK-P9-BE-NEXT:    stfiwx f0, 0, r3
+; CHECK-P9-BE-NEXT:    blr
 entry:
   %vecext = extractelement <4 x i32> %a, i32 2
   %arrayidx = getelementptr inbounds i32, i32* %ap, i64 3
@@ -314,23 +448,30 @@ define <4 x i32> @testi3(<4 x i32> returned %a, <4 x i32> %b, i32* nocapture %ap
 ; CHECK-LABEL: testi3:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    xxsldwi vs0, vs34, vs34, 3
-; CHECK-NEXT:    mfvsrwz r3, f0
-; CHECK-NEXT:    stw r3, 12(r7)
+; CHECK-NEXT:    addi r3, r7, 12
+; CHECK-NEXT:    stfiwx f0, 0, r3
 ; CHECK-NEXT:    blr
 ;
 ; CHECK-BE-LABEL: testi3:
 ; CHECK-BE:       # %bb.0: # %entry
-; CHECK-BE-NEXT:    xxswapd vs0, vs34
-; CHECK-BE-NEXT:    mfvsrwz r3, f0
-; CHECK-BE-NEXT:    stw r3, 12(r7)
+; CHECK-BE-NEXT:    xxsldwi vs0, vs34, vs34, 2
+; CHECK-BE-NEXT:    addi r3, r7, 12
+; CHECK-BE-NEXT:    stfiwx f0, 0, r3
 ; CHECK-BE-NEXT:    blr
 ;
 ; CHECK-P9-LABEL: testi3:
 ; CHECK-P9:       # %bb.0: # %entry
-; CHECK-P9-NEXT:    li r3, 12
-; CHECK-P9-NEXT:    vextuwrx r3, r3, v2
-; CHECK-P9-NEXT:    stw r3, 12(r7)
+; CHECK-P9-NEXT:    xxsldwi vs0, vs34, vs34, 3
+; CHECK-P9-NEXT:    addi r3, r7, 12
+; CHECK-P9-NEXT:    stfiwx f0, 0, r3
 ; CHECK-P9-NEXT:    blr
+;
+; CHECK-P9-BE-LABEL: testi3:
+; CHECK-P9-BE:       # %bb.0: # %entry
+; CHECK-P9-BE-NEXT:    xxsldwi vs0, vs34, vs34, 2
+; CHECK-P9-BE-NEXT:    addi r3, r7, 12
+; CHECK-P9-BE-NEXT:    stfiwx f0, 0, r3
+; CHECK-P9-BE-NEXT:    blr
 entry:
   %vecext = extractelement <4 x i32> %a, i32 3
   %arrayidx = getelementptr inbounds i32, i32* %ap, i64 3
diff --git a/test/CodeGen/PowerPC/f128-aggregates.ll b/test/CodeGen/PowerPC/f128-aggregates.ll
index d671c76f5f48f728016bf4ed2252a49a48c0a38a..9d161037763d92555922fc0b605c1730ad59472a 100644
--- a/test/CodeGen/PowerPC/f128-aggregates.ll
+++ b/test/CodeGen/PowerPC/f128-aggregates.ll
@@ -82,7 +82,6 @@ define fp128 @testStruct_03(%struct.With9fp128params* byval nocapture readonly
                             align 16 %a) {
 ; CHECK-LABEL: testStruct_03:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    lxv v2, 128(r1)
 ; CHECK-NEXT:    std r10, 88(r1)
 ; CHECK-NEXT:    std r9, 80(r1)
 ; CHECK-NEXT:    std r8, 72(r1)
@@ -91,11 +90,11 @@ define fp128 @testStruct_03(%struct.With9fp128params* byval nocapture readonly
 ; CHECK-NEXT:    std r5, 48(r1)
 ; CHECK-NEXT:    std r4, 40(r1)
 ; CHECK-NEXT:    std r3, 32(r1)
+; CHECK-NEXT:    lxv v2, 128(r1)
 ; CHECK-NEXT:    blr
 
 ; CHECK-BE-LABEL: testStruct_03:
 ; CHECK-BE:       # %bb.0: # %entry
-; CHECK-BE-NEXT:    lxv v2, 144(r1)
 ; CHECK-BE-NEXT:    std r10, 104(r1)
 ; CHECK-BE-NEXT:    std r9, 96(r1)
 ; CHECK-BE-NEXT:    std r8, 88(r1)
@@ -104,6 +103,7 @@ define fp128 @testStruct_03(%struct.With9fp128params* byval nocapture readonly
 ; CHECK-BE-NEXT:    std r5, 64(r1)
 ; CHECK-BE-NEXT:    std r4, 56(r1)
 ; CHECK-BE-NEXT:    std r3, 48(r1)
+; CHECK-BE-NEXT:    lxv v2, 144(r1)
 ; CHECK-BE-NEXT:    blr
 entry:
   %a7 = getelementptr inbounds %struct.With9fp128params,
@@ -228,12 +228,12 @@ entry:
 define fp128 @testMixedAggregate_03([4 x i128] %sa.coerce) {
 ; CHECK-LABEL: testMixedAggregate_03:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-DAG:     mtvsrwa v2, r3
-; CHECK-DAG:     mtvsrdd v3, r6, r5
-; CHECK:         mtvsrd v4, r10
+; CHECK:         mtvsrwa v2, r3
 ; CHECK:         xscvsdqp v2, v2
-; CHECK-DAG:     xscvsdqp v[[REG:[0-9]+]], v4
-; CHECK-DAG:     xsaddqp v2, v3, v2
+; CHECK:         mtvsrdd v3, r6, r5
+; CHECK:         xsaddqp v2, v3, v2
+; CHECK:         mtvsrd v[[REG1:[0-9]+]], r10
+; CHECK:         xscvsdqp v[[REG:[0-9]+]], v[[REG1]]
 ; CHECK:         xsaddqp v2, v2, v[[REG]]
 ; CHECK-NEXT:    blr
 entry:
@@ -260,11 +260,11 @@ define fp128 @testNestedAggregate(%struct.MixedC* byval nocapture readonly align
 ; CHECK-NEXT:    std r7, 64(r1)
 ; CHECK-NEXT:    std r10, 88(r1)
 ; CHECK-NEXT:    std r9, 80(r1)
-; CHECK-NEXT:    lxv v2, 64(r1)
 ; CHECK-NEXT:    std r6, 56(r1)
 ; CHECK-NEXT:    std r5, 48(r1)
 ; CHECK-NEXT:    std r4, 40(r1)
 ; CHECK-NEXT:    std r3, 32(r1)
+; CHECK-NEXT:    lxv v2, 64(r1)
 ; CHECK-NEXT:    blr
 
 ; CHECK-BE-LABEL: testNestedAggregate:
@@ -273,11 +273,11 @@ define fp128 @testNestedAggregate(%struct.MixedC* byval nocapture readonly align
 ; CHECK-BE-NEXT:    std r7, 80(r1)
 ; CHECK-BE-NEXT:    std r10, 104(r1)
 ; CHECK-BE-NEXT:    std r9, 96(r1)
-; CHECK-BE-NEXT:    lxv v2, 80(r1)
 ; CHECK-BE-NEXT:    std r6, 72(r1)
 ; CHECK-BE-NEXT:    std r5, 64(r1)
 ; CHECK-BE-NEXT:    std r4, 56(r1)
 ; CHECK-BE-NEXT:    std r3, 48(r1)
+; CHECK-BE-NEXT:    lxv v2, 80(r1)
 ; CHECK-BE-NEXT:    blr
 entry:
   %c = getelementptr inbounds %struct.MixedC, %struct.MixedC* %a, i64 0, i32 1, i32 1
@@ -337,25 +337,25 @@ entry:
 define fp128 @sum_float128(i32 signext %count, ...) {
 ; CHECK-LABEL: sum_float128:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    addis r11, r2, .LCPI17_0@toc@ha
-; CHECK-NEXT:    cmpwi cr0, r3, 1
 ; CHECK-NEXT:    std r10, 88(r1)
 ; CHECK-NEXT:    std r9, 80(r1)
 ; CHECK-NEXT:    std r8, 72(r1)
 ; CHECK-NEXT:    std r7, 64(r1)
 ; CHECK-NEXT:    std r6, 56(r1)
-; CHECK-NEXT:    std r5, 48(r1)
+; CHECK-NEXT:    cmpwi cr0, r3, 1
 ; CHECK-NEXT:    std r4, 40(r1)
-; CHECK-NEXT:    addi r11, r11, .LCPI17_0@toc@l
-; CHECK-NEXT:    lxvx v2, 0, r11
+; CHECK-NEXT:    addis [[REG:r[0-9]+]], r2, .LCPI17_0@toc@ha
+; CHECK-NEXT:    addi [[REG1:r[0-9]+]], [[REG]], .LCPI17_0@toc@l
+; CHECK-NEXT:    lxvx v2, 0, [[REG1]]
+; CHECK-NEXT:    std r5, 48(r1)
 ; CHECK-NEXT:    bltlr cr0
 ; CHECK-NEXT:  # %bb.1: # %if.end
 ; CHECK-NEXT:    addi r3, r1, 40
 ; CHECK-NEXT:    lxvx v3, 0, r3
 ; CHECK-NEXT:    xsaddqp v2, v3, v2
+; CHECK-NEXT:    addi [[REG2:r[0-9]+]], r1, 72
+; CHECK-NEXT:    std [[REG2]], -8(r1)
 ; CHECK-NEXT:    lxv v3, 16(r3)
-; CHECK-NEXT:    addi r3, r1, 72
-; CHECK-NEXT:    std r3, -8(r1)
 ; CHECK-NEXT:    xsaddqp v2, v2, v3
 ; CHECK-NEXT:    blr
 entry:
diff --git a/test/CodeGen/PowerPC/f128-arith.ll b/test/CodeGen/PowerPC/f128-arith.ll
index 955d59d40d5e086869d1461cb48bcce9294785cd..fde472d810c068f6e44759ac248a4471108c5140 100644
--- a/test/CodeGen/PowerPC/f128-arith.ll
+++ b/test/CodeGen/PowerPC/f128-arith.ll
@@ -283,7 +283,7 @@ define void @qp_powi(fp128* nocapture readonly %a, i32* nocapture readonly %b,
                      fp128* nocapture %res) {
 ; CHECK-LABEL: qp_powi:
 ; CHECK:         lxv v2, 0(r3)
-; CHECK:         lwz r3, 0(r4)
+; CHECK:         lwz r5, 0(r4)
 ; CHECK:         bl __powikf2
 ; CHECK:         blr
 entry:
diff --git a/test/CodeGen/PowerPC/f128-conv.ll b/test/CodeGen/PowerPC/f128-conv.ll
index ef433bd3e45ecbd00f4d313eadb6a7b6e552a1ab..4c64341d63497203eae476ae34eb7071d6e2e035 100644
--- a/test/CodeGen/PowerPC/f128-conv.ll
+++ b/test/CodeGen/PowerPC/f128-conv.ll
@@ -444,10 +444,10 @@ define void @qpConv2dp_03(double* nocapture %res, i32 signext %idx) {
 ; CHECK-LABEL: qpConv2dp_03:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    addis r5, r2, .LC7@toc@ha
-; CHECK-NEXT:    sldi r4, r4, 3
 ; CHECK-NEXT:    ld r5, .LC7@toc@l(r5)
 ; CHECK-NEXT:    lxvx v2, 0, r5
 ; CHECK-NEXT:    xscvqpdp v2, v2
+; CHECK-NEXT:    sldi r4, r4, 3
 ; CHECK-NEXT:    stxsdx v2, r3, r4
 ; CHECK-NEXT:    blr
 entry:
@@ -517,11 +517,11 @@ define void @qpConv2sp_03(float* nocapture %res, i32 signext %idx) {
 ; CHECK-LABEL: qpConv2sp_03:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    addis r5, r2, .LC7@toc@ha
-; CHECK-NEXT:    sldi r4, r4, 2
 ; CHECK-NEXT:    ld r5, .LC7@toc@l(r5)
 ; CHECK-NEXT:    lxv v2, 48(r5)
 ; CHECK-NEXT:    xscvqpdpo v2, v2
 ; CHECK-NEXT:    xsrsp f0, v2
+; CHECK-NEXT:    sldi r4, r4, 2
 ; CHECK-NEXT:    stfsx f0, r3, r4
 ; CHECK-NEXT:    blr
 entry:
@@ -609,8 +609,8 @@ define void @dpConv2qp_03(fp128* nocapture %res, i32 signext %idx, double %a) {
 ; CHECK-LABEL: dpConv2qp_03:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    xscpsgndp v2, f1, f1
-; CHECK-NEXT:    sldi r4, r4, 4
-; CHECK-NEXT:    xscvdpqp v2, v2
+; CHECK-DAG:     sldi r4, r4, 4
+; CHECK-DAG:     xscvdpqp v2, v2
 ; CHECK-NEXT:    stxvx v2, r3, r4
 ; CHECK-NEXT:    blr
 entry:
@@ -689,8 +689,8 @@ define void @spConv2qp_03(fp128* nocapture %res, i32 signext %idx, float %a) {
 ; CHECK-LABEL: spConv2qp_03:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    xscpsgndp v2, f1, f1
-; CHECK-NEXT:    sldi r4, r4, 4
-; CHECK-NEXT:    xscvdpqp v2, v2
+; CHECK-DAG:     sldi r4, r4, 4
+; CHECK-DAG:     xscvdpqp v2, v2
 ; CHECK-NEXT:    stxvx v2, r3, r4
 ; CHECK-NEXT:    blr
 entry:
diff --git a/test/CodeGen/PowerPC/f128-passByValue.ll b/test/CodeGen/PowerPC/f128-passByValue.ll
index 467e5553ad1f272a25aae5a386b60c0b42dafc02..7d400b85310abb8f2ef64f0a7703c928c4c8eafb 100644
--- a/test/CodeGen/PowerPC/f128-passByValue.ll
+++ b/test/CodeGen/PowerPC/f128-passByValue.ll
@@ -63,8 +63,8 @@ define fp128 @fp128Array(fp128* nocapture readonly %farray,
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    sldi r4, r4, 4
 ; CHECK-NEXT:    lxv v2, 0(r3)
-; CHECK-NEXT:    add r4, r3, r4
-; CHECK-NEXT:    lxv v3, -16(r4)
+; CHECK-NEXT:    add [[REG:r[0-9]+]], r3, r4
+; CHECK-NEXT:    lxv v3, -16([[REG]])
 ; CHECK-NEXT:    xsaddqp v2, v2, v3
 ; CHECK-NEXT:    blr
                          i32 signext %loopcnt, fp128* nocapture readnone %sum) {
@@ -85,7 +85,6 @@ define fp128 @maxVecParam(fp128 %p1, fp128 %p2, fp128 %p3, fp128 %p4, fp128 %p5,
 ; CHECK-LABEL: maxVecParam:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    xsaddqp v2, v2, v3
-; CHECK-NEXT:    lxv v[[REG0:[0-9]+]], 224(r1)
 ; CHECK-NEXT:    xsaddqp v2, v2, v4
 ; CHECK-NEXT:    xsaddqp v2, v2, v5
 ; CHECK-NEXT:    xsaddqp v2, v2, v6
@@ -96,6 +95,7 @@ define fp128 @maxVecParam(fp128 %p1, fp128 %p2, fp128 %p3, fp128 %p4, fp128 %p5,
 ; CHECK-NEXT:    xsaddqp v2, v2, v11
 ; CHECK-NEXT:    xsaddqp v2, v2, v12
 ; CHECK-NEXT:    xsaddqp v2, v2, v13
+; CHECK-NEXT:    lxv v[[REG0:[0-9]+]], 224(r1)
 ; CHECK-NEXT:    xssubqp v2, v2, v[[REG0]]
 ; CHECK-NEXT:    blr
                           fp128 %p6, fp128 %p7, fp128 %p8, fp128 %p9, fp128 %p10,
@@ -121,9 +121,9 @@ entry:
 define fp128 @mixParam_01(fp128 %a, i32 signext %i, fp128 %b) {
 ; CHECK-LABEL: mixParam_01:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    mtvsrwa v4, r5
-; CHECK-NEXT:    xsaddqp v2, v2, v3
-; CHECK-NEXT:    xscvsdqp v[[REG0:[0-9]+]], v4
+; CHECK-DAG:     mtvsrwa [[REG1:v[0-9]+]], r5
+; CHECK-DAG:     xsaddqp v2, v2, v3
+; CHECK-NEXT:    xscvsdqp v[[REG0:[0-9]+]], [[REG1]]
 ; CHECK-NEXT:    xsaddqp v2, v2, v[[REG0]]
 ; CHECK-NEXT:    blr
 entry:
@@ -136,8 +136,8 @@ entry:
 define fastcc fp128 @mixParam_01f(fp128 %a, i32 signext %i, fp128 %b) {
 ; CHECK-LABEL: mixParam_01f:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    mtvsrwa v[[REG0:[0-9]+]], r3
-; CHECK-NEXT:    xsaddqp v2, v2, v3
+; CHECK-DAG:     mtvsrwa v[[REG0:[0-9]+]], r3
+; CHECK-DAG:     xsaddqp v2, v2, v3
 ; CHECK-NEXT:    xscvsdqp v[[REG1:[0-9]+]], v[[REG0]]
 ; CHECK-NEXT:    xsaddqp v2, v2, v[[REG1]]
 ; CHECK-NEXT:    blr
@@ -152,17 +152,17 @@ entry:
 define fp128 @mixParam_02(fp128 %p1, double %p2, i64* nocapture %p3,
 ; CHECK-LABEL: mixParam_02:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-DAG:     lwz r3, 96(r1)
+; CHECK:         lwz r3, 96(r1)
 ; CHECK:         add r4, r7, r9
-; CHECK-NEXT:    xscpsgndp v[[REG0:[0-9]+]], f1, f1
-; CHECK-DAG:     add r4, r4, r10
+; CHECK:         add r4, r4, r10
+; CHECK:         add r3, r4, r3
+; CHECK:         clrldi r3, r3, 32
+; CHECK:         std r3, 0(r6)
+; CHECK:         lxv v[[REG1:[0-9]+]], 0(r8)
+; CHECK:         xscpsgndp v[[REG0:[0-9]+]], f1, f1
 ; CHECK:         xscvdpqp v[[REG0]], v[[REG0]]
-; CHECK-NEXT:    add r3, r4, r3
-; CHECK-NEXT:    clrldi r3, r3, 32
-; CHECK-NEXT:    std r3, 0(r6)
-; CHECK-NEXT:    lxv v[[REG1:[0-9]+]], 0(r8)
-; CHECK-NEXT:    xsaddqp v2, v[[REG1]], v2
-; CHECK-NEXT:    xsaddqp v2, v2, v3
+; CHECK:         xsaddqp v2, v[[REG1]], v2
+; CHECK:         xsaddqp v2, v2, v3
 ; CHECK-NEXT:    blr
                           i16 signext %p4, fp128* nocapture readonly %p5,
                           i32 signext %p6, i8 zeroext %p7, i32 zeroext %p8) {
@@ -186,13 +186,13 @@ define fastcc fp128 @mixParam_02f(fp128 %p1, double %p2, i64* nocapture %p3,
 ; CHECK-LABEL: mixParam_02f:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    add r4, r4, r6
-; CHECK-NEXT:    xscpsgndp v[[REG0:[0-9]+]], f1, f1
 ; CHECK-NEXT:    add r4, r4, r7
-; CHECK-NEXT:    xscvdpqp v[[REG0]], v[[REG0]]
 ; CHECK-NEXT:    add r4, r4, r8
 ; CHECK-NEXT:    clrldi r4, r4, 32
-; CHECK-NEXT:    std r4, 0(r3)
-; CHECK-NEXT:    lxv v[[REG1:[0-9]+]], 0(r5)
+; CHECK-DAG:     std r4, 0(r3)
+; CHECK-DAG:     lxv v[[REG1:[0-9]+]], 0(r5)
+; CHECK-NEXT:    xscpsgndp v[[REG0:[0-9]+]], f1, f1
+; CHECK-NEXT:    xscvdpqp v[[REG0]], v[[REG0]]
 ; CHECK-NEXT:    xsaddqp v2, v[[REG1]], v2
 ; CHECK-NEXT:    xsaddqp v2, v2, v[[REG0]] 
 ; CHECK-NEXT:    blr
@@ -219,11 +219,11 @@ define void @mixParam_03(fp128 %f1, double* nocapture %d1, <4 x i32> %vec1,
 ; CHECK-LABEL: mixParam_03:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-DAG:     ld r3, 104(r1)
-; CHECK-DAG:     mtvsrwa v[[REG2:[0-9]+]], r10
 ; CHECK-DAG:     stxv v2, 0(r9)
-; CHECK-DAG:     xscvsdqp v[[REG1:[0-9]+]], v[[REG2]]
 ; CHECK:         stxvx v3, 0, r3
-; CHECK-NEXT:    lxv v2, 0(r9)
+; CHECK:         mtvsrwa v[[REG2:[0-9]+]], r10
+; CHECK-DAG:     xscvsdqp v[[REG1:[0-9]+]], v[[REG2]]
+; CHECK-DAG:     lxv v2, 0(r9)
 ; CHECK-NEXT:    xsaddqp v2, v2, v[[REG1]]
 ; CHECK-NEXT:    xscvqpdp v2, v2
 ; CHECK-NEXT:    stxsd v2, 0(r5)
@@ -245,10 +245,10 @@ entry:
 define fastcc void @mixParam_03f(fp128 %f1, double* nocapture %d1, <4 x i32> %vec1,
 ; CHECK-LABEL: mixParam_03f:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    mtvsrwa v[[REG0:[0-9]+]], r5
-; CHECK-NEXT:    stxv v[[REG1:[0-9]+]], 0(r4)
-; CHECK-NEXT:    stxv v[[REG2:[0-9]+]], 0(r7)
-; CHECK-NEXT:    lxv v[[REG1]], 0(r4)
+; CHECK-DAG:     mtvsrwa v[[REG0:[0-9]+]], r5
+; CHECK-DAG:     stxv v[[REG1:[0-9]+]], 0(r4)
+; CHECK-DAG:     stxv v[[REG2:[0-9]+]], 0(r7)
+; CHECK-DAG:     lxv v[[REG1]], 0(r4)
 ; CHECK-NEXT:    xscvsdqp v[[REG3:[0-9]+]], v[[REG0]]
 ; CHECK-NEXT:    xsaddqp v[[REG4:[0-9]+]], v[[REG1]], v[[REG3]]
 ; CHECK-NEXT:    xscvqpdp v2, v[[REG4]]
diff --git a/test/CodeGen/PowerPC/f128-truncateNconv.ll b/test/CodeGen/PowerPC/f128-truncateNconv.ll
index d346683d502263ffbb8e4b9f5bdfe5604b6cbca0..ebded9dcccbe5cdabd34049b7e362ec1653a33bd 100644
--- a/test/CodeGen/PowerPC/f128-truncateNconv.ll
+++ b/test/CodeGen/PowerPC/f128-truncateNconv.ll
@@ -53,10 +53,10 @@ entry:
   ret i64 %conv
 
 ; CHECK-LABEL: qpConv2sdw_03
+; CHECK: lxv v[[REG:[0-9]+]], 0(r3)
 ; CHECK: addis r[[REG0:[0-9]+]], r2, .LC0@toc@ha
-; CHECK-DAG: ld r[[REG0]], .LC0@toc@l(r[[REG0]])
-; CHECK-DAG: lxv v[[REG1:[0-9]+]], 16(r[[REG0]])
-; CHECK-DAG: lxv v[[REG:[0-9]+]], 0(r3)
+; CHECK: ld r[[REG0]], .LC0@toc@l(r[[REG0]])
+; CHECK: lxv v[[REG1:[0-9]+]], 16(r[[REG0]])
 ; CHECK: xsaddqp v[[REG]], v[[REG]], v[[REG1]]
 ; CHECK-NEXT: xscvqpsdz v[[CONV:[0-9]+]], v[[REG]]
 ; CHECK-NEXT: mfvsrd r3, v[[CONV]]
@@ -97,7 +97,7 @@ entry:
 
 ; CHECK-LABEL: qpConv2sdw_testXForm
 ; CHECK: xscvqpsdz v[[CONV:[0-9]+]],
-; CHECK-NEXT: stxsdx v[[CONV]], r3, r4
+; CHECK: stxsdx v[[CONV]], r3, r4
 ; CHECK-NEXT: blr
 }
 
@@ -146,10 +146,10 @@ entry:
   ret i64 %conv
 
 ; CHECK-LABEL: qpConv2udw_03
+; CHECK: lxv v[[REG:[0-9]+]], 0(r3)
 ; CHECK: addis r[[REG0:[0-9]+]], r2, .LC0@toc@ha
 ; CHECK-DAG: ld r[[REG0]], .LC0@toc@l(r[[REG0]])
 ; CHECK-DAG: lxv v[[REG1:[0-9]+]], 16(r[[REG0]])
-; CHECK-DAG: lxv v[[REG:[0-9]+]], 0(r3)
 ; CHECK: xsaddqp v[[REG]], v[[REG]], v[[REG1]]
 ; CHECK-NEXT: xscvqpudz v[[CONV:[0-9]+]], v[[REG]]
 ; CHECK-NEXT: mfvsrd r3, v[[CONV]]
@@ -190,7 +190,7 @@ entry:
 
 ; CHECK-LABEL: qpConv2udw_testXForm
 ; CHECK: xscvqpudz v[[CONV:[0-9]+]],
-; CHECK-NEXT: stxsdx v[[CONV]], r3, r4
+; CHECK: stxsdx v[[CONV]], r3, r4
 ; CHECK-NEXT: blr
 }
 
@@ -240,10 +240,10 @@ entry:
   ret i32 %conv
 
 ; CHECK-LABEL: qpConv2sw_03
+; CHECK: lxv v[[REG:[0-9]+]], 0(r3)
 ; CHECK: addis r[[REG0:[0-9]+]], r2, .LC0@toc@ha
 ; CHECK-DAG: ld r[[REG0]], .LC0@toc@l(r[[REG0]])
 ; CHECK-DAG: lxv v[[REG1:[0-9]+]], 16(r[[REG0]])
-; CHECK-DAG: lxv v[[REG:[0-9]+]], 0(r3)
 ; CHECK-NEXT: xsaddqp v[[ADD:[0-9]+]], v[[REG]], v[[REG1]]
 ; CHECK-NEXT: xscvqpswz v[[CONV:[0-9]+]], v[[ADD]]
 ; CHECK-NEXT: mfvsrwz r[[REG2:[0-9]+]], v[[CONV]]
@@ -316,10 +316,10 @@ entry:
   ret i32 %conv
 
 ; CHECK-LABEL: qpConv2uw_03
+; CHECK: lxv v[[REG:[0-9]+]], 0(r3)
 ; CHECK: addis r[[REG0:[0-9]+]], r2, .LC0@toc@ha
 ; CHECK-DAG: ld r[[REG0]], .LC0@toc@l(r[[REG0]])
 ; CHECK-DAG: lxv v[[REG1:[0-9]+]], 16(r[[REG0]])
-; CHECK-DAG: lxv v[[REG:[0-9]+]], 0(r3)
 ; CHECK-NEXT: xsaddqp v[[ADD:[0-9]+]], v[[REG]], v[[REG1]]
 ; CHECK-NEXT: xscvqpuwz v[[CONV:[0-9]+]], v[[ADD]]
 ; CHECK-NEXT: mfvsrwz r3, v[[CONV]]
@@ -355,7 +355,7 @@ define signext i16 @qpConv2shw(fp128* nocapture readonly %a) {
 ; CHECK-NEXT:    lxv v2, 0(r3)
 ; CHECK-NEXT:    xscvqpswz v2, v2
 ; CHECK-NEXT:    mfvsrwz r3, v2
-; CHECK-NEXT:    extsh r3, r3
+; CHECK-NEXT:    extsw r3, r3
 ; CHECK-NEXT:    blr
 entry:
   %0 = load fp128, fp128* %a, align 16
@@ -386,14 +386,14 @@ entry:
 define signext i16 @qpConv2shw_03(fp128* nocapture readonly %a) {
 ; CHECK-LABEL: qpConv2shw_03:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    addis r4, r2, .LC0@toc@ha
 ; CHECK-NEXT:    lxv v2, 0(r3)
-; CHECK-NEXT:    ld r4, .LC0@toc@l(r4)
-; CHECK-NEXT:    lxv v3, 16(r4)
+; CHECK-NEXT:    addis [[REG:r[0-9]+]], r2, .LC0@toc@ha
+; CHECK-NEXT:    ld [[REG1:r[0-9]+]], .LC0@toc@l([[REG]])
+; CHECK-NEXT:    lxv v3, 16([[REG1]])
 ; CHECK-NEXT:    xsaddqp v2, v2, v3
 ; CHECK-NEXT:    xscvqpswz v2, v2
 ; CHECK-NEXT:    mfvsrwz r3, v2
-; CHECK-NEXT:    extsh r3, r3
+; CHECK-NEXT:    extsw r3, r3
 ; CHECK-NEXT:    blr
 entry:
   %0 = load fp128, fp128* %a, align 16
@@ -463,10 +463,10 @@ entry:
 define zeroext i16 @qpConv2uhw_03(fp128* nocapture readonly %a) {
 ; CHECK-LABEL: qpConv2uhw_03:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    addis r4, r2, .LC0@toc@ha
 ; CHECK-NEXT:    lxv v2, 0(r3)
-; CHECK-NEXT:    ld r4, .LC0@toc@l(r4)
-; CHECK-NEXT:    lxv v3, 16(r4)
+; CHECK-NEXT:    addis [[REG:r[0-9]+]], r2, .LC0@toc@ha
+; CHECK-NEXT:    ld [[REG1:r[0-9]+]], .LC0@toc@l([[REG]])
+; CHECK-NEXT:    lxv v3, 16([[REG1]])
 ; CHECK-NEXT:    xsaddqp v2, v2, v3
 ; CHECK-NEXT:    xscvqpswz v2, v2
 ; CHECK-NEXT:    mfvsrwz r3, v2
@@ -509,7 +509,7 @@ define signext i8 @qpConv2sb(fp128* nocapture readonly %a) {
 ; CHECK-NEXT:    lxv v2, 0(r3)
 ; CHECK-NEXT:    xscvqpswz v2, v2
 ; CHECK-NEXT:    mfvsrwz r3, v2
-; CHECK-NEXT:    extsb r3, r3
+; CHECK-NEXT:    extsw r3, r3
 ; CHECK-NEXT:    blr
 entry:
   %0 = load fp128, fp128* %a, align 16
@@ -540,14 +540,14 @@ entry:
 define signext i8 @qpConv2sb_03(fp128* nocapture readonly %a) {
 ; CHECK-LABEL: qpConv2sb_03:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    addis r4, r2, .LC0@toc@ha
 ; CHECK-NEXT:    lxv v2, 0(r3)
-; CHECK-NEXT:    ld r4, .LC0@toc@l(r4)
-; CHECK-NEXT:    lxv v3, 16(r4)
+; CHECK-NEXT:    addis [[REG:r[0-9]+]], r2, .LC0@toc@ha
+; CHECK-NEXT:    ld [[REG1:r[0-9]+]], .LC0@toc@l([[REG]])
+; CHECK-NEXT:    lxv v3, 16([[REG1]])
 ; CHECK-NEXT:    xsaddqp v2, v2, v3
 ; CHECK-NEXT:    xscvqpswz v2, v2
 ; CHECK-NEXT:    mfvsrwz r3, v2
-; CHECK-NEXT:    extsb r3, r3
+; CHECK-NEXT:    extsw r3, r3
 ; CHECK-NEXT:    blr
 entry:
   %0 = load fp128, fp128* %a, align 16
@@ -617,10 +617,10 @@ entry:
 define zeroext i8 @qpConv2ub_03(fp128* nocapture readonly %a) {
 ; CHECK-LABEL: qpConv2ub_03:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    addis r4, r2, .LC0@toc@ha
 ; CHECK-NEXT:    lxv v2, 0(r3)
-; CHECK-NEXT:    ld r4, .LC0@toc@l(r4)
-; CHECK-NEXT:    lxv v3, 16(r4)
+; CHECK-NEXT:    addis [[REG:r[0-9]+]], r2, .LC0@toc@ha
+; CHECK-NEXT:    ld [[REG1:r[0-9]+]], .LC0@toc@l([[REG]])
+; CHECK-NEXT:    lxv v3, 16([[REG1]])
 ; CHECK-NEXT:    xsaddqp v2, v2, v3
 ; CHECK-NEXT:    xscvqpswz v2, v2
 ; CHECK-NEXT:    mfvsrwz r3, v2
diff --git a/test/CodeGen/PowerPC/fast-isel-cmp-imm.ll b/test/CodeGen/PowerPC/fast-isel-cmp-imm.ll
index 004b82f09b87aee3d7e3397c20bc7e35599b1a33..df8d37d4d3675e4644cb1a869a35d51dfb4fbbb9 100644
--- a/test/CodeGen/PowerPC/fast-isel-cmp-imm.ll
+++ b/test/CodeGen/PowerPC/fast-isel-cmp-imm.ll
@@ -1,17 +1,21 @@
-; FIXME: FastISel currently returns false if it hits code that uses VSX
-; registers and with -fast-isel-abort=1 turned on the test case will then fail.
-; When fastisel better supports VSX fix up this test case.
-;
 ; RUN: llc < %s -O0 -verify-machineinstrs -fast-isel-abort=1 -mtriple=powerpc64-unknown-linux-gnu -mcpu=pwr7 -mattr=-vsx | FileCheck %s --check-prefix=ELF64
+; RUN: llc < %s -O0 -verify-machineinstrs -fast-isel-abort=1 -mtriple=powerpc64le-unknown-linux-gnu -mattr=+vsx | FileCheck %s --check-prefix=VSX
 ; RUN: llc < %s -O0 -verify-machineinstrs -fast-isel-abort=1 -mtriple=powerpc-unknown-linux-gnu -mcpu=e500 -mattr=spe | FileCheck %s --check-prefix=SPE
+
+declare void @foo()
+
 define void @t1a(float %a) nounwind {
 entry:
-; ELF64: t1a
-; SPE: t1a
+; ELF64-LABEL: @t1a
+; SPE-LABEL: @t1a
+; VSX-LABEL: @t1a
   %cmp = fcmp oeq float %a, 0.000000e+00
 ; ELF64: addis
 ; ELF64: lfs
 ; ELF64: fcmpu
+; VSX: addis
+; VSX: lfs
+; VSX: fcmpu
 ; SPE: efscmpeq
   br i1 %cmp, label %if.then, label %if.end
 
@@ -23,16 +27,41 @@ if.end:                                           ; preds = %if.then, %entry
   ret void
 }
 
-declare void @foo()
-
 define void @t1b(float %a) nounwind {
 entry:
-; ELF64: t1b
-; SPE: t1b
+; ELF64-LABEL: @t1b
+; SPE-LABEL: @t1b
+; VSX-LABEL: @t1b
   %cmp = fcmp oeq float %a, -0.000000e+00
 ; ELF64: addis
 ; ELF64: lfs
 ; ELF64: fcmpu
+; VSX: addis
+; VSX: lfs
+; VSX: fcmpu
+; SPE: efscmpeq
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  call void @foo()
+  br label %if.end
+
+if.end:                                           ; preds = %if.then, %entry
+  ret void
+}
+
+define void @t1c(float %a) nounwind {
+entry:
+; ELF64-LABEL: @t1c
+; SPE-LABEL: @t1c
+; VSX-LABEL: @t1c
+  %cmp = fcmp oeq float -0.000000e+00, %a
+; ELF64: addis
+; ELF64: lfs
+; ELF64: fcmpu
+; VSX: addis
+; VSX: lfs
+; VSX: fcmpu
 ; SPE: efscmpeq
   br i1 %cmp, label %if.then, label %if.end
 
@@ -46,12 +75,16 @@ if.end:                                           ; preds = %if.then, %entry
 
 define void @t2a(double %a) nounwind {
 entry:
-; ELF64: t2a
-; SPE: t2a
+; ELF64-LABEL: @t2a
+; SPE-LABEL: @t2a
+; VSX-LABEL: @t2a
   %cmp = fcmp oeq double %a, 0.000000e+00
 ; ELF64: addis
 ; ELF64: lfd
 ; ELF64: fcmpu
+; VSX: addis
+; VSX: lfd
+; VSX: xscmpudp
 ; SPE: efdcmpeq
   br i1 %cmp, label %if.then, label %if.end
 
@@ -65,12 +98,39 @@ if.end:                                           ; preds = %if.then, %entry
 
 define void @t2b(double %a) nounwind {
 entry:
-; ELF64: t2b
-; SPE: t2b
+; ELF64-LABEL: @t2b
+; SPE-LABEL: @t2b
+; VSX-LABEL: @t2b
   %cmp = fcmp oeq double %a, -0.000000e+00
 ; ELF64: addis
 ; ELF64: lfd
 ; ELF64: fcmpu
+; VSX: addis
+; VSX: lfd
+; VSX: xscmpudp
+; SPE: efdcmpeq
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  call void @foo()
+  br label %if.end
+
+if.end:                                           ; preds = %if.then, %entry
+  ret void
+}
+
+define void @t2c(double %a) nounwind {
+entry:
+; ELF64-LABEL: @t2c
+; SPE-LABEL: @t2c
+; VSX-LABEL: @t2c
+  %cmp = fcmp oeq double -0.000000e+00, %a
+; ELF64: addis
+; ELF64: lfd
+; ELF64: fcmpu
+; VSX: addis
+; VSX: lfd
+; VSX: xscmpudp
 ; SPE: efdcmpeq
   br i1 %cmp, label %if.then, label %if.end
 
@@ -84,7 +144,7 @@ if.end:                                           ; preds = %if.then, %entry
 
 define void @t4(i8 signext %a) nounwind {
 entry:
-; ELF64: t4
+; ELF64-LABEL: @t4
   %cmp = icmp eq i8 %a, -1
 ; ELF64: extsb
 ; ELF64: cmpwi
@@ -100,7 +160,7 @@ if.end:                                           ; preds = %if.then, %entry
 
 define void @t5(i8 zeroext %a) nounwind {
 entry:
-; ELF64: t5
+; ELF64-LABEL: @t5
   %cmp = icmp eq i8 %a, 1
 ; ELF64: extsb
 ; ELF64: cmpwi
@@ -114,9 +174,25 @@ if.end:                                           ; preds = %if.then, %entry
   ret void
 }
 
+define void @t5a(i8 zeroext %a) nounwind {
+entry:
+; ELF64-LABEL: @t5a
+  %cmp = icmp eq i8 1, %a
+; ELF64: extsb
+; ELF64: cmpw
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  call void @foo()
+  br label %if.end
+
+if.end:                                           ; preds = %if.then, %entry
+  ret void
+}
+
 define void @t6(i16 signext %a) nounwind {
 entry:
-; ELF64: t6
+; ELF64-LABEL: @t6
   %cmp = icmp eq i16 %a, -1
 ; ELF64: extsh
 ; ELF64: cmpwi
@@ -132,7 +208,7 @@ if.end:                                           ; preds = %if.then, %entry
 
 define void @t7(i16 zeroext %a) nounwind {
 entry:
-; ELF64: t7
+; ELF64-LABEL: @t7
   %cmp = icmp eq i16 %a, 1
 ; ELF64: extsh
 ; ELF64: cmpwi
@@ -146,9 +222,25 @@ if.end:                                           ; preds = %if.then, %entry
   ret void
 }
 
+define void @t7a(i16 zeroext %a) nounwind {
+entry:
+; ELF64-LABEL: @t7a
+  %cmp = icmp eq i16 1, %a
+; ELF64: extsh
+; ELF64: cmpw
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  call void @foo()
+  br label %if.end
+
+if.end:                                           ; preds = %if.then, %entry
+  ret void
+}
+
 define void @t8(i32 %a) nounwind {
 entry:
-; ELF64: t8
+; ELF64-LABEL: @t8
   %cmp = icmp eq i32 %a, -1
 ; ELF64: cmpwi
   br i1 %cmp, label %if.then, label %if.end
@@ -163,7 +255,7 @@ if.end:                                           ; preds = %if.then, %entry
 
 define void @t9(i32 %a) nounwind {
 entry:
-; ELF64: t9
+; ELF64-LABEL: @t9
   %cmp = icmp eq i32 %a, 1
 ; ELF64: cmpwi
   br i1 %cmp, label %if.then, label %if.end
@@ -178,7 +270,7 @@ if.end:                                           ; preds = %if.then, %entry
 
 define void @t10(i32 %a) nounwind {
 entry:
-; ELF64: t10
+; ELF64-LABEL: @t10
   %cmp = icmp eq i32 %a, 384
 ; ELF64: cmpwi
   br i1 %cmp, label %if.then, label %if.end
@@ -193,7 +285,7 @@ if.end:                                           ; preds = %if.then, %entry
 
 define void @t11(i32 %a) nounwind {
 entry:
-; ELF64: t11
+; ELF64-LABEL: @t11
   %cmp = icmp eq i32 %a, 4096
 ; ELF64: cmpwi
   br i1 %cmp, label %if.then, label %if.end
@@ -206,9 +298,24 @@ if.end:                                           ; preds = %if.then, %entry
   ret void
 }
 
+define void @t11a(i32 %a) nounwind {
+entry:
+; ELF64-LABEL: @t11a
+  %cmp = icmp eq i32 4096, %a
+; ELF64: cmpw
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  call void @foo()
+  br label %if.end
+
+if.end:                                           ; preds = %if.then, %entry
+  ret void
+}
+
 define void @t12(i8 %a) nounwind {
 entry:
-; ELF64: t12
+; ELF64-LABEL: @t12
   %cmp = icmp ugt i8 %a, -113
 ; ELF64: clrlwi
 ; ELF64: cmplwi
@@ -224,7 +331,7 @@ if.end:                                           ; preds = %if.then, %entry
 
 define void @t13() nounwind ssp {
 entry:
-; ELF64: t13
+; ELF64-LABEL: @t13
   %cmp = icmp slt i32 -123, -2147483648
 ; ELF64: li
 ; ELF64: lis
@@ -240,7 +347,7 @@ if.end:                                           ; preds = %entry
 
 define void @t14(i64 %a) nounwind {
 entry:
-; ELF64: t14
+; ELF64-LABEL: @t14
   %cmp = icmp eq i64 %a, -1
 ; ELF64: cmpdi
   br i1 %cmp, label %if.then, label %if.end
@@ -255,7 +362,7 @@ if.end:                                           ; preds = %if.then, %entry
 
 define void @t15(i64 %a) nounwind {
 entry:
-; ELF64: t15
+; ELF64-LABEL: @t15
   %cmp = icmp eq i64 %a, 1
 ; ELF64: cmpdi
   br i1 %cmp, label %if.then, label %if.end
@@ -270,7 +377,7 @@ if.end:                                           ; preds = %if.then, %entry
 
 define void @t16(i64 %a) nounwind {
 entry:
-; ELF64: t16
+; ELF64-LABEL: @t16
   %cmp = icmp eq i64 %a, 384
 ; ELF64: cmpdi
   br i1 %cmp, label %if.then, label %if.end
@@ -285,7 +392,7 @@ if.end:                                           ; preds = %if.then, %entry
 
 define void @t17(i64 %a) nounwind {
 entry:
-; ELF64: t17
+; ELF64-LABEL: @t17
   %cmp = icmp eq i64 %a, 32768
 ; Extra operand so we don't match on cmpdi.
 ; ELF64: cmpd {{[0-9]+}}
@@ -299,3 +406,19 @@ if.end:                                           ; preds = %if.then, %entry
   ret void
 }
 
+define void @t17a(i64 %a) nounwind {
+entry:
+; ELF64-LABEL: @t17a
+  %cmp = icmp eq i64 32768, %a
+; Extra operand so we don't match on cmpdi.
+; ELF64: cmpd {{[0-9]+}}
+  br i1 %cmp, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  call void @foo()
+  br label %if.end
+
+if.end:                                           ; preds = %if.then, %entry
+  ret void
+}
+
diff --git a/test/CodeGen/PowerPC/fast-isel-fcmp-nan.ll b/test/CodeGen/PowerPC/fast-isel-fcmp-nan.ll
index f060395bb2485f5ecbe2c332176154ad709b1f2c..84d4614e56ac144cbd08c1624f5863bb16f8bf87 100644
--- a/test/CodeGen/PowerPC/fast-isel-fcmp-nan.ll
+++ b/test/CodeGen/PowerPC/fast-isel-fcmp-nan.ll
@@ -1,4 +1,4 @@
-; RUN: llc -mtriple powerpc64le-unknown-linux-gnu -fast-isel -O0 < %s | FileCheck %s
+; RUN: llc -verify-machineinstrs -mtriple powerpc64le-unknown-linux-gnu -fast-isel -O0 < %s | FileCheck %s
 
 define i1 @TestULT(double %t0) {
 ; CHECK-LABEL: TestULT:
@@ -17,7 +17,7 @@ good:
 
 define i1 @TestULE(double %t0) {
 ; CHECK-LABEL: TestULE:
-; CHECK: fcmpu
+; CHECK: xscmpudp
 ; CHECK-NEXT: ble
 ; CHECK: blr
 entry:
@@ -33,7 +33,7 @@ good:
 
 define i1 @TestUNE(double %t0) {
 ; CHECK-LABEL: TestUNE:
-; CHECK: fcmpu
+; CHECK: xscmpudp
 ; CHECK-NEXT: bne
 ; CHECK: blr
 entry:
@@ -79,7 +79,7 @@ good:
 
 define i1 @TestUGE(double %t0) {
 ; CHECK-LABEL: TestUGE:
-; CHECK: fcmpu
+; CHECK: xscmpudp
 ; CHECK-NEXT: bge
 ; CHECK: blr
 entry:
@@ -95,7 +95,7 @@ good:
 
 define i1 @TestOLT(double %t0) {
 ; CHECK-LABEL: TestOLT:
-; CHECK: fcmpu
+; CHECK: xscmpudp
 ; CHECK-NEXT: blt
 ; CHECK: blr
 entry:
@@ -141,7 +141,7 @@ good:
 
 define i1 @TestOEQ(double %t0) {
 ; CHECK-LABEL: TestOEQ:
-; CHECK: fcmpu
+; CHECK: xscmpudp
 ; CHECK-NEXT: beq
 ; CHECK: blr
 entry:
@@ -157,7 +157,7 @@ good:
 
 define i1 @TestOGT(double %t0) {
 ; CHECK-LABEL: TestOGT:
-; CHECK: fcmpu
+; CHECK: xscmpudp
 ; CHECK-NEXT: bgt
 ; CHECK: blr
 entry:
diff --git a/test/CodeGen/PowerPC/licm-remat.ll b/test/CodeGen/PowerPC/licm-remat.ll
index e72a8b0cd3e4e39338c4dcbcf30b7007e6f6d4de..045f7a4c07ce12f3b2b443c26eaf1a50078ed311 100644
--- a/test/CodeGen/PowerPC/licm-remat.ll
+++ b/test/CodeGen/PowerPC/licm-remat.ll
@@ -24,8 +24,8 @@ define linkonce_odr void @ZN6snappyDecompressor_(%"class.snappy::SnappyDecompres
 ; CHECK-DAG:   addi 25, 3, _ZN6snappy8internalL8wordmaskE@toc@l
 ; CHECK-DAG:   addis 5, 2, _ZN6snappy8internalL10char_tableE@toc@ha
 ; CHECK-DAG:   addi 24, 5, _ZN6snappy8internalL10char_tableE@toc@l
-; CHECK:       b .LBB0_2
-; CHECK:       .LBB0_2: # %for.cond
+; CHECK:       b .[[LABEL1:[A-Z0-9_]+]]
+; CHECK:       .[[LABEL1]]: # %for.cond
 ; CHECK-NOT:   addis {{[0-9]+}}, 2, _ZN6snappy8internalL8wordmaskE@toc@ha
 ; CHECK-NOT:   addis {{[0-9]+}}, 2, _ZN6snappy8internalL10char_tableE@toc@ha
 ; CHECK:       bctrl
diff --git a/test/CodeGen/PowerPC/memcmpIR.ll b/test/CodeGen/PowerPC/memcmpIR.ll
index 9888519d8b6121a8ad8b89e94f1f6ebe38ee15c8..4c0de720fc66b67f3e814a7b2d1982f6b6607a64 100644
--- a/test/CodeGen/PowerPC/memcmpIR.ll
+++ b/test/CodeGen/PowerPC/memcmpIR.ll
@@ -17,10 +17,14 @@ entry:
   ; CHECK-NEXT: br label %endblock
 
   ; CHECK-LABEL: loadbb1:{{.*}}
-  ; CHECK: [[GEP1:%[0-9]+]] = getelementptr i64, i64* {{.*}}, i64 1
-  ; CHECK-NEXT: [[GEP2:%[0-9]+]] = getelementptr i64, i64* {{.*}}, i64 1
-  ; CHECK-NEXT: [[LOAD1:%[0-9]+]] = load i64, i64* [[GEP1]]
-  ; CHECK-NEXT: [[LOAD2:%[0-9]+]] = load i64, i64* [[GEP2]]
+  ; CHECK: [[BCC1:%[0-9]+]] = bitcast i32* {{.*}} to i8*
+  ; CHECK-NEXT: [[BCC2:%[0-9]+]] = bitcast i32* {{.*}} to i8*
+  ; CHECK-NEXT: [[GEP1:%[0-9]+]] = getelementptr i8, i8* [[BCC2]], i8 8
+  ; CHECK-NEXT: [[BCL1:%[0-9]+]] = bitcast i8* [[GEP1]] to i64*
+  ; CHECK-NEXT: [[GEP2:%[0-9]+]] = getelementptr i8, i8* [[BCC1]], i8 8
+  ; CHECK-NEXT: [[BCL2:%[0-9]+]] = bitcast i8* [[GEP2]] to i64*
+  ; CHECK-NEXT: [[LOAD1:%[0-9]+]] = load i64, i64* [[BCL1]]
+  ; CHECK-NEXT: [[LOAD2:%[0-9]+]] = load i64, i64* [[BCL2]]
   ; CHECK-NEXT: [[BSWAP1:%[0-9]+]] = call i64 @llvm.bswap.i64(i64 [[LOAD1]])
   ; CHECK-NEXT: [[BSWAP2:%[0-9]+]] = call i64 @llvm.bswap.i64(i64 [[LOAD2]])
   ; CHECK-NEXT: [[ICMP:%[0-9]+]] = icmp eq i64 [[BSWAP1]], [[BSWAP2]]
@@ -38,10 +42,14 @@ entry:
   ; CHECK-BE-NEXT: br label %endblock
 
   ; CHECK-BE-LABEL: loadbb1:{{.*}}
-  ; CHECK-BE: [[GEP1:%[0-9]+]] = getelementptr i64, i64* {{.*}}, i64 1
-  ; CHECK-BE-NEXT: [[GEP2:%[0-9]+]] = getelementptr i64, i64* {{.*}}, i64 1
-  ; CHECK-BE-NEXT: [[LOAD1:%[0-9]+]] = load i64, i64* [[GEP1]]
-  ; CHECK-BE-NEXT: [[LOAD2:%[0-9]+]] = load i64, i64* [[GEP2]]
+  ; CHECK-BE: [[BCC1:%[0-9]+]] = bitcast i32* {{.*}} to i8*
+  ; CHECK-BE-NEXT: [[BCC2:%[0-9]+]] = bitcast i32* {{.*}} to i8*
+  ; CHECK-BE-NEXT: [[GEP1:%[0-9]+]] = getelementptr i8, i8* [[BCC2]], i8 8
+  ; CHECK-BE-NEXT: [[BCL1:%[0-9]+]] = bitcast i8* [[GEP1]] to i64*
+  ; CHECK-BE-NEXT: [[GEP2:%[0-9]+]] = getelementptr i8, i8* [[BCC1]], i8 8
+  ; CHECK-BE-NEXT: [[BCL2:%[0-9]+]] = bitcast i8* [[GEP2]] to i64*
+  ; CHECK-BE-NEXT: [[LOAD1:%[0-9]+]] = load i64, i64* [[BCL1]]
+  ; CHECK-BE-NEXT: [[LOAD2:%[0-9]+]] = load i64, i64* [[BCL2]]
   ; CHECK-BE-NEXT: [[ICMP:%[0-9]+]] = icmp eq i64 [[LOAD1]], [[LOAD2]]
   ; CHECK-BE-NEXT:  br i1 [[ICMP]], label %endblock, label %res_block
 
diff --git a/test/CodeGen/PowerPC/mi-scheduling-lhs.ll b/test/CodeGen/PowerPC/mi-scheduling-lhs.ll
index 4de384dce6ec4b421c3d16b49acaec45f3475aeb..2e09cb1dd630d9b73b7674df94dfa18a2bc8b643 100644
--- a/test/CodeGen/PowerPC/mi-scheduling-lhs.ll
+++ b/test/CodeGen/PowerPC/mi-scheduling-lhs.ll
@@ -40,8 +40,8 @@ for.end:                                          ; preds = %for.body, %entry
 ; CHECK: li [[REG4:[0-9]+]], 5
 ; CHECK: [[LAB:[a-z0-9A-Z_.]+]]:
 ; CHECK: ld [[REG2:[0-9]+]], a@toc@l([[REG1]])
-; CHECK: ld [[REG3:[0-9]+]], 0([[REG2]])
 ; CHECK: stw [[REG4]], 8([[REG2]])
+; CHECK: ld [[REG3:[0-9]+]], 0([[REG2]])
 ; CHECK: stw [[REG4]], 8([[REG3]]) 
 ; CHECK: std [[REG3]], 0([[REG3]])
 ; CHECK: bdnz [[LAB]]
diff --git a/test/CodeGen/PowerPC/mulld.ll b/test/CodeGen/PowerPC/mulld.ll
index e42286d31b47155649a39896332b3c79a83cbbc4..c867d9ae8e2713a7e58e9d82f475606157a5c41a 100644
--- a/test/CodeGen/PowerPC/mulld.ll
+++ b/test/CodeGen/PowerPC/mulld.ll
@@ -11,8 +11,8 @@ define void @bn_mul_comba8(i64* nocapture %r, i64* nocapture readonly %a, i64* n
 ; CHECK-LABEL: bn_mul_comba8:
 ; CHECK:    mulhdu
 ; CHECK-NEXT:    mulld
-; CHECK-NEXT:    mulhdu
-; CHECK-NEXT:    mulld
+; CHECK:         mulhdu
+; CHECK:         mulld
 ; CHECK-NEXT:    mulhdu
 
 
diff --git a/test/CodeGen/PowerPC/pow.75.ll b/test/CodeGen/PowerPC/pow.75.ll
new file mode 100644
index 0000000000000000000000000000000000000000..c05afd30234019574cec86d9f676707b31434399
--- /dev/null
+++ b/test/CodeGen/PowerPC/pow.75.ll
@@ -0,0 +1,48 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=powerpc64le-unknown-unknown -debug 2>&1 | FileCheck %s
+; REQUIRES: asserts
+
+declare float @llvm.pow.f32(float, float)
+declare <4 x float> @llvm.pow.v4f32(<4 x float>, <4 x float>)
+declare double @llvm.pow.f64(double, double)
+declare <2 x double> @llvm.pow.v2f64(<2 x double>, <2 x double>)
+
+define float @pow_f32_three_fourth_fmf(float %x) nounwind {
+; CHECK: Combining: {{.*}}: f32 = fpow ninf nsz afn [[X:t[0-9]+]], ConstantFP:f32<7.500000e-01>
+; CHECK-NEXT: Creating new node: [[SQRT:t[0-9]+]]: f32 = fsqrt ninf nsz afn [[X]]
+; CHECK-NEXT: Creating new node: [[SQRTSQRT:t[0-9]+]]: f32 = fsqrt ninf nsz afn [[SQRT]]
+; CHECK-NEXT: Creating new node: [[R:t[0-9]+]]: f32 = fmul ninf nsz afn [[SQRT]], [[SQRTSQRT]]
+; CHECK-NEXT:  ... into: [[R]]: f32 = fmul ninf nsz afn [[SQRT]], [[SQRTSQRT]]
+  %r = call nsz ninf afn float @llvm.pow.f32(float %x, float 7.5e-01)
+  ret float %r
+}
+
+define double @pow_f64_three_fourth_fmf(double %x) nounwind {
+; CHECK: Combining: {{.*}}: f64 = fpow ninf nsz afn [[X:t[0-9]+]], ConstantFP:f64<7.500000e-01>
+; CHECK-NEXT: Creating new node: [[SQRT:t[0-9]+]]: f64 = fsqrt ninf nsz afn [[X]]
+; CHECK-NEXT: Creating new node: [[SQRTSQRT:t[0-9]+]]: f64 = fsqrt ninf nsz afn [[SQRT]]
+; CHECK-NEXT: Creating new node: [[R:t[0-9]+]]: f64 = fmul ninf nsz afn [[SQRT]], [[SQRTSQRT]]
+; CHECK-NEXT:  ... into: [[R]]: f64 = fmul ninf nsz afn [[SQRT]], [[SQRTSQRT]]
+  %r = call nsz ninf afn double @llvm.pow.f64(double %x, double 7.5e-01)
+  ret double %r
+}
+
+define <4 x float> @pow_v4f32_three_fourth_fmf(<4 x float> %x) nounwind {
+; CHECK: Combining: {{.*}}: v4f32 = fpow nnan ninf nsz arcp contract afn reassoc [[X:t[0-9]+]], {{.*}}
+; CHECK-NEXT: Creating new node: [[SQRT:t[0-9]+]]: v4f32 = fsqrt nnan ninf nsz arcp contract afn reassoc [[X]]
+; CHECK-NEXT: Creating new node: [[SQRTSQRT:t[0-9]+]]: v4f32 = fsqrt nnan ninf nsz arcp contract afn reassoc [[SQRT]]
+; CHECK-NEXT: Creating new node: [[R:t[0-9]+]]: v4f32 = fmul nnan ninf nsz arcp contract afn reassoc [[SQRT]], [[SQRTSQRT]]
+; CHECK-NEXT:  ... into: [[R]]: v4f32 = fmul nnan ninf nsz arcp contract afn reassoc [[SQRT]], [[SQRTSQRT]]
+  %r = call fast <4 x float> @llvm.pow.v4f32(<4 x float> %x, <4 x float> <float 7.5e-1, float 7.5e-1, float 7.5e-01, float 7.5e-01>)
+  ret <4 x float> %r
+}
+
+define <2 x double> @pow_v2f64_three_fourth_fmf(<2 x double> %x) nounwind {
+; CHECK: Combining: {{.*}}: v2f64 = fpow nnan ninf nsz arcp contract afn reassoc [[X:t[0-9]+]], {{.*}}
+; CHECK-NEXT: Creating new node: [[SQRT:t[0-9]+]]: v2f64 = fsqrt nnan ninf nsz arcp contract afn reassoc [[X]]
+; CHECK-NEXT: Creating new node: [[SQRTSQRT:t[0-9]+]]: v2f64 = fsqrt nnan ninf nsz arcp contract afn reassoc [[SQRT]]
+; CHECK-NEXT: Creating new node: [[R:t[0-9]+]]: v2f64 = fmul nnan ninf nsz arcp contract afn reassoc [[SQRT]], [[SQRTSQRT]]
+; CHECK-NEXT:  ... into: [[R]]: v2f64 = fmul nnan ninf nsz arcp contract afn reassoc [[SQRT]], [[SQRTSQRT]]
+  %r = call fast <2 x double> @llvm.pow.v2f64(<2 x double> %x, <2 x double> <double 7.5e-1, double 7.5e-1>)
+  ret <2 x double> %r
+}
diff --git a/test/CodeGen/PowerPC/ppc-prologue.ll b/test/CodeGen/PowerPC/ppc-prologue.ll
index 8675318b0c450bafbbdd260047a7368bc7f2603a..342728cda379c62cdf26a7b0ddc0731f48fb8e58 100644
--- a/test/CodeGen/PowerPC/ppc-prologue.ll
+++ b/test/CodeGen/PowerPC/ppc-prologue.ll
@@ -1,4 +1,4 @@
-; RUN: llc -verify-machineinstrs < %s -mtriple=powerpc-unknown-linux-gnu -disable-fp-elim | FileCheck %s
+; RUN: llc -verify-machineinstrs < %s -mtriple=powerpc-unknown-linux-gnu -frame-pointer=all | FileCheck %s
 
 define i32 @_Z4funci(i32 %a) ssp {
 ; CHECK:       mflr 0
diff --git a/test/CodeGen/PowerPC/ppc64-P9-setb.ll b/test/CodeGen/PowerPC/ppc64-P9-setb.ll
index 0006c07fa3addf8b127876a764287e868227b78f..d141e4aae074e79c864b9214b309e58b00976296 100644
--- a/test/CodeGen/PowerPC/ppc64-P9-setb.ll
+++ b/test/CodeGen/PowerPC/ppc64-P9-setb.ll
@@ -744,14 +744,14 @@ define i64 @setb28(i64 %a, i64 %b) {
 ; CHECK-NOT: addic
 ; CHECK-NOT: subfe
 ; CHECK-NOT: isel
-; CHECK: extsh
+; CHECK: extsw
 ; CHECK: blr
 ; CHECK-PWR8-LABEL: setb28
 ; CHECK-PWR8-DAG: cmpd
 ; CHECK-PWR8-DAG: addic
 ; CHECK-PWR8-DAG: subfe
 ; CHECK-PWR8: isel
-; CHECK-PWR8: extsh
+; CHECK-PWR8: extsw
 ; CHECK-PWR8: blr
 }
 
diff --git a/test/CodeGen/PowerPC/ppc64-anyregcc-crash.ll b/test/CodeGen/PowerPC/ppc64-anyregcc-crash.ll
index dfa6ec058b9274002d8cd5d5f2a4401e27f8fb2a..e19ab11e43f39f71bde46a795b7e494c7bfc12cd 100644
--- a/test/CodeGen/PowerPC/ppc64-anyregcc-crash.ll
+++ b/test/CodeGen/PowerPC/ppc64-anyregcc-crash.ll
@@ -1,4 +1,4 @@
-; RUN: not llc < %s -mtriple=powerpc64-unknown-linux-gnu 2>&1 | FileCheck %s
+; RUN: not llc -verify-machineinstrs < %s -mtriple=powerpc64-unknown-linux-gnu 2>&1 | FileCheck %s
 ;
 ; Check that misuse of anyregcc results in a compile time error.
 
diff --git a/test/CodeGen/PowerPC/ppc64-anyregcc.ll b/test/CodeGen/PowerPC/ppc64-anyregcc.ll
index 06ec561a45ddc1cd094c99711c1b2b184d4b716e..b8c62c3f9362edcff1b854f0511f51ff6375721b 100644
--- a/test/CodeGen/PowerPC/ppc64-anyregcc.ll
+++ b/test/CodeGen/PowerPC/ppc64-anyregcc.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s | FileCheck %s
+; RUN: llc -verify-machineinstrs < %s | FileCheck %s
 target datalayout = "E-m:e-i64:64-n32:64"
 target triple = "powerpc64-unknown-linux-gnu"
 
diff --git a/test/CodeGen/PowerPC/ppc64-patchpoint.ll b/test/CodeGen/PowerPC/ppc64-patchpoint.ll
index d10ea98cd1a77d741012551365df5bc92900e850..7a6f5acb921480ee5ec43ab0ce9958df223e8457 100644
--- a/test/CodeGen/PowerPC/ppc64-patchpoint.ll
+++ b/test/CodeGen/PowerPC/ppc64-patchpoint.ll
@@ -1,7 +1,7 @@
-; RUN: llc                             < %s | FileCheck %s -check-prefix=CHECK -check-prefix=CHECK-BE
-; RUN: llc -fast-isel -fast-isel-abort=1 < %s | FileCheck %s -check-prefix=CHECK -check-prefix=CHECK-BE
-; RUN: llc -mtriple=powerpc64le-unknown-linux-gnu                             < %s | FileCheck %s -check-prefix=CHECK -check-prefix=CHECK-LE
-; RUN: llc -mtriple=powerpc64le-unknown-linux-gnu -fast-isel -fast-isel-abort=1 < %s | FileCheck %s -check-prefix=CHECK -check-prefix=CHECK-LE
+; RUN: llc -verify-machineinstrs < %s | FileCheck %s -check-prefix=CHECK -check-prefix=CHECK-BE
+; RUN: llc -fast-isel -fast-isel-abort=1 -verify-machineinstrs < %s | FileCheck %s -check-prefix=CHECK -check-prefix=CHECK-BE
+; RUN: llc -mtriple=powerpc64le-unknown-linux-gnu -verify-machineinstrs < %s | FileCheck %s -check-prefix=CHECK -check-prefix=CHECK-LE
+; RUN: llc -mtriple=powerpc64le-unknown-linux-gnu -fast-isel -fast-isel-abort=1 -verify-machineinstrs < %s | FileCheck %s -check-prefix=CHECK -check-prefix=CHECK-LE
 
 target triple = "powerpc64-unknown-linux-gnu"
 
diff --git a/test/CodeGen/PowerPC/ppc64-stackmap.ll b/test/CodeGen/PowerPC/ppc64-stackmap.ll
index 5abc2a2a2173055cca8d0921ee02390a5ac3a7ee..8b2466bc06014cae7cb95593a97598e65ed3c23c 100644
--- a/test/CodeGen/PowerPC/ppc64-stackmap.ll
+++ b/test/CodeGen/PowerPC/ppc64-stackmap.ll
@@ -1,4 +1,4 @@
-; RUN: llc                             < %s | FileCheck %s
+; RUN: llc -verify-machineinstrs < %s | FileCheck %s
 ;
 ; Note: Print verbose stackmaps using -debug-only=stackmaps.
 
diff --git a/test/CodeGen/PowerPC/pr24546.ll b/test/CodeGen/PowerPC/pr24546.ll
index 91de922f7f1ec1e53725fc1fa68a856ed355cfaf..172c9fd8fbee096023ed0702db6f9fd4681c5081 100644
--- a/test/CodeGen/PowerPC/pr24546.ll
+++ b/test/CodeGen/PowerPC/pr24546.ll
@@ -56,7 +56,7 @@ attributes #3 = { nounwind }
 !llvm.module.flags = !{!29, !30}
 !llvm.ident = !{!31}
 
-!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 3.8.0 (git://github.com/llvm-mirror/clang.git e0848b6353721eb1b278a5bbea257bbf6316251e) (git://github.com/llvm-mirror/llvm.git 8724a428dfd5e78d7865bb01783708e83f9ed128)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, retainedTypes: !3, globals: !23)
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 3.8.0", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, retainedTypes: !3, globals: !23)
 !1 = !DIFile(filename: "testcase.i", directory: "/tmp/glibc.build")
 !2 = !{}
 !3 = !{!4}
@@ -86,7 +86,7 @@ attributes #3 = { nounwind }
 !28 = !DISubrange(count: 23)
 !29 = !{i32 2, !"Dwarf Version", i32 4}
 !30 = !{i32 2, !"Debug Info Version", i32 3}
-!31 = !{!"clang version 3.8.0 (git://github.com/llvm-mirror/clang.git e0848b6353721eb1b278a5bbea257bbf6316251e) (git://github.com/llvm-mirror/llvm.git 8724a428dfd5e78d7865bb01783708e83f9ed128)"}
+!31 = !{!"clang version 3.8.0"}
 !32 = !DILocation(line: 21, column: 32, scope: !33)
 !33 = distinct !DILexicalBlock(scope: !6, file: !1, line: 21, column: 6)
 !34 = !DILocation(line: 22, column: 15, scope: !35)
diff --git a/test/CodeGen/PowerPC/pr35688.ll b/test/CodeGen/PowerPC/pr35688.ll
index 832cd43eb95c3f0b5ff829409c6a106151fa54c8..098573ec1b008ae194d1ec557365baa6a8de15f4 100644
--- a/test/CodeGen/PowerPC/pr35688.ll
+++ b/test/CodeGen/PowerPC/pr35688.ll
@@ -1,6 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-unknown < %s  | \
+; RUN: llc -enable-mssa-loop-dependency=false -verify-machineinstrs -mtriple=powerpc64le-unknown-unknown < %s  | \
 ; RUN:   FileCheck %s
+; RUN: llc -enable-mssa-loop-dependency=true -verify-machineinstrs -mtriple=powerpc64le-unknown-unknown < %s  | \
+; RUN:   FileCheck %s --check-prefix=MSSA
 ; Function Attrs: nounwind
 define void @ec_GFp_nistp256_points_mul() {
 ; CHECK-LABEL: ec_GFp_nistp256_points_mul:
@@ -14,6 +16,19 @@ define void @ec_GFp_nistp256_points_mul() {
 ; CHECK:    subfc 5, 5, 7
 ; CHECK:    subfe 5, 4, 6
 ; CHECK:    sradi 5, 5, 63
+
+; With MemorySSA, everything is taken out of the loop by licm.
+; Loads and stores to undef are treated as non-aliasing.
+; MSSA-LABEL: ec_GFp_nistp256_points_mul
+; MSSA:     ld 3, 0(3)
+; MSSA:     li 4, 0
+; MSSA:     subfic 5, 3, 0
+; MSSA:     subfze 5, 4
+; MSSA:     sradi 5, 5, 63
+; MSSA:     subfc 3, 3, 5
+; MSSA:     subfe 3, 4, 5
+; MSSA:     sradi 3, 3, 63
+; MSSA:     std 3, 0(3)
 entry:
   br label %fe_cmovznz.exit.i534.i.15
 
@@ -32,3 +47,4 @@ fe_cmovznz.exit.i534.i.15:                        ; preds = %fe_cmovznz.exit.i53
   store i64 %conv4.i60.i.i, i64* undef, align 16
   br label %fe_cmovznz.exit.i534.i.15
 }
+
diff --git a/test/CodeGen/PowerPC/pr39815.ll b/test/CodeGen/PowerPC/pr39815.ll
new file mode 100644
index 0000000000000000000000000000000000000000..a01c8be8634073477ace760b515807c27be69ae3
--- /dev/null
+++ b/test/CodeGen/PowerPC/pr39815.ll
@@ -0,0 +1,31 @@
+; RUN: llc -mcpu=pwr9 -mtriple=powerpc64le-unknown-linux-gnu < %s \
+; RUN:   -verify-machineinstrs | FileCheck %s
+
+@b = common dso_local local_unnamed_addr global i64* null, align 8
+@a = common dso_local local_unnamed_addr global i8 0, align 1
+
+define void @testADDEPromoteResult() {
+entry:
+  %0 = load i64*, i64** @b, align 8
+  %1 = load i64, i64* %0, align 8
+  %cmp = icmp ne i64* %0, null
+  %conv1 = zext i1 %cmp to i64
+  %add = add nsw i64 %1, %conv1
+  %2 = trunc i64 %add to i8
+  %conv2 = and i8 %2, 5
+  store i8 %conv2, i8* @a, align 1
+  ret void
+
+; CHECK-LABEL: @testADDEPromoteResult
+; CHECK:      # %bb.0:
+; CHECK-DAG:   addis [[REG1:[0-9]+]], [[REG2:[0-9]+]], [[VAR1:[a-z0-9A-Z_.]+]]@toc@ha
+; CHECK-DAG:   ld [[REG3:[0-9]+]], [[VAR1]]@toc@l([[REG1]])
+; CHECK-DAG:   lbz [[REG4:[0-9]+]], 0([[REG3]])
+; CHECK-DAG:   addic [[REG5:[0-9]+]], [[REG3]], -1
+; CHECK-DAG:   extsb [[REG6:[0-9]+]], [[REG4]]
+; CHECK-DAG:   addze [[REG7:[0-9]+]], [[REG6]]
+; CHECK-DAG:   addis [[REG8:[0-9]+]], [[REG2]], [[VAR2:[a-z0-9A-Z_.]+]]@toc@ha
+; CHECK-DAG:   andi. [[REG9:[0-9]+]], [[REG7]], 5
+; CHECK-DAG:   stb [[REG9]], [[VAR2]]@toc@l([[REG8]])
+; CHECK:       blr
+}
diff --git a/test/CodeGen/PowerPC/pre-inc-disable.ll b/test/CodeGen/PowerPC/pre-inc-disable.ll
index 52fc7fe8746cffc2206fe1d0b1906f33aee71657..97ebe188c85fc35d182811da48957c5c6f866efc 100644
--- a/test/CodeGen/PowerPC/pre-inc-disable.ll
+++ b/test/CodeGen/PowerPC/pre-inc-disable.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mcpu=pwr9 -O3 -verify-machineinstrs -ppc-vsr-nums-as-vr \
 ; RUN:     -ppc-asm-full-reg-names -mtriple=powerpc64le-unknown-linux-gnu \
 ; RUN:     < %s | FileCheck %s
@@ -9,109 +10,121 @@
 ; Function Attrs: norecurse nounwind readonly
 define signext i32 @test_pre_inc_disable_1(i8* nocapture readonly %pix1, i32 signext %i_stride_pix1, i8* nocapture readonly %pix2) {
 ; CHECK-LABEL: test_pre_inc_disable_1:
-; CHECK:   # %bb.0: # %entry
-; CHECK:    addis r6, r2
-; CHECK:    addis r7, r2,
-; CHECK:    lfd f0, 0(r5)
-; CHECK:    xxlxor v4, v4, v4
-; CHECK:    addi r5, r6,
-; CHECK:    addi r6, r7,
-; CHECK:    lxvx v2, 0, r5
-; CHECK:    lxvx v3, 0, r6
-; CHECK:    xxpermdi v5, f0, f0, 2
-; CHECK-DAG: vperm v[[VR1:[0-9]+]], v4, v5, v2
-; CHECK-DAG: vperm v[[VR2:[0-9]+]], v5, v4, v3
-; CHECK-DAG: xvnegsp v[[VR3:[0-9]+]], v[[VR1]]
-; CHECK-DAG: xvnegsp v[[VR4:[0-9]+]], v[[VR2]]
-
-; CHECK:  .LBB0_1: # %for.cond1.preheader
-; CHECK:    lfd f0, 0(r3)
-; CHECK:    xxpermdi v1, f0, f0, 2
-; CHECK:    vperm v6, v1, v4, v3
-; CHECK:    vperm v1, v4, v1, v2
-; CHECK-DAG:    xvnegsp v6, v6
-; CHECK-DAG:    xvnegsp v1, v1
-; CHECK-DAG: vabsduw v1, v1, v[[VR3]]
-; CHECK-DAG: vabsduw v6, v6, v[[VR4]]
-; CHECK:    vadduwm v1, v6, v1
-; CHECK:    xxswapd v6, v1
-; CHECK:    vadduwm v1, v1, v6
-; CHECK:    xxspltw v6, v1, 2
-; CHECK:    vadduwm v1, v1, v6
-; CHECK:    vextuwrx r7, r6, v1
-; CHECK:    ldux r8, r3, r4
-; CHECK:    add r3, r3, r4
-; CHECK:    add r5, r7, r5
-; CHECK:    mtvsrd f0, r8
-; CHECK:    xxswapd v1, vs0
-; CHECK:    vperm v6, v1, v4, v3
-; CHECK:    vperm v1, v4, v1, v2
-; CHECK-DAG: xvnegsp v6, v6
-; CHECK-DAG: xvnegsp v1, v1
-; CHECK-DAG: vabsduw v1, v1, v[[VR3]]
-; CHECK-DAG: vabsduw v6, v6, v[[VR4]]
-; CHECK:    vadduwm v1, v6, v1
-; CHECK:    xxswapd v6, v1
-; CHECK:    vadduwm v1, v1, v6
-; CHECK:    xxspltw v6, v1, 2
-; CHECK:    vadduwm v1, v1, v6
-; CHECK:    vextuwrx r8, r6, v1
-; CHECK:    add r5, r8, r5
-; CHECK:    bdnz .LBB0_1
-; CHECK:    extsw r3, r5
-; CHECK:    blr
-
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lfd f0, 0(r5)
+; CHECK-NEXT:    addis r5, r2, .LCPI0_0@toc@ha
+; CHECK-NEXT:    addi r5, r5, .LCPI0_0@toc@l
+; CHECK-NEXT:    lxvx v2, 0, r5
+; CHECK-NEXT:    addis r5, r2, .LCPI0_1@toc@ha
+; CHECK-NEXT:    addi r5, r5, .LCPI0_1@toc@l
+; CHECK-NEXT:    lxvx v4, 0, r5
+; CHECK-NEXT:    xxpermdi v5, f0, f0, 2
+; CHECK-NEXT:    xxlxor v3, v3, v3
+; CHECK-NEXT:    li r5, 4
+; CHECK-NEXT:    vperm v0, v3, v5, v2
+; CHECK-NEXT:    mtctr r5
+; CHECK-NEXT:    li r5, 0
+; CHECK-NEXT:    vperm v1, v5, v3, v4
+; CHECK-NEXT:    li r6, 0
+; CHECK-NEXT:    xvnegsp v5, v0
+; CHECK-NEXT:    xvnegsp v0, v1
+; CHECK-NEXT:    .p2align 4
+; CHECK-NEXT:  .LBB0_1: # %for.cond1.preheader
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    lfd f0, 0(r3)
+; CHECK-NEXT:    xxpermdi v1, f0, f0, 2
+; CHECK-NEXT:    vperm v6, v3, v1, v2
+; CHECK-NEXT:    vperm v1, v1, v3, v4
+; CHECK-NEXT:    xvnegsp v6, v6
+; CHECK-NEXT:    xvnegsp v1, v1
+; CHECK-NEXT:    vabsduw v6, v6, v5
+; CHECK-NEXT:    vabsduw v1, v1, v0
+; CHECK-NEXT:    vadduwm v1, v1, v6
+; CHECK-NEXT:    xxswapd v6, v1
+; CHECK-NEXT:    vadduwm v1, v1, v6
+; CHECK-NEXT:    xxspltw v6, v1, 2
+; CHECK-NEXT:    vadduwm v1, v1, v6
+; CHECK-NEXT:    vextuwrx r7, r5, v1
+; CHECK-NEXT:    ldux r8, r3, r4
+; CHECK-NEXT:    add r3, r3, r4
+; CHECK-NEXT:    add r6, r7, r6
+; CHECK-NEXT:    mtvsrd f0, r8
+; CHECK-NEXT:    xxswapd v1, vs0
+; CHECK-NEXT:    vperm v6, v3, v1, v2
+; CHECK-NEXT:    vperm v1, v1, v3, v4
+; CHECK-NEXT:    xvnegsp v6, v6
+; CHECK-NEXT:    xvnegsp v1, v1
+; CHECK-NEXT:    vabsduw v6, v6, v5
+; CHECK-NEXT:    vabsduw v1, v1, v0
+; CHECK-NEXT:    vadduwm v1, v1, v6
+; CHECK-NEXT:    xxswapd v6, v1
+; CHECK-NEXT:    vadduwm v1, v1, v6
+; CHECK-NEXT:    xxspltw v6, v1, 2
+; CHECK-NEXT:    vadduwm v1, v1, v6
+; CHECK-NEXT:    vextuwrx r7, r5, v1
+; CHECK-NEXT:    add r6, r7, r6
+; CHECK-NEXT:    bdnz .LBB0_1
+; CHECK-NEXT:  # %bb.2: # %for.cond.cleanup
+; CHECK-NEXT:    extsw r3, r6
+; CHECK-NEXT:    blr
+;
 ; P9BE-LABEL: test_pre_inc_disable_1:
-; P9BE:    addis r6, r2,
-; P9BE:    addis r7, r2,
-; P9BE:    lfd f0, 0(r5)
-; P9BE:    xxlxor v4, v4, v4
-; P9BE:    addi r5, r6,
-; P9BE:    addi r6, r7,
-; P9BE:    lxvx v2, 0, r5
-; P9BE:    lxvx v3, 0, r6
-; P9BE:    xxlor v5, vs0, vs0
-; P9BE:    li r6, 0
-; P9BE-DAG: vperm v[[VR1:[0-9]+]], v4, v5, v2
-; P9BE-DAG: vperm v[[VR2:[0-9]+]], v4, v5, v3
-; P9BE-DAG: xvnegsp v[[VR3:[0-9]+]], v[[VR1]]
-; P9BE-DAG: xvnegsp v[[VR4:[0-9]+]], v[[VR2]]
-
-; P9BE:  .LBB0_1: # %for.cond1.preheader
-; P9BE:    lfd f0, 0(r3)
-; P9BE:    xxlor v1, vs0, vs0
-; P9BE:    vperm v6, v4, v1, v3
-; P9BE:    vperm v1, v4, v1, v2
-; P9BE-DAG: xvnegsp v6, v6
-; P9BE-DAG: xvnegsp v1, v1
-; P9BE-DAG: vabsduw v1, v1, v[[VR3]]
-; P9BE-DAG: vabsduw v6, v6, v[[VR4]]
-; P9BE:    vadduwm v1, v6, v1
-; P9BE:    xxswapd v6, v1
-; P9BE:    vadduwm v1, v1, v6
-; P9BE:    xxspltw v6, v1, 1
-; P9BE:    vadduwm v1, v1, v6
-; P9BE:    vextuwlx r[[GR1:[0-9]+]], r6, v1
-; P9BE:    ldux r[[GR2:[0-9]+]], r3, r4
-; P9BE:    add r3, r3, r4
-; P9BE:    add r5, r[[GR1]], r5
-; P9BE:    mtvsrd v1, r[[GR2]]
-; P9BE:    vperm v6, v4, v1, v3
-; P9BE:    vperm v1, v4, v1, v2
-; P9BE-DAG: xvnegsp v6, v6
-; P9BE-DAG: xvnegsp v1, v1
-; P9BE-DAG: vabsduw v1, v1, v[[VR3]]
-; P9BE-DAG: vabsduw v6, v6, v[[VR4]]
-; P9BE:    vadduwm v1, v6, v1
-; P9BE:    xxswapd v6, v1
-; P9BE:    vadduwm v1, v1, v6
-; P9BE:    xxspltw v6, v1, 1
-; P9BE:    vadduwm v1, v1, v6
-; P9BE:    vextuwlx r8, r6, v1
-; P9BE:    add r5, r8, r5
-; P9BE:    bdnz .LBB0_1
-; P9BE:    extsw r3, r5
-; P9BE:    blr
+; P9BE:       # %bb.0: # %entry
+; P9BE-NEXT:    lfd f0, 0(r5)
+; P9BE-NEXT:    addis r5, r2, .LCPI0_0@toc@ha
+; P9BE-NEXT:    addi r5, r5, .LCPI0_0@toc@l
+; P9BE-NEXT:    lxvx v2, 0, r5
+; P9BE-NEXT:    addis r5, r2, .LCPI0_1@toc@ha
+; P9BE-NEXT:    addi r5, r5, .LCPI0_1@toc@l
+; P9BE-NEXT:    lxvx v4, 0, r5
+; P9BE-NEXT:    li r5, 4
+; P9BE-NEXT:    xxlor v5, vs0, vs0
+; P9BE-NEXT:    xxlxor v3, v3, v3
+; P9BE-NEXT:    vperm v0, v3, v5, v2
+; P9BE-NEXT:    mtctr r5
+; P9BE-NEXT:    li r5, 0
+; P9BE-NEXT:    vperm v1, v3, v5, v4
+; P9BE-NEXT:    li r6, 0
+; P9BE-NEXT:    xvnegsp v5, v0
+; P9BE-NEXT:    xvnegsp v0, v1
+; P9BE-NEXT:    .p2align 4
+; P9BE-NEXT:  .LBB0_1: # %for.cond1.preheader
+; P9BE-NEXT:    # =>This Inner Loop Header: Depth=1
+; P9BE-NEXT:    lfd f0, 0(r3)
+; P9BE-NEXT:    xxlor v1, vs0, vs0
+; P9BE-NEXT:    vperm v6, v3, v1, v4
+; P9BE-NEXT:    vperm v1, v3, v1, v2
+; P9BE-NEXT:    xvnegsp v1, v1
+; P9BE-NEXT:    xvnegsp v6, v6
+; P9BE-NEXT:    vabsduw v1, v1, v5
+; P9BE-NEXT:    vabsduw v6, v6, v0
+; P9BE-NEXT:    vadduwm v1, v6, v1
+; P9BE-NEXT:    xxswapd v6, v1
+; P9BE-NEXT:    vadduwm v1, v1, v6
+; P9BE-NEXT:    xxspltw v6, v1, 1
+; P9BE-NEXT:    vadduwm v1, v1, v6
+; P9BE-NEXT:    vextuwlx r7, r5, v1
+; P9BE-NEXT:    add r6, r7, r6
+; P9BE-NEXT:    ldux r7, r3, r4
+; P9BE-NEXT:    add r3, r3, r4
+; P9BE-NEXT:    mtvsrd v1, r7
+; P9BE-NEXT:    vperm v6, v3, v1, v2
+; P9BE-NEXT:    vperm v1, v3, v1, v4
+; P9BE-NEXT:    xvnegsp v6, v6
+; P9BE-NEXT:    xvnegsp v1, v1
+; P9BE-NEXT:    vabsduw v6, v6, v5
+; P9BE-NEXT:    vabsduw v1, v1, v0
+; P9BE-NEXT:    vadduwm v1, v1, v6
+; P9BE-NEXT:    xxswapd v6, v1
+; P9BE-NEXT:    vadduwm v1, v1, v6
+; P9BE-NEXT:    xxspltw v6, v1, 1
+; P9BE-NEXT:    vadduwm v1, v1, v6
+; P9BE-NEXT:    vextuwlx r7, r5, v1
+; P9BE-NEXT:    add r6, r7, r6
+; P9BE-NEXT:    bdnz .LBB0_1
+; P9BE-NEXT:  # %bb.2: # %for.cond.cleanup
+; P9BE-NEXT:    extsw r3, r6
+; P9BE-NEXT:    blr
 entry:
   %idx.ext = sext i32 %i_stride_pix1 to i64
   %0 = bitcast i8* %pix2 to <8 x i8>*
@@ -166,58 +179,62 @@ for.cond.cleanup:                                 ; preds = %for.cond1.preheader
 ; Function Attrs: norecurse nounwind readonly
 define signext i32 @test_pre_inc_disable_2(i8* nocapture readonly %pix1, i8* nocapture readonly %pix2) {
 ; CHECK-LABEL: test_pre_inc_disable_2:
-; CHECK:    addis r5, r2,
-; CHECK:    addis r6, r2,
-; CHECK:    lfd f0, 0(r3)
-; CHECK:    lfd f1, 0(r4)
-; CHECK:    xxlxor v0, v0, v0
-; CHECK:    addi r3, r5, .LCPI1_0@toc@l
-; CHECK:    addi r4, r6, .LCPI1_1@toc@l
-; CHECK:    lxvx v2, 0, r3
-; CHECK:    lxvx v3, 0, r4
-; CHECK:    xxpermdi v4, f0, f0, 2
-; CHECK:    xxpermdi v5, f1, f1, 2
-; CHECK:    vperm v1, v4, v0, v2
-; CHECK:    vperm v4, v0, v4, v3
-; CHECK:    vperm v2, v5, v0, v2
-; CHECK:    vperm v3, v0, v5, v3
-; CHECK:    vabsduw v3, v4, v3
-; CHECK:    vabsduw v2, v1, v2
-; CHECK:    vadduwm v2, v2, v3
-; CHECK:    xxswapd v3, v2
-; CHECK:    vadduwm v2, v2, v3
-; CHECK:    xxspltw v3, v2, 2
-; CHECK:    vadduwm v2, v2, v3
-; CHECK:    vextuwrx r3, r3, v2
-; CHECK:    extsw r3, r3
-; CHECK:    blr
-
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lfd f0, 0(r3)
+; CHECK-NEXT:    addis r3, r2, .LCPI1_0@toc@ha
+; CHECK-NEXT:    addi r3, r3, .LCPI1_0@toc@l
+; CHECK-NEXT:    lxvx v4, 0, r3
+; CHECK-NEXT:    addis r3, r2, .LCPI1_1@toc@ha
+; CHECK-NEXT:    xxpermdi v2, f0, f0, 2
+; CHECK-NEXT:    lfd f0, 0(r4)
+; CHECK-NEXT:    addi r3, r3, .LCPI1_1@toc@l
+; CHECK-NEXT:    xxlxor v3, v3, v3
+; CHECK-NEXT:    lxvx v0, 0, r3
+; CHECK-NEXT:    xxpermdi v1, f0, f0, 2
+; CHECK-NEXT:    vperm v5, v2, v3, v4
+; CHECK-NEXT:    vperm v2, v3, v2, v0
+; CHECK-NEXT:    vperm v0, v3, v1, v0
+; CHECK-NEXT:    vperm v3, v1, v3, v4
+; CHECK-NEXT:    vabsduw v2, v2, v0
+; CHECK-NEXT:    vabsduw v3, v5, v3
+; CHECK-NEXT:    vadduwm v2, v3, v2
+; CHECK-NEXT:    xxswapd v3, v2
+; CHECK-NEXT:    li r3, 0
+; CHECK-NEXT:    vadduwm v2, v2, v3
+; CHECK-NEXT:    xxspltw v3, v2, 2
+; CHECK-NEXT:    vadduwm v2, v2, v3
+; CHECK-NEXT:    vextuwrx r3, r3, v2
+; CHECK-NEXT:    extsw r3, r3
+; CHECK-NEXT:    blr
+;
 ; P9BE-LABEL: test_pre_inc_disable_2:
-; P9BE:    addis r5, r2,
-; P9BE:    addis r6, r2,
-; P9BE:    lfd f0, 0(r3)
-; P9BE:    lfd f1, 0(r4)
-; P9BE:    xxlxor v5, v5, v5
-; P9BE:    addi r3, r5,
-; P9BE:    addi r4, r6,
-; P9BE:    lxvx v2, 0, r3
-; P9BE:    lxvx v3, 0, r4
-; P9BE:    xxlor v4, vs0, vs0
-; P9BE:    xxlor v0, vs1, vs1
-; P9BE:    vperm v1, v5, v4, v2
-; P9BE:    vperm v4, v5, v4, v3
-; P9BE:    vperm v2, v5, v0, v2
-; P9BE:    vperm v3, v5, v0, v3
-; P9BE:    vabsduw v3, v4, v3
-; P9BE:    vabsduw v2, v1, v2
-; P9BE:    vadduwm v2, v2, v3
-; P9BE:    xxswapd v3, v2
-; P9BE:    vadduwm v2, v2, v3
-; P9BE:    xxspltw v3, v2, 1
-; P9BE:    vadduwm v2, v2, v3
-; P9BE:    vextuwlx r3, r3, v2
-; P9BE:    extsw r3, r3
-; P9BE:    blr
+; P9BE:       # %bb.0: # %entry
+; P9BE-NEXT:    lfd f0, 0(r3)
+; P9BE-NEXT:    addis r3, r2, .LCPI1_0@toc@ha
+; P9BE-NEXT:    addi r3, r3, .LCPI1_0@toc@l
+; P9BE-NEXT:    lxvx v4, 0, r3
+; P9BE-NEXT:    addis r3, r2, .LCPI1_1@toc@ha
+; P9BE-NEXT:    addi r3, r3, .LCPI1_1@toc@l
+; P9BE-NEXT:    xxlor v2, vs0, vs0
+; P9BE-NEXT:    lfd f0, 0(r4)
+; P9BE-NEXT:    lxvx v0, 0, r3
+; P9BE-NEXT:    xxlxor v3, v3, v3
+; P9BE-NEXT:    xxlor v1, vs0, vs0
+; P9BE-NEXT:    vperm v5, v3, v2, v4
+; P9BE-NEXT:    vperm v2, v3, v2, v0
+; P9BE-NEXT:    vperm v0, v3, v1, v0
+; P9BE-NEXT:    vperm v3, v3, v1, v4
+; P9BE-NEXT:    vabsduw v2, v2, v0
+; P9BE-NEXT:    vabsduw v3, v5, v3
+; P9BE-NEXT:    vadduwm v2, v3, v2
+; P9BE-NEXT:    xxswapd v3, v2
+; P9BE-NEXT:    vadduwm v2, v2, v3
+; P9BE-NEXT:    xxspltw v3, v2, 1
+; P9BE-NEXT:    vadduwm v2, v2, v3
+; P9BE-NEXT:    li r3, 0
+; P9BE-NEXT:    vextuwlx r3, r3, v2
+; P9BE-NEXT:    extsw r3, r3
+; P9BE-NEXT:    blr
 entry:
   %0 = bitcast i8* %pix1 to <8 x i8>*
   %1 = load <8 x i8>, <8 x i8>* %0, align 1
diff --git a/test/CodeGen/PowerPC/remove-implicit-use.mir b/test/CodeGen/PowerPC/remove-implicit-use.mir
new file mode 100644
index 0000000000000000000000000000000000000000..9a70ce317852f6c03ca6b5c30a692409156b2a77
--- /dev/null
+++ b/test/CodeGen/PowerPC/remove-implicit-use.mir
@@ -0,0 +1,78 @@
+# RUN: llc -mtriple=powerpc64le-unknown-unknown -start-after=ppc-mi-peepholes \ 
+# RUN: -stop-before=ppc-expand-isel -verify-machineinstrs %s -o - | FileCheck %s
+--- |
+  ; ModuleID = 'a.ll'
+  source_filename = "a.c"   
+  target datalayout = "e-m:e-i64:64-n32:64"
+  target triple = "powerpc64le-unknown-linux-gnu"
+
+  ; Function Attrs: norecurse nounwind readnone
+  define signext i32 @test(i32 signext %a, i32 signext %b, i32 signext %c) local_unnamed_addr #0 {
+  entry:
+    %cmp = icmp sgt i32 %a, %b
+    %add = add nsw i32 %c, %b
+    %cond = select i1 %cmp, i32 %a, i32 %add
+    ret i32 %cond
+  }
+
+  
+  !llvm.module.flags = !{!0, !1}
+  !llvm.ident = !{!2}
+  
+  !0 = !{i32 1, !"wchar_size", i32 4}
+  !1 = !{i32 7, !"PIC Level", i32 2}
+  !2 = !{!"clang version 8.0.0 (trunk 347251)"}
+
+...
+---
+name:            test
+alignment:       4
+exposesReturnsTwice: false
+legalized:       false
+regBankSelected: false
+selected:        false
+failedISel:      false
+tracksRegLiveness: true
+hasWinCFI:       false
+registers:       []
+liveins:         
+  - { reg: '$x3', virtual-reg: '' }
+  - { reg: '$x4', virtual-reg: '' }
+  - { reg: '$x5', virtual-reg: '' }
+frameInfo:       
+  isFrameAddressTaken: false
+  isReturnAddressTaken: false
+  hasStackMap:     false
+  hasPatchPoint:   false
+  stackSize:       0
+  offsetAdjustment: 0
+  maxAlignment:    0
+  adjustsStack:    false
+  hasCalls:        false
+  stackProtector:  ''
+  maxCallFrameSize: 0
+  cvBytesOfCalleeSavedRegisters: 0
+  hasOpaqueSPAdjustment: false
+  hasVAStart:      false
+  hasMustTailInVarArgFunc: false
+  localFrameSize:  0
+  savePoint:       ''
+  restorePoint:    ''
+fixedStack:      []
+stack:           []
+constants:       []
+body:             |
+  bb.0.entry:
+    liveins: $x3, $x5
+
+    renamable $r4 = LI 0
+    renamable $r5 = nsw ADD4 killed renamable $r5, renamable $r5, implicit $x5
+    renamable $cr0 = CMPW renamable $r3, killed renamable $r4, implicit $x4
+    ; CHECK: ADD4
+    ; CHECK-NOT: implicit $x4
+    renamable $r3 = ISEL killed renamable $r3, killed renamable $r5, killed renamable $cr0gt, implicit $cr0, implicit $x3
+    renamable $x3 = EXTSW_32_64 killed renamable $r3
+    BLR8 implicit $lr8, implicit $rm, implicit killed $x3
+
+...
+
diff --git a/test/CodeGen/PowerPC/sat-add.ll b/test/CodeGen/PowerPC/sat-add.ll
new file mode 100644
index 0000000000000000000000000000000000000000..7f4e1b8cc2acd31ce3da514b5e7570509461ba0f
--- /dev/null
+++ b/test/CodeGen/PowerPC/sat-add.ll
@@ -0,0 +1,785 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=powerpc64le-- -verify-machineinstrs | FileCheck %s
+
+; There are at least 3 potential patterns corresponding to an unsigned saturated add: min, cmp with sum, cmp with not.
+; Test each of those patterns with i8/i16/i32/i64.
+; Test each of those with a constant operand and a variable operand.
+; Test each of those with a 128-bit vector type.
+
+define i8 @unsigned_sat_constant_i8_using_min(i8 %x) {
+; CHECK-LABEL: unsigned_sat_constant_i8_using_min:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    clrlwi 5, 3, 24
+; CHECK-NEXT:    li 4, -43
+; CHECK-NEXT:    cmplwi 5, 213
+; CHECK-NEXT:    isel 3, 3, 4, 0
+; CHECK-NEXT:    addi 3, 3, 42
+; CHECK-NEXT:    blr
+  %c = icmp ult i8 %x, -43
+  %s = select i1 %c, i8 %x, i8 -43
+  %r = add i8 %s, 42
+  ret i8 %r
+}
+
+define i8 @unsigned_sat_constant_i8_using_cmp_sum(i8 %x) {
+; CHECK-LABEL: unsigned_sat_constant_i8_using_cmp_sum:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    rlwinm 3, 3, 0, 24, 31
+; CHECK-NEXT:    addi 3, 3, 42
+; CHECK-NEXT:    andi. 4, 3, 256
+; CHECK-NEXT:    li 4, -1
+; CHECK-NEXT:    isel 3, 3, 4, 2
+; CHECK-NEXT:    blr
+  %a = add i8 %x, 42
+  %c = icmp ugt i8 %x, %a
+  %r = select i1 %c, i8 -1, i8 %a
+  ret i8 %r
+}
+
+define i8 @unsigned_sat_constant_i8_using_cmp_notval(i8 %x) {
+; CHECK-LABEL: unsigned_sat_constant_i8_using_cmp_notval:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    clrlwi 5, 3, 24
+; CHECK-NEXT:    li 4, -1
+; CHECK-NEXT:    addi 3, 3, 42
+; CHECK-NEXT:    cmplwi 5, 213
+; CHECK-NEXT:    isel 3, 4, 3, 1
+; CHECK-NEXT:    blr
+  %a = add i8 %x, 42
+  %c = icmp ugt i8 %x, -43
+  %r = select i1 %c, i8 -1, i8 %a
+  ret i8 %r
+}
+
+define i16 @unsigned_sat_constant_i16_using_min(i16 %x) {
+; CHECK-LABEL: unsigned_sat_constant_i16_using_min:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    clrlwi 5, 3, 16
+; CHECK-NEXT:    li 4, -43
+; CHECK-NEXT:    cmplwi 5, 65493
+; CHECK-NEXT:    isel 3, 3, 4, 0
+; CHECK-NEXT:    addi 3, 3, 42
+; CHECK-NEXT:    blr
+  %c = icmp ult i16 %x, -43
+  %s = select i1 %c, i16 %x, i16 -43
+  %r = add i16 %s, 42
+  ret i16 %r
+}
+
+define i16 @unsigned_sat_constant_i16_using_cmp_sum(i16 %x) {
+; CHECK-LABEL: unsigned_sat_constant_i16_using_cmp_sum:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    rlwinm 3, 3, 0, 16, 31
+; CHECK-NEXT:    addi 3, 3, 42
+; CHECK-NEXT:    andis. 4, 3, 1
+; CHECK-NEXT:    li 4, -1
+; CHECK-NEXT:    isel 3, 3, 4, 2
+; CHECK-NEXT:    blr
+  %a = add i16 %x, 42
+  %c = icmp ugt i16 %x, %a
+  %r = select i1 %c, i16 -1, i16 %a
+  ret i16 %r
+}
+
+define i16 @unsigned_sat_constant_i16_using_cmp_notval(i16 %x) {
+; CHECK-LABEL: unsigned_sat_constant_i16_using_cmp_notval:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    clrlwi 5, 3, 16
+; CHECK-NEXT:    li 4, -1
+; CHECK-NEXT:    addi 3, 3, 42
+; CHECK-NEXT:    cmplwi 5, 65493
+; CHECK-NEXT:    isel 3, 4, 3, 1
+; CHECK-NEXT:    blr
+  %a = add i16 %x, 42
+  %c = icmp ugt i16 %x, -43
+  %r = select i1 %c, i16 -1, i16 %a
+  ret i16 %r
+}
+
+define i32 @unsigned_sat_constant_i32_using_min(i32 %x) {
+; CHECK-LABEL: unsigned_sat_constant_i32_using_min:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    li 4, -43
+; CHECK-NEXT:    cmplw 3, 4
+; CHECK-NEXT:    isel 3, 3, 4, 0
+; CHECK-NEXT:    addi 3, 3, 42
+; CHECK-NEXT:    blr
+  %c = icmp ult i32 %x, -43
+  %s = select i1 %c, i32 %x, i32 -43
+  %r = add i32 %s, 42
+  ret i32 %r
+}
+
+define i32 @unsigned_sat_constant_i32_using_cmp_sum(i32 %x) {
+; CHECK-LABEL: unsigned_sat_constant_i32_using_cmp_sum:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi 5, 3, 42
+; CHECK-NEXT:    li 4, -1
+; CHECK-NEXT:    cmplw 0, 5, 3
+; CHECK-NEXT:    isel 3, 4, 5, 0
+; CHECK-NEXT:    blr
+  %a = add i32 %x, 42
+  %c = icmp ugt i32 %x, %a
+  %r = select i1 %c, i32 -1, i32 %a
+  ret i32 %r
+}
+
+define i32 @unsigned_sat_constant_i32_using_cmp_notval(i32 %x) {
+; CHECK-LABEL: unsigned_sat_constant_i32_using_cmp_notval:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    li 4, -43
+; CHECK-NEXT:    addi 5, 3, 42
+; CHECK-NEXT:    cmplw 0, 3, 4
+; CHECK-NEXT:    li 3, -1
+; CHECK-NEXT:    isel 3, 3, 5, 1
+; CHECK-NEXT:    blr
+  %a = add i32 %x, 42
+  %c = icmp ugt i32 %x, -43
+  %r = select i1 %c, i32 -1, i32 %a
+  ret i32 %r
+}
+
+define i64 @unsigned_sat_constant_i64_using_min(i64 %x) {
+; CHECK-LABEL: unsigned_sat_constant_i64_using_min:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    li 4, -43
+; CHECK-NEXT:    cmpld 3, 4
+; CHECK-NEXT:    isel 3, 3, 4, 0
+; CHECK-NEXT:    addi 3, 3, 42
+; CHECK-NEXT:    blr
+  %c = icmp ult i64 %x, -43
+  %s = select i1 %c, i64 %x, i64 -43
+  %r = add i64 %s, 42
+  ret i64 %r
+}
+
+define i64 @unsigned_sat_constant_i64_using_cmp_sum(i64 %x) {
+; CHECK-LABEL: unsigned_sat_constant_i64_using_cmp_sum:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi 5, 3, 42
+; CHECK-NEXT:    li 4, -1
+; CHECK-NEXT:    cmpld 5, 3
+; CHECK-NEXT:    isel 3, 4, 5, 0
+; CHECK-NEXT:    blr
+  %a = add i64 %x, 42
+  %c = icmp ugt i64 %x, %a
+  %r = select i1 %c, i64 -1, i64 %a
+  ret i64 %r
+}
+
+define i64 @unsigned_sat_constant_i64_using_cmp_notval(i64 %x) {
+; CHECK-LABEL: unsigned_sat_constant_i64_using_cmp_notval:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    li 4, -43
+; CHECK-NEXT:    addi 5, 3, 42
+; CHECK-NEXT:    cmpld 3, 4
+; CHECK-NEXT:    li 3, -1
+; CHECK-NEXT:    isel 3, 3, 5, 1
+; CHECK-NEXT:    blr
+  %a = add i64 %x, 42
+  %c = icmp ugt i64 %x, -43
+  %r = select i1 %c, i64 -1, i64 %a
+  ret i64 %r
+}
+
+define i8 @unsigned_sat_variable_i8_using_min(i8 %x, i8 %y) {
+; CHECK-LABEL: unsigned_sat_variable_i8_using_min:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    nor 5, 4, 4
+; CHECK-NEXT:    clrlwi 6, 3, 24
+; CHECK-NEXT:    clrlwi 7, 5, 24
+; CHECK-NEXT:    cmplw 6, 7
+; CHECK-NEXT:    isel 3, 3, 5, 0
+; CHECK-NEXT:    add 3, 3, 4
+; CHECK-NEXT:    blr
+  %noty = xor i8 %y, -1
+  %c = icmp ult i8 %x, %noty
+  %s = select i1 %c, i8 %x, i8 %noty
+  %r = add i8 %s, %y
+  ret i8 %r
+}
+
+define i8 @unsigned_sat_variable_i8_using_cmp_sum(i8 %x, i8 %y) {
+; CHECK-LABEL: unsigned_sat_variable_i8_using_cmp_sum:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    rlwinm 4, 4, 0, 24, 31
+; CHECK-NEXT:    rlwinm 3, 3, 0, 24, 31
+; CHECK-NEXT:    add 3, 3, 4
+; CHECK-NEXT:    andi. 4, 3, 256
+; CHECK-NEXT:    li 4, -1
+; CHECK-NEXT:    isel 3, 3, 4, 2
+; CHECK-NEXT:    blr
+  %a = add i8 %x, %y
+  %c = icmp ugt i8 %x, %a
+  %r = select i1 %c, i8 -1, i8 %a
+  ret i8 %r
+}
+
+define i8 @unsigned_sat_variable_i8_using_cmp_notval(i8 %x, i8 %y) {
+; CHECK-LABEL: unsigned_sat_variable_i8_using_cmp_notval:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    nor 6, 4, 4
+; CHECK-NEXT:    clrlwi 7, 3, 24
+; CHECK-NEXT:    li 5, -1
+; CHECK-NEXT:    add 3, 3, 4
+; CHECK-NEXT:    clrlwi 6, 6, 24
+; CHECK-NEXT:    cmplw 7, 6
+; CHECK-NEXT:    isel 3, 5, 3, 1
+; CHECK-NEXT:    blr
+  %noty = xor i8 %y, -1
+  %a = add i8 %x, %y
+  %c = icmp ugt i8 %x, %noty
+  %r = select i1 %c, i8 -1, i8 %a
+  ret i8 %r
+}
+
+define i16 @unsigned_sat_variable_i16_using_min(i16 %x, i16 %y) {
+; CHECK-LABEL: unsigned_sat_variable_i16_using_min:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    nor 5, 4, 4
+; CHECK-NEXT:    clrlwi 6, 3, 16
+; CHECK-NEXT:    clrlwi 7, 5, 16
+; CHECK-NEXT:    cmplw 6, 7
+; CHECK-NEXT:    isel 3, 3, 5, 0
+; CHECK-NEXT:    add 3, 3, 4
+; CHECK-NEXT:    blr
+  %noty = xor i16 %y, -1
+  %c = icmp ult i16 %x, %noty
+  %s = select i1 %c, i16 %x, i16 %noty
+  %r = add i16 %s, %y
+  ret i16 %r
+}
+
+define i16 @unsigned_sat_variable_i16_using_cmp_sum(i16 %x, i16 %y) {
+; CHECK-LABEL: unsigned_sat_variable_i16_using_cmp_sum:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    rlwinm 4, 4, 0, 16, 31
+; CHECK-NEXT:    rlwinm 3, 3, 0, 16, 31
+; CHECK-NEXT:    add 3, 3, 4
+; CHECK-NEXT:    andis. 4, 3, 1
+; CHECK-NEXT:    li 4, -1
+; CHECK-NEXT:    isel 3, 3, 4, 2
+; CHECK-NEXT:    blr
+  %a = add i16 %x, %y
+  %c = icmp ugt i16 %x, %a
+  %r = select i1 %c, i16 -1, i16 %a
+  ret i16 %r
+}
+
+define i16 @unsigned_sat_variable_i16_using_cmp_notval(i16 %x, i16 %y) {
+; CHECK-LABEL: unsigned_sat_variable_i16_using_cmp_notval:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    nor 6, 4, 4
+; CHECK-NEXT:    clrlwi 7, 3, 16
+; CHECK-NEXT:    li 5, -1
+; CHECK-NEXT:    add 3, 3, 4
+; CHECK-NEXT:    clrlwi 6, 6, 16
+; CHECK-NEXT:    cmplw 7, 6
+; CHECK-NEXT:    isel 3, 5, 3, 1
+; CHECK-NEXT:    blr
+  %noty = xor i16 %y, -1
+  %a = add i16 %x, %y
+  %c = icmp ugt i16 %x, %noty
+  %r = select i1 %c, i16 -1, i16 %a
+  ret i16 %r
+}
+
+define i32 @unsigned_sat_variable_i32_using_min(i32 %x, i32 %y) {
+; CHECK-LABEL: unsigned_sat_variable_i32_using_min:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    nor 5, 4, 4
+; CHECK-NEXT:    cmplw 3, 5
+; CHECK-NEXT:    isel 3, 3, 5, 0
+; CHECK-NEXT:    add 3, 3, 4
+; CHECK-NEXT:    blr
+  %noty = xor i32 %y, -1
+  %c = icmp ult i32 %x, %noty
+  %s = select i1 %c, i32 %x, i32 %noty
+  %r = add i32 %s, %y
+  ret i32 %r
+}
+
+define i32 @unsigned_sat_variable_i32_using_cmp_sum(i32 %x, i32 %y) {
+; CHECK-LABEL: unsigned_sat_variable_i32_using_cmp_sum:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    add 4, 3, 4
+; CHECK-NEXT:    li 5, -1
+; CHECK-NEXT:    cmplw 0, 4, 3
+; CHECK-NEXT:    isel 3, 5, 4, 0
+; CHECK-NEXT:    blr
+  %a = add i32 %x, %y
+  %c = icmp ugt i32 %x, %a
+  %r = select i1 %c, i32 -1, i32 %a
+  ret i32 %r
+}
+
+define i32 @unsigned_sat_variable_i32_using_cmp_notval(i32 %x, i32 %y) {
+; CHECK-LABEL: unsigned_sat_variable_i32_using_cmp_notval:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    nor 6, 4, 4
+; CHECK-NEXT:    li 5, -1
+; CHECK-NEXT:    cmplw 3, 6
+; CHECK-NEXT:    add 3, 3, 4
+; CHECK-NEXT:    isel 3, 5, 3, 1
+; CHECK-NEXT:    blr
+  %noty = xor i32 %y, -1
+  %a = add i32 %x, %y
+  %c = icmp ugt i32 %x, %noty
+  %r = select i1 %c, i32 -1, i32 %a
+  ret i32 %r
+}
+
+define i64 @unsigned_sat_variable_i64_using_min(i64 %x, i64 %y) {
+; CHECK-LABEL: unsigned_sat_variable_i64_using_min:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    not 5, 4
+; CHECK-NEXT:    cmpld 3, 5
+; CHECK-NEXT:    isel 3, 3, 5, 0
+; CHECK-NEXT:    add 3, 3, 4
+; CHECK-NEXT:    blr
+  %noty = xor i64 %y, -1
+  %c = icmp ult i64 %x, %noty
+  %s = select i1 %c, i64 %x, i64 %noty
+  %r = add i64 %s, %y
+  ret i64 %r
+}
+
+define i64 @unsigned_sat_variable_i64_using_cmp_sum(i64 %x, i64 %y) {
+; CHECK-LABEL: unsigned_sat_variable_i64_using_cmp_sum:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    add 4, 3, 4
+; CHECK-NEXT:    li 5, -1
+; CHECK-NEXT:    cmpld 4, 3
+; CHECK-NEXT:    isel 3, 5, 4, 0
+; CHECK-NEXT:    blr
+  %a = add i64 %x, %y
+  %c = icmp ugt i64 %x, %a
+  %r = select i1 %c, i64 -1, i64 %a
+  ret i64 %r
+}
+
+define i64 @unsigned_sat_variable_i64_using_cmp_notval(i64 %x, i64 %y) {
+; CHECK-LABEL: unsigned_sat_variable_i64_using_cmp_notval:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    not 6, 4
+; CHECK-NEXT:    li 5, -1
+; CHECK-NEXT:    cmpld 3, 6
+; CHECK-NEXT:    add 3, 3, 4
+; CHECK-NEXT:    isel 3, 5, 3, 1
+; CHECK-NEXT:    blr
+  %noty = xor i64 %y, -1
+  %a = add i64 %x, %y
+  %c = icmp ugt i64 %x, %noty
+  %r = select i1 %c, i64 -1, i64 %a
+  ret i64 %r
+}
+
+define <16 x i8> @unsigned_sat_constant_v16i8_using_min(<16 x i8> %x) {
+; CHECK-LABEL: unsigned_sat_constant_v16i8_using_min:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addis 3, 2, .LCPI24_0@toc@ha
+; CHECK-NEXT:    addi 3, 3, .LCPI24_0@toc@l
+; CHECK-NEXT:    lvx 3, 0, 3
+; CHECK-NEXT:    addis 3, 2, .LCPI24_1@toc@ha
+; CHECK-NEXT:    addi 3, 3, .LCPI24_1@toc@l
+; CHECK-NEXT:    vcmpgtub 4, 3, 2
+; CHECK-NEXT:    xxsel 34, 35, 34, 36
+; CHECK-NEXT:    lvx 3, 0, 3
+; CHECK-NEXT:    vaddubm 2, 2, 3
+; CHECK-NEXT:    blr
+  %c = icmp ult <16 x i8> %x, <i8 -43, i8 -43, i8 -43, i8 -43, i8 -43, i8 -43, i8 -43, i8 -43, i8 -43, i8 -43, i8 -43, i8 -43, i8 -43, i8 -43, i8 -43, i8 -43>
+  %s = select <16 x i1> %c, <16 x i8> %x, <16 x i8> <i8 -43, i8 -43, i8 -43, i8 -43, i8 -43, i8 -43, i8 -43, i8 -43, i8 -43, i8 -43, i8 -43, i8 -43, i8 -43, i8 -43, i8 -43, i8 -43>
+  %r = add <16 x i8> %s, <i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42>
+  ret <16 x i8> %r
+}
+
+define <16 x i8> @unsigned_sat_constant_v16i8_using_cmp_sum(<16 x i8> %x) {
+; CHECK-LABEL: unsigned_sat_constant_v16i8_using_cmp_sum:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addis 3, 2, .LCPI25_0@toc@ha
+; CHECK-NEXT:    vspltisb 4, -1
+; CHECK-NEXT:    addi 3, 3, .LCPI25_0@toc@l
+; CHECK-NEXT:    lvx 3, 0, 3
+; CHECK-NEXT:    vaddubm 3, 2, 3
+; CHECK-NEXT:    vcmpgtub 2, 2, 3
+; CHECK-NEXT:    xxsel 34, 35, 36, 34
+; CHECK-NEXT:    blr
+  %a = add <16 x i8> %x, <i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42>
+  %c = icmp ugt <16 x i8> %x, %a
+  %r = select <16 x i1> %c, <16 x i8> <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>, <16 x i8> %a
+  ret <16 x i8> %r
+}
+
+define <16 x i8> @unsigned_sat_constant_v16i8_using_cmp_notval(<16 x i8> %x) {
+; CHECK-LABEL: unsigned_sat_constant_v16i8_using_cmp_notval:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addis 3, 2, .LCPI26_1@toc@ha
+; CHECK-NEXT:    vspltisb 5, -1
+; CHECK-NEXT:    addi 3, 3, .LCPI26_1@toc@l
+; CHECK-NEXT:    lvx 3, 0, 3
+; CHECK-NEXT:    addis 3, 2, .LCPI26_0@toc@ha
+; CHECK-NEXT:    addi 3, 3, .LCPI26_0@toc@l
+; CHECK-NEXT:    vcmpgtub 3, 2, 3
+; CHECK-NEXT:    lvx 4, 0, 3
+; CHECK-NEXT:    vaddubm 2, 2, 4
+; CHECK-NEXT:    xxsel 34, 34, 37, 35
+; CHECK-NEXT:    blr
+  %a = add <16 x i8> %x, <i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42>
+  %c = icmp ugt <16 x i8> %x, <i8 -43, i8 -43, i8 -43, i8 -43, i8 -43, i8 -43, i8 -43, i8 -43, i8 -43, i8 -43, i8 -43, i8 -43, i8 -43, i8 -43, i8 -43, i8 -43>
+  %r = select <16 x i1> %c, <16 x i8> <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>, <16 x i8> %a
+  ret <16 x i8> %r
+}
+
+define <8 x i16> @unsigned_sat_constant_v8i16_using_min(<8 x i16> %x) {
+; CHECK-LABEL: unsigned_sat_constant_v8i16_using_min:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addis 3, 2, .LCPI27_0@toc@ha
+; CHECK-NEXT:    addi 3, 3, .LCPI27_0@toc@l
+; CHECK-NEXT:    lvx 3, 0, 3
+; CHECK-NEXT:    addis 3, 2, .LCPI27_1@toc@ha
+; CHECK-NEXT:    addi 3, 3, .LCPI27_1@toc@l
+; CHECK-NEXT:    vcmpgtuh 4, 3, 2
+; CHECK-NEXT:    xxsel 34, 35, 34, 36
+; CHECK-NEXT:    lvx 3, 0, 3
+; CHECK-NEXT:    vadduhm 2, 2, 3
+; CHECK-NEXT:    blr
+  %c = icmp ult <8 x i16> %x, <i16 -43, i16 -43, i16 -43, i16 -43, i16 -43, i16 -43, i16 -43, i16 -43>
+  %s = select <8 x i1> %c, <8 x i16> %x, <8 x i16> <i16 -43, i16 -43, i16 -43, i16 -43, i16 -43, i16 -43, i16 -43, i16 -43>
+  %r = add <8 x i16> %s, <i16 42, i16 42, i16 42, i16 42, i16 42, i16 42, i16 42, i16 42>
+  ret <8 x i16> %r
+}
+
+define <8 x i16> @unsigned_sat_constant_v8i16_using_cmp_sum(<8 x i16> %x) {
+; CHECK-LABEL: unsigned_sat_constant_v8i16_using_cmp_sum:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addis 3, 2, .LCPI28_0@toc@ha
+; CHECK-NEXT:    vspltisb 4, -1
+; CHECK-NEXT:    addi 3, 3, .LCPI28_0@toc@l
+; CHECK-NEXT:    lvx 3, 0, 3
+; CHECK-NEXT:    vadduhm 3, 2, 3
+; CHECK-NEXT:    vcmpgtuh 2, 2, 3
+; CHECK-NEXT:    xxsel 34, 35, 36, 34
+; CHECK-NEXT:    blr
+  %a = add <8 x i16> %x, <i16 42, i16 42, i16 42, i16 42, i16 42, i16 42, i16 42, i16 42>
+  %c = icmp ugt <8 x i16> %x, %a
+  %r = select <8 x i1> %c, <8 x i16> <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>, <8 x i16> %a
+  ret <8 x i16> %r
+}
+
+define <8 x i16> @unsigned_sat_constant_v8i16_using_cmp_notval(<8 x i16> %x) {
+; CHECK-LABEL: unsigned_sat_constant_v8i16_using_cmp_notval:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addis 3, 2, .LCPI29_1@toc@ha
+; CHECK-NEXT:    vspltisb 5, -1
+; CHECK-NEXT:    addi 3, 3, .LCPI29_1@toc@l
+; CHECK-NEXT:    lvx 3, 0, 3
+; CHECK-NEXT:    addis 3, 2, .LCPI29_0@toc@ha
+; CHECK-NEXT:    addi 3, 3, .LCPI29_0@toc@l
+; CHECK-NEXT:    vcmpgtuh 3, 2, 3
+; CHECK-NEXT:    lvx 4, 0, 3
+; CHECK-NEXT:    vadduhm 2, 2, 4
+; CHECK-NEXT:    xxsel 34, 34, 37, 35
+; CHECK-NEXT:    blr
+  %a = add <8 x i16> %x, <i16 42, i16 42, i16 42, i16 42, i16 42, i16 42, i16 42, i16 42>
+  %c = icmp ugt <8 x i16> %x, <i16 -43, i16 -43, i16 -43, i16 -43, i16 -43, i16 -43, i16 -43, i16 -43>
+  %r = select <8 x i1> %c, <8 x i16> <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>, <8 x i16> %a
+  ret <8 x i16> %r
+}
+
+define <4 x i32> @unsigned_sat_constant_v4i32_using_min(<4 x i32> %x) {
+; CHECK-LABEL: unsigned_sat_constant_v4i32_using_min:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addis 3, 2, .LCPI30_0@toc@ha
+; CHECK-NEXT:    addi 3, 3, .LCPI30_0@toc@l
+; CHECK-NEXT:    lvx 3, 0, 3
+; CHECK-NEXT:    addis 3, 2, .LCPI30_1@toc@ha
+; CHECK-NEXT:    addi 3, 3, .LCPI30_1@toc@l
+; CHECK-NEXT:    vcmpgtuw 4, 3, 2
+; CHECK-NEXT:    xxsel 34, 35, 34, 36
+; CHECK-NEXT:    lvx 3, 0, 3
+; CHECK-NEXT:    vadduwm 2, 2, 3
+; CHECK-NEXT:    blr
+  %c = icmp ult <4 x i32> %x, <i32 -43, i32 -43, i32 -43, i32 -43>
+  %s = select <4 x i1> %c, <4 x i32> %x, <4 x i32> <i32 -43, i32 -43, i32 -43, i32 -43>
+  %r = add <4 x i32> %s, <i32 42, i32 42, i32 42, i32 42>
+  ret <4 x i32> %r
+}
+
+define <4 x i32> @unsigned_sat_constant_v4i32_using_cmp_sum(<4 x i32> %x) {
+; CHECK-LABEL: unsigned_sat_constant_v4i32_using_cmp_sum:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addis 3, 2, .LCPI31_0@toc@ha
+; CHECK-NEXT:    vspltisb 4, -1
+; CHECK-NEXT:    addi 3, 3, .LCPI31_0@toc@l
+; CHECK-NEXT:    lvx 3, 0, 3
+; CHECK-NEXT:    vadduwm 3, 2, 3
+; CHECK-NEXT:    vcmpgtuw 2, 2, 3
+; CHECK-NEXT:    xxsel 34, 35, 36, 34
+; CHECK-NEXT:    blr
+  %a = add <4 x i32> %x, <i32 42, i32 42, i32 42, i32 42>
+  %c = icmp ugt <4 x i32> %x, %a
+  %r = select <4 x i1> %c, <4 x i32> <i32 -1, i32 -1, i32 -1, i32 -1>, <4 x i32> %a
+  ret <4 x i32> %r
+}
+
+define <4 x i32> @unsigned_sat_constant_v4i32_using_cmp_notval(<4 x i32> %x) {
+; CHECK-LABEL: unsigned_sat_constant_v4i32_using_cmp_notval:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addis 3, 2, .LCPI32_1@toc@ha
+; CHECK-NEXT:    vspltisb 5, -1
+; CHECK-NEXT:    addi 3, 3, .LCPI32_1@toc@l
+; CHECK-NEXT:    lvx 3, 0, 3
+; CHECK-NEXT:    addis 3, 2, .LCPI32_0@toc@ha
+; CHECK-NEXT:    addi 3, 3, .LCPI32_0@toc@l
+; CHECK-NEXT:    vcmpgtuw 3, 2, 3
+; CHECK-NEXT:    lvx 4, 0, 3
+; CHECK-NEXT:    vadduwm 2, 2, 4
+; CHECK-NEXT:    xxsel 34, 34, 37, 35
+; CHECK-NEXT:    blr
+  %a = add <4 x i32> %x, <i32 42, i32 42, i32 42, i32 42>
+  %c = icmp ugt <4 x i32> %x, <i32 -43, i32 -43, i32 -43, i32 -43>
+  %r = select <4 x i1> %c, <4 x i32> <i32 -1, i32 -1, i32 -1, i32 -1>, <4 x i32> %a
+  ret <4 x i32> %r
+}
+
+define <2 x i64> @unsigned_sat_constant_v2i64_using_min(<2 x i64> %x) {
+; CHECK-LABEL: unsigned_sat_constant_v2i64_using_min:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addis 3, 2, .LCPI33_0@toc@ha
+; CHECK-NEXT:    addi 3, 3, .LCPI33_0@toc@l
+; CHECK-NEXT:    lxvd2x 0, 0, 3
+; CHECK-NEXT:    addis 3, 2, .LCPI33_1@toc@ha
+; CHECK-NEXT:    addi 3, 3, .LCPI33_1@toc@l
+; CHECK-NEXT:    xxswapd 35, 0
+; CHECK-NEXT:    lxvd2x 0, 0, 3
+; CHECK-NEXT:    vcmpgtud 4, 3, 2
+; CHECK-NEXT:    xxsel 34, 35, 34, 36
+; CHECK-NEXT:    xxswapd 35, 0
+; CHECK-NEXT:    vaddudm 2, 2, 3
+; CHECK-NEXT:    blr
+  %c = icmp ult <2 x i64> %x, <i64 -43, i64 -43>
+  %s = select <2 x i1> %c, <2 x i64> %x, <2 x i64> <i64 -43, i64 -43>
+  %r = add <2 x i64> %s, <i64 42, i64 42>
+  ret <2 x i64> %r
+}
+
+define <2 x i64> @unsigned_sat_constant_v2i64_using_cmp_sum(<2 x i64> %x) {
+; CHECK-LABEL: unsigned_sat_constant_v2i64_using_cmp_sum:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addis 3, 2, .LCPI34_0@toc@ha
+; CHECK-NEXT:    vspltisb 4, -1
+; CHECK-NEXT:    addi 3, 3, .LCPI34_0@toc@l
+; CHECK-NEXT:    lxvd2x 0, 0, 3
+; CHECK-NEXT:    xxswapd 35, 0
+; CHECK-NEXT:    vaddudm 3, 2, 3
+; CHECK-NEXT:    vcmpgtud 2, 2, 3
+; CHECK-NEXT:    xxsel 34, 35, 36, 34
+; CHECK-NEXT:    blr
+  %a = add <2 x i64> %x, <i64 42, i64 42>
+  %c = icmp ugt <2 x i64> %x, %a
+  %r = select <2 x i1> %c, <2 x i64> <i64 -1, i64 -1>, <2 x i64> %a
+  ret <2 x i64> %r
+}
+
+define <2 x i64> @unsigned_sat_constant_v2i64_using_cmp_notval(<2 x i64> %x) {
+; CHECK-LABEL: unsigned_sat_constant_v2i64_using_cmp_notval:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addis 3, 2, .LCPI35_1@toc@ha
+; CHECK-NEXT:    vspltisb 5, -1
+; CHECK-NEXT:    addi 3, 3, .LCPI35_1@toc@l
+; CHECK-NEXT:    lxvd2x 0, 0, 3
+; CHECK-NEXT:    addis 3, 2, .LCPI35_0@toc@ha
+; CHECK-NEXT:    addi 3, 3, .LCPI35_0@toc@l
+; CHECK-NEXT:    xxswapd 35, 0
+; CHECK-NEXT:    lxvd2x 0, 0, 3
+; CHECK-NEXT:    vcmpgtud 3, 2, 3
+; CHECK-NEXT:    xxswapd 36, 0
+; CHECK-NEXT:    vaddudm 2, 2, 4
+; CHECK-NEXT:    xxsel 34, 34, 37, 35
+; CHECK-NEXT:    blr
+  %a = add <2 x i64> %x, <i64 42, i64 42>
+  %c = icmp ugt <2 x i64> %x, <i64 -43, i64 -43>
+  %r = select <2 x i1> %c, <2 x i64> <i64 -1, i64 -1>, <2 x i64> %a
+  ret <2 x i64> %r
+}
+
+define <16 x i8> @unsigned_sat_variable_v16i8_using_min(<16 x i8> %x, <16 x i8> %y) {
+; CHECK-LABEL: unsigned_sat_variable_v16i8_using_min:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xxlnor 36, 35, 35
+; CHECK-NEXT:    vcmpgtub 5, 4, 2
+; CHECK-NEXT:    xxsel 34, 36, 34, 37
+; CHECK-NEXT:    vaddubm 2, 2, 3
+; CHECK-NEXT:    blr
+  %noty = xor <16 x i8> %y, <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>
+  %c = icmp ult <16 x i8> %x, %noty
+  %s = select <16 x i1> %c, <16 x i8> %x, <16 x i8> %noty
+  %r = add <16 x i8> %s, %y
+  ret <16 x i8> %r
+}
+
+define <16 x i8> @unsigned_sat_variable_v16i8_using_cmp_sum(<16 x i8> %x, <16 x i8> %y) {
+; CHECK-LABEL: unsigned_sat_variable_v16i8_using_cmp_sum:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vaddubm 3, 2, 3
+; CHECK-NEXT:    vspltisb 4, -1
+; CHECK-NEXT:    vcmpgtub 2, 2, 3
+; CHECK-NEXT:    xxsel 34, 35, 36, 34
+; CHECK-NEXT:    blr
+  %a = add <16 x i8> %x, %y
+  %c = icmp ugt <16 x i8> %x, %a
+  %r = select <16 x i1> %c, <16 x i8> <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>, <16 x i8> %a
+  ret <16 x i8> %r
+}
+
+define <16 x i8> @unsigned_sat_variable_v16i8_using_cmp_notval(<16 x i8> %x, <16 x i8> %y) {
+; CHECK-LABEL: unsigned_sat_variable_v16i8_using_cmp_notval:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xxlnor 36, 35, 35
+; CHECK-NEXT:    vspltisb 5, -1
+; CHECK-NEXT:    vcmpgtub 4, 2, 4
+; CHECK-NEXT:    vaddubm 2, 2, 3
+; CHECK-NEXT:    xxsel 34, 34, 37, 36
+; CHECK-NEXT:    blr
+  %noty = xor <16 x i8> %y, <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>
+  %a = add <16 x i8> %x, %y
+  %c = icmp ugt <16 x i8> %x, %noty
+  %r = select <16 x i1> %c, <16 x i8> <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>, <16 x i8> %a
+  ret <16 x i8> %r
+}
+
+define <8 x i16> @unsigned_sat_variable_v8i16_using_min(<8 x i16> %x, <8 x i16> %y) {
+; CHECK-LABEL: unsigned_sat_variable_v8i16_using_min:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xxlnor 36, 35, 35
+; CHECK-NEXT:    vcmpgtuh 5, 4, 2
+; CHECK-NEXT:    xxsel 34, 36, 34, 37
+; CHECK-NEXT:    vadduhm 2, 2, 3
+; CHECK-NEXT:    blr
+  %noty = xor <8 x i16> %y, <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>
+  %c = icmp ult <8 x i16> %x, %noty
+  %s = select <8 x i1> %c, <8 x i16> %x, <8 x i16> %noty
+  %r = add <8 x i16> %s, %y
+  ret <8 x i16> %r
+}
+
+define <8 x i16> @unsigned_sat_variable_v8i16_using_cmp_sum(<8 x i16> %x, <8 x i16> %y) {
+; CHECK-LABEL: unsigned_sat_variable_v8i16_using_cmp_sum:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vadduhm 3, 2, 3
+; CHECK-NEXT:    vspltisb 4, -1
+; CHECK-NEXT:    vcmpgtuh 2, 2, 3
+; CHECK-NEXT:    xxsel 34, 35, 36, 34
+; CHECK-NEXT:    blr
+  %a = add <8 x i16> %x, %y
+  %c = icmp ugt <8 x i16> %x, %a
+  %r = select <8 x i1> %c, <8 x i16> <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>, <8 x i16> %a
+  ret <8 x i16> %r
+}
+
+define <8 x i16> @unsigned_sat_variable_v8i16_using_cmp_notval(<8 x i16> %x, <8 x i16> %y) {
+; CHECK-LABEL: unsigned_sat_variable_v8i16_using_cmp_notval:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xxlnor 36, 35, 35
+; CHECK-NEXT:    vspltisb 5, -1
+; CHECK-NEXT:    vcmpgtuh 4, 2, 4
+; CHECK-NEXT:    vadduhm 2, 2, 3
+; CHECK-NEXT:    xxsel 34, 34, 37, 36
+; CHECK-NEXT:    blr
+  %noty = xor <8 x i16> %y, <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>
+  %a = add <8 x i16> %x, %y
+  %c = icmp ugt <8 x i16> %x, %noty
+  %r = select <8 x i1> %c, <8 x i16> <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>, <8 x i16> %a
+  ret <8 x i16> %r
+}
+
+define <4 x i32> @unsigned_sat_variable_v4i32_using_min(<4 x i32> %x, <4 x i32> %y) {
+; CHECK-LABEL: unsigned_sat_variable_v4i32_using_min:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xxlnor 36, 35, 35
+; CHECK-NEXT:    vcmpgtuw 5, 4, 2
+; CHECK-NEXT:    xxsel 34, 36, 34, 37
+; CHECK-NEXT:    vadduwm 2, 2, 3
+; CHECK-NEXT:    blr
+  %noty = xor <4 x i32> %y, <i32 -1, i32 -1, i32 -1, i32 -1>
+  %c = icmp ult <4 x i32> %x, %noty
+  %s = select <4 x i1> %c, <4 x i32> %x, <4 x i32> %noty
+  %r = add <4 x i32> %s, %y
+  ret <4 x i32> %r
+}
+
+define <4 x i32> @unsigned_sat_variable_v4i32_using_cmp_sum(<4 x i32> %x, <4 x i32> %y) {
+; CHECK-LABEL: unsigned_sat_variable_v4i32_using_cmp_sum:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vadduwm 3, 2, 3
+; CHECK-NEXT:    vspltisb 4, -1
+; CHECK-NEXT:    vcmpgtuw 2, 2, 3
+; CHECK-NEXT:    xxsel 34, 35, 36, 34
+; CHECK-NEXT:    blr
+  %a = add <4 x i32> %x, %y
+  %c = icmp ugt <4 x i32> %x, %a
+  %r = select <4 x i1> %c, <4 x i32> <i32 -1, i32 -1, i32 -1, i32 -1>, <4 x i32> %a
+  ret <4 x i32> %r
+}
+
+define <4 x i32> @unsigned_sat_variable_v4i32_using_cmp_notval(<4 x i32> %x, <4 x i32> %y) {
+; CHECK-LABEL: unsigned_sat_variable_v4i32_using_cmp_notval:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xxlnor 36, 35, 35
+; CHECK-NEXT:    vspltisb 5, -1
+; CHECK-NEXT:    vcmpgtuw 4, 2, 4
+; CHECK-NEXT:    vadduwm 2, 2, 3
+; CHECK-NEXT:    xxsel 34, 34, 37, 36
+; CHECK-NEXT:    blr
+  %noty = xor <4 x i32> %y, <i32 -1, i32 -1, i32 -1, i32 -1>
+  %a = add <4 x i32> %x, %y
+  %c = icmp ugt <4 x i32> %x, %noty
+  %r = select <4 x i1> %c, <4 x i32> <i32 -1, i32 -1, i32 -1, i32 -1>, <4 x i32> %a
+  ret <4 x i32> %r
+}
+
+define <2 x i64> @unsigned_sat_variable_v2i64_using_min(<2 x i64> %x, <2 x i64> %y) {
+; CHECK-LABEL: unsigned_sat_variable_v2i64_using_min:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xxlnor 36, 35, 35
+; CHECK-NEXT:    vcmpgtud 5, 4, 2
+; CHECK-NEXT:    xxsel 34, 36, 34, 37
+; CHECK-NEXT:    vaddudm 2, 2, 3
+; CHECK-NEXT:    blr
+  %noty = xor <2 x i64> %y, <i64 -1, i64 -1>
+  %c = icmp ult <2 x i64> %x, %noty
+  %s = select <2 x i1> %c, <2 x i64> %x, <2 x i64> %noty
+  %r = add <2 x i64> %s, %y
+  ret <2 x i64> %r
+}
+
+define <2 x i64> @unsigned_sat_variable_v2i64_using_cmp_sum(<2 x i64> %x, <2 x i64> %y) {
+; CHECK-LABEL: unsigned_sat_variable_v2i64_using_cmp_sum:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vaddudm 3, 2, 3
+; CHECK-NEXT:    vspltisb 4, -1
+; CHECK-NEXT:    vcmpgtud 2, 2, 3
+; CHECK-NEXT:    xxsel 34, 35, 36, 34
+; CHECK-NEXT:    blr
+  %a = add <2 x i64> %x, %y
+  %c = icmp ugt <2 x i64> %x, %a
+  %r = select <2 x i1> %c, <2 x i64> <i64 -1, i64 -1>, <2 x i64> %a
+  ret <2 x i64> %r
+}
+
+define <2 x i64> @unsigned_sat_variable_v2i64_using_cmp_notval(<2 x i64> %x, <2 x i64> %y) {
+; CHECK-LABEL: unsigned_sat_variable_v2i64_using_cmp_notval:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xxlnor 36, 35, 35
+; CHECK-NEXT:    vspltisb 5, -1
+; CHECK-NEXT:    vcmpgtud 4, 2, 4
+; CHECK-NEXT:    vaddudm 2, 2, 3
+; CHECK-NEXT:    xxsel 34, 34, 37, 36
+; CHECK-NEXT:    blr
+  %noty = xor <2 x i64> %y, <i64 -1, i64 -1>
+  %a = add <2 x i64> %x, %y
+  %c = icmp ugt <2 x i64> %x, %noty
+  %r = select <2 x i1> %c, <2 x i64> <i64 -1, i64 -1>, <2 x i64> %a
+  ret <2 x i64> %r
+}
+
diff --git a/test/CodeGen/PowerPC/scalar_vector_test_1.ll b/test/CodeGen/PowerPC/scalar_vector_test_1.ll
index d8f44f1211db3ec09b214ea4cd411f556aaf448b..5ad28e0f96e7a95e5f72d3ffb5d81d884be6647c 100644
--- a/test/CodeGen/PowerPC/scalar_vector_test_1.ll
+++ b/test/CodeGen/PowerPC/scalar_vector_test_1.ll
@@ -7,6 +7,7 @@
 ; RUN: llc -mcpu=pwr8 -verify-machineinstrs -ppc-vsr-nums-as-vr -ppc-asm-full-reg-names \
 ; RUN:    -mtriple=powerpc64-unknown-linux-gnu < %s | FileCheck %s --check-prefix=P8BE
 
+
 ; Function Attrs: norecurse nounwind readonly
 define <2 x i64> @s2v_test1(i64* nocapture readonly %int64, <2 x i64> %vec) {
 ; P9LE-LABEL: s2v_test1:
diff --git a/test/CodeGen/PowerPC/scalar_vector_test_2.ll b/test/CodeGen/PowerPC/scalar_vector_test_2.ll
index da1b8bcaa3d0282ef1d495f693cc4e2b4b204ca6..af695c58f1b113ad5863a9d33668bd87bbbfa6eb 100644
--- a/test/CodeGen/PowerPC/scalar_vector_test_2.ll
+++ b/test/CodeGen/PowerPC/scalar_vector_test_2.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mcpu=pwr9 -verify-machineinstrs -ppc-vsr-nums-as-vr -ppc-asm-full-reg-names \
 ; RUN:    -mtriple=powerpc64le-unknown-linux-gnu < %s | FileCheck %s --check-prefix=P9LE
 ; RUN: llc -mcpu=pwr9 -verify-machineinstrs -ppc-vsr-nums-as-vr -ppc-asm-full-reg-names \
@@ -15,11 +16,10 @@ define void @test_liwzx1(<1 x float>* %A, <1 x float>* %B, <1 x float>* %C) {
 ; P9LE-NEXT:    xxpermdi vs0, f0, f0, 2
 ; P9LE-NEXT:    xxpermdi vs1, f1, f1, 2
 ; P9LE-NEXT:    xvaddsp vs0, vs0, vs1
-; P9LE-NEXT:    xxsldwi vs0, vs0, vs0, 3
-; P9LE-NEXT:    xscvspdpn f0, vs0
-; P9LE-NEXT:    stfs f0, 0(r5)
+; P9LE-NEXT:    xxsldwi vs0, vs0, vs0, 2
+; P9LE-NEXT:    stfiwx f0, 0, r5
 ; P9LE-NEXT:    blr
-
+;
 ; P9BE-LABEL: test_liwzx1:
 ; P9BE:       # %bb.0:
 ; P9BE-NEXT:    lfiwzx f0, 0, r3
@@ -27,10 +27,10 @@ define void @test_liwzx1(<1 x float>* %A, <1 x float>* %B, <1 x float>* %C) {
 ; P9BE-NEXT:    xxsldwi vs0, f0, f0, 1
 ; P9BE-NEXT:    xxsldwi vs1, f1, f1, 1
 ; P9BE-NEXT:    xvaddsp vs0, vs0, vs1
-; P9BE-NEXT:    xscvspdpn f0, vs0
-; P9BE-NEXT:    stfs f0, 0(r5)
+; P9BE-NEXT:    xxsldwi vs0, vs0, vs0, 3
+; P9BE-NEXT:    stfiwx f0, 0, r5
 ; P9BE-NEXT:    blr
-
+;
 ; P8LE-LABEL: test_liwzx1:
 ; P8LE:       # %bb.0:
 ; P8LE-NEXT:    lfiwzx f0, 0, r3
@@ -38,11 +38,10 @@ define void @test_liwzx1(<1 x float>* %A, <1 x float>* %B, <1 x float>* %C) {
 ; P8LE-NEXT:    xxpermdi vs0, f0, f0, 2
 ; P8LE-NEXT:    xxpermdi vs1, f1, f1, 2
 ; P8LE-NEXT:    xvaddsp vs0, vs0, vs1
-; P8LE-NEXT:    xxsldwi vs0, vs0, vs0, 3
-; P8LE-NEXT:    xscvspdpn f0, vs0
-; P8LE-NEXT:    stfsx f0, 0, r5
+; P8LE-NEXT:    xxsldwi vs0, vs0, vs0, 2
+; P8LE-NEXT:    stfiwx f0, 0, r5
 ; P8LE-NEXT:    blr
-
+;
 ; P8BE-LABEL: test_liwzx1:
 ; P8BE:       # %bb.0:
 ; P8BE-NEXT:    lfiwzx f0, 0, r3
@@ -50,9 +49,12 @@ define void @test_liwzx1(<1 x float>* %A, <1 x float>* %B, <1 x float>* %C) {
 ; P8BE-NEXT:    xxsldwi vs0, f0, f0, 1
 ; P8BE-NEXT:    xxsldwi vs1, f1, f1, 1
 ; P8BE-NEXT:    xvaddsp vs0, vs0, vs1
-; P8BE-NEXT:    xscvspdpn f0, vs0
-; P8BE-NEXT:    stfsx f0, 0, r5
+; P8BE-NEXT:    xxsldwi vs0, vs0, vs0, 3
+; P8BE-NEXT:    stfiwx f0, 0, r5
 ; P8BE-NEXT:    blr
+
+
+
   %a = load <1 x float>, <1 x float>* %A
   %b = load <1 x float>, <1 x float>* %B
   %X = fadd <1 x float> %a, %b
@@ -65,27 +67,26 @@ define <1 x float>* @test_liwzx2(<1 x float>* %A, <1 x float>* %B, <1 x float>*
 ; P9LE:       # %bb.0:
 ; P9LE-NEXT:    lfiwzx f0, 0, r3
 ; P9LE-NEXT:    lfiwzx f1, 0, r4
-; P9LE-NEXT:    mr r3, r5
 ; P9LE-NEXT:    xxpermdi vs0, f0, f0, 2
 ; P9LE-NEXT:    xxpermdi vs1, f1, f1, 2
 ; P9LE-NEXT:    xvsubsp vs0, vs0, vs1
-; P9LE-NEXT:    xxsldwi vs0, vs0, vs0, 3
-; P9LE-NEXT:    xscvspdpn f0, vs0
-; P9LE-NEXT:    stfs f0, 0(r5)
+; P9LE-NEXT:    xxsldwi vs0, vs0, vs0, 2
+; P9LE-NEXT:    mr r3, r5
+; P9LE-NEXT:    stfiwx f0, 0, r5
 ; P9LE-NEXT:    blr
-
+;
 ; P9BE-LABEL: test_liwzx2:
 ; P9BE:       # %bb.0:
 ; P9BE-NEXT:    lfiwzx f0, 0, r3
 ; P9BE-NEXT:    lfiwzx f1, 0, r4
-; P9BE-NEXT:    mr r3, r5
 ; P9BE-NEXT:    xxsldwi vs0, f0, f0, 1
 ; P9BE-NEXT:    xxsldwi vs1, f1, f1, 1
 ; P9BE-NEXT:    xvsubsp vs0, vs0, vs1
-; P9BE-NEXT:    xscvspdpn f0, vs0
-; P9BE-NEXT:    stfs f0, 0(r5)
+; P9BE-NEXT:    xxsldwi vs0, vs0, vs0, 3
+; P9BE-NEXT:    mr r3, r5
+; P9BE-NEXT:    stfiwx f0, 0, r5
 ; P9BE-NEXT:    blr
-
+;
 ; P8LE-LABEL: test_liwzx2:
 ; P8LE:       # %bb.0:
 ; P8LE-NEXT:    lfiwzx f0, 0, r3
@@ -94,11 +95,10 @@ define <1 x float>* @test_liwzx2(<1 x float>* %A, <1 x float>* %B, <1 x float>*
 ; P8LE-NEXT:    xxpermdi vs0, f0, f0, 2
 ; P8LE-NEXT:    xxpermdi vs1, f1, f1, 2
 ; P8LE-NEXT:    xvsubsp vs0, vs0, vs1
-; P8LE-NEXT:    xxsldwi vs0, vs0, vs0, 3
-; P8LE-NEXT:    xscvspdpn f0, vs0
-; P8LE-NEXT:    stfsx f0, 0, r5
+; P8LE-NEXT:    xxsldwi vs0, vs0, vs0, 2
+; P8LE-NEXT:    stfiwx f0, 0, r5
 ; P8LE-NEXT:    blr
-
+;
 ; P8BE-LABEL: test_liwzx2:
 ; P8BE:       # %bb.0:
 ; P8BE-NEXT:    lfiwzx f0, 0, r3
@@ -107,9 +107,14 @@ define <1 x float>* @test_liwzx2(<1 x float>* %A, <1 x float>* %B, <1 x float>*
 ; P8BE-NEXT:    xxsldwi vs0, f0, f0, 1
 ; P8BE-NEXT:    xxsldwi vs1, f1, f1, 1
 ; P8BE-NEXT:    xvsubsp vs0, vs0, vs1
-; P8BE-NEXT:    xscvspdpn f0, vs0
-; P8BE-NEXT:    stfsx f0, 0, r5
+; P8BE-NEXT:    xxsldwi vs0, vs0, vs0, 3
+; P8BE-NEXT:    stfiwx f0, 0, r5
 ; P8BE-NEXT:    blr
+
+
+
+
+
   %a = load <1 x float>, <1 x float>* %A
   %b = load <1 x float>, <1 x float>* %B
   %X = fsub <1 x float> %a, %b
diff --git a/test/CodeGen/PowerPC/scalar_vector_test_4.ll b/test/CodeGen/PowerPC/scalar_vector_test_4.ll
index aaaf0ba60f1db26003bdbce00e8185a79299b6df..2a7e177869fa324893071752a06728294fad3264 100644
--- a/test/CodeGen/PowerPC/scalar_vector_test_4.ll
+++ b/test/CodeGen/PowerPC/scalar_vector_test_4.ll
@@ -172,8 +172,8 @@ define <2 x float> @s2v_test_f2(float* nocapture readonly %f64, <2 x float> %vec
 ; P9LE-LABEL: s2v_test_f2:
 ; P9LE:       # %bb.0: # %entry
 ; P9LE-NEXT:    addi r3, r3, 4
-; P9LE-NEXT:    xxspltw v2, v2, 2
-; P9LE-NEXT:    lfiwzx f0, 0, r3
+; P9LE-DAG:     xxspltw v2, v2, 2
+; P9LE-DAG:     lfiwzx f0, 0, r3
 ; P9LE-NEXT:    xxpermdi v3, f0, f0, 2
 ; P9LE-NEXT:    vmrglw v2, v2, v3
 ; P9LE-NEXT:    blr
@@ -181,8 +181,8 @@ define <2 x float> @s2v_test_f2(float* nocapture readonly %f64, <2 x float> %vec
 ; P9BE-LABEL: s2v_test_f2:
 ; P9BE:       # %bb.0: # %entry
 ; P9BE:       addi r3, r3, 4
-; P9BE:       xxspltw v2, v2, 1
-; P9BE:       lfiwzx f0, 0, r3
+; P9BE-DAG:   xxspltw v2, v2, 1
+; P9BE-DAG:   lfiwzx f0, 0, r3
 ; P9BE-NEXT:  xxsldwi v3, f0, f0, 1
 ; P9BE:       vmrghw v2, v3, v2
 ; P9BE-NEXT:  blr
@@ -216,18 +216,18 @@ define <2 x float> @s2v_test_f3(float* nocapture readonly %f64, <2 x float> %vec
 ; P9LE-LABEL: s2v_test_f3:
 ; P9LE:       # %bb.0: # %entry
 ; P9LE-NEXT:    sldi r4, r7, 2
-; P9LE-NEXT:    xxspltw v2, v2, 2
 ; P9LE-NEXT:    lfiwzx f0, r3, r4
-; P9LE-NEXT:    xxpermdi v3, f0, f0, 2
+; P9LE-DAG:     xxspltw v2, v2, 2
+; P9LE-DAG:     xxpermdi v3, f0, f0, 2
 ; P9LE-NEXT:    vmrglw v2, v2, v3
 ; P9LE-NEXT:    blr
 
 ; P9BE-LABEL: s2v_test_f3:
 ; P9BE:       # %bb.0: # %entry
 ; P9BE:         sldi r4, r7, 2
-; P9BE:         xxspltw v2, v2, 1
 ; P9BE:         lfiwzx f0, r3, r4
-; P9BE-NEXT:    xxsldwi v3, f0, f0, 1
+; P9BE-DAG:     xxspltw v2, v2, 1
+; P9BE-DAG:     xxsldwi v3, f0, f0, 1
 ; P9BE:         vmrghw v2, v3, v2
 ; P9BE-NEXT:    blr
 
@@ -261,18 +261,18 @@ define <2 x float> @s2v_test_f4(float* nocapture readonly %f64, <2 x float> %vec
 ; P9LE-LABEL: s2v_test_f4:
 ; P9LE:       # %bb.0: # %entry
 ; P9LE-NEXT:    addi r3, r3, 4
-; P9LE-NEXT:    xxspltw v2, v2, 2
 ; P9LE-NEXT:    lfiwzx f0, 0, r3
-; P9LE-NEXT:    xxpermdi v3, f0, f0, 2
+; P9LE-DAG:    xxspltw v2, v2, 2
+; P9LE-DAG:    xxpermdi v3, f0, f0, 2
 ; P9LE-NEXT:    vmrglw v2, v2, v3
 ; P9LE-NEXT:    blr
 
 ; P9BE-LABEL: s2v_test_f4:
 ; P9BE:       # %bb.0: # %entry
 ; P9BE:         addi r3, r3, 4
-; P9BE:         xxspltw v2, v2, 1
 ; P9BE:         lfiwzx f0, 0, r3
-; P9BE-NEXT:    xxsldwi v3, f0, f0, 1
+; P9BE-DAG:     xxspltw v2, v2, 1
+; P9BE-DAG:     xxsldwi v3, f0, f0, 1
 ; P9BE:         vmrghw v2, v3, v2
 ; P9BE-NEXT:    blr
 
diff --git a/test/CodeGen/PowerPC/select-i1-vs-i1.ll b/test/CodeGen/PowerPC/select-i1-vs-i1.ll
index a2df1828afc8965da0929ce1a6992dfefb8e88a2..51bd26d6828cc594461605ade46b8fe0a8635241 100644
--- a/test/CodeGen/PowerPC/select-i1-vs-i1.ll
+++ b/test/CodeGen/PowerPC/select-i1-vs-i1.ll
@@ -928,10 +928,8 @@ entry:
 ; CHECK-LABEL: @testv4floateq
 ; CHECK-DAG: fcmpu {{[0-9]+}}, 3, 4
 ; CHECK-DAG: fcmpu {{[0-9]+}}, 1, 2
-; CHECK: crxor [[REG1:[0-9]+]], {{[0-9]+}}, {{[0-9]+}}
-; CHECK: bc 12, [[REG1]], .LBB[[BB1:[0-9_]+]]
-; CHECK: vmr 3, 2
-; CHECK: .LBB[[BB1]]
+; CHECK: creqv [[REG1:[0-9]+]], {{[0-9]+}}, {{[0-9]+}}
+; CHECK: bclr 12, [[REG1]], 0
 ; CHECK: vmr 2, 3
 ; CHECK: blr
 }
@@ -1065,7 +1063,7 @@ entry:
 ; CHECK-DAG: fcmpu {{[0-9]+}}, 3, 4
 ; CHECK: bc 12, 2, .LBB[[BB:[0-9_]+]]
 ; CHECK-DAG: fcmpu {{[0-9]+}}, 1, 2
-; CHECK: bc 4, 2, .LBB[[BB]]
+; CHECK: bclr 12, 2, 0
 ; CHECK: .LBB[[BB]]:
 ; CHECK: vmr 2, 3
 ; CHECK: blr
@@ -1083,7 +1081,7 @@ entry:
 ; CHECK-DAG: fcmpu {{[0-9]+}}, 3, 4
 ; CHECK: bc 4, 2, .LBB[[BB:[0-9_]+]]
 ; CHECK-DAG: fcmpu {{[0-9]+}}, 1, 2
-; CHECK: bc 12, 2, .LBB[[BB]]
+; CHECK: bclr 4, 2, 0
 ; CHECK: .LBB[[BB]]:
 ; CHECK: vmr 2, 3
 ; CHECK: blr
@@ -1134,10 +1132,8 @@ entry:
 ; CHECK-LABEL: @testv2doubleeq
 ; CHECK-DAG: fcmpu {{[0-9]+}}, 3, 4
 ; CHECK-DAG: fcmpu {{[0-9]+}}, 1, 2
-; CHECK: crxor [[REG1:[0-9]+]], {{[0-9]+}}, {{[0-9]+}}
-; CHECK: bc 12, [[REG1]], .LBB[[BB55:[0-9_]+]]
-; CHECK: vmr 3, 2
-; CHECK: .LBB[[BB55]]
+; CHECK: creqv [[REG1:[0-9]+]], {{[0-9]+}}, {{[0-9]+}}
+; CHECK: bclr 12, [[REG1]], 0
 ; CHECK: vmr 2, 3
 ; CHECK: blr
 }
@@ -1188,7 +1184,7 @@ entry:
 ; CHECK-DAG: fcmpu {{[0-9]+}}, 3, 4
 ; CHECK: bc 4, 2, .LBB[[BB:[0-9_]+]]
 ; CHECK-DAG: fcmpu {{[0-9]+}}, 1, 2
-; CHECK: bc 12, 2, .LBB[[BB]]
+; CHECK: bclr 4, 2, 0
 ; CHECK: .LBB[[BB]]
 ; CHECK: vmr 2, 3
 ; CHECK: blr
@@ -1206,7 +1202,7 @@ entry:
 ; CHECK-DAG: fcmpu {{[0-9]+}}, 3, 4
 ; CHECK: bc 12, 2, .LBB[[BB:[0-9_]+]]
 ; CHECK-DAG: fcmpu {{[0-9]+}}, 1, 2
-; CHECK: bc 4, 2, .LBB[[BB]]
+; CHECK: bclr 12, 2, 0
 ; CHECK: .LBB[[BB]]
 ; CHECK: vmr 2, 3
 ; CHECK: blr
diff --git a/test/CodeGen/PowerPC/setcc-logic.ll b/test/CodeGen/PowerPC/setcc-logic.ll
index e883531d0094be910905b76dfd839666b0269004..87f73e7b2e8777d0112a8b6d2a54a629b1694d0f 100644
--- a/test/CodeGen/PowerPC/setcc-logic.ll
+++ b/test/CodeGen/PowerPC/setcc-logic.ll
@@ -478,3 +478,33 @@ define <4 x i1> @and_eq_vec(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c, <4 x i32>
   ret <4 x i1> %and
 }
 
+define i1 @or_icmps_const_1bit_diff(i64 %x) {
+; CHECK-LABEL: or_icmps_const_1bit_diff:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    li 4, -5
+; CHECK-NEXT:    addi 3, 3, -13
+; CHECK-NEXT:    and 3, 3, 4
+; CHECK-NEXT:    cntlzd 3, 3
+; CHECK-NEXT:    rldicl 3, 3, 58, 63
+; CHECK-NEXT:    blr
+  %a = icmp eq i64 %x, 17
+  %b = icmp eq i64 %x, 13
+  %r = or i1 %a, %b
+  ret i1 %r
+}
+
+define i1 @and_icmps_const_1bit_diff(i32 %x) {
+; CHECK-LABEL: and_icmps_const_1bit_diff:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addi 3, 3, -4625
+; CHECK-NEXT:    rlwinm 3, 3, 0, 28, 26
+; CHECK-NEXT:    cntlzw 3, 3
+; CHECK-NEXT:    nor 3, 3, 3
+; CHECK-NEXT:    rlwinm 3, 3, 27, 31, 31
+; CHECK-NEXT:    blr
+  %a = icmp ne i32 %x, 4625
+  %b = icmp ne i32 %x, 4641
+  %r = and i1 %a, %b
+  ret i1 %r
+}
+
diff --git a/test/CodeGen/PowerPC/setcr_bc3.mir b/test/CodeGen/PowerPC/setcr_bc3.mir
new file mode 100644
index 0000000000000000000000000000000000000000..49df90a7224ec9720d17ab179ce71f58274b1b0a
--- /dev/null
+++ b/test/CodeGen/PowerPC/setcr_bc3.mir
@@ -0,0 +1,108 @@
+# RUN: llc -verify-machineinstrs -start-before=ppc-pre-emit-peephole %s -o - | FileCheck %s
+--- |
+  target datalayout = "e-m:e-i64:64-n32:64"
+  target triple = "powerpc64le-unknown-linux-gnu"
+
+  declare signext i32 @callee(i32 signext)
+
+  define signext i32 @func(i32 signext %v) {
+    ret i32 0
+  }
+...
+---
+name:            func
+alignment:       4
+exposesReturnsTwice: false
+legalized:       false
+regBankSelected: false
+selected:        false
+failedISel:      false
+tracksRegLiveness: true
+registers:
+liveins:
+  - { reg: '$x3', virtual-reg: '' }
+frameInfo:
+  isFrameAddressTaken: false
+  isReturnAddressTaken: false
+  hasStackMap:     false
+  hasPatchPoint:   false
+  stackSize:       48
+  offsetAdjustment: 0
+  maxAlignment:    0
+  adjustsStack:    true
+  hasCalls:        true
+  stackProtector:  ''
+  maxCallFrameSize: 32
+  hasOpaqueSPAdjustment: false
+  hasVAStart:      false
+  hasMustTailInVarArgFunc: false
+  localFrameSize:  0
+  savePoint:       ''
+  restorePoint:    ''
+fixedStack:
+  - { id: 0, type: spill-slot, offset: -16, size: 8, alignment: 16, stack-id: 0, 
+      callee-saved-register: '$x30', callee-saved-restored: true, debug-info-variable: '', 
+      debug-info-expression: '', debug-info-location: '' }
+stack:
+constants:
+
+body:             |
+  bb.0:
+    successors: %bb.2(0x30000000), %bb.1(0x50000000)
+    liveins: $x3, $x30
+
+    ; bc should be converted into b
+    ; CHECK-LABEL: func
+    ; CHECK: # %bb.1
+    ; CHECK: creqv
+    ; CHECK-NOT: bc
+    ; CHECK: b .LBB0_3
+    ; CHECK: .LBB0_2
+
+    $x0 = MFLR8 implicit $lr8
+    STD killed $x0, 16, $x1
+    $x1 = STDU $x1, -48, $x1
+    STD killed $x30, 32, $x1 :: (store 8 into %fixed-stack.0, align 16)
+    $x30 = OR8 $x3, $x3
+    BL8_NOP @callee, csr_svr464_altivec, implicit-def dead $lr8, implicit $rm, implicit killed $x3, implicit $x2, implicit-def $r1, implicit-def $x3
+    renamable $cr0 = CMPLWI renamable $r3, 0
+    BCC 76, killed renamable $cr0, %bb.2
+
+  bb.1:
+    successors: %bb.5(0x40000000), %bb.2(0x40000000)
+    liveins: $x3
+
+    renamable $x3 = EXTSW_32_64 killed renamable $r3, implicit $x3
+    BL8_NOP @callee, csr_svr464_altivec, implicit-def dead $lr8, implicit $rm, implicit killed $x3, implicit $x2, implicit-def $r1, implicit-def $x3
+    renamable $cr2un = CRSET
+    $cr2gt = CROR $cr2un, $cr2un
+    $x30 = OR8 killed $x3, $x3
+    BC killed renamable $cr2un, %bb.5
+    B %bb.2
+
+  bb.4:
+    successors: %bb.5(0x80000000)
+    liveins: $x30
+
+    $x3 = LI8 0
+    BL8_NOP @callee, csr_svr464_altivec, implicit-def dead $lr8, implicit $rm, implicit killed $x3, implicit $x2, implicit-def $r1, implicit-def dead $x3
+
+  bb.5:
+    liveins: $x30, $cr0gt
+
+    renamable $x3 = EXTSW_32_64 killed renamable $r30, implicit $x30
+    $x30 = LD 32, $x1 :: (load 8 from %fixed-stack.0, align 16)
+    $x1 = ADDI8 $x1, 48
+    $x0 = LD 16, $x1
+    MTLR8 killed $x0, implicit-def $lr8
+    BLR8 implicit $lr8, implicit $rm, implicit killed $x3
+
+  bb.2:
+    successors: %bb.5(0x40000000), %bb.4(0x40000000)
+    liveins: $x30
+
+    renamable $cr0 = CMPWI renamable $r30, -1
+    BCn killed renamable $cr0gt, %bb.4
+    B %bb.5
+
+...
diff --git a/test/CodeGen/PowerPC/signbit-shift.ll b/test/CodeGen/PowerPC/signbit-shift.ll
index 41d250e924e2f4ade46c8edfa426157b8860db20..d87a24128b9c06941cbc543c692a7fecb81ebca2 100644
--- a/test/CodeGen/PowerPC/signbit-shift.ll
+++ b/test/CodeGen/PowerPC/signbit-shift.ll
@@ -69,9 +69,8 @@ define i32 @sext_ifpos(i32 %x) {
 define i32 @add_sext_ifpos(i32 %x) {
 ; CHECK-LABEL: add_sext_ifpos:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    nor 3, 3, 3
-; CHECK-NEXT:    srawi 3, 3, 31
-; CHECK-NEXT:    addi 3, 3, 42
+; CHECK-NEXT:    srwi 3, 3, 31
+; CHECK-NEXT:    addi 3, 3, 41
 ; CHECK-NEXT:    blr
   %c = icmp sgt i32 %x, -1
   %e = sext i1 %c to i32
diff --git a/test/CodeGen/PowerPC/stack-realign.ll b/test/CodeGen/PowerPC/stack-realign.ll
index d92f93b517fde19baf395311bbdfb371c9d1238a..21de346ab57e844c2d90e7ccfd5910b7fc0dd67c 100644
--- a/test/CodeGen/PowerPC/stack-realign.ll
+++ b/test/CodeGen/PowerPC/stack-realign.ll
@@ -1,7 +1,7 @@
 ; RUN: llc -mtriple=powerpc64-unknown-linux-gnu -mcpu=pwr7 < %s | FileCheck %s
-; RUN: llc -mtriple=powerpc64-unknown-linux-gnu -mcpu=pwr7 -disable-fp-elim < %s | FileCheck -check-prefix=CHECK-FP %s
-; RUN: llc -mtriple=powerpc-unknown-linux-gnu -disable-fp-elim < %s | FileCheck -check-prefix=CHECK-32 %s
-; RUN: llc -mtriple=powerpc-unknown-linux-gnu -disable-fp-elim -relocation-model=pic < %s | FileCheck -check-prefix=CHECK-32-PIC %s
+; RUN: llc -mtriple=powerpc64-unknown-linux-gnu -mcpu=pwr7 -frame-pointer=all < %s | FileCheck -check-prefix=CHECK-FP %s
+; RUN: llc -mtriple=powerpc-unknown-linux-gnu -frame-pointer=all < %s | FileCheck -check-prefix=CHECK-32 %s
+; RUN: llc -mtriple=powerpc-unknown-linux-gnu -frame-pointer=all -relocation-model=pic < %s | FileCheck -check-prefix=CHECK-32-PIC %s
 target datalayout = "E-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-f128:128:128-v128:128:128-n32:64"
 target triple = "powerpc64-unknown-linux-gnu"
 
diff --git a/test/CodeGen/PowerPC/store_fptoi.ll b/test/CodeGen/PowerPC/store_fptoi.ll
index bac874a621178c0411fb63003fd86a9fa1de5ae3..0978061377a54022287d730ca3824eb1a139c50a 100644
--- a/test/CodeGen/PowerPC/store_fptoi.ll
+++ b/test/CodeGen/PowerPC/store_fptoi.ll
@@ -296,8 +296,8 @@ entry:
 
 ; CHECK-LABEL: spConv2sdw_x
 ; CHECK: lfs [[LD:[0-9]+]], 0(3)
-; CHECK-NEXT: sldi [[REG:[0-9]+]], 5, 3
-; CHECK-NEXT: xscvdpsxds [[CONV:[0-9]+]], [[LD]]
+; CHECK-DAG:  sldi [[REG:[0-9]+]], 5, 3
+; CHECK-DAG:  xscvdpsxds [[CONV:[0-9]+]], [[LD]]
 ; CHECK-NEXT: stxsdx [[CONV]], 4, [[REG]]
 ; CHECK-NEXT: blr
 
@@ -322,8 +322,8 @@ entry:
 
 ; CHECK-LABEL: spConv2sw_x
 ; CHECK: lfs [[LD:[0-9]+]], 0(3)
-; CHECK-NEXT: sldi [[REG:[0-9]+]], 5, 2
-; CHECK-NEXT: xscvdpsxws [[CONV:[0-9]+]], [[LD]]
+; CHECK-DAG: sldi [[REG:[0-9]+]], 5, 2
+; CHECK-DAG: xscvdpsxws [[CONV:[0-9]+]], [[LD]]
 ; CHECK-NEXT: stfiwx [[CONV]], 4, [[REG]]
 ; CHECK-NEXT: blr
 
@@ -348,8 +348,8 @@ entry:
 
 ; CHECK-LABEL: spConv2shw_x
 ; CHECK: lfs [[LD:[0-9]+]], 0(3)
-; CHECK: sldi [[REG:[0-9]+]], 5, 1
-; CHECK: xscvdpsxws [[CONV:[0-9]+]], [[LD]]
+; CHECK-DAG: sldi [[REG:[0-9]+]], 5, 1
+; CHECK-DAG: xscvdpsxws [[CONV:[0-9]+]], [[LD]]
 ; CHECK-NEXT: stxsihx [[CONV]], 4, [[REG]]
 ; CHECK-NEXT: blr
 
@@ -680,8 +680,8 @@ entry:
 
 ; CHECK-LABEL: spConv2udw_x
 ; CHECK: lfs [[LD:[0-9]+]], 0(3)
-; CHECK-NEXT: sldi [[REG:[0-9]+]], 5, 3
-; CHECK-NEXT: xscvdpuxds [[CONV:[0-9]+]], [[LD]]
+; CHECK-DAG: sldi [[REG:[0-9]+]], 5, 3
+; CHECK-DAG: xscvdpuxds [[CONV:[0-9]+]], [[LD]]
 ; CHECK-NEXT: stxsdx [[CONV]], 4, [[REG]]
 ; CHECK-NEXT: blr
 
@@ -706,8 +706,8 @@ entry:
 
 ; CHECK-LABEL: spConv2uw_x
 ; CHECK: lfs [[LD:[0-9]+]], 0(3)
-; CHECK-NEXT: sldi [[REG:[0-9]+]], 5, 2
-; CHECK-NEXT: xscvdpuxws [[CONV:[0-9]+]], [[LD]]
+; CHECK-DAG: sldi [[REG:[0-9]+]], 5, 2
+; CHECK-DAG: xscvdpuxws [[CONV:[0-9]+]], [[LD]]
 ; CHECK-NEXT: stfiwx [[CONV]], 4, [[REG]]
 ; CHECK-NEXT: blr
 
@@ -732,8 +732,8 @@ entry:
 
 ; CHECK-LABEL: spConv2uhw_x
 ; CHECK: lfs [[LD:[0-9]+]], 0(3)
-; CHECK: sldi [[REG:[0-9]+]], 5, 1
-; CHECK: xscvdpuxws [[CONV:[0-9]+]], [[LD]]
+; CHECK-DAG: sldi [[REG:[0-9]+]], 5, 1
+; CHECK-DAG: xscvdpuxws [[CONV:[0-9]+]], [[LD]]
 ; CHECK-NEXT: stxsihx [[CONV]], 4, [[REG]]
 ; CHECK-NEXT: blr
 
diff --git a/test/CodeGen/PowerPC/structsinmem.ll b/test/CodeGen/PowerPC/structsinmem.ll
index bbe82894a67aa49a52c0f71cc51c99733c0ef91d..b994aae6eea9a42893f3cca4b78f54e558684fc1 100644
--- a/test/CodeGen/PowerPC/structsinmem.ll
+++ b/test/CodeGen/PowerPC/structsinmem.ll
@@ -1,4 +1,4 @@
-; RUN: llc -verify-machineinstrs -mcpu=ppc64 -O0 -disable-fp-elim -fast-isel=false < %s | FileCheck %s
+; RUN: llc -verify-machineinstrs -mcpu=ppc64 -O0 -frame-pointer=all -fast-isel=false < %s | FileCheck %s
 
 target datalayout = "E-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v128:128:128-n32:64"
 target triple = "powerpc64-unknown-linux-gnu"
diff --git a/test/CodeGen/PowerPC/structsinregs.ll b/test/CodeGen/PowerPC/structsinregs.ll
index 52976ca784533a1b9e1880de6ffef3e5d93e3eca..08f8ba90fd1e75e9f7558d4bdfb08f3fd04b7d53 100644
--- a/test/CodeGen/PowerPC/structsinregs.ll
+++ b/test/CodeGen/PowerPC/structsinregs.ll
@@ -1,4 +1,4 @@
-; RUN: llc -verify-machineinstrs -mcpu=ppc64 -O0 -disable-fp-elim -fast-isel=false < %s | FileCheck %s
+; RUN: llc -verify-machineinstrs -mcpu=ppc64 -O0 -frame-pointer=all -fast-isel=false < %s | FileCheck %s
 
 target datalayout = "E-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v128:128:128-n32:64"
 target triple = "powerpc64-unknown-linux-gnu"
diff --git a/test/CodeGen/PowerPC/stwu-sched.ll b/test/CodeGen/PowerPC/stwu-sched.ll
index a8a8896570682a34fba6eb670576df7cc1ab994a..1c5cf0b1739580b03173488c18dcae1c9300c14a 100644
--- a/test/CodeGen/PowerPC/stwu-sched.ll
+++ b/test/CodeGen/PowerPC/stwu-sched.ll
@@ -11,8 +11,8 @@
 ; Function Attrs: norecurse nounwind writeonly
 define void @initCombList(%0* nocapture, i32 signext) local_unnamed_addr #0 {
 ; CHECK-LABEL: initCombList:
-; CHECK:   addi 3, 3, -8
-; CHECK-NEXT: stwu 5, 64(4)
+; CHECK: addi 4, 4, -8
+; CHECK: stwu 5, 64(3)
 
 ; CHECK-ITIN-LABEL: initCombList:
 ; CHECK-ITIN: stwu 5, 64(4)
diff --git a/test/CodeGen/PowerPC/swaps-le-6.ll b/test/CodeGen/PowerPC/swaps-le-6.ll
index 0811287a792a2ecd3ad3fd8c962f43da481e549c..a7467219d7388f6a36f41949b91cfc310dcef866 100644
--- a/test/CodeGen/PowerPC/swaps-le-6.ll
+++ b/test/CodeGen/PowerPC/swaps-le-6.ll
@@ -9,7 +9,7 @@
 
 ; RUN: llc -relocation-model=pic -mcpu=pwr9 -mtriple=powerpc64le-unknown-linux-gnu -O3 \
 ; RUN:   -ppc-vsr-nums-as-vr -ppc-asm-full-reg-names -verify-machineinstrs \
-; RUN:   -mattr=-power9-vector < %s | FileCheck %s
+; RUN:   -mattr=-power9-vector < %s | FileCheck %s --check-prefix=CHECK-P9-NOVECTOR
 
 ; These tests verify that VSX swap optimization works when loading a scalar
 ; into a vector register.
@@ -31,18 +31,29 @@ define void @bar0() {
 ; CHECK:     stxvd2x vs0, 0, r3
 ; CHECK:     blr
 ;
+; CHECK-P9-NOVECTOR-LABEL: bar0:
+; CHECK-P9-NOVECTOR:   # %bb.0: # %entry
+; CHECK-P9-NOVECTOR:     addis r3, r2, .LC0@toc@ha
+; CHECK-P9-NOVECTOR:     ld r3, .LC0@toc@l(r3)
+; CHECK-P9-NOVECTOR:     addis r3, r2, .LC1@toc@ha
+; CHECK-P9-NOVECTOR:     addis r3, r2, .LC2@toc@ha
+; CHECK-P9-NOVECTOR:     ld r3, .LC2@toc@l(r3)
+; CHECK-P9-NOVECTOR:     xxpermdi vs0, vs1, vs0, 1
+; CHECK-P9-NOVECTOR:     stxvd2x vs0, 0, r3
+; CHECK-P9-NOVECTOR:     blr
+;
 ; CHECK-P9-LABEL: bar0:
 ; CHECK-P9:   # %bb.0: # %entry
 ; CHECK-P9:     addis r3, r2, .LC0@toc@ha
-; CHECK-P9:     addis r4, r2, .LC1@toc@ha
 ; CHECK-P9:     ld r3, .LC0@toc@l(r3)
-; CHECK-P9:     ld r4, .LC1@toc@l(r4)
-; CHECK-P9:     lfd f0, 0(r3)
-; CHECK-P9:     lxvx vs1, 0, r4
+; CHECK-P9:     lxvx vs0, 0, r3
+; CHECK-P9:     addis r3, r2, .LC1@toc@ha
+; CHECK-P9:     ld r3, .LC1@toc@l(r3)
+; CHECK-P9:     lfd f1, 0(r3)
 ; CHECK-P9:     addis r3, r2, .LC2@toc@ha
 ; CHECK-P9:     ld r3, .LC2@toc@l(r3)
-; CHECK-P9:     xxpermdi vs0, f0, f0, 2
-; CHECK-P9:     xxpermdi vs0, vs1, vs0, 1
+; CHECK-P9:     xxpermdi vs1, f1, f1, 2
+; CHECK-P9:     xxpermdi vs0, vs0, vs1, 1
 ; CHECK-P9:     stxvx vs0, 0, r3
 ; CHECK-P9:     blr
 entry:
@@ -65,18 +76,29 @@ define void @bar1() {
 ; CHECK:     stxvd2x vs0, 0, r3
 ; CHECK:     blr
 ;
+; CHECK-P9-NOVECTOR-LABEL: bar1:
+; CHECK-P9-NOVECTOR:   # %bb.0: # %entry
+; CHECK-P9-NOVECTOR:     addis r3, r2, .LC0@toc@ha
+; CHECK-P9-NOVECTOR:     ld r3, .LC0@toc@l(r3)
+; CHECK-P9-NOVECTOR:     addis r3, r2, .LC1@toc@ha
+; CHECK-P9-NOVECTOR:     addis r3, r2, .LC2@toc@ha
+; CHECK-P9-NOVECTOR:     ld r3, .LC2@toc@l(r3)
+; CHECK-P9-NOVECTOR:     xxmrghd vs0, vs0, vs1
+; CHECK-P9-NOVECTOR:     stxvd2x vs0, 0, r3
+; CHECK-P9-NOVECTOR:     blr
+;
 ; CHECK-P9-LABEL: bar1:
 ; CHECK-P9:   # %bb.0: # %entry
 ; CHECK-P9:     addis r3, r2, .LC0@toc@ha
-; CHECK-P9:     addis r4, r2, .LC1@toc@ha
 ; CHECK-P9:     ld r3, .LC0@toc@l(r3)
-; CHECK-P9:     ld r4, .LC1@toc@l(r4)
-; CHECK-P9:     lfd f0, 0(r3)
-; CHECK-P9:     lxvx vs1, 0, r4
+; CHECK-P9:     lxvx vs0, 0, r3
+; CHECK-P9:     addis r3, r2, .LC1@toc@ha
+; CHECK-P9:     ld r3, .LC1@toc@l(r3)
+; CHECK-P9:     lfd f1, 0(r3)
 ; CHECK-P9:     addis r3, r2, .LC2@toc@ha
 ; CHECK-P9:     ld r3, .LC2@toc@l(r3)
-; CHECK-P9:     xxpermdi vs0, f0, f0, 2
-; CHECK-P9:     xxmrgld vs0, vs0, vs1
+; CHECK-P9:     xxpermdi vs1, f1, f1, 2
+; CHECK-P9:     xxmrgld vs0, vs1, vs0
 ; CHECK-P9:     stxvx vs0, 0, r3
 ; CHECK-P9:     blr
 entry:
diff --git a/test/CodeGen/PowerPC/vec-itofp.ll b/test/CodeGen/PowerPC/vec-itofp.ll
index 852b7c822ad8fcd63898847463f85b4243836de3..d9528e235743b11e5e554c47f03b572b32f0494a 100644
--- a/test/CodeGen/PowerPC/vec-itofp.ll
+++ b/test/CodeGen/PowerPC/vec-itofp.ll
@@ -16,12 +16,12 @@ entry:
   ret void
 ; CHECK-P9-LABEL: @test8
 ; CHECK-P9: vperm
-; CHECK-P9: vperm
-; CHECK-P9: vperm
-; CHECK-P9: vperm
 ; CHECK-P9: xvcvuxddp
+; CHECK-P9: vperm
 ; CHECK-P9: xvcvuxddp
+; CHECK-P9: vperm
 ; CHECK-P9: xvcvuxddp
+; CHECK-P9: vperm
 ; CHECK-P9: xvcvuxddp
 ; CHECK-P8-LABEL: @test8
 ; CHECK-P8: vperm
@@ -42,8 +42,8 @@ entry:
   ret void
 ; CHECK-P9-LABEL: @test4
 ; CHECK-P9: vperm
-; CHECK-P9: vperm
 ; CHECK-P9: xvcvuxddp
+; CHECK-P9: vperm
 ; CHECK-P9: xvcvuxddp
 ; CHECK-P8-LABEL: @test4
 ; CHECK-P8: vperm
@@ -113,16 +113,16 @@ entry:
   ret void
 ; CHECK-P9-LABEL: @stest8
 ; CHECK-P9: vperm
-; CHECK-P9: vperm
-; CHECK-P9: vperm
-; CHECK-P9: vperm
-; CHECK-P9: vextsh2d
-; CHECK-P9: vextsh2d
-; CHECK-P9: vextsh2d
 ; CHECK-P9: vextsh2d
 ; CHECK-P9: xvcvsxddp
+; CHECK-P9: vperm
+; CHECK-P9: vextsh2d
 ; CHECK-P9: xvcvsxddp
+; CHECK-P9: vperm
+; CHECK-P9: vextsh2d
 ; CHECK-P9: xvcvsxddp
+; CHECK-P9: vperm
+; CHECK-P9: vextsh2d
 ; CHECK-P9: xvcvsxddp
 }
 
@@ -134,10 +134,10 @@ entry:
   ret void
 ; CHECK-P9-LABEL: @stest4
 ; CHECK-P9: vperm
-; CHECK-P9: vperm
-; CHECK-P9: vextsh2d
 ; CHECK-P9: vextsh2d
 ; CHECK-P9: xvcvsxddp
+; CHECK-P9: vperm
+; CHECK-P9: vextsh2d
 ; CHECK-P9: xvcvsxddp
 }
 
diff --git a/test/CodeGen/PowerPC/vec-trunc.ll b/test/CodeGen/PowerPC/vec-trunc.ll
new file mode 100644
index 0000000000000000000000000000000000000000..6df696a38e91027af46e2c3ca18093962540a837
--- /dev/null
+++ b/test/CodeGen/PowerPC/vec-trunc.ll
@@ -0,0 +1,197 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu \
+; RUN:     -mattr=+vsx -ppc-asm-full-reg-names -ppc-vsr-nums-as-vr < %s | \
+; RUN: FileCheck %s
+; RUN: llc -verify-machineinstrs -mtriple=powerpc64-unknown-linux-gnu \
+; RUN:     -mattr=+vsx -ppc-asm-full-reg-names -ppc-vsr-nums-as-vr < %s | \
+; RUN: FileCheck %s --check-prefix=CHECK-BE
+
+define void @test8i8(<8 x i8>* nocapture %Sink, <8 x i16>* nocapture readonly %SrcPtr) {
+; CHECK-LABEL: test8i8:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lvx v2, 0, r4
+; CHECK-NEXT:    vpkuhum v2, v2, v2
+; CHECK-NEXT:    xxswapd vs0, v2
+; CHECK-NEXT:    stfdx f0, 0, r3
+; CHECK-NEXT:    blr
+;
+; CHECK-BE-LABEL: test8i8:
+; CHECK-BE:       # %bb.0: # %entry
+; CHECK-BE-NEXT:    lxvw4x v2, 0, r4
+; CHECK-BE-NEXT:    addi r5, r1, -16
+; CHECK-BE-NEXT:    vpkuhum v2, v2, v2
+; CHECK-BE-NEXT:    stxvd2x v2, 0, r5
+; CHECK-BE-NEXT:    ld r4, -16(r1)
+; CHECK-BE-NEXT:    std r4, 0(r3)
+; CHECK-BE-NEXT:    blr
+entry:
+  %0 = load <8 x i16>, <8 x i16>* %SrcPtr, align 16
+  %1 = trunc <8 x i16> %0 to <8 x i8>
+  store <8 x i8> %1, <8 x i8>* %Sink, align 16
+  ret void
+}
+
+define void @test4i8(<4 x i8>* nocapture %Sink, <4 x i16>* nocapture readonly %SrcPtr) {
+; CHECK-LABEL: test4i8:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lvx v2, 0, r4
+; CHECK-NEXT:    vpkuhum v2, v2, v2
+; CHECK-NEXT:    xxsldwi vs0, v2, v2, 2
+; CHECK-NEXT:    stfiwx f0, 0, r3
+; CHECK-NEXT:    blr
+;
+; CHECK-BE-LABEL: test4i8:
+; CHECK-BE:       # %bb.0: # %entry
+; CHECK-BE-NEXT:    lxvw4x v2, 0, r4
+; CHECK-BE-NEXT:    addi r5, r1, -16
+; CHECK-BE-NEXT:    vpkuhum v2, v2, v2
+; CHECK-BE-NEXT:    stxvw4x v2, 0, r5
+; CHECK-BE-NEXT:    lwz r4, -16(r1)
+; CHECK-BE-NEXT:    stw r4, 0(r3)
+; CHECK-BE-NEXT:    blr
+entry:
+  %0 = load <4 x i16>, <4 x i16>* %SrcPtr, align 16
+  %1 = trunc <4 x i16> %0 to <4 x i8>
+  store <4 x i8> %1, <4 x i8>* %Sink, align 16
+  ret void
+}
+
+define void @test4i8w(<4 x i8>* nocapture %Sink, <4 x i32>* nocapture readonly %SrcPtr) {
+; CHECK-LABEL: test4i8w:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    addis r5, r2, .LCPI2_0@toc@ha
+; CHECK-NEXT:    lvx v3, 0, r4
+; CHECK-NEXT:    addi r5, r5, .LCPI2_0@toc@l
+; CHECK-NEXT:    lvx v2, 0, r5
+; CHECK-NEXT:    vperm v2, v3, v3, v2
+; CHECK-NEXT:    xxsldwi vs0, v2, v2, 2
+; CHECK-NEXT:    stfiwx f0, 0, r3
+; CHECK-NEXT:    blr
+;
+; CHECK-BE-LABEL: test4i8w:
+; CHECK-BE:       # %bb.0: # %entry
+; CHECK-BE-NEXT:    addis r5, r2, .LCPI2_0@toc@ha
+; CHECK-BE-NEXT:    lxvw4x v2, 0, r4
+; CHECK-BE-NEXT:    addi r4, r5, .LCPI2_0@toc@l
+; CHECK-BE-NEXT:    lxvw4x v3, 0, r4
+; CHECK-BE-NEXT:    addi r4, r1, -16
+; CHECK-BE-NEXT:    vperm v2, v2, v2, v3
+; CHECK-BE-NEXT:    stxvw4x v2, 0, r4
+; CHECK-BE-NEXT:    lwz r4, -16(r1)
+; CHECK-BE-NEXT:    stw r4, 0(r3)
+; CHECK-BE-NEXT:    blr
+entry:
+  %0 = load <4 x i32>, <4 x i32>* %SrcPtr, align 16
+  %1 = trunc <4 x i32> %0 to <4 x i8>
+  store <4 x i8> %1, <4 x i8>* %Sink, align 16
+  ret void
+}
+
+define void @test2i8(<2 x i8>* nocapture %Sink, <2 x i16>* nocapture readonly %SrcPtr) {
+; CHECK-LABEL: test2i8:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lvx v2, 0, r4
+; CHECK-NEXT:    vpkuhum v2, v2, v2
+; CHECK-NEXT:    xxswapd vs0, v2
+; CHECK-NEXT:    mfvsrd r4, f0
+; CHECK-NEXT:    clrldi r4, r4, 48
+; CHECK-NEXT:    sth r4, 0(r3)
+; CHECK-NEXT:    blr
+;
+; CHECK-BE-LABEL: test2i8:
+; CHECK-BE:       # %bb.0: # %entry
+; CHECK-BE-NEXT:    lxvw4x v2, 0, r4
+; CHECK-BE-NEXT:    addi r5, r1, -16
+; CHECK-BE-NEXT:    vpkuhum v2, v2, v2
+; CHECK-BE-NEXT:    stxvw4x v2, 0, r5
+; CHECK-BE-NEXT:    lhz r4, -16(r1)
+; CHECK-BE-NEXT:    sth r4, 0(r3)
+; CHECK-BE-NEXT:    blr
+entry:
+  %0 = load <2 x i16>, <2 x i16>* %SrcPtr, align 16
+  %1 = trunc <2 x i16> %0 to <2 x i8>
+  store <2 x i8> %1, <2 x i8>* %Sink, align 16
+  ret void
+}
+
+define void @test4i16(<4 x i16>* nocapture %Sink, <4 x i32>* nocapture readonly %SrcPtr) {
+; CHECK-LABEL: test4i16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lvx v2, 0, r4
+; CHECK-NEXT:    vpkuwum v2, v2, v2
+; CHECK-NEXT:    xxswapd vs0, v2
+; CHECK-NEXT:    stfdx f0, 0, r3
+; CHECK-NEXT:    blr
+;
+; CHECK-BE-LABEL: test4i16:
+; CHECK-BE:       # %bb.0: # %entry
+; CHECK-BE-NEXT:    lxvw4x v2, 0, r4
+; CHECK-BE-NEXT:    addi r5, r1, -16
+; CHECK-BE-NEXT:    vpkuwum v2, v2, v2
+; CHECK-BE-NEXT:    stxvd2x v2, 0, r5
+; CHECK-BE-NEXT:    ld r4, -16(r1)
+; CHECK-BE-NEXT:    std r4, 0(r3)
+; CHECK-BE-NEXT:    blr
+entry:
+  %0 = load <4 x i32>, <4 x i32>* %SrcPtr, align 16
+  %1 = trunc <4 x i32> %0 to <4 x i16>
+  store <4 x i16> %1, <4 x i16>* %Sink, align 16
+  ret void
+}
+
+define void @test2i16(<2 x i16>* nocapture %Sink, <2 x i32>* nocapture readonly %SrcPtr) {
+; CHECK-LABEL: test2i16:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lvx v2, 0, r4
+; CHECK-NEXT:    vpkuwum v2, v2, v2
+; CHECK-NEXT:    xxsldwi vs0, v2, v2, 2
+; CHECK-NEXT:    stfiwx f0, 0, r3
+; CHECK-NEXT:    blr
+;
+; CHECK-BE-LABEL: test2i16:
+; CHECK-BE:       # %bb.0: # %entry
+; CHECK-BE-NEXT:    lxvw4x v2, 0, r4
+; CHECK-BE-NEXT:    addi r5, r1, -16
+; CHECK-BE-NEXT:    vpkuwum v2, v2, v2
+; CHECK-BE-NEXT:    stxvw4x v2, 0, r5
+; CHECK-BE-NEXT:    lwz r4, -16(r1)
+; CHECK-BE-NEXT:    stw r4, 0(r3)
+; CHECK-BE-NEXT:    blr
+entry:
+  %0 = load <2 x i32>, <2 x i32>* %SrcPtr, align 16
+  %1 = trunc <2 x i32> %0 to <2 x i16>
+  store <2 x i16> %1, <2 x i16>* %Sink, align 16
+  ret void
+}
+
+define void @test2i16d(<2 x i16>* nocapture %Sink, <2 x i64>* nocapture readonly %SrcPtr) {
+; CHECK-LABEL: test2i16d:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lxvd2x vs0, 0, r4
+; CHECK-NEXT:    addis r5, r2, .LCPI6_0@toc@ha
+; CHECK-NEXT:    addi r4, r5, .LCPI6_0@toc@l
+; CHECK-NEXT:    lvx v3, 0, r4
+; CHECK-NEXT:    xxswapd v2, vs0
+; CHECK-NEXT:    vperm v2, v2, v2, v3
+; CHECK-NEXT:    xxsldwi vs0, v2, v2, 2
+; CHECK-NEXT:    stfiwx f0, 0, r3
+; CHECK-NEXT:    blr
+;
+; CHECK-BE-LABEL: test2i16d:
+; CHECK-BE:       # %bb.0: # %entry
+; CHECK-BE-NEXT:    addis r5, r2, .LCPI6_0@toc@ha
+; CHECK-BE-NEXT:    lxvw4x v2, 0, r4
+; CHECK-BE-NEXT:    addi r4, r5, .LCPI6_0@toc@l
+; CHECK-BE-NEXT:    lxvw4x v3, 0, r4
+; CHECK-BE-NEXT:    addi r4, r1, -16
+; CHECK-BE-NEXT:    vperm v2, v2, v2, v3
+; CHECK-BE-NEXT:    stxvw4x v2, 0, r4
+; CHECK-BE-NEXT:    lwz r4, -16(r1)
+; CHECK-BE-NEXT:    stw r4, 0(r3)
+; CHECK-BE-NEXT:    blr
+entry:
+  %0 = load <2 x i64>, <2 x i64>* %SrcPtr, align 16
+  %1 = trunc <2 x i64> %0 to <2 x i16>
+  store <2 x i16> %1, <2 x i16>* %Sink, align 16
+  ret void
+}
diff --git a/test/CodeGen/PowerPC/vec_conv_fp32_to_i16_elts.ll b/test/CodeGen/PowerPC/vec_conv_fp32_to_i16_elts.ll
index 9977410f9979366eac61b476cbac900d0b52010f..631e3c4fd43081ef40dbbef040e3a73d62e3aea9 100644
--- a/test/CodeGen/PowerPC/vec_conv_fp32_to_i16_elts.ll
+++ b/test/CodeGen/PowerPC/vec_conv_fp32_to_i16_elts.ll
@@ -36,35 +36,35 @@ define i32 @test2elt(i64 %a.coerce) local_unnamed_addr #0 {
 ; CHECK-P9-NEXT:    xxswapd v2, vs0
 ; CHECK-P9-NEXT:    xscvspdpn f0, vs0
 ; CHECK-P9-NEXT:    xxsldwi vs1, v2, v2, 3
-; CHECK-P9-NEXT:    xscvdpsxws f0, f0
 ; CHECK-P9-NEXT:    xscvspdpn f1, vs1
 ; CHECK-P9-NEXT:    xscvdpsxws f1, f1
-; CHECK-P9-NEXT:    mfvsrwz r4, f0
+; CHECK-P9-NEXT:    xscvdpsxws f0, f0
 ; CHECK-P9-NEXT:    mfvsrwz r3, f1
-; CHECK-P9-NEXT:    mtvsrd f1, r4
+; CHECK-P9-NEXT:    mtvsrd f1, r3
+; CHECK-P9-NEXT:    mfvsrwz r3, f0
 ; CHECK-P9-NEXT:    mtvsrd f0, r3
-; CHECK-P9-NEXT:    xxswapd v3, vs1
-; CHECK-P9-NEXT:    li r3, 0
-; CHECK-P9-NEXT:    xxswapd v2, vs0
+; CHECK-P9-NEXT:    xxswapd v2, vs1
+; CHECK-P9-NEXT:    xxswapd v3, vs0
 ; CHECK-P9-NEXT:    vmrglh v2, v3, v2
+; CHECK-P9-NEXT:    li r3, 0
 ; CHECK-P9-NEXT:    vextuwrx r3, r3, v2
 ; CHECK-P9-NEXT:    blr
 ;
 ; CHECK-BE-LABEL: test2elt:
 ; CHECK-BE:       # %bb.0: # %entry
 ; CHECK-BE-NEXT:    mtvsrd f0, r3
-; CHECK-BE-NEXT:    xxsldwi vs1, vs0, vs0, 1
+; CHECK-BE-NEXT:    xscvspdpn f1, vs0
+; CHECK-BE-NEXT:    xxsldwi vs0, vs0, vs0, 1
+; CHECK-BE-NEXT:    xscvdpsxws f1, f1
 ; CHECK-BE-NEXT:    xscvspdpn f0, vs0
-; CHECK-BE-NEXT:    xscvspdpn f1, vs1
 ; CHECK-BE-NEXT:    xscvdpsxws f0, f0
-; CHECK-BE-NEXT:    xscvdpsxws f1, f1
-; CHECK-BE-NEXT:    mfvsrwz r3, f0
+; CHECK-BE-NEXT:    mfvsrwz r3, f1
 ; CHECK-BE-NEXT:    sldi r3, r3, 48
-; CHECK-BE-NEXT:    mfvsrwz r4, f1
 ; CHECK-BE-NEXT:    mtvsrd v2, r3
+; CHECK-BE-NEXT:    mfvsrwz r3, f0
+; CHECK-BE-NEXT:    sldi r3, r3, 48
+; CHECK-BE-NEXT:    mtvsrd v3, r3
 ; CHECK-BE-NEXT:    li r3, 0
-; CHECK-BE-NEXT:    sldi r4, r4, 48
-; CHECK-BE-NEXT:    mtvsrd v3, r4
 ; CHECK-BE-NEXT:    vmrghh v2, v2, v3
 ; CHECK-BE-NEXT:    vextuwlx r3, r3, v2
 ; CHECK-BE-NEXT:    blr
@@ -111,62 +111,62 @@ define i64 @test4elt(<4 x float> %a) local_unnamed_addr #1 {
 ; CHECK-P9-LABEL: test4elt:
 ; CHECK-P9:       # %bb.0: # %entry
 ; CHECK-P9-NEXT:    xxsldwi vs0, v2, v2, 3
-; CHECK-P9-NEXT:    xxswapd vs1, v2
-; CHECK-P9-NEXT:    xxsldwi vs2, v2, v2, 1
-; CHECK-P9-NEXT:    xscvspdpn f3, v2
 ; CHECK-P9-NEXT:    xscvspdpn f0, vs0
-; CHECK-P9-NEXT:    xscvspdpn f1, vs1
-; CHECK-P9-NEXT:    xscvspdpn f2, vs2
-; CHECK-P9-NEXT:    xscvdpsxws f3, f3
 ; CHECK-P9-NEXT:    xscvdpsxws f0, f0
-; CHECK-P9-NEXT:    xscvdpsxws f1, f1
-; CHECK-P9-NEXT:    xscvdpsxws f2, f2
-; CHECK-P9-NEXT:    mfvsrwz r5, f3
 ; CHECK-P9-NEXT:    mfvsrwz r3, f0
-; CHECK-P9-NEXT:    mfvsrwz r4, f1
-; CHECK-P9-NEXT:    mfvsrwz r6, f2
-; CHECK-P9-NEXT:    mtvsrd f2, r5
 ; CHECK-P9-NEXT:    mtvsrd f0, r3
-; CHECK-P9-NEXT:    mtvsrd f1, r4
-; CHECK-P9-NEXT:    mtvsrd f3, r6
-; CHECK-P9-NEXT:    xxswapd v4, vs2
+; CHECK-P9-NEXT:    xxswapd v3, vs0
+; CHECK-P9-NEXT:    xxswapd vs0, v2
+; CHECK-P9-NEXT:    xscvspdpn f0, vs0
+; CHECK-P9-NEXT:    xscvdpsxws f0, f0
+; CHECK-P9-NEXT:    mfvsrwz r3, f0
+; CHECK-P9-NEXT:    mtvsrd f0, r3
+; CHECK-P9-NEXT:    xxswapd v4, vs0
+; CHECK-P9-NEXT:    xscvspdpn f0, v2
+; CHECK-P9-NEXT:    xscvdpsxws f0, f0
+; CHECK-P9-NEXT:    mfvsrwz r3, f0
+; CHECK-P9-NEXT:    mtvsrd f0, r3
+; CHECK-P9-NEXT:    vmrglh v3, v4, v3
+; CHECK-P9-NEXT:    xxswapd v4, vs0
+; CHECK-P9-NEXT:    xxsldwi vs0, v2, v2, 1
+; CHECK-P9-NEXT:    xscvspdpn f0, vs0
+; CHECK-P9-NEXT:    xscvdpsxws f0, f0
+; CHECK-P9-NEXT:    mfvsrwz r3, f0
+; CHECK-P9-NEXT:    mtvsrd f0, r3
 ; CHECK-P9-NEXT:    xxswapd v2, vs0
-; CHECK-P9-NEXT:    xxswapd v3, vs1
-; CHECK-P9-NEXT:    xxswapd v5, vs3
-; CHECK-P9-NEXT:    vmrglh v2, v3, v2
-; CHECK-P9-NEXT:    vmrglh v3, v4, v5
-; CHECK-P9-NEXT:    vmrglw v2, v3, v2
+; CHECK-P9-NEXT:    vmrglh v2, v4, v2
+; CHECK-P9-NEXT:    vmrglw v2, v2, v3
 ; CHECK-P9-NEXT:    mfvsrld r3, v2
 ; CHECK-P9-NEXT:    blr
 ;
 ; CHECK-BE-LABEL: test4elt:
 ; CHECK-BE:       # %bb.0: # %entry
 ; CHECK-BE-NEXT:    xxsldwi vs0, v2, v2, 3
-; CHECK-BE-NEXT:    xxswapd vs1, v2
-; CHECK-BE-NEXT:    xxsldwi vs2, v2, v2, 1
-; CHECK-BE-NEXT:    xscvspdpn f3, v2
 ; CHECK-BE-NEXT:    xscvspdpn f0, vs0
-; CHECK-BE-NEXT:    xscvspdpn f1, vs1
-; CHECK-BE-NEXT:    xscvspdpn f2, vs2
-; CHECK-BE-NEXT:    xscvdpsxws f3, f3
 ; CHECK-BE-NEXT:    xscvdpsxws f0, f0
-; CHECK-BE-NEXT:    xscvdpsxws f1, f1
-; CHECK-BE-NEXT:    xscvdpsxws f2, f2
-; CHECK-BE-NEXT:    mfvsrwz r5, f3
-; CHECK-BE-NEXT:    sldi r5, r5, 48
 ; CHECK-BE-NEXT:    mfvsrwz r3, f0
-; CHECK-BE-NEXT:    mfvsrwz r4, f1
-; CHECK-BE-NEXT:    mfvsrwz r6, f2
-; CHECK-BE-NEXT:    mtvsrd v4, r5
+; CHECK-BE-NEXT:    xxswapd vs0, v2
+; CHECK-BE-NEXT:    sldi r3, r3, 48
+; CHECK-BE-NEXT:    xscvspdpn f0, vs0
+; CHECK-BE-NEXT:    mtvsrd v3, r3
+; CHECK-BE-NEXT:    xscvdpsxws f0, f0
+; CHECK-BE-NEXT:    mfvsrwz r3, f0
+; CHECK-BE-NEXT:    xscvspdpn f0, v2
+; CHECK-BE-NEXT:    sldi r3, r3, 48
+; CHECK-BE-NEXT:    xscvdpsxws f0, f0
+; CHECK-BE-NEXT:    mtvsrd v4, r3
+; CHECK-BE-NEXT:    vmrghh v3, v4, v3
+; CHECK-BE-NEXT:    mfvsrwz r3, f0
+; CHECK-BE-NEXT:    xxsldwi vs0, v2, v2, 1
+; CHECK-BE-NEXT:    sldi r3, r3, 48
+; CHECK-BE-NEXT:    xscvspdpn f0, vs0
+; CHECK-BE-NEXT:    mtvsrd v4, r3
+; CHECK-BE-NEXT:    xscvdpsxws f0, f0
+; CHECK-BE-NEXT:    mfvsrwz r3, f0
 ; CHECK-BE-NEXT:    sldi r3, r3, 48
-; CHECK-BE-NEXT:    sldi r4, r4, 48
-; CHECK-BE-NEXT:    sldi r6, r6, 48
 ; CHECK-BE-NEXT:    mtvsrd v2, r3
-; CHECK-BE-NEXT:    mtvsrd v3, r4
-; CHECK-BE-NEXT:    mtvsrd v5, r6
-; CHECK-BE-NEXT:    vmrghh v2, v3, v2
-; CHECK-BE-NEXT:    vmrghh v3, v4, v5
-; CHECK-BE-NEXT:    vmrghw v2, v3, v2
+; CHECK-BE-NEXT:    vmrghh v2, v4, v2
+; CHECK-BE-NEXT:    vmrghw v2, v2, v3
 ; CHECK-BE-NEXT:    mfvsrd r3, v2
 ; CHECK-BE-NEXT:    blr
 entry:
@@ -238,119 +238,119 @@ define <8 x i16> @test8elt(<8 x float>* nocapture readonly) local_unnamed_addr #
 ;
 ; CHECK-P9-LABEL: test8elt:
 ; CHECK-P9:       # %bb.0: # %entry
-; CHECK-P9-NEXT:    lxv vs0, 16(r3)
 ; CHECK-P9-NEXT:    lxv vs1, 0(r3)
 ; CHECK-P9-NEXT:    xxsldwi vs2, vs1, vs1, 3
-; CHECK-P9-NEXT:    xxswapd vs3, vs1
-; CHECK-P9-NEXT:    xxsldwi vs4, vs1, vs1, 1
-; CHECK-P9-NEXT:    xxsldwi vs5, vs0, vs0, 3
-; CHECK-P9-NEXT:    xxswapd vs6, vs0
-; CHECK-P9-NEXT:    xxsldwi vs7, vs0, vs0, 1
-; CHECK-P9-NEXT:    xscvspdpn f1, vs1
-; CHECK-P9-NEXT:    xscvspdpn f0, vs0
 ; CHECK-P9-NEXT:    xscvspdpn f2, vs2
-; CHECK-P9-NEXT:    xscvspdpn f3, vs3
-; CHECK-P9-NEXT:    xscvspdpn f4, vs4
-; CHECK-P9-NEXT:    xscvspdpn f5, vs5
-; CHECK-P9-NEXT:    xscvspdpn f6, vs6
-; CHECK-P9-NEXT:    xscvspdpn f7, vs7
-; CHECK-P9-NEXT:    xscvdpsxws f1, f1
-; CHECK-P9-NEXT:    xscvdpsxws f0, f0
 ; CHECK-P9-NEXT:    xscvdpsxws f2, f2
-; CHECK-P9-NEXT:    xscvdpsxws f3, f3
-; CHECK-P9-NEXT:    xscvdpsxws f4, f4
-; CHECK-P9-NEXT:    xscvdpsxws f5, f5
-; CHECK-P9-NEXT:    xscvdpsxws f6, f6
-; CHECK-P9-NEXT:    xscvdpsxws f7, f7
-; CHECK-P9-NEXT:    mfvsrwz r5, f1
-; CHECK-P9-NEXT:    mfvsrwz r9, f0
+; CHECK-P9-NEXT:    lxv vs0, 16(r3)
 ; CHECK-P9-NEXT:    mfvsrwz r3, f2
-; CHECK-P9-NEXT:    mfvsrwz r4, f3
-; CHECK-P9-NEXT:    mfvsrwz r6, f4
-; CHECK-P9-NEXT:    mfvsrwz r7, f5
-; CHECK-P9-NEXT:    mfvsrwz r8, f6
-; CHECK-P9-NEXT:    mfvsrwz r10, f7
-; CHECK-P9-NEXT:    mtvsrd f2, r5
-; CHECK-P9-NEXT:    mtvsrd f6, r9
-; CHECK-P9-NEXT:    mtvsrd f0, r3
-; CHECK-P9-NEXT:    mtvsrd f1, r4
-; CHECK-P9-NEXT:    mtvsrd f3, r6
-; CHECK-P9-NEXT:    mtvsrd f4, r7
-; CHECK-P9-NEXT:    mtvsrd f5, r8
-; CHECK-P9-NEXT:    mtvsrd f7, r10
-; CHECK-P9-NEXT:    xxswapd v4, vs2
-; CHECK-P9-NEXT:    xxswapd v6, vs6
-; CHECK-P9-NEXT:    xxswapd v2, vs0
-; CHECK-P9-NEXT:    xxswapd v3, vs1
-; CHECK-P9-NEXT:    xxswapd v5, vs3
-; CHECK-P9-NEXT:    xxswapd v0, vs4
-; CHECK-P9-NEXT:    xxswapd v1, vs5
-; CHECK-P9-NEXT:    xxswapd v7, vs7
+; CHECK-P9-NEXT:    mtvsrd f2, r3
+; CHECK-P9-NEXT:    xxswapd v2, vs2
+; CHECK-P9-NEXT:    xxswapd vs2, vs1
+; CHECK-P9-NEXT:    xscvspdpn f2, vs2
+; CHECK-P9-NEXT:    xscvdpsxws f2, f2
+; CHECK-P9-NEXT:    mfvsrwz r3, f2
+; CHECK-P9-NEXT:    mtvsrd f2, r3
+; CHECK-P9-NEXT:    xxswapd v3, vs2
+; CHECK-P9-NEXT:    xscvspdpn f2, vs1
+; CHECK-P9-NEXT:    xxsldwi vs1, vs1, vs1, 1
+; CHECK-P9-NEXT:    xscvspdpn f1, vs1
+; CHECK-P9-NEXT:    xscvdpsxws f2, f2
+; CHECK-P9-NEXT:    xscvdpsxws f1, f1
+; CHECK-P9-NEXT:    mfvsrwz r3, f2
+; CHECK-P9-NEXT:    mtvsrd f2, r3
+; CHECK-P9-NEXT:    mfvsrwz r3, f1
+; CHECK-P9-NEXT:    mtvsrd f1, r3
+; CHECK-P9-NEXT:    xxswapd v4, vs1
+; CHECK-P9-NEXT:    xxsldwi vs1, vs0, vs0, 3
+; CHECK-P9-NEXT:    xscvspdpn f1, vs1
+; CHECK-P9-NEXT:    xscvdpsxws f1, f1
 ; CHECK-P9-NEXT:    vmrglh v2, v3, v2
-; CHECK-P9-NEXT:    vmrglh v3, v4, v5
-; CHECK-P9-NEXT:    vmrglh v4, v1, v0
-; CHECK-P9-NEXT:    vmrglh v5, v6, v7
+; CHECK-P9-NEXT:    xxswapd v3, vs2
+; CHECK-P9-NEXT:    vmrglh v3, v3, v4
 ; CHECK-P9-NEXT:    vmrglw v2, v3, v2
-; CHECK-P9-NEXT:    vmrglw v3, v5, v4
+; CHECK-P9-NEXT:    mfvsrwz r3, f1
+; CHECK-P9-NEXT:    mtvsrd f1, r3
+; CHECK-P9-NEXT:    xxswapd v3, vs1
+; CHECK-P9-NEXT:    xxswapd vs1, vs0
+; CHECK-P9-NEXT:    xscvspdpn f1, vs1
+; CHECK-P9-NEXT:    xscvdpsxws f1, f1
+; CHECK-P9-NEXT:    mfvsrwz r3, f1
+; CHECK-P9-NEXT:    mtvsrd f1, r3
+; CHECK-P9-NEXT:    xxswapd v4, vs1
+; CHECK-P9-NEXT:    xscvspdpn f1, vs0
+; CHECK-P9-NEXT:    xxsldwi vs0, vs0, vs0, 1
+; CHECK-P9-NEXT:    xscvspdpn f0, vs0
+; CHECK-P9-NEXT:    xscvdpsxws f1, f1
+; CHECK-P9-NEXT:    xscvdpsxws f0, f0
+; CHECK-P9-NEXT:    mfvsrwz r3, f1
+; CHECK-P9-NEXT:    mtvsrd f1, r3
+; CHECK-P9-NEXT:    mfvsrwz r3, f0
+; CHECK-P9-NEXT:    mtvsrd f0, r3
+; CHECK-P9-NEXT:    vmrglh v3, v4, v3
+; CHECK-P9-NEXT:    xxswapd v4, vs1
+; CHECK-P9-NEXT:    xxswapd v5, vs0
+; CHECK-P9-NEXT:    vmrglh v4, v4, v5
+; CHECK-P9-NEXT:    vmrglw v3, v4, v3
 ; CHECK-P9-NEXT:    xxmrgld v2, v3, v2
 ; CHECK-P9-NEXT:    blr
 ;
 ; CHECK-BE-LABEL: test8elt:
 ; CHECK-BE:       # %bb.0: # %entry
-; CHECK-BE-NEXT:    lxv vs0, 0(r3)
 ; CHECK-BE-NEXT:    lxv vs1, 16(r3)
 ; CHECK-BE-NEXT:    xxsldwi vs2, vs1, vs1, 3
-; CHECK-BE-NEXT:    xxswapd vs3, vs1
-; CHECK-BE-NEXT:    xxsldwi vs4, vs1, vs1, 1
-; CHECK-BE-NEXT:    xxsldwi vs5, vs0, vs0, 3
-; CHECK-BE-NEXT:    xxswapd vs6, vs0
-; CHECK-BE-NEXT:    xxsldwi vs7, vs0, vs0, 1
-; CHECK-BE-NEXT:    xscvspdpn f1, vs1
-; CHECK-BE-NEXT:    xscvspdpn f0, vs0
 ; CHECK-BE-NEXT:    xscvspdpn f2, vs2
-; CHECK-BE-NEXT:    xscvspdpn f3, vs3
-; CHECK-BE-NEXT:    xscvspdpn f4, vs4
-; CHECK-BE-NEXT:    xscvspdpn f5, vs5
-; CHECK-BE-NEXT:    xscvspdpn f6, vs6
-; CHECK-BE-NEXT:    xscvspdpn f7, vs7
-; CHECK-BE-NEXT:    xscvdpsxws f1, f1
-; CHECK-BE-NEXT:    xscvdpsxws f0, f0
 ; CHECK-BE-NEXT:    xscvdpsxws f2, f2
-; CHECK-BE-NEXT:    xscvdpsxws f3, f3
-; CHECK-BE-NEXT:    xscvdpsxws f4, f4
-; CHECK-BE-NEXT:    xscvdpsxws f5, f5
-; CHECK-BE-NEXT:    xscvdpsxws f6, f6
-; CHECK-BE-NEXT:    xscvdpsxws f7, f7
-; CHECK-BE-NEXT:    mfvsrwz r5, f1
-; CHECK-BE-NEXT:    mfvsrwz r9, f0
-; CHECK-BE-NEXT:    sldi r5, r5, 48
-; CHECK-BE-NEXT:    sldi r9, r9, 48
+; CHECK-BE-NEXT:    lxv vs0, 0(r3)
 ; CHECK-BE-NEXT:    mfvsrwz r3, f2
-; CHECK-BE-NEXT:    mfvsrwz r4, f3
-; CHECK-BE-NEXT:    mfvsrwz r6, f4
-; CHECK-BE-NEXT:    mfvsrwz r7, f5
-; CHECK-BE-NEXT:    mfvsrwz r8, f6
-; CHECK-BE-NEXT:    mfvsrwz r10, f7
-; CHECK-BE-NEXT:    mtvsrd v4, r5
-; CHECK-BE-NEXT:    mtvsrd v6, r9
+; CHECK-BE-NEXT:    xxswapd vs2, vs1
 ; CHECK-BE-NEXT:    sldi r3, r3, 48
-; CHECK-BE-NEXT:    sldi r4, r4, 48
-; CHECK-BE-NEXT:    sldi r6, r6, 48
-; CHECK-BE-NEXT:    sldi r7, r7, 48
-; CHECK-BE-NEXT:    sldi r8, r8, 48
-; CHECK-BE-NEXT:    sldi r10, r10, 48
+; CHECK-BE-NEXT:    xscvspdpn f2, vs2
 ; CHECK-BE-NEXT:    mtvsrd v2, r3
-; CHECK-BE-NEXT:    mtvsrd v3, r4
-; CHECK-BE-NEXT:    mtvsrd v5, r6
-; CHECK-BE-NEXT:    mtvsrd v0, r7
-; CHECK-BE-NEXT:    mtvsrd v1, r8
-; CHECK-BE-NEXT:    mtvsrd v7, r10
+; CHECK-BE-NEXT:    xscvdpsxws f2, f2
+; CHECK-BE-NEXT:    mfvsrwz r3, f2
+; CHECK-BE-NEXT:    xscvspdpn f2, vs1
+; CHECK-BE-NEXT:    xxsldwi vs1, vs1, vs1, 1
+; CHECK-BE-NEXT:    sldi r3, r3, 48
+; CHECK-BE-NEXT:    xscvdpsxws f2, f2
+; CHECK-BE-NEXT:    xscvspdpn f1, vs1
+; CHECK-BE-NEXT:    mtvsrd v3, r3
+; CHECK-BE-NEXT:    xscvdpsxws f1, f1
 ; CHECK-BE-NEXT:    vmrghh v2, v3, v2
-; CHECK-BE-NEXT:    vmrghh v3, v4, v5
-; CHECK-BE-NEXT:    vmrghh v4, v1, v0
-; CHECK-BE-NEXT:    vmrghh v5, v6, v7
+; CHECK-BE-NEXT:    mfvsrwz r3, f2
+; CHECK-BE-NEXT:    sldi r3, r3, 48
+; CHECK-BE-NEXT:    mtvsrd v3, r3
+; CHECK-BE-NEXT:    mfvsrwz r3, f1
+; CHECK-BE-NEXT:    xxsldwi vs1, vs0, vs0, 3
+; CHECK-BE-NEXT:    sldi r3, r3, 48
+; CHECK-BE-NEXT:    xscvspdpn f1, vs1
+; CHECK-BE-NEXT:    mtvsrd v4, r3
+; CHECK-BE-NEXT:    xscvdpsxws f1, f1
+; CHECK-BE-NEXT:    mfvsrwz r3, f1
+; CHECK-BE-NEXT:    xxswapd vs1, vs0
+; CHECK-BE-NEXT:    xscvspdpn f1, vs1
+; CHECK-BE-NEXT:    xscvdpsxws f1, f1
+; CHECK-BE-NEXT:    vmrghh v3, v3, v4
+; CHECK-BE-NEXT:    sldi r3, r3, 48
 ; CHECK-BE-NEXT:    vmrghw v2, v3, v2
-; CHECK-BE-NEXT:    vmrghw v3, v5, v4
+; CHECK-BE-NEXT:    mtvsrd v3, r3
+; CHECK-BE-NEXT:    mfvsrwz r3, f1
+; CHECK-BE-NEXT:    xscvspdpn f1, vs0
+; CHECK-BE-NEXT:    xxsldwi vs0, vs0, vs0, 1
+; CHECK-BE-NEXT:    sldi r3, r3, 48
+; CHECK-BE-NEXT:    xscvdpsxws f1, f1
+; CHECK-BE-NEXT:    xscvspdpn f0, vs0
+; CHECK-BE-NEXT:    mtvsrd v4, r3
+; CHECK-BE-NEXT:    xscvdpsxws f0, f0
+; CHECK-BE-NEXT:    vmrghh v3, v4, v3
+; CHECK-BE-NEXT:    mfvsrwz r3, f1
+; CHECK-BE-NEXT:    sldi r3, r3, 48
+; CHECK-BE-NEXT:    mtvsrd v4, r3
+; CHECK-BE-NEXT:    mfvsrwz r3, f0
+; CHECK-BE-NEXT:    sldi r3, r3, 48
+; CHECK-BE-NEXT:    mtvsrd v5, r3
+; CHECK-BE-NEXT:    vmrghh v4, v4, v5
+; CHECK-BE-NEXT:    vmrghw v3, v4, v3
 ; CHECK-BE-NEXT:    xxmrghd v2, v3, v2
 ; CHECK-BE-NEXT:    blr
 entry:
@@ -481,259 +481,235 @@ define void @test16elt(<16 x i16>* noalias nocapture sret %agg.result, <16 x flo
 ;
 ; CHECK-P9-LABEL: test16elt:
 ; CHECK-P9:       # %bb.0: # %entry
-; CHECK-P9-NEXT:    lxv vs2, 16(r4)
-; CHECK-P9-NEXT:    lxv vs3, 0(r4)
-; CHECK-P9-NEXT:    lxv vs0, 48(r4)
-; CHECK-P9-NEXT:    lxv vs1, 32(r4)
-; CHECK-P9-NEXT:    std r25, -56(r1) # 8-byte Folded Spill
-; CHECK-P9-NEXT:    std r26, -48(r1) # 8-byte Folded Spill
-; CHECK-P9-NEXT:    std r27, -40(r1) # 8-byte Folded Spill
-; CHECK-P9-NEXT:    std r28, -32(r1) # 8-byte Folded Spill
-; CHECK-P9-NEXT:    std r29, -24(r1) # 8-byte Folded Spill
-; CHECK-P9-NEXT:    std r30, -16(r1) # 8-byte Folded Spill
-; CHECK-P9-NEXT:    xxsldwi vs4, vs3, vs3, 3
-; CHECK-P9-NEXT:    xxswapd vs5, vs3
-; CHECK-P9-NEXT:    xxsldwi vs6, vs3, vs3, 1
-; CHECK-P9-NEXT:    xxsldwi vs7, vs2, vs2, 3
-; CHECK-P9-NEXT:    xxswapd vs8, vs2
-; CHECK-P9-NEXT:    xxsldwi vs9, vs2, vs2, 1
-; CHECK-P9-NEXT:    xxsldwi vs10, vs1, vs1, 3
-; CHECK-P9-NEXT:    xxswapd vs11, vs1
-; CHECK-P9-NEXT:    xxsldwi vs12, vs1, vs1, 1
-; CHECK-P9-NEXT:    xxsldwi vs13, vs0, vs0, 3
-; CHECK-P9-NEXT:    xxswapd v2, vs0
-; CHECK-P9-NEXT:    xxsldwi v3, vs0, vs0, 1
-; CHECK-P9-NEXT:    xscvspdpn f3, vs3
-; CHECK-P9-NEXT:    xscvspdpn f2, vs2
-; CHECK-P9-NEXT:    xscvspdpn f1, vs1
-; CHECK-P9-NEXT:    xscvspdpn f0, vs0
+; CHECK-P9-NEXT:    lxv vs1, 0(r4)
+; CHECK-P9-NEXT:    lxv vs3, 16(r4)
+; CHECK-P9-NEXT:    xscvspdpn f5, vs1
+; CHECK-P9-NEXT:    xxsldwi vs2, vs1, vs1, 3
+; CHECK-P9-NEXT:    xscvspdpn f8, vs3
+; CHECK-P9-NEXT:    xxswapd vs4, vs1
+; CHECK-P9-NEXT:    xxsldwi vs1, vs1, vs1, 1
 ; CHECK-P9-NEXT:    xscvspdpn f4, vs4
-; CHECK-P9-NEXT:    xscvspdpn f5, vs5
+; CHECK-P9-NEXT:    xscvdpsxws f5, f5
+; CHECK-P9-NEXT:    xscvspdpn f2, vs2
+; CHECK-P9-NEXT:    xscvdpsxws f8, f8
+; CHECK-P9-NEXT:    xxsldwi vs6, vs3, vs3, 3
+; CHECK-P9-NEXT:    xxswapd vs7, vs3
 ; CHECK-P9-NEXT:    xscvspdpn f6, vs6
+; CHECK-P9-NEXT:    xxsldwi vs3, vs3, vs3, 1
 ; CHECK-P9-NEXT:    xscvspdpn f7, vs7
-; CHECK-P9-NEXT:    xscvspdpn f8, vs8
-; CHECK-P9-NEXT:    xscvspdpn f9, vs9
-; CHECK-P9-NEXT:    xscvspdpn f10, vs10
-; CHECK-P9-NEXT:    xscvspdpn f11, vs11
-; CHECK-P9-NEXT:    xscvspdpn f12, vs12
-; CHECK-P9-NEXT:    xscvspdpn f13, vs13
-; CHECK-P9-NEXT:    xscvspdpn v2, v2
-; CHECK-P9-NEXT:    xscvspdpn v3, v3
-; CHECK-P9-NEXT:    xscvdpsxws f3, f3
+; CHECK-P9-NEXT:    xscvspdpn f3, vs3
 ; CHECK-P9-NEXT:    xscvdpsxws f2, f2
-; CHECK-P9-NEXT:    xscvdpsxws f1, f1
-; CHECK-P9-NEXT:    xscvdpsxws f0, f0
+; CHECK-P9-NEXT:    xscvspdpn f1, vs1
 ; CHECK-P9-NEXT:    xscvdpsxws f4, f4
-; CHECK-P9-NEXT:    xscvdpsxws f5, f5
 ; CHECK-P9-NEXT:    xscvdpsxws f6, f6
+; CHECK-P9-NEXT:    mfvsrwz r5, f5
+; CHECK-P9-NEXT:    xscvdpsxws f1, f1
 ; CHECK-P9-NEXT:    xscvdpsxws f7, f7
-; CHECK-P9-NEXT:    xscvdpsxws f8, f8
+; CHECK-P9-NEXT:    xscvdpsxws f3, f3
+; CHECK-P9-NEXT:    mtvsrd f5, r5
+; CHECK-P9-NEXT:    mfvsrwz r5, f8
+; CHECK-P9-NEXT:    mtvsrd f8, r5
+; CHECK-P9-NEXT:    mfvsrwz r5, f2
+; CHECK-P9-NEXT:    lxv vs0, 32(r4)
+; CHECK-P9-NEXT:    xxsldwi vs9, vs0, vs0, 3
+; CHECK-P9-NEXT:    xxswapd vs10, vs0
+; CHECK-P9-NEXT:    xscvspdpn f9, vs9
+; CHECK-P9-NEXT:    xscvspdpn f10, vs10
 ; CHECK-P9-NEXT:    xscvdpsxws f9, f9
 ; CHECK-P9-NEXT:    xscvdpsxws f10, f10
-; CHECK-P9-NEXT:    xscvdpsxws f11, f11
-; CHECK-P9-NEXT:    xscvdpsxws f12, f12
-; CHECK-P9-NEXT:    xscvdpsxws f13, f13
-; CHECK-P9-NEXT:    xscvdpsxws v2, v2
-; CHECK-P9-NEXT:    xscvdpsxws v3, v3
-; CHECK-P9-NEXT:    mfvsrwz r4, f3
-; CHECK-P9-NEXT:    mfvsrwz r5, f2
-; CHECK-P9-NEXT:    mfvsrwz r12, f1
-; CHECK-P9-NEXT:    mfvsrwz r0, f0
-; CHECK-P9-NEXT:    mfvsrwz r6, f4
-; CHECK-P9-NEXT:    mfvsrwz r7, f5
-; CHECK-P9-NEXT:    mfvsrwz r8, f6
-; CHECK-P9-NEXT:    mfvsrwz r9, f7
-; CHECK-P9-NEXT:    mfvsrwz r10, f8
-; CHECK-P9-NEXT:    mfvsrwz r11, f9
-; CHECK-P9-NEXT:    mfvsrwz r30, f10
-; CHECK-P9-NEXT:    mfvsrwz r29, f11
-; CHECK-P9-NEXT:    mfvsrwz r28, f12
-; CHECK-P9-NEXT:    mfvsrwz r27, f13
-; CHECK-P9-NEXT:    mfvsrwz r26, v2
-; CHECK-P9-NEXT:    mfvsrwz r25, v3
-; CHECK-P9-NEXT:    mtvsrd f0, r4
+; CHECK-P9-NEXT:    mtvsrd f2, r5
+; CHECK-P9-NEXT:    mfvsrwz r5, f4
+; CHECK-P9-NEXT:    mtvsrd f4, r5
+; CHECK-P9-NEXT:    mfvsrwz r5, f1
 ; CHECK-P9-NEXT:    mtvsrd f1, r5
-; CHECK-P9-NEXT:    mtvsrd f8, r12
-; CHECK-P9-NEXT:    mtvsrd f9, r0
-; CHECK-P9-NEXT:    mtvsrd f2, r6
-; CHECK-P9-NEXT:    mtvsrd f3, r7
-; CHECK-P9-NEXT:    mtvsrd f4, r8
-; CHECK-P9-NEXT:    mtvsrd f5, r9
-; CHECK-P9-NEXT:    mtvsrd f6, r10
-; CHECK-P9-NEXT:    mtvsrd f7, r11
-; CHECK-P9-NEXT:    mtvsrd f10, r30
-; CHECK-P9-NEXT:    ld r30, -16(r1) # 8-byte Folded Reload
-; CHECK-P9-NEXT:    mtvsrd f11, r29
-; CHECK-P9-NEXT:    ld r29, -24(r1) # 8-byte Folded Reload
-; CHECK-P9-NEXT:    mtvsrd f12, r28
-; CHECK-P9-NEXT:    ld r28, -32(r1) # 8-byte Folded Reload
-; CHECK-P9-NEXT:    mtvsrd f13, r27
-; CHECK-P9-NEXT:    ld r27, -40(r1) # 8-byte Folded Reload
-; CHECK-P9-NEXT:    mtvsrd v2, r26
-; CHECK-P9-NEXT:    ld r26, -48(r1) # 8-byte Folded Reload
-; CHECK-P9-NEXT:    mtvsrd v3, r25
-; CHECK-P9-NEXT:    ld r25, -56(r1) # 8-byte Folded Reload
-; CHECK-P9-NEXT:    xxswapd v4, vs0
-; CHECK-P9-NEXT:    xxswapd v5, vs2
+; CHECK-P9-NEXT:    mfvsrwz r5, f6
+; CHECK-P9-NEXT:    xxswapd v2, vs2
+; CHECK-P9-NEXT:    xxswapd v3, vs4
+; CHECK-P9-NEXT:    xscvspdpn f2, vs0
+; CHECK-P9-NEXT:    xxsldwi vs0, vs0, vs0, 1
+; CHECK-P9-NEXT:    xscvspdpn f0, vs0
+; CHECK-P9-NEXT:    xscvdpsxws f2, f2
+; CHECK-P9-NEXT:    mtvsrd f6, r5
+; CHECK-P9-NEXT:    mfvsrwz r5, f7
+; CHECK-P9-NEXT:    xxswapd v4, vs1
+; CHECK-P9-NEXT:    lxv vs1, 48(r4)
+; CHECK-P9-NEXT:    vmrglh v2, v3, v2
+; CHECK-P9-NEXT:    xxswapd v3, vs5
+; CHECK-P9-NEXT:    mtvsrd f7, r5
+; CHECK-P9-NEXT:    mfvsrwz r5, f3
+; CHECK-P9-NEXT:    vmrglh v3, v3, v4
+; CHECK-P9-NEXT:    xxswapd v4, vs6
+; CHECK-P9-NEXT:    xxswapd v5, vs7
+; CHECK-P9-NEXT:    mtvsrd f3, r5
+; CHECK-P9-NEXT:    xscvdpsxws f0, f0
 ; CHECK-P9-NEXT:    xxswapd v0, vs3
-; CHECK-P9-NEXT:    xxswapd v1, vs4
-; CHECK-P9-NEXT:    xxswapd v6, vs5
-; CHECK-P9-NEXT:    xxswapd v7, vs6
-; CHECK-P9-NEXT:    xxswapd v8, vs1
-; CHECK-P9-NEXT:    xxswapd v9, vs7
-; CHECK-P9-NEXT:    xxswapd v10, vs10
-; CHECK-P9-NEXT:    xxswapd v11, vs11
-; CHECK-P9-NEXT:    xxswapd v12, vs8
-; CHECK-P9-NEXT:    xxswapd v13, vs12
-; CHECK-P9-NEXT:    xxswapd v14, vs13
-; CHECK-P9-NEXT:    xxswapd v2, v2
-; CHECK-P9-NEXT:    xxswapd v15, vs9
-; CHECK-P9-NEXT:    xxswapd v3, v3
-; CHECK-P9-NEXT:    vmrglh v5, v0, v5
-; CHECK-P9-NEXT:    vmrglh v4, v4, v1
-; CHECK-P9-NEXT:    vmrglh v0, v7, v6
-; CHECK-P9-NEXT:    vmrglh v1, v8, v9
-; CHECK-P9-NEXT:    vmrglh v6, v11, v10
-; CHECK-P9-NEXT:    vmrglh v7, v12, v13
-; CHECK-P9-NEXT:    vmrglh v2, v2, v14
-; CHECK-P9-NEXT:    vmrglh v3, v15, v3
-; CHECK-P9-NEXT:    vmrglw v4, v4, v5
-; CHECK-P9-NEXT:    vmrglw v5, v1, v0
-; CHECK-P9-NEXT:    vmrglw v0, v7, v6
+; CHECK-P9-NEXT:    vmrglh v4, v5, v4
+; CHECK-P9-NEXT:    xxswapd v5, vs8
+; CHECK-P9-NEXT:    vmrglh v5, v5, v0
+; CHECK-P9-NEXT:    mfvsrwz r4, f2
+; CHECK-P9-NEXT:    mtvsrd f2, r4
+; CHECK-P9-NEXT:    mfvsrwz r4, f0
 ; CHECK-P9-NEXT:    vmrglw v2, v3, v2
-; CHECK-P9-NEXT:    xxmrgld vs0, v5, v4
-; CHECK-P9-NEXT:    xxmrgld vs1, v2, v0
-; CHECK-P9-NEXT:    stxv vs0, 0(r3)
-; CHECK-P9-NEXT:    stxv vs1, 16(r3)
+; CHECK-P9-NEXT:    mtvsrd f0, r4
+; CHECK-P9-NEXT:    vmrglw v3, v5, v4
+; CHECK-P9-NEXT:    xxswapd v4, vs2
+; CHECK-P9-NEXT:    xxmrgld vs2, v3, v2
+; CHECK-P9-NEXT:    xxswapd v2, vs0
+; CHECK-P9-NEXT:    xxsldwi vs0, vs1, vs1, 3
+; CHECK-P9-NEXT:    xscvspdpn f0, vs0
+; CHECK-P9-NEXT:    xscvdpsxws f0, f0
+; CHECK-P9-NEXT:    mfvsrwz r4, f0
+; CHECK-P9-NEXT:    mtvsrd f0, r4
+; CHECK-P9-NEXT:    xxswapd v3, vs0
+; CHECK-P9-NEXT:    xxswapd vs0, vs1
+; CHECK-P9-NEXT:    xscvspdpn f0, vs0
+; CHECK-P9-NEXT:    xscvdpsxws f0, f0
+; CHECK-P9-NEXT:    mfvsrwz r4, f0
+; CHECK-P9-NEXT:    mtvsrd f0, r4
+; CHECK-P9-NEXT:    vmrglh v2, v4, v2
+; CHECK-P9-NEXT:    xxswapd v4, vs0
+; CHECK-P9-NEXT:    xscvspdpn f0, vs1
+; CHECK-P9-NEXT:    xscvdpsxws f0, f0
+; CHECK-P9-NEXT:    mfvsrwz r4, f0
+; CHECK-P9-NEXT:    mtvsrd f0, r4
+; CHECK-P9-NEXT:    vmrglh v3, v4, v3
+; CHECK-P9-NEXT:    xxswapd v4, vs0
+; CHECK-P9-NEXT:    xxsldwi vs0, vs1, vs1, 1
+; CHECK-P9-NEXT:    xscvspdpn f0, vs0
+; CHECK-P9-NEXT:    xscvdpsxws f0, f0
+; CHECK-P9-NEXT:    mfvsrwz r5, f9
+; CHECK-P9-NEXT:    mtvsrd f9, r5
+; CHECK-P9-NEXT:    mfvsrwz r5, f10
+; CHECK-P9-NEXT:    mtvsrd f10, r5
+; CHECK-P9-NEXT:    xxswapd v0, vs9
+; CHECK-P9-NEXT:    xxswapd v1, vs10
+; CHECK-P9-NEXT:    vmrglh v0, v1, v0
+; CHECK-P9-NEXT:    vmrglw v2, v2, v0
+; CHECK-P9-NEXT:    stxv vs2, 0(r3)
+; CHECK-P9-NEXT:    mfvsrwz r4, f0
+; CHECK-P9-NEXT:    mtvsrd f0, r4
+; CHECK-P9-NEXT:    xxswapd v5, vs0
+; CHECK-P9-NEXT:    vmrglh v4, v4, v5
+; CHECK-P9-NEXT:    vmrglw v3, v4, v3
+; CHECK-P9-NEXT:    xxmrgld vs0, v3, v2
+; CHECK-P9-NEXT:    stxv vs0, 16(r3)
 ; CHECK-P9-NEXT:    blr
 ;
 ; CHECK-BE-LABEL: test16elt:
 ; CHECK-BE:       # %bb.0: # %entry
-; CHECK-BE-NEXT:    lxv vs2, 0(r4)
-; CHECK-BE-NEXT:    lxv vs3, 16(r4)
-; CHECK-BE-NEXT:    lxv vs0, 32(r4)
-; CHECK-BE-NEXT:    lxv vs1, 48(r4)
-; CHECK-BE-NEXT:    std r25, -56(r1) # 8-byte Folded Spill
-; CHECK-BE-NEXT:    std r26, -48(r1) # 8-byte Folded Spill
-; CHECK-BE-NEXT:    std r27, -40(r1) # 8-byte Folded Spill
-; CHECK-BE-NEXT:    std r28, -32(r1) # 8-byte Folded Spill
-; CHECK-BE-NEXT:    std r29, -24(r1) # 8-byte Folded Spill
-; CHECK-BE-NEXT:    std r30, -16(r1) # 8-byte Folded Spill
-; CHECK-BE-NEXT:    xxsldwi vs4, vs3, vs3, 3
-; CHECK-BE-NEXT:    xxswapd vs5, vs3
-; CHECK-BE-NEXT:    xxsldwi vs6, vs3, vs3, 1
-; CHECK-BE-NEXT:    xxsldwi vs7, vs2, vs2, 3
-; CHECK-BE-NEXT:    xxswapd vs8, vs2
-; CHECK-BE-NEXT:    xxsldwi vs9, vs2, vs2, 1
-; CHECK-BE-NEXT:    xxsldwi vs10, vs1, vs1, 3
-; CHECK-BE-NEXT:    xxswapd vs11, vs1
-; CHECK-BE-NEXT:    xxsldwi vs12, vs1, vs1, 1
-; CHECK-BE-NEXT:    xxsldwi vs13, vs0, vs0, 3
-; CHECK-BE-NEXT:    xxswapd v2, vs0
-; CHECK-BE-NEXT:    xxsldwi v3, vs0, vs0, 1
-; CHECK-BE-NEXT:    xscvspdpn f3, vs3
+; CHECK-BE-NEXT:    lxv vs1, 16(r4)
+; CHECK-BE-NEXT:    xxsldwi vs2, vs1, vs1, 3
 ; CHECK-BE-NEXT:    xscvspdpn f2, vs2
-; CHECK-BE-NEXT:    xscvspdpn f1, vs1
-; CHECK-BE-NEXT:    xscvspdpn f0, vs0
-; CHECK-BE-NEXT:    xscvspdpn f4, vs4
-; CHECK-BE-NEXT:    xscvspdpn f5, vs5
-; CHECK-BE-NEXT:    xscvspdpn f6, vs6
-; CHECK-BE-NEXT:    xscvspdpn f7, vs7
-; CHECK-BE-NEXT:    xscvspdpn f8, vs8
-; CHECK-BE-NEXT:    xscvspdpn f9, vs9
-; CHECK-BE-NEXT:    xscvspdpn f10, vs10
-; CHECK-BE-NEXT:    xscvspdpn f11, vs11
-; CHECK-BE-NEXT:    xscvspdpn f12, vs12
-; CHECK-BE-NEXT:    xscvspdpn f13, vs13
-; CHECK-BE-NEXT:    xscvspdpn v2, v2
-; CHECK-BE-NEXT:    xscvspdpn v3, v3
+; CHECK-BE-NEXT:    xxswapd vs3, vs1
+; CHECK-BE-NEXT:    xscvspdpn f3, vs3
+; CHECK-BE-NEXT:    xscvdpsxws f2, f2
 ; CHECK-BE-NEXT:    xscvdpsxws f3, f3
+; CHECK-BE-NEXT:    mfvsrwz r5, f2
+; CHECK-BE-NEXT:    xscvspdpn f4, vs1
+; CHECK-BE-NEXT:    xxsldwi vs1, vs1, vs1, 1
+; CHECK-BE-NEXT:    xscvspdpn f1, vs1
+; CHECK-BE-NEXT:    xscvdpsxws f1, f1
+; CHECK-BE-NEXT:    sldi r5, r5, 48
+; CHECK-BE-NEXT:    mtvsrd v2, r5
+; CHECK-BE-NEXT:    mfvsrwz r5, f3
+; CHECK-BE-NEXT:    xscvdpsxws f3, f4
+; CHECK-BE-NEXT:    lxv vs0, 0(r4)
+; CHECK-BE-NEXT:    xxsldwi vs2, vs0, vs0, 3
+; CHECK-BE-NEXT:    xscvspdpn f2, vs2
+; CHECK-BE-NEXT:    sldi r5, r5, 48
+; CHECK-BE-NEXT:    mtvsrd v3, r5
+; CHECK-BE-NEXT:    vmrghh v2, v3, v2
+; CHECK-BE-NEXT:    mfvsrwz r5, f3
+; CHECK-BE-NEXT:    sldi r5, r5, 48
+; CHECK-BE-NEXT:    mtvsrd v3, r5
+; CHECK-BE-NEXT:    mfvsrwz r5, f1
+; CHECK-BE-NEXT:    xxswapd vs1, vs0
 ; CHECK-BE-NEXT:    xscvdpsxws f2, f2
+; CHECK-BE-NEXT:    sldi r5, r5, 48
+; CHECK-BE-NEXT:    xscvspdpn f1, vs1
+; CHECK-BE-NEXT:    mtvsrd v4, r5
+; CHECK-BE-NEXT:    mfvsrwz r5, f2
+; CHECK-BE-NEXT:    xscvdpsxws f1, f1
+; CHECK-BE-NEXT:    sldi r5, r5, 48
+; CHECK-BE-NEXT:    vmrghh v3, v3, v4
+; CHECK-BE-NEXT:    mtvsrd v4, r5
+; CHECK-BE-NEXT:    vmrghw v2, v3, v2
+; CHECK-BE-NEXT:    mfvsrwz r5, f1
+; CHECK-BE-NEXT:    xscvspdpn f1, vs0
+; CHECK-BE-NEXT:    xxsldwi vs0, vs0, vs0, 1
+; CHECK-BE-NEXT:    sldi r5, r5, 48
 ; CHECK-BE-NEXT:    xscvdpsxws f1, f1
+; CHECK-BE-NEXT:    xscvspdpn f0, vs0
+; CHECK-BE-NEXT:    mtvsrd v5, r5
 ; CHECK-BE-NEXT:    xscvdpsxws f0, f0
-; CHECK-BE-NEXT:    xscvdpsxws f4, f4
+; CHECK-BE-NEXT:    vmrghh v4, v5, v4
+; CHECK-BE-NEXT:    mfvsrwz r5, f1
+; CHECK-BE-NEXT:    lxv vs1, 48(r4)
+; CHECK-BE-NEXT:    sldi r5, r5, 48
+; CHECK-BE-NEXT:    mtvsrd v5, r5
+; CHECK-BE-NEXT:    mfvsrwz r5, f0
+; CHECK-BE-NEXT:    lxv vs0, 32(r4)
+; CHECK-BE-NEXT:    xscvspdpn f5, vs1
+; CHECK-BE-NEXT:    xxsldwi vs2, vs1, vs1, 3
+; CHECK-BE-NEXT:    xscvspdpn f2, vs2
 ; CHECK-BE-NEXT:    xscvdpsxws f5, f5
-; CHECK-BE-NEXT:    xscvdpsxws f6, f6
-; CHECK-BE-NEXT:    xscvdpsxws f7, f7
-; CHECK-BE-NEXT:    xscvdpsxws f8, f8
-; CHECK-BE-NEXT:    xscvdpsxws f9, f9
-; CHECK-BE-NEXT:    xscvdpsxws f10, f10
-; CHECK-BE-NEXT:    xscvdpsxws f11, f11
-; CHECK-BE-NEXT:    xscvdpsxws f12, f12
-; CHECK-BE-NEXT:    xscvdpsxws f13, f13
-; CHECK-BE-NEXT:    xscvdpsxws v2, v2
-; CHECK-BE-NEXT:    xscvdpsxws v3, v3
-; CHECK-BE-NEXT:    mfvsrwz r4, f3
-; CHECK-BE-NEXT:    mfvsrwz r5, f2
-; CHECK-BE-NEXT:    mfvsrwz r12, f1
-; CHECK-BE-NEXT:    mfvsrwz r0, f0
-; CHECK-BE-NEXT:    mfvsrwz r6, f4
-; CHECK-BE-NEXT:    mfvsrwz r7, f5
-; CHECK-BE-NEXT:    mfvsrwz r8, f6
-; CHECK-BE-NEXT:    mfvsrwz r9, f7
-; CHECK-BE-NEXT:    mfvsrwz r10, f8
-; CHECK-BE-NEXT:    mfvsrwz r11, f9
-; CHECK-BE-NEXT:    mfvsrwz r30, f10
-; CHECK-BE-NEXT:    mfvsrwz r29, f11
-; CHECK-BE-NEXT:    mfvsrwz r28, f12
-; CHECK-BE-NEXT:    mfvsrwz r27, f13
-; CHECK-BE-NEXT:    mfvsrwz r26, v2
-; CHECK-BE-NEXT:    mfvsrwz r25, v3
-; CHECK-BE-NEXT:    sldi r4, r4, 48
 ; CHECK-BE-NEXT:    sldi r5, r5, 48
-; CHECK-BE-NEXT:    sldi r12, r12, 48
-; CHECK-BE-NEXT:    sldi r0, r0, 48
-; CHECK-BE-NEXT:    sldi r6, r6, 48
-; CHECK-BE-NEXT:    sldi r7, r7, 48
-; CHECK-BE-NEXT:    sldi r8, r8, 48
-; CHECK-BE-NEXT:    sldi r9, r9, 48
-; CHECK-BE-NEXT:    sldi r10, r10, 48
-; CHECK-BE-NEXT:    sldi r11, r11, 48
-; CHECK-BE-NEXT:    sldi r30, r30, 48
-; CHECK-BE-NEXT:    sldi r29, r29, 48
-; CHECK-BE-NEXT:    sldi r28, r28, 48
-; CHECK-BE-NEXT:    sldi r27, r27, 48
-; CHECK-BE-NEXT:    sldi r26, r26, 48
-; CHECK-BE-NEXT:    sldi r25, r25, 48
+; CHECK-BE-NEXT:    xxswapd vs3, vs1
+; CHECK-BE-NEXT:    mtvsrd v0, r5
+; CHECK-BE-NEXT:    vmrghh v5, v5, v0
+; CHECK-BE-NEXT:    xscvspdpn f3, vs3
+; CHECK-BE-NEXT:    xxsldwi vs1, vs1, vs1, 1
+; CHECK-BE-NEXT:    xscvspdpn f1, vs1
+; CHECK-BE-NEXT:    xscvdpsxws f2, f2
+; CHECK-BE-NEXT:    vmrghw v3, v5, v4
+; CHECK-BE-NEXT:    xscvdpsxws f3, f3
+; CHECK-BE-NEXT:    mfvsrwz r4, f5
+; CHECK-BE-NEXT:    xxmrghd vs4, v3, v2
+; CHECK-BE-NEXT:    sldi r4, r4, 48
 ; CHECK-BE-NEXT:    mtvsrd v2, r4
-; CHECK-BE-NEXT:    mtvsrd v3, r5
-; CHECK-BE-NEXT:    mtvsrd v10, r12
-; CHECK-BE-NEXT:    mtvsrd v14, r0
-; CHECK-BE-NEXT:    mtvsrd v4, r6
-; CHECK-BE-NEXT:    mtvsrd v5, r7
-; CHECK-BE-NEXT:    mtvsrd v0, r8
-; CHECK-BE-NEXT:    mtvsrd v1, r9
-; CHECK-BE-NEXT:    mtvsrd v6, r10
-; CHECK-BE-NEXT:    mtvsrd v7, r11
-; CHECK-BE-NEXT:    mtvsrd v8, r30
-; CHECK-BE-NEXT:    ld r30, -16(r1) # 8-byte Folded Reload
-; CHECK-BE-NEXT:    mtvsrd v9, r29
-; CHECK-BE-NEXT:    ld r29, -24(r1) # 8-byte Folded Reload
-; CHECK-BE-NEXT:    mtvsrd v11, r28
-; CHECK-BE-NEXT:    ld r28, -32(r1) # 8-byte Folded Reload
-; CHECK-BE-NEXT:    mtvsrd v12, r27
-; CHECK-BE-NEXT:    ld r27, -40(r1) # 8-byte Folded Reload
-; CHECK-BE-NEXT:    mtvsrd v13, r26
-; CHECK-BE-NEXT:    ld r26, -48(r1) # 8-byte Folded Reload
-; CHECK-BE-NEXT:    mtvsrd v15, r25
-; CHECK-BE-NEXT:    ld r25, -56(r1) # 8-byte Folded Reload
-; CHECK-BE-NEXT:    vmrghh v4, v5, v4
-; CHECK-BE-NEXT:    vmrghh v2, v2, v0
-; CHECK-BE-NEXT:    vmrghh v5, v6, v1
-; CHECK-BE-NEXT:    vmrghh v3, v3, v7
-; CHECK-BE-NEXT:    vmrghh v0, v9, v8
-; CHECK-BE-NEXT:    vmrghh v1, v10, v11
-; CHECK-BE-NEXT:    vmrghh v6, v13, v12
-; CHECK-BE-NEXT:    vmrghh v7, v14, v15
-; CHECK-BE-NEXT:    vmrghw v2, v2, v4
-; CHECK-BE-NEXT:    vmrghw v3, v3, v5
-; CHECK-BE-NEXT:    vmrghw v4, v1, v0
-; CHECK-BE-NEXT:    vmrghw v5, v7, v6
-; CHECK-BE-NEXT:    xxmrghd vs0, v3, v2
-; CHECK-BE-NEXT:    xxmrghd vs1, v5, v4
-; CHECK-BE-NEXT:    stxv vs0, 0(r3)
-; CHECK-BE-NEXT:    stxv vs1, 16(r3)
-; CHECK-BE-NEXT:    blr
+; CHECK-BE-NEXT:    mfvsrwz r4, f2
+; CHECK-BE-NEXT:    xscvdpsxws f1, f1
+; CHECK-BE-NEXT:    stxv vs4, 0(r3)
+; CHECK-BE-NEXT:    sldi r4, r4, 48
+; CHECK-BE-NEXT:    mtvsrd v3, r4
+; CHECK-BE-NEXT:    mfvsrwz r4, f3
+; CHECK-BE-NEXT:    sldi r4, r4, 48
+; CHECK-BE-NEXT:    mtvsrd v4, r4
+; CHECK-BE-NEXT:    mfvsrwz r4, f1
+; CHECK-BE-NEXT:    xxsldwi vs1, vs0, vs0, 3
+; CHECK-BE-NEXT:    sldi r4, r4, 48
+; CHECK-BE-NEXT:    xscvspdpn f1, vs1
+; CHECK-BE-NEXT:    xscvdpsxws f1, f1
+; CHECK-BE-NEXT:    vmrghh v3, v4, v3
+; CHECK-BE-NEXT:    mtvsrd v4, r4
+; CHECK-BE-NEXT:    mfvsrwz r4, f1
+; CHECK-BE-NEXT:    xxswapd vs1, vs0
+; CHECK-BE-NEXT:    xscvspdpn f1, vs1
+; CHECK-BE-NEXT:    xscvdpsxws f1, f1
+; CHECK-BE-NEXT:    vmrghh v2, v2, v4
+; CHECK-BE-NEXT:    sldi r4, r4, 48
+; CHECK-BE-NEXT:    vmrghw v2, v2, v3
+; CHECK-BE-NEXT:    mtvsrd v3, r4
+; CHECK-BE-NEXT:    mfvsrwz r4, f1
+; CHECK-BE-NEXT:    xscvspdpn f1, vs0
+; CHECK-BE-NEXT:    xxsldwi vs0, vs0, vs0, 1
+; CHECK-BE-NEXT:    sldi r4, r4, 48
+; CHECK-BE-NEXT:    xscvdpsxws f1, f1
+; CHECK-BE-NEXT:    xscvspdpn f0, vs0
+; CHECK-BE-NEXT:    mtvsrd v4, r4
+; CHECK-BE-NEXT:    xscvdpsxws f0, f0
+; CHECK-BE-NEXT:    vmrghh v3, v4, v3
+; CHECK-BE-NEXT:    mfvsrwz r4, f1
+; CHECK-BE-NEXT:    sldi r4, r4, 48
+; CHECK-BE-NEXT:    mtvsrd v4, r4
+; CHECK-BE-NEXT:    mfvsrwz r4, f0
+; CHECK-BE-NEXT:    sldi r4, r4, 48
+; CHECK-BE-NEXT:    mtvsrd v5, r4
+; CHECK-BE-NEXT:    vmrghh v4, v4, v5
+; CHECK-BE-NEXT:    vmrghw v3, v4, v3
+; CHECK-BE-NEXT:    xxmrghd vs0, v3, v2
+; CHECK-BE-NEXT:    stxv vs0, 16(r3)
+; CHECK-BE-NEXT:    blr
 entry:
   %a = load <16 x float>, <16 x float>* %0, align 64
   %1 = fptoui <16 x float> %a to <16 x i16>
@@ -768,35 +744,35 @@ define i32 @test2elt_signed(i64 %a.coerce) local_unnamed_addr #0 {
 ; CHECK-P9-NEXT:    xxswapd v2, vs0
 ; CHECK-P9-NEXT:    xscvspdpn f0, vs0
 ; CHECK-P9-NEXT:    xxsldwi vs1, v2, v2, 3
-; CHECK-P9-NEXT:    xscvdpsxws f0, f0
 ; CHECK-P9-NEXT:    xscvspdpn f1, vs1
 ; CHECK-P9-NEXT:    xscvdpsxws f1, f1
-; CHECK-P9-NEXT:    mfvsrwz r4, f0
+; CHECK-P9-NEXT:    xscvdpsxws f0, f0
 ; CHECK-P9-NEXT:    mfvsrwz r3, f1
-; CHECK-P9-NEXT:    mtvsrd f1, r4
+; CHECK-P9-NEXT:    mtvsrd f1, r3
+; CHECK-P9-NEXT:    mfvsrwz r3, f0
 ; CHECK-P9-NEXT:    mtvsrd f0, r3
-; CHECK-P9-NEXT:    xxswapd v3, vs1
-; CHECK-P9-NEXT:    li r3, 0
-; CHECK-P9-NEXT:    xxswapd v2, vs0
+; CHECK-P9-NEXT:    xxswapd v2, vs1
+; CHECK-P9-NEXT:    xxswapd v3, vs0
 ; CHECK-P9-NEXT:    vmrglh v2, v3, v2
+; CHECK-P9-NEXT:    li r3, 0
 ; CHECK-P9-NEXT:    vextuwrx r3, r3, v2
 ; CHECK-P9-NEXT:    blr
 ;
 ; CHECK-BE-LABEL: test2elt_signed:
 ; CHECK-BE:       # %bb.0: # %entry
 ; CHECK-BE-NEXT:    mtvsrd f0, r3
-; CHECK-BE-NEXT:    xxsldwi vs1, vs0, vs0, 1
+; CHECK-BE-NEXT:    xscvspdpn f1, vs0
+; CHECK-BE-NEXT:    xxsldwi vs0, vs0, vs0, 1
+; CHECK-BE-NEXT:    xscvdpsxws f1, f1
 ; CHECK-BE-NEXT:    xscvspdpn f0, vs0
-; CHECK-BE-NEXT:    xscvspdpn f1, vs1
 ; CHECK-BE-NEXT:    xscvdpsxws f0, f0
-; CHECK-BE-NEXT:    xscvdpsxws f1, f1
-; CHECK-BE-NEXT:    mfvsrwz r3, f0
+; CHECK-BE-NEXT:    mfvsrwz r3, f1
 ; CHECK-BE-NEXT:    sldi r3, r3, 48
-; CHECK-BE-NEXT:    mfvsrwz r4, f1
 ; CHECK-BE-NEXT:    mtvsrd v2, r3
+; CHECK-BE-NEXT:    mfvsrwz r3, f0
+; CHECK-BE-NEXT:    sldi r3, r3, 48
+; CHECK-BE-NEXT:    mtvsrd v3, r3
 ; CHECK-BE-NEXT:    li r3, 0
-; CHECK-BE-NEXT:    sldi r4, r4, 48
-; CHECK-BE-NEXT:    mtvsrd v3, r4
 ; CHECK-BE-NEXT:    vmrghh v2, v2, v3
 ; CHECK-BE-NEXT:    vextuwlx r3, r3, v2
 ; CHECK-BE-NEXT:    blr
@@ -843,62 +819,62 @@ define i64 @test4elt_signed(<4 x float> %a) local_unnamed_addr #1 {
 ; CHECK-P9-LABEL: test4elt_signed:
 ; CHECK-P9:       # %bb.0: # %entry
 ; CHECK-P9-NEXT:    xxsldwi vs0, v2, v2, 3
-; CHECK-P9-NEXT:    xxswapd vs1, v2
-; CHECK-P9-NEXT:    xxsldwi vs2, v2, v2, 1
-; CHECK-P9-NEXT:    xscvspdpn f3, v2
 ; CHECK-P9-NEXT:    xscvspdpn f0, vs0
-; CHECK-P9-NEXT:    xscvspdpn f1, vs1
-; CHECK-P9-NEXT:    xscvspdpn f2, vs2
-; CHECK-P9-NEXT:    xscvdpsxws f3, f3
 ; CHECK-P9-NEXT:    xscvdpsxws f0, f0
-; CHECK-P9-NEXT:    xscvdpsxws f1, f1
-; CHECK-P9-NEXT:    xscvdpsxws f2, f2
-; CHECK-P9-NEXT:    mfvsrwz r5, f3
 ; CHECK-P9-NEXT:    mfvsrwz r3, f0
-; CHECK-P9-NEXT:    mfvsrwz r4, f1
-; CHECK-P9-NEXT:    mfvsrwz r6, f2
-; CHECK-P9-NEXT:    mtvsrd f2, r5
 ; CHECK-P9-NEXT:    mtvsrd f0, r3
-; CHECK-P9-NEXT:    mtvsrd f1, r4
-; CHECK-P9-NEXT:    mtvsrd f3, r6
-; CHECK-P9-NEXT:    xxswapd v4, vs2
+; CHECK-P9-NEXT:    xxswapd v3, vs0
+; CHECK-P9-NEXT:    xxswapd vs0, v2
+; CHECK-P9-NEXT:    xscvspdpn f0, vs0
+; CHECK-P9-NEXT:    xscvdpsxws f0, f0
+; CHECK-P9-NEXT:    mfvsrwz r3, f0
+; CHECK-P9-NEXT:    mtvsrd f0, r3
+; CHECK-P9-NEXT:    xxswapd v4, vs0
+; CHECK-P9-NEXT:    xscvspdpn f0, v2
+; CHECK-P9-NEXT:    xscvdpsxws f0, f0
+; CHECK-P9-NEXT:    mfvsrwz r3, f0
+; CHECK-P9-NEXT:    mtvsrd f0, r3
+; CHECK-P9-NEXT:    vmrglh v3, v4, v3
+; CHECK-P9-NEXT:    xxswapd v4, vs0
+; CHECK-P9-NEXT:    xxsldwi vs0, v2, v2, 1
+; CHECK-P9-NEXT:    xscvspdpn f0, vs0
+; CHECK-P9-NEXT:    xscvdpsxws f0, f0
+; CHECK-P9-NEXT:    mfvsrwz r3, f0
+; CHECK-P9-NEXT:    mtvsrd f0, r3
 ; CHECK-P9-NEXT:    xxswapd v2, vs0
-; CHECK-P9-NEXT:    xxswapd v3, vs1
-; CHECK-P9-NEXT:    xxswapd v5, vs3
-; CHECK-P9-NEXT:    vmrglh v2, v3, v2
-; CHECK-P9-NEXT:    vmrglh v3, v4, v5
-; CHECK-P9-NEXT:    vmrglw v2, v3, v2
+; CHECK-P9-NEXT:    vmrglh v2, v4, v2
+; CHECK-P9-NEXT:    vmrglw v2, v2, v3
 ; CHECK-P9-NEXT:    mfvsrld r3, v2
 ; CHECK-P9-NEXT:    blr
 ;
 ; CHECK-BE-LABEL: test4elt_signed:
 ; CHECK-BE:       # %bb.0: # %entry
 ; CHECK-BE-NEXT:    xxsldwi vs0, v2, v2, 3
-; CHECK-BE-NEXT:    xxswapd vs1, v2
-; CHECK-BE-NEXT:    xxsldwi vs2, v2, v2, 1
-; CHECK-BE-NEXT:    xscvspdpn f3, v2
 ; CHECK-BE-NEXT:    xscvspdpn f0, vs0
-; CHECK-BE-NEXT:    xscvspdpn f1, vs1
-; CHECK-BE-NEXT:    xscvspdpn f2, vs2
-; CHECK-BE-NEXT:    xscvdpsxws f3, f3
 ; CHECK-BE-NEXT:    xscvdpsxws f0, f0
-; CHECK-BE-NEXT:    xscvdpsxws f1, f1
-; CHECK-BE-NEXT:    xscvdpsxws f2, f2
-; CHECK-BE-NEXT:    mfvsrwz r5, f3
-; CHECK-BE-NEXT:    sldi r5, r5, 48
 ; CHECK-BE-NEXT:    mfvsrwz r3, f0
-; CHECK-BE-NEXT:    mfvsrwz r4, f1
-; CHECK-BE-NEXT:    mfvsrwz r6, f2
-; CHECK-BE-NEXT:    mtvsrd v4, r5
+; CHECK-BE-NEXT:    xxswapd vs0, v2
+; CHECK-BE-NEXT:    sldi r3, r3, 48
+; CHECK-BE-NEXT:    xscvspdpn f0, vs0
+; CHECK-BE-NEXT:    mtvsrd v3, r3
+; CHECK-BE-NEXT:    xscvdpsxws f0, f0
+; CHECK-BE-NEXT:    mfvsrwz r3, f0
+; CHECK-BE-NEXT:    xscvspdpn f0, v2
+; CHECK-BE-NEXT:    sldi r3, r3, 48
+; CHECK-BE-NEXT:    xscvdpsxws f0, f0
+; CHECK-BE-NEXT:    mtvsrd v4, r3
+; CHECK-BE-NEXT:    vmrghh v3, v4, v3
+; CHECK-BE-NEXT:    mfvsrwz r3, f0
+; CHECK-BE-NEXT:    xxsldwi vs0, v2, v2, 1
+; CHECK-BE-NEXT:    sldi r3, r3, 48
+; CHECK-BE-NEXT:    xscvspdpn f0, vs0
+; CHECK-BE-NEXT:    mtvsrd v4, r3
+; CHECK-BE-NEXT:    xscvdpsxws f0, f0
+; CHECK-BE-NEXT:    mfvsrwz r3, f0
 ; CHECK-BE-NEXT:    sldi r3, r3, 48
-; CHECK-BE-NEXT:    sldi r4, r4, 48
-; CHECK-BE-NEXT:    sldi r6, r6, 48
 ; CHECK-BE-NEXT:    mtvsrd v2, r3
-; CHECK-BE-NEXT:    mtvsrd v3, r4
-; CHECK-BE-NEXT:    mtvsrd v5, r6
-; CHECK-BE-NEXT:    vmrghh v2, v3, v2
-; CHECK-BE-NEXT:    vmrghh v3, v4, v5
-; CHECK-BE-NEXT:    vmrghw v2, v3, v2
+; CHECK-BE-NEXT:    vmrghh v2, v4, v2
+; CHECK-BE-NEXT:    vmrghw v2, v2, v3
 ; CHECK-BE-NEXT:    mfvsrd r3, v2
 ; CHECK-BE-NEXT:    blr
 entry:
@@ -970,119 +946,119 @@ define <8 x i16> @test8elt_signed(<8 x float>* nocapture readonly) local_unnamed
 ;
 ; CHECK-P9-LABEL: test8elt_signed:
 ; CHECK-P9:       # %bb.0: # %entry
-; CHECK-P9-NEXT:    lxv vs0, 16(r3)
 ; CHECK-P9-NEXT:    lxv vs1, 0(r3)
 ; CHECK-P9-NEXT:    xxsldwi vs2, vs1, vs1, 3
-; CHECK-P9-NEXT:    xxswapd vs3, vs1
-; CHECK-P9-NEXT:    xxsldwi vs4, vs1, vs1, 1
-; CHECK-P9-NEXT:    xxsldwi vs5, vs0, vs0, 3
-; CHECK-P9-NEXT:    xxswapd vs6, vs0
-; CHECK-P9-NEXT:    xxsldwi vs7, vs0, vs0, 1
-; CHECK-P9-NEXT:    xscvspdpn f1, vs1
-; CHECK-P9-NEXT:    xscvspdpn f0, vs0
 ; CHECK-P9-NEXT:    xscvspdpn f2, vs2
-; CHECK-P9-NEXT:    xscvspdpn f3, vs3
-; CHECK-P9-NEXT:    xscvspdpn f4, vs4
-; CHECK-P9-NEXT:    xscvspdpn f5, vs5
-; CHECK-P9-NEXT:    xscvspdpn f6, vs6
-; CHECK-P9-NEXT:    xscvspdpn f7, vs7
-; CHECK-P9-NEXT:    xscvdpsxws f1, f1
-; CHECK-P9-NEXT:    xscvdpsxws f0, f0
 ; CHECK-P9-NEXT:    xscvdpsxws f2, f2
-; CHECK-P9-NEXT:    xscvdpsxws f3, f3
-; CHECK-P9-NEXT:    xscvdpsxws f4, f4
-; CHECK-P9-NEXT:    xscvdpsxws f5, f5
-; CHECK-P9-NEXT:    xscvdpsxws f6, f6
-; CHECK-P9-NEXT:    xscvdpsxws f7, f7
-; CHECK-P9-NEXT:    mfvsrwz r5, f1
-; CHECK-P9-NEXT:    mfvsrwz r9, f0
+; CHECK-P9-NEXT:    lxv vs0, 16(r3)
 ; CHECK-P9-NEXT:    mfvsrwz r3, f2
-; CHECK-P9-NEXT:    mfvsrwz r4, f3
-; CHECK-P9-NEXT:    mfvsrwz r6, f4
-; CHECK-P9-NEXT:    mfvsrwz r7, f5
-; CHECK-P9-NEXT:    mfvsrwz r8, f6
-; CHECK-P9-NEXT:    mfvsrwz r10, f7
-; CHECK-P9-NEXT:    mtvsrd f2, r5
-; CHECK-P9-NEXT:    mtvsrd f6, r9
-; CHECK-P9-NEXT:    mtvsrd f0, r3
-; CHECK-P9-NEXT:    mtvsrd f1, r4
-; CHECK-P9-NEXT:    mtvsrd f3, r6
-; CHECK-P9-NEXT:    mtvsrd f4, r7
-; CHECK-P9-NEXT:    mtvsrd f5, r8
-; CHECK-P9-NEXT:    mtvsrd f7, r10
-; CHECK-P9-NEXT:    xxswapd v4, vs2
-; CHECK-P9-NEXT:    xxswapd v6, vs6
-; CHECK-P9-NEXT:    xxswapd v2, vs0
-; CHECK-P9-NEXT:    xxswapd v3, vs1
-; CHECK-P9-NEXT:    xxswapd v5, vs3
-; CHECK-P9-NEXT:    xxswapd v0, vs4
-; CHECK-P9-NEXT:    xxswapd v1, vs5
-; CHECK-P9-NEXT:    xxswapd v7, vs7
+; CHECK-P9-NEXT:    mtvsrd f2, r3
+; CHECK-P9-NEXT:    xxswapd v2, vs2
+; CHECK-P9-NEXT:    xxswapd vs2, vs1
+; CHECK-P9-NEXT:    xscvspdpn f2, vs2
+; CHECK-P9-NEXT:    xscvdpsxws f2, f2
+; CHECK-P9-NEXT:    mfvsrwz r3, f2
+; CHECK-P9-NEXT:    mtvsrd f2, r3
+; CHECK-P9-NEXT:    xxswapd v3, vs2
+; CHECK-P9-NEXT:    xscvspdpn f2, vs1
+; CHECK-P9-NEXT:    xxsldwi vs1, vs1, vs1, 1
+; CHECK-P9-NEXT:    xscvspdpn f1, vs1
+; CHECK-P9-NEXT:    xscvdpsxws f2, f2
+; CHECK-P9-NEXT:    xscvdpsxws f1, f1
+; CHECK-P9-NEXT:    mfvsrwz r3, f2
+; CHECK-P9-NEXT:    mtvsrd f2, r3
+; CHECK-P9-NEXT:    mfvsrwz r3, f1
+; CHECK-P9-NEXT:    mtvsrd f1, r3
+; CHECK-P9-NEXT:    xxswapd v4, vs1
+; CHECK-P9-NEXT:    xxsldwi vs1, vs0, vs0, 3
+; CHECK-P9-NEXT:    xscvspdpn f1, vs1
+; CHECK-P9-NEXT:    xscvdpsxws f1, f1
 ; CHECK-P9-NEXT:    vmrglh v2, v3, v2
-; CHECK-P9-NEXT:    vmrglh v3, v4, v5
-; CHECK-P9-NEXT:    vmrglh v4, v1, v0
-; CHECK-P9-NEXT:    vmrglh v5, v6, v7
+; CHECK-P9-NEXT:    xxswapd v3, vs2
+; CHECK-P9-NEXT:    vmrglh v3, v3, v4
 ; CHECK-P9-NEXT:    vmrglw v2, v3, v2
-; CHECK-P9-NEXT:    vmrglw v3, v5, v4
+; CHECK-P9-NEXT:    mfvsrwz r3, f1
+; CHECK-P9-NEXT:    mtvsrd f1, r3
+; CHECK-P9-NEXT:    xxswapd v3, vs1
+; CHECK-P9-NEXT:    xxswapd vs1, vs0
+; CHECK-P9-NEXT:    xscvspdpn f1, vs1
+; CHECK-P9-NEXT:    xscvdpsxws f1, f1
+; CHECK-P9-NEXT:    mfvsrwz r3, f1
+; CHECK-P9-NEXT:    mtvsrd f1, r3
+; CHECK-P9-NEXT:    xxswapd v4, vs1
+; CHECK-P9-NEXT:    xscvspdpn f1, vs0
+; CHECK-P9-NEXT:    xxsldwi vs0, vs0, vs0, 1
+; CHECK-P9-NEXT:    xscvspdpn f0, vs0
+; CHECK-P9-NEXT:    xscvdpsxws f1, f1
+; CHECK-P9-NEXT:    xscvdpsxws f0, f0
+; CHECK-P9-NEXT:    mfvsrwz r3, f1
+; CHECK-P9-NEXT:    mtvsrd f1, r3
+; CHECK-P9-NEXT:    mfvsrwz r3, f0
+; CHECK-P9-NEXT:    mtvsrd f0, r3
+; CHECK-P9-NEXT:    vmrglh v3, v4, v3
+; CHECK-P9-NEXT:    xxswapd v4, vs1
+; CHECK-P9-NEXT:    xxswapd v5, vs0
+; CHECK-P9-NEXT:    vmrglh v4, v4, v5
+; CHECK-P9-NEXT:    vmrglw v3, v4, v3
 ; CHECK-P9-NEXT:    xxmrgld v2, v3, v2
 ; CHECK-P9-NEXT:    blr
 ;
 ; CHECK-BE-LABEL: test8elt_signed:
 ; CHECK-BE:       # %bb.0: # %entry
-; CHECK-BE-NEXT:    lxv vs0, 0(r3)
 ; CHECK-BE-NEXT:    lxv vs1, 16(r3)
 ; CHECK-BE-NEXT:    xxsldwi vs2, vs1, vs1, 3
-; CHECK-BE-NEXT:    xxswapd vs3, vs1
-; CHECK-BE-NEXT:    xxsldwi vs4, vs1, vs1, 1
-; CHECK-BE-NEXT:    xxsldwi vs5, vs0, vs0, 3
-; CHECK-BE-NEXT:    xxswapd vs6, vs0
-; CHECK-BE-NEXT:    xxsldwi vs7, vs0, vs0, 1
-; CHECK-BE-NEXT:    xscvspdpn f1, vs1
-; CHECK-BE-NEXT:    xscvspdpn f0, vs0
 ; CHECK-BE-NEXT:    xscvspdpn f2, vs2
-; CHECK-BE-NEXT:    xscvspdpn f3, vs3
-; CHECK-BE-NEXT:    xscvspdpn f4, vs4
-; CHECK-BE-NEXT:    xscvspdpn f5, vs5
-; CHECK-BE-NEXT:    xscvspdpn f6, vs6
-; CHECK-BE-NEXT:    xscvspdpn f7, vs7
-; CHECK-BE-NEXT:    xscvdpsxws f1, f1
-; CHECK-BE-NEXT:    xscvdpsxws f0, f0
 ; CHECK-BE-NEXT:    xscvdpsxws f2, f2
-; CHECK-BE-NEXT:    xscvdpsxws f3, f3
-; CHECK-BE-NEXT:    xscvdpsxws f4, f4
-; CHECK-BE-NEXT:    xscvdpsxws f5, f5
-; CHECK-BE-NEXT:    xscvdpsxws f6, f6
-; CHECK-BE-NEXT:    xscvdpsxws f7, f7
-; CHECK-BE-NEXT:    mfvsrwz r5, f1
-; CHECK-BE-NEXT:    mfvsrwz r9, f0
-; CHECK-BE-NEXT:    sldi r5, r5, 48
-; CHECK-BE-NEXT:    sldi r9, r9, 48
+; CHECK-BE-NEXT:    lxv vs0, 0(r3)
 ; CHECK-BE-NEXT:    mfvsrwz r3, f2
-; CHECK-BE-NEXT:    mfvsrwz r4, f3
-; CHECK-BE-NEXT:    mfvsrwz r6, f4
-; CHECK-BE-NEXT:    mfvsrwz r7, f5
-; CHECK-BE-NEXT:    mfvsrwz r8, f6
-; CHECK-BE-NEXT:    mfvsrwz r10, f7
-; CHECK-BE-NEXT:    mtvsrd v4, r5
-; CHECK-BE-NEXT:    mtvsrd v6, r9
+; CHECK-BE-NEXT:    xxswapd vs2, vs1
 ; CHECK-BE-NEXT:    sldi r3, r3, 48
-; CHECK-BE-NEXT:    sldi r4, r4, 48
-; CHECK-BE-NEXT:    sldi r6, r6, 48
-; CHECK-BE-NEXT:    sldi r7, r7, 48
-; CHECK-BE-NEXT:    sldi r8, r8, 48
-; CHECK-BE-NEXT:    sldi r10, r10, 48
+; CHECK-BE-NEXT:    xscvspdpn f2, vs2
 ; CHECK-BE-NEXT:    mtvsrd v2, r3
-; CHECK-BE-NEXT:    mtvsrd v3, r4
-; CHECK-BE-NEXT:    mtvsrd v5, r6
-; CHECK-BE-NEXT:    mtvsrd v0, r7
-; CHECK-BE-NEXT:    mtvsrd v1, r8
-; CHECK-BE-NEXT:    mtvsrd v7, r10
+; CHECK-BE-NEXT:    xscvdpsxws f2, f2
+; CHECK-BE-NEXT:    mfvsrwz r3, f2
+; CHECK-BE-NEXT:    xscvspdpn f2, vs1
+; CHECK-BE-NEXT:    xxsldwi vs1, vs1, vs1, 1
+; CHECK-BE-NEXT:    sldi r3, r3, 48
+; CHECK-BE-NEXT:    xscvdpsxws f2, f2
+; CHECK-BE-NEXT:    xscvspdpn f1, vs1
+; CHECK-BE-NEXT:    mtvsrd v3, r3
+; CHECK-BE-NEXT:    xscvdpsxws f1, f1
 ; CHECK-BE-NEXT:    vmrghh v2, v3, v2
-; CHECK-BE-NEXT:    vmrghh v3, v4, v5
-; CHECK-BE-NEXT:    vmrghh v4, v1, v0
-; CHECK-BE-NEXT:    vmrghh v5, v6, v7
+; CHECK-BE-NEXT:    mfvsrwz r3, f2
+; CHECK-BE-NEXT:    sldi r3, r3, 48
+; CHECK-BE-NEXT:    mtvsrd v3, r3
+; CHECK-BE-NEXT:    mfvsrwz r3, f1
+; CHECK-BE-NEXT:    xxsldwi vs1, vs0, vs0, 3
+; CHECK-BE-NEXT:    sldi r3, r3, 48
+; CHECK-BE-NEXT:    xscvspdpn f1, vs1
+; CHECK-BE-NEXT:    mtvsrd v4, r3
+; CHECK-BE-NEXT:    xscvdpsxws f1, f1
+; CHECK-BE-NEXT:    mfvsrwz r3, f1
+; CHECK-BE-NEXT:    xxswapd vs1, vs0
+; CHECK-BE-NEXT:    xscvspdpn f1, vs1
+; CHECK-BE-NEXT:    xscvdpsxws f1, f1
+; CHECK-BE-NEXT:    vmrghh v3, v3, v4
+; CHECK-BE-NEXT:    sldi r3, r3, 48
 ; CHECK-BE-NEXT:    vmrghw v2, v3, v2
-; CHECK-BE-NEXT:    vmrghw v3, v5, v4
+; CHECK-BE-NEXT:    mtvsrd v3, r3
+; CHECK-BE-NEXT:    mfvsrwz r3, f1
+; CHECK-BE-NEXT:    xscvspdpn f1, vs0
+; CHECK-BE-NEXT:    xxsldwi vs0, vs0, vs0, 1
+; CHECK-BE-NEXT:    sldi r3, r3, 48
+; CHECK-BE-NEXT:    xscvdpsxws f1, f1
+; CHECK-BE-NEXT:    xscvspdpn f0, vs0
+; CHECK-BE-NEXT:    mtvsrd v4, r3
+; CHECK-BE-NEXT:    xscvdpsxws f0, f0
+; CHECK-BE-NEXT:    vmrghh v3, v4, v3
+; CHECK-BE-NEXT:    mfvsrwz r3, f1
+; CHECK-BE-NEXT:    sldi r3, r3, 48
+; CHECK-BE-NEXT:    mtvsrd v4, r3
+; CHECK-BE-NEXT:    mfvsrwz r3, f0
+; CHECK-BE-NEXT:    sldi r3, r3, 48
+; CHECK-BE-NEXT:    mtvsrd v5, r3
+; CHECK-BE-NEXT:    vmrghh v4, v4, v5
+; CHECK-BE-NEXT:    vmrghw v3, v4, v3
 ; CHECK-BE-NEXT:    xxmrghd v2, v3, v2
 ; CHECK-BE-NEXT:    blr
 entry:
@@ -1213,258 +1189,234 @@ define void @test16elt_signed(<16 x i16>* noalias nocapture sret %agg.result, <1
 ;
 ; CHECK-P9-LABEL: test16elt_signed:
 ; CHECK-P9:       # %bb.0: # %entry
-; CHECK-P9-NEXT:    lxv vs2, 16(r4)
-; CHECK-P9-NEXT:    lxv vs3, 0(r4)
-; CHECK-P9-NEXT:    lxv vs0, 48(r4)
-; CHECK-P9-NEXT:    lxv vs1, 32(r4)
-; CHECK-P9-NEXT:    std r25, -56(r1) # 8-byte Folded Spill
-; CHECK-P9-NEXT:    std r26, -48(r1) # 8-byte Folded Spill
-; CHECK-P9-NEXT:    std r27, -40(r1) # 8-byte Folded Spill
-; CHECK-P9-NEXT:    std r28, -32(r1) # 8-byte Folded Spill
-; CHECK-P9-NEXT:    std r29, -24(r1) # 8-byte Folded Spill
-; CHECK-P9-NEXT:    std r30, -16(r1) # 8-byte Folded Spill
-; CHECK-P9-NEXT:    xxsldwi vs4, vs3, vs3, 3
-; CHECK-P9-NEXT:    xxswapd vs5, vs3
-; CHECK-P9-NEXT:    xxsldwi vs6, vs3, vs3, 1
-; CHECK-P9-NEXT:    xxsldwi vs7, vs2, vs2, 3
-; CHECK-P9-NEXT:    xxswapd vs8, vs2
-; CHECK-P9-NEXT:    xxsldwi vs9, vs2, vs2, 1
-; CHECK-P9-NEXT:    xxsldwi vs10, vs1, vs1, 3
-; CHECK-P9-NEXT:    xxswapd vs11, vs1
-; CHECK-P9-NEXT:    xxsldwi vs12, vs1, vs1, 1
-; CHECK-P9-NEXT:    xxsldwi vs13, vs0, vs0, 3
-; CHECK-P9-NEXT:    xxswapd v2, vs0
-; CHECK-P9-NEXT:    xxsldwi v3, vs0, vs0, 1
-; CHECK-P9-NEXT:    xscvspdpn f3, vs3
-; CHECK-P9-NEXT:    xscvspdpn f2, vs2
-; CHECK-P9-NEXT:    xscvspdpn f1, vs1
-; CHECK-P9-NEXT:    xscvspdpn f0, vs0
+; CHECK-P9-NEXT:    lxv vs1, 0(r4)
+; CHECK-P9-NEXT:    lxv vs3, 16(r4)
+; CHECK-P9-NEXT:    xscvspdpn f5, vs1
+; CHECK-P9-NEXT:    xxsldwi vs2, vs1, vs1, 3
+; CHECK-P9-NEXT:    xscvspdpn f8, vs3
+; CHECK-P9-NEXT:    xxswapd vs4, vs1
+; CHECK-P9-NEXT:    xxsldwi vs1, vs1, vs1, 1
 ; CHECK-P9-NEXT:    xscvspdpn f4, vs4
-; CHECK-P9-NEXT:    xscvspdpn f5, vs5
+; CHECK-P9-NEXT:    xscvdpsxws f5, f5
+; CHECK-P9-NEXT:    xscvspdpn f2, vs2
+; CHECK-P9-NEXT:    xscvdpsxws f8, f8
+; CHECK-P9-NEXT:    xxsldwi vs6, vs3, vs3, 3
+; CHECK-P9-NEXT:    xxswapd vs7, vs3
 ; CHECK-P9-NEXT:    xscvspdpn f6, vs6
+; CHECK-P9-NEXT:    xxsldwi vs3, vs3, vs3, 1
 ; CHECK-P9-NEXT:    xscvspdpn f7, vs7
-; CHECK-P9-NEXT:    xscvspdpn f8, vs8
-; CHECK-P9-NEXT:    xscvspdpn f9, vs9
-; CHECK-P9-NEXT:    xscvspdpn f10, vs10
-; CHECK-P9-NEXT:    xscvspdpn f11, vs11
-; CHECK-P9-NEXT:    xscvspdpn f12, vs12
-; CHECK-P9-NEXT:    xscvspdpn f13, vs13
-; CHECK-P9-NEXT:    xscvspdpn v2, v2
-; CHECK-P9-NEXT:    xscvspdpn v3, v3
-; CHECK-P9-NEXT:    xscvdpsxws f3, f3
+; CHECK-P9-NEXT:    xscvspdpn f3, vs3
 ; CHECK-P9-NEXT:    xscvdpsxws f2, f2
-; CHECK-P9-NEXT:    xscvdpsxws f1, f1
-; CHECK-P9-NEXT:    xscvdpsxws f0, f0
+; CHECK-P9-NEXT:    xscvspdpn f1, vs1
 ; CHECK-P9-NEXT:    xscvdpsxws f4, f4
-; CHECK-P9-NEXT:    xscvdpsxws f5, f5
 ; CHECK-P9-NEXT:    xscvdpsxws f6, f6
+; CHECK-P9-NEXT:    mfvsrwz r5, f5
+; CHECK-P9-NEXT:    xscvdpsxws f1, f1
 ; CHECK-P9-NEXT:    xscvdpsxws f7, f7
-; CHECK-P9-NEXT:    xscvdpsxws f8, f8
+; CHECK-P9-NEXT:    xscvdpsxws f3, f3
+; CHECK-P9-NEXT:    mtvsrd f5, r5
+; CHECK-P9-NEXT:    mfvsrwz r5, f8
+; CHECK-P9-NEXT:    mtvsrd f8, r5
+; CHECK-P9-NEXT:    mfvsrwz r5, f2
+; CHECK-P9-NEXT:    lxv vs0, 32(r4)
+; CHECK-P9-NEXT:    xxsldwi vs9, vs0, vs0, 3
+; CHECK-P9-NEXT:    xxswapd vs10, vs0
+; CHECK-P9-NEXT:    xscvspdpn f9, vs9
+; CHECK-P9-NEXT:    xscvspdpn f10, vs10
 ; CHECK-P9-NEXT:    xscvdpsxws f9, f9
 ; CHECK-P9-NEXT:    xscvdpsxws f10, f10
-; CHECK-P9-NEXT:    xscvdpsxws f11, f11
-; CHECK-P9-NEXT:    xscvdpsxws f12, f12
-; CHECK-P9-NEXT:    xscvdpsxws f13, f13
-; CHECK-P9-NEXT:    xscvdpsxws v2, v2
-; CHECK-P9-NEXT:    xscvdpsxws v3, v3
-; CHECK-P9-NEXT:    mfvsrwz r4, f3
-; CHECK-P9-NEXT:    mfvsrwz r5, f2
-; CHECK-P9-NEXT:    mfvsrwz r12, f1
-; CHECK-P9-NEXT:    mfvsrwz r0, f0
-; CHECK-P9-NEXT:    mfvsrwz r6, f4
-; CHECK-P9-NEXT:    mfvsrwz r7, f5
-; CHECK-P9-NEXT:    mfvsrwz r8, f6
-; CHECK-P9-NEXT:    mfvsrwz r9, f7
-; CHECK-P9-NEXT:    mfvsrwz r10, f8
-; CHECK-P9-NEXT:    mfvsrwz r11, f9
-; CHECK-P9-NEXT:    mfvsrwz r30, f10
-; CHECK-P9-NEXT:    mfvsrwz r29, f11
-; CHECK-P9-NEXT:    mfvsrwz r28, f12
-; CHECK-P9-NEXT:    mfvsrwz r27, f13
-; CHECK-P9-NEXT:    mfvsrwz r26, v2
-; CHECK-P9-NEXT:    mfvsrwz r25, v3
-; CHECK-P9-NEXT:    mtvsrd f0, r4
+; CHECK-P9-NEXT:    mtvsrd f2, r5
+; CHECK-P9-NEXT:    mfvsrwz r5, f4
+; CHECK-P9-NEXT:    mtvsrd f4, r5
+; CHECK-P9-NEXT:    mfvsrwz r5, f1
 ; CHECK-P9-NEXT:    mtvsrd f1, r5
-; CHECK-P9-NEXT:    mtvsrd f8, r12
-; CHECK-P9-NEXT:    mtvsrd f9, r0
-; CHECK-P9-NEXT:    mtvsrd f2, r6
-; CHECK-P9-NEXT:    mtvsrd f3, r7
-; CHECK-P9-NEXT:    mtvsrd f4, r8
-; CHECK-P9-NEXT:    mtvsrd f5, r9
-; CHECK-P9-NEXT:    mtvsrd f6, r10
-; CHECK-P9-NEXT:    mtvsrd f7, r11
-; CHECK-P9-NEXT:    mtvsrd f10, r30
-; CHECK-P9-NEXT:    ld r30, -16(r1) # 8-byte Folded Reload
-; CHECK-P9-NEXT:    mtvsrd f11, r29
-; CHECK-P9-NEXT:    ld r29, -24(r1) # 8-byte Folded Reload
-; CHECK-P9-NEXT:    mtvsrd f12, r28
-; CHECK-P9-NEXT:    ld r28, -32(r1) # 8-byte Folded Reload
-; CHECK-P9-NEXT:    mtvsrd f13, r27
-; CHECK-P9-NEXT:    ld r27, -40(r1) # 8-byte Folded Reload
-; CHECK-P9-NEXT:    mtvsrd v2, r26
-; CHECK-P9-NEXT:    ld r26, -48(r1) # 8-byte Folded Reload
-; CHECK-P9-NEXT:    mtvsrd v3, r25
-; CHECK-P9-NEXT:    ld r25, -56(r1) # 8-byte Folded Reload
-; CHECK-P9-NEXT:    xxswapd v4, vs0
-; CHECK-P9-NEXT:    xxswapd v5, vs2
+; CHECK-P9-NEXT:    mfvsrwz r5, f6
+; CHECK-P9-NEXT:    xxswapd v2, vs2
+; CHECK-P9-NEXT:    xxswapd v3, vs4
+; CHECK-P9-NEXT:    xscvspdpn f2, vs0
+; CHECK-P9-NEXT:    xxsldwi vs0, vs0, vs0, 1
+; CHECK-P9-NEXT:    xscvspdpn f0, vs0
+; CHECK-P9-NEXT:    xscvdpsxws f2, f2
+; CHECK-P9-NEXT:    mtvsrd f6, r5
+; CHECK-P9-NEXT:    mfvsrwz r5, f7
+; CHECK-P9-NEXT:    xxswapd v4, vs1
+; CHECK-P9-NEXT:    lxv vs1, 48(r4)
+; CHECK-P9-NEXT:    vmrglh v2, v3, v2
+; CHECK-P9-NEXT:    xxswapd v3, vs5
+; CHECK-P9-NEXT:    mtvsrd f7, r5
+; CHECK-P9-NEXT:    mfvsrwz r5, f3
+; CHECK-P9-NEXT:    vmrglh v3, v3, v4
+; CHECK-P9-NEXT:    xxswapd v4, vs6
+; CHECK-P9-NEXT:    xxswapd v5, vs7
+; CHECK-P9-NEXT:    mtvsrd f3, r5
+; CHECK-P9-NEXT:    xscvdpsxws f0, f0
 ; CHECK-P9-NEXT:    xxswapd v0, vs3
-; CHECK-P9-NEXT:    xxswapd v1, vs4
-; CHECK-P9-NEXT:    xxswapd v6, vs5
-; CHECK-P9-NEXT:    xxswapd v7, vs6
-; CHECK-P9-NEXT:    xxswapd v8, vs1
-; CHECK-P9-NEXT:    xxswapd v9, vs7
-; CHECK-P9-NEXT:    xxswapd v10, vs10
-; CHECK-P9-NEXT:    xxswapd v11, vs11
-; CHECK-P9-NEXT:    xxswapd v12, vs8
-; CHECK-P9-NEXT:    xxswapd v13, vs12
-; CHECK-P9-NEXT:    xxswapd v14, vs13
-; CHECK-P9-NEXT:    xxswapd v2, v2
-; CHECK-P9-NEXT:    xxswapd v15, vs9
-; CHECK-P9-NEXT:    xxswapd v3, v3
-; CHECK-P9-NEXT:    vmrglh v5, v0, v5
-; CHECK-P9-NEXT:    vmrglh v4, v4, v1
-; CHECK-P9-NEXT:    vmrglh v0, v7, v6
-; CHECK-P9-NEXT:    vmrglh v1, v8, v9
-; CHECK-P9-NEXT:    vmrglh v6, v11, v10
-; CHECK-P9-NEXT:    vmrglh v7, v12, v13
-; CHECK-P9-NEXT:    vmrglh v2, v2, v14
-; CHECK-P9-NEXT:    vmrglh v3, v15, v3
-; CHECK-P9-NEXT:    vmrglw v4, v4, v5
-; CHECK-P9-NEXT:    vmrglw v5, v1, v0
-; CHECK-P9-NEXT:    vmrglw v0, v7, v6
+; CHECK-P9-NEXT:    vmrglh v4, v5, v4
+; CHECK-P9-NEXT:    xxswapd v5, vs8
+; CHECK-P9-NEXT:    vmrglh v5, v5, v0
+; CHECK-P9-NEXT:    mfvsrwz r4, f2
+; CHECK-P9-NEXT:    mtvsrd f2, r4
+; CHECK-P9-NEXT:    mfvsrwz r4, f0
 ; CHECK-P9-NEXT:    vmrglw v2, v3, v2
-; CHECK-P9-NEXT:    xxmrgld vs0, v5, v4
-; CHECK-P9-NEXT:    xxmrgld vs1, v2, v0
-; CHECK-P9-NEXT:    stxv vs0, 0(r3)
-; CHECK-P9-NEXT:    stxv vs1, 16(r3)
+; CHECK-P9-NEXT:    mtvsrd f0, r4
+; CHECK-P9-NEXT:    vmrglw v3, v5, v4
+; CHECK-P9-NEXT:    xxswapd v4, vs2
+; CHECK-P9-NEXT:    xxmrgld vs2, v3, v2
+; CHECK-P9-NEXT:    xxswapd v2, vs0
+; CHECK-P9-NEXT:    xxsldwi vs0, vs1, vs1, 3
+; CHECK-P9-NEXT:    xscvspdpn f0, vs0
+; CHECK-P9-NEXT:    xscvdpsxws f0, f0
+; CHECK-P9-NEXT:    mfvsrwz r4, f0
+; CHECK-P9-NEXT:    mtvsrd f0, r4
+; CHECK-P9-NEXT:    xxswapd v3, vs0
+; CHECK-P9-NEXT:    xxswapd vs0, vs1
+; CHECK-P9-NEXT:    xscvspdpn f0, vs0
+; CHECK-P9-NEXT:    xscvdpsxws f0, f0
+; CHECK-P9-NEXT:    mfvsrwz r4, f0
+; CHECK-P9-NEXT:    mtvsrd f0, r4
+; CHECK-P9-NEXT:    vmrglh v2, v4, v2
+; CHECK-P9-NEXT:    xxswapd v4, vs0
+; CHECK-P9-NEXT:    xscvspdpn f0, vs1
+; CHECK-P9-NEXT:    xscvdpsxws f0, f0
+; CHECK-P9-NEXT:    mfvsrwz r4, f0
+; CHECK-P9-NEXT:    mtvsrd f0, r4
+; CHECK-P9-NEXT:    vmrglh v3, v4, v3
+; CHECK-P9-NEXT:    xxswapd v4, vs0
+; CHECK-P9-NEXT:    xxsldwi vs0, vs1, vs1, 1
+; CHECK-P9-NEXT:    xscvspdpn f0, vs0
+; CHECK-P9-NEXT:    xscvdpsxws f0, f0
+; CHECK-P9-NEXT:    mfvsrwz r5, f9
+; CHECK-P9-NEXT:    mtvsrd f9, r5
+; CHECK-P9-NEXT:    mfvsrwz r5, f10
+; CHECK-P9-NEXT:    mtvsrd f10, r5
+; CHECK-P9-NEXT:    xxswapd v0, vs9
+; CHECK-P9-NEXT:    xxswapd v1, vs10
+; CHECK-P9-NEXT:    vmrglh v0, v1, v0
+; CHECK-P9-NEXT:    vmrglw v2, v2, v0
+; CHECK-P9-NEXT:    stxv vs2, 0(r3)
+; CHECK-P9-NEXT:    mfvsrwz r4, f0
+; CHECK-P9-NEXT:    mtvsrd f0, r4
+; CHECK-P9-NEXT:    xxswapd v5, vs0
+; CHECK-P9-NEXT:    vmrglh v4, v4, v5
+; CHECK-P9-NEXT:    vmrglw v3, v4, v3
+; CHECK-P9-NEXT:    xxmrgld vs0, v3, v2
+; CHECK-P9-NEXT:    stxv vs0, 16(r3)
 ; CHECK-P9-NEXT:    blr
 ;
 ; CHECK-BE-LABEL: test16elt_signed:
 ; CHECK-BE:       # %bb.0: # %entry
-; CHECK-BE-NEXT:    lxv vs2, 0(r4)
-; CHECK-BE-NEXT:    lxv vs3, 16(r4)
-; CHECK-BE-NEXT:    lxv vs0, 32(r4)
-; CHECK-BE-NEXT:    lxv vs1, 48(r4)
-; CHECK-BE-NEXT:    std r25, -56(r1) # 8-byte Folded Spill
-; CHECK-BE-NEXT:    std r26, -48(r1) # 8-byte Folded Spill
-; CHECK-BE-NEXT:    std r27, -40(r1) # 8-byte Folded Spill
-; CHECK-BE-NEXT:    std r28, -32(r1) # 8-byte Folded Spill
-; CHECK-BE-NEXT:    std r29, -24(r1) # 8-byte Folded Spill
-; CHECK-BE-NEXT:    std r30, -16(r1) # 8-byte Folded Spill
-; CHECK-BE-NEXT:    xxsldwi vs4, vs3, vs3, 3
-; CHECK-BE-NEXT:    xxswapd vs5, vs3
-; CHECK-BE-NEXT:    xxsldwi vs6, vs3, vs3, 1
-; CHECK-BE-NEXT:    xxsldwi vs7, vs2, vs2, 3
-; CHECK-BE-NEXT:    xxswapd vs8, vs2
-; CHECK-BE-NEXT:    xxsldwi vs9, vs2, vs2, 1
-; CHECK-BE-NEXT:    xxsldwi vs10, vs1, vs1, 3
-; CHECK-BE-NEXT:    xxswapd vs11, vs1
-; CHECK-BE-NEXT:    xxsldwi vs12, vs1, vs1, 1
-; CHECK-BE-NEXT:    xxsldwi vs13, vs0, vs0, 3
-; CHECK-BE-NEXT:    xxswapd v2, vs0
-; CHECK-BE-NEXT:    xxsldwi v3, vs0, vs0, 1
-; CHECK-BE-NEXT:    xscvspdpn f3, vs3
+; CHECK-BE-NEXT:    lxv vs1, 16(r4)
+; CHECK-BE-NEXT:    xxsldwi vs2, vs1, vs1, 3
 ; CHECK-BE-NEXT:    xscvspdpn f2, vs2
-; CHECK-BE-NEXT:    xscvspdpn f1, vs1
-; CHECK-BE-NEXT:    xscvspdpn f0, vs0
-; CHECK-BE-NEXT:    xscvspdpn f4, vs4
-; CHECK-BE-NEXT:    xscvspdpn f5, vs5
-; CHECK-BE-NEXT:    xscvspdpn f6, vs6
-; CHECK-BE-NEXT:    xscvspdpn f7, vs7
-; CHECK-BE-NEXT:    xscvspdpn f8, vs8
-; CHECK-BE-NEXT:    xscvspdpn f9, vs9
-; CHECK-BE-NEXT:    xscvspdpn f10, vs10
-; CHECK-BE-NEXT:    xscvspdpn f11, vs11
-; CHECK-BE-NEXT:    xscvspdpn f12, vs12
-; CHECK-BE-NEXT:    xscvspdpn f13, vs13
-; CHECK-BE-NEXT:    xscvspdpn v2, v2
-; CHECK-BE-NEXT:    xscvspdpn v3, v3
+; CHECK-BE-NEXT:    xxswapd vs3, vs1
+; CHECK-BE-NEXT:    xscvspdpn f3, vs3
+; CHECK-BE-NEXT:    xscvdpsxws f2, f2
 ; CHECK-BE-NEXT:    xscvdpsxws f3, f3
+; CHECK-BE-NEXT:    mfvsrwz r5, f2
+; CHECK-BE-NEXT:    xscvspdpn f4, vs1
+; CHECK-BE-NEXT:    xxsldwi vs1, vs1, vs1, 1
+; CHECK-BE-NEXT:    xscvspdpn f1, vs1
+; CHECK-BE-NEXT:    xscvdpsxws f1, f1
+; CHECK-BE-NEXT:    sldi r5, r5, 48
+; CHECK-BE-NEXT:    mtvsrd v2, r5
+; CHECK-BE-NEXT:    mfvsrwz r5, f3
+; CHECK-BE-NEXT:    xscvdpsxws f3, f4
+; CHECK-BE-NEXT:    lxv vs0, 0(r4)
+; CHECK-BE-NEXT:    xxsldwi vs2, vs0, vs0, 3
+; CHECK-BE-NEXT:    xscvspdpn f2, vs2
+; CHECK-BE-NEXT:    sldi r5, r5, 48
+; CHECK-BE-NEXT:    mtvsrd v3, r5
+; CHECK-BE-NEXT:    vmrghh v2, v3, v2
+; CHECK-BE-NEXT:    mfvsrwz r5, f3
+; CHECK-BE-NEXT:    sldi r5, r5, 48
+; CHECK-BE-NEXT:    mtvsrd v3, r5
+; CHECK-BE-NEXT:    mfvsrwz r5, f1
+; CHECK-BE-NEXT:    xxswapd vs1, vs0
 ; CHECK-BE-NEXT:    xscvdpsxws f2, f2
+; CHECK-BE-NEXT:    sldi r5, r5, 48
+; CHECK-BE-NEXT:    xscvspdpn f1, vs1
+; CHECK-BE-NEXT:    mtvsrd v4, r5
+; CHECK-BE-NEXT:    mfvsrwz r5, f2
 ; CHECK-BE-NEXT:    xscvdpsxws f1, f1
+; CHECK-BE-NEXT:    sldi r5, r5, 48
+; CHECK-BE-NEXT:    vmrghh v3, v3, v4
+; CHECK-BE-NEXT:    mtvsrd v4, r5
+; CHECK-BE-NEXT:    vmrghw v2, v3, v2
+; CHECK-BE-NEXT:    mfvsrwz r5, f1
+; CHECK-BE-NEXT:    xscvspdpn f1, vs0
+; CHECK-BE-NEXT:    xxsldwi vs0, vs0, vs0, 1
+; CHECK-BE-NEXT:    sldi r5, r5, 48
+; CHECK-BE-NEXT:    xscvdpsxws f1, f1
+; CHECK-BE-NEXT:    xscvspdpn f0, vs0
+; CHECK-BE-NEXT:    mtvsrd v5, r5
 ; CHECK-BE-NEXT:    xscvdpsxws f0, f0
-; CHECK-BE-NEXT:    xscvdpsxws f4, f4
+; CHECK-BE-NEXT:    vmrghh v4, v5, v4
+; CHECK-BE-NEXT:    mfvsrwz r5, f1
+; CHECK-BE-NEXT:    lxv vs1, 48(r4)
+; CHECK-BE-NEXT:    sldi r5, r5, 48
+; CHECK-BE-NEXT:    mtvsrd v5, r5
+; CHECK-BE-NEXT:    mfvsrwz r5, f0
+; CHECK-BE-NEXT:    lxv vs0, 32(r4)
+; CHECK-BE-NEXT:    xscvspdpn f5, vs1
+; CHECK-BE-NEXT:    xxsldwi vs2, vs1, vs1, 3
+; CHECK-BE-NEXT:    xscvspdpn f2, vs2
 ; CHECK-BE-NEXT:    xscvdpsxws f5, f5
-; CHECK-BE-NEXT:    xscvdpsxws f6, f6
-; CHECK-BE-NEXT:    xscvdpsxws f7, f7
-; CHECK-BE-NEXT:    xscvdpsxws f8, f8
-; CHECK-BE-NEXT:    xscvdpsxws f9, f9
-; CHECK-BE-NEXT:    xscvdpsxws f10, f10
-; CHECK-BE-NEXT:    xscvdpsxws f11, f11
-; CHECK-BE-NEXT:    xscvdpsxws f12, f12
-; CHECK-BE-NEXT:    xscvdpsxws f13, f13
-; CHECK-BE-NEXT:    xscvdpsxws v2, v2
-; CHECK-BE-NEXT:    xscvdpsxws v3, v3
-; CHECK-BE-NEXT:    mfvsrwz r4, f3
-; CHECK-BE-NEXT:    mfvsrwz r5, f2
-; CHECK-BE-NEXT:    mfvsrwz r12, f1
-; CHECK-BE-NEXT:    mfvsrwz r0, f0
-; CHECK-BE-NEXT:    mfvsrwz r6, f4
-; CHECK-BE-NEXT:    mfvsrwz r7, f5
-; CHECK-BE-NEXT:    mfvsrwz r8, f6
-; CHECK-BE-NEXT:    mfvsrwz r9, f7
-; CHECK-BE-NEXT:    mfvsrwz r10, f8
-; CHECK-BE-NEXT:    mfvsrwz r11, f9
-; CHECK-BE-NEXT:    mfvsrwz r30, f10
-; CHECK-BE-NEXT:    mfvsrwz r29, f11
-; CHECK-BE-NEXT:    mfvsrwz r28, f12
-; CHECK-BE-NEXT:    mfvsrwz r27, f13
-; CHECK-BE-NEXT:    mfvsrwz r26, v2
-; CHECK-BE-NEXT:    mfvsrwz r25, v3
-; CHECK-BE-NEXT:    sldi r4, r4, 48
 ; CHECK-BE-NEXT:    sldi r5, r5, 48
-; CHECK-BE-NEXT:    sldi r12, r12, 48
-; CHECK-BE-NEXT:    sldi r0, r0, 48
-; CHECK-BE-NEXT:    sldi r6, r6, 48
-; CHECK-BE-NEXT:    sldi r7, r7, 48
-; CHECK-BE-NEXT:    sldi r8, r8, 48
-; CHECK-BE-NEXT:    sldi r9, r9, 48
-; CHECK-BE-NEXT:    sldi r10, r10, 48
-; CHECK-BE-NEXT:    sldi r11, r11, 48
-; CHECK-BE-NEXT:    sldi r30, r30, 48
-; CHECK-BE-NEXT:    sldi r29, r29, 48
-; CHECK-BE-NEXT:    sldi r28, r28, 48
-; CHECK-BE-NEXT:    sldi r27, r27, 48
-; CHECK-BE-NEXT:    sldi r26, r26, 48
-; CHECK-BE-NEXT:    sldi r25, r25, 48
+; CHECK-BE-NEXT:    xxswapd vs3, vs1
+; CHECK-BE-NEXT:    mtvsrd v0, r5
+; CHECK-BE-NEXT:    vmrghh v5, v5, v0
+; CHECK-BE-NEXT:    xscvspdpn f3, vs3
+; CHECK-BE-NEXT:    xxsldwi vs1, vs1, vs1, 1
+; CHECK-BE-NEXT:    xscvspdpn f1, vs1
+; CHECK-BE-NEXT:    xscvdpsxws f2, f2
+; CHECK-BE-NEXT:    vmrghw v3, v5, v4
+; CHECK-BE-NEXT:    xscvdpsxws f3, f3
+; CHECK-BE-NEXT:    mfvsrwz r4, f5
+; CHECK-BE-NEXT:    xxmrghd vs4, v3, v2
+; CHECK-BE-NEXT:    sldi r4, r4, 48
 ; CHECK-BE-NEXT:    mtvsrd v2, r4
-; CHECK-BE-NEXT:    mtvsrd v3, r5
-; CHECK-BE-NEXT:    mtvsrd v10, r12
-; CHECK-BE-NEXT:    mtvsrd v14, r0
-; CHECK-BE-NEXT:    mtvsrd v4, r6
-; CHECK-BE-NEXT:    mtvsrd v5, r7
-; CHECK-BE-NEXT:    mtvsrd v0, r8
-; CHECK-BE-NEXT:    mtvsrd v1, r9
-; CHECK-BE-NEXT:    mtvsrd v6, r10
-; CHECK-BE-NEXT:    mtvsrd v7, r11
-; CHECK-BE-NEXT:    mtvsrd v8, r30
-; CHECK-BE-NEXT:    ld r30, -16(r1) # 8-byte Folded Reload
-; CHECK-BE-NEXT:    mtvsrd v9, r29
-; CHECK-BE-NEXT:    ld r29, -24(r1) # 8-byte Folded Reload
-; CHECK-BE-NEXT:    mtvsrd v11, r28
-; CHECK-BE-NEXT:    ld r28, -32(r1) # 8-byte Folded Reload
-; CHECK-BE-NEXT:    mtvsrd v12, r27
-; CHECK-BE-NEXT:    ld r27, -40(r1) # 8-byte Folded Reload
-; CHECK-BE-NEXT:    mtvsrd v13, r26
-; CHECK-BE-NEXT:    ld r26, -48(r1) # 8-byte Folded Reload
-; CHECK-BE-NEXT:    mtvsrd v15, r25
-; CHECK-BE-NEXT:    ld r25, -56(r1) # 8-byte Folded Reload
-; CHECK-BE-NEXT:    vmrghh v4, v5, v4
-; CHECK-BE-NEXT:    vmrghh v2, v2, v0
-; CHECK-BE-NEXT:    vmrghh v5, v6, v1
-; CHECK-BE-NEXT:    vmrghh v3, v3, v7
-; CHECK-BE-NEXT:    vmrghh v0, v9, v8
-; CHECK-BE-NEXT:    vmrghh v1, v10, v11
-; CHECK-BE-NEXT:    vmrghh v6, v13, v12
-; CHECK-BE-NEXT:    vmrghh v7, v14, v15
-; CHECK-BE-NEXT:    vmrghw v2, v2, v4
-; CHECK-BE-NEXT:    vmrghw v3, v3, v5
-; CHECK-BE-NEXT:    vmrghw v4, v1, v0
-; CHECK-BE-NEXT:    vmrghw v5, v7, v6
+; CHECK-BE-NEXT:    mfvsrwz r4, f2
+; CHECK-BE-NEXT:    xscvdpsxws f1, f1
+; CHECK-BE-NEXT:    stxv vs4, 0(r3)
+; CHECK-BE-NEXT:    sldi r4, r4, 48
+; CHECK-BE-NEXT:    mtvsrd v3, r4
+; CHECK-BE-NEXT:    mfvsrwz r4, f3
+; CHECK-BE-NEXT:    sldi r4, r4, 48
+; CHECK-BE-NEXT:    mtvsrd v4, r4
+; CHECK-BE-NEXT:    mfvsrwz r4, f1
+; CHECK-BE-NEXT:    xxsldwi vs1, vs0, vs0, 3
+; CHECK-BE-NEXT:    sldi r4, r4, 48
+; CHECK-BE-NEXT:    xscvspdpn f1, vs1
+; CHECK-BE-NEXT:    xscvdpsxws f1, f1
+; CHECK-BE-NEXT:    vmrghh v3, v4, v3
+; CHECK-BE-NEXT:    mtvsrd v4, r4
+; CHECK-BE-NEXT:    mfvsrwz r4, f1
+; CHECK-BE-NEXT:    xxswapd vs1, vs0
+; CHECK-BE-NEXT:    xscvspdpn f1, vs1
+; CHECK-BE-NEXT:    xscvdpsxws f1, f1
+; CHECK-BE-NEXT:    vmrghh v2, v2, v4
+; CHECK-BE-NEXT:    sldi r4, r4, 48
+; CHECK-BE-NEXT:    vmrghw v2, v2, v3
+; CHECK-BE-NEXT:    mtvsrd v3, r4
+; CHECK-BE-NEXT:    mfvsrwz r4, f1
+; CHECK-BE-NEXT:    xscvspdpn f1, vs0
+; CHECK-BE-NEXT:    xxsldwi vs0, vs0, vs0, 1
+; CHECK-BE-NEXT:    sldi r4, r4, 48
+; CHECK-BE-NEXT:    xscvdpsxws f1, f1
+; CHECK-BE-NEXT:    xscvspdpn f0, vs0
+; CHECK-BE-NEXT:    mtvsrd v4, r4
+; CHECK-BE-NEXT:    xscvdpsxws f0, f0
+; CHECK-BE-NEXT:    vmrghh v3, v4, v3
+; CHECK-BE-NEXT:    mfvsrwz r4, f1
+; CHECK-BE-NEXT:    sldi r4, r4, 48
+; CHECK-BE-NEXT:    mtvsrd v4, r4
+; CHECK-BE-NEXT:    mfvsrwz r4, f0
+; CHECK-BE-NEXT:    sldi r4, r4, 48
+; CHECK-BE-NEXT:    mtvsrd v5, r4
+; CHECK-BE-NEXT:    vmrghh v4, v4, v5
+; CHECK-BE-NEXT:    vmrghw v3, v4, v3
 ; CHECK-BE-NEXT:    xxmrghd vs0, v3, v2
-; CHECK-BE-NEXT:    xxmrghd vs1, v5, v4
-; CHECK-BE-NEXT:    stxv vs0, 0(r3)
-; CHECK-BE-NEXT:    stxv vs1, 16(r3)
+; CHECK-BE-NEXT:    stxv vs0, 16(r3)
 ; CHECK-BE-NEXT:    blr
 entry:
   %a = load <16 x float>, <16 x float>* %0, align 64
diff --git a/test/CodeGen/PowerPC/vec_conv_fp32_to_i64_elts.ll b/test/CodeGen/PowerPC/vec_conv_fp32_to_i64_elts.ll
index 3a733f0e934a000b30318eeedb39cebfe329c91f..6b945d4688062d5a49be9fe98561e92b37b3d14c 100644
--- a/test/CodeGen/PowerPC/vec_conv_fp32_to_i64_elts.ll
+++ b/test/CodeGen/PowerPC/vec_conv_fp32_to_i64_elts.ll
@@ -35,10 +35,10 @@ define <2 x i64> @test2elt(i64 %a.coerce) local_unnamed_addr #0 {
 ; CHECK-BE-LABEL: test2elt:
 ; CHECK-BE:       # %bb.0: # %entry
 ; CHECK-BE-NEXT:    mtvsrd f0, r3
-; CHECK-BE-NEXT:    xxsldwi vs1, vs0, vs0, 1
+; CHECK-BE-NEXT:    xscvspdpn f1, vs0
+; CHECK-BE-NEXT:    xxsldwi vs0, vs0, vs0, 1
 ; CHECK-BE-NEXT:    xscvspdpn f0, vs0
-; CHECK-BE-NEXT:    xscvspdpn f1, vs1
-; CHECK-BE-NEXT:    xxmrghd vs0, vs0, vs1
+; CHECK-BE-NEXT:    xxmrghd vs0, vs1, vs0
 ; CHECK-BE-NEXT:    xvcvdpuxds v2, vs0
 ; CHECK-BE-NEXT:    blr
 entry:
@@ -72,34 +72,34 @@ define void @test4elt(<4 x i64>* noalias nocapture sret %agg.result, <4 x float>
 ; CHECK-P9:       # %bb.0: # %entry
 ; CHECK-P9-NEXT:    xxsldwi vs0, v2, v2, 3
 ; CHECK-P9-NEXT:    xxswapd vs1, v2
-; CHECK-P9-NEXT:    xxsldwi vs2, v2, v2, 1
-; CHECK-P9-NEXT:    xscvspdpn f3, v2
 ; CHECK-P9-NEXT:    xscvspdpn f0, vs0
 ; CHECK-P9-NEXT:    xscvspdpn f1, vs1
+; CHECK-P9-NEXT:    xxsldwi vs2, v2, v2, 1
 ; CHECK-P9-NEXT:    xscvspdpn f2, vs2
 ; CHECK-P9-NEXT:    xxmrghd vs0, vs1, vs0
-; CHECK-P9-NEXT:    xxmrghd vs1, vs3, vs2
+; CHECK-P9-NEXT:    xscvspdpn f1, v2
+; CHECK-P9-NEXT:    xxmrghd vs1, vs1, vs2
 ; CHECK-P9-NEXT:    xvcvdpuxds vs0, vs0
 ; CHECK-P9-NEXT:    xvcvdpuxds vs1, vs1
-; CHECK-P9-NEXT:    stxv vs1, 16(r3)
 ; CHECK-P9-NEXT:    stxv vs0, 0(r3)
+; CHECK-P9-NEXT:    stxv vs1, 16(r3)
 ; CHECK-P9-NEXT:    blr
 ;
 ; CHECK-BE-LABEL: test4elt:
 ; CHECK-BE:       # %bb.0: # %entry
-; CHECK-BE-NEXT:    xxsldwi vs0, v2, v2, 1
-; CHECK-BE-NEXT:    xxsldwi vs1, v2, v2, 3
+; CHECK-BE-NEXT:    xxsldwi vs1, v2, v2, 1
+; CHECK-BE-NEXT:    xscvspdpn f0, v2
 ; CHECK-BE-NEXT:    xxswapd vs2, v2
-; CHECK-BE-NEXT:    xscvspdpn f3, v2
-; CHECK-BE-NEXT:    xscvspdpn f0, vs0
+; CHECK-BE-NEXT:    xscvspdpn f1, vs1
+; CHECK-BE-NEXT:    xxmrghd vs0, vs0, vs1
+; CHECK-BE-NEXT:    xxsldwi vs1, v2, v2, 3
 ; CHECK-BE-NEXT:    xscvspdpn f1, vs1
 ; CHECK-BE-NEXT:    xscvspdpn f2, vs2
-; CHECK-BE-NEXT:    xxmrghd vs0, vs3, vs0
-; CHECK-BE-NEXT:    xxmrghd vs1, vs2, vs1
 ; CHECK-BE-NEXT:    xvcvdpuxds vs0, vs0
+; CHECK-BE-NEXT:    xxmrghd vs1, vs2, vs1
 ; CHECK-BE-NEXT:    xvcvdpuxds vs1, vs1
-; CHECK-BE-NEXT:    stxv vs1, 16(r3)
 ; CHECK-BE-NEXT:    stxv vs0, 0(r3)
+; CHECK-BE-NEXT:    stxv vs1, 16(r3)
 ; CHECK-BE-NEXT:    blr
 entry:
   %0 = fptoui <4 x float> %a to <4 x i64>
@@ -149,66 +149,66 @@ define void @test8elt(<8 x i64>* noalias nocapture sret %agg.result, <8 x float>
 ;
 ; CHECK-P9-LABEL: test8elt:
 ; CHECK-P9:       # %bb.0: # %entry
-; CHECK-P9-NEXT:    lxv vs0, 16(r4)
-; CHECK-P9-NEXT:    lxv vs1, 0(r4)
-; CHECK-P9-NEXT:    xxsldwi vs2, vs1, vs1, 3
-; CHECK-P9-NEXT:    xxswapd vs3, vs1
-; CHECK-P9-NEXT:    xxsldwi vs4, vs1, vs1, 1
-; CHECK-P9-NEXT:    xxsldwi vs5, vs0, vs0, 3
-; CHECK-P9-NEXT:    xxswapd vs6, vs0
-; CHECK-P9-NEXT:    xxsldwi vs7, vs0, vs0, 1
+; CHECK-P9-NEXT:    lxv vs0, 0(r4)
+; CHECK-P9-NEXT:    xxsldwi vs1, vs0, vs0, 3
+; CHECK-P9-NEXT:    xxswapd vs2, vs0
 ; CHECK-P9-NEXT:    xscvspdpn f1, vs1
-; CHECK-P9-NEXT:    xscvspdpn f0, vs0
 ; CHECK-P9-NEXT:    xscvspdpn f2, vs2
+; CHECK-P9-NEXT:    xscvspdpn f3, vs0
+; CHECK-P9-NEXT:    xxsldwi vs0, vs0, vs0, 1
+; CHECK-P9-NEXT:    xscvspdpn f0, vs0
+; CHECK-P9-NEXT:    xxmrghd vs1, vs2, vs1
+; CHECK-P9-NEXT:    lxv vs2, 16(r4)
+; CHECK-P9-NEXT:    xxmrghd vs0, vs3, vs0
+; CHECK-P9-NEXT:    xvcvdpuxds vs1, vs1
+; CHECK-P9-NEXT:    xvcvdpuxds vs0, vs0
+; CHECK-P9-NEXT:    xxsldwi vs3, vs2, vs2, 3
+; CHECK-P9-NEXT:    xxswapd vs4, vs2
 ; CHECK-P9-NEXT:    xscvspdpn f3, vs3
 ; CHECK-P9-NEXT:    xscvspdpn f4, vs4
-; CHECK-P9-NEXT:    xscvspdpn f5, vs5
-; CHECK-P9-NEXT:    xscvspdpn f6, vs6
-; CHECK-P9-NEXT:    xscvspdpn f7, vs7
-; CHECK-P9-NEXT:    xxmrghd vs2, vs3, vs2
-; CHECK-P9-NEXT:    xxmrghd vs1, vs1, vs4
-; CHECK-P9-NEXT:    xxmrghd vs3, vs6, vs5
-; CHECK-P9-NEXT:    xxmrghd vs0, vs0, vs7
-; CHECK-P9-NEXT:    xvcvdpuxds vs2, vs2
-; CHECK-P9-NEXT:    xvcvdpuxds vs1, vs1
+; CHECK-P9-NEXT:    stxv vs0, 16(r3)
+; CHECK-P9-NEXT:    xxmrghd vs3, vs4, vs3
+; CHECK-P9-NEXT:    xscvspdpn f4, vs2
+; CHECK-P9-NEXT:    xxsldwi vs2, vs2, vs2, 1
+; CHECK-P9-NEXT:    xscvspdpn f2, vs2
 ; CHECK-P9-NEXT:    xvcvdpuxds vs3, vs3
-; CHECK-P9-NEXT:    xvcvdpuxds vs0, vs0
-; CHECK-P9-NEXT:    stxv vs0, 48(r3)
+; CHECK-P9-NEXT:    xxmrghd vs2, vs4, vs2
+; CHECK-P9-NEXT:    xvcvdpuxds vs2, vs2
 ; CHECK-P9-NEXT:    stxv vs3, 32(r3)
-; CHECK-P9-NEXT:    stxv vs1, 16(r3)
-; CHECK-P9-NEXT:    stxv vs2, 0(r3)
+; CHECK-P9-NEXT:    stxv vs2, 48(r3)
+; CHECK-P9-NEXT:    stxv vs1, 0(r3)
 ; CHECK-P9-NEXT:    blr
 ;
 ; CHECK-BE-LABEL: test8elt:
 ; CHECK-BE:       # %bb.0: # %entry
-; CHECK-BE-NEXT:    lxv vs0, 16(r4)
 ; CHECK-BE-NEXT:    lxv vs1, 0(r4)
-; CHECK-BE-NEXT:    xxsldwi vs2, vs1, vs1, 1
+; CHECK-BE-NEXT:    xxsldwi vs3, vs1, vs1, 1
+; CHECK-BE-NEXT:    xscvspdpn f2, vs1
+; CHECK-BE-NEXT:    xscvspdpn f3, vs3
+; CHECK-BE-NEXT:    lxv vs0, 16(r4)
+; CHECK-BE-NEXT:    xxsldwi vs4, vs0, vs0, 1
+; CHECK-BE-NEXT:    xscvspdpn f4, vs4
+; CHECK-BE-NEXT:    xxmrghd vs2, vs2, vs3
 ; CHECK-BE-NEXT:    xxsldwi vs3, vs1, vs1, 3
-; CHECK-BE-NEXT:    xxswapd vs4, vs1
-; CHECK-BE-NEXT:    xxsldwi vs5, vs0, vs0, 1
-; CHECK-BE-NEXT:    xxsldwi vs6, vs0, vs0, 3
-; CHECK-BE-NEXT:    xxswapd vs7, vs0
+; CHECK-BE-NEXT:    xxswapd vs1, vs1
+; CHECK-BE-NEXT:    xscvspdpn f3, vs3
 ; CHECK-BE-NEXT:    xscvspdpn f1, vs1
+; CHECK-BE-NEXT:    xxmrghd vs1, vs1, vs3
+; CHECK-BE-NEXT:    xscvspdpn f3, vs0
+; CHECK-BE-NEXT:    xxmrghd vs3, vs3, vs4
+; CHECK-BE-NEXT:    xxsldwi vs4, vs0, vs0, 3
+; CHECK-BE-NEXT:    xxswapd vs0, vs0
 ; CHECK-BE-NEXT:    xscvspdpn f0, vs0
-; CHECK-BE-NEXT:    xscvspdpn f2, vs2
-; CHECK-BE-NEXT:    xscvspdpn f3, vs3
 ; CHECK-BE-NEXT:    xscvspdpn f4, vs4
-; CHECK-BE-NEXT:    xscvspdpn f5, vs5
-; CHECK-BE-NEXT:    xscvspdpn f6, vs6
-; CHECK-BE-NEXT:    xscvspdpn f7, vs7
-; CHECK-BE-NEXT:    xxmrghd vs1, vs1, vs2
-; CHECK-BE-NEXT:    xxmrghd vs2, vs4, vs3
-; CHECK-BE-NEXT:    xxmrghd vs0, vs0, vs5
-; CHECK-BE-NEXT:    xxmrghd vs3, vs7, vs6
-; CHECK-BE-NEXT:    xvcvdpuxds vs1, vs1
+; CHECK-BE-NEXT:    xxmrghd vs0, vs0, vs4
 ; CHECK-BE-NEXT:    xvcvdpuxds vs2, vs2
-; CHECK-BE-NEXT:    xvcvdpuxds vs0, vs0
+; CHECK-BE-NEXT:    xvcvdpuxds vs1, vs1
 ; CHECK-BE-NEXT:    xvcvdpuxds vs3, vs3
-; CHECK-BE-NEXT:    stxv vs3, 48(r3)
-; CHECK-BE-NEXT:    stxv vs0, 32(r3)
-; CHECK-BE-NEXT:    stxv vs2, 16(r3)
-; CHECK-BE-NEXT:    stxv vs1, 0(r3)
+; CHECK-BE-NEXT:    stxv vs1, 16(r3)
+; CHECK-BE-NEXT:    xvcvdpuxds vs0, vs0
+; CHECK-BE-NEXT:    stxv vs3, 32(r3)
+; CHECK-BE-NEXT:    stxv vs0, 48(r3)
+; CHECK-BE-NEXT:    stxv vs2, 0(r3)
 ; CHECK-BE-NEXT:    blr
 entry:
   %a = load <8 x float>, <8 x float>* %0, align 32
@@ -295,130 +295,122 @@ define void @test16elt(<16 x i64>* noalias nocapture sret %agg.result, <16 x flo
 ;
 ; CHECK-P9-LABEL: test16elt:
 ; CHECK-P9:       # %bb.0: # %entry
-; CHECK-P9-NEXT:    lxv vs0, 16(r4)
-; CHECK-P9-NEXT:    lxv vs1, 0(r4)
-; CHECK-P9-NEXT:    lxv vs2, 48(r4)
-; CHECK-P9-NEXT:    lxv vs3, 32(r4)
-; CHECK-P9-NEXT:    stfd f30, -16(r1) # 8-byte Folded Spill
-; CHECK-P9-NEXT:    stfd f31, -8(r1) # 8-byte Folded Spill
-; CHECK-P9-NEXT:    xxsldwi vs4, vs1, vs1, 3
-; CHECK-P9-NEXT:    xxswapd vs5, vs1
-; CHECK-P9-NEXT:    xxsldwi vs6, vs1, vs1, 1
-; CHECK-P9-NEXT:    xxsldwi vs7, vs0, vs0, 3
-; CHECK-P9-NEXT:    xxswapd vs8, vs0
-; CHECK-P9-NEXT:    xxsldwi vs9, vs0, vs0, 1
-; CHECK-P9-NEXT:    xxsldwi vs10, vs3, vs3, 3
-; CHECK-P9-NEXT:    xxswapd vs11, vs3
-; CHECK-P9-NEXT:    xxsldwi vs12, vs3, vs3, 1
-; CHECK-P9-NEXT:    xxsldwi vs13, vs2, vs2, 3
-; CHECK-P9-NEXT:    xxswapd v2, vs2
-; CHECK-P9-NEXT:    xxsldwi v3, vs2, vs2, 1
-; CHECK-P9-NEXT:    xscvspdpn f1, vs1
-; CHECK-P9-NEXT:    xscvspdpn f0, vs0
-; CHECK-P9-NEXT:    xscvspdpn f3, vs3
-; CHECK-P9-NEXT:    xscvspdpn f2, vs2
-; CHECK-P9-NEXT:    xscvspdpn f4, vs4
+; CHECK-P9-NEXT:    lxv vs4, 16(r4)
+; CHECK-P9-NEXT:    xxsldwi vs5, vs4, vs4, 3
+; CHECK-P9-NEXT:    xxswapd vs6, vs4
+; CHECK-P9-NEXT:    lxv vs0, 0(r4)
+; CHECK-P9-NEXT:    xxsldwi vs1, vs0, vs0, 3
+; CHECK-P9-NEXT:    xxswapd vs2, vs0
 ; CHECK-P9-NEXT:    xscvspdpn f5, vs5
 ; CHECK-P9-NEXT:    xscvspdpn f6, vs6
+; CHECK-P9-NEXT:    xxmrghd vs5, vs6, vs5
+; CHECK-P9-NEXT:    xscvspdpn f6, vs4
+; CHECK-P9-NEXT:    xxsldwi vs4, vs4, vs4, 1
+; CHECK-P9-NEXT:    lxv vs3, 32(r4)
+; CHECK-P9-NEXT:    xscvspdpn f2, vs2
+; CHECK-P9-NEXT:    xxswapd vs7, vs3
 ; CHECK-P9-NEXT:    xscvspdpn f7, vs7
-; CHECK-P9-NEXT:    xscvspdpn f8, vs8
-; CHECK-P9-NEXT:    xscvspdpn f9, vs9
-; CHECK-P9-NEXT:    xscvspdpn f10, vs10
-; CHECK-P9-NEXT:    xscvspdpn f11, vs11
-; CHECK-P9-NEXT:    xscvspdpn f12, vs12
-; CHECK-P9-NEXT:    xscvspdpn f13, vs13
-; CHECK-P9-NEXT:    xscvspdpn f31, v2
-; CHECK-P9-NEXT:    xscvspdpn f30, v3
-; CHECK-P9-NEXT:    xxmrghd vs4, vs5, vs4
-; CHECK-P9-NEXT:    xxmrghd vs1, vs1, vs6
-; CHECK-P9-NEXT:    xxmrghd vs5, vs8, vs7
-; CHECK-P9-NEXT:    xxmrghd vs0, vs0, vs9
-; CHECK-P9-NEXT:    xxmrghd vs6, vs11, vs10
-; CHECK-P9-NEXT:    xxmrghd vs3, vs3, vs12
-; CHECK-P9-NEXT:    xxmrghd vs7, vs31, vs13
-; CHECK-P9-NEXT:    lfd f31, -8(r1) # 8-byte Folded Reload
-; CHECK-P9-NEXT:    xxmrghd vs2, vs2, vs30
-; CHECK-P9-NEXT:    lfd f30, -16(r1) # 8-byte Folded Reload
-; CHECK-P9-NEXT:    xvcvdpuxds vs4, vs4
+; CHECK-P9-NEXT:    xscvspdpn f4, vs4
+; CHECK-P9-NEXT:    xscvspdpn f1, vs1
+; CHECK-P9-NEXT:    xxmrghd vs1, vs2, vs1
+; CHECK-P9-NEXT:    xscvspdpn f2, vs0
+; CHECK-P9-NEXT:    xxsldwi vs0, vs0, vs0, 1
+; CHECK-P9-NEXT:    xscvspdpn f0, vs0
+; CHECK-P9-NEXT:    xxmrghd vs0, vs2, vs0
+; CHECK-P9-NEXT:    xxmrghd vs4, vs6, vs4
+; CHECK-P9-NEXT:    xxsldwi vs6, vs3, vs3, 3
 ; CHECK-P9-NEXT:    xvcvdpuxds vs1, vs1
 ; CHECK-P9-NEXT:    xvcvdpuxds vs5, vs5
+; CHECK-P9-NEXT:    xscvspdpn f6, vs6
+; CHECK-P9-NEXT:    xxmrghd vs6, vs7, vs6
+; CHECK-P9-NEXT:    xscvspdpn f7, vs3
+; CHECK-P9-NEXT:    xxsldwi vs3, vs3, vs3, 1
+; CHECK-P9-NEXT:    lxv vs2, 48(r4)
+; CHECK-P9-NEXT:    xxswapd vs8, vs2
+; CHECK-P9-NEXT:    xscvspdpn f8, vs8
 ; CHECK-P9-NEXT:    xvcvdpuxds vs0, vs0
+; CHECK-P9-NEXT:    stxv vs5, 32(r3)
 ; CHECK-P9-NEXT:    xvcvdpuxds vs6, vs6
+; CHECK-P9-NEXT:    xscvspdpn f3, vs3
+; CHECK-P9-NEXT:    xxmrghd vs3, vs7, vs3
+; CHECK-P9-NEXT:    xxsldwi vs7, vs2, vs2, 3
+; CHECK-P9-NEXT:    xscvspdpn f7, vs7
+; CHECK-P9-NEXT:    xxmrghd vs7, vs8, vs7
+; CHECK-P9-NEXT:    xscvspdpn f8, vs2
+; CHECK-P9-NEXT:    xxsldwi vs2, vs2, vs2, 1
+; CHECK-P9-NEXT:    stxv vs6, 64(r3)
+; CHECK-P9-NEXT:    xvcvdpuxds vs4, vs4
 ; CHECK-P9-NEXT:    xvcvdpuxds vs3, vs3
 ; CHECK-P9-NEXT:    xvcvdpuxds vs7, vs7
+; CHECK-P9-NEXT:    xscvspdpn f2, vs2
+; CHECK-P9-NEXT:    stxv vs3, 80(r3)
+; CHECK-P9-NEXT:    xxmrghd vs2, vs8, vs2
 ; CHECK-P9-NEXT:    xvcvdpuxds vs2, vs2
-; CHECK-P9-NEXT:    stxv vs0, 48(r3)
-; CHECK-P9-NEXT:    stxv vs5, 32(r3)
-; CHECK-P9-NEXT:    stxv vs1, 16(r3)
-; CHECK-P9-NEXT:    stxv vs4, 0(r3)
 ; CHECK-P9-NEXT:    stxv vs2, 112(r3)
 ; CHECK-P9-NEXT:    stxv vs7, 96(r3)
-; CHECK-P9-NEXT:    stxv vs3, 80(r3)
-; CHECK-P9-NEXT:    stxv vs6, 64(r3)
+; CHECK-P9-NEXT:    stxv vs4, 48(r3)
+; CHECK-P9-NEXT:    stxv vs0, 16(r3)
+; CHECK-P9-NEXT:    stxv vs1, 0(r3)
 ; CHECK-P9-NEXT:    blr
 ;
 ; CHECK-BE-LABEL: test16elt:
 ; CHECK-BE:       # %bb.0: # %entry
-; CHECK-BE-NEXT:    lxv vs0, 16(r4)
-; CHECK-BE-NEXT:    lxv vs1, 0(r4)
-; CHECK-BE-NEXT:    lxv vs2, 48(r4)
-; CHECK-BE-NEXT:    lxv vs3, 32(r4)
-; CHECK-BE-NEXT:    stfd f30, -16(r1) # 8-byte Folded Spill
-; CHECK-BE-NEXT:    stfd f31, -8(r1) # 8-byte Folded Spill
-; CHECK-BE-NEXT:    xxsldwi vs4, vs1, vs1, 1
-; CHECK-BE-NEXT:    xxsldwi vs5, vs1, vs1, 3
-; CHECK-BE-NEXT:    xxswapd vs6, vs1
-; CHECK-BE-NEXT:    xxsldwi vs7, vs0, vs0, 1
-; CHECK-BE-NEXT:    xxsldwi vs8, vs0, vs0, 3
-; CHECK-BE-NEXT:    xxswapd vs9, vs0
-; CHECK-BE-NEXT:    xxsldwi vs10, vs3, vs3, 1
-; CHECK-BE-NEXT:    xxsldwi vs11, vs3, vs3, 3
-; CHECK-BE-NEXT:    xxswapd vs12, vs3
-; CHECK-BE-NEXT:    xxsldwi vs13, vs2, vs2, 1
-; CHECK-BE-NEXT:    xxsldwi v2, vs2, vs2, 3
-; CHECK-BE-NEXT:    xxswapd v3, vs2
-; CHECK-BE-NEXT:    xscvspdpn f1, vs1
-; CHECK-BE-NEXT:    xscvspdpn f0, vs0
-; CHECK-BE-NEXT:    xscvspdpn f3, vs3
-; CHECK-BE-NEXT:    xscvspdpn f2, vs2
-; CHECK-BE-NEXT:    xscvspdpn f4, vs4
+; CHECK-BE-NEXT:    lxv vs0, 0(r4)
+; CHECK-BE-NEXT:    lxv vs4, 16(r4)
+; CHECK-BE-NEXT:    xxsldwi vs2, vs0, vs0, 1
+; CHECK-BE-NEXT:    xscvspdpn f1, vs0
+; CHECK-BE-NEXT:    xxsldwi vs5, vs0, vs0, 3
+; CHECK-BE-NEXT:    xxswapd vs0, vs0
 ; CHECK-BE-NEXT:    xscvspdpn f5, vs5
+; CHECK-BE-NEXT:    xscvspdpn f0, vs0
+; CHECK-BE-NEXT:    xxsldwi vs6, vs4, vs4, 1
 ; CHECK-BE-NEXT:    xscvspdpn f6, vs6
+; CHECK-BE-NEXT:    xxmrghd vs0, vs0, vs5
+; CHECK-BE-NEXT:    xscvspdpn f5, vs4
+; CHECK-BE-NEXT:    lxv vs3, 32(r4)
+; CHECK-BE-NEXT:    xxsldwi vs7, vs3, vs3, 1
 ; CHECK-BE-NEXT:    xscvspdpn f7, vs7
-; CHECK-BE-NEXT:    xscvspdpn f8, vs8
-; CHECK-BE-NEXT:    xscvspdpn f9, vs9
-; CHECK-BE-NEXT:    xscvspdpn f10, vs10
-; CHECK-BE-NEXT:    xscvspdpn f11, vs11
-; CHECK-BE-NEXT:    xscvspdpn f12, vs12
-; CHECK-BE-NEXT:    xscvspdpn f13, vs13
-; CHECK-BE-NEXT:    xscvspdpn f31, v2
-; CHECK-BE-NEXT:    xscvspdpn f30, v3
-; CHECK-BE-NEXT:    xxmrghd vs1, vs1, vs4
-; CHECK-BE-NEXT:    xxmrghd vs4, vs6, vs5
-; CHECK-BE-NEXT:    xxmrghd vs0, vs0, vs7
-; CHECK-BE-NEXT:    xxmrghd vs5, vs9, vs8
-; CHECK-BE-NEXT:    xxmrghd vs3, vs3, vs10
-; CHECK-BE-NEXT:    xxmrghd vs6, vs12, vs11
-; CHECK-BE-NEXT:    xxmrghd vs2, vs2, vs13
-; CHECK-BE-NEXT:    xxmrghd vs7, vs30, vs31
-; CHECK-BE-NEXT:    lfd f31, -8(r1) # 8-byte Folded Reload
-; CHECK-BE-NEXT:    lfd f30, -16(r1) # 8-byte Folded Reload
+; CHECK-BE-NEXT:    xxmrghd vs5, vs5, vs6
+; CHECK-BE-NEXT:    xxsldwi vs6, vs4, vs4, 3
+; CHECK-BE-NEXT:    xxswapd vs4, vs4
+; CHECK-BE-NEXT:    xscvspdpn f6, vs6
+; CHECK-BE-NEXT:    xscvspdpn f4, vs4
+; CHECK-BE-NEXT:    xscvspdpn f2, vs2
+; CHECK-BE-NEXT:    xxmrghd vs1, vs1, vs2
+; CHECK-BE-NEXT:    lxv vs2, 48(r4)
+; CHECK-BE-NEXT:    xxsldwi vs8, vs2, vs2, 1
 ; CHECK-BE-NEXT:    xvcvdpuxds vs1, vs1
-; CHECK-BE-NEXT:    xvcvdpuxds vs4, vs4
 ; CHECK-BE-NEXT:    xvcvdpuxds vs0, vs0
 ; CHECK-BE-NEXT:    xvcvdpuxds vs5, vs5
-; CHECK-BE-NEXT:    xvcvdpuxds vs3, vs3
+; CHECK-BE-NEXT:    xscvspdpn f8, vs8
+; CHECK-BE-NEXT:    xxmrghd vs4, vs4, vs6
+; CHECK-BE-NEXT:    xscvspdpn f6, vs3
+; CHECK-BE-NEXT:    stxv vs0, 16(r3)
+; CHECK-BE-NEXT:    xxmrghd vs6, vs6, vs7
+; CHECK-BE-NEXT:    xxsldwi vs7, vs3, vs3, 3
+; CHECK-BE-NEXT:    xxswapd vs3, vs3
+; CHECK-BE-NEXT:    xscvspdpn f7, vs7
+; CHECK-BE-NEXT:    xscvspdpn f3, vs3
+; CHECK-BE-NEXT:    xxmrghd vs3, vs3, vs7
+; CHECK-BE-NEXT:    xscvspdpn f7, vs2
+; CHECK-BE-NEXT:    xxmrghd vs7, vs7, vs8
+; CHECK-BE-NEXT:    xxsldwi vs8, vs2, vs2, 3
+; CHECK-BE-NEXT:    xxswapd vs2, vs2
+; CHECK-BE-NEXT:    xscvspdpn f8, vs8
+; CHECK-BE-NEXT:    xscvspdpn f2, vs2
+; CHECK-BE-NEXT:    xxmrghd vs2, vs2, vs8
+; CHECK-BE-NEXT:    stxv vs5, 32(r3)
+; CHECK-BE-NEXT:    xvcvdpuxds vs4, vs4
 ; CHECK-BE-NEXT:    xvcvdpuxds vs6, vs6
-; CHECK-BE-NEXT:    xvcvdpuxds vs2, vs2
+; CHECK-BE-NEXT:    xvcvdpuxds vs3, vs3
 ; CHECK-BE-NEXT:    xvcvdpuxds vs7, vs7
-; CHECK-BE-NEXT:    stxv vs5, 48(r3)
-; CHECK-BE-NEXT:    stxv vs0, 32(r3)
-; CHECK-BE-NEXT:    stxv vs4, 16(r3)
+; CHECK-BE-NEXT:    stxv vs3, 80(r3)
+; CHECK-BE-NEXT:    stxv vs7, 96(r3)
+; CHECK-BE-NEXT:    xvcvdpuxds vs2, vs2
+; CHECK-BE-NEXT:    stxv vs2, 112(r3)
+; CHECK-BE-NEXT:    stxv vs6, 64(r3)
+; CHECK-BE-NEXT:    stxv vs4, 48(r3)
 ; CHECK-BE-NEXT:    stxv vs1, 0(r3)
-; CHECK-BE-NEXT:    stxv vs7, 112(r3)
-; CHECK-BE-NEXT:    stxv vs2, 96(r3)
-; CHECK-BE-NEXT:    stxv vs6, 80(r3)
-; CHECK-BE-NEXT:    stxv vs3, 64(r3)
 ; CHECK-BE-NEXT:    blr
 entry:
   %a = load <16 x float>, <16 x float>* %0, align 64
@@ -453,10 +445,10 @@ define <2 x i64> @test2elt_signed(i64 %a.coerce) local_unnamed_addr #0 {
 ; CHECK-BE-LABEL: test2elt_signed:
 ; CHECK-BE:       # %bb.0: # %entry
 ; CHECK-BE-NEXT:    mtvsrd f0, r3
-; CHECK-BE-NEXT:    xxsldwi vs1, vs0, vs0, 1
+; CHECK-BE-NEXT:    xscvspdpn f1, vs0
+; CHECK-BE-NEXT:    xxsldwi vs0, vs0, vs0, 1
 ; CHECK-BE-NEXT:    xscvspdpn f0, vs0
-; CHECK-BE-NEXT:    xscvspdpn f1, vs1
-; CHECK-BE-NEXT:    xxmrghd vs0, vs0, vs1
+; CHECK-BE-NEXT:    xxmrghd vs0, vs1, vs0
 ; CHECK-BE-NEXT:    xvcvdpuxds v2, vs0
 ; CHECK-BE-NEXT:    blr
 entry:
@@ -490,34 +482,34 @@ define void @test4elt_signed(<4 x i64>* noalias nocapture sret %agg.result, <4 x
 ; CHECK-P9:       # %bb.0: # %entry
 ; CHECK-P9-NEXT:    xxsldwi vs0, v2, v2, 3
 ; CHECK-P9-NEXT:    xxswapd vs1, v2
-; CHECK-P9-NEXT:    xxsldwi vs2, v2, v2, 1
-; CHECK-P9-NEXT:    xscvspdpn f3, v2
 ; CHECK-P9-NEXT:    xscvspdpn f0, vs0
 ; CHECK-P9-NEXT:    xscvspdpn f1, vs1
+; CHECK-P9-NEXT:    xxsldwi vs2, v2, v2, 1
 ; CHECK-P9-NEXT:    xscvspdpn f2, vs2
 ; CHECK-P9-NEXT:    xxmrghd vs0, vs1, vs0
-; CHECK-P9-NEXT:    xxmrghd vs1, vs3, vs2
+; CHECK-P9-NEXT:    xscvspdpn f1, v2
+; CHECK-P9-NEXT:    xxmrghd vs1, vs1, vs2
 ; CHECK-P9-NEXT:    xvcvdpuxds vs0, vs0
 ; CHECK-P9-NEXT:    xvcvdpuxds vs1, vs1
-; CHECK-P9-NEXT:    stxv vs1, 16(r3)
 ; CHECK-P9-NEXT:    stxv vs0, 0(r3)
+; CHECK-P9-NEXT:    stxv vs1, 16(r3)
 ; CHECK-P9-NEXT:    blr
 ;
 ; CHECK-BE-LABEL: test4elt_signed:
 ; CHECK-BE:       # %bb.0: # %entry
-; CHECK-BE-NEXT:    xxsldwi vs0, v2, v2, 1
-; CHECK-BE-NEXT:    xxsldwi vs1, v2, v2, 3
+; CHECK-BE-NEXT:    xxsldwi vs1, v2, v2, 1
+; CHECK-BE-NEXT:    xscvspdpn f0, v2
 ; CHECK-BE-NEXT:    xxswapd vs2, v2
-; CHECK-BE-NEXT:    xscvspdpn f3, v2
-; CHECK-BE-NEXT:    xscvspdpn f0, vs0
+; CHECK-BE-NEXT:    xscvspdpn f1, vs1
+; CHECK-BE-NEXT:    xxmrghd vs0, vs0, vs1
+; CHECK-BE-NEXT:    xxsldwi vs1, v2, v2, 3
 ; CHECK-BE-NEXT:    xscvspdpn f1, vs1
 ; CHECK-BE-NEXT:    xscvspdpn f2, vs2
-; CHECK-BE-NEXT:    xxmrghd vs0, vs3, vs0
-; CHECK-BE-NEXT:    xxmrghd vs1, vs2, vs1
 ; CHECK-BE-NEXT:    xvcvdpuxds vs0, vs0
+; CHECK-BE-NEXT:    xxmrghd vs1, vs2, vs1
 ; CHECK-BE-NEXT:    xvcvdpuxds vs1, vs1
-; CHECK-BE-NEXT:    stxv vs1, 16(r3)
 ; CHECK-BE-NEXT:    stxv vs0, 0(r3)
+; CHECK-BE-NEXT:    stxv vs1, 16(r3)
 ; CHECK-BE-NEXT:    blr
 entry:
   %0 = fptoui <4 x float> %a to <4 x i64>
@@ -567,66 +559,66 @@ define void @test8elt_signed(<8 x i64>* noalias nocapture sret %agg.result, <8 x
 ;
 ; CHECK-P9-LABEL: test8elt_signed:
 ; CHECK-P9:       # %bb.0: # %entry
-; CHECK-P9-NEXT:    lxv vs0, 16(r4)
-; CHECK-P9-NEXT:    lxv vs1, 0(r4)
-; CHECK-P9-NEXT:    xxsldwi vs2, vs1, vs1, 3
-; CHECK-P9-NEXT:    xxswapd vs3, vs1
-; CHECK-P9-NEXT:    xxsldwi vs4, vs1, vs1, 1
-; CHECK-P9-NEXT:    xxsldwi vs5, vs0, vs0, 3
-; CHECK-P9-NEXT:    xxswapd vs6, vs0
-; CHECK-P9-NEXT:    xxsldwi vs7, vs0, vs0, 1
+; CHECK-P9-NEXT:    lxv vs0, 0(r4)
+; CHECK-P9-NEXT:    xxsldwi vs1, vs0, vs0, 3
+; CHECK-P9-NEXT:    xxswapd vs2, vs0
 ; CHECK-P9-NEXT:    xscvspdpn f1, vs1
-; CHECK-P9-NEXT:    xscvspdpn f0, vs0
 ; CHECK-P9-NEXT:    xscvspdpn f2, vs2
+; CHECK-P9-NEXT:    xscvspdpn f3, vs0
+; CHECK-P9-NEXT:    xxsldwi vs0, vs0, vs0, 1
+; CHECK-P9-NEXT:    xscvspdpn f0, vs0
+; CHECK-P9-NEXT:    xxmrghd vs1, vs2, vs1
+; CHECK-P9-NEXT:    lxv vs2, 16(r4)
+; CHECK-P9-NEXT:    xxmrghd vs0, vs3, vs0
+; CHECK-P9-NEXT:    xvcvdpuxds vs1, vs1
+; CHECK-P9-NEXT:    xvcvdpuxds vs0, vs0
+; CHECK-P9-NEXT:    xxsldwi vs3, vs2, vs2, 3
+; CHECK-P9-NEXT:    xxswapd vs4, vs2
 ; CHECK-P9-NEXT:    xscvspdpn f3, vs3
 ; CHECK-P9-NEXT:    xscvspdpn f4, vs4
-; CHECK-P9-NEXT:    xscvspdpn f5, vs5
-; CHECK-P9-NEXT:    xscvspdpn f6, vs6
-; CHECK-P9-NEXT:    xscvspdpn f7, vs7
-; CHECK-P9-NEXT:    xxmrghd vs2, vs3, vs2
-; CHECK-P9-NEXT:    xxmrghd vs1, vs1, vs4
-; CHECK-P9-NEXT:    xxmrghd vs3, vs6, vs5
-; CHECK-P9-NEXT:    xxmrghd vs0, vs0, vs7
-; CHECK-P9-NEXT:    xvcvdpuxds vs2, vs2
-; CHECK-P9-NEXT:    xvcvdpuxds vs1, vs1
+; CHECK-P9-NEXT:    stxv vs0, 16(r3)
+; CHECK-P9-NEXT:    xxmrghd vs3, vs4, vs3
+; CHECK-P9-NEXT:    xscvspdpn f4, vs2
+; CHECK-P9-NEXT:    xxsldwi vs2, vs2, vs2, 1
+; CHECK-P9-NEXT:    xscvspdpn f2, vs2
 ; CHECK-P9-NEXT:    xvcvdpuxds vs3, vs3
-; CHECK-P9-NEXT:    xvcvdpuxds vs0, vs0
-; CHECK-P9-NEXT:    stxv vs0, 48(r3)
+; CHECK-P9-NEXT:    xxmrghd vs2, vs4, vs2
+; CHECK-P9-NEXT:    xvcvdpuxds vs2, vs2
 ; CHECK-P9-NEXT:    stxv vs3, 32(r3)
-; CHECK-P9-NEXT:    stxv vs1, 16(r3)
-; CHECK-P9-NEXT:    stxv vs2, 0(r3)
+; CHECK-P9-NEXT:    stxv vs2, 48(r3)
+; CHECK-P9-NEXT:    stxv vs1, 0(r3)
 ; CHECK-P9-NEXT:    blr
 ;
 ; CHECK-BE-LABEL: test8elt_signed:
 ; CHECK-BE:       # %bb.0: # %entry
-; CHECK-BE-NEXT:    lxv vs0, 16(r4)
 ; CHECK-BE-NEXT:    lxv vs1, 0(r4)
-; CHECK-BE-NEXT:    xxsldwi vs2, vs1, vs1, 1
+; CHECK-BE-NEXT:    xxsldwi vs3, vs1, vs1, 1
+; CHECK-BE-NEXT:    xscvspdpn f2, vs1
+; CHECK-BE-NEXT:    xscvspdpn f3, vs3
+; CHECK-BE-NEXT:    lxv vs0, 16(r4)
+; CHECK-BE-NEXT:    xxsldwi vs4, vs0, vs0, 1
+; CHECK-BE-NEXT:    xscvspdpn f4, vs4
+; CHECK-BE-NEXT:    xxmrghd vs2, vs2, vs3
 ; CHECK-BE-NEXT:    xxsldwi vs3, vs1, vs1, 3
-; CHECK-BE-NEXT:    xxswapd vs4, vs1
-; CHECK-BE-NEXT:    xxsldwi vs5, vs0, vs0, 1
-; CHECK-BE-NEXT:    xxsldwi vs6, vs0, vs0, 3
-; CHECK-BE-NEXT:    xxswapd vs7, vs0
+; CHECK-BE-NEXT:    xxswapd vs1, vs1
+; CHECK-BE-NEXT:    xscvspdpn f3, vs3
 ; CHECK-BE-NEXT:    xscvspdpn f1, vs1
+; CHECK-BE-NEXT:    xxmrghd vs1, vs1, vs3
+; CHECK-BE-NEXT:    xscvspdpn f3, vs0
+; CHECK-BE-NEXT:    xxmrghd vs3, vs3, vs4
+; CHECK-BE-NEXT:    xxsldwi vs4, vs0, vs0, 3
+; CHECK-BE-NEXT:    xxswapd vs0, vs0
 ; CHECK-BE-NEXT:    xscvspdpn f0, vs0
-; CHECK-BE-NEXT:    xscvspdpn f2, vs2
-; CHECK-BE-NEXT:    xscvspdpn f3, vs3
 ; CHECK-BE-NEXT:    xscvspdpn f4, vs4
-; CHECK-BE-NEXT:    xscvspdpn f5, vs5
-; CHECK-BE-NEXT:    xscvspdpn f6, vs6
-; CHECK-BE-NEXT:    xscvspdpn f7, vs7
-; CHECK-BE-NEXT:    xxmrghd vs1, vs1, vs2
-; CHECK-BE-NEXT:    xxmrghd vs2, vs4, vs3
-; CHECK-BE-NEXT:    xxmrghd vs0, vs0, vs5
-; CHECK-BE-NEXT:    xxmrghd vs3, vs7, vs6
-; CHECK-BE-NEXT:    xvcvdpuxds vs1, vs1
+; CHECK-BE-NEXT:    xxmrghd vs0, vs0, vs4
 ; CHECK-BE-NEXT:    xvcvdpuxds vs2, vs2
-; CHECK-BE-NEXT:    xvcvdpuxds vs0, vs0
+; CHECK-BE-NEXT:    xvcvdpuxds vs1, vs1
 ; CHECK-BE-NEXT:    xvcvdpuxds vs3, vs3
-; CHECK-BE-NEXT:    stxv vs3, 48(r3)
-; CHECK-BE-NEXT:    stxv vs0, 32(r3)
-; CHECK-BE-NEXT:    stxv vs2, 16(r3)
-; CHECK-BE-NEXT:    stxv vs1, 0(r3)
+; CHECK-BE-NEXT:    stxv vs1, 16(r3)
+; CHECK-BE-NEXT:    xvcvdpuxds vs0, vs0
+; CHECK-BE-NEXT:    stxv vs3, 32(r3)
+; CHECK-BE-NEXT:    stxv vs0, 48(r3)
+; CHECK-BE-NEXT:    stxv vs2, 0(r3)
 ; CHECK-BE-NEXT:    blr
 entry:
   %a = load <8 x float>, <8 x float>* %0, align 32
@@ -713,130 +705,122 @@ define void @test16elt_signed(<16 x i64>* noalias nocapture sret %agg.result, <1
 ;
 ; CHECK-P9-LABEL: test16elt_signed:
 ; CHECK-P9:       # %bb.0: # %entry
-; CHECK-P9-NEXT:    lxv vs0, 16(r4)
-; CHECK-P9-NEXT:    lxv vs1, 0(r4)
-; CHECK-P9-NEXT:    lxv vs2, 48(r4)
-; CHECK-P9-NEXT:    lxv vs3, 32(r4)
-; CHECK-P9-NEXT:    stfd f30, -16(r1) # 8-byte Folded Spill
-; CHECK-P9-NEXT:    stfd f31, -8(r1) # 8-byte Folded Spill
-; CHECK-P9-NEXT:    xxsldwi vs4, vs1, vs1, 3
-; CHECK-P9-NEXT:    xxswapd vs5, vs1
-; CHECK-P9-NEXT:    xxsldwi vs6, vs1, vs1, 1
-; CHECK-P9-NEXT:    xxsldwi vs7, vs0, vs0, 3
-; CHECK-P9-NEXT:    xxswapd vs8, vs0
-; CHECK-P9-NEXT:    xxsldwi vs9, vs0, vs0, 1
-; CHECK-P9-NEXT:    xxsldwi vs10, vs3, vs3, 3
-; CHECK-P9-NEXT:    xxswapd vs11, vs3
-; CHECK-P9-NEXT:    xxsldwi vs12, vs3, vs3, 1
-; CHECK-P9-NEXT:    xxsldwi vs13, vs2, vs2, 3
-; CHECK-P9-NEXT:    xxswapd v2, vs2
-; CHECK-P9-NEXT:    xxsldwi v3, vs2, vs2, 1
-; CHECK-P9-NEXT:    xscvspdpn f1, vs1
-; CHECK-P9-NEXT:    xscvspdpn f0, vs0
-; CHECK-P9-NEXT:    xscvspdpn f3, vs3
-; CHECK-P9-NEXT:    xscvspdpn f2, vs2
-; CHECK-P9-NEXT:    xscvspdpn f4, vs4
+; CHECK-P9-NEXT:    lxv vs4, 16(r4)
+; CHECK-P9-NEXT:    xxsldwi vs5, vs4, vs4, 3
+; CHECK-P9-NEXT:    xxswapd vs6, vs4
+; CHECK-P9-NEXT:    lxv vs0, 0(r4)
+; CHECK-P9-NEXT:    xxsldwi vs1, vs0, vs0, 3
+; CHECK-P9-NEXT:    xxswapd vs2, vs0
 ; CHECK-P9-NEXT:    xscvspdpn f5, vs5
 ; CHECK-P9-NEXT:    xscvspdpn f6, vs6
+; CHECK-P9-NEXT:    xxmrghd vs5, vs6, vs5
+; CHECK-P9-NEXT:    xscvspdpn f6, vs4
+; CHECK-P9-NEXT:    xxsldwi vs4, vs4, vs4, 1
+; CHECK-P9-NEXT:    lxv vs3, 32(r4)
+; CHECK-P9-NEXT:    xscvspdpn f2, vs2
+; CHECK-P9-NEXT:    xxswapd vs7, vs3
 ; CHECK-P9-NEXT:    xscvspdpn f7, vs7
-; CHECK-P9-NEXT:    xscvspdpn f8, vs8
-; CHECK-P9-NEXT:    xscvspdpn f9, vs9
-; CHECK-P9-NEXT:    xscvspdpn f10, vs10
-; CHECK-P9-NEXT:    xscvspdpn f11, vs11
-; CHECK-P9-NEXT:    xscvspdpn f12, vs12
-; CHECK-P9-NEXT:    xscvspdpn f13, vs13
-; CHECK-P9-NEXT:    xscvspdpn f31, v2
-; CHECK-P9-NEXT:    xscvspdpn f30, v3
-; CHECK-P9-NEXT:    xxmrghd vs4, vs5, vs4
-; CHECK-P9-NEXT:    xxmrghd vs1, vs1, vs6
-; CHECK-P9-NEXT:    xxmrghd vs5, vs8, vs7
-; CHECK-P9-NEXT:    xxmrghd vs0, vs0, vs9
-; CHECK-P9-NEXT:    xxmrghd vs6, vs11, vs10
-; CHECK-P9-NEXT:    xxmrghd vs3, vs3, vs12
-; CHECK-P9-NEXT:    xxmrghd vs7, vs31, vs13
-; CHECK-P9-NEXT:    lfd f31, -8(r1) # 8-byte Folded Reload
-; CHECK-P9-NEXT:    xxmrghd vs2, vs2, vs30
-; CHECK-P9-NEXT:    lfd f30, -16(r1) # 8-byte Folded Reload
-; CHECK-P9-NEXT:    xvcvdpuxds vs4, vs4
+; CHECK-P9-NEXT:    xscvspdpn f4, vs4
+; CHECK-P9-NEXT:    xscvspdpn f1, vs1
+; CHECK-P9-NEXT:    xxmrghd vs1, vs2, vs1
+; CHECK-P9-NEXT:    xscvspdpn f2, vs0
+; CHECK-P9-NEXT:    xxsldwi vs0, vs0, vs0, 1
+; CHECK-P9-NEXT:    xscvspdpn f0, vs0
+; CHECK-P9-NEXT:    xxmrghd vs0, vs2, vs0
+; CHECK-P9-NEXT:    xxmrghd vs4, vs6, vs4
+; CHECK-P9-NEXT:    xxsldwi vs6, vs3, vs3, 3
 ; CHECK-P9-NEXT:    xvcvdpuxds vs1, vs1
 ; CHECK-P9-NEXT:    xvcvdpuxds vs5, vs5
+; CHECK-P9-NEXT:    xscvspdpn f6, vs6
+; CHECK-P9-NEXT:    xxmrghd vs6, vs7, vs6
+; CHECK-P9-NEXT:    xscvspdpn f7, vs3
+; CHECK-P9-NEXT:    xxsldwi vs3, vs3, vs3, 1
+; CHECK-P9-NEXT:    lxv vs2, 48(r4)
+; CHECK-P9-NEXT:    xxswapd vs8, vs2
+; CHECK-P9-NEXT:    xscvspdpn f8, vs8
 ; CHECK-P9-NEXT:    xvcvdpuxds vs0, vs0
+; CHECK-P9-NEXT:    stxv vs5, 32(r3)
 ; CHECK-P9-NEXT:    xvcvdpuxds vs6, vs6
+; CHECK-P9-NEXT:    xscvspdpn f3, vs3
+; CHECK-P9-NEXT:    xxmrghd vs3, vs7, vs3
+; CHECK-P9-NEXT:    xxsldwi vs7, vs2, vs2, 3
+; CHECK-P9-NEXT:    xscvspdpn f7, vs7
+; CHECK-P9-NEXT:    xxmrghd vs7, vs8, vs7
+; CHECK-P9-NEXT:    xscvspdpn f8, vs2
+; CHECK-P9-NEXT:    xxsldwi vs2, vs2, vs2, 1
+; CHECK-P9-NEXT:    stxv vs6, 64(r3)
+; CHECK-P9-NEXT:    xvcvdpuxds vs4, vs4
 ; CHECK-P9-NEXT:    xvcvdpuxds vs3, vs3
 ; CHECK-P9-NEXT:    xvcvdpuxds vs7, vs7
+; CHECK-P9-NEXT:    xscvspdpn f2, vs2
+; CHECK-P9-NEXT:    stxv vs3, 80(r3)
+; CHECK-P9-NEXT:    xxmrghd vs2, vs8, vs2
 ; CHECK-P9-NEXT:    xvcvdpuxds vs2, vs2
-; CHECK-P9-NEXT:    stxv vs0, 48(r3)
-; CHECK-P9-NEXT:    stxv vs5, 32(r3)
-; CHECK-P9-NEXT:    stxv vs1, 16(r3)
-; CHECK-P9-NEXT:    stxv vs4, 0(r3)
 ; CHECK-P9-NEXT:    stxv vs2, 112(r3)
 ; CHECK-P9-NEXT:    stxv vs7, 96(r3)
-; CHECK-P9-NEXT:    stxv vs3, 80(r3)
-; CHECK-P9-NEXT:    stxv vs6, 64(r3)
+; CHECK-P9-NEXT:    stxv vs4, 48(r3)
+; CHECK-P9-NEXT:    stxv vs0, 16(r3)
+; CHECK-P9-NEXT:    stxv vs1, 0(r3)
 ; CHECK-P9-NEXT:    blr
 ;
 ; CHECK-BE-LABEL: test16elt_signed:
 ; CHECK-BE:       # %bb.0: # %entry
-; CHECK-BE-NEXT:    lxv vs0, 16(r4)
-; CHECK-BE-NEXT:    lxv vs1, 0(r4)
-; CHECK-BE-NEXT:    lxv vs2, 48(r4)
-; CHECK-BE-NEXT:    lxv vs3, 32(r4)
-; CHECK-BE-NEXT:    stfd f30, -16(r1) # 8-byte Folded Spill
-; CHECK-BE-NEXT:    stfd f31, -8(r1) # 8-byte Folded Spill
-; CHECK-BE-NEXT:    xxsldwi vs4, vs1, vs1, 1
-; CHECK-BE-NEXT:    xxsldwi vs5, vs1, vs1, 3
-; CHECK-BE-NEXT:    xxswapd vs6, vs1
-; CHECK-BE-NEXT:    xxsldwi vs7, vs0, vs0, 1
-; CHECK-BE-NEXT:    xxsldwi vs8, vs0, vs0, 3
-; CHECK-BE-NEXT:    xxswapd vs9, vs0
-; CHECK-BE-NEXT:    xxsldwi vs10, vs3, vs3, 1
-; CHECK-BE-NEXT:    xxsldwi vs11, vs3, vs3, 3
-; CHECK-BE-NEXT:    xxswapd vs12, vs3
-; CHECK-BE-NEXT:    xxsldwi vs13, vs2, vs2, 1
-; CHECK-BE-NEXT:    xxsldwi v2, vs2, vs2, 3
-; CHECK-BE-NEXT:    xxswapd v3, vs2
-; CHECK-BE-NEXT:    xscvspdpn f1, vs1
-; CHECK-BE-NEXT:    xscvspdpn f0, vs0
-; CHECK-BE-NEXT:    xscvspdpn f3, vs3
-; CHECK-BE-NEXT:    xscvspdpn f2, vs2
-; CHECK-BE-NEXT:    xscvspdpn f4, vs4
+; CHECK-BE-NEXT:    lxv vs0, 0(r4)
+; CHECK-BE-NEXT:    lxv vs4, 16(r4)
+; CHECK-BE-NEXT:    xxsldwi vs2, vs0, vs0, 1
+; CHECK-BE-NEXT:    xscvspdpn f1, vs0
+; CHECK-BE-NEXT:    xxsldwi vs5, vs0, vs0, 3
+; CHECK-BE-NEXT:    xxswapd vs0, vs0
 ; CHECK-BE-NEXT:    xscvspdpn f5, vs5
+; CHECK-BE-NEXT:    xscvspdpn f0, vs0
+; CHECK-BE-NEXT:    xxsldwi vs6, vs4, vs4, 1
 ; CHECK-BE-NEXT:    xscvspdpn f6, vs6
+; CHECK-BE-NEXT:    xxmrghd vs0, vs0, vs5
+; CHECK-BE-NEXT:    xscvspdpn f5, vs4
+; CHECK-BE-NEXT:    lxv vs3, 32(r4)
+; CHECK-BE-NEXT:    xxsldwi vs7, vs3, vs3, 1
 ; CHECK-BE-NEXT:    xscvspdpn f7, vs7
-; CHECK-BE-NEXT:    xscvspdpn f8, vs8
-; CHECK-BE-NEXT:    xscvspdpn f9, vs9
-; CHECK-BE-NEXT:    xscvspdpn f10, vs10
-; CHECK-BE-NEXT:    xscvspdpn f11, vs11
-; CHECK-BE-NEXT:    xscvspdpn f12, vs12
-; CHECK-BE-NEXT:    xscvspdpn f13, vs13
-; CHECK-BE-NEXT:    xscvspdpn f31, v2
-; CHECK-BE-NEXT:    xscvspdpn f30, v3
-; CHECK-BE-NEXT:    xxmrghd vs1, vs1, vs4
-; CHECK-BE-NEXT:    xxmrghd vs4, vs6, vs5
-; CHECK-BE-NEXT:    xxmrghd vs0, vs0, vs7
-; CHECK-BE-NEXT:    xxmrghd vs5, vs9, vs8
-; CHECK-BE-NEXT:    xxmrghd vs3, vs3, vs10
-; CHECK-BE-NEXT:    xxmrghd vs6, vs12, vs11
-; CHECK-BE-NEXT:    xxmrghd vs2, vs2, vs13
-; CHECK-BE-NEXT:    xxmrghd vs7, vs30, vs31
-; CHECK-BE-NEXT:    lfd f31, -8(r1) # 8-byte Folded Reload
-; CHECK-BE-NEXT:    lfd f30, -16(r1) # 8-byte Folded Reload
+; CHECK-BE-NEXT:    xxmrghd vs5, vs5, vs6
+; CHECK-BE-NEXT:    xxsldwi vs6, vs4, vs4, 3
+; CHECK-BE-NEXT:    xxswapd vs4, vs4
+; CHECK-BE-NEXT:    xscvspdpn f6, vs6
+; CHECK-BE-NEXT:    xscvspdpn f4, vs4
+; CHECK-BE-NEXT:    xscvspdpn f2, vs2
+; CHECK-BE-NEXT:    xxmrghd vs1, vs1, vs2
+; CHECK-BE-NEXT:    lxv vs2, 48(r4)
+; CHECK-BE-NEXT:    xxsldwi vs8, vs2, vs2, 1
 ; CHECK-BE-NEXT:    xvcvdpuxds vs1, vs1
-; CHECK-BE-NEXT:    xvcvdpuxds vs4, vs4
 ; CHECK-BE-NEXT:    xvcvdpuxds vs0, vs0
 ; CHECK-BE-NEXT:    xvcvdpuxds vs5, vs5
-; CHECK-BE-NEXT:    xvcvdpuxds vs3, vs3
+; CHECK-BE-NEXT:    xscvspdpn f8, vs8
+; CHECK-BE-NEXT:    xxmrghd vs4, vs4, vs6
+; CHECK-BE-NEXT:    xscvspdpn f6, vs3
+; CHECK-BE-NEXT:    stxv vs0, 16(r3)
+; CHECK-BE-NEXT:    xxmrghd vs6, vs6, vs7
+; CHECK-BE-NEXT:    xxsldwi vs7, vs3, vs3, 3
+; CHECK-BE-NEXT:    xxswapd vs3, vs3
+; CHECK-BE-NEXT:    xscvspdpn f7, vs7
+; CHECK-BE-NEXT:    xscvspdpn f3, vs3
+; CHECK-BE-NEXT:    xxmrghd vs3, vs3, vs7
+; CHECK-BE-NEXT:    xscvspdpn f7, vs2
+; CHECK-BE-NEXT:    xxmrghd vs7, vs7, vs8
+; CHECK-BE-NEXT:    xxsldwi vs8, vs2, vs2, 3
+; CHECK-BE-NEXT:    xxswapd vs2, vs2
+; CHECK-BE-NEXT:    xscvspdpn f8, vs8
+; CHECK-BE-NEXT:    xscvspdpn f2, vs2
+; CHECK-BE-NEXT:    xxmrghd vs2, vs2, vs8
+; CHECK-BE-NEXT:    stxv vs5, 32(r3)
+; CHECK-BE-NEXT:    xvcvdpuxds vs4, vs4
 ; CHECK-BE-NEXT:    xvcvdpuxds vs6, vs6
-; CHECK-BE-NEXT:    xvcvdpuxds vs2, vs2
+; CHECK-BE-NEXT:    xvcvdpuxds vs3, vs3
 ; CHECK-BE-NEXT:    xvcvdpuxds vs7, vs7
-; CHECK-BE-NEXT:    stxv vs5, 48(r3)
-; CHECK-BE-NEXT:    stxv vs0, 32(r3)
-; CHECK-BE-NEXT:    stxv vs4, 16(r3)
+; CHECK-BE-NEXT:    stxv vs3, 80(r3)
+; CHECK-BE-NEXT:    stxv vs7, 96(r3)
+; CHECK-BE-NEXT:    xvcvdpuxds vs2, vs2
+; CHECK-BE-NEXT:    stxv vs2, 112(r3)
+; CHECK-BE-NEXT:    stxv vs6, 64(r3)
+; CHECK-BE-NEXT:    stxv vs4, 48(r3)
 ; CHECK-BE-NEXT:    stxv vs1, 0(r3)
-; CHECK-BE-NEXT:    stxv vs7, 112(r3)
-; CHECK-BE-NEXT:    stxv vs2, 96(r3)
-; CHECK-BE-NEXT:    stxv vs6, 80(r3)
-; CHECK-BE-NEXT:    stxv vs3, 64(r3)
 ; CHECK-BE-NEXT:    blr
 entry:
   %a = load <16 x float>, <16 x float>* %0, align 64
diff --git a/test/CodeGen/PowerPC/vec_conv_fp32_to_i8_elts.ll b/test/CodeGen/PowerPC/vec_conv_fp32_to_i8_elts.ll
index 708e844b549df704d556a0f3f8281080307e0543..de38a82a3a8039e13b5f2623d1ceb008bcaf65c4 100644
--- a/test/CodeGen/PowerPC/vec_conv_fp32_to_i8_elts.ll
+++ b/test/CodeGen/PowerPC/vec_conv_fp32_to_i8_elts.ll
@@ -39,18 +39,18 @@ define i16 @test2elt(i64 %a.coerce) local_unnamed_addr #0 {
 ; CHECK-P9-NEXT:    xxswapd v2, vs0
 ; CHECK-P9-NEXT:    xscvspdpn f0, vs0
 ; CHECK-P9-NEXT:    xxsldwi vs1, v2, v2, 3
-; CHECK-P9-NEXT:    xscvdpsxws f0, f0
 ; CHECK-P9-NEXT:    xscvspdpn f1, vs1
 ; CHECK-P9-NEXT:    xscvdpsxws f1, f1
-; CHECK-P9-NEXT:    mfvsrwz r4, f0
+; CHECK-P9-NEXT:    xscvdpsxws f0, f0
 ; CHECK-P9-NEXT:    mfvsrwz r3, f1
-; CHECK-P9-NEXT:    mtvsrd f1, r4
+; CHECK-P9-NEXT:    mtvsrd f1, r3
+; CHECK-P9-NEXT:    mfvsrwz r3, f0
 ; CHECK-P9-NEXT:    mtvsrd f0, r3
-; CHECK-P9-NEXT:    xxswapd v3, vs1
-; CHECK-P9-NEXT:    addi r3, r1, -2
-; CHECK-P9-NEXT:    xxswapd v2, vs0
+; CHECK-P9-NEXT:    xxswapd v2, vs1
+; CHECK-P9-NEXT:    xxswapd v3, vs0
 ; CHECK-P9-NEXT:    vmrglb v2, v3, v2
 ; CHECK-P9-NEXT:    vsldoi v2, v2, v2, 8
+; CHECK-P9-NEXT:    addi r3, r1, -2
 ; CHECK-P9-NEXT:    stxsihx v2, 0, r3
 ; CHECK-P9-NEXT:    lhz r3, -2(r1)
 ; CHECK-P9-NEXT:    blr
@@ -58,18 +58,18 @@ define i16 @test2elt(i64 %a.coerce) local_unnamed_addr #0 {
 ; CHECK-BE-LABEL: test2elt:
 ; CHECK-BE:       # %bb.0: # %entry
 ; CHECK-BE-NEXT:    mtvsrd f0, r3
-; CHECK-BE-NEXT:    xxsldwi vs1, vs0, vs0, 1
+; CHECK-BE-NEXT:    xscvspdpn f1, vs0
+; CHECK-BE-NEXT:    xxsldwi vs0, vs0, vs0, 1
+; CHECK-BE-NEXT:    xscvdpsxws f1, f1
 ; CHECK-BE-NEXT:    xscvspdpn f0, vs0
-; CHECK-BE-NEXT:    xscvspdpn f1, vs1
 ; CHECK-BE-NEXT:    xscvdpsxws f0, f0
-; CHECK-BE-NEXT:    xscvdpsxws f1, f1
-; CHECK-BE-NEXT:    mfvsrwz r3, f0
+; CHECK-BE-NEXT:    mfvsrwz r3, f1
 ; CHECK-BE-NEXT:    sldi r3, r3, 56
-; CHECK-BE-NEXT:    mfvsrwz r4, f1
 ; CHECK-BE-NEXT:    mtvsrd v2, r3
+; CHECK-BE-NEXT:    mfvsrwz r3, f0
+; CHECK-BE-NEXT:    sldi r3, r3, 56
+; CHECK-BE-NEXT:    mtvsrd v3, r3
 ; CHECK-BE-NEXT:    addi r3, r1, -2
-; CHECK-BE-NEXT:    sldi r4, r4, 56
-; CHECK-BE-NEXT:    mtvsrd v3, r4
 ; CHECK-BE-NEXT:    vmrghb v2, v2, v3
 ; CHECK-BE-NEXT:    vsldoi v2, v2, v2, 10
 ; CHECK-BE-NEXT:    stxsihx v2, 0, r3
@@ -118,64 +118,64 @@ define i32 @test4elt(<4 x float> %a) local_unnamed_addr #1 {
 ; CHECK-P9-LABEL: test4elt:
 ; CHECK-P9:       # %bb.0: # %entry
 ; CHECK-P9-NEXT:    xxsldwi vs0, v2, v2, 3
-; CHECK-P9-NEXT:    xxswapd vs1, v2
-; CHECK-P9-NEXT:    xxsldwi vs2, v2, v2, 1
-; CHECK-P9-NEXT:    xscvspdpn f3, v2
 ; CHECK-P9-NEXT:    xscvspdpn f0, vs0
-; CHECK-P9-NEXT:    xscvspdpn f1, vs1
-; CHECK-P9-NEXT:    xscvspdpn f2, vs2
-; CHECK-P9-NEXT:    xscvdpsxws f3, f3
 ; CHECK-P9-NEXT:    xscvdpsxws f0, f0
-; CHECK-P9-NEXT:    xscvdpsxws f1, f1
-; CHECK-P9-NEXT:    xscvdpsxws f2, f2
-; CHECK-P9-NEXT:    mfvsrwz r5, f3
 ; CHECK-P9-NEXT:    mfvsrwz r3, f0
-; CHECK-P9-NEXT:    mfvsrwz r4, f1
-; CHECK-P9-NEXT:    mfvsrwz r6, f2
-; CHECK-P9-NEXT:    mtvsrd f2, r5
 ; CHECK-P9-NEXT:    mtvsrd f0, r3
-; CHECK-P9-NEXT:    mtvsrd f1, r4
-; CHECK-P9-NEXT:    mtvsrd f3, r6
-; CHECK-P9-NEXT:    xxswapd v4, vs2
+; CHECK-P9-NEXT:    xxswapd v3, vs0
+; CHECK-P9-NEXT:    xxswapd vs0, v2
+; CHECK-P9-NEXT:    xscvspdpn f0, vs0
+; CHECK-P9-NEXT:    xscvdpsxws f0, f0
+; CHECK-P9-NEXT:    mfvsrwz r3, f0
+; CHECK-P9-NEXT:    mtvsrd f0, r3
+; CHECK-P9-NEXT:    xxswapd v4, vs0
+; CHECK-P9-NEXT:    xscvspdpn f0, v2
+; CHECK-P9-NEXT:    xscvdpsxws f0, f0
+; CHECK-P9-NEXT:    mfvsrwz r3, f0
+; CHECK-P9-NEXT:    mtvsrd f0, r3
+; CHECK-P9-NEXT:    vmrglb v3, v4, v3
+; CHECK-P9-NEXT:    xxswapd v4, vs0
+; CHECK-P9-NEXT:    xxsldwi vs0, v2, v2, 1
+; CHECK-P9-NEXT:    xscvspdpn f0, vs0
+; CHECK-P9-NEXT:    xscvdpsxws f0, f0
+; CHECK-P9-NEXT:    mfvsrwz r3, f0
+; CHECK-P9-NEXT:    mtvsrd f0, r3
 ; CHECK-P9-NEXT:    li r3, 0
 ; CHECK-P9-NEXT:    xxswapd v2, vs0
-; CHECK-P9-NEXT:    xxswapd v3, vs1
-; CHECK-P9-NEXT:    xxswapd v5, vs3
-; CHECK-P9-NEXT:    vmrglb v2, v3, v2
-; CHECK-P9-NEXT:    vmrglb v3, v4, v5
-; CHECK-P9-NEXT:    vmrglh v2, v3, v2
+; CHECK-P9-NEXT:    vmrglb v2, v4, v2
+; CHECK-P9-NEXT:    vmrglh v2, v2, v3
 ; CHECK-P9-NEXT:    vextuwrx r3, r3, v2
 ; CHECK-P9-NEXT:    blr
 ;
 ; CHECK-BE-LABEL: test4elt:
 ; CHECK-BE:       # %bb.0: # %entry
 ; CHECK-BE-NEXT:    xxsldwi vs0, v2, v2, 3
-; CHECK-BE-NEXT:    xxswapd vs1, v2
-; CHECK-BE-NEXT:    xxsldwi vs2, v2, v2, 1
-; CHECK-BE-NEXT:    xscvspdpn f3, v2
 ; CHECK-BE-NEXT:    xscvspdpn f0, vs0
-; CHECK-BE-NEXT:    xscvspdpn f1, vs1
-; CHECK-BE-NEXT:    xscvspdpn f2, vs2
-; CHECK-BE-NEXT:    xscvdpsxws f3, f3
 ; CHECK-BE-NEXT:    xscvdpsxws f0, f0
-; CHECK-BE-NEXT:    xscvdpsxws f1, f1
-; CHECK-BE-NEXT:    xscvdpsxws f2, f2
-; CHECK-BE-NEXT:    mfvsrwz r5, f3
-; CHECK-BE-NEXT:    sldi r5, r5, 56
 ; CHECK-BE-NEXT:    mfvsrwz r3, f0
-; CHECK-BE-NEXT:    mfvsrwz r4, f1
-; CHECK-BE-NEXT:    mfvsrwz r6, f2
-; CHECK-BE-NEXT:    mtvsrd v4, r5
+; CHECK-BE-NEXT:    xxswapd vs0, v2
+; CHECK-BE-NEXT:    sldi r3, r3, 56
+; CHECK-BE-NEXT:    xscvspdpn f0, vs0
+; CHECK-BE-NEXT:    mtvsrd v3, r3
+; CHECK-BE-NEXT:    xscvdpsxws f0, f0
+; CHECK-BE-NEXT:    mfvsrwz r3, f0
+; CHECK-BE-NEXT:    xscvspdpn f0, v2
+; CHECK-BE-NEXT:    sldi r3, r3, 56
+; CHECK-BE-NEXT:    xscvdpsxws f0, f0
+; CHECK-BE-NEXT:    mtvsrd v4, r3
+; CHECK-BE-NEXT:    vmrghb v3, v4, v3
+; CHECK-BE-NEXT:    mfvsrwz r3, f0
+; CHECK-BE-NEXT:    xxsldwi vs0, v2, v2, 1
+; CHECK-BE-NEXT:    sldi r3, r3, 56
+; CHECK-BE-NEXT:    xscvspdpn f0, vs0
+; CHECK-BE-NEXT:    mtvsrd v4, r3
+; CHECK-BE-NEXT:    xscvdpsxws f0, f0
+; CHECK-BE-NEXT:    mfvsrwz r3, f0
 ; CHECK-BE-NEXT:    sldi r3, r3, 56
-; CHECK-BE-NEXT:    sldi r4, r4, 56
-; CHECK-BE-NEXT:    sldi r6, r6, 56
 ; CHECK-BE-NEXT:    mtvsrd v2, r3
-; CHECK-BE-NEXT:    mtvsrd v3, r4
-; CHECK-BE-NEXT:    mtvsrd v5, r6
 ; CHECK-BE-NEXT:    li r3, 0
-; CHECK-BE-NEXT:    vmrghb v2, v3, v2
-; CHECK-BE-NEXT:    vmrghb v3, v4, v5
-; CHECK-BE-NEXT:    vmrghh v2, v3, v2
+; CHECK-BE-NEXT:    vmrghb v2, v4, v2
+; CHECK-BE-NEXT:    vmrghh v2, v2, v3
 ; CHECK-BE-NEXT:    vextuwlx r3, r3, v2
 ; CHECK-BE-NEXT:    blr
 entry:
@@ -249,120 +249,120 @@ define i64 @test8elt(<8 x float>* nocapture readonly) local_unnamed_addr #2 {
 ;
 ; CHECK-P9-LABEL: test8elt:
 ; CHECK-P9:       # %bb.0: # %entry
-; CHECK-P9-NEXT:    lxv vs0, 16(r3)
 ; CHECK-P9-NEXT:    lxv vs1, 0(r3)
 ; CHECK-P9-NEXT:    xxsldwi vs2, vs1, vs1, 3
-; CHECK-P9-NEXT:    xxswapd vs3, vs1
-; CHECK-P9-NEXT:    xxsldwi vs4, vs1, vs1, 1
-; CHECK-P9-NEXT:    xxsldwi vs5, vs0, vs0, 3
-; CHECK-P9-NEXT:    xxswapd vs6, vs0
-; CHECK-P9-NEXT:    xxsldwi vs7, vs0, vs0, 1
-; CHECK-P9-NEXT:    xscvspdpn f1, vs1
-; CHECK-P9-NEXT:    xscvspdpn f0, vs0
 ; CHECK-P9-NEXT:    xscvspdpn f2, vs2
-; CHECK-P9-NEXT:    xscvspdpn f3, vs3
-; CHECK-P9-NEXT:    xscvspdpn f4, vs4
-; CHECK-P9-NEXT:    xscvspdpn f5, vs5
-; CHECK-P9-NEXT:    xscvspdpn f6, vs6
-; CHECK-P9-NEXT:    xscvspdpn f7, vs7
-; CHECK-P9-NEXT:    xscvdpsxws f1, f1
-; CHECK-P9-NEXT:    xscvdpsxws f0, f0
 ; CHECK-P9-NEXT:    xscvdpsxws f2, f2
-; CHECK-P9-NEXT:    xscvdpsxws f3, f3
-; CHECK-P9-NEXT:    xscvdpsxws f4, f4
-; CHECK-P9-NEXT:    xscvdpsxws f5, f5
-; CHECK-P9-NEXT:    xscvdpsxws f6, f6
-; CHECK-P9-NEXT:    xscvdpsxws f7, f7
-; CHECK-P9-NEXT:    mfvsrwz r5, f1
-; CHECK-P9-NEXT:    mfvsrwz r9, f0
+; CHECK-P9-NEXT:    lxv vs0, 16(r3)
 ; CHECK-P9-NEXT:    mfvsrwz r3, f2
-; CHECK-P9-NEXT:    mfvsrwz r4, f3
-; CHECK-P9-NEXT:    mfvsrwz r6, f4
-; CHECK-P9-NEXT:    mfvsrwz r7, f5
-; CHECK-P9-NEXT:    mfvsrwz r8, f6
-; CHECK-P9-NEXT:    mfvsrwz r10, f7
-; CHECK-P9-NEXT:    mtvsrd f2, r5
-; CHECK-P9-NEXT:    mtvsrd f6, r9
-; CHECK-P9-NEXT:    mtvsrd f0, r3
-; CHECK-P9-NEXT:    mtvsrd f1, r4
-; CHECK-P9-NEXT:    mtvsrd f3, r6
-; CHECK-P9-NEXT:    mtvsrd f4, r7
-; CHECK-P9-NEXT:    mtvsrd f5, r8
-; CHECK-P9-NEXT:    mtvsrd f7, r10
-; CHECK-P9-NEXT:    xxswapd v4, vs2
-; CHECK-P9-NEXT:    xxswapd v6, vs6
-; CHECK-P9-NEXT:    xxswapd v2, vs0
-; CHECK-P9-NEXT:    xxswapd v3, vs1
-; CHECK-P9-NEXT:    xxswapd v5, vs3
-; CHECK-P9-NEXT:    xxswapd v0, vs4
-; CHECK-P9-NEXT:    xxswapd v1, vs5
-; CHECK-P9-NEXT:    xxswapd v7, vs7
+; CHECK-P9-NEXT:    mtvsrd f2, r3
+; CHECK-P9-NEXT:    xxswapd v2, vs2
+; CHECK-P9-NEXT:    xxswapd vs2, vs1
+; CHECK-P9-NEXT:    xscvspdpn f2, vs2
+; CHECK-P9-NEXT:    xscvdpsxws f2, f2
+; CHECK-P9-NEXT:    mfvsrwz r3, f2
+; CHECK-P9-NEXT:    mtvsrd f2, r3
+; CHECK-P9-NEXT:    xxswapd v3, vs2
+; CHECK-P9-NEXT:    xscvspdpn f2, vs1
+; CHECK-P9-NEXT:    xxsldwi vs1, vs1, vs1, 1
+; CHECK-P9-NEXT:    xscvspdpn f1, vs1
+; CHECK-P9-NEXT:    xscvdpsxws f2, f2
+; CHECK-P9-NEXT:    xscvdpsxws f1, f1
+; CHECK-P9-NEXT:    mfvsrwz r3, f2
+; CHECK-P9-NEXT:    mtvsrd f2, r3
+; CHECK-P9-NEXT:    mfvsrwz r3, f1
+; CHECK-P9-NEXT:    mtvsrd f1, r3
+; CHECK-P9-NEXT:    xxswapd v4, vs1
+; CHECK-P9-NEXT:    xxsldwi vs1, vs0, vs0, 3
+; CHECK-P9-NEXT:    xscvspdpn f1, vs1
+; CHECK-P9-NEXT:    xscvdpsxws f1, f1
 ; CHECK-P9-NEXT:    vmrglb v2, v3, v2
-; CHECK-P9-NEXT:    vmrglb v3, v4, v5
-; CHECK-P9-NEXT:    vmrglb v4, v1, v0
-; CHECK-P9-NEXT:    vmrglb v5, v6, v7
+; CHECK-P9-NEXT:    xxswapd v3, vs2
+; CHECK-P9-NEXT:    vmrglb v3, v3, v4
 ; CHECK-P9-NEXT:    vmrglh v2, v3, v2
-; CHECK-P9-NEXT:    vmrglh v3, v5, v4
+; CHECK-P9-NEXT:    mfvsrwz r3, f1
+; CHECK-P9-NEXT:    mtvsrd f1, r3
+; CHECK-P9-NEXT:    xxswapd v3, vs1
+; CHECK-P9-NEXT:    xxswapd vs1, vs0
+; CHECK-P9-NEXT:    xscvspdpn f1, vs1
+; CHECK-P9-NEXT:    xscvdpsxws f1, f1
+; CHECK-P9-NEXT:    mfvsrwz r3, f1
+; CHECK-P9-NEXT:    mtvsrd f1, r3
+; CHECK-P9-NEXT:    xxswapd v4, vs1
+; CHECK-P9-NEXT:    xscvspdpn f1, vs0
+; CHECK-P9-NEXT:    xxsldwi vs0, vs0, vs0, 1
+; CHECK-P9-NEXT:    xscvspdpn f0, vs0
+; CHECK-P9-NEXT:    xscvdpsxws f1, f1
+; CHECK-P9-NEXT:    xscvdpsxws f0, f0
+; CHECK-P9-NEXT:    mfvsrwz r3, f1
+; CHECK-P9-NEXT:    mtvsrd f1, r3
+; CHECK-P9-NEXT:    mfvsrwz r3, f0
+; CHECK-P9-NEXT:    mtvsrd f0, r3
+; CHECK-P9-NEXT:    vmrglb v3, v4, v3
+; CHECK-P9-NEXT:    xxswapd v4, vs1
+; CHECK-P9-NEXT:    xxswapd v5, vs0
+; CHECK-P9-NEXT:    vmrglb v4, v4, v5
+; CHECK-P9-NEXT:    vmrglh v3, v4, v3
 ; CHECK-P9-NEXT:    vmrglw v2, v3, v2
 ; CHECK-P9-NEXT:    mfvsrld r3, v2
 ; CHECK-P9-NEXT:    blr
 ;
 ; CHECK-BE-LABEL: test8elt:
 ; CHECK-BE:       # %bb.0: # %entry
-; CHECK-BE-NEXT:    lxv vs0, 0(r3)
 ; CHECK-BE-NEXT:    lxv vs1, 16(r3)
 ; CHECK-BE-NEXT:    xxsldwi vs2, vs1, vs1, 3
-; CHECK-BE-NEXT:    xxswapd vs3, vs1
-; CHECK-BE-NEXT:    xxsldwi vs4, vs1, vs1, 1
-; CHECK-BE-NEXT:    xxsldwi vs5, vs0, vs0, 3
-; CHECK-BE-NEXT:    xxswapd vs6, vs0
-; CHECK-BE-NEXT:    xxsldwi vs7, vs0, vs0, 1
-; CHECK-BE-NEXT:    xscvspdpn f1, vs1
-; CHECK-BE-NEXT:    xscvspdpn f0, vs0
 ; CHECK-BE-NEXT:    xscvspdpn f2, vs2
-; CHECK-BE-NEXT:    xscvspdpn f3, vs3
-; CHECK-BE-NEXT:    xscvspdpn f4, vs4
-; CHECK-BE-NEXT:    xscvspdpn f5, vs5
-; CHECK-BE-NEXT:    xscvspdpn f6, vs6
-; CHECK-BE-NEXT:    xscvspdpn f7, vs7
-; CHECK-BE-NEXT:    xscvdpsxws f1, f1
-; CHECK-BE-NEXT:    xscvdpsxws f0, f0
 ; CHECK-BE-NEXT:    xscvdpsxws f2, f2
-; CHECK-BE-NEXT:    xscvdpsxws f3, f3
-; CHECK-BE-NEXT:    xscvdpsxws f4, f4
-; CHECK-BE-NEXT:    xscvdpsxws f5, f5
-; CHECK-BE-NEXT:    xscvdpsxws f6, f6
-; CHECK-BE-NEXT:    xscvdpsxws f7, f7
-; CHECK-BE-NEXT:    mfvsrwz r5, f1
-; CHECK-BE-NEXT:    mfvsrwz r9, f0
-; CHECK-BE-NEXT:    sldi r5, r5, 56
-; CHECK-BE-NEXT:    sldi r9, r9, 56
+; CHECK-BE-NEXT:    lxv vs0, 0(r3)
 ; CHECK-BE-NEXT:    mfvsrwz r3, f2
-; CHECK-BE-NEXT:    mfvsrwz r4, f3
-; CHECK-BE-NEXT:    mfvsrwz r6, f4
-; CHECK-BE-NEXT:    mfvsrwz r7, f5
-; CHECK-BE-NEXT:    mfvsrwz r8, f6
-; CHECK-BE-NEXT:    mfvsrwz r10, f7
-; CHECK-BE-NEXT:    mtvsrd v4, r5
-; CHECK-BE-NEXT:    mtvsrd v6, r9
+; CHECK-BE-NEXT:    xxswapd vs2, vs1
 ; CHECK-BE-NEXT:    sldi r3, r3, 56
-; CHECK-BE-NEXT:    sldi r4, r4, 56
-; CHECK-BE-NEXT:    sldi r6, r6, 56
-; CHECK-BE-NEXT:    sldi r7, r7, 56
-; CHECK-BE-NEXT:    sldi r8, r8, 56
-; CHECK-BE-NEXT:    sldi r10, r10, 56
+; CHECK-BE-NEXT:    xscvspdpn f2, vs2
 ; CHECK-BE-NEXT:    mtvsrd v2, r3
-; CHECK-BE-NEXT:    mtvsrd v3, r4
-; CHECK-BE-NEXT:    mtvsrd v5, r6
-; CHECK-BE-NEXT:    mtvsrd v0, r7
-; CHECK-BE-NEXT:    mtvsrd v1, r8
-; CHECK-BE-NEXT:    mtvsrd v7, r10
+; CHECK-BE-NEXT:    xscvdpsxws f2, f2
+; CHECK-BE-NEXT:    mfvsrwz r3, f2
+; CHECK-BE-NEXT:    xscvspdpn f2, vs1
+; CHECK-BE-NEXT:    xxsldwi vs1, vs1, vs1, 1
+; CHECK-BE-NEXT:    sldi r3, r3, 56
+; CHECK-BE-NEXT:    xscvdpsxws f2, f2
+; CHECK-BE-NEXT:    xscvspdpn f1, vs1
+; CHECK-BE-NEXT:    mtvsrd v3, r3
+; CHECK-BE-NEXT:    xscvdpsxws f1, f1
 ; CHECK-BE-NEXT:    vmrghb v2, v3, v2
-; CHECK-BE-NEXT:    vmrghb v3, v4, v5
-; CHECK-BE-NEXT:    vmrghb v4, v1, v0
-; CHECK-BE-NEXT:    vmrghb v5, v6, v7
+; CHECK-BE-NEXT:    mfvsrwz r3, f2
+; CHECK-BE-NEXT:    sldi r3, r3, 56
+; CHECK-BE-NEXT:    mtvsrd v3, r3
+; CHECK-BE-NEXT:    mfvsrwz r3, f1
+; CHECK-BE-NEXT:    xxsldwi vs1, vs0, vs0, 3
+; CHECK-BE-NEXT:    sldi r3, r3, 56
+; CHECK-BE-NEXT:    xscvspdpn f1, vs1
+; CHECK-BE-NEXT:    mtvsrd v4, r3
+; CHECK-BE-NEXT:    xscvdpsxws f1, f1
+; CHECK-BE-NEXT:    mfvsrwz r3, f1
+; CHECK-BE-NEXT:    xxswapd vs1, vs0
+; CHECK-BE-NEXT:    xscvspdpn f1, vs1
+; CHECK-BE-NEXT:    xscvdpsxws f1, f1
+; CHECK-BE-NEXT:    vmrghb v3, v3, v4
+; CHECK-BE-NEXT:    sldi r3, r3, 56
 ; CHECK-BE-NEXT:    vmrghh v2, v3, v2
-; CHECK-BE-NEXT:    vmrghh v3, v5, v4
+; CHECK-BE-NEXT:    mtvsrd v3, r3
+; CHECK-BE-NEXT:    mfvsrwz r3, f1
+; CHECK-BE-NEXT:    xscvspdpn f1, vs0
+; CHECK-BE-NEXT:    xxsldwi vs0, vs0, vs0, 1
+; CHECK-BE-NEXT:    sldi r3, r3, 56
+; CHECK-BE-NEXT:    xscvdpsxws f1, f1
+; CHECK-BE-NEXT:    xscvspdpn f0, vs0
+; CHECK-BE-NEXT:    mtvsrd v4, r3
+; CHECK-BE-NEXT:    xscvdpsxws f0, f0
+; CHECK-BE-NEXT:    vmrghb v3, v4, v3
+; CHECK-BE-NEXT:    mfvsrwz r3, f1
+; CHECK-BE-NEXT:    sldi r3, r3, 56
+; CHECK-BE-NEXT:    mtvsrd v4, r3
+; CHECK-BE-NEXT:    mfvsrwz r3, f0
+; CHECK-BE-NEXT:    sldi r3, r3, 56
+; CHECK-BE-NEXT:    mtvsrd v5, r3
+; CHECK-BE-NEXT:    vmrghb v4, v4, v5
+; CHECK-BE-NEXT:    vmrghh v3, v4, v3
 ; CHECK-BE-NEXT:    vmrghw v2, v3, v2
 ; CHECK-BE-NEXT:    mfvsrd r3, v2
 ; CHECK-BE-NEXT:    blr
@@ -494,251 +494,231 @@ define <16 x i8> @test16elt(<16 x float>* nocapture readonly) local_unnamed_addr
 ;
 ; CHECK-P9-LABEL: test16elt:
 ; CHECK-P9:       # %bb.0: # %entry
-; CHECK-P9-NEXT:    lxv vs2, 16(r3)
-; CHECK-P9-NEXT:    lxv vs3, 0(r3)
-; CHECK-P9-NEXT:    lxv vs0, 48(r3)
-; CHECK-P9-NEXT:    lxv vs1, 32(r3)
-; CHECK-P9-NEXT:    std r26, -48(r1) # 8-byte Folded Spill
-; CHECK-P9-NEXT:    std r27, -40(r1) # 8-byte Folded Spill
-; CHECK-P9-NEXT:    std r28, -32(r1) # 8-byte Folded Spill
-; CHECK-P9-NEXT:    std r29, -24(r1) # 8-byte Folded Spill
-; CHECK-P9-NEXT:    std r30, -16(r1) # 8-byte Folded Spill
-; CHECK-P9-NEXT:    xxsldwi vs4, vs3, vs3, 3
-; CHECK-P9-NEXT:    xxswapd vs5, vs3
-; CHECK-P9-NEXT:    xxsldwi vs6, vs3, vs3, 1
-; CHECK-P9-NEXT:    xxsldwi vs7, vs2, vs2, 3
-; CHECK-P9-NEXT:    xxswapd vs8, vs2
-; CHECK-P9-NEXT:    xxsldwi vs9, vs2, vs2, 1
-; CHECK-P9-NEXT:    xxsldwi vs10, vs1, vs1, 3
-; CHECK-P9-NEXT:    xxswapd vs11, vs1
-; CHECK-P9-NEXT:    xxsldwi vs12, vs1, vs1, 1
-; CHECK-P9-NEXT:    xxsldwi vs13, vs0, vs0, 3
-; CHECK-P9-NEXT:    xxswapd v2, vs0
-; CHECK-P9-NEXT:    xxsldwi v3, vs0, vs0, 1
-; CHECK-P9-NEXT:    xscvspdpn f3, vs3
-; CHECK-P9-NEXT:    xscvspdpn f2, vs2
+; CHECK-P9-NEXT:    lxv vs0, 0(r3)
+; CHECK-P9-NEXT:    xxsldwi vs1, vs0, vs0, 3
 ; CHECK-P9-NEXT:    xscvspdpn f1, vs1
+; CHECK-P9-NEXT:    xscvdpsxws f1, f1
+; CHECK-P9-NEXT:    mfvsrwz r4, f1
+; CHECK-P9-NEXT:    mtvsrd f1, r4
+; CHECK-P9-NEXT:    xxswapd v2, vs1
+; CHECK-P9-NEXT:    xxswapd vs1, vs0
+; CHECK-P9-NEXT:    xscvspdpn f1, vs1
+; CHECK-P9-NEXT:    xscvdpsxws f1, f1
+; CHECK-P9-NEXT:    lxv vs2, 48(r3)
+; CHECK-P9-NEXT:    lxv vs3, 32(r3)
+; CHECK-P9-NEXT:    lxv vs4, 16(r3)
+; CHECK-P9-NEXT:    mfvsrwz r3, f1
+; CHECK-P9-NEXT:    mtvsrd f1, r3
+; CHECK-P9-NEXT:    xxswapd v3, vs1
+; CHECK-P9-NEXT:    xscvspdpn f1, vs0
+; CHECK-P9-NEXT:    xxsldwi vs0, vs0, vs0, 1
 ; CHECK-P9-NEXT:    xscvspdpn f0, vs0
-; CHECK-P9-NEXT:    xscvspdpn f4, vs4
-; CHECK-P9-NEXT:    xscvspdpn f5, vs5
-; CHECK-P9-NEXT:    xscvspdpn f6, vs6
-; CHECK-P9-NEXT:    xscvspdpn f7, vs7
-; CHECK-P9-NEXT:    xscvspdpn f8, vs8
-; CHECK-P9-NEXT:    xscvspdpn f9, vs9
-; CHECK-P9-NEXT:    xscvspdpn f10, vs10
-; CHECK-P9-NEXT:    xscvspdpn f11, vs11
-; CHECK-P9-NEXT:    xscvspdpn f12, vs12
-; CHECK-P9-NEXT:    xscvspdpn f13, vs13
-; CHECK-P9-NEXT:    xscvspdpn v2, v2
-; CHECK-P9-NEXT:    xscvspdpn v3, v3
-; CHECK-P9-NEXT:    xscvdpsxws f3, f3
-; CHECK-P9-NEXT:    xscvdpsxws f2, f2
 ; CHECK-P9-NEXT:    xscvdpsxws f1, f1
 ; CHECK-P9-NEXT:    xscvdpsxws f0, f0
-; CHECK-P9-NEXT:    xscvdpsxws f4, f4
-; CHECK-P9-NEXT:    xscvdpsxws f5, f5
-; CHECK-P9-NEXT:    xscvdpsxws f6, f6
-; CHECK-P9-NEXT:    xscvdpsxws f7, f7
-; CHECK-P9-NEXT:    xscvdpsxws f8, f8
-; CHECK-P9-NEXT:    xscvdpsxws f9, f9
-; CHECK-P9-NEXT:    xscvdpsxws f10, f10
-; CHECK-P9-NEXT:    xscvdpsxws f11, f11
-; CHECK-P9-NEXT:    xscvdpsxws f12, f12
-; CHECK-P9-NEXT:    xscvdpsxws f13, f13
-; CHECK-P9-NEXT:    xscvdpsxws v2, v2
-; CHECK-P9-NEXT:    xscvdpsxws v3, v3
-; CHECK-P9-NEXT:    mfvsrwz r3, f3
-; CHECK-P9-NEXT:    mfvsrwz r4, f2
-; CHECK-P9-NEXT:    mfvsrwz r11, f1
-; CHECK-P9-NEXT:    mfvsrwz r12, f0
-; CHECK-P9-NEXT:    mfvsrwz r5, f4
-; CHECK-P9-NEXT:    mfvsrwz r6, f5
-; CHECK-P9-NEXT:    mfvsrwz r7, f6
-; CHECK-P9-NEXT:    mfvsrwz r8, f7
-; CHECK-P9-NEXT:    mfvsrwz r9, f8
-; CHECK-P9-NEXT:    mfvsrwz r10, f9
-; CHECK-P9-NEXT:    mfvsrwz r0, f10
-; CHECK-P9-NEXT:    mfvsrwz r30, f11
-; CHECK-P9-NEXT:    mfvsrwz r29, f12
-; CHECK-P9-NEXT:    mfvsrwz r28, f13
-; CHECK-P9-NEXT:    mfvsrwz r27, v2
-; CHECK-P9-NEXT:    mfvsrwz r26, v3
+; CHECK-P9-NEXT:    mfvsrwz r3, f1
+; CHECK-P9-NEXT:    mtvsrd f1, r3
+; CHECK-P9-NEXT:    mfvsrwz r3, f0
 ; CHECK-P9-NEXT:    mtvsrd f0, r3
-; CHECK-P9-NEXT:    mtvsrd f1, r4
-; CHECK-P9-NEXT:    mtvsrd f8, r11
-; CHECK-P9-NEXT:    mtvsrd f9, r12
-; CHECK-P9-NEXT:    mtvsrd f2, r5
-; CHECK-P9-NEXT:    mtvsrd f3, r6
-; CHECK-P9-NEXT:    mtvsrd f4, r7
-; CHECK-P9-NEXT:    mtvsrd f5, r8
-; CHECK-P9-NEXT:    mtvsrd f6, r9
-; CHECK-P9-NEXT:    mtvsrd f7, r10
-; CHECK-P9-NEXT:    mtvsrd f10, r0
-; CHECK-P9-NEXT:    mtvsrd f11, r30
-; CHECK-P9-NEXT:    ld r30, -16(r1) # 8-byte Folded Reload
-; CHECK-P9-NEXT:    mtvsrd f12, r29
-; CHECK-P9-NEXT:    ld r29, -24(r1) # 8-byte Folded Reload
-; CHECK-P9-NEXT:    mtvsrd f13, r28
-; CHECK-P9-NEXT:    ld r28, -32(r1) # 8-byte Folded Reload
-; CHECK-P9-NEXT:    mtvsrd v2, r27
-; CHECK-P9-NEXT:    ld r27, -40(r1) # 8-byte Folded Reload
-; CHECK-P9-NEXT:    mtvsrd v3, r26
-; CHECK-P9-NEXT:    ld r26, -48(r1) # 8-byte Folded Reload
 ; CHECK-P9-NEXT:    xxswapd v4, vs0
-; CHECK-P9-NEXT:    xxswapd v5, vs2
-; CHECK-P9-NEXT:    xxswapd v0, vs3
-; CHECK-P9-NEXT:    xxswapd v1, vs4
-; CHECK-P9-NEXT:    xxswapd v6, vs5
-; CHECK-P9-NEXT:    xxswapd v7, vs6
-; CHECK-P9-NEXT:    xxswapd v8, vs1
-; CHECK-P9-NEXT:    xxswapd v9, vs7
-; CHECK-P9-NEXT:    xxswapd v10, vs10
-; CHECK-P9-NEXT:    xxswapd v11, vs11
-; CHECK-P9-NEXT:    xxswapd v12, vs8
-; CHECK-P9-NEXT:    xxswapd v13, vs12
-; CHECK-P9-NEXT:    xxswapd v14, vs13
-; CHECK-P9-NEXT:    xxswapd v2, v2
-; CHECK-P9-NEXT:    xxswapd v15, vs9
-; CHECK-P9-NEXT:    xxswapd v3, v3
-; CHECK-P9-NEXT:    vmrglb v5, v0, v5
-; CHECK-P9-NEXT:    vmrglb v4, v4, v1
-; CHECK-P9-NEXT:    vmrglb v0, v7, v6
-; CHECK-P9-NEXT:    vmrglb v1, v8, v9
-; CHECK-P9-NEXT:    vmrglb v6, v11, v10
-; CHECK-P9-NEXT:    vmrglb v7, v12, v13
-; CHECK-P9-NEXT:    vmrglb v2, v2, v14
-; CHECK-P9-NEXT:    vmrglb v3, v15, v3
-; CHECK-P9-NEXT:    vmrglh v4, v4, v5
-; CHECK-P9-NEXT:    vmrglh v5, v1, v0
-; CHECK-P9-NEXT:    vmrglh v0, v7, v6
+; CHECK-P9-NEXT:    xxsldwi vs0, vs4, vs4, 3
+; CHECK-P9-NEXT:    xscvspdpn f0, vs0
+; CHECK-P9-NEXT:    xscvdpsxws f0, f0
+; CHECK-P9-NEXT:    vmrglb v2, v3, v2
+; CHECK-P9-NEXT:    xxswapd v3, vs1
+; CHECK-P9-NEXT:    vmrglb v3, v3, v4
 ; CHECK-P9-NEXT:    vmrglh v2, v3, v2
-; CHECK-P9-NEXT:    vmrglw v3, v5, v4
-; CHECK-P9-NEXT:    vmrglw v2, v2, v0
-; CHECK-P9-NEXT:    xxmrgld v2, v2, v3
+; CHECK-P9-NEXT:    mfvsrwz r3, f0
+; CHECK-P9-NEXT:    mtvsrd f0, r3
+; CHECK-P9-NEXT:    xxswapd v3, vs0
+; CHECK-P9-NEXT:    xxswapd vs0, vs4
+; CHECK-P9-NEXT:    xscvspdpn f0, vs0
+; CHECK-P9-NEXT:    xscvdpsxws f0, f0
+; CHECK-P9-NEXT:    mfvsrwz r3, f0
+; CHECK-P9-NEXT:    mtvsrd f0, r3
+; CHECK-P9-NEXT:    xxswapd v4, vs0
+; CHECK-P9-NEXT:    xscvspdpn f0, vs4
+; CHECK-P9-NEXT:    xscvdpsxws f0, f0
+; CHECK-P9-NEXT:    mfvsrwz r3, f0
+; CHECK-P9-NEXT:    mtvsrd f0, r3
+; CHECK-P9-NEXT:    vmrglb v3, v4, v3
+; CHECK-P9-NEXT:    xxswapd v4, vs0
+; CHECK-P9-NEXT:    xxsldwi vs0, vs4, vs4, 1
+; CHECK-P9-NEXT:    xscvspdpn f0, vs0
+; CHECK-P9-NEXT:    xscvdpsxws f0, f0
+; CHECK-P9-NEXT:    mfvsrwz r3, f0
+; CHECK-P9-NEXT:    mtvsrd f0, r3
+; CHECK-P9-NEXT:    xxswapd v5, vs0
+; CHECK-P9-NEXT:    xxsldwi vs0, vs3, vs3, 3
+; CHECK-P9-NEXT:    xscvspdpn f0, vs0
+; CHECK-P9-NEXT:    xscvdpsxws f0, f0
+; CHECK-P9-NEXT:    vmrglb v4, v4, v5
+; CHECK-P9-NEXT:    vmrglh v3, v4, v3
+; CHECK-P9-NEXT:    vmrglw v2, v3, v2
+; CHECK-P9-NEXT:    mfvsrwz r3, f0
+; CHECK-P9-NEXT:    mtvsrd f0, r3
+; CHECK-P9-NEXT:    xxswapd v3, vs0
+; CHECK-P9-NEXT:    xxswapd vs0, vs3
+; CHECK-P9-NEXT:    xscvspdpn f0, vs0
+; CHECK-P9-NEXT:    xscvdpsxws f0, f0
+; CHECK-P9-NEXT:    mfvsrwz r3, f0
+; CHECK-P9-NEXT:    mtvsrd f0, r3
+; CHECK-P9-NEXT:    xxswapd v4, vs0
+; CHECK-P9-NEXT:    xscvspdpn f0, vs3
+; CHECK-P9-NEXT:    xscvdpsxws f0, f0
+; CHECK-P9-NEXT:    mfvsrwz r3, f0
+; CHECK-P9-NEXT:    mtvsrd f0, r3
+; CHECK-P9-NEXT:    vmrglb v3, v4, v3
+; CHECK-P9-NEXT:    xxswapd v4, vs0
+; CHECK-P9-NEXT:    xxsldwi vs0, vs3, vs3, 1
+; CHECK-P9-NEXT:    xscvspdpn f0, vs0
+; CHECK-P9-NEXT:    xscvdpsxws f0, f0
+; CHECK-P9-NEXT:    mfvsrwz r3, f0
+; CHECK-P9-NEXT:    mtvsrd f0, r3
+; CHECK-P9-NEXT:    xxswapd v5, vs0
+; CHECK-P9-NEXT:    xxsldwi vs0, vs2, vs2, 3
+; CHECK-P9-NEXT:    xscvspdpn f0, vs0
+; CHECK-P9-NEXT:    xscvdpsxws f0, f0
+; CHECK-P9-NEXT:    mfvsrwz r3, f0
+; CHECK-P9-NEXT:    vmrglb v4, v4, v5
+; CHECK-P9-NEXT:    vmrglh v3, v4, v3
+; CHECK-P9-NEXT:    mtvsrd f0, r3
+; CHECK-P9-NEXT:    xxswapd v4, vs0
+; CHECK-P9-NEXT:    xxswapd vs0, vs2
+; CHECK-P9-NEXT:    xscvspdpn f0, vs0
+; CHECK-P9-NEXT:    xscvdpsxws f0, f0
+; CHECK-P9-NEXT:    mfvsrwz r3, f0
+; CHECK-P9-NEXT:    mtvsrd f0, r3
+; CHECK-P9-NEXT:    xxswapd v5, vs0
+; CHECK-P9-NEXT:    xscvspdpn f0, vs2
+; CHECK-P9-NEXT:    xscvdpsxws f0, f0
+; CHECK-P9-NEXT:    mfvsrwz r3, f0
+; CHECK-P9-NEXT:    mtvsrd f0, r3
+; CHECK-P9-NEXT:    vmrglb v4, v5, v4
+; CHECK-P9-NEXT:    xxswapd v5, vs0
+; CHECK-P9-NEXT:    xxsldwi vs0, vs2, vs2, 1
+; CHECK-P9-NEXT:    xscvspdpn f0, vs0
+; CHECK-P9-NEXT:    xscvdpsxws f0, f0
+; CHECK-P9-NEXT:    mfvsrwz r3, f0
+; CHECK-P9-NEXT:    mtvsrd f0, r3
+; CHECK-P9-NEXT:    xxswapd v0, vs0
+; CHECK-P9-NEXT:    vmrglb v5, v5, v0
+; CHECK-P9-NEXT:    vmrglh v4, v5, v4
+; CHECK-P9-NEXT:    vmrglw v3, v4, v3
+; CHECK-P9-NEXT:    xxmrgld v2, v3, v2
 ; CHECK-P9-NEXT:    blr
 ;
 ; CHECK-BE-LABEL: test16elt:
 ; CHECK-BE:       # %bb.0: # %entry
-; CHECK-BE-NEXT:    lxv vs2, 32(r3)
 ; CHECK-BE-NEXT:    lxv vs3, 48(r3)
+; CHECK-BE-NEXT:    xxsldwi vs4, vs3, vs3, 3
+; CHECK-BE-NEXT:    xscvspdpn f4, vs4
+; CHECK-BE-NEXT:    xscvdpsxws f4, f4
 ; CHECK-BE-NEXT:    lxv vs0, 0(r3)
 ; CHECK-BE-NEXT:    lxv vs1, 16(r3)
-; CHECK-BE-NEXT:    std r26, -48(r1) # 8-byte Folded Spill
-; CHECK-BE-NEXT:    std r27, -40(r1) # 8-byte Folded Spill
-; CHECK-BE-NEXT:    std r28, -32(r1) # 8-byte Folded Spill
-; CHECK-BE-NEXT:    std r29, -24(r1) # 8-byte Folded Spill
-; CHECK-BE-NEXT:    std r30, -16(r1) # 8-byte Folded Spill
-; CHECK-BE-NEXT:    xxsldwi vs4, vs3, vs3, 3
-; CHECK-BE-NEXT:    xxswapd vs5, vs3
-; CHECK-BE-NEXT:    xxsldwi vs6, vs3, vs3, 1
-; CHECK-BE-NEXT:    xxsldwi vs7, vs2, vs2, 3
-; CHECK-BE-NEXT:    xxswapd vs8, vs2
-; CHECK-BE-NEXT:    xxsldwi vs9, vs2, vs2, 1
-; CHECK-BE-NEXT:    xxsldwi vs10, vs1, vs1, 3
-; CHECK-BE-NEXT:    xxswapd vs11, vs1
-; CHECK-BE-NEXT:    xxsldwi vs12, vs1, vs1, 1
-; CHECK-BE-NEXT:    xxsldwi vs13, vs0, vs0, 3
-; CHECK-BE-NEXT:    xxswapd v2, vs0
-; CHECK-BE-NEXT:    xxsldwi v3, vs0, vs0, 1
-; CHECK-BE-NEXT:    xscvspdpn f3, vs3
-; CHECK-BE-NEXT:    xscvspdpn f2, vs2
-; CHECK-BE-NEXT:    xscvspdpn f1, vs1
-; CHECK-BE-NEXT:    xscvspdpn f0, vs0
+; CHECK-BE-NEXT:    lxv vs2, 32(r3)
+; CHECK-BE-NEXT:    mfvsrwz r3, f4
+; CHECK-BE-NEXT:    xxswapd vs4, vs3
+; CHECK-BE-NEXT:    sldi r3, r3, 56
 ; CHECK-BE-NEXT:    xscvspdpn f4, vs4
-; CHECK-BE-NEXT:    xscvspdpn f5, vs5
-; CHECK-BE-NEXT:    xscvspdpn f6, vs6
-; CHECK-BE-NEXT:    xscvspdpn f7, vs7
-; CHECK-BE-NEXT:    xscvspdpn f8, vs8
-; CHECK-BE-NEXT:    xscvspdpn f9, vs9
-; CHECK-BE-NEXT:    xscvspdpn f10, vs10
-; CHECK-BE-NEXT:    xscvspdpn f11, vs11
-; CHECK-BE-NEXT:    xscvspdpn f12, vs12
-; CHECK-BE-NEXT:    xscvspdpn f13, vs13
-; CHECK-BE-NEXT:    xscvspdpn v2, v2
-; CHECK-BE-NEXT:    xscvspdpn v3, v3
+; CHECK-BE-NEXT:    mtvsrd v2, r3
+; CHECK-BE-NEXT:    xscvdpsxws f4, f4
+; CHECK-BE-NEXT:    mfvsrwz r3, f4
+; CHECK-BE-NEXT:    xscvspdpn f4, vs3
+; CHECK-BE-NEXT:    xxsldwi vs3, vs3, vs3, 1
+; CHECK-BE-NEXT:    sldi r3, r3, 56
+; CHECK-BE-NEXT:    xscvdpsxws f4, f4
+; CHECK-BE-NEXT:    xscvspdpn f3, vs3
+; CHECK-BE-NEXT:    mtvsrd v3, r3
 ; CHECK-BE-NEXT:    xscvdpsxws f3, f3
+; CHECK-BE-NEXT:    vmrghb v2, v3, v2
+; CHECK-BE-NEXT:    mfvsrwz r3, f4
+; CHECK-BE-NEXT:    sldi r3, r3, 56
+; CHECK-BE-NEXT:    mtvsrd v3, r3
+; CHECK-BE-NEXT:    mfvsrwz r3, f3
+; CHECK-BE-NEXT:    xxsldwi vs3, vs2, vs2, 3
+; CHECK-BE-NEXT:    sldi r3, r3, 56
+; CHECK-BE-NEXT:    xscvspdpn f3, vs3
+; CHECK-BE-NEXT:    mtvsrd v4, r3
+; CHECK-BE-NEXT:    xscvdpsxws f3, f3
+; CHECK-BE-NEXT:    mfvsrwz r3, f3
+; CHECK-BE-NEXT:    xxswapd vs3, vs2
+; CHECK-BE-NEXT:    xscvspdpn f3, vs3
+; CHECK-BE-NEXT:    xscvdpsxws f3, f3
+; CHECK-BE-NEXT:    vmrghb v3, v3, v4
+; CHECK-BE-NEXT:    sldi r3, r3, 56
+; CHECK-BE-NEXT:    vmrghh v2, v3, v2
+; CHECK-BE-NEXT:    mtvsrd v3, r3
+; CHECK-BE-NEXT:    mfvsrwz r3, f3
+; CHECK-BE-NEXT:    xscvspdpn f3, vs2
+; CHECK-BE-NEXT:    xxsldwi vs2, vs2, vs2, 1
+; CHECK-BE-NEXT:    sldi r3, r3, 56
+; CHECK-BE-NEXT:    xscvdpsxws f3, f3
+; CHECK-BE-NEXT:    xscvspdpn f2, vs2
+; CHECK-BE-NEXT:    mtvsrd v4, r3
 ; CHECK-BE-NEXT:    xscvdpsxws f2, f2
-; CHECK-BE-NEXT:    xscvdpsxws f1, f1
-; CHECK-BE-NEXT:    xscvdpsxws f0, f0
-; CHECK-BE-NEXT:    xscvdpsxws f4, f4
-; CHECK-BE-NEXT:    xscvdpsxws f5, f5
-; CHECK-BE-NEXT:    xscvdpsxws f6, f6
-; CHECK-BE-NEXT:    xscvdpsxws f7, f7
-; CHECK-BE-NEXT:    xscvdpsxws f8, f8
-; CHECK-BE-NEXT:    xscvdpsxws f9, f9
-; CHECK-BE-NEXT:    xscvdpsxws f10, f10
-; CHECK-BE-NEXT:    xscvdpsxws f11, f11
-; CHECK-BE-NEXT:    xscvdpsxws f12, f12
-; CHECK-BE-NEXT:    xscvdpsxws f13, f13
-; CHECK-BE-NEXT:    xscvdpsxws v2, v2
-; CHECK-BE-NEXT:    xscvdpsxws v3, v3
+; CHECK-BE-NEXT:    vmrghb v3, v4, v3
 ; CHECK-BE-NEXT:    mfvsrwz r3, f3
-; CHECK-BE-NEXT:    mfvsrwz r4, f2
-; CHECK-BE-NEXT:    mfvsrwz r11, f1
-; CHECK-BE-NEXT:    mfvsrwz r12, f0
-; CHECK-BE-NEXT:    mfvsrwz r5, f4
-; CHECK-BE-NEXT:    mfvsrwz r6, f5
-; CHECK-BE-NEXT:    mfvsrwz r7, f6
-; CHECK-BE-NEXT:    mfvsrwz r8, f7
-; CHECK-BE-NEXT:    mfvsrwz r9, f8
-; CHECK-BE-NEXT:    mfvsrwz r10, f9
-; CHECK-BE-NEXT:    mfvsrwz r0, f10
-; CHECK-BE-NEXT:    mfvsrwz r30, f11
-; CHECK-BE-NEXT:    mfvsrwz r29, f12
-; CHECK-BE-NEXT:    mfvsrwz r28, f13
-; CHECK-BE-NEXT:    mfvsrwz r27, v2
-; CHECK-BE-NEXT:    mfvsrwz r26, v3
 ; CHECK-BE-NEXT:    sldi r3, r3, 56
-; CHECK-BE-NEXT:    sldi r4, r4, 56
-; CHECK-BE-NEXT:    sldi r11, r11, 56
-; CHECK-BE-NEXT:    sldi r12, r12, 56
-; CHECK-BE-NEXT:    sldi r5, r5, 56
-; CHECK-BE-NEXT:    sldi r6, r6, 56
-; CHECK-BE-NEXT:    sldi r7, r7, 56
-; CHECK-BE-NEXT:    sldi r8, r8, 56
-; CHECK-BE-NEXT:    sldi r9, r9, 56
-; CHECK-BE-NEXT:    sldi r10, r10, 56
-; CHECK-BE-NEXT:    sldi r0, r0, 56
-; CHECK-BE-NEXT:    sldi r30, r30, 56
-; CHECK-BE-NEXT:    sldi r29, r29, 56
-; CHECK-BE-NEXT:    sldi r28, r28, 56
-; CHECK-BE-NEXT:    sldi r27, r27, 56
-; CHECK-BE-NEXT:    sldi r26, r26, 56
-; CHECK-BE-NEXT:    mtvsrd v2, r3
-; CHECK-BE-NEXT:    mtvsrd v3, r4
-; CHECK-BE-NEXT:    mtvsrd v10, r11
-; CHECK-BE-NEXT:    mtvsrd v14, r12
-; CHECK-BE-NEXT:    mtvsrd v4, r5
-; CHECK-BE-NEXT:    mtvsrd v5, r6
-; CHECK-BE-NEXT:    mtvsrd v0, r7
-; CHECK-BE-NEXT:    mtvsrd v1, r8
-; CHECK-BE-NEXT:    mtvsrd v6, r9
-; CHECK-BE-NEXT:    mtvsrd v7, r10
-; CHECK-BE-NEXT:    mtvsrd v8, r0
-; CHECK-BE-NEXT:    mtvsrd v9, r30
-; CHECK-BE-NEXT:    ld r30, -16(r1) # 8-byte Folded Reload
-; CHECK-BE-NEXT:    mtvsrd v11, r29
-; CHECK-BE-NEXT:    ld r29, -24(r1) # 8-byte Folded Reload
-; CHECK-BE-NEXT:    mtvsrd v12, r28
-; CHECK-BE-NEXT:    ld r28, -32(r1) # 8-byte Folded Reload
-; CHECK-BE-NEXT:    mtvsrd v13, r27
-; CHECK-BE-NEXT:    ld r27, -40(r1) # 8-byte Folded Reload
-; CHECK-BE-NEXT:    mtvsrd v15, r26
-; CHECK-BE-NEXT:    ld r26, -48(r1) # 8-byte Folded Reload
-; CHECK-BE-NEXT:    vmrghb v4, v5, v4
-; CHECK-BE-NEXT:    vmrghb v2, v2, v0
-; CHECK-BE-NEXT:    vmrghb v5, v6, v1
-; CHECK-BE-NEXT:    vmrghb v3, v3, v7
-; CHECK-BE-NEXT:    vmrghb v0, v9, v8
-; CHECK-BE-NEXT:    vmrghb v1, v10, v11
-; CHECK-BE-NEXT:    vmrghb v6, v13, v12
-; CHECK-BE-NEXT:    vmrghb v7, v14, v15
-; CHECK-BE-NEXT:    vmrghh v2, v2, v4
-; CHECK-BE-NEXT:    vmrghh v3, v3, v5
-; CHECK-BE-NEXT:    vmrghh v4, v1, v0
-; CHECK-BE-NEXT:    vmrghh v5, v7, v6
+; CHECK-BE-NEXT:    mtvsrd v4, r3
+; CHECK-BE-NEXT:    mfvsrwz r3, f2
+; CHECK-BE-NEXT:    xxsldwi vs2, vs1, vs1, 3
+; CHECK-BE-NEXT:    sldi r3, r3, 56
+; CHECK-BE-NEXT:    xscvspdpn f2, vs2
+; CHECK-BE-NEXT:    mtvsrd v5, r3
+; CHECK-BE-NEXT:    xscvdpsxws f2, f2
+; CHECK-BE-NEXT:    mfvsrwz r3, f2
+; CHECK-BE-NEXT:    xxswapd vs2, vs1
+; CHECK-BE-NEXT:    xscvspdpn f2, vs2
+; CHECK-BE-NEXT:    xscvdpsxws f2, f2
+; CHECK-BE-NEXT:    vmrghb v4, v4, v5
+; CHECK-BE-NEXT:    vmrghh v3, v4, v3
+; CHECK-BE-NEXT:    sldi r3, r3, 56
 ; CHECK-BE-NEXT:    vmrghw v2, v3, v2
-; CHECK-BE-NEXT:    vmrghw v3, v5, v4
+; CHECK-BE-NEXT:    mtvsrd v3, r3
+; CHECK-BE-NEXT:    mfvsrwz r3, f2
+; CHECK-BE-NEXT:    xscvspdpn f2, vs1
+; CHECK-BE-NEXT:    xxsldwi vs1, vs1, vs1, 1
+; CHECK-BE-NEXT:    sldi r3, r3, 56
+; CHECK-BE-NEXT:    xscvdpsxws f2, f2
+; CHECK-BE-NEXT:    xscvspdpn f1, vs1
+; CHECK-BE-NEXT:    mtvsrd v4, r3
+; CHECK-BE-NEXT:    xscvdpsxws f1, f1
+; CHECK-BE-NEXT:    vmrghb v3, v4, v3
+; CHECK-BE-NEXT:    mfvsrwz r3, f2
+; CHECK-BE-NEXT:    sldi r3, r3, 56
+; CHECK-BE-NEXT:    mtvsrd v4, r3
+; CHECK-BE-NEXT:    mfvsrwz r3, f1
+; CHECK-BE-NEXT:    xxsldwi vs1, vs0, vs0, 3
+; CHECK-BE-NEXT:    sldi r3, r3, 56
+; CHECK-BE-NEXT:    xscvspdpn f1, vs1
+; CHECK-BE-NEXT:    mtvsrd v5, r3
+; CHECK-BE-NEXT:    xscvdpsxws f1, f1
+; CHECK-BE-NEXT:    mfvsrwz r3, f1
+; CHECK-BE-NEXT:    xxswapd vs1, vs0
+; CHECK-BE-NEXT:    xscvspdpn f1, vs1
+; CHECK-BE-NEXT:    xscvdpsxws f1, f1
+; CHECK-BE-NEXT:    vmrghb v4, v4, v5
+; CHECK-BE-NEXT:    sldi r3, r3, 56
+; CHECK-BE-NEXT:    vmrghh v3, v4, v3
+; CHECK-BE-NEXT:    mtvsrd v4, r3
+; CHECK-BE-NEXT:    mfvsrwz r3, f1
+; CHECK-BE-NEXT:    xscvspdpn f1, vs0
+; CHECK-BE-NEXT:    xxsldwi vs0, vs0, vs0, 1
+; CHECK-BE-NEXT:    sldi r3, r3, 56
+; CHECK-BE-NEXT:    xscvdpsxws f1, f1
+; CHECK-BE-NEXT:    xscvspdpn f0, vs0
+; CHECK-BE-NEXT:    mtvsrd v5, r3
+; CHECK-BE-NEXT:    xscvdpsxws f0, f0
+; CHECK-BE-NEXT:    vmrghb v4, v5, v4
+; CHECK-BE-NEXT:    mfvsrwz r3, f1
+; CHECK-BE-NEXT:    sldi r3, r3, 56
+; CHECK-BE-NEXT:    mtvsrd v5, r3
+; CHECK-BE-NEXT:    mfvsrwz r3, f0
+; CHECK-BE-NEXT:    sldi r3, r3, 56
+; CHECK-BE-NEXT:    mtvsrd v0, r3
+; CHECK-BE-NEXT:    vmrghb v5, v5, v0
+; CHECK-BE-NEXT:    vmrghh v4, v5, v4
+; CHECK-BE-NEXT:    vmrghw v3, v4, v3
 ; CHECK-BE-NEXT:    xxmrghd v2, v3, v2
 ; CHECK-BE-NEXT:    blr
 entry:
@@ -777,18 +757,18 @@ define i16 @test2elt_signed(i64 %a.coerce) local_unnamed_addr #0 {
 ; CHECK-P9-NEXT:    xxswapd v2, vs0
 ; CHECK-P9-NEXT:    xscvspdpn f0, vs0
 ; CHECK-P9-NEXT:    xxsldwi vs1, v2, v2, 3
-; CHECK-P9-NEXT:    xscvdpsxws f0, f0
 ; CHECK-P9-NEXT:    xscvspdpn f1, vs1
 ; CHECK-P9-NEXT:    xscvdpsxws f1, f1
-; CHECK-P9-NEXT:    mfvsrwz r4, f0
+; CHECK-P9-NEXT:    xscvdpsxws f0, f0
 ; CHECK-P9-NEXT:    mfvsrwz r3, f1
-; CHECK-P9-NEXT:    mtvsrd f1, r4
+; CHECK-P9-NEXT:    mtvsrd f1, r3
+; CHECK-P9-NEXT:    mfvsrwz r3, f0
 ; CHECK-P9-NEXT:    mtvsrd f0, r3
-; CHECK-P9-NEXT:    xxswapd v3, vs1
-; CHECK-P9-NEXT:    addi r3, r1, -2
-; CHECK-P9-NEXT:    xxswapd v2, vs0
+; CHECK-P9-NEXT:    xxswapd v2, vs1
+; CHECK-P9-NEXT:    xxswapd v3, vs0
 ; CHECK-P9-NEXT:    vmrglb v2, v3, v2
 ; CHECK-P9-NEXT:    vsldoi v2, v2, v2, 8
+; CHECK-P9-NEXT:    addi r3, r1, -2
 ; CHECK-P9-NEXT:    stxsihx v2, 0, r3
 ; CHECK-P9-NEXT:    lhz r3, -2(r1)
 ; CHECK-P9-NEXT:    blr
@@ -796,18 +776,18 @@ define i16 @test2elt_signed(i64 %a.coerce) local_unnamed_addr #0 {
 ; CHECK-BE-LABEL: test2elt_signed:
 ; CHECK-BE:       # %bb.0: # %entry
 ; CHECK-BE-NEXT:    mtvsrd f0, r3
-; CHECK-BE-NEXT:    xxsldwi vs1, vs0, vs0, 1
+; CHECK-BE-NEXT:    xscvspdpn f1, vs0
+; CHECK-BE-NEXT:    xxsldwi vs0, vs0, vs0, 1
+; CHECK-BE-NEXT:    xscvdpsxws f1, f1
 ; CHECK-BE-NEXT:    xscvspdpn f0, vs0
-; CHECK-BE-NEXT:    xscvspdpn f1, vs1
 ; CHECK-BE-NEXT:    xscvdpsxws f0, f0
-; CHECK-BE-NEXT:    xscvdpsxws f1, f1
-; CHECK-BE-NEXT:    mfvsrwz r3, f0
+; CHECK-BE-NEXT:    mfvsrwz r3, f1
 ; CHECK-BE-NEXT:    sldi r3, r3, 56
-; CHECK-BE-NEXT:    mfvsrwz r4, f1
 ; CHECK-BE-NEXT:    mtvsrd v2, r3
+; CHECK-BE-NEXT:    mfvsrwz r3, f0
+; CHECK-BE-NEXT:    sldi r3, r3, 56
+; CHECK-BE-NEXT:    mtvsrd v3, r3
 ; CHECK-BE-NEXT:    addi r3, r1, -2
-; CHECK-BE-NEXT:    sldi r4, r4, 56
-; CHECK-BE-NEXT:    mtvsrd v3, r4
 ; CHECK-BE-NEXT:    vmrghb v2, v2, v3
 ; CHECK-BE-NEXT:    vsldoi v2, v2, v2, 10
 ; CHECK-BE-NEXT:    stxsihx v2, 0, r3
@@ -856,64 +836,64 @@ define i32 @test4elt_signed(<4 x float> %a) local_unnamed_addr #1 {
 ; CHECK-P9-LABEL: test4elt_signed:
 ; CHECK-P9:       # %bb.0: # %entry
 ; CHECK-P9-NEXT:    xxsldwi vs0, v2, v2, 3
-; CHECK-P9-NEXT:    xxswapd vs1, v2
-; CHECK-P9-NEXT:    xxsldwi vs2, v2, v2, 1
-; CHECK-P9-NEXT:    xscvspdpn f3, v2
 ; CHECK-P9-NEXT:    xscvspdpn f0, vs0
-; CHECK-P9-NEXT:    xscvspdpn f1, vs1
-; CHECK-P9-NEXT:    xscvspdpn f2, vs2
-; CHECK-P9-NEXT:    xscvdpsxws f3, f3
 ; CHECK-P9-NEXT:    xscvdpsxws f0, f0
-; CHECK-P9-NEXT:    xscvdpsxws f1, f1
-; CHECK-P9-NEXT:    xscvdpsxws f2, f2
-; CHECK-P9-NEXT:    mfvsrwz r5, f3
 ; CHECK-P9-NEXT:    mfvsrwz r3, f0
-; CHECK-P9-NEXT:    mfvsrwz r4, f1
-; CHECK-P9-NEXT:    mfvsrwz r6, f2
-; CHECK-P9-NEXT:    mtvsrd f2, r5
 ; CHECK-P9-NEXT:    mtvsrd f0, r3
-; CHECK-P9-NEXT:    mtvsrd f1, r4
-; CHECK-P9-NEXT:    mtvsrd f3, r6
-; CHECK-P9-NEXT:    xxswapd v4, vs2
+; CHECK-P9-NEXT:    xxswapd v3, vs0
+; CHECK-P9-NEXT:    xxswapd vs0, v2
+; CHECK-P9-NEXT:    xscvspdpn f0, vs0
+; CHECK-P9-NEXT:    xscvdpsxws f0, f0
+; CHECK-P9-NEXT:    mfvsrwz r3, f0
+; CHECK-P9-NEXT:    mtvsrd f0, r3
+; CHECK-P9-NEXT:    xxswapd v4, vs0
+; CHECK-P9-NEXT:    xscvspdpn f0, v2
+; CHECK-P9-NEXT:    xscvdpsxws f0, f0
+; CHECK-P9-NEXT:    mfvsrwz r3, f0
+; CHECK-P9-NEXT:    mtvsrd f0, r3
+; CHECK-P9-NEXT:    vmrglb v3, v4, v3
+; CHECK-P9-NEXT:    xxswapd v4, vs0
+; CHECK-P9-NEXT:    xxsldwi vs0, v2, v2, 1
+; CHECK-P9-NEXT:    xscvspdpn f0, vs0
+; CHECK-P9-NEXT:    xscvdpsxws f0, f0
+; CHECK-P9-NEXT:    mfvsrwz r3, f0
+; CHECK-P9-NEXT:    mtvsrd f0, r3
 ; CHECK-P9-NEXT:    li r3, 0
 ; CHECK-P9-NEXT:    xxswapd v2, vs0
-; CHECK-P9-NEXT:    xxswapd v3, vs1
-; CHECK-P9-NEXT:    xxswapd v5, vs3
-; CHECK-P9-NEXT:    vmrglb v2, v3, v2
-; CHECK-P9-NEXT:    vmrglb v3, v4, v5
-; CHECK-P9-NEXT:    vmrglh v2, v3, v2
+; CHECK-P9-NEXT:    vmrglb v2, v4, v2
+; CHECK-P9-NEXT:    vmrglh v2, v2, v3
 ; CHECK-P9-NEXT:    vextuwrx r3, r3, v2
 ; CHECK-P9-NEXT:    blr
 ;
 ; CHECK-BE-LABEL: test4elt_signed:
 ; CHECK-BE:       # %bb.0: # %entry
 ; CHECK-BE-NEXT:    xxsldwi vs0, v2, v2, 3
-; CHECK-BE-NEXT:    xxswapd vs1, v2
-; CHECK-BE-NEXT:    xxsldwi vs2, v2, v2, 1
-; CHECK-BE-NEXT:    xscvspdpn f3, v2
 ; CHECK-BE-NEXT:    xscvspdpn f0, vs0
-; CHECK-BE-NEXT:    xscvspdpn f1, vs1
-; CHECK-BE-NEXT:    xscvspdpn f2, vs2
-; CHECK-BE-NEXT:    xscvdpsxws f3, f3
 ; CHECK-BE-NEXT:    xscvdpsxws f0, f0
-; CHECK-BE-NEXT:    xscvdpsxws f1, f1
-; CHECK-BE-NEXT:    xscvdpsxws f2, f2
-; CHECK-BE-NEXT:    mfvsrwz r5, f3
-; CHECK-BE-NEXT:    sldi r5, r5, 56
 ; CHECK-BE-NEXT:    mfvsrwz r3, f0
-; CHECK-BE-NEXT:    mfvsrwz r4, f1
-; CHECK-BE-NEXT:    mfvsrwz r6, f2
-; CHECK-BE-NEXT:    mtvsrd v4, r5
+; CHECK-BE-NEXT:    xxswapd vs0, v2
+; CHECK-BE-NEXT:    sldi r3, r3, 56
+; CHECK-BE-NEXT:    xscvspdpn f0, vs0
+; CHECK-BE-NEXT:    mtvsrd v3, r3
+; CHECK-BE-NEXT:    xscvdpsxws f0, f0
+; CHECK-BE-NEXT:    mfvsrwz r3, f0
+; CHECK-BE-NEXT:    xscvspdpn f0, v2
+; CHECK-BE-NEXT:    sldi r3, r3, 56
+; CHECK-BE-NEXT:    xscvdpsxws f0, f0
+; CHECK-BE-NEXT:    mtvsrd v4, r3
+; CHECK-BE-NEXT:    vmrghb v3, v4, v3
+; CHECK-BE-NEXT:    mfvsrwz r3, f0
+; CHECK-BE-NEXT:    xxsldwi vs0, v2, v2, 1
+; CHECK-BE-NEXT:    sldi r3, r3, 56
+; CHECK-BE-NEXT:    xscvspdpn f0, vs0
+; CHECK-BE-NEXT:    mtvsrd v4, r3
+; CHECK-BE-NEXT:    xscvdpsxws f0, f0
+; CHECK-BE-NEXT:    mfvsrwz r3, f0
 ; CHECK-BE-NEXT:    sldi r3, r3, 56
-; CHECK-BE-NEXT:    sldi r4, r4, 56
-; CHECK-BE-NEXT:    sldi r6, r6, 56
 ; CHECK-BE-NEXT:    mtvsrd v2, r3
-; CHECK-BE-NEXT:    mtvsrd v3, r4
-; CHECK-BE-NEXT:    mtvsrd v5, r6
 ; CHECK-BE-NEXT:    li r3, 0
-; CHECK-BE-NEXT:    vmrghb v2, v3, v2
-; CHECK-BE-NEXT:    vmrghb v3, v4, v5
-; CHECK-BE-NEXT:    vmrghh v2, v3, v2
+; CHECK-BE-NEXT:    vmrghb v2, v4, v2
+; CHECK-BE-NEXT:    vmrghh v2, v2, v3
 ; CHECK-BE-NEXT:    vextuwlx r3, r3, v2
 ; CHECK-BE-NEXT:    blr
 entry:
@@ -987,120 +967,120 @@ define i64 @test8elt_signed(<8 x float>* nocapture readonly) local_unnamed_addr
 ;
 ; CHECK-P9-LABEL: test8elt_signed:
 ; CHECK-P9:       # %bb.0: # %entry
-; CHECK-P9-NEXT:    lxv vs0, 16(r3)
 ; CHECK-P9-NEXT:    lxv vs1, 0(r3)
 ; CHECK-P9-NEXT:    xxsldwi vs2, vs1, vs1, 3
-; CHECK-P9-NEXT:    xxswapd vs3, vs1
-; CHECK-P9-NEXT:    xxsldwi vs4, vs1, vs1, 1
-; CHECK-P9-NEXT:    xxsldwi vs5, vs0, vs0, 3
-; CHECK-P9-NEXT:    xxswapd vs6, vs0
-; CHECK-P9-NEXT:    xxsldwi vs7, vs0, vs0, 1
-; CHECK-P9-NEXT:    xscvspdpn f1, vs1
-; CHECK-P9-NEXT:    xscvspdpn f0, vs0
 ; CHECK-P9-NEXT:    xscvspdpn f2, vs2
-; CHECK-P9-NEXT:    xscvspdpn f3, vs3
-; CHECK-P9-NEXT:    xscvspdpn f4, vs4
-; CHECK-P9-NEXT:    xscvspdpn f5, vs5
-; CHECK-P9-NEXT:    xscvspdpn f6, vs6
-; CHECK-P9-NEXT:    xscvspdpn f7, vs7
-; CHECK-P9-NEXT:    xscvdpsxws f1, f1
-; CHECK-P9-NEXT:    xscvdpsxws f0, f0
 ; CHECK-P9-NEXT:    xscvdpsxws f2, f2
-; CHECK-P9-NEXT:    xscvdpsxws f3, f3
-; CHECK-P9-NEXT:    xscvdpsxws f4, f4
-; CHECK-P9-NEXT:    xscvdpsxws f5, f5
-; CHECK-P9-NEXT:    xscvdpsxws f6, f6
-; CHECK-P9-NEXT:    xscvdpsxws f7, f7
-; CHECK-P9-NEXT:    mfvsrwz r5, f1
-; CHECK-P9-NEXT:    mfvsrwz r9, f0
+; CHECK-P9-NEXT:    lxv vs0, 16(r3)
 ; CHECK-P9-NEXT:    mfvsrwz r3, f2
-; CHECK-P9-NEXT:    mfvsrwz r4, f3
-; CHECK-P9-NEXT:    mfvsrwz r6, f4
-; CHECK-P9-NEXT:    mfvsrwz r7, f5
-; CHECK-P9-NEXT:    mfvsrwz r8, f6
-; CHECK-P9-NEXT:    mfvsrwz r10, f7
-; CHECK-P9-NEXT:    mtvsrd f2, r5
-; CHECK-P9-NEXT:    mtvsrd f6, r9
-; CHECK-P9-NEXT:    mtvsrd f0, r3
-; CHECK-P9-NEXT:    mtvsrd f1, r4
-; CHECK-P9-NEXT:    mtvsrd f3, r6
-; CHECK-P9-NEXT:    mtvsrd f4, r7
-; CHECK-P9-NEXT:    mtvsrd f5, r8
-; CHECK-P9-NEXT:    mtvsrd f7, r10
-; CHECK-P9-NEXT:    xxswapd v4, vs2
-; CHECK-P9-NEXT:    xxswapd v6, vs6
-; CHECK-P9-NEXT:    xxswapd v2, vs0
-; CHECK-P9-NEXT:    xxswapd v3, vs1
-; CHECK-P9-NEXT:    xxswapd v5, vs3
-; CHECK-P9-NEXT:    xxswapd v0, vs4
-; CHECK-P9-NEXT:    xxswapd v1, vs5
-; CHECK-P9-NEXT:    xxswapd v7, vs7
+; CHECK-P9-NEXT:    mtvsrd f2, r3
+; CHECK-P9-NEXT:    xxswapd v2, vs2
+; CHECK-P9-NEXT:    xxswapd vs2, vs1
+; CHECK-P9-NEXT:    xscvspdpn f2, vs2
+; CHECK-P9-NEXT:    xscvdpsxws f2, f2
+; CHECK-P9-NEXT:    mfvsrwz r3, f2
+; CHECK-P9-NEXT:    mtvsrd f2, r3
+; CHECK-P9-NEXT:    xxswapd v3, vs2
+; CHECK-P9-NEXT:    xscvspdpn f2, vs1
+; CHECK-P9-NEXT:    xxsldwi vs1, vs1, vs1, 1
+; CHECK-P9-NEXT:    xscvspdpn f1, vs1
+; CHECK-P9-NEXT:    xscvdpsxws f2, f2
+; CHECK-P9-NEXT:    xscvdpsxws f1, f1
+; CHECK-P9-NEXT:    mfvsrwz r3, f2
+; CHECK-P9-NEXT:    mtvsrd f2, r3
+; CHECK-P9-NEXT:    mfvsrwz r3, f1
+; CHECK-P9-NEXT:    mtvsrd f1, r3
+; CHECK-P9-NEXT:    xxswapd v4, vs1
+; CHECK-P9-NEXT:    xxsldwi vs1, vs0, vs0, 3
+; CHECK-P9-NEXT:    xscvspdpn f1, vs1
+; CHECK-P9-NEXT:    xscvdpsxws f1, f1
 ; CHECK-P9-NEXT:    vmrglb v2, v3, v2
-; CHECK-P9-NEXT:    vmrglb v3, v4, v5
-; CHECK-P9-NEXT:    vmrglb v4, v1, v0
-; CHECK-P9-NEXT:    vmrglb v5, v6, v7
+; CHECK-P9-NEXT:    xxswapd v3, vs2
+; CHECK-P9-NEXT:    vmrglb v3, v3, v4
 ; CHECK-P9-NEXT:    vmrglh v2, v3, v2
-; CHECK-P9-NEXT:    vmrglh v3, v5, v4
+; CHECK-P9-NEXT:    mfvsrwz r3, f1
+; CHECK-P9-NEXT:    mtvsrd f1, r3
+; CHECK-P9-NEXT:    xxswapd v3, vs1
+; CHECK-P9-NEXT:    xxswapd vs1, vs0
+; CHECK-P9-NEXT:    xscvspdpn f1, vs1
+; CHECK-P9-NEXT:    xscvdpsxws f1, f1
+; CHECK-P9-NEXT:    mfvsrwz r3, f1
+; CHECK-P9-NEXT:    mtvsrd f1, r3
+; CHECK-P9-NEXT:    xxswapd v4, vs1
+; CHECK-P9-NEXT:    xscvspdpn f1, vs0
+; CHECK-P9-NEXT:    xxsldwi vs0, vs0, vs0, 1
+; CHECK-P9-NEXT:    xscvspdpn f0, vs0
+; CHECK-P9-NEXT:    xscvdpsxws f1, f1
+; CHECK-P9-NEXT:    xscvdpsxws f0, f0
+; CHECK-P9-NEXT:    mfvsrwz r3, f1
+; CHECK-P9-NEXT:    mtvsrd f1, r3
+; CHECK-P9-NEXT:    mfvsrwz r3, f0
+; CHECK-P9-NEXT:    mtvsrd f0, r3
+; CHECK-P9-NEXT:    vmrglb v3, v4, v3
+; CHECK-P9-NEXT:    xxswapd v4, vs1
+; CHECK-P9-NEXT:    xxswapd v5, vs0
+; CHECK-P9-NEXT:    vmrglb v4, v4, v5
+; CHECK-P9-NEXT:    vmrglh v3, v4, v3
 ; CHECK-P9-NEXT:    vmrglw v2, v3, v2
 ; CHECK-P9-NEXT:    mfvsrld r3, v2
 ; CHECK-P9-NEXT:    blr
 ;
 ; CHECK-BE-LABEL: test8elt_signed:
 ; CHECK-BE:       # %bb.0: # %entry
-; CHECK-BE-NEXT:    lxv vs0, 0(r3)
 ; CHECK-BE-NEXT:    lxv vs1, 16(r3)
 ; CHECK-BE-NEXT:    xxsldwi vs2, vs1, vs1, 3
-; CHECK-BE-NEXT:    xxswapd vs3, vs1
-; CHECK-BE-NEXT:    xxsldwi vs4, vs1, vs1, 1
-; CHECK-BE-NEXT:    xxsldwi vs5, vs0, vs0, 3
-; CHECK-BE-NEXT:    xxswapd vs6, vs0
-; CHECK-BE-NEXT:    xxsldwi vs7, vs0, vs0, 1
-; CHECK-BE-NEXT:    xscvspdpn f1, vs1
-; CHECK-BE-NEXT:    xscvspdpn f0, vs0
 ; CHECK-BE-NEXT:    xscvspdpn f2, vs2
-; CHECK-BE-NEXT:    xscvspdpn f3, vs3
-; CHECK-BE-NEXT:    xscvspdpn f4, vs4
-; CHECK-BE-NEXT:    xscvspdpn f5, vs5
-; CHECK-BE-NEXT:    xscvspdpn f6, vs6
-; CHECK-BE-NEXT:    xscvspdpn f7, vs7
-; CHECK-BE-NEXT:    xscvdpsxws f1, f1
-; CHECK-BE-NEXT:    xscvdpsxws f0, f0
 ; CHECK-BE-NEXT:    xscvdpsxws f2, f2
-; CHECK-BE-NEXT:    xscvdpsxws f3, f3
-; CHECK-BE-NEXT:    xscvdpsxws f4, f4
-; CHECK-BE-NEXT:    xscvdpsxws f5, f5
-; CHECK-BE-NEXT:    xscvdpsxws f6, f6
-; CHECK-BE-NEXT:    xscvdpsxws f7, f7
-; CHECK-BE-NEXT:    mfvsrwz r5, f1
-; CHECK-BE-NEXT:    mfvsrwz r9, f0
-; CHECK-BE-NEXT:    sldi r5, r5, 56
-; CHECK-BE-NEXT:    sldi r9, r9, 56
+; CHECK-BE-NEXT:    lxv vs0, 0(r3)
 ; CHECK-BE-NEXT:    mfvsrwz r3, f2
-; CHECK-BE-NEXT:    mfvsrwz r4, f3
-; CHECK-BE-NEXT:    mfvsrwz r6, f4
-; CHECK-BE-NEXT:    mfvsrwz r7, f5
-; CHECK-BE-NEXT:    mfvsrwz r8, f6
-; CHECK-BE-NEXT:    mfvsrwz r10, f7
-; CHECK-BE-NEXT:    mtvsrd v4, r5
-; CHECK-BE-NEXT:    mtvsrd v6, r9
+; CHECK-BE-NEXT:    xxswapd vs2, vs1
 ; CHECK-BE-NEXT:    sldi r3, r3, 56
-; CHECK-BE-NEXT:    sldi r4, r4, 56
-; CHECK-BE-NEXT:    sldi r6, r6, 56
-; CHECK-BE-NEXT:    sldi r7, r7, 56
-; CHECK-BE-NEXT:    sldi r8, r8, 56
-; CHECK-BE-NEXT:    sldi r10, r10, 56
+; CHECK-BE-NEXT:    xscvspdpn f2, vs2
 ; CHECK-BE-NEXT:    mtvsrd v2, r3
-; CHECK-BE-NEXT:    mtvsrd v3, r4
-; CHECK-BE-NEXT:    mtvsrd v5, r6
-; CHECK-BE-NEXT:    mtvsrd v0, r7
-; CHECK-BE-NEXT:    mtvsrd v1, r8
-; CHECK-BE-NEXT:    mtvsrd v7, r10
+; CHECK-BE-NEXT:    xscvdpsxws f2, f2
+; CHECK-BE-NEXT:    mfvsrwz r3, f2
+; CHECK-BE-NEXT:    xscvspdpn f2, vs1
+; CHECK-BE-NEXT:    xxsldwi vs1, vs1, vs1, 1
+; CHECK-BE-NEXT:    sldi r3, r3, 56
+; CHECK-BE-NEXT:    xscvdpsxws f2, f2
+; CHECK-BE-NEXT:    xscvspdpn f1, vs1
+; CHECK-BE-NEXT:    mtvsrd v3, r3
+; CHECK-BE-NEXT:    xscvdpsxws f1, f1
 ; CHECK-BE-NEXT:    vmrghb v2, v3, v2
-; CHECK-BE-NEXT:    vmrghb v3, v4, v5
-; CHECK-BE-NEXT:    vmrghb v4, v1, v0
-; CHECK-BE-NEXT:    vmrghb v5, v6, v7
+; CHECK-BE-NEXT:    mfvsrwz r3, f2
+; CHECK-BE-NEXT:    sldi r3, r3, 56
+; CHECK-BE-NEXT:    mtvsrd v3, r3
+; CHECK-BE-NEXT:    mfvsrwz r3, f1
+; CHECK-BE-NEXT:    xxsldwi vs1, vs0, vs0, 3
+; CHECK-BE-NEXT:    sldi r3, r3, 56
+; CHECK-BE-NEXT:    xscvspdpn f1, vs1
+; CHECK-BE-NEXT:    mtvsrd v4, r3
+; CHECK-BE-NEXT:    xscvdpsxws f1, f1
+; CHECK-BE-NEXT:    mfvsrwz r3, f1
+; CHECK-BE-NEXT:    xxswapd vs1, vs0
+; CHECK-BE-NEXT:    xscvspdpn f1, vs1
+; CHECK-BE-NEXT:    xscvdpsxws f1, f1
+; CHECK-BE-NEXT:    vmrghb v3, v3, v4
+; CHECK-BE-NEXT:    sldi r3, r3, 56
 ; CHECK-BE-NEXT:    vmrghh v2, v3, v2
-; CHECK-BE-NEXT:    vmrghh v3, v5, v4
+; CHECK-BE-NEXT:    mtvsrd v3, r3
+; CHECK-BE-NEXT:    mfvsrwz r3, f1
+; CHECK-BE-NEXT:    xscvspdpn f1, vs0
+; CHECK-BE-NEXT:    xxsldwi vs0, vs0, vs0, 1
+; CHECK-BE-NEXT:    sldi r3, r3, 56
+; CHECK-BE-NEXT:    xscvdpsxws f1, f1
+; CHECK-BE-NEXT:    xscvspdpn f0, vs0
+; CHECK-BE-NEXT:    mtvsrd v4, r3
+; CHECK-BE-NEXT:    xscvdpsxws f0, f0
+; CHECK-BE-NEXT:    vmrghb v3, v4, v3
+; CHECK-BE-NEXT:    mfvsrwz r3, f1
+; CHECK-BE-NEXT:    sldi r3, r3, 56
+; CHECK-BE-NEXT:    mtvsrd v4, r3
+; CHECK-BE-NEXT:    mfvsrwz r3, f0
+; CHECK-BE-NEXT:    sldi r3, r3, 56
+; CHECK-BE-NEXT:    mtvsrd v5, r3
+; CHECK-BE-NEXT:    vmrghb v4, v4, v5
+; CHECK-BE-NEXT:    vmrghh v3, v4, v3
 ; CHECK-BE-NEXT:    vmrghw v2, v3, v2
 ; CHECK-BE-NEXT:    mfvsrd r3, v2
 ; CHECK-BE-NEXT:    blr
@@ -1232,251 +1212,231 @@ define <16 x i8> @test16elt_signed(<16 x float>* nocapture readonly) local_unnam
 ;
 ; CHECK-P9-LABEL: test16elt_signed:
 ; CHECK-P9:       # %bb.0: # %entry
-; CHECK-P9-NEXT:    lxv vs2, 16(r3)
-; CHECK-P9-NEXT:    lxv vs3, 0(r3)
-; CHECK-P9-NEXT:    lxv vs0, 48(r3)
-; CHECK-P9-NEXT:    lxv vs1, 32(r3)
-; CHECK-P9-NEXT:    std r26, -48(r1) # 8-byte Folded Spill
-; CHECK-P9-NEXT:    std r27, -40(r1) # 8-byte Folded Spill
-; CHECK-P9-NEXT:    std r28, -32(r1) # 8-byte Folded Spill
-; CHECK-P9-NEXT:    std r29, -24(r1) # 8-byte Folded Spill
-; CHECK-P9-NEXT:    std r30, -16(r1) # 8-byte Folded Spill
-; CHECK-P9-NEXT:    xxsldwi vs4, vs3, vs3, 3
-; CHECK-P9-NEXT:    xxswapd vs5, vs3
-; CHECK-P9-NEXT:    xxsldwi vs6, vs3, vs3, 1
-; CHECK-P9-NEXT:    xxsldwi vs7, vs2, vs2, 3
-; CHECK-P9-NEXT:    xxswapd vs8, vs2
-; CHECK-P9-NEXT:    xxsldwi vs9, vs2, vs2, 1
-; CHECK-P9-NEXT:    xxsldwi vs10, vs1, vs1, 3
-; CHECK-P9-NEXT:    xxswapd vs11, vs1
-; CHECK-P9-NEXT:    xxsldwi vs12, vs1, vs1, 1
-; CHECK-P9-NEXT:    xxsldwi vs13, vs0, vs0, 3
-; CHECK-P9-NEXT:    xxswapd v2, vs0
-; CHECK-P9-NEXT:    xxsldwi v3, vs0, vs0, 1
-; CHECK-P9-NEXT:    xscvspdpn f3, vs3
-; CHECK-P9-NEXT:    xscvspdpn f2, vs2
+; CHECK-P9-NEXT:    lxv vs0, 0(r3)
+; CHECK-P9-NEXT:    xxsldwi vs1, vs0, vs0, 3
 ; CHECK-P9-NEXT:    xscvspdpn f1, vs1
+; CHECK-P9-NEXT:    xscvdpsxws f1, f1
+; CHECK-P9-NEXT:    mfvsrwz r4, f1
+; CHECK-P9-NEXT:    mtvsrd f1, r4
+; CHECK-P9-NEXT:    xxswapd v2, vs1
+; CHECK-P9-NEXT:    xxswapd vs1, vs0
+; CHECK-P9-NEXT:    xscvspdpn f1, vs1
+; CHECK-P9-NEXT:    xscvdpsxws f1, f1
+; CHECK-P9-NEXT:    lxv vs2, 48(r3)
+; CHECK-P9-NEXT:    lxv vs3, 32(r3)
+; CHECK-P9-NEXT:    lxv vs4, 16(r3)
+; CHECK-P9-NEXT:    mfvsrwz r3, f1
+; CHECK-P9-NEXT:    mtvsrd f1, r3
+; CHECK-P9-NEXT:    xxswapd v3, vs1
+; CHECK-P9-NEXT:    xscvspdpn f1, vs0
+; CHECK-P9-NEXT:    xxsldwi vs0, vs0, vs0, 1
 ; CHECK-P9-NEXT:    xscvspdpn f0, vs0
-; CHECK-P9-NEXT:    xscvspdpn f4, vs4
-; CHECK-P9-NEXT:    xscvspdpn f5, vs5
-; CHECK-P9-NEXT:    xscvspdpn f6, vs6
-; CHECK-P9-NEXT:    xscvspdpn f7, vs7
-; CHECK-P9-NEXT:    xscvspdpn f8, vs8
-; CHECK-P9-NEXT:    xscvspdpn f9, vs9
-; CHECK-P9-NEXT:    xscvspdpn f10, vs10
-; CHECK-P9-NEXT:    xscvspdpn f11, vs11
-; CHECK-P9-NEXT:    xscvspdpn f12, vs12
-; CHECK-P9-NEXT:    xscvspdpn f13, vs13
-; CHECK-P9-NEXT:    xscvspdpn v2, v2
-; CHECK-P9-NEXT:    xscvspdpn v3, v3
-; CHECK-P9-NEXT:    xscvdpsxws f3, f3
-; CHECK-P9-NEXT:    xscvdpsxws f2, f2
 ; CHECK-P9-NEXT:    xscvdpsxws f1, f1
 ; CHECK-P9-NEXT:    xscvdpsxws f0, f0
-; CHECK-P9-NEXT:    xscvdpsxws f4, f4
-; CHECK-P9-NEXT:    xscvdpsxws f5, f5
-; CHECK-P9-NEXT:    xscvdpsxws f6, f6
-; CHECK-P9-NEXT:    xscvdpsxws f7, f7
-; CHECK-P9-NEXT:    xscvdpsxws f8, f8
-; CHECK-P9-NEXT:    xscvdpsxws f9, f9
-; CHECK-P9-NEXT:    xscvdpsxws f10, f10
-; CHECK-P9-NEXT:    xscvdpsxws f11, f11
-; CHECK-P9-NEXT:    xscvdpsxws f12, f12
-; CHECK-P9-NEXT:    xscvdpsxws f13, f13
-; CHECK-P9-NEXT:    xscvdpsxws v2, v2
-; CHECK-P9-NEXT:    xscvdpsxws v3, v3
-; CHECK-P9-NEXT:    mfvsrwz r3, f3
-; CHECK-P9-NEXT:    mfvsrwz r4, f2
-; CHECK-P9-NEXT:    mfvsrwz r11, f1
-; CHECK-P9-NEXT:    mfvsrwz r12, f0
-; CHECK-P9-NEXT:    mfvsrwz r5, f4
-; CHECK-P9-NEXT:    mfvsrwz r6, f5
-; CHECK-P9-NEXT:    mfvsrwz r7, f6
-; CHECK-P9-NEXT:    mfvsrwz r8, f7
-; CHECK-P9-NEXT:    mfvsrwz r9, f8
-; CHECK-P9-NEXT:    mfvsrwz r10, f9
-; CHECK-P9-NEXT:    mfvsrwz r0, f10
-; CHECK-P9-NEXT:    mfvsrwz r30, f11
-; CHECK-P9-NEXT:    mfvsrwz r29, f12
-; CHECK-P9-NEXT:    mfvsrwz r28, f13
-; CHECK-P9-NEXT:    mfvsrwz r27, v2
-; CHECK-P9-NEXT:    mfvsrwz r26, v3
+; CHECK-P9-NEXT:    mfvsrwz r3, f1
+; CHECK-P9-NEXT:    mtvsrd f1, r3
+; CHECK-P9-NEXT:    mfvsrwz r3, f0
 ; CHECK-P9-NEXT:    mtvsrd f0, r3
-; CHECK-P9-NEXT:    mtvsrd f1, r4
-; CHECK-P9-NEXT:    mtvsrd f8, r11
-; CHECK-P9-NEXT:    mtvsrd f9, r12
-; CHECK-P9-NEXT:    mtvsrd f2, r5
-; CHECK-P9-NEXT:    mtvsrd f3, r6
-; CHECK-P9-NEXT:    mtvsrd f4, r7
-; CHECK-P9-NEXT:    mtvsrd f5, r8
-; CHECK-P9-NEXT:    mtvsrd f6, r9
-; CHECK-P9-NEXT:    mtvsrd f7, r10
-; CHECK-P9-NEXT:    mtvsrd f10, r0
-; CHECK-P9-NEXT:    mtvsrd f11, r30
-; CHECK-P9-NEXT:    ld r30, -16(r1) # 8-byte Folded Reload
-; CHECK-P9-NEXT:    mtvsrd f12, r29
-; CHECK-P9-NEXT:    ld r29, -24(r1) # 8-byte Folded Reload
-; CHECK-P9-NEXT:    mtvsrd f13, r28
-; CHECK-P9-NEXT:    ld r28, -32(r1) # 8-byte Folded Reload
-; CHECK-P9-NEXT:    mtvsrd v2, r27
-; CHECK-P9-NEXT:    ld r27, -40(r1) # 8-byte Folded Reload
-; CHECK-P9-NEXT:    mtvsrd v3, r26
-; CHECK-P9-NEXT:    ld r26, -48(r1) # 8-byte Folded Reload
 ; CHECK-P9-NEXT:    xxswapd v4, vs0
-; CHECK-P9-NEXT:    xxswapd v5, vs2
-; CHECK-P9-NEXT:    xxswapd v0, vs3
-; CHECK-P9-NEXT:    xxswapd v1, vs4
-; CHECK-P9-NEXT:    xxswapd v6, vs5
-; CHECK-P9-NEXT:    xxswapd v7, vs6
-; CHECK-P9-NEXT:    xxswapd v8, vs1
-; CHECK-P9-NEXT:    xxswapd v9, vs7
-; CHECK-P9-NEXT:    xxswapd v10, vs10
-; CHECK-P9-NEXT:    xxswapd v11, vs11
-; CHECK-P9-NEXT:    xxswapd v12, vs8
-; CHECK-P9-NEXT:    xxswapd v13, vs12
-; CHECK-P9-NEXT:    xxswapd v14, vs13
-; CHECK-P9-NEXT:    xxswapd v2, v2
-; CHECK-P9-NEXT:    xxswapd v15, vs9
-; CHECK-P9-NEXT:    xxswapd v3, v3
-; CHECK-P9-NEXT:    vmrglb v5, v0, v5
-; CHECK-P9-NEXT:    vmrglb v4, v4, v1
-; CHECK-P9-NEXT:    vmrglb v0, v7, v6
-; CHECK-P9-NEXT:    vmrglb v1, v8, v9
-; CHECK-P9-NEXT:    vmrglb v6, v11, v10
-; CHECK-P9-NEXT:    vmrglb v7, v12, v13
-; CHECK-P9-NEXT:    vmrglb v2, v2, v14
-; CHECK-P9-NEXT:    vmrglb v3, v15, v3
-; CHECK-P9-NEXT:    vmrglh v4, v4, v5
-; CHECK-P9-NEXT:    vmrglh v5, v1, v0
-; CHECK-P9-NEXT:    vmrglh v0, v7, v6
+; CHECK-P9-NEXT:    xxsldwi vs0, vs4, vs4, 3
+; CHECK-P9-NEXT:    xscvspdpn f0, vs0
+; CHECK-P9-NEXT:    xscvdpsxws f0, f0
+; CHECK-P9-NEXT:    vmrglb v2, v3, v2
+; CHECK-P9-NEXT:    xxswapd v3, vs1
+; CHECK-P9-NEXT:    vmrglb v3, v3, v4
 ; CHECK-P9-NEXT:    vmrglh v2, v3, v2
-; CHECK-P9-NEXT:    vmrglw v3, v5, v4
-; CHECK-P9-NEXT:    vmrglw v2, v2, v0
-; CHECK-P9-NEXT:    xxmrgld v2, v2, v3
+; CHECK-P9-NEXT:    mfvsrwz r3, f0
+; CHECK-P9-NEXT:    mtvsrd f0, r3
+; CHECK-P9-NEXT:    xxswapd v3, vs0
+; CHECK-P9-NEXT:    xxswapd vs0, vs4
+; CHECK-P9-NEXT:    xscvspdpn f0, vs0
+; CHECK-P9-NEXT:    xscvdpsxws f0, f0
+; CHECK-P9-NEXT:    mfvsrwz r3, f0
+; CHECK-P9-NEXT:    mtvsrd f0, r3
+; CHECK-P9-NEXT:    xxswapd v4, vs0
+; CHECK-P9-NEXT:    xscvspdpn f0, vs4
+; CHECK-P9-NEXT:    xscvdpsxws f0, f0
+; CHECK-P9-NEXT:    mfvsrwz r3, f0
+; CHECK-P9-NEXT:    mtvsrd f0, r3
+; CHECK-P9-NEXT:    vmrglb v3, v4, v3
+; CHECK-P9-NEXT:    xxswapd v4, vs0
+; CHECK-P9-NEXT:    xxsldwi vs0, vs4, vs4, 1
+; CHECK-P9-NEXT:    xscvspdpn f0, vs0
+; CHECK-P9-NEXT:    xscvdpsxws f0, f0
+; CHECK-P9-NEXT:    mfvsrwz r3, f0
+; CHECK-P9-NEXT:    mtvsrd f0, r3
+; CHECK-P9-NEXT:    xxswapd v5, vs0
+; CHECK-P9-NEXT:    xxsldwi vs0, vs3, vs3, 3
+; CHECK-P9-NEXT:    xscvspdpn f0, vs0
+; CHECK-P9-NEXT:    xscvdpsxws f0, f0
+; CHECK-P9-NEXT:    vmrglb v4, v4, v5
+; CHECK-P9-NEXT:    vmrglh v3, v4, v3
+; CHECK-P9-NEXT:    vmrglw v2, v3, v2
+; CHECK-P9-NEXT:    mfvsrwz r3, f0
+; CHECK-P9-NEXT:    mtvsrd f0, r3
+; CHECK-P9-NEXT:    xxswapd v3, vs0
+; CHECK-P9-NEXT:    xxswapd vs0, vs3
+; CHECK-P9-NEXT:    xscvspdpn f0, vs0
+; CHECK-P9-NEXT:    xscvdpsxws f0, f0
+; CHECK-P9-NEXT:    mfvsrwz r3, f0
+; CHECK-P9-NEXT:    mtvsrd f0, r3
+; CHECK-P9-NEXT:    xxswapd v4, vs0
+; CHECK-P9-NEXT:    xscvspdpn f0, vs3
+; CHECK-P9-NEXT:    xscvdpsxws f0, f0
+; CHECK-P9-NEXT:    mfvsrwz r3, f0
+; CHECK-P9-NEXT:    mtvsrd f0, r3
+; CHECK-P9-NEXT:    vmrglb v3, v4, v3
+; CHECK-P9-NEXT:    xxswapd v4, vs0
+; CHECK-P9-NEXT:    xxsldwi vs0, vs3, vs3, 1
+; CHECK-P9-NEXT:    xscvspdpn f0, vs0
+; CHECK-P9-NEXT:    xscvdpsxws f0, f0
+; CHECK-P9-NEXT:    mfvsrwz r3, f0
+; CHECK-P9-NEXT:    mtvsrd f0, r3
+; CHECK-P9-NEXT:    xxswapd v5, vs0
+; CHECK-P9-NEXT:    xxsldwi vs0, vs2, vs2, 3
+; CHECK-P9-NEXT:    xscvspdpn f0, vs0
+; CHECK-P9-NEXT:    xscvdpsxws f0, f0
+; CHECK-P9-NEXT:    mfvsrwz r3, f0
+; CHECK-P9-NEXT:    vmrglb v4, v4, v5
+; CHECK-P9-NEXT:    vmrglh v3, v4, v3
+; CHECK-P9-NEXT:    mtvsrd f0, r3
+; CHECK-P9-NEXT:    xxswapd v4, vs0
+; CHECK-P9-NEXT:    xxswapd vs0, vs2
+; CHECK-P9-NEXT:    xscvspdpn f0, vs0
+; CHECK-P9-NEXT:    xscvdpsxws f0, f0
+; CHECK-P9-NEXT:    mfvsrwz r3, f0
+; CHECK-P9-NEXT:    mtvsrd f0, r3
+; CHECK-P9-NEXT:    xxswapd v5, vs0
+; CHECK-P9-NEXT:    xscvspdpn f0, vs2
+; CHECK-P9-NEXT:    xscvdpsxws f0, f0
+; CHECK-P9-NEXT:    mfvsrwz r3, f0
+; CHECK-P9-NEXT:    mtvsrd f0, r3
+; CHECK-P9-NEXT:    vmrglb v4, v5, v4
+; CHECK-P9-NEXT:    xxswapd v5, vs0
+; CHECK-P9-NEXT:    xxsldwi vs0, vs2, vs2, 1
+; CHECK-P9-NEXT:    xscvspdpn f0, vs0
+; CHECK-P9-NEXT:    xscvdpsxws f0, f0
+; CHECK-P9-NEXT:    mfvsrwz r3, f0
+; CHECK-P9-NEXT:    mtvsrd f0, r3
+; CHECK-P9-NEXT:    xxswapd v0, vs0
+; CHECK-P9-NEXT:    vmrglb v5, v5, v0
+; CHECK-P9-NEXT:    vmrglh v4, v5, v4
+; CHECK-P9-NEXT:    vmrglw v3, v4, v3
+; CHECK-P9-NEXT:    xxmrgld v2, v3, v2
 ; CHECK-P9-NEXT:    blr
 ;
 ; CHECK-BE-LABEL: test16elt_signed:
 ; CHECK-BE:       # %bb.0: # %entry
-; CHECK-BE-NEXT:    lxv vs2, 32(r3)
 ; CHECK-BE-NEXT:    lxv vs3, 48(r3)
+; CHECK-BE-NEXT:    xxsldwi vs4, vs3, vs3, 3
+; CHECK-BE-NEXT:    xscvspdpn f4, vs4
+; CHECK-BE-NEXT:    xscvdpsxws f4, f4
 ; CHECK-BE-NEXT:    lxv vs0, 0(r3)
 ; CHECK-BE-NEXT:    lxv vs1, 16(r3)
-; CHECK-BE-NEXT:    std r26, -48(r1) # 8-byte Folded Spill
-; CHECK-BE-NEXT:    std r27, -40(r1) # 8-byte Folded Spill
-; CHECK-BE-NEXT:    std r28, -32(r1) # 8-byte Folded Spill
-; CHECK-BE-NEXT:    std r29, -24(r1) # 8-byte Folded Spill
-; CHECK-BE-NEXT:    std r30, -16(r1) # 8-byte Folded Spill
-; CHECK-BE-NEXT:    xxsldwi vs4, vs3, vs3, 3
-; CHECK-BE-NEXT:    xxswapd vs5, vs3
-; CHECK-BE-NEXT:    xxsldwi vs6, vs3, vs3, 1
-; CHECK-BE-NEXT:    xxsldwi vs7, vs2, vs2, 3
-; CHECK-BE-NEXT:    xxswapd vs8, vs2
-; CHECK-BE-NEXT:    xxsldwi vs9, vs2, vs2, 1
-; CHECK-BE-NEXT:    xxsldwi vs10, vs1, vs1, 3
-; CHECK-BE-NEXT:    xxswapd vs11, vs1
-; CHECK-BE-NEXT:    xxsldwi vs12, vs1, vs1, 1
-; CHECK-BE-NEXT:    xxsldwi vs13, vs0, vs0, 3
-; CHECK-BE-NEXT:    xxswapd v2, vs0
-; CHECK-BE-NEXT:    xxsldwi v3, vs0, vs0, 1
-; CHECK-BE-NEXT:    xscvspdpn f3, vs3
-; CHECK-BE-NEXT:    xscvspdpn f2, vs2
-; CHECK-BE-NEXT:    xscvspdpn f1, vs1
-; CHECK-BE-NEXT:    xscvspdpn f0, vs0
+; CHECK-BE-NEXT:    lxv vs2, 32(r3)
+; CHECK-BE-NEXT:    mfvsrwz r3, f4
+; CHECK-BE-NEXT:    xxswapd vs4, vs3
+; CHECK-BE-NEXT:    sldi r3, r3, 56
 ; CHECK-BE-NEXT:    xscvspdpn f4, vs4
-; CHECK-BE-NEXT:    xscvspdpn f5, vs5
-; CHECK-BE-NEXT:    xscvspdpn f6, vs6
-; CHECK-BE-NEXT:    xscvspdpn f7, vs7
-; CHECK-BE-NEXT:    xscvspdpn f8, vs8
-; CHECK-BE-NEXT:    xscvspdpn f9, vs9
-; CHECK-BE-NEXT:    xscvspdpn f10, vs10
-; CHECK-BE-NEXT:    xscvspdpn f11, vs11
-; CHECK-BE-NEXT:    xscvspdpn f12, vs12
-; CHECK-BE-NEXT:    xscvspdpn f13, vs13
-; CHECK-BE-NEXT:    xscvspdpn v2, v2
-; CHECK-BE-NEXT:    xscvspdpn v3, v3
+; CHECK-BE-NEXT:    mtvsrd v2, r3
+; CHECK-BE-NEXT:    xscvdpsxws f4, f4
+; CHECK-BE-NEXT:    mfvsrwz r3, f4
+; CHECK-BE-NEXT:    xscvspdpn f4, vs3
+; CHECK-BE-NEXT:    xxsldwi vs3, vs3, vs3, 1
+; CHECK-BE-NEXT:    sldi r3, r3, 56
+; CHECK-BE-NEXT:    xscvdpsxws f4, f4
+; CHECK-BE-NEXT:    xscvspdpn f3, vs3
+; CHECK-BE-NEXT:    mtvsrd v3, r3
 ; CHECK-BE-NEXT:    xscvdpsxws f3, f3
+; CHECK-BE-NEXT:    vmrghb v2, v3, v2
+; CHECK-BE-NEXT:    mfvsrwz r3, f4
+; CHECK-BE-NEXT:    sldi r3, r3, 56
+; CHECK-BE-NEXT:    mtvsrd v3, r3
+; CHECK-BE-NEXT:    mfvsrwz r3, f3
+; CHECK-BE-NEXT:    xxsldwi vs3, vs2, vs2, 3
+; CHECK-BE-NEXT:    sldi r3, r3, 56
+; CHECK-BE-NEXT:    xscvspdpn f3, vs3
+; CHECK-BE-NEXT:    mtvsrd v4, r3
+; CHECK-BE-NEXT:    xscvdpsxws f3, f3
+; CHECK-BE-NEXT:    mfvsrwz r3, f3
+; CHECK-BE-NEXT:    xxswapd vs3, vs2
+; CHECK-BE-NEXT:    xscvspdpn f3, vs3
+; CHECK-BE-NEXT:    xscvdpsxws f3, f3
+; CHECK-BE-NEXT:    vmrghb v3, v3, v4
+; CHECK-BE-NEXT:    sldi r3, r3, 56
+; CHECK-BE-NEXT:    vmrghh v2, v3, v2
+; CHECK-BE-NEXT:    mtvsrd v3, r3
+; CHECK-BE-NEXT:    mfvsrwz r3, f3
+; CHECK-BE-NEXT:    xscvspdpn f3, vs2
+; CHECK-BE-NEXT:    xxsldwi vs2, vs2, vs2, 1
+; CHECK-BE-NEXT:    sldi r3, r3, 56
+; CHECK-BE-NEXT:    xscvdpsxws f3, f3
+; CHECK-BE-NEXT:    xscvspdpn f2, vs2
+; CHECK-BE-NEXT:    mtvsrd v4, r3
 ; CHECK-BE-NEXT:    xscvdpsxws f2, f2
-; CHECK-BE-NEXT:    xscvdpsxws f1, f1
-; CHECK-BE-NEXT:    xscvdpsxws f0, f0
-; CHECK-BE-NEXT:    xscvdpsxws f4, f4
-; CHECK-BE-NEXT:    xscvdpsxws f5, f5
-; CHECK-BE-NEXT:    xscvdpsxws f6, f6
-; CHECK-BE-NEXT:    xscvdpsxws f7, f7
-; CHECK-BE-NEXT:    xscvdpsxws f8, f8
-; CHECK-BE-NEXT:    xscvdpsxws f9, f9
-; CHECK-BE-NEXT:    xscvdpsxws f10, f10
-; CHECK-BE-NEXT:    xscvdpsxws f11, f11
-; CHECK-BE-NEXT:    xscvdpsxws f12, f12
-; CHECK-BE-NEXT:    xscvdpsxws f13, f13
-; CHECK-BE-NEXT:    xscvdpsxws v2, v2
-; CHECK-BE-NEXT:    xscvdpsxws v3, v3
+; CHECK-BE-NEXT:    vmrghb v3, v4, v3
 ; CHECK-BE-NEXT:    mfvsrwz r3, f3
-; CHECK-BE-NEXT:    mfvsrwz r4, f2
-; CHECK-BE-NEXT:    mfvsrwz r11, f1
-; CHECK-BE-NEXT:    mfvsrwz r12, f0
-; CHECK-BE-NEXT:    mfvsrwz r5, f4
-; CHECK-BE-NEXT:    mfvsrwz r6, f5
-; CHECK-BE-NEXT:    mfvsrwz r7, f6
-; CHECK-BE-NEXT:    mfvsrwz r8, f7
-; CHECK-BE-NEXT:    mfvsrwz r9, f8
-; CHECK-BE-NEXT:    mfvsrwz r10, f9
-; CHECK-BE-NEXT:    mfvsrwz r0, f10
-; CHECK-BE-NEXT:    mfvsrwz r30, f11
-; CHECK-BE-NEXT:    mfvsrwz r29, f12
-; CHECK-BE-NEXT:    mfvsrwz r28, f13
-; CHECK-BE-NEXT:    mfvsrwz r27, v2
-; CHECK-BE-NEXT:    mfvsrwz r26, v3
 ; CHECK-BE-NEXT:    sldi r3, r3, 56
-; CHECK-BE-NEXT:    sldi r4, r4, 56
-; CHECK-BE-NEXT:    sldi r11, r11, 56
-; CHECK-BE-NEXT:    sldi r12, r12, 56
-; CHECK-BE-NEXT:    sldi r5, r5, 56
-; CHECK-BE-NEXT:    sldi r6, r6, 56
-; CHECK-BE-NEXT:    sldi r7, r7, 56
-; CHECK-BE-NEXT:    sldi r8, r8, 56
-; CHECK-BE-NEXT:    sldi r9, r9, 56
-; CHECK-BE-NEXT:    sldi r10, r10, 56
-; CHECK-BE-NEXT:    sldi r0, r0, 56
-; CHECK-BE-NEXT:    sldi r30, r30, 56
-; CHECK-BE-NEXT:    sldi r29, r29, 56
-; CHECK-BE-NEXT:    sldi r28, r28, 56
-; CHECK-BE-NEXT:    sldi r27, r27, 56
-; CHECK-BE-NEXT:    sldi r26, r26, 56
-; CHECK-BE-NEXT:    mtvsrd v2, r3
-; CHECK-BE-NEXT:    mtvsrd v3, r4
-; CHECK-BE-NEXT:    mtvsrd v10, r11
-; CHECK-BE-NEXT:    mtvsrd v14, r12
-; CHECK-BE-NEXT:    mtvsrd v4, r5
-; CHECK-BE-NEXT:    mtvsrd v5, r6
-; CHECK-BE-NEXT:    mtvsrd v0, r7
-; CHECK-BE-NEXT:    mtvsrd v1, r8
-; CHECK-BE-NEXT:    mtvsrd v6, r9
-; CHECK-BE-NEXT:    mtvsrd v7, r10
-; CHECK-BE-NEXT:    mtvsrd v8, r0
-; CHECK-BE-NEXT:    mtvsrd v9, r30
-; CHECK-BE-NEXT:    ld r30, -16(r1) # 8-byte Folded Reload
-; CHECK-BE-NEXT:    mtvsrd v11, r29
-; CHECK-BE-NEXT:    ld r29, -24(r1) # 8-byte Folded Reload
-; CHECK-BE-NEXT:    mtvsrd v12, r28
-; CHECK-BE-NEXT:    ld r28, -32(r1) # 8-byte Folded Reload
-; CHECK-BE-NEXT:    mtvsrd v13, r27
-; CHECK-BE-NEXT:    ld r27, -40(r1) # 8-byte Folded Reload
-; CHECK-BE-NEXT:    mtvsrd v15, r26
-; CHECK-BE-NEXT:    ld r26, -48(r1) # 8-byte Folded Reload
-; CHECK-BE-NEXT:    vmrghb v4, v5, v4
-; CHECK-BE-NEXT:    vmrghb v2, v2, v0
-; CHECK-BE-NEXT:    vmrghb v5, v6, v1
-; CHECK-BE-NEXT:    vmrghb v3, v3, v7
-; CHECK-BE-NEXT:    vmrghb v0, v9, v8
-; CHECK-BE-NEXT:    vmrghb v1, v10, v11
-; CHECK-BE-NEXT:    vmrghb v6, v13, v12
-; CHECK-BE-NEXT:    vmrghb v7, v14, v15
-; CHECK-BE-NEXT:    vmrghh v2, v2, v4
-; CHECK-BE-NEXT:    vmrghh v3, v3, v5
-; CHECK-BE-NEXT:    vmrghh v4, v1, v0
-; CHECK-BE-NEXT:    vmrghh v5, v7, v6
+; CHECK-BE-NEXT:    mtvsrd v4, r3
+; CHECK-BE-NEXT:    mfvsrwz r3, f2
+; CHECK-BE-NEXT:    xxsldwi vs2, vs1, vs1, 3
+; CHECK-BE-NEXT:    sldi r3, r3, 56
+; CHECK-BE-NEXT:    xscvspdpn f2, vs2
+; CHECK-BE-NEXT:    mtvsrd v5, r3
+; CHECK-BE-NEXT:    xscvdpsxws f2, f2
+; CHECK-BE-NEXT:    mfvsrwz r3, f2
+; CHECK-BE-NEXT:    xxswapd vs2, vs1
+; CHECK-BE-NEXT:    xscvspdpn f2, vs2
+; CHECK-BE-NEXT:    xscvdpsxws f2, f2
+; CHECK-BE-NEXT:    vmrghb v4, v4, v5
+; CHECK-BE-NEXT:    vmrghh v3, v4, v3
+; CHECK-BE-NEXT:    sldi r3, r3, 56
 ; CHECK-BE-NEXT:    vmrghw v2, v3, v2
-; CHECK-BE-NEXT:    vmrghw v3, v5, v4
+; CHECK-BE-NEXT:    mtvsrd v3, r3
+; CHECK-BE-NEXT:    mfvsrwz r3, f2
+; CHECK-BE-NEXT:    xscvspdpn f2, vs1
+; CHECK-BE-NEXT:    xxsldwi vs1, vs1, vs1, 1
+; CHECK-BE-NEXT:    sldi r3, r3, 56
+; CHECK-BE-NEXT:    xscvdpsxws f2, f2
+; CHECK-BE-NEXT:    xscvspdpn f1, vs1
+; CHECK-BE-NEXT:    mtvsrd v4, r3
+; CHECK-BE-NEXT:    xscvdpsxws f1, f1
+; CHECK-BE-NEXT:    vmrghb v3, v4, v3
+; CHECK-BE-NEXT:    mfvsrwz r3, f2
+; CHECK-BE-NEXT:    sldi r3, r3, 56
+; CHECK-BE-NEXT:    mtvsrd v4, r3
+; CHECK-BE-NEXT:    mfvsrwz r3, f1
+; CHECK-BE-NEXT:    xxsldwi vs1, vs0, vs0, 3
+; CHECK-BE-NEXT:    sldi r3, r3, 56
+; CHECK-BE-NEXT:    xscvspdpn f1, vs1
+; CHECK-BE-NEXT:    mtvsrd v5, r3
+; CHECK-BE-NEXT:    xscvdpsxws f1, f1
+; CHECK-BE-NEXT:    mfvsrwz r3, f1
+; CHECK-BE-NEXT:    xxswapd vs1, vs0
+; CHECK-BE-NEXT:    xscvspdpn f1, vs1
+; CHECK-BE-NEXT:    xscvdpsxws f1, f1
+; CHECK-BE-NEXT:    vmrghb v4, v4, v5
+; CHECK-BE-NEXT:    sldi r3, r3, 56
+; CHECK-BE-NEXT:    vmrghh v3, v4, v3
+; CHECK-BE-NEXT:    mtvsrd v4, r3
+; CHECK-BE-NEXT:    mfvsrwz r3, f1
+; CHECK-BE-NEXT:    xscvspdpn f1, vs0
+; CHECK-BE-NEXT:    xxsldwi vs0, vs0, vs0, 1
+; CHECK-BE-NEXT:    sldi r3, r3, 56
+; CHECK-BE-NEXT:    xscvdpsxws f1, f1
+; CHECK-BE-NEXT:    xscvspdpn f0, vs0
+; CHECK-BE-NEXT:    mtvsrd v5, r3
+; CHECK-BE-NEXT:    xscvdpsxws f0, f0
+; CHECK-BE-NEXT:    vmrghb v4, v5, v4
+; CHECK-BE-NEXT:    mfvsrwz r3, f1
+; CHECK-BE-NEXT:    sldi r3, r3, 56
+; CHECK-BE-NEXT:    mtvsrd v5, r3
+; CHECK-BE-NEXT:    mfvsrwz r3, f0
+; CHECK-BE-NEXT:    sldi r3, r3, 56
+; CHECK-BE-NEXT:    mtvsrd v0, r3
+; CHECK-BE-NEXT:    vmrghb v5, v5, v0
+; CHECK-BE-NEXT:    vmrghh v4, v5, v4
+; CHECK-BE-NEXT:    vmrghw v3, v4, v3
 ; CHECK-BE-NEXT:    xxmrghd v2, v3, v2
 ; CHECK-BE-NEXT:    blr
 entry:
diff --git a/test/CodeGen/PowerPC/vec_conv_fp64_to_i16_elts.ll b/test/CodeGen/PowerPC/vec_conv_fp64_to_i16_elts.ll
index a713b52a740a8c51e86228952dfe7a64568e1d86..9e87876f81f15c0a778c562385cefae80fefc889 100644
--- a/test/CodeGen/PowerPC/vec_conv_fp64_to_i16_elts.ll
+++ b/test/CodeGen/PowerPC/vec_conv_fp64_to_i16_elts.ll
@@ -28,33 +28,33 @@ define i32 @test2elt(<2 x double> %a) local_unnamed_addr #0 {
 ;
 ; CHECK-P9-LABEL: test2elt:
 ; CHECK-P9:       # %bb.0: # %entry
+; CHECK-P9-NEXT:    xscvdpsxws f0, v2
+; CHECK-P9-NEXT:    mfvsrwz r3, f0
+; CHECK-P9-NEXT:    mtvsrd f0, r3
+; CHECK-P9-NEXT:    xxswapd v3, vs0
 ; CHECK-P9-NEXT:    xxswapd vs0, v2
-; CHECK-P9-NEXT:    xscvdpsxws f1, v2
 ; CHECK-P9-NEXT:    xscvdpsxws f0, f0
-; CHECK-P9-NEXT:    mfvsrwz r3, f1
-; CHECK-P9-NEXT:    mfvsrwz r4, f0
+; CHECK-P9-NEXT:    mfvsrwz r3, f0
 ; CHECK-P9-NEXT:    mtvsrd f0, r3
 ; CHECK-P9-NEXT:    li r3, 0
-; CHECK-P9-NEXT:    mtvsrd f1, r4
 ; CHECK-P9-NEXT:    xxswapd v2, vs0
-; CHECK-P9-NEXT:    xxswapd v3, vs1
-; CHECK-P9-NEXT:    vmrglh v2, v2, v3
+; CHECK-P9-NEXT:    vmrglh v2, v3, v2
 ; CHECK-P9-NEXT:    vextuwrx r3, r3, v2
 ; CHECK-P9-NEXT:    blr
 ;
 ; CHECK-BE-LABEL: test2elt:
 ; CHECK-BE:       # %bb.0: # %entry
+; CHECK-BE-NEXT:    xscvdpsxws f0, v2
+; CHECK-BE-NEXT:    mfvsrwz r3, f0
 ; CHECK-BE-NEXT:    xxswapd vs0, v2
-; CHECK-BE-NEXT:    xscvdpsxws f1, v2
+; CHECK-BE-NEXT:    sldi r3, r3, 48
 ; CHECK-BE-NEXT:    xscvdpsxws f0, f0
-; CHECK-BE-NEXT:    mfvsrwz r3, f1
+; CHECK-BE-NEXT:    mtvsrd v3, r3
+; CHECK-BE-NEXT:    mfvsrwz r3, f0
 ; CHECK-BE-NEXT:    sldi r3, r3, 48
-; CHECK-BE-NEXT:    mfvsrwz r4, f0
 ; CHECK-BE-NEXT:    mtvsrd v2, r3
 ; CHECK-BE-NEXT:    li r3, 0
-; CHECK-BE-NEXT:    sldi r4, r4, 48
-; CHECK-BE-NEXT:    mtvsrd v3, r4
-; CHECK-BE-NEXT:    vmrghh v2, v2, v3
+; CHECK-BE-NEXT:    vmrghh v2, v3, v2
 ; CHECK-BE-NEXT:    vextuwlx r3, r3, v2
 ; CHECK-BE-NEXT:    blr
 entry:
@@ -96,56 +96,56 @@ define i64 @test4elt(<4 x double>* nocapture readonly) local_unnamed_addr #1 {
 ;
 ; CHECK-P9-LABEL: test4elt:
 ; CHECK-P9:       # %bb.0: # %entry
-; CHECK-P9-NEXT:    lxv vs0, 16(r3)
 ; CHECK-P9-NEXT:    lxv vs1, 0(r3)
-; CHECK-P9-NEXT:    xxswapd vs2, vs1
-; CHECK-P9-NEXT:    xxswapd vs3, vs0
+; CHECK-P9-NEXT:    xscvdpsxws f2, f1
+; CHECK-P9-NEXT:    xxswapd vs1, vs1
 ; CHECK-P9-NEXT:    xscvdpsxws f1, f1
+; CHECK-P9-NEXT:    lxv vs0, 16(r3)
+; CHECK-P9-NEXT:    mfvsrwz r3, f2
+; CHECK-P9-NEXT:    mtvsrd f2, r3
+; CHECK-P9-NEXT:    mfvsrwz r3, f1
+; CHECK-P9-NEXT:    xxswapd v2, vs2
+; CHECK-P9-NEXT:    mtvsrd f1, r3
+; CHECK-P9-NEXT:    xxswapd v3, vs1
+; CHECK-P9-NEXT:    xscvdpsxws f1, f0
+; CHECK-P9-NEXT:    xxswapd vs0, vs0
 ; CHECK-P9-NEXT:    xscvdpsxws f0, f0
-; CHECK-P9-NEXT:    xscvdpsxws f2, f2
-; CHECK-P9-NEXT:    xscvdpsxws f3, f3
 ; CHECK-P9-NEXT:    mfvsrwz r3, f1
-; CHECK-P9-NEXT:    mfvsrwz r5, f0
+; CHECK-P9-NEXT:    mtvsrd f1, r3
+; CHECK-P9-NEXT:    mfvsrwz r3, f0
 ; CHECK-P9-NEXT:    mtvsrd f0, r3
-; CHECK-P9-NEXT:    mfvsrwz r4, f2
-; CHECK-P9-NEXT:    mfvsrwz r6, f3
-; CHECK-P9-NEXT:    mtvsrd f2, r5
-; CHECK-P9-NEXT:    xxswapd v2, vs0
-; CHECK-P9-NEXT:    mtvsrd f1, r4
-; CHECK-P9-NEXT:    mtvsrd f3, r6
-; CHECK-P9-NEXT:    xxswapd v4, vs2
-; CHECK-P9-NEXT:    xxswapd v3, vs1
-; CHECK-P9-NEXT:    xxswapd v5, vs3
 ; CHECK-P9-NEXT:    vmrglh v2, v2, v3
-; CHECK-P9-NEXT:    vmrglh v3, v4, v5
+; CHECK-P9-NEXT:    xxswapd v3, vs1
+; CHECK-P9-NEXT:    xxswapd v4, vs0
+; CHECK-P9-NEXT:    vmrglh v3, v3, v4
 ; CHECK-P9-NEXT:    vmrglw v2, v3, v2
 ; CHECK-P9-NEXT:    mfvsrld r3, v2
 ; CHECK-P9-NEXT:    blr
 ;
 ; CHECK-BE-LABEL: test4elt:
 ; CHECK-BE:       # %bb.0: # %entry
-; CHECK-BE-NEXT:    lxv vs0, 0(r3)
 ; CHECK-BE-NEXT:    lxv vs1, 16(r3)
-; CHECK-BE-NEXT:    xxswapd vs2, vs1
-; CHECK-BE-NEXT:    xxswapd vs3, vs0
+; CHECK-BE-NEXT:    xscvdpsxws f2, f1
+; CHECK-BE-NEXT:    xxswapd vs1, vs1
 ; CHECK-BE-NEXT:    xscvdpsxws f1, f1
-; CHECK-BE-NEXT:    xscvdpsxws f0, f0
-; CHECK-BE-NEXT:    xscvdpsxws f2, f2
-; CHECK-BE-NEXT:    xscvdpsxws f3, f3
-; CHECK-BE-NEXT:    mfvsrwz r3, f1
-; CHECK-BE-NEXT:    mfvsrwz r5, f0
+; CHECK-BE-NEXT:    lxv vs0, 0(r3)
+; CHECK-BE-NEXT:    mfvsrwz r3, f2
 ; CHECK-BE-NEXT:    sldi r3, r3, 48
-; CHECK-BE-NEXT:    sldi r5, r5, 48
-; CHECK-BE-NEXT:    mfvsrwz r4, f2
-; CHECK-BE-NEXT:    mfvsrwz r6, f3
 ; CHECK-BE-NEXT:    mtvsrd v2, r3
-; CHECK-BE-NEXT:    mtvsrd v4, r5
-; CHECK-BE-NEXT:    sldi r4, r4, 48
-; CHECK-BE-NEXT:    sldi r6, r6, 48
-; CHECK-BE-NEXT:    mtvsrd v3, r4
-; CHECK-BE-NEXT:    mtvsrd v5, r6
+; CHECK-BE-NEXT:    mfvsrwz r3, f1
+; CHECK-BE-NEXT:    xscvdpsxws f1, f0
+; CHECK-BE-NEXT:    xxswapd vs0, vs0
+; CHECK-BE-NEXT:    sldi r3, r3, 48
+; CHECK-BE-NEXT:    xscvdpsxws f0, f0
+; CHECK-BE-NEXT:    mtvsrd v3, r3
 ; CHECK-BE-NEXT:    vmrghh v2, v2, v3
-; CHECK-BE-NEXT:    vmrghh v3, v4, v5
+; CHECK-BE-NEXT:    mfvsrwz r3, f1
+; CHECK-BE-NEXT:    sldi r3, r3, 48
+; CHECK-BE-NEXT:    mtvsrd v3, r3
+; CHECK-BE-NEXT:    mfvsrwz r3, f0
+; CHECK-BE-NEXT:    sldi r3, r3, 48
+; CHECK-BE-NEXT:    mtvsrd v4, r3
+; CHECK-BE-NEXT:    vmrghh v3, v3, v4
 ; CHECK-BE-NEXT:    vmrghw v2, v3, v2
 ; CHECK-BE-NEXT:    mfvsrd r3, v2
 ; CHECK-BE-NEXT:    blr
@@ -213,103 +213,103 @@ define <8 x i16> @test8elt(<8 x double>* nocapture readonly) local_unnamed_addr
 ;
 ; CHECK-P9-LABEL: test8elt:
 ; CHECK-P9:       # %bb.0: # %entry
+; CHECK-P9-NEXT:    lxv vs3, 0(r3)
+; CHECK-P9-NEXT:    xscvdpsxws f4, f3
+; CHECK-P9-NEXT:    xxswapd vs3, vs3
+; CHECK-P9-NEXT:    xscvdpsxws f3, f3
 ; CHECK-P9-NEXT:    lxv vs0, 48(r3)
 ; CHECK-P9-NEXT:    lxv vs1, 32(r3)
 ; CHECK-P9-NEXT:    lxv vs2, 16(r3)
-; CHECK-P9-NEXT:    lxv vs3, 0(r3)
-; CHECK-P9-NEXT:    xxswapd vs4, vs3
-; CHECK-P9-NEXT:    xxswapd vs5, vs2
-; CHECK-P9-NEXT:    xxswapd vs6, vs1
-; CHECK-P9-NEXT:    xxswapd vs7, vs0
-; CHECK-P9-NEXT:    xscvdpsxws f3, f3
+; CHECK-P9-NEXT:    mfvsrwz r3, f4
+; CHECK-P9-NEXT:    mtvsrd f4, r3
+; CHECK-P9-NEXT:    mfvsrwz r3, f3
+; CHECK-P9-NEXT:    xxswapd v2, vs4
+; CHECK-P9-NEXT:    mtvsrd f3, r3
+; CHECK-P9-NEXT:    xxswapd v3, vs3
+; CHECK-P9-NEXT:    xscvdpsxws f3, f2
+; CHECK-P9-NEXT:    xxswapd vs2, vs2
 ; CHECK-P9-NEXT:    xscvdpsxws f2, f2
-; CHECK-P9-NEXT:    xscvdpsxws f1, f1
-; CHECK-P9-NEXT:    xscvdpsxws f0, f0
-; CHECK-P9-NEXT:    xscvdpsxws f4, f4
-; CHECK-P9-NEXT:    xscvdpsxws f5, f5
-; CHECK-P9-NEXT:    xscvdpsxws f6, f6
-; CHECK-P9-NEXT:    xscvdpsxws f7, f7
 ; CHECK-P9-NEXT:    mfvsrwz r3, f3
-; CHECK-P9-NEXT:    mfvsrwz r5, f2
-; CHECK-P9-NEXT:    mfvsrwz r7, f1
-; CHECK-P9-NEXT:    mfvsrwz r9, f0
-; CHECK-P9-NEXT:    mtvsrd f0, r3
-; CHECK-P9-NEXT:    mtvsrd f2, r5
-; CHECK-P9-NEXT:    mfvsrwz r4, f4
-; CHECK-P9-NEXT:    mfvsrwz r6, f5
-; CHECK-P9-NEXT:    mfvsrwz r8, f6
-; CHECK-P9-NEXT:    mfvsrwz r10, f7
-; CHECK-P9-NEXT:    mtvsrd f4, r7
-; CHECK-P9-NEXT:    mtvsrd f6, r9
-; CHECK-P9-NEXT:    xxswapd v2, vs0
+; CHECK-P9-NEXT:    mtvsrd f3, r3
+; CHECK-P9-NEXT:    mfvsrwz r3, f2
+; CHECK-P9-NEXT:    mtvsrd f2, r3
 ; CHECK-P9-NEXT:    xxswapd v4, vs2
-; CHECK-P9-NEXT:    mtvsrd f1, r4
-; CHECK-P9-NEXT:    mtvsrd f3, r6
-; CHECK-P9-NEXT:    mtvsrd f5, r8
-; CHECK-P9-NEXT:    mtvsrd f7, r10
-; CHECK-P9-NEXT:    xxswapd v0, vs4
-; CHECK-P9-NEXT:    xxswapd v6, vs6
-; CHECK-P9-NEXT:    xxswapd v3, vs1
-; CHECK-P9-NEXT:    xxswapd v5, vs3
-; CHECK-P9-NEXT:    xxswapd v1, vs5
-; CHECK-P9-NEXT:    xxswapd v7, vs7
+; CHECK-P9-NEXT:    xscvdpsxws f2, f1
+; CHECK-P9-NEXT:    xxswapd vs1, vs1
+; CHECK-P9-NEXT:    xscvdpsxws f1, f1
+; CHECK-P9-NEXT:    mfvsrwz r3, f2
+; CHECK-P9-NEXT:    mtvsrd f2, r3
+; CHECK-P9-NEXT:    mfvsrwz r3, f1
 ; CHECK-P9-NEXT:    vmrglh v2, v2, v3
-; CHECK-P9-NEXT:    vmrglh v3, v4, v5
-; CHECK-P9-NEXT:    vmrglh v4, v0, v1
-; CHECK-P9-NEXT:    vmrglh v5, v6, v7
+; CHECK-P9-NEXT:    xxswapd v3, vs3
+; CHECK-P9-NEXT:    vmrglh v3, v3, v4
 ; CHECK-P9-NEXT:    vmrglw v2, v3, v2
-; CHECK-P9-NEXT:    vmrglw v3, v5, v4
+; CHECK-P9-NEXT:    xxswapd v3, vs2
+; CHECK-P9-NEXT:    mtvsrd f1, r3
+; CHECK-P9-NEXT:    xxswapd v4, vs1
+; CHECK-P9-NEXT:    xscvdpsxws f1, f0
+; CHECK-P9-NEXT:    xxswapd vs0, vs0
+; CHECK-P9-NEXT:    xscvdpsxws f0, f0
+; CHECK-P9-NEXT:    mfvsrwz r3, f1
+; CHECK-P9-NEXT:    mtvsrd f1, r3
+; CHECK-P9-NEXT:    mfvsrwz r3, f0
+; CHECK-P9-NEXT:    mtvsrd f0, r3
+; CHECK-P9-NEXT:    vmrglh v3, v3, v4
+; CHECK-P9-NEXT:    xxswapd v4, vs1
+; CHECK-P9-NEXT:    xxswapd v5, vs0
+; CHECK-P9-NEXT:    vmrglh v4, v4, v5
+; CHECK-P9-NEXT:    vmrglw v3, v4, v3
 ; CHECK-P9-NEXT:    xxmrgld v2, v3, v2
 ; CHECK-P9-NEXT:    blr
 ;
 ; CHECK-BE-LABEL: test8elt:
 ; CHECK-BE:       # %bb.0: # %entry
-; CHECK-BE-NEXT:    lxv vs0, 0(r3)
-; CHECK-BE-NEXT:    lxv vs1, 16(r3)
-; CHECK-BE-NEXT:    lxv vs2, 32(r3)
 ; CHECK-BE-NEXT:    lxv vs3, 48(r3)
-; CHECK-BE-NEXT:    xxswapd vs4, vs3
-; CHECK-BE-NEXT:    xxswapd vs5, vs2
-; CHECK-BE-NEXT:    xxswapd vs6, vs1
-; CHECK-BE-NEXT:    xxswapd vs7, vs0
+; CHECK-BE-NEXT:    xscvdpsxws f4, f3
+; CHECK-BE-NEXT:    xxswapd vs3, vs3
 ; CHECK-BE-NEXT:    xscvdpsxws f3, f3
-; CHECK-BE-NEXT:    xscvdpsxws f2, f2
-; CHECK-BE-NEXT:    xscvdpsxws f1, f1
-; CHECK-BE-NEXT:    xscvdpsxws f0, f0
-; CHECK-BE-NEXT:    xscvdpsxws f4, f4
-; CHECK-BE-NEXT:    xscvdpsxws f5, f5
-; CHECK-BE-NEXT:    xscvdpsxws f6, f6
-; CHECK-BE-NEXT:    xscvdpsxws f7, f7
-; CHECK-BE-NEXT:    mfvsrwz r3, f3
-; CHECK-BE-NEXT:    mfvsrwz r5, f2
-; CHECK-BE-NEXT:    mfvsrwz r7, f1
-; CHECK-BE-NEXT:    mfvsrwz r9, f0
+; CHECK-BE-NEXT:    lxv vs2, 32(r3)
+; CHECK-BE-NEXT:    lxv vs0, 0(r3)
+; CHECK-BE-NEXT:    lxv vs1, 16(r3)
+; CHECK-BE-NEXT:    mfvsrwz r3, f4
 ; CHECK-BE-NEXT:    sldi r3, r3, 48
-; CHECK-BE-NEXT:    sldi r5, r5, 48
-; CHECK-BE-NEXT:    sldi r7, r7, 48
-; CHECK-BE-NEXT:    sldi r9, r9, 48
-; CHECK-BE-NEXT:    mfvsrwz r4, f4
-; CHECK-BE-NEXT:    mfvsrwz r6, f5
-; CHECK-BE-NEXT:    mfvsrwz r8, f6
-; CHECK-BE-NEXT:    mfvsrwz r10, f7
 ; CHECK-BE-NEXT:    mtvsrd v2, r3
-; CHECK-BE-NEXT:    mtvsrd v4, r5
-; CHECK-BE-NEXT:    mtvsrd v0, r7
-; CHECK-BE-NEXT:    mtvsrd v6, r9
-; CHECK-BE-NEXT:    sldi r4, r4, 48
-; CHECK-BE-NEXT:    sldi r6, r6, 48
-; CHECK-BE-NEXT:    sldi r8, r8, 48
-; CHECK-BE-NEXT:    sldi r10, r10, 48
-; CHECK-BE-NEXT:    mtvsrd v3, r4
-; CHECK-BE-NEXT:    mtvsrd v5, r6
-; CHECK-BE-NEXT:    mtvsrd v1, r8
-; CHECK-BE-NEXT:    mtvsrd v7, r10
+; CHECK-BE-NEXT:    mfvsrwz r3, f3
+; CHECK-BE-NEXT:    xscvdpsxws f3, f2
+; CHECK-BE-NEXT:    xxswapd vs2, vs2
+; CHECK-BE-NEXT:    sldi r3, r3, 48
+; CHECK-BE-NEXT:    xscvdpsxws f2, f2
+; CHECK-BE-NEXT:    mtvsrd v3, r3
 ; CHECK-BE-NEXT:    vmrghh v2, v2, v3
-; CHECK-BE-NEXT:    vmrghh v3, v4, v5
-; CHECK-BE-NEXT:    vmrghh v4, v0, v1
-; CHECK-BE-NEXT:    vmrghh v5, v6, v7
+; CHECK-BE-NEXT:    mfvsrwz r3, f3
+; CHECK-BE-NEXT:    sldi r3, r3, 48
+; CHECK-BE-NEXT:    mtvsrd v3, r3
+; CHECK-BE-NEXT:    mfvsrwz r3, f2
+; CHECK-BE-NEXT:    xscvdpsxws f2, f1
+; CHECK-BE-NEXT:    xxswapd vs1, vs1
+; CHECK-BE-NEXT:    sldi r3, r3, 48
+; CHECK-BE-NEXT:    xscvdpsxws f1, f1
+; CHECK-BE-NEXT:    mtvsrd v4, r3
+; CHECK-BE-NEXT:    vmrghh v3, v3, v4
+; CHECK-BE-NEXT:    mfvsrwz r3, f2
+; CHECK-BE-NEXT:    sldi r3, r3, 48
 ; CHECK-BE-NEXT:    vmrghw v2, v3, v2
-; CHECK-BE-NEXT:    vmrghw v3, v5, v4
+; CHECK-BE-NEXT:    mtvsrd v3, r3
+; CHECK-BE-NEXT:    mfvsrwz r3, f1
+; CHECK-BE-NEXT:    xscvdpsxws f1, f0
+; CHECK-BE-NEXT:    xxswapd vs0, vs0
+; CHECK-BE-NEXT:    sldi r3, r3, 48
+; CHECK-BE-NEXT:    xscvdpsxws f0, f0
+; CHECK-BE-NEXT:    mtvsrd v4, r3
+; CHECK-BE-NEXT:    vmrghh v3, v3, v4
+; CHECK-BE-NEXT:    mfvsrwz r3, f1
+; CHECK-BE-NEXT:    sldi r3, r3, 48
+; CHECK-BE-NEXT:    mtvsrd v4, r3
+; CHECK-BE-NEXT:    mfvsrwz r3, f0
+; CHECK-BE-NEXT:    sldi r3, r3, 48
+; CHECK-BE-NEXT:    mtvsrd v5, r3
+; CHECK-BE-NEXT:    vmrghh v4, v4, v5
+; CHECK-BE-NEXT:    vmrghw v3, v4, v3
 ; CHECK-BE-NEXT:    xxmrghd v2, v3, v2
 ; CHECK-BE-NEXT:    blr
 entry:
@@ -428,226 +428,202 @@ define void @test16elt(<16 x i16>* noalias nocapture sret %agg.result, <16 x dou
 ;
 ; CHECK-P9-LABEL: test16elt:
 ; CHECK-P9:       # %bb.0: # %entry
-; CHECK-P9-NEXT:    lxv vs2, 48(r4)
-; CHECK-P9-NEXT:    lxv vs4, 32(r4)
-; CHECK-P9-NEXT:    lxv vs5, 16(r4)
-; CHECK-P9-NEXT:    lxv vs6, 0(r4)
-; CHECK-P9-NEXT:    lxv vs0, 112(r4)
-; CHECK-P9-NEXT:    lxv vs1, 96(r4)
-; CHECK-P9-NEXT:    lxv vs3, 80(r4)
-; CHECK-P9-NEXT:    lxv vs7, 64(r4)
-; CHECK-P9-NEXT:    std r25, -56(r1) # 8-byte Folded Spill
-; CHECK-P9-NEXT:    std r26, -48(r1) # 8-byte Folded Spill
-; CHECK-P9-NEXT:    std r27, -40(r1) # 8-byte Folded Spill
-; CHECK-P9-NEXT:    std r28, -32(r1) # 8-byte Folded Spill
-; CHECK-P9-NEXT:    std r29, -24(r1) # 8-byte Folded Spill
-; CHECK-P9-NEXT:    std r30, -16(r1) # 8-byte Folded Spill
-; CHECK-P9-NEXT:    xxswapd vs8, vs6
-; CHECK-P9-NEXT:    xxswapd vs9, vs5
-; CHECK-P9-NEXT:    xxswapd vs10, vs4
-; CHECK-P9-NEXT:    xxswapd vs11, vs2
-; CHECK-P9-NEXT:    xxswapd vs12, vs7
-; CHECK-P9-NEXT:    xxswapd vs13, vs3
-; CHECK-P9-NEXT:    xxswapd v2, vs1
-; CHECK-P9-NEXT:    xxswapd v3, vs0
-; CHECK-P9-NEXT:    xscvdpsxws f6, f6
-; CHECK-P9-NEXT:    xscvdpsxws f5, f5
+; CHECK-P9-NEXT:    lxv vs4, 0(r4)
+; CHECK-P9-NEXT:    lxv vs3, 16(r4)
+; CHECK-P9-NEXT:    lxv vs2, 32(r4)
+; CHECK-P9-NEXT:    xscvdpsxws f5, f4
+; CHECK-P9-NEXT:    lxv vs1, 48(r4)
+; CHECK-P9-NEXT:    xscvdpsxws f6, f3
+; CHECK-P9-NEXT:    lxv vs0, 64(r4)
+; CHECK-P9-NEXT:    xscvdpsxws f7, f2
+; CHECK-P9-NEXT:    xscvdpsxws f8, f1
+; CHECK-P9-NEXT:    xxswapd vs4, vs4
 ; CHECK-P9-NEXT:    xscvdpsxws f4, f4
-; CHECK-P9-NEXT:    xscvdpsxws f2, f2
-; CHECK-P9-NEXT:    xscvdpsxws f7, f7
+; CHECK-P9-NEXT:    mfvsrwz r5, f5
+; CHECK-P9-NEXT:    xscvdpsxws f9, f0
+; CHECK-P9-NEXT:    xxswapd vs3, vs3
 ; CHECK-P9-NEXT:    xscvdpsxws f3, f3
+; CHECK-P9-NEXT:    mtvsrd f5, r5
+; CHECK-P9-NEXT:    mfvsrwz r5, f6
+; CHECK-P9-NEXT:    xxswapd vs2, vs2
+; CHECK-P9-NEXT:    xscvdpsxws f2, f2
+; CHECK-P9-NEXT:    mtvsrd f6, r5
+; CHECK-P9-NEXT:    mfvsrwz r5, f7
+; CHECK-P9-NEXT:    mtvsrd f7, r5
+; CHECK-P9-NEXT:    mfvsrwz r5, f8
+; CHECK-P9-NEXT:    mtvsrd f8, r5
+; CHECK-P9-NEXT:    mfvsrwz r5, f9
+; CHECK-P9-NEXT:    mtvsrd f9, r5
+; CHECK-P9-NEXT:    mfvsrwz r5, f4
+; CHECK-P9-NEXT:    mtvsrd f4, r5
+; CHECK-P9-NEXT:    mfvsrwz r5, f3
+; CHECK-P9-NEXT:    xxswapd vs1, vs1
 ; CHECK-P9-NEXT:    xscvdpsxws f1, f1
+; CHECK-P9-NEXT:    xxswapd v2, vs5
+; CHECK-P9-NEXT:    xxswapd v5, vs8
+; CHECK-P9-NEXT:    xxswapd v0, vs9
+; CHECK-P9-NEXT:    mtvsrd f3, r5
+; CHECK-P9-NEXT:    mfvsrwz r5, f2
+; CHECK-P9-NEXT:    mtvsrd f2, r5
+; CHECK-P9-NEXT:    xxswapd vs0, vs0
 ; CHECK-P9-NEXT:    xscvdpsxws f0, f0
-; CHECK-P9-NEXT:    xscvdpsxws f8, f8
-; CHECK-P9-NEXT:    xscvdpsxws f9, f9
-; CHECK-P9-NEXT:    xscvdpsxws f10, f10
-; CHECK-P9-NEXT:    xscvdpsxws f11, f11
-; CHECK-P9-NEXT:    xscvdpsxws f12, f12
-; CHECK-P9-NEXT:    xscvdpsxws f13, f13
-; CHECK-P9-NEXT:    xscvdpsxws v2, v2
-; CHECK-P9-NEXT:    xscvdpsxws v3, v3
-; CHECK-P9-NEXT:    mfvsrwz r4, f6
-; CHECK-P9-NEXT:    mfvsrwz r5, f5
-; CHECK-P9-NEXT:    mfvsrwz r6, f4
-; CHECK-P9-NEXT:    mfvsrwz r7, f2
-; CHECK-P9-NEXT:    mfvsrwz r12, f7
-; CHECK-P9-NEXT:    mfvsrwz r0, f3
-; CHECK-P9-NEXT:    mfvsrwz r30, f1
-; CHECK-P9-NEXT:    mfvsrwz r29, f0
-; CHECK-P9-NEXT:    mfvsrwz r8, f8
-; CHECK-P9-NEXT:    mfvsrwz r9, f9
-; CHECK-P9-NEXT:    mfvsrwz r10, f10
-; CHECK-P9-NEXT:    mfvsrwz r11, f11
-; CHECK-P9-NEXT:    mfvsrwz r28, f12
-; CHECK-P9-NEXT:    mfvsrwz r27, f13
-; CHECK-P9-NEXT:    mfvsrwz r26, v2
-; CHECK-P9-NEXT:    mfvsrwz r25, v3
-; CHECK-P9-NEXT:    mtvsrd f0, r4
+; CHECK-P9-NEXT:    xxswapd v1, vs2
+; CHECK-P9-NEXT:    lxv vs2, 80(r4)
+; CHECK-P9-NEXT:    xxswapd v3, vs4
+; CHECK-P9-NEXT:    vmrglh v2, v2, v3
+; CHECK-P9-NEXT:    xxswapd v3, vs6
+; CHECK-P9-NEXT:    xxswapd v4, vs3
+; CHECK-P9-NEXT:    xscvdpsxws f3, f2
+; CHECK-P9-NEXT:    xxswapd vs2, vs2
+; CHECK-P9-NEXT:    mfvsrwz r5, f1
+; CHECK-P9-NEXT:    vmrglh v3, v3, v4
+; CHECK-P9-NEXT:    xxswapd v4, vs7
 ; CHECK-P9-NEXT:    mtvsrd f1, r5
-; CHECK-P9-NEXT:    mtvsrd f2, r6
-; CHECK-P9-NEXT:    mtvsrd f3, r7
-; CHECK-P9-NEXT:    mtvsrd f8, r12
-; CHECK-P9-NEXT:    mtvsrd f9, r0
-; CHECK-P9-NEXT:    mtvsrd f10, r30
-; CHECK-P9-NEXT:    mtvsrd f11, r29
-; CHECK-P9-NEXT:    mtvsrd f4, r8
-; CHECK-P9-NEXT:    mtvsrd f5, r9
-; CHECK-P9-NEXT:    mtvsrd f6, r10
-; CHECK-P9-NEXT:    mtvsrd f7, r11
-; CHECK-P9-NEXT:    mtvsrd f12, r28
-; CHECK-P9-NEXT:    mtvsrd f13, r27
-; CHECK-P9-NEXT:    mtvsrd v2, r26
-; CHECK-P9-NEXT:    mtvsrd v3, r25
-; CHECK-P9-NEXT:    xxswapd v4, vs0
-; CHECK-P9-NEXT:    xxswapd v5, vs1
-; CHECK-P9-NEXT:    xxswapd v0, vs2
-; CHECK-P9-NEXT:    xxswapd v1, vs3
-; CHECK-P9-NEXT:    ld r30, -16(r1) # 8-byte Folded Reload
-; CHECK-P9-NEXT:    ld r29, -24(r1) # 8-byte Folded Reload
-; CHECK-P9-NEXT:    ld r28, -32(r1) # 8-byte Folded Reload
-; CHECK-P9-NEXT:    ld r27, -40(r1) # 8-byte Folded Reload
-; CHECK-P9-NEXT:    xxswapd v6, vs4
-; CHECK-P9-NEXT:    xxswapd v7, vs5
-; CHECK-P9-NEXT:    ld r26, -48(r1) # 8-byte Folded Reload
-; CHECK-P9-NEXT:    ld r25, -56(r1) # 8-byte Folded Reload
-; CHECK-P9-NEXT:    xxswapd v8, vs6
-; CHECK-P9-NEXT:    xxswapd v9, vs7
-; CHECK-P9-NEXT:    xxswapd v10, vs8
-; CHECK-P9-NEXT:    xxswapd v11, vs12
-; CHECK-P9-NEXT:    xxswapd v12, vs9
-; CHECK-P9-NEXT:    xxswapd v13, vs13
-; CHECK-P9-NEXT:    xxswapd v14, vs10
-; CHECK-P9-NEXT:    xxswapd v2, v2
-; CHECK-P9-NEXT:    xxswapd v15, vs11
-; CHECK-P9-NEXT:    xxswapd v3, v3
-; CHECK-P9-NEXT:    vmrglh v4, v4, v6
-; CHECK-P9-NEXT:    vmrglh v5, v5, v7
-; CHECK-P9-NEXT:    vmrglh v0, v0, v8
-; CHECK-P9-NEXT:    vmrglh v1, v1, v9
-; CHECK-P9-NEXT:    vmrglh v6, v10, v11
-; CHECK-P9-NEXT:    vmrglh v7, v12, v13
-; CHECK-P9-NEXT:    vmrglh v2, v14, v2
-; CHECK-P9-NEXT:    vmrglh v3, v15, v3
-; CHECK-P9-NEXT:    vmrglw v4, v5, v4
-; CHECK-P9-NEXT:    vmrglw v5, v1, v0
-; CHECK-P9-NEXT:    vmrglw v0, v7, v6
+; CHECK-P9-NEXT:    mfvsrwz r5, f0
+; CHECK-P9-NEXT:    vmrglh v4, v4, v1
+; CHECK-P9-NEXT:    xxswapd v1, vs1
+; CHECK-P9-NEXT:    mtvsrd f0, r5
+; CHECK-P9-NEXT:    vmrglh v5, v5, v1
+; CHECK-P9-NEXT:    xscvdpsxws f2, f2
+; CHECK-P9-NEXT:    xxswapd v1, vs0
+; CHECK-P9-NEXT:    lxv vs0, 112(r4)
+; CHECK-P9-NEXT:    lxv vs1, 96(r4)
+; CHECK-P9-NEXT:    mfvsrwz r4, f3
+; CHECK-P9-NEXT:    mtvsrd f3, r4
+; CHECK-P9-NEXT:    mfvsrwz r4, f2
 ; CHECK-P9-NEXT:    vmrglw v2, v3, v2
-; CHECK-P9-NEXT:    xxmrgld vs0, v5, v4
-; CHECK-P9-NEXT:    xxmrgld vs1, v2, v0
-; CHECK-P9-NEXT:    stxv vs0, 0(r3)
-; CHECK-P9-NEXT:    stxv vs1, 16(r3)
+; CHECK-P9-NEXT:    vmrglw v3, v5, v4
+; CHECK-P9-NEXT:    xxmrgld vs4, v3, v2
+; CHECK-P9-NEXT:    xxswapd v2, vs3
+; CHECK-P9-NEXT:    vmrglh v0, v0, v1
+; CHECK-P9-NEXT:    mtvsrd f2, r4
+; CHECK-P9-NEXT:    xxswapd v3, vs2
+; CHECK-P9-NEXT:    xscvdpsxws f2, f1
+; CHECK-P9-NEXT:    xxswapd vs1, vs1
+; CHECK-P9-NEXT:    xscvdpsxws f1, f1
+; CHECK-P9-NEXT:    mfvsrwz r4, f2
+; CHECK-P9-NEXT:    mtvsrd f2, r4
+; CHECK-P9-NEXT:    mfvsrwz r4, f1
+; CHECK-P9-NEXT:    mtvsrd f1, r4
+; CHECK-P9-NEXT:    xxswapd v4, vs1
+; CHECK-P9-NEXT:    xscvdpsxws f1, f0
+; CHECK-P9-NEXT:    xxswapd vs0, vs0
+; CHECK-P9-NEXT:    xscvdpsxws f0, f0
+; CHECK-P9-NEXT:    mfvsrwz r4, f1
+; CHECK-P9-NEXT:    mtvsrd f1, r4
+; CHECK-P9-NEXT:    mfvsrwz r4, f0
+; CHECK-P9-NEXT:    vmrglh v2, v2, v3
+; CHECK-P9-NEXT:    xxswapd v3, vs2
+; CHECK-P9-NEXT:    vmrglh v3, v3, v4
+; CHECK-P9-NEXT:    xxswapd v4, vs1
+; CHECK-P9-NEXT:    vmrglw v2, v2, v0
+; CHECK-P9-NEXT:    mtvsrd f0, r4
+; CHECK-P9-NEXT:    xxswapd v5, vs0
+; CHECK-P9-NEXT:    vmrglh v4, v4, v5
+; CHECK-P9-NEXT:    vmrglw v3, v4, v3
+; CHECK-P9-NEXT:    xxmrgld vs0, v3, v2
+; CHECK-P9-NEXT:    stxv vs0, 16(r3)
+; CHECK-P9-NEXT:    stxv vs4, 0(r3)
 ; CHECK-P9-NEXT:    blr
 ;
 ; CHECK-BE-LABEL: test16elt:
 ; CHECK-BE:       # %bb.0: # %entry
-; CHECK-BE-NEXT:    lxv vs2, 0(r4)
-; CHECK-BE-NEXT:    lxv vs4, 16(r4)
-; CHECK-BE-NEXT:    lxv vs5, 32(r4)
-; CHECK-BE-NEXT:    lxv vs6, 48(r4)
-; CHECK-BE-NEXT:    lxv vs0, 64(r4)
-; CHECK-BE-NEXT:    lxv vs1, 80(r4)
-; CHECK-BE-NEXT:    lxv vs3, 96(r4)
-; CHECK-BE-NEXT:    lxv vs7, 112(r4)
-; CHECK-BE-NEXT:    std r25, -56(r1) # 8-byte Folded Spill
-; CHECK-BE-NEXT:    std r26, -48(r1) # 8-byte Folded Spill
-; CHECK-BE-NEXT:    std r27, -40(r1) # 8-byte Folded Spill
-; CHECK-BE-NEXT:    std r28, -32(r1) # 8-byte Folded Spill
-; CHECK-BE-NEXT:    std r29, -24(r1) # 8-byte Folded Spill
-; CHECK-BE-NEXT:    std r30, -16(r1) # 8-byte Folded Spill
-; CHECK-BE-NEXT:    xxswapd vs8, vs6
-; CHECK-BE-NEXT:    xxswapd vs9, vs5
-; CHECK-BE-NEXT:    xxswapd vs10, vs4
-; CHECK-BE-NEXT:    xxswapd vs11, vs2
-; CHECK-BE-NEXT:    xxswapd vs12, vs7
-; CHECK-BE-NEXT:    xxswapd vs13, vs3
-; CHECK-BE-NEXT:    xxswapd v2, vs1
-; CHECK-BE-NEXT:    xxswapd v3, vs0
-; CHECK-BE-NEXT:    xscvdpsxws f6, f6
-; CHECK-BE-NEXT:    xscvdpsxws f5, f5
+; CHECK-BE-NEXT:    lxv vs4, 48(r4)
+; CHECK-BE-NEXT:    xscvdpsxws f5, f4
+; CHECK-BE-NEXT:    xxswapd vs4, vs4
+; CHECK-BE-NEXT:    lxv vs3, 32(r4)
+; CHECK-BE-NEXT:    xscvdpsxws f6, f3
+; CHECK-BE-NEXT:    xxswapd vs3, vs3
 ; CHECK-BE-NEXT:    xscvdpsxws f4, f4
-; CHECK-BE-NEXT:    xscvdpsxws f2, f2
-; CHECK-BE-NEXT:    xscvdpsxws f7, f7
+; CHECK-BE-NEXT:    mfvsrwz r5, f5
+; CHECK-BE-NEXT:    sldi r5, r5, 48
+; CHECK-BE-NEXT:    lxv vs2, 16(r4)
 ; CHECK-BE-NEXT:    xscvdpsxws f3, f3
+; CHECK-BE-NEXT:    xscvdpsxws f7, f2
+; CHECK-BE-NEXT:    xxswapd vs2, vs2
+; CHECK-BE-NEXT:    xscvdpsxws f2, f2
+; CHECK-BE-NEXT:    mtvsrd v2, r5
+; CHECK-BE-NEXT:    mfvsrwz r5, f4
+; CHECK-BE-NEXT:    sldi r5, r5, 48
+; CHECK-BE-NEXT:    lxv vs1, 0(r4)
+; CHECK-BE-NEXT:    xscvdpsxws f4, f1
+; CHECK-BE-NEXT:    xxswapd vs1, vs1
 ; CHECK-BE-NEXT:    xscvdpsxws f1, f1
-; CHECK-BE-NEXT:    xscvdpsxws f0, f0
-; CHECK-BE-NEXT:    xscvdpsxws f8, f8
-; CHECK-BE-NEXT:    xscvdpsxws f9, f9
-; CHECK-BE-NEXT:    xscvdpsxws f10, f10
-; CHECK-BE-NEXT:    xscvdpsxws f11, f11
-; CHECK-BE-NEXT:    xscvdpsxws f12, f12
-; CHECK-BE-NEXT:    xscvdpsxws f13, f13
-; CHECK-BE-NEXT:    xscvdpsxws v2, v2
-; CHECK-BE-NEXT:    xscvdpsxws v3, v3
-; CHECK-BE-NEXT:    mfvsrwz r4, f6
-; CHECK-BE-NEXT:    mfvsrwz r5, f5
-; CHECK-BE-NEXT:    mfvsrwz r6, f4
-; CHECK-BE-NEXT:    mfvsrwz r7, f2
-; CHECK-BE-NEXT:    mfvsrwz r12, f7
-; CHECK-BE-NEXT:    mfvsrwz r0, f3
-; CHECK-BE-NEXT:    mfvsrwz r30, f1
-; CHECK-BE-NEXT:    mfvsrwz r29, f0
-; CHECK-BE-NEXT:    mfvsrwz r8, f8
-; CHECK-BE-NEXT:    mfvsrwz r9, f9
-; CHECK-BE-NEXT:    mfvsrwz r10, f10
-; CHECK-BE-NEXT:    mfvsrwz r11, f11
-; CHECK-BE-NEXT:    mfvsrwz r28, f12
-; CHECK-BE-NEXT:    mfvsrwz r27, f13
-; CHECK-BE-NEXT:    mfvsrwz r26, v2
-; CHECK-BE-NEXT:    mfvsrwz r25, v3
-; CHECK-BE-NEXT:    sldi r4, r4, 48
+; CHECK-BE-NEXT:    mtvsrd v3, r5
+; CHECK-BE-NEXT:    mfvsrwz r5, f6
 ; CHECK-BE-NEXT:    sldi r5, r5, 48
-; CHECK-BE-NEXT:    sldi r6, r6, 48
-; CHECK-BE-NEXT:    sldi r7, r7, 48
-; CHECK-BE-NEXT:    sldi r12, r12, 48
-; CHECK-BE-NEXT:    sldi r0, r0, 48
-; CHECK-BE-NEXT:    sldi r30, r30, 48
-; CHECK-BE-NEXT:    sldi r29, r29, 48
-; CHECK-BE-NEXT:    sldi r8, r8, 48
-; CHECK-BE-NEXT:    sldi r9, r9, 48
-; CHECK-BE-NEXT:    sldi r10, r10, 48
-; CHECK-BE-NEXT:    sldi r11, r11, 48
-; CHECK-BE-NEXT:    sldi r28, r28, 48
-; CHECK-BE-NEXT:    sldi r27, r27, 48
-; CHECK-BE-NEXT:    sldi r26, r26, 48
-; CHECK-BE-NEXT:    sldi r25, r25, 48
-; CHECK-BE-NEXT:    mtvsrd v2, r4
+; CHECK-BE-NEXT:    lxv vs0, 112(r4)
+; CHECK-BE-NEXT:    vmrghh v2, v2, v3
 ; CHECK-BE-NEXT:    mtvsrd v3, r5
-; CHECK-BE-NEXT:    mtvsrd v4, r6
-; CHECK-BE-NEXT:    mtvsrd v5, r7
-; CHECK-BE-NEXT:    mtvsrd v8, r12
-; CHECK-BE-NEXT:    mtvsrd v10, r0
-; CHECK-BE-NEXT:    mtvsrd v12, r30
-; CHECK-BE-NEXT:    ld r30, -16(r1) # 8-byte Folded Reload
-; CHECK-BE-NEXT:    mtvsrd v0, r8
-; CHECK-BE-NEXT:    mtvsrd v1, r9
-; CHECK-BE-NEXT:    mtvsrd v6, r10
-; CHECK-BE-NEXT:    mtvsrd v7, r11
-; CHECK-BE-NEXT:    mtvsrd v9, r28
-; CHECK-BE-NEXT:    ld r28, -32(r1) # 8-byte Folded Reload
-; CHECK-BE-NEXT:    mtvsrd v11, r27
-; CHECK-BE-NEXT:    ld r27, -40(r1) # 8-byte Folded Reload
-; CHECK-BE-NEXT:    mtvsrd v13, r26
-; CHECK-BE-NEXT:    mtvsrd v14, r29
-; CHECK-BE-NEXT:    ld r29, -24(r1) # 8-byte Folded Reload
-; CHECK-BE-NEXT:    ld r26, -48(r1) # 8-byte Folded Reload
-; CHECK-BE-NEXT:    mtvsrd v15, r25
-; CHECK-BE-NEXT:    ld r25, -56(r1) # 8-byte Folded Reload
-; CHECK-BE-NEXT:    vmrghh v2, v2, v0
-; CHECK-BE-NEXT:    vmrghh v3, v3, v1
-; CHECK-BE-NEXT:    vmrghh v4, v4, v6
-; CHECK-BE-NEXT:    vmrghh v5, v5, v7
-; CHECK-BE-NEXT:    vmrghh v0, v8, v9
-; CHECK-BE-NEXT:    vmrghh v1, v10, v11
-; CHECK-BE-NEXT:    vmrghh v6, v12, v13
-; CHECK-BE-NEXT:    vmrghh v7, v14, v15
+; CHECK-BE-NEXT:    mfvsrwz r5, f3
+; CHECK-BE-NEXT:    xscvdpsxws f3, f0
+; CHECK-BE-NEXT:    xxswapd vs0, vs0
+; CHECK-BE-NEXT:    sldi r5, r5, 48
+; CHECK-BE-NEXT:    xscvdpsxws f0, f0
+; CHECK-BE-NEXT:    mtvsrd v4, r5
+; CHECK-BE-NEXT:    mfvsrwz r5, f7
+; CHECK-BE-NEXT:    sldi r5, r5, 48
+; CHECK-BE-NEXT:    vmrghh v3, v3, v4
+; CHECK-BE-NEXT:    mtvsrd v4, r5
+; CHECK-BE-NEXT:    mfvsrwz r5, f4
 ; CHECK-BE-NEXT:    vmrghw v2, v3, v2
+; CHECK-BE-NEXT:    sldi r5, r5, 48
+; CHECK-BE-NEXT:    mtvsrd v5, r5
+; CHECK-BE-NEXT:    mfvsrwz r5, f3
+; CHECK-BE-NEXT:    sldi r5, r5, 48
+; CHECK-BE-NEXT:    mtvsrd v0, r5
+; CHECK-BE-NEXT:    mfvsrwz r5, f2
+; CHECK-BE-NEXT:    lxv vs2, 96(r4)
+; CHECK-BE-NEXT:    sldi r5, r5, 48
+; CHECK-BE-NEXT:    mtvsrd v1, r5
+; CHECK-BE-NEXT:    mfvsrwz r5, f1
+; CHECK-BE-NEXT:    lxv vs1, 80(r4)
+; CHECK-BE-NEXT:    xscvdpsxws f3, f2
+; CHECK-BE-NEXT:    xxswapd vs2, vs2
+; CHECK-BE-NEXT:    sldi r5, r5, 48
+; CHECK-BE-NEXT:    vmrghh v4, v4, v1
+; CHECK-BE-NEXT:    mtvsrd v1, r5
+; CHECK-BE-NEXT:    xscvdpsxws f2, f2
+; CHECK-BE-NEXT:    vmrghh v5, v5, v1
+; CHECK-BE-NEXT:    mfvsrwz r5, f0
+; CHECK-BE-NEXT:    lxv vs0, 64(r4)
+; CHECK-BE-NEXT:    mfvsrwz r4, f3
+; CHECK-BE-NEXT:    sldi r4, r4, 48
 ; CHECK-BE-NEXT:    vmrghw v3, v5, v4
-; CHECK-BE-NEXT:    vmrghw v4, v1, v0
-; CHECK-BE-NEXT:    vmrghw v5, v7, v6
+; CHECK-BE-NEXT:    xxmrghd vs3, v3, v2
+; CHECK-BE-NEXT:    mtvsrd v2, r4
+; CHECK-BE-NEXT:    mfvsrwz r4, f2
+; CHECK-BE-NEXT:    xscvdpsxws f2, f1
+; CHECK-BE-NEXT:    xxswapd vs1, vs1
+; CHECK-BE-NEXT:    sldi r4, r4, 48
+; CHECK-BE-NEXT:    xscvdpsxws f1, f1
+; CHECK-BE-NEXT:    mtvsrd v3, r4
+; CHECK-BE-NEXT:    vmrghh v2, v2, v3
+; CHECK-BE-NEXT:    mfvsrwz r4, f2
+; CHECK-BE-NEXT:    sldi r4, r4, 48
+; CHECK-BE-NEXT:    mtvsrd v3, r4
+; CHECK-BE-NEXT:    mfvsrwz r4, f1
+; CHECK-BE-NEXT:    xscvdpsxws f1, f0
+; CHECK-BE-NEXT:    xxswapd vs0, vs0
+; CHECK-BE-NEXT:    sldi r4, r4, 48
+; CHECK-BE-NEXT:    xscvdpsxws f0, f0
+; CHECK-BE-NEXT:    mtvsrd v4, r4
+; CHECK-BE-NEXT:    vmrghh v3, v3, v4
+; CHECK-BE-NEXT:    mfvsrwz r4, f1
+; CHECK-BE-NEXT:    sldi r4, r4, 48
+; CHECK-BE-NEXT:    mtvsrd v4, r4
+; CHECK-BE-NEXT:    mfvsrwz r4, f0
+; CHECK-BE-NEXT:    sldi r5, r5, 48
+; CHECK-BE-NEXT:    mtvsrd v1, r5
+; CHECK-BE-NEXT:    vmrghh v0, v0, v1
+; CHECK-BE-NEXT:    vmrghw v2, v2, v0
+; CHECK-BE-NEXT:    stxv vs3, 0(r3)
+; CHECK-BE-NEXT:    sldi r4, r4, 48
+; CHECK-BE-NEXT:    mtvsrd v5, r4
+; CHECK-BE-NEXT:    vmrghh v4, v4, v5
+; CHECK-BE-NEXT:    vmrghw v3, v4, v3
 ; CHECK-BE-NEXT:    xxmrghd vs0, v3, v2
-; CHECK-BE-NEXT:    xxmrghd vs1, v5, v4
-; CHECK-BE-NEXT:    stxv vs0, 0(r3)
-; CHECK-BE-NEXT:    stxv vs1, 16(r3)
+; CHECK-BE-NEXT:    stxv vs0, 16(r3)
 ; CHECK-BE-NEXT:    blr
 entry:
   %a = load <16 x double>, <16 x double>* %0, align 128
@@ -675,33 +651,33 @@ define i32 @test2elt_signed(<2 x double> %a) local_unnamed_addr #0 {
 ;
 ; CHECK-P9-LABEL: test2elt_signed:
 ; CHECK-P9:       # %bb.0: # %entry
+; CHECK-P9-NEXT:    xscvdpsxws f0, v2
+; CHECK-P9-NEXT:    mfvsrwz r3, f0
+; CHECK-P9-NEXT:    mtvsrd f0, r3
+; CHECK-P9-NEXT:    xxswapd v3, vs0
 ; CHECK-P9-NEXT:    xxswapd vs0, v2
-; CHECK-P9-NEXT:    xscvdpsxws f1, v2
 ; CHECK-P9-NEXT:    xscvdpsxws f0, f0
-; CHECK-P9-NEXT:    mfvsrwz r3, f1
-; CHECK-P9-NEXT:    mfvsrwz r4, f0
+; CHECK-P9-NEXT:    mfvsrwz r3, f0
 ; CHECK-P9-NEXT:    mtvsrd f0, r3
 ; CHECK-P9-NEXT:    li r3, 0
-; CHECK-P9-NEXT:    mtvsrd f1, r4
 ; CHECK-P9-NEXT:    xxswapd v2, vs0
-; CHECK-P9-NEXT:    xxswapd v3, vs1
-; CHECK-P9-NEXT:    vmrglh v2, v2, v3
+; CHECK-P9-NEXT:    vmrglh v2, v3, v2
 ; CHECK-P9-NEXT:    vextuwrx r3, r3, v2
 ; CHECK-P9-NEXT:    blr
 ;
 ; CHECK-BE-LABEL: test2elt_signed:
 ; CHECK-BE:       # %bb.0: # %entry
+; CHECK-BE-NEXT:    xscvdpsxws f0, v2
+; CHECK-BE-NEXT:    mfvsrwz r3, f0
 ; CHECK-BE-NEXT:    xxswapd vs0, v2
-; CHECK-BE-NEXT:    xscvdpsxws f1, v2
+; CHECK-BE-NEXT:    sldi r3, r3, 48
 ; CHECK-BE-NEXT:    xscvdpsxws f0, f0
-; CHECK-BE-NEXT:    mfvsrwz r3, f1
+; CHECK-BE-NEXT:    mtvsrd v3, r3
+; CHECK-BE-NEXT:    mfvsrwz r3, f0
 ; CHECK-BE-NEXT:    sldi r3, r3, 48
-; CHECK-BE-NEXT:    mfvsrwz r4, f0
 ; CHECK-BE-NEXT:    mtvsrd v2, r3
 ; CHECK-BE-NEXT:    li r3, 0
-; CHECK-BE-NEXT:    sldi r4, r4, 48
-; CHECK-BE-NEXT:    mtvsrd v3, r4
-; CHECK-BE-NEXT:    vmrghh v2, v2, v3
+; CHECK-BE-NEXT:    vmrghh v2, v3, v2
 ; CHECK-BE-NEXT:    vextuwlx r3, r3, v2
 ; CHECK-BE-NEXT:    blr
 entry:
@@ -743,56 +719,56 @@ define i64 @test4elt_signed(<4 x double>* nocapture readonly) local_unnamed_addr
 ;
 ; CHECK-P9-LABEL: test4elt_signed:
 ; CHECK-P9:       # %bb.0: # %entry
-; CHECK-P9-NEXT:    lxv vs0, 16(r3)
 ; CHECK-P9-NEXT:    lxv vs1, 0(r3)
-; CHECK-P9-NEXT:    xxswapd vs2, vs1
-; CHECK-P9-NEXT:    xxswapd vs3, vs0
+; CHECK-P9-NEXT:    xscvdpsxws f2, f1
+; CHECK-P9-NEXT:    xxswapd vs1, vs1
 ; CHECK-P9-NEXT:    xscvdpsxws f1, f1
+; CHECK-P9-NEXT:    lxv vs0, 16(r3)
+; CHECK-P9-NEXT:    mfvsrwz r3, f2
+; CHECK-P9-NEXT:    mtvsrd f2, r3
+; CHECK-P9-NEXT:    mfvsrwz r3, f1
+; CHECK-P9-NEXT:    xxswapd v2, vs2
+; CHECK-P9-NEXT:    mtvsrd f1, r3
+; CHECK-P9-NEXT:    xxswapd v3, vs1
+; CHECK-P9-NEXT:    xscvdpsxws f1, f0
+; CHECK-P9-NEXT:    xxswapd vs0, vs0
 ; CHECK-P9-NEXT:    xscvdpsxws f0, f0
-; CHECK-P9-NEXT:    xscvdpsxws f2, f2
-; CHECK-P9-NEXT:    xscvdpsxws f3, f3
 ; CHECK-P9-NEXT:    mfvsrwz r3, f1
-; CHECK-P9-NEXT:    mfvsrwz r5, f0
+; CHECK-P9-NEXT:    mtvsrd f1, r3
+; CHECK-P9-NEXT:    mfvsrwz r3, f0
 ; CHECK-P9-NEXT:    mtvsrd f0, r3
-; CHECK-P9-NEXT:    mfvsrwz r4, f2
-; CHECK-P9-NEXT:    mfvsrwz r6, f3
-; CHECK-P9-NEXT:    mtvsrd f2, r5
-; CHECK-P9-NEXT:    xxswapd v2, vs0
-; CHECK-P9-NEXT:    mtvsrd f1, r4
-; CHECK-P9-NEXT:    mtvsrd f3, r6
-; CHECK-P9-NEXT:    xxswapd v4, vs2
-; CHECK-P9-NEXT:    xxswapd v3, vs1
-; CHECK-P9-NEXT:    xxswapd v5, vs3
 ; CHECK-P9-NEXT:    vmrglh v2, v2, v3
-; CHECK-P9-NEXT:    vmrglh v3, v4, v5
+; CHECK-P9-NEXT:    xxswapd v3, vs1
+; CHECK-P9-NEXT:    xxswapd v4, vs0
+; CHECK-P9-NEXT:    vmrglh v3, v3, v4
 ; CHECK-P9-NEXT:    vmrglw v2, v3, v2
 ; CHECK-P9-NEXT:    mfvsrld r3, v2
 ; CHECK-P9-NEXT:    blr
 ;
 ; CHECK-BE-LABEL: test4elt_signed:
 ; CHECK-BE:       # %bb.0: # %entry
-; CHECK-BE-NEXT:    lxv vs0, 0(r3)
 ; CHECK-BE-NEXT:    lxv vs1, 16(r3)
-; CHECK-BE-NEXT:    xxswapd vs2, vs1
-; CHECK-BE-NEXT:    xxswapd vs3, vs0
+; CHECK-BE-NEXT:    xscvdpsxws f2, f1
+; CHECK-BE-NEXT:    xxswapd vs1, vs1
 ; CHECK-BE-NEXT:    xscvdpsxws f1, f1
-; CHECK-BE-NEXT:    xscvdpsxws f0, f0
-; CHECK-BE-NEXT:    xscvdpsxws f2, f2
-; CHECK-BE-NEXT:    xscvdpsxws f3, f3
-; CHECK-BE-NEXT:    mfvsrwz r3, f1
-; CHECK-BE-NEXT:    mfvsrwz r5, f0
+; CHECK-BE-NEXT:    lxv vs0, 0(r3)
+; CHECK-BE-NEXT:    mfvsrwz r3, f2
 ; CHECK-BE-NEXT:    sldi r3, r3, 48
-; CHECK-BE-NEXT:    sldi r5, r5, 48
-; CHECK-BE-NEXT:    mfvsrwz r4, f2
-; CHECK-BE-NEXT:    mfvsrwz r6, f3
 ; CHECK-BE-NEXT:    mtvsrd v2, r3
-; CHECK-BE-NEXT:    mtvsrd v4, r5
-; CHECK-BE-NEXT:    sldi r4, r4, 48
-; CHECK-BE-NEXT:    sldi r6, r6, 48
-; CHECK-BE-NEXT:    mtvsrd v3, r4
-; CHECK-BE-NEXT:    mtvsrd v5, r6
+; CHECK-BE-NEXT:    mfvsrwz r3, f1
+; CHECK-BE-NEXT:    xscvdpsxws f1, f0
+; CHECK-BE-NEXT:    xxswapd vs0, vs0
+; CHECK-BE-NEXT:    sldi r3, r3, 48
+; CHECK-BE-NEXT:    xscvdpsxws f0, f0
+; CHECK-BE-NEXT:    mtvsrd v3, r3
 ; CHECK-BE-NEXT:    vmrghh v2, v2, v3
-; CHECK-BE-NEXT:    vmrghh v3, v4, v5
+; CHECK-BE-NEXT:    mfvsrwz r3, f1
+; CHECK-BE-NEXT:    sldi r3, r3, 48
+; CHECK-BE-NEXT:    mtvsrd v3, r3
+; CHECK-BE-NEXT:    mfvsrwz r3, f0
+; CHECK-BE-NEXT:    sldi r3, r3, 48
+; CHECK-BE-NEXT:    mtvsrd v4, r3
+; CHECK-BE-NEXT:    vmrghh v3, v3, v4
 ; CHECK-BE-NEXT:    vmrghw v2, v3, v2
 ; CHECK-BE-NEXT:    mfvsrd r3, v2
 ; CHECK-BE-NEXT:    blr
@@ -860,103 +836,103 @@ define <8 x i16> @test8elt_signed(<8 x double>* nocapture readonly) local_unname
 ;
 ; CHECK-P9-LABEL: test8elt_signed:
 ; CHECK-P9:       # %bb.0: # %entry
+; CHECK-P9-NEXT:    lxv vs3, 0(r3)
+; CHECK-P9-NEXT:    xscvdpsxws f4, f3
+; CHECK-P9-NEXT:    xxswapd vs3, vs3
+; CHECK-P9-NEXT:    xscvdpsxws f3, f3
 ; CHECK-P9-NEXT:    lxv vs0, 48(r3)
 ; CHECK-P9-NEXT:    lxv vs1, 32(r3)
 ; CHECK-P9-NEXT:    lxv vs2, 16(r3)
-; CHECK-P9-NEXT:    lxv vs3, 0(r3)
-; CHECK-P9-NEXT:    xxswapd vs4, vs3
-; CHECK-P9-NEXT:    xxswapd vs5, vs2
-; CHECK-P9-NEXT:    xxswapd vs6, vs1
-; CHECK-P9-NEXT:    xxswapd vs7, vs0
-; CHECK-P9-NEXT:    xscvdpsxws f3, f3
+; CHECK-P9-NEXT:    mfvsrwz r3, f4
+; CHECK-P9-NEXT:    mtvsrd f4, r3
+; CHECK-P9-NEXT:    mfvsrwz r3, f3
+; CHECK-P9-NEXT:    xxswapd v2, vs4
+; CHECK-P9-NEXT:    mtvsrd f3, r3
+; CHECK-P9-NEXT:    xxswapd v3, vs3
+; CHECK-P9-NEXT:    xscvdpsxws f3, f2
+; CHECK-P9-NEXT:    xxswapd vs2, vs2
 ; CHECK-P9-NEXT:    xscvdpsxws f2, f2
-; CHECK-P9-NEXT:    xscvdpsxws f1, f1
-; CHECK-P9-NEXT:    xscvdpsxws f0, f0
-; CHECK-P9-NEXT:    xscvdpsxws f4, f4
-; CHECK-P9-NEXT:    xscvdpsxws f5, f5
-; CHECK-P9-NEXT:    xscvdpsxws f6, f6
-; CHECK-P9-NEXT:    xscvdpsxws f7, f7
 ; CHECK-P9-NEXT:    mfvsrwz r3, f3
-; CHECK-P9-NEXT:    mfvsrwz r5, f2
-; CHECK-P9-NEXT:    mfvsrwz r7, f1
-; CHECK-P9-NEXT:    mfvsrwz r9, f0
-; CHECK-P9-NEXT:    mtvsrd f0, r3
-; CHECK-P9-NEXT:    mtvsrd f2, r5
-; CHECK-P9-NEXT:    mfvsrwz r4, f4
-; CHECK-P9-NEXT:    mfvsrwz r6, f5
-; CHECK-P9-NEXT:    mfvsrwz r8, f6
-; CHECK-P9-NEXT:    mfvsrwz r10, f7
-; CHECK-P9-NEXT:    mtvsrd f4, r7
-; CHECK-P9-NEXT:    mtvsrd f6, r9
-; CHECK-P9-NEXT:    xxswapd v2, vs0
+; CHECK-P9-NEXT:    mtvsrd f3, r3
+; CHECK-P9-NEXT:    mfvsrwz r3, f2
+; CHECK-P9-NEXT:    mtvsrd f2, r3
 ; CHECK-P9-NEXT:    xxswapd v4, vs2
-; CHECK-P9-NEXT:    mtvsrd f1, r4
-; CHECK-P9-NEXT:    mtvsrd f3, r6
-; CHECK-P9-NEXT:    mtvsrd f5, r8
-; CHECK-P9-NEXT:    mtvsrd f7, r10
-; CHECK-P9-NEXT:    xxswapd v0, vs4
-; CHECK-P9-NEXT:    xxswapd v6, vs6
-; CHECK-P9-NEXT:    xxswapd v3, vs1
-; CHECK-P9-NEXT:    xxswapd v5, vs3
-; CHECK-P9-NEXT:    xxswapd v1, vs5
-; CHECK-P9-NEXT:    xxswapd v7, vs7
+; CHECK-P9-NEXT:    xscvdpsxws f2, f1
+; CHECK-P9-NEXT:    xxswapd vs1, vs1
+; CHECK-P9-NEXT:    xscvdpsxws f1, f1
+; CHECK-P9-NEXT:    mfvsrwz r3, f2
+; CHECK-P9-NEXT:    mtvsrd f2, r3
+; CHECK-P9-NEXT:    mfvsrwz r3, f1
 ; CHECK-P9-NEXT:    vmrglh v2, v2, v3
-; CHECK-P9-NEXT:    vmrglh v3, v4, v5
-; CHECK-P9-NEXT:    vmrglh v4, v0, v1
-; CHECK-P9-NEXT:    vmrglh v5, v6, v7
+; CHECK-P9-NEXT:    xxswapd v3, vs3
+; CHECK-P9-NEXT:    vmrglh v3, v3, v4
 ; CHECK-P9-NEXT:    vmrglw v2, v3, v2
-; CHECK-P9-NEXT:    vmrglw v3, v5, v4
+; CHECK-P9-NEXT:    xxswapd v3, vs2
+; CHECK-P9-NEXT:    mtvsrd f1, r3
+; CHECK-P9-NEXT:    xxswapd v4, vs1
+; CHECK-P9-NEXT:    xscvdpsxws f1, f0
+; CHECK-P9-NEXT:    xxswapd vs0, vs0
+; CHECK-P9-NEXT:    xscvdpsxws f0, f0
+; CHECK-P9-NEXT:    mfvsrwz r3, f1
+; CHECK-P9-NEXT:    mtvsrd f1, r3
+; CHECK-P9-NEXT:    mfvsrwz r3, f0
+; CHECK-P9-NEXT:    mtvsrd f0, r3
+; CHECK-P9-NEXT:    vmrglh v3, v3, v4
+; CHECK-P9-NEXT:    xxswapd v4, vs1
+; CHECK-P9-NEXT:    xxswapd v5, vs0
+; CHECK-P9-NEXT:    vmrglh v4, v4, v5
+; CHECK-P9-NEXT:    vmrglw v3, v4, v3
 ; CHECK-P9-NEXT:    xxmrgld v2, v3, v2
 ; CHECK-P9-NEXT:    blr
 ;
 ; CHECK-BE-LABEL: test8elt_signed:
 ; CHECK-BE:       # %bb.0: # %entry
-; CHECK-BE-NEXT:    lxv vs0, 0(r3)
-; CHECK-BE-NEXT:    lxv vs1, 16(r3)
-; CHECK-BE-NEXT:    lxv vs2, 32(r3)
 ; CHECK-BE-NEXT:    lxv vs3, 48(r3)
-; CHECK-BE-NEXT:    xxswapd vs4, vs3
-; CHECK-BE-NEXT:    xxswapd vs5, vs2
-; CHECK-BE-NEXT:    xxswapd vs6, vs1
-; CHECK-BE-NEXT:    xxswapd vs7, vs0
+; CHECK-BE-NEXT:    xscvdpsxws f4, f3
+; CHECK-BE-NEXT:    xxswapd vs3, vs3
 ; CHECK-BE-NEXT:    xscvdpsxws f3, f3
-; CHECK-BE-NEXT:    xscvdpsxws f2, f2
-; CHECK-BE-NEXT:    xscvdpsxws f1, f1
-; CHECK-BE-NEXT:    xscvdpsxws f0, f0
-; CHECK-BE-NEXT:    xscvdpsxws f4, f4
-; CHECK-BE-NEXT:    xscvdpsxws f5, f5
-; CHECK-BE-NEXT:    xscvdpsxws f6, f6
-; CHECK-BE-NEXT:    xscvdpsxws f7, f7
-; CHECK-BE-NEXT:    mfvsrwz r3, f3
-; CHECK-BE-NEXT:    mfvsrwz r5, f2
-; CHECK-BE-NEXT:    mfvsrwz r7, f1
-; CHECK-BE-NEXT:    mfvsrwz r9, f0
+; CHECK-BE-NEXT:    lxv vs2, 32(r3)
+; CHECK-BE-NEXT:    lxv vs0, 0(r3)
+; CHECK-BE-NEXT:    lxv vs1, 16(r3)
+; CHECK-BE-NEXT:    mfvsrwz r3, f4
 ; CHECK-BE-NEXT:    sldi r3, r3, 48
-; CHECK-BE-NEXT:    sldi r5, r5, 48
-; CHECK-BE-NEXT:    sldi r7, r7, 48
-; CHECK-BE-NEXT:    sldi r9, r9, 48
-; CHECK-BE-NEXT:    mfvsrwz r4, f4
-; CHECK-BE-NEXT:    mfvsrwz r6, f5
-; CHECK-BE-NEXT:    mfvsrwz r8, f6
-; CHECK-BE-NEXT:    mfvsrwz r10, f7
 ; CHECK-BE-NEXT:    mtvsrd v2, r3
-; CHECK-BE-NEXT:    mtvsrd v4, r5
-; CHECK-BE-NEXT:    mtvsrd v0, r7
-; CHECK-BE-NEXT:    mtvsrd v6, r9
-; CHECK-BE-NEXT:    sldi r4, r4, 48
-; CHECK-BE-NEXT:    sldi r6, r6, 48
-; CHECK-BE-NEXT:    sldi r8, r8, 48
-; CHECK-BE-NEXT:    sldi r10, r10, 48
-; CHECK-BE-NEXT:    mtvsrd v3, r4
-; CHECK-BE-NEXT:    mtvsrd v5, r6
-; CHECK-BE-NEXT:    mtvsrd v1, r8
-; CHECK-BE-NEXT:    mtvsrd v7, r10
+; CHECK-BE-NEXT:    mfvsrwz r3, f3
+; CHECK-BE-NEXT:    xscvdpsxws f3, f2
+; CHECK-BE-NEXT:    xxswapd vs2, vs2
+; CHECK-BE-NEXT:    sldi r3, r3, 48
+; CHECK-BE-NEXT:    xscvdpsxws f2, f2
+; CHECK-BE-NEXT:    mtvsrd v3, r3
 ; CHECK-BE-NEXT:    vmrghh v2, v2, v3
-; CHECK-BE-NEXT:    vmrghh v3, v4, v5
-; CHECK-BE-NEXT:    vmrghh v4, v0, v1
-; CHECK-BE-NEXT:    vmrghh v5, v6, v7
+; CHECK-BE-NEXT:    mfvsrwz r3, f3
+; CHECK-BE-NEXT:    sldi r3, r3, 48
+; CHECK-BE-NEXT:    mtvsrd v3, r3
+; CHECK-BE-NEXT:    mfvsrwz r3, f2
+; CHECK-BE-NEXT:    xscvdpsxws f2, f1
+; CHECK-BE-NEXT:    xxswapd vs1, vs1
+; CHECK-BE-NEXT:    sldi r3, r3, 48
+; CHECK-BE-NEXT:    xscvdpsxws f1, f1
+; CHECK-BE-NEXT:    mtvsrd v4, r3
+; CHECK-BE-NEXT:    vmrghh v3, v3, v4
+; CHECK-BE-NEXT:    mfvsrwz r3, f2
+; CHECK-BE-NEXT:    sldi r3, r3, 48
 ; CHECK-BE-NEXT:    vmrghw v2, v3, v2
-; CHECK-BE-NEXT:    vmrghw v3, v5, v4
+; CHECK-BE-NEXT:    mtvsrd v3, r3
+; CHECK-BE-NEXT:    mfvsrwz r3, f1
+; CHECK-BE-NEXT:    xscvdpsxws f1, f0
+; CHECK-BE-NEXT:    xxswapd vs0, vs0
+; CHECK-BE-NEXT:    sldi r3, r3, 48
+; CHECK-BE-NEXT:    xscvdpsxws f0, f0
+; CHECK-BE-NEXT:    mtvsrd v4, r3
+; CHECK-BE-NEXT:    vmrghh v3, v3, v4
+; CHECK-BE-NEXT:    mfvsrwz r3, f1
+; CHECK-BE-NEXT:    sldi r3, r3, 48
+; CHECK-BE-NEXT:    mtvsrd v4, r3
+; CHECK-BE-NEXT:    mfvsrwz r3, f0
+; CHECK-BE-NEXT:    sldi r3, r3, 48
+; CHECK-BE-NEXT:    mtvsrd v5, r3
+; CHECK-BE-NEXT:    vmrghh v4, v4, v5
+; CHECK-BE-NEXT:    vmrghw v3, v4, v3
 ; CHECK-BE-NEXT:    xxmrghd v2, v3, v2
 ; CHECK-BE-NEXT:    blr
 entry:
@@ -1075,226 +1051,202 @@ define void @test16elt_signed(<16 x i16>* noalias nocapture sret %agg.result, <1
 ;
 ; CHECK-P9-LABEL: test16elt_signed:
 ; CHECK-P9:       # %bb.0: # %entry
-; CHECK-P9-NEXT:    lxv vs2, 48(r4)
-; CHECK-P9-NEXT:    lxv vs4, 32(r4)
-; CHECK-P9-NEXT:    lxv vs5, 16(r4)
-; CHECK-P9-NEXT:    lxv vs6, 0(r4)
-; CHECK-P9-NEXT:    lxv vs0, 112(r4)
-; CHECK-P9-NEXT:    lxv vs1, 96(r4)
-; CHECK-P9-NEXT:    lxv vs3, 80(r4)
-; CHECK-P9-NEXT:    lxv vs7, 64(r4)
-; CHECK-P9-NEXT:    std r25, -56(r1) # 8-byte Folded Spill
-; CHECK-P9-NEXT:    std r26, -48(r1) # 8-byte Folded Spill
-; CHECK-P9-NEXT:    std r27, -40(r1) # 8-byte Folded Spill
-; CHECK-P9-NEXT:    std r28, -32(r1) # 8-byte Folded Spill
-; CHECK-P9-NEXT:    std r29, -24(r1) # 8-byte Folded Spill
-; CHECK-P9-NEXT:    std r30, -16(r1) # 8-byte Folded Spill
-; CHECK-P9-NEXT:    xxswapd vs8, vs6
-; CHECK-P9-NEXT:    xxswapd vs9, vs5
-; CHECK-P9-NEXT:    xxswapd vs10, vs4
-; CHECK-P9-NEXT:    xxswapd vs11, vs2
-; CHECK-P9-NEXT:    xxswapd vs12, vs7
-; CHECK-P9-NEXT:    xxswapd vs13, vs3
-; CHECK-P9-NEXT:    xxswapd v2, vs1
-; CHECK-P9-NEXT:    xxswapd v3, vs0
-; CHECK-P9-NEXT:    xscvdpsxws f6, f6
-; CHECK-P9-NEXT:    xscvdpsxws f5, f5
+; CHECK-P9-NEXT:    lxv vs4, 0(r4)
+; CHECK-P9-NEXT:    lxv vs3, 16(r4)
+; CHECK-P9-NEXT:    lxv vs2, 32(r4)
+; CHECK-P9-NEXT:    xscvdpsxws f5, f4
+; CHECK-P9-NEXT:    lxv vs1, 48(r4)
+; CHECK-P9-NEXT:    xscvdpsxws f6, f3
+; CHECK-P9-NEXT:    lxv vs0, 64(r4)
+; CHECK-P9-NEXT:    xscvdpsxws f7, f2
+; CHECK-P9-NEXT:    xscvdpsxws f8, f1
+; CHECK-P9-NEXT:    xxswapd vs4, vs4
 ; CHECK-P9-NEXT:    xscvdpsxws f4, f4
-; CHECK-P9-NEXT:    xscvdpsxws f2, f2
-; CHECK-P9-NEXT:    xscvdpsxws f7, f7
+; CHECK-P9-NEXT:    mfvsrwz r5, f5
+; CHECK-P9-NEXT:    xscvdpsxws f9, f0
+; CHECK-P9-NEXT:    xxswapd vs3, vs3
 ; CHECK-P9-NEXT:    xscvdpsxws f3, f3
+; CHECK-P9-NEXT:    mtvsrd f5, r5
+; CHECK-P9-NEXT:    mfvsrwz r5, f6
+; CHECK-P9-NEXT:    xxswapd vs2, vs2
+; CHECK-P9-NEXT:    xscvdpsxws f2, f2
+; CHECK-P9-NEXT:    mtvsrd f6, r5
+; CHECK-P9-NEXT:    mfvsrwz r5, f7
+; CHECK-P9-NEXT:    mtvsrd f7, r5
+; CHECK-P9-NEXT:    mfvsrwz r5, f8
+; CHECK-P9-NEXT:    mtvsrd f8, r5
+; CHECK-P9-NEXT:    mfvsrwz r5, f9
+; CHECK-P9-NEXT:    mtvsrd f9, r5
+; CHECK-P9-NEXT:    mfvsrwz r5, f4
+; CHECK-P9-NEXT:    mtvsrd f4, r5
+; CHECK-P9-NEXT:    mfvsrwz r5, f3
+; CHECK-P9-NEXT:    xxswapd vs1, vs1
 ; CHECK-P9-NEXT:    xscvdpsxws f1, f1
+; CHECK-P9-NEXT:    xxswapd v2, vs5
+; CHECK-P9-NEXT:    xxswapd v5, vs8
+; CHECK-P9-NEXT:    xxswapd v0, vs9
+; CHECK-P9-NEXT:    mtvsrd f3, r5
+; CHECK-P9-NEXT:    mfvsrwz r5, f2
+; CHECK-P9-NEXT:    mtvsrd f2, r5
+; CHECK-P9-NEXT:    xxswapd vs0, vs0
 ; CHECK-P9-NEXT:    xscvdpsxws f0, f0
-; CHECK-P9-NEXT:    xscvdpsxws f8, f8
-; CHECK-P9-NEXT:    xscvdpsxws f9, f9
-; CHECK-P9-NEXT:    xscvdpsxws f10, f10
-; CHECK-P9-NEXT:    xscvdpsxws f11, f11
-; CHECK-P9-NEXT:    xscvdpsxws f12, f12
-; CHECK-P9-NEXT:    xscvdpsxws f13, f13
-; CHECK-P9-NEXT:    xscvdpsxws v2, v2
-; CHECK-P9-NEXT:    xscvdpsxws v3, v3
-; CHECK-P9-NEXT:    mfvsrwz r4, f6
-; CHECK-P9-NEXT:    mfvsrwz r5, f5
-; CHECK-P9-NEXT:    mfvsrwz r6, f4
-; CHECK-P9-NEXT:    mfvsrwz r7, f2
-; CHECK-P9-NEXT:    mfvsrwz r12, f7
-; CHECK-P9-NEXT:    mfvsrwz r0, f3
-; CHECK-P9-NEXT:    mfvsrwz r30, f1
-; CHECK-P9-NEXT:    mfvsrwz r29, f0
-; CHECK-P9-NEXT:    mfvsrwz r8, f8
-; CHECK-P9-NEXT:    mfvsrwz r9, f9
-; CHECK-P9-NEXT:    mfvsrwz r10, f10
-; CHECK-P9-NEXT:    mfvsrwz r11, f11
-; CHECK-P9-NEXT:    mfvsrwz r28, f12
-; CHECK-P9-NEXT:    mfvsrwz r27, f13
-; CHECK-P9-NEXT:    mfvsrwz r26, v2
-; CHECK-P9-NEXT:    mfvsrwz r25, v3
-; CHECK-P9-NEXT:    mtvsrd f0, r4
+; CHECK-P9-NEXT:    xxswapd v1, vs2
+; CHECK-P9-NEXT:    lxv vs2, 80(r4)
+; CHECK-P9-NEXT:    xxswapd v3, vs4
+; CHECK-P9-NEXT:    vmrglh v2, v2, v3
+; CHECK-P9-NEXT:    xxswapd v3, vs6
+; CHECK-P9-NEXT:    xxswapd v4, vs3
+; CHECK-P9-NEXT:    xscvdpsxws f3, f2
+; CHECK-P9-NEXT:    xxswapd vs2, vs2
+; CHECK-P9-NEXT:    mfvsrwz r5, f1
+; CHECK-P9-NEXT:    vmrglh v3, v3, v4
+; CHECK-P9-NEXT:    xxswapd v4, vs7
 ; CHECK-P9-NEXT:    mtvsrd f1, r5
-; CHECK-P9-NEXT:    mtvsrd f2, r6
-; CHECK-P9-NEXT:    mtvsrd f3, r7
-; CHECK-P9-NEXT:    mtvsrd f8, r12
-; CHECK-P9-NEXT:    mtvsrd f9, r0
-; CHECK-P9-NEXT:    mtvsrd f10, r30
-; CHECK-P9-NEXT:    mtvsrd f11, r29
-; CHECK-P9-NEXT:    mtvsrd f4, r8
-; CHECK-P9-NEXT:    mtvsrd f5, r9
-; CHECK-P9-NEXT:    mtvsrd f6, r10
-; CHECK-P9-NEXT:    mtvsrd f7, r11
-; CHECK-P9-NEXT:    mtvsrd f12, r28
-; CHECK-P9-NEXT:    mtvsrd f13, r27
-; CHECK-P9-NEXT:    mtvsrd v2, r26
-; CHECK-P9-NEXT:    mtvsrd v3, r25
-; CHECK-P9-NEXT:    xxswapd v4, vs0
-; CHECK-P9-NEXT:    xxswapd v5, vs1
-; CHECK-P9-NEXT:    xxswapd v0, vs2
-; CHECK-P9-NEXT:    xxswapd v1, vs3
-; CHECK-P9-NEXT:    ld r30, -16(r1) # 8-byte Folded Reload
-; CHECK-P9-NEXT:    ld r29, -24(r1) # 8-byte Folded Reload
-; CHECK-P9-NEXT:    ld r28, -32(r1) # 8-byte Folded Reload
-; CHECK-P9-NEXT:    ld r27, -40(r1) # 8-byte Folded Reload
-; CHECK-P9-NEXT:    xxswapd v6, vs4
-; CHECK-P9-NEXT:    xxswapd v7, vs5
-; CHECK-P9-NEXT:    ld r26, -48(r1) # 8-byte Folded Reload
-; CHECK-P9-NEXT:    ld r25, -56(r1) # 8-byte Folded Reload
-; CHECK-P9-NEXT:    xxswapd v8, vs6
-; CHECK-P9-NEXT:    xxswapd v9, vs7
-; CHECK-P9-NEXT:    xxswapd v10, vs8
-; CHECK-P9-NEXT:    xxswapd v11, vs12
-; CHECK-P9-NEXT:    xxswapd v12, vs9
-; CHECK-P9-NEXT:    xxswapd v13, vs13
-; CHECK-P9-NEXT:    xxswapd v14, vs10
-; CHECK-P9-NEXT:    xxswapd v2, v2
-; CHECK-P9-NEXT:    xxswapd v15, vs11
-; CHECK-P9-NEXT:    xxswapd v3, v3
-; CHECK-P9-NEXT:    vmrglh v4, v4, v6
-; CHECK-P9-NEXT:    vmrglh v5, v5, v7
-; CHECK-P9-NEXT:    vmrglh v0, v0, v8
-; CHECK-P9-NEXT:    vmrglh v1, v1, v9
-; CHECK-P9-NEXT:    vmrglh v6, v10, v11
-; CHECK-P9-NEXT:    vmrglh v7, v12, v13
-; CHECK-P9-NEXT:    vmrglh v2, v14, v2
-; CHECK-P9-NEXT:    vmrglh v3, v15, v3
-; CHECK-P9-NEXT:    vmrglw v4, v5, v4
-; CHECK-P9-NEXT:    vmrglw v5, v1, v0
-; CHECK-P9-NEXT:    vmrglw v0, v7, v6
+; CHECK-P9-NEXT:    mfvsrwz r5, f0
+; CHECK-P9-NEXT:    vmrglh v4, v4, v1
+; CHECK-P9-NEXT:    xxswapd v1, vs1
+; CHECK-P9-NEXT:    mtvsrd f0, r5
+; CHECK-P9-NEXT:    vmrglh v5, v5, v1
+; CHECK-P9-NEXT:    xscvdpsxws f2, f2
+; CHECK-P9-NEXT:    xxswapd v1, vs0
+; CHECK-P9-NEXT:    lxv vs0, 112(r4)
+; CHECK-P9-NEXT:    lxv vs1, 96(r4)
+; CHECK-P9-NEXT:    mfvsrwz r4, f3
+; CHECK-P9-NEXT:    mtvsrd f3, r4
+; CHECK-P9-NEXT:    mfvsrwz r4, f2
 ; CHECK-P9-NEXT:    vmrglw v2, v3, v2
-; CHECK-P9-NEXT:    xxmrgld vs0, v5, v4
-; CHECK-P9-NEXT:    xxmrgld vs1, v2, v0
-; CHECK-P9-NEXT:    stxv vs0, 0(r3)
-; CHECK-P9-NEXT:    stxv vs1, 16(r3)
+; CHECK-P9-NEXT:    vmrglw v3, v5, v4
+; CHECK-P9-NEXT:    xxmrgld vs4, v3, v2
+; CHECK-P9-NEXT:    xxswapd v2, vs3
+; CHECK-P9-NEXT:    vmrglh v0, v0, v1
+; CHECK-P9-NEXT:    mtvsrd f2, r4
+; CHECK-P9-NEXT:    xxswapd v3, vs2
+; CHECK-P9-NEXT:    xscvdpsxws f2, f1
+; CHECK-P9-NEXT:    xxswapd vs1, vs1
+; CHECK-P9-NEXT:    xscvdpsxws f1, f1
+; CHECK-P9-NEXT:    mfvsrwz r4, f2
+; CHECK-P9-NEXT:    mtvsrd f2, r4
+; CHECK-P9-NEXT:    mfvsrwz r4, f1
+; CHECK-P9-NEXT:    mtvsrd f1, r4
+; CHECK-P9-NEXT:    xxswapd v4, vs1
+; CHECK-P9-NEXT:    xscvdpsxws f1, f0
+; CHECK-P9-NEXT:    xxswapd vs0, vs0
+; CHECK-P9-NEXT:    xscvdpsxws f0, f0
+; CHECK-P9-NEXT:    mfvsrwz r4, f1
+; CHECK-P9-NEXT:    mtvsrd f1, r4
+; CHECK-P9-NEXT:    mfvsrwz r4, f0
+; CHECK-P9-NEXT:    vmrglh v2, v2, v3
+; CHECK-P9-NEXT:    xxswapd v3, vs2
+; CHECK-P9-NEXT:    vmrglh v3, v3, v4
+; CHECK-P9-NEXT:    xxswapd v4, vs1
+; CHECK-P9-NEXT:    vmrglw v2, v2, v0
+; CHECK-P9-NEXT:    mtvsrd f0, r4
+; CHECK-P9-NEXT:    xxswapd v5, vs0
+; CHECK-P9-NEXT:    vmrglh v4, v4, v5
+; CHECK-P9-NEXT:    vmrglw v3, v4, v3
+; CHECK-P9-NEXT:    xxmrgld vs0, v3, v2
+; CHECK-P9-NEXT:    stxv vs0, 16(r3)
+; CHECK-P9-NEXT:    stxv vs4, 0(r3)
 ; CHECK-P9-NEXT:    blr
 ;
 ; CHECK-BE-LABEL: test16elt_signed:
 ; CHECK-BE:       # %bb.0: # %entry
-; CHECK-BE-NEXT:    lxv vs2, 0(r4)
-; CHECK-BE-NEXT:    lxv vs4, 16(r4)
-; CHECK-BE-NEXT:    lxv vs5, 32(r4)
-; CHECK-BE-NEXT:    lxv vs6, 48(r4)
-; CHECK-BE-NEXT:    lxv vs0, 64(r4)
-; CHECK-BE-NEXT:    lxv vs1, 80(r4)
-; CHECK-BE-NEXT:    lxv vs3, 96(r4)
-; CHECK-BE-NEXT:    lxv vs7, 112(r4)
-; CHECK-BE-NEXT:    std r25, -56(r1) # 8-byte Folded Spill
-; CHECK-BE-NEXT:    std r26, -48(r1) # 8-byte Folded Spill
-; CHECK-BE-NEXT:    std r27, -40(r1) # 8-byte Folded Spill
-; CHECK-BE-NEXT:    std r28, -32(r1) # 8-byte Folded Spill
-; CHECK-BE-NEXT:    std r29, -24(r1) # 8-byte Folded Spill
-; CHECK-BE-NEXT:    std r30, -16(r1) # 8-byte Folded Spill
-; CHECK-BE-NEXT:    xxswapd vs8, vs6
-; CHECK-BE-NEXT:    xxswapd vs9, vs5
-; CHECK-BE-NEXT:    xxswapd vs10, vs4
-; CHECK-BE-NEXT:    xxswapd vs11, vs2
-; CHECK-BE-NEXT:    xxswapd vs12, vs7
-; CHECK-BE-NEXT:    xxswapd vs13, vs3
-; CHECK-BE-NEXT:    xxswapd v2, vs1
-; CHECK-BE-NEXT:    xxswapd v3, vs0
-; CHECK-BE-NEXT:    xscvdpsxws f6, f6
-; CHECK-BE-NEXT:    xscvdpsxws f5, f5
+; CHECK-BE-NEXT:    lxv vs4, 48(r4)
+; CHECK-BE-NEXT:    xscvdpsxws f5, f4
+; CHECK-BE-NEXT:    xxswapd vs4, vs4
+; CHECK-BE-NEXT:    lxv vs3, 32(r4)
+; CHECK-BE-NEXT:    xscvdpsxws f6, f3
+; CHECK-BE-NEXT:    xxswapd vs3, vs3
 ; CHECK-BE-NEXT:    xscvdpsxws f4, f4
-; CHECK-BE-NEXT:    xscvdpsxws f2, f2
-; CHECK-BE-NEXT:    xscvdpsxws f7, f7
+; CHECK-BE-NEXT:    mfvsrwz r5, f5
+; CHECK-BE-NEXT:    sldi r5, r5, 48
+; CHECK-BE-NEXT:    lxv vs2, 16(r4)
 ; CHECK-BE-NEXT:    xscvdpsxws f3, f3
+; CHECK-BE-NEXT:    xscvdpsxws f7, f2
+; CHECK-BE-NEXT:    xxswapd vs2, vs2
+; CHECK-BE-NEXT:    xscvdpsxws f2, f2
+; CHECK-BE-NEXT:    mtvsrd v2, r5
+; CHECK-BE-NEXT:    mfvsrwz r5, f4
+; CHECK-BE-NEXT:    sldi r5, r5, 48
+; CHECK-BE-NEXT:    lxv vs1, 0(r4)
+; CHECK-BE-NEXT:    xscvdpsxws f4, f1
+; CHECK-BE-NEXT:    xxswapd vs1, vs1
 ; CHECK-BE-NEXT:    xscvdpsxws f1, f1
-; CHECK-BE-NEXT:    xscvdpsxws f0, f0
-; CHECK-BE-NEXT:    xscvdpsxws f8, f8
-; CHECK-BE-NEXT:    xscvdpsxws f9, f9
-; CHECK-BE-NEXT:    xscvdpsxws f10, f10
-; CHECK-BE-NEXT:    xscvdpsxws f11, f11
-; CHECK-BE-NEXT:    xscvdpsxws f12, f12
-; CHECK-BE-NEXT:    xscvdpsxws f13, f13
-; CHECK-BE-NEXT:    xscvdpsxws v2, v2
-; CHECK-BE-NEXT:    xscvdpsxws v3, v3
-; CHECK-BE-NEXT:    mfvsrwz r4, f6
-; CHECK-BE-NEXT:    mfvsrwz r5, f5
-; CHECK-BE-NEXT:    mfvsrwz r6, f4
-; CHECK-BE-NEXT:    mfvsrwz r7, f2
-; CHECK-BE-NEXT:    mfvsrwz r12, f7
-; CHECK-BE-NEXT:    mfvsrwz r0, f3
-; CHECK-BE-NEXT:    mfvsrwz r30, f1
-; CHECK-BE-NEXT:    mfvsrwz r29, f0
-; CHECK-BE-NEXT:    mfvsrwz r8, f8
-; CHECK-BE-NEXT:    mfvsrwz r9, f9
-; CHECK-BE-NEXT:    mfvsrwz r10, f10
-; CHECK-BE-NEXT:    mfvsrwz r11, f11
-; CHECK-BE-NEXT:    mfvsrwz r28, f12
-; CHECK-BE-NEXT:    mfvsrwz r27, f13
-; CHECK-BE-NEXT:    mfvsrwz r26, v2
-; CHECK-BE-NEXT:    mfvsrwz r25, v3
-; CHECK-BE-NEXT:    sldi r4, r4, 48
+; CHECK-BE-NEXT:    mtvsrd v3, r5
+; CHECK-BE-NEXT:    mfvsrwz r5, f6
 ; CHECK-BE-NEXT:    sldi r5, r5, 48
-; CHECK-BE-NEXT:    sldi r6, r6, 48
-; CHECK-BE-NEXT:    sldi r7, r7, 48
-; CHECK-BE-NEXT:    sldi r12, r12, 48
-; CHECK-BE-NEXT:    sldi r0, r0, 48
-; CHECK-BE-NEXT:    sldi r30, r30, 48
-; CHECK-BE-NEXT:    sldi r29, r29, 48
-; CHECK-BE-NEXT:    sldi r8, r8, 48
-; CHECK-BE-NEXT:    sldi r9, r9, 48
-; CHECK-BE-NEXT:    sldi r10, r10, 48
-; CHECK-BE-NEXT:    sldi r11, r11, 48
-; CHECK-BE-NEXT:    sldi r28, r28, 48
-; CHECK-BE-NEXT:    sldi r27, r27, 48
-; CHECK-BE-NEXT:    sldi r26, r26, 48
-; CHECK-BE-NEXT:    sldi r25, r25, 48
-; CHECK-BE-NEXT:    mtvsrd v2, r4
+; CHECK-BE-NEXT:    lxv vs0, 112(r4)
+; CHECK-BE-NEXT:    vmrghh v2, v2, v3
 ; CHECK-BE-NEXT:    mtvsrd v3, r5
-; CHECK-BE-NEXT:    mtvsrd v4, r6
-; CHECK-BE-NEXT:    mtvsrd v5, r7
-; CHECK-BE-NEXT:    mtvsrd v8, r12
-; CHECK-BE-NEXT:    mtvsrd v10, r0
-; CHECK-BE-NEXT:    mtvsrd v12, r30
-; CHECK-BE-NEXT:    ld r30, -16(r1) # 8-byte Folded Reload
-; CHECK-BE-NEXT:    mtvsrd v0, r8
-; CHECK-BE-NEXT:    mtvsrd v1, r9
-; CHECK-BE-NEXT:    mtvsrd v6, r10
-; CHECK-BE-NEXT:    mtvsrd v7, r11
-; CHECK-BE-NEXT:    mtvsrd v9, r28
-; CHECK-BE-NEXT:    ld r28, -32(r1) # 8-byte Folded Reload
-; CHECK-BE-NEXT:    mtvsrd v11, r27
-; CHECK-BE-NEXT:    ld r27, -40(r1) # 8-byte Folded Reload
-; CHECK-BE-NEXT:    mtvsrd v13, r26
-; CHECK-BE-NEXT:    mtvsrd v14, r29
-; CHECK-BE-NEXT:    ld r29, -24(r1) # 8-byte Folded Reload
-; CHECK-BE-NEXT:    ld r26, -48(r1) # 8-byte Folded Reload
-; CHECK-BE-NEXT:    mtvsrd v15, r25
-; CHECK-BE-NEXT:    ld r25, -56(r1) # 8-byte Folded Reload
-; CHECK-BE-NEXT:    vmrghh v2, v2, v0
-; CHECK-BE-NEXT:    vmrghh v3, v3, v1
-; CHECK-BE-NEXT:    vmrghh v4, v4, v6
-; CHECK-BE-NEXT:    vmrghh v5, v5, v7
-; CHECK-BE-NEXT:    vmrghh v0, v8, v9
-; CHECK-BE-NEXT:    vmrghh v1, v10, v11
-; CHECK-BE-NEXT:    vmrghh v6, v12, v13
-; CHECK-BE-NEXT:    vmrghh v7, v14, v15
+; CHECK-BE-NEXT:    mfvsrwz r5, f3
+; CHECK-BE-NEXT:    xscvdpsxws f3, f0
+; CHECK-BE-NEXT:    xxswapd vs0, vs0
+; CHECK-BE-NEXT:    sldi r5, r5, 48
+; CHECK-BE-NEXT:    xscvdpsxws f0, f0
+; CHECK-BE-NEXT:    mtvsrd v4, r5
+; CHECK-BE-NEXT:    mfvsrwz r5, f7
+; CHECK-BE-NEXT:    sldi r5, r5, 48
+; CHECK-BE-NEXT:    vmrghh v3, v3, v4
+; CHECK-BE-NEXT:    mtvsrd v4, r5
+; CHECK-BE-NEXT:    mfvsrwz r5, f4
 ; CHECK-BE-NEXT:    vmrghw v2, v3, v2
+; CHECK-BE-NEXT:    sldi r5, r5, 48
+; CHECK-BE-NEXT:    mtvsrd v5, r5
+; CHECK-BE-NEXT:    mfvsrwz r5, f3
+; CHECK-BE-NEXT:    sldi r5, r5, 48
+; CHECK-BE-NEXT:    mtvsrd v0, r5
+; CHECK-BE-NEXT:    mfvsrwz r5, f2
+; CHECK-BE-NEXT:    lxv vs2, 96(r4)
+; CHECK-BE-NEXT:    sldi r5, r5, 48
+; CHECK-BE-NEXT:    mtvsrd v1, r5
+; CHECK-BE-NEXT:    mfvsrwz r5, f1
+; CHECK-BE-NEXT:    lxv vs1, 80(r4)
+; CHECK-BE-NEXT:    xscvdpsxws f3, f2
+; CHECK-BE-NEXT:    xxswapd vs2, vs2
+; CHECK-BE-NEXT:    sldi r5, r5, 48
+; CHECK-BE-NEXT:    vmrghh v4, v4, v1
+; CHECK-BE-NEXT:    mtvsrd v1, r5
+; CHECK-BE-NEXT:    xscvdpsxws f2, f2
+; CHECK-BE-NEXT:    vmrghh v5, v5, v1
+; CHECK-BE-NEXT:    mfvsrwz r5, f0
+; CHECK-BE-NEXT:    lxv vs0, 64(r4)
+; CHECK-BE-NEXT:    mfvsrwz r4, f3
+; CHECK-BE-NEXT:    sldi r4, r4, 48
 ; CHECK-BE-NEXT:    vmrghw v3, v5, v4
-; CHECK-BE-NEXT:    vmrghw v4, v1, v0
-; CHECK-BE-NEXT:    vmrghw v5, v7, v6
+; CHECK-BE-NEXT:    xxmrghd vs3, v3, v2
+; CHECK-BE-NEXT:    mtvsrd v2, r4
+; CHECK-BE-NEXT:    mfvsrwz r4, f2
+; CHECK-BE-NEXT:    xscvdpsxws f2, f1
+; CHECK-BE-NEXT:    xxswapd vs1, vs1
+; CHECK-BE-NEXT:    sldi r4, r4, 48
+; CHECK-BE-NEXT:    xscvdpsxws f1, f1
+; CHECK-BE-NEXT:    mtvsrd v3, r4
+; CHECK-BE-NEXT:    vmrghh v2, v2, v3
+; CHECK-BE-NEXT:    mfvsrwz r4, f2
+; CHECK-BE-NEXT:    sldi r4, r4, 48
+; CHECK-BE-NEXT:    mtvsrd v3, r4
+; CHECK-BE-NEXT:    mfvsrwz r4, f1
+; CHECK-BE-NEXT:    xscvdpsxws f1, f0
+; CHECK-BE-NEXT:    xxswapd vs0, vs0
+; CHECK-BE-NEXT:    sldi r4, r4, 48
+; CHECK-BE-NEXT:    xscvdpsxws f0, f0
+; CHECK-BE-NEXT:    mtvsrd v4, r4
+; CHECK-BE-NEXT:    vmrghh v3, v3, v4
+; CHECK-BE-NEXT:    mfvsrwz r4, f1
+; CHECK-BE-NEXT:    sldi r4, r4, 48
+; CHECK-BE-NEXT:    mtvsrd v4, r4
+; CHECK-BE-NEXT:    mfvsrwz r4, f0
+; CHECK-BE-NEXT:    sldi r5, r5, 48
+; CHECK-BE-NEXT:    mtvsrd v1, r5
+; CHECK-BE-NEXT:    vmrghh v0, v0, v1
+; CHECK-BE-NEXT:    vmrghw v2, v2, v0
+; CHECK-BE-NEXT:    stxv vs3, 0(r3)
+; CHECK-BE-NEXT:    sldi r4, r4, 48
+; CHECK-BE-NEXT:    mtvsrd v5, r4
+; CHECK-BE-NEXT:    vmrghh v4, v4, v5
+; CHECK-BE-NEXT:    vmrghw v3, v4, v3
 ; CHECK-BE-NEXT:    xxmrghd vs0, v3, v2
-; CHECK-BE-NEXT:    xxmrghd vs1, v5, v4
-; CHECK-BE-NEXT:    stxv vs0, 0(r3)
-; CHECK-BE-NEXT:    stxv vs1, 16(r3)
+; CHECK-BE-NEXT:    stxv vs0, 16(r3)
 ; CHECK-BE-NEXT:    blr
 entry:
   %a = load <16 x double>, <16 x double>* %0, align 128
diff --git a/test/CodeGen/PowerPC/vec_conv_fp64_to_i32_elts.ll b/test/CodeGen/PowerPC/vec_conv_fp64_to_i32_elts.ll
index 5f35f808fb89d3862ed9043578ef2e588f66bfa2..f3431f862b1450dc7546b14e7056d2b765927730 100644
--- a/test/CodeGen/PowerPC/vec_conv_fp64_to_i32_elts.ll
+++ b/test/CodeGen/PowerPC/vec_conv_fp64_to_i32_elts.ll
@@ -28,27 +28,27 @@ define i64 @test2elt(<2 x double> %a) local_unnamed_addr #0 {
 ;
 ; CHECK-P9-LABEL: test2elt:
 ; CHECK-P9:       # %bb.0: # %entry
+; CHECK-P9-NEXT:    xscvdpuxws f0, v2
+; CHECK-P9-NEXT:    mfvsrwz r3, f0
 ; CHECK-P9-NEXT:    xxswapd vs0, v2
-; CHECK-P9-NEXT:    xscvdpuxws f1, v2
+; CHECK-P9-NEXT:    mtvsrws v3, r3
 ; CHECK-P9-NEXT:    xscvdpuxws f0, f0
-; CHECK-P9-NEXT:    mfvsrwz r3, f1
+; CHECK-P9-NEXT:    mfvsrwz r3, f0
 ; CHECK-P9-NEXT:    mtvsrws v2, r3
-; CHECK-P9-NEXT:    mfvsrwz r4, f0
-; CHECK-P9-NEXT:    mtvsrws v3, r4
-; CHECK-P9-NEXT:    vmrglw v2, v2, v3
+; CHECK-P9-NEXT:    vmrglw v2, v3, v2
 ; CHECK-P9-NEXT:    mfvsrld r3, v2
 ; CHECK-P9-NEXT:    blr
 ;
 ; CHECK-BE-LABEL: test2elt:
 ; CHECK-BE:       # %bb.0: # %entry
+; CHECK-BE-NEXT:    xscvdpuxws f0, v2
+; CHECK-BE-NEXT:    mfvsrwz r3, f0
 ; CHECK-BE-NEXT:    xxswapd vs0, v2
-; CHECK-BE-NEXT:    xscvdpuxws f1, v2
+; CHECK-BE-NEXT:    mtvsrws v3, r3
 ; CHECK-BE-NEXT:    xscvdpuxws f0, f0
-; CHECK-BE-NEXT:    mfvsrwz r3, f1
+; CHECK-BE-NEXT:    mfvsrwz r3, f0
 ; CHECK-BE-NEXT:    mtvsrws v2, r3
-; CHECK-BE-NEXT:    mfvsrwz r4, f0
-; CHECK-BE-NEXT:    mtvsrws v3, r4
-; CHECK-BE-NEXT:    vmrghw v2, v2, v3
+; CHECK-BE-NEXT:    vmrghw v2, v3, v2
 ; CHECK-BE-NEXT:    mfvsrd r3, v2
 ; CHECK-BE-NEXT:    blr
 entry:
@@ -129,42 +129,42 @@ define void @test8elt(<8 x i32>* noalias nocapture sret %agg.result, <8 x double
 ;
 ; CHECK-P9-LABEL: test8elt:
 ; CHECK-P9:       # %bb.0: # %entry
-; CHECK-P9-NEXT:    lxv vs0, 32(r4)
-; CHECK-P9-NEXT:    lxv vs1, 48(r4)
 ; CHECK-P9-NEXT:    lxv vs2, 0(r4)
 ; CHECK-P9-NEXT:    lxv vs3, 16(r4)
 ; CHECK-P9-NEXT:    xxmrgld vs4, vs3, vs2
 ; CHECK-P9-NEXT:    xxmrghd vs2, vs3, vs2
-; CHECK-P9-NEXT:    xxmrgld vs3, vs1, vs0
-; CHECK-P9-NEXT:    xxmrghd vs0, vs1, vs0
+; CHECK-P9-NEXT:    lxv vs0, 32(r4)
+; CHECK-P9-NEXT:    lxv vs1, 48(r4)
 ; CHECK-P9-NEXT:    xvcvdpuxws v2, vs4
 ; CHECK-P9-NEXT:    xvcvdpuxws v3, vs2
-; CHECK-P9-NEXT:    xvcvdpuxws v4, vs3
-; CHECK-P9-NEXT:    xvcvdpuxws v5, vs0
+; CHECK-P9-NEXT:    xxmrgld vs2, vs1, vs0
+; CHECK-P9-NEXT:    xxmrghd vs0, vs1, vs0
+; CHECK-P9-NEXT:    xvcvdpuxws v4, vs0
 ; CHECK-P9-NEXT:    vmrgew v2, v3, v2
-; CHECK-P9-NEXT:    vmrgew v3, v5, v4
-; CHECK-P9-NEXT:    stxv v3, 16(r3)
+; CHECK-P9-NEXT:    xvcvdpuxws v3, vs2
 ; CHECK-P9-NEXT:    stxv v2, 0(r3)
+; CHECK-P9-NEXT:    vmrgew v3, v4, v3
+; CHECK-P9-NEXT:    stxv v3, 16(r3)
 ; CHECK-P9-NEXT:    blr
 ;
 ; CHECK-BE-LABEL: test8elt:
 ; CHECK-BE:       # %bb.0: # %entry
-; CHECK-BE-NEXT:    lxv vs0, 48(r4)
-; CHECK-BE-NEXT:    lxv vs1, 32(r4)
 ; CHECK-BE-NEXT:    lxv vs2, 16(r4)
 ; CHECK-BE-NEXT:    lxv vs3, 0(r4)
 ; CHECK-BE-NEXT:    xxmrgld vs4, vs3, vs2
 ; CHECK-BE-NEXT:    xxmrghd vs2, vs3, vs2
-; CHECK-BE-NEXT:    xxmrgld vs3, vs1, vs0
-; CHECK-BE-NEXT:    xxmrghd vs0, vs1, vs0
+; CHECK-BE-NEXT:    lxv vs0, 48(r4)
+; CHECK-BE-NEXT:    lxv vs1, 32(r4)
 ; CHECK-BE-NEXT:    xvcvdpuxws v2, vs4
 ; CHECK-BE-NEXT:    xvcvdpuxws v3, vs2
-; CHECK-BE-NEXT:    xvcvdpuxws v4, vs3
-; CHECK-BE-NEXT:    xvcvdpuxws v5, vs0
+; CHECK-BE-NEXT:    xxmrgld vs2, vs1, vs0
+; CHECK-BE-NEXT:    xxmrghd vs0, vs1, vs0
+; CHECK-BE-NEXT:    xvcvdpuxws v4, vs0
 ; CHECK-BE-NEXT:    vmrgew v2, v3, v2
-; CHECK-BE-NEXT:    vmrgew v3, v5, v4
-; CHECK-BE-NEXT:    stxv v3, 16(r3)
+; CHECK-BE-NEXT:    xvcvdpuxws v3, vs2
 ; CHECK-BE-NEXT:    stxv v2, 0(r3)
+; CHECK-BE-NEXT:    vmrgew v3, v4, v3
+; CHECK-BE-NEXT:    stxv v3, 16(r3)
 ; CHECK-BE-NEXT:    blr
 entry:
   %a = load <8 x double>, <8 x double>* %0, align 64
@@ -227,74 +227,74 @@ define void @test16elt(<16 x i32>* noalias nocapture sret %agg.result, <16 x dou
 ;
 ; CHECK-P9-LABEL: test16elt:
 ; CHECK-P9:       # %bb.0: # %entry
-; CHECK-P9-NEXT:    lxv vs0, 32(r4)
-; CHECK-P9-NEXT:    lxv vs1, 48(r4)
-; CHECK-P9-NEXT:    lxv vs2, 0(r4)
-; CHECK-P9-NEXT:    lxv vs3, 16(r4)
-; CHECK-P9-NEXT:    lxv vs4, 96(r4)
-; CHECK-P9-NEXT:    lxv vs5, 112(r4)
-; CHECK-P9-NEXT:    lxv vs6, 64(r4)
-; CHECK-P9-NEXT:    lxv vs7, 80(r4)
-; CHECK-P9-NEXT:    xxmrgld vs8, vs3, vs2
-; CHECK-P9-NEXT:    xxmrghd vs2, vs3, vs2
-; CHECK-P9-NEXT:    xxmrgld vs3, vs1, vs0
-; CHECK-P9-NEXT:    xxmrghd vs0, vs1, vs0
-; CHECK-P9-NEXT:    xxmrgld vs1, vs7, vs6
+; CHECK-P9-NEXT:    lxv vs6, 0(r4)
+; CHECK-P9-NEXT:    lxv vs7, 16(r4)
+; CHECK-P9-NEXT:    xxmrgld vs8, vs7, vs6
 ; CHECK-P9-NEXT:    xxmrghd vs6, vs7, vs6
+; CHECK-P9-NEXT:    lxv vs4, 32(r4)
+; CHECK-P9-NEXT:    lxv vs5, 48(r4)
 ; CHECK-P9-NEXT:    xxmrgld vs7, vs5, vs4
 ; CHECK-P9-NEXT:    xxmrghd vs4, vs5, vs4
 ; CHECK-P9-NEXT:    xvcvdpuxws v2, vs8
-; CHECK-P9-NEXT:    xvcvdpuxws v3, vs2
-; CHECK-P9-NEXT:    xvcvdpuxws v4, vs3
-; CHECK-P9-NEXT:    xvcvdpuxws v5, vs0
-; CHECK-P9-NEXT:    xvcvdpuxws v0, vs1
-; CHECK-P9-NEXT:    xvcvdpuxws v1, vs6
-; CHECK-P9-NEXT:    xvcvdpuxws v6, vs7
-; CHECK-P9-NEXT:    xvcvdpuxws v7, vs4
+; CHECK-P9-NEXT:    xvcvdpuxws v3, vs6
+; CHECK-P9-NEXT:    lxv vs2, 64(r4)
+; CHECK-P9-NEXT:    lxv vs3, 80(r4)
+; CHECK-P9-NEXT:    xvcvdpuxws v4, vs7
 ; CHECK-P9-NEXT:    vmrgew v2, v3, v2
-; CHECK-P9-NEXT:    vmrgew v3, v5, v4
-; CHECK-P9-NEXT:    vmrgew v4, v1, v0
-; CHECK-P9-NEXT:    vmrgew v5, v7, v6
-; CHECK-P9-NEXT:    stxv v3, 16(r3)
+; CHECK-P9-NEXT:    xvcvdpuxws v3, vs4
+; CHECK-P9-NEXT:    xxmrgld vs4, vs3, vs2
+; CHECK-P9-NEXT:    xxmrghd vs2, vs3, vs2
+; CHECK-P9-NEXT:    lxv vs0, 96(r4)
+; CHECK-P9-NEXT:    lxv vs1, 112(r4)
 ; CHECK-P9-NEXT:    stxv v2, 0(r3)
-; CHECK-P9-NEXT:    stxv v5, 48(r3)
+; CHECK-P9-NEXT:    xvcvdpuxws v5, vs2
+; CHECK-P9-NEXT:    xxmrgld vs2, vs1, vs0
+; CHECK-P9-NEXT:    xxmrghd vs0, vs1, vs0
+; CHECK-P9-NEXT:    xvcvdpuxws v0, vs0
+; CHECK-P9-NEXT:    vmrgew v3, v3, v4
+; CHECK-P9-NEXT:    xvcvdpuxws v4, vs4
+; CHECK-P9-NEXT:    stxv v3, 16(r3)
+; CHECK-P9-NEXT:    vmrgew v4, v5, v4
 ; CHECK-P9-NEXT:    stxv v4, 32(r3)
+; CHECK-P9-NEXT:    xvcvdpuxws v5, vs2
+; CHECK-P9-NEXT:    vmrgew v5, v0, v5
+; CHECK-P9-NEXT:    stxv v5, 48(r3)
 ; CHECK-P9-NEXT:    blr
 ;
 ; CHECK-BE-LABEL: test16elt:
 ; CHECK-BE:       # %bb.0: # %entry
-; CHECK-BE-NEXT:    lxv vs0, 48(r4)
-; CHECK-BE-NEXT:    lxv vs1, 32(r4)
-; CHECK-BE-NEXT:    lxv vs2, 16(r4)
-; CHECK-BE-NEXT:    lxv vs3, 0(r4)
-; CHECK-BE-NEXT:    lxv vs4, 112(r4)
-; CHECK-BE-NEXT:    lxv vs5, 96(r4)
-; CHECK-BE-NEXT:    lxv vs6, 80(r4)
-; CHECK-BE-NEXT:    lxv vs7, 64(r4)
-; CHECK-BE-NEXT:    xxmrgld vs8, vs3, vs2
-; CHECK-BE-NEXT:    xxmrghd vs2, vs3, vs2
-; CHECK-BE-NEXT:    xxmrgld vs3, vs1, vs0
-; CHECK-BE-NEXT:    xxmrghd vs0, vs1, vs0
-; CHECK-BE-NEXT:    xxmrgld vs1, vs7, vs6
+; CHECK-BE-NEXT:    lxv vs6, 16(r4)
+; CHECK-BE-NEXT:    lxv vs7, 0(r4)
+; CHECK-BE-NEXT:    xxmrgld vs8, vs7, vs6
 ; CHECK-BE-NEXT:    xxmrghd vs6, vs7, vs6
+; CHECK-BE-NEXT:    lxv vs4, 48(r4)
+; CHECK-BE-NEXT:    lxv vs5, 32(r4)
 ; CHECK-BE-NEXT:    xxmrgld vs7, vs5, vs4
 ; CHECK-BE-NEXT:    xxmrghd vs4, vs5, vs4
 ; CHECK-BE-NEXT:    xvcvdpuxws v2, vs8
-; CHECK-BE-NEXT:    xvcvdpuxws v3, vs2
-; CHECK-BE-NEXT:    xvcvdpuxws v4, vs3
-; CHECK-BE-NEXT:    xvcvdpuxws v5, vs0
-; CHECK-BE-NEXT:    xvcvdpuxws v0, vs1
-; CHECK-BE-NEXT:    xvcvdpuxws v1, vs6
-; CHECK-BE-NEXT:    xvcvdpuxws v6, vs7
-; CHECK-BE-NEXT:    xvcvdpuxws v7, vs4
+; CHECK-BE-NEXT:    xvcvdpuxws v3, vs6
+; CHECK-BE-NEXT:    lxv vs2, 80(r4)
+; CHECK-BE-NEXT:    lxv vs3, 64(r4)
+; CHECK-BE-NEXT:    xvcvdpuxws v4, vs7
 ; CHECK-BE-NEXT:    vmrgew v2, v3, v2
-; CHECK-BE-NEXT:    vmrgew v3, v5, v4
-; CHECK-BE-NEXT:    vmrgew v4, v1, v0
-; CHECK-BE-NEXT:    vmrgew v5, v7, v6
-; CHECK-BE-NEXT:    stxv v3, 16(r3)
+; CHECK-BE-NEXT:    xvcvdpuxws v3, vs4
+; CHECK-BE-NEXT:    xxmrgld vs4, vs3, vs2
+; CHECK-BE-NEXT:    xxmrghd vs2, vs3, vs2
+; CHECK-BE-NEXT:    lxv vs0, 112(r4)
+; CHECK-BE-NEXT:    lxv vs1, 96(r4)
 ; CHECK-BE-NEXT:    stxv v2, 0(r3)
-; CHECK-BE-NEXT:    stxv v5, 48(r3)
+; CHECK-BE-NEXT:    xvcvdpuxws v5, vs2
+; CHECK-BE-NEXT:    xxmrgld vs2, vs1, vs0
+; CHECK-BE-NEXT:    xxmrghd vs0, vs1, vs0
+; CHECK-BE-NEXT:    xvcvdpuxws v0, vs0
+; CHECK-BE-NEXT:    vmrgew v3, v3, v4
+; CHECK-BE-NEXT:    xvcvdpuxws v4, vs4
+; CHECK-BE-NEXT:    stxv v3, 16(r3)
+; CHECK-BE-NEXT:    vmrgew v4, v5, v4
 ; CHECK-BE-NEXT:    stxv v4, 32(r3)
+; CHECK-BE-NEXT:    xvcvdpuxws v5, vs2
+; CHECK-BE-NEXT:    vmrgew v5, v0, v5
+; CHECK-BE-NEXT:    stxv v5, 48(r3)
 ; CHECK-BE-NEXT:    blr
 entry:
   %a = load <16 x double>, <16 x double>* %0, align 128
@@ -322,27 +322,27 @@ define i64 @test2elt_signed(<2 x double> %a) local_unnamed_addr #0 {
 ;
 ; CHECK-P9-LABEL: test2elt_signed:
 ; CHECK-P9:       # %bb.0: # %entry
+; CHECK-P9-NEXT:    xscvdpsxws f0, v2
+; CHECK-P9-NEXT:    mfvsrwz r3, f0
 ; CHECK-P9-NEXT:    xxswapd vs0, v2
-; CHECK-P9-NEXT:    xscvdpsxws f1, v2
+; CHECK-P9-NEXT:    mtvsrws v3, r3
 ; CHECK-P9-NEXT:    xscvdpsxws f0, f0
-; CHECK-P9-NEXT:    mfvsrwz r3, f1
+; CHECK-P9-NEXT:    mfvsrwz r3, f0
 ; CHECK-P9-NEXT:    mtvsrws v2, r3
-; CHECK-P9-NEXT:    mfvsrwz r4, f0
-; CHECK-P9-NEXT:    mtvsrws v3, r4
-; CHECK-P9-NEXT:    vmrglw v2, v2, v3
+; CHECK-P9-NEXT:    vmrglw v2, v3, v2
 ; CHECK-P9-NEXT:    mfvsrld r3, v2
 ; CHECK-P9-NEXT:    blr
 ;
 ; CHECK-BE-LABEL: test2elt_signed:
 ; CHECK-BE:       # %bb.0: # %entry
+; CHECK-BE-NEXT:    xscvdpsxws f0, v2
+; CHECK-BE-NEXT:    mfvsrwz r3, f0
 ; CHECK-BE-NEXT:    xxswapd vs0, v2
-; CHECK-BE-NEXT:    xscvdpsxws f1, v2
+; CHECK-BE-NEXT:    mtvsrws v3, r3
 ; CHECK-BE-NEXT:    xscvdpsxws f0, f0
-; CHECK-BE-NEXT:    mfvsrwz r3, f1
+; CHECK-BE-NEXT:    mfvsrwz r3, f0
 ; CHECK-BE-NEXT:    mtvsrws v2, r3
-; CHECK-BE-NEXT:    mfvsrwz r4, f0
-; CHECK-BE-NEXT:    mtvsrws v3, r4
-; CHECK-BE-NEXT:    vmrghw v2, v2, v3
+; CHECK-BE-NEXT:    vmrghw v2, v3, v2
 ; CHECK-BE-NEXT:    mfvsrd r3, v2
 ; CHECK-BE-NEXT:    blr
 entry:
@@ -423,42 +423,42 @@ define void @test8elt_signed(<8 x i32>* noalias nocapture sret %agg.result, <8 x
 ;
 ; CHECK-P9-LABEL: test8elt_signed:
 ; CHECK-P9:       # %bb.0: # %entry
-; CHECK-P9-NEXT:    lxv vs0, 32(r4)
-; CHECK-P9-NEXT:    lxv vs1, 48(r4)
 ; CHECK-P9-NEXT:    lxv vs2, 0(r4)
 ; CHECK-P9-NEXT:    lxv vs3, 16(r4)
 ; CHECK-P9-NEXT:    xxmrgld vs4, vs3, vs2
 ; CHECK-P9-NEXT:    xxmrghd vs2, vs3, vs2
-; CHECK-P9-NEXT:    xxmrgld vs3, vs1, vs0
-; CHECK-P9-NEXT:    xxmrghd vs0, vs1, vs0
+; CHECK-P9-NEXT:    lxv vs0, 32(r4)
+; CHECK-P9-NEXT:    lxv vs1, 48(r4)
 ; CHECK-P9-NEXT:    xvcvdpsxws v2, vs4
 ; CHECK-P9-NEXT:    xvcvdpsxws v3, vs2
-; CHECK-P9-NEXT:    xvcvdpsxws v4, vs3
-; CHECK-P9-NEXT:    xvcvdpsxws v5, vs0
+; CHECK-P9-NEXT:    xxmrgld vs2, vs1, vs0
+; CHECK-P9-NEXT:    xxmrghd vs0, vs1, vs0
+; CHECK-P9-NEXT:    xvcvdpsxws v4, vs0
 ; CHECK-P9-NEXT:    vmrgew v2, v3, v2
-; CHECK-P9-NEXT:    vmrgew v3, v5, v4
-; CHECK-P9-NEXT:    stxv v3, 16(r3)
+; CHECK-P9-NEXT:    xvcvdpsxws v3, vs2
 ; CHECK-P9-NEXT:    stxv v2, 0(r3)
+; CHECK-P9-NEXT:    vmrgew v3, v4, v3
+; CHECK-P9-NEXT:    stxv v3, 16(r3)
 ; CHECK-P9-NEXT:    blr
 ;
 ; CHECK-BE-LABEL: test8elt_signed:
 ; CHECK-BE:       # %bb.0: # %entry
-; CHECK-BE-NEXT:    lxv vs0, 48(r4)
-; CHECK-BE-NEXT:    lxv vs1, 32(r4)
 ; CHECK-BE-NEXT:    lxv vs2, 16(r4)
 ; CHECK-BE-NEXT:    lxv vs3, 0(r4)
 ; CHECK-BE-NEXT:    xxmrgld vs4, vs3, vs2
 ; CHECK-BE-NEXT:    xxmrghd vs2, vs3, vs2
-; CHECK-BE-NEXT:    xxmrgld vs3, vs1, vs0
-; CHECK-BE-NEXT:    xxmrghd vs0, vs1, vs0
+; CHECK-BE-NEXT:    lxv vs0, 48(r4)
+; CHECK-BE-NEXT:    lxv vs1, 32(r4)
 ; CHECK-BE-NEXT:    xvcvdpsxws v2, vs4
 ; CHECK-BE-NEXT:    xvcvdpsxws v3, vs2
-; CHECK-BE-NEXT:    xvcvdpsxws v4, vs3
-; CHECK-BE-NEXT:    xvcvdpsxws v5, vs0
+; CHECK-BE-NEXT:    xxmrgld vs2, vs1, vs0
+; CHECK-BE-NEXT:    xxmrghd vs0, vs1, vs0
+; CHECK-BE-NEXT:    xvcvdpsxws v4, vs0
 ; CHECK-BE-NEXT:    vmrgew v2, v3, v2
-; CHECK-BE-NEXT:    vmrgew v3, v5, v4
-; CHECK-BE-NEXT:    stxv v3, 16(r3)
+; CHECK-BE-NEXT:    xvcvdpsxws v3, vs2
 ; CHECK-BE-NEXT:    stxv v2, 0(r3)
+; CHECK-BE-NEXT:    vmrgew v3, v4, v3
+; CHECK-BE-NEXT:    stxv v3, 16(r3)
 ; CHECK-BE-NEXT:    blr
 entry:
   %a = load <8 x double>, <8 x double>* %0, align 64
@@ -521,74 +521,74 @@ define void @test16elt_signed(<16 x i32>* noalias nocapture sret %agg.result, <1
 ;
 ; CHECK-P9-LABEL: test16elt_signed:
 ; CHECK-P9:       # %bb.0: # %entry
-; CHECK-P9-NEXT:    lxv vs0, 32(r4)
-; CHECK-P9-NEXT:    lxv vs1, 48(r4)
-; CHECK-P9-NEXT:    lxv vs2, 0(r4)
-; CHECK-P9-NEXT:    lxv vs3, 16(r4)
-; CHECK-P9-NEXT:    lxv vs4, 96(r4)
-; CHECK-P9-NEXT:    lxv vs5, 112(r4)
-; CHECK-P9-NEXT:    lxv vs6, 64(r4)
-; CHECK-P9-NEXT:    lxv vs7, 80(r4)
-; CHECK-P9-NEXT:    xxmrgld vs8, vs3, vs2
-; CHECK-P9-NEXT:    xxmrghd vs2, vs3, vs2
-; CHECK-P9-NEXT:    xxmrgld vs3, vs1, vs0
-; CHECK-P9-NEXT:    xxmrghd vs0, vs1, vs0
-; CHECK-P9-NEXT:    xxmrgld vs1, vs7, vs6
+; CHECK-P9-NEXT:    lxv vs6, 0(r4)
+; CHECK-P9-NEXT:    lxv vs7, 16(r4)
+; CHECK-P9-NEXT:    xxmrgld vs8, vs7, vs6
 ; CHECK-P9-NEXT:    xxmrghd vs6, vs7, vs6
+; CHECK-P9-NEXT:    lxv vs4, 32(r4)
+; CHECK-P9-NEXT:    lxv vs5, 48(r4)
 ; CHECK-P9-NEXT:    xxmrgld vs7, vs5, vs4
 ; CHECK-P9-NEXT:    xxmrghd vs4, vs5, vs4
 ; CHECK-P9-NEXT:    xvcvdpsxws v2, vs8
-; CHECK-P9-NEXT:    xvcvdpsxws v3, vs2
-; CHECK-P9-NEXT:    xvcvdpsxws v4, vs3
-; CHECK-P9-NEXT:    xvcvdpsxws v5, vs0
-; CHECK-P9-NEXT:    xvcvdpsxws v0, vs1
-; CHECK-P9-NEXT:    xvcvdpsxws v1, vs6
-; CHECK-P9-NEXT:    xvcvdpsxws v6, vs7
-; CHECK-P9-NEXT:    xvcvdpsxws v7, vs4
+; CHECK-P9-NEXT:    xvcvdpsxws v3, vs6
+; CHECK-P9-NEXT:    lxv vs2, 64(r4)
+; CHECK-P9-NEXT:    lxv vs3, 80(r4)
+; CHECK-P9-NEXT:    xvcvdpsxws v4, vs7
 ; CHECK-P9-NEXT:    vmrgew v2, v3, v2
-; CHECK-P9-NEXT:    vmrgew v3, v5, v4
-; CHECK-P9-NEXT:    vmrgew v4, v1, v0
-; CHECK-P9-NEXT:    vmrgew v5, v7, v6
-; CHECK-P9-NEXT:    stxv v3, 16(r3)
+; CHECK-P9-NEXT:    xvcvdpsxws v3, vs4
+; CHECK-P9-NEXT:    xxmrgld vs4, vs3, vs2
+; CHECK-P9-NEXT:    xxmrghd vs2, vs3, vs2
+; CHECK-P9-NEXT:    lxv vs0, 96(r4)
+; CHECK-P9-NEXT:    lxv vs1, 112(r4)
 ; CHECK-P9-NEXT:    stxv v2, 0(r3)
-; CHECK-P9-NEXT:    stxv v5, 48(r3)
+; CHECK-P9-NEXT:    xvcvdpsxws v5, vs2
+; CHECK-P9-NEXT:    xxmrgld vs2, vs1, vs0
+; CHECK-P9-NEXT:    xxmrghd vs0, vs1, vs0
+; CHECK-P9-NEXT:    xvcvdpsxws v0, vs0
+; CHECK-P9-NEXT:    vmrgew v3, v3, v4
+; CHECK-P9-NEXT:    xvcvdpsxws v4, vs4
+; CHECK-P9-NEXT:    stxv v3, 16(r3)
+; CHECK-P9-NEXT:    vmrgew v4, v5, v4
 ; CHECK-P9-NEXT:    stxv v4, 32(r3)
+; CHECK-P9-NEXT:    xvcvdpsxws v5, vs2
+; CHECK-P9-NEXT:    vmrgew v5, v0, v5
+; CHECK-P9-NEXT:    stxv v5, 48(r3)
 ; CHECK-P9-NEXT:    blr
 ;
 ; CHECK-BE-LABEL: test16elt_signed:
 ; CHECK-BE:       # %bb.0: # %entry
-; CHECK-BE-NEXT:    lxv vs0, 48(r4)
-; CHECK-BE-NEXT:    lxv vs1, 32(r4)
-; CHECK-BE-NEXT:    lxv vs2, 16(r4)
-; CHECK-BE-NEXT:    lxv vs3, 0(r4)
-; CHECK-BE-NEXT:    lxv vs4, 112(r4)
-; CHECK-BE-NEXT:    lxv vs5, 96(r4)
-; CHECK-BE-NEXT:    lxv vs6, 80(r4)
-; CHECK-BE-NEXT:    lxv vs7, 64(r4)
-; CHECK-BE-NEXT:    xxmrgld vs8, vs3, vs2
-; CHECK-BE-NEXT:    xxmrghd vs2, vs3, vs2
-; CHECK-BE-NEXT:    xxmrgld vs3, vs1, vs0
-; CHECK-BE-NEXT:    xxmrghd vs0, vs1, vs0
-; CHECK-BE-NEXT:    xxmrgld vs1, vs7, vs6
+; CHECK-BE-NEXT:    lxv vs6, 16(r4)
+; CHECK-BE-NEXT:    lxv vs7, 0(r4)
+; CHECK-BE-NEXT:    xxmrgld vs8, vs7, vs6
 ; CHECK-BE-NEXT:    xxmrghd vs6, vs7, vs6
+; CHECK-BE-NEXT:    lxv vs4, 48(r4)
+; CHECK-BE-NEXT:    lxv vs5, 32(r4)
 ; CHECK-BE-NEXT:    xxmrgld vs7, vs5, vs4
 ; CHECK-BE-NEXT:    xxmrghd vs4, vs5, vs4
 ; CHECK-BE-NEXT:    xvcvdpsxws v2, vs8
-; CHECK-BE-NEXT:    xvcvdpsxws v3, vs2
-; CHECK-BE-NEXT:    xvcvdpsxws v4, vs3
-; CHECK-BE-NEXT:    xvcvdpsxws v5, vs0
-; CHECK-BE-NEXT:    xvcvdpsxws v0, vs1
-; CHECK-BE-NEXT:    xvcvdpsxws v1, vs6
-; CHECK-BE-NEXT:    xvcvdpsxws v6, vs7
-; CHECK-BE-NEXT:    xvcvdpsxws v7, vs4
+; CHECK-BE-NEXT:    xvcvdpsxws v3, vs6
+; CHECK-BE-NEXT:    lxv vs2, 80(r4)
+; CHECK-BE-NEXT:    lxv vs3, 64(r4)
+; CHECK-BE-NEXT:    xvcvdpsxws v4, vs7
 ; CHECK-BE-NEXT:    vmrgew v2, v3, v2
-; CHECK-BE-NEXT:    vmrgew v3, v5, v4
-; CHECK-BE-NEXT:    vmrgew v4, v1, v0
-; CHECK-BE-NEXT:    vmrgew v5, v7, v6
-; CHECK-BE-NEXT:    stxv v3, 16(r3)
+; CHECK-BE-NEXT:    xvcvdpsxws v3, vs4
+; CHECK-BE-NEXT:    xxmrgld vs4, vs3, vs2
+; CHECK-BE-NEXT:    xxmrghd vs2, vs3, vs2
+; CHECK-BE-NEXT:    lxv vs0, 112(r4)
+; CHECK-BE-NEXT:    lxv vs1, 96(r4)
 ; CHECK-BE-NEXT:    stxv v2, 0(r3)
-; CHECK-BE-NEXT:    stxv v5, 48(r3)
+; CHECK-BE-NEXT:    xvcvdpsxws v5, vs2
+; CHECK-BE-NEXT:    xxmrgld vs2, vs1, vs0
+; CHECK-BE-NEXT:    xxmrghd vs0, vs1, vs0
+; CHECK-BE-NEXT:    xvcvdpsxws v0, vs0
+; CHECK-BE-NEXT:    vmrgew v3, v3, v4
+; CHECK-BE-NEXT:    xvcvdpsxws v4, vs4
+; CHECK-BE-NEXT:    stxv v3, 16(r3)
+; CHECK-BE-NEXT:    vmrgew v4, v5, v4
 ; CHECK-BE-NEXT:    stxv v4, 32(r3)
+; CHECK-BE-NEXT:    xvcvdpsxws v5, vs2
+; CHECK-BE-NEXT:    vmrgew v5, v0, v5
+; CHECK-BE-NEXT:    stxv v5, 48(r3)
 ; CHECK-BE-NEXT:    blr
 entry:
   %a = load <16 x double>, <16 x double>* %0, align 128
diff --git a/test/CodeGen/PowerPC/vec_conv_fp64_to_i8_elts.ll b/test/CodeGen/PowerPC/vec_conv_fp64_to_i8_elts.ll
index ef7b9c190bc23d331201b2c4fb83a11ac3ea72fc..5e4751e554e1b29acee9aaea3e871816ca6669d5 100644
--- a/test/CodeGen/PowerPC/vec_conv_fp64_to_i8_elts.ll
+++ b/test/CodeGen/PowerPC/vec_conv_fp64_to_i8_elts.ll
@@ -31,17 +31,17 @@ define i16 @test2elt(<2 x double> %a) local_unnamed_addr #0 {
 ;
 ; CHECK-P9-LABEL: test2elt:
 ; CHECK-P9:       # %bb.0: # %entry
+; CHECK-P9-NEXT:    xscvdpsxws f0, v2
+; CHECK-P9-NEXT:    mfvsrwz r3, f0
+; CHECK-P9-NEXT:    mtvsrd f0, r3
+; CHECK-P9-NEXT:    xxswapd v3, vs0
 ; CHECK-P9-NEXT:    xxswapd vs0, v2
-; CHECK-P9-NEXT:    xscvdpsxws f1, v2
 ; CHECK-P9-NEXT:    xscvdpsxws f0, f0
-; CHECK-P9-NEXT:    mfvsrwz r3, f1
-; CHECK-P9-NEXT:    mfvsrwz r4, f0
+; CHECK-P9-NEXT:    mfvsrwz r3, f0
 ; CHECK-P9-NEXT:    mtvsrd f0, r3
 ; CHECK-P9-NEXT:    addi r3, r1, -2
-; CHECK-P9-NEXT:    mtvsrd f1, r4
 ; CHECK-P9-NEXT:    xxswapd v2, vs0
-; CHECK-P9-NEXT:    xxswapd v3, vs1
-; CHECK-P9-NEXT:    vmrglb v2, v2, v3
+; CHECK-P9-NEXT:    vmrglb v2, v3, v2
 ; CHECK-P9-NEXT:    vsldoi v2, v2, v2, 8
 ; CHECK-P9-NEXT:    stxsihx v2, 0, r3
 ; CHECK-P9-NEXT:    lhz r3, -2(r1)
@@ -49,17 +49,17 @@ define i16 @test2elt(<2 x double> %a) local_unnamed_addr #0 {
 ;
 ; CHECK-BE-LABEL: test2elt:
 ; CHECK-BE:       # %bb.0: # %entry
+; CHECK-BE-NEXT:    xscvdpsxws f0, v2
+; CHECK-BE-NEXT:    mfvsrwz r3, f0
 ; CHECK-BE-NEXT:    xxswapd vs0, v2
-; CHECK-BE-NEXT:    xscvdpsxws f1, v2
+; CHECK-BE-NEXT:    sldi r3, r3, 56
 ; CHECK-BE-NEXT:    xscvdpsxws f0, f0
-; CHECK-BE-NEXT:    mfvsrwz r3, f1
+; CHECK-BE-NEXT:    mtvsrd v3, r3
+; CHECK-BE-NEXT:    mfvsrwz r3, f0
 ; CHECK-BE-NEXT:    sldi r3, r3, 56
-; CHECK-BE-NEXT:    mfvsrwz r4, f0
 ; CHECK-BE-NEXT:    mtvsrd v2, r3
 ; CHECK-BE-NEXT:    addi r3, r1, -2
-; CHECK-BE-NEXT:    sldi r4, r4, 56
-; CHECK-BE-NEXT:    mtvsrd v3, r4
-; CHECK-BE-NEXT:    vmrghb v2, v2, v3
+; CHECK-BE-NEXT:    vmrghb v2, v3, v2
 ; CHECK-BE-NEXT:    vsldoi v2, v2, v2, 10
 ; CHECK-BE-NEXT:    stxsihx v2, 0, r3
 ; CHECK-BE-NEXT:    lhz r3, -2(r1)
@@ -103,58 +103,58 @@ define i32 @test4elt(<4 x double>* nocapture readonly) local_unnamed_addr #1 {
 ;
 ; CHECK-P9-LABEL: test4elt:
 ; CHECK-P9:       # %bb.0: # %entry
-; CHECK-P9-NEXT:    lxv vs0, 16(r3)
 ; CHECK-P9-NEXT:    lxv vs1, 0(r3)
-; CHECK-P9-NEXT:    xxswapd vs2, vs1
-; CHECK-P9-NEXT:    xxswapd vs3, vs0
+; CHECK-P9-NEXT:    xscvdpsxws f2, f1
+; CHECK-P9-NEXT:    xxswapd vs1, vs1
 ; CHECK-P9-NEXT:    xscvdpsxws f1, f1
+; CHECK-P9-NEXT:    lxv vs0, 16(r3)
+; CHECK-P9-NEXT:    mfvsrwz r3, f2
+; CHECK-P9-NEXT:    mtvsrd f2, r3
+; CHECK-P9-NEXT:    mfvsrwz r3, f1
+; CHECK-P9-NEXT:    xxswapd v2, vs2
+; CHECK-P9-NEXT:    mtvsrd f1, r3
+; CHECK-P9-NEXT:    xxswapd v3, vs1
+; CHECK-P9-NEXT:    xscvdpsxws f1, f0
+; CHECK-P9-NEXT:    xxswapd vs0, vs0
 ; CHECK-P9-NEXT:    xscvdpsxws f0, f0
-; CHECK-P9-NEXT:    xscvdpsxws f2, f2
-; CHECK-P9-NEXT:    xscvdpsxws f3, f3
 ; CHECK-P9-NEXT:    mfvsrwz r3, f1
-; CHECK-P9-NEXT:    mfvsrwz r5, f0
+; CHECK-P9-NEXT:    mtvsrd f1, r3
+; CHECK-P9-NEXT:    mfvsrwz r3, f0
 ; CHECK-P9-NEXT:    mtvsrd f0, r3
-; CHECK-P9-NEXT:    li r3, 0
-; CHECK-P9-NEXT:    mfvsrwz r4, f2
-; CHECK-P9-NEXT:    mfvsrwz r6, f3
-; CHECK-P9-NEXT:    mtvsrd f2, r5
-; CHECK-P9-NEXT:    xxswapd v2, vs0
-; CHECK-P9-NEXT:    mtvsrd f1, r4
-; CHECK-P9-NEXT:    mtvsrd f3, r6
-; CHECK-P9-NEXT:    xxswapd v4, vs2
-; CHECK-P9-NEXT:    xxswapd v3, vs1
-; CHECK-P9-NEXT:    xxswapd v5, vs3
 ; CHECK-P9-NEXT:    vmrglb v2, v2, v3
-; CHECK-P9-NEXT:    vmrglb v3, v4, v5
+; CHECK-P9-NEXT:    xxswapd v3, vs1
+; CHECK-P9-NEXT:    xxswapd v4, vs0
+; CHECK-P9-NEXT:    vmrglb v3, v3, v4
 ; CHECK-P9-NEXT:    vmrglh v2, v3, v2
+; CHECK-P9-NEXT:    li r3, 0
 ; CHECK-P9-NEXT:    vextuwrx r3, r3, v2
 ; CHECK-P9-NEXT:    blr
 ;
 ; CHECK-BE-LABEL: test4elt:
 ; CHECK-BE:       # %bb.0: # %entry
-; CHECK-BE-NEXT:    lxv vs0, 0(r3)
 ; CHECK-BE-NEXT:    lxv vs1, 16(r3)
-; CHECK-BE-NEXT:    xxswapd vs2, vs1
-; CHECK-BE-NEXT:    xxswapd vs3, vs0
+; CHECK-BE-NEXT:    xscvdpsxws f2, f1
+; CHECK-BE-NEXT:    xxswapd vs1, vs1
 ; CHECK-BE-NEXT:    xscvdpsxws f1, f1
+; CHECK-BE-NEXT:    lxv vs0, 0(r3)
+; CHECK-BE-NEXT:    mfvsrwz r3, f2
+; CHECK-BE-NEXT:    sldi r3, r3, 56
+; CHECK-BE-NEXT:    mtvsrd v2, r3
+; CHECK-BE-NEXT:    mfvsrwz r3, f1
+; CHECK-BE-NEXT:    xscvdpsxws f1, f0
+; CHECK-BE-NEXT:    xxswapd vs0, vs0
+; CHECK-BE-NEXT:    sldi r3, r3, 56
 ; CHECK-BE-NEXT:    xscvdpsxws f0, f0
-; CHECK-BE-NEXT:    xscvdpsxws f2, f2
-; CHECK-BE-NEXT:    xscvdpsxws f3, f3
+; CHECK-BE-NEXT:    mtvsrd v3, r3
+; CHECK-BE-NEXT:    vmrghb v2, v2, v3
 ; CHECK-BE-NEXT:    mfvsrwz r3, f1
-; CHECK-BE-NEXT:    mfvsrwz r5, f0
 ; CHECK-BE-NEXT:    sldi r3, r3, 56
-; CHECK-BE-NEXT:    sldi r5, r5, 56
-; CHECK-BE-NEXT:    mfvsrwz r4, f2
-; CHECK-BE-NEXT:    mfvsrwz r6, f3
-; CHECK-BE-NEXT:    mtvsrd v2, r3
-; CHECK-BE-NEXT:    mtvsrd v4, r5
+; CHECK-BE-NEXT:    mtvsrd v3, r3
+; CHECK-BE-NEXT:    mfvsrwz r3, f0
+; CHECK-BE-NEXT:    sldi r3, r3, 56
+; CHECK-BE-NEXT:    mtvsrd v4, r3
 ; CHECK-BE-NEXT:    li r3, 0
-; CHECK-BE-NEXT:    sldi r4, r4, 56
-; CHECK-BE-NEXT:    sldi r6, r6, 56
-; CHECK-BE-NEXT:    mtvsrd v3, r4
-; CHECK-BE-NEXT:    mtvsrd v5, r6
-; CHECK-BE-NEXT:    vmrghb v2, v2, v3
-; CHECK-BE-NEXT:    vmrghb v3, v4, v5
+; CHECK-BE-NEXT:    vmrghb v3, v3, v4
 ; CHECK-BE-NEXT:    vmrghh v2, v3, v2
 ; CHECK-BE-NEXT:    vextuwlx r3, r3, v2
 ; CHECK-BE-NEXT:    blr
@@ -224,104 +224,104 @@ define i64 @test8elt(<8 x double>* nocapture readonly) local_unnamed_addr #1 {
 ;
 ; CHECK-P9-LABEL: test8elt:
 ; CHECK-P9:       # %bb.0: # %entry
+; CHECK-P9-NEXT:    lxv vs3, 0(r3)
+; CHECK-P9-NEXT:    xscvdpsxws f4, f3
+; CHECK-P9-NEXT:    xxswapd vs3, vs3
+; CHECK-P9-NEXT:    xscvdpsxws f3, f3
 ; CHECK-P9-NEXT:    lxv vs0, 48(r3)
 ; CHECK-P9-NEXT:    lxv vs1, 32(r3)
 ; CHECK-P9-NEXT:    lxv vs2, 16(r3)
-; CHECK-P9-NEXT:    lxv vs3, 0(r3)
-; CHECK-P9-NEXT:    xxswapd vs4, vs3
-; CHECK-P9-NEXT:    xxswapd vs5, vs2
-; CHECK-P9-NEXT:    xxswapd vs6, vs1
-; CHECK-P9-NEXT:    xxswapd vs7, vs0
-; CHECK-P9-NEXT:    xscvdpsxws f3, f3
+; CHECK-P9-NEXT:    mfvsrwz r3, f4
+; CHECK-P9-NEXT:    mtvsrd f4, r3
+; CHECK-P9-NEXT:    mfvsrwz r3, f3
+; CHECK-P9-NEXT:    xxswapd v2, vs4
+; CHECK-P9-NEXT:    mtvsrd f3, r3
+; CHECK-P9-NEXT:    xxswapd v3, vs3
+; CHECK-P9-NEXT:    xscvdpsxws f3, f2
+; CHECK-P9-NEXT:    xxswapd vs2, vs2
 ; CHECK-P9-NEXT:    xscvdpsxws f2, f2
-; CHECK-P9-NEXT:    xscvdpsxws f1, f1
-; CHECK-P9-NEXT:    xscvdpsxws f0, f0
-; CHECK-P9-NEXT:    xscvdpsxws f4, f4
-; CHECK-P9-NEXT:    xscvdpsxws f5, f5
-; CHECK-P9-NEXT:    xscvdpsxws f6, f6
-; CHECK-P9-NEXT:    xscvdpsxws f7, f7
 ; CHECK-P9-NEXT:    mfvsrwz r3, f3
-; CHECK-P9-NEXT:    mfvsrwz r5, f2
-; CHECK-P9-NEXT:    mfvsrwz r7, f1
-; CHECK-P9-NEXT:    mfvsrwz r9, f0
-; CHECK-P9-NEXT:    mtvsrd f0, r3
-; CHECK-P9-NEXT:    mtvsrd f2, r5
-; CHECK-P9-NEXT:    mfvsrwz r4, f4
-; CHECK-P9-NEXT:    mfvsrwz r6, f5
-; CHECK-P9-NEXT:    mfvsrwz r8, f6
-; CHECK-P9-NEXT:    mfvsrwz r10, f7
-; CHECK-P9-NEXT:    mtvsrd f4, r7
-; CHECK-P9-NEXT:    mtvsrd f6, r9
-; CHECK-P9-NEXT:    xxswapd v2, vs0
+; CHECK-P9-NEXT:    mtvsrd f3, r3
+; CHECK-P9-NEXT:    mfvsrwz r3, f2
+; CHECK-P9-NEXT:    mtvsrd f2, r3
 ; CHECK-P9-NEXT:    xxswapd v4, vs2
-; CHECK-P9-NEXT:    mtvsrd f1, r4
-; CHECK-P9-NEXT:    mtvsrd f3, r6
-; CHECK-P9-NEXT:    mtvsrd f5, r8
-; CHECK-P9-NEXT:    mtvsrd f7, r10
-; CHECK-P9-NEXT:    xxswapd v0, vs4
-; CHECK-P9-NEXT:    xxswapd v6, vs6
-; CHECK-P9-NEXT:    xxswapd v3, vs1
-; CHECK-P9-NEXT:    xxswapd v5, vs3
-; CHECK-P9-NEXT:    xxswapd v1, vs5
-; CHECK-P9-NEXT:    xxswapd v7, vs7
+; CHECK-P9-NEXT:    xscvdpsxws f2, f1
+; CHECK-P9-NEXT:    xxswapd vs1, vs1
+; CHECK-P9-NEXT:    xscvdpsxws f1, f1
+; CHECK-P9-NEXT:    mfvsrwz r3, f2
+; CHECK-P9-NEXT:    mtvsrd f2, r3
+; CHECK-P9-NEXT:    mfvsrwz r3, f1
 ; CHECK-P9-NEXT:    vmrglb v2, v2, v3
-; CHECK-P9-NEXT:    vmrglb v3, v4, v5
-; CHECK-P9-NEXT:    vmrglb v4, v0, v1
-; CHECK-P9-NEXT:    vmrglb v5, v6, v7
+; CHECK-P9-NEXT:    xxswapd v3, vs3
+; CHECK-P9-NEXT:    vmrglb v3, v3, v4
 ; CHECK-P9-NEXT:    vmrglh v2, v3, v2
-; CHECK-P9-NEXT:    vmrglh v3, v5, v4
+; CHECK-P9-NEXT:    xxswapd v3, vs2
+; CHECK-P9-NEXT:    mtvsrd f1, r3
+; CHECK-P9-NEXT:    xxswapd v4, vs1
+; CHECK-P9-NEXT:    xscvdpsxws f1, f0
+; CHECK-P9-NEXT:    xxswapd vs0, vs0
+; CHECK-P9-NEXT:    xscvdpsxws f0, f0
+; CHECK-P9-NEXT:    mfvsrwz r3, f1
+; CHECK-P9-NEXT:    mtvsrd f1, r3
+; CHECK-P9-NEXT:    mfvsrwz r3, f0
+; CHECK-P9-NEXT:    mtvsrd f0, r3
+; CHECK-P9-NEXT:    vmrglb v3, v3, v4
+; CHECK-P9-NEXT:    xxswapd v4, vs1
+; CHECK-P9-NEXT:    xxswapd v5, vs0
+; CHECK-P9-NEXT:    vmrglb v4, v4, v5
+; CHECK-P9-NEXT:    vmrglh v3, v4, v3
 ; CHECK-P9-NEXT:    vmrglw v2, v3, v2
 ; CHECK-P9-NEXT:    mfvsrld r3, v2
 ; CHECK-P9-NEXT:    blr
 ;
 ; CHECK-BE-LABEL: test8elt:
 ; CHECK-BE:       # %bb.0: # %entry
-; CHECK-BE-NEXT:    lxv vs0, 0(r3)
-; CHECK-BE-NEXT:    lxv vs1, 16(r3)
-; CHECK-BE-NEXT:    lxv vs2, 32(r3)
 ; CHECK-BE-NEXT:    lxv vs3, 48(r3)
-; CHECK-BE-NEXT:    xxswapd vs4, vs3
-; CHECK-BE-NEXT:    xxswapd vs5, vs2
-; CHECK-BE-NEXT:    xxswapd vs6, vs1
-; CHECK-BE-NEXT:    xxswapd vs7, vs0
+; CHECK-BE-NEXT:    xscvdpsxws f4, f3
+; CHECK-BE-NEXT:    xxswapd vs3, vs3
 ; CHECK-BE-NEXT:    xscvdpsxws f3, f3
-; CHECK-BE-NEXT:    xscvdpsxws f2, f2
-; CHECK-BE-NEXT:    xscvdpsxws f1, f1
-; CHECK-BE-NEXT:    xscvdpsxws f0, f0
-; CHECK-BE-NEXT:    xscvdpsxws f4, f4
-; CHECK-BE-NEXT:    xscvdpsxws f5, f5
-; CHECK-BE-NEXT:    xscvdpsxws f6, f6
-; CHECK-BE-NEXT:    xscvdpsxws f7, f7
-; CHECK-BE-NEXT:    mfvsrwz r3, f3
-; CHECK-BE-NEXT:    mfvsrwz r5, f2
-; CHECK-BE-NEXT:    mfvsrwz r7, f1
-; CHECK-BE-NEXT:    mfvsrwz r9, f0
+; CHECK-BE-NEXT:    lxv vs2, 32(r3)
+; CHECK-BE-NEXT:    lxv vs0, 0(r3)
+; CHECK-BE-NEXT:    lxv vs1, 16(r3)
+; CHECK-BE-NEXT:    mfvsrwz r3, f4
 ; CHECK-BE-NEXT:    sldi r3, r3, 56
-; CHECK-BE-NEXT:    sldi r5, r5, 56
-; CHECK-BE-NEXT:    sldi r7, r7, 56
-; CHECK-BE-NEXT:    sldi r9, r9, 56
-; CHECK-BE-NEXT:    mfvsrwz r4, f4
-; CHECK-BE-NEXT:    mfvsrwz r6, f5
-; CHECK-BE-NEXT:    mfvsrwz r8, f6
-; CHECK-BE-NEXT:    mfvsrwz r10, f7
 ; CHECK-BE-NEXT:    mtvsrd v2, r3
-; CHECK-BE-NEXT:    mtvsrd v4, r5
-; CHECK-BE-NEXT:    mtvsrd v0, r7
-; CHECK-BE-NEXT:    mtvsrd v6, r9
-; CHECK-BE-NEXT:    sldi r4, r4, 56
-; CHECK-BE-NEXT:    sldi r6, r6, 56
-; CHECK-BE-NEXT:    sldi r8, r8, 56
-; CHECK-BE-NEXT:    sldi r10, r10, 56
-; CHECK-BE-NEXT:    mtvsrd v3, r4
-; CHECK-BE-NEXT:    mtvsrd v5, r6
-; CHECK-BE-NEXT:    mtvsrd v1, r8
-; CHECK-BE-NEXT:    mtvsrd v7, r10
+; CHECK-BE-NEXT:    mfvsrwz r3, f3
+; CHECK-BE-NEXT:    xscvdpsxws f3, f2
+; CHECK-BE-NEXT:    xxswapd vs2, vs2
+; CHECK-BE-NEXT:    sldi r3, r3, 56
+; CHECK-BE-NEXT:    xscvdpsxws f2, f2
+; CHECK-BE-NEXT:    mtvsrd v3, r3
 ; CHECK-BE-NEXT:    vmrghb v2, v2, v3
-; CHECK-BE-NEXT:    vmrghb v3, v4, v5
-; CHECK-BE-NEXT:    vmrghb v4, v0, v1
-; CHECK-BE-NEXT:    vmrghb v5, v6, v7
+; CHECK-BE-NEXT:    mfvsrwz r3, f3
+; CHECK-BE-NEXT:    sldi r3, r3, 56
+; CHECK-BE-NEXT:    mtvsrd v3, r3
+; CHECK-BE-NEXT:    mfvsrwz r3, f2
+; CHECK-BE-NEXT:    xscvdpsxws f2, f1
+; CHECK-BE-NEXT:    xxswapd vs1, vs1
+; CHECK-BE-NEXT:    sldi r3, r3, 56
+; CHECK-BE-NEXT:    xscvdpsxws f1, f1
+; CHECK-BE-NEXT:    mtvsrd v4, r3
+; CHECK-BE-NEXT:    vmrghb v3, v3, v4
+; CHECK-BE-NEXT:    mfvsrwz r3, f2
+; CHECK-BE-NEXT:    sldi r3, r3, 56
 ; CHECK-BE-NEXT:    vmrghh v2, v3, v2
-; CHECK-BE-NEXT:    vmrghh v3, v5, v4
+; CHECK-BE-NEXT:    mtvsrd v3, r3
+; CHECK-BE-NEXT:    mfvsrwz r3, f1
+; CHECK-BE-NEXT:    xscvdpsxws f1, f0
+; CHECK-BE-NEXT:    xxswapd vs0, vs0
+; CHECK-BE-NEXT:    sldi r3, r3, 56
+; CHECK-BE-NEXT:    xscvdpsxws f0, f0
+; CHECK-BE-NEXT:    mtvsrd v4, r3
+; CHECK-BE-NEXT:    vmrghb v3, v3, v4
+; CHECK-BE-NEXT:    mfvsrwz r3, f1
+; CHECK-BE-NEXT:    sldi r3, r3, 56
+; CHECK-BE-NEXT:    mtvsrd v4, r3
+; CHECK-BE-NEXT:    mfvsrwz r3, f0
+; CHECK-BE-NEXT:    sldi r3, r3, 56
+; CHECK-BE-NEXT:    mtvsrd v5, r3
+; CHECK-BE-NEXT:    vmrghb v4, v4, v5
+; CHECK-BE-NEXT:    vmrghh v3, v4, v3
 ; CHECK-BE-NEXT:    vmrghw v2, v3, v2
 ; CHECK-BE-NEXT:    mfvsrd r3, v2
 ; CHECK-BE-NEXT:    blr
@@ -441,219 +441,199 @@ define <16 x i8> @test16elt(<16 x double>* nocapture readonly) local_unnamed_add
 ;
 ; CHECK-P9-LABEL: test16elt:
 ; CHECK-P9:       # %bb.0: # %entry
-; CHECK-P9-NEXT:    lxv vs2, 48(r3)
-; CHECK-P9-NEXT:    lxv vs3, 32(r3)
-; CHECK-P9-NEXT:    lxv vs4, 16(r3)
-; CHECK-P9-NEXT:    lxv vs5, 0(r3)
+; CHECK-P9-NEXT:    lxv vs7, 0(r3)
+; CHECK-P9-NEXT:    xscvdpsxws f8, f7
+; CHECK-P9-NEXT:    xxswapd vs7, vs7
+; CHECK-P9-NEXT:    xscvdpsxws f7, f7
 ; CHECK-P9-NEXT:    lxv vs0, 112(r3)
 ; CHECK-P9-NEXT:    lxv vs1, 96(r3)
-; CHECK-P9-NEXT:    lxv vs6, 80(r3)
-; CHECK-P9-NEXT:    lxv vs7, 64(r3)
-; CHECK-P9-NEXT:    std r26, -48(r1) # 8-byte Folded Spill
-; CHECK-P9-NEXT:    std r27, -40(r1) # 8-byte Folded Spill
-; CHECK-P9-NEXT:    std r28, -32(r1) # 8-byte Folded Spill
-; CHECK-P9-NEXT:    std r29, -24(r1) # 8-byte Folded Spill
-; CHECK-P9-NEXT:    std r30, -16(r1) # 8-byte Folded Spill
-; CHECK-P9-NEXT:    xxswapd vs8, vs5
-; CHECK-P9-NEXT:    xxswapd vs9, vs4
-; CHECK-P9-NEXT:    xxswapd vs10, vs3
-; CHECK-P9-NEXT:    xxswapd vs11, vs2
-; CHECK-P9-NEXT:    xxswapd vs12, vs7
-; CHECK-P9-NEXT:    xxswapd vs13, vs6
-; CHECK-P9-NEXT:    xxswapd v2, vs1
-; CHECK-P9-NEXT:    xxswapd v3, vs0
+; CHECK-P9-NEXT:    lxv vs2, 80(r3)
+; CHECK-P9-NEXT:    lxv vs3, 64(r3)
+; CHECK-P9-NEXT:    lxv vs4, 48(r3)
+; CHECK-P9-NEXT:    lxv vs5, 32(r3)
+; CHECK-P9-NEXT:    lxv vs6, 16(r3)
+; CHECK-P9-NEXT:    mfvsrwz r3, f8
+; CHECK-P9-NEXT:    mtvsrd f8, r3
+; CHECK-P9-NEXT:    mfvsrwz r3, f7
+; CHECK-P9-NEXT:    xxswapd v2, vs8
+; CHECK-P9-NEXT:    mtvsrd f7, r3
+; CHECK-P9-NEXT:    xxswapd v3, vs7
+; CHECK-P9-NEXT:    xscvdpsxws f7, f6
+; CHECK-P9-NEXT:    xxswapd vs6, vs6
+; CHECK-P9-NEXT:    xscvdpsxws f6, f6
+; CHECK-P9-NEXT:    mfvsrwz r3, f7
+; CHECK-P9-NEXT:    mtvsrd f7, r3
+; CHECK-P9-NEXT:    mfvsrwz r3, f6
+; CHECK-P9-NEXT:    mtvsrd f6, r3
+; CHECK-P9-NEXT:    xxswapd v4, vs6
+; CHECK-P9-NEXT:    xscvdpsxws f6, f5
+; CHECK-P9-NEXT:    xxswapd vs5, vs5
 ; CHECK-P9-NEXT:    xscvdpsxws f5, f5
+; CHECK-P9-NEXT:    mfvsrwz r3, f6
+; CHECK-P9-NEXT:    mtvsrd f6, r3
+; CHECK-P9-NEXT:    mfvsrwz r3, f5
+; CHECK-P9-NEXT:    vmrglb v2, v2, v3
+; CHECK-P9-NEXT:    xxswapd v3, vs7
+; CHECK-P9-NEXT:    vmrglb v3, v3, v4
+; CHECK-P9-NEXT:    vmrglh v2, v3, v2
+; CHECK-P9-NEXT:    xxswapd v3, vs6
+; CHECK-P9-NEXT:    mtvsrd f5, r3
+; CHECK-P9-NEXT:    xxswapd v4, vs5
+; CHECK-P9-NEXT:    xscvdpsxws f5, f4
+; CHECK-P9-NEXT:    xxswapd vs4, vs4
 ; CHECK-P9-NEXT:    xscvdpsxws f4, f4
+; CHECK-P9-NEXT:    mfvsrwz r3, f5
+; CHECK-P9-NEXT:    mtvsrd f5, r3
+; CHECK-P9-NEXT:    mfvsrwz r3, f4
+; CHECK-P9-NEXT:    mtvsrd f4, r3
+; CHECK-P9-NEXT:    xxswapd v5, vs4
+; CHECK-P9-NEXT:    xscvdpsxws f4, f3
+; CHECK-P9-NEXT:    xxswapd vs3, vs3
 ; CHECK-P9-NEXT:    xscvdpsxws f3, f3
+; CHECK-P9-NEXT:    vmrglb v3, v3, v4
+; CHECK-P9-NEXT:    xxswapd v4, vs5
+; CHECK-P9-NEXT:    vmrglb v4, v4, v5
+; CHECK-P9-NEXT:    vmrglh v3, v4, v3
+; CHECK-P9-NEXT:    mfvsrwz r3, f4
+; CHECK-P9-NEXT:    mtvsrd f4, r3
+; CHECK-P9-NEXT:    mfvsrwz r3, f3
+; CHECK-P9-NEXT:    mtvsrd f3, r3
+; CHECK-P9-NEXT:    xxswapd v4, vs3
+; CHECK-P9-NEXT:    xscvdpsxws f3, f2
+; CHECK-P9-NEXT:    xxswapd vs2, vs2
 ; CHECK-P9-NEXT:    xscvdpsxws f2, f2
-; CHECK-P9-NEXT:    xscvdpsxws f7, f7
-; CHECK-P9-NEXT:    xscvdpsxws f6, f6
+; CHECK-P9-NEXT:    mfvsrwz r3, f3
+; CHECK-P9-NEXT:    mtvsrd f3, r3
+; CHECK-P9-NEXT:    mfvsrwz r3, f2
+; CHECK-P9-NEXT:    mtvsrd f2, r3
+; CHECK-P9-NEXT:    xxswapd v5, vs2
+; CHECK-P9-NEXT:    xscvdpsxws f2, f1
+; CHECK-P9-NEXT:    xxswapd vs1, vs1
 ; CHECK-P9-NEXT:    xscvdpsxws f1, f1
+; CHECK-P9-NEXT:    vmrglw v2, v3, v2
+; CHECK-P9-NEXT:    xxswapd v3, vs4
+; CHECK-P9-NEXT:    vmrglb v3, v3, v4
+; CHECK-P9-NEXT:    xxswapd v4, vs3
+; CHECK-P9-NEXT:    vmrglb v4, v4, v5
+; CHECK-P9-NEXT:    vmrglh v3, v4, v3
+; CHECK-P9-NEXT:    mfvsrwz r3, f2
+; CHECK-P9-NEXT:    mtvsrd f2, r3
+; CHECK-P9-NEXT:    mfvsrwz r3, f1
+; CHECK-P9-NEXT:    xxswapd v4, vs2
+; CHECK-P9-NEXT:    mtvsrd f1, r3
+; CHECK-P9-NEXT:    xxswapd v5, vs1
+; CHECK-P9-NEXT:    xscvdpsxws f1, f0
+; CHECK-P9-NEXT:    xxswapd vs0, vs0
 ; CHECK-P9-NEXT:    xscvdpsxws f0, f0
-; CHECK-P9-NEXT:    xscvdpsxws f8, f8
-; CHECK-P9-NEXT:    xscvdpsxws f9, f9
-; CHECK-P9-NEXT:    xscvdpsxws f10, f10
-; CHECK-P9-NEXT:    xscvdpsxws f11, f11
-; CHECK-P9-NEXT:    xscvdpsxws f12, f12
-; CHECK-P9-NEXT:    xscvdpsxws f13, f13
-; CHECK-P9-NEXT:    xscvdpsxws v2, v2
-; CHECK-P9-NEXT:    xscvdpsxws v3, v3
-; CHECK-P9-NEXT:    mfvsrwz r3, f5
-; CHECK-P9-NEXT:    mfvsrwz r4, f4
-; CHECK-P9-NEXT:    mfvsrwz r5, f3
-; CHECK-P9-NEXT:    mfvsrwz r6, f2
-; CHECK-P9-NEXT:    mfvsrwz r11, f7
-; CHECK-P9-NEXT:    mfvsrwz r12, f6
-; CHECK-P9-NEXT:    mfvsrwz r0, f1
-; CHECK-P9-NEXT:    mfvsrwz r30, f0
-; CHECK-P9-NEXT:    mfvsrwz r7, f8
-; CHECK-P9-NEXT:    mfvsrwz r8, f9
-; CHECK-P9-NEXT:    mfvsrwz r9, f10
-; CHECK-P9-NEXT:    mfvsrwz r10, f11
-; CHECK-P9-NEXT:    mfvsrwz r29, f12
-; CHECK-P9-NEXT:    mfvsrwz r28, f13
-; CHECK-P9-NEXT:    mfvsrwz r27, v2
-; CHECK-P9-NEXT:    mfvsrwz r26, v3
+; CHECK-P9-NEXT:    mfvsrwz r3, f1
+; CHECK-P9-NEXT:    mtvsrd f1, r3
+; CHECK-P9-NEXT:    mfvsrwz r3, f0
 ; CHECK-P9-NEXT:    mtvsrd f0, r3
-; CHECK-P9-NEXT:    mtvsrd f1, r4
-; CHECK-P9-NEXT:    mtvsrd f2, r5
-; CHECK-P9-NEXT:    mtvsrd f3, r6
-; CHECK-P9-NEXT:    mtvsrd f8, r11
-; CHECK-P9-NEXT:    mtvsrd f9, r12
-; CHECK-P9-NEXT:    mtvsrd f10, r0
-; CHECK-P9-NEXT:    mtvsrd f11, r30
-; CHECK-P9-NEXT:    mtvsrd f4, r7
-; CHECK-P9-NEXT:    mtvsrd f5, r8
-; CHECK-P9-NEXT:    mtvsrd f6, r9
-; CHECK-P9-NEXT:    mtvsrd f7, r10
-; CHECK-P9-NEXT:    mtvsrd f12, r29
-; CHECK-P9-NEXT:    mtvsrd f13, r28
-; CHECK-P9-NEXT:    mtvsrd v2, r27
-; CHECK-P9-NEXT:    mtvsrd v3, r26
-; CHECK-P9-NEXT:    xxswapd v4, vs0
+; CHECK-P9-NEXT:    vmrglb v4, v4, v5
 ; CHECK-P9-NEXT:    xxswapd v5, vs1
-; CHECK-P9-NEXT:    xxswapd v0, vs2
-; CHECK-P9-NEXT:    xxswapd v1, vs3
-; CHECK-P9-NEXT:    ld r30, -16(r1) # 8-byte Folded Reload
-; CHECK-P9-NEXT:    ld r29, -24(r1) # 8-byte Folded Reload
-; CHECK-P9-NEXT:    ld r28, -32(r1) # 8-byte Folded Reload
-; CHECK-P9-NEXT:    ld r27, -40(r1) # 8-byte Folded Reload
-; CHECK-P9-NEXT:    xxswapd v6, vs4
-; CHECK-P9-NEXT:    xxswapd v7, vs5
-; CHECK-P9-NEXT:    ld r26, -48(r1) # 8-byte Folded Reload
-; CHECK-P9-NEXT:    xxswapd v8, vs6
-; CHECK-P9-NEXT:    xxswapd v9, vs7
-; CHECK-P9-NEXT:    xxswapd v10, vs8
-; CHECK-P9-NEXT:    xxswapd v11, vs12
-; CHECK-P9-NEXT:    xxswapd v12, vs9
-; CHECK-P9-NEXT:    xxswapd v13, vs13
-; CHECK-P9-NEXT:    xxswapd v14, vs10
-; CHECK-P9-NEXT:    xxswapd v2, v2
-; CHECK-P9-NEXT:    xxswapd v15, vs11
-; CHECK-P9-NEXT:    xxswapd v3, v3
-; CHECK-P9-NEXT:    vmrglb v4, v4, v6
-; CHECK-P9-NEXT:    vmrglb v5, v5, v7
-; CHECK-P9-NEXT:    vmrglb v0, v0, v8
-; CHECK-P9-NEXT:    vmrglb v1, v1, v9
-; CHECK-P9-NEXT:    vmrglb v6, v10, v11
-; CHECK-P9-NEXT:    vmrglb v7, v12, v13
-; CHECK-P9-NEXT:    vmrglb v2, v14, v2
-; CHECK-P9-NEXT:    vmrglb v3, v15, v3
+; CHECK-P9-NEXT:    xxswapd v0, vs0
+; CHECK-P9-NEXT:    vmrglb v5, v5, v0
 ; CHECK-P9-NEXT:    vmrglh v4, v5, v4
-; CHECK-P9-NEXT:    vmrglh v5, v1, v0
-; CHECK-P9-NEXT:    vmrglh v0, v7, v6
-; CHECK-P9-NEXT:    vmrglh v2, v3, v2
-; CHECK-P9-NEXT:    vmrglw v3, v5, v4
-; CHECK-P9-NEXT:    vmrglw v2, v2, v0
-; CHECK-P9-NEXT:    xxmrgld v2, v2, v3
+; CHECK-P9-NEXT:    vmrglw v3, v4, v3
+; CHECK-P9-NEXT:    xxmrgld v2, v3, v2
 ; CHECK-P9-NEXT:    blr
 ;
 ; CHECK-BE-LABEL: test16elt:
 ; CHECK-BE:       # %bb.0: # %entry
-; CHECK-BE-NEXT:    lxv vs2, 64(r3)
-; CHECK-BE-NEXT:    lxv vs3, 80(r3)
-; CHECK-BE-NEXT:    lxv vs4, 96(r3)
-; CHECK-BE-NEXT:    lxv vs5, 112(r3)
+; CHECK-BE-NEXT:    lxv vs7, 112(r3)
+; CHECK-BE-NEXT:    xscvdpsxws f8, f7
+; CHECK-BE-NEXT:    xxswapd vs7, vs7
+; CHECK-BE-NEXT:    xscvdpsxws f7, f7
+; CHECK-BE-NEXT:    lxv vs6, 96(r3)
 ; CHECK-BE-NEXT:    lxv vs0, 0(r3)
 ; CHECK-BE-NEXT:    lxv vs1, 16(r3)
-; CHECK-BE-NEXT:    lxv vs6, 32(r3)
-; CHECK-BE-NEXT:    lxv vs7, 48(r3)
-; CHECK-BE-NEXT:    std r26, -48(r1) # 8-byte Folded Spill
-; CHECK-BE-NEXT:    std r27, -40(r1) # 8-byte Folded Spill
-; CHECK-BE-NEXT:    std r28, -32(r1) # 8-byte Folded Spill
-; CHECK-BE-NEXT:    std r29, -24(r1) # 8-byte Folded Spill
-; CHECK-BE-NEXT:    std r30, -16(r1) # 8-byte Folded Spill
-; CHECK-BE-NEXT:    xxswapd vs8, vs5
-; CHECK-BE-NEXT:    xxswapd vs9, vs4
-; CHECK-BE-NEXT:    xxswapd vs10, vs3
-; CHECK-BE-NEXT:    xxswapd vs11, vs2
-; CHECK-BE-NEXT:    xxswapd vs12, vs7
-; CHECK-BE-NEXT:    xxswapd vs13, vs6
-; CHECK-BE-NEXT:    xxswapd v2, vs1
-; CHECK-BE-NEXT:    xxswapd v3, vs0
+; CHECK-BE-NEXT:    lxv vs2, 32(r3)
+; CHECK-BE-NEXT:    lxv vs3, 48(r3)
+; CHECK-BE-NEXT:    lxv vs4, 64(r3)
+; CHECK-BE-NEXT:    lxv vs5, 80(r3)
+; CHECK-BE-NEXT:    mfvsrwz r3, f8
+; CHECK-BE-NEXT:    sldi r3, r3, 56
+; CHECK-BE-NEXT:    mtvsrd v2, r3
+; CHECK-BE-NEXT:    mfvsrwz r3, f7
+; CHECK-BE-NEXT:    xscvdpsxws f7, f6
+; CHECK-BE-NEXT:    xxswapd vs6, vs6
+; CHECK-BE-NEXT:    sldi r3, r3, 56
+; CHECK-BE-NEXT:    xscvdpsxws f6, f6
+; CHECK-BE-NEXT:    mtvsrd v3, r3
+; CHECK-BE-NEXT:    vmrghb v2, v2, v3
+; CHECK-BE-NEXT:    mfvsrwz r3, f7
+; CHECK-BE-NEXT:    sldi r3, r3, 56
+; CHECK-BE-NEXT:    mtvsrd v3, r3
+; CHECK-BE-NEXT:    mfvsrwz r3, f6
+; CHECK-BE-NEXT:    xscvdpsxws f6, f5
+; CHECK-BE-NEXT:    xxswapd vs5, vs5
+; CHECK-BE-NEXT:    sldi r3, r3, 56
 ; CHECK-BE-NEXT:    xscvdpsxws f5, f5
+; CHECK-BE-NEXT:    mtvsrd v4, r3
+; CHECK-BE-NEXT:    vmrghb v3, v3, v4
+; CHECK-BE-NEXT:    mfvsrwz r3, f6
+; CHECK-BE-NEXT:    sldi r3, r3, 56
+; CHECK-BE-NEXT:    vmrghh v2, v3, v2
+; CHECK-BE-NEXT:    mtvsrd v3, r3
+; CHECK-BE-NEXT:    mfvsrwz r3, f5
+; CHECK-BE-NEXT:    xscvdpsxws f5, f4
+; CHECK-BE-NEXT:    xxswapd vs4, vs4
+; CHECK-BE-NEXT:    sldi r3, r3, 56
 ; CHECK-BE-NEXT:    xscvdpsxws f4, f4
+; CHECK-BE-NEXT:    mtvsrd v4, r3
+; CHECK-BE-NEXT:    vmrghb v3, v3, v4
+; CHECK-BE-NEXT:    mfvsrwz r3, f5
+; CHECK-BE-NEXT:    sldi r3, r3, 56
+; CHECK-BE-NEXT:    mtvsrd v4, r3
+; CHECK-BE-NEXT:    mfvsrwz r3, f4
+; CHECK-BE-NEXT:    xscvdpsxws f4, f3
+; CHECK-BE-NEXT:    xxswapd vs3, vs3
+; CHECK-BE-NEXT:    sldi r3, r3, 56
 ; CHECK-BE-NEXT:    xscvdpsxws f3, f3
+; CHECK-BE-NEXT:    mtvsrd v5, r3
+; CHECK-BE-NEXT:    vmrghb v4, v4, v5
+; CHECK-BE-NEXT:    mfvsrwz r3, f4
+; CHECK-BE-NEXT:    vmrghh v3, v4, v3
+; CHECK-BE-NEXT:    sldi r3, r3, 56
+; CHECK-BE-NEXT:    vmrghw v2, v3, v2
+; CHECK-BE-NEXT:    mtvsrd v3, r3
+; CHECK-BE-NEXT:    mfvsrwz r3, f3
+; CHECK-BE-NEXT:    xscvdpsxws f3, f2
+; CHECK-BE-NEXT:    xxswapd vs2, vs2
+; CHECK-BE-NEXT:    sldi r3, r3, 56
 ; CHECK-BE-NEXT:    xscvdpsxws f2, f2
-; CHECK-BE-NEXT:    xscvdpsxws f7, f7
-; CHECK-BE-NEXT:    xscvdpsxws f6, f6
+; CHECK-BE-NEXT:    mtvsrd v4, r3
+; CHECK-BE-NEXT:    vmrghb v3, v3, v4
+; CHECK-BE-NEXT:    mfvsrwz r3, f3
+; CHECK-BE-NEXT:    sldi r3, r3, 56
+; CHECK-BE-NEXT:    mtvsrd v4, r3
+; CHECK-BE-NEXT:    mfvsrwz r3, f2
+; CHECK-BE-NEXT:    xscvdpsxws f2, f1
+; CHECK-BE-NEXT:    xxswapd vs1, vs1
+; CHECK-BE-NEXT:    sldi r3, r3, 56
 ; CHECK-BE-NEXT:    xscvdpsxws f1, f1
+; CHECK-BE-NEXT:    mtvsrd v5, r3
+; CHECK-BE-NEXT:    vmrghb v4, v4, v5
+; CHECK-BE-NEXT:    mfvsrwz r3, f2
+; CHECK-BE-NEXT:    sldi r3, r3, 56
+; CHECK-BE-NEXT:    vmrghh v3, v4, v3
+; CHECK-BE-NEXT:    mtvsrd v4, r3
+; CHECK-BE-NEXT:    mfvsrwz r3, f1
+; CHECK-BE-NEXT:    xscvdpsxws f1, f0
+; CHECK-BE-NEXT:    xxswapd vs0, vs0
+; CHECK-BE-NEXT:    sldi r3, r3, 56
 ; CHECK-BE-NEXT:    xscvdpsxws f0, f0
-; CHECK-BE-NEXT:    xscvdpsxws f8, f8
-; CHECK-BE-NEXT:    xscvdpsxws f9, f9
-; CHECK-BE-NEXT:    xscvdpsxws f10, f10
-; CHECK-BE-NEXT:    xscvdpsxws f11, f11
-; CHECK-BE-NEXT:    xscvdpsxws f12, f12
-; CHECK-BE-NEXT:    xscvdpsxws f13, f13
-; CHECK-BE-NEXT:    xscvdpsxws v2, v2
-; CHECK-BE-NEXT:    xscvdpsxws v3, v3
-; CHECK-BE-NEXT:    mfvsrwz r3, f5
-; CHECK-BE-NEXT:    mfvsrwz r4, f4
-; CHECK-BE-NEXT:    mfvsrwz r5, f3
-; CHECK-BE-NEXT:    mfvsrwz r6, f2
-; CHECK-BE-NEXT:    mfvsrwz r11, f7
-; CHECK-BE-NEXT:    mfvsrwz r12, f6
-; CHECK-BE-NEXT:    mfvsrwz r0, f1
-; CHECK-BE-NEXT:    mfvsrwz r30, f0
-; CHECK-BE-NEXT:    mfvsrwz r7, f8
-; CHECK-BE-NEXT:    mfvsrwz r8, f9
-; CHECK-BE-NEXT:    mfvsrwz r9, f10
-; CHECK-BE-NEXT:    mfvsrwz r10, f11
-; CHECK-BE-NEXT:    mfvsrwz r29, f12
-; CHECK-BE-NEXT:    mfvsrwz r28, f13
-; CHECK-BE-NEXT:    mfvsrwz r27, v2
-; CHECK-BE-NEXT:    mfvsrwz r26, v3
+; CHECK-BE-NEXT:    mtvsrd v5, r3
+; CHECK-BE-NEXT:    vmrghb v4, v4, v5
+; CHECK-BE-NEXT:    mfvsrwz r3, f1
 ; CHECK-BE-NEXT:    sldi r3, r3, 56
-; CHECK-BE-NEXT:    sldi r4, r4, 56
-; CHECK-BE-NEXT:    sldi r5, r5, 56
-; CHECK-BE-NEXT:    sldi r6, r6, 56
-; CHECK-BE-NEXT:    sldi r11, r11, 56
-; CHECK-BE-NEXT:    sldi r12, r12, 56
-; CHECK-BE-NEXT:    sldi r0, r0, 56
-; CHECK-BE-NEXT:    sldi r30, r30, 56
-; CHECK-BE-NEXT:    sldi r7, r7, 56
-; CHECK-BE-NEXT:    sldi r8, r8, 56
-; CHECK-BE-NEXT:    sldi r9, r9, 56
-; CHECK-BE-NEXT:    sldi r10, r10, 56
-; CHECK-BE-NEXT:    sldi r29, r29, 56
-; CHECK-BE-NEXT:    sldi r28, r28, 56
-; CHECK-BE-NEXT:    sldi r27, r27, 56
-; CHECK-BE-NEXT:    sldi r26, r26, 56
-; CHECK-BE-NEXT:    mtvsrd v2, r3
-; CHECK-BE-NEXT:    mtvsrd v3, r4
-; CHECK-BE-NEXT:    mtvsrd v4, r5
-; CHECK-BE-NEXT:    mtvsrd v5, r6
-; CHECK-BE-NEXT:    mtvsrd v8, r11
-; CHECK-BE-NEXT:    mtvsrd v10, r12
-; CHECK-BE-NEXT:    mtvsrd v12, r0
-; CHECK-BE-NEXT:    mtvsrd v14, r30
-; CHECK-BE-NEXT:    mtvsrd v0, r7
-; CHECK-BE-NEXT:    mtvsrd v1, r8
-; CHECK-BE-NEXT:    mtvsrd v6, r9
-; CHECK-BE-NEXT:    mtvsrd v7, r10
-; CHECK-BE-NEXT:    ld r30, -16(r1) # 8-byte Folded Reload
-; CHECK-BE-NEXT:    mtvsrd v9, r29
-; CHECK-BE-NEXT:    ld r29, -24(r1) # 8-byte Folded Reload
-; CHECK-BE-NEXT:    mtvsrd v11, r28
-; CHECK-BE-NEXT:    mtvsrd v13, r27
-; CHECK-BE-NEXT:    mtvsrd v15, r26
-; CHECK-BE-NEXT:    ld r28, -32(r1) # 8-byte Folded Reload
-; CHECK-BE-NEXT:    ld r27, -40(r1) # 8-byte Folded Reload
-; CHECK-BE-NEXT:    ld r26, -48(r1) # 8-byte Folded Reload
-; CHECK-BE-NEXT:    vmrghb v2, v2, v0
-; CHECK-BE-NEXT:    vmrghb v3, v3, v1
-; CHECK-BE-NEXT:    vmrghb v4, v4, v6
-; CHECK-BE-NEXT:    vmrghb v5, v5, v7
-; CHECK-BE-NEXT:    vmrghb v0, v8, v9
-; CHECK-BE-NEXT:    vmrghb v1, v10, v11
-; CHECK-BE-NEXT:    vmrghb v6, v12, v13
-; CHECK-BE-NEXT:    vmrghb v7, v14, v15
-; CHECK-BE-NEXT:    vmrghh v2, v3, v2
-; CHECK-BE-NEXT:    vmrghh v3, v5, v4
-; CHECK-BE-NEXT:    vmrghh v4, v1, v0
-; CHECK-BE-NEXT:    vmrghh v5, v7, v6
-; CHECK-BE-NEXT:    vmrghw v2, v3, v2
-; CHECK-BE-NEXT:    vmrghw v3, v5, v4
+; CHECK-BE-NEXT:    mtvsrd v5, r3
+; CHECK-BE-NEXT:    mfvsrwz r3, f0
+; CHECK-BE-NEXT:    sldi r3, r3, 56
+; CHECK-BE-NEXT:    mtvsrd v0, r3
+; CHECK-BE-NEXT:    vmrghb v5, v5, v0
+; CHECK-BE-NEXT:    vmrghh v4, v5, v4
+; CHECK-BE-NEXT:    vmrghw v3, v4, v3
 ; CHECK-BE-NEXT:    xxmrghd v2, v3, v2
 ; CHECK-BE-NEXT:    blr
 entry:
@@ -684,17 +664,17 @@ define i16 @test2elt_signed(<2 x double> %a) local_unnamed_addr #0 {
 ;
 ; CHECK-P9-LABEL: test2elt_signed:
 ; CHECK-P9:       # %bb.0: # %entry
+; CHECK-P9-NEXT:    xscvdpsxws f0, v2
+; CHECK-P9-NEXT:    mfvsrwz r3, f0
+; CHECK-P9-NEXT:    mtvsrd f0, r3
+; CHECK-P9-NEXT:    xxswapd v3, vs0
 ; CHECK-P9-NEXT:    xxswapd vs0, v2
-; CHECK-P9-NEXT:    xscvdpsxws f1, v2
 ; CHECK-P9-NEXT:    xscvdpsxws f0, f0
-; CHECK-P9-NEXT:    mfvsrwz r3, f1
-; CHECK-P9-NEXT:    mfvsrwz r4, f0
+; CHECK-P9-NEXT:    mfvsrwz r3, f0
 ; CHECK-P9-NEXT:    mtvsrd f0, r3
 ; CHECK-P9-NEXT:    addi r3, r1, -2
-; CHECK-P9-NEXT:    mtvsrd f1, r4
 ; CHECK-P9-NEXT:    xxswapd v2, vs0
-; CHECK-P9-NEXT:    xxswapd v3, vs1
-; CHECK-P9-NEXT:    vmrglb v2, v2, v3
+; CHECK-P9-NEXT:    vmrglb v2, v3, v2
 ; CHECK-P9-NEXT:    vsldoi v2, v2, v2, 8
 ; CHECK-P9-NEXT:    stxsihx v2, 0, r3
 ; CHECK-P9-NEXT:    lhz r3, -2(r1)
@@ -702,17 +682,17 @@ define i16 @test2elt_signed(<2 x double> %a) local_unnamed_addr #0 {
 ;
 ; CHECK-BE-LABEL: test2elt_signed:
 ; CHECK-BE:       # %bb.0: # %entry
+; CHECK-BE-NEXT:    xscvdpsxws f0, v2
+; CHECK-BE-NEXT:    mfvsrwz r3, f0
 ; CHECK-BE-NEXT:    xxswapd vs0, v2
-; CHECK-BE-NEXT:    xscvdpsxws f1, v2
+; CHECK-BE-NEXT:    sldi r3, r3, 56
 ; CHECK-BE-NEXT:    xscvdpsxws f0, f0
-; CHECK-BE-NEXT:    mfvsrwz r3, f1
+; CHECK-BE-NEXT:    mtvsrd v3, r3
+; CHECK-BE-NEXT:    mfvsrwz r3, f0
 ; CHECK-BE-NEXT:    sldi r3, r3, 56
-; CHECK-BE-NEXT:    mfvsrwz r4, f0
 ; CHECK-BE-NEXT:    mtvsrd v2, r3
 ; CHECK-BE-NEXT:    addi r3, r1, -2
-; CHECK-BE-NEXT:    sldi r4, r4, 56
-; CHECK-BE-NEXT:    mtvsrd v3, r4
-; CHECK-BE-NEXT:    vmrghb v2, v2, v3
+; CHECK-BE-NEXT:    vmrghb v2, v3, v2
 ; CHECK-BE-NEXT:    vsldoi v2, v2, v2, 10
 ; CHECK-BE-NEXT:    stxsihx v2, 0, r3
 ; CHECK-BE-NEXT:    lhz r3, -2(r1)
@@ -756,58 +736,58 @@ define i32 @test4elt_signed(<4 x double>* nocapture readonly) local_unnamed_addr
 ;
 ; CHECK-P9-LABEL: test4elt_signed:
 ; CHECK-P9:       # %bb.0: # %entry
-; CHECK-P9-NEXT:    lxv vs0, 16(r3)
 ; CHECK-P9-NEXT:    lxv vs1, 0(r3)
-; CHECK-P9-NEXT:    xxswapd vs2, vs1
-; CHECK-P9-NEXT:    xxswapd vs3, vs0
+; CHECK-P9-NEXT:    xscvdpsxws f2, f1
+; CHECK-P9-NEXT:    xxswapd vs1, vs1
 ; CHECK-P9-NEXT:    xscvdpsxws f1, f1
+; CHECK-P9-NEXT:    lxv vs0, 16(r3)
+; CHECK-P9-NEXT:    mfvsrwz r3, f2
+; CHECK-P9-NEXT:    mtvsrd f2, r3
+; CHECK-P9-NEXT:    mfvsrwz r3, f1
+; CHECK-P9-NEXT:    xxswapd v2, vs2
+; CHECK-P9-NEXT:    mtvsrd f1, r3
+; CHECK-P9-NEXT:    xxswapd v3, vs1
+; CHECK-P9-NEXT:    xscvdpsxws f1, f0
+; CHECK-P9-NEXT:    xxswapd vs0, vs0
 ; CHECK-P9-NEXT:    xscvdpsxws f0, f0
-; CHECK-P9-NEXT:    xscvdpsxws f2, f2
-; CHECK-P9-NEXT:    xscvdpsxws f3, f3
 ; CHECK-P9-NEXT:    mfvsrwz r3, f1
-; CHECK-P9-NEXT:    mfvsrwz r5, f0
+; CHECK-P9-NEXT:    mtvsrd f1, r3
+; CHECK-P9-NEXT:    mfvsrwz r3, f0
 ; CHECK-P9-NEXT:    mtvsrd f0, r3
-; CHECK-P9-NEXT:    li r3, 0
-; CHECK-P9-NEXT:    mfvsrwz r4, f2
-; CHECK-P9-NEXT:    mfvsrwz r6, f3
-; CHECK-P9-NEXT:    mtvsrd f2, r5
-; CHECK-P9-NEXT:    xxswapd v2, vs0
-; CHECK-P9-NEXT:    mtvsrd f1, r4
-; CHECK-P9-NEXT:    mtvsrd f3, r6
-; CHECK-P9-NEXT:    xxswapd v4, vs2
-; CHECK-P9-NEXT:    xxswapd v3, vs1
-; CHECK-P9-NEXT:    xxswapd v5, vs3
 ; CHECK-P9-NEXT:    vmrglb v2, v2, v3
-; CHECK-P9-NEXT:    vmrglb v3, v4, v5
+; CHECK-P9-NEXT:    xxswapd v3, vs1
+; CHECK-P9-NEXT:    xxswapd v4, vs0
+; CHECK-P9-NEXT:    vmrglb v3, v3, v4
 ; CHECK-P9-NEXT:    vmrglh v2, v3, v2
+; CHECK-P9-NEXT:    li r3, 0
 ; CHECK-P9-NEXT:    vextuwrx r3, r3, v2
 ; CHECK-P9-NEXT:    blr
 ;
 ; CHECK-BE-LABEL: test4elt_signed:
 ; CHECK-BE:       # %bb.0: # %entry
-; CHECK-BE-NEXT:    lxv vs0, 0(r3)
 ; CHECK-BE-NEXT:    lxv vs1, 16(r3)
-; CHECK-BE-NEXT:    xxswapd vs2, vs1
-; CHECK-BE-NEXT:    xxswapd vs3, vs0
+; CHECK-BE-NEXT:    xscvdpsxws f2, f1
+; CHECK-BE-NEXT:    xxswapd vs1, vs1
 ; CHECK-BE-NEXT:    xscvdpsxws f1, f1
+; CHECK-BE-NEXT:    lxv vs0, 0(r3)
+; CHECK-BE-NEXT:    mfvsrwz r3, f2
+; CHECK-BE-NEXT:    sldi r3, r3, 56
+; CHECK-BE-NEXT:    mtvsrd v2, r3
+; CHECK-BE-NEXT:    mfvsrwz r3, f1
+; CHECK-BE-NEXT:    xscvdpsxws f1, f0
+; CHECK-BE-NEXT:    xxswapd vs0, vs0
+; CHECK-BE-NEXT:    sldi r3, r3, 56
 ; CHECK-BE-NEXT:    xscvdpsxws f0, f0
-; CHECK-BE-NEXT:    xscvdpsxws f2, f2
-; CHECK-BE-NEXT:    xscvdpsxws f3, f3
+; CHECK-BE-NEXT:    mtvsrd v3, r3
+; CHECK-BE-NEXT:    vmrghb v2, v2, v3
 ; CHECK-BE-NEXT:    mfvsrwz r3, f1
-; CHECK-BE-NEXT:    mfvsrwz r5, f0
 ; CHECK-BE-NEXT:    sldi r3, r3, 56
-; CHECK-BE-NEXT:    sldi r5, r5, 56
-; CHECK-BE-NEXT:    mfvsrwz r4, f2
-; CHECK-BE-NEXT:    mfvsrwz r6, f3
-; CHECK-BE-NEXT:    mtvsrd v2, r3
-; CHECK-BE-NEXT:    mtvsrd v4, r5
+; CHECK-BE-NEXT:    mtvsrd v3, r3
+; CHECK-BE-NEXT:    mfvsrwz r3, f0
+; CHECK-BE-NEXT:    sldi r3, r3, 56
+; CHECK-BE-NEXT:    mtvsrd v4, r3
 ; CHECK-BE-NEXT:    li r3, 0
-; CHECK-BE-NEXT:    sldi r4, r4, 56
-; CHECK-BE-NEXT:    sldi r6, r6, 56
-; CHECK-BE-NEXT:    mtvsrd v3, r4
-; CHECK-BE-NEXT:    mtvsrd v5, r6
-; CHECK-BE-NEXT:    vmrghb v2, v2, v3
-; CHECK-BE-NEXT:    vmrghb v3, v4, v5
+; CHECK-BE-NEXT:    vmrghb v3, v3, v4
 ; CHECK-BE-NEXT:    vmrghh v2, v3, v2
 ; CHECK-BE-NEXT:    vextuwlx r3, r3, v2
 ; CHECK-BE-NEXT:    blr
@@ -877,104 +857,104 @@ define i64 @test8elt_signed(<8 x double>* nocapture readonly) local_unnamed_addr
 ;
 ; CHECK-P9-LABEL: test8elt_signed:
 ; CHECK-P9:       # %bb.0: # %entry
+; CHECK-P9-NEXT:    lxv vs3, 0(r3)
+; CHECK-P9-NEXT:    xscvdpsxws f4, f3
+; CHECK-P9-NEXT:    xxswapd vs3, vs3
+; CHECK-P9-NEXT:    xscvdpsxws f3, f3
 ; CHECK-P9-NEXT:    lxv vs0, 48(r3)
 ; CHECK-P9-NEXT:    lxv vs1, 32(r3)
 ; CHECK-P9-NEXT:    lxv vs2, 16(r3)
-; CHECK-P9-NEXT:    lxv vs3, 0(r3)
-; CHECK-P9-NEXT:    xxswapd vs4, vs3
-; CHECK-P9-NEXT:    xxswapd vs5, vs2
-; CHECK-P9-NEXT:    xxswapd vs6, vs1
-; CHECK-P9-NEXT:    xxswapd vs7, vs0
-; CHECK-P9-NEXT:    xscvdpsxws f3, f3
+; CHECK-P9-NEXT:    mfvsrwz r3, f4
+; CHECK-P9-NEXT:    mtvsrd f4, r3
+; CHECK-P9-NEXT:    mfvsrwz r3, f3
+; CHECK-P9-NEXT:    xxswapd v2, vs4
+; CHECK-P9-NEXT:    mtvsrd f3, r3
+; CHECK-P9-NEXT:    xxswapd v3, vs3
+; CHECK-P9-NEXT:    xscvdpsxws f3, f2
+; CHECK-P9-NEXT:    xxswapd vs2, vs2
 ; CHECK-P9-NEXT:    xscvdpsxws f2, f2
-; CHECK-P9-NEXT:    xscvdpsxws f1, f1
-; CHECK-P9-NEXT:    xscvdpsxws f0, f0
-; CHECK-P9-NEXT:    xscvdpsxws f4, f4
-; CHECK-P9-NEXT:    xscvdpsxws f5, f5
-; CHECK-P9-NEXT:    xscvdpsxws f6, f6
-; CHECK-P9-NEXT:    xscvdpsxws f7, f7
 ; CHECK-P9-NEXT:    mfvsrwz r3, f3
-; CHECK-P9-NEXT:    mfvsrwz r5, f2
-; CHECK-P9-NEXT:    mfvsrwz r7, f1
-; CHECK-P9-NEXT:    mfvsrwz r9, f0
-; CHECK-P9-NEXT:    mtvsrd f0, r3
-; CHECK-P9-NEXT:    mtvsrd f2, r5
-; CHECK-P9-NEXT:    mfvsrwz r4, f4
-; CHECK-P9-NEXT:    mfvsrwz r6, f5
-; CHECK-P9-NEXT:    mfvsrwz r8, f6
-; CHECK-P9-NEXT:    mfvsrwz r10, f7
-; CHECK-P9-NEXT:    mtvsrd f4, r7
-; CHECK-P9-NEXT:    mtvsrd f6, r9
-; CHECK-P9-NEXT:    xxswapd v2, vs0
+; CHECK-P9-NEXT:    mtvsrd f3, r3
+; CHECK-P9-NEXT:    mfvsrwz r3, f2
+; CHECK-P9-NEXT:    mtvsrd f2, r3
 ; CHECK-P9-NEXT:    xxswapd v4, vs2
-; CHECK-P9-NEXT:    mtvsrd f1, r4
-; CHECK-P9-NEXT:    mtvsrd f3, r6
-; CHECK-P9-NEXT:    mtvsrd f5, r8
-; CHECK-P9-NEXT:    mtvsrd f7, r10
-; CHECK-P9-NEXT:    xxswapd v0, vs4
-; CHECK-P9-NEXT:    xxswapd v6, vs6
-; CHECK-P9-NEXT:    xxswapd v3, vs1
-; CHECK-P9-NEXT:    xxswapd v5, vs3
-; CHECK-P9-NEXT:    xxswapd v1, vs5
-; CHECK-P9-NEXT:    xxswapd v7, vs7
+; CHECK-P9-NEXT:    xscvdpsxws f2, f1
+; CHECK-P9-NEXT:    xxswapd vs1, vs1
+; CHECK-P9-NEXT:    xscvdpsxws f1, f1
+; CHECK-P9-NEXT:    mfvsrwz r3, f2
+; CHECK-P9-NEXT:    mtvsrd f2, r3
+; CHECK-P9-NEXT:    mfvsrwz r3, f1
 ; CHECK-P9-NEXT:    vmrglb v2, v2, v3
-; CHECK-P9-NEXT:    vmrglb v3, v4, v5
-; CHECK-P9-NEXT:    vmrglb v4, v0, v1
-; CHECK-P9-NEXT:    vmrglb v5, v6, v7
+; CHECK-P9-NEXT:    xxswapd v3, vs3
+; CHECK-P9-NEXT:    vmrglb v3, v3, v4
 ; CHECK-P9-NEXT:    vmrglh v2, v3, v2
-; CHECK-P9-NEXT:    vmrglh v3, v5, v4
+; CHECK-P9-NEXT:    xxswapd v3, vs2
+; CHECK-P9-NEXT:    mtvsrd f1, r3
+; CHECK-P9-NEXT:    xxswapd v4, vs1
+; CHECK-P9-NEXT:    xscvdpsxws f1, f0
+; CHECK-P9-NEXT:    xxswapd vs0, vs0
+; CHECK-P9-NEXT:    xscvdpsxws f0, f0
+; CHECK-P9-NEXT:    mfvsrwz r3, f1
+; CHECK-P9-NEXT:    mtvsrd f1, r3
+; CHECK-P9-NEXT:    mfvsrwz r3, f0
+; CHECK-P9-NEXT:    mtvsrd f0, r3
+; CHECK-P9-NEXT:    vmrglb v3, v3, v4
+; CHECK-P9-NEXT:    xxswapd v4, vs1
+; CHECK-P9-NEXT:    xxswapd v5, vs0
+; CHECK-P9-NEXT:    vmrglb v4, v4, v5
+; CHECK-P9-NEXT:    vmrglh v3, v4, v3
 ; CHECK-P9-NEXT:    vmrglw v2, v3, v2
 ; CHECK-P9-NEXT:    mfvsrld r3, v2
 ; CHECK-P9-NEXT:    blr
 ;
 ; CHECK-BE-LABEL: test8elt_signed:
 ; CHECK-BE:       # %bb.0: # %entry
-; CHECK-BE-NEXT:    lxv vs0, 0(r3)
-; CHECK-BE-NEXT:    lxv vs1, 16(r3)
-; CHECK-BE-NEXT:    lxv vs2, 32(r3)
 ; CHECK-BE-NEXT:    lxv vs3, 48(r3)
-; CHECK-BE-NEXT:    xxswapd vs4, vs3
-; CHECK-BE-NEXT:    xxswapd vs5, vs2
-; CHECK-BE-NEXT:    xxswapd vs6, vs1
-; CHECK-BE-NEXT:    xxswapd vs7, vs0
+; CHECK-BE-NEXT:    xscvdpsxws f4, f3
+; CHECK-BE-NEXT:    xxswapd vs3, vs3
 ; CHECK-BE-NEXT:    xscvdpsxws f3, f3
-; CHECK-BE-NEXT:    xscvdpsxws f2, f2
-; CHECK-BE-NEXT:    xscvdpsxws f1, f1
-; CHECK-BE-NEXT:    xscvdpsxws f0, f0
-; CHECK-BE-NEXT:    xscvdpsxws f4, f4
-; CHECK-BE-NEXT:    xscvdpsxws f5, f5
-; CHECK-BE-NEXT:    xscvdpsxws f6, f6
-; CHECK-BE-NEXT:    xscvdpsxws f7, f7
-; CHECK-BE-NEXT:    mfvsrwz r3, f3
-; CHECK-BE-NEXT:    mfvsrwz r5, f2
-; CHECK-BE-NEXT:    mfvsrwz r7, f1
-; CHECK-BE-NEXT:    mfvsrwz r9, f0
+; CHECK-BE-NEXT:    lxv vs2, 32(r3)
+; CHECK-BE-NEXT:    lxv vs0, 0(r3)
+; CHECK-BE-NEXT:    lxv vs1, 16(r3)
+; CHECK-BE-NEXT:    mfvsrwz r3, f4
 ; CHECK-BE-NEXT:    sldi r3, r3, 56
-; CHECK-BE-NEXT:    sldi r5, r5, 56
-; CHECK-BE-NEXT:    sldi r7, r7, 56
-; CHECK-BE-NEXT:    sldi r9, r9, 56
-; CHECK-BE-NEXT:    mfvsrwz r4, f4
-; CHECK-BE-NEXT:    mfvsrwz r6, f5
-; CHECK-BE-NEXT:    mfvsrwz r8, f6
-; CHECK-BE-NEXT:    mfvsrwz r10, f7
 ; CHECK-BE-NEXT:    mtvsrd v2, r3
-; CHECK-BE-NEXT:    mtvsrd v4, r5
-; CHECK-BE-NEXT:    mtvsrd v0, r7
-; CHECK-BE-NEXT:    mtvsrd v6, r9
-; CHECK-BE-NEXT:    sldi r4, r4, 56
-; CHECK-BE-NEXT:    sldi r6, r6, 56
-; CHECK-BE-NEXT:    sldi r8, r8, 56
-; CHECK-BE-NEXT:    sldi r10, r10, 56
-; CHECK-BE-NEXT:    mtvsrd v3, r4
-; CHECK-BE-NEXT:    mtvsrd v5, r6
-; CHECK-BE-NEXT:    mtvsrd v1, r8
-; CHECK-BE-NEXT:    mtvsrd v7, r10
+; CHECK-BE-NEXT:    mfvsrwz r3, f3
+; CHECK-BE-NEXT:    xscvdpsxws f3, f2
+; CHECK-BE-NEXT:    xxswapd vs2, vs2
+; CHECK-BE-NEXT:    sldi r3, r3, 56
+; CHECK-BE-NEXT:    xscvdpsxws f2, f2
+; CHECK-BE-NEXT:    mtvsrd v3, r3
 ; CHECK-BE-NEXT:    vmrghb v2, v2, v3
-; CHECK-BE-NEXT:    vmrghb v3, v4, v5
-; CHECK-BE-NEXT:    vmrghb v4, v0, v1
-; CHECK-BE-NEXT:    vmrghb v5, v6, v7
+; CHECK-BE-NEXT:    mfvsrwz r3, f3
+; CHECK-BE-NEXT:    sldi r3, r3, 56
+; CHECK-BE-NEXT:    mtvsrd v3, r3
+; CHECK-BE-NEXT:    mfvsrwz r3, f2
+; CHECK-BE-NEXT:    xscvdpsxws f2, f1
+; CHECK-BE-NEXT:    xxswapd vs1, vs1
+; CHECK-BE-NEXT:    sldi r3, r3, 56
+; CHECK-BE-NEXT:    xscvdpsxws f1, f1
+; CHECK-BE-NEXT:    mtvsrd v4, r3
+; CHECK-BE-NEXT:    vmrghb v3, v3, v4
+; CHECK-BE-NEXT:    mfvsrwz r3, f2
+; CHECK-BE-NEXT:    sldi r3, r3, 56
 ; CHECK-BE-NEXT:    vmrghh v2, v3, v2
-; CHECK-BE-NEXT:    vmrghh v3, v5, v4
+; CHECK-BE-NEXT:    mtvsrd v3, r3
+; CHECK-BE-NEXT:    mfvsrwz r3, f1
+; CHECK-BE-NEXT:    xscvdpsxws f1, f0
+; CHECK-BE-NEXT:    xxswapd vs0, vs0
+; CHECK-BE-NEXT:    sldi r3, r3, 56
+; CHECK-BE-NEXT:    xscvdpsxws f0, f0
+; CHECK-BE-NEXT:    mtvsrd v4, r3
+; CHECK-BE-NEXT:    vmrghb v3, v3, v4
+; CHECK-BE-NEXT:    mfvsrwz r3, f1
+; CHECK-BE-NEXT:    sldi r3, r3, 56
+; CHECK-BE-NEXT:    mtvsrd v4, r3
+; CHECK-BE-NEXT:    mfvsrwz r3, f0
+; CHECK-BE-NEXT:    sldi r3, r3, 56
+; CHECK-BE-NEXT:    mtvsrd v5, r3
+; CHECK-BE-NEXT:    vmrghb v4, v4, v5
+; CHECK-BE-NEXT:    vmrghh v3, v4, v3
 ; CHECK-BE-NEXT:    vmrghw v2, v3, v2
 ; CHECK-BE-NEXT:    mfvsrd r3, v2
 ; CHECK-BE-NEXT:    blr
@@ -1094,219 +1074,199 @@ define <16 x i8> @test16elt_signed(<16 x double>* nocapture readonly) local_unna
 ;
 ; CHECK-P9-LABEL: test16elt_signed:
 ; CHECK-P9:       # %bb.0: # %entry
-; CHECK-P9-NEXT:    lxv vs2, 48(r3)
-; CHECK-P9-NEXT:    lxv vs3, 32(r3)
-; CHECK-P9-NEXT:    lxv vs4, 16(r3)
-; CHECK-P9-NEXT:    lxv vs5, 0(r3)
+; CHECK-P9-NEXT:    lxv vs7, 0(r3)
+; CHECK-P9-NEXT:    xscvdpsxws f8, f7
+; CHECK-P9-NEXT:    xxswapd vs7, vs7
+; CHECK-P9-NEXT:    xscvdpsxws f7, f7
 ; CHECK-P9-NEXT:    lxv vs0, 112(r3)
 ; CHECK-P9-NEXT:    lxv vs1, 96(r3)
-; CHECK-P9-NEXT:    lxv vs6, 80(r3)
-; CHECK-P9-NEXT:    lxv vs7, 64(r3)
-; CHECK-P9-NEXT:    std r26, -48(r1) # 8-byte Folded Spill
-; CHECK-P9-NEXT:    std r27, -40(r1) # 8-byte Folded Spill
-; CHECK-P9-NEXT:    std r28, -32(r1) # 8-byte Folded Spill
-; CHECK-P9-NEXT:    std r29, -24(r1) # 8-byte Folded Spill
-; CHECK-P9-NEXT:    std r30, -16(r1) # 8-byte Folded Spill
-; CHECK-P9-NEXT:    xxswapd vs8, vs5
-; CHECK-P9-NEXT:    xxswapd vs9, vs4
-; CHECK-P9-NEXT:    xxswapd vs10, vs3
-; CHECK-P9-NEXT:    xxswapd vs11, vs2
-; CHECK-P9-NEXT:    xxswapd vs12, vs7
-; CHECK-P9-NEXT:    xxswapd vs13, vs6
-; CHECK-P9-NEXT:    xxswapd v2, vs1
-; CHECK-P9-NEXT:    xxswapd v3, vs0
+; CHECK-P9-NEXT:    lxv vs2, 80(r3)
+; CHECK-P9-NEXT:    lxv vs3, 64(r3)
+; CHECK-P9-NEXT:    lxv vs4, 48(r3)
+; CHECK-P9-NEXT:    lxv vs5, 32(r3)
+; CHECK-P9-NEXT:    lxv vs6, 16(r3)
+; CHECK-P9-NEXT:    mfvsrwz r3, f8
+; CHECK-P9-NEXT:    mtvsrd f8, r3
+; CHECK-P9-NEXT:    mfvsrwz r3, f7
+; CHECK-P9-NEXT:    xxswapd v2, vs8
+; CHECK-P9-NEXT:    mtvsrd f7, r3
+; CHECK-P9-NEXT:    xxswapd v3, vs7
+; CHECK-P9-NEXT:    xscvdpsxws f7, f6
+; CHECK-P9-NEXT:    xxswapd vs6, vs6
+; CHECK-P9-NEXT:    xscvdpsxws f6, f6
+; CHECK-P9-NEXT:    mfvsrwz r3, f7
+; CHECK-P9-NEXT:    mtvsrd f7, r3
+; CHECK-P9-NEXT:    mfvsrwz r3, f6
+; CHECK-P9-NEXT:    mtvsrd f6, r3
+; CHECK-P9-NEXT:    xxswapd v4, vs6
+; CHECK-P9-NEXT:    xscvdpsxws f6, f5
+; CHECK-P9-NEXT:    xxswapd vs5, vs5
 ; CHECK-P9-NEXT:    xscvdpsxws f5, f5
+; CHECK-P9-NEXT:    mfvsrwz r3, f6
+; CHECK-P9-NEXT:    mtvsrd f6, r3
+; CHECK-P9-NEXT:    mfvsrwz r3, f5
+; CHECK-P9-NEXT:    vmrglb v2, v2, v3
+; CHECK-P9-NEXT:    xxswapd v3, vs7
+; CHECK-P9-NEXT:    vmrglb v3, v3, v4
+; CHECK-P9-NEXT:    vmrglh v2, v3, v2
+; CHECK-P9-NEXT:    xxswapd v3, vs6
+; CHECK-P9-NEXT:    mtvsrd f5, r3
+; CHECK-P9-NEXT:    xxswapd v4, vs5
+; CHECK-P9-NEXT:    xscvdpsxws f5, f4
+; CHECK-P9-NEXT:    xxswapd vs4, vs4
 ; CHECK-P9-NEXT:    xscvdpsxws f4, f4
+; CHECK-P9-NEXT:    mfvsrwz r3, f5
+; CHECK-P9-NEXT:    mtvsrd f5, r3
+; CHECK-P9-NEXT:    mfvsrwz r3, f4
+; CHECK-P9-NEXT:    mtvsrd f4, r3
+; CHECK-P9-NEXT:    xxswapd v5, vs4
+; CHECK-P9-NEXT:    xscvdpsxws f4, f3
+; CHECK-P9-NEXT:    xxswapd vs3, vs3
 ; CHECK-P9-NEXT:    xscvdpsxws f3, f3
+; CHECK-P9-NEXT:    vmrglb v3, v3, v4
+; CHECK-P9-NEXT:    xxswapd v4, vs5
+; CHECK-P9-NEXT:    vmrglb v4, v4, v5
+; CHECK-P9-NEXT:    vmrglh v3, v4, v3
+; CHECK-P9-NEXT:    mfvsrwz r3, f4
+; CHECK-P9-NEXT:    mtvsrd f4, r3
+; CHECK-P9-NEXT:    mfvsrwz r3, f3
+; CHECK-P9-NEXT:    mtvsrd f3, r3
+; CHECK-P9-NEXT:    xxswapd v4, vs3
+; CHECK-P9-NEXT:    xscvdpsxws f3, f2
+; CHECK-P9-NEXT:    xxswapd vs2, vs2
 ; CHECK-P9-NEXT:    xscvdpsxws f2, f2
-; CHECK-P9-NEXT:    xscvdpsxws f7, f7
-; CHECK-P9-NEXT:    xscvdpsxws f6, f6
+; CHECK-P9-NEXT:    mfvsrwz r3, f3
+; CHECK-P9-NEXT:    mtvsrd f3, r3
+; CHECK-P9-NEXT:    mfvsrwz r3, f2
+; CHECK-P9-NEXT:    mtvsrd f2, r3
+; CHECK-P9-NEXT:    xxswapd v5, vs2
+; CHECK-P9-NEXT:    xscvdpsxws f2, f1
+; CHECK-P9-NEXT:    xxswapd vs1, vs1
 ; CHECK-P9-NEXT:    xscvdpsxws f1, f1
+; CHECK-P9-NEXT:    vmrglw v2, v3, v2
+; CHECK-P9-NEXT:    xxswapd v3, vs4
+; CHECK-P9-NEXT:    vmrglb v3, v3, v4
+; CHECK-P9-NEXT:    xxswapd v4, vs3
+; CHECK-P9-NEXT:    vmrglb v4, v4, v5
+; CHECK-P9-NEXT:    vmrglh v3, v4, v3
+; CHECK-P9-NEXT:    mfvsrwz r3, f2
+; CHECK-P9-NEXT:    mtvsrd f2, r3
+; CHECK-P9-NEXT:    mfvsrwz r3, f1
+; CHECK-P9-NEXT:    xxswapd v4, vs2
+; CHECK-P9-NEXT:    mtvsrd f1, r3
+; CHECK-P9-NEXT:    xxswapd v5, vs1
+; CHECK-P9-NEXT:    xscvdpsxws f1, f0
+; CHECK-P9-NEXT:    xxswapd vs0, vs0
 ; CHECK-P9-NEXT:    xscvdpsxws f0, f0
-; CHECK-P9-NEXT:    xscvdpsxws f8, f8
-; CHECK-P9-NEXT:    xscvdpsxws f9, f9
-; CHECK-P9-NEXT:    xscvdpsxws f10, f10
-; CHECK-P9-NEXT:    xscvdpsxws f11, f11
-; CHECK-P9-NEXT:    xscvdpsxws f12, f12
-; CHECK-P9-NEXT:    xscvdpsxws f13, f13
-; CHECK-P9-NEXT:    xscvdpsxws v2, v2
-; CHECK-P9-NEXT:    xscvdpsxws v3, v3
-; CHECK-P9-NEXT:    mfvsrwz r3, f5
-; CHECK-P9-NEXT:    mfvsrwz r4, f4
-; CHECK-P9-NEXT:    mfvsrwz r5, f3
-; CHECK-P9-NEXT:    mfvsrwz r6, f2
-; CHECK-P9-NEXT:    mfvsrwz r11, f7
-; CHECK-P9-NEXT:    mfvsrwz r12, f6
-; CHECK-P9-NEXT:    mfvsrwz r0, f1
-; CHECK-P9-NEXT:    mfvsrwz r30, f0
-; CHECK-P9-NEXT:    mfvsrwz r7, f8
-; CHECK-P9-NEXT:    mfvsrwz r8, f9
-; CHECK-P9-NEXT:    mfvsrwz r9, f10
-; CHECK-P9-NEXT:    mfvsrwz r10, f11
-; CHECK-P9-NEXT:    mfvsrwz r29, f12
-; CHECK-P9-NEXT:    mfvsrwz r28, f13
-; CHECK-P9-NEXT:    mfvsrwz r27, v2
-; CHECK-P9-NEXT:    mfvsrwz r26, v3
+; CHECK-P9-NEXT:    mfvsrwz r3, f1
+; CHECK-P9-NEXT:    mtvsrd f1, r3
+; CHECK-P9-NEXT:    mfvsrwz r3, f0
 ; CHECK-P9-NEXT:    mtvsrd f0, r3
-; CHECK-P9-NEXT:    mtvsrd f1, r4
-; CHECK-P9-NEXT:    mtvsrd f2, r5
-; CHECK-P9-NEXT:    mtvsrd f3, r6
-; CHECK-P9-NEXT:    mtvsrd f8, r11
-; CHECK-P9-NEXT:    mtvsrd f9, r12
-; CHECK-P9-NEXT:    mtvsrd f10, r0
-; CHECK-P9-NEXT:    mtvsrd f11, r30
-; CHECK-P9-NEXT:    mtvsrd f4, r7
-; CHECK-P9-NEXT:    mtvsrd f5, r8
-; CHECK-P9-NEXT:    mtvsrd f6, r9
-; CHECK-P9-NEXT:    mtvsrd f7, r10
-; CHECK-P9-NEXT:    mtvsrd f12, r29
-; CHECK-P9-NEXT:    mtvsrd f13, r28
-; CHECK-P9-NEXT:    mtvsrd v2, r27
-; CHECK-P9-NEXT:    mtvsrd v3, r26
-; CHECK-P9-NEXT:    xxswapd v4, vs0
+; CHECK-P9-NEXT:    vmrglb v4, v4, v5
 ; CHECK-P9-NEXT:    xxswapd v5, vs1
-; CHECK-P9-NEXT:    xxswapd v0, vs2
-; CHECK-P9-NEXT:    xxswapd v1, vs3
-; CHECK-P9-NEXT:    ld r30, -16(r1) # 8-byte Folded Reload
-; CHECK-P9-NEXT:    ld r29, -24(r1) # 8-byte Folded Reload
-; CHECK-P9-NEXT:    ld r28, -32(r1) # 8-byte Folded Reload
-; CHECK-P9-NEXT:    ld r27, -40(r1) # 8-byte Folded Reload
-; CHECK-P9-NEXT:    xxswapd v6, vs4
-; CHECK-P9-NEXT:    xxswapd v7, vs5
-; CHECK-P9-NEXT:    ld r26, -48(r1) # 8-byte Folded Reload
-; CHECK-P9-NEXT:    xxswapd v8, vs6
-; CHECK-P9-NEXT:    xxswapd v9, vs7
-; CHECK-P9-NEXT:    xxswapd v10, vs8
-; CHECK-P9-NEXT:    xxswapd v11, vs12
-; CHECK-P9-NEXT:    xxswapd v12, vs9
-; CHECK-P9-NEXT:    xxswapd v13, vs13
-; CHECK-P9-NEXT:    xxswapd v14, vs10
-; CHECK-P9-NEXT:    xxswapd v2, v2
-; CHECK-P9-NEXT:    xxswapd v15, vs11
-; CHECK-P9-NEXT:    xxswapd v3, v3
-; CHECK-P9-NEXT:    vmrglb v4, v4, v6
-; CHECK-P9-NEXT:    vmrglb v5, v5, v7
-; CHECK-P9-NEXT:    vmrglb v0, v0, v8
-; CHECK-P9-NEXT:    vmrglb v1, v1, v9
-; CHECK-P9-NEXT:    vmrglb v6, v10, v11
-; CHECK-P9-NEXT:    vmrglb v7, v12, v13
-; CHECK-P9-NEXT:    vmrglb v2, v14, v2
-; CHECK-P9-NEXT:    vmrglb v3, v15, v3
+; CHECK-P9-NEXT:    xxswapd v0, vs0
+; CHECK-P9-NEXT:    vmrglb v5, v5, v0
 ; CHECK-P9-NEXT:    vmrglh v4, v5, v4
-; CHECK-P9-NEXT:    vmrglh v5, v1, v0
-; CHECK-P9-NEXT:    vmrglh v0, v7, v6
-; CHECK-P9-NEXT:    vmrglh v2, v3, v2
-; CHECK-P9-NEXT:    vmrglw v3, v5, v4
-; CHECK-P9-NEXT:    vmrglw v2, v2, v0
-; CHECK-P9-NEXT:    xxmrgld v2, v2, v3
+; CHECK-P9-NEXT:    vmrglw v3, v4, v3
+; CHECK-P9-NEXT:    xxmrgld v2, v3, v2
 ; CHECK-P9-NEXT:    blr
 ;
 ; CHECK-BE-LABEL: test16elt_signed:
 ; CHECK-BE:       # %bb.0: # %entry
-; CHECK-BE-NEXT:    lxv vs2, 64(r3)
-; CHECK-BE-NEXT:    lxv vs3, 80(r3)
-; CHECK-BE-NEXT:    lxv vs4, 96(r3)
-; CHECK-BE-NEXT:    lxv vs5, 112(r3)
+; CHECK-BE-NEXT:    lxv vs7, 112(r3)
+; CHECK-BE-NEXT:    xscvdpsxws f8, f7
+; CHECK-BE-NEXT:    xxswapd vs7, vs7
+; CHECK-BE-NEXT:    xscvdpsxws f7, f7
+; CHECK-BE-NEXT:    lxv vs6, 96(r3)
 ; CHECK-BE-NEXT:    lxv vs0, 0(r3)
 ; CHECK-BE-NEXT:    lxv vs1, 16(r3)
-; CHECK-BE-NEXT:    lxv vs6, 32(r3)
-; CHECK-BE-NEXT:    lxv vs7, 48(r3)
-; CHECK-BE-NEXT:    std r26, -48(r1) # 8-byte Folded Spill
-; CHECK-BE-NEXT:    std r27, -40(r1) # 8-byte Folded Spill
-; CHECK-BE-NEXT:    std r28, -32(r1) # 8-byte Folded Spill
-; CHECK-BE-NEXT:    std r29, -24(r1) # 8-byte Folded Spill
-; CHECK-BE-NEXT:    std r30, -16(r1) # 8-byte Folded Spill
-; CHECK-BE-NEXT:    xxswapd vs8, vs5
-; CHECK-BE-NEXT:    xxswapd vs9, vs4
-; CHECK-BE-NEXT:    xxswapd vs10, vs3
-; CHECK-BE-NEXT:    xxswapd vs11, vs2
-; CHECK-BE-NEXT:    xxswapd vs12, vs7
-; CHECK-BE-NEXT:    xxswapd vs13, vs6
-; CHECK-BE-NEXT:    xxswapd v2, vs1
-; CHECK-BE-NEXT:    xxswapd v3, vs0
+; CHECK-BE-NEXT:    lxv vs2, 32(r3)
+; CHECK-BE-NEXT:    lxv vs3, 48(r3)
+; CHECK-BE-NEXT:    lxv vs4, 64(r3)
+; CHECK-BE-NEXT:    lxv vs5, 80(r3)
+; CHECK-BE-NEXT:    mfvsrwz r3, f8
+; CHECK-BE-NEXT:    sldi r3, r3, 56
+; CHECK-BE-NEXT:    mtvsrd v2, r3
+; CHECK-BE-NEXT:    mfvsrwz r3, f7
+; CHECK-BE-NEXT:    xscvdpsxws f7, f6
+; CHECK-BE-NEXT:    xxswapd vs6, vs6
+; CHECK-BE-NEXT:    sldi r3, r3, 56
+; CHECK-BE-NEXT:    xscvdpsxws f6, f6
+; CHECK-BE-NEXT:    mtvsrd v3, r3
+; CHECK-BE-NEXT:    vmrghb v2, v2, v3
+; CHECK-BE-NEXT:    mfvsrwz r3, f7
+; CHECK-BE-NEXT:    sldi r3, r3, 56
+; CHECK-BE-NEXT:    mtvsrd v3, r3
+; CHECK-BE-NEXT:    mfvsrwz r3, f6
+; CHECK-BE-NEXT:    xscvdpsxws f6, f5
+; CHECK-BE-NEXT:    xxswapd vs5, vs5
+; CHECK-BE-NEXT:    sldi r3, r3, 56
 ; CHECK-BE-NEXT:    xscvdpsxws f5, f5
+; CHECK-BE-NEXT:    mtvsrd v4, r3
+; CHECK-BE-NEXT:    vmrghb v3, v3, v4
+; CHECK-BE-NEXT:    mfvsrwz r3, f6
+; CHECK-BE-NEXT:    sldi r3, r3, 56
+; CHECK-BE-NEXT:    vmrghh v2, v3, v2
+; CHECK-BE-NEXT:    mtvsrd v3, r3
+; CHECK-BE-NEXT:    mfvsrwz r3, f5
+; CHECK-BE-NEXT:    xscvdpsxws f5, f4
+; CHECK-BE-NEXT:    xxswapd vs4, vs4
+; CHECK-BE-NEXT:    sldi r3, r3, 56
 ; CHECK-BE-NEXT:    xscvdpsxws f4, f4
+; CHECK-BE-NEXT:    mtvsrd v4, r3
+; CHECK-BE-NEXT:    vmrghb v3, v3, v4
+; CHECK-BE-NEXT:    mfvsrwz r3, f5
+; CHECK-BE-NEXT:    sldi r3, r3, 56
+; CHECK-BE-NEXT:    mtvsrd v4, r3
+; CHECK-BE-NEXT:    mfvsrwz r3, f4
+; CHECK-BE-NEXT:    xscvdpsxws f4, f3
+; CHECK-BE-NEXT:    xxswapd vs3, vs3
+; CHECK-BE-NEXT:    sldi r3, r3, 56
 ; CHECK-BE-NEXT:    xscvdpsxws f3, f3
+; CHECK-BE-NEXT:    mtvsrd v5, r3
+; CHECK-BE-NEXT:    vmrghb v4, v4, v5
+; CHECK-BE-NEXT:    mfvsrwz r3, f4
+; CHECK-BE-NEXT:    vmrghh v3, v4, v3
+; CHECK-BE-NEXT:    sldi r3, r3, 56
+; CHECK-BE-NEXT:    vmrghw v2, v3, v2
+; CHECK-BE-NEXT:    mtvsrd v3, r3
+; CHECK-BE-NEXT:    mfvsrwz r3, f3
+; CHECK-BE-NEXT:    xscvdpsxws f3, f2
+; CHECK-BE-NEXT:    xxswapd vs2, vs2
+; CHECK-BE-NEXT:    sldi r3, r3, 56
 ; CHECK-BE-NEXT:    xscvdpsxws f2, f2
-; CHECK-BE-NEXT:    xscvdpsxws f7, f7
-; CHECK-BE-NEXT:    xscvdpsxws f6, f6
+; CHECK-BE-NEXT:    mtvsrd v4, r3
+; CHECK-BE-NEXT:    vmrghb v3, v3, v4
+; CHECK-BE-NEXT:    mfvsrwz r3, f3
+; CHECK-BE-NEXT:    sldi r3, r3, 56
+; CHECK-BE-NEXT:    mtvsrd v4, r3
+; CHECK-BE-NEXT:    mfvsrwz r3, f2
+; CHECK-BE-NEXT:    xscvdpsxws f2, f1
+; CHECK-BE-NEXT:    xxswapd vs1, vs1
+; CHECK-BE-NEXT:    sldi r3, r3, 56
 ; CHECK-BE-NEXT:    xscvdpsxws f1, f1
+; CHECK-BE-NEXT:    mtvsrd v5, r3
+; CHECK-BE-NEXT:    vmrghb v4, v4, v5
+; CHECK-BE-NEXT:    mfvsrwz r3, f2
+; CHECK-BE-NEXT:    sldi r3, r3, 56
+; CHECK-BE-NEXT:    vmrghh v3, v4, v3
+; CHECK-BE-NEXT:    mtvsrd v4, r3
+; CHECK-BE-NEXT:    mfvsrwz r3, f1
+; CHECK-BE-NEXT:    xscvdpsxws f1, f0
+; CHECK-BE-NEXT:    xxswapd vs0, vs0
+; CHECK-BE-NEXT:    sldi r3, r3, 56
 ; CHECK-BE-NEXT:    xscvdpsxws f0, f0
-; CHECK-BE-NEXT:    xscvdpsxws f8, f8
-; CHECK-BE-NEXT:    xscvdpsxws f9, f9
-; CHECK-BE-NEXT:    xscvdpsxws f10, f10
-; CHECK-BE-NEXT:    xscvdpsxws f11, f11
-; CHECK-BE-NEXT:    xscvdpsxws f12, f12
-; CHECK-BE-NEXT:    xscvdpsxws f13, f13
-; CHECK-BE-NEXT:    xscvdpsxws v2, v2
-; CHECK-BE-NEXT:    xscvdpsxws v3, v3
-; CHECK-BE-NEXT:    mfvsrwz r3, f5
-; CHECK-BE-NEXT:    mfvsrwz r4, f4
-; CHECK-BE-NEXT:    mfvsrwz r5, f3
-; CHECK-BE-NEXT:    mfvsrwz r6, f2
-; CHECK-BE-NEXT:    mfvsrwz r11, f7
-; CHECK-BE-NEXT:    mfvsrwz r12, f6
-; CHECK-BE-NEXT:    mfvsrwz r0, f1
-; CHECK-BE-NEXT:    mfvsrwz r30, f0
-; CHECK-BE-NEXT:    mfvsrwz r7, f8
-; CHECK-BE-NEXT:    mfvsrwz r8, f9
-; CHECK-BE-NEXT:    mfvsrwz r9, f10
-; CHECK-BE-NEXT:    mfvsrwz r10, f11
-; CHECK-BE-NEXT:    mfvsrwz r29, f12
-; CHECK-BE-NEXT:    mfvsrwz r28, f13
-; CHECK-BE-NEXT:    mfvsrwz r27, v2
-; CHECK-BE-NEXT:    mfvsrwz r26, v3
+; CHECK-BE-NEXT:    mtvsrd v5, r3
+; CHECK-BE-NEXT:    vmrghb v4, v4, v5
+; CHECK-BE-NEXT:    mfvsrwz r3, f1
 ; CHECK-BE-NEXT:    sldi r3, r3, 56
-; CHECK-BE-NEXT:    sldi r4, r4, 56
-; CHECK-BE-NEXT:    sldi r5, r5, 56
-; CHECK-BE-NEXT:    sldi r6, r6, 56
-; CHECK-BE-NEXT:    sldi r11, r11, 56
-; CHECK-BE-NEXT:    sldi r12, r12, 56
-; CHECK-BE-NEXT:    sldi r0, r0, 56
-; CHECK-BE-NEXT:    sldi r30, r30, 56
-; CHECK-BE-NEXT:    sldi r7, r7, 56
-; CHECK-BE-NEXT:    sldi r8, r8, 56
-; CHECK-BE-NEXT:    sldi r9, r9, 56
-; CHECK-BE-NEXT:    sldi r10, r10, 56
-; CHECK-BE-NEXT:    sldi r29, r29, 56
-; CHECK-BE-NEXT:    sldi r28, r28, 56
-; CHECK-BE-NEXT:    sldi r27, r27, 56
-; CHECK-BE-NEXT:    sldi r26, r26, 56
-; CHECK-BE-NEXT:    mtvsrd v2, r3
-; CHECK-BE-NEXT:    mtvsrd v3, r4
-; CHECK-BE-NEXT:    mtvsrd v4, r5
-; CHECK-BE-NEXT:    mtvsrd v5, r6
-; CHECK-BE-NEXT:    mtvsrd v8, r11
-; CHECK-BE-NEXT:    mtvsrd v10, r12
-; CHECK-BE-NEXT:    mtvsrd v12, r0
-; CHECK-BE-NEXT:    mtvsrd v14, r30
-; CHECK-BE-NEXT:    mtvsrd v0, r7
-; CHECK-BE-NEXT:    mtvsrd v1, r8
-; CHECK-BE-NEXT:    mtvsrd v6, r9
-; CHECK-BE-NEXT:    mtvsrd v7, r10
-; CHECK-BE-NEXT:    ld r30, -16(r1) # 8-byte Folded Reload
-; CHECK-BE-NEXT:    mtvsrd v9, r29
-; CHECK-BE-NEXT:    ld r29, -24(r1) # 8-byte Folded Reload
-; CHECK-BE-NEXT:    mtvsrd v11, r28
-; CHECK-BE-NEXT:    mtvsrd v13, r27
-; CHECK-BE-NEXT:    mtvsrd v15, r26
-; CHECK-BE-NEXT:    ld r28, -32(r1) # 8-byte Folded Reload
-; CHECK-BE-NEXT:    ld r27, -40(r1) # 8-byte Folded Reload
-; CHECK-BE-NEXT:    ld r26, -48(r1) # 8-byte Folded Reload
-; CHECK-BE-NEXT:    vmrghb v2, v2, v0
-; CHECK-BE-NEXT:    vmrghb v3, v3, v1
-; CHECK-BE-NEXT:    vmrghb v4, v4, v6
-; CHECK-BE-NEXT:    vmrghb v5, v5, v7
-; CHECK-BE-NEXT:    vmrghb v0, v8, v9
-; CHECK-BE-NEXT:    vmrghb v1, v10, v11
-; CHECK-BE-NEXT:    vmrghb v6, v12, v13
-; CHECK-BE-NEXT:    vmrghb v7, v14, v15
-; CHECK-BE-NEXT:    vmrghh v2, v3, v2
-; CHECK-BE-NEXT:    vmrghh v3, v5, v4
-; CHECK-BE-NEXT:    vmrghh v4, v1, v0
-; CHECK-BE-NEXT:    vmrghh v5, v7, v6
-; CHECK-BE-NEXT:    vmrghw v2, v3, v2
-; CHECK-BE-NEXT:    vmrghw v3, v5, v4
+; CHECK-BE-NEXT:    mtvsrd v5, r3
+; CHECK-BE-NEXT:    mfvsrwz r3, f0
+; CHECK-BE-NEXT:    sldi r3, r3, 56
+; CHECK-BE-NEXT:    mtvsrd v0, r3
+; CHECK-BE-NEXT:    vmrghb v5, v5, v0
+; CHECK-BE-NEXT:    vmrghh v4, v5, v4
+; CHECK-BE-NEXT:    vmrghw v3, v4, v3
 ; CHECK-BE-NEXT:    xxmrghd v2, v3, v2
 ; CHECK-BE-NEXT:    blr
 entry:
diff --git a/test/CodeGen/PowerPC/vec_conv_fp_to_i_8byte_elts.ll b/test/CodeGen/PowerPC/vec_conv_fp_to_i_8byte_elts.ll
index a5097b070c134eff1ad0a2964a77304ebf050316..a059e56bc3f66db41a0cc575d5bc666fa79bca26 100644
--- a/test/CodeGen/PowerPC/vec_conv_fp_to_i_8byte_elts.ll
+++ b/test/CodeGen/PowerPC/vec_conv_fp_to_i_8byte_elts.ll
@@ -163,58 +163,58 @@ define void @test16elt(<16 x i64>* noalias nocapture sret %agg.result, <16 x dou
 ;
 ; CHECK-P9-LABEL: test16elt:
 ; CHECK-P9:       # %bb.0: # %entry
-; CHECK-P9-NEXT:    lxv vs0, 48(r4)
-; CHECK-P9-NEXT:    lxv vs1, 32(r4)
-; CHECK-P9-NEXT:    lxv vs2, 16(r4)
-; CHECK-P9-NEXT:    lxv vs3, 0(r4)
-; CHECK-P9-NEXT:    lxv vs4, 112(r4)
-; CHECK-P9-NEXT:    lxv vs5, 96(r4)
-; CHECK-P9-NEXT:    lxv vs6, 80(r4)
-; CHECK-P9-NEXT:    lxv vs7, 64(r4)
+; CHECK-P9-NEXT:    lxv vs0, 112(r4)
+; CHECK-P9-NEXT:    lxv vs1, 96(r4)
+; CHECK-P9-NEXT:    lxv vs2, 80(r4)
+; CHECK-P9-NEXT:    lxv vs3, 64(r4)
+; CHECK-P9-NEXT:    lxv vs4, 48(r4)
+; CHECK-P9-NEXT:    xvcvdpuxds vs4, vs4
+; CHECK-P9-NEXT:    lxv vs5, 32(r4)
+; CHECK-P9-NEXT:    lxv vs6, 16(r4)
+; CHECK-P9-NEXT:    lxv vs7, 0(r4)
+; CHECK-P9-NEXT:    xvcvdpuxds vs7, vs7
+; CHECK-P9-NEXT:    xvcvdpuxds vs6, vs6
+; CHECK-P9-NEXT:    xvcvdpuxds vs5, vs5
 ; CHECK-P9-NEXT:    xvcvdpuxds vs3, vs3
 ; CHECK-P9-NEXT:    xvcvdpuxds vs2, vs2
 ; CHECK-P9-NEXT:    xvcvdpuxds vs1, vs1
 ; CHECK-P9-NEXT:    xvcvdpuxds vs0, vs0
-; CHECK-P9-NEXT:    xvcvdpuxds vs7, vs7
-; CHECK-P9-NEXT:    xvcvdpuxds vs6, vs6
-; CHECK-P9-NEXT:    xvcvdpuxds vs5, vs5
-; CHECK-P9-NEXT:    xvcvdpuxds vs4, vs4
-; CHECK-P9-NEXT:    stxv vs0, 48(r3)
-; CHECK-P9-NEXT:    stxv vs1, 32(r3)
-; CHECK-P9-NEXT:    stxv vs2, 16(r3)
-; CHECK-P9-NEXT:    stxv vs3, 0(r3)
-; CHECK-P9-NEXT:    stxv vs4, 112(r3)
-; CHECK-P9-NEXT:    stxv vs5, 96(r3)
-; CHECK-P9-NEXT:    stxv vs6, 80(r3)
-; CHECK-P9-NEXT:    stxv vs7, 64(r3)
+; CHECK-P9-NEXT:    stxv vs0, 112(r3)
+; CHECK-P9-NEXT:    stxv vs1, 96(r3)
+; CHECK-P9-NEXT:    stxv vs2, 80(r3)
+; CHECK-P9-NEXT:    stxv vs3, 64(r3)
+; CHECK-P9-NEXT:    stxv vs4, 48(r3)
+; CHECK-P9-NEXT:    stxv vs5, 32(r3)
+; CHECK-P9-NEXT:    stxv vs6, 16(r3)
+; CHECK-P9-NEXT:    stxv vs7, 0(r3)
 ; CHECK-P9-NEXT:    blr
 ;
 ; CHECK-BE-LABEL: test16elt:
 ; CHECK-BE:       # %bb.0: # %entry
-; CHECK-BE-NEXT:    lxv vs0, 48(r4)
-; CHECK-BE-NEXT:    lxv vs1, 32(r4)
-; CHECK-BE-NEXT:    lxv vs2, 16(r4)
-; CHECK-BE-NEXT:    lxv vs3, 0(r4)
-; CHECK-BE-NEXT:    lxv vs4, 112(r4)
-; CHECK-BE-NEXT:    lxv vs5, 96(r4)
-; CHECK-BE-NEXT:    lxv vs6, 80(r4)
-; CHECK-BE-NEXT:    lxv vs7, 64(r4)
+; CHECK-BE-NEXT:    lxv vs0, 112(r4)
+; CHECK-BE-NEXT:    lxv vs1, 96(r4)
+; CHECK-BE-NEXT:    lxv vs2, 80(r4)
+; CHECK-BE-NEXT:    lxv vs3, 64(r4)
+; CHECK-BE-NEXT:    lxv vs4, 48(r4)
+; CHECK-BE-NEXT:    xvcvdpuxds vs4, vs4
+; CHECK-BE-NEXT:    lxv vs5, 32(r4)
+; CHECK-BE-NEXT:    lxv vs6, 16(r4)
+; CHECK-BE-NEXT:    lxv vs7, 0(r4)
+; CHECK-BE-NEXT:    xvcvdpuxds vs7, vs7
+; CHECK-BE-NEXT:    xvcvdpuxds vs6, vs6
+; CHECK-BE-NEXT:    xvcvdpuxds vs5, vs5
 ; CHECK-BE-NEXT:    xvcvdpuxds vs3, vs3
 ; CHECK-BE-NEXT:    xvcvdpuxds vs2, vs2
 ; CHECK-BE-NEXT:    xvcvdpuxds vs1, vs1
 ; CHECK-BE-NEXT:    xvcvdpuxds vs0, vs0
-; CHECK-BE-NEXT:    xvcvdpuxds vs7, vs7
-; CHECK-BE-NEXT:    xvcvdpuxds vs6, vs6
-; CHECK-BE-NEXT:    xvcvdpuxds vs5, vs5
-; CHECK-BE-NEXT:    xvcvdpuxds vs4, vs4
-; CHECK-BE-NEXT:    stxv vs0, 48(r3)
-; CHECK-BE-NEXT:    stxv vs1, 32(r3)
-; CHECK-BE-NEXT:    stxv vs2, 16(r3)
-; CHECK-BE-NEXT:    stxv vs3, 0(r3)
-; CHECK-BE-NEXT:    stxv vs4, 112(r3)
-; CHECK-BE-NEXT:    stxv vs5, 96(r3)
-; CHECK-BE-NEXT:    stxv vs6, 80(r3)
-; CHECK-BE-NEXT:    stxv vs7, 64(r3)
+; CHECK-BE-NEXT:    stxv vs0, 112(r3)
+; CHECK-BE-NEXT:    stxv vs1, 96(r3)
+; CHECK-BE-NEXT:    stxv vs2, 80(r3)
+; CHECK-BE-NEXT:    stxv vs3, 64(r3)
+; CHECK-BE-NEXT:    stxv vs4, 48(r3)
+; CHECK-BE-NEXT:    stxv vs5, 32(r3)
+; CHECK-BE-NEXT:    stxv vs6, 16(r3)
+; CHECK-BE-NEXT:    stxv vs7, 0(r3)
 ; CHECK-BE-NEXT:    blr
 entry:
   %a = load <16 x double>, <16 x double>* %0, align 128
@@ -377,58 +377,58 @@ define void @test16elt_signed(<16 x i64>* noalias nocapture sret %agg.result, <1
 ;
 ; CHECK-P9-LABEL: test16elt_signed:
 ; CHECK-P9:       # %bb.0: # %entry
-; CHECK-P9-NEXT:    lxv vs0, 48(r4)
-; CHECK-P9-NEXT:    lxv vs1, 32(r4)
-; CHECK-P9-NEXT:    lxv vs2, 16(r4)
-; CHECK-P9-NEXT:    lxv vs3, 0(r4)
-; CHECK-P9-NEXT:    lxv vs4, 112(r4)
-; CHECK-P9-NEXT:    lxv vs5, 96(r4)
-; CHECK-P9-NEXT:    lxv vs6, 80(r4)
-; CHECK-P9-NEXT:    lxv vs7, 64(r4)
+; CHECK-P9-NEXT:    lxv vs0, 112(r4)
+; CHECK-P9-NEXT:    lxv vs1, 96(r4)
+; CHECK-P9-NEXT:    lxv vs2, 80(r4)
+; CHECK-P9-NEXT:    lxv vs3, 64(r4)
+; CHECK-P9-NEXT:    lxv vs4, 48(r4)
+; CHECK-P9-NEXT:    xvcvdpsxds vs4, vs4
+; CHECK-P9-NEXT:    lxv vs5, 32(r4)
+; CHECK-P9-NEXT:    lxv vs6, 16(r4)
+; CHECK-P9-NEXT:    lxv vs7, 0(r4)
+; CHECK-P9-NEXT:    xvcvdpsxds vs7, vs7
+; CHECK-P9-NEXT:    xvcvdpsxds vs6, vs6
+; CHECK-P9-NEXT:    xvcvdpsxds vs5, vs5
 ; CHECK-P9-NEXT:    xvcvdpsxds vs3, vs3
 ; CHECK-P9-NEXT:    xvcvdpsxds vs2, vs2
 ; CHECK-P9-NEXT:    xvcvdpsxds vs1, vs1
 ; CHECK-P9-NEXT:    xvcvdpsxds vs0, vs0
-; CHECK-P9-NEXT:    xvcvdpsxds vs7, vs7
-; CHECK-P9-NEXT:    xvcvdpsxds vs6, vs6
-; CHECK-P9-NEXT:    xvcvdpsxds vs5, vs5
-; CHECK-P9-NEXT:    xvcvdpsxds vs4, vs4
-; CHECK-P9-NEXT:    stxv vs0, 48(r3)
-; CHECK-P9-NEXT:    stxv vs1, 32(r3)
-; CHECK-P9-NEXT:    stxv vs2, 16(r3)
-; CHECK-P9-NEXT:    stxv vs3, 0(r3)
-; CHECK-P9-NEXT:    stxv vs4, 112(r3)
-; CHECK-P9-NEXT:    stxv vs5, 96(r3)
-; CHECK-P9-NEXT:    stxv vs6, 80(r3)
-; CHECK-P9-NEXT:    stxv vs7, 64(r3)
+; CHECK-P9-NEXT:    stxv vs0, 112(r3)
+; CHECK-P9-NEXT:    stxv vs1, 96(r3)
+; CHECK-P9-NEXT:    stxv vs2, 80(r3)
+; CHECK-P9-NEXT:    stxv vs3, 64(r3)
+; CHECK-P9-NEXT:    stxv vs4, 48(r3)
+; CHECK-P9-NEXT:    stxv vs5, 32(r3)
+; CHECK-P9-NEXT:    stxv vs6, 16(r3)
+; CHECK-P9-NEXT:    stxv vs7, 0(r3)
 ; CHECK-P9-NEXT:    blr
 ;
 ; CHECK-BE-LABEL: test16elt_signed:
 ; CHECK-BE:       # %bb.0: # %entry
-; CHECK-BE-NEXT:    lxv vs0, 48(r4)
-; CHECK-BE-NEXT:    lxv vs1, 32(r4)
-; CHECK-BE-NEXT:    lxv vs2, 16(r4)
-; CHECK-BE-NEXT:    lxv vs3, 0(r4)
-; CHECK-BE-NEXT:    lxv vs4, 112(r4)
-; CHECK-BE-NEXT:    lxv vs5, 96(r4)
-; CHECK-BE-NEXT:    lxv vs6, 80(r4)
-; CHECK-BE-NEXT:    lxv vs7, 64(r4)
+; CHECK-BE-NEXT:    lxv vs0, 112(r4)
+; CHECK-BE-NEXT:    lxv vs1, 96(r4)
+; CHECK-BE-NEXT:    lxv vs2, 80(r4)
+; CHECK-BE-NEXT:    lxv vs3, 64(r4)
+; CHECK-BE-NEXT:    lxv vs4, 48(r4)
+; CHECK-BE-NEXT:    xvcvdpsxds vs4, vs4
+; CHECK-BE-NEXT:    lxv vs5, 32(r4)
+; CHECK-BE-NEXT:    lxv vs6, 16(r4)
+; CHECK-BE-NEXT:    lxv vs7, 0(r4)
+; CHECK-BE-NEXT:    xvcvdpsxds vs7, vs7
+; CHECK-BE-NEXT:    xvcvdpsxds vs6, vs6
+; CHECK-BE-NEXT:    xvcvdpsxds vs5, vs5
 ; CHECK-BE-NEXT:    xvcvdpsxds vs3, vs3
 ; CHECK-BE-NEXT:    xvcvdpsxds vs2, vs2
 ; CHECK-BE-NEXT:    xvcvdpsxds vs1, vs1
 ; CHECK-BE-NEXT:    xvcvdpsxds vs0, vs0
-; CHECK-BE-NEXT:    xvcvdpsxds vs7, vs7
-; CHECK-BE-NEXT:    xvcvdpsxds vs6, vs6
-; CHECK-BE-NEXT:    xvcvdpsxds vs5, vs5
-; CHECK-BE-NEXT:    xvcvdpsxds vs4, vs4
-; CHECK-BE-NEXT:    stxv vs0, 48(r3)
-; CHECK-BE-NEXT:    stxv vs1, 32(r3)
-; CHECK-BE-NEXT:    stxv vs2, 16(r3)
-; CHECK-BE-NEXT:    stxv vs3, 0(r3)
-; CHECK-BE-NEXT:    stxv vs4, 112(r3)
-; CHECK-BE-NEXT:    stxv vs5, 96(r3)
-; CHECK-BE-NEXT:    stxv vs6, 80(r3)
-; CHECK-BE-NEXT:    stxv vs7, 64(r3)
+; CHECK-BE-NEXT:    stxv vs0, 112(r3)
+; CHECK-BE-NEXT:    stxv vs1, 96(r3)
+; CHECK-BE-NEXT:    stxv vs2, 80(r3)
+; CHECK-BE-NEXT:    stxv vs3, 64(r3)
+; CHECK-BE-NEXT:    stxv vs4, 48(r3)
+; CHECK-BE-NEXT:    stxv vs5, 32(r3)
+; CHECK-BE-NEXT:    stxv vs6, 16(r3)
+; CHECK-BE-NEXT:    stxv vs7, 0(r3)
 ; CHECK-BE-NEXT:    blr
 entry:
   %a = load <16 x double>, <16 x double>* %0, align 128
diff --git a/test/CodeGen/PowerPC/vec_conv_i16_to_fp32_elts.ll b/test/CodeGen/PowerPC/vec_conv_i16_to_fp32_elts.ll
index b5221552ecd03ba21455aa50b09e9b577edf803c..f8538ac4cda6c3894e90838aeee32ece3b5e2b6b 100644
--- a/test/CodeGen/PowerPC/vec_conv_i16_to_fp32_elts.ll
+++ b/test/CodeGen/PowerPC/vec_conv_i16_to_fp32_elts.ll
@@ -35,20 +35,20 @@ define i64 @test2elt(i32 %a.coerce) local_unnamed_addr #0 {
 ; CHECK-P9:       # %bb.0: # %entry
 ; CHECK-P9-NEXT:    mtvsrws v2, r3
 ; CHECK-P9-NEXT:    li r3, 0
-; CHECK-P9-NEXT:    li r4, 2
 ; CHECK-P9-NEXT:    vextuhrx r3, r3, v2
-; CHECK-P9-NEXT:    vextuhrx r4, r4, v2
 ; CHECK-P9-NEXT:    rlwinm r3, r3, 0, 16, 31
-; CHECK-P9-NEXT:    rlwinm r4, r4, 0, 16, 31
 ; CHECK-P9-NEXT:    mtvsrwz f0, r3
-; CHECK-P9-NEXT:    mtvsrwz f1, r4
+; CHECK-P9-NEXT:    li r3, 2
+; CHECK-P9-NEXT:    xscvuxdsp f0, f0
+; CHECK-P9-NEXT:    xscvdpspn vs0, f0
+; CHECK-P9-NEXT:    vextuhrx r3, r3, v2
+; CHECK-P9-NEXT:    rlwinm r3, r3, 0, 16, 31
+; CHECK-P9-NEXT:    xxsldwi v3, vs0, vs0, 1
+; CHECK-P9-NEXT:    mtvsrwz f0, r3
 ; CHECK-P9-NEXT:    xscvuxdsp f0, f0
-; CHECK-P9-NEXT:    xscvuxdsp f1, f1
 ; CHECK-P9-NEXT:    xscvdpspn vs0, f0
-; CHECK-P9-NEXT:    xscvdpspn vs1, f1
 ; CHECK-P9-NEXT:    xxsldwi v2, vs0, vs0, 1
-; CHECK-P9-NEXT:    xxsldwi v3, vs1, vs1, 1
-; CHECK-P9-NEXT:    vmrglw v2, v3, v2
+; CHECK-P9-NEXT:    vmrglw v2, v2, v3
 ; CHECK-P9-NEXT:    mfvsrld r3, v2
 ; CHECK-P9-NEXT:    blr
 ;
@@ -56,18 +56,18 @@ define i64 @test2elt(i32 %a.coerce) local_unnamed_addr #0 {
 ; CHECK-BE:       # %bb.0: # %entry
 ; CHECK-BE-NEXT:    mtvsrws v2, r3
 ; CHECK-BE-NEXT:    li r3, 2
-; CHECK-BE-NEXT:    li r4, 0
 ; CHECK-BE-NEXT:    vextuhlx r3, r3, v2
-; CHECK-BE-NEXT:    vextuhlx r4, r4, v2
 ; CHECK-BE-NEXT:    rlwinm r3, r3, 0, 16, 31
-; CHECK-BE-NEXT:    rlwinm r4, r4, 0, 16, 31
 ; CHECK-BE-NEXT:    mtvsrwz f0, r3
-; CHECK-BE-NEXT:    mtvsrwz f1, r4
+; CHECK-BE-NEXT:    li r3, 0
+; CHECK-BE-NEXT:    xscvuxdsp f0, f0
+; CHECK-BE-NEXT:    vextuhlx r3, r3, v2
+; CHECK-BE-NEXT:    rlwinm r3, r3, 0, 16, 31
+; CHECK-BE-NEXT:    xscvdpspn v3, f0
+; CHECK-BE-NEXT:    mtvsrwz f0, r3
 ; CHECK-BE-NEXT:    xscvuxdsp f0, f0
-; CHECK-BE-NEXT:    xscvuxdsp f1, f1
 ; CHECK-BE-NEXT:    xscvdpspn v2, f0
-; CHECK-BE-NEXT:    xscvdpspn v3, f1
-; CHECK-BE-NEXT:    vmrghw v2, v3, v2
+; CHECK-BE-NEXT:    vmrghw v2, v2, v3
 ; CHECK-BE-NEXT:    mfvsrd r3, v2
 ; CHECK-BE-NEXT:    blr
 entry:
@@ -80,90 +80,37 @@ entry:
 define <4 x float> @test4elt(i64 %a.coerce) local_unnamed_addr #1 {
 ; CHECK-P8-LABEL: test4elt:
 ; CHECK-P8:       # %bb.0: # %entry
+; CHECK-P8-NEXT:    addis r4, r2, .LCPI1_0@toc@ha
 ; CHECK-P8-NEXT:    mtvsrd f0, r3
-; CHECK-P8-NEXT:    mfvsrd r3, f0
-; CHECK-P8-NEXT:    clrldi r4, r3, 48
-; CHECK-P8-NEXT:    rldicl r5, r3, 32, 48
-; CHECK-P8-NEXT:    rlwinm r4, r4, 0, 16, 31
-; CHECK-P8-NEXT:    rlwinm r5, r5, 0, 16, 31
-; CHECK-P8-NEXT:    mtvsrwz f0, r4
-; CHECK-P8-NEXT:    rldicl r4, r3, 48, 48
-; CHECK-P8-NEXT:    rldicl r3, r3, 16, 48
-; CHECK-P8-NEXT:    rlwinm r4, r4, 0, 16, 31
-; CHECK-P8-NEXT:    rlwinm r3, r3, 0, 16, 31
-; CHECK-P8-NEXT:    mtvsrwz f1, r5
-; CHECK-P8-NEXT:    mtvsrwz f2, r4
-; CHECK-P8-NEXT:    mtvsrwz f3, r3
-; CHECK-P8-NEXT:    xscvuxdsp f0, f0
-; CHECK-P8-NEXT:    xscvuxdsp f1, f1
-; CHECK-P8-NEXT:    xscvuxdsp f2, f2
-; CHECK-P8-NEXT:    xscvuxdsp f3, f3
-; CHECK-P8-NEXT:    xxmrghd vs0, vs1, vs0
-; CHECK-P8-NEXT:    xxmrghd vs1, vs3, vs2
-; CHECK-P8-NEXT:    xvcvdpsp v2, vs0
-; CHECK-P8-NEXT:    xvcvdpsp v3, vs1
-; CHECK-P8-NEXT:    vmrgew v2, v3, v2
+; CHECK-P8-NEXT:    addi r3, r4, .LCPI1_0@toc@l
+; CHECK-P8-NEXT:    xxlxor v4, v4, v4
+; CHECK-P8-NEXT:    xxswapd v2, vs0
+; CHECK-P8-NEXT:    lvx v3, 0, r3
+; CHECK-P8-NEXT:    vperm v2, v4, v2, v3
+; CHECK-P8-NEXT:    xvcvuxwsp v2, v2
 ; CHECK-P8-NEXT:    blr
 ;
 ; CHECK-P9-LABEL: test4elt:
 ; CHECK-P9:       # %bb.0: # %entry
 ; CHECK-P9-NEXT:    mtvsrd f0, r3
-; CHECK-P9-NEXT:    li r3, 0
-; CHECK-P9-NEXT:    li r4, 4
-; CHECK-P9-NEXT:    li r5, 2
-; CHECK-P9-NEXT:    li r6, 6
+; CHECK-P9-NEXT:    addis r3, r2, .LCPI1_0@toc@ha
+; CHECK-P9-NEXT:    addi r3, r3, .LCPI1_0@toc@l
+; CHECK-P9-NEXT:    lxvx v3, 0, r3
 ; CHECK-P9-NEXT:    xxswapd v2, vs0
-; CHECK-P9-NEXT:    vextuhrx r3, r3, v2
-; CHECK-P9-NEXT:    vextuhrx r4, r4, v2
-; CHECK-P9-NEXT:    vextuhrx r5, r5, v2
-; CHECK-P9-NEXT:    vextuhrx r6, r6, v2
-; CHECK-P9-NEXT:    rlwinm r3, r3, 0, 16, 31
-; CHECK-P9-NEXT:    rlwinm r4, r4, 0, 16, 31
-; CHECK-P9-NEXT:    rlwinm r5, r5, 0, 16, 31
-; CHECK-P9-NEXT:    rlwinm r6, r6, 0, 16, 31
-; CHECK-P9-NEXT:    mtvsrwz f0, r3
-; CHECK-P9-NEXT:    mtvsrwz f1, r4
-; CHECK-P9-NEXT:    mtvsrwz f2, r5
-; CHECK-P9-NEXT:    mtvsrwz f3, r6
-; CHECK-P9-NEXT:    xscvuxdsp f0, f0
-; CHECK-P9-NEXT:    xscvuxdsp f1, f1
-; CHECK-P9-NEXT:    xscvuxdsp f2, f2
-; CHECK-P9-NEXT:    xscvuxdsp f3, f3
-; CHECK-P9-NEXT:    xxmrghd vs0, vs1, vs0
-; CHECK-P9-NEXT:    xxmrghd vs1, vs3, vs2
-; CHECK-P9-NEXT:    xvcvdpsp v2, vs0
-; CHECK-P9-NEXT:    xvcvdpsp v3, vs1
-; CHECK-P9-NEXT:    vmrgew v2, v3, v2
+; CHECK-P9-NEXT:    xxlxor v4, v4, v4
+; CHECK-P9-NEXT:    vperm v2, v4, v2, v3
+; CHECK-P9-NEXT:    xvcvuxwsp v2, v2
 ; CHECK-P9-NEXT:    blr
 ;
 ; CHECK-BE-LABEL: test4elt:
 ; CHECK-BE:       # %bb.0: # %entry
-; CHECK-BE-NEXT:    li r4, 6
 ; CHECK-BE-NEXT:    mtvsrd v2, r3
-; CHECK-BE-NEXT:    li r3, 2
-; CHECK-BE-NEXT:    li r5, 4
-; CHECK-BE-NEXT:    li r6, 0
-; CHECK-BE-NEXT:    vextuhlx r4, r4, v2
-; CHECK-BE-NEXT:    vextuhlx r3, r3, v2
-; CHECK-BE-NEXT:    vextuhlx r5, r5, v2
-; CHECK-BE-NEXT:    vextuhlx r6, r6, v2
-; CHECK-BE-NEXT:    rlwinm r4, r4, 0, 16, 31
-; CHECK-BE-NEXT:    rlwinm r3, r3, 0, 16, 31
-; CHECK-BE-NEXT:    rlwinm r5, r5, 0, 16, 31
-; CHECK-BE-NEXT:    rlwinm r6, r6, 0, 16, 31
-; CHECK-BE-NEXT:    mtvsrwz f0, r4
-; CHECK-BE-NEXT:    mtvsrwz f1, r3
-; CHECK-BE-NEXT:    mtvsrwz f2, r5
-; CHECK-BE-NEXT:    mtvsrwz f3, r6
-; CHECK-BE-NEXT:    xscvuxdsp f0, f0
-; CHECK-BE-NEXT:    xscvuxdsp f1, f1
-; CHECK-BE-NEXT:    xscvuxdsp f2, f2
-; CHECK-BE-NEXT:    xscvuxdsp f3, f3
-; CHECK-BE-NEXT:    xxmrghd vs0, vs1, vs0
-; CHECK-BE-NEXT:    xxmrghd vs1, vs3, vs2
-; CHECK-BE-NEXT:    xvcvdpsp v2, vs0
-; CHECK-BE-NEXT:    xvcvdpsp v3, vs1
-; CHECK-BE-NEXT:    vmrgew v2, v3, v2
+; CHECK-BE-NEXT:    addis r3, r2, .LCPI1_0@toc@ha
+; CHECK-BE-NEXT:    addi r3, r3, .LCPI1_0@toc@l
+; CHECK-BE-NEXT:    lxvx v3, 0, r3
+; CHECK-BE-NEXT:    xxlxor v4, v4, v4
+; CHECK-BE-NEXT:    vperm v2, v2, v4, v3
+; CHECK-BE-NEXT:    xvcvuxwsp v2, v2
 ; CHECK-BE-NEXT:    blr
 entry:
   %0 = bitcast i64 %a.coerce to <4 x i16>
@@ -174,166 +121,54 @@ entry:
 define void @test8elt(<8 x float>* noalias nocapture sret %agg.result, <8 x i16> %a) local_unnamed_addr #2 {
 ; CHECK-P8-LABEL: test8elt:
 ; CHECK-P8:       # %bb.0: # %entry
-; CHECK-P8-NEXT:    mfvsrd r5, v2
-; CHECK-P8-NEXT:    xxswapd vs0, v2
+; CHECK-P8-NEXT:    addis r4, r2, .LCPI2_0@toc@ha
+; CHECK-P8-NEXT:    addis r5, r2, .LCPI2_1@toc@ha
+; CHECK-P8-NEXT:    xxlxor v4, v4, v4
+; CHECK-P8-NEXT:    addi r4, r4, .LCPI2_0@toc@l
+; CHECK-P8-NEXT:    lvx v3, 0, r4
+; CHECK-P8-NEXT:    addi r4, r5, .LCPI2_1@toc@l
+; CHECK-P8-NEXT:    lvx v5, 0, r4
 ; CHECK-P8-NEXT:    li r4, 16
-; CHECK-P8-NEXT:    clrldi r6, r5, 48
-; CHECK-P8-NEXT:    rldicl r7, r5, 32, 48
-; CHECK-P8-NEXT:    rlwinm r6, r6, 0, 16, 31
-; CHECK-P8-NEXT:    mfvsrd r8, f0
-; CHECK-P8-NEXT:    rlwinm r7, r7, 0, 16, 31
-; CHECK-P8-NEXT:    mtvsrwz f1, r6
-; CHECK-P8-NEXT:    rldicl r6, r5, 48, 48
-; CHECK-P8-NEXT:    rldicl r5, r5, 16, 48
-; CHECK-P8-NEXT:    rlwinm r6, r6, 0, 16, 31
-; CHECK-P8-NEXT:    rlwinm r5, r5, 0, 16, 31
-; CHECK-P8-NEXT:    mtvsrwz f0, r7
-; CHECK-P8-NEXT:    mtvsrwz f2, r6
-; CHECK-P8-NEXT:    clrldi r6, r8, 48
-; CHECK-P8-NEXT:    mtvsrwz f3, r5
-; CHECK-P8-NEXT:    rldicl r5, r8, 32, 48
-; CHECK-P8-NEXT:    rlwinm r6, r6, 0, 16, 31
-; CHECK-P8-NEXT:    mtvsrwz f4, r6
-; CHECK-P8-NEXT:    rldicl r6, r8, 48, 48
-; CHECK-P8-NEXT:    rlwinm r5, r5, 0, 16, 31
-; CHECK-P8-NEXT:    mtvsrwz f5, r5
-; CHECK-P8-NEXT:    rlwinm r5, r6, 0, 16, 31
-; CHECK-P8-NEXT:    mtvsrwz f6, r5
-; CHECK-P8-NEXT:    rldicl r5, r8, 16, 48
-; CHECK-P8-NEXT:    rlwinm r5, r5, 0, 16, 31
-; CHECK-P8-NEXT:    xscvuxdsp f1, f1
-; CHECK-P8-NEXT:    mtvsrwz f7, r5
-; CHECK-P8-NEXT:    xscvuxdsp f0, f0
-; CHECK-P8-NEXT:    xscvuxdsp f2, f2
-; CHECK-P8-NEXT:    xscvuxdsp f4, f4
-; CHECK-P8-NEXT:    xscvuxdsp f5, f5
-; CHECK-P8-NEXT:    xscvuxdsp f6, f6
-; CHECK-P8-NEXT:    xscvuxdsp f7, f7
-; CHECK-P8-NEXT:    xscvuxdsp f3, f3
-; CHECK-P8-NEXT:    xxmrghd vs0, vs0, vs1
-; CHECK-P8-NEXT:    xxmrghd vs1, vs5, vs4
-; CHECK-P8-NEXT:    xvcvdpsp v2, vs0
-; CHECK-P8-NEXT:    xxmrghd vs4, vs7, vs6
-; CHECK-P8-NEXT:    xxmrghd vs2, vs3, vs2
-; CHECK-P8-NEXT:    xvcvdpsp v3, vs1
-; CHECK-P8-NEXT:    xvcvdpsp v4, vs4
-; CHECK-P8-NEXT:    xvcvdpsp v5, vs2
-; CHECK-P8-NEXT:    vmrgew v3, v4, v3
-; CHECK-P8-NEXT:    vmrgew v2, v5, v2
+; CHECK-P8-NEXT:    vperm v3, v4, v2, v3
+; CHECK-P8-NEXT:    vperm v2, v4, v2, v5
+; CHECK-P8-NEXT:    xvcvuxwsp v3, v3
+; CHECK-P8-NEXT:    xvcvuxwsp v2, v2
 ; CHECK-P8-NEXT:    stvx v3, 0, r3
 ; CHECK-P8-NEXT:    stvx v2, r3, r4
 ; CHECK-P8-NEXT:    blr
 ;
 ; CHECK-P9-LABEL: test8elt:
 ; CHECK-P9:       # %bb.0: # %entry
-; CHECK-P9-NEXT:    li r4, 8
-; CHECK-P9-NEXT:    li r5, 12
-; CHECK-P9-NEXT:    li r6, 10
-; CHECK-P9-NEXT:    li r7, 14
-; CHECK-P9-NEXT:    li r8, 0
-; CHECK-P9-NEXT:    li r9, 4
-; CHECK-P9-NEXT:    li r10, 2
-; CHECK-P9-NEXT:    li r11, 6
-; CHECK-P9-NEXT:    vextuhrx r4, r4, v2
-; CHECK-P9-NEXT:    vextuhrx r5, r5, v2
-; CHECK-P9-NEXT:    vextuhrx r6, r6, v2
-; CHECK-P9-NEXT:    vextuhrx r7, r7, v2
-; CHECK-P9-NEXT:    vextuhrx r8, r8, v2
-; CHECK-P9-NEXT:    vextuhrx r9, r9, v2
-; CHECK-P9-NEXT:    vextuhrx r10, r10, v2
-; CHECK-P9-NEXT:    vextuhrx r11, r11, v2
-; CHECK-P9-NEXT:    rlwinm r4, r4, 0, 16, 31
-; CHECK-P9-NEXT:    rlwinm r5, r5, 0, 16, 31
-; CHECK-P9-NEXT:    rlwinm r6, r6, 0, 16, 31
-; CHECK-P9-NEXT:    rlwinm r7, r7, 0, 16, 31
-; CHECK-P9-NEXT:    rlwinm r8, r8, 0, 16, 31
-; CHECK-P9-NEXT:    rlwinm r9, r9, 0, 16, 31
-; CHECK-P9-NEXT:    rlwinm r10, r10, 0, 16, 31
-; CHECK-P9-NEXT:    rlwinm r11, r11, 0, 16, 31
-; CHECK-P9-NEXT:    mtvsrwz f0, r4
-; CHECK-P9-NEXT:    mtvsrwz f1, r5
-; CHECK-P9-NEXT:    mtvsrwz f2, r6
-; CHECK-P9-NEXT:    mtvsrwz f3, r7
-; CHECK-P9-NEXT:    mtvsrwz f4, r8
-; CHECK-P9-NEXT:    mtvsrwz f5, r9
-; CHECK-P9-NEXT:    mtvsrwz f6, r10
-; CHECK-P9-NEXT:    mtvsrwz f7, r11
-; CHECK-P9-NEXT:    xscvuxdsp f0, f0
-; CHECK-P9-NEXT:    xscvuxdsp f1, f1
-; CHECK-P9-NEXT:    xscvuxdsp f2, f2
-; CHECK-P9-NEXT:    xscvuxdsp f3, f3
-; CHECK-P9-NEXT:    xscvuxdsp f4, f4
-; CHECK-P9-NEXT:    xscvuxdsp f5, f5
-; CHECK-P9-NEXT:    xscvuxdsp f6, f6
-; CHECK-P9-NEXT:    xscvuxdsp f7, f7
-; CHECK-P9-NEXT:    xxmrghd vs0, vs1, vs0
-; CHECK-P9-NEXT:    xxmrghd vs1, vs3, vs2
-; CHECK-P9-NEXT:    xxmrghd vs2, vs5, vs4
-; CHECK-P9-NEXT:    xxmrghd vs3, vs7, vs6
-; CHECK-P9-NEXT:    xvcvdpsp v2, vs0
-; CHECK-P9-NEXT:    xvcvdpsp v3, vs1
-; CHECK-P9-NEXT:    xvcvdpsp v4, vs2
-; CHECK-P9-NEXT:    xvcvdpsp v5, vs3
-; CHECK-P9-NEXT:    vmrgew v2, v3, v2
-; CHECK-P9-NEXT:    vmrgew v3, v5, v4
-; CHECK-P9-NEXT:    stxv v3, 0(r3)
-; CHECK-P9-NEXT:    stxv v2, 16(r3)
+; CHECK-P9-NEXT:    addis r4, r2, .LCPI2_0@toc@ha
+; CHECK-P9-NEXT:    addi r4, r4, .LCPI2_0@toc@l
+; CHECK-P9-NEXT:    lxvx v3, 0, r4
+; CHECK-P9-NEXT:    xxlxor v4, v4, v4
+; CHECK-P9-NEXT:    addis r4, r2, .LCPI2_1@toc@ha
+; CHECK-P9-NEXT:    addi r4, r4, .LCPI2_1@toc@l
+; CHECK-P9-NEXT:    vperm v3, v4, v2, v3
+; CHECK-P9-NEXT:    xvcvuxwsp vs0, v3
+; CHECK-P9-NEXT:    lxvx v3, 0, r4
+; CHECK-P9-NEXT:    vperm v2, v4, v2, v3
+; CHECK-P9-NEXT:    stxv vs0, 0(r3)
+; CHECK-P9-NEXT:    xvcvuxwsp vs1, v2
+; CHECK-P9-NEXT:    stxv vs1, 16(r3)
 ; CHECK-P9-NEXT:    blr
 ;
 ; CHECK-BE-LABEL: test8elt:
 ; CHECK-BE:       # %bb.0: # %entry
-; CHECK-BE-NEXT:    li r4, 12
-; CHECK-BE-NEXT:    li r5, 8
-; CHECK-BE-NEXT:    li r6, 10
-; CHECK-BE-NEXT:    li r7, 14
-; CHECK-BE-NEXT:    li r8, 6
-; CHECK-BE-NEXT:    li r9, 2
-; CHECK-BE-NEXT:    li r10, 4
-; CHECK-BE-NEXT:    li r11, 0
-; CHECK-BE-NEXT:    vextuhlx r4, r4, v2
-; CHECK-BE-NEXT:    vextuhlx r5, r5, v2
-; CHECK-BE-NEXT:    vextuhlx r6, r6, v2
-; CHECK-BE-NEXT:    vextuhlx r7, r7, v2
-; CHECK-BE-NEXT:    vextuhlx r8, r8, v2
-; CHECK-BE-NEXT:    vextuhlx r9, r9, v2
-; CHECK-BE-NEXT:    vextuhlx r10, r10, v2
-; CHECK-BE-NEXT:    vextuhlx r11, r11, v2
-; CHECK-BE-NEXT:    rlwinm r4, r4, 0, 16, 31
-; CHECK-BE-NEXT:    rlwinm r5, r5, 0, 16, 31
-; CHECK-BE-NEXT:    rlwinm r6, r6, 0, 16, 31
-; CHECK-BE-NEXT:    rlwinm r7, r7, 0, 16, 31
-; CHECK-BE-NEXT:    rlwinm r8, r8, 0, 16, 31
-; CHECK-BE-NEXT:    rlwinm r9, r9, 0, 16, 31
-; CHECK-BE-NEXT:    rlwinm r10, r10, 0, 16, 31
-; CHECK-BE-NEXT:    rlwinm r11, r11, 0, 16, 31
-; CHECK-BE-NEXT:    mtvsrwz f0, r4
-; CHECK-BE-NEXT:    mtvsrwz f1, r5
-; CHECK-BE-NEXT:    mtvsrwz f2, r6
-; CHECK-BE-NEXT:    mtvsrwz f3, r7
-; CHECK-BE-NEXT:    mtvsrwz f4, r8
-; CHECK-BE-NEXT:    mtvsrwz f5, r9
-; CHECK-BE-NEXT:    mtvsrwz f6, r10
-; CHECK-BE-NEXT:    mtvsrwz f7, r11
-; CHECK-BE-NEXT:    xscvuxdsp f0, f0
-; CHECK-BE-NEXT:    xscvuxdsp f1, f1
-; CHECK-BE-NEXT:    xscvuxdsp f2, f2
-; CHECK-BE-NEXT:    xscvuxdsp f3, f3
-; CHECK-BE-NEXT:    xscvuxdsp f4, f4
-; CHECK-BE-NEXT:    xscvuxdsp f5, f5
-; CHECK-BE-NEXT:    xscvuxdsp f6, f6
-; CHECK-BE-NEXT:    xscvuxdsp f7, f7
-; CHECK-BE-NEXT:    xxmrghd vs0, vs1, vs0
-; CHECK-BE-NEXT:    xxmrghd vs1, vs2, vs3
-; CHECK-BE-NEXT:    xxmrghd vs2, vs5, vs4
-; CHECK-BE-NEXT:    xxmrghd vs3, vs7, vs6
-; CHECK-BE-NEXT:    xvcvdpsp v2, vs0
-; CHECK-BE-NEXT:    xvcvdpsp v3, vs1
-; CHECK-BE-NEXT:    xvcvdpsp v4, vs2
-; CHECK-BE-NEXT:    xvcvdpsp v5, vs3
-; CHECK-BE-NEXT:    vmrgew v2, v2, v3
-; CHECK-BE-NEXT:    vmrgew v3, v5, v4
-; CHECK-BE-NEXT:    stxv v3, 0(r3)
-; CHECK-BE-NEXT:    stxv v2, 16(r3)
+; CHECK-BE-NEXT:    addis r4, r2, .LCPI2_0@toc@ha
+; CHECK-BE-NEXT:    addi r4, r4, .LCPI2_0@toc@l
+; CHECK-BE-NEXT:    lxvx v3, 0, r4
+; CHECK-BE-NEXT:    xxlxor v4, v4, v4
+; CHECK-BE-NEXT:    addis r4, r2, .LCPI2_1@toc@ha
+; CHECK-BE-NEXT:    addi r4, r4, .LCPI2_1@toc@l
+; CHECK-BE-NEXT:    vperm v3, v2, v4, v3
+; CHECK-BE-NEXT:    xvcvuxwsp vs0, v3
+; CHECK-BE-NEXT:    lxvx v3, 0, r4
+; CHECK-BE-NEXT:    vperm v2, v4, v2, v3
+; CHECK-BE-NEXT:    stxv vs0, 0(r3)
+; CHECK-BE-NEXT:    xvcvuxwsp vs1, v2
+; CHECK-BE-NEXT:    stxv vs1, 16(r3)
 ; CHECK-BE-NEXT:    blr
 entry:
   %0 = uitofp <8 x i16> %a to <8 x float>
@@ -344,341 +179,80 @@ entry:
 define void @test16elt(<16 x float>* noalias nocapture sret %agg.result, <16 x i16>* nocapture readonly) local_unnamed_addr #3 {
 ; CHECK-P8-LABEL: test16elt:
 ; CHECK-P8:       # %bb.0: # %entry
+; CHECK-P8-NEXT:    addis r5, r2, .LCPI3_0@toc@ha
+; CHECK-P8-NEXT:    addis r6, r2, .LCPI3_1@toc@ha
+; CHECK-P8-NEXT:    xxlxor v3, v3, v3
+; CHECK-P8-NEXT:    lvx v4, 0, r4
+; CHECK-P8-NEXT:    addi r5, r5, .LCPI3_0@toc@l
+; CHECK-P8-NEXT:    addi r6, r6, .LCPI3_1@toc@l
+; CHECK-P8-NEXT:    lvx v2, 0, r5
 ; CHECK-P8-NEXT:    li r5, 16
-; CHECK-P8-NEXT:    lvx v3, 0, r4
-; CHECK-P8-NEXT:    lvx v2, r4, r5
-; CHECK-P8-NEXT:    mfvsrd r7, v3
-; CHECK-P8-NEXT:    xxswapd vs8, v3
-; CHECK-P8-NEXT:    mfvsrd r6, v2
-; CHECK-P8-NEXT:    xxswapd vs2, v2
-; CHECK-P8-NEXT:    clrldi r4, r6, 48
-; CHECK-P8-NEXT:    rldicl r8, r6, 32, 48
-; CHECK-P8-NEXT:    rlwinm r4, r4, 0, 16, 31
-; CHECK-P8-NEXT:    rlwinm r8, r8, 0, 16, 31
-; CHECK-P8-NEXT:    mtvsrwz f0, r4
-; CHECK-P8-NEXT:    rldicl r4, r6, 48, 48
-; CHECK-P8-NEXT:    rldicl r6, r6, 16, 48
-; CHECK-P8-NEXT:    rlwinm r4, r4, 0, 16, 31
-; CHECK-P8-NEXT:    mtvsrwz f1, r8
-; CHECK-P8-NEXT:    clrldi r8, r7, 48
-; CHECK-P8-NEXT:    rlwinm r6, r6, 0, 16, 31
-; CHECK-P8-NEXT:    mtvsrwz f3, r4
-; CHECK-P8-NEXT:    rlwinm r4, r8, 0, 16, 31
-; CHECK-P8-NEXT:    mtvsrwz f4, r6
-; CHECK-P8-NEXT:    rldicl r6, r7, 32, 48
-; CHECK-P8-NEXT:    mtvsrwz f5, r4
-; CHECK-P8-NEXT:    rldicl r4, r7, 48, 48
-; CHECK-P8-NEXT:    rlwinm r6, r6, 0, 16, 31
-; CHECK-P8-NEXT:    mfvsrd r8, f2
-; CHECK-P8-NEXT:    rlwinm r4, r4, 0, 16, 31
-; CHECK-P8-NEXT:    mtvsrwz f2, r6
-; CHECK-P8-NEXT:    rldicl r6, r7, 16, 48
-; CHECK-P8-NEXT:    mtvsrwz f6, r4
-; CHECK-P8-NEXT:    clrldi r4, r8, 48
-; CHECK-P8-NEXT:    rlwinm r6, r6, 0, 16, 31
-; CHECK-P8-NEXT:    rlwinm r4, r4, 0, 16, 31
-; CHECK-P8-NEXT:    mtvsrwz f7, r6
-; CHECK-P8-NEXT:    rldicl r6, r8, 32, 48
-; CHECK-P8-NEXT:    mtvsrwz f9, r4
-; CHECK-P8-NEXT:    rldicl r4, r8, 48, 48
-; CHECK-P8-NEXT:    rlwinm r6, r6, 0, 16, 31
-; CHECK-P8-NEXT:    rlwinm r4, r4, 0, 16, 31
-; CHECK-P8-NEXT:    mtvsrwz f10, r6
-; CHECK-P8-NEXT:    rldicl r6, r8, 16, 48
-; CHECK-P8-NEXT:    mtvsrwz f11, r4
-; CHECK-P8-NEXT:    rlwinm r6, r6, 0, 16, 31
-; CHECK-P8-NEXT:    mfvsrd r4, f8
-; CHECK-P8-NEXT:    mtvsrwz f8, r6
-; CHECK-P8-NEXT:    clrldi r6, r4, 48
-; CHECK-P8-NEXT:    xscvuxdsp f0, f0
-; CHECK-P8-NEXT:    rlwinm r6, r6, 0, 16, 31
-; CHECK-P8-NEXT:    xscvuxdsp f1, f1
-; CHECK-P8-NEXT:    xscvuxdsp f3, f3
-; CHECK-P8-NEXT:    xscvuxdsp f4, f4
-; CHECK-P8-NEXT:    mtvsrwz f12, r6
-; CHECK-P8-NEXT:    rldicl r6, r4, 32, 48
-; CHECK-P8-NEXT:    rlwinm r6, r6, 0, 16, 31
-; CHECK-P8-NEXT:    xscvuxdsp f5, f5
-; CHECK-P8-NEXT:    mtvsrwz f13, r6
-; CHECK-P8-NEXT:    rldicl r6, r4, 48, 48
-; CHECK-P8-NEXT:    rldicl r4, r4, 16, 48
-; CHECK-P8-NEXT:    xscvuxdsp f2, f2
-; CHECK-P8-NEXT:    rlwinm r6, r6, 0, 16, 31
-; CHECK-P8-NEXT:    rlwinm r4, r4, 0, 16, 31
-; CHECK-P8-NEXT:    xxmrghd vs0, vs1, vs0
-; CHECK-P8-NEXT:    mtvsrwz v2, r6
-; CHECK-P8-NEXT:    mtvsrwz v3, r4
-; CHECK-P8-NEXT:    xxmrghd vs3, vs4, vs3
+; CHECK-P8-NEXT:    lvx v0, 0, r6
+; CHECK-P8-NEXT:    li r6, 32
+; CHECK-P8-NEXT:    lvx v5, r4, r5
 ; CHECK-P8-NEXT:    li r4, 48
-; CHECK-P8-NEXT:    xscvuxdsp f6, f6
-; CHECK-P8-NEXT:    xscvuxdsp f7, f7
-; CHECK-P8-NEXT:    xscvuxdsp f9, f9
-; CHECK-P8-NEXT:    xscvuxdsp f10, f10
-; CHECK-P8-NEXT:    xxmrghd vs2, vs2, vs5
-; CHECK-P8-NEXT:    xscvuxdsp f11, f11
-; CHECK-P8-NEXT:    xscvuxdsp f8, f8
-; CHECK-P8-NEXT:    xscvuxdsp f12, f12
-; CHECK-P8-NEXT:    xscvuxdsp f13, f13
-; CHECK-P8-NEXT:    xxmrghd vs5, vs7, vs6
-; CHECK-P8-NEXT:    xscvuxdsp f1, v2
-; CHECK-P8-NEXT:    xscvuxdsp f4, v3
-; CHECK-P8-NEXT:    xvcvdpsp v2, vs0
-; CHECK-P8-NEXT:    xxmrghd vs0, vs10, vs9
-; CHECK-P8-NEXT:    xvcvdpsp v3, vs3
-; CHECK-P8-NEXT:    xxmrghd vs3, vs8, vs11
-; CHECK-P8-NEXT:    xvcvdpsp v4, vs2
-; CHECK-P8-NEXT:    xxmrghd vs2, vs13, vs12
-; CHECK-P8-NEXT:    xvcvdpsp v5, vs5
-; CHECK-P8-NEXT:    xvcvdpsp v0, vs0
-; CHECK-P8-NEXT:    xxmrghd vs1, vs4, vs1
-; CHECK-P8-NEXT:    xvcvdpsp v1, vs3
-; CHECK-P8-NEXT:    xvcvdpsp v6, vs2
-; CHECK-P8-NEXT:    vmrgew v2, v3, v2
-; CHECK-P8-NEXT:    xvcvdpsp v7, vs1
-; CHECK-P8-NEXT:    vmrgew v3, v5, v4
-; CHECK-P8-NEXT:    vmrgew v4, v1, v0
-; CHECK-P8-NEXT:    stvx v2, r3, r4
-; CHECK-P8-NEXT:    li r4, 32
-; CHECK-P8-NEXT:    vmrgew v5, v7, v6
+; CHECK-P8-NEXT:    vperm v1, v3, v4, v2
+; CHECK-P8-NEXT:    vperm v2, v3, v5, v2
+; CHECK-P8-NEXT:    vperm v5, v3, v5, v0
+; CHECK-P8-NEXT:    vperm v3, v3, v4, v0
+; CHECK-P8-NEXT:    xvcvuxwsp v4, v1
+; CHECK-P8-NEXT:    xvcvuxwsp v2, v2
+; CHECK-P8-NEXT:    xvcvuxwsp v5, v5
+; CHECK-P8-NEXT:    xvcvuxwsp v3, v3
+; CHECK-P8-NEXT:    stvx v4, 0, r3
+; CHECK-P8-NEXT:    stvx v2, r3, r6
+; CHECK-P8-NEXT:    stvx v5, r3, r4
 ; CHECK-P8-NEXT:    stvx v3, r3, r5
-; CHECK-P8-NEXT:    stvx v4, r3, r4
-; CHECK-P8-NEXT:    stvx v5, 0, r3
 ; CHECK-P8-NEXT:    blr
 ;
 ; CHECK-P9-LABEL: test16elt:
 ; CHECK-P9:       # %bb.0: # %entry
-; CHECK-P9-NEXT:    lxv v3, 0(r4)
 ; CHECK-P9-NEXT:    lxv v2, 16(r4)
-; CHECK-P9-NEXT:    li r4, 0
-; CHECK-P9-NEXT:    li r5, 4
-; CHECK-P9-NEXT:    li r6, 2
-; CHECK-P9-NEXT:    li r7, 6
-; CHECK-P9-NEXT:    li r8, 8
-; CHECK-P9-NEXT:    std r25, -72(r1) # 8-byte Folded Spill
-; CHECK-P9-NEXT:    li r9, 12
-; CHECK-P9-NEXT:    li r10, 10
-; CHECK-P9-NEXT:    li r11, 14
-; CHECK-P9-NEXT:    std r26, -64(r1) # 8-byte Folded Spill
-; CHECK-P9-NEXT:    std r27, -56(r1) # 8-byte Folded Spill
-; CHECK-P9-NEXT:    std r28, -48(r1) # 8-byte Folded Spill
-; CHECK-P9-NEXT:    std r29, -40(r1) # 8-byte Folded Spill
-; CHECK-P9-NEXT:    std r30, -32(r1) # 8-byte Folded Spill
-; CHECK-P9-NEXT:    stfd f30, -16(r1) # 8-byte Folded Spill
-; CHECK-P9-NEXT:    stfd f31, -8(r1) # 8-byte Folded Spill
-; CHECK-P9-NEXT:    vextuhrx r12, r4, v3
-; CHECK-P9-NEXT:    vextuhrx r0, r5, v3
-; CHECK-P9-NEXT:    vextuhrx r30, r6, v3
-; CHECK-P9-NEXT:    vextuhrx r29, r7, v3
-; CHECK-P9-NEXT:    vextuhrx r28, r8, v3
-; CHECK-P9-NEXT:    vextuhrx r27, r9, v3
-; CHECK-P9-NEXT:    vextuhrx r26, r10, v3
-; CHECK-P9-NEXT:    vextuhrx r25, r11, v3
-; CHECK-P9-NEXT:    vextuhrx r4, r4, v2
-; CHECK-P9-NEXT:    vextuhrx r5, r5, v2
-; CHECK-P9-NEXT:    vextuhrx r6, r6, v2
-; CHECK-P9-NEXT:    vextuhrx r7, r7, v2
-; CHECK-P9-NEXT:    vextuhrx r8, r8, v2
-; CHECK-P9-NEXT:    vextuhrx r9, r9, v2
-; CHECK-P9-NEXT:    vextuhrx r10, r10, v2
-; CHECK-P9-NEXT:    vextuhrx r11, r11, v2
-; CHECK-P9-NEXT:    rlwinm r12, r12, 0, 16, 31
-; CHECK-P9-NEXT:    rlwinm r0, r0, 0, 16, 31
-; CHECK-P9-NEXT:    rlwinm r30, r30, 0, 16, 31
-; CHECK-P9-NEXT:    rlwinm r29, r29, 0, 16, 31
-; CHECK-P9-NEXT:    rlwinm r28, r28, 0, 16, 31
-; CHECK-P9-NEXT:    rlwinm r27, r27, 0, 16, 31
-; CHECK-P9-NEXT:    rlwinm r26, r26, 0, 16, 31
-; CHECK-P9-NEXT:    rlwinm r25, r25, 0, 16, 31
-; CHECK-P9-NEXT:    rlwinm r4, r4, 0, 16, 31
-; CHECK-P9-NEXT:    rlwinm r5, r5, 0, 16, 31
-; CHECK-P9-NEXT:    rlwinm r6, r6, 0, 16, 31
-; CHECK-P9-NEXT:    rlwinm r7, r7, 0, 16, 31
-; CHECK-P9-NEXT:    rlwinm r8, r8, 0, 16, 31
-; CHECK-P9-NEXT:    rlwinm r9, r9, 0, 16, 31
-; CHECK-P9-NEXT:    rlwinm r10, r10, 0, 16, 31
-; CHECK-P9-NEXT:    rlwinm r11, r11, 0, 16, 31
-; CHECK-P9-NEXT:    mtvsrwz f0, r12
-; CHECK-P9-NEXT:    mtvsrwz f1, r0
-; CHECK-P9-NEXT:    mtvsrwz f2, r30
-; CHECK-P9-NEXT:    ld r30, -32(r1) # 8-byte Folded Reload
-; CHECK-P9-NEXT:    mtvsrwz f3, r29
-; CHECK-P9-NEXT:    ld r29, -40(r1) # 8-byte Folded Reload
-; CHECK-P9-NEXT:    mtvsrwz f4, r28
-; CHECK-P9-NEXT:    ld r28, -48(r1) # 8-byte Folded Reload
-; CHECK-P9-NEXT:    mtvsrwz f5, r27
-; CHECK-P9-NEXT:    ld r27, -56(r1) # 8-byte Folded Reload
-; CHECK-P9-NEXT:    mtvsrwz f6, r26
-; CHECK-P9-NEXT:    ld r26, -64(r1) # 8-byte Folded Reload
-; CHECK-P9-NEXT:    mtvsrwz f7, r25
-; CHECK-P9-NEXT:    ld r25, -72(r1) # 8-byte Folded Reload
-; CHECK-P9-NEXT:    mtvsrwz f8, r4
-; CHECK-P9-NEXT:    mtvsrwz f9, r5
-; CHECK-P9-NEXT:    mtvsrwz f10, r6
-; CHECK-P9-NEXT:    mtvsrwz f11, r7
-; CHECK-P9-NEXT:    mtvsrwz f12, r8
-; CHECK-P9-NEXT:    mtvsrwz f13, r9
-; CHECK-P9-NEXT:    mtvsrwz v2, r10
-; CHECK-P9-NEXT:    mtvsrwz v3, r11
-; CHECK-P9-NEXT:    xscvuxdsp f0, f0
-; CHECK-P9-NEXT:    xscvuxdsp f1, f1
-; CHECK-P9-NEXT:    xscvuxdsp f2, f2
-; CHECK-P9-NEXT:    xscvuxdsp f3, f3
-; CHECK-P9-NEXT:    xscvuxdsp f4, f4
-; CHECK-P9-NEXT:    xscvuxdsp f5, f5
-; CHECK-P9-NEXT:    xscvuxdsp f6, f6
-; CHECK-P9-NEXT:    xscvuxdsp f7, f7
-; CHECK-P9-NEXT:    xscvuxdsp f8, f8
-; CHECK-P9-NEXT:    xscvuxdsp f9, f9
-; CHECK-P9-NEXT:    xscvuxdsp f10, f10
-; CHECK-P9-NEXT:    xscvuxdsp f11, f11
-; CHECK-P9-NEXT:    xscvuxdsp f12, f12
-; CHECK-P9-NEXT:    xscvuxdsp f13, f13
-; CHECK-P9-NEXT:    xscvuxdsp f31, v2
-; CHECK-P9-NEXT:    xscvuxdsp f30, v3
-; CHECK-P9-NEXT:    xxmrghd vs0, vs1, vs0
-; CHECK-P9-NEXT:    xxmrghd vs1, vs3, vs2
-; CHECK-P9-NEXT:    xxmrghd vs2, vs5, vs4
-; CHECK-P9-NEXT:    xxmrghd vs3, vs7, vs6
-; CHECK-P9-NEXT:    xxmrghd vs4, vs9, vs8
-; CHECK-P9-NEXT:    xxmrghd vs5, vs11, vs10
-; CHECK-P9-NEXT:    xxmrghd vs6, vs13, vs12
-; CHECK-P9-NEXT:    xxmrghd vs7, vs30, vs31
-; CHECK-P9-NEXT:    lfd f31, -8(r1) # 8-byte Folded Reload
-; CHECK-P9-NEXT:    lfd f30, -16(r1) # 8-byte Folded Reload
-; CHECK-P9-NEXT:    xvcvdpsp v2, vs0
-; CHECK-P9-NEXT:    xvcvdpsp v3, vs1
-; CHECK-P9-NEXT:    xvcvdpsp v4, vs2
-; CHECK-P9-NEXT:    xvcvdpsp v5, vs3
-; CHECK-P9-NEXT:    xvcvdpsp v0, vs4
-; CHECK-P9-NEXT:    xvcvdpsp v1, vs5
-; CHECK-P9-NEXT:    xvcvdpsp v6, vs6
-; CHECK-P9-NEXT:    xvcvdpsp v7, vs7
-; CHECK-P9-NEXT:    vmrgew v2, v3, v2
-; CHECK-P9-NEXT:    vmrgew v3, v5, v4
-; CHECK-P9-NEXT:    vmrgew v4, v1, v0
-; CHECK-P9-NEXT:    vmrgew v5, v7, v6
-; CHECK-P9-NEXT:    stxv v3, 16(r3)
-; CHECK-P9-NEXT:    stxv v2, 0(r3)
-; CHECK-P9-NEXT:    stxv v5, 48(r3)
-; CHECK-P9-NEXT:    stxv v4, 32(r3)
+; CHECK-P9-NEXT:    lxv v3, 0(r4)
+; CHECK-P9-NEXT:    addis r4, r2, .LCPI3_0@toc@ha
+; CHECK-P9-NEXT:    addi r4, r4, .LCPI3_0@toc@l
+; CHECK-P9-NEXT:    lxvx v4, 0, r4
+; CHECK-P9-NEXT:    xxlxor v5, v5, v5
+; CHECK-P9-NEXT:    addis r4, r2, .LCPI3_1@toc@ha
+; CHECK-P9-NEXT:    addi r4, r4, .LCPI3_1@toc@l
+; CHECK-P9-NEXT:    vperm v0, v5, v3, v4
+; CHECK-P9-NEXT:    xvcvuxwsp vs0, v0
+; CHECK-P9-NEXT:    lxvx v0, 0, r4
+; CHECK-P9-NEXT:    vperm v3, v5, v3, v0
+; CHECK-P9-NEXT:    stxv vs0, 0(r3)
+; CHECK-P9-NEXT:    xvcvuxwsp vs1, v3
+; CHECK-P9-NEXT:    vperm v3, v5, v2, v4
+; CHECK-P9-NEXT:    vperm v2, v5, v2, v0
+; CHECK-P9-NEXT:    xvcvuxwsp vs2, v3
+; CHECK-P9-NEXT:    xvcvuxwsp vs3, v2
+; CHECK-P9-NEXT:    stxv vs1, 16(r3)
+; CHECK-P9-NEXT:    stxv vs3, 48(r3)
+; CHECK-P9-NEXT:    stxv vs2, 32(r3)
 ; CHECK-P9-NEXT:    blr
 ;
 ; CHECK-BE-LABEL: test16elt:
 ; CHECK-BE:       # %bb.0: # %entry
-; CHECK-BE-NEXT:    lxv v3, 0(r4)
 ; CHECK-BE-NEXT:    lxv v2, 16(r4)
-; CHECK-BE-NEXT:    li r4, 6
-; CHECK-BE-NEXT:    li r5, 2
-; CHECK-BE-NEXT:    li r6, 4
-; CHECK-BE-NEXT:    li r7, 0
-; CHECK-BE-NEXT:    li r8, 14
-; CHECK-BE-NEXT:    std r25, -72(r1) # 8-byte Folded Spill
-; CHECK-BE-NEXT:    li r9, 10
-; CHECK-BE-NEXT:    li r10, 12
-; CHECK-BE-NEXT:    li r11, 8
-; CHECK-BE-NEXT:    std r26, -64(r1) # 8-byte Folded Spill
-; CHECK-BE-NEXT:    std r27, -56(r1) # 8-byte Folded Spill
-; CHECK-BE-NEXT:    std r28, -48(r1) # 8-byte Folded Spill
-; CHECK-BE-NEXT:    std r29, -40(r1) # 8-byte Folded Spill
-; CHECK-BE-NEXT:    std r30, -32(r1) # 8-byte Folded Spill
-; CHECK-BE-NEXT:    stfd f30, -16(r1) # 8-byte Folded Spill
-; CHECK-BE-NEXT:    stfd f31, -8(r1) # 8-byte Folded Spill
-; CHECK-BE-NEXT:    vextuhlx r12, r4, v3
-; CHECK-BE-NEXT:    vextuhlx r0, r5, v3
-; CHECK-BE-NEXT:    vextuhlx r30, r6, v3
-; CHECK-BE-NEXT:    vextuhlx r29, r7, v3
-; CHECK-BE-NEXT:    vextuhlx r28, r8, v3
-; CHECK-BE-NEXT:    vextuhlx r27, r9, v3
-; CHECK-BE-NEXT:    vextuhlx r26, r10, v3
-; CHECK-BE-NEXT:    vextuhlx r25, r11, v3
-; CHECK-BE-NEXT:    vextuhlx r4, r4, v2
-; CHECK-BE-NEXT:    vextuhlx r5, r5, v2
-; CHECK-BE-NEXT:    vextuhlx r6, r6, v2
-; CHECK-BE-NEXT:    vextuhlx r7, r7, v2
-; CHECK-BE-NEXT:    vextuhlx r8, r8, v2
-; CHECK-BE-NEXT:    vextuhlx r9, r9, v2
-; CHECK-BE-NEXT:    vextuhlx r10, r10, v2
-; CHECK-BE-NEXT:    vextuhlx r11, r11, v2
-; CHECK-BE-NEXT:    rlwinm r12, r12, 0, 16, 31
-; CHECK-BE-NEXT:    rlwinm r0, r0, 0, 16, 31
-; CHECK-BE-NEXT:    rlwinm r30, r30, 0, 16, 31
-; CHECK-BE-NEXT:    rlwinm r29, r29, 0, 16, 31
-; CHECK-BE-NEXT:    rlwinm r28, r28, 0, 16, 31
-; CHECK-BE-NEXT:    rlwinm r27, r27, 0, 16, 31
-; CHECK-BE-NEXT:    rlwinm r26, r26, 0, 16, 31
-; CHECK-BE-NEXT:    rlwinm r25, r25, 0, 16, 31
-; CHECK-BE-NEXT:    rlwinm r4, r4, 0, 16, 31
-; CHECK-BE-NEXT:    rlwinm r5, r5, 0, 16, 31
-; CHECK-BE-NEXT:    rlwinm r6, r6, 0, 16, 31
-; CHECK-BE-NEXT:    rlwinm r7, r7, 0, 16, 31
-; CHECK-BE-NEXT:    rlwinm r8, r8, 0, 16, 31
-; CHECK-BE-NEXT:    rlwinm r9, r9, 0, 16, 31
-; CHECK-BE-NEXT:    rlwinm r10, r10, 0, 16, 31
-; CHECK-BE-NEXT:    rlwinm r11, r11, 0, 16, 31
-; CHECK-BE-NEXT:    mtvsrwz f0, r12
-; CHECK-BE-NEXT:    mtvsrwz f1, r0
-; CHECK-BE-NEXT:    mtvsrwz f2, r30
-; CHECK-BE-NEXT:    ld r30, -32(r1) # 8-byte Folded Reload
-; CHECK-BE-NEXT:    mtvsrwz f3, r29
-; CHECK-BE-NEXT:    ld r29, -40(r1) # 8-byte Folded Reload
-; CHECK-BE-NEXT:    mtvsrwz f4, r28
-; CHECK-BE-NEXT:    ld r28, -48(r1) # 8-byte Folded Reload
-; CHECK-BE-NEXT:    mtvsrwz f5, r27
-; CHECK-BE-NEXT:    ld r27, -56(r1) # 8-byte Folded Reload
-; CHECK-BE-NEXT:    mtvsrwz f6, r26
-; CHECK-BE-NEXT:    ld r26, -64(r1) # 8-byte Folded Reload
-; CHECK-BE-NEXT:    mtvsrwz f7, r25
-; CHECK-BE-NEXT:    ld r25, -72(r1) # 8-byte Folded Reload
-; CHECK-BE-NEXT:    mtvsrwz f8, r4
-; CHECK-BE-NEXT:    mtvsrwz f9, r5
-; CHECK-BE-NEXT:    mtvsrwz f10, r6
-; CHECK-BE-NEXT:    mtvsrwz f11, r7
-; CHECK-BE-NEXT:    mtvsrwz f12, r8
-; CHECK-BE-NEXT:    mtvsrwz f13, r9
-; CHECK-BE-NEXT:    mtvsrwz v2, r10
-; CHECK-BE-NEXT:    mtvsrwz v3, r11
-; CHECK-BE-NEXT:    xscvuxdsp f0, f0
-; CHECK-BE-NEXT:    xscvuxdsp f1, f1
-; CHECK-BE-NEXT:    xscvuxdsp f2, f2
-; CHECK-BE-NEXT:    xscvuxdsp f3, f3
-; CHECK-BE-NEXT:    xscvuxdsp f4, f4
-; CHECK-BE-NEXT:    xscvuxdsp f5, f5
-; CHECK-BE-NEXT:    xscvuxdsp f6, f6
-; CHECK-BE-NEXT:    xscvuxdsp f7, f7
-; CHECK-BE-NEXT:    xscvuxdsp f8, f8
-; CHECK-BE-NEXT:    xscvuxdsp f9, f9
-; CHECK-BE-NEXT:    xscvuxdsp f10, f10
-; CHECK-BE-NEXT:    xscvuxdsp f11, f11
-; CHECK-BE-NEXT:    xscvuxdsp f12, f12
-; CHECK-BE-NEXT:    xscvuxdsp f13, f13
-; CHECK-BE-NEXT:    xscvuxdsp f31, v2
-; CHECK-BE-NEXT:    xscvuxdsp f30, v3
-; CHECK-BE-NEXT:    xxmrghd vs0, vs1, vs0
-; CHECK-BE-NEXT:    xxmrghd vs1, vs3, vs2
-; CHECK-BE-NEXT:    xxmrghd vs2, vs5, vs4
-; CHECK-BE-NEXT:    xxmrghd vs3, vs7, vs6
-; CHECK-BE-NEXT:    xxmrghd vs4, vs9, vs8
-; CHECK-BE-NEXT:    xxmrghd vs5, vs11, vs10
-; CHECK-BE-NEXT:    xxmrghd vs6, vs13, vs12
-; CHECK-BE-NEXT:    xxmrghd vs7, vs30, vs31
-; CHECK-BE-NEXT:    lfd f31, -8(r1) # 8-byte Folded Reload
-; CHECK-BE-NEXT:    lfd f30, -16(r1) # 8-byte Folded Reload
-; CHECK-BE-NEXT:    xvcvdpsp v2, vs0
-; CHECK-BE-NEXT:    xvcvdpsp v3, vs1
-; CHECK-BE-NEXT:    xvcvdpsp v4, vs2
-; CHECK-BE-NEXT:    xvcvdpsp v5, vs3
-; CHECK-BE-NEXT:    xvcvdpsp v0, vs4
-; CHECK-BE-NEXT:    xvcvdpsp v1, vs5
-; CHECK-BE-NEXT:    xvcvdpsp v6, vs6
-; CHECK-BE-NEXT:    xvcvdpsp v7, vs7
-; CHECK-BE-NEXT:    vmrgew v2, v3, v2
-; CHECK-BE-NEXT:    vmrgew v3, v5, v4
-; CHECK-BE-NEXT:    vmrgew v4, v1, v0
-; CHECK-BE-NEXT:    vmrgew v5, v7, v6
-; CHECK-BE-NEXT:    stxv v3, 16(r3)
-; CHECK-BE-NEXT:    stxv v2, 0(r3)
-; CHECK-BE-NEXT:    stxv v5, 48(r3)
-; CHECK-BE-NEXT:    stxv v4, 32(r3)
+; CHECK-BE-NEXT:    lxv v3, 0(r4)
+; CHECK-BE-NEXT:    addis r4, r2, .LCPI3_0@toc@ha
+; CHECK-BE-NEXT:    addi r4, r4, .LCPI3_0@toc@l
+; CHECK-BE-NEXT:    lxvx v4, 0, r4
+; CHECK-BE-NEXT:    xxlxor v5, v5, v5
+; CHECK-BE-NEXT:    addis r4, r2, .LCPI3_1@toc@ha
+; CHECK-BE-NEXT:    addi r4, r4, .LCPI3_1@toc@l
+; CHECK-BE-NEXT:    vperm v0, v3, v5, v4
+; CHECK-BE-NEXT:    xvcvuxwsp vs0, v0
+; CHECK-BE-NEXT:    lxvx v0, 0, r4
+; CHECK-BE-NEXT:    vperm v3, v5, v3, v0
+; CHECK-BE-NEXT:    stxv vs0, 0(r3)
+; CHECK-BE-NEXT:    xvcvuxwsp vs1, v3
+; CHECK-BE-NEXT:    vperm v3, v2, v5, v4
+; CHECK-BE-NEXT:    vperm v2, v5, v2, v0
+; CHECK-BE-NEXT:    xvcvuxwsp vs2, v3
+; CHECK-BE-NEXT:    xvcvuxwsp vs3, v2
+; CHECK-BE-NEXT:    stxv vs1, 16(r3)
+; CHECK-BE-NEXT:    stxv vs3, 48(r3)
+; CHECK-BE-NEXT:    stxv vs2, 32(r3)
 ; CHECK-BE-NEXT:    blr
 entry:
   %a = load <16 x i16>, <16 x i16>* %0, align 32
@@ -713,20 +287,20 @@ define i64 @test2elt_signed(i32 %a.coerce) local_unnamed_addr #0 {
 ; CHECK-P9:       # %bb.0: # %entry
 ; CHECK-P9-NEXT:    mtvsrws v2, r3
 ; CHECK-P9-NEXT:    li r3, 0
-; CHECK-P9-NEXT:    li r4, 2
 ; CHECK-P9-NEXT:    vextuhrx r3, r3, v2
-; CHECK-P9-NEXT:    vextuhrx r4, r4, v2
 ; CHECK-P9-NEXT:    extsh r3, r3
-; CHECK-P9-NEXT:    extsh r4, r4
 ; CHECK-P9-NEXT:    mtvsrwa f0, r3
-; CHECK-P9-NEXT:    mtvsrwa f1, r4
+; CHECK-P9-NEXT:    li r3, 2
+; CHECK-P9-NEXT:    xscvsxdsp f0, f0
+; CHECK-P9-NEXT:    xscvdpspn vs0, f0
+; CHECK-P9-NEXT:    vextuhrx r3, r3, v2
+; CHECK-P9-NEXT:    extsh r3, r3
+; CHECK-P9-NEXT:    xxsldwi v3, vs0, vs0, 1
+; CHECK-P9-NEXT:    mtvsrwa f0, r3
 ; CHECK-P9-NEXT:    xscvsxdsp f0, f0
-; CHECK-P9-NEXT:    xscvsxdsp f1, f1
 ; CHECK-P9-NEXT:    xscvdpspn vs0, f0
-; CHECK-P9-NEXT:    xscvdpspn vs1, f1
 ; CHECK-P9-NEXT:    xxsldwi v2, vs0, vs0, 1
-; CHECK-P9-NEXT:    xxsldwi v3, vs1, vs1, 1
-; CHECK-P9-NEXT:    vmrglw v2, v3, v2
+; CHECK-P9-NEXT:    vmrglw v2, v2, v3
 ; CHECK-P9-NEXT:    mfvsrld r3, v2
 ; CHECK-P9-NEXT:    blr
 ;
@@ -734,18 +308,18 @@ define i64 @test2elt_signed(i32 %a.coerce) local_unnamed_addr #0 {
 ; CHECK-BE:       # %bb.0: # %entry
 ; CHECK-BE-NEXT:    mtvsrws v2, r3
 ; CHECK-BE-NEXT:    li r3, 2
-; CHECK-BE-NEXT:    li r4, 0
 ; CHECK-BE-NEXT:    vextuhlx r3, r3, v2
-; CHECK-BE-NEXT:    vextuhlx r4, r4, v2
 ; CHECK-BE-NEXT:    extsh r3, r3
-; CHECK-BE-NEXT:    extsh r4, r4
 ; CHECK-BE-NEXT:    mtvsrwa f0, r3
-; CHECK-BE-NEXT:    mtvsrwa f1, r4
+; CHECK-BE-NEXT:    li r3, 0
+; CHECK-BE-NEXT:    xscvsxdsp f0, f0
+; CHECK-BE-NEXT:    vextuhlx r3, r3, v2
+; CHECK-BE-NEXT:    extsh r3, r3
+; CHECK-BE-NEXT:    xscvdpspn v3, f0
+; CHECK-BE-NEXT:    mtvsrwa f0, r3
 ; CHECK-BE-NEXT:    xscvsxdsp f0, f0
-; CHECK-BE-NEXT:    xscvsxdsp f1, f1
 ; CHECK-BE-NEXT:    xscvdpspn v2, f0
-; CHECK-BE-NEXT:    xscvdpspn v3, f1
-; CHECK-BE-NEXT:    vmrghw v2, v3, v2
+; CHECK-BE-NEXT:    vmrghw v2, v2, v3
 ; CHECK-BE-NEXT:    mfvsrd r3, v2
 ; CHECK-BE-NEXT:    blr
 entry:
@@ -759,89 +333,30 @@ define <4 x float> @test4elt_signed(i64 %a.coerce) local_unnamed_addr #1 {
 ; CHECK-P8-LABEL: test4elt_signed:
 ; CHECK-P8:       # %bb.0: # %entry
 ; CHECK-P8-NEXT:    mtvsrd f0, r3
-; CHECK-P8-NEXT:    mfvsrd r3, f0
-; CHECK-P8-NEXT:    clrldi r4, r3, 48
-; CHECK-P8-NEXT:    rldicl r5, r3, 32, 48
-; CHECK-P8-NEXT:    extsh r4, r4
-; CHECK-P8-NEXT:    extsh r5, r5
-; CHECK-P8-NEXT:    mtvsrwa f0, r4
-; CHECK-P8-NEXT:    rldicl r4, r3, 48, 48
-; CHECK-P8-NEXT:    rldicl r3, r3, 16, 48
-; CHECK-P8-NEXT:    extsh r4, r4
-; CHECK-P8-NEXT:    extsh r3, r3
-; CHECK-P8-NEXT:    mtvsrwa f1, r5
-; CHECK-P8-NEXT:    mtvsrwa f2, r4
-; CHECK-P8-NEXT:    mtvsrwa f3, r3
-; CHECK-P8-NEXT:    xscvsxdsp f0, f0
-; CHECK-P8-NEXT:    xscvsxdsp f1, f1
-; CHECK-P8-NEXT:    xscvsxdsp f2, f2
-; CHECK-P8-NEXT:    xscvsxdsp f3, f3
-; CHECK-P8-NEXT:    xxmrghd vs0, vs1, vs0
-; CHECK-P8-NEXT:    xxmrghd vs1, vs3, vs2
-; CHECK-P8-NEXT:    xvcvdpsp v2, vs0
-; CHECK-P8-NEXT:    xvcvdpsp v3, vs1
-; CHECK-P8-NEXT:    vmrgew v2, v3, v2
+; CHECK-P8-NEXT:    vspltisw v3, 8
+; CHECK-P8-NEXT:    xxswapd v2, vs0
+; CHECK-P8-NEXT:    vadduwm v3, v3, v3
+; CHECK-P8-NEXT:    vmrglh v2, v2, v2
+; CHECK-P8-NEXT:    vslw v2, v2, v3
+; CHECK-P8-NEXT:    vsraw v2, v2, v3
+; CHECK-P8-NEXT:    xvcvsxwsp v2, v2
 ; CHECK-P8-NEXT:    blr
 ;
 ; CHECK-P9-LABEL: test4elt_signed:
 ; CHECK-P9:       # %bb.0: # %entry
 ; CHECK-P9-NEXT:    mtvsrd f0, r3
-; CHECK-P9-NEXT:    li r3, 0
-; CHECK-P9-NEXT:    li r4, 4
-; CHECK-P9-NEXT:    li r5, 2
-; CHECK-P9-NEXT:    li r6, 6
 ; CHECK-P9-NEXT:    xxswapd v2, vs0
-; CHECK-P9-NEXT:    vextuhrx r3, r3, v2
-; CHECK-P9-NEXT:    vextuhrx r4, r4, v2
-; CHECK-P9-NEXT:    vextuhrx r5, r5, v2
-; CHECK-P9-NEXT:    vextuhrx r6, r6, v2
-; CHECK-P9-NEXT:    extsh r3, r3
-; CHECK-P9-NEXT:    extsh r4, r4
-; CHECK-P9-NEXT:    extsh r5, r5
-; CHECK-P9-NEXT:    extsh r6, r6
-; CHECK-P9-NEXT:    mtvsrwa f0, r3
-; CHECK-P9-NEXT:    mtvsrwa f1, r4
-; CHECK-P9-NEXT:    mtvsrwa f2, r5
-; CHECK-P9-NEXT:    mtvsrwa f3, r6
-; CHECK-P9-NEXT:    xscvsxdsp f0, f0
-; CHECK-P9-NEXT:    xscvsxdsp f1, f1
-; CHECK-P9-NEXT:    xscvsxdsp f2, f2
-; CHECK-P9-NEXT:    xscvsxdsp f3, f3
-; CHECK-P9-NEXT:    xxmrghd vs0, vs1, vs0
-; CHECK-P9-NEXT:    xxmrghd vs1, vs3, vs2
-; CHECK-P9-NEXT:    xvcvdpsp v2, vs0
-; CHECK-P9-NEXT:    xvcvdpsp v3, vs1
-; CHECK-P9-NEXT:    vmrgew v2, v3, v2
+; CHECK-P9-NEXT:    vmrglh v2, v2, v2
+; CHECK-P9-NEXT:    vextsh2w v2, v2
+; CHECK-P9-NEXT:    xvcvsxwsp v2, v2
 ; CHECK-P9-NEXT:    blr
 ;
 ; CHECK-BE-LABEL: test4elt_signed:
 ; CHECK-BE:       # %bb.0: # %entry
-; CHECK-BE-NEXT:    li r4, 6
 ; CHECK-BE-NEXT:    mtvsrd v2, r3
-; CHECK-BE-NEXT:    li r3, 2
-; CHECK-BE-NEXT:    li r5, 4
-; CHECK-BE-NEXT:    li r6, 0
-; CHECK-BE-NEXT:    vextuhlx r4, r4, v2
-; CHECK-BE-NEXT:    vextuhlx r3, r3, v2
-; CHECK-BE-NEXT:    vextuhlx r5, r5, v2
-; CHECK-BE-NEXT:    vextuhlx r6, r6, v2
-; CHECK-BE-NEXT:    extsh r4, r4
-; CHECK-BE-NEXT:    extsh r3, r3
-; CHECK-BE-NEXT:    extsh r5, r5
-; CHECK-BE-NEXT:    extsh r6, r6
-; CHECK-BE-NEXT:    mtvsrwa f0, r4
-; CHECK-BE-NEXT:    mtvsrwa f1, r3
-; CHECK-BE-NEXT:    mtvsrwa f2, r5
-; CHECK-BE-NEXT:    mtvsrwa f3, r6
-; CHECK-BE-NEXT:    xscvsxdsp f0, f0
-; CHECK-BE-NEXT:    xscvsxdsp f1, f1
-; CHECK-BE-NEXT:    xscvsxdsp f2, f2
-; CHECK-BE-NEXT:    xscvsxdsp f3, f3
-; CHECK-BE-NEXT:    xxmrghd vs0, vs1, vs0
-; CHECK-BE-NEXT:    xxmrghd vs1, vs3, vs2
-; CHECK-BE-NEXT:    xvcvdpsp v2, vs0
-; CHECK-BE-NEXT:    xvcvdpsp v3, vs1
-; CHECK-BE-NEXT:    vmrgew v2, v3, v2
+; CHECK-BE-NEXT:    vmrghh v2, v2, v2
+; CHECK-BE-NEXT:    vextsh2w v2, v2
+; CHECK-BE-NEXT:    xvcvsxwsp v2, v2
 ; CHECK-BE-NEXT:    blr
 entry:
   %0 = bitcast i64 %a.coerce to <4 x i16>
@@ -852,166 +367,47 @@ entry:
 define void @test8elt_signed(<8 x float>* noalias nocapture sret %agg.result, <8 x i16> %a) local_unnamed_addr #2 {
 ; CHECK-P8-LABEL: test8elt_signed:
 ; CHECK-P8:       # %bb.0: # %entry
-; CHECK-P8-NEXT:    mfvsrd r5, v2
-; CHECK-P8-NEXT:    xxswapd vs0, v2
+; CHECK-P8-NEXT:    vmrglh v4, v2, v2
+; CHECK-P8-NEXT:    vspltisw v3, 8
 ; CHECK-P8-NEXT:    li r4, 16
-; CHECK-P8-NEXT:    clrldi r6, r5, 48
-; CHECK-P8-NEXT:    rldicl r7, r5, 32, 48
-; CHECK-P8-NEXT:    extsh r6, r6
-; CHECK-P8-NEXT:    mfvsrd r8, f0
-; CHECK-P8-NEXT:    extsh r7, r7
-; CHECK-P8-NEXT:    mtvsrwa f1, r6
-; CHECK-P8-NEXT:    rldicl r6, r5, 48, 48
-; CHECK-P8-NEXT:    rldicl r5, r5, 16, 48
-; CHECK-P8-NEXT:    extsh r6, r6
-; CHECK-P8-NEXT:    extsh r5, r5
-; CHECK-P8-NEXT:    mtvsrwa f0, r7
-; CHECK-P8-NEXT:    mtvsrwa f2, r6
-; CHECK-P8-NEXT:    clrldi r6, r8, 48
-; CHECK-P8-NEXT:    mtvsrwa f3, r5
-; CHECK-P8-NEXT:    rldicl r5, r8, 32, 48
-; CHECK-P8-NEXT:    extsh r6, r6
-; CHECK-P8-NEXT:    mtvsrwa f4, r6
-; CHECK-P8-NEXT:    rldicl r6, r8, 48, 48
-; CHECK-P8-NEXT:    extsh r5, r5
-; CHECK-P8-NEXT:    mtvsrwa f5, r5
-; CHECK-P8-NEXT:    extsh r5, r6
-; CHECK-P8-NEXT:    mtvsrwa f6, r5
-; CHECK-P8-NEXT:    rldicl r5, r8, 16, 48
-; CHECK-P8-NEXT:    extsh r5, r5
-; CHECK-P8-NEXT:    xscvsxdsp f1, f1
-; CHECK-P8-NEXT:    mtvsrwa f7, r5
-; CHECK-P8-NEXT:    xscvsxdsp f0, f0
-; CHECK-P8-NEXT:    xscvsxdsp f2, f2
-; CHECK-P8-NEXT:    xscvsxdsp f4, f4
-; CHECK-P8-NEXT:    xscvsxdsp f5, f5
-; CHECK-P8-NEXT:    xscvsxdsp f6, f6
-; CHECK-P8-NEXT:    xscvsxdsp f7, f7
-; CHECK-P8-NEXT:    xscvsxdsp f3, f3
-; CHECK-P8-NEXT:    xxmrghd vs0, vs0, vs1
-; CHECK-P8-NEXT:    xxmrghd vs1, vs5, vs4
-; CHECK-P8-NEXT:    xvcvdpsp v2, vs0
-; CHECK-P8-NEXT:    xxmrghd vs4, vs7, vs6
-; CHECK-P8-NEXT:    xxmrghd vs2, vs3, vs2
-; CHECK-P8-NEXT:    xvcvdpsp v3, vs1
-; CHECK-P8-NEXT:    xvcvdpsp v4, vs4
-; CHECK-P8-NEXT:    xvcvdpsp v5, vs2
-; CHECK-P8-NEXT:    vmrgew v3, v4, v3
-; CHECK-P8-NEXT:    vmrgew v2, v5, v2
+; CHECK-P8-NEXT:    vmrghh v2, v2, v2
+; CHECK-P8-NEXT:    vadduwm v3, v3, v3
+; CHECK-P8-NEXT:    vslw v4, v4, v3
+; CHECK-P8-NEXT:    vslw v2, v2, v3
+; CHECK-P8-NEXT:    vsraw v4, v4, v3
+; CHECK-P8-NEXT:    vsraw v2, v2, v3
+; CHECK-P8-NEXT:    xvcvsxwsp v3, v4
+; CHECK-P8-NEXT:    xvcvsxwsp v2, v2
 ; CHECK-P8-NEXT:    stvx v3, 0, r3
 ; CHECK-P8-NEXT:    stvx v2, r3, r4
 ; CHECK-P8-NEXT:    blr
 ;
 ; CHECK-P9-LABEL: test8elt_signed:
 ; CHECK-P9:       # %bb.0: # %entry
-; CHECK-P9-NEXT:    li r4, 8
-; CHECK-P9-NEXT:    li r5, 12
-; CHECK-P9-NEXT:    li r6, 10
-; CHECK-P9-NEXT:    li r7, 14
-; CHECK-P9-NEXT:    li r8, 0
-; CHECK-P9-NEXT:    li r9, 4
-; CHECK-P9-NEXT:    li r10, 2
-; CHECK-P9-NEXT:    li r11, 6
-; CHECK-P9-NEXT:    vextuhrx r4, r4, v2
-; CHECK-P9-NEXT:    vextuhrx r5, r5, v2
-; CHECK-P9-NEXT:    vextuhrx r6, r6, v2
-; CHECK-P9-NEXT:    vextuhrx r7, r7, v2
-; CHECK-P9-NEXT:    vextuhrx r8, r8, v2
-; CHECK-P9-NEXT:    vextuhrx r9, r9, v2
-; CHECK-P9-NEXT:    vextuhrx r10, r10, v2
-; CHECK-P9-NEXT:    vextuhrx r11, r11, v2
-; CHECK-P9-NEXT:    extsh r4, r4
-; CHECK-P9-NEXT:    extsh r5, r5
-; CHECK-P9-NEXT:    extsh r6, r6
-; CHECK-P9-NEXT:    extsh r7, r7
-; CHECK-P9-NEXT:    extsh r8, r8
-; CHECK-P9-NEXT:    extsh r9, r9
-; CHECK-P9-NEXT:    extsh r10, r10
-; CHECK-P9-NEXT:    extsh r11, r11
-; CHECK-P9-NEXT:    mtvsrwa f0, r4
-; CHECK-P9-NEXT:    mtvsrwa f1, r5
-; CHECK-P9-NEXT:    mtvsrwa f2, r6
-; CHECK-P9-NEXT:    mtvsrwa f3, r7
-; CHECK-P9-NEXT:    mtvsrwa f4, r8
-; CHECK-P9-NEXT:    mtvsrwa f5, r9
-; CHECK-P9-NEXT:    mtvsrwa f6, r10
-; CHECK-P9-NEXT:    mtvsrwa f7, r11
-; CHECK-P9-NEXT:    xscvsxdsp f0, f0
-; CHECK-P9-NEXT:    xscvsxdsp f1, f1
-; CHECK-P9-NEXT:    xscvsxdsp f2, f2
-; CHECK-P9-NEXT:    xscvsxdsp f3, f3
-; CHECK-P9-NEXT:    xscvsxdsp f4, f4
-; CHECK-P9-NEXT:    xscvsxdsp f5, f5
-; CHECK-P9-NEXT:    xscvsxdsp f6, f6
-; CHECK-P9-NEXT:    xscvsxdsp f7, f7
-; CHECK-P9-NEXT:    xxmrghd vs0, vs1, vs0
-; CHECK-P9-NEXT:    xxmrghd vs1, vs3, vs2
-; CHECK-P9-NEXT:    xxmrghd vs2, vs5, vs4
-; CHECK-P9-NEXT:    xxmrghd vs3, vs7, vs6
-; CHECK-P9-NEXT:    xvcvdpsp v2, vs0
-; CHECK-P9-NEXT:    xvcvdpsp v3, vs1
-; CHECK-P9-NEXT:    xvcvdpsp v4, vs2
-; CHECK-P9-NEXT:    xvcvdpsp v5, vs3
-; CHECK-P9-NEXT:    vmrgew v2, v3, v2
-; CHECK-P9-NEXT:    vmrgew v3, v5, v4
-; CHECK-P9-NEXT:    stxv v3, 0(r3)
-; CHECK-P9-NEXT:    stxv v2, 16(r3)
+; CHECK-P9-NEXT:    vmrglh v3, v2, v2
+; CHECK-P9-NEXT:    vmrghh v2, v2, v2
+; CHECK-P9-NEXT:    vextsh2w v3, v3
+; CHECK-P9-NEXT:    vextsh2w v2, v2
+; CHECK-P9-NEXT:    xvcvsxwsp vs0, v3
+; CHECK-P9-NEXT:    xvcvsxwsp vs1, v2
+; CHECK-P9-NEXT:    stxv vs1, 16(r3)
+; CHECK-P9-NEXT:    stxv vs0, 0(r3)
 ; CHECK-P9-NEXT:    blr
 ;
 ; CHECK-BE-LABEL: test8elt_signed:
 ; CHECK-BE:       # %bb.0: # %entry
-; CHECK-BE-NEXT:    li r4, 12
-; CHECK-BE-NEXT:    li r5, 8
-; CHECK-BE-NEXT:    li r6, 10
-; CHECK-BE-NEXT:    li r7, 14
-; CHECK-BE-NEXT:    li r8, 6
-; CHECK-BE-NEXT:    li r9, 2
-; CHECK-BE-NEXT:    li r10, 4
-; CHECK-BE-NEXT:    li r11, 0
-; CHECK-BE-NEXT:    vextuhlx r4, r4, v2
-; CHECK-BE-NEXT:    vextuhlx r5, r5, v2
-; CHECK-BE-NEXT:    vextuhlx r6, r6, v2
-; CHECK-BE-NEXT:    vextuhlx r7, r7, v2
-; CHECK-BE-NEXT:    vextuhlx r8, r8, v2
-; CHECK-BE-NEXT:    vextuhlx r9, r9, v2
-; CHECK-BE-NEXT:    vextuhlx r10, r10, v2
-; CHECK-BE-NEXT:    vextuhlx r11, r11, v2
-; CHECK-BE-NEXT:    extsh r4, r4
-; CHECK-BE-NEXT:    extsh r5, r5
-; CHECK-BE-NEXT:    extsh r6, r6
-; CHECK-BE-NEXT:    extsh r7, r7
-; CHECK-BE-NEXT:    extsh r8, r8
-; CHECK-BE-NEXT:    extsh r9, r9
-; CHECK-BE-NEXT:    extsh r10, r10
-; CHECK-BE-NEXT:    extsh r11, r11
-; CHECK-BE-NEXT:    mtvsrwa f0, r4
-; CHECK-BE-NEXT:    mtvsrwa f1, r5
-; CHECK-BE-NEXT:    mtvsrwa f2, r6
-; CHECK-BE-NEXT:    mtvsrwa f3, r7
-; CHECK-BE-NEXT:    mtvsrwa f4, r8
-; CHECK-BE-NEXT:    mtvsrwa f5, r9
-; CHECK-BE-NEXT:    mtvsrwa f6, r10
-; CHECK-BE-NEXT:    mtvsrwa f7, r11
-; CHECK-BE-NEXT:    xscvsxdsp f0, f0
-; CHECK-BE-NEXT:    xscvsxdsp f1, f1
-; CHECK-BE-NEXT:    xscvsxdsp f2, f2
-; CHECK-BE-NEXT:    xscvsxdsp f3, f3
-; CHECK-BE-NEXT:    xscvsxdsp f4, f4
-; CHECK-BE-NEXT:    xscvsxdsp f5, f5
-; CHECK-BE-NEXT:    xscvsxdsp f6, f6
-; CHECK-BE-NEXT:    xscvsxdsp f7, f7
-; CHECK-BE-NEXT:    xxmrghd vs0, vs1, vs0
-; CHECK-BE-NEXT:    xxmrghd vs1, vs2, vs3
-; CHECK-BE-NEXT:    xxmrghd vs2, vs5, vs4
-; CHECK-BE-NEXT:    xxmrghd vs3, vs7, vs6
-; CHECK-BE-NEXT:    xvcvdpsp v2, vs0
-; CHECK-BE-NEXT:    xvcvdpsp v3, vs1
-; CHECK-BE-NEXT:    xvcvdpsp v4, vs2
-; CHECK-BE-NEXT:    xvcvdpsp v5, vs3
-; CHECK-BE-NEXT:    vmrgew v2, v2, v3
-; CHECK-BE-NEXT:    vmrgew v3, v5, v4
-; CHECK-BE-NEXT:    stxv v3, 0(r3)
-; CHECK-BE-NEXT:    stxv v2, 16(r3)
+; CHECK-BE-NEXT:    addis r4, r2, .LCPI6_0@toc@ha
+; CHECK-BE-NEXT:    addi r4, r4, .LCPI6_0@toc@l
+; CHECK-BE-NEXT:    lxvx v3, 0, r4
+; CHECK-BE-NEXT:    xxlxor v4, v4, v4
+; CHECK-BE-NEXT:    vperm v3, v4, v2, v3
+; CHECK-BE-NEXT:    vmrghh v2, v2, v2
+; CHECK-BE-NEXT:    vextsh2w v3, v3
+; CHECK-BE-NEXT:    vextsh2w v2, v2
+; CHECK-BE-NEXT:    xvcvsxwsp vs0, v3
+; CHECK-BE-NEXT:    xvcvsxwsp vs1, v2
+; CHECK-BE-NEXT:    stxv vs1, 0(r3)
+; CHECK-BE-NEXT:    stxv vs0, 16(r3)
 ; CHECK-BE-NEXT:    blr
 entry:
   %0 = sitofp <8 x i16> %a to <8 x float>
@@ -1023,340 +419,80 @@ define void @test16elt_signed(<16 x float>* noalias nocapture sret %agg.result,
 ; CHECK-P8-LABEL: test16elt_signed:
 ; CHECK-P8:       # %bb.0: # %entry
 ; CHECK-P8-NEXT:    li r5, 16
-; CHECK-P8-NEXT:    lvx v3, 0, r4
-; CHECK-P8-NEXT:    lvx v2, r4, r5
-; CHECK-P8-NEXT:    mfvsrd r7, v3
-; CHECK-P8-NEXT:    xxswapd vs8, v3
-; CHECK-P8-NEXT:    mfvsrd r6, v2
-; CHECK-P8-NEXT:    xxswapd vs2, v2
-; CHECK-P8-NEXT:    clrldi r4, r6, 48
-; CHECK-P8-NEXT:    rldicl r8, r6, 32, 48
-; CHECK-P8-NEXT:    extsh r4, r4
-; CHECK-P8-NEXT:    extsh r8, r8
-; CHECK-P8-NEXT:    mtvsrwa f0, r4
-; CHECK-P8-NEXT:    rldicl r4, r6, 48, 48
-; CHECK-P8-NEXT:    rldicl r6, r6, 16, 48
-; CHECK-P8-NEXT:    extsh r4, r4
-; CHECK-P8-NEXT:    mtvsrwa f1, r8
-; CHECK-P8-NEXT:    clrldi r8, r7, 48
-; CHECK-P8-NEXT:    extsh r6, r6
-; CHECK-P8-NEXT:    mtvsrwa f3, r4
-; CHECK-P8-NEXT:    extsh r4, r8
-; CHECK-P8-NEXT:    mtvsrwa f4, r6
-; CHECK-P8-NEXT:    rldicl r6, r7, 32, 48
-; CHECK-P8-NEXT:    mtvsrwa f5, r4
-; CHECK-P8-NEXT:    rldicl r4, r7, 48, 48
-; CHECK-P8-NEXT:    extsh r6, r6
-; CHECK-P8-NEXT:    mfvsrd r8, f2
-; CHECK-P8-NEXT:    extsh r4, r4
-; CHECK-P8-NEXT:    mtvsrwa f2, r6
-; CHECK-P8-NEXT:    rldicl r6, r7, 16, 48
-; CHECK-P8-NEXT:    mtvsrwa f6, r4
-; CHECK-P8-NEXT:    clrldi r4, r8, 48
-; CHECK-P8-NEXT:    extsh r6, r6
-; CHECK-P8-NEXT:    extsh r4, r4
-; CHECK-P8-NEXT:    mtvsrwa f7, r6
-; CHECK-P8-NEXT:    rldicl r6, r8, 32, 48
-; CHECK-P8-NEXT:    mtvsrwa f9, r4
-; CHECK-P8-NEXT:    rldicl r4, r8, 48, 48
-; CHECK-P8-NEXT:    extsh r6, r6
-; CHECK-P8-NEXT:    extsh r4, r4
-; CHECK-P8-NEXT:    mtvsrwa f10, r6
-; CHECK-P8-NEXT:    rldicl r6, r8, 16, 48
-; CHECK-P8-NEXT:    mtvsrwa f11, r4
-; CHECK-P8-NEXT:    extsh r6, r6
-; CHECK-P8-NEXT:    mfvsrd r4, f8
-; CHECK-P8-NEXT:    mtvsrwa f8, r6
-; CHECK-P8-NEXT:    clrldi r6, r4, 48
-; CHECK-P8-NEXT:    xscvsxdsp f0, f0
-; CHECK-P8-NEXT:    extsh r6, r6
-; CHECK-P8-NEXT:    xscvsxdsp f1, f1
-; CHECK-P8-NEXT:    xscvsxdsp f3, f3
-; CHECK-P8-NEXT:    xscvsxdsp f4, f4
-; CHECK-P8-NEXT:    mtvsrwa f12, r6
-; CHECK-P8-NEXT:    rldicl r6, r4, 32, 48
-; CHECK-P8-NEXT:    extsh r6, r6
-; CHECK-P8-NEXT:    xscvsxdsp f5, f5
-; CHECK-P8-NEXT:    mtvsrwa f13, r6
-; CHECK-P8-NEXT:    rldicl r6, r4, 48, 48
-; CHECK-P8-NEXT:    rldicl r4, r4, 16, 48
-; CHECK-P8-NEXT:    xscvsxdsp f2, f2
-; CHECK-P8-NEXT:    extsh r6, r6
-; CHECK-P8-NEXT:    extsh r4, r4
-; CHECK-P8-NEXT:    xxmrghd vs0, vs1, vs0
-; CHECK-P8-NEXT:    mtvsrwa v2, r6
-; CHECK-P8-NEXT:    mtvsrwa v3, r4
-; CHECK-P8-NEXT:    xxmrghd vs3, vs4, vs3
+; CHECK-P8-NEXT:    lvx v2, 0, r4
+; CHECK-P8-NEXT:    vspltisw v5, 8
+; CHECK-P8-NEXT:    li r6, 32
+; CHECK-P8-NEXT:    lvx v3, r4, r5
 ; CHECK-P8-NEXT:    li r4, 48
-; CHECK-P8-NEXT:    xscvsxdsp f6, f6
-; CHECK-P8-NEXT:    xscvsxdsp f7, f7
-; CHECK-P8-NEXT:    xscvsxdsp f9, f9
-; CHECK-P8-NEXT:    xscvsxdsp f10, f10
-; CHECK-P8-NEXT:    xxmrghd vs2, vs2, vs5
-; CHECK-P8-NEXT:    xscvsxdsp f11, f11
-; CHECK-P8-NEXT:    xscvsxdsp f8, f8
-; CHECK-P8-NEXT:    xscvsxdsp f12, f12
-; CHECK-P8-NEXT:    xscvsxdsp f13, f13
-; CHECK-P8-NEXT:    xxmrghd vs5, vs7, vs6
-; CHECK-P8-NEXT:    xscvsxdsp f1, v2
-; CHECK-P8-NEXT:    xscvsxdsp f4, v3
-; CHECK-P8-NEXT:    xvcvdpsp v2, vs0
-; CHECK-P8-NEXT:    xxmrghd vs0, vs10, vs9
-; CHECK-P8-NEXT:    xvcvdpsp v3, vs3
-; CHECK-P8-NEXT:    xxmrghd vs3, vs8, vs11
-; CHECK-P8-NEXT:    xvcvdpsp v4, vs2
-; CHECK-P8-NEXT:    xxmrghd vs2, vs13, vs12
-; CHECK-P8-NEXT:    xvcvdpsp v5, vs5
-; CHECK-P8-NEXT:    xvcvdpsp v0, vs0
-; CHECK-P8-NEXT:    xxmrghd vs1, vs4, vs1
-; CHECK-P8-NEXT:    xvcvdpsp v1, vs3
-; CHECK-P8-NEXT:    xvcvdpsp v6, vs2
-; CHECK-P8-NEXT:    vmrgew v2, v3, v2
-; CHECK-P8-NEXT:    xvcvdpsp v7, vs1
-; CHECK-P8-NEXT:    vmrgew v3, v5, v4
-; CHECK-P8-NEXT:    vmrgew v4, v1, v0
-; CHECK-P8-NEXT:    stvx v2, r3, r4
-; CHECK-P8-NEXT:    li r4, 32
-; CHECK-P8-NEXT:    vmrgew v5, v7, v6
-; CHECK-P8-NEXT:    stvx v3, r3, r5
-; CHECK-P8-NEXT:    stvx v4, r3, r4
-; CHECK-P8-NEXT:    stvx v5, 0, r3
+; CHECK-P8-NEXT:    vmrglh v4, v2, v2
+; CHECK-P8-NEXT:    vmrglh v0, v3, v3
+; CHECK-P8-NEXT:    vmrghh v3, v3, v3
+; CHECK-P8-NEXT:    vmrghh v2, v2, v2
+; CHECK-P8-NEXT:    vadduwm v5, v5, v5
+; CHECK-P8-NEXT:    vslw v4, v4, v5
+; CHECK-P8-NEXT:    vslw v0, v0, v5
+; CHECK-P8-NEXT:    vslw v3, v3, v5
+; CHECK-P8-NEXT:    vslw v2, v2, v5
+; CHECK-P8-NEXT:    vsraw v4, v4, v5
+; CHECK-P8-NEXT:    vsraw v0, v0, v5
+; CHECK-P8-NEXT:    vsraw v3, v3, v5
+; CHECK-P8-NEXT:    vsraw v2, v2, v5
+; CHECK-P8-NEXT:    xvcvsxwsp v4, v4
+; CHECK-P8-NEXT:    xvcvsxwsp v5, v0
+; CHECK-P8-NEXT:    xvcvsxwsp v3, v3
+; CHECK-P8-NEXT:    xvcvsxwsp v2, v2
+; CHECK-P8-NEXT:    stvx v4, 0, r3
+; CHECK-P8-NEXT:    stvx v5, r3, r6
+; CHECK-P8-NEXT:    stvx v3, r3, r4
+; CHECK-P8-NEXT:    stvx v2, r3, r5
 ; CHECK-P8-NEXT:    blr
 ;
 ; CHECK-P9-LABEL: test16elt_signed:
 ; CHECK-P9:       # %bb.0: # %entry
 ; CHECK-P9-NEXT:    lxv v3, 0(r4)
 ; CHECK-P9-NEXT:    lxv v2, 16(r4)
-; CHECK-P9-NEXT:    li r4, 0
-; CHECK-P9-NEXT:    li r5, 4
-; CHECK-P9-NEXT:    li r6, 2
-; CHECK-P9-NEXT:    li r7, 6
-; CHECK-P9-NEXT:    li r8, 8
-; CHECK-P9-NEXT:    std r25, -72(r1) # 8-byte Folded Spill
-; CHECK-P9-NEXT:    li r9, 12
-; CHECK-P9-NEXT:    li r10, 10
-; CHECK-P9-NEXT:    li r11, 14
-; CHECK-P9-NEXT:    std r26, -64(r1) # 8-byte Folded Spill
-; CHECK-P9-NEXT:    std r27, -56(r1) # 8-byte Folded Spill
-; CHECK-P9-NEXT:    std r28, -48(r1) # 8-byte Folded Spill
-; CHECK-P9-NEXT:    std r29, -40(r1) # 8-byte Folded Spill
-; CHECK-P9-NEXT:    std r30, -32(r1) # 8-byte Folded Spill
-; CHECK-P9-NEXT:    stfd f30, -16(r1) # 8-byte Folded Spill
-; CHECK-P9-NEXT:    stfd f31, -8(r1) # 8-byte Folded Spill
-; CHECK-P9-NEXT:    vextuhrx r12, r4, v3
-; CHECK-P9-NEXT:    vextuhrx r0, r5, v3
-; CHECK-P9-NEXT:    vextuhrx r30, r6, v3
-; CHECK-P9-NEXT:    vextuhrx r29, r7, v3
-; CHECK-P9-NEXT:    vextuhrx r28, r8, v3
-; CHECK-P9-NEXT:    vextuhrx r27, r9, v3
-; CHECK-P9-NEXT:    vextuhrx r26, r10, v3
-; CHECK-P9-NEXT:    vextuhrx r25, r11, v3
-; CHECK-P9-NEXT:    vextuhrx r4, r4, v2
-; CHECK-P9-NEXT:    vextuhrx r5, r5, v2
-; CHECK-P9-NEXT:    vextuhrx r6, r6, v2
-; CHECK-P9-NEXT:    vextuhrx r7, r7, v2
-; CHECK-P9-NEXT:    vextuhrx r8, r8, v2
-; CHECK-P9-NEXT:    vextuhrx r9, r9, v2
-; CHECK-P9-NEXT:    vextuhrx r10, r10, v2
-; CHECK-P9-NEXT:    vextuhrx r11, r11, v2
-; CHECK-P9-NEXT:    extsh r12, r12
-; CHECK-P9-NEXT:    extsh r0, r0
-; CHECK-P9-NEXT:    extsh r30, r30
-; CHECK-P9-NEXT:    extsh r29, r29
-; CHECK-P9-NEXT:    extsh r28, r28
-; CHECK-P9-NEXT:    extsh r27, r27
-; CHECK-P9-NEXT:    extsh r26, r26
-; CHECK-P9-NEXT:    extsh r25, r25
-; CHECK-P9-NEXT:    extsh r4, r4
-; CHECK-P9-NEXT:    extsh r5, r5
-; CHECK-P9-NEXT:    extsh r6, r6
-; CHECK-P9-NEXT:    extsh r7, r7
-; CHECK-P9-NEXT:    extsh r8, r8
-; CHECK-P9-NEXT:    extsh r9, r9
-; CHECK-P9-NEXT:    extsh r10, r10
-; CHECK-P9-NEXT:    extsh r11, r11
-; CHECK-P9-NEXT:    mtvsrwa f0, r12
-; CHECK-P9-NEXT:    mtvsrwa f1, r0
-; CHECK-P9-NEXT:    mtvsrwa f2, r30
-; CHECK-P9-NEXT:    ld r30, -32(r1) # 8-byte Folded Reload
-; CHECK-P9-NEXT:    mtvsrwa f3, r29
-; CHECK-P9-NEXT:    ld r29, -40(r1) # 8-byte Folded Reload
-; CHECK-P9-NEXT:    mtvsrwa f4, r28
-; CHECK-P9-NEXT:    ld r28, -48(r1) # 8-byte Folded Reload
-; CHECK-P9-NEXT:    mtvsrwa f5, r27
-; CHECK-P9-NEXT:    ld r27, -56(r1) # 8-byte Folded Reload
-; CHECK-P9-NEXT:    mtvsrwa f6, r26
-; CHECK-P9-NEXT:    ld r26, -64(r1) # 8-byte Folded Reload
-; CHECK-P9-NEXT:    mtvsrwa f7, r25
-; CHECK-P9-NEXT:    ld r25, -72(r1) # 8-byte Folded Reload
-; CHECK-P9-NEXT:    mtvsrwa f8, r4
-; CHECK-P9-NEXT:    mtvsrwa f9, r5
-; CHECK-P9-NEXT:    mtvsrwa f10, r6
-; CHECK-P9-NEXT:    mtvsrwa f11, r7
-; CHECK-P9-NEXT:    mtvsrwa f12, r8
-; CHECK-P9-NEXT:    mtvsrwa f13, r9
-; CHECK-P9-NEXT:    mtvsrwa v2, r10
-; CHECK-P9-NEXT:    mtvsrwa v3, r11
-; CHECK-P9-NEXT:    xscvsxdsp f0, f0
-; CHECK-P9-NEXT:    xscvsxdsp f1, f1
-; CHECK-P9-NEXT:    xscvsxdsp f2, f2
-; CHECK-P9-NEXT:    xscvsxdsp f3, f3
-; CHECK-P9-NEXT:    xscvsxdsp f4, f4
-; CHECK-P9-NEXT:    xscvsxdsp f5, f5
-; CHECK-P9-NEXT:    xscvsxdsp f6, f6
-; CHECK-P9-NEXT:    xscvsxdsp f7, f7
-; CHECK-P9-NEXT:    xscvsxdsp f8, f8
-; CHECK-P9-NEXT:    xscvsxdsp f9, f9
-; CHECK-P9-NEXT:    xscvsxdsp f10, f10
-; CHECK-P9-NEXT:    xscvsxdsp f11, f11
-; CHECK-P9-NEXT:    xscvsxdsp f12, f12
-; CHECK-P9-NEXT:    xscvsxdsp f13, f13
-; CHECK-P9-NEXT:    xscvsxdsp f31, v2
-; CHECK-P9-NEXT:    xscvsxdsp f30, v3
-; CHECK-P9-NEXT:    xxmrghd vs0, vs1, vs0
-; CHECK-P9-NEXT:    xxmrghd vs1, vs3, vs2
-; CHECK-P9-NEXT:    xxmrghd vs2, vs5, vs4
-; CHECK-P9-NEXT:    xxmrghd vs3, vs7, vs6
-; CHECK-P9-NEXT:    xxmrghd vs4, vs9, vs8
-; CHECK-P9-NEXT:    xxmrghd vs5, vs11, vs10
-; CHECK-P9-NEXT:    xxmrghd vs6, vs13, vs12
-; CHECK-P9-NEXT:    xxmrghd vs7, vs30, vs31
-; CHECK-P9-NEXT:    lfd f31, -8(r1) # 8-byte Folded Reload
-; CHECK-P9-NEXT:    lfd f30, -16(r1) # 8-byte Folded Reload
-; CHECK-P9-NEXT:    xvcvdpsp v2, vs0
-; CHECK-P9-NEXT:    xvcvdpsp v3, vs1
-; CHECK-P9-NEXT:    xvcvdpsp v4, vs2
-; CHECK-P9-NEXT:    xvcvdpsp v5, vs3
-; CHECK-P9-NEXT:    xvcvdpsp v0, vs4
-; CHECK-P9-NEXT:    xvcvdpsp v1, vs5
-; CHECK-P9-NEXT:    xvcvdpsp v6, vs6
-; CHECK-P9-NEXT:    xvcvdpsp v7, vs7
-; CHECK-P9-NEXT:    vmrgew v2, v3, v2
-; CHECK-P9-NEXT:    vmrgew v3, v5, v4
-; CHECK-P9-NEXT:    vmrgew v4, v1, v0
-; CHECK-P9-NEXT:    vmrgew v5, v7, v6
-; CHECK-P9-NEXT:    stxv v3, 16(r3)
-; CHECK-P9-NEXT:    stxv v2, 0(r3)
-; CHECK-P9-NEXT:    stxv v5, 48(r3)
-; CHECK-P9-NEXT:    stxv v4, 32(r3)
+; CHECK-P9-NEXT:    vmrglh v4, v3, v3
+; CHECK-P9-NEXT:    vmrghh v3, v3, v3
+; CHECK-P9-NEXT:    vextsh2w v3, v3
+; CHECK-P9-NEXT:    xvcvsxwsp vs1, v3
+; CHECK-P9-NEXT:    vmrglh v3, v2, v2
+; CHECK-P9-NEXT:    vmrghh v2, v2, v2
+; CHECK-P9-NEXT:    vextsh2w v4, v4
+; CHECK-P9-NEXT:    xvcvsxwsp vs0, v4
+; CHECK-P9-NEXT:    vextsh2w v3, v3
+; CHECK-P9-NEXT:    vextsh2w v2, v2
+; CHECK-P9-NEXT:    xvcvsxwsp vs2, v3
+; CHECK-P9-NEXT:    xvcvsxwsp vs3, v2
+; CHECK-P9-NEXT:    stxv vs1, 16(r3)
+; CHECK-P9-NEXT:    stxv vs0, 0(r3)
+; CHECK-P9-NEXT:    stxv vs3, 48(r3)
+; CHECK-P9-NEXT:    stxv vs2, 32(r3)
 ; CHECK-P9-NEXT:    blr
 ;
 ; CHECK-BE-LABEL: test16elt_signed:
 ; CHECK-BE:       # %bb.0: # %entry
-; CHECK-BE-NEXT:    lxv v3, 0(r4)
 ; CHECK-BE-NEXT:    lxv v2, 16(r4)
-; CHECK-BE-NEXT:    li r4, 6
-; CHECK-BE-NEXT:    li r5, 2
-; CHECK-BE-NEXT:    li r6, 4
-; CHECK-BE-NEXT:    li r7, 0
-; CHECK-BE-NEXT:    li r8, 14
-; CHECK-BE-NEXT:    std r25, -72(r1) # 8-byte Folded Spill
-; CHECK-BE-NEXT:    li r9, 10
-; CHECK-BE-NEXT:    li r10, 12
-; CHECK-BE-NEXT:    li r11, 8
-; CHECK-BE-NEXT:    std r26, -64(r1) # 8-byte Folded Spill
-; CHECK-BE-NEXT:    std r27, -56(r1) # 8-byte Folded Spill
-; CHECK-BE-NEXT:    std r28, -48(r1) # 8-byte Folded Spill
-; CHECK-BE-NEXT:    std r29, -40(r1) # 8-byte Folded Spill
-; CHECK-BE-NEXT:    std r30, -32(r1) # 8-byte Folded Spill
-; CHECK-BE-NEXT:    stfd f30, -16(r1) # 8-byte Folded Spill
-; CHECK-BE-NEXT:    stfd f31, -8(r1) # 8-byte Folded Spill
-; CHECK-BE-NEXT:    vextuhlx r12, r4, v3
-; CHECK-BE-NEXT:    vextuhlx r0, r5, v3
-; CHECK-BE-NEXT:    vextuhlx r30, r6, v3
-; CHECK-BE-NEXT:    vextuhlx r29, r7, v3
-; CHECK-BE-NEXT:    vextuhlx r28, r8, v3
-; CHECK-BE-NEXT:    vextuhlx r27, r9, v3
-; CHECK-BE-NEXT:    vextuhlx r26, r10, v3
-; CHECK-BE-NEXT:    vextuhlx r25, r11, v3
-; CHECK-BE-NEXT:    vextuhlx r4, r4, v2
-; CHECK-BE-NEXT:    vextuhlx r5, r5, v2
-; CHECK-BE-NEXT:    vextuhlx r6, r6, v2
-; CHECK-BE-NEXT:    vextuhlx r7, r7, v2
-; CHECK-BE-NEXT:    vextuhlx r8, r8, v2
-; CHECK-BE-NEXT:    vextuhlx r9, r9, v2
-; CHECK-BE-NEXT:    vextuhlx r10, r10, v2
-; CHECK-BE-NEXT:    vextuhlx r11, r11, v2
-; CHECK-BE-NEXT:    extsh r12, r12
-; CHECK-BE-NEXT:    extsh r0, r0
-; CHECK-BE-NEXT:    extsh r30, r30
-; CHECK-BE-NEXT:    extsh r29, r29
-; CHECK-BE-NEXT:    extsh r28, r28
-; CHECK-BE-NEXT:    extsh r27, r27
-; CHECK-BE-NEXT:    extsh r26, r26
-; CHECK-BE-NEXT:    extsh r25, r25
-; CHECK-BE-NEXT:    extsh r4, r4
-; CHECK-BE-NEXT:    extsh r5, r5
-; CHECK-BE-NEXT:    extsh r6, r6
-; CHECK-BE-NEXT:    extsh r7, r7
-; CHECK-BE-NEXT:    extsh r8, r8
-; CHECK-BE-NEXT:    extsh r9, r9
-; CHECK-BE-NEXT:    extsh r10, r10
-; CHECK-BE-NEXT:    extsh r11, r11
-; CHECK-BE-NEXT:    mtvsrwa f0, r12
-; CHECK-BE-NEXT:    mtvsrwa f1, r0
-; CHECK-BE-NEXT:    mtvsrwa f2, r30
-; CHECK-BE-NEXT:    ld r30, -32(r1) # 8-byte Folded Reload
-; CHECK-BE-NEXT:    mtvsrwa f3, r29
-; CHECK-BE-NEXT:    ld r29, -40(r1) # 8-byte Folded Reload
-; CHECK-BE-NEXT:    mtvsrwa f4, r28
-; CHECK-BE-NEXT:    ld r28, -48(r1) # 8-byte Folded Reload
-; CHECK-BE-NEXT:    mtvsrwa f5, r27
-; CHECK-BE-NEXT:    ld r27, -56(r1) # 8-byte Folded Reload
-; CHECK-BE-NEXT:    mtvsrwa f6, r26
-; CHECK-BE-NEXT:    ld r26, -64(r1) # 8-byte Folded Reload
-; CHECK-BE-NEXT:    mtvsrwa f7, r25
-; CHECK-BE-NEXT:    ld r25, -72(r1) # 8-byte Folded Reload
-; CHECK-BE-NEXT:    mtvsrwa f8, r4
-; CHECK-BE-NEXT:    mtvsrwa f9, r5
-; CHECK-BE-NEXT:    mtvsrwa f10, r6
-; CHECK-BE-NEXT:    mtvsrwa f11, r7
-; CHECK-BE-NEXT:    mtvsrwa f12, r8
-; CHECK-BE-NEXT:    mtvsrwa f13, r9
-; CHECK-BE-NEXT:    mtvsrwa v2, r10
-; CHECK-BE-NEXT:    mtvsrwa v3, r11
-; CHECK-BE-NEXT:    xscvsxdsp f0, f0
-; CHECK-BE-NEXT:    xscvsxdsp f1, f1
-; CHECK-BE-NEXT:    xscvsxdsp f2, f2
-; CHECK-BE-NEXT:    xscvsxdsp f3, f3
-; CHECK-BE-NEXT:    xscvsxdsp f4, f4
-; CHECK-BE-NEXT:    xscvsxdsp f5, f5
-; CHECK-BE-NEXT:    xscvsxdsp f6, f6
-; CHECK-BE-NEXT:    xscvsxdsp f7, f7
-; CHECK-BE-NEXT:    xscvsxdsp f8, f8
-; CHECK-BE-NEXT:    xscvsxdsp f9, f9
-; CHECK-BE-NEXT:    xscvsxdsp f10, f10
-; CHECK-BE-NEXT:    xscvsxdsp f11, f11
-; CHECK-BE-NEXT:    xscvsxdsp f12, f12
-; CHECK-BE-NEXT:    xscvsxdsp f13, f13
-; CHECK-BE-NEXT:    xscvsxdsp f31, v2
-; CHECK-BE-NEXT:    xscvsxdsp f30, v3
-; CHECK-BE-NEXT:    xxmrghd vs0, vs1, vs0
-; CHECK-BE-NEXT:    xxmrghd vs1, vs3, vs2
-; CHECK-BE-NEXT:    xxmrghd vs2, vs5, vs4
-; CHECK-BE-NEXT:    xxmrghd vs3, vs7, vs6
-; CHECK-BE-NEXT:    xxmrghd vs4, vs9, vs8
-; CHECK-BE-NEXT:    xxmrghd vs5, vs11, vs10
-; CHECK-BE-NEXT:    xxmrghd vs6, vs13, vs12
-; CHECK-BE-NEXT:    xxmrghd vs7, vs30, vs31
-; CHECK-BE-NEXT:    lfd f31, -8(r1) # 8-byte Folded Reload
-; CHECK-BE-NEXT:    lfd f30, -16(r1) # 8-byte Folded Reload
-; CHECK-BE-NEXT:    xvcvdpsp v2, vs0
-; CHECK-BE-NEXT:    xvcvdpsp v3, vs1
-; CHECK-BE-NEXT:    xvcvdpsp v4, vs2
-; CHECK-BE-NEXT:    xvcvdpsp v5, vs3
-; CHECK-BE-NEXT:    xvcvdpsp v0, vs4
-; CHECK-BE-NEXT:    xvcvdpsp v1, vs5
-; CHECK-BE-NEXT:    xvcvdpsp v6, vs6
-; CHECK-BE-NEXT:    xvcvdpsp v7, vs7
-; CHECK-BE-NEXT:    vmrgew v2, v3, v2
-; CHECK-BE-NEXT:    vmrgew v3, v5, v4
-; CHECK-BE-NEXT:    vmrgew v4, v1, v0
-; CHECK-BE-NEXT:    vmrgew v5, v7, v6
-; CHECK-BE-NEXT:    stxv v3, 16(r3)
-; CHECK-BE-NEXT:    stxv v2, 0(r3)
-; CHECK-BE-NEXT:    stxv v5, 48(r3)
-; CHECK-BE-NEXT:    stxv v4, 32(r3)
+; CHECK-BE-NEXT:    lxv v3, 0(r4)
+; CHECK-BE-NEXT:    addis r4, r2, .LCPI7_0@toc@ha
+; CHECK-BE-NEXT:    addi r4, r4, .LCPI7_0@toc@l
+; CHECK-BE-NEXT:    lxvx v4, 0, r4
+; CHECK-BE-NEXT:    xxlxor v5, v5, v5
+; CHECK-BE-NEXT:    vperm v0, v5, v3, v4
+; CHECK-BE-NEXT:    vperm v4, v5, v2, v4
+; CHECK-BE-NEXT:    vmrghh v3, v3, v3
+; CHECK-BE-NEXT:    vmrghh v2, v2, v2
+; CHECK-BE-NEXT:    vextsh2w v0, v0
+; CHECK-BE-NEXT:    vextsh2w v4, v4
+; CHECK-BE-NEXT:    vextsh2w v3, v3
+; CHECK-BE-NEXT:    vextsh2w v2, v2
+; CHECK-BE-NEXT:    xvcvsxwsp vs0, v0
+; CHECK-BE-NEXT:    xvcvsxwsp vs1, v4
+; CHECK-BE-NEXT:    xvcvsxwsp vs2, v3
+; CHECK-BE-NEXT:    xvcvsxwsp vs3, v2
+; CHECK-BE-NEXT:    stxv vs3, 32(r3)
+; CHECK-BE-NEXT:    stxv vs2, 0(r3)
+; CHECK-BE-NEXT:    stxv vs1, 48(r3)
+; CHECK-BE-NEXT:    stxv vs0, 16(r3)
 ; CHECK-BE-NEXT:    blr
 entry:
   %a = load <16 x i16>, <16 x i16>* %0, align 32
diff --git a/test/CodeGen/PowerPC/vec_conv_i16_to_fp64_elts.ll b/test/CodeGen/PowerPC/vec_conv_i16_to_fp64_elts.ll
index a8f814c52189da26c16e7c3c3dbf7239034daf84..f03d0696f217d79e062fcd6180021e3965077f5f 100644
--- a/test/CodeGen/PowerPC/vec_conv_i16_to_fp64_elts.ll
+++ b/test/CodeGen/PowerPC/vec_conv_i16_to_fp64_elts.ll
@@ -24,23 +24,23 @@ define <2 x double> @test2elt(i32 %a.coerce) local_unnamed_addr #0 {
 ;
 ; CHECK-P9-LABEL: test2elt:
 ; CHECK-P9:       # %bb.0: # %entry
-; CHECK-P9-NEXT:    addis r4, r2, .LCPI0_0@toc@ha
-; CHECK-P9-NEXT:    mtvsrws v3, r3
+; CHECK-P9-NEXT:    mtvsrws v2, r3
+; CHECK-P9-NEXT:    addis r3, r2, .LCPI0_0@toc@ha
+; CHECK-P9-NEXT:    addi r3, r3, .LCPI0_0@toc@l
+; CHECK-P9-NEXT:    lxvx v3, 0, r3
 ; CHECK-P9-NEXT:    xxlxor v4, v4, v4
-; CHECK-P9-NEXT:    addi r4, r4, .LCPI0_0@toc@l
-; CHECK-P9-NEXT:    lxvx v2, 0, r4
-; CHECK-P9-NEXT:    vperm v2, v4, v3, v2
+; CHECK-P9-NEXT:    vperm v2, v4, v2, v3
 ; CHECK-P9-NEXT:    xvcvuxddp v2, v2
 ; CHECK-P9-NEXT:    blr
 ;
 ; CHECK-BE-LABEL: test2elt:
 ; CHECK-BE:       # %bb.0: # %entry
-; CHECK-BE-NEXT:    addis r4, r2, .LCPI0_0@toc@ha
-; CHECK-BE-NEXT:    mtvsrws v3, r3
+; CHECK-BE-NEXT:    mtvsrws v2, r3
+; CHECK-BE-NEXT:    addis r3, r2, .LCPI0_0@toc@ha
+; CHECK-BE-NEXT:    addi r3, r3, .LCPI0_0@toc@l
+; CHECK-BE-NEXT:    lxvx v3, 0, r3
 ; CHECK-BE-NEXT:    xxlxor v4, v4, v4
-; CHECK-BE-NEXT:    addi r4, r4, .LCPI0_0@toc@l
-; CHECK-BE-NEXT:    lxvx v2, 0, r4
-; CHECK-BE-NEXT:    vperm v2, v3, v4, v2
+; CHECK-BE-NEXT:    vperm v2, v2, v4, v3
 ; CHECK-BE-NEXT:    xvcvuxddp v2, v2
 ; CHECK-BE-NEXT:    blr
 entry:
@@ -74,39 +74,39 @@ define void @test4elt(<4 x double>* noalias nocapture sret %agg.result, i64 %a.c
 ;
 ; CHECK-P9-LABEL: test4elt:
 ; CHECK-P9:       # %bb.0: # %entry
-; CHECK-P9-NEXT:    addis r5, r2, .LCPI1_0@toc@ha
-; CHECK-P9-NEXT:    addis r6, r2, .LCPI1_1@toc@ha
 ; CHECK-P9-NEXT:    mtvsrd f0, r4
-; CHECK-P9-NEXT:    xxlxor v5, v5, v5
-; CHECK-P9-NEXT:    addi r5, r5, .LCPI1_0@toc@l
-; CHECK-P9-NEXT:    addi r6, r6, .LCPI1_1@toc@l
-; CHECK-P9-NEXT:    xxswapd v4, vs0
-; CHECK-P9-NEXT:    lxvx v2, 0, r5
-; CHECK-P9-NEXT:    lxvx v3, 0, r6
-; CHECK-P9-NEXT:    vperm v2, v5, v4, v2
-; CHECK-P9-NEXT:    vperm v3, v5, v4, v3
-; CHECK-P9-NEXT:    xvcvuxddp vs0, v2
-; CHECK-P9-NEXT:    xvcvuxddp vs1, v3
-; CHECK-P9-NEXT:    stxv vs1, 16(r3)
+; CHECK-P9-NEXT:    addis r4, r2, .LCPI1_0@toc@ha
+; CHECK-P9-NEXT:    addi r4, r4, .LCPI1_0@toc@l
+; CHECK-P9-NEXT:    lxvx v3, 0, r4
+; CHECK-P9-NEXT:    xxswapd v2, vs0
+; CHECK-P9-NEXT:    xxlxor v4, v4, v4
+; CHECK-P9-NEXT:    addis r4, r2, .LCPI1_1@toc@ha
+; CHECK-P9-NEXT:    addi r4, r4, .LCPI1_1@toc@l
+; CHECK-P9-NEXT:    vperm v3, v4, v2, v3
+; CHECK-P9-NEXT:    xvcvuxddp vs0, v3
+; CHECK-P9-NEXT:    lxvx v3, 0, r4
+; CHECK-P9-NEXT:    vperm v2, v4, v2, v3
 ; CHECK-P9-NEXT:    stxv vs0, 0(r3)
+; CHECK-P9-NEXT:    xvcvuxddp vs1, v2
+; CHECK-P9-NEXT:    stxv vs1, 16(r3)
 ; CHECK-P9-NEXT:    blr
 ;
 ; CHECK-BE-LABEL: test4elt:
 ; CHECK-BE:       # %bb.0: # %entry
-; CHECK-BE-NEXT:    addis r5, r2, .LCPI1_0@toc@ha
-; CHECK-BE-NEXT:    addis r6, r2, .LCPI1_1@toc@ha
-; CHECK-BE-NEXT:    mtvsrd v4, r4
-; CHECK-BE-NEXT:    xxlxor v5, v5, v5
-; CHECK-BE-NEXT:    addi r5, r5, .LCPI1_0@toc@l
-; CHECK-BE-NEXT:    addi r6, r6, .LCPI1_1@toc@l
-; CHECK-BE-NEXT:    lxvx v2, 0, r5
-; CHECK-BE-NEXT:    lxvx v3, 0, r6
-; CHECK-BE-NEXT:    vperm v2, v4, v5, v2
-; CHECK-BE-NEXT:    vperm v3, v5, v4, v3
-; CHECK-BE-NEXT:    xvcvuxddp vs0, v2
-; CHECK-BE-NEXT:    xvcvuxddp vs1, v3
-; CHECK-BE-NEXT:    stxv vs1, 16(r3)
+; CHECK-BE-NEXT:    mtvsrd v2, r4
+; CHECK-BE-NEXT:    addis r4, r2, .LCPI1_0@toc@ha
+; CHECK-BE-NEXT:    addi r4, r4, .LCPI1_0@toc@l
+; CHECK-BE-NEXT:    lxvx v3, 0, r4
+; CHECK-BE-NEXT:    xxlxor v4, v4, v4
+; CHECK-BE-NEXT:    addis r4, r2, .LCPI1_1@toc@ha
+; CHECK-BE-NEXT:    addi r4, r4, .LCPI1_1@toc@l
+; CHECK-BE-NEXT:    vperm v3, v2, v4, v3
+; CHECK-BE-NEXT:    xvcvuxddp vs0, v3
+; CHECK-BE-NEXT:    lxvx v3, 0, r4
+; CHECK-BE-NEXT:    vperm v2, v4, v2, v3
 ; CHECK-BE-NEXT:    stxv vs0, 0(r3)
+; CHECK-BE-NEXT:    xvcvuxddp vs1, v2
+; CHECK-BE-NEXT:    stxv vs1, 16(r3)
 ; CHECK-BE-NEXT:    blr
 entry:
   %0 = bitcast i64 %a.coerce to <4 x i16>
@@ -155,59 +155,59 @@ define void @test8elt(<8 x double>* noalias nocapture sret %agg.result, <8 x i16
 ; CHECK-P9-LABEL: test8elt:
 ; CHECK-P9:       # %bb.0: # %entry
 ; CHECK-P9-NEXT:    addis r4, r2, .LCPI2_0@toc@ha
-; CHECK-P9-NEXT:    addis r5, r2, .LCPI2_1@toc@ha
-; CHECK-P9-NEXT:    addis r6, r2, .LCPI2_2@toc@ha
-; CHECK-P9-NEXT:    addis r7, r2, .LCPI2_3@toc@ha
-; CHECK-P9-NEXT:    xxlxor v1, v1, v1
 ; CHECK-P9-NEXT:    addi r4, r4, .LCPI2_0@toc@l
-; CHECK-P9-NEXT:    addi r5, r5, .LCPI2_1@toc@l
-; CHECK-P9-NEXT:    addi r6, r6, .LCPI2_2@toc@l
-; CHECK-P9-NEXT:    addi r7, r7, .LCPI2_3@toc@l
 ; CHECK-P9-NEXT:    lxvx v3, 0, r4
-; CHECK-P9-NEXT:    lxvx v4, 0, r5
-; CHECK-P9-NEXT:    lxvx v5, 0, r6
-; CHECK-P9-NEXT:    lxvx v0, 0, r7
-; CHECK-P9-NEXT:    vperm v3, v1, v2, v3
-; CHECK-P9-NEXT:    vperm v4, v1, v2, v4
-; CHECK-P9-NEXT:    vperm v5, v1, v2, v5
-; CHECK-P9-NEXT:    vperm v2, v1, v2, v0
+; CHECK-P9-NEXT:    xxlxor v4, v4, v4
+; CHECK-P9-NEXT:    addis r4, r2, .LCPI2_1@toc@ha
+; CHECK-P9-NEXT:    addi r4, r4, .LCPI2_1@toc@l
+; CHECK-P9-NEXT:    vperm v3, v4, v2, v3
 ; CHECK-P9-NEXT:    xvcvuxddp vs0, v3
-; CHECK-P9-NEXT:    xvcvuxddp vs1, v4
-; CHECK-P9-NEXT:    xvcvuxddp vs2, v5
+; CHECK-P9-NEXT:    lxvx v3, 0, r4
+; CHECK-P9-NEXT:    addis r4, r2, .LCPI2_2@toc@ha
+; CHECK-P9-NEXT:    addi r4, r4, .LCPI2_2@toc@l
+; CHECK-P9-NEXT:    vperm v3, v4, v2, v3
+; CHECK-P9-NEXT:    stxv vs0, 0(r3)
+; CHECK-P9-NEXT:    xvcvuxddp vs1, v3
+; CHECK-P9-NEXT:    lxvx v3, 0, r4
+; CHECK-P9-NEXT:    addis r4, r2, .LCPI2_3@toc@ha
+; CHECK-P9-NEXT:    addi r4, r4, .LCPI2_3@toc@l
+; CHECK-P9-NEXT:    vperm v3, v4, v2, v3
+; CHECK-P9-NEXT:    stxv vs1, 16(r3)
+; CHECK-P9-NEXT:    xvcvuxddp vs2, v3
+; CHECK-P9-NEXT:    lxvx v3, 0, r4
+; CHECK-P9-NEXT:    vperm v2, v4, v2, v3
+; CHECK-P9-NEXT:    stxv vs2, 32(r3)
 ; CHECK-P9-NEXT:    xvcvuxddp vs3, v2
 ; CHECK-P9-NEXT:    stxv vs3, 48(r3)
-; CHECK-P9-NEXT:    stxv vs2, 32(r3)
-; CHECK-P9-NEXT:    stxv vs1, 16(r3)
-; CHECK-P9-NEXT:    stxv vs0, 0(r3)
 ; CHECK-P9-NEXT:    blr
 ;
 ; CHECK-BE-LABEL: test8elt:
 ; CHECK-BE:       # %bb.0: # %entry
 ; CHECK-BE-NEXT:    addis r4, r2, .LCPI2_0@toc@ha
-; CHECK-BE-NEXT:    addis r5, r2, .LCPI2_1@toc@ha
-; CHECK-BE-NEXT:    addis r6, r2, .LCPI2_2@toc@ha
-; CHECK-BE-NEXT:    addis r7, r2, .LCPI2_3@toc@ha
-; CHECK-BE-NEXT:    xxlxor v1, v1, v1
 ; CHECK-BE-NEXT:    addi r4, r4, .LCPI2_0@toc@l
-; CHECK-BE-NEXT:    addi r5, r5, .LCPI2_1@toc@l
-; CHECK-BE-NEXT:    addi r6, r6, .LCPI2_2@toc@l
-; CHECK-BE-NEXT:    addi r7, r7, .LCPI2_3@toc@l
 ; CHECK-BE-NEXT:    lxvx v3, 0, r4
-; CHECK-BE-NEXT:    lxvx v4, 0, r5
-; CHECK-BE-NEXT:    lxvx v5, 0, r6
-; CHECK-BE-NEXT:    lxvx v0, 0, r7
-; CHECK-BE-NEXT:    vperm v3, v2, v1, v3
-; CHECK-BE-NEXT:    vperm v4, v1, v2, v4
-; CHECK-BE-NEXT:    vperm v5, v1, v2, v5
-; CHECK-BE-NEXT:    vperm v2, v1, v2, v0
+; CHECK-BE-NEXT:    xxlxor v4, v4, v4
+; CHECK-BE-NEXT:    addis r4, r2, .LCPI2_1@toc@ha
+; CHECK-BE-NEXT:    addi r4, r4, .LCPI2_1@toc@l
+; CHECK-BE-NEXT:    vperm v3, v2, v4, v3
 ; CHECK-BE-NEXT:    xvcvuxddp vs0, v3
-; CHECK-BE-NEXT:    xvcvuxddp vs1, v4
-; CHECK-BE-NEXT:    xvcvuxddp vs2, v5
+; CHECK-BE-NEXT:    lxvx v3, 0, r4
+; CHECK-BE-NEXT:    addis r4, r2, .LCPI2_2@toc@ha
+; CHECK-BE-NEXT:    addi r4, r4, .LCPI2_2@toc@l
+; CHECK-BE-NEXT:    vperm v3, v4, v2, v3
+; CHECK-BE-NEXT:    stxv vs0, 0(r3)
+; CHECK-BE-NEXT:    xvcvuxddp vs1, v3
+; CHECK-BE-NEXT:    lxvx v3, 0, r4
+; CHECK-BE-NEXT:    addis r4, r2, .LCPI2_3@toc@ha
+; CHECK-BE-NEXT:    addi r4, r4, .LCPI2_3@toc@l
+; CHECK-BE-NEXT:    vperm v3, v4, v2, v3
+; CHECK-BE-NEXT:    stxv vs1, 16(r3)
+; CHECK-BE-NEXT:    xvcvuxddp vs2, v3
+; CHECK-BE-NEXT:    lxvx v3, 0, r4
+; CHECK-BE-NEXT:    vperm v2, v4, v2, v3
+; CHECK-BE-NEXT:    stxv vs2, 32(r3)
 ; CHECK-BE-NEXT:    xvcvuxddp vs3, v2
 ; CHECK-BE-NEXT:    stxv vs3, 48(r3)
-; CHECK-BE-NEXT:    stxv vs2, 32(r3)
-; CHECK-BE-NEXT:    stxv vs1, 16(r3)
-; CHECK-BE-NEXT:    stxv vs0, 0(r3)
 ; CHECK-BE-NEXT:    blr
 entry:
   %0 = uitofp <8 x i16> %a to <8 x double>
@@ -276,88 +276,88 @@ define void @test16elt(<16 x double>* noalias nocapture sret %agg.result, <16 x
 ;
 ; CHECK-P9-LABEL: test16elt:
 ; CHECK-P9:       # %bb.0: # %entry
-; CHECK-P9-NEXT:    addis r5, r2, .LCPI3_0@toc@ha
-; CHECK-P9-NEXT:    addis r6, r2, .LCPI3_1@toc@ha
-; CHECK-P9-NEXT:    addis r7, r2, .LCPI3_2@toc@ha
-; CHECK-P9-NEXT:    addis r8, r2, .LCPI3_3@toc@ha
-; CHECK-P9-NEXT:    lxv v0, 0(r4)
-; CHECK-P9-NEXT:    lxv v1, 16(r4)
-; CHECK-P9-NEXT:    xxlxor v6, v6, v6
-; CHECK-P9-NEXT:    addi r5, r5, .LCPI3_0@toc@l
-; CHECK-P9-NEXT:    addi r6, r6, .LCPI3_1@toc@l
-; CHECK-P9-NEXT:    addi r7, r7, .LCPI3_2@toc@l
-; CHECK-P9-NEXT:    addi r8, r8, .LCPI3_3@toc@l
-; CHECK-P9-NEXT:    lxvx v2, 0, r5
-; CHECK-P9-NEXT:    lxvx v3, 0, r6
-; CHECK-P9-NEXT:    lxvx v4, 0, r7
-; CHECK-P9-NEXT:    lxvx v5, 0, r8
-; CHECK-P9-NEXT:    vperm v7, v6, v0, v2
-; CHECK-P9-NEXT:    vperm v8, v6, v0, v3
-; CHECK-P9-NEXT:    vperm v9, v6, v0, v4
-; CHECK-P9-NEXT:    vperm v0, v6, v0, v5
-; CHECK-P9-NEXT:    vperm v2, v6, v1, v2
-; CHECK-P9-NEXT:    vperm v3, v6, v1, v3
-; CHECK-P9-NEXT:    vperm v4, v6, v1, v4
-; CHECK-P9-NEXT:    vperm v5, v6, v1, v5
-; CHECK-P9-NEXT:    xvcvuxddp vs0, v7
-; CHECK-P9-NEXT:    xvcvuxddp vs1, v8
-; CHECK-P9-NEXT:    xvcvuxddp vs2, v9
-; CHECK-P9-NEXT:    xvcvuxddp vs3, v0
-; CHECK-P9-NEXT:    xvcvuxddp vs4, v2
+; CHECK-P9-NEXT:    lxv v2, 16(r4)
+; CHECK-P9-NEXT:    lxv v3, 0(r4)
+; CHECK-P9-NEXT:    addis r4, r2, .LCPI3_0@toc@ha
+; CHECK-P9-NEXT:    addi r4, r4, .LCPI3_0@toc@l
+; CHECK-P9-NEXT:    lxvx v4, 0, r4
+; CHECK-P9-NEXT:    xxlxor v5, v5, v5
+; CHECK-P9-NEXT:    addis r4, r2, .LCPI3_1@toc@ha
+; CHECK-P9-NEXT:    addi r4, r4, .LCPI3_1@toc@l
+; CHECK-P9-NEXT:    vperm v0, v5, v3, v4
+; CHECK-P9-NEXT:    xvcvuxddp vs0, v0
+; CHECK-P9-NEXT:    lxvx v0, 0, r4
+; CHECK-P9-NEXT:    addis r4, r2, .LCPI3_2@toc@ha
+; CHECK-P9-NEXT:    addi r4, r4, .LCPI3_2@toc@l
+; CHECK-P9-NEXT:    vperm v1, v5, v3, v0
+; CHECK-P9-NEXT:    stxv vs0, 0(r3)
+; CHECK-P9-NEXT:    xvcvuxddp vs1, v1
+; CHECK-P9-NEXT:    lxvx v1, 0, r4
+; CHECK-P9-NEXT:    addis r4, r2, .LCPI3_3@toc@ha
+; CHECK-P9-NEXT:    addi r4, r4, .LCPI3_3@toc@l
+; CHECK-P9-NEXT:    vperm v6, v5, v3, v1
+; CHECK-P9-NEXT:    stxv vs1, 16(r3)
+; CHECK-P9-NEXT:    xvcvuxddp vs2, v6
+; CHECK-P9-NEXT:    lxvx v6, 0, r4
+; CHECK-P9-NEXT:    vperm v3, v5, v3, v6
+; CHECK-P9-NEXT:    stxv vs2, 32(r3)
+; CHECK-P9-NEXT:    xvcvuxddp vs3, v3
+; CHECK-P9-NEXT:    vperm v3, v5, v2, v4
+; CHECK-P9-NEXT:    xvcvuxddp vs4, v3
+; CHECK-P9-NEXT:    vperm v3, v5, v2, v0
 ; CHECK-P9-NEXT:    xvcvuxddp vs5, v3
-; CHECK-P9-NEXT:    xvcvuxddp vs6, v4
-; CHECK-P9-NEXT:    xvcvuxddp vs7, v5
+; CHECK-P9-NEXT:    vperm v3, v5, v2, v1
+; CHECK-P9-NEXT:    vperm v2, v5, v2, v6
 ; CHECK-P9-NEXT:    stxv vs3, 48(r3)
-; CHECK-P9-NEXT:    stxv vs2, 32(r3)
-; CHECK-P9-NEXT:    stxv vs1, 16(r3)
-; CHECK-P9-NEXT:    stxv vs0, 0(r3)
+; CHECK-P9-NEXT:    xvcvuxddp vs6, v3
+; CHECK-P9-NEXT:    xvcvuxddp vs7, v2
+; CHECK-P9-NEXT:    stxv vs4, 64(r3)
+; CHECK-P9-NEXT:    stxv vs5, 80(r3)
 ; CHECK-P9-NEXT:    stxv vs7, 112(r3)
 ; CHECK-P9-NEXT:    stxv vs6, 96(r3)
-; CHECK-P9-NEXT:    stxv vs5, 80(r3)
-; CHECK-P9-NEXT:    stxv vs4, 64(r3)
 ; CHECK-P9-NEXT:    blr
 ;
 ; CHECK-BE-LABEL: test16elt:
 ; CHECK-BE:       # %bb.0: # %entry
-; CHECK-BE-NEXT:    addis r5, r2, .LCPI3_0@toc@ha
-; CHECK-BE-NEXT:    addis r6, r2, .LCPI3_1@toc@ha
-; CHECK-BE-NEXT:    addis r7, r2, .LCPI3_2@toc@ha
-; CHECK-BE-NEXT:    addis r8, r2, .LCPI3_3@toc@ha
-; CHECK-BE-NEXT:    lxv v0, 0(r4)
-; CHECK-BE-NEXT:    lxv v1, 16(r4)
-; CHECK-BE-NEXT:    xxlxor v6, v6, v6
-; CHECK-BE-NEXT:    addi r5, r5, .LCPI3_0@toc@l
-; CHECK-BE-NEXT:    addi r6, r6, .LCPI3_1@toc@l
-; CHECK-BE-NEXT:    addi r7, r7, .LCPI3_2@toc@l
-; CHECK-BE-NEXT:    addi r8, r8, .LCPI3_3@toc@l
-; CHECK-BE-NEXT:    lxvx v2, 0, r5
-; CHECK-BE-NEXT:    lxvx v3, 0, r6
-; CHECK-BE-NEXT:    lxvx v4, 0, r7
-; CHECK-BE-NEXT:    lxvx v5, 0, r8
-; CHECK-BE-NEXT:    vperm v7, v0, v6, v2
-; CHECK-BE-NEXT:    vperm v8, v6, v0, v3
-; CHECK-BE-NEXT:    vperm v9, v6, v0, v4
-; CHECK-BE-NEXT:    vperm v0, v6, v0, v5
-; CHECK-BE-NEXT:    vperm v2, v1, v6, v2
-; CHECK-BE-NEXT:    vperm v3, v6, v1, v3
-; CHECK-BE-NEXT:    vperm v4, v6, v1, v4
-; CHECK-BE-NEXT:    vperm v5, v6, v1, v5
-; CHECK-BE-NEXT:    xvcvuxddp vs0, v7
-; CHECK-BE-NEXT:    xvcvuxddp vs1, v8
-; CHECK-BE-NEXT:    xvcvuxddp vs2, v9
-; CHECK-BE-NEXT:    xvcvuxddp vs3, v0
-; CHECK-BE-NEXT:    xvcvuxddp vs4, v2
+; CHECK-BE-NEXT:    lxv v2, 16(r4)
+; CHECK-BE-NEXT:    lxv v3, 0(r4)
+; CHECK-BE-NEXT:    addis r4, r2, .LCPI3_0@toc@ha
+; CHECK-BE-NEXT:    addi r4, r4, .LCPI3_0@toc@l
+; CHECK-BE-NEXT:    lxvx v4, 0, r4
+; CHECK-BE-NEXT:    xxlxor v5, v5, v5
+; CHECK-BE-NEXT:    addis r4, r2, .LCPI3_1@toc@ha
+; CHECK-BE-NEXT:    addi r4, r4, .LCPI3_1@toc@l
+; CHECK-BE-NEXT:    vperm v0, v3, v5, v4
+; CHECK-BE-NEXT:    xvcvuxddp vs0, v0
+; CHECK-BE-NEXT:    lxvx v0, 0, r4
+; CHECK-BE-NEXT:    addis r4, r2, .LCPI3_2@toc@ha
+; CHECK-BE-NEXT:    addi r4, r4, .LCPI3_2@toc@l
+; CHECK-BE-NEXT:    vperm v1, v5, v3, v0
+; CHECK-BE-NEXT:    stxv vs0, 0(r3)
+; CHECK-BE-NEXT:    xvcvuxddp vs1, v1
+; CHECK-BE-NEXT:    lxvx v1, 0, r4
+; CHECK-BE-NEXT:    addis r4, r2, .LCPI3_3@toc@ha
+; CHECK-BE-NEXT:    addi r4, r4, .LCPI3_3@toc@l
+; CHECK-BE-NEXT:    vperm v6, v5, v3, v1
+; CHECK-BE-NEXT:    stxv vs1, 16(r3)
+; CHECK-BE-NEXT:    xvcvuxddp vs2, v6
+; CHECK-BE-NEXT:    lxvx v6, 0, r4
+; CHECK-BE-NEXT:    vperm v3, v5, v3, v6
+; CHECK-BE-NEXT:    stxv vs2, 32(r3)
+; CHECK-BE-NEXT:    xvcvuxddp vs3, v3
+; CHECK-BE-NEXT:    vperm v3, v2, v5, v4
+; CHECK-BE-NEXT:    xvcvuxddp vs4, v3
+; CHECK-BE-NEXT:    vperm v3, v5, v2, v0
 ; CHECK-BE-NEXT:    xvcvuxddp vs5, v3
-; CHECK-BE-NEXT:    xvcvuxddp vs6, v4
-; CHECK-BE-NEXT:    xvcvuxddp vs7, v5
+; CHECK-BE-NEXT:    vperm v3, v5, v2, v1
+; CHECK-BE-NEXT:    vperm v2, v5, v2, v6
 ; CHECK-BE-NEXT:    stxv vs3, 48(r3)
-; CHECK-BE-NEXT:    stxv vs2, 32(r3)
-; CHECK-BE-NEXT:    stxv vs1, 16(r3)
-; CHECK-BE-NEXT:    stxv vs0, 0(r3)
+; CHECK-BE-NEXT:    xvcvuxddp vs6, v3
+; CHECK-BE-NEXT:    xvcvuxddp vs7, v2
+; CHECK-BE-NEXT:    stxv vs4, 64(r3)
+; CHECK-BE-NEXT:    stxv vs5, 80(r3)
 ; CHECK-BE-NEXT:    stxv vs7, 112(r3)
 ; CHECK-BE-NEXT:    stxv vs6, 96(r3)
-; CHECK-BE-NEXT:    stxv vs5, 80(r3)
-; CHECK-BE-NEXT:    stxv vs4, 64(r3)
 ; CHECK-BE-NEXT:    blr
 entry:
   %a = load <16 x i16>, <16 x i16>* %0, align 32
@@ -369,37 +369,39 @@ entry:
 define <2 x double> @test2elt_signed(i32 %a.coerce) local_unnamed_addr #0 {
 ; CHECK-P8-LABEL: test2elt_signed:
 ; CHECK-P8:       # %bb.0: # %entry
+; CHECK-P8-NEXT:    addis r4, r2, .LCPI4_0@toc@ha
 ; CHECK-P8-NEXT:    mtvsrd f0, r3
-; CHECK-P8-NEXT:    mfvsrd r3, f0
-; CHECK-P8-NEXT:    clrldi r4, r3, 48
-; CHECK-P8-NEXT:    rldicl r3, r3, 48, 48
-; CHECK-P8-NEXT:    extsh r4, r4
-; CHECK-P8-NEXT:    extsh r3, r3
-; CHECK-P8-NEXT:    mtvsrwa f0, r4
-; CHECK-P8-NEXT:    mtvsrwa f1, r3
-; CHECK-P8-NEXT:    xscvsxddp f0, f0
-; CHECK-P8-NEXT:    xscvsxddp f1, f1
-; CHECK-P8-NEXT:    xxmrghd v2, vs1, vs0
+; CHECK-P8-NEXT:    addi r3, r4, .LCPI4_0@toc@l
+; CHECK-P8-NEXT:    xxswapd v2, vs0
+; CHECK-P8-NEXT:    lvx v3, 0, r3
+; CHECK-P8-NEXT:    addis r3, r2, .LCPI4_1@toc@ha
+; CHECK-P8-NEXT:    addi r3, r3, .LCPI4_1@toc@l
+; CHECK-P8-NEXT:    lxvd2x vs0, 0, r3
+; CHECK-P8-NEXT:    vperm v2, v2, v2, v3
+; CHECK-P8-NEXT:    xxswapd v3, vs0
+; CHECK-P8-NEXT:    vsld v2, v2, v3
+; CHECK-P8-NEXT:    vsrad v2, v2, v3
+; CHECK-P8-NEXT:    xvcvsxddp v2, v2
 ; CHECK-P8-NEXT:    blr
 ;
 ; CHECK-P9-LABEL: test2elt_signed:
 ; CHECK-P9:       # %bb.0: # %entry
-; CHECK-P9-NEXT:    addis r4, r2, .LCPI4_0@toc@ha
-; CHECK-P9-NEXT:    mtvsrws v3, r3
-; CHECK-P9-NEXT:    addi r4, r4, .LCPI4_0@toc@l
-; CHECK-P9-NEXT:    lxvx v2, 0, r4
-; CHECK-P9-NEXT:    vperm v2, v3, v3, v2
+; CHECK-P9-NEXT:    mtvsrws v2, r3
+; CHECK-P9-NEXT:    addis r3, r2, .LCPI4_0@toc@ha
+; CHECK-P9-NEXT:    addi r3, r3, .LCPI4_0@toc@l
+; CHECK-P9-NEXT:    lxvx v3, 0, r3
+; CHECK-P9-NEXT:    vperm v2, v2, v2, v3
 ; CHECK-P9-NEXT:    vextsh2d v2, v2
 ; CHECK-P9-NEXT:    xvcvsxddp v2, v2
 ; CHECK-P9-NEXT:    blr
 ;
 ; CHECK-BE-LABEL: test2elt_signed:
 ; CHECK-BE:       # %bb.0: # %entry
-; CHECK-BE-NEXT:    addis r4, r2, .LCPI4_0@toc@ha
-; CHECK-BE-NEXT:    mtvsrws v3, r3
-; CHECK-BE-NEXT:    addi r4, r4, .LCPI4_0@toc@l
-; CHECK-BE-NEXT:    lxvx v2, 0, r4
-; CHECK-BE-NEXT:    vperm v2, v3, v3, v2
+; CHECK-BE-NEXT:    mtvsrws v2, r3
+; CHECK-BE-NEXT:    addis r3, r2, .LCPI4_0@toc@ha
+; CHECK-BE-NEXT:    addi r3, r3, .LCPI4_0@toc@l
+; CHECK-BE-NEXT:    lxvx v3, 0, r3
+; CHECK-BE-NEXT:    vperm v2, v2, v2, v3
 ; CHECK-BE-NEXT:    vextsh2d v2, v2
 ; CHECK-BE-NEXT:    xvcvsxddp v2, v2
 ; CHECK-BE-NEXT:    blr
@@ -412,27 +414,27 @@ entry:
 define void @test4elt_signed(<4 x double>* noalias nocapture sret %agg.result, i64 %a.coerce) local_unnamed_addr #1 {
 ; CHECK-P8-LABEL: test4elt_signed:
 ; CHECK-P8:       # %bb.0: # %entry
+; CHECK-P8-NEXT:    addis r5, r2, .LCPI5_0@toc@ha
 ; CHECK-P8-NEXT:    mtvsrd f0, r4
-; CHECK-P8-NEXT:    mfvsrd r4, f0
-; CHECK-P8-NEXT:    clrldi r5, r4, 48
-; CHECK-P8-NEXT:    rldicl r6, r4, 48, 48
-; CHECK-P8-NEXT:    extsh r5, r5
-; CHECK-P8-NEXT:    extsh r6, r6
-; CHECK-P8-NEXT:    mtvsrwa f0, r5
-; CHECK-P8-NEXT:    rldicl r5, r4, 32, 48
-; CHECK-P8-NEXT:    rldicl r4, r4, 16, 48
-; CHECK-P8-NEXT:    extsh r5, r5
-; CHECK-P8-NEXT:    extsh r4, r4
-; CHECK-P8-NEXT:    mtvsrwa f1, r6
-; CHECK-P8-NEXT:    mtvsrwa f2, r5
-; CHECK-P8-NEXT:    mtvsrwa f3, r4
+; CHECK-P8-NEXT:    addis r4, r2, .LCPI5_2@toc@ha
+; CHECK-P8-NEXT:    addi r5, r5, .LCPI5_0@toc@l
+; CHECK-P8-NEXT:    addi r4, r4, .LCPI5_2@toc@l
+; CHECK-P8-NEXT:    lvx v2, 0, r5
+; CHECK-P8-NEXT:    xxswapd v3, vs0
+; CHECK-P8-NEXT:    lvx v4, 0, r4
+; CHECK-P8-NEXT:    addis r4, r2, .LCPI5_1@toc@ha
+; CHECK-P8-NEXT:    addi r4, r4, .LCPI5_1@toc@l
+; CHECK-P8-NEXT:    lxvd2x vs0, 0, r4
 ; CHECK-P8-NEXT:    li r4, 16
-; CHECK-P8-NEXT:    xscvsxddp f0, f0
-; CHECK-P8-NEXT:    xscvsxddp f1, f1
-; CHECK-P8-NEXT:    xscvsxddp f2, f2
-; CHECK-P8-NEXT:    xscvsxddp f3, f3
-; CHECK-P8-NEXT:    xxmrghd vs0, vs1, vs0
-; CHECK-P8-NEXT:    xxmrghd vs1, vs3, vs2
+; CHECK-P8-NEXT:    vperm v2, v3, v3, v2
+; CHECK-P8-NEXT:    vperm v3, v3, v3, v4
+; CHECK-P8-NEXT:    xxswapd v4, vs0
+; CHECK-P8-NEXT:    vsld v2, v2, v4
+; CHECK-P8-NEXT:    vsld v3, v3, v4
+; CHECK-P8-NEXT:    vsrad v2, v2, v4
+; CHECK-P8-NEXT:    vsrad v3, v3, v4
+; CHECK-P8-NEXT:    xvcvsxddp vs0, v2
+; CHECK-P8-NEXT:    xvcvsxddp vs1, v3
 ; CHECK-P8-NEXT:    xxswapd vs0, vs0
 ; CHECK-P8-NEXT:    xxswapd vs1, vs1
 ; CHECK-P8-NEXT:    stxvd2x vs1, r3, r4
@@ -441,41 +443,41 @@ define void @test4elt_signed(<4 x double>* noalias nocapture sret %agg.result, i
 ;
 ; CHECK-P9-LABEL: test4elt_signed:
 ; CHECK-P9:       # %bb.0: # %entry
-; CHECK-P9-NEXT:    addis r5, r2, .LCPI5_0@toc@ha
-; CHECK-P9-NEXT:    addis r6, r2, .LCPI5_1@toc@ha
 ; CHECK-P9-NEXT:    mtvsrd f0, r4
-; CHECK-P9-NEXT:    addi r5, r5, .LCPI5_0@toc@l
-; CHECK-P9-NEXT:    addi r6, r6, .LCPI5_1@toc@l
-; CHECK-P9-NEXT:    xxswapd v4, vs0
-; CHECK-P9-NEXT:    lxvx v2, 0, r5
-; CHECK-P9-NEXT:    lxvx v3, 0, r6
-; CHECK-P9-NEXT:    vperm v2, v4, v4, v2
-; CHECK-P9-NEXT:    vperm v3, v4, v4, v3
-; CHECK-P9-NEXT:    vextsh2d v2, v2
+; CHECK-P9-NEXT:    addis r4, r2, .LCPI5_0@toc@ha
+; CHECK-P9-NEXT:    addi r4, r4, .LCPI5_0@toc@l
+; CHECK-P9-NEXT:    lxvx v3, 0, r4
+; CHECK-P9-NEXT:    xxswapd v2, vs0
+; CHECK-P9-NEXT:    vperm v3, v2, v2, v3
+; CHECK-P9-NEXT:    addis r4, r2, .LCPI5_1@toc@ha
+; CHECK-P9-NEXT:    addi r4, r4, .LCPI5_1@toc@l
 ; CHECK-P9-NEXT:    vextsh2d v3, v3
-; CHECK-P9-NEXT:    xvcvsxddp vs0, v2
-; CHECK-P9-NEXT:    xvcvsxddp vs1, v3
-; CHECK-P9-NEXT:    stxv vs1, 16(r3)
+; CHECK-P9-NEXT:    xvcvsxddp vs0, v3
+; CHECK-P9-NEXT:    lxvx v3, 0, r4
+; CHECK-P9-NEXT:    vperm v2, v2, v2, v3
 ; CHECK-P9-NEXT:    stxv vs0, 0(r3)
+; CHECK-P9-NEXT:    vextsh2d v2, v2
+; CHECK-P9-NEXT:    xvcvsxddp vs1, v2
+; CHECK-P9-NEXT:    stxv vs1, 16(r3)
 ; CHECK-P9-NEXT:    blr
 ;
 ; CHECK-BE-LABEL: test4elt_signed:
 ; CHECK-BE:       # %bb.0: # %entry
-; CHECK-BE-NEXT:    addis r5, r2, .LCPI5_0@toc@ha
-; CHECK-BE-NEXT:    addis r6, r2, .LCPI5_1@toc@ha
-; CHECK-BE-NEXT:    mtvsrd v4, r4
-; CHECK-BE-NEXT:    xxlxor v5, v5, v5
-; CHECK-BE-NEXT:    addi r5, r5, .LCPI5_0@toc@l
-; CHECK-BE-NEXT:    addi r6, r6, .LCPI5_1@toc@l
-; CHECK-BE-NEXT:    lxvx v2, 0, r5
-; CHECK-BE-NEXT:    lxvx v3, 0, r6
-; CHECK-BE-NEXT:    vperm v2, v5, v4, v2
-; CHECK-BE-NEXT:    vperm v3, v4, v4, v3
-; CHECK-BE-NEXT:    vextsh2d v2, v2
+; CHECK-BE-NEXT:    mtvsrd v2, r4
+; CHECK-BE-NEXT:    addis r4, r2, .LCPI5_0@toc@ha
+; CHECK-BE-NEXT:    addi r4, r4, .LCPI5_0@toc@l
+; CHECK-BE-NEXT:    lxvx v4, 0, r4
+; CHECK-BE-NEXT:    xxlxor v3, v3, v3
+; CHECK-BE-NEXT:    vperm v3, v3, v2, v4
+; CHECK-BE-NEXT:    addis r4, r2, .LCPI5_1@toc@ha
 ; CHECK-BE-NEXT:    vextsh2d v3, v3
-; CHECK-BE-NEXT:    xvcvsxddp vs0, v2
-; CHECK-BE-NEXT:    xvcvsxddp vs1, v3
+; CHECK-BE-NEXT:    addi r4, r4, .LCPI5_1@toc@l
+; CHECK-BE-NEXT:    xvcvsxddp vs0, v3
+; CHECK-BE-NEXT:    lxvx v3, 0, r4
+; CHECK-BE-NEXT:    vperm v2, v2, v2, v3
 ; CHECK-BE-NEXT:    stxv vs0, 16(r3)
+; CHECK-BE-NEXT:    vextsh2d v2, v2
+; CHECK-BE-NEXT:    xvcvsxddp vs1, v2
 ; CHECK-BE-NEXT:    stxv vs1, 0(r3)
 ; CHECK-BE-NEXT:    blr
 entry:
@@ -488,121 +490,114 @@ entry:
 define void @test8elt_signed(<8 x double>* noalias nocapture sret %agg.result, <8 x i16> %a) local_unnamed_addr #2 {
 ; CHECK-P8-LABEL: test8elt_signed:
 ; CHECK-P8:       # %bb.0: # %entry
-; CHECK-P8-NEXT:    mfvsrd r4, v2
-; CHECK-P8-NEXT:    xxswapd vs0, v2
-; CHECK-P8-NEXT:    clrldi r5, r4, 48
-; CHECK-P8-NEXT:    rldicl r6, r4, 48, 48
-; CHECK-P8-NEXT:    extsh r5, r5
-; CHECK-P8-NEXT:    mfvsrd r7, f0
-; CHECK-P8-NEXT:    extsh r6, r6
-; CHECK-P8-NEXT:    mtvsrwa f1, r5
-; CHECK-P8-NEXT:    rldicl r5, r4, 32, 48
-; CHECK-P8-NEXT:    rldicl r4, r4, 16, 48
-; CHECK-P8-NEXT:    extsh r5, r5
-; CHECK-P8-NEXT:    extsh r4, r4
-; CHECK-P8-NEXT:    mtvsrwa f0, r6
-; CHECK-P8-NEXT:    mtvsrwa f2, r5
-; CHECK-P8-NEXT:    clrldi r5, r7, 48
-; CHECK-P8-NEXT:    mtvsrwa f3, r4
-; CHECK-P8-NEXT:    extsh r4, r5
-; CHECK-P8-NEXT:    rldicl r5, r7, 16, 48
-; CHECK-P8-NEXT:    mtvsrwa f4, r4
-; CHECK-P8-NEXT:    rldicl r4, r7, 48, 48
-; CHECK-P8-NEXT:    extsh r5, r5
-; CHECK-P8-NEXT:    extsh r4, r4
-; CHECK-P8-NEXT:    mtvsrwa f7, r5
-; CHECK-P8-NEXT:    li r5, 32
-; CHECK-P8-NEXT:    mtvsrwa f5, r4
-; CHECK-P8-NEXT:    rldicl r4, r7, 32, 48
-; CHECK-P8-NEXT:    extsh r4, r4
-; CHECK-P8-NEXT:    xscvsxddp f1, f1
-; CHECK-P8-NEXT:    mtvsrwa f6, r4
+; CHECK-P8-NEXT:    addis r5, r2, .LCPI6_2@toc@ha
+; CHECK-P8-NEXT:    addis r4, r2, .LCPI6_0@toc@ha
+; CHECK-P8-NEXT:    addis r6, r2, .LCPI6_3@toc@ha
+; CHECK-P8-NEXT:    addi r5, r5, .LCPI6_2@toc@l
+; CHECK-P8-NEXT:    addi r4, r4, .LCPI6_0@toc@l
+; CHECK-P8-NEXT:    addi r6, r6, .LCPI6_3@toc@l
+; CHECK-P8-NEXT:    lvx v4, 0, r5
+; CHECK-P8-NEXT:    addis r5, r2, .LCPI6_4@toc@ha
+; CHECK-P8-NEXT:    lvx v3, 0, r4
+; CHECK-P8-NEXT:    lvx v5, 0, r6
+; CHECK-P8-NEXT:    addis r4, r2, .LCPI6_1@toc@ha
+; CHECK-P8-NEXT:    addi r5, r5, .LCPI6_4@toc@l
+; CHECK-P8-NEXT:    addi r4, r4, .LCPI6_1@toc@l
+; CHECK-P8-NEXT:    lvx v0, 0, r5
+; CHECK-P8-NEXT:    lxvd2x vs0, 0, r4
 ; CHECK-P8-NEXT:    li r4, 48
-; CHECK-P8-NEXT:    xscvsxddp f0, f0
-; CHECK-P8-NEXT:    xscvsxddp f2, f2
-; CHECK-P8-NEXT:    xscvsxddp f3, f3
-; CHECK-P8-NEXT:    xscvsxddp f4, f4
-; CHECK-P8-NEXT:    xscvsxddp f5, f5
-; CHECK-P8-NEXT:    xscvsxddp f6, f6
-; CHECK-P8-NEXT:    xscvsxddp f7, f7
-; CHECK-P8-NEXT:    xxmrghd vs0, vs0, vs1
-; CHECK-P8-NEXT:    xxmrghd vs1, vs3, vs2
+; CHECK-P8-NEXT:    li r5, 32
+; CHECK-P8-NEXT:    vperm v3, v2, v2, v3
+; CHECK-P8-NEXT:    vperm v4, v2, v2, v4
+; CHECK-P8-NEXT:    vperm v5, v2, v2, v5
+; CHECK-P8-NEXT:    vperm v2, v2, v2, v0
+; CHECK-P8-NEXT:    xxswapd v0, vs0
+; CHECK-P8-NEXT:    vsld v3, v3, v0
+; CHECK-P8-NEXT:    vsld v4, v4, v0
+; CHECK-P8-NEXT:    vsld v5, v5, v0
+; CHECK-P8-NEXT:    vsld v2, v2, v0
+; CHECK-P8-NEXT:    vsrad v3, v3, v0
+; CHECK-P8-NEXT:    vsrad v2, v2, v0
+; CHECK-P8-NEXT:    vsrad v4, v4, v0
+; CHECK-P8-NEXT:    vsrad v5, v5, v0
+; CHECK-P8-NEXT:    xvcvsxddp vs2, v2
+; CHECK-P8-NEXT:    xvcvsxddp vs0, v3
+; CHECK-P8-NEXT:    xvcvsxddp vs1, v5
+; CHECK-P8-NEXT:    xvcvsxddp vs3, v4
+; CHECK-P8-NEXT:    xxswapd vs2, vs2
 ; CHECK-P8-NEXT:    xxswapd vs0, vs0
-; CHECK-P8-NEXT:    xxmrghd vs2, vs5, vs4
 ; CHECK-P8-NEXT:    xxswapd vs1, vs1
-; CHECK-P8-NEXT:    xxmrghd vs3, vs7, vs6
-; CHECK-P8-NEXT:    xxswapd vs2, vs2
-; CHECK-P8-NEXT:    stxvd2x vs1, r3, r4
-; CHECK-P8-NEXT:    li r4, 16
-; CHECK-P8-NEXT:    stxvd2x vs0, r3, r5
 ; CHECK-P8-NEXT:    xxswapd vs3, vs3
+; CHECK-P8-NEXT:    stxvd2x vs2, r3, r4
+; CHECK-P8-NEXT:    li r4, 16
+; CHECK-P8-NEXT:    stxvd2x vs1, r3, r5
 ; CHECK-P8-NEXT:    stxvd2x vs3, r3, r4
-; CHECK-P8-NEXT:    stxvd2x vs2, 0, r3
+; CHECK-P8-NEXT:    stxvd2x vs0, 0, r3
 ; CHECK-P8-NEXT:    blr
 ;
 ; CHECK-P9-LABEL: test8elt_signed:
 ; CHECK-P9:       # %bb.0: # %entry
 ; CHECK-P9-NEXT:    addis r4, r2, .LCPI6_0@toc@ha
-; CHECK-P9-NEXT:    addis r5, r2, .LCPI6_1@toc@ha
-; CHECK-P9-NEXT:    addis r6, r2, .LCPI6_2@toc@ha
-; CHECK-P9-NEXT:    addis r7, r2, .LCPI6_3@toc@ha
 ; CHECK-P9-NEXT:    addi r4, r4, .LCPI6_0@toc@l
-; CHECK-P9-NEXT:    addi r5, r5, .LCPI6_1@toc@l
-; CHECK-P9-NEXT:    addi r6, r6, .LCPI6_2@toc@l
-; CHECK-P9-NEXT:    addi r7, r7, .LCPI6_3@toc@l
 ; CHECK-P9-NEXT:    lxvx v3, 0, r4
-; CHECK-P9-NEXT:    lxvx v4, 0, r5
-; CHECK-P9-NEXT:    lxvx v5, 0, r6
-; CHECK-P9-NEXT:    lxvx v0, 0, r7
+; CHECK-P9-NEXT:    addis r4, r2, .LCPI6_1@toc@ha
+; CHECK-P9-NEXT:    addi r4, r4, .LCPI6_1@toc@l
 ; CHECK-P9-NEXT:    vperm v3, v2, v2, v3
-; CHECK-P9-NEXT:    vperm v4, v2, v2, v4
-; CHECK-P9-NEXT:    vperm v5, v2, v2, v5
-; CHECK-P9-NEXT:    vperm v2, v2, v2, v0
 ; CHECK-P9-NEXT:    vextsh2d v3, v3
-; CHECK-P9-NEXT:    vextsh2d v4, v4
-; CHECK-P9-NEXT:    vextsh2d v5, v5
-; CHECK-P9-NEXT:    vextsh2d v2, v2
 ; CHECK-P9-NEXT:    xvcvsxddp vs0, v3
-; CHECK-P9-NEXT:    xvcvsxddp vs1, v4
-; CHECK-P9-NEXT:    xvcvsxddp vs2, v5
+; CHECK-P9-NEXT:    lxvx v3, 0, r4
+; CHECK-P9-NEXT:    addis r4, r2, .LCPI6_2@toc@ha
+; CHECK-P9-NEXT:    addi r4, r4, .LCPI6_2@toc@l
+; CHECK-P9-NEXT:    vperm v3, v2, v2, v3
+; CHECK-P9-NEXT:    stxv vs0, 0(r3)
+; CHECK-P9-NEXT:    vextsh2d v3, v3
+; CHECK-P9-NEXT:    xvcvsxddp vs1, v3
+; CHECK-P9-NEXT:    lxvx v3, 0, r4
+; CHECK-P9-NEXT:    addis r4, r2, .LCPI6_3@toc@ha
+; CHECK-P9-NEXT:    addi r4, r4, .LCPI6_3@toc@l
+; CHECK-P9-NEXT:    vperm v3, v2, v2, v3
+; CHECK-P9-NEXT:    stxv vs1, 16(r3)
+; CHECK-P9-NEXT:    vextsh2d v3, v3
+; CHECK-P9-NEXT:    xvcvsxddp vs2, v3
+; CHECK-P9-NEXT:    lxvx v3, 0, r4
+; CHECK-P9-NEXT:    vperm v2, v2, v2, v3
+; CHECK-P9-NEXT:    stxv vs2, 32(r3)
+; CHECK-P9-NEXT:    vextsh2d v2, v2
 ; CHECK-P9-NEXT:    xvcvsxddp vs3, v2
 ; CHECK-P9-NEXT:    stxv vs3, 48(r3)
-; CHECK-P9-NEXT:    stxv vs2, 32(r3)
-; CHECK-P9-NEXT:    stxv vs1, 16(r3)
-; CHECK-P9-NEXT:    stxv vs0, 0(r3)
 ; CHECK-P9-NEXT:    blr
 ;
 ; CHECK-BE-LABEL: test8elt_signed:
 ; CHECK-BE:       # %bb.0: # %entry
 ; CHECK-BE-NEXT:    addis r4, r2, .LCPI6_0@toc@ha
-; CHECK-BE-NEXT:    addis r5, r2, .LCPI6_1@toc@ha
-; CHECK-BE-NEXT:    addis r6, r2, .LCPI6_2@toc@ha
-; CHECK-BE-NEXT:    addis r7, r2, .LCPI6_3@toc@ha
-; CHECK-BE-NEXT:    xxlxor v1, v1, v1
 ; CHECK-BE-NEXT:    addi r4, r4, .LCPI6_0@toc@l
-; CHECK-BE-NEXT:    addi r5, r5, .LCPI6_1@toc@l
-; CHECK-BE-NEXT:    addi r6, r6, .LCPI6_2@toc@l
-; CHECK-BE-NEXT:    addi r7, r7, .LCPI6_3@toc@l
 ; CHECK-BE-NEXT:    lxvx v3, 0, r4
-; CHECK-BE-NEXT:    lxvx v4, 0, r5
-; CHECK-BE-NEXT:    lxvx v5, 0, r6
-; CHECK-BE-NEXT:    lxvx v0, 0, r7
-; CHECK-BE-NEXT:    vperm v3, v1, v2, v3
-; CHECK-BE-NEXT:    vperm v4, v1, v2, v4
-; CHECK-BE-NEXT:    vperm v5, v2, v2, v5
-; CHECK-BE-NEXT:    vperm v2, v2, v2, v0
+; CHECK-BE-NEXT:    xxlxor v4, v4, v4
+; CHECK-BE-NEXT:    vperm v3, v4, v2, v3
+; CHECK-BE-NEXT:    addis r4, r2, .LCPI6_1@toc@ha
+; CHECK-BE-NEXT:    addi r4, r4, .LCPI6_1@toc@l
 ; CHECK-BE-NEXT:    vextsh2d v3, v3
-; CHECK-BE-NEXT:    vextsh2d v4, v4
-; CHECK-BE-NEXT:    vextsh2d v5, v5
-; CHECK-BE-NEXT:    vextsh2d v2, v2
 ; CHECK-BE-NEXT:    xvcvsxddp vs0, v3
-; CHECK-BE-NEXT:    xvcvsxddp vs1, v4
-; CHECK-BE-NEXT:    xvcvsxddp vs2, v5
-; CHECK-BE-NEXT:    xvcvsxddp vs3, v2
-; CHECK-BE-NEXT:    stxv vs1, 48(r3)
-; CHECK-BE-NEXT:    stxv vs3, 32(r3)
+; CHECK-BE-NEXT:    lxvx v3, 0, r4
+; CHECK-BE-NEXT:    addis r4, r2, .LCPI6_2@toc@ha
+; CHECK-BE-NEXT:    addi r4, r4, .LCPI6_2@toc@l
+; CHECK-BE-NEXT:    vperm v3, v4, v2, v3
 ; CHECK-BE-NEXT:    stxv vs0, 16(r3)
+; CHECK-BE-NEXT:    vextsh2d v3, v3
+; CHECK-BE-NEXT:    xvcvsxddp vs1, v3
+; CHECK-BE-NEXT:    lxvx v3, 0, r4
+; CHECK-BE-NEXT:    addis r4, r2, .LCPI6_3@toc@ha
+; CHECK-BE-NEXT:    addi r4, r4, .LCPI6_3@toc@l
+; CHECK-BE-NEXT:    vperm v3, v2, v2, v3
+; CHECK-BE-NEXT:    stxv vs1, 48(r3)
+; CHECK-BE-NEXT:    vextsh2d v3, v3
+; CHECK-BE-NEXT:    xvcvsxddp vs2, v3
+; CHECK-BE-NEXT:    lxvx v3, 0, r4
+; CHECK-BE-NEXT:    vperm v2, v2, v2, v3
 ; CHECK-BE-NEXT:    stxv vs2, 0(r3)
+; CHECK-BE-NEXT:    vextsh2d v2, v2
+; CHECK-BE-NEXT:    xvcvsxddp vs3, v2
+; CHECK-BE-NEXT:    stxv vs3, 32(r3)
 ; CHECK-BE-NEXT:    blr
 entry:
   %0 = sitofp <8 x i16> %a to <8 x double>
@@ -613,212 +608,180 @@ entry:
 define void @test16elt_signed(<16 x double>* noalias nocapture sret %agg.result, <16 x i16>* nocapture readonly) local_unnamed_addr #3 {
 ; CHECK-P8-LABEL: test16elt_signed:
 ; CHECK-P8:       # %bb.0: # %entry
-; CHECK-P8-NEXT:    li r5, 16
-; CHECK-P8-NEXT:    lvx v3, 0, r4
-; CHECK-P8-NEXT:    stfd f31, -8(r1) # 8-byte Folded Spill
-; CHECK-P8-NEXT:    lvx v2, r4, r5
-; CHECK-P8-NEXT:    mfvsrd r7, v3
-; CHECK-P8-NEXT:    xxswapd vs8, v3
-; CHECK-P8-NEXT:    mfvsrd r6, v2
-; CHECK-P8-NEXT:    xxswapd vs2, v2
-; CHECK-P8-NEXT:    clrldi r4, r6, 48
-; CHECK-P8-NEXT:    rldicl r8, r6, 48, 48
-; CHECK-P8-NEXT:    extsh r4, r4
-; CHECK-P8-NEXT:    extsh r8, r8
-; CHECK-P8-NEXT:    mtvsrwa f0, r4
-; CHECK-P8-NEXT:    rldicl r4, r6, 32, 48
-; CHECK-P8-NEXT:    rldicl r6, r6, 16, 48
-; CHECK-P8-NEXT:    mtvsrwa f1, r8
-; CHECK-P8-NEXT:    extsh r4, r4
-; CHECK-P8-NEXT:    clrldi r8, r7, 48
-; CHECK-P8-NEXT:    extsh r6, r6
-; CHECK-P8-NEXT:    mtvsrwa f3, r4
-; CHECK-P8-NEXT:    extsh r4, r8
-; CHECK-P8-NEXT:    mtvsrwa f4, r6
-; CHECK-P8-NEXT:    rldicl r6, r7, 48, 48
-; CHECK-P8-NEXT:    mtvsrwa f5, r4
-; CHECK-P8-NEXT:    rldicl r4, r7, 32, 48
-; CHECK-P8-NEXT:    extsh r6, r6
-; CHECK-P8-NEXT:    mfvsrd r8, f2
-; CHECK-P8-NEXT:    extsh r4, r4
-; CHECK-P8-NEXT:    mtvsrwa f2, r6
-; CHECK-P8-NEXT:    rldicl r6, r7, 16, 48
-; CHECK-P8-NEXT:    mtvsrwa f6, r4
-; CHECK-P8-NEXT:    clrldi r4, r8, 48
-; CHECK-P8-NEXT:    extsh r6, r6
-; CHECK-P8-NEXT:    extsh r4, r4
-; CHECK-P8-NEXT:    mtvsrwa f7, r6
-; CHECK-P8-NEXT:    rldicl r6, r8, 48, 48
-; CHECK-P8-NEXT:    mtvsrwa f9, r4
-; CHECK-P8-NEXT:    rldicl r4, r8, 32, 48
-; CHECK-P8-NEXT:    extsh r6, r6
-; CHECK-P8-NEXT:    extsh r4, r4
-; CHECK-P8-NEXT:    mtvsrwa f10, r6
-; CHECK-P8-NEXT:    rldicl r6, r8, 16, 48
-; CHECK-P8-NEXT:    mtvsrwa f11, r4
-; CHECK-P8-NEXT:    extsh r6, r6
-; CHECK-P8-NEXT:    mfvsrd r4, f8
-; CHECK-P8-NEXT:    mtvsrwa f8, r6
-; CHECK-P8-NEXT:    clrldi r6, r4, 48
-; CHECK-P8-NEXT:    xscvsxddp f3, f3
-; CHECK-P8-NEXT:    extsh r6, r6
-; CHECK-P8-NEXT:    xscvsxddp f4, f4
-; CHECK-P8-NEXT:    mtvsrwa f12, r6
-; CHECK-P8-NEXT:    rldicl r6, r4, 48, 48
-; CHECK-P8-NEXT:    extsh r6, r6
-; CHECK-P8-NEXT:    xscvsxddp f0, f0
-; CHECK-P8-NEXT:    mtvsrwa f13, r6
-; CHECK-P8-NEXT:    rldicl r6, r4, 32, 48
-; CHECK-P8-NEXT:    rldicl r4, r4, 16, 48
-; CHECK-P8-NEXT:    xscvsxddp f1, f1
-; CHECK-P8-NEXT:    extsh r6, r6
-; CHECK-P8-NEXT:    extsh r4, r4
-; CHECK-P8-NEXT:    xscvsxddp f5, f5
-; CHECK-P8-NEXT:    xscvsxddp f2, f2
-; CHECK-P8-NEXT:    xxmrghd vs3, vs4, vs3
-; CHECK-P8-NEXT:    mtvsrwa v2, r6
-; CHECK-P8-NEXT:    li r6, 32
-; CHECK-P8-NEXT:    mtvsrwa v3, r4
+; CHECK-P8-NEXT:    addis r5, r2, .LCPI7_0@toc@ha
+; CHECK-P8-NEXT:    addis r6, r2, .LCPI7_2@toc@ha
+; CHECK-P8-NEXT:    lvx v4, 0, r4
+; CHECK-P8-NEXT:    addi r5, r5, .LCPI7_0@toc@l
+; CHECK-P8-NEXT:    addi r6, r6, .LCPI7_2@toc@l
+; CHECK-P8-NEXT:    lvx v2, 0, r5
+; CHECK-P8-NEXT:    addis r5, r2, .LCPI7_3@toc@ha
+; CHECK-P8-NEXT:    lvx v3, 0, r6
+; CHECK-P8-NEXT:    addis r6, r2, .LCPI7_4@toc@ha
+; CHECK-P8-NEXT:    addi r5, r5, .LCPI7_3@toc@l
+; CHECK-P8-NEXT:    addi r6, r6, .LCPI7_4@toc@l
+; CHECK-P8-NEXT:    lvx v5, 0, r5
+; CHECK-P8-NEXT:    lvx v0, 0, r6
+; CHECK-P8-NEXT:    li r6, 16
+; CHECK-P8-NEXT:    addis r5, r2, .LCPI7_1@toc@ha
+; CHECK-P8-NEXT:    lvx v7, r4, r6
+; CHECK-P8-NEXT:    addi r5, r5, .LCPI7_1@toc@l
+; CHECK-P8-NEXT:    vperm v1, v4, v4, v2
 ; CHECK-P8-NEXT:    li r4, 112
-; CHECK-P8-NEXT:    xscvsxddp f6, f6
-; CHECK-P8-NEXT:    xscvsxddp f7, f7
-; CHECK-P8-NEXT:    xxmrghd vs0, vs1, vs0
-; CHECK-P8-NEXT:    xscvsxddp f9, f9
-; CHECK-P8-NEXT:    xscvsxddp f10, f10
-; CHECK-P8-NEXT:    xxmrghd vs1, vs2, vs5
-; CHECK-P8-NEXT:    xscvsxddp f11, f11
-; CHECK-P8-NEXT:    xxswapd vs2, vs3
-; CHECK-P8-NEXT:    xscvsxddp f8, f8
-; CHECK-P8-NEXT:    xxswapd vs0, vs0
-; CHECK-P8-NEXT:    xscvsxddp f12, f12
-; CHECK-P8-NEXT:    xxswapd vs1, vs1
-; CHECK-P8-NEXT:    xscvsxddp f13, f13
-; CHECK-P8-NEXT:    xxmrghd vs3, vs7, vs6
-; CHECK-P8-NEXT:    xscvsxddp f4, v2
-; CHECK-P8-NEXT:    stxvd2x vs2, r3, r4
-; CHECK-P8-NEXT:    li r4, 96
-; CHECK-P8-NEXT:    xscvsxddp f31, v3
-; CHECK-P8-NEXT:    xxmrghd vs5, vs10, vs9
+; CHECK-P8-NEXT:    vperm v6, v4, v4, v3
+; CHECK-P8-NEXT:    lxvd2x vs0, 0, r5
+; CHECK-P8-NEXT:    li r5, 96
+; CHECK-P8-NEXT:    vperm v8, v4, v4, v5
+; CHECK-P8-NEXT:    vperm v4, v4, v4, v0
+; CHECK-P8-NEXT:    vperm v5, v7, v7, v5
+; CHECK-P8-NEXT:    xxswapd v9, vs0
+; CHECK-P8-NEXT:    vperm v0, v7, v7, v0
+; CHECK-P8-NEXT:    vperm v2, v7, v7, v2
+; CHECK-P8-NEXT:    vperm v3, v7, v7, v3
+; CHECK-P8-NEXT:    vsld v1, v1, v9
+; CHECK-P8-NEXT:    vsld v6, v6, v9
+; CHECK-P8-NEXT:    vsld v5, v5, v9
+; CHECK-P8-NEXT:    vsld v0, v0, v9
+; CHECK-P8-NEXT:    vsld v2, v2, v9
+; CHECK-P8-NEXT:    vsld v3, v3, v9
+; CHECK-P8-NEXT:    vsrad v5, v5, v9
+; CHECK-P8-NEXT:    vsrad v0, v0, v9
+; CHECK-P8-NEXT:    vsld v7, v8, v9
+; CHECK-P8-NEXT:    vsld v4, v4, v9
+; CHECK-P8-NEXT:    vsrad v2, v2, v9
+; CHECK-P8-NEXT:    vsrad v3, v3, v9
+; CHECK-P8-NEXT:    xvcvsxddp vs2, v5
+; CHECK-P8-NEXT:    xvcvsxddp vs3, v0
+; CHECK-P8-NEXT:    vsrad v1, v1, v9
+; CHECK-P8-NEXT:    vsrad v6, v6, v9
+; CHECK-P8-NEXT:    vsrad v7, v7, v9
+; CHECK-P8-NEXT:    vsrad v4, v4, v9
+; CHECK-P8-NEXT:    xvcvsxddp vs1, v2
+; CHECK-P8-NEXT:    xxswapd vs2, vs2
+; CHECK-P8-NEXT:    xvcvsxddp vs4, v3
 ; CHECK-P8-NEXT:    xxswapd vs3, vs3
-; CHECK-P8-NEXT:    stxvd2x vs0, r3, r4
-; CHECK-P8-NEXT:    li r4, 48
-; CHECK-P8-NEXT:    xxmrghd vs6, vs8, vs11
-; CHECK-P8-NEXT:    xxmrghd vs7, vs13, vs12
+; CHECK-P8-NEXT:    xvcvsxddp vs0, v7
+; CHECK-P8-NEXT:    xvcvsxddp vs5, v4
+; CHECK-P8-NEXT:    xvcvsxddp vs6, v1
 ; CHECK-P8-NEXT:    stxvd2x vs3, r3, r4
 ; CHECK-P8-NEXT:    li r4, 80
-; CHECK-P8-NEXT:    xxswapd vs0, vs6
-; CHECK-P8-NEXT:    stxvd2x vs1, r3, r6
-; CHECK-P8-NEXT:    li r6, 64
-; CHECK-P8-NEXT:    xxmrghd vs2, vs31, vs4
-; CHECK-P8-NEXT:    xxswapd vs4, vs5
-; CHECK-P8-NEXT:    xxswapd vs5, vs7
-; CHECK-P8-NEXT:    stxvd2x vs0, r3, r4
-; CHECK-P8-NEXT:    xxswapd vs2, vs2
-; CHECK-P8-NEXT:    stxvd2x vs4, r3, r6
+; CHECK-P8-NEXT:    xvcvsxddp vs7, v6
 ; CHECK-P8-NEXT:    stxvd2x vs2, r3, r5
-; CHECK-P8-NEXT:    stxvd2x vs5, 0, r3
-; CHECK-P8-NEXT:    lfd f31, -8(r1) # 8-byte Folded Reload
+; CHECK-P8-NEXT:    li r5, 64
+; CHECK-P8-NEXT:    xxswapd vs1, vs1
+; CHECK-P8-NEXT:    xxswapd vs4, vs4
+; CHECK-P8-NEXT:    xxswapd vs0, vs0
+; CHECK-P8-NEXT:    xxswapd vs5, vs5
+; CHECK-P8-NEXT:    xxswapd vs3, vs6
+; CHECK-P8-NEXT:    stxvd2x vs4, r3, r4
+; CHECK-P8-NEXT:    li r4, 48
+; CHECK-P8-NEXT:    xxswapd vs2, vs7
+; CHECK-P8-NEXT:    stxvd2x vs1, r3, r5
+; CHECK-P8-NEXT:    li r5, 32
+; CHECK-P8-NEXT:    stxvd2x vs5, r3, r4
+; CHECK-P8-NEXT:    stxvd2x vs0, r3, r5
+; CHECK-P8-NEXT:    stxvd2x vs2, r3, r6
+; CHECK-P8-NEXT:    stxvd2x vs3, 0, r3
 ; CHECK-P8-NEXT:    blr
 ;
 ; CHECK-P9-LABEL: test16elt_signed:
 ; CHECK-P9:       # %bb.0: # %entry
 ; CHECK-P9-NEXT:    addis r5, r2, .LCPI7_0@toc@ha
-; CHECK-P9-NEXT:    addis r6, r2, .LCPI7_1@toc@ha
-; CHECK-P9-NEXT:    addis r7, r2, .LCPI7_2@toc@ha
-; CHECK-P9-NEXT:    addis r8, r2, .LCPI7_3@toc@ha
-; CHECK-P9-NEXT:    lxv v0, 0(r4)
-; CHECK-P9-NEXT:    lxv v1, 16(r4)
 ; CHECK-P9-NEXT:    addi r5, r5, .LCPI7_0@toc@l
-; CHECK-P9-NEXT:    addi r6, r6, .LCPI7_1@toc@l
-; CHECK-P9-NEXT:    addi r7, r7, .LCPI7_2@toc@l
-; CHECK-P9-NEXT:    addi r8, r8, .LCPI7_3@toc@l
-; CHECK-P9-NEXT:    lxvx v2, 0, r5
-; CHECK-P9-NEXT:    lxvx v3, 0, r6
-; CHECK-P9-NEXT:    lxvx v4, 0, r7
-; CHECK-P9-NEXT:    lxvx v5, 0, r8
-; CHECK-P9-NEXT:    vperm v6, v0, v0, v2
-; CHECK-P9-NEXT:    vperm v7, v0, v0, v3
-; CHECK-P9-NEXT:    vperm v8, v0, v0, v4
-; CHECK-P9-NEXT:    vperm v0, v0, v0, v5
-; CHECK-P9-NEXT:    vperm v2, v1, v1, v2
-; CHECK-P9-NEXT:    vperm v3, v1, v1, v3
-; CHECK-P9-NEXT:    vperm v4, v1, v1, v4
-; CHECK-P9-NEXT:    vperm v5, v1, v1, v5
-; CHECK-P9-NEXT:    vextsh2d v1, v6
-; CHECK-P9-NEXT:    vextsh2d v6, v7
-; CHECK-P9-NEXT:    vextsh2d v7, v8
-; CHECK-P9-NEXT:    vextsh2d v0, v0
-; CHECK-P9-NEXT:    vextsh2d v2, v2
-; CHECK-P9-NEXT:    vextsh2d v3, v3
+; CHECK-P9-NEXT:    lxv v2, 0(r4)
+; CHECK-P9-NEXT:    lxvx v3, 0, r5
+; CHECK-P9-NEXT:    addis r5, r2, .LCPI7_1@toc@ha
+; CHECK-P9-NEXT:    addi r5, r5, .LCPI7_1@toc@l
+; CHECK-P9-NEXT:    lxvx v5, 0, r5
+; CHECK-P9-NEXT:    addis r5, r2, .LCPI7_2@toc@ha
+; CHECK-P9-NEXT:    vperm v4, v2, v2, v3
+; CHECK-P9-NEXT:    addi r5, r5, .LCPI7_2@toc@l
 ; CHECK-P9-NEXT:    vextsh2d v4, v4
-; CHECK-P9-NEXT:    vextsh2d v5, v5
-; CHECK-P9-NEXT:    xvcvsxddp vs0, v1
-; CHECK-P9-NEXT:    xvcvsxddp vs1, v6
-; CHECK-P9-NEXT:    xvcvsxddp vs2, v7
-; CHECK-P9-NEXT:    xvcvsxddp vs3, v0
-; CHECK-P9-NEXT:    xvcvsxddp vs4, v2
-; CHECK-P9-NEXT:    xvcvsxddp vs5, v3
-; CHECK-P9-NEXT:    xvcvsxddp vs6, v4
-; CHECK-P9-NEXT:    xvcvsxddp vs7, v5
+; CHECK-P9-NEXT:    lxvx v0, 0, r5
+; CHECK-P9-NEXT:    addis r5, r2, .LCPI7_3@toc@ha
+; CHECK-P9-NEXT:    xvcvsxddp vs0, v4
+; CHECK-P9-NEXT:    vperm v4, v2, v2, v5
+; CHECK-P9-NEXT:    addi r5, r5, .LCPI7_3@toc@l
+; CHECK-P9-NEXT:    lxvx v1, 0, r5
+; CHECK-P9-NEXT:    vextsh2d v4, v4
+; CHECK-P9-NEXT:    xvcvsxddp vs1, v4
+; CHECK-P9-NEXT:    vperm v4, v2, v2, v0
+; CHECK-P9-NEXT:    vperm v2, v2, v2, v1
+; CHECK-P9-NEXT:    vextsh2d v4, v4
+; CHECK-P9-NEXT:    xvcvsxddp vs2, v4
+; CHECK-P9-NEXT:    lxv v4, 16(r4)
+; CHECK-P9-NEXT:    vextsh2d v2, v2
+; CHECK-P9-NEXT:    xvcvsxddp vs3, v2
+; CHECK-P9-NEXT:    vperm v2, v4, v4, v3
+; CHECK-P9-NEXT:    vextsh2d v2, v2
 ; CHECK-P9-NEXT:    stxv vs3, 48(r3)
+; CHECK-P9-NEXT:    xvcvsxddp vs4, v2
+; CHECK-P9-NEXT:    vperm v2, v4, v4, v5
+; CHECK-P9-NEXT:    vextsh2d v2, v2
+; CHECK-P9-NEXT:    xvcvsxddp vs5, v2
+; CHECK-P9-NEXT:    vperm v2, v4, v4, v0
+; CHECK-P9-NEXT:    stxv vs4, 64(r3)
+; CHECK-P9-NEXT:    stxv vs5, 80(r3)
+; CHECK-P9-NEXT:    vextsh2d v2, v2
+; CHECK-P9-NEXT:    xvcvsxddp vs6, v2
+; CHECK-P9-NEXT:    vperm v2, v4, v4, v1
+; CHECK-P9-NEXT:    vextsh2d v2, v2
+; CHECK-P9-NEXT:    stxv vs6, 96(r3)
+; CHECK-P9-NEXT:    xvcvsxddp vs7, v2
+; CHECK-P9-NEXT:    stxv vs7, 112(r3)
 ; CHECK-P9-NEXT:    stxv vs2, 32(r3)
 ; CHECK-P9-NEXT:    stxv vs1, 16(r3)
 ; CHECK-P9-NEXT:    stxv vs0, 0(r3)
-; CHECK-P9-NEXT:    stxv vs7, 112(r3)
-; CHECK-P9-NEXT:    stxv vs6, 96(r3)
-; CHECK-P9-NEXT:    stxv vs5, 80(r3)
-; CHECK-P9-NEXT:    stxv vs4, 64(r3)
 ; CHECK-P9-NEXT:    blr
 ;
 ; CHECK-BE-LABEL: test16elt_signed:
 ; CHECK-BE:       # %bb.0: # %entry
 ; CHECK-BE-NEXT:    addis r5, r2, .LCPI7_0@toc@ha
-; CHECK-BE-NEXT:    addis r6, r2, .LCPI7_1@toc@ha
-; CHECK-BE-NEXT:    addis r7, r2, .LCPI7_2@toc@ha
-; CHECK-BE-NEXT:    addis r8, r2, .LCPI7_3@toc@ha
-; CHECK-BE-NEXT:    lxv v2, 16(r4)
-; CHECK-BE-NEXT:    lxv v3, 0(r4)
-; CHECK-BE-NEXT:    xxlxor v6, v6, v6
 ; CHECK-BE-NEXT:    addi r5, r5, .LCPI7_0@toc@l
-; CHECK-BE-NEXT:    addi r6, r6, .LCPI7_1@toc@l
-; CHECK-BE-NEXT:    addi r7, r7, .LCPI7_2@toc@l
-; CHECK-BE-NEXT:    addi r8, r8, .LCPI7_3@toc@l
-; CHECK-BE-NEXT:    lxvx v4, 0, r5
-; CHECK-BE-NEXT:    lxvx v5, 0, r6
-; CHECK-BE-NEXT:    lxvx v0, 0, r7
-; CHECK-BE-NEXT:    lxvx v1, 0, r8
-; CHECK-BE-NEXT:    vperm v7, v6, v3, v4
-; CHECK-BE-NEXT:    vperm v8, v6, v3, v5
-; CHECK-BE-NEXT:    vperm v4, v6, v2, v4
-; CHECK-BE-NEXT:    vperm v5, v6, v2, v5
-; CHECK-BE-NEXT:    vperm v6, v3, v3, v0
-; CHECK-BE-NEXT:    vperm v3, v3, v3, v1
-; CHECK-BE-NEXT:    vperm v0, v2, v2, v0
-; CHECK-BE-NEXT:    vperm v2, v2, v2, v1
-; CHECK-BE-NEXT:    vextsh2d v1, v7
-; CHECK-BE-NEXT:    vextsh2d v7, v8
-; CHECK-BE-NEXT:    vextsh2d v4, v4
-; CHECK-BE-NEXT:    vextsh2d v5, v5
-; CHECK-BE-NEXT:    vextsh2d v6, v6
-; CHECK-BE-NEXT:    vextsh2d v3, v3
+; CHECK-BE-NEXT:    lxvx v2, 0, r5
+; CHECK-BE-NEXT:    lxv v4, 0(r4)
+; CHECK-BE-NEXT:    lxv v1, 16(r4)
+; CHECK-BE-NEXT:    addis r5, r2, .LCPI7_1@toc@ha
+; CHECK-BE-NEXT:    addi r5, r5, .LCPI7_1@toc@l
+; CHECK-BE-NEXT:    addis r4, r2, .LCPI7_2@toc@ha
+; CHECK-BE-NEXT:    xxlxor v5, v5, v5
+; CHECK-BE-NEXT:    vperm v0, v5, v4, v2
+; CHECK-BE-NEXT:    lxvx v3, 0, r5
+; CHECK-BE-NEXT:    vperm v2, v5, v1, v2
+; CHECK-BE-NEXT:    vextsh2d v2, v2
+; CHECK-BE-NEXT:    addi r4, r4, .LCPI7_2@toc@l
 ; CHECK-BE-NEXT:    vextsh2d v0, v0
+; CHECK-BE-NEXT:    xvcvsxddp vs2, v2
+; CHECK-BE-NEXT:    vperm v2, v5, v1, v3
 ; CHECK-BE-NEXT:    vextsh2d v2, v2
-; CHECK-BE-NEXT:    xvcvsxddp vs0, v1
-; CHECK-BE-NEXT:    xvcvsxddp vs1, v7
-; CHECK-BE-NEXT:    xvcvsxddp vs2, v4
-; CHECK-BE-NEXT:    xvcvsxddp vs3, v5
-; CHECK-BE-NEXT:    xvcvsxddp vs4, v6
-; CHECK-BE-NEXT:    xvcvsxddp vs5, v3
-; CHECK-BE-NEXT:    xvcvsxddp vs6, v0
-; CHECK-BE-NEXT:    xvcvsxddp vs7, v2
-; CHECK-BE-NEXT:    stxv vs3, 112(r3)
 ; CHECK-BE-NEXT:    stxv vs2, 80(r3)
+; CHECK-BE-NEXT:    xvcvsxddp vs3, v2
+; CHECK-BE-NEXT:    lxvx v2, 0, r4
+; CHECK-BE-NEXT:    xvcvsxddp vs0, v0
+; CHECK-BE-NEXT:    vperm v0, v5, v4, v3
+; CHECK-BE-NEXT:    vperm v3, v4, v4, v2
+; CHECK-BE-NEXT:    addis r4, r2, .LCPI7_3@toc@ha
+; CHECK-BE-NEXT:    vextsh2d v0, v0
+; CHECK-BE-NEXT:    xvcvsxddp vs1, v0
 ; CHECK-BE-NEXT:    stxv vs1, 48(r3)
+; CHECK-BE-NEXT:    vextsh2d v3, v3
+; CHECK-BE-NEXT:    addi r4, r4, .LCPI7_3@toc@l
+; CHECK-BE-NEXT:    xvcvsxddp vs4, v3
+; CHECK-BE-NEXT:    lxvx v3, 0, r4
+; CHECK-BE-NEXT:    vperm v2, v1, v1, v2
+; CHECK-BE-NEXT:    vextsh2d v2, v2
+; CHECK-BE-NEXT:    xvcvsxddp vs6, v2
+; CHECK-BE-NEXT:    vperm v2, v1, v1, v3
+; CHECK-BE-NEXT:    vperm v4, v4, v4, v3
+; CHECK-BE-NEXT:    vextsh2d v4, v4
+; CHECK-BE-NEXT:    vextsh2d v2, v2
+; CHECK-BE-NEXT:    xvcvsxddp vs7, v2
+; CHECK-BE-NEXT:    xvcvsxddp vs5, v4
+; CHECK-BE-NEXT:    stxv vs3, 112(r3)
+; CHECK-BE-NEXT:    stxv vs6, 64(r3)
 ; CHECK-BE-NEXT:    stxv vs0, 16(r3)
+; CHECK-BE-NEXT:    stxv vs4, 0(r3)
 ; CHECK-BE-NEXT:    stxv vs7, 96(r3)
-; CHECK-BE-NEXT:    stxv vs6, 64(r3)
 ; CHECK-BE-NEXT:    stxv vs5, 32(r3)
-; CHECK-BE-NEXT:    stxv vs4, 0(r3)
 ; CHECK-BE-NEXT:    blr
 entry:
   %a = load <16 x i16>, <16 x i16>* %0, align 32
diff --git a/test/CodeGen/PowerPC/vec_conv_i32_to_fp64_elts.ll b/test/CodeGen/PowerPC/vec_conv_i32_to_fp64_elts.ll
index 39b88bab3986d5c119d9c2bd00aa23e8202e2a7d..2609cb343004314ff7d9d34e8d752defa8cead79 100644
--- a/test/CodeGen/PowerPC/vec_conv_i32_to_fp64_elts.ll
+++ b/test/CodeGen/PowerPC/vec_conv_i32_to_fp64_elts.ll
@@ -105,38 +105,38 @@ define void @test8elt(<8 x double>* noalias nocapture sret %agg.result, <8 x i32
 ;
 ; CHECK-P9-LABEL: test8elt:
 ; CHECK-P9:       # %bb.0: # %entry
-; CHECK-P9-NEXT:    lxv vs0, 16(r4)
 ; CHECK-P9-NEXT:    lxv vs1, 0(r4)
 ; CHECK-P9-NEXT:    xxmrglw v2, vs1, vs1
-; CHECK-P9-NEXT:    xxmrghw v3, vs1, vs1
-; CHECK-P9-NEXT:    xxmrglw v4, vs0, vs0
-; CHECK-P9-NEXT:    xxmrghw v5, vs0, vs0
+; CHECK-P9-NEXT:    lxv vs0, 16(r4)
+; CHECK-P9-NEXT:    xvcvuxwdp vs2, v2
+; CHECK-P9-NEXT:    xxmrghw v2, vs1, vs1
+; CHECK-P9-NEXT:    xvcvuxwdp vs1, v2
+; CHECK-P9-NEXT:    xxmrglw v2, vs0, vs0
+; CHECK-P9-NEXT:    xvcvuxwdp vs3, v2
+; CHECK-P9-NEXT:    xxmrghw v2, vs0, vs0
+; CHECK-P9-NEXT:    stxv vs2, 0(r3)
 ; CHECK-P9-NEXT:    xvcvuxwdp vs0, v2
-; CHECK-P9-NEXT:    xvcvuxwdp vs1, v3
-; CHECK-P9-NEXT:    xvcvuxwdp vs2, v4
-; CHECK-P9-NEXT:    xvcvuxwdp vs3, v5
-; CHECK-P9-NEXT:    stxv vs3, 48(r3)
-; CHECK-P9-NEXT:    stxv vs2, 32(r3)
 ; CHECK-P9-NEXT:    stxv vs1, 16(r3)
-; CHECK-P9-NEXT:    stxv vs0, 0(r3)
+; CHECK-P9-NEXT:    stxv vs3, 32(r3)
+; CHECK-P9-NEXT:    stxv vs0, 48(r3)
 ; CHECK-P9-NEXT:    blr
 ;
 ; CHECK-BE-LABEL: test8elt:
 ; CHECK-BE:       # %bb.0: # %entry
-; CHECK-BE-NEXT:    lxv vs0, 16(r4)
 ; CHECK-BE-NEXT:    lxv vs1, 0(r4)
 ; CHECK-BE-NEXT:    xxmrghw v2, vs1, vs1
-; CHECK-BE-NEXT:    xxmrglw v3, vs1, vs1
-; CHECK-BE-NEXT:    xxmrghw v4, vs0, vs0
-; CHECK-BE-NEXT:    xxmrglw v5, vs0, vs0
+; CHECK-BE-NEXT:    lxv vs0, 16(r4)
+; CHECK-BE-NEXT:    xvcvuxwdp vs2, v2
+; CHECK-BE-NEXT:    xxmrglw v2, vs1, vs1
+; CHECK-BE-NEXT:    xvcvuxwdp vs1, v2
+; CHECK-BE-NEXT:    xxmrghw v2, vs0, vs0
+; CHECK-BE-NEXT:    xvcvuxwdp vs3, v2
+; CHECK-BE-NEXT:    xxmrglw v2, vs0, vs0
+; CHECK-BE-NEXT:    stxv vs2, 0(r3)
 ; CHECK-BE-NEXT:    xvcvuxwdp vs0, v2
-; CHECK-BE-NEXT:    xvcvuxwdp vs1, v3
-; CHECK-BE-NEXT:    xvcvuxwdp vs2, v4
-; CHECK-BE-NEXT:    xvcvuxwdp vs3, v5
-; CHECK-BE-NEXT:    stxv vs3, 48(r3)
-; CHECK-BE-NEXT:    stxv vs2, 32(r3)
 ; CHECK-BE-NEXT:    stxv vs1, 16(r3)
-; CHECK-BE-NEXT:    stxv vs0, 0(r3)
+; CHECK-BE-NEXT:    stxv vs3, 32(r3)
+; CHECK-BE-NEXT:    stxv vs0, 48(r3)
 ; CHECK-BE-NEXT:    blr
 entry:
   %a = load <8 x i32>, <8 x i32>* %0, align 32
@@ -195,66 +195,66 @@ define void @test16elt(<16 x double>* noalias nocapture sret %agg.result, <16 x
 ;
 ; CHECK-P9-LABEL: test16elt:
 ; CHECK-P9:       # %bb.0: # %entry
-; CHECK-P9-NEXT:    lxv vs0, 16(r4)
-; CHECK-P9-NEXT:    lxv vs1, 0(r4)
-; CHECK-P9-NEXT:    lxv vs2, 48(r4)
-; CHECK-P9-NEXT:    lxv vs3, 32(r4)
-; CHECK-P9-NEXT:    xxmrglw v2, vs1, vs1
-; CHECK-P9-NEXT:    xxmrghw v3, vs1, vs1
-; CHECK-P9-NEXT:    xxmrglw v4, vs0, vs0
-; CHECK-P9-NEXT:    xxmrghw v5, vs0, vs0
-; CHECK-P9-NEXT:    xxmrglw v0, vs3, vs3
-; CHECK-P9-NEXT:    xxmrghw v1, vs3, vs3
-; CHECK-P9-NEXT:    xxmrglw v6, vs2, vs2
-; CHECK-P9-NEXT:    xxmrghw v7, vs2, vs2
+; CHECK-P9-NEXT:    lxv vs0, 0(r4)
+; CHECK-P9-NEXT:    xxmrglw v2, vs0, vs0
+; CHECK-P9-NEXT:    lxv vs2, 16(r4)
+; CHECK-P9-NEXT:    lxv vs4, 48(r4)
+; CHECK-P9-NEXT:    xvcvuxwdp vs1, v2
+; CHECK-P9-NEXT:    xxmrghw v2, vs0, vs0
+; CHECK-P9-NEXT:    lxv vs5, 32(r4)
 ; CHECK-P9-NEXT:    xvcvuxwdp vs0, v2
-; CHECK-P9-NEXT:    xvcvuxwdp vs1, v3
-; CHECK-P9-NEXT:    xvcvuxwdp vs2, v4
-; CHECK-P9-NEXT:    xvcvuxwdp vs3, v5
-; CHECK-P9-NEXT:    xvcvuxwdp vs4, v0
-; CHECK-P9-NEXT:    xvcvuxwdp vs5, v1
-; CHECK-P9-NEXT:    xvcvuxwdp vs6, v6
-; CHECK-P9-NEXT:    xvcvuxwdp vs7, v7
-; CHECK-P9-NEXT:    stxv vs3, 48(r3)
-; CHECK-P9-NEXT:    stxv vs2, 32(r3)
-; CHECK-P9-NEXT:    stxv vs1, 16(r3)
-; CHECK-P9-NEXT:    stxv vs0, 0(r3)
-; CHECK-P9-NEXT:    stxv vs7, 112(r3)
-; CHECK-P9-NEXT:    stxv vs6, 96(r3)
+; CHECK-P9-NEXT:    xxmrglw v2, vs2, vs2
+; CHECK-P9-NEXT:    xvcvuxwdp vs3, v2
+; CHECK-P9-NEXT:    xxmrghw v2, vs2, vs2
+; CHECK-P9-NEXT:    stxv vs1, 0(r3)
+; CHECK-P9-NEXT:    stxv vs0, 16(r3)
+; CHECK-P9-NEXT:    xvcvuxwdp vs2, v2
+; CHECK-P9-NEXT:    xxmrglw v2, vs5, vs5
+; CHECK-P9-NEXT:    xvcvuxwdp vs6, v2
+; CHECK-P9-NEXT:    xxmrghw v2, vs5, vs5
+; CHECK-P9-NEXT:    stxv vs3, 32(r3)
+; CHECK-P9-NEXT:    stxv vs2, 48(r3)
+; CHECK-P9-NEXT:    xvcvuxwdp vs5, v2
+; CHECK-P9-NEXT:    xxmrglw v2, vs4, vs4
+; CHECK-P9-NEXT:    xvcvuxwdp vs7, v2
+; CHECK-P9-NEXT:    xxmrghw v2, vs4, vs4
+; CHECK-P9-NEXT:    stxv vs6, 64(r3)
 ; CHECK-P9-NEXT:    stxv vs5, 80(r3)
-; CHECK-P9-NEXT:    stxv vs4, 64(r3)
+; CHECK-P9-NEXT:    xvcvuxwdp vs4, v2
+; CHECK-P9-NEXT:    stxv vs7, 96(r3)
+; CHECK-P9-NEXT:    stxv vs4, 112(r3)
 ; CHECK-P9-NEXT:    blr
 ;
 ; CHECK-BE-LABEL: test16elt:
 ; CHECK-BE:       # %bb.0: # %entry
-; CHECK-BE-NEXT:    lxv vs0, 16(r4)
-; CHECK-BE-NEXT:    lxv vs1, 0(r4)
-; CHECK-BE-NEXT:    lxv vs2, 48(r4)
-; CHECK-BE-NEXT:    lxv vs3, 32(r4)
-; CHECK-BE-NEXT:    xxmrghw v2, vs1, vs1
-; CHECK-BE-NEXT:    xxmrglw v3, vs1, vs1
-; CHECK-BE-NEXT:    xxmrghw v4, vs0, vs0
-; CHECK-BE-NEXT:    xxmrglw v5, vs0, vs0
-; CHECK-BE-NEXT:    xxmrghw v0, vs3, vs3
-; CHECK-BE-NEXT:    xxmrglw v1, vs3, vs3
-; CHECK-BE-NEXT:    xxmrghw v6, vs2, vs2
-; CHECK-BE-NEXT:    xxmrglw v7, vs2, vs2
+; CHECK-BE-NEXT:    lxv vs0, 0(r4)
+; CHECK-BE-NEXT:    xxmrghw v2, vs0, vs0
+; CHECK-BE-NEXT:    lxv vs2, 16(r4)
+; CHECK-BE-NEXT:    lxv vs4, 48(r4)
+; CHECK-BE-NEXT:    xvcvuxwdp vs1, v2
+; CHECK-BE-NEXT:    xxmrglw v2, vs0, vs0
+; CHECK-BE-NEXT:    lxv vs5, 32(r4)
 ; CHECK-BE-NEXT:    xvcvuxwdp vs0, v2
-; CHECK-BE-NEXT:    xvcvuxwdp vs1, v3
-; CHECK-BE-NEXT:    xvcvuxwdp vs2, v4
-; CHECK-BE-NEXT:    xvcvuxwdp vs3, v5
-; CHECK-BE-NEXT:    xvcvuxwdp vs4, v0
-; CHECK-BE-NEXT:    xvcvuxwdp vs5, v1
-; CHECK-BE-NEXT:    xvcvuxwdp vs6, v6
-; CHECK-BE-NEXT:    xvcvuxwdp vs7, v7
-; CHECK-BE-NEXT:    stxv vs3, 48(r3)
-; CHECK-BE-NEXT:    stxv vs2, 32(r3)
-; CHECK-BE-NEXT:    stxv vs1, 16(r3)
-; CHECK-BE-NEXT:    stxv vs0, 0(r3)
-; CHECK-BE-NEXT:    stxv vs7, 112(r3)
-; CHECK-BE-NEXT:    stxv vs6, 96(r3)
+; CHECK-BE-NEXT:    xxmrghw v2, vs2, vs2
+; CHECK-BE-NEXT:    xvcvuxwdp vs3, v2
+; CHECK-BE-NEXT:    xxmrglw v2, vs2, vs2
+; CHECK-BE-NEXT:    stxv vs1, 0(r3)
+; CHECK-BE-NEXT:    stxv vs0, 16(r3)
+; CHECK-BE-NEXT:    xvcvuxwdp vs2, v2
+; CHECK-BE-NEXT:    xxmrghw v2, vs5, vs5
+; CHECK-BE-NEXT:    xvcvuxwdp vs6, v2
+; CHECK-BE-NEXT:    xxmrglw v2, vs5, vs5
+; CHECK-BE-NEXT:    stxv vs3, 32(r3)
+; CHECK-BE-NEXT:    stxv vs2, 48(r3)
+; CHECK-BE-NEXT:    xvcvuxwdp vs5, v2
+; CHECK-BE-NEXT:    xxmrghw v2, vs4, vs4
+; CHECK-BE-NEXT:    xvcvuxwdp vs7, v2
+; CHECK-BE-NEXT:    xxmrglw v2, vs4, vs4
+; CHECK-BE-NEXT:    stxv vs6, 64(r3)
 ; CHECK-BE-NEXT:    stxv vs5, 80(r3)
-; CHECK-BE-NEXT:    stxv vs4, 64(r3)
+; CHECK-BE-NEXT:    xvcvuxwdp vs4, v2
+; CHECK-BE-NEXT:    stxv vs7, 96(r3)
+; CHECK-BE-NEXT:    stxv vs4, 112(r3)
 ; CHECK-BE-NEXT:    blr
 entry:
   %a = load <16 x i32>, <16 x i32>* %0, align 64
@@ -359,38 +359,38 @@ define void @test8elt_signed(<8 x double>* noalias nocapture sret %agg.result, <
 ;
 ; CHECK-P9-LABEL: test8elt_signed:
 ; CHECK-P9:       # %bb.0: # %entry
-; CHECK-P9-NEXT:    lxv vs0, 16(r4)
 ; CHECK-P9-NEXT:    lxv vs1, 0(r4)
 ; CHECK-P9-NEXT:    xxmrglw v2, vs1, vs1
-; CHECK-P9-NEXT:    xxmrghw v3, vs1, vs1
-; CHECK-P9-NEXT:    xxmrglw v4, vs0, vs0
-; CHECK-P9-NEXT:    xxmrghw v5, vs0, vs0
+; CHECK-P9-NEXT:    lxv vs0, 16(r4)
+; CHECK-P9-NEXT:    xvcvsxwdp vs2, v2
+; CHECK-P9-NEXT:    xxmrghw v2, vs1, vs1
+; CHECK-P9-NEXT:    xvcvsxwdp vs1, v2
+; CHECK-P9-NEXT:    xxmrglw v2, vs0, vs0
+; CHECK-P9-NEXT:    xvcvsxwdp vs3, v2
+; CHECK-P9-NEXT:    xxmrghw v2, vs0, vs0
+; CHECK-P9-NEXT:    stxv vs2, 0(r3)
 ; CHECK-P9-NEXT:    xvcvsxwdp vs0, v2
-; CHECK-P9-NEXT:    xvcvsxwdp vs1, v3
-; CHECK-P9-NEXT:    xvcvsxwdp vs2, v4
-; CHECK-P9-NEXT:    xvcvsxwdp vs3, v5
-; CHECK-P9-NEXT:    stxv vs3, 48(r3)
-; CHECK-P9-NEXT:    stxv vs2, 32(r3)
 ; CHECK-P9-NEXT:    stxv vs1, 16(r3)
-; CHECK-P9-NEXT:    stxv vs0, 0(r3)
+; CHECK-P9-NEXT:    stxv vs3, 32(r3)
+; CHECK-P9-NEXT:    stxv vs0, 48(r3)
 ; CHECK-P9-NEXT:    blr
 ;
 ; CHECK-BE-LABEL: test8elt_signed:
 ; CHECK-BE:       # %bb.0: # %entry
-; CHECK-BE-NEXT:    lxv vs0, 16(r4)
 ; CHECK-BE-NEXT:    lxv vs1, 0(r4)
 ; CHECK-BE-NEXT:    xxmrghw v2, vs1, vs1
-; CHECK-BE-NEXT:    xxmrglw v3, vs1, vs1
-; CHECK-BE-NEXT:    xxmrghw v4, vs0, vs0
-; CHECK-BE-NEXT:    xxmrglw v5, vs0, vs0
+; CHECK-BE-NEXT:    lxv vs0, 16(r4)
+; CHECK-BE-NEXT:    xvcvsxwdp vs2, v2
+; CHECK-BE-NEXT:    xxmrglw v2, vs1, vs1
+; CHECK-BE-NEXT:    xvcvsxwdp vs1, v2
+; CHECK-BE-NEXT:    xxmrghw v2, vs0, vs0
+; CHECK-BE-NEXT:    xvcvsxwdp vs3, v2
+; CHECK-BE-NEXT:    xxmrglw v2, vs0, vs0
+; CHECK-BE-NEXT:    stxv vs2, 0(r3)
 ; CHECK-BE-NEXT:    xvcvsxwdp vs0, v2
-; CHECK-BE-NEXT:    xvcvsxwdp vs1, v3
-; CHECK-BE-NEXT:    xvcvsxwdp vs2, v4
-; CHECK-BE-NEXT:    xvcvsxwdp vs3, v5
-; CHECK-BE-NEXT:    stxv vs3, 48(r3)
-; CHECK-BE-NEXT:    stxv vs2, 32(r3)
 ; CHECK-BE-NEXT:    stxv vs1, 16(r3)
-; CHECK-BE-NEXT:    stxv vs0, 0(r3)
+; CHECK-BE-NEXT:    stxv vs3, 32(r3)
+; CHECK-BE-NEXT:    stxv vs0, 48(r3)
 ; CHECK-BE-NEXT:    blr
 entry:
   %a = load <8 x i32>, <8 x i32>* %0, align 32
@@ -449,66 +449,66 @@ define void @test16elt_signed(<16 x double>* noalias nocapture sret %agg.result,
 ;
 ; CHECK-P9-LABEL: test16elt_signed:
 ; CHECK-P9:       # %bb.0: # %entry
-; CHECK-P9-NEXT:    lxv vs0, 16(r4)
-; CHECK-P9-NEXT:    lxv vs1, 0(r4)
-; CHECK-P9-NEXT:    lxv vs2, 48(r4)
-; CHECK-P9-NEXT:    lxv vs3, 32(r4)
-; CHECK-P9-NEXT:    xxmrglw v2, vs1, vs1
-; CHECK-P9-NEXT:    xxmrghw v3, vs1, vs1
-; CHECK-P9-NEXT:    xxmrglw v4, vs0, vs0
-; CHECK-P9-NEXT:    xxmrghw v5, vs0, vs0
-; CHECK-P9-NEXT:    xxmrglw v0, vs3, vs3
-; CHECK-P9-NEXT:    xxmrghw v1, vs3, vs3
-; CHECK-P9-NEXT:    xxmrglw v6, vs2, vs2
-; CHECK-P9-NEXT:    xxmrghw v7, vs2, vs2
+; CHECK-P9-NEXT:    lxv vs0, 0(r4)
+; CHECK-P9-NEXT:    xxmrglw v2, vs0, vs0
+; CHECK-P9-NEXT:    lxv vs2, 16(r4)
+; CHECK-P9-NEXT:    lxv vs4, 48(r4)
+; CHECK-P9-NEXT:    xvcvsxwdp vs1, v2
+; CHECK-P9-NEXT:    xxmrghw v2, vs0, vs0
+; CHECK-P9-NEXT:    lxv vs5, 32(r4)
 ; CHECK-P9-NEXT:    xvcvsxwdp vs0, v2
-; CHECK-P9-NEXT:    xvcvsxwdp vs1, v3
-; CHECK-P9-NEXT:    xvcvsxwdp vs2, v4
-; CHECK-P9-NEXT:    xvcvsxwdp vs3, v5
-; CHECK-P9-NEXT:    xvcvsxwdp vs4, v0
-; CHECK-P9-NEXT:    xvcvsxwdp vs5, v1
-; CHECK-P9-NEXT:    xvcvsxwdp vs6, v6
-; CHECK-P9-NEXT:    xvcvsxwdp vs7, v7
-; CHECK-P9-NEXT:    stxv vs3, 48(r3)
-; CHECK-P9-NEXT:    stxv vs2, 32(r3)
-; CHECK-P9-NEXT:    stxv vs1, 16(r3)
-; CHECK-P9-NEXT:    stxv vs0, 0(r3)
-; CHECK-P9-NEXT:    stxv vs7, 112(r3)
-; CHECK-P9-NEXT:    stxv vs6, 96(r3)
+; CHECK-P9-NEXT:    xxmrglw v2, vs2, vs2
+; CHECK-P9-NEXT:    xvcvsxwdp vs3, v2
+; CHECK-P9-NEXT:    xxmrghw v2, vs2, vs2
+; CHECK-P9-NEXT:    stxv vs1, 0(r3)
+; CHECK-P9-NEXT:    stxv vs0, 16(r3)
+; CHECK-P9-NEXT:    xvcvsxwdp vs2, v2
+; CHECK-P9-NEXT:    xxmrglw v2, vs5, vs5
+; CHECK-P9-NEXT:    xvcvsxwdp vs6, v2
+; CHECK-P9-NEXT:    xxmrghw v2, vs5, vs5
+; CHECK-P9-NEXT:    stxv vs3, 32(r3)
+; CHECK-P9-NEXT:    stxv vs2, 48(r3)
+; CHECK-P9-NEXT:    xvcvsxwdp vs5, v2
+; CHECK-P9-NEXT:    xxmrglw v2, vs4, vs4
+; CHECK-P9-NEXT:    xvcvsxwdp vs7, v2
+; CHECK-P9-NEXT:    xxmrghw v2, vs4, vs4
+; CHECK-P9-NEXT:    stxv vs6, 64(r3)
 ; CHECK-P9-NEXT:    stxv vs5, 80(r3)
-; CHECK-P9-NEXT:    stxv vs4, 64(r3)
+; CHECK-P9-NEXT:    xvcvsxwdp vs4, v2
+; CHECK-P9-NEXT:    stxv vs7, 96(r3)
+; CHECK-P9-NEXT:    stxv vs4, 112(r3)
 ; CHECK-P9-NEXT:    blr
 ;
 ; CHECK-BE-LABEL: test16elt_signed:
 ; CHECK-BE:       # %bb.0: # %entry
-; CHECK-BE-NEXT:    lxv vs0, 16(r4)
-; CHECK-BE-NEXT:    lxv vs1, 0(r4)
-; CHECK-BE-NEXT:    lxv vs2, 48(r4)
-; CHECK-BE-NEXT:    lxv vs3, 32(r4)
-; CHECK-BE-NEXT:    xxmrghw v2, vs1, vs1
-; CHECK-BE-NEXT:    xxmrglw v3, vs1, vs1
-; CHECK-BE-NEXT:    xxmrghw v4, vs0, vs0
-; CHECK-BE-NEXT:    xxmrglw v5, vs0, vs0
-; CHECK-BE-NEXT:    xxmrghw v0, vs3, vs3
-; CHECK-BE-NEXT:    xxmrglw v1, vs3, vs3
-; CHECK-BE-NEXT:    xxmrghw v6, vs2, vs2
-; CHECK-BE-NEXT:    xxmrglw v7, vs2, vs2
+; CHECK-BE-NEXT:    lxv vs0, 0(r4)
+; CHECK-BE-NEXT:    xxmrghw v2, vs0, vs0
+; CHECK-BE-NEXT:    lxv vs2, 16(r4)
+; CHECK-BE-NEXT:    lxv vs4, 48(r4)
+; CHECK-BE-NEXT:    xvcvsxwdp vs1, v2
+; CHECK-BE-NEXT:    xxmrglw v2, vs0, vs0
+; CHECK-BE-NEXT:    lxv vs5, 32(r4)
 ; CHECK-BE-NEXT:    xvcvsxwdp vs0, v2
-; CHECK-BE-NEXT:    xvcvsxwdp vs1, v3
-; CHECK-BE-NEXT:    xvcvsxwdp vs2, v4
-; CHECK-BE-NEXT:    xvcvsxwdp vs3, v5
-; CHECK-BE-NEXT:    xvcvsxwdp vs4, v0
-; CHECK-BE-NEXT:    xvcvsxwdp vs5, v1
-; CHECK-BE-NEXT:    xvcvsxwdp vs6, v6
-; CHECK-BE-NEXT:    xvcvsxwdp vs7, v7
-; CHECK-BE-NEXT:    stxv vs3, 48(r3)
-; CHECK-BE-NEXT:    stxv vs2, 32(r3)
-; CHECK-BE-NEXT:    stxv vs1, 16(r3)
-; CHECK-BE-NEXT:    stxv vs0, 0(r3)
-; CHECK-BE-NEXT:    stxv vs7, 112(r3)
-; CHECK-BE-NEXT:    stxv vs6, 96(r3)
+; CHECK-BE-NEXT:    xxmrghw v2, vs2, vs2
+; CHECK-BE-NEXT:    xvcvsxwdp vs3, v2
+; CHECK-BE-NEXT:    xxmrglw v2, vs2, vs2
+; CHECK-BE-NEXT:    stxv vs1, 0(r3)
+; CHECK-BE-NEXT:    stxv vs0, 16(r3)
+; CHECK-BE-NEXT:    xvcvsxwdp vs2, v2
+; CHECK-BE-NEXT:    xxmrghw v2, vs5, vs5
+; CHECK-BE-NEXT:    xvcvsxwdp vs6, v2
+; CHECK-BE-NEXT:    xxmrglw v2, vs5, vs5
+; CHECK-BE-NEXT:    stxv vs3, 32(r3)
+; CHECK-BE-NEXT:    stxv vs2, 48(r3)
+; CHECK-BE-NEXT:    xvcvsxwdp vs5, v2
+; CHECK-BE-NEXT:    xxmrghw v2, vs4, vs4
+; CHECK-BE-NEXT:    xvcvsxwdp vs7, v2
+; CHECK-BE-NEXT:    xxmrglw v2, vs4, vs4
+; CHECK-BE-NEXT:    stxv vs6, 64(r3)
 ; CHECK-BE-NEXT:    stxv vs5, 80(r3)
-; CHECK-BE-NEXT:    stxv vs4, 64(r3)
+; CHECK-BE-NEXT:    xvcvsxwdp vs4, v2
+; CHECK-BE-NEXT:    stxv vs7, 96(r3)
+; CHECK-BE-NEXT:    stxv vs4, 112(r3)
 ; CHECK-BE-NEXT:    blr
 entry:
   %a = load <16 x i32>, <16 x i32>* %0, align 64
diff --git a/test/CodeGen/PowerPC/vec_conv_i64_to_fp32_elts.ll b/test/CodeGen/PowerPC/vec_conv_i64_to_fp32_elts.ll
index cae556e27c630a9e7763fdbe5cb8445484adc81b..08e6f70bbe6d9529c7797a0fa04fd999c326c932 100644
--- a/test/CodeGen/PowerPC/vec_conv_i64_to_fp32_elts.ll
+++ b/test/CodeGen/PowerPC/vec_conv_i64_to_fp32_elts.ll
@@ -28,14 +28,14 @@ define i64 @test2elt(<2 x i64> %a) local_unnamed_addr #0 {
 ; CHECK-P9-LABEL: test2elt:
 ; CHECK-P9:       # %bb.0: # %entry
 ; CHECK-P9-NEXT:    xxswapd vs0, v2
-; CHECK-P9-NEXT:    xxlor vs1, v2, v2
-; CHECK-P9-NEXT:    xscvuxdsp f1, f1
 ; CHECK-P9-NEXT:    xscvuxdsp f0, f0
-; CHECK-P9-NEXT:    xscvdpspn vs1, f1
 ; CHECK-P9-NEXT:    xscvdpspn vs0, f0
-; CHECK-P9-NEXT:    xxsldwi v3, vs1, vs1, 1
+; CHECK-P9-NEXT:    xxsldwi v3, vs0, vs0, 1
+; CHECK-P9-NEXT:    xxlor vs0, v2, v2
+; CHECK-P9-NEXT:    xscvuxdsp f0, f0
+; CHECK-P9-NEXT:    xscvdpspn vs0, f0
 ; CHECK-P9-NEXT:    xxsldwi v2, vs0, vs0, 1
-; CHECK-P9-NEXT:    vmrglw v2, v3, v2
+; CHECK-P9-NEXT:    vmrglw v2, v2, v3
 ; CHECK-P9-NEXT:    mfvsrld r3, v2
 ; CHECK-P9-NEXT:    blr
 ;
@@ -62,51 +62,35 @@ define <4 x float> @test4elt(<4 x i64>* nocapture readonly) local_unnamed_addr #
 ; CHECK-P8-NEXT:    li r4, 16
 ; CHECK-P8-NEXT:    lxvd2x vs1, 0, r3
 ; CHECK-P8-NEXT:    lxvd2x vs0, r3, r4
-; CHECK-P8-NEXT:    xxswapd vs3, vs1
-; CHECK-P8-NEXT:    xscvuxdsp f1, f1
-; CHECK-P8-NEXT:    xxswapd vs2, vs0
-; CHECK-P8-NEXT:    xscvuxdsp f0, f0
-; CHECK-P8-NEXT:    xscvuxdsp f3, f3
-; CHECK-P8-NEXT:    xscvuxdsp f2, f2
-; CHECK-P8-NEXT:    xxmrghd vs0, vs0, vs1
-; CHECK-P8-NEXT:    xxmrghd vs1, vs2, vs3
-; CHECK-P8-NEXT:    xvcvdpsp v2, vs0
-; CHECK-P8-NEXT:    xvcvdpsp v3, vs1
-; CHECK-P8-NEXT:    vmrgew v2, v3, v2
+; CHECK-P8-NEXT:    xxswapd v3, vs1
+; CHECK-P8-NEXT:    xxswapd v2, vs0
+; CHECK-P8-NEXT:    xvcvuxdsp vs1, v3
+; CHECK-P8-NEXT:    xvcvuxdsp vs0, v2
+; CHECK-P8-NEXT:    xxsldwi v3, vs1, vs1, 3
+; CHECK-P8-NEXT:    xxsldwi v2, vs0, vs0, 3
+; CHECK-P8-NEXT:    vpkudum v2, v2, v3
 ; CHECK-P8-NEXT:    blr
 ;
 ; CHECK-P9-LABEL: test4elt:
 ; CHECK-P9:       # %bb.0: # %entry
-; CHECK-P9-NEXT:    lxv vs0, 16(r3)
-; CHECK-P9-NEXT:    lxv vs1, 0(r3)
-; CHECK-P9-NEXT:    xxswapd vs2, vs1
-; CHECK-P9-NEXT:    xxswapd vs3, vs0
-; CHECK-P9-NEXT:    xscvuxdsp f1, f1
-; CHECK-P9-NEXT:    xscvuxdsp f0, f0
-; CHECK-P9-NEXT:    xscvuxdsp f2, f2
-; CHECK-P9-NEXT:    xscvuxdsp f3, f3
-; CHECK-P9-NEXT:    xxmrghd vs0, vs0, vs1
-; CHECK-P9-NEXT:    xxmrghd vs2, vs3, vs2
-; CHECK-P9-NEXT:    xvcvdpsp v3, vs0
-; CHECK-P9-NEXT:    xvcvdpsp v2, vs2
-; CHECK-P9-NEXT:    vmrgew v2, v3, v2
+; CHECK-P9-NEXT:    lxv v3, 0(r3)
+; CHECK-P9-NEXT:    xvcvuxdsp vs0, v3
+; CHECK-P9-NEXT:    lxv v2, 16(r3)
+; CHECK-P9-NEXT:    xxsldwi v3, vs0, vs0, 3
+; CHECK-P9-NEXT:    xvcvuxdsp vs0, v2
+; CHECK-P9-NEXT:    xxsldwi v2, vs0, vs0, 3
+; CHECK-P9-NEXT:    vpkudum v2, v2, v3
 ; CHECK-P9-NEXT:    blr
 ;
 ; CHECK-BE-LABEL: test4elt:
 ; CHECK-BE:       # %bb.0: # %entry
-; CHECK-BE-NEXT:    lxv vs0, 0(r3)
-; CHECK-BE-NEXT:    lxv vs1, 16(r3)
-; CHECK-BE-NEXT:    xxswapd vs2, vs1
-; CHECK-BE-NEXT:    xxswapd vs3, vs0
-; CHECK-BE-NEXT:    xscvuxdsp f1, f1
-; CHECK-BE-NEXT:    xscvuxdsp f0, f0
-; CHECK-BE-NEXT:    xscvuxdsp f2, f2
-; CHECK-BE-NEXT:    xscvuxdsp f3, f3
-; CHECK-BE-NEXT:    xxmrghd vs0, vs0, vs1
-; CHECK-BE-NEXT:    xxmrghd vs1, vs3, vs2
-; CHECK-BE-NEXT:    xvcvdpsp v2, vs0
-; CHECK-BE-NEXT:    xvcvdpsp v3, vs1
-; CHECK-BE-NEXT:    vmrgew v2, v2, v3
+; CHECK-BE-NEXT:    lxv v3, 16(r3)
+; CHECK-BE-NEXT:    xvcvuxdsp vs0, v3
+; CHECK-BE-NEXT:    lxv v2, 0(r3)
+; CHECK-BE-NEXT:    xxsldwi v3, vs0, vs0, 3
+; CHECK-BE-NEXT:    xvcvuxdsp vs0, v2
+; CHECK-BE-NEXT:    xxsldwi v2, vs0, vs0, 3
+; CHECK-BE-NEXT:    vpkudum v2, v2, v3
 ; CHECK-BE-NEXT:    blr
 entry:
   %a = load <4 x i64>, <4 x i64>* %0, align 32
@@ -124,94 +108,62 @@ define void @test8elt(<8 x float>* noalias nocapture sret %agg.result, <8 x i64>
 ; CHECK-P8-NEXT:    li r5, 16
 ; CHECK-P8-NEXT:    lxvd2x vs1, r4, r6
 ; CHECK-P8-NEXT:    lxvd2x vs2, r4, r5
-; CHECK-P8-NEXT:    xxswapd vs7, vs3
-; CHECK-P8-NEXT:    xscvuxdsp f3, f3
-; CHECK-P8-NEXT:    xxswapd vs4, vs0
-; CHECK-P8-NEXT:    xscvuxdsp f0, f0
-; CHECK-P8-NEXT:    xxswapd vs5, vs1
-; CHECK-P8-NEXT:    xscvuxdsp f1, f1
-; CHECK-P8-NEXT:    xxswapd vs6, vs2
-; CHECK-P8-NEXT:    xscvuxdsp f2, f2
-; CHECK-P8-NEXT:    xscvuxdsp f4, f4
-; CHECK-P8-NEXT:    xscvuxdsp f5, f5
-; CHECK-P8-NEXT:    xscvuxdsp f6, f6
-; CHECK-P8-NEXT:    xscvuxdsp f7, f7
-; CHECK-P8-NEXT:    xxmrghd vs0, vs1, vs0
-; CHECK-P8-NEXT:    xxmrghd vs1, vs2, vs3
-; CHECK-P8-NEXT:    xxmrghd vs2, vs5, vs4
-; CHECK-P8-NEXT:    xvcvdpsp v2, vs0
-; CHECK-P8-NEXT:    xvcvdpsp v3, vs1
-; CHECK-P8-NEXT:    xxmrghd vs0, vs6, vs7
-; CHECK-P8-NEXT:    xvcvdpsp v4, vs2
-; CHECK-P8-NEXT:    xvcvdpsp v5, vs0
-; CHECK-P8-NEXT:    vmrgew v2, v4, v2
-; CHECK-P8-NEXT:    vmrgew v3, v5, v3
+; CHECK-P8-NEXT:    xxswapd v5, vs3
+; CHECK-P8-NEXT:    xxswapd v2, vs0
+; CHECK-P8-NEXT:    xxswapd v3, vs1
+; CHECK-P8-NEXT:    xxswapd v4, vs2
+; CHECK-P8-NEXT:    xvcvuxdsp vs3, v5
+; CHECK-P8-NEXT:    xvcvuxdsp vs0, v2
+; CHECK-P8-NEXT:    xvcvuxdsp vs1, v3
+; CHECK-P8-NEXT:    xvcvuxdsp vs2, v4
+; CHECK-P8-NEXT:    xxsldwi v5, vs3, vs3, 3
+; CHECK-P8-NEXT:    xxsldwi v2, vs0, vs0, 3
+; CHECK-P8-NEXT:    xxsldwi v3, vs1, vs1, 3
+; CHECK-P8-NEXT:    xxsldwi v4, vs2, vs2, 3
+; CHECK-P8-NEXT:    vpkudum v2, v3, v2
+; CHECK-P8-NEXT:    vpkudum v3, v4, v5
 ; CHECK-P8-NEXT:    stvx v2, r3, r5
 ; CHECK-P8-NEXT:    stvx v3, 0, r3
 ; CHECK-P8-NEXT:    blr
 ;
 ; CHECK-P9-LABEL: test8elt:
 ; CHECK-P9:       # %bb.0: # %entry
-; CHECK-P9-NEXT:    lxv vs0, 48(r4)
-; CHECK-P9-NEXT:    lxv vs1, 32(r4)
-; CHECK-P9-NEXT:    lxv vs2, 16(r4)
-; CHECK-P9-NEXT:    lxv vs3, 0(r4)
-; CHECK-P9-NEXT:    xxswapd vs4, vs3
-; CHECK-P9-NEXT:    xxswapd vs5, vs2
-; CHECK-P9-NEXT:    xxswapd vs6, vs1
-; CHECK-P9-NEXT:    xxswapd vs7, vs0
-; CHECK-P9-NEXT:    xscvuxdsp f3, f3
-; CHECK-P9-NEXT:    xscvuxdsp f2, f2
-; CHECK-P9-NEXT:    xscvuxdsp f1, f1
-; CHECK-P9-NEXT:    xscvuxdsp f0, f0
-; CHECK-P9-NEXT:    xscvuxdsp f4, f4
-; CHECK-P9-NEXT:    xscvuxdsp f5, f5
-; CHECK-P9-NEXT:    xscvuxdsp f6, f6
-; CHECK-P9-NEXT:    xscvuxdsp f7, f7
-; CHECK-P9-NEXT:    xxmrghd vs2, vs2, vs3
-; CHECK-P9-NEXT:    xxmrghd vs0, vs0, vs1
-; CHECK-P9-NEXT:    xxmrghd vs4, vs5, vs4
-; CHECK-P9-NEXT:    xxmrghd vs3, vs7, vs6
-; CHECK-P9-NEXT:    xvcvdpsp v3, vs2
-; CHECK-P9-NEXT:    xvcvdpsp v5, vs0
-; CHECK-P9-NEXT:    xvcvdpsp v2, vs4
-; CHECK-P9-NEXT:    xvcvdpsp v4, vs3
-; CHECK-P9-NEXT:    vmrgew v2, v3, v2
-; CHECK-P9-NEXT:    vmrgew v3, v5, v4
-; CHECK-P9-NEXT:    stxv v3, 16(r3)
-; CHECK-P9-NEXT:    stxv v2, 0(r3)
+; CHECK-P9-NEXT:    lxv v5, 0(r4)
+; CHECK-P9-NEXT:    xvcvuxdsp vs0, v5
+; CHECK-P9-NEXT:    lxv v4, 16(r4)
+; CHECK-P9-NEXT:    xxsldwi v5, vs0, vs0, 3
+; CHECK-P9-NEXT:    xvcvuxdsp vs0, v4
+; CHECK-P9-NEXT:    lxv v3, 32(r4)
+; CHECK-P9-NEXT:    xxsldwi v4, vs0, vs0, 3
+; CHECK-P9-NEXT:    xvcvuxdsp vs0, v3
+; CHECK-P9-NEXT:    lxv v2, 48(r4)
+; CHECK-P9-NEXT:    vpkudum v3, v4, v5
+; CHECK-P9-NEXT:    stxv v3, 0(r3)
+; CHECK-P9-NEXT:    xxsldwi v4, vs0, vs0, 3
+; CHECK-P9-NEXT:    xvcvuxdsp vs0, v2
+; CHECK-P9-NEXT:    xxsldwi v2, vs0, vs0, 3
+; CHECK-P9-NEXT:    vpkudum v2, v2, v4
+; CHECK-P9-NEXT:    stxv v2, 16(r3)
 ; CHECK-P9-NEXT:    blr
 ;
 ; CHECK-BE-LABEL: test8elt:
 ; CHECK-BE:       # %bb.0: # %entry
-; CHECK-BE-NEXT:    lxv vs0, 32(r4)
-; CHECK-BE-NEXT:    lxv vs1, 48(r4)
-; CHECK-BE-NEXT:    lxv vs2, 0(r4)
-; CHECK-BE-NEXT:    lxv vs3, 16(r4)
-; CHECK-BE-NEXT:    xxswapd vs4, vs3
-; CHECK-BE-NEXT:    xxswapd vs5, vs2
-; CHECK-BE-NEXT:    xxswapd vs6, vs1
-; CHECK-BE-NEXT:    xxswapd vs7, vs0
-; CHECK-BE-NEXT:    xscvuxdsp f3, f3
-; CHECK-BE-NEXT:    xscvuxdsp f2, f2
-; CHECK-BE-NEXT:    xscvuxdsp f1, f1
-; CHECK-BE-NEXT:    xscvuxdsp f0, f0
-; CHECK-BE-NEXT:    xscvuxdsp f4, f4
-; CHECK-BE-NEXT:    xscvuxdsp f5, f5
-; CHECK-BE-NEXT:    xscvuxdsp f6, f6
-; CHECK-BE-NEXT:    xscvuxdsp f7, f7
-; CHECK-BE-NEXT:    xxmrghd vs2, vs2, vs3
-; CHECK-BE-NEXT:    xxmrghd vs0, vs0, vs1
-; CHECK-BE-NEXT:    xxmrghd vs3, vs5, vs4
-; CHECK-BE-NEXT:    xxmrghd vs1, vs7, vs6
-; CHECK-BE-NEXT:    xvcvdpsp v2, vs2
-; CHECK-BE-NEXT:    xvcvdpsp v4, vs0
-; CHECK-BE-NEXT:    xvcvdpsp v3, vs3
-; CHECK-BE-NEXT:    xvcvdpsp v5, vs1
-; CHECK-BE-NEXT:    vmrgew v2, v2, v3
-; CHECK-BE-NEXT:    vmrgew v3, v4, v5
-; CHECK-BE-NEXT:    stxv v3, 16(r3)
-; CHECK-BE-NEXT:    stxv v2, 0(r3)
+; CHECK-BE-NEXT:    lxv v5, 16(r4)
+; CHECK-BE-NEXT:    xvcvuxdsp vs0, v5
+; CHECK-BE-NEXT:    lxv v4, 0(r4)
+; CHECK-BE-NEXT:    xxsldwi v5, vs0, vs0, 3
+; CHECK-BE-NEXT:    xvcvuxdsp vs0, v4
+; CHECK-BE-NEXT:    lxv v3, 48(r4)
+; CHECK-BE-NEXT:    xxsldwi v4, vs0, vs0, 3
+; CHECK-BE-NEXT:    xvcvuxdsp vs0, v3
+; CHECK-BE-NEXT:    lxv v2, 32(r4)
+; CHECK-BE-NEXT:    vpkudum v3, v4, v5
+; CHECK-BE-NEXT:    stxv v3, 0(r3)
+; CHECK-BE-NEXT:    xxsldwi v4, vs0, vs0, 3
+; CHECK-BE-NEXT:    xvcvuxdsp vs0, v2
+; CHECK-BE-NEXT:    xxsldwi v2, vs0, vs0, 3
+; CHECK-BE-NEXT:    vpkudum v2, v2, v4
+; CHECK-BE-NEXT:    stxv v2, 16(r3)
 ; CHECK-BE-NEXT:    blr
 entry:
   %a = load <8 x i64>, <8 x i64>* %0, align 64
@@ -223,69 +175,49 @@ entry:
 define void @test16elt(<16 x float>* noalias nocapture sret %agg.result, <16 x i64>* nocapture readonly) local_unnamed_addr #2 {
 ; CHECK-P8-LABEL: test16elt:
 ; CHECK-P8:       # %bb.0: # %entry
-; CHECK-P8-NEXT:    li r7, 64
 ; CHECK-P8-NEXT:    li r5, 32
 ; CHECK-P8-NEXT:    li r6, 48
-; CHECK-P8-NEXT:    lxvd2x vs11, 0, r4
-; CHECK-P8-NEXT:    stfd f30, -16(r1) # 8-byte Folded Spill
-; CHECK-P8-NEXT:    lxvd2x vs8, r4, r7
-; CHECK-P8-NEXT:    li r7, 80
-; CHECK-P8-NEXT:    lxvd2x vs6, r4, r5
-; CHECK-P8-NEXT:    stfd f31, -8(r1) # 8-byte Folded Spill
-; CHECK-P8-NEXT:    lxvd2x vs7, r4, r6
+; CHECK-P8-NEXT:    li r7, 64
+; CHECK-P8-NEXT:    lxvd2x vs4, 0, r4
+; CHECK-P8-NEXT:    lxvd2x vs0, r4, r5
+; CHECK-P8-NEXT:    lxvd2x vs1, r4, r6
 ; CHECK-P8-NEXT:    lxvd2x vs2, r4, r7
-; CHECK-P8-NEXT:    li r7, 96
+; CHECK-P8-NEXT:    li r7, 80
 ; CHECK-P8-NEXT:    lxvd2x vs3, r4, r7
+; CHECK-P8-NEXT:    li r7, 96
+; CHECK-P8-NEXT:    xxswapd v2, vs0
+; CHECK-P8-NEXT:    lxvd2x vs0, r4, r7
 ; CHECK-P8-NEXT:    li r7, 112
-; CHECK-P8-NEXT:    xscvuxdsp f30, f11
-; CHECK-P8-NEXT:    xxswapd vs11, vs11
-; CHECK-P8-NEXT:    lxvd2x vs4, r4, r7
+; CHECK-P8-NEXT:    xxswapd v3, vs1
+; CHECK-P8-NEXT:    lxvd2x vs1, r4, r7
 ; CHECK-P8-NEXT:    li r7, 16
-; CHECK-P8-NEXT:    xscvuxdsp f0, f6
-; CHECK-P8-NEXT:    xxswapd vs6, vs6
-; CHECK-P8-NEXT:    xscvuxdsp f1, f7
-; CHECK-P8-NEXT:    lxvd2x vs9, r4, r7
-; CHECK-P8-NEXT:    xxswapd vs7, vs7
-; CHECK-P8-NEXT:    xscvuxdsp f5, f8
-; CHECK-P8-NEXT:    xxswapd vs8, vs8
-; CHECK-P8-NEXT:    xscvuxdsp f10, f2
-; CHECK-P8-NEXT:    xxswapd vs2, vs2
-; CHECK-P8-NEXT:    xscvuxdsp f12, f3
-; CHECK-P8-NEXT:    xxswapd vs3, vs3
-; CHECK-P8-NEXT:    xscvuxdsp f13, f4
-; CHECK-P8-NEXT:    xxswapd vs4, vs4
-; CHECK-P8-NEXT:    xscvuxdsp f31, f9
-; CHECK-P8-NEXT:    xxswapd vs9, vs9
-; CHECK-P8-NEXT:    xscvuxdsp f6, f6
-; CHECK-P8-NEXT:    xxmrghd vs0, vs1, vs0
-; CHECK-P8-NEXT:    xscvuxdsp f7, f7
-; CHECK-P8-NEXT:    xscvuxdsp f8, f8
-; CHECK-P8-NEXT:    xxmrghd vs5, vs10, vs5
-; CHECK-P8-NEXT:    xscvuxdsp f2, f2
-; CHECK-P8-NEXT:    xscvuxdsp f3, f3
-; CHECK-P8-NEXT:    xxmrghd vs10, vs13, vs12
-; CHECK-P8-NEXT:    xscvuxdsp f4, f4
-; CHECK-P8-NEXT:    xscvuxdsp f1, f9
-; CHECK-P8-NEXT:    xscvuxdsp f9, f11
-; CHECK-P8-NEXT:    xxmrghd vs11, vs31, vs30
-; CHECK-P8-NEXT:    lfd f31, -8(r1) # 8-byte Folded Reload
-; CHECK-P8-NEXT:    lfd f30, -16(r1) # 8-byte Folded Reload
-; CHECK-P8-NEXT:    xvcvdpsp v2, vs0
-; CHECK-P8-NEXT:    xxmrghd vs0, vs7, vs6
-; CHECK-P8-NEXT:    xxmrghd vs2, vs2, vs8
-; CHECK-P8-NEXT:    xvcvdpsp v3, vs5
-; CHECK-P8-NEXT:    xvcvdpsp v4, vs10
-; CHECK-P8-NEXT:    xxmrghd vs3, vs4, vs3
-; CHECK-P8-NEXT:    xvcvdpsp v5, vs11
-; CHECK-P8-NEXT:    xvcvdpsp v0, vs0
-; CHECK-P8-NEXT:    xxmrghd vs1, vs1, vs9
-; CHECK-P8-NEXT:    xvcvdpsp v1, vs2
-; CHECK-P8-NEXT:    xvcvdpsp v6, vs3
-; CHECK-P8-NEXT:    xvcvdpsp v7, vs1
-; CHECK-P8-NEXT:    vmrgew v2, v0, v2
-; CHECK-P8-NEXT:    vmrgew v3, v1, v3
-; CHECK-P8-NEXT:    vmrgew v4, v6, v4
-; CHECK-P8-NEXT:    vmrgew v5, v7, v5
+; CHECK-P8-NEXT:    xxswapd v4, vs2
+; CHECK-P8-NEXT:    lxvd2x vs2, r4, r7
+; CHECK-P8-NEXT:    xxswapd v5, vs3
+; CHECK-P8-NEXT:    xvcvuxdsp vs3, v2
+; CHECK-P8-NEXT:    xxswapd v2, vs0
+; CHECK-P8-NEXT:    xvcvuxdsp vs0, v3
+; CHECK-P8-NEXT:    xxswapd v3, vs1
+; CHECK-P8-NEXT:    xvcvuxdsp vs1, v4
+; CHECK-P8-NEXT:    xxswapd v4, vs2
+; CHECK-P8-NEXT:    xvcvuxdsp vs2, v5
+; CHECK-P8-NEXT:    xxswapd v5, vs4
+; CHECK-P8-NEXT:    xvcvuxdsp vs4, v2
+; CHECK-P8-NEXT:    xvcvuxdsp vs5, v3
+; CHECK-P8-NEXT:    xvcvuxdsp vs6, v4
+; CHECK-P8-NEXT:    xxsldwi v2, vs3, vs3, 3
+; CHECK-P8-NEXT:    xvcvuxdsp vs7, v5
+; CHECK-P8-NEXT:    xxsldwi v3, vs0, vs0, 3
+; CHECK-P8-NEXT:    xxsldwi v4, vs1, vs1, 3
+; CHECK-P8-NEXT:    xxsldwi v5, vs2, vs2, 3
+; CHECK-P8-NEXT:    xxsldwi v0, vs4, vs4, 3
+; CHECK-P8-NEXT:    vpkudum v2, v3, v2
+; CHECK-P8-NEXT:    xxsldwi v1, vs5, vs5, 3
+; CHECK-P8-NEXT:    xxsldwi v6, vs6, vs6, 3
+; CHECK-P8-NEXT:    vpkudum v3, v5, v4
+; CHECK-P8-NEXT:    xxsldwi v7, vs7, vs7, 3
+; CHECK-P8-NEXT:    vpkudum v4, v1, v0
+; CHECK-P8-NEXT:    vpkudum v5, v6, v7
 ; CHECK-P8-NEXT:    stvx v2, r3, r7
 ; CHECK-P8-NEXT:    stvx v3, r3, r5
 ; CHECK-P8-NEXT:    stvx v4, r3, r6
@@ -294,130 +226,74 @@ define void @test16elt(<16 x float>* noalias nocapture sret %agg.result, <16 x i
 ;
 ; CHECK-P9-LABEL: test16elt:
 ; CHECK-P9:       # %bb.0: # %entry
-; CHECK-P9-NEXT:    lxv vs4, 48(r4)
-; CHECK-P9-NEXT:    lxv vs5, 32(r4)
-; CHECK-P9-NEXT:    lxv vs6, 16(r4)
-; CHECK-P9-NEXT:    lxv vs7, 0(r4)
-; CHECK-P9-NEXT:    lxv vs8, 112(r4)
-; CHECK-P9-NEXT:    lxv vs9, 96(r4)
-; CHECK-P9-NEXT:    lxv vs10, 80(r4)
-; CHECK-P9-NEXT:    lxv vs11, 64(r4)
-; CHECK-P9-NEXT:    stfd f30, -16(r1) # 8-byte Folded Spill
-; CHECK-P9-NEXT:    stfd f31, -8(r1) # 8-byte Folded Spill
-; CHECK-P9-NEXT:    xxswapd vs0, vs7
-; CHECK-P9-NEXT:    xxswapd vs1, vs6
-; CHECK-P9-NEXT:    xxswapd vs2, vs5
-; CHECK-P9-NEXT:    xxswapd vs3, vs4
-; CHECK-P9-NEXT:    xxswapd vs12, vs11
-; CHECK-P9-NEXT:    xxswapd vs13, vs10
-; CHECK-P9-NEXT:    xxswapd vs31, vs9
-; CHECK-P9-NEXT:    xxswapd vs30, vs8
-; CHECK-P9-NEXT:    xscvuxdsp f7, f7
-; CHECK-P9-NEXT:    xscvuxdsp f6, f6
-; CHECK-P9-NEXT:    xscvuxdsp f5, f5
-; CHECK-P9-NEXT:    xscvuxdsp f4, f4
-; CHECK-P9-NEXT:    xscvuxdsp f11, f11
-; CHECK-P9-NEXT:    xscvuxdsp f10, f10
-; CHECK-P9-NEXT:    xscvuxdsp f9, f9
-; CHECK-P9-NEXT:    xscvuxdsp f8, f8
-; CHECK-P9-NEXT:    xscvuxdsp f0, f0
-; CHECK-P9-NEXT:    xscvuxdsp f1, f1
-; CHECK-P9-NEXT:    xscvuxdsp f2, f2
-; CHECK-P9-NEXT:    xscvuxdsp f3, f3
-; CHECK-P9-NEXT:    xscvuxdsp f12, f12
-; CHECK-P9-NEXT:    xscvuxdsp f13, f13
-; CHECK-P9-NEXT:    xscvuxdsp f31, f31
-; CHECK-P9-NEXT:    xscvuxdsp f30, f30
-; CHECK-P9-NEXT:    xxmrghd vs6, vs6, vs7
-; CHECK-P9-NEXT:    xxmrghd vs4, vs4, vs5
-; CHECK-P9-NEXT:    xxmrghd vs0, vs1, vs0
-; CHECK-P9-NEXT:    xxmrghd vs1, vs3, vs2
-; CHECK-P9-NEXT:    xxmrghd vs2, vs10, vs11
-; CHECK-P9-NEXT:    xxmrghd vs3, vs8, vs9
-; CHECK-P9-NEXT:    xxmrghd vs5, vs13, vs12
-; CHECK-P9-NEXT:    xxmrghd vs7, vs30, vs31
-; CHECK-P9-NEXT:    lfd f31, -8(r1) # 8-byte Folded Reload
-; CHECK-P9-NEXT:    lfd f30, -16(r1) # 8-byte Folded Reload
-; CHECK-P9-NEXT:    xvcvdpsp v2, vs6
-; CHECK-P9-NEXT:    xvcvdpsp v3, vs4
-; CHECK-P9-NEXT:    xvcvdpsp v4, vs0
-; CHECK-P9-NEXT:    xvcvdpsp v5, vs1
-; CHECK-P9-NEXT:    xvcvdpsp v0, vs5
-; CHECK-P9-NEXT:    xvcvdpsp v1, vs2
-; CHECK-P9-NEXT:    xvcvdpsp v6, vs7
-; CHECK-P9-NEXT:    xvcvdpsp v7, vs3
-; CHECK-P9-NEXT:    vmrgew v2, v2, v4
-; CHECK-P9-NEXT:    vmrgew v3, v3, v5
-; CHECK-P9-NEXT:    vmrgew v4, v1, v0
-; CHECK-P9-NEXT:    vmrgew v5, v7, v6
+; CHECK-P9-NEXT:    lxv v7, 0(r4)
+; CHECK-P9-NEXT:    xvcvuxdsp vs0, v7
+; CHECK-P9-NEXT:    lxv v6, 16(r4)
+; CHECK-P9-NEXT:    xxsldwi v7, vs0, vs0, 3
+; CHECK-P9-NEXT:    xvcvuxdsp vs0, v6
+; CHECK-P9-NEXT:    lxv v1, 32(r4)
+; CHECK-P9-NEXT:    xxsldwi v6, vs0, vs0, 3
+; CHECK-P9-NEXT:    xvcvuxdsp vs0, v1
+; CHECK-P9-NEXT:    lxv v0, 48(r4)
+; CHECK-P9-NEXT:    vpkudum v1, v6, v7
+; CHECK-P9-NEXT:    xxsldwi v6, vs0, vs0, 3
+; CHECK-P9-NEXT:    xvcvuxdsp vs0, v0
+; CHECK-P9-NEXT:    lxv v5, 64(r4)
+; CHECK-P9-NEXT:    xxsldwi v0, vs0, vs0, 3
+; CHECK-P9-NEXT:    xvcvuxdsp vs0, v5
+; CHECK-P9-NEXT:    lxv v4, 80(r4)
+; CHECK-P9-NEXT:    vpkudum v0, v0, v6
+; CHECK-P9-NEXT:    xxsldwi v5, vs0, vs0, 3
+; CHECK-P9-NEXT:    lxv v3, 96(r4)
+; CHECK-P9-NEXT:    xvcvuxdsp vs0, v4
+; CHECK-P9-NEXT:    xxsldwi v4, vs0, vs0, 3
+; CHECK-P9-NEXT:    xvcvuxdsp vs0, v3
+; CHECK-P9-NEXT:    lxv v2, 112(r4)
+; CHECK-P9-NEXT:    stxv v0, 16(r3)
+; CHECK-P9-NEXT:    stxv v1, 0(r3)
+; CHECK-P9-NEXT:    vpkudum v4, v4, v5
 ; CHECK-P9-NEXT:    stxv v4, 32(r3)
-; CHECK-P9-NEXT:    stxv v3, 16(r3)
-; CHECK-P9-NEXT:    stxv v2, 0(r3)
-; CHECK-P9-NEXT:    stxv v5, 48(r3)
+; CHECK-P9-NEXT:    xxsldwi v3, vs0, vs0, 3
+; CHECK-P9-NEXT:    xvcvuxdsp vs0, v2
+; CHECK-P9-NEXT:    xxsldwi v2, vs0, vs0, 3
+; CHECK-P9-NEXT:    vpkudum v2, v2, v3
+; CHECK-P9-NEXT:    stxv v2, 48(r3)
 ; CHECK-P9-NEXT:    blr
 ;
 ; CHECK-BE-LABEL: test16elt:
 ; CHECK-BE:       # %bb.0: # %entry
-; CHECK-BE-NEXT:    lxv vs2, 32(r4)
-; CHECK-BE-NEXT:    lxv vs3, 48(r4)
-; CHECK-BE-NEXT:    lxv vs4, 0(r4)
-; CHECK-BE-NEXT:    lxv vs5, 16(r4)
-; CHECK-BE-NEXT:    lxv vs6, 96(r4)
-; CHECK-BE-NEXT:    lxv vs7, 112(r4)
-; CHECK-BE-NEXT:    lxv vs8, 64(r4)
-; CHECK-BE-NEXT:    lxv vs9, 80(r4)
-; CHECK-BE-NEXT:    stfd f30, -16(r1) # 8-byte Folded Spill
-; CHECK-BE-NEXT:    stfd f31, -8(r1) # 8-byte Folded Spill
-; CHECK-BE-NEXT:    xxswapd vs0, vs5
-; CHECK-BE-NEXT:    xxswapd vs1, vs4
-; CHECK-BE-NEXT:    xxswapd vs10, vs3
-; CHECK-BE-NEXT:    xxswapd vs11, vs2
-; CHECK-BE-NEXT:    xxswapd vs12, vs9
-; CHECK-BE-NEXT:    xxswapd vs13, vs8
-; CHECK-BE-NEXT:    xxswapd vs31, vs7
-; CHECK-BE-NEXT:    xxswapd vs30, vs6
-; CHECK-BE-NEXT:    xscvuxdsp f5, f5
-; CHECK-BE-NEXT:    xscvuxdsp f4, f4
-; CHECK-BE-NEXT:    xscvuxdsp f3, f3
-; CHECK-BE-NEXT:    xscvuxdsp f2, f2
-; CHECK-BE-NEXT:    xscvuxdsp f9, f9
-; CHECK-BE-NEXT:    xscvuxdsp f8, f8
-; CHECK-BE-NEXT:    xscvuxdsp f7, f7
-; CHECK-BE-NEXT:    xscvuxdsp f6, f6
-; CHECK-BE-NEXT:    xscvuxdsp f0, f0
-; CHECK-BE-NEXT:    xscvuxdsp f1, f1
-; CHECK-BE-NEXT:    xscvuxdsp f10, f10
-; CHECK-BE-NEXT:    xscvuxdsp f11, f11
-; CHECK-BE-NEXT:    xscvuxdsp f12, f12
-; CHECK-BE-NEXT:    xscvuxdsp f13, f13
-; CHECK-BE-NEXT:    xscvuxdsp f31, f31
-; CHECK-BE-NEXT:    xscvuxdsp f30, f30
-; CHECK-BE-NEXT:    xxmrghd vs4, vs4, vs5
-; CHECK-BE-NEXT:    xxmrghd vs2, vs2, vs3
-; CHECK-BE-NEXT:    xxmrghd vs3, vs8, vs9
-; CHECK-BE-NEXT:    xxmrghd vs5, vs6, vs7
-; CHECK-BE-NEXT:    xxmrghd vs0, vs1, vs0
-; CHECK-BE-NEXT:    xxmrghd vs1, vs11, vs10
-; CHECK-BE-NEXT:    xxmrghd vs6, vs13, vs12
-; CHECK-BE-NEXT:    xxmrghd vs7, vs30, vs31
-; CHECK-BE-NEXT:    lfd f31, -8(r1) # 8-byte Folded Reload
-; CHECK-BE-NEXT:    lfd f30, -16(r1) # 8-byte Folded Reload
-; CHECK-BE-NEXT:    xvcvdpsp v2, vs4
-; CHECK-BE-NEXT:    xvcvdpsp v3, vs2
-; CHECK-BE-NEXT:    xvcvdpsp v0, vs3
-; CHECK-BE-NEXT:    xvcvdpsp v6, vs5
-; CHECK-BE-NEXT:    xvcvdpsp v4, vs0
-; CHECK-BE-NEXT:    xvcvdpsp v5, vs1
-; CHECK-BE-NEXT:    xvcvdpsp v1, vs6
-; CHECK-BE-NEXT:    xvcvdpsp v7, vs7
-; CHECK-BE-NEXT:    vmrgew v2, v2, v4
-; CHECK-BE-NEXT:    vmrgew v3, v3, v5
-; CHECK-BE-NEXT:    vmrgew v4, v0, v1
-; CHECK-BE-NEXT:    vmrgew v5, v6, v7
-; CHECK-BE-NEXT:    stxv v5, 48(r3)
+; CHECK-BE-NEXT:    lxv v7, 16(r4)
+; CHECK-BE-NEXT:    xvcvuxdsp vs0, v7
+; CHECK-BE-NEXT:    lxv v6, 0(r4)
+; CHECK-BE-NEXT:    xxsldwi v7, vs0, vs0, 3
+; CHECK-BE-NEXT:    xvcvuxdsp vs0, v6
+; CHECK-BE-NEXT:    lxv v1, 48(r4)
+; CHECK-BE-NEXT:    xxsldwi v6, vs0, vs0, 3
+; CHECK-BE-NEXT:    xvcvuxdsp vs0, v1
+; CHECK-BE-NEXT:    lxv v0, 32(r4)
+; CHECK-BE-NEXT:    vpkudum v1, v6, v7
+; CHECK-BE-NEXT:    xxsldwi v6, vs0, vs0, 3
+; CHECK-BE-NEXT:    xvcvuxdsp vs0, v0
+; CHECK-BE-NEXT:    lxv v5, 80(r4)
+; CHECK-BE-NEXT:    xxsldwi v0, vs0, vs0, 3
+; CHECK-BE-NEXT:    xvcvuxdsp vs0, v5
+; CHECK-BE-NEXT:    lxv v4, 64(r4)
+; CHECK-BE-NEXT:    vpkudum v0, v0, v6
+; CHECK-BE-NEXT:    xxsldwi v5, vs0, vs0, 3
+; CHECK-BE-NEXT:    lxv v3, 112(r4)
+; CHECK-BE-NEXT:    xvcvuxdsp vs0, v4
+; CHECK-BE-NEXT:    xxsldwi v4, vs0, vs0, 3
+; CHECK-BE-NEXT:    xvcvuxdsp vs0, v3
+; CHECK-BE-NEXT:    lxv v2, 96(r4)
+; CHECK-BE-NEXT:    stxv v0, 16(r3)
+; CHECK-BE-NEXT:    stxv v1, 0(r3)
+; CHECK-BE-NEXT:    vpkudum v4, v4, v5
 ; CHECK-BE-NEXT:    stxv v4, 32(r3)
-; CHECK-BE-NEXT:    stxv v3, 16(r3)
-; CHECK-BE-NEXT:    stxv v2, 0(r3)
+; CHECK-BE-NEXT:    xxsldwi v3, vs0, vs0, 3
+; CHECK-BE-NEXT:    xvcvuxdsp vs0, v2
+; CHECK-BE-NEXT:    xxsldwi v2, vs0, vs0, 3
+; CHECK-BE-NEXT:    vpkudum v2, v2, v3
+; CHECK-BE-NEXT:    stxv v2, 48(r3)
 ; CHECK-BE-NEXT:    blr
 entry:
   %a = load <16 x i64>, <16 x i64>* %0, align 128
@@ -445,14 +321,14 @@ define i64 @test2elt_signed(<2 x i64> %a) local_unnamed_addr #0 {
 ; CHECK-P9-LABEL: test2elt_signed:
 ; CHECK-P9:       # %bb.0: # %entry
 ; CHECK-P9-NEXT:    xxswapd vs0, v2
-; CHECK-P9-NEXT:    xxlor vs1, v2, v2
-; CHECK-P9-NEXT:    xscvsxdsp f1, f1
 ; CHECK-P9-NEXT:    xscvsxdsp f0, f0
-; CHECK-P9-NEXT:    xscvdpspn vs1, f1
 ; CHECK-P9-NEXT:    xscvdpspn vs0, f0
-; CHECK-P9-NEXT:    xxsldwi v3, vs1, vs1, 1
+; CHECK-P9-NEXT:    xxsldwi v3, vs0, vs0, 1
+; CHECK-P9-NEXT:    xxlor vs0, v2, v2
+; CHECK-P9-NEXT:    xscvsxdsp f0, f0
+; CHECK-P9-NEXT:    xscvdpspn vs0, f0
 ; CHECK-P9-NEXT:    xxsldwi v2, vs0, vs0, 1
-; CHECK-P9-NEXT:    vmrglw v2, v3, v2
+; CHECK-P9-NEXT:    vmrglw v2, v2, v3
 ; CHECK-P9-NEXT:    mfvsrld r3, v2
 ; CHECK-P9-NEXT:    blr
 ;
@@ -479,51 +355,35 @@ define <4 x float> @test4elt_signed(<4 x i64>* nocapture readonly) local_unnamed
 ; CHECK-P8-NEXT:    li r4, 16
 ; CHECK-P8-NEXT:    lxvd2x vs1, 0, r3
 ; CHECK-P8-NEXT:    lxvd2x vs0, r3, r4
-; CHECK-P8-NEXT:    xxswapd vs3, vs1
-; CHECK-P8-NEXT:    xscvsxdsp f1, f1
-; CHECK-P8-NEXT:    xxswapd vs2, vs0
-; CHECK-P8-NEXT:    xscvsxdsp f0, f0
-; CHECK-P8-NEXT:    xscvsxdsp f3, f3
-; CHECK-P8-NEXT:    xscvsxdsp f2, f2
-; CHECK-P8-NEXT:    xxmrghd vs0, vs0, vs1
-; CHECK-P8-NEXT:    xxmrghd vs1, vs2, vs3
-; CHECK-P8-NEXT:    xvcvdpsp v2, vs0
-; CHECK-P8-NEXT:    xvcvdpsp v3, vs1
-; CHECK-P8-NEXT:    vmrgew v2, v3, v2
+; CHECK-P8-NEXT:    xxswapd v3, vs1
+; CHECK-P8-NEXT:    xxswapd v2, vs0
+; CHECK-P8-NEXT:    xvcvsxdsp vs1, v3
+; CHECK-P8-NEXT:    xvcvsxdsp vs0, v2
+; CHECK-P8-NEXT:    xxsldwi v3, vs1, vs1, 3
+; CHECK-P8-NEXT:    xxsldwi v2, vs0, vs0, 3
+; CHECK-P8-NEXT:    vpkudum v2, v2, v3
 ; CHECK-P8-NEXT:    blr
 ;
 ; CHECK-P9-LABEL: test4elt_signed:
 ; CHECK-P9:       # %bb.0: # %entry
-; CHECK-P9-NEXT:    lxv vs0, 16(r3)
-; CHECK-P9-NEXT:    lxv vs1, 0(r3)
-; CHECK-P9-NEXT:    xxswapd vs2, vs1
-; CHECK-P9-NEXT:    xxswapd vs3, vs0
-; CHECK-P9-NEXT:    xscvsxdsp f1, f1
-; CHECK-P9-NEXT:    xscvsxdsp f0, f0
-; CHECK-P9-NEXT:    xscvsxdsp f2, f2
-; CHECK-P9-NEXT:    xscvsxdsp f3, f3
-; CHECK-P9-NEXT:    xxmrghd vs0, vs0, vs1
-; CHECK-P9-NEXT:    xxmrghd vs2, vs3, vs2
-; CHECK-P9-NEXT:    xvcvdpsp v3, vs0
-; CHECK-P9-NEXT:    xvcvdpsp v2, vs2
-; CHECK-P9-NEXT:    vmrgew v2, v3, v2
+; CHECK-P9-NEXT:    lxv v3, 0(r3)
+; CHECK-P9-NEXT:    xvcvsxdsp vs0, v3
+; CHECK-P9-NEXT:    lxv v2, 16(r3)
+; CHECK-P9-NEXT:    xxsldwi v3, vs0, vs0, 3
+; CHECK-P9-NEXT:    xvcvsxdsp vs0, v2
+; CHECK-P9-NEXT:    xxsldwi v2, vs0, vs0, 3
+; CHECK-P9-NEXT:    vpkudum v2, v2, v3
 ; CHECK-P9-NEXT:    blr
 ;
 ; CHECK-BE-LABEL: test4elt_signed:
 ; CHECK-BE:       # %bb.0: # %entry
-; CHECK-BE-NEXT:    lxv vs0, 0(r3)
-; CHECK-BE-NEXT:    lxv vs1, 16(r3)
-; CHECK-BE-NEXT:    xxswapd vs2, vs1
-; CHECK-BE-NEXT:    xxswapd vs3, vs0
-; CHECK-BE-NEXT:    xscvsxdsp f1, f1
-; CHECK-BE-NEXT:    xscvsxdsp f0, f0
-; CHECK-BE-NEXT:    xscvsxdsp f2, f2
-; CHECK-BE-NEXT:    xscvsxdsp f3, f3
-; CHECK-BE-NEXT:    xxmrghd vs0, vs0, vs1
-; CHECK-BE-NEXT:    xxmrghd vs1, vs3, vs2
-; CHECK-BE-NEXT:    xvcvdpsp v2, vs0
-; CHECK-BE-NEXT:    xvcvdpsp v3, vs1
-; CHECK-BE-NEXT:    vmrgew v2, v2, v3
+; CHECK-BE-NEXT:    lxv v3, 16(r3)
+; CHECK-BE-NEXT:    xvcvsxdsp vs0, v3
+; CHECK-BE-NEXT:    lxv v2, 0(r3)
+; CHECK-BE-NEXT:    xxsldwi v3, vs0, vs0, 3
+; CHECK-BE-NEXT:    xvcvsxdsp vs0, v2
+; CHECK-BE-NEXT:    xxsldwi v2, vs0, vs0, 3
+; CHECK-BE-NEXT:    vpkudum v2, v2, v3
 ; CHECK-BE-NEXT:    blr
 entry:
   %a = load <4 x i64>, <4 x i64>* %0, align 32
@@ -541,94 +401,62 @@ define void @test8elt_signed(<8 x float>* noalias nocapture sret %agg.result, <8
 ; CHECK-P8-NEXT:    li r5, 16
 ; CHECK-P8-NEXT:    lxvd2x vs1, r4, r6
 ; CHECK-P8-NEXT:    lxvd2x vs2, r4, r5
-; CHECK-P8-NEXT:    xxswapd vs7, vs3
-; CHECK-P8-NEXT:    xscvsxdsp f3, f3
-; CHECK-P8-NEXT:    xxswapd vs4, vs0
-; CHECK-P8-NEXT:    xscvsxdsp f0, f0
-; CHECK-P8-NEXT:    xxswapd vs5, vs1
-; CHECK-P8-NEXT:    xscvsxdsp f1, f1
-; CHECK-P8-NEXT:    xxswapd vs6, vs2
-; CHECK-P8-NEXT:    xscvsxdsp f2, f2
-; CHECK-P8-NEXT:    xscvsxdsp f4, f4
-; CHECK-P8-NEXT:    xscvsxdsp f5, f5
-; CHECK-P8-NEXT:    xscvsxdsp f6, f6
-; CHECK-P8-NEXT:    xscvsxdsp f7, f7
-; CHECK-P8-NEXT:    xxmrghd vs0, vs1, vs0
-; CHECK-P8-NEXT:    xxmrghd vs1, vs2, vs3
-; CHECK-P8-NEXT:    xxmrghd vs2, vs5, vs4
-; CHECK-P8-NEXT:    xvcvdpsp v2, vs0
-; CHECK-P8-NEXT:    xvcvdpsp v3, vs1
-; CHECK-P8-NEXT:    xxmrghd vs0, vs6, vs7
-; CHECK-P8-NEXT:    xvcvdpsp v4, vs2
-; CHECK-P8-NEXT:    xvcvdpsp v5, vs0
-; CHECK-P8-NEXT:    vmrgew v2, v4, v2
-; CHECK-P8-NEXT:    vmrgew v3, v5, v3
+; CHECK-P8-NEXT:    xxswapd v5, vs3
+; CHECK-P8-NEXT:    xxswapd v2, vs0
+; CHECK-P8-NEXT:    xxswapd v3, vs1
+; CHECK-P8-NEXT:    xxswapd v4, vs2
+; CHECK-P8-NEXT:    xvcvsxdsp vs3, v5
+; CHECK-P8-NEXT:    xvcvsxdsp vs0, v2
+; CHECK-P8-NEXT:    xvcvsxdsp vs1, v3
+; CHECK-P8-NEXT:    xvcvsxdsp vs2, v4
+; CHECK-P8-NEXT:    xxsldwi v5, vs3, vs3, 3
+; CHECK-P8-NEXT:    xxsldwi v2, vs0, vs0, 3
+; CHECK-P8-NEXT:    xxsldwi v3, vs1, vs1, 3
+; CHECK-P8-NEXT:    xxsldwi v4, vs2, vs2, 3
+; CHECK-P8-NEXT:    vpkudum v2, v3, v2
+; CHECK-P8-NEXT:    vpkudum v3, v4, v5
 ; CHECK-P8-NEXT:    stvx v2, r3, r5
 ; CHECK-P8-NEXT:    stvx v3, 0, r3
 ; CHECK-P8-NEXT:    blr
 ;
 ; CHECK-P9-LABEL: test8elt_signed:
 ; CHECK-P9:       # %bb.0: # %entry
-; CHECK-P9-NEXT:    lxv vs0, 48(r4)
-; CHECK-P9-NEXT:    lxv vs1, 32(r4)
-; CHECK-P9-NEXT:    lxv vs2, 16(r4)
-; CHECK-P9-NEXT:    lxv vs3, 0(r4)
-; CHECK-P9-NEXT:    xxswapd vs4, vs3
-; CHECK-P9-NEXT:    xxswapd vs5, vs2
-; CHECK-P9-NEXT:    xxswapd vs6, vs1
-; CHECK-P9-NEXT:    xxswapd vs7, vs0
-; CHECK-P9-NEXT:    xscvsxdsp f3, f3
-; CHECK-P9-NEXT:    xscvsxdsp f2, f2
-; CHECK-P9-NEXT:    xscvsxdsp f1, f1
-; CHECK-P9-NEXT:    xscvsxdsp f0, f0
-; CHECK-P9-NEXT:    xscvsxdsp f4, f4
-; CHECK-P9-NEXT:    xscvsxdsp f5, f5
-; CHECK-P9-NEXT:    xscvsxdsp f6, f6
-; CHECK-P9-NEXT:    xscvsxdsp f7, f7
-; CHECK-P9-NEXT:    xxmrghd vs2, vs2, vs3
-; CHECK-P9-NEXT:    xxmrghd vs0, vs0, vs1
-; CHECK-P9-NEXT:    xxmrghd vs4, vs5, vs4
-; CHECK-P9-NEXT:    xxmrghd vs3, vs7, vs6
-; CHECK-P9-NEXT:    xvcvdpsp v3, vs2
-; CHECK-P9-NEXT:    xvcvdpsp v5, vs0
-; CHECK-P9-NEXT:    xvcvdpsp v2, vs4
-; CHECK-P9-NEXT:    xvcvdpsp v4, vs3
-; CHECK-P9-NEXT:    vmrgew v2, v3, v2
-; CHECK-P9-NEXT:    vmrgew v3, v5, v4
-; CHECK-P9-NEXT:    stxv v3, 16(r3)
-; CHECK-P9-NEXT:    stxv v2, 0(r3)
+; CHECK-P9-NEXT:    lxv v5, 0(r4)
+; CHECK-P9-NEXT:    xvcvsxdsp vs0, v5
+; CHECK-P9-NEXT:    lxv v4, 16(r4)
+; CHECK-P9-NEXT:    xxsldwi v5, vs0, vs0, 3
+; CHECK-P9-NEXT:    xvcvsxdsp vs0, v4
+; CHECK-P9-NEXT:    lxv v3, 32(r4)
+; CHECK-P9-NEXT:    xxsldwi v4, vs0, vs0, 3
+; CHECK-P9-NEXT:    xvcvsxdsp vs0, v3
+; CHECK-P9-NEXT:    lxv v2, 48(r4)
+; CHECK-P9-NEXT:    vpkudum v3, v4, v5
+; CHECK-P9-NEXT:    stxv v3, 0(r3)
+; CHECK-P9-NEXT:    xxsldwi v4, vs0, vs0, 3
+; CHECK-P9-NEXT:    xvcvsxdsp vs0, v2
+; CHECK-P9-NEXT:    xxsldwi v2, vs0, vs0, 3
+; CHECK-P9-NEXT:    vpkudum v2, v2, v4
+; CHECK-P9-NEXT:    stxv v2, 16(r3)
 ; CHECK-P9-NEXT:    blr
 ;
 ; CHECK-BE-LABEL: test8elt_signed:
 ; CHECK-BE:       # %bb.0: # %entry
-; CHECK-BE-NEXT:    lxv vs0, 32(r4)
-; CHECK-BE-NEXT:    lxv vs1, 48(r4)
-; CHECK-BE-NEXT:    lxv vs2, 0(r4)
-; CHECK-BE-NEXT:    lxv vs3, 16(r4)
-; CHECK-BE-NEXT:    xxswapd vs4, vs3
-; CHECK-BE-NEXT:    xxswapd vs5, vs2
-; CHECK-BE-NEXT:    xxswapd vs6, vs1
-; CHECK-BE-NEXT:    xxswapd vs7, vs0
-; CHECK-BE-NEXT:    xscvsxdsp f3, f3
-; CHECK-BE-NEXT:    xscvsxdsp f2, f2
-; CHECK-BE-NEXT:    xscvsxdsp f1, f1
-; CHECK-BE-NEXT:    xscvsxdsp f0, f0
-; CHECK-BE-NEXT:    xscvsxdsp f4, f4
-; CHECK-BE-NEXT:    xscvsxdsp f5, f5
-; CHECK-BE-NEXT:    xscvsxdsp f6, f6
-; CHECK-BE-NEXT:    xscvsxdsp f7, f7
-; CHECK-BE-NEXT:    xxmrghd vs2, vs2, vs3
-; CHECK-BE-NEXT:    xxmrghd vs0, vs0, vs1
-; CHECK-BE-NEXT:    xxmrghd vs3, vs5, vs4
-; CHECK-BE-NEXT:    xxmrghd vs1, vs7, vs6
-; CHECK-BE-NEXT:    xvcvdpsp v2, vs2
-; CHECK-BE-NEXT:    xvcvdpsp v4, vs0
-; CHECK-BE-NEXT:    xvcvdpsp v3, vs3
-; CHECK-BE-NEXT:    xvcvdpsp v5, vs1
-; CHECK-BE-NEXT:    vmrgew v2, v2, v3
-; CHECK-BE-NEXT:    vmrgew v3, v4, v5
-; CHECK-BE-NEXT:    stxv v3, 16(r3)
-; CHECK-BE-NEXT:    stxv v2, 0(r3)
+; CHECK-BE-NEXT:    lxv v5, 16(r4)
+; CHECK-BE-NEXT:    xvcvsxdsp vs0, v5
+; CHECK-BE-NEXT:    lxv v4, 0(r4)
+; CHECK-BE-NEXT:    xxsldwi v5, vs0, vs0, 3
+; CHECK-BE-NEXT:    xvcvsxdsp vs0, v4
+; CHECK-BE-NEXT:    lxv v3, 48(r4)
+; CHECK-BE-NEXT:    xxsldwi v4, vs0, vs0, 3
+; CHECK-BE-NEXT:    xvcvsxdsp vs0, v3
+; CHECK-BE-NEXT:    lxv v2, 32(r4)
+; CHECK-BE-NEXT:    vpkudum v3, v4, v5
+; CHECK-BE-NEXT:    stxv v3, 0(r3)
+; CHECK-BE-NEXT:    xxsldwi v4, vs0, vs0, 3
+; CHECK-BE-NEXT:    xvcvsxdsp vs0, v2
+; CHECK-BE-NEXT:    xxsldwi v2, vs0, vs0, 3
+; CHECK-BE-NEXT:    vpkudum v2, v2, v4
+; CHECK-BE-NEXT:    stxv v2, 16(r3)
 ; CHECK-BE-NEXT:    blr
 entry:
   %a = load <8 x i64>, <8 x i64>* %0, align 64
@@ -640,69 +468,49 @@ entry:
 define void @test16elt_signed(<16 x float>* noalias nocapture sret %agg.result, <16 x i64>* nocapture readonly) local_unnamed_addr #2 {
 ; CHECK-P8-LABEL: test16elt_signed:
 ; CHECK-P8:       # %bb.0: # %entry
-; CHECK-P8-NEXT:    li r7, 64
 ; CHECK-P8-NEXT:    li r5, 32
 ; CHECK-P8-NEXT:    li r6, 48
-; CHECK-P8-NEXT:    lxvd2x vs11, 0, r4
-; CHECK-P8-NEXT:    stfd f30, -16(r1) # 8-byte Folded Spill
-; CHECK-P8-NEXT:    lxvd2x vs8, r4, r7
-; CHECK-P8-NEXT:    li r7, 80
-; CHECK-P8-NEXT:    lxvd2x vs6, r4, r5
-; CHECK-P8-NEXT:    stfd f31, -8(r1) # 8-byte Folded Spill
-; CHECK-P8-NEXT:    lxvd2x vs7, r4, r6
+; CHECK-P8-NEXT:    li r7, 64
+; CHECK-P8-NEXT:    lxvd2x vs4, 0, r4
+; CHECK-P8-NEXT:    lxvd2x vs0, r4, r5
+; CHECK-P8-NEXT:    lxvd2x vs1, r4, r6
 ; CHECK-P8-NEXT:    lxvd2x vs2, r4, r7
-; CHECK-P8-NEXT:    li r7, 96
+; CHECK-P8-NEXT:    li r7, 80
 ; CHECK-P8-NEXT:    lxvd2x vs3, r4, r7
+; CHECK-P8-NEXT:    li r7, 96
+; CHECK-P8-NEXT:    xxswapd v2, vs0
+; CHECK-P8-NEXT:    lxvd2x vs0, r4, r7
 ; CHECK-P8-NEXT:    li r7, 112
-; CHECK-P8-NEXT:    xscvsxdsp f30, f11
-; CHECK-P8-NEXT:    xxswapd vs11, vs11
-; CHECK-P8-NEXT:    lxvd2x vs4, r4, r7
+; CHECK-P8-NEXT:    xxswapd v3, vs1
+; CHECK-P8-NEXT:    lxvd2x vs1, r4, r7
 ; CHECK-P8-NEXT:    li r7, 16
-; CHECK-P8-NEXT:    xscvsxdsp f0, f6
-; CHECK-P8-NEXT:    xxswapd vs6, vs6
-; CHECK-P8-NEXT:    xscvsxdsp f1, f7
-; CHECK-P8-NEXT:    lxvd2x vs9, r4, r7
-; CHECK-P8-NEXT:    xxswapd vs7, vs7
-; CHECK-P8-NEXT:    xscvsxdsp f5, f8
-; CHECK-P8-NEXT:    xxswapd vs8, vs8
-; CHECK-P8-NEXT:    xscvsxdsp f10, f2
-; CHECK-P8-NEXT:    xxswapd vs2, vs2
-; CHECK-P8-NEXT:    xscvsxdsp f12, f3
-; CHECK-P8-NEXT:    xxswapd vs3, vs3
-; CHECK-P8-NEXT:    xscvsxdsp f13, f4
-; CHECK-P8-NEXT:    xxswapd vs4, vs4
-; CHECK-P8-NEXT:    xscvsxdsp f31, f9
-; CHECK-P8-NEXT:    xxswapd vs9, vs9
-; CHECK-P8-NEXT:    xscvsxdsp f6, f6
-; CHECK-P8-NEXT:    xxmrghd vs0, vs1, vs0
-; CHECK-P8-NEXT:    xscvsxdsp f7, f7
-; CHECK-P8-NEXT:    xscvsxdsp f8, f8
-; CHECK-P8-NEXT:    xxmrghd vs5, vs10, vs5
-; CHECK-P8-NEXT:    xscvsxdsp f2, f2
-; CHECK-P8-NEXT:    xscvsxdsp f3, f3
-; CHECK-P8-NEXT:    xxmrghd vs10, vs13, vs12
-; CHECK-P8-NEXT:    xscvsxdsp f4, f4
-; CHECK-P8-NEXT:    xscvsxdsp f1, f9
-; CHECK-P8-NEXT:    xscvsxdsp f9, f11
-; CHECK-P8-NEXT:    xxmrghd vs11, vs31, vs30
-; CHECK-P8-NEXT:    lfd f31, -8(r1) # 8-byte Folded Reload
-; CHECK-P8-NEXT:    lfd f30, -16(r1) # 8-byte Folded Reload
-; CHECK-P8-NEXT:    xvcvdpsp v2, vs0
-; CHECK-P8-NEXT:    xxmrghd vs0, vs7, vs6
-; CHECK-P8-NEXT:    xxmrghd vs2, vs2, vs8
-; CHECK-P8-NEXT:    xvcvdpsp v3, vs5
-; CHECK-P8-NEXT:    xvcvdpsp v4, vs10
-; CHECK-P8-NEXT:    xxmrghd vs3, vs4, vs3
-; CHECK-P8-NEXT:    xvcvdpsp v5, vs11
-; CHECK-P8-NEXT:    xvcvdpsp v0, vs0
-; CHECK-P8-NEXT:    xxmrghd vs1, vs1, vs9
-; CHECK-P8-NEXT:    xvcvdpsp v1, vs2
-; CHECK-P8-NEXT:    xvcvdpsp v6, vs3
-; CHECK-P8-NEXT:    xvcvdpsp v7, vs1
-; CHECK-P8-NEXT:    vmrgew v2, v0, v2
-; CHECK-P8-NEXT:    vmrgew v3, v1, v3
-; CHECK-P8-NEXT:    vmrgew v4, v6, v4
-; CHECK-P8-NEXT:    vmrgew v5, v7, v5
+; CHECK-P8-NEXT:    xxswapd v4, vs2
+; CHECK-P8-NEXT:    lxvd2x vs2, r4, r7
+; CHECK-P8-NEXT:    xxswapd v5, vs3
+; CHECK-P8-NEXT:    xvcvsxdsp vs3, v2
+; CHECK-P8-NEXT:    xxswapd v2, vs0
+; CHECK-P8-NEXT:    xvcvsxdsp vs0, v3
+; CHECK-P8-NEXT:    xxswapd v3, vs1
+; CHECK-P8-NEXT:    xvcvsxdsp vs1, v4
+; CHECK-P8-NEXT:    xxswapd v4, vs2
+; CHECK-P8-NEXT:    xvcvsxdsp vs2, v5
+; CHECK-P8-NEXT:    xxswapd v5, vs4
+; CHECK-P8-NEXT:    xvcvsxdsp vs4, v2
+; CHECK-P8-NEXT:    xvcvsxdsp vs5, v3
+; CHECK-P8-NEXT:    xvcvsxdsp vs6, v4
+; CHECK-P8-NEXT:    xxsldwi v2, vs3, vs3, 3
+; CHECK-P8-NEXT:    xvcvsxdsp vs7, v5
+; CHECK-P8-NEXT:    xxsldwi v3, vs0, vs0, 3
+; CHECK-P8-NEXT:    xxsldwi v4, vs1, vs1, 3
+; CHECK-P8-NEXT:    xxsldwi v5, vs2, vs2, 3
+; CHECK-P8-NEXT:    xxsldwi v0, vs4, vs4, 3
+; CHECK-P8-NEXT:    vpkudum v2, v3, v2
+; CHECK-P8-NEXT:    xxsldwi v1, vs5, vs5, 3
+; CHECK-P8-NEXT:    xxsldwi v6, vs6, vs6, 3
+; CHECK-P8-NEXT:    vpkudum v3, v5, v4
+; CHECK-P8-NEXT:    xxsldwi v7, vs7, vs7, 3
+; CHECK-P8-NEXT:    vpkudum v4, v1, v0
+; CHECK-P8-NEXT:    vpkudum v5, v6, v7
 ; CHECK-P8-NEXT:    stvx v2, r3, r7
 ; CHECK-P8-NEXT:    stvx v3, r3, r5
 ; CHECK-P8-NEXT:    stvx v4, r3, r6
@@ -711,130 +519,74 @@ define void @test16elt_signed(<16 x float>* noalias nocapture sret %agg.result,
 ;
 ; CHECK-P9-LABEL: test16elt_signed:
 ; CHECK-P9:       # %bb.0: # %entry
-; CHECK-P9-NEXT:    lxv vs4, 48(r4)
-; CHECK-P9-NEXT:    lxv vs5, 32(r4)
-; CHECK-P9-NEXT:    lxv vs6, 16(r4)
-; CHECK-P9-NEXT:    lxv vs7, 0(r4)
-; CHECK-P9-NEXT:    lxv vs8, 112(r4)
-; CHECK-P9-NEXT:    lxv vs9, 96(r4)
-; CHECK-P9-NEXT:    lxv vs10, 80(r4)
-; CHECK-P9-NEXT:    lxv vs11, 64(r4)
-; CHECK-P9-NEXT:    stfd f30, -16(r1) # 8-byte Folded Spill
-; CHECK-P9-NEXT:    stfd f31, -8(r1) # 8-byte Folded Spill
-; CHECK-P9-NEXT:    xxswapd vs0, vs7
-; CHECK-P9-NEXT:    xxswapd vs1, vs6
-; CHECK-P9-NEXT:    xxswapd vs2, vs5
-; CHECK-P9-NEXT:    xxswapd vs3, vs4
-; CHECK-P9-NEXT:    xxswapd vs12, vs11
-; CHECK-P9-NEXT:    xxswapd vs13, vs10
-; CHECK-P9-NEXT:    xxswapd vs31, vs9
-; CHECK-P9-NEXT:    xxswapd vs30, vs8
-; CHECK-P9-NEXT:    xscvsxdsp f7, f7
-; CHECK-P9-NEXT:    xscvsxdsp f6, f6
-; CHECK-P9-NEXT:    xscvsxdsp f5, f5
-; CHECK-P9-NEXT:    xscvsxdsp f4, f4
-; CHECK-P9-NEXT:    xscvsxdsp f11, f11
-; CHECK-P9-NEXT:    xscvsxdsp f10, f10
-; CHECK-P9-NEXT:    xscvsxdsp f9, f9
-; CHECK-P9-NEXT:    xscvsxdsp f8, f8
-; CHECK-P9-NEXT:    xscvsxdsp f0, f0
-; CHECK-P9-NEXT:    xscvsxdsp f1, f1
-; CHECK-P9-NEXT:    xscvsxdsp f2, f2
-; CHECK-P9-NEXT:    xscvsxdsp f3, f3
-; CHECK-P9-NEXT:    xscvsxdsp f12, f12
-; CHECK-P9-NEXT:    xscvsxdsp f13, f13
-; CHECK-P9-NEXT:    xscvsxdsp f31, f31
-; CHECK-P9-NEXT:    xscvsxdsp f30, f30
-; CHECK-P9-NEXT:    xxmrghd vs6, vs6, vs7
-; CHECK-P9-NEXT:    xxmrghd vs4, vs4, vs5
-; CHECK-P9-NEXT:    xxmrghd vs0, vs1, vs0
-; CHECK-P9-NEXT:    xxmrghd vs1, vs3, vs2
-; CHECK-P9-NEXT:    xxmrghd vs2, vs10, vs11
-; CHECK-P9-NEXT:    xxmrghd vs3, vs8, vs9
-; CHECK-P9-NEXT:    xxmrghd vs5, vs13, vs12
-; CHECK-P9-NEXT:    xxmrghd vs7, vs30, vs31
-; CHECK-P9-NEXT:    lfd f31, -8(r1) # 8-byte Folded Reload
-; CHECK-P9-NEXT:    lfd f30, -16(r1) # 8-byte Folded Reload
-; CHECK-P9-NEXT:    xvcvdpsp v2, vs6
-; CHECK-P9-NEXT:    xvcvdpsp v3, vs4
-; CHECK-P9-NEXT:    xvcvdpsp v4, vs0
-; CHECK-P9-NEXT:    xvcvdpsp v5, vs1
-; CHECK-P9-NEXT:    xvcvdpsp v0, vs5
-; CHECK-P9-NEXT:    xvcvdpsp v1, vs2
-; CHECK-P9-NEXT:    xvcvdpsp v6, vs7
-; CHECK-P9-NEXT:    xvcvdpsp v7, vs3
-; CHECK-P9-NEXT:    vmrgew v2, v2, v4
-; CHECK-P9-NEXT:    vmrgew v3, v3, v5
-; CHECK-P9-NEXT:    vmrgew v4, v1, v0
-; CHECK-P9-NEXT:    vmrgew v5, v7, v6
+; CHECK-P9-NEXT:    lxv v7, 0(r4)
+; CHECK-P9-NEXT:    xvcvsxdsp vs0, v7
+; CHECK-P9-NEXT:    lxv v6, 16(r4)
+; CHECK-P9-NEXT:    xxsldwi v7, vs0, vs0, 3
+; CHECK-P9-NEXT:    xvcvsxdsp vs0, v6
+; CHECK-P9-NEXT:    lxv v1, 32(r4)
+; CHECK-P9-NEXT:    xxsldwi v6, vs0, vs0, 3
+; CHECK-P9-NEXT:    xvcvsxdsp vs0, v1
+; CHECK-P9-NEXT:    lxv v0, 48(r4)
+; CHECK-P9-NEXT:    vpkudum v1, v6, v7
+; CHECK-P9-NEXT:    xxsldwi v6, vs0, vs0, 3
+; CHECK-P9-NEXT:    xvcvsxdsp vs0, v0
+; CHECK-P9-NEXT:    lxv v5, 64(r4)
+; CHECK-P9-NEXT:    xxsldwi v0, vs0, vs0, 3
+; CHECK-P9-NEXT:    xvcvsxdsp vs0, v5
+; CHECK-P9-NEXT:    lxv v4, 80(r4)
+; CHECK-P9-NEXT:    vpkudum v0, v0, v6
+; CHECK-P9-NEXT:    xxsldwi v5, vs0, vs0, 3
+; CHECK-P9-NEXT:    lxv v3, 96(r4)
+; CHECK-P9-NEXT:    xvcvsxdsp vs0, v4
+; CHECK-P9-NEXT:    xxsldwi v4, vs0, vs0, 3
+; CHECK-P9-NEXT:    xvcvsxdsp vs0, v3
+; CHECK-P9-NEXT:    lxv v2, 112(r4)
+; CHECK-P9-NEXT:    stxv v0, 16(r3)
+; CHECK-P9-NEXT:    stxv v1, 0(r3)
+; CHECK-P9-NEXT:    vpkudum v4, v4, v5
 ; CHECK-P9-NEXT:    stxv v4, 32(r3)
-; CHECK-P9-NEXT:    stxv v3, 16(r3)
-; CHECK-P9-NEXT:    stxv v2, 0(r3)
-; CHECK-P9-NEXT:    stxv v5, 48(r3)
+; CHECK-P9-NEXT:    xxsldwi v3, vs0, vs0, 3
+; CHECK-P9-NEXT:    xvcvsxdsp vs0, v2
+; CHECK-P9-NEXT:    xxsldwi v2, vs0, vs0, 3
+; CHECK-P9-NEXT:    vpkudum v2, v2, v3
+; CHECK-P9-NEXT:    stxv v2, 48(r3)
 ; CHECK-P9-NEXT:    blr
 ;
 ; CHECK-BE-LABEL: test16elt_signed:
 ; CHECK-BE:       # %bb.0: # %entry
-; CHECK-BE-NEXT:    lxv vs2, 32(r4)
-; CHECK-BE-NEXT:    lxv vs3, 48(r4)
-; CHECK-BE-NEXT:    lxv vs4, 0(r4)
-; CHECK-BE-NEXT:    lxv vs5, 16(r4)
-; CHECK-BE-NEXT:    lxv vs6, 96(r4)
-; CHECK-BE-NEXT:    lxv vs7, 112(r4)
-; CHECK-BE-NEXT:    lxv vs8, 64(r4)
-; CHECK-BE-NEXT:    lxv vs9, 80(r4)
-; CHECK-BE-NEXT:    stfd f30, -16(r1) # 8-byte Folded Spill
-; CHECK-BE-NEXT:    stfd f31, -8(r1) # 8-byte Folded Spill
-; CHECK-BE-NEXT:    xxswapd vs0, vs5
-; CHECK-BE-NEXT:    xxswapd vs1, vs4
-; CHECK-BE-NEXT:    xxswapd vs10, vs3
-; CHECK-BE-NEXT:    xxswapd vs11, vs2
-; CHECK-BE-NEXT:    xxswapd vs12, vs9
-; CHECK-BE-NEXT:    xxswapd vs13, vs8
-; CHECK-BE-NEXT:    xxswapd vs31, vs7
-; CHECK-BE-NEXT:    xxswapd vs30, vs6
-; CHECK-BE-NEXT:    xscvsxdsp f5, f5
-; CHECK-BE-NEXT:    xscvsxdsp f4, f4
-; CHECK-BE-NEXT:    xscvsxdsp f3, f3
-; CHECK-BE-NEXT:    xscvsxdsp f2, f2
-; CHECK-BE-NEXT:    xscvsxdsp f9, f9
-; CHECK-BE-NEXT:    xscvsxdsp f8, f8
-; CHECK-BE-NEXT:    xscvsxdsp f7, f7
-; CHECK-BE-NEXT:    xscvsxdsp f6, f6
-; CHECK-BE-NEXT:    xscvsxdsp f0, f0
-; CHECK-BE-NEXT:    xscvsxdsp f1, f1
-; CHECK-BE-NEXT:    xscvsxdsp f10, f10
-; CHECK-BE-NEXT:    xscvsxdsp f11, f11
-; CHECK-BE-NEXT:    xscvsxdsp f12, f12
-; CHECK-BE-NEXT:    xscvsxdsp f13, f13
-; CHECK-BE-NEXT:    xscvsxdsp f31, f31
-; CHECK-BE-NEXT:    xscvsxdsp f30, f30
-; CHECK-BE-NEXT:    xxmrghd vs4, vs4, vs5
-; CHECK-BE-NEXT:    xxmrghd vs2, vs2, vs3
-; CHECK-BE-NEXT:    xxmrghd vs3, vs8, vs9
-; CHECK-BE-NEXT:    xxmrghd vs5, vs6, vs7
-; CHECK-BE-NEXT:    xxmrghd vs0, vs1, vs0
-; CHECK-BE-NEXT:    xxmrghd vs1, vs11, vs10
-; CHECK-BE-NEXT:    xxmrghd vs6, vs13, vs12
-; CHECK-BE-NEXT:    xxmrghd vs7, vs30, vs31
-; CHECK-BE-NEXT:    lfd f31, -8(r1) # 8-byte Folded Reload
-; CHECK-BE-NEXT:    lfd f30, -16(r1) # 8-byte Folded Reload
-; CHECK-BE-NEXT:    xvcvdpsp v2, vs4
-; CHECK-BE-NEXT:    xvcvdpsp v3, vs2
-; CHECK-BE-NEXT:    xvcvdpsp v0, vs3
-; CHECK-BE-NEXT:    xvcvdpsp v6, vs5
-; CHECK-BE-NEXT:    xvcvdpsp v4, vs0
-; CHECK-BE-NEXT:    xvcvdpsp v5, vs1
-; CHECK-BE-NEXT:    xvcvdpsp v1, vs6
-; CHECK-BE-NEXT:    xvcvdpsp v7, vs7
-; CHECK-BE-NEXT:    vmrgew v2, v2, v4
-; CHECK-BE-NEXT:    vmrgew v3, v3, v5
-; CHECK-BE-NEXT:    vmrgew v4, v0, v1
-; CHECK-BE-NEXT:    vmrgew v5, v6, v7
-; CHECK-BE-NEXT:    stxv v5, 48(r3)
+; CHECK-BE-NEXT:    lxv v7, 16(r4)
+; CHECK-BE-NEXT:    xvcvsxdsp vs0, v7
+; CHECK-BE-NEXT:    lxv v6, 0(r4)
+; CHECK-BE-NEXT:    xxsldwi v7, vs0, vs0, 3
+; CHECK-BE-NEXT:    xvcvsxdsp vs0, v6
+; CHECK-BE-NEXT:    lxv v1, 48(r4)
+; CHECK-BE-NEXT:    xxsldwi v6, vs0, vs0, 3
+; CHECK-BE-NEXT:    xvcvsxdsp vs0, v1
+; CHECK-BE-NEXT:    lxv v0, 32(r4)
+; CHECK-BE-NEXT:    vpkudum v1, v6, v7
+; CHECK-BE-NEXT:    xxsldwi v6, vs0, vs0, 3
+; CHECK-BE-NEXT:    xvcvsxdsp vs0, v0
+; CHECK-BE-NEXT:    lxv v5, 80(r4)
+; CHECK-BE-NEXT:    xxsldwi v0, vs0, vs0, 3
+; CHECK-BE-NEXT:    xvcvsxdsp vs0, v5
+; CHECK-BE-NEXT:    lxv v4, 64(r4)
+; CHECK-BE-NEXT:    vpkudum v0, v0, v6
+; CHECK-BE-NEXT:    xxsldwi v5, vs0, vs0, 3
+; CHECK-BE-NEXT:    lxv v3, 112(r4)
+; CHECK-BE-NEXT:    xvcvsxdsp vs0, v4
+; CHECK-BE-NEXT:    xxsldwi v4, vs0, vs0, 3
+; CHECK-BE-NEXT:    xvcvsxdsp vs0, v3
+; CHECK-BE-NEXT:    lxv v2, 96(r4)
+; CHECK-BE-NEXT:    stxv v0, 16(r3)
+; CHECK-BE-NEXT:    stxv v1, 0(r3)
+; CHECK-BE-NEXT:    vpkudum v4, v4, v5
 ; CHECK-BE-NEXT:    stxv v4, 32(r3)
-; CHECK-BE-NEXT:    stxv v3, 16(r3)
-; CHECK-BE-NEXT:    stxv v2, 0(r3)
+; CHECK-BE-NEXT:    xxsldwi v3, vs0, vs0, 3
+; CHECK-BE-NEXT:    xvcvsxdsp vs0, v2
+; CHECK-BE-NEXT:    xxsldwi v2, vs0, vs0, 3
+; CHECK-BE-NEXT:    vpkudum v2, v2, v3
+; CHECK-BE-NEXT:    stxv v2, 48(r3)
 ; CHECK-BE-NEXT:    blr
 entry:
   %a = load <16 x i64>, <16 x i64>* %0, align 128
diff --git a/test/CodeGen/PowerPC/vec_conv_i8_to_fp32_elts.ll b/test/CodeGen/PowerPC/vec_conv_i8_to_fp32_elts.ll
index 9d44a88c6a23e0e70d488cc62ebc1ccacf9838e6..432598fb8b761d91cb437f6f49c1c559d65dddb8 100644
--- a/test/CodeGen/PowerPC/vec_conv_i8_to_fp32_elts.ll
+++ b/test/CodeGen/PowerPC/vec_conv_i8_to_fp32_elts.ll
@@ -35,20 +35,20 @@ define i64 @test2elt(i16 %a.coerce) local_unnamed_addr #0 {
 ; CHECK-P9:       # %bb.0: # %entry
 ; CHECK-P9-NEXT:    mtvsrws v2, r3
 ; CHECK-P9-NEXT:    li r3, 0
-; CHECK-P9-NEXT:    li r4, 1
 ; CHECK-P9-NEXT:    vextubrx r3, r3, v2
-; CHECK-P9-NEXT:    vextubrx r4, r4, v2
 ; CHECK-P9-NEXT:    rlwinm r3, r3, 0, 24, 31
-; CHECK-P9-NEXT:    rlwinm r4, r4, 0, 24, 31
 ; CHECK-P9-NEXT:    mtvsrwz f0, r3
-; CHECK-P9-NEXT:    mtvsrwz f1, r4
+; CHECK-P9-NEXT:    li r3, 1
+; CHECK-P9-NEXT:    xscvuxdsp f0, f0
+; CHECK-P9-NEXT:    xscvdpspn vs0, f0
+; CHECK-P9-NEXT:    vextubrx r3, r3, v2
+; CHECK-P9-NEXT:    rlwinm r3, r3, 0, 24, 31
+; CHECK-P9-NEXT:    xxsldwi v3, vs0, vs0, 1
+; CHECK-P9-NEXT:    mtvsrwz f0, r3
 ; CHECK-P9-NEXT:    xscvuxdsp f0, f0
-; CHECK-P9-NEXT:    xscvuxdsp f1, f1
 ; CHECK-P9-NEXT:    xscvdpspn vs0, f0
-; CHECK-P9-NEXT:    xscvdpspn vs1, f1
 ; CHECK-P9-NEXT:    xxsldwi v2, vs0, vs0, 1
-; CHECK-P9-NEXT:    xxsldwi v3, vs1, vs1, 1
-; CHECK-P9-NEXT:    vmrglw v2, v3, v2
+; CHECK-P9-NEXT:    vmrglw v2, v2, v3
 ; CHECK-P9-NEXT:    mfvsrld r3, v2
 ; CHECK-P9-NEXT:    blr
 ;
@@ -56,18 +56,18 @@ define i64 @test2elt(i16 %a.coerce) local_unnamed_addr #0 {
 ; CHECK-BE:       # %bb.0: # %entry
 ; CHECK-BE-NEXT:    mtvsrws v2, r3
 ; CHECK-BE-NEXT:    li r3, 1
-; CHECK-BE-NEXT:    li r4, 0
 ; CHECK-BE-NEXT:    vextublx r3, r3, v2
-; CHECK-BE-NEXT:    vextublx r4, r4, v2
 ; CHECK-BE-NEXT:    rlwinm r3, r3, 0, 24, 31
-; CHECK-BE-NEXT:    rlwinm r4, r4, 0, 24, 31
 ; CHECK-BE-NEXT:    mtvsrwz f0, r3
-; CHECK-BE-NEXT:    mtvsrwz f1, r4
+; CHECK-BE-NEXT:    li r3, 0
+; CHECK-BE-NEXT:    xscvuxdsp f0, f0
+; CHECK-BE-NEXT:    vextublx r3, r3, v2
+; CHECK-BE-NEXT:    rlwinm r3, r3, 0, 24, 31
+; CHECK-BE-NEXT:    xscvdpspn v3, f0
+; CHECK-BE-NEXT:    mtvsrwz f0, r3
 ; CHECK-BE-NEXT:    xscvuxdsp f0, f0
-; CHECK-BE-NEXT:    xscvuxdsp f1, f1
 ; CHECK-BE-NEXT:    xscvdpspn v2, f0
-; CHECK-BE-NEXT:    xscvdpspn v3, f1
-; CHECK-BE-NEXT:    vmrghw v2, v3, v2
+; CHECK-BE-NEXT:    vmrghw v2, v2, v3
 ; CHECK-BE-NEXT:    mfvsrd r3, v2
 ; CHECK-BE-NEXT:    blr
 entry:
@@ -80,89 +80,36 @@ entry:
 define <4 x float> @test4elt(i32 %a.coerce) local_unnamed_addr #1 {
 ; CHECK-P8-LABEL: test4elt:
 ; CHECK-P8:       # %bb.0: # %entry
+; CHECK-P8-NEXT:    addis r4, r2, .LCPI1_0@toc@ha
 ; CHECK-P8-NEXT:    mtvsrd f0, r3
-; CHECK-P8-NEXT:    mfvsrd r3, f0
-; CHECK-P8-NEXT:    clrldi r4, r3, 56
-; CHECK-P8-NEXT:    rldicl r5, r3, 48, 56
-; CHECK-P8-NEXT:    rlwinm r4, r4, 0, 24, 31
-; CHECK-P8-NEXT:    rlwinm r5, r5, 0, 24, 31
-; CHECK-P8-NEXT:    mtvsrwz f0, r4
-; CHECK-P8-NEXT:    rldicl r4, r3, 56, 56
-; CHECK-P8-NEXT:    rldicl r3, r3, 40, 56
-; CHECK-P8-NEXT:    rlwinm r4, r4, 0, 24, 31
-; CHECK-P8-NEXT:    rlwinm r3, r3, 0, 24, 31
-; CHECK-P8-NEXT:    mtvsrwz f1, r5
-; CHECK-P8-NEXT:    mtvsrwz f2, r4
-; CHECK-P8-NEXT:    mtvsrwz f3, r3
-; CHECK-P8-NEXT:    xscvuxdsp f0, f0
-; CHECK-P8-NEXT:    xscvuxdsp f1, f1
-; CHECK-P8-NEXT:    xscvuxdsp f2, f2
-; CHECK-P8-NEXT:    xscvuxdsp f3, f3
-; CHECK-P8-NEXT:    xxmrghd vs0, vs1, vs0
-; CHECK-P8-NEXT:    xxmrghd vs1, vs3, vs2
-; CHECK-P8-NEXT:    xvcvdpsp v2, vs0
-; CHECK-P8-NEXT:    xvcvdpsp v3, vs1
-; CHECK-P8-NEXT:    vmrgew v2, v3, v2
+; CHECK-P8-NEXT:    addi r3, r4, .LCPI1_0@toc@l
+; CHECK-P8-NEXT:    xxlxor v4, v4, v4
+; CHECK-P8-NEXT:    xxswapd v2, vs0
+; CHECK-P8-NEXT:    lvx v3, 0, r3
+; CHECK-P8-NEXT:    vperm v2, v4, v2, v3
+; CHECK-P8-NEXT:    xvcvuxwsp v2, v2
 ; CHECK-P8-NEXT:    blr
 ;
 ; CHECK-P9-LABEL: test4elt:
 ; CHECK-P9:       # %bb.0: # %entry
 ; CHECK-P9-NEXT:    mtvsrws v2, r3
-; CHECK-P9-NEXT:    li r3, 0
-; CHECK-P9-NEXT:    li r4, 2
-; CHECK-P9-NEXT:    li r5, 1
-; CHECK-P9-NEXT:    li r6, 3
-; CHECK-P9-NEXT:    vextubrx r3, r3, v2
-; CHECK-P9-NEXT:    vextubrx r4, r4, v2
-; CHECK-P9-NEXT:    vextubrx r5, r5, v2
-; CHECK-P9-NEXT:    vextubrx r6, r6, v2
-; CHECK-P9-NEXT:    rlwinm r3, r3, 0, 24, 31
-; CHECK-P9-NEXT:    rlwinm r4, r4, 0, 24, 31
-; CHECK-P9-NEXT:    rlwinm r5, r5, 0, 24, 31
-; CHECK-P9-NEXT:    rlwinm r6, r6, 0, 24, 31
-; CHECK-P9-NEXT:    mtvsrwz f0, r3
-; CHECK-P9-NEXT:    mtvsrwz f1, r4
-; CHECK-P9-NEXT:    mtvsrwz f2, r5
-; CHECK-P9-NEXT:    mtvsrwz f3, r6
-; CHECK-P9-NEXT:    xscvuxdsp f0, f0
-; CHECK-P9-NEXT:    xscvuxdsp f1, f1
-; CHECK-P9-NEXT:    xscvuxdsp f2, f2
-; CHECK-P9-NEXT:    xscvuxdsp f3, f3
-; CHECK-P9-NEXT:    xxmrghd vs0, vs1, vs0
-; CHECK-P9-NEXT:    xxmrghd vs1, vs3, vs2
-; CHECK-P9-NEXT:    xvcvdpsp v2, vs0
-; CHECK-P9-NEXT:    xvcvdpsp v3, vs1
-; CHECK-P9-NEXT:    vmrgew v2, v3, v2
+; CHECK-P9-NEXT:    addis r3, r2, .LCPI1_0@toc@ha
+; CHECK-P9-NEXT:    addi r3, r3, .LCPI1_0@toc@l
+; CHECK-P9-NEXT:    lxvx v3, 0, r3
+; CHECK-P9-NEXT:    xxlxor v4, v4, v4
+; CHECK-P9-NEXT:    vperm v2, v4, v2, v3
+; CHECK-P9-NEXT:    xvcvuxwsp v2, v2
 ; CHECK-P9-NEXT:    blr
 ;
 ; CHECK-BE-LABEL: test4elt:
 ; CHECK-BE:       # %bb.0: # %entry
 ; CHECK-BE-NEXT:    mtvsrws v2, r3
-; CHECK-BE-NEXT:    li r3, 3
-; CHECK-BE-NEXT:    li r4, 1
-; CHECK-BE-NEXT:    li r5, 2
-; CHECK-BE-NEXT:    li r6, 0
-; CHECK-BE-NEXT:    vextublx r3, r3, v2
-; CHECK-BE-NEXT:    vextublx r4, r4, v2
-; CHECK-BE-NEXT:    vextublx r5, r5, v2
-; CHECK-BE-NEXT:    vextublx r6, r6, v2
-; CHECK-BE-NEXT:    rlwinm r3, r3, 0, 24, 31
-; CHECK-BE-NEXT:    rlwinm r4, r4, 0, 24, 31
-; CHECK-BE-NEXT:    rlwinm r5, r5, 0, 24, 31
-; CHECK-BE-NEXT:    rlwinm r6, r6, 0, 24, 31
-; CHECK-BE-NEXT:    mtvsrwz f0, r3
-; CHECK-BE-NEXT:    mtvsrwz f1, r4
-; CHECK-BE-NEXT:    mtvsrwz f2, r5
-; CHECK-BE-NEXT:    mtvsrwz f3, r6
-; CHECK-BE-NEXT:    xscvuxdsp f0, f0
-; CHECK-BE-NEXT:    xscvuxdsp f1, f1
-; CHECK-BE-NEXT:    xscvuxdsp f2, f2
-; CHECK-BE-NEXT:    xscvuxdsp f3, f3
-; CHECK-BE-NEXT:    xxmrghd vs0, vs1, vs0
-; CHECK-BE-NEXT:    xxmrghd vs1, vs3, vs2
-; CHECK-BE-NEXT:    xvcvdpsp v2, vs0
-; CHECK-BE-NEXT:    xvcvdpsp v3, vs1
-; CHECK-BE-NEXT:    vmrgew v2, v3, v2
+; CHECK-BE-NEXT:    addis r3, r2, .LCPI1_0@toc@ha
+; CHECK-BE-NEXT:    addi r3, r3, .LCPI1_0@toc@l
+; CHECK-BE-NEXT:    lxvx v3, 0, r3
+; CHECK-BE-NEXT:    xxlxor v4, v4, v4
+; CHECK-BE-NEXT:    vperm v2, v2, v4, v3
+; CHECK-BE-NEXT:    xvcvuxwsp v2, v2
 ; CHECK-BE-NEXT:    blr
 entry:
   %0 = bitcast i32 %a.coerce to <4 x i8>
@@ -173,168 +120,59 @@ entry:
 define void @test8elt(<8 x float>* noalias nocapture sret %agg.result, i64 %a.coerce) local_unnamed_addr #2 {
 ; CHECK-P8-LABEL: test8elt:
 ; CHECK-P8:       # %bb.0: # %entry
+; CHECK-P8-NEXT:    addis r5, r2, .LCPI2_0@toc@ha
 ; CHECK-P8-NEXT:    mtvsrd f0, r4
-; CHECK-P8-NEXT:    li r5, 16
-; CHECK-P8-NEXT:    mfvsrd r4, f0
-; CHECK-P8-NEXT:    clrldi r6, r4, 56
-; CHECK-P8-NEXT:    rldicl r7, r4, 48, 56
-; CHECK-P8-NEXT:    rlwinm r6, r6, 0, 24, 31
-; CHECK-P8-NEXT:    rlwinm r7, r7, 0, 24, 31
-; CHECK-P8-NEXT:    mtvsrwz f0, r6
-; CHECK-P8-NEXT:    rldicl r6, r4, 56, 56
-; CHECK-P8-NEXT:    rlwinm r6, r6, 0, 24, 31
-; CHECK-P8-NEXT:    mtvsrwz f1, r7
-; CHECK-P8-NEXT:    rldicl r7, r4, 40, 56
-; CHECK-P8-NEXT:    mtvsrwz f2, r6
-; CHECK-P8-NEXT:    rldicl r6, r4, 32, 56
-; CHECK-P8-NEXT:    rlwinm r7, r7, 0, 24, 31
-; CHECK-P8-NEXT:    rlwinm r6, r6, 0, 24, 31
-; CHECK-P8-NEXT:    mtvsrwz f3, r7
-; CHECK-P8-NEXT:    rldicl r7, r4, 16, 56
-; CHECK-P8-NEXT:    mtvsrwz f4, r6
-; CHECK-P8-NEXT:    rldicl r6, r4, 24, 56
-; CHECK-P8-NEXT:    rldicl r4, r4, 8, 56
-; CHECK-P8-NEXT:    rlwinm r7, r7, 0, 24, 31
-; CHECK-P8-NEXT:    rlwinm r6, r6, 0, 24, 31
-; CHECK-P8-NEXT:    rlwinm r4, r4, 0, 24, 31
-; CHECK-P8-NEXT:    mtvsrwz f5, r7
-; CHECK-P8-NEXT:    mtvsrwz f6, r6
-; CHECK-P8-NEXT:    mtvsrwz f7, r4
-; CHECK-P8-NEXT:    xscvuxdsp f0, f0
-; CHECK-P8-NEXT:    xscvuxdsp f1, f1
-; CHECK-P8-NEXT:    xscvuxdsp f2, f2
-; CHECK-P8-NEXT:    xscvuxdsp f3, f3
-; CHECK-P8-NEXT:    xscvuxdsp f4, f4
-; CHECK-P8-NEXT:    xscvuxdsp f5, f5
-; CHECK-P8-NEXT:    xscvuxdsp f6, f6
-; CHECK-P8-NEXT:    xscvuxdsp f7, f7
-; CHECK-P8-NEXT:    xxmrghd vs0, vs1, vs0
-; CHECK-P8-NEXT:    xxmrghd vs1, vs3, vs2
-; CHECK-P8-NEXT:    xvcvdpsp v2, vs0
-; CHECK-P8-NEXT:    xxmrghd vs2, vs5, vs4
-; CHECK-P8-NEXT:    xvcvdpsp v3, vs1
-; CHECK-P8-NEXT:    xxmrghd vs3, vs7, vs6
-; CHECK-P8-NEXT:    xvcvdpsp v4, vs2
-; CHECK-P8-NEXT:    xvcvdpsp v5, vs3
-; CHECK-P8-NEXT:    vmrgew v2, v3, v2
-; CHECK-P8-NEXT:    vmrgew v3, v5, v4
+; CHECK-P8-NEXT:    addis r4, r2, .LCPI2_1@toc@ha
+; CHECK-P8-NEXT:    addi r5, r5, .LCPI2_0@toc@l
+; CHECK-P8-NEXT:    addi r4, r4, .LCPI2_1@toc@l
+; CHECK-P8-NEXT:    xxlxor v4, v4, v4
+; CHECK-P8-NEXT:    lvx v2, 0, r5
+; CHECK-P8-NEXT:    xxswapd v3, vs0
+; CHECK-P8-NEXT:    lvx v5, 0, r4
+; CHECK-P8-NEXT:    li r4, 16
+; CHECK-P8-NEXT:    vperm v2, v4, v3, v2
+; CHECK-P8-NEXT:    vperm v3, v4, v3, v5
+; CHECK-P8-NEXT:    xvcvuxwsp v2, v2
+; CHECK-P8-NEXT:    xvcvuxwsp v3, v3
 ; CHECK-P8-NEXT:    stvx v2, 0, r3
-; CHECK-P8-NEXT:    stvx v3, r3, r5
+; CHECK-P8-NEXT:    stvx v3, r3, r4
 ; CHECK-P8-NEXT:    blr
 ;
 ; CHECK-P9-LABEL: test8elt:
 ; CHECK-P9:       # %bb.0: # %entry
 ; CHECK-P9-NEXT:    mtvsrd f0, r4
-; CHECK-P9-NEXT:    li r4, 0
-; CHECK-P9-NEXT:    li r5, 2
-; CHECK-P9-NEXT:    li r6, 1
-; CHECK-P9-NEXT:    li r7, 3
-; CHECK-P9-NEXT:    li r8, 4
-; CHECK-P9-NEXT:    li r9, 6
-; CHECK-P9-NEXT:    li r10, 5
-; CHECK-P9-NEXT:    li r11, 7
+; CHECK-P9-NEXT:    addis r4, r2, .LCPI2_0@toc@ha
+; CHECK-P9-NEXT:    addi r4, r4, .LCPI2_0@toc@l
+; CHECK-P9-NEXT:    lxvx v3, 0, r4
 ; CHECK-P9-NEXT:    xxswapd v2, vs0
-; CHECK-P9-NEXT:    vextubrx r4, r4, v2
-; CHECK-P9-NEXT:    vextubrx r5, r5, v2
-; CHECK-P9-NEXT:    vextubrx r6, r6, v2
-; CHECK-P9-NEXT:    vextubrx r7, r7, v2
-; CHECK-P9-NEXT:    vextubrx r8, r8, v2
-; CHECK-P9-NEXT:    vextubrx r9, r9, v2
-; CHECK-P9-NEXT:    vextubrx r10, r10, v2
-; CHECK-P9-NEXT:    vextubrx r11, r11, v2
-; CHECK-P9-NEXT:    rlwinm r4, r4, 0, 24, 31
-; CHECK-P9-NEXT:    rlwinm r5, r5, 0, 24, 31
-; CHECK-P9-NEXT:    rlwinm r6, r6, 0, 24, 31
-; CHECK-P9-NEXT:    rlwinm r7, r7, 0, 24, 31
-; CHECK-P9-NEXT:    rlwinm r8, r8, 0, 24, 31
-; CHECK-P9-NEXT:    rlwinm r9, r9, 0, 24, 31
-; CHECK-P9-NEXT:    rlwinm r10, r10, 0, 24, 31
-; CHECK-P9-NEXT:    rlwinm r11, r11, 0, 24, 31
-; CHECK-P9-NEXT:    mtvsrwz f0, r4
-; CHECK-P9-NEXT:    mtvsrwz f1, r5
-; CHECK-P9-NEXT:    mtvsrwz f2, r6
-; CHECK-P9-NEXT:    mtvsrwz f3, r7
-; CHECK-P9-NEXT:    mtvsrwz f4, r8
-; CHECK-P9-NEXT:    mtvsrwz f5, r9
-; CHECK-P9-NEXT:    mtvsrwz f6, r10
-; CHECK-P9-NEXT:    mtvsrwz f7, r11
-; CHECK-P9-NEXT:    xscvuxdsp f0, f0
-; CHECK-P9-NEXT:    xscvuxdsp f1, f1
-; CHECK-P9-NEXT:    xscvuxdsp f2, f2
-; CHECK-P9-NEXT:    xscvuxdsp f3, f3
-; CHECK-P9-NEXT:    xscvuxdsp f4, f4
-; CHECK-P9-NEXT:    xscvuxdsp f5, f5
-; CHECK-P9-NEXT:    xscvuxdsp f6, f6
-; CHECK-P9-NEXT:    xscvuxdsp f7, f7
-; CHECK-P9-NEXT:    xxmrghd vs0, vs1, vs0
-; CHECK-P9-NEXT:    xxmrghd vs1, vs3, vs2
-; CHECK-P9-NEXT:    xxmrghd vs2, vs5, vs4
-; CHECK-P9-NEXT:    xxmrghd vs3, vs7, vs6
-; CHECK-P9-NEXT:    xvcvdpsp v2, vs0
-; CHECK-P9-NEXT:    xvcvdpsp v3, vs1
-; CHECK-P9-NEXT:    xvcvdpsp v4, vs2
-; CHECK-P9-NEXT:    xvcvdpsp v5, vs3
-; CHECK-P9-NEXT:    vmrgew v2, v3, v2
-; CHECK-P9-NEXT:    vmrgew v3, v5, v4
-; CHECK-P9-NEXT:    stxv v3, 16(r3)
-; CHECK-P9-NEXT:    stxv v2, 0(r3)
+; CHECK-P9-NEXT:    xxlxor v4, v4, v4
+; CHECK-P9-NEXT:    addis r4, r2, .LCPI2_1@toc@ha
+; CHECK-P9-NEXT:    addi r4, r4, .LCPI2_1@toc@l
+; CHECK-P9-NEXT:    vperm v3, v4, v2, v3
+; CHECK-P9-NEXT:    xvcvuxwsp vs0, v3
+; CHECK-P9-NEXT:    lxvx v3, 0, r4
+; CHECK-P9-NEXT:    vperm v2, v4, v2, v3
+; CHECK-P9-NEXT:    stxv vs0, 0(r3)
+; CHECK-P9-NEXT:    xvcvuxwsp vs1, v2
+; CHECK-P9-NEXT:    stxv vs1, 16(r3)
 ; CHECK-P9-NEXT:    blr
 ;
 ; CHECK-BE-LABEL: test8elt:
 ; CHECK-BE:       # %bb.0: # %entry
-; CHECK-BE-NEXT:    li r5, 3
 ; CHECK-BE-NEXT:    mtvsrd v2, r4
-; CHECK-BE-NEXT:    li r4, 1
-; CHECK-BE-NEXT:    li r6, 2
-; CHECK-BE-NEXT:    li r7, 0
-; CHECK-BE-NEXT:    li r8, 7
-; CHECK-BE-NEXT:    li r9, 5
-; CHECK-BE-NEXT:    li r10, 6
-; CHECK-BE-NEXT:    li r11, 4
-; CHECK-BE-NEXT:    vextublx r5, r5, v2
-; CHECK-BE-NEXT:    vextublx r4, r4, v2
-; CHECK-BE-NEXT:    vextublx r6, r6, v2
-; CHECK-BE-NEXT:    vextublx r7, r7, v2
-; CHECK-BE-NEXT:    vextublx r8, r8, v2
-; CHECK-BE-NEXT:    vextublx r9, r9, v2
-; CHECK-BE-NEXT:    vextublx r10, r10, v2
-; CHECK-BE-NEXT:    vextublx r11, r11, v2
-; CHECK-BE-NEXT:    rlwinm r5, r5, 0, 24, 31
-; CHECK-BE-NEXT:    rlwinm r4, r4, 0, 24, 31
-; CHECK-BE-NEXT:    rlwinm r6, r6, 0, 24, 31
-; CHECK-BE-NEXT:    rlwinm r7, r7, 0, 24, 31
-; CHECK-BE-NEXT:    rlwinm r8, r8, 0, 24, 31
-; CHECK-BE-NEXT:    rlwinm r9, r9, 0, 24, 31
-; CHECK-BE-NEXT:    rlwinm r10, r10, 0, 24, 31
-; CHECK-BE-NEXT:    rlwinm r11, r11, 0, 24, 31
-; CHECK-BE-NEXT:    mtvsrwz f0, r5
-; CHECK-BE-NEXT:    mtvsrwz f1, r4
-; CHECK-BE-NEXT:    mtvsrwz f2, r6
-; CHECK-BE-NEXT:    mtvsrwz f3, r7
-; CHECK-BE-NEXT:    mtvsrwz f4, r8
-; CHECK-BE-NEXT:    mtvsrwz f5, r9
-; CHECK-BE-NEXT:    mtvsrwz f6, r10
-; CHECK-BE-NEXT:    mtvsrwz f7, r11
-; CHECK-BE-NEXT:    xscvuxdsp f0, f0
-; CHECK-BE-NEXT:    xscvuxdsp f1, f1
-; CHECK-BE-NEXT:    xscvuxdsp f2, f2
-; CHECK-BE-NEXT:    xscvuxdsp f3, f3
-; CHECK-BE-NEXT:    xscvuxdsp f4, f4
-; CHECK-BE-NEXT:    xscvuxdsp f5, f5
-; CHECK-BE-NEXT:    xscvuxdsp f6, f6
-; CHECK-BE-NEXT:    xscvuxdsp f7, f7
-; CHECK-BE-NEXT:    xxmrghd vs0, vs1, vs0
-; CHECK-BE-NEXT:    xxmrghd vs1, vs3, vs2
-; CHECK-BE-NEXT:    xxmrghd vs2, vs5, vs4
-; CHECK-BE-NEXT:    xxmrghd vs3, vs7, vs6
-; CHECK-BE-NEXT:    xvcvdpsp v2, vs0
-; CHECK-BE-NEXT:    xvcvdpsp v3, vs1
-; CHECK-BE-NEXT:    xvcvdpsp v4, vs2
-; CHECK-BE-NEXT:    xvcvdpsp v5, vs3
-; CHECK-BE-NEXT:    vmrgew v2, v3, v2
-; CHECK-BE-NEXT:    vmrgew v3, v5, v4
-; CHECK-BE-NEXT:    stxv v2, 0(r3)
-; CHECK-BE-NEXT:    stxv v3, 16(r3)
+; CHECK-BE-NEXT:    addis r4, r2, .LCPI2_0@toc@ha
+; CHECK-BE-NEXT:    addi r4, r4, .LCPI2_0@toc@l
+; CHECK-BE-NEXT:    lxvx v3, 0, r4
+; CHECK-BE-NEXT:    xxlxor v4, v4, v4
+; CHECK-BE-NEXT:    addis r4, r2, .LCPI2_1@toc@ha
+; CHECK-BE-NEXT:    addi r4, r4, .LCPI2_1@toc@l
+; CHECK-BE-NEXT:    vperm v3, v2, v4, v3
+; CHECK-BE-NEXT:    xvcvuxwsp vs0, v3
+; CHECK-BE-NEXT:    lxvx v3, 0, r4
+; CHECK-BE-NEXT:    vperm v2, v4, v2, v3
+; CHECK-BE-NEXT:    stxv vs0, 0(r3)
+; CHECK-BE-NEXT:    xvcvuxwsp vs1, v2
+; CHECK-BE-NEXT:    stxv vs1, 16(r3)
 ; CHECK-BE-NEXT:    blr
 entry:
   %0 = bitcast i64 %a.coerce to <8 x i8>
@@ -346,348 +184,92 @@ entry:
 define void @test16elt(<16 x float>* noalias nocapture sret %agg.result, <16 x i8> %a) local_unnamed_addr #3 {
 ; CHECK-P8-LABEL: test16elt:
 ; CHECK-P8:       # %bb.0: # %entry
-; CHECK-P8-NEXT:    mfvsrd r4, v2
-; CHECK-P8-NEXT:    xxswapd vs2, v2
-; CHECK-P8-NEXT:    clrldi r5, r4, 56
-; CHECK-P8-NEXT:    rldicl r6, r4, 48, 56
-; CHECK-P8-NEXT:    rlwinm r5, r5, 0, 24, 31
-; CHECK-P8-NEXT:    rlwinm r6, r6, 0, 24, 31
-; CHECK-P8-NEXT:    mtvsrwz f0, r5
-; CHECK-P8-NEXT:    rldicl r5, r4, 40, 56
-; CHECK-P8-NEXT:    rldicl r7, r4, 56, 56
-; CHECK-P8-NEXT:    mtvsrwz f1, r6
-; CHECK-P8-NEXT:    rldicl r6, r4, 32, 56
-; CHECK-P8-NEXT:    rlwinm r5, r5, 0, 24, 31
-; CHECK-P8-NEXT:    rlwinm r7, r7, 0, 24, 31
-; CHECK-P8-NEXT:    rlwinm r6, r6, 0, 24, 31
-; CHECK-P8-NEXT:    mtvsrwz f4, r5
-; CHECK-P8-NEXT:    rldicl r5, r4, 16, 56
-; CHECK-P8-NEXT:    mtvsrwz f3, r7
-; CHECK-P8-NEXT:    rlwinm r5, r5, 0, 24, 31
-; CHECK-P8-NEXT:    mtvsrwz f5, r6
-; CHECK-P8-NEXT:    rldicl r6, r4, 24, 56
-; CHECK-P8-NEXT:    rldicl r4, r4, 8, 56
-; CHECK-P8-NEXT:    mfvsrd r7, f2
-; CHECK-P8-NEXT:    rlwinm r4, r4, 0, 24, 31
-; CHECK-P8-NEXT:    mtvsrwz f2, r5
-; CHECK-P8-NEXT:    rlwinm r5, r6, 0, 24, 31
-; CHECK-P8-NEXT:    mtvsrwz f6, r5
-; CHECK-P8-NEXT:    clrldi r5, r7, 56
-; CHECK-P8-NEXT:    mtvsrwz f7, r4
-; CHECK-P8-NEXT:    rldicl r4, r7, 48, 56
-; CHECK-P8-NEXT:    rlwinm r5, r5, 0, 24, 31
-; CHECK-P8-NEXT:    mtvsrwz f8, r5
-; CHECK-P8-NEXT:    rldicl r5, r7, 56, 56
-; CHECK-P8-NEXT:    rlwinm r4, r4, 0, 24, 31
-; CHECK-P8-NEXT:    mtvsrwz f9, r4
-; CHECK-P8-NEXT:    rlwinm r4, r5, 0, 24, 31
-; CHECK-P8-NEXT:    rldicl r5, r7, 8, 56
-; CHECK-P8-NEXT:    mtvsrwz f10, r4
-; CHECK-P8-NEXT:    rldicl r4, r7, 40, 56
-; CHECK-P8-NEXT:    rlwinm r5, r5, 0, 24, 31
-; CHECK-P8-NEXT:    rlwinm r4, r4, 0, 24, 31
-; CHECK-P8-NEXT:    xscvuxdsp f0, f0
-; CHECK-P8-NEXT:    mtvsrwz f11, r4
-; CHECK-P8-NEXT:    rldicl r4, r7, 32, 56
-; CHECK-P8-NEXT:    rlwinm r4, r4, 0, 24, 31
-; CHECK-P8-NEXT:    xscvuxdsp f1, f1
-; CHECK-P8-NEXT:    xscvuxdsp f3, f3
-; CHECK-P8-NEXT:    xscvuxdsp f4, f4
-; CHECK-P8-NEXT:    mtvsrwz f12, r4
-; CHECK-P8-NEXT:    rldicl r4, r7, 16, 56
-; CHECK-P8-NEXT:    rlwinm r4, r4, 0, 24, 31
-; CHECK-P8-NEXT:    xscvuxdsp f5, f5
-; CHECK-P8-NEXT:    mtvsrwz f13, r4
-; CHECK-P8-NEXT:    rldicl r4, r7, 24, 56
-; CHECK-P8-NEXT:    xscvuxdsp f2, f2
-; CHECK-P8-NEXT:    rlwinm r4, r4, 0, 24, 31
-; CHECK-P8-NEXT:    xxmrghd vs0, vs1, vs0
-; CHECK-P8-NEXT:    mtvsrwz v2, r4
+; CHECK-P8-NEXT:    addis r4, r2, .LCPI3_0@toc@ha
+; CHECK-P8-NEXT:    addis r5, r2, .LCPI3_2@toc@ha
+; CHECK-P8-NEXT:    xxlxor v4, v4, v4
+; CHECK-P8-NEXT:    addi r4, r4, .LCPI3_0@toc@l
+; CHECK-P8-NEXT:    addi r5, r5, .LCPI3_2@toc@l
+; CHECK-P8-NEXT:    lvx v3, 0, r4
+; CHECK-P8-NEXT:    addis r4, r2, .LCPI3_3@toc@ha
+; CHECK-P8-NEXT:    lvx v5, 0, r5
+; CHECK-P8-NEXT:    addis r5, r2, .LCPI3_1@toc@ha
+; CHECK-P8-NEXT:    addi r4, r4, .LCPI3_3@toc@l
+; CHECK-P8-NEXT:    addi r5, r5, .LCPI3_1@toc@l
+; CHECK-P8-NEXT:    lvx v0, 0, r4
+; CHECK-P8-NEXT:    lvx v1, 0, r5
 ; CHECK-P8-NEXT:    li r4, 48
-; CHECK-P8-NEXT:    mtvsrwz v3, r5
-; CHECK-P8-NEXT:    xxmrghd vs3, vs4, vs3
 ; CHECK-P8-NEXT:    li r5, 32
-; CHECK-P8-NEXT:    xscvuxdsp f6, f6
-; CHECK-P8-NEXT:    xscvuxdsp f7, f7
-; CHECK-P8-NEXT:    xscvuxdsp f8, f8
-; CHECK-P8-NEXT:    xscvuxdsp f9, f9
-; CHECK-P8-NEXT:    xxmrghd vs2, vs2, vs5
-; CHECK-P8-NEXT:    xscvuxdsp f10, f10
-; CHECK-P8-NEXT:    xscvuxdsp f11, f11
-; CHECK-P8-NEXT:    xscvuxdsp f12, f12
-; CHECK-P8-NEXT:    xscvuxdsp f13, f13
-; CHECK-P8-NEXT:    xxmrghd vs5, vs7, vs6
-; CHECK-P8-NEXT:    xscvuxdsp f1, v2
-; CHECK-P8-NEXT:    xscvuxdsp f4, v3
-; CHECK-P8-NEXT:    xvcvdpsp v2, vs0
-; CHECK-P8-NEXT:    xxmrghd vs0, vs9, vs8
-; CHECK-P8-NEXT:    xvcvdpsp v3, vs3
-; CHECK-P8-NEXT:    xxmrghd vs3, vs11, vs10
-; CHECK-P8-NEXT:    xvcvdpsp v4, vs2
-; CHECK-P8-NEXT:    xxmrghd vs2, vs13, vs12
-; CHECK-P8-NEXT:    xvcvdpsp v5, vs5
-; CHECK-P8-NEXT:    xvcvdpsp v0, vs0
-; CHECK-P8-NEXT:    xxmrghd vs1, vs4, vs1
-; CHECK-P8-NEXT:    xvcvdpsp v1, vs3
-; CHECK-P8-NEXT:    xvcvdpsp v6, vs2
-; CHECK-P8-NEXT:    vmrgew v2, v3, v2
-; CHECK-P8-NEXT:    xvcvdpsp v7, vs1
-; CHECK-P8-NEXT:    vmrgew v3, v5, v4
-; CHECK-P8-NEXT:    vmrgew v4, v1, v0
-; CHECK-P8-NEXT:    stvx v2, r3, r5
-; CHECK-P8-NEXT:    vmrgew v5, v7, v6
-; CHECK-P8-NEXT:    stvx v3, r3, r4
-; CHECK-P8-NEXT:    li r4, 16
-; CHECK-P8-NEXT:    stvx v4, 0, r3
+; CHECK-P8-NEXT:    vperm v5, v4, v2, v5
+; CHECK-P8-NEXT:    vperm v3, v4, v2, v3
+; CHECK-P8-NEXT:    vperm v0, v4, v2, v0
+; CHECK-P8-NEXT:    vperm v2, v4, v2, v1
+; CHECK-P8-NEXT:    xvcvuxwsp v4, v5
+; CHECK-P8-NEXT:    xvcvuxwsp v3, v3
+; CHECK-P8-NEXT:    xvcvuxwsp v5, v0
+; CHECK-P8-NEXT:    xvcvuxwsp v2, v2
+; CHECK-P8-NEXT:    stvx v4, r3, r5
+; CHECK-P8-NEXT:    stvx v3, 0, r3
 ; CHECK-P8-NEXT:    stvx v5, r3, r4
+; CHECK-P8-NEXT:    li r4, 16
+; CHECK-P8-NEXT:    stvx v2, r3, r4
 ; CHECK-P8-NEXT:    blr
 ;
 ; CHECK-P9-LABEL: test16elt:
 ; CHECK-P9:       # %bb.0: # %entry
-; CHECK-P9-NEXT:    std r25, -72(r1) # 8-byte Folded Spill
-; CHECK-P9-NEXT:    std r26, -64(r1) # 8-byte Folded Spill
-; CHECK-P9-NEXT:    std r27, -56(r1) # 8-byte Folded Spill
-; CHECK-P9-NEXT:    std r28, -48(r1) # 8-byte Folded Spill
-; CHECK-P9-NEXT:    std r29, -40(r1) # 8-byte Folded Spill
-; CHECK-P9-NEXT:    li r4, 0
-; CHECK-P9-NEXT:    std r30, -32(r1) # 8-byte Folded Spill
-; CHECK-P9-NEXT:    li r5, 2
-; CHECK-P9-NEXT:    li r6, 1
-; CHECK-P9-NEXT:    li r7, 3
-; CHECK-P9-NEXT:    li r8, 4
-; CHECK-P9-NEXT:    li r9, 6
-; CHECK-P9-NEXT:    li r10, 5
-; CHECK-P9-NEXT:    li r11, 7
-; CHECK-P9-NEXT:    li r12, 8
-; CHECK-P9-NEXT:    li r0, 10
-; CHECK-P9-NEXT:    li r30, 9
-; CHECK-P9-NEXT:    li r29, 11
-; CHECK-P9-NEXT:    li r28, 12
-; CHECK-P9-NEXT:    li r27, 14
-; CHECK-P9-NEXT:    li r26, 13
-; CHECK-P9-NEXT:    li r25, 15
-; CHECK-P9-NEXT:    stfd f30, -16(r1) # 8-byte Folded Spill
-; CHECK-P9-NEXT:    stfd f31, -8(r1) # 8-byte Folded Spill
-; CHECK-P9-NEXT:    vextubrx r4, r4, v2
-; CHECK-P9-NEXT:    vextubrx r5, r5, v2
-; CHECK-P9-NEXT:    vextubrx r6, r6, v2
-; CHECK-P9-NEXT:    vextubrx r7, r7, v2
-; CHECK-P9-NEXT:    vextubrx r8, r8, v2
-; CHECK-P9-NEXT:    vextubrx r9, r9, v2
-; CHECK-P9-NEXT:    vextubrx r10, r10, v2
-; CHECK-P9-NEXT:    vextubrx r11, r11, v2
-; CHECK-P9-NEXT:    vextubrx r12, r12, v2
-; CHECK-P9-NEXT:    vextubrx r0, r0, v2
-; CHECK-P9-NEXT:    vextubrx r30, r30, v2
-; CHECK-P9-NEXT:    vextubrx r29, r29, v2
-; CHECK-P9-NEXT:    vextubrx r28, r28, v2
-; CHECK-P9-NEXT:    vextubrx r27, r27, v2
-; CHECK-P9-NEXT:    vextubrx r26, r26, v2
-; CHECK-P9-NEXT:    vextubrx r25, r25, v2
-; CHECK-P9-NEXT:    rlwinm r4, r4, 0, 24, 31
-; CHECK-P9-NEXT:    rlwinm r5, r5, 0, 24, 31
-; CHECK-P9-NEXT:    rlwinm r6, r6, 0, 24, 31
-; CHECK-P9-NEXT:    rlwinm r7, r7, 0, 24, 31
-; CHECK-P9-NEXT:    rlwinm r8, r8, 0, 24, 31
-; CHECK-P9-NEXT:    rlwinm r9, r9, 0, 24, 31
-; CHECK-P9-NEXT:    rlwinm r10, r10, 0, 24, 31
-; CHECK-P9-NEXT:    rlwinm r11, r11, 0, 24, 31
-; CHECK-P9-NEXT:    rlwinm r12, r12, 0, 24, 31
-; CHECK-P9-NEXT:    rlwinm r0, r0, 0, 24, 31
-; CHECK-P9-NEXT:    rlwinm r30, r30, 0, 24, 31
-; CHECK-P9-NEXT:    rlwinm r29, r29, 0, 24, 31
-; CHECK-P9-NEXT:    rlwinm r28, r28, 0, 24, 31
-; CHECK-P9-NEXT:    rlwinm r27, r27, 0, 24, 31
-; CHECK-P9-NEXT:    rlwinm r26, r26, 0, 24, 31
-; CHECK-P9-NEXT:    rlwinm r25, r25, 0, 24, 31
-; CHECK-P9-NEXT:    mtvsrwz f0, r4
-; CHECK-P9-NEXT:    mtvsrwz f1, r5
-; CHECK-P9-NEXT:    mtvsrwz f2, r6
-; CHECK-P9-NEXT:    mtvsrwz f3, r7
-; CHECK-P9-NEXT:    mtvsrwz f4, r8
-; CHECK-P9-NEXT:    mtvsrwz f5, r9
-; CHECK-P9-NEXT:    mtvsrwz f6, r10
-; CHECK-P9-NEXT:    mtvsrwz f7, r11
-; CHECK-P9-NEXT:    mtvsrwz f8, r12
-; CHECK-P9-NEXT:    mtvsrwz f9, r0
-; CHECK-P9-NEXT:    mtvsrwz f10, r30
-; CHECK-P9-NEXT:    ld r30, -32(r1) # 8-byte Folded Reload
-; CHECK-P9-NEXT:    mtvsrwz f11, r29
-; CHECK-P9-NEXT:    ld r29, -40(r1) # 8-byte Folded Reload
-; CHECK-P9-NEXT:    mtvsrwz f12, r28
-; CHECK-P9-NEXT:    ld r28, -48(r1) # 8-byte Folded Reload
-; CHECK-P9-NEXT:    mtvsrwz f13, r27
-; CHECK-P9-NEXT:    ld r27, -56(r1) # 8-byte Folded Reload
-; CHECK-P9-NEXT:    mtvsrwz v2, r26
-; CHECK-P9-NEXT:    ld r26, -64(r1) # 8-byte Folded Reload
-; CHECK-P9-NEXT:    mtvsrwz v3, r25
-; CHECK-P9-NEXT:    ld r25, -72(r1) # 8-byte Folded Reload
-; CHECK-P9-NEXT:    xscvuxdsp f0, f0
-; CHECK-P9-NEXT:    xscvuxdsp f1, f1
-; CHECK-P9-NEXT:    xscvuxdsp f2, f2
-; CHECK-P9-NEXT:    xscvuxdsp f3, f3
-; CHECK-P9-NEXT:    xscvuxdsp f4, f4
-; CHECK-P9-NEXT:    xscvuxdsp f5, f5
-; CHECK-P9-NEXT:    xscvuxdsp f6, f6
-; CHECK-P9-NEXT:    xscvuxdsp f7, f7
-; CHECK-P9-NEXT:    xscvuxdsp f8, f8
-; CHECK-P9-NEXT:    xscvuxdsp f9, f9
-; CHECK-P9-NEXT:    xscvuxdsp f10, f10
-; CHECK-P9-NEXT:    xscvuxdsp f11, f11
-; CHECK-P9-NEXT:    xscvuxdsp f12, f12
-; CHECK-P9-NEXT:    xscvuxdsp f13, f13
-; CHECK-P9-NEXT:    xscvuxdsp f31, v2
-; CHECK-P9-NEXT:    xscvuxdsp f30, v3
-; CHECK-P9-NEXT:    xxmrghd vs0, vs1, vs0
-; CHECK-P9-NEXT:    xxmrghd vs1, vs3, vs2
-; CHECK-P9-NEXT:    xxmrghd vs2, vs5, vs4
-; CHECK-P9-NEXT:    xxmrghd vs3, vs7, vs6
-; CHECK-P9-NEXT:    xxmrghd vs4, vs9, vs8
-; CHECK-P9-NEXT:    xxmrghd vs5, vs11, vs10
-; CHECK-P9-NEXT:    xxmrghd vs6, vs13, vs12
-; CHECK-P9-NEXT:    xxmrghd vs7, vs30, vs31
-; CHECK-P9-NEXT:    lfd f31, -8(r1) # 8-byte Folded Reload
-; CHECK-P9-NEXT:    lfd f30, -16(r1) # 8-byte Folded Reload
-; CHECK-P9-NEXT:    xvcvdpsp v2, vs0
-; CHECK-P9-NEXT:    xvcvdpsp v3, vs1
-; CHECK-P9-NEXT:    xvcvdpsp v4, vs2
-; CHECK-P9-NEXT:    xvcvdpsp v5, vs3
-; CHECK-P9-NEXT:    xvcvdpsp v0, vs4
-; CHECK-P9-NEXT:    xvcvdpsp v1, vs5
-; CHECK-P9-NEXT:    xvcvdpsp v6, vs6
-; CHECK-P9-NEXT:    xvcvdpsp v7, vs7
-; CHECK-P9-NEXT:    vmrgew v2, v3, v2
-; CHECK-P9-NEXT:    vmrgew v3, v5, v4
-; CHECK-P9-NEXT:    vmrgew v4, v1, v0
-; CHECK-P9-NEXT:    vmrgew v5, v7, v6
-; CHECK-P9-NEXT:    stxv v3, 16(r3)
-; CHECK-P9-NEXT:    stxv v2, 0(r3)
-; CHECK-P9-NEXT:    stxv v5, 48(r3)
-; CHECK-P9-NEXT:    stxv v4, 32(r3)
+; CHECK-P9-NEXT:    addis r4, r2, .LCPI3_0@toc@ha
+; CHECK-P9-NEXT:    addi r4, r4, .LCPI3_0@toc@l
+; CHECK-P9-NEXT:    lxvx v3, 0, r4
+; CHECK-P9-NEXT:    xxlxor v4, v4, v4
+; CHECK-P9-NEXT:    addis r4, r2, .LCPI3_1@toc@ha
+; CHECK-P9-NEXT:    addi r4, r4, .LCPI3_1@toc@l
+; CHECK-P9-NEXT:    vperm v3, v4, v2, v3
+; CHECK-P9-NEXT:    xvcvuxwsp vs0, v3
+; CHECK-P9-NEXT:    lxvx v3, 0, r4
+; CHECK-P9-NEXT:    addis r4, r2, .LCPI3_2@toc@ha
+; CHECK-P9-NEXT:    addi r4, r4, .LCPI3_2@toc@l
+; CHECK-P9-NEXT:    vperm v3, v4, v2, v3
+; CHECK-P9-NEXT:    stxv vs0, 0(r3)
+; CHECK-P9-NEXT:    xvcvuxwsp vs1, v3
+; CHECK-P9-NEXT:    lxvx v3, 0, r4
+; CHECK-P9-NEXT:    addis r4, r2, .LCPI3_3@toc@ha
+; CHECK-P9-NEXT:    addi r4, r4, .LCPI3_3@toc@l
+; CHECK-P9-NEXT:    vperm v3, v4, v2, v3
+; CHECK-P9-NEXT:    stxv vs1, 16(r3)
+; CHECK-P9-NEXT:    xvcvuxwsp vs2, v3
+; CHECK-P9-NEXT:    lxvx v3, 0, r4
+; CHECK-P9-NEXT:    vperm v2, v4, v2, v3
+; CHECK-P9-NEXT:    stxv vs2, 32(r3)
+; CHECK-P9-NEXT:    xvcvuxwsp vs3, v2
+; CHECK-P9-NEXT:    stxv vs3, 48(r3)
 ; CHECK-P9-NEXT:    blr
 ;
 ; CHECK-BE-LABEL: test16elt:
 ; CHECK-BE:       # %bb.0: # %entry
-; CHECK-BE-NEXT:    std r25, -72(r1) # 8-byte Folded Spill
-; CHECK-BE-NEXT:    std r26, -64(r1) # 8-byte Folded Spill
-; CHECK-BE-NEXT:    std r27, -56(r1) # 8-byte Folded Spill
-; CHECK-BE-NEXT:    std r28, -48(r1) # 8-byte Folded Spill
-; CHECK-BE-NEXT:    std r29, -40(r1) # 8-byte Folded Spill
-; CHECK-BE-NEXT:    li r4, 3
-; CHECK-BE-NEXT:    std r30, -32(r1) # 8-byte Folded Spill
-; CHECK-BE-NEXT:    li r5, 1
-; CHECK-BE-NEXT:    li r6, 2
-; CHECK-BE-NEXT:    li r7, 0
-; CHECK-BE-NEXT:    li r8, 7
-; CHECK-BE-NEXT:    li r9, 5
-; CHECK-BE-NEXT:    li r10, 6
-; CHECK-BE-NEXT:    li r11, 4
-; CHECK-BE-NEXT:    li r12, 11
-; CHECK-BE-NEXT:    li r0, 9
-; CHECK-BE-NEXT:    li r30, 10
-; CHECK-BE-NEXT:    li r29, 8
-; CHECK-BE-NEXT:    li r28, 15
-; CHECK-BE-NEXT:    li r27, 13
-; CHECK-BE-NEXT:    li r26, 14
-; CHECK-BE-NEXT:    li r25, 12
-; CHECK-BE-NEXT:    stfd f30, -16(r1) # 8-byte Folded Spill
-; CHECK-BE-NEXT:    stfd f31, -8(r1) # 8-byte Folded Spill
-; CHECK-BE-NEXT:    vextublx r4, r4, v2
-; CHECK-BE-NEXT:    vextublx r5, r5, v2
-; CHECK-BE-NEXT:    vextublx r6, r6, v2
-; CHECK-BE-NEXT:    vextublx r7, r7, v2
-; CHECK-BE-NEXT:    vextublx r8, r8, v2
-; CHECK-BE-NEXT:    vextublx r9, r9, v2
-; CHECK-BE-NEXT:    vextublx r10, r10, v2
-; CHECK-BE-NEXT:    vextublx r11, r11, v2
-; CHECK-BE-NEXT:    vextublx r12, r12, v2
-; CHECK-BE-NEXT:    vextublx r0, r0, v2
-; CHECK-BE-NEXT:    vextublx r30, r30, v2
-; CHECK-BE-NEXT:    vextublx r29, r29, v2
-; CHECK-BE-NEXT:    vextublx r28, r28, v2
-; CHECK-BE-NEXT:    vextublx r27, r27, v2
-; CHECK-BE-NEXT:    vextublx r26, r26, v2
-; CHECK-BE-NEXT:    vextublx r25, r25, v2
-; CHECK-BE-NEXT:    rlwinm r4, r4, 0, 24, 31
-; CHECK-BE-NEXT:    rlwinm r5, r5, 0, 24, 31
-; CHECK-BE-NEXT:    rlwinm r6, r6, 0, 24, 31
-; CHECK-BE-NEXT:    rlwinm r7, r7, 0, 24, 31
-; CHECK-BE-NEXT:    rlwinm r8, r8, 0, 24, 31
-; CHECK-BE-NEXT:    rlwinm r9, r9, 0, 24, 31
-; CHECK-BE-NEXT:    rlwinm r10, r10, 0, 24, 31
-; CHECK-BE-NEXT:    rlwinm r11, r11, 0, 24, 31
-; CHECK-BE-NEXT:    rlwinm r12, r12, 0, 24, 31
-; CHECK-BE-NEXT:    rlwinm r0, r0, 0, 24, 31
-; CHECK-BE-NEXT:    rlwinm r30, r30, 0, 24, 31
-; CHECK-BE-NEXT:    rlwinm r29, r29, 0, 24, 31
-; CHECK-BE-NEXT:    rlwinm r28, r28, 0, 24, 31
-; CHECK-BE-NEXT:    rlwinm r27, r27, 0, 24, 31
-; CHECK-BE-NEXT:    rlwinm r26, r26, 0, 24, 31
-; CHECK-BE-NEXT:    rlwinm r25, r25, 0, 24, 31
-; CHECK-BE-NEXT:    mtvsrwz f0, r4
-; CHECK-BE-NEXT:    mtvsrwz f1, r5
-; CHECK-BE-NEXT:    mtvsrwz f2, r6
-; CHECK-BE-NEXT:    mtvsrwz f3, r7
-; CHECK-BE-NEXT:    mtvsrwz f4, r8
-; CHECK-BE-NEXT:    mtvsrwz f5, r9
-; CHECK-BE-NEXT:    mtvsrwz f6, r10
-; CHECK-BE-NEXT:    mtvsrwz f7, r11
-; CHECK-BE-NEXT:    mtvsrwz f8, r12
-; CHECK-BE-NEXT:    mtvsrwz f9, r0
-; CHECK-BE-NEXT:    mtvsrwz f10, r30
-; CHECK-BE-NEXT:    ld r30, -32(r1) # 8-byte Folded Reload
-; CHECK-BE-NEXT:    mtvsrwz f11, r29
-; CHECK-BE-NEXT:    ld r29, -40(r1) # 8-byte Folded Reload
-; CHECK-BE-NEXT:    mtvsrwz f12, r28
-; CHECK-BE-NEXT:    ld r28, -48(r1) # 8-byte Folded Reload
-; CHECK-BE-NEXT:    mtvsrwz f13, r27
-; CHECK-BE-NEXT:    ld r27, -56(r1) # 8-byte Folded Reload
-; CHECK-BE-NEXT:    mtvsrwz v2, r26
-; CHECK-BE-NEXT:    ld r26, -64(r1) # 8-byte Folded Reload
-; CHECK-BE-NEXT:    mtvsrwz v3, r25
-; CHECK-BE-NEXT:    ld r25, -72(r1) # 8-byte Folded Reload
-; CHECK-BE-NEXT:    xscvuxdsp f0, f0
-; CHECK-BE-NEXT:    xscvuxdsp f1, f1
-; CHECK-BE-NEXT:    xscvuxdsp f2, f2
-; CHECK-BE-NEXT:    xscvuxdsp f3, f3
-; CHECK-BE-NEXT:    xscvuxdsp f4, f4
-; CHECK-BE-NEXT:    xscvuxdsp f5, f5
-; CHECK-BE-NEXT:    xscvuxdsp f6, f6
-; CHECK-BE-NEXT:    xscvuxdsp f7, f7
-; CHECK-BE-NEXT:    xscvuxdsp f8, f8
-; CHECK-BE-NEXT:    xscvuxdsp f9, f9
-; CHECK-BE-NEXT:    xscvuxdsp f10, f10
-; CHECK-BE-NEXT:    xscvuxdsp f11, f11
-; CHECK-BE-NEXT:    xscvuxdsp f12, f12
-; CHECK-BE-NEXT:    xscvuxdsp f13, f13
-; CHECK-BE-NEXT:    xscvuxdsp f31, v2
-; CHECK-BE-NEXT:    xscvuxdsp f30, v3
-; CHECK-BE-NEXT:    xxmrghd vs0, vs1, vs0
-; CHECK-BE-NEXT:    xxmrghd vs1, vs3, vs2
-; CHECK-BE-NEXT:    xxmrghd vs2, vs5, vs4
-; CHECK-BE-NEXT:    xxmrghd vs3, vs7, vs6
-; CHECK-BE-NEXT:    xxmrghd vs4, vs9, vs8
-; CHECK-BE-NEXT:    xxmrghd vs5, vs11, vs10
-; CHECK-BE-NEXT:    xxmrghd vs6, vs13, vs12
-; CHECK-BE-NEXT:    xxmrghd vs7, vs30, vs31
-; CHECK-BE-NEXT:    lfd f31, -8(r1) # 8-byte Folded Reload
-; CHECK-BE-NEXT:    lfd f30, -16(r1) # 8-byte Folded Reload
-; CHECK-BE-NEXT:    xvcvdpsp v2, vs0
-; CHECK-BE-NEXT:    xvcvdpsp v3, vs1
-; CHECK-BE-NEXT:    xvcvdpsp v4, vs2
-; CHECK-BE-NEXT:    xvcvdpsp v5, vs3
-; CHECK-BE-NEXT:    xvcvdpsp v0, vs4
-; CHECK-BE-NEXT:    xvcvdpsp v1, vs5
-; CHECK-BE-NEXT:    xvcvdpsp v6, vs6
-; CHECK-BE-NEXT:    xvcvdpsp v7, vs7
-; CHECK-BE-NEXT:    vmrgew v2, v3, v2
-; CHECK-BE-NEXT:    vmrgew v3, v5, v4
-; CHECK-BE-NEXT:    vmrgew v4, v1, v0
-; CHECK-BE-NEXT:    vmrgew v5, v7, v6
-; CHECK-BE-NEXT:    stxv v3, 16(r3)
-; CHECK-BE-NEXT:    stxv v2, 0(r3)
-; CHECK-BE-NEXT:    stxv v5, 48(r3)
-; CHECK-BE-NEXT:    stxv v4, 32(r3)
+; CHECK-BE-NEXT:    addis r4, r2, .LCPI3_0@toc@ha
+; CHECK-BE-NEXT:    addi r4, r4, .LCPI3_0@toc@l
+; CHECK-BE-NEXT:    lxvx v3, 0, r4
+; CHECK-BE-NEXT:    xxlxor v4, v4, v4
+; CHECK-BE-NEXT:    addis r4, r2, .LCPI3_1@toc@ha
+; CHECK-BE-NEXT:    addi r4, r4, .LCPI3_1@toc@l
+; CHECK-BE-NEXT:    vperm v3, v2, v4, v3
+; CHECK-BE-NEXT:    xvcvuxwsp vs0, v3
+; CHECK-BE-NEXT:    lxvx v3, 0, r4
+; CHECK-BE-NEXT:    addis r4, r2, .LCPI3_2@toc@ha
+; CHECK-BE-NEXT:    addi r4, r4, .LCPI3_2@toc@l
+; CHECK-BE-NEXT:    vperm v3, v4, v2, v3
+; CHECK-BE-NEXT:    stxv vs0, 0(r3)
+; CHECK-BE-NEXT:    xvcvuxwsp vs1, v3
+; CHECK-BE-NEXT:    lxvx v3, 0, r4
+; CHECK-BE-NEXT:    addis r4, r2, .LCPI3_3@toc@ha
+; CHECK-BE-NEXT:    addi r4, r4, .LCPI3_3@toc@l
+; CHECK-BE-NEXT:    vperm v3, v4, v2, v3
+; CHECK-BE-NEXT:    stxv vs1, 16(r3)
+; CHECK-BE-NEXT:    xvcvuxwsp vs2, v3
+; CHECK-BE-NEXT:    lxvx v3, 0, r4
+; CHECK-BE-NEXT:    vperm v2, v4, v2, v3
+; CHECK-BE-NEXT:    stxv vs2, 32(r3)
+; CHECK-BE-NEXT:    xvcvuxwsp vs3, v2
+; CHECK-BE-NEXT:    stxv vs3, 48(r3)
 ; CHECK-BE-NEXT:    blr
 entry:
   %0 = uitofp <16 x i8> %a to <16 x float>
@@ -721,20 +303,20 @@ define i64 @test2elt_signed(i16 %a.coerce) local_unnamed_addr #0 {
 ; CHECK-P9:       # %bb.0: # %entry
 ; CHECK-P9-NEXT:    mtvsrws v2, r3
 ; CHECK-P9-NEXT:    li r3, 0
-; CHECK-P9-NEXT:    li r4, 1
 ; CHECK-P9-NEXT:    vextubrx r3, r3, v2
-; CHECK-P9-NEXT:    vextubrx r4, r4, v2
 ; CHECK-P9-NEXT:    extsb r3, r3
-; CHECK-P9-NEXT:    extsb r4, r4
 ; CHECK-P9-NEXT:    mtvsrwa f0, r3
-; CHECK-P9-NEXT:    mtvsrwa f1, r4
+; CHECK-P9-NEXT:    li r3, 1
+; CHECK-P9-NEXT:    xscvsxdsp f0, f0
+; CHECK-P9-NEXT:    xscvdpspn vs0, f0
+; CHECK-P9-NEXT:    vextubrx r3, r3, v2
+; CHECK-P9-NEXT:    extsb r3, r3
+; CHECK-P9-NEXT:    xxsldwi v3, vs0, vs0, 1
+; CHECK-P9-NEXT:    mtvsrwa f0, r3
 ; CHECK-P9-NEXT:    xscvsxdsp f0, f0
-; CHECK-P9-NEXT:    xscvsxdsp f1, f1
 ; CHECK-P9-NEXT:    xscvdpspn vs0, f0
-; CHECK-P9-NEXT:    xscvdpspn vs1, f1
 ; CHECK-P9-NEXT:    xxsldwi v2, vs0, vs0, 1
-; CHECK-P9-NEXT:    xxsldwi v3, vs1, vs1, 1
-; CHECK-P9-NEXT:    vmrglw v2, v3, v2
+; CHECK-P9-NEXT:    vmrglw v2, v2, v3
 ; CHECK-P9-NEXT:    mfvsrld r3, v2
 ; CHECK-P9-NEXT:    blr
 ;
@@ -742,18 +324,18 @@ define i64 @test2elt_signed(i16 %a.coerce) local_unnamed_addr #0 {
 ; CHECK-BE:       # %bb.0: # %entry
 ; CHECK-BE-NEXT:    mtvsrws v2, r3
 ; CHECK-BE-NEXT:    li r3, 1
-; CHECK-BE-NEXT:    li r4, 0
 ; CHECK-BE-NEXT:    vextublx r3, r3, v2
-; CHECK-BE-NEXT:    vextublx r4, r4, v2
 ; CHECK-BE-NEXT:    extsb r3, r3
-; CHECK-BE-NEXT:    extsb r4, r4
 ; CHECK-BE-NEXT:    mtvsrwa f0, r3
-; CHECK-BE-NEXT:    mtvsrwa f1, r4
+; CHECK-BE-NEXT:    li r3, 0
+; CHECK-BE-NEXT:    xscvsxdsp f0, f0
+; CHECK-BE-NEXT:    vextublx r3, r3, v2
+; CHECK-BE-NEXT:    extsb r3, r3
+; CHECK-BE-NEXT:    xscvdpspn v3, f0
+; CHECK-BE-NEXT:    mtvsrwa f0, r3
 ; CHECK-BE-NEXT:    xscvsxdsp f0, f0
-; CHECK-BE-NEXT:    xscvsxdsp f1, f1
 ; CHECK-BE-NEXT:    xscvdpspn v2, f0
-; CHECK-BE-NEXT:    xscvdpspn v3, f1
-; CHECK-BE-NEXT:    vmrghw v2, v3, v2
+; CHECK-BE-NEXT:    vmrghw v2, v2, v3
 ; CHECK-BE-NEXT:    mfvsrd r3, v2
 ; CHECK-BE-NEXT:    blr
 entry:
@@ -766,89 +348,39 @@ entry:
 define <4 x float> @test4elt_signed(i32 %a.coerce) local_unnamed_addr #1 {
 ; CHECK-P8-LABEL: test4elt_signed:
 ; CHECK-P8:       # %bb.0: # %entry
+; CHECK-P8-NEXT:    addis r4, r2, .LCPI5_0@toc@ha
 ; CHECK-P8-NEXT:    mtvsrd f0, r3
-; CHECK-P8-NEXT:    mfvsrd r3, f0
-; CHECK-P8-NEXT:    clrldi r4, r3, 56
-; CHECK-P8-NEXT:    rldicl r5, r3, 48, 56
-; CHECK-P8-NEXT:    extsb r4, r4
-; CHECK-P8-NEXT:    extsb r5, r5
-; CHECK-P8-NEXT:    mtvsrwa f0, r4
-; CHECK-P8-NEXT:    rldicl r4, r3, 56, 56
-; CHECK-P8-NEXT:    rldicl r3, r3, 40, 56
-; CHECK-P8-NEXT:    extsb r4, r4
-; CHECK-P8-NEXT:    extsb r3, r3
-; CHECK-P8-NEXT:    mtvsrwa f1, r5
-; CHECK-P8-NEXT:    mtvsrwa f2, r4
-; CHECK-P8-NEXT:    mtvsrwa f3, r3
-; CHECK-P8-NEXT:    xscvsxdsp f0, f0
-; CHECK-P8-NEXT:    xscvsxdsp f1, f1
-; CHECK-P8-NEXT:    xscvsxdsp f2, f2
-; CHECK-P8-NEXT:    xscvsxdsp f3, f3
-; CHECK-P8-NEXT:    xxmrghd vs0, vs1, vs0
-; CHECK-P8-NEXT:    xxmrghd vs1, vs3, vs2
-; CHECK-P8-NEXT:    xvcvdpsp v2, vs0
-; CHECK-P8-NEXT:    xvcvdpsp v3, vs1
-; CHECK-P8-NEXT:    vmrgew v2, v3, v2
+; CHECK-P8-NEXT:    addi r3, r4, .LCPI5_0@toc@l
+; CHECK-P8-NEXT:    xxswapd v2, vs0
+; CHECK-P8-NEXT:    lvx v3, 0, r3
+; CHECK-P8-NEXT:    vperm v2, v2, v2, v3
+; CHECK-P8-NEXT:    vspltisw v3, 12
+; CHECK-P8-NEXT:    vadduwm v3, v3, v3
+; CHECK-P8-NEXT:    vslw v2, v2, v3
+; CHECK-P8-NEXT:    vsraw v2, v2, v3
+; CHECK-P8-NEXT:    xvcvsxwsp v2, v2
 ; CHECK-P8-NEXT:    blr
 ;
 ; CHECK-P9-LABEL: test4elt_signed:
 ; CHECK-P9:       # %bb.0: # %entry
 ; CHECK-P9-NEXT:    mtvsrws v2, r3
-; CHECK-P9-NEXT:    li r3, 0
-; CHECK-P9-NEXT:    li r4, 2
-; CHECK-P9-NEXT:    li r5, 1
-; CHECK-P9-NEXT:    li r6, 3
-; CHECK-P9-NEXT:    vextubrx r3, r3, v2
-; CHECK-P9-NEXT:    vextubrx r4, r4, v2
-; CHECK-P9-NEXT:    vextubrx r5, r5, v2
-; CHECK-P9-NEXT:    vextubrx r6, r6, v2
-; CHECK-P9-NEXT:    extsb r3, r3
-; CHECK-P9-NEXT:    extsb r4, r4
-; CHECK-P9-NEXT:    extsb r5, r5
-; CHECK-P9-NEXT:    extsb r6, r6
-; CHECK-P9-NEXT:    mtvsrwa f0, r3
-; CHECK-P9-NEXT:    mtvsrwa f1, r4
-; CHECK-P9-NEXT:    mtvsrwa f2, r5
-; CHECK-P9-NEXT:    mtvsrwa f3, r6
-; CHECK-P9-NEXT:    xscvsxdsp f0, f0
-; CHECK-P9-NEXT:    xscvsxdsp f1, f1
-; CHECK-P9-NEXT:    xscvsxdsp f2, f2
-; CHECK-P9-NEXT:    xscvsxdsp f3, f3
-; CHECK-P9-NEXT:    xxmrghd vs0, vs1, vs0
-; CHECK-P9-NEXT:    xxmrghd vs1, vs3, vs2
-; CHECK-P9-NEXT:    xvcvdpsp v2, vs0
-; CHECK-P9-NEXT:    xvcvdpsp v3, vs1
-; CHECK-P9-NEXT:    vmrgew v2, v3, v2
+; CHECK-P9-NEXT:    addis r3, r2, .LCPI5_0@toc@ha
+; CHECK-P9-NEXT:    addi r3, r3, .LCPI5_0@toc@l
+; CHECK-P9-NEXT:    lxvx v3, 0, r3
+; CHECK-P9-NEXT:    vperm v2, v2, v2, v3
+; CHECK-P9-NEXT:    vextsb2w v2, v2
+; CHECK-P9-NEXT:    xvcvsxwsp v2, v2
 ; CHECK-P9-NEXT:    blr
 ;
 ; CHECK-BE-LABEL: test4elt_signed:
 ; CHECK-BE:       # %bb.0: # %entry
 ; CHECK-BE-NEXT:    mtvsrws v2, r3
-; CHECK-BE-NEXT:    li r3, 3
-; CHECK-BE-NEXT:    li r4, 1
-; CHECK-BE-NEXT:    li r5, 2
-; CHECK-BE-NEXT:    li r6, 0
-; CHECK-BE-NEXT:    vextublx r3, r3, v2
-; CHECK-BE-NEXT:    vextublx r4, r4, v2
-; CHECK-BE-NEXT:    vextublx r5, r5, v2
-; CHECK-BE-NEXT:    vextublx r6, r6, v2
-; CHECK-BE-NEXT:    extsb r3, r3
-; CHECK-BE-NEXT:    extsb r4, r4
-; CHECK-BE-NEXT:    extsb r5, r5
-; CHECK-BE-NEXT:    extsb r6, r6
-; CHECK-BE-NEXT:    mtvsrwa f0, r3
-; CHECK-BE-NEXT:    mtvsrwa f1, r4
-; CHECK-BE-NEXT:    mtvsrwa f2, r5
-; CHECK-BE-NEXT:    mtvsrwa f3, r6
-; CHECK-BE-NEXT:    xscvsxdsp f0, f0
-; CHECK-BE-NEXT:    xscvsxdsp f1, f1
-; CHECK-BE-NEXT:    xscvsxdsp f2, f2
-; CHECK-BE-NEXT:    xscvsxdsp f3, f3
-; CHECK-BE-NEXT:    xxmrghd vs0, vs1, vs0
-; CHECK-BE-NEXT:    xxmrghd vs1, vs3, vs2
-; CHECK-BE-NEXT:    xvcvdpsp v2, vs0
-; CHECK-BE-NEXT:    xvcvdpsp v3, vs1
-; CHECK-BE-NEXT:    vmrgew v2, v3, v2
+; CHECK-BE-NEXT:    addis r3, r2, .LCPI5_0@toc@ha
+; CHECK-BE-NEXT:    addi r3, r3, .LCPI5_0@toc@l
+; CHECK-BE-NEXT:    lxvx v3, 0, r3
+; CHECK-BE-NEXT:    vperm v2, v2, v2, v3
+; CHECK-BE-NEXT:    vextsb2w v2, v2
+; CHECK-BE-NEXT:    xvcvsxwsp v2, v2
 ; CHECK-BE-NEXT:    blr
 entry:
   %0 = bitcast i32 %a.coerce to <4 x i8>
@@ -859,168 +391,67 @@ entry:
 define void @test8elt_signed(<8 x float>* noalias nocapture sret %agg.result, i64 %a.coerce) local_unnamed_addr #2 {
 ; CHECK-P8-LABEL: test8elt_signed:
 ; CHECK-P8:       # %bb.0: # %entry
+; CHECK-P8-NEXT:    addis r5, r2, .LCPI6_0@toc@ha
 ; CHECK-P8-NEXT:    mtvsrd f0, r4
-; CHECK-P8-NEXT:    li r5, 16
-; CHECK-P8-NEXT:    mfvsrd r4, f0
-; CHECK-P8-NEXT:    clrldi r6, r4, 56
-; CHECK-P8-NEXT:    rldicl r7, r4, 48, 56
-; CHECK-P8-NEXT:    extsb r6, r6
-; CHECK-P8-NEXT:    extsb r7, r7
-; CHECK-P8-NEXT:    mtvsrwa f0, r6
-; CHECK-P8-NEXT:    rldicl r6, r4, 56, 56
-; CHECK-P8-NEXT:    extsb r6, r6
-; CHECK-P8-NEXT:    mtvsrwa f1, r7
-; CHECK-P8-NEXT:    rldicl r7, r4, 40, 56
-; CHECK-P8-NEXT:    mtvsrwa f2, r6
-; CHECK-P8-NEXT:    rldicl r6, r4, 32, 56
-; CHECK-P8-NEXT:    extsb r7, r7
-; CHECK-P8-NEXT:    extsb r6, r6
-; CHECK-P8-NEXT:    mtvsrwa f3, r7
-; CHECK-P8-NEXT:    rldicl r7, r4, 16, 56
-; CHECK-P8-NEXT:    mtvsrwa f4, r6
-; CHECK-P8-NEXT:    rldicl r6, r4, 24, 56
-; CHECK-P8-NEXT:    rldicl r4, r4, 8, 56
-; CHECK-P8-NEXT:    extsb r7, r7
-; CHECK-P8-NEXT:    extsb r6, r6
-; CHECK-P8-NEXT:    extsb r4, r4
-; CHECK-P8-NEXT:    mtvsrwa f5, r7
-; CHECK-P8-NEXT:    mtvsrwa f6, r6
-; CHECK-P8-NEXT:    mtvsrwa f7, r4
-; CHECK-P8-NEXT:    xscvsxdsp f0, f0
-; CHECK-P8-NEXT:    xscvsxdsp f1, f1
-; CHECK-P8-NEXT:    xscvsxdsp f2, f2
-; CHECK-P8-NEXT:    xscvsxdsp f3, f3
-; CHECK-P8-NEXT:    xscvsxdsp f4, f4
-; CHECK-P8-NEXT:    xscvsxdsp f5, f5
-; CHECK-P8-NEXT:    xscvsxdsp f6, f6
-; CHECK-P8-NEXT:    xscvsxdsp f7, f7
-; CHECK-P8-NEXT:    xxmrghd vs0, vs1, vs0
-; CHECK-P8-NEXT:    xxmrghd vs1, vs3, vs2
-; CHECK-P8-NEXT:    xvcvdpsp v2, vs0
-; CHECK-P8-NEXT:    xxmrghd vs2, vs5, vs4
-; CHECK-P8-NEXT:    xvcvdpsp v3, vs1
-; CHECK-P8-NEXT:    xxmrghd vs3, vs7, vs6
-; CHECK-P8-NEXT:    xvcvdpsp v4, vs2
-; CHECK-P8-NEXT:    xvcvdpsp v5, vs3
-; CHECK-P8-NEXT:    vmrgew v2, v3, v2
-; CHECK-P8-NEXT:    vmrgew v3, v5, v4
+; CHECK-P8-NEXT:    addis r4, r2, .LCPI6_1@toc@ha
+; CHECK-P8-NEXT:    vspltisw v5, 12
+; CHECK-P8-NEXT:    addi r5, r5, .LCPI6_0@toc@l
+; CHECK-P8-NEXT:    addi r4, r4, .LCPI6_1@toc@l
+; CHECK-P8-NEXT:    lvx v2, 0, r5
+; CHECK-P8-NEXT:    xxswapd v3, vs0
+; CHECK-P8-NEXT:    lvx v4, 0, r4
+; CHECK-P8-NEXT:    li r4, 16
+; CHECK-P8-NEXT:    vperm v2, v3, v3, v2
+; CHECK-P8-NEXT:    vperm v3, v3, v3, v4
+; CHECK-P8-NEXT:    vadduwm v4, v5, v5
+; CHECK-P8-NEXT:    vslw v2, v2, v4
+; CHECK-P8-NEXT:    vslw v3, v3, v4
+; CHECK-P8-NEXT:    vsraw v2, v2, v4
+; CHECK-P8-NEXT:    vsraw v3, v3, v4
+; CHECK-P8-NEXT:    xvcvsxwsp v2, v2
+; CHECK-P8-NEXT:    xvcvsxwsp v3, v3
 ; CHECK-P8-NEXT:    stvx v2, 0, r3
-; CHECK-P8-NEXT:    stvx v3, r3, r5
+; CHECK-P8-NEXT:    stvx v3, r3, r4
 ; CHECK-P8-NEXT:    blr
 ;
 ; CHECK-P9-LABEL: test8elt_signed:
 ; CHECK-P9:       # %bb.0: # %entry
 ; CHECK-P9-NEXT:    mtvsrd f0, r4
-; CHECK-P9-NEXT:    li r4, 0
-; CHECK-P9-NEXT:    li r5, 2
-; CHECK-P9-NEXT:    li r6, 1
-; CHECK-P9-NEXT:    li r7, 3
-; CHECK-P9-NEXT:    li r8, 4
-; CHECK-P9-NEXT:    li r9, 6
-; CHECK-P9-NEXT:    li r10, 5
-; CHECK-P9-NEXT:    li r11, 7
+; CHECK-P9-NEXT:    addis r4, r2, .LCPI6_0@toc@ha
+; CHECK-P9-NEXT:    addi r4, r4, .LCPI6_0@toc@l
+; CHECK-P9-NEXT:    lxvx v3, 0, r4
 ; CHECK-P9-NEXT:    xxswapd v2, vs0
-; CHECK-P9-NEXT:    vextubrx r4, r4, v2
-; CHECK-P9-NEXT:    vextubrx r5, r5, v2
-; CHECK-P9-NEXT:    vextubrx r6, r6, v2
-; CHECK-P9-NEXT:    vextubrx r7, r7, v2
-; CHECK-P9-NEXT:    vextubrx r8, r8, v2
-; CHECK-P9-NEXT:    vextubrx r9, r9, v2
-; CHECK-P9-NEXT:    vextubrx r10, r10, v2
-; CHECK-P9-NEXT:    vextubrx r11, r11, v2
-; CHECK-P9-NEXT:    extsb r4, r4
-; CHECK-P9-NEXT:    extsb r5, r5
-; CHECK-P9-NEXT:    extsb r6, r6
-; CHECK-P9-NEXT:    extsb r7, r7
-; CHECK-P9-NEXT:    extsb r8, r8
-; CHECK-P9-NEXT:    extsb r9, r9
-; CHECK-P9-NEXT:    extsb r10, r10
-; CHECK-P9-NEXT:    extsb r11, r11
-; CHECK-P9-NEXT:    mtvsrwa f0, r4
-; CHECK-P9-NEXT:    mtvsrwa f1, r5
-; CHECK-P9-NEXT:    mtvsrwa f2, r6
-; CHECK-P9-NEXT:    mtvsrwa f3, r7
-; CHECK-P9-NEXT:    mtvsrwa f4, r8
-; CHECK-P9-NEXT:    mtvsrwa f5, r9
-; CHECK-P9-NEXT:    mtvsrwa f6, r10
-; CHECK-P9-NEXT:    mtvsrwa f7, r11
-; CHECK-P9-NEXT:    xscvsxdsp f0, f0
-; CHECK-P9-NEXT:    xscvsxdsp f1, f1
-; CHECK-P9-NEXT:    xscvsxdsp f2, f2
-; CHECK-P9-NEXT:    xscvsxdsp f3, f3
-; CHECK-P9-NEXT:    xscvsxdsp f4, f4
-; CHECK-P9-NEXT:    xscvsxdsp f5, f5
-; CHECK-P9-NEXT:    xscvsxdsp f6, f6
-; CHECK-P9-NEXT:    xscvsxdsp f7, f7
-; CHECK-P9-NEXT:    xxmrghd vs0, vs1, vs0
-; CHECK-P9-NEXT:    xxmrghd vs1, vs3, vs2
-; CHECK-P9-NEXT:    xxmrghd vs2, vs5, vs4
-; CHECK-P9-NEXT:    xxmrghd vs3, vs7, vs6
-; CHECK-P9-NEXT:    xvcvdpsp v2, vs0
-; CHECK-P9-NEXT:    xvcvdpsp v3, vs1
-; CHECK-P9-NEXT:    xvcvdpsp v4, vs2
-; CHECK-P9-NEXT:    xvcvdpsp v5, vs3
-; CHECK-P9-NEXT:    vmrgew v2, v3, v2
-; CHECK-P9-NEXT:    vmrgew v3, v5, v4
-; CHECK-P9-NEXT:    stxv v3, 16(r3)
-; CHECK-P9-NEXT:    stxv v2, 0(r3)
+; CHECK-P9-NEXT:    vperm v3, v2, v2, v3
+; CHECK-P9-NEXT:    addis r4, r2, .LCPI6_1@toc@ha
+; CHECK-P9-NEXT:    addi r4, r4, .LCPI6_1@toc@l
+; CHECK-P9-NEXT:    vextsb2w v3, v3
+; CHECK-P9-NEXT:    xvcvsxwsp vs0, v3
+; CHECK-P9-NEXT:    lxvx v3, 0, r4
+; CHECK-P9-NEXT:    vperm v2, v2, v2, v3
+; CHECK-P9-NEXT:    stxv vs0, 0(r3)
+; CHECK-P9-NEXT:    vextsb2w v2, v2
+; CHECK-P9-NEXT:    xvcvsxwsp vs1, v2
+; CHECK-P9-NEXT:    stxv vs1, 16(r3)
 ; CHECK-P9-NEXT:    blr
 ;
 ; CHECK-BE-LABEL: test8elt_signed:
 ; CHECK-BE:       # %bb.0: # %entry
-; CHECK-BE-NEXT:    li r5, 3
 ; CHECK-BE-NEXT:    mtvsrd v2, r4
-; CHECK-BE-NEXT:    li r4, 1
-; CHECK-BE-NEXT:    li r6, 2
-; CHECK-BE-NEXT:    li r7, 0
-; CHECK-BE-NEXT:    li r8, 7
-; CHECK-BE-NEXT:    li r9, 5
-; CHECK-BE-NEXT:    li r10, 6
-; CHECK-BE-NEXT:    li r11, 4
-; CHECK-BE-NEXT:    vextublx r5, r5, v2
-; CHECK-BE-NEXT:    vextublx r4, r4, v2
-; CHECK-BE-NEXT:    vextublx r6, r6, v2
-; CHECK-BE-NEXT:    vextublx r7, r7, v2
-; CHECK-BE-NEXT:    vextublx r8, r8, v2
-; CHECK-BE-NEXT:    vextublx r9, r9, v2
-; CHECK-BE-NEXT:    vextublx r10, r10, v2
-; CHECK-BE-NEXT:    vextublx r11, r11, v2
-; CHECK-BE-NEXT:    extsb r5, r5
-; CHECK-BE-NEXT:    extsb r4, r4
-; CHECK-BE-NEXT:    extsb r6, r6
-; CHECK-BE-NEXT:    extsb r7, r7
-; CHECK-BE-NEXT:    extsb r8, r8
-; CHECK-BE-NEXT:    extsb r9, r9
-; CHECK-BE-NEXT:    extsb r10, r10
-; CHECK-BE-NEXT:    extsb r11, r11
-; CHECK-BE-NEXT:    mtvsrwa f0, r5
-; CHECK-BE-NEXT:    mtvsrwa f1, r4
-; CHECK-BE-NEXT:    mtvsrwa f2, r6
-; CHECK-BE-NEXT:    mtvsrwa f3, r7
-; CHECK-BE-NEXT:    mtvsrwa f4, r8
-; CHECK-BE-NEXT:    mtvsrwa f5, r9
-; CHECK-BE-NEXT:    mtvsrwa f6, r10
-; CHECK-BE-NEXT:    mtvsrwa f7, r11
-; CHECK-BE-NEXT:    xscvsxdsp f0, f0
-; CHECK-BE-NEXT:    xscvsxdsp f1, f1
-; CHECK-BE-NEXT:    xscvsxdsp f2, f2
-; CHECK-BE-NEXT:    xscvsxdsp f3, f3
-; CHECK-BE-NEXT:    xscvsxdsp f4, f4
-; CHECK-BE-NEXT:    xscvsxdsp f5, f5
-; CHECK-BE-NEXT:    xscvsxdsp f6, f6
-; CHECK-BE-NEXT:    xscvsxdsp f7, f7
-; CHECK-BE-NEXT:    xxmrghd vs0, vs1, vs0
-; CHECK-BE-NEXT:    xxmrghd vs1, vs3, vs2
-; CHECK-BE-NEXT:    xxmrghd vs2, vs5, vs4
-; CHECK-BE-NEXT:    xxmrghd vs3, vs7, vs6
-; CHECK-BE-NEXT:    xvcvdpsp v2, vs0
-; CHECK-BE-NEXT:    xvcvdpsp v3, vs1
-; CHECK-BE-NEXT:    xvcvdpsp v4, vs2
-; CHECK-BE-NEXT:    xvcvdpsp v5, vs3
-; CHECK-BE-NEXT:    vmrgew v2, v3, v2
-; CHECK-BE-NEXT:    vmrgew v3, v5, v4
-; CHECK-BE-NEXT:    stxv v2, 0(r3)
-; CHECK-BE-NEXT:    stxv v3, 16(r3)
+; CHECK-BE-NEXT:    addis r4, r2, .LCPI6_0@toc@ha
+; CHECK-BE-NEXT:    addi r4, r4, .LCPI6_0@toc@l
+; CHECK-BE-NEXT:    lxvx v4, 0, r4
+; CHECK-BE-NEXT:    xxlxor v3, v3, v3
+; CHECK-BE-NEXT:    vperm v3, v3, v2, v4
+; CHECK-BE-NEXT:    addis r4, r2, .LCPI6_1@toc@ha
+; CHECK-BE-NEXT:    vextsb2w v3, v3
+; CHECK-BE-NEXT:    addi r4, r4, .LCPI6_1@toc@l
+; CHECK-BE-NEXT:    xvcvsxwsp vs0, v3
+; CHECK-BE-NEXT:    lxvx v3, 0, r4
+; CHECK-BE-NEXT:    vperm v2, v2, v2, v3
+; CHECK-BE-NEXT:    stxv vs0, 16(r3)
+; CHECK-BE-NEXT:    vextsb2w v2, v2
+; CHECK-BE-NEXT:    xvcvsxwsp vs1, v2
+; CHECK-BE-NEXT:    stxv vs1, 0(r3)
 ; CHECK-BE-NEXT:    blr
 entry:
   %0 = bitcast i64 %a.coerce to <8 x i8>
@@ -1032,348 +463,108 @@ entry:
 define void @test16elt_signed(<16 x float>* noalias nocapture sret %agg.result, <16 x i8> %a) local_unnamed_addr #3 {
 ; CHECK-P8-LABEL: test16elt_signed:
 ; CHECK-P8:       # %bb.0: # %entry
-; CHECK-P8-NEXT:    mfvsrd r4, v2
-; CHECK-P8-NEXT:    xxswapd vs2, v2
-; CHECK-P8-NEXT:    clrldi r5, r4, 56
-; CHECK-P8-NEXT:    rldicl r6, r4, 48, 56
-; CHECK-P8-NEXT:    extsb r5, r5
-; CHECK-P8-NEXT:    extsb r6, r6
-; CHECK-P8-NEXT:    mtvsrwa f0, r5
-; CHECK-P8-NEXT:    rldicl r5, r4, 40, 56
-; CHECK-P8-NEXT:    rldicl r7, r4, 56, 56
-; CHECK-P8-NEXT:    mtvsrwa f1, r6
-; CHECK-P8-NEXT:    rldicl r6, r4, 32, 56
-; CHECK-P8-NEXT:    extsb r5, r5
-; CHECK-P8-NEXT:    extsb r7, r7
-; CHECK-P8-NEXT:    extsb r6, r6
-; CHECK-P8-NEXT:    mtvsrwa f4, r5
-; CHECK-P8-NEXT:    rldicl r5, r4, 16, 56
-; CHECK-P8-NEXT:    mtvsrwa f3, r7
-; CHECK-P8-NEXT:    extsb r5, r5
-; CHECK-P8-NEXT:    mtvsrwa f5, r6
-; CHECK-P8-NEXT:    rldicl r6, r4, 24, 56
-; CHECK-P8-NEXT:    rldicl r4, r4, 8, 56
-; CHECK-P8-NEXT:    mfvsrd r7, f2
-; CHECK-P8-NEXT:    extsb r4, r4
-; CHECK-P8-NEXT:    mtvsrwa f2, r5
-; CHECK-P8-NEXT:    extsb r5, r6
-; CHECK-P8-NEXT:    mtvsrwa f6, r5
-; CHECK-P8-NEXT:    clrldi r5, r7, 56
-; CHECK-P8-NEXT:    mtvsrwa f7, r4
-; CHECK-P8-NEXT:    rldicl r4, r7, 48, 56
-; CHECK-P8-NEXT:    extsb r5, r5
-; CHECK-P8-NEXT:    mtvsrwa f8, r5
-; CHECK-P8-NEXT:    rldicl r5, r7, 56, 56
-; CHECK-P8-NEXT:    extsb r4, r4
-; CHECK-P8-NEXT:    mtvsrwa f9, r4
-; CHECK-P8-NEXT:    extsb r4, r5
-; CHECK-P8-NEXT:    rldicl r5, r7, 8, 56
-; CHECK-P8-NEXT:    mtvsrwa f10, r4
-; CHECK-P8-NEXT:    rldicl r4, r7, 40, 56
-; CHECK-P8-NEXT:    extsb r5, r5
-; CHECK-P8-NEXT:    extsb r4, r4
-; CHECK-P8-NEXT:    xscvsxdsp f0, f0
-; CHECK-P8-NEXT:    mtvsrwa f11, r4
-; CHECK-P8-NEXT:    rldicl r4, r7, 32, 56
-; CHECK-P8-NEXT:    extsb r4, r4
-; CHECK-P8-NEXT:    xscvsxdsp f1, f1
-; CHECK-P8-NEXT:    xscvsxdsp f3, f3
-; CHECK-P8-NEXT:    xscvsxdsp f4, f4
-; CHECK-P8-NEXT:    mtvsrwa f12, r4
-; CHECK-P8-NEXT:    rldicl r4, r7, 16, 56
-; CHECK-P8-NEXT:    extsb r4, r4
-; CHECK-P8-NEXT:    xscvsxdsp f5, f5
-; CHECK-P8-NEXT:    mtvsrwa f13, r4
-; CHECK-P8-NEXT:    rldicl r4, r7, 24, 56
-; CHECK-P8-NEXT:    xscvsxdsp f2, f2
-; CHECK-P8-NEXT:    extsb r4, r4
-; CHECK-P8-NEXT:    xxmrghd vs0, vs1, vs0
-; CHECK-P8-NEXT:    mtvsrwa v2, r4
+; CHECK-P8-NEXT:    addis r4, r2, .LCPI7_0@toc@ha
+; CHECK-P8-NEXT:    addis r5, r2, .LCPI7_2@toc@ha
+; CHECK-P8-NEXT:    vspltisw v1, 12
+; CHECK-P8-NEXT:    addi r4, r4, .LCPI7_0@toc@l
+; CHECK-P8-NEXT:    addi r5, r5, .LCPI7_2@toc@l
+; CHECK-P8-NEXT:    lvx v3, 0, r4
+; CHECK-P8-NEXT:    addis r4, r2, .LCPI7_3@toc@ha
+; CHECK-P8-NEXT:    lvx v4, 0, r5
+; CHECK-P8-NEXT:    addis r5, r2, .LCPI7_1@toc@ha
+; CHECK-P8-NEXT:    addi r4, r4, .LCPI7_3@toc@l
+; CHECK-P8-NEXT:    addi r5, r5, .LCPI7_1@toc@l
+; CHECK-P8-NEXT:    lvx v5, 0, r4
+; CHECK-P8-NEXT:    lvx v0, 0, r5
 ; CHECK-P8-NEXT:    li r4, 48
-; CHECK-P8-NEXT:    mtvsrwa v3, r5
-; CHECK-P8-NEXT:    xxmrghd vs3, vs4, vs3
 ; CHECK-P8-NEXT:    li r5, 32
-; CHECK-P8-NEXT:    xscvsxdsp f6, f6
-; CHECK-P8-NEXT:    xscvsxdsp f7, f7
-; CHECK-P8-NEXT:    xscvsxdsp f8, f8
-; CHECK-P8-NEXT:    xscvsxdsp f9, f9
-; CHECK-P8-NEXT:    xxmrghd vs2, vs2, vs5
-; CHECK-P8-NEXT:    xscvsxdsp f10, f10
-; CHECK-P8-NEXT:    xscvsxdsp f11, f11
-; CHECK-P8-NEXT:    xscvsxdsp f12, f12
-; CHECK-P8-NEXT:    xscvsxdsp f13, f13
-; CHECK-P8-NEXT:    xxmrghd vs5, vs7, vs6
-; CHECK-P8-NEXT:    xscvsxdsp f1, v2
-; CHECK-P8-NEXT:    xscvsxdsp f4, v3
-; CHECK-P8-NEXT:    xvcvdpsp v2, vs0
-; CHECK-P8-NEXT:    xxmrghd vs0, vs9, vs8
-; CHECK-P8-NEXT:    xvcvdpsp v3, vs3
-; CHECK-P8-NEXT:    xxmrghd vs3, vs11, vs10
-; CHECK-P8-NEXT:    xvcvdpsp v4, vs2
-; CHECK-P8-NEXT:    xxmrghd vs2, vs13, vs12
-; CHECK-P8-NEXT:    xvcvdpsp v5, vs5
-; CHECK-P8-NEXT:    xvcvdpsp v0, vs0
-; CHECK-P8-NEXT:    xxmrghd vs1, vs4, vs1
-; CHECK-P8-NEXT:    xvcvdpsp v1, vs3
-; CHECK-P8-NEXT:    xvcvdpsp v6, vs2
-; CHECK-P8-NEXT:    vmrgew v2, v3, v2
-; CHECK-P8-NEXT:    xvcvdpsp v7, vs1
-; CHECK-P8-NEXT:    vmrgew v3, v5, v4
-; CHECK-P8-NEXT:    vmrgew v4, v1, v0
-; CHECK-P8-NEXT:    stvx v2, r3, r5
-; CHECK-P8-NEXT:    vmrgew v5, v7, v6
-; CHECK-P8-NEXT:    stvx v3, r3, r4
-; CHECK-P8-NEXT:    li r4, 16
-; CHECK-P8-NEXT:    stvx v4, 0, r3
+; CHECK-P8-NEXT:    vperm v3, v2, v2, v3
+; CHECK-P8-NEXT:    vperm v4, v2, v2, v4
+; CHECK-P8-NEXT:    vperm v5, v2, v2, v5
+; CHECK-P8-NEXT:    vperm v2, v2, v2, v0
+; CHECK-P8-NEXT:    vadduwm v0, v1, v1
+; CHECK-P8-NEXT:    vslw v3, v3, v0
+; CHECK-P8-NEXT:    vslw v4, v4, v0
+; CHECK-P8-NEXT:    vslw v5, v5, v0
+; CHECK-P8-NEXT:    vslw v2, v2, v0
+; CHECK-P8-NEXT:    vsraw v3, v3, v0
+; CHECK-P8-NEXT:    vsraw v4, v4, v0
+; CHECK-P8-NEXT:    vsraw v5, v5, v0
+; CHECK-P8-NEXT:    vsraw v2, v2, v0
+; CHECK-P8-NEXT:    xvcvsxwsp v3, v3
+; CHECK-P8-NEXT:    xvcvsxwsp v4, v4
+; CHECK-P8-NEXT:    xvcvsxwsp v5, v5
+; CHECK-P8-NEXT:    xvcvsxwsp v2, v2
+; CHECK-P8-NEXT:    stvx v3, 0, r3
+; CHECK-P8-NEXT:    stvx v4, r3, r5
 ; CHECK-P8-NEXT:    stvx v5, r3, r4
+; CHECK-P8-NEXT:    li r4, 16
+; CHECK-P8-NEXT:    stvx v2, r3, r4
 ; CHECK-P8-NEXT:    blr
 ;
 ; CHECK-P9-LABEL: test16elt_signed:
 ; CHECK-P9:       # %bb.0: # %entry
-; CHECK-P9-NEXT:    std r25, -72(r1) # 8-byte Folded Spill
-; CHECK-P9-NEXT:    std r26, -64(r1) # 8-byte Folded Spill
-; CHECK-P9-NEXT:    std r27, -56(r1) # 8-byte Folded Spill
-; CHECK-P9-NEXT:    std r28, -48(r1) # 8-byte Folded Spill
-; CHECK-P9-NEXT:    std r29, -40(r1) # 8-byte Folded Spill
-; CHECK-P9-NEXT:    li r4, 0
-; CHECK-P9-NEXT:    std r30, -32(r1) # 8-byte Folded Spill
-; CHECK-P9-NEXT:    li r5, 2
-; CHECK-P9-NEXT:    li r6, 1
-; CHECK-P9-NEXT:    li r7, 3
-; CHECK-P9-NEXT:    li r8, 4
-; CHECK-P9-NEXT:    li r9, 6
-; CHECK-P9-NEXT:    li r10, 5
-; CHECK-P9-NEXT:    li r11, 7
-; CHECK-P9-NEXT:    li r12, 8
-; CHECK-P9-NEXT:    li r0, 10
-; CHECK-P9-NEXT:    li r30, 9
-; CHECK-P9-NEXT:    li r29, 11
-; CHECK-P9-NEXT:    li r28, 12
-; CHECK-P9-NEXT:    li r27, 14
-; CHECK-P9-NEXT:    li r26, 13
-; CHECK-P9-NEXT:    li r25, 15
-; CHECK-P9-NEXT:    stfd f30, -16(r1) # 8-byte Folded Spill
-; CHECK-P9-NEXT:    stfd f31, -8(r1) # 8-byte Folded Spill
-; CHECK-P9-NEXT:    vextubrx r4, r4, v2
-; CHECK-P9-NEXT:    vextubrx r5, r5, v2
-; CHECK-P9-NEXT:    vextubrx r6, r6, v2
-; CHECK-P9-NEXT:    vextubrx r7, r7, v2
-; CHECK-P9-NEXT:    vextubrx r8, r8, v2
-; CHECK-P9-NEXT:    vextubrx r9, r9, v2
-; CHECK-P9-NEXT:    vextubrx r10, r10, v2
-; CHECK-P9-NEXT:    vextubrx r11, r11, v2
-; CHECK-P9-NEXT:    vextubrx r12, r12, v2
-; CHECK-P9-NEXT:    vextubrx r0, r0, v2
-; CHECK-P9-NEXT:    vextubrx r30, r30, v2
-; CHECK-P9-NEXT:    vextubrx r29, r29, v2
-; CHECK-P9-NEXT:    vextubrx r28, r28, v2
-; CHECK-P9-NEXT:    vextubrx r27, r27, v2
-; CHECK-P9-NEXT:    vextubrx r26, r26, v2
-; CHECK-P9-NEXT:    vextubrx r25, r25, v2
-; CHECK-P9-NEXT:    extsb r4, r4
-; CHECK-P9-NEXT:    extsb r5, r5
-; CHECK-P9-NEXT:    extsb r6, r6
-; CHECK-P9-NEXT:    extsb r7, r7
-; CHECK-P9-NEXT:    extsb r8, r8
-; CHECK-P9-NEXT:    extsb r9, r9
-; CHECK-P9-NEXT:    extsb r10, r10
-; CHECK-P9-NEXT:    extsb r11, r11
-; CHECK-P9-NEXT:    extsb r12, r12
-; CHECK-P9-NEXT:    extsb r0, r0
-; CHECK-P9-NEXT:    extsb r30, r30
-; CHECK-P9-NEXT:    extsb r29, r29
-; CHECK-P9-NEXT:    extsb r28, r28
-; CHECK-P9-NEXT:    extsb r27, r27
-; CHECK-P9-NEXT:    extsb r26, r26
-; CHECK-P9-NEXT:    extsb r25, r25
-; CHECK-P9-NEXT:    mtvsrwa f0, r4
-; CHECK-P9-NEXT:    mtvsrwa f1, r5
-; CHECK-P9-NEXT:    mtvsrwa f2, r6
-; CHECK-P9-NEXT:    mtvsrwa f3, r7
-; CHECK-P9-NEXT:    mtvsrwa f4, r8
-; CHECK-P9-NEXT:    mtvsrwa f5, r9
-; CHECK-P9-NEXT:    mtvsrwa f6, r10
-; CHECK-P9-NEXT:    mtvsrwa f7, r11
-; CHECK-P9-NEXT:    mtvsrwa f8, r12
-; CHECK-P9-NEXT:    mtvsrwa f9, r0
-; CHECK-P9-NEXT:    mtvsrwa f10, r30
-; CHECK-P9-NEXT:    ld r30, -32(r1) # 8-byte Folded Reload
-; CHECK-P9-NEXT:    mtvsrwa f11, r29
-; CHECK-P9-NEXT:    ld r29, -40(r1) # 8-byte Folded Reload
-; CHECK-P9-NEXT:    mtvsrwa f12, r28
-; CHECK-P9-NEXT:    ld r28, -48(r1) # 8-byte Folded Reload
-; CHECK-P9-NEXT:    mtvsrwa f13, r27
-; CHECK-P9-NEXT:    ld r27, -56(r1) # 8-byte Folded Reload
-; CHECK-P9-NEXT:    mtvsrwa v2, r26
-; CHECK-P9-NEXT:    ld r26, -64(r1) # 8-byte Folded Reload
-; CHECK-P9-NEXT:    mtvsrwa v3, r25
-; CHECK-P9-NEXT:    ld r25, -72(r1) # 8-byte Folded Reload
-; CHECK-P9-NEXT:    xscvsxdsp f0, f0
-; CHECK-P9-NEXT:    xscvsxdsp f1, f1
-; CHECK-P9-NEXT:    xscvsxdsp f2, f2
-; CHECK-P9-NEXT:    xscvsxdsp f3, f3
-; CHECK-P9-NEXT:    xscvsxdsp f4, f4
-; CHECK-P9-NEXT:    xscvsxdsp f5, f5
-; CHECK-P9-NEXT:    xscvsxdsp f6, f6
-; CHECK-P9-NEXT:    xscvsxdsp f7, f7
-; CHECK-P9-NEXT:    xscvsxdsp f8, f8
-; CHECK-P9-NEXT:    xscvsxdsp f9, f9
-; CHECK-P9-NEXT:    xscvsxdsp f10, f10
-; CHECK-P9-NEXT:    xscvsxdsp f11, f11
-; CHECK-P9-NEXT:    xscvsxdsp f12, f12
-; CHECK-P9-NEXT:    xscvsxdsp f13, f13
-; CHECK-P9-NEXT:    xscvsxdsp f31, v2
-; CHECK-P9-NEXT:    xscvsxdsp f30, v3
-; CHECK-P9-NEXT:    xxmrghd vs0, vs1, vs0
-; CHECK-P9-NEXT:    xxmrghd vs1, vs3, vs2
-; CHECK-P9-NEXT:    xxmrghd vs2, vs5, vs4
-; CHECK-P9-NEXT:    xxmrghd vs3, vs7, vs6
-; CHECK-P9-NEXT:    xxmrghd vs4, vs9, vs8
-; CHECK-P9-NEXT:    xxmrghd vs5, vs11, vs10
-; CHECK-P9-NEXT:    xxmrghd vs6, vs13, vs12
-; CHECK-P9-NEXT:    xxmrghd vs7, vs30, vs31
-; CHECK-P9-NEXT:    lfd f31, -8(r1) # 8-byte Folded Reload
-; CHECK-P9-NEXT:    lfd f30, -16(r1) # 8-byte Folded Reload
-; CHECK-P9-NEXT:    xvcvdpsp v2, vs0
-; CHECK-P9-NEXT:    xvcvdpsp v3, vs1
-; CHECK-P9-NEXT:    xvcvdpsp v4, vs2
-; CHECK-P9-NEXT:    xvcvdpsp v5, vs3
-; CHECK-P9-NEXT:    xvcvdpsp v0, vs4
-; CHECK-P9-NEXT:    xvcvdpsp v1, vs5
-; CHECK-P9-NEXT:    xvcvdpsp v6, vs6
-; CHECK-P9-NEXT:    xvcvdpsp v7, vs7
-; CHECK-P9-NEXT:    vmrgew v2, v3, v2
-; CHECK-P9-NEXT:    vmrgew v3, v5, v4
-; CHECK-P9-NEXT:    vmrgew v4, v1, v0
-; CHECK-P9-NEXT:    vmrgew v5, v7, v6
-; CHECK-P9-NEXT:    stxv v3, 16(r3)
-; CHECK-P9-NEXT:    stxv v2, 0(r3)
-; CHECK-P9-NEXT:    stxv v5, 48(r3)
-; CHECK-P9-NEXT:    stxv v4, 32(r3)
+; CHECK-P9-NEXT:    addis r4, r2, .LCPI7_0@toc@ha
+; CHECK-P9-NEXT:    addi r4, r4, .LCPI7_0@toc@l
+; CHECK-P9-NEXT:    lxvx v3, 0, r4
+; CHECK-P9-NEXT:    addis r4, r2, .LCPI7_1@toc@ha
+; CHECK-P9-NEXT:    addi r4, r4, .LCPI7_1@toc@l
+; CHECK-P9-NEXT:    vperm v3, v2, v2, v3
+; CHECK-P9-NEXT:    vextsb2w v3, v3
+; CHECK-P9-NEXT:    xvcvsxwsp vs0, v3
+; CHECK-P9-NEXT:    lxvx v3, 0, r4
+; CHECK-P9-NEXT:    addis r4, r2, .LCPI7_2@toc@ha
+; CHECK-P9-NEXT:    addi r4, r4, .LCPI7_2@toc@l
+; CHECK-P9-NEXT:    vperm v3, v2, v2, v3
+; CHECK-P9-NEXT:    stxv vs0, 0(r3)
+; CHECK-P9-NEXT:    vextsb2w v3, v3
+; CHECK-P9-NEXT:    xvcvsxwsp vs1, v3
+; CHECK-P9-NEXT:    lxvx v3, 0, r4
+; CHECK-P9-NEXT:    addis r4, r2, .LCPI7_3@toc@ha
+; CHECK-P9-NEXT:    addi r4, r4, .LCPI7_3@toc@l
+; CHECK-P9-NEXT:    vperm v3, v2, v2, v3
+; CHECK-P9-NEXT:    stxv vs1, 16(r3)
+; CHECK-P9-NEXT:    vextsb2w v3, v3
+; CHECK-P9-NEXT:    xvcvsxwsp vs2, v3
+; CHECK-P9-NEXT:    lxvx v3, 0, r4
+; CHECK-P9-NEXT:    vperm v2, v2, v2, v3
+; CHECK-P9-NEXT:    stxv vs2, 32(r3)
+; CHECK-P9-NEXT:    vextsb2w v2, v2
+; CHECK-P9-NEXT:    xvcvsxwsp vs3, v2
+; CHECK-P9-NEXT:    stxv vs3, 48(r3)
 ; CHECK-P9-NEXT:    blr
 ;
 ; CHECK-BE-LABEL: test16elt_signed:
 ; CHECK-BE:       # %bb.0: # %entry
-; CHECK-BE-NEXT:    std r25, -72(r1) # 8-byte Folded Spill
-; CHECK-BE-NEXT:    std r26, -64(r1) # 8-byte Folded Spill
-; CHECK-BE-NEXT:    std r27, -56(r1) # 8-byte Folded Spill
-; CHECK-BE-NEXT:    std r28, -48(r1) # 8-byte Folded Spill
-; CHECK-BE-NEXT:    std r29, -40(r1) # 8-byte Folded Spill
-; CHECK-BE-NEXT:    li r4, 3
-; CHECK-BE-NEXT:    std r30, -32(r1) # 8-byte Folded Spill
-; CHECK-BE-NEXT:    li r5, 1
-; CHECK-BE-NEXT:    li r6, 2
-; CHECK-BE-NEXT:    li r7, 0
-; CHECK-BE-NEXT:    li r8, 7
-; CHECK-BE-NEXT:    li r9, 5
-; CHECK-BE-NEXT:    li r10, 6
-; CHECK-BE-NEXT:    li r11, 4
-; CHECK-BE-NEXT:    li r12, 11
-; CHECK-BE-NEXT:    li r0, 9
-; CHECK-BE-NEXT:    li r30, 10
-; CHECK-BE-NEXT:    li r29, 8
-; CHECK-BE-NEXT:    li r28, 15
-; CHECK-BE-NEXT:    li r27, 13
-; CHECK-BE-NEXT:    li r26, 14
-; CHECK-BE-NEXT:    li r25, 12
-; CHECK-BE-NEXT:    stfd f30, -16(r1) # 8-byte Folded Spill
-; CHECK-BE-NEXT:    stfd f31, -8(r1) # 8-byte Folded Spill
-; CHECK-BE-NEXT:    vextublx r4, r4, v2
-; CHECK-BE-NEXT:    vextublx r5, r5, v2
-; CHECK-BE-NEXT:    vextublx r6, r6, v2
-; CHECK-BE-NEXT:    vextublx r7, r7, v2
-; CHECK-BE-NEXT:    vextublx r8, r8, v2
-; CHECK-BE-NEXT:    vextublx r9, r9, v2
-; CHECK-BE-NEXT:    vextublx r10, r10, v2
-; CHECK-BE-NEXT:    vextublx r11, r11, v2
-; CHECK-BE-NEXT:    vextublx r12, r12, v2
-; CHECK-BE-NEXT:    vextublx r0, r0, v2
-; CHECK-BE-NEXT:    vextublx r30, r30, v2
-; CHECK-BE-NEXT:    vextublx r29, r29, v2
-; CHECK-BE-NEXT:    vextublx r28, r28, v2
-; CHECK-BE-NEXT:    vextublx r27, r27, v2
-; CHECK-BE-NEXT:    vextublx r26, r26, v2
-; CHECK-BE-NEXT:    vextublx r25, r25, v2
-; CHECK-BE-NEXT:    extsb r4, r4
-; CHECK-BE-NEXT:    extsb r5, r5
-; CHECK-BE-NEXT:    extsb r6, r6
-; CHECK-BE-NEXT:    extsb r7, r7
-; CHECK-BE-NEXT:    extsb r8, r8
-; CHECK-BE-NEXT:    extsb r9, r9
-; CHECK-BE-NEXT:    extsb r10, r10
-; CHECK-BE-NEXT:    extsb r11, r11
-; CHECK-BE-NEXT:    extsb r12, r12
-; CHECK-BE-NEXT:    extsb r0, r0
-; CHECK-BE-NEXT:    extsb r30, r30
-; CHECK-BE-NEXT:    extsb r29, r29
-; CHECK-BE-NEXT:    extsb r28, r28
-; CHECK-BE-NEXT:    extsb r27, r27
-; CHECK-BE-NEXT:    extsb r26, r26
-; CHECK-BE-NEXT:    extsb r25, r25
-; CHECK-BE-NEXT:    mtvsrwa f0, r4
-; CHECK-BE-NEXT:    mtvsrwa f1, r5
-; CHECK-BE-NEXT:    mtvsrwa f2, r6
-; CHECK-BE-NEXT:    mtvsrwa f3, r7
-; CHECK-BE-NEXT:    mtvsrwa f4, r8
-; CHECK-BE-NEXT:    mtvsrwa f5, r9
-; CHECK-BE-NEXT:    mtvsrwa f6, r10
-; CHECK-BE-NEXT:    mtvsrwa f7, r11
-; CHECK-BE-NEXT:    mtvsrwa f8, r12
-; CHECK-BE-NEXT:    mtvsrwa f9, r0
-; CHECK-BE-NEXT:    mtvsrwa f10, r30
-; CHECK-BE-NEXT:    ld r30, -32(r1) # 8-byte Folded Reload
-; CHECK-BE-NEXT:    mtvsrwa f11, r29
-; CHECK-BE-NEXT:    ld r29, -40(r1) # 8-byte Folded Reload
-; CHECK-BE-NEXT:    mtvsrwa f12, r28
-; CHECK-BE-NEXT:    ld r28, -48(r1) # 8-byte Folded Reload
-; CHECK-BE-NEXT:    mtvsrwa f13, r27
-; CHECK-BE-NEXT:    ld r27, -56(r1) # 8-byte Folded Reload
-; CHECK-BE-NEXT:    mtvsrwa v2, r26
-; CHECK-BE-NEXT:    ld r26, -64(r1) # 8-byte Folded Reload
-; CHECK-BE-NEXT:    mtvsrwa v3, r25
-; CHECK-BE-NEXT:    ld r25, -72(r1) # 8-byte Folded Reload
-; CHECK-BE-NEXT:    xscvsxdsp f0, f0
-; CHECK-BE-NEXT:    xscvsxdsp f1, f1
-; CHECK-BE-NEXT:    xscvsxdsp f2, f2
-; CHECK-BE-NEXT:    xscvsxdsp f3, f3
-; CHECK-BE-NEXT:    xscvsxdsp f4, f4
-; CHECK-BE-NEXT:    xscvsxdsp f5, f5
-; CHECK-BE-NEXT:    xscvsxdsp f6, f6
-; CHECK-BE-NEXT:    xscvsxdsp f7, f7
-; CHECK-BE-NEXT:    xscvsxdsp f8, f8
-; CHECK-BE-NEXT:    xscvsxdsp f9, f9
-; CHECK-BE-NEXT:    xscvsxdsp f10, f10
-; CHECK-BE-NEXT:    xscvsxdsp f11, f11
-; CHECK-BE-NEXT:    xscvsxdsp f12, f12
-; CHECK-BE-NEXT:    xscvsxdsp f13, f13
-; CHECK-BE-NEXT:    xscvsxdsp f31, v2
-; CHECK-BE-NEXT:    xscvsxdsp f30, v3
-; CHECK-BE-NEXT:    xxmrghd vs0, vs1, vs0
-; CHECK-BE-NEXT:    xxmrghd vs1, vs3, vs2
-; CHECK-BE-NEXT:    xxmrghd vs2, vs5, vs4
-; CHECK-BE-NEXT:    xxmrghd vs3, vs7, vs6
-; CHECK-BE-NEXT:    xxmrghd vs4, vs9, vs8
-; CHECK-BE-NEXT:    xxmrghd vs5, vs11, vs10
-; CHECK-BE-NEXT:    xxmrghd vs6, vs13, vs12
-; CHECK-BE-NEXT:    xxmrghd vs7, vs30, vs31
-; CHECK-BE-NEXT:    lfd f31, -8(r1) # 8-byte Folded Reload
-; CHECK-BE-NEXT:    lfd f30, -16(r1) # 8-byte Folded Reload
-; CHECK-BE-NEXT:    xvcvdpsp v2, vs0
-; CHECK-BE-NEXT:    xvcvdpsp v3, vs1
-; CHECK-BE-NEXT:    xvcvdpsp v4, vs2
-; CHECK-BE-NEXT:    xvcvdpsp v5, vs3
-; CHECK-BE-NEXT:    xvcvdpsp v0, vs4
-; CHECK-BE-NEXT:    xvcvdpsp v1, vs5
-; CHECK-BE-NEXT:    xvcvdpsp v6, vs6
-; CHECK-BE-NEXT:    xvcvdpsp v7, vs7
-; CHECK-BE-NEXT:    vmrgew v2, v3, v2
-; CHECK-BE-NEXT:    vmrgew v3, v5, v4
-; CHECK-BE-NEXT:    vmrgew v4, v1, v0
-; CHECK-BE-NEXT:    vmrgew v5, v7, v6
-; CHECK-BE-NEXT:    stxv v3, 16(r3)
-; CHECK-BE-NEXT:    stxv v2, 0(r3)
-; CHECK-BE-NEXT:    stxv v5, 48(r3)
-; CHECK-BE-NEXT:    stxv v4, 32(r3)
+; CHECK-BE-NEXT:    addis r4, r2, .LCPI7_0@toc@ha
+; CHECK-BE-NEXT:    addi r4, r4, .LCPI7_0@toc@l
+; CHECK-BE-NEXT:    lxvx v3, 0, r4
+; CHECK-BE-NEXT:    xxlxor v4, v4, v4
+; CHECK-BE-NEXT:    vperm v3, v4, v2, v3
+; CHECK-BE-NEXT:    addis r4, r2, .LCPI7_1@toc@ha
+; CHECK-BE-NEXT:    addi r4, r4, .LCPI7_1@toc@l
+; CHECK-BE-NEXT:    vextsb2w v3, v3
+; CHECK-BE-NEXT:    xvcvsxwsp vs0, v3
+; CHECK-BE-NEXT:    lxvx v3, 0, r4
+; CHECK-BE-NEXT:    addis r4, r2, .LCPI7_2@toc@ha
+; CHECK-BE-NEXT:    addi r4, r4, .LCPI7_2@toc@l
+; CHECK-BE-NEXT:    vperm v3, v4, v2, v3
+; CHECK-BE-NEXT:    stxv vs0, 16(r3)
+; CHECK-BE-NEXT:    vextsb2w v3, v3
+; CHECK-BE-NEXT:    xvcvsxwsp vs1, v3
+; CHECK-BE-NEXT:    lxvx v3, 0, r4
+; CHECK-BE-NEXT:    addis r4, r2, .LCPI7_3@toc@ha
+; CHECK-BE-NEXT:    addi r4, r4, .LCPI7_3@toc@l
+; CHECK-BE-NEXT:    vperm v3, v2, v2, v3
+; CHECK-BE-NEXT:    stxv vs1, 48(r3)
+; CHECK-BE-NEXT:    vextsb2w v3, v3
+; CHECK-BE-NEXT:    xvcvsxwsp vs2, v3
+; CHECK-BE-NEXT:    lxvx v3, 0, r4
+; CHECK-BE-NEXT:    vperm v2, v2, v2, v3
+; CHECK-BE-NEXT:    stxv vs2, 0(r3)
+; CHECK-BE-NEXT:    vextsb2w v2, v2
+; CHECK-BE-NEXT:    xvcvsxwsp vs3, v2
+; CHECK-BE-NEXT:    stxv vs3, 32(r3)
 ; CHECK-BE-NEXT:    blr
 entry:
   %0 = sitofp <16 x i8> %a to <16 x float>
diff --git a/test/CodeGen/PowerPC/vec_conv_i8_to_fp64_elts.ll b/test/CodeGen/PowerPC/vec_conv_i8_to_fp64_elts.ll
index bafc18bece2140fdd882256c8117d23d94f9d499..1a9790106db77cbc0e5e1b09a2561df2074acf08 100644
--- a/test/CodeGen/PowerPC/vec_conv_i8_to_fp64_elts.ll
+++ b/test/CodeGen/PowerPC/vec_conv_i8_to_fp64_elts.ll
@@ -12,49 +12,36 @@
 define <2 x double> @test2elt(i16 %a.coerce) local_unnamed_addr #0 {
 ; CHECK-P8-LABEL: test2elt:
 ; CHECK-P8:       # %bb.0: # %entry
+; CHECK-P8-NEXT:    addis r4, r2, .LCPI0_0@toc@ha
 ; CHECK-P8-NEXT:    mtvsrd f0, r3
-; CHECK-P8-NEXT:    mfvsrd r3, f0
-; CHECK-P8-NEXT:    clrldi r4, r3, 56
-; CHECK-P8-NEXT:    rldicl r3, r3, 56, 56
-; CHECK-P8-NEXT:    rlwinm r4, r4, 0, 24, 31
-; CHECK-P8-NEXT:    rlwinm r3, r3, 0, 24, 31
-; CHECK-P8-NEXT:    mtvsrwz f0, r4
-; CHECK-P8-NEXT:    mtvsrwz f1, r3
-; CHECK-P8-NEXT:    xscvuxddp f0, f0
-; CHECK-P8-NEXT:    xscvuxddp f1, f1
-; CHECK-P8-NEXT:    xxmrghd v2, vs1, vs0
+; CHECK-P8-NEXT:    addi r3, r4, .LCPI0_0@toc@l
+; CHECK-P8-NEXT:    xxlxor v4, v4, v4
+; CHECK-P8-NEXT:    xxswapd v2, vs0
+; CHECK-P8-NEXT:    lvx v3, 0, r3
+; CHECK-P8-NEXT:    vperm v2, v4, v2, v3
+; CHECK-P8-NEXT:    xvcvuxddp v2, v2
 ; CHECK-P8-NEXT:    blr
 ;
 ; CHECK-P9-LABEL: test2elt:
 ; CHECK-P9:       # %bb.0: # %entry
 ; CHECK-P9-NEXT:    mtvsrws v2, r3
-; CHECK-P9-NEXT:    li r3, 0
-; CHECK-P9-NEXT:    li r4, 1
-; CHECK-P9-NEXT:    vextubrx r3, r3, v2
-; CHECK-P9-NEXT:    vextubrx r4, r4, v2
-; CHECK-P9-NEXT:    rlwinm r3, r3, 0, 24, 31
-; CHECK-P9-NEXT:    rlwinm r4, r4, 0, 24, 31
-; CHECK-P9-NEXT:    mtvsrwz f0, r3
-; CHECK-P9-NEXT:    mtvsrwz f1, r4
-; CHECK-P9-NEXT:    xscvuxddp f0, f0
-; CHECK-P9-NEXT:    xscvuxddp f1, f1
-; CHECK-P9-NEXT:    xxmrghd v2, vs1, vs0
+; CHECK-P9-NEXT:    addis r3, r2, .LCPI0_0@toc@ha
+; CHECK-P9-NEXT:    addi r3, r3, .LCPI0_0@toc@l
+; CHECK-P9-NEXT:    lxvx v3, 0, r3
+; CHECK-P9-NEXT:    xxlxor v4, v4, v4
+; CHECK-P9-NEXT:    vperm v2, v4, v2, v3
+; CHECK-P9-NEXT:    xvcvuxddp v2, v2
 ; CHECK-P9-NEXT:    blr
 ;
 ; CHECK-BE-LABEL: test2elt:
 ; CHECK-BE:       # %bb.0: # %entry
 ; CHECK-BE-NEXT:    mtvsrws v2, r3
-; CHECK-BE-NEXT:    li r3, 1
-; CHECK-BE-NEXT:    li r4, 0
-; CHECK-BE-NEXT:    vextublx r3, r3, v2
-; CHECK-BE-NEXT:    vextublx r4, r4, v2
-; CHECK-BE-NEXT:    rlwinm r3, r3, 0, 24, 31
-; CHECK-BE-NEXT:    rlwinm r4, r4, 0, 24, 31
-; CHECK-BE-NEXT:    mtvsrwz f0, r3
-; CHECK-BE-NEXT:    mtvsrwz f1, r4
-; CHECK-BE-NEXT:    xscvuxddp f0, f0
-; CHECK-BE-NEXT:    xscvuxddp f1, f1
-; CHECK-BE-NEXT:    xxmrghd v2, vs1, vs0
+; CHECK-BE-NEXT:    addis r3, r2, .LCPI0_0@toc@ha
+; CHECK-BE-NEXT:    addi r3, r3, .LCPI0_0@toc@l
+; CHECK-BE-NEXT:    lxvx v3, 0, r3
+; CHECK-BE-NEXT:    xxlxor v4, v4, v4
+; CHECK-BE-NEXT:    vperm v2, v2, v4, v3
+; CHECK-BE-NEXT:    xvcvuxddp v2, v2
 ; CHECK-BE-NEXT:    blr
 entry:
   %0 = bitcast i16 %a.coerce to <2 x i8>
@@ -65,27 +52,20 @@ entry:
 define void @test4elt(<4 x double>* noalias nocapture sret %agg.result, i32 %a.coerce) local_unnamed_addr #1 {
 ; CHECK-P8-LABEL: test4elt:
 ; CHECK-P8:       # %bb.0: # %entry
+; CHECK-P8-NEXT:    addis r5, r2, .LCPI1_0@toc@ha
 ; CHECK-P8-NEXT:    mtvsrd f0, r4
-; CHECK-P8-NEXT:    mfvsrd r4, f0
-; CHECK-P8-NEXT:    clrldi r5, r4, 56
-; CHECK-P8-NEXT:    rldicl r6, r4, 56, 56
-; CHECK-P8-NEXT:    rlwinm r5, r5, 0, 24, 31
-; CHECK-P8-NEXT:    rlwinm r6, r6, 0, 24, 31
-; CHECK-P8-NEXT:    mtvsrwz f0, r5
-; CHECK-P8-NEXT:    rldicl r5, r4, 48, 56
-; CHECK-P8-NEXT:    rldicl r4, r4, 40, 56
-; CHECK-P8-NEXT:    rlwinm r5, r5, 0, 24, 31
-; CHECK-P8-NEXT:    rlwinm r4, r4, 0, 24, 31
-; CHECK-P8-NEXT:    mtvsrwz f1, r6
-; CHECK-P8-NEXT:    mtvsrwz f2, r5
-; CHECK-P8-NEXT:    mtvsrwz f3, r4
+; CHECK-P8-NEXT:    addis r4, r2, .LCPI1_1@toc@ha
+; CHECK-P8-NEXT:    addi r5, r5, .LCPI1_0@toc@l
+; CHECK-P8-NEXT:    addi r4, r4, .LCPI1_1@toc@l
+; CHECK-P8-NEXT:    xxlxor v4, v4, v4
+; CHECK-P8-NEXT:    lvx v2, 0, r5
+; CHECK-P8-NEXT:    xxswapd v3, vs0
+; CHECK-P8-NEXT:    lvx v5, 0, r4
 ; CHECK-P8-NEXT:    li r4, 16
-; CHECK-P8-NEXT:    xscvuxddp f0, f0
-; CHECK-P8-NEXT:    xscvuxddp f1, f1
-; CHECK-P8-NEXT:    xscvuxddp f2, f2
-; CHECK-P8-NEXT:    xscvuxddp f3, f3
-; CHECK-P8-NEXT:    xxmrghd vs0, vs1, vs0
-; CHECK-P8-NEXT:    xxmrghd vs1, vs3, vs2
+; CHECK-P8-NEXT:    vperm v2, v4, v3, v2
+; CHECK-P8-NEXT:    vperm v3, v4, v3, v5
+; CHECK-P8-NEXT:    xvcvuxddp vs0, v2
+; CHECK-P8-NEXT:    xvcvuxddp vs1, v3
 ; CHECK-P8-NEXT:    xxswapd vs0, vs0
 ; CHECK-P8-NEXT:    xxswapd vs1, vs1
 ; CHECK-P8-NEXT:    stxvd2x vs1, r3, r4
@@ -95,59 +75,37 @@ define void @test4elt(<4 x double>* noalias nocapture sret %agg.result, i32 %a.c
 ; CHECK-P9-LABEL: test4elt:
 ; CHECK-P9:       # %bb.0: # %entry
 ; CHECK-P9-NEXT:    mtvsrws v2, r4
-; CHECK-P9-NEXT:    li r4, 0
-; CHECK-P9-NEXT:    li r5, 1
-; CHECK-P9-NEXT:    li r6, 2
-; CHECK-P9-NEXT:    li r7, 3
-; CHECK-P9-NEXT:    vextubrx r4, r4, v2
-; CHECK-P9-NEXT:    vextubrx r5, r5, v2
-; CHECK-P9-NEXT:    vextubrx r6, r6, v2
-; CHECK-P9-NEXT:    vextubrx r7, r7, v2
-; CHECK-P9-NEXT:    rlwinm r4, r4, 0, 24, 31
-; CHECK-P9-NEXT:    rlwinm r5, r5, 0, 24, 31
-; CHECK-P9-NEXT:    rlwinm r6, r6, 0, 24, 31
-; CHECK-P9-NEXT:    rlwinm r7, r7, 0, 24, 31
-; CHECK-P9-NEXT:    mtvsrwz f0, r4
-; CHECK-P9-NEXT:    mtvsrwz f1, r5
-; CHECK-P9-NEXT:    mtvsrwz f2, r6
-; CHECK-P9-NEXT:    mtvsrwz f3, r7
-; CHECK-P9-NEXT:    xscvuxddp f0, f0
-; CHECK-P9-NEXT:    xscvuxddp f1, f1
-; CHECK-P9-NEXT:    xscvuxddp f2, f2
-; CHECK-P9-NEXT:    xscvuxddp f3, f3
-; CHECK-P9-NEXT:    xxmrghd vs0, vs1, vs0
-; CHECK-P9-NEXT:    xxmrghd vs1, vs3, vs2
-; CHECK-P9-NEXT:    stxv vs1, 16(r3)
+; CHECK-P9-NEXT:    addis r4, r2, .LCPI1_0@toc@ha
+; CHECK-P9-NEXT:    addi r4, r4, .LCPI1_0@toc@l
+; CHECK-P9-NEXT:    lxvx v3, 0, r4
+; CHECK-P9-NEXT:    xxlxor v4, v4, v4
+; CHECK-P9-NEXT:    addis r4, r2, .LCPI1_1@toc@ha
+; CHECK-P9-NEXT:    addi r4, r4, .LCPI1_1@toc@l
+; CHECK-P9-NEXT:    vperm v3, v4, v2, v3
+; CHECK-P9-NEXT:    xvcvuxddp vs0, v3
+; CHECK-P9-NEXT:    lxvx v3, 0, r4
+; CHECK-P9-NEXT:    vperm v2, v4, v2, v3
 ; CHECK-P9-NEXT:    stxv vs0, 0(r3)
+; CHECK-P9-NEXT:    xvcvuxddp vs1, v2
+; CHECK-P9-NEXT:    stxv vs1, 16(r3)
 ; CHECK-P9-NEXT:    blr
 ;
 ; CHECK-BE-LABEL: test4elt:
 ; CHECK-BE:       # %bb.0: # %entry
 ; CHECK-BE-NEXT:    mtvsrws v2, r4
-; CHECK-BE-NEXT:    li r4, 1
-; CHECK-BE-NEXT:    li r5, 0
-; CHECK-BE-NEXT:    li r6, 3
-; CHECK-BE-NEXT:    li r7, 2
-; CHECK-BE-NEXT:    vextublx r4, r4, v2
-; CHECK-BE-NEXT:    vextublx r5, r5, v2
-; CHECK-BE-NEXT:    vextublx r6, r6, v2
-; CHECK-BE-NEXT:    vextublx r7, r7, v2
-; CHECK-BE-NEXT:    rlwinm r4, r4, 0, 24, 31
-; CHECK-BE-NEXT:    rlwinm r5, r5, 0, 24, 31
-; CHECK-BE-NEXT:    rlwinm r6, r6, 0, 24, 31
-; CHECK-BE-NEXT:    rlwinm r7, r7, 0, 24, 31
-; CHECK-BE-NEXT:    mtvsrwz f0, r4
-; CHECK-BE-NEXT:    mtvsrwz f1, r5
-; CHECK-BE-NEXT:    mtvsrwz f2, r6
-; CHECK-BE-NEXT:    mtvsrwz f3, r7
-; CHECK-BE-NEXT:    xscvuxddp f0, f0
-; CHECK-BE-NEXT:    xscvuxddp f1, f1
-; CHECK-BE-NEXT:    xscvuxddp f2, f2
-; CHECK-BE-NEXT:    xscvuxddp f3, f3
-; CHECK-BE-NEXT:    xxmrghd vs0, vs1, vs0
-; CHECK-BE-NEXT:    xxmrghd vs1, vs3, vs2
-; CHECK-BE-NEXT:    stxv vs1, 16(r3)
+; CHECK-BE-NEXT:    addis r4, r2, .LCPI1_0@toc@ha
+; CHECK-BE-NEXT:    addi r4, r4, .LCPI1_0@toc@l
+; CHECK-BE-NEXT:    lxvx v3, 0, r4
+; CHECK-BE-NEXT:    xxlxor v4, v4, v4
+; CHECK-BE-NEXT:    addis r4, r2, .LCPI1_1@toc@ha
+; CHECK-BE-NEXT:    addi r4, r4, .LCPI1_1@toc@l
+; CHECK-BE-NEXT:    vperm v3, v2, v4, v3
+; CHECK-BE-NEXT:    xvcvuxddp vs0, v3
+; CHECK-BE-NEXT:    lxvx v3, 0, r4
+; CHECK-BE-NEXT:    vperm v2, v4, v2, v3
 ; CHECK-BE-NEXT:    stxv vs0, 0(r3)
+; CHECK-BE-NEXT:    xvcvuxddp vs1, v2
+; CHECK-BE-NEXT:    stxv vs1, 16(r3)
 ; CHECK-BE-NEXT:    blr
 entry:
   %0 = bitcast i32 %a.coerce to <4 x i8>
@@ -159,161 +117,100 @@ entry:
 define void @test8elt(<8 x double>* noalias nocapture sret %agg.result, i64 %a.coerce) local_unnamed_addr #1 {
 ; CHECK-P8-LABEL: test8elt:
 ; CHECK-P8:       # %bb.0: # %entry
+; CHECK-P8-NEXT:    addis r5, r2, .LCPI2_0@toc@ha
 ; CHECK-P8-NEXT:    mtvsrd f0, r4
-; CHECK-P8-NEXT:    mfvsrd r4, f0
-; CHECK-P8-NEXT:    clrldi r5, r4, 56
-; CHECK-P8-NEXT:    rldicl r6, r4, 56, 56
-; CHECK-P8-NEXT:    rlwinm r5, r5, 0, 24, 31
-; CHECK-P8-NEXT:    rlwinm r6, r6, 0, 24, 31
-; CHECK-P8-NEXT:    mtvsrwz f0, r5
-; CHECK-P8-NEXT:    rldicl r5, r4, 48, 56
-; CHECK-P8-NEXT:    rlwinm r5, r5, 0, 24, 31
-; CHECK-P8-NEXT:    mtvsrwz f1, r6
-; CHECK-P8-NEXT:    rldicl r6, r4, 40, 56
-; CHECK-P8-NEXT:    mtvsrwz f2, r5
-; CHECK-P8-NEXT:    rldicl r5, r4, 32, 56
-; CHECK-P8-NEXT:    rlwinm r6, r6, 0, 24, 31
-; CHECK-P8-NEXT:    rlwinm r5, r5, 0, 24, 31
-; CHECK-P8-NEXT:    mtvsrwz f3, r6
-; CHECK-P8-NEXT:    rldicl r6, r4, 24, 56
-; CHECK-P8-NEXT:    mtvsrwz f4, r5
-; CHECK-P8-NEXT:    rldicl r5, r4, 16, 56
-; CHECK-P8-NEXT:    rldicl r4, r4, 8, 56
-; CHECK-P8-NEXT:    rlwinm r6, r6, 0, 24, 31
-; CHECK-P8-NEXT:    rlwinm r5, r5, 0, 24, 31
-; CHECK-P8-NEXT:    rlwinm r4, r4, 0, 24, 31
-; CHECK-P8-NEXT:    mtvsrwz f5, r6
-; CHECK-P8-NEXT:    mtvsrwz f6, r5
-; CHECK-P8-NEXT:    li r5, 32
-; CHECK-P8-NEXT:    mtvsrwz f7, r4
+; CHECK-P8-NEXT:    addis r4, r2, .LCPI2_2@toc@ha
+; CHECK-P8-NEXT:    addi r5, r5, .LCPI2_0@toc@l
+; CHECK-P8-NEXT:    addi r4, r4, .LCPI2_2@toc@l
+; CHECK-P8-NEXT:    xxlxor v4, v4, v4
+; CHECK-P8-NEXT:    lvx v2, 0, r5
+; CHECK-P8-NEXT:    addis r5, r2, .LCPI2_3@toc@ha
+; CHECK-P8-NEXT:    lvx v5, 0, r4
+; CHECK-P8-NEXT:    addis r4, r2, .LCPI2_1@toc@ha
+; CHECK-P8-NEXT:    xxswapd v3, vs0
+; CHECK-P8-NEXT:    addi r5, r5, .LCPI2_3@toc@l
+; CHECK-P8-NEXT:    addi r4, r4, .LCPI2_1@toc@l
+; CHECK-P8-NEXT:    lvx v0, 0, r5
+; CHECK-P8-NEXT:    lvx v1, 0, r4
 ; CHECK-P8-NEXT:    li r4, 48
-; CHECK-P8-NEXT:    xscvuxddp f4, f4
-; CHECK-P8-NEXT:    xscvuxddp f5, f5
-; CHECK-P8-NEXT:    xscvuxddp f6, f6
-; CHECK-P8-NEXT:    xscvuxddp f7, f7
-; CHECK-P8-NEXT:    xscvuxddp f0, f0
-; CHECK-P8-NEXT:    xscvuxddp f1, f1
-; CHECK-P8-NEXT:    xscvuxddp f2, f2
-; CHECK-P8-NEXT:    xscvuxddp f3, f3
-; CHECK-P8-NEXT:    xxmrghd vs4, vs5, vs4
-; CHECK-P8-NEXT:    xxmrghd vs5, vs7, vs6
-; CHECK-P8-NEXT:    xxmrghd vs0, vs1, vs0
-; CHECK-P8-NEXT:    xxmrghd vs1, vs3, vs2
-; CHECK-P8-NEXT:    xxswapd vs2, vs5
-; CHECK-P8-NEXT:    xxswapd vs3, vs4
+; CHECK-P8-NEXT:    li r5, 32
+; CHECK-P8-NEXT:    vperm v2, v4, v3, v2
+; CHECK-P8-NEXT:    vperm v5, v4, v3, v5
+; CHECK-P8-NEXT:    vperm v0, v4, v3, v0
+; CHECK-P8-NEXT:    vperm v3, v4, v3, v1
+; CHECK-P8-NEXT:    xvcvuxddp vs0, v2
+; CHECK-P8-NEXT:    xvcvuxddp vs1, v5
+; CHECK-P8-NEXT:    xvcvuxddp vs2, v0
+; CHECK-P8-NEXT:    xvcvuxddp vs3, v3
 ; CHECK-P8-NEXT:    xxswapd vs0, vs0
 ; CHECK-P8-NEXT:    xxswapd vs1, vs1
+; CHECK-P8-NEXT:    xxswapd vs2, vs2
+; CHECK-P8-NEXT:    xxswapd vs3, vs3
 ; CHECK-P8-NEXT:    stxvd2x vs2, r3, r4
 ; CHECK-P8-NEXT:    li r4, 16
-; CHECK-P8-NEXT:    stxvd2x vs3, r3, r5
-; CHECK-P8-NEXT:    stxvd2x vs1, r3, r4
+; CHECK-P8-NEXT:    stxvd2x vs1, r3, r5
+; CHECK-P8-NEXT:    stxvd2x vs3, r3, r4
 ; CHECK-P8-NEXT:    stxvd2x vs0, 0, r3
 ; CHECK-P8-NEXT:    blr
 ;
 ; CHECK-P9-LABEL: test8elt:
 ; CHECK-P9:       # %bb.0: # %entry
 ; CHECK-P9-NEXT:    mtvsrd f0, r4
-; CHECK-P9-NEXT:    li r4, 0
-; CHECK-P9-NEXT:    li r5, 1
-; CHECK-P9-NEXT:    li r6, 2
-; CHECK-P9-NEXT:    li r7, 3
-; CHECK-P9-NEXT:    li r8, 4
-; CHECK-P9-NEXT:    li r9, 5
-; CHECK-P9-NEXT:    li r10, 6
-; CHECK-P9-NEXT:    li r11, 7
+; CHECK-P9-NEXT:    addis r4, r2, .LCPI2_0@toc@ha
+; CHECK-P9-NEXT:    addi r4, r4, .LCPI2_0@toc@l
+; CHECK-P9-NEXT:    lxvx v3, 0, r4
 ; CHECK-P9-NEXT:    xxswapd v2, vs0
-; CHECK-P9-NEXT:    vextubrx r4, r4, v2
-; CHECK-P9-NEXT:    vextubrx r5, r5, v2
-; CHECK-P9-NEXT:    vextubrx r6, r6, v2
-; CHECK-P9-NEXT:    vextubrx r7, r7, v2
-; CHECK-P9-NEXT:    vextubrx r8, r8, v2
-; CHECK-P9-NEXT:    vextubrx r9, r9, v2
-; CHECK-P9-NEXT:    vextubrx r10, r10, v2
-; CHECK-P9-NEXT:    vextubrx r11, r11, v2
-; CHECK-P9-NEXT:    rlwinm r4, r4, 0, 24, 31
-; CHECK-P9-NEXT:    rlwinm r5, r5, 0, 24, 31
-; CHECK-P9-NEXT:    rlwinm r6, r6, 0, 24, 31
-; CHECK-P9-NEXT:    rlwinm r7, r7, 0, 24, 31
-; CHECK-P9-NEXT:    rlwinm r8, r8, 0, 24, 31
-; CHECK-P9-NEXT:    rlwinm r9, r9, 0, 24, 31
-; CHECK-P9-NEXT:    rlwinm r10, r10, 0, 24, 31
-; CHECK-P9-NEXT:    rlwinm r11, r11, 0, 24, 31
-; CHECK-P9-NEXT:    mtvsrwz f0, r4
-; CHECK-P9-NEXT:    mtvsrwz f1, r5
-; CHECK-P9-NEXT:    mtvsrwz f2, r6
-; CHECK-P9-NEXT:    mtvsrwz f3, r7
-; CHECK-P9-NEXT:    mtvsrwz f4, r8
-; CHECK-P9-NEXT:    mtvsrwz f5, r9
-; CHECK-P9-NEXT:    mtvsrwz f6, r10
-; CHECK-P9-NEXT:    mtvsrwz f7, r11
-; CHECK-P9-NEXT:    xscvuxddp f0, f0
-; CHECK-P9-NEXT:    xscvuxddp f1, f1
-; CHECK-P9-NEXT:    xscvuxddp f2, f2
-; CHECK-P9-NEXT:    xscvuxddp f3, f3
-; CHECK-P9-NEXT:    xscvuxddp f4, f4
-; CHECK-P9-NEXT:    xscvuxddp f5, f5
-; CHECK-P9-NEXT:    xscvuxddp f6, f6
-; CHECK-P9-NEXT:    xscvuxddp f7, f7
-; CHECK-P9-NEXT:    xxmrghd vs0, vs1, vs0
-; CHECK-P9-NEXT:    xxmrghd vs1, vs3, vs2
-; CHECK-P9-NEXT:    xxmrghd vs2, vs5, vs4
-; CHECK-P9-NEXT:    xxmrghd vs3, vs7, vs6
-; CHECK-P9-NEXT:    stxv vs3, 48(r3)
-; CHECK-P9-NEXT:    stxv vs2, 32(r3)
-; CHECK-P9-NEXT:    stxv vs1, 16(r3)
+; CHECK-P9-NEXT:    xxlxor v4, v4, v4
+; CHECK-P9-NEXT:    addis r4, r2, .LCPI2_1@toc@ha
+; CHECK-P9-NEXT:    addi r4, r4, .LCPI2_1@toc@l
+; CHECK-P9-NEXT:    vperm v3, v4, v2, v3
+; CHECK-P9-NEXT:    xvcvuxddp vs0, v3
+; CHECK-P9-NEXT:    lxvx v3, 0, r4
+; CHECK-P9-NEXT:    addis r4, r2, .LCPI2_2@toc@ha
+; CHECK-P9-NEXT:    addi r4, r4, .LCPI2_2@toc@l
+; CHECK-P9-NEXT:    vperm v3, v4, v2, v3
 ; CHECK-P9-NEXT:    stxv vs0, 0(r3)
+; CHECK-P9-NEXT:    xvcvuxddp vs1, v3
+; CHECK-P9-NEXT:    lxvx v3, 0, r4
+; CHECK-P9-NEXT:    addis r4, r2, .LCPI2_3@toc@ha
+; CHECK-P9-NEXT:    addi r4, r4, .LCPI2_3@toc@l
+; CHECK-P9-NEXT:    vperm v3, v4, v2, v3
+; CHECK-P9-NEXT:    stxv vs1, 16(r3)
+; CHECK-P9-NEXT:    xvcvuxddp vs2, v3
+; CHECK-P9-NEXT:    lxvx v3, 0, r4
+; CHECK-P9-NEXT:    vperm v2, v4, v2, v3
+; CHECK-P9-NEXT:    stxv vs2, 32(r3)
+; CHECK-P9-NEXT:    xvcvuxddp vs3, v2
+; CHECK-P9-NEXT:    stxv vs3, 48(r3)
 ; CHECK-P9-NEXT:    blr
 ;
 ; CHECK-BE-LABEL: test8elt:
 ; CHECK-BE:       # %bb.0: # %entry
-; CHECK-BE-NEXT:    li r5, 1
 ; CHECK-BE-NEXT:    mtvsrd v2, r4
-; CHECK-BE-NEXT:    li r4, 0
-; CHECK-BE-NEXT:    li r6, 3
-; CHECK-BE-NEXT:    li r7, 2
-; CHECK-BE-NEXT:    li r8, 5
-; CHECK-BE-NEXT:    li r9, 4
-; CHECK-BE-NEXT:    li r10, 7
-; CHECK-BE-NEXT:    li r11, 6
-; CHECK-BE-NEXT:    vextublx r5, r5, v2
-; CHECK-BE-NEXT:    vextublx r4, r4, v2
-; CHECK-BE-NEXT:    vextublx r6, r6, v2
-; CHECK-BE-NEXT:    vextublx r7, r7, v2
-; CHECK-BE-NEXT:    vextublx r8, r8, v2
-; CHECK-BE-NEXT:    vextublx r9, r9, v2
-; CHECK-BE-NEXT:    vextublx r10, r10, v2
-; CHECK-BE-NEXT:    vextublx r11, r11, v2
-; CHECK-BE-NEXT:    rlwinm r5, r5, 0, 24, 31
-; CHECK-BE-NEXT:    rlwinm r4, r4, 0, 24, 31
-; CHECK-BE-NEXT:    rlwinm r6, r6, 0, 24, 31
-; CHECK-BE-NEXT:    rlwinm r7, r7, 0, 24, 31
-; CHECK-BE-NEXT:    rlwinm r8, r8, 0, 24, 31
-; CHECK-BE-NEXT:    rlwinm r9, r9, 0, 24, 31
-; CHECK-BE-NEXT:    rlwinm r10, r10, 0, 24, 31
-; CHECK-BE-NEXT:    rlwinm r11, r11, 0, 24, 31
-; CHECK-BE-NEXT:    mtvsrwz f0, r5
-; CHECK-BE-NEXT:    mtvsrwz f1, r4
-; CHECK-BE-NEXT:    mtvsrwz f2, r6
-; CHECK-BE-NEXT:    mtvsrwz f3, r7
-; CHECK-BE-NEXT:    mtvsrwz f4, r8
-; CHECK-BE-NEXT:    mtvsrwz f5, r9
-; CHECK-BE-NEXT:    mtvsrwz f6, r10
-; CHECK-BE-NEXT:    mtvsrwz f7, r11
-; CHECK-BE-NEXT:    xscvuxddp f0, f0
-; CHECK-BE-NEXT:    xscvuxddp f1, f1
-; CHECK-BE-NEXT:    xscvuxddp f2, f2
-; CHECK-BE-NEXT:    xscvuxddp f3, f3
-; CHECK-BE-NEXT:    xscvuxddp f4, f4
-; CHECK-BE-NEXT:    xscvuxddp f5, f5
-; CHECK-BE-NEXT:    xscvuxddp f6, f6
-; CHECK-BE-NEXT:    xscvuxddp f7, f7
-; CHECK-BE-NEXT:    xxmrghd vs0, vs1, vs0
-; CHECK-BE-NEXT:    xxmrghd vs1, vs3, vs2
-; CHECK-BE-NEXT:    xxmrghd vs2, vs5, vs4
-; CHECK-BE-NEXT:    xxmrghd vs3, vs7, vs6
-; CHECK-BE-NEXT:    stxv vs2, 32(r3)
-; CHECK-BE-NEXT:    stxv vs1, 16(r3)
+; CHECK-BE-NEXT:    addis r4, r2, .LCPI2_0@toc@ha
+; CHECK-BE-NEXT:    addi r4, r4, .LCPI2_0@toc@l
+; CHECK-BE-NEXT:    lxvx v3, 0, r4
+; CHECK-BE-NEXT:    xxlxor v4, v4, v4
+; CHECK-BE-NEXT:    addis r4, r2, .LCPI2_1@toc@ha
+; CHECK-BE-NEXT:    addi r4, r4, .LCPI2_1@toc@l
+; CHECK-BE-NEXT:    vperm v3, v2, v4, v3
+; CHECK-BE-NEXT:    xvcvuxddp vs0, v3
+; CHECK-BE-NEXT:    lxvx v3, 0, r4
+; CHECK-BE-NEXT:    addis r4, r2, .LCPI2_2@toc@ha
+; CHECK-BE-NEXT:    addi r4, r4, .LCPI2_2@toc@l
+; CHECK-BE-NEXT:    vperm v3, v4, v2, v3
 ; CHECK-BE-NEXT:    stxv vs0, 0(r3)
+; CHECK-BE-NEXT:    xvcvuxddp vs1, v3
+; CHECK-BE-NEXT:    lxvx v3, 0, r4
+; CHECK-BE-NEXT:    addis r4, r2, .LCPI2_3@toc@ha
+; CHECK-BE-NEXT:    addi r4, r4, .LCPI2_3@toc@l
+; CHECK-BE-NEXT:    vperm v3, v4, v2, v3
+; CHECK-BE-NEXT:    stxv vs1, 16(r3)
+; CHECK-BE-NEXT:    xvcvuxddp vs2, v3
+; CHECK-BE-NEXT:    lxvx v3, 0, r4
+; CHECK-BE-NEXT:    vperm v2, v4, v2, v3
+; CHECK-BE-NEXT:    stxv vs2, 32(r3)
+; CHECK-BE-NEXT:    xvcvuxddp vs3, v2
 ; CHECK-BE-NEXT:    stxv vs3, 48(r3)
 ; CHECK-BE-NEXT:    blr
 entry:
@@ -326,338 +223,176 @@ entry:
 define void @test16elt(<16 x double>* noalias nocapture sret %agg.result, <16 x i8> %a) local_unnamed_addr #2 {
 ; CHECK-P8-LABEL: test16elt:
 ; CHECK-P8:       # %bb.0: # %entry
-; CHECK-P8-NEXT:    mfvsrd r5, v2
-; CHECK-P8-NEXT:    xxswapd vs2, v2
+; CHECK-P8-NEXT:    addis r4, r2, .LCPI3_0@toc@ha
+; CHECK-P8-NEXT:    addis r5, r2, .LCPI3_1@toc@ha
+; CHECK-P8-NEXT:    xxlxor v4, v4, v4
+; CHECK-P8-NEXT:    addi r4, r4, .LCPI3_0@toc@l
+; CHECK-P8-NEXT:    addi r5, r5, .LCPI3_1@toc@l
+; CHECK-P8-NEXT:    lvx v3, 0, r4
+; CHECK-P8-NEXT:    addis r4, r2, .LCPI3_2@toc@ha
+; CHECK-P8-NEXT:    lvx v5, 0, r5
+; CHECK-P8-NEXT:    addis r5, r2, .LCPI3_4@toc@ha
+; CHECK-P8-NEXT:    addi r4, r4, .LCPI3_2@toc@l
+; CHECK-P8-NEXT:    addi r5, r5, .LCPI3_4@toc@l
+; CHECK-P8-NEXT:    lvx v0, 0, r4
+; CHECK-P8-NEXT:    addis r4, r2, .LCPI3_6@toc@ha
+; CHECK-P8-NEXT:    lvx v1, 0, r5
+; CHECK-P8-NEXT:    addis r5, r2, .LCPI3_7@toc@ha
+; CHECK-P8-NEXT:    addi r4, r4, .LCPI3_6@toc@l
+; CHECK-P8-NEXT:    addi r5, r5, .LCPI3_7@toc@l
+; CHECK-P8-NEXT:    vperm v3, v4, v2, v3
+; CHECK-P8-NEXT:    lvx v6, 0, r4
+; CHECK-P8-NEXT:    addis r4, r2, .LCPI3_5@toc@ha
+; CHECK-P8-NEXT:    lvx v7, 0, r5
+; CHECK-P8-NEXT:    addis r5, r2, .LCPI3_3@toc@ha
+; CHECK-P8-NEXT:    vperm v5, v4, v2, v5
+; CHECK-P8-NEXT:    addi r4, r4, .LCPI3_5@toc@l
+; CHECK-P8-NEXT:    addi r5, r5, .LCPI3_3@toc@l
+; CHECK-P8-NEXT:    vperm v0, v4, v2, v0
+; CHECK-P8-NEXT:    lvx v8, 0, r4
+; CHECK-P8-NEXT:    lvx v9, 0, r5
+; CHECK-P8-NEXT:    vperm v1, v4, v2, v1
 ; CHECK-P8-NEXT:    li r4, 112
-; CHECK-P8-NEXT:    stfd f31, -8(r1) # 8-byte Folded Spill
-; CHECK-P8-NEXT:    clrldi r6, r5, 56
-; CHECK-P8-NEXT:    rldicl r7, r5, 56, 56
-; CHECK-P8-NEXT:    rlwinm r6, r6, 0, 24, 31
-; CHECK-P8-NEXT:    rlwinm r7, r7, 0, 24, 31
-; CHECK-P8-NEXT:    mtvsrwz f0, r6
-; CHECK-P8-NEXT:    rldicl r6, r5, 40, 56
-; CHECK-P8-NEXT:    rldicl r8, r5, 48, 56
-; CHECK-P8-NEXT:    mtvsrwz f1, r7
-; CHECK-P8-NEXT:    rldicl r7, r5, 32, 56
-; CHECK-P8-NEXT:    rlwinm r6, r6, 0, 24, 31
-; CHECK-P8-NEXT:    rlwinm r8, r8, 0, 24, 31
-; CHECK-P8-NEXT:    rlwinm r7, r7, 0, 24, 31
-; CHECK-P8-NEXT:    mtvsrwz f4, r6
-; CHECK-P8-NEXT:    rldicl r6, r5, 24, 56
-; CHECK-P8-NEXT:    mtvsrwz f3, r8
-; CHECK-P8-NEXT:    rlwinm r6, r6, 0, 24, 31
-; CHECK-P8-NEXT:    mtvsrwz f5, r7
-; CHECK-P8-NEXT:    rldicl r7, r5, 16, 56
-; CHECK-P8-NEXT:    rldicl r5, r5, 8, 56
-; CHECK-P8-NEXT:    mfvsrd r8, f2
-; CHECK-P8-NEXT:    rlwinm r5, r5, 0, 24, 31
-; CHECK-P8-NEXT:    mtvsrwz f2, r6
-; CHECK-P8-NEXT:    rlwinm r6, r7, 0, 24, 31
-; CHECK-P8-NEXT:    mtvsrwz f6, r6
-; CHECK-P8-NEXT:    clrldi r6, r8, 56
-; CHECK-P8-NEXT:    mtvsrwz f7, r5
-; CHECK-P8-NEXT:    rldicl r5, r8, 56, 56
-; CHECK-P8-NEXT:    rlwinm r6, r6, 0, 24, 31
-; CHECK-P8-NEXT:    mtvsrwz f8, r6
-; CHECK-P8-NEXT:    rldicl r6, r8, 48, 56
-; CHECK-P8-NEXT:    rlwinm r5, r5, 0, 24, 31
-; CHECK-P8-NEXT:    mtvsrwz f9, r5
-; CHECK-P8-NEXT:    rldicl r5, r8, 40, 56
-; CHECK-P8-NEXT:    rlwinm r6, r6, 0, 24, 31
-; CHECK-P8-NEXT:    mtvsrwz f10, r6
-; CHECK-P8-NEXT:    rldicl r6, r8, 32, 56
-; CHECK-P8-NEXT:    rlwinm r5, r5, 0, 24, 31
-; CHECK-P8-NEXT:    mtvsrwz f11, r5
-; CHECK-P8-NEXT:    rldicl r5, r8, 24, 56
-; CHECK-P8-NEXT:    rlwinm r6, r6, 0, 24, 31
-; CHECK-P8-NEXT:    mtvsrwz f12, r6
-; CHECK-P8-NEXT:    rldicl r6, r8, 16, 56
-; CHECK-P8-NEXT:    rlwinm r5, r5, 0, 24, 31
-; CHECK-P8-NEXT:    xscvuxddp f6, f6
-; CHECK-P8-NEXT:    xscvuxddp f7, f7
-; CHECK-P8-NEXT:    mtvsrwz f13, r5
-; CHECK-P8-NEXT:    rlwinm r5, r6, 0, 24, 31
-; CHECK-P8-NEXT:    mtvsrwz v2, r5
-; CHECK-P8-NEXT:    rldicl r5, r8, 8, 56
-; CHECK-P8-NEXT:    xscvuxddp f5, f5
-; CHECK-P8-NEXT:    rlwinm r5, r5, 0, 24, 31
-; CHECK-P8-NEXT:    xscvuxddp f2, f2
-; CHECK-P8-NEXT:    xscvuxddp f0, f0
-; CHECK-P8-NEXT:    xscvuxddp f1, f1
-; CHECK-P8-NEXT:    xxmrghd vs6, vs7, vs6
-; CHECK-P8-NEXT:    mtvsrwz v3, r5
-; CHECK-P8-NEXT:    li r5, 64
-; CHECK-P8-NEXT:    xscvuxddp f3, f3
-; CHECK-P8-NEXT:    xscvuxddp f4, f4
-; CHECK-P8-NEXT:    xscvuxddp f31, v2
-; CHECK-P8-NEXT:    xxmrghd vs2, vs2, vs5
-; CHECK-P8-NEXT:    xscvuxddp f7, v3
-; CHECK-P8-NEXT:    xscvuxddp f8, f8
-; CHECK-P8-NEXT:    xxmrghd vs0, vs1, vs0
-; CHECK-P8-NEXT:    xscvuxddp f9, f9
-; CHECK-P8-NEXT:    xxswapd vs1, vs6
-; CHECK-P8-NEXT:    xscvuxddp f10, f10
+; CHECK-P8-NEXT:    li r5, 96
+; CHECK-P8-NEXT:    vperm v6, v4, v2, v6
+; CHECK-P8-NEXT:    vperm v7, v4, v2, v7
+; CHECK-P8-NEXT:    vperm v8, v4, v2, v8
+; CHECK-P8-NEXT:    vperm v2, v4, v2, v9
+; CHECK-P8-NEXT:    xvcvuxddp vs0, v0
+; CHECK-P8-NEXT:    xvcvuxddp vs1, v1
+; CHECK-P8-NEXT:    xvcvuxddp vs2, v6
+; CHECK-P8-NEXT:    xvcvuxddp vs3, v7
+; CHECK-P8-NEXT:    xvcvuxddp vs4, v8
+; CHECK-P8-NEXT:    xvcvuxddp vs5, v2
+; CHECK-P8-NEXT:    xvcvuxddp vs6, v3
+; CHECK-P8-NEXT:    xxswapd vs0, vs0
+; CHECK-P8-NEXT:    xvcvuxddp vs7, v5
+; CHECK-P8-NEXT:    xxswapd vs1, vs1
 ; CHECK-P8-NEXT:    xxswapd vs2, vs2
-; CHECK-P8-NEXT:    xscvuxddp f12, f12
-; CHECK-P8-NEXT:    xxmrghd vs3, vs4, vs3
-; CHECK-P8-NEXT:    xscvuxddp f13, f13
-; CHECK-P8-NEXT:    stxvd2x vs1, r3, r4
-; CHECK-P8-NEXT:    li r4, 96
-; CHECK-P8-NEXT:    xscvuxddp f11, f11
-; CHECK-P8-NEXT:    xxmrghd vs6, vs7, vs31
 ; CHECK-P8-NEXT:    xxswapd vs3, vs3
-; CHECK-P8-NEXT:    xxswapd vs0, vs0
-; CHECK-P8-NEXT:    xxmrghd vs4, vs9, vs8
-; CHECK-P8-NEXT:    stxvd2x vs2, r3, r4
-; CHECK-P8-NEXT:    li r4, 80
-; CHECK-P8-NEXT:    xxswapd vs2, vs6
+; CHECK-P8-NEXT:    xxswapd vs4, vs4
+; CHECK-P8-NEXT:    xxswapd vs5, vs5
 ; CHECK-P8-NEXT:    stxvd2x vs3, r3, r4
+; CHECK-P8-NEXT:    stxvd2x vs2, r3, r5
+; CHECK-P8-NEXT:    li r4, 80
+; CHECK-P8-NEXT:    li r5, 64
+; CHECK-P8-NEXT:    xxswapd vs2, vs7
+; CHECK-P8-NEXT:    xxswapd vs3, vs6
+; CHECK-P8-NEXT:    stxvd2x vs4, r3, r4
 ; CHECK-P8-NEXT:    li r4, 48
-; CHECK-P8-NEXT:    stxvd2x vs0, r3, r5
+; CHECK-P8-NEXT:    stxvd2x vs1, r3, r5
 ; CHECK-P8-NEXT:    li r5, 32
-; CHECK-P8-NEXT:    xxmrghd vs5, vs13, vs12
-; CHECK-P8-NEXT:    xxswapd vs4, vs4
-; CHECK-P8-NEXT:    xxmrghd vs1, vs11, vs10
-; CHECK-P8-NEXT:    stxvd2x vs2, r3, r4
+; CHECK-P8-NEXT:    stxvd2x vs5, r3, r4
 ; CHECK-P8-NEXT:    li r4, 16
-; CHECK-P8-NEXT:    xxswapd vs5, vs5
-; CHECK-P8-NEXT:    xxswapd vs1, vs1
-; CHECK-P8-NEXT:    stxvd2x vs5, r3, r5
-; CHECK-P8-NEXT:    stxvd2x vs1, r3, r4
-; CHECK-P8-NEXT:    stxvd2x vs4, 0, r3
-; CHECK-P8-NEXT:    lfd f31, -8(r1) # 8-byte Folded Reload
+; CHECK-P8-NEXT:    stxvd2x vs0, r3, r5
+; CHECK-P8-NEXT:    stxvd2x vs2, r3, r4
+; CHECK-P8-NEXT:    stxvd2x vs3, 0, r3
 ; CHECK-P8-NEXT:    blr
 ;
 ; CHECK-P9-LABEL: test16elt:
 ; CHECK-P9:       # %bb.0: # %entry
-; CHECK-P9-NEXT:    std r25, -72(r1) # 8-byte Folded Spill
-; CHECK-P9-NEXT:    std r26, -64(r1) # 8-byte Folded Spill
-; CHECK-P9-NEXT:    std r27, -56(r1) # 8-byte Folded Spill
-; CHECK-P9-NEXT:    std r28, -48(r1) # 8-byte Folded Spill
-; CHECK-P9-NEXT:    std r29, -40(r1) # 8-byte Folded Spill
-; CHECK-P9-NEXT:    li r4, 0
-; CHECK-P9-NEXT:    std r30, -32(r1) # 8-byte Folded Spill
-; CHECK-P9-NEXT:    li r5, 1
-; CHECK-P9-NEXT:    li r6, 2
-; CHECK-P9-NEXT:    li r7, 3
-; CHECK-P9-NEXT:    li r8, 4
-; CHECK-P9-NEXT:    li r9, 5
-; CHECK-P9-NEXT:    li r10, 6
-; CHECK-P9-NEXT:    li r11, 7
-; CHECK-P9-NEXT:    li r12, 8
-; CHECK-P9-NEXT:    li r0, 9
-; CHECK-P9-NEXT:    li r30, 10
-; CHECK-P9-NEXT:    li r29, 11
-; CHECK-P9-NEXT:    li r28, 12
-; CHECK-P9-NEXT:    li r27, 13
-; CHECK-P9-NEXT:    li r26, 14
-; CHECK-P9-NEXT:    li r25, 15
-; CHECK-P9-NEXT:    stfd f30, -16(r1) # 8-byte Folded Spill
-; CHECK-P9-NEXT:    stfd f31, -8(r1) # 8-byte Folded Spill
-; CHECK-P9-NEXT:    vextubrx r4, r4, v2
-; CHECK-P9-NEXT:    vextubrx r5, r5, v2
-; CHECK-P9-NEXT:    vextubrx r6, r6, v2
-; CHECK-P9-NEXT:    vextubrx r7, r7, v2
-; CHECK-P9-NEXT:    vextubrx r8, r8, v2
-; CHECK-P9-NEXT:    vextubrx r9, r9, v2
-; CHECK-P9-NEXT:    vextubrx r10, r10, v2
-; CHECK-P9-NEXT:    vextubrx r11, r11, v2
-; CHECK-P9-NEXT:    vextubrx r12, r12, v2
-; CHECK-P9-NEXT:    vextubrx r0, r0, v2
-; CHECK-P9-NEXT:    vextubrx r30, r30, v2
-; CHECK-P9-NEXT:    vextubrx r29, r29, v2
-; CHECK-P9-NEXT:    vextubrx r28, r28, v2
-; CHECK-P9-NEXT:    vextubrx r27, r27, v2
-; CHECK-P9-NEXT:    vextubrx r26, r26, v2
-; CHECK-P9-NEXT:    vextubrx r25, r25, v2
-; CHECK-P9-NEXT:    rlwinm r4, r4, 0, 24, 31
-; CHECK-P9-NEXT:    rlwinm r5, r5, 0, 24, 31
-; CHECK-P9-NEXT:    rlwinm r6, r6, 0, 24, 31
-; CHECK-P9-NEXT:    rlwinm r7, r7, 0, 24, 31
-; CHECK-P9-NEXT:    rlwinm r8, r8, 0, 24, 31
-; CHECK-P9-NEXT:    rlwinm r9, r9, 0, 24, 31
-; CHECK-P9-NEXT:    rlwinm r10, r10, 0, 24, 31
-; CHECK-P9-NEXT:    rlwinm r11, r11, 0, 24, 31
-; CHECK-P9-NEXT:    rlwinm r12, r12, 0, 24, 31
-; CHECK-P9-NEXT:    rlwinm r0, r0, 0, 24, 31
-; CHECK-P9-NEXT:    rlwinm r30, r30, 0, 24, 31
-; CHECK-P9-NEXT:    rlwinm r29, r29, 0, 24, 31
-; CHECK-P9-NEXT:    rlwinm r28, r28, 0, 24, 31
-; CHECK-P9-NEXT:    rlwinm r27, r27, 0, 24, 31
-; CHECK-P9-NEXT:    rlwinm r26, r26, 0, 24, 31
-; CHECK-P9-NEXT:    rlwinm r25, r25, 0, 24, 31
-; CHECK-P9-NEXT:    mtvsrwz f0, r4
-; CHECK-P9-NEXT:    mtvsrwz f1, r5
-; CHECK-P9-NEXT:    mtvsrwz f2, r6
-; CHECK-P9-NEXT:    mtvsrwz f3, r7
-; CHECK-P9-NEXT:    mtvsrwz f4, r8
-; CHECK-P9-NEXT:    mtvsrwz f5, r9
-; CHECK-P9-NEXT:    mtvsrwz f6, r10
-; CHECK-P9-NEXT:    mtvsrwz f7, r11
-; CHECK-P9-NEXT:    mtvsrwz f8, r12
-; CHECK-P9-NEXT:    mtvsrwz f9, r0
-; CHECK-P9-NEXT:    mtvsrwz f10, r30
-; CHECK-P9-NEXT:    ld r30, -32(r1) # 8-byte Folded Reload
-; CHECK-P9-NEXT:    mtvsrwz f11, r29
-; CHECK-P9-NEXT:    ld r29, -40(r1) # 8-byte Folded Reload
-; CHECK-P9-NEXT:    mtvsrwz f12, r28
-; CHECK-P9-NEXT:    ld r28, -48(r1) # 8-byte Folded Reload
-; CHECK-P9-NEXT:    mtvsrwz f13, r27
-; CHECK-P9-NEXT:    ld r27, -56(r1) # 8-byte Folded Reload
-; CHECK-P9-NEXT:    mtvsrwz v2, r26
-; CHECK-P9-NEXT:    ld r26, -64(r1) # 8-byte Folded Reload
-; CHECK-P9-NEXT:    mtvsrwz v3, r25
-; CHECK-P9-NEXT:    ld r25, -72(r1) # 8-byte Folded Reload
-; CHECK-P9-NEXT:    xscvuxddp f0, f0
-; CHECK-P9-NEXT:    xscvuxddp f1, f1
-; CHECK-P9-NEXT:    xscvuxddp f2, f2
-; CHECK-P9-NEXT:    xscvuxddp f3, f3
-; CHECK-P9-NEXT:    xscvuxddp f4, f4
-; CHECK-P9-NEXT:    xscvuxddp f5, f5
-; CHECK-P9-NEXT:    xscvuxddp f6, f6
-; CHECK-P9-NEXT:    xscvuxddp f7, f7
-; CHECK-P9-NEXT:    xscvuxddp f8, f8
-; CHECK-P9-NEXT:    xscvuxddp f9, f9
-; CHECK-P9-NEXT:    xscvuxddp f10, f10
-; CHECK-P9-NEXT:    xscvuxddp f11, f11
-; CHECK-P9-NEXT:    xscvuxddp f12, f12
-; CHECK-P9-NEXT:    xscvuxddp f13, f13
-; CHECK-P9-NEXT:    xscvuxddp f31, v2
-; CHECK-P9-NEXT:    xscvuxddp f30, v3
-; CHECK-P9-NEXT:    xxmrghd vs0, vs1, vs0
-; CHECK-P9-NEXT:    xxmrghd vs1, vs3, vs2
-; CHECK-P9-NEXT:    xxmrghd vs2, vs5, vs4
-; CHECK-P9-NEXT:    xxmrghd vs3, vs7, vs6
-; CHECK-P9-NEXT:    xxmrghd vs4, vs9, vs8
-; CHECK-P9-NEXT:    xxmrghd vs5, vs11, vs10
-; CHECK-P9-NEXT:    xxmrghd vs6, vs13, vs12
-; CHECK-P9-NEXT:    xxmrghd vs7, vs30, vs31
-; CHECK-P9-NEXT:    lfd f31, -8(r1) # 8-byte Folded Reload
-; CHECK-P9-NEXT:    lfd f30, -16(r1) # 8-byte Folded Reload
+; CHECK-P9-NEXT:    addis r4, r2, .LCPI3_0@toc@ha
+; CHECK-P9-NEXT:    addi r4, r4, .LCPI3_0@toc@l
+; CHECK-P9-NEXT:    lxvx v3, 0, r4
+; CHECK-P9-NEXT:    xxlxor v4, v4, v4
+; CHECK-P9-NEXT:    addis r4, r2, .LCPI3_1@toc@ha
+; CHECK-P9-NEXT:    addi r4, r4, .LCPI3_1@toc@l
+; CHECK-P9-NEXT:    vperm v3, v4, v2, v3
+; CHECK-P9-NEXT:    xvcvuxddp vs0, v3
+; CHECK-P9-NEXT:    lxvx v3, 0, r4
+; CHECK-P9-NEXT:    addis r4, r2, .LCPI3_2@toc@ha
+; CHECK-P9-NEXT:    addi r4, r4, .LCPI3_2@toc@l
+; CHECK-P9-NEXT:    vperm v3, v4, v2, v3
 ; CHECK-P9-NEXT:    stxv vs0, 0(r3)
-; CHECK-P9-NEXT:    stxv vs4, 64(r3)
-; CHECK-P9-NEXT:    stxv vs3, 48(r3)
-; CHECK-P9-NEXT:    stxv vs2, 32(r3)
+; CHECK-P9-NEXT:    xvcvuxddp vs1, v3
+; CHECK-P9-NEXT:    lxvx v3, 0, r4
+; CHECK-P9-NEXT:    addis r4, r2, .LCPI3_3@toc@ha
+; CHECK-P9-NEXT:    addi r4, r4, .LCPI3_3@toc@l
+; CHECK-P9-NEXT:    vperm v3, v4, v2, v3
 ; CHECK-P9-NEXT:    stxv vs1, 16(r3)
-; CHECK-P9-NEXT:    stxv vs7, 112(r3)
-; CHECK-P9-NEXT:    stxv vs6, 96(r3)
+; CHECK-P9-NEXT:    xvcvuxddp vs2, v3
+; CHECK-P9-NEXT:    lxvx v3, 0, r4
+; CHECK-P9-NEXT:    addis r4, r2, .LCPI3_4@toc@ha
+; CHECK-P9-NEXT:    addi r4, r4, .LCPI3_4@toc@l
+; CHECK-P9-NEXT:    vperm v3, v4, v2, v3
+; CHECK-P9-NEXT:    stxv vs2, 32(r3)
+; CHECK-P9-NEXT:    xvcvuxddp vs3, v3
+; CHECK-P9-NEXT:    lxvx v3, 0, r4
+; CHECK-P9-NEXT:    addis r4, r2, .LCPI3_5@toc@ha
+; CHECK-P9-NEXT:    addi r4, r4, .LCPI3_5@toc@l
+; CHECK-P9-NEXT:    vperm v3, v4, v2, v3
+; CHECK-P9-NEXT:    stxv vs3, 48(r3)
+; CHECK-P9-NEXT:    xvcvuxddp vs4, v3
+; CHECK-P9-NEXT:    lxvx v3, 0, r4
+; CHECK-P9-NEXT:    addis r4, r2, .LCPI3_6@toc@ha
+; CHECK-P9-NEXT:    addi r4, r4, .LCPI3_6@toc@l
+; CHECK-P9-NEXT:    vperm v3, v4, v2, v3
+; CHECK-P9-NEXT:    stxv vs4, 64(r3)
+; CHECK-P9-NEXT:    xvcvuxddp vs5, v3
+; CHECK-P9-NEXT:    lxvx v3, 0, r4
+; CHECK-P9-NEXT:    addis r4, r2, .LCPI3_7@toc@ha
+; CHECK-P9-NEXT:    addi r4, r4, .LCPI3_7@toc@l
+; CHECK-P9-NEXT:    vperm v3, v4, v2, v3
 ; CHECK-P9-NEXT:    stxv vs5, 80(r3)
+; CHECK-P9-NEXT:    xvcvuxddp vs6, v3
+; CHECK-P9-NEXT:    lxvx v3, 0, r4
+; CHECK-P9-NEXT:    vperm v2, v4, v2, v3
+; CHECK-P9-NEXT:    stxv vs6, 96(r3)
+; CHECK-P9-NEXT:    xvcvuxddp vs7, v2
+; CHECK-P9-NEXT:    stxv vs7, 112(r3)
 ; CHECK-P9-NEXT:    blr
 ;
 ; CHECK-BE-LABEL: test16elt:
 ; CHECK-BE:       # %bb.0: # %entry
-; CHECK-BE-NEXT:    std r25, -72(r1) # 8-byte Folded Spill
-; CHECK-BE-NEXT:    std r26, -64(r1) # 8-byte Folded Spill
-; CHECK-BE-NEXT:    std r27, -56(r1) # 8-byte Folded Spill
-; CHECK-BE-NEXT:    std r28, -48(r1) # 8-byte Folded Spill
-; CHECK-BE-NEXT:    std r29, -40(r1) # 8-byte Folded Spill
-; CHECK-BE-NEXT:    li r4, 1
-; CHECK-BE-NEXT:    std r30, -32(r1) # 8-byte Folded Spill
-; CHECK-BE-NEXT:    li r5, 0
-; CHECK-BE-NEXT:    li r6, 3
-; CHECK-BE-NEXT:    li r7, 2
-; CHECK-BE-NEXT:    li r8, 5
-; CHECK-BE-NEXT:    li r9, 4
-; CHECK-BE-NEXT:    li r10, 7
-; CHECK-BE-NEXT:    li r11, 6
-; CHECK-BE-NEXT:    li r12, 9
-; CHECK-BE-NEXT:    li r0, 8
-; CHECK-BE-NEXT:    li r30, 11
-; CHECK-BE-NEXT:    li r29, 10
-; CHECK-BE-NEXT:    li r28, 13
-; CHECK-BE-NEXT:    li r27, 12
-; CHECK-BE-NEXT:    li r26, 15
-; CHECK-BE-NEXT:    li r25, 14
-; CHECK-BE-NEXT:    stfd f30, -16(r1) # 8-byte Folded Spill
-; CHECK-BE-NEXT:    stfd f31, -8(r1) # 8-byte Folded Spill
-; CHECK-BE-NEXT:    vextublx r4, r4, v2
-; CHECK-BE-NEXT:    vextublx r5, r5, v2
-; CHECK-BE-NEXT:    vextublx r6, r6, v2
-; CHECK-BE-NEXT:    vextublx r7, r7, v2
-; CHECK-BE-NEXT:    vextublx r8, r8, v2
-; CHECK-BE-NEXT:    vextublx r9, r9, v2
-; CHECK-BE-NEXT:    vextublx r10, r10, v2
-; CHECK-BE-NEXT:    vextublx r11, r11, v2
-; CHECK-BE-NEXT:    vextublx r12, r12, v2
-; CHECK-BE-NEXT:    vextublx r0, r0, v2
-; CHECK-BE-NEXT:    vextublx r30, r30, v2
-; CHECK-BE-NEXT:    vextublx r29, r29, v2
-; CHECK-BE-NEXT:    vextublx r28, r28, v2
-; CHECK-BE-NEXT:    vextublx r27, r27, v2
-; CHECK-BE-NEXT:    vextublx r26, r26, v2
-; CHECK-BE-NEXT:    vextublx r25, r25, v2
-; CHECK-BE-NEXT:    rlwinm r4, r4, 0, 24, 31
-; CHECK-BE-NEXT:    rlwinm r5, r5, 0, 24, 31
-; CHECK-BE-NEXT:    rlwinm r6, r6, 0, 24, 31
-; CHECK-BE-NEXT:    rlwinm r7, r7, 0, 24, 31
-; CHECK-BE-NEXT:    rlwinm r8, r8, 0, 24, 31
-; CHECK-BE-NEXT:    rlwinm r9, r9, 0, 24, 31
-; CHECK-BE-NEXT:    rlwinm r10, r10, 0, 24, 31
-; CHECK-BE-NEXT:    rlwinm r11, r11, 0, 24, 31
-; CHECK-BE-NEXT:    rlwinm r12, r12, 0, 24, 31
-; CHECK-BE-NEXT:    rlwinm r0, r0, 0, 24, 31
-; CHECK-BE-NEXT:    rlwinm r30, r30, 0, 24, 31
-; CHECK-BE-NEXT:    rlwinm r29, r29, 0, 24, 31
-; CHECK-BE-NEXT:    rlwinm r28, r28, 0, 24, 31
-; CHECK-BE-NEXT:    rlwinm r27, r27, 0, 24, 31
-; CHECK-BE-NEXT:    rlwinm r26, r26, 0, 24, 31
-; CHECK-BE-NEXT:    rlwinm r25, r25, 0, 24, 31
-; CHECK-BE-NEXT:    mtvsrwz f0, r4
-; CHECK-BE-NEXT:    mtvsrwz f1, r5
-; CHECK-BE-NEXT:    mtvsrwz f2, r6
-; CHECK-BE-NEXT:    mtvsrwz f3, r7
-; CHECK-BE-NEXT:    mtvsrwz f4, r8
-; CHECK-BE-NEXT:    mtvsrwz f5, r9
-; CHECK-BE-NEXT:    mtvsrwz f6, r10
-; CHECK-BE-NEXT:    mtvsrwz f7, r11
-; CHECK-BE-NEXT:    mtvsrwz f8, r12
-; CHECK-BE-NEXT:    mtvsrwz f9, r0
-; CHECK-BE-NEXT:    mtvsrwz f10, r30
-; CHECK-BE-NEXT:    ld r30, -32(r1) # 8-byte Folded Reload
-; CHECK-BE-NEXT:    mtvsrwz f11, r29
-; CHECK-BE-NEXT:    ld r29, -40(r1) # 8-byte Folded Reload
-; CHECK-BE-NEXT:    mtvsrwz f12, r28
-; CHECK-BE-NEXT:    ld r28, -48(r1) # 8-byte Folded Reload
-; CHECK-BE-NEXT:    mtvsrwz f13, r27
-; CHECK-BE-NEXT:    ld r27, -56(r1) # 8-byte Folded Reload
-; CHECK-BE-NEXT:    mtvsrwz v2, r26
-; CHECK-BE-NEXT:    ld r26, -64(r1) # 8-byte Folded Reload
-; CHECK-BE-NEXT:    mtvsrwz v3, r25
-; CHECK-BE-NEXT:    ld r25, -72(r1) # 8-byte Folded Reload
-; CHECK-BE-NEXT:    xscvuxddp f0, f0
-; CHECK-BE-NEXT:    xscvuxddp f1, f1
-; CHECK-BE-NEXT:    xscvuxddp f2, f2
-; CHECK-BE-NEXT:    xscvuxddp f3, f3
-; CHECK-BE-NEXT:    xscvuxddp f4, f4
-; CHECK-BE-NEXT:    xscvuxddp f5, f5
-; CHECK-BE-NEXT:    xscvuxddp f6, f6
-; CHECK-BE-NEXT:    xscvuxddp f7, f7
-; CHECK-BE-NEXT:    xscvuxddp f8, f8
-; CHECK-BE-NEXT:    xscvuxddp f9, f9
-; CHECK-BE-NEXT:    xscvuxddp f10, f10
-; CHECK-BE-NEXT:    xscvuxddp f11, f11
-; CHECK-BE-NEXT:    xscvuxddp f12, f12
-; CHECK-BE-NEXT:    xscvuxddp f13, f13
-; CHECK-BE-NEXT:    xscvuxddp f31, v2
-; CHECK-BE-NEXT:    xscvuxddp f30, v3
-; CHECK-BE-NEXT:    xxmrghd vs0, vs1, vs0
-; CHECK-BE-NEXT:    xxmrghd vs1, vs3, vs2
-; CHECK-BE-NEXT:    xxmrghd vs2, vs5, vs4
-; CHECK-BE-NEXT:    xxmrghd vs3, vs7, vs6
-; CHECK-BE-NEXT:    xxmrghd vs4, vs9, vs8
-; CHECK-BE-NEXT:    xxmrghd vs5, vs11, vs10
-; CHECK-BE-NEXT:    xxmrghd vs6, vs13, vs12
-; CHECK-BE-NEXT:    xxmrghd vs7, vs30, vs31
-; CHECK-BE-NEXT:    lfd f31, -8(r1) # 8-byte Folded Reload
-; CHECK-BE-NEXT:    lfd f30, -16(r1) # 8-byte Folded Reload
+; CHECK-BE-NEXT:    addis r4, r2, .LCPI3_0@toc@ha
+; CHECK-BE-NEXT:    addi r4, r4, .LCPI3_0@toc@l
+; CHECK-BE-NEXT:    lxvx v3, 0, r4
+; CHECK-BE-NEXT:    xxlxor v4, v4, v4
+; CHECK-BE-NEXT:    addis r4, r2, .LCPI3_1@toc@ha
+; CHECK-BE-NEXT:    addi r4, r4, .LCPI3_1@toc@l
+; CHECK-BE-NEXT:    vperm v3, v2, v4, v3
+; CHECK-BE-NEXT:    xvcvuxddp vs0, v3
+; CHECK-BE-NEXT:    lxvx v3, 0, r4
+; CHECK-BE-NEXT:    addis r4, r2, .LCPI3_2@toc@ha
+; CHECK-BE-NEXT:    addi r4, r4, .LCPI3_2@toc@l
+; CHECK-BE-NEXT:    vperm v3, v4, v2, v3
 ; CHECK-BE-NEXT:    stxv vs0, 0(r3)
-; CHECK-BE-NEXT:    stxv vs4, 64(r3)
-; CHECK-BE-NEXT:    stxv vs3, 48(r3)
-; CHECK-BE-NEXT:    stxv vs2, 32(r3)
+; CHECK-BE-NEXT:    xvcvuxddp vs1, v3
+; CHECK-BE-NEXT:    lxvx v3, 0, r4
+; CHECK-BE-NEXT:    addis r4, r2, .LCPI3_3@toc@ha
+; CHECK-BE-NEXT:    addi r4, r4, .LCPI3_3@toc@l
+; CHECK-BE-NEXT:    vperm v3, v4, v2, v3
 ; CHECK-BE-NEXT:    stxv vs1, 16(r3)
-; CHECK-BE-NEXT:    stxv vs7, 112(r3)
-; CHECK-BE-NEXT:    stxv vs6, 96(r3)
+; CHECK-BE-NEXT:    xvcvuxddp vs2, v3
+; CHECK-BE-NEXT:    lxvx v3, 0, r4
+; CHECK-BE-NEXT:    addis r4, r2, .LCPI3_4@toc@ha
+; CHECK-BE-NEXT:    addi r4, r4, .LCPI3_4@toc@l
+; CHECK-BE-NEXT:    vperm v3, v4, v2, v3
+; CHECK-BE-NEXT:    stxv vs2, 32(r3)
+; CHECK-BE-NEXT:    xvcvuxddp vs3, v3
+; CHECK-BE-NEXT:    lxvx v3, 0, r4
+; CHECK-BE-NEXT:    addis r4, r2, .LCPI3_5@toc@ha
+; CHECK-BE-NEXT:    addi r4, r4, .LCPI3_5@toc@l
+; CHECK-BE-NEXT:    vperm v3, v4, v2, v3
+; CHECK-BE-NEXT:    stxv vs3, 48(r3)
+; CHECK-BE-NEXT:    xvcvuxddp vs4, v3
+; CHECK-BE-NEXT:    lxvx v3, 0, r4
+; CHECK-BE-NEXT:    addis r4, r2, .LCPI3_6@toc@ha
+; CHECK-BE-NEXT:    addi r4, r4, .LCPI3_6@toc@l
+; CHECK-BE-NEXT:    vperm v3, v4, v2, v3
+; CHECK-BE-NEXT:    stxv vs4, 64(r3)
+; CHECK-BE-NEXT:    xvcvuxddp vs5, v3
+; CHECK-BE-NEXT:    lxvx v3, 0, r4
+; CHECK-BE-NEXT:    addis r4, r2, .LCPI3_7@toc@ha
+; CHECK-BE-NEXT:    addi r4, r4, .LCPI3_7@toc@l
+; CHECK-BE-NEXT:    vperm v3, v4, v2, v3
 ; CHECK-BE-NEXT:    stxv vs5, 80(r3)
+; CHECK-BE-NEXT:    xvcvuxddp vs6, v3
+; CHECK-BE-NEXT:    lxvx v3, 0, r4
+; CHECK-BE-NEXT:    vperm v2, v4, v2, v3
+; CHECK-BE-NEXT:    stxv vs6, 96(r3)
+; CHECK-BE-NEXT:    xvcvuxddp vs7, v2
+; CHECK-BE-NEXT:    stxv vs7, 112(r3)
 ; CHECK-BE-NEXT:    blr
 entry:
   %0 = uitofp <16 x i8> %a to <16 x double>
@@ -668,49 +403,41 @@ entry:
 define <2 x double> @test2elt_signed(i16 %a.coerce) local_unnamed_addr #0 {
 ; CHECK-P8-LABEL: test2elt_signed:
 ; CHECK-P8:       # %bb.0: # %entry
+; CHECK-P8-NEXT:    addis r4, r2, .LCPI4_0@toc@ha
 ; CHECK-P8-NEXT:    mtvsrd f0, r3
-; CHECK-P8-NEXT:    mfvsrd r3, f0
-; CHECK-P8-NEXT:    clrldi r4, r3, 56
-; CHECK-P8-NEXT:    rldicl r3, r3, 56, 56
-; CHECK-P8-NEXT:    extsb r4, r4
-; CHECK-P8-NEXT:    extsb r3, r3
-; CHECK-P8-NEXT:    mtvsrwa f0, r4
-; CHECK-P8-NEXT:    mtvsrwa f1, r3
-; CHECK-P8-NEXT:    xscvsxddp f0, f0
-; CHECK-P8-NEXT:    xscvsxddp f1, f1
-; CHECK-P8-NEXT:    xxmrghd v2, vs1, vs0
+; CHECK-P8-NEXT:    addi r3, r4, .LCPI4_0@toc@l
+; CHECK-P8-NEXT:    xxswapd v2, vs0
+; CHECK-P8-NEXT:    lvx v3, 0, r3
+; CHECK-P8-NEXT:    addis r3, r2, .LCPI4_1@toc@ha
+; CHECK-P8-NEXT:    addi r3, r3, .LCPI4_1@toc@l
+; CHECK-P8-NEXT:    lxvd2x vs0, 0, r3
+; CHECK-P8-NEXT:    vperm v2, v2, v2, v3
+; CHECK-P8-NEXT:    xxswapd v3, vs0
+; CHECK-P8-NEXT:    vsld v2, v2, v3
+; CHECK-P8-NEXT:    vsrad v2, v2, v3
+; CHECK-P8-NEXT:    xvcvsxddp v2, v2
 ; CHECK-P8-NEXT:    blr
 ;
 ; CHECK-P9-LABEL: test2elt_signed:
 ; CHECK-P9:       # %bb.0: # %entry
 ; CHECK-P9-NEXT:    mtvsrws v2, r3
-; CHECK-P9-NEXT:    li r3, 0
-; CHECK-P9-NEXT:    li r4, 1
-; CHECK-P9-NEXT:    vextubrx r3, r3, v2
-; CHECK-P9-NEXT:    vextubrx r4, r4, v2
-; CHECK-P9-NEXT:    extsb r3, r3
-; CHECK-P9-NEXT:    extsb r4, r4
-; CHECK-P9-NEXT:    mtvsrwa f0, r3
-; CHECK-P9-NEXT:    mtvsrwa f1, r4
-; CHECK-P9-NEXT:    xscvsxddp f0, f0
-; CHECK-P9-NEXT:    xscvsxddp f1, f1
-; CHECK-P9-NEXT:    xxmrghd v2, vs1, vs0
+; CHECK-P9-NEXT:    addis r3, r2, .LCPI4_0@toc@ha
+; CHECK-P9-NEXT:    addi r3, r3, .LCPI4_0@toc@l
+; CHECK-P9-NEXT:    lxvx v3, 0, r3
+; CHECK-P9-NEXT:    vperm v2, v2, v2, v3
+; CHECK-P9-NEXT:    vextsb2d v2, v2
+; CHECK-P9-NEXT:    xvcvsxddp v2, v2
 ; CHECK-P9-NEXT:    blr
 ;
 ; CHECK-BE-LABEL: test2elt_signed:
 ; CHECK-BE:       # %bb.0: # %entry
 ; CHECK-BE-NEXT:    mtvsrws v2, r3
-; CHECK-BE-NEXT:    li r3, 1
-; CHECK-BE-NEXT:    li r4, 0
-; CHECK-BE-NEXT:    vextublx r3, r3, v2
-; CHECK-BE-NEXT:    vextublx r4, r4, v2
-; CHECK-BE-NEXT:    extsb r3, r3
-; CHECK-BE-NEXT:    extsb r4, r4
-; CHECK-BE-NEXT:    mtvsrwa f0, r3
-; CHECK-BE-NEXT:    mtvsrwa f1, r4
-; CHECK-BE-NEXT:    xscvsxddp f0, f0
-; CHECK-BE-NEXT:    xscvsxddp f1, f1
-; CHECK-BE-NEXT:    xxmrghd v2, vs1, vs0
+; CHECK-BE-NEXT:    addis r3, r2, .LCPI4_0@toc@ha
+; CHECK-BE-NEXT:    addi r3, r3, .LCPI4_0@toc@l
+; CHECK-BE-NEXT:    lxvx v3, 0, r3
+; CHECK-BE-NEXT:    vperm v2, v2, v2, v3
+; CHECK-BE-NEXT:    vextsb2d v2, v2
+; CHECK-BE-NEXT:    xvcvsxddp v2, v2
 ; CHECK-BE-NEXT:    blr
 entry:
   %0 = bitcast i16 %a.coerce to <2 x i8>
@@ -721,27 +448,27 @@ entry:
 define void @test4elt_signed(<4 x double>* noalias nocapture sret %agg.result, i32 %a.coerce) local_unnamed_addr #1 {
 ; CHECK-P8-LABEL: test4elt_signed:
 ; CHECK-P8:       # %bb.0: # %entry
+; CHECK-P8-NEXT:    addis r5, r2, .LCPI5_0@toc@ha
 ; CHECK-P8-NEXT:    mtvsrd f0, r4
-; CHECK-P8-NEXT:    mfvsrd r4, f0
-; CHECK-P8-NEXT:    clrldi r5, r4, 56
-; CHECK-P8-NEXT:    rldicl r6, r4, 56, 56
-; CHECK-P8-NEXT:    extsb r5, r5
-; CHECK-P8-NEXT:    extsb r6, r6
-; CHECK-P8-NEXT:    mtvsrwa f0, r5
-; CHECK-P8-NEXT:    rldicl r5, r4, 48, 56
-; CHECK-P8-NEXT:    rldicl r4, r4, 40, 56
-; CHECK-P8-NEXT:    extsb r5, r5
-; CHECK-P8-NEXT:    extsb r4, r4
-; CHECK-P8-NEXT:    mtvsrwa f1, r6
-; CHECK-P8-NEXT:    mtvsrwa f2, r5
-; CHECK-P8-NEXT:    mtvsrwa f3, r4
+; CHECK-P8-NEXT:    addis r4, r2, .LCPI5_2@toc@ha
+; CHECK-P8-NEXT:    addi r5, r5, .LCPI5_0@toc@l
+; CHECK-P8-NEXT:    addi r4, r4, .LCPI5_2@toc@l
+; CHECK-P8-NEXT:    lvx v2, 0, r5
+; CHECK-P8-NEXT:    xxswapd v3, vs0
+; CHECK-P8-NEXT:    lvx v4, 0, r4
+; CHECK-P8-NEXT:    addis r4, r2, .LCPI5_1@toc@ha
+; CHECK-P8-NEXT:    addi r4, r4, .LCPI5_1@toc@l
+; CHECK-P8-NEXT:    lxvd2x vs0, 0, r4
 ; CHECK-P8-NEXT:    li r4, 16
-; CHECK-P8-NEXT:    xscvsxddp f0, f0
-; CHECK-P8-NEXT:    xscvsxddp f1, f1
-; CHECK-P8-NEXT:    xscvsxddp f2, f2
-; CHECK-P8-NEXT:    xscvsxddp f3, f3
-; CHECK-P8-NEXT:    xxmrghd vs0, vs1, vs0
-; CHECK-P8-NEXT:    xxmrghd vs1, vs3, vs2
+; CHECK-P8-NEXT:    vperm v2, v3, v3, v2
+; CHECK-P8-NEXT:    vperm v3, v3, v3, v4
+; CHECK-P8-NEXT:    xxswapd v4, vs0
+; CHECK-P8-NEXT:    vsld v2, v2, v4
+; CHECK-P8-NEXT:    vsld v3, v3, v4
+; CHECK-P8-NEXT:    vsrad v2, v2, v4
+; CHECK-P8-NEXT:    vsrad v3, v3, v4
+; CHECK-P8-NEXT:    xvcvsxddp vs0, v2
+; CHECK-P8-NEXT:    xvcvsxddp vs1, v3
 ; CHECK-P8-NEXT:    xxswapd vs0, vs0
 ; CHECK-P8-NEXT:    xxswapd vs1, vs1
 ; CHECK-P8-NEXT:    stxvd2x vs1, r3, r4
@@ -751,59 +478,40 @@ define void @test4elt_signed(<4 x double>* noalias nocapture sret %agg.result, i
 ; CHECK-P9-LABEL: test4elt_signed:
 ; CHECK-P9:       # %bb.0: # %entry
 ; CHECK-P9-NEXT:    mtvsrws v2, r4
-; CHECK-P9-NEXT:    li r4, 0
-; CHECK-P9-NEXT:    li r5, 1
-; CHECK-P9-NEXT:    li r6, 2
-; CHECK-P9-NEXT:    li r7, 3
-; CHECK-P9-NEXT:    vextubrx r4, r4, v2
-; CHECK-P9-NEXT:    vextubrx r5, r5, v2
-; CHECK-P9-NEXT:    vextubrx r6, r6, v2
-; CHECK-P9-NEXT:    vextubrx r7, r7, v2
-; CHECK-P9-NEXT:    extsb r4, r4
-; CHECK-P9-NEXT:    extsb r5, r5
-; CHECK-P9-NEXT:    extsb r6, r6
-; CHECK-P9-NEXT:    extsb r7, r7
-; CHECK-P9-NEXT:    mtvsrwa f0, r4
-; CHECK-P9-NEXT:    mtvsrwa f1, r5
-; CHECK-P9-NEXT:    mtvsrwa f2, r6
-; CHECK-P9-NEXT:    mtvsrwa f3, r7
-; CHECK-P9-NEXT:    xscvsxddp f0, f0
-; CHECK-P9-NEXT:    xscvsxddp f1, f1
-; CHECK-P9-NEXT:    xscvsxddp f2, f2
-; CHECK-P9-NEXT:    xscvsxddp f3, f3
-; CHECK-P9-NEXT:    xxmrghd vs0, vs1, vs0
-; CHECK-P9-NEXT:    xxmrghd vs1, vs3, vs2
-; CHECK-P9-NEXT:    stxv vs1, 16(r3)
+; CHECK-P9-NEXT:    addis r4, r2, .LCPI5_0@toc@ha
+; CHECK-P9-NEXT:    addi r4, r4, .LCPI5_0@toc@l
+; CHECK-P9-NEXT:    lxvx v3, 0, r4
+; CHECK-P9-NEXT:    addis r4, r2, .LCPI5_1@toc@ha
+; CHECK-P9-NEXT:    addi r4, r4, .LCPI5_1@toc@l
+; CHECK-P9-NEXT:    vperm v3, v2, v2, v3
+; CHECK-P9-NEXT:    vextsb2d v3, v3
+; CHECK-P9-NEXT:    xvcvsxddp vs0, v3
+; CHECK-P9-NEXT:    lxvx v3, 0, r4
+; CHECK-P9-NEXT:    vperm v2, v2, v2, v3
 ; CHECK-P9-NEXT:    stxv vs0, 0(r3)
+; CHECK-P9-NEXT:    vextsb2d v2, v2
+; CHECK-P9-NEXT:    xvcvsxddp vs1, v2
+; CHECK-P9-NEXT:    stxv vs1, 16(r3)
 ; CHECK-P9-NEXT:    blr
 ;
 ; CHECK-BE-LABEL: test4elt_signed:
 ; CHECK-BE:       # %bb.0: # %entry
 ; CHECK-BE-NEXT:    mtvsrws v2, r4
-; CHECK-BE-NEXT:    li r4, 1
-; CHECK-BE-NEXT:    li r5, 0
-; CHECK-BE-NEXT:    li r6, 3
-; CHECK-BE-NEXT:    li r7, 2
-; CHECK-BE-NEXT:    vextublx r4, r4, v2
-; CHECK-BE-NEXT:    vextublx r5, r5, v2
-; CHECK-BE-NEXT:    vextublx r6, r6, v2
-; CHECK-BE-NEXT:    vextublx r7, r7, v2
-; CHECK-BE-NEXT:    extsb r4, r4
-; CHECK-BE-NEXT:    extsb r5, r5
-; CHECK-BE-NEXT:    extsb r6, r6
-; CHECK-BE-NEXT:    extsb r7, r7
-; CHECK-BE-NEXT:    mtvsrwa f0, r4
-; CHECK-BE-NEXT:    mtvsrwa f1, r5
-; CHECK-BE-NEXT:    mtvsrwa f2, r6
-; CHECK-BE-NEXT:    mtvsrwa f3, r7
-; CHECK-BE-NEXT:    xscvsxddp f0, f0
-; CHECK-BE-NEXT:    xscvsxddp f1, f1
-; CHECK-BE-NEXT:    xscvsxddp f2, f2
-; CHECK-BE-NEXT:    xscvsxddp f3, f3
-; CHECK-BE-NEXT:    xxmrghd vs0, vs1, vs0
-; CHECK-BE-NEXT:    xxmrghd vs1, vs3, vs2
-; CHECK-BE-NEXT:    stxv vs1, 16(r3)
-; CHECK-BE-NEXT:    stxv vs0, 0(r3)
+; CHECK-BE-NEXT:    addis r4, r2, .LCPI5_0@toc@ha
+; CHECK-BE-NEXT:    addi r4, r4, .LCPI5_0@toc@l
+; CHECK-BE-NEXT:    lxvx v4, 0, r4
+; CHECK-BE-NEXT:    xxlxor v3, v3, v3
+; CHECK-BE-NEXT:    vperm v3, v3, v2, v4
+; CHECK-BE-NEXT:    addis r4, r2, .LCPI5_1@toc@ha
+; CHECK-BE-NEXT:    vextsb2d v3, v3
+; CHECK-BE-NEXT:    addi r4, r4, .LCPI5_1@toc@l
+; CHECK-BE-NEXT:    xvcvsxddp vs0, v3
+; CHECK-BE-NEXT:    lxvx v3, 0, r4
+; CHECK-BE-NEXT:    vperm v2, v2, v2, v3
+; CHECK-BE-NEXT:    stxv vs0, 16(r3)
+; CHECK-BE-NEXT:    vextsb2d v2, v2
+; CHECK-BE-NEXT:    xvcvsxddp vs1, v2
+; CHECK-BE-NEXT:    stxv vs1, 0(r3)
 ; CHECK-BE-NEXT:    blr
 entry:
   %0 = bitcast i32 %a.coerce to <4 x i8>
@@ -816,161 +524,118 @@ define void @test8elt_signed(<8 x double>* noalias nocapture sret %agg.result, i
 ; CHECK-P8-LABEL: test8elt_signed:
 ; CHECK-P8:       # %bb.0: # %entry
 ; CHECK-P8-NEXT:    mtvsrd f0, r4
-; CHECK-P8-NEXT:    mfvsrd r4, f0
-; CHECK-P8-NEXT:    clrldi r5, r4, 56
-; CHECK-P8-NEXT:    rldicl r6, r4, 56, 56
-; CHECK-P8-NEXT:    extsb r5, r5
-; CHECK-P8-NEXT:    extsb r6, r6
-; CHECK-P8-NEXT:    mtvsrwa f0, r5
-; CHECK-P8-NEXT:    rldicl r5, r4, 48, 56
-; CHECK-P8-NEXT:    extsb r5, r5
-; CHECK-P8-NEXT:    mtvsrwa f1, r6
-; CHECK-P8-NEXT:    rldicl r6, r4, 40, 56
-; CHECK-P8-NEXT:    mtvsrwa f2, r5
-; CHECK-P8-NEXT:    rldicl r5, r4, 32, 56
-; CHECK-P8-NEXT:    extsb r6, r6
-; CHECK-P8-NEXT:    extsb r5, r5
-; CHECK-P8-NEXT:    mtvsrwa f3, r6
-; CHECK-P8-NEXT:    rldicl r6, r4, 24, 56
-; CHECK-P8-NEXT:    mtvsrwa f4, r5
-; CHECK-P8-NEXT:    rldicl r5, r4, 16, 56
-; CHECK-P8-NEXT:    rldicl r4, r4, 8, 56
-; CHECK-P8-NEXT:    extsb r6, r6
-; CHECK-P8-NEXT:    extsb r5, r5
-; CHECK-P8-NEXT:    extsb r4, r4
-; CHECK-P8-NEXT:    mtvsrwa f5, r6
-; CHECK-P8-NEXT:    mtvsrwa f6, r5
-; CHECK-P8-NEXT:    li r5, 32
-; CHECK-P8-NEXT:    mtvsrwa f7, r4
+; CHECK-P8-NEXT:    addis r4, r2, .LCPI6_2@toc@ha
+; CHECK-P8-NEXT:    addis r5, r2, .LCPI6_0@toc@ha
+; CHECK-P8-NEXT:    addis r6, r2, .LCPI6_3@toc@ha
+; CHECK-P8-NEXT:    addi r4, r4, .LCPI6_2@toc@l
+; CHECK-P8-NEXT:    addi r5, r5, .LCPI6_0@toc@l
+; CHECK-P8-NEXT:    addi r6, r6, .LCPI6_3@toc@l
+; CHECK-P8-NEXT:    lvx v4, 0, r4
+; CHECK-P8-NEXT:    addis r4, r2, .LCPI6_4@toc@ha
+; CHECK-P8-NEXT:    lvx v2, 0, r5
+; CHECK-P8-NEXT:    xxswapd v3, vs0
+; CHECK-P8-NEXT:    lvx v5, 0, r6
+; CHECK-P8-NEXT:    addis r5, r2, .LCPI6_1@toc@ha
+; CHECK-P8-NEXT:    addi r4, r4, .LCPI6_4@toc@l
+; CHECK-P8-NEXT:    addi r5, r5, .LCPI6_1@toc@l
+; CHECK-P8-NEXT:    lvx v0, 0, r4
+; CHECK-P8-NEXT:    lxvd2x vs0, 0, r5
 ; CHECK-P8-NEXT:    li r4, 48
-; CHECK-P8-NEXT:    xscvsxddp f4, f4
-; CHECK-P8-NEXT:    xscvsxddp f5, f5
-; CHECK-P8-NEXT:    xscvsxddp f6, f6
-; CHECK-P8-NEXT:    xscvsxddp f7, f7
-; CHECK-P8-NEXT:    xscvsxddp f0, f0
-; CHECK-P8-NEXT:    xscvsxddp f1, f1
-; CHECK-P8-NEXT:    xscvsxddp f2, f2
-; CHECK-P8-NEXT:    xscvsxddp f3, f3
-; CHECK-P8-NEXT:    xxmrghd vs4, vs5, vs4
-; CHECK-P8-NEXT:    xxmrghd vs5, vs7, vs6
-; CHECK-P8-NEXT:    xxmrghd vs0, vs1, vs0
-; CHECK-P8-NEXT:    xxmrghd vs1, vs3, vs2
-; CHECK-P8-NEXT:    xxswapd vs2, vs5
-; CHECK-P8-NEXT:    xxswapd vs3, vs4
+; CHECK-P8-NEXT:    li r5, 32
+; CHECK-P8-NEXT:    vperm v2, v3, v3, v2
+; CHECK-P8-NEXT:    vperm v4, v3, v3, v4
+; CHECK-P8-NEXT:    vperm v5, v3, v3, v5
+; CHECK-P8-NEXT:    vperm v3, v3, v3, v0
+; CHECK-P8-NEXT:    xxswapd v0, vs0
+; CHECK-P8-NEXT:    vsld v2, v2, v0
+; CHECK-P8-NEXT:    vsld v4, v4, v0
+; CHECK-P8-NEXT:    vsld v5, v5, v0
+; CHECK-P8-NEXT:    vsld v3, v3, v0
+; CHECK-P8-NEXT:    vsrad v2, v2, v0
+; CHECK-P8-NEXT:    vsrad v3, v3, v0
+; CHECK-P8-NEXT:    vsrad v4, v4, v0
+; CHECK-P8-NEXT:    vsrad v5, v5, v0
+; CHECK-P8-NEXT:    xvcvsxddp vs2, v3
+; CHECK-P8-NEXT:    xvcvsxddp vs0, v2
+; CHECK-P8-NEXT:    xvcvsxddp vs1, v5
+; CHECK-P8-NEXT:    xvcvsxddp vs3, v4
+; CHECK-P8-NEXT:    xxswapd vs2, vs2
 ; CHECK-P8-NEXT:    xxswapd vs0, vs0
 ; CHECK-P8-NEXT:    xxswapd vs1, vs1
+; CHECK-P8-NEXT:    xxswapd vs3, vs3
 ; CHECK-P8-NEXT:    stxvd2x vs2, r3, r4
 ; CHECK-P8-NEXT:    li r4, 16
-; CHECK-P8-NEXT:    stxvd2x vs3, r3, r5
-; CHECK-P8-NEXT:    stxvd2x vs1, r3, r4
+; CHECK-P8-NEXT:    stxvd2x vs1, r3, r5
+; CHECK-P8-NEXT:    stxvd2x vs3, r3, r4
 ; CHECK-P8-NEXT:    stxvd2x vs0, 0, r3
 ; CHECK-P8-NEXT:    blr
 ;
 ; CHECK-P9-LABEL: test8elt_signed:
 ; CHECK-P9:       # %bb.0: # %entry
 ; CHECK-P9-NEXT:    mtvsrd f0, r4
-; CHECK-P9-NEXT:    li r4, 0
-; CHECK-P9-NEXT:    li r5, 1
-; CHECK-P9-NEXT:    li r6, 2
-; CHECK-P9-NEXT:    li r7, 3
-; CHECK-P9-NEXT:    li r8, 4
-; CHECK-P9-NEXT:    li r9, 5
-; CHECK-P9-NEXT:    li r10, 6
-; CHECK-P9-NEXT:    li r11, 7
+; CHECK-P9-NEXT:    addis r4, r2, .LCPI6_0@toc@ha
+; CHECK-P9-NEXT:    addi r4, r4, .LCPI6_0@toc@l
+; CHECK-P9-NEXT:    lxvx v3, 0, r4
 ; CHECK-P9-NEXT:    xxswapd v2, vs0
-; CHECK-P9-NEXT:    vextubrx r4, r4, v2
-; CHECK-P9-NEXT:    vextubrx r5, r5, v2
-; CHECK-P9-NEXT:    vextubrx r6, r6, v2
-; CHECK-P9-NEXT:    vextubrx r7, r7, v2
-; CHECK-P9-NEXT:    vextubrx r8, r8, v2
-; CHECK-P9-NEXT:    vextubrx r9, r9, v2
-; CHECK-P9-NEXT:    vextubrx r10, r10, v2
-; CHECK-P9-NEXT:    vextubrx r11, r11, v2
-; CHECK-P9-NEXT:    extsb r4, r4
-; CHECK-P9-NEXT:    extsb r5, r5
-; CHECK-P9-NEXT:    extsb r6, r6
-; CHECK-P9-NEXT:    extsb r7, r7
-; CHECK-P9-NEXT:    extsb r8, r8
-; CHECK-P9-NEXT:    extsb r9, r9
-; CHECK-P9-NEXT:    extsb r10, r10
-; CHECK-P9-NEXT:    extsb r11, r11
-; CHECK-P9-NEXT:    mtvsrwa f0, r4
-; CHECK-P9-NEXT:    mtvsrwa f1, r5
-; CHECK-P9-NEXT:    mtvsrwa f2, r6
-; CHECK-P9-NEXT:    mtvsrwa f3, r7
-; CHECK-P9-NEXT:    mtvsrwa f4, r8
-; CHECK-P9-NEXT:    mtvsrwa f5, r9
-; CHECK-P9-NEXT:    mtvsrwa f6, r10
-; CHECK-P9-NEXT:    mtvsrwa f7, r11
-; CHECK-P9-NEXT:    xscvsxddp f0, f0
-; CHECK-P9-NEXT:    xscvsxddp f1, f1
-; CHECK-P9-NEXT:    xscvsxddp f2, f2
-; CHECK-P9-NEXT:    xscvsxddp f3, f3
-; CHECK-P9-NEXT:    xscvsxddp f4, f4
-; CHECK-P9-NEXT:    xscvsxddp f5, f5
-; CHECK-P9-NEXT:    xscvsxddp f6, f6
-; CHECK-P9-NEXT:    xscvsxddp f7, f7
-; CHECK-P9-NEXT:    xxmrghd vs0, vs1, vs0
-; CHECK-P9-NEXT:    xxmrghd vs1, vs3, vs2
-; CHECK-P9-NEXT:    xxmrghd vs2, vs5, vs4
-; CHECK-P9-NEXT:    xxmrghd vs3, vs7, vs6
-; CHECK-P9-NEXT:    stxv vs3, 48(r3)
-; CHECK-P9-NEXT:    stxv vs2, 32(r3)
-; CHECK-P9-NEXT:    stxv vs1, 16(r3)
+; CHECK-P9-NEXT:    vperm v3, v2, v2, v3
+; CHECK-P9-NEXT:    addis r4, r2, .LCPI6_1@toc@ha
+; CHECK-P9-NEXT:    addi r4, r4, .LCPI6_1@toc@l
+; CHECK-P9-NEXT:    vextsb2d v3, v3
+; CHECK-P9-NEXT:    xvcvsxddp vs0, v3
+; CHECK-P9-NEXT:    lxvx v3, 0, r4
+; CHECK-P9-NEXT:    addis r4, r2, .LCPI6_2@toc@ha
+; CHECK-P9-NEXT:    addi r4, r4, .LCPI6_2@toc@l
+; CHECK-P9-NEXT:    vperm v3, v2, v2, v3
 ; CHECK-P9-NEXT:    stxv vs0, 0(r3)
+; CHECK-P9-NEXT:    vextsb2d v3, v3
+; CHECK-P9-NEXT:    xvcvsxddp vs1, v3
+; CHECK-P9-NEXT:    lxvx v3, 0, r4
+; CHECK-P9-NEXT:    addis r4, r2, .LCPI6_3@toc@ha
+; CHECK-P9-NEXT:    addi r4, r4, .LCPI6_3@toc@l
+; CHECK-P9-NEXT:    vperm v3, v2, v2, v3
+; CHECK-P9-NEXT:    stxv vs1, 16(r3)
+; CHECK-P9-NEXT:    vextsb2d v3, v3
+; CHECK-P9-NEXT:    xvcvsxddp vs2, v3
+; CHECK-P9-NEXT:    lxvx v3, 0, r4
+; CHECK-P9-NEXT:    vperm v2, v2, v2, v3
+; CHECK-P9-NEXT:    stxv vs2, 32(r3)
+; CHECK-P9-NEXT:    vextsb2d v2, v2
+; CHECK-P9-NEXT:    xvcvsxddp vs3, v2
+; CHECK-P9-NEXT:    stxv vs3, 48(r3)
 ; CHECK-P9-NEXT:    blr
 ;
 ; CHECK-BE-LABEL: test8elt_signed:
 ; CHECK-BE:       # %bb.0: # %entry
-; CHECK-BE-NEXT:    li r5, 1
 ; CHECK-BE-NEXT:    mtvsrd v2, r4
-; CHECK-BE-NEXT:    li r4, 0
-; CHECK-BE-NEXT:    li r6, 3
-; CHECK-BE-NEXT:    li r7, 2
-; CHECK-BE-NEXT:    li r8, 5
-; CHECK-BE-NEXT:    li r9, 4
-; CHECK-BE-NEXT:    li r10, 7
-; CHECK-BE-NEXT:    li r11, 6
-; CHECK-BE-NEXT:    vextublx r5, r5, v2
-; CHECK-BE-NEXT:    vextublx r4, r4, v2
-; CHECK-BE-NEXT:    vextublx r6, r6, v2
-; CHECK-BE-NEXT:    vextublx r7, r7, v2
-; CHECK-BE-NEXT:    vextublx r8, r8, v2
-; CHECK-BE-NEXT:    vextublx r9, r9, v2
-; CHECK-BE-NEXT:    vextublx r10, r10, v2
-; CHECK-BE-NEXT:    vextublx r11, r11, v2
-; CHECK-BE-NEXT:    extsb r5, r5
-; CHECK-BE-NEXT:    extsb r4, r4
-; CHECK-BE-NEXT:    extsb r6, r6
-; CHECK-BE-NEXT:    extsb r7, r7
-; CHECK-BE-NEXT:    extsb r8, r8
-; CHECK-BE-NEXT:    extsb r9, r9
-; CHECK-BE-NEXT:    extsb r10, r10
-; CHECK-BE-NEXT:    extsb r11, r11
-; CHECK-BE-NEXT:    mtvsrwa f0, r5
-; CHECK-BE-NEXT:    mtvsrwa f1, r4
-; CHECK-BE-NEXT:    mtvsrwa f2, r6
-; CHECK-BE-NEXT:    mtvsrwa f3, r7
-; CHECK-BE-NEXT:    mtvsrwa f4, r8
-; CHECK-BE-NEXT:    mtvsrwa f5, r9
-; CHECK-BE-NEXT:    mtvsrwa f6, r10
-; CHECK-BE-NEXT:    mtvsrwa f7, r11
-; CHECK-BE-NEXT:    xscvsxddp f0, f0
-; CHECK-BE-NEXT:    xscvsxddp f1, f1
-; CHECK-BE-NEXT:    xscvsxddp f2, f2
-; CHECK-BE-NEXT:    xscvsxddp f3, f3
-; CHECK-BE-NEXT:    xscvsxddp f4, f4
-; CHECK-BE-NEXT:    xscvsxddp f5, f5
-; CHECK-BE-NEXT:    xscvsxddp f6, f6
-; CHECK-BE-NEXT:    xscvsxddp f7, f7
-; CHECK-BE-NEXT:    xxmrghd vs0, vs1, vs0
-; CHECK-BE-NEXT:    xxmrghd vs1, vs3, vs2
-; CHECK-BE-NEXT:    xxmrghd vs2, vs5, vs4
-; CHECK-BE-NEXT:    xxmrghd vs3, vs7, vs6
-; CHECK-BE-NEXT:    stxv vs2, 32(r3)
-; CHECK-BE-NEXT:    stxv vs1, 16(r3)
-; CHECK-BE-NEXT:    stxv vs0, 0(r3)
-; CHECK-BE-NEXT:    stxv vs3, 48(r3)
+; CHECK-BE-NEXT:    addis r4, r2, .LCPI6_0@toc@ha
+; CHECK-BE-NEXT:    addi r4, r4, .LCPI6_0@toc@l
+; CHECK-BE-NEXT:    lxvx v3, 0, r4
+; CHECK-BE-NEXT:    xxlxor v4, v4, v4
+; CHECK-BE-NEXT:    vperm v3, v4, v2, v3
+; CHECK-BE-NEXT:    addis r4, r2, .LCPI6_1@toc@ha
+; CHECK-BE-NEXT:    vextsb2d v3, v3
+; CHECK-BE-NEXT:    addi r4, r4, .LCPI6_1@toc@l
+; CHECK-BE-NEXT:    xvcvsxddp vs0, v3
+; CHECK-BE-NEXT:    lxvx v3, 0, r4
+; CHECK-BE-NEXT:    addis r4, r2, .LCPI6_2@toc@ha
+; CHECK-BE-NEXT:    addi r4, r4, .LCPI6_2@toc@l
+; CHECK-BE-NEXT:    vperm v3, v4, v2, v3
+; CHECK-BE-NEXT:    stxv vs0, 16(r3)
+; CHECK-BE-NEXT:    vextsb2d v3, v3
+; CHECK-BE-NEXT:    xvcvsxddp vs1, v3
+; CHECK-BE-NEXT:    lxvx v3, 0, r4
+; CHECK-BE-NEXT:    addis r4, r2, .LCPI6_3@toc@ha
+; CHECK-BE-NEXT:    addi r4, r4, .LCPI6_3@toc@l
+; CHECK-BE-NEXT:    vperm v3, v2, v2, v3
+; CHECK-BE-NEXT:    stxv vs1, 48(r3)
+; CHECK-BE-NEXT:    vextsb2d v3, v3
+; CHECK-BE-NEXT:    xvcvsxddp vs2, v3
+; CHECK-BE-NEXT:    lxvx v3, 0, r4
+; CHECK-BE-NEXT:    vperm v2, v2, v2, v3
+; CHECK-BE-NEXT:    stxv vs2, 0(r3)
+; CHECK-BE-NEXT:    vextsb2d v2, v2
+; CHECK-BE-NEXT:    xvcvsxddp vs3, v2
+; CHECK-BE-NEXT:    stxv vs3, 32(r3)
 ; CHECK-BE-NEXT:    blr
 entry:
   %0 = bitcast i64 %a.coerce to <8 x i8>
@@ -982,338 +647,210 @@ entry:
 define void @test16elt_signed(<16 x double>* noalias nocapture sret %agg.result, <16 x i8> %a) local_unnamed_addr #2 {
 ; CHECK-P8-LABEL: test16elt_signed:
 ; CHECK-P8:       # %bb.0: # %entry
-; CHECK-P8-NEXT:    mfvsrd r5, v2
-; CHECK-P8-NEXT:    xxswapd vs2, v2
+; CHECK-P8-NEXT:    addis r4, r2, .LCPI7_0@toc@ha
+; CHECK-P8-NEXT:    addis r5, r2, .LCPI7_2@toc@ha
+; CHECK-P8-NEXT:    addis r6, r2, .LCPI7_3@toc@ha
+; CHECK-P8-NEXT:    addi r4, r4, .LCPI7_0@toc@l
+; CHECK-P8-NEXT:    addi r5, r5, .LCPI7_2@toc@l
+; CHECK-P8-NEXT:    addi r6, r6, .LCPI7_3@toc@l
+; CHECK-P8-NEXT:    lvx v3, 0, r4
+; CHECK-P8-NEXT:    addis r4, r2, .LCPI7_4@toc@ha
+; CHECK-P8-NEXT:    lvx v4, 0, r5
+; CHECK-P8-NEXT:    addis r5, r2, .LCPI7_5@toc@ha
+; CHECK-P8-NEXT:    lvx v5, 0, r6
+; CHECK-P8-NEXT:    addis r6, r2, .LCPI7_1@toc@ha
+; CHECK-P8-NEXT:    addi r4, r4, .LCPI7_4@toc@l
+; CHECK-P8-NEXT:    addi r5, r5, .LCPI7_5@toc@l
+; CHECK-P8-NEXT:    addi r6, r6, .LCPI7_1@toc@l
+; CHECK-P8-NEXT:    lvx v0, 0, r4
+; CHECK-P8-NEXT:    addis r4, r2, .LCPI7_6@toc@ha
+; CHECK-P8-NEXT:    lvx v1, 0, r5
+; CHECK-P8-NEXT:    addis r5, r2, .LCPI7_7@toc@ha
+; CHECK-P8-NEXT:    lxvd2x vs0, 0, r6
+; CHECK-P8-NEXT:    addi r4, r4, .LCPI7_6@toc@l
+; CHECK-P8-NEXT:    addi r5, r5, .LCPI7_7@toc@l
+; CHECK-P8-NEXT:    vperm v3, v2, v2, v3
+; CHECK-P8-NEXT:    lvx v6, 0, r4
+; CHECK-P8-NEXT:    addis r4, r2, .LCPI7_8@toc@ha
+; CHECK-P8-NEXT:    lvx v7, 0, r5
+; CHECK-P8-NEXT:    vperm v4, v2, v2, v4
+; CHECK-P8-NEXT:    li r5, 96
+; CHECK-P8-NEXT:    addi r4, r4, .LCPI7_8@toc@l
+; CHECK-P8-NEXT:    vperm v5, v2, v2, v5
+; CHECK-P8-NEXT:    xxswapd v9, vs0
+; CHECK-P8-NEXT:    lvx v8, 0, r4
+; CHECK-P8-NEXT:    vperm v0, v2, v2, v0
 ; CHECK-P8-NEXT:    li r4, 112
-; CHECK-P8-NEXT:    stfd f31, -8(r1) # 8-byte Folded Spill
-; CHECK-P8-NEXT:    clrldi r6, r5, 56
-; CHECK-P8-NEXT:    rldicl r7, r5, 56, 56
-; CHECK-P8-NEXT:    extsb r6, r6
-; CHECK-P8-NEXT:    extsb r7, r7
-; CHECK-P8-NEXT:    mtvsrwa f0, r6
-; CHECK-P8-NEXT:    rldicl r6, r5, 40, 56
-; CHECK-P8-NEXT:    rldicl r8, r5, 48, 56
-; CHECK-P8-NEXT:    mtvsrwa f1, r7
-; CHECK-P8-NEXT:    rldicl r7, r5, 32, 56
-; CHECK-P8-NEXT:    extsb r6, r6
-; CHECK-P8-NEXT:    extsb r8, r8
-; CHECK-P8-NEXT:    extsb r7, r7
-; CHECK-P8-NEXT:    mtvsrwa f4, r6
-; CHECK-P8-NEXT:    rldicl r6, r5, 24, 56
-; CHECK-P8-NEXT:    mtvsrwa f3, r8
-; CHECK-P8-NEXT:    extsb r6, r6
-; CHECK-P8-NEXT:    mtvsrwa f5, r7
-; CHECK-P8-NEXT:    rldicl r7, r5, 16, 56
-; CHECK-P8-NEXT:    rldicl r5, r5, 8, 56
-; CHECK-P8-NEXT:    mfvsrd r8, f2
-; CHECK-P8-NEXT:    extsb r5, r5
-; CHECK-P8-NEXT:    mtvsrwa f2, r6
-; CHECK-P8-NEXT:    extsb r6, r7
-; CHECK-P8-NEXT:    mtvsrwa f6, r6
-; CHECK-P8-NEXT:    clrldi r6, r8, 56
-; CHECK-P8-NEXT:    mtvsrwa f7, r5
-; CHECK-P8-NEXT:    rldicl r5, r8, 56, 56
-; CHECK-P8-NEXT:    extsb r6, r6
-; CHECK-P8-NEXT:    mtvsrwa f8, r6
-; CHECK-P8-NEXT:    rldicl r6, r8, 48, 56
-; CHECK-P8-NEXT:    extsb r5, r5
-; CHECK-P8-NEXT:    mtvsrwa f9, r5
-; CHECK-P8-NEXT:    rldicl r5, r8, 40, 56
-; CHECK-P8-NEXT:    extsb r6, r6
-; CHECK-P8-NEXT:    mtvsrwa f10, r6
-; CHECK-P8-NEXT:    rldicl r6, r8, 32, 56
-; CHECK-P8-NEXT:    extsb r5, r5
-; CHECK-P8-NEXT:    mtvsrwa f11, r5
-; CHECK-P8-NEXT:    rldicl r5, r8, 24, 56
-; CHECK-P8-NEXT:    extsb r6, r6
-; CHECK-P8-NEXT:    mtvsrwa f12, r6
-; CHECK-P8-NEXT:    rldicl r6, r8, 16, 56
-; CHECK-P8-NEXT:    extsb r5, r5
-; CHECK-P8-NEXT:    xscvsxddp f6, f6
-; CHECK-P8-NEXT:    xscvsxddp f7, f7
-; CHECK-P8-NEXT:    mtvsrwa f13, r5
-; CHECK-P8-NEXT:    extsb r5, r6
-; CHECK-P8-NEXT:    mtvsrwa v2, r5
-; CHECK-P8-NEXT:    rldicl r5, r8, 8, 56
-; CHECK-P8-NEXT:    xscvsxddp f5, f5
-; CHECK-P8-NEXT:    extsb r5, r5
-; CHECK-P8-NEXT:    xscvsxddp f2, f2
-; CHECK-P8-NEXT:    xscvsxddp f0, f0
-; CHECK-P8-NEXT:    xscvsxddp f1, f1
-; CHECK-P8-NEXT:    xxmrghd vs6, vs7, vs6
-; CHECK-P8-NEXT:    mtvsrwa v3, r5
-; CHECK-P8-NEXT:    li r5, 64
-; CHECK-P8-NEXT:    xscvsxddp f3, f3
-; CHECK-P8-NEXT:    xscvsxddp f4, f4
-; CHECK-P8-NEXT:    xscvsxddp f31, v2
-; CHECK-P8-NEXT:    xxmrghd vs2, vs2, vs5
-; CHECK-P8-NEXT:    xscvsxddp f7, v3
-; CHECK-P8-NEXT:    xscvsxddp f8, f8
-; CHECK-P8-NEXT:    xxmrghd vs0, vs1, vs0
-; CHECK-P8-NEXT:    xscvsxddp f9, f9
-; CHECK-P8-NEXT:    xxswapd vs1, vs6
-; CHECK-P8-NEXT:    xscvsxddp f10, f10
+; CHECK-P8-NEXT:    vperm v1, v2, v2, v1
+; CHECK-P8-NEXT:    vperm v6, v2, v2, v6
+; CHECK-P8-NEXT:    vperm v7, v2, v2, v7
+; CHECK-P8-NEXT:    vperm v2, v2, v2, v8
+; CHECK-P8-NEXT:    vsld v3, v3, v9
+; CHECK-P8-NEXT:    vsld v0, v0, v9
+; CHECK-P8-NEXT:    vsld v1, v1, v9
+; CHECK-P8-NEXT:    vsld v6, v6, v9
+; CHECK-P8-NEXT:    vsld v7, v7, v9
+; CHECK-P8-NEXT:    vsld v2, v2, v9
+; CHECK-P8-NEXT:    vsrad v7, v7, v9
+; CHECK-P8-NEXT:    vsrad v2, v2, v9
+; CHECK-P8-NEXT:    vsld v4, v4, v9
+; CHECK-P8-NEXT:    vsld v5, v5, v9
+; CHECK-P8-NEXT:    vsrad v6, v6, v9
+; CHECK-P8-NEXT:    vsrad v0, v0, v9
+; CHECK-P8-NEXT:    vsrad v1, v1, v9
+; CHECK-P8-NEXT:    xvcvsxddp vs2, v7
+; CHECK-P8-NEXT:    xvcvsxddp vs3, v2
+; CHECK-P8-NEXT:    vsrad v3, v3, v9
+; CHECK-P8-NEXT:    vsrad v4, v4, v9
+; CHECK-P8-NEXT:    vsrad v5, v5, v9
+; CHECK-P8-NEXT:    xvcvsxddp vs4, v6
+; CHECK-P8-NEXT:    xvcvsxddp vs1, v1
 ; CHECK-P8-NEXT:    xxswapd vs2, vs2
-; CHECK-P8-NEXT:    xscvsxddp f12, f12
-; CHECK-P8-NEXT:    xxmrghd vs3, vs4, vs3
-; CHECK-P8-NEXT:    xscvsxddp f13, f13
-; CHECK-P8-NEXT:    stxvd2x vs1, r3, r4
-; CHECK-P8-NEXT:    li r4, 96
-; CHECK-P8-NEXT:    xscvsxddp f11, f11
-; CHECK-P8-NEXT:    xxmrghd vs6, vs7, vs31
+; CHECK-P8-NEXT:    xvcvsxddp vs5, v0
 ; CHECK-P8-NEXT:    xxswapd vs3, vs3
-; CHECK-P8-NEXT:    xxswapd vs0, vs0
-; CHECK-P8-NEXT:    xxmrghd vs4, vs9, vs8
-; CHECK-P8-NEXT:    stxvd2x vs2, r3, r4
-; CHECK-P8-NEXT:    li r4, 80
-; CHECK-P8-NEXT:    xxswapd vs2, vs6
+; CHECK-P8-NEXT:    xvcvsxddp vs0, v5
+; CHECK-P8-NEXT:    xvcvsxddp vs6, v3
+; CHECK-P8-NEXT:    xvcvsxddp vs7, v4
 ; CHECK-P8-NEXT:    stxvd2x vs3, r3, r4
+; CHECK-P8-NEXT:    li r4, 80
+; CHECK-P8-NEXT:    xxswapd vs4, vs4
+; CHECK-P8-NEXT:    stxvd2x vs2, r3, r5
+; CHECK-P8-NEXT:    li r5, 64
+; CHECK-P8-NEXT:    xxswapd vs1, vs1
+; CHECK-P8-NEXT:    xxswapd vs5, vs5
+; CHECK-P8-NEXT:    xxswapd vs0, vs0
+; CHECK-P8-NEXT:    stxvd2x vs4, r3, r4
 ; CHECK-P8-NEXT:    li r4, 48
-; CHECK-P8-NEXT:    stxvd2x vs0, r3, r5
+; CHECK-P8-NEXT:    xxswapd vs3, vs6
+; CHECK-P8-NEXT:    stxvd2x vs1, r3, r5
 ; CHECK-P8-NEXT:    li r5, 32
-; CHECK-P8-NEXT:    xxmrghd vs5, vs13, vs12
-; CHECK-P8-NEXT:    xxswapd vs4, vs4
-; CHECK-P8-NEXT:    xxmrghd vs1, vs11, vs10
-; CHECK-P8-NEXT:    stxvd2x vs2, r3, r4
+; CHECK-P8-NEXT:    xxswapd vs2, vs7
+; CHECK-P8-NEXT:    stxvd2x vs5, r3, r4
 ; CHECK-P8-NEXT:    li r4, 16
-; CHECK-P8-NEXT:    xxswapd vs5, vs5
-; CHECK-P8-NEXT:    xxswapd vs1, vs1
-; CHECK-P8-NEXT:    stxvd2x vs5, r3, r5
-; CHECK-P8-NEXT:    stxvd2x vs1, r3, r4
-; CHECK-P8-NEXT:    stxvd2x vs4, 0, r3
-; CHECK-P8-NEXT:    lfd f31, -8(r1) # 8-byte Folded Reload
+; CHECK-P8-NEXT:    stxvd2x vs0, r3, r5
+; CHECK-P8-NEXT:    stxvd2x vs2, r3, r4
+; CHECK-P8-NEXT:    stxvd2x vs3, 0, r3
 ; CHECK-P8-NEXT:    blr
 ;
 ; CHECK-P9-LABEL: test16elt_signed:
 ; CHECK-P9:       # %bb.0: # %entry
-; CHECK-P9-NEXT:    std r25, -72(r1) # 8-byte Folded Spill
-; CHECK-P9-NEXT:    std r26, -64(r1) # 8-byte Folded Spill
-; CHECK-P9-NEXT:    std r27, -56(r1) # 8-byte Folded Spill
-; CHECK-P9-NEXT:    std r28, -48(r1) # 8-byte Folded Spill
-; CHECK-P9-NEXT:    std r29, -40(r1) # 8-byte Folded Spill
-; CHECK-P9-NEXT:    li r4, 0
-; CHECK-P9-NEXT:    std r30, -32(r1) # 8-byte Folded Spill
-; CHECK-P9-NEXT:    li r5, 1
-; CHECK-P9-NEXT:    li r6, 2
-; CHECK-P9-NEXT:    li r7, 3
-; CHECK-P9-NEXT:    li r8, 4
-; CHECK-P9-NEXT:    li r9, 5
-; CHECK-P9-NEXT:    li r10, 6
-; CHECK-P9-NEXT:    li r11, 7
-; CHECK-P9-NEXT:    li r12, 8
-; CHECK-P9-NEXT:    li r0, 9
-; CHECK-P9-NEXT:    li r30, 10
-; CHECK-P9-NEXT:    li r29, 11
-; CHECK-P9-NEXT:    li r28, 12
-; CHECK-P9-NEXT:    li r27, 13
-; CHECK-P9-NEXT:    li r26, 14
-; CHECK-P9-NEXT:    li r25, 15
-; CHECK-P9-NEXT:    stfd f30, -16(r1) # 8-byte Folded Spill
-; CHECK-P9-NEXT:    stfd f31, -8(r1) # 8-byte Folded Spill
-; CHECK-P9-NEXT:    vextubrx r4, r4, v2
-; CHECK-P9-NEXT:    vextubrx r5, r5, v2
-; CHECK-P9-NEXT:    vextubrx r6, r6, v2
-; CHECK-P9-NEXT:    vextubrx r7, r7, v2
-; CHECK-P9-NEXT:    vextubrx r8, r8, v2
-; CHECK-P9-NEXT:    vextubrx r9, r9, v2
-; CHECK-P9-NEXT:    vextubrx r10, r10, v2
-; CHECK-P9-NEXT:    vextubrx r11, r11, v2
-; CHECK-P9-NEXT:    vextubrx r12, r12, v2
-; CHECK-P9-NEXT:    vextubrx r0, r0, v2
-; CHECK-P9-NEXT:    vextubrx r30, r30, v2
-; CHECK-P9-NEXT:    vextubrx r29, r29, v2
-; CHECK-P9-NEXT:    vextubrx r28, r28, v2
-; CHECK-P9-NEXT:    vextubrx r27, r27, v2
-; CHECK-P9-NEXT:    vextubrx r26, r26, v2
-; CHECK-P9-NEXT:    vextubrx r25, r25, v2
-; CHECK-P9-NEXT:    extsb r4, r4
-; CHECK-P9-NEXT:    extsb r5, r5
-; CHECK-P9-NEXT:    extsb r6, r6
-; CHECK-P9-NEXT:    extsb r7, r7
-; CHECK-P9-NEXT:    extsb r8, r8
-; CHECK-P9-NEXT:    extsb r9, r9
-; CHECK-P9-NEXT:    extsb r10, r10
-; CHECK-P9-NEXT:    extsb r11, r11
-; CHECK-P9-NEXT:    extsb r12, r12
-; CHECK-P9-NEXT:    extsb r0, r0
-; CHECK-P9-NEXT:    extsb r30, r30
-; CHECK-P9-NEXT:    extsb r29, r29
-; CHECK-P9-NEXT:    extsb r28, r28
-; CHECK-P9-NEXT:    extsb r27, r27
-; CHECK-P9-NEXT:    extsb r26, r26
-; CHECK-P9-NEXT:    extsb r25, r25
-; CHECK-P9-NEXT:    mtvsrwa f0, r4
-; CHECK-P9-NEXT:    mtvsrwa f1, r5
-; CHECK-P9-NEXT:    mtvsrwa f2, r6
-; CHECK-P9-NEXT:    mtvsrwa f3, r7
-; CHECK-P9-NEXT:    mtvsrwa f4, r8
-; CHECK-P9-NEXT:    mtvsrwa f5, r9
-; CHECK-P9-NEXT:    mtvsrwa f6, r10
-; CHECK-P9-NEXT:    mtvsrwa f7, r11
-; CHECK-P9-NEXT:    mtvsrwa f8, r12
-; CHECK-P9-NEXT:    mtvsrwa f9, r0
-; CHECK-P9-NEXT:    mtvsrwa f10, r30
-; CHECK-P9-NEXT:    ld r30, -32(r1) # 8-byte Folded Reload
-; CHECK-P9-NEXT:    mtvsrwa f11, r29
-; CHECK-P9-NEXT:    ld r29, -40(r1) # 8-byte Folded Reload
-; CHECK-P9-NEXT:    mtvsrwa f12, r28
-; CHECK-P9-NEXT:    ld r28, -48(r1) # 8-byte Folded Reload
-; CHECK-P9-NEXT:    mtvsrwa f13, r27
-; CHECK-P9-NEXT:    ld r27, -56(r1) # 8-byte Folded Reload
-; CHECK-P9-NEXT:    mtvsrwa v2, r26
-; CHECK-P9-NEXT:    ld r26, -64(r1) # 8-byte Folded Reload
-; CHECK-P9-NEXT:    mtvsrwa v3, r25
-; CHECK-P9-NEXT:    ld r25, -72(r1) # 8-byte Folded Reload
-; CHECK-P9-NEXT:    xscvsxddp f0, f0
-; CHECK-P9-NEXT:    xscvsxddp f1, f1
-; CHECK-P9-NEXT:    xscvsxddp f2, f2
-; CHECK-P9-NEXT:    xscvsxddp f3, f3
-; CHECK-P9-NEXT:    xscvsxddp f4, f4
-; CHECK-P9-NEXT:    xscvsxddp f5, f5
-; CHECK-P9-NEXT:    xscvsxddp f6, f6
-; CHECK-P9-NEXT:    xscvsxddp f7, f7
-; CHECK-P9-NEXT:    xscvsxddp f8, f8
-; CHECK-P9-NEXT:    xscvsxddp f9, f9
-; CHECK-P9-NEXT:    xscvsxddp f10, f10
-; CHECK-P9-NEXT:    xscvsxddp f11, f11
-; CHECK-P9-NEXT:    xscvsxddp f12, f12
-; CHECK-P9-NEXT:    xscvsxddp f13, f13
-; CHECK-P9-NEXT:    xscvsxddp f31, v2
-; CHECK-P9-NEXT:    xscvsxddp f30, v3
-; CHECK-P9-NEXT:    xxmrghd vs0, vs1, vs0
-; CHECK-P9-NEXT:    xxmrghd vs1, vs3, vs2
-; CHECK-P9-NEXT:    xxmrghd vs2, vs5, vs4
-; CHECK-P9-NEXT:    xxmrghd vs3, vs7, vs6
-; CHECK-P9-NEXT:    xxmrghd vs4, vs9, vs8
-; CHECK-P9-NEXT:    xxmrghd vs5, vs11, vs10
-; CHECK-P9-NEXT:    xxmrghd vs6, vs13, vs12
-; CHECK-P9-NEXT:    xxmrghd vs7, vs30, vs31
-; CHECK-P9-NEXT:    lfd f31, -8(r1) # 8-byte Folded Reload
-; CHECK-P9-NEXT:    lfd f30, -16(r1) # 8-byte Folded Reload
+; CHECK-P9-NEXT:    addis r4, r2, .LCPI7_0@toc@ha
+; CHECK-P9-NEXT:    addi r4, r4, .LCPI7_0@toc@l
+; CHECK-P9-NEXT:    lxvx v3, 0, r4
+; CHECK-P9-NEXT:    addis r4, r2, .LCPI7_1@toc@ha
+; CHECK-P9-NEXT:    addi r4, r4, .LCPI7_1@toc@l
+; CHECK-P9-NEXT:    vperm v3, v2, v2, v3
+; CHECK-P9-NEXT:    vextsb2d v3, v3
+; CHECK-P9-NEXT:    xvcvsxddp vs0, v3
+; CHECK-P9-NEXT:    lxvx v3, 0, r4
+; CHECK-P9-NEXT:    addis r4, r2, .LCPI7_2@toc@ha
+; CHECK-P9-NEXT:    addi r4, r4, .LCPI7_2@toc@l
+; CHECK-P9-NEXT:    vperm v3, v2, v2, v3
 ; CHECK-P9-NEXT:    stxv vs0, 0(r3)
-; CHECK-P9-NEXT:    stxv vs4, 64(r3)
-; CHECK-P9-NEXT:    stxv vs3, 48(r3)
-; CHECK-P9-NEXT:    stxv vs2, 32(r3)
+; CHECK-P9-NEXT:    vextsb2d v3, v3
+; CHECK-P9-NEXT:    xvcvsxddp vs1, v3
+; CHECK-P9-NEXT:    lxvx v3, 0, r4
+; CHECK-P9-NEXT:    addis r4, r2, .LCPI7_3@toc@ha
+; CHECK-P9-NEXT:    addi r4, r4, .LCPI7_3@toc@l
+; CHECK-P9-NEXT:    vperm v3, v2, v2, v3
 ; CHECK-P9-NEXT:    stxv vs1, 16(r3)
-; CHECK-P9-NEXT:    stxv vs7, 112(r3)
-; CHECK-P9-NEXT:    stxv vs6, 96(r3)
+; CHECK-P9-NEXT:    vextsb2d v3, v3
+; CHECK-P9-NEXT:    xvcvsxddp vs2, v3
+; CHECK-P9-NEXT:    lxvx v3, 0, r4
+; CHECK-P9-NEXT:    addis r4, r2, .LCPI7_4@toc@ha
+; CHECK-P9-NEXT:    addi r4, r4, .LCPI7_4@toc@l
+; CHECK-P9-NEXT:    vperm v3, v2, v2, v3
+; CHECK-P9-NEXT:    stxv vs2, 32(r3)
+; CHECK-P9-NEXT:    vextsb2d v3, v3
+; CHECK-P9-NEXT:    xvcvsxddp vs3, v3
+; CHECK-P9-NEXT:    lxvx v3, 0, r4
+; CHECK-P9-NEXT:    addis r4, r2, .LCPI7_5@toc@ha
+; CHECK-P9-NEXT:    addi r4, r4, .LCPI7_5@toc@l
+; CHECK-P9-NEXT:    vperm v3, v2, v2, v3
+; CHECK-P9-NEXT:    stxv vs3, 48(r3)
+; CHECK-P9-NEXT:    vextsb2d v3, v3
+; CHECK-P9-NEXT:    xvcvsxddp vs4, v3
+; CHECK-P9-NEXT:    lxvx v3, 0, r4
+; CHECK-P9-NEXT:    addis r4, r2, .LCPI7_6@toc@ha
+; CHECK-P9-NEXT:    addi r4, r4, .LCPI7_6@toc@l
+; CHECK-P9-NEXT:    vperm v3, v2, v2, v3
+; CHECK-P9-NEXT:    stxv vs4, 64(r3)
+; CHECK-P9-NEXT:    vextsb2d v3, v3
+; CHECK-P9-NEXT:    xvcvsxddp vs5, v3
+; CHECK-P9-NEXT:    lxvx v3, 0, r4
+; CHECK-P9-NEXT:    addis r4, r2, .LCPI7_7@toc@ha
+; CHECK-P9-NEXT:    addi r4, r4, .LCPI7_7@toc@l
+; CHECK-P9-NEXT:    vperm v3, v2, v2, v3
 ; CHECK-P9-NEXT:    stxv vs5, 80(r3)
+; CHECK-P9-NEXT:    vextsb2d v3, v3
+; CHECK-P9-NEXT:    xvcvsxddp vs6, v3
+; CHECK-P9-NEXT:    lxvx v3, 0, r4
+; CHECK-P9-NEXT:    vperm v2, v2, v2, v3
+; CHECK-P9-NEXT:    stxv vs6, 96(r3)
+; CHECK-P9-NEXT:    vextsb2d v2, v2
+; CHECK-P9-NEXT:    xvcvsxddp vs7, v2
+; CHECK-P9-NEXT:    stxv vs7, 112(r3)
 ; CHECK-P9-NEXT:    blr
 ;
 ; CHECK-BE-LABEL: test16elt_signed:
 ; CHECK-BE:       # %bb.0: # %entry
-; CHECK-BE-NEXT:    std r25, -72(r1) # 8-byte Folded Spill
-; CHECK-BE-NEXT:    std r26, -64(r1) # 8-byte Folded Spill
-; CHECK-BE-NEXT:    std r27, -56(r1) # 8-byte Folded Spill
-; CHECK-BE-NEXT:    std r28, -48(r1) # 8-byte Folded Spill
-; CHECK-BE-NEXT:    std r29, -40(r1) # 8-byte Folded Spill
-; CHECK-BE-NEXT:    li r4, 1
-; CHECK-BE-NEXT:    std r30, -32(r1) # 8-byte Folded Spill
-; CHECK-BE-NEXT:    li r5, 0
-; CHECK-BE-NEXT:    li r6, 3
-; CHECK-BE-NEXT:    li r7, 2
-; CHECK-BE-NEXT:    li r8, 5
-; CHECK-BE-NEXT:    li r9, 4
-; CHECK-BE-NEXT:    li r10, 7
-; CHECK-BE-NEXT:    li r11, 6
-; CHECK-BE-NEXT:    li r12, 9
-; CHECK-BE-NEXT:    li r0, 8
-; CHECK-BE-NEXT:    li r30, 11
-; CHECK-BE-NEXT:    li r29, 10
-; CHECK-BE-NEXT:    li r28, 13
-; CHECK-BE-NEXT:    li r27, 12
-; CHECK-BE-NEXT:    li r26, 15
-; CHECK-BE-NEXT:    li r25, 14
-; CHECK-BE-NEXT:    stfd f30, -16(r1) # 8-byte Folded Spill
-; CHECK-BE-NEXT:    stfd f31, -8(r1) # 8-byte Folded Spill
-; CHECK-BE-NEXT:    vextublx r4, r4, v2
-; CHECK-BE-NEXT:    vextublx r5, r5, v2
-; CHECK-BE-NEXT:    vextublx r6, r6, v2
-; CHECK-BE-NEXT:    vextublx r7, r7, v2
-; CHECK-BE-NEXT:    vextublx r8, r8, v2
-; CHECK-BE-NEXT:    vextublx r9, r9, v2
-; CHECK-BE-NEXT:    vextublx r10, r10, v2
-; CHECK-BE-NEXT:    vextublx r11, r11, v2
-; CHECK-BE-NEXT:    vextublx r12, r12, v2
-; CHECK-BE-NEXT:    vextublx r0, r0, v2
-; CHECK-BE-NEXT:    vextublx r30, r30, v2
-; CHECK-BE-NEXT:    vextublx r29, r29, v2
-; CHECK-BE-NEXT:    vextublx r28, r28, v2
-; CHECK-BE-NEXT:    vextublx r27, r27, v2
-; CHECK-BE-NEXT:    vextublx r26, r26, v2
-; CHECK-BE-NEXT:    vextublx r25, r25, v2
-; CHECK-BE-NEXT:    extsb r4, r4
-; CHECK-BE-NEXT:    extsb r5, r5
-; CHECK-BE-NEXT:    extsb r6, r6
-; CHECK-BE-NEXT:    extsb r7, r7
-; CHECK-BE-NEXT:    extsb r8, r8
-; CHECK-BE-NEXT:    extsb r9, r9
-; CHECK-BE-NEXT:    extsb r10, r10
-; CHECK-BE-NEXT:    extsb r11, r11
-; CHECK-BE-NEXT:    extsb r12, r12
-; CHECK-BE-NEXT:    extsb r0, r0
-; CHECK-BE-NEXT:    extsb r30, r30
-; CHECK-BE-NEXT:    extsb r29, r29
-; CHECK-BE-NEXT:    extsb r28, r28
-; CHECK-BE-NEXT:    extsb r27, r27
-; CHECK-BE-NEXT:    extsb r26, r26
-; CHECK-BE-NEXT:    extsb r25, r25
-; CHECK-BE-NEXT:    mtvsrwa f0, r4
-; CHECK-BE-NEXT:    mtvsrwa f1, r5
-; CHECK-BE-NEXT:    mtvsrwa f2, r6
-; CHECK-BE-NEXT:    mtvsrwa f3, r7
-; CHECK-BE-NEXT:    mtvsrwa f4, r8
-; CHECK-BE-NEXT:    mtvsrwa f5, r9
-; CHECK-BE-NEXT:    mtvsrwa f6, r10
-; CHECK-BE-NEXT:    mtvsrwa f7, r11
-; CHECK-BE-NEXT:    mtvsrwa f8, r12
-; CHECK-BE-NEXT:    mtvsrwa f9, r0
-; CHECK-BE-NEXT:    mtvsrwa f10, r30
-; CHECK-BE-NEXT:    ld r30, -32(r1) # 8-byte Folded Reload
-; CHECK-BE-NEXT:    mtvsrwa f11, r29
-; CHECK-BE-NEXT:    ld r29, -40(r1) # 8-byte Folded Reload
-; CHECK-BE-NEXT:    mtvsrwa f12, r28
-; CHECK-BE-NEXT:    ld r28, -48(r1) # 8-byte Folded Reload
-; CHECK-BE-NEXT:    mtvsrwa f13, r27
-; CHECK-BE-NEXT:    ld r27, -56(r1) # 8-byte Folded Reload
-; CHECK-BE-NEXT:    mtvsrwa v2, r26
-; CHECK-BE-NEXT:    ld r26, -64(r1) # 8-byte Folded Reload
-; CHECK-BE-NEXT:    mtvsrwa v3, r25
-; CHECK-BE-NEXT:    ld r25, -72(r1) # 8-byte Folded Reload
-; CHECK-BE-NEXT:    xscvsxddp f0, f0
-; CHECK-BE-NEXT:    xscvsxddp f1, f1
-; CHECK-BE-NEXT:    xscvsxddp f2, f2
-; CHECK-BE-NEXT:    xscvsxddp f3, f3
-; CHECK-BE-NEXT:    xscvsxddp f4, f4
-; CHECK-BE-NEXT:    xscvsxddp f5, f5
-; CHECK-BE-NEXT:    xscvsxddp f6, f6
-; CHECK-BE-NEXT:    xscvsxddp f7, f7
-; CHECK-BE-NEXT:    xscvsxddp f8, f8
-; CHECK-BE-NEXT:    xscvsxddp f9, f9
-; CHECK-BE-NEXT:    xscvsxddp f10, f10
-; CHECK-BE-NEXT:    xscvsxddp f11, f11
-; CHECK-BE-NEXT:    xscvsxddp f12, f12
-; CHECK-BE-NEXT:    xscvsxddp f13, f13
-; CHECK-BE-NEXT:    xscvsxddp f31, v2
-; CHECK-BE-NEXT:    xscvsxddp f30, v3
-; CHECK-BE-NEXT:    xxmrghd vs0, vs1, vs0
-; CHECK-BE-NEXT:    xxmrghd vs1, vs3, vs2
-; CHECK-BE-NEXT:    xxmrghd vs2, vs5, vs4
-; CHECK-BE-NEXT:    xxmrghd vs3, vs7, vs6
-; CHECK-BE-NEXT:    xxmrghd vs4, vs9, vs8
-; CHECK-BE-NEXT:    xxmrghd vs5, vs11, vs10
-; CHECK-BE-NEXT:    xxmrghd vs6, vs13, vs12
-; CHECK-BE-NEXT:    xxmrghd vs7, vs30, vs31
-; CHECK-BE-NEXT:    lfd f31, -8(r1) # 8-byte Folded Reload
-; CHECK-BE-NEXT:    lfd f30, -16(r1) # 8-byte Folded Reload
-; CHECK-BE-NEXT:    stxv vs0, 0(r3)
-; CHECK-BE-NEXT:    stxv vs4, 64(r3)
-; CHECK-BE-NEXT:    stxv vs3, 48(r3)
-; CHECK-BE-NEXT:    stxv vs2, 32(r3)
-; CHECK-BE-NEXT:    stxv vs1, 16(r3)
-; CHECK-BE-NEXT:    stxv vs7, 112(r3)
-; CHECK-BE-NEXT:    stxv vs6, 96(r3)
-; CHECK-BE-NEXT:    stxv vs5, 80(r3)
+; CHECK-BE-NEXT:    addis r4, r2, .LCPI7_0@toc@ha
+; CHECK-BE-NEXT:    addi r4, r4, .LCPI7_0@toc@l
+; CHECK-BE-NEXT:    lxvx v4, 0, r4
+; CHECK-BE-NEXT:    xxlxor v3, v3, v3
+; CHECK-BE-NEXT:    vperm v4, v3, v2, v4
+; CHECK-BE-NEXT:    addis r4, r2, .LCPI7_1@toc@ha
+; CHECK-BE-NEXT:    addi r4, r4, .LCPI7_1@toc@l
+; CHECK-BE-NEXT:    vextsb2d v4, v4
+; CHECK-BE-NEXT:    xvcvsxddp vs0, v4
+; CHECK-BE-NEXT:    lxvx v4, 0, r4
+; CHECK-BE-NEXT:    addis r4, r2, .LCPI7_2@toc@ha
+; CHECK-BE-NEXT:    addi r4, r4, .LCPI7_2@toc@l
+; CHECK-BE-NEXT:    vperm v4, v3, v2, v4
+; CHECK-BE-NEXT:    stxv vs0, 16(r3)
+; CHECK-BE-NEXT:    vextsb2d v4, v4
+; CHECK-BE-NEXT:    xvcvsxddp vs1, v4
+; CHECK-BE-NEXT:    lxvx v4, 0, r4
+; CHECK-BE-NEXT:    addis r4, r2, .LCPI7_3@toc@ha
+; CHECK-BE-NEXT:    addi r4, r4, .LCPI7_3@toc@l
+; CHECK-BE-NEXT:    vperm v4, v3, v2, v4
+; CHECK-BE-NEXT:    stxv vs1, 48(r3)
+; CHECK-BE-NEXT:    vextsb2d v4, v4
+; CHECK-BE-NEXT:    xvcvsxddp vs2, v4
+; CHECK-BE-NEXT:    lxvx v4, 0, r4
+; CHECK-BE-NEXT:    addis r4, r2, .LCPI7_4@toc@ha
+; CHECK-BE-NEXT:    addi r4, r4, .LCPI7_4@toc@l
+; CHECK-BE-NEXT:    vperm v3, v3, v2, v4
+; CHECK-BE-NEXT:    stxv vs2, 80(r3)
+; CHECK-BE-NEXT:    vextsb2d v3, v3
+; CHECK-BE-NEXT:    xvcvsxddp vs3, v3
+; CHECK-BE-NEXT:    lxvx v3, 0, r4
+; CHECK-BE-NEXT:    addis r4, r2, .LCPI7_5@toc@ha
+; CHECK-BE-NEXT:    addi r4, r4, .LCPI7_5@toc@l
+; CHECK-BE-NEXT:    vperm v3, v2, v2, v3
+; CHECK-BE-NEXT:    stxv vs3, 112(r3)
+; CHECK-BE-NEXT:    vextsb2d v3, v3
+; CHECK-BE-NEXT:    xvcvsxddp vs4, v3
+; CHECK-BE-NEXT:    lxvx v3, 0, r4
+; CHECK-BE-NEXT:    addis r4, r2, .LCPI7_6@toc@ha
+; CHECK-BE-NEXT:    addi r4, r4, .LCPI7_6@toc@l
+; CHECK-BE-NEXT:    vperm v3, v2, v2, v3
+; CHECK-BE-NEXT:    stxv vs4, 0(r3)
+; CHECK-BE-NEXT:    vextsb2d v3, v3
+; CHECK-BE-NEXT:    xvcvsxddp vs5, v3
+; CHECK-BE-NEXT:    lxvx v3, 0, r4
+; CHECK-BE-NEXT:    addis r4, r2, .LCPI7_7@toc@ha
+; CHECK-BE-NEXT:    addi r4, r4, .LCPI7_7@toc@l
+; CHECK-BE-NEXT:    vperm v3, v2, v2, v3
+; CHECK-BE-NEXT:    stxv vs5, 32(r3)
+; CHECK-BE-NEXT:    vextsb2d v3, v3
+; CHECK-BE-NEXT:    xvcvsxddp vs6, v3
+; CHECK-BE-NEXT:    lxvx v3, 0, r4
+; CHECK-BE-NEXT:    vperm v2, v2, v2, v3
+; CHECK-BE-NEXT:    stxv vs6, 64(r3)
+; CHECK-BE-NEXT:    vextsb2d v2, v2
+; CHECK-BE-NEXT:    xvcvsxddp vs7, v2
+; CHECK-BE-NEXT:    stxv vs7, 96(r3)
 ; CHECK-BE-NEXT:    blr
 entry:
   %0 = sitofp <16 x i8> %a to <16 x double>
diff --git a/test/CodeGen/PowerPC/vec_conv_i_to_fp_8byte_elts.ll b/test/CodeGen/PowerPC/vec_conv_i_to_fp_8byte_elts.ll
index 5bf15e3ae4f22bcbdcbf97b7670e73be9da9e99f..45e306ec6bf4f5b81ebc9c6df3e196ae86abd27d 100644
--- a/test/CodeGen/PowerPC/vec_conv_i_to_fp_8byte_elts.ll
+++ b/test/CodeGen/PowerPC/vec_conv_i_to_fp_8byte_elts.ll
@@ -163,58 +163,58 @@ define void @test16elt(<16 x double>* noalias nocapture sret %agg.result, <16 x
 ;
 ; CHECK-P9-LABEL: test16elt:
 ; CHECK-P9:       # %bb.0: # %entry
-; CHECK-P9-NEXT:    lxv v2, 48(r4)
-; CHECK-P9-NEXT:    lxv v3, 32(r4)
-; CHECK-P9-NEXT:    lxv v4, 16(r4)
-; CHECK-P9-NEXT:    lxv v5, 0(r4)
-; CHECK-P9-NEXT:    lxv v0, 112(r4)
-; CHECK-P9-NEXT:    lxv v1, 96(r4)
-; CHECK-P9-NEXT:    lxv v6, 80(r4)
-; CHECK-P9-NEXT:    lxv v7, 64(r4)
-; CHECK-P9-NEXT:    xvcvuxddp vs0, v5
-; CHECK-P9-NEXT:    xvcvuxddp vs1, v4
-; CHECK-P9-NEXT:    xvcvuxddp vs2, v3
-; CHECK-P9-NEXT:    xvcvuxddp vs3, v2
-; CHECK-P9-NEXT:    xvcvuxddp vs4, v7
-; CHECK-P9-NEXT:    xvcvuxddp vs5, v6
-; CHECK-P9-NEXT:    xvcvuxddp vs6, v1
-; CHECK-P9-NEXT:    xvcvuxddp vs7, v0
-; CHECK-P9-NEXT:    stxv vs3, 48(r3)
-; CHECK-P9-NEXT:    stxv vs2, 32(r3)
-; CHECK-P9-NEXT:    stxv vs1, 16(r3)
-; CHECK-P9-NEXT:    stxv vs0, 0(r3)
+; CHECK-P9-NEXT:    lxv v2, 112(r4)
+; CHECK-P9-NEXT:    lxv v3, 96(r4)
+; CHECK-P9-NEXT:    lxv v4, 80(r4)
+; CHECK-P9-NEXT:    lxv v5, 64(r4)
+; CHECK-P9-NEXT:    lxv v0, 48(r4)
+; CHECK-P9-NEXT:    xvcvuxddp vs3, v0
+; CHECK-P9-NEXT:    lxv v1, 32(r4)
+; CHECK-P9-NEXT:    lxv v6, 16(r4)
+; CHECK-P9-NEXT:    lxv v7, 0(r4)
+; CHECK-P9-NEXT:    xvcvuxddp vs0, v7
+; CHECK-P9-NEXT:    xvcvuxddp vs1, v6
+; CHECK-P9-NEXT:    xvcvuxddp vs2, v1
+; CHECK-P9-NEXT:    xvcvuxddp vs4, v5
+; CHECK-P9-NEXT:    xvcvuxddp vs5, v4
+; CHECK-P9-NEXT:    xvcvuxddp vs6, v3
+; CHECK-P9-NEXT:    xvcvuxddp vs7, v2
 ; CHECK-P9-NEXT:    stxv vs7, 112(r3)
 ; CHECK-P9-NEXT:    stxv vs6, 96(r3)
 ; CHECK-P9-NEXT:    stxv vs5, 80(r3)
 ; CHECK-P9-NEXT:    stxv vs4, 64(r3)
+; CHECK-P9-NEXT:    stxv vs3, 48(r3)
+; CHECK-P9-NEXT:    stxv vs2, 32(r3)
+; CHECK-P9-NEXT:    stxv vs1, 16(r3)
+; CHECK-P9-NEXT:    stxv vs0, 0(r3)
 ; CHECK-P9-NEXT:    blr
 ;
 ; CHECK-BE-LABEL: test16elt:
 ; CHECK-BE:       # %bb.0: # %entry
-; CHECK-BE-NEXT:    lxv v2, 48(r4)
-; CHECK-BE-NEXT:    lxv v3, 32(r4)
-; CHECK-BE-NEXT:    lxv v4, 16(r4)
-; CHECK-BE-NEXT:    lxv v5, 0(r4)
-; CHECK-BE-NEXT:    lxv v0, 112(r4)
-; CHECK-BE-NEXT:    lxv v1, 96(r4)
-; CHECK-BE-NEXT:    lxv v6, 80(r4)
-; CHECK-BE-NEXT:    lxv v7, 64(r4)
-; CHECK-BE-NEXT:    xvcvuxddp vs0, v5
-; CHECK-BE-NEXT:    xvcvuxddp vs1, v4
-; CHECK-BE-NEXT:    xvcvuxddp vs2, v3
-; CHECK-BE-NEXT:    xvcvuxddp vs3, v2
-; CHECK-BE-NEXT:    xvcvuxddp vs4, v7
-; CHECK-BE-NEXT:    xvcvuxddp vs5, v6
-; CHECK-BE-NEXT:    xvcvuxddp vs6, v1
-; CHECK-BE-NEXT:    xvcvuxddp vs7, v0
-; CHECK-BE-NEXT:    stxv vs3, 48(r3)
-; CHECK-BE-NEXT:    stxv vs2, 32(r3)
-; CHECK-BE-NEXT:    stxv vs1, 16(r3)
-; CHECK-BE-NEXT:    stxv vs0, 0(r3)
+; CHECK-BE-NEXT:    lxv v2, 112(r4)
+; CHECK-BE-NEXT:    lxv v3, 96(r4)
+; CHECK-BE-NEXT:    lxv v4, 80(r4)
+; CHECK-BE-NEXT:    lxv v5, 64(r4)
+; CHECK-BE-NEXT:    lxv v0, 48(r4)
+; CHECK-BE-NEXT:    xvcvuxddp vs3, v0
+; CHECK-BE-NEXT:    lxv v1, 32(r4)
+; CHECK-BE-NEXT:    lxv v6, 16(r4)
+; CHECK-BE-NEXT:    lxv v7, 0(r4)
+; CHECK-BE-NEXT:    xvcvuxddp vs0, v7
+; CHECK-BE-NEXT:    xvcvuxddp vs1, v6
+; CHECK-BE-NEXT:    xvcvuxddp vs2, v1
+; CHECK-BE-NEXT:    xvcvuxddp vs4, v5
+; CHECK-BE-NEXT:    xvcvuxddp vs5, v4
+; CHECK-BE-NEXT:    xvcvuxddp vs6, v3
+; CHECK-BE-NEXT:    xvcvuxddp vs7, v2
 ; CHECK-BE-NEXT:    stxv vs7, 112(r3)
 ; CHECK-BE-NEXT:    stxv vs6, 96(r3)
 ; CHECK-BE-NEXT:    stxv vs5, 80(r3)
 ; CHECK-BE-NEXT:    stxv vs4, 64(r3)
+; CHECK-BE-NEXT:    stxv vs3, 48(r3)
+; CHECK-BE-NEXT:    stxv vs2, 32(r3)
+; CHECK-BE-NEXT:    stxv vs1, 16(r3)
+; CHECK-BE-NEXT:    stxv vs0, 0(r3)
 ; CHECK-BE-NEXT:    blr
 entry:
   %a = load <16 x i64>, <16 x i64>* %0, align 128
@@ -377,58 +377,58 @@ define void @test16elt_signed(<16 x double>* noalias nocapture sret %agg.result,
 ;
 ; CHECK-P9-LABEL: test16elt_signed:
 ; CHECK-P9:       # %bb.0: # %entry
-; CHECK-P9-NEXT:    lxv v2, 48(r4)
-; CHECK-P9-NEXT:    lxv v3, 32(r4)
-; CHECK-P9-NEXT:    lxv v4, 16(r4)
-; CHECK-P9-NEXT:    lxv v5, 0(r4)
-; CHECK-P9-NEXT:    lxv v0, 112(r4)
-; CHECK-P9-NEXT:    lxv v1, 96(r4)
-; CHECK-P9-NEXT:    lxv v6, 80(r4)
-; CHECK-P9-NEXT:    lxv v7, 64(r4)
-; CHECK-P9-NEXT:    xvcvsxddp vs0, v5
-; CHECK-P9-NEXT:    xvcvsxddp vs1, v4
-; CHECK-P9-NEXT:    xvcvsxddp vs2, v3
-; CHECK-P9-NEXT:    xvcvsxddp vs3, v2
-; CHECK-P9-NEXT:    xvcvsxddp vs4, v7
-; CHECK-P9-NEXT:    xvcvsxddp vs5, v6
-; CHECK-P9-NEXT:    xvcvsxddp vs6, v1
-; CHECK-P9-NEXT:    xvcvsxddp vs7, v0
-; CHECK-P9-NEXT:    stxv vs3, 48(r3)
-; CHECK-P9-NEXT:    stxv vs2, 32(r3)
-; CHECK-P9-NEXT:    stxv vs1, 16(r3)
-; CHECK-P9-NEXT:    stxv vs0, 0(r3)
+; CHECK-P9-NEXT:    lxv v2, 112(r4)
+; CHECK-P9-NEXT:    lxv v3, 96(r4)
+; CHECK-P9-NEXT:    lxv v4, 80(r4)
+; CHECK-P9-NEXT:    lxv v5, 64(r4)
+; CHECK-P9-NEXT:    lxv v0, 48(r4)
+; CHECK-P9-NEXT:    xvcvsxddp vs3, v0
+; CHECK-P9-NEXT:    lxv v1, 32(r4)
+; CHECK-P9-NEXT:    lxv v6, 16(r4)
+; CHECK-P9-NEXT:    lxv v7, 0(r4)
+; CHECK-P9-NEXT:    xvcvsxddp vs0, v7
+; CHECK-P9-NEXT:    xvcvsxddp vs1, v6
+; CHECK-P9-NEXT:    xvcvsxddp vs2, v1
+; CHECK-P9-NEXT:    xvcvsxddp vs4, v5
+; CHECK-P9-NEXT:    xvcvsxddp vs5, v4
+; CHECK-P9-NEXT:    xvcvsxddp vs6, v3
+; CHECK-P9-NEXT:    xvcvsxddp vs7, v2
 ; CHECK-P9-NEXT:    stxv vs7, 112(r3)
 ; CHECK-P9-NEXT:    stxv vs6, 96(r3)
 ; CHECK-P9-NEXT:    stxv vs5, 80(r3)
 ; CHECK-P9-NEXT:    stxv vs4, 64(r3)
+; CHECK-P9-NEXT:    stxv vs3, 48(r3)
+; CHECK-P9-NEXT:    stxv vs2, 32(r3)
+; CHECK-P9-NEXT:    stxv vs1, 16(r3)
+; CHECK-P9-NEXT:    stxv vs0, 0(r3)
 ; CHECK-P9-NEXT:    blr
 ;
 ; CHECK-BE-LABEL: test16elt_signed:
 ; CHECK-BE:       # %bb.0: # %entry
-; CHECK-BE-NEXT:    lxv v2, 48(r4)
-; CHECK-BE-NEXT:    lxv v3, 32(r4)
-; CHECK-BE-NEXT:    lxv v4, 16(r4)
-; CHECK-BE-NEXT:    lxv v5, 0(r4)
-; CHECK-BE-NEXT:    lxv v0, 112(r4)
-; CHECK-BE-NEXT:    lxv v1, 96(r4)
-; CHECK-BE-NEXT:    lxv v6, 80(r4)
-; CHECK-BE-NEXT:    lxv v7, 64(r4)
-; CHECK-BE-NEXT:    xvcvsxddp vs0, v5
-; CHECK-BE-NEXT:    xvcvsxddp vs1, v4
-; CHECK-BE-NEXT:    xvcvsxddp vs2, v3
-; CHECK-BE-NEXT:    xvcvsxddp vs3, v2
-; CHECK-BE-NEXT:    xvcvsxddp vs4, v7
-; CHECK-BE-NEXT:    xvcvsxddp vs5, v6
-; CHECK-BE-NEXT:    xvcvsxddp vs6, v1
-; CHECK-BE-NEXT:    xvcvsxddp vs7, v0
-; CHECK-BE-NEXT:    stxv vs3, 48(r3)
-; CHECK-BE-NEXT:    stxv vs2, 32(r3)
-; CHECK-BE-NEXT:    stxv vs1, 16(r3)
-; CHECK-BE-NEXT:    stxv vs0, 0(r3)
+; CHECK-BE-NEXT:    lxv v2, 112(r4)
+; CHECK-BE-NEXT:    lxv v3, 96(r4)
+; CHECK-BE-NEXT:    lxv v4, 80(r4)
+; CHECK-BE-NEXT:    lxv v5, 64(r4)
+; CHECK-BE-NEXT:    lxv v0, 48(r4)
+; CHECK-BE-NEXT:    xvcvsxddp vs3, v0
+; CHECK-BE-NEXT:    lxv v1, 32(r4)
+; CHECK-BE-NEXT:    lxv v6, 16(r4)
+; CHECK-BE-NEXT:    lxv v7, 0(r4)
+; CHECK-BE-NEXT:    xvcvsxddp vs0, v7
+; CHECK-BE-NEXT:    xvcvsxddp vs1, v6
+; CHECK-BE-NEXT:    xvcvsxddp vs2, v1
+; CHECK-BE-NEXT:    xvcvsxddp vs4, v5
+; CHECK-BE-NEXT:    xvcvsxddp vs5, v4
+; CHECK-BE-NEXT:    xvcvsxddp vs6, v3
+; CHECK-BE-NEXT:    xvcvsxddp vs7, v2
 ; CHECK-BE-NEXT:    stxv vs7, 112(r3)
 ; CHECK-BE-NEXT:    stxv vs6, 96(r3)
 ; CHECK-BE-NEXT:    stxv vs5, 80(r3)
 ; CHECK-BE-NEXT:    stxv vs4, 64(r3)
+; CHECK-BE-NEXT:    stxv vs3, 48(r3)
+; CHECK-BE-NEXT:    stxv vs2, 32(r3)
+; CHECK-BE-NEXT:    stxv vs1, 16(r3)
+; CHECK-BE-NEXT:    stxv vs0, 0(r3)
 ; CHECK-BE-NEXT:    blr
 entry:
   %a = load <16 x i64>, <16 x i64>* %0, align 128
diff --git a/test/CodeGen/PowerPC/vsx-p9.ll b/test/CodeGen/PowerPC/vsx-p9.ll
index d7bea34f7546c85cb1a675fcba7a711c017ac1f0..0dec6ddbcb44d6574b1cc49a2312837d07e0eec3 100644
--- a/test/CodeGen/PowerPC/vsx-p9.ll
+++ b/test/CodeGen/PowerPC/vsx-p9.ll
@@ -37,7 +37,7 @@ entry:
   %add.i = add <16 x i8> %1, %0
   tail call void (...) @sink(<16 x i8> %add.i)
 ; CHECK: lxvx 34, 0, 3
-; CHECK: lxvx 35, 0, 4
+; CHECK: lxvx 35, 0, 3 
 ; CHECK: vaddubm 2, 3, 2
 ; CHECK: stxv 34,
 ; CHECK: bl sink
@@ -46,7 +46,7 @@ entry:
   %add.i22 = add <16 x i8> %3, %2
   tail call void (...) @sink(<16 x i8> %add.i22)
 ; CHECK: lxvx 34, 0, 3
-; CHECK: lxvx 35, 0, 4
+; CHECK: lxvx 35, 0, 3 
 ; CHECK: vaddubm 2, 3, 2
 ; CHECK: stxv 34,
 ; CHECK: bl sink
@@ -55,7 +55,7 @@ entry:
   %add.i21 = add <8 x i16> %5, %4
   tail call void (...) @sink(<8 x i16> %add.i21)
 ; CHECK: lxvx 34, 0, 3
-; CHECK: lxvx 35, 0, 4
+; CHECK: lxvx 35, 0, 3 
 ; CHECK: vadduhm 2, 3, 2
 ; CHECK: stxv 34,
 ; CHECK: bl sink
@@ -64,7 +64,7 @@ entry:
   %add.i20 = add <8 x i16> %7, %6
   tail call void (...) @sink(<8 x i16> %add.i20)
 ; CHECK: lxvx 34, 0, 3
-; CHECK: lxvx 35, 0, 4
+; CHECK: lxvx 35, 0, 3 
 ; CHECK: vadduhm 2, 3, 2
 ; CHECK: stxv 34,
 ; CHECK: bl sink
@@ -73,7 +73,7 @@ entry:
   %add.i19 = add <4 x i32> %9, %8
   tail call void (...) @sink(<4 x i32> %add.i19)
 ; CHECK: lxvx 34, 0, 3
-; CHECK: lxvx 35, 0, 4
+; CHECK: lxvx 35, 0, 3 
 ; CHECK: vadduwm 2, 3, 2
 ; CHECK: stxv 34,
 ; CHECK: bl sink
@@ -82,7 +82,7 @@ entry:
   %add.i18 = add <4 x i32> %11, %10
   tail call void (...) @sink(<4 x i32> %add.i18)
 ; CHECK: lxvx 34, 0, 3
-; CHECK: lxvx 35, 0, 4
+; CHECK: lxvx 35, 0, 3 
 ; CHECK: vadduwm 2, 3, 2
 ; CHECK: stxv 34,
 ; CHECK: bl sink
@@ -91,7 +91,7 @@ entry:
   %add.i17 = add <2 x i64> %13, %12
   tail call void (...) @sink(<2 x i64> %add.i17)
 ; CHECK: lxvx 34, 0, 3
-; CHECK: lxvx 35, 0, 4
+; CHECK: lxvx 35, 0, 3 
 ; CHECK: vaddudm 2, 3, 2
 ; CHECK: stxv 34,
 ; CHECK: bl sink
@@ -100,7 +100,7 @@ entry:
   %add.i16 = add <2 x i64> %15, %14
   tail call void (...) @sink(<2 x i64> %add.i16)
 ; CHECK: lxvx 34, 0, 3
-; CHECK: lxvx 35, 0, 4
+; CHECK: lxvx 35, 0, 3 
 ; CHECK: vaddudm 2, 3, 2
 ; CHECK: stxv 34,
 ; CHECK: bl sink
@@ -109,7 +109,7 @@ entry:
   %add.i15 = add <1 x i128> %17, %16
   tail call void (...) @sink(<1 x i128> %add.i15)
 ; CHECK: lxvx 34, 0, 3
-; CHECK: lxvx 35, 0, 4
+; CHECK: lxvx 35, 0, 3 
 ; CHECK: vadduqm 2, 3, 2
 ; CHECK: stxv 34,
 ; CHECK: bl sink
@@ -118,7 +118,7 @@ entry:
   %add.i14 = add <1 x i128> %19, %18
   tail call void (...) @sink(<1 x i128> %add.i14)
 ; CHECK: lxvx 34, 0, 3
-; CHECK: lxvx 35, 0, 4
+; CHECK: lxvx 35, 0, 3 
 ; CHECK: vadduqm 2, 3, 2
 ; CHECK: stxv 34,
 ; CHECK: bl sink
@@ -127,7 +127,7 @@ entry:
   %add.i13 = fadd <4 x float> %20, %21
   tail call void (...) @sink(<4 x float> %add.i13)
 ; CHECK: lxvx 0, 0, 3
-; CHECK: lxvx 1, 0, 4
+; CHECK: lxvx 1, 0, 3 
 ; CHECK: xvaddsp 34, 0, 1
 ; CHECK: stxv 34,
 ; CHECK: bl sink
@@ -136,7 +136,7 @@ entry:
   %add.i12 = fadd <2 x double> %22, %23
   tail call void (...) @sink(<2 x double> %add.i12)
 ; CHECK: lxvx 0, 0, 3
-; CHECK: lxvx 1, 0, 4
+; CHECK: lxvx 1, 0, 3 
 ; CHECK: xvadddp 0, 0, 1
 ; CHECK: stxv 0,
 ; CHECK: bl sink
diff --git a/test/CodeGen/PowerPC/vsx-self-copy.ll b/test/CodeGen/PowerPC/vsx-self-copy.ll
index 787ac4b77164b2fc04da2a9042cc9a63ab9284e9..6a6008d23928e3cfe495474a8233a52e8c948d17 100644
--- a/test/CodeGen/PowerPC/vsx-self-copy.ll
+++ b/test/CodeGen/PowerPC/vsx-self-copy.ll
@@ -1,5 +1,5 @@
-; RUN: llc -mcpu=pwr7 -mattr=+vsx < %s | FileCheck %s
-; RUN: llc -mcpu=pwr7 -mattr=+vsx -fast-isel -O0 < %s | FileCheck %s
+; RUN: llc -mcpu=pwr7 -mattr=+vsx < %s -verify-machineinstrs | FileCheck %s
+; RUN: llc -mcpu=pwr7 -mattr=+vsx -fast-isel -O0 < %s -verify-machineinstrs | FileCheck %s
 target datalayout = "E-m:e-i64:64-n32:64"
 target triple = "powerpc64-unknown-linux-gnu"
 
diff --git a/test/CodeGen/PowerPC/vsx-spill.ll b/test/CodeGen/PowerPC/vsx-spill.ll
index d46664ba98d97d36d02c8e69d49a900995e0910e..6c87861be15653131d178b60ebd5528f237a0b04 100644
--- a/test/CodeGen/PowerPC/vsx-spill.ll
+++ b/test/CodeGen/PowerPC/vsx-spill.ll
@@ -93,8 +93,8 @@ entry:
 
 ; CHECK-P9-REG-LABEL: foo3
 ; CHECK-P9-REG: stdu r1, -400(r1)
-; CHECK-P9-REG: lfd f30, 384(r1)
-; CHECK-P9-REG: xsadddp f1, f0, f0
+; CHECK-P9-REG-DAG: lfd f30, 384(r1)
+; CHECK-P9-REG-DAG: xsadddp f1, f0, f0
 
 ; CHECK-P9-FISL-LABEL: foo3
 ; CHECK-P9-FISL: stdu r1, -400(r1)
diff --git a/test/CodeGen/PowerPC/vsx.ll b/test/CodeGen/PowerPC/vsx.ll
index d6a5ed37040f5f2da50be591401f9820bd308593..4b1c2ed9d7a8166cd93d42b970cacce692aab062 100644
--- a/test/CodeGen/PowerPC/vsx.ll
+++ b/test/CodeGen/PowerPC/vsx.ll
@@ -1099,20 +1099,15 @@ define <2 x double> @test69(<2 x i16> %a) {
   ret <2 x double> %w
 
 ; CHECK-LABEL: @test69
-; CHECK-DAG: lfiwax f0, 0, r3
-; CHECK-DAG: lfiwax f1, 0, r3
-; CHECK-DAG: xscvsxddp f0, f0
-; CHECK-DAG: xscvsxddp f1, f1
-; CHECK: xxmrghd v2, vs1, vs0
+; CHECK-DAG: lxvd2x v2, 0, r3
+; CHECK-DAG: xvcvsxddp v2, v2
 ; CHECK: blr
 
 ; CHECK-LE-LABEL: @test69
-; CHECK-LE: mfvsrd
-; CHECK-LE: mtvsrwa
-; CHECK-LE: mtvsrwa
-; CHECK-LE: xscvsxddp
-; CHECK-LE: xscvsxddp
-; CHECK-LE: xxmrghd
+; CHECK-LE: vperm
+; CHECK-LE: vsld
+; CHECK-LE: vsrad
+; CHECK-LE: xvcvsxddp v2, v2
 ; CHECK-LE: blr
 }
 
@@ -1122,20 +1117,15 @@ define <2 x double> @test70(<2 x i8> %a) {
   ret <2 x double> %w
 
 ; CHECK-LABEL: @test70
-; CHECK-DAG: lfiwax f0, 0, r3
-; CHECK-DAG: lfiwax f1, 0, r3
-; CHECK-DAG: xscvsxddp f0, f0
-; CHECK-DAG: xscvsxddp f1, f1
-; CHECK: xxmrghd v2, vs1, vs0
+; CHECK-DAG: lxvd2x v2, 0, r3
+; CHECK-DAG: xvcvsxddp v2, v2
 ; CHECK: blr
 
 ; CHECK-LE-LABEL: @test70
-; CHECK-LE: mfvsrd
-; CHECK-LE: mtvsrwa
-; CHECK-LE: mtvsrwa
-; CHECK-LE: xscvsxddp
-; CHECK-LE: xscvsxddp
-; CHECK-LE: xxmrghd
+; CHECK-LE: vperm
+; CHECK-LE: vsld
+; CHECK-LE: vsrad
+; CHECK-LE: xvcvsxddp v2, v2
 ; CHECK-LE: blr
 }
 
diff --git a/test/CodeGen/PowerPC/vsx_insert_extract_le.ll b/test/CodeGen/PowerPC/vsx_insert_extract_le.ll
index ef7d8f3500756d92ccfb710f9266d01346c8c913..0c081f744c60769f9e72f7830d8bf869a5730a46 100644
--- a/test/CodeGen/PowerPC/vsx_insert_extract_le.ll
+++ b/test/CodeGen/PowerPC/vsx_insert_extract_le.ll
@@ -31,10 +31,10 @@ define <2 x double> @testi0(<2 x double>* %p1, double* %p2) {
 ;
 ; CHECK-P9-LABEL: testi0:
 ; CHECK-P9:       # %bb.0:
-; CHECK-P9-NEXT:    lfd f0, 0(r4)
-; CHECK-P9-NEXT:    lxv vs1, 0(r3)
-; CHECK-P9-NEXT:    xxpermdi vs0, f0, f0, 2
-; CHECK-P9-NEXT:    xxpermdi v2, vs1, vs0, 1
+; CHECK-P9-NEXT:    lfd [[REG:f[0-9]+]], 0(r4)
+; CHECK-P9-NEXT:    lxv [[REG1:vs[0-9]+]], 0(r3)
+; CHECK-P9-NEXT:    xxpermdi [[REG2:vs[0-9]+]], [[REG]], [[REG]], 2
+; CHECK-P9-NEXT:    xxpermdi v2, [[REG1]], [[REG2]], 1
 ; CHECK-P9-NEXT:    blr
   %v = load <2 x double>, <2 x double>* %p1
   %s = load double, double* %p2
@@ -65,10 +65,10 @@ define <2 x double> @testi1(<2 x double>* %p1, double* %p2) {
 ;
 ; CHECK-P9-LABEL: testi1:
 ; CHECK-P9:       # %bb.0:
-; CHECK-P9-NEXT:    lfd f0, 0(r4)
-; CHECK-P9-NEXT:    lxv vs1, 0(r3)
-; CHECK-P9-NEXT:    xxpermdi vs0, f0, f0, 2
-; CHECK-P9-NEXT:    xxmrgld v2, vs0, vs1
+; CHECK-P9-NEXT:    lfd [[REG:f[0-9]+]], 0(r4)
+; CHECK-P9-NEXT:    lxv [[REG1:vs[0-9]+]], 0(r3)
+; CHECK-P9-NEXT:    xxpermdi [[REG2:vs[0-9]+]], [[REG]], [[REG]], 2
+; CHECK-P9-NEXT:    xxmrgld v2, [[REG2]], [[REG1]] 
 ; CHECK-P9-NEXT:    blr
   %v = load <2 x double>, <2 x double>* %p1
   %s = load double, double* %p2
diff --git a/test/CodeGen/RISCV/alu32.ll b/test/CodeGen/RISCV/alu32.ll
index 3776e53c30692af3363c5d9ae27b6f496be103e5..d6f667c9c6f2bf49656ddbfc7a92f4c16c26af04 100644
--- a/test/CodeGen/RISCV/alu32.ll
+++ b/test/CodeGen/RISCV/alu32.ll
@@ -181,7 +181,7 @@ define i32 @sll(i32 %a, i32 %b) nounwind {
 ;
 ; RV64I-LABEL: sll:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    sll a0, a0, a1
+; RV64I-NEXT:    sllw a0, a0, a1
 ; RV64I-NEXT:    ret
   %1 = shl i32 %a, %b
   ret i32 %1
@@ -235,8 +235,6 @@ define i32 @xor(i32 %a, i32 %b) nounwind {
   ret i32 %1
 }
 
-; TODO: should select srlw for RV64.
-
 define i32 @srl(i32 %a, i32 %b) nounwind {
 ; RV32I-LABEL: srl:
 ; RV32I:       # %bb.0:
@@ -245,16 +243,12 @@ define i32 @srl(i32 %a, i32 %b) nounwind {
 ;
 ; RV64I-LABEL: srl:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    slli a0, a0, 32
-; RV64I-NEXT:    srli a0, a0, 32
-; RV64I-NEXT:    srl a0, a0, a1
+; RV64I-NEXT:    srlw a0, a0, a1
 ; RV64I-NEXT:    ret
   %1 = lshr i32 %a, %b
   ret i32 %1
 }
 
-; TODO: should select sraw for RV64.
-
 define i32 @sra(i32 %a, i32 %b) nounwind {
 ; RV32I-LABEL: sra:
 ; RV32I:       # %bb.0:
@@ -263,8 +257,7 @@ define i32 @sra(i32 %a, i32 %b) nounwind {
 ;
 ; RV64I-LABEL: sra:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    sext.w a0, a0
-; RV64I-NEXT:    sra a0, a0, a1
+; RV64I-NEXT:    sraw a0, a0, a1
 ; RV64I-NEXT:    ret
   %1 = ashr i32 %a, %b
   ret i32 %1
diff --git a/test/CodeGen/RISCV/alu64.ll b/test/CodeGen/RISCV/alu64.ll
index 021211b6f365d3301afd95adcc0c5cbeffe1ecbb..e66d1d62e555d08e11238a7eef4632332dabb151 100644
--- a/test/CodeGen/RISCV/alu64.ll
+++ b/test/CodeGen/RISCV/alu64.ll
@@ -444,13 +444,10 @@ define signext i32 @subw(i32 signext %a, i32 signext %b) {
   ret i32 %1
 }
 
-; TODO: should select sllw for RV64.
-
 define signext i32 @sllw(i32 signext %a, i32 zeroext %b) {
 ; RV64I-LABEL: sllw:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    sll a0, a0, a1
-; RV64I-NEXT:    sext.w a0, a0
+; RV64I-NEXT:    sllw a0, a0, a1
 ; RV64I-NEXT:    ret
 ;
 ; RV32I-LABEL: sllw:
@@ -461,15 +458,10 @@ define signext i32 @sllw(i32 signext %a, i32 zeroext %b) {
   ret i32 %1
 }
 
-; TODO: should select srlw for RV64.
-
 define signext i32 @srlw(i32 signext %a, i32 zeroext %b) {
 ; RV64I-LABEL: srlw:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    slli a0, a0, 32
-; RV64I-NEXT:    srli a0, a0, 32
-; RV64I-NEXT:    srl a0, a0, a1
-; RV64I-NEXT:    sext.w a0, a0
+; RV64I-NEXT:    srlw a0, a0, a1
 ; RV64I-NEXT:    ret
 ;
 ; RV32I-LABEL: srlw:
@@ -480,13 +472,10 @@ define signext i32 @srlw(i32 signext %a, i32 zeroext %b) {
   ret i32 %1
 }
 
-; TODO: should select sraw for RV64.
-
 define signext i32 @sraw(i64 %a, i32 zeroext %b) {
 ; RV64I-LABEL: sraw:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    sext.w a0, a0
-; RV64I-NEXT:    sra a0, a0, a1
+; RV64I-NEXT:    sraw a0, a0, a1
 ; RV64I-NEXT:    ret
 ;
 ; RV32I-LABEL: sraw:
diff --git a/test/CodeGen/RISCV/atomic-cmpxchg.ll b/test/CodeGen/RISCV/atomic-cmpxchg.ll
index 85cd1691366401f77b38420eafae92c27f345118..c1113cdfb4d34dfbb341a5ef7bffc4b65589bf6f 100644
--- a/test/CodeGen/RISCV/atomic-cmpxchg.ll
+++ b/test/CodeGen/RISCV/atomic-cmpxchg.ll
@@ -3,6 +3,10 @@
 ; RUN:   | FileCheck -check-prefix=RV32I %s
 ; RUN: llc -mtriple=riscv32 -mattr=+a -verify-machineinstrs < %s \
 ; RUN:   | FileCheck -check-prefix=RV32IA %s
+; RUN: llc -mtriple=riscv64 -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=RV64I %s
+; RUN: llc -mtriple=riscv64 -mattr=+a -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=RV64IA %s
 
 define void @cmpxchg_i8_monotonic_monotonic(i8* %ptr, i8 %cmp, i8 %val) {
 ; RV32I-LABEL: cmpxchg_i8_monotonic_monotonic:
@@ -41,6 +45,43 @@ define void @cmpxchg_i8_monotonic_monotonic(i8* %ptr, i8 %cmp, i8 %val) {
 ; RV32IA-NEXT:    bnez a5, .LBB0_1
 ; RV32IA-NEXT:  .LBB0_3:
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: cmpxchg_i8_monotonic_monotonic:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sb a1, 7(sp)
+; RV64I-NEXT:    addi a1, sp, 7
+; RV64I-NEXT:    mv a3, zero
+; RV64I-NEXT:    mv a4, zero
+; RV64I-NEXT:    call __atomic_compare_exchange_1
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: cmpxchg_i8_monotonic_monotonic:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    slli a3, a0, 3
+; RV64IA-NEXT:    andi a3, a3, 24
+; RV64IA-NEXT:    addi a4, zero, 255
+; RV64IA-NEXT:    sllw a4, a4, a3
+; RV64IA-NEXT:    andi a2, a2, 255
+; RV64IA-NEXT:    sllw a2, a2, a3
+; RV64IA-NEXT:    andi a1, a1, 255
+; RV64IA-NEXT:    sllw a1, a1, a3
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:  .LBB0_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.w a3, (a0)
+; RV64IA-NEXT:    and a5, a3, a4
+; RV64IA-NEXT:    bne a5, a1, .LBB0_3
+; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB0_1 Depth=1
+; RV64IA-NEXT:    xor a5, a3, a2
+; RV64IA-NEXT:    and a5, a5, a4
+; RV64IA-NEXT:    xor a5, a3, a5
+; RV64IA-NEXT:    sc.w a5, a5, (a0)
+; RV64IA-NEXT:    bnez a5, .LBB0_1
+; RV64IA-NEXT:  .LBB0_3:
+; RV64IA-NEXT:    ret
   %res = cmpxchg i8* %ptr, i8 %cmp, i8 %val monotonic monotonic
   ret void
 }
@@ -82,6 +123,43 @@ define void @cmpxchg_i8_acquire_monotonic(i8* %ptr, i8 %cmp, i8 %val) {
 ; RV32IA-NEXT:    bnez a5, .LBB1_1
 ; RV32IA-NEXT:  .LBB1_3:
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: cmpxchg_i8_acquire_monotonic:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sb a1, 7(sp)
+; RV64I-NEXT:    addi a1, sp, 7
+; RV64I-NEXT:    addi a3, zero, 2
+; RV64I-NEXT:    mv a4, zero
+; RV64I-NEXT:    call __atomic_compare_exchange_1
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: cmpxchg_i8_acquire_monotonic:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    slli a3, a0, 3
+; RV64IA-NEXT:    andi a3, a3, 24
+; RV64IA-NEXT:    addi a4, zero, 255
+; RV64IA-NEXT:    sllw a4, a4, a3
+; RV64IA-NEXT:    andi a2, a2, 255
+; RV64IA-NEXT:    sllw a2, a2, a3
+; RV64IA-NEXT:    andi a1, a1, 255
+; RV64IA-NEXT:    sllw a1, a1, a3
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:  .LBB1_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.w.aq a3, (a0)
+; RV64IA-NEXT:    and a5, a3, a4
+; RV64IA-NEXT:    bne a5, a1, .LBB1_3
+; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB1_1 Depth=1
+; RV64IA-NEXT:    xor a5, a3, a2
+; RV64IA-NEXT:    and a5, a5, a4
+; RV64IA-NEXT:    xor a5, a3, a5
+; RV64IA-NEXT:    sc.w a5, a5, (a0)
+; RV64IA-NEXT:    bnez a5, .LBB1_1
+; RV64IA-NEXT:  .LBB1_3:
+; RV64IA-NEXT:    ret
   %res = cmpxchg i8* %ptr, i8 %cmp, i8 %val acquire monotonic
   ret void
 }
@@ -94,7 +172,7 @@ define void @cmpxchg_i8_acquire_acquire(i8* %ptr, i8 %cmp, i8 %val) {
 ; RV32I-NEXT:    sb a1, 11(sp)
 ; RV32I-NEXT:    addi a1, sp, 11
 ; RV32I-NEXT:    addi a3, zero, 2
-; RV32I-NEXT:    mv a4, a3
+; RV32I-NEXT:    addi a4, zero, 2
 ; RV32I-NEXT:    call __atomic_compare_exchange_1
 ; RV32I-NEXT:    lw ra, 12(sp)
 ; RV32I-NEXT:    addi sp, sp, 16
@@ -123,6 +201,43 @@ define void @cmpxchg_i8_acquire_acquire(i8* %ptr, i8 %cmp, i8 %val) {
 ; RV32IA-NEXT:    bnez a5, .LBB2_1
 ; RV32IA-NEXT:  .LBB2_3:
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: cmpxchg_i8_acquire_acquire:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sb a1, 7(sp)
+; RV64I-NEXT:    addi a1, sp, 7
+; RV64I-NEXT:    addi a3, zero, 2
+; RV64I-NEXT:    addi a4, zero, 2
+; RV64I-NEXT:    call __atomic_compare_exchange_1
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: cmpxchg_i8_acquire_acquire:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    slli a3, a0, 3
+; RV64IA-NEXT:    andi a3, a3, 24
+; RV64IA-NEXT:    addi a4, zero, 255
+; RV64IA-NEXT:    sllw a4, a4, a3
+; RV64IA-NEXT:    andi a2, a2, 255
+; RV64IA-NEXT:    sllw a2, a2, a3
+; RV64IA-NEXT:    andi a1, a1, 255
+; RV64IA-NEXT:    sllw a1, a1, a3
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:  .LBB2_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.w.aq a3, (a0)
+; RV64IA-NEXT:    and a5, a3, a4
+; RV64IA-NEXT:    bne a5, a1, .LBB2_3
+; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB2_1 Depth=1
+; RV64IA-NEXT:    xor a5, a3, a2
+; RV64IA-NEXT:    and a5, a5, a4
+; RV64IA-NEXT:    xor a5, a3, a5
+; RV64IA-NEXT:    sc.w a5, a5, (a0)
+; RV64IA-NEXT:    bnez a5, .LBB2_1
+; RV64IA-NEXT:  .LBB2_3:
+; RV64IA-NEXT:    ret
   %res = cmpxchg i8* %ptr, i8 %cmp, i8 %val acquire acquire
   ret void
 }
@@ -164,6 +279,43 @@ define void @cmpxchg_i8_release_monotonic(i8* %ptr, i8 %cmp, i8 %val) {
 ; RV32IA-NEXT:    bnez a5, .LBB3_1
 ; RV32IA-NEXT:  .LBB3_3:
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: cmpxchg_i8_release_monotonic:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sb a1, 7(sp)
+; RV64I-NEXT:    addi a1, sp, 7
+; RV64I-NEXT:    addi a3, zero, 3
+; RV64I-NEXT:    mv a4, zero
+; RV64I-NEXT:    call __atomic_compare_exchange_1
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: cmpxchg_i8_release_monotonic:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    slli a3, a0, 3
+; RV64IA-NEXT:    andi a3, a3, 24
+; RV64IA-NEXT:    addi a4, zero, 255
+; RV64IA-NEXT:    sllw a4, a4, a3
+; RV64IA-NEXT:    andi a2, a2, 255
+; RV64IA-NEXT:    sllw a2, a2, a3
+; RV64IA-NEXT:    andi a1, a1, 255
+; RV64IA-NEXT:    sllw a1, a1, a3
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:  .LBB3_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.w a3, (a0)
+; RV64IA-NEXT:    and a5, a3, a4
+; RV64IA-NEXT:    bne a5, a1, .LBB3_3
+; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB3_1 Depth=1
+; RV64IA-NEXT:    xor a5, a3, a2
+; RV64IA-NEXT:    and a5, a5, a4
+; RV64IA-NEXT:    xor a5, a3, a5
+; RV64IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV64IA-NEXT:    bnez a5, .LBB3_1
+; RV64IA-NEXT:  .LBB3_3:
+; RV64IA-NEXT:    ret
   %res = cmpxchg i8* %ptr, i8 %cmp, i8 %val release monotonic
   ret void
 }
@@ -205,6 +357,43 @@ define void @cmpxchg_i8_release_acquire(i8* %ptr, i8 %cmp, i8 %val) {
 ; RV32IA-NEXT:    bnez a5, .LBB4_1
 ; RV32IA-NEXT:  .LBB4_3:
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: cmpxchg_i8_release_acquire:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sb a1, 7(sp)
+; RV64I-NEXT:    addi a1, sp, 7
+; RV64I-NEXT:    addi a3, zero, 3
+; RV64I-NEXT:    addi a4, zero, 2
+; RV64I-NEXT:    call __atomic_compare_exchange_1
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: cmpxchg_i8_release_acquire:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    slli a3, a0, 3
+; RV64IA-NEXT:    andi a3, a3, 24
+; RV64IA-NEXT:    addi a4, zero, 255
+; RV64IA-NEXT:    sllw a4, a4, a3
+; RV64IA-NEXT:    andi a2, a2, 255
+; RV64IA-NEXT:    sllw a2, a2, a3
+; RV64IA-NEXT:    andi a1, a1, 255
+; RV64IA-NEXT:    sllw a1, a1, a3
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:  .LBB4_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.w a3, (a0)
+; RV64IA-NEXT:    and a5, a3, a4
+; RV64IA-NEXT:    bne a5, a1, .LBB4_3
+; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB4_1 Depth=1
+; RV64IA-NEXT:    xor a5, a3, a2
+; RV64IA-NEXT:    and a5, a5, a4
+; RV64IA-NEXT:    xor a5, a3, a5
+; RV64IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV64IA-NEXT:    bnez a5, .LBB4_1
+; RV64IA-NEXT:  .LBB4_3:
+; RV64IA-NEXT:    ret
   %res = cmpxchg i8* %ptr, i8 %cmp, i8 %val release acquire
   ret void
 }
@@ -246,6 +435,43 @@ define void @cmpxchg_i8_acq_rel_monotonic(i8* %ptr, i8 %cmp, i8 %val) {
 ; RV32IA-NEXT:    bnez a5, .LBB5_1
 ; RV32IA-NEXT:  .LBB5_3:
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: cmpxchg_i8_acq_rel_monotonic:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sb a1, 7(sp)
+; RV64I-NEXT:    addi a1, sp, 7
+; RV64I-NEXT:    addi a3, zero, 4
+; RV64I-NEXT:    mv a4, zero
+; RV64I-NEXT:    call __atomic_compare_exchange_1
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: cmpxchg_i8_acq_rel_monotonic:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    slli a3, a0, 3
+; RV64IA-NEXT:    andi a3, a3, 24
+; RV64IA-NEXT:    addi a4, zero, 255
+; RV64IA-NEXT:    sllw a4, a4, a3
+; RV64IA-NEXT:    andi a2, a2, 255
+; RV64IA-NEXT:    sllw a2, a2, a3
+; RV64IA-NEXT:    andi a1, a1, 255
+; RV64IA-NEXT:    sllw a1, a1, a3
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:  .LBB5_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.w.aq a3, (a0)
+; RV64IA-NEXT:    and a5, a3, a4
+; RV64IA-NEXT:    bne a5, a1, .LBB5_3
+; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB5_1 Depth=1
+; RV64IA-NEXT:    xor a5, a3, a2
+; RV64IA-NEXT:    and a5, a5, a4
+; RV64IA-NEXT:    xor a5, a3, a5
+; RV64IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV64IA-NEXT:    bnez a5, .LBB5_1
+; RV64IA-NEXT:  .LBB5_3:
+; RV64IA-NEXT:    ret
   %res = cmpxchg i8* %ptr, i8 %cmp, i8 %val acq_rel monotonic
   ret void
 }
@@ -287,6 +513,43 @@ define void @cmpxchg_i8_acq_rel_acquire(i8* %ptr, i8 %cmp, i8 %val) {
 ; RV32IA-NEXT:    bnez a5, .LBB6_1
 ; RV32IA-NEXT:  .LBB6_3:
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: cmpxchg_i8_acq_rel_acquire:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sb a1, 7(sp)
+; RV64I-NEXT:    addi a1, sp, 7
+; RV64I-NEXT:    addi a3, zero, 4
+; RV64I-NEXT:    addi a4, zero, 2
+; RV64I-NEXT:    call __atomic_compare_exchange_1
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: cmpxchg_i8_acq_rel_acquire:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    slli a3, a0, 3
+; RV64IA-NEXT:    andi a3, a3, 24
+; RV64IA-NEXT:    addi a4, zero, 255
+; RV64IA-NEXT:    sllw a4, a4, a3
+; RV64IA-NEXT:    andi a2, a2, 255
+; RV64IA-NEXT:    sllw a2, a2, a3
+; RV64IA-NEXT:    andi a1, a1, 255
+; RV64IA-NEXT:    sllw a1, a1, a3
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:  .LBB6_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.w.aq a3, (a0)
+; RV64IA-NEXT:    and a5, a3, a4
+; RV64IA-NEXT:    bne a5, a1, .LBB6_3
+; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB6_1 Depth=1
+; RV64IA-NEXT:    xor a5, a3, a2
+; RV64IA-NEXT:    and a5, a5, a4
+; RV64IA-NEXT:    xor a5, a3, a5
+; RV64IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV64IA-NEXT:    bnez a5, .LBB6_1
+; RV64IA-NEXT:  .LBB6_3:
+; RV64IA-NEXT:    ret
   %res = cmpxchg i8* %ptr, i8 %cmp, i8 %val acq_rel acquire
   ret void
 }
@@ -328,6 +591,43 @@ define void @cmpxchg_i8_seq_cst_monotonic(i8* %ptr, i8 %cmp, i8 %val) {
 ; RV32IA-NEXT:    bnez a5, .LBB7_1
 ; RV32IA-NEXT:  .LBB7_3:
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: cmpxchg_i8_seq_cst_monotonic:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sb a1, 7(sp)
+; RV64I-NEXT:    addi a1, sp, 7
+; RV64I-NEXT:    addi a3, zero, 5
+; RV64I-NEXT:    mv a4, zero
+; RV64I-NEXT:    call __atomic_compare_exchange_1
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: cmpxchg_i8_seq_cst_monotonic:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    slli a3, a0, 3
+; RV64IA-NEXT:    andi a3, a3, 24
+; RV64IA-NEXT:    addi a4, zero, 255
+; RV64IA-NEXT:    sllw a4, a4, a3
+; RV64IA-NEXT:    andi a2, a2, 255
+; RV64IA-NEXT:    sllw a2, a2, a3
+; RV64IA-NEXT:    andi a1, a1, 255
+; RV64IA-NEXT:    sllw a1, a1, a3
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:  .LBB7_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.w.aqrl a3, (a0)
+; RV64IA-NEXT:    and a5, a3, a4
+; RV64IA-NEXT:    bne a5, a1, .LBB7_3
+; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB7_1 Depth=1
+; RV64IA-NEXT:    xor a5, a3, a2
+; RV64IA-NEXT:    and a5, a5, a4
+; RV64IA-NEXT:    xor a5, a3, a5
+; RV64IA-NEXT:    sc.w.aqrl a5, a5, (a0)
+; RV64IA-NEXT:    bnez a5, .LBB7_1
+; RV64IA-NEXT:  .LBB7_3:
+; RV64IA-NEXT:    ret
   %res = cmpxchg i8* %ptr, i8 %cmp, i8 %val seq_cst monotonic
   ret void
 }
@@ -369,6 +669,43 @@ define void @cmpxchg_i8_seq_cst_acquire(i8* %ptr, i8 %cmp, i8 %val) {
 ; RV32IA-NEXT:    bnez a5, .LBB8_1
 ; RV32IA-NEXT:  .LBB8_3:
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: cmpxchg_i8_seq_cst_acquire:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sb a1, 7(sp)
+; RV64I-NEXT:    addi a1, sp, 7
+; RV64I-NEXT:    addi a3, zero, 5
+; RV64I-NEXT:    addi a4, zero, 2
+; RV64I-NEXT:    call __atomic_compare_exchange_1
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: cmpxchg_i8_seq_cst_acquire:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    slli a3, a0, 3
+; RV64IA-NEXT:    andi a3, a3, 24
+; RV64IA-NEXT:    addi a4, zero, 255
+; RV64IA-NEXT:    sllw a4, a4, a3
+; RV64IA-NEXT:    andi a2, a2, 255
+; RV64IA-NEXT:    sllw a2, a2, a3
+; RV64IA-NEXT:    andi a1, a1, 255
+; RV64IA-NEXT:    sllw a1, a1, a3
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:  .LBB8_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.w.aqrl a3, (a0)
+; RV64IA-NEXT:    and a5, a3, a4
+; RV64IA-NEXT:    bne a5, a1, .LBB8_3
+; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB8_1 Depth=1
+; RV64IA-NEXT:    xor a5, a3, a2
+; RV64IA-NEXT:    and a5, a5, a4
+; RV64IA-NEXT:    xor a5, a3, a5
+; RV64IA-NEXT:    sc.w.aqrl a5, a5, (a0)
+; RV64IA-NEXT:    bnez a5, .LBB8_1
+; RV64IA-NEXT:  .LBB8_3:
+; RV64IA-NEXT:    ret
   %res = cmpxchg i8* %ptr, i8 %cmp, i8 %val seq_cst acquire
   ret void
 }
@@ -381,7 +718,7 @@ define void @cmpxchg_i8_seq_cst_seq_cst(i8* %ptr, i8 %cmp, i8 %val) {
 ; RV32I-NEXT:    sb a1, 11(sp)
 ; RV32I-NEXT:    addi a1, sp, 11
 ; RV32I-NEXT:    addi a3, zero, 5
-; RV32I-NEXT:    mv a4, a3
+; RV32I-NEXT:    addi a4, zero, 5
 ; RV32I-NEXT:    call __atomic_compare_exchange_1
 ; RV32I-NEXT:    lw ra, 12(sp)
 ; RV32I-NEXT:    addi sp, sp, 16
@@ -410,6 +747,43 @@ define void @cmpxchg_i8_seq_cst_seq_cst(i8* %ptr, i8 %cmp, i8 %val) {
 ; RV32IA-NEXT:    bnez a5, .LBB9_1
 ; RV32IA-NEXT:  .LBB9_3:
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: cmpxchg_i8_seq_cst_seq_cst:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sb a1, 7(sp)
+; RV64I-NEXT:    addi a1, sp, 7
+; RV64I-NEXT:    addi a3, zero, 5
+; RV64I-NEXT:    addi a4, zero, 5
+; RV64I-NEXT:    call __atomic_compare_exchange_1
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: cmpxchg_i8_seq_cst_seq_cst:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    slli a3, a0, 3
+; RV64IA-NEXT:    andi a3, a3, 24
+; RV64IA-NEXT:    addi a4, zero, 255
+; RV64IA-NEXT:    sllw a4, a4, a3
+; RV64IA-NEXT:    andi a2, a2, 255
+; RV64IA-NEXT:    sllw a2, a2, a3
+; RV64IA-NEXT:    andi a1, a1, 255
+; RV64IA-NEXT:    sllw a1, a1, a3
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:  .LBB9_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.w.aqrl a3, (a0)
+; RV64IA-NEXT:    and a5, a3, a4
+; RV64IA-NEXT:    bne a5, a1, .LBB9_3
+; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB9_1 Depth=1
+; RV64IA-NEXT:    xor a5, a3, a2
+; RV64IA-NEXT:    and a5, a5, a4
+; RV64IA-NEXT:    xor a5, a3, a5
+; RV64IA-NEXT:    sc.w.aqrl a5, a5, (a0)
+; RV64IA-NEXT:    bnez a5, .LBB9_1
+; RV64IA-NEXT:  .LBB9_3:
+; RV64IA-NEXT:    ret
   %res = cmpxchg i8* %ptr, i8 %cmp, i8 %val seq_cst seq_cst
   ret void
 }
@@ -452,6 +826,44 @@ define void @cmpxchg_i16_monotonic_monotonic(i16* %ptr, i16 %cmp, i16 %val) {
 ; RV32IA-NEXT:    bnez a5, .LBB10_1
 ; RV32IA-NEXT:  .LBB10_3:
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: cmpxchg_i16_monotonic_monotonic:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sh a1, 6(sp)
+; RV64I-NEXT:    addi a1, sp, 6
+; RV64I-NEXT:    mv a3, zero
+; RV64I-NEXT:    mv a4, zero
+; RV64I-NEXT:    call __atomic_compare_exchange_2
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: cmpxchg_i16_monotonic_monotonic:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    lui a3, 16
+; RV64IA-NEXT:    addiw a3, a3, -1
+; RV64IA-NEXT:    and a1, a1, a3
+; RV64IA-NEXT:    and a2, a2, a3
+; RV64IA-NEXT:    slli a4, a0, 3
+; RV64IA-NEXT:    andi a4, a4, 24
+; RV64IA-NEXT:    sllw a3, a3, a4
+; RV64IA-NEXT:    sllw a2, a2, a4
+; RV64IA-NEXT:    sllw a1, a1, a4
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:  .LBB10_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.w a4, (a0)
+; RV64IA-NEXT:    and a5, a4, a3
+; RV64IA-NEXT:    bne a5, a1, .LBB10_3
+; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB10_1 Depth=1
+; RV64IA-NEXT:    xor a5, a4, a2
+; RV64IA-NEXT:    and a5, a5, a3
+; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:    sc.w a5, a5, (a0)
+; RV64IA-NEXT:    bnez a5, .LBB10_1
+; RV64IA-NEXT:  .LBB10_3:
+; RV64IA-NEXT:    ret
   %res = cmpxchg i16* %ptr, i16 %cmp, i16 %val monotonic monotonic
   ret void
 }
@@ -494,6 +906,44 @@ define void @cmpxchg_i16_acquire_monotonic(i16* %ptr, i16 %cmp, i16 %val) {
 ; RV32IA-NEXT:    bnez a5, .LBB11_1
 ; RV32IA-NEXT:  .LBB11_3:
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: cmpxchg_i16_acquire_monotonic:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sh a1, 6(sp)
+; RV64I-NEXT:    addi a1, sp, 6
+; RV64I-NEXT:    addi a3, zero, 2
+; RV64I-NEXT:    mv a4, zero
+; RV64I-NEXT:    call __atomic_compare_exchange_2
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: cmpxchg_i16_acquire_monotonic:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    lui a3, 16
+; RV64IA-NEXT:    addiw a3, a3, -1
+; RV64IA-NEXT:    and a1, a1, a3
+; RV64IA-NEXT:    and a2, a2, a3
+; RV64IA-NEXT:    slli a4, a0, 3
+; RV64IA-NEXT:    andi a4, a4, 24
+; RV64IA-NEXT:    sllw a3, a3, a4
+; RV64IA-NEXT:    sllw a2, a2, a4
+; RV64IA-NEXT:    sllw a1, a1, a4
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:  .LBB11_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.w.aq a4, (a0)
+; RV64IA-NEXT:    and a5, a4, a3
+; RV64IA-NEXT:    bne a5, a1, .LBB11_3
+; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB11_1 Depth=1
+; RV64IA-NEXT:    xor a5, a4, a2
+; RV64IA-NEXT:    and a5, a5, a3
+; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:    sc.w a5, a5, (a0)
+; RV64IA-NEXT:    bnez a5, .LBB11_1
+; RV64IA-NEXT:  .LBB11_3:
+; RV64IA-NEXT:    ret
   %res = cmpxchg i16* %ptr, i16 %cmp, i16 %val acquire monotonic
   ret void
 }
@@ -506,7 +956,7 @@ define void @cmpxchg_i16_acquire_acquire(i16* %ptr, i16 %cmp, i16 %val) {
 ; RV32I-NEXT:    sh a1, 10(sp)
 ; RV32I-NEXT:    addi a1, sp, 10
 ; RV32I-NEXT:    addi a3, zero, 2
-; RV32I-NEXT:    mv a4, a3
+; RV32I-NEXT:    addi a4, zero, 2
 ; RV32I-NEXT:    call __atomic_compare_exchange_2
 ; RV32I-NEXT:    lw ra, 12(sp)
 ; RV32I-NEXT:    addi sp, sp, 16
@@ -536,6 +986,44 @@ define void @cmpxchg_i16_acquire_acquire(i16* %ptr, i16 %cmp, i16 %val) {
 ; RV32IA-NEXT:    bnez a5, .LBB12_1
 ; RV32IA-NEXT:  .LBB12_3:
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: cmpxchg_i16_acquire_acquire:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sh a1, 6(sp)
+; RV64I-NEXT:    addi a1, sp, 6
+; RV64I-NEXT:    addi a3, zero, 2
+; RV64I-NEXT:    addi a4, zero, 2
+; RV64I-NEXT:    call __atomic_compare_exchange_2
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: cmpxchg_i16_acquire_acquire:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    lui a3, 16
+; RV64IA-NEXT:    addiw a3, a3, -1
+; RV64IA-NEXT:    and a1, a1, a3
+; RV64IA-NEXT:    and a2, a2, a3
+; RV64IA-NEXT:    slli a4, a0, 3
+; RV64IA-NEXT:    andi a4, a4, 24
+; RV64IA-NEXT:    sllw a3, a3, a4
+; RV64IA-NEXT:    sllw a2, a2, a4
+; RV64IA-NEXT:    sllw a1, a1, a4
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:  .LBB12_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.w.aq a4, (a0)
+; RV64IA-NEXT:    and a5, a4, a3
+; RV64IA-NEXT:    bne a5, a1, .LBB12_3
+; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB12_1 Depth=1
+; RV64IA-NEXT:    xor a5, a4, a2
+; RV64IA-NEXT:    and a5, a5, a3
+; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:    sc.w a5, a5, (a0)
+; RV64IA-NEXT:    bnez a5, .LBB12_1
+; RV64IA-NEXT:  .LBB12_3:
+; RV64IA-NEXT:    ret
   %res = cmpxchg i16* %ptr, i16 %cmp, i16 %val acquire acquire
   ret void
 }
@@ -578,6 +1066,44 @@ define void @cmpxchg_i16_release_monotonic(i16* %ptr, i16 %cmp, i16 %val) {
 ; RV32IA-NEXT:    bnez a5, .LBB13_1
 ; RV32IA-NEXT:  .LBB13_3:
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: cmpxchg_i16_release_monotonic:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sh a1, 6(sp)
+; RV64I-NEXT:    addi a1, sp, 6
+; RV64I-NEXT:    addi a3, zero, 3
+; RV64I-NEXT:    mv a4, zero
+; RV64I-NEXT:    call __atomic_compare_exchange_2
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: cmpxchg_i16_release_monotonic:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    lui a3, 16
+; RV64IA-NEXT:    addiw a3, a3, -1
+; RV64IA-NEXT:    and a1, a1, a3
+; RV64IA-NEXT:    and a2, a2, a3
+; RV64IA-NEXT:    slli a4, a0, 3
+; RV64IA-NEXT:    andi a4, a4, 24
+; RV64IA-NEXT:    sllw a3, a3, a4
+; RV64IA-NEXT:    sllw a2, a2, a4
+; RV64IA-NEXT:    sllw a1, a1, a4
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:  .LBB13_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.w a4, (a0)
+; RV64IA-NEXT:    and a5, a4, a3
+; RV64IA-NEXT:    bne a5, a1, .LBB13_3
+; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB13_1 Depth=1
+; RV64IA-NEXT:    xor a5, a4, a2
+; RV64IA-NEXT:    and a5, a5, a3
+; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV64IA-NEXT:    bnez a5, .LBB13_1
+; RV64IA-NEXT:  .LBB13_3:
+; RV64IA-NEXT:    ret
   %res = cmpxchg i16* %ptr, i16 %cmp, i16 %val release monotonic
   ret void
 }
@@ -620,6 +1146,44 @@ define void @cmpxchg_i16_release_acquire(i16* %ptr, i16 %cmp, i16 %val) {
 ; RV32IA-NEXT:    bnez a5, .LBB14_1
 ; RV32IA-NEXT:  .LBB14_3:
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: cmpxchg_i16_release_acquire:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sh a1, 6(sp)
+; RV64I-NEXT:    addi a1, sp, 6
+; RV64I-NEXT:    addi a3, zero, 3
+; RV64I-NEXT:    addi a4, zero, 2
+; RV64I-NEXT:    call __atomic_compare_exchange_2
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: cmpxchg_i16_release_acquire:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    lui a3, 16
+; RV64IA-NEXT:    addiw a3, a3, -1
+; RV64IA-NEXT:    and a1, a1, a3
+; RV64IA-NEXT:    and a2, a2, a3
+; RV64IA-NEXT:    slli a4, a0, 3
+; RV64IA-NEXT:    andi a4, a4, 24
+; RV64IA-NEXT:    sllw a3, a3, a4
+; RV64IA-NEXT:    sllw a2, a2, a4
+; RV64IA-NEXT:    sllw a1, a1, a4
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:  .LBB14_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.w a4, (a0)
+; RV64IA-NEXT:    and a5, a4, a3
+; RV64IA-NEXT:    bne a5, a1, .LBB14_3
+; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB14_1 Depth=1
+; RV64IA-NEXT:    xor a5, a4, a2
+; RV64IA-NEXT:    and a5, a5, a3
+; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV64IA-NEXT:    bnez a5, .LBB14_1
+; RV64IA-NEXT:  .LBB14_3:
+; RV64IA-NEXT:    ret
   %res = cmpxchg i16* %ptr, i16 %cmp, i16 %val release acquire
   ret void
 }
@@ -662,6 +1226,44 @@ define void @cmpxchg_i16_acq_rel_monotonic(i16* %ptr, i16 %cmp, i16 %val) {
 ; RV32IA-NEXT:    bnez a5, .LBB15_1
 ; RV32IA-NEXT:  .LBB15_3:
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: cmpxchg_i16_acq_rel_monotonic:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sh a1, 6(sp)
+; RV64I-NEXT:    addi a1, sp, 6
+; RV64I-NEXT:    addi a3, zero, 4
+; RV64I-NEXT:    mv a4, zero
+; RV64I-NEXT:    call __atomic_compare_exchange_2
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: cmpxchg_i16_acq_rel_monotonic:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    lui a3, 16
+; RV64IA-NEXT:    addiw a3, a3, -1
+; RV64IA-NEXT:    and a1, a1, a3
+; RV64IA-NEXT:    and a2, a2, a3
+; RV64IA-NEXT:    slli a4, a0, 3
+; RV64IA-NEXT:    andi a4, a4, 24
+; RV64IA-NEXT:    sllw a3, a3, a4
+; RV64IA-NEXT:    sllw a2, a2, a4
+; RV64IA-NEXT:    sllw a1, a1, a4
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:  .LBB15_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.w.aq a4, (a0)
+; RV64IA-NEXT:    and a5, a4, a3
+; RV64IA-NEXT:    bne a5, a1, .LBB15_3
+; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB15_1 Depth=1
+; RV64IA-NEXT:    xor a5, a4, a2
+; RV64IA-NEXT:    and a5, a5, a3
+; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV64IA-NEXT:    bnez a5, .LBB15_1
+; RV64IA-NEXT:  .LBB15_3:
+; RV64IA-NEXT:    ret
   %res = cmpxchg i16* %ptr, i16 %cmp, i16 %val acq_rel monotonic
   ret void
 }
@@ -704,6 +1306,44 @@ define void @cmpxchg_i16_acq_rel_acquire(i16* %ptr, i16 %cmp, i16 %val) {
 ; RV32IA-NEXT:    bnez a5, .LBB16_1
 ; RV32IA-NEXT:  .LBB16_3:
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: cmpxchg_i16_acq_rel_acquire:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sh a1, 6(sp)
+; RV64I-NEXT:    addi a1, sp, 6
+; RV64I-NEXT:    addi a3, zero, 4
+; RV64I-NEXT:    addi a4, zero, 2
+; RV64I-NEXT:    call __atomic_compare_exchange_2
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: cmpxchg_i16_acq_rel_acquire:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    lui a3, 16
+; RV64IA-NEXT:    addiw a3, a3, -1
+; RV64IA-NEXT:    and a1, a1, a3
+; RV64IA-NEXT:    and a2, a2, a3
+; RV64IA-NEXT:    slli a4, a0, 3
+; RV64IA-NEXT:    andi a4, a4, 24
+; RV64IA-NEXT:    sllw a3, a3, a4
+; RV64IA-NEXT:    sllw a2, a2, a4
+; RV64IA-NEXT:    sllw a1, a1, a4
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:  .LBB16_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.w.aq a4, (a0)
+; RV64IA-NEXT:    and a5, a4, a3
+; RV64IA-NEXT:    bne a5, a1, .LBB16_3
+; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB16_1 Depth=1
+; RV64IA-NEXT:    xor a5, a4, a2
+; RV64IA-NEXT:    and a5, a5, a3
+; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV64IA-NEXT:    bnez a5, .LBB16_1
+; RV64IA-NEXT:  .LBB16_3:
+; RV64IA-NEXT:    ret
   %res = cmpxchg i16* %ptr, i16 %cmp, i16 %val acq_rel acquire
   ret void
 }
@@ -746,6 +1386,44 @@ define void @cmpxchg_i16_seq_cst_monotonic(i16* %ptr, i16 %cmp, i16 %val) {
 ; RV32IA-NEXT:    bnez a5, .LBB17_1
 ; RV32IA-NEXT:  .LBB17_3:
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: cmpxchg_i16_seq_cst_monotonic:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sh a1, 6(sp)
+; RV64I-NEXT:    addi a1, sp, 6
+; RV64I-NEXT:    addi a3, zero, 5
+; RV64I-NEXT:    mv a4, zero
+; RV64I-NEXT:    call __atomic_compare_exchange_2
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: cmpxchg_i16_seq_cst_monotonic:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    lui a3, 16
+; RV64IA-NEXT:    addiw a3, a3, -1
+; RV64IA-NEXT:    and a1, a1, a3
+; RV64IA-NEXT:    and a2, a2, a3
+; RV64IA-NEXT:    slli a4, a0, 3
+; RV64IA-NEXT:    andi a4, a4, 24
+; RV64IA-NEXT:    sllw a3, a3, a4
+; RV64IA-NEXT:    sllw a2, a2, a4
+; RV64IA-NEXT:    sllw a1, a1, a4
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:  .LBB17_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.w.aqrl a4, (a0)
+; RV64IA-NEXT:    and a5, a4, a3
+; RV64IA-NEXT:    bne a5, a1, .LBB17_3
+; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB17_1 Depth=1
+; RV64IA-NEXT:    xor a5, a4, a2
+; RV64IA-NEXT:    and a5, a5, a3
+; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:    sc.w.aqrl a5, a5, (a0)
+; RV64IA-NEXT:    bnez a5, .LBB17_1
+; RV64IA-NEXT:  .LBB17_3:
+; RV64IA-NEXT:    ret
   %res = cmpxchg i16* %ptr, i16 %cmp, i16 %val seq_cst monotonic
   ret void
 }
@@ -788,6 +1466,44 @@ define void @cmpxchg_i16_seq_cst_acquire(i16* %ptr, i16 %cmp, i16 %val) {
 ; RV32IA-NEXT:    bnez a5, .LBB18_1
 ; RV32IA-NEXT:  .LBB18_3:
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: cmpxchg_i16_seq_cst_acquire:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sh a1, 6(sp)
+; RV64I-NEXT:    addi a1, sp, 6
+; RV64I-NEXT:    addi a3, zero, 5
+; RV64I-NEXT:    addi a4, zero, 2
+; RV64I-NEXT:    call __atomic_compare_exchange_2
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: cmpxchg_i16_seq_cst_acquire:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    lui a3, 16
+; RV64IA-NEXT:    addiw a3, a3, -1
+; RV64IA-NEXT:    and a1, a1, a3
+; RV64IA-NEXT:    and a2, a2, a3
+; RV64IA-NEXT:    slli a4, a0, 3
+; RV64IA-NEXT:    andi a4, a4, 24
+; RV64IA-NEXT:    sllw a3, a3, a4
+; RV64IA-NEXT:    sllw a2, a2, a4
+; RV64IA-NEXT:    sllw a1, a1, a4
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:  .LBB18_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.w.aqrl a4, (a0)
+; RV64IA-NEXT:    and a5, a4, a3
+; RV64IA-NEXT:    bne a5, a1, .LBB18_3
+; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB18_1 Depth=1
+; RV64IA-NEXT:    xor a5, a4, a2
+; RV64IA-NEXT:    and a5, a5, a3
+; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:    sc.w.aqrl a5, a5, (a0)
+; RV64IA-NEXT:    bnez a5, .LBB18_1
+; RV64IA-NEXT:  .LBB18_3:
+; RV64IA-NEXT:    ret
   %res = cmpxchg i16* %ptr, i16 %cmp, i16 %val seq_cst acquire
   ret void
 }
@@ -800,7 +1516,7 @@ define void @cmpxchg_i16_seq_cst_seq_cst(i16* %ptr, i16 %cmp, i16 %val) {
 ; RV32I-NEXT:    sh a1, 10(sp)
 ; RV32I-NEXT:    addi a1, sp, 10
 ; RV32I-NEXT:    addi a3, zero, 5
-; RV32I-NEXT:    mv a4, a3
+; RV32I-NEXT:    addi a4, zero, 5
 ; RV32I-NEXT:    call __atomic_compare_exchange_2
 ; RV32I-NEXT:    lw ra, 12(sp)
 ; RV32I-NEXT:    addi sp, sp, 16
@@ -830,6 +1546,44 @@ define void @cmpxchg_i16_seq_cst_seq_cst(i16* %ptr, i16 %cmp, i16 %val) {
 ; RV32IA-NEXT:    bnez a5, .LBB19_1
 ; RV32IA-NEXT:  .LBB19_3:
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: cmpxchg_i16_seq_cst_seq_cst:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sh a1, 6(sp)
+; RV64I-NEXT:    addi a1, sp, 6
+; RV64I-NEXT:    addi a3, zero, 5
+; RV64I-NEXT:    addi a4, zero, 5
+; RV64I-NEXT:    call __atomic_compare_exchange_2
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: cmpxchg_i16_seq_cst_seq_cst:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    lui a3, 16
+; RV64IA-NEXT:    addiw a3, a3, -1
+; RV64IA-NEXT:    and a1, a1, a3
+; RV64IA-NEXT:    and a2, a2, a3
+; RV64IA-NEXT:    slli a4, a0, 3
+; RV64IA-NEXT:    andi a4, a4, 24
+; RV64IA-NEXT:    sllw a3, a3, a4
+; RV64IA-NEXT:    sllw a2, a2, a4
+; RV64IA-NEXT:    sllw a1, a1, a4
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:  .LBB19_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.w.aqrl a4, (a0)
+; RV64IA-NEXT:    and a5, a4, a3
+; RV64IA-NEXT:    bne a5, a1, .LBB19_3
+; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB19_1 Depth=1
+; RV64IA-NEXT:    xor a5, a4, a2
+; RV64IA-NEXT:    and a5, a5, a3
+; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:    sc.w.aqrl a5, a5, (a0)
+; RV64IA-NEXT:    bnez a5, .LBB19_1
+; RV64IA-NEXT:  .LBB19_3:
+; RV64IA-NEXT:    ret
   %res = cmpxchg i16* %ptr, i16 %cmp, i16 %val seq_cst seq_cst
   ret void
 }
@@ -858,6 +1612,30 @@ define void @cmpxchg_i32_monotonic_monotonic(i32* %ptr, i32 %cmp, i32 %val) {
 ; RV32IA-NEXT:    bnez a4, .LBB20_1
 ; RV32IA-NEXT:  .LBB20_3:
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: cmpxchg_i32_monotonic_monotonic:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sw a1, 4(sp)
+; RV64I-NEXT:    addi a1, sp, 4
+; RV64I-NEXT:    mv a3, zero
+; RV64I-NEXT:    mv a4, zero
+; RV64I-NEXT:    call __atomic_compare_exchange_4
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: cmpxchg_i32_monotonic_monotonic:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:  .LBB20_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.w a3, (a0)
+; RV64IA-NEXT:    bne a3, a1, .LBB20_3
+; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB20_1 Depth=1
+; RV64IA-NEXT:    sc.w a4, a2, (a0)
+; RV64IA-NEXT:    bnez a4, .LBB20_1
+; RV64IA-NEXT:  .LBB20_3:
+; RV64IA-NEXT:    ret
   %res = cmpxchg i32* %ptr, i32 %cmp, i32 %val monotonic monotonic
   ret void
 }
@@ -886,6 +1664,30 @@ define void @cmpxchg_i32_acquire_monotonic(i32* %ptr, i32 %cmp, i32 %val) {
 ; RV32IA-NEXT:    bnez a4, .LBB21_1
 ; RV32IA-NEXT:  .LBB21_3:
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: cmpxchg_i32_acquire_monotonic:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sw a1, 4(sp)
+; RV64I-NEXT:    addi a1, sp, 4
+; RV64I-NEXT:    addi a3, zero, 2
+; RV64I-NEXT:    mv a4, zero
+; RV64I-NEXT:    call __atomic_compare_exchange_4
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: cmpxchg_i32_acquire_monotonic:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:  .LBB21_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.w.aq a3, (a0)
+; RV64IA-NEXT:    bne a3, a1, .LBB21_3
+; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB21_1 Depth=1
+; RV64IA-NEXT:    sc.w a4, a2, (a0)
+; RV64IA-NEXT:    bnez a4, .LBB21_1
+; RV64IA-NEXT:  .LBB21_3:
+; RV64IA-NEXT:    ret
   %res = cmpxchg i32* %ptr, i32 %cmp, i32 %val acquire monotonic
   ret void
 }
@@ -898,7 +1700,7 @@ define void @cmpxchg_i32_acquire_acquire(i32* %ptr, i32 %cmp, i32 %val) {
 ; RV32I-NEXT:    sw a1, 8(sp)
 ; RV32I-NEXT:    addi a1, sp, 8
 ; RV32I-NEXT:    addi a3, zero, 2
-; RV32I-NEXT:    mv a4, a3
+; RV32I-NEXT:    addi a4, zero, 2
 ; RV32I-NEXT:    call __atomic_compare_exchange_4
 ; RV32I-NEXT:    lw ra, 12(sp)
 ; RV32I-NEXT:    addi sp, sp, 16
@@ -914,6 +1716,30 @@ define void @cmpxchg_i32_acquire_acquire(i32* %ptr, i32 %cmp, i32 %val) {
 ; RV32IA-NEXT:    bnez a4, .LBB22_1
 ; RV32IA-NEXT:  .LBB22_3:
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: cmpxchg_i32_acquire_acquire:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sw a1, 4(sp)
+; RV64I-NEXT:    addi a1, sp, 4
+; RV64I-NEXT:    addi a3, zero, 2
+; RV64I-NEXT:    addi a4, zero, 2
+; RV64I-NEXT:    call __atomic_compare_exchange_4
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: cmpxchg_i32_acquire_acquire:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:  .LBB22_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.w.aq a3, (a0)
+; RV64IA-NEXT:    bne a3, a1, .LBB22_3
+; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB22_1 Depth=1
+; RV64IA-NEXT:    sc.w a4, a2, (a0)
+; RV64IA-NEXT:    bnez a4, .LBB22_1
+; RV64IA-NEXT:  .LBB22_3:
+; RV64IA-NEXT:    ret
   %res = cmpxchg i32* %ptr, i32 %cmp, i32 %val acquire acquire
   ret void
 }
@@ -942,6 +1768,30 @@ define void @cmpxchg_i32_release_monotonic(i32* %ptr, i32 %cmp, i32 %val) {
 ; RV32IA-NEXT:    bnez a4, .LBB23_1
 ; RV32IA-NEXT:  .LBB23_3:
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: cmpxchg_i32_release_monotonic:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sw a1, 4(sp)
+; RV64I-NEXT:    addi a1, sp, 4
+; RV64I-NEXT:    addi a3, zero, 3
+; RV64I-NEXT:    mv a4, zero
+; RV64I-NEXT:    call __atomic_compare_exchange_4
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: cmpxchg_i32_release_monotonic:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:  .LBB23_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.w a3, (a0)
+; RV64IA-NEXT:    bne a3, a1, .LBB23_3
+; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB23_1 Depth=1
+; RV64IA-NEXT:    sc.w.rl a4, a2, (a0)
+; RV64IA-NEXT:    bnez a4, .LBB23_1
+; RV64IA-NEXT:  .LBB23_3:
+; RV64IA-NEXT:    ret
   %res = cmpxchg i32* %ptr, i32 %cmp, i32 %val release monotonic
   ret void
 }
@@ -970,6 +1820,30 @@ define void @cmpxchg_i32_release_acquire(i32* %ptr, i32 %cmp, i32 %val) {
 ; RV32IA-NEXT:    bnez a4, .LBB24_1
 ; RV32IA-NEXT:  .LBB24_3:
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: cmpxchg_i32_release_acquire:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sw a1, 4(sp)
+; RV64I-NEXT:    addi a1, sp, 4
+; RV64I-NEXT:    addi a3, zero, 3
+; RV64I-NEXT:    addi a4, zero, 2
+; RV64I-NEXT:    call __atomic_compare_exchange_4
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: cmpxchg_i32_release_acquire:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:  .LBB24_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.w a3, (a0)
+; RV64IA-NEXT:    bne a3, a1, .LBB24_3
+; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB24_1 Depth=1
+; RV64IA-NEXT:    sc.w.rl a4, a2, (a0)
+; RV64IA-NEXT:    bnez a4, .LBB24_1
+; RV64IA-NEXT:  .LBB24_3:
+; RV64IA-NEXT:    ret
   %res = cmpxchg i32* %ptr, i32 %cmp, i32 %val release acquire
   ret void
 }
@@ -998,6 +1872,30 @@ define void @cmpxchg_i32_acq_rel_monotonic(i32* %ptr, i32 %cmp, i32 %val) {
 ; RV32IA-NEXT:    bnez a4, .LBB25_1
 ; RV32IA-NEXT:  .LBB25_3:
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: cmpxchg_i32_acq_rel_monotonic:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sw a1, 4(sp)
+; RV64I-NEXT:    addi a1, sp, 4
+; RV64I-NEXT:    addi a3, zero, 4
+; RV64I-NEXT:    mv a4, zero
+; RV64I-NEXT:    call __atomic_compare_exchange_4
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: cmpxchg_i32_acq_rel_monotonic:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:  .LBB25_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.w.aq a3, (a0)
+; RV64IA-NEXT:    bne a3, a1, .LBB25_3
+; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB25_1 Depth=1
+; RV64IA-NEXT:    sc.w.rl a4, a2, (a0)
+; RV64IA-NEXT:    bnez a4, .LBB25_1
+; RV64IA-NEXT:  .LBB25_3:
+; RV64IA-NEXT:    ret
   %res = cmpxchg i32* %ptr, i32 %cmp, i32 %val acq_rel monotonic
   ret void
 }
@@ -1026,6 +1924,30 @@ define void @cmpxchg_i32_acq_rel_acquire(i32* %ptr, i32 %cmp, i32 %val) {
 ; RV32IA-NEXT:    bnez a4, .LBB26_1
 ; RV32IA-NEXT:  .LBB26_3:
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: cmpxchg_i32_acq_rel_acquire:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sw a1, 4(sp)
+; RV64I-NEXT:    addi a1, sp, 4
+; RV64I-NEXT:    addi a3, zero, 4
+; RV64I-NEXT:    addi a4, zero, 2
+; RV64I-NEXT:    call __atomic_compare_exchange_4
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: cmpxchg_i32_acq_rel_acquire:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:  .LBB26_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.w.aq a3, (a0)
+; RV64IA-NEXT:    bne a3, a1, .LBB26_3
+; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB26_1 Depth=1
+; RV64IA-NEXT:    sc.w.rl a4, a2, (a0)
+; RV64IA-NEXT:    bnez a4, .LBB26_1
+; RV64IA-NEXT:  .LBB26_3:
+; RV64IA-NEXT:    ret
   %res = cmpxchg i32* %ptr, i32 %cmp, i32 %val acq_rel acquire
   ret void
 }
@@ -1054,6 +1976,30 @@ define void @cmpxchg_i32_seq_cst_monotonic(i32* %ptr, i32 %cmp, i32 %val) {
 ; RV32IA-NEXT:    bnez a4, .LBB27_1
 ; RV32IA-NEXT:  .LBB27_3:
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: cmpxchg_i32_seq_cst_monotonic:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sw a1, 4(sp)
+; RV64I-NEXT:    addi a1, sp, 4
+; RV64I-NEXT:    addi a3, zero, 5
+; RV64I-NEXT:    mv a4, zero
+; RV64I-NEXT:    call __atomic_compare_exchange_4
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: cmpxchg_i32_seq_cst_monotonic:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:  .LBB27_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.w.aqrl a3, (a0)
+; RV64IA-NEXT:    bne a3, a1, .LBB27_3
+; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB27_1 Depth=1
+; RV64IA-NEXT:    sc.w.aqrl a4, a2, (a0)
+; RV64IA-NEXT:    bnez a4, .LBB27_1
+; RV64IA-NEXT:  .LBB27_3:
+; RV64IA-NEXT:    ret
   %res = cmpxchg i32* %ptr, i32 %cmp, i32 %val seq_cst monotonic
   ret void
 }
@@ -1082,6 +2028,30 @@ define void @cmpxchg_i32_seq_cst_acquire(i32* %ptr, i32 %cmp, i32 %val) {
 ; RV32IA-NEXT:    bnez a4, .LBB28_1
 ; RV32IA-NEXT:  .LBB28_3:
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: cmpxchg_i32_seq_cst_acquire:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sw a1, 4(sp)
+; RV64I-NEXT:    addi a1, sp, 4
+; RV64I-NEXT:    addi a3, zero, 5
+; RV64I-NEXT:    addi a4, zero, 2
+; RV64I-NEXT:    call __atomic_compare_exchange_4
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: cmpxchg_i32_seq_cst_acquire:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:  .LBB28_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.w.aqrl a3, (a0)
+; RV64IA-NEXT:    bne a3, a1, .LBB28_3
+; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB28_1 Depth=1
+; RV64IA-NEXT:    sc.w.aqrl a4, a2, (a0)
+; RV64IA-NEXT:    bnez a4, .LBB28_1
+; RV64IA-NEXT:  .LBB28_3:
+; RV64IA-NEXT:    ret
   %res = cmpxchg i32* %ptr, i32 %cmp, i32 %val seq_cst acquire
   ret void
 }
@@ -1094,7 +2064,7 @@ define void @cmpxchg_i32_seq_cst_seq_cst(i32* %ptr, i32 %cmp, i32 %val) {
 ; RV32I-NEXT:    sw a1, 8(sp)
 ; RV32I-NEXT:    addi a1, sp, 8
 ; RV32I-NEXT:    addi a3, zero, 5
-; RV32I-NEXT:    mv a4, a3
+; RV32I-NEXT:    addi a4, zero, 5
 ; RV32I-NEXT:    call __atomic_compare_exchange_4
 ; RV32I-NEXT:    lw ra, 12(sp)
 ; RV32I-NEXT:    addi sp, sp, 16
@@ -1110,6 +2080,30 @@ define void @cmpxchg_i32_seq_cst_seq_cst(i32* %ptr, i32 %cmp, i32 %val) {
 ; RV32IA-NEXT:    bnez a4, .LBB29_1
 ; RV32IA-NEXT:  .LBB29_3:
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: cmpxchg_i32_seq_cst_seq_cst:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sw a1, 4(sp)
+; RV64I-NEXT:    addi a1, sp, 4
+; RV64I-NEXT:    addi a3, zero, 5
+; RV64I-NEXT:    addi a4, zero, 5
+; RV64I-NEXT:    call __atomic_compare_exchange_4
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: cmpxchg_i32_seq_cst_seq_cst:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:  .LBB29_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.w.aqrl a3, (a0)
+; RV64IA-NEXT:    bne a3, a1, .LBB29_3
+; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB29_1 Depth=1
+; RV64IA-NEXT:    sc.w.aqrl a4, a2, (a0)
+; RV64IA-NEXT:    bnez a4, .LBB29_1
+; RV64IA-NEXT:  .LBB29_3:
+; RV64IA-NEXT:    ret
   %res = cmpxchg i32* %ptr, i32 %cmp, i32 %val seq_cst seq_cst
   ret void
 }
@@ -1146,6 +2140,30 @@ define void @cmpxchg_i64_monotonic_monotonic(i64* %ptr, i64 %cmp, i64 %val) {
 ; RV32IA-NEXT:    lw ra, 12(sp)
 ; RV32IA-NEXT:    addi sp, sp, 16
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: cmpxchg_i64_monotonic_monotonic:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd a1, 0(sp)
+; RV64I-NEXT:    mv a1, sp
+; RV64I-NEXT:    mv a3, zero
+; RV64I-NEXT:    mv a4, zero
+; RV64I-NEXT:    call __atomic_compare_exchange_8
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: cmpxchg_i64_monotonic_monotonic:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:  .LBB30_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.d a3, (a0)
+; RV64IA-NEXT:    bne a3, a1, .LBB30_3
+; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB30_1 Depth=1
+; RV64IA-NEXT:    sc.d a4, a2, (a0)
+; RV64IA-NEXT:    bnez a4, .LBB30_1
+; RV64IA-NEXT:  .LBB30_3:
+; RV64IA-NEXT:    ret
   %res = cmpxchg i64* %ptr, i64 %cmp, i64 %val monotonic monotonic
   ret void
 }
@@ -1155,13 +2173,12 @@ define void @cmpxchg_i64_acquire_monotonic(i64* %ptr, i64 %cmp, i64 %val) {
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
 ; RV32I-NEXT:    sw ra, 12(sp)
-; RV32I-NEXT:    mv a5, a4
 ; RV32I-NEXT:    sw a2, 4(sp)
 ; RV32I-NEXT:    sw a1, 0(sp)
 ; RV32I-NEXT:    mv a1, sp
-; RV32I-NEXT:    addi a4, zero, 2
 ; RV32I-NEXT:    mv a2, a3
-; RV32I-NEXT:    mv a3, a5
+; RV32I-NEXT:    mv a3, a4
+; RV32I-NEXT:    addi  a4, zero, 2
 ; RV32I-NEXT:    mv a5, zero
 ; RV32I-NEXT:    call __atomic_compare_exchange_8
 ; RV32I-NEXT:    lw ra, 12(sp)
@@ -1172,18 +2189,41 @@ define void @cmpxchg_i64_acquire_monotonic(i64* %ptr, i64 %cmp, i64 %val) {
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    addi sp, sp, -16
 ; RV32IA-NEXT:    sw ra, 12(sp)
-; RV32IA-NEXT:    mv a5, a4
 ; RV32IA-NEXT:    sw a2, 4(sp)
 ; RV32IA-NEXT:    sw a1, 0(sp)
 ; RV32IA-NEXT:    mv a1, sp
-; RV32IA-NEXT:    addi a4, zero, 2
 ; RV32IA-NEXT:    mv a2, a3
-; RV32IA-NEXT:    mv a3, a5
+; RV32IA-NEXT:    mv a3, a4
+; RV32IA-NEXT:    addi a4, zero, 2
 ; RV32IA-NEXT:    mv a5, zero
 ; RV32IA-NEXT:    call __atomic_compare_exchange_8
 ; RV32IA-NEXT:    lw ra, 12(sp)
 ; RV32IA-NEXT:    addi sp, sp, 16
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: cmpxchg_i64_acquire_monotonic:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd a1, 0(sp)
+; RV64I-NEXT:    mv a1, sp
+; RV64I-NEXT:    addi a3, zero, 2
+; RV64I-NEXT:    mv a4, zero
+; RV64I-NEXT:    call __atomic_compare_exchange_8
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: cmpxchg_i64_acquire_monotonic:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:  .LBB31_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.d.aq a3, (a0)
+; RV64IA-NEXT:    bne a3, a1, .LBB31_3
+; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB31_1 Depth=1
+; RV64IA-NEXT:    sc.d a4, a2, (a0)
+; RV64IA-NEXT:    bnez a4, .LBB31_1
+; RV64IA-NEXT:  .LBB31_3:
+; RV64IA-NEXT:    ret
   %res = cmpxchg i64* %ptr, i64 %cmp, i64 %val acquire monotonic
   ret void
 }
@@ -1196,10 +2236,10 @@ define void @cmpxchg_i64_acquire_acquire(i64* %ptr, i64 %cmp, i64 %val) {
 ; RV32I-NEXT:    sw a2, 4(sp)
 ; RV32I-NEXT:    sw a1, 0(sp)
 ; RV32I-NEXT:    mv a1, sp
-; RV32I-NEXT:    addi a5, zero, 2
 ; RV32I-NEXT:    mv a2, a3
 ; RV32I-NEXT:    mv a3, a4
-; RV32I-NEXT:    mv a4, a5
+; RV32I-NEXT:    addi a4, zero, 2
+; RV32I-NEXT:    addi a5, zero, 2
 ; RV32I-NEXT:    call __atomic_compare_exchange_8
 ; RV32I-NEXT:    lw ra, 12(sp)
 ; RV32I-NEXT:    addi sp, sp, 16
@@ -1212,14 +2252,38 @@ define void @cmpxchg_i64_acquire_acquire(i64* %ptr, i64 %cmp, i64 %val) {
 ; RV32IA-NEXT:    sw a2, 4(sp)
 ; RV32IA-NEXT:    sw a1, 0(sp)
 ; RV32IA-NEXT:    mv a1, sp
-; RV32IA-NEXT:    addi a5, zero, 2
 ; RV32IA-NEXT:    mv a2, a3
 ; RV32IA-NEXT:    mv a3, a4
-; RV32IA-NEXT:    mv a4, a5
+; RV32IA-NEXT:    addi a4, zero, 2
+; RV32IA-NEXT:    addi a5, zero, 2
 ; RV32IA-NEXT:    call __atomic_compare_exchange_8
 ; RV32IA-NEXT:    lw ra, 12(sp)
 ; RV32IA-NEXT:    addi sp, sp, 16
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: cmpxchg_i64_acquire_acquire:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd a1, 0(sp)
+; RV64I-NEXT:    mv a1, sp
+; RV64I-NEXT:    addi a3, zero, 2
+; RV64I-NEXT:    addi a4, zero, 2
+; RV64I-NEXT:    call __atomic_compare_exchange_8
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: cmpxchg_i64_acquire_acquire:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:  .LBB32_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.d.aq a3, (a0)
+; RV64IA-NEXT:    bne a3, a1, .LBB32_3
+; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB32_1 Depth=1
+; RV64IA-NEXT:    sc.d a4, a2, (a0)
+; RV64IA-NEXT:    bnez a4, .LBB32_1
+; RV64IA-NEXT:  .LBB32_3:
+; RV64IA-NEXT:    ret
   %res = cmpxchg i64* %ptr, i64 %cmp, i64 %val acquire acquire
   ret void
 }
@@ -1229,13 +2293,12 @@ define void @cmpxchg_i64_release_monotonic(i64* %ptr, i64 %cmp, i64 %val) {
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
 ; RV32I-NEXT:    sw ra, 12(sp)
-; RV32I-NEXT:    mv a5, a4
 ; RV32I-NEXT:    sw a2, 4(sp)
 ; RV32I-NEXT:    sw a1, 0(sp)
 ; RV32I-NEXT:    mv a1, sp
-; RV32I-NEXT:    addi a4, zero, 3
 ; RV32I-NEXT:    mv a2, a3
-; RV32I-NEXT:    mv a3, a5
+; RV32I-NEXT:    mv a3, a4
+; RV32I-NEXT:    addi a4, zero, 3
 ; RV32I-NEXT:    mv a5, zero
 ; RV32I-NEXT:    call __atomic_compare_exchange_8
 ; RV32I-NEXT:    lw ra, 12(sp)
@@ -1246,18 +2309,41 @@ define void @cmpxchg_i64_release_monotonic(i64* %ptr, i64 %cmp, i64 %val) {
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    addi sp, sp, -16
 ; RV32IA-NEXT:    sw ra, 12(sp)
-; RV32IA-NEXT:    mv a5, a4
 ; RV32IA-NEXT:    sw a2, 4(sp)
 ; RV32IA-NEXT:    sw a1, 0(sp)
 ; RV32IA-NEXT:    mv a1, sp
-; RV32IA-NEXT:    addi a4, zero, 3
 ; RV32IA-NEXT:    mv a2, a3
-; RV32IA-NEXT:    mv a3, a5
+; RV32IA-NEXT:    mv a3, a4
+; RV32IA-NEXT:    addi a4, zero, 3
 ; RV32IA-NEXT:    mv a5, zero
 ; RV32IA-NEXT:    call __atomic_compare_exchange_8
 ; RV32IA-NEXT:    lw ra, 12(sp)
 ; RV32IA-NEXT:    addi sp, sp, 16
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: cmpxchg_i64_release_monotonic:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd a1, 0(sp)
+; RV64I-NEXT:    mv a1, sp
+; RV64I-NEXT:    addi a3, zero, 3
+; RV64I-NEXT:    mv a4, zero
+; RV64I-NEXT:    call __atomic_compare_exchange_8
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: cmpxchg_i64_release_monotonic:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:  .LBB33_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.d a3, (a0)
+; RV64IA-NEXT:    bne a3, a1, .LBB33_3
+; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB33_1 Depth=1
+; RV64IA-NEXT:    sc.d.rl a4, a2, (a0)
+; RV64IA-NEXT:    bnez a4, .LBB33_1
+; RV64IA-NEXT:  .LBB33_3:
+; RV64IA-NEXT:    ret
   %res = cmpxchg i64* %ptr, i64 %cmp, i64 %val release monotonic
   ret void
 }
@@ -1267,14 +2353,13 @@ define void @cmpxchg_i64_release_acquire(i64* %ptr, i64 %cmp, i64 %val) {
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
 ; RV32I-NEXT:    sw ra, 12(sp)
-; RV32I-NEXT:    mv a6, a4
 ; RV32I-NEXT:    sw a2, 4(sp)
 ; RV32I-NEXT:    sw a1, 0(sp)
 ; RV32I-NEXT:    mv a1, sp
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:    mv a3, a4
 ; RV32I-NEXT:    addi a4, zero, 3
 ; RV32I-NEXT:    addi a5, zero, 2
-; RV32I-NEXT:    mv a2, a3
-; RV32I-NEXT:    mv a3, a6
 ; RV32I-NEXT:    call __atomic_compare_exchange_8
 ; RV32I-NEXT:    lw ra, 12(sp)
 ; RV32I-NEXT:    addi sp, sp, 16
@@ -1284,18 +2369,41 @@ define void @cmpxchg_i64_release_acquire(i64* %ptr, i64 %cmp, i64 %val) {
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    addi sp, sp, -16
 ; RV32IA-NEXT:    sw ra, 12(sp)
-; RV32IA-NEXT:    mv a6, a4
 ; RV32IA-NEXT:    sw a2, 4(sp)
 ; RV32IA-NEXT:    sw a1, 0(sp)
 ; RV32IA-NEXT:    mv a1, sp
+; RV32IA-NEXT:    mv a2, a3
+; RV32IA-NEXT:    mv a3, a4
 ; RV32IA-NEXT:    addi a4, zero, 3
 ; RV32IA-NEXT:    addi a5, zero, 2
-; RV32IA-NEXT:    mv a2, a3
-; RV32IA-NEXT:    mv a3, a6
 ; RV32IA-NEXT:    call __atomic_compare_exchange_8
 ; RV32IA-NEXT:    lw ra, 12(sp)
 ; RV32IA-NEXT:    addi sp, sp, 16
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: cmpxchg_i64_release_acquire:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd a1, 0(sp)
+; RV64I-NEXT:    mv a1, sp
+; RV64I-NEXT:    addi a3, zero, 3
+; RV64I-NEXT:    addi a4, zero, 2
+; RV64I-NEXT:    call __atomic_compare_exchange_8
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: cmpxchg_i64_release_acquire:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:  .LBB34_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.d a3, (a0)
+; RV64IA-NEXT:    bne a3, a1, .LBB34_3
+; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB34_1 Depth=1
+; RV64IA-NEXT:    sc.d.rl a4, a2, (a0)
+; RV64IA-NEXT:    bnez a4, .LBB34_1
+; RV64IA-NEXT:  .LBB34_3:
+; RV64IA-NEXT:    ret
   %res = cmpxchg i64* %ptr, i64 %cmp, i64 %val release acquire
   ret void
 }
@@ -1305,13 +2413,12 @@ define void @cmpxchg_i64_acq_rel_monotonic(i64* %ptr, i64 %cmp, i64 %val) {
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
 ; RV32I-NEXT:    sw ra, 12(sp)
-; RV32I-NEXT:    mv a5, a4
 ; RV32I-NEXT:    sw a2, 4(sp)
 ; RV32I-NEXT:    sw a1, 0(sp)
 ; RV32I-NEXT:    mv a1, sp
-; RV32I-NEXT:    addi a4, zero, 4
 ; RV32I-NEXT:    mv a2, a3
-; RV32I-NEXT:    mv a3, a5
+; RV32I-NEXT:    mv a3, a4
+; RV32I-NEXT:    addi a4, zero, 4
 ; RV32I-NEXT:    mv a5, zero
 ; RV32I-NEXT:    call __atomic_compare_exchange_8
 ; RV32I-NEXT:    lw ra, 12(sp)
@@ -1322,18 +2429,41 @@ define void @cmpxchg_i64_acq_rel_monotonic(i64* %ptr, i64 %cmp, i64 %val) {
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    addi sp, sp, -16
 ; RV32IA-NEXT:    sw ra, 12(sp)
-; RV32IA-NEXT:    mv a5, a4
 ; RV32IA-NEXT:    sw a2, 4(sp)
 ; RV32IA-NEXT:    sw a1, 0(sp)
 ; RV32IA-NEXT:    mv a1, sp
-; RV32IA-NEXT:    addi a4, zero, 4
 ; RV32IA-NEXT:    mv a2, a3
-; RV32IA-NEXT:    mv a3, a5
+; RV32IA-NEXT:    mv a3, a4
+; RV32IA-NEXT:    addi a4, zero, 4
 ; RV32IA-NEXT:    mv a5, zero
 ; RV32IA-NEXT:    call __atomic_compare_exchange_8
 ; RV32IA-NEXT:    lw ra, 12(sp)
 ; RV32IA-NEXT:    addi sp, sp, 16
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: cmpxchg_i64_acq_rel_monotonic:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd a1, 0(sp)
+; RV64I-NEXT:    mv a1, sp
+; RV64I-NEXT:    addi a3, zero, 4
+; RV64I-NEXT:    mv a4, zero
+; RV64I-NEXT:    call __atomic_compare_exchange_8
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: cmpxchg_i64_acq_rel_monotonic:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:  .LBB35_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.d.aq a3, (a0)
+; RV64IA-NEXT:    bne a3, a1, .LBB35_3
+; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB35_1 Depth=1
+; RV64IA-NEXT:    sc.d.rl a4, a2, (a0)
+; RV64IA-NEXT:    bnez a4, .LBB35_1
+; RV64IA-NEXT:  .LBB35_3:
+; RV64IA-NEXT:    ret
   %res = cmpxchg i64* %ptr, i64 %cmp, i64 %val acq_rel monotonic
   ret void
 }
@@ -1343,14 +2473,13 @@ define void @cmpxchg_i64_acq_rel_acquire(i64* %ptr, i64 %cmp, i64 %val) {
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
 ; RV32I-NEXT:    sw ra, 12(sp)
-; RV32I-NEXT:    mv a6, a4
 ; RV32I-NEXT:    sw a2, 4(sp)
 ; RV32I-NEXT:    sw a1, 0(sp)
 ; RV32I-NEXT:    mv a1, sp
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:    mv a3, a4
 ; RV32I-NEXT:    addi a4, zero, 4
 ; RV32I-NEXT:    addi a5, zero, 2
-; RV32I-NEXT:    mv a2, a3
-; RV32I-NEXT:    mv a3, a6
 ; RV32I-NEXT:    call __atomic_compare_exchange_8
 ; RV32I-NEXT:    lw ra, 12(sp)
 ; RV32I-NEXT:    addi sp, sp, 16
@@ -1360,18 +2489,41 @@ define void @cmpxchg_i64_acq_rel_acquire(i64* %ptr, i64 %cmp, i64 %val) {
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    addi sp, sp, -16
 ; RV32IA-NEXT:    sw ra, 12(sp)
-; RV32IA-NEXT:    mv a6, a4
 ; RV32IA-NEXT:    sw a2, 4(sp)
 ; RV32IA-NEXT:    sw a1, 0(sp)
 ; RV32IA-NEXT:    mv a1, sp
+; RV32IA-NEXT:    mv a2, a3
+; RV32IA-NEXT:    mv a3, a4
 ; RV32IA-NEXT:    addi a4, zero, 4
 ; RV32IA-NEXT:    addi a5, zero, 2
-; RV32IA-NEXT:    mv a2, a3
-; RV32IA-NEXT:    mv a3, a6
 ; RV32IA-NEXT:    call __atomic_compare_exchange_8
 ; RV32IA-NEXT:    lw ra, 12(sp)
 ; RV32IA-NEXT:    addi sp, sp, 16
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: cmpxchg_i64_acq_rel_acquire:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd a1, 0(sp)
+; RV64I-NEXT:    mv a1, sp
+; RV64I-NEXT:    addi a3, zero, 4
+; RV64I-NEXT:    addi a4, zero, 2
+; RV64I-NEXT:    call __atomic_compare_exchange_8
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: cmpxchg_i64_acq_rel_acquire:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:  .LBB36_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.d.aq a3, (a0)
+; RV64IA-NEXT:    bne a3, a1, .LBB36_3
+; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB36_1 Depth=1
+; RV64IA-NEXT:    sc.d.rl a4, a2, (a0)
+; RV64IA-NEXT:    bnez a4, .LBB36_1
+; RV64IA-NEXT:  .LBB36_3:
+; RV64IA-NEXT:    ret
   %res = cmpxchg i64* %ptr, i64 %cmp, i64 %val acq_rel acquire
   ret void
 }
@@ -1381,13 +2533,12 @@ define void @cmpxchg_i64_seq_cst_monotonic(i64* %ptr, i64 %cmp, i64 %val) {
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
 ; RV32I-NEXT:    sw ra, 12(sp)
-; RV32I-NEXT:    mv a5, a4
 ; RV32I-NEXT:    sw a2, 4(sp)
 ; RV32I-NEXT:    sw a1, 0(sp)
 ; RV32I-NEXT:    mv a1, sp
-; RV32I-NEXT:    addi a4, zero, 5
 ; RV32I-NEXT:    mv a2, a3
-; RV32I-NEXT:    mv a3, a5
+; RV32I-NEXT:    mv a3, a4
+; RV32I-NEXT:    addi a4, zero, 5
 ; RV32I-NEXT:    mv a5, zero
 ; RV32I-NEXT:    call __atomic_compare_exchange_8
 ; RV32I-NEXT:    lw ra, 12(sp)
@@ -1398,18 +2549,41 @@ define void @cmpxchg_i64_seq_cst_monotonic(i64* %ptr, i64 %cmp, i64 %val) {
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    addi sp, sp, -16
 ; RV32IA-NEXT:    sw ra, 12(sp)
-; RV32IA-NEXT:    mv a5, a4
 ; RV32IA-NEXT:    sw a2, 4(sp)
 ; RV32IA-NEXT:    sw a1, 0(sp)
 ; RV32IA-NEXT:    mv a1, sp
-; RV32IA-NEXT:    addi a4, zero, 5
 ; RV32IA-NEXT:    mv a2, a3
-; RV32IA-NEXT:    mv a3, a5
+; RV32IA-NEXT:    mv a3, a4
+; RV32IA-NEXT:    addi a4, zero, 5
 ; RV32IA-NEXT:    mv a5, zero
 ; RV32IA-NEXT:    call __atomic_compare_exchange_8
 ; RV32IA-NEXT:    lw ra, 12(sp)
 ; RV32IA-NEXT:    addi sp, sp, 16
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: cmpxchg_i64_seq_cst_monotonic:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd a1, 0(sp)
+; RV64I-NEXT:    mv a1, sp
+; RV64I-NEXT:    addi a3, zero, 5
+; RV64I-NEXT:    mv a4, zero
+; RV64I-NEXT:    call __atomic_compare_exchange_8
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: cmpxchg_i64_seq_cst_monotonic:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:  .LBB37_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.d.aqrl a3, (a0)
+; RV64IA-NEXT:    bne a3, a1, .LBB37_3
+; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB37_1 Depth=1
+; RV64IA-NEXT:    sc.d.aqrl a4, a2, (a0)
+; RV64IA-NEXT:    bnez a4, .LBB37_1
+; RV64IA-NEXT:  .LBB37_3:
+; RV64IA-NEXT:    ret
   %res = cmpxchg i64* %ptr, i64 %cmp, i64 %val seq_cst monotonic
   ret void
 }
@@ -1419,14 +2593,13 @@ define void @cmpxchg_i64_seq_cst_acquire(i64* %ptr, i64 %cmp, i64 %val) {
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
 ; RV32I-NEXT:    sw ra, 12(sp)
-; RV32I-NEXT:    mv a6, a4
 ; RV32I-NEXT:    sw a2, 4(sp)
 ; RV32I-NEXT:    sw a1, 0(sp)
 ; RV32I-NEXT:    mv a1, sp
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:    mv a3, a4
 ; RV32I-NEXT:    addi a4, zero, 5
 ; RV32I-NEXT:    addi a5, zero, 2
-; RV32I-NEXT:    mv a2, a3
-; RV32I-NEXT:    mv a3, a6
 ; RV32I-NEXT:    call __atomic_compare_exchange_8
 ; RV32I-NEXT:    lw ra, 12(sp)
 ; RV32I-NEXT:    addi sp, sp, 16
@@ -1436,18 +2609,41 @@ define void @cmpxchg_i64_seq_cst_acquire(i64* %ptr, i64 %cmp, i64 %val) {
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    addi sp, sp, -16
 ; RV32IA-NEXT:    sw ra, 12(sp)
-; RV32IA-NEXT:    mv a6, a4
 ; RV32IA-NEXT:    sw a2, 4(sp)
 ; RV32IA-NEXT:    sw a1, 0(sp)
 ; RV32IA-NEXT:    mv a1, sp
+; RV32IA-NEXT:    mv a2, a3
+; RV32IA-NEXT:    mv a3, a4
 ; RV32IA-NEXT:    addi a4, zero, 5
 ; RV32IA-NEXT:    addi a5, zero, 2
-; RV32IA-NEXT:    mv a2, a3
-; RV32IA-NEXT:    mv a3, a6
 ; RV32IA-NEXT:    call __atomic_compare_exchange_8
 ; RV32IA-NEXT:    lw ra, 12(sp)
 ; RV32IA-NEXT:    addi sp, sp, 16
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: cmpxchg_i64_seq_cst_acquire:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd a1, 0(sp)
+; RV64I-NEXT:    mv a1, sp
+; RV64I-NEXT:    addi a3, zero, 5
+; RV64I-NEXT:    addi a4, zero, 2
+; RV64I-NEXT:    call __atomic_compare_exchange_8
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: cmpxchg_i64_seq_cst_acquire:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:  .LBB38_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.d.aqrl a3, (a0)
+; RV64IA-NEXT:    bne a3, a1, .LBB38_3
+; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB38_1 Depth=1
+; RV64IA-NEXT:    sc.d.aqrl a4, a2, (a0)
+; RV64IA-NEXT:    bnez a4, .LBB38_1
+; RV64IA-NEXT:  .LBB38_3:
+; RV64IA-NEXT:    ret
   %res = cmpxchg i64* %ptr, i64 %cmp, i64 %val seq_cst acquire
   ret void
 }
@@ -1460,10 +2656,10 @@ define void @cmpxchg_i64_seq_cst_seq_cst(i64* %ptr, i64 %cmp, i64 %val) {
 ; RV32I-NEXT:    sw a2, 4(sp)
 ; RV32I-NEXT:    sw a1, 0(sp)
 ; RV32I-NEXT:    mv a1, sp
-; RV32I-NEXT:    addi a5, zero, 5
 ; RV32I-NEXT:    mv a2, a3
 ; RV32I-NEXT:    mv a3, a4
-; RV32I-NEXT:    mv a4, a5
+; RV32I-NEXT:    addi a4, zero, 5
+; RV32I-NEXT:    addi a5, zero, 5
 ; RV32I-NEXT:    call __atomic_compare_exchange_8
 ; RV32I-NEXT:    lw ra, 12(sp)
 ; RV32I-NEXT:    addi sp, sp, 16
@@ -1476,14 +2672,38 @@ define void @cmpxchg_i64_seq_cst_seq_cst(i64* %ptr, i64 %cmp, i64 %val) {
 ; RV32IA-NEXT:    sw a2, 4(sp)
 ; RV32IA-NEXT:    sw a1, 0(sp)
 ; RV32IA-NEXT:    mv a1, sp
-; RV32IA-NEXT:    addi a5, zero, 5
 ; RV32IA-NEXT:    mv a2, a3
 ; RV32IA-NEXT:    mv a3, a4
-; RV32IA-NEXT:    mv a4, a5
+; RV32IA-NEXT:    addi a4, zero, 5
+; RV32IA-NEXT:    addi a5, zero, 5
 ; RV32IA-NEXT:    call __atomic_compare_exchange_8
 ; RV32IA-NEXT:    lw ra, 12(sp)
 ; RV32IA-NEXT:    addi sp, sp, 16
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: cmpxchg_i64_seq_cst_seq_cst:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sd a1, 0(sp)
+; RV64I-NEXT:    mv a1, sp
+; RV64I-NEXT:    addi a3, zero, 5
+; RV64I-NEXT:    addi a4, zero, 5
+; RV64I-NEXT:    call __atomic_compare_exchange_8
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: cmpxchg_i64_seq_cst_seq_cst:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:  .LBB39_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.d.aqrl a3, (a0)
+; RV64IA-NEXT:    bne a3, a1, .LBB39_3
+; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB39_1 Depth=1
+; RV64IA-NEXT:    sc.d.aqrl a4, a2, (a0)
+; RV64IA-NEXT:    bnez a4, .LBB39_1
+; RV64IA-NEXT:  .LBB39_3:
+; RV64IA-NEXT:    ret
   %res = cmpxchg i64* %ptr, i64 %cmp, i64 %val seq_cst seq_cst
   ret void
 }
diff --git a/test/CodeGen/RISCV/atomic-fence.ll b/test/CodeGen/RISCV/atomic-fence.ll
index 174977d98d6465cc6b17598276a19998dd12f238..120f65bfcfdd53decf5d772296d399864426dcb5 100644
--- a/test/CodeGen/RISCV/atomic-fence.ll
+++ b/test/CodeGen/RISCV/atomic-fence.ll
@@ -3,12 +3,21 @@
 ; RUN:   | FileCheck -check-prefix=RV32I %s
 ; RUN: llc -mtriple=riscv32 -mattr=+a -verify-machineinstrs < %s \
 ; RUN:   | FileCheck -check-prefix=RV32I %s
+; RUN: llc -mtriple=riscv64 -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=RV64I %s
+; RUN: llc -mtriple=riscv64 -mattr=+a -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=RV64I %s
 
 define void @fence_acquire() nounwind {
 ; RV32I-LABEL: fence_acquire:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    fence r, rw
 ; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: fence_acquire:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    fence r, rw
+; RV64I-NEXT:    ret
   fence acquire
   ret void
 }
@@ -18,6 +27,11 @@ define void @fence_release() nounwind {
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    fence rw, w
 ; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: fence_release:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    fence rw, w
+; RV64I-NEXT:    ret
   fence release
   ret void
 }
@@ -27,6 +41,11 @@ define void @fence_acq_rel() nounwind {
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    fence.tso
 ; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: fence_acq_rel:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    fence.tso
+; RV64I-NEXT:    ret
   fence acq_rel
   ret void
 }
@@ -36,6 +55,11 @@ define void @fence_seq_cst() nounwind {
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    fence rw, rw
 ; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: fence_seq_cst:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    fence rw, rw
+; RV64I-NEXT:    ret
   fence seq_cst
   ret void
 }
diff --git a/test/CodeGen/RISCV/atomic-load-store.ll b/test/CodeGen/RISCV/atomic-load-store.ll
index ba2b594c1c769d042e36748bd021c99d8e64a387..da6f986eb752f243dcc03d6b0f3241e6ede9b709 100644
--- a/test/CodeGen/RISCV/atomic-load-store.ll
+++ b/test/CodeGen/RISCV/atomic-load-store.ll
@@ -3,6 +3,10 @@
 ; RUN:   | FileCheck -check-prefix=RV32I %s
 ; RUN: llc -mtriple=riscv32 -mattr=+a -verify-machineinstrs < %s \
 ; RUN:   | FileCheck -check-prefix=RV32IA %s
+; RUN: llc -mtriple=riscv64 -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=RV64I %s
+; RUN: llc -mtriple=riscv64 -mattr=+a -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=RV64IA %s
 
 define i8 @atomic_load_i8_unordered(i8 *%a) nounwind {
 ; RV32I-LABEL: atomic_load_i8_unordered:
@@ -19,6 +23,21 @@ define i8 @atomic_load_i8_unordered(i8 *%a) nounwind {
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    lb a0, 0(a0)
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomic_load_i8_unordered:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    mv a1, zero
+; RV64I-NEXT:    call __atomic_load_1
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomic_load_i8_unordered:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    lb a0, 0(a0)
+; RV64IA-NEXT:    ret
   %1 = load atomic i8, i8* %a unordered, align 1
   ret i8 %1
 }
@@ -38,6 +57,21 @@ define i8 @atomic_load_i8_monotonic(i8 *%a) nounwind {
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    lb a0, 0(a0)
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomic_load_i8_monotonic:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    mv a1, zero
+; RV64I-NEXT:    call __atomic_load_1
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomic_load_i8_monotonic:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    lb a0, 0(a0)
+; RV64IA-NEXT:    ret
   %1 = load atomic i8, i8* %a monotonic, align 1
   ret i8 %1
 }
@@ -58,6 +92,22 @@ define i8 @atomic_load_i8_acquire(i8 *%a) nounwind {
 ; RV32IA-NEXT:    lb a0, 0(a0)
 ; RV32IA-NEXT:    fence r, rw
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomic_load_i8_acquire:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a1, zero, 2
+; RV64I-NEXT:    call __atomic_load_1
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomic_load_i8_acquire:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    lb a0, 0(a0)
+; RV64IA-NEXT:    fence r, rw
+; RV64IA-NEXT:    ret
   %1 = load atomic i8, i8* %a acquire, align 1
   ret i8 %1
 }
@@ -79,6 +129,23 @@ define i8 @atomic_load_i8_seq_cst(i8 *%a) nounwind {
 ; RV32IA-NEXT:    lb a0, 0(a0)
 ; RV32IA-NEXT:    fence r, rw
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomic_load_i8_seq_cst:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a1, zero, 5
+; RV64I-NEXT:    call __atomic_load_1
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomic_load_i8_seq_cst:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    fence rw, rw
+; RV64IA-NEXT:    lb a0, 0(a0)
+; RV64IA-NEXT:    fence r, rw
+; RV64IA-NEXT:    ret
   %1 = load atomic i8, i8* %a seq_cst, align 1
   ret i8 %1
 }
@@ -98,6 +165,21 @@ define i16 @atomic_load_i16_unordered(i16 *%a) nounwind {
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    lh a0, 0(a0)
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomic_load_i16_unordered:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    mv a1, zero
+; RV64I-NEXT:    call __atomic_load_2
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomic_load_i16_unordered:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    lh a0, 0(a0)
+; RV64IA-NEXT:    ret
   %1 = load atomic i16, i16* %a unordered, align 2
   ret i16 %1
 }
@@ -117,6 +199,21 @@ define i16 @atomic_load_i16_monotonic(i16 *%a) nounwind {
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    lh a0, 0(a0)
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomic_load_i16_monotonic:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    mv a1, zero
+; RV64I-NEXT:    call __atomic_load_2
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomic_load_i16_monotonic:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    lh a0, 0(a0)
+; RV64IA-NEXT:    ret
   %1 = load atomic i16, i16* %a monotonic, align 2
   ret i16 %1
 }
@@ -137,6 +234,22 @@ define i16 @atomic_load_i16_acquire(i16 *%a) nounwind {
 ; RV32IA-NEXT:    lh a0, 0(a0)
 ; RV32IA-NEXT:    fence r, rw
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomic_load_i16_acquire:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a1, zero, 2
+; RV64I-NEXT:    call __atomic_load_2
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomic_load_i16_acquire:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    lh a0, 0(a0)
+; RV64IA-NEXT:    fence r, rw
+; RV64IA-NEXT:    ret
   %1 = load atomic i16, i16* %a acquire, align 2
   ret i16 %1
 }
@@ -158,6 +271,23 @@ define i16 @atomic_load_i16_seq_cst(i16 *%a) nounwind {
 ; RV32IA-NEXT:    lh a0, 0(a0)
 ; RV32IA-NEXT:    fence r, rw
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomic_load_i16_seq_cst:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a1, zero, 5
+; RV64I-NEXT:    call __atomic_load_2
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomic_load_i16_seq_cst:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    fence rw, rw
+; RV64IA-NEXT:    lh a0, 0(a0)
+; RV64IA-NEXT:    fence r, rw
+; RV64IA-NEXT:    ret
   %1 = load atomic i16, i16* %a seq_cst, align 2
   ret i16 %1
 }
@@ -177,6 +307,21 @@ define i32 @atomic_load_i32_unordered(i32 *%a) nounwind {
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    lw a0, 0(a0)
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomic_load_i32_unordered:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    mv a1, zero
+; RV64I-NEXT:    call __atomic_load_4
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomic_load_i32_unordered:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    lw a0, 0(a0)
+; RV64IA-NEXT:    ret
   %1 = load atomic i32, i32* %a unordered, align 4
   ret i32 %1
 }
@@ -196,6 +341,21 @@ define i32 @atomic_load_i32_monotonic(i32 *%a) nounwind {
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    lw a0, 0(a0)
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomic_load_i32_monotonic:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    mv a1, zero
+; RV64I-NEXT:    call __atomic_load_4
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomic_load_i32_monotonic:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    lw a0, 0(a0)
+; RV64IA-NEXT:    ret
   %1 = load atomic i32, i32* %a monotonic, align 4
   ret i32 %1
 }
@@ -216,6 +376,22 @@ define i32 @atomic_load_i32_acquire(i32 *%a) nounwind {
 ; RV32IA-NEXT:    lw a0, 0(a0)
 ; RV32IA-NEXT:    fence r, rw
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomic_load_i32_acquire:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a1, zero, 2
+; RV64I-NEXT:    call __atomic_load_4
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomic_load_i32_acquire:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    lw a0, 0(a0)
+; RV64IA-NEXT:    fence r, rw
+; RV64IA-NEXT:    ret
   %1 = load atomic i32, i32* %a acquire, align 4
   ret i32 %1
 }
@@ -237,6 +413,23 @@ define i32 @atomic_load_i32_seq_cst(i32 *%a) nounwind {
 ; RV32IA-NEXT:    lw a0, 0(a0)
 ; RV32IA-NEXT:    fence r, rw
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomic_load_i32_seq_cst:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a1, zero, 5
+; RV64I-NEXT:    call __atomic_load_4
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomic_load_i32_seq_cst:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    fence rw, rw
+; RV64IA-NEXT:    lw a0, 0(a0)
+; RV64IA-NEXT:    fence r, rw
+; RV64IA-NEXT:    ret
   %1 = load atomic i32, i32* %a seq_cst, align 4
   ret i32 %1
 }
@@ -261,6 +454,21 @@ define i64 @atomic_load_i64_unordered(i64 *%a) nounwind {
 ; RV32IA-NEXT:    lw ra, 12(sp)
 ; RV32IA-NEXT:    addi sp, sp, 16
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomic_load_i64_unordered:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    mv a1, zero
+; RV64I-NEXT:    call __atomic_load_8
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomic_load_i64_unordered:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    ld a0, 0(a0)
+; RV64IA-NEXT:    ret
   %1 = load atomic i64, i64* %a unordered, align 8
   ret i64 %1
 }
@@ -285,6 +493,21 @@ define i64 @atomic_load_i64_monotonic(i64 *%a) nounwind {
 ; RV32IA-NEXT:    lw ra, 12(sp)
 ; RV32IA-NEXT:    addi sp, sp, 16
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomic_load_i64_monotonic:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    mv a1, zero
+; RV64I-NEXT:    call __atomic_load_8
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomic_load_i64_monotonic:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    ld a0, 0(a0)
+; RV64IA-NEXT:    ret
   %1 = load atomic i64, i64* %a monotonic, align 8
   ret i64 %1
 }
@@ -309,6 +532,22 @@ define i64 @atomic_load_i64_acquire(i64 *%a) nounwind {
 ; RV32IA-NEXT:    lw ra, 12(sp)
 ; RV32IA-NEXT:    addi sp, sp, 16
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomic_load_i64_acquire:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a1, zero, 2
+; RV64I-NEXT:    call __atomic_load_8
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomic_load_i64_acquire:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    ld a0, 0(a0)
+; RV64IA-NEXT:    fence r, rw
+; RV64IA-NEXT:    ret
   %1 = load atomic i64, i64* %a acquire, align 8
   ret i64 %1
 }
@@ -333,6 +572,23 @@ define i64 @atomic_load_i64_seq_cst(i64 *%a) nounwind {
 ; RV32IA-NEXT:    lw ra, 12(sp)
 ; RV32IA-NEXT:    addi sp, sp, 16
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomic_load_i64_seq_cst:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a1, zero, 5
+; RV64I-NEXT:    call __atomic_load_8
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomic_load_i64_seq_cst:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    fence rw, rw
+; RV64IA-NEXT:    ld a0, 0(a0)
+; RV64IA-NEXT:    fence r, rw
+; RV64IA-NEXT:    ret
   %1 = load atomic i64, i64* %a seq_cst, align 8
   ret i64 %1
 }
@@ -352,6 +608,21 @@ define void @atomic_store_i8_unordered(i8 *%a, i8 %b) nounwind {
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    sb a1, 0(a0)
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomic_store_i8_unordered:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    mv a2, zero
+; RV64I-NEXT:    call __atomic_store_1
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomic_store_i8_unordered:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    sb a1, 0(a0)
+; RV64IA-NEXT:    ret
   store atomic i8 %b, i8* %a unordered, align 1
   ret void
 }
@@ -371,6 +642,21 @@ define void @atomic_store_i8_monotonic(i8 *%a, i8 %b) nounwind {
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    sb a1, 0(a0)
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomic_store_i8_monotonic:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    mv a2, zero
+; RV64I-NEXT:    call __atomic_store_1
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomic_store_i8_monotonic:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    sb a1, 0(a0)
+; RV64IA-NEXT:    ret
   store atomic i8 %b, i8* %a monotonic, align 1
   ret void
 }
@@ -391,6 +677,22 @@ define void @atomic_store_i8_release(i8 *%a, i8 %b) nounwind {
 ; RV32IA-NEXT:    fence rw, w
 ; RV32IA-NEXT:    sb a1, 0(a0)
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomic_store_i8_release:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a2, zero, 3
+; RV64I-NEXT:    call __atomic_store_1
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomic_store_i8_release:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    fence rw, w
+; RV64IA-NEXT:    sb a1, 0(a0)
+; RV64IA-NEXT:    ret
   store atomic i8 %b, i8* %a release, align 1
   ret void
 }
@@ -411,6 +713,22 @@ define void @atomic_store_i8_seq_cst(i8 *%a, i8 %b) nounwind {
 ; RV32IA-NEXT:    fence rw, w
 ; RV32IA-NEXT:    sb a1, 0(a0)
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomic_store_i8_seq_cst:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a2, zero, 5
+; RV64I-NEXT:    call __atomic_store_1
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomic_store_i8_seq_cst:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    fence rw, w
+; RV64IA-NEXT:    sb a1, 0(a0)
+; RV64IA-NEXT:    ret
   store atomic i8 %b, i8* %a seq_cst, align 1
   ret void
 }
@@ -430,6 +748,21 @@ define void @atomic_store_i16_unordered(i16 *%a, i16 %b) nounwind {
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    sh a1, 0(a0)
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomic_store_i16_unordered:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    mv a2, zero
+; RV64I-NEXT:    call __atomic_store_2
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomic_store_i16_unordered:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    sh a1, 0(a0)
+; RV64IA-NEXT:    ret
   store atomic i16 %b, i16* %a unordered, align 2
   ret void
 }
@@ -449,6 +782,21 @@ define void @atomic_store_i16_monotonic(i16 *%a, i16 %b) nounwind {
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    sh a1, 0(a0)
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomic_store_i16_monotonic:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    mv a2, zero
+; RV64I-NEXT:    call __atomic_store_2
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomic_store_i16_monotonic:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    sh a1, 0(a0)
+; RV64IA-NEXT:    ret
   store atomic i16 %b, i16* %a monotonic, align 2
   ret void
 }
@@ -469,6 +817,22 @@ define void @atomic_store_i16_release(i16 *%a, i16 %b) nounwind {
 ; RV32IA-NEXT:    fence rw, w
 ; RV32IA-NEXT:    sh a1, 0(a0)
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomic_store_i16_release:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a2, zero, 3
+; RV64I-NEXT:    call __atomic_store_2
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomic_store_i16_release:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    fence rw, w
+; RV64IA-NEXT:    sh a1, 0(a0)
+; RV64IA-NEXT:    ret
   store atomic i16 %b, i16* %a release, align 2
   ret void
 }
@@ -489,6 +853,22 @@ define void @atomic_store_i16_seq_cst(i16 *%a, i16 %b) nounwind {
 ; RV32IA-NEXT:    fence rw, w
 ; RV32IA-NEXT:    sh a1, 0(a0)
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomic_store_i16_seq_cst:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a2, zero, 5
+; RV64I-NEXT:    call __atomic_store_2
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomic_store_i16_seq_cst:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    fence rw, w
+; RV64IA-NEXT:    sh a1, 0(a0)
+; RV64IA-NEXT:    ret
   store atomic i16 %b, i16* %a seq_cst, align 2
   ret void
 }
@@ -508,6 +888,21 @@ define void @atomic_store_i32_unordered(i32 *%a, i32 %b) nounwind {
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    sw a1, 0(a0)
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomic_store_i32_unordered:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    mv a2, zero
+; RV64I-NEXT:    call __atomic_store_4
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomic_store_i32_unordered:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    sw a1, 0(a0)
+; RV64IA-NEXT:    ret
   store atomic i32 %b, i32* %a unordered, align 4
   ret void
 }
@@ -527,6 +922,21 @@ define void @atomic_store_i32_monotonic(i32 *%a, i32 %b) nounwind {
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    sw a1, 0(a0)
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomic_store_i32_monotonic:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    mv a2, zero
+; RV64I-NEXT:    call __atomic_store_4
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomic_store_i32_monotonic:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    sw a1, 0(a0)
+; RV64IA-NEXT:    ret
   store atomic i32 %b, i32* %a monotonic, align 4
   ret void
 }
@@ -547,6 +957,22 @@ define void @atomic_store_i32_release(i32 *%a, i32 %b) nounwind {
 ; RV32IA-NEXT:    fence rw, w
 ; RV32IA-NEXT:    sw a1, 0(a0)
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomic_store_i32_release:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a2, zero, 3
+; RV64I-NEXT:    call __atomic_store_4
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomic_store_i32_release:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    fence rw, w
+; RV64IA-NEXT:    sw a1, 0(a0)
+; RV64IA-NEXT:    ret
   store atomic i32 %b, i32* %a release, align 4
   ret void
 }
@@ -567,6 +993,22 @@ define void @atomic_store_i32_seq_cst(i32 *%a, i32 %b) nounwind {
 ; RV32IA-NEXT:    fence rw, w
 ; RV32IA-NEXT:    sw a1, 0(a0)
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomic_store_i32_seq_cst:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a2, zero, 5
+; RV64I-NEXT:    call __atomic_store_4
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomic_store_i32_seq_cst:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    fence rw, w
+; RV64IA-NEXT:    sw a1, 0(a0)
+; RV64IA-NEXT:    ret
   store atomic i32 %b, i32* %a seq_cst, align 4
   ret void
 }
@@ -591,6 +1033,21 @@ define void @atomic_store_i64_unordered(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    lw ra, 12(sp)
 ; RV32IA-NEXT:    addi sp, sp, 16
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomic_store_i64_unordered:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    mv a2, zero
+; RV64I-NEXT:    call __atomic_store_8
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomic_store_i64_unordered:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    sd a1, 0(a0)
+; RV64IA-NEXT:    ret
   store atomic i64 %b, i64* %a unordered, align 8
   ret void
 }
@@ -615,6 +1072,21 @@ define void @atomic_store_i64_monotonic(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    lw ra, 12(sp)
 ; RV32IA-NEXT:    addi sp, sp, 16
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomic_store_i64_monotonic:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    mv a2, zero
+; RV64I-NEXT:    call __atomic_store_8
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomic_store_i64_monotonic:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    sd a1, 0(a0)
+; RV64IA-NEXT:    ret
   store atomic i64 %b, i64* %a monotonic, align 8
   ret void
 }
@@ -639,6 +1111,22 @@ define void @atomic_store_i64_release(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    lw ra, 12(sp)
 ; RV32IA-NEXT:    addi sp, sp, 16
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomic_store_i64_release:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a2, zero, 3
+; RV64I-NEXT:    call __atomic_store_8
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomic_store_i64_release:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    fence rw, w
+; RV64IA-NEXT:    sd a1, 0(a0)
+; RV64IA-NEXT:    ret
   store atomic i64 %b, i64* %a release, align 8
   ret void
 }
@@ -663,6 +1151,22 @@ define void @atomic_store_i64_seq_cst(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    lw ra, 12(sp)
 ; RV32IA-NEXT:    addi sp, sp, 16
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomic_store_i64_seq_cst:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a2, zero, 5
+; RV64I-NEXT:    call __atomic_store_8
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomic_store_i64_seq_cst:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    fence rw, w
+; RV64IA-NEXT:    sd a1, 0(a0)
+; RV64IA-NEXT:    ret
   store atomic i64 %b, i64* %a seq_cst, align 8
   ret void
 }
diff --git a/test/CodeGen/RISCV/atomic-rmw.ll b/test/CodeGen/RISCV/atomic-rmw.ll
index f27afbba7ccca8b9fe85ff6bb1c690101deeec75..9e98af6037c763e8104d4f2b6b7928f38ed8563c 100644
--- a/test/CodeGen/RISCV/atomic-rmw.ll
+++ b/test/CodeGen/RISCV/atomic-rmw.ll
@@ -3,6 +3,10 @@
 ; RUN:   | FileCheck -check-prefix=RV32I %s
 ; RUN: llc -mtriple=riscv32 -mattr=+a -verify-machineinstrs < %s \
 ; RUN:   | FileCheck -check-prefix=RV32IA %s
+; RUN: llc -mtriple=riscv64 -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=RV64I %s
+; RUN: llc -mtriple=riscv64 -mattr=+a -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=RV64IA %s
 
 define i8 @atomicrmw_xchg_i8_monotonic(i8* %a, i8 %b) {
 ; RV32I-LABEL: atomicrmw_xchg_i8_monotonic:
@@ -35,6 +39,37 @@ define i8 @atomicrmw_xchg_i8_monotonic(i8* %a, i8 %b) {
 ; RV32IA-NEXT:  # %bb.2:
 ; RV32IA-NEXT:    srl a0, a4, a2
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_xchg_i8_monotonic:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    mv a2, zero
+; RV64I-NEXT:    call __atomic_exchange_1
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_xchg_i8_monotonic:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    slli a2, a0, 3
+; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    addi a3, zero, 255
+; RV64IA-NEXT:    sllw a3, a3, a2
+; RV64IA-NEXT:    andi a1, a1, 255
+; RV64IA-NEXT:    sllw a1, a1, a2
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:  .LBB0_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.w a4, (a0)
+; RV64IA-NEXT:    add a5, zero, a1
+; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:    and a5, a5, a3
+; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:    sc.w a5, a5, (a0)
+; RV64IA-NEXT:    bnez a5, .LBB0_1
+; RV64IA-NEXT:  # %bb.2:
+; RV64IA-NEXT:    srlw a0, a4, a2
+; RV64IA-NEXT:    ret
   %1 = atomicrmw xchg i8* %a, i8 %b monotonic
   ret i8 %1
 }
@@ -70,6 +105,37 @@ define i8 @atomicrmw_xchg_i8_acquire(i8* %a, i8 %b) {
 ; RV32IA-NEXT:  # %bb.2:
 ; RV32IA-NEXT:    srl a0, a4, a2
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_xchg_i8_acquire:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a2, zero, 2
+; RV64I-NEXT:    call __atomic_exchange_1
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_xchg_i8_acquire:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    slli a2, a0, 3
+; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    addi a3, zero, 255
+; RV64IA-NEXT:    sllw a3, a3, a2
+; RV64IA-NEXT:    andi a1, a1, 255
+; RV64IA-NEXT:    sllw a1, a1, a2
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:  .LBB1_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.w.aq a4, (a0)
+; RV64IA-NEXT:    add a5, zero, a1
+; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:    and a5, a5, a3
+; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:    sc.w a5, a5, (a0)
+; RV64IA-NEXT:    bnez a5, .LBB1_1
+; RV64IA-NEXT:  # %bb.2:
+; RV64IA-NEXT:    srlw a0, a4, a2
+; RV64IA-NEXT:    ret
   %1 = atomicrmw xchg i8* %a, i8 %b acquire
   ret i8 %1
 }
@@ -105,6 +171,37 @@ define i8 @atomicrmw_xchg_i8_release(i8* %a, i8 %b) {
 ; RV32IA-NEXT:  # %bb.2:
 ; RV32IA-NEXT:    srl a0, a4, a2
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_xchg_i8_release:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a2, zero, 3
+; RV64I-NEXT:    call __atomic_exchange_1
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_xchg_i8_release:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    slli a2, a0, 3
+; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    addi a3, zero, 255
+; RV64IA-NEXT:    sllw a3, a3, a2
+; RV64IA-NEXT:    andi a1, a1, 255
+; RV64IA-NEXT:    sllw a1, a1, a2
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:  .LBB2_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.w a4, (a0)
+; RV64IA-NEXT:    add a5, zero, a1
+; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:    and a5, a5, a3
+; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV64IA-NEXT:    bnez a5, .LBB2_1
+; RV64IA-NEXT:  # %bb.2:
+; RV64IA-NEXT:    srlw a0, a4, a2
+; RV64IA-NEXT:    ret
   %1 = atomicrmw xchg i8* %a, i8 %b release
   ret i8 %1
 }
@@ -140,6 +237,37 @@ define i8 @atomicrmw_xchg_i8_acq_rel(i8* %a, i8 %b) {
 ; RV32IA-NEXT:  # %bb.2:
 ; RV32IA-NEXT:    srl a0, a4, a2
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_xchg_i8_acq_rel:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a2, zero, 4
+; RV64I-NEXT:    call __atomic_exchange_1
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_xchg_i8_acq_rel:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    slli a2, a0, 3
+; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    addi a3, zero, 255
+; RV64IA-NEXT:    sllw a3, a3, a2
+; RV64IA-NEXT:    andi a1, a1, 255
+; RV64IA-NEXT:    sllw a1, a1, a2
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:  .LBB3_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.w.aq a4, (a0)
+; RV64IA-NEXT:    add a5, zero, a1
+; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:    and a5, a5, a3
+; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV64IA-NEXT:    bnez a5, .LBB3_1
+; RV64IA-NEXT:  # %bb.2:
+; RV64IA-NEXT:    srlw a0, a4, a2
+; RV64IA-NEXT:    ret
   %1 = atomicrmw xchg i8* %a, i8 %b acq_rel
   ret i8 %1
 }
@@ -175,6 +303,37 @@ define i8 @atomicrmw_xchg_i8_seq_cst(i8* %a, i8 %b) {
 ; RV32IA-NEXT:  # %bb.2:
 ; RV32IA-NEXT:    srl a0, a4, a2
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_xchg_i8_seq_cst:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a2, zero, 5
+; RV64I-NEXT:    call __atomic_exchange_1
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_xchg_i8_seq_cst:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    slli a2, a0, 3
+; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    addi a3, zero, 255
+; RV64IA-NEXT:    sllw a3, a3, a2
+; RV64IA-NEXT:    andi a1, a1, 255
+; RV64IA-NEXT:    sllw a1, a1, a2
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:  .LBB4_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.w.aqrl a4, (a0)
+; RV64IA-NEXT:    add a5, zero, a1
+; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:    and a5, a5, a3
+; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:    sc.w.aqrl a5, a5, (a0)
+; RV64IA-NEXT:    bnez a5, .LBB4_1
+; RV64IA-NEXT:  # %bb.2:
+; RV64IA-NEXT:    srlw a0, a4, a2
+; RV64IA-NEXT:    ret
   %1 = atomicrmw xchg i8* %a, i8 %b seq_cst
   ret i8 %1
 }
@@ -210,6 +369,37 @@ define i8 @atomicrmw_add_i8_monotonic(i8 *%a, i8 %b) nounwind {
 ; RV32IA-NEXT:  # %bb.2:
 ; RV32IA-NEXT:    srl a0, a4, a2
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_add_i8_monotonic:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    mv a2, zero
+; RV64I-NEXT:    call __atomic_fetch_add_1
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_add_i8_monotonic:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    slli a2, a0, 3
+; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    addi a3, zero, 255
+; RV64IA-NEXT:    sllw a3, a3, a2
+; RV64IA-NEXT:    andi a1, a1, 255
+; RV64IA-NEXT:    sllw a1, a1, a2
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:  .LBB5_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.w a4, (a0)
+; RV64IA-NEXT:    add a5, a4, a1
+; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:    and a5, a5, a3
+; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:    sc.w a5, a5, (a0)
+; RV64IA-NEXT:    bnez a5, .LBB5_1
+; RV64IA-NEXT:  # %bb.2:
+; RV64IA-NEXT:    srlw a0, a4, a2
+; RV64IA-NEXT:    ret
   %1 = atomicrmw add i8* %a, i8 %b monotonic
   ret i8 %1
 }
@@ -245,6 +435,37 @@ define i8 @atomicrmw_add_i8_acquire(i8 *%a, i8 %b) nounwind {
 ; RV32IA-NEXT:  # %bb.2:
 ; RV32IA-NEXT:    srl a0, a4, a2
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_add_i8_acquire:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a2, zero, 2
+; RV64I-NEXT:    call __atomic_fetch_add_1
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_add_i8_acquire:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    slli a2, a0, 3
+; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    addi a3, zero, 255
+; RV64IA-NEXT:    sllw a3, a3, a2
+; RV64IA-NEXT:    andi a1, a1, 255
+; RV64IA-NEXT:    sllw a1, a1, a2
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:  .LBB6_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.w.aq a4, (a0)
+; RV64IA-NEXT:    add a5, a4, a1
+; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:    and a5, a5, a3
+; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:    sc.w a5, a5, (a0)
+; RV64IA-NEXT:    bnez a5, .LBB6_1
+; RV64IA-NEXT:  # %bb.2:
+; RV64IA-NEXT:    srlw a0, a4, a2
+; RV64IA-NEXT:    ret
   %1 = atomicrmw add i8* %a, i8 %b acquire
   ret i8 %1
 }
@@ -280,6 +501,37 @@ define i8 @atomicrmw_add_i8_release(i8 *%a, i8 %b) nounwind {
 ; RV32IA-NEXT:  # %bb.2:
 ; RV32IA-NEXT:    srl a0, a4, a2
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_add_i8_release:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a2, zero, 3
+; RV64I-NEXT:    call __atomic_fetch_add_1
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_add_i8_release:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    slli a2, a0, 3
+; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    addi a3, zero, 255
+; RV64IA-NEXT:    sllw a3, a3, a2
+; RV64IA-NEXT:    andi a1, a1, 255
+; RV64IA-NEXT:    sllw a1, a1, a2
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:  .LBB7_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.w a4, (a0)
+; RV64IA-NEXT:    add a5, a4, a1
+; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:    and a5, a5, a3
+; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV64IA-NEXT:    bnez a5, .LBB7_1
+; RV64IA-NEXT:  # %bb.2:
+; RV64IA-NEXT:    srlw a0, a4, a2
+; RV64IA-NEXT:    ret
   %1 = atomicrmw add i8* %a, i8 %b release
   ret i8 %1
 }
@@ -315,6 +567,37 @@ define i8 @atomicrmw_add_i8_acq_rel(i8 *%a, i8 %b) nounwind {
 ; RV32IA-NEXT:  # %bb.2:
 ; RV32IA-NEXT:    srl a0, a4, a2
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_add_i8_acq_rel:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a2, zero, 4
+; RV64I-NEXT:    call __atomic_fetch_add_1
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_add_i8_acq_rel:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    slli a2, a0, 3
+; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    addi a3, zero, 255
+; RV64IA-NEXT:    sllw a3, a3, a2
+; RV64IA-NEXT:    andi a1, a1, 255
+; RV64IA-NEXT:    sllw a1, a1, a2
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:  .LBB8_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.w.aq a4, (a0)
+; RV64IA-NEXT:    add a5, a4, a1
+; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:    and a5, a5, a3
+; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV64IA-NEXT:    bnez a5, .LBB8_1
+; RV64IA-NEXT:  # %bb.2:
+; RV64IA-NEXT:    srlw a0, a4, a2
+; RV64IA-NEXT:    ret
   %1 = atomicrmw add i8* %a, i8 %b acq_rel
   ret i8 %1
 }
@@ -350,6 +633,37 @@ define i8 @atomicrmw_add_i8_seq_cst(i8 *%a, i8 %b) nounwind {
 ; RV32IA-NEXT:  # %bb.2:
 ; RV32IA-NEXT:    srl a0, a4, a2
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_add_i8_seq_cst:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a2, zero, 5
+; RV64I-NEXT:    call __atomic_fetch_add_1
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_add_i8_seq_cst:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    slli a2, a0, 3
+; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    addi a3, zero, 255
+; RV64IA-NEXT:    sllw a3, a3, a2
+; RV64IA-NEXT:    andi a1, a1, 255
+; RV64IA-NEXT:    sllw a1, a1, a2
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:  .LBB9_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.w.aqrl a4, (a0)
+; RV64IA-NEXT:    add a5, a4, a1
+; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:    and a5, a5, a3
+; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:    sc.w.aqrl a5, a5, (a0)
+; RV64IA-NEXT:    bnez a5, .LBB9_1
+; RV64IA-NEXT:  # %bb.2:
+; RV64IA-NEXT:    srlw a0, a4, a2
+; RV64IA-NEXT:    ret
   %1 = atomicrmw add i8* %a, i8 %b seq_cst
   ret i8 %1
 }
@@ -385,6 +699,37 @@ define i8 @atomicrmw_sub_i8_monotonic(i8* %a, i8 %b) {
 ; RV32IA-NEXT:  # %bb.2:
 ; RV32IA-NEXT:    srl a0, a4, a2
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_sub_i8_monotonic:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    mv a2, zero
+; RV64I-NEXT:    call __atomic_fetch_sub_1
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_sub_i8_monotonic:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    slli a2, a0, 3
+; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    addi a3, zero, 255
+; RV64IA-NEXT:    sllw a3, a3, a2
+; RV64IA-NEXT:    andi a1, a1, 255
+; RV64IA-NEXT:    sllw a1, a1, a2
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:  .LBB10_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.w a4, (a0)
+; RV64IA-NEXT:    sub a5, a4, a1
+; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:    and a5, a5, a3
+; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:    sc.w a5, a5, (a0)
+; RV64IA-NEXT:    bnez a5, .LBB10_1
+; RV64IA-NEXT:  # %bb.2:
+; RV64IA-NEXT:    srlw a0, a4, a2
+; RV64IA-NEXT:    ret
   %1 = atomicrmw sub i8* %a, i8 %b monotonic
   ret i8 %1
 }
@@ -420,6 +765,37 @@ define i8 @atomicrmw_sub_i8_acquire(i8* %a, i8 %b) {
 ; RV32IA-NEXT:  # %bb.2:
 ; RV32IA-NEXT:    srl a0, a4, a2
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_sub_i8_acquire:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a2, zero, 2
+; RV64I-NEXT:    call __atomic_fetch_sub_1
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_sub_i8_acquire:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    slli a2, a0, 3
+; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    addi a3, zero, 255
+; RV64IA-NEXT:    sllw a3, a3, a2
+; RV64IA-NEXT:    andi a1, a1, 255
+; RV64IA-NEXT:    sllw a1, a1, a2
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:  .LBB11_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.w.aq a4, (a0)
+; RV64IA-NEXT:    sub a5, a4, a1
+; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:    and a5, a5, a3
+; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:    sc.w a5, a5, (a0)
+; RV64IA-NEXT:    bnez a5, .LBB11_1
+; RV64IA-NEXT:  # %bb.2:
+; RV64IA-NEXT:    srlw a0, a4, a2
+; RV64IA-NEXT:    ret
   %1 = atomicrmw sub i8* %a, i8 %b acquire
   ret i8 %1
 }
@@ -455,6 +831,37 @@ define i8 @atomicrmw_sub_i8_release(i8* %a, i8 %b) {
 ; RV32IA-NEXT:  # %bb.2:
 ; RV32IA-NEXT:    srl a0, a4, a2
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_sub_i8_release:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a2, zero, 3
+; RV64I-NEXT:    call __atomic_fetch_sub_1
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_sub_i8_release:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    slli a2, a0, 3
+; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    addi a3, zero, 255
+; RV64IA-NEXT:    sllw a3, a3, a2
+; RV64IA-NEXT:    andi a1, a1, 255
+; RV64IA-NEXT:    sllw a1, a1, a2
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:  .LBB12_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.w a4, (a0)
+; RV64IA-NEXT:    sub a5, a4, a1
+; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:    and a5, a5, a3
+; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV64IA-NEXT:    bnez a5, .LBB12_1
+; RV64IA-NEXT:  # %bb.2:
+; RV64IA-NEXT:    srlw a0, a4, a2
+; RV64IA-NEXT:    ret
   %1 = atomicrmw sub i8* %a, i8 %b release
   ret i8 %1
 }
@@ -490,6 +897,37 @@ define i8 @atomicrmw_sub_i8_acq_rel(i8* %a, i8 %b) {
 ; RV32IA-NEXT:  # %bb.2:
 ; RV32IA-NEXT:    srl a0, a4, a2
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_sub_i8_acq_rel:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a2, zero, 4
+; RV64I-NEXT:    call __atomic_fetch_sub_1
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_sub_i8_acq_rel:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    slli a2, a0, 3
+; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    addi a3, zero, 255
+; RV64IA-NEXT:    sllw a3, a3, a2
+; RV64IA-NEXT:    andi a1, a1, 255
+; RV64IA-NEXT:    sllw a1, a1, a2
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:  .LBB13_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.w.aq a4, (a0)
+; RV64IA-NEXT:    sub a5, a4, a1
+; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:    and a5, a5, a3
+; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV64IA-NEXT:    bnez a5, .LBB13_1
+; RV64IA-NEXT:  # %bb.2:
+; RV64IA-NEXT:    srlw a0, a4, a2
+; RV64IA-NEXT:    ret
   %1 = atomicrmw sub i8* %a, i8 %b acq_rel
   ret i8 %1
 }
@@ -525,6 +963,37 @@ define i8 @atomicrmw_sub_i8_seq_cst(i8* %a, i8 %b) {
 ; RV32IA-NEXT:  # %bb.2:
 ; RV32IA-NEXT:    srl a0, a4, a2
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_sub_i8_seq_cst:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a2, zero, 5
+; RV64I-NEXT:    call __atomic_fetch_sub_1
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_sub_i8_seq_cst:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    slli a2, a0, 3
+; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    addi a3, zero, 255
+; RV64IA-NEXT:    sllw a3, a3, a2
+; RV64IA-NEXT:    andi a1, a1, 255
+; RV64IA-NEXT:    sllw a1, a1, a2
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:  .LBB14_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.w.aqrl a4, (a0)
+; RV64IA-NEXT:    sub a5, a4, a1
+; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:    and a5, a5, a3
+; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:    sc.w.aqrl a5, a5, (a0)
+; RV64IA-NEXT:    bnez a5, .LBB14_1
+; RV64IA-NEXT:  # %bb.2:
+; RV64IA-NEXT:    srlw a0, a4, a2
+; RV64IA-NEXT:    ret
   %1 = atomicrmw sub i8* %a, i8 %b seq_cst
   ret i8 %1
 }
@@ -554,6 +1023,31 @@ define i8 @atomicrmw_and_i8_monotonic(i8 *%a, i8 %b) nounwind {
 ; RV32IA-NEXT:    amoand.w a0, a1, (a0)
 ; RV32IA-NEXT:    srl a0, a0, a2
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_and_i8_monotonic:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    mv a2, zero
+; RV64I-NEXT:    call __atomic_fetch_and_1
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_and_i8_monotonic:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    andi a1, a1, 255
+; RV64IA-NEXT:    slli a2, a0, 3
+; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    sllw a1, a1, a2
+; RV64IA-NEXT:    addi a3, zero, 255
+; RV64IA-NEXT:    sllw a3, a3, a2
+; RV64IA-NEXT:    not a3, a3
+; RV64IA-NEXT:    or a1, a3, a1
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    amoand.w a0, a1, (a0)
+; RV64IA-NEXT:    srlw a0, a0, a2
+; RV64IA-NEXT:    ret
   %1 = atomicrmw and i8* %a, i8 %b monotonic
   ret i8 %1
 }
@@ -583,6 +1077,31 @@ define i8 @atomicrmw_and_i8_acquire(i8 *%a, i8 %b) nounwind {
 ; RV32IA-NEXT:    amoand.w.aq a0, a1, (a0)
 ; RV32IA-NEXT:    srl a0, a0, a2
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_and_i8_acquire:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a2, zero, 2
+; RV64I-NEXT:    call __atomic_fetch_and_1
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_and_i8_acquire:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    andi a1, a1, 255
+; RV64IA-NEXT:    slli a2, a0, 3
+; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    sllw a1, a1, a2
+; RV64IA-NEXT:    addi a3, zero, 255
+; RV64IA-NEXT:    sllw a3, a3, a2
+; RV64IA-NEXT:    not a3, a3
+; RV64IA-NEXT:    or a1, a3, a1
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    amoand.w.aq a0, a1, (a0)
+; RV64IA-NEXT:    srlw a0, a0, a2
+; RV64IA-NEXT:    ret
   %1 = atomicrmw and i8* %a, i8 %b acquire
   ret i8 %1
 }
@@ -612,6 +1131,31 @@ define i8 @atomicrmw_and_i8_release(i8 *%a, i8 %b) nounwind {
 ; RV32IA-NEXT:    amoand.w.rl a0, a1, (a0)
 ; RV32IA-NEXT:    srl a0, a0, a2
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_and_i8_release:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a2, zero, 3
+; RV64I-NEXT:    call __atomic_fetch_and_1
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_and_i8_release:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    andi a1, a1, 255
+; RV64IA-NEXT:    slli a2, a0, 3
+; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    sllw a1, a1, a2
+; RV64IA-NEXT:    addi a3, zero, 255
+; RV64IA-NEXT:    sllw a3, a3, a2
+; RV64IA-NEXT:    not a3, a3
+; RV64IA-NEXT:    or a1, a3, a1
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    amoand.w.rl a0, a1, (a0)
+; RV64IA-NEXT:    srlw a0, a0, a2
+; RV64IA-NEXT:    ret
   %1 = atomicrmw and i8* %a, i8 %b release
   ret i8 %1
 }
@@ -641,6 +1185,31 @@ define i8 @atomicrmw_and_i8_acq_rel(i8 *%a, i8 %b) nounwind {
 ; RV32IA-NEXT:    amoand.w.aqrl a0, a1, (a0)
 ; RV32IA-NEXT:    srl a0, a0, a2
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_and_i8_acq_rel:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a2, zero, 4
+; RV64I-NEXT:    call __atomic_fetch_and_1
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_and_i8_acq_rel:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    andi a1, a1, 255
+; RV64IA-NEXT:    slli a2, a0, 3
+; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    sllw a1, a1, a2
+; RV64IA-NEXT:    addi a3, zero, 255
+; RV64IA-NEXT:    sllw a3, a3, a2
+; RV64IA-NEXT:    not a3, a3
+; RV64IA-NEXT:    or a1, a3, a1
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    amoand.w.aqrl a0, a1, (a0)
+; RV64IA-NEXT:    srlw a0, a0, a2
+; RV64IA-NEXT:    ret
   %1 = atomicrmw and i8* %a, i8 %b acq_rel
   ret i8 %1
 }
@@ -670,6 +1239,31 @@ define i8 @atomicrmw_and_i8_seq_cst(i8 *%a, i8 %b) nounwind {
 ; RV32IA-NEXT:    amoand.w.aqrl a0, a1, (a0)
 ; RV32IA-NEXT:    srl a0, a0, a2
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_and_i8_seq_cst:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a2, zero, 5
+; RV64I-NEXT:    call __atomic_fetch_and_1
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_and_i8_seq_cst:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    andi a1, a1, 255
+; RV64IA-NEXT:    slli a2, a0, 3
+; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    sllw a1, a1, a2
+; RV64IA-NEXT:    addi a3, zero, 255
+; RV64IA-NEXT:    sllw a3, a3, a2
+; RV64IA-NEXT:    not a3, a3
+; RV64IA-NEXT:    or a1, a3, a1
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    amoand.w.aqrl a0, a1, (a0)
+; RV64IA-NEXT:    srlw a0, a0, a2
+; RV64IA-NEXT:    ret
   %1 = atomicrmw and i8* %a, i8 %b seq_cst
   ret i8 %1
 }
@@ -706,6 +1300,38 @@ define i8 @atomicrmw_nand_i8_monotonic(i8* %a, i8 %b) {
 ; RV32IA-NEXT:  # %bb.2:
 ; RV32IA-NEXT:    srl a0, a4, a2
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_nand_i8_monotonic:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    mv a2, zero
+; RV64I-NEXT:    call __atomic_fetch_nand_1
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_nand_i8_monotonic:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    slli a2, a0, 3
+; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    addi a3, zero, 255
+; RV64IA-NEXT:    sllw a3, a3, a2
+; RV64IA-NEXT:    andi a1, a1, 255
+; RV64IA-NEXT:    sllw a1, a1, a2
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:  .LBB20_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.w a4, (a0)
+; RV64IA-NEXT:    and a5, a4, a1
+; RV64IA-NEXT:    not a5, a5
+; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:    and a5, a5, a3
+; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:    sc.w a5, a5, (a0)
+; RV64IA-NEXT:    bnez a5, .LBB20_1
+; RV64IA-NEXT:  # %bb.2:
+; RV64IA-NEXT:    srlw a0, a4, a2
+; RV64IA-NEXT:    ret
   %1 = atomicrmw nand i8* %a, i8 %b monotonic
   ret i8 %1
 }
@@ -742,6 +1368,38 @@ define i8 @atomicrmw_nand_i8_acquire(i8* %a, i8 %b) {
 ; RV32IA-NEXT:  # %bb.2:
 ; RV32IA-NEXT:    srl a0, a4, a2
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_nand_i8_acquire:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a2, zero, 2
+; RV64I-NEXT:    call __atomic_fetch_nand_1
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_nand_i8_acquire:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    slli a2, a0, 3
+; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    addi a3, zero, 255
+; RV64IA-NEXT:    sllw a3, a3, a2
+; RV64IA-NEXT:    andi a1, a1, 255
+; RV64IA-NEXT:    sllw a1, a1, a2
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:  .LBB21_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.w.aq a4, (a0)
+; RV64IA-NEXT:    and a5, a4, a1
+; RV64IA-NEXT:    not a5, a5
+; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:    and a5, a5, a3
+; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:    sc.w a5, a5, (a0)
+; RV64IA-NEXT:    bnez a5, .LBB21_1
+; RV64IA-NEXT:  # %bb.2:
+; RV64IA-NEXT:    srlw a0, a4, a2
+; RV64IA-NEXT:    ret
   %1 = atomicrmw nand i8* %a, i8 %b acquire
   ret i8 %1
 }
@@ -778,6 +1436,38 @@ define i8 @atomicrmw_nand_i8_release(i8* %a, i8 %b) {
 ; RV32IA-NEXT:  # %bb.2:
 ; RV32IA-NEXT:    srl a0, a4, a2
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_nand_i8_release:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a2, zero, 3
+; RV64I-NEXT:    call __atomic_fetch_nand_1
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_nand_i8_release:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    slli a2, a0, 3
+; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    addi a3, zero, 255
+; RV64IA-NEXT:    sllw a3, a3, a2
+; RV64IA-NEXT:    andi a1, a1, 255
+; RV64IA-NEXT:    sllw a1, a1, a2
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:  .LBB22_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.w a4, (a0)
+; RV64IA-NEXT:    and a5, a4, a1
+; RV64IA-NEXT:    not a5, a5
+; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:    and a5, a5, a3
+; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV64IA-NEXT:    bnez a5, .LBB22_1
+; RV64IA-NEXT:  # %bb.2:
+; RV64IA-NEXT:    srlw a0, a4, a2
+; RV64IA-NEXT:    ret
   %1 = atomicrmw nand i8* %a, i8 %b release
   ret i8 %1
 }
@@ -814,6 +1504,38 @@ define i8 @atomicrmw_nand_i8_acq_rel(i8* %a, i8 %b) {
 ; RV32IA-NEXT:  # %bb.2:
 ; RV32IA-NEXT:    srl a0, a4, a2
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_nand_i8_acq_rel:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a2, zero, 4
+; RV64I-NEXT:    call __atomic_fetch_nand_1
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_nand_i8_acq_rel:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    slli a2, a0, 3
+; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    addi a3, zero, 255
+; RV64IA-NEXT:    sllw a3, a3, a2
+; RV64IA-NEXT:    andi a1, a1, 255
+; RV64IA-NEXT:    sllw a1, a1, a2
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:  .LBB23_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.w.aq a4, (a0)
+; RV64IA-NEXT:    and a5, a4, a1
+; RV64IA-NEXT:    not a5, a5
+; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:    and a5, a5, a3
+; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV64IA-NEXT:    bnez a5, .LBB23_1
+; RV64IA-NEXT:  # %bb.2:
+; RV64IA-NEXT:    srlw a0, a4, a2
+; RV64IA-NEXT:    ret
   %1 = atomicrmw nand i8* %a, i8 %b acq_rel
   ret i8 %1
 }
@@ -850,6 +1572,38 @@ define i8 @atomicrmw_nand_i8_seq_cst(i8* %a, i8 %b) {
 ; RV32IA-NEXT:  # %bb.2:
 ; RV32IA-NEXT:    srl a0, a4, a2
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_nand_i8_seq_cst:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a2, zero, 5
+; RV64I-NEXT:    call __atomic_fetch_nand_1
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_nand_i8_seq_cst:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    slli a2, a0, 3
+; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    addi a3, zero, 255
+; RV64IA-NEXT:    sllw a3, a3, a2
+; RV64IA-NEXT:    andi a1, a1, 255
+; RV64IA-NEXT:    sllw a1, a1, a2
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:  .LBB24_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.w.aqrl a4, (a0)
+; RV64IA-NEXT:    and a5, a4, a1
+; RV64IA-NEXT:    not a5, a5
+; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:    and a5, a5, a3
+; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:    sc.w.aqrl a5, a5, (a0)
+; RV64IA-NEXT:    bnez a5, .LBB24_1
+; RV64IA-NEXT:  # %bb.2:
+; RV64IA-NEXT:    srlw a0, a4, a2
+; RV64IA-NEXT:    ret
   %1 = atomicrmw nand i8* %a, i8 %b seq_cst
   ret i8 %1
 }
@@ -875,6 +1629,27 @@ define i8 @atomicrmw_or_i8_monotonic(i8 *%a, i8 %b) nounwind {
 ; RV32IA-NEXT:    amoor.w a0, a1, (a0)
 ; RV32IA-NEXT:    srl a0, a0, a2
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_or_i8_monotonic:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    mv a2, zero
+; RV64I-NEXT:    call __atomic_fetch_or_1
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_or_i8_monotonic:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    andi a1, a1, 255
+; RV64IA-NEXT:    slli a2, a0, 3
+; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    sllw a1, a1, a2
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    amoor.w a0, a1, (a0)
+; RV64IA-NEXT:    srlw a0, a0, a2
+; RV64IA-NEXT:    ret
   %1 = atomicrmw or i8* %a, i8 %b monotonic
   ret i8 %1
 }
@@ -900,6 +1675,27 @@ define i8 @atomicrmw_or_i8_acquire(i8 *%a, i8 %b) nounwind {
 ; RV32IA-NEXT:    amoor.w.aq a0, a1, (a0)
 ; RV32IA-NEXT:    srl a0, a0, a2
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_or_i8_acquire:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a2, zero, 2
+; RV64I-NEXT:    call __atomic_fetch_or_1
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_or_i8_acquire:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    andi a1, a1, 255
+; RV64IA-NEXT:    slli a2, a0, 3
+; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    sllw a1, a1, a2
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    amoor.w.aq a0, a1, (a0)
+; RV64IA-NEXT:    srlw a0, a0, a2
+; RV64IA-NEXT:    ret
   %1 = atomicrmw or i8* %a, i8 %b acquire
   ret i8 %1
 }
@@ -925,6 +1721,27 @@ define i8 @atomicrmw_or_i8_release(i8 *%a, i8 %b) nounwind {
 ; RV32IA-NEXT:    amoor.w.rl a0, a1, (a0)
 ; RV32IA-NEXT:    srl a0, a0, a2
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_or_i8_release:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a2, zero, 3
+; RV64I-NEXT:    call __atomic_fetch_or_1
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_or_i8_release:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    andi a1, a1, 255
+; RV64IA-NEXT:    slli a2, a0, 3
+; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    sllw a1, a1, a2
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    amoor.w.rl a0, a1, (a0)
+; RV64IA-NEXT:    srlw a0, a0, a2
+; RV64IA-NEXT:    ret
   %1 = atomicrmw or i8* %a, i8 %b release
   ret i8 %1
 }
@@ -950,6 +1767,27 @@ define i8 @atomicrmw_or_i8_acq_rel(i8 *%a, i8 %b) nounwind {
 ; RV32IA-NEXT:    amoor.w.aqrl a0, a1, (a0)
 ; RV32IA-NEXT:    srl a0, a0, a2
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_or_i8_acq_rel:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a2, zero, 4
+; RV64I-NEXT:    call __atomic_fetch_or_1
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_or_i8_acq_rel:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    andi a1, a1, 255
+; RV64IA-NEXT:    slli a2, a0, 3
+; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    sllw a1, a1, a2
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    amoor.w.aqrl a0, a1, (a0)
+; RV64IA-NEXT:    srlw a0, a0, a2
+; RV64IA-NEXT:    ret
   %1 = atomicrmw or i8* %a, i8 %b acq_rel
   ret i8 %1
 }
@@ -975,6 +1813,27 @@ define i8 @atomicrmw_or_i8_seq_cst(i8 *%a, i8 %b) nounwind {
 ; RV32IA-NEXT:    amoor.w.aqrl a0, a1, (a0)
 ; RV32IA-NEXT:    srl a0, a0, a2
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_or_i8_seq_cst:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a2, zero, 5
+; RV64I-NEXT:    call __atomic_fetch_or_1
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_or_i8_seq_cst:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    andi a1, a1, 255
+; RV64IA-NEXT:    slli a2, a0, 3
+; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    sllw a1, a1, a2
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    amoor.w.aqrl a0, a1, (a0)
+; RV64IA-NEXT:    srlw a0, a0, a2
+; RV64IA-NEXT:    ret
   %1 = atomicrmw or i8* %a, i8 %b seq_cst
   ret i8 %1
 }
@@ -1000,6 +1859,27 @@ define i8 @atomicrmw_xor_i8_monotonic(i8 *%a, i8 %b) nounwind {
 ; RV32IA-NEXT:    amoxor.w a0, a1, (a0)
 ; RV32IA-NEXT:    srl a0, a0, a2
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_xor_i8_monotonic:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    mv a2, zero
+; RV64I-NEXT:    call __atomic_fetch_xor_1
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_xor_i8_monotonic:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    andi a1, a1, 255
+; RV64IA-NEXT:    slli a2, a0, 3
+; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    sllw a1, a1, a2
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    amoxor.w a0, a1, (a0)
+; RV64IA-NEXT:    srlw a0, a0, a2
+; RV64IA-NEXT:    ret
   %1 = atomicrmw xor i8* %a, i8 %b monotonic
   ret i8 %1
 }
@@ -1025,6 +1905,27 @@ define i8 @atomicrmw_xor_i8_acquire(i8 *%a, i8 %b) nounwind {
 ; RV32IA-NEXT:    amoxor.w.aq a0, a1, (a0)
 ; RV32IA-NEXT:    srl a0, a0, a2
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_xor_i8_acquire:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a2, zero, 2
+; RV64I-NEXT:    call __atomic_fetch_xor_1
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_xor_i8_acquire:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    andi a1, a1, 255
+; RV64IA-NEXT:    slli a2, a0, 3
+; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    sllw a1, a1, a2
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    amoxor.w.aq a0, a1, (a0)
+; RV64IA-NEXT:    srlw a0, a0, a2
+; RV64IA-NEXT:    ret
   %1 = atomicrmw xor i8* %a, i8 %b acquire
   ret i8 %1
 }
@@ -1050,6 +1951,27 @@ define i8 @atomicrmw_xor_i8_release(i8 *%a, i8 %b) nounwind {
 ; RV32IA-NEXT:    amoxor.w.rl a0, a1, (a0)
 ; RV32IA-NEXT:    srl a0, a0, a2
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_xor_i8_release:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a2, zero, 3
+; RV64I-NEXT:    call __atomic_fetch_xor_1
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_xor_i8_release:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    andi a1, a1, 255
+; RV64IA-NEXT:    slli a2, a0, 3
+; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    sllw a1, a1, a2
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    amoxor.w.rl a0, a1, (a0)
+; RV64IA-NEXT:    srlw a0, a0, a2
+; RV64IA-NEXT:    ret
   %1 = atomicrmw xor i8* %a, i8 %b release
   ret i8 %1
 }
@@ -1075,6 +1997,27 @@ define i8 @atomicrmw_xor_i8_acq_rel(i8 *%a, i8 %b) nounwind {
 ; RV32IA-NEXT:    amoxor.w.aqrl a0, a1, (a0)
 ; RV32IA-NEXT:    srl a0, a0, a2
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_xor_i8_acq_rel:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a2, zero, 4
+; RV64I-NEXT:    call __atomic_fetch_xor_1
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_xor_i8_acq_rel:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    andi a1, a1, 255
+; RV64IA-NEXT:    slli a2, a0, 3
+; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    sllw a1, a1, a2
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    amoxor.w.aqrl a0, a1, (a0)
+; RV64IA-NEXT:    srlw a0, a0, a2
+; RV64IA-NEXT:    ret
   %1 = atomicrmw xor i8* %a, i8 %b acq_rel
   ret i8 %1
 }
@@ -1100,6 +2043,27 @@ define i8 @atomicrmw_xor_i8_seq_cst(i8 *%a, i8 %b) nounwind {
 ; RV32IA-NEXT:    amoxor.w.aqrl a0, a1, (a0)
 ; RV32IA-NEXT:    srl a0, a0, a2
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_xor_i8_seq_cst:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a2, zero, 5
+; RV64I-NEXT:    call __atomic_fetch_xor_1
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_xor_i8_seq_cst:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    andi a1, a1, 255
+; RV64IA-NEXT:    slli a2, a0, 3
+; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    sllw a1, a1, a2
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    amoxor.w.aqrl a0, a1, (a0)
+; RV64IA-NEXT:    srlw a0, a0, a2
+; RV64IA-NEXT:    ret
   %1 = atomicrmw xor i8* %a, i8 %b seq_cst
   ret i8 %1
 }
@@ -1177,6 +2141,79 @@ define i8 @atomicrmw_max_i8_monotonic(i8 *%a, i8 %b) nounwind {
 ; RV32IA-NEXT:  # %bb.4:
 ; RV32IA-NEXT:    srl a0, a5, a2
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_max_i8_monotonic:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -48
+; RV64I-NEXT:    sd ra, 40(sp)
+; RV64I-NEXT:    sd s1, 32(sp)
+; RV64I-NEXT:    sd s2, 24(sp)
+; RV64I-NEXT:    sd s3, 16(sp)
+; RV64I-NEXT:    sd s4, 8(sp)
+; RV64I-NEXT:    mv s2, a1
+; RV64I-NEXT:    mv s4, a0
+; RV64I-NEXT:    lbu a0, 0(a0)
+; RV64I-NEXT:    slli a1, a1, 56
+; RV64I-NEXT:    srai s1, a1, 56
+; RV64I-NEXT:    addi s3, sp, 7
+; RV64I-NEXT:  .LBB35_1: # %atomicrmw.start
+; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV64I-NEXT:    slli a1, a0, 56
+; RV64I-NEXT:    srai a1, a1, 56
+; RV64I-NEXT:    mv a2, a0
+; RV64I-NEXT:    blt s1, a1, .LBB35_3
+; RV64I-NEXT:  # %bb.2: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB35_1 Depth=1
+; RV64I-NEXT:    mv a2, s2
+; RV64I-NEXT:  .LBB35_3: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB35_1 Depth=1
+; RV64I-NEXT:    sb a0, 7(sp)
+; RV64I-NEXT:    mv a0, s4
+; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    mv a3, zero
+; RV64I-NEXT:    mv a4, zero
+; RV64I-NEXT:    call __atomic_compare_exchange_1
+; RV64I-NEXT:    mv a1, a0
+; RV64I-NEXT:    lb a0, 7(sp)
+; RV64I-NEXT:    beqz a1, .LBB35_1
+; RV64I-NEXT:  # %bb.4: # %atomicrmw.end
+; RV64I-NEXT:    ld s4, 8(sp)
+; RV64I-NEXT:    ld s3, 16(sp)
+; RV64I-NEXT:    ld s2, 24(sp)
+; RV64I-NEXT:    ld s1, 32(sp)
+; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    addi sp, sp, 48
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_max_i8_monotonic:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    slli a2, a0, 3
+; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    addi a3, zero, 56
+; RV64IA-NEXT:    sub a6, a3, a2
+; RV64IA-NEXT:    addi a4, zero, 255
+; RV64IA-NEXT:    sllw a7, a4, a2
+; RV64IA-NEXT:    slli a1, a1, 56
+; RV64IA-NEXT:    srai a1, a1, 56
+; RV64IA-NEXT:    sllw a1, a1, a2
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:  .LBB35_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.w a5, (a0)
+; RV64IA-NEXT:    and a4, a5, a7
+; RV64IA-NEXT:    mv a3, a5
+; RV64IA-NEXT:    sll a4, a4, a6
+; RV64IA-NEXT:    sra a4, a4, a6
+; RV64IA-NEXT:    bge a4, a1, .LBB35_3
+; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB35_1 Depth=1
+; RV64IA-NEXT:    xor a3, a5, a1
+; RV64IA-NEXT:    and a3, a3, a7
+; RV64IA-NEXT:    xor a3, a5, a3
+; RV64IA-NEXT:  .LBB35_3: # in Loop: Header=BB35_1 Depth=1
+; RV64IA-NEXT:    sc.w a3, a3, (a0)
+; RV64IA-NEXT:    bnez a3, .LBB35_1
+; RV64IA-NEXT:  # %bb.4:
+; RV64IA-NEXT:    srlw a0, a5, a2
+; RV64IA-NEXT:    ret
   %1 = atomicrmw max i8* %a, i8 %b monotonic
   ret i8 %1
 }
@@ -1190,36 +2227,33 @@ define i8 @atomicrmw_max_i8_acquire(i8 *%a, i8 %b) nounwind {
 ; RV32I-NEXT:    sw s2, 20(sp)
 ; RV32I-NEXT:    sw s3, 16(sp)
 ; RV32I-NEXT:    sw s4, 12(sp)
-; RV32I-NEXT:    sw s5, 8(sp)
 ; RV32I-NEXT:    mv s2, a1
 ; RV32I-NEXT:    mv s4, a0
 ; RV32I-NEXT:    lbu a0, 0(a0)
 ; RV32I-NEXT:    slli a1, a1, 24
-; RV32I-NEXT:    srai s5, a1, 24
-; RV32I-NEXT:    addi s3, sp, 7
-; RV32I-NEXT:    addi s1, zero, 2
+; RV32I-NEXT:    srai s1, a1, 24
+; RV32I-NEXT:    addi s3, sp, 11
 ; RV32I-NEXT:  .LBB36_1: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
 ; RV32I-NEXT:    slli a1, a0, 24
 ; RV32I-NEXT:    srai a1, a1, 24
 ; RV32I-NEXT:    mv a2, a0
-; RV32I-NEXT:    blt s5, a1, .LBB36_3
+; RV32I-NEXT:    blt s1, a1, .LBB36_3
 ; RV32I-NEXT:  # %bb.2: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB36_1 Depth=1
 ; RV32I-NEXT:    mv a2, s2
 ; RV32I-NEXT:  .LBB36_3: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB36_1 Depth=1
-; RV32I-NEXT:    sb a0, 7(sp)
+; RV32I-NEXT:    sb a0, 11(sp)
 ; RV32I-NEXT:    mv a0, s4
 ; RV32I-NEXT:    mv a1, s3
-; RV32I-NEXT:    mv a3, s1
-; RV32I-NEXT:    mv a4, s1
+; RV32I-NEXT:    addi a3, zero, 2
+; RV32I-NEXT:    addi a4, zero, 2
 ; RV32I-NEXT:    call __atomic_compare_exchange_1
 ; RV32I-NEXT:    mv a1, a0
-; RV32I-NEXT:    lb a0, 7(sp)
+; RV32I-NEXT:    lb a0, 11(sp)
 ; RV32I-NEXT:    beqz a1, .LBB36_1
 ; RV32I-NEXT:  # %bb.4: # %atomicrmw.end
-; RV32I-NEXT:    lw s5, 8(sp)
 ; RV32I-NEXT:    lw s4, 12(sp)
 ; RV32I-NEXT:    lw s3, 16(sp)
 ; RV32I-NEXT:    lw s2, 20(sp)
@@ -1257,6 +2291,79 @@ define i8 @atomicrmw_max_i8_acquire(i8 *%a, i8 %b) nounwind {
 ; RV32IA-NEXT:  # %bb.4:
 ; RV32IA-NEXT:    srl a0, a5, a2
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_max_i8_acquire:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -48
+; RV64I-NEXT:    sd ra, 40(sp)
+; RV64I-NEXT:    sd s1, 32(sp)
+; RV64I-NEXT:    sd s2, 24(sp)
+; RV64I-NEXT:    sd s3, 16(sp)
+; RV64I-NEXT:    sd s4, 8(sp)
+; RV64I-NEXT:    mv s2, a1
+; RV64I-NEXT:    mv s4, a0
+; RV64I-NEXT:    lbu a0, 0(a0)
+; RV64I-NEXT:    slli a1, a1, 56
+; RV64I-NEXT:    srai s1, a1, 56
+; RV64I-NEXT:    addi s3, sp, 7
+; RV64I-NEXT:  .LBB36_1: # %atomicrmw.start
+; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV64I-NEXT:    slli a1, a0, 56
+; RV64I-NEXT:    srai a1, a1, 56
+; RV64I-NEXT:    mv a2, a0
+; RV64I-NEXT:    blt s1, a1, .LBB36_3
+; RV64I-NEXT:  # %bb.2: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB36_1 Depth=1
+; RV64I-NEXT:    mv a2, s2
+; RV64I-NEXT:  .LBB36_3: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB36_1 Depth=1
+; RV64I-NEXT:    sb a0, 7(sp)
+; RV64I-NEXT:    mv a0, s4
+; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    addi a3, zero, 2
+; RV64I-NEXT:    addi a4, zero, 2
+; RV64I-NEXT:    call __atomic_compare_exchange_1
+; RV64I-NEXT:    mv a1, a0
+; RV64I-NEXT:    lb a0, 7(sp)
+; RV64I-NEXT:    beqz a1, .LBB36_1
+; RV64I-NEXT:  # %bb.4: # %atomicrmw.end
+; RV64I-NEXT:    ld s4, 8(sp)
+; RV64I-NEXT:    ld s3, 16(sp)
+; RV64I-NEXT:    ld s2, 24(sp)
+; RV64I-NEXT:    ld s1, 32(sp)
+; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    addi sp, sp, 48
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_max_i8_acquire:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    slli a2, a0, 3
+; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    addi a3, zero, 56
+; RV64IA-NEXT:    sub a6, a3, a2
+; RV64IA-NEXT:    addi a4, zero, 255
+; RV64IA-NEXT:    sllw a7, a4, a2
+; RV64IA-NEXT:    slli a1, a1, 56
+; RV64IA-NEXT:    srai a1, a1, 56
+; RV64IA-NEXT:    sllw a1, a1, a2
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:  .LBB36_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.w.aq a5, (a0)
+; RV64IA-NEXT:    and a4, a5, a7
+; RV64IA-NEXT:    mv a3, a5
+; RV64IA-NEXT:    sll a4, a4, a6
+; RV64IA-NEXT:    sra a4, a4, a6
+; RV64IA-NEXT:    bge a4, a1, .LBB36_3
+; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB36_1 Depth=1
+; RV64IA-NEXT:    xor a3, a5, a1
+; RV64IA-NEXT:    and a3, a3, a7
+; RV64IA-NEXT:    xor a3, a5, a3
+; RV64IA-NEXT:  .LBB36_3: # in Loop: Header=BB36_1 Depth=1
+; RV64IA-NEXT:    sc.w a3, a3, (a0)
+; RV64IA-NEXT:    bnez a3, .LBB36_1
+; RV64IA-NEXT:  # %bb.4:
+; RV64IA-NEXT:    srlw a0, a5, a2
+; RV64IA-NEXT:    ret
   %1 = atomicrmw max i8* %a, i8 %b acquire
   ret i8 %1
 }
@@ -1270,14 +2377,12 @@ define i8 @atomicrmw_max_i8_release(i8 *%a, i8 %b) nounwind {
 ; RV32I-NEXT:    sw s2, 20(sp)
 ; RV32I-NEXT:    sw s3, 16(sp)
 ; RV32I-NEXT:    sw s4, 12(sp)
-; RV32I-NEXT:    sw s5, 8(sp)
 ; RV32I-NEXT:    mv s2, a1
-; RV32I-NEXT:    mv s5, a0
+; RV32I-NEXT:    mv s4, a0
 ; RV32I-NEXT:    lbu a0, 0(a0)
 ; RV32I-NEXT:    slli a1, a1, 24
 ; RV32I-NEXT:    srai s1, a1, 24
-; RV32I-NEXT:    addi s3, sp, 7
-; RV32I-NEXT:    addi s4, zero, 3
+; RV32I-NEXT:    addi s3, sp, 11
 ; RV32I-NEXT:  .LBB37_1: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
 ; RV32I-NEXT:    slli a1, a0, 24
@@ -1289,17 +2394,16 @@ define i8 @atomicrmw_max_i8_release(i8 *%a, i8 %b) nounwind {
 ; RV32I-NEXT:    mv a2, s2
 ; RV32I-NEXT:  .LBB37_3: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB37_1 Depth=1
-; RV32I-NEXT:    sb a0, 7(sp)
-; RV32I-NEXT:    mv a0, s5
+; RV32I-NEXT:    sb a0, 11(sp)
+; RV32I-NEXT:    mv a0, s4
 ; RV32I-NEXT:    mv a1, s3
-; RV32I-NEXT:    mv a3, s4
+; RV32I-NEXT:    addi a3, zero, 3
 ; RV32I-NEXT:    mv a4, zero
 ; RV32I-NEXT:    call __atomic_compare_exchange_1
 ; RV32I-NEXT:    mv a1, a0
-; RV32I-NEXT:    lb a0, 7(sp)
+; RV32I-NEXT:    lb a0, 11(sp)
 ; RV32I-NEXT:    beqz a1, .LBB37_1
 ; RV32I-NEXT:  # %bb.4: # %atomicrmw.end
-; RV32I-NEXT:    lw s5, 8(sp)
 ; RV32I-NEXT:    lw s4, 12(sp)
 ; RV32I-NEXT:    lw s3, 16(sp)
 ; RV32I-NEXT:    lw s2, 20(sp)
@@ -1337,6 +2441,79 @@ define i8 @atomicrmw_max_i8_release(i8 *%a, i8 %b) nounwind {
 ; RV32IA-NEXT:  # %bb.4:
 ; RV32IA-NEXT:    srl a0, a5, a2
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_max_i8_release:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -48
+; RV64I-NEXT:    sd ra, 40(sp)
+; RV64I-NEXT:    sd s1, 32(sp)
+; RV64I-NEXT:    sd s2, 24(sp)
+; RV64I-NEXT:    sd s3, 16(sp)
+; RV64I-NEXT:    sd s4, 8(sp)
+; RV64I-NEXT:    mv s2, a1
+; RV64I-NEXT:    mv s4, a0
+; RV64I-NEXT:    lbu a0, 0(a0)
+; RV64I-NEXT:    slli a1, a1, 56
+; RV64I-NEXT:    srai s1, a1, 56
+; RV64I-NEXT:    addi s3, sp, 7
+; RV64I-NEXT:  .LBB37_1: # %atomicrmw.start
+; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV64I-NEXT:    slli a1, a0, 56
+; RV64I-NEXT:    srai a1, a1, 56
+; RV64I-NEXT:    mv a2, a0
+; RV64I-NEXT:    blt s1, a1, .LBB37_3
+; RV64I-NEXT:  # %bb.2: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB37_1 Depth=1
+; RV64I-NEXT:    mv a2, s2
+; RV64I-NEXT:  .LBB37_3: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB37_1 Depth=1
+; RV64I-NEXT:    sb a0, 7(sp)
+; RV64I-NEXT:    mv a0, s4
+; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    addi a3, zero, 3
+; RV64I-NEXT:    mv a4, zero
+; RV64I-NEXT:    call __atomic_compare_exchange_1
+; RV64I-NEXT:    mv a1, a0
+; RV64I-NEXT:    lb a0, 7(sp)
+; RV64I-NEXT:    beqz a1, .LBB37_1
+; RV64I-NEXT:  # %bb.4: # %atomicrmw.end
+; RV64I-NEXT:    ld s4, 8(sp)
+; RV64I-NEXT:    ld s3, 16(sp)
+; RV64I-NEXT:    ld s2, 24(sp)
+; RV64I-NEXT:    ld s1, 32(sp)
+; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    addi sp, sp, 48
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_max_i8_release:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    slli a2, a0, 3
+; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    addi a3, zero, 56
+; RV64IA-NEXT:    sub a6, a3, a2
+; RV64IA-NEXT:    addi a4, zero, 255
+; RV64IA-NEXT:    sllw a7, a4, a2
+; RV64IA-NEXT:    slli a1, a1, 56
+; RV64IA-NEXT:    srai a1, a1, 56
+; RV64IA-NEXT:    sllw a1, a1, a2
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:  .LBB37_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.w a5, (a0)
+; RV64IA-NEXT:    and a4, a5, a7
+; RV64IA-NEXT:    mv a3, a5
+; RV64IA-NEXT:    sll a4, a4, a6
+; RV64IA-NEXT:    sra a4, a4, a6
+; RV64IA-NEXT:    bge a4, a1, .LBB37_3
+; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB37_1 Depth=1
+; RV64IA-NEXT:    xor a3, a5, a1
+; RV64IA-NEXT:    and a3, a3, a7
+; RV64IA-NEXT:    xor a3, a5, a3
+; RV64IA-NEXT:  .LBB37_3: # in Loop: Header=BB37_1 Depth=1
+; RV64IA-NEXT:    sc.w.rl a3, a3, (a0)
+; RV64IA-NEXT:    bnez a3, .LBB37_1
+; RV64IA-NEXT:  # %bb.4:
+; RV64IA-NEXT:    srlw a0, a5, a2
+; RV64IA-NEXT:    ret
   %1 = atomicrmw max i8* %a, i8 %b release
   ret i8 %1
 }
@@ -1350,16 +2527,12 @@ define i8 @atomicrmw_max_i8_acq_rel(i8 *%a, i8 %b) nounwind {
 ; RV32I-NEXT:    sw s2, 20(sp)
 ; RV32I-NEXT:    sw s3, 16(sp)
 ; RV32I-NEXT:    sw s4, 12(sp)
-; RV32I-NEXT:    sw s5, 8(sp)
-; RV32I-NEXT:    sw s6, 4(sp)
 ; RV32I-NEXT:    mv s2, a1
-; RV32I-NEXT:    mv s6, a0
+; RV32I-NEXT:    mv s4, a0
 ; RV32I-NEXT:    lbu a0, 0(a0)
 ; RV32I-NEXT:    slli a1, a1, 24
 ; RV32I-NEXT:    srai s1, a1, 24
-; RV32I-NEXT:    addi s3, sp, 3
-; RV32I-NEXT:    addi s4, zero, 4
-; RV32I-NEXT:    addi s5, zero, 2
+; RV32I-NEXT:    addi s3, sp, 11
 ; RV32I-NEXT:  .LBB38_1: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
 ; RV32I-NEXT:    slli a1, a0, 24
@@ -1371,18 +2544,16 @@ define i8 @atomicrmw_max_i8_acq_rel(i8 *%a, i8 %b) nounwind {
 ; RV32I-NEXT:    mv a2, s2
 ; RV32I-NEXT:  .LBB38_3: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB38_1 Depth=1
-; RV32I-NEXT:    sb a0, 3(sp)
-; RV32I-NEXT:    mv a0, s6
+; RV32I-NEXT:    sb a0, 11(sp)
+; RV32I-NEXT:    mv a0, s4
 ; RV32I-NEXT:    mv a1, s3
-; RV32I-NEXT:    mv a3, s4
-; RV32I-NEXT:    mv a4, s5
+; RV32I-NEXT:    addi a3, zero, 4
+; RV32I-NEXT:    addi a4, zero, 2
 ; RV32I-NEXT:    call __atomic_compare_exchange_1
 ; RV32I-NEXT:    mv a1, a0
-; RV32I-NEXT:    lb a0, 3(sp)
+; RV32I-NEXT:    lb a0, 11(sp)
 ; RV32I-NEXT:    beqz a1, .LBB38_1
 ; RV32I-NEXT:  # %bb.4: # %atomicrmw.end
-; RV32I-NEXT:    lw s6, 4(sp)
-; RV32I-NEXT:    lw s5, 8(sp)
 ; RV32I-NEXT:    lw s4, 12(sp)
 ; RV32I-NEXT:    lw s3, 16(sp)
 ; RV32I-NEXT:    lw s2, 20(sp)
@@ -1420,6 +2591,79 @@ define i8 @atomicrmw_max_i8_acq_rel(i8 *%a, i8 %b) nounwind {
 ; RV32IA-NEXT:  # %bb.4:
 ; RV32IA-NEXT:    srl a0, a5, a2
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_max_i8_acq_rel:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -48
+; RV64I-NEXT:    sd ra, 40(sp)
+; RV64I-NEXT:    sd s1, 32(sp)
+; RV64I-NEXT:    sd s2, 24(sp)
+; RV64I-NEXT:    sd s3, 16(sp)
+; RV64I-NEXT:    sd s4, 8(sp)
+; RV64I-NEXT:    mv s2, a1
+; RV64I-NEXT:    mv s4, a0
+; RV64I-NEXT:    lbu a0, 0(a0)
+; RV64I-NEXT:    slli a1, a1, 56
+; RV64I-NEXT:    srai s1, a1, 56
+; RV64I-NEXT:    addi s3, sp, 7
+; RV64I-NEXT:  .LBB38_1: # %atomicrmw.start
+; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV64I-NEXT:    slli a1, a0, 56
+; RV64I-NEXT:    srai a1, a1, 56
+; RV64I-NEXT:    mv a2, a0
+; RV64I-NEXT:    blt s1, a1, .LBB38_3
+; RV64I-NEXT:  # %bb.2: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB38_1 Depth=1
+; RV64I-NEXT:    mv a2, s2
+; RV64I-NEXT:  .LBB38_3: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB38_1 Depth=1
+; RV64I-NEXT:    sb a0, 7(sp)
+; RV64I-NEXT:    mv a0, s4
+; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    addi a3, zero, 4
+; RV64I-NEXT:    addi a4, zero, 2
+; RV64I-NEXT:    call __atomic_compare_exchange_1
+; RV64I-NEXT:    mv a1, a0
+; RV64I-NEXT:    lb a0, 7(sp)
+; RV64I-NEXT:    beqz a1, .LBB38_1
+; RV64I-NEXT:  # %bb.4: # %atomicrmw.end
+; RV64I-NEXT:    ld s4, 8(sp)
+; RV64I-NEXT:    ld s3, 16(sp)
+; RV64I-NEXT:    ld s2, 24(sp)
+; RV64I-NEXT:    ld s1, 32(sp)
+; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    addi sp, sp, 48
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_max_i8_acq_rel:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    slli a2, a0, 3
+; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    addi a3, zero, 56
+; RV64IA-NEXT:    sub a6, a3, a2
+; RV64IA-NEXT:    addi a4, zero, 255
+; RV64IA-NEXT:    sllw a7, a4, a2
+; RV64IA-NEXT:    slli a1, a1, 56
+; RV64IA-NEXT:    srai a1, a1, 56
+; RV64IA-NEXT:    sllw a1, a1, a2
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:  .LBB38_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.w.aq a5, (a0)
+; RV64IA-NEXT:    and a4, a5, a7
+; RV64IA-NEXT:    mv a3, a5
+; RV64IA-NEXT:    sll a4, a4, a6
+; RV64IA-NEXT:    sra a4, a4, a6
+; RV64IA-NEXT:    bge a4, a1, .LBB38_3
+; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB38_1 Depth=1
+; RV64IA-NEXT:    xor a3, a5, a1
+; RV64IA-NEXT:    and a3, a3, a7
+; RV64IA-NEXT:    xor a3, a5, a3
+; RV64IA-NEXT:  .LBB38_3: # in Loop: Header=BB38_1 Depth=1
+; RV64IA-NEXT:    sc.w.rl a3, a3, (a0)
+; RV64IA-NEXT:    bnez a3, .LBB38_1
+; RV64IA-NEXT:  # %bb.4:
+; RV64IA-NEXT:    srlw a0, a5, a2
+; RV64IA-NEXT:    ret
   %1 = atomicrmw max i8* %a, i8 %b acq_rel
   ret i8 %1
 }
@@ -1433,36 +2677,33 @@ define i8 @atomicrmw_max_i8_seq_cst(i8 *%a, i8 %b) nounwind {
 ; RV32I-NEXT:    sw s2, 20(sp)
 ; RV32I-NEXT:    sw s3, 16(sp)
 ; RV32I-NEXT:    sw s4, 12(sp)
-; RV32I-NEXT:    sw s5, 8(sp)
 ; RV32I-NEXT:    mv s2, a1
 ; RV32I-NEXT:    mv s4, a0
 ; RV32I-NEXT:    lbu a0, 0(a0)
 ; RV32I-NEXT:    slli a1, a1, 24
-; RV32I-NEXT:    srai s5, a1, 24
-; RV32I-NEXT:    addi s3, sp, 7
-; RV32I-NEXT:    addi s1, zero, 5
+; RV32I-NEXT:    srai s1, a1, 24
+; RV32I-NEXT:    addi s3, sp, 11
 ; RV32I-NEXT:  .LBB39_1: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
 ; RV32I-NEXT:    slli a1, a0, 24
 ; RV32I-NEXT:    srai a1, a1, 24
 ; RV32I-NEXT:    mv a2, a0
-; RV32I-NEXT:    blt s5, a1, .LBB39_3
+; RV32I-NEXT:    blt s1, a1, .LBB39_3
 ; RV32I-NEXT:  # %bb.2: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB39_1 Depth=1
 ; RV32I-NEXT:    mv a2, s2
 ; RV32I-NEXT:  .LBB39_3: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB39_1 Depth=1
-; RV32I-NEXT:    sb a0, 7(sp)
+; RV32I-NEXT:    sb a0, 11(sp)
 ; RV32I-NEXT:    mv a0, s4
 ; RV32I-NEXT:    mv a1, s3
-; RV32I-NEXT:    mv a3, s1
-; RV32I-NEXT:    mv a4, s1
+; RV32I-NEXT:    addi a3, zero, 5
+; RV32I-NEXT:    addi a4, zero, 5
 ; RV32I-NEXT:    call __atomic_compare_exchange_1
 ; RV32I-NEXT:    mv a1, a0
-; RV32I-NEXT:    lb a0, 7(sp)
+; RV32I-NEXT:    lb a0, 11(sp)
 ; RV32I-NEXT:    beqz a1, .LBB39_1
 ; RV32I-NEXT:  # %bb.4: # %atomicrmw.end
-; RV32I-NEXT:    lw s5, 8(sp)
 ; RV32I-NEXT:    lw s4, 12(sp)
 ; RV32I-NEXT:    lw s3, 16(sp)
 ; RV32I-NEXT:    lw s2, 20(sp)
@@ -1500,6 +2741,79 @@ define i8 @atomicrmw_max_i8_seq_cst(i8 *%a, i8 %b) nounwind {
 ; RV32IA-NEXT:  # %bb.4:
 ; RV32IA-NEXT:    srl a0, a5, a2
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_max_i8_seq_cst:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -48
+; RV64I-NEXT:    sd ra, 40(sp)
+; RV64I-NEXT:    sd s1, 32(sp)
+; RV64I-NEXT:    sd s2, 24(sp)
+; RV64I-NEXT:    sd s3, 16(sp)
+; RV64I-NEXT:    sd s4, 8(sp)
+; RV64I-NEXT:    mv s2, a1
+; RV64I-NEXT:    mv s4, a0
+; RV64I-NEXT:    lbu a0, 0(a0)
+; RV64I-NEXT:    slli a1, a1, 56
+; RV64I-NEXT:    srai s1, a1, 56
+; RV64I-NEXT:    addi s3, sp, 7
+; RV64I-NEXT:  .LBB39_1: # %atomicrmw.start
+; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV64I-NEXT:    slli a1, a0, 56
+; RV64I-NEXT:    srai a1, a1, 56
+; RV64I-NEXT:    mv a2, a0
+; RV64I-NEXT:    blt s1, a1, .LBB39_3
+; RV64I-NEXT:  # %bb.2: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB39_1 Depth=1
+; RV64I-NEXT:    mv a2, s2
+; RV64I-NEXT:  .LBB39_3: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB39_1 Depth=1
+; RV64I-NEXT:    sb a0, 7(sp)
+; RV64I-NEXT:    mv a0, s4
+; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    addi a3, zero, 5
+; RV64I-NEXT:    addi a4, zero, 5
+; RV64I-NEXT:    call __atomic_compare_exchange_1
+; RV64I-NEXT:    mv a1, a0
+; RV64I-NEXT:    lb a0, 7(sp)
+; RV64I-NEXT:    beqz a1, .LBB39_1
+; RV64I-NEXT:  # %bb.4: # %atomicrmw.end
+; RV64I-NEXT:    ld s4, 8(sp)
+; RV64I-NEXT:    ld s3, 16(sp)
+; RV64I-NEXT:    ld s2, 24(sp)
+; RV64I-NEXT:    ld s1, 32(sp)
+; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    addi sp, sp, 48
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_max_i8_seq_cst:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    slli a2, a0, 3
+; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    addi a3, zero, 56
+; RV64IA-NEXT:    sub a6, a3, a2
+; RV64IA-NEXT:    addi a4, zero, 255
+; RV64IA-NEXT:    sllw a7, a4, a2
+; RV64IA-NEXT:    slli a1, a1, 56
+; RV64IA-NEXT:    srai a1, a1, 56
+; RV64IA-NEXT:    sllw a1, a1, a2
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:  .LBB39_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.w.aqrl a5, (a0)
+; RV64IA-NEXT:    and a4, a5, a7
+; RV64IA-NEXT:    mv a3, a5
+; RV64IA-NEXT:    sll a4, a4, a6
+; RV64IA-NEXT:    sra a4, a4, a6
+; RV64IA-NEXT:    bge a4, a1, .LBB39_3
+; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB39_1 Depth=1
+; RV64IA-NEXT:    xor a3, a5, a1
+; RV64IA-NEXT:    and a3, a3, a7
+; RV64IA-NEXT:    xor a3, a5, a3
+; RV64IA-NEXT:  .LBB39_3: # in Loop: Header=BB39_1 Depth=1
+; RV64IA-NEXT:    sc.w.aqrl a3, a3, (a0)
+; RV64IA-NEXT:    bnez a3, .LBB39_1
+; RV64IA-NEXT:  # %bb.4:
+; RV64IA-NEXT:    srlw a0, a5, a2
+; RV64IA-NEXT:    ret
   %1 = atomicrmw max i8* %a, i8 %b seq_cst
   ret i8 %1
 }
@@ -1577,6 +2891,79 @@ define i8 @atomicrmw_min_i8_monotonic(i8 *%a, i8 %b) nounwind {
 ; RV32IA-NEXT:  # %bb.4:
 ; RV32IA-NEXT:    srl a0, a5, a2
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_min_i8_monotonic:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -48
+; RV64I-NEXT:    sd ra, 40(sp)
+; RV64I-NEXT:    sd s1, 32(sp)
+; RV64I-NEXT:    sd s2, 24(sp)
+; RV64I-NEXT:    sd s3, 16(sp)
+; RV64I-NEXT:    sd s4, 8(sp)
+; RV64I-NEXT:    mv s2, a1
+; RV64I-NEXT:    mv s4, a0
+; RV64I-NEXT:    lbu a0, 0(a0)
+; RV64I-NEXT:    slli a1, a1, 56
+; RV64I-NEXT:    srai s1, a1, 56
+; RV64I-NEXT:    addi s3, sp, 7
+; RV64I-NEXT:  .LBB40_1: # %atomicrmw.start
+; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV64I-NEXT:    slli a1, a0, 56
+; RV64I-NEXT:    srai a1, a1, 56
+; RV64I-NEXT:    mv a2, a0
+; RV64I-NEXT:    bge s1, a1, .LBB40_3
+; RV64I-NEXT:  # %bb.2: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB40_1 Depth=1
+; RV64I-NEXT:    mv a2, s2
+; RV64I-NEXT:  .LBB40_3: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB40_1 Depth=1
+; RV64I-NEXT:    sb a0, 7(sp)
+; RV64I-NEXT:    mv a0, s4
+; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    mv a3, zero
+; RV64I-NEXT:    mv a4, zero
+; RV64I-NEXT:    call __atomic_compare_exchange_1
+; RV64I-NEXT:    mv a1, a0
+; RV64I-NEXT:    lb a0, 7(sp)
+; RV64I-NEXT:    beqz a1, .LBB40_1
+; RV64I-NEXT:  # %bb.4: # %atomicrmw.end
+; RV64I-NEXT:    ld s4, 8(sp)
+; RV64I-NEXT:    ld s3, 16(sp)
+; RV64I-NEXT:    ld s2, 24(sp)
+; RV64I-NEXT:    ld s1, 32(sp)
+; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    addi sp, sp, 48
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_min_i8_monotonic:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    slli a2, a0, 3
+; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    addi a3, zero, 56
+; RV64IA-NEXT:    sub a6, a3, a2
+; RV64IA-NEXT:    addi a4, zero, 255
+; RV64IA-NEXT:    sllw a7, a4, a2
+; RV64IA-NEXT:    slli a1, a1, 56
+; RV64IA-NEXT:    srai a1, a1, 56
+; RV64IA-NEXT:    sllw a1, a1, a2
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:  .LBB40_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.w a5, (a0)
+; RV64IA-NEXT:    and a4, a5, a7
+; RV64IA-NEXT:    mv a3, a5
+; RV64IA-NEXT:    sll a4, a4, a6
+; RV64IA-NEXT:    sra a4, a4, a6
+; RV64IA-NEXT:    bge a1, a4, .LBB40_3
+; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB40_1 Depth=1
+; RV64IA-NEXT:    xor a3, a5, a1
+; RV64IA-NEXT:    and a3, a3, a7
+; RV64IA-NEXT:    xor a3, a5, a3
+; RV64IA-NEXT:  .LBB40_3: # in Loop: Header=BB40_1 Depth=1
+; RV64IA-NEXT:    sc.w a3, a3, (a0)
+; RV64IA-NEXT:    bnez a3, .LBB40_1
+; RV64IA-NEXT:  # %bb.4:
+; RV64IA-NEXT:    srlw a0, a5, a2
+; RV64IA-NEXT:    ret
   %1 = atomicrmw min i8* %a, i8 %b monotonic
   ret i8 %1
 }
@@ -1590,36 +2977,33 @@ define i8 @atomicrmw_min_i8_acquire(i8 *%a, i8 %b) nounwind {
 ; RV32I-NEXT:    sw s2, 20(sp)
 ; RV32I-NEXT:    sw s3, 16(sp)
 ; RV32I-NEXT:    sw s4, 12(sp)
-; RV32I-NEXT:    sw s5, 8(sp)
 ; RV32I-NEXT:    mv s2, a1
 ; RV32I-NEXT:    mv s4, a0
 ; RV32I-NEXT:    lbu a0, 0(a0)
 ; RV32I-NEXT:    slli a1, a1, 24
-; RV32I-NEXT:    srai s5, a1, 24
-; RV32I-NEXT:    addi s3, sp, 7
-; RV32I-NEXT:    addi s1, zero, 2
+; RV32I-NEXT:    srai s1, a1, 24
+; RV32I-NEXT:    addi s3, sp, 11
 ; RV32I-NEXT:  .LBB41_1: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
 ; RV32I-NEXT:    slli a1, a0, 24
 ; RV32I-NEXT:    srai a1, a1, 24
 ; RV32I-NEXT:    mv a2, a0
-; RV32I-NEXT:    bge s5, a1, .LBB41_3
+; RV32I-NEXT:    bge s1, a1, .LBB41_3
 ; RV32I-NEXT:  # %bb.2: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB41_1 Depth=1
 ; RV32I-NEXT:    mv a2, s2
 ; RV32I-NEXT:  .LBB41_3: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB41_1 Depth=1
-; RV32I-NEXT:    sb a0, 7(sp)
+; RV32I-NEXT:    sb a0, 11(sp)
 ; RV32I-NEXT:    mv a0, s4
 ; RV32I-NEXT:    mv a1, s3
-; RV32I-NEXT:    mv a3, s1
-; RV32I-NEXT:    mv a4, s1
+; RV32I-NEXT:    addi a3, zero, 2
+; RV32I-NEXT:    addi a4, zero, 2
 ; RV32I-NEXT:    call __atomic_compare_exchange_1
 ; RV32I-NEXT:    mv a1, a0
-; RV32I-NEXT:    lb a0, 7(sp)
+; RV32I-NEXT:    lb a0, 11(sp)
 ; RV32I-NEXT:    beqz a1, .LBB41_1
 ; RV32I-NEXT:  # %bb.4: # %atomicrmw.end
-; RV32I-NEXT:    lw s5, 8(sp)
 ; RV32I-NEXT:    lw s4, 12(sp)
 ; RV32I-NEXT:    lw s3, 16(sp)
 ; RV32I-NEXT:    lw s2, 20(sp)
@@ -1657,6 +3041,79 @@ define i8 @atomicrmw_min_i8_acquire(i8 *%a, i8 %b) nounwind {
 ; RV32IA-NEXT:  # %bb.4:
 ; RV32IA-NEXT:    srl a0, a5, a2
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_min_i8_acquire:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -48
+; RV64I-NEXT:    sd ra, 40(sp)
+; RV64I-NEXT:    sd s1, 32(sp)
+; RV64I-NEXT:    sd s2, 24(sp)
+; RV64I-NEXT:    sd s3, 16(sp)
+; RV64I-NEXT:    sd s4, 8(sp)
+; RV64I-NEXT:    mv s2, a1
+; RV64I-NEXT:    mv s4, a0
+; RV64I-NEXT:    lbu a0, 0(a0)
+; RV64I-NEXT:    slli a1, a1, 56
+; RV64I-NEXT:    srai s1, a1, 56
+; RV64I-NEXT:    addi s3, sp, 7
+; RV64I-NEXT:  .LBB41_1: # %atomicrmw.start
+; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV64I-NEXT:    slli a1, a0, 56
+; RV64I-NEXT:    srai a1, a1, 56
+; RV64I-NEXT:    mv a2, a0
+; RV64I-NEXT:    bge s1, a1, .LBB41_3
+; RV64I-NEXT:  # %bb.2: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB41_1 Depth=1
+; RV64I-NEXT:    mv a2, s2
+; RV64I-NEXT:  .LBB41_3: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB41_1 Depth=1
+; RV64I-NEXT:    sb a0, 7(sp)
+; RV64I-NEXT:    mv a0, s4
+; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    addi a3, zero, 2
+; RV64I-NEXT:    addi a4, zero, 2
+; RV64I-NEXT:    call __atomic_compare_exchange_1
+; RV64I-NEXT:    mv a1, a0
+; RV64I-NEXT:    lb a0, 7(sp)
+; RV64I-NEXT:    beqz a1, .LBB41_1
+; RV64I-NEXT:  # %bb.4: # %atomicrmw.end
+; RV64I-NEXT:    ld s4, 8(sp)
+; RV64I-NEXT:    ld s3, 16(sp)
+; RV64I-NEXT:    ld s2, 24(sp)
+; RV64I-NEXT:    ld s1, 32(sp)
+; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    addi sp, sp, 48
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_min_i8_acquire:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    slli a2, a0, 3
+; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    addi a3, zero, 56
+; RV64IA-NEXT:    sub a6, a3, a2
+; RV64IA-NEXT:    addi a4, zero, 255
+; RV64IA-NEXT:    sllw a7, a4, a2
+; RV64IA-NEXT:    slli a1, a1, 56
+; RV64IA-NEXT:    srai a1, a1, 56
+; RV64IA-NEXT:    sllw a1, a1, a2
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:  .LBB41_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.w.aq a5, (a0)
+; RV64IA-NEXT:    and a4, a5, a7
+; RV64IA-NEXT:    mv a3, a5
+; RV64IA-NEXT:    sll a4, a4, a6
+; RV64IA-NEXT:    sra a4, a4, a6
+; RV64IA-NEXT:    bge a1, a4, .LBB41_3
+; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB41_1 Depth=1
+; RV64IA-NEXT:    xor a3, a5, a1
+; RV64IA-NEXT:    and a3, a3, a7
+; RV64IA-NEXT:    xor a3, a5, a3
+; RV64IA-NEXT:  .LBB41_3: # in Loop: Header=BB41_1 Depth=1
+; RV64IA-NEXT:    sc.w a3, a3, (a0)
+; RV64IA-NEXT:    bnez a3, .LBB41_1
+; RV64IA-NEXT:  # %bb.4:
+; RV64IA-NEXT:    srlw a0, a5, a2
+; RV64IA-NEXT:    ret
   %1 = atomicrmw min i8* %a, i8 %b acquire
   ret i8 %1
 }
@@ -1670,14 +3127,12 @@ define i8 @atomicrmw_min_i8_release(i8 *%a, i8 %b) nounwind {
 ; RV32I-NEXT:    sw s2, 20(sp)
 ; RV32I-NEXT:    sw s3, 16(sp)
 ; RV32I-NEXT:    sw s4, 12(sp)
-; RV32I-NEXT:    sw s5, 8(sp)
 ; RV32I-NEXT:    mv s2, a1
-; RV32I-NEXT:    mv s5, a0
+; RV32I-NEXT:    mv s4, a0
 ; RV32I-NEXT:    lbu a0, 0(a0)
 ; RV32I-NEXT:    slli a1, a1, 24
 ; RV32I-NEXT:    srai s1, a1, 24
-; RV32I-NEXT:    addi s3, sp, 7
-; RV32I-NEXT:    addi s4, zero, 3
+; RV32I-NEXT:    addi s3, sp, 11
 ; RV32I-NEXT:  .LBB42_1: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
 ; RV32I-NEXT:    slli a1, a0, 24
@@ -1689,17 +3144,16 @@ define i8 @atomicrmw_min_i8_release(i8 *%a, i8 %b) nounwind {
 ; RV32I-NEXT:    mv a2, s2
 ; RV32I-NEXT:  .LBB42_3: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB42_1 Depth=1
-; RV32I-NEXT:    sb a0, 7(sp)
-; RV32I-NEXT:    mv a0, s5
+; RV32I-NEXT:    sb a0, 11(sp)
+; RV32I-NEXT:    mv a0, s4
 ; RV32I-NEXT:    mv a1, s3
-; RV32I-NEXT:    mv a3, s4
+; RV32I-NEXT:    addi a3, zero, 3
 ; RV32I-NEXT:    mv a4, zero
 ; RV32I-NEXT:    call __atomic_compare_exchange_1
 ; RV32I-NEXT:    mv a1, a0
-; RV32I-NEXT:    lb a0, 7(sp)
+; RV32I-NEXT:    lb a0, 11(sp)
 ; RV32I-NEXT:    beqz a1, .LBB42_1
 ; RV32I-NEXT:  # %bb.4: # %atomicrmw.end
-; RV32I-NEXT:    lw s5, 8(sp)
 ; RV32I-NEXT:    lw s4, 12(sp)
 ; RV32I-NEXT:    lw s3, 16(sp)
 ; RV32I-NEXT:    lw s2, 20(sp)
@@ -1737,6 +3191,79 @@ define i8 @atomicrmw_min_i8_release(i8 *%a, i8 %b) nounwind {
 ; RV32IA-NEXT:  # %bb.4:
 ; RV32IA-NEXT:    srl a0, a5, a2
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_min_i8_release:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -48
+; RV64I-NEXT:    sd ra, 40(sp)
+; RV64I-NEXT:    sd s1, 32(sp)
+; RV64I-NEXT:    sd s2, 24(sp)
+; RV64I-NEXT:    sd s3, 16(sp)
+; RV64I-NEXT:    sd s4, 8(sp)
+; RV64I-NEXT:    mv s2, a1
+; RV64I-NEXT:    mv s4, a0
+; RV64I-NEXT:    lbu a0, 0(a0)
+; RV64I-NEXT:    slli a1, a1, 56
+; RV64I-NEXT:    srai s1, a1, 56
+; RV64I-NEXT:    addi s3, sp, 7
+; RV64I-NEXT:  .LBB42_1: # %atomicrmw.start
+; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV64I-NEXT:    slli a1, a0, 56
+; RV64I-NEXT:    srai a1, a1, 56
+; RV64I-NEXT:    mv a2, a0
+; RV64I-NEXT:    bge s1, a1, .LBB42_3
+; RV64I-NEXT:  # %bb.2: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB42_1 Depth=1
+; RV64I-NEXT:    mv a2, s2
+; RV64I-NEXT:  .LBB42_3: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB42_1 Depth=1
+; RV64I-NEXT:    sb a0, 7(sp)
+; RV64I-NEXT:    mv a0, s4
+; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    addi a3, zero, 3
+; RV64I-NEXT:    mv a4, zero
+; RV64I-NEXT:    call __atomic_compare_exchange_1
+; RV64I-NEXT:    mv a1, a0
+; RV64I-NEXT:    lb a0, 7(sp)
+; RV64I-NEXT:    beqz a1, .LBB42_1
+; RV64I-NEXT:  # %bb.4: # %atomicrmw.end
+; RV64I-NEXT:    ld s4, 8(sp)
+; RV64I-NEXT:    ld s3, 16(sp)
+; RV64I-NEXT:    ld s2, 24(sp)
+; RV64I-NEXT:    ld s1, 32(sp)
+; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    addi sp, sp, 48
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_min_i8_release:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    slli a2, a0, 3
+; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    addi a3, zero, 56
+; RV64IA-NEXT:    sub a6, a3, a2
+; RV64IA-NEXT:    addi a4, zero, 255
+; RV64IA-NEXT:    sllw a7, a4, a2
+; RV64IA-NEXT:    slli a1, a1, 56
+; RV64IA-NEXT:    srai a1, a1, 56
+; RV64IA-NEXT:    sllw a1, a1, a2
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:  .LBB42_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.w a5, (a0)
+; RV64IA-NEXT:    and a4, a5, a7
+; RV64IA-NEXT:    mv a3, a5
+; RV64IA-NEXT:    sll a4, a4, a6
+; RV64IA-NEXT:    sra a4, a4, a6
+; RV64IA-NEXT:    bge a1, a4, .LBB42_3
+; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB42_1 Depth=1
+; RV64IA-NEXT:    xor a3, a5, a1
+; RV64IA-NEXT:    and a3, a3, a7
+; RV64IA-NEXT:    xor a3, a5, a3
+; RV64IA-NEXT:  .LBB42_3: # in Loop: Header=BB42_1 Depth=1
+; RV64IA-NEXT:    sc.w.rl a3, a3, (a0)
+; RV64IA-NEXT:    bnez a3, .LBB42_1
+; RV64IA-NEXT:  # %bb.4:
+; RV64IA-NEXT:    srlw a0, a5, a2
+; RV64IA-NEXT:    ret
   %1 = atomicrmw min i8* %a, i8 %b release
   ret i8 %1
 }
@@ -1750,16 +3277,12 @@ define i8 @atomicrmw_min_i8_acq_rel(i8 *%a, i8 %b) nounwind {
 ; RV32I-NEXT:    sw s2, 20(sp)
 ; RV32I-NEXT:    sw s3, 16(sp)
 ; RV32I-NEXT:    sw s4, 12(sp)
-; RV32I-NEXT:    sw s5, 8(sp)
-; RV32I-NEXT:    sw s6, 4(sp)
 ; RV32I-NEXT:    mv s2, a1
-; RV32I-NEXT:    mv s6, a0
+; RV32I-NEXT:    mv s4, a0
 ; RV32I-NEXT:    lbu a0, 0(a0)
 ; RV32I-NEXT:    slli a1, a1, 24
 ; RV32I-NEXT:    srai s1, a1, 24
-; RV32I-NEXT:    addi s3, sp, 3
-; RV32I-NEXT:    addi s4, zero, 4
-; RV32I-NEXT:    addi s5, zero, 2
+; RV32I-NEXT:    addi s3, sp, 11
 ; RV32I-NEXT:  .LBB43_1: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
 ; RV32I-NEXT:    slli a1, a0, 24
@@ -1771,18 +3294,16 @@ define i8 @atomicrmw_min_i8_acq_rel(i8 *%a, i8 %b) nounwind {
 ; RV32I-NEXT:    mv a2, s2
 ; RV32I-NEXT:  .LBB43_3: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB43_1 Depth=1
-; RV32I-NEXT:    sb a0, 3(sp)
-; RV32I-NEXT:    mv a0, s6
+; RV32I-NEXT:    sb a0, 11(sp)
+; RV32I-NEXT:    mv a0, s4
 ; RV32I-NEXT:    mv a1, s3
-; RV32I-NEXT:    mv a3, s4
-; RV32I-NEXT:    mv a4, s5
+; RV32I-NEXT:    addi a3, zero, 4
+; RV32I-NEXT:    addi a4, zero, 2
 ; RV32I-NEXT:    call __atomic_compare_exchange_1
 ; RV32I-NEXT:    mv a1, a0
-; RV32I-NEXT:    lb a0, 3(sp)
+; RV32I-NEXT:    lb a0, 11(sp)
 ; RV32I-NEXT:    beqz a1, .LBB43_1
 ; RV32I-NEXT:  # %bb.4: # %atomicrmw.end
-; RV32I-NEXT:    lw s6, 4(sp)
-; RV32I-NEXT:    lw s5, 8(sp)
 ; RV32I-NEXT:    lw s4, 12(sp)
 ; RV32I-NEXT:    lw s3, 16(sp)
 ; RV32I-NEXT:    lw s2, 20(sp)
@@ -1820,6 +3341,79 @@ define i8 @atomicrmw_min_i8_acq_rel(i8 *%a, i8 %b) nounwind {
 ; RV32IA-NEXT:  # %bb.4:
 ; RV32IA-NEXT:    srl a0, a5, a2
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_min_i8_acq_rel:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -48
+; RV64I-NEXT:    sd ra, 40(sp)
+; RV64I-NEXT:    sd s1, 32(sp)
+; RV64I-NEXT:    sd s2, 24(sp)
+; RV64I-NEXT:    sd s3, 16(sp)
+; RV64I-NEXT:    sd s4, 8(sp)
+; RV64I-NEXT:    mv s2, a1
+; RV64I-NEXT:    mv s4, a0
+; RV64I-NEXT:    lbu a0, 0(a0)
+; RV64I-NEXT:    slli a1, a1, 56
+; RV64I-NEXT:    srai s1, a1, 56
+; RV64I-NEXT:    addi s3, sp, 7
+; RV64I-NEXT:  .LBB43_1: # %atomicrmw.start
+; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV64I-NEXT:    slli a1, a0, 56
+; RV64I-NEXT:    srai a1, a1, 56
+; RV64I-NEXT:    mv a2, a0
+; RV64I-NEXT:    bge s1, a1, .LBB43_3
+; RV64I-NEXT:  # %bb.2: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB43_1 Depth=1
+; RV64I-NEXT:    mv a2, s2
+; RV64I-NEXT:  .LBB43_3: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB43_1 Depth=1
+; RV64I-NEXT:    sb a0, 7(sp)
+; RV64I-NEXT:    mv a0, s4
+; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    addi a3, zero, 4
+; RV64I-NEXT:    addi a4, zero, 2
+; RV64I-NEXT:    call __atomic_compare_exchange_1
+; RV64I-NEXT:    mv a1, a0
+; RV64I-NEXT:    lb a0, 7(sp)
+; RV64I-NEXT:    beqz a1, .LBB43_1
+; RV64I-NEXT:  # %bb.4: # %atomicrmw.end
+; RV64I-NEXT:    ld s4, 8(sp)
+; RV64I-NEXT:    ld s3, 16(sp)
+; RV64I-NEXT:    ld s2, 24(sp)
+; RV64I-NEXT:    ld s1, 32(sp)
+; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    addi sp, sp, 48
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_min_i8_acq_rel:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    slli a2, a0, 3
+; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    addi a3, zero, 56
+; RV64IA-NEXT:    sub a6, a3, a2
+; RV64IA-NEXT:    addi a4, zero, 255
+; RV64IA-NEXT:    sllw a7, a4, a2
+; RV64IA-NEXT:    slli a1, a1, 56
+; RV64IA-NEXT:    srai a1, a1, 56
+; RV64IA-NEXT:    sllw a1, a1, a2
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:  .LBB43_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.w.aq a5, (a0)
+; RV64IA-NEXT:    and a4, a5, a7
+; RV64IA-NEXT:    mv a3, a5
+; RV64IA-NEXT:    sll a4, a4, a6
+; RV64IA-NEXT:    sra a4, a4, a6
+; RV64IA-NEXT:    bge a1, a4, .LBB43_3
+; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB43_1 Depth=1
+; RV64IA-NEXT:    xor a3, a5, a1
+; RV64IA-NEXT:    and a3, a3, a7
+; RV64IA-NEXT:    xor a3, a5, a3
+; RV64IA-NEXT:  .LBB43_3: # in Loop: Header=BB43_1 Depth=1
+; RV64IA-NEXT:    sc.w.rl a3, a3, (a0)
+; RV64IA-NEXT:    bnez a3, .LBB43_1
+; RV64IA-NEXT:  # %bb.4:
+; RV64IA-NEXT:    srlw a0, a5, a2
+; RV64IA-NEXT:    ret
   %1 = atomicrmw min i8* %a, i8 %b acq_rel
   ret i8 %1
 }
@@ -1833,36 +3427,33 @@ define i8 @atomicrmw_min_i8_seq_cst(i8 *%a, i8 %b) nounwind {
 ; RV32I-NEXT:    sw s2, 20(sp)
 ; RV32I-NEXT:    sw s3, 16(sp)
 ; RV32I-NEXT:    sw s4, 12(sp)
-; RV32I-NEXT:    sw s5, 8(sp)
 ; RV32I-NEXT:    mv s2, a1
 ; RV32I-NEXT:    mv s4, a0
 ; RV32I-NEXT:    lbu a0, 0(a0)
 ; RV32I-NEXT:    slli a1, a1, 24
-; RV32I-NEXT:    srai s5, a1, 24
-; RV32I-NEXT:    addi s3, sp, 7
-; RV32I-NEXT:    addi s1, zero, 5
+; RV32I-NEXT:    srai s1, a1, 24
+; RV32I-NEXT:    addi s3, sp, 11
 ; RV32I-NEXT:  .LBB44_1: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
 ; RV32I-NEXT:    slli a1, a0, 24
 ; RV32I-NEXT:    srai a1, a1, 24
 ; RV32I-NEXT:    mv a2, a0
-; RV32I-NEXT:    bge s5, a1, .LBB44_3
+; RV32I-NEXT:    bge s1, a1, .LBB44_3
 ; RV32I-NEXT:  # %bb.2: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB44_1 Depth=1
 ; RV32I-NEXT:    mv a2, s2
 ; RV32I-NEXT:  .LBB44_3: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB44_1 Depth=1
-; RV32I-NEXT:    sb a0, 7(sp)
+; RV32I-NEXT:    sb a0, 11(sp)
 ; RV32I-NEXT:    mv a0, s4
 ; RV32I-NEXT:    mv a1, s3
-; RV32I-NEXT:    mv a3, s1
-; RV32I-NEXT:    mv a4, s1
+; RV32I-NEXT:    addi a3, zero, 5
+; RV32I-NEXT:    addi a4, zero, 5
 ; RV32I-NEXT:    call __atomic_compare_exchange_1
 ; RV32I-NEXT:    mv a1, a0
-; RV32I-NEXT:    lb a0, 7(sp)
+; RV32I-NEXT:    lb a0, 11(sp)
 ; RV32I-NEXT:    beqz a1, .LBB44_1
 ; RV32I-NEXT:  # %bb.4: # %atomicrmw.end
-; RV32I-NEXT:    lw s5, 8(sp)
 ; RV32I-NEXT:    lw s4, 12(sp)
 ; RV32I-NEXT:    lw s3, 16(sp)
 ; RV32I-NEXT:    lw s2, 20(sp)
@@ -1900,6 +3491,79 @@ define i8 @atomicrmw_min_i8_seq_cst(i8 *%a, i8 %b) nounwind {
 ; RV32IA-NEXT:  # %bb.4:
 ; RV32IA-NEXT:    srl a0, a5, a2
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_min_i8_seq_cst:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -48
+; RV64I-NEXT:    sd ra, 40(sp)
+; RV64I-NEXT:    sd s1, 32(sp)
+; RV64I-NEXT:    sd s2, 24(sp)
+; RV64I-NEXT:    sd s3, 16(sp)
+; RV64I-NEXT:    sd s4, 8(sp)
+; RV64I-NEXT:    mv s2, a1
+; RV64I-NEXT:    mv s4, a0
+; RV64I-NEXT:    lbu a0, 0(a0)
+; RV64I-NEXT:    slli a1, a1, 56
+; RV64I-NEXT:    srai s1, a1, 56
+; RV64I-NEXT:    addi s3, sp, 7
+; RV64I-NEXT:  .LBB44_1: # %atomicrmw.start
+; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV64I-NEXT:    slli a1, a0, 56
+; RV64I-NEXT:    srai a1, a1, 56
+; RV64I-NEXT:    mv a2, a0
+; RV64I-NEXT:    bge s1, a1, .LBB44_3
+; RV64I-NEXT:  # %bb.2: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB44_1 Depth=1
+; RV64I-NEXT:    mv a2, s2
+; RV64I-NEXT:  .LBB44_3: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB44_1 Depth=1
+; RV64I-NEXT:    sb a0, 7(sp)
+; RV64I-NEXT:    mv a0, s4
+; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    addi a3, zero, 5
+; RV64I-NEXT:    addi a4, zero, 5
+; RV64I-NEXT:    call __atomic_compare_exchange_1
+; RV64I-NEXT:    mv a1, a0
+; RV64I-NEXT:    lb a0, 7(sp)
+; RV64I-NEXT:    beqz a1, .LBB44_1
+; RV64I-NEXT:  # %bb.4: # %atomicrmw.end
+; RV64I-NEXT:    ld s4, 8(sp)
+; RV64I-NEXT:    ld s3, 16(sp)
+; RV64I-NEXT:    ld s2, 24(sp)
+; RV64I-NEXT:    ld s1, 32(sp)
+; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    addi sp, sp, 48
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_min_i8_seq_cst:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    slli a2, a0, 3
+; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    addi a3, zero, 56
+; RV64IA-NEXT:    sub a6, a3, a2
+; RV64IA-NEXT:    addi a4, zero, 255
+; RV64IA-NEXT:    sllw a7, a4, a2
+; RV64IA-NEXT:    slli a1, a1, 56
+; RV64IA-NEXT:    srai a1, a1, 56
+; RV64IA-NEXT:    sllw a1, a1, a2
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:  .LBB44_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.w.aqrl a5, (a0)
+; RV64IA-NEXT:    and a4, a5, a7
+; RV64IA-NEXT:    mv a3, a5
+; RV64IA-NEXT:    sll a4, a4, a6
+; RV64IA-NEXT:    sra a4, a4, a6
+; RV64IA-NEXT:    bge a1, a4, .LBB44_3
+; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB44_1 Depth=1
+; RV64IA-NEXT:    xor a3, a5, a1
+; RV64IA-NEXT:    and a3, a3, a7
+; RV64IA-NEXT:    xor a3, a5, a3
+; RV64IA-NEXT:  .LBB44_3: # in Loop: Header=BB44_1 Depth=1
+; RV64IA-NEXT:    sc.w.aqrl a3, a3, (a0)
+; RV64IA-NEXT:    bnez a3, .LBB44_1
+; RV64IA-NEXT:  # %bb.4:
+; RV64IA-NEXT:    srlw a0, a5, a2
+; RV64IA-NEXT:    ret
   %1 = atomicrmw min i8* %a, i8 %b seq_cst
   ret i8 %1
 }
@@ -1970,6 +3634,72 @@ define i8 @atomicrmw_umax_i8_monotonic(i8 *%a, i8 %b) nounwind {
 ; RV32IA-NEXT:  # %bb.4:
 ; RV32IA-NEXT:    srl a0, a4, a2
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_umax_i8_monotonic:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -48
+; RV64I-NEXT:    sd ra, 40(sp)
+; RV64I-NEXT:    sd s1, 32(sp)
+; RV64I-NEXT:    sd s2, 24(sp)
+; RV64I-NEXT:    sd s3, 16(sp)
+; RV64I-NEXT:    sd s4, 8(sp)
+; RV64I-NEXT:    mv s2, a1
+; RV64I-NEXT:    mv s4, a0
+; RV64I-NEXT:    lbu a0, 0(a0)
+; RV64I-NEXT:    andi s1, a1, 255
+; RV64I-NEXT:    addi s3, sp, 7
+; RV64I-NEXT:  .LBB45_1: # %atomicrmw.start
+; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV64I-NEXT:    andi a1, a0, 255
+; RV64I-NEXT:    mv a2, a0
+; RV64I-NEXT:    bltu s1, a1, .LBB45_3
+; RV64I-NEXT:  # %bb.2: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB45_1 Depth=1
+; RV64I-NEXT:    mv a2, s2
+; RV64I-NEXT:  .LBB45_3: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB45_1 Depth=1
+; RV64I-NEXT:    sb a0, 7(sp)
+; RV64I-NEXT:    mv a0, s4
+; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    mv a3, zero
+; RV64I-NEXT:    mv a4, zero
+; RV64I-NEXT:    call __atomic_compare_exchange_1
+; RV64I-NEXT:    mv a1, a0
+; RV64I-NEXT:    lb a0, 7(sp)
+; RV64I-NEXT:    beqz a1, .LBB45_1
+; RV64I-NEXT:  # %bb.4: # %atomicrmw.end
+; RV64I-NEXT:    ld s4, 8(sp)
+; RV64I-NEXT:    ld s3, 16(sp)
+; RV64I-NEXT:    ld s2, 24(sp)
+; RV64I-NEXT:    ld s1, 32(sp)
+; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    addi sp, sp, 48
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_umax_i8_monotonic:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    slli a2, a0, 3
+; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    addi a3, zero, 255
+; RV64IA-NEXT:    sllw a6, a3, a2
+; RV64IA-NEXT:    andi a1, a1, 255
+; RV64IA-NEXT:    sllw a1, a1, a2
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:  .LBB45_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.w a4, (a0)
+; RV64IA-NEXT:    and a3, a4, a6
+; RV64IA-NEXT:    mv a5, a4
+; RV64IA-NEXT:    bgeu a3, a1, .LBB45_3
+; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB45_1 Depth=1
+; RV64IA-NEXT:    xor a5, a4, a1
+; RV64IA-NEXT:    and a5, a5, a6
+; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:  .LBB45_3: # in Loop: Header=BB45_1 Depth=1
+; RV64IA-NEXT:    sc.w a5, a5, (a0)
+; RV64IA-NEXT:    bnez a5, .LBB45_1
+; RV64IA-NEXT:  # %bb.4:
+; RV64IA-NEXT:    srlw a0, a4, a2
+; RV64IA-NEXT:    ret
   %1 = atomicrmw umax i8* %a, i8 %b monotonic
   ret i8 %1
 }
@@ -1983,34 +3713,31 @@ define i8 @atomicrmw_umax_i8_acquire(i8 *%a, i8 %b) nounwind {
 ; RV32I-NEXT:    sw s2, 20(sp)
 ; RV32I-NEXT:    sw s3, 16(sp)
 ; RV32I-NEXT:    sw s4, 12(sp)
-; RV32I-NEXT:    sw s5, 8(sp)
 ; RV32I-NEXT:    mv s2, a1
 ; RV32I-NEXT:    mv s4, a0
 ; RV32I-NEXT:    lbu a0, 0(a0)
-; RV32I-NEXT:    andi s5, a1, 255
-; RV32I-NEXT:    addi s3, sp, 7
-; RV32I-NEXT:    addi s1, zero, 2
+; RV32I-NEXT:    andi s1, a1, 255
+; RV32I-NEXT:    addi s3, sp, 11
 ; RV32I-NEXT:  .LBB46_1: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
 ; RV32I-NEXT:    andi a1, a0, 255
 ; RV32I-NEXT:    mv a2, a0
-; RV32I-NEXT:    bltu s5, a1, .LBB46_3
+; RV32I-NEXT:    bltu s1, a1, .LBB46_3
 ; RV32I-NEXT:  # %bb.2: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB46_1 Depth=1
 ; RV32I-NEXT:    mv a2, s2
 ; RV32I-NEXT:  .LBB46_3: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB46_1 Depth=1
-; RV32I-NEXT:    sb a0, 7(sp)
+; RV32I-NEXT:    sb a0, 11(sp)
 ; RV32I-NEXT:    mv a0, s4
 ; RV32I-NEXT:    mv a1, s3
-; RV32I-NEXT:    mv a3, s1
-; RV32I-NEXT:    mv a4, s1
+; RV32I-NEXT:    addi a3, zero, 2
+; RV32I-NEXT:    addi a4, zero, 2
 ; RV32I-NEXT:    call __atomic_compare_exchange_1
 ; RV32I-NEXT:    mv a1, a0
-; RV32I-NEXT:    lb a0, 7(sp)
+; RV32I-NEXT:    lb a0, 11(sp)
 ; RV32I-NEXT:    beqz a1, .LBB46_1
 ; RV32I-NEXT:  # %bb.4: # %atomicrmw.end
-; RV32I-NEXT:    lw s5, 8(sp)
 ; RV32I-NEXT:    lw s4, 12(sp)
 ; RV32I-NEXT:    lw s3, 16(sp)
 ; RV32I-NEXT:    lw s2, 20(sp)
@@ -2043,6 +3770,72 @@ define i8 @atomicrmw_umax_i8_acquire(i8 *%a, i8 %b) nounwind {
 ; RV32IA-NEXT:  # %bb.4:
 ; RV32IA-NEXT:    srl a0, a4, a2
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_umax_i8_acquire:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -48
+; RV64I-NEXT:    sd ra, 40(sp)
+; RV64I-NEXT:    sd s1, 32(sp)
+; RV64I-NEXT:    sd s2, 24(sp)
+; RV64I-NEXT:    sd s3, 16(sp)
+; RV64I-NEXT:    sd s4, 8(sp)
+; RV64I-NEXT:    mv s2, a1
+; RV64I-NEXT:    mv s4, a0
+; RV64I-NEXT:    lbu a0, 0(a0)
+; RV64I-NEXT:    andi s1, a1, 255
+; RV64I-NEXT:    addi s3, sp, 7
+; RV64I-NEXT:  .LBB46_1: # %atomicrmw.start
+; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV64I-NEXT:    andi a1, a0, 255
+; RV64I-NEXT:    mv a2, a0
+; RV64I-NEXT:    bltu s1, a1, .LBB46_3
+; RV64I-NEXT:  # %bb.2: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB46_1 Depth=1
+; RV64I-NEXT:    mv a2, s2
+; RV64I-NEXT:  .LBB46_3: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB46_1 Depth=1
+; RV64I-NEXT:    sb a0, 7(sp)
+; RV64I-NEXT:    mv a0, s4
+; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    addi a3, zero, 2
+; RV64I-NEXT:    addi a4, zero, 2
+; RV64I-NEXT:    call __atomic_compare_exchange_1
+; RV64I-NEXT:    mv a1, a0
+; RV64I-NEXT:    lb a0, 7(sp)
+; RV64I-NEXT:    beqz a1, .LBB46_1
+; RV64I-NEXT:  # %bb.4: # %atomicrmw.end
+; RV64I-NEXT:    ld s4, 8(sp)
+; RV64I-NEXT:    ld s3, 16(sp)
+; RV64I-NEXT:    ld s2, 24(sp)
+; RV64I-NEXT:    ld s1, 32(sp)
+; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    addi sp, sp, 48
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_umax_i8_acquire:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    slli a2, a0, 3
+; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    addi a3, zero, 255
+; RV64IA-NEXT:    sllw a6, a3, a2
+; RV64IA-NEXT:    andi a1, a1, 255
+; RV64IA-NEXT:    sllw a1, a1, a2
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:  .LBB46_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.w.aq a4, (a0)
+; RV64IA-NEXT:    and a3, a4, a6
+; RV64IA-NEXT:    mv a5, a4
+; RV64IA-NEXT:    bgeu a3, a1, .LBB46_3
+; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB46_1 Depth=1
+; RV64IA-NEXT:    xor a5, a4, a1
+; RV64IA-NEXT:    and a5, a5, a6
+; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:  .LBB46_3: # in Loop: Header=BB46_1 Depth=1
+; RV64IA-NEXT:    sc.w a5, a5, (a0)
+; RV64IA-NEXT:    bnez a5, .LBB46_1
+; RV64IA-NEXT:  # %bb.4:
+; RV64IA-NEXT:    srlw a0, a4, a2
+; RV64IA-NEXT:    ret
   %1 = atomicrmw umax i8* %a, i8 %b acquire
   ret i8 %1
 }
@@ -2056,13 +3849,11 @@ define i8 @atomicrmw_umax_i8_release(i8 *%a, i8 %b) nounwind {
 ; RV32I-NEXT:    sw s2, 20(sp)
 ; RV32I-NEXT:    sw s3, 16(sp)
 ; RV32I-NEXT:    sw s4, 12(sp)
-; RV32I-NEXT:    sw s5, 8(sp)
 ; RV32I-NEXT:    mv s2, a1
-; RV32I-NEXT:    mv s5, a0
+; RV32I-NEXT:    mv s4, a0
 ; RV32I-NEXT:    lbu a0, 0(a0)
 ; RV32I-NEXT:    andi s1, a1, 255
-; RV32I-NEXT:    addi s3, sp, 7
-; RV32I-NEXT:    addi s4, zero, 3
+; RV32I-NEXT:    addi s3, sp, 11
 ; RV32I-NEXT:  .LBB47_1: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
 ; RV32I-NEXT:    andi a1, a0, 255
@@ -2073,17 +3864,16 @@ define i8 @atomicrmw_umax_i8_release(i8 *%a, i8 %b) nounwind {
 ; RV32I-NEXT:    mv a2, s2
 ; RV32I-NEXT:  .LBB47_3: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB47_1 Depth=1
-; RV32I-NEXT:    sb a0, 7(sp)
-; RV32I-NEXT:    mv a0, s5
+; RV32I-NEXT:    sb a0, 11(sp)
+; RV32I-NEXT:    mv a0, s4
 ; RV32I-NEXT:    mv a1, s3
-; RV32I-NEXT:    mv a3, s4
+; RV32I-NEXT:    addi a3, zero, 3
 ; RV32I-NEXT:    mv a4, zero
 ; RV32I-NEXT:    call __atomic_compare_exchange_1
 ; RV32I-NEXT:    mv a1, a0
-; RV32I-NEXT:    lb a0, 7(sp)
+; RV32I-NEXT:    lb a0, 11(sp)
 ; RV32I-NEXT:    beqz a1, .LBB47_1
 ; RV32I-NEXT:  # %bb.4: # %atomicrmw.end
-; RV32I-NEXT:    lw s5, 8(sp)
 ; RV32I-NEXT:    lw s4, 12(sp)
 ; RV32I-NEXT:    lw s3, 16(sp)
 ; RV32I-NEXT:    lw s2, 20(sp)
@@ -2116,6 +3906,72 @@ define i8 @atomicrmw_umax_i8_release(i8 *%a, i8 %b) nounwind {
 ; RV32IA-NEXT:  # %bb.4:
 ; RV32IA-NEXT:    srl a0, a4, a2
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_umax_i8_release:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -48
+; RV64I-NEXT:    sd ra, 40(sp)
+; RV64I-NEXT:    sd s1, 32(sp)
+; RV64I-NEXT:    sd s2, 24(sp)
+; RV64I-NEXT:    sd s3, 16(sp)
+; RV64I-NEXT:    sd s4, 8(sp)
+; RV64I-NEXT:    mv s2, a1
+; RV64I-NEXT:    mv s4, a0
+; RV64I-NEXT:    lbu a0, 0(a0)
+; RV64I-NEXT:    andi s1, a1, 255
+; RV64I-NEXT:    addi s3, sp, 7
+; RV64I-NEXT:  .LBB47_1: # %atomicrmw.start
+; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV64I-NEXT:    andi a1, a0, 255
+; RV64I-NEXT:    mv a2, a0
+; RV64I-NEXT:    bltu s1, a1, .LBB47_3
+; RV64I-NEXT:  # %bb.2: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB47_1 Depth=1
+; RV64I-NEXT:    mv a2, s2
+; RV64I-NEXT:  .LBB47_3: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB47_1 Depth=1
+; RV64I-NEXT:    sb a0, 7(sp)
+; RV64I-NEXT:    mv a0, s4
+; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    addi a3, zero, 3
+; RV64I-NEXT:    mv a4, zero
+; RV64I-NEXT:    call __atomic_compare_exchange_1
+; RV64I-NEXT:    mv a1, a0
+; RV64I-NEXT:    lb a0, 7(sp)
+; RV64I-NEXT:    beqz a1, .LBB47_1
+; RV64I-NEXT:  # %bb.4: # %atomicrmw.end
+; RV64I-NEXT:    ld s4, 8(sp)
+; RV64I-NEXT:    ld s3, 16(sp)
+; RV64I-NEXT:    ld s2, 24(sp)
+; RV64I-NEXT:    ld s1, 32(sp)
+; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    addi sp, sp, 48
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_umax_i8_release:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    slli a2, a0, 3
+; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    addi a3, zero, 255
+; RV64IA-NEXT:    sllw a6, a3, a2
+; RV64IA-NEXT:    andi a1, a1, 255
+; RV64IA-NEXT:    sllw a1, a1, a2
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:  .LBB47_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.w a4, (a0)
+; RV64IA-NEXT:    and a3, a4, a6
+; RV64IA-NEXT:    mv a5, a4
+; RV64IA-NEXT:    bgeu a3, a1, .LBB47_3
+; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB47_1 Depth=1
+; RV64IA-NEXT:    xor a5, a4, a1
+; RV64IA-NEXT:    and a5, a5, a6
+; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:  .LBB47_3: # in Loop: Header=BB47_1 Depth=1
+; RV64IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV64IA-NEXT:    bnez a5, .LBB47_1
+; RV64IA-NEXT:  # %bb.4:
+; RV64IA-NEXT:    srlw a0, a4, a2
+; RV64IA-NEXT:    ret
   %1 = atomicrmw umax i8* %a, i8 %b release
   ret i8 %1
 }
@@ -2129,15 +3985,11 @@ define i8 @atomicrmw_umax_i8_acq_rel(i8 *%a, i8 %b) nounwind {
 ; RV32I-NEXT:    sw s2, 20(sp)
 ; RV32I-NEXT:    sw s3, 16(sp)
 ; RV32I-NEXT:    sw s4, 12(sp)
-; RV32I-NEXT:    sw s5, 8(sp)
-; RV32I-NEXT:    sw s6, 4(sp)
 ; RV32I-NEXT:    mv s2, a1
-; RV32I-NEXT:    mv s6, a0
+; RV32I-NEXT:    mv s4, a0
 ; RV32I-NEXT:    lbu a0, 0(a0)
 ; RV32I-NEXT:    andi s1, a1, 255
-; RV32I-NEXT:    addi s3, sp, 3
-; RV32I-NEXT:    addi s4, zero, 4
-; RV32I-NEXT:    addi s5, zero, 2
+; RV32I-NEXT:    addi s3, sp, 11
 ; RV32I-NEXT:  .LBB48_1: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
 ; RV32I-NEXT:    andi a1, a0, 255
@@ -2148,18 +4000,16 @@ define i8 @atomicrmw_umax_i8_acq_rel(i8 *%a, i8 %b) nounwind {
 ; RV32I-NEXT:    mv a2, s2
 ; RV32I-NEXT:  .LBB48_3: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB48_1 Depth=1
-; RV32I-NEXT:    sb a0, 3(sp)
-; RV32I-NEXT:    mv a0, s6
+; RV32I-NEXT:    sb a0, 11(sp)
+; RV32I-NEXT:    mv a0, s4
 ; RV32I-NEXT:    mv a1, s3
-; RV32I-NEXT:    mv a3, s4
-; RV32I-NEXT:    mv a4, s5
+; RV32I-NEXT:    addi a3, zero, 4
+; RV32I-NEXT:    addi a4, zero, 2
 ; RV32I-NEXT:    call __atomic_compare_exchange_1
 ; RV32I-NEXT:    mv a1, a0
-; RV32I-NEXT:    lb a0, 3(sp)
+; RV32I-NEXT:    lb a0, 11(sp)
 ; RV32I-NEXT:    beqz a1, .LBB48_1
 ; RV32I-NEXT:  # %bb.4: # %atomicrmw.end
-; RV32I-NEXT:    lw s6, 4(sp)
-; RV32I-NEXT:    lw s5, 8(sp)
 ; RV32I-NEXT:    lw s4, 12(sp)
 ; RV32I-NEXT:    lw s3, 16(sp)
 ; RV32I-NEXT:    lw s2, 20(sp)
@@ -2192,6 +4042,72 @@ define i8 @atomicrmw_umax_i8_acq_rel(i8 *%a, i8 %b) nounwind {
 ; RV32IA-NEXT:  # %bb.4:
 ; RV32IA-NEXT:    srl a0, a4, a2
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_umax_i8_acq_rel:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -48
+; RV64I-NEXT:    sd ra, 40(sp)
+; RV64I-NEXT:    sd s1, 32(sp)
+; RV64I-NEXT:    sd s2, 24(sp)
+; RV64I-NEXT:    sd s3, 16(sp)
+; RV64I-NEXT:    sd s4, 8(sp)
+; RV64I-NEXT:    mv s2, a1
+; RV64I-NEXT:    mv s4, a0
+; RV64I-NEXT:    lbu a0, 0(a0)
+; RV64I-NEXT:    andi s1, a1, 255
+; RV64I-NEXT:    addi s3, sp, 7
+; RV64I-NEXT:  .LBB48_1: # %atomicrmw.start
+; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV64I-NEXT:    andi a1, a0, 255
+; RV64I-NEXT:    mv a2, a0
+; RV64I-NEXT:    bltu s1, a1, .LBB48_3
+; RV64I-NEXT:  # %bb.2: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB48_1 Depth=1
+; RV64I-NEXT:    mv a2, s2
+; RV64I-NEXT:  .LBB48_3: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB48_1 Depth=1
+; RV64I-NEXT:    sb a0, 7(sp)
+; RV64I-NEXT:    mv a0, s4
+; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    addi a3, zero, 4
+; RV64I-NEXT:    addi a4, zero, 2
+; RV64I-NEXT:    call __atomic_compare_exchange_1
+; RV64I-NEXT:    mv a1, a0
+; RV64I-NEXT:    lb a0, 7(sp)
+; RV64I-NEXT:    beqz a1, .LBB48_1
+; RV64I-NEXT:  # %bb.4: # %atomicrmw.end
+; RV64I-NEXT:    ld s4, 8(sp)
+; RV64I-NEXT:    ld s3, 16(sp)
+; RV64I-NEXT:    ld s2, 24(sp)
+; RV64I-NEXT:    ld s1, 32(sp)
+; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    addi sp, sp, 48
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_umax_i8_acq_rel:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    slli a2, a0, 3
+; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    addi a3, zero, 255
+; RV64IA-NEXT:    sllw a6, a3, a2
+; RV64IA-NEXT:    andi a1, a1, 255
+; RV64IA-NEXT:    sllw a1, a1, a2
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:  .LBB48_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.w.aq a4, (a0)
+; RV64IA-NEXT:    and a3, a4, a6
+; RV64IA-NEXT:    mv a5, a4
+; RV64IA-NEXT:    bgeu a3, a1, .LBB48_3
+; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB48_1 Depth=1
+; RV64IA-NEXT:    xor a5, a4, a1
+; RV64IA-NEXT:    and a5, a5, a6
+; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:  .LBB48_3: # in Loop: Header=BB48_1 Depth=1
+; RV64IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV64IA-NEXT:    bnez a5, .LBB48_1
+; RV64IA-NEXT:  # %bb.4:
+; RV64IA-NEXT:    srlw a0, a4, a2
+; RV64IA-NEXT:    ret
   %1 = atomicrmw umax i8* %a, i8 %b acq_rel
   ret i8 %1
 }
@@ -2205,34 +4121,31 @@ define i8 @atomicrmw_umax_i8_seq_cst(i8 *%a, i8 %b) nounwind {
 ; RV32I-NEXT:    sw s2, 20(sp)
 ; RV32I-NEXT:    sw s3, 16(sp)
 ; RV32I-NEXT:    sw s4, 12(sp)
-; RV32I-NEXT:    sw s5, 8(sp)
 ; RV32I-NEXT:    mv s2, a1
 ; RV32I-NEXT:    mv s4, a0
 ; RV32I-NEXT:    lbu a0, 0(a0)
-; RV32I-NEXT:    andi s5, a1, 255
-; RV32I-NEXT:    addi s3, sp, 7
-; RV32I-NEXT:    addi s1, zero, 5
+; RV32I-NEXT:    andi s1, a1, 255
+; RV32I-NEXT:    addi s3, sp, 11
 ; RV32I-NEXT:  .LBB49_1: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
 ; RV32I-NEXT:    andi a1, a0, 255
 ; RV32I-NEXT:    mv a2, a0
-; RV32I-NEXT:    bltu s5, a1, .LBB49_3
+; RV32I-NEXT:    bltu s1, a1, .LBB49_3
 ; RV32I-NEXT:  # %bb.2: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB49_1 Depth=1
 ; RV32I-NEXT:    mv a2, s2
 ; RV32I-NEXT:  .LBB49_3: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB49_1 Depth=1
-; RV32I-NEXT:    sb a0, 7(sp)
+; RV32I-NEXT:    sb a0, 11(sp)
 ; RV32I-NEXT:    mv a0, s4
 ; RV32I-NEXT:    mv a1, s3
-; RV32I-NEXT:    mv a3, s1
-; RV32I-NEXT:    mv a4, s1
+; RV32I-NEXT:    addi a3, zero, 5
+; RV32I-NEXT:    addi a4, zero, 5
 ; RV32I-NEXT:    call __atomic_compare_exchange_1
 ; RV32I-NEXT:    mv a1, a0
-; RV32I-NEXT:    lb a0, 7(sp)
+; RV32I-NEXT:    lb a0, 11(sp)
 ; RV32I-NEXT:    beqz a1, .LBB49_1
 ; RV32I-NEXT:  # %bb.4: # %atomicrmw.end
-; RV32I-NEXT:    lw s5, 8(sp)
 ; RV32I-NEXT:    lw s4, 12(sp)
 ; RV32I-NEXT:    lw s3, 16(sp)
 ; RV32I-NEXT:    lw s2, 20(sp)
@@ -2265,6 +4178,72 @@ define i8 @atomicrmw_umax_i8_seq_cst(i8 *%a, i8 %b) nounwind {
 ; RV32IA-NEXT:  # %bb.4:
 ; RV32IA-NEXT:    srl a0, a4, a2
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_umax_i8_seq_cst:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -48
+; RV64I-NEXT:    sd ra, 40(sp)
+; RV64I-NEXT:    sd s1, 32(sp)
+; RV64I-NEXT:    sd s2, 24(sp)
+; RV64I-NEXT:    sd s3, 16(sp)
+; RV64I-NEXT:    sd s4, 8(sp)
+; RV64I-NEXT:    mv s2, a1
+; RV64I-NEXT:    mv s4, a0
+; RV64I-NEXT:    lbu a0, 0(a0)
+; RV64I-NEXT:    andi s1, a1, 255
+; RV64I-NEXT:    addi s3, sp, 7
+; RV64I-NEXT:  .LBB49_1: # %atomicrmw.start
+; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV64I-NEXT:    andi a1, a0, 255
+; RV64I-NEXT:    mv a2, a0
+; RV64I-NEXT:    bltu s1, a1, .LBB49_3
+; RV64I-NEXT:  # %bb.2: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB49_1 Depth=1
+; RV64I-NEXT:    mv a2, s2
+; RV64I-NEXT:  .LBB49_3: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB49_1 Depth=1
+; RV64I-NEXT:    sb a0, 7(sp)
+; RV64I-NEXT:    mv a0, s4
+; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    addi a3, zero, 5
+; RV64I-NEXT:    addi a4, zero, 5
+; RV64I-NEXT:    call __atomic_compare_exchange_1
+; RV64I-NEXT:    mv a1, a0
+; RV64I-NEXT:    lb a0, 7(sp)
+; RV64I-NEXT:    beqz a1, .LBB49_1
+; RV64I-NEXT:  # %bb.4: # %atomicrmw.end
+; RV64I-NEXT:    ld s4, 8(sp)
+; RV64I-NEXT:    ld s3, 16(sp)
+; RV64I-NEXT:    ld s2, 24(sp)
+; RV64I-NEXT:    ld s1, 32(sp)
+; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    addi sp, sp, 48
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_umax_i8_seq_cst:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    slli a2, a0, 3
+; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    addi a3, zero, 255
+; RV64IA-NEXT:    sllw a6, a3, a2
+; RV64IA-NEXT:    andi a1, a1, 255
+; RV64IA-NEXT:    sllw a1, a1, a2
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:  .LBB49_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.w.aqrl a4, (a0)
+; RV64IA-NEXT:    and a3, a4, a6
+; RV64IA-NEXT:    mv a5, a4
+; RV64IA-NEXT:    bgeu a3, a1, .LBB49_3
+; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB49_1 Depth=1
+; RV64IA-NEXT:    xor a5, a4, a1
+; RV64IA-NEXT:    and a5, a5, a6
+; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:  .LBB49_3: # in Loop: Header=BB49_1 Depth=1
+; RV64IA-NEXT:    sc.w.aqrl a5, a5, (a0)
+; RV64IA-NEXT:    bnez a5, .LBB49_1
+; RV64IA-NEXT:  # %bb.4:
+; RV64IA-NEXT:    srlw a0, a4, a2
+; RV64IA-NEXT:    ret
   %1 = atomicrmw umax i8* %a, i8 %b seq_cst
   ret i8 %1
 }
@@ -2335,6 +4314,72 @@ define i8 @atomicrmw_umin_i8_monotonic(i8 *%a, i8 %b) nounwind {
 ; RV32IA-NEXT:  # %bb.4:
 ; RV32IA-NEXT:    srl a0, a4, a2
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_umin_i8_monotonic:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -48
+; RV64I-NEXT:    sd ra, 40(sp)
+; RV64I-NEXT:    sd s1, 32(sp)
+; RV64I-NEXT:    sd s2, 24(sp)
+; RV64I-NEXT:    sd s3, 16(sp)
+; RV64I-NEXT:    sd s4, 8(sp)
+; RV64I-NEXT:    mv s2, a1
+; RV64I-NEXT:    mv s4, a0
+; RV64I-NEXT:    lbu a0, 0(a0)
+; RV64I-NEXT:    andi s1, a1, 255
+; RV64I-NEXT:    addi s3, sp, 7
+; RV64I-NEXT:  .LBB50_1: # %atomicrmw.start
+; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV64I-NEXT:    andi a1, a0, 255
+; RV64I-NEXT:    mv a2, a0
+; RV64I-NEXT:    bgeu s1, a1, .LBB50_3
+; RV64I-NEXT:  # %bb.2: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB50_1 Depth=1
+; RV64I-NEXT:    mv a2, s2
+; RV64I-NEXT:  .LBB50_3: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB50_1 Depth=1
+; RV64I-NEXT:    sb a0, 7(sp)
+; RV64I-NEXT:    mv a0, s4
+; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    mv a3, zero
+; RV64I-NEXT:    mv a4, zero
+; RV64I-NEXT:    call __atomic_compare_exchange_1
+; RV64I-NEXT:    mv a1, a0
+; RV64I-NEXT:    lb a0, 7(sp)
+; RV64I-NEXT:    beqz a1, .LBB50_1
+; RV64I-NEXT:  # %bb.4: # %atomicrmw.end
+; RV64I-NEXT:    ld s4, 8(sp)
+; RV64I-NEXT:    ld s3, 16(sp)
+; RV64I-NEXT:    ld s2, 24(sp)
+; RV64I-NEXT:    ld s1, 32(sp)
+; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    addi sp, sp, 48
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_umin_i8_monotonic:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    slli a2, a0, 3
+; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    addi a3, zero, 255
+; RV64IA-NEXT:    sllw a6, a3, a2
+; RV64IA-NEXT:    andi a1, a1, 255
+; RV64IA-NEXT:    sllw a1, a1, a2
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:  .LBB50_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.w a4, (a0)
+; RV64IA-NEXT:    and a3, a4, a6
+; RV64IA-NEXT:    mv a5, a4
+; RV64IA-NEXT:    bgeu a1, a3, .LBB50_3
+; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB50_1 Depth=1
+; RV64IA-NEXT:    xor a5, a4, a1
+; RV64IA-NEXT:    and a5, a5, a6
+; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:  .LBB50_3: # in Loop: Header=BB50_1 Depth=1
+; RV64IA-NEXT:    sc.w a5, a5, (a0)
+; RV64IA-NEXT:    bnez a5, .LBB50_1
+; RV64IA-NEXT:  # %bb.4:
+; RV64IA-NEXT:    srlw a0, a4, a2
+; RV64IA-NEXT:    ret
   %1 = atomicrmw umin i8* %a, i8 %b monotonic
   ret i8 %1
 }
@@ -2348,34 +4393,31 @@ define i8 @atomicrmw_umin_i8_acquire(i8 *%a, i8 %b) nounwind {
 ; RV32I-NEXT:    sw s2, 20(sp)
 ; RV32I-NEXT:    sw s3, 16(sp)
 ; RV32I-NEXT:    sw s4, 12(sp)
-; RV32I-NEXT:    sw s5, 8(sp)
 ; RV32I-NEXT:    mv s2, a1
 ; RV32I-NEXT:    mv s4, a0
 ; RV32I-NEXT:    lbu a0, 0(a0)
-; RV32I-NEXT:    andi s5, a1, 255
-; RV32I-NEXT:    addi s3, sp, 7
-; RV32I-NEXT:    addi s1, zero, 2
+; RV32I-NEXT:    andi s1, a1, 255
+; RV32I-NEXT:    addi s3, sp, 11
 ; RV32I-NEXT:  .LBB51_1: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
 ; RV32I-NEXT:    andi a1, a0, 255
 ; RV32I-NEXT:    mv a2, a0
-; RV32I-NEXT:    bgeu s5, a1, .LBB51_3
+; RV32I-NEXT:    bgeu s1, a1, .LBB51_3
 ; RV32I-NEXT:  # %bb.2: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB51_1 Depth=1
 ; RV32I-NEXT:    mv a2, s2
 ; RV32I-NEXT:  .LBB51_3: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB51_1 Depth=1
-; RV32I-NEXT:    sb a0, 7(sp)
+; RV32I-NEXT:    sb a0, 11(sp)
 ; RV32I-NEXT:    mv a0, s4
 ; RV32I-NEXT:    mv a1, s3
-; RV32I-NEXT:    mv a3, s1
-; RV32I-NEXT:    mv a4, s1
+; RV32I-NEXT:    addi a3, zero, 2
+; RV32I-NEXT:    addi a4, zero, 2
 ; RV32I-NEXT:    call __atomic_compare_exchange_1
 ; RV32I-NEXT:    mv a1, a0
-; RV32I-NEXT:    lb a0, 7(sp)
+; RV32I-NEXT:    lb a0, 11(sp)
 ; RV32I-NEXT:    beqz a1, .LBB51_1
 ; RV32I-NEXT:  # %bb.4: # %atomicrmw.end
-; RV32I-NEXT:    lw s5, 8(sp)
 ; RV32I-NEXT:    lw s4, 12(sp)
 ; RV32I-NEXT:    lw s3, 16(sp)
 ; RV32I-NEXT:    lw s2, 20(sp)
@@ -2408,6 +4450,72 @@ define i8 @atomicrmw_umin_i8_acquire(i8 *%a, i8 %b) nounwind {
 ; RV32IA-NEXT:  # %bb.4:
 ; RV32IA-NEXT:    srl a0, a4, a2
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_umin_i8_acquire:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -48
+; RV64I-NEXT:    sd ra, 40(sp)
+; RV64I-NEXT:    sd s1, 32(sp)
+; RV64I-NEXT:    sd s2, 24(sp)
+; RV64I-NEXT:    sd s3, 16(sp)
+; RV64I-NEXT:    sd s4, 8(sp)
+; RV64I-NEXT:    mv s2, a1
+; RV64I-NEXT:    mv s4, a0
+; RV64I-NEXT:    lbu a0, 0(a0)
+; RV64I-NEXT:    andi s1, a1, 255
+; RV64I-NEXT:    addi s3, sp, 7
+; RV64I-NEXT:  .LBB51_1: # %atomicrmw.start
+; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV64I-NEXT:    andi a1, a0, 255
+; RV64I-NEXT:    mv a2, a0
+; RV64I-NEXT:    bgeu s1, a1, .LBB51_3
+; RV64I-NEXT:  # %bb.2: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB51_1 Depth=1
+; RV64I-NEXT:    mv a2, s2
+; RV64I-NEXT:  .LBB51_3: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB51_1 Depth=1
+; RV64I-NEXT:    sb a0, 7(sp)
+; RV64I-NEXT:    mv a0, s4
+; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    addi a3, zero, 2
+; RV64I-NEXT:    addi a4, zero, 2
+; RV64I-NEXT:    call __atomic_compare_exchange_1
+; RV64I-NEXT:    mv a1, a0
+; RV64I-NEXT:    lb a0, 7(sp)
+; RV64I-NEXT:    beqz a1, .LBB51_1
+; RV64I-NEXT:  # %bb.4: # %atomicrmw.end
+; RV64I-NEXT:    ld s4, 8(sp)
+; RV64I-NEXT:    ld s3, 16(sp)
+; RV64I-NEXT:    ld s2, 24(sp)
+; RV64I-NEXT:    ld s1, 32(sp)
+; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    addi sp, sp, 48
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_umin_i8_acquire:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    slli a2, a0, 3
+; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    addi a3, zero, 255
+; RV64IA-NEXT:    sllw a6, a3, a2
+; RV64IA-NEXT:    andi a1, a1, 255
+; RV64IA-NEXT:    sllw a1, a1, a2
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:  .LBB51_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.w.aq a4, (a0)
+; RV64IA-NEXT:    and a3, a4, a6
+; RV64IA-NEXT:    mv a5, a4
+; RV64IA-NEXT:    bgeu a1, a3, .LBB51_3
+; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB51_1 Depth=1
+; RV64IA-NEXT:    xor a5, a4, a1
+; RV64IA-NEXT:    and a5, a5, a6
+; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:  .LBB51_3: # in Loop: Header=BB51_1 Depth=1
+; RV64IA-NEXT:    sc.w a5, a5, (a0)
+; RV64IA-NEXT:    bnez a5, .LBB51_1
+; RV64IA-NEXT:  # %bb.4:
+; RV64IA-NEXT:    srlw a0, a4, a2
+; RV64IA-NEXT:    ret
   %1 = atomicrmw umin i8* %a, i8 %b acquire
   ret i8 %1
 }
@@ -2421,13 +4529,11 @@ define i8 @atomicrmw_umin_i8_release(i8 *%a, i8 %b) nounwind {
 ; RV32I-NEXT:    sw s2, 20(sp)
 ; RV32I-NEXT:    sw s3, 16(sp)
 ; RV32I-NEXT:    sw s4, 12(sp)
-; RV32I-NEXT:    sw s5, 8(sp)
 ; RV32I-NEXT:    mv s2, a1
-; RV32I-NEXT:    mv s5, a0
+; RV32I-NEXT:    mv s4, a0
 ; RV32I-NEXT:    lbu a0, 0(a0)
 ; RV32I-NEXT:    andi s1, a1, 255
-; RV32I-NEXT:    addi s3, sp, 7
-; RV32I-NEXT:    addi s4, zero, 3
+; RV32I-NEXT:    addi s3, sp, 11
 ; RV32I-NEXT:  .LBB52_1: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
 ; RV32I-NEXT:    andi a1, a0, 255
@@ -2438,17 +4544,16 @@ define i8 @atomicrmw_umin_i8_release(i8 *%a, i8 %b) nounwind {
 ; RV32I-NEXT:    mv a2, s2
 ; RV32I-NEXT:  .LBB52_3: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB52_1 Depth=1
-; RV32I-NEXT:    sb a0, 7(sp)
-; RV32I-NEXT:    mv a0, s5
+; RV32I-NEXT:    sb a0, 11(sp)
+; RV32I-NEXT:    mv a0, s4
 ; RV32I-NEXT:    mv a1, s3
-; RV32I-NEXT:    mv a3, s4
+; RV32I-NEXT:    addi a3, zero, 3
 ; RV32I-NEXT:    mv a4, zero
 ; RV32I-NEXT:    call __atomic_compare_exchange_1
 ; RV32I-NEXT:    mv a1, a0
-; RV32I-NEXT:    lb a0, 7(sp)
+; RV32I-NEXT:    lb a0, 11(sp)
 ; RV32I-NEXT:    beqz a1, .LBB52_1
 ; RV32I-NEXT:  # %bb.4: # %atomicrmw.end
-; RV32I-NEXT:    lw s5, 8(sp)
 ; RV32I-NEXT:    lw s4, 12(sp)
 ; RV32I-NEXT:    lw s3, 16(sp)
 ; RV32I-NEXT:    lw s2, 20(sp)
@@ -2481,6 +4586,72 @@ define i8 @atomicrmw_umin_i8_release(i8 *%a, i8 %b) nounwind {
 ; RV32IA-NEXT:  # %bb.4:
 ; RV32IA-NEXT:    srl a0, a4, a2
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_umin_i8_release:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -48
+; RV64I-NEXT:    sd ra, 40(sp)
+; RV64I-NEXT:    sd s1, 32(sp)
+; RV64I-NEXT:    sd s2, 24(sp)
+; RV64I-NEXT:    sd s3, 16(sp)
+; RV64I-NEXT:    sd s4, 8(sp)
+; RV64I-NEXT:    mv s2, a1
+; RV64I-NEXT:    mv s4, a0
+; RV64I-NEXT:    lbu a0, 0(a0)
+; RV64I-NEXT:    andi s1, a1, 255
+; RV64I-NEXT:    addi s3, sp, 7
+; RV64I-NEXT:  .LBB52_1: # %atomicrmw.start
+; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV64I-NEXT:    andi a1, a0, 255
+; RV64I-NEXT:    mv a2, a0
+; RV64I-NEXT:    bgeu s1, a1, .LBB52_3
+; RV64I-NEXT:  # %bb.2: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB52_1 Depth=1
+; RV64I-NEXT:    mv a2, s2
+; RV64I-NEXT:  .LBB52_3: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB52_1 Depth=1
+; RV64I-NEXT:    sb a0, 7(sp)
+; RV64I-NEXT:    mv a0, s4
+; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    addi a3, zero, 3
+; RV64I-NEXT:    mv a4, zero
+; RV64I-NEXT:    call __atomic_compare_exchange_1
+; RV64I-NEXT:    mv a1, a0
+; RV64I-NEXT:    lb a0, 7(sp)
+; RV64I-NEXT:    beqz a1, .LBB52_1
+; RV64I-NEXT:  # %bb.4: # %atomicrmw.end
+; RV64I-NEXT:    ld s4, 8(sp)
+; RV64I-NEXT:    ld s3, 16(sp)
+; RV64I-NEXT:    ld s2, 24(sp)
+; RV64I-NEXT:    ld s1, 32(sp)
+; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    addi sp, sp, 48
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_umin_i8_release:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    slli a2, a0, 3
+; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    addi a3, zero, 255
+; RV64IA-NEXT:    sllw a6, a3, a2
+; RV64IA-NEXT:    andi a1, a1, 255
+; RV64IA-NEXT:    sllw a1, a1, a2
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:  .LBB52_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.w a4, (a0)
+; RV64IA-NEXT:    and a3, a4, a6
+; RV64IA-NEXT:    mv a5, a4
+; RV64IA-NEXT:    bgeu a1, a3, .LBB52_3
+; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB52_1 Depth=1
+; RV64IA-NEXT:    xor a5, a4, a1
+; RV64IA-NEXT:    and a5, a5, a6
+; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:  .LBB52_3: # in Loop: Header=BB52_1 Depth=1
+; RV64IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV64IA-NEXT:    bnez a5, .LBB52_1
+; RV64IA-NEXT:  # %bb.4:
+; RV64IA-NEXT:    srlw a0, a4, a2
+; RV64IA-NEXT:    ret
   %1 = atomicrmw umin i8* %a, i8 %b release
   ret i8 %1
 }
@@ -2494,15 +4665,11 @@ define i8 @atomicrmw_umin_i8_acq_rel(i8 *%a, i8 %b) nounwind {
 ; RV32I-NEXT:    sw s2, 20(sp)
 ; RV32I-NEXT:    sw s3, 16(sp)
 ; RV32I-NEXT:    sw s4, 12(sp)
-; RV32I-NEXT:    sw s5, 8(sp)
-; RV32I-NEXT:    sw s6, 4(sp)
 ; RV32I-NEXT:    mv s2, a1
-; RV32I-NEXT:    mv s6, a0
+; RV32I-NEXT:    mv s4, a0
 ; RV32I-NEXT:    lbu a0, 0(a0)
 ; RV32I-NEXT:    andi s1, a1, 255
-; RV32I-NEXT:    addi s3, sp, 3
-; RV32I-NEXT:    addi s4, zero, 4
-; RV32I-NEXT:    addi s5, zero, 2
+; RV32I-NEXT:    addi s3, sp, 11
 ; RV32I-NEXT:  .LBB53_1: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
 ; RV32I-NEXT:    andi a1, a0, 255
@@ -2513,18 +4680,16 @@ define i8 @atomicrmw_umin_i8_acq_rel(i8 *%a, i8 %b) nounwind {
 ; RV32I-NEXT:    mv a2, s2
 ; RV32I-NEXT:  .LBB53_3: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB53_1 Depth=1
-; RV32I-NEXT:    sb a0, 3(sp)
-; RV32I-NEXT:    mv a0, s6
+; RV32I-NEXT:    sb a0, 11(sp)
+; RV32I-NEXT:    mv a0, s4
 ; RV32I-NEXT:    mv a1, s3
-; RV32I-NEXT:    mv a3, s4
-; RV32I-NEXT:    mv a4, s5
+; RV32I-NEXT:    addi a3, zero, 4
+; RV32I-NEXT:    addi a4, zero, 2
 ; RV32I-NEXT:    call __atomic_compare_exchange_1
 ; RV32I-NEXT:    mv a1, a0
-; RV32I-NEXT:    lb a0, 3(sp)
+; RV32I-NEXT:    lb a0, 11(sp)
 ; RV32I-NEXT:    beqz a1, .LBB53_1
 ; RV32I-NEXT:  # %bb.4: # %atomicrmw.end
-; RV32I-NEXT:    lw s6, 4(sp)
-; RV32I-NEXT:    lw s5, 8(sp)
 ; RV32I-NEXT:    lw s4, 12(sp)
 ; RV32I-NEXT:    lw s3, 16(sp)
 ; RV32I-NEXT:    lw s2, 20(sp)
@@ -2557,6 +4722,72 @@ define i8 @atomicrmw_umin_i8_acq_rel(i8 *%a, i8 %b) nounwind {
 ; RV32IA-NEXT:  # %bb.4:
 ; RV32IA-NEXT:    srl a0, a4, a2
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_umin_i8_acq_rel:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -48
+; RV64I-NEXT:    sd ra, 40(sp)
+; RV64I-NEXT:    sd s1, 32(sp)
+; RV64I-NEXT:    sd s2, 24(sp)
+; RV64I-NEXT:    sd s3, 16(sp)
+; RV64I-NEXT:    sd s4, 8(sp)
+; RV64I-NEXT:    mv s2, a1
+; RV64I-NEXT:    mv s4, a0
+; RV64I-NEXT:    lbu a0, 0(a0)
+; RV64I-NEXT:    andi s1, a1, 255
+; RV64I-NEXT:    addi s3, sp, 7
+; RV64I-NEXT:  .LBB53_1: # %atomicrmw.start
+; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV64I-NEXT:    andi a1, a0, 255
+; RV64I-NEXT:    mv a2, a0
+; RV64I-NEXT:    bgeu s1, a1, .LBB53_3
+; RV64I-NEXT:  # %bb.2: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB53_1 Depth=1
+; RV64I-NEXT:    mv a2, s2
+; RV64I-NEXT:  .LBB53_3: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB53_1 Depth=1
+; RV64I-NEXT:    sb a0, 7(sp)
+; RV64I-NEXT:    mv a0, s4
+; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    addi a3, zero, 4
+; RV64I-NEXT:    addi a4, zero, 2
+; RV64I-NEXT:    call __atomic_compare_exchange_1
+; RV64I-NEXT:    mv a1, a0
+; RV64I-NEXT:    lb a0, 7(sp)
+; RV64I-NEXT:    beqz a1, .LBB53_1
+; RV64I-NEXT:  # %bb.4: # %atomicrmw.end
+; RV64I-NEXT:    ld s4, 8(sp)
+; RV64I-NEXT:    ld s3, 16(sp)
+; RV64I-NEXT:    ld s2, 24(sp)
+; RV64I-NEXT:    ld s1, 32(sp)
+; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    addi sp, sp, 48
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_umin_i8_acq_rel:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    slli a2, a0, 3
+; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    addi a3, zero, 255
+; RV64IA-NEXT:    sllw a6, a3, a2
+; RV64IA-NEXT:    andi a1, a1, 255
+; RV64IA-NEXT:    sllw a1, a1, a2
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:  .LBB53_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.w.aq a4, (a0)
+; RV64IA-NEXT:    and a3, a4, a6
+; RV64IA-NEXT:    mv a5, a4
+; RV64IA-NEXT:    bgeu a1, a3, .LBB53_3
+; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB53_1 Depth=1
+; RV64IA-NEXT:    xor a5, a4, a1
+; RV64IA-NEXT:    and a5, a5, a6
+; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:  .LBB53_3: # in Loop: Header=BB53_1 Depth=1
+; RV64IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV64IA-NEXT:    bnez a5, .LBB53_1
+; RV64IA-NEXT:  # %bb.4:
+; RV64IA-NEXT:    srlw a0, a4, a2
+; RV64IA-NEXT:    ret
   %1 = atomicrmw umin i8* %a, i8 %b acq_rel
   ret i8 %1
 }
@@ -2570,34 +4801,31 @@ define i8 @atomicrmw_umin_i8_seq_cst(i8 *%a, i8 %b) nounwind {
 ; RV32I-NEXT:    sw s2, 20(sp)
 ; RV32I-NEXT:    sw s3, 16(sp)
 ; RV32I-NEXT:    sw s4, 12(sp)
-; RV32I-NEXT:    sw s5, 8(sp)
 ; RV32I-NEXT:    mv s2, a1
 ; RV32I-NEXT:    mv s4, a0
 ; RV32I-NEXT:    lbu a0, 0(a0)
-; RV32I-NEXT:    andi s5, a1, 255
-; RV32I-NEXT:    addi s3, sp, 7
-; RV32I-NEXT:    addi s1, zero, 5
+; RV32I-NEXT:    andi s1, a1, 255
+; RV32I-NEXT:    addi s3, sp, 11
 ; RV32I-NEXT:  .LBB54_1: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
 ; RV32I-NEXT:    andi a1, a0, 255
 ; RV32I-NEXT:    mv a2, a0
-; RV32I-NEXT:    bgeu s5, a1, .LBB54_3
+; RV32I-NEXT:    bgeu s1, a1, .LBB54_3
 ; RV32I-NEXT:  # %bb.2: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB54_1 Depth=1
 ; RV32I-NEXT:    mv a2, s2
 ; RV32I-NEXT:  .LBB54_3: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB54_1 Depth=1
-; RV32I-NEXT:    sb a0, 7(sp)
+; RV32I-NEXT:    sb a0, 11(sp)
 ; RV32I-NEXT:    mv a0, s4
 ; RV32I-NEXT:    mv a1, s3
-; RV32I-NEXT:    mv a3, s1
-; RV32I-NEXT:    mv a4, s1
+; RV32I-NEXT:    addi a3, zero, 5
+; RV32I-NEXT:    addi a4, zero, 5
 ; RV32I-NEXT:    call __atomic_compare_exchange_1
 ; RV32I-NEXT:    mv a1, a0
-; RV32I-NEXT:    lb a0, 7(sp)
+; RV32I-NEXT:    lb a0, 11(sp)
 ; RV32I-NEXT:    beqz a1, .LBB54_1
 ; RV32I-NEXT:  # %bb.4: # %atomicrmw.end
-; RV32I-NEXT:    lw s5, 8(sp)
 ; RV32I-NEXT:    lw s4, 12(sp)
 ; RV32I-NEXT:    lw s3, 16(sp)
 ; RV32I-NEXT:    lw s2, 20(sp)
@@ -2630,6 +4858,72 @@ define i8 @atomicrmw_umin_i8_seq_cst(i8 *%a, i8 %b) nounwind {
 ; RV32IA-NEXT:  # %bb.4:
 ; RV32IA-NEXT:    srl a0, a4, a2
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_umin_i8_seq_cst:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -48
+; RV64I-NEXT:    sd ra, 40(sp)
+; RV64I-NEXT:    sd s1, 32(sp)
+; RV64I-NEXT:    sd s2, 24(sp)
+; RV64I-NEXT:    sd s3, 16(sp)
+; RV64I-NEXT:    sd s4, 8(sp)
+; RV64I-NEXT:    mv s2, a1
+; RV64I-NEXT:    mv s4, a0
+; RV64I-NEXT:    lbu a0, 0(a0)
+; RV64I-NEXT:    andi s1, a1, 255
+; RV64I-NEXT:    addi s3, sp, 7
+; RV64I-NEXT:  .LBB54_1: # %atomicrmw.start
+; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV64I-NEXT:    andi a1, a0, 255
+; RV64I-NEXT:    mv a2, a0
+; RV64I-NEXT:    bgeu s1, a1, .LBB54_3
+; RV64I-NEXT:  # %bb.2: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB54_1 Depth=1
+; RV64I-NEXT:    mv a2, s2
+; RV64I-NEXT:  .LBB54_3: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB54_1 Depth=1
+; RV64I-NEXT:    sb a0, 7(sp)
+; RV64I-NEXT:    mv a0, s4
+; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    addi a3, zero, 5
+; RV64I-NEXT:    addi a4, zero, 5
+; RV64I-NEXT:    call __atomic_compare_exchange_1
+; RV64I-NEXT:    mv a1, a0
+; RV64I-NEXT:    lb a0, 7(sp)
+; RV64I-NEXT:    beqz a1, .LBB54_1
+; RV64I-NEXT:  # %bb.4: # %atomicrmw.end
+; RV64I-NEXT:    ld s4, 8(sp)
+; RV64I-NEXT:    ld s3, 16(sp)
+; RV64I-NEXT:    ld s2, 24(sp)
+; RV64I-NEXT:    ld s1, 32(sp)
+; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    addi sp, sp, 48
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_umin_i8_seq_cst:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    slli a2, a0, 3
+; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    addi a3, zero, 255
+; RV64IA-NEXT:    sllw a6, a3, a2
+; RV64IA-NEXT:    andi a1, a1, 255
+; RV64IA-NEXT:    sllw a1, a1, a2
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:  .LBB54_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.w.aqrl a4, (a0)
+; RV64IA-NEXT:    and a3, a4, a6
+; RV64IA-NEXT:    mv a5, a4
+; RV64IA-NEXT:    bgeu a1, a3, .LBB54_3
+; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB54_1 Depth=1
+; RV64IA-NEXT:    xor a5, a4, a1
+; RV64IA-NEXT:    and a5, a5, a6
+; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:  .LBB54_3: # in Loop: Header=BB54_1 Depth=1
+; RV64IA-NEXT:    sc.w.aqrl a5, a5, (a0)
+; RV64IA-NEXT:    bnez a5, .LBB54_1
+; RV64IA-NEXT:  # %bb.4:
+; RV64IA-NEXT:    srlw a0, a4, a2
+; RV64IA-NEXT:    ret
   %1 = atomicrmw umin i8* %a, i8 %b seq_cst
   ret i8 %1
 }
@@ -2666,6 +4960,38 @@ define i16 @atomicrmw_xchg_i16_monotonic(i16* %a, i16 %b) {
 ; RV32IA-NEXT:  # %bb.2:
 ; RV32IA-NEXT:    srl a0, a4, a3
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_xchg_i16_monotonic:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    mv a2, zero
+; RV64I-NEXT:    call __atomic_exchange_2
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_xchg_i16_monotonic:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    lui a2, 16
+; RV64IA-NEXT:    addiw a2, a2, -1
+; RV64IA-NEXT:    and a1, a1, a2
+; RV64IA-NEXT:    slli a3, a0, 3
+; RV64IA-NEXT:    andi a3, a3, 24
+; RV64IA-NEXT:    sllw a2, a2, a3
+; RV64IA-NEXT:    sllw a1, a1, a3
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:  .LBB55_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.w a4, (a0)
+; RV64IA-NEXT:    add a5, zero, a1
+; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:    and a5, a5, a2
+; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:    sc.w a5, a5, (a0)
+; RV64IA-NEXT:    bnez a5, .LBB55_1
+; RV64IA-NEXT:  # %bb.2:
+; RV64IA-NEXT:    srlw a0, a4, a3
+; RV64IA-NEXT:    ret
   %1 = atomicrmw xchg i16* %a, i16 %b monotonic
   ret i16 %1
 }
@@ -2702,6 +5028,38 @@ define i16 @atomicrmw_xchg_i16_acquire(i16* %a, i16 %b) {
 ; RV32IA-NEXT:  # %bb.2:
 ; RV32IA-NEXT:    srl a0, a4, a3
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_xchg_i16_acquire:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a2, zero, 2
+; RV64I-NEXT:    call __atomic_exchange_2
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_xchg_i16_acquire:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    lui a2, 16
+; RV64IA-NEXT:    addiw a2, a2, -1
+; RV64IA-NEXT:    and a1, a1, a2
+; RV64IA-NEXT:    slli a3, a0, 3
+; RV64IA-NEXT:    andi a3, a3, 24
+; RV64IA-NEXT:    sllw a2, a2, a3
+; RV64IA-NEXT:    sllw a1, a1, a3
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:  .LBB56_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.w.aq a4, (a0)
+; RV64IA-NEXT:    add a5, zero, a1
+; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:    and a5, a5, a2
+; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:    sc.w a5, a5, (a0)
+; RV64IA-NEXT:    bnez a5, .LBB56_1
+; RV64IA-NEXT:  # %bb.2:
+; RV64IA-NEXT:    srlw a0, a4, a3
+; RV64IA-NEXT:    ret
   %1 = atomicrmw xchg i16* %a, i16 %b acquire
   ret i16 %1
 }
@@ -2738,6 +5096,38 @@ define i16 @atomicrmw_xchg_i16_release(i16* %a, i16 %b) {
 ; RV32IA-NEXT:  # %bb.2:
 ; RV32IA-NEXT:    srl a0, a4, a3
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_xchg_i16_release:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a2, zero, 3
+; RV64I-NEXT:    call __atomic_exchange_2
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_xchg_i16_release:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    lui a2, 16
+; RV64IA-NEXT:    addiw a2, a2, -1
+; RV64IA-NEXT:    and a1, a1, a2
+; RV64IA-NEXT:    slli a3, a0, 3
+; RV64IA-NEXT:    andi a3, a3, 24
+; RV64IA-NEXT:    sllw a2, a2, a3
+; RV64IA-NEXT:    sllw a1, a1, a3
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:  .LBB57_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.w a4, (a0)
+; RV64IA-NEXT:    add a5, zero, a1
+; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:    and a5, a5, a2
+; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV64IA-NEXT:    bnez a5, .LBB57_1
+; RV64IA-NEXT:  # %bb.2:
+; RV64IA-NEXT:    srlw a0, a4, a3
+; RV64IA-NEXT:    ret
   %1 = atomicrmw xchg i16* %a, i16 %b release
   ret i16 %1
 }
@@ -2774,6 +5164,38 @@ define i16 @atomicrmw_xchg_i16_acq_rel(i16* %a, i16 %b) {
 ; RV32IA-NEXT:  # %bb.2:
 ; RV32IA-NEXT:    srl a0, a4, a3
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_xchg_i16_acq_rel:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a2, zero, 4
+; RV64I-NEXT:    call __atomic_exchange_2
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_xchg_i16_acq_rel:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    lui a2, 16
+; RV64IA-NEXT:    addiw a2, a2, -1
+; RV64IA-NEXT:    and a1, a1, a2
+; RV64IA-NEXT:    slli a3, a0, 3
+; RV64IA-NEXT:    andi a3, a3, 24
+; RV64IA-NEXT:    sllw a2, a2, a3
+; RV64IA-NEXT:    sllw a1, a1, a3
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:  .LBB58_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.w.aq a4, (a0)
+; RV64IA-NEXT:    add a5, zero, a1
+; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:    and a5, a5, a2
+; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV64IA-NEXT:    bnez a5, .LBB58_1
+; RV64IA-NEXT:  # %bb.2:
+; RV64IA-NEXT:    srlw a0, a4, a3
+; RV64IA-NEXT:    ret
   %1 = atomicrmw xchg i16* %a, i16 %b acq_rel
   ret i16 %1
 }
@@ -2810,6 +5232,38 @@ define i16 @atomicrmw_xchg_i16_seq_cst(i16* %a, i16 %b) {
 ; RV32IA-NEXT:  # %bb.2:
 ; RV32IA-NEXT:    srl a0, a4, a3
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_xchg_i16_seq_cst:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a2, zero, 5
+; RV64I-NEXT:    call __atomic_exchange_2
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_xchg_i16_seq_cst:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    lui a2, 16
+; RV64IA-NEXT:    addiw a2, a2, -1
+; RV64IA-NEXT:    and a1, a1, a2
+; RV64IA-NEXT:    slli a3, a0, 3
+; RV64IA-NEXT:    andi a3, a3, 24
+; RV64IA-NEXT:    sllw a2, a2, a3
+; RV64IA-NEXT:    sllw a1, a1, a3
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:  .LBB59_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.w.aqrl a4, (a0)
+; RV64IA-NEXT:    add a5, zero, a1
+; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:    and a5, a5, a2
+; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:    sc.w.aqrl a5, a5, (a0)
+; RV64IA-NEXT:    bnez a5, .LBB59_1
+; RV64IA-NEXT:  # %bb.2:
+; RV64IA-NEXT:    srlw a0, a4, a3
+; RV64IA-NEXT:    ret
   %1 = atomicrmw xchg i16* %a, i16 %b seq_cst
   ret i16 %1
 }
@@ -2846,6 +5300,38 @@ define i16 @atomicrmw_add_i16_monotonic(i16 *%a, i16 %b) nounwind {
 ; RV32IA-NEXT:  # %bb.2:
 ; RV32IA-NEXT:    srl a0, a4, a3
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_add_i16_monotonic:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    mv a2, zero
+; RV64I-NEXT:    call __atomic_fetch_add_2
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_add_i16_monotonic:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    lui a2, 16
+; RV64IA-NEXT:    addiw a2, a2, -1
+; RV64IA-NEXT:    and a1, a1, a2
+; RV64IA-NEXT:    slli a3, a0, 3
+; RV64IA-NEXT:    andi a3, a3, 24
+; RV64IA-NEXT:    sllw a2, a2, a3
+; RV64IA-NEXT:    sllw a1, a1, a3
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:  .LBB60_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.w a4, (a0)
+; RV64IA-NEXT:    add a5, a4, a1
+; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:    and a5, a5, a2
+; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:    sc.w a5, a5, (a0)
+; RV64IA-NEXT:    bnez a5, .LBB60_1
+; RV64IA-NEXT:  # %bb.2:
+; RV64IA-NEXT:    srlw a0, a4, a3
+; RV64IA-NEXT:    ret
   %1 = atomicrmw add i16* %a, i16 %b monotonic
   ret i16 %1
 }
@@ -2882,6 +5368,38 @@ define i16 @atomicrmw_add_i16_acquire(i16 *%a, i16 %b) nounwind {
 ; RV32IA-NEXT:  # %bb.2:
 ; RV32IA-NEXT:    srl a0, a4, a3
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_add_i16_acquire:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a2, zero, 2
+; RV64I-NEXT:    call __atomic_fetch_add_2
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_add_i16_acquire:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    lui a2, 16
+; RV64IA-NEXT:    addiw a2, a2, -1
+; RV64IA-NEXT:    and a1, a1, a2
+; RV64IA-NEXT:    slli a3, a0, 3
+; RV64IA-NEXT:    andi a3, a3, 24
+; RV64IA-NEXT:    sllw a2, a2, a3
+; RV64IA-NEXT:    sllw a1, a1, a3
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:  .LBB61_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.w.aq a4, (a0)
+; RV64IA-NEXT:    add a5, a4, a1
+; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:    and a5, a5, a2
+; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:    sc.w a5, a5, (a0)
+; RV64IA-NEXT:    bnez a5, .LBB61_1
+; RV64IA-NEXT:  # %bb.2:
+; RV64IA-NEXT:    srlw a0, a4, a3
+; RV64IA-NEXT:    ret
   %1 = atomicrmw add i16* %a, i16 %b acquire
   ret i16 %1
 }
@@ -2918,6 +5436,38 @@ define i16 @atomicrmw_add_i16_release(i16 *%a, i16 %b) nounwind {
 ; RV32IA-NEXT:  # %bb.2:
 ; RV32IA-NEXT:    srl a0, a4, a3
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_add_i16_release:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a2, zero, 3
+; RV64I-NEXT:    call __atomic_fetch_add_2
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_add_i16_release:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    lui a2, 16
+; RV64IA-NEXT:    addiw a2, a2, -1
+; RV64IA-NEXT:    and a1, a1, a2
+; RV64IA-NEXT:    slli a3, a0, 3
+; RV64IA-NEXT:    andi a3, a3, 24
+; RV64IA-NEXT:    sllw a2, a2, a3
+; RV64IA-NEXT:    sllw a1, a1, a3
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:  .LBB62_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.w a4, (a0)
+; RV64IA-NEXT:    add a5, a4, a1
+; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:    and a5, a5, a2
+; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV64IA-NEXT:    bnez a5, .LBB62_1
+; RV64IA-NEXT:  # %bb.2:
+; RV64IA-NEXT:    srlw a0, a4, a3
+; RV64IA-NEXT:    ret
   %1 = atomicrmw add i16* %a, i16 %b release
   ret i16 %1
 }
@@ -2954,6 +5504,38 @@ define i16 @atomicrmw_add_i16_acq_rel(i16 *%a, i16 %b) nounwind {
 ; RV32IA-NEXT:  # %bb.2:
 ; RV32IA-NEXT:    srl a0, a4, a3
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_add_i16_acq_rel:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a2, zero, 4
+; RV64I-NEXT:    call __atomic_fetch_add_2
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_add_i16_acq_rel:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    lui a2, 16
+; RV64IA-NEXT:    addiw a2, a2, -1
+; RV64IA-NEXT:    and a1, a1, a2
+; RV64IA-NEXT:    slli a3, a0, 3
+; RV64IA-NEXT:    andi a3, a3, 24
+; RV64IA-NEXT:    sllw a2, a2, a3
+; RV64IA-NEXT:    sllw a1, a1, a3
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:  .LBB63_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.w.aq a4, (a0)
+; RV64IA-NEXT:    add a5, a4, a1
+; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:    and a5, a5, a2
+; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV64IA-NEXT:    bnez a5, .LBB63_1
+; RV64IA-NEXT:  # %bb.2:
+; RV64IA-NEXT:    srlw a0, a4, a3
+; RV64IA-NEXT:    ret
   %1 = atomicrmw add i16* %a, i16 %b acq_rel
   ret i16 %1
 }
@@ -2990,6 +5572,38 @@ define i16 @atomicrmw_add_i16_seq_cst(i16 *%a, i16 %b) nounwind {
 ; RV32IA-NEXT:  # %bb.2:
 ; RV32IA-NEXT:    srl a0, a4, a3
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_add_i16_seq_cst:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a2, zero, 5
+; RV64I-NEXT:    call __atomic_fetch_add_2
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_add_i16_seq_cst:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    lui a2, 16
+; RV64IA-NEXT:    addiw a2, a2, -1
+; RV64IA-NEXT:    and a1, a1, a2
+; RV64IA-NEXT:    slli a3, a0, 3
+; RV64IA-NEXT:    andi a3, a3, 24
+; RV64IA-NEXT:    sllw a2, a2, a3
+; RV64IA-NEXT:    sllw a1, a1, a3
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:  .LBB64_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.w.aqrl a4, (a0)
+; RV64IA-NEXT:    add a5, a4, a1
+; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:    and a5, a5, a2
+; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:    sc.w.aqrl a5, a5, (a0)
+; RV64IA-NEXT:    bnez a5, .LBB64_1
+; RV64IA-NEXT:  # %bb.2:
+; RV64IA-NEXT:    srlw a0, a4, a3
+; RV64IA-NEXT:    ret
   %1 = atomicrmw add i16* %a, i16 %b seq_cst
   ret i16 %1
 }
@@ -3026,6 +5640,38 @@ define i16 @atomicrmw_sub_i16_monotonic(i16* %a, i16 %b) {
 ; RV32IA-NEXT:  # %bb.2:
 ; RV32IA-NEXT:    srl a0, a4, a3
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_sub_i16_monotonic:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    mv a2, zero
+; RV64I-NEXT:    call __atomic_fetch_sub_2
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_sub_i16_monotonic:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    lui a2, 16
+; RV64IA-NEXT:    addiw a2, a2, -1
+; RV64IA-NEXT:    and a1, a1, a2
+; RV64IA-NEXT:    slli a3, a0, 3
+; RV64IA-NEXT:    andi a3, a3, 24
+; RV64IA-NEXT:    sllw a2, a2, a3
+; RV64IA-NEXT:    sllw a1, a1, a3
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:  .LBB65_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.w a4, (a0)
+; RV64IA-NEXT:    sub a5, a4, a1
+; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:    and a5, a5, a2
+; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:    sc.w a5, a5, (a0)
+; RV64IA-NEXT:    bnez a5, .LBB65_1
+; RV64IA-NEXT:  # %bb.2:
+; RV64IA-NEXT:    srlw a0, a4, a3
+; RV64IA-NEXT:    ret
   %1 = atomicrmw sub i16* %a, i16 %b monotonic
   ret i16 %1
 }
@@ -3062,6 +5708,38 @@ define i16 @atomicrmw_sub_i16_acquire(i16* %a, i16 %b) {
 ; RV32IA-NEXT:  # %bb.2:
 ; RV32IA-NEXT:    srl a0, a4, a3
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_sub_i16_acquire:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a2, zero, 2
+; RV64I-NEXT:    call __atomic_fetch_sub_2
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_sub_i16_acquire:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    lui a2, 16
+; RV64IA-NEXT:    addiw a2, a2, -1
+; RV64IA-NEXT:    and a1, a1, a2
+; RV64IA-NEXT:    slli a3, a0, 3
+; RV64IA-NEXT:    andi a3, a3, 24
+; RV64IA-NEXT:    sllw a2, a2, a3
+; RV64IA-NEXT:    sllw a1, a1, a3
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:  .LBB66_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.w.aq a4, (a0)
+; RV64IA-NEXT:    sub a5, a4, a1
+; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:    and a5, a5, a2
+; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:    sc.w a5, a5, (a0)
+; RV64IA-NEXT:    bnez a5, .LBB66_1
+; RV64IA-NEXT:  # %bb.2:
+; RV64IA-NEXT:    srlw a0, a4, a3
+; RV64IA-NEXT:    ret
   %1 = atomicrmw sub i16* %a, i16 %b acquire
   ret i16 %1
 }
@@ -3098,6 +5776,38 @@ define i16 @atomicrmw_sub_i16_release(i16* %a, i16 %b) {
 ; RV32IA-NEXT:  # %bb.2:
 ; RV32IA-NEXT:    srl a0, a4, a3
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_sub_i16_release:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a2, zero, 3
+; RV64I-NEXT:    call __atomic_fetch_sub_2
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_sub_i16_release:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    lui a2, 16
+; RV64IA-NEXT:    addiw a2, a2, -1
+; RV64IA-NEXT:    and a1, a1, a2
+; RV64IA-NEXT:    slli a3, a0, 3
+; RV64IA-NEXT:    andi a3, a3, 24
+; RV64IA-NEXT:    sllw a2, a2, a3
+; RV64IA-NEXT:    sllw a1, a1, a3
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:  .LBB67_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.w a4, (a0)
+; RV64IA-NEXT:    sub a5, a4, a1
+; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:    and a5, a5, a2
+; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV64IA-NEXT:    bnez a5, .LBB67_1
+; RV64IA-NEXT:  # %bb.2:
+; RV64IA-NEXT:    srlw a0, a4, a3
+; RV64IA-NEXT:    ret
   %1 = atomicrmw sub i16* %a, i16 %b release
   ret i16 %1
 }
@@ -3134,6 +5844,38 @@ define i16 @atomicrmw_sub_i16_acq_rel(i16* %a, i16 %b) {
 ; RV32IA-NEXT:  # %bb.2:
 ; RV32IA-NEXT:    srl a0, a4, a3
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_sub_i16_acq_rel:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a2, zero, 4
+; RV64I-NEXT:    call __atomic_fetch_sub_2
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_sub_i16_acq_rel:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    lui a2, 16
+; RV64IA-NEXT:    addiw a2, a2, -1
+; RV64IA-NEXT:    and a1, a1, a2
+; RV64IA-NEXT:    slli a3, a0, 3
+; RV64IA-NEXT:    andi a3, a3, 24
+; RV64IA-NEXT:    sllw a2, a2, a3
+; RV64IA-NEXT:    sllw a1, a1, a3
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:  .LBB68_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.w.aq a4, (a0)
+; RV64IA-NEXT:    sub a5, a4, a1
+; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:    and a5, a5, a2
+; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV64IA-NEXT:    bnez a5, .LBB68_1
+; RV64IA-NEXT:  # %bb.2:
+; RV64IA-NEXT:    srlw a0, a4, a3
+; RV64IA-NEXT:    ret
   %1 = atomicrmw sub i16* %a, i16 %b acq_rel
   ret i16 %1
 }
@@ -3170,6 +5912,38 @@ define i16 @atomicrmw_sub_i16_seq_cst(i16* %a, i16 %b) {
 ; RV32IA-NEXT:  # %bb.2:
 ; RV32IA-NEXT:    srl a0, a4, a3
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_sub_i16_seq_cst:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a2, zero, 5
+; RV64I-NEXT:    call __atomic_fetch_sub_2
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_sub_i16_seq_cst:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    lui a2, 16
+; RV64IA-NEXT:    addiw a2, a2, -1
+; RV64IA-NEXT:    and a1, a1, a2
+; RV64IA-NEXT:    slli a3, a0, 3
+; RV64IA-NEXT:    andi a3, a3, 24
+; RV64IA-NEXT:    sllw a2, a2, a3
+; RV64IA-NEXT:    sllw a1, a1, a3
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:  .LBB69_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.w.aqrl a4, (a0)
+; RV64IA-NEXT:    sub a5, a4, a1
+; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:    and a5, a5, a2
+; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:    sc.w.aqrl a5, a5, (a0)
+; RV64IA-NEXT:    bnez a5, .LBB69_1
+; RV64IA-NEXT:  # %bb.2:
+; RV64IA-NEXT:    srlw a0, a4, a3
+; RV64IA-NEXT:    ret
   %1 = atomicrmw sub i16* %a, i16 %b seq_cst
   ret i16 %1
 }
@@ -3200,6 +5974,32 @@ define i16 @atomicrmw_and_i16_monotonic(i16 *%a, i16 %b) nounwind {
 ; RV32IA-NEXT:    amoand.w a0, a1, (a0)
 ; RV32IA-NEXT:    srl a0, a0, a3
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_and_i16_monotonic:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    mv a2, zero
+; RV64I-NEXT:    call __atomic_fetch_and_2
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_and_i16_monotonic:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    lui a2, 16
+; RV64IA-NEXT:    addiw a2, a2, -1
+; RV64IA-NEXT:    and a1, a1, a2
+; RV64IA-NEXT:    slli a3, a0, 3
+; RV64IA-NEXT:    andi a3, a3, 24
+; RV64IA-NEXT:    sllw a1, a1, a3
+; RV64IA-NEXT:    sllw a2, a2, a3
+; RV64IA-NEXT:    not a2, a2
+; RV64IA-NEXT:    or a1, a2, a1
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    amoand.w a0, a1, (a0)
+; RV64IA-NEXT:    srlw a0, a0, a3
+; RV64IA-NEXT:    ret
   %1 = atomicrmw and i16* %a, i16 %b monotonic
   ret i16 %1
 }
@@ -3230,6 +6030,32 @@ define i16 @atomicrmw_and_i16_acquire(i16 *%a, i16 %b) nounwind {
 ; RV32IA-NEXT:    amoand.w.aq a0, a1, (a0)
 ; RV32IA-NEXT:    srl a0, a0, a3
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_and_i16_acquire:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a2, zero, 2
+; RV64I-NEXT:    call __atomic_fetch_and_2
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_and_i16_acquire:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    lui a2, 16
+; RV64IA-NEXT:    addiw a2, a2, -1
+; RV64IA-NEXT:    and a1, a1, a2
+; RV64IA-NEXT:    slli a3, a0, 3
+; RV64IA-NEXT:    andi a3, a3, 24
+; RV64IA-NEXT:    sllw a1, a1, a3
+; RV64IA-NEXT:    sllw a2, a2, a3
+; RV64IA-NEXT:    not a2, a2
+; RV64IA-NEXT:    or a1, a2, a1
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    amoand.w.aq a0, a1, (a0)
+; RV64IA-NEXT:    srlw a0, a0, a3
+; RV64IA-NEXT:    ret
   %1 = atomicrmw and i16* %a, i16 %b acquire
   ret i16 %1
 }
@@ -3260,6 +6086,32 @@ define i16 @atomicrmw_and_i16_release(i16 *%a, i16 %b) nounwind {
 ; RV32IA-NEXT:    amoand.w.rl a0, a1, (a0)
 ; RV32IA-NEXT:    srl a0, a0, a3
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_and_i16_release:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a2, zero, 3
+; RV64I-NEXT:    call __atomic_fetch_and_2
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_and_i16_release:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    lui a2, 16
+; RV64IA-NEXT:    addiw a2, a2, -1
+; RV64IA-NEXT:    and a1, a1, a2
+; RV64IA-NEXT:    slli a3, a0, 3
+; RV64IA-NEXT:    andi a3, a3, 24
+; RV64IA-NEXT:    sllw a1, a1, a3
+; RV64IA-NEXT:    sllw a2, a2, a3
+; RV64IA-NEXT:    not a2, a2
+; RV64IA-NEXT:    or a1, a2, a1
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    amoand.w.rl a0, a1, (a0)
+; RV64IA-NEXT:    srlw a0, a0, a3
+; RV64IA-NEXT:    ret
   %1 = atomicrmw and i16* %a, i16 %b release
   ret i16 %1
 }
@@ -3290,6 +6142,32 @@ define i16 @atomicrmw_and_i16_acq_rel(i16 *%a, i16 %b) nounwind {
 ; RV32IA-NEXT:    amoand.w.aqrl a0, a1, (a0)
 ; RV32IA-NEXT:    srl a0, a0, a3
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_and_i16_acq_rel:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a2, zero, 4
+; RV64I-NEXT:    call __atomic_fetch_and_2
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_and_i16_acq_rel:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    lui a2, 16
+; RV64IA-NEXT:    addiw a2, a2, -1
+; RV64IA-NEXT:    and a1, a1, a2
+; RV64IA-NEXT:    slli a3, a0, 3
+; RV64IA-NEXT:    andi a3, a3, 24
+; RV64IA-NEXT:    sllw a1, a1, a3
+; RV64IA-NEXT:    sllw a2, a2, a3
+; RV64IA-NEXT:    not a2, a2
+; RV64IA-NEXT:    or a1, a2, a1
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    amoand.w.aqrl a0, a1, (a0)
+; RV64IA-NEXT:    srlw a0, a0, a3
+; RV64IA-NEXT:    ret
   %1 = atomicrmw and i16* %a, i16 %b acq_rel
   ret i16 %1
 }
@@ -3320,6 +6198,32 @@ define i16 @atomicrmw_and_i16_seq_cst(i16 *%a, i16 %b) nounwind {
 ; RV32IA-NEXT:    amoand.w.aqrl a0, a1, (a0)
 ; RV32IA-NEXT:    srl a0, a0, a3
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_and_i16_seq_cst:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a2, zero, 5
+; RV64I-NEXT:    call __atomic_fetch_and_2
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_and_i16_seq_cst:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    lui a2, 16
+; RV64IA-NEXT:    addiw a2, a2, -1
+; RV64IA-NEXT:    and a1, a1, a2
+; RV64IA-NEXT:    slli a3, a0, 3
+; RV64IA-NEXT:    andi a3, a3, 24
+; RV64IA-NEXT:    sllw a1, a1, a3
+; RV64IA-NEXT:    sllw a2, a2, a3
+; RV64IA-NEXT:    not a2, a2
+; RV64IA-NEXT:    or a1, a2, a1
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    amoand.w.aqrl a0, a1, (a0)
+; RV64IA-NEXT:    srlw a0, a0, a3
+; RV64IA-NEXT:    ret
   %1 = atomicrmw and i16* %a, i16 %b seq_cst
   ret i16 %1
 }
@@ -3357,6 +6261,39 @@ define i16 @atomicrmw_nand_i16_monotonic(i16* %a, i16 %b) {
 ; RV32IA-NEXT:  # %bb.2:
 ; RV32IA-NEXT:    srl a0, a4, a3
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_nand_i16_monotonic:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    mv a2, zero
+; RV64I-NEXT:    call __atomic_fetch_nand_2
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_nand_i16_monotonic:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    lui a2, 16
+; RV64IA-NEXT:    addiw a2, a2, -1
+; RV64IA-NEXT:    and a1, a1, a2
+; RV64IA-NEXT:    slli a3, a0, 3
+; RV64IA-NEXT:    andi a3, a3, 24
+; RV64IA-NEXT:    sllw a2, a2, a3
+; RV64IA-NEXT:    sllw a1, a1, a3
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:  .LBB75_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.w a4, (a0)
+; RV64IA-NEXT:    and a5, a4, a1
+; RV64IA-NEXT:    not a5, a5
+; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:    and a5, a5, a2
+; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:    sc.w a5, a5, (a0)
+; RV64IA-NEXT:    bnez a5, .LBB75_1
+; RV64IA-NEXT:  # %bb.2:
+; RV64IA-NEXT:    srlw a0, a4, a3
+; RV64IA-NEXT:    ret
   %1 = atomicrmw nand i16* %a, i16 %b monotonic
   ret i16 %1
 }
@@ -3394,6 +6331,39 @@ define i16 @atomicrmw_nand_i16_acquire(i16* %a, i16 %b) {
 ; RV32IA-NEXT:  # %bb.2:
 ; RV32IA-NEXT:    srl a0, a4, a3
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_nand_i16_acquire:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a2, zero, 2
+; RV64I-NEXT:    call __atomic_fetch_nand_2
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_nand_i16_acquire:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    lui a2, 16
+; RV64IA-NEXT:    addiw a2, a2, -1
+; RV64IA-NEXT:    and a1, a1, a2
+; RV64IA-NEXT:    slli a3, a0, 3
+; RV64IA-NEXT:    andi a3, a3, 24
+; RV64IA-NEXT:    sllw a2, a2, a3
+; RV64IA-NEXT:    sllw a1, a1, a3
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:  .LBB76_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.w.aq a4, (a0)
+; RV64IA-NEXT:    and a5, a4, a1
+; RV64IA-NEXT:    not a5, a5
+; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:    and a5, a5, a2
+; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:    sc.w a5, a5, (a0)
+; RV64IA-NEXT:    bnez a5, .LBB76_1
+; RV64IA-NEXT:  # %bb.2:
+; RV64IA-NEXT:    srlw a0, a4, a3
+; RV64IA-NEXT:    ret
   %1 = atomicrmw nand i16* %a, i16 %b acquire
   ret i16 %1
 }
@@ -3431,6 +6401,39 @@ define i16 @atomicrmw_nand_i16_release(i16* %a, i16 %b) {
 ; RV32IA-NEXT:  # %bb.2:
 ; RV32IA-NEXT:    srl a0, a4, a3
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_nand_i16_release:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a2, zero, 3
+; RV64I-NEXT:    call __atomic_fetch_nand_2
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_nand_i16_release:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    lui a2, 16
+; RV64IA-NEXT:    addiw a2, a2, -1
+; RV64IA-NEXT:    and a1, a1, a2
+; RV64IA-NEXT:    slli a3, a0, 3
+; RV64IA-NEXT:    andi a3, a3, 24
+; RV64IA-NEXT:    sllw a2, a2, a3
+; RV64IA-NEXT:    sllw a1, a1, a3
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:  .LBB77_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.w a4, (a0)
+; RV64IA-NEXT:    and a5, a4, a1
+; RV64IA-NEXT:    not a5, a5
+; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:    and a5, a5, a2
+; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV64IA-NEXT:    bnez a5, .LBB77_1
+; RV64IA-NEXT:  # %bb.2:
+; RV64IA-NEXT:    srlw a0, a4, a3
+; RV64IA-NEXT:    ret
   %1 = atomicrmw nand i16* %a, i16 %b release
   ret i16 %1
 }
@@ -3468,6 +6471,39 @@ define i16 @atomicrmw_nand_i16_acq_rel(i16* %a, i16 %b) {
 ; RV32IA-NEXT:  # %bb.2:
 ; RV32IA-NEXT:    srl a0, a4, a3
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_nand_i16_acq_rel:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a2, zero, 4
+; RV64I-NEXT:    call __atomic_fetch_nand_2
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_nand_i16_acq_rel:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    lui a2, 16
+; RV64IA-NEXT:    addiw a2, a2, -1
+; RV64IA-NEXT:    and a1, a1, a2
+; RV64IA-NEXT:    slli a3, a0, 3
+; RV64IA-NEXT:    andi a3, a3, 24
+; RV64IA-NEXT:    sllw a2, a2, a3
+; RV64IA-NEXT:    sllw a1, a1, a3
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:  .LBB78_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.w.aq a4, (a0)
+; RV64IA-NEXT:    and a5, a4, a1
+; RV64IA-NEXT:    not a5, a5
+; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:    and a5, a5, a2
+; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV64IA-NEXT:    bnez a5, .LBB78_1
+; RV64IA-NEXT:  # %bb.2:
+; RV64IA-NEXT:    srlw a0, a4, a3
+; RV64IA-NEXT:    ret
   %1 = atomicrmw nand i16* %a, i16 %b acq_rel
   ret i16 %1
 }
@@ -3505,6 +6541,39 @@ define i16 @atomicrmw_nand_i16_seq_cst(i16* %a, i16 %b) {
 ; RV32IA-NEXT:  # %bb.2:
 ; RV32IA-NEXT:    srl a0, a4, a3
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_nand_i16_seq_cst:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a2, zero, 5
+; RV64I-NEXT:    call __atomic_fetch_nand_2
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_nand_i16_seq_cst:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    lui a2, 16
+; RV64IA-NEXT:    addiw a2, a2, -1
+; RV64IA-NEXT:    and a1, a1, a2
+; RV64IA-NEXT:    slli a3, a0, 3
+; RV64IA-NEXT:    andi a3, a3, 24
+; RV64IA-NEXT:    sllw a2, a2, a3
+; RV64IA-NEXT:    sllw a1, a1, a3
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:  .LBB79_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.w.aqrl a4, (a0)
+; RV64IA-NEXT:    and a5, a4, a1
+; RV64IA-NEXT:    not a5, a5
+; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:    and a5, a5, a2
+; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:    sc.w.aqrl a5, a5, (a0)
+; RV64IA-NEXT:    bnez a5, .LBB79_1
+; RV64IA-NEXT:  # %bb.2:
+; RV64IA-NEXT:    srlw a0, a4, a3
+; RV64IA-NEXT:    ret
   %1 = atomicrmw nand i16* %a, i16 %b seq_cst
   ret i16 %1
 }
@@ -3532,6 +6601,29 @@ define i16 @atomicrmw_or_i16_monotonic(i16 *%a, i16 %b) nounwind {
 ; RV32IA-NEXT:    amoor.w a0, a1, (a0)
 ; RV32IA-NEXT:    srl a0, a0, a2
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_or_i16_monotonic:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    mv a2, zero
+; RV64I-NEXT:    call __atomic_fetch_or_2
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_or_i16_monotonic:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    lui a2, 16
+; RV64IA-NEXT:    addiw a2, a2, -1
+; RV64IA-NEXT:    and a1, a1, a2
+; RV64IA-NEXT:    slli a2, a0, 3
+; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    sllw a1, a1, a2
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    amoor.w a0, a1, (a0)
+; RV64IA-NEXT:    srlw a0, a0, a2
+; RV64IA-NEXT:    ret
   %1 = atomicrmw or i16* %a, i16 %b monotonic
   ret i16 %1
 }
@@ -3559,6 +6651,29 @@ define i16 @atomicrmw_or_i16_acquire(i16 *%a, i16 %b) nounwind {
 ; RV32IA-NEXT:    amoor.w.aq a0, a1, (a0)
 ; RV32IA-NEXT:    srl a0, a0, a2
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_or_i16_acquire:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a2, zero, 2
+; RV64I-NEXT:    call __atomic_fetch_or_2
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_or_i16_acquire:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    lui a2, 16
+; RV64IA-NEXT:    addiw a2, a2, -1
+; RV64IA-NEXT:    and a1, a1, a2
+; RV64IA-NEXT:    slli a2, a0, 3
+; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    sllw a1, a1, a2
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    amoor.w.aq a0, a1, (a0)
+; RV64IA-NEXT:    srlw a0, a0, a2
+; RV64IA-NEXT:    ret
   %1 = atomicrmw or i16* %a, i16 %b acquire
   ret i16 %1
 }
@@ -3586,6 +6701,29 @@ define i16 @atomicrmw_or_i16_release(i16 *%a, i16 %b) nounwind {
 ; RV32IA-NEXT:    amoor.w.rl a0, a1, (a0)
 ; RV32IA-NEXT:    srl a0, a0, a2
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_or_i16_release:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a2, zero, 3
+; RV64I-NEXT:    call __atomic_fetch_or_2
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_or_i16_release:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    lui a2, 16
+; RV64IA-NEXT:    addiw a2, a2, -1
+; RV64IA-NEXT:    and a1, a1, a2
+; RV64IA-NEXT:    slli a2, a0, 3
+; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    sllw a1, a1, a2
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    amoor.w.rl a0, a1, (a0)
+; RV64IA-NEXT:    srlw a0, a0, a2
+; RV64IA-NEXT:    ret
   %1 = atomicrmw or i16* %a, i16 %b release
   ret i16 %1
 }
@@ -3613,6 +6751,29 @@ define i16 @atomicrmw_or_i16_acq_rel(i16 *%a, i16 %b) nounwind {
 ; RV32IA-NEXT:    amoor.w.aqrl a0, a1, (a0)
 ; RV32IA-NEXT:    srl a0, a0, a2
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_or_i16_acq_rel:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a2, zero, 4
+; RV64I-NEXT:    call __atomic_fetch_or_2
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_or_i16_acq_rel:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    lui a2, 16
+; RV64IA-NEXT:    addiw a2, a2, -1
+; RV64IA-NEXT:    and a1, a1, a2
+; RV64IA-NEXT:    slli a2, a0, 3
+; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    sllw a1, a1, a2
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    amoor.w.aqrl a0, a1, (a0)
+; RV64IA-NEXT:    srlw a0, a0, a2
+; RV64IA-NEXT:    ret
   %1 = atomicrmw or i16* %a, i16 %b acq_rel
   ret i16 %1
 }
@@ -3640,6 +6801,29 @@ define i16 @atomicrmw_or_i16_seq_cst(i16 *%a, i16 %b) nounwind {
 ; RV32IA-NEXT:    amoor.w.aqrl a0, a1, (a0)
 ; RV32IA-NEXT:    srl a0, a0, a2
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_or_i16_seq_cst:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a2, zero, 5
+; RV64I-NEXT:    call __atomic_fetch_or_2
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_or_i16_seq_cst:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    lui a2, 16
+; RV64IA-NEXT:    addiw a2, a2, -1
+; RV64IA-NEXT:    and a1, a1, a2
+; RV64IA-NEXT:    slli a2, a0, 3
+; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    sllw a1, a1, a2
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    amoor.w.aqrl a0, a1, (a0)
+; RV64IA-NEXT:    srlw a0, a0, a2
+; RV64IA-NEXT:    ret
   %1 = atomicrmw or i16* %a, i16 %b seq_cst
   ret i16 %1
 }
@@ -3667,6 +6851,29 @@ define i16 @atomicrmw_xor_i16_monotonic(i16 *%a, i16 %b) nounwind {
 ; RV32IA-NEXT:    amoxor.w a0, a1, (a0)
 ; RV32IA-NEXT:    srl a0, a0, a2
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_xor_i16_monotonic:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    mv a2, zero
+; RV64I-NEXT:    call __atomic_fetch_xor_2
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_xor_i16_monotonic:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    lui a2, 16
+; RV64IA-NEXT:    addiw a2, a2, -1
+; RV64IA-NEXT:    and a1, a1, a2
+; RV64IA-NEXT:    slli a2, a0, 3
+; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    sllw a1, a1, a2
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    amoxor.w a0, a1, (a0)
+; RV64IA-NEXT:    srlw a0, a0, a2
+; RV64IA-NEXT:    ret
   %1 = atomicrmw xor i16* %a, i16 %b monotonic
   ret i16 %1
 }
@@ -3694,6 +6901,29 @@ define i16 @atomicrmw_xor_i16_acquire(i16 *%a, i16 %b) nounwind {
 ; RV32IA-NEXT:    amoxor.w.aq a0, a1, (a0)
 ; RV32IA-NEXT:    srl a0, a0, a2
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_xor_i16_acquire:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a2, zero, 2
+; RV64I-NEXT:    call __atomic_fetch_xor_2
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_xor_i16_acquire:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    lui a2, 16
+; RV64IA-NEXT:    addiw a2, a2, -1
+; RV64IA-NEXT:    and a1, a1, a2
+; RV64IA-NEXT:    slli a2, a0, 3
+; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    sllw a1, a1, a2
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    amoxor.w.aq a0, a1, (a0)
+; RV64IA-NEXT:    srlw a0, a0, a2
+; RV64IA-NEXT:    ret
   %1 = atomicrmw xor i16* %a, i16 %b acquire
   ret i16 %1
 }
@@ -3721,6 +6951,29 @@ define i16 @atomicrmw_xor_i16_release(i16 *%a, i16 %b) nounwind {
 ; RV32IA-NEXT:    amoxor.w.rl a0, a1, (a0)
 ; RV32IA-NEXT:    srl a0, a0, a2
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_xor_i16_release:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a2, zero, 3
+; RV64I-NEXT:    call __atomic_fetch_xor_2
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_xor_i16_release:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    lui a2, 16
+; RV64IA-NEXT:    addiw a2, a2, -1
+; RV64IA-NEXT:    and a1, a1, a2
+; RV64IA-NEXT:    slli a2, a0, 3
+; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    sllw a1, a1, a2
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    amoxor.w.rl a0, a1, (a0)
+; RV64IA-NEXT:    srlw a0, a0, a2
+; RV64IA-NEXT:    ret
   %1 = atomicrmw xor i16* %a, i16 %b release
   ret i16 %1
 }
@@ -3748,6 +7001,29 @@ define i16 @atomicrmw_xor_i16_acq_rel(i16 *%a, i16 %b) nounwind {
 ; RV32IA-NEXT:    amoxor.w.aqrl a0, a1, (a0)
 ; RV32IA-NEXT:    srl a0, a0, a2
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_xor_i16_acq_rel:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a2, zero, 4
+; RV64I-NEXT:    call __atomic_fetch_xor_2
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_xor_i16_acq_rel:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    lui a2, 16
+; RV64IA-NEXT:    addiw a2, a2, -1
+; RV64IA-NEXT:    and a1, a1, a2
+; RV64IA-NEXT:    slli a2, a0, 3
+; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    sllw a1, a1, a2
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    amoxor.w.aqrl a0, a1, (a0)
+; RV64IA-NEXT:    srlw a0, a0, a2
+; RV64IA-NEXT:    ret
   %1 = atomicrmw xor i16* %a, i16 %b acq_rel
   ret i16 %1
 }
@@ -3775,6 +7051,29 @@ define i16 @atomicrmw_xor_i16_seq_cst(i16 *%a, i16 %b) nounwind {
 ; RV32IA-NEXT:    amoxor.w.aqrl a0, a1, (a0)
 ; RV32IA-NEXT:    srl a0, a0, a2
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_xor_i16_seq_cst:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a2, zero, 5
+; RV64I-NEXT:    call __atomic_fetch_xor_2
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_xor_i16_seq_cst:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    lui a2, 16
+; RV64IA-NEXT:    addiw a2, a2, -1
+; RV64IA-NEXT:    and a1, a1, a2
+; RV64IA-NEXT:    slli a2, a0, 3
+; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    sllw a1, a1, a2
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    amoxor.w.aqrl a0, a1, (a0)
+; RV64IA-NEXT:    srlw a0, a0, a2
+; RV64IA-NEXT:    ret
   %1 = atomicrmw xor i16* %a, i16 %b seq_cst
   ret i16 %1
 }
@@ -3853,6 +7152,80 @@ define i16 @atomicrmw_max_i16_monotonic(i16 *%a, i16 %b) nounwind {
 ; RV32IA-NEXT:  # %bb.4:
 ; RV32IA-NEXT:    srl a0, a5, a2
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_max_i16_monotonic:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -48
+; RV64I-NEXT:    sd ra, 40(sp)
+; RV64I-NEXT:    sd s1, 32(sp)
+; RV64I-NEXT:    sd s2, 24(sp)
+; RV64I-NEXT:    sd s3, 16(sp)
+; RV64I-NEXT:    sd s4, 8(sp)
+; RV64I-NEXT:    mv s2, a1
+; RV64I-NEXT:    mv s4, a0
+; RV64I-NEXT:    lhu a0, 0(a0)
+; RV64I-NEXT:    slli a1, a1, 48
+; RV64I-NEXT:    srai s1, a1, 48
+; RV64I-NEXT:    addi s3, sp, 6
+; RV64I-NEXT:  .LBB90_1: # %atomicrmw.start
+; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV64I-NEXT:    slli a1, a0, 48
+; RV64I-NEXT:    srai a1, a1, 48
+; RV64I-NEXT:    mv a2, a0
+; RV64I-NEXT:    blt s1, a1, .LBB90_3
+; RV64I-NEXT:  # %bb.2: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB90_1 Depth=1
+; RV64I-NEXT:    mv a2, s2
+; RV64I-NEXT:  .LBB90_3: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB90_1 Depth=1
+; RV64I-NEXT:    sh a0, 6(sp)
+; RV64I-NEXT:    mv a0, s4
+; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    mv a3, zero
+; RV64I-NEXT:    mv a4, zero
+; RV64I-NEXT:    call __atomic_compare_exchange_2
+; RV64I-NEXT:    mv a1, a0
+; RV64I-NEXT:    lh a0, 6(sp)
+; RV64I-NEXT:    beqz a1, .LBB90_1
+; RV64I-NEXT:  # %bb.4: # %atomicrmw.end
+; RV64I-NEXT:    ld s4, 8(sp)
+; RV64I-NEXT:    ld s3, 16(sp)
+; RV64I-NEXT:    ld s2, 24(sp)
+; RV64I-NEXT:    ld s1, 32(sp)
+; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    addi sp, sp, 48
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_max_i16_monotonic:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    slli a2, a0, 3
+; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    addi a3, zero, 48
+; RV64IA-NEXT:    sub a6, a3, a2
+; RV64IA-NEXT:    lui a4, 16
+; RV64IA-NEXT:    addiw a4, a4, -1
+; RV64IA-NEXT:    sllw a7, a4, a2
+; RV64IA-NEXT:    slli a1, a1, 48
+; RV64IA-NEXT:    srai a1, a1, 48
+; RV64IA-NEXT:    sllw a1, a1, a2
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:  .LBB90_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.w a5, (a0)
+; RV64IA-NEXT:    and a4, a5, a7
+; RV64IA-NEXT:    mv a3, a5
+; RV64IA-NEXT:    sll a4, a4, a6
+; RV64IA-NEXT:    sra a4, a4, a6
+; RV64IA-NEXT:    bge a4, a1, .LBB90_3
+; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB90_1 Depth=1
+; RV64IA-NEXT:    xor a3, a5, a1
+; RV64IA-NEXT:    and a3, a3, a7
+; RV64IA-NEXT:    xor a3, a5, a3
+; RV64IA-NEXT:  .LBB90_3: # in Loop: Header=BB90_1 Depth=1
+; RV64IA-NEXT:    sc.w a3, a3, (a0)
+; RV64IA-NEXT:    bnez a3, .LBB90_1
+; RV64IA-NEXT:  # %bb.4:
+; RV64IA-NEXT:    srlw a0, a5, a2
+; RV64IA-NEXT:    ret
   %1 = atomicrmw max i16* %a, i16 %b monotonic
   ret i16 %1
 }
@@ -3866,36 +7239,33 @@ define i16 @atomicrmw_max_i16_acquire(i16 *%a, i16 %b) nounwind {
 ; RV32I-NEXT:    sw s2, 20(sp)
 ; RV32I-NEXT:    sw s3, 16(sp)
 ; RV32I-NEXT:    sw s4, 12(sp)
-; RV32I-NEXT:    sw s5, 8(sp)
 ; RV32I-NEXT:    mv s2, a1
 ; RV32I-NEXT:    mv s4, a0
 ; RV32I-NEXT:    lhu a0, 0(a0)
 ; RV32I-NEXT:    slli a1, a1, 16
-; RV32I-NEXT:    srai s5, a1, 16
-; RV32I-NEXT:    addi s3, sp, 6
-; RV32I-NEXT:    addi s1, zero, 2
+; RV32I-NEXT:    srai s1, a1, 16
+; RV32I-NEXT:    addi s3, sp, 10
 ; RV32I-NEXT:  .LBB91_1: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
 ; RV32I-NEXT:    slli a1, a0, 16
 ; RV32I-NEXT:    srai a1, a1, 16
 ; RV32I-NEXT:    mv a2, a0
-; RV32I-NEXT:    blt s5, a1, .LBB91_3
+; RV32I-NEXT:    blt s1, a1, .LBB91_3
 ; RV32I-NEXT:  # %bb.2: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB91_1 Depth=1
 ; RV32I-NEXT:    mv a2, s2
 ; RV32I-NEXT:  .LBB91_3: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB91_1 Depth=1
-; RV32I-NEXT:    sh a0, 6(sp)
+; RV32I-NEXT:    sh a0, 10(sp)
 ; RV32I-NEXT:    mv a0, s4
 ; RV32I-NEXT:    mv a1, s3
-; RV32I-NEXT:    mv a3, s1
-; RV32I-NEXT:    mv a4, s1
+; RV32I-NEXT:    addi a3, zero, 2
+; RV32I-NEXT:    addi a4, zero, 2
 ; RV32I-NEXT:    call __atomic_compare_exchange_2
 ; RV32I-NEXT:    mv a1, a0
-; RV32I-NEXT:    lh a0, 6(sp)
+; RV32I-NEXT:    lh a0, 10(sp)
 ; RV32I-NEXT:    beqz a1, .LBB91_1
 ; RV32I-NEXT:  # %bb.4: # %atomicrmw.end
-; RV32I-NEXT:    lw s5, 8(sp)
 ; RV32I-NEXT:    lw s4, 12(sp)
 ; RV32I-NEXT:    lw s3, 16(sp)
 ; RV32I-NEXT:    lw s2, 20(sp)
@@ -3934,6 +7304,80 @@ define i16 @atomicrmw_max_i16_acquire(i16 *%a, i16 %b) nounwind {
 ; RV32IA-NEXT:  # %bb.4:
 ; RV32IA-NEXT:    srl a0, a5, a2
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_max_i16_acquire:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -48
+; RV64I-NEXT:    sd ra, 40(sp)
+; RV64I-NEXT:    sd s1, 32(sp)
+; RV64I-NEXT:    sd s2, 24(sp)
+; RV64I-NEXT:    sd s3, 16(sp)
+; RV64I-NEXT:    sd s4, 8(sp)
+; RV64I-NEXT:    mv s2, a1
+; RV64I-NEXT:    mv s4, a0
+; RV64I-NEXT:    lhu a0, 0(a0)
+; RV64I-NEXT:    slli a1, a1, 48
+; RV64I-NEXT:    srai s1, a1, 48
+; RV64I-NEXT:    addi s3, sp, 6
+; RV64I-NEXT:  .LBB91_1: # %atomicrmw.start
+; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV64I-NEXT:    slli a1, a0, 48
+; RV64I-NEXT:    srai a1, a1, 48
+; RV64I-NEXT:    mv a2, a0
+; RV64I-NEXT:    blt s1, a1, .LBB91_3
+; RV64I-NEXT:  # %bb.2: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB91_1 Depth=1
+; RV64I-NEXT:    mv a2, s2
+; RV64I-NEXT:  .LBB91_3: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB91_1 Depth=1
+; RV64I-NEXT:    sh a0, 6(sp)
+; RV64I-NEXT:    mv a0, s4
+; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    addi a3, zero, 2
+; RV64I-NEXT:    addi a4, zero, 2
+; RV64I-NEXT:    call __atomic_compare_exchange_2
+; RV64I-NEXT:    mv a1, a0
+; RV64I-NEXT:    lh a0, 6(sp)
+; RV64I-NEXT:    beqz a1, .LBB91_1
+; RV64I-NEXT:  # %bb.4: # %atomicrmw.end
+; RV64I-NEXT:    ld s4, 8(sp)
+; RV64I-NEXT:    ld s3, 16(sp)
+; RV64I-NEXT:    ld s2, 24(sp)
+; RV64I-NEXT:    ld s1, 32(sp)
+; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    addi sp, sp, 48
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_max_i16_acquire:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    slli a2, a0, 3
+; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    addi a3, zero, 48
+; RV64IA-NEXT:    sub a6, a3, a2
+; RV64IA-NEXT:    lui a4, 16
+; RV64IA-NEXT:    addiw a4, a4, -1
+; RV64IA-NEXT:    sllw a7, a4, a2
+; RV64IA-NEXT:    slli a1, a1, 48
+; RV64IA-NEXT:    srai a1, a1, 48
+; RV64IA-NEXT:    sllw a1, a1, a2
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:  .LBB91_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.w.aq a5, (a0)
+; RV64IA-NEXT:    and a4, a5, a7
+; RV64IA-NEXT:    mv a3, a5
+; RV64IA-NEXT:    sll a4, a4, a6
+; RV64IA-NEXT:    sra a4, a4, a6
+; RV64IA-NEXT:    bge a4, a1, .LBB91_3
+; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB91_1 Depth=1
+; RV64IA-NEXT:    xor a3, a5, a1
+; RV64IA-NEXT:    and a3, a3, a7
+; RV64IA-NEXT:    xor a3, a5, a3
+; RV64IA-NEXT:  .LBB91_3: # in Loop: Header=BB91_1 Depth=1
+; RV64IA-NEXT:    sc.w a3, a3, (a0)
+; RV64IA-NEXT:    bnez a3, .LBB91_1
+; RV64IA-NEXT:  # %bb.4:
+; RV64IA-NEXT:    srlw a0, a5, a2
+; RV64IA-NEXT:    ret
   %1 = atomicrmw max i16* %a, i16 %b acquire
   ret i16 %1
 }
@@ -3947,14 +7391,12 @@ define i16 @atomicrmw_max_i16_release(i16 *%a, i16 %b) nounwind {
 ; RV32I-NEXT:    sw s2, 20(sp)
 ; RV32I-NEXT:    sw s3, 16(sp)
 ; RV32I-NEXT:    sw s4, 12(sp)
-; RV32I-NEXT:    sw s5, 8(sp)
 ; RV32I-NEXT:    mv s2, a1
-; RV32I-NEXT:    mv s5, a0
+; RV32I-NEXT:    mv s4, a0
 ; RV32I-NEXT:    lhu a0, 0(a0)
 ; RV32I-NEXT:    slli a1, a1, 16
 ; RV32I-NEXT:    srai s1, a1, 16
-; RV32I-NEXT:    addi s3, sp, 6
-; RV32I-NEXT:    addi s4, zero, 3
+; RV32I-NEXT:    addi s3, sp, 10
 ; RV32I-NEXT:  .LBB92_1: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
 ; RV32I-NEXT:    slli a1, a0, 16
@@ -3966,17 +7408,16 @@ define i16 @atomicrmw_max_i16_release(i16 *%a, i16 %b) nounwind {
 ; RV32I-NEXT:    mv a2, s2
 ; RV32I-NEXT:  .LBB92_3: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB92_1 Depth=1
-; RV32I-NEXT:    sh a0, 6(sp)
-; RV32I-NEXT:    mv a0, s5
+; RV32I-NEXT:    sh a0, 10(sp)
+; RV32I-NEXT:    mv a0, s4
 ; RV32I-NEXT:    mv a1, s3
-; RV32I-NEXT:    mv a3, s4
+; RV32I-NEXT:    addi a3, zero, 3
 ; RV32I-NEXT:    mv a4, zero
 ; RV32I-NEXT:    call __atomic_compare_exchange_2
 ; RV32I-NEXT:    mv a1, a0
-; RV32I-NEXT:    lh a0, 6(sp)
+; RV32I-NEXT:    lh a0, 10(sp)
 ; RV32I-NEXT:    beqz a1, .LBB92_1
 ; RV32I-NEXT:  # %bb.4: # %atomicrmw.end
-; RV32I-NEXT:    lw s5, 8(sp)
 ; RV32I-NEXT:    lw s4, 12(sp)
 ; RV32I-NEXT:    lw s3, 16(sp)
 ; RV32I-NEXT:    lw s2, 20(sp)
@@ -4015,6 +7456,80 @@ define i16 @atomicrmw_max_i16_release(i16 *%a, i16 %b) nounwind {
 ; RV32IA-NEXT:  # %bb.4:
 ; RV32IA-NEXT:    srl a0, a5, a2
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_max_i16_release:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -48
+; RV64I-NEXT:    sd ra, 40(sp)
+; RV64I-NEXT:    sd s1, 32(sp)
+; RV64I-NEXT:    sd s2, 24(sp)
+; RV64I-NEXT:    sd s3, 16(sp)
+; RV64I-NEXT:    sd s4, 8(sp)
+; RV64I-NEXT:    mv s2, a1
+; RV64I-NEXT:    mv s4, a0
+; RV64I-NEXT:    lhu a0, 0(a0)
+; RV64I-NEXT:    slli a1, a1, 48
+; RV64I-NEXT:    srai s1, a1, 48
+; RV64I-NEXT:    addi s3, sp, 6
+; RV64I-NEXT:  .LBB92_1: # %atomicrmw.start
+; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV64I-NEXT:    slli a1, a0, 48
+; RV64I-NEXT:    srai a1, a1, 48
+; RV64I-NEXT:    mv a2, a0
+; RV64I-NEXT:    blt s1, a1, .LBB92_3
+; RV64I-NEXT:  # %bb.2: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB92_1 Depth=1
+; RV64I-NEXT:    mv a2, s2
+; RV64I-NEXT:  .LBB92_3: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB92_1 Depth=1
+; RV64I-NEXT:    sh a0, 6(sp)
+; RV64I-NEXT:    mv a0, s4
+; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    addi a3, zero, 3
+; RV64I-NEXT:    mv a4, zero
+; RV64I-NEXT:    call __atomic_compare_exchange_2
+; RV64I-NEXT:    mv a1, a0
+; RV64I-NEXT:    lh a0, 6(sp)
+; RV64I-NEXT:    beqz a1, .LBB92_1
+; RV64I-NEXT:  # %bb.4: # %atomicrmw.end
+; RV64I-NEXT:    ld s4, 8(sp)
+; RV64I-NEXT:    ld s3, 16(sp)
+; RV64I-NEXT:    ld s2, 24(sp)
+; RV64I-NEXT:    ld s1, 32(sp)
+; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    addi sp, sp, 48
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_max_i16_release:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    slli a2, a0, 3
+; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    addi a3, zero, 48
+; RV64IA-NEXT:    sub a6, a3, a2
+; RV64IA-NEXT:    lui a4, 16
+; RV64IA-NEXT:    addiw a4, a4, -1
+; RV64IA-NEXT:    sllw a7, a4, a2
+; RV64IA-NEXT:    slli a1, a1, 48
+; RV64IA-NEXT:    srai a1, a1, 48
+; RV64IA-NEXT:    sllw a1, a1, a2
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:  .LBB92_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.w a5, (a0)
+; RV64IA-NEXT:    and a4, a5, a7
+; RV64IA-NEXT:    mv a3, a5
+; RV64IA-NEXT:    sll a4, a4, a6
+; RV64IA-NEXT:    sra a4, a4, a6
+; RV64IA-NEXT:    bge a4, a1, .LBB92_3
+; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB92_1 Depth=1
+; RV64IA-NEXT:    xor a3, a5, a1
+; RV64IA-NEXT:    and a3, a3, a7
+; RV64IA-NEXT:    xor a3, a5, a3
+; RV64IA-NEXT:  .LBB92_3: # in Loop: Header=BB92_1 Depth=1
+; RV64IA-NEXT:    sc.w.rl a3, a3, (a0)
+; RV64IA-NEXT:    bnez a3, .LBB92_1
+; RV64IA-NEXT:  # %bb.4:
+; RV64IA-NEXT:    srlw a0, a5, a2
+; RV64IA-NEXT:    ret
   %1 = atomicrmw max i16* %a, i16 %b release
   ret i16 %1
 }
@@ -4028,16 +7543,12 @@ define i16 @atomicrmw_max_i16_acq_rel(i16 *%a, i16 %b) nounwind {
 ; RV32I-NEXT:    sw s2, 20(sp)
 ; RV32I-NEXT:    sw s3, 16(sp)
 ; RV32I-NEXT:    sw s4, 12(sp)
-; RV32I-NEXT:    sw s5, 8(sp)
-; RV32I-NEXT:    sw s6, 4(sp)
 ; RV32I-NEXT:    mv s2, a1
-; RV32I-NEXT:    mv s6, a0
+; RV32I-NEXT:    mv s4, a0
 ; RV32I-NEXT:    lhu a0, 0(a0)
 ; RV32I-NEXT:    slli a1, a1, 16
 ; RV32I-NEXT:    srai s1, a1, 16
-; RV32I-NEXT:    addi s3, sp, 2
-; RV32I-NEXT:    addi s4, zero, 4
-; RV32I-NEXT:    addi s5, zero, 2
+; RV32I-NEXT:    addi s3, sp, 10
 ; RV32I-NEXT:  .LBB93_1: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
 ; RV32I-NEXT:    slli a1, a0, 16
@@ -4049,18 +7560,16 @@ define i16 @atomicrmw_max_i16_acq_rel(i16 *%a, i16 %b) nounwind {
 ; RV32I-NEXT:    mv a2, s2
 ; RV32I-NEXT:  .LBB93_3: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB93_1 Depth=1
-; RV32I-NEXT:    sh a0, 2(sp)
-; RV32I-NEXT:    mv a0, s6
+; RV32I-NEXT:    sh a0, 10(sp)
+; RV32I-NEXT:    mv a0, s4
 ; RV32I-NEXT:    mv a1, s3
-; RV32I-NEXT:    mv a3, s4
-; RV32I-NEXT:    mv a4, s5
+; RV32I-NEXT:    addi a3, zero, 4
+; RV32I-NEXT:    addi a4, zero, 2
 ; RV32I-NEXT:    call __atomic_compare_exchange_2
 ; RV32I-NEXT:    mv a1, a0
-; RV32I-NEXT:    lh a0, 2(sp)
+; RV32I-NEXT:    lh a0, 10(sp)
 ; RV32I-NEXT:    beqz a1, .LBB93_1
 ; RV32I-NEXT:  # %bb.4: # %atomicrmw.end
-; RV32I-NEXT:    lw s6, 4(sp)
-; RV32I-NEXT:    lw s5, 8(sp)
 ; RV32I-NEXT:    lw s4, 12(sp)
 ; RV32I-NEXT:    lw s3, 16(sp)
 ; RV32I-NEXT:    lw s2, 20(sp)
@@ -4099,6 +7608,80 @@ define i16 @atomicrmw_max_i16_acq_rel(i16 *%a, i16 %b) nounwind {
 ; RV32IA-NEXT:  # %bb.4:
 ; RV32IA-NEXT:    srl a0, a5, a2
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_max_i16_acq_rel:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -48
+; RV64I-NEXT:    sd ra, 40(sp)
+; RV64I-NEXT:    sd s1, 32(sp)
+; RV64I-NEXT:    sd s2, 24(sp)
+; RV64I-NEXT:    sd s3, 16(sp)
+; RV64I-NEXT:    sd s4, 8(sp)
+; RV64I-NEXT:    mv s2, a1
+; RV64I-NEXT:    mv s4, a0
+; RV64I-NEXT:    lhu a0, 0(a0)
+; RV64I-NEXT:    slli a1, a1, 48
+; RV64I-NEXT:    srai s1, a1, 48
+; RV64I-NEXT:    addi s3, sp, 6
+; RV64I-NEXT:  .LBB93_1: # %atomicrmw.start
+; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV64I-NEXT:    slli a1, a0, 48
+; RV64I-NEXT:    srai a1, a1, 48
+; RV64I-NEXT:    mv a2, a0
+; RV64I-NEXT:    blt s1, a1, .LBB93_3
+; RV64I-NEXT:  # %bb.2: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB93_1 Depth=1
+; RV64I-NEXT:    mv a2, s2
+; RV64I-NEXT:  .LBB93_3: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB93_1 Depth=1
+; RV64I-NEXT:    sh a0, 6(sp)
+; RV64I-NEXT:    mv a0, s4
+; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    addi a3, zero, 4
+; RV64I-NEXT:    addi a4, zero, 2
+; RV64I-NEXT:    call __atomic_compare_exchange_2
+; RV64I-NEXT:    mv a1, a0
+; RV64I-NEXT:    lh a0, 6(sp)
+; RV64I-NEXT:    beqz a1, .LBB93_1
+; RV64I-NEXT:  # %bb.4: # %atomicrmw.end
+; RV64I-NEXT:    ld s4, 8(sp)
+; RV64I-NEXT:    ld s3, 16(sp)
+; RV64I-NEXT:    ld s2, 24(sp)
+; RV64I-NEXT:    ld s1, 32(sp)
+; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    addi sp, sp, 48
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_max_i16_acq_rel:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    slli a2, a0, 3
+; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    addi a3, zero, 48
+; RV64IA-NEXT:    sub a6, a3, a2
+; RV64IA-NEXT:    lui a4, 16
+; RV64IA-NEXT:    addiw a4, a4, -1
+; RV64IA-NEXT:    sllw a7, a4, a2
+; RV64IA-NEXT:    slli a1, a1, 48
+; RV64IA-NEXT:    srai a1, a1, 48
+; RV64IA-NEXT:    sllw a1, a1, a2
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:  .LBB93_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.w.aq a5, (a0)
+; RV64IA-NEXT:    and a4, a5, a7
+; RV64IA-NEXT:    mv a3, a5
+; RV64IA-NEXT:    sll a4, a4, a6
+; RV64IA-NEXT:    sra a4, a4, a6
+; RV64IA-NEXT:    bge a4, a1, .LBB93_3
+; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB93_1 Depth=1
+; RV64IA-NEXT:    xor a3, a5, a1
+; RV64IA-NEXT:    and a3, a3, a7
+; RV64IA-NEXT:    xor a3, a5, a3
+; RV64IA-NEXT:  .LBB93_3: # in Loop: Header=BB93_1 Depth=1
+; RV64IA-NEXT:    sc.w.rl a3, a3, (a0)
+; RV64IA-NEXT:    bnez a3, .LBB93_1
+; RV64IA-NEXT:  # %bb.4:
+; RV64IA-NEXT:    srlw a0, a5, a2
+; RV64IA-NEXT:    ret
   %1 = atomicrmw max i16* %a, i16 %b acq_rel
   ret i16 %1
 }
@@ -4112,36 +7695,33 @@ define i16 @atomicrmw_max_i16_seq_cst(i16 *%a, i16 %b) nounwind {
 ; RV32I-NEXT:    sw s2, 20(sp)
 ; RV32I-NEXT:    sw s3, 16(sp)
 ; RV32I-NEXT:    sw s4, 12(sp)
-; RV32I-NEXT:    sw s5, 8(sp)
 ; RV32I-NEXT:    mv s2, a1
 ; RV32I-NEXT:    mv s4, a0
 ; RV32I-NEXT:    lhu a0, 0(a0)
 ; RV32I-NEXT:    slli a1, a1, 16
-; RV32I-NEXT:    srai s5, a1, 16
-; RV32I-NEXT:    addi s3, sp, 6
-; RV32I-NEXT:    addi s1, zero, 5
+; RV32I-NEXT:    srai s1, a1, 16
+; RV32I-NEXT:    addi s3, sp, 10
 ; RV32I-NEXT:  .LBB94_1: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
 ; RV32I-NEXT:    slli a1, a0, 16
 ; RV32I-NEXT:    srai a1, a1, 16
 ; RV32I-NEXT:    mv a2, a0
-; RV32I-NEXT:    blt s5, a1, .LBB94_3
+; RV32I-NEXT:    blt s1, a1, .LBB94_3
 ; RV32I-NEXT:  # %bb.2: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB94_1 Depth=1
 ; RV32I-NEXT:    mv a2, s2
 ; RV32I-NEXT:  .LBB94_3: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB94_1 Depth=1
-; RV32I-NEXT:    sh a0, 6(sp)
+; RV32I-NEXT:    sh a0, 10(sp)
 ; RV32I-NEXT:    mv a0, s4
 ; RV32I-NEXT:    mv a1, s3
-; RV32I-NEXT:    mv a3, s1
-; RV32I-NEXT:    mv a4, s1
+; RV32I-NEXT:    addi a3, zero, 5
+; RV32I-NEXT:    addi a4, zero, 5
 ; RV32I-NEXT:    call __atomic_compare_exchange_2
 ; RV32I-NEXT:    mv a1, a0
-; RV32I-NEXT:    lh a0, 6(sp)
+; RV32I-NEXT:    lh a0, 10(sp)
 ; RV32I-NEXT:    beqz a1, .LBB94_1
 ; RV32I-NEXT:  # %bb.4: # %atomicrmw.end
-; RV32I-NEXT:    lw s5, 8(sp)
 ; RV32I-NEXT:    lw s4, 12(sp)
 ; RV32I-NEXT:    lw s3, 16(sp)
 ; RV32I-NEXT:    lw s2, 20(sp)
@@ -4180,6 +7760,80 @@ define i16 @atomicrmw_max_i16_seq_cst(i16 *%a, i16 %b) nounwind {
 ; RV32IA-NEXT:  # %bb.4:
 ; RV32IA-NEXT:    srl a0, a5, a2
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_max_i16_seq_cst:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -48
+; RV64I-NEXT:    sd ra, 40(sp)
+; RV64I-NEXT:    sd s1, 32(sp)
+; RV64I-NEXT:    sd s2, 24(sp)
+; RV64I-NEXT:    sd s3, 16(sp)
+; RV64I-NEXT:    sd s4, 8(sp)
+; RV64I-NEXT:    mv s2, a1
+; RV64I-NEXT:    mv s4, a0
+; RV64I-NEXT:    lhu a0, 0(a0)
+; RV64I-NEXT:    slli a1, a1, 48
+; RV64I-NEXT:    srai s1, a1, 48
+; RV64I-NEXT:    addi s3, sp, 6
+; RV64I-NEXT:  .LBB94_1: # %atomicrmw.start
+; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV64I-NEXT:    slli a1, a0, 48
+; RV64I-NEXT:    srai a1, a1, 48
+; RV64I-NEXT:    mv a2, a0
+; RV64I-NEXT:    blt s1, a1, .LBB94_3
+; RV64I-NEXT:  # %bb.2: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB94_1 Depth=1
+; RV64I-NEXT:    mv a2, s2
+; RV64I-NEXT:  .LBB94_3: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB94_1 Depth=1
+; RV64I-NEXT:    sh a0, 6(sp)
+; RV64I-NEXT:    mv a0, s4
+; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    addi a3, zero, 5
+; RV64I-NEXT:    addi a4, zero, 5
+; RV64I-NEXT:    call __atomic_compare_exchange_2
+; RV64I-NEXT:    mv a1, a0
+; RV64I-NEXT:    lh a0, 6(sp)
+; RV64I-NEXT:    beqz a1, .LBB94_1
+; RV64I-NEXT:  # %bb.4: # %atomicrmw.end
+; RV64I-NEXT:    ld s4, 8(sp)
+; RV64I-NEXT:    ld s3, 16(sp)
+; RV64I-NEXT:    ld s2, 24(sp)
+; RV64I-NEXT:    ld s1, 32(sp)
+; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    addi sp, sp, 48
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_max_i16_seq_cst:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    slli a2, a0, 3
+; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    addi a3, zero, 48
+; RV64IA-NEXT:    sub a6, a3, a2
+; RV64IA-NEXT:    lui a4, 16
+; RV64IA-NEXT:    addiw a4, a4, -1
+; RV64IA-NEXT:    sllw a7, a4, a2
+; RV64IA-NEXT:    slli a1, a1, 48
+; RV64IA-NEXT:    srai a1, a1, 48
+; RV64IA-NEXT:    sllw a1, a1, a2
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:  .LBB94_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.w.aqrl a5, (a0)
+; RV64IA-NEXT:    and a4, a5, a7
+; RV64IA-NEXT:    mv a3, a5
+; RV64IA-NEXT:    sll a4, a4, a6
+; RV64IA-NEXT:    sra a4, a4, a6
+; RV64IA-NEXT:    bge a4, a1, .LBB94_3
+; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB94_1 Depth=1
+; RV64IA-NEXT:    xor a3, a5, a1
+; RV64IA-NEXT:    and a3, a3, a7
+; RV64IA-NEXT:    xor a3, a5, a3
+; RV64IA-NEXT:  .LBB94_3: # in Loop: Header=BB94_1 Depth=1
+; RV64IA-NEXT:    sc.w.aqrl a3, a3, (a0)
+; RV64IA-NEXT:    bnez a3, .LBB94_1
+; RV64IA-NEXT:  # %bb.4:
+; RV64IA-NEXT:    srlw a0, a5, a2
+; RV64IA-NEXT:    ret
   %1 = atomicrmw max i16* %a, i16 %b seq_cst
   ret i16 %1
 }
@@ -4258,6 +7912,80 @@ define i16 @atomicrmw_min_i16_monotonic(i16 *%a, i16 %b) nounwind {
 ; RV32IA-NEXT:  # %bb.4:
 ; RV32IA-NEXT:    srl a0, a5, a2
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_min_i16_monotonic:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -48
+; RV64I-NEXT:    sd ra, 40(sp)
+; RV64I-NEXT:    sd s1, 32(sp)
+; RV64I-NEXT:    sd s2, 24(sp)
+; RV64I-NEXT:    sd s3, 16(sp)
+; RV64I-NEXT:    sd s4, 8(sp)
+; RV64I-NEXT:    mv s2, a1
+; RV64I-NEXT:    mv s4, a0
+; RV64I-NEXT:    lhu a0, 0(a0)
+; RV64I-NEXT:    slli a1, a1, 48
+; RV64I-NEXT:    srai s1, a1, 48
+; RV64I-NEXT:    addi s3, sp, 6
+; RV64I-NEXT:  .LBB95_1: # %atomicrmw.start
+; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV64I-NEXT:    slli a1, a0, 48
+; RV64I-NEXT:    srai a1, a1, 48
+; RV64I-NEXT:    mv a2, a0
+; RV64I-NEXT:    bge s1, a1, .LBB95_3
+; RV64I-NEXT:  # %bb.2: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB95_1 Depth=1
+; RV64I-NEXT:    mv a2, s2
+; RV64I-NEXT:  .LBB95_3: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB95_1 Depth=1
+; RV64I-NEXT:    sh a0, 6(sp)
+; RV64I-NEXT:    mv a0, s4
+; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    mv a3, zero
+; RV64I-NEXT:    mv a4, zero
+; RV64I-NEXT:    call __atomic_compare_exchange_2
+; RV64I-NEXT:    mv a1, a0
+; RV64I-NEXT:    lh a0, 6(sp)
+; RV64I-NEXT:    beqz a1, .LBB95_1
+; RV64I-NEXT:  # %bb.4: # %atomicrmw.end
+; RV64I-NEXT:    ld s4, 8(sp)
+; RV64I-NEXT:    ld s3, 16(sp)
+; RV64I-NEXT:    ld s2, 24(sp)
+; RV64I-NEXT:    ld s1, 32(sp)
+; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    addi sp, sp, 48
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_min_i16_monotonic:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    slli a2, a0, 3
+; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    addi a3, zero, 48
+; RV64IA-NEXT:    sub a6, a3, a2
+; RV64IA-NEXT:    lui a4, 16
+; RV64IA-NEXT:    addiw a4, a4, -1
+; RV64IA-NEXT:    sllw a7, a4, a2
+; RV64IA-NEXT:    slli a1, a1, 48
+; RV64IA-NEXT:    srai a1, a1, 48
+; RV64IA-NEXT:    sllw a1, a1, a2
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:  .LBB95_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.w a5, (a0)
+; RV64IA-NEXT:    and a4, a5, a7
+; RV64IA-NEXT:    mv a3, a5
+; RV64IA-NEXT:    sll a4, a4, a6
+; RV64IA-NEXT:    sra a4, a4, a6
+; RV64IA-NEXT:    bge a1, a4, .LBB95_3
+; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB95_1 Depth=1
+; RV64IA-NEXT:    xor a3, a5, a1
+; RV64IA-NEXT:    and a3, a3, a7
+; RV64IA-NEXT:    xor a3, a5, a3
+; RV64IA-NEXT:  .LBB95_3: # in Loop: Header=BB95_1 Depth=1
+; RV64IA-NEXT:    sc.w a3, a3, (a0)
+; RV64IA-NEXT:    bnez a3, .LBB95_1
+; RV64IA-NEXT:  # %bb.4:
+; RV64IA-NEXT:    srlw a0, a5, a2
+; RV64IA-NEXT:    ret
   %1 = atomicrmw min i16* %a, i16 %b monotonic
   ret i16 %1
 }
@@ -4271,36 +7999,33 @@ define i16 @atomicrmw_min_i16_acquire(i16 *%a, i16 %b) nounwind {
 ; RV32I-NEXT:    sw s2, 20(sp)
 ; RV32I-NEXT:    sw s3, 16(sp)
 ; RV32I-NEXT:    sw s4, 12(sp)
-; RV32I-NEXT:    sw s5, 8(sp)
 ; RV32I-NEXT:    mv s2, a1
 ; RV32I-NEXT:    mv s4, a0
 ; RV32I-NEXT:    lhu a0, 0(a0)
 ; RV32I-NEXT:    slli a1, a1, 16
-; RV32I-NEXT:    srai s5, a1, 16
-; RV32I-NEXT:    addi s3, sp, 6
-; RV32I-NEXT:    addi s1, zero, 2
+; RV32I-NEXT:    srai s1, a1, 16
+; RV32I-NEXT:    addi s3, sp, 10
 ; RV32I-NEXT:  .LBB96_1: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
 ; RV32I-NEXT:    slli a1, a0, 16
 ; RV32I-NEXT:    srai a1, a1, 16
 ; RV32I-NEXT:    mv a2, a0
-; RV32I-NEXT:    bge s5, a1, .LBB96_3
+; RV32I-NEXT:    bge s1, a1, .LBB96_3
 ; RV32I-NEXT:  # %bb.2: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB96_1 Depth=1
 ; RV32I-NEXT:    mv a2, s2
 ; RV32I-NEXT:  .LBB96_3: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB96_1 Depth=1
-; RV32I-NEXT:    sh a0, 6(sp)
+; RV32I-NEXT:    sh a0, 10(sp)
 ; RV32I-NEXT:    mv a0, s4
 ; RV32I-NEXT:    mv a1, s3
-; RV32I-NEXT:    mv a3, s1
-; RV32I-NEXT:    mv a4, s1
+; RV32I-NEXT:    addi a3, zero, 2
+; RV32I-NEXT:    addi a4, zero, 2
 ; RV32I-NEXT:    call __atomic_compare_exchange_2
 ; RV32I-NEXT:    mv a1, a0
-; RV32I-NEXT:    lh a0, 6(sp)
+; RV32I-NEXT:    lh a0, 10(sp)
 ; RV32I-NEXT:    beqz a1, .LBB96_1
 ; RV32I-NEXT:  # %bb.4: # %atomicrmw.end
-; RV32I-NEXT:    lw s5, 8(sp)
 ; RV32I-NEXT:    lw s4, 12(sp)
 ; RV32I-NEXT:    lw s3, 16(sp)
 ; RV32I-NEXT:    lw s2, 20(sp)
@@ -4339,6 +8064,80 @@ define i16 @atomicrmw_min_i16_acquire(i16 *%a, i16 %b) nounwind {
 ; RV32IA-NEXT:  # %bb.4:
 ; RV32IA-NEXT:    srl a0, a5, a2
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_min_i16_acquire:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -48
+; RV64I-NEXT:    sd ra, 40(sp)
+; RV64I-NEXT:    sd s1, 32(sp)
+; RV64I-NEXT:    sd s2, 24(sp)
+; RV64I-NEXT:    sd s3, 16(sp)
+; RV64I-NEXT:    sd s4, 8(sp)
+; RV64I-NEXT:    mv s2, a1
+; RV64I-NEXT:    mv s4, a0
+; RV64I-NEXT:    lhu a0, 0(a0)
+; RV64I-NEXT:    slli a1, a1, 48
+; RV64I-NEXT:    srai s1, a1, 48
+; RV64I-NEXT:    addi s3, sp, 6
+; RV64I-NEXT:  .LBB96_1: # %atomicrmw.start
+; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV64I-NEXT:    slli a1, a0, 48
+; RV64I-NEXT:    srai a1, a1, 48
+; RV64I-NEXT:    mv a2, a0
+; RV64I-NEXT:    bge s1, a1, .LBB96_3
+; RV64I-NEXT:  # %bb.2: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB96_1 Depth=1
+; RV64I-NEXT:    mv a2, s2
+; RV64I-NEXT:  .LBB96_3: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB96_1 Depth=1
+; RV64I-NEXT:    sh a0, 6(sp)
+; RV64I-NEXT:    mv a0, s4
+; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    addi a3, zero, 2
+; RV64I-NEXT:    addi a4, zero, 2
+; RV64I-NEXT:    call __atomic_compare_exchange_2
+; RV64I-NEXT:    mv a1, a0
+; RV64I-NEXT:    lh a0, 6(sp)
+; RV64I-NEXT:    beqz a1, .LBB96_1
+; RV64I-NEXT:  # %bb.4: # %atomicrmw.end
+; RV64I-NEXT:    ld s4, 8(sp)
+; RV64I-NEXT:    ld s3, 16(sp)
+; RV64I-NEXT:    ld s2, 24(sp)
+; RV64I-NEXT:    ld s1, 32(sp)
+; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    addi sp, sp, 48
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_min_i16_acquire:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    slli a2, a0, 3
+; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    addi a3, zero, 48
+; RV64IA-NEXT:    sub a6, a3, a2
+; RV64IA-NEXT:    lui a4, 16
+; RV64IA-NEXT:    addiw a4, a4, -1
+; RV64IA-NEXT:    sllw a7, a4, a2
+; RV64IA-NEXT:    slli a1, a1, 48
+; RV64IA-NEXT:    srai a1, a1, 48
+; RV64IA-NEXT:    sllw a1, a1, a2
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:  .LBB96_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.w.aq a5, (a0)
+; RV64IA-NEXT:    and a4, a5, a7
+; RV64IA-NEXT:    mv a3, a5
+; RV64IA-NEXT:    sll a4, a4, a6
+; RV64IA-NEXT:    sra a4, a4, a6
+; RV64IA-NEXT:    bge a1, a4, .LBB96_3
+; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB96_1 Depth=1
+; RV64IA-NEXT:    xor a3, a5, a1
+; RV64IA-NEXT:    and a3, a3, a7
+; RV64IA-NEXT:    xor a3, a5, a3
+; RV64IA-NEXT:  .LBB96_3: # in Loop: Header=BB96_1 Depth=1
+; RV64IA-NEXT:    sc.w a3, a3, (a0)
+; RV64IA-NEXT:    bnez a3, .LBB96_1
+; RV64IA-NEXT:  # %bb.4:
+; RV64IA-NEXT:    srlw a0, a5, a2
+; RV64IA-NEXT:    ret
   %1 = atomicrmw min i16* %a, i16 %b acquire
   ret i16 %1
 }
@@ -4352,14 +8151,12 @@ define i16 @atomicrmw_min_i16_release(i16 *%a, i16 %b) nounwind {
 ; RV32I-NEXT:    sw s2, 20(sp)
 ; RV32I-NEXT:    sw s3, 16(sp)
 ; RV32I-NEXT:    sw s4, 12(sp)
-; RV32I-NEXT:    sw s5, 8(sp)
 ; RV32I-NEXT:    mv s2, a1
-; RV32I-NEXT:    mv s5, a0
+; RV32I-NEXT:    mv s4, a0
 ; RV32I-NEXT:    lhu a0, 0(a0)
 ; RV32I-NEXT:    slli a1, a1, 16
 ; RV32I-NEXT:    srai s1, a1, 16
-; RV32I-NEXT:    addi s3, sp, 6
-; RV32I-NEXT:    addi s4, zero, 3
+; RV32I-NEXT:    addi s3, sp, 10
 ; RV32I-NEXT:  .LBB97_1: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
 ; RV32I-NEXT:    slli a1, a0, 16
@@ -4371,17 +8168,16 @@ define i16 @atomicrmw_min_i16_release(i16 *%a, i16 %b) nounwind {
 ; RV32I-NEXT:    mv a2, s2
 ; RV32I-NEXT:  .LBB97_3: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB97_1 Depth=1
-; RV32I-NEXT:    sh a0, 6(sp)
-; RV32I-NEXT:    mv a0, s5
+; RV32I-NEXT:    sh a0, 10(sp)
+; RV32I-NEXT:    mv a0, s4
 ; RV32I-NEXT:    mv a1, s3
-; RV32I-NEXT:    mv a3, s4
+; RV32I-NEXT:    addi a3, zero, 3
 ; RV32I-NEXT:    mv a4, zero
 ; RV32I-NEXT:    call __atomic_compare_exchange_2
 ; RV32I-NEXT:    mv a1, a0
-; RV32I-NEXT:    lh a0, 6(sp)
+; RV32I-NEXT:    lh a0, 10(sp)
 ; RV32I-NEXT:    beqz a1, .LBB97_1
 ; RV32I-NEXT:  # %bb.4: # %atomicrmw.end
-; RV32I-NEXT:    lw s5, 8(sp)
 ; RV32I-NEXT:    lw s4, 12(sp)
 ; RV32I-NEXT:    lw s3, 16(sp)
 ; RV32I-NEXT:    lw s2, 20(sp)
@@ -4420,6 +8216,80 @@ define i16 @atomicrmw_min_i16_release(i16 *%a, i16 %b) nounwind {
 ; RV32IA-NEXT:  # %bb.4:
 ; RV32IA-NEXT:    srl a0, a5, a2
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_min_i16_release:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -48
+; RV64I-NEXT:    sd ra, 40(sp)
+; RV64I-NEXT:    sd s1, 32(sp)
+; RV64I-NEXT:    sd s2, 24(sp)
+; RV64I-NEXT:    sd s3, 16(sp)
+; RV64I-NEXT:    sd s4, 8(sp)
+; RV64I-NEXT:    mv s2, a1
+; RV64I-NEXT:    mv s4, a0
+; RV64I-NEXT:    lhu a0, 0(a0)
+; RV64I-NEXT:    slli a1, a1, 48
+; RV64I-NEXT:    srai s1, a1, 48
+; RV64I-NEXT:    addi s3, sp, 6
+; RV64I-NEXT:  .LBB97_1: # %atomicrmw.start
+; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV64I-NEXT:    slli a1, a0, 48
+; RV64I-NEXT:    srai a1, a1, 48
+; RV64I-NEXT:    mv a2, a0
+; RV64I-NEXT:    bge s1, a1, .LBB97_3
+; RV64I-NEXT:  # %bb.2: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB97_1 Depth=1
+; RV64I-NEXT:    mv a2, s2
+; RV64I-NEXT:  .LBB97_3: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB97_1 Depth=1
+; RV64I-NEXT:    sh a0, 6(sp)
+; RV64I-NEXT:    mv a0, s4
+; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    addi a3, zero, 3
+; RV64I-NEXT:    mv a4, zero
+; RV64I-NEXT:    call __atomic_compare_exchange_2
+; RV64I-NEXT:    mv a1, a0
+; RV64I-NEXT:    lh a0, 6(sp)
+; RV64I-NEXT:    beqz a1, .LBB97_1
+; RV64I-NEXT:  # %bb.4: # %atomicrmw.end
+; RV64I-NEXT:    ld s4, 8(sp)
+; RV64I-NEXT:    ld s3, 16(sp)
+; RV64I-NEXT:    ld s2, 24(sp)
+; RV64I-NEXT:    ld s1, 32(sp)
+; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    addi sp, sp, 48
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_min_i16_release:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    slli a2, a0, 3
+; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    addi a3, zero, 48
+; RV64IA-NEXT:    sub a6, a3, a2
+; RV64IA-NEXT:    lui a4, 16
+; RV64IA-NEXT:    addiw a4, a4, -1
+; RV64IA-NEXT:    sllw a7, a4, a2
+; RV64IA-NEXT:    slli a1, a1, 48
+; RV64IA-NEXT:    srai a1, a1, 48
+; RV64IA-NEXT:    sllw a1, a1, a2
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:  .LBB97_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.w a5, (a0)
+; RV64IA-NEXT:    and a4, a5, a7
+; RV64IA-NEXT:    mv a3, a5
+; RV64IA-NEXT:    sll a4, a4, a6
+; RV64IA-NEXT:    sra a4, a4, a6
+; RV64IA-NEXT:    bge a1, a4, .LBB97_3
+; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB97_1 Depth=1
+; RV64IA-NEXT:    xor a3, a5, a1
+; RV64IA-NEXT:    and a3, a3, a7
+; RV64IA-NEXT:    xor a3, a5, a3
+; RV64IA-NEXT:  .LBB97_3: # in Loop: Header=BB97_1 Depth=1
+; RV64IA-NEXT:    sc.w.rl a3, a3, (a0)
+; RV64IA-NEXT:    bnez a3, .LBB97_1
+; RV64IA-NEXT:  # %bb.4:
+; RV64IA-NEXT:    srlw a0, a5, a2
+; RV64IA-NEXT:    ret
   %1 = atomicrmw min i16* %a, i16 %b release
   ret i16 %1
 }
@@ -4433,16 +8303,12 @@ define i16 @atomicrmw_min_i16_acq_rel(i16 *%a, i16 %b) nounwind {
 ; RV32I-NEXT:    sw s2, 20(sp)
 ; RV32I-NEXT:    sw s3, 16(sp)
 ; RV32I-NEXT:    sw s4, 12(sp)
-; RV32I-NEXT:    sw s5, 8(sp)
-; RV32I-NEXT:    sw s6, 4(sp)
 ; RV32I-NEXT:    mv s2, a1
-; RV32I-NEXT:    mv s6, a0
+; RV32I-NEXT:    mv s4, a0
 ; RV32I-NEXT:    lhu a0, 0(a0)
 ; RV32I-NEXT:    slli a1, a1, 16
 ; RV32I-NEXT:    srai s1, a1, 16
-; RV32I-NEXT:    addi s3, sp, 2
-; RV32I-NEXT:    addi s4, zero, 4
-; RV32I-NEXT:    addi s5, zero, 2
+; RV32I-NEXT:    addi s3, sp, 10
 ; RV32I-NEXT:  .LBB98_1: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
 ; RV32I-NEXT:    slli a1, a0, 16
@@ -4454,18 +8320,16 @@ define i16 @atomicrmw_min_i16_acq_rel(i16 *%a, i16 %b) nounwind {
 ; RV32I-NEXT:    mv a2, s2
 ; RV32I-NEXT:  .LBB98_3: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB98_1 Depth=1
-; RV32I-NEXT:    sh a0, 2(sp)
-; RV32I-NEXT:    mv a0, s6
+; RV32I-NEXT:    sh a0, 10(sp)
+; RV32I-NEXT:    mv a0, s4
 ; RV32I-NEXT:    mv a1, s3
-; RV32I-NEXT:    mv a3, s4
-; RV32I-NEXT:    mv a4, s5
+; RV32I-NEXT:    addi a3, zero, 4
+; RV32I-NEXT:    addi a4, zero, 2
 ; RV32I-NEXT:    call __atomic_compare_exchange_2
 ; RV32I-NEXT:    mv a1, a0
-; RV32I-NEXT:    lh a0, 2(sp)
+; RV32I-NEXT:    lh a0, 10(sp)
 ; RV32I-NEXT:    beqz a1, .LBB98_1
 ; RV32I-NEXT:  # %bb.4: # %atomicrmw.end
-; RV32I-NEXT:    lw s6, 4(sp)
-; RV32I-NEXT:    lw s5, 8(sp)
 ; RV32I-NEXT:    lw s4, 12(sp)
 ; RV32I-NEXT:    lw s3, 16(sp)
 ; RV32I-NEXT:    lw s2, 20(sp)
@@ -4504,6 +8368,80 @@ define i16 @atomicrmw_min_i16_acq_rel(i16 *%a, i16 %b) nounwind {
 ; RV32IA-NEXT:  # %bb.4:
 ; RV32IA-NEXT:    srl a0, a5, a2
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_min_i16_acq_rel:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -48
+; RV64I-NEXT:    sd ra, 40(sp)
+; RV64I-NEXT:    sd s1, 32(sp)
+; RV64I-NEXT:    sd s2, 24(sp)
+; RV64I-NEXT:    sd s3, 16(sp)
+; RV64I-NEXT:    sd s4, 8(sp)
+; RV64I-NEXT:    mv s2, a1
+; RV64I-NEXT:    mv s4, a0
+; RV64I-NEXT:    lhu a0, 0(a0)
+; RV64I-NEXT:    slli a1, a1, 48
+; RV64I-NEXT:    srai s1, a1, 48
+; RV64I-NEXT:    addi s3, sp, 6
+; RV64I-NEXT:  .LBB98_1: # %atomicrmw.start
+; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV64I-NEXT:    slli a1, a0, 48
+; RV64I-NEXT:    srai a1, a1, 48
+; RV64I-NEXT:    mv a2, a0
+; RV64I-NEXT:    bge s1, a1, .LBB98_3
+; RV64I-NEXT:  # %bb.2: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB98_1 Depth=1
+; RV64I-NEXT:    mv a2, s2
+; RV64I-NEXT:  .LBB98_3: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB98_1 Depth=1
+; RV64I-NEXT:    sh a0, 6(sp)
+; RV64I-NEXT:    mv a0, s4
+; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    addi a3, zero, 4
+; RV64I-NEXT:    addi a4, zero, 2
+; RV64I-NEXT:    call __atomic_compare_exchange_2
+; RV64I-NEXT:    mv a1, a0
+; RV64I-NEXT:    lh a0, 6(sp)
+; RV64I-NEXT:    beqz a1, .LBB98_1
+; RV64I-NEXT:  # %bb.4: # %atomicrmw.end
+; RV64I-NEXT:    ld s4, 8(sp)
+; RV64I-NEXT:    ld s3, 16(sp)
+; RV64I-NEXT:    ld s2, 24(sp)
+; RV64I-NEXT:    ld s1, 32(sp)
+; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    addi sp, sp, 48
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_min_i16_acq_rel:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    slli a2, a0, 3
+; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    addi a3, zero, 48
+; RV64IA-NEXT:    sub a6, a3, a2
+; RV64IA-NEXT:    lui a4, 16
+; RV64IA-NEXT:    addiw a4, a4, -1
+; RV64IA-NEXT:    sllw a7, a4, a2
+; RV64IA-NEXT:    slli a1, a1, 48
+; RV64IA-NEXT:    srai a1, a1, 48
+; RV64IA-NEXT:    sllw a1, a1, a2
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:  .LBB98_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.w.aq a5, (a0)
+; RV64IA-NEXT:    and a4, a5, a7
+; RV64IA-NEXT:    mv a3, a5
+; RV64IA-NEXT:    sll a4, a4, a6
+; RV64IA-NEXT:    sra a4, a4, a6
+; RV64IA-NEXT:    bge a1, a4, .LBB98_3
+; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB98_1 Depth=1
+; RV64IA-NEXT:    xor a3, a5, a1
+; RV64IA-NEXT:    and a3, a3, a7
+; RV64IA-NEXT:    xor a3, a5, a3
+; RV64IA-NEXT:  .LBB98_3: # in Loop: Header=BB98_1 Depth=1
+; RV64IA-NEXT:    sc.w.rl a3, a3, (a0)
+; RV64IA-NEXT:    bnez a3, .LBB98_1
+; RV64IA-NEXT:  # %bb.4:
+; RV64IA-NEXT:    srlw a0, a5, a2
+; RV64IA-NEXT:    ret
   %1 = atomicrmw min i16* %a, i16 %b acq_rel
   ret i16 %1
 }
@@ -4517,36 +8455,33 @@ define i16 @atomicrmw_min_i16_seq_cst(i16 *%a, i16 %b) nounwind {
 ; RV32I-NEXT:    sw s2, 20(sp)
 ; RV32I-NEXT:    sw s3, 16(sp)
 ; RV32I-NEXT:    sw s4, 12(sp)
-; RV32I-NEXT:    sw s5, 8(sp)
 ; RV32I-NEXT:    mv s2, a1
 ; RV32I-NEXT:    mv s4, a0
 ; RV32I-NEXT:    lhu a0, 0(a0)
 ; RV32I-NEXT:    slli a1, a1, 16
-; RV32I-NEXT:    srai s5, a1, 16
-; RV32I-NEXT:    addi s3, sp, 6
-; RV32I-NEXT:    addi s1, zero, 5
+; RV32I-NEXT:    srai s1, a1, 16
+; RV32I-NEXT:    addi s3, sp, 10
 ; RV32I-NEXT:  .LBB99_1: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
 ; RV32I-NEXT:    slli a1, a0, 16
 ; RV32I-NEXT:    srai a1, a1, 16
 ; RV32I-NEXT:    mv a2, a0
-; RV32I-NEXT:    bge s5, a1, .LBB99_3
+; RV32I-NEXT:    bge s1, a1, .LBB99_3
 ; RV32I-NEXT:  # %bb.2: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB99_1 Depth=1
 ; RV32I-NEXT:    mv a2, s2
 ; RV32I-NEXT:  .LBB99_3: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB99_1 Depth=1
-; RV32I-NEXT:    sh a0, 6(sp)
+; RV32I-NEXT:    sh a0, 10(sp)
 ; RV32I-NEXT:    mv a0, s4
 ; RV32I-NEXT:    mv a1, s3
-; RV32I-NEXT:    mv a3, s1
-; RV32I-NEXT:    mv a4, s1
+; RV32I-NEXT:    addi a3, zero, 5
+; RV32I-NEXT:    addi a4, zero, 5
 ; RV32I-NEXT:    call __atomic_compare_exchange_2
 ; RV32I-NEXT:    mv a1, a0
-; RV32I-NEXT:    lh a0, 6(sp)
+; RV32I-NEXT:    lh a0, 10(sp)
 ; RV32I-NEXT:    beqz a1, .LBB99_1
 ; RV32I-NEXT:  # %bb.4: # %atomicrmw.end
-; RV32I-NEXT:    lw s5, 8(sp)
 ; RV32I-NEXT:    lw s4, 12(sp)
 ; RV32I-NEXT:    lw s3, 16(sp)
 ; RV32I-NEXT:    lw s2, 20(sp)
@@ -4585,6 +8520,80 @@ define i16 @atomicrmw_min_i16_seq_cst(i16 *%a, i16 %b) nounwind {
 ; RV32IA-NEXT:  # %bb.4:
 ; RV32IA-NEXT:    srl a0, a5, a2
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_min_i16_seq_cst:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -48
+; RV64I-NEXT:    sd ra, 40(sp)
+; RV64I-NEXT:    sd s1, 32(sp)
+; RV64I-NEXT:    sd s2, 24(sp)
+; RV64I-NEXT:    sd s3, 16(sp)
+; RV64I-NEXT:    sd s4, 8(sp)
+; RV64I-NEXT:    mv s2, a1
+; RV64I-NEXT:    mv s4, a0
+; RV64I-NEXT:    lhu a0, 0(a0)
+; RV64I-NEXT:    slli a1, a1, 48
+; RV64I-NEXT:    srai s1, a1, 48
+; RV64I-NEXT:    addi s3, sp, 6
+; RV64I-NEXT:  .LBB99_1: # %atomicrmw.start
+; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV64I-NEXT:    slli a1, a0, 48
+; RV64I-NEXT:    srai a1, a1, 48
+; RV64I-NEXT:    mv a2, a0
+; RV64I-NEXT:    bge s1, a1, .LBB99_3
+; RV64I-NEXT:  # %bb.2: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB99_1 Depth=1
+; RV64I-NEXT:    mv a2, s2
+; RV64I-NEXT:  .LBB99_3: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB99_1 Depth=1
+; RV64I-NEXT:    sh a0, 6(sp)
+; RV64I-NEXT:    mv a0, s4
+; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    addi a3, zero, 5
+; RV64I-NEXT:    addi a4, zero, 5
+; RV64I-NEXT:    call __atomic_compare_exchange_2
+; RV64I-NEXT:    mv a1, a0
+; RV64I-NEXT:    lh a0, 6(sp)
+; RV64I-NEXT:    beqz a1, .LBB99_1
+; RV64I-NEXT:  # %bb.4: # %atomicrmw.end
+; RV64I-NEXT:    ld s4, 8(sp)
+; RV64I-NEXT:    ld s3, 16(sp)
+; RV64I-NEXT:    ld s2, 24(sp)
+; RV64I-NEXT:    ld s1, 32(sp)
+; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    addi sp, sp, 48
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_min_i16_seq_cst:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    slli a2, a0, 3
+; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    addi a3, zero, 48
+; RV64IA-NEXT:    sub a6, a3, a2
+; RV64IA-NEXT:    lui a4, 16
+; RV64IA-NEXT:    addiw a4, a4, -1
+; RV64IA-NEXT:    sllw a7, a4, a2
+; RV64IA-NEXT:    slli a1, a1, 48
+; RV64IA-NEXT:    srai a1, a1, 48
+; RV64IA-NEXT:    sllw a1, a1, a2
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:  .LBB99_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.w.aqrl a5, (a0)
+; RV64IA-NEXT:    and a4, a5, a7
+; RV64IA-NEXT:    mv a3, a5
+; RV64IA-NEXT:    sll a4, a4, a6
+; RV64IA-NEXT:    sra a4, a4, a6
+; RV64IA-NEXT:    bge a1, a4, .LBB99_3
+; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB99_1 Depth=1
+; RV64IA-NEXT:    xor a3, a5, a1
+; RV64IA-NEXT:    and a3, a3, a7
+; RV64IA-NEXT:    xor a3, a5, a3
+; RV64IA-NEXT:  .LBB99_3: # in Loop: Header=BB99_1 Depth=1
+; RV64IA-NEXT:    sc.w.aqrl a3, a3, (a0)
+; RV64IA-NEXT:    bnez a3, .LBB99_1
+; RV64IA-NEXT:  # %bb.4:
+; RV64IA-NEXT:    srlw a0, a5, a2
+; RV64IA-NEXT:    ret
   %1 = atomicrmw min i16* %a, i16 %b seq_cst
   ret i16 %1
 }
@@ -4660,6 +8669,77 @@ define i16 @atomicrmw_umax_i16_monotonic(i16 *%a, i16 %b) nounwind {
 ; RV32IA-NEXT:  # %bb.4:
 ; RV32IA-NEXT:    srl a0, a4, a3
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_umax_i16_monotonic:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -64
+; RV64I-NEXT:    sd ra, 56(sp)
+; RV64I-NEXT:    sd s1, 48(sp)
+; RV64I-NEXT:    sd s2, 40(sp)
+; RV64I-NEXT:    sd s3, 32(sp)
+; RV64I-NEXT:    sd s4, 24(sp)
+; RV64I-NEXT:    sd s5, 16(sp)
+; RV64I-NEXT:    mv s2, a1
+; RV64I-NEXT:    mv s4, a0
+; RV64I-NEXT:    lhu a0, 0(a0)
+; RV64I-NEXT:    lui a1, 16
+; RV64I-NEXT:    addiw s1, a1, -1
+; RV64I-NEXT:    and s5, s2, s1
+; RV64I-NEXT:    addi s3, sp, 14
+; RV64I-NEXT:  .LBB100_1: # %atomicrmw.start
+; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV64I-NEXT:    and a1, a0, s1
+; RV64I-NEXT:    mv a2, a0
+; RV64I-NEXT:    bltu s5, a1, .LBB100_3
+; RV64I-NEXT:  # %bb.2: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB100_1 Depth=1
+; RV64I-NEXT:    mv a2, s2
+; RV64I-NEXT:  .LBB100_3: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB100_1 Depth=1
+; RV64I-NEXT:    sh a0, 14(sp)
+; RV64I-NEXT:    mv a0, s4
+; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    mv a3, zero
+; RV64I-NEXT:    mv a4, zero
+; RV64I-NEXT:    call __atomic_compare_exchange_2
+; RV64I-NEXT:    mv a1, a0
+; RV64I-NEXT:    lh a0, 14(sp)
+; RV64I-NEXT:    beqz a1, .LBB100_1
+; RV64I-NEXT:  # %bb.4: # %atomicrmw.end
+; RV64I-NEXT:    ld s5, 16(sp)
+; RV64I-NEXT:    ld s4, 24(sp)
+; RV64I-NEXT:    ld s3, 32(sp)
+; RV64I-NEXT:    ld s2, 40(sp)
+; RV64I-NEXT:    ld s1, 48(sp)
+; RV64I-NEXT:    ld ra, 56(sp)
+; RV64I-NEXT:    addi sp, sp, 64
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_umax_i16_monotonic:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    lui a2, 16
+; RV64IA-NEXT:    addiw a2, a2, -1
+; RV64IA-NEXT:    and a1, a1, a2
+; RV64IA-NEXT:    slli a3, a0, 3
+; RV64IA-NEXT:    andi a3, a3, 24
+; RV64IA-NEXT:    sllw a6, a2, a3
+; RV64IA-NEXT:    sllw a1, a1, a3
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:  .LBB100_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.w a4, (a0)
+; RV64IA-NEXT:    and a2, a4, a6
+; RV64IA-NEXT:    mv a5, a4
+; RV64IA-NEXT:    bgeu a2, a1, .LBB100_3
+; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB100_1 Depth=1
+; RV64IA-NEXT:    xor a5, a4, a1
+; RV64IA-NEXT:    and a5, a5, a6
+; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:  .LBB100_3: # in Loop: Header=BB100_1 Depth=1
+; RV64IA-NEXT:    sc.w a5, a5, (a0)
+; RV64IA-NEXT:    bnez a5, .LBB100_1
+; RV64IA-NEXT:  # %bb.4:
+; RV64IA-NEXT:    srlw a0, a4, a3
+; RV64IA-NEXT:    ret
   %1 = atomicrmw umax i16* %a, i16 %b monotonic
   ret i16 %1
 }
@@ -4674,36 +8754,33 @@ define i16 @atomicrmw_umax_i16_acquire(i16 *%a, i16 %b) nounwind {
 ; RV32I-NEXT:    sw s3, 16(sp)
 ; RV32I-NEXT:    sw s4, 12(sp)
 ; RV32I-NEXT:    sw s5, 8(sp)
-; RV32I-NEXT:    sw s6, 4(sp)
 ; RV32I-NEXT:    mv s2, a1
 ; RV32I-NEXT:    mv s4, a0
 ; RV32I-NEXT:    lhu a0, 0(a0)
 ; RV32I-NEXT:    lui a1, 16
-; RV32I-NEXT:    addi s5, a1, -1
-; RV32I-NEXT:    and s6, s2, s5
-; RV32I-NEXT:    addi s3, sp, 2
-; RV32I-NEXT:    addi s1, zero, 2
+; RV32I-NEXT:    addi s1, a1, -1
+; RV32I-NEXT:    and s5, s2, s1
+; RV32I-NEXT:    addi s3, sp, 6
 ; RV32I-NEXT:  .LBB101_1: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    and a1, a0, s5
+; RV32I-NEXT:    and a1, a0, s1
 ; RV32I-NEXT:    mv a2, a0
-; RV32I-NEXT:    bltu s6, a1, .LBB101_3
+; RV32I-NEXT:    bltu s5, a1, .LBB101_3
 ; RV32I-NEXT:  # %bb.2: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB101_1 Depth=1
 ; RV32I-NEXT:    mv a2, s2
 ; RV32I-NEXT:  .LBB101_3: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB101_1 Depth=1
-; RV32I-NEXT:    sh a0, 2(sp)
+; RV32I-NEXT:    sh a0, 6(sp)
 ; RV32I-NEXT:    mv a0, s4
 ; RV32I-NEXT:    mv a1, s3
-; RV32I-NEXT:    mv a3, s1
-; RV32I-NEXT:    mv a4, s1
+; RV32I-NEXT:    addi a3, zero, 2
+; RV32I-NEXT:    addi a4, zero, 2
 ; RV32I-NEXT:    call __atomic_compare_exchange_2
 ; RV32I-NEXT:    mv a1, a0
-; RV32I-NEXT:    lh a0, 2(sp)
+; RV32I-NEXT:    lh a0, 6(sp)
 ; RV32I-NEXT:    beqz a1, .LBB101_1
 ; RV32I-NEXT:  # %bb.4: # %atomicrmw.end
-; RV32I-NEXT:    lw s6, 4(sp)
 ; RV32I-NEXT:    lw s5, 8(sp)
 ; RV32I-NEXT:    lw s4, 12(sp)
 ; RV32I-NEXT:    lw s3, 16(sp)
@@ -4738,6 +8815,77 @@ define i16 @atomicrmw_umax_i16_acquire(i16 *%a, i16 %b) nounwind {
 ; RV32IA-NEXT:  # %bb.4:
 ; RV32IA-NEXT:    srl a0, a4, a3
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_umax_i16_acquire:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -64
+; RV64I-NEXT:    sd ra, 56(sp)
+; RV64I-NEXT:    sd s1, 48(sp)
+; RV64I-NEXT:    sd s2, 40(sp)
+; RV64I-NEXT:    sd s3, 32(sp)
+; RV64I-NEXT:    sd s4, 24(sp)
+; RV64I-NEXT:    sd s5, 16(sp)
+; RV64I-NEXT:    mv s2, a1
+; RV64I-NEXT:    mv s4, a0
+; RV64I-NEXT:    lhu a0, 0(a0)
+; RV64I-NEXT:    lui a1, 16
+; RV64I-NEXT:    addiw s1, a1, -1
+; RV64I-NEXT:    and s5, s2, s1
+; RV64I-NEXT:    addi s3, sp, 14
+; RV64I-NEXT:  .LBB101_1: # %atomicrmw.start
+; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV64I-NEXT:    and a1, a0, s1
+; RV64I-NEXT:    mv a2, a0
+; RV64I-NEXT:    bltu s5, a1, .LBB101_3
+; RV64I-NEXT:  # %bb.2: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB101_1 Depth=1
+; RV64I-NEXT:    mv a2, s2
+; RV64I-NEXT:  .LBB101_3: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB101_1 Depth=1
+; RV64I-NEXT:    sh a0, 14(sp)
+; RV64I-NEXT:    mv a0, s4
+; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    addi a3, zero, 2
+; RV64I-NEXT:    addi a4, zero, 2
+; RV64I-NEXT:    call __atomic_compare_exchange_2
+; RV64I-NEXT:    mv a1, a0
+; RV64I-NEXT:    lh a0, 14(sp)
+; RV64I-NEXT:    beqz a1, .LBB101_1
+; RV64I-NEXT:  # %bb.4: # %atomicrmw.end
+; RV64I-NEXT:    ld s5, 16(sp)
+; RV64I-NEXT:    ld s4, 24(sp)
+; RV64I-NEXT:    ld s3, 32(sp)
+; RV64I-NEXT:    ld s2, 40(sp)
+; RV64I-NEXT:    ld s1, 48(sp)
+; RV64I-NEXT:    ld ra, 56(sp)
+; RV64I-NEXT:    addi sp, sp, 64
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_umax_i16_acquire:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    lui a2, 16
+; RV64IA-NEXT:    addiw a2, a2, -1
+; RV64IA-NEXT:    and a1, a1, a2
+; RV64IA-NEXT:    slli a3, a0, 3
+; RV64IA-NEXT:    andi a3, a3, 24
+; RV64IA-NEXT:    sllw a6, a2, a3
+; RV64IA-NEXT:    sllw a1, a1, a3
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:  .LBB101_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.w.aq a4, (a0)
+; RV64IA-NEXT:    and a2, a4, a6
+; RV64IA-NEXT:    mv a5, a4
+; RV64IA-NEXT:    bgeu a2, a1, .LBB101_3
+; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB101_1 Depth=1
+; RV64IA-NEXT:    xor a5, a4, a1
+; RV64IA-NEXT:    and a5, a5, a6
+; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:  .LBB101_3: # in Loop: Header=BB101_1 Depth=1
+; RV64IA-NEXT:    sc.w a5, a5, (a0)
+; RV64IA-NEXT:    bnez a5, .LBB101_1
+; RV64IA-NEXT:  # %bb.4:
+; RV64IA-NEXT:    srlw a0, a4, a3
+; RV64IA-NEXT:    ret
   %1 = atomicrmw umax i16* %a, i16 %b acquire
   ret i16 %1
 }
@@ -4752,36 +8900,33 @@ define i16 @atomicrmw_umax_i16_release(i16 *%a, i16 %b) nounwind {
 ; RV32I-NEXT:    sw s3, 16(sp)
 ; RV32I-NEXT:    sw s4, 12(sp)
 ; RV32I-NEXT:    sw s5, 8(sp)
-; RV32I-NEXT:    sw s6, 4(sp)
 ; RV32I-NEXT:    mv s2, a1
-; RV32I-NEXT:    mv s5, a0
+; RV32I-NEXT:    mv s4, a0
 ; RV32I-NEXT:    lhu a0, 0(a0)
 ; RV32I-NEXT:    lui a1, 16
 ; RV32I-NEXT:    addi s1, a1, -1
-; RV32I-NEXT:    and s6, s2, s1
-; RV32I-NEXT:    addi s3, sp, 2
-; RV32I-NEXT:    addi s4, zero, 3
+; RV32I-NEXT:    and s5, s2, s1
+; RV32I-NEXT:    addi s3, sp, 6
 ; RV32I-NEXT:  .LBB102_1: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
 ; RV32I-NEXT:    and a1, a0, s1
 ; RV32I-NEXT:    mv a2, a0
-; RV32I-NEXT:    bltu s6, a1, .LBB102_3
+; RV32I-NEXT:    bltu s5, a1, .LBB102_3
 ; RV32I-NEXT:  # %bb.2: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB102_1 Depth=1
 ; RV32I-NEXT:    mv a2, s2
 ; RV32I-NEXT:  .LBB102_3: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB102_1 Depth=1
-; RV32I-NEXT:    sh a0, 2(sp)
-; RV32I-NEXT:    mv a0, s5
+; RV32I-NEXT:    sh a0, 6(sp)
+; RV32I-NEXT:    mv a0, s4
 ; RV32I-NEXT:    mv a1, s3
-; RV32I-NEXT:    mv a3, s4
+; RV32I-NEXT:    addi a3, zero, 3
 ; RV32I-NEXT:    mv a4, zero
 ; RV32I-NEXT:    call __atomic_compare_exchange_2
 ; RV32I-NEXT:    mv a1, a0
-; RV32I-NEXT:    lh a0, 2(sp)
+; RV32I-NEXT:    lh a0, 6(sp)
 ; RV32I-NEXT:    beqz a1, .LBB102_1
 ; RV32I-NEXT:  # %bb.4: # %atomicrmw.end
-; RV32I-NEXT:    lw s6, 4(sp)
 ; RV32I-NEXT:    lw s5, 8(sp)
 ; RV32I-NEXT:    lw s4, 12(sp)
 ; RV32I-NEXT:    lw s3, 16(sp)
@@ -4816,6 +8961,77 @@ define i16 @atomicrmw_umax_i16_release(i16 *%a, i16 %b) nounwind {
 ; RV32IA-NEXT:  # %bb.4:
 ; RV32IA-NEXT:    srl a0, a4, a3
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_umax_i16_release:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -64
+; RV64I-NEXT:    sd ra, 56(sp)
+; RV64I-NEXT:    sd s1, 48(sp)
+; RV64I-NEXT:    sd s2, 40(sp)
+; RV64I-NEXT:    sd s3, 32(sp)
+; RV64I-NEXT:    sd s4, 24(sp)
+; RV64I-NEXT:    sd s5, 16(sp)
+; RV64I-NEXT:    mv s2, a1
+; RV64I-NEXT:    mv s4, a0
+; RV64I-NEXT:    lhu a0, 0(a0)
+; RV64I-NEXT:    lui a1, 16
+; RV64I-NEXT:    addiw s1, a1, -1
+; RV64I-NEXT:    and s5, s2, s1
+; RV64I-NEXT:    addi s3, sp, 14
+; RV64I-NEXT:  .LBB102_1: # %atomicrmw.start
+; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV64I-NEXT:    and a1, a0, s1
+; RV64I-NEXT:    mv a2, a0
+; RV64I-NEXT:    bltu s5, a1, .LBB102_3
+; RV64I-NEXT:  # %bb.2: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB102_1 Depth=1
+; RV64I-NEXT:    mv a2, s2
+; RV64I-NEXT:  .LBB102_3: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB102_1 Depth=1
+; RV64I-NEXT:    sh a0, 14(sp)
+; RV64I-NEXT:    mv a0, s4
+; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    addi a3, zero, 3
+; RV64I-NEXT:    mv a4, zero
+; RV64I-NEXT:    call __atomic_compare_exchange_2
+; RV64I-NEXT:    mv a1, a0
+; RV64I-NEXT:    lh a0, 14(sp)
+; RV64I-NEXT:    beqz a1, .LBB102_1
+; RV64I-NEXT:  # %bb.4: # %atomicrmw.end
+; RV64I-NEXT:    ld s5, 16(sp)
+; RV64I-NEXT:    ld s4, 24(sp)
+; RV64I-NEXT:    ld s3, 32(sp)
+; RV64I-NEXT:    ld s2, 40(sp)
+; RV64I-NEXT:    ld s1, 48(sp)
+; RV64I-NEXT:    ld ra, 56(sp)
+; RV64I-NEXT:    addi sp, sp, 64
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_umax_i16_release:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    lui a2, 16
+; RV64IA-NEXT:    addiw a2, a2, -1
+; RV64IA-NEXT:    and a1, a1, a2
+; RV64IA-NEXT:    slli a3, a0, 3
+; RV64IA-NEXT:    andi a3, a3, 24
+; RV64IA-NEXT:    sllw a6, a2, a3
+; RV64IA-NEXT:    sllw a1, a1, a3
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:  .LBB102_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.w a4, (a0)
+; RV64IA-NEXT:    and a2, a4, a6
+; RV64IA-NEXT:    mv a5, a4
+; RV64IA-NEXT:    bgeu a2, a1, .LBB102_3
+; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB102_1 Depth=1
+; RV64IA-NEXT:    xor a5, a4, a1
+; RV64IA-NEXT:    and a5, a5, a6
+; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:  .LBB102_3: # in Loop: Header=BB102_1 Depth=1
+; RV64IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV64IA-NEXT:    bnez a5, .LBB102_1
+; RV64IA-NEXT:  # %bb.4:
+; RV64IA-NEXT:    srlw a0, a4, a3
+; RV64IA-NEXT:    ret
   %1 = atomicrmw umax i16* %a, i16 %b release
   ret i16 %1
 }
@@ -4823,53 +9039,47 @@ define i16 @atomicrmw_umax_i16_release(i16 *%a, i16 %b) nounwind {
 define i16 @atomicrmw_umax_i16_acq_rel(i16 *%a, i16 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_umax_i16_acq_rel:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    addi sp, sp, -48
-; RV32I-NEXT:    sw ra, 44(sp)
-; RV32I-NEXT:    sw s1, 40(sp)
-; RV32I-NEXT:    sw s2, 36(sp)
-; RV32I-NEXT:    sw s3, 32(sp)
-; RV32I-NEXT:    sw s4, 28(sp)
-; RV32I-NEXT:    sw s5, 24(sp)
-; RV32I-NEXT:    sw s6, 20(sp)
-; RV32I-NEXT:    sw s7, 16(sp)
+; RV32I-NEXT:    addi sp, sp, -32
+; RV32I-NEXT:    sw ra, 28(sp)
+; RV32I-NEXT:    sw s1, 24(sp)
+; RV32I-NEXT:    sw s2, 20(sp)
+; RV32I-NEXT:    sw s3, 16(sp)
+; RV32I-NEXT:    sw s4, 12(sp)
+; RV32I-NEXT:    sw s5, 8(sp)
 ; RV32I-NEXT:    mv s2, a1
-; RV32I-NEXT:    mv s6, a0
+; RV32I-NEXT:    mv s4, a0
 ; RV32I-NEXT:    lhu a0, 0(a0)
 ; RV32I-NEXT:    lui a1, 16
 ; RV32I-NEXT:    addi s1, a1, -1
-; RV32I-NEXT:    and s7, s2, s1
-; RV32I-NEXT:    addi s3, sp, 14
-; RV32I-NEXT:    addi s4, zero, 4
-; RV32I-NEXT:    addi s5, zero, 2
+; RV32I-NEXT:    and s5, s2, s1
+; RV32I-NEXT:    addi s3, sp, 6
 ; RV32I-NEXT:  .LBB103_1: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
 ; RV32I-NEXT:    and a1, a0, s1
 ; RV32I-NEXT:    mv a2, a0
-; RV32I-NEXT:    bltu s7, a1, .LBB103_3
+; RV32I-NEXT:    bltu s5, a1, .LBB103_3
 ; RV32I-NEXT:  # %bb.2: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB103_1 Depth=1
 ; RV32I-NEXT:    mv a2, s2
 ; RV32I-NEXT:  .LBB103_3: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB103_1 Depth=1
-; RV32I-NEXT:    sh a0, 14(sp)
-; RV32I-NEXT:    mv a0, s6
+; RV32I-NEXT:    sh a0, 6(sp)
+; RV32I-NEXT:    mv a0, s4
 ; RV32I-NEXT:    mv a1, s3
-; RV32I-NEXT:    mv a3, s4
-; RV32I-NEXT:    mv a4, s5
+; RV32I-NEXT:    addi a3, zero, 4
+; RV32I-NEXT:    addi a4, zero, 2
 ; RV32I-NEXT:    call __atomic_compare_exchange_2
 ; RV32I-NEXT:    mv a1, a0
-; RV32I-NEXT:    lh a0, 14(sp)
+; RV32I-NEXT:    lh a0, 6(sp)
 ; RV32I-NEXT:    beqz a1, .LBB103_1
 ; RV32I-NEXT:  # %bb.4: # %atomicrmw.end
-; RV32I-NEXT:    lw s7, 16(sp)
-; RV32I-NEXT:    lw s6, 20(sp)
-; RV32I-NEXT:    lw s5, 24(sp)
-; RV32I-NEXT:    lw s4, 28(sp)
-; RV32I-NEXT:    lw s3, 32(sp)
-; RV32I-NEXT:    lw s2, 36(sp)
-; RV32I-NEXT:    lw s1, 40(sp)
-; RV32I-NEXT:    lw ra, 44(sp)
-; RV32I-NEXT:    addi sp, sp, 48
+; RV32I-NEXT:    lw s5, 8(sp)
+; RV32I-NEXT:    lw s4, 12(sp)
+; RV32I-NEXT:    lw s3, 16(sp)
+; RV32I-NEXT:    lw s2, 20(sp)
+; RV32I-NEXT:    lw s1, 24(sp)
+; RV32I-NEXT:    lw ra, 28(sp)
+; RV32I-NEXT:    addi sp, sp, 32
 ; RV32I-NEXT:    ret
 ;
 ; RV32IA-LABEL: atomicrmw_umax_i16_acq_rel:
@@ -4897,6 +9107,77 @@ define i16 @atomicrmw_umax_i16_acq_rel(i16 *%a, i16 %b) nounwind {
 ; RV32IA-NEXT:  # %bb.4:
 ; RV32IA-NEXT:    srl a0, a4, a3
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_umax_i16_acq_rel:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -64
+; RV64I-NEXT:    sd ra, 56(sp)
+; RV64I-NEXT:    sd s1, 48(sp)
+; RV64I-NEXT:    sd s2, 40(sp)
+; RV64I-NEXT:    sd s3, 32(sp)
+; RV64I-NEXT:    sd s4, 24(sp)
+; RV64I-NEXT:    sd s5, 16(sp)
+; RV64I-NEXT:    mv s2, a1
+; RV64I-NEXT:    mv s4, a0
+; RV64I-NEXT:    lhu a0, 0(a0)
+; RV64I-NEXT:    lui a1, 16
+; RV64I-NEXT:    addiw s1, a1, -1
+; RV64I-NEXT:    and s5, s2, s1
+; RV64I-NEXT:    addi s3, sp, 14
+; RV64I-NEXT:  .LBB103_1: # %atomicrmw.start
+; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV64I-NEXT:    and a1, a0, s1
+; RV64I-NEXT:    mv a2, a0
+; RV64I-NEXT:    bltu s5, a1, .LBB103_3
+; RV64I-NEXT:  # %bb.2: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB103_1 Depth=1
+; RV64I-NEXT:    mv a2, s2
+; RV64I-NEXT:  .LBB103_3: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB103_1 Depth=1
+; RV64I-NEXT:    sh a0, 14(sp)
+; RV64I-NEXT:    mv a0, s4
+; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    addi a3, zero, 4
+; RV64I-NEXT:    addi a4, zero, 2
+; RV64I-NEXT:    call __atomic_compare_exchange_2
+; RV64I-NEXT:    mv a1, a0
+; RV64I-NEXT:    lh a0, 14(sp)
+; RV64I-NEXT:    beqz a1, .LBB103_1
+; RV64I-NEXT:  # %bb.4: # %atomicrmw.end
+; RV64I-NEXT:    ld s5, 16(sp)
+; RV64I-NEXT:    ld s4, 24(sp)
+; RV64I-NEXT:    ld s3, 32(sp)
+; RV64I-NEXT:    ld s2, 40(sp)
+; RV64I-NEXT:    ld s1, 48(sp)
+; RV64I-NEXT:    ld ra, 56(sp)
+; RV64I-NEXT:    addi sp, sp, 64
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_umax_i16_acq_rel:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    lui a2, 16
+; RV64IA-NEXT:    addiw a2, a2, -1
+; RV64IA-NEXT:    and a1, a1, a2
+; RV64IA-NEXT:    slli a3, a0, 3
+; RV64IA-NEXT:    andi a3, a3, 24
+; RV64IA-NEXT:    sllw a6, a2, a3
+; RV64IA-NEXT:    sllw a1, a1, a3
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:  .LBB103_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.w.aq a4, (a0)
+; RV64IA-NEXT:    and a2, a4, a6
+; RV64IA-NEXT:    mv a5, a4
+; RV64IA-NEXT:    bgeu a2, a1, .LBB103_3
+; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB103_1 Depth=1
+; RV64IA-NEXT:    xor a5, a4, a1
+; RV64IA-NEXT:    and a5, a5, a6
+; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:  .LBB103_3: # in Loop: Header=BB103_1 Depth=1
+; RV64IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV64IA-NEXT:    bnez a5, .LBB103_1
+; RV64IA-NEXT:  # %bb.4:
+; RV64IA-NEXT:    srlw a0, a4, a3
+; RV64IA-NEXT:    ret
   %1 = atomicrmw umax i16* %a, i16 %b acq_rel
   ret i16 %1
 }
@@ -4911,36 +9192,33 @@ define i16 @atomicrmw_umax_i16_seq_cst(i16 *%a, i16 %b) nounwind {
 ; RV32I-NEXT:    sw s3, 16(sp)
 ; RV32I-NEXT:    sw s4, 12(sp)
 ; RV32I-NEXT:    sw s5, 8(sp)
-; RV32I-NEXT:    sw s6, 4(sp)
 ; RV32I-NEXT:    mv s2, a1
 ; RV32I-NEXT:    mv s4, a0
 ; RV32I-NEXT:    lhu a0, 0(a0)
 ; RV32I-NEXT:    lui a1, 16
-; RV32I-NEXT:    addi s5, a1, -1
-; RV32I-NEXT:    and s6, s2, s5
-; RV32I-NEXT:    addi s3, sp, 2
-; RV32I-NEXT:    addi s1, zero, 5
+; RV32I-NEXT:    addi s1, a1, -1
+; RV32I-NEXT:    and s5, s2, s1
+; RV32I-NEXT:    addi s3, sp, 6
 ; RV32I-NEXT:  .LBB104_1: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    and a1, a0, s5
+; RV32I-NEXT:    and a1, a0, s1
 ; RV32I-NEXT:    mv a2, a0
-; RV32I-NEXT:    bltu s6, a1, .LBB104_3
+; RV32I-NEXT:    bltu s5, a1, .LBB104_3
 ; RV32I-NEXT:  # %bb.2: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB104_1 Depth=1
 ; RV32I-NEXT:    mv a2, s2
 ; RV32I-NEXT:  .LBB104_3: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB104_1 Depth=1
-; RV32I-NEXT:    sh a0, 2(sp)
+; RV32I-NEXT:    sh a0, 6(sp)
 ; RV32I-NEXT:    mv a0, s4
 ; RV32I-NEXT:    mv a1, s3
-; RV32I-NEXT:    mv a3, s1
-; RV32I-NEXT:    mv a4, s1
+; RV32I-NEXT:    addi a3, zero, 5
+; RV32I-NEXT:    addi a4, zero, 5
 ; RV32I-NEXT:    call __atomic_compare_exchange_2
 ; RV32I-NEXT:    mv a1, a0
-; RV32I-NEXT:    lh a0, 2(sp)
+; RV32I-NEXT:    lh a0, 6(sp)
 ; RV32I-NEXT:    beqz a1, .LBB104_1
 ; RV32I-NEXT:  # %bb.4: # %atomicrmw.end
-; RV32I-NEXT:    lw s6, 4(sp)
 ; RV32I-NEXT:    lw s5, 8(sp)
 ; RV32I-NEXT:    lw s4, 12(sp)
 ; RV32I-NEXT:    lw s3, 16(sp)
@@ -4975,6 +9253,77 @@ define i16 @atomicrmw_umax_i16_seq_cst(i16 *%a, i16 %b) nounwind {
 ; RV32IA-NEXT:  # %bb.4:
 ; RV32IA-NEXT:    srl a0, a4, a3
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_umax_i16_seq_cst:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -64
+; RV64I-NEXT:    sd ra, 56(sp)
+; RV64I-NEXT:    sd s1, 48(sp)
+; RV64I-NEXT:    sd s2, 40(sp)
+; RV64I-NEXT:    sd s3, 32(sp)
+; RV64I-NEXT:    sd s4, 24(sp)
+; RV64I-NEXT:    sd s5, 16(sp)
+; RV64I-NEXT:    mv s2, a1
+; RV64I-NEXT:    mv s4, a0
+; RV64I-NEXT:    lhu a0, 0(a0)
+; RV64I-NEXT:    lui a1, 16
+; RV64I-NEXT:    addiw s1, a1, -1
+; RV64I-NEXT:    and s5, s2, s1
+; RV64I-NEXT:    addi s3, sp, 14
+; RV64I-NEXT:  .LBB104_1: # %atomicrmw.start
+; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV64I-NEXT:    and a1, a0, s1
+; RV64I-NEXT:    mv a2, a0
+; RV64I-NEXT:    bltu s5, a1, .LBB104_3
+; RV64I-NEXT:  # %bb.2: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB104_1 Depth=1
+; RV64I-NEXT:    mv a2, s2
+; RV64I-NEXT:  .LBB104_3: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB104_1 Depth=1
+; RV64I-NEXT:    sh a0, 14(sp)
+; RV64I-NEXT:    mv a0, s4
+; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    addi a3, zero, 5
+; RV64I-NEXT:    addi a4, zero, 5
+; RV64I-NEXT:    call __atomic_compare_exchange_2
+; RV64I-NEXT:    mv a1, a0
+; RV64I-NEXT:    lh a0, 14(sp)
+; RV64I-NEXT:    beqz a1, .LBB104_1
+; RV64I-NEXT:  # %bb.4: # %atomicrmw.end
+; RV64I-NEXT:    ld s5, 16(sp)
+; RV64I-NEXT:    ld s4, 24(sp)
+; RV64I-NEXT:    ld s3, 32(sp)
+; RV64I-NEXT:    ld s2, 40(sp)
+; RV64I-NEXT:    ld s1, 48(sp)
+; RV64I-NEXT:    ld ra, 56(sp)
+; RV64I-NEXT:    addi sp, sp, 64
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_umax_i16_seq_cst:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    lui a2, 16
+; RV64IA-NEXT:    addiw a2, a2, -1
+; RV64IA-NEXT:    and a1, a1, a2
+; RV64IA-NEXT:    slli a3, a0, 3
+; RV64IA-NEXT:    andi a3, a3, 24
+; RV64IA-NEXT:    sllw a6, a2, a3
+; RV64IA-NEXT:    sllw a1, a1, a3
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:  .LBB104_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.w.aqrl a4, (a0)
+; RV64IA-NEXT:    and a2, a4, a6
+; RV64IA-NEXT:    mv a5, a4
+; RV64IA-NEXT:    bgeu a2, a1, .LBB104_3
+; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB104_1 Depth=1
+; RV64IA-NEXT:    xor a5, a4, a1
+; RV64IA-NEXT:    and a5, a5, a6
+; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:  .LBB104_3: # in Loop: Header=BB104_1 Depth=1
+; RV64IA-NEXT:    sc.w.aqrl a5, a5, (a0)
+; RV64IA-NEXT:    bnez a5, .LBB104_1
+; RV64IA-NEXT:  # %bb.4:
+; RV64IA-NEXT:    srlw a0, a4, a3
+; RV64IA-NEXT:    ret
   %1 = atomicrmw umax i16* %a, i16 %b seq_cst
   ret i16 %1
 }
@@ -5050,6 +9399,77 @@ define i16 @atomicrmw_umin_i16_monotonic(i16 *%a, i16 %b) nounwind {
 ; RV32IA-NEXT:  # %bb.4:
 ; RV32IA-NEXT:    srl a0, a4, a3
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_umin_i16_monotonic:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -64
+; RV64I-NEXT:    sd ra, 56(sp)
+; RV64I-NEXT:    sd s1, 48(sp)
+; RV64I-NEXT:    sd s2, 40(sp)
+; RV64I-NEXT:    sd s3, 32(sp)
+; RV64I-NEXT:    sd s4, 24(sp)
+; RV64I-NEXT:    sd s5, 16(sp)
+; RV64I-NEXT:    mv s2, a1
+; RV64I-NEXT:    mv s4, a0
+; RV64I-NEXT:    lhu a0, 0(a0)
+; RV64I-NEXT:    lui a1, 16
+; RV64I-NEXT:    addiw s1, a1, -1
+; RV64I-NEXT:    and s5, s2, s1
+; RV64I-NEXT:    addi s3, sp, 14
+; RV64I-NEXT:  .LBB105_1: # %atomicrmw.start
+; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV64I-NEXT:    and a1, a0, s1
+; RV64I-NEXT:    mv a2, a0
+; RV64I-NEXT:    bgeu s5, a1, .LBB105_3
+; RV64I-NEXT:  # %bb.2: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB105_1 Depth=1
+; RV64I-NEXT:    mv a2, s2
+; RV64I-NEXT:  .LBB105_3: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB105_1 Depth=1
+; RV64I-NEXT:    sh a0, 14(sp)
+; RV64I-NEXT:    mv a0, s4
+; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    mv a3, zero
+; RV64I-NEXT:    mv a4, zero
+; RV64I-NEXT:    call __atomic_compare_exchange_2
+; RV64I-NEXT:    mv a1, a0
+; RV64I-NEXT:    lh a0, 14(sp)
+; RV64I-NEXT:    beqz a1, .LBB105_1
+; RV64I-NEXT:  # %bb.4: # %atomicrmw.end
+; RV64I-NEXT:    ld s5, 16(sp)
+; RV64I-NEXT:    ld s4, 24(sp)
+; RV64I-NEXT:    ld s3, 32(sp)
+; RV64I-NEXT:    ld s2, 40(sp)
+; RV64I-NEXT:    ld s1, 48(sp)
+; RV64I-NEXT:    ld ra, 56(sp)
+; RV64I-NEXT:    addi sp, sp, 64
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_umin_i16_monotonic:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    lui a2, 16
+; RV64IA-NEXT:    addiw a2, a2, -1
+; RV64IA-NEXT:    and a1, a1, a2
+; RV64IA-NEXT:    slli a3, a0, 3
+; RV64IA-NEXT:    andi a3, a3, 24
+; RV64IA-NEXT:    sllw a6, a2, a3
+; RV64IA-NEXT:    sllw a1, a1, a3
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:  .LBB105_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.w a4, (a0)
+; RV64IA-NEXT:    and a2, a4, a6
+; RV64IA-NEXT:    mv a5, a4
+; RV64IA-NEXT:    bgeu a1, a2, .LBB105_3
+; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB105_1 Depth=1
+; RV64IA-NEXT:    xor a5, a4, a1
+; RV64IA-NEXT:    and a5, a5, a6
+; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:  .LBB105_3: # in Loop: Header=BB105_1 Depth=1
+; RV64IA-NEXT:    sc.w a5, a5, (a0)
+; RV64IA-NEXT:    bnez a5, .LBB105_1
+; RV64IA-NEXT:  # %bb.4:
+; RV64IA-NEXT:    srlw a0, a4, a3
+; RV64IA-NEXT:    ret
   %1 = atomicrmw umin i16* %a, i16 %b monotonic
   ret i16 %1
 }
@@ -5064,36 +9484,33 @@ define i16 @atomicrmw_umin_i16_acquire(i16 *%a, i16 %b) nounwind {
 ; RV32I-NEXT:    sw s3, 16(sp)
 ; RV32I-NEXT:    sw s4, 12(sp)
 ; RV32I-NEXT:    sw s5, 8(sp)
-; RV32I-NEXT:    sw s6, 4(sp)
 ; RV32I-NEXT:    mv s2, a1
 ; RV32I-NEXT:    mv s4, a0
 ; RV32I-NEXT:    lhu a0, 0(a0)
 ; RV32I-NEXT:    lui a1, 16
-; RV32I-NEXT:    addi s5, a1, -1
-; RV32I-NEXT:    and s6, s2, s5
-; RV32I-NEXT:    addi s3, sp, 2
-; RV32I-NEXT:    addi s1, zero, 2
+; RV32I-NEXT:    addi s1, a1, -1
+; RV32I-NEXT:    and s5, s2, s1
+; RV32I-NEXT:    addi s3, sp, 6
 ; RV32I-NEXT:  .LBB106_1: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    and a1, a0, s5
+; RV32I-NEXT:    and a1, a0, s1
 ; RV32I-NEXT:    mv a2, a0
-; RV32I-NEXT:    bgeu s6, a1, .LBB106_3
+; RV32I-NEXT:    bgeu s5, a1, .LBB106_3
 ; RV32I-NEXT:  # %bb.2: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB106_1 Depth=1
 ; RV32I-NEXT:    mv a2, s2
 ; RV32I-NEXT:  .LBB106_3: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB106_1 Depth=1
-; RV32I-NEXT:    sh a0, 2(sp)
+; RV32I-NEXT:    sh a0, 6(sp)
 ; RV32I-NEXT:    mv a0, s4
 ; RV32I-NEXT:    mv a1, s3
-; RV32I-NEXT:    mv a3, s1
-; RV32I-NEXT:    mv a4, s1
+; RV32I-NEXT:    addi a3, zero, 2
+; RV32I-NEXT:    addi a4, zero, 2
 ; RV32I-NEXT:    call __atomic_compare_exchange_2
 ; RV32I-NEXT:    mv a1, a0
-; RV32I-NEXT:    lh a0, 2(sp)
+; RV32I-NEXT:    lh a0, 6(sp)
 ; RV32I-NEXT:    beqz a1, .LBB106_1
 ; RV32I-NEXT:  # %bb.4: # %atomicrmw.end
-; RV32I-NEXT:    lw s6, 4(sp)
 ; RV32I-NEXT:    lw s5, 8(sp)
 ; RV32I-NEXT:    lw s4, 12(sp)
 ; RV32I-NEXT:    lw s3, 16(sp)
@@ -5128,6 +9545,77 @@ define i16 @atomicrmw_umin_i16_acquire(i16 *%a, i16 %b) nounwind {
 ; RV32IA-NEXT:  # %bb.4:
 ; RV32IA-NEXT:    srl a0, a4, a3
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_umin_i16_acquire:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -64
+; RV64I-NEXT:    sd ra, 56(sp)
+; RV64I-NEXT:    sd s1, 48(sp)
+; RV64I-NEXT:    sd s2, 40(sp)
+; RV64I-NEXT:    sd s3, 32(sp)
+; RV64I-NEXT:    sd s4, 24(sp)
+; RV64I-NEXT:    sd s5, 16(sp)
+; RV64I-NEXT:    mv s2, a1
+; RV64I-NEXT:    mv s4, a0
+; RV64I-NEXT:    lhu a0, 0(a0)
+; RV64I-NEXT:    lui a1, 16
+; RV64I-NEXT:    addiw s1, a1, -1
+; RV64I-NEXT:    and s5, s2, s1
+; RV64I-NEXT:    addi s3, sp, 14
+; RV64I-NEXT:  .LBB106_1: # %atomicrmw.start
+; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV64I-NEXT:    and a1, a0, s1
+; RV64I-NEXT:    mv a2, a0
+; RV64I-NEXT:    bgeu s5, a1, .LBB106_3
+; RV64I-NEXT:  # %bb.2: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB106_1 Depth=1
+; RV64I-NEXT:    mv a2, s2
+; RV64I-NEXT:  .LBB106_3: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB106_1 Depth=1
+; RV64I-NEXT:    sh a0, 14(sp)
+; RV64I-NEXT:    mv a0, s4
+; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    addi a3, zero, 2
+; RV64I-NEXT:    addi a4, zero, 2
+; RV64I-NEXT:    call __atomic_compare_exchange_2
+; RV64I-NEXT:    mv a1, a0
+; RV64I-NEXT:    lh a0, 14(sp)
+; RV64I-NEXT:    beqz a1, .LBB106_1
+; RV64I-NEXT:  # %bb.4: # %atomicrmw.end
+; RV64I-NEXT:    ld s5, 16(sp)
+; RV64I-NEXT:    ld s4, 24(sp)
+; RV64I-NEXT:    ld s3, 32(sp)
+; RV64I-NEXT:    ld s2, 40(sp)
+; RV64I-NEXT:    ld s1, 48(sp)
+; RV64I-NEXT:    ld ra, 56(sp)
+; RV64I-NEXT:    addi sp, sp, 64
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_umin_i16_acquire:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    lui a2, 16
+; RV64IA-NEXT:    addiw a2, a2, -1
+; RV64IA-NEXT:    and a1, a1, a2
+; RV64IA-NEXT:    slli a3, a0, 3
+; RV64IA-NEXT:    andi a3, a3, 24
+; RV64IA-NEXT:    sllw a6, a2, a3
+; RV64IA-NEXT:    sllw a1, a1, a3
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:  .LBB106_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.w.aq a4, (a0)
+; RV64IA-NEXT:    and a2, a4, a6
+; RV64IA-NEXT:    mv a5, a4
+; RV64IA-NEXT:    bgeu a1, a2, .LBB106_3
+; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB106_1 Depth=1
+; RV64IA-NEXT:    xor a5, a4, a1
+; RV64IA-NEXT:    and a5, a5, a6
+; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:  .LBB106_3: # in Loop: Header=BB106_1 Depth=1
+; RV64IA-NEXT:    sc.w a5, a5, (a0)
+; RV64IA-NEXT:    bnez a5, .LBB106_1
+; RV64IA-NEXT:  # %bb.4:
+; RV64IA-NEXT:    srlw a0, a4, a3
+; RV64IA-NEXT:    ret
   %1 = atomicrmw umin i16* %a, i16 %b acquire
   ret i16 %1
 }
@@ -5142,36 +9630,33 @@ define i16 @atomicrmw_umin_i16_release(i16 *%a, i16 %b) nounwind {
 ; RV32I-NEXT:    sw s3, 16(sp)
 ; RV32I-NEXT:    sw s4, 12(sp)
 ; RV32I-NEXT:    sw s5, 8(sp)
-; RV32I-NEXT:    sw s6, 4(sp)
 ; RV32I-NEXT:    mv s2, a1
-; RV32I-NEXT:    mv s5, a0
+; RV32I-NEXT:    mv s4, a0
 ; RV32I-NEXT:    lhu a0, 0(a0)
 ; RV32I-NEXT:    lui a1, 16
 ; RV32I-NEXT:    addi s1, a1, -1
-; RV32I-NEXT:    and s6, s2, s1
-; RV32I-NEXT:    addi s3, sp, 2
-; RV32I-NEXT:    addi s4, zero, 3
+; RV32I-NEXT:    and s5, s2, s1
+; RV32I-NEXT:    addi s3, sp, 6
 ; RV32I-NEXT:  .LBB107_1: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
 ; RV32I-NEXT:    and a1, a0, s1
 ; RV32I-NEXT:    mv a2, a0
-; RV32I-NEXT:    bgeu s6, a1, .LBB107_3
+; RV32I-NEXT:    bgeu s5, a1, .LBB107_3
 ; RV32I-NEXT:  # %bb.2: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB107_1 Depth=1
 ; RV32I-NEXT:    mv a2, s2
 ; RV32I-NEXT:  .LBB107_3: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB107_1 Depth=1
-; RV32I-NEXT:    sh a0, 2(sp)
-; RV32I-NEXT:    mv a0, s5
+; RV32I-NEXT:    sh a0, 6(sp)
+; RV32I-NEXT:    mv a0, s4
 ; RV32I-NEXT:    mv a1, s3
-; RV32I-NEXT:    mv a3, s4
+; RV32I-NEXT:    addi a3, zero, 3
 ; RV32I-NEXT:    mv a4, zero
 ; RV32I-NEXT:    call __atomic_compare_exchange_2
 ; RV32I-NEXT:    mv a1, a0
-; RV32I-NEXT:    lh a0, 2(sp)
+; RV32I-NEXT:    lh a0, 6(sp)
 ; RV32I-NEXT:    beqz a1, .LBB107_1
 ; RV32I-NEXT:  # %bb.4: # %atomicrmw.end
-; RV32I-NEXT:    lw s6, 4(sp)
 ; RV32I-NEXT:    lw s5, 8(sp)
 ; RV32I-NEXT:    lw s4, 12(sp)
 ; RV32I-NEXT:    lw s3, 16(sp)
@@ -5206,6 +9691,77 @@ define i16 @atomicrmw_umin_i16_release(i16 *%a, i16 %b) nounwind {
 ; RV32IA-NEXT:  # %bb.4:
 ; RV32IA-NEXT:    srl a0, a4, a3
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_umin_i16_release:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -64
+; RV64I-NEXT:    sd ra, 56(sp)
+; RV64I-NEXT:    sd s1, 48(sp)
+; RV64I-NEXT:    sd s2, 40(sp)
+; RV64I-NEXT:    sd s3, 32(sp)
+; RV64I-NEXT:    sd s4, 24(sp)
+; RV64I-NEXT:    sd s5, 16(sp)
+; RV64I-NEXT:    mv s2, a1
+; RV64I-NEXT:    mv s4, a0
+; RV64I-NEXT:    lhu a0, 0(a0)
+; RV64I-NEXT:    lui a1, 16
+; RV64I-NEXT:    addiw s1, a1, -1
+; RV64I-NEXT:    and s5, s2, s1
+; RV64I-NEXT:    addi s3, sp, 14
+; RV64I-NEXT:  .LBB107_1: # %atomicrmw.start
+; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV64I-NEXT:    and a1, a0, s1
+; RV64I-NEXT:    mv a2, a0
+; RV64I-NEXT:    bgeu s5, a1, .LBB107_3
+; RV64I-NEXT:  # %bb.2: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB107_1 Depth=1
+; RV64I-NEXT:    mv a2, s2
+; RV64I-NEXT:  .LBB107_3: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB107_1 Depth=1
+; RV64I-NEXT:    sh a0, 14(sp)
+; RV64I-NEXT:    mv a0, s4
+; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    addi a3, zero, 3
+; RV64I-NEXT:    mv a4, zero
+; RV64I-NEXT:    call __atomic_compare_exchange_2
+; RV64I-NEXT:    mv a1, a0
+; RV64I-NEXT:    lh a0, 14(sp)
+; RV64I-NEXT:    beqz a1, .LBB107_1
+; RV64I-NEXT:  # %bb.4: # %atomicrmw.end
+; RV64I-NEXT:    ld s5, 16(sp)
+; RV64I-NEXT:    ld s4, 24(sp)
+; RV64I-NEXT:    ld s3, 32(sp)
+; RV64I-NEXT:    ld s2, 40(sp)
+; RV64I-NEXT:    ld s1, 48(sp)
+; RV64I-NEXT:    ld ra, 56(sp)
+; RV64I-NEXT:    addi sp, sp, 64
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_umin_i16_release:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    lui a2, 16
+; RV64IA-NEXT:    addiw a2, a2, -1
+; RV64IA-NEXT:    and a1, a1, a2
+; RV64IA-NEXT:    slli a3, a0, 3
+; RV64IA-NEXT:    andi a3, a3, 24
+; RV64IA-NEXT:    sllw a6, a2, a3
+; RV64IA-NEXT:    sllw a1, a1, a3
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:  .LBB107_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.w a4, (a0)
+; RV64IA-NEXT:    and a2, a4, a6
+; RV64IA-NEXT:    mv a5, a4
+; RV64IA-NEXT:    bgeu a1, a2, .LBB107_3
+; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB107_1 Depth=1
+; RV64IA-NEXT:    xor a5, a4, a1
+; RV64IA-NEXT:    and a5, a5, a6
+; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:  .LBB107_3: # in Loop: Header=BB107_1 Depth=1
+; RV64IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV64IA-NEXT:    bnez a5, .LBB107_1
+; RV64IA-NEXT:  # %bb.4:
+; RV64IA-NEXT:    srlw a0, a4, a3
+; RV64IA-NEXT:    ret
   %1 = atomicrmw umin i16* %a, i16 %b release
   ret i16 %1
 }
@@ -5213,53 +9769,47 @@ define i16 @atomicrmw_umin_i16_release(i16 *%a, i16 %b) nounwind {
 define i16 @atomicrmw_umin_i16_acq_rel(i16 *%a, i16 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_umin_i16_acq_rel:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    addi sp, sp, -48
-; RV32I-NEXT:    sw ra, 44(sp)
-; RV32I-NEXT:    sw s1, 40(sp)
-; RV32I-NEXT:    sw s2, 36(sp)
-; RV32I-NEXT:    sw s3, 32(sp)
-; RV32I-NEXT:    sw s4, 28(sp)
-; RV32I-NEXT:    sw s5, 24(sp)
-; RV32I-NEXT:    sw s6, 20(sp)
-; RV32I-NEXT:    sw s7, 16(sp)
+; RV32I-NEXT:    addi sp, sp, -32
+; RV32I-NEXT:    sw ra, 28(sp)
+; RV32I-NEXT:    sw s1, 24(sp)
+; RV32I-NEXT:    sw s2, 20(sp)
+; RV32I-NEXT:    sw s3, 16(sp)
+; RV32I-NEXT:    sw s4, 12(sp)
+; RV32I-NEXT:    sw s5, 8(sp)
 ; RV32I-NEXT:    mv s2, a1
-; RV32I-NEXT:    mv s6, a0
+; RV32I-NEXT:    mv s4, a0
 ; RV32I-NEXT:    lhu a0, 0(a0)
 ; RV32I-NEXT:    lui a1, 16
 ; RV32I-NEXT:    addi s1, a1, -1
-; RV32I-NEXT:    and s7, s2, s1
-; RV32I-NEXT:    addi s3, sp, 14
-; RV32I-NEXT:    addi s4, zero, 4
-; RV32I-NEXT:    addi s5, zero, 2
+; RV32I-NEXT:    and s5, s2, s1
+; RV32I-NEXT:    addi s3, sp, 6
 ; RV32I-NEXT:  .LBB108_1: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
 ; RV32I-NEXT:    and a1, a0, s1
 ; RV32I-NEXT:    mv a2, a0
-; RV32I-NEXT:    bgeu s7, a1, .LBB108_3
+; RV32I-NEXT:    bgeu s5, a1, .LBB108_3
 ; RV32I-NEXT:  # %bb.2: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB108_1 Depth=1
 ; RV32I-NEXT:    mv a2, s2
 ; RV32I-NEXT:  .LBB108_3: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB108_1 Depth=1
-; RV32I-NEXT:    sh a0, 14(sp)
-; RV32I-NEXT:    mv a0, s6
+; RV32I-NEXT:    sh a0, 6(sp)
+; RV32I-NEXT:    mv a0, s4
 ; RV32I-NEXT:    mv a1, s3
-; RV32I-NEXT:    mv a3, s4
-; RV32I-NEXT:    mv a4, s5
+; RV32I-NEXT:    addi a3, zero, 4
+; RV32I-NEXT:    addi a4, zero, 2
 ; RV32I-NEXT:    call __atomic_compare_exchange_2
 ; RV32I-NEXT:    mv a1, a0
-; RV32I-NEXT:    lh a0, 14(sp)
+; RV32I-NEXT:    lh a0, 6(sp)
 ; RV32I-NEXT:    beqz a1, .LBB108_1
 ; RV32I-NEXT:  # %bb.4: # %atomicrmw.end
-; RV32I-NEXT:    lw s7, 16(sp)
-; RV32I-NEXT:    lw s6, 20(sp)
-; RV32I-NEXT:    lw s5, 24(sp)
-; RV32I-NEXT:    lw s4, 28(sp)
-; RV32I-NEXT:    lw s3, 32(sp)
-; RV32I-NEXT:    lw s2, 36(sp)
-; RV32I-NEXT:    lw s1, 40(sp)
-; RV32I-NEXT:    lw ra, 44(sp)
-; RV32I-NEXT:    addi sp, sp, 48
+; RV32I-NEXT:    lw s5, 8(sp)
+; RV32I-NEXT:    lw s4, 12(sp)
+; RV32I-NEXT:    lw s3, 16(sp)
+; RV32I-NEXT:    lw s2, 20(sp)
+; RV32I-NEXT:    lw s1, 24(sp)
+; RV32I-NEXT:    lw ra, 28(sp)
+; RV32I-NEXT:    addi sp, sp, 32
 ; RV32I-NEXT:    ret
 ;
 ; RV32IA-LABEL: atomicrmw_umin_i16_acq_rel:
@@ -5287,6 +9837,77 @@ define i16 @atomicrmw_umin_i16_acq_rel(i16 *%a, i16 %b) nounwind {
 ; RV32IA-NEXT:  # %bb.4:
 ; RV32IA-NEXT:    srl a0, a4, a3
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_umin_i16_acq_rel:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -64
+; RV64I-NEXT:    sd ra, 56(sp)
+; RV64I-NEXT:    sd s1, 48(sp)
+; RV64I-NEXT:    sd s2, 40(sp)
+; RV64I-NEXT:    sd s3, 32(sp)
+; RV64I-NEXT:    sd s4, 24(sp)
+; RV64I-NEXT:    sd s5, 16(sp)
+; RV64I-NEXT:    mv s2, a1
+; RV64I-NEXT:    mv s4, a0
+; RV64I-NEXT:    lhu a0, 0(a0)
+; RV64I-NEXT:    lui a1, 16
+; RV64I-NEXT:    addiw s1, a1, -1
+; RV64I-NEXT:    and s5, s2, s1
+; RV64I-NEXT:    addi s3, sp, 14
+; RV64I-NEXT:  .LBB108_1: # %atomicrmw.start
+; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV64I-NEXT:    and a1, a0, s1
+; RV64I-NEXT:    mv a2, a0
+; RV64I-NEXT:    bgeu s5, a1, .LBB108_3
+; RV64I-NEXT:  # %bb.2: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB108_1 Depth=1
+; RV64I-NEXT:    mv a2, s2
+; RV64I-NEXT:  .LBB108_3: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB108_1 Depth=1
+; RV64I-NEXT:    sh a0, 14(sp)
+; RV64I-NEXT:    mv a0, s4
+; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    addi a3, zero, 4
+; RV64I-NEXT:    addi a4, zero, 2
+; RV64I-NEXT:    call __atomic_compare_exchange_2
+; RV64I-NEXT:    mv a1, a0
+; RV64I-NEXT:    lh a0, 14(sp)
+; RV64I-NEXT:    beqz a1, .LBB108_1
+; RV64I-NEXT:  # %bb.4: # %atomicrmw.end
+; RV64I-NEXT:    ld s5, 16(sp)
+; RV64I-NEXT:    ld s4, 24(sp)
+; RV64I-NEXT:    ld s3, 32(sp)
+; RV64I-NEXT:    ld s2, 40(sp)
+; RV64I-NEXT:    ld s1, 48(sp)
+; RV64I-NEXT:    ld ra, 56(sp)
+; RV64I-NEXT:    addi sp, sp, 64
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_umin_i16_acq_rel:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    lui a2, 16
+; RV64IA-NEXT:    addiw a2, a2, -1
+; RV64IA-NEXT:    and a1, a1, a2
+; RV64IA-NEXT:    slli a3, a0, 3
+; RV64IA-NEXT:    andi a3, a3, 24
+; RV64IA-NEXT:    sllw a6, a2, a3
+; RV64IA-NEXT:    sllw a1, a1, a3
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:  .LBB108_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.w.aq a4, (a0)
+; RV64IA-NEXT:    and a2, a4, a6
+; RV64IA-NEXT:    mv a5, a4
+; RV64IA-NEXT:    bgeu a1, a2, .LBB108_3
+; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB108_1 Depth=1
+; RV64IA-NEXT:    xor a5, a4, a1
+; RV64IA-NEXT:    and a5, a5, a6
+; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:  .LBB108_3: # in Loop: Header=BB108_1 Depth=1
+; RV64IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV64IA-NEXT:    bnez a5, .LBB108_1
+; RV64IA-NEXT:  # %bb.4:
+; RV64IA-NEXT:    srlw a0, a4, a3
+; RV64IA-NEXT:    ret
   %1 = atomicrmw umin i16* %a, i16 %b acq_rel
   ret i16 %1
 }
@@ -5301,36 +9922,33 @@ define i16 @atomicrmw_umin_i16_seq_cst(i16 *%a, i16 %b) nounwind {
 ; RV32I-NEXT:    sw s3, 16(sp)
 ; RV32I-NEXT:    sw s4, 12(sp)
 ; RV32I-NEXT:    sw s5, 8(sp)
-; RV32I-NEXT:    sw s6, 4(sp)
 ; RV32I-NEXT:    mv s2, a1
 ; RV32I-NEXT:    mv s4, a0
 ; RV32I-NEXT:    lhu a0, 0(a0)
 ; RV32I-NEXT:    lui a1, 16
-; RV32I-NEXT:    addi s5, a1, -1
-; RV32I-NEXT:    and s6, s2, s5
-; RV32I-NEXT:    addi s3, sp, 2
-; RV32I-NEXT:    addi s1, zero, 5
+; RV32I-NEXT:    addi s1, a1, -1
+; RV32I-NEXT:    and s5, s2, s1
+; RV32I-NEXT:    addi s3, sp, 6
 ; RV32I-NEXT:  .LBB109_1: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    and a1, a0, s5
+; RV32I-NEXT:    and a1, a0, s1
 ; RV32I-NEXT:    mv a2, a0
-; RV32I-NEXT:    bgeu s6, a1, .LBB109_3
+; RV32I-NEXT:    bgeu s5, a1, .LBB109_3
 ; RV32I-NEXT:  # %bb.2: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB109_1 Depth=1
 ; RV32I-NEXT:    mv a2, s2
 ; RV32I-NEXT:  .LBB109_3: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB109_1 Depth=1
-; RV32I-NEXT:    sh a0, 2(sp)
+; RV32I-NEXT:    sh a0, 6(sp)
 ; RV32I-NEXT:    mv a0, s4
 ; RV32I-NEXT:    mv a1, s3
-; RV32I-NEXT:    mv a3, s1
-; RV32I-NEXT:    mv a4, s1
+; RV32I-NEXT:    addi a3, zero, 5
+; RV32I-NEXT:    addi a4, zero, 5
 ; RV32I-NEXT:    call __atomic_compare_exchange_2
 ; RV32I-NEXT:    mv a1, a0
-; RV32I-NEXT:    lh a0, 2(sp)
+; RV32I-NEXT:    lh a0, 6(sp)
 ; RV32I-NEXT:    beqz a1, .LBB109_1
 ; RV32I-NEXT:  # %bb.4: # %atomicrmw.end
-; RV32I-NEXT:    lw s6, 4(sp)
 ; RV32I-NEXT:    lw s5, 8(sp)
 ; RV32I-NEXT:    lw s4, 12(sp)
 ; RV32I-NEXT:    lw s3, 16(sp)
@@ -5365,6 +9983,77 @@ define i16 @atomicrmw_umin_i16_seq_cst(i16 *%a, i16 %b) nounwind {
 ; RV32IA-NEXT:  # %bb.4:
 ; RV32IA-NEXT:    srl a0, a4, a3
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_umin_i16_seq_cst:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -64
+; RV64I-NEXT:    sd ra, 56(sp)
+; RV64I-NEXT:    sd s1, 48(sp)
+; RV64I-NEXT:    sd s2, 40(sp)
+; RV64I-NEXT:    sd s3, 32(sp)
+; RV64I-NEXT:    sd s4, 24(sp)
+; RV64I-NEXT:    sd s5, 16(sp)
+; RV64I-NEXT:    mv s2, a1
+; RV64I-NEXT:    mv s4, a0
+; RV64I-NEXT:    lhu a0, 0(a0)
+; RV64I-NEXT:    lui a1, 16
+; RV64I-NEXT:    addiw s1, a1, -1
+; RV64I-NEXT:    and s5, s2, s1
+; RV64I-NEXT:    addi s3, sp, 14
+; RV64I-NEXT:  .LBB109_1: # %atomicrmw.start
+; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV64I-NEXT:    and a1, a0, s1
+; RV64I-NEXT:    mv a2, a0
+; RV64I-NEXT:    bgeu s5, a1, .LBB109_3
+; RV64I-NEXT:  # %bb.2: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB109_1 Depth=1
+; RV64I-NEXT:    mv a2, s2
+; RV64I-NEXT:  .LBB109_3: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB109_1 Depth=1
+; RV64I-NEXT:    sh a0, 14(sp)
+; RV64I-NEXT:    mv a0, s4
+; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    addi a3, zero, 5
+; RV64I-NEXT:    addi a4, zero, 5
+; RV64I-NEXT:    call __atomic_compare_exchange_2
+; RV64I-NEXT:    mv a1, a0
+; RV64I-NEXT:    lh a0, 14(sp)
+; RV64I-NEXT:    beqz a1, .LBB109_1
+; RV64I-NEXT:  # %bb.4: # %atomicrmw.end
+; RV64I-NEXT:    ld s5, 16(sp)
+; RV64I-NEXT:    ld s4, 24(sp)
+; RV64I-NEXT:    ld s3, 32(sp)
+; RV64I-NEXT:    ld s2, 40(sp)
+; RV64I-NEXT:    ld s1, 48(sp)
+; RV64I-NEXT:    ld ra, 56(sp)
+; RV64I-NEXT:    addi sp, sp, 64
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_umin_i16_seq_cst:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    lui a2, 16
+; RV64IA-NEXT:    addiw a2, a2, -1
+; RV64IA-NEXT:    and a1, a1, a2
+; RV64IA-NEXT:    slli a3, a0, 3
+; RV64IA-NEXT:    andi a3, a3, 24
+; RV64IA-NEXT:    sllw a6, a2, a3
+; RV64IA-NEXT:    sllw a1, a1, a3
+; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:  .LBB109_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.w.aqrl a4, (a0)
+; RV64IA-NEXT:    and a2, a4, a6
+; RV64IA-NEXT:    mv a5, a4
+; RV64IA-NEXT:    bgeu a1, a2, .LBB109_3
+; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB109_1 Depth=1
+; RV64IA-NEXT:    xor a5, a4, a1
+; RV64IA-NEXT:    and a5, a5, a6
+; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:  .LBB109_3: # in Loop: Header=BB109_1 Depth=1
+; RV64IA-NEXT:    sc.w.aqrl a5, a5, (a0)
+; RV64IA-NEXT:    bnez a5, .LBB109_1
+; RV64IA-NEXT:  # %bb.4:
+; RV64IA-NEXT:    srlw a0, a4, a3
+; RV64IA-NEXT:    ret
   %1 = atomicrmw umin i16* %a, i16 %b seq_cst
   ret i16 %1
 }
@@ -5384,6 +10073,21 @@ define i32 @atomicrmw_xchg_i32_monotonic(i32* %a, i32 %b) {
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    amoswap.w a0, a1, (a0)
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_xchg_i32_monotonic:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    mv a2, zero
+; RV64I-NEXT:    call __atomic_exchange_4
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_xchg_i32_monotonic:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    amoswap.w a0, a1, (a0)
+; RV64IA-NEXT:    ret
   %1 = atomicrmw xchg i32* %a, i32 %b monotonic
   ret i32 %1
 }
@@ -5403,6 +10107,21 @@ define i32 @atomicrmw_xchg_i32_acquire(i32* %a, i32 %b) {
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    amoswap.w.aq a0, a1, (a0)
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_xchg_i32_acquire:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a2, zero, 2
+; RV64I-NEXT:    call __atomic_exchange_4
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_xchg_i32_acquire:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    amoswap.w.aq a0, a1, (a0)
+; RV64IA-NEXT:    ret
   %1 = atomicrmw xchg i32* %a, i32 %b acquire
   ret i32 %1
 }
@@ -5422,6 +10141,21 @@ define i32 @atomicrmw_xchg_i32_release(i32* %a, i32 %b) {
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    amoswap.w.rl a0, a1, (a0)
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_xchg_i32_release:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a2, zero, 3
+; RV64I-NEXT:    call __atomic_exchange_4
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_xchg_i32_release:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    amoswap.w.rl a0, a1, (a0)
+; RV64IA-NEXT:    ret
   %1 = atomicrmw xchg i32* %a, i32 %b release
   ret i32 %1
 }
@@ -5441,6 +10175,21 @@ define i32 @atomicrmw_xchg_i32_acq_rel(i32* %a, i32 %b) {
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    amoswap.w.aqrl a0, a1, (a0)
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_xchg_i32_acq_rel:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a2, zero, 4
+; RV64I-NEXT:    call __atomic_exchange_4
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_xchg_i32_acq_rel:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    amoswap.w.aqrl a0, a1, (a0)
+; RV64IA-NEXT:    ret
   %1 = atomicrmw xchg i32* %a, i32 %b acq_rel
   ret i32 %1
 }
@@ -5460,6 +10209,21 @@ define i32 @atomicrmw_xchg_i32_seq_cst(i32* %a, i32 %b) {
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    amoswap.w.aqrl a0, a1, (a0)
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_xchg_i32_seq_cst:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a2, zero, 5
+; RV64I-NEXT:    call __atomic_exchange_4
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_xchg_i32_seq_cst:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    amoswap.w.aqrl a0, a1, (a0)
+; RV64IA-NEXT:    ret
   %1 = atomicrmw xchg i32* %a, i32 %b seq_cst
   ret i32 %1
 }
@@ -5479,6 +10243,21 @@ define i32 @atomicrmw_add_i32_monotonic(i32 *%a, i32 %b) nounwind {
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    amoadd.w a0, a1, (a0)
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_add_i32_monotonic:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    mv a2, zero
+; RV64I-NEXT:    call __atomic_fetch_add_4
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_add_i32_monotonic:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    amoadd.w a0, a1, (a0)
+; RV64IA-NEXT:    ret
   %1 = atomicrmw add i32* %a, i32 %b monotonic
   ret i32 %1
 }
@@ -5498,6 +10277,21 @@ define i32 @atomicrmw_add_i32_acquire(i32 *%a, i32 %b) nounwind {
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    amoadd.w.aq a0, a1, (a0)
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_add_i32_acquire:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a2, zero, 2
+; RV64I-NEXT:    call __atomic_fetch_add_4
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_add_i32_acquire:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    amoadd.w.aq a0, a1, (a0)
+; RV64IA-NEXT:    ret
   %1 = atomicrmw add i32* %a, i32 %b acquire
   ret i32 %1
 }
@@ -5517,6 +10311,21 @@ define i32 @atomicrmw_add_i32_release(i32 *%a, i32 %b) nounwind {
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    amoadd.w.rl a0, a1, (a0)
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_add_i32_release:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a2, zero, 3
+; RV64I-NEXT:    call __atomic_fetch_add_4
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_add_i32_release:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    amoadd.w.rl a0, a1, (a0)
+; RV64IA-NEXT:    ret
   %1 = atomicrmw add i32* %a, i32 %b release
   ret i32 %1
 }
@@ -5536,6 +10345,21 @@ define i32 @atomicrmw_add_i32_acq_rel(i32 *%a, i32 %b) nounwind {
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    amoadd.w.aqrl a0, a1, (a0)
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_add_i32_acq_rel:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a2, zero, 4
+; RV64I-NEXT:    call __atomic_fetch_add_4
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_add_i32_acq_rel:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    amoadd.w.aqrl a0, a1, (a0)
+; RV64IA-NEXT:    ret
   %1 = atomicrmw add i32* %a, i32 %b acq_rel
   ret i32 %1
 }
@@ -5555,6 +10379,21 @@ define i32 @atomicrmw_add_i32_seq_cst(i32 *%a, i32 %b) nounwind {
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    amoadd.w.aqrl a0, a1, (a0)
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_add_i32_seq_cst:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a2, zero, 5
+; RV64I-NEXT:    call __atomic_fetch_add_4
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_add_i32_seq_cst:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    amoadd.w.aqrl a0, a1, (a0)
+; RV64IA-NEXT:    ret
   %1 = atomicrmw add i32* %a, i32 %b seq_cst
   ret i32 %1
 }
@@ -5575,6 +10414,22 @@ define i32 @atomicrmw_sub_i32_monotonic(i32* %a, i32 %b) {
 ; RV32IA-NEXT:    neg a1, a1
 ; RV32IA-NEXT:    amoadd.w a0, a1, (a0)
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_sub_i32_monotonic:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    mv a2, zero
+; RV64I-NEXT:    call __atomic_fetch_sub_4
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_sub_i32_monotonic:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    neg a1, a1
+; RV64IA-NEXT:    amoadd.w a0, a1, (a0)
+; RV64IA-NEXT:    ret
   %1 = atomicrmw sub i32* %a, i32 %b monotonic
   ret i32 %1
 }
@@ -5595,6 +10450,22 @@ define i32 @atomicrmw_sub_i32_acquire(i32* %a, i32 %b) {
 ; RV32IA-NEXT:    neg a1, a1
 ; RV32IA-NEXT:    amoadd.w.aq a0, a1, (a0)
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_sub_i32_acquire:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a2, zero, 2
+; RV64I-NEXT:    call __atomic_fetch_sub_4
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_sub_i32_acquire:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    neg a1, a1
+; RV64IA-NEXT:    amoadd.w.aq a0, a1, (a0)
+; RV64IA-NEXT:    ret
   %1 = atomicrmw sub i32* %a, i32 %b acquire
   ret i32 %1
 }
@@ -5615,6 +10486,22 @@ define i32 @atomicrmw_sub_i32_release(i32* %a, i32 %b) {
 ; RV32IA-NEXT:    neg a1, a1
 ; RV32IA-NEXT:    amoadd.w.rl a0, a1, (a0)
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_sub_i32_release:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a2, zero, 3
+; RV64I-NEXT:    call __atomic_fetch_sub_4
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_sub_i32_release:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    neg a1, a1
+; RV64IA-NEXT:    amoadd.w.rl a0, a1, (a0)
+; RV64IA-NEXT:    ret
   %1 = atomicrmw sub i32* %a, i32 %b release
   ret i32 %1
 }
@@ -5635,6 +10522,22 @@ define i32 @atomicrmw_sub_i32_acq_rel(i32* %a, i32 %b) {
 ; RV32IA-NEXT:    neg a1, a1
 ; RV32IA-NEXT:    amoadd.w.aqrl a0, a1, (a0)
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_sub_i32_acq_rel:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a2, zero, 4
+; RV64I-NEXT:    call __atomic_fetch_sub_4
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_sub_i32_acq_rel:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    neg a1, a1
+; RV64IA-NEXT:    amoadd.w.aqrl a0, a1, (a0)
+; RV64IA-NEXT:    ret
   %1 = atomicrmw sub i32* %a, i32 %b acq_rel
   ret i32 %1
 }
@@ -5655,6 +10558,22 @@ define i32 @atomicrmw_sub_i32_seq_cst(i32* %a, i32 %b) {
 ; RV32IA-NEXT:    neg a1, a1
 ; RV32IA-NEXT:    amoadd.w.aqrl a0, a1, (a0)
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_sub_i32_seq_cst:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a2, zero, 5
+; RV64I-NEXT:    call __atomic_fetch_sub_4
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_sub_i32_seq_cst:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    neg a1, a1
+; RV64IA-NEXT:    amoadd.w.aqrl a0, a1, (a0)
+; RV64IA-NEXT:    ret
   %1 = atomicrmw sub i32* %a, i32 %b seq_cst
   ret i32 %1
 }
@@ -5674,6 +10593,21 @@ define i32 @atomicrmw_and_i32_monotonic(i32 *%a, i32 %b) nounwind {
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    amoand.w a0, a1, (a0)
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_and_i32_monotonic:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    mv a2, zero
+; RV64I-NEXT:    call __atomic_fetch_and_4
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_and_i32_monotonic:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    amoand.w a0, a1, (a0)
+; RV64IA-NEXT:    ret
   %1 = atomicrmw and i32* %a, i32 %b monotonic
   ret i32 %1
 }
@@ -5693,6 +10627,21 @@ define i32 @atomicrmw_and_i32_acquire(i32 *%a, i32 %b) nounwind {
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    amoand.w.aq a0, a1, (a0)
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_and_i32_acquire:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a2, zero, 2
+; RV64I-NEXT:    call __atomic_fetch_and_4
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_and_i32_acquire:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    amoand.w.aq a0, a1, (a0)
+; RV64IA-NEXT:    ret
   %1 = atomicrmw and i32* %a, i32 %b acquire
   ret i32 %1
 }
@@ -5712,6 +10661,21 @@ define i32 @atomicrmw_and_i32_release(i32 *%a, i32 %b) nounwind {
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    amoand.w.rl a0, a1, (a0)
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_and_i32_release:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a2, zero, 3
+; RV64I-NEXT:    call __atomic_fetch_and_4
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_and_i32_release:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    amoand.w.rl a0, a1, (a0)
+; RV64IA-NEXT:    ret
   %1 = atomicrmw and i32* %a, i32 %b release
   ret i32 %1
 }
@@ -5731,6 +10695,21 @@ define i32 @atomicrmw_and_i32_acq_rel(i32 *%a, i32 %b) nounwind {
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    amoand.w.aqrl a0, a1, (a0)
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_and_i32_acq_rel:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a2, zero, 4
+; RV64I-NEXT:    call __atomic_fetch_and_4
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_and_i32_acq_rel:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    amoand.w.aqrl a0, a1, (a0)
+; RV64IA-NEXT:    ret
   %1 = atomicrmw and i32* %a, i32 %b acq_rel
   ret i32 %1
 }
@@ -5750,6 +10729,21 @@ define i32 @atomicrmw_and_i32_seq_cst(i32 *%a, i32 %b) nounwind {
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    amoand.w.aqrl a0, a1, (a0)
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_and_i32_seq_cst:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a2, zero, 5
+; RV64I-NEXT:    call __atomic_fetch_and_4
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_and_i32_seq_cst:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    amoand.w.aqrl a0, a1, (a0)
+; RV64IA-NEXT:    ret
   %1 = atomicrmw and i32* %a, i32 %b seq_cst
   ret i32 %1
 }
@@ -5776,6 +10770,28 @@ define i32 @atomicrmw_nand_i32_monotonic(i32* %a, i32 %b) {
 ; RV32IA-NEXT:  # %bb.2:
 ; RV32IA-NEXT:    mv a0, a2
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_nand_i32_monotonic:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    mv a2, zero
+; RV64I-NEXT:    call __atomic_fetch_nand_4
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_nand_i32_monotonic:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:  .LBB130_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.w a2, (a0)
+; RV64IA-NEXT:    and a3, a2, a1
+; RV64IA-NEXT:    not a3, a3
+; RV64IA-NEXT:    sc.w a3, a3, (a0)
+; RV64IA-NEXT:    bnez a3, .LBB130_1
+; RV64IA-NEXT:  # %bb.2:
+; RV64IA-NEXT:    mv a0, a2
+; RV64IA-NEXT:    ret
   %1 = atomicrmw nand i32* %a, i32 %b monotonic
   ret i32 %1
 }
@@ -5802,6 +10818,28 @@ define i32 @atomicrmw_nand_i32_acquire(i32* %a, i32 %b) {
 ; RV32IA-NEXT:  # %bb.2:
 ; RV32IA-NEXT:    mv a0, a2
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_nand_i32_acquire:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a2, zero, 2
+; RV64I-NEXT:    call __atomic_fetch_nand_4
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_nand_i32_acquire:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:  .LBB131_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.w.aq a2, (a0)
+; RV64IA-NEXT:    and a3, a2, a1
+; RV64IA-NEXT:    not a3, a3
+; RV64IA-NEXT:    sc.w a3, a3, (a0)
+; RV64IA-NEXT:    bnez a3, .LBB131_1
+; RV64IA-NEXT:  # %bb.2:
+; RV64IA-NEXT:    mv a0, a2
+; RV64IA-NEXT:    ret
   %1 = atomicrmw nand i32* %a, i32 %b acquire
   ret i32 %1
 }
@@ -5828,6 +10866,28 @@ define i32 @atomicrmw_nand_i32_release(i32* %a, i32 %b) {
 ; RV32IA-NEXT:  # %bb.2:
 ; RV32IA-NEXT:    mv a0, a2
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_nand_i32_release:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a2, zero, 3
+; RV64I-NEXT:    call __atomic_fetch_nand_4
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_nand_i32_release:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:  .LBB132_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.w a2, (a0)
+; RV64IA-NEXT:    and a3, a2, a1
+; RV64IA-NEXT:    not a3, a3
+; RV64IA-NEXT:    sc.w.rl a3, a3, (a0)
+; RV64IA-NEXT:    bnez a3, .LBB132_1
+; RV64IA-NEXT:  # %bb.2:
+; RV64IA-NEXT:    mv a0, a2
+; RV64IA-NEXT:    ret
   %1 = atomicrmw nand i32* %a, i32 %b release
   ret i32 %1
 }
@@ -5854,6 +10914,28 @@ define i32 @atomicrmw_nand_i32_acq_rel(i32* %a, i32 %b) {
 ; RV32IA-NEXT:  # %bb.2:
 ; RV32IA-NEXT:    mv a0, a2
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_nand_i32_acq_rel:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a2, zero, 4
+; RV64I-NEXT:    call __atomic_fetch_nand_4
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_nand_i32_acq_rel:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:  .LBB133_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.w.aq a2, (a0)
+; RV64IA-NEXT:    and a3, a2, a1
+; RV64IA-NEXT:    not a3, a3
+; RV64IA-NEXT:    sc.w.rl a3, a3, (a0)
+; RV64IA-NEXT:    bnez a3, .LBB133_1
+; RV64IA-NEXT:  # %bb.2:
+; RV64IA-NEXT:    mv a0, a2
+; RV64IA-NEXT:    ret
   %1 = atomicrmw nand i32* %a, i32 %b acq_rel
   ret i32 %1
 }
@@ -5880,6 +10962,28 @@ define i32 @atomicrmw_nand_i32_seq_cst(i32* %a, i32 %b) {
 ; RV32IA-NEXT:  # %bb.2:
 ; RV32IA-NEXT:    mv a0, a2
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_nand_i32_seq_cst:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a2, zero, 5
+; RV64I-NEXT:    call __atomic_fetch_nand_4
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_nand_i32_seq_cst:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:  .LBB134_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.w.aqrl a2, (a0)
+; RV64IA-NEXT:    and a3, a2, a1
+; RV64IA-NEXT:    not a3, a3
+; RV64IA-NEXT:    sc.w.aqrl a3, a3, (a0)
+; RV64IA-NEXT:    bnez a3, .LBB134_1
+; RV64IA-NEXT:  # %bb.2:
+; RV64IA-NEXT:    mv a0, a2
+; RV64IA-NEXT:    ret
   %1 = atomicrmw nand i32* %a, i32 %b seq_cst
   ret i32 %1
 }
@@ -5899,6 +11003,21 @@ define i32 @atomicrmw_or_i32_monotonic(i32 *%a, i32 %b) nounwind {
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    amoor.w a0, a1, (a0)
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_or_i32_monotonic:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    mv a2, zero
+; RV64I-NEXT:    call __atomic_fetch_or_4
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_or_i32_monotonic:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    amoor.w a0, a1, (a0)
+; RV64IA-NEXT:    ret
   %1 = atomicrmw or i32* %a, i32 %b monotonic
   ret i32 %1
 }
@@ -5918,6 +11037,21 @@ define i32 @atomicrmw_or_i32_acquire(i32 *%a, i32 %b) nounwind {
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    amoor.w.aq a0, a1, (a0)
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_or_i32_acquire:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a2, zero, 2
+; RV64I-NEXT:    call __atomic_fetch_or_4
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_or_i32_acquire:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    amoor.w.aq a0, a1, (a0)
+; RV64IA-NEXT:    ret
   %1 = atomicrmw or i32* %a, i32 %b acquire
   ret i32 %1
 }
@@ -5937,6 +11071,21 @@ define i32 @atomicrmw_or_i32_release(i32 *%a, i32 %b) nounwind {
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    amoor.w.rl a0, a1, (a0)
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_or_i32_release:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a2, zero, 3
+; RV64I-NEXT:    call __atomic_fetch_or_4
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_or_i32_release:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    amoor.w.rl a0, a1, (a0)
+; RV64IA-NEXT:    ret
   %1 = atomicrmw or i32* %a, i32 %b release
   ret i32 %1
 }
@@ -5956,6 +11105,21 @@ define i32 @atomicrmw_or_i32_acq_rel(i32 *%a, i32 %b) nounwind {
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    amoor.w.aqrl a0, a1, (a0)
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_or_i32_acq_rel:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a2, zero, 4
+; RV64I-NEXT:    call __atomic_fetch_or_4
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_or_i32_acq_rel:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    amoor.w.aqrl a0, a1, (a0)
+; RV64IA-NEXT:    ret
   %1 = atomicrmw or i32* %a, i32 %b acq_rel
   ret i32 %1
 }
@@ -5975,6 +11139,21 @@ define i32 @atomicrmw_or_i32_seq_cst(i32 *%a, i32 %b) nounwind {
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    amoor.w.aqrl a0, a1, (a0)
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_or_i32_seq_cst:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a2, zero, 5
+; RV64I-NEXT:    call __atomic_fetch_or_4
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_or_i32_seq_cst:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    amoor.w.aqrl a0, a1, (a0)
+; RV64IA-NEXT:    ret
   %1 = atomicrmw or i32* %a, i32 %b seq_cst
   ret i32 %1
 }
@@ -5994,6 +11173,21 @@ define i32 @atomicrmw_xor_i32_monotonic(i32 *%a, i32 %b) nounwind {
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    amoxor.w a0, a1, (a0)
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_xor_i32_monotonic:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    mv a2, zero
+; RV64I-NEXT:    call __atomic_fetch_xor_4
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_xor_i32_monotonic:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    amoxor.w a0, a1, (a0)
+; RV64IA-NEXT:    ret
   %1 = atomicrmw xor i32* %a, i32 %b monotonic
   ret i32 %1
 }
@@ -6013,6 +11207,21 @@ define i32 @atomicrmw_xor_i32_acquire(i32 *%a, i32 %b) nounwind {
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    amoxor.w.aq a0, a1, (a0)
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_xor_i32_acquire:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a2, zero, 2
+; RV64I-NEXT:    call __atomic_fetch_xor_4
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_xor_i32_acquire:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    amoxor.w.aq a0, a1, (a0)
+; RV64IA-NEXT:    ret
   %1 = atomicrmw xor i32* %a, i32 %b acquire
   ret i32 %1
 }
@@ -6032,6 +11241,21 @@ define i32 @atomicrmw_xor_i32_release(i32 *%a, i32 %b) nounwind {
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    amoxor.w.rl a0, a1, (a0)
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_xor_i32_release:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a2, zero, 3
+; RV64I-NEXT:    call __atomic_fetch_xor_4
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_xor_i32_release:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    amoxor.w.rl a0, a1, (a0)
+; RV64IA-NEXT:    ret
   %1 = atomicrmw xor i32* %a, i32 %b release
   ret i32 %1
 }
@@ -6051,6 +11275,21 @@ define i32 @atomicrmw_xor_i32_acq_rel(i32 *%a, i32 %b) nounwind {
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    amoxor.w.aqrl a0, a1, (a0)
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_xor_i32_acq_rel:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a2, zero, 4
+; RV64I-NEXT:    call __atomic_fetch_xor_4
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_xor_i32_acq_rel:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    amoxor.w.aqrl a0, a1, (a0)
+; RV64IA-NEXT:    ret
   %1 = atomicrmw xor i32* %a, i32 %b acq_rel
   ret i32 %1
 }
@@ -6070,6 +11309,21 @@ define i32 @atomicrmw_xor_i32_seq_cst(i32 *%a, i32 %b) nounwind {
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    amoxor.w.aqrl a0, a1, (a0)
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_xor_i32_seq_cst:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a2, zero, 5
+; RV64I-NEXT:    call __atomic_fetch_xor_4
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_xor_i32_seq_cst:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    amoxor.w.aqrl a0, a1, (a0)
+; RV64IA-NEXT:    ret
   %1 = atomicrmw xor i32* %a, i32 %b seq_cst
   ret i32 %1
 }
@@ -6115,6 +11369,52 @@ define i32 @atomicrmw_max_i32_monotonic(i32 *%a, i32 %b) nounwind {
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    amomax.w a0, a1, (a0)
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_max_i32_monotonic:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -48
+; RV64I-NEXT:    sd ra, 40(sp)
+; RV64I-NEXT:    sd s1, 32(sp)
+; RV64I-NEXT:    sd s2, 24(sp)
+; RV64I-NEXT:    sd s3, 16(sp)
+; RV64I-NEXT:    sd s4, 8(sp)
+; RV64I-NEXT:    mv s2, a1
+; RV64I-NEXT:    mv s4, a0
+; RV64I-NEXT:    lwu a0, 0(a0)
+; RV64I-NEXT:    sext.w s1, a1
+; RV64I-NEXT:    addi s3, sp, 4
+; RV64I-NEXT:  .LBB145_1: # %atomicrmw.start
+; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV64I-NEXT:    sext.w a1, a0
+; RV64I-NEXT:    mv a2, a0
+; RV64I-NEXT:    blt s1, a1, .LBB145_3
+; RV64I-NEXT:  # %bb.2: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB145_1 Depth=1
+; RV64I-NEXT:    mv a2, s2
+; RV64I-NEXT:  .LBB145_3: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB145_1 Depth=1
+; RV64I-NEXT:    sw a0, 4(sp)
+; RV64I-NEXT:    mv a0, s4
+; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    mv a3, zero
+; RV64I-NEXT:    mv a4, zero
+; RV64I-NEXT:    call __atomic_compare_exchange_4
+; RV64I-NEXT:    mv a1, a0
+; RV64I-NEXT:    lw a0, 4(sp)
+; RV64I-NEXT:    beqz a1, .LBB145_1
+; RV64I-NEXT:  # %bb.4: # %atomicrmw.end
+; RV64I-NEXT:    ld s4, 8(sp)
+; RV64I-NEXT:    ld s3, 16(sp)
+; RV64I-NEXT:    ld s2, 24(sp)
+; RV64I-NEXT:    ld s1, 32(sp)
+; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    addi sp, sp, 48
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_max_i32_monotonic:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    amomax.w a0, a1, (a0)
+; RV64IA-NEXT:    ret
   %1 = atomicrmw max i32* %a, i32 %b monotonic
   ret i32 %1
 }
@@ -6127,15 +11427,13 @@ define i32 @atomicrmw_max_i32_acquire(i32 *%a, i32 %b) nounwind {
 ; RV32I-NEXT:    sw s1, 24(sp)
 ; RV32I-NEXT:    sw s2, 20(sp)
 ; RV32I-NEXT:    sw s3, 16(sp)
-; RV32I-NEXT:    sw s4, 12(sp)
 ; RV32I-NEXT:    mv s1, a1
 ; RV32I-NEXT:    mv s2, a0
 ; RV32I-NEXT:    lw a2, 0(a0)
-; RV32I-NEXT:    addi s3, sp, 8
-; RV32I-NEXT:    addi s4, zero, 2
+; RV32I-NEXT:    addi s3, sp, 12
 ; RV32I-NEXT:  .LBB146_1: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    sw a2, 8(sp)
+; RV32I-NEXT:    sw a2, 12(sp)
 ; RV32I-NEXT:    blt s1, a2, .LBB146_3
 ; RV32I-NEXT:  # %bb.2: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB146_1 Depth=1
@@ -6144,14 +11442,13 @@ define i32 @atomicrmw_max_i32_acquire(i32 *%a, i32 %b) nounwind {
 ; RV32I-NEXT:    # in Loop: Header=BB146_1 Depth=1
 ; RV32I-NEXT:    mv a0, s2
 ; RV32I-NEXT:    mv a1, s3
-; RV32I-NEXT:    mv a3, s4
-; RV32I-NEXT:    mv a4, s4
+; RV32I-NEXT:    addi a3, zero, 2
+; RV32I-NEXT:    addi a4, zero, 2
 ; RV32I-NEXT:    call __atomic_compare_exchange_4
-; RV32I-NEXT:    lw a2, 8(sp)
+; RV32I-NEXT:    lw a2, 12(sp)
 ; RV32I-NEXT:    beqz a0, .LBB146_1
 ; RV32I-NEXT:  # %bb.4: # %atomicrmw.end
 ; RV32I-NEXT:    mv a0, a2
-; RV32I-NEXT:    lw s4, 12(sp)
 ; RV32I-NEXT:    lw s3, 16(sp)
 ; RV32I-NEXT:    lw s2, 20(sp)
 ; RV32I-NEXT:    lw s1, 24(sp)
@@ -6163,6 +11460,52 @@ define i32 @atomicrmw_max_i32_acquire(i32 *%a, i32 %b) nounwind {
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    amomax.w.aq a0, a1, (a0)
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_max_i32_acquire:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -48
+; RV64I-NEXT:    sd ra, 40(sp)
+; RV64I-NEXT:    sd s1, 32(sp)
+; RV64I-NEXT:    sd s2, 24(sp)
+; RV64I-NEXT:    sd s3, 16(sp)
+; RV64I-NEXT:    sd s4, 8(sp)
+; RV64I-NEXT:    mv s2, a1
+; RV64I-NEXT:    mv s4, a0
+; RV64I-NEXT:    lwu a0, 0(a0)
+; RV64I-NEXT:    sext.w s1, a1
+; RV64I-NEXT:    addi s3, sp, 4
+; RV64I-NEXT:  .LBB146_1: # %atomicrmw.start
+; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV64I-NEXT:    sext.w a1, a0
+; RV64I-NEXT:    mv a2, a0
+; RV64I-NEXT:    blt s1, a1, .LBB146_3
+; RV64I-NEXT:  # %bb.2: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB146_1 Depth=1
+; RV64I-NEXT:    mv a2, s2
+; RV64I-NEXT:  .LBB146_3: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB146_1 Depth=1
+; RV64I-NEXT:    sw a0, 4(sp)
+; RV64I-NEXT:    mv a0, s4
+; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    addi a3, zero, 2
+; RV64I-NEXT:    addi a4, zero, 2
+; RV64I-NEXT:    call __atomic_compare_exchange_4
+; RV64I-NEXT:    mv a1, a0
+; RV64I-NEXT:    lw a0, 4(sp)
+; RV64I-NEXT:    beqz a1, .LBB146_1
+; RV64I-NEXT:  # %bb.4: # %atomicrmw.end
+; RV64I-NEXT:    ld s4, 8(sp)
+; RV64I-NEXT:    ld s3, 16(sp)
+; RV64I-NEXT:    ld s2, 24(sp)
+; RV64I-NEXT:    ld s1, 32(sp)
+; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    addi sp, sp, 48
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_max_i32_acquire:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    amomax.w.aq a0, a1, (a0)
+; RV64IA-NEXT:    ret
   %1 = atomicrmw max i32* %a, i32 %b acquire
   ret i32 %1
 }
@@ -6175,15 +11518,13 @@ define i32 @atomicrmw_max_i32_release(i32 *%a, i32 %b) nounwind {
 ; RV32I-NEXT:    sw s1, 24(sp)
 ; RV32I-NEXT:    sw s2, 20(sp)
 ; RV32I-NEXT:    sw s3, 16(sp)
-; RV32I-NEXT:    sw s4, 12(sp)
 ; RV32I-NEXT:    mv s1, a1
 ; RV32I-NEXT:    mv s2, a0
 ; RV32I-NEXT:    lw a2, 0(a0)
-; RV32I-NEXT:    addi s3, sp, 8
-; RV32I-NEXT:    addi s4, zero, 3
+; RV32I-NEXT:    addi s3, sp, 12
 ; RV32I-NEXT:  .LBB147_1: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    sw a2, 8(sp)
+; RV32I-NEXT:    sw a2, 12(sp)
 ; RV32I-NEXT:    blt s1, a2, .LBB147_3
 ; RV32I-NEXT:  # %bb.2: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB147_1 Depth=1
@@ -6192,14 +11533,13 @@ define i32 @atomicrmw_max_i32_release(i32 *%a, i32 %b) nounwind {
 ; RV32I-NEXT:    # in Loop: Header=BB147_1 Depth=1
 ; RV32I-NEXT:    mv a0, s2
 ; RV32I-NEXT:    mv a1, s3
-; RV32I-NEXT:    mv a3, s4
+; RV32I-NEXT:    addi a3, zero, 3
 ; RV32I-NEXT:    mv a4, zero
 ; RV32I-NEXT:    call __atomic_compare_exchange_4
-; RV32I-NEXT:    lw a2, 8(sp)
+; RV32I-NEXT:    lw a2, 12(sp)
 ; RV32I-NEXT:    beqz a0, .LBB147_1
 ; RV32I-NEXT:  # %bb.4: # %atomicrmw.end
 ; RV32I-NEXT:    mv a0, a2
-; RV32I-NEXT:    lw s4, 12(sp)
 ; RV32I-NEXT:    lw s3, 16(sp)
 ; RV32I-NEXT:    lw s2, 20(sp)
 ; RV32I-NEXT:    lw s1, 24(sp)
@@ -6211,6 +11551,52 @@ define i32 @atomicrmw_max_i32_release(i32 *%a, i32 %b) nounwind {
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    amomax.w.rl a0, a1, (a0)
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_max_i32_release:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -48
+; RV64I-NEXT:    sd ra, 40(sp)
+; RV64I-NEXT:    sd s1, 32(sp)
+; RV64I-NEXT:    sd s2, 24(sp)
+; RV64I-NEXT:    sd s3, 16(sp)
+; RV64I-NEXT:    sd s4, 8(sp)
+; RV64I-NEXT:    mv s2, a1
+; RV64I-NEXT:    mv s4, a0
+; RV64I-NEXT:    lwu a0, 0(a0)
+; RV64I-NEXT:    sext.w s1, a1
+; RV64I-NEXT:    addi s3, sp, 4
+; RV64I-NEXT:  .LBB147_1: # %atomicrmw.start
+; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV64I-NEXT:    sext.w a1, a0
+; RV64I-NEXT:    mv a2, a0
+; RV64I-NEXT:    blt s1, a1, .LBB147_3
+; RV64I-NEXT:  # %bb.2: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB147_1 Depth=1
+; RV64I-NEXT:    mv a2, s2
+; RV64I-NEXT:  .LBB147_3: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB147_1 Depth=1
+; RV64I-NEXT:    sw a0, 4(sp)
+; RV64I-NEXT:    mv a0, s4
+; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    addi a3, zero, 3
+; RV64I-NEXT:    mv a4, zero
+; RV64I-NEXT:    call __atomic_compare_exchange_4
+; RV64I-NEXT:    mv a1, a0
+; RV64I-NEXT:    lw a0, 4(sp)
+; RV64I-NEXT:    beqz a1, .LBB147_1
+; RV64I-NEXT:  # %bb.4: # %atomicrmw.end
+; RV64I-NEXT:    ld s4, 8(sp)
+; RV64I-NEXT:    ld s3, 16(sp)
+; RV64I-NEXT:    ld s2, 24(sp)
+; RV64I-NEXT:    ld s1, 32(sp)
+; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    addi sp, sp, 48
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_max_i32_release:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    amomax.w.rl a0, a1, (a0)
+; RV64IA-NEXT:    ret
   %1 = atomicrmw max i32* %a, i32 %b release
   ret i32 %1
 }
@@ -6223,17 +11609,13 @@ define i32 @atomicrmw_max_i32_acq_rel(i32 *%a, i32 %b) nounwind {
 ; RV32I-NEXT:    sw s1, 24(sp)
 ; RV32I-NEXT:    sw s2, 20(sp)
 ; RV32I-NEXT:    sw s3, 16(sp)
-; RV32I-NEXT:    sw s4, 12(sp)
-; RV32I-NEXT:    sw s5, 8(sp)
 ; RV32I-NEXT:    mv s1, a1
 ; RV32I-NEXT:    mv s2, a0
 ; RV32I-NEXT:    lw a2, 0(a0)
-; RV32I-NEXT:    addi s3, sp, 4
-; RV32I-NEXT:    addi s4, zero, 4
-; RV32I-NEXT:    addi s5, zero, 2
+; RV32I-NEXT:    addi s3, sp, 12
 ; RV32I-NEXT:  .LBB148_1: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    sw a2, 4(sp)
+; RV32I-NEXT:    sw a2, 12(sp)
 ; RV32I-NEXT:    blt s1, a2, .LBB148_3
 ; RV32I-NEXT:  # %bb.2: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB148_1 Depth=1
@@ -6242,15 +11624,13 @@ define i32 @atomicrmw_max_i32_acq_rel(i32 *%a, i32 %b) nounwind {
 ; RV32I-NEXT:    # in Loop: Header=BB148_1 Depth=1
 ; RV32I-NEXT:    mv a0, s2
 ; RV32I-NEXT:    mv a1, s3
-; RV32I-NEXT:    mv a3, s4
-; RV32I-NEXT:    mv a4, s5
+; RV32I-NEXT:    addi a3, zero, 4
+; RV32I-NEXT:    addi a4, zero, 2
 ; RV32I-NEXT:    call __atomic_compare_exchange_4
-; RV32I-NEXT:    lw a2, 4(sp)
+; RV32I-NEXT:    lw a2, 12(sp)
 ; RV32I-NEXT:    beqz a0, .LBB148_1
 ; RV32I-NEXT:  # %bb.4: # %atomicrmw.end
 ; RV32I-NEXT:    mv a0, a2
-; RV32I-NEXT:    lw s5, 8(sp)
-; RV32I-NEXT:    lw s4, 12(sp)
 ; RV32I-NEXT:    lw s3, 16(sp)
 ; RV32I-NEXT:    lw s2, 20(sp)
 ; RV32I-NEXT:    lw s1, 24(sp)
@@ -6262,6 +11642,52 @@ define i32 @atomicrmw_max_i32_acq_rel(i32 *%a, i32 %b) nounwind {
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    amomax.w.aqrl a0, a1, (a0)
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_max_i32_acq_rel:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -48
+; RV64I-NEXT:    sd ra, 40(sp)
+; RV64I-NEXT:    sd s1, 32(sp)
+; RV64I-NEXT:    sd s2, 24(sp)
+; RV64I-NEXT:    sd s3, 16(sp)
+; RV64I-NEXT:    sd s4, 8(sp)
+; RV64I-NEXT:    mv s2, a1
+; RV64I-NEXT:    mv s4, a0
+; RV64I-NEXT:    lwu a0, 0(a0)
+; RV64I-NEXT:    sext.w s1, a1
+; RV64I-NEXT:    addi s3, sp, 4
+; RV64I-NEXT:  .LBB148_1: # %atomicrmw.start
+; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV64I-NEXT:    sext.w a1, a0
+; RV64I-NEXT:    mv a2, a0
+; RV64I-NEXT:    blt s1, a1, .LBB148_3
+; RV64I-NEXT:  # %bb.2: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB148_1 Depth=1
+; RV64I-NEXT:    mv a2, s2
+; RV64I-NEXT:  .LBB148_3: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB148_1 Depth=1
+; RV64I-NEXT:    sw a0, 4(sp)
+; RV64I-NEXT:    mv a0, s4
+; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    addi a3, zero, 4
+; RV64I-NEXT:    addi a4, zero, 2
+; RV64I-NEXT:    call __atomic_compare_exchange_4
+; RV64I-NEXT:    mv a1, a0
+; RV64I-NEXT:    lw a0, 4(sp)
+; RV64I-NEXT:    beqz a1, .LBB148_1
+; RV64I-NEXT:  # %bb.4: # %atomicrmw.end
+; RV64I-NEXT:    ld s4, 8(sp)
+; RV64I-NEXT:    ld s3, 16(sp)
+; RV64I-NEXT:    ld s2, 24(sp)
+; RV64I-NEXT:    ld s1, 32(sp)
+; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    addi sp, sp, 48
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_max_i32_acq_rel:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    amomax.w.aqrl a0, a1, (a0)
+; RV64IA-NEXT:    ret
   %1 = atomicrmw max i32* %a, i32 %b acq_rel
   ret i32 %1
 }
@@ -6274,15 +11700,13 @@ define i32 @atomicrmw_max_i32_seq_cst(i32 *%a, i32 %b) nounwind {
 ; RV32I-NEXT:    sw s1, 24(sp)
 ; RV32I-NEXT:    sw s2, 20(sp)
 ; RV32I-NEXT:    sw s3, 16(sp)
-; RV32I-NEXT:    sw s4, 12(sp)
 ; RV32I-NEXT:    mv s1, a1
 ; RV32I-NEXT:    mv s2, a0
 ; RV32I-NEXT:    lw a2, 0(a0)
-; RV32I-NEXT:    addi s3, sp, 8
-; RV32I-NEXT:    addi s4, zero, 5
+; RV32I-NEXT:    addi s3, sp, 12
 ; RV32I-NEXT:  .LBB149_1: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    sw a2, 8(sp)
+; RV32I-NEXT:    sw a2, 12(sp)
 ; RV32I-NEXT:    blt s1, a2, .LBB149_3
 ; RV32I-NEXT:  # %bb.2: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB149_1 Depth=1
@@ -6291,14 +11715,13 @@ define i32 @atomicrmw_max_i32_seq_cst(i32 *%a, i32 %b) nounwind {
 ; RV32I-NEXT:    # in Loop: Header=BB149_1 Depth=1
 ; RV32I-NEXT:    mv a0, s2
 ; RV32I-NEXT:    mv a1, s3
-; RV32I-NEXT:    mv a3, s4
-; RV32I-NEXT:    mv a4, s4
+; RV32I-NEXT:    addi a3, zero, 5
+; RV32I-NEXT:    addi a4, zero, 5
 ; RV32I-NEXT:    call __atomic_compare_exchange_4
-; RV32I-NEXT:    lw a2, 8(sp)
+; RV32I-NEXT:    lw a2, 12(sp)
 ; RV32I-NEXT:    beqz a0, .LBB149_1
 ; RV32I-NEXT:  # %bb.4: # %atomicrmw.end
 ; RV32I-NEXT:    mv a0, a2
-; RV32I-NEXT:    lw s4, 12(sp)
 ; RV32I-NEXT:    lw s3, 16(sp)
 ; RV32I-NEXT:    lw s2, 20(sp)
 ; RV32I-NEXT:    lw s1, 24(sp)
@@ -6310,6 +11733,52 @@ define i32 @atomicrmw_max_i32_seq_cst(i32 *%a, i32 %b) nounwind {
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    amomax.w.aqrl a0, a1, (a0)
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_max_i32_seq_cst:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -48
+; RV64I-NEXT:    sd ra, 40(sp)
+; RV64I-NEXT:    sd s1, 32(sp)
+; RV64I-NEXT:    sd s2, 24(sp)
+; RV64I-NEXT:    sd s3, 16(sp)
+; RV64I-NEXT:    sd s4, 8(sp)
+; RV64I-NEXT:    mv s2, a1
+; RV64I-NEXT:    mv s4, a0
+; RV64I-NEXT:    lwu a0, 0(a0)
+; RV64I-NEXT:    sext.w s1, a1
+; RV64I-NEXT:    addi s3, sp, 4
+; RV64I-NEXT:  .LBB149_1: # %atomicrmw.start
+; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV64I-NEXT:    sext.w a1, a0
+; RV64I-NEXT:    mv a2, a0
+; RV64I-NEXT:    blt s1, a1, .LBB149_3
+; RV64I-NEXT:  # %bb.2: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB149_1 Depth=1
+; RV64I-NEXT:    mv a2, s2
+; RV64I-NEXT:  .LBB149_3: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB149_1 Depth=1
+; RV64I-NEXT:    sw a0, 4(sp)
+; RV64I-NEXT:    mv a0, s4
+; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    addi a3, zero, 5
+; RV64I-NEXT:    addi a4, zero, 5
+; RV64I-NEXT:    call __atomic_compare_exchange_4
+; RV64I-NEXT:    mv a1, a0
+; RV64I-NEXT:    lw a0, 4(sp)
+; RV64I-NEXT:    beqz a1, .LBB149_1
+; RV64I-NEXT:  # %bb.4: # %atomicrmw.end
+; RV64I-NEXT:    ld s4, 8(sp)
+; RV64I-NEXT:    ld s3, 16(sp)
+; RV64I-NEXT:    ld s2, 24(sp)
+; RV64I-NEXT:    ld s1, 32(sp)
+; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    addi sp, sp, 48
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_max_i32_seq_cst:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    amomax.w.aqrl a0, a1, (a0)
+; RV64IA-NEXT:    ret
   %1 = atomicrmw max i32* %a, i32 %b seq_cst
   ret i32 %1
 }
@@ -6355,6 +11824,52 @@ define i32 @atomicrmw_min_i32_monotonic(i32 *%a, i32 %b) nounwind {
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    amomin.w a0, a1, (a0)
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_min_i32_monotonic:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -48
+; RV64I-NEXT:    sd ra, 40(sp)
+; RV64I-NEXT:    sd s1, 32(sp)
+; RV64I-NEXT:    sd s2, 24(sp)
+; RV64I-NEXT:    sd s3, 16(sp)
+; RV64I-NEXT:    sd s4, 8(sp)
+; RV64I-NEXT:    mv s2, a1
+; RV64I-NEXT:    mv s4, a0
+; RV64I-NEXT:    lwu a0, 0(a0)
+; RV64I-NEXT:    sext.w s1, a1
+; RV64I-NEXT:    addi s3, sp, 4
+; RV64I-NEXT:  .LBB150_1: # %atomicrmw.start
+; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV64I-NEXT:    sext.w a1, a0
+; RV64I-NEXT:    mv a2, a0
+; RV64I-NEXT:    bge s1, a1, .LBB150_3
+; RV64I-NEXT:  # %bb.2: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB150_1 Depth=1
+; RV64I-NEXT:    mv a2, s2
+; RV64I-NEXT:  .LBB150_3: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB150_1 Depth=1
+; RV64I-NEXT:    sw a0, 4(sp)
+; RV64I-NEXT:    mv a0, s4
+; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    mv a3, zero
+; RV64I-NEXT:    mv a4, zero
+; RV64I-NEXT:    call __atomic_compare_exchange_4
+; RV64I-NEXT:    mv a1, a0
+; RV64I-NEXT:    lw a0, 4(sp)
+; RV64I-NEXT:    beqz a1, .LBB150_1
+; RV64I-NEXT:  # %bb.4: # %atomicrmw.end
+; RV64I-NEXT:    ld s4, 8(sp)
+; RV64I-NEXT:    ld s3, 16(sp)
+; RV64I-NEXT:    ld s2, 24(sp)
+; RV64I-NEXT:    ld s1, 32(sp)
+; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    addi sp, sp, 48
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_min_i32_monotonic:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    amomin.w a0, a1, (a0)
+; RV64IA-NEXT:    ret
   %1 = atomicrmw min i32* %a, i32 %b monotonic
   ret i32 %1
 }
@@ -6367,15 +11882,13 @@ define i32 @atomicrmw_min_i32_acquire(i32 *%a, i32 %b) nounwind {
 ; RV32I-NEXT:    sw s1, 24(sp)
 ; RV32I-NEXT:    sw s2, 20(sp)
 ; RV32I-NEXT:    sw s3, 16(sp)
-; RV32I-NEXT:    sw s4, 12(sp)
 ; RV32I-NEXT:    mv s1, a1
 ; RV32I-NEXT:    mv s2, a0
 ; RV32I-NEXT:    lw a2, 0(a0)
-; RV32I-NEXT:    addi s3, sp, 8
-; RV32I-NEXT:    addi s4, zero, 2
+; RV32I-NEXT:    addi s3, sp, 12
 ; RV32I-NEXT:  .LBB151_1: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    sw a2, 8(sp)
+; RV32I-NEXT:    sw a2, 12(sp)
 ; RV32I-NEXT:    bge s1, a2, .LBB151_3
 ; RV32I-NEXT:  # %bb.2: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB151_1 Depth=1
@@ -6384,14 +11897,13 @@ define i32 @atomicrmw_min_i32_acquire(i32 *%a, i32 %b) nounwind {
 ; RV32I-NEXT:    # in Loop: Header=BB151_1 Depth=1
 ; RV32I-NEXT:    mv a0, s2
 ; RV32I-NEXT:    mv a1, s3
-; RV32I-NEXT:    mv a3, s4
-; RV32I-NEXT:    mv a4, s4
+; RV32I-NEXT:    addi a3, zero, 2
+; RV32I-NEXT:    addi a4, zero, 2
 ; RV32I-NEXT:    call __atomic_compare_exchange_4
-; RV32I-NEXT:    lw a2, 8(sp)
+; RV32I-NEXT:    lw a2, 12(sp)
 ; RV32I-NEXT:    beqz a0, .LBB151_1
 ; RV32I-NEXT:  # %bb.4: # %atomicrmw.end
 ; RV32I-NEXT:    mv a0, a2
-; RV32I-NEXT:    lw s4, 12(sp)
 ; RV32I-NEXT:    lw s3, 16(sp)
 ; RV32I-NEXT:    lw s2, 20(sp)
 ; RV32I-NEXT:    lw s1, 24(sp)
@@ -6403,6 +11915,52 @@ define i32 @atomicrmw_min_i32_acquire(i32 *%a, i32 %b) nounwind {
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    amomin.w.aq a0, a1, (a0)
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_min_i32_acquire:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -48
+; RV64I-NEXT:    sd ra, 40(sp)
+; RV64I-NEXT:    sd s1, 32(sp)
+; RV64I-NEXT:    sd s2, 24(sp)
+; RV64I-NEXT:    sd s3, 16(sp)
+; RV64I-NEXT:    sd s4, 8(sp)
+; RV64I-NEXT:    mv s2, a1
+; RV64I-NEXT:    mv s4, a0
+; RV64I-NEXT:    lwu a0, 0(a0)
+; RV64I-NEXT:    sext.w s1, a1
+; RV64I-NEXT:    addi s3, sp, 4
+; RV64I-NEXT:  .LBB151_1: # %atomicrmw.start
+; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV64I-NEXT:    sext.w a1, a0
+; RV64I-NEXT:    mv a2, a0
+; RV64I-NEXT:    bge s1, a1, .LBB151_3
+; RV64I-NEXT:  # %bb.2: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB151_1 Depth=1
+; RV64I-NEXT:    mv a2, s2
+; RV64I-NEXT:  .LBB151_3: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB151_1 Depth=1
+; RV64I-NEXT:    sw a0, 4(sp)
+; RV64I-NEXT:    mv a0, s4
+; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    addi a3, zero, 2
+; RV64I-NEXT:    addi a4, zero, 2
+; RV64I-NEXT:    call __atomic_compare_exchange_4
+; RV64I-NEXT:    mv a1, a0
+; RV64I-NEXT:    lw a0, 4(sp)
+; RV64I-NEXT:    beqz a1, .LBB151_1
+; RV64I-NEXT:  # %bb.4: # %atomicrmw.end
+; RV64I-NEXT:    ld s4, 8(sp)
+; RV64I-NEXT:    ld s3, 16(sp)
+; RV64I-NEXT:    ld s2, 24(sp)
+; RV64I-NEXT:    ld s1, 32(sp)
+; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    addi sp, sp, 48
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_min_i32_acquire:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    amomin.w.aq a0, a1, (a0)
+; RV64IA-NEXT:    ret
   %1 = atomicrmw min i32* %a, i32 %b acquire
   ret i32 %1
 }
@@ -6415,15 +11973,13 @@ define i32 @atomicrmw_min_i32_release(i32 *%a, i32 %b) nounwind {
 ; RV32I-NEXT:    sw s1, 24(sp)
 ; RV32I-NEXT:    sw s2, 20(sp)
 ; RV32I-NEXT:    sw s3, 16(sp)
-; RV32I-NEXT:    sw s4, 12(sp)
 ; RV32I-NEXT:    mv s1, a1
 ; RV32I-NEXT:    mv s2, a0
 ; RV32I-NEXT:    lw a2, 0(a0)
-; RV32I-NEXT:    addi s3, sp, 8
-; RV32I-NEXT:    addi s4, zero, 3
+; RV32I-NEXT:    addi s3, sp, 12
 ; RV32I-NEXT:  .LBB152_1: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    sw a2, 8(sp)
+; RV32I-NEXT:    sw a2, 12(sp)
 ; RV32I-NEXT:    bge s1, a2, .LBB152_3
 ; RV32I-NEXT:  # %bb.2: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB152_1 Depth=1
@@ -6432,14 +11988,13 @@ define i32 @atomicrmw_min_i32_release(i32 *%a, i32 %b) nounwind {
 ; RV32I-NEXT:    # in Loop: Header=BB152_1 Depth=1
 ; RV32I-NEXT:    mv a0, s2
 ; RV32I-NEXT:    mv a1, s3
-; RV32I-NEXT:    mv a3, s4
+; RV32I-NEXT:    addi a3, zero, 3
 ; RV32I-NEXT:    mv a4, zero
 ; RV32I-NEXT:    call __atomic_compare_exchange_4
-; RV32I-NEXT:    lw a2, 8(sp)
+; RV32I-NEXT:    lw a2, 12(sp)
 ; RV32I-NEXT:    beqz a0, .LBB152_1
 ; RV32I-NEXT:  # %bb.4: # %atomicrmw.end
 ; RV32I-NEXT:    mv a0, a2
-; RV32I-NEXT:    lw s4, 12(sp)
 ; RV32I-NEXT:    lw s3, 16(sp)
 ; RV32I-NEXT:    lw s2, 20(sp)
 ; RV32I-NEXT:    lw s1, 24(sp)
@@ -6451,6 +12006,52 @@ define i32 @atomicrmw_min_i32_release(i32 *%a, i32 %b) nounwind {
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    amomin.w.rl a0, a1, (a0)
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_min_i32_release:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -48
+; RV64I-NEXT:    sd ra, 40(sp)
+; RV64I-NEXT:    sd s1, 32(sp)
+; RV64I-NEXT:    sd s2, 24(sp)
+; RV64I-NEXT:    sd s3, 16(sp)
+; RV64I-NEXT:    sd s4, 8(sp)
+; RV64I-NEXT:    mv s2, a1
+; RV64I-NEXT:    mv s4, a0
+; RV64I-NEXT:    lwu a0, 0(a0)
+; RV64I-NEXT:    sext.w s1, a1
+; RV64I-NEXT:    addi s3, sp, 4
+; RV64I-NEXT:  .LBB152_1: # %atomicrmw.start
+; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV64I-NEXT:    sext.w a1, a0
+; RV64I-NEXT:    mv a2, a0
+; RV64I-NEXT:    bge s1, a1, .LBB152_3
+; RV64I-NEXT:  # %bb.2: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB152_1 Depth=1
+; RV64I-NEXT:    mv a2, s2
+; RV64I-NEXT:  .LBB152_3: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB152_1 Depth=1
+; RV64I-NEXT:    sw a0, 4(sp)
+; RV64I-NEXT:    mv a0, s4
+; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    addi a3, zero, 3
+; RV64I-NEXT:    mv a4, zero
+; RV64I-NEXT:    call __atomic_compare_exchange_4
+; RV64I-NEXT:    mv a1, a0
+; RV64I-NEXT:    lw a0, 4(sp)
+; RV64I-NEXT:    beqz a1, .LBB152_1
+; RV64I-NEXT:  # %bb.4: # %atomicrmw.end
+; RV64I-NEXT:    ld s4, 8(sp)
+; RV64I-NEXT:    ld s3, 16(sp)
+; RV64I-NEXT:    ld s2, 24(sp)
+; RV64I-NEXT:    ld s1, 32(sp)
+; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    addi sp, sp, 48
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_min_i32_release:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    amomin.w.rl a0, a1, (a0)
+; RV64IA-NEXT:    ret
   %1 = atomicrmw min i32* %a, i32 %b release
   ret i32 %1
 }
@@ -6463,17 +12064,13 @@ define i32 @atomicrmw_min_i32_acq_rel(i32 *%a, i32 %b) nounwind {
 ; RV32I-NEXT:    sw s1, 24(sp)
 ; RV32I-NEXT:    sw s2, 20(sp)
 ; RV32I-NEXT:    sw s3, 16(sp)
-; RV32I-NEXT:    sw s4, 12(sp)
-; RV32I-NEXT:    sw s5, 8(sp)
 ; RV32I-NEXT:    mv s1, a1
 ; RV32I-NEXT:    mv s2, a0
 ; RV32I-NEXT:    lw a2, 0(a0)
-; RV32I-NEXT:    addi s3, sp, 4
-; RV32I-NEXT:    addi s4, zero, 4
-; RV32I-NEXT:    addi s5, zero, 2
+; RV32I-NEXT:    addi s3, sp, 12
 ; RV32I-NEXT:  .LBB153_1: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    sw a2, 4(sp)
+; RV32I-NEXT:    sw a2, 12(sp)
 ; RV32I-NEXT:    bge s1, a2, .LBB153_3
 ; RV32I-NEXT:  # %bb.2: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB153_1 Depth=1
@@ -6482,15 +12079,13 @@ define i32 @atomicrmw_min_i32_acq_rel(i32 *%a, i32 %b) nounwind {
 ; RV32I-NEXT:    # in Loop: Header=BB153_1 Depth=1
 ; RV32I-NEXT:    mv a0, s2
 ; RV32I-NEXT:    mv a1, s3
-; RV32I-NEXT:    mv a3, s4
-; RV32I-NEXT:    mv a4, s5
+; RV32I-NEXT:    addi a3, zero, 4
+; RV32I-NEXT:    addi a4, zero, 2
 ; RV32I-NEXT:    call __atomic_compare_exchange_4
-; RV32I-NEXT:    lw a2, 4(sp)
+; RV32I-NEXT:    lw a2, 12(sp)
 ; RV32I-NEXT:    beqz a0, .LBB153_1
 ; RV32I-NEXT:  # %bb.4: # %atomicrmw.end
 ; RV32I-NEXT:    mv a0, a2
-; RV32I-NEXT:    lw s5, 8(sp)
-; RV32I-NEXT:    lw s4, 12(sp)
 ; RV32I-NEXT:    lw s3, 16(sp)
 ; RV32I-NEXT:    lw s2, 20(sp)
 ; RV32I-NEXT:    lw s1, 24(sp)
@@ -6502,6 +12097,52 @@ define i32 @atomicrmw_min_i32_acq_rel(i32 *%a, i32 %b) nounwind {
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    amomin.w.aqrl a0, a1, (a0)
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_min_i32_acq_rel:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -48
+; RV64I-NEXT:    sd ra, 40(sp)
+; RV64I-NEXT:    sd s1, 32(sp)
+; RV64I-NEXT:    sd s2, 24(sp)
+; RV64I-NEXT:    sd s3, 16(sp)
+; RV64I-NEXT:    sd s4, 8(sp)
+; RV64I-NEXT:    mv s2, a1
+; RV64I-NEXT:    mv s4, a0
+; RV64I-NEXT:    lwu a0, 0(a0)
+; RV64I-NEXT:    sext.w s1, a1
+; RV64I-NEXT:    addi s3, sp, 4
+; RV64I-NEXT:  .LBB153_1: # %atomicrmw.start
+; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV64I-NEXT:    sext.w a1, a0
+; RV64I-NEXT:    mv a2, a0
+; RV64I-NEXT:    bge s1, a1, .LBB153_3
+; RV64I-NEXT:  # %bb.2: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB153_1 Depth=1
+; RV64I-NEXT:    mv a2, s2
+; RV64I-NEXT:  .LBB153_3: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB153_1 Depth=1
+; RV64I-NEXT:    sw a0, 4(sp)
+; RV64I-NEXT:    mv a0, s4
+; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    addi a3, zero, 4
+; RV64I-NEXT:    addi a4, zero, 2
+; RV64I-NEXT:    call __atomic_compare_exchange_4
+; RV64I-NEXT:    mv a1, a0
+; RV64I-NEXT:    lw a0, 4(sp)
+; RV64I-NEXT:    beqz a1, .LBB153_1
+; RV64I-NEXT:  # %bb.4: # %atomicrmw.end
+; RV64I-NEXT:    ld s4, 8(sp)
+; RV64I-NEXT:    ld s3, 16(sp)
+; RV64I-NEXT:    ld s2, 24(sp)
+; RV64I-NEXT:    ld s1, 32(sp)
+; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    addi sp, sp, 48
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_min_i32_acq_rel:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    amomin.w.aqrl a0, a1, (a0)
+; RV64IA-NEXT:    ret
   %1 = atomicrmw min i32* %a, i32 %b acq_rel
   ret i32 %1
 }
@@ -6514,15 +12155,13 @@ define i32 @atomicrmw_min_i32_seq_cst(i32 *%a, i32 %b) nounwind {
 ; RV32I-NEXT:    sw s1, 24(sp)
 ; RV32I-NEXT:    sw s2, 20(sp)
 ; RV32I-NEXT:    sw s3, 16(sp)
-; RV32I-NEXT:    sw s4, 12(sp)
 ; RV32I-NEXT:    mv s1, a1
 ; RV32I-NEXT:    mv s2, a0
 ; RV32I-NEXT:    lw a2, 0(a0)
-; RV32I-NEXT:    addi s3, sp, 8
-; RV32I-NEXT:    addi s4, zero, 5
+; RV32I-NEXT:    addi s3, sp, 12
 ; RV32I-NEXT:  .LBB154_1: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    sw a2, 8(sp)
+; RV32I-NEXT:    sw a2, 12(sp)
 ; RV32I-NEXT:    bge s1, a2, .LBB154_3
 ; RV32I-NEXT:  # %bb.2: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB154_1 Depth=1
@@ -6531,14 +12170,13 @@ define i32 @atomicrmw_min_i32_seq_cst(i32 *%a, i32 %b) nounwind {
 ; RV32I-NEXT:    # in Loop: Header=BB154_1 Depth=1
 ; RV32I-NEXT:    mv a0, s2
 ; RV32I-NEXT:    mv a1, s3
-; RV32I-NEXT:    mv a3, s4
-; RV32I-NEXT:    mv a4, s4
+; RV32I-NEXT:    addi a3, zero, 5
+; RV32I-NEXT:    addi a4, zero, 5
 ; RV32I-NEXT:    call __atomic_compare_exchange_4
-; RV32I-NEXT:    lw a2, 8(sp)
+; RV32I-NEXT:    lw a2, 12(sp)
 ; RV32I-NEXT:    beqz a0, .LBB154_1
 ; RV32I-NEXT:  # %bb.4: # %atomicrmw.end
 ; RV32I-NEXT:    mv a0, a2
-; RV32I-NEXT:    lw s4, 12(sp)
 ; RV32I-NEXT:    lw s3, 16(sp)
 ; RV32I-NEXT:    lw s2, 20(sp)
 ; RV32I-NEXT:    lw s1, 24(sp)
@@ -6550,6 +12188,52 @@ define i32 @atomicrmw_min_i32_seq_cst(i32 *%a, i32 %b) nounwind {
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    amomin.w.aqrl a0, a1, (a0)
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_min_i32_seq_cst:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -48
+; RV64I-NEXT:    sd ra, 40(sp)
+; RV64I-NEXT:    sd s1, 32(sp)
+; RV64I-NEXT:    sd s2, 24(sp)
+; RV64I-NEXT:    sd s3, 16(sp)
+; RV64I-NEXT:    sd s4, 8(sp)
+; RV64I-NEXT:    mv s2, a1
+; RV64I-NEXT:    mv s4, a0
+; RV64I-NEXT:    lwu a0, 0(a0)
+; RV64I-NEXT:    sext.w s1, a1
+; RV64I-NEXT:    addi s3, sp, 4
+; RV64I-NEXT:  .LBB154_1: # %atomicrmw.start
+; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV64I-NEXT:    sext.w a1, a0
+; RV64I-NEXT:    mv a2, a0
+; RV64I-NEXT:    bge s1, a1, .LBB154_3
+; RV64I-NEXT:  # %bb.2: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB154_1 Depth=1
+; RV64I-NEXT:    mv a2, s2
+; RV64I-NEXT:  .LBB154_3: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB154_1 Depth=1
+; RV64I-NEXT:    sw a0, 4(sp)
+; RV64I-NEXT:    mv a0, s4
+; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    addi a3, zero, 5
+; RV64I-NEXT:    addi a4, zero, 5
+; RV64I-NEXT:    call __atomic_compare_exchange_4
+; RV64I-NEXT:    mv a1, a0
+; RV64I-NEXT:    lw a0, 4(sp)
+; RV64I-NEXT:    beqz a1, .LBB154_1
+; RV64I-NEXT:  # %bb.4: # %atomicrmw.end
+; RV64I-NEXT:    ld s4, 8(sp)
+; RV64I-NEXT:    ld s3, 16(sp)
+; RV64I-NEXT:    ld s2, 24(sp)
+; RV64I-NEXT:    ld s1, 32(sp)
+; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    addi sp, sp, 48
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_min_i32_seq_cst:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    amomin.w.aqrl a0, a1, (a0)
+; RV64IA-NEXT:    ret
   %1 = atomicrmw min i32* %a, i32 %b seq_cst
   ret i32 %1
 }
@@ -6595,6 +12279,52 @@ define i32 @atomicrmw_umax_i32_monotonic(i32 *%a, i32 %b) nounwind {
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    amomaxu.w a0, a1, (a0)
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_umax_i32_monotonic:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -48
+; RV64I-NEXT:    sd ra, 40(sp)
+; RV64I-NEXT:    sd s1, 32(sp)
+; RV64I-NEXT:    sd s2, 24(sp)
+; RV64I-NEXT:    sd s3, 16(sp)
+; RV64I-NEXT:    sd s4, 8(sp)
+; RV64I-NEXT:    mv s2, a1
+; RV64I-NEXT:    mv s4, a0
+; RV64I-NEXT:    lwu a0, 0(a0)
+; RV64I-NEXT:    sext.w s1, a1
+; RV64I-NEXT:    addi s3, sp, 4
+; RV64I-NEXT:  .LBB155_1: # %atomicrmw.start
+; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV64I-NEXT:    sext.w a1, a0
+; RV64I-NEXT:    mv a2, a0
+; RV64I-NEXT:    bltu s1, a1, .LBB155_3
+; RV64I-NEXT:  # %bb.2: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB155_1 Depth=1
+; RV64I-NEXT:    mv a2, s2
+; RV64I-NEXT:  .LBB155_3: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB155_1 Depth=1
+; RV64I-NEXT:    sw a0, 4(sp)
+; RV64I-NEXT:    mv a0, s4
+; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    mv a3, zero
+; RV64I-NEXT:    mv a4, zero
+; RV64I-NEXT:    call __atomic_compare_exchange_4
+; RV64I-NEXT:    mv a1, a0
+; RV64I-NEXT:    lw a0, 4(sp)
+; RV64I-NEXT:    beqz a1, .LBB155_1
+; RV64I-NEXT:  # %bb.4: # %atomicrmw.end
+; RV64I-NEXT:    ld s4, 8(sp)
+; RV64I-NEXT:    ld s3, 16(sp)
+; RV64I-NEXT:    ld s2, 24(sp)
+; RV64I-NEXT:    ld s1, 32(sp)
+; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    addi sp, sp, 48
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_umax_i32_monotonic:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    amomaxu.w a0, a1, (a0)
+; RV64IA-NEXT:    ret
   %1 = atomicrmw umax i32* %a, i32 %b monotonic
   ret i32 %1
 }
@@ -6607,15 +12337,13 @@ define i32 @atomicrmw_umax_i32_acquire(i32 *%a, i32 %b) nounwind {
 ; RV32I-NEXT:    sw s1, 24(sp)
 ; RV32I-NEXT:    sw s2, 20(sp)
 ; RV32I-NEXT:    sw s3, 16(sp)
-; RV32I-NEXT:    sw s4, 12(sp)
 ; RV32I-NEXT:    mv s1, a1
 ; RV32I-NEXT:    mv s2, a0
 ; RV32I-NEXT:    lw a2, 0(a0)
-; RV32I-NEXT:    addi s3, sp, 8
-; RV32I-NEXT:    addi s4, zero, 2
+; RV32I-NEXT:    addi s3, sp, 12
 ; RV32I-NEXT:  .LBB156_1: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    sw a2, 8(sp)
+; RV32I-NEXT:    sw a2, 12(sp)
 ; RV32I-NEXT:    bltu s1, a2, .LBB156_3
 ; RV32I-NEXT:  # %bb.2: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB156_1 Depth=1
@@ -6624,14 +12352,13 @@ define i32 @atomicrmw_umax_i32_acquire(i32 *%a, i32 %b) nounwind {
 ; RV32I-NEXT:    # in Loop: Header=BB156_1 Depth=1
 ; RV32I-NEXT:    mv a0, s2
 ; RV32I-NEXT:    mv a1, s3
-; RV32I-NEXT:    mv a3, s4
-; RV32I-NEXT:    mv a4, s4
+; RV32I-NEXT:    addi a3, zero, 2
+; RV32I-NEXT:    addi a4, zero, 2
 ; RV32I-NEXT:    call __atomic_compare_exchange_4
-; RV32I-NEXT:    lw a2, 8(sp)
+; RV32I-NEXT:    lw a2, 12(sp)
 ; RV32I-NEXT:    beqz a0, .LBB156_1
 ; RV32I-NEXT:  # %bb.4: # %atomicrmw.end
 ; RV32I-NEXT:    mv a0, a2
-; RV32I-NEXT:    lw s4, 12(sp)
 ; RV32I-NEXT:    lw s3, 16(sp)
 ; RV32I-NEXT:    lw s2, 20(sp)
 ; RV32I-NEXT:    lw s1, 24(sp)
@@ -6643,6 +12370,52 @@ define i32 @atomicrmw_umax_i32_acquire(i32 *%a, i32 %b) nounwind {
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    amomaxu.w.aq a0, a1, (a0)
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_umax_i32_acquire:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -48
+; RV64I-NEXT:    sd ra, 40(sp)
+; RV64I-NEXT:    sd s1, 32(sp)
+; RV64I-NEXT:    sd s2, 24(sp)
+; RV64I-NEXT:    sd s3, 16(sp)
+; RV64I-NEXT:    sd s4, 8(sp)
+; RV64I-NEXT:    mv s2, a1
+; RV64I-NEXT:    mv s4, a0
+; RV64I-NEXT:    lwu a0, 0(a0)
+; RV64I-NEXT:    sext.w s1, a1
+; RV64I-NEXT:    addi s3, sp, 4
+; RV64I-NEXT:  .LBB156_1: # %atomicrmw.start
+; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV64I-NEXT:    sext.w a1, a0
+; RV64I-NEXT:    mv a2, a0
+; RV64I-NEXT:    bltu s1, a1, .LBB156_3
+; RV64I-NEXT:  # %bb.2: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB156_1 Depth=1
+; RV64I-NEXT:    mv a2, s2
+; RV64I-NEXT:  .LBB156_3: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB156_1 Depth=1
+; RV64I-NEXT:    sw a0, 4(sp)
+; RV64I-NEXT:    mv a0, s4
+; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    addi a3, zero, 2
+; RV64I-NEXT:    addi a4, zero, 2
+; RV64I-NEXT:    call __atomic_compare_exchange_4
+; RV64I-NEXT:    mv a1, a0
+; RV64I-NEXT:    lw a0, 4(sp)
+; RV64I-NEXT:    beqz a1, .LBB156_1
+; RV64I-NEXT:  # %bb.4: # %atomicrmw.end
+; RV64I-NEXT:    ld s4, 8(sp)
+; RV64I-NEXT:    ld s3, 16(sp)
+; RV64I-NEXT:    ld s2, 24(sp)
+; RV64I-NEXT:    ld s1, 32(sp)
+; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    addi sp, sp, 48
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_umax_i32_acquire:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    amomaxu.w.aq a0, a1, (a0)
+; RV64IA-NEXT:    ret
   %1 = atomicrmw umax i32* %a, i32 %b acquire
   ret i32 %1
 }
@@ -6655,15 +12428,13 @@ define i32 @atomicrmw_umax_i32_release(i32 *%a, i32 %b) nounwind {
 ; RV32I-NEXT:    sw s1, 24(sp)
 ; RV32I-NEXT:    sw s2, 20(sp)
 ; RV32I-NEXT:    sw s3, 16(sp)
-; RV32I-NEXT:    sw s4, 12(sp)
 ; RV32I-NEXT:    mv s1, a1
 ; RV32I-NEXT:    mv s2, a0
 ; RV32I-NEXT:    lw a2, 0(a0)
-; RV32I-NEXT:    addi s3, sp, 8
-; RV32I-NEXT:    addi s4, zero, 3
+; RV32I-NEXT:    addi s3, sp, 12
 ; RV32I-NEXT:  .LBB157_1: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    sw a2, 8(sp)
+; RV32I-NEXT:    sw a2, 12(sp)
 ; RV32I-NEXT:    bltu s1, a2, .LBB157_3
 ; RV32I-NEXT:  # %bb.2: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB157_1 Depth=1
@@ -6672,14 +12443,13 @@ define i32 @atomicrmw_umax_i32_release(i32 *%a, i32 %b) nounwind {
 ; RV32I-NEXT:    # in Loop: Header=BB157_1 Depth=1
 ; RV32I-NEXT:    mv a0, s2
 ; RV32I-NEXT:    mv a1, s3
-; RV32I-NEXT:    mv a3, s4
+; RV32I-NEXT:    addi a3, zero, 3
 ; RV32I-NEXT:    mv a4, zero
 ; RV32I-NEXT:    call __atomic_compare_exchange_4
-; RV32I-NEXT:    lw a2, 8(sp)
+; RV32I-NEXT:    lw a2, 12(sp)
 ; RV32I-NEXT:    beqz a0, .LBB157_1
 ; RV32I-NEXT:  # %bb.4: # %atomicrmw.end
 ; RV32I-NEXT:    mv a0, a2
-; RV32I-NEXT:    lw s4, 12(sp)
 ; RV32I-NEXT:    lw s3, 16(sp)
 ; RV32I-NEXT:    lw s2, 20(sp)
 ; RV32I-NEXT:    lw s1, 24(sp)
@@ -6691,6 +12461,52 @@ define i32 @atomicrmw_umax_i32_release(i32 *%a, i32 %b) nounwind {
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    amomaxu.w.rl a0, a1, (a0)
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_umax_i32_release:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -48
+; RV64I-NEXT:    sd ra, 40(sp)
+; RV64I-NEXT:    sd s1, 32(sp)
+; RV64I-NEXT:    sd s2, 24(sp)
+; RV64I-NEXT:    sd s3, 16(sp)
+; RV64I-NEXT:    sd s4, 8(sp)
+; RV64I-NEXT:    mv s2, a1
+; RV64I-NEXT:    mv s4, a0
+; RV64I-NEXT:    lwu a0, 0(a0)
+; RV64I-NEXT:    sext.w s1, a1
+; RV64I-NEXT:    addi s3, sp, 4
+; RV64I-NEXT:  .LBB157_1: # %atomicrmw.start
+; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV64I-NEXT:    sext.w a1, a0
+; RV64I-NEXT:    mv a2, a0
+; RV64I-NEXT:    bltu s1, a1, .LBB157_3
+; RV64I-NEXT:  # %bb.2: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB157_1 Depth=1
+; RV64I-NEXT:    mv a2, s2
+; RV64I-NEXT:  .LBB157_3: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB157_1 Depth=1
+; RV64I-NEXT:    sw a0, 4(sp)
+; RV64I-NEXT:    mv a0, s4
+; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    addi a3, zero, 3
+; RV64I-NEXT:    mv a4, zero
+; RV64I-NEXT:    call __atomic_compare_exchange_4
+; RV64I-NEXT:    mv a1, a0
+; RV64I-NEXT:    lw a0, 4(sp)
+; RV64I-NEXT:    beqz a1, .LBB157_1
+; RV64I-NEXT:  # %bb.4: # %atomicrmw.end
+; RV64I-NEXT:    ld s4, 8(sp)
+; RV64I-NEXT:    ld s3, 16(sp)
+; RV64I-NEXT:    ld s2, 24(sp)
+; RV64I-NEXT:    ld s1, 32(sp)
+; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    addi sp, sp, 48
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_umax_i32_release:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    amomaxu.w.rl a0, a1, (a0)
+; RV64IA-NEXT:    ret
   %1 = atomicrmw umax i32* %a, i32 %b release
   ret i32 %1
 }
@@ -6703,17 +12519,13 @@ define i32 @atomicrmw_umax_i32_acq_rel(i32 *%a, i32 %b) nounwind {
 ; RV32I-NEXT:    sw s1, 24(sp)
 ; RV32I-NEXT:    sw s2, 20(sp)
 ; RV32I-NEXT:    sw s3, 16(sp)
-; RV32I-NEXT:    sw s4, 12(sp)
-; RV32I-NEXT:    sw s5, 8(sp)
 ; RV32I-NEXT:    mv s1, a1
 ; RV32I-NEXT:    mv s2, a0
 ; RV32I-NEXT:    lw a2, 0(a0)
-; RV32I-NEXT:    addi s3, sp, 4
-; RV32I-NEXT:    addi s4, zero, 4
-; RV32I-NEXT:    addi s5, zero, 2
+; RV32I-NEXT:    addi s3, sp, 12
 ; RV32I-NEXT:  .LBB158_1: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    sw a2, 4(sp)
+; RV32I-NEXT:    sw a2, 12(sp)
 ; RV32I-NEXT:    bltu s1, a2, .LBB158_3
 ; RV32I-NEXT:  # %bb.2: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB158_1 Depth=1
@@ -6722,15 +12534,13 @@ define i32 @atomicrmw_umax_i32_acq_rel(i32 *%a, i32 %b) nounwind {
 ; RV32I-NEXT:    # in Loop: Header=BB158_1 Depth=1
 ; RV32I-NEXT:    mv a0, s2
 ; RV32I-NEXT:    mv a1, s3
-; RV32I-NEXT:    mv a3, s4
-; RV32I-NEXT:    mv a4, s5
+; RV32I-NEXT:    addi a3, zero, 4
+; RV32I-NEXT:    addi a4, zero, 2
 ; RV32I-NEXT:    call __atomic_compare_exchange_4
-; RV32I-NEXT:    lw a2, 4(sp)
+; RV32I-NEXT:    lw a2, 12(sp)
 ; RV32I-NEXT:    beqz a0, .LBB158_1
 ; RV32I-NEXT:  # %bb.4: # %atomicrmw.end
 ; RV32I-NEXT:    mv a0, a2
-; RV32I-NEXT:    lw s5, 8(sp)
-; RV32I-NEXT:    lw s4, 12(sp)
 ; RV32I-NEXT:    lw s3, 16(sp)
 ; RV32I-NEXT:    lw s2, 20(sp)
 ; RV32I-NEXT:    lw s1, 24(sp)
@@ -6742,6 +12552,52 @@ define i32 @atomicrmw_umax_i32_acq_rel(i32 *%a, i32 %b) nounwind {
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    amomaxu.w.aqrl a0, a1, (a0)
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_umax_i32_acq_rel:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -48
+; RV64I-NEXT:    sd ra, 40(sp)
+; RV64I-NEXT:    sd s1, 32(sp)
+; RV64I-NEXT:    sd s2, 24(sp)
+; RV64I-NEXT:    sd s3, 16(sp)
+; RV64I-NEXT:    sd s4, 8(sp)
+; RV64I-NEXT:    mv s2, a1
+; RV64I-NEXT:    mv s4, a0
+; RV64I-NEXT:    lwu a0, 0(a0)
+; RV64I-NEXT:    sext.w s1, a1
+; RV64I-NEXT:    addi s3, sp, 4
+; RV64I-NEXT:  .LBB158_1: # %atomicrmw.start
+; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV64I-NEXT:    sext.w a1, a0
+; RV64I-NEXT:    mv a2, a0
+; RV64I-NEXT:    bltu s1, a1, .LBB158_3
+; RV64I-NEXT:  # %bb.2: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB158_1 Depth=1
+; RV64I-NEXT:    mv a2, s2
+; RV64I-NEXT:  .LBB158_3: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB158_1 Depth=1
+; RV64I-NEXT:    sw a0, 4(sp)
+; RV64I-NEXT:    mv a0, s4
+; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    addi a3, zero, 4
+; RV64I-NEXT:    addi a4, zero, 2
+; RV64I-NEXT:    call __atomic_compare_exchange_4
+; RV64I-NEXT:    mv a1, a0
+; RV64I-NEXT:    lw a0, 4(sp)
+; RV64I-NEXT:    beqz a1, .LBB158_1
+; RV64I-NEXT:  # %bb.4: # %atomicrmw.end
+; RV64I-NEXT:    ld s4, 8(sp)
+; RV64I-NEXT:    ld s3, 16(sp)
+; RV64I-NEXT:    ld s2, 24(sp)
+; RV64I-NEXT:    ld s1, 32(sp)
+; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    addi sp, sp, 48
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_umax_i32_acq_rel:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    amomaxu.w.aqrl a0, a1, (a0)
+; RV64IA-NEXT:    ret
   %1 = atomicrmw umax i32* %a, i32 %b acq_rel
   ret i32 %1
 }
@@ -6754,15 +12610,13 @@ define i32 @atomicrmw_umax_i32_seq_cst(i32 *%a, i32 %b) nounwind {
 ; RV32I-NEXT:    sw s1, 24(sp)
 ; RV32I-NEXT:    sw s2, 20(sp)
 ; RV32I-NEXT:    sw s3, 16(sp)
-; RV32I-NEXT:    sw s4, 12(sp)
 ; RV32I-NEXT:    mv s1, a1
 ; RV32I-NEXT:    mv s2, a0
 ; RV32I-NEXT:    lw a2, 0(a0)
-; RV32I-NEXT:    addi s3, sp, 8
-; RV32I-NEXT:    addi s4, zero, 5
+; RV32I-NEXT:    addi s3, sp, 12
 ; RV32I-NEXT:  .LBB159_1: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    sw a2, 8(sp)
+; RV32I-NEXT:    sw a2, 12(sp)
 ; RV32I-NEXT:    bltu s1, a2, .LBB159_3
 ; RV32I-NEXT:  # %bb.2: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB159_1 Depth=1
@@ -6771,14 +12625,13 @@ define i32 @atomicrmw_umax_i32_seq_cst(i32 *%a, i32 %b) nounwind {
 ; RV32I-NEXT:    # in Loop: Header=BB159_1 Depth=1
 ; RV32I-NEXT:    mv a0, s2
 ; RV32I-NEXT:    mv a1, s3
-; RV32I-NEXT:    mv a3, s4
-; RV32I-NEXT:    mv a4, s4
+; RV32I-NEXT:    addi a3, zero, 5
+; RV32I-NEXT:    addi a4, zero, 5
 ; RV32I-NEXT:    call __atomic_compare_exchange_4
-; RV32I-NEXT:    lw a2, 8(sp)
+; RV32I-NEXT:    lw a2, 12(sp)
 ; RV32I-NEXT:    beqz a0, .LBB159_1
 ; RV32I-NEXT:  # %bb.4: # %atomicrmw.end
 ; RV32I-NEXT:    mv a0, a2
-; RV32I-NEXT:    lw s4, 12(sp)
 ; RV32I-NEXT:    lw s3, 16(sp)
 ; RV32I-NEXT:    lw s2, 20(sp)
 ; RV32I-NEXT:    lw s1, 24(sp)
@@ -6790,6 +12643,52 @@ define i32 @atomicrmw_umax_i32_seq_cst(i32 *%a, i32 %b) nounwind {
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    amomaxu.w.aqrl a0, a1, (a0)
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_umax_i32_seq_cst:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -48
+; RV64I-NEXT:    sd ra, 40(sp)
+; RV64I-NEXT:    sd s1, 32(sp)
+; RV64I-NEXT:    sd s2, 24(sp)
+; RV64I-NEXT:    sd s3, 16(sp)
+; RV64I-NEXT:    sd s4, 8(sp)
+; RV64I-NEXT:    mv s2, a1
+; RV64I-NEXT:    mv s4, a0
+; RV64I-NEXT:    lwu a0, 0(a0)
+; RV64I-NEXT:    sext.w s1, a1
+; RV64I-NEXT:    addi s3, sp, 4
+; RV64I-NEXT:  .LBB159_1: # %atomicrmw.start
+; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV64I-NEXT:    sext.w a1, a0
+; RV64I-NEXT:    mv a2, a0
+; RV64I-NEXT:    bltu s1, a1, .LBB159_3
+; RV64I-NEXT:  # %bb.2: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB159_1 Depth=1
+; RV64I-NEXT:    mv a2, s2
+; RV64I-NEXT:  .LBB159_3: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB159_1 Depth=1
+; RV64I-NEXT:    sw a0, 4(sp)
+; RV64I-NEXT:    mv a0, s4
+; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    addi a3, zero, 5
+; RV64I-NEXT:    addi a4, zero, 5
+; RV64I-NEXT:    call __atomic_compare_exchange_4
+; RV64I-NEXT:    mv a1, a0
+; RV64I-NEXT:    lw a0, 4(sp)
+; RV64I-NEXT:    beqz a1, .LBB159_1
+; RV64I-NEXT:  # %bb.4: # %atomicrmw.end
+; RV64I-NEXT:    ld s4, 8(sp)
+; RV64I-NEXT:    ld s3, 16(sp)
+; RV64I-NEXT:    ld s2, 24(sp)
+; RV64I-NEXT:    ld s1, 32(sp)
+; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    addi sp, sp, 48
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_umax_i32_seq_cst:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    amomaxu.w.aqrl a0, a1, (a0)
+; RV64IA-NEXT:    ret
   %1 = atomicrmw umax i32* %a, i32 %b seq_cst
   ret i32 %1
 }
@@ -6835,6 +12734,52 @@ define i32 @atomicrmw_umin_i32_monotonic(i32 *%a, i32 %b) nounwind {
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    amominu.w a0, a1, (a0)
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_umin_i32_monotonic:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -48
+; RV64I-NEXT:    sd ra, 40(sp)
+; RV64I-NEXT:    sd s1, 32(sp)
+; RV64I-NEXT:    sd s2, 24(sp)
+; RV64I-NEXT:    sd s3, 16(sp)
+; RV64I-NEXT:    sd s4, 8(sp)
+; RV64I-NEXT:    mv s2, a1
+; RV64I-NEXT:    mv s4, a0
+; RV64I-NEXT:    lwu a0, 0(a0)
+; RV64I-NEXT:    sext.w s1, a1
+; RV64I-NEXT:    addi s3, sp, 4
+; RV64I-NEXT:  .LBB160_1: # %atomicrmw.start
+; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV64I-NEXT:    sext.w a1, a0
+; RV64I-NEXT:    mv a2, a0
+; RV64I-NEXT:    bgeu s1, a1, .LBB160_3
+; RV64I-NEXT:  # %bb.2: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB160_1 Depth=1
+; RV64I-NEXT:    mv a2, s2
+; RV64I-NEXT:  .LBB160_3: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB160_1 Depth=1
+; RV64I-NEXT:    sw a0, 4(sp)
+; RV64I-NEXT:    mv a0, s4
+; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    mv a3, zero
+; RV64I-NEXT:    mv a4, zero
+; RV64I-NEXT:    call __atomic_compare_exchange_4
+; RV64I-NEXT:    mv a1, a0
+; RV64I-NEXT:    lw a0, 4(sp)
+; RV64I-NEXT:    beqz a1, .LBB160_1
+; RV64I-NEXT:  # %bb.4: # %atomicrmw.end
+; RV64I-NEXT:    ld s4, 8(sp)
+; RV64I-NEXT:    ld s3, 16(sp)
+; RV64I-NEXT:    ld s2, 24(sp)
+; RV64I-NEXT:    ld s1, 32(sp)
+; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    addi sp, sp, 48
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_umin_i32_monotonic:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    amominu.w a0, a1, (a0)
+; RV64IA-NEXT:    ret
   %1 = atomicrmw umin i32* %a, i32 %b monotonic
   ret i32 %1
 }
@@ -6847,15 +12792,13 @@ define i32 @atomicrmw_umin_i32_acquire(i32 *%a, i32 %b) nounwind {
 ; RV32I-NEXT:    sw s1, 24(sp)
 ; RV32I-NEXT:    sw s2, 20(sp)
 ; RV32I-NEXT:    sw s3, 16(sp)
-; RV32I-NEXT:    sw s4, 12(sp)
 ; RV32I-NEXT:    mv s1, a1
 ; RV32I-NEXT:    mv s2, a0
 ; RV32I-NEXT:    lw a2, 0(a0)
-; RV32I-NEXT:    addi s3, sp, 8
-; RV32I-NEXT:    addi s4, zero, 2
+; RV32I-NEXT:    addi s3, sp, 12
 ; RV32I-NEXT:  .LBB161_1: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    sw a2, 8(sp)
+; RV32I-NEXT:    sw a2, 12(sp)
 ; RV32I-NEXT:    bgeu s1, a2, .LBB161_3
 ; RV32I-NEXT:  # %bb.2: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB161_1 Depth=1
@@ -6864,14 +12807,13 @@ define i32 @atomicrmw_umin_i32_acquire(i32 *%a, i32 %b) nounwind {
 ; RV32I-NEXT:    # in Loop: Header=BB161_1 Depth=1
 ; RV32I-NEXT:    mv a0, s2
 ; RV32I-NEXT:    mv a1, s3
-; RV32I-NEXT:    mv a3, s4
-; RV32I-NEXT:    mv a4, s4
+; RV32I-NEXT:    addi a3, zero, 2
+; RV32I-NEXT:    addi a4, zero, 2
 ; RV32I-NEXT:    call __atomic_compare_exchange_4
-; RV32I-NEXT:    lw a2, 8(sp)
+; RV32I-NEXT:    lw a2, 12(sp)
 ; RV32I-NEXT:    beqz a0, .LBB161_1
 ; RV32I-NEXT:  # %bb.4: # %atomicrmw.end
 ; RV32I-NEXT:    mv a0, a2
-; RV32I-NEXT:    lw s4, 12(sp)
 ; RV32I-NEXT:    lw s3, 16(sp)
 ; RV32I-NEXT:    lw s2, 20(sp)
 ; RV32I-NEXT:    lw s1, 24(sp)
@@ -6883,6 +12825,52 @@ define i32 @atomicrmw_umin_i32_acquire(i32 *%a, i32 %b) nounwind {
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    amominu.w.aq a0, a1, (a0)
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_umin_i32_acquire:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -48
+; RV64I-NEXT:    sd ra, 40(sp)
+; RV64I-NEXT:    sd s1, 32(sp)
+; RV64I-NEXT:    sd s2, 24(sp)
+; RV64I-NEXT:    sd s3, 16(sp)
+; RV64I-NEXT:    sd s4, 8(sp)
+; RV64I-NEXT:    mv s2, a1
+; RV64I-NEXT:    mv s4, a0
+; RV64I-NEXT:    lwu a0, 0(a0)
+; RV64I-NEXT:    sext.w s1, a1
+; RV64I-NEXT:    addi s3, sp, 4
+; RV64I-NEXT:  .LBB161_1: # %atomicrmw.start
+; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV64I-NEXT:    sext.w a1, a0
+; RV64I-NEXT:    mv a2, a0
+; RV64I-NEXT:    bgeu s1, a1, .LBB161_3
+; RV64I-NEXT:  # %bb.2: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB161_1 Depth=1
+; RV64I-NEXT:    mv a2, s2
+; RV64I-NEXT:  .LBB161_3: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB161_1 Depth=1
+; RV64I-NEXT:    sw a0, 4(sp)
+; RV64I-NEXT:    mv a0, s4
+; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    addi a3, zero, 2
+; RV64I-NEXT:    addi a4, zero, 2
+; RV64I-NEXT:    call __atomic_compare_exchange_4
+; RV64I-NEXT:    mv a1, a0
+; RV64I-NEXT:    lw a0, 4(sp)
+; RV64I-NEXT:    beqz a1, .LBB161_1
+; RV64I-NEXT:  # %bb.4: # %atomicrmw.end
+; RV64I-NEXT:    ld s4, 8(sp)
+; RV64I-NEXT:    ld s3, 16(sp)
+; RV64I-NEXT:    ld s2, 24(sp)
+; RV64I-NEXT:    ld s1, 32(sp)
+; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    addi sp, sp, 48
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_umin_i32_acquire:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    amominu.w.aq a0, a1, (a0)
+; RV64IA-NEXT:    ret
   %1 = atomicrmw umin i32* %a, i32 %b acquire
   ret i32 %1
 }
@@ -6895,15 +12883,13 @@ define i32 @atomicrmw_umin_i32_release(i32 *%a, i32 %b) nounwind {
 ; RV32I-NEXT:    sw s1, 24(sp)
 ; RV32I-NEXT:    sw s2, 20(sp)
 ; RV32I-NEXT:    sw s3, 16(sp)
-; RV32I-NEXT:    sw s4, 12(sp)
 ; RV32I-NEXT:    mv s1, a1
 ; RV32I-NEXT:    mv s2, a0
 ; RV32I-NEXT:    lw a2, 0(a0)
-; RV32I-NEXT:    addi s3, sp, 8
-; RV32I-NEXT:    addi s4, zero, 3
+; RV32I-NEXT:    addi s3, sp, 12
 ; RV32I-NEXT:  .LBB162_1: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    sw a2, 8(sp)
+; RV32I-NEXT:    sw a2, 12(sp)
 ; RV32I-NEXT:    bgeu s1, a2, .LBB162_3
 ; RV32I-NEXT:  # %bb.2: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB162_1 Depth=1
@@ -6912,14 +12898,13 @@ define i32 @atomicrmw_umin_i32_release(i32 *%a, i32 %b) nounwind {
 ; RV32I-NEXT:    # in Loop: Header=BB162_1 Depth=1
 ; RV32I-NEXT:    mv a0, s2
 ; RV32I-NEXT:    mv a1, s3
-; RV32I-NEXT:    mv a3, s4
+; RV32I-NEXT:    addi a3, zero, 3
 ; RV32I-NEXT:    mv a4, zero
 ; RV32I-NEXT:    call __atomic_compare_exchange_4
-; RV32I-NEXT:    lw a2, 8(sp)
+; RV32I-NEXT:    lw a2, 12(sp)
 ; RV32I-NEXT:    beqz a0, .LBB162_1
 ; RV32I-NEXT:  # %bb.4: # %atomicrmw.end
 ; RV32I-NEXT:    mv a0, a2
-; RV32I-NEXT:    lw s4, 12(sp)
 ; RV32I-NEXT:    lw s3, 16(sp)
 ; RV32I-NEXT:    lw s2, 20(sp)
 ; RV32I-NEXT:    lw s1, 24(sp)
@@ -6931,6 +12916,52 @@ define i32 @atomicrmw_umin_i32_release(i32 *%a, i32 %b) nounwind {
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    amominu.w.rl a0, a1, (a0)
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_umin_i32_release:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -48
+; RV64I-NEXT:    sd ra, 40(sp)
+; RV64I-NEXT:    sd s1, 32(sp)
+; RV64I-NEXT:    sd s2, 24(sp)
+; RV64I-NEXT:    sd s3, 16(sp)
+; RV64I-NEXT:    sd s4, 8(sp)
+; RV64I-NEXT:    mv s2, a1
+; RV64I-NEXT:    mv s4, a0
+; RV64I-NEXT:    lwu a0, 0(a0)
+; RV64I-NEXT:    sext.w s1, a1
+; RV64I-NEXT:    addi s3, sp, 4
+; RV64I-NEXT:  .LBB162_1: # %atomicrmw.start
+; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV64I-NEXT:    sext.w a1, a0
+; RV64I-NEXT:    mv a2, a0
+; RV64I-NEXT:    bgeu s1, a1, .LBB162_3
+; RV64I-NEXT:  # %bb.2: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB162_1 Depth=1
+; RV64I-NEXT:    mv a2, s2
+; RV64I-NEXT:  .LBB162_3: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB162_1 Depth=1
+; RV64I-NEXT:    sw a0, 4(sp)
+; RV64I-NEXT:    mv a0, s4
+; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    addi a3, zero, 3
+; RV64I-NEXT:    mv a4, zero
+; RV64I-NEXT:    call __atomic_compare_exchange_4
+; RV64I-NEXT:    mv a1, a0
+; RV64I-NEXT:    lw a0, 4(sp)
+; RV64I-NEXT:    beqz a1, .LBB162_1
+; RV64I-NEXT:  # %bb.4: # %atomicrmw.end
+; RV64I-NEXT:    ld s4, 8(sp)
+; RV64I-NEXT:    ld s3, 16(sp)
+; RV64I-NEXT:    ld s2, 24(sp)
+; RV64I-NEXT:    ld s1, 32(sp)
+; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    addi sp, sp, 48
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_umin_i32_release:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    amominu.w.rl a0, a1, (a0)
+; RV64IA-NEXT:    ret
   %1 = atomicrmw umin i32* %a, i32 %b release
   ret i32 %1
 }
@@ -6943,17 +12974,13 @@ define i32 @atomicrmw_umin_i32_acq_rel(i32 *%a, i32 %b) nounwind {
 ; RV32I-NEXT:    sw s1, 24(sp)
 ; RV32I-NEXT:    sw s2, 20(sp)
 ; RV32I-NEXT:    sw s3, 16(sp)
-; RV32I-NEXT:    sw s4, 12(sp)
-; RV32I-NEXT:    sw s5, 8(sp)
 ; RV32I-NEXT:    mv s1, a1
 ; RV32I-NEXT:    mv s2, a0
 ; RV32I-NEXT:    lw a2, 0(a0)
-; RV32I-NEXT:    addi s3, sp, 4
-; RV32I-NEXT:    addi s4, zero, 4
-; RV32I-NEXT:    addi s5, zero, 2
+; RV32I-NEXT:    addi s3, sp, 12
 ; RV32I-NEXT:  .LBB163_1: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    sw a2, 4(sp)
+; RV32I-NEXT:    sw a2, 12(sp)
 ; RV32I-NEXT:    bgeu s1, a2, .LBB163_3
 ; RV32I-NEXT:  # %bb.2: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB163_1 Depth=1
@@ -6962,15 +12989,13 @@ define i32 @atomicrmw_umin_i32_acq_rel(i32 *%a, i32 %b) nounwind {
 ; RV32I-NEXT:    # in Loop: Header=BB163_1 Depth=1
 ; RV32I-NEXT:    mv a0, s2
 ; RV32I-NEXT:    mv a1, s3
-; RV32I-NEXT:    mv a3, s4
-; RV32I-NEXT:    mv a4, s5
+; RV32I-NEXT:    addi a3, zero, 4
+; RV32I-NEXT:    addi a4, zero, 2
 ; RV32I-NEXT:    call __atomic_compare_exchange_4
-; RV32I-NEXT:    lw a2, 4(sp)
+; RV32I-NEXT:    lw a2, 12(sp)
 ; RV32I-NEXT:    beqz a0, .LBB163_1
 ; RV32I-NEXT:  # %bb.4: # %atomicrmw.end
 ; RV32I-NEXT:    mv a0, a2
-; RV32I-NEXT:    lw s5, 8(sp)
-; RV32I-NEXT:    lw s4, 12(sp)
 ; RV32I-NEXT:    lw s3, 16(sp)
 ; RV32I-NEXT:    lw s2, 20(sp)
 ; RV32I-NEXT:    lw s1, 24(sp)
@@ -6982,6 +13007,52 @@ define i32 @atomicrmw_umin_i32_acq_rel(i32 *%a, i32 %b) nounwind {
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    amominu.w.aqrl a0, a1, (a0)
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_umin_i32_acq_rel:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -48
+; RV64I-NEXT:    sd ra, 40(sp)
+; RV64I-NEXT:    sd s1, 32(sp)
+; RV64I-NEXT:    sd s2, 24(sp)
+; RV64I-NEXT:    sd s3, 16(sp)
+; RV64I-NEXT:    sd s4, 8(sp)
+; RV64I-NEXT:    mv s2, a1
+; RV64I-NEXT:    mv s4, a0
+; RV64I-NEXT:    lwu a0, 0(a0)
+; RV64I-NEXT:    sext.w s1, a1
+; RV64I-NEXT:    addi s3, sp, 4
+; RV64I-NEXT:  .LBB163_1: # %atomicrmw.start
+; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV64I-NEXT:    sext.w a1, a0
+; RV64I-NEXT:    mv a2, a0
+; RV64I-NEXT:    bgeu s1, a1, .LBB163_3
+; RV64I-NEXT:  # %bb.2: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB163_1 Depth=1
+; RV64I-NEXT:    mv a2, s2
+; RV64I-NEXT:  .LBB163_3: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB163_1 Depth=1
+; RV64I-NEXT:    sw a0, 4(sp)
+; RV64I-NEXT:    mv a0, s4
+; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    addi a3, zero, 4
+; RV64I-NEXT:    addi a4, zero, 2
+; RV64I-NEXT:    call __atomic_compare_exchange_4
+; RV64I-NEXT:    mv a1, a0
+; RV64I-NEXT:    lw a0, 4(sp)
+; RV64I-NEXT:    beqz a1, .LBB163_1
+; RV64I-NEXT:  # %bb.4: # %atomicrmw.end
+; RV64I-NEXT:    ld s4, 8(sp)
+; RV64I-NEXT:    ld s3, 16(sp)
+; RV64I-NEXT:    ld s2, 24(sp)
+; RV64I-NEXT:    ld s1, 32(sp)
+; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    addi sp, sp, 48
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_umin_i32_acq_rel:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    amominu.w.aqrl a0, a1, (a0)
+; RV64IA-NEXT:    ret
   %1 = atomicrmw umin i32* %a, i32 %b acq_rel
   ret i32 %1
 }
@@ -6994,15 +13065,13 @@ define i32 @atomicrmw_umin_i32_seq_cst(i32 *%a, i32 %b) nounwind {
 ; RV32I-NEXT:    sw s1, 24(sp)
 ; RV32I-NEXT:    sw s2, 20(sp)
 ; RV32I-NEXT:    sw s3, 16(sp)
-; RV32I-NEXT:    sw s4, 12(sp)
 ; RV32I-NEXT:    mv s1, a1
 ; RV32I-NEXT:    mv s2, a0
 ; RV32I-NEXT:    lw a2, 0(a0)
-; RV32I-NEXT:    addi s3, sp, 8
-; RV32I-NEXT:    addi s4, zero, 5
+; RV32I-NEXT:    addi s3, sp, 12
 ; RV32I-NEXT:  .LBB164_1: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    sw a2, 8(sp)
+; RV32I-NEXT:    sw a2, 12(sp)
 ; RV32I-NEXT:    bgeu s1, a2, .LBB164_3
 ; RV32I-NEXT:  # %bb.2: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB164_1 Depth=1
@@ -7011,14 +13080,13 @@ define i32 @atomicrmw_umin_i32_seq_cst(i32 *%a, i32 %b) nounwind {
 ; RV32I-NEXT:    # in Loop: Header=BB164_1 Depth=1
 ; RV32I-NEXT:    mv a0, s2
 ; RV32I-NEXT:    mv a1, s3
-; RV32I-NEXT:    mv a3, s4
-; RV32I-NEXT:    mv a4, s4
+; RV32I-NEXT:    addi a3, zero, 5
+; RV32I-NEXT:    addi a4, zero, 5
 ; RV32I-NEXT:    call __atomic_compare_exchange_4
-; RV32I-NEXT:    lw a2, 8(sp)
+; RV32I-NEXT:    lw a2, 12(sp)
 ; RV32I-NEXT:    beqz a0, .LBB164_1
 ; RV32I-NEXT:  # %bb.4: # %atomicrmw.end
 ; RV32I-NEXT:    mv a0, a2
-; RV32I-NEXT:    lw s4, 12(sp)
 ; RV32I-NEXT:    lw s3, 16(sp)
 ; RV32I-NEXT:    lw s2, 20(sp)
 ; RV32I-NEXT:    lw s1, 24(sp)
@@ -7030,6 +13098,52 @@ define i32 @atomicrmw_umin_i32_seq_cst(i32 *%a, i32 %b) nounwind {
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    amominu.w.aqrl a0, a1, (a0)
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_umin_i32_seq_cst:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -48
+; RV64I-NEXT:    sd ra, 40(sp)
+; RV64I-NEXT:    sd s1, 32(sp)
+; RV64I-NEXT:    sd s2, 24(sp)
+; RV64I-NEXT:    sd s3, 16(sp)
+; RV64I-NEXT:    sd s4, 8(sp)
+; RV64I-NEXT:    mv s2, a1
+; RV64I-NEXT:    mv s4, a0
+; RV64I-NEXT:    lwu a0, 0(a0)
+; RV64I-NEXT:    sext.w s1, a1
+; RV64I-NEXT:    addi s3, sp, 4
+; RV64I-NEXT:  .LBB164_1: # %atomicrmw.start
+; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV64I-NEXT:    sext.w a1, a0
+; RV64I-NEXT:    mv a2, a0
+; RV64I-NEXT:    bgeu s1, a1, .LBB164_3
+; RV64I-NEXT:  # %bb.2: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB164_1 Depth=1
+; RV64I-NEXT:    mv a2, s2
+; RV64I-NEXT:  .LBB164_3: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB164_1 Depth=1
+; RV64I-NEXT:    sw a0, 4(sp)
+; RV64I-NEXT:    mv a0, s4
+; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    addi a3, zero, 5
+; RV64I-NEXT:    addi a4, zero, 5
+; RV64I-NEXT:    call __atomic_compare_exchange_4
+; RV64I-NEXT:    mv a1, a0
+; RV64I-NEXT:    lw a0, 4(sp)
+; RV64I-NEXT:    beqz a1, .LBB164_1
+; RV64I-NEXT:  # %bb.4: # %atomicrmw.end
+; RV64I-NEXT:    ld s4, 8(sp)
+; RV64I-NEXT:    ld s3, 16(sp)
+; RV64I-NEXT:    ld s2, 24(sp)
+; RV64I-NEXT:    ld s1, 32(sp)
+; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    addi sp, sp, 48
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_umin_i32_seq_cst:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    amominu.w.aqrl a0, a1, (a0)
+; RV64IA-NEXT:    ret
   %1 = atomicrmw umin i32* %a, i32 %b seq_cst
   ret i32 %1
 }
@@ -7054,6 +13168,21 @@ define i64 @atomicrmw_xchg_i64_monotonic(i64* %a, i64 %b) {
 ; RV32IA-NEXT:    lw ra, 12(sp)
 ; RV32IA-NEXT:    addi sp, sp, 16
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_xchg_i64_monotonic:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    mv a2, zero
+; RV64I-NEXT:    call __atomic_exchange_8
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_xchg_i64_monotonic:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    amoswap.d a0, a1, (a0)
+; RV64IA-NEXT:    ret
   %1 = atomicrmw xchg i64* %a, i64 %b monotonic
   ret i64 %1
 }
@@ -7078,6 +13207,21 @@ define i64 @atomicrmw_xchg_i64_acquire(i64* %a, i64 %b) {
 ; RV32IA-NEXT:    lw ra, 12(sp)
 ; RV32IA-NEXT:    addi sp, sp, 16
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_xchg_i64_acquire:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a2, zero, 2
+; RV64I-NEXT:    call __atomic_exchange_8
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_xchg_i64_acquire:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    amoswap.d.aq a0, a1, (a0)
+; RV64IA-NEXT:    ret
   %1 = atomicrmw xchg i64* %a, i64 %b acquire
   ret i64 %1
 }
@@ -7102,6 +13246,21 @@ define i64 @atomicrmw_xchg_i64_release(i64* %a, i64 %b) {
 ; RV32IA-NEXT:    lw ra, 12(sp)
 ; RV32IA-NEXT:    addi sp, sp, 16
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_xchg_i64_release:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a2, zero, 3
+; RV64I-NEXT:    call __atomic_exchange_8
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_xchg_i64_release:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    amoswap.d.rl a0, a1, (a0)
+; RV64IA-NEXT:    ret
   %1 = atomicrmw xchg i64* %a, i64 %b release
   ret i64 %1
 }
@@ -7126,6 +13285,21 @@ define i64 @atomicrmw_xchg_i64_acq_rel(i64* %a, i64 %b) {
 ; RV32IA-NEXT:    lw ra, 12(sp)
 ; RV32IA-NEXT:    addi sp, sp, 16
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_xchg_i64_acq_rel:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a2, zero, 4
+; RV64I-NEXT:    call __atomic_exchange_8
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_xchg_i64_acq_rel:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    amoswap.d.aqrl a0, a1, (a0)
+; RV64IA-NEXT:    ret
   %1 = atomicrmw xchg i64* %a, i64 %b acq_rel
   ret i64 %1
 }
@@ -7150,6 +13324,21 @@ define i64 @atomicrmw_xchg_i64_seq_cst(i64* %a, i64 %b) {
 ; RV32IA-NEXT:    lw ra, 12(sp)
 ; RV32IA-NEXT:    addi sp, sp, 16
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_xchg_i64_seq_cst:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a2, zero, 5
+; RV64I-NEXT:    call __atomic_exchange_8
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_xchg_i64_seq_cst:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    amoswap.d.aqrl a0, a1, (a0)
+; RV64IA-NEXT:    ret
   %1 = atomicrmw xchg i64* %a, i64 %b seq_cst
   ret i64 %1
 }
@@ -7174,6 +13363,21 @@ define i64 @atomicrmw_add_i64_monotonic(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    lw ra, 12(sp)
 ; RV32IA-NEXT:    addi sp, sp, 16
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_add_i64_monotonic:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    mv a2, zero
+; RV64I-NEXT:    call __atomic_fetch_add_8
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_add_i64_monotonic:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    amoadd.d a0, a1, (a0)
+; RV64IA-NEXT:    ret
   %1 = atomicrmw add i64* %a, i64 %b monotonic
   ret i64 %1
 }
@@ -7198,6 +13402,21 @@ define i64 @atomicrmw_add_i64_acquire(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    lw ra, 12(sp)
 ; RV32IA-NEXT:    addi sp, sp, 16
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_add_i64_acquire:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a2, zero, 2
+; RV64I-NEXT:    call __atomic_fetch_add_8
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_add_i64_acquire:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    amoadd.d.aq a0, a1, (a0)
+; RV64IA-NEXT:    ret
   %1 = atomicrmw add i64* %a, i64 %b acquire
   ret i64 %1
 }
@@ -7222,6 +13441,21 @@ define i64 @atomicrmw_add_i64_release(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    lw ra, 12(sp)
 ; RV32IA-NEXT:    addi sp, sp, 16
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_add_i64_release:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a2, zero, 3
+; RV64I-NEXT:    call __atomic_fetch_add_8
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_add_i64_release:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    amoadd.d.rl a0, a1, (a0)
+; RV64IA-NEXT:    ret
   %1 = atomicrmw add i64* %a, i64 %b release
   ret i64 %1
 }
@@ -7246,6 +13480,21 @@ define i64 @atomicrmw_add_i64_acq_rel(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    lw ra, 12(sp)
 ; RV32IA-NEXT:    addi sp, sp, 16
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_add_i64_acq_rel:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a2, zero, 4
+; RV64I-NEXT:    call __atomic_fetch_add_8
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_add_i64_acq_rel:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    amoadd.d.aqrl a0, a1, (a0)
+; RV64IA-NEXT:    ret
   %1 = atomicrmw add i64* %a, i64 %b acq_rel
   ret i64 %1
 }
@@ -7270,6 +13519,21 @@ define i64 @atomicrmw_add_i64_seq_cst(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    lw ra, 12(sp)
 ; RV32IA-NEXT:    addi sp, sp, 16
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_add_i64_seq_cst:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a2, zero, 5
+; RV64I-NEXT:    call __atomic_fetch_add_8
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_add_i64_seq_cst:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    amoadd.d.aqrl a0, a1, (a0)
+; RV64IA-NEXT:    ret
   %1 = atomicrmw add i64* %a, i64 %b seq_cst
   ret i64 %1
 }
@@ -7294,6 +13558,22 @@ define i64 @atomicrmw_sub_i64_monotonic(i64* %a, i64 %b) {
 ; RV32IA-NEXT:    lw ra, 12(sp)
 ; RV32IA-NEXT:    addi sp, sp, 16
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_sub_i64_monotonic:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    mv a2, zero
+; RV64I-NEXT:    call __atomic_fetch_sub_8
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_sub_i64_monotonic:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    neg a1, a1
+; RV64IA-NEXT:    amoadd.d a0, a1, (a0)
+; RV64IA-NEXT:    ret
   %1 = atomicrmw sub i64* %a, i64 %b monotonic
   ret i64 %1
 }
@@ -7318,6 +13598,22 @@ define i64 @atomicrmw_sub_i64_acquire(i64* %a, i64 %b) {
 ; RV32IA-NEXT:    lw ra, 12(sp)
 ; RV32IA-NEXT:    addi sp, sp, 16
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_sub_i64_acquire:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a2, zero, 2
+; RV64I-NEXT:    call __atomic_fetch_sub_8
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_sub_i64_acquire:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    neg a1, a1
+; RV64IA-NEXT:    amoadd.d.aq a0, a1, (a0)
+; RV64IA-NEXT:    ret
   %1 = atomicrmw sub i64* %a, i64 %b acquire
   ret i64 %1
 }
@@ -7342,6 +13638,22 @@ define i64 @atomicrmw_sub_i64_release(i64* %a, i64 %b) {
 ; RV32IA-NEXT:    lw ra, 12(sp)
 ; RV32IA-NEXT:    addi sp, sp, 16
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_sub_i64_release:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a2, zero, 3
+; RV64I-NEXT:    call __atomic_fetch_sub_8
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_sub_i64_release:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    neg a1, a1
+; RV64IA-NEXT:    amoadd.d.rl a0, a1, (a0)
+; RV64IA-NEXT:    ret
   %1 = atomicrmw sub i64* %a, i64 %b release
   ret i64 %1
 }
@@ -7366,6 +13678,22 @@ define i64 @atomicrmw_sub_i64_acq_rel(i64* %a, i64 %b) {
 ; RV32IA-NEXT:    lw ra, 12(sp)
 ; RV32IA-NEXT:    addi sp, sp, 16
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_sub_i64_acq_rel:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a2, zero, 4
+; RV64I-NEXT:    call __atomic_fetch_sub_8
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_sub_i64_acq_rel:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    neg a1, a1
+; RV64IA-NEXT:    amoadd.d.aqrl a0, a1, (a0)
+; RV64IA-NEXT:    ret
   %1 = atomicrmw sub i64* %a, i64 %b acq_rel
   ret i64 %1
 }
@@ -7390,6 +13718,22 @@ define i64 @atomicrmw_sub_i64_seq_cst(i64* %a, i64 %b) {
 ; RV32IA-NEXT:    lw ra, 12(sp)
 ; RV32IA-NEXT:    addi sp, sp, 16
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_sub_i64_seq_cst:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a2, zero, 5
+; RV64I-NEXT:    call __atomic_fetch_sub_8
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_sub_i64_seq_cst:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    neg a1, a1
+; RV64IA-NEXT:    amoadd.d.aqrl a0, a1, (a0)
+; RV64IA-NEXT:    ret
   %1 = atomicrmw sub i64* %a, i64 %b seq_cst
   ret i64 %1
 }
@@ -7414,6 +13758,21 @@ define i64 @atomicrmw_and_i64_monotonic(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    lw ra, 12(sp)
 ; RV32IA-NEXT:    addi sp, sp, 16
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_and_i64_monotonic:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    mv a2, zero
+; RV64I-NEXT:    call __atomic_fetch_and_8
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_and_i64_monotonic:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    amoand.d a0, a1, (a0)
+; RV64IA-NEXT:    ret
   %1 = atomicrmw and i64* %a, i64 %b monotonic
   ret i64 %1
 }
@@ -7438,6 +13797,21 @@ define i64 @atomicrmw_and_i64_acquire(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    lw ra, 12(sp)
 ; RV32IA-NEXT:    addi sp, sp, 16
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_and_i64_acquire:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a2, zero, 2
+; RV64I-NEXT:    call __atomic_fetch_and_8
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_and_i64_acquire:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    amoand.d.aq a0, a1, (a0)
+; RV64IA-NEXT:    ret
   %1 = atomicrmw and i64* %a, i64 %b acquire
   ret i64 %1
 }
@@ -7462,6 +13836,21 @@ define i64 @atomicrmw_and_i64_release(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    lw ra, 12(sp)
 ; RV32IA-NEXT:    addi sp, sp, 16
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_and_i64_release:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a2, zero, 3
+; RV64I-NEXT:    call __atomic_fetch_and_8
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_and_i64_release:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    amoand.d.rl a0, a1, (a0)
+; RV64IA-NEXT:    ret
   %1 = atomicrmw and i64* %a, i64 %b release
   ret i64 %1
 }
@@ -7486,6 +13875,21 @@ define i64 @atomicrmw_and_i64_acq_rel(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    lw ra, 12(sp)
 ; RV32IA-NEXT:    addi sp, sp, 16
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_and_i64_acq_rel:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a2, zero, 4
+; RV64I-NEXT:    call __atomic_fetch_and_8
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_and_i64_acq_rel:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    amoand.d.aqrl a0, a1, (a0)
+; RV64IA-NEXT:    ret
   %1 = atomicrmw and i64* %a, i64 %b acq_rel
   ret i64 %1
 }
@@ -7510,6 +13914,21 @@ define i64 @atomicrmw_and_i64_seq_cst(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    lw ra, 12(sp)
 ; RV32IA-NEXT:    addi sp, sp, 16
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_and_i64_seq_cst:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a2, zero, 5
+; RV64I-NEXT:    call __atomic_fetch_and_8
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_and_i64_seq_cst:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    amoand.d.aqrl a0, a1, (a0)
+; RV64IA-NEXT:    ret
   %1 = atomicrmw and i64* %a, i64 %b seq_cst
   ret i64 %1
 }
@@ -7534,6 +13953,28 @@ define i64 @atomicrmw_nand_i64_monotonic(i64* %a, i64 %b) {
 ; RV32IA-NEXT:    lw ra, 12(sp)
 ; RV32IA-NEXT:    addi sp, sp, 16
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_nand_i64_monotonic:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    mv a2, zero
+; RV64I-NEXT:    call __atomic_fetch_nand_8
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_nand_i64_monotonic:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:  .LBB185_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.d a2, (a0)
+; RV64IA-NEXT:    and a3, a2, a1
+; RV64IA-NEXT:    not a3, a3
+; RV64IA-NEXT:    sc.d a3, a3, (a0)
+; RV64IA-NEXT:    bnez a3, .LBB185_1
+; RV64IA-NEXT:  # %bb.2:
+; RV64IA-NEXT:    mv a0, a2
+; RV64IA-NEXT:    ret
   %1 = atomicrmw nand i64* %a, i64 %b monotonic
   ret i64 %1
 }
@@ -7558,6 +13999,28 @@ define i64 @atomicrmw_nand_i64_acquire(i64* %a, i64 %b) {
 ; RV32IA-NEXT:    lw ra, 12(sp)
 ; RV32IA-NEXT:    addi sp, sp, 16
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_nand_i64_acquire:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a2, zero, 2
+; RV64I-NEXT:    call __atomic_fetch_nand_8
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_nand_i64_acquire:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:  .LBB186_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.d.aq a2, (a0)
+; RV64IA-NEXT:    and a3, a2, a1
+; RV64IA-NEXT:    not a3, a3
+; RV64IA-NEXT:    sc.d a3, a3, (a0)
+; RV64IA-NEXT:    bnez a3, .LBB186_1
+; RV64IA-NEXT:  # %bb.2:
+; RV64IA-NEXT:    mv a0, a2
+; RV64IA-NEXT:    ret
   %1 = atomicrmw nand i64* %a, i64 %b acquire
   ret i64 %1
 }
@@ -7582,6 +14045,28 @@ define i64 @atomicrmw_nand_i64_release(i64* %a, i64 %b) {
 ; RV32IA-NEXT:    lw ra, 12(sp)
 ; RV32IA-NEXT:    addi sp, sp, 16
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_nand_i64_release:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a2, zero, 3
+; RV64I-NEXT:    call __atomic_fetch_nand_8
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_nand_i64_release:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:  .LBB187_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.d a2, (a0)
+; RV64IA-NEXT:    and a3, a2, a1
+; RV64IA-NEXT:    not a3, a3
+; RV64IA-NEXT:    sc.d.rl a3, a3, (a0)
+; RV64IA-NEXT:    bnez a3, .LBB187_1
+; RV64IA-NEXT:  # %bb.2:
+; RV64IA-NEXT:    mv a0, a2
+; RV64IA-NEXT:    ret
   %1 = atomicrmw nand i64* %a, i64 %b release
   ret i64 %1
 }
@@ -7606,6 +14091,28 @@ define i64 @atomicrmw_nand_i64_acq_rel(i64* %a, i64 %b) {
 ; RV32IA-NEXT:    lw ra, 12(sp)
 ; RV32IA-NEXT:    addi sp, sp, 16
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_nand_i64_acq_rel:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a2, zero, 4
+; RV64I-NEXT:    call __atomic_fetch_nand_8
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_nand_i64_acq_rel:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:  .LBB188_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.d.aq a2, (a0)
+; RV64IA-NEXT:    and a3, a2, a1
+; RV64IA-NEXT:    not a3, a3
+; RV64IA-NEXT:    sc.d.rl a3, a3, (a0)
+; RV64IA-NEXT:    bnez a3, .LBB188_1
+; RV64IA-NEXT:  # %bb.2:
+; RV64IA-NEXT:    mv a0, a2
+; RV64IA-NEXT:    ret
   %1 = atomicrmw nand i64* %a, i64 %b acq_rel
   ret i64 %1
 }
@@ -7630,6 +14137,28 @@ define i64 @atomicrmw_nand_i64_seq_cst(i64* %a, i64 %b) {
 ; RV32IA-NEXT:    lw ra, 12(sp)
 ; RV32IA-NEXT:    addi sp, sp, 16
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_nand_i64_seq_cst:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a2, zero, 5
+; RV64I-NEXT:    call __atomic_fetch_nand_8
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_nand_i64_seq_cst:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:  .LBB189_1: # =>This Inner Loop Header: Depth=1
+; RV64IA-NEXT:    lr.d.aqrl a2, (a0)
+; RV64IA-NEXT:    and a3, a2, a1
+; RV64IA-NEXT:    not a3, a3
+; RV64IA-NEXT:    sc.d.aqrl a3, a3, (a0)
+; RV64IA-NEXT:    bnez a3, .LBB189_1
+; RV64IA-NEXT:  # %bb.2:
+; RV64IA-NEXT:    mv a0, a2
+; RV64IA-NEXT:    ret
   %1 = atomicrmw nand i64* %a, i64 %b seq_cst
   ret i64 %1
 }
@@ -7654,6 +14183,21 @@ define i64 @atomicrmw_or_i64_monotonic(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    lw ra, 12(sp)
 ; RV32IA-NEXT:    addi sp, sp, 16
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_or_i64_monotonic:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    mv a2, zero
+; RV64I-NEXT:    call __atomic_fetch_or_8
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_or_i64_monotonic:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    amoor.d a0, a1, (a0)
+; RV64IA-NEXT:    ret
   %1 = atomicrmw or i64* %a, i64 %b monotonic
   ret i64 %1
 }
@@ -7678,6 +14222,21 @@ define i64 @atomicrmw_or_i64_acquire(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    lw ra, 12(sp)
 ; RV32IA-NEXT:    addi sp, sp, 16
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_or_i64_acquire:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a2, zero, 2
+; RV64I-NEXT:    call __atomic_fetch_or_8
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_or_i64_acquire:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    amoor.d.aq a0, a1, (a0)
+; RV64IA-NEXT:    ret
   %1 = atomicrmw or i64* %a, i64 %b acquire
   ret i64 %1
 }
@@ -7702,6 +14261,21 @@ define i64 @atomicrmw_or_i64_release(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    lw ra, 12(sp)
 ; RV32IA-NEXT:    addi sp, sp, 16
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_or_i64_release:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a2, zero, 3
+; RV64I-NEXT:    call __atomic_fetch_or_8
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_or_i64_release:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    amoor.d.rl a0, a1, (a0)
+; RV64IA-NEXT:    ret
   %1 = atomicrmw or i64* %a, i64 %b release
   ret i64 %1
 }
@@ -7726,6 +14300,21 @@ define i64 @atomicrmw_or_i64_acq_rel(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    lw ra, 12(sp)
 ; RV32IA-NEXT:    addi sp, sp, 16
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_or_i64_acq_rel:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a2, zero, 4
+; RV64I-NEXT:    call __atomic_fetch_or_8
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_or_i64_acq_rel:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    amoor.d.aqrl a0, a1, (a0)
+; RV64IA-NEXT:    ret
   %1 = atomicrmw or i64* %a, i64 %b acq_rel
   ret i64 %1
 }
@@ -7750,6 +14339,21 @@ define i64 @atomicrmw_or_i64_seq_cst(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    lw ra, 12(sp)
 ; RV32IA-NEXT:    addi sp, sp, 16
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_or_i64_seq_cst:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a2, zero, 5
+; RV64I-NEXT:    call __atomic_fetch_or_8
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_or_i64_seq_cst:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    amoor.d.aqrl a0, a1, (a0)
+; RV64IA-NEXT:    ret
   %1 = atomicrmw or i64* %a, i64 %b seq_cst
   ret i64 %1
 }
@@ -7774,6 +14378,21 @@ define i64 @atomicrmw_xor_i64_monotonic(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    lw ra, 12(sp)
 ; RV32IA-NEXT:    addi sp, sp, 16
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_xor_i64_monotonic:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    mv a2, zero
+; RV64I-NEXT:    call __atomic_fetch_xor_8
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_xor_i64_monotonic:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    amoxor.d a0, a1, (a0)
+; RV64IA-NEXT:    ret
   %1 = atomicrmw xor i64* %a, i64 %b monotonic
   ret i64 %1
 }
@@ -7798,6 +14417,21 @@ define i64 @atomicrmw_xor_i64_acquire(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    lw ra, 12(sp)
 ; RV32IA-NEXT:    addi sp, sp, 16
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_xor_i64_acquire:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a2, zero, 2
+; RV64I-NEXT:    call __atomic_fetch_xor_8
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_xor_i64_acquire:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    amoxor.d.aq a0, a1, (a0)
+; RV64IA-NEXT:    ret
   %1 = atomicrmw xor i64* %a, i64 %b acquire
   ret i64 %1
 }
@@ -7822,6 +14456,21 @@ define i64 @atomicrmw_xor_i64_release(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    lw ra, 12(sp)
 ; RV32IA-NEXT:    addi sp, sp, 16
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_xor_i64_release:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a2, zero, 3
+; RV64I-NEXT:    call __atomic_fetch_xor_8
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_xor_i64_release:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    amoxor.d.rl a0, a1, (a0)
+; RV64IA-NEXT:    ret
   %1 = atomicrmw xor i64* %a, i64 %b release
   ret i64 %1
 }
@@ -7846,6 +14495,21 @@ define i64 @atomicrmw_xor_i64_acq_rel(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    lw ra, 12(sp)
 ; RV32IA-NEXT:    addi sp, sp, 16
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_xor_i64_acq_rel:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a2, zero, 4
+; RV64I-NEXT:    call __atomic_fetch_xor_8
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_xor_i64_acq_rel:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    amoxor.d.aqrl a0, a1, (a0)
+; RV64IA-NEXT:    ret
   %1 = atomicrmw xor i64* %a, i64 %b acq_rel
   ret i64 %1
 }
@@ -7870,6 +14534,21 @@ define i64 @atomicrmw_xor_i64_seq_cst(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    lw ra, 12(sp)
 ; RV32IA-NEXT:    addi sp, sp, 16
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_xor_i64_seq_cst:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a2, zero, 5
+; RV64I-NEXT:    call __atomic_fetch_xor_8
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_xor_i64_seq_cst:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    amoxor.d.aqrl a0, a1, (a0)
+; RV64IA-NEXT:    ret
   %1 = atomicrmw xor i64* %a, i64 %b seq_cst
   ret i64 %1
 }
@@ -7990,6 +14669,47 @@ define i64 @atomicrmw_max_i64_monotonic(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    lw ra, 28(sp)
 ; RV32IA-NEXT:    addi sp, sp, 32
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_max_i64_monotonic:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -48
+; RV64I-NEXT:    sd ra, 40(sp)
+; RV64I-NEXT:    sd s1, 32(sp)
+; RV64I-NEXT:    sd s2, 24(sp)
+; RV64I-NEXT:    sd s3, 16(sp)
+; RV64I-NEXT:    mv s1, a1
+; RV64I-NEXT:    mv s2, a0
+; RV64I-NEXT:    ld a2, 0(a0)
+; RV64I-NEXT:    addi s3, sp, 8
+; RV64I-NEXT:  .LBB200_1: # %atomicrmw.start
+; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV64I-NEXT:    sd a2, 8(sp)
+; RV64I-NEXT:    blt s1, a2, .LBB200_3
+; RV64I-NEXT:  # %bb.2: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB200_1 Depth=1
+; RV64I-NEXT:    mv a2, s1
+; RV64I-NEXT:  .LBB200_3: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB200_1 Depth=1
+; RV64I-NEXT:    mv a0, s2
+; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    mv a3, zero
+; RV64I-NEXT:    mv a4, zero
+; RV64I-NEXT:    call __atomic_compare_exchange_8
+; RV64I-NEXT:    ld a2, 8(sp)
+; RV64I-NEXT:    beqz a0, .LBB200_1
+; RV64I-NEXT:  # %bb.4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a2
+; RV64I-NEXT:    ld s3, 16(sp)
+; RV64I-NEXT:    ld s2, 24(sp)
+; RV64I-NEXT:    ld s1, 32(sp)
+; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    addi sp, sp, 48
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_max_i64_monotonic:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    amomax.d a0, a1, (a0)
+; RV64IA-NEXT:    ret
   %1 = atomicrmw max i64* %a, i64 %b monotonic
   ret i64 %1
 }
@@ -8003,14 +14723,12 @@ define i64 @atomicrmw_max_i64_acquire(i64 *%a, i64 %b) nounwind {
 ; RV32I-NEXT:    sw s2, 20(sp)
 ; RV32I-NEXT:    sw s3, 16(sp)
 ; RV32I-NEXT:    sw s4, 12(sp)
-; RV32I-NEXT:    sw s5, 8(sp)
 ; RV32I-NEXT:    mv s1, a2
 ; RV32I-NEXT:    mv s2, a1
 ; RV32I-NEXT:    mv s3, a0
 ; RV32I-NEXT:    lw a1, 4(a0)
 ; RV32I-NEXT:    lw a2, 0(a0)
 ; RV32I-NEXT:    mv s4, sp
-; RV32I-NEXT:    addi s5, zero, 2
 ; RV32I-NEXT:  .LBB201_1: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
 ; RV32I-NEXT:    beq a1, s1, .LBB201_3
@@ -8039,15 +14757,14 @@ define i64 @atomicrmw_max_i64_acquire(i64 *%a, i64 %b) nounwind {
 ; RV32I-NEXT:    sw a1, 4(sp)
 ; RV32I-NEXT:    mv a0, s3
 ; RV32I-NEXT:    mv a1, s4
-; RV32I-NEXT:    mv a4, s5
-; RV32I-NEXT:    mv a5, s5
+; RV32I-NEXT:    addi a4, zero, 2
+; RV32I-NEXT:    addi a5, zero, 2
 ; RV32I-NEXT:    call __atomic_compare_exchange_8
 ; RV32I-NEXT:    lw a1, 4(sp)
 ; RV32I-NEXT:    lw a2, 0(sp)
 ; RV32I-NEXT:    beqz a0, .LBB201_1
 ; RV32I-NEXT:  # %bb.8: # %atomicrmw.end
 ; RV32I-NEXT:    mv a0, a2
-; RV32I-NEXT:    lw s5, 8(sp)
 ; RV32I-NEXT:    lw s4, 12(sp)
 ; RV32I-NEXT:    lw s3, 16(sp)
 ; RV32I-NEXT:    lw s2, 20(sp)
@@ -8064,14 +14781,12 @@ define i64 @atomicrmw_max_i64_acquire(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    sw s2, 20(sp)
 ; RV32IA-NEXT:    sw s3, 16(sp)
 ; RV32IA-NEXT:    sw s4, 12(sp)
-; RV32IA-NEXT:    sw s5, 8(sp)
 ; RV32IA-NEXT:    mv s1, a2
 ; RV32IA-NEXT:    mv s2, a1
 ; RV32IA-NEXT:    mv s3, a0
 ; RV32IA-NEXT:    lw a1, 4(a0)
 ; RV32IA-NEXT:    lw a2, 0(a0)
 ; RV32IA-NEXT:    mv s4, sp
-; RV32IA-NEXT:    addi s5, zero, 2
 ; RV32IA-NEXT:  .LBB201_1: # %atomicrmw.start
 ; RV32IA-NEXT:    # =>This Inner Loop Header: Depth=1
 ; RV32IA-NEXT:    beq a1, s1, .LBB201_3
@@ -8100,15 +14815,14 @@ define i64 @atomicrmw_max_i64_acquire(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    sw a1, 4(sp)
 ; RV32IA-NEXT:    mv a0, s3
 ; RV32IA-NEXT:    mv a1, s4
-; RV32IA-NEXT:    mv a4, s5
-; RV32IA-NEXT:    mv a5, s5
+; RV32IA-NEXT:    addi a4, zero, 2
+; RV32IA-NEXT:    addi a5, zero, 2
 ; RV32IA-NEXT:    call __atomic_compare_exchange_8
 ; RV32IA-NEXT:    lw a1, 4(sp)
 ; RV32IA-NEXT:    lw a2, 0(sp)
 ; RV32IA-NEXT:    beqz a0, .LBB201_1
 ; RV32IA-NEXT:  # %bb.8: # %atomicrmw.end
 ; RV32IA-NEXT:    mv a0, a2
-; RV32IA-NEXT:    lw s5, 8(sp)
 ; RV32IA-NEXT:    lw s4, 12(sp)
 ; RV32IA-NEXT:    lw s3, 16(sp)
 ; RV32IA-NEXT:    lw s2, 20(sp)
@@ -8116,6 +14830,47 @@ define i64 @atomicrmw_max_i64_acquire(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    lw ra, 28(sp)
 ; RV32IA-NEXT:    addi sp, sp, 32
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_max_i64_acquire:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -48
+; RV64I-NEXT:    sd ra, 40(sp)
+; RV64I-NEXT:    sd s1, 32(sp)
+; RV64I-NEXT:    sd s2, 24(sp)
+; RV64I-NEXT:    sd s3, 16(sp)
+; RV64I-NEXT:    mv s1, a1
+; RV64I-NEXT:    mv s2, a0
+; RV64I-NEXT:    ld a2, 0(a0)
+; RV64I-NEXT:    addi s3, sp, 8
+; RV64I-NEXT:  .LBB201_1: # %atomicrmw.start
+; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV64I-NEXT:    sd a2, 8(sp)
+; RV64I-NEXT:    blt s1, a2, .LBB201_3
+; RV64I-NEXT:  # %bb.2: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB201_1 Depth=1
+; RV64I-NEXT:    mv a2, s1
+; RV64I-NEXT:  .LBB201_3: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB201_1 Depth=1
+; RV64I-NEXT:    mv a0, s2
+; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    addi a3, zero, 2
+; RV64I-NEXT:    addi a4, zero, 2
+; RV64I-NEXT:    call __atomic_compare_exchange_8
+; RV64I-NEXT:    ld a2, 8(sp)
+; RV64I-NEXT:    beqz a0, .LBB201_1
+; RV64I-NEXT:  # %bb.4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a2
+; RV64I-NEXT:    ld s3, 16(sp)
+; RV64I-NEXT:    ld s2, 24(sp)
+; RV64I-NEXT:    ld s1, 32(sp)
+; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    addi sp, sp, 48
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_max_i64_acquire:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    amomax.d.aq a0, a1, (a0)
+; RV64IA-NEXT:    ret
   %1 = atomicrmw max i64* %a, i64 %b acquire
   ret i64 %1
 }
@@ -8129,14 +14884,12 @@ define i64 @atomicrmw_max_i64_release(i64 *%a, i64 %b) nounwind {
 ; RV32I-NEXT:    sw s2, 20(sp)
 ; RV32I-NEXT:    sw s3, 16(sp)
 ; RV32I-NEXT:    sw s4, 12(sp)
-; RV32I-NEXT:    sw s5, 8(sp)
 ; RV32I-NEXT:    mv s1, a2
 ; RV32I-NEXT:    mv s2, a1
 ; RV32I-NEXT:    mv s3, a0
 ; RV32I-NEXT:    lw a1, 4(a0)
 ; RV32I-NEXT:    lw a2, 0(a0)
 ; RV32I-NEXT:    mv s4, sp
-; RV32I-NEXT:    addi s5, zero, 3
 ; RV32I-NEXT:  .LBB202_1: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
 ; RV32I-NEXT:    beq a1, s1, .LBB202_3
@@ -8165,7 +14918,7 @@ define i64 @atomicrmw_max_i64_release(i64 *%a, i64 %b) nounwind {
 ; RV32I-NEXT:    sw a1, 4(sp)
 ; RV32I-NEXT:    mv a0, s3
 ; RV32I-NEXT:    mv a1, s4
-; RV32I-NEXT:    mv a4, s5
+; RV32I-NEXT:    addi a4, zero, 3
 ; RV32I-NEXT:    mv a5, zero
 ; RV32I-NEXT:    call __atomic_compare_exchange_8
 ; RV32I-NEXT:    lw a1, 4(sp)
@@ -8173,7 +14926,6 @@ define i64 @atomicrmw_max_i64_release(i64 *%a, i64 %b) nounwind {
 ; RV32I-NEXT:    beqz a0, .LBB202_1
 ; RV32I-NEXT:  # %bb.8: # %atomicrmw.end
 ; RV32I-NEXT:    mv a0, a2
-; RV32I-NEXT:    lw s5, 8(sp)
 ; RV32I-NEXT:    lw s4, 12(sp)
 ; RV32I-NEXT:    lw s3, 16(sp)
 ; RV32I-NEXT:    lw s2, 20(sp)
@@ -8190,14 +14942,12 @@ define i64 @atomicrmw_max_i64_release(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    sw s2, 20(sp)
 ; RV32IA-NEXT:    sw s3, 16(sp)
 ; RV32IA-NEXT:    sw s4, 12(sp)
-; RV32IA-NEXT:    sw s5, 8(sp)
 ; RV32IA-NEXT:    mv s1, a2
 ; RV32IA-NEXT:    mv s2, a1
 ; RV32IA-NEXT:    mv s3, a0
 ; RV32IA-NEXT:    lw a1, 4(a0)
 ; RV32IA-NEXT:    lw a2, 0(a0)
 ; RV32IA-NEXT:    mv s4, sp
-; RV32IA-NEXT:    addi s5, zero, 3
 ; RV32IA-NEXT:  .LBB202_1: # %atomicrmw.start
 ; RV32IA-NEXT:    # =>This Inner Loop Header: Depth=1
 ; RV32IA-NEXT:    beq a1, s1, .LBB202_3
@@ -8226,7 +14976,7 @@ define i64 @atomicrmw_max_i64_release(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    sw a1, 4(sp)
 ; RV32IA-NEXT:    mv a0, s3
 ; RV32IA-NEXT:    mv a1, s4
-; RV32IA-NEXT:    mv a4, s5
+; RV32IA-NEXT:    addi a4, zero, 3
 ; RV32IA-NEXT:    mv a5, zero
 ; RV32IA-NEXT:    call __atomic_compare_exchange_8
 ; RV32IA-NEXT:    lw a1, 4(sp)
@@ -8234,7 +14984,6 @@ define i64 @atomicrmw_max_i64_release(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    beqz a0, .LBB202_1
 ; RV32IA-NEXT:  # %bb.8: # %atomicrmw.end
 ; RV32IA-NEXT:    mv a0, a2
-; RV32IA-NEXT:    lw s5, 8(sp)
 ; RV32IA-NEXT:    lw s4, 12(sp)
 ; RV32IA-NEXT:    lw s3, 16(sp)
 ; RV32IA-NEXT:    lw s2, 20(sp)
@@ -8242,6 +14991,47 @@ define i64 @atomicrmw_max_i64_release(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    lw ra, 28(sp)
 ; RV32IA-NEXT:    addi sp, sp, 32
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_max_i64_release:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -48
+; RV64I-NEXT:    sd ra, 40(sp)
+; RV64I-NEXT:    sd s1, 32(sp)
+; RV64I-NEXT:    sd s2, 24(sp)
+; RV64I-NEXT:    sd s3, 16(sp)
+; RV64I-NEXT:    mv s1, a1
+; RV64I-NEXT:    mv s2, a0
+; RV64I-NEXT:    ld a2, 0(a0)
+; RV64I-NEXT:    addi s3, sp, 8
+; RV64I-NEXT:  .LBB202_1: # %atomicrmw.start
+; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV64I-NEXT:    sd a2, 8(sp)
+; RV64I-NEXT:    blt s1, a2, .LBB202_3
+; RV64I-NEXT:  # %bb.2: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB202_1 Depth=1
+; RV64I-NEXT:    mv a2, s1
+; RV64I-NEXT:  .LBB202_3: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB202_1 Depth=1
+; RV64I-NEXT:    mv a0, s2
+; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    addi a3, zero, 3
+; RV64I-NEXT:    mv a4, zero
+; RV64I-NEXT:    call __atomic_compare_exchange_8
+; RV64I-NEXT:    ld a2, 8(sp)
+; RV64I-NEXT:    beqz a0, .LBB202_1
+; RV64I-NEXT:  # %bb.4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a2
+; RV64I-NEXT:    ld s3, 16(sp)
+; RV64I-NEXT:    ld s2, 24(sp)
+; RV64I-NEXT:    ld s1, 32(sp)
+; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    addi sp, sp, 48
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_max_i64_release:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    amomax.d.rl a0, a1, (a0)
+; RV64IA-NEXT:    ret
   %1 = atomicrmw max i64* %a, i64 %b release
   ret i64 %1
 }
@@ -8249,34 +15039,30 @@ define i64 @atomicrmw_max_i64_release(i64 *%a, i64 %b) nounwind {
 define i64 @atomicrmw_max_i64_acq_rel(i64 *%a, i64 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_max_i64_acq_rel:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    addi sp, sp, -48
-; RV32I-NEXT:    sw ra, 44(sp)
-; RV32I-NEXT:    sw s1, 40(sp)
-; RV32I-NEXT:    sw s2, 36(sp)
-; RV32I-NEXT:    sw s3, 32(sp)
-; RV32I-NEXT:    sw s4, 28(sp)
-; RV32I-NEXT:    sw s5, 24(sp)
-; RV32I-NEXT:    sw s6, 20(sp)
+; RV32I-NEXT:    addi sp, sp, -32
+; RV32I-NEXT:    sw ra, 28(sp)
+; RV32I-NEXT:    sw s1, 24(sp)
+; RV32I-NEXT:    sw s2, 20(sp)
+; RV32I-NEXT:    sw s3, 16(sp)
+; RV32I-NEXT:    sw s4, 12(sp)
 ; RV32I-NEXT:    mv s1, a2
 ; RV32I-NEXT:    mv s2, a1
 ; RV32I-NEXT:    mv s3, a0
 ; RV32I-NEXT:    lw a1, 4(a0)
 ; RV32I-NEXT:    lw a2, 0(a0)
-; RV32I-NEXT:    addi s4, sp, 8
-; RV32I-NEXT:    addi s5, zero, 4
-; RV32I-NEXT:    addi s6, zero, 2
+; RV32I-NEXT:    mv s4, sp
 ; RV32I-NEXT:  .LBB203_1: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
 ; RV32I-NEXT:    beq a1, s1, .LBB203_3
 ; RV32I-NEXT:  # %bb.2: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB203_1 Depth=1
 ; RV32I-NEXT:    slt a0, s1, a1
-; RV32I-NEXT:    sw a2, 8(sp)
+; RV32I-NEXT:    sw a2, 0(sp)
 ; RV32I-NEXT:    beqz a0, .LBB203_4
 ; RV32I-NEXT:    j .LBB203_5
 ; RV32I-NEXT:  .LBB203_3: # in Loop: Header=BB203_1 Depth=1
 ; RV32I-NEXT:    sltu a0, s2, a2
-; RV32I-NEXT:    sw a2, 8(sp)
+; RV32I-NEXT:    sw a2, 0(sp)
 ; RV32I-NEXT:    bnez a0, .LBB203_5
 ; RV32I-NEXT:  .LBB203_4: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB203_1 Depth=1
@@ -8290,57 +15076,51 @@ define i64 @atomicrmw_max_i64_acq_rel(i64 *%a, i64 %b) nounwind {
 ; RV32I-NEXT:    mv a3, s1
 ; RV32I-NEXT:  .LBB203_7: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB203_1 Depth=1
-; RV32I-NEXT:    sw a1, 12(sp)
+; RV32I-NEXT:    sw a1, 4(sp)
 ; RV32I-NEXT:    mv a0, s3
 ; RV32I-NEXT:    mv a1, s4
-; RV32I-NEXT:    mv a4, s5
-; RV32I-NEXT:    mv a5, s6
+; RV32I-NEXT:    addi a4, zero, 4
+; RV32I-NEXT:    addi a5, zero, 2
 ; RV32I-NEXT:    call __atomic_compare_exchange_8
-; RV32I-NEXT:    lw a1, 12(sp)
-; RV32I-NEXT:    lw a2, 8(sp)
+; RV32I-NEXT:    lw a1, 4(sp)
+; RV32I-NEXT:    lw a2, 0(sp)
 ; RV32I-NEXT:    beqz a0, .LBB203_1
 ; RV32I-NEXT:  # %bb.8: # %atomicrmw.end
 ; RV32I-NEXT:    mv a0, a2
-; RV32I-NEXT:    lw s6, 20(sp)
-; RV32I-NEXT:    lw s5, 24(sp)
-; RV32I-NEXT:    lw s4, 28(sp)
-; RV32I-NEXT:    lw s3, 32(sp)
-; RV32I-NEXT:    lw s2, 36(sp)
-; RV32I-NEXT:    lw s1, 40(sp)
-; RV32I-NEXT:    lw ra, 44(sp)
-; RV32I-NEXT:    addi sp, sp, 48
+; RV32I-NEXT:    lw s4, 12(sp)
+; RV32I-NEXT:    lw s3, 16(sp)
+; RV32I-NEXT:    lw s2, 20(sp)
+; RV32I-NEXT:    lw s1, 24(sp)
+; RV32I-NEXT:    lw ra, 28(sp)
+; RV32I-NEXT:    addi sp, sp, 32
 ; RV32I-NEXT:    ret
 ;
 ; RV32IA-LABEL: atomicrmw_max_i64_acq_rel:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    addi sp, sp, -48
-; RV32IA-NEXT:    sw ra, 44(sp)
-; RV32IA-NEXT:    sw s1, 40(sp)
-; RV32IA-NEXT:    sw s2, 36(sp)
-; RV32IA-NEXT:    sw s3, 32(sp)
-; RV32IA-NEXT:    sw s4, 28(sp)
-; RV32IA-NEXT:    sw s5, 24(sp)
-; RV32IA-NEXT:    sw s6, 20(sp)
+; RV32IA-NEXT:    addi sp, sp, -32
+; RV32IA-NEXT:    sw ra, 28(sp)
+; RV32IA-NEXT:    sw s1, 24(sp)
+; RV32IA-NEXT:    sw s2, 20(sp)
+; RV32IA-NEXT:    sw s3, 16(sp)
+; RV32IA-NEXT:    sw s4, 12(sp)
 ; RV32IA-NEXT:    mv s1, a2
 ; RV32IA-NEXT:    mv s2, a1
 ; RV32IA-NEXT:    mv s3, a0
 ; RV32IA-NEXT:    lw a1, 4(a0)
 ; RV32IA-NEXT:    lw a2, 0(a0)
-; RV32IA-NEXT:    addi s4, sp, 8
-; RV32IA-NEXT:    addi s5, zero, 4
-; RV32IA-NEXT:    addi s6, zero, 2
+; RV32IA-NEXT:    mv s4, sp
 ; RV32IA-NEXT:  .LBB203_1: # %atomicrmw.start
 ; RV32IA-NEXT:    # =>This Inner Loop Header: Depth=1
 ; RV32IA-NEXT:    beq a1, s1, .LBB203_3
 ; RV32IA-NEXT:  # %bb.2: # %atomicrmw.start
 ; RV32IA-NEXT:    # in Loop: Header=BB203_1 Depth=1
 ; RV32IA-NEXT:    slt a0, s1, a1
-; RV32IA-NEXT:    sw a2, 8(sp)
+; RV32IA-NEXT:    sw a2, 0(sp)
 ; RV32IA-NEXT:    beqz a0, .LBB203_4
 ; RV32IA-NEXT:    j .LBB203_5
 ; RV32IA-NEXT:  .LBB203_3: # in Loop: Header=BB203_1 Depth=1
 ; RV32IA-NEXT:    sltu a0, s2, a2
-; RV32IA-NEXT:    sw a2, 8(sp)
+; RV32IA-NEXT:    sw a2, 0(sp)
 ; RV32IA-NEXT:    bnez a0, .LBB203_5
 ; RV32IA-NEXT:  .LBB203_4: # %atomicrmw.start
 ; RV32IA-NEXT:    # in Loop: Header=BB203_1 Depth=1
@@ -8354,26 +15134,65 @@ define i64 @atomicrmw_max_i64_acq_rel(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    mv a3, s1
 ; RV32IA-NEXT:  .LBB203_7: # %atomicrmw.start
 ; RV32IA-NEXT:    # in Loop: Header=BB203_1 Depth=1
-; RV32IA-NEXT:    sw a1, 12(sp)
+; RV32IA-NEXT:    sw a1, 4(sp)
 ; RV32IA-NEXT:    mv a0, s3
 ; RV32IA-NEXT:    mv a1, s4
-; RV32IA-NEXT:    mv a4, s5
-; RV32IA-NEXT:    mv a5, s6
+; RV32IA-NEXT:    addi a4, zero, 4
+; RV32IA-NEXT:    addi a5, zero, 2
 ; RV32IA-NEXT:    call __atomic_compare_exchange_8
-; RV32IA-NEXT:    lw a1, 12(sp)
-; RV32IA-NEXT:    lw a2, 8(sp)
+; RV32IA-NEXT:    lw a1, 4(sp)
+; RV32IA-NEXT:    lw a2, 0(sp)
 ; RV32IA-NEXT:    beqz a0, .LBB203_1
 ; RV32IA-NEXT:  # %bb.8: # %atomicrmw.end
 ; RV32IA-NEXT:    mv a0, a2
-; RV32IA-NEXT:    lw s6, 20(sp)
-; RV32IA-NEXT:    lw s5, 24(sp)
-; RV32IA-NEXT:    lw s4, 28(sp)
-; RV32IA-NEXT:    lw s3, 32(sp)
-; RV32IA-NEXT:    lw s2, 36(sp)
-; RV32IA-NEXT:    lw s1, 40(sp)
-; RV32IA-NEXT:    lw ra, 44(sp)
-; RV32IA-NEXT:    addi sp, sp, 48
+; RV32IA-NEXT:    lw s4, 12(sp)
+; RV32IA-NEXT:    lw s3, 16(sp)
+; RV32IA-NEXT:    lw s2, 20(sp)
+; RV32IA-NEXT:    lw s1, 24(sp)
+; RV32IA-NEXT:    lw ra, 28(sp)
+; RV32IA-NEXT:    addi sp, sp, 32
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_max_i64_acq_rel:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -48
+; RV64I-NEXT:    sd ra, 40(sp)
+; RV64I-NEXT:    sd s1, 32(sp)
+; RV64I-NEXT:    sd s2, 24(sp)
+; RV64I-NEXT:    sd s3, 16(sp)
+; RV64I-NEXT:    mv s1, a1
+; RV64I-NEXT:    mv s2, a0
+; RV64I-NEXT:    ld a2, 0(a0)
+; RV64I-NEXT:    addi s3, sp, 8
+; RV64I-NEXT:  .LBB203_1: # %atomicrmw.start
+; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV64I-NEXT:    sd a2, 8(sp)
+; RV64I-NEXT:    blt s1, a2, .LBB203_3
+; RV64I-NEXT:  # %bb.2: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB203_1 Depth=1
+; RV64I-NEXT:    mv a2, s1
+; RV64I-NEXT:  .LBB203_3: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB203_1 Depth=1
+; RV64I-NEXT:    mv a0, s2
+; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    addi a3, zero, 4
+; RV64I-NEXT:    addi a4, zero, 2
+; RV64I-NEXT:    call __atomic_compare_exchange_8
+; RV64I-NEXT:    ld a2, 8(sp)
+; RV64I-NEXT:    beqz a0, .LBB203_1
+; RV64I-NEXT:  # %bb.4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a2
+; RV64I-NEXT:    ld s3, 16(sp)
+; RV64I-NEXT:    ld s2, 24(sp)
+; RV64I-NEXT:    ld s1, 32(sp)
+; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    addi sp, sp, 48
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_max_i64_acq_rel:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    amomax.d.aqrl a0, a1, (a0)
+; RV64IA-NEXT:    ret
   %1 = atomicrmw max i64* %a, i64 %b acq_rel
   ret i64 %1
 }
@@ -8387,14 +15206,12 @@ define i64 @atomicrmw_max_i64_seq_cst(i64 *%a, i64 %b) nounwind {
 ; RV32I-NEXT:    sw s2, 20(sp)
 ; RV32I-NEXT:    sw s3, 16(sp)
 ; RV32I-NEXT:    sw s4, 12(sp)
-; RV32I-NEXT:    sw s5, 8(sp)
 ; RV32I-NEXT:    mv s1, a2
 ; RV32I-NEXT:    mv s2, a1
 ; RV32I-NEXT:    mv s3, a0
 ; RV32I-NEXT:    lw a1, 4(a0)
 ; RV32I-NEXT:    lw a2, 0(a0)
 ; RV32I-NEXT:    mv s4, sp
-; RV32I-NEXT:    addi s5, zero, 5
 ; RV32I-NEXT:  .LBB204_1: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
 ; RV32I-NEXT:    beq a1, s1, .LBB204_3
@@ -8423,15 +15240,14 @@ define i64 @atomicrmw_max_i64_seq_cst(i64 *%a, i64 %b) nounwind {
 ; RV32I-NEXT:    sw a1, 4(sp)
 ; RV32I-NEXT:    mv a0, s3
 ; RV32I-NEXT:    mv a1, s4
-; RV32I-NEXT:    mv a4, s5
-; RV32I-NEXT:    mv a5, s5
+; RV32I-NEXT:    addi a4, zero, 5
+; RV32I-NEXT:    addi a5, zero, 5
 ; RV32I-NEXT:    call __atomic_compare_exchange_8
 ; RV32I-NEXT:    lw a1, 4(sp)
 ; RV32I-NEXT:    lw a2, 0(sp)
 ; RV32I-NEXT:    beqz a0, .LBB204_1
 ; RV32I-NEXT:  # %bb.8: # %atomicrmw.end
 ; RV32I-NEXT:    mv a0, a2
-; RV32I-NEXT:    lw s5, 8(sp)
 ; RV32I-NEXT:    lw s4, 12(sp)
 ; RV32I-NEXT:    lw s3, 16(sp)
 ; RV32I-NEXT:    lw s2, 20(sp)
@@ -8448,14 +15264,12 @@ define i64 @atomicrmw_max_i64_seq_cst(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    sw s2, 20(sp)
 ; RV32IA-NEXT:    sw s3, 16(sp)
 ; RV32IA-NEXT:    sw s4, 12(sp)
-; RV32IA-NEXT:    sw s5, 8(sp)
 ; RV32IA-NEXT:    mv s1, a2
 ; RV32IA-NEXT:    mv s2, a1
 ; RV32IA-NEXT:    mv s3, a0
 ; RV32IA-NEXT:    lw a1, 4(a0)
 ; RV32IA-NEXT:    lw a2, 0(a0)
 ; RV32IA-NEXT:    mv s4, sp
-; RV32IA-NEXT:    addi s5, zero, 5
 ; RV32IA-NEXT:  .LBB204_1: # %atomicrmw.start
 ; RV32IA-NEXT:    # =>This Inner Loop Header: Depth=1
 ; RV32IA-NEXT:    beq a1, s1, .LBB204_3
@@ -8484,15 +15298,14 @@ define i64 @atomicrmw_max_i64_seq_cst(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    sw a1, 4(sp)
 ; RV32IA-NEXT:    mv a0, s3
 ; RV32IA-NEXT:    mv a1, s4
-; RV32IA-NEXT:    mv a4, s5
-; RV32IA-NEXT:    mv a5, s5
+; RV32IA-NEXT:    addi a4, zero, 5
+; RV32IA-NEXT:    addi a5, zero, 5
 ; RV32IA-NEXT:    call __atomic_compare_exchange_8
 ; RV32IA-NEXT:    lw a1, 4(sp)
 ; RV32IA-NEXT:    lw a2, 0(sp)
 ; RV32IA-NEXT:    beqz a0, .LBB204_1
 ; RV32IA-NEXT:  # %bb.8: # %atomicrmw.end
 ; RV32IA-NEXT:    mv a0, a2
-; RV32IA-NEXT:    lw s5, 8(sp)
 ; RV32IA-NEXT:    lw s4, 12(sp)
 ; RV32IA-NEXT:    lw s3, 16(sp)
 ; RV32IA-NEXT:    lw s2, 20(sp)
@@ -8500,6 +15313,47 @@ define i64 @atomicrmw_max_i64_seq_cst(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    lw ra, 28(sp)
 ; RV32IA-NEXT:    addi sp, sp, 32
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_max_i64_seq_cst:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -48
+; RV64I-NEXT:    sd ra, 40(sp)
+; RV64I-NEXT:    sd s1, 32(sp)
+; RV64I-NEXT:    sd s2, 24(sp)
+; RV64I-NEXT:    sd s3, 16(sp)
+; RV64I-NEXT:    mv s1, a1
+; RV64I-NEXT:    mv s2, a0
+; RV64I-NEXT:    ld a2, 0(a0)
+; RV64I-NEXT:    addi s3, sp, 8
+; RV64I-NEXT:  .LBB204_1: # %atomicrmw.start
+; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV64I-NEXT:    sd a2, 8(sp)
+; RV64I-NEXT:    blt s1, a2, .LBB204_3
+; RV64I-NEXT:  # %bb.2: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB204_1 Depth=1
+; RV64I-NEXT:    mv a2, s1
+; RV64I-NEXT:  .LBB204_3: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB204_1 Depth=1
+; RV64I-NEXT:    mv a0, s2
+; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    addi a3, zero, 5
+; RV64I-NEXT:    addi a4, zero, 5
+; RV64I-NEXT:    call __atomic_compare_exchange_8
+; RV64I-NEXT:    ld a2, 8(sp)
+; RV64I-NEXT:    beqz a0, .LBB204_1
+; RV64I-NEXT:  # %bb.4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a2
+; RV64I-NEXT:    ld s3, 16(sp)
+; RV64I-NEXT:    ld s2, 24(sp)
+; RV64I-NEXT:    ld s1, 32(sp)
+; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    addi sp, sp, 48
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_max_i64_seq_cst:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    amomax.d.aqrl a0, a1, (a0)
+; RV64IA-NEXT:    ret
   %1 = atomicrmw max i64* %a, i64 %b seq_cst
   ret i64 %1
 }
@@ -8622,6 +15476,47 @@ define i64 @atomicrmw_min_i64_monotonic(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    lw ra, 28(sp)
 ; RV32IA-NEXT:    addi sp, sp, 32
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_min_i64_monotonic:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -48
+; RV64I-NEXT:    sd ra, 40(sp)
+; RV64I-NEXT:    sd s1, 32(sp)
+; RV64I-NEXT:    sd s2, 24(sp)
+; RV64I-NEXT:    sd s3, 16(sp)
+; RV64I-NEXT:    mv s1, a1
+; RV64I-NEXT:    mv s2, a0
+; RV64I-NEXT:    ld a2, 0(a0)
+; RV64I-NEXT:    addi s3, sp, 8
+; RV64I-NEXT:  .LBB205_1: # %atomicrmw.start
+; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV64I-NEXT:    sd a2, 8(sp)
+; RV64I-NEXT:    bge s1, a2, .LBB205_3
+; RV64I-NEXT:  # %bb.2: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB205_1 Depth=1
+; RV64I-NEXT:    mv a2, s1
+; RV64I-NEXT:  .LBB205_3: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB205_1 Depth=1
+; RV64I-NEXT:    mv a0, s2
+; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    mv a3, zero
+; RV64I-NEXT:    mv a4, zero
+; RV64I-NEXT:    call __atomic_compare_exchange_8
+; RV64I-NEXT:    ld a2, 8(sp)
+; RV64I-NEXT:    beqz a0, .LBB205_1
+; RV64I-NEXT:  # %bb.4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a2
+; RV64I-NEXT:    ld s3, 16(sp)
+; RV64I-NEXT:    ld s2, 24(sp)
+; RV64I-NEXT:    ld s1, 32(sp)
+; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    addi sp, sp, 48
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_min_i64_monotonic:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    amomin.d a0, a1, (a0)
+; RV64IA-NEXT:    ret
   %1 = atomicrmw min i64* %a, i64 %b monotonic
   ret i64 %1
 }
@@ -8635,14 +15530,12 @@ define i64 @atomicrmw_min_i64_acquire(i64 *%a, i64 %b) nounwind {
 ; RV32I-NEXT:    sw s2, 20(sp)
 ; RV32I-NEXT:    sw s3, 16(sp)
 ; RV32I-NEXT:    sw s4, 12(sp)
-; RV32I-NEXT:    sw s5, 8(sp)
 ; RV32I-NEXT:    mv s1, a2
 ; RV32I-NEXT:    mv s2, a1
 ; RV32I-NEXT:    mv s3, a0
 ; RV32I-NEXT:    lw a1, 4(a0)
 ; RV32I-NEXT:    lw a2, 0(a0)
 ; RV32I-NEXT:    mv s4, sp
-; RV32I-NEXT:    addi s5, zero, 2
 ; RV32I-NEXT:  .LBB206_1: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
 ; RV32I-NEXT:    beq a1, s1, .LBB206_3
@@ -8672,15 +15565,14 @@ define i64 @atomicrmw_min_i64_acquire(i64 *%a, i64 %b) nounwind {
 ; RV32I-NEXT:    sw a1, 4(sp)
 ; RV32I-NEXT:    mv a0, s3
 ; RV32I-NEXT:    mv a1, s4
-; RV32I-NEXT:    mv a4, s5
-; RV32I-NEXT:    mv a5, s5
+; RV32I-NEXT:    addi a4, zero, 2
+; RV32I-NEXT:    addi a5, zero, 2
 ; RV32I-NEXT:    call __atomic_compare_exchange_8
 ; RV32I-NEXT:    lw a1, 4(sp)
 ; RV32I-NEXT:    lw a2, 0(sp)
 ; RV32I-NEXT:    beqz a0, .LBB206_1
 ; RV32I-NEXT:  # %bb.9: # %atomicrmw.end
 ; RV32I-NEXT:    mv a0, a2
-; RV32I-NEXT:    lw s5, 8(sp)
 ; RV32I-NEXT:    lw s4, 12(sp)
 ; RV32I-NEXT:    lw s3, 16(sp)
 ; RV32I-NEXT:    lw s2, 20(sp)
@@ -8697,14 +15589,12 @@ define i64 @atomicrmw_min_i64_acquire(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    sw s2, 20(sp)
 ; RV32IA-NEXT:    sw s3, 16(sp)
 ; RV32IA-NEXT:    sw s4, 12(sp)
-; RV32IA-NEXT:    sw s5, 8(sp)
 ; RV32IA-NEXT:    mv s1, a2
 ; RV32IA-NEXT:    mv s2, a1
 ; RV32IA-NEXT:    mv s3, a0
 ; RV32IA-NEXT:    lw a1, 4(a0)
 ; RV32IA-NEXT:    lw a2, 0(a0)
 ; RV32IA-NEXT:    mv s4, sp
-; RV32IA-NEXT:    addi s5, zero, 2
 ; RV32IA-NEXT:  .LBB206_1: # %atomicrmw.start
 ; RV32IA-NEXT:    # =>This Inner Loop Header: Depth=1
 ; RV32IA-NEXT:    beq a1, s1, .LBB206_3
@@ -8734,15 +15624,14 @@ define i64 @atomicrmw_min_i64_acquire(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    sw a1, 4(sp)
 ; RV32IA-NEXT:    mv a0, s3
 ; RV32IA-NEXT:    mv a1, s4
-; RV32IA-NEXT:    mv a4, s5
-; RV32IA-NEXT:    mv a5, s5
+; RV32IA-NEXT:    addi a4, zero, 2
+; RV32IA-NEXT:    addi a5, zero, 2
 ; RV32IA-NEXT:    call __atomic_compare_exchange_8
 ; RV32IA-NEXT:    lw a1, 4(sp)
 ; RV32IA-NEXT:    lw a2, 0(sp)
 ; RV32IA-NEXT:    beqz a0, .LBB206_1
 ; RV32IA-NEXT:  # %bb.9: # %atomicrmw.end
 ; RV32IA-NEXT:    mv a0, a2
-; RV32IA-NEXT:    lw s5, 8(sp)
 ; RV32IA-NEXT:    lw s4, 12(sp)
 ; RV32IA-NEXT:    lw s3, 16(sp)
 ; RV32IA-NEXT:    lw s2, 20(sp)
@@ -8750,6 +15639,47 @@ define i64 @atomicrmw_min_i64_acquire(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    lw ra, 28(sp)
 ; RV32IA-NEXT:    addi sp, sp, 32
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_min_i64_acquire:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -48
+; RV64I-NEXT:    sd ra, 40(sp)
+; RV64I-NEXT:    sd s1, 32(sp)
+; RV64I-NEXT:    sd s2, 24(sp)
+; RV64I-NEXT:    sd s3, 16(sp)
+; RV64I-NEXT:    mv s1, a1
+; RV64I-NEXT:    mv s2, a0
+; RV64I-NEXT:    ld a2, 0(a0)
+; RV64I-NEXT:    addi s3, sp, 8
+; RV64I-NEXT:  .LBB206_1: # %atomicrmw.start
+; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV64I-NEXT:    sd a2, 8(sp)
+; RV64I-NEXT:    bge s1, a2, .LBB206_3
+; RV64I-NEXT:  # %bb.2: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB206_1 Depth=1
+; RV64I-NEXT:    mv a2, s1
+; RV64I-NEXT:  .LBB206_3: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB206_1 Depth=1
+; RV64I-NEXT:    mv a0, s2
+; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    addi a3, zero, 2
+; RV64I-NEXT:    addi a4, zero, 2
+; RV64I-NEXT:    call __atomic_compare_exchange_8
+; RV64I-NEXT:    ld a2, 8(sp)
+; RV64I-NEXT:    beqz a0, .LBB206_1
+; RV64I-NEXT:  # %bb.4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a2
+; RV64I-NEXT:    ld s3, 16(sp)
+; RV64I-NEXT:    ld s2, 24(sp)
+; RV64I-NEXT:    ld s1, 32(sp)
+; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    addi sp, sp, 48
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_min_i64_acquire:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    amomin.d.aq a0, a1, (a0)
+; RV64IA-NEXT:    ret
   %1 = atomicrmw min i64* %a, i64 %b acquire
   ret i64 %1
 }
@@ -8763,14 +15693,12 @@ define i64 @atomicrmw_min_i64_release(i64 *%a, i64 %b) nounwind {
 ; RV32I-NEXT:    sw s2, 20(sp)
 ; RV32I-NEXT:    sw s3, 16(sp)
 ; RV32I-NEXT:    sw s4, 12(sp)
-; RV32I-NEXT:    sw s5, 8(sp)
 ; RV32I-NEXT:    mv s1, a2
 ; RV32I-NEXT:    mv s2, a1
 ; RV32I-NEXT:    mv s3, a0
 ; RV32I-NEXT:    lw a1, 4(a0)
 ; RV32I-NEXT:    lw a2, 0(a0)
 ; RV32I-NEXT:    mv s4, sp
-; RV32I-NEXT:    addi s5, zero, 3
 ; RV32I-NEXT:  .LBB207_1: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
 ; RV32I-NEXT:    beq a1, s1, .LBB207_3
@@ -8800,7 +15728,7 @@ define i64 @atomicrmw_min_i64_release(i64 *%a, i64 %b) nounwind {
 ; RV32I-NEXT:    sw a1, 4(sp)
 ; RV32I-NEXT:    mv a0, s3
 ; RV32I-NEXT:    mv a1, s4
-; RV32I-NEXT:    mv a4, s5
+; RV32I-NEXT:    addi a4, zero, 3
 ; RV32I-NEXT:    mv a5, zero
 ; RV32I-NEXT:    call __atomic_compare_exchange_8
 ; RV32I-NEXT:    lw a1, 4(sp)
@@ -8808,7 +15736,6 @@ define i64 @atomicrmw_min_i64_release(i64 *%a, i64 %b) nounwind {
 ; RV32I-NEXT:    beqz a0, .LBB207_1
 ; RV32I-NEXT:  # %bb.9: # %atomicrmw.end
 ; RV32I-NEXT:    mv a0, a2
-; RV32I-NEXT:    lw s5, 8(sp)
 ; RV32I-NEXT:    lw s4, 12(sp)
 ; RV32I-NEXT:    lw s3, 16(sp)
 ; RV32I-NEXT:    lw s2, 20(sp)
@@ -8825,14 +15752,12 @@ define i64 @atomicrmw_min_i64_release(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    sw s2, 20(sp)
 ; RV32IA-NEXT:    sw s3, 16(sp)
 ; RV32IA-NEXT:    sw s4, 12(sp)
-; RV32IA-NEXT:    sw s5, 8(sp)
 ; RV32IA-NEXT:    mv s1, a2
 ; RV32IA-NEXT:    mv s2, a1
 ; RV32IA-NEXT:    mv s3, a0
 ; RV32IA-NEXT:    lw a1, 4(a0)
 ; RV32IA-NEXT:    lw a2, 0(a0)
 ; RV32IA-NEXT:    mv s4, sp
-; RV32IA-NEXT:    addi s5, zero, 3
 ; RV32IA-NEXT:  .LBB207_1: # %atomicrmw.start
 ; RV32IA-NEXT:    # =>This Inner Loop Header: Depth=1
 ; RV32IA-NEXT:    beq a1, s1, .LBB207_3
@@ -8862,7 +15787,7 @@ define i64 @atomicrmw_min_i64_release(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    sw a1, 4(sp)
 ; RV32IA-NEXT:    mv a0, s3
 ; RV32IA-NEXT:    mv a1, s4
-; RV32IA-NEXT:    mv a4, s5
+; RV32IA-NEXT:    addi a4, zero, 3
 ; RV32IA-NEXT:    mv a5, zero
 ; RV32IA-NEXT:    call __atomic_compare_exchange_8
 ; RV32IA-NEXT:    lw a1, 4(sp)
@@ -8870,7 +15795,6 @@ define i64 @atomicrmw_min_i64_release(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    beqz a0, .LBB207_1
 ; RV32IA-NEXT:  # %bb.9: # %atomicrmw.end
 ; RV32IA-NEXT:    mv a0, a2
-; RV32IA-NEXT:    lw s5, 8(sp)
 ; RV32IA-NEXT:    lw s4, 12(sp)
 ; RV32IA-NEXT:    lw s3, 16(sp)
 ; RV32IA-NEXT:    lw s2, 20(sp)
@@ -8878,6 +15802,47 @@ define i64 @atomicrmw_min_i64_release(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    lw ra, 28(sp)
 ; RV32IA-NEXT:    addi sp, sp, 32
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_min_i64_release:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -48
+; RV64I-NEXT:    sd ra, 40(sp)
+; RV64I-NEXT:    sd s1, 32(sp)
+; RV64I-NEXT:    sd s2, 24(sp)
+; RV64I-NEXT:    sd s3, 16(sp)
+; RV64I-NEXT:    mv s1, a1
+; RV64I-NEXT:    mv s2, a0
+; RV64I-NEXT:    ld a2, 0(a0)
+; RV64I-NEXT:    addi s3, sp, 8
+; RV64I-NEXT:  .LBB207_1: # %atomicrmw.start
+; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV64I-NEXT:    sd a2, 8(sp)
+; RV64I-NEXT:    bge s1, a2, .LBB207_3
+; RV64I-NEXT:  # %bb.2: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB207_1 Depth=1
+; RV64I-NEXT:    mv a2, s1
+; RV64I-NEXT:  .LBB207_3: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB207_1 Depth=1
+; RV64I-NEXT:    mv a0, s2
+; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    addi a3, zero, 3
+; RV64I-NEXT:    mv a4, zero
+; RV64I-NEXT:    call __atomic_compare_exchange_8
+; RV64I-NEXT:    ld a2, 8(sp)
+; RV64I-NEXT:    beqz a0, .LBB207_1
+; RV64I-NEXT:  # %bb.4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a2
+; RV64I-NEXT:    ld s3, 16(sp)
+; RV64I-NEXT:    ld s2, 24(sp)
+; RV64I-NEXT:    ld s1, 32(sp)
+; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    addi sp, sp, 48
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_min_i64_release:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    amomin.d.rl a0, a1, (a0)
+; RV64IA-NEXT:    ret
   %1 = atomicrmw min i64* %a, i64 %b release
   ret i64 %1
 }
@@ -8885,22 +15850,18 @@ define i64 @atomicrmw_min_i64_release(i64 *%a, i64 %b) nounwind {
 define i64 @atomicrmw_min_i64_acq_rel(i64 *%a, i64 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_min_i64_acq_rel:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    addi sp, sp, -48
-; RV32I-NEXT:    sw ra, 44(sp)
-; RV32I-NEXT:    sw s1, 40(sp)
-; RV32I-NEXT:    sw s2, 36(sp)
-; RV32I-NEXT:    sw s3, 32(sp)
-; RV32I-NEXT:    sw s4, 28(sp)
-; RV32I-NEXT:    sw s5, 24(sp)
-; RV32I-NEXT:    sw s6, 20(sp)
+; RV32I-NEXT:    addi sp, sp, -32
+; RV32I-NEXT:    sw ra, 28(sp)
+; RV32I-NEXT:    sw s1, 24(sp)
+; RV32I-NEXT:    sw s2, 20(sp)
+; RV32I-NEXT:    sw s3, 16(sp)
+; RV32I-NEXT:    sw s4, 12(sp)
 ; RV32I-NEXT:    mv s1, a2
 ; RV32I-NEXT:    mv s2, a1
 ; RV32I-NEXT:    mv s3, a0
 ; RV32I-NEXT:    lw a1, 4(a0)
 ; RV32I-NEXT:    lw a2, 0(a0)
-; RV32I-NEXT:    addi s4, sp, 8
-; RV32I-NEXT:    addi s5, zero, 4
-; RV32I-NEXT:    addi s6, zero, 2
+; RV32I-NEXT:    mv s4, sp
 ; RV32I-NEXT:  .LBB208_1: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
 ; RV32I-NEXT:    beq a1, s1, .LBB208_3
@@ -8913,7 +15874,7 @@ define i64 @atomicrmw_min_i64_acq_rel(i64 *%a, i64 %b) nounwind {
 ; RV32I-NEXT:  .LBB208_4: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB208_1 Depth=1
 ; RV32I-NEXT:    xori a0, a0, 1
-; RV32I-NEXT:    sw a2, 8(sp)
+; RV32I-NEXT:    sw a2, 0(sp)
 ; RV32I-NEXT:    bnez a0, .LBB208_6
 ; RV32I-NEXT:  # %bb.5: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB208_1 Depth=1
@@ -8927,45 +15888,39 @@ define i64 @atomicrmw_min_i64_acq_rel(i64 *%a, i64 %b) nounwind {
 ; RV32I-NEXT:    mv a3, s1
 ; RV32I-NEXT:  .LBB208_8: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB208_1 Depth=1
-; RV32I-NEXT:    sw a1, 12(sp)
+; RV32I-NEXT:    sw a1, 4(sp)
 ; RV32I-NEXT:    mv a0, s3
 ; RV32I-NEXT:    mv a1, s4
-; RV32I-NEXT:    mv a4, s5
-; RV32I-NEXT:    mv a5, s6
+; RV32I-NEXT:    addi a4, zero, 4
+; RV32I-NEXT:    addi a5, zero, 2
 ; RV32I-NEXT:    call __atomic_compare_exchange_8
-; RV32I-NEXT:    lw a1, 12(sp)
-; RV32I-NEXT:    lw a2, 8(sp)
+; RV32I-NEXT:    lw a1, 4(sp)
+; RV32I-NEXT:    lw a2, 0(sp)
 ; RV32I-NEXT:    beqz a0, .LBB208_1
 ; RV32I-NEXT:  # %bb.9: # %atomicrmw.end
 ; RV32I-NEXT:    mv a0, a2
-; RV32I-NEXT:    lw s6, 20(sp)
-; RV32I-NEXT:    lw s5, 24(sp)
-; RV32I-NEXT:    lw s4, 28(sp)
-; RV32I-NEXT:    lw s3, 32(sp)
-; RV32I-NEXT:    lw s2, 36(sp)
-; RV32I-NEXT:    lw s1, 40(sp)
-; RV32I-NEXT:    lw ra, 44(sp)
-; RV32I-NEXT:    addi sp, sp, 48
+; RV32I-NEXT:    lw s4, 12(sp)
+; RV32I-NEXT:    lw s3, 16(sp)
+; RV32I-NEXT:    lw s2, 20(sp)
+; RV32I-NEXT:    lw s1, 24(sp)
+; RV32I-NEXT:    lw ra, 28(sp)
+; RV32I-NEXT:    addi sp, sp, 32
 ; RV32I-NEXT:    ret
 ;
 ; RV32IA-LABEL: atomicrmw_min_i64_acq_rel:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    addi sp, sp, -48
-; RV32IA-NEXT:    sw ra, 44(sp)
-; RV32IA-NEXT:    sw s1, 40(sp)
-; RV32IA-NEXT:    sw s2, 36(sp)
-; RV32IA-NEXT:    sw s3, 32(sp)
-; RV32IA-NEXT:    sw s4, 28(sp)
-; RV32IA-NEXT:    sw s5, 24(sp)
-; RV32IA-NEXT:    sw s6, 20(sp)
+; RV32IA-NEXT:    addi sp, sp, -32
+; RV32IA-NEXT:    sw ra, 28(sp)
+; RV32IA-NEXT:    sw s1, 24(sp)
+; RV32IA-NEXT:    sw s2, 20(sp)
+; RV32IA-NEXT:    sw s3, 16(sp)
+; RV32IA-NEXT:    sw s4, 12(sp)
 ; RV32IA-NEXT:    mv s1, a2
 ; RV32IA-NEXT:    mv s2, a1
 ; RV32IA-NEXT:    mv s3, a0
 ; RV32IA-NEXT:    lw a1, 4(a0)
 ; RV32IA-NEXT:    lw a2, 0(a0)
-; RV32IA-NEXT:    addi s4, sp, 8
-; RV32IA-NEXT:    addi s5, zero, 4
-; RV32IA-NEXT:    addi s6, zero, 2
+; RV32IA-NEXT:    mv s4, sp
 ; RV32IA-NEXT:  .LBB208_1: # %atomicrmw.start
 ; RV32IA-NEXT:    # =>This Inner Loop Header: Depth=1
 ; RV32IA-NEXT:    beq a1, s1, .LBB208_3
@@ -8978,7 +15933,7 @@ define i64 @atomicrmw_min_i64_acq_rel(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:  .LBB208_4: # %atomicrmw.start
 ; RV32IA-NEXT:    # in Loop: Header=BB208_1 Depth=1
 ; RV32IA-NEXT:    xori a0, a0, 1
-; RV32IA-NEXT:    sw a2, 8(sp)
+; RV32IA-NEXT:    sw a2, 0(sp)
 ; RV32IA-NEXT:    bnez a0, .LBB208_6
 ; RV32IA-NEXT:  # %bb.5: # %atomicrmw.start
 ; RV32IA-NEXT:    # in Loop: Header=BB208_1 Depth=1
@@ -8992,26 +15947,65 @@ define i64 @atomicrmw_min_i64_acq_rel(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    mv a3, s1
 ; RV32IA-NEXT:  .LBB208_8: # %atomicrmw.start
 ; RV32IA-NEXT:    # in Loop: Header=BB208_1 Depth=1
-; RV32IA-NEXT:    sw a1, 12(sp)
+; RV32IA-NEXT:    sw a1, 4(sp)
 ; RV32IA-NEXT:    mv a0, s3
 ; RV32IA-NEXT:    mv a1, s4
-; RV32IA-NEXT:    mv a4, s5
-; RV32IA-NEXT:    mv a5, s6
+; RV32IA-NEXT:    addi a4, zero, 4
+; RV32IA-NEXT:    addi a5, zero, 2
 ; RV32IA-NEXT:    call __atomic_compare_exchange_8
-; RV32IA-NEXT:    lw a1, 12(sp)
-; RV32IA-NEXT:    lw a2, 8(sp)
+; RV32IA-NEXT:    lw a1, 4(sp)
+; RV32IA-NEXT:    lw a2, 0(sp)
 ; RV32IA-NEXT:    beqz a0, .LBB208_1
 ; RV32IA-NEXT:  # %bb.9: # %atomicrmw.end
 ; RV32IA-NEXT:    mv a0, a2
-; RV32IA-NEXT:    lw s6, 20(sp)
-; RV32IA-NEXT:    lw s5, 24(sp)
-; RV32IA-NEXT:    lw s4, 28(sp)
-; RV32IA-NEXT:    lw s3, 32(sp)
-; RV32IA-NEXT:    lw s2, 36(sp)
-; RV32IA-NEXT:    lw s1, 40(sp)
-; RV32IA-NEXT:    lw ra, 44(sp)
-; RV32IA-NEXT:    addi sp, sp, 48
+; RV32IA-NEXT:    lw s4, 12(sp)
+; RV32IA-NEXT:    lw s3, 16(sp)
+; RV32IA-NEXT:    lw s2, 20(sp)
+; RV32IA-NEXT:    lw s1, 24(sp)
+; RV32IA-NEXT:    lw ra, 28(sp)
+; RV32IA-NEXT:    addi sp, sp, 32
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_min_i64_acq_rel:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -48
+; RV64I-NEXT:    sd ra, 40(sp)
+; RV64I-NEXT:    sd s1, 32(sp)
+; RV64I-NEXT:    sd s2, 24(sp)
+; RV64I-NEXT:    sd s3, 16(sp)
+; RV64I-NEXT:    mv s1, a1
+; RV64I-NEXT:    mv s2, a0
+; RV64I-NEXT:    ld a2, 0(a0)
+; RV64I-NEXT:    addi s3, sp, 8
+; RV64I-NEXT:  .LBB208_1: # %atomicrmw.start
+; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV64I-NEXT:    sd a2, 8(sp)
+; RV64I-NEXT:    bge s1, a2, .LBB208_3
+; RV64I-NEXT:  # %bb.2: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB208_1 Depth=1
+; RV64I-NEXT:    mv a2, s1
+; RV64I-NEXT:  .LBB208_3: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB208_1 Depth=1
+; RV64I-NEXT:    mv a0, s2
+; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    addi a3, zero, 4
+; RV64I-NEXT:    addi a4, zero, 2
+; RV64I-NEXT:    call __atomic_compare_exchange_8
+; RV64I-NEXT:    ld a2, 8(sp)
+; RV64I-NEXT:    beqz a0, .LBB208_1
+; RV64I-NEXT:  # %bb.4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a2
+; RV64I-NEXT:    ld s3, 16(sp)
+; RV64I-NEXT:    ld s2, 24(sp)
+; RV64I-NEXT:    ld s1, 32(sp)
+; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    addi sp, sp, 48
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_min_i64_acq_rel:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    amomin.d.aqrl a0, a1, (a0)
+; RV64IA-NEXT:    ret
   %1 = atomicrmw min i64* %a, i64 %b acq_rel
   ret i64 %1
 }
@@ -9025,14 +16019,12 @@ define i64 @atomicrmw_min_i64_seq_cst(i64 *%a, i64 %b) nounwind {
 ; RV32I-NEXT:    sw s2, 20(sp)
 ; RV32I-NEXT:    sw s3, 16(sp)
 ; RV32I-NEXT:    sw s4, 12(sp)
-; RV32I-NEXT:    sw s5, 8(sp)
 ; RV32I-NEXT:    mv s1, a2
 ; RV32I-NEXT:    mv s2, a1
 ; RV32I-NEXT:    mv s3, a0
 ; RV32I-NEXT:    lw a1, 4(a0)
 ; RV32I-NEXT:    lw a2, 0(a0)
 ; RV32I-NEXT:    mv s4, sp
-; RV32I-NEXT:    addi s5, zero, 5
 ; RV32I-NEXT:  .LBB209_1: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
 ; RV32I-NEXT:    beq a1, s1, .LBB209_3
@@ -9062,15 +16054,14 @@ define i64 @atomicrmw_min_i64_seq_cst(i64 *%a, i64 %b) nounwind {
 ; RV32I-NEXT:    sw a1, 4(sp)
 ; RV32I-NEXT:    mv a0, s3
 ; RV32I-NEXT:    mv a1, s4
-; RV32I-NEXT:    mv a4, s5
-; RV32I-NEXT:    mv a5, s5
+; RV32I-NEXT:    addi a4, zero, 5
+; RV32I-NEXT:    addi a5, zero, 5
 ; RV32I-NEXT:    call __atomic_compare_exchange_8
 ; RV32I-NEXT:    lw a1, 4(sp)
 ; RV32I-NEXT:    lw a2, 0(sp)
 ; RV32I-NEXT:    beqz a0, .LBB209_1
 ; RV32I-NEXT:  # %bb.9: # %atomicrmw.end
 ; RV32I-NEXT:    mv a0, a2
-; RV32I-NEXT:    lw s5, 8(sp)
 ; RV32I-NEXT:    lw s4, 12(sp)
 ; RV32I-NEXT:    lw s3, 16(sp)
 ; RV32I-NEXT:    lw s2, 20(sp)
@@ -9087,14 +16078,12 @@ define i64 @atomicrmw_min_i64_seq_cst(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    sw s2, 20(sp)
 ; RV32IA-NEXT:    sw s3, 16(sp)
 ; RV32IA-NEXT:    sw s4, 12(sp)
-; RV32IA-NEXT:    sw s5, 8(sp)
 ; RV32IA-NEXT:    mv s1, a2
 ; RV32IA-NEXT:    mv s2, a1
 ; RV32IA-NEXT:    mv s3, a0
 ; RV32IA-NEXT:    lw a1, 4(a0)
 ; RV32IA-NEXT:    lw a2, 0(a0)
 ; RV32IA-NEXT:    mv s4, sp
-; RV32IA-NEXT:    addi s5, zero, 5
 ; RV32IA-NEXT:  .LBB209_1: # %atomicrmw.start
 ; RV32IA-NEXT:    # =>This Inner Loop Header: Depth=1
 ; RV32IA-NEXT:    beq a1, s1, .LBB209_3
@@ -9124,15 +16113,14 @@ define i64 @atomicrmw_min_i64_seq_cst(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    sw a1, 4(sp)
 ; RV32IA-NEXT:    mv a0, s3
 ; RV32IA-NEXT:    mv a1, s4
-; RV32IA-NEXT:    mv a4, s5
-; RV32IA-NEXT:    mv a5, s5
+; RV32IA-NEXT:    addi a4, zero, 5
+; RV32IA-NEXT:    addi a5, zero, 5
 ; RV32IA-NEXT:    call __atomic_compare_exchange_8
 ; RV32IA-NEXT:    lw a1, 4(sp)
 ; RV32IA-NEXT:    lw a2, 0(sp)
 ; RV32IA-NEXT:    beqz a0, .LBB209_1
 ; RV32IA-NEXT:  # %bb.9: # %atomicrmw.end
 ; RV32IA-NEXT:    mv a0, a2
-; RV32IA-NEXT:    lw s5, 8(sp)
 ; RV32IA-NEXT:    lw s4, 12(sp)
 ; RV32IA-NEXT:    lw s3, 16(sp)
 ; RV32IA-NEXT:    lw s2, 20(sp)
@@ -9140,6 +16128,47 @@ define i64 @atomicrmw_min_i64_seq_cst(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    lw ra, 28(sp)
 ; RV32IA-NEXT:    addi sp, sp, 32
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_min_i64_seq_cst:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -48
+; RV64I-NEXT:    sd ra, 40(sp)
+; RV64I-NEXT:    sd s1, 32(sp)
+; RV64I-NEXT:    sd s2, 24(sp)
+; RV64I-NEXT:    sd s3, 16(sp)
+; RV64I-NEXT:    mv s1, a1
+; RV64I-NEXT:    mv s2, a0
+; RV64I-NEXT:    ld a2, 0(a0)
+; RV64I-NEXT:    addi s3, sp, 8
+; RV64I-NEXT:  .LBB209_1: # %atomicrmw.start
+; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV64I-NEXT:    sd a2, 8(sp)
+; RV64I-NEXT:    bge s1, a2, .LBB209_3
+; RV64I-NEXT:  # %bb.2: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB209_1 Depth=1
+; RV64I-NEXT:    mv a2, s1
+; RV64I-NEXT:  .LBB209_3: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB209_1 Depth=1
+; RV64I-NEXT:    mv a0, s2
+; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    addi a3, zero, 5
+; RV64I-NEXT:    addi a4, zero, 5
+; RV64I-NEXT:    call __atomic_compare_exchange_8
+; RV64I-NEXT:    ld a2, 8(sp)
+; RV64I-NEXT:    beqz a0, .LBB209_1
+; RV64I-NEXT:  # %bb.4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a2
+; RV64I-NEXT:    ld s3, 16(sp)
+; RV64I-NEXT:    ld s2, 24(sp)
+; RV64I-NEXT:    ld s1, 32(sp)
+; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    addi sp, sp, 48
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_min_i64_seq_cst:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    amomin.d.aqrl a0, a1, (a0)
+; RV64IA-NEXT:    ret
   %1 = atomicrmw min i64* %a, i64 %b seq_cst
   ret i64 %1
 }
@@ -9260,6 +16289,47 @@ define i64 @atomicrmw_umax_i64_monotonic(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    lw ra, 28(sp)
 ; RV32IA-NEXT:    addi sp, sp, 32
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_umax_i64_monotonic:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -48
+; RV64I-NEXT:    sd ra, 40(sp)
+; RV64I-NEXT:    sd s1, 32(sp)
+; RV64I-NEXT:    sd s2, 24(sp)
+; RV64I-NEXT:    sd s3, 16(sp)
+; RV64I-NEXT:    mv s1, a1
+; RV64I-NEXT:    mv s2, a0
+; RV64I-NEXT:    ld a2, 0(a0)
+; RV64I-NEXT:    addi s3, sp, 8
+; RV64I-NEXT:  .LBB210_1: # %atomicrmw.start
+; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV64I-NEXT:    sd a2, 8(sp)
+; RV64I-NEXT:    bltu s1, a2, .LBB210_3
+; RV64I-NEXT:  # %bb.2: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB210_1 Depth=1
+; RV64I-NEXT:    mv a2, s1
+; RV64I-NEXT:  .LBB210_3: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB210_1 Depth=1
+; RV64I-NEXT:    mv a0, s2
+; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    mv a3, zero
+; RV64I-NEXT:    mv a4, zero
+; RV64I-NEXT:    call __atomic_compare_exchange_8
+; RV64I-NEXT:    ld a2, 8(sp)
+; RV64I-NEXT:    beqz a0, .LBB210_1
+; RV64I-NEXT:  # %bb.4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a2
+; RV64I-NEXT:    ld s3, 16(sp)
+; RV64I-NEXT:    ld s2, 24(sp)
+; RV64I-NEXT:    ld s1, 32(sp)
+; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    addi sp, sp, 48
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_umax_i64_monotonic:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    amomaxu.d a0, a1, (a0)
+; RV64IA-NEXT:    ret
   %1 = atomicrmw umax i64* %a, i64 %b monotonic
   ret i64 %1
 }
@@ -9273,14 +16343,12 @@ define i64 @atomicrmw_umax_i64_acquire(i64 *%a, i64 %b) nounwind {
 ; RV32I-NEXT:    sw s2, 20(sp)
 ; RV32I-NEXT:    sw s3, 16(sp)
 ; RV32I-NEXT:    sw s4, 12(sp)
-; RV32I-NEXT:    sw s5, 8(sp)
 ; RV32I-NEXT:    mv s1, a2
 ; RV32I-NEXT:    mv s2, a1
 ; RV32I-NEXT:    mv s3, a0
 ; RV32I-NEXT:    lw a1, 4(a0)
 ; RV32I-NEXT:    lw a2, 0(a0)
 ; RV32I-NEXT:    mv s4, sp
-; RV32I-NEXT:    addi s5, zero, 2
 ; RV32I-NEXT:  .LBB211_1: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
 ; RV32I-NEXT:    beq a1, s1, .LBB211_3
@@ -9309,15 +16377,14 @@ define i64 @atomicrmw_umax_i64_acquire(i64 *%a, i64 %b) nounwind {
 ; RV32I-NEXT:    sw a1, 4(sp)
 ; RV32I-NEXT:    mv a0, s3
 ; RV32I-NEXT:    mv a1, s4
-; RV32I-NEXT:    mv a4, s5
-; RV32I-NEXT:    mv a5, s5
+; RV32I-NEXT:    addi a4, zero, 2
+; RV32I-NEXT:    addi a5, zero, 2
 ; RV32I-NEXT:    call __atomic_compare_exchange_8
 ; RV32I-NEXT:    lw a1, 4(sp)
 ; RV32I-NEXT:    lw a2, 0(sp)
 ; RV32I-NEXT:    beqz a0, .LBB211_1
 ; RV32I-NEXT:  # %bb.8: # %atomicrmw.end
 ; RV32I-NEXT:    mv a0, a2
-; RV32I-NEXT:    lw s5, 8(sp)
 ; RV32I-NEXT:    lw s4, 12(sp)
 ; RV32I-NEXT:    lw s3, 16(sp)
 ; RV32I-NEXT:    lw s2, 20(sp)
@@ -9334,14 +16401,12 @@ define i64 @atomicrmw_umax_i64_acquire(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    sw s2, 20(sp)
 ; RV32IA-NEXT:    sw s3, 16(sp)
 ; RV32IA-NEXT:    sw s4, 12(sp)
-; RV32IA-NEXT:    sw s5, 8(sp)
 ; RV32IA-NEXT:    mv s1, a2
 ; RV32IA-NEXT:    mv s2, a1
 ; RV32IA-NEXT:    mv s3, a0
 ; RV32IA-NEXT:    lw a1, 4(a0)
 ; RV32IA-NEXT:    lw a2, 0(a0)
 ; RV32IA-NEXT:    mv s4, sp
-; RV32IA-NEXT:    addi s5, zero, 2
 ; RV32IA-NEXT:  .LBB211_1: # %atomicrmw.start
 ; RV32IA-NEXT:    # =>This Inner Loop Header: Depth=1
 ; RV32IA-NEXT:    beq a1, s1, .LBB211_3
@@ -9370,15 +16435,14 @@ define i64 @atomicrmw_umax_i64_acquire(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    sw a1, 4(sp)
 ; RV32IA-NEXT:    mv a0, s3
 ; RV32IA-NEXT:    mv a1, s4
-; RV32IA-NEXT:    mv a4, s5
-; RV32IA-NEXT:    mv a5, s5
+; RV32IA-NEXT:    addi a4, zero, 2
+; RV32IA-NEXT:    addi a5, zero, 2
 ; RV32IA-NEXT:    call __atomic_compare_exchange_8
 ; RV32IA-NEXT:    lw a1, 4(sp)
 ; RV32IA-NEXT:    lw a2, 0(sp)
 ; RV32IA-NEXT:    beqz a0, .LBB211_1
 ; RV32IA-NEXT:  # %bb.8: # %atomicrmw.end
 ; RV32IA-NEXT:    mv a0, a2
-; RV32IA-NEXT:    lw s5, 8(sp)
 ; RV32IA-NEXT:    lw s4, 12(sp)
 ; RV32IA-NEXT:    lw s3, 16(sp)
 ; RV32IA-NEXT:    lw s2, 20(sp)
@@ -9386,6 +16450,47 @@ define i64 @atomicrmw_umax_i64_acquire(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    lw ra, 28(sp)
 ; RV32IA-NEXT:    addi sp, sp, 32
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_umax_i64_acquire:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -48
+; RV64I-NEXT:    sd ra, 40(sp)
+; RV64I-NEXT:    sd s1, 32(sp)
+; RV64I-NEXT:    sd s2, 24(sp)
+; RV64I-NEXT:    sd s3, 16(sp)
+; RV64I-NEXT:    mv s1, a1
+; RV64I-NEXT:    mv s2, a0
+; RV64I-NEXT:    ld a2, 0(a0)
+; RV64I-NEXT:    addi s3, sp, 8
+; RV64I-NEXT:  .LBB211_1: # %atomicrmw.start
+; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV64I-NEXT:    sd a2, 8(sp)
+; RV64I-NEXT:    bltu s1, a2, .LBB211_3
+; RV64I-NEXT:  # %bb.2: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB211_1 Depth=1
+; RV64I-NEXT:    mv a2, s1
+; RV64I-NEXT:  .LBB211_3: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB211_1 Depth=1
+; RV64I-NEXT:    mv a0, s2
+; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    addi a3, zero, 2
+; RV64I-NEXT:    addi a4, zero, 2
+; RV64I-NEXT:    call __atomic_compare_exchange_8
+; RV64I-NEXT:    ld a2, 8(sp)
+; RV64I-NEXT:    beqz a0, .LBB211_1
+; RV64I-NEXT:  # %bb.4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a2
+; RV64I-NEXT:    ld s3, 16(sp)
+; RV64I-NEXT:    ld s2, 24(sp)
+; RV64I-NEXT:    ld s1, 32(sp)
+; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    addi sp, sp, 48
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_umax_i64_acquire:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    amomaxu.d.aq a0, a1, (a0)
+; RV64IA-NEXT:    ret
   %1 = atomicrmw umax i64* %a, i64 %b acquire
   ret i64 %1
 }
@@ -9399,14 +16504,12 @@ define i64 @atomicrmw_umax_i64_release(i64 *%a, i64 %b) nounwind {
 ; RV32I-NEXT:    sw s2, 20(sp)
 ; RV32I-NEXT:    sw s3, 16(sp)
 ; RV32I-NEXT:    sw s4, 12(sp)
-; RV32I-NEXT:    sw s5, 8(sp)
 ; RV32I-NEXT:    mv s1, a2
 ; RV32I-NEXT:    mv s2, a1
 ; RV32I-NEXT:    mv s3, a0
 ; RV32I-NEXT:    lw a1, 4(a0)
 ; RV32I-NEXT:    lw a2, 0(a0)
 ; RV32I-NEXT:    mv s4, sp
-; RV32I-NEXT:    addi s5, zero, 3
 ; RV32I-NEXT:  .LBB212_1: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
 ; RV32I-NEXT:    beq a1, s1, .LBB212_3
@@ -9435,7 +16538,7 @@ define i64 @atomicrmw_umax_i64_release(i64 *%a, i64 %b) nounwind {
 ; RV32I-NEXT:    sw a1, 4(sp)
 ; RV32I-NEXT:    mv a0, s3
 ; RV32I-NEXT:    mv a1, s4
-; RV32I-NEXT:    mv a4, s5
+; RV32I-NEXT:    addi a4, zero, 3
 ; RV32I-NEXT:    mv a5, zero
 ; RV32I-NEXT:    call __atomic_compare_exchange_8
 ; RV32I-NEXT:    lw a1, 4(sp)
@@ -9443,7 +16546,6 @@ define i64 @atomicrmw_umax_i64_release(i64 *%a, i64 %b) nounwind {
 ; RV32I-NEXT:    beqz a0, .LBB212_1
 ; RV32I-NEXT:  # %bb.8: # %atomicrmw.end
 ; RV32I-NEXT:    mv a0, a2
-; RV32I-NEXT:    lw s5, 8(sp)
 ; RV32I-NEXT:    lw s4, 12(sp)
 ; RV32I-NEXT:    lw s3, 16(sp)
 ; RV32I-NEXT:    lw s2, 20(sp)
@@ -9460,14 +16562,12 @@ define i64 @atomicrmw_umax_i64_release(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    sw s2, 20(sp)
 ; RV32IA-NEXT:    sw s3, 16(sp)
 ; RV32IA-NEXT:    sw s4, 12(sp)
-; RV32IA-NEXT:    sw s5, 8(sp)
 ; RV32IA-NEXT:    mv s1, a2
 ; RV32IA-NEXT:    mv s2, a1
 ; RV32IA-NEXT:    mv s3, a0
 ; RV32IA-NEXT:    lw a1, 4(a0)
 ; RV32IA-NEXT:    lw a2, 0(a0)
 ; RV32IA-NEXT:    mv s4, sp
-; RV32IA-NEXT:    addi s5, zero, 3
 ; RV32IA-NEXT:  .LBB212_1: # %atomicrmw.start
 ; RV32IA-NEXT:    # =>This Inner Loop Header: Depth=1
 ; RV32IA-NEXT:    beq a1, s1, .LBB212_3
@@ -9496,7 +16596,7 @@ define i64 @atomicrmw_umax_i64_release(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    sw a1, 4(sp)
 ; RV32IA-NEXT:    mv a0, s3
 ; RV32IA-NEXT:    mv a1, s4
-; RV32IA-NEXT:    mv a4, s5
+; RV32IA-NEXT:    addi a4, zero, 3
 ; RV32IA-NEXT:    mv a5, zero
 ; RV32IA-NEXT:    call __atomic_compare_exchange_8
 ; RV32IA-NEXT:    lw a1, 4(sp)
@@ -9504,7 +16604,6 @@ define i64 @atomicrmw_umax_i64_release(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    beqz a0, .LBB212_1
 ; RV32IA-NEXT:  # %bb.8: # %atomicrmw.end
 ; RV32IA-NEXT:    mv a0, a2
-; RV32IA-NEXT:    lw s5, 8(sp)
 ; RV32IA-NEXT:    lw s4, 12(sp)
 ; RV32IA-NEXT:    lw s3, 16(sp)
 ; RV32IA-NEXT:    lw s2, 20(sp)
@@ -9512,6 +16611,47 @@ define i64 @atomicrmw_umax_i64_release(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    lw ra, 28(sp)
 ; RV32IA-NEXT:    addi sp, sp, 32
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_umax_i64_release:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -48
+; RV64I-NEXT:    sd ra, 40(sp)
+; RV64I-NEXT:    sd s1, 32(sp)
+; RV64I-NEXT:    sd s2, 24(sp)
+; RV64I-NEXT:    sd s3, 16(sp)
+; RV64I-NEXT:    mv s1, a1
+; RV64I-NEXT:    mv s2, a0
+; RV64I-NEXT:    ld a2, 0(a0)
+; RV64I-NEXT:    addi s3, sp, 8
+; RV64I-NEXT:  .LBB212_1: # %atomicrmw.start
+; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV64I-NEXT:    sd a2, 8(sp)
+; RV64I-NEXT:    bltu s1, a2, .LBB212_3
+; RV64I-NEXT:  # %bb.2: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB212_1 Depth=1
+; RV64I-NEXT:    mv a2, s1
+; RV64I-NEXT:  .LBB212_3: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB212_1 Depth=1
+; RV64I-NEXT:    mv a0, s2
+; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    addi a3, zero, 3
+; RV64I-NEXT:    mv a4, zero
+; RV64I-NEXT:    call __atomic_compare_exchange_8
+; RV64I-NEXT:    ld a2, 8(sp)
+; RV64I-NEXT:    beqz a0, .LBB212_1
+; RV64I-NEXT:  # %bb.4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a2
+; RV64I-NEXT:    ld s3, 16(sp)
+; RV64I-NEXT:    ld s2, 24(sp)
+; RV64I-NEXT:    ld s1, 32(sp)
+; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    addi sp, sp, 48
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_umax_i64_release:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    amomaxu.d.rl a0, a1, (a0)
+; RV64IA-NEXT:    ret
   %1 = atomicrmw umax i64* %a, i64 %b release
   ret i64 %1
 }
@@ -9519,34 +16659,30 @@ define i64 @atomicrmw_umax_i64_release(i64 *%a, i64 %b) nounwind {
 define i64 @atomicrmw_umax_i64_acq_rel(i64 *%a, i64 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_umax_i64_acq_rel:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    addi sp, sp, -48
-; RV32I-NEXT:    sw ra, 44(sp)
-; RV32I-NEXT:    sw s1, 40(sp)
-; RV32I-NEXT:    sw s2, 36(sp)
-; RV32I-NEXT:    sw s3, 32(sp)
-; RV32I-NEXT:    sw s4, 28(sp)
-; RV32I-NEXT:    sw s5, 24(sp)
-; RV32I-NEXT:    sw s6, 20(sp)
+; RV32I-NEXT:    addi sp, sp, -32
+; RV32I-NEXT:    sw ra, 28(sp)
+; RV32I-NEXT:    sw s1, 24(sp)
+; RV32I-NEXT:    sw s2, 20(sp)
+; RV32I-NEXT:    sw s3, 16(sp)
+; RV32I-NEXT:    sw s4, 12(sp)
 ; RV32I-NEXT:    mv s1, a2
 ; RV32I-NEXT:    mv s2, a1
 ; RV32I-NEXT:    mv s3, a0
 ; RV32I-NEXT:    lw a1, 4(a0)
 ; RV32I-NEXT:    lw a2, 0(a0)
-; RV32I-NEXT:    addi s4, sp, 8
-; RV32I-NEXT:    addi s5, zero, 4
-; RV32I-NEXT:    addi s6, zero, 2
+; RV32I-NEXT:    mv s4, sp
 ; RV32I-NEXT:  .LBB213_1: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
 ; RV32I-NEXT:    beq a1, s1, .LBB213_3
 ; RV32I-NEXT:  # %bb.2: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB213_1 Depth=1
 ; RV32I-NEXT:    sltu a0, s1, a1
-; RV32I-NEXT:    sw a2, 8(sp)
+; RV32I-NEXT:    sw a2, 0(sp)
 ; RV32I-NEXT:    beqz a0, .LBB213_4
 ; RV32I-NEXT:    j .LBB213_5
 ; RV32I-NEXT:  .LBB213_3: # in Loop: Header=BB213_1 Depth=1
 ; RV32I-NEXT:    sltu a0, s2, a2
-; RV32I-NEXT:    sw a2, 8(sp)
+; RV32I-NEXT:    sw a2, 0(sp)
 ; RV32I-NEXT:    bnez a0, .LBB213_5
 ; RV32I-NEXT:  .LBB213_4: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB213_1 Depth=1
@@ -9560,57 +16696,51 @@ define i64 @atomicrmw_umax_i64_acq_rel(i64 *%a, i64 %b) nounwind {
 ; RV32I-NEXT:    mv a3, s1
 ; RV32I-NEXT:  .LBB213_7: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB213_1 Depth=1
-; RV32I-NEXT:    sw a1, 12(sp)
+; RV32I-NEXT:    sw a1, 4(sp)
 ; RV32I-NEXT:    mv a0, s3
 ; RV32I-NEXT:    mv a1, s4
-; RV32I-NEXT:    mv a4, s5
-; RV32I-NEXT:    mv a5, s6
+; RV32I-NEXT:    addi a4, zero, 4
+; RV32I-NEXT:    addi a5, zero, 2
 ; RV32I-NEXT:    call __atomic_compare_exchange_8
-; RV32I-NEXT:    lw a1, 12(sp)
-; RV32I-NEXT:    lw a2, 8(sp)
+; RV32I-NEXT:    lw a1, 4(sp)
+; RV32I-NEXT:    lw a2, 0(sp)
 ; RV32I-NEXT:    beqz a0, .LBB213_1
 ; RV32I-NEXT:  # %bb.8: # %atomicrmw.end
 ; RV32I-NEXT:    mv a0, a2
-; RV32I-NEXT:    lw s6, 20(sp)
-; RV32I-NEXT:    lw s5, 24(sp)
-; RV32I-NEXT:    lw s4, 28(sp)
-; RV32I-NEXT:    lw s3, 32(sp)
-; RV32I-NEXT:    lw s2, 36(sp)
-; RV32I-NEXT:    lw s1, 40(sp)
-; RV32I-NEXT:    lw ra, 44(sp)
-; RV32I-NEXT:    addi sp, sp, 48
+; RV32I-NEXT:    lw s4, 12(sp)
+; RV32I-NEXT:    lw s3, 16(sp)
+; RV32I-NEXT:    lw s2, 20(sp)
+; RV32I-NEXT:    lw s1, 24(sp)
+; RV32I-NEXT:    lw ra, 28(sp)
+; RV32I-NEXT:    addi sp, sp, 32
 ; RV32I-NEXT:    ret
 ;
 ; RV32IA-LABEL: atomicrmw_umax_i64_acq_rel:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    addi sp, sp, -48
-; RV32IA-NEXT:    sw ra, 44(sp)
-; RV32IA-NEXT:    sw s1, 40(sp)
-; RV32IA-NEXT:    sw s2, 36(sp)
-; RV32IA-NEXT:    sw s3, 32(sp)
-; RV32IA-NEXT:    sw s4, 28(sp)
-; RV32IA-NEXT:    sw s5, 24(sp)
-; RV32IA-NEXT:    sw s6, 20(sp)
+; RV32IA-NEXT:    addi sp, sp, -32
+; RV32IA-NEXT:    sw ra, 28(sp)
+; RV32IA-NEXT:    sw s1, 24(sp)
+; RV32IA-NEXT:    sw s2, 20(sp)
+; RV32IA-NEXT:    sw s3, 16(sp)
+; RV32IA-NEXT:    sw s4, 12(sp)
 ; RV32IA-NEXT:    mv s1, a2
 ; RV32IA-NEXT:    mv s2, a1
 ; RV32IA-NEXT:    mv s3, a0
 ; RV32IA-NEXT:    lw a1, 4(a0)
 ; RV32IA-NEXT:    lw a2, 0(a0)
-; RV32IA-NEXT:    addi s4, sp, 8
-; RV32IA-NEXT:    addi s5, zero, 4
-; RV32IA-NEXT:    addi s6, zero, 2
+; RV32IA-NEXT:    mv s4, sp
 ; RV32IA-NEXT:  .LBB213_1: # %atomicrmw.start
 ; RV32IA-NEXT:    # =>This Inner Loop Header: Depth=1
 ; RV32IA-NEXT:    beq a1, s1, .LBB213_3
 ; RV32IA-NEXT:  # %bb.2: # %atomicrmw.start
 ; RV32IA-NEXT:    # in Loop: Header=BB213_1 Depth=1
 ; RV32IA-NEXT:    sltu a0, s1, a1
-; RV32IA-NEXT:    sw a2, 8(sp)
+; RV32IA-NEXT:    sw a2, 0(sp)
 ; RV32IA-NEXT:    beqz a0, .LBB213_4
 ; RV32IA-NEXT:    j .LBB213_5
 ; RV32IA-NEXT:  .LBB213_3: # in Loop: Header=BB213_1 Depth=1
 ; RV32IA-NEXT:    sltu a0, s2, a2
-; RV32IA-NEXT:    sw a2, 8(sp)
+; RV32IA-NEXT:    sw a2, 0(sp)
 ; RV32IA-NEXT:    bnez a0, .LBB213_5
 ; RV32IA-NEXT:  .LBB213_4: # %atomicrmw.start
 ; RV32IA-NEXT:    # in Loop: Header=BB213_1 Depth=1
@@ -9624,26 +16754,65 @@ define i64 @atomicrmw_umax_i64_acq_rel(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    mv a3, s1
 ; RV32IA-NEXT:  .LBB213_7: # %atomicrmw.start
 ; RV32IA-NEXT:    # in Loop: Header=BB213_1 Depth=1
-; RV32IA-NEXT:    sw a1, 12(sp)
+; RV32IA-NEXT:    sw a1, 4(sp)
 ; RV32IA-NEXT:    mv a0, s3
 ; RV32IA-NEXT:    mv a1, s4
-; RV32IA-NEXT:    mv a4, s5
-; RV32IA-NEXT:    mv a5, s6
+; RV32IA-NEXT:    addi a4, zero, 4
+; RV32IA-NEXT:    addi a5, zero, 2
 ; RV32IA-NEXT:    call __atomic_compare_exchange_8
-; RV32IA-NEXT:    lw a1, 12(sp)
-; RV32IA-NEXT:    lw a2, 8(sp)
+; RV32IA-NEXT:    lw a1, 4(sp)
+; RV32IA-NEXT:    lw a2, 0(sp)
 ; RV32IA-NEXT:    beqz a0, .LBB213_1
 ; RV32IA-NEXT:  # %bb.8: # %atomicrmw.end
 ; RV32IA-NEXT:    mv a0, a2
-; RV32IA-NEXT:    lw s6, 20(sp)
-; RV32IA-NEXT:    lw s5, 24(sp)
-; RV32IA-NEXT:    lw s4, 28(sp)
-; RV32IA-NEXT:    lw s3, 32(sp)
-; RV32IA-NEXT:    lw s2, 36(sp)
-; RV32IA-NEXT:    lw s1, 40(sp)
-; RV32IA-NEXT:    lw ra, 44(sp)
-; RV32IA-NEXT:    addi sp, sp, 48
+; RV32IA-NEXT:    lw s4, 12(sp)
+; RV32IA-NEXT:    lw s3, 16(sp)
+; RV32IA-NEXT:    lw s2, 20(sp)
+; RV32IA-NEXT:    lw s1, 24(sp)
+; RV32IA-NEXT:    lw ra, 28(sp)
+; RV32IA-NEXT:    addi sp, sp, 32
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_umax_i64_acq_rel:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -48
+; RV64I-NEXT:    sd ra, 40(sp)
+; RV64I-NEXT:    sd s1, 32(sp)
+; RV64I-NEXT:    sd s2, 24(sp)
+; RV64I-NEXT:    sd s3, 16(sp)
+; RV64I-NEXT:    mv s1, a1
+; RV64I-NEXT:    mv s2, a0
+; RV64I-NEXT:    ld a2, 0(a0)
+; RV64I-NEXT:    addi s3, sp, 8
+; RV64I-NEXT:  .LBB213_1: # %atomicrmw.start
+; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV64I-NEXT:    sd a2, 8(sp)
+; RV64I-NEXT:    bltu s1, a2, .LBB213_3
+; RV64I-NEXT:  # %bb.2: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB213_1 Depth=1
+; RV64I-NEXT:    mv a2, s1
+; RV64I-NEXT:  .LBB213_3: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB213_1 Depth=1
+; RV64I-NEXT:    mv a0, s2
+; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    addi a3, zero, 4
+; RV64I-NEXT:    addi a4, zero, 2
+; RV64I-NEXT:    call __atomic_compare_exchange_8
+; RV64I-NEXT:    ld a2, 8(sp)
+; RV64I-NEXT:    beqz a0, .LBB213_1
+; RV64I-NEXT:  # %bb.4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a2
+; RV64I-NEXT:    ld s3, 16(sp)
+; RV64I-NEXT:    ld s2, 24(sp)
+; RV64I-NEXT:    ld s1, 32(sp)
+; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    addi sp, sp, 48
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_umax_i64_acq_rel:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    amomaxu.d.aqrl a0, a1, (a0)
+; RV64IA-NEXT:    ret
   %1 = atomicrmw umax i64* %a, i64 %b acq_rel
   ret i64 %1
 }
@@ -9657,14 +16826,12 @@ define i64 @atomicrmw_umax_i64_seq_cst(i64 *%a, i64 %b) nounwind {
 ; RV32I-NEXT:    sw s2, 20(sp)
 ; RV32I-NEXT:    sw s3, 16(sp)
 ; RV32I-NEXT:    sw s4, 12(sp)
-; RV32I-NEXT:    sw s5, 8(sp)
 ; RV32I-NEXT:    mv s1, a2
 ; RV32I-NEXT:    mv s2, a1
 ; RV32I-NEXT:    mv s3, a0
 ; RV32I-NEXT:    lw a1, 4(a0)
 ; RV32I-NEXT:    lw a2, 0(a0)
 ; RV32I-NEXT:    mv s4, sp
-; RV32I-NEXT:    addi s5, zero, 5
 ; RV32I-NEXT:  .LBB214_1: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
 ; RV32I-NEXT:    beq a1, s1, .LBB214_3
@@ -9693,15 +16860,14 @@ define i64 @atomicrmw_umax_i64_seq_cst(i64 *%a, i64 %b) nounwind {
 ; RV32I-NEXT:    sw a1, 4(sp)
 ; RV32I-NEXT:    mv a0, s3
 ; RV32I-NEXT:    mv a1, s4
-; RV32I-NEXT:    mv a4, s5
-; RV32I-NEXT:    mv a5, s5
+; RV32I-NEXT:    addi a4, zero, 5
+; RV32I-NEXT:    addi a5, zero, 5
 ; RV32I-NEXT:    call __atomic_compare_exchange_8
 ; RV32I-NEXT:    lw a1, 4(sp)
 ; RV32I-NEXT:    lw a2, 0(sp)
 ; RV32I-NEXT:    beqz a0, .LBB214_1
 ; RV32I-NEXT:  # %bb.8: # %atomicrmw.end
 ; RV32I-NEXT:    mv a0, a2
-; RV32I-NEXT:    lw s5, 8(sp)
 ; RV32I-NEXT:    lw s4, 12(sp)
 ; RV32I-NEXT:    lw s3, 16(sp)
 ; RV32I-NEXT:    lw s2, 20(sp)
@@ -9718,14 +16884,12 @@ define i64 @atomicrmw_umax_i64_seq_cst(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    sw s2, 20(sp)
 ; RV32IA-NEXT:    sw s3, 16(sp)
 ; RV32IA-NEXT:    sw s4, 12(sp)
-; RV32IA-NEXT:    sw s5, 8(sp)
 ; RV32IA-NEXT:    mv s1, a2
 ; RV32IA-NEXT:    mv s2, a1
 ; RV32IA-NEXT:    mv s3, a0
 ; RV32IA-NEXT:    lw a1, 4(a0)
 ; RV32IA-NEXT:    lw a2, 0(a0)
 ; RV32IA-NEXT:    mv s4, sp
-; RV32IA-NEXT:    addi s5, zero, 5
 ; RV32IA-NEXT:  .LBB214_1: # %atomicrmw.start
 ; RV32IA-NEXT:    # =>This Inner Loop Header: Depth=1
 ; RV32IA-NEXT:    beq a1, s1, .LBB214_3
@@ -9754,15 +16918,14 @@ define i64 @atomicrmw_umax_i64_seq_cst(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    sw a1, 4(sp)
 ; RV32IA-NEXT:    mv a0, s3
 ; RV32IA-NEXT:    mv a1, s4
-; RV32IA-NEXT:    mv a4, s5
-; RV32IA-NEXT:    mv a5, s5
+; RV32IA-NEXT:    addi a4, zero, 5
+; RV32IA-NEXT:    addi a5, zero, 5
 ; RV32IA-NEXT:    call __atomic_compare_exchange_8
 ; RV32IA-NEXT:    lw a1, 4(sp)
 ; RV32IA-NEXT:    lw a2, 0(sp)
 ; RV32IA-NEXT:    beqz a0, .LBB214_1
 ; RV32IA-NEXT:  # %bb.8: # %atomicrmw.end
 ; RV32IA-NEXT:    mv a0, a2
-; RV32IA-NEXT:    lw s5, 8(sp)
 ; RV32IA-NEXT:    lw s4, 12(sp)
 ; RV32IA-NEXT:    lw s3, 16(sp)
 ; RV32IA-NEXT:    lw s2, 20(sp)
@@ -9770,6 +16933,47 @@ define i64 @atomicrmw_umax_i64_seq_cst(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    lw ra, 28(sp)
 ; RV32IA-NEXT:    addi sp, sp, 32
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_umax_i64_seq_cst:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -48
+; RV64I-NEXT:    sd ra, 40(sp)
+; RV64I-NEXT:    sd s1, 32(sp)
+; RV64I-NEXT:    sd s2, 24(sp)
+; RV64I-NEXT:    sd s3, 16(sp)
+; RV64I-NEXT:    mv s1, a1
+; RV64I-NEXT:    mv s2, a0
+; RV64I-NEXT:    ld a2, 0(a0)
+; RV64I-NEXT:    addi s3, sp, 8
+; RV64I-NEXT:  .LBB214_1: # %atomicrmw.start
+; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV64I-NEXT:    sd a2, 8(sp)
+; RV64I-NEXT:    bltu s1, a2, .LBB214_3
+; RV64I-NEXT:  # %bb.2: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB214_1 Depth=1
+; RV64I-NEXT:    mv a2, s1
+; RV64I-NEXT:  .LBB214_3: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB214_1 Depth=1
+; RV64I-NEXT:    mv a0, s2
+; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    addi a3, zero, 5
+; RV64I-NEXT:    addi a4, zero, 5
+; RV64I-NEXT:    call __atomic_compare_exchange_8
+; RV64I-NEXT:    ld a2, 8(sp)
+; RV64I-NEXT:    beqz a0, .LBB214_1
+; RV64I-NEXT:  # %bb.4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a2
+; RV64I-NEXT:    ld s3, 16(sp)
+; RV64I-NEXT:    ld s2, 24(sp)
+; RV64I-NEXT:    ld s1, 32(sp)
+; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    addi sp, sp, 48
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_umax_i64_seq_cst:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    amomaxu.d.aqrl a0, a1, (a0)
+; RV64IA-NEXT:    ret
   %1 = atomicrmw umax i64* %a, i64 %b seq_cst
   ret i64 %1
 }
@@ -9892,6 +17096,47 @@ define i64 @atomicrmw_umin_i64_monotonic(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    lw ra, 28(sp)
 ; RV32IA-NEXT:    addi sp, sp, 32
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_umin_i64_monotonic:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -48
+; RV64I-NEXT:    sd ra, 40(sp)
+; RV64I-NEXT:    sd s1, 32(sp)
+; RV64I-NEXT:    sd s2, 24(sp)
+; RV64I-NEXT:    sd s3, 16(sp)
+; RV64I-NEXT:    mv s1, a1
+; RV64I-NEXT:    mv s2, a0
+; RV64I-NEXT:    ld a2, 0(a0)
+; RV64I-NEXT:    addi s3, sp, 8
+; RV64I-NEXT:  .LBB215_1: # %atomicrmw.start
+; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV64I-NEXT:    sd a2, 8(sp)
+; RV64I-NEXT:    bgeu s1, a2, .LBB215_3
+; RV64I-NEXT:  # %bb.2: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB215_1 Depth=1
+; RV64I-NEXT:    mv a2, s1
+; RV64I-NEXT:  .LBB215_3: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB215_1 Depth=1
+; RV64I-NEXT:    mv a0, s2
+; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    mv a3, zero
+; RV64I-NEXT:    mv a4, zero
+; RV64I-NEXT:    call __atomic_compare_exchange_8
+; RV64I-NEXT:    ld a2, 8(sp)
+; RV64I-NEXT:    beqz a0, .LBB215_1
+; RV64I-NEXT:  # %bb.4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a2
+; RV64I-NEXT:    ld s3, 16(sp)
+; RV64I-NEXT:    ld s2, 24(sp)
+; RV64I-NEXT:    ld s1, 32(sp)
+; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    addi sp, sp, 48
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_umin_i64_monotonic:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    amominu.d a0, a1, (a0)
+; RV64IA-NEXT:    ret
   %1 = atomicrmw umin i64* %a, i64 %b monotonic
   ret i64 %1
 }
@@ -9905,14 +17150,12 @@ define i64 @atomicrmw_umin_i64_acquire(i64 *%a, i64 %b) nounwind {
 ; RV32I-NEXT:    sw s2, 20(sp)
 ; RV32I-NEXT:    sw s3, 16(sp)
 ; RV32I-NEXT:    sw s4, 12(sp)
-; RV32I-NEXT:    sw s5, 8(sp)
 ; RV32I-NEXT:    mv s1, a2
 ; RV32I-NEXT:    mv s2, a1
 ; RV32I-NEXT:    mv s3, a0
 ; RV32I-NEXT:    lw a1, 4(a0)
 ; RV32I-NEXT:    lw a2, 0(a0)
 ; RV32I-NEXT:    mv s4, sp
-; RV32I-NEXT:    addi s5, zero, 2
 ; RV32I-NEXT:  .LBB216_1: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
 ; RV32I-NEXT:    beq a1, s1, .LBB216_3
@@ -9942,15 +17185,14 @@ define i64 @atomicrmw_umin_i64_acquire(i64 *%a, i64 %b) nounwind {
 ; RV32I-NEXT:    sw a1, 4(sp)
 ; RV32I-NEXT:    mv a0, s3
 ; RV32I-NEXT:    mv a1, s4
-; RV32I-NEXT:    mv a4, s5
-; RV32I-NEXT:    mv a5, s5
+; RV32I-NEXT:    addi a4, zero, 2
+; RV32I-NEXT:    addi a5, zero, 2
 ; RV32I-NEXT:    call __atomic_compare_exchange_8
 ; RV32I-NEXT:    lw a1, 4(sp)
 ; RV32I-NEXT:    lw a2, 0(sp)
 ; RV32I-NEXT:    beqz a0, .LBB216_1
 ; RV32I-NEXT:  # %bb.9: # %atomicrmw.end
 ; RV32I-NEXT:    mv a0, a2
-; RV32I-NEXT:    lw s5, 8(sp)
 ; RV32I-NEXT:    lw s4, 12(sp)
 ; RV32I-NEXT:    lw s3, 16(sp)
 ; RV32I-NEXT:    lw s2, 20(sp)
@@ -9967,14 +17209,12 @@ define i64 @atomicrmw_umin_i64_acquire(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    sw s2, 20(sp)
 ; RV32IA-NEXT:    sw s3, 16(sp)
 ; RV32IA-NEXT:    sw s4, 12(sp)
-; RV32IA-NEXT:    sw s5, 8(sp)
 ; RV32IA-NEXT:    mv s1, a2
 ; RV32IA-NEXT:    mv s2, a1
 ; RV32IA-NEXT:    mv s3, a0
 ; RV32IA-NEXT:    lw a1, 4(a0)
 ; RV32IA-NEXT:    lw a2, 0(a0)
 ; RV32IA-NEXT:    mv s4, sp
-; RV32IA-NEXT:    addi s5, zero, 2
 ; RV32IA-NEXT:  .LBB216_1: # %atomicrmw.start
 ; RV32IA-NEXT:    # =>This Inner Loop Header: Depth=1
 ; RV32IA-NEXT:    beq a1, s1, .LBB216_3
@@ -10004,15 +17244,14 @@ define i64 @atomicrmw_umin_i64_acquire(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    sw a1, 4(sp)
 ; RV32IA-NEXT:    mv a0, s3
 ; RV32IA-NEXT:    mv a1, s4
-; RV32IA-NEXT:    mv a4, s5
-; RV32IA-NEXT:    mv a5, s5
+; RV32IA-NEXT:    addi a4, zero, 2
+; RV32IA-NEXT:    addi a5, zero, 2
 ; RV32IA-NEXT:    call __atomic_compare_exchange_8
 ; RV32IA-NEXT:    lw a1, 4(sp)
 ; RV32IA-NEXT:    lw a2, 0(sp)
 ; RV32IA-NEXT:    beqz a0, .LBB216_1
 ; RV32IA-NEXT:  # %bb.9: # %atomicrmw.end
 ; RV32IA-NEXT:    mv a0, a2
-; RV32IA-NEXT:    lw s5, 8(sp)
 ; RV32IA-NEXT:    lw s4, 12(sp)
 ; RV32IA-NEXT:    lw s3, 16(sp)
 ; RV32IA-NEXT:    lw s2, 20(sp)
@@ -10020,6 +17259,47 @@ define i64 @atomicrmw_umin_i64_acquire(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    lw ra, 28(sp)
 ; RV32IA-NEXT:    addi sp, sp, 32
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_umin_i64_acquire:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -48
+; RV64I-NEXT:    sd ra, 40(sp)
+; RV64I-NEXT:    sd s1, 32(sp)
+; RV64I-NEXT:    sd s2, 24(sp)
+; RV64I-NEXT:    sd s3, 16(sp)
+; RV64I-NEXT:    mv s1, a1
+; RV64I-NEXT:    mv s2, a0
+; RV64I-NEXT:    ld a2, 0(a0)
+; RV64I-NEXT:    addi s3, sp, 8
+; RV64I-NEXT:  .LBB216_1: # %atomicrmw.start
+; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV64I-NEXT:    sd a2, 8(sp)
+; RV64I-NEXT:    bgeu s1, a2, .LBB216_3
+; RV64I-NEXT:  # %bb.2: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB216_1 Depth=1
+; RV64I-NEXT:    mv a2, s1
+; RV64I-NEXT:  .LBB216_3: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB216_1 Depth=1
+; RV64I-NEXT:    mv a0, s2
+; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    addi a3, zero, 2
+; RV64I-NEXT:    addi a4, zero, 2
+; RV64I-NEXT:    call __atomic_compare_exchange_8
+; RV64I-NEXT:    ld a2, 8(sp)
+; RV64I-NEXT:    beqz a0, .LBB216_1
+; RV64I-NEXT:  # %bb.4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a2
+; RV64I-NEXT:    ld s3, 16(sp)
+; RV64I-NEXT:    ld s2, 24(sp)
+; RV64I-NEXT:    ld s1, 32(sp)
+; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    addi sp, sp, 48
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_umin_i64_acquire:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    amominu.d.aq a0, a1, (a0)
+; RV64IA-NEXT:    ret
   %1 = atomicrmw umin i64* %a, i64 %b acquire
   ret i64 %1
 }
@@ -10033,14 +17313,12 @@ define i64 @atomicrmw_umin_i64_release(i64 *%a, i64 %b) nounwind {
 ; RV32I-NEXT:    sw s2, 20(sp)
 ; RV32I-NEXT:    sw s3, 16(sp)
 ; RV32I-NEXT:    sw s4, 12(sp)
-; RV32I-NEXT:    sw s5, 8(sp)
 ; RV32I-NEXT:    mv s1, a2
 ; RV32I-NEXT:    mv s2, a1
 ; RV32I-NEXT:    mv s3, a0
 ; RV32I-NEXT:    lw a1, 4(a0)
 ; RV32I-NEXT:    lw a2, 0(a0)
 ; RV32I-NEXT:    mv s4, sp
-; RV32I-NEXT:    addi s5, zero, 3
 ; RV32I-NEXT:  .LBB217_1: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
 ; RV32I-NEXT:    beq a1, s1, .LBB217_3
@@ -10070,7 +17348,7 @@ define i64 @atomicrmw_umin_i64_release(i64 *%a, i64 %b) nounwind {
 ; RV32I-NEXT:    sw a1, 4(sp)
 ; RV32I-NEXT:    mv a0, s3
 ; RV32I-NEXT:    mv a1, s4
-; RV32I-NEXT:    mv a4, s5
+; RV32I-NEXT:    addi a4, zero, 3
 ; RV32I-NEXT:    mv a5, zero
 ; RV32I-NEXT:    call __atomic_compare_exchange_8
 ; RV32I-NEXT:    lw a1, 4(sp)
@@ -10078,7 +17356,6 @@ define i64 @atomicrmw_umin_i64_release(i64 *%a, i64 %b) nounwind {
 ; RV32I-NEXT:    beqz a0, .LBB217_1
 ; RV32I-NEXT:  # %bb.9: # %atomicrmw.end
 ; RV32I-NEXT:    mv a0, a2
-; RV32I-NEXT:    lw s5, 8(sp)
 ; RV32I-NEXT:    lw s4, 12(sp)
 ; RV32I-NEXT:    lw s3, 16(sp)
 ; RV32I-NEXT:    lw s2, 20(sp)
@@ -10095,14 +17372,12 @@ define i64 @atomicrmw_umin_i64_release(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    sw s2, 20(sp)
 ; RV32IA-NEXT:    sw s3, 16(sp)
 ; RV32IA-NEXT:    sw s4, 12(sp)
-; RV32IA-NEXT:    sw s5, 8(sp)
 ; RV32IA-NEXT:    mv s1, a2
 ; RV32IA-NEXT:    mv s2, a1
 ; RV32IA-NEXT:    mv s3, a0
 ; RV32IA-NEXT:    lw a1, 4(a0)
 ; RV32IA-NEXT:    lw a2, 0(a0)
 ; RV32IA-NEXT:    mv s4, sp
-; RV32IA-NEXT:    addi s5, zero, 3
 ; RV32IA-NEXT:  .LBB217_1: # %atomicrmw.start
 ; RV32IA-NEXT:    # =>This Inner Loop Header: Depth=1
 ; RV32IA-NEXT:    beq a1, s1, .LBB217_3
@@ -10132,7 +17407,7 @@ define i64 @atomicrmw_umin_i64_release(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    sw a1, 4(sp)
 ; RV32IA-NEXT:    mv a0, s3
 ; RV32IA-NEXT:    mv a1, s4
-; RV32IA-NEXT:    mv a4, s5
+; RV32IA-NEXT:    addi a4, zero, 3
 ; RV32IA-NEXT:    mv a5, zero
 ; RV32IA-NEXT:    call __atomic_compare_exchange_8
 ; RV32IA-NEXT:    lw a1, 4(sp)
@@ -10140,7 +17415,6 @@ define i64 @atomicrmw_umin_i64_release(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    beqz a0, .LBB217_1
 ; RV32IA-NEXT:  # %bb.9: # %atomicrmw.end
 ; RV32IA-NEXT:    mv a0, a2
-; RV32IA-NEXT:    lw s5, 8(sp)
 ; RV32IA-NEXT:    lw s4, 12(sp)
 ; RV32IA-NEXT:    lw s3, 16(sp)
 ; RV32IA-NEXT:    lw s2, 20(sp)
@@ -10148,6 +17422,47 @@ define i64 @atomicrmw_umin_i64_release(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    lw ra, 28(sp)
 ; RV32IA-NEXT:    addi sp, sp, 32
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_umin_i64_release:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -48
+; RV64I-NEXT:    sd ra, 40(sp)
+; RV64I-NEXT:    sd s1, 32(sp)
+; RV64I-NEXT:    sd s2, 24(sp)
+; RV64I-NEXT:    sd s3, 16(sp)
+; RV64I-NEXT:    mv s1, a1
+; RV64I-NEXT:    mv s2, a0
+; RV64I-NEXT:    ld a2, 0(a0)
+; RV64I-NEXT:    addi s3, sp, 8
+; RV64I-NEXT:  .LBB217_1: # %atomicrmw.start
+; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV64I-NEXT:    sd a2, 8(sp)
+; RV64I-NEXT:    bgeu s1, a2, .LBB217_3
+; RV64I-NEXT:  # %bb.2: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB217_1 Depth=1
+; RV64I-NEXT:    mv a2, s1
+; RV64I-NEXT:  .LBB217_3: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB217_1 Depth=1
+; RV64I-NEXT:    mv a0, s2
+; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    addi a3, zero, 3
+; RV64I-NEXT:    mv a4, zero
+; RV64I-NEXT:    call __atomic_compare_exchange_8
+; RV64I-NEXT:    ld a2, 8(sp)
+; RV64I-NEXT:    beqz a0, .LBB217_1
+; RV64I-NEXT:  # %bb.4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a2
+; RV64I-NEXT:    ld s3, 16(sp)
+; RV64I-NEXT:    ld s2, 24(sp)
+; RV64I-NEXT:    ld s1, 32(sp)
+; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    addi sp, sp, 48
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_umin_i64_release:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    amominu.d.rl a0, a1, (a0)
+; RV64IA-NEXT:    ret
   %1 = atomicrmw umin i64* %a, i64 %b release
   ret i64 %1
 }
@@ -10155,22 +17470,18 @@ define i64 @atomicrmw_umin_i64_release(i64 *%a, i64 %b) nounwind {
 define i64 @atomicrmw_umin_i64_acq_rel(i64 *%a, i64 %b) nounwind {
 ; RV32I-LABEL: atomicrmw_umin_i64_acq_rel:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    addi sp, sp, -48
-; RV32I-NEXT:    sw ra, 44(sp)
-; RV32I-NEXT:    sw s1, 40(sp)
-; RV32I-NEXT:    sw s2, 36(sp)
-; RV32I-NEXT:    sw s3, 32(sp)
-; RV32I-NEXT:    sw s4, 28(sp)
-; RV32I-NEXT:    sw s5, 24(sp)
-; RV32I-NEXT:    sw s6, 20(sp)
+; RV32I-NEXT:    addi sp, sp, -32
+; RV32I-NEXT:    sw ra, 28(sp)
+; RV32I-NEXT:    sw s1, 24(sp)
+; RV32I-NEXT:    sw s2, 20(sp)
+; RV32I-NEXT:    sw s3, 16(sp)
+; RV32I-NEXT:    sw s4, 12(sp)
 ; RV32I-NEXT:    mv s1, a2
 ; RV32I-NEXT:    mv s2, a1
 ; RV32I-NEXT:    mv s3, a0
 ; RV32I-NEXT:    lw a1, 4(a0)
 ; RV32I-NEXT:    lw a2, 0(a0)
-; RV32I-NEXT:    addi s4, sp, 8
-; RV32I-NEXT:    addi s5, zero, 4
-; RV32I-NEXT:    addi s6, zero, 2
+; RV32I-NEXT:    mv s4, sp
 ; RV32I-NEXT:  .LBB218_1: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
 ; RV32I-NEXT:    beq a1, s1, .LBB218_3
@@ -10183,7 +17494,7 @@ define i64 @atomicrmw_umin_i64_acq_rel(i64 *%a, i64 %b) nounwind {
 ; RV32I-NEXT:  .LBB218_4: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB218_1 Depth=1
 ; RV32I-NEXT:    xori a0, a0, 1
-; RV32I-NEXT:    sw a2, 8(sp)
+; RV32I-NEXT:    sw a2, 0(sp)
 ; RV32I-NEXT:    bnez a0, .LBB218_6
 ; RV32I-NEXT:  # %bb.5: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB218_1 Depth=1
@@ -10197,45 +17508,39 @@ define i64 @atomicrmw_umin_i64_acq_rel(i64 *%a, i64 %b) nounwind {
 ; RV32I-NEXT:    mv a3, s1
 ; RV32I-NEXT:  .LBB218_8: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB218_1 Depth=1
-; RV32I-NEXT:    sw a1, 12(sp)
+; RV32I-NEXT:    sw a1, 4(sp)
 ; RV32I-NEXT:    mv a0, s3
 ; RV32I-NEXT:    mv a1, s4
-; RV32I-NEXT:    mv a4, s5
-; RV32I-NEXT:    mv a5, s6
+; RV32I-NEXT:    addi a4, zero, 4
+; RV32I-NEXT:    addi a5, zero, 2
 ; RV32I-NEXT:    call __atomic_compare_exchange_8
-; RV32I-NEXT:    lw a1, 12(sp)
-; RV32I-NEXT:    lw a2, 8(sp)
+; RV32I-NEXT:    lw a1, 4(sp)
+; RV32I-NEXT:    lw a2, 0(sp)
 ; RV32I-NEXT:    beqz a0, .LBB218_1
 ; RV32I-NEXT:  # %bb.9: # %atomicrmw.end
 ; RV32I-NEXT:    mv a0, a2
-; RV32I-NEXT:    lw s6, 20(sp)
-; RV32I-NEXT:    lw s5, 24(sp)
-; RV32I-NEXT:    lw s4, 28(sp)
-; RV32I-NEXT:    lw s3, 32(sp)
-; RV32I-NEXT:    lw s2, 36(sp)
-; RV32I-NEXT:    lw s1, 40(sp)
-; RV32I-NEXT:    lw ra, 44(sp)
-; RV32I-NEXT:    addi sp, sp, 48
+; RV32I-NEXT:    lw s4, 12(sp)
+; RV32I-NEXT:    lw s3, 16(sp)
+; RV32I-NEXT:    lw s2, 20(sp)
+; RV32I-NEXT:    lw s1, 24(sp)
+; RV32I-NEXT:    lw ra, 28(sp)
+; RV32I-NEXT:    addi sp, sp, 32
 ; RV32I-NEXT:    ret
 ;
 ; RV32IA-LABEL: atomicrmw_umin_i64_acq_rel:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    addi sp, sp, -48
-; RV32IA-NEXT:    sw ra, 44(sp)
-; RV32IA-NEXT:    sw s1, 40(sp)
-; RV32IA-NEXT:    sw s2, 36(sp)
-; RV32IA-NEXT:    sw s3, 32(sp)
-; RV32IA-NEXT:    sw s4, 28(sp)
-; RV32IA-NEXT:    sw s5, 24(sp)
-; RV32IA-NEXT:    sw s6, 20(sp)
+; RV32IA-NEXT:    addi sp, sp, -32
+; RV32IA-NEXT:    sw ra, 28(sp)
+; RV32IA-NEXT:    sw s1, 24(sp)
+; RV32IA-NEXT:    sw s2, 20(sp)
+; RV32IA-NEXT:    sw s3, 16(sp)
+; RV32IA-NEXT:    sw s4, 12(sp)
 ; RV32IA-NEXT:    mv s1, a2
 ; RV32IA-NEXT:    mv s2, a1
 ; RV32IA-NEXT:    mv s3, a0
 ; RV32IA-NEXT:    lw a1, 4(a0)
 ; RV32IA-NEXT:    lw a2, 0(a0)
-; RV32IA-NEXT:    addi s4, sp, 8
-; RV32IA-NEXT:    addi s5, zero, 4
-; RV32IA-NEXT:    addi s6, zero, 2
+; RV32IA-NEXT:    mv s4, sp
 ; RV32IA-NEXT:  .LBB218_1: # %atomicrmw.start
 ; RV32IA-NEXT:    # =>This Inner Loop Header: Depth=1
 ; RV32IA-NEXT:    beq a1, s1, .LBB218_3
@@ -10248,7 +17553,7 @@ define i64 @atomicrmw_umin_i64_acq_rel(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:  .LBB218_4: # %atomicrmw.start
 ; RV32IA-NEXT:    # in Loop: Header=BB218_1 Depth=1
 ; RV32IA-NEXT:    xori a0, a0, 1
-; RV32IA-NEXT:    sw a2, 8(sp)
+; RV32IA-NEXT:    sw a2, 0(sp)
 ; RV32IA-NEXT:    bnez a0, .LBB218_6
 ; RV32IA-NEXT:  # %bb.5: # %atomicrmw.start
 ; RV32IA-NEXT:    # in Loop: Header=BB218_1 Depth=1
@@ -10262,26 +17567,65 @@ define i64 @atomicrmw_umin_i64_acq_rel(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    mv a3, s1
 ; RV32IA-NEXT:  .LBB218_8: # %atomicrmw.start
 ; RV32IA-NEXT:    # in Loop: Header=BB218_1 Depth=1
-; RV32IA-NEXT:    sw a1, 12(sp)
+; RV32IA-NEXT:    sw a1, 4(sp)
 ; RV32IA-NEXT:    mv a0, s3
 ; RV32IA-NEXT:    mv a1, s4
-; RV32IA-NEXT:    mv a4, s5
-; RV32IA-NEXT:    mv a5, s6
+; RV32IA-NEXT:    addi a4, zero, 4
+; RV32IA-NEXT:    addi a5, zero, 2
 ; RV32IA-NEXT:    call __atomic_compare_exchange_8
-; RV32IA-NEXT:    lw a1, 12(sp)
-; RV32IA-NEXT:    lw a2, 8(sp)
+; RV32IA-NEXT:    lw a1, 4(sp)
+; RV32IA-NEXT:    lw a2, 0(sp)
 ; RV32IA-NEXT:    beqz a0, .LBB218_1
 ; RV32IA-NEXT:  # %bb.9: # %atomicrmw.end
 ; RV32IA-NEXT:    mv a0, a2
-; RV32IA-NEXT:    lw s6, 20(sp)
-; RV32IA-NEXT:    lw s5, 24(sp)
-; RV32IA-NEXT:    lw s4, 28(sp)
-; RV32IA-NEXT:    lw s3, 32(sp)
-; RV32IA-NEXT:    lw s2, 36(sp)
-; RV32IA-NEXT:    lw s1, 40(sp)
-; RV32IA-NEXT:    lw ra, 44(sp)
-; RV32IA-NEXT:    addi sp, sp, 48
+; RV32IA-NEXT:    lw s4, 12(sp)
+; RV32IA-NEXT:    lw s3, 16(sp)
+; RV32IA-NEXT:    lw s2, 20(sp)
+; RV32IA-NEXT:    lw s1, 24(sp)
+; RV32IA-NEXT:    lw ra, 28(sp)
+; RV32IA-NEXT:    addi sp, sp, 32
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_umin_i64_acq_rel:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -48
+; RV64I-NEXT:    sd ra, 40(sp)
+; RV64I-NEXT:    sd s1, 32(sp)
+; RV64I-NEXT:    sd s2, 24(sp)
+; RV64I-NEXT:    sd s3, 16(sp)
+; RV64I-NEXT:    mv s1, a1
+; RV64I-NEXT:    mv s2, a0
+; RV64I-NEXT:    ld a2, 0(a0)
+; RV64I-NEXT:    addi s3, sp, 8
+; RV64I-NEXT:  .LBB218_1: # %atomicrmw.start
+; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV64I-NEXT:    sd a2, 8(sp)
+; RV64I-NEXT:    bgeu s1, a2, .LBB218_3
+; RV64I-NEXT:  # %bb.2: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB218_1 Depth=1
+; RV64I-NEXT:    mv a2, s1
+; RV64I-NEXT:  .LBB218_3: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB218_1 Depth=1
+; RV64I-NEXT:    mv a0, s2
+; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    addi a3, zero, 4
+; RV64I-NEXT:    addi a4, zero, 2
+; RV64I-NEXT:    call __atomic_compare_exchange_8
+; RV64I-NEXT:    ld a2, 8(sp)
+; RV64I-NEXT:    beqz a0, .LBB218_1
+; RV64I-NEXT:  # %bb.4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a2
+; RV64I-NEXT:    ld s3, 16(sp)
+; RV64I-NEXT:    ld s2, 24(sp)
+; RV64I-NEXT:    ld s1, 32(sp)
+; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    addi sp, sp, 48
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_umin_i64_acq_rel:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    amominu.d.aqrl a0, a1, (a0)
+; RV64IA-NEXT:    ret
   %1 = atomicrmw umin i64* %a, i64 %b acq_rel
   ret i64 %1
 }
@@ -10295,14 +17639,12 @@ define i64 @atomicrmw_umin_i64_seq_cst(i64 *%a, i64 %b) nounwind {
 ; RV32I-NEXT:    sw s2, 20(sp)
 ; RV32I-NEXT:    sw s3, 16(sp)
 ; RV32I-NEXT:    sw s4, 12(sp)
-; RV32I-NEXT:    sw s5, 8(sp)
 ; RV32I-NEXT:    mv s1, a2
 ; RV32I-NEXT:    mv s2, a1
 ; RV32I-NEXT:    mv s3, a0
 ; RV32I-NEXT:    lw a1, 4(a0)
 ; RV32I-NEXT:    lw a2, 0(a0)
 ; RV32I-NEXT:    mv s4, sp
-; RV32I-NEXT:    addi s5, zero, 5
 ; RV32I-NEXT:  .LBB219_1: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
 ; RV32I-NEXT:    beq a1, s1, .LBB219_3
@@ -10332,15 +17674,14 @@ define i64 @atomicrmw_umin_i64_seq_cst(i64 *%a, i64 %b) nounwind {
 ; RV32I-NEXT:    sw a1, 4(sp)
 ; RV32I-NEXT:    mv a0, s3
 ; RV32I-NEXT:    mv a1, s4
-; RV32I-NEXT:    mv a4, s5
-; RV32I-NEXT:    mv a5, s5
+; RV32I-NEXT:    addi a4, zero, 5
+; RV32I-NEXT:    addi a5, zero, 5
 ; RV32I-NEXT:    call __atomic_compare_exchange_8
 ; RV32I-NEXT:    lw a1, 4(sp)
 ; RV32I-NEXT:    lw a2, 0(sp)
 ; RV32I-NEXT:    beqz a0, .LBB219_1
 ; RV32I-NEXT:  # %bb.9: # %atomicrmw.end
 ; RV32I-NEXT:    mv a0, a2
-; RV32I-NEXT:    lw s5, 8(sp)
 ; RV32I-NEXT:    lw s4, 12(sp)
 ; RV32I-NEXT:    lw s3, 16(sp)
 ; RV32I-NEXT:    lw s2, 20(sp)
@@ -10357,14 +17698,12 @@ define i64 @atomicrmw_umin_i64_seq_cst(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    sw s2, 20(sp)
 ; RV32IA-NEXT:    sw s3, 16(sp)
 ; RV32IA-NEXT:    sw s4, 12(sp)
-; RV32IA-NEXT:    sw s5, 8(sp)
 ; RV32IA-NEXT:    mv s1, a2
 ; RV32IA-NEXT:    mv s2, a1
 ; RV32IA-NEXT:    mv s3, a0
 ; RV32IA-NEXT:    lw a1, 4(a0)
 ; RV32IA-NEXT:    lw a2, 0(a0)
 ; RV32IA-NEXT:    mv s4, sp
-; RV32IA-NEXT:    addi s5, zero, 5
 ; RV32IA-NEXT:  .LBB219_1: # %atomicrmw.start
 ; RV32IA-NEXT:    # =>This Inner Loop Header: Depth=1
 ; RV32IA-NEXT:    beq a1, s1, .LBB219_3
@@ -10394,15 +17733,14 @@ define i64 @atomicrmw_umin_i64_seq_cst(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    sw a1, 4(sp)
 ; RV32IA-NEXT:    mv a0, s3
 ; RV32IA-NEXT:    mv a1, s4
-; RV32IA-NEXT:    mv a4, s5
-; RV32IA-NEXT:    mv a5, s5
+; RV32IA-NEXT:    addi a4, zero, 5
+; RV32IA-NEXT:    addi a5, zero, 5
 ; RV32IA-NEXT:    call __atomic_compare_exchange_8
 ; RV32IA-NEXT:    lw a1, 4(sp)
 ; RV32IA-NEXT:    lw a2, 0(sp)
 ; RV32IA-NEXT:    beqz a0, .LBB219_1
 ; RV32IA-NEXT:  # %bb.9: # %atomicrmw.end
 ; RV32IA-NEXT:    mv a0, a2
-; RV32IA-NEXT:    lw s5, 8(sp)
 ; RV32IA-NEXT:    lw s4, 12(sp)
 ; RV32IA-NEXT:    lw s3, 16(sp)
 ; RV32IA-NEXT:    lw s2, 20(sp)
@@ -10410,6 +17748,47 @@ define i64 @atomicrmw_umin_i64_seq_cst(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    lw ra, 28(sp)
 ; RV32IA-NEXT:    addi sp, sp, 32
 ; RV32IA-NEXT:    ret
+;
+; RV64I-LABEL: atomicrmw_umin_i64_seq_cst:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -48
+; RV64I-NEXT:    sd ra, 40(sp)
+; RV64I-NEXT:    sd s1, 32(sp)
+; RV64I-NEXT:    sd s2, 24(sp)
+; RV64I-NEXT:    sd s3, 16(sp)
+; RV64I-NEXT:    mv s1, a1
+; RV64I-NEXT:    mv s2, a0
+; RV64I-NEXT:    ld a2, 0(a0)
+; RV64I-NEXT:    addi s3, sp, 8
+; RV64I-NEXT:  .LBB219_1: # %atomicrmw.start
+; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
+; RV64I-NEXT:    sd a2, 8(sp)
+; RV64I-NEXT:    bgeu s1, a2, .LBB219_3
+; RV64I-NEXT:  # %bb.2: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB219_1 Depth=1
+; RV64I-NEXT:    mv a2, s1
+; RV64I-NEXT:  .LBB219_3: # %atomicrmw.start
+; RV64I-NEXT:    # in Loop: Header=BB219_1 Depth=1
+; RV64I-NEXT:    mv a0, s2
+; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    addi a3, zero, 5
+; RV64I-NEXT:    addi a4, zero, 5
+; RV64I-NEXT:    call __atomic_compare_exchange_8
+; RV64I-NEXT:    ld a2, 8(sp)
+; RV64I-NEXT:    beqz a0, .LBB219_1
+; RV64I-NEXT:  # %bb.4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a2
+; RV64I-NEXT:    ld s3, 16(sp)
+; RV64I-NEXT:    ld s2, 24(sp)
+; RV64I-NEXT:    ld s1, 32(sp)
+; RV64I-NEXT:    ld ra, 40(sp)
+; RV64I-NEXT:    addi sp, sp, 48
+; RV64I-NEXT:    ret
+;
+; RV64IA-LABEL: atomicrmw_umin_i64_seq_cst:
+; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    amominu.d.aqrl a0, a1, (a0)
+; RV64IA-NEXT:    ret
   %1 = atomicrmw umin i64* %a, i64 %b seq_cst
   ret i64 %1
 }
diff --git a/test/CodeGen/RISCV/calling-conv-rv32f-ilp32.ll b/test/CodeGen/RISCV/calling-conv-rv32f-ilp32.ll
index 849e18a81bbd669e78cb8166e61bb824ef6814a7..9a68f0006668dff9c0f368524a4f82c36d0fbd1d 100644
--- a/test/CodeGen/RISCV/calling-conv-rv32f-ilp32.ll
+++ b/test/CodeGen/RISCV/calling-conv-rv32f-ilp32.ll
@@ -36,12 +36,12 @@ define float @caller_onstack_f32_noop(float %a) nounwind {
 ; RV32IF-NEXT:    lui a0, 264704
 ; RV32IF-NEXT:    sw a0, 0(sp)
 ; RV32IF-NEXT:    addi a0, zero, 1
-; RV32IF-NEXT:    addi a2, zero, 2
-; RV32IF-NEXT:    addi a4, zero, 3
-; RV32IF-NEXT:    addi a6, zero, 4
 ; RV32IF-NEXT:    mv a1, zero
+; RV32IF-NEXT:    addi a2, zero, 2
 ; RV32IF-NEXT:    mv a3, zero
+; RV32IF-NEXT:    addi a4, zero, 3
 ; RV32IF-NEXT:    mv a5, zero
+; RV32IF-NEXT:    addi a6, zero, 4
 ; RV32IF-NEXT:    mv a7, zero
 ; RV32IF-NEXT:    call onstack_f32_noop
 ; RV32IF-NEXT:    lw ra, 12(sp)
@@ -63,12 +63,12 @@ define float @caller_onstack_f32_fadd(float %a, float %b) nounwind {
 ; RV32IF-NEXT:    fadd.s ft0, ft0, ft1
 ; RV32IF-NEXT:    fsw ft0, 0(sp)
 ; RV32IF-NEXT:    addi a0, zero, 1
-; RV32IF-NEXT:    addi a2, zero, 2
-; RV32IF-NEXT:    addi a4, zero, 3
-; RV32IF-NEXT:    addi a6, zero, 4
 ; RV32IF-NEXT:    mv a1, zero
+; RV32IF-NEXT:    addi a2, zero, 2
 ; RV32IF-NEXT:    mv a3, zero
+; RV32IF-NEXT:    addi a4, zero, 3
 ; RV32IF-NEXT:    mv a5, zero
+; RV32IF-NEXT:    addi a6, zero, 4
 ; RV32IF-NEXT:    mv a7, zero
 ; RV32IF-NEXT:    call onstack_f32_noop
 ; RV32IF-NEXT:    lw ra, 12(sp)
diff --git a/test/CodeGen/RISCV/calling-conv.ll b/test/CodeGen/RISCV/calling-conv.ll
index 536859372831dae4674d3a0278de726fd742ea43..8de776377605d439a1fc288893cb1eec102f20f7 100644
--- a/test/CodeGen/RISCV/calling-conv.ll
+++ b/test/CodeGen/RISCV/calling-conv.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=riscv32 -verify-machineinstrs < %s \
 ; RUN:   | FileCheck -check-prefix=RV32I-FPELIM %s
-; RUN: llc -mtriple=riscv32 -verify-machineinstrs -disable-fp-elim < %s \
+; RUN: llc -mtriple=riscv32 -verify-machineinstrs -frame-pointer=all < %s \
 ; RUN:   | FileCheck -check-prefix=RV32I-WITHFP %s
 
 ; As well as calling convention details, we check that ra and fp are
@@ -83,11 +83,11 @@ define i32 @caller_scalars() nounwind {
 ; RV32I-FPELIM-NEXT:    sw ra, 12(sp)
 ; RV32I-FPELIM-NEXT:    addi a0, zero, 1
 ; RV32I-FPELIM-NEXT:    addi a1, zero, 2
+; RV32I-FPELIM-NEXT:    mv a2, zero
 ; RV32I-FPELIM-NEXT:    addi a3, zero, 3
 ; RV32I-FPELIM-NEXT:    addi a4, zero, 4
-; RV32I-FPELIM-NEXT:    lui a6, 262464
-; RV32I-FPELIM-NEXT:    mv a2, zero
 ; RV32I-FPELIM-NEXT:    mv a5, zero
+; RV32I-FPELIM-NEXT:    lui a6, 262464
 ; RV32I-FPELIM-NEXT:    call callee_scalars
 ; RV32I-FPELIM-NEXT:    lw ra, 12(sp)
 ; RV32I-FPELIM-NEXT:    addi sp, sp, 16
@@ -101,11 +101,11 @@ define i32 @caller_scalars() nounwind {
 ; RV32I-WITHFP-NEXT:    addi s0, sp, 16
 ; RV32I-WITHFP-NEXT:    addi a0, zero, 1
 ; RV32I-WITHFP-NEXT:    addi a1, zero, 2
+; RV32I-WITHFP-NEXT:    mv a2, zero
 ; RV32I-WITHFP-NEXT:    addi a3, zero, 3
 ; RV32I-WITHFP-NEXT:    addi a4, zero, 4
-; RV32I-WITHFP-NEXT:    lui a6, 262464
-; RV32I-WITHFP-NEXT:    mv a2, zero
 ; RV32I-WITHFP-NEXT:    mv a5, zero
+; RV32I-WITHFP-NEXT:    lui a6, 262464
 ; RV32I-WITHFP-NEXT:    call callee_scalars
 ; RV32I-WITHFP-NEXT:    lw s0, 8(sp)
 ; RV32I-WITHFP-NEXT:    lw ra, 12(sp)
@@ -296,6 +296,7 @@ define i32 @caller_large_scalars_exhausted_regs() nounwind {
 ; RV32I-FPELIM-NEXT:    sw zero, 44(sp)
 ; RV32I-FPELIM-NEXT:    addi a0, zero, 8
 ; RV32I-FPELIM-NEXT:    sw a0, 40(sp)
+; RV32I-FPELIM-NEXT:    addi a7, sp, 40
 ; RV32I-FPELIM-NEXT:    addi a0, zero, 1
 ; RV32I-FPELIM-NEXT:    addi a1, zero, 2
 ; RV32I-FPELIM-NEXT:    addi a2, zero, 3
@@ -303,7 +304,6 @@ define i32 @caller_large_scalars_exhausted_regs() nounwind {
 ; RV32I-FPELIM-NEXT:    addi a4, zero, 5
 ; RV32I-FPELIM-NEXT:    addi a5, zero, 6
 ; RV32I-FPELIM-NEXT:    addi a6, zero, 7
-; RV32I-FPELIM-NEXT:    addi a7, sp, 40
 ; RV32I-FPELIM-NEXT:    call callee_large_scalars_exhausted_regs
 ; RV32I-FPELIM-NEXT:    lw ra, 60(sp)
 ; RV32I-FPELIM-NEXT:    addi sp, sp, 64
@@ -329,6 +329,7 @@ define i32 @caller_large_scalars_exhausted_regs() nounwind {
 ; RV32I-WITHFP-NEXT:    sw zero, -20(s0)
 ; RV32I-WITHFP-NEXT:    addi a0, zero, 8
 ; RV32I-WITHFP-NEXT:    sw a0, -24(s0)
+; RV32I-WITHFP-NEXT:    addi a7, s0, -24
 ; RV32I-WITHFP-NEXT:    addi a0, zero, 1
 ; RV32I-WITHFP-NEXT:    addi a1, zero, 2
 ; RV32I-WITHFP-NEXT:    addi a2, zero, 3
@@ -336,7 +337,6 @@ define i32 @caller_large_scalars_exhausted_regs() nounwind {
 ; RV32I-WITHFP-NEXT:    addi a4, zero, 5
 ; RV32I-WITHFP-NEXT:    addi a5, zero, 6
 ; RV32I-WITHFP-NEXT:    addi a6, zero, 7
-; RV32I-WITHFP-NEXT:    addi a7, s0, -24
 ; RV32I-WITHFP-NEXT:    call callee_large_scalars_exhausted_regs
 ; RV32I-WITHFP-NEXT:    lw s0, 56(sp)
 ; RV32I-WITHFP-NEXT:    lw ra, 60(sp)
@@ -459,10 +459,10 @@ define i32 @caller_many_scalars() nounwind {
 ; RV32I-FPELIM-NEXT:    addi a1, zero, 2
 ; RV32I-FPELIM-NEXT:    addi a2, zero, 3
 ; RV32I-FPELIM-NEXT:    addi a3, zero, 4
+; RV32I-FPELIM-NEXT:    mv a4, zero
 ; RV32I-FPELIM-NEXT:    addi a5, zero, 5
 ; RV32I-FPELIM-NEXT:    addi a6, zero, 6
 ; RV32I-FPELIM-NEXT:    addi a7, zero, 7
-; RV32I-FPELIM-NEXT:    mv a4, zero
 ; RV32I-FPELIM-NEXT:    call callee_many_scalars
 ; RV32I-FPELIM-NEXT:    lw ra, 12(sp)
 ; RV32I-FPELIM-NEXT:    addi sp, sp, 16
@@ -481,10 +481,10 @@ define i32 @caller_many_scalars() nounwind {
 ; RV32I-WITHFP-NEXT:    addi a1, zero, 2
 ; RV32I-WITHFP-NEXT:    addi a2, zero, 3
 ; RV32I-WITHFP-NEXT:    addi a3, zero, 4
+; RV32I-WITHFP-NEXT:    mv a4, zero
 ; RV32I-WITHFP-NEXT:    addi a5, zero, 5
 ; RV32I-WITHFP-NEXT:    addi a6, zero, 6
 ; RV32I-WITHFP-NEXT:    addi a7, zero, 7
-; RV32I-WITHFP-NEXT:    mv a4, zero
 ; RV32I-WITHFP-NEXT:    call callee_many_scalars
 ; RV32I-WITHFP-NEXT:    lw s0, 8(sp)
 ; RV32I-WITHFP-NEXT:    lw ra, 12(sp)
@@ -734,9 +734,9 @@ define void @caller_aligned_stack() nounwind {
 ; RV32I-FPELIM-NEXT:    sw a0, 32(sp)
 ; RV32I-FPELIM-NEXT:    lui a0, 688509
 ; RV32I-FPELIM-NEXT:    addi a5, a0, -2048
+; RV32I-FPELIM-NEXT:    addi a2, sp, 32
 ; RV32I-FPELIM-NEXT:    addi a0, zero, 1
 ; RV32I-FPELIM-NEXT:    addi a1, zero, 11
-; RV32I-FPELIM-NEXT:    addi a2, sp, 32
 ; RV32I-FPELIM-NEXT:    addi a3, zero, 12
 ; RV32I-FPELIM-NEXT:    addi a4, zero, 13
 ; RV32I-FPELIM-NEXT:    addi a6, zero, 4
@@ -780,9 +780,9 @@ define void @caller_aligned_stack() nounwind {
 ; RV32I-WITHFP-NEXT:    sw a0, -32(s0)
 ; RV32I-WITHFP-NEXT:    lui a0, 688509
 ; RV32I-WITHFP-NEXT:    addi a5, a0, -2048
+; RV32I-WITHFP-NEXT:    addi a2, s0, -32
 ; RV32I-WITHFP-NEXT:    addi a0, zero, 1
 ; RV32I-WITHFP-NEXT:    addi a1, zero, 11
-; RV32I-WITHFP-NEXT:    addi a2, s0, -32
 ; RV32I-WITHFP-NEXT:    addi a3, zero, 12
 ; RV32I-WITHFP-NEXT:    addi a4, zero, 13
 ; RV32I-WITHFP-NEXT:    addi a6, zero, 4
diff --git a/test/CodeGen/RISCV/div.ll b/test/CodeGen/RISCV/div.ll
index da9df17ccd4ffa5e172ed9cfe18068be62561f05..1fd0084f3ac3a3c33919a3ca64310365f4b0a7e4 100644
--- a/test/CodeGen/RISCV/div.ll
+++ b/test/CodeGen/RISCV/div.ll
@@ -3,6 +3,10 @@
 ; RUN:   | FileCheck -check-prefix=RV32I %s
 ; RUN: llc -mtriple=riscv32 -mattr=+m -verify-machineinstrs < %s \
 ; RUN:   | FileCheck -check-prefix=RV32IM %s
+; RUN: llc -mtriple=riscv64 -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=RV64I %s
+; RUN: llc -mtriple=riscv64 -mattr=+m -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=RV64IM %s
 
 define i32 @udiv(i32 %a, i32 %b) nounwind {
 ; RV32I-LABEL: udiv:
@@ -18,6 +22,24 @@ define i32 @udiv(i32 %a, i32 %b) nounwind {
 ; RV32IM:       # %bb.0:
 ; RV32IM-NEXT:    divu a0, a0, a1
 ; RV32IM-NEXT:    ret
+;
+; RV64I-LABEL: udiv:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    slli a0, a0, 32
+; RV64I-NEXT:    srli a0, a0, 32
+; RV64I-NEXT:    slli a1, a1, 32
+; RV64I-NEXT:    srli a1, a1, 32
+; RV64I-NEXT:    call __udivdi3
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IM-LABEL: udiv:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    divuw a0, a0, a1
+; RV64IM-NEXT:    ret
   %1 = udiv i32 %a, %b
   ret i32 %1
 }
@@ -40,6 +62,34 @@ define i32 @udiv_constant(i32 %a) nounwind {
 ; RV32IM-NEXT:    mulhu a0, a0, a1
 ; RV32IM-NEXT:    srli a0, a0, 2
 ; RV32IM-NEXT:    ret
+;
+; RV64I-LABEL: udiv_constant:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    slli a0, a0, 32
+; RV64I-NEXT:    srli a0, a0, 32
+; RV64I-NEXT:    addi a1, zero, 5
+; RV64I-NEXT:    call __udivdi3
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IM-LABEL: udiv_constant:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    slli a0, a0, 32
+; RV64IM-NEXT:    srli a0, a0, 32
+; RV64IM-NEXT:    lui a1, 1035469
+; RV64IM-NEXT:    addiw a1, a1, -819
+; RV64IM-NEXT:    slli a1, a1, 12
+; RV64IM-NEXT:    addi a1, a1, -819
+; RV64IM-NEXT:    slli a1, a1, 12
+; RV64IM-NEXT:    addi a1, a1, -819
+; RV64IM-NEXT:    slli a1, a1, 12
+; RV64IM-NEXT:    addi a1, a1, -819
+; RV64IM-NEXT:    mulhu a0, a0, a1
+; RV64IM-NEXT:    srli a0, a0, 2
+; RV64IM-NEXT:    ret
   %1 = udiv i32 %a, 5
   ret i32 %1
 }
@@ -54,6 +104,16 @@ define i32 @udiv_pow2(i32 %a) nounwind {
 ; RV32IM:       # %bb.0:
 ; RV32IM-NEXT:    srli a0, a0, 3
 ; RV32IM-NEXT:    ret
+;
+; RV64I-LABEL: udiv_pow2:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    srliw a0, a0, 3
+; RV64I-NEXT:    ret
+;
+; RV64IM-LABEL: udiv_pow2:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    srliw a0, a0, 3
+; RV64IM-NEXT:    ret
   %1 = udiv i32 %a, 8
   ret i32 %1
 }
@@ -76,6 +136,20 @@ define i64 @udiv64(i64 %a, i64 %b) nounwind {
 ; RV32IM-NEXT:    lw ra, 12(sp)
 ; RV32IM-NEXT:    addi sp, sp, 16
 ; RV32IM-NEXT:    ret
+;
+; RV64I-LABEL: udiv64:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    call __udivdi3
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IM-LABEL: udiv64:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    divu a0, a0, a1
+; RV64IM-NEXT:    ret
   %1 = udiv i64 %a, %b
   ret i64 %1
 }
@@ -102,6 +176,30 @@ define i64 @udiv64_constant(i64 %a) nounwind {
 ; RV32IM-NEXT:    lw ra, 12(sp)
 ; RV32IM-NEXT:    addi sp, sp, 16
 ; RV32IM-NEXT:    ret
+;
+; RV64I-LABEL: udiv64_constant:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a1, zero, 5
+; RV64I-NEXT:    call __udivdi3
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IM-LABEL: udiv64_constant:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    lui a1, 1035469
+; RV64IM-NEXT:    addiw a1, a1, -819
+; RV64IM-NEXT:    slli a1, a1, 12
+; RV64IM-NEXT:    addi a1, a1, -819
+; RV64IM-NEXT:    slli a1, a1, 12
+; RV64IM-NEXT:    addi a1, a1, -819
+; RV64IM-NEXT:    slli a1, a1, 12
+; RV64IM-NEXT:    addi a1, a1, -819
+; RV64IM-NEXT:    mulhu a0, a0, a1
+; RV64IM-NEXT:    srli a0, a0, 2
+; RV64IM-NEXT:    ret
   %1 = udiv i64 %a, 5
   ret i64 %1
 }
@@ -120,6 +218,22 @@ define i32 @sdiv(i32 %a, i32 %b) nounwind {
 ; RV32IM:       # %bb.0:
 ; RV32IM-NEXT:    div a0, a0, a1
 ; RV32IM-NEXT:    ret
+;
+; RV64I-LABEL: sdiv:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sext.w a0, a0
+; RV64I-NEXT:    sext.w a1, a1
+; RV64I-NEXT:    call __divdi3
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IM-LABEL: sdiv:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    divw a0, a0, a1
+; RV64IM-NEXT:    ret
   %1 = sdiv i32 %a, %b
   ret i32 %1
 }
@@ -144,6 +258,34 @@ define i32 @sdiv_constant(i32 %a) nounwind {
 ; RV32IM-NEXT:    srai a0, a0, 1
 ; RV32IM-NEXT:    add a0, a0, a1
 ; RV32IM-NEXT:    ret
+;
+; RV64I-LABEL: sdiv_constant:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sext.w a0, a0
+; RV64I-NEXT:    addi a1, zero, 5
+; RV64I-NEXT:    call __divdi3
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IM-LABEL: sdiv_constant:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    sext.w a0, a0
+; RV64IM-NEXT:    lui a1, 13107
+; RV64IM-NEXT:    addiw a1, a1, 819
+; RV64IM-NEXT:    slli a1, a1, 12
+; RV64IM-NEXT:    addi a1, a1, 819
+; RV64IM-NEXT:    slli a1, a1, 12
+; RV64IM-NEXT:    addi a1, a1, 819
+; RV64IM-NEXT:    slli a1, a1, 13
+; RV64IM-NEXT:    addi a1, a1, 1639
+; RV64IM-NEXT:    mulh a0, a0, a1
+; RV64IM-NEXT:    srli a1, a0, 63
+; RV64IM-NEXT:    srai a0, a0, 1
+; RV64IM-NEXT:    add a0, a0, a1
+; RV64IM-NEXT:    ret
   %1 = sdiv i32 %a, 5
   ret i32 %1
 }
@@ -164,6 +306,24 @@ define i32 @sdiv_pow2(i32 %a) nounwind {
 ; RV32IM-NEXT:    add a0, a0, a1
 ; RV32IM-NEXT:    srai a0, a0, 3
 ; RV32IM-NEXT:    ret
+;
+; RV64I-LABEL: sdiv_pow2:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    sext.w a1, a0
+; RV64I-NEXT:    srli a1, a1, 60
+; RV64I-NEXT:    andi a1, a1, 7
+; RV64I-NEXT:    add a0, a0, a1
+; RV64I-NEXT:    sraiw a0, a0, 3
+; RV64I-NEXT:    ret
+;
+; RV64IM-LABEL: sdiv_pow2:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    sext.w a1, a0
+; RV64IM-NEXT:    srli a1, a1, 60
+; RV64IM-NEXT:    andi a1, a1, 7
+; RV64IM-NEXT:    add a0, a0, a1
+; RV64IM-NEXT:    sraiw a0, a0, 3
+; RV64IM-NEXT:    ret
   %1 = sdiv i32 %a, 8
   ret i32 %1
 }
@@ -186,6 +346,20 @@ define i64 @sdiv64(i64 %a, i64 %b) nounwind {
 ; RV32IM-NEXT:    lw ra, 12(sp)
 ; RV32IM-NEXT:    addi sp, sp, 16
 ; RV32IM-NEXT:    ret
+;
+; RV64I-LABEL: sdiv64:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    call __divdi3
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IM-LABEL: sdiv64:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    div a0, a0, a1
+; RV64IM-NEXT:    ret
   %1 = sdiv i64 %a, %b
   ret i64 %1
 }
@@ -212,6 +386,83 @@ define i64 @sdiv64_constant(i64 %a) nounwind {
 ; RV32IM-NEXT:    lw ra, 12(sp)
 ; RV32IM-NEXT:    addi sp, sp, 16
 ; RV32IM-NEXT:    ret
+;
+; RV64I-LABEL: sdiv64_constant:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a1, zero, 5
+; RV64I-NEXT:    call __divdi3
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IM-LABEL: sdiv64_constant:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    lui a1, 13107
+; RV64IM-NEXT:    addiw a1, a1, 819
+; RV64IM-NEXT:    slli a1, a1, 12
+; RV64IM-NEXT:    addi a1, a1, 819
+; RV64IM-NEXT:    slli a1, a1, 12
+; RV64IM-NEXT:    addi a1, a1, 819
+; RV64IM-NEXT:    slli a1, a1, 13
+; RV64IM-NEXT:    addi a1, a1, 1639
+; RV64IM-NEXT:    mulh a0, a0, a1
+; RV64IM-NEXT:    srli a1, a0, 63
+; RV64IM-NEXT:    srai a0, a0, 1
+; RV64IM-NEXT:    add a0, a0, a1
+; RV64IM-NEXT:    ret
   %1 = sdiv i64 %a, 5
   ret i64 %1
 }
+
+; Although this sdiv has two sexti32 operands, it shouldn't compile to divw on
+; RV64M as that wouldn't produce the correct result for e.g. INT_MIN/-1.
+
+define i64 @sdiv64_sext_operands(i32 %a, i32 %b) nounwind {
+; RV32I-LABEL: sdiv64_sext_operands:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -16
+; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    mv a2, a1
+; RV32I-NEXT:    srai a1, a0, 31
+; RV32I-NEXT:    srai a3, a2, 31
+; RV32I-NEXT:    call __divdi3
+; RV32I-NEXT:    lw ra, 12(sp)
+; RV32I-NEXT:    addi sp, sp, 16
+; RV32I-NEXT:    ret
+;
+; RV32IM-LABEL: sdiv64_sext_operands:
+; RV32IM:       # %bb.0:
+; RV32IM-NEXT:    addi sp, sp, -16
+; RV32IM-NEXT:    sw ra, 12(sp)
+; RV32IM-NEXT:    mv a2, a1
+; RV32IM-NEXT:    srai a1, a0, 31
+; RV32IM-NEXT:    srai a3, a2, 31
+; RV32IM-NEXT:    call __divdi3
+; RV32IM-NEXT:    lw ra, 12(sp)
+; RV32IM-NEXT:    addi sp, sp, 16
+; RV32IM-NEXT:    ret
+;
+; RV64I-LABEL: sdiv64_sext_operands:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sext.w a0, a0
+; RV64I-NEXT:    sext.w a1, a1
+; RV64I-NEXT:    call __divdi3
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IM-LABEL: sdiv64_sext_operands:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    sext.w a1, a1
+; RV64IM-NEXT:    sext.w a0, a0
+; RV64IM-NEXT:    div a0, a0, a1
+; RV64IM-NEXT:    ret
+  %1 = sext i32 %a to i64
+  %2 = sext i32 %b to i64
+  %3 = sdiv i64 %1, %2
+  ret i64 %3
+}
diff --git a/test/CodeGen/RISCV/double-arith.ll b/test/CodeGen/RISCV/double-arith.ll
index cd3a1d963783236718f29b951806ee289ea4725a..2b696210ae1dfa98790108a5b67a17e14e33ec62 100644
--- a/test/CodeGen/RISCV/double-arith.ll
+++ b/test/CodeGen/RISCV/double-arith.ll
@@ -1,6 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=riscv32 -mattr=+d -verify-machineinstrs < %s \
 ; RUN:   | FileCheck -check-prefix=RV32IFD %s
+; RUN: llc -mtriple=riscv64 -mattr=+d -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=RV64IFD %s
 
 ; These tests are each targeted at a particular RISC-V FPU instruction. Most
 ; other files in this folder exercise LLVM IR instructions that don't directly
@@ -22,6 +24,14 @@ define double @fadd_d(double %a, double %b) nounwind {
 ; RV32IFD-NEXT:    lw a1, 12(sp)
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: fadd_d:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    fmv.d.x ft0, a1
+; RV64IFD-NEXT:    fmv.d.x ft1, a0
+; RV64IFD-NEXT:    fadd.d ft0, ft1, ft0
+; RV64IFD-NEXT:    fmv.x.d a0, ft0
+; RV64IFD-NEXT:    ret
   %1 = fadd double %a, %b
   ret double %1
 }
@@ -42,6 +52,14 @@ define double @fsub_d(double %a, double %b) nounwind {
 ; RV32IFD-NEXT:    lw a1, 12(sp)
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: fsub_d:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    fmv.d.x ft0, a1
+; RV64IFD-NEXT:    fmv.d.x ft1, a0
+; RV64IFD-NEXT:    fsub.d ft0, ft1, ft0
+; RV64IFD-NEXT:    fmv.x.d a0, ft0
+; RV64IFD-NEXT:    ret
   %1 = fsub double %a, %b
   ret double %1
 }
@@ -62,6 +80,14 @@ define double @fmul_d(double %a, double %b) nounwind {
 ; RV32IFD-NEXT:    lw a1, 12(sp)
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: fmul_d:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    fmv.d.x ft0, a1
+; RV64IFD-NEXT:    fmv.d.x ft1, a0
+; RV64IFD-NEXT:    fmul.d ft0, ft1, ft0
+; RV64IFD-NEXT:    fmv.x.d a0, ft0
+; RV64IFD-NEXT:    ret
   %1 = fmul double %a, %b
   ret double %1
 }
@@ -82,11 +108,19 @@ define double @fdiv_d(double %a, double %b) nounwind {
 ; RV32IFD-NEXT:    lw a1, 12(sp)
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: fdiv_d:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    fmv.d.x ft0, a1
+; RV64IFD-NEXT:    fmv.d.x ft1, a0
+; RV64IFD-NEXT:    fdiv.d ft0, ft1, ft0
+; RV64IFD-NEXT:    fmv.x.d a0, ft0
+; RV64IFD-NEXT:    ret
   %1 = fdiv double %a, %b
   ret double %1
 }
 
-declare double @llvm.sqrt.f32(double)
+declare double @llvm.sqrt.f64(double)
 
 define double @fsqrt_d(double %a) nounwind {
 ; RV32IFD-LABEL: fsqrt_d:
@@ -101,11 +135,18 @@ define double @fsqrt_d(double %a) nounwind {
 ; RV32IFD-NEXT:    lw a1, 12(sp)
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
-  %1 = call double @llvm.sqrt.f32(double %a)
+;
+; RV64IFD-LABEL: fsqrt_d:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    fmv.d.x ft0, a0
+; RV64IFD-NEXT:    fsqrt.d ft0, ft0
+; RV64IFD-NEXT:    fmv.x.d a0, ft0
+; RV64IFD-NEXT:    ret
+  %1 = call double @llvm.sqrt.f64(double %a)
   ret double %1
 }
 
-declare double @llvm.copysign.f32(double, double)
+declare double @llvm.copysign.f64(double, double)
 
 define double @fsgnj_d(double %a, double %b) nounwind {
 ; RV32IFD-LABEL: fsgnj_d:
@@ -123,28 +164,51 @@ define double @fsgnj_d(double %a, double %b) nounwind {
 ; RV32IFD-NEXT:    lw a1, 12(sp)
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
-  %1 = call double @llvm.copysign.f32(double %a, double %b)
+;
+; RV64IFD-LABEL: fsgnj_d:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    fmv.d.x ft0, a1
+; RV64IFD-NEXT:    fmv.d.x ft1, a0
+; RV64IFD-NEXT:    fsgnj.d ft0, ft1, ft0
+; RV64IFD-NEXT:    fmv.x.d a0, ft0
+; RV64IFD-NEXT:    ret
+  %1 = call double @llvm.copysign.f64(double %a, double %b)
   ret double %1
 }
 
-define double @fneg_d(double %a) nounwind {
+; This function performs extra work to ensure that
+; DAGCombiner::visitBITCAST doesn't replace the fneg with an xor.
+define i32 @fneg_d(double %a, double %b) nounwind {
 ; RV32IFD-LABEL: fneg_d:
 ; RV32IFD:       # %bb.0:
 ; RV32IFD-NEXT:    addi sp, sp, -16
 ; RV32IFD-NEXT:    sw a0, 8(sp)
 ; RV32IFD-NEXT:    sw a1, 12(sp)
 ; RV32IFD-NEXT:    fld ft0, 8(sp)
-; RV32IFD-NEXT:    fneg.d ft0, ft0
-; RV32IFD-NEXT:    fsd ft0, 8(sp)
-; RV32IFD-NEXT:    lw a0, 8(sp)
-; RV32IFD-NEXT:    lw a1, 12(sp)
+; RV32IFD-NEXT:    fadd.d ft0, ft0, ft0
+; RV32IFD-NEXT:    fneg.d ft1, ft0
+; RV32IFD-NEXT:    feq.d a0, ft0, ft1
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
-  %1 = fsub double -0.0, %a
-  ret double %1
+;
+; RV64IFD-LABEL: fneg_d:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    fmv.d.x ft0, a0
+; RV64IFD-NEXT:    fadd.d ft0, ft0, ft0
+; RV64IFD-NEXT:    fneg.d ft1, ft0
+; RV64IFD-NEXT:    feq.d a0, ft0, ft1
+; RV64IFD-NEXT:    ret
+  %1 = fadd double %a, %a
+  %2 = fneg double %1
+  %3 = fcmp oeq double %1, %2
+  %4 = zext i1 %3 to i32
+  ret i32 %4
 }
 
 define double @fsgnjn_d(double %a, double %b) nounwind {
+; TODO: fsgnjn.s isn't selected on RV64 because DAGCombiner::visitBITCAST will
+; convert (bitconvert (fneg x)) to a xor.
+;
 ; RV32IFD-LABEL: fsgnjn_d:
 ; RV32IFD:       # %bb.0:
 ; RV32IFD-NEXT:    addi sp, sp, -16
@@ -160,31 +224,61 @@ define double @fsgnjn_d(double %a, double %b) nounwind {
 ; RV32IFD-NEXT:    lw a1, 12(sp)
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: fsgnjn_d:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    addi a2, zero, -1
+; RV64IFD-NEXT:    slli a2, a2, 63
+; RV64IFD-NEXT:    xor a1, a1, a2
+; RV64IFD-NEXT:    fmv.d.x ft0, a1
+; RV64IFD-NEXT:    fmv.d.x ft1, a0
+; RV64IFD-NEXT:    fsgnj.d ft0, ft1, ft0
+; RV64IFD-NEXT:    fmv.x.d a0, ft0
+; RV64IFD-NEXT:    ret
   %1 = fsub double -0.0, %b
-  %2 = call double @llvm.copysign.f32(double %a, double %1)
+  %2 = call double @llvm.copysign.f64(double %a, double %1)
   ret double %2
 }
 
-declare double @llvm.fabs.f32(double)
+declare double @llvm.fabs.f64(double)
 
-define double @fabs_d(double %a) nounwind {
+; This function performs extra work to ensure that
+; DAGCombiner::visitBITCAST doesn't replace the fabs with an and.
+define double @fabs_d(double %a, double %b) nounwind {
 ; RV32IFD-LABEL: fabs_d:
 ; RV32IFD:       # %bb.0:
 ; RV32IFD-NEXT:    addi sp, sp, -16
+; RV32IFD-NEXT:    sw a2, 8(sp)
+; RV32IFD-NEXT:    sw a3, 12(sp)
+; RV32IFD-NEXT:    fld ft0, 8(sp)
 ; RV32IFD-NEXT:    sw a0, 8(sp)
 ; RV32IFD-NEXT:    sw a1, 12(sp)
-; RV32IFD-NEXT:    fld ft0, 8(sp)
-; RV32IFD-NEXT:    fabs.d ft0, ft0
+; RV32IFD-NEXT:    fld ft1, 8(sp)
+; RV32IFD-NEXT:    fadd.d ft0, ft1, ft0
+; RV32IFD-NEXT:    fabs.d ft1, ft0
+; RV32IFD-NEXT:    fadd.d ft0, ft1, ft0
 ; RV32IFD-NEXT:    fsd ft0, 8(sp)
 ; RV32IFD-NEXT:    lw a0, 8(sp)
 ; RV32IFD-NEXT:    lw a1, 12(sp)
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
-  %1 = call double @llvm.fabs.f32(double %a)
-  ret double %1
+;
+; RV64IFD-LABEL: fabs_d:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    fmv.d.x ft0, a1
+; RV64IFD-NEXT:    fmv.d.x ft1, a0
+; RV64IFD-NEXT:    fadd.d ft0, ft1, ft0
+; RV64IFD-NEXT:    fabs.d ft1, ft0
+; RV64IFD-NEXT:    fadd.d ft0, ft1, ft0
+; RV64IFD-NEXT:    fmv.x.d a0, ft0
+; RV64IFD-NEXT:    ret
+  %1 = fadd double %a, %b
+  %2 = call double @llvm.fabs.f64(double %1)
+  %3 = fadd double %2, %1
+  ret double %3
 }
 
-declare double @llvm.minnum.f32(double, double)
+declare double @llvm.minnum.f64(double, double)
 
 define double @fmin_d(double %a, double %b) nounwind {
 ; RV32IFD-LABEL: fmin_d:
@@ -202,11 +296,19 @@ define double @fmin_d(double %a, double %b) nounwind {
 ; RV32IFD-NEXT:    lw a1, 12(sp)
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
-  %1 = call double @llvm.minnum.f32(double %a, double %b)
+;
+; RV64IFD-LABEL: fmin_d:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    fmv.d.x ft0, a1
+; RV64IFD-NEXT:    fmv.d.x ft1, a0
+; RV64IFD-NEXT:    fmin.d ft0, ft1, ft0
+; RV64IFD-NEXT:    fmv.x.d a0, ft0
+; RV64IFD-NEXT:    ret
+  %1 = call double @llvm.minnum.f64(double %a, double %b)
   ret double %1
 }
 
-declare double @llvm.maxnum.f32(double, double)
+declare double @llvm.maxnum.f64(double, double)
 
 define double @fmax_d(double %a, double %b) nounwind {
 ; RV32IFD-LABEL: fmax_d:
@@ -224,7 +326,15 @@ define double @fmax_d(double %a, double %b) nounwind {
 ; RV32IFD-NEXT:    lw a1, 12(sp)
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
-  %1 = call double @llvm.maxnum.f32(double %a, double %b)
+;
+; RV64IFD-LABEL: fmax_d:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    fmv.d.x ft0, a1
+; RV64IFD-NEXT:    fmv.d.x ft1, a0
+; RV64IFD-NEXT:    fmax.d ft0, ft1, ft0
+; RV64IFD-NEXT:    fmv.x.d a0, ft0
+; RV64IFD-NEXT:    ret
+  %1 = call double @llvm.maxnum.f64(double %a, double %b)
   ret double %1
 }
 
@@ -241,6 +351,13 @@ define i32 @feq_d(double %a, double %b) nounwind {
 ; RV32IFD-NEXT:    feq.d a0, ft1, ft0
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: feq_d:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    fmv.d.x ft0, a1
+; RV64IFD-NEXT:    fmv.d.x ft1, a0
+; RV64IFD-NEXT:    feq.d a0, ft1, ft0
+; RV64IFD-NEXT:    ret
   %1 = fcmp oeq double %a, %b
   %2 = zext i1 %1 to i32
   ret i32 %2
@@ -259,6 +376,13 @@ define i32 @flt_d(double %a, double %b) nounwind {
 ; RV32IFD-NEXT:    flt.d a0, ft1, ft0
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: flt_d:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    fmv.d.x ft0, a1
+; RV64IFD-NEXT:    fmv.d.x ft1, a0
+; RV64IFD-NEXT:    flt.d a0, ft1, ft0
+; RV64IFD-NEXT:    ret
   %1 = fcmp olt double %a, %b
   %2 = zext i1 %1 to i32
   ret i32 %2
@@ -277,6 +401,13 @@ define i32 @fle_d(double %a, double %b) nounwind {
 ; RV32IFD-NEXT:    fle.d a0, ft1, ft0
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: fle_d:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    fmv.d.x ft0, a1
+; RV64IFD-NEXT:    fmv.d.x ft1, a0
+; RV64IFD-NEXT:    fle.d a0, ft1, ft0
+; RV64IFD-NEXT:    ret
   %1 = fcmp ole double %a, %b
   %2 = zext i1 %1 to i32
   ret i32 %2
@@ -303,6 +434,15 @@ define double @fmadd_d(double %a, double %b, double %c) nounwind {
 ; RV32IFD-NEXT:    lw a1, 12(sp)
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: fmadd_d:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    fmv.d.x ft0, a2
+; RV64IFD-NEXT:    fmv.d.x ft1, a1
+; RV64IFD-NEXT:    fmv.d.x ft2, a0
+; RV64IFD-NEXT:    fmadd.d ft0, ft2, ft1, ft0
+; RV64IFD-NEXT:    fmv.x.d a0, ft0
+; RV64IFD-NEXT:    ret
   %1 = call double @llvm.fma.f64(double %a, double %b, double %c)
   ret double %1
 }
@@ -330,6 +470,19 @@ define double @fmsub_d(double %a, double %b, double %c) nounwind {
 ; RV32IFD-NEXT:    lw a1, 12(sp)
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: fmsub_d:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    fmv.d.x ft0, a2
+; RV64IFD-NEXT:    lui a2, %hi(.LCPI15_0)
+; RV64IFD-NEXT:    addi a2, a2, %lo(.LCPI15_0)
+; RV64IFD-NEXT:    fld ft1, 0(a2)
+; RV64IFD-NEXT:    fadd.d ft0, ft0, ft1
+; RV64IFD-NEXT:    fmv.d.x ft1, a1
+; RV64IFD-NEXT:    fmv.d.x ft2, a0
+; RV64IFD-NEXT:    fmsub.d ft0, ft2, ft1, ft0
+; RV64IFD-NEXT:    fmv.x.d a0, ft0
+; RV64IFD-NEXT:    ret
   %c_ = fadd double 0.0, %c ; avoid negation using xor
   %negc = fsub double -0.0, %c_
   %1 = call double @llvm.fma.f64(double %a, double %b, double %negc)
@@ -360,6 +513,20 @@ define double @fnmadd_d(double %a, double %b, double %c) nounwind {
 ; RV32IFD-NEXT:    lw a1, 12(sp)
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: fnmadd_d:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    fmv.d.x ft0, a2
+; RV64IFD-NEXT:    lui a2, %hi(.LCPI16_0)
+; RV64IFD-NEXT:    addi a2, a2, %lo(.LCPI16_0)
+; RV64IFD-NEXT:    fld ft1, 0(a2)
+; RV64IFD-NEXT:    fadd.d ft0, ft0, ft1
+; RV64IFD-NEXT:    fmv.d.x ft2, a0
+; RV64IFD-NEXT:    fadd.d ft1, ft2, ft1
+; RV64IFD-NEXT:    fmv.d.x ft2, a1
+; RV64IFD-NEXT:    fnmadd.d ft0, ft1, ft2, ft0
+; RV64IFD-NEXT:    fmv.x.d a0, ft0
+; RV64IFD-NEXT:    ret
   %a_ = fadd double 0.0, %a
   %c_ = fadd double 0.0, %c
   %nega = fsub double -0.0, %a_
@@ -391,6 +558,19 @@ define double @fnmsub_d(double %a, double %b, double %c) nounwind {
 ; RV32IFD-NEXT:    lw a1, 12(sp)
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: fnmsub_d:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    fmv.d.x ft0, a0
+; RV64IFD-NEXT:    lui a0, %hi(.LCPI17_0)
+; RV64IFD-NEXT:    addi a0, a0, %lo(.LCPI17_0)
+; RV64IFD-NEXT:    fld ft1, 0(a0)
+; RV64IFD-NEXT:    fadd.d ft0, ft0, ft1
+; RV64IFD-NEXT:    fmv.d.x ft1, a2
+; RV64IFD-NEXT:    fmv.d.x ft2, a1
+; RV64IFD-NEXT:    fnmsub.d ft0, ft0, ft2, ft1
+; RV64IFD-NEXT:    fmv.x.d a0, ft0
+; RV64IFD-NEXT:    ret
   %a_ = fadd double 0.0, %a
   %nega = fsub double -0.0, %a_
   %1 = call double @llvm.fma.f64(double %nega, double %b, double %c)
diff --git a/test/CodeGen/RISCV/double-bitmanip-dagcombines.ll b/test/CodeGen/RISCV/double-bitmanip-dagcombines.ll
new file mode 100644
index 0000000000000000000000000000000000000000..626a239e298a0c74478a304cd49515c025e44cb6
--- /dev/null
+++ b/test/CodeGen/RISCV/double-bitmanip-dagcombines.ll
@@ -0,0 +1,143 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv32 -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=RV32I %s
+; RUN: llc -mtriple=riscv32 -mattr=+d -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=RV32IFD %s
+; RUN: llc -mtriple=riscv64 -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=RV64I %s
+; RUN: llc -mtriple=riscv64 -mattr=+d -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=RV64IFD %s
+;
+; This file tests cases where simple floating point operations can be
+; profitably handled though bit manipulation if a soft-float ABI is being used
+; (e.g. fneg implemented by XORing the sign bit). This is typically handled in
+; DAGCombiner::visitBITCAST, but this target-independent code may not trigger
+; in cases where we perform custom legalisation (e.g. RV32IFD).
+
+; TODO: Add an appropriate target-specific DAG combine that can handle
+; RISCVISD::SplitF64/BuildPairF64 used for RV32IFD.
+
+define double @fneg(double %a) nounwind {
+; RV32I-LABEL: fneg:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    lui a2, 524288
+; RV32I-NEXT:    xor a1, a1, a2
+; RV32I-NEXT:    ret
+;
+; RV32IFD-LABEL: fneg:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    lui a2, 524288
+; RV32IFD-NEXT:    xor a1, a1, a2
+; RV32IFD-NEXT:    ret
+;
+; RV64I-LABEL: fneg:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi a1, zero, -1
+; RV64I-NEXT:    slli a1, a1, 63
+; RV64I-NEXT:    xor a0, a0, a1
+; RV64I-NEXT:    ret
+;
+; RV64IFD-LABEL: fneg:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    addi a1, zero, -1
+; RV64IFD-NEXT:    slli a1, a1, 63
+; RV64IFD-NEXT:    xor a0, a0, a1
+; RV64IFD-NEXT:    ret
+  %1 = fneg double %a
+  ret double %1
+}
+
+declare double @llvm.fabs.f64(double)
+
+define double @fabs(double %a) nounwind {
+; RV32I-LABEL: fabs:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    lui a2, 524288
+; RV32I-NEXT:    addi a2, a2, -1
+; RV32I-NEXT:    and a1, a1, a2
+; RV32I-NEXT:    ret
+;
+; RV32IFD-LABEL: fabs:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    lui a2, 524288
+; RV32IFD-NEXT:    addi a2, a2, -1
+; RV32IFD-NEXT:    and a1, a1, a2
+; RV32IFD-NEXT:    ret
+;
+; RV64I-LABEL: fabs:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi a1, zero, -1
+; RV64I-NEXT:    slli a1, a1, 63
+; RV64I-NEXT:    addi a1, a1, -1
+; RV64I-NEXT:    and a0, a0, a1
+; RV64I-NEXT:    ret
+;
+; RV64IFD-LABEL: fabs:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    addi a1, zero, -1
+; RV64IFD-NEXT:    slli a1, a1, 63
+; RV64IFD-NEXT:    addi a1, a1, -1
+; RV64IFD-NEXT:    and a0, a0, a1
+; RV64IFD-NEXT:    ret
+  %1 = call double @llvm.fabs.f64(double %a)
+  ret double %1
+}
+
+declare double @llvm.copysign.f64(double, double)
+
+; DAGTypeLegalizer::SoftenFloatRes_FCOPYSIGN will convert to bitwise
+; operations if floating point isn't supported. A combine could be written to
+; do the same even when f64 is legal.
+
+define double @fcopysign_fneg(double %a, double %b) nounwind {
+; RV32I-LABEL: fcopysign_fneg:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    not a2, a3
+; RV32I-NEXT:    lui a3, 524288
+; RV32I-NEXT:    and a2, a2, a3
+; RV32I-NEXT:    addi a3, a3, -1
+; RV32I-NEXT:    and a1, a1, a3
+; RV32I-NEXT:    or a1, a1, a2
+; RV32I-NEXT:    ret
+;
+; RV32IFD-LABEL: fcopysign_fneg:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    addi sp, sp, -16
+; RV32IFD-NEXT:    sw a2, 8(sp)
+; RV32IFD-NEXT:    sw a3, 12(sp)
+; RV32IFD-NEXT:    fld ft0, 8(sp)
+; RV32IFD-NEXT:    sw a0, 8(sp)
+; RV32IFD-NEXT:    sw a1, 12(sp)
+; RV32IFD-NEXT:    fld ft1, 8(sp)
+; RV32IFD-NEXT:    fsgnjn.d ft0, ft1, ft0
+; RV32IFD-NEXT:    fsd ft0, 8(sp)
+; RV32IFD-NEXT:    lw a0, 8(sp)
+; RV32IFD-NEXT:    lw a1, 12(sp)
+; RV32IFD-NEXT:    addi sp, sp, 16
+; RV32IFD-NEXT:    ret
+;
+; RV64I-LABEL: fcopysign_fneg:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi a2, zero, -1
+; RV64I-NEXT:    slli a2, a2, 63
+; RV64I-NEXT:    not a1, a1
+; RV64I-NEXT:    and a1, a1, a2
+; RV64I-NEXT:    addi a2, a2, -1
+; RV64I-NEXT:    and a0, a0, a2
+; RV64I-NEXT:    or a0, a0, a1
+; RV64I-NEXT:    ret
+;
+; RV64IFD-LABEL: fcopysign_fneg:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    addi a2, zero, -1
+; RV64IFD-NEXT:    slli a2, a2, 63
+; RV64IFD-NEXT:    xor a1, a1, a2
+; RV64IFD-NEXT:    fmv.d.x ft0, a1
+; RV64IFD-NEXT:    fmv.d.x ft1, a0
+; RV64IFD-NEXT:    fsgnj.d ft0, ft1, ft0
+; RV64IFD-NEXT:    fmv.x.d a0, ft0
+; RV64IFD-NEXT:    ret
+  %1 = fneg double %b
+  %2 = call double @llvm.copysign.f64(double %a, double %1)
+  ret double %2
+}
diff --git a/test/CodeGen/RISCV/double-br-fcmp.ll b/test/CodeGen/RISCV/double-br-fcmp.ll
index d1055c665864f386f4c20e8322df172534a43309..275c4aa05540a44852c1c83cd6d2e39c22fd4f9b 100644
--- a/test/CodeGen/RISCV/double-br-fcmp.ll
+++ b/test/CodeGen/RISCV/double-br-fcmp.ll
@@ -1,6 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=riscv32 -mattr=+d -verify-machineinstrs < %s \
 ; RUN:   | FileCheck -check-prefix=RV32IFD %s
+; RUN: llc -mtriple=riscv64 -mattr=+d -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=RV64IFD %s
 
 declare void @abort()
 declare void @exit(i32)
@@ -18,6 +20,19 @@ define void @br_fcmp_false(double %a, double %b) nounwind {
 ; RV32IFD-NEXT:    ret
 ; RV32IFD-NEXT:  .LBB0_2: # %if.else
 ; RV32IFD-NEXT:    call abort
+;
+; RV64IFD-LABEL: br_fcmp_false:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    addi sp, sp, -16
+; RV64IFD-NEXT:    sd ra, 8(sp)
+; RV64IFD-NEXT:    addi a0, zero, 1
+; RV64IFD-NEXT:    bnez a0, .LBB0_2
+; RV64IFD-NEXT:  # %bb.1: # %if.then
+; RV64IFD-NEXT:    ld ra, 8(sp)
+; RV64IFD-NEXT:    addi sp, sp, 16
+; RV64IFD-NEXT:    ret
+; RV64IFD-NEXT:  .LBB0_2: # %if.else
+; RV64IFD-NEXT:    call abort
   %1 = fcmp false double %a, %b
   br i1 %1, label %if.then, label %if.else
 if.then:
@@ -46,6 +61,21 @@ define void @br_fcmp_oeq(double %a, double %b) nounwind {
 ; RV32IFD-NEXT:    ret
 ; RV32IFD-NEXT:  .LBB1_2: # %if.then
 ; RV32IFD-NEXT:    call abort
+;
+; RV64IFD-LABEL: br_fcmp_oeq:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    addi sp, sp, -16
+; RV64IFD-NEXT:    sd ra, 8(sp)
+; RV64IFD-NEXT:    fmv.d.x ft0, a1
+; RV64IFD-NEXT:    fmv.d.x ft1, a0
+; RV64IFD-NEXT:    feq.d a0, ft1, ft0
+; RV64IFD-NEXT:    bnez a0, .LBB1_2
+; RV64IFD-NEXT:  # %bb.1: # %if.else
+; RV64IFD-NEXT:    ld ra, 8(sp)
+; RV64IFD-NEXT:    addi sp, sp, 16
+; RV64IFD-NEXT:    ret
+; RV64IFD-NEXT:  .LBB1_2: # %if.then
+; RV64IFD-NEXT:    call abort
   %1 = fcmp oeq double %a, %b
   br i1 %1, label %if.then, label %if.else
 if.else:
@@ -78,6 +108,22 @@ define void @br_fcmp_oeq_alt(double %a, double %b) nounwind {
 ; RV32IFD-NEXT:    ret
 ; RV32IFD-NEXT:  .LBB2_2: # %if.then
 ; RV32IFD-NEXT:    call abort
+;
+; RV64IFD-LABEL: br_fcmp_oeq_alt:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    addi sp, sp, -16
+; RV64IFD-NEXT:    sd ra, 8(sp)
+; RV64IFD-NEXT:    fmv.d.x ft0, a1
+; RV64IFD-NEXT:    fmv.d.x ft1, a0
+; RV64IFD-NEXT:    feq.d a0, ft1, ft0
+; RV64IFD-NEXT:    xori a0, a0, 1
+; RV64IFD-NEXT:    beqz a0, .LBB2_2
+; RV64IFD-NEXT:  # %bb.1: # %if.else
+; RV64IFD-NEXT:    ld ra, 8(sp)
+; RV64IFD-NEXT:    addi sp, sp, 16
+; RV64IFD-NEXT:    ret
+; RV64IFD-NEXT:  .LBB2_2: # %if.then
+; RV64IFD-NEXT:    call abort
   %1 = fcmp oeq double %a, %b
   br i1 %1, label %if.then, label %if.else
 if.then:
@@ -106,6 +152,21 @@ define void @br_fcmp_ogt(double %a, double %b) nounwind {
 ; RV32IFD-NEXT:    ret
 ; RV32IFD-NEXT:  .LBB3_2: # %if.then
 ; RV32IFD-NEXT:    call abort
+;
+; RV64IFD-LABEL: br_fcmp_ogt:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    addi sp, sp, -16
+; RV64IFD-NEXT:    sd ra, 8(sp)
+; RV64IFD-NEXT:    fmv.d.x ft0, a0
+; RV64IFD-NEXT:    fmv.d.x ft1, a1
+; RV64IFD-NEXT:    flt.d a0, ft1, ft0
+; RV64IFD-NEXT:    bnez a0, .LBB3_2
+; RV64IFD-NEXT:  # %bb.1: # %if.else
+; RV64IFD-NEXT:    ld ra, 8(sp)
+; RV64IFD-NEXT:    addi sp, sp, 16
+; RV64IFD-NEXT:    ret
+; RV64IFD-NEXT:  .LBB3_2: # %if.then
+; RV64IFD-NEXT:    call abort
   %1 = fcmp ogt double %a, %b
   br i1 %1, label %if.then, label %if.else
 if.else:
@@ -134,6 +195,21 @@ define void @br_fcmp_oge(double %a, double %b) nounwind {
 ; RV32IFD-NEXT:    ret
 ; RV32IFD-NEXT:  .LBB4_2: # %if.then
 ; RV32IFD-NEXT:    call abort
+;
+; RV64IFD-LABEL: br_fcmp_oge:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    addi sp, sp, -16
+; RV64IFD-NEXT:    sd ra, 8(sp)
+; RV64IFD-NEXT:    fmv.d.x ft0, a0
+; RV64IFD-NEXT:    fmv.d.x ft1, a1
+; RV64IFD-NEXT:    fle.d a0, ft1, ft0
+; RV64IFD-NEXT:    bnez a0, .LBB4_2
+; RV64IFD-NEXT:  # %bb.1: # %if.else
+; RV64IFD-NEXT:    ld ra, 8(sp)
+; RV64IFD-NEXT:    addi sp, sp, 16
+; RV64IFD-NEXT:    ret
+; RV64IFD-NEXT:  .LBB4_2: # %if.then
+; RV64IFD-NEXT:    call abort
   %1 = fcmp oge double %a, %b
   br i1 %1, label %if.then, label %if.else
 if.else:
@@ -162,6 +238,21 @@ define void @br_fcmp_olt(double %a, double %b) nounwind {
 ; RV32IFD-NEXT:    ret
 ; RV32IFD-NEXT:  .LBB5_2: # %if.then
 ; RV32IFD-NEXT:    call abort
+;
+; RV64IFD-LABEL: br_fcmp_olt:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    addi sp, sp, -16
+; RV64IFD-NEXT:    sd ra, 8(sp)
+; RV64IFD-NEXT:    fmv.d.x ft0, a1
+; RV64IFD-NEXT:    fmv.d.x ft1, a0
+; RV64IFD-NEXT:    flt.d a0, ft1, ft0
+; RV64IFD-NEXT:    bnez a0, .LBB5_2
+; RV64IFD-NEXT:  # %bb.1: # %if.else
+; RV64IFD-NEXT:    ld ra, 8(sp)
+; RV64IFD-NEXT:    addi sp, sp, 16
+; RV64IFD-NEXT:    ret
+; RV64IFD-NEXT:  .LBB5_2: # %if.then
+; RV64IFD-NEXT:    call abort
   %1 = fcmp olt double %a, %b
   br i1 %1, label %if.then, label %if.else
 if.else:
@@ -190,6 +281,21 @@ define void @br_fcmp_ole(double %a, double %b) nounwind {
 ; RV32IFD-NEXT:    ret
 ; RV32IFD-NEXT:  .LBB6_2: # %if.then
 ; RV32IFD-NEXT:    call abort
+;
+; RV64IFD-LABEL: br_fcmp_ole:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    addi sp, sp, -16
+; RV64IFD-NEXT:    sd ra, 8(sp)
+; RV64IFD-NEXT:    fmv.d.x ft0, a1
+; RV64IFD-NEXT:    fmv.d.x ft1, a0
+; RV64IFD-NEXT:    fle.d a0, ft1, ft0
+; RV64IFD-NEXT:    bnez a0, .LBB6_2
+; RV64IFD-NEXT:  # %bb.1: # %if.else
+; RV64IFD-NEXT:    ld ra, 8(sp)
+; RV64IFD-NEXT:    addi sp, sp, 16
+; RV64IFD-NEXT:    ret
+; RV64IFD-NEXT:  .LBB6_2: # %if.then
+; RV64IFD-NEXT:    call abort
   %1 = fcmp ole double %a, %b
   br i1 %1, label %if.then, label %if.else
 if.else:
@@ -226,6 +332,28 @@ define void @br_fcmp_one(double %a, double %b) nounwind {
 ; RV32IFD-NEXT:    ret
 ; RV32IFD-NEXT:  .LBB7_2: # %if.then
 ; RV32IFD-NEXT:    call abort
+;
+; RV64IFD-LABEL: br_fcmp_one:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    addi sp, sp, -16
+; RV64IFD-NEXT:    sd ra, 8(sp)
+; RV64IFD-NEXT:    fmv.d.x ft0, a0
+; RV64IFD-NEXT:    fmv.d.x ft1, a1
+; RV64IFD-NEXT:    feq.d a0, ft1, ft1
+; RV64IFD-NEXT:    feq.d a1, ft0, ft0
+; RV64IFD-NEXT:    and a0, a1, a0
+; RV64IFD-NEXT:    feq.d a1, ft0, ft1
+; RV64IFD-NEXT:    not a1, a1
+; RV64IFD-NEXT:    seqz a0, a0
+; RV64IFD-NEXT:    xori a0, a0, 1
+; RV64IFD-NEXT:    and a0, a1, a0
+; RV64IFD-NEXT:    bnez a0, .LBB7_2
+; RV64IFD-NEXT:  # %bb.1: # %if.else
+; RV64IFD-NEXT:    ld ra, 8(sp)
+; RV64IFD-NEXT:    addi sp, sp, 16
+; RV64IFD-NEXT:    ret
+; RV64IFD-NEXT:  .LBB7_2: # %if.then
+; RV64IFD-NEXT:    call abort
   %1 = fcmp one double %a, %b
   br i1 %1, label %if.then, label %if.else
 if.else:
@@ -258,6 +386,25 @@ define void @br_fcmp_ord(double %a, double %b) nounwind {
 ; RV32IFD-NEXT:    ret
 ; RV32IFD-NEXT:  .LBB8_2: # %if.then
 ; RV32IFD-NEXT:    call abort
+;
+; RV64IFD-LABEL: br_fcmp_ord:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    addi sp, sp, -16
+; RV64IFD-NEXT:    sd ra, 8(sp)
+; RV64IFD-NEXT:    fmv.d.x ft0, a1
+; RV64IFD-NEXT:    feq.d a1, ft0, ft0
+; RV64IFD-NEXT:    fmv.d.x ft0, a0
+; RV64IFD-NEXT:    feq.d a0, ft0, ft0
+; RV64IFD-NEXT:    and a0, a0, a1
+; RV64IFD-NEXT:    seqz a0, a0
+; RV64IFD-NEXT:    xori a0, a0, 1
+; RV64IFD-NEXT:    bnez a0, .LBB8_2
+; RV64IFD-NEXT:  # %bb.1: # %if.else
+; RV64IFD-NEXT:    ld ra, 8(sp)
+; RV64IFD-NEXT:    addi sp, sp, 16
+; RV64IFD-NEXT:    ret
+; RV64IFD-NEXT:  .LBB8_2: # %if.then
+; RV64IFD-NEXT:    call abort
   %1 = fcmp ord double %a, %b
   br i1 %1, label %if.then, label %if.else
 if.else:
@@ -291,6 +438,26 @@ define void @br_fcmp_ueq(double %a, double %b) nounwind {
 ; RV32IFD-NEXT:    ret
 ; RV32IFD-NEXT:  .LBB9_2: # %if.then
 ; RV32IFD-NEXT:    call abort
+;
+; RV64IFD-LABEL: br_fcmp_ueq:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    addi sp, sp, -16
+; RV64IFD-NEXT:    sd ra, 8(sp)
+; RV64IFD-NEXT:    fmv.d.x ft0, a1
+; RV64IFD-NEXT:    fmv.d.x ft1, a0
+; RV64IFD-NEXT:    feq.d a0, ft1, ft0
+; RV64IFD-NEXT:    feq.d a1, ft0, ft0
+; RV64IFD-NEXT:    feq.d a2, ft1, ft1
+; RV64IFD-NEXT:    and a1, a2, a1
+; RV64IFD-NEXT:    seqz a1, a1
+; RV64IFD-NEXT:    or a0, a0, a1
+; RV64IFD-NEXT:    bnez a0, .LBB9_2
+; RV64IFD-NEXT:  # %bb.1: # %if.else
+; RV64IFD-NEXT:    ld ra, 8(sp)
+; RV64IFD-NEXT:    addi sp, sp, 16
+; RV64IFD-NEXT:    ret
+; RV64IFD-NEXT:  .LBB9_2: # %if.then
+; RV64IFD-NEXT:    call abort
   %1 = fcmp ueq double %a, %b
   br i1 %1, label %if.then, label %if.else
 if.else:
@@ -320,6 +487,22 @@ define void @br_fcmp_ugt(double %a, double %b) nounwind {
 ; RV32IFD-NEXT:    ret
 ; RV32IFD-NEXT:  .LBB10_2: # %if.then
 ; RV32IFD-NEXT:    call abort
+;
+; RV64IFD-LABEL: br_fcmp_ugt:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    addi sp, sp, -16
+; RV64IFD-NEXT:    sd ra, 8(sp)
+; RV64IFD-NEXT:    fmv.d.x ft0, a1
+; RV64IFD-NEXT:    fmv.d.x ft1, a0
+; RV64IFD-NEXT:    fle.d a0, ft1, ft0
+; RV64IFD-NEXT:    xori a0, a0, 1
+; RV64IFD-NEXT:    bnez a0, .LBB10_2
+; RV64IFD-NEXT:  # %bb.1: # %if.else
+; RV64IFD-NEXT:    ld ra, 8(sp)
+; RV64IFD-NEXT:    addi sp, sp, 16
+; RV64IFD-NEXT:    ret
+; RV64IFD-NEXT:  .LBB10_2: # %if.then
+; RV64IFD-NEXT:    call abort
   %1 = fcmp ugt double %a, %b
   br i1 %1, label %if.then, label %if.else
 if.else:
@@ -349,6 +532,22 @@ define void @br_fcmp_uge(double %a, double %b) nounwind {
 ; RV32IFD-NEXT:    ret
 ; RV32IFD-NEXT:  .LBB11_2: # %if.then
 ; RV32IFD-NEXT:    call abort
+;
+; RV64IFD-LABEL: br_fcmp_uge:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    addi sp, sp, -16
+; RV64IFD-NEXT:    sd ra, 8(sp)
+; RV64IFD-NEXT:    fmv.d.x ft0, a1
+; RV64IFD-NEXT:    fmv.d.x ft1, a0
+; RV64IFD-NEXT:    flt.d a0, ft1, ft0
+; RV64IFD-NEXT:    xori a0, a0, 1
+; RV64IFD-NEXT:    bnez a0, .LBB11_2
+; RV64IFD-NEXT:  # %bb.1: # %if.else
+; RV64IFD-NEXT:    ld ra, 8(sp)
+; RV64IFD-NEXT:    addi sp, sp, 16
+; RV64IFD-NEXT:    ret
+; RV64IFD-NEXT:  .LBB11_2: # %if.then
+; RV64IFD-NEXT:    call abort
   %1 = fcmp uge double %a, %b
   br i1 %1, label %if.then, label %if.else
 if.else:
@@ -378,6 +577,22 @@ define void @br_fcmp_ult(double %a, double %b) nounwind {
 ; RV32IFD-NEXT:    ret
 ; RV32IFD-NEXT:  .LBB12_2: # %if.then
 ; RV32IFD-NEXT:    call abort
+;
+; RV64IFD-LABEL: br_fcmp_ult:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    addi sp, sp, -16
+; RV64IFD-NEXT:    sd ra, 8(sp)
+; RV64IFD-NEXT:    fmv.d.x ft0, a0
+; RV64IFD-NEXT:    fmv.d.x ft1, a1
+; RV64IFD-NEXT:    fle.d a0, ft1, ft0
+; RV64IFD-NEXT:    xori a0, a0, 1
+; RV64IFD-NEXT:    bnez a0, .LBB12_2
+; RV64IFD-NEXT:  # %bb.1: # %if.else
+; RV64IFD-NEXT:    ld ra, 8(sp)
+; RV64IFD-NEXT:    addi sp, sp, 16
+; RV64IFD-NEXT:    ret
+; RV64IFD-NEXT:  .LBB12_2: # %if.then
+; RV64IFD-NEXT:    call abort
   %1 = fcmp ult double %a, %b
   br i1 %1, label %if.then, label %if.else
 if.else:
@@ -407,6 +622,22 @@ define void @br_fcmp_ule(double %a, double %b) nounwind {
 ; RV32IFD-NEXT:    ret
 ; RV32IFD-NEXT:  .LBB13_2: # %if.then
 ; RV32IFD-NEXT:    call abort
+;
+; RV64IFD-LABEL: br_fcmp_ule:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    addi sp, sp, -16
+; RV64IFD-NEXT:    sd ra, 8(sp)
+; RV64IFD-NEXT:    fmv.d.x ft0, a0
+; RV64IFD-NEXT:    fmv.d.x ft1, a1
+; RV64IFD-NEXT:    flt.d a0, ft1, ft0
+; RV64IFD-NEXT:    xori a0, a0, 1
+; RV64IFD-NEXT:    bnez a0, .LBB13_2
+; RV64IFD-NEXT:  # %bb.1: # %if.else
+; RV64IFD-NEXT:    ld ra, 8(sp)
+; RV64IFD-NEXT:    addi sp, sp, 16
+; RV64IFD-NEXT:    ret
+; RV64IFD-NEXT:  .LBB13_2: # %if.then
+; RV64IFD-NEXT:    call abort
   %1 = fcmp ule double %a, %b
   br i1 %1, label %if.then, label %if.else
 if.else:
@@ -436,6 +667,22 @@ define void @br_fcmp_une(double %a, double %b) nounwind {
 ; RV32IFD-NEXT:    ret
 ; RV32IFD-NEXT:  .LBB14_2: # %if.then
 ; RV32IFD-NEXT:    call abort
+;
+; RV64IFD-LABEL: br_fcmp_une:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    addi sp, sp, -16
+; RV64IFD-NEXT:    sd ra, 8(sp)
+; RV64IFD-NEXT:    fmv.d.x ft0, a1
+; RV64IFD-NEXT:    fmv.d.x ft1, a0
+; RV64IFD-NEXT:    feq.d a0, ft1, ft0
+; RV64IFD-NEXT:    xori a0, a0, 1
+; RV64IFD-NEXT:    bnez a0, .LBB14_2
+; RV64IFD-NEXT:  # %bb.1: # %if.else
+; RV64IFD-NEXT:    ld ra, 8(sp)
+; RV64IFD-NEXT:    addi sp, sp, 16
+; RV64IFD-NEXT:    ret
+; RV64IFD-NEXT:  .LBB14_2: # %if.then
+; RV64IFD-NEXT:    call abort
   %1 = fcmp une double %a, %b
   br i1 %1, label %if.then, label %if.else
 if.else:
@@ -468,6 +715,24 @@ define void @br_fcmp_uno(double %a, double %b) nounwind {
 ; RV32IFD-NEXT:    ret
 ; RV32IFD-NEXT:  .LBB15_2: # %if.then
 ; RV32IFD-NEXT:    call abort
+;
+; RV64IFD-LABEL: br_fcmp_uno:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    addi sp, sp, -16
+; RV64IFD-NEXT:    sd ra, 8(sp)
+; RV64IFD-NEXT:    fmv.d.x ft0, a1
+; RV64IFD-NEXT:    feq.d a1, ft0, ft0
+; RV64IFD-NEXT:    fmv.d.x ft0, a0
+; RV64IFD-NEXT:    feq.d a0, ft0, ft0
+; RV64IFD-NEXT:    and a0, a0, a1
+; RV64IFD-NEXT:    seqz a0, a0
+; RV64IFD-NEXT:    bnez a0, .LBB15_2
+; RV64IFD-NEXT:  # %bb.1: # %if.else
+; RV64IFD-NEXT:    ld ra, 8(sp)
+; RV64IFD-NEXT:    addi sp, sp, 16
+; RV64IFD-NEXT:    ret
+; RV64IFD-NEXT:  .LBB15_2: # %if.then
+; RV64IFD-NEXT:    call abort
   %1 = fcmp uno double %a, %b
   br i1 %1, label %if.then, label %if.else
 if.else:
@@ -490,6 +755,19 @@ define void @br_fcmp_true(double %a, double %b) nounwind {
 ; RV32IFD-NEXT:    ret
 ; RV32IFD-NEXT:  .LBB16_2: # %if.then
 ; RV32IFD-NEXT:    call abort
+;
+; RV64IFD-LABEL: br_fcmp_true:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    addi sp, sp, -16
+; RV64IFD-NEXT:    sd ra, 8(sp)
+; RV64IFD-NEXT:    addi a0, zero, 1
+; RV64IFD-NEXT:    bnez a0, .LBB16_2
+; RV64IFD-NEXT:  # %bb.1: # %if.else
+; RV64IFD-NEXT:    ld ra, 8(sp)
+; RV64IFD-NEXT:    addi sp, sp, 16
+; RV64IFD-NEXT:    ret
+; RV64IFD-NEXT:  .LBB16_2: # %if.then
+; RV64IFD-NEXT:    call abort
   %1 = fcmp true double %a, %b
   br i1 %1, label %if.then, label %if.else
 if.else:
diff --git a/test/CodeGen/RISCV/double-calling-conv.ll b/test/CodeGen/RISCV/double-calling-conv.ll
index 5e9382fdf696ce4f8bc6901dba21c331a8605d04..2fe79418922eddac42ec30776555185e2be5c614 100644
--- a/test/CodeGen/RISCV/double-calling-conv.ll
+++ b/test/CodeGen/RISCV/double-calling-conv.ll
@@ -95,8 +95,8 @@ define double @caller_double_split_reg_stack() nounwind {
 ; RV32IFD-NEXT:    lw a6, 20(sp)
 ; RV32IFD-NEXT:    addi a0, zero, 1
 ; RV32IFD-NEXT:    addi a1, zero, 2
-; RV32IFD-NEXT:    addi a3, zero, 3
 ; RV32IFD-NEXT:    mv a2, zero
+; RV32IFD-NEXT:    addi a3, zero, 3
 ; RV32IFD-NEXT:    mv a4, zero
 ; RV32IFD-NEXT:    call callee_double_split_reg_stack
 ; RV32IFD-NEXT:    lw ra, 28(sp)
@@ -138,12 +138,12 @@ define double @caller_double_stack() nounwind {
 ; RV32IFD-NEXT:    sw a0, 0(sp)
 ; RV32IFD-NEXT:    sw a0, 8(sp)
 ; RV32IFD-NEXT:    addi a0, zero, 1
-; RV32IFD-NEXT:    addi a2, zero, 2
-; RV32IFD-NEXT:    addi a4, zero, 3
-; RV32IFD-NEXT:    addi a6, zero, 4
 ; RV32IFD-NEXT:    mv a1, zero
+; RV32IFD-NEXT:    addi a2, zero, 2
 ; RV32IFD-NEXT:    mv a3, zero
+; RV32IFD-NEXT:    addi a4, zero, 3
 ; RV32IFD-NEXT:    mv a5, zero
+; RV32IFD-NEXT:    addi a6, zero, 4
 ; RV32IFD-NEXT:    mv a7, zero
 ; RV32IFD-NEXT:    call callee_double_stack
 ; RV32IFD-NEXT:    lw ra, 28(sp)
diff --git a/test/CodeGen/RISCV/double-convert.ll b/test/CodeGen/RISCV/double-convert.ll
index 44de78dc74025fad04a920a05b200f9afc88baca..1d30fdadf4830f35b53d07e682f79eb8c9257e74 100644
--- a/test/CodeGen/RISCV/double-convert.ll
+++ b/test/CodeGen/RISCV/double-convert.ll
@@ -1,6 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=riscv32 -mattr=+d -verify-machineinstrs < %s \
 ; RUN:   | FileCheck -check-prefix=RV32IFD %s
+; RUN: llc -mtriple=riscv64 -mattr=+d -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=RV64IFD %s
 
 define float @fcvt_s_d(double %a) nounwind {
 ; RV32IFD-LABEL: fcvt_s_d:
@@ -13,6 +15,13 @@ define float @fcvt_s_d(double %a) nounwind {
 ; RV32IFD-NEXT:    fmv.x.w a0, ft0
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: fcvt_s_d:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    fmv.d.x ft0, a0
+; RV64IFD-NEXT:    fcvt.s.d ft0, ft0
+; RV64IFD-NEXT:    fmv.x.w a0, ft0
+; RV64IFD-NEXT:    ret
   %1 = fptrunc double %a to float
   ret float %1
 }
@@ -28,10 +37,19 @@ define double @fcvt_d_s(float %a) nounwind {
 ; RV32IFD-NEXT:    lw a1, 12(sp)
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: fcvt_d_s:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    fmv.w.x ft0, a0
+; RV64IFD-NEXT:    fcvt.d.s ft0, ft0
+; RV64IFD-NEXT:    fmv.x.d a0, ft0
+; RV64IFD-NEXT:    ret
   %1 = fpext float %a to double
   ret double %1
 }
 
+; For RV64D, fcvt.l.d is semantically equivalent to fcvt.w.d in this case
+; because fptosi will produce poison if the result doesn't fit into an i32.
 define i32 @fcvt_w_d(double %a) nounwind {
 ; RV32IFD-LABEL: fcvt_w_d:
 ; RV32IFD:       # %bb.0:
@@ -42,10 +60,18 @@ define i32 @fcvt_w_d(double %a) nounwind {
 ; RV32IFD-NEXT:    fcvt.w.d a0, ft0, rtz
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: fcvt_w_d:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    fmv.d.x ft0, a0
+; RV64IFD-NEXT:    fcvt.l.d a0, ft0, rtz
+; RV64IFD-NEXT:    ret
   %1 = fptosi double %a to i32
   ret i32 %1
 }
 
+; For RV64D, fcvt.lu.d is semantically equivalent to fcvt.wu.d in this case
+; because fptosi will produce poison if the result doesn't fit into an i32.
 define i32 @fcvt_wu_d(double %a) nounwind {
 ; RV32IFD-LABEL: fcvt_wu_d:
 ; RV32IFD:       # %bb.0:
@@ -56,6 +82,12 @@ define i32 @fcvt_wu_d(double %a) nounwind {
 ; RV32IFD-NEXT:    fcvt.wu.d a0, ft0, rtz
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: fcvt_wu_d:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    fmv.d.x ft0, a0
+; RV64IFD-NEXT:    fcvt.lu.d a0, ft0, rtz
+; RV64IFD-NEXT:    ret
   %1 = fptoui double %a to i32
   ret i32 %1
 }
@@ -70,6 +102,12 @@ define double @fcvt_d_w(i32 %a) nounwind {
 ; RV32IFD-NEXT:    lw a1, 12(sp)
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: fcvt_d_w:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    fcvt.d.w ft0, a0
+; RV64IFD-NEXT:    fmv.x.d a0, ft0
+; RV64IFD-NEXT:    ret
   %1 = sitofp i32 %a to double
   ret double %1
 }
@@ -84,6 +122,148 @@ define double @fcvt_d_wu(i32 %a) nounwind {
 ; RV32IFD-NEXT:    lw a1, 12(sp)
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: fcvt_d_wu:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    fcvt.d.wu ft0, a0
+; RV64IFD-NEXT:    fmv.x.d a0, ft0
+; RV64IFD-NEXT:    ret
   %1 = uitofp i32 %a to double
   ret double %1
 }
+
+define i64 @fcvt_l_d(double %a) nounwind {
+; RV32IFD-LABEL: fcvt_l_d:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    addi sp, sp, -16
+; RV32IFD-NEXT:    sw ra, 12(sp)
+; RV32IFD-NEXT:    call __fixdfdi
+; RV32IFD-NEXT:    lw ra, 12(sp)
+; RV32IFD-NEXT:    addi sp, sp, 16
+; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: fcvt_l_d:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    fmv.d.x ft0, a0
+; RV64IFD-NEXT:    fcvt.l.d a0, ft0, rtz
+; RV64IFD-NEXT:    ret
+  %1 = fptosi double %a to i64
+  ret i64 %1
+}
+
+define i64 @fcvt_lu_d(double %a) nounwind {
+; RV32IFD-LABEL: fcvt_lu_d:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    addi sp, sp, -16
+; RV32IFD-NEXT:    sw ra, 12(sp)
+; RV32IFD-NEXT:    call __fixunsdfdi
+; RV32IFD-NEXT:    lw ra, 12(sp)
+; RV32IFD-NEXT:    addi sp, sp, 16
+; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: fcvt_lu_d:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    fmv.d.x ft0, a0
+; RV64IFD-NEXT:    fcvt.lu.d a0, ft0, rtz
+; RV64IFD-NEXT:    ret
+  %1 = fptoui double %a to i64
+  ret i64 %1
+}
+
+define i64 @fmv_x_d(double %a, double %b) nounwind {
+; RV32IFD-LABEL: fmv_x_d:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    addi sp, sp, -16
+; RV32IFD-NEXT:    sw a2, 0(sp)
+; RV32IFD-NEXT:    sw a3, 4(sp)
+; RV32IFD-NEXT:    fld ft0, 0(sp)
+; RV32IFD-NEXT:    sw a0, 0(sp)
+; RV32IFD-NEXT:    sw a1, 4(sp)
+; RV32IFD-NEXT:    fld ft1, 0(sp)
+; RV32IFD-NEXT:    fadd.d ft0, ft1, ft0
+; RV32IFD-NEXT:    fsd ft0, 8(sp)
+; RV32IFD-NEXT:    lw a0, 8(sp)
+; RV32IFD-NEXT:    lw a1, 12(sp)
+; RV32IFD-NEXT:    addi sp, sp, 16
+; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: fmv_x_d:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    fmv.d.x ft0, a1
+; RV64IFD-NEXT:    fmv.d.x ft1, a0
+; RV64IFD-NEXT:    fadd.d ft0, ft1, ft0
+; RV64IFD-NEXT:    fmv.x.d a0, ft0
+; RV64IFD-NEXT:    ret
+  %1 = fadd double %a, %b
+  %2 = bitcast double %1 to i64
+  ret i64 %2
+}
+
+define double @fcvt_d_l(i64 %a) nounwind {
+; RV32IFD-LABEL: fcvt_d_l:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    addi sp, sp, -16
+; RV32IFD-NEXT:    sw ra, 12(sp)
+; RV32IFD-NEXT:    call __floatdidf
+; RV32IFD-NEXT:    lw ra, 12(sp)
+; RV32IFD-NEXT:    addi sp, sp, 16
+; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: fcvt_d_l:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    fcvt.d.l ft0, a0
+; RV64IFD-NEXT:    fmv.x.d a0, ft0
+; RV64IFD-NEXT:    ret
+  %1 = sitofp i64 %a to double
+  ret double %1
+}
+
+define double @fcvt_d_lu(i64 %a) nounwind {
+; RV32IFD-LABEL: fcvt_d_lu:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    addi sp, sp, -16
+; RV32IFD-NEXT:    sw ra, 12(sp)
+; RV32IFD-NEXT:    call __floatundidf
+; RV32IFD-NEXT:    lw ra, 12(sp)
+; RV32IFD-NEXT:    addi sp, sp, 16
+; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: fcvt_d_lu:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    fcvt.d.lu ft0, a0
+; RV64IFD-NEXT:    fmv.x.d a0, ft0
+; RV64IFD-NEXT:    ret
+  %1 = uitofp i64 %a to double
+  ret double %1
+}
+
+define double @fmv_d_x(i64 %a, i64 %b) nounwind {
+; Ensure fmv.w.x is generated even for a soft double calling convention
+; RV32IFD-LABEL: fmv_d_x:
+; RV32IFD:       # %bb.0:
+; RV32IFD-NEXT:    addi sp, sp, -32
+; RV32IFD-NEXT:    sw a3, 20(sp)
+; RV32IFD-NEXT:    sw a2, 16(sp)
+; RV32IFD-NEXT:    sw a1, 28(sp)
+; RV32IFD-NEXT:    sw a0, 24(sp)
+; RV32IFD-NEXT:    fld ft0, 16(sp)
+; RV32IFD-NEXT:    fld ft1, 24(sp)
+; RV32IFD-NEXT:    fadd.d ft0, ft1, ft0
+; RV32IFD-NEXT:    fsd ft0, 8(sp)
+; RV32IFD-NEXT:    lw a0, 8(sp)
+; RV32IFD-NEXT:    lw a1, 12(sp)
+; RV32IFD-NEXT:    addi sp, sp, 32
+; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: fmv_d_x:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    fmv.d.x ft0, a1
+; RV64IFD-NEXT:    fmv.d.x ft1, a0
+; RV64IFD-NEXT:    fadd.d ft0, ft1, ft0
+; RV64IFD-NEXT:    fmv.x.d a0, ft0
+; RV64IFD-NEXT:    ret
+  %1 = bitcast i64 %a to double
+  %2 = bitcast i64 %b to double
+  %3 = fadd double %1, %2
+  ret double %3
+}
diff --git a/test/CodeGen/RISCV/double-fcmp.ll b/test/CodeGen/RISCV/double-fcmp.ll
index adcd804122b820587eaef02b9147565dba91345d..25c8f6d96fb80ab94af2d51ac56fae9cd18338ab 100644
--- a/test/CodeGen/RISCV/double-fcmp.ll
+++ b/test/CodeGen/RISCV/double-fcmp.ll
@@ -1,12 +1,19 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=riscv32 -mattr=+d -verify-machineinstrs < %s \
 ; RUN:   | FileCheck -check-prefix=RV32IFD %s
+; RUN: llc -mtriple=riscv64 -mattr=+d -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=RV64IFD %s
 
 define i32 @fcmp_false(double %a, double %b) nounwind {
 ; RV32IFD-LABEL: fcmp_false:
 ; RV32IFD:       # %bb.0:
 ; RV32IFD-NEXT:    mv a0, zero
 ; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: fcmp_false:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    mv a0, zero
+; RV64IFD-NEXT:    ret
   %1 = fcmp false double %a, %b
   %2 = zext i1 %1 to i32
   ret i32 %2
@@ -25,6 +32,13 @@ define i32 @fcmp_oeq(double %a, double %b) nounwind {
 ; RV32IFD-NEXT:    feq.d a0, ft1, ft0
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: fcmp_oeq:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    fmv.d.x ft0, a1
+; RV64IFD-NEXT:    fmv.d.x ft1, a0
+; RV64IFD-NEXT:    feq.d a0, ft1, ft0
+; RV64IFD-NEXT:    ret
   %1 = fcmp oeq double %a, %b
   %2 = zext i1 %1 to i32
   ret i32 %2
@@ -43,6 +57,13 @@ define i32 @fcmp_ogt(double %a, double %b) nounwind {
 ; RV32IFD-NEXT:    flt.d a0, ft1, ft0
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: fcmp_ogt:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    fmv.d.x ft0, a0
+; RV64IFD-NEXT:    fmv.d.x ft1, a1
+; RV64IFD-NEXT:    flt.d a0, ft1, ft0
+; RV64IFD-NEXT:    ret
   %1 = fcmp ogt double %a, %b
   %2 = zext i1 %1 to i32
   ret i32 %2
@@ -61,6 +82,13 @@ define i32 @fcmp_oge(double %a, double %b) nounwind {
 ; RV32IFD-NEXT:    fle.d a0, ft1, ft0
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: fcmp_oge:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    fmv.d.x ft0, a0
+; RV64IFD-NEXT:    fmv.d.x ft1, a1
+; RV64IFD-NEXT:    fle.d a0, ft1, ft0
+; RV64IFD-NEXT:    ret
   %1 = fcmp oge double %a, %b
   %2 = zext i1 %1 to i32
   ret i32 %2
@@ -79,6 +107,13 @@ define i32 @fcmp_olt(double %a, double %b) nounwind {
 ; RV32IFD-NEXT:    flt.d a0, ft1, ft0
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: fcmp_olt:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    fmv.d.x ft0, a1
+; RV64IFD-NEXT:    fmv.d.x ft1, a0
+; RV64IFD-NEXT:    flt.d a0, ft1, ft0
+; RV64IFD-NEXT:    ret
   %1 = fcmp olt double %a, %b
   %2 = zext i1 %1 to i32
   ret i32 %2
@@ -97,6 +132,13 @@ define i32 @fcmp_ole(double %a, double %b) nounwind {
 ; RV32IFD-NEXT:    fle.d a0, ft1, ft0
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: fcmp_ole:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    fmv.d.x ft0, a1
+; RV64IFD-NEXT:    fmv.d.x ft1, a0
+; RV64IFD-NEXT:    fle.d a0, ft1, ft0
+; RV64IFD-NEXT:    ret
   %1 = fcmp ole double %a, %b
   %2 = zext i1 %1 to i32
   ret i32 %2
@@ -122,6 +164,20 @@ define i32 @fcmp_one(double %a, double %b) nounwind {
 ; RV32IFD-NEXT:    and a0, a1, a0
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: fcmp_one:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    fmv.d.x ft0, a0
+; RV64IFD-NEXT:    fmv.d.x ft1, a1
+; RV64IFD-NEXT:    feq.d a0, ft1, ft1
+; RV64IFD-NEXT:    feq.d a1, ft0, ft0
+; RV64IFD-NEXT:    and a0, a1, a0
+; RV64IFD-NEXT:    feq.d a1, ft0, ft1
+; RV64IFD-NEXT:    not a1, a1
+; RV64IFD-NEXT:    seqz a0, a0
+; RV64IFD-NEXT:    xori a0, a0, 1
+; RV64IFD-NEXT:    and a0, a1, a0
+; RV64IFD-NEXT:    ret
   %1 = fcmp one double %a, %b
   %2 = zext i1 %1 to i32
   ret i32 %2
@@ -144,6 +200,17 @@ define i32 @fcmp_ord(double %a, double %b) nounwind {
 ; RV32IFD-NEXT:    xori a0, a0, 1
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: fcmp_ord:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    fmv.d.x ft0, a1
+; RV64IFD-NEXT:    feq.d a1, ft0, ft0
+; RV64IFD-NEXT:    fmv.d.x ft0, a0
+; RV64IFD-NEXT:    feq.d a0, ft0, ft0
+; RV64IFD-NEXT:    and a0, a0, a1
+; RV64IFD-NEXT:    seqz a0, a0
+; RV64IFD-NEXT:    xori a0, a0, 1
+; RV64IFD-NEXT:    ret
   %1 = fcmp ord double %a, %b
   %2 = zext i1 %1 to i32
   ret i32 %2
@@ -167,6 +234,18 @@ define i32 @fcmp_ueq(double %a, double %b) nounwind {
 ; RV32IFD-NEXT:    or a0, a0, a1
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: fcmp_ueq:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    fmv.d.x ft0, a1
+; RV64IFD-NEXT:    fmv.d.x ft1, a0
+; RV64IFD-NEXT:    feq.d a0, ft1, ft0
+; RV64IFD-NEXT:    feq.d a1, ft0, ft0
+; RV64IFD-NEXT:    feq.d a2, ft1, ft1
+; RV64IFD-NEXT:    and a1, a2, a1
+; RV64IFD-NEXT:    seqz a1, a1
+; RV64IFD-NEXT:    or a0, a0, a1
+; RV64IFD-NEXT:    ret
   %1 = fcmp ueq double %a, %b
   %2 = zext i1 %1 to i32
   ret i32 %2
@@ -186,6 +265,14 @@ define i32 @fcmp_ugt(double %a, double %b) nounwind {
 ; RV32IFD-NEXT:    xori a0, a0, 1
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: fcmp_ugt:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    fmv.d.x ft0, a1
+; RV64IFD-NEXT:    fmv.d.x ft1, a0
+; RV64IFD-NEXT:    fle.d a0, ft1, ft0
+; RV64IFD-NEXT:    xori a0, a0, 1
+; RV64IFD-NEXT:    ret
   %1 = fcmp ugt double %a, %b
   %2 = zext i1 %1 to i32
   ret i32 %2
@@ -205,6 +292,14 @@ define i32 @fcmp_uge(double %a, double %b) nounwind {
 ; RV32IFD-NEXT:    xori a0, a0, 1
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: fcmp_uge:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    fmv.d.x ft0, a1
+; RV64IFD-NEXT:    fmv.d.x ft1, a0
+; RV64IFD-NEXT:    flt.d a0, ft1, ft0
+; RV64IFD-NEXT:    xori a0, a0, 1
+; RV64IFD-NEXT:    ret
   %1 = fcmp uge double %a, %b
   %2 = zext i1 %1 to i32
   ret i32 %2
@@ -224,6 +319,14 @@ define i32 @fcmp_ult(double %a, double %b) nounwind {
 ; RV32IFD-NEXT:    xori a0, a0, 1
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: fcmp_ult:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    fmv.d.x ft0, a0
+; RV64IFD-NEXT:    fmv.d.x ft1, a1
+; RV64IFD-NEXT:    fle.d a0, ft1, ft0
+; RV64IFD-NEXT:    xori a0, a0, 1
+; RV64IFD-NEXT:    ret
   %1 = fcmp ult double %a, %b
   %2 = zext i1 %1 to i32
   ret i32 %2
@@ -243,6 +346,14 @@ define i32 @fcmp_ule(double %a, double %b) nounwind {
 ; RV32IFD-NEXT:    xori a0, a0, 1
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: fcmp_ule:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    fmv.d.x ft0, a0
+; RV64IFD-NEXT:    fmv.d.x ft1, a1
+; RV64IFD-NEXT:    flt.d a0, ft1, ft0
+; RV64IFD-NEXT:    xori a0, a0, 1
+; RV64IFD-NEXT:    ret
   %1 = fcmp ule double %a, %b
   %2 = zext i1 %1 to i32
   ret i32 %2
@@ -262,6 +373,14 @@ define i32 @fcmp_une(double %a, double %b) nounwind {
 ; RV32IFD-NEXT:    xori a0, a0, 1
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: fcmp_une:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    fmv.d.x ft0, a1
+; RV64IFD-NEXT:    fmv.d.x ft1, a0
+; RV64IFD-NEXT:    feq.d a0, ft1, ft0
+; RV64IFD-NEXT:    xori a0, a0, 1
+; RV64IFD-NEXT:    ret
   %1 = fcmp une double %a, %b
   %2 = zext i1 %1 to i32
   ret i32 %2
@@ -283,6 +402,16 @@ define i32 @fcmp_uno(double %a, double %b) nounwind {
 ; RV32IFD-NEXT:    seqz a0, a0
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: fcmp_uno:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    fmv.d.x ft0, a1
+; RV64IFD-NEXT:    feq.d a1, ft0, ft0
+; RV64IFD-NEXT:    fmv.d.x ft0, a0
+; RV64IFD-NEXT:    feq.d a0, ft0, ft0
+; RV64IFD-NEXT:    and a0, a0, a1
+; RV64IFD-NEXT:    seqz a0, a0
+; RV64IFD-NEXT:    ret
   %1 = fcmp uno double %a, %b
   %2 = zext i1 %1 to i32
   ret i32 %2
@@ -293,6 +422,11 @@ define i32 @fcmp_true(double %a, double %b) nounwind {
 ; RV32IFD:       # %bb.0:
 ; RV32IFD-NEXT:    addi a0, zero, 1
 ; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: fcmp_true:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    addi a0, zero, 1
+; RV64IFD-NEXT:    ret
   %1 = fcmp true double %a, %b
   %2 = zext i1 %1 to i32
   ret i32 %2
diff --git a/test/CodeGen/RISCV/double-imm.ll b/test/CodeGen/RISCV/double-imm.ll
index a652d862175aec52c72fbaf010c8f48a1e122122..8a763b82202a41a6512c211e3dfc71c2bc14fc3f 100644
--- a/test/CodeGen/RISCV/double-imm.ll
+++ b/test/CodeGen/RISCV/double-imm.ll
@@ -1,8 +1,13 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=riscv32 -mattr=+d -verify-machineinstrs < %s \
 ; RUN:   | FileCheck -check-prefix=RV32IFD %s
+; RUN: llc -mtriple=riscv64 -mattr=+d -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=RV64IFD %s
 
 define double @double_imm() nounwind {
+; TODO: Should probably prefer fld or ld on RV64 rather than materialising an
+; expensive constant.
+;
 ; RV32IFD-LABEL: double_imm:
 ; RV32IFD:       # %bb.0:
 ; RV32IFD-NEXT:    addi sp, sp, -16
@@ -14,6 +19,18 @@ define double @double_imm() nounwind {
 ; RV32IFD-NEXT:    lw a1, 12(sp)
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: double_imm:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    lui a0, 512
+; RV64IFD-NEXT:    addiw a0, a0, 1169
+; RV64IFD-NEXT:    slli a0, a0, 15
+; RV64IFD-NEXT:    addi a0, a0, -299
+; RV64IFD-NEXT:    slli a0, a0, 14
+; RV64IFD-NEXT:    addi a0, a0, 1091
+; RV64IFD-NEXT:    slli a0, a0, 12
+; RV64IFD-NEXT:    addi a0, a0, -744
+; RV64IFD-NEXT:    ret
   ret double 3.1415926535897931159979634685441851615905761718750
 }
 
@@ -33,6 +50,16 @@ define double @double_imm_op(double %a) nounwind {
 ; RV32IFD-NEXT:    lw a1, 12(sp)
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: double_imm_op:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    fmv.d.x ft0, a0
+; RV64IFD-NEXT:    lui a0, %hi(.LCPI1_0)
+; RV64IFD-NEXT:    addi a0, a0, %lo(.LCPI1_0)
+; RV64IFD-NEXT:    fld ft1, 0(a0)
+; RV64IFD-NEXT:    fadd.d ft0, ft0, ft1
+; RV64IFD-NEXT:    fmv.x.d a0, ft0
+; RV64IFD-NEXT:    ret
   %1 = fadd double %a, 1.0
   ret double %1
 }
diff --git a/test/CodeGen/RISCV/double-intrinsics.ll b/test/CodeGen/RISCV/double-intrinsics.ll
index cd14c1932d4712726db2bdec96cd07a1dd45a485..354601b91a8cecb486e3171d90a5d3cf63f1f662 100644
--- a/test/CodeGen/RISCV/double-intrinsics.ll
+++ b/test/CodeGen/RISCV/double-intrinsics.ll
@@ -1,6 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=riscv32 -mattr=+d -verify-machineinstrs < %s \
 ; RUN:   | FileCheck -check-prefix=RV32IFD %s
+; RUN: llc -mtriple=riscv64 -mattr=+d -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=RV64IFD %s
 
 declare double @llvm.sqrt.f64(double)
 
@@ -17,6 +19,13 @@ define double @sqrt_f64(double %a) nounwind {
 ; RV32IFD-NEXT:    lw a1, 12(sp)
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: sqrt_f64:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    fmv.d.x ft0, a0
+; RV64IFD-NEXT:    fsqrt.d ft0, ft0
+; RV64IFD-NEXT:    fmv.x.d a0, ft0
+; RV64IFD-NEXT:    ret
   %1 = call double @llvm.sqrt.f64(double %a)
   ret double %1
 }
@@ -32,6 +41,16 @@ define double @powi_f64(double %a, i32 %b) nounwind {
 ; RV32IFD-NEXT:    lw ra, 12(sp)
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: powi_f64:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    addi sp, sp, -16
+; RV64IFD-NEXT:    sd ra, 8(sp)
+; RV64IFD-NEXT:    sext.w a1, a1
+; RV64IFD-NEXT:    call __powidf2
+; RV64IFD-NEXT:    ld ra, 8(sp)
+; RV64IFD-NEXT:    addi sp, sp, 16
+; RV64IFD-NEXT:    ret
   %1 = call double @llvm.powi.f64(double %a, i32 %b)
   ret double %1
 }
@@ -47,6 +66,15 @@ define double @sin_f64(double %a) nounwind {
 ; RV32IFD-NEXT:    lw ra, 12(sp)
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: sin_f64:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    addi sp, sp, -16
+; RV64IFD-NEXT:    sd ra, 8(sp)
+; RV64IFD-NEXT:    call sin
+; RV64IFD-NEXT:    ld ra, 8(sp)
+; RV64IFD-NEXT:    addi sp, sp, 16
+; RV64IFD-NEXT:    ret
   %1 = call double @llvm.sin.f64(double %a)
   ret double %1
 }
@@ -62,6 +90,15 @@ define double @cos_f64(double %a) nounwind {
 ; RV32IFD-NEXT:    lw ra, 12(sp)
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: cos_f64:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    addi sp, sp, -16
+; RV64IFD-NEXT:    sd ra, 8(sp)
+; RV64IFD-NEXT:    call cos
+; RV64IFD-NEXT:    ld ra, 8(sp)
+; RV64IFD-NEXT:    addi sp, sp, 16
+; RV64IFD-NEXT:    ret
   %1 = call double @llvm.cos.f64(double %a)
   ret double %1
 }
@@ -101,6 +138,27 @@ define double @sincos_f64(double %a) nounwind {
 ; RV32IFD-NEXT:    lw ra, 28(sp)
 ; RV32IFD-NEXT:    addi sp, sp, 32
 ; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: sincos_f64:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    addi sp, sp, -32
+; RV64IFD-NEXT:    sd ra, 24(sp)
+; RV64IFD-NEXT:    sd s1, 16(sp)
+; RV64IFD-NEXT:    sd s2, 8(sp)
+; RV64IFD-NEXT:    mv s1, a0
+; RV64IFD-NEXT:    call sin
+; RV64IFD-NEXT:    mv s2, a0
+; RV64IFD-NEXT:    mv a0, s1
+; RV64IFD-NEXT:    call cos
+; RV64IFD-NEXT:    fmv.d.x ft0, a0
+; RV64IFD-NEXT:    fmv.d.x ft1, s2
+; RV64IFD-NEXT:    fadd.d ft0, ft1, ft0
+; RV64IFD-NEXT:    fmv.x.d a0, ft0
+; RV64IFD-NEXT:    ld s2, 8(sp)
+; RV64IFD-NEXT:    ld s1, 16(sp)
+; RV64IFD-NEXT:    ld ra, 24(sp)
+; RV64IFD-NEXT:    addi sp, sp, 32
+; RV64IFD-NEXT:    ret
   %1 = call double @llvm.sin.f64(double %a)
   %2 = call double @llvm.cos.f64(double %a)
   %3 = fadd double %1, %2
@@ -118,6 +176,15 @@ define double @pow_f64(double %a, double %b) nounwind {
 ; RV32IFD-NEXT:    lw ra, 12(sp)
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: pow_f64:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    addi sp, sp, -16
+; RV64IFD-NEXT:    sd ra, 8(sp)
+; RV64IFD-NEXT:    call pow
+; RV64IFD-NEXT:    ld ra, 8(sp)
+; RV64IFD-NEXT:    addi sp, sp, 16
+; RV64IFD-NEXT:    ret
   %1 = call double @llvm.pow.f64(double %a, double %b)
   ret double %1
 }
@@ -133,6 +200,15 @@ define double @exp_f64(double %a) nounwind {
 ; RV32IFD-NEXT:    lw ra, 12(sp)
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: exp_f64:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    addi sp, sp, -16
+; RV64IFD-NEXT:    sd ra, 8(sp)
+; RV64IFD-NEXT:    call exp
+; RV64IFD-NEXT:    ld ra, 8(sp)
+; RV64IFD-NEXT:    addi sp, sp, 16
+; RV64IFD-NEXT:    ret
   %1 = call double @llvm.exp.f64(double %a)
   ret double %1
 }
@@ -148,6 +224,15 @@ define double @exp2_f64(double %a) nounwind {
 ; RV32IFD-NEXT:    lw ra, 12(sp)
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: exp2_f64:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    addi sp, sp, -16
+; RV64IFD-NEXT:    sd ra, 8(sp)
+; RV64IFD-NEXT:    call exp2
+; RV64IFD-NEXT:    ld ra, 8(sp)
+; RV64IFD-NEXT:    addi sp, sp, 16
+; RV64IFD-NEXT:    ret
   %1 = call double @llvm.exp2.f64(double %a)
   ret double %1
 }
@@ -163,6 +248,15 @@ define double @log_f64(double %a) nounwind {
 ; RV32IFD-NEXT:    lw ra, 12(sp)
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: log_f64:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    addi sp, sp, -16
+; RV64IFD-NEXT:    sd ra, 8(sp)
+; RV64IFD-NEXT:    call log
+; RV64IFD-NEXT:    ld ra, 8(sp)
+; RV64IFD-NEXT:    addi sp, sp, 16
+; RV64IFD-NEXT:    ret
   %1 = call double @llvm.log.f64(double %a)
   ret double %1
 }
@@ -178,6 +272,15 @@ define double @log10_f64(double %a) nounwind {
 ; RV32IFD-NEXT:    lw ra, 12(sp)
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: log10_f64:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    addi sp, sp, -16
+; RV64IFD-NEXT:    sd ra, 8(sp)
+; RV64IFD-NEXT:    call log10
+; RV64IFD-NEXT:    ld ra, 8(sp)
+; RV64IFD-NEXT:    addi sp, sp, 16
+; RV64IFD-NEXT:    ret
   %1 = call double @llvm.log10.f64(double %a)
   ret double %1
 }
@@ -193,6 +296,15 @@ define double @log2_f64(double %a) nounwind {
 ; RV32IFD-NEXT:    lw ra, 12(sp)
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: log2_f64:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    addi sp, sp, -16
+; RV64IFD-NEXT:    sd ra, 8(sp)
+; RV64IFD-NEXT:    call log2
+; RV64IFD-NEXT:    ld ra, 8(sp)
+; RV64IFD-NEXT:    addi sp, sp, 16
+; RV64IFD-NEXT:    ret
   %1 = call double @llvm.log2.f64(double %a)
   ret double %1
 }
@@ -218,6 +330,15 @@ define double @fma_f64(double %a, double %b, double %c) nounwind {
 ; RV32IFD-NEXT:    lw a1, 12(sp)
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: fma_f64:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    fmv.d.x ft0, a2
+; RV64IFD-NEXT:    fmv.d.x ft1, a1
+; RV64IFD-NEXT:    fmv.d.x ft2, a0
+; RV64IFD-NEXT:    fmadd.d ft0, ft2, ft1, ft0
+; RV64IFD-NEXT:    fmv.x.d a0, ft0
+; RV64IFD-NEXT:    ret
   %1 = call double @llvm.fma.f64(double %a, double %b, double %c)
   ret double %1
 }
@@ -245,6 +366,16 @@ define double @fmuladd_f64(double %a, double %b, double %c) nounwind {
 ; RV32IFD-NEXT:    lw a1, 12(sp)
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: fmuladd_f64:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    fmv.d.x ft0, a1
+; RV64IFD-NEXT:    fmv.d.x ft1, a0
+; RV64IFD-NEXT:    fmul.d ft0, ft1, ft0
+; RV64IFD-NEXT:    fmv.d.x ft1, a2
+; RV64IFD-NEXT:    fadd.d ft0, ft0, ft1
+; RV64IFD-NEXT:    fmv.x.d a0, ft0
+; RV64IFD-NEXT:    ret
   %1 = call double @llvm.fmuladd.f64(double %a, double %b, double %c)
   ret double %1
 }
@@ -254,16 +385,18 @@ declare double @llvm.fabs.f64(double)
 define double @fabs_f64(double %a) nounwind {
 ; RV32IFD-LABEL: fabs_f64:
 ; RV32IFD:       # %bb.0:
-; RV32IFD-NEXT:    addi sp, sp, -16
-; RV32IFD-NEXT:    sw a0, 8(sp)
-; RV32IFD-NEXT:    sw a1, 12(sp)
-; RV32IFD-NEXT:    fld ft0, 8(sp)
-; RV32IFD-NEXT:    fabs.d ft0, ft0
-; RV32IFD-NEXT:    fsd ft0, 8(sp)
-; RV32IFD-NEXT:    lw a0, 8(sp)
-; RV32IFD-NEXT:    lw a1, 12(sp)
-; RV32IFD-NEXT:    addi sp, sp, 16
+; RV32IFD-NEXT:    lui a2, 524288
+; RV32IFD-NEXT:    addi a2, a2, -1
+; RV32IFD-NEXT:    and a1, a1, a2
 ; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: fabs_f64:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    addi a1, zero, -1
+; RV64IFD-NEXT:    slli a1, a1, 63
+; RV64IFD-NEXT:    addi a1, a1, -1
+; RV64IFD-NEXT:    and a0, a0, a1
+; RV64IFD-NEXT:    ret
   %1 = call double @llvm.fabs.f64(double %a)
   ret double %1
 }
@@ -286,6 +419,14 @@ define double @minnum_f64(double %a, double %b) nounwind {
 ; RV32IFD-NEXT:    lw a1, 12(sp)
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: minnum_f64:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    fmv.d.x ft0, a1
+; RV64IFD-NEXT:    fmv.d.x ft1, a0
+; RV64IFD-NEXT:    fmin.d ft0, ft1, ft0
+; RV64IFD-NEXT:    fmv.x.d a0, ft0
+; RV64IFD-NEXT:    ret
   %1 = call double @llvm.minnum.f64(double %a, double %b)
   ret double %1
 }
@@ -308,6 +449,14 @@ define double @maxnum_f64(double %a, double %b) nounwind {
 ; RV32IFD-NEXT:    lw a1, 12(sp)
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: maxnum_f64:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    fmv.d.x ft0, a1
+; RV64IFD-NEXT:    fmv.d.x ft1, a0
+; RV64IFD-NEXT:    fmax.d ft0, ft1, ft0
+; RV64IFD-NEXT:    fmv.x.d a0, ft0
+; RV64IFD-NEXT:    ret
   %1 = call double @llvm.maxnum.f64(double %a, double %b)
   ret double %1
 }
@@ -347,6 +496,14 @@ define double @copysign_f64(double %a, double %b) nounwind {
 ; RV32IFD-NEXT:    lw a1, 12(sp)
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: copysign_f64:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    fmv.d.x ft0, a1
+; RV64IFD-NEXT:    fmv.d.x ft1, a0
+; RV64IFD-NEXT:    fsgnj.d ft0, ft1, ft0
+; RV64IFD-NEXT:    fmv.x.d a0, ft0
+; RV64IFD-NEXT:    ret
   %1 = call double @llvm.copysign.f64(double %a, double %b)
   ret double %1
 }
@@ -362,6 +519,15 @@ define double @floor_f64(double %a) nounwind {
 ; RV32IFD-NEXT:    lw ra, 12(sp)
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: floor_f64:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    addi sp, sp, -16
+; RV64IFD-NEXT:    sd ra, 8(sp)
+; RV64IFD-NEXT:    call floor
+; RV64IFD-NEXT:    ld ra, 8(sp)
+; RV64IFD-NEXT:    addi sp, sp, 16
+; RV64IFD-NEXT:    ret
   %1 = call double @llvm.floor.f64(double %a)
   ret double %1
 }
@@ -377,6 +543,15 @@ define double @ceil_f64(double %a) nounwind {
 ; RV32IFD-NEXT:    lw ra, 12(sp)
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: ceil_f64:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    addi sp, sp, -16
+; RV64IFD-NEXT:    sd ra, 8(sp)
+; RV64IFD-NEXT:    call ceil
+; RV64IFD-NEXT:    ld ra, 8(sp)
+; RV64IFD-NEXT:    addi sp, sp, 16
+; RV64IFD-NEXT:    ret
   %1 = call double @llvm.ceil.f64(double %a)
   ret double %1
 }
@@ -392,6 +567,15 @@ define double @trunc_f64(double %a) nounwind {
 ; RV32IFD-NEXT:    lw ra, 12(sp)
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: trunc_f64:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    addi sp, sp, -16
+; RV64IFD-NEXT:    sd ra, 8(sp)
+; RV64IFD-NEXT:    call trunc
+; RV64IFD-NEXT:    ld ra, 8(sp)
+; RV64IFD-NEXT:    addi sp, sp, 16
+; RV64IFD-NEXT:    ret
   %1 = call double @llvm.trunc.f64(double %a)
   ret double %1
 }
@@ -407,6 +591,15 @@ define double @rint_f64(double %a) nounwind {
 ; RV32IFD-NEXT:    lw ra, 12(sp)
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: rint_f64:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    addi sp, sp, -16
+; RV64IFD-NEXT:    sd ra, 8(sp)
+; RV64IFD-NEXT:    call rint
+; RV64IFD-NEXT:    ld ra, 8(sp)
+; RV64IFD-NEXT:    addi sp, sp, 16
+; RV64IFD-NEXT:    ret
   %1 = call double @llvm.rint.f64(double %a)
   ret double %1
 }
@@ -422,6 +615,15 @@ define double @nearbyint_f64(double %a) nounwind {
 ; RV32IFD-NEXT:    lw ra, 12(sp)
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: nearbyint_f64:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    addi sp, sp, -16
+; RV64IFD-NEXT:    sd ra, 8(sp)
+; RV64IFD-NEXT:    call nearbyint
+; RV64IFD-NEXT:    ld ra, 8(sp)
+; RV64IFD-NEXT:    addi sp, sp, 16
+; RV64IFD-NEXT:    ret
   %1 = call double @llvm.nearbyint.f64(double %a)
   ret double %1
 }
@@ -437,6 +639,15 @@ define double @round_f64(double %a) nounwind {
 ; RV32IFD-NEXT:    lw ra, 12(sp)
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: round_f64:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    addi sp, sp, -16
+; RV64IFD-NEXT:    sd ra, 8(sp)
+; RV64IFD-NEXT:    call round
+; RV64IFD-NEXT:    ld ra, 8(sp)
+; RV64IFD-NEXT:    addi sp, sp, 16
+; RV64IFD-NEXT:    ret
   %1 = call double @llvm.round.f64(double %a)
   ret double %1
 }
diff --git a/test/CodeGen/RISCV/double-mem.ll b/test/CodeGen/RISCV/double-mem.ll
index 5b20447ed73ea2ea78d2d48b760d6aa5c9ae7883..a7a93de8a50e03af9219b1c55a2c5cae87c3edaa 100644
--- a/test/CodeGen/RISCV/double-mem.ll
+++ b/test/CodeGen/RISCV/double-mem.ll
@@ -1,6 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=riscv32 -mattr=+d -verify-machineinstrs < %s \
 ; RUN:   | FileCheck -check-prefix=RV32IFD %s
+; RUN: llc -mtriple=riscv64 -mattr=+d -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=RV64IFD %s
 
 define double @fld(double *%a) nounwind {
 ; RV32IFD-LABEL: fld:
@@ -14,6 +16,14 @@ define double @fld(double *%a) nounwind {
 ; RV32IFD-NEXT:    lw a1, 12(sp)
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: fld:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    fld ft0, 24(a0)
+; RV64IFD-NEXT:    fld ft1, 0(a0)
+; RV64IFD-NEXT:    fadd.d ft0, ft1, ft0
+; RV64IFD-NEXT:    fmv.x.d a0, ft0
+; RV64IFD-NEXT:    ret
   %1 = load double, double* %a
   %2 = getelementptr double, double* %a, i32 3
   %3 = load double, double* %2
@@ -38,6 +48,15 @@ define void @fsd(double *%a, double %b, double %c) nounwind {
 ; RV32IFD-NEXT:    fsd ft0, 0(a0)
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: fsd:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    fmv.d.x ft0, a2
+; RV64IFD-NEXT:    fmv.d.x ft1, a1
+; RV64IFD-NEXT:    fadd.d ft0, ft1, ft0
+; RV64IFD-NEXT:    fsd ft0, 64(a0)
+; RV64IFD-NEXT:    fsd ft0, 0(a0)
+; RV64IFD-NEXT:    ret
 ; Use %b and %c in an FP op to ensure floating point registers are used, even
 ; for the soft float ABI
   %1 = fadd double %b, %c
@@ -72,6 +91,20 @@ define double @fld_fsd_global(double %a, double %b) nounwind {
 ; RV32IFD-NEXT:    lw a1, 12(sp)
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: fld_fsd_global:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    fmv.d.x ft0, a1
+; RV64IFD-NEXT:    fmv.d.x ft1, a0
+; RV64IFD-NEXT:    fadd.d ft0, ft1, ft0
+; RV64IFD-NEXT:    lui a0, %hi(G)
+; RV64IFD-NEXT:    fld ft1, %lo(G)(a0)
+; RV64IFD-NEXT:    fsd ft0, %lo(G)(a0)
+; RV64IFD-NEXT:    addi a0, a0, %lo(G)
+; RV64IFD-NEXT:    fld ft1, 72(a0)
+; RV64IFD-NEXT:    fsd ft0, 72(a0)
+; RV64IFD-NEXT:    fmv.x.d a0, ft0
+; RV64IFD-NEXT:    ret
 ; Use %a and %b in an FP op to ensure floating point registers are used, even
 ; for the soft float ABI
   %1 = fadd double %a, %b
@@ -100,6 +133,18 @@ define double @fld_fsd_constant(double %a) nounwind {
 ; RV32IFD-NEXT:    lw a1, 12(sp)
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: fld_fsd_constant:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    fmv.d.x ft0, a0
+; RV64IFD-NEXT:    lui a0, 56
+; RV64IFD-NEXT:    addiw a0, a0, -1353
+; RV64IFD-NEXT:    slli a0, a0, 14
+; RV64IFD-NEXT:    fld ft1, -273(a0)
+; RV64IFD-NEXT:    fadd.d ft0, ft0, ft1
+; RV64IFD-NEXT:    fsd ft0, -273(a0)
+; RV64IFD-NEXT:    fmv.x.d a0, ft0
+; RV64IFD-NEXT:    ret
   %1 = inttoptr i32 3735928559 to double*
   %2 = load volatile double, double* %1
   %3 = fadd double %a, %2
@@ -133,6 +178,23 @@ define double @fld_stack(double %a) nounwind {
 ; RV32IFD-NEXT:    lw ra, 28(sp)
 ; RV32IFD-NEXT:    addi sp, sp, 32
 ; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: fld_stack:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    addi sp, sp, -32
+; RV64IFD-NEXT:    sd ra, 24(sp)
+; RV64IFD-NEXT:    sd s1, 16(sp)
+; RV64IFD-NEXT:    mv s1, a0
+; RV64IFD-NEXT:    addi a0, sp, 8
+; RV64IFD-NEXT:    call notdead
+; RV64IFD-NEXT:    fmv.d.x ft0, s1
+; RV64IFD-NEXT:    fld ft1, 8(sp)
+; RV64IFD-NEXT:    fadd.d ft0, ft1, ft0
+; RV64IFD-NEXT:    fmv.x.d a0, ft0
+; RV64IFD-NEXT:    ld s1, 16(sp)
+; RV64IFD-NEXT:    ld ra, 24(sp)
+; RV64IFD-NEXT:    addi sp, sp, 32
+; RV64IFD-NEXT:    ret
   %1 = alloca double, align 8
   %2 = bitcast double* %1 to i8*
   call void @notdead(i8* %2)
@@ -159,6 +221,20 @@ define void @fsd_stack(double %a, double %b) nounwind {
 ; RV32IFD-NEXT:    lw ra, 28(sp)
 ; RV32IFD-NEXT:    addi sp, sp, 32
 ; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: fsd_stack:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    addi sp, sp, -16
+; RV64IFD-NEXT:    sd ra, 8(sp)
+; RV64IFD-NEXT:    fmv.d.x ft0, a1
+; RV64IFD-NEXT:    fmv.d.x ft1, a0
+; RV64IFD-NEXT:    fadd.d ft0, ft1, ft0
+; RV64IFD-NEXT:    fsd ft0, 0(sp)
+; RV64IFD-NEXT:    mv a0, sp
+; RV64IFD-NEXT:    call notdead
+; RV64IFD-NEXT:    ld ra, 8(sp)
+; RV64IFD-NEXT:    addi sp, sp, 16
+; RV64IFD-NEXT:    ret
   %1 = fadd double %a, %b ; force store from FPR64
   %2 = alloca double, align 8
   store double %1, double* %2
@@ -179,6 +255,13 @@ define void @fsd_trunc(float* %a, double %b) nounwind noinline optnone {
 ; RV32IFD-NEXT:    fsw ft0, 0(a0)
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: fsd_trunc:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    fmv.d.x ft0, a1
+; RV64IFD-NEXT:    fcvt.s.d ft0, ft0
+; RV64IFD-NEXT:    fsw ft0, 0(a0)
+; RV64IFD-NEXT:    ret
   %1 = fptrunc double %b to float
   store float %1, float* %a, align 4
   ret void
diff --git a/test/CodeGen/RISCV/double-select-fcmp.ll b/test/CodeGen/RISCV/double-select-fcmp.ll
index fa10ee257245255e23d54607204f6c20b6ae69d5..4503c2365c816cd37d3440bc649eab89a9e79c31 100644
--- a/test/CodeGen/RISCV/double-select-fcmp.ll
+++ b/test/CodeGen/RISCV/double-select-fcmp.ll
@@ -1,6 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=riscv32 -mattr=+d -verify-machineinstrs < %s \
 ; RUN:   | FileCheck -check-prefix=RV32IFD %s
+; RUN: llc -mtriple=riscv64 -mattr=+d -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=RV64IFD %s
 
 define double @select_fcmp_false(double %a, double %b) nounwind {
 ; RV32IFD-LABEL: select_fcmp_false:
@@ -8,6 +10,11 @@ define double @select_fcmp_false(double %a, double %b) nounwind {
 ; RV32IFD-NEXT:    mv a1, a3
 ; RV32IFD-NEXT:    mv a0, a2
 ; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: select_fcmp_false:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    mv a0, a1
+; RV64IFD-NEXT:    ret
   %1 = fcmp false double %a, %b
   %2 = select i1 %1, double %a, double %b
   ret double %2
@@ -33,6 +40,18 @@ define double @select_fcmp_oeq(double %a, double %b) nounwind {
 ; RV32IFD-NEXT:    lw a1, 12(sp)
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: select_fcmp_oeq:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    fmv.d.x ft1, a1
+; RV64IFD-NEXT:    fmv.d.x ft0, a0
+; RV64IFD-NEXT:    feq.d a0, ft0, ft1
+; RV64IFD-NEXT:    bnez a0, .LBB1_2
+; RV64IFD-NEXT:  # %bb.1:
+; RV64IFD-NEXT:    fmv.d ft0, ft1
+; RV64IFD-NEXT:  .LBB1_2:
+; RV64IFD-NEXT:    fmv.x.d a0, ft0
+; RV64IFD-NEXT:    ret
   %1 = fcmp oeq double %a, %b
   %2 = select i1 %1, double %a, double %b
   ret double %2
@@ -58,6 +77,18 @@ define double @select_fcmp_ogt(double %a, double %b) nounwind {
 ; RV32IFD-NEXT:    lw a1, 12(sp)
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: select_fcmp_ogt:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    fmv.d.x ft0, a0
+; RV64IFD-NEXT:    fmv.d.x ft1, a1
+; RV64IFD-NEXT:    flt.d a0, ft1, ft0
+; RV64IFD-NEXT:    bnez a0, .LBB2_2
+; RV64IFD-NEXT:  # %bb.1:
+; RV64IFD-NEXT:    fmv.d ft0, ft1
+; RV64IFD-NEXT:  .LBB2_2:
+; RV64IFD-NEXT:    fmv.x.d a0, ft0
+; RV64IFD-NEXT:    ret
   %1 = fcmp ogt double %a, %b
   %2 = select i1 %1, double %a, double %b
   ret double %2
@@ -83,6 +114,18 @@ define double @select_fcmp_oge(double %a, double %b) nounwind {
 ; RV32IFD-NEXT:    lw a1, 12(sp)
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: select_fcmp_oge:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    fmv.d.x ft0, a0
+; RV64IFD-NEXT:    fmv.d.x ft1, a1
+; RV64IFD-NEXT:    fle.d a0, ft1, ft0
+; RV64IFD-NEXT:    bnez a0, .LBB3_2
+; RV64IFD-NEXT:  # %bb.1:
+; RV64IFD-NEXT:    fmv.d ft0, ft1
+; RV64IFD-NEXT:  .LBB3_2:
+; RV64IFD-NEXT:    fmv.x.d a0, ft0
+; RV64IFD-NEXT:    ret
   %1 = fcmp oge double %a, %b
   %2 = select i1 %1, double %a, double %b
   ret double %2
@@ -108,6 +151,18 @@ define double @select_fcmp_olt(double %a, double %b) nounwind {
 ; RV32IFD-NEXT:    lw a1, 12(sp)
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: select_fcmp_olt:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    fmv.d.x ft1, a1
+; RV64IFD-NEXT:    fmv.d.x ft0, a0
+; RV64IFD-NEXT:    flt.d a0, ft0, ft1
+; RV64IFD-NEXT:    bnez a0, .LBB4_2
+; RV64IFD-NEXT:  # %bb.1:
+; RV64IFD-NEXT:    fmv.d ft0, ft1
+; RV64IFD-NEXT:  .LBB4_2:
+; RV64IFD-NEXT:    fmv.x.d a0, ft0
+; RV64IFD-NEXT:    ret
   %1 = fcmp olt double %a, %b
   %2 = select i1 %1, double %a, double %b
   ret double %2
@@ -133,6 +188,18 @@ define double @select_fcmp_ole(double %a, double %b) nounwind {
 ; RV32IFD-NEXT:    lw a1, 12(sp)
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: select_fcmp_ole:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    fmv.d.x ft1, a1
+; RV64IFD-NEXT:    fmv.d.x ft0, a0
+; RV64IFD-NEXT:    fle.d a0, ft0, ft1
+; RV64IFD-NEXT:    bnez a0, .LBB5_2
+; RV64IFD-NEXT:  # %bb.1:
+; RV64IFD-NEXT:    fmv.d ft0, ft1
+; RV64IFD-NEXT:  .LBB5_2:
+; RV64IFD-NEXT:    fmv.x.d a0, ft0
+; RV64IFD-NEXT:    ret
   %1 = fcmp ole double %a, %b
   %2 = select i1 %1, double %a, double %b
   ret double %2
@@ -166,6 +233,25 @@ define double @select_fcmp_one(double %a, double %b) nounwind {
 ; RV32IFD-NEXT:    lw a1, 12(sp)
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: select_fcmp_one:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    fmv.d.x ft0, a0
+; RV64IFD-NEXT:    fmv.d.x ft1, a1
+; RV64IFD-NEXT:    feq.d a0, ft1, ft1
+; RV64IFD-NEXT:    feq.d a1, ft0, ft0
+; RV64IFD-NEXT:    and a0, a1, a0
+; RV64IFD-NEXT:    feq.d a1, ft0, ft1
+; RV64IFD-NEXT:    not a1, a1
+; RV64IFD-NEXT:    seqz a0, a0
+; RV64IFD-NEXT:    xori a0, a0, 1
+; RV64IFD-NEXT:    and a0, a1, a0
+; RV64IFD-NEXT:    bnez a0, .LBB6_2
+; RV64IFD-NEXT:  # %bb.1:
+; RV64IFD-NEXT:    fmv.d ft0, ft1
+; RV64IFD-NEXT:  .LBB6_2:
+; RV64IFD-NEXT:    fmv.x.d a0, ft0
+; RV64IFD-NEXT:    ret
   %1 = fcmp one double %a, %b
   %2 = select i1 %1, double %a, double %b
   ret double %2
@@ -195,6 +281,22 @@ define double @select_fcmp_ord(double %a, double %b) nounwind {
 ; RV32IFD-NEXT:    lw a1, 12(sp)
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: select_fcmp_ord:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    fmv.d.x ft0, a0
+; RV64IFD-NEXT:    fmv.d.x ft1, a1
+; RV64IFD-NEXT:    feq.d a0, ft1, ft1
+; RV64IFD-NEXT:    feq.d a1, ft0, ft0
+; RV64IFD-NEXT:    and a0, a1, a0
+; RV64IFD-NEXT:    seqz a0, a0
+; RV64IFD-NEXT:    xori a0, a0, 1
+; RV64IFD-NEXT:    bnez a0, .LBB7_2
+; RV64IFD-NEXT:  # %bb.1:
+; RV64IFD-NEXT:    fmv.d ft0, ft1
+; RV64IFD-NEXT:  .LBB7_2:
+; RV64IFD-NEXT:    fmv.x.d a0, ft0
+; RV64IFD-NEXT:    ret
   %1 = fcmp ord double %a, %b
   %2 = select i1 %1, double %a, double %b
   ret double %2
@@ -225,6 +327,23 @@ define double @select_fcmp_ueq(double %a, double %b) nounwind {
 ; RV32IFD-NEXT:    lw a1, 12(sp)
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: select_fcmp_ueq:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    fmv.d.x ft0, a0
+; RV64IFD-NEXT:    fmv.d.x ft1, a1
+; RV64IFD-NEXT:    feq.d a0, ft1, ft1
+; RV64IFD-NEXT:    feq.d a1, ft0, ft0
+; RV64IFD-NEXT:    and a0, a1, a0
+; RV64IFD-NEXT:    seqz a0, a0
+; RV64IFD-NEXT:    feq.d a1, ft0, ft1
+; RV64IFD-NEXT:    or a0, a1, a0
+; RV64IFD-NEXT:    bnez a0, .LBB8_2
+; RV64IFD-NEXT:  # %bb.1:
+; RV64IFD-NEXT:    fmv.d ft0, ft1
+; RV64IFD-NEXT:  .LBB8_2:
+; RV64IFD-NEXT:    fmv.x.d a0, ft0
+; RV64IFD-NEXT:    ret
   %1 = fcmp ueq double %a, %b
   %2 = select i1 %1, double %a, double %b
   ret double %2
@@ -251,6 +370,19 @@ define double @select_fcmp_ugt(double %a, double %b) nounwind {
 ; RV32IFD-NEXT:    lw a1, 12(sp)
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: select_fcmp_ugt:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    fmv.d.x ft1, a1
+; RV64IFD-NEXT:    fmv.d.x ft0, a0
+; RV64IFD-NEXT:    fle.d a0, ft0, ft1
+; RV64IFD-NEXT:    xori a0, a0, 1
+; RV64IFD-NEXT:    bnez a0, .LBB9_2
+; RV64IFD-NEXT:  # %bb.1:
+; RV64IFD-NEXT:    fmv.d ft0, ft1
+; RV64IFD-NEXT:  .LBB9_2:
+; RV64IFD-NEXT:    fmv.x.d a0, ft0
+; RV64IFD-NEXT:    ret
   %1 = fcmp ugt double %a, %b
   %2 = select i1 %1, double %a, double %b
   ret double %2
@@ -277,6 +409,19 @@ define double @select_fcmp_uge(double %a, double %b) nounwind {
 ; RV32IFD-NEXT:    lw a1, 12(sp)
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: select_fcmp_uge:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    fmv.d.x ft1, a1
+; RV64IFD-NEXT:    fmv.d.x ft0, a0
+; RV64IFD-NEXT:    flt.d a0, ft0, ft1
+; RV64IFD-NEXT:    xori a0, a0, 1
+; RV64IFD-NEXT:    bnez a0, .LBB10_2
+; RV64IFD-NEXT:  # %bb.1:
+; RV64IFD-NEXT:    fmv.d ft0, ft1
+; RV64IFD-NEXT:  .LBB10_2:
+; RV64IFD-NEXT:    fmv.x.d a0, ft0
+; RV64IFD-NEXT:    ret
   %1 = fcmp uge double %a, %b
   %2 = select i1 %1, double %a, double %b
   ret double %2
@@ -303,6 +448,19 @@ define double @select_fcmp_ult(double %a, double %b) nounwind {
 ; RV32IFD-NEXT:    lw a1, 12(sp)
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: select_fcmp_ult:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    fmv.d.x ft0, a0
+; RV64IFD-NEXT:    fmv.d.x ft1, a1
+; RV64IFD-NEXT:    fle.d a0, ft1, ft0
+; RV64IFD-NEXT:    xori a0, a0, 1
+; RV64IFD-NEXT:    bnez a0, .LBB11_2
+; RV64IFD-NEXT:  # %bb.1:
+; RV64IFD-NEXT:    fmv.d ft0, ft1
+; RV64IFD-NEXT:  .LBB11_2:
+; RV64IFD-NEXT:    fmv.x.d a0, ft0
+; RV64IFD-NEXT:    ret
   %1 = fcmp ult double %a, %b
   %2 = select i1 %1, double %a, double %b
   ret double %2
@@ -329,6 +487,19 @@ define double @select_fcmp_ule(double %a, double %b) nounwind {
 ; RV32IFD-NEXT:    lw a1, 12(sp)
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: select_fcmp_ule:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    fmv.d.x ft0, a0
+; RV64IFD-NEXT:    fmv.d.x ft1, a1
+; RV64IFD-NEXT:    flt.d a0, ft1, ft0
+; RV64IFD-NEXT:    xori a0, a0, 1
+; RV64IFD-NEXT:    bnez a0, .LBB12_2
+; RV64IFD-NEXT:  # %bb.1:
+; RV64IFD-NEXT:    fmv.d ft0, ft1
+; RV64IFD-NEXT:  .LBB12_2:
+; RV64IFD-NEXT:    fmv.x.d a0, ft0
+; RV64IFD-NEXT:    ret
   %1 = fcmp ule double %a, %b
   %2 = select i1 %1, double %a, double %b
   ret double %2
@@ -355,6 +526,19 @@ define double @select_fcmp_une(double %a, double %b) nounwind {
 ; RV32IFD-NEXT:    lw a1, 12(sp)
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: select_fcmp_une:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    fmv.d.x ft1, a1
+; RV64IFD-NEXT:    fmv.d.x ft0, a0
+; RV64IFD-NEXT:    feq.d a0, ft0, ft1
+; RV64IFD-NEXT:    xori a0, a0, 1
+; RV64IFD-NEXT:    bnez a0, .LBB13_2
+; RV64IFD-NEXT:  # %bb.1:
+; RV64IFD-NEXT:    fmv.d ft0, ft1
+; RV64IFD-NEXT:  .LBB13_2:
+; RV64IFD-NEXT:    fmv.x.d a0, ft0
+; RV64IFD-NEXT:    ret
   %1 = fcmp une double %a, %b
   %2 = select i1 %1, double %a, double %b
   ret double %2
@@ -384,6 +568,21 @@ define double @select_fcmp_uno(double %a, double %b) nounwind {
 ; RV32IFD-NEXT:    lw a1, 12(sp)
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: select_fcmp_uno:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    fmv.d.x ft0, a0
+; RV64IFD-NEXT:    fmv.d.x ft1, a1
+; RV64IFD-NEXT:    feq.d a0, ft1, ft1
+; RV64IFD-NEXT:    feq.d a1, ft0, ft0
+; RV64IFD-NEXT:    and a0, a1, a0
+; RV64IFD-NEXT:    seqz a0, a0
+; RV64IFD-NEXT:    bnez a0, .LBB14_2
+; RV64IFD-NEXT:  # %bb.1:
+; RV64IFD-NEXT:    fmv.d ft0, ft1
+; RV64IFD-NEXT:  .LBB14_2:
+; RV64IFD-NEXT:    fmv.x.d a0, ft0
+; RV64IFD-NEXT:    ret
   %1 = fcmp uno double %a, %b
   %2 = select i1 %1, double %a, double %b
   ret double %2
@@ -393,6 +592,10 @@ define double @select_fcmp_true(double %a, double %b) nounwind {
 ; RV32IFD-LABEL: select_fcmp_true:
 ; RV32IFD:       # %bb.0:
 ; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: select_fcmp_true:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    ret
   %1 = fcmp true double %a, %b
   %2 = select i1 %1, double %a, double %b
   ret double %2
@@ -417,6 +620,18 @@ define i32 @i32_select_fcmp_oeq(double %a, double %b, i32 %c, i32 %d) nounwind {
 ; RV32IFD-NEXT:    mv a0, a4
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: i32_select_fcmp_oeq:
+; RV64IFD:       # %bb.0:
+; RV64IFD-NEXT:    fmv.d.x ft0, a1
+; RV64IFD-NEXT:    fmv.d.x ft1, a0
+; RV64IFD-NEXT:    feq.d a0, ft1, ft0
+; RV64IFD-NEXT:    bnez a0, .LBB16_2
+; RV64IFD-NEXT:  # %bb.1:
+; RV64IFD-NEXT:    mv a2, a3
+; RV64IFD-NEXT:  .LBB16_2:
+; RV64IFD-NEXT:    mv a0, a2
+; RV64IFD-NEXT:    ret
   %1 = fcmp oeq double %a, %b
   %2 = select i1 %1, i32 %c, i32 %d
   ret i32 %2
diff --git a/test/CodeGen/RISCV/double-stack-spill-restore.ll b/test/CodeGen/RISCV/double-stack-spill-restore.ll
index b5c0b991e6cc00cbe1af0c4fb11bf7b83de4fb1d..28a9e12dadd0e99ef6eb329aaf7b69d0b2f28fb3 100644
--- a/test/CodeGen/RISCV/double-stack-spill-restore.ll
+++ b/test/CodeGen/RISCV/double-stack-spill-restore.ll
@@ -1,6 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=riscv32 -mattr=+d -verify-machineinstrs < %s \
 ; RUN:   | FileCheck -check-prefix=RV32IFD %s
+; RUN: llc -mtriple=riscv64 -mattr=+d -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=RV64IFD %s
 
 define double @func(double %d, i32 %n) nounwind {
 ; RV32IFD-LABEL: func:
@@ -30,6 +32,28 @@ define double @func(double %d, i32 %n) nounwind {
 ; RV32IFD-NEXT:    lw ra, 28(sp)
 ; RV32IFD-NEXT:    addi sp, sp, 32
 ; RV32IFD-NEXT:    ret
+;
+; RV64IFD-LABEL: func:
+; RV64IFD:       # %bb.0: # %entry
+; RV64IFD-NEXT:    addi sp, sp, -16
+; RV64IFD-NEXT:    sd ra, 8(sp)
+; RV64IFD-NEXT:    fmv.d.x ft0, a0
+; RV64IFD-NEXT:    slli a0, a1, 32
+; RV64IFD-NEXT:    srli a0, a0, 32
+; RV64IFD-NEXT:    beqz a0, .LBB0_2
+; RV64IFD-NEXT:  # %bb.1: # %if.else
+; RV64IFD-NEXT:    addi a1, a1, -1
+; RV64IFD-NEXT:    fmv.x.d a0, ft0
+; RV64IFD-NEXT:    fsd ft0, 0(sp)
+; RV64IFD-NEXT:    call func
+; RV64IFD-NEXT:    fmv.d.x ft0, a0
+; RV64IFD-NEXT:    fld ft1, 0(sp)
+; RV64IFD-NEXT:    fadd.d ft0, ft0, ft1
+; RV64IFD-NEXT:  .LBB0_2: # %return
+; RV64IFD-NEXT:    fmv.x.d a0, ft0
+; RV64IFD-NEXT:    ld ra, 8(sp)
+; RV64IFD-NEXT:    addi sp, sp, 16
+; RV64IFD-NEXT:    ret
 entry:
   %cmp = icmp eq i32 %n, 0
   br i1 %cmp, label %return, label %if.else
diff --git a/test/CodeGen/RISCV/float-arith.ll b/test/CodeGen/RISCV/float-arith.ll
index ab874476541ec69d98e3090d095268296b7e21fe..a668b7e494274708c009773b1db489d9c7fba74e 100644
--- a/test/CodeGen/RISCV/float-arith.ll
+++ b/test/CodeGen/RISCV/float-arith.ll
@@ -1,6 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=riscv32 -mattr=+f -verify-machineinstrs < %s \
 ; RUN:   | FileCheck -check-prefix=RV32IF %s
+; RUN: llc -mtriple=riscv64 -mattr=+f -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=RV64IF %s
 
 ; These tests are each targeted at a particular RISC-V FPU instruction. Most
 ; other files in this folder exercise LLVM IR instructions that don't directly
@@ -14,6 +16,14 @@ define float @fadd_s(float %a, float %b) nounwind {
 ; RV32IF-NEXT:    fadd.s ft0, ft1, ft0
 ; RV32IF-NEXT:    fmv.x.w a0, ft0
 ; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: fadd_s:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fmv.w.x ft0, a1
+; RV64IF-NEXT:    fmv.w.x ft1, a0
+; RV64IF-NEXT:    fadd.s ft0, ft1, ft0
+; RV64IF-NEXT:    fmv.x.w a0, ft0
+; RV64IF-NEXT:    ret
   %1 = fadd float %a, %b
   ret float %1
 }
@@ -26,6 +36,14 @@ define float @fsub_s(float %a, float %b) nounwind {
 ; RV32IF-NEXT:    fsub.s ft0, ft1, ft0
 ; RV32IF-NEXT:    fmv.x.w a0, ft0
 ; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: fsub_s:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fmv.w.x ft0, a1
+; RV64IF-NEXT:    fmv.w.x ft1, a0
+; RV64IF-NEXT:    fsub.s ft0, ft1, ft0
+; RV64IF-NEXT:    fmv.x.w a0, ft0
+; RV64IF-NEXT:    ret
   %1 = fsub float %a, %b
   ret float %1
 }
@@ -38,6 +56,14 @@ define float @fmul_s(float %a, float %b) nounwind {
 ; RV32IF-NEXT:    fmul.s ft0, ft1, ft0
 ; RV32IF-NEXT:    fmv.x.w a0, ft0
 ; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: fmul_s:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fmv.w.x ft0, a1
+; RV64IF-NEXT:    fmv.w.x ft1, a0
+; RV64IF-NEXT:    fmul.s ft0, ft1, ft0
+; RV64IF-NEXT:    fmv.x.w a0, ft0
+; RV64IF-NEXT:    ret
   %1 = fmul float %a, %b
   ret float %1
 }
@@ -50,6 +76,14 @@ define float @fdiv_s(float %a, float %b) nounwind {
 ; RV32IF-NEXT:    fdiv.s ft0, ft1, ft0
 ; RV32IF-NEXT:    fmv.x.w a0, ft0
 ; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: fdiv_s:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fmv.w.x ft0, a1
+; RV64IF-NEXT:    fmv.w.x ft1, a0
+; RV64IF-NEXT:    fdiv.s ft0, ft1, ft0
+; RV64IF-NEXT:    fmv.x.w a0, ft0
+; RV64IF-NEXT:    ret
   %1 = fdiv float %a, %b
   ret float %1
 }
@@ -63,6 +97,13 @@ define float @fsqrt_s(float %a) nounwind {
 ; RV32IF-NEXT:    fsqrt.s ft0, ft0
 ; RV32IF-NEXT:    fmv.x.w a0, ft0
 ; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: fsqrt_s:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fmv.w.x ft0, a0
+; RV64IF-NEXT:    fsqrt.s ft0, ft0
+; RV64IF-NEXT:    fmv.x.w a0, ft0
+; RV64IF-NEXT:    ret
   %1 = call float @llvm.sqrt.f32(float %a)
   ret float %1
 }
@@ -77,54 +118,97 @@ define float @fsgnj_s(float %a, float %b) nounwind {
 ; RV32IF-NEXT:    fsgnj.s ft0, ft1, ft0
 ; RV32IF-NEXT:    fmv.x.w a0, ft0
 ; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: fsgnj_s:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fmv.w.x ft0, a1
+; RV64IF-NEXT:    fmv.w.x ft1, a0
+; RV64IF-NEXT:    fsgnj.s ft0, ft1, ft0
+; RV64IF-NEXT:    fmv.x.w a0, ft0
+; RV64IF-NEXT:    ret
   %1 = call float @llvm.copysign.f32(float %a, float %b)
   ret float %1
 }
 
-define float @fneg_s(float %a) nounwind {
-; TODO: doesn't test the fneg selection pattern because
-; DAGCombiner::visitBITCAST will generate a xor on the incoming integer
-; argument
+; This function performs extra work to ensure that
+; DAGCombiner::visitBITCAST doesn't replace the fneg with an xor.
+define i32 @fneg_s(float %a, float %b) nounwind {
 ; RV32IF-LABEL: fneg_s:
 ; RV32IF:       # %bb.0:
-; RV32IF-NEXT:    lui a1, 524288
-; RV32IF-NEXT:    xor a0, a0, a1
+; RV32IF-NEXT:    fmv.w.x ft0, a0
+; RV32IF-NEXT:    fadd.s ft0, ft0, ft0
+; RV32IF-NEXT:    fneg.s ft1, ft0
+; RV32IF-NEXT:    feq.s a0, ft0, ft1
 ; RV32IF-NEXT:    ret
-  %1 = fsub float -0.0, %a
-  ret float %1
+;
+; RV64IF-LABEL: fneg_s:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fmv.w.x ft0, a0
+; RV64IF-NEXT:    fadd.s ft0, ft0, ft0
+; RV64IF-NEXT:    fneg.s ft1, ft0
+; RV64IF-NEXT:    feq.s a0, ft0, ft1
+; RV64IF-NEXT:    ret
+  %1 = fadd float %a, %a
+  %2 = fneg float %1
+  %3 = fcmp oeq float %1, %2
+  %4 = zext i1 %3 to i32
+  ret i32 %4
 }
 
+; This function performs extra work to ensure that
+; DAGCombiner::visitBITCAST doesn't replace the fneg with an xor.
 define float @fsgnjn_s(float %a, float %b) nounwind {
-; TODO: fsgnjn.s isn't selected because DAGCombiner::visitBITCAST will convert
-; (bitconvert (fneg x)) to a xor
 ; RV32IF-LABEL: fsgnjn_s:
 ; RV32IF:       # %bb.0:
-; RV32IF-NEXT:    lui a2, 524288
-; RV32IF-NEXT:    xor a1, a1, a2
 ; RV32IF-NEXT:    fmv.w.x ft0, a1
 ; RV32IF-NEXT:    fmv.w.x ft1, a0
-; RV32IF-NEXT:    fsgnj.s ft0, ft1, ft0
+; RV32IF-NEXT:    fadd.s ft0, ft1, ft0
+; RV32IF-NEXT:    fsgnjn.s ft0, ft1, ft0
 ; RV32IF-NEXT:    fmv.x.w a0, ft0
 ; RV32IF-NEXT:    ret
-  %1 = fsub float -0.0, %b
-  %2 = call float @llvm.copysign.f32(float %a, float %1)
-  ret float %2
+;
+; RV64IF-LABEL: fsgnjn_s:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fmv.w.x ft0, a1
+; RV64IF-NEXT:    fmv.w.x ft1, a0
+; RV64IF-NEXT:    fadd.s ft0, ft1, ft0
+; RV64IF-NEXT:    fsgnjn.s ft0, ft1, ft0
+; RV64IF-NEXT:    fmv.x.w a0, ft0
+; RV64IF-NEXT:    ret
+  %1 = fadd float %a, %b
+  %2 = fneg float %1
+  %3 = call float @llvm.copysign.f32(float %a, float %2)
+  ret float %3
 }
 
 declare float @llvm.fabs.f32(float)
 
-define float @fabs_s(float %a) nounwind {
-; TODO: doesn't test the fabs selection pattern because
-; DAGCombiner::visitBITCAST will generate an and on the incoming integer
-; argument
+; This function performs extra work to ensure that
+; DAGCombiner::visitBITCAST doesn't replace the fabs with an and.
+define float @fabs_s(float %a, float %b) nounwind {
 ; RV32IF-LABEL: fabs_s:
 ; RV32IF:       # %bb.0:
-; RV32IF-NEXT:    lui a1, 524288
-; RV32IF-NEXT:    addi a1, a1, -1
-; RV32IF-NEXT:    and a0, a0, a1
+; RV32IF-NEXT:    fmv.w.x ft0, a1
+; RV32IF-NEXT:    fmv.w.x ft1, a0
+; RV32IF-NEXT:    fadd.s ft0, ft1, ft0
+; RV32IF-NEXT:    fabs.s ft1, ft0
+; RV32IF-NEXT:    fadd.s ft0, ft1, ft0
+; RV32IF-NEXT:    fmv.x.w a0, ft0
 ; RV32IF-NEXT:    ret
-  %1 = call float @llvm.fabs.f32(float %a)
-  ret float %1
+;
+; RV64IF-LABEL: fabs_s:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fmv.w.x ft0, a1
+; RV64IF-NEXT:    fmv.w.x ft1, a0
+; RV64IF-NEXT:    fadd.s ft0, ft1, ft0
+; RV64IF-NEXT:    fabs.s ft1, ft0
+; RV64IF-NEXT:    fadd.s ft0, ft1, ft0
+; RV64IF-NEXT:    fmv.x.w a0, ft0
+; RV64IF-NEXT:    ret
+  %1 = fadd float %a, %b
+  %2 = call float @llvm.fabs.f32(float %1)
+  %3 = fadd float %2, %1
+  ret float %3
 }
 
 declare float @llvm.minnum.f32(float, float)
@@ -137,6 +221,14 @@ define float @fmin_s(float %a, float %b) nounwind {
 ; RV32IF-NEXT:    fmin.s ft0, ft1, ft0
 ; RV32IF-NEXT:    fmv.x.w a0, ft0
 ; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: fmin_s:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fmv.w.x ft0, a1
+; RV64IF-NEXT:    fmv.w.x ft1, a0
+; RV64IF-NEXT:    fmin.s ft0, ft1, ft0
+; RV64IF-NEXT:    fmv.x.w a0, ft0
+; RV64IF-NEXT:    ret
   %1 = call float @llvm.minnum.f32(float %a, float %b)
   ret float %1
 }
@@ -151,6 +243,14 @@ define float @fmax_s(float %a, float %b) nounwind {
 ; RV32IF-NEXT:    fmax.s ft0, ft1, ft0
 ; RV32IF-NEXT:    fmv.x.w a0, ft0
 ; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: fmax_s:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fmv.w.x ft0, a1
+; RV64IF-NEXT:    fmv.w.x ft1, a0
+; RV64IF-NEXT:    fmax.s ft0, ft1, ft0
+; RV64IF-NEXT:    fmv.x.w a0, ft0
+; RV64IF-NEXT:    ret
   %1 = call float @llvm.maxnum.f32(float %a, float %b)
   ret float %1
 }
@@ -162,6 +262,13 @@ define i32 @feq_s(float %a, float %b) nounwind {
 ; RV32IF-NEXT:    fmv.w.x ft1, a0
 ; RV32IF-NEXT:    feq.s a0, ft1, ft0
 ; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: feq_s:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fmv.w.x ft0, a1
+; RV64IF-NEXT:    fmv.w.x ft1, a0
+; RV64IF-NEXT:    feq.s a0, ft1, ft0
+; RV64IF-NEXT:    ret
   %1 = fcmp oeq float %a, %b
   %2 = zext i1 %1 to i32
   ret i32 %2
@@ -174,6 +281,13 @@ define i32 @flt_s(float %a, float %b) nounwind {
 ; RV32IF-NEXT:    fmv.w.x ft1, a0
 ; RV32IF-NEXT:    flt.s a0, ft1, ft0
 ; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: flt_s:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fmv.w.x ft0, a1
+; RV64IF-NEXT:    fmv.w.x ft1, a0
+; RV64IF-NEXT:    flt.s a0, ft1, ft0
+; RV64IF-NEXT:    ret
   %1 = fcmp olt float %a, %b
   %2 = zext i1 %1 to i32
   ret i32 %2
@@ -186,6 +300,13 @@ define i32 @fle_s(float %a, float %b) nounwind {
 ; RV32IF-NEXT:    fmv.w.x ft1, a0
 ; RV32IF-NEXT:    fle.s a0, ft1, ft0
 ; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: fle_s:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fmv.w.x ft0, a1
+; RV64IF-NEXT:    fmv.w.x ft1, a0
+; RV64IF-NEXT:    fle.s a0, ft1, ft0
+; RV64IF-NEXT:    ret
   %1 = fcmp ole float %a, %b
   %2 = zext i1 %1 to i32
   ret i32 %2
@@ -202,6 +323,15 @@ define float @fmadd_s(float %a, float %b, float %c) nounwind {
 ; RV32IF-NEXT:    fmadd.s ft0, ft2, ft1, ft0
 ; RV32IF-NEXT:    fmv.x.w a0, ft0
 ; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: fmadd_s:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fmv.w.x ft0, a2
+; RV64IF-NEXT:    fmv.w.x ft1, a1
+; RV64IF-NEXT:    fmv.w.x ft2, a0
+; RV64IF-NEXT:    fmadd.s ft0, ft2, ft1, ft0
+; RV64IF-NEXT:    fmv.x.w a0, ft0
+; RV64IF-NEXT:    ret
   %1 = call float @llvm.fma.f32(float %a, float %b, float %c)
   ret float %1
 }
@@ -219,6 +349,19 @@ define float @fmsub_s(float %a, float %b, float %c) nounwind {
 ; RV32IF-NEXT:    fmsub.s ft0, ft2, ft1, ft0
 ; RV32IF-NEXT:    fmv.x.w a0, ft0
 ; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: fmsub_s:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fmv.w.x ft0, a2
+; RV64IF-NEXT:    lui a2, %hi(.LCPI15_0)
+; RV64IF-NEXT:    addi a2, a2, %lo(.LCPI15_0)
+; RV64IF-NEXT:    flw ft1, 0(a2)
+; RV64IF-NEXT:    fadd.s ft0, ft0, ft1
+; RV64IF-NEXT:    fmv.w.x ft1, a1
+; RV64IF-NEXT:    fmv.w.x ft2, a0
+; RV64IF-NEXT:    fmsub.s ft0, ft2, ft1, ft0
+; RV64IF-NEXT:    fmv.x.w a0, ft0
+; RV64IF-NEXT:    ret
   %c_ = fadd float 0.0, %c ; avoid negation using xor
   %negc = fsub float -0.0, %c_
   %1 = call float @llvm.fma.f32(float %a, float %b, float %negc)
@@ -239,6 +382,20 @@ define float @fnmadd_s(float %a, float %b, float %c) nounwind {
 ; RV32IF-NEXT:    fnmadd.s ft0, ft1, ft2, ft0
 ; RV32IF-NEXT:    fmv.x.w a0, ft0
 ; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: fnmadd_s:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fmv.w.x ft0, a2
+; RV64IF-NEXT:    lui a2, %hi(.LCPI16_0)
+; RV64IF-NEXT:    addi a2, a2, %lo(.LCPI16_0)
+; RV64IF-NEXT:    flw ft1, 0(a2)
+; RV64IF-NEXT:    fadd.s ft0, ft0, ft1
+; RV64IF-NEXT:    fmv.w.x ft2, a0
+; RV64IF-NEXT:    fadd.s ft1, ft2, ft1
+; RV64IF-NEXT:    fmv.w.x ft2, a1
+; RV64IF-NEXT:    fnmadd.s ft0, ft1, ft2, ft0
+; RV64IF-NEXT:    fmv.x.w a0, ft0
+; RV64IF-NEXT:    ret
   %a_ = fadd float 0.0, %a
   %c_ = fadd float 0.0, %c
   %nega = fsub float -0.0, %a_
@@ -260,6 +417,19 @@ define float @fnmsub_s(float %a, float %b, float %c) nounwind {
 ; RV32IF-NEXT:    fnmsub.s ft0, ft0, ft2, ft1
 ; RV32IF-NEXT:    fmv.x.w a0, ft0
 ; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: fnmsub_s:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fmv.w.x ft0, a0
+; RV64IF-NEXT:    lui a0, %hi(.LCPI17_0)
+; RV64IF-NEXT:    addi a0, a0, %lo(.LCPI17_0)
+; RV64IF-NEXT:    flw ft1, 0(a0)
+; RV64IF-NEXT:    fadd.s ft0, ft0, ft1
+; RV64IF-NEXT:    fmv.w.x ft1, a2
+; RV64IF-NEXT:    fmv.w.x ft2, a1
+; RV64IF-NEXT:    fnmsub.s ft0, ft0, ft2, ft1
+; RV64IF-NEXT:    fmv.x.w a0, ft0
+; RV64IF-NEXT:    ret
   %a_ = fadd float 0.0, %a
   %nega = fsub float -0.0, %a_
   %1 = call float @llvm.fma.f32(float %nega, float %b, float %c)
diff --git a/test/CodeGen/RISCV/float-bitmanip-dagcombines.ll b/test/CodeGen/RISCV/float-bitmanip-dagcombines.ll
new file mode 100644
index 0000000000000000000000000000000000000000..215fe5839ef7e160d1ee25d16ef8aabb9a1272b9
--- /dev/null
+++ b/test/CodeGen/RISCV/float-bitmanip-dagcombines.ll
@@ -0,0 +1,126 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv32 -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=RV32I %s
+; RUN: llc -mtriple=riscv32 -mattr=+f -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=RV32IF %s
+; RUN: llc -mtriple=riscv64 -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=RV64I %s
+; RUN: llc -mtriple=riscv64 -mattr=+f -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=RV64IF %s
+
+; This file tests cases where simple floating point operations can be
+; profitably handled though bit manipulation if a soft-float ABI is being used
+; (e.g. fneg implemented by XORing the sign bit). This is typically handled in
+; DAGCombiner::visitBITCAST, but this target-independent code may not trigger
+; in cases where we perform custom legalisation (e.g. RV64F).
+
+define float @fneg(float %a) nounwind {
+; RV32I-LABEL: fneg:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    lui a1, 524288
+; RV32I-NEXT:    xor a0, a0, a1
+; RV32I-NEXT:    ret
+;
+; RV32IF-LABEL: fneg:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    lui a1, 524288
+; RV32IF-NEXT:    xor a0, a0, a1
+; RV32IF-NEXT:    ret
+;
+; RV64I-LABEL: fneg:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    lui a1, 524288
+; RV64I-NEXT:    xor a0, a0, a1
+; RV64I-NEXT:    ret
+;
+; RV64IF-LABEL: fneg:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    lui a1, 524288
+; RV64IF-NEXT:    xor a0, a0, a1
+; RV64IF-NEXT:    ret
+  %1 = fneg float %a
+  ret float %1
+}
+
+declare float @llvm.fabs.f32(float)
+
+define float @fabs(float %a) nounwind {
+; RV32I-LABEL: fabs:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    lui a1, 524288
+; RV32I-NEXT:    addi a1, a1, -1
+; RV32I-NEXT:    and a0, a0, a1
+; RV32I-NEXT:    ret
+;
+; RV32IF-LABEL: fabs:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    lui a1, 524288
+; RV32IF-NEXT:    addi a1, a1, -1
+; RV32IF-NEXT:    and a0, a0, a1
+; RV32IF-NEXT:    ret
+;
+; RV64I-LABEL: fabs:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    lui a1, 524288
+; RV64I-NEXT:    addiw a1, a1, -1
+; RV64I-NEXT:    and a0, a0, a1
+; RV64I-NEXT:    ret
+;
+; RV64IF-LABEL: fabs:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    lui a1, 524288
+; RV64IF-NEXT:    addiw a1, a1, -1
+; RV64IF-NEXT:    and a0, a0, a1
+; RV64IF-NEXT:    ret
+  %1 = call float @llvm.fabs.f32(float %a)
+  ret float %1
+}
+
+declare float @llvm.copysign.f32(float, float)
+
+; DAGTypeLegalizer::SoftenFloatRes_FCOPYSIGN will convert to bitwise
+; operations if floating point isn't supported. A combine could be written to
+; do the same even when f32 is legal.
+
+define float @fcopysign_fneg(float %a, float %b) nounwind {
+; RV32I-LABEL: fcopysign_fneg:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    not a1, a1
+; RV32I-NEXT:    lui a2, 524288
+; RV32I-NEXT:    and a1, a1, a2
+; RV32I-NEXT:    addi a2, a2, -1
+; RV32I-NEXT:    and a0, a0, a2
+; RV32I-NEXT:    or a0, a0, a1
+; RV32I-NEXT:    ret
+;
+; RV32IF-LABEL: fcopysign_fneg:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    lui a2, 524288
+; RV32IF-NEXT:    xor a1, a1, a2
+; RV32IF-NEXT:    fmv.w.x ft0, a1
+; RV32IF-NEXT:    fmv.w.x ft1, a0
+; RV32IF-NEXT:    fsgnj.s ft0, ft1, ft0
+; RV32IF-NEXT:    fmv.x.w a0, ft0
+; RV32IF-NEXT:    ret
+;
+; RV64I-LABEL: fcopysign_fneg:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    not a1, a1
+; RV64I-NEXT:    lui a2, 524288
+; RV64I-NEXT:    and a1, a1, a2
+; RV64I-NEXT:    addiw a2, a2, -1
+; RV64I-NEXT:    and a0, a0, a2
+; RV64I-NEXT:    or a0, a0, a1
+; RV64I-NEXT:    ret
+;
+; RV64IF-LABEL: fcopysign_fneg:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fmv.w.x ft0, a1
+; RV64IF-NEXT:    fmv.w.x ft1, a0
+; RV64IF-NEXT:    fsgnjn.s ft0, ft1, ft0
+; RV64IF-NEXT:    fmv.x.w a0, ft0
+; RV64IF-NEXT:    ret
+  %1 = fneg float %b
+  %2 = call float @llvm.copysign.f32(float %a, float %1)
+  ret float %2
+}
diff --git a/test/CodeGen/RISCV/float-br-fcmp.ll b/test/CodeGen/RISCV/float-br-fcmp.ll
index 5804bd0fd1f82f43e1631ef259a418203ee5d618..0e78c91bedf73fcec904ab8fcece08ca7a870dda 100644
--- a/test/CodeGen/RISCV/float-br-fcmp.ll
+++ b/test/CodeGen/RISCV/float-br-fcmp.ll
@@ -1,6 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=riscv32 -mattr=+f -verify-machineinstrs < %s \
 ; RUN:   | FileCheck -check-prefix=RV32IF %s
+; RUN: llc -mtriple=riscv64 -mattr=+f -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=RV64IF %s
 
 declare void @abort()
 declare void @exit(i32)
@@ -19,6 +21,19 @@ define void @br_fcmp_false(float %a, float %b) nounwind {
 ; RV32IF-NEXT:    ret
 ; RV32IF-NEXT:  .LBB0_2: # %if.else
 ; RV32IF-NEXT:    call abort
+;
+; RV64IF-LABEL: br_fcmp_false:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    addi sp, sp, -16
+; RV64IF-NEXT:    sd ra, 8(sp)
+; RV64IF-NEXT:    addi a0, zero, 1
+; RV64IF-NEXT:    bnez a0, .LBB0_2
+; RV64IF-NEXT:  # %bb.1: # %if.then
+; RV64IF-NEXT:    ld ra, 8(sp)
+; RV64IF-NEXT:    addi sp, sp, 16
+; RV64IF-NEXT:    ret
+; RV64IF-NEXT:  .LBB0_2: # %if.else
+; RV64IF-NEXT:    call abort
   %1 = fcmp false float %a, %b
   br i1 %1, label %if.then, label %if.else
 if.then:
@@ -43,6 +58,21 @@ define void @br_fcmp_oeq(float %a, float %b) nounwind {
 ; RV32IF-NEXT:    ret
 ; RV32IF-NEXT:  .LBB1_2: # %if.then
 ; RV32IF-NEXT:    call abort
+;
+; RV64IF-LABEL: br_fcmp_oeq:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    addi sp, sp, -16
+; RV64IF-NEXT:    sd ra, 8(sp)
+; RV64IF-NEXT:    fmv.w.x ft0, a1
+; RV64IF-NEXT:    fmv.w.x ft1, a0
+; RV64IF-NEXT:    feq.s a0, ft1, ft0
+; RV64IF-NEXT:    bnez a0, .LBB1_2
+; RV64IF-NEXT:  # %bb.1: # %if.else
+; RV64IF-NEXT:    ld ra, 8(sp)
+; RV64IF-NEXT:    addi sp, sp, 16
+; RV64IF-NEXT:    ret
+; RV64IF-NEXT:  .LBB1_2: # %if.then
+; RV64IF-NEXT:    call abort
   %1 = fcmp oeq float %a, %b
   br i1 %1, label %if.then, label %if.else
 if.else:
@@ -71,6 +101,22 @@ define void @br_fcmp_oeq_alt(float %a, float %b) nounwind {
 ; RV32IF-NEXT:    ret
 ; RV32IF-NEXT:  .LBB2_2: # %if.then
 ; RV32IF-NEXT:    call abort
+;
+; RV64IF-LABEL: br_fcmp_oeq_alt:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    addi sp, sp, -16
+; RV64IF-NEXT:    sd ra, 8(sp)
+; RV64IF-NEXT:    fmv.w.x ft0, a1
+; RV64IF-NEXT:    fmv.w.x ft1, a0
+; RV64IF-NEXT:    feq.s a0, ft1, ft0
+; RV64IF-NEXT:    xori a0, a0, 1
+; RV64IF-NEXT:    beqz a0, .LBB2_2
+; RV64IF-NEXT:  # %bb.1: # %if.else
+; RV64IF-NEXT:    ld ra, 8(sp)
+; RV64IF-NEXT:    addi sp, sp, 16
+; RV64IF-NEXT:    ret
+; RV64IF-NEXT:  .LBB2_2: # %if.then
+; RV64IF-NEXT:    call abort
   %1 = fcmp oeq float %a, %b
   br i1 %1, label %if.then, label %if.else
 if.then:
@@ -95,6 +141,21 @@ define void @br_fcmp_ogt(float %a, float %b) nounwind {
 ; RV32IF-NEXT:    ret
 ; RV32IF-NEXT:  .LBB3_2: # %if.then
 ; RV32IF-NEXT:    call abort
+;
+; RV64IF-LABEL: br_fcmp_ogt:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    addi sp, sp, -16
+; RV64IF-NEXT:    sd ra, 8(sp)
+; RV64IF-NEXT:    fmv.w.x ft0, a0
+; RV64IF-NEXT:    fmv.w.x ft1, a1
+; RV64IF-NEXT:    flt.s a0, ft1, ft0
+; RV64IF-NEXT:    bnez a0, .LBB3_2
+; RV64IF-NEXT:  # %bb.1: # %if.else
+; RV64IF-NEXT:    ld ra, 8(sp)
+; RV64IF-NEXT:    addi sp, sp, 16
+; RV64IF-NEXT:    ret
+; RV64IF-NEXT:  .LBB3_2: # %if.then
+; RV64IF-NEXT:    call abort
   %1 = fcmp ogt float %a, %b
   br i1 %1, label %if.then, label %if.else
 if.else:
@@ -119,6 +180,21 @@ define void @br_fcmp_oge(float %a, float %b) nounwind {
 ; RV32IF-NEXT:    ret
 ; RV32IF-NEXT:  .LBB4_2: # %if.then
 ; RV32IF-NEXT:    call abort
+;
+; RV64IF-LABEL: br_fcmp_oge:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    addi sp, sp, -16
+; RV64IF-NEXT:    sd ra, 8(sp)
+; RV64IF-NEXT:    fmv.w.x ft0, a0
+; RV64IF-NEXT:    fmv.w.x ft1, a1
+; RV64IF-NEXT:    fle.s a0, ft1, ft0
+; RV64IF-NEXT:    bnez a0, .LBB4_2
+; RV64IF-NEXT:  # %bb.1: # %if.else
+; RV64IF-NEXT:    ld ra, 8(sp)
+; RV64IF-NEXT:    addi sp, sp, 16
+; RV64IF-NEXT:    ret
+; RV64IF-NEXT:  .LBB4_2: # %if.then
+; RV64IF-NEXT:    call abort
   %1 = fcmp oge float %a, %b
   br i1 %1, label %if.then, label %if.else
 if.else:
@@ -143,6 +219,21 @@ define void @br_fcmp_olt(float %a, float %b) nounwind {
 ; RV32IF-NEXT:    ret
 ; RV32IF-NEXT:  .LBB5_2: # %if.then
 ; RV32IF-NEXT:    call abort
+;
+; RV64IF-LABEL: br_fcmp_olt:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    addi sp, sp, -16
+; RV64IF-NEXT:    sd ra, 8(sp)
+; RV64IF-NEXT:    fmv.w.x ft0, a1
+; RV64IF-NEXT:    fmv.w.x ft1, a0
+; RV64IF-NEXT:    flt.s a0, ft1, ft0
+; RV64IF-NEXT:    bnez a0, .LBB5_2
+; RV64IF-NEXT:  # %bb.1: # %if.else
+; RV64IF-NEXT:    ld ra, 8(sp)
+; RV64IF-NEXT:    addi sp, sp, 16
+; RV64IF-NEXT:    ret
+; RV64IF-NEXT:  .LBB5_2: # %if.then
+; RV64IF-NEXT:    call abort
   %1 = fcmp olt float %a, %b
   br i1 %1, label %if.then, label %if.else
 if.else:
@@ -167,6 +258,21 @@ define void @br_fcmp_ole(float %a, float %b) nounwind {
 ; RV32IF-NEXT:    ret
 ; RV32IF-NEXT:  .LBB6_2: # %if.then
 ; RV32IF-NEXT:    call abort
+;
+; RV64IF-LABEL: br_fcmp_ole:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    addi sp, sp, -16
+; RV64IF-NEXT:    sd ra, 8(sp)
+; RV64IF-NEXT:    fmv.w.x ft0, a1
+; RV64IF-NEXT:    fmv.w.x ft1, a0
+; RV64IF-NEXT:    fle.s a0, ft1, ft0
+; RV64IF-NEXT:    bnez a0, .LBB6_2
+; RV64IF-NEXT:  # %bb.1: # %if.else
+; RV64IF-NEXT:    ld ra, 8(sp)
+; RV64IF-NEXT:    addi sp, sp, 16
+; RV64IF-NEXT:    ret
+; RV64IF-NEXT:  .LBB6_2: # %if.then
+; RV64IF-NEXT:    call abort
   %1 = fcmp ole float %a, %b
   br i1 %1, label %if.then, label %if.else
 if.else:
@@ -199,6 +305,28 @@ define void @br_fcmp_one(float %a, float %b) nounwind {
 ; RV32IF-NEXT:    ret
 ; RV32IF-NEXT:  .LBB7_2: # %if.then
 ; RV32IF-NEXT:    call abort
+;
+; RV64IF-LABEL: br_fcmp_one:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    addi sp, sp, -16
+; RV64IF-NEXT:    sd ra, 8(sp)
+; RV64IF-NEXT:    fmv.w.x ft0, a0
+; RV64IF-NEXT:    fmv.w.x ft1, a1
+; RV64IF-NEXT:    feq.s a0, ft1, ft1
+; RV64IF-NEXT:    feq.s a1, ft0, ft0
+; RV64IF-NEXT:    and a0, a1, a0
+; RV64IF-NEXT:    feq.s a1, ft0, ft1
+; RV64IF-NEXT:    not a1, a1
+; RV64IF-NEXT:    seqz a0, a0
+; RV64IF-NEXT:    xori a0, a0, 1
+; RV64IF-NEXT:    and a0, a1, a0
+; RV64IF-NEXT:    bnez a0, .LBB7_2
+; RV64IF-NEXT:  # %bb.1: # %if.else
+; RV64IF-NEXT:    ld ra, 8(sp)
+; RV64IF-NEXT:    addi sp, sp, 16
+; RV64IF-NEXT:    ret
+; RV64IF-NEXT:  .LBB7_2: # %if.then
+; RV64IF-NEXT:    call abort
   %1 = fcmp one float %a, %b
   br i1 %1, label %if.then, label %if.else
 if.else:
@@ -227,6 +355,25 @@ define void @br_fcmp_ord(float %a, float %b) nounwind {
 ; RV32IF-NEXT:    ret
 ; RV32IF-NEXT:  .LBB8_2: # %if.then
 ; RV32IF-NEXT:    call abort
+;
+; RV64IF-LABEL: br_fcmp_ord:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    addi sp, sp, -16
+; RV64IF-NEXT:    sd ra, 8(sp)
+; RV64IF-NEXT:    fmv.w.x ft0, a1
+; RV64IF-NEXT:    feq.s a1, ft0, ft0
+; RV64IF-NEXT:    fmv.w.x ft0, a0
+; RV64IF-NEXT:    feq.s a0, ft0, ft0
+; RV64IF-NEXT:    and a0, a0, a1
+; RV64IF-NEXT:    seqz a0, a0
+; RV64IF-NEXT:    xori a0, a0, 1
+; RV64IF-NEXT:    bnez a0, .LBB8_2
+; RV64IF-NEXT:  # %bb.1: # %if.else
+; RV64IF-NEXT:    ld ra, 8(sp)
+; RV64IF-NEXT:    addi sp, sp, 16
+; RV64IF-NEXT:    ret
+; RV64IF-NEXT:  .LBB8_2: # %if.then
+; RV64IF-NEXT:    call abort
   %1 = fcmp ord float %a, %b
   br i1 %1, label %if.then, label %if.else
 if.else:
@@ -256,6 +403,26 @@ define void @br_fcmp_ueq(float %a, float %b) nounwind {
 ; RV32IF-NEXT:    ret
 ; RV32IF-NEXT:  .LBB9_2: # %if.then
 ; RV32IF-NEXT:    call abort
+;
+; RV64IF-LABEL: br_fcmp_ueq:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    addi sp, sp, -16
+; RV64IF-NEXT:    sd ra, 8(sp)
+; RV64IF-NEXT:    fmv.w.x ft0, a1
+; RV64IF-NEXT:    fmv.w.x ft1, a0
+; RV64IF-NEXT:    feq.s a0, ft1, ft0
+; RV64IF-NEXT:    feq.s a1, ft0, ft0
+; RV64IF-NEXT:    feq.s a2, ft1, ft1
+; RV64IF-NEXT:    and a1, a2, a1
+; RV64IF-NEXT:    seqz a1, a1
+; RV64IF-NEXT:    or a0, a0, a1
+; RV64IF-NEXT:    bnez a0, .LBB9_2
+; RV64IF-NEXT:  # %bb.1: # %if.else
+; RV64IF-NEXT:    ld ra, 8(sp)
+; RV64IF-NEXT:    addi sp, sp, 16
+; RV64IF-NEXT:    ret
+; RV64IF-NEXT:  .LBB9_2: # %if.then
+; RV64IF-NEXT:    call abort
   %1 = fcmp ueq float %a, %b
   br i1 %1, label %if.then, label %if.else
 if.else:
@@ -281,6 +448,22 @@ define void @br_fcmp_ugt(float %a, float %b) nounwind {
 ; RV32IF-NEXT:    ret
 ; RV32IF-NEXT:  .LBB10_2: # %if.then
 ; RV32IF-NEXT:    call abort
+;
+; RV64IF-LABEL: br_fcmp_ugt:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    addi sp, sp, -16
+; RV64IF-NEXT:    sd ra, 8(sp)
+; RV64IF-NEXT:    fmv.w.x ft0, a1
+; RV64IF-NEXT:    fmv.w.x ft1, a0
+; RV64IF-NEXT:    fle.s a0, ft1, ft0
+; RV64IF-NEXT:    xori a0, a0, 1
+; RV64IF-NEXT:    bnez a0, .LBB10_2
+; RV64IF-NEXT:  # %bb.1: # %if.else
+; RV64IF-NEXT:    ld ra, 8(sp)
+; RV64IF-NEXT:    addi sp, sp, 16
+; RV64IF-NEXT:    ret
+; RV64IF-NEXT:  .LBB10_2: # %if.then
+; RV64IF-NEXT:    call abort
   %1 = fcmp ugt float %a, %b
   br i1 %1, label %if.then, label %if.else
 if.else:
@@ -306,6 +489,22 @@ define void @br_fcmp_uge(float %a, float %b) nounwind {
 ; RV32IF-NEXT:    ret
 ; RV32IF-NEXT:  .LBB11_2: # %if.then
 ; RV32IF-NEXT:    call abort
+;
+; RV64IF-LABEL: br_fcmp_uge:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    addi sp, sp, -16
+; RV64IF-NEXT:    sd ra, 8(sp)
+; RV64IF-NEXT:    fmv.w.x ft0, a1
+; RV64IF-NEXT:    fmv.w.x ft1, a0
+; RV64IF-NEXT:    flt.s a0, ft1, ft0
+; RV64IF-NEXT:    xori a0, a0, 1
+; RV64IF-NEXT:    bnez a0, .LBB11_2
+; RV64IF-NEXT:  # %bb.1: # %if.else
+; RV64IF-NEXT:    ld ra, 8(sp)
+; RV64IF-NEXT:    addi sp, sp, 16
+; RV64IF-NEXT:    ret
+; RV64IF-NEXT:  .LBB11_2: # %if.then
+; RV64IF-NEXT:    call abort
   %1 = fcmp uge float %a, %b
   br i1 %1, label %if.then, label %if.else
 if.else:
@@ -331,6 +530,22 @@ define void @br_fcmp_ult(float %a, float %b) nounwind {
 ; RV32IF-NEXT:    ret
 ; RV32IF-NEXT:  .LBB12_2: # %if.then
 ; RV32IF-NEXT:    call abort
+;
+; RV64IF-LABEL: br_fcmp_ult:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    addi sp, sp, -16
+; RV64IF-NEXT:    sd ra, 8(sp)
+; RV64IF-NEXT:    fmv.w.x ft0, a0
+; RV64IF-NEXT:    fmv.w.x ft1, a1
+; RV64IF-NEXT:    fle.s a0, ft1, ft0
+; RV64IF-NEXT:    xori a0, a0, 1
+; RV64IF-NEXT:    bnez a0, .LBB12_2
+; RV64IF-NEXT:  # %bb.1: # %if.else
+; RV64IF-NEXT:    ld ra, 8(sp)
+; RV64IF-NEXT:    addi sp, sp, 16
+; RV64IF-NEXT:    ret
+; RV64IF-NEXT:  .LBB12_2: # %if.then
+; RV64IF-NEXT:    call abort
   %1 = fcmp ult float %a, %b
   br i1 %1, label %if.then, label %if.else
 if.else:
@@ -356,6 +571,22 @@ define void @br_fcmp_ule(float %a, float %b) nounwind {
 ; RV32IF-NEXT:    ret
 ; RV32IF-NEXT:  .LBB13_2: # %if.then
 ; RV32IF-NEXT:    call abort
+;
+; RV64IF-LABEL: br_fcmp_ule:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    addi sp, sp, -16
+; RV64IF-NEXT:    sd ra, 8(sp)
+; RV64IF-NEXT:    fmv.w.x ft0, a0
+; RV64IF-NEXT:    fmv.w.x ft1, a1
+; RV64IF-NEXT:    flt.s a0, ft1, ft0
+; RV64IF-NEXT:    xori a0, a0, 1
+; RV64IF-NEXT:    bnez a0, .LBB13_2
+; RV64IF-NEXT:  # %bb.1: # %if.else
+; RV64IF-NEXT:    ld ra, 8(sp)
+; RV64IF-NEXT:    addi sp, sp, 16
+; RV64IF-NEXT:    ret
+; RV64IF-NEXT:  .LBB13_2: # %if.then
+; RV64IF-NEXT:    call abort
   %1 = fcmp ule float %a, %b
   br i1 %1, label %if.then, label %if.else
 if.else:
@@ -381,6 +612,22 @@ define void @br_fcmp_une(float %a, float %b) nounwind {
 ; RV32IF-NEXT:    ret
 ; RV32IF-NEXT:  .LBB14_2: # %if.then
 ; RV32IF-NEXT:    call abort
+;
+; RV64IF-LABEL: br_fcmp_une:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    addi sp, sp, -16
+; RV64IF-NEXT:    sd ra, 8(sp)
+; RV64IF-NEXT:    fmv.w.x ft0, a1
+; RV64IF-NEXT:    fmv.w.x ft1, a0
+; RV64IF-NEXT:    feq.s a0, ft1, ft0
+; RV64IF-NEXT:    xori a0, a0, 1
+; RV64IF-NEXT:    bnez a0, .LBB14_2
+; RV64IF-NEXT:  # %bb.1: # %if.else
+; RV64IF-NEXT:    ld ra, 8(sp)
+; RV64IF-NEXT:    addi sp, sp, 16
+; RV64IF-NEXT:    ret
+; RV64IF-NEXT:  .LBB14_2: # %if.then
+; RV64IF-NEXT:    call abort
   %1 = fcmp une float %a, %b
   br i1 %1, label %if.then, label %if.else
 if.else:
@@ -409,6 +656,24 @@ define void @br_fcmp_uno(float %a, float %b) nounwind {
 ; RV32IF-NEXT:    ret
 ; RV32IF-NEXT:  .LBB15_2: # %if.then
 ; RV32IF-NEXT:    call abort
+;
+; RV64IF-LABEL: br_fcmp_uno:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    addi sp, sp, -16
+; RV64IF-NEXT:    sd ra, 8(sp)
+; RV64IF-NEXT:    fmv.w.x ft0, a1
+; RV64IF-NEXT:    feq.s a1, ft0, ft0
+; RV64IF-NEXT:    fmv.w.x ft0, a0
+; RV64IF-NEXT:    feq.s a0, ft0, ft0
+; RV64IF-NEXT:    and a0, a0, a1
+; RV64IF-NEXT:    seqz a0, a0
+; RV64IF-NEXT:    bnez a0, .LBB15_2
+; RV64IF-NEXT:  # %bb.1: # %if.else
+; RV64IF-NEXT:    ld ra, 8(sp)
+; RV64IF-NEXT:    addi sp, sp, 16
+; RV64IF-NEXT:    ret
+; RV64IF-NEXT:  .LBB15_2: # %if.then
+; RV64IF-NEXT:    call abort
   %1 = fcmp uno float %a, %b
   br i1 %1, label %if.then, label %if.else
 if.else:
@@ -431,6 +696,19 @@ define void @br_fcmp_true(float %a, float %b) nounwind {
 ; RV32IF-NEXT:    ret
 ; RV32IF-NEXT:  .LBB16_2: # %if.then
 ; RV32IF-NEXT:    call abort
+;
+; RV64IF-LABEL: br_fcmp_true:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    addi sp, sp, -16
+; RV64IF-NEXT:    sd ra, 8(sp)
+; RV64IF-NEXT:    addi a0, zero, 1
+; RV64IF-NEXT:    bnez a0, .LBB16_2
+; RV64IF-NEXT:  # %bb.1: # %if.else
+; RV64IF-NEXT:    ld ra, 8(sp)
+; RV64IF-NEXT:    addi sp, sp, 16
+; RV64IF-NEXT:    ret
+; RV64IF-NEXT:  .LBB16_2: # %if.then
+; RV64IF-NEXT:    call abort
   %1 = fcmp true float %a, %b
   br i1 %1, label %if.then, label %if.else
 if.else:
@@ -471,6 +749,38 @@ define i32 @br_fcmp_store_load_stack_slot(float %a, float %b) nounwind {
 ; RV32IF-NEXT:    ret
 ; RV32IF-NEXT:  .LBB17_3: # %if.then
 ; RV32IF-NEXT:    call abort
+;
+; RV64IF-LABEL: br_fcmp_store_load_stack_slot:
+; RV64IF:       # %bb.0: # %entry
+; RV64IF-NEXT:    addi sp, sp, -32
+; RV64IF-NEXT:    sd ra, 24(sp)
+; RV64IF-NEXT:    sd s1, 16(sp)
+; RV64IF-NEXT:    lui a0, %hi(.LCPI17_0)
+; RV64IF-NEXT:    addi a0, a0, %lo(.LCPI17_0)
+; RV64IF-NEXT:    flw ft0, 0(a0)
+; RV64IF-NEXT:    fsw ft0, 12(sp)
+; RV64IF-NEXT:    fmv.x.w s1, ft0
+; RV64IF-NEXT:    mv a0, s1
+; RV64IF-NEXT:    call dummy
+; RV64IF-NEXT:    fmv.w.x ft0, a0
+; RV64IF-NEXT:    flw ft1, 12(sp)
+; RV64IF-NEXT:    feq.s a0, ft0, ft1
+; RV64IF-NEXT:    beqz a0, .LBB17_3
+; RV64IF-NEXT:  # %bb.1: # %if.end
+; RV64IF-NEXT:    mv a0, s1
+; RV64IF-NEXT:    call dummy
+; RV64IF-NEXT:    fmv.w.x ft0, a0
+; RV64IF-NEXT:    flw ft1, 12(sp)
+; RV64IF-NEXT:    feq.s a0, ft0, ft1
+; RV64IF-NEXT:    beqz a0, .LBB17_3
+; RV64IF-NEXT:  # %bb.2: # %if.end4
+; RV64IF-NEXT:    mv a0, zero
+; RV64IF-NEXT:    ld s1, 16(sp)
+; RV64IF-NEXT:    ld ra, 24(sp)
+; RV64IF-NEXT:    addi sp, sp, 32
+; RV64IF-NEXT:    ret
+; RV64IF-NEXT:  .LBB17_3: # %if.then
+; RV64IF-NEXT:    call abort
 entry:
   %call = call float @dummy(float 0.000000e+00)
   %cmp = fcmp une float %call, 0.000000e+00
diff --git a/test/CodeGen/RISCV/float-convert.ll b/test/CodeGen/RISCV/float-convert.ll
index d6e67cc6a8e1b5970943ed32610f789351759aca..296e9d468fe7e4557c617c13b864aeeca2474f64 100644
--- a/test/CodeGen/RISCV/float-convert.ll
+++ b/test/CodeGen/RISCV/float-convert.ll
@@ -1,23 +1,41 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=riscv32 -mattr=+f -verify-machineinstrs < %s \
 ; RUN:   | FileCheck -check-prefix=RV32IF %s
+; RUN: llc -mtriple=riscv64 -mattr=+f -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=RV64IF %s
 
+; For RV64F, fcvt.l.s is semantically equivalent to fcvt.w.s in this case
+; because fptosi will produce poison if the result doesn't fit into an i32.
 define i32 @fcvt_w_s(float %a) nounwind {
 ; RV32IF-LABEL: fcvt_w_s:
 ; RV32IF:       # %bb.0:
 ; RV32IF-NEXT:    fmv.w.x ft0, a0
 ; RV32IF-NEXT:    fcvt.w.s a0, ft0, rtz
 ; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: fcvt_w_s:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fmv.w.x ft0, a0
+; RV64IF-NEXT:    fcvt.l.s a0, ft0, rtz
+; RV64IF-NEXT:    ret
   %1 = fptosi float %a to i32
   ret i32 %1
 }
 
+; For RV64F, fcvt.lu.s is semantically equivalent to fcvt.wu.s in this case
+; because fptoui will produce poison if the result doesn't fit into an i32.
 define i32 @fcvt_wu_s(float %a) nounwind {
 ; RV32IF-LABEL: fcvt_wu_s:
 ; RV32IF:       # %bb.0:
 ; RV32IF-NEXT:    fmv.w.x ft0, a0
 ; RV32IF-NEXT:    fcvt.wu.s a0, ft0, rtz
 ; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: fcvt_wu_s:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fmv.w.x ft0, a0
+; RV64IF-NEXT:    fcvt.lu.s a0, ft0, rtz
+; RV64IF-NEXT:    ret
   %1 = fptoui float %a to i32
   ret i32 %1
 }
@@ -30,6 +48,14 @@ define i32 @fmv_x_w(float %a, float %b) nounwind {
 ; RV32IF-NEXT:    fadd.s ft0, ft1, ft0
 ; RV32IF-NEXT:    fmv.x.w a0, ft0
 ; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: fmv_x_w:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fmv.w.x ft0, a1
+; RV64IF-NEXT:    fmv.w.x ft1, a0
+; RV64IF-NEXT:    fadd.s ft0, ft1, ft0
+; RV64IF-NEXT:    fmv.x.w a0, ft0
+; RV64IF-NEXT:    ret
 ; Ensure fmv.x.w is generated even for a soft float calling convention
   %1 = fadd float %a, %b
   %2 = bitcast float %1 to i32
@@ -42,6 +68,12 @@ define float @fcvt_s_w(i32 %a) nounwind {
 ; RV32IF-NEXT:    fcvt.s.w ft0, a0
 ; RV32IF-NEXT:    fmv.x.w a0, ft0
 ; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: fcvt_s_w:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fcvt.s.w ft0, a0
+; RV64IF-NEXT:    fmv.x.w a0, ft0
+; RV64IF-NEXT:    ret
   %1 = sitofp i32 %a to float
   ret float %1
 }
@@ -52,6 +84,12 @@ define float @fcvt_s_wu(i32 %a) nounwind {
 ; RV32IF-NEXT:    fcvt.s.wu ft0, a0
 ; RV32IF-NEXT:    fmv.x.w a0, ft0
 ; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: fcvt_s_wu:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fcvt.s.wu ft0, a0
+; RV64IF-NEXT:    fmv.x.w a0, ft0
+; RV64IF-NEXT:    ret
   %1 = uitofp i32 %a to float
   ret float %1
 }
@@ -64,9 +102,93 @@ define float @fmv_w_x(i32 %a, i32 %b) nounwind {
 ; RV32IF-NEXT:    fadd.s ft0, ft1, ft0
 ; RV32IF-NEXT:    fmv.x.w a0, ft0
 ; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: fmv_w_x:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fmv.w.x ft0, a1
+; RV64IF-NEXT:    fmv.w.x ft1, a0
+; RV64IF-NEXT:    fadd.s ft0, ft1, ft0
+; RV64IF-NEXT:    fmv.x.w a0, ft0
+; RV64IF-NEXT:    ret
 ; Ensure fmv.w.x is generated even for a soft float calling convention
   %1 = bitcast i32 %a to float
   %2 = bitcast i32 %b to float
   %3 = fadd float %1, %2
   ret float %3
 }
+
+define i64 @fcvt_l_s(float %a) nounwind {
+; RV32IF-LABEL: fcvt_l_s:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    addi sp, sp, -16
+; RV32IF-NEXT:    sw ra, 12(sp)
+; RV32IF-NEXT:    call __fixsfdi
+; RV32IF-NEXT:    lw ra, 12(sp)
+; RV32IF-NEXT:    addi sp, sp, 16
+; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: fcvt_l_s:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fmv.w.x ft0, a0
+; RV64IF-NEXT:    fcvt.l.s a0, ft0, rtz
+; RV64IF-NEXT:    ret
+  %1 = fptosi float %a to i64
+  ret i64 %1
+}
+
+define i64 @fcvt_lu_s(float %a) nounwind {
+; RV32IF-LABEL: fcvt_lu_s:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    addi sp, sp, -16
+; RV32IF-NEXT:    sw ra, 12(sp)
+; RV32IF-NEXT:    call __fixunssfdi
+; RV32IF-NEXT:    lw ra, 12(sp)
+; RV32IF-NEXT:    addi sp, sp, 16
+; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: fcvt_lu_s:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fmv.w.x ft0, a0
+; RV64IF-NEXT:    fcvt.lu.s a0, ft0, rtz
+; RV64IF-NEXT:    ret
+  %1 = fptoui float %a to i64
+  ret i64 %1
+}
+
+define float @fcvt_s_l(i64 %a) nounwind {
+; RV32IF-LABEL: fcvt_s_l:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    addi sp, sp, -16
+; RV32IF-NEXT:    sw ra, 12(sp)
+; RV32IF-NEXT:    call __floatdisf
+; RV32IF-NEXT:    lw ra, 12(sp)
+; RV32IF-NEXT:    addi sp, sp, 16
+; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: fcvt_s_l:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fcvt.s.l ft0, a0
+; RV64IF-NEXT:    fmv.x.w a0, ft0
+; RV64IF-NEXT:    ret
+  %1 = sitofp i64 %a to float
+  ret float %1
+}
+
+define float @fcvt_s_lu(i64 %a) nounwind {
+; RV32IF-LABEL: fcvt_s_lu:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    addi sp, sp, -16
+; RV32IF-NEXT:    sw ra, 12(sp)
+; RV32IF-NEXT:    call __floatundisf
+; RV32IF-NEXT:    lw ra, 12(sp)
+; RV32IF-NEXT:    addi sp, sp, 16
+; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: fcvt_s_lu:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fcvt.s.lu ft0, a0
+; RV64IF-NEXT:    fmv.x.w a0, ft0
+; RV64IF-NEXT:    ret
+  %1 = uitofp i64 %a to float
+  ret float %1
+}
diff --git a/test/CodeGen/RISCV/float-fcmp.ll b/test/CodeGen/RISCV/float-fcmp.ll
index c8942a90eb76291face486a997db40b690aaf0cf..16f441847911b73a3a543e9df39991009c37448e 100644
--- a/test/CodeGen/RISCV/float-fcmp.ll
+++ b/test/CodeGen/RISCV/float-fcmp.ll
@@ -1,12 +1,19 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=riscv32 -mattr=+f -verify-machineinstrs < %s \
 ; RUN:   | FileCheck -check-prefix=RV32IF %s
+; RUN: llc -mtriple=riscv64 -mattr=+f -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=RV64IF %s
 
 define i32 @fcmp_false(float %a, float %b) nounwind {
 ; RV32IF-LABEL: fcmp_false:
 ; RV32IF:       # %bb.0:
 ; RV32IF-NEXT:    mv a0, zero
 ; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: fcmp_false:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    mv a0, zero
+; RV64IF-NEXT:    ret
   %1 = fcmp false float %a, %b
   %2 = zext i1 %1 to i32
   ret i32 %2
@@ -19,6 +26,13 @@ define i32 @fcmp_oeq(float %a, float %b) nounwind {
 ; RV32IF-NEXT:    fmv.w.x ft1, a0
 ; RV32IF-NEXT:    feq.s a0, ft1, ft0
 ; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: fcmp_oeq:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fmv.w.x ft0, a1
+; RV64IF-NEXT:    fmv.w.x ft1, a0
+; RV64IF-NEXT:    feq.s a0, ft1, ft0
+; RV64IF-NEXT:    ret
   %1 = fcmp oeq float %a, %b
   %2 = zext i1 %1 to i32
   ret i32 %2
@@ -31,6 +45,13 @@ define i32 @fcmp_ogt(float %a, float %b) nounwind {
 ; RV32IF-NEXT:    fmv.w.x ft1, a1
 ; RV32IF-NEXT:    flt.s a0, ft1, ft0
 ; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: fcmp_ogt:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fmv.w.x ft0, a0
+; RV64IF-NEXT:    fmv.w.x ft1, a1
+; RV64IF-NEXT:    flt.s a0, ft1, ft0
+; RV64IF-NEXT:    ret
   %1 = fcmp ogt float %a, %b
   %2 = zext i1 %1 to i32
   ret i32 %2
@@ -43,6 +64,13 @@ define i32 @fcmp_oge(float %a, float %b) nounwind {
 ; RV32IF-NEXT:    fmv.w.x ft1, a1
 ; RV32IF-NEXT:    fle.s a0, ft1, ft0
 ; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: fcmp_oge:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fmv.w.x ft0, a0
+; RV64IF-NEXT:    fmv.w.x ft1, a1
+; RV64IF-NEXT:    fle.s a0, ft1, ft0
+; RV64IF-NEXT:    ret
   %1 = fcmp oge float %a, %b
   %2 = zext i1 %1 to i32
   ret i32 %2
@@ -55,6 +83,13 @@ define i32 @fcmp_olt(float %a, float %b) nounwind {
 ; RV32IF-NEXT:    fmv.w.x ft1, a0
 ; RV32IF-NEXT:    flt.s a0, ft1, ft0
 ; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: fcmp_olt:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fmv.w.x ft0, a1
+; RV64IF-NEXT:    fmv.w.x ft1, a0
+; RV64IF-NEXT:    flt.s a0, ft1, ft0
+; RV64IF-NEXT:    ret
   %1 = fcmp olt float %a, %b
   %2 = zext i1 %1 to i32
   ret i32 %2
@@ -67,6 +102,13 @@ define i32 @fcmp_ole(float %a, float %b) nounwind {
 ; RV32IF-NEXT:    fmv.w.x ft1, a0
 ; RV32IF-NEXT:    fle.s a0, ft1, ft0
 ; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: fcmp_ole:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fmv.w.x ft0, a1
+; RV64IF-NEXT:    fmv.w.x ft1, a0
+; RV64IF-NEXT:    fle.s a0, ft1, ft0
+; RV64IF-NEXT:    ret
   %1 = fcmp ole float %a, %b
   %2 = zext i1 %1 to i32
   ret i32 %2
@@ -86,6 +128,20 @@ define i32 @fcmp_one(float %a, float %b) nounwind {
 ; RV32IF-NEXT:    xori a0, a0, 1
 ; RV32IF-NEXT:    and a0, a1, a0
 ; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: fcmp_one:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fmv.w.x ft0, a0
+; RV64IF-NEXT:    fmv.w.x ft1, a1
+; RV64IF-NEXT:    feq.s a0, ft1, ft1
+; RV64IF-NEXT:    feq.s a1, ft0, ft0
+; RV64IF-NEXT:    and a0, a1, a0
+; RV64IF-NEXT:    feq.s a1, ft0, ft1
+; RV64IF-NEXT:    not a1, a1
+; RV64IF-NEXT:    seqz a0, a0
+; RV64IF-NEXT:    xori a0, a0, 1
+; RV64IF-NEXT:    and a0, a1, a0
+; RV64IF-NEXT:    ret
   %1 = fcmp one float %a, %b
   %2 = zext i1 %1 to i32
   ret i32 %2
@@ -102,6 +158,17 @@ define i32 @fcmp_ord(float %a, float %b) nounwind {
 ; RV32IF-NEXT:    seqz a0, a0
 ; RV32IF-NEXT:    xori a0, a0, 1
 ; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: fcmp_ord:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fmv.w.x ft0, a1
+; RV64IF-NEXT:    feq.s a1, ft0, ft0
+; RV64IF-NEXT:    fmv.w.x ft0, a0
+; RV64IF-NEXT:    feq.s a0, ft0, ft0
+; RV64IF-NEXT:    and a0, a0, a1
+; RV64IF-NEXT:    seqz a0, a0
+; RV64IF-NEXT:    xori a0, a0, 1
+; RV64IF-NEXT:    ret
   %1 = fcmp ord float %a, %b
   %2 = zext i1 %1 to i32
   ret i32 %2
@@ -119,6 +186,18 @@ define i32 @fcmp_ueq(float %a, float %b) nounwind {
 ; RV32IF-NEXT:    seqz a1, a1
 ; RV32IF-NEXT:    or a0, a0, a1
 ; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: fcmp_ueq:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fmv.w.x ft0, a1
+; RV64IF-NEXT:    fmv.w.x ft1, a0
+; RV64IF-NEXT:    feq.s a0, ft1, ft0
+; RV64IF-NEXT:    feq.s a1, ft0, ft0
+; RV64IF-NEXT:    feq.s a2, ft1, ft1
+; RV64IF-NEXT:    and a1, a2, a1
+; RV64IF-NEXT:    seqz a1, a1
+; RV64IF-NEXT:    or a0, a0, a1
+; RV64IF-NEXT:    ret
   %1 = fcmp ueq float %a, %b
   %2 = zext i1 %1 to i32
   ret i32 %2
@@ -132,6 +211,14 @@ define i32 @fcmp_ugt(float %a, float %b) nounwind {
 ; RV32IF-NEXT:    fle.s a0, ft1, ft0
 ; RV32IF-NEXT:    xori a0, a0, 1
 ; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: fcmp_ugt:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fmv.w.x ft0, a1
+; RV64IF-NEXT:    fmv.w.x ft1, a0
+; RV64IF-NEXT:    fle.s a0, ft1, ft0
+; RV64IF-NEXT:    xori a0, a0, 1
+; RV64IF-NEXT:    ret
   %1 = fcmp ugt float %a, %b
   %2 = zext i1 %1 to i32
   ret i32 %2
@@ -145,6 +232,14 @@ define i32 @fcmp_uge(float %a, float %b) nounwind {
 ; RV32IF-NEXT:    flt.s a0, ft1, ft0
 ; RV32IF-NEXT:    xori a0, a0, 1
 ; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: fcmp_uge:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fmv.w.x ft0, a1
+; RV64IF-NEXT:    fmv.w.x ft1, a0
+; RV64IF-NEXT:    flt.s a0, ft1, ft0
+; RV64IF-NEXT:    xori a0, a0, 1
+; RV64IF-NEXT:    ret
   %1 = fcmp uge float %a, %b
   %2 = zext i1 %1 to i32
   ret i32 %2
@@ -158,6 +253,14 @@ define i32 @fcmp_ult(float %a, float %b) nounwind {
 ; RV32IF-NEXT:    fle.s a0, ft1, ft0
 ; RV32IF-NEXT:    xori a0, a0, 1
 ; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: fcmp_ult:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fmv.w.x ft0, a0
+; RV64IF-NEXT:    fmv.w.x ft1, a1
+; RV64IF-NEXT:    fle.s a0, ft1, ft0
+; RV64IF-NEXT:    xori a0, a0, 1
+; RV64IF-NEXT:    ret
   %1 = fcmp ult float %a, %b
   %2 = zext i1 %1 to i32
   ret i32 %2
@@ -171,6 +274,14 @@ define i32 @fcmp_ule(float %a, float %b) nounwind {
 ; RV32IF-NEXT:    flt.s a0, ft1, ft0
 ; RV32IF-NEXT:    xori a0, a0, 1
 ; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: fcmp_ule:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fmv.w.x ft0, a0
+; RV64IF-NEXT:    fmv.w.x ft1, a1
+; RV64IF-NEXT:    flt.s a0, ft1, ft0
+; RV64IF-NEXT:    xori a0, a0, 1
+; RV64IF-NEXT:    ret
   %1 = fcmp ule float %a, %b
   %2 = zext i1 %1 to i32
   ret i32 %2
@@ -184,6 +295,14 @@ define i32 @fcmp_une(float %a, float %b) nounwind {
 ; RV32IF-NEXT:    feq.s a0, ft1, ft0
 ; RV32IF-NEXT:    xori a0, a0, 1
 ; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: fcmp_une:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fmv.w.x ft0, a1
+; RV64IF-NEXT:    fmv.w.x ft1, a0
+; RV64IF-NEXT:    feq.s a0, ft1, ft0
+; RV64IF-NEXT:    xori a0, a0, 1
+; RV64IF-NEXT:    ret
   %1 = fcmp une float %a, %b
   %2 = zext i1 %1 to i32
   ret i32 %2
@@ -199,6 +318,16 @@ define i32 @fcmp_uno(float %a, float %b) nounwind {
 ; RV32IF-NEXT:    and a0, a0, a1
 ; RV32IF-NEXT:    seqz a0, a0
 ; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: fcmp_uno:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fmv.w.x ft0, a1
+; RV64IF-NEXT:    feq.s a1, ft0, ft0
+; RV64IF-NEXT:    fmv.w.x ft0, a0
+; RV64IF-NEXT:    feq.s a0, ft0, ft0
+; RV64IF-NEXT:    and a0, a0, a1
+; RV64IF-NEXT:    seqz a0, a0
+; RV64IF-NEXT:    ret
   %1 = fcmp uno float %a, %b
   %2 = zext i1 %1 to i32
   ret i32 %2
@@ -209,6 +338,11 @@ define i32 @fcmp_true(float %a, float %b) nounwind {
 ; RV32IF:       # %bb.0:
 ; RV32IF-NEXT:    addi a0, zero, 1
 ; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: fcmp_true:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    addi a0, zero, 1
+; RV64IF-NEXT:    ret
   %1 = fcmp true float %a, %b
   %2 = zext i1 %1 to i32
   ret i32 %2
diff --git a/test/CodeGen/RISCV/float-imm.ll b/test/CodeGen/RISCV/float-imm.ll
index a8d032571e128cd0ed0046dbed904e14eb8e554d..09d9c3c933a12c06f6800738150792e4d2939830 100644
--- a/test/CodeGen/RISCV/float-imm.ll
+++ b/test/CodeGen/RISCV/float-imm.ll
@@ -1,13 +1,24 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=riscv32 -mattr=+f -verify-machineinstrs < %s \
 ; RUN:   | FileCheck -check-prefix=RV32IF %s
+; RUN: llc -mtriple=riscv64 -mattr=+f -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=RV64IF %s
 
+; TODO: constant pool shouldn't be necessary for RV64IF.
 define float @float_imm() nounwind {
 ; RV32IF-LABEL: float_imm:
 ; RV32IF:       # %bb.0:
 ; RV32IF-NEXT:    lui a0, 263313
 ; RV32IF-NEXT:    addi a0, a0, -37
 ; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: float_imm:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    lui a0, %hi(.LCPI0_0)
+; RV64IF-NEXT:    addi a0, a0, %lo(.LCPI0_0)
+; RV64IF-NEXT:    flw ft0, 0(a0)
+; RV64IF-NEXT:    fmv.x.w a0, ft0
+; RV64IF-NEXT:    ret
   ret float 3.14159274101257324218750
 }
 
@@ -22,6 +33,16 @@ define float @float_imm_op(float %a) nounwind {
 ; RV32IF-NEXT:    fadd.s ft0, ft0, ft1
 ; RV32IF-NEXT:    fmv.x.w a0, ft0
 ; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: float_imm_op:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fmv.w.x ft0, a0
+; RV64IF-NEXT:    lui a0, %hi(.LCPI1_0)
+; RV64IF-NEXT:    addi a0, a0, %lo(.LCPI1_0)
+; RV64IF-NEXT:    flw ft1, 0(a0)
+; RV64IF-NEXT:    fadd.s ft0, ft0, ft1
+; RV64IF-NEXT:    fmv.x.w a0, ft0
+; RV64IF-NEXT:    ret
   %1 = fadd float %a, 1.0
   ret float %1
 }
diff --git a/test/CodeGen/RISCV/float-intrinsics.ll b/test/CodeGen/RISCV/float-intrinsics.ll
index 57f3a28ee60e6b237e8ac56d007964b1ac2d32ae..20248415468a0702223433a781d22b5126abb327 100644
--- a/test/CodeGen/RISCV/float-intrinsics.ll
+++ b/test/CodeGen/RISCV/float-intrinsics.ll
@@ -3,6 +3,10 @@
 ; RUN:   | FileCheck -check-prefix=RV32IF %s
 ; RUN: llc -mtriple=riscv32 -mattr=+d -verify-machineinstrs < %s \
 ; RUN:   | FileCheck -check-prefix=RV32IF %s
+; RUN: llc -mtriple=riscv64 -mattr=+f -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=RV64IF %s
+; RUN: llc -mtriple=riscv64 -mattr=+d -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=RV64IF %s
 
 declare float @llvm.sqrt.f32(float)
 
@@ -13,6 +17,13 @@ define float @sqrt_f32(float %a) nounwind {
 ; RV32IF-NEXT:    fsqrt.s ft0, ft0
 ; RV32IF-NEXT:    fmv.x.w a0, ft0
 ; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: sqrt_f32:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fmv.w.x ft0, a0
+; RV64IF-NEXT:    fsqrt.s ft0, ft0
+; RV64IF-NEXT:    fmv.x.w a0, ft0
+; RV64IF-NEXT:    ret
   %1 = call float @llvm.sqrt.f32(float %a)
   ret float %1
 }
@@ -28,6 +39,16 @@ define float @powi_f32(float %a, i32 %b) nounwind {
 ; RV32IF-NEXT:    lw ra, 12(sp)
 ; RV32IF-NEXT:    addi sp, sp, 16
 ; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: powi_f32:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    addi sp, sp, -16
+; RV64IF-NEXT:    sd ra, 8(sp)
+; RV64IF-NEXT:    sext.w a1, a1
+; RV64IF-NEXT:    call __powisf2
+; RV64IF-NEXT:    ld ra, 8(sp)
+; RV64IF-NEXT:    addi sp, sp, 16
+; RV64IF-NEXT:    ret
   %1 = call float @llvm.powi.f32(float %a, i32 %b)
   ret float %1
 }
@@ -43,6 +64,15 @@ define float @sin_f32(float %a) nounwind {
 ; RV32IF-NEXT:    lw ra, 12(sp)
 ; RV32IF-NEXT:    addi sp, sp, 16
 ; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: sin_f32:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    addi sp, sp, -16
+; RV64IF-NEXT:    sd ra, 8(sp)
+; RV64IF-NEXT:    call sinf
+; RV64IF-NEXT:    ld ra, 8(sp)
+; RV64IF-NEXT:    addi sp, sp, 16
+; RV64IF-NEXT:    ret
   %1 = call float @llvm.sin.f32(float %a)
   ret float %1
 }
@@ -58,6 +88,15 @@ define float @cos_f32(float %a) nounwind {
 ; RV32IF-NEXT:    lw ra, 12(sp)
 ; RV32IF-NEXT:    addi sp, sp, 16
 ; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: cos_f32:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    addi sp, sp, -16
+; RV64IF-NEXT:    sd ra, 8(sp)
+; RV64IF-NEXT:    call cosf
+; RV64IF-NEXT:    ld ra, 8(sp)
+; RV64IF-NEXT:    addi sp, sp, 16
+; RV64IF-NEXT:    ret
   %1 = call float @llvm.cos.f32(float %a)
   ret float %1
 }
@@ -84,6 +123,27 @@ define float @sincos_f32(float %a) nounwind {
 ; RV32IF-NEXT:    lw ra, 12(sp)
 ; RV32IF-NEXT:    addi sp, sp, 16
 ; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: sincos_f32:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    addi sp, sp, -32
+; RV64IF-NEXT:    sd ra, 24(sp)
+; RV64IF-NEXT:    sd s1, 16(sp)
+; RV64IF-NEXT:    sd s2, 8(sp)
+; RV64IF-NEXT:    mv s1, a0
+; RV64IF-NEXT:    call sinf
+; RV64IF-NEXT:    mv s2, a0
+; RV64IF-NEXT:    mv a0, s1
+; RV64IF-NEXT:    call cosf
+; RV64IF-NEXT:    fmv.w.x ft0, a0
+; RV64IF-NEXT:    fmv.w.x ft1, s2
+; RV64IF-NEXT:    fadd.s ft0, ft1, ft0
+; RV64IF-NEXT:    fmv.x.w a0, ft0
+; RV64IF-NEXT:    ld s2, 8(sp)
+; RV64IF-NEXT:    ld s1, 16(sp)
+; RV64IF-NEXT:    ld ra, 24(sp)
+; RV64IF-NEXT:    addi sp, sp, 32
+; RV64IF-NEXT:    ret
   %1 = call float @llvm.sin.f32(float %a)
   %2 = call float @llvm.cos.f32(float %a)
   %3 = fadd float %1, %2
@@ -101,6 +161,15 @@ define float @pow_f32(float %a, float %b) nounwind {
 ; RV32IF-NEXT:    lw ra, 12(sp)
 ; RV32IF-NEXT:    addi sp, sp, 16
 ; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: pow_f32:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    addi sp, sp, -16
+; RV64IF-NEXT:    sd ra, 8(sp)
+; RV64IF-NEXT:    call powf
+; RV64IF-NEXT:    ld ra, 8(sp)
+; RV64IF-NEXT:    addi sp, sp, 16
+; RV64IF-NEXT:    ret
   %1 = call float @llvm.pow.f32(float %a, float %b)
   ret float %1
 }
@@ -116,6 +185,15 @@ define float @exp_f32(float %a) nounwind {
 ; RV32IF-NEXT:    lw ra, 12(sp)
 ; RV32IF-NEXT:    addi sp, sp, 16
 ; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: exp_f32:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    addi sp, sp, -16
+; RV64IF-NEXT:    sd ra, 8(sp)
+; RV64IF-NEXT:    call expf
+; RV64IF-NEXT:    ld ra, 8(sp)
+; RV64IF-NEXT:    addi sp, sp, 16
+; RV64IF-NEXT:    ret
   %1 = call float @llvm.exp.f32(float %a)
   ret float %1
 }
@@ -131,6 +209,15 @@ define float @exp2_f32(float %a) nounwind {
 ; RV32IF-NEXT:    lw ra, 12(sp)
 ; RV32IF-NEXT:    addi sp, sp, 16
 ; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: exp2_f32:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    addi sp, sp, -16
+; RV64IF-NEXT:    sd ra, 8(sp)
+; RV64IF-NEXT:    call exp2f
+; RV64IF-NEXT:    ld ra, 8(sp)
+; RV64IF-NEXT:    addi sp, sp, 16
+; RV64IF-NEXT:    ret
   %1 = call float @llvm.exp2.f32(float %a)
   ret float %1
 }
@@ -146,6 +233,15 @@ define float @log_f32(float %a) nounwind {
 ; RV32IF-NEXT:    lw ra, 12(sp)
 ; RV32IF-NEXT:    addi sp, sp, 16
 ; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: log_f32:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    addi sp, sp, -16
+; RV64IF-NEXT:    sd ra, 8(sp)
+; RV64IF-NEXT:    call logf
+; RV64IF-NEXT:    ld ra, 8(sp)
+; RV64IF-NEXT:    addi sp, sp, 16
+; RV64IF-NEXT:    ret
   %1 = call float @llvm.log.f32(float %a)
   ret float %1
 }
@@ -161,6 +257,15 @@ define float @log10_f32(float %a) nounwind {
 ; RV32IF-NEXT:    lw ra, 12(sp)
 ; RV32IF-NEXT:    addi sp, sp, 16
 ; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: log10_f32:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    addi sp, sp, -16
+; RV64IF-NEXT:    sd ra, 8(sp)
+; RV64IF-NEXT:    call log10f
+; RV64IF-NEXT:    ld ra, 8(sp)
+; RV64IF-NEXT:    addi sp, sp, 16
+; RV64IF-NEXT:    ret
   %1 = call float @llvm.log10.f32(float %a)
   ret float %1
 }
@@ -176,6 +281,15 @@ define float @log2_f32(float %a) nounwind {
 ; RV32IF-NEXT:    lw ra, 12(sp)
 ; RV32IF-NEXT:    addi sp, sp, 16
 ; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: log2_f32:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    addi sp, sp, -16
+; RV64IF-NEXT:    sd ra, 8(sp)
+; RV64IF-NEXT:    call log2f
+; RV64IF-NEXT:    ld ra, 8(sp)
+; RV64IF-NEXT:    addi sp, sp, 16
+; RV64IF-NEXT:    ret
   %1 = call float @llvm.log2.f32(float %a)
   ret float %1
 }
@@ -191,6 +305,15 @@ define float @fma_f32(float %a, float %b, float %c) nounwind {
 ; RV32IF-NEXT:    fmadd.s ft0, ft2, ft1, ft0
 ; RV32IF-NEXT:    fmv.x.w a0, ft0
 ; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: fma_f32:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fmv.w.x ft0, a2
+; RV64IF-NEXT:    fmv.w.x ft1, a1
+; RV64IF-NEXT:    fmv.w.x ft2, a0
+; RV64IF-NEXT:    fmadd.s ft0, ft2, ft1, ft0
+; RV64IF-NEXT:    fmv.x.w a0, ft0
+; RV64IF-NEXT:    ret
   %1 = call float @llvm.fma.f32(float %a, float %b, float %c)
   ret float %1
 }
@@ -208,6 +331,16 @@ define float @fmuladd_f32(float %a, float %b, float %c) nounwind {
 ; RV32IF-NEXT:    fadd.s ft0, ft0, ft1
 ; RV32IF-NEXT:    fmv.x.w a0, ft0
 ; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: fmuladd_f32:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fmv.w.x ft0, a1
+; RV64IF-NEXT:    fmv.w.x ft1, a0
+; RV64IF-NEXT:    fmul.s ft0, ft1, ft0
+; RV64IF-NEXT:    fmv.w.x ft1, a2
+; RV64IF-NEXT:    fadd.s ft0, ft0, ft1
+; RV64IF-NEXT:    fmv.x.w a0, ft0
+; RV64IF-NEXT:    ret
   %1 = call float @llvm.fmuladd.f32(float %a, float %b, float %c)
   ret float %1
 }
@@ -221,6 +354,13 @@ define float @fabs_f32(float %a) nounwind {
 ; RV32IF-NEXT:    addi a1, a1, -1
 ; RV32IF-NEXT:    and a0, a0, a1
 ; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: fabs_f32:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    lui a1, 524288
+; RV64IF-NEXT:    addiw a1, a1, -1
+; RV64IF-NEXT:    and a0, a0, a1
+; RV64IF-NEXT:    ret
   %1 = call float @llvm.fabs.f32(float %a)
   ret float %1
 }
@@ -235,6 +375,14 @@ define float @minnum_f32(float %a, float %b) nounwind {
 ; RV32IF-NEXT:    fmin.s ft0, ft1, ft0
 ; RV32IF-NEXT:    fmv.x.w a0, ft0
 ; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: minnum_f32:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fmv.w.x ft0, a1
+; RV64IF-NEXT:    fmv.w.x ft1, a0
+; RV64IF-NEXT:    fmin.s ft0, ft1, ft0
+; RV64IF-NEXT:    fmv.x.w a0, ft0
+; RV64IF-NEXT:    ret
   %1 = call float @llvm.minnum.f32(float %a, float %b)
   ret float %1
 }
@@ -249,6 +397,14 @@ define float @maxnum_f32(float %a, float %b) nounwind {
 ; RV32IF-NEXT:    fmax.s ft0, ft1, ft0
 ; RV32IF-NEXT:    fmv.x.w a0, ft0
 ; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: maxnum_f32:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fmv.w.x ft0, a1
+; RV64IF-NEXT:    fmv.w.x ft1, a0
+; RV64IF-NEXT:    fmax.s ft0, ft1, ft0
+; RV64IF-NEXT:    fmv.x.w a0, ft0
+; RV64IF-NEXT:    ret
   %1 = call float @llvm.maxnum.f32(float %a, float %b)
   ret float %1
 }
@@ -280,6 +436,14 @@ define float @copysign_f32(float %a, float %b) nounwind {
 ; RV32IF-NEXT:    fsgnj.s ft0, ft1, ft0
 ; RV32IF-NEXT:    fmv.x.w a0, ft0
 ; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: copysign_f32:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fmv.w.x ft0, a1
+; RV64IF-NEXT:    fmv.w.x ft1, a0
+; RV64IF-NEXT:    fsgnj.s ft0, ft1, ft0
+; RV64IF-NEXT:    fmv.x.w a0, ft0
+; RV64IF-NEXT:    ret
   %1 = call float @llvm.copysign.f32(float %a, float %b)
   ret float %1
 }
@@ -295,6 +459,15 @@ define float @floor_f32(float %a) nounwind {
 ; RV32IF-NEXT:    lw ra, 12(sp)
 ; RV32IF-NEXT:    addi sp, sp, 16
 ; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: floor_f32:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    addi sp, sp, -16
+; RV64IF-NEXT:    sd ra, 8(sp)
+; RV64IF-NEXT:    call floorf
+; RV64IF-NEXT:    ld ra, 8(sp)
+; RV64IF-NEXT:    addi sp, sp, 16
+; RV64IF-NEXT:    ret
   %1 = call float @llvm.floor.f32(float %a)
   ret float %1
 }
@@ -310,6 +483,15 @@ define float @ceil_f32(float %a) nounwind {
 ; RV32IF-NEXT:    lw ra, 12(sp)
 ; RV32IF-NEXT:    addi sp, sp, 16
 ; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: ceil_f32:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    addi sp, sp, -16
+; RV64IF-NEXT:    sd ra, 8(sp)
+; RV64IF-NEXT:    call ceilf
+; RV64IF-NEXT:    ld ra, 8(sp)
+; RV64IF-NEXT:    addi sp, sp, 16
+; RV64IF-NEXT:    ret
   %1 = call float @llvm.ceil.f32(float %a)
   ret float %1
 }
@@ -325,6 +507,15 @@ define float @trunc_f32(float %a) nounwind {
 ; RV32IF-NEXT:    lw ra, 12(sp)
 ; RV32IF-NEXT:    addi sp, sp, 16
 ; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: trunc_f32:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    addi sp, sp, -16
+; RV64IF-NEXT:    sd ra, 8(sp)
+; RV64IF-NEXT:    call truncf
+; RV64IF-NEXT:    ld ra, 8(sp)
+; RV64IF-NEXT:    addi sp, sp, 16
+; RV64IF-NEXT:    ret
   %1 = call float @llvm.trunc.f32(float %a)
   ret float %1
 }
@@ -340,6 +531,15 @@ define float @rint_f32(float %a) nounwind {
 ; RV32IF-NEXT:    lw ra, 12(sp)
 ; RV32IF-NEXT:    addi sp, sp, 16
 ; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: rint_f32:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    addi sp, sp, -16
+; RV64IF-NEXT:    sd ra, 8(sp)
+; RV64IF-NEXT:    call rintf
+; RV64IF-NEXT:    ld ra, 8(sp)
+; RV64IF-NEXT:    addi sp, sp, 16
+; RV64IF-NEXT:    ret
   %1 = call float @llvm.rint.f32(float %a)
   ret float %1
 }
@@ -355,6 +555,15 @@ define float @nearbyint_f32(float %a) nounwind {
 ; RV32IF-NEXT:    lw ra, 12(sp)
 ; RV32IF-NEXT:    addi sp, sp, 16
 ; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: nearbyint_f32:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    addi sp, sp, -16
+; RV64IF-NEXT:    sd ra, 8(sp)
+; RV64IF-NEXT:    call nearbyintf
+; RV64IF-NEXT:    ld ra, 8(sp)
+; RV64IF-NEXT:    addi sp, sp, 16
+; RV64IF-NEXT:    ret
   %1 = call float @llvm.nearbyint.f32(float %a)
   ret float %1
 }
@@ -370,6 +579,15 @@ define float @round_f32(float %a) nounwind {
 ; RV32IF-NEXT:    lw ra, 12(sp)
 ; RV32IF-NEXT:    addi sp, sp, 16
 ; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: round_f32:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    addi sp, sp, -16
+; RV64IF-NEXT:    sd ra, 8(sp)
+; RV64IF-NEXT:    call roundf
+; RV64IF-NEXT:    ld ra, 8(sp)
+; RV64IF-NEXT:    addi sp, sp, 16
+; RV64IF-NEXT:    ret
   %1 = call float @llvm.round.f32(float %a)
   ret float %1
 }
diff --git a/test/CodeGen/RISCV/float-mem.ll b/test/CodeGen/RISCV/float-mem.ll
index 95a9d73d5f753dd136db8e933d8d6360a1d8a853..b90ee601b74a39f322cffbfaa03ea5873afc9723 100644
--- a/test/CodeGen/RISCV/float-mem.ll
+++ b/test/CodeGen/RISCV/float-mem.ll
@@ -1,6 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=riscv32 -mattr=+f -verify-machineinstrs < %s \
 ; RUN:   | FileCheck -check-prefix=RV32IF %s
+; RUN: llc -mtriple=riscv64 -mattr=+f -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=RV64IF %s
 
 define float @flw(float *%a) nounwind {
 ; RV32IF-LABEL: flw:
@@ -10,6 +12,14 @@ define float @flw(float *%a) nounwind {
 ; RV32IF-NEXT:    fadd.s ft0, ft1, ft0
 ; RV32IF-NEXT:    fmv.x.w a0, ft0
 ; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: flw:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    flw ft0, 12(a0)
+; RV64IF-NEXT:    flw ft1, 0(a0)
+; RV64IF-NEXT:    fadd.s ft0, ft1, ft0
+; RV64IF-NEXT:    fmv.x.w a0, ft0
+; RV64IF-NEXT:    ret
   %1 = load float, float* %a
   %2 = getelementptr float, float* %a, i32 3
   %3 = load float, float* %2
@@ -30,6 +40,15 @@ define void @fsw(float *%a, float %b, float %c) nounwind {
 ; RV32IF-NEXT:    fsw ft0, 32(a0)
 ; RV32IF-NEXT:    fsw ft0, 0(a0)
 ; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: fsw:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fmv.w.x ft0, a2
+; RV64IF-NEXT:    fmv.w.x ft1, a1
+; RV64IF-NEXT:    fadd.s ft0, ft1, ft0
+; RV64IF-NEXT:    fsw ft0, 32(a0)
+; RV64IF-NEXT:    fsw ft0, 0(a0)
+; RV64IF-NEXT:    ret
   %1 = fadd float %b, %c
   store float %1, float* %a
   %2 = getelementptr float, float* %a, i32 8
@@ -56,6 +75,20 @@ define float @flw_fsw_global(float %a, float %b) nounwind {
 ; RV32IF-NEXT:    fsw ft0, 36(a0)
 ; RV32IF-NEXT:    fmv.x.w a0, ft0
 ; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: flw_fsw_global:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fmv.w.x ft0, a1
+; RV64IF-NEXT:    fmv.w.x ft1, a0
+; RV64IF-NEXT:    fadd.s ft0, ft1, ft0
+; RV64IF-NEXT:    lui a0, %hi(G)
+; RV64IF-NEXT:    flw ft1, %lo(G)(a0)
+; RV64IF-NEXT:    fsw ft0, %lo(G)(a0)
+; RV64IF-NEXT:    addi a0, a0, %lo(G)
+; RV64IF-NEXT:    flw ft1, 36(a0)
+; RV64IF-NEXT:    fsw ft0, 36(a0)
+; RV64IF-NEXT:    fmv.x.w a0, ft0
+; RV64IF-NEXT:    ret
   %1 = fadd float %a, %b
   %2 = load volatile float, float* @G
   store float %1, float* @G
@@ -76,6 +109,18 @@ define float @flw_fsw_constant(float %a) nounwind {
 ; RV32IF-NEXT:    fsw ft0, -273(a0)
 ; RV32IF-NEXT:    fmv.x.w a0, ft0
 ; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: flw_fsw_constant:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fmv.w.x ft0, a0
+; RV64IF-NEXT:    lui a0, 56
+; RV64IF-NEXT:    addiw a0, a0, -1353
+; RV64IF-NEXT:    slli a0, a0, 14
+; RV64IF-NEXT:    flw ft1, -273(a0)
+; RV64IF-NEXT:    fadd.s ft0, ft0, ft1
+; RV64IF-NEXT:    fsw ft0, -273(a0)
+; RV64IF-NEXT:    fmv.x.w a0, ft0
+; RV64IF-NEXT:    ret
   %1 = inttoptr i32 3735928559 to float*
   %2 = load volatile float, float* %1
   %3 = fadd float %a, %2
@@ -102,6 +147,23 @@ define float @flw_stack(float %a) nounwind {
 ; RV32IF-NEXT:    lw ra, 12(sp)
 ; RV32IF-NEXT:    addi sp, sp, 16
 ; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: flw_stack:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    addi sp, sp, -32
+; RV64IF-NEXT:    sd ra, 24(sp)
+; RV64IF-NEXT:    sd s1, 16(sp)
+; RV64IF-NEXT:    mv s1, a0
+; RV64IF-NEXT:    addi a0, sp, 12
+; RV64IF-NEXT:    call notdead
+; RV64IF-NEXT:    fmv.w.x ft0, s1
+; RV64IF-NEXT:    flw ft1, 12(sp)
+; RV64IF-NEXT:    fadd.s ft0, ft1, ft0
+; RV64IF-NEXT:    fmv.x.w a0, ft0
+; RV64IF-NEXT:    ld s1, 16(sp)
+; RV64IF-NEXT:    ld ra, 24(sp)
+; RV64IF-NEXT:    addi sp, sp, 32
+; RV64IF-NEXT:    ret
   %1 = alloca float, align 4
   %2 = bitcast float* %1 to i8*
   call void @notdead(i8* %2)
@@ -124,6 +186,20 @@ define void @fsw_stack(float %a, float %b) nounwind {
 ; RV32IF-NEXT:    lw ra, 12(sp)
 ; RV32IF-NEXT:    addi sp, sp, 16
 ; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: fsw_stack:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    addi sp, sp, -16
+; RV64IF-NEXT:    sd ra, 8(sp)
+; RV64IF-NEXT:    fmv.w.x ft0, a1
+; RV64IF-NEXT:    fmv.w.x ft1, a0
+; RV64IF-NEXT:    fadd.s ft0, ft1, ft0
+; RV64IF-NEXT:    fsw ft0, 4(sp)
+; RV64IF-NEXT:    addi a0, sp, 4
+; RV64IF-NEXT:    call notdead
+; RV64IF-NEXT:    ld ra, 8(sp)
+; RV64IF-NEXT:    addi sp, sp, 16
+; RV64IF-NEXT:    ret
   %1 = fadd float %a, %b ; force store from FPR32
   %2 = alloca float, align 4
   store float %1, float* %2
diff --git a/test/CodeGen/RISCV/float-select-fcmp.ll b/test/CodeGen/RISCV/float-select-fcmp.ll
index 59d4a3f078b0392386f4ba9b56c163e54c2d7bf7..3659ed2d17504f53d7dfdd6e0befda192dec2515 100644
--- a/test/CodeGen/RISCV/float-select-fcmp.ll
+++ b/test/CodeGen/RISCV/float-select-fcmp.ll
@@ -1,12 +1,19 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=riscv32 -mattr=+f -verify-machineinstrs < %s \
 ; RUN:   | FileCheck -check-prefix=RV32IF %s
+; RUN: llc -mtriple=riscv64 -mattr=+f -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=RV64IF %s
 
 define float @select_fcmp_false(float %a, float %b) nounwind {
 ; RV32IF-LABEL: select_fcmp_false:
 ; RV32IF:       # %bb.0:
 ; RV32IF-NEXT:    mv a0, a1
 ; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: select_fcmp_false:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    mv a0, a1
+; RV64IF-NEXT:    ret
   %1 = fcmp false float %a, %b
   %2 = select i1 %1, float %a, float %b
   ret float %2
@@ -24,6 +31,18 @@ define float @select_fcmp_oeq(float %a, float %b) nounwind {
 ; RV32IF-NEXT:  .LBB1_2:
 ; RV32IF-NEXT:    fmv.x.w a0, ft0
 ; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: select_fcmp_oeq:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fmv.w.x ft1, a1
+; RV64IF-NEXT:    fmv.w.x ft0, a0
+; RV64IF-NEXT:    feq.s a0, ft0, ft1
+; RV64IF-NEXT:    bnez a0, .LBB1_2
+; RV64IF-NEXT:  # %bb.1:
+; RV64IF-NEXT:    fmv.s ft0, ft1
+; RV64IF-NEXT:  .LBB1_2:
+; RV64IF-NEXT:    fmv.x.w a0, ft0
+; RV64IF-NEXT:    ret
   %1 = fcmp oeq float %a, %b
   %2 = select i1 %1, float %a, float %b
   ret float %2
@@ -41,6 +60,18 @@ define float @select_fcmp_ogt(float %a, float %b) nounwind {
 ; RV32IF-NEXT:  .LBB2_2:
 ; RV32IF-NEXT:    fmv.x.w a0, ft0
 ; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: select_fcmp_ogt:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fmv.w.x ft0, a0
+; RV64IF-NEXT:    fmv.w.x ft1, a1
+; RV64IF-NEXT:    flt.s a0, ft1, ft0
+; RV64IF-NEXT:    bnez a0, .LBB2_2
+; RV64IF-NEXT:  # %bb.1:
+; RV64IF-NEXT:    fmv.s ft0, ft1
+; RV64IF-NEXT:  .LBB2_2:
+; RV64IF-NEXT:    fmv.x.w a0, ft0
+; RV64IF-NEXT:    ret
   %1 = fcmp ogt float %a, %b
   %2 = select i1 %1, float %a, float %b
   ret float %2
@@ -58,6 +89,18 @@ define float @select_fcmp_oge(float %a, float %b) nounwind {
 ; RV32IF-NEXT:  .LBB3_2:
 ; RV32IF-NEXT:    fmv.x.w a0, ft0
 ; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: select_fcmp_oge:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fmv.w.x ft0, a0
+; RV64IF-NEXT:    fmv.w.x ft1, a1
+; RV64IF-NEXT:    fle.s a0, ft1, ft0
+; RV64IF-NEXT:    bnez a0, .LBB3_2
+; RV64IF-NEXT:  # %bb.1:
+; RV64IF-NEXT:    fmv.s ft0, ft1
+; RV64IF-NEXT:  .LBB3_2:
+; RV64IF-NEXT:    fmv.x.w a0, ft0
+; RV64IF-NEXT:    ret
   %1 = fcmp oge float %a, %b
   %2 = select i1 %1, float %a, float %b
   ret float %2
@@ -75,6 +118,18 @@ define float @select_fcmp_olt(float %a, float %b) nounwind {
 ; RV32IF-NEXT:  .LBB4_2:
 ; RV32IF-NEXT:    fmv.x.w a0, ft0
 ; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: select_fcmp_olt:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fmv.w.x ft1, a1
+; RV64IF-NEXT:    fmv.w.x ft0, a0
+; RV64IF-NEXT:    flt.s a0, ft0, ft1
+; RV64IF-NEXT:    bnez a0, .LBB4_2
+; RV64IF-NEXT:  # %bb.1:
+; RV64IF-NEXT:    fmv.s ft0, ft1
+; RV64IF-NEXT:  .LBB4_2:
+; RV64IF-NEXT:    fmv.x.w a0, ft0
+; RV64IF-NEXT:    ret
   %1 = fcmp olt float %a, %b
   %2 = select i1 %1, float %a, float %b
   ret float %2
@@ -92,6 +147,18 @@ define float @select_fcmp_ole(float %a, float %b) nounwind {
 ; RV32IF-NEXT:  .LBB5_2:
 ; RV32IF-NEXT:    fmv.x.w a0, ft0
 ; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: select_fcmp_ole:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fmv.w.x ft1, a1
+; RV64IF-NEXT:    fmv.w.x ft0, a0
+; RV64IF-NEXT:    fle.s a0, ft0, ft1
+; RV64IF-NEXT:    bnez a0, .LBB5_2
+; RV64IF-NEXT:  # %bb.1:
+; RV64IF-NEXT:    fmv.s ft0, ft1
+; RV64IF-NEXT:  .LBB5_2:
+; RV64IF-NEXT:    fmv.x.w a0, ft0
+; RV64IF-NEXT:    ret
   %1 = fcmp ole float %a, %b
   %2 = select i1 %1, float %a, float %b
   ret float %2
@@ -117,6 +184,25 @@ define float @select_fcmp_one(float %a, float %b) nounwind {
 ; RV32IF-NEXT:  .LBB6_2:
 ; RV32IF-NEXT:    fmv.x.w a0, ft0
 ; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: select_fcmp_one:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fmv.w.x ft0, a0
+; RV64IF-NEXT:    fmv.w.x ft1, a1
+; RV64IF-NEXT:    feq.s a0, ft1, ft1
+; RV64IF-NEXT:    feq.s a1, ft0, ft0
+; RV64IF-NEXT:    and a0, a1, a0
+; RV64IF-NEXT:    feq.s a1, ft0, ft1
+; RV64IF-NEXT:    not a1, a1
+; RV64IF-NEXT:    seqz a0, a0
+; RV64IF-NEXT:    xori a0, a0, 1
+; RV64IF-NEXT:    and a0, a1, a0
+; RV64IF-NEXT:    bnez a0, .LBB6_2
+; RV64IF-NEXT:  # %bb.1:
+; RV64IF-NEXT:    fmv.s ft0, ft1
+; RV64IF-NEXT:  .LBB6_2:
+; RV64IF-NEXT:    fmv.x.w a0, ft0
+; RV64IF-NEXT:    ret
   %1 = fcmp one float %a, %b
   %2 = select i1 %1, float %a, float %b
   ret float %2
@@ -138,6 +224,22 @@ define float @select_fcmp_ord(float %a, float %b) nounwind {
 ; RV32IF-NEXT:  .LBB7_2:
 ; RV32IF-NEXT:    fmv.x.w a0, ft0
 ; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: select_fcmp_ord:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fmv.w.x ft0, a0
+; RV64IF-NEXT:    fmv.w.x ft1, a1
+; RV64IF-NEXT:    feq.s a0, ft1, ft1
+; RV64IF-NEXT:    feq.s a1, ft0, ft0
+; RV64IF-NEXT:    and a0, a1, a0
+; RV64IF-NEXT:    seqz a0, a0
+; RV64IF-NEXT:    xori a0, a0, 1
+; RV64IF-NEXT:    bnez a0, .LBB7_2
+; RV64IF-NEXT:  # %bb.1:
+; RV64IF-NEXT:    fmv.s ft0, ft1
+; RV64IF-NEXT:  .LBB7_2:
+; RV64IF-NEXT:    fmv.x.w a0, ft0
+; RV64IF-NEXT:    ret
   %1 = fcmp ord float %a, %b
   %2 = select i1 %1, float %a, float %b
   ret float %2
@@ -160,6 +262,23 @@ define float @select_fcmp_ueq(float %a, float %b) nounwind {
 ; RV32IF-NEXT:  .LBB8_2:
 ; RV32IF-NEXT:    fmv.x.w a0, ft0
 ; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: select_fcmp_ueq:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fmv.w.x ft0, a0
+; RV64IF-NEXT:    fmv.w.x ft1, a1
+; RV64IF-NEXT:    feq.s a0, ft1, ft1
+; RV64IF-NEXT:    feq.s a1, ft0, ft0
+; RV64IF-NEXT:    and a0, a1, a0
+; RV64IF-NEXT:    seqz a0, a0
+; RV64IF-NEXT:    feq.s a1, ft0, ft1
+; RV64IF-NEXT:    or a0, a1, a0
+; RV64IF-NEXT:    bnez a0, .LBB8_2
+; RV64IF-NEXT:  # %bb.1:
+; RV64IF-NEXT:    fmv.s ft0, ft1
+; RV64IF-NEXT:  .LBB8_2:
+; RV64IF-NEXT:    fmv.x.w a0, ft0
+; RV64IF-NEXT:    ret
   %1 = fcmp ueq float %a, %b
   %2 = select i1 %1, float %a, float %b
   ret float %2
@@ -178,6 +297,19 @@ define float @select_fcmp_ugt(float %a, float %b) nounwind {
 ; RV32IF-NEXT:  .LBB9_2:
 ; RV32IF-NEXT:    fmv.x.w a0, ft0
 ; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: select_fcmp_ugt:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fmv.w.x ft1, a1
+; RV64IF-NEXT:    fmv.w.x ft0, a0
+; RV64IF-NEXT:    fle.s a0, ft0, ft1
+; RV64IF-NEXT:    xori a0, a0, 1
+; RV64IF-NEXT:    bnez a0, .LBB9_2
+; RV64IF-NEXT:  # %bb.1:
+; RV64IF-NEXT:    fmv.s ft0, ft1
+; RV64IF-NEXT:  .LBB9_2:
+; RV64IF-NEXT:    fmv.x.w a0, ft0
+; RV64IF-NEXT:    ret
   %1 = fcmp ugt float %a, %b
   %2 = select i1 %1, float %a, float %b
   ret float %2
@@ -196,6 +328,19 @@ define float @select_fcmp_uge(float %a, float %b) nounwind {
 ; RV32IF-NEXT:  .LBB10_2:
 ; RV32IF-NEXT:    fmv.x.w a0, ft0
 ; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: select_fcmp_uge:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fmv.w.x ft1, a1
+; RV64IF-NEXT:    fmv.w.x ft0, a0
+; RV64IF-NEXT:    flt.s a0, ft0, ft1
+; RV64IF-NEXT:    xori a0, a0, 1
+; RV64IF-NEXT:    bnez a0, .LBB10_2
+; RV64IF-NEXT:  # %bb.1:
+; RV64IF-NEXT:    fmv.s ft0, ft1
+; RV64IF-NEXT:  .LBB10_2:
+; RV64IF-NEXT:    fmv.x.w a0, ft0
+; RV64IF-NEXT:    ret
   %1 = fcmp uge float %a, %b
   %2 = select i1 %1, float %a, float %b
   ret float %2
@@ -214,6 +359,19 @@ define float @select_fcmp_ult(float %a, float %b) nounwind {
 ; RV32IF-NEXT:  .LBB11_2:
 ; RV32IF-NEXT:    fmv.x.w a0, ft0
 ; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: select_fcmp_ult:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fmv.w.x ft0, a0
+; RV64IF-NEXT:    fmv.w.x ft1, a1
+; RV64IF-NEXT:    fle.s a0, ft1, ft0
+; RV64IF-NEXT:    xori a0, a0, 1
+; RV64IF-NEXT:    bnez a0, .LBB11_2
+; RV64IF-NEXT:  # %bb.1:
+; RV64IF-NEXT:    fmv.s ft0, ft1
+; RV64IF-NEXT:  .LBB11_2:
+; RV64IF-NEXT:    fmv.x.w a0, ft0
+; RV64IF-NEXT:    ret
   %1 = fcmp ult float %a, %b
   %2 = select i1 %1, float %a, float %b
   ret float %2
@@ -232,6 +390,19 @@ define float @select_fcmp_ule(float %a, float %b) nounwind {
 ; RV32IF-NEXT:  .LBB12_2:
 ; RV32IF-NEXT:    fmv.x.w a0, ft0
 ; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: select_fcmp_ule:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fmv.w.x ft0, a0
+; RV64IF-NEXT:    fmv.w.x ft1, a1
+; RV64IF-NEXT:    flt.s a0, ft1, ft0
+; RV64IF-NEXT:    xori a0, a0, 1
+; RV64IF-NEXT:    bnez a0, .LBB12_2
+; RV64IF-NEXT:  # %bb.1:
+; RV64IF-NEXT:    fmv.s ft0, ft1
+; RV64IF-NEXT:  .LBB12_2:
+; RV64IF-NEXT:    fmv.x.w a0, ft0
+; RV64IF-NEXT:    ret
   %1 = fcmp ule float %a, %b
   %2 = select i1 %1, float %a, float %b
   ret float %2
@@ -250,6 +421,19 @@ define float @select_fcmp_une(float %a, float %b) nounwind {
 ; RV32IF-NEXT:  .LBB13_2:
 ; RV32IF-NEXT:    fmv.x.w a0, ft0
 ; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: select_fcmp_une:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fmv.w.x ft1, a1
+; RV64IF-NEXT:    fmv.w.x ft0, a0
+; RV64IF-NEXT:    feq.s a0, ft0, ft1
+; RV64IF-NEXT:    xori a0, a0, 1
+; RV64IF-NEXT:    bnez a0, .LBB13_2
+; RV64IF-NEXT:  # %bb.1:
+; RV64IF-NEXT:    fmv.s ft0, ft1
+; RV64IF-NEXT:  .LBB13_2:
+; RV64IF-NEXT:    fmv.x.w a0, ft0
+; RV64IF-NEXT:    ret
   %1 = fcmp une float %a, %b
   %2 = select i1 %1, float %a, float %b
   ret float %2
@@ -271,6 +455,21 @@ define float @select_fcmp_uno(float %a, float %b) nounwind {
 ; RV32IF-NEXT:  .LBB14_2:
 ; RV32IF-NEXT:    fmv.x.w a0, ft0
 ; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: select_fcmp_uno:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fmv.w.x ft0, a0
+; RV64IF-NEXT:    fmv.w.x ft1, a1
+; RV64IF-NEXT:    feq.s a0, ft1, ft1
+; RV64IF-NEXT:    feq.s a1, ft0, ft0
+; RV64IF-NEXT:    and a0, a1, a0
+; RV64IF-NEXT:    seqz a0, a0
+; RV64IF-NEXT:    bnez a0, .LBB14_2
+; RV64IF-NEXT:  # %bb.1:
+; RV64IF-NEXT:    fmv.s ft0, ft1
+; RV64IF-NEXT:  .LBB14_2:
+; RV64IF-NEXT:    fmv.x.w a0, ft0
+; RV64IF-NEXT:    ret
   %1 = fcmp uno float %a, %b
   %2 = select i1 %1, float %a, float %b
   ret float %2
@@ -280,6 +479,10 @@ define float @select_fcmp_true(float %a, float %b) nounwind {
 ; RV32IF-LABEL: select_fcmp_true:
 ; RV32IF:       # %bb.0:
 ; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: select_fcmp_true:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    ret
   %1 = fcmp true float %a, %b
   %2 = select i1 %1, float %a, float %b
   ret float %2
@@ -298,6 +501,18 @@ define i32 @i32_select_fcmp_oeq(float %a, float %b, i32 %c, i32 %d) nounwind {
 ; RV32IF-NEXT:  .LBB16_2:
 ; RV32IF-NEXT:    mv a0, a2
 ; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: i32_select_fcmp_oeq:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fmv.w.x ft0, a1
+; RV64IF-NEXT:    fmv.w.x ft1, a0
+; RV64IF-NEXT:    feq.s a0, ft1, ft0
+; RV64IF-NEXT:    bnez a0, .LBB16_2
+; RV64IF-NEXT:  # %bb.1:
+; RV64IF-NEXT:    mv a2, a3
+; RV64IF-NEXT:  .LBB16_2:
+; RV64IF-NEXT:    mv a0, a2
+; RV64IF-NEXT:    ret
   %1 = fcmp oeq float %a, %b
   %2 = select i1 %1, i32 %c, i32 %d
   ret i32 %2
diff --git a/test/CodeGen/RISCV/frame.ll b/test/CodeGen/RISCV/frame.ll
index a1f58095188a65f3c66d02ddd4a1d63242eabf63..e8a833f37f83b20d9a140594b5efebdfaac2b9b8 100644
--- a/test/CodeGen/RISCV/frame.ll
+++ b/test/CodeGen/RISCV/frame.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=riscv32 -verify-machineinstrs < %s \
 ; RUN:   | FileCheck -check-prefix=RV32I-FPELIM %s
-; RUN: llc -mtriple=riscv32 -disable-fp-elim -verify-machineinstrs < %s \
+; RUN: llc -mtriple=riscv32 -frame-pointer=all -verify-machineinstrs < %s \
 ; RUN:   | FileCheck -check-prefix=RV32I-WITHFP %s
 
 %struct.key_t = type { i32, [16 x i8] }
diff --git a/test/CodeGen/RISCV/imm.ll b/test/CodeGen/RISCV/imm.ll
index 872b6d5a9864b1b42c81e4573681b63f7198e2d6..b733b5921658dc575ce1ef923a72aa8d13de1df3 100644
--- a/test/CodeGen/RISCV/imm.ll
+++ b/test/CodeGen/RISCV/imm.ll
@@ -142,8 +142,8 @@ define i64 @imm64_2() nounwind {
 define i64 @imm64_3() nounwind {
 ; RV32I-LABEL: imm64_3:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    addi a1, zero, 1
 ; RV32I-NEXT:    mv a0, zero
+; RV32I-NEXT:    addi a1, zero, 1
 ; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: imm64_3:
@@ -157,8 +157,8 @@ define i64 @imm64_3() nounwind {
 define i64 @imm64_4() nounwind {
 ; RV32I-LABEL: imm64_4:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    lui a1, 524288
 ; RV32I-NEXT:    mv a0, zero
+; RV32I-NEXT:    lui a1, 524288
 ; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: imm64_4:
@@ -172,8 +172,8 @@ define i64 @imm64_4() nounwind {
 define i64 @imm64_5() nounwind {
 ; RV32I-LABEL: imm64_5:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    lui a1, 524288
 ; RV32I-NEXT:    mv a0, zero
+; RV32I-NEXT:    lui a1, 524288
 ; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: imm64_5:
@@ -249,7 +249,7 @@ define i64 @imm64_9() nounwind {
 ; RV32I-LABEL: imm64_9:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi a0, zero, -1
-; RV32I-NEXT:    mv a1, a0
+; RV32I-NEXT:    addi  a1, zero, -1
 ; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: imm64_9:
diff --git a/test/CodeGen/RISCV/jumptable.ll b/test/CodeGen/RISCV/jumptable.ll
index f54f51edadae3e3adac327bf0341f788a9f00fb8..f849dc959abc4844b6089c92453d18c3c0461526 100644
--- a/test/CodeGen/RISCV/jumptable.ll
+++ b/test/CodeGen/RISCV/jumptable.ll
@@ -8,31 +8,31 @@ define void @jt(i32 %in, i32* %out) {
 ; RV32I-NEXT:    addi a2, zero, 2
 ; RV32I-NEXT:    blt a2, a0, .LBB0_4
 ; RV32I-NEXT:  # %bb.1: # %entry
-; RV32I-NEXT:    addi a3, zero, 1
-; RV32I-NEXT:    beq a0, a3, .LBB0_7
+; RV32I-NEXT:    addi a2, zero, 1
+; RV32I-NEXT:    beq a0, a2, .LBB0_7
 ; RV32I-NEXT:  # %bb.2: # %entry
-; RV32I-NEXT:    bne a0, a2, .LBB0_9
+; RV32I-NEXT:    addi a2, zero, 2
+; RV32I-NEXT:    bne a0, a2, .LBB0_10
 ; RV32I-NEXT:  # %bb.3: # %bb2
 ; RV32I-NEXT:    addi a0, zero, 3
-; RV32I-NEXT:    sw a0, 0(a1)
-; RV32I-NEXT:    ret
+; RV32I-NEXT:    j .LBB0_9
 ; RV32I-NEXT:  .LBB0_4: # %entry
-; RV32I-NEXT:    addi a3, zero, 3
-; RV32I-NEXT:    beq a0, a3, .LBB0_8
+; RV32I-NEXT:    addi a2, zero, 3
+; RV32I-NEXT:    beq a0, a2, .LBB0_8
 ; RV32I-NEXT:  # %bb.5: # %entry
 ; RV32I-NEXT:    addi a2, zero, 4
-; RV32I-NEXT:    bne a0, a2, .LBB0_9
+; RV32I-NEXT:    bne a0, a2, .LBB0_10
 ; RV32I-NEXT:  # %bb.6: # %bb4
 ; RV32I-NEXT:    addi a0, zero, 1
-; RV32I-NEXT:    sw a0, 0(a1)
-; RV32I-NEXT:    ret
+; RV32I-NEXT:    j .LBB0_9
 ; RV32I-NEXT:  .LBB0_7: # %bb1
 ; RV32I-NEXT:    addi a0, zero, 4
-; RV32I-NEXT:    sw a0, 0(a1)
-; RV32I-NEXT:    ret
+; RV32I-NEXT:    j .LBB0_9
 ; RV32I-NEXT:  .LBB0_8: # %bb3
-; RV32I-NEXT:    sw a2, 0(a1)
+; RV32I-NEXT:    addi a0, zero, 2
 ; RV32I-NEXT:  .LBB0_9: # %exit
+; RV32I-NEXT:    sw a0, 0(a1)
+; RV32I-NEXT:  .LBB0_10: # %exit
 ; RV32I-NEXT:    ret
 entry:
   switch i32 %in, label %exit [
diff --git a/test/CodeGen/RISCV/large-stack.ll b/test/CodeGen/RISCV/large-stack.ll
index 51130fc2a94a04aec631353569d27eeacc9d2d26..3e86871d457a59426217f5efcef34fc758193598 100644
--- a/test/CodeGen/RISCV/large-stack.ll
+++ b/test/CodeGen/RISCV/large-stack.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=riscv32 -verify-machineinstrs < %s \
 ; RUN:   | FileCheck -check-prefix=RV32I-FPELIM %s
-; RUN: llc -mtriple=riscv32 -verify-machineinstrs -disable-fp-elim < %s \
+; RUN: llc -mtriple=riscv32 -verify-machineinstrs -frame-pointer=all < %s \
 ; RUN:   | FileCheck -check-prefix=RV32I-WITHFP %s
 
 ; TODO: the quality of the generated code is poor
diff --git a/test/CodeGen/RISCV/legalize-fneg.ll b/test/CodeGen/RISCV/legalize-fneg.ll
new file mode 100644
index 0000000000000000000000000000000000000000..a38daaa7eebf4a88026b592568c0829cb3d5debb
--- /dev/null
+++ b/test/CodeGen/RISCV/legalize-fneg.ll
@@ -0,0 +1,105 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv32 -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=RV32 %s
+; RUN: llc -mtriple=riscv64 -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=RV64 %s
+; RUN: llc -mtriple=riscv32 -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=NOLIB %s
+; RUN: llc -mtriple=riscv64 -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=NOLIB %s
+
+define void @test1(float* %a, float* %b) {
+; RV32-LABEL: test1:
+; RV32:       # %bb.0: # %entry
+; RV32-NEXT:    lw a1, 0(a1)
+; RV32-NEXT:    lui a2, 524288
+; RV32-NEXT:    xor a1, a1, a2
+; RV32-NEXT:    sw a1, 0(a0)
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: test1:
+; RV64:       # %bb.0: # %entry
+; RV64-NEXT:    addi a2, zero, 1
+; RV64-NEXT:    slli a2, a2, 31
+; RV64-NEXT:    lw a1, 0(a1)
+; RV64-NEXT:    xor a1, a1, a2
+; RV64-NEXT:    sw a1, 0(a0)
+; RV64-NEXT:    ret
+
+; NOLIB-LABEL: test1:
+; NOLIB:         xor
+; NOLIB-NOT:     call __subsf3
+entry:
+  %0 = load float, float* %b
+  %neg = fneg float %0
+  store float %neg, float* %a
+  ret void
+}
+
+define void @test2(double* %a, double* %b) {
+; RV32-LABEL: test2:
+; RV32:       # %bb.0: # %entry
+; RV32-NEXT:    lw a2, 4(a1)
+; RV32-NEXT:    lw a1, 0(a1)
+; RV32-NEXT:    sw a1, 0(a0)
+; RV32-NEXT:    lui a1, 524288
+; RV32-NEXT:    xor a1, a2, a1
+; RV32-NEXT:    sw a1, 4(a0)
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: test2:
+; RV64:       # %bb.0: # %entry
+; RV64-NEXT:    addi a2, zero, -1
+; RV64-NEXT:    slli a2, a2, 63
+; RV64-NEXT:    ld a1, 0(a1)
+; RV64-NEXT:    xor a1, a1, a2
+; RV64-NEXT:    sd a1, 0(a0)
+; RV64-NEXT:    ret
+
+; NOLIB-LABEL: test2:
+; NOLIB:         xor
+; NOLIB-NOT:     call __subdf3
+entry:
+  %0 = load double, double* %b
+  %neg = fneg double %0
+  store double %neg, double* %a
+  ret void
+}
+
+define void @test3(fp128* %a, fp128* %b) {
+; RV32-LABEL: test3:
+; RV32:       # %bb.0: # %entry
+; RV32-NEXT:    lw a2, 12(a1)
+; RV32-NEXT:    lw a3, 0(a1)
+; RV32-NEXT:    lw a4, 4(a1)
+; RV32-NEXT:    lw a1, 8(a1)
+; RV32-NEXT:    sw a1, 8(a0)
+; RV32-NEXT:    sw a4, 4(a0)
+; RV32-NEXT:    sw a3, 0(a0)
+; RV32-NEXT:    lui a1, 524288
+; RV32-NEXT:    xor a1, a2, a1
+; RV32-NEXT:    sw a1, 12(a0)
+; RV32-NEXT:    ret
+;
+; RV64-LABEL: test3:
+; RV64:       # %bb.0: # %entry
+; RV64-NEXT:    ld a2, 8(a1)
+; RV64-NEXT:    ld a1, 0(a1)
+; RV64-NEXT:    sd a1, 0(a0)
+; RV64-NEXT:    addi a1, zero, -1
+; RV64-NEXT:    slli a1, a1, 63
+; RV64-NEXT:    xor a1, a2, a1
+; RV64-NEXT:    sd a1, 8(a0)
+; RV64-NEXT:    ret
+
+; NOLIB-LABEL: test3:
+; NOLIB:         xor
+; NOLIB-NOT:     call __subtf3
+entry:
+  %0 = load fp128, fp128* %b
+  %neg = fneg fp128 %0
+  store fp128 %neg, fp128* %a
+  ret void
+}
+
+
diff --git a/test/CodeGen/RISCV/mul.ll b/test/CodeGen/RISCV/mul.ll
index 444a75f3751275f05c47f86c087459e200a04991..9bf95bece61f1de49e839667bb31c346ec947291 100644
--- a/test/CodeGen/RISCV/mul.ll
+++ b/test/CodeGen/RISCV/mul.ll
@@ -3,8 +3,12 @@
 ; RUN:   | FileCheck -check-prefix=RV32I %s
 ; RUN: llc -mtriple=riscv32 -mattr=+m -verify-machineinstrs < %s \
 ; RUN:   | FileCheck -check-prefix=RV32IM %s
+; RUN: llc -mtriple=riscv64 -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=RV64I %s
+; RUN: llc -mtriple=riscv64 -mattr=+m -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=RV64IM %s
 
-define i32 @square(i32 %a) nounwind {
+define signext i32 @square(i32 %a) nounwind {
 ; RV32I-LABEL: square:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -19,11 +23,27 @@ define i32 @square(i32 %a) nounwind {
 ; RV32IM:       # %bb.0:
 ; RV32IM-NEXT:    mul a0, a0, a0
 ; RV32IM-NEXT:    ret
+;
+; RV64I-LABEL: square:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    mv a1, a0
+; RV64I-NEXT:    call __muldi3
+; RV64I-NEXT:    sext.w a0, a0
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IM-LABEL: square:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    mulw a0, a0, a0
+; RV64IM-NEXT:    ret
   %1 = mul i32 %a, %a
   ret i32 %1
 }
 
-define i32 @mul(i32 %a, i32 %b) nounwind {
+define signext i32 @mul(i32 %a, i32 %b) nounwind {
 ; RV32I-LABEL: mul:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -37,11 +57,26 @@ define i32 @mul(i32 %a, i32 %b) nounwind {
 ; RV32IM:       # %bb.0:
 ; RV32IM-NEXT:    mul a0, a0, a1
 ; RV32IM-NEXT:    ret
+;
+; RV64I-LABEL: mul:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    call __muldi3
+; RV64I-NEXT:    sext.w a0, a0
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IM-LABEL: mul:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    mulw a0, a0, a1
+; RV64IM-NEXT:    ret
   %1 = mul i32 %a, %b
   ret i32 %1
 }
 
-define i32 @mul_constant(i32 %a) nounwind {
+define signext i32 @mul_constant(i32 %a) nounwind {
 ; RV32I-LABEL: mul_constant:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -57,6 +92,23 @@ define i32 @mul_constant(i32 %a) nounwind {
 ; RV32IM-NEXT:    addi a1, zero, 5
 ; RV32IM-NEXT:    mul a0, a0, a1
 ; RV32IM-NEXT:    ret
+;
+; RV64I-LABEL: mul_constant:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a1, zero, 5
+; RV64I-NEXT:    call __muldi3
+; RV64I-NEXT:    sext.w a0, a0
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IM-LABEL: mul_constant:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    addi a1, zero, 5
+; RV64IM-NEXT:    mulw a0, a0, a1
+; RV64IM-NEXT:    ret
   %1 = mul i32 %a, 5
   ret i32 %1
 }
@@ -71,6 +123,16 @@ define i32 @mul_pow2(i32 %a) nounwind {
 ; RV32IM:       # %bb.0:
 ; RV32IM-NEXT:    slli a0, a0, 3
 ; RV32IM-NEXT:    ret
+;
+; RV64I-LABEL: mul_pow2:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    slli a0, a0, 3
+; RV64I-NEXT:    ret
+;
+; RV64IM-LABEL: mul_pow2:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    slli a0, a0, 3
+; RV64IM-NEXT:    ret
   %1 = mul i32 %a, 8
   ret i32 %1
 }
@@ -94,6 +156,20 @@ define i64 @mul64(i64 %a, i64 %b) nounwind {
 ; RV32IM-NEXT:    add a1, a3, a1
 ; RV32IM-NEXT:    mul a0, a0, a2
 ; RV32IM-NEXT:    ret
+;
+; RV64I-LABEL: mul64:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    call __muldi3
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IM-LABEL: mul64:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    mul a0, a0, a1
+; RV64IM-NEXT:    ret
   %1 = mul i64 %a, %b
   ret i64 %1
 }
@@ -118,6 +194,22 @@ define i64 @mul64_constant(i64 %a) nounwind {
 ; RV32IM-NEXT:    add a1, a3, a1
 ; RV32IM-NEXT:    mul a0, a0, a2
 ; RV32IM-NEXT:    ret
+;
+; RV64I-LABEL: mul64_constant:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    addi a1, zero, 5
+; RV64I-NEXT:    call __muldi3
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IM-LABEL: mul64_constant:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    addi a1, zero, 5
+; RV64IM-NEXT:    mul a0, a0, a1
+; RV64IM-NEXT:    ret
   %1 = mul i64 %a, 5
   ret i64 %1
 }
@@ -140,6 +232,26 @@ define i32 @mulhs(i32 %a, i32 %b) nounwind {
 ; RV32IM:       # %bb.0:
 ; RV32IM-NEXT:    mulh a0, a0, a1
 ; RV32IM-NEXT:    ret
+;
+; RV64I-LABEL: mulhs:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sext.w a0, a0
+; RV64I-NEXT:    sext.w a1, a1
+; RV64I-NEXT:    call __muldi3
+; RV64I-NEXT:    srli a0, a0, 32
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IM-LABEL: mulhs:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    sext.w a1, a1
+; RV64IM-NEXT:    sext.w a0, a0
+; RV64IM-NEXT:    mul a0, a0, a1
+; RV64IM-NEXT:    srli a0, a0, 32
+; RV64IM-NEXT:    ret
   %1 = sext i32 %a to i64
   %2 = sext i32 %b to i64
   %3 = mul i64 %1, %2
@@ -148,7 +260,7 @@ define i32 @mulhs(i32 %a, i32 %b) nounwind {
   ret i32 %5
 }
 
-define i32 @mulhu(i32 %a, i32 %b) nounwind {
+define zeroext i32 @mulhu(i32 zeroext %a, i32 zeroext %b) nounwind {
 ; RV32I-LABEL: mulhu:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
@@ -166,6 +278,22 @@ define i32 @mulhu(i32 %a, i32 %b) nounwind {
 ; RV32IM:       # %bb.0:
 ; RV32IM-NEXT:    mulhu a0, a0, a1
 ; RV32IM-NEXT:    ret
+;
+; RV64I-LABEL: mulhu:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    call __muldi3
+; RV64I-NEXT:    srli a0, a0, 32
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IM-LABEL: mulhu:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    mul a0, a0, a1
+; RV64IM-NEXT:    srli a0, a0, 32
+; RV64IM-NEXT:    ret
   %1 = zext i32 %a to i64
   %2 = zext i32 %b to i64
   %3 = mul i64 %1, %2
diff --git a/test/CodeGen/RISCV/pr40333.ll b/test/CodeGen/RISCV/pr40333.ll
new file mode 100644
index 0000000000000000000000000000000000000000..79e24e3296b1f161c4f900b7774039ce5faa53ed
--- /dev/null
+++ b/test/CodeGen/RISCV/pr40333.ll
@@ -0,0 +1,20 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv64 -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=RV64I %s
+
+; This test case is significantly simplified from the submitted .ll but
+; demonstrates the same issue. At the time of this problem report, an infinite
+; loop would be created in DAGCombine, converting ANY_EXTEND to SIGN_EXTEND
+; and back again.
+
+define signext i8 @foo(i32 %a, i32 %b) nounwind {
+; RV64I-LABEL: foo:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    srlw a0, a0, a1
+; RV64I-NEXT:    slli a0, a0, 56
+; RV64I-NEXT:    srai a0, a0, 56
+; RV64I-NEXT:    ret
+ %1 = lshr i32 %a, %b
+ %2 = trunc i32 %1 to i8
+ ret i8 %2
+}
diff --git a/test/CodeGen/RISCV/rem.ll b/test/CodeGen/RISCV/rem.ll
index f37931f448e47465ffc306bd830080d5f8573f44..505a351db0c4ac384cad9132f6e299eeb547d952 100644
--- a/test/CodeGen/RISCV/rem.ll
+++ b/test/CodeGen/RISCV/rem.ll
@@ -3,6 +3,10 @@
 ; RUN:   | FileCheck -check-prefix=RV32I %s
 ; RUN: llc -mtriple=riscv32 -mattr=+m -verify-machineinstrs < %s \
 ; RUN:   | FileCheck -check-prefix=RV32IM %s
+; RUN: llc -mtriple=riscv64 -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=RV64I %s
+; RUN: llc -mtriple=riscv64 -mattr=+m -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=RV64IM %s
 
 define i32 @urem(i32 %a, i32 %b) nounwind {
 ; RV32I-LABEL: urem:
@@ -18,6 +22,24 @@ define i32 @urem(i32 %a, i32 %b) nounwind {
 ; RV32IM:       # %bb.0:
 ; RV32IM-NEXT:    remu a0, a0, a1
 ; RV32IM-NEXT:    ret
+;
+; RV64I-LABEL: urem:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    slli a0, a0, 32
+; RV64I-NEXT:    srli a0, a0, 32
+; RV64I-NEXT:    slli a1, a1, 32
+; RV64I-NEXT:    srli a1, a1, 32
+; RV64I-NEXT:    call __umoddi3
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IM-LABEL: urem:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    remuw a0, a0, a1
+; RV64IM-NEXT:    ret
   %1 = urem i32 %a, %b
   ret i32 %1
 }
@@ -36,6 +58,22 @@ define i32 @srem(i32 %a, i32 %b) nounwind {
 ; RV32IM:       # %bb.0:
 ; RV32IM-NEXT:    rem a0, a0, a1
 ; RV32IM-NEXT:    ret
+;
+; RV64I-LABEL: srem:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi sp, sp, -16
+; RV64I-NEXT:    sd ra, 8(sp)
+; RV64I-NEXT:    sext.w a0, a0
+; RV64I-NEXT:    sext.w a1, a1
+; RV64I-NEXT:    call __moddi3
+; RV64I-NEXT:    ld ra, 8(sp)
+; RV64I-NEXT:    addi sp, sp, 16
+; RV64I-NEXT:    ret
+;
+; RV64IM-LABEL: srem:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    remw a0, a0, a1
+; RV64IM-NEXT:    ret
   %1 = srem i32 %a, %b
   ret i32 %1
 }
diff --git a/test/CodeGen/RISCV/remat.ll b/test/CodeGen/RISCV/remat.ll
index 3b146615bf0fb05057e1c1b9032a5897b1560b15..a1b8a6ce9e032c8a6a736e980fe43b76d0636480 100644
--- a/test/CodeGen/RISCV/remat.ll
+++ b/test/CodeGen/RISCV/remat.ll
@@ -36,78 +36,77 @@ define i32 @test() nounwind {
 ; RV32I-NEXT:    sw s9, 8(sp)
 ; RV32I-NEXT:    sw s10, 4(sp)
 ; RV32I-NEXT:    sw s11, 0(sp)
-; RV32I-NEXT:    lui s3, %hi(a)
-; RV32I-NEXT:    lw a0, %lo(a)(s3)
+; RV32I-NEXT:    lui s9, %hi(a)
+; RV32I-NEXT:    lw a0, %lo(a)(s9)
 ; RV32I-NEXT:    beqz a0, .LBB0_11
 ; RV32I-NEXT:  # %bb.1: # %for.body.preheader
-; RV32I-NEXT:    lui s5, %hi(k)
-; RV32I-NEXT:    lui s6, %hi(j)
-; RV32I-NEXT:    lui s7, %hi(i)
-; RV32I-NEXT:    lui s9, %hi(g)
-; RV32I-NEXT:    lui s10, %hi(f)
-; RV32I-NEXT:    lui s1, %hi(e)
-; RV32I-NEXT:    lui s8, %hi(d)
-; RV32I-NEXT:    addi s11, zero, 32
-; RV32I-NEXT:    lui s2, %hi(c)
-; RV32I-NEXT:    lui s4, %hi(b)
+; RV32I-NEXT:    lui s2, %hi(l)
+; RV32I-NEXT:    lui s3, %hi(k)
+; RV32I-NEXT:    lui s4, %hi(j)
+; RV32I-NEXT:    lui s5, %hi(i)
+; RV32I-NEXT:    lui s7, %hi(g)
+; RV32I-NEXT:    lui s8, %hi(f)
+; RV32I-NEXT:    lui s10, %hi(e)
+; RV32I-NEXT:    lui s1, %hi(d)
+; RV32I-NEXT:    lui s11, %hi(c)
+; RV32I-NEXT:    lui s6, %hi(b)
 ; RV32I-NEXT:  .LBB0_2: # %for.body
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    lui a1, %hi(l)
-; RV32I-NEXT:    lw a1, %lo(l)(a1)
+; RV32I-NEXT:    lw a1, %lo(l)(s2)
 ; RV32I-NEXT:    beqz a1, .LBB0_4
 ; RV32I-NEXT:  # %bb.3: # %if.then
 ; RV32I-NEXT:    # in Loop: Header=BB0_2 Depth=1
-; RV32I-NEXT:    lw a4, %lo(e)(s1)
-; RV32I-NEXT:    lw a3, %lo(d)(s8)
-; RV32I-NEXT:    lw a2, %lo(c)(s2)
-; RV32I-NEXT:    lw a1, %lo(b)(s4)
-; RV32I-NEXT:    mv a5, s11
+; RV32I-NEXT:    lw a4, %lo(e)(s10)
+; RV32I-NEXT:    lw a3, %lo(d)(s1)
+; RV32I-NEXT:    lw a2, %lo(c)(s11)
+; RV32I-NEXT:    lw a1, %lo(b)(s6)
+; RV32I-NEXT:    addi a5, zero, 32
 ; RV32I-NEXT:    call foo
 ; RV32I-NEXT:  .LBB0_4: # %if.end
 ; RV32I-NEXT:    # in Loop: Header=BB0_2 Depth=1
-; RV32I-NEXT:    lw a0, %lo(k)(s5)
+; RV32I-NEXT:    lw a0, %lo(k)(s3)
 ; RV32I-NEXT:    beqz a0, .LBB0_6
 ; RV32I-NEXT:  # %bb.5: # %if.then3
 ; RV32I-NEXT:    # in Loop: Header=BB0_2 Depth=1
-; RV32I-NEXT:    lw a4, %lo(f)(s10)
-; RV32I-NEXT:    lw a3, %lo(e)(s1)
-; RV32I-NEXT:    lw a2, %lo(d)(s8)
-; RV32I-NEXT:    lw a1, %lo(c)(s2)
-; RV32I-NEXT:    lw a0, %lo(b)(s4)
+; RV32I-NEXT:    lw a4, %lo(f)(s8)
+; RV32I-NEXT:    lw a3, %lo(e)(s10)
+; RV32I-NEXT:    lw a2, %lo(d)(s1)
+; RV32I-NEXT:    lw a1, %lo(c)(s11)
+; RV32I-NEXT:    lw a0, %lo(b)(s6)
 ; RV32I-NEXT:    addi a5, zero, 64
 ; RV32I-NEXT:    call foo
 ; RV32I-NEXT:  .LBB0_6: # %if.end5
 ; RV32I-NEXT:    # in Loop: Header=BB0_2 Depth=1
-; RV32I-NEXT:    lw a0, %lo(j)(s6)
+; RV32I-NEXT:    lw a0, %lo(j)(s4)
 ; RV32I-NEXT:    beqz a0, .LBB0_8
 ; RV32I-NEXT:  # %bb.7: # %if.then7
 ; RV32I-NEXT:    # in Loop: Header=BB0_2 Depth=1
-; RV32I-NEXT:    lw a4, %lo(g)(s9)
-; RV32I-NEXT:    lw a3, %lo(f)(s10)
-; RV32I-NEXT:    lw a2, %lo(e)(s1)
-; RV32I-NEXT:    lw a1, %lo(d)(s8)
-; RV32I-NEXT:    lw a0, %lo(c)(s2)
-; RV32I-NEXT:    mv a5, s11
+; RV32I-NEXT:    lw a4, %lo(g)(s7)
+; RV32I-NEXT:    lw a3, %lo(f)(s8)
+; RV32I-NEXT:    lw a2, %lo(e)(s10)
+; RV32I-NEXT:    lw a1, %lo(d)(s1)
+; RV32I-NEXT:    lw a0, %lo(c)(s11)
+; RV32I-NEXT:    addi a5, zero, 32
 ; RV32I-NEXT:    call foo
 ; RV32I-NEXT:  .LBB0_8: # %if.end9
 ; RV32I-NEXT:    # in Loop: Header=BB0_2 Depth=1
-; RV32I-NEXT:    lw a0, %lo(i)(s7)
+; RV32I-NEXT:    lw a0, %lo(i)(s5)
 ; RV32I-NEXT:    beqz a0, .LBB0_10
 ; RV32I-NEXT:  # %bb.9: # %if.then11
 ; RV32I-NEXT:    # in Loop: Header=BB0_2 Depth=1
 ; RV32I-NEXT:    lui a0, %hi(h)
 ; RV32I-NEXT:    lw a4, %lo(h)(a0)
-; RV32I-NEXT:    lw a3, %lo(g)(s9)
-; RV32I-NEXT:    lw a2, %lo(f)(s10)
-; RV32I-NEXT:    lw a1, %lo(e)(s1)
-; RV32I-NEXT:    lw a0, %lo(d)(s8)
-; RV32I-NEXT:    mv a5, s11
+; RV32I-NEXT:    lw a3, %lo(g)(s7)
+; RV32I-NEXT:    lw a2, %lo(f)(s8)
+; RV32I-NEXT:    lw a1, %lo(e)(s10)
+; RV32I-NEXT:    lw a0, %lo(d)(s1)
+; RV32I-NEXT:    addi a5, zero, 32
 ; RV32I-NEXT:    call foo
 ; RV32I-NEXT:  .LBB0_10: # %for.inc
 ; RV32I-NEXT:    # in Loop: Header=BB0_2 Depth=1
-; RV32I-NEXT:    lw a0, %lo(a)(s3)
+; RV32I-NEXT:    lw a0, %lo(a)(s9)
 ; RV32I-NEXT:    addi a0, a0, -1
-; RV32I-NEXT:    sw a0, %lo(a)(s3)
+; RV32I-NEXT:    sw a0, %lo(a)(s9)
 ; RV32I-NEXT:    bnez a0, .LBB0_2
 ; RV32I-NEXT:  .LBB0_11: # %for.end
 ; RV32I-NEXT:    addi a0, zero, 1
diff --git a/test/CodeGen/RISCV/rv32i-rv64i-float-double.ll b/test/CodeGen/RISCV/rv32i-rv64i-float-double.ll
new file mode 100644
index 0000000000000000000000000000000000000000..8d5b6dd56d7667bbe1a1c723ce280e37c51918b1
--- /dev/null
+++ b/test/CodeGen/RISCV/rv32i-rv64i-float-double.ll
@@ -0,0 +1,86 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv32 -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=RV32IF %s
+; RUN: llc -mtriple=riscv64 -verify-machineinstrs < %s \
+; RUN:   | FileCheck -check-prefix=RV64IF %s
+
+; This file provides a simple sanity check of float and double operations for
+; RV32I and RV64I. This is primarily intended to ensure that custom
+; legalisation or DAG combines aren't incorrectly triggered when the F
+; extension isn't enabled.
+
+; TODO: f32 parameters on RV64 with a soft-float ABI are anyext.
+
+define float @float_test(float %a, float %b) nounwind {
+; RV32IF-LABEL: float_test:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    addi sp, sp, -16
+; RV32IF-NEXT:    sw ra, 12(sp)
+; RV32IF-NEXT:    sw s1, 8(sp)
+; RV32IF-NEXT:    mv s1, a1
+; RV32IF-NEXT:    call __addsf3
+; RV32IF-NEXT:    mv a1, s1
+; RV32IF-NEXT:    call __divsf3
+; RV32IF-NEXT:    lw s1, 8(sp)
+; RV32IF-NEXT:    lw ra, 12(sp)
+; RV32IF-NEXT:    addi sp, sp, 16
+; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: float_test:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    addi sp, sp, -16
+; RV64IF-NEXT:    sd ra, 8(sp)
+; RV64IF-NEXT:    sd s1, 0(sp)
+; RV64IF-NEXT:    slli a0, a0, 32
+; RV64IF-NEXT:    srli a0, a0, 32
+; RV64IF-NEXT:    slli a1, a1, 32
+; RV64IF-NEXT:    srli s1, a1, 32
+; RV64IF-NEXT:    mv a1, s1
+; RV64IF-NEXT:    call __addsf3
+; RV64IF-NEXT:    mv a1, s1
+; RV64IF-NEXT:    call __divsf3
+; RV64IF-NEXT:    ld s1, 0(sp)
+; RV64IF-NEXT:    ld ra, 8(sp)
+; RV64IF-NEXT:    addi sp, sp, 16
+; RV64IF-NEXT:    ret
+  %1 = fadd float %a, %b
+  %2 = fdiv float %1, %b
+  ret float %2
+}
+
+define double @double_test(double %a, double %b) nounwind {
+; RV32IF-LABEL: double_test:
+; RV32IF:       # %bb.0:
+; RV32IF-NEXT:    addi sp, sp, -16
+; RV32IF-NEXT:    sw ra, 12(sp)
+; RV32IF-NEXT:    sw s1, 8(sp)
+; RV32IF-NEXT:    sw s2, 4(sp)
+; RV32IF-NEXT:    mv s2, a3
+; RV32IF-NEXT:    mv s1, a2
+; RV32IF-NEXT:    call __adddf3
+; RV32IF-NEXT:    mv a2, s1
+; RV32IF-NEXT:    mv a3, s2
+; RV32IF-NEXT:    call __divdf3
+; RV32IF-NEXT:    lw s2, 4(sp)
+; RV32IF-NEXT:    lw s1, 8(sp)
+; RV32IF-NEXT:    lw ra, 12(sp)
+; RV32IF-NEXT:    addi sp, sp, 16
+; RV32IF-NEXT:    ret
+;
+; RV64IF-LABEL: double_test:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    addi sp, sp, -16
+; RV64IF-NEXT:    sd ra, 8(sp)
+; RV64IF-NEXT:    sd s1, 0(sp)
+; RV64IF-NEXT:    mv s1, a1
+; RV64IF-NEXT:    call __adddf3
+; RV64IF-NEXT:    mv a1, s1
+; RV64IF-NEXT:    call __divdf3
+; RV64IF-NEXT:    ld s1, 0(sp)
+; RV64IF-NEXT:    ld ra, 8(sp)
+; RV64IF-NEXT:    addi sp, sp, 16
+; RV64IF-NEXT:    ret
+  %1 = fadd double %a, %b
+  %2 = fdiv double %1, %b
+  ret double %2
+}
diff --git a/test/CodeGen/RISCV/rv64d-double-convert.ll b/test/CodeGen/RISCV/rv64d-double-convert.ll
new file mode 100644
index 0000000000000000000000000000000000000000..da3fcf678e364961e948f4364fa43b9d65287d33
--- /dev/null
+++ b/test/CodeGen/RISCV/rv64d-double-convert.ll
@@ -0,0 +1,130 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv64 -mattr=+d -verify-machineinstrs < %s \
+; RUN:   | FileCheck %s -check-prefix=RV64ID
+
+; This file exhaustively checks double<->i32 conversions. In general,
+; fcvt.l[u].d can be selected instead of fcvt.w[u].d because poison is
+; generated for an fpto[s|u]i conversion if the result doesn't fit in the
+; target type.
+
+define i32 @aext_fptosi(double %a) nounwind {
+; RV64ID-LABEL: aext_fptosi:
+; RV64ID:       # %bb.0:
+; RV64ID-NEXT:    fmv.d.x ft0, a0
+; RV64ID-NEXT:    fcvt.l.d a0, ft0, rtz
+; RV64ID-NEXT:    ret
+  %1 = fptosi double %a to i32
+  ret i32 %1
+}
+
+define signext i32 @sext_fptosi(double %a) nounwind {
+; RV64ID-LABEL: sext_fptosi:
+; RV64ID:       # %bb.0:
+; RV64ID-NEXT:    fmv.d.x ft0, a0
+; RV64ID-NEXT:    fcvt.l.d a0, ft0, rtz
+; RV64ID-NEXT:    ret
+  %1 = fptosi double %a to i32
+  ret i32 %1
+}
+
+define zeroext i32 @zext_fptosi(double %a) nounwind {
+; RV64ID-LABEL: zext_fptosi:
+; RV64ID:       # %bb.0:
+; RV64ID-NEXT:    fmv.d.x ft0, a0
+; RV64ID-NEXT:    fcvt.l.d a0, ft0, rtz
+; RV64ID-NEXT:    slli a0, a0, 32
+; RV64ID-NEXT:    srli a0, a0, 32
+; RV64ID-NEXT:    ret
+  %1 = fptosi double %a to i32
+  ret i32 %1
+}
+
+define i32 @aext_fptoui(double %a) nounwind {
+; RV64ID-LABEL: aext_fptoui:
+; RV64ID:       # %bb.0:
+; RV64ID-NEXT:    fmv.d.x ft0, a0
+; RV64ID-NEXT:    fcvt.lu.d a0, ft0, rtz
+; RV64ID-NEXT:    ret
+  %1 = fptoui double %a to i32
+  ret i32 %1
+}
+
+define signext i32 @sext_fptoui(double %a) nounwind {
+; RV64ID-LABEL: sext_fptoui:
+; RV64ID:       # %bb.0:
+; RV64ID-NEXT:    fmv.d.x ft0, a0
+; RV64ID-NEXT:    fcvt.wu.d a0, ft0, rtz
+; RV64ID-NEXT:    ret
+  %1 = fptoui double %a to i32
+  ret i32 %1
+}
+
+define zeroext i32 @zext_fptoui(double %a) nounwind {
+; RV64ID-LABEL: zext_fptoui:
+; RV64ID:       # %bb.0:
+; RV64ID-NEXT:    fmv.d.x ft0, a0
+; RV64ID-NEXT:    fcvt.lu.d a0, ft0, rtz
+; RV64ID-NEXT:    ret
+  %1 = fptoui double %a to i32
+  ret i32 %1
+}
+
+define double @uitofp_aext_i32_to_f64(i32 %a) nounwind {
+; RV64ID-LABEL: uitofp_aext_i32_to_f64:
+; RV64ID:       # %bb.0:
+; RV64ID-NEXT:    fcvt.d.wu ft0, a0
+; RV64ID-NEXT:    fmv.x.d a0, ft0
+; RV64ID-NEXT:    ret
+  %1 = uitofp i32 %a to double
+  ret double %1
+}
+
+define double @uitofp_sext_i32_to_f64(i32 signext %a) nounwind {
+; RV64ID-LABEL: uitofp_sext_i32_to_f64:
+; RV64ID:       # %bb.0:
+; RV64ID-NEXT:    fcvt.d.wu ft0, a0
+; RV64ID-NEXT:    fmv.x.d a0, ft0
+; RV64ID-NEXT:    ret
+  %1 = uitofp i32 %a to double
+  ret double %1
+}
+
+define double @uitofp_zext_i32_to_f64(i32 zeroext %a) nounwind {
+; RV64ID-LABEL: uitofp_zext_i32_to_f64:
+; RV64ID:       # %bb.0:
+; RV64ID-NEXT:    fcvt.d.wu ft0, a0
+; RV64ID-NEXT:    fmv.x.d a0, ft0
+; RV64ID-NEXT:    ret
+  %1 = uitofp i32 %a to double
+  ret double %1
+}
+
+define double @sitofp_aext_i32_to_f64(i32 %a) nounwind {
+; RV64ID-LABEL: sitofp_aext_i32_to_f64:
+; RV64ID:       # %bb.0:
+; RV64ID-NEXT:    fcvt.d.w ft0, a0
+; RV64ID-NEXT:    fmv.x.d a0, ft0
+; RV64ID-NEXT:    ret
+  %1 = sitofp i32 %a to double
+  ret double %1
+}
+
+define double @sitofp_sext_i32_to_f64(i32 signext %a) nounwind {
+; RV64ID-LABEL: sitofp_sext_i32_to_f64:
+; RV64ID:       # %bb.0:
+; RV64ID-NEXT:    fcvt.d.l ft0, a0
+; RV64ID-NEXT:    fmv.x.d a0, ft0
+; RV64ID-NEXT:    ret
+  %1 = sitofp i32 %a to double
+  ret double %1
+}
+
+define double @sitofp_zext_i32_to_f64(i32 zeroext %a) nounwind {
+; RV64ID-LABEL: sitofp_zext_i32_to_f64:
+; RV64ID:       # %bb.0:
+; RV64ID-NEXT:    fcvt.d.w ft0, a0
+; RV64ID-NEXT:    fmv.x.d a0, ft0
+; RV64ID-NEXT:    ret
+  %1 = sitofp i32 %a to double
+  ret double %1
+}
diff --git a/test/CodeGen/RISCV/rv64f-float-convert.ll b/test/CodeGen/RISCV/rv64f-float-convert.ll
new file mode 100644
index 0000000000000000000000000000000000000000..5f8952085ae89242573b6ff3bb89741b4eb9b364
--- /dev/null
+++ b/test/CodeGen/RISCV/rv64f-float-convert.ll
@@ -0,0 +1,187 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv64 -mattr=+f -verify-machineinstrs < %s \
+; RUN:   | FileCheck %s -check-prefix=RV64IF
+
+; This file exhaustively checks float<->i32 conversions. In general,
+; fcvt.l[u].s can be selected instead of fcvt.w[u].s because poison is
+; generated for an fpto[s|u]i conversion if the result doesn't fit in the
+; target type.
+
+define i32 @aext_fptosi(float %a) nounwind {
+; RV64IF-LABEL: aext_fptosi:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fmv.w.x ft0, a0
+; RV64IF-NEXT:    fcvt.l.s a0, ft0, rtz
+; RV64IF-NEXT:    ret
+  %1 = fptosi float %a to i32
+  ret i32 %1
+}
+
+define signext i32 @sext_fptosi(float %a) nounwind {
+; RV64IF-LABEL: sext_fptosi:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fmv.w.x ft0, a0
+; RV64IF-NEXT:    fcvt.l.s a0, ft0, rtz
+; RV64IF-NEXT:    ret
+  %1 = fptosi float %a to i32
+  ret i32 %1
+}
+
+define zeroext i32 @zext_fptosi(float %a) nounwind {
+; RV64IF-LABEL: zext_fptosi:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fmv.w.x ft0, a0
+; RV64IF-NEXT:    fcvt.l.s a0, ft0, rtz
+; RV64IF-NEXT:    slli a0, a0, 32
+; RV64IF-NEXT:    srli a0, a0, 32
+; RV64IF-NEXT:    ret
+  %1 = fptosi float %a to i32
+  ret i32 %1
+}
+
+define i32 @aext_fptoui(float %a) nounwind {
+; RV64IF-LABEL: aext_fptoui:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fmv.w.x ft0, a0
+; RV64IF-NEXT:    fcvt.lu.s a0, ft0, rtz
+; RV64IF-NEXT:    ret
+  %1 = fptoui float %a to i32
+  ret i32 %1
+}
+
+define signext i32 @sext_fptoui(float %a) nounwind {
+; RV64IF-LABEL: sext_fptoui:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fmv.w.x ft0, a0
+; RV64IF-NEXT:    fcvt.wu.s a0, ft0, rtz
+; RV64IF-NEXT:    ret
+  %1 = fptoui float %a to i32
+  ret i32 %1
+}
+
+define zeroext i32 @zext_fptoui(float %a) nounwind {
+; RV64IF-LABEL: zext_fptoui:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fmv.w.x ft0, a0
+; RV64IF-NEXT:    fcvt.lu.s a0, ft0, rtz
+; RV64IF-NEXT:    ret
+  %1 = fptoui float %a to i32
+  ret i32 %1
+}
+
+define i32 @bcvt_f32_to_aext_i32(float %a, float %b) nounwind {
+; RV64IF-LABEL: bcvt_f32_to_aext_i32:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fmv.w.x ft0, a1
+; RV64IF-NEXT:    fmv.w.x ft1, a0
+; RV64IF-NEXT:    fadd.s ft0, ft1, ft0
+; RV64IF-NEXT:    fmv.x.w a0, ft0
+; RV64IF-NEXT:    ret
+  %1 = fadd float %a, %b
+  %2 = bitcast float %1 to i32
+  ret i32 %2
+}
+
+define signext i32 @bcvt_f32_to_sext_i32(float %a, float %b) nounwind {
+; RV64IF-LABEL: bcvt_f32_to_sext_i32:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fmv.w.x ft0, a1
+; RV64IF-NEXT:    fmv.w.x ft1, a0
+; RV64IF-NEXT:    fadd.s ft0, ft1, ft0
+; RV64IF-NEXT:    fmv.x.w a0, ft0
+; RV64IF-NEXT:    ret
+  %1 = fadd float %a, %b
+  %2 = bitcast float %1 to i32
+  ret i32 %2
+}
+
+define zeroext i32 @bcvt_f32_to_zext_i32(float %a, float %b) nounwind {
+; RV64IF-LABEL: bcvt_f32_to_zext_i32:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fmv.w.x ft0, a1
+; RV64IF-NEXT:    fmv.w.x ft1, a0
+; RV64IF-NEXT:    fadd.s ft0, ft1, ft0
+; RV64IF-NEXT:    fmv.x.w a0, ft0
+; RV64IF-NEXT:    slli a0, a0, 32
+; RV64IF-NEXT:    srli a0, a0, 32
+; RV64IF-NEXT:    ret
+  %1 = fadd float %a, %b
+  %2 = bitcast float %1 to i32
+  ret i32 %2
+}
+
+define float @bcvt_i64_to_f32_via_i32(i64 %a, i64 %b) nounwind {
+; RV64IF-LABEL: bcvt_i64_to_f32_via_i32:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fmv.w.x ft0, a1
+; RV64IF-NEXT:    fmv.w.x ft1, a0
+; RV64IF-NEXT:    fadd.s ft0, ft1, ft0
+; RV64IF-NEXT:    fmv.x.w a0, ft0
+; RV64IF-NEXT:    ret
+  %1 = trunc i64 %a to i32
+  %2 = trunc i64 %b to i32
+  %3 = bitcast i32 %1 to float
+  %4 = bitcast i32 %2 to float
+  %5 = fadd float %3, %4
+  ret float %5
+}
+
+define float @uitofp_aext_i32_to_f32(i32 %a) nounwind {
+; RV64IF-LABEL: uitofp_aext_i32_to_f32:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fcvt.s.wu ft0, a0
+; RV64IF-NEXT:    fmv.x.w a0, ft0
+; RV64IF-NEXT:    ret
+  %1 = uitofp i32 %a to float
+  ret float %1
+}
+
+define float @uitofp_sext_i32_to_f32(i32 signext %a) nounwind {
+; RV64IF-LABEL: uitofp_sext_i32_to_f32:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fcvt.s.wu ft0, a0
+; RV64IF-NEXT:    fmv.x.w a0, ft0
+; RV64IF-NEXT:    ret
+  %1 = uitofp i32 %a to float
+  ret float %1
+}
+
+define float @uitofp_zext_i32_to_f32(i32 zeroext %a) nounwind {
+; RV64IF-LABEL: uitofp_zext_i32_to_f32:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fcvt.s.wu ft0, a0
+; RV64IF-NEXT:    fmv.x.w a0, ft0
+; RV64IF-NEXT:    ret
+  %1 = uitofp i32 %a to float
+  ret float %1
+}
+
+define float @sitofp_aext_i32_to_f32(i32 %a) nounwind {
+; RV64IF-LABEL: sitofp_aext_i32_to_f32:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fcvt.s.w ft0, a0
+; RV64IF-NEXT:    fmv.x.w a0, ft0
+; RV64IF-NEXT:    ret
+  %1 = sitofp i32 %a to float
+  ret float %1
+}
+
+define float @sitofp_sext_i32_to_f32(i32 signext %a) nounwind {
+; RV64IF-LABEL: sitofp_sext_i32_to_f32:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fcvt.s.l ft0, a0
+; RV64IF-NEXT:    fmv.x.w a0, ft0
+; RV64IF-NEXT:    ret
+  %1 = sitofp i32 %a to float
+  ret float %1
+}
+
+define float @sitofp_zext_i32_to_f32(i32 zeroext %a) nounwind {
+; RV64IF-LABEL: sitofp_zext_i32_to_f32:
+; RV64IF:       # %bb.0:
+; RV64IF-NEXT:    fcvt.s.w ft0, a0
+; RV64IF-NEXT:    fmv.x.w a0, ft0
+; RV64IF-NEXT:    ret
+  %1 = sitofp i32 %a to float
+  ret float %1
+}
diff --git a/test/CodeGen/RISCV/rv64i-exhaustive-w-insts.ll b/test/CodeGen/RISCV/rv64i-exhaustive-w-insts.ll
index 52a59c04a3253cba4973c8508266d12e478449bd..b7f513d6d22fd4ca558368290866c03393cb3892 100644
--- a/test/CodeGen/RISCV/rv64i-exhaustive-w-insts.ll
+++ b/test/CodeGen/RISCV/rv64i-exhaustive-w-insts.ll
@@ -546,7 +546,7 @@ define zeroext i32 @zext_subw_zext_zext(i32 zeroext %a, i32 zeroext %b) nounwind
 define i32 @aext_sllw_aext_aext(i32 %a, i32 %b) nounwind {
 ; RV64I-LABEL: aext_sllw_aext_aext:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    sll a0, a0, a1
+; RV64I-NEXT:    sllw a0, a0, a1
 ; RV64I-NEXT:    ret
   %1 = shl i32 %a, %b
   ret i32 %1
@@ -555,7 +555,7 @@ define i32 @aext_sllw_aext_aext(i32 %a, i32 %b) nounwind {
 define i32 @aext_sllw_aext_sext(i32 %a, i32 signext %b) nounwind {
 ; RV64I-LABEL: aext_sllw_aext_sext:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    sll a0, a0, a1
+; RV64I-NEXT:    sllw a0, a0, a1
 ; RV64I-NEXT:    ret
   %1 = shl i32 %a, %b
   ret i32 %1
@@ -564,7 +564,7 @@ define i32 @aext_sllw_aext_sext(i32 %a, i32 signext %b) nounwind {
 define i32 @aext_sllw_aext_zext(i32 %a, i32 zeroext %b) nounwind {
 ; RV64I-LABEL: aext_sllw_aext_zext:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    sll a0, a0, a1
+; RV64I-NEXT:    sllw a0, a0, a1
 ; RV64I-NEXT:    ret
   %1 = shl i32 %a, %b
   ret i32 %1
@@ -573,7 +573,7 @@ define i32 @aext_sllw_aext_zext(i32 %a, i32 zeroext %b) nounwind {
 define i32 @aext_sllw_sext_aext(i32 signext %a, i32 %b) nounwind {
 ; RV64I-LABEL: aext_sllw_sext_aext:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    sll a0, a0, a1
+; RV64I-NEXT:    sllw a0, a0, a1
 ; RV64I-NEXT:    ret
   %1 = shl i32 %a, %b
   ret i32 %1
@@ -582,7 +582,7 @@ define i32 @aext_sllw_sext_aext(i32 signext %a, i32 %b) nounwind {
 define i32 @aext_sllw_sext_sext(i32 signext %a, i32 signext %b) nounwind {
 ; RV64I-LABEL: aext_sllw_sext_sext:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    sll a0, a0, a1
+; RV64I-NEXT:    sllw a0, a0, a1
 ; RV64I-NEXT:    ret
   %1 = shl i32 %a, %b
   ret i32 %1
@@ -591,7 +591,7 @@ define i32 @aext_sllw_sext_sext(i32 signext %a, i32 signext %b) nounwind {
 define i32 @aext_sllw_sext_zext(i32 signext %a, i32 zeroext %b) nounwind {
 ; RV64I-LABEL: aext_sllw_sext_zext:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    sll a0, a0, a1
+; RV64I-NEXT:    sllw a0, a0, a1
 ; RV64I-NEXT:    ret
   %1 = shl i32 %a, %b
   ret i32 %1
@@ -600,7 +600,7 @@ define i32 @aext_sllw_sext_zext(i32 signext %a, i32 zeroext %b) nounwind {
 define i32 @aext_sllw_zext_aext(i32 zeroext %a, i32 %b) nounwind {
 ; RV64I-LABEL: aext_sllw_zext_aext:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    sll a0, a0, a1
+; RV64I-NEXT:    sllw a0, a0, a1
 ; RV64I-NEXT:    ret
   %1 = shl i32 %a, %b
   ret i32 %1
@@ -609,7 +609,7 @@ define i32 @aext_sllw_zext_aext(i32 zeroext %a, i32 %b) nounwind {
 define i32 @aext_sllw_zext_sext(i32 zeroext %a, i32 signext %b) nounwind {
 ; RV64I-LABEL: aext_sllw_zext_sext:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    sll a0, a0, a1
+; RV64I-NEXT:    sllw a0, a0, a1
 ; RV64I-NEXT:    ret
   %1 = shl i32 %a, %b
   ret i32 %1
@@ -618,19 +618,16 @@ define i32 @aext_sllw_zext_sext(i32 zeroext %a, i32 signext %b) nounwind {
 define i32 @aext_sllw_zext_zext(i32 zeroext %a, i32 zeroext %b) nounwind {
 ; RV64I-LABEL: aext_sllw_zext_zext:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    sll a0, a0, a1
+; RV64I-NEXT:    sllw a0, a0, a1
 ; RV64I-NEXT:    ret
   %1 = shl i32 %a, %b
   ret i32 %1
 }
 
-; TODO: Select sllw for all cases witha signext result.
-
 define signext i32 @sext_sllw_aext_aext(i32 %a, i32 %b) nounwind {
 ; RV64I-LABEL: sext_sllw_aext_aext:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    sll a0, a0, a1
-; RV64I-NEXT:    sext.w a0, a0
+; RV64I-NEXT:    sllw a0, a0, a1
 ; RV64I-NEXT:    ret
   %1 = shl i32 %a, %b
   ret i32 %1
@@ -639,8 +636,7 @@ define signext i32 @sext_sllw_aext_aext(i32 %a, i32 %b) nounwind {
 define signext i32 @sext_sllw_aext_sext(i32 %a, i32 signext %b) nounwind {
 ; RV64I-LABEL: sext_sllw_aext_sext:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    sll a0, a0, a1
-; RV64I-NEXT:    sext.w a0, a0
+; RV64I-NEXT:    sllw a0, a0, a1
 ; RV64I-NEXT:    ret
   %1 = shl i32 %a, %b
   ret i32 %1
@@ -649,8 +645,7 @@ define signext i32 @sext_sllw_aext_sext(i32 %a, i32 signext %b) nounwind {
 define signext i32 @sext_sllw_aext_zext(i32 %a, i32 zeroext %b) nounwind {
 ; RV64I-LABEL: sext_sllw_aext_zext:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    sll a0, a0, a1
-; RV64I-NEXT:    sext.w a0, a0
+; RV64I-NEXT:    sllw a0, a0, a1
 ; RV64I-NEXT:    ret
   %1 = shl i32 %a, %b
   ret i32 %1
@@ -659,8 +654,7 @@ define signext i32 @sext_sllw_aext_zext(i32 %a, i32 zeroext %b) nounwind {
 define signext i32 @sext_sllw_sext_aext(i32 signext %a, i32 %b) nounwind {
 ; RV64I-LABEL: sext_sllw_sext_aext:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    sll a0, a0, a1
-; RV64I-NEXT:    sext.w a0, a0
+; RV64I-NEXT:    sllw a0, a0, a1
 ; RV64I-NEXT:    ret
   %1 = shl i32 %a, %b
   ret i32 %1
@@ -669,8 +663,7 @@ define signext i32 @sext_sllw_sext_aext(i32 signext %a, i32 %b) nounwind {
 define signext i32 @sext_sllw_sext_sext(i32 signext %a, i32 signext %b) nounwind {
 ; RV64I-LABEL: sext_sllw_sext_sext:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    sll a0, a0, a1
-; RV64I-NEXT:    sext.w a0, a0
+; RV64I-NEXT:    sllw a0, a0, a1
 ; RV64I-NEXT:    ret
   %1 = shl i32 %a, %b
   ret i32 %1
@@ -679,8 +672,7 @@ define signext i32 @sext_sllw_sext_sext(i32 signext %a, i32 signext %b) nounwind
 define signext i32 @sext_sllw_sext_zext(i32 signext %a, i32 zeroext %b) nounwind {
 ; RV64I-LABEL: sext_sllw_sext_zext:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    sll a0, a0, a1
-; RV64I-NEXT:    sext.w a0, a0
+; RV64I-NEXT:    sllw a0, a0, a1
 ; RV64I-NEXT:    ret
   %1 = shl i32 %a, %b
   ret i32 %1
@@ -689,8 +681,7 @@ define signext i32 @sext_sllw_sext_zext(i32 signext %a, i32 zeroext %b) nounwind
 define signext i32 @sext_sllw_zext_aext(i32 zeroext %a, i32 %b) nounwind {
 ; RV64I-LABEL: sext_sllw_zext_aext:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    sll a0, a0, a1
-; RV64I-NEXT:    sext.w a0, a0
+; RV64I-NEXT:    sllw a0, a0, a1
 ; RV64I-NEXT:    ret
   %1 = shl i32 %a, %b
   ret i32 %1
@@ -699,8 +690,7 @@ define signext i32 @sext_sllw_zext_aext(i32 zeroext %a, i32 %b) nounwind {
 define signext i32 @sext_sllw_zext_sext(i32 zeroext %a, i32 signext %b) nounwind {
 ; RV64I-LABEL: sext_sllw_zext_sext:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    sll a0, a0, a1
-; RV64I-NEXT:    sext.w a0, a0
+; RV64I-NEXT:    sllw a0, a0, a1
 ; RV64I-NEXT:    ret
   %1 = shl i32 %a, %b
   ret i32 %1
@@ -709,8 +699,7 @@ define signext i32 @sext_sllw_zext_sext(i32 zeroext %a, i32 signext %b) nounwind
 define signext i32 @sext_sllw_zext_zext(i32 zeroext %a, i32 zeroext %b) nounwind {
 ; RV64I-LABEL: sext_sllw_zext_zext:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    sll a0, a0, a1
-; RV64I-NEXT:    sext.w a0, a0
+; RV64I-NEXT:    sllw a0, a0, a1
 ; RV64I-NEXT:    ret
   %1 = shl i32 %a, %b
   ret i32 %1
@@ -721,7 +710,7 @@ define signext i32 @sext_sllw_zext_zext(i32 zeroext %a, i32 zeroext %b) nounwind
 define zeroext i32 @zext_sllw_aext_aext(i32 %a, i32 %b) nounwind {
 ; RV64I-LABEL: zext_sllw_aext_aext:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    sll a0, a0, a1
+; RV64I-NEXT:    sllw a0, a0, a1
 ; RV64I-NEXT:    slli a0, a0, 32
 ; RV64I-NEXT:    srli a0, a0, 32
 ; RV64I-NEXT:    ret
@@ -732,7 +721,7 @@ define zeroext i32 @zext_sllw_aext_aext(i32 %a, i32 %b) nounwind {
 define zeroext i32 @zext_sllw_aext_sext(i32 %a, i32 signext %b) nounwind {
 ; RV64I-LABEL: zext_sllw_aext_sext:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    sll a0, a0, a1
+; RV64I-NEXT:    sllw a0, a0, a1
 ; RV64I-NEXT:    slli a0, a0, 32
 ; RV64I-NEXT:    srli a0, a0, 32
 ; RV64I-NEXT:    ret
@@ -743,7 +732,7 @@ define zeroext i32 @zext_sllw_aext_sext(i32 %a, i32 signext %b) nounwind {
 define zeroext i32 @zext_sllw_aext_zext(i32 %a, i32 zeroext %b) nounwind {
 ; RV64I-LABEL: zext_sllw_aext_zext:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    sll a0, a0, a1
+; RV64I-NEXT:    sllw a0, a0, a1
 ; RV64I-NEXT:    slli a0, a0, 32
 ; RV64I-NEXT:    srli a0, a0, 32
 ; RV64I-NEXT:    ret
@@ -754,7 +743,7 @@ define zeroext i32 @zext_sllw_aext_zext(i32 %a, i32 zeroext %b) nounwind {
 define zeroext i32 @zext_sllw_sext_aext(i32 signext %a, i32 %b) nounwind {
 ; RV64I-LABEL: zext_sllw_sext_aext:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    sll a0, a0, a1
+; RV64I-NEXT:    sllw a0, a0, a1
 ; RV64I-NEXT:    slli a0, a0, 32
 ; RV64I-NEXT:    srli a0, a0, 32
 ; RV64I-NEXT:    ret
@@ -765,7 +754,7 @@ define zeroext i32 @zext_sllw_sext_aext(i32 signext %a, i32 %b) nounwind {
 define zeroext i32 @zext_sllw_sext_sext(i32 signext %a, i32 signext %b) nounwind {
 ; RV64I-LABEL: zext_sllw_sext_sext:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    sll a0, a0, a1
+; RV64I-NEXT:    sllw a0, a0, a1
 ; RV64I-NEXT:    slli a0, a0, 32
 ; RV64I-NEXT:    srli a0, a0, 32
 ; RV64I-NEXT:    ret
@@ -776,7 +765,7 @@ define zeroext i32 @zext_sllw_sext_sext(i32 signext %a, i32 signext %b) nounwind
 define zeroext i32 @zext_sllw_sext_zext(i32 signext %a, i32 zeroext %b) nounwind {
 ; RV64I-LABEL: zext_sllw_sext_zext:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    sll a0, a0, a1
+; RV64I-NEXT:    sllw a0, a0, a1
 ; RV64I-NEXT:    slli a0, a0, 32
 ; RV64I-NEXT:    srli a0, a0, 32
 ; RV64I-NEXT:    ret
@@ -787,7 +776,7 @@ define zeroext i32 @zext_sllw_sext_zext(i32 signext %a, i32 zeroext %b) nounwind
 define zeroext i32 @zext_sllw_zext_aext(i32 zeroext %a, i32 %b) nounwind {
 ; RV64I-LABEL: zext_sllw_zext_aext:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    sll a0, a0, a1
+; RV64I-NEXT:    sllw a0, a0, a1
 ; RV64I-NEXT:    slli a0, a0, 32
 ; RV64I-NEXT:    srli a0, a0, 32
 ; RV64I-NEXT:    ret
@@ -798,7 +787,7 @@ define zeroext i32 @zext_sllw_zext_aext(i32 zeroext %a, i32 %b) nounwind {
 define zeroext i32 @zext_sllw_zext_sext(i32 zeroext %a, i32 signext %b) nounwind {
 ; RV64I-LABEL: zext_sllw_zext_sext:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    sll a0, a0, a1
+; RV64I-NEXT:    sllw a0, a0, a1
 ; RV64I-NEXT:    slli a0, a0, 32
 ; RV64I-NEXT:    srli a0, a0, 32
 ; RV64I-NEXT:    ret
@@ -809,7 +798,7 @@ define zeroext i32 @zext_sllw_zext_sext(i32 zeroext %a, i32 signext %b) nounwind
 define zeroext i32 @zext_sllw_zext_zext(i32 zeroext %a, i32 zeroext %b) nounwind {
 ; RV64I-LABEL: zext_sllw_zext_zext:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    sll a0, a0, a1
+; RV64I-NEXT:    sllw a0, a0, a1
 ; RV64I-NEXT:    slli a0, a0, 32
 ; RV64I-NEXT:    srli a0, a0, 32
 ; RV64I-NEXT:    ret
@@ -817,14 +806,10 @@ define zeroext i32 @zext_sllw_zext_zext(i32 zeroext %a, i32 zeroext %b) nounwind
   ret i32 %1
 }
 
-; TODO: srlw should be selected for 32-bit lshr with variable arguments.
-
 define i32 @aext_srlw_aext_aext(i32 %a, i32 %b) nounwind {
 ; RV64I-LABEL: aext_srlw_aext_aext:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    slli a0, a0, 32
-; RV64I-NEXT:    srli a0, a0, 32
-; RV64I-NEXT:    srl a0, a0, a1
+; RV64I-NEXT:    srlw a0, a0, a1
 ; RV64I-NEXT:    ret
   %1 = lshr i32 %a, %b
   ret i32 %1
@@ -833,9 +818,7 @@ define i32 @aext_srlw_aext_aext(i32 %a, i32 %b) nounwind {
 define i32 @aext_srlw_aext_sext(i32 %a, i32 signext %b) nounwind {
 ; RV64I-LABEL: aext_srlw_aext_sext:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    slli a0, a0, 32
-; RV64I-NEXT:    srli a0, a0, 32
-; RV64I-NEXT:    srl a0, a0, a1
+; RV64I-NEXT:    srlw a0, a0, a1
 ; RV64I-NEXT:    ret
   %1 = lshr i32 %a, %b
   ret i32 %1
@@ -844,9 +827,7 @@ define i32 @aext_srlw_aext_sext(i32 %a, i32 signext %b) nounwind {
 define i32 @aext_srlw_aext_zext(i32 %a, i32 zeroext %b) nounwind {
 ; RV64I-LABEL: aext_srlw_aext_zext:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    slli a0, a0, 32
-; RV64I-NEXT:    srli a0, a0, 32
-; RV64I-NEXT:    srl a0, a0, a1
+; RV64I-NEXT:    srlw a0, a0, a1
 ; RV64I-NEXT:    ret
   %1 = lshr i32 %a, %b
   ret i32 %1
@@ -855,9 +836,7 @@ define i32 @aext_srlw_aext_zext(i32 %a, i32 zeroext %b) nounwind {
 define i32 @aext_srlw_sext_aext(i32 signext %a, i32 %b) nounwind {
 ; RV64I-LABEL: aext_srlw_sext_aext:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    slli a0, a0, 32
-; RV64I-NEXT:    srli a0, a0, 32
-; RV64I-NEXT:    srl a0, a0, a1
+; RV64I-NEXT:    srlw a0, a0, a1
 ; RV64I-NEXT:    ret
   %1 = lshr i32 %a, %b
   ret i32 %1
@@ -866,9 +845,7 @@ define i32 @aext_srlw_sext_aext(i32 signext %a, i32 %b) nounwind {
 define i32 @aext_srlw_sext_sext(i32 signext %a, i32 signext %b) nounwind {
 ; RV64I-LABEL: aext_srlw_sext_sext:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    slli a0, a0, 32
-; RV64I-NEXT:    srli a0, a0, 32
-; RV64I-NEXT:    srl a0, a0, a1
+; RV64I-NEXT:    srlw a0, a0, a1
 ; RV64I-NEXT:    ret
   %1 = lshr i32 %a, %b
   ret i32 %1
@@ -877,9 +854,7 @@ define i32 @aext_srlw_sext_sext(i32 signext %a, i32 signext %b) nounwind {
 define i32 @aext_srlw_sext_zext(i32 signext %a, i32 zeroext %b) nounwind {
 ; RV64I-LABEL: aext_srlw_sext_zext:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    slli a0, a0, 32
-; RV64I-NEXT:    srli a0, a0, 32
-; RV64I-NEXT:    srl a0, a0, a1
+; RV64I-NEXT:    srlw a0, a0, a1
 ; RV64I-NEXT:    ret
   %1 = lshr i32 %a, %b
   ret i32 %1
@@ -888,7 +863,7 @@ define i32 @aext_srlw_sext_zext(i32 signext %a, i32 zeroext %b) nounwind {
 define i32 @aext_srlw_zext_aext(i32 zeroext %a, i32 %b) nounwind {
 ; RV64I-LABEL: aext_srlw_zext_aext:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    srl a0, a0, a1
+; RV64I-NEXT:    srlw a0, a0, a1
 ; RV64I-NEXT:    ret
   %1 = lshr i32 %a, %b
   ret i32 %1
@@ -897,7 +872,7 @@ define i32 @aext_srlw_zext_aext(i32 zeroext %a, i32 %b) nounwind {
 define i32 @aext_srlw_zext_sext(i32 zeroext %a, i32 signext %b) nounwind {
 ; RV64I-LABEL: aext_srlw_zext_sext:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    srl a0, a0, a1
+; RV64I-NEXT:    srlw a0, a0, a1
 ; RV64I-NEXT:    ret
   %1 = lshr i32 %a, %b
   ret i32 %1
@@ -906,7 +881,7 @@ define i32 @aext_srlw_zext_sext(i32 zeroext %a, i32 signext %b) nounwind {
 define i32 @aext_srlw_zext_zext(i32 zeroext %a, i32 zeroext %b) nounwind {
 ; RV64I-LABEL: aext_srlw_zext_zext:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    srl a0, a0, a1
+; RV64I-NEXT:    srlw a0, a0, a1
 ; RV64I-NEXT:    ret
   %1 = lshr i32 %a, %b
   ret i32 %1
@@ -915,10 +890,7 @@ define i32 @aext_srlw_zext_zext(i32 zeroext %a, i32 zeroext %b) nounwind {
 define signext i32 @sext_srlw_aext_aext(i32 %a, i32 %b) nounwind {
 ; RV64I-LABEL: sext_srlw_aext_aext:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    slli a0, a0, 32
-; RV64I-NEXT:    srli a0, a0, 32
-; RV64I-NEXT:    srl a0, a0, a1
-; RV64I-NEXT:    sext.w a0, a0
+; RV64I-NEXT:    srlw a0, a0, a1
 ; RV64I-NEXT:    ret
   %1 = lshr i32 %a, %b
   ret i32 %1
@@ -927,10 +899,7 @@ define signext i32 @sext_srlw_aext_aext(i32 %a, i32 %b) nounwind {
 define signext i32 @sext_srlw_aext_sext(i32 %a, i32 signext %b) nounwind {
 ; RV64I-LABEL: sext_srlw_aext_sext:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    slli a0, a0, 32
-; RV64I-NEXT:    srli a0, a0, 32
-; RV64I-NEXT:    srl a0, a0, a1
-; RV64I-NEXT:    sext.w a0, a0
+; RV64I-NEXT:    srlw a0, a0, a1
 ; RV64I-NEXT:    ret
   %1 = lshr i32 %a, %b
   ret i32 %1
@@ -939,10 +908,7 @@ define signext i32 @sext_srlw_aext_sext(i32 %a, i32 signext %b) nounwind {
 define signext i32 @sext_srlw_aext_zext(i32 %a, i32 zeroext %b) nounwind {
 ; RV64I-LABEL: sext_srlw_aext_zext:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    slli a0, a0, 32
-; RV64I-NEXT:    srli a0, a0, 32
-; RV64I-NEXT:    srl a0, a0, a1
-; RV64I-NEXT:    sext.w a0, a0
+; RV64I-NEXT:    srlw a0, a0, a1
 ; RV64I-NEXT:    ret
   %1 = lshr i32 %a, %b
   ret i32 %1
@@ -951,10 +917,7 @@ define signext i32 @sext_srlw_aext_zext(i32 %a, i32 zeroext %b) nounwind {
 define signext i32 @sext_srlw_sext_aext(i32 signext %a, i32 %b) nounwind {
 ; RV64I-LABEL: sext_srlw_sext_aext:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    slli a0, a0, 32
-; RV64I-NEXT:    srli a0, a0, 32
-; RV64I-NEXT:    srl a0, a0, a1
-; RV64I-NEXT:    sext.w a0, a0
+; RV64I-NEXT:    srlw a0, a0, a1
 ; RV64I-NEXT:    ret
   %1 = lshr i32 %a, %b
   ret i32 %1
@@ -963,10 +926,7 @@ define signext i32 @sext_srlw_sext_aext(i32 signext %a, i32 %b) nounwind {
 define signext i32 @sext_srlw_sext_sext(i32 signext %a, i32 signext %b) nounwind {
 ; RV64I-LABEL: sext_srlw_sext_sext:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    slli a0, a0, 32
-; RV64I-NEXT:    srli a0, a0, 32
-; RV64I-NEXT:    srl a0, a0, a1
-; RV64I-NEXT:    sext.w a0, a0
+; RV64I-NEXT:    srlw a0, a0, a1
 ; RV64I-NEXT:    ret
   %1 = lshr i32 %a, %b
   ret i32 %1
@@ -975,10 +935,7 @@ define signext i32 @sext_srlw_sext_sext(i32 signext %a, i32 signext %b) nounwind
 define signext i32 @sext_srlw_sext_zext(i32 signext %a, i32 zeroext %b) nounwind {
 ; RV64I-LABEL: sext_srlw_sext_zext:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    slli a0, a0, 32
-; RV64I-NEXT:    srli a0, a0, 32
-; RV64I-NEXT:    srl a0, a0, a1
-; RV64I-NEXT:    sext.w a0, a0
+; RV64I-NEXT:    srlw a0, a0, a1
 ; RV64I-NEXT:    ret
   %1 = lshr i32 %a, %b
   ret i32 %1
@@ -987,8 +944,7 @@ define signext i32 @sext_srlw_sext_zext(i32 signext %a, i32 zeroext %b) nounwind
 define signext i32 @sext_srlw_zext_aext(i32 zeroext %a, i32 %b) nounwind {
 ; RV64I-LABEL: sext_srlw_zext_aext:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    srl a0, a0, a1
-; RV64I-NEXT:    sext.w a0, a0
+; RV64I-NEXT:    srlw a0, a0, a1
 ; RV64I-NEXT:    ret
   %1 = lshr i32 %a, %b
   ret i32 %1
@@ -997,8 +953,7 @@ define signext i32 @sext_srlw_zext_aext(i32 zeroext %a, i32 %b) nounwind {
 define signext i32 @sext_srlw_zext_sext(i32 zeroext %a, i32 signext %b) nounwind {
 ; RV64I-LABEL: sext_srlw_zext_sext:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    srl a0, a0, a1
-; RV64I-NEXT:    sext.w a0, a0
+; RV64I-NEXT:    srlw a0, a0, a1
 ; RV64I-NEXT:    ret
   %1 = lshr i32 %a, %b
   ret i32 %1
@@ -1007,8 +962,7 @@ define signext i32 @sext_srlw_zext_sext(i32 zeroext %a, i32 signext %b) nounwind
 define signext i32 @sext_srlw_zext_zext(i32 zeroext %a, i32 zeroext %b) nounwind {
 ; RV64I-LABEL: sext_srlw_zext_zext:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    srl a0, a0, a1
-; RV64I-NEXT:    sext.w a0, a0
+; RV64I-NEXT:    srlw a0, a0, a1
 ; RV64I-NEXT:    ret
   %1 = lshr i32 %a, %b
   ret i32 %1
@@ -1017,9 +971,7 @@ define signext i32 @sext_srlw_zext_zext(i32 zeroext %a, i32 zeroext %b) nounwind
 define zeroext i32 @zext_srlw_aext_aext(i32 %a, i32 %b) nounwind {
 ; RV64I-LABEL: zext_srlw_aext_aext:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    slli a0, a0, 32
-; RV64I-NEXT:    srli a0, a0, 32
-; RV64I-NEXT:    srl a0, a0, a1
+; RV64I-NEXT:    srlw a0, a0, a1
 ; RV64I-NEXT:    slli a0, a0, 32
 ; RV64I-NEXT:    srli a0, a0, 32
 ; RV64I-NEXT:    ret
@@ -1030,9 +982,7 @@ define zeroext i32 @zext_srlw_aext_aext(i32 %a, i32 %b) nounwind {
 define zeroext i32 @zext_srlw_aext_sext(i32 %a, i32 signext %b) nounwind {
 ; RV64I-LABEL: zext_srlw_aext_sext:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    slli a0, a0, 32
-; RV64I-NEXT:    srli a0, a0, 32
-; RV64I-NEXT:    srl a0, a0, a1
+; RV64I-NEXT:    srlw a0, a0, a1
 ; RV64I-NEXT:    slli a0, a0, 32
 ; RV64I-NEXT:    srli a0, a0, 32
 ; RV64I-NEXT:    ret
@@ -1043,9 +993,7 @@ define zeroext i32 @zext_srlw_aext_sext(i32 %a, i32 signext %b) nounwind {
 define zeroext i32 @zext_srlw_aext_zext(i32 %a, i32 zeroext %b) nounwind {
 ; RV64I-LABEL: zext_srlw_aext_zext:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    slli a0, a0, 32
-; RV64I-NEXT:    srli a0, a0, 32
-; RV64I-NEXT:    srl a0, a0, a1
+; RV64I-NEXT:    srlw a0, a0, a1
 ; RV64I-NEXT:    slli a0, a0, 32
 ; RV64I-NEXT:    srli a0, a0, 32
 ; RV64I-NEXT:    ret
@@ -1056,9 +1004,7 @@ define zeroext i32 @zext_srlw_aext_zext(i32 %a, i32 zeroext %b) nounwind {
 define zeroext i32 @zext_srlw_sext_aext(i32 signext %a, i32 %b) nounwind {
 ; RV64I-LABEL: zext_srlw_sext_aext:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    slli a0, a0, 32
-; RV64I-NEXT:    srli a0, a0, 32
-; RV64I-NEXT:    srl a0, a0, a1
+; RV64I-NEXT:    srlw a0, a0, a1
 ; RV64I-NEXT:    slli a0, a0, 32
 ; RV64I-NEXT:    srli a0, a0, 32
 ; RV64I-NEXT:    ret
@@ -1069,9 +1015,7 @@ define zeroext i32 @zext_srlw_sext_aext(i32 signext %a, i32 %b) nounwind {
 define zeroext i32 @zext_srlw_sext_sext(i32 signext %a, i32 signext %b) nounwind {
 ; RV64I-LABEL: zext_srlw_sext_sext:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    slli a0, a0, 32
-; RV64I-NEXT:    srli a0, a0, 32
-; RV64I-NEXT:    srl a0, a0, a1
+; RV64I-NEXT:    srlw a0, a0, a1
 ; RV64I-NEXT:    slli a0, a0, 32
 ; RV64I-NEXT:    srli a0, a0, 32
 ; RV64I-NEXT:    ret
@@ -1082,9 +1026,7 @@ define zeroext i32 @zext_srlw_sext_sext(i32 signext %a, i32 signext %b) nounwind
 define zeroext i32 @zext_srlw_sext_zext(i32 signext %a, i32 zeroext %b) nounwind {
 ; RV64I-LABEL: zext_srlw_sext_zext:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    slli a0, a0, 32
-; RV64I-NEXT:    srli a0, a0, 32
-; RV64I-NEXT:    srl a0, a0, a1
+; RV64I-NEXT:    srlw a0, a0, a1
 ; RV64I-NEXT:    slli a0, a0, 32
 ; RV64I-NEXT:    srli a0, a0, 32
 ; RV64I-NEXT:    ret
@@ -1095,7 +1037,7 @@ define zeroext i32 @zext_srlw_sext_zext(i32 signext %a, i32 zeroext %b) nounwind
 define zeroext i32 @zext_srlw_zext_aext(i32 zeroext %a, i32 %b) nounwind {
 ; RV64I-LABEL: zext_srlw_zext_aext:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    srl a0, a0, a1
+; RV64I-NEXT:    srlw a0, a0, a1
 ; RV64I-NEXT:    slli a0, a0, 32
 ; RV64I-NEXT:    srli a0, a0, 32
 ; RV64I-NEXT:    ret
@@ -1106,7 +1048,7 @@ define zeroext i32 @zext_srlw_zext_aext(i32 zeroext %a, i32 %b) nounwind {
 define zeroext i32 @zext_srlw_zext_sext(i32 zeroext %a, i32 signext %b) nounwind {
 ; RV64I-LABEL: zext_srlw_zext_sext:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    srl a0, a0, a1
+; RV64I-NEXT:    srlw a0, a0, a1
 ; RV64I-NEXT:    slli a0, a0, 32
 ; RV64I-NEXT:    srli a0, a0, 32
 ; RV64I-NEXT:    ret
@@ -1117,7 +1059,7 @@ define zeroext i32 @zext_srlw_zext_sext(i32 zeroext %a, i32 signext %b) nounwind
 define zeroext i32 @zext_srlw_zext_zext(i32 zeroext %a, i32 zeroext %b) nounwind {
 ; RV64I-LABEL: zext_srlw_zext_zext:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    srl a0, a0, a1
+; RV64I-NEXT:    srlw a0, a0, a1
 ; RV64I-NEXT:    slli a0, a0, 32
 ; RV64I-NEXT:    srli a0, a0, 32
 ; RV64I-NEXT:    ret
@@ -1125,14 +1067,10 @@ define zeroext i32 @zext_srlw_zext_zext(i32 zeroext %a, i32 zeroext %b) nounwind
   ret i32 %1
 }
 
-; TODO: sraw should be selected if the first operand is not sign-extended. If the
-; first operand is sign-extended, sra is equivalent for the test cases below.
-
 define i32 @aext_sraw_aext_aext(i32 %a, i32 %b) nounwind {
 ; RV64I-LABEL: aext_sraw_aext_aext:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    sext.w a0, a0
-; RV64I-NEXT:    sra a0, a0, a1
+; RV64I-NEXT:    sraw a0, a0, a1
 ; RV64I-NEXT:    ret
   %1 = ashr i32 %a, %b
   ret i32 %1
@@ -1141,8 +1079,7 @@ define i32 @aext_sraw_aext_aext(i32 %a, i32 %b) nounwind {
 define i32 @aext_sraw_aext_sext(i32 %a, i32 signext %b) nounwind {
 ; RV64I-LABEL: aext_sraw_aext_sext:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    sext.w a0, a0
-; RV64I-NEXT:    sra a0, a0, a1
+; RV64I-NEXT:    sraw a0, a0, a1
 ; RV64I-NEXT:    ret
   %1 = ashr i32 %a, %b
   ret i32 %1
@@ -1151,8 +1088,7 @@ define i32 @aext_sraw_aext_sext(i32 %a, i32 signext %b) nounwind {
 define i32 @aext_sraw_aext_zext(i32 %a, i32 zeroext %b) nounwind {
 ; RV64I-LABEL: aext_sraw_aext_zext:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    sext.w a0, a0
-; RV64I-NEXT:    sra a0, a0, a1
+; RV64I-NEXT:    sraw a0, a0, a1
 ; RV64I-NEXT:    ret
   %1 = ashr i32 %a, %b
   ret i32 %1
@@ -1161,7 +1097,7 @@ define i32 @aext_sraw_aext_zext(i32 %a, i32 zeroext %b) nounwind {
 define i32 @aext_sraw_sext_aext(i32 signext %a, i32 %b) nounwind {
 ; RV64I-LABEL: aext_sraw_sext_aext:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    sra a0, a0, a1
+; RV64I-NEXT:    sraw a0, a0, a1
 ; RV64I-NEXT:    ret
   %1 = ashr i32 %a, %b
   ret i32 %1
@@ -1170,7 +1106,7 @@ define i32 @aext_sraw_sext_aext(i32 signext %a, i32 %b) nounwind {
 define i32 @aext_sraw_sext_sext(i32 signext %a, i32 signext %b) nounwind {
 ; RV64I-LABEL: aext_sraw_sext_sext:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    sra a0, a0, a1
+; RV64I-NEXT:    sraw a0, a0, a1
 ; RV64I-NEXT:    ret
   %1 = ashr i32 %a, %b
   ret i32 %1
@@ -1179,7 +1115,7 @@ define i32 @aext_sraw_sext_sext(i32 signext %a, i32 signext %b) nounwind {
 define i32 @aext_sraw_sext_zext(i32 signext %a, i32 zeroext %b) nounwind {
 ; RV64I-LABEL: aext_sraw_sext_zext:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    sra a0, a0, a1
+; RV64I-NEXT:    sraw a0, a0, a1
 ; RV64I-NEXT:    ret
   %1 = ashr i32 %a, %b
   ret i32 %1
@@ -1188,8 +1124,7 @@ define i32 @aext_sraw_sext_zext(i32 signext %a, i32 zeroext %b) nounwind {
 define i32 @aext_sraw_zext_aext(i32 zeroext %a, i32 %b) nounwind {
 ; RV64I-LABEL: aext_sraw_zext_aext:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    sext.w a0, a0
-; RV64I-NEXT:    sra a0, a0, a1
+; RV64I-NEXT:    sraw a0, a0, a1
 ; RV64I-NEXT:    ret
   %1 = ashr i32 %a, %b
   ret i32 %1
@@ -1198,8 +1133,7 @@ define i32 @aext_sraw_zext_aext(i32 zeroext %a, i32 %b) nounwind {
 define i32 @aext_sraw_zext_sext(i32 zeroext %a, i32 signext %b) nounwind {
 ; RV64I-LABEL: aext_sraw_zext_sext:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    sext.w a0, a0
-; RV64I-NEXT:    sra a0, a0, a1
+; RV64I-NEXT:    sraw a0, a0, a1
 ; RV64I-NEXT:    ret
   %1 = ashr i32 %a, %b
   ret i32 %1
@@ -1208,8 +1142,7 @@ define i32 @aext_sraw_zext_sext(i32 zeroext %a, i32 signext %b) nounwind {
 define i32 @aext_sraw_zext_zext(i32 zeroext %a, i32 zeroext %b) nounwind {
 ; RV64I-LABEL: aext_sraw_zext_zext:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    sext.w a0, a0
-; RV64I-NEXT:    sra a0, a0, a1
+; RV64I-NEXT:    sraw a0, a0, a1
 ; RV64I-NEXT:    ret
   %1 = ashr i32 %a, %b
   ret i32 %1
@@ -1218,8 +1151,7 @@ define i32 @aext_sraw_zext_zext(i32 zeroext %a, i32 zeroext %b) nounwind {
 define signext i32 @sext_sraw_aext_aext(i32 %a, i32 %b) nounwind {
 ; RV64I-LABEL: sext_sraw_aext_aext:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    sext.w a0, a0
-; RV64I-NEXT:    sra a0, a0, a1
+; RV64I-NEXT:    sraw a0, a0, a1
 ; RV64I-NEXT:    ret
   %1 = ashr i32 %a, %b
   ret i32 %1
@@ -1228,8 +1160,7 @@ define signext i32 @sext_sraw_aext_aext(i32 %a, i32 %b) nounwind {
 define signext i32 @sext_sraw_aext_sext(i32 %a, i32 signext %b) nounwind {
 ; RV64I-LABEL: sext_sraw_aext_sext:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    sext.w a0, a0
-; RV64I-NEXT:    sra a0, a0, a1
+; RV64I-NEXT:    sraw a0, a0, a1
 ; RV64I-NEXT:    ret
   %1 = ashr i32 %a, %b
   ret i32 %1
@@ -1238,8 +1169,7 @@ define signext i32 @sext_sraw_aext_sext(i32 %a, i32 signext %b) nounwind {
 define signext i32 @sext_sraw_aext_zext(i32 %a, i32 zeroext %b) nounwind {
 ; RV64I-LABEL: sext_sraw_aext_zext:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    sext.w a0, a0
-; RV64I-NEXT:    sra a0, a0, a1
+; RV64I-NEXT:    sraw a0, a0, a1
 ; RV64I-NEXT:    ret
   %1 = ashr i32 %a, %b
   ret i32 %1
@@ -1248,7 +1178,7 @@ define signext i32 @sext_sraw_aext_zext(i32 %a, i32 zeroext %b) nounwind {
 define signext i32 @sext_sraw_sext_aext(i32 signext %a, i32 %b) nounwind {
 ; RV64I-LABEL: sext_sraw_sext_aext:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    sra a0, a0, a1
+; RV64I-NEXT:    sraw a0, a0, a1
 ; RV64I-NEXT:    ret
   %1 = ashr i32 %a, %b
   ret i32 %1
@@ -1257,7 +1187,7 @@ define signext i32 @sext_sraw_sext_aext(i32 signext %a, i32 %b) nounwind {
 define signext i32 @sext_sraw_sext_sext(i32 signext %a, i32 signext %b) nounwind {
 ; RV64I-LABEL: sext_sraw_sext_sext:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    sra a0, a0, a1
+; RV64I-NEXT:    sraw a0, a0, a1
 ; RV64I-NEXT:    ret
   %1 = ashr i32 %a, %b
   ret i32 %1
@@ -1266,7 +1196,7 @@ define signext i32 @sext_sraw_sext_sext(i32 signext %a, i32 signext %b) nounwind
 define signext i32 @sext_sraw_sext_zext(i32 signext %a, i32 zeroext %b) nounwind {
 ; RV64I-LABEL: sext_sraw_sext_zext:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    sra a0, a0, a1
+; RV64I-NEXT:    sraw a0, a0, a1
 ; RV64I-NEXT:    ret
   %1 = ashr i32 %a, %b
   ret i32 %1
@@ -1275,8 +1205,7 @@ define signext i32 @sext_sraw_sext_zext(i32 signext %a, i32 zeroext %b) nounwind
 define signext i32 @sext_sraw_zext_aext(i32 zeroext %a, i32 %b) nounwind {
 ; RV64I-LABEL: sext_sraw_zext_aext:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    sext.w a0, a0
-; RV64I-NEXT:    sra a0, a0, a1
+; RV64I-NEXT:    sraw a0, a0, a1
 ; RV64I-NEXT:    ret
   %1 = ashr i32 %a, %b
   ret i32 %1
@@ -1285,8 +1214,7 @@ define signext i32 @sext_sraw_zext_aext(i32 zeroext %a, i32 %b) nounwind {
 define signext i32 @sext_sraw_zext_sext(i32 zeroext %a, i32 signext %b) nounwind {
 ; RV64I-LABEL: sext_sraw_zext_sext:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    sext.w a0, a0
-; RV64I-NEXT:    sra a0, a0, a1
+; RV64I-NEXT:    sraw a0, a0, a1
 ; RV64I-NEXT:    ret
   %1 = ashr i32 %a, %b
   ret i32 %1
@@ -1295,8 +1223,7 @@ define signext i32 @sext_sraw_zext_sext(i32 zeroext %a, i32 signext %b) nounwind
 define signext i32 @sext_sraw_zext_zext(i32 zeroext %a, i32 zeroext %b) nounwind {
 ; RV64I-LABEL: sext_sraw_zext_zext:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    sext.w a0, a0
-; RV64I-NEXT:    sra a0, a0, a1
+; RV64I-NEXT:    sraw a0, a0, a1
 ; RV64I-NEXT:    ret
   %1 = ashr i32 %a, %b
   ret i32 %1
@@ -1305,8 +1232,7 @@ define signext i32 @sext_sraw_zext_zext(i32 zeroext %a, i32 zeroext %b) nounwind
 define zeroext i32 @zext_sraw_aext_aext(i32 %a, i32 %b) nounwind {
 ; RV64I-LABEL: zext_sraw_aext_aext:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    sext.w a0, a0
-; RV64I-NEXT:    sra a0, a0, a1
+; RV64I-NEXT:    sraw a0, a0, a1
 ; RV64I-NEXT:    slli a0, a0, 32
 ; RV64I-NEXT:    srli a0, a0, 32
 ; RV64I-NEXT:    ret
@@ -1317,8 +1243,7 @@ define zeroext i32 @zext_sraw_aext_aext(i32 %a, i32 %b) nounwind {
 define zeroext i32 @zext_sraw_aext_sext(i32 %a, i32 signext %b) nounwind {
 ; RV64I-LABEL: zext_sraw_aext_sext:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    sext.w a0, a0
-; RV64I-NEXT:    sra a0, a0, a1
+; RV64I-NEXT:    sraw a0, a0, a1
 ; RV64I-NEXT:    slli a0, a0, 32
 ; RV64I-NEXT:    srli a0, a0, 32
 ; RV64I-NEXT:    ret
@@ -1329,8 +1254,7 @@ define zeroext i32 @zext_sraw_aext_sext(i32 %a, i32 signext %b) nounwind {
 define zeroext i32 @zext_sraw_aext_zext(i32 %a, i32 zeroext %b) nounwind {
 ; RV64I-LABEL: zext_sraw_aext_zext:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    sext.w a0, a0
-; RV64I-NEXT:    sra a0, a0, a1
+; RV64I-NEXT:    sraw a0, a0, a1
 ; RV64I-NEXT:    slli a0, a0, 32
 ; RV64I-NEXT:    srli a0, a0, 32
 ; RV64I-NEXT:    ret
@@ -1341,7 +1265,7 @@ define zeroext i32 @zext_sraw_aext_zext(i32 %a, i32 zeroext %b) nounwind {
 define zeroext i32 @zext_sraw_sext_aext(i32 signext %a, i32 %b) nounwind {
 ; RV64I-LABEL: zext_sraw_sext_aext:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    sra a0, a0, a1
+; RV64I-NEXT:    sraw a0, a0, a1
 ; RV64I-NEXT:    slli a0, a0, 32
 ; RV64I-NEXT:    srli a0, a0, 32
 ; RV64I-NEXT:    ret
@@ -1352,7 +1276,7 @@ define zeroext i32 @zext_sraw_sext_aext(i32 signext %a, i32 %b) nounwind {
 define zeroext i32 @zext_sraw_sext_sext(i32 signext %a, i32 signext %b) nounwind {
 ; RV64I-LABEL: zext_sraw_sext_sext:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    sra a0, a0, a1
+; RV64I-NEXT:    sraw a0, a0, a1
 ; RV64I-NEXT:    slli a0, a0, 32
 ; RV64I-NEXT:    srli a0, a0, 32
 ; RV64I-NEXT:    ret
@@ -1363,7 +1287,7 @@ define zeroext i32 @zext_sraw_sext_sext(i32 signext %a, i32 signext %b) nounwind
 define zeroext i32 @zext_sraw_sext_zext(i32 signext %a, i32 zeroext %b) nounwind {
 ; RV64I-LABEL: zext_sraw_sext_zext:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    sra a0, a0, a1
+; RV64I-NEXT:    sraw a0, a0, a1
 ; RV64I-NEXT:    slli a0, a0, 32
 ; RV64I-NEXT:    srli a0, a0, 32
 ; RV64I-NEXT:    ret
@@ -1374,8 +1298,7 @@ define zeroext i32 @zext_sraw_sext_zext(i32 signext %a, i32 zeroext %b) nounwind
 define zeroext i32 @zext_sraw_zext_aext(i32 zeroext %a, i32 %b) nounwind {
 ; RV64I-LABEL: zext_sraw_zext_aext:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    sext.w a0, a0
-; RV64I-NEXT:    sra a0, a0, a1
+; RV64I-NEXT:    sraw a0, a0, a1
 ; RV64I-NEXT:    slli a0, a0, 32
 ; RV64I-NEXT:    srli a0, a0, 32
 ; RV64I-NEXT:    ret
@@ -1386,8 +1309,7 @@ define zeroext i32 @zext_sraw_zext_aext(i32 zeroext %a, i32 %b) nounwind {
 define zeroext i32 @zext_sraw_zext_sext(i32 zeroext %a, i32 signext %b) nounwind {
 ; RV64I-LABEL: zext_sraw_zext_sext:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    sext.w a0, a0
-; RV64I-NEXT:    sra a0, a0, a1
+; RV64I-NEXT:    sraw a0, a0, a1
 ; RV64I-NEXT:    slli a0, a0, 32
 ; RV64I-NEXT:    srli a0, a0, 32
 ; RV64I-NEXT:    ret
@@ -1398,8 +1320,7 @@ define zeroext i32 @zext_sraw_zext_sext(i32 zeroext %a, i32 signext %b) nounwind
 define zeroext i32 @zext_sraw_zext_zext(i32 zeroext %a, i32 zeroext %b) nounwind {
 ; RV64I-LABEL: zext_sraw_zext_zext:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    sext.w a0, a0
-; RV64I-NEXT:    sra a0, a0, a1
+; RV64I-NEXT:    sraw a0, a0, a1
 ; RV64I-NEXT:    slli a0, a0, 32
 ; RV64I-NEXT:    srli a0, a0, 32
 ; RV64I-NEXT:    ret
diff --git a/test/CodeGen/RISCV/rv64m-exhaustive-w-insts.ll b/test/CodeGen/RISCV/rv64m-exhaustive-w-insts.ll
new file mode 100644
index 0000000000000000000000000000000000000000..980c5813944e9cc1e46cb18d7c67daa6e1dc24c2
--- /dev/null
+++ b/test/CodeGen/RISCV/rv64m-exhaustive-w-insts.ll
@@ -0,0 +1,1308 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv64 -mattr=+m -verify-machineinstrs < %s \
+; RUN:   | FileCheck %s -check-prefix=RV64IM
+
+; The patterns for the 'W' suffixed RV64M instructions have the potential of
+; missing cases. This file checks all the variants of
+; sign-extended/zero-extended/any-extended inputs and outputs.
+
+define i32 @aext_mulw_aext_aext(i32 %a, i32 %b) nounwind {
+; RV64IM-LABEL: aext_mulw_aext_aext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    mul a0, a0, a1
+; RV64IM-NEXT:    ret
+  %1 = mul i32 %a, %b
+  ret i32 %1
+}
+
+define i32 @aext_mulw_aext_sext(i32 %a, i32 signext %b) nounwind {
+; RV64IM-LABEL: aext_mulw_aext_sext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    mul a0, a0, a1
+; RV64IM-NEXT:    ret
+  %1 = mul i32 %a, %b
+  ret i32 %1
+}
+
+define i32 @aext_mulw_aext_zext(i32 %a, i32 zeroext %b) nounwind {
+; RV64IM-LABEL: aext_mulw_aext_zext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    mul a0, a0, a1
+; RV64IM-NEXT:    ret
+  %1 = mul i32 %a, %b
+  ret i32 %1
+}
+
+define i32 @aext_mulw_sext_aext(i32 signext %a, i32 %b) nounwind {
+; RV64IM-LABEL: aext_mulw_sext_aext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    mul a0, a0, a1
+; RV64IM-NEXT:    ret
+  %1 = mul i32 %a, %b
+  ret i32 %1
+}
+
+define i32 @aext_mulw_sext_sext(i32 signext %a, i32 signext %b) nounwind {
+; RV64IM-LABEL: aext_mulw_sext_sext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    mul a0, a0, a1
+; RV64IM-NEXT:    ret
+  %1 = mul i32 %a, %b
+  ret i32 %1
+}
+
+define i32 @aext_mulw_sext_zext(i32 signext %a, i32 zeroext %b) nounwind {
+; RV64IM-LABEL: aext_mulw_sext_zext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    mul a0, a0, a1
+; RV64IM-NEXT:    ret
+  %1 = mul i32 %a, %b
+  ret i32 %1
+}
+
+define i32 @aext_mulw_zext_aext(i32 zeroext %a, i32 %b) nounwind {
+; RV64IM-LABEL: aext_mulw_zext_aext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    mul a0, a0, a1
+; RV64IM-NEXT:    ret
+  %1 = mul i32 %a, %b
+  ret i32 %1
+}
+
+define i32 @aext_mulw_zext_sext(i32 zeroext %a, i32 signext %b) nounwind {
+; RV64IM-LABEL: aext_mulw_zext_sext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    mul a0, a0, a1
+; RV64IM-NEXT:    ret
+  %1 = mul i32 %a, %b
+  ret i32 %1
+}
+
+define i32 @aext_mulw_zext_zext(i32 zeroext %a, i32 zeroext %b) nounwind {
+; RV64IM-LABEL: aext_mulw_zext_zext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    mul a0, a0, a1
+; RV64IM-NEXT:    ret
+  %1 = mul i32 %a, %b
+  ret i32 %1
+}
+
+define signext i32 @sext_mulw_aext_aext(i32 %a, i32 %b) nounwind {
+; RV64IM-LABEL: sext_mulw_aext_aext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    mulw a0, a0, a1
+; RV64IM-NEXT:    ret
+  %1 = mul i32 %a, %b
+  ret i32 %1
+}
+
+define signext i32 @sext_mulw_aext_sext(i32 %a, i32 signext %b) nounwind {
+; RV64IM-LABEL: sext_mulw_aext_sext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    mulw a0, a0, a1
+; RV64IM-NEXT:    ret
+  %1 = mul i32 %a, %b
+  ret i32 %1
+}
+
+define signext i32 @sext_mulw_aext_zext(i32 %a, i32 zeroext %b) nounwind {
+; RV64IM-LABEL: sext_mulw_aext_zext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    mulw a0, a0, a1
+; RV64IM-NEXT:    ret
+  %1 = mul i32 %a, %b
+  ret i32 %1
+}
+
+define signext i32 @sext_mulw_sext_aext(i32 signext %a, i32 %b) nounwind {
+; RV64IM-LABEL: sext_mulw_sext_aext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    mulw a0, a0, a1
+; RV64IM-NEXT:    ret
+  %1 = mul i32 %a, %b
+  ret i32 %1
+}
+
+define signext i32 @sext_mulw_sext_sext(i32 signext %a, i32 signext %b) nounwind {
+; RV64IM-LABEL: sext_mulw_sext_sext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    mulw a0, a0, a1
+; RV64IM-NEXT:    ret
+  %1 = mul i32 %a, %b
+  ret i32 %1
+}
+
+define signext i32 @sext_mulw_sext_zext(i32 signext %a, i32 zeroext %b) nounwind {
+; RV64IM-LABEL: sext_mulw_sext_zext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    mulw a0, a0, a1
+; RV64IM-NEXT:    ret
+  %1 = mul i32 %a, %b
+  ret i32 %1
+}
+
+define signext i32 @sext_mulw_zext_aext(i32 zeroext %a, i32 %b) nounwind {
+; RV64IM-LABEL: sext_mulw_zext_aext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    mulw a0, a0, a1
+; RV64IM-NEXT:    ret
+  %1 = mul i32 %a, %b
+  ret i32 %1
+}
+
+define signext i32 @sext_mulw_zext_sext(i32 zeroext %a, i32 signext %b) nounwind {
+; RV64IM-LABEL: sext_mulw_zext_sext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    mulw a0, a0, a1
+; RV64IM-NEXT:    ret
+  %1 = mul i32 %a, %b
+  ret i32 %1
+}
+
+define signext i32 @sext_mulw_zext_zext(i32 zeroext %a, i32 zeroext %b) nounwind {
+; RV64IM-LABEL: sext_mulw_zext_zext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    mulw a0, a0, a1
+; RV64IM-NEXT:    ret
+  %1 = mul i32 %a, %b
+  ret i32 %1
+}
+
+define zeroext i32 @zext_mulw_aext_aext(i32 %a, i32 %b) nounwind {
+; RV64IM-LABEL: zext_mulw_aext_aext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    mul a0, a0, a1
+; RV64IM-NEXT:    slli a0, a0, 32
+; RV64IM-NEXT:    srli a0, a0, 32
+; RV64IM-NEXT:    ret
+  %1 = mul i32 %a, %b
+  ret i32 %1
+}
+
+define zeroext i32 @zext_mulw_aext_sext(i32 %a, i32 signext %b) nounwind {
+; RV64IM-LABEL: zext_mulw_aext_sext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    mul a0, a0, a1
+; RV64IM-NEXT:    slli a0, a0, 32
+; RV64IM-NEXT:    srli a0, a0, 32
+; RV64IM-NEXT:    ret
+  %1 = mul i32 %a, %b
+  ret i32 %1
+}
+
+define zeroext i32 @zext_mulw_aext_zext(i32 %a, i32 zeroext %b) nounwind {
+; RV64IM-LABEL: zext_mulw_aext_zext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    mul a0, a0, a1
+; RV64IM-NEXT:    slli a0, a0, 32
+; RV64IM-NEXT:    srli a0, a0, 32
+; RV64IM-NEXT:    ret
+  %1 = mul i32 %a, %b
+  ret i32 %1
+}
+
+define zeroext i32 @zext_mulw_sext_aext(i32 signext %a, i32 %b) nounwind {
+; RV64IM-LABEL: zext_mulw_sext_aext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    mul a0, a0, a1
+; RV64IM-NEXT:    slli a0, a0, 32
+; RV64IM-NEXT:    srli a0, a0, 32
+; RV64IM-NEXT:    ret
+  %1 = mul i32 %a, %b
+  ret i32 %1
+}
+
+define zeroext i32 @zext_mulw_sext_sext(i32 signext %a, i32 signext %b) nounwind {
+; RV64IM-LABEL: zext_mulw_sext_sext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    mul a0, a0, a1
+; RV64IM-NEXT:    slli a0, a0, 32
+; RV64IM-NEXT:    srli a0, a0, 32
+; RV64IM-NEXT:    ret
+  %1 = mul i32 %a, %b
+  ret i32 %1
+}
+
+define zeroext i32 @zext_mulw_sext_zext(i32 signext %a, i32 zeroext %b) nounwind {
+; RV64IM-LABEL: zext_mulw_sext_zext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    mul a0, a0, a1
+; RV64IM-NEXT:    slli a0, a0, 32
+; RV64IM-NEXT:    srli a0, a0, 32
+; RV64IM-NEXT:    ret
+  %1 = mul i32 %a, %b
+  ret i32 %1
+}
+
+define zeroext i32 @zext_mulw_zext_aext(i32 zeroext %a, i32 %b) nounwind {
+; RV64IM-LABEL: zext_mulw_zext_aext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    mul a0, a0, a1
+; RV64IM-NEXT:    slli a0, a0, 32
+; RV64IM-NEXT:    srli a0, a0, 32
+; RV64IM-NEXT:    ret
+  %1 = mul i32 %a, %b
+  ret i32 %1
+}
+
+define zeroext i32 @zext_mulw_zext_sext(i32 zeroext %a, i32 signext %b) nounwind {
+; RV64IM-LABEL: zext_mulw_zext_sext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    mul a0, a0, a1
+; RV64IM-NEXT:    slli a0, a0, 32
+; RV64IM-NEXT:    srli a0, a0, 32
+; RV64IM-NEXT:    ret
+  %1 = mul i32 %a, %b
+  ret i32 %1
+}
+
+define zeroext i32 @zext_mulw_zext_zext(i32 zeroext %a, i32 zeroext %b) nounwind {
+; RV64IM-LABEL: zext_mulw_zext_zext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    mul a0, a0, a1
+; RV64IM-NEXT:    slli a0, a0, 32
+; RV64IM-NEXT:    srli a0, a0, 32
+; RV64IM-NEXT:    ret
+  %1 = mul i32 %a, %b
+  ret i32 %1
+}
+
+define i32 @aext_divuw_aext_aext(i32 %a, i32 %b) nounwind {
+; RV64IM-LABEL: aext_divuw_aext_aext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    divuw a0, a0, a1
+; RV64IM-NEXT:    ret
+  %1 = udiv i32 %a, %b
+  ret i32 %1
+}
+
+define i32 @aext_divuw_aext_sext(i32 %a, i32 signext %b) nounwind {
+; RV64IM-LABEL: aext_divuw_aext_sext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    divuw a0, a0, a1
+; RV64IM-NEXT:    ret
+  %1 = udiv i32 %a, %b
+  ret i32 %1
+}
+
+define i32 @aext_divuw_aext_zext(i32 %a, i32 zeroext %b) nounwind {
+; RV64IM-LABEL: aext_divuw_aext_zext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    divuw a0, a0, a1
+; RV64IM-NEXT:    ret
+  %1 = udiv i32 %a, %b
+  ret i32 %1
+}
+
+define i32 @aext_divuw_sext_aext(i32 signext %a, i32 %b) nounwind {
+; RV64IM-LABEL: aext_divuw_sext_aext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    divuw a0, a0, a1
+; RV64IM-NEXT:    ret
+  %1 = udiv i32 %a, %b
+  ret i32 %1
+}
+
+define i32 @aext_divuw_sext_sext(i32 signext %a, i32 signext %b) nounwind {
+; RV64IM-LABEL: aext_divuw_sext_sext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    divuw a0, a0, a1
+; RV64IM-NEXT:    ret
+  %1 = udiv i32 %a, %b
+  ret i32 %1
+}
+
+define i32 @aext_divuw_sext_zext(i32 signext %a, i32 zeroext %b) nounwind {
+; RV64IM-LABEL: aext_divuw_sext_zext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    divuw a0, a0, a1
+; RV64IM-NEXT:    ret
+  %1 = udiv i32 %a, %b
+  ret i32 %1
+}
+
+define i32 @aext_divuw_zext_aext(i32 zeroext %a, i32 %b) nounwind {
+; RV64IM-LABEL: aext_divuw_zext_aext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    divuw a0, a0, a1
+; RV64IM-NEXT:    ret
+  %1 = udiv i32 %a, %b
+  ret i32 %1
+}
+
+define i32 @aext_divuw_zext_sext(i32 zeroext %a, i32 signext %b) nounwind {
+; RV64IM-LABEL: aext_divuw_zext_sext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    divuw a0, a0, a1
+; RV64IM-NEXT:    ret
+  %1 = udiv i32 %a, %b
+  ret i32 %1
+}
+
+define i32 @aext_divuw_zext_zext(i32 zeroext %a, i32 zeroext %b) nounwind {
+; RV64IM-LABEL: aext_divuw_zext_zext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    divuw a0, a0, a1
+; RV64IM-NEXT:    ret
+  %1 = udiv i32 %a, %b
+  ret i32 %1
+}
+
+define signext i32 @sext_divuw_aext_aext(i32 %a, i32 %b) nounwind {
+; RV64IM-LABEL: sext_divuw_aext_aext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    divuw a0, a0, a1
+; RV64IM-NEXT:    ret
+  %1 = udiv i32 %a, %b
+  ret i32 %1
+}
+
+define signext i32 @sext_divuw_aext_sext(i32 %a, i32 signext %b) nounwind {
+; RV64IM-LABEL: sext_divuw_aext_sext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    divuw a0, a0, a1
+; RV64IM-NEXT:    ret
+  %1 = udiv i32 %a, %b
+  ret i32 %1
+}
+
+define signext i32 @sext_divuw_aext_zext(i32 %a, i32 zeroext %b) nounwind {
+; RV64IM-LABEL: sext_divuw_aext_zext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    divuw a0, a0, a1
+; RV64IM-NEXT:    ret
+  %1 = udiv i32 %a, %b
+  ret i32 %1
+}
+
+define signext i32 @sext_divuw_sext_aext(i32 signext %a, i32 %b) nounwind {
+; RV64IM-LABEL: sext_divuw_sext_aext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    divuw a0, a0, a1
+; RV64IM-NEXT:    ret
+  %1 = udiv i32 %a, %b
+  ret i32 %1
+}
+
+define signext i32 @sext_divuw_sext_sext(i32 signext %a, i32 signext %b) nounwind {
+; RV64IM-LABEL: sext_divuw_sext_sext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    divuw a0, a0, a1
+; RV64IM-NEXT:    ret
+  %1 = udiv i32 %a, %b
+  ret i32 %1
+}
+
+define signext i32 @sext_divuw_sext_zext(i32 signext %a, i32 zeroext %b) nounwind {
+; RV64IM-LABEL: sext_divuw_sext_zext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    divuw a0, a0, a1
+; RV64IM-NEXT:    ret
+  %1 = udiv i32 %a, %b
+  ret i32 %1
+}
+
+define signext i32 @sext_divuw_zext_aext(i32 zeroext %a, i32 %b) nounwind {
+; RV64IM-LABEL: sext_divuw_zext_aext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    divuw a0, a0, a1
+; RV64IM-NEXT:    ret
+  %1 = udiv i32 %a, %b
+  ret i32 %1
+}
+
+define signext i32 @sext_divuw_zext_sext(i32 zeroext %a, i32 signext %b) nounwind {
+; RV64IM-LABEL: sext_divuw_zext_sext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    divuw a0, a0, a1
+; RV64IM-NEXT:    ret
+  %1 = udiv i32 %a, %b
+  ret i32 %1
+}
+
+define signext i32 @sext_divuw_zext_zext(i32 zeroext %a, i32 zeroext %b) nounwind {
+; RV64IM-LABEL: sext_divuw_zext_zext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    divuw a0, a0, a1
+; RV64IM-NEXT:    ret
+  %1 = udiv i32 %a, %b
+  ret i32 %1
+}
+
+define zeroext i32 @zext_divuw_aext_aext(i32 %a, i32 %b) nounwind {
+; RV64IM-LABEL: zext_divuw_aext_aext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    divuw a0, a0, a1
+; RV64IM-NEXT:    slli a0, a0, 32
+; RV64IM-NEXT:    srli a0, a0, 32
+; RV64IM-NEXT:    ret
+  %1 = udiv i32 %a, %b
+  ret i32 %1
+}
+
+define zeroext i32 @zext_divuw_aext_sext(i32 %a, i32 signext %b) nounwind {
+; RV64IM-LABEL: zext_divuw_aext_sext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    divuw a0, a0, a1
+; RV64IM-NEXT:    slli a0, a0, 32
+; RV64IM-NEXT:    srli a0, a0, 32
+; RV64IM-NEXT:    ret
+  %1 = udiv i32 %a, %b
+  ret i32 %1
+}
+
+define zeroext i32 @zext_divuw_aext_zext(i32 %a, i32 zeroext %b) nounwind {
+; RV64IM-LABEL: zext_divuw_aext_zext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    divuw a0, a0, a1
+; RV64IM-NEXT:    slli a0, a0, 32
+; RV64IM-NEXT:    srli a0, a0, 32
+; RV64IM-NEXT:    ret
+  %1 = udiv i32 %a, %b
+  ret i32 %1
+}
+
+define zeroext i32 @zext_divuw_sext_aext(i32 signext %a, i32 %b) nounwind {
+; RV64IM-LABEL: zext_divuw_sext_aext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    divuw a0, a0, a1
+; RV64IM-NEXT:    slli a0, a0, 32
+; RV64IM-NEXT:    srli a0, a0, 32
+; RV64IM-NEXT:    ret
+  %1 = udiv i32 %a, %b
+  ret i32 %1
+}
+
+define zeroext i32 @zext_divuw_sext_sext(i32 signext %a, i32 signext %b) nounwind {
+; RV64IM-LABEL: zext_divuw_sext_sext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    divuw a0, a0, a1
+; RV64IM-NEXT:    slli a0, a0, 32
+; RV64IM-NEXT:    srli a0, a0, 32
+; RV64IM-NEXT:    ret
+  %1 = udiv i32 %a, %b
+  ret i32 %1
+}
+
+define zeroext i32 @zext_divuw_sext_zext(i32 signext %a, i32 zeroext %b) nounwind {
+; RV64IM-LABEL: zext_divuw_sext_zext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    divuw a0, a0, a1
+; RV64IM-NEXT:    slli a0, a0, 32
+; RV64IM-NEXT:    srli a0, a0, 32
+; RV64IM-NEXT:    ret
+  %1 = udiv i32 %a, %b
+  ret i32 %1
+}
+
+define zeroext i32 @zext_divuw_zext_aext(i32 zeroext %a, i32 %b) nounwind {
+; RV64IM-LABEL: zext_divuw_zext_aext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    divuw a0, a0, a1
+; RV64IM-NEXT:    slli a0, a0, 32
+; RV64IM-NEXT:    srli a0, a0, 32
+; RV64IM-NEXT:    ret
+  %1 = udiv i32 %a, %b
+  ret i32 %1
+}
+
+define zeroext i32 @zext_divuw_zext_sext(i32 zeroext %a, i32 signext %b) nounwind {
+; RV64IM-LABEL: zext_divuw_zext_sext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    divuw a0, a0, a1
+; RV64IM-NEXT:    slli a0, a0, 32
+; RV64IM-NEXT:    srli a0, a0, 32
+; RV64IM-NEXT:    ret
+  %1 = udiv i32 %a, %b
+  ret i32 %1
+}
+
+define zeroext i32 @zext_divuw_zext_zext(i32 zeroext %a, i32 zeroext %b) nounwind {
+; RV64IM-LABEL: zext_divuw_zext_zext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    divu a0, a0, a1
+; RV64IM-NEXT:    ret
+  %1 = udiv i32 %a, %b
+  ret i32 %1
+}
+
+define i32 @aext_divw_aext_aext(i32 %a, i32 %b) nounwind {
+; RV64IM-LABEL: aext_divw_aext_aext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    divw a0, a0, a1
+; RV64IM-NEXT:    ret
+  %1 = sdiv i32 %a, %b
+  ret i32 %1
+}
+
+define i32 @aext_divw_aext_sext(i32 %a, i32 signext %b) nounwind {
+; RV64IM-LABEL: aext_divw_aext_sext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    divw a0, a0, a1
+; RV64IM-NEXT:    ret
+  %1 = sdiv i32 %a, %b
+  ret i32 %1
+}
+
+define i32 @aext_divw_aext_zext(i32 %a, i32 zeroext %b) nounwind {
+; RV64IM-LABEL: aext_divw_aext_zext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    divw a0, a0, a1
+; RV64IM-NEXT:    ret
+  %1 = sdiv i32 %a, %b
+  ret i32 %1
+}
+
+define i32 @aext_divw_sext_aext(i32 signext %a, i32 %b) nounwind {
+; RV64IM-LABEL: aext_divw_sext_aext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    divw a0, a0, a1
+; RV64IM-NEXT:    ret
+  %1 = sdiv i32 %a, %b
+  ret i32 %1
+}
+
+define i32 @aext_divw_sext_sext(i32 signext %a, i32 signext %b) nounwind {
+; RV64IM-LABEL: aext_divw_sext_sext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    divw a0, a0, a1
+; RV64IM-NEXT:    ret
+  %1 = sdiv i32 %a, %b
+  ret i32 %1
+}
+
+define i32 @aext_divw_sext_zext(i32 signext %a, i32 zeroext %b) nounwind {
+; RV64IM-LABEL: aext_divw_sext_zext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    divw a0, a0, a1
+; RV64IM-NEXT:    ret
+  %1 = sdiv i32 %a, %b
+  ret i32 %1
+}
+
+define i32 @aext_divw_zext_aext(i32 zeroext %a, i32 %b) nounwind {
+; RV64IM-LABEL: aext_divw_zext_aext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    divw a0, a0, a1
+; RV64IM-NEXT:    ret
+  %1 = sdiv i32 %a, %b
+  ret i32 %1
+}
+
+define i32 @aext_divw_zext_sext(i32 zeroext %a, i32 signext %b) nounwind {
+; RV64IM-LABEL: aext_divw_zext_sext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    divw a0, a0, a1
+; RV64IM-NEXT:    ret
+  %1 = sdiv i32 %a, %b
+  ret i32 %1
+}
+
+define i32 @aext_divw_zext_zext(i32 zeroext %a, i32 zeroext %b) nounwind {
+; RV64IM-LABEL: aext_divw_zext_zext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    divw a0, a0, a1
+; RV64IM-NEXT:    ret
+  %1 = sdiv i32 %a, %b
+  ret i32 %1
+}
+
+define signext i32 @sext_divw_aext_aext(i32 %a, i32 %b) nounwind {
+; RV64IM-LABEL: sext_divw_aext_aext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    divw a0, a0, a1
+; RV64IM-NEXT:    ret
+  %1 = sdiv i32 %a, %b
+  ret i32 %1
+}
+
+define signext i32 @sext_divw_aext_sext(i32 %a, i32 signext %b) nounwind {
+; RV64IM-LABEL: sext_divw_aext_sext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    divw a0, a0, a1
+; RV64IM-NEXT:    ret
+  %1 = sdiv i32 %a, %b
+  ret i32 %1
+}
+
+define signext i32 @sext_divw_aext_zext(i32 %a, i32 zeroext %b) nounwind {
+; RV64IM-LABEL: sext_divw_aext_zext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    divw a0, a0, a1
+; RV64IM-NEXT:    ret
+  %1 = sdiv i32 %a, %b
+  ret i32 %1
+}
+
+define signext i32 @sext_divw_sext_aext(i32 signext %a, i32 %b) nounwind {
+; RV64IM-LABEL: sext_divw_sext_aext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    divw a0, a0, a1
+; RV64IM-NEXT:    ret
+  %1 = sdiv i32 %a, %b
+  ret i32 %1
+}
+
+define signext i32 @sext_divw_sext_sext(i32 signext %a, i32 signext %b) nounwind {
+; RV64IM-LABEL: sext_divw_sext_sext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    divw a0, a0, a1
+; RV64IM-NEXT:    ret
+  %1 = sdiv i32 %a, %b
+  ret i32 %1
+}
+
+define signext i32 @sext_divw_sext_zext(i32 signext %a, i32 zeroext %b) nounwind {
+; RV64IM-LABEL: sext_divw_sext_zext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    divw a0, a0, a1
+; RV64IM-NEXT:    ret
+  %1 = sdiv i32 %a, %b
+  ret i32 %1
+}
+
+define signext i32 @sext_divw_zext_aext(i32 zeroext %a, i32 %b) nounwind {
+; RV64IM-LABEL: sext_divw_zext_aext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    divw a0, a0, a1
+; RV64IM-NEXT:    ret
+  %1 = sdiv i32 %a, %b
+  ret i32 %1
+}
+
+define signext i32 @sext_divw_zext_sext(i32 zeroext %a, i32 signext %b) nounwind {
+; RV64IM-LABEL: sext_divw_zext_sext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    divw a0, a0, a1
+; RV64IM-NEXT:    ret
+  %1 = sdiv i32 %a, %b
+  ret i32 %1
+}
+
+define signext i32 @sext_divw_zext_zext(i32 zeroext %a, i32 zeroext %b) nounwind {
+; RV64IM-LABEL: sext_divw_zext_zext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    divw a0, a0, a1
+; RV64IM-NEXT:    ret
+  %1 = sdiv i32 %a, %b
+  ret i32 %1
+}
+
+define zeroext i32 @zext_divw_aext_aext(i32 %a, i32 %b) nounwind {
+; RV64IM-LABEL: zext_divw_aext_aext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    divw a0, a0, a1
+; RV64IM-NEXT:    slli a0, a0, 32
+; RV64IM-NEXT:    srli a0, a0, 32
+; RV64IM-NEXT:    ret
+  %1 = sdiv i32 %a, %b
+  ret i32 %1
+}
+
+define zeroext i32 @zext_divw_aext_sext(i32 %a, i32 signext %b) nounwind {
+; RV64IM-LABEL: zext_divw_aext_sext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    divw a0, a0, a1
+; RV64IM-NEXT:    slli a0, a0, 32
+; RV64IM-NEXT:    srli a0, a0, 32
+; RV64IM-NEXT:    ret
+  %1 = sdiv i32 %a, %b
+  ret i32 %1
+}
+
+define zeroext i32 @zext_divw_aext_zext(i32 %a, i32 zeroext %b) nounwind {
+; RV64IM-LABEL: zext_divw_aext_zext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    divw a0, a0, a1
+; RV64IM-NEXT:    slli a0, a0, 32
+; RV64IM-NEXT:    srli a0, a0, 32
+; RV64IM-NEXT:    ret
+  %1 = sdiv i32 %a, %b
+  ret i32 %1
+}
+
+define zeroext i32 @zext_divw_sext_aext(i32 signext %a, i32 %b) nounwind {
+; RV64IM-LABEL: zext_divw_sext_aext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    divw a0, a0, a1
+; RV64IM-NEXT:    slli a0, a0, 32
+; RV64IM-NEXT:    srli a0, a0, 32
+; RV64IM-NEXT:    ret
+  %1 = sdiv i32 %a, %b
+  ret i32 %1
+}
+
+define zeroext i32 @zext_divw_sext_sext(i32 signext %a, i32 signext %b) nounwind {
+; RV64IM-LABEL: zext_divw_sext_sext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    divw a0, a0, a1
+; RV64IM-NEXT:    slli a0, a0, 32
+; RV64IM-NEXT:    srli a0, a0, 32
+; RV64IM-NEXT:    ret
+  %1 = sdiv i32 %a, %b
+  ret i32 %1
+}
+
+define zeroext i32 @zext_divw_sext_zext(i32 signext %a, i32 zeroext %b) nounwind {
+; RV64IM-LABEL: zext_divw_sext_zext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    divw a0, a0, a1
+; RV64IM-NEXT:    slli a0, a0, 32
+; RV64IM-NEXT:    srli a0, a0, 32
+; RV64IM-NEXT:    ret
+  %1 = sdiv i32 %a, %b
+  ret i32 %1
+}
+
+define zeroext i32 @zext_divw_zext_aext(i32 zeroext %a, i32 %b) nounwind {
+; RV64IM-LABEL: zext_divw_zext_aext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    divw a0, a0, a1
+; RV64IM-NEXT:    slli a0, a0, 32
+; RV64IM-NEXT:    srli a0, a0, 32
+; RV64IM-NEXT:    ret
+  %1 = sdiv i32 %a, %b
+  ret i32 %1
+}
+
+define zeroext i32 @zext_divw_zext_sext(i32 zeroext %a, i32 signext %b) nounwind {
+; RV64IM-LABEL: zext_divw_zext_sext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    divw a0, a0, a1
+; RV64IM-NEXT:    slli a0, a0, 32
+; RV64IM-NEXT:    srli a0, a0, 32
+; RV64IM-NEXT:    ret
+  %1 = sdiv i32 %a, %b
+  ret i32 %1
+}
+
+define zeroext i32 @zext_divw_zext_zext(i32 zeroext %a, i32 zeroext %b) nounwind {
+; RV64IM-LABEL: zext_divw_zext_zext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    divw a0, a0, a1
+; RV64IM-NEXT:    slli a0, a0, 32
+; RV64IM-NEXT:    srli a0, a0, 32
+; RV64IM-NEXT:    ret
+  %1 = sdiv i32 %a, %b
+  ret i32 %1
+}
+
+define i32 @aext_remw_aext_aext(i32 %a, i32 %b) nounwind {
+; RV64IM-LABEL: aext_remw_aext_aext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    remw a0, a0, a1
+; RV64IM-NEXT:    ret
+  %1 = srem i32 %a, %b
+  ret i32 %1
+}
+
+define i32 @aext_remw_aext_sext(i32 %a, i32 signext %b) nounwind {
+; RV64IM-LABEL: aext_remw_aext_sext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    remw a0, a0, a1
+; RV64IM-NEXT:    ret
+  %1 = srem i32 %a, %b
+  ret i32 %1
+}
+
+define i32 @aext_remw_aext_zext(i32 %a, i32 zeroext %b) nounwind {
+; RV64IM-LABEL: aext_remw_aext_zext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    remw a0, a0, a1
+; RV64IM-NEXT:    ret
+  %1 = srem i32 %a, %b
+  ret i32 %1
+}
+
+define i32 @aext_remw_sext_aext(i32 signext %a, i32 %b) nounwind {
+; RV64IM-LABEL: aext_remw_sext_aext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    remw a0, a0, a1
+; RV64IM-NEXT:    ret
+  %1 = srem i32 %a, %b
+  ret i32 %1
+}
+
+define i32 @aext_remw_sext_sext(i32 signext %a, i32 signext %b) nounwind {
+; RV64IM-LABEL: aext_remw_sext_sext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    remw a0, a0, a1
+; RV64IM-NEXT:    ret
+  %1 = srem i32 %a, %b
+  ret i32 %1
+}
+
+define i32 @aext_remw_sext_zext(i32 signext %a, i32 zeroext %b) nounwind {
+; RV64IM-LABEL: aext_remw_sext_zext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    remw a0, a0, a1
+; RV64IM-NEXT:    ret
+  %1 = srem i32 %a, %b
+  ret i32 %1
+}
+
+define i32 @aext_remw_zext_aext(i32 zeroext %a, i32 %b) nounwind {
+; RV64IM-LABEL: aext_remw_zext_aext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    remw a0, a0, a1
+; RV64IM-NEXT:    ret
+  %1 = srem i32 %a, %b
+  ret i32 %1
+}
+
+define i32 @aext_remw_zext_sext(i32 zeroext %a, i32 signext %b) nounwind {
+; RV64IM-LABEL: aext_remw_zext_sext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    remw a0, a0, a1
+; RV64IM-NEXT:    ret
+  %1 = srem i32 %a, %b
+  ret i32 %1
+}
+
+define i32 @aext_remw_zext_zext(i32 zeroext %a, i32 zeroext %b) nounwind {
+; RV64IM-LABEL: aext_remw_zext_zext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    remw a0, a0, a1
+; RV64IM-NEXT:    ret
+  %1 = srem i32 %a, %b
+  ret i32 %1
+}
+
+define signext i32 @sext_remw_aext_aext(i32 %a, i32 %b) nounwind {
+; RV64IM-LABEL: sext_remw_aext_aext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    remw a0, a0, a1
+; RV64IM-NEXT:    ret
+  %1 = srem i32 %a, %b
+  ret i32 %1
+}
+
+define signext i32 @sext_remw_aext_sext(i32 %a, i32 signext %b) nounwind {
+; RV64IM-LABEL: sext_remw_aext_sext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    remw a0, a0, a1
+; RV64IM-NEXT:    ret
+  %1 = srem i32 %a, %b
+  ret i32 %1
+}
+
+define signext i32 @sext_remw_aext_zext(i32 %a, i32 zeroext %b) nounwind {
+; RV64IM-LABEL: sext_remw_aext_zext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    remw a0, a0, a1
+; RV64IM-NEXT:    ret
+  %1 = srem i32 %a, %b
+  ret i32 %1
+}
+
+define signext i32 @sext_remw_sext_aext(i32 signext %a, i32 %b) nounwind {
+; RV64IM-LABEL: sext_remw_sext_aext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    remw a0, a0, a1
+; RV64IM-NEXT:    ret
+  %1 = srem i32 %a, %b
+  ret i32 %1
+}
+
+define signext i32 @sext_remw_sext_sext(i32 signext %a, i32 signext %b) nounwind {
+; RV64IM-LABEL: sext_remw_sext_sext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    remw a0, a0, a1
+; RV64IM-NEXT:    ret
+  %1 = srem i32 %a, %b
+  ret i32 %1
+}
+
+define signext i32 @sext_remw_sext_zext(i32 signext %a, i32 zeroext %b) nounwind {
+; RV64IM-LABEL: sext_remw_sext_zext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    remw a0, a0, a1
+; RV64IM-NEXT:    ret
+  %1 = srem i32 %a, %b
+  ret i32 %1
+}
+
+define signext i32 @sext_remw_zext_aext(i32 zeroext %a, i32 %b) nounwind {
+; RV64IM-LABEL: sext_remw_zext_aext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    remw a0, a0, a1
+; RV64IM-NEXT:    ret
+  %1 = srem i32 %a, %b
+  ret i32 %1
+}
+
+define signext i32 @sext_remw_zext_sext(i32 zeroext %a, i32 signext %b) nounwind {
+; RV64IM-LABEL: sext_remw_zext_sext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    remw a0, a0, a1
+; RV64IM-NEXT:    ret
+  %1 = srem i32 %a, %b
+  ret i32 %1
+}
+
+define signext i32 @sext_remw_zext_zext(i32 zeroext %a, i32 zeroext %b) nounwind {
+; RV64IM-LABEL: sext_remw_zext_zext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    remw a0, a0, a1
+; RV64IM-NEXT:    ret
+  %1 = srem i32 %a, %b
+  ret i32 %1
+}
+
+define zeroext i32 @zext_remw_aext_aext(i32 %a, i32 %b) nounwind {
+; RV64IM-LABEL: zext_remw_aext_aext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    remw a0, a0, a1
+; RV64IM-NEXT:    slli a0, a0, 32
+; RV64IM-NEXT:    srli a0, a0, 32
+; RV64IM-NEXT:    ret
+  %1 = srem i32 %a, %b
+  ret i32 %1
+}
+
+define zeroext i32 @zext_remw_aext_sext(i32 %a, i32 signext %b) nounwind {
+; RV64IM-LABEL: zext_remw_aext_sext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    remw a0, a0, a1
+; RV64IM-NEXT:    slli a0, a0, 32
+; RV64IM-NEXT:    srli a0, a0, 32
+; RV64IM-NEXT:    ret
+  %1 = srem i32 %a, %b
+  ret i32 %1
+}
+
+define zeroext i32 @zext_remw_aext_zext(i32 %a, i32 zeroext %b) nounwind {
+; RV64IM-LABEL: zext_remw_aext_zext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    remw a0, a0, a1
+; RV64IM-NEXT:    slli a0, a0, 32
+; RV64IM-NEXT:    srli a0, a0, 32
+; RV64IM-NEXT:    ret
+  %1 = srem i32 %a, %b
+  ret i32 %1
+}
+
+define zeroext i32 @zext_remw_sext_aext(i32 signext %a, i32 %b) nounwind {
+; RV64IM-LABEL: zext_remw_sext_aext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    remw a0, a0, a1
+; RV64IM-NEXT:    slli a0, a0, 32
+; RV64IM-NEXT:    srli a0, a0, 32
+; RV64IM-NEXT:    ret
+  %1 = srem i32 %a, %b
+  ret i32 %1
+}
+
+define zeroext i32 @zext_remw_sext_sext(i32 signext %a, i32 signext %b) nounwind {
+; RV64IM-LABEL: zext_remw_sext_sext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    remw a0, a0, a1
+; RV64IM-NEXT:    slli a0, a0, 32
+; RV64IM-NEXT:    srli a0, a0, 32
+; RV64IM-NEXT:    ret
+  %1 = srem i32 %a, %b
+  ret i32 %1
+}
+
+define zeroext i32 @zext_remw_sext_zext(i32 signext %a, i32 zeroext %b) nounwind {
+; RV64IM-LABEL: zext_remw_sext_zext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    remw a0, a0, a1
+; RV64IM-NEXT:    slli a0, a0, 32
+; RV64IM-NEXT:    srli a0, a0, 32
+; RV64IM-NEXT:    ret
+  %1 = srem i32 %a, %b
+  ret i32 %1
+}
+
+define zeroext i32 @zext_remw_zext_aext(i32 zeroext %a, i32 %b) nounwind {
+; RV64IM-LABEL: zext_remw_zext_aext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    remw a0, a0, a1
+; RV64IM-NEXT:    slli a0, a0, 32
+; RV64IM-NEXT:    srli a0, a0, 32
+; RV64IM-NEXT:    ret
+  %1 = srem i32 %a, %b
+  ret i32 %1
+}
+
+define zeroext i32 @zext_remw_zext_sext(i32 zeroext %a, i32 signext %b) nounwind {
+; RV64IM-LABEL: zext_remw_zext_sext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    remw a0, a0, a1
+; RV64IM-NEXT:    slli a0, a0, 32
+; RV64IM-NEXT:    srli a0, a0, 32
+; RV64IM-NEXT:    ret
+  %1 = srem i32 %a, %b
+  ret i32 %1
+}
+
+define zeroext i32 @zext_remw_zext_zext(i32 zeroext %a, i32 zeroext %b) nounwind {
+; RV64IM-LABEL: zext_remw_zext_zext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    remw a0, a0, a1
+; RV64IM-NEXT:    slli a0, a0, 32
+; RV64IM-NEXT:    srli a0, a0, 32
+; RV64IM-NEXT:    ret
+  %1 = srem i32 %a, %b
+  ret i32 %1
+}
+
+define i32 @aext_remuw_aext_aext(i32 %a, i32 %b) nounwind {
+; RV64IM-LABEL: aext_remuw_aext_aext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    remuw a0, a0, a1
+; RV64IM-NEXT:    ret
+  %1 = urem i32 %a, %b
+  ret i32 %1
+}
+
+define i32 @aext_remuw_aext_sext(i32 %a, i32 signext %b) nounwind {
+; RV64IM-LABEL: aext_remuw_aext_sext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    remuw a0, a0, a1
+; RV64IM-NEXT:    ret
+  %1 = urem i32 %a, %b
+  ret i32 %1
+}
+
+define i32 @aext_remuw_aext_zext(i32 %a, i32 zeroext %b) nounwind {
+; RV64IM-LABEL: aext_remuw_aext_zext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    remuw a0, a0, a1
+; RV64IM-NEXT:    ret
+  %1 = urem i32 %a, %b
+  ret i32 %1
+}
+
+define i32 @aext_remuw_sext_aext(i32 signext %a, i32 %b) nounwind {
+; RV64IM-LABEL: aext_remuw_sext_aext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    remuw a0, a0, a1
+; RV64IM-NEXT:    ret
+  %1 = urem i32 %a, %b
+  ret i32 %1
+}
+
+define i32 @aext_remuw_sext_sext(i32 signext %a, i32 signext %b) nounwind {
+; RV64IM-LABEL: aext_remuw_sext_sext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    remuw a0, a0, a1
+; RV64IM-NEXT:    ret
+  %1 = urem i32 %a, %b
+  ret i32 %1
+}
+
+define i32 @aext_remuw_sext_zext(i32 signext %a, i32 zeroext %b) nounwind {
+; RV64IM-LABEL: aext_remuw_sext_zext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    remuw a0, a0, a1
+; RV64IM-NEXT:    ret
+  %1 = urem i32 %a, %b
+  ret i32 %1
+}
+
+define i32 @aext_remuw_zext_aext(i32 zeroext %a, i32 %b) nounwind {
+; RV64IM-LABEL: aext_remuw_zext_aext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    remuw a0, a0, a1
+; RV64IM-NEXT:    ret
+  %1 = urem i32 %a, %b
+  ret i32 %1
+}
+
+define i32 @aext_remuw_zext_sext(i32 zeroext %a, i32 signext %b) nounwind {
+; RV64IM-LABEL: aext_remuw_zext_sext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    remuw a0, a0, a1
+; RV64IM-NEXT:    ret
+  %1 = urem i32 %a, %b
+  ret i32 %1
+}
+
+define i32 @aext_remuw_zext_zext(i32 zeroext %a, i32 zeroext %b) nounwind {
+; RV64IM-LABEL: aext_remuw_zext_zext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    remuw a0, a0, a1
+; RV64IM-NEXT:    ret
+  %1 = urem i32 %a, %b
+  ret i32 %1
+}
+
+define signext i32 @sext_remuw_aext_aext(i32 %a, i32 %b) nounwind {
+; RV64IM-LABEL: sext_remuw_aext_aext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    remuw a0, a0, a1
+; RV64IM-NEXT:    ret
+  %1 = urem i32 %a, %b
+  ret i32 %1
+}
+
+define signext i32 @sext_remuw_aext_sext(i32 %a, i32 signext %b) nounwind {
+; RV64IM-LABEL: sext_remuw_aext_sext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    remuw a0, a0, a1
+; RV64IM-NEXT:    ret
+  %1 = urem i32 %a, %b
+  ret i32 %1
+}
+
+define signext i32 @sext_remuw_aext_zext(i32 %a, i32 zeroext %b) nounwind {
+; RV64IM-LABEL: sext_remuw_aext_zext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    remuw a0, a0, a1
+; RV64IM-NEXT:    ret
+  %1 = urem i32 %a, %b
+  ret i32 %1
+}
+
+define signext i32 @sext_remuw_sext_aext(i32 signext %a, i32 %b) nounwind {
+; RV64IM-LABEL: sext_remuw_sext_aext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    remuw a0, a0, a1
+; RV64IM-NEXT:    ret
+  %1 = urem i32 %a, %b
+  ret i32 %1
+}
+
+define signext i32 @sext_remuw_sext_sext(i32 signext %a, i32 signext %b) nounwind {
+; RV64IM-LABEL: sext_remuw_sext_sext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    remuw a0, a0, a1
+; RV64IM-NEXT:    ret
+  %1 = urem i32 %a, %b
+  ret i32 %1
+}
+
+define signext i32 @sext_remuw_sext_zext(i32 signext %a, i32 zeroext %b) nounwind {
+; RV64IM-LABEL: sext_remuw_sext_zext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    remuw a0, a0, a1
+; RV64IM-NEXT:    ret
+  %1 = urem i32 %a, %b
+  ret i32 %1
+}
+
+define signext i32 @sext_remuw_zext_aext(i32 zeroext %a, i32 %b) nounwind {
+; RV64IM-LABEL: sext_remuw_zext_aext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    remuw a0, a0, a1
+; RV64IM-NEXT:    ret
+  %1 = urem i32 %a, %b
+  ret i32 %1
+}
+
+define signext i32 @sext_remuw_zext_sext(i32 zeroext %a, i32 signext %b) nounwind {
+; RV64IM-LABEL: sext_remuw_zext_sext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    remuw a0, a0, a1
+; RV64IM-NEXT:    ret
+  %1 = urem i32 %a, %b
+  ret i32 %1
+}
+
+define signext i32 @sext_remuw_zext_zext(i32 zeroext %a, i32 zeroext %b) nounwind {
+; RV64IM-LABEL: sext_remuw_zext_zext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    remuw a0, a0, a1
+; RV64IM-NEXT:    ret
+  %1 = urem i32 %a, %b
+  ret i32 %1
+}
+
+define zeroext i32 @zext_remuw_aext_aext(i32 %a, i32 %b) nounwind {
+; RV64IM-LABEL: zext_remuw_aext_aext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    remuw a0, a0, a1
+; RV64IM-NEXT:    slli a0, a0, 32
+; RV64IM-NEXT:    srli a0, a0, 32
+; RV64IM-NEXT:    ret
+  %1 = urem i32 %a, %b
+  ret i32 %1
+}
+
+define zeroext i32 @zext_remuw_aext_sext(i32 %a, i32 signext %b) nounwind {
+; RV64IM-LABEL: zext_remuw_aext_sext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    remuw a0, a0, a1
+; RV64IM-NEXT:    slli a0, a0, 32
+; RV64IM-NEXT:    srli a0, a0, 32
+; RV64IM-NEXT:    ret
+  %1 = urem i32 %a, %b
+  ret i32 %1
+}
+
+define zeroext i32 @zext_remuw_aext_zext(i32 %a, i32 zeroext %b) nounwind {
+; RV64IM-LABEL: zext_remuw_aext_zext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    remuw a0, a0, a1
+; RV64IM-NEXT:    slli a0, a0, 32
+; RV64IM-NEXT:    srli a0, a0, 32
+; RV64IM-NEXT:    ret
+  %1 = urem i32 %a, %b
+  ret i32 %1
+}
+
+define zeroext i32 @zext_remuw_sext_aext(i32 signext %a, i32 %b) nounwind {
+; RV64IM-LABEL: zext_remuw_sext_aext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    remuw a0, a0, a1
+; RV64IM-NEXT:    slli a0, a0, 32
+; RV64IM-NEXT:    srli a0, a0, 32
+; RV64IM-NEXT:    ret
+  %1 = urem i32 %a, %b
+  ret i32 %1
+}
+
+define zeroext i32 @zext_remuw_sext_sext(i32 signext %a, i32 signext %b) nounwind {
+; RV64IM-LABEL: zext_remuw_sext_sext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    remuw a0, a0, a1
+; RV64IM-NEXT:    slli a0, a0, 32
+; RV64IM-NEXT:    srli a0, a0, 32
+; RV64IM-NEXT:    ret
+  %1 = urem i32 %a, %b
+  ret i32 %1
+}
+
+define zeroext i32 @zext_remuw_sext_zext(i32 signext %a, i32 zeroext %b) nounwind {
+; RV64IM-LABEL: zext_remuw_sext_zext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    remuw a0, a0, a1
+; RV64IM-NEXT:    slli a0, a0, 32
+; RV64IM-NEXT:    srli a0, a0, 32
+; RV64IM-NEXT:    ret
+  %1 = urem i32 %a, %b
+  ret i32 %1
+}
+
+define zeroext i32 @zext_remuw_zext_aext(i32 zeroext %a, i32 %b) nounwind {
+; RV64IM-LABEL: zext_remuw_zext_aext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    remuw a0, a0, a1
+; RV64IM-NEXT:    slli a0, a0, 32
+; RV64IM-NEXT:    srli a0, a0, 32
+; RV64IM-NEXT:    ret
+  %1 = urem i32 %a, %b
+  ret i32 %1
+}
+
+define zeroext i32 @zext_remuw_zext_sext(i32 zeroext %a, i32 signext %b) nounwind {
+; RV64IM-LABEL: zext_remuw_zext_sext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    remuw a0, a0, a1
+; RV64IM-NEXT:    slli a0, a0, 32
+; RV64IM-NEXT:    srli a0, a0, 32
+; RV64IM-NEXT:    ret
+  %1 = urem i32 %a, %b
+  ret i32 %1
+}
+
+define zeroext i32 @zext_remuw_zext_zext(i32 zeroext %a, i32 zeroext %b) nounwind {
+; RV64IM-LABEL: zext_remuw_zext_zext:
+; RV64IM:       # %bb.0:
+; RV64IM-NEXT:    remu a0, a0, a1
+; RV64IM-NEXT:    ret
+  %1 = urem i32 %a, %b
+  ret i32 %1
+}
diff --git a/test/CodeGen/RISCV/vararg.ll b/test/CodeGen/RISCV/vararg.ll
index 77f8f300956af0821fd57f54b618bda37702c78e..60286a76e041c18be711d7a105c4e38bdf8c5bbe 100644
--- a/test/CodeGen/RISCV/vararg.ll
+++ b/test/CodeGen/RISCV/vararg.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=riscv32 -verify-machineinstrs < %s \
 ; RUN:   | FileCheck -check-prefix=RV32I-FPELIM %s
-; RUN: llc -mtriple=riscv32 -verify-machineinstrs -disable-fp-elim < %s \
+; RUN: llc -mtriple=riscv32 -verify-machineinstrs -frame-pointer=all < %s \
 ; RUN:   | FileCheck -check-prefix=RV32I-WITHFP %s
 
 declare void @llvm.va_start(i8*)
@@ -185,9 +185,9 @@ define void @va1_caller() nounwind {
 ; RV32I-FPELIM:       # %bb.0:
 ; RV32I-FPELIM-NEXT:    addi sp, sp, -16
 ; RV32I-FPELIM-NEXT:    sw ra, 12(sp)
+; RV32I-FPELIM-NEXT:    mv a2, zero
 ; RV32I-FPELIM-NEXT:    lui a3, 261888
 ; RV32I-FPELIM-NEXT:    addi a4, zero, 2
-; RV32I-FPELIM-NEXT:    mv a2, zero
 ; RV32I-FPELIM-NEXT:    call va1
 ; RV32I-FPELIM-NEXT:    lw ra, 12(sp)
 ; RV32I-FPELIM-NEXT:    addi sp, sp, 16
@@ -199,9 +199,9 @@ define void @va1_caller() nounwind {
 ; RV32I-WITHFP-NEXT:    sw ra, 12(sp)
 ; RV32I-WITHFP-NEXT:    sw s0, 8(sp)
 ; RV32I-WITHFP-NEXT:    addi s0, sp, 16
+; RV32I-WITHFP-NEXT:    mv a2, zero
 ; RV32I-WITHFP-NEXT:    lui a3, 261888
 ; RV32I-WITHFP-NEXT:    addi a4, zero, 2
-; RV32I-WITHFP-NEXT:    mv a2, zero
 ; RV32I-WITHFP-NEXT:    call va1
 ; RV32I-WITHFP-NEXT:    lw s0, 8(sp)
 ; RV32I-WITHFP-NEXT:    lw ra, 12(sp)
@@ -335,8 +335,8 @@ define void @va2_caller() nounwind {
 ; RV32I-FPELIM:       # %bb.0:
 ; RV32I-FPELIM-NEXT:    addi sp, sp, -16
 ; RV32I-FPELIM-NEXT:    sw ra, 12(sp)
-; RV32I-FPELIM-NEXT:    lui a3, 261888
 ; RV32I-FPELIM-NEXT:    mv a2, zero
+; RV32I-FPELIM-NEXT:    lui a3, 261888
 ; RV32I-FPELIM-NEXT:    call va2
 ; RV32I-FPELIM-NEXT:    lw ra, 12(sp)
 ; RV32I-FPELIM-NEXT:    addi sp, sp, 16
@@ -348,8 +348,8 @@ define void @va2_caller() nounwind {
 ; RV32I-WITHFP-NEXT:    sw ra, 12(sp)
 ; RV32I-WITHFP-NEXT:    sw s0, 8(sp)
 ; RV32I-WITHFP-NEXT:    addi s0, sp, 16
-; RV32I-WITHFP-NEXT:    lui a3, 261888
 ; RV32I-WITHFP-NEXT:    mv a2, zero
+; RV32I-WITHFP-NEXT:    lui a3, 261888
 ; RV32I-WITHFP-NEXT:    call va2
 ; RV32I-WITHFP-NEXT:    lw s0, 8(sp)
 ; RV32I-WITHFP-NEXT:    lw ra, 12(sp)
@@ -498,10 +498,10 @@ define void @va3_caller() nounwind {
 ; RV32I-FPELIM-NEXT:    addi sp, sp, -16
 ; RV32I-FPELIM-NEXT:    sw ra, 12(sp)
 ; RV32I-FPELIM-NEXT:    addi a0, zero, 2
-; RV32I-FPELIM-NEXT:    lui a2, 261888
-; RV32I-FPELIM-NEXT:    lui a5, 262144
 ; RV32I-FPELIM-NEXT:    mv a1, zero
+; RV32I-FPELIM-NEXT:    lui a2, 261888
 ; RV32I-FPELIM-NEXT:    mv a4, zero
+; RV32I-FPELIM-NEXT:    lui a5, 262144
 ; RV32I-FPELIM-NEXT:    call va3
 ; RV32I-FPELIM-NEXT:    lw ra, 12(sp)
 ; RV32I-FPELIM-NEXT:    addi sp, sp, 16
@@ -514,10 +514,10 @@ define void @va3_caller() nounwind {
 ; RV32I-WITHFP-NEXT:    sw s0, 8(sp)
 ; RV32I-WITHFP-NEXT:    addi s0, sp, 16
 ; RV32I-WITHFP-NEXT:    addi a0, zero, 2
-; RV32I-WITHFP-NEXT:    lui a2, 261888
-; RV32I-WITHFP-NEXT:    lui a5, 262144
 ; RV32I-WITHFP-NEXT:    mv a1, zero
+; RV32I-WITHFP-NEXT:    lui a2, 261888
 ; RV32I-WITHFP-NEXT:    mv a4, zero
+; RV32I-WITHFP-NEXT:    lui a5, 262144
 ; RV32I-WITHFP-NEXT:    call va3
 ; RV32I-WITHFP-NEXT:    lw s0, 8(sp)
 ; RV32I-WITHFP-NEXT:    lw ra, 12(sp)
@@ -707,9 +707,9 @@ define void @va5_aligned_stack_caller() nounwind {
 ; RV32I-FPELIM-NEXT:    sw a0, 32(sp)
 ; RV32I-FPELIM-NEXT:    lui a0, 688509
 ; RV32I-FPELIM-NEXT:    addi a6, a0, -2048
+; RV32I-FPELIM-NEXT:    addi a2, sp, 32
 ; RV32I-FPELIM-NEXT:    addi a0, zero, 1
 ; RV32I-FPELIM-NEXT:    addi a1, zero, 11
-; RV32I-FPELIM-NEXT:    addi a2, sp, 32
 ; RV32I-FPELIM-NEXT:    addi a3, zero, 12
 ; RV32I-FPELIM-NEXT:    addi a4, zero, 13
 ; RV32I-FPELIM-NEXT:    addi a7, zero, 4
@@ -752,9 +752,9 @@ define void @va5_aligned_stack_caller() nounwind {
 ; RV32I-WITHFP-NEXT:    sw a0, -32(s0)
 ; RV32I-WITHFP-NEXT:    lui a0, 688509
 ; RV32I-WITHFP-NEXT:    addi a6, a0, -2048
+; RV32I-WITHFP-NEXT:    addi a2, s0, -32
 ; RV32I-WITHFP-NEXT:    addi a0, zero, 1
 ; RV32I-WITHFP-NEXT:    addi a1, zero, 11
-; RV32I-WITHFP-NEXT:    addi a2, s0, -32
 ; RV32I-WITHFP-NEXT:    addi a3, zero, 12
 ; RV32I-WITHFP-NEXT:    addi a4, zero, 13
 ; RV32I-WITHFP-NEXT:    addi a7, zero, 4
diff --git a/test/CodeGen/SPARC/empty-functions.ll b/test/CodeGen/SPARC/empty-functions.ll
index 974df232033a5e0dc04beade97a4f88df845d7ef..797bbdad54e6b4b66d29a2f77d397c5641afb7eb 100644
--- a/test/CodeGen/SPARC/empty-functions.ll
+++ b/test/CodeGen/SPARC/empty-functions.ll
@@ -1,5 +1,5 @@
 ; RUN: llc < %s -mtriple=sparc-linux-gnu | FileCheck -check-prefix=LINUX-NO-FP %s
-; RUN: llc < %s -mtriple=sparc-linux-gnu -disable-fp-elim | FileCheck -check-prefix=LINUX-FP %s
+; RUN: llc < %s -mtriple=sparc-linux-gnu -frame-pointer=all | FileCheck -check-prefix=LINUX-FP %s
 
 define void @func() {
 entry:
diff --git a/test/CodeGen/SystemZ/Large/branch-01.ll b/test/CodeGen/SystemZ/Large/branch-01.ll
index 17d9c49a6568a2c0aaba92cc7929efa8787ccc15..ed505471c0a9527836cc26b65d05f96b699bd8ac 100644
--- a/test/CodeGen/SystemZ/Large/branch-01.ll
+++ b/test/CodeGen/SystemZ/Large/branch-01.ll
@@ -11889,7 +11889,7 @@ attributes #3 = { nounwind }
 
 !llvm.ident = !{!0}
 
-!0 = !{!"clang version 7.0.0 (http://llvm.org/git/clang.git a73a299e6b81e72cb50aa8ec5e8b04de7f4e1f81) (http://llvm.org/git/llvm.git 94cb0130bcb7244300ebde720d4e03b6910a1f1f)"}
+!0 = !{!"clang version 7.0.0"}
 !1 = !{!2, !4}
 !2 = distinct !{!2, !3, !"func_62: %agg.result"}
 !3 = distinct !{!3, !"func_62"}
diff --git a/test/CodeGen/SystemZ/Large/branch-range-01.py b/test/CodeGen/SystemZ/Large/branch-range-01.py
index 344d26121afb1ba488dcfab0aa9ac4dab39eb969..49ce623129a791e3f1065e6140ed2e4000eabb32 100644
--- a/test/CodeGen/SystemZ/Large/branch-range-01.py
+++ b/test/CodeGen/SystemZ/Large/branch-range-01.py
@@ -67,42 +67,44 @@
 # CHECK: c %r4, 136(%r3)
 # CHECK: jge [[LABEL]]
 
+from __future__ import print_function
+
 branch_blocks = 10
 main_size = 0xffd8
 
-print '@global = global i32 0'
+print('@global = global i32 0')
 
-print 'define void @f1(i8 *%base, i32 *%stop, i32 %limit) {'
-print 'entry:'
-print '  br label %before0'
-print ''
+print('define void @f1(i8 *%base, i32 *%stop, i32 %limit) {')
+print('entry:')
+print('  br label %before0')
+print('')
 
-for i in xrange(branch_blocks):
+for i in range(branch_blocks):
     next = 'before%d' % (i + 1) if i + 1 < branch_blocks else 'main'
-    print 'before%d:' % i
-    print '  %%bstop%d = getelementptr i32, i32 *%%stop, i64 %d' % (i, i)
-    print '  %%bcur%d = load i32 , i32 *%%bstop%d' % (i, i)
-    print '  %%btest%d = icmp eq i32 %%limit, %%bcur%d' % (i, i)
-    print '  br i1 %%btest%d, label %%after0, label %%%s' % (i, next)
-    print ''
+    print('before%d:' % i)
+    print('  %%bstop%d = getelementptr i32, i32 *%%stop, i64 %d' % (i, i))
+    print('  %%bcur%d = load i32 , i32 *%%bstop%d' % (i, i))
+    print('  %%btest%d = icmp eq i32 %%limit, %%bcur%d' % (i, i))
+    print('  br i1 %%btest%d, label %%after0, label %%%s' % (i, next))
+    print('')
 
-print '%s:' % next
+print('%s:' % next)
 a, b = 1, 1
-for i in xrange(0, main_size, 6):
+for i in range(0, main_size, 6):
     a, b = b, a + b
     offset = 4096 + b % 500000
     value = a % 256
-    print '  %%ptr%d = getelementptr i8, i8 *%%base, i64 %d' % (i, offset)
-    print '  store volatile i8 %d, i8 *%%ptr%d' % (value, i)
+    print('  %%ptr%d = getelementptr i8, i8 *%%base, i64 %d' % (i, offset))
+    print('  store volatile i8 %d, i8 *%%ptr%d' % (value, i))
 
-for i in xrange(branch_blocks):
-    print '  %%astop%d = getelementptr i32, i32 *%%stop, i64 %d' % (i, i + 25)
-    print '  %%acur%d = load i32 , i32 *%%astop%d' % (i, i)
-    print '  %%atest%d = icmp eq i32 %%limit, %%acur%d' % (i, i)
-    print '  br i1 %%atest%d, label %%main, label %%after%d' % (i, i)
-    print ''
-    print 'after%d:' % i
+for i in range(branch_blocks):
+    print('  %%astop%d = getelementptr i32, i32 *%%stop, i64 %d' % (i, i + 25))
+    print('  %%acur%d = load i32 , i32 *%%astop%d' % (i, i))
+    print('  %%atest%d = icmp eq i32 %%limit, %%acur%d' % (i, i))
+    print('  br i1 %%atest%d, label %%main, label %%after%d' % (i, i))
+    print('')
+    print('after%d:' % i)
 
-print '  %dummy = load volatile i32, i32 *@global'
-print '  ret void'
-print '}'
+print('  %dummy = load volatile i32, i32 *@global')
+print('  ret void')
+print('}')
diff --git a/test/CodeGen/SystemZ/Large/branch-range-02.py b/test/CodeGen/SystemZ/Large/branch-range-02.py
index 7f7b09954a6194f1a7afa990b2992a3824aaa6ff..155f742dca67b0d7d67fa5d8ec6bbc37dcc9e83b 100644
--- a/test/CodeGen/SystemZ/Large/branch-range-02.py
+++ b/test/CodeGen/SystemZ/Large/branch-range-02.py
@@ -56,27 +56,29 @@
 # CHECK: c %r4, 1036(%r3)
 # CHECK: jge [[LABEL]]
 
+from __future__ import print_function
+
 blocks = 256 + 4
 
-print 'define void @f1(i8 *%base, i32 *%stop, i32 %limit) {'
-print 'entry:'
-print '  br label %b0'
-print ''
+print('define void @f1(i8 *%base, i32 *%stop, i32 %limit) {')
+print('entry:')
+print('  br label %b0')
+print('')
 
 a, b = 1, 1
-for i in xrange(blocks):
+for i in range(blocks):
     a, b = b, a + b
     value = a % 256
     next = 'b%d' % (i + 1) if i + 1 < blocks else 'end'
     other = 'end' if 2 * i < blocks else 'b0'
-    print 'b%d:' % i
-    print '  store volatile i8 %d, i8 *%%base' % value
-    print '  %%astop%d = getelementptr i32, i32 *%%stop, i64 %d' % (i, i)
-    print '  %%acur%d = load i32 , i32 *%%astop%d' % (i, i)
-    print '  %%atest%d = icmp eq i32 %%limit, %%acur%d' % (i, i)
-    print '  br i1 %%atest%d, label %%%s, label %%%s' % (i, other, next)
+    print('b%d:' % i)
+    print('  store volatile i8 %d, i8 *%%base' % value)
+    print('  %%astop%d = getelementptr i32, i32 *%%stop, i64 %d' % (i, i))
+    print('  %%acur%d = load i32 , i32 *%%astop%d' % (i, i))
+    print('  %%atest%d = icmp eq i32 %%limit, %%acur%d' % (i, i))
+    print('  br i1 %%atest%d, label %%%s, label %%%s' % (i, other, next))
 
-print ''
-print '%s:' % next
-print '  ret void'
-print '}'
+print('')
+print('%s:' % next)
+print('  ret void')
+print('}')
diff --git a/test/CodeGen/SystemZ/Large/branch-range-03.py b/test/CodeGen/SystemZ/Large/branch-range-03.py
index 75c9ea4a05104c29905656064aec32e42a91368e..f58bedf5e0d9a1e3752c8a1ea5c8f24e788aaa76 100644
--- a/test/CodeGen/SystemZ/Large/branch-range-03.py
+++ b/test/CodeGen/SystemZ/Large/branch-range-03.py
@@ -67,44 +67,46 @@
 # CHECK: cr %r4, [[REG]]
 # CHECK: jge [[LABEL]]
 
+from __future__ import print_function
+
 branch_blocks = 8
 main_size = 0xffcc
 
-print '@global = global i32 0'
+print('@global = global i32 0')
 
-print 'define void @f1(i8 *%base, i8 *%stop, i32 %limit) {'
-print 'entry:'
-print '  br label %before0'
-print ''
+print('define void @f1(i8 *%base, i8 *%stop, i32 %limit) {')
+print('entry:')
+print('  br label %before0')
+print('')
 
-for i in xrange(branch_blocks):
+for i in range(branch_blocks):
     next = 'before%d' % (i + 1) if i + 1 < branch_blocks else 'main'
-    print 'before%d:' % i
-    print '  %%bstop%d = getelementptr i8, i8 *%%stop, i64 %d' % (i, i)
-    print '  %%bcur%d = load i8 , i8 *%%bstop%d' % (i, i)
-    print '  %%bext%d = sext i8 %%bcur%d to i32' % (i, i)
-    print '  %%btest%d = icmp eq i32 %%limit, %%bext%d' % (i, i)
-    print '  br i1 %%btest%d, label %%after0, label %%%s' % (i, next)
-    print ''
+    print('before%d:' % i)
+    print('  %%bstop%d = getelementptr i8, i8 *%%stop, i64 %d' % (i, i))
+    print('  %%bcur%d = load i8 , i8 *%%bstop%d' % (i, i))
+    print('  %%bext%d = sext i8 %%bcur%d to i32' % (i, i))
+    print('  %%btest%d = icmp eq i32 %%limit, %%bext%d' % (i, i))
+    print('  br i1 %%btest%d, label %%after0, label %%%s' % (i, next))
+    print('')
 
-print '%s:' % next
+print('%s:' % next)
 a, b = 1, 1
-for i in xrange(0, main_size, 6):
+for i in range(0, main_size, 6):
     a, b = b, a + b
     offset = 4096 + b % 500000
     value = a % 256
-    print '  %%ptr%d = getelementptr i8, i8 *%%base, i64 %d' % (i, offset)
-    print '  store volatile i8 %d, i8 *%%ptr%d' % (value, i)
+    print('  %%ptr%d = getelementptr i8, i8 *%%base, i64 %d' % (i, offset))
+    print('  store volatile i8 %d, i8 *%%ptr%d' % (value, i))
 
-for i in xrange(branch_blocks):
-    print '  %%astop%d = getelementptr i8, i8 *%%stop, i64 %d' % (i, i + 25)
-    print '  %%acur%d = load i8 , i8 *%%astop%d' % (i, i)
-    print '  %%aext%d = sext i8 %%acur%d to i32' % (i, i)
-    print '  %%atest%d = icmp eq i32 %%limit, %%aext%d' % (i, i)
-    print '  br i1 %%atest%d, label %%main, label %%after%d' % (i, i)
-    print ''
-    print 'after%d:' % i
+for i in range(branch_blocks):
+    print('  %%astop%d = getelementptr i8, i8 *%%stop, i64 %d' % (i, i + 25))
+    print('  %%acur%d = load i8 , i8 *%%astop%d' % (i, i))
+    print('  %%aext%d = sext i8 %%acur%d to i32' % (i, i))
+    print('  %%atest%d = icmp eq i32 %%limit, %%aext%d' % (i, i))
+    print('  br i1 %%atest%d, label %%main, label %%after%d' % (i, i))
+    print('')
+    print('after%d:' % i)
 
-print '  %dummy = load volatile i32, i32 *@global'
-print '  ret void'
-print '}'
+print('  %dummy = load volatile i32, i32 *@global')
+print('  ret void')
+print('}')
diff --git a/test/CodeGen/SystemZ/Large/branch-range-04.py b/test/CodeGen/SystemZ/Large/branch-range-04.py
index d475c9565e41d2dec2adb5e352e78cf7feebd6ce..6aca0a229221df964d998bae791ae97a69567e2c 100644
--- a/test/CodeGen/SystemZ/Large/branch-range-04.py
+++ b/test/CodeGen/SystemZ/Large/branch-range-04.py
@@ -71,44 +71,46 @@
 # CHECK: cgr %r4, [[REG]]
 # CHECK: jge [[LABEL]]
 
+from __future__ import print_function
+
 branch_blocks = 8
 main_size = 0xffcc
 
-print '@global = global i32 0'
+print('@global = global i32 0')
 
-print 'define void @f1(i8 *%base, i8 *%stop, i64 %limit) {'
-print 'entry:'
-print '  br label %before0'
-print ''
+print('define void @f1(i8 *%base, i8 *%stop, i64 %limit) {')
+print('entry:')
+print('  br label %before0')
+print('')
 
-for i in xrange(branch_blocks):
+for i in range(branch_blocks):
     next = 'before%d' % (i + 1) if i + 1 < branch_blocks else 'main'
-    print 'before%d:' % i
-    print '  %%bstop%d = getelementptr i8, i8 *%%stop, i64 %d' % (i, i)
-    print '  %%bcur%d = load i8 , i8 *%%bstop%d' % (i, i)
-    print '  %%bext%d = sext i8 %%bcur%d to i64' % (i, i)
-    print '  %%btest%d = icmp eq i64 %%limit, %%bext%d' % (i, i)
-    print '  br i1 %%btest%d, label %%after0, label %%%s' % (i, next)
-    print ''
+    print('before%d:' % i)
+    print('  %%bstop%d = getelementptr i8, i8 *%%stop, i64 %d' % (i, i))
+    print('  %%bcur%d = load i8 , i8 *%%bstop%d' % (i, i))
+    print('  %%bext%d = sext i8 %%bcur%d to i64' % (i, i))
+    print('  %%btest%d = icmp eq i64 %%limit, %%bext%d' % (i, i))
+    print('  br i1 %%btest%d, label %%after0, label %%%s' % (i, next))
+    print('')
 
-print '%s:' % next
+print('%s:' % next)
 a, b = 1, 1
-for i in xrange(0, main_size, 6):
+for i in range(0, main_size, 6):
     a, b = b, a + b
     offset = 4096 + b % 500000
     value = a % 256
-    print '  %%ptr%d = getelementptr i8, i8 *%%base, i64 %d' % (i, offset)
-    print '  store volatile i8 %d, i8 *%%ptr%d' % (value, i)
+    print('  %%ptr%d = getelementptr i8, i8 *%%base, i64 %d' % (i, offset))
+    print('  store volatile i8 %d, i8 *%%ptr%d' % (value, i))
 
-for i in xrange(branch_blocks):
-    print '  %%astop%d = getelementptr i8, i8 *%%stop, i64 %d' % (i, i + 25)
-    print '  %%acur%d = load i8 , i8 *%%astop%d' % (i, i)
-    print '  %%aext%d = sext i8 %%acur%d to i64' % (i, i)
-    print '  %%atest%d = icmp eq i64 %%limit, %%aext%d' % (i, i)
-    print '  br i1 %%atest%d, label %%main, label %%after%d' % (i, i)
-    print ''
-    print 'after%d:' % i
+for i in range(branch_blocks):
+    print('  %%astop%d = getelementptr i8, i8 *%%stop, i64 %d' % (i, i + 25))
+    print('  %%acur%d = load i8 , i8 *%%astop%d' % (i, i))
+    print('  %%aext%d = sext i8 %%acur%d to i64' % (i, i))
+    print('  %%atest%d = icmp eq i64 %%limit, %%aext%d' % (i, i))
+    print('  br i1 %%atest%d, label %%main, label %%after%d' % (i, i))
+    print('')
+    print('after%d:' % i)
 
-print '  %dummy = load volatile i32, i32 *@global'
-print '  ret void'
-print '}'
+print('  %dummy = load volatile i32, i32 *@global')
+print('  ret void')
+print('}')
diff --git a/test/CodeGen/SystemZ/Large/branch-range-05.py b/test/CodeGen/SystemZ/Large/branch-range-05.py
index 0a56eff85e1a7cdf49c32d8101ac7f1ff944ade7..326fe8b077c52796ce172eba0b8ff48439a78894 100644
--- a/test/CodeGen/SystemZ/Large/branch-range-05.py
+++ b/test/CodeGen/SystemZ/Large/branch-range-05.py
@@ -71,42 +71,44 @@
 # CHECK: chi [[REG]], 107
 # CHECK: jgl [[LABEL]]
 
+from __future__ import print_function
+
 branch_blocks = 8
 main_size = 0xffcc
 
-print '@global = global i32 0'
+print('@global = global i32 0')
 
-print 'define void @f1(i8 *%base, i8 *%stop) {'
-print 'entry:'
-print '  br label %before0'
-print ''
+print('define void @f1(i8 *%base, i8 *%stop) {')
+print('entry:')
+print('  br label %before0')
+print('')
 
-for i in xrange(branch_blocks):
+for i in range(branch_blocks):
     next = 'before%d' % (i + 1) if i + 1 < branch_blocks else 'main'
-    print 'before%d:' % i
-    print '  %%bcur%d = load i8 , i8 *%%stop' % i
-    print '  %%bext%d = sext i8 %%bcur%d to i32' % (i, i)
-    print '  %%btest%d = icmp slt i32 %%bext%d, %d' % (i, i, i + 50)
-    print '  br i1 %%btest%d, label %%after0, label %%%s' % (i, next)
-    print ''
+    print('before%d:' % i)
+    print('  %%bcur%d = load i8 , i8 *%%stop' % i)
+    print('  %%bext%d = sext i8 %%bcur%d to i32' % (i, i))
+    print('  %%btest%d = icmp slt i32 %%bext%d, %d' % (i, i, i + 50))
+    print('  br i1 %%btest%d, label %%after0, label %%%s' % (i, next))
+    print('')
 
-print '%s:' % next
+print('%s:' % next)
 a, b = 1, 1
-for i in xrange(0, main_size, 6):
+for i in range(0, main_size, 6):
     a, b = b, a + b
     offset = 4096 + b % 500000
     value = a % 256
-    print '  %%ptr%d = getelementptr i8, i8 *%%base, i64 %d' % (i, offset)
-    print '  store volatile i8 %d, i8 *%%ptr%d' % (value, i)
+    print('  %%ptr%d = getelementptr i8, i8 *%%base, i64 %d' % (i, offset))
+    print('  store volatile i8 %d, i8 *%%ptr%d' % (value, i))
 
-for i in xrange(branch_blocks):
-    print '  %%acur%d = load i8 , i8 *%%stop' % i
-    print '  %%aext%d = sext i8 %%acur%d to i32' % (i, i)
-    print '  %%atest%d = icmp slt i32 %%aext%d, %d' % (i, i, i + 100)
-    print '  br i1 %%atest%d, label %%main, label %%after%d' % (i, i)
-    print ''
-    print 'after%d:' % i
+for i in range(branch_blocks):
+    print('  %%acur%d = load i8 , i8 *%%stop' % i)
+    print('  %%aext%d = sext i8 %%acur%d to i32' % (i, i))
+    print('  %%atest%d = icmp slt i32 %%aext%d, %d' % (i, i, i + 100))
+    print('  br i1 %%atest%d, label %%main, label %%after%d' % (i, i))
+    print('')
+    print('after%d:' % i)
 
-print '  %dummy = load volatile i32, i32 *@global'
-print '  ret void'
-print '}'
+print('  %dummy = load volatile i32, i32 *@global')
+print('  ret void')
+print('}')
diff --git a/test/CodeGen/SystemZ/Large/branch-range-06.py b/test/CodeGen/SystemZ/Large/branch-range-06.py
index 5b054345b083db93d655a495af07246f2667b9ae..44cc5c6fdf8928d8a90723dc3425e73f6fa64c69 100644
--- a/test/CodeGen/SystemZ/Large/branch-range-06.py
+++ b/test/CodeGen/SystemZ/Large/branch-range-06.py
@@ -71,42 +71,44 @@
 # CHECK: cghi [[REG]], 107
 # CHECK: jgl [[LABEL]]
 
+from __future__ import print_function
+
 branch_blocks = 8
 main_size = 0xffcc
 
-print '@global = global i32 0'
+print('@global = global i32 0')
 
-print 'define void @f1(i8 *%base, i8 *%stop) {'
-print 'entry:'
-print '  br label %before0'
-print ''
+print('define void @f1(i8 *%base, i8 *%stop) {')
+print('entry:')
+print('  br label %before0')
+print('')
 
-for i in xrange(branch_blocks):
+for i in range(branch_blocks):
     next = 'before%d' % (i + 1) if i + 1 < branch_blocks else 'main'
-    print 'before%d:' % i
-    print '  %%bcur%d = load i8 , i8 *%%stop' % i
-    print '  %%bext%d = sext i8 %%bcur%d to i64' % (i, i)
-    print '  %%btest%d = icmp slt i64 %%bext%d, %d' % (i, i, i + 50)
-    print '  br i1 %%btest%d, label %%after0, label %%%s' % (i, next)
-    print ''
+    print('before%d:' % i)
+    print('  %%bcur%d = load i8 , i8 *%%stop' % i)
+    print('  %%bext%d = sext i8 %%bcur%d to i64' % (i, i))
+    print('  %%btest%d = icmp slt i64 %%bext%d, %d' % (i, i, i + 50))
+    print('  br i1 %%btest%d, label %%after0, label %%%s' % (i, next))
+    print('')
 
-print '%s:' % next
+print('%s:' % next)
 a, b = 1, 1
-for i in xrange(0, main_size, 6):
+for i in range(0, main_size, 6):
     a, b = b, a + b
     offset = 4096 + b % 500000
     value = a % 256
-    print '  %%ptr%d = getelementptr i8, i8 *%%base, i64 %d' % (i, offset)
-    print '  store volatile i8 %d, i8 *%%ptr%d' % (value, i)
+    print('  %%ptr%d = getelementptr i8, i8 *%%base, i64 %d' % (i, offset))
+    print('  store volatile i8 %d, i8 *%%ptr%d' % (value, i))
 
-for i in xrange(branch_blocks):
-    print '  %%acur%d = load i8 , i8 *%%stop' % i
-    print '  %%aext%d = sext i8 %%acur%d to i64' % (i, i)
-    print '  %%atest%d = icmp slt i64 %%aext%d, %d' % (i, i, i + 100)
-    print '  br i1 %%atest%d, label %%main, label %%after%d' % (i, i)
-    print ''
-    print 'after%d:' % i
+for i in range(branch_blocks):
+    print('  %%acur%d = load i8 , i8 *%%stop' % i)
+    print('  %%aext%d = sext i8 %%acur%d to i64' % (i, i))
+    print('  %%atest%d = icmp slt i64 %%aext%d, %d' % (i, i, i + 100))
+    print('  br i1 %%atest%d, label %%main, label %%after%d' % (i, i))
+    print('')
+    print('after%d:' % i)
 
-print '  %dummy = load volatile i32, i32 *@global'
-print '  ret void'
-print '}'
+print('  %dummy = load volatile i32, i32 *@global')
+print('  ret void')
+print('}')
diff --git a/test/CodeGen/SystemZ/Large/branch-range-07.py b/test/CodeGen/SystemZ/Large/branch-range-07.py
index c5fef10577f6c862452368e34ad6cc499538d324..71e958868a7d22c0bec7d2a87b858d6d2e86c772 100644
--- a/test/CodeGen/SystemZ/Large/branch-range-07.py
+++ b/test/CodeGen/SystemZ/Large/branch-range-07.py
@@ -32,37 +32,39 @@
 # CHECK: ahi {{%r[0-9]+}}, -1
 # CHECK: jglh
 
+from __future__ import print_function
+
 branch_blocks = 8
 main_size = 0xffd8
 
-print 'define void @f1(i8 *%base, i32 *%counts) {'
-print 'entry:'
+print('define void @f1(i8 *%base, i32 *%counts) {')
+print('entry:')
 
-for i in xrange(branch_blocks - 1, -1, -1):
-    print '  %%countptr%d = getelementptr i32, i32 *%%counts, i64 %d' % (i, i)
-    print '  %%initcount%d = load i32 , i32 *%%countptr%d' % (i, i)
-    print '  br label %%loop%d' % i
+for i in range(branch_blocks - 1, -1, -1):
+    print('  %%countptr%d = getelementptr i32, i32 *%%counts, i64 %d' % (i, i))
+    print('  %%initcount%d = load i32 , i32 *%%countptr%d' % (i, i))
+    print('  br label %%loop%d' % i)
     
-    print 'loop%d:' % i
+    print('loop%d:' % i)
     block1 = 'entry' if i == branch_blocks - 1 else 'loop%d' % (i + 1)
     block2 = 'loop0' if i == 0 else 'after%d' % (i - 1)
-    print ('  %%count%d = phi i32 [ %%initcount%d, %%%s ],'
-           ' [ %%nextcount%d, %%%s ]' % (i, i, block1, i, block2))
+    print(('  %%count%d = phi i32 [ %%initcount%d, %%%s ],'
+           ' [ %%nextcount%d, %%%s ]' % (i, i, block1, i, block2)))
 
 a, b = 1, 1
-for i in xrange(0, main_size, 6):
+for i in range(0, main_size, 6):
     a, b = b, a + b
     offset = 4096 + b % 500000
     value = a % 256
-    print '  %%ptr%d = getelementptr i8, i8 *%%base, i64 %d' % (i, offset)
-    print '  store volatile i8 %d, i8 *%%ptr%d' % (value, i)
+    print('  %%ptr%d = getelementptr i8, i8 *%%base, i64 %d' % (i, offset))
+    print('  store volatile i8 %d, i8 *%%ptr%d' % (value, i))
 
-for i in xrange(branch_blocks):
-    print '  %%nextcount%d = add i32 %%count%d, -1' % (i, i)
-    print '  %%test%d = icmp ne i32 %%nextcount%d, 0' % (i, i)
-    print '  br i1 %%test%d, label %%loop%d, label %%after%d' % (i, i, i)
-    print ''
-    print 'after%d:' % i
+for i in range(branch_blocks):
+    print('  %%nextcount%d = add i32 %%count%d, -1' % (i, i))
+    print('  %%test%d = icmp ne i32 %%nextcount%d, 0' % (i, i))
+    print('  br i1 %%test%d, label %%loop%d, label %%after%d' % (i, i, i))
+    print('')
+    print('after%d:' % i)
 
-print '  ret void'
-print '}'
+print('  ret void')
+print('}')
diff --git a/test/CodeGen/SystemZ/Large/branch-range-08.py b/test/CodeGen/SystemZ/Large/branch-range-08.py
index 8b6b67398337b839453223065226788710f05d58..2ef6859fc5c8b25a8feb8a8a106696fa26548c87 100644
--- a/test/CodeGen/SystemZ/Large/branch-range-08.py
+++ b/test/CodeGen/SystemZ/Large/branch-range-08.py
@@ -33,37 +33,39 @@
 # CHECK: aghi {{%r[0-9]+}}, -1
 # CHECK: jglh
 
+from __future__ import print_function
+
 branch_blocks = 8
 main_size = 0xffd8
 
-print 'define void @f1(i8 *%base, i64 *%counts) {'
-print 'entry:'
+print('define void @f1(i8 *%base, i64 *%counts) {')
+print('entry:')
 
-for i in xrange(branch_blocks - 1, -1, -1):
-    print '  %%countptr%d = getelementptr i64, i64 *%%counts, i64 %d' % (i, i)
-    print '  %%initcount%d = load i64 , i64 *%%countptr%d' % (i, i)
-    print '  br label %%loop%d' % i
+for i in range(branch_blocks - 1, -1, -1):
+    print('  %%countptr%d = getelementptr i64, i64 *%%counts, i64 %d' % (i, i))
+    print('  %%initcount%d = load i64 , i64 *%%countptr%d' % (i, i))
+    print('  br label %%loop%d' % i)
     
-    print 'loop%d:' % i
+    print('loop%d:' % i)
     block1 = 'entry' if i == branch_blocks - 1 else 'loop%d' % (i + 1)
     block2 = 'loop0' if i == 0 else 'after%d' % (i - 1)
-    print ('  %%count%d = phi i64 [ %%initcount%d, %%%s ],'
-           ' [ %%nextcount%d, %%%s ]' % (i, i, block1, i, block2))
+    print(('  %%count%d = phi i64 [ %%initcount%d, %%%s ],'
+           ' [ %%nextcount%d, %%%s ]' % (i, i, block1, i, block2)))
 
 a, b = 1, 1
-for i in xrange(0, main_size, 6):
+for i in range(0, main_size, 6):
     a, b = b, a + b
     offset = 4096 + b % 500000
     value = a % 256
-    print '  %%ptr%d = getelementptr i8, i8 *%%base, i64 %d' % (i, offset)
-    print '  store volatile i8 %d, i8 *%%ptr%d' % (value, i)
+    print('  %%ptr%d = getelementptr i8, i8 *%%base, i64 %d' % (i, offset))
+    print('  store volatile i8 %d, i8 *%%ptr%d' % (value, i))
 
-for i in xrange(branch_blocks):
-    print '  %%nextcount%d = add i64 %%count%d, -1' % (i, i)
-    print '  %%test%d = icmp ne i64 %%nextcount%d, 0' % (i, i)
-    print '  br i1 %%test%d, label %%loop%d, label %%after%d' % (i, i, i)
-    print ''
-    print 'after%d:' % i
+for i in range(branch_blocks):
+    print('  %%nextcount%d = add i64 %%count%d, -1' % (i, i))
+    print('  %%test%d = icmp ne i64 %%nextcount%d, 0' % (i, i))
+    print('  br i1 %%test%d, label %%loop%d, label %%after%d' % (i, i, i))
+    print('')
+    print('after%d:' % i)
 
-print '  ret void'
-print '}'
+print('  ret void')
+print('}')
diff --git a/test/CodeGen/SystemZ/Large/branch-range-09.py b/test/CodeGen/SystemZ/Large/branch-range-09.py
index 6b568a6e64098485c42e2f1ba4aadef14820a0d7..09d5d691548d82f7610277247deb7dbfaa6c429e 100644
--- a/test/CodeGen/SystemZ/Large/branch-range-09.py
+++ b/test/CodeGen/SystemZ/Large/branch-range-09.py
@@ -67,44 +67,46 @@
 # CHECK: clr %r4, [[REG]]
 # CHECK: jgl [[LABEL]]
 
+from __future__ import print_function
+
 branch_blocks = 8
 main_size = 0xffcc
 
-print '@global = global i32 0'
+print('@global = global i32 0')
 
-print 'define void @f1(i8 *%base, i8 *%stop, i32 %limit) {'
-print 'entry:'
-print '  br label %before0'
-print ''
+print('define void @f1(i8 *%base, i8 *%stop, i32 %limit) {')
+print('entry:')
+print('  br label %before0')
+print('')
 
-for i in xrange(branch_blocks):
+for i in range(branch_blocks):
     next = 'before%d' % (i + 1) if i + 1 < branch_blocks else 'main'
-    print 'before%d:' % i
-    print '  %%bstop%d = getelementptr i8, i8 *%%stop, i64 %d' % (i, i)
-    print '  %%bcur%d = load i8 , i8 *%%bstop%d' % (i, i)
-    print '  %%bext%d = sext i8 %%bcur%d to i32' % (i, i)
-    print '  %%btest%d = icmp ult i32 %%limit, %%bext%d' % (i, i)
-    print '  br i1 %%btest%d, label %%after0, label %%%s' % (i, next)
-    print ''
+    print('before%d:' % i)
+    print('  %%bstop%d = getelementptr i8, i8 *%%stop, i64 %d' % (i, i))
+    print('  %%bcur%d = load i8 , i8 *%%bstop%d' % (i, i))
+    print('  %%bext%d = sext i8 %%bcur%d to i32' % (i, i))
+    print('  %%btest%d = icmp ult i32 %%limit, %%bext%d' % (i, i))
+    print('  br i1 %%btest%d, label %%after0, label %%%s' % (i, next))
+    print('')
 
-print '%s:' % next
+print('%s:' % next)
 a, b = 1, 1
-for i in xrange(0, main_size, 6):
+for i in range(0, main_size, 6):
     a, b = b, a + b
     offset = 4096 + b % 500000
     value = a % 256
-    print '  %%ptr%d = getelementptr i8, i8 *%%base, i64 %d' % (i, offset)
-    print '  store volatile i8 %d, i8 *%%ptr%d' % (value, i)
+    print('  %%ptr%d = getelementptr i8, i8 *%%base, i64 %d' % (i, offset))
+    print('  store volatile i8 %d, i8 *%%ptr%d' % (value, i))
 
-for i in xrange(branch_blocks):
-    print '  %%astop%d = getelementptr i8, i8 *%%stop, i64 %d' % (i, i + 25)
-    print '  %%acur%d = load i8 , i8 *%%astop%d' % (i, i)
-    print '  %%aext%d = sext i8 %%acur%d to i32' % (i, i)
-    print '  %%atest%d = icmp ult i32 %%limit, %%aext%d' % (i, i)
-    print '  br i1 %%atest%d, label %%main, label %%after%d' % (i, i)
-    print ''
-    print 'after%d:' % i
+for i in range(branch_blocks):
+    print('  %%astop%d = getelementptr i8, i8 *%%stop, i64 %d' % (i, i + 25))
+    print('  %%acur%d = load i8 , i8 *%%astop%d' % (i, i))
+    print('  %%aext%d = sext i8 %%acur%d to i32' % (i, i))
+    print('  %%atest%d = icmp ult i32 %%limit, %%aext%d' % (i, i))
+    print('  br i1 %%atest%d, label %%main, label %%after%d' % (i, i))
+    print('')
+    print('after%d:' % i)
 
-print '  %dummy = load volatile i32, i32 *@global'
-print '  ret void'
-print '}'
+print('  %dummy = load volatile i32, i32 *@global')
+print('  ret void')
+print('}')
diff --git a/test/CodeGen/SystemZ/Large/branch-range-10.py b/test/CodeGen/SystemZ/Large/branch-range-10.py
index c6f8945e294070cdb07fea29921dbc4a26aea320..258989b748f49fc67cd56c915ef69a3c46bbd403 100644
--- a/test/CodeGen/SystemZ/Large/branch-range-10.py
+++ b/test/CodeGen/SystemZ/Large/branch-range-10.py
@@ -71,44 +71,46 @@
 # CHECK: clgr %r4, [[REG]]
 # CHECK: jgl [[LABEL]]
 
+from __future__ import print_function
+
 branch_blocks = 8
 main_size = 0xffcc
 
-print '@global = global i32 0'
+print('@global = global i32 0')
 
-print 'define void @f1(i8 *%base, i8 *%stop, i64 %limit) {'
-print 'entry:'
-print '  br label %before0'
-print ''
+print('define void @f1(i8 *%base, i8 *%stop, i64 %limit) {')
+print('entry:')
+print('  br label %before0')
+print('')
 
-for i in xrange(branch_blocks):
+for i in range(branch_blocks):
     next = 'before%d' % (i + 1) if i + 1 < branch_blocks else 'main'
-    print 'before%d:' % i
-    print '  %%bstop%d = getelementptr i8, i8 *%%stop, i64 %d' % (i, i)
-    print '  %%bcur%d = load i8 , i8 *%%bstop%d' % (i, i)
-    print '  %%bext%d = sext i8 %%bcur%d to i64' % (i, i)
-    print '  %%btest%d = icmp ult i64 %%limit, %%bext%d' % (i, i)
-    print '  br i1 %%btest%d, label %%after0, label %%%s' % (i, next)
-    print ''
+    print('before%d:' % i)
+    print('  %%bstop%d = getelementptr i8, i8 *%%stop, i64 %d' % (i, i))
+    print('  %%bcur%d = load i8 , i8 *%%bstop%d' % (i, i))
+    print('  %%bext%d = sext i8 %%bcur%d to i64' % (i, i))
+    print('  %%btest%d = icmp ult i64 %%limit, %%bext%d' % (i, i))
+    print('  br i1 %%btest%d, label %%after0, label %%%s' % (i, next))
+    print('')
 
-print '%s:' % next
+print('%s:' % next)
 a, b = 1, 1
-for i in xrange(0, main_size, 6):
+for i in range(0, main_size, 6):
     a, b = b, a + b
     offset = 4096 + b % 500000
     value = a % 256
-    print '  %%ptr%d = getelementptr i8, i8 *%%base, i64 %d' % (i, offset)
-    print '  store volatile i8 %d, i8 *%%ptr%d' % (value, i)
+    print('  %%ptr%d = getelementptr i8, i8 *%%base, i64 %d' % (i, offset))
+    print('  store volatile i8 %d, i8 *%%ptr%d' % (value, i))
 
-for i in xrange(branch_blocks):
-    print '  %%astop%d = getelementptr i8, i8 *%%stop, i64 %d' % (i, i + 25)
-    print '  %%acur%d = load i8 , i8 *%%astop%d' % (i, i)
-    print '  %%aext%d = sext i8 %%acur%d to i64' % (i, i)
-    print '  %%atest%d = icmp ult i64 %%limit, %%aext%d' % (i, i)
-    print '  br i1 %%atest%d, label %%main, label %%after%d' % (i, i)
-    print ''
-    print 'after%d:' % i
+for i in range(branch_blocks):
+    print('  %%astop%d = getelementptr i8, i8 *%%stop, i64 %d' % (i, i + 25))
+    print('  %%acur%d = load i8 , i8 *%%astop%d' % (i, i))
+    print('  %%aext%d = sext i8 %%acur%d to i64' % (i, i))
+    print('  %%atest%d = icmp ult i64 %%limit, %%aext%d' % (i, i))
+    print('  br i1 %%atest%d, label %%main, label %%after%d' % (i, i))
+    print('')
+    print('after%d:' % i)
 
-print '  %dummy = load volatile i32, i32 *@global'
-print '  ret void'
-print '}'
+print('  %dummy = load volatile i32, i32 *@global')
+print('  ret void')
+print('}')
diff --git a/test/CodeGen/SystemZ/Large/branch-range-11.py b/test/CodeGen/SystemZ/Large/branch-range-11.py
index 10466df8baa703c710b21a976631696129acbadf..22776aaabf0eb752c8486d06430d54bac1aa1f6d 100644
--- a/test/CodeGen/SystemZ/Large/branch-range-11.py
+++ b/test/CodeGen/SystemZ/Large/branch-range-11.py
@@ -87,44 +87,46 @@
 # CHECK: clfi [[REG]], 107
 # CHECK: jgl [[LABEL]]
 
+from __future__ import print_function
+
 branch_blocks = 8
 main_size = 0xffc6
 
-print '@global = global i32 0'
+print('@global = global i32 0')
 
-print 'define void @f1(i8 *%base, i32 *%stopa, i32 *%stopb) {'
-print 'entry:'
-print '  br label %before0'
-print ''
+print('define void @f1(i8 *%base, i32 *%stopa, i32 *%stopb) {')
+print('entry:')
+print('  br label %before0')
+print('')
 
-for i in xrange(branch_blocks):
+for i in range(branch_blocks):
     next = 'before%d' % (i + 1) if i + 1 < branch_blocks else 'main'
-    print 'before%d:' % i
-    print '  %%bcur%da = load i32 , i32 *%%stopa' % i
-    print '  %%bcur%db = load i32 , i32 *%%stopb' % i
-    print '  %%bsub%d = sub i32 %%bcur%da, %%bcur%db' % (i, i, i)
-    print '  %%btest%d = icmp ult i32 %%bsub%d, %d' % (i, i, i + 50)
-    print '  br i1 %%btest%d, label %%after0, label %%%s' % (i, next)
-    print ''
+    print('before%d:' % i)
+    print('  %%bcur%da = load i32 , i32 *%%stopa' % i)
+    print('  %%bcur%db = load i32 , i32 *%%stopb' % i)
+    print('  %%bsub%d = sub i32 %%bcur%da, %%bcur%db' % (i, i, i))
+    print('  %%btest%d = icmp ult i32 %%bsub%d, %d' % (i, i, i + 50))
+    print('  br i1 %%btest%d, label %%after0, label %%%s' % (i, next))
+    print('')
 
-print '%s:' % next
+print('%s:' % next)
 a, b = 1, 1
-for i in xrange(0, main_size, 6):
+for i in range(0, main_size, 6):
     a, b = b, a + b
     offset = 4096 + b % 500000
     value = a % 256
-    print '  %%ptr%d = getelementptr i8, i8 *%%base, i64 %d' % (i, offset)
-    print '  store volatile i8 %d, i8 *%%ptr%d' % (value, i)
+    print('  %%ptr%d = getelementptr i8, i8 *%%base, i64 %d' % (i, offset))
+    print('  store volatile i8 %d, i8 *%%ptr%d' % (value, i))
 
-for i in xrange(branch_blocks):
-    print '  %%acur%da = load i32 , i32 *%%stopa' % i
-    print '  %%acur%db = load i32 , i32 *%%stopb' % i
-    print '  %%asub%d = sub i32 %%acur%da, %%acur%db' % (i, i, i)
-    print '  %%atest%d = icmp ult i32 %%asub%d, %d' % (i, i, i + 100)
-    print '  br i1 %%atest%d, label %%main, label %%after%d' % (i, i)
-    print ''
-    print 'after%d:' % i
+for i in range(branch_blocks):
+    print('  %%acur%da = load i32 , i32 *%%stopa' % i)
+    print('  %%acur%db = load i32 , i32 *%%stopb' % i)
+    print('  %%asub%d = sub i32 %%acur%da, %%acur%db' % (i, i, i))
+    print('  %%atest%d = icmp ult i32 %%asub%d, %d' % (i, i, i + 100))
+    print('  br i1 %%atest%d, label %%main, label %%after%d' % (i, i))
+    print('')
+    print('after%d:' % i)
 
-print '  %dummy = load volatile i32, i32 *@global'
-print '  ret void'
-print '}'
+print('  %dummy = load volatile i32, i32 *@global')
+print('  ret void')
+print('}')
diff --git a/test/CodeGen/SystemZ/Large/branch-range-12.py b/test/CodeGen/SystemZ/Large/branch-range-12.py
index 809483a9fcd2739f024618ae42ec8e7d1de7d385..0911953a47a3abf19b3a1eef46d2c21c2a86bb4f 100644
--- a/test/CodeGen/SystemZ/Large/branch-range-12.py
+++ b/test/CodeGen/SystemZ/Large/branch-range-12.py
@@ -87,44 +87,46 @@
 # CHECK: clgfi [[REG]], 107
 # CHECK: jgl [[LABEL]]
 
+from __future__ import print_function
+
 branch_blocks = 8
 main_size = 0xffb4
 
-print '@global = global i32 0'
+print('@global = global i32 0')
 
-print 'define void @f1(i8 *%base, i64 *%stopa, i64 *%stopb) {'
-print 'entry:'
-print '  br label %before0'
-print ''
+print('define void @f1(i8 *%base, i64 *%stopa, i64 *%stopb) {')
+print('entry:')
+print('  br label %before0')
+print('')
 
-for i in xrange(branch_blocks):
+for i in range(branch_blocks):
     next = 'before%d' % (i + 1) if i + 1 < branch_blocks else 'main'
-    print 'before%d:' % i
-    print '  %%bcur%da = load i64 , i64 *%%stopa' % i
-    print '  %%bcur%db = load i64 , i64 *%%stopb' % i
-    print '  %%bsub%d = sub i64 %%bcur%da, %%bcur%db' % (i, i, i)
-    print '  %%btest%d = icmp ult i64 %%bsub%d, %d' % (i, i, i + 50)
-    print '  br i1 %%btest%d, label %%after0, label %%%s' % (i, next)
-    print ''
+    print('before%d:' % i)
+    print('  %%bcur%da = load i64 , i64 *%%stopa' % i)
+    print('  %%bcur%db = load i64 , i64 *%%stopb' % i)
+    print('  %%bsub%d = sub i64 %%bcur%da, %%bcur%db' % (i, i, i))
+    print('  %%btest%d = icmp ult i64 %%bsub%d, %d' % (i, i, i + 50))
+    print('  br i1 %%btest%d, label %%after0, label %%%s' % (i, next))
+    print('')
 
-print '%s:' % next
+print('%s:' % next)
 a, b = 1, 1
-for i in xrange(0, main_size, 6):
+for i in range(0, main_size, 6):
     a, b = b, a + b
     offset = 4096 + b % 500000
     value = a % 256
-    print '  %%ptr%d = getelementptr i8, i8 *%%base, i64 %d' % (i, offset)
-    print '  store volatile i8 %d, i8 *%%ptr%d' % (value, i)
+    print('  %%ptr%d = getelementptr i8, i8 *%%base, i64 %d' % (i, offset))
+    print('  store volatile i8 %d, i8 *%%ptr%d' % (value, i))
 
-for i in xrange(branch_blocks):
-    print '  %%acur%da = load i64 , i64 *%%stopa' % i
-    print '  %%acur%db = load i64 , i64 *%%stopb' % i
-    print '  %%asub%d = sub i64 %%acur%da, %%acur%db' % (i, i, i)
-    print '  %%atest%d = icmp ult i64 %%asub%d, %d' % (i, i, i + 100)
-    print '  br i1 %%atest%d, label %%main, label %%after%d' % (i, i)
-    print ''
-    print 'after%d:' % i
+for i in range(branch_blocks):
+    print('  %%acur%da = load i64 , i64 *%%stopa' % i)
+    print('  %%acur%db = load i64 , i64 *%%stopb' % i)
+    print('  %%asub%d = sub i64 %%acur%da, %%acur%db' % (i, i, i))
+    print('  %%atest%d = icmp ult i64 %%asub%d, %d' % (i, i, i + 100))
+    print('  br i1 %%atest%d, label %%main, label %%after%d' % (i, i))
+    print('')
+    print('after%d:' % i)
 
-print '  %dummy = load volatile i32, i32 *@global'
-print '  ret void'
-print '}'
+print('  %dummy = load volatile i32, i32 *@global')
+print('  ret void')
+print('}')
diff --git a/test/CodeGen/SystemZ/Large/spill-01.py b/test/CodeGen/SystemZ/Large/spill-01.py
index f59f607d5beb99c891be64c1f9c30061db11a5b1..2831804df9e575a06b87fbc4540fa535aed85b30 100644
--- a/test/CodeGen/SystemZ/Large/spill-01.py
+++ b/test/CodeGen/SystemZ/Large/spill-01.py
@@ -18,23 +18,26 @@
 # CHECK: lay [[REG:%r[0-5]]], 4096(%r15)
 # CHECK: mvc {{[0-9]+}}(8,{{%r[0-9]+}}), 8([[REG]])
 # CHECK: br %r14
+
+from __future__ import print_function
+
 count = 500
 
-print 'declare void @foo()'
-print ''
-print 'define void @f1(i64 *%base0, i64 *%base1) {'
+print('declare void @foo()')
+print('')
+print('define void @f1(i64 *%base0, i64 *%base1) {')
 
 for i in range(count):
-    print '  %%ptr%d = getelementptr i64, i64 *%%base%d, i64 %d' % (i, i % 2, i / 2)
-    print '  %%val%d = load i64 , i64 *%%ptr%d' % (i, i)
-    print ''
+    print('  %%ptr%d = getelementptr i64, i64 *%%base%d, i64 %d' % (i, i % 2, i / 2))
+    print('  %%val%d = load i64 , i64 *%%ptr%d' % (i, i))
+    print('')
 
-print '  call void @foo()'
-print ''
+print('  call void @foo()')
+print('')
 
 for i in range(count):
-    print '  store i64 %%val%d, i64 *%%ptr%d' % (i, i)
+    print('  store i64 %%val%d, i64 *%%ptr%d' % (i, i))
 
-print ''
-print '  ret void'
-print '}'
+print('')
+print('  ret void')
+print('}')
diff --git a/test/CodeGen/SystemZ/Large/spill-02.py b/test/CodeGen/SystemZ/Large/spill-02.py
index 4ccfa11a0d3e6a62a8cecf52e0bf55f6db86896f..f8359932efb2cfae0a7fad9aed13c7b885d97c87 100644
--- a/test/CodeGen/SystemZ/Large/spill-02.py
+++ b/test/CodeGen/SystemZ/Large/spill-02.py
@@ -19,55 +19,58 @@
 # the first 8168 bytes to be used for the call.  160 of these bytes are
 # allocated for the ABI frame.  There are also 5 argument registers, one of
 # which is used as a base pointer.
+
+from __future__ import print_function
+
 args = (8168 - 160) / 8 + (5 - 1)
 
-print 'declare i64 *@foo(i64 *%s)' % (', i64' * args)
-print 'declare void @bar(i64 *)'
-print ''
-print 'define i64 @f1(i64 %foo) {'
-print 'entry:'
+print('declare i64 *@foo(i64 *%s)' % (', i64' * args))
+print('declare void @bar(i64 *)')
+print('')
+print('define i64 @f1(i64 %foo) {')
+print('entry:')
 
 # Make the allocation big, so that it goes at the top of the frame.
-print '  %array = alloca [1000 x i64]'
-print '  %area = getelementptr [1000 x i64], [1000 x i64] *%array, i64 0, i64 0'
-print '  %%base = call i64 *@foo(i64 *%%area%s)' % (', i64 0' * args)
-print ''
+print('  %array = alloca [1000 x i64]')
+print('  %area = getelementptr [1000 x i64], [1000 x i64] *%array, i64 0, i64 0')
+print('  %%base = call i64 *@foo(i64 *%%area%s)' % (', i64 0' * args))
+print('')
 
 # Make sure all GPRs are used.  One is needed for the stack pointer and
 # another for %base, so we need 14 live values.
 count = 14
 for i in range(count):
-    print '  %%ptr%d = getelementptr i64, i64 *%%base, i64 %d' % (i, i / 2)
-    print '  %%val%d = load volatile i64 , i64 *%%ptr%d' % (i, i)
-    print ''
+    print('  %%ptr%d = getelementptr i64, i64 *%%base, i64 %d' % (i, i / 2))
+    print('  %%val%d = load volatile i64 , i64 *%%ptr%d' % (i, i))
+    print('')
 
 # Encourage the register allocator to give preference to these %vals
 # by using them several times.
 for j in range(4):
     for i in range(count):
-        print '  store volatile i64 %%val%d, i64 *%%ptr%d' % (i, i)
-    print ''
+        print('  store volatile i64 %%val%d, i64 *%%ptr%d' % (i, i))
+    print('')
 
 # Copy the incoming argument, which we expect to be spilled, to the frame
 # index for the alloca area.  Also throw in a volatile store, so that this
 # block cannot be reordered with the surrounding code.
-print '  %cond = icmp eq i64 %val0, %val1'
-print '  br i1 %cond, label %skip, label %fallthru'
-print ''
-print 'fallthru:'
-print '  store i64 %foo, i64 *%area'
-print '  store volatile i64 %val0, i64 *%ptr0'
-print '  br label %skip'
-print ''
-print 'skip:'
+print('  %cond = icmp eq i64 %val0, %val1')
+print('  br i1 %cond, label %skip, label %fallthru')
+print('')
+print('fallthru:')
+print('  store i64 %foo, i64 *%area')
+print('  store volatile i64 %val0, i64 *%ptr0')
+print('  br label %skip')
+print('')
+print('skip:')
 
 # Use each %val a few more times to emphasise the point, and to make sure
 # that they are live across the store of %foo.
 for j in range(4):
     for i in range(count):
-        print '  store volatile i64 %%val%d, i64 *%%ptr%d' % (i, i)
-    print ''
+        print('  store volatile i64 %%val%d, i64 *%%ptr%d' % (i, i))
+    print('')
 
-print '  call void @bar(i64 *%area)'
-print '  ret i64 0'
-print '}'
+print('  call void @bar(i64 *%area)')
+print('  ret i64 0')
+print('}')
diff --git a/test/CodeGen/SystemZ/buildvector-00.ll b/test/CodeGen/SystemZ/buildvector-00.ll
new file mode 100644
index 0000000000000000000000000000000000000000..dc1c0ffd4f684acf610e0f9ef2fd7e38a64a7a01
--- /dev/null
+++ b/test/CodeGen/SystemZ/buildvector-00.ll
@@ -0,0 +1,36 @@
+; Test that the dag combiner can understand that some vector operands are
+; all-zeros and then optimize the logical operations.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
+
+define void @f1() {
+; CHECK-LABEL: f1:
+; CHECK: vno
+; CHECK-NOT: vno
+
+bb:
+  %tmp = shufflevector <2 x i64> undef, <2 x i64> undef, <2 x i32> zeroinitializer
+  br label %bb1
+
+bb1:                                              ; preds = %bb
+  %tmp2 = load i64, i64* undef, align 8
+  %tmp3 = insertelement <2 x i64> undef, i64 %tmp2, i32 1
+  %tmp4 = icmp ne <2 x i64> undef, zeroinitializer
+  %tmp5 = xor <2 x i1> %tmp4, zeroinitializer
+  %tmp6 = xor <2 x i1> zeroinitializer, %tmp5
+  %tmp7 = and <2 x i64> %tmp3, %tmp
+  %tmp8 = icmp ne <2 x i64> %tmp7, zeroinitializer
+  %tmp9 = xor <2 x i1> zeroinitializer, %tmp8
+  %tmp10 = icmp ne <2 x i64> undef, zeroinitializer
+  %tmp11 = xor <2 x i1> %tmp10, %tmp9
+  %tmp12 = and <2 x i1> %tmp6, %tmp11
+  %tmp13 = extractelement <2 x i1> %tmp12, i32 0
+  br i1 %tmp13, label %bb14, label %bb15
+
+bb14:                                             ; preds = %bb1
+  store i64 undef, i64* undef, align 8
+  br label %bb15
+
+bb15:                                             ; preds = %bb14, %bb1
+  unreachable
+}
diff --git a/test/CodeGen/SystemZ/cond-move-05.mir b/test/CodeGen/SystemZ/cond-move-05.mir
index 95507c9e7653d9972c23e156cc17f8be961fe068..de33f49734f010db97cddfd6e16b77a67654adf8 100644
--- a/test/CodeGen/SystemZ/cond-move-05.mir
+++ b/test/CodeGen/SystemZ/cond-move-05.mir
@@ -38,7 +38,7 @@
   
   !llvm.ident = !{!0}
   
-  !0 = !{!"clang version 6.0.0 (http://llvm.org/git/clang.git d80246686d6ad2a749d11470afbbd1bbe4d1b561) (http://llvm.org/git/llvm.git 1693ef38604a3ad9c3da656d2b58a77312207b01)"}
+  !0 = !{!"clang version 6.0.0"}
 
 ...
 
diff --git a/test/CodeGen/SystemZ/debuginstr-00.mir b/test/CodeGen/SystemZ/debuginstr-00.mir
new file mode 100644
index 0000000000000000000000000000000000000000..53a43d03a22a6c73e972fb365f12892dac11bef4
--- /dev/null
+++ b/test/CodeGen/SystemZ/debuginstr-00.mir
@@ -0,0 +1,85 @@
+# Check that the backend can produce a branch-on-count instruction also in
+# the presence of a DEBUG_VALUE machine instruction.
+#
+# RUN: llc %s -mtriple=s390x-linux-gnu -mcpu=z13 -start-before=postrapseudos \
+# RUN:   -o - 2>&1 | FileCheck %s
+#
+# CHECK: brctg
+
+--- |
+  
+  define dso_local void @put_charge_groups_in_box() local_unnamed_addr #0 !dbg !5 {
+  bb:
+    br label %bb2
+  
+  bb2:                                              ; preds = %bb2, %bb3, %bb
+    br i1 undef, label %bb2, label %bb3.preheader
+  
+  bb3.preheader:                                    ; preds = %bb2
+    br label %bb3
+  
+  bb3:                                              ; preds = %bb3.preheader, %bb3
+    %tmp4 = phi i64 [ %tmp6, %bb3 ], [ 2, %bb3.preheader ]
+    %tmp6 = add nsw i64 %tmp4, -1
+    ; Keep the dbg metadata live by referencing it in the IR.
+    call void @llvm.dbg.value(metadata i32 0, i64 0, metadata !7, metadata !DIExpression())
+    %tmp8 = icmp eq i64 %tmp6, 0
+    br i1 %tmp8, label %bb2, label %bb3
+  }
+  
+  declare void @llvm.dbg.value(metadata, i64, metadata, metadata) #1
+  
+  declare void @llvm.stackprotector(i8*, i8**) #2
+  
+  attributes #0 = { "target-cpu"="z13" "use-soft-float"="false" }
+  attributes #1 = { nounwind readnone speculatable "target-cpu"="z13" }
+  attributes #2 = { nounwind }
+  
+  !llvm.dbg.cu = !{!0}
+  !llvm.module.flags = !{!3, !4}
+  
+  !0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "llvm", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !2)
+  !1 = !DIFile(filename: "test.ll", directory: "/tmp")
+  !2 = !{}
+  !3 = !{i32 2, !"Dwarf Version", i32 4}
+  !4 = !{i32 2, !"Debug Info Version", i32 3}
+  !5 = distinct !DISubprogram(name: "test_dbg_value", scope: !1, file: !1, line: 1, type: !6, isLocal: false, isDefinition: true, scopeLine: 1, flags: DIFlagPrototyped, isOptimized: false, unit: !0, retainedNodes: !2)
+  !6 = !DISubroutineType(types: !2)
+  !7 = !DILocalVariable(name: "in", arg: 1, scope: !5, file: !1, line: 1, type: !8)
+  !8 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+  !9 = !DILocation(line: 1, column: 1, scope: !5)
+
+...
+---
+name:            put_charge_groups_in_box
+alignment:       4
+tracksRegLiveness: true
+frameInfo:       
+  maxCallFrameSize: 0
+body:             |
+  bb.0.bb:
+    renamable $r0l = LHIMux 0
+  
+  bb.1.bb2:
+    liveins: $r0l
+  
+    CHIMux renamable $r0l, 0, implicit-def $cc
+    BRC 14, 6, %bb.1, implicit killed $cc
+  
+  bb.2.bb3.preheader:
+    liveins: $r0l
+  
+    renamable $r1d = LGHI 2
+  
+  bb.3.bb3:
+    successors: %bb.1(0x04000000), %bb.3(0x7c000000)
+    liveins: $r1d, $r0l
+  
+    renamable $r1d = nsw AGHI killed renamable $r1d, -1, implicit-def dead $cc
+    DBG_VALUE $r1d, $noreg, !7, !DIExpression(), debug-location !9
+    CGHI renamable $r1d, 0, implicit-def $cc
+    DBG_VALUE $r1d, $noreg, !7, !DIExpression(), debug-location !9
+    BRC 14, 6, %bb.3, implicit $cc
+    J %bb.1
+
+...
diff --git a/test/CodeGen/SystemZ/debuginstr-01.mir b/test/CodeGen/SystemZ/debuginstr-01.mir
new file mode 100644
index 0000000000000000000000000000000000000000..bcc7f547941b4cd41b8835e2cefec75945503711
--- /dev/null
+++ b/test/CodeGen/SystemZ/debuginstr-01.mir
@@ -0,0 +1,80 @@
+# Check that the backend can produce a load-and-trap instruction also in
+# the presence of a DEBUG_VALUE machine instruction.
+#
+# RUN: llc %s -mtriple=s390x-linux-gnu -mcpu=zEC12 -start-before=postrapseudos \
+# RUN:   -o - 2>&1 | FileCheck %s
+#
+# CHECK: lat
+
+--- |
+  
+  define i32 @f1(i32* %ptr) #1 {
+  entry:
+    %val = load i32, i32* %ptr
+    ; Keep the dbg metadata live by referencing it in the IR.
+    call void @llvm.dbg.value(metadata i32 0, i64 0, metadata !7, metadata !DIExpression()), !dbg !9
+    %cmp = icmp eq i32 %val, 0
+    br i1 %cmp, label %if.then, label %if.end
+  
+  if.then:                                          ; preds = %entry
+    tail call void @llvm.trap()
+    unreachable
+  
+  if.end:                                           ; preds = %entry
+    ret i32 %val
+  }
+  
+  declare void @llvm.trap() #0
+  declare void @llvm.dbg.value(metadata, i64, metadata, metadata)
+  
+  declare void @llvm.stackprotector(i8*, i8**) #2
+
+  attributes #0 = { cold noreturn nounwind "target-cpu"="zEC12" }
+  attributes #1 = { "target-cpu"="zEC12" }
+  attributes #2 = { nounwind }
+
+  !llvm.dbg.cu = !{!0}
+  !llvm.module.flags = !{!3, !4}
+
+  !0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "llvm", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !2)
+  !1 = !DIFile(filename: "test.ll", directory: "/tmp")
+  !2 = !{}
+  !3 = !{i32 2, !"Dwarf Version", i32 4}
+  !4 = !{i32 2, !"Debug Info Version", i32 3}
+  !5 = distinct !DISubprogram(name: "test_dbg_value", scope: !1, file: !1, line: 1, type: !6, isLocal: false, isDefinition: true, scopeLine: 1, flags: DIFlagPrototyped, isOptimized: false, unit: !0, retainedNodes: !2)
+  !6 = !DISubroutineType(types: !2)
+  !7 = !DILocalVariable(name: "in", arg: 1, scope: !5, file: !1, line: 1, type: !8)
+  !8 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+  !9 = !DILocation(line: 1, column: 1, scope: !5)
+
+...
+---
+name:            f1
+alignment:       4
+tracksRegLiveness: true
+liveins:         
+  - { reg: '$r2d' }
+frameInfo:       
+  maxCallFrameSize: 0
+body:             |
+  bb.0.entry:
+    successors: %bb.1(0x00000001), %bb.2(0x7fffffff)
+    liveins: $r2d
+  
+    renamable $r2l = LMux killed renamable $r2d, 0, $noreg :: (load 4 from %ir.ptr)
+    DBG_VALUE $noreg, $r2l, !7, !DIExpression(), debug-location !9
+    CHIMux renamable $r2l, 0, implicit-def $cc
+    DBG_VALUE $noreg, $r2l, !7, !DIExpression(), debug-location !9
+    BRC 14, 6, %bb.2, implicit killed $cc
+  
+  bb.1.if.then:
+    successors: 
+  
+    Trap
+  
+  bb.2.if.end:
+    liveins: $r2l
+  
+    Return implicit $r2l
+
+...
diff --git a/test/CodeGen/SystemZ/debuginstr-02.mir b/test/CodeGen/SystemZ/debuginstr-02.mir
new file mode 100644
index 0000000000000000000000000000000000000000..205e8b9095a0c8bd06be94c625d1bf7d7dfaa2ce
--- /dev/null
+++ b/test/CodeGen/SystemZ/debuginstr-02.mir
@@ -0,0 +1,93 @@
+# Check that the backend can handle consecutive select instructions also in
+# the presence of DEBUG_VALUE machine instructions.
+#
+# RUN: llc %s -verify-machineinstrs -mtriple=s390x-linux-gnu -mcpu=z13 \
+# RUN:   -start-before=expand-isel-pseudos -o - 2>&1 | FileCheck %s
+#
+# CHECK-LABEL: %bb.1:
+# CHECK:       ldr
+# CHECK-NEXT:  ldr
+# CHECK-NEXT:  ldr
+
+--- |
+  ; ModuleID = 'tc.ll'
+  source_filename = "tc.ll"
+  target datalayout = "E-m:e-i1:8:16-i8:8:16-i64:64-f128:64-v128:64-a:8:16-n32:64"
+  
+  define float @fun(i1 %cmp, float %arg0, float %arg1, float %arg2, float %arg3, float %arg4) #0 {
+    %fs0 = select i1 %cmp, float %arg0, float %arg1
+    ; Keep the dbg metadata live by referencing it in the IR.
+    call void @llvm.dbg.value(metadata i32 0, i64 0, metadata !7, metadata !DIExpression()), !dbg !9
+    %fs1 = select i1 %cmp, float %arg2, float %arg3
+    call void @llvm.dbg.value(metadata i32 0, i64 0, metadata !7, metadata !DIExpression()), !dbg !9
+    %fs2 = select i1 %cmp, float %arg0, float %arg3
+    %s0 = fadd float %fs0, %fs1
+    %s1 = fadd float %s0, %fs2
+    ret float %s1
+  }
+  
+  declare void @llvm.dbg.value(metadata, i64, metadata, metadata) #1
+  
+  attributes #0 = { "target-cpu"="z13" }
+
+  !llvm.dbg.cu = !{!0}
+  !llvm.module.flags = !{!3, !4}
+  
+  !0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "llvm", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !2)
+  !1 = !DIFile(filename: "test.ll", directory: "/tmp")
+  !2 = !{}
+  !3 = !{i32 2, !"Dwarf Version", i32 4}
+  !4 = !{i32 2, !"Debug Info Version", i32 3}
+  !5 = distinct !DISubprogram(name: "test_dbg_value", scope: !1, file: !1, line: 1, type: !6, isLocal: false, isDefinition: true, scopeLine: 1, flags: DIFlagPrototyped, isOptimized: false, unit: !0, retainedNodes: !2)
+  !6 = !DISubroutineType(types: !2)
+  !7 = !DILocalVariable(name: "in", arg: 1, scope: !5, file: !1, line: 1, type: !8)
+  !8 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+  !9 = !DILocation(line: 1, column: 1, scope: !5)
+
+...
+---
+name:            fun
+alignment:       4
+tracksRegLiveness: true
+registers:       
+  - { id: 0, class: gr32bit }
+  - { id: 1, class: fp32bit }
+  - { id: 2, class: fp32bit }
+  - { id: 3, class: fp32bit }
+  - { id: 4, class: fp32bit }
+  - { id: 5, class: fp32bit }
+  - { id: 6, class: fp32bit }
+  - { id: 7, class: fp32bit }
+  - { id: 8, class: fp32bit }
+  - { id: 9, class: fp32bit }
+liveins:         
+  - { reg: '$r2l', virtual-reg: '%0' }
+  - { reg: '$f0s', virtual-reg: '%1' }
+  - { reg: '$f2s', virtual-reg: '%2' }
+  - { reg: '$f4s', virtual-reg: '%3' }
+  - { reg: '$f6s', virtual-reg: '%4' }
+frameInfo:       
+  maxAlignment:    8
+fixedStack:      
+  - { id: 0, size: 4, alignment: 8, stack-id: 0, isImmutable: true }
+body:             |
+  bb.0 (%ir-block.0):
+    liveins: $r2l, $f0s, $f2s, $f4s, $f6s
+  
+    %4:fp32bit = COPY $f6s
+    %3:fp32bit = COPY $f4s
+    %2:fp32bit = COPY $f2s
+    %1:fp32bit = COPY $f0s
+    %0:gr32bit = COPY $r2l
+    TMLMux %0, 1, implicit-def $cc
+    %5:fp32bit = SelectVR32 %1, %2, 15, 7, implicit $cc
+    DBG_VALUE %5, $noreg, !7, !DIExpression(), debug-location !9
+    %6:fp32bit = SelectVR32 %3, %4, 15, 7, implicit $cc
+    DBG_VALUE %6, $noreg, !7, !DIExpression(), debug-location !9
+    %7:fp32bit = SelectVR32 %1, %4, 15, 7, implicit $cc
+    %8:fp32bit = AEBR %5, killed %6, implicit-def dead $cc
+    %9:fp32bit = AEBR %8, killed %7, implicit-def dead $cc
+    $f0s = COPY %9
+    Return implicit $f0s
+
+...
diff --git a/test/CodeGen/SystemZ/debuginstr-cgp.mir b/test/CodeGen/SystemZ/debuginstr-cgp.mir
new file mode 100644
index 0000000000000000000000000000000000000000..8d76c33f8ff54401e25731dce02b318c8d799ae0
--- /dev/null
+++ b/test/CodeGen/SystemZ/debuginstr-cgp.mir
@@ -0,0 +1,171 @@
+# Check that the codegenprepare succeeds in dupRetToEnableTailCallOpts() also
+# in the presence of a call to @llvm.dbg.value()
+#
+# RUN: llc %s -mtriple=s390x-linux-gnu -mcpu=z13 -start-before=codegenprepare \
+# RUN:   -stop-after codegenprepare -o - | FileCheck %s
+#
+# CHECK-LABEL: bb2:
+# CHECK:       ret
+# CHECK-LABEL: bb4:
+# CHECK:       ret
+
+
+# Generated with:
+#
+# bin/llc -mtriple=s390x-linux-gnu -mcpu=z13 -stop-before codegenprepare -simplify-mir
+#
+# %0 = type { i32 (...)**, i16, %1* }
+# %1 = type { i32 (...)** }
+# %2 = type { i32 (...)**, %1*, i8, i32, i32, i32, i16, i32, i16, i32, i16*, %3*, %6*, %9 }
+# %3 = type { %4 }
+# %4 = type { i32 (...)**, i8, i32, i32, %5**, %1* }
+# %5 = type { i32, i32 }
+# %6 = type { %7*, %0*, %0*, %0*, %0*, %0*, %0*, %0*, %0*, %0*, %0*, %0*, %0*, %0*, %0*, %1* }
+# %7 = type { %8 }
+# %8 = type { i32 (...)**, i8, i32, i32, %0**, %1* }
+# %9 = type { i8* }
+# %10 = type { %0, i32, i32, %0* }
+#
+# define %0* @Fun(%2* %arg) !dbg !7 {
+# bb:
+#   switch i32 undef, label %bb3 [
+#     i32 58, label %bb1
+#     i32 41, label %bb2
+#   ], !dbg !14
+#
+# bb1:                                              ; preds = %bb
+#   br label %bb4, !dbg !15
+#
+# bb2:                                              ; preds = %bb
+#   %tmp = tail call %10* @hoge(%6* undef, %0* undef, i32 signext 0, i32 signext 0), !dbg !16
+#   call void @llvm.dbg.value(metadata %10* %tmp, metadata !10, metadata !DIExpression()), !dbg !16
+#   br label %bb4, !dbg !17
+#
+# bb3:                                              ; preds = %bb
+#   unreachable, !dbg !18
+#
+# bb4:                                              ; preds = %bb2, %bb1
+#   %tmp5 = phi %10* [ undef, %bb1 ], [ %tmp, %bb2 ], !dbg !19
+#   call void @llvm.dbg.value(metadata %10* %tmp5, metadata !12, metadata !DIExpression()), !dbg !19
+#   %tmp6 = bitcast %10* %tmp5 to %0*, !dbg !20
+#   call void @llvm.dbg.value(metadata %0* %tmp6, metadata !13, metadata !DIExpression()), !dbg !20
+#   ret %0* %tmp6, !dbg !21
+# }
+#
+# declare %10* @hoge(%6*, %0*, i32, i32)
+#
+# ; Function Attrs: nounwind readnone speculatable
+# declare void @llvm.dbg.value(metadata, metadata, metadata) #1
+#
+# attributes #0 = { "use-soft-float"="false" }
+# attributes #1 = { nounwind readnone speculatable }
+#
+# !llvm.module.flags = !{!0, !1}
+# !llvm.dbg.cu = !{!2}
+# !llvm.debugify = !{!5, !6}
+#
+# !0 = !{i32 2, !"Debug Info Version", i32 3}
+# !1 = !{i32 1, !"wchar_size", i32 4}
+# !2 = distinct !DICompileUnit(language: DW_LANG_C, file: !3, producer: "debugify", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !4)
+# !3 = !DIFile(filename: "tc.ll", directory: "/")
+# !4 = !{}
+# !5 = !{i32 8}
+# !6 = !{i32 3}
+# !7 = distinct !DISubprogram(name: "eggs", linkageName: "eggs", scope: null, file: !3, line: 1, type: !8, scopeLine: 1, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !2, retainedNodes: !9)
+# !8 = !DISubroutineType(types: !4)
+# !9 = !{!10, !12, !13}
+# !10 = !DILocalVariable(name: "1", scope: !7, file: !3, line: 3, type: !11)
+# !11 = !DIBasicType(name: "ty64", size: 64, encoding: DW_ATE_unsigned)
+# !12 = !DILocalVariable(name: "2", scope: !7, file: !3, line: 6, type: !11)
+# !13 = !DILocalVariable(name: "3", scope: !7, file: !3, line: 7, type: !11)
+# !14 = !DILocation(line: 1, column: 1, scope: !7)
+# !15 = !DILocation(line: 2, column: 1, scope: !7)
+# !16 = !DILocation(line: 3, column: 1, scope: !7)
+# !17 = !DILocation(line: 4, column: 1, scope: !7)
+# !18 = !DILocation(line: 5, column: 1, scope: !7)
+# !19 = !DILocation(line: 6, column: 1, scope: !7)
+# !20 = !DILocation(line: 7, column: 1, scope: !7)
+# !21 = !DILocation(line: 8, column: 1, scope: !7)
+
+
+--- |
+  
+  %0 = type { i32 (...)**, i16, %1* }
+  %1 = type { i32 (...)** }
+  %2 = type { i32 (...)**, %1*, i8, i32, i32, i32, i16, i32, i16, i32, i16*, %3*, %6*, %9 }
+  %3 = type { %4 }
+  %4 = type { i32 (...)**, i8, i32, i32, %5**, %1* }
+  %5 = type { i32, i32 }
+  %6 = type { %7*, %0*, %0*, %0*, %0*, %0*, %0*, %0*, %0*, %0*, %0*, %0*, %0*, %0*, %0*, %1* }
+  %7 = type { %8 }
+  %8 = type { i32 (...)**, i8, i32, i32, %0**, %1* }
+  %9 = type { i8* }
+  %10 = type { %0, i32, i32, %0* }
+  
+  define %0* @Fun(%2* %arg) #0 !dbg !7 {
+  bb:
+    switch i32 undef, label %bb3 [
+      i32 58, label %bb1
+      i32 41, label %bb2
+    ], !dbg !14
+  
+  bb1:                                              ; preds = %bb
+    br label %bb4, !dbg !15
+  
+  bb2:                                              ; preds = %bb
+    %tmp = tail call %10* @hoge(%6* undef, %0* undef, i32 signext 0, i32 signext 0), !dbg !16
+    call void @llvm.dbg.value(metadata %10* %tmp, metadata !10, metadata !DIExpression()), !dbg !16
+    br label %bb4, !dbg !17
+  
+  bb3:                                              ; preds = %bb
+    unreachable, !dbg !18
+  
+  bb4:                                              ; preds = %bb2, %bb1
+    %tmp5 = phi %10* [ undef, %bb1 ], [ %tmp, %bb2 ], !dbg !19
+    call void @llvm.dbg.value(metadata %10* %tmp5, metadata !12, metadata !DIExpression()), !dbg !19
+    %tmp6 = bitcast %10* %tmp5 to %0*, !dbg !20
+    call void @llvm.dbg.value(metadata %0* %tmp6, metadata !13, metadata !DIExpression()), !dbg !20
+    ret %0* %tmp6, !dbg !21
+  }
+  
+  declare %10* @hoge(%6*, %0*, i32, i32) #0
+  
+  ; Function Attrs: nounwind readnone speculatable
+  declare void @llvm.dbg.value(metadata, metadata, metadata) #1
+  
+  attributes #0 = { "target-cpu"="z13" }
+  attributes #1 = { nounwind readnone speculatable "target-cpu"="z13" }
+  
+  !llvm.module.flags = !{!0, !1}
+  !llvm.dbg.cu = !{!2}
+  !llvm.debugify = !{!5, !6}
+  
+  !0 = !{i32 2, !"Debug Info Version", i32 3}
+  !1 = !{i32 1, !"wchar_size", i32 4}
+  !2 = distinct !DICompileUnit(language: DW_LANG_C, file: !3, producer: "debugify", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !4)
+  !3 = !DIFile(filename: "tc.ll", directory: "/")
+  !4 = !{}
+  !5 = !{i32 8}
+  !6 = !{i32 3}
+  !7 = distinct !DISubprogram(name: "eggs", linkageName: "eggs", scope: null, file: !3, line: 1, type: !8, scopeLine: 1, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !2, retainedNodes: !9)
+  !8 = !DISubroutineType(types: !4)
+  !9 = !{!10, !12, !13}
+  !10 = !DILocalVariable(name: "1", scope: !7, file: !3, line: 3, type: !11)
+  !11 = !DIBasicType(name: "ty64", size: 64, encoding: DW_ATE_unsigned)
+  !12 = !DILocalVariable(name: "2", scope: !7, file: !3, line: 6, type: !11)
+  !13 = !DILocalVariable(name: "3", scope: !7, file: !3, line: 7, type: !11)
+  !14 = !DILocation(line: 1, column: 1, scope: !7)
+  !15 = !DILocation(line: 2, column: 1, scope: !7)
+  !16 = !DILocation(line: 3, column: 1, scope: !7)
+  !17 = !DILocation(line: 4, column: 1, scope: !7)
+  !18 = !DILocation(line: 5, column: 1, scope: !7)
+  !19 = !DILocation(line: 6, column: 1, scope: !7)
+  !20 = !DILocation(line: 7, column: 1, scope: !7)
+  !21 = !DILocation(line: 8, column: 1, scope: !7)
+
+...
+---
+name:            Fun
+alignment:       4
+tracksRegLiveness: true
+...
diff --git a/test/CodeGen/SystemZ/fp-const-12.ll b/test/CodeGen/SystemZ/fp-const-12.ll
new file mode 100644
index 0000000000000000000000000000000000000000..f22156ba40e8b8ec34111286f0518e48d80dd125
--- /dev/null
+++ b/test/CodeGen/SystemZ/fp-const-12.ll
@@ -0,0 +1,63 @@
+; Test loads of FP constants with VGM.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
+
+define double @f1() {
+; CHECK-LABEL: f1:
+; CHECK: vgmg %v0, 2, 11
+  ret double 1.0
+}
+
+define double @f2() {
+; CHECK-LABEL: f2:
+; CHECK: vgmg %v0, 1, 1
+  ret double 2.0
+}
+
+define double @f3() {
+; CHECK-LABEL: f3:
+; CHECK: vgmg %v0, 0, 1
+  ret double -2.0
+}
+
+define double @f4() {
+; CHECK-LABEL: f4:
+; CHECK: vgmg %v0, 2, 10
+  ret double 0.5
+}
+
+define double @f5() {
+; CHECK-LABEL: f5:
+; CHECK: vgmg %v0, 2, 9
+  ret double 0.125
+}
+
+define float @f6() {
+; CHECK-LABEL: f6:
+; CHECK: vgmf %v0, 2, 8
+  ret float 1.0
+}
+
+define float @f7() {
+; CHECK-LABEL: f7:
+; CHECK: vgmf %v0, 1, 1
+  ret float 2.0
+}
+
+define float @f8() {
+; CHECK-LABEL: f8:
+; CHECK: vgmf %v0, 0, 1
+  ret float -2.0
+}
+
+define float @f9() {
+; CHECK-LABEL: f9:
+; CHECK: vgmf %v0, 2, 7
+  ret float 0.5
+}
+
+define float @f10() {
+; CHECK-LABEL: f10:
+; CHECK: vgmf %v0, 2, 6
+  ret float 0.125
+}
diff --git a/test/CodeGen/SystemZ/frame-07.ll b/test/CodeGen/SystemZ/frame-07.ll
index b042d9a205ddfdf8317e6f014df69a4b2838c408..c5c36a6a5de4844714b6199ac35db99fbe038a7e 100644
--- a/test/CodeGen/SystemZ/frame-07.ll
+++ b/test/CodeGen/SystemZ/frame-07.ll
@@ -1,7 +1,7 @@
 ; Test the saving and restoring of FPRs in large frames.
 ;
 ; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z10 | FileCheck -check-prefix=CHECK-NOFP %s
-; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z10 -disable-fp-elim | FileCheck -check-prefix=CHECK-FP %s
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z10 -frame-pointer=all | FileCheck -check-prefix=CHECK-FP %s
 
 ; Test a frame size that requires some FPRs to be saved and loaded using
 ; the 20-bit STDY and LDY while others can use the 12-bit STD and LD.
diff --git a/test/CodeGen/SystemZ/frame-09.ll b/test/CodeGen/SystemZ/frame-09.ll
index d6ebc0d46fb576b32116602340c01bedfa008421..8dbd47aa9fa03c83e15175e7d228f0ed592a2b36 100644
--- a/test/CodeGen/SystemZ/frame-09.ll
+++ b/test/CodeGen/SystemZ/frame-09.ll
@@ -1,6 +1,6 @@
 ; Test the handling of the frame pointer (%r11).
 ;
-; RUN: llc < %s -mtriple=s390x-linux-gnu -disable-fp-elim | FileCheck %s
+; RUN: llc < %s -mtriple=s390x-linux-gnu -frame-pointer=all | FileCheck %s
 
 ; We should always initialise %r11 when FP elimination is disabled.
 ; We don't need to allocate any more than the caller-provided 160-byte
diff --git a/test/CodeGen/SystemZ/frame-13.ll b/test/CodeGen/SystemZ/frame-13.ll
index 15503fd44363ac0a3db3fd84362c6ea785d1fe6a..1fb840bc5ba98f5bdcd94fc30cacf16aaa5ed3b9 100644
--- a/test/CodeGen/SystemZ/frame-13.ll
+++ b/test/CodeGen/SystemZ/frame-13.ll
@@ -4,7 +4,7 @@
 ;
 ; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z10 | \
 ; RUN:   FileCheck -check-prefix=CHECK-NOFP %s
-; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z10 -disable-fp-elim | \
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z10 -frame-pointer=all | \
 ; RUN:   FileCheck -check-prefix=CHECK-FP %s
 
 ; This file tests what happens when a displacement is converted from
diff --git a/test/CodeGen/SystemZ/frame-14.ll b/test/CodeGen/SystemZ/frame-14.ll
index d6235d1318fe5b4204c5fdcae5996f2182b88f89..e70731249b42bed56815c435279408e908e342dd 100644
--- a/test/CodeGen/SystemZ/frame-14.ll
+++ b/test/CodeGen/SystemZ/frame-14.ll
@@ -5,7 +5,7 @@
 ;
 ; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z10 | \
 ; RUN:   FileCheck -check-prefix=CHECK-NOFP %s
-; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z10 -disable-fp-elim | \
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z10 -frame-pointer=all | \
 ; RUN:   FileCheck -check-prefix=CHECK-FP %s
 ;
 ; This file tests what happens when a displacement is converted from
diff --git a/test/CodeGen/SystemZ/frame-15.ll b/test/CodeGen/SystemZ/frame-15.ll
index e86ef7ba35a128fc131992a20473f47598e1e13e..0595b5403b05c56cb73cf1d05d8e9690236cdfdf 100644
--- a/test/CodeGen/SystemZ/frame-15.ll
+++ b/test/CodeGen/SystemZ/frame-15.ll
@@ -4,7 +4,7 @@
 ;
 ; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z10 | \
 ; RUN:   FileCheck -check-prefix=CHECK-NOFP %s
-; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z10 -disable-fp-elim | \
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z10 -frame-pointer=all | \
 ; RUN:   FileCheck -check-prefix=CHECK-FP %s
 
 declare void @foo(float *%ptr1, float *%ptr2)
diff --git a/test/CodeGen/SystemZ/frame-16.ll b/test/CodeGen/SystemZ/frame-16.ll
index c99c2a302f10ed42e2af1945c0a6ec315a022acc..ae8a041ae1107ca1cce9032738e511434d26a14f 100644
--- a/test/CodeGen/SystemZ/frame-16.ll
+++ b/test/CodeGen/SystemZ/frame-16.ll
@@ -5,7 +5,7 @@
 ;
 ; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z10 | \
 ; RUN:   FileCheck -check-prefix=CHECK-NOFP %s
-; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z10 -disable-fp-elim | \
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z10 -frame-pointer=all | \
 ; RUN:   FileCheck -check-prefix=CHECK-FP %s
 
 ; This file tests what happens when a displacement is converted from
diff --git a/test/CodeGen/SystemZ/memcmp-01.ll b/test/CodeGen/SystemZ/memcmp-01.ll
index ac980e49d60bc195be4f7f21fee352e350407946..740a86750dd8a6f2cd00d65b2bf3bb163020b876 100644
--- a/test/CodeGen/SystemZ/memcmp-01.ll
+++ b/test/CodeGen/SystemZ/memcmp-01.ll
@@ -16,10 +16,10 @@ define i32 @f1(i8 *%src1, i8 *%src2) {
 ; Check a case where the result is used as an integer.
 define i32 @f2(i8 *%src1, i8 *%src2) {
 ; CHECK-LABEL: f2:
-; CHECK: clc 0(2,%r2), 0(%r3)
-; CHECK: ipm [[REG:%r[0-5]]]
-; CHECK: srl [[REG]], 28
-; CHECK: rll %r2, [[REG]], 31
+; CHECK: clc 0(2,%r3), 0(%r2)
+; CHECK: ipm %r2
+; CHECK: sll %r2, 2
+; CHECK: sra %r2, 30
 ; CHECK: br %r14
   %res = call i32 @memcmp(i8 *%src1, i8 *%src2, i64 2)
   ret i32 %res
@@ -28,7 +28,7 @@ define i32 @f2(i8 *%src1, i8 *%src2) {
 ; Check a case where the result is tested for equality.
 define void @f3(i8 *%src1, i8 *%src2, i32 *%dest) {
 ; CHECK-LABEL: f3:
-; CHECK: clc 0(3,%r2), 0(%r3)
+; CHECK: clc 0(3,%r3), 0(%r2)
 ; CHECK-NEXT: ber %r14
 ; CHECK: br %r14
   %res = call i32 @memcmp(i8 *%src1, i8 *%src2, i64 3)
@@ -46,7 +46,7 @@ exit:
 ; Check a case where the result is tested for inequality.
 define void @f4(i8 *%src1, i8 *%src2, i32 *%dest) {
 ; CHECK-LABEL: f4:
-; CHECK: clc 0(4,%r2), 0(%r3)
+; CHECK: clc 0(4,%r3), 0(%r2)
 ; CHECK-NEXT: blhr %r14
 ; CHECK: br %r14
 entry:
@@ -65,8 +65,8 @@ exit:
 ; Check a case where the result is tested via slt.
 define void @f5(i8 *%src1, i8 *%src2, i32 *%dest) {
 ; CHECK-LABEL: f5:
-; CHECK: clc 0(5,%r2), 0(%r3)
-; CHECK-NEXT: blr %r14
+; CHECK: clc 0(5,%r3), 0(%r2)
+; CHECK-NEXT: bhr %r14
 ; CHECK: br %r14
 entry:
   %res = call i32 @memcmp(i8 *%src1, i8 *%src2, i64 5)
@@ -84,8 +84,8 @@ exit:
 ; Check a case where the result is tested for sgt.
 define void @f6(i8 *%src1, i8 *%src2, i32 *%dest) {
 ; CHECK-LABEL: f6:
-; CHECK: clc 0(6,%r2), 0(%r3)
-; CHECK-NEXT: bhr %r14
+; CHECK: clc 0(6,%r3), 0(%r2)
+; CHECK-NEXT: blr %r14
 ; CHECK: br %r14
 entry:
   %res = call i32 @memcmp(i8 *%src1, i8 *%src2, i64 6)
@@ -104,10 +104,10 @@ exit:
 ; an integer and for branching.
 define i32 @f7(i8 *%src1, i8 *%src2, i32 *%dest) {
 ; CHECK-LABEL: f7:
-; CHECK: clc 0(256,%r2), 0(%r3)
-; CHECK: ipm [[REG:%r[0-5]]]
-; CHECK: srl [[REG]], 28
-; CHECK: rll %r2, [[REG]], 31
+; CHECK: clc 0(256,%r3), 0(%r2)
+; CHECK: ipm %r2
+; CHECK: sll %r2, 2
+; CHECK: sra %r2, 30
 ; CHECK: blr %r14
 ; CHECK: br %r14
 entry:
@@ -126,9 +126,9 @@ exit:
 ; 257 bytes needs two CLCs.
 define i32 @f8(i8 *%src1, i8 *%src2) {
 ; CHECK-LABEL: f8:
-; CHECK: clc 0(256,%r2), 0(%r3)
+; CHECK: clc 0(256,%r3), 0(%r2)
 ; CHECK: jlh [[LABEL:\..*]]
-; CHECK: clc 256(1,%r2), 256(%r3)
+; CHECK: clc 256(1,%r3), 256(%r2)
 ; CHECK: [[LABEL]]:
 ; CHECK: ipm [[REG:%r[0-5]]]
 ; CHECK: br %r14
@@ -139,11 +139,11 @@ define i32 @f8(i8 *%src1, i8 *%src2) {
 ; Test a comparison of 258 bytes in which the CC result can be used directly.
 define void @f9(i8 *%src1, i8 *%src2, i32 *%dest) {
 ; CHECK-LABEL: f9:
-; CHECK: clc 0(256,%r2), 0(%r3)
+; CHECK: clc 0(256,%r3), 0(%r2)
 ; CHECK: jlh [[LABEL:\..*]]
-; CHECK: clc 256(1,%r2), 256(%r3)
+; CHECK: clc 256(1,%r3), 256(%r2)
 ; CHECK: [[LABEL]]:
-; CHECK-NEXT: blr %r14
+; CHECK-NEXT: bhr %r14
 ; CHECK: br %r14
 entry:
   %res = call i32 @memcmp(i8 *%src1, i8 *%src2, i64 257)
@@ -161,9 +161,9 @@ exit:
 ; Test the largest size that can use two CLCs.
 define i32 @f10(i8 *%src1, i8 *%src2) {
 ; CHECK-LABEL: f10:
-; CHECK: clc 0(256,%r2), 0(%r3)
+; CHECK: clc 0(256,%r3), 0(%r2)
 ; CHECK: jlh [[LABEL:\..*]]
-; CHECK: clc 256(256,%r2), 256(%r3)
+; CHECK: clc 256(256,%r3), 256(%r2)
 ; CHECK: [[LABEL]]:
 ; CHECK: ipm [[REG:%r[0-5]]]
 ; CHECK: br %r14
@@ -174,11 +174,11 @@ define i32 @f10(i8 *%src1, i8 *%src2) {
 ; Test the smallest size that needs 3 CLCs.
 define i32 @f11(i8 *%src1, i8 *%src2) {
 ; CHECK-LABEL: f11:
-; CHECK: clc 0(256,%r2), 0(%r3)
+; CHECK: clc 0(256,%r3), 0(%r2)
 ; CHECK: jlh [[LABEL:\..*]]
-; CHECK: clc 256(256,%r2), 256(%r3)
+; CHECK: clc 256(256,%r3), 256(%r2)
 ; CHECK: jlh [[LABEL]]
-; CHECK: clc 512(1,%r2), 512(%r3)
+; CHECK: clc 512(1,%r3), 512(%r2)
 ; CHECK: [[LABEL]]:
 ; CHECK: ipm [[REG:%r[0-5]]]
 ; CHECK: br %r14
@@ -189,11 +189,11 @@ define i32 @f11(i8 *%src1, i8 *%src2) {
 ; Test the largest size than can use 3 CLCs.
 define i32 @f12(i8 *%src1, i8 *%src2) {
 ; CHECK-LABEL: f12:
-; CHECK: clc 0(256,%r2), 0(%r3)
+; CHECK: clc 0(256,%r3), 0(%r2)
 ; CHECK: jlh [[LABEL:\..*]]
-; CHECK: clc 256(256,%r2), 256(%r3)
+; CHECK: clc 256(256,%r3), 256(%r2)
 ; CHECK: jlh [[LABEL]]
-; CHECK: clc 512(256,%r2), 512(%r3)
+; CHECK: clc 512(256,%r3), 512(%r2)
 ; CHECK: [[LABEL]]:
 ; CHECK: ipm [[REG:%r[0-5]]]
 ; CHECK: br %r14
@@ -207,12 +207,12 @@ define i32 @f13(i8 *%src1, i8 *%src2) {
 ; CHECK-LABEL: f13:
 ; CHECK: lghi [[COUNT:%r[0-5]]], 3
 ; CHECK: [[LOOP:.L[^:]*]]:
-; CHECK: clc 0(256,%r2), 0(%r3)
+; CHECK: clc 0(256,%r3), 0(%r2)
 ; CHECK: jlh [[LABEL:\..*]]
 ; CHECK-DAG: la %r2, 256(%r2)
 ; CHECK-DAG: la %r3, 256(%r3)
 ; CHECK: brctg [[COUNT]], [[LOOP]]
-; CHECK: clc 0(1,%r2), 0(%r3)
+; CHECK: clc 0(1,%r3), 0(%r2)
 ; CHECK: [[LABEL]]:
 ; CHECK: ipm [[REG:%r[0-5]]]
 ; CHECK: br %r14
diff --git a/test/CodeGen/SystemZ/scalar-ctlz.ll b/test/CodeGen/SystemZ/scalar-ctlz.ll
new file mode 100644
index 0000000000000000000000000000000000000000..15a0e37cbf6ed98527d2da5ea562c97c308378c9
--- /dev/null
+++ b/test/CodeGen/SystemZ/scalar-ctlz.ll
@@ -0,0 +1,107 @@
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
+;
+; FIXME: two consecutive immediate adds not fused in i16/i8 functions.
+
+declare i64 @llvm.ctlz.i64(i64, i1)
+declare i32 @llvm.ctlz.i32(i32, i1)
+declare i16 @llvm.ctlz.i16(i16, i1)
+declare i8 @llvm.ctlz.i8(i8, i1)
+
+define i64 @f0(i64 %arg) {
+; CHECK-LABEL: f0:
+; CHECK-LABEL: %bb.0:
+; CHECK-NOT:   %bb.1:
+; CHECK: flogr
+  %1 = tail call i64 @llvm.ctlz.i64(i64 %arg, i1 false)
+  ret i64 %1
+}
+
+define i64 @f1(i64 %arg) {
+; CHECK-LABEL: f1:
+; CHECK-LABEL: %bb.0:
+; CHECK-NEXT: flogr
+; CHECK-NEXT: # kill
+; CHECK-NEXT: br %r14
+  %1 = tail call i64 @llvm.ctlz.i64(i64 %arg, i1 true)
+  ret i64 %1
+}
+
+define i32 @f2(i32 %arg) {
+; CHECK-LABEL: f2:
+; CHECK-LABEL: %bb.0:
+; CHECK-NEXT: llgfr %r0, %r2
+; CHECK-NEXT: flogr %r2, %r0
+; CHECK-NEXT: aghi  %r2, -32
+; CHECK-NEXT: # kill
+; CHECK-NEXT: br %r14
+  %1 = tail call i32 @llvm.ctlz.i32(i32 %arg, i1 false)
+  ret i32 %1
+}
+
+define i32 @f3(i32 %arg) {
+; CHECK-LABEL: f3:
+; CHECK-LABEL: %bb.0:
+; CHECK-NEXT: llgfr %r0, %r2
+; CHECK-NEXT: flogr %r2, %r0
+; CHECK-NEXT: aghi  %r2, -32
+; CHECK-NEXT: # kill
+; CHECK-NEXT: br %r14
+  %1 = tail call i32 @llvm.ctlz.i32(i32 %arg, i1 true)
+  ret i32 %1
+}
+
+define i16 @f4(i16 %arg) {
+; CHECK-LABEL: f4:
+; CHECK-LABEL: %bb.0:
+; CHECK-NEXT: # kill
+; CHECK-NEXT: llghr %r0, %r2
+; CHECK-NEXT: flogr %r2, %r0
+; CHECK-NEXT: aghi  %r2, -32
+; CHECK-NEXT: ahi   %r2, -16
+; CHECK-NEXT: # kill
+; CHECK-NEXT: br %r14
+  %1 = tail call i16 @llvm.ctlz.i16(i16 %arg, i1 false)
+  ret i16 %1
+}
+
+define i16 @f5(i16 %arg) {
+; CHECK-LABEL: f5:
+; CHECK-LABEL: %bb.0:
+; CHECK-NEXT: # kill
+; CHECK-NEXT: llghr %r0, %r2
+; CHECK-NEXT: flogr %r2, %r0
+; CHECK-NEXT: aghi  %r2, -32
+; CHECK-NEXT: ahi   %r2, -16
+; CHECK-NEXT: # kill
+; CHECK-NEXT: br %r14
+  %1 = tail call i16 @llvm.ctlz.i16(i16 %arg, i1 true)
+  ret i16 %1
+}
+
+define i8 @f6(i8 %arg) {
+; CHECK-LABEL: f6:
+; CHECK-LABEL: %bb.0:
+; CHECK-NEXT: # kill
+; CHECK-NEXT: llgcr %r0, %r2
+; CHECK-NEXT: flogr %r2, %r0
+; CHECK-NEXT: aghi  %r2, -32
+; CHECK-NEXT: ahi   %r2, -24
+; CHECK-NEXT: # kill
+; CHECK-NEXT: br %r14
+  %1 = tail call i8 @llvm.ctlz.i8(i8 %arg, i1 false)
+  ret i8 %1
+}
+
+define i8 @f7(i8 %arg) {
+; CHECK-LABEL: f7:
+; CHECK-LABEL: %bb.0:
+; CHECK-NEXT: # kill
+; CHECK-NEXT: llgcr %r0, %r2
+; CHECK-NEXT: flogr %r2, %r0
+; CHECK-NEXT: aghi  %r2, -32
+; CHECK-NEXT: ahi   %r2, -24
+; CHECK-NEXT: # kill
+; CHECK-NEXT: br %r14
+  %1 = tail call i8 @llvm.ctlz.i8(i8 %arg, i1 true)
+  ret i8 %1
+}
diff --git a/test/CodeGen/SystemZ/strcmp-01.ll b/test/CodeGen/SystemZ/strcmp-01.ll
index ef05d832e73e9550890675d9a868022fc7346e8e..a3e3bbbb23be7f3ee57edd98831d3ba180613b39 100644
--- a/test/CodeGen/SystemZ/strcmp-01.ll
+++ b/test/CodeGen/SystemZ/strcmp-01.ll
@@ -9,12 +9,12 @@ define i32 @f1(i8 *%src1, i8 *%src2) {
 ; CHECK-LABEL: f1:
 ; CHECK: lhi %r0, 0
 ; CHECK: [[LABEL:\.[^:]*]]:
-; CHECK: clst %r2, %r3
+; CHECK: clst %r3, %r2
 ; CHECK-NEXT: jo [[LABEL]]
 ; CHECK-NEXT: %bb.{{[0-9]+}}
-; CHECK-NEXT: ipm [[REG:%r[0-5]]]
-; CHECK: srl [[REG]], 28
-; CHECK: rll %r2, [[REG]], 31
+; CHECK-NEXT: ipm %r2
+; CHECK: sll %r2, 2
+; CHECK: sra %r2, 30
 ; CHECK: br %r14
   %res = call i32 @strcmp(i8 *%src1, i8 *%src2)
   ret i32 %res
@@ -25,7 +25,7 @@ define void @f2(i8 *%src1, i8 *%src2, i32 *%dest) {
 ; CHECK-LABEL: f2:
 ; CHECK: lhi %r0, 0
 ; CHECK: [[LABEL:\.[^:]*]]:
-; CHECK: clst %r2, %r3
+; CHECK: clst %r3, %r2
 ; CHECK-NEXT: jo [[LABEL]]
 ; CHECK-NEXT: %bb.{{[0-9]+}}
 ; CHECK-NEXT: ber %r14
@@ -48,12 +48,12 @@ define i32 @f3(i8 *%src1, i8 *%src2, i32 *%dest) {
 ; CHECK-LABEL: f3:
 ; CHECK: lhi %r0, 0
 ; CHECK: [[LABEL:\.[^:]*]]:
-; CHECK: clst %r2, %r3
+; CHECK: clst %r3, %r2
 ; CHECK-NEXT: jo [[LABEL]]
 ; CHECK-NEXT: %bb.{{[0-9]+}}
-; CHECK-NEXT: ipm [[REG:%r[0-5]]]
-; CHECK: srl [[REG]], 28
-; CHECK: rll %r2, [[REG]], 31
+; CHECK-NEXT: ipm %r2
+; CHECK: sll %r2, 2
+; CHECK: sra %r2, 30
 ; CHECK: blr %r14
 ; CHECK: br %r14
 entry:
diff --git a/test/CodeGen/SystemZ/subregliveness-01.ll b/test/CodeGen/SystemZ/subregliveness-01.ll
index 8863e228ec7db345632072404f6a8359600b06ca..5813fd30921eb2f6eec9ab1ff93344111693922f 100644
--- a/test/CodeGen/SystemZ/subregliveness-01.ll
+++ b/test/CodeGen/SystemZ/subregliveness-01.ll
@@ -36,7 +36,7 @@ attributes #0 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disa
 
 !llvm.ident = !{!0}
 
-!0 = !{!"clang version 8.0.0 (http://llvm.org/git/clang.git c0a5e830f198cf42d29f72f1ec06fbf4c5210e2c) (http://llvm.org/git/llvm.git ffc8c538b70b678031b8617f61f83ee120bcb884)"}
+!0 = !{!"clang version 8.0.0"}
 !1 = !{!2}
 !2 = distinct !{!2, !3, !"func_1: %agg.result"}
 !3 = distinct !{!3, !"func_1"}
diff --git a/test/CodeGen/SystemZ/subregliveness-02.ll b/test/CodeGen/SystemZ/subregliveness-02.ll
index abdc223bcb81d2d871153d484793efd0f43e1bab..8f972e18a66470fb3ae8ef0908ee3dc54e849c23 100644
--- a/test/CodeGen/SystemZ/subregliveness-02.ll
+++ b/test/CodeGen/SystemZ/subregliveness-02.ll
@@ -1,7 +1,7 @@
 ; RUN: llc -mtriple=s390x-linux-gnu -mcpu=z13 -systemz-subreg-liveness < %s | FileCheck %s
 
 ; Check for successful compilation.
-; CHECK: meeb %f0, 0(%r1)
+; CHECK: meebr %f1, %f0
 
 target datalayout = "E-m:e-i1:8:16-i8:8:16-i64:64-f128:64-v128:64-a:8:16-n32:64"
 target triple = "s390x-ibm-linux"
diff --git a/test/CodeGen/SystemZ/subregliveness-06.mir b/test/CodeGen/SystemZ/subregliveness-06.mir
index fcb1d34dbafe75d6af6716134fb1048c2900f015..01e3341ead95c3ca0835689c5b690438ef2d88d4 100644
--- a/test/CodeGen/SystemZ/subregliveness-06.mir
+++ b/test/CodeGen/SystemZ/subregliveness-06.mir
@@ -128,7 +128,7 @@
   
   !llvm.ident = !{!0}
   
-  !0 = !{!"clang version 8.0.0 (http://llvm.org/git/clang.git c0a5e830f198cf42d29f72f1ec06fbf4c5210e2c) (http://llvm.org/git/llvm.git ffc8c538b70b678031b8617f61f83ee120bcb884)"}
+  !0 = !{!"clang version 8.0.0"}
   !1 = !{!2, !2, i64 0}
   !2 = !{!"omnipotent char", !3, i64 0}
   !3 = !{!"Simple C/C++ TBAA"}
diff --git a/test/CodeGen/SystemZ/vec-const-05.ll b/test/CodeGen/SystemZ/vec-const-05.ll
index 55f3cdd59027a43a0e8140020989e6a46ca5b179..719280e9d608776c519efa75f1c535969dc3a31f 100644
--- a/test/CodeGen/SystemZ/vec-const-05.ll
+++ b/test/CodeGen/SystemZ/vec-const-05.ll
@@ -1,63 +1,28 @@
-; Test vector byte masks, v4f32 version.
+; Test vector byte masks, v4f32 version. Only all-zero vectors are handled.
 ;
 ; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
 
 ; Test an all-zeros vector.
-define <4 x float> @f1() {
-; CHECK-LABEL: f1:
+define <4 x float> @f0() {
+; CHECK-LABEL: f0:
 ; CHECK: vgbm %v24, 0
 ; CHECK: br %r14
   ret <4 x float> zeroinitializer
 }
 
-; Test an all-ones vector.
-define <4 x float> @f2() {
-; CHECK-LABEL: f2:
-; CHECK: vgbm %v24, 65535
-; CHECK: br %r14
-  ret <4 x float> <float 0xffffffffe0000000, float 0xffffffffe0000000,
-                   float 0xffffffffe0000000, float 0xffffffffe0000000>
-}
-
-; Test a mixed vector (mask 0xc731).
-define <4 x float> @f3() {
-; CHECK-LABEL: f3:
-; CHECK: vgbm %v24, 50993
-; CHECK: br %r14
-  ret <4 x float> <float 0xffffe00000000000, float 0x381fffffe0000000,
-                   float 0x379fffe000000000, float 0x371fe00000000000>
-}
-
-; Test that undefs are treated as zero (mask 0xc031).
-define <4 x float> @f4() {
-; CHECK-LABEL: f4:
-; CHECK: vgbm %v24, 49201
-; CHECK: br %r14
-  ret <4 x float> <float 0xffffe00000000000, float undef,
-                   float 0x379fffe000000000, float 0x371fe00000000000>
-}
-
-; Test that we don't use VGBM if one of the bytes is not 0 or 0xff.
-define <4 x float> @f5() {
-; CHECK-LABEL: f5:
-; CHECK-NOT: vgbm
+; Test that undefs are treated as zero.
+define <4 x float> @f1() {
+; CHECK-LABEL: f1:
+; CHECK: vgbm %v24, 0
 ; CHECK: br %r14
-  ret <4 x float> <float 0xffffe00000000000, float 0x381fffffc0000000,
-                   float 0x379fffe000000000, float 0x371fe00000000000>
+  ret <4 x float> <float zeroinitializer, float undef,
+                   float zeroinitializer, float undef>
 }
 
 ; Test an all-zeros v2f32 that gets promoted to v4f32.
-define <2 x float> @f6() {
-; CHECK-LABEL: f6:
+define <2 x float> @f2() {
+; CHECK-LABEL: f2:
 ; CHECK: vgbm %v24, 0
 ; CHECK: br %r14
   ret <2 x float> zeroinitializer
 }
-
-; Test a mixed v2f32 that gets promoted to v4f32 (mask 0xc700).
-define <2 x float> @f7() {
-; CHECK-LABEL: f7:
-; CHECK: vgbm %v24, 50944
-; CHECK: br %r14
-  ret <2 x float> <float 0xffffe00000000000, float 0x381fffffe0000000>
-}
diff --git a/test/CodeGen/SystemZ/vec-const-06.ll b/test/CodeGen/SystemZ/vec-const-06.ll
index be53a0581ec256e3dd1262385b070facc44dd0f8..6144e0f4910f459f078f7d55277048d7d0de2c67 100644
--- a/test/CodeGen/SystemZ/vec-const-06.ll
+++ b/test/CodeGen/SystemZ/vec-const-06.ll
@@ -1,43 +1,19 @@
-; Test vector byte masks, v2f64 version.
+; Test vector byte masks, v2f64 version. Only all-zero vectors are handled.
 ;
 ; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
 
 ; Test an all-zeros vector.
-define <2 x double> @f1() {
-; CHECK-LABEL: f1:
+define <2 x double> @f0() {
+; CHECK-LABEL: f0:
 ; CHECK: vgbm %v24, 0
 ; CHECK: br %r14
   ret <2 x double> zeroinitializer
 }
 
-; Test an all-ones vector.
-define <2 x double> @f2() {
-; CHECK-LABEL: f2:
-; CHECK: vgbm %v24, 65535
-; CHECK: br %r14
-  ret <2 x double> <double 0xffffffffffffffff, double 0xffffffffffffffff>
-}
-
-; Test a mixed vector (mask 0x8c76).
-define <2 x double> @f3() {
-; CHECK-LABEL: f3:
-; CHECK: vgbm %v24, 35958
-; CHECK: br %r14
-  ret <2 x double> <double 0xff000000ffff0000, double 0x00ffffff00ffff00>
-}
-
-; Test that undefs are treated as zero (mask 0x8c00).
-define <2 x double> @f4() {
-; CHECK-LABEL: f4:
-; CHECK: vgbm %v24, 35840
-; CHECK: br %r14
-  ret <2 x double> <double 0xff000000ffff0000, double undef>
-}
-
-; Test that we don't use VGBM if one of the bytes is not 0 or 0xff.
-define <2 x double> @f5() {
-; CHECK-LABEL: f5:
-; CHECK-NOT: vgbm
+; Test that undefs are treated as zero.
+define <2 x double> @f1() {
+; CHECK-LABEL: f1:
+; CHECK: vgbm %v24, 0
 ; CHECK: br %r14
-  ret <2 x double> <double 0xfe000000ffff0000, double 0x00ffffff00ffff00>
+  ret <2 x double> <double zeroinitializer, double undef>
 }
diff --git a/test/CodeGen/SystemZ/vec-conv-02.ll b/test/CodeGen/SystemZ/vec-conv-02.ll
index ab84389f3c8e8cd5091f532ac11db94674dd98ac..d4c0f72641827d6a41784e11201992435e51e3d0 100644
--- a/test/CodeGen/SystemZ/vec-conv-02.ll
+++ b/test/CodeGen/SystemZ/vec-conv-02.ll
@@ -15,19 +15,30 @@ define void @f1(<2 x double> %val, <2 x float> *%ptr) {
 ; Test conversion of an f64 in a vector register to an f32.
 define float @f2(<2 x double> %vec) {
 ; CHECK-LABEL: f2:
-; CHECK: wledb %f0, %v24
+; CHECK: wledb %f0, %v24, 0, 0
 ; CHECK: br %r14
   %scalar = extractelement <2 x double> %vec, i32 0
   %ret = fptrunc double %scalar to float
   ret float %ret
 }
 
-; Test conversion of an f32 in a vector register to an f64.
-define double @f3(<4 x float> %vec) {
+; Test cases where even elements of a v4f32 are converted to f64s.
+define <2 x double> @f3(<4 x float> %vec) {
 ; CHECK-LABEL: f3:
+; CHECK: vldeb %v24, {{%v[0-9]+}}
+; CHECK: br %r14
+  %shuffle = shufflevector <4 x float> %vec, <4 x float> undef, <2 x i32> <i32 0, i32 2>
+  %res = fpext <2 x float> %shuffle to <2 x double>
+  ret <2 x double> %res
+}
+
+; Test conversion of an f32 in a vector register to an f64.
+define double @f4(<4 x float> %vec) {
+; CHECK-LABEL: f4:
 ; CHECK: wldeb %f0, %v24
 ; CHECK: br %r14
   %scalar = extractelement <4 x float> %vec, i32 0
   %ret = fpext float %scalar to double
   ret double %ret
 }
+
diff --git a/test/CodeGen/SystemZ/vec-move-08.ll b/test/CodeGen/SystemZ/vec-move-08.ll
index 5396a1edec6abc0384d1baf0effbd110ee664f22..7c4a16c95403d344be6c9b97ecbc115e96a8c4a4 100644
--- a/test/CodeGen/SystemZ/vec-move-08.ll
+++ b/test/CodeGen/SystemZ/vec-move-08.ll
@@ -442,3 +442,35 @@ define <2 x double> @f39(<2 x double> %val, <2 x i64> %index, i64 %base) {
   %ret = insertelement <2 x double> %val, double %element, i32 1
   ret <2 x double> %ret
 }
+
+; Test a v4i32 gather where the load is chained.
+define void @f40(<4 x i32> %val, <4 x i32> %index, i64 %base, <4 x i32> *%res) {
+; CHECK-LABEL: f40:
+; CHECK: vgef %v24, 0(%v26,%r2), 1
+; CHECK: vst %v24, 0(%r3)
+; CHECK: br %r14
+  %elem = extractelement <4 x i32> %index, i32 1
+  %ext = zext i32 %elem to i64
+  %add = add i64 %base, %ext
+  %ptr = inttoptr i64 %add to i32 *
+  %element = load i32, i32 *%ptr
+  %ret = insertelement <4 x i32> %val, i32 %element, i32 1
+  store <4 x i32> %ret, <4 x i32> *%res
+  ret void
+}
+
+; Test a v2i64 gather where the load is chained.
+define void @f41(<2 x i64> %val, <2 x i64> %index, i64 %base, <2 x i64> *%res) {
+; CHECK-LABEL: f41:
+; CHECK: vgeg %v24, 0(%v26,%r2), 1
+; CHECK: vst %v24, 0(%r3)
+; CHECK: br %r14
+  %elem = extractelement <2 x i64> %index, i32 1
+  %add = add i64 %base, %elem
+  %ptr = inttoptr i64 %add to i64 *
+  %element = load i64, i64 *%ptr
+  %ret = insertelement <2 x i64> %val, i64 %element, i32 1
+  store <2 x i64> %ret, <2 x i64> *%res
+  ret void
+}
+
diff --git a/test/CodeGen/SystemZ/vec-move-14.ll b/test/CodeGen/SystemZ/vec-move-14.ll
index e41eb9da034653dd9f21ed6b6c19a6df05af7974..e6415e84c9c403896d85a7e537ab281440e99ac1 100644
--- a/test/CodeGen/SystemZ/vec-move-14.ll
+++ b/test/CodeGen/SystemZ/vec-move-14.ll
@@ -94,3 +94,45 @@ define <2 x double> @f9(double *%ptr) {
   %ret = insertelement <2 x double> zeroinitializer, double %val, i32 0
   ret <2 x double> %ret
 }
+
+; Test VLLEZF with a float when the result is stored to memory.
+define void @f10(float *%ptr, <4 x float> *%res) {
+; CHECK-LABEL: f10:
+; CHECK: vllezf [[REG:%v[0-9]+]], 0(%r2)
+; CHECK: vst [[REG]], 0(%r3)
+; CHECK: br %r14
+  %val = load float, float *%ptr
+  %ret = insertelement <4 x float> zeroinitializer, float %val, i32 1
+  store <4 x float> %ret, <4 x float> *%res
+  ret void
+}
+
+; Test VLLEZG with a double when the result is stored to memory.
+define void @f11(double *%ptr, <2 x double> *%res) {
+; CHECK-LABEL: f11:
+; CHECK: vllezg [[REG:%v[0-9]+]], 0(%r2)
+; CHECK: vst [[REG]], 0(%r3)
+; CHECK: br %r14
+  %val = load double, double *%ptr
+  %ret = insertelement <2 x double> zeroinitializer, double %val, i32 0
+  store <2 x double> %ret, <2 x double> *%res
+  ret void
+}
+
+; Test VLLEZG when the zeroinitializer is shared.
+define void @f12(i64 *%ptr, <2 x i64> *%res) {
+; CHECK-LABEL: f12:
+; CHECK: vllezg [[REG:%v[0-9]+]], 0(%r2)
+; CHECK: vst [[REG]], 0(%r3)
+; CHECK: vllezg [[REG1:%v[0-9]+]], 0(%r2)
+; CHECK: vst [[REG1]], 0(%r3)
+; CHECK: br %r14
+  %val = load volatile i64, i64 *%ptr
+  %ret = insertelement <2 x i64> zeroinitializer, i64 %val, i32 0
+  store volatile <2 x i64> %ret, <2 x i64> *%res
+  %val1 = load volatile i64, i64 *%ptr
+  %ret1 = insertelement <2 x i64> zeroinitializer, i64 %val1, i32 0
+  store volatile <2 x i64> %ret1, <2 x i64> *%res
+  ret void
+}
+
diff --git a/test/CodeGen/SystemZ/vec-move-18.ll b/test/CodeGen/SystemZ/vec-move-18.ll
index 5d3d09d83ef154005c2a1b802a721030edfad334..9bb61934ff2658f5ac7665de6f6a85def494ceec 100644
--- a/test/CodeGen/SystemZ/vec-move-18.ll
+++ b/test/CodeGen/SystemZ/vec-move-18.ll
@@ -22,3 +22,15 @@ define <4 x float> @f2(float *%ptr) {
   ret <4 x float> %ret
 }
 
+; Test VLLEZLF with a float when the result is stored to memory.
+define void @f3(float *%ptr, <4 x float> *%res) {
+; CHECK-LABEL: f3:
+; CHECK: vllezlf [[REG:%v[0-9]+]], 0(%r2)
+; CHECK: vst [[REG]], 0(%r3)
+; CHECK: br %r14
+  %val = load float, float *%ptr
+  %ret = insertelement <4 x float> zeroinitializer, float %val, i32 0
+  store <4 x float> %ret, <4 x float> *%res
+  ret void
+}
+
diff --git a/test/CodeGen/Thumb/2009-07-27-PEIAssert.ll b/test/CodeGen/Thumb/2009-07-27-PEIAssert.ll
index aaca3a7f3db215bd9cde077ce7e53517dac09e18..f0de64db007190a953eaa43747d29d232f3498be 100644
--- a/test/CodeGen/Thumb/2009-07-27-PEIAssert.ll
+++ b/test/CodeGen/Thumb/2009-07-27-PEIAssert.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -mtriple=thumbv6-apple-darwin -relocation-model=pic -disable-fp-elim
+; RUN: llc < %s -mtriple=thumbv6-apple-darwin -relocation-model=pic -frame-pointer=all
 
 	%struct.LinkList = type { i32, %struct.LinkList* }
 	%struct.List = type { i32, i32* }
diff --git a/test/CodeGen/Thumb/2009-08-20-ISelBug.ll b/test/CodeGen/Thumb/2009-08-20-ISelBug.ll
index 86d7023957269c1930b6a4a2f996c33f14bc118c..5c4b34d946c66e3ba30380700451bf98803e0671 100644
--- a/test/CodeGen/Thumb/2009-08-20-ISelBug.ll
+++ b/test/CodeGen/Thumb/2009-08-20-ISelBug.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -mtriple=thumbv6-apple-darwin -relocation-model=pic -disable-fp-elim -mattr=+v6 -verify-machineinstrs | FileCheck %s
+; RUN: llc < %s -mtriple=thumbv6-apple-darwin -relocation-model=pic -frame-pointer=all -mattr=+v6 -verify-machineinstrs | FileCheck %s
 ; rdar://7157006
 
 %struct.FILE = type { i8*, i32, i32, i16, i16, %struct.__sbuf, i32, i8*, i32 (i8*)*, i32 (i8*, i8*, i32)*, i64 (i8*, i64, i32)*, i32 (i8*, i8*, i32)*, %struct.__sbuf, %struct.__sFILEX*, i32, [3 x i8], [1 x i8], %struct.__sbuf, i32, i64 }
diff --git a/test/CodeGen/Thumb/frame-access.ll b/test/CodeGen/Thumb/frame-access.ll
index 7fae4470f8f928e3662ec76b526275ac7e93287a..9cbed5ed33fd71e4326d0033a0330d5ec0614c7b 100644
--- a/test/CodeGen/Thumb/frame-access.ll
+++ b/test/CodeGen/Thumb/frame-access.ll
@@ -1,4 +1,4 @@
-; RUN: llc -mtriple=thumbv6m-eabi -disable-fp-elim=false %s -o - | FileCheck %s
+; RUN: llc -mtriple=thumbv6m-eabi -frame-pointer=none %s -o - | FileCheck %s
 
 ; struct S { int x[128]; } s;
 ; int f(int *, int, int, int, struct S);
diff --git a/test/CodeGen/Thumb/frame_thumb.ll b/test/CodeGen/Thumb/frame_thumb.ll
index 6cc4dd12f6a7b4828498b65c0300426948a7738e..3cff8e20a20d0122dc3c9e7effb074ce7f88614e 100644
--- a/test/CodeGen/Thumb/frame_thumb.ll
+++ b/test/CodeGen/Thumb/frame_thumb.ll
@@ -1,7 +1,7 @@
 ; RUN: llc < %s -mtriple=thumb-apple-darwin \
-; RUN:     -disable-fp-elim | not grep "r11"
+; RUN:     -frame-pointer=all | not grep "r11"
 ; RUN: llc < %s -mtriple=thumb-linux-gnueabi \
-; RUN:     -disable-fp-elim | not grep "r11"
+; RUN:     -frame-pointer=all | not grep "r11"
 
 define i32 @f() {
 entry:
diff --git a/test/CodeGen/Thumb/opt-phis.mir b/test/CodeGen/Thumb/opt-phis.mir
new file mode 100644
index 0000000000000000000000000000000000000000..e3d0a8bb1f71d7c0610798bad1bdd2b0af7fffc7
--- /dev/null
+++ b/test/CodeGen/Thumb/opt-phis.mir
@@ -0,0 +1,38 @@
+# RUN: llc -mtriple thumbv6m-none-eabi -run-pass=opt-phis -verify-machineinstrs -o - %s | FileCheck %s
+--- |
+  target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64"
+  target triple = "thumbv6m-arm-none-eabi"
+
+  define i32 @funca() { unreachable }
+
+...
+---
+name:            funca
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    successors: %bb.2(0x40000000), %bb.3(0x40000000)
+
+    %0:tgpr, dead $cpsr = tMOVi8 255, 14, $noreg
+    %1:tgpr, dead $cpsr = tMOVi8 128, 14, $noreg
+    tCMPi8 %0, 0, 14, $noreg, implicit-def $cpsr
+    tBcc %bb.3, 1, $cpsr
+
+  bb.2:
+    successors: %bb.3(0x80000000)
+
+  bb.3:
+    successors: %bb.0(0x80000000)
+
+    %2:tgpr = PHI %0, %bb.2, %0, %bb.0
+    %3:tgpr, $cpsr = tSUBrr %1, killed %2, 14, $noreg
+    %4:tgpr, $cpsr = tADDrr killed %3, killed %0, 14, $noreg
+    tB %bb.0, 14, $noreg
+
+# Make sure the kill flags are not incorrect
+# CHECK-LABEL: name: funca
+# CHECK-NOT: PHI
+# CHECK: tSUBrr %1, %0
+# CHECK: tADDrr killed %3, %0
+
+...
diff --git a/test/CodeGen/Thumb/push.ll b/test/CodeGen/Thumb/push.ll
index 4f4ffed7bbd0f757512b4ebb78e08cb244d0a732..74629effd147a5ea72480136f7972b03564da51f 100644
--- a/test/CodeGen/Thumb/push.ll
+++ b/test/CodeGen/Thumb/push.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -mtriple=thumb-apple-darwin -disable-fp-elim | FileCheck %s
+; RUN: llc < %s -mtriple=thumb-apple-darwin -frame-pointer=all | FileCheck %s
 ; rdar://7268481
 
 define void @t() nounwind {
diff --git a/test/CodeGen/Thumb/shift-and.ll b/test/CodeGen/Thumb/shift-and.ll
index 01f6b84d0c36c3ddce8cfcd8f933587e4db253d2..b8e3416adbb865abdf789b831da8c40542c78b6d 100644
--- a/test/CodeGen/Thumb/shift-and.ll
+++ b/test/CodeGen/Thumb/shift-and.ll
@@ -45,9 +45,8 @@ entry:
 define i32 @test4(i32 %x) {
 ; CHECK-LABEL: test4:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    lsls r0, r0, #4
-; CHECK-NEXT:    movs r1, #112
-; CHECK-NEXT:    bics r0, r1
+; CHECK-NEXT:    lsrs r0, r0, #3
+; CHECK-NEXT:    lsls r0, r0, #7
 ; CHECK-NEXT:    bx lr
 entry:
   %0 = shl i32 %x, 4
@@ -84,9 +83,8 @@ entry:
 define i32 @test7(i32 %x) {
 ; CHECK-LABEL: test7:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    lsrs r1, r0, #29
-; CHECK-NEXT:    movs r0, #4
-; CHECK-NEXT:    ands r0, r1
+; CHECK-NEXT:    lsrs r0, r0, #31
+; CHECK-NEXT:    lsls r0, r0, #2
 ; CHECK-NEXT:    bx lr
 entry:
   %0 = lshr i32 %x, 29
@@ -110,9 +108,8 @@ entry:
 define i32 @test9(i32 %x) {
 ; CHECK-LABEL: test9:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    lsrs r0, r0, #2
-; CHECK-NEXT:    movs r1, #1
-; CHECK-NEXT:    bics r0, r1
+; CHECK-NEXT:    lsrs r0, r0, #3
+; CHECK-NEXT:    lsls r0, r0, #1
 ; CHECK-NEXT:    bx lr
 entry:
   %and = lshr i32 %x, 2
@@ -131,3 +128,133 @@ entry:
   %shr = and i32 %0, 255
   ret i32 %shr
 }
+
+define i32 @test11(i32 %x) {
+; CHECK-LABEL: test11:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    lsrs r0, r0, #24
+; CHECK-NEXT:    lsls r0, r0, #2
+; CHECK-NEXT:    bx lr
+entry:
+  %shl = lshr i32 %x, 22
+  %and = and i32 %shl, 1020
+  ret i32 %and
+}
+
+define i32 @test12(i32 %x) {
+; CHECK-LABEL: test12:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    lsrs r0, r0, #3
+; CHECK-NEXT:    lsls r0, r0, #4
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = shl i32 %x, 1
+  %shr = and i32 %0, -16
+  ret i32 %shr
+}
+
+define i32 @test13(i32 %x) {
+; CHECK-LABEL: test13:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    lsrs r0, r0, #3
+; CHECK-NEXT:    lsls r0, r0, #4
+; CHECK-NEXT:    bx lr
+entry:
+  %shr = lshr i32 %x, 3
+  %shl = shl i32 %shr, 4
+  ret i32 %shl
+}
+
+define i32 @test14(i32 %x) {
+; CHECK-LABEL: test14:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    lsrs r0, r0, #6
+; CHECK-NEXT:    lsls r0, r0, #10
+; CHECK-NEXT:    bx lr
+entry:
+  %shl = shl i32 %x, 4
+  %and = and i32 %shl, -1024
+  ret i32 %and
+}
+
+define i32 @test15(i32 %x) {
+; CHECK-LABEL: test15:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    lsrs r0, r0, #4
+; CHECK-NEXT:    lsls r0, r0, #3
+; CHECK-NEXT:    bx lr
+entry:
+  %shr = lshr i32 %x, 4
+  %shl = shl i32 %shr, 3
+  ret i32 %shl
+}
+
+define i32 @test16(i32 %x) {
+; CHECK-LABEL: test16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    lsls r0, r0, #28
+; CHECK-NEXT:    lsrs r0, r0, #26
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = and i32 %x, 15
+  %shl = shl i32 %0, 2
+  ret i32 %shl
+}
+
+define i32* @test17(i32* %p, i32 %x) {
+; CHECK-LABEL: test17:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    lsls r1, r1, #28
+; CHECK-NEXT:    lsrs r1, r1, #26
+; CHECK-NEXT:    adds r0, r0, r1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = and i32 %x, 15
+  %shl = getelementptr i32, i32* %p, i32 %0
+  ret i32* %shl
+}
+
+define i32* @test18(i32* %p, i32 %x) {
+; CHECK-LABEL: test18:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    adds r1, r1, #1
+; CHECK-NEXT:    lsls r1, r1, #28
+; CHECK-NEXT:    lsrs r1, r1, #26
+; CHECK-NEXT:    adds r0, r0, r1
+; CHECK-NEXT:    bx lr
+entry:
+  %xx = add i32 %x, 1
+  %0 = and i32 %xx, 15
+  %shl = getelementptr i32, i32* %p, i32 %0
+  ret i32* %shl
+}
+
+define i32* @test19(i32* %p, i32 %x) {
+; CHECK-LABEL: test19:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    subs r1, r1, #1
+; CHECK-NEXT:    lsls r1, r1, #28
+; CHECK-NEXT:    lsrs r1, r1, #26
+; CHECK-NEXT:    adds r0, r0, r1
+; CHECK-NEXT:    bx lr
+entry:
+  %xx = sub i32 %x, 1
+  %0 = and i32 %xx, 15
+  %shl = getelementptr i32, i32* %p, i32 %0
+  ret i32* %shl
+}
+
+define i32* @test20(i32* %p, i32 %x) {
+; CHECK-LABEL: test20:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    subs r1, r1, #1
+; CHECK-NEXT:    lsls r1, r1, #28
+; CHECK-NEXT:    lsrs r1, r1, #26
+; CHECK-NEXT:    adds r0, r0, r1
+; CHECK-NEXT:    bx lr
+entry:
+  %xx = add i32 %x, 15
+  %0 = and i32 %xx, 15
+  %shl = getelementptr i32, i32* %p, i32 %0
+  ret i32* %shl
+}
diff --git a/test/CodeGen/Thumb/umulo-32-legalisation-lowering.ll b/test/CodeGen/Thumb/umulo-32-legalisation-lowering.ll
new file mode 100644
index 0000000000000000000000000000000000000000..fb0d441e0c313bd597ce1577c38ea0561f997a51
--- /dev/null
+++ b/test/CodeGen/Thumb/umulo-32-legalisation-lowering.ll
@@ -0,0 +1,43 @@
+; RUN: llc < %s -mtriple=thumb-eabi   | FileCheck %s --check-prefixes CHECK,CHECK-LITTLE
+; RUN: llc < %s -mtriple=thumbeb-eabi | FileCheck %s --check-prefixes CHECK,CHECK-BIG
+
+define i1 @umulo32(i32 %l, i32 %r) unnamed_addr #0 {
+; CHECK-LABEL: umulo32:
+; CHECK:                @ %bb.0: @ %start
+; CHECK-NEXT:           .save {r7, lr}
+; CHECK-NEXT:           push {r7, lr}
+; CHECK-LITTLE-NEXT:    movs r2, r1
+; CHECK-LITTLE-NEXT:    movs r1, #0
+; CHECK-NEXT:           movs r3, r1
+; CHECK-BIG-NEXT:       movs r1, r0
+; CHECK-BIG-NEXT:       movs r0, #0
+; CHECK-BIG-NEXT:       movs r2, r0
+; CHECK-NEXT:           bl __aeabi_lmul
+; CHECK-LITTLE-NEXT:    cmp  r1, #0
+; CHECK-LITTLE-NEXT:    bne  .LBB0_2
+; CHECK-LITTLE-NEXT:    @ %bb.1:
+; CHECK-LITTLE-NEXT:    movs r0, r1
+; CHECK-LITTLE-NEXT:    b    .LBB0_3
+; CHECK-LITTLE-NEXT:    .LBB0_2:
+; CHECK-LITTLE-NEXT:    movs r0, #1
+; CHECK-LITTLE-NEXT:    .LBB0_3:
+; CHECK-BIG-NEXT:       cmp  r0, #0
+; CHECK-BIG-NEXT:       beq  .LBB0_2
+; CHECK-BIG-NEXT:       @ %bb.1:
+; CHECK-BIG-NEXT:       movs r0, #1
+; CHECK-BIG-NEXT:       .LBB0_2:
+; CHECK-NEXT:           pop {r7}
+; CHECK-NEXT:           pop {r1}
+; CHECK-NEXT:           bx r1
+start:
+  %0 = tail call { i32, i1 } @llvm.umul.with.overflow.i32(i32 %l, i32 %r) #2
+  %1 = extractvalue { i32, i1 } %0, 1
+  ret i1 %1
+}
+
+; Function Attrs: nounwind readnone speculatable
+declare { i32, i1 } @llvm.umul.with.overflow.i32(i32, i32) #1
+
+attributes #0 = { nounwind readnone uwtable }
+attributes #1 = { nounwind readnone speculatable }
+attributes #2 = { nounwind }
diff --git a/test/CodeGen/Thumb/uxth.ll b/test/CodeGen/Thumb/uxth.ll
new file mode 100644
index 0000000000000000000000000000000000000000..e1e5117b394e7ed7a65a68d721b207e10c0ea0ac
--- /dev/null
+++ b/test/CodeGen/Thumb/uxth.ll
@@ -0,0 +1,159 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv6m-eabi -asm-verbose=false %s -o - | FileCheck %s --check-prefix=V6M
+
+define i32 @uxth(i32 %x) {
+; V6M-LABEL: uxth:
+; V6M:         uxth r0, r0
+; V6M-NEXT:    bx lr
+  %xn = and i32 %x, 65535
+  ret i32 %xn
+}
+
+define i32 @uxtb(i32 %x) {
+; V6M-LABEL: uxtb:
+; V6M:         uxtb r0, r0
+; V6M-NEXT:    bx lr
+  %xn = and i32 %x, 255
+  ret i32 %xn
+}
+
+define i32 @uxth_2(i32 %x, i32 %y) {
+; V6M-LABEL: uxth_2:
+; V6M:         uxth r1, r1
+; V6M-NEXT:    uxth r0, r0
+; V6M-NEXT:    muls r0, r1, r0
+; V6M-NEXT:    bx lr
+  %xn = and i32 %x, 65535
+  %yn = and i32 %y, 65535
+  %r = mul nuw i32 %xn, %yn
+  ret i32 %r
+}
+
+define i32 @uxtb_2(i32 %x, i32 %y) {
+; V6M-LABEL: uxtb_2:
+; V6M:         uxtb r1, r1
+; V6M-NEXT:    uxtb r0, r0
+; V6M-NEXT:    muls r0, r1, r0
+; V6M-NEXT:    bx lr
+  %xn = and i32 %x, 255
+  %yn = and i32 %y, 255
+  %r = mul nuw nsw i32 %xn, %yn
+  ret i32 %r
+}
+
+define void @uxth_loop(i32* %a, i32 %n) {
+; V6M-LABEL: uxth_loop:
+; V6M:       .LBB4_1:
+; V6M-NEXT:    ldrh r2, [r0]
+; V6M-NEXT:    stm r0!, {r2}
+; V6M-NEXT:    subs r1, r1, #1
+; V6M-NEXT:    bne .LBB4_1
+; V6M-NEXT:    bx lr
+entry:
+  br label %for.body
+
+for.body:
+  %i = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32* %a, i32 %i
+  %x = load i32, i32* %arrayidx
+  %xn = and i32 %x, 65535
+  store i32 %xn, i32* %arrayidx
+  %inc = add nuw nsw i32 %i, 1
+  %exitcond = icmp eq i32 %inc, %n
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:
+  ret void
+}
+
+define void @uxth_loop_2(i32* %a, i32 %n) {
+; V6M-LABEL: uxth_loop_2:
+; V6M:         .save {r4, lr}
+; V6M-NEXT:    push {r4, lr}
+; V6M-NEXT:    movs r2, #0
+; V6M-NEXT:  .LBB5_1:
+; V6M-NEXT:    uxth r3, r2
+; V6M-NEXT:    ldrh r4, [r0]
+; V6M-NEXT:    muls r4, r3, r4
+; V6M-NEXT:    stm r0!, {r4}
+; V6M-NEXT:    adds r2, r2, #1
+; V6M-NEXT:    cmp r1, r2
+; V6M-NEXT:    bne .LBB5_1
+; V6M-NEXT:    pop {r4, pc}
+entry:
+  br label %for.body
+
+for.body:
+  %i = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32* %a, i32 %i
+  %x = load i32, i32* %arrayidx
+  %xn = and i32 %x, 65535
+  %in = and i32 %i, 65535
+  %s = mul i32 %xn, %in
+  store i32 %s, i32* %arrayidx
+  %inc = add nuw nsw i32 %i, 1
+  %exitcond = icmp eq i32 %inc, %n
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:
+  ret void
+}
+
+define void @uxtb_loop(i32* %a, i32 %n) {
+; V6M-LABEL: uxtb_loop:
+; V6M:       .LBB6_1:
+; V6M-NEXT:    ldrb r2, [r0]
+; V6M-NEXT:    stm r0!, {r2}
+; V6M-NEXT:    subs r1, r1, #1
+; V6M-NEXT:    bne .LBB6_1
+; V6M-NEXT:    bx lr
+entry:
+  br label %for.body
+
+for.body:
+  %i = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32* %a, i32 %i
+  %x = load i32, i32* %arrayidx
+  %xn = and i32 %x, 255
+  store i32 %xn, i32* %arrayidx
+  %inc = add nuw nsw i32 %i, 1
+  %exitcond = icmp eq i32 %inc, %n
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:
+  ret void
+}
+
+define void @uxtb_loop_2(i32* %a, i32 %n) {
+; V6M-LABEL: uxtb_loop_2:
+; V6M:         .save {r4, lr}
+; V6M-NEXT:    push {r4, lr}
+; V6M-NEXT:    movs r2, #0
+; V6M-NEXT:  .LBB7_1:
+; V6M-NEXT:    uxtb r3, r2
+; V6M-NEXT:    ldrb r4, [r0]
+; V6M-NEXT:    muls r4, r3, r4
+; V6M-NEXT:    stm r0!, {r4}
+; V6M-NEXT:    adds r2, r2, #1
+; V6M-NEXT:    cmp r1, r2
+; V6M-NEXT:    bne .LBB7_1
+; V6M-NEXT:    pop {r4, pc}
+entry:
+  br label %for.body
+
+for.body:
+  %i = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32* %a, i32 %i
+  %x = load i32, i32* %arrayidx
+  %xn = and i32 %x, 255
+  %in = and i32 %i, 255
+  %s = mul i32 %xn, %in
+  store i32 %s, i32* %arrayidx
+  %inc = add nuw nsw i32 %i, 1
+  %exitcond = icmp eq i32 %inc, %n
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+
+for.cond.cleanup:
+  ret void
+}
+
diff --git a/test/CodeGen/Thumb2/2009-07-30-PEICrash.ll b/test/CodeGen/Thumb2/2009-07-30-PEICrash.ll
index 77d2991160d96f8e155d1811ad3cf4c93f7e91ad..1b4dbb326194bee6b0a521b7a32b063b4c4a1a75 100644
--- a/test/CodeGen/Thumb2/2009-07-30-PEICrash.ll
+++ b/test/CodeGen/Thumb2/2009-07-30-PEICrash.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -mtriple=thumbv7-apple-darwin9 -mcpu=cortex-a8 -relocation-model=pic -disable-fp-elim
+; RUN: llc < %s -mtriple=thumbv7-apple-darwin9 -mcpu=cortex-a8 -relocation-model=pic -frame-pointer=all
 
 	%struct.FILE = type { i8*, i32, i32, i16, i16, %struct.__sbuf, i32, i8*, i32 (i8*)*, i32 (i8*, i8*, i32)*, i64 (i8*, i64, i32)*, i32 (i8*, i8*, i32)*, %struct.__sbuf, %struct.__sFILEX*, i32, [3 x i8], [1 x i8], %struct.__sbuf, i32, i64 }
 	%struct.JHUFF_TBL = type { [17 x i8], [256 x i8], i32 }
diff --git a/test/CodeGen/Thumb2/2009-08-01-WrongLDRBOpc.ll b/test/CodeGen/Thumb2/2009-08-01-WrongLDRBOpc.ll
index 4a99e280e7abd5de6c44e91a0a6b81998978441b..1eafa5b2042c6b2a2b1ef41b8da3039941ec0735 100644
--- a/test/CodeGen/Thumb2/2009-08-01-WrongLDRBOpc.ll
+++ b/test/CodeGen/Thumb2/2009-08-01-WrongLDRBOpc.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -mtriple=thumbv7-apple-darwin9 -mcpu=cortex-a8 -relocation-model=pic -disable-fp-elim -arm-atomic-cfg-tidy=0 | FileCheck %s
+; RUN: llc < %s -mtriple=thumbv7-apple-darwin9 -mcpu=cortex-a8 -relocation-model=pic -frame-pointer=all -arm-atomic-cfg-tidy=0 | FileCheck %s
 
 @csize = external global [100 x [20 x [4 x i8]]]		; <[100 x [20 x [4 x i8]]]*> [#uses=1]
 @vsize = external global [100 x [20 x [4 x i8]]]		; <[100 x [20 x [4 x i8]]]*> [#uses=1]
diff --git a/test/CodeGen/Thumb2/2009-08-02-CoalescerBug.ll b/test/CodeGen/Thumb2/2009-08-02-CoalescerBug.ll
index 55b0921779d4edc902b8c512f99757dbc82a88ec..85d8d5c51d77da2d6eb9fdff9f7276b265d3dbb4 100644
--- a/test/CodeGen/Thumb2/2009-08-02-CoalescerBug.ll
+++ b/test/CodeGen/Thumb2/2009-08-02-CoalescerBug.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -mtriple=thumbv7-apple-darwin9 -mcpu=cortex-a8 -relocation-model=pic -disable-fp-elim
+; RUN: llc < %s -mtriple=thumbv7-apple-darwin9 -mcpu=cortex-a8 -relocation-model=pic -frame-pointer=all
 
 	%0 = type { void (%"struct.xalanc_1_8::FormatterToXML"*, i16)*, i32 }		; type %0
 	%1 = type { void (%"struct.xalanc_1_8::FormatterToXML"*, i16*)*, i32 }		; type %1
diff --git a/test/CodeGen/Thumb2/2009-08-04-CoalescerBug.ll b/test/CodeGen/Thumb2/2009-08-04-CoalescerBug.ll
index ccec979bf8cf52ac761828aaf1e0b70c68cb07c7..1be2c963a888a313ef91ba7639d78d168eac91ea 100644
--- a/test/CodeGen/Thumb2/2009-08-04-CoalescerBug.ll
+++ b/test/CodeGen/Thumb2/2009-08-04-CoalescerBug.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -mtriple=thumbv7-apple-darwin -mcpu=cortex-a8 -relocation-model=pic -disable-fp-elim
+; RUN: llc < %s -mtriple=thumbv7-apple-darwin -mcpu=cortex-a8 -relocation-model=pic -frame-pointer=all
 
 	%0 = type { %struct.GAP }		; type %0
 	%1 = type { i16, i8, i8 }		; type %1
diff --git a/test/CodeGen/Thumb2/2009-08-04-ScavengerAssert.ll b/test/CodeGen/Thumb2/2009-08-04-ScavengerAssert.ll
index 89f47d9d26f94f79473ada6bb7d7b9cd8f2dc984..5656d78423949a9d9a3d86cc49b60d54677e4305 100644
--- a/test/CodeGen/Thumb2/2009-08-04-ScavengerAssert.ll
+++ b/test/CodeGen/Thumb2/2009-08-04-ScavengerAssert.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -mtriple=thumbv7-apple-darwin -mcpu=cortex-a8 -relocation-model=pic -disable-fp-elim -O3
+; RUN: llc < %s -mtriple=thumbv7-apple-darwin -mcpu=cortex-a8 -relocation-model=pic -frame-pointer=all -O3
 
 	%0 = type { i16, i8, i8 }		; type %0
 	%1 = type { [2 x i32], [2 x i32] }		; type %1
diff --git a/test/CodeGen/Thumb2/2009-08-21-PostRAKill4.ll b/test/CodeGen/Thumb2/2009-08-21-PostRAKill4.ll
index 04dcb9d03ad5019edc4656e2f6500fa9875a91b3..ff6375d52f64e8ee53ab0b309d17c1ea2f5bdec8 100644
--- a/test/CodeGen/Thumb2/2009-08-21-PostRAKill4.ll
+++ b/test/CodeGen/Thumb2/2009-08-21-PostRAKill4.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -asm-verbose=false -O3 -relocation-model=pic -disable-fp-elim -mtriple=thumbv7-apple-darwin -mcpu=cortex-a8 -post-RA-scheduler
+; RUN: llc < %s -asm-verbose=false -O3 -relocation-model=pic -frame-pointer=all -mtriple=thumbv7-apple-darwin -mcpu=cortex-a8 -post-RA-scheduler
 
 ; ModuleID = '<stdin>'
 target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:32-f32:32:32-f64:32:32-v64:64:64-v128:128:128-a0:0:64"
diff --git a/test/CodeGen/Thumb2/2009-09-01-PostRAProlog.ll b/test/CodeGen/Thumb2/2009-09-01-PostRAProlog.ll
index 5b7107604e6f73ea492c1495ac5411affc468f55..ee1f03e2c428db17901305ea645e979b6f10262b 100644
--- a/test/CodeGen/Thumb2/2009-09-01-PostRAProlog.ll
+++ b/test/CodeGen/Thumb2/2009-09-01-PostRAProlog.ll
@@ -1,4 +1,4 @@
-; RUN: llc -asm-verbose=false -O3 -relocation-model=pic -disable-fp-elim -mtriple=thumbv7-apple-darwin -mcpu=cortex-a8 < %s | FileCheck %s
+; RUN: llc -asm-verbose=false -O3 -relocation-model=pic -frame-pointer=all -mtriple=thumbv7-apple-darwin -mcpu=cortex-a8 < %s | FileCheck %s
 
 target datalayout = "e-p:32:32:32-i1:8:32-i8:8:32-i16:16:32-i32:32:32-i64:32:32-f32:32:32-f64:32:32-v64:64:64-v128:128:128-a0:0:32"
 target triple = "thumbv7-apple-darwin9"
diff --git a/test/CodeGen/Thumb2/2009-11-01-CopyReg2RegBug.ll b/test/CodeGen/Thumb2/2009-11-01-CopyReg2RegBug.ll
index 4588018539535666311ac35d846c749a7907bf0c..7363a4a8eb5a4606f2ce5dbf848a1337ea096a86 100644
--- a/test/CodeGen/Thumb2/2009-11-01-CopyReg2RegBug.ll
+++ b/test/CodeGen/Thumb2/2009-11-01-CopyReg2RegBug.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -mtriple=thumbv7-apple-darwin -relocation-model=pic -disable-fp-elim -mcpu=cortex-a8
+; RUN: llc < %s -mtriple=thumbv7-apple-darwin -relocation-model=pic -frame-pointer=all -mcpu=cortex-a8
 
 define void @get_initial_mb16x16_cost() nounwind {
 entry:
diff --git a/test/CodeGen/Thumb2/2010-02-24-BigStack.ll b/test/CodeGen/Thumb2/2010-02-24-BigStack.ll
index 2b53747f99681711da1a815a01c882e26269e259..86e434a3043348c8bc63cc56b3524c52c7b1c19b 100644
--- a/test/CodeGen/Thumb2/2010-02-24-BigStack.ll
+++ b/test/CodeGen/Thumb2/2010-02-24-BigStack.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -O0 -relocation-model=pic -disable-fp-elim -mcpu=cortex-a8 -mattr=+vfp2
+; RUN: llc < %s -O0 -relocation-model=pic -frame-pointer=all -mcpu=cortex-a8 -mattr=+vfp2
 ; This test creates a big stack frame without spilling any callee-saved registers.
 ; Make sure the whole stack frame is addrerssable wiothout scavenger crashes.
 target datalayout = "e-p:32:32:32-i1:8:32-i8:8:32-i16:16:32-i32:32:32-i64:32:32-f32:32:32-f64:32:32-v64:64:64-v128:128:128-a0:0:32-n32"
diff --git a/test/CodeGen/Thumb2/2010-06-19-ITBlockCrash.ll b/test/CodeGen/Thumb2/2010-06-19-ITBlockCrash.ll
index eba2e584ddc060a1b99708551b9c8efa514500fd..42039161fbb00f3a41e961061cdd8ff37d7a1c0a 100644
--- a/test/CodeGen/Thumb2/2010-06-19-ITBlockCrash.ll
+++ b/test/CodeGen/Thumb2/2010-06-19-ITBlockCrash.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -mtriple=thumbv7-apple-darwin -O3 -relocation-model=pic -disable-fp-elim -mcpu=cortex-a8
+; RUN: llc < %s -mtriple=thumbv7-apple-darwin -O3 -relocation-model=pic -frame-pointer=all -mcpu=cortex-a8
 ; rdar://8110842
 
 declare arm_apcscc i32 @__maskrune(i32, i32)
diff --git a/test/CodeGen/Thumb2/2011-12-16-T2SizeReduceAssert.ll b/test/CodeGen/Thumb2/2011-12-16-T2SizeReduceAssert.ll
index 7c8802ddad670297ef3e5c62028f8c43938e7a87..854b42522e7c7f9af660fdd98757840187f445f1 100644
--- a/test/CodeGen/Thumb2/2011-12-16-T2SizeReduceAssert.ll
+++ b/test/CodeGen/Thumb2/2011-12-16-T2SizeReduceAssert.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -mtriple=thumbv7-apple-ios -relocation-model=pic -disable-fp-elim -mcpu=cortex-a8
+; RUN: llc < %s -mtriple=thumbv7-apple-ios -relocation-model=pic -frame-pointer=all -mcpu=cortex-a8
 ; RUN: llc < %s -mtriple=thumbv8-none-linux-gnueabi
 
 %struct.LIST_NODE.0.16 = type { %struct.LIST_NODE.0.16*, i8* }
diff --git a/test/CodeGen/Thumb2/2012-01-13-CBNZBug.ll b/test/CodeGen/Thumb2/2012-01-13-CBNZBug.ll
index e5be8df086309b5ae94fec538ea0119650c56a99..99daf7e0acf5f592e76398a1a616cfcdaacf47a6 100644
--- a/test/CodeGen/Thumb2/2012-01-13-CBNZBug.ll
+++ b/test/CodeGen/Thumb2/2012-01-13-CBNZBug.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -mtriple=thumbv7-apple-ios -relocation-model=pic -disable-fp-elim -mcpu=cortex-a8 | FileCheck %s
+; RUN: llc < %s -mtriple=thumbv7-apple-ios -relocation-model=pic -frame-pointer=all -mcpu=cortex-a8 | FileCheck %s
 ; rdar://10676853
 
 %struct.Dict_node_struct = type { i8*, %struct.Word_file_struct*, %struct.Exp_struct*, %struct.Dict_node_struct*, %struct.Dict_node_struct* }
diff --git a/test/CodeGen/Thumb2/frameless.ll b/test/CodeGen/Thumb2/frameless.ll
index fa8d5d87dfca374ae3f08085fcc6bd5e9a1b4da5..01e0414de37d93197e9a46f730b97917d0990b74 100644
--- a/test/CodeGen/Thumb2/frameless.ll
+++ b/test/CodeGen/Thumb2/frameless.ll
@@ -1,5 +1,5 @@
-; RUN: llc < %s -mtriple=thumbv7-apple-darwin -disable-fp-elim | not grep mov
-; RUN: llc < %s -mtriple=thumbv7-linux -disable-fp-elim | not grep mov
+; RUN: llc < %s -mtriple=thumbv7-apple-darwin -frame-pointer=all | not grep mov
+; RUN: llc < %s -mtriple=thumbv7-linux -frame-pointer=all | not grep mov
 
 define void @t() nounwind readnone {
   ret void
diff --git a/test/CodeGen/Thumb2/frameless2.ll b/test/CodeGen/Thumb2/frameless2.ll
index 374335421a540aa6f3cb2c2d2100cab3e96ab6ec..c8aa82f6030b161724380d6518736082b6c67757 100644
--- a/test/CodeGen/Thumb2/frameless2.ll
+++ b/test/CodeGen/Thumb2/frameless2.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -mtriple=thumbv7-apple-darwin -disable-fp-elim | not grep r7
+; RUN: llc < %s -mtriple=thumbv7-apple-darwin -frame-pointer=all | not grep r7
 
 %struct.noise3 = type { [3 x [17 x i32]] }
 %struct.noiseguard = type { i32, i32, i32 }
diff --git a/test/CodeGen/Thumb2/ldr-str-imm12.ll b/test/CodeGen/Thumb2/ldr-str-imm12.ll
index c6d00d4c1e11bfe0342304f6c0e1bc79d137dbb9..c1f7de3dbdfecec1bcd4ab34a3c79abbdc4a4b88 100644
--- a/test/CodeGen/Thumb2/ldr-str-imm12.ll
+++ b/test/CodeGen/Thumb2/ldr-str-imm12.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -mtriple=thumbv7-apple-darwin -arm-atomic-cfg-tidy=0 -mcpu=cortex-a8 -relocation-model=pic -disable-fp-elim | FileCheck %s
+; RUN: llc < %s -mtriple=thumbv7-apple-darwin -arm-atomic-cfg-tidy=0 -mcpu=cortex-a8 -relocation-model=pic -frame-pointer=all | FileCheck %s
 ; rdar://7352504
 ; Make sure we use "str r9, [sp, #+28]" instead of "sub.w r4, r7, #256" followed by "str r9, [r4, #-32]".
 
diff --git a/test/CodeGen/Thumb2/machine-licm.ll b/test/CodeGen/Thumb2/machine-licm.ll
index 1f638751345b943d0c220849bfcdc07fa62a38be..d2438ac4242ef6f3d9a7b9d6ba5c9199a3d19dbf 100644
--- a/test/CodeGen/Thumb2/machine-licm.ll
+++ b/test/CodeGen/Thumb2/machine-licm.ll
@@ -1,5 +1,5 @@
-; RUN: llc < %s -mtriple=thumbv7-apple-darwin -mcpu=cortex-a8 -relocation-model=dynamic-no-pic -disable-fp-elim | FileCheck %s
-; RUN: llc < %s -mtriple=thumbv7-apple-darwin -mcpu=cortex-a8 -relocation-model=pic -disable-fp-elim | FileCheck %s --check-prefix=PIC
+; RUN: llc < %s -mtriple=thumbv7-apple-darwin -mcpu=cortex-a8 -relocation-model=dynamic-no-pic -frame-pointer=all | FileCheck %s
+; RUN: llc < %s -mtriple=thumbv7-apple-darwin -mcpu=cortex-a8 -relocation-model=pic -frame-pointer=all | FileCheck %s --check-prefix=PIC
 ; rdar://7353541
 ; rdar://7354376
 
diff --git a/test/CodeGen/Thumb2/t2-teq-reduce.mir b/test/CodeGen/Thumb2/t2-teq-reduce.mir
new file mode 100644
index 0000000000000000000000000000000000000000..d85d233aa459c8a6b6334abc4d3714e56a78c4b3
--- /dev/null
+++ b/test/CodeGen/Thumb2/t2-teq-reduce.mir
@@ -0,0 +1,267 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -run-pass=t2-reduce-size %s -o - | FileCheck %s
+
+--- |
+  target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64"
+  target triple = "thumbv8m.main"
+
+  %list_head = type { %list_head*, %list_data* }
+  %list_data = type { i16, i16 }
+
+  define %list_head* @reg_reg_it_block(%list_head* %a, i16 zeroext %b) {
+  entry:
+    br label %while.begin
+
+  while.begin:                                      ; preds = %while.body.end, %entry
+    %list.addr.i = phi %list_head* [ %ld.5, %while.body.end ], [ %a, %entry ]
+    %info.i = getelementptr inbounds %list_head, %list_head* %list.addr.i, i32 0, i32 1
+    %ld.0 = load %list_data*, %list_data** %info.i, align 4
+    %data16.i1 = bitcast %list_data* %ld.0 to i16*
+    %ld.1 = load i16, i16* %data16.i1, align 2
+    %xor.1 = xor i16 %ld.1, %b
+    %cmp.i = icmp eq i16 %xor.1, 0
+    br i1 %cmp.i, label %exit, label %while.body.a
+
+  while.body.a:                                     ; preds = %while.begin
+    %next.i2 = bitcast %list_head* %list.addr.i to %list_head**
+    %ld.2 = load %list_head*, %list_head** %next.i2, align 4
+    %cmp.i.1 = icmp eq %list_head* %ld.2, null
+    br i1 %cmp.i.1, label %exit, label %it.block
+
+  it.block:                                         ; preds = %while.body.a
+    %info.i.1 = getelementptr inbounds %list_head, %list_head* %ld.2, i32 0, i32 1
+    %ld.3 = load %list_data*, %list_data** %info.i.1, align 4
+    %data16.i.13 = bitcast %list_data* %ld.3 to i16*
+    %ld.4 = load i16, i16* %data16.i.13, align 2
+    %xor.2 = xor i16 %ld.4, %b
+    %cmp.i.2 = icmp eq i16 %xor.2, 0
+    br i1 %cmp.i.2, label %exit, label %while.body.end
+
+  while.body.end:                                   ; preds = %it.block
+    %next.i.14 = bitcast %list_head* %ld.2 to %list_head**
+    %ld.5 = load %list_head*, %list_head** %next.i.14, align 4
+    %cmp.i.3 = icmp eq %list_head* %ld.5, null
+    br i1 %cmp.i.3, label %exit, label %while.begin
+
+  exit:                                             ; preds = %while.body.end, %it.block, %while.body.a, %while.begin
+    %res = phi %list_head* [ %list.addr.i, %while.begin ], [ %ld.2, %while.body.a ], [ %ld.2, %it.block ], [ %ld.5, %while.body.end ]
+    ret %list_head* %res
+  }
+
+  define i16 @op_not_killed(%list_head* %a, i16 zeroext %b) {
+  entry:
+    br label %while.begin
+
+  while.begin:                                      ; preds = %while.body.end, %entry
+    %list.addr.i = phi %list_head* [ %ld.5, %while.body.end ], [ %a, %entry ]
+    %info.i = getelementptr inbounds %list_head, %list_head* %list.addr.i, i32 0, i32 1
+    %ld.0 = load %list_data*, %list_data** %info.i, align 4
+    %data16.i1 = bitcast %list_data* %ld.0 to i16*
+    %ld.1 = load i16, i16* %data16.i1, align 2
+    %xor.1 = xor i16 %ld.1, %b
+    %cmp.i = icmp eq i16 %xor.1, 0
+    br i1 %cmp.i, label %exit, label %while.body.a
+
+  while.body.a:                                     ; preds = %while.begin
+    %next.i2 = bitcast %list_head* %list.addr.i to %list_head**
+    %ld.2 = load %list_head*, %list_head** %next.i2, align 4
+    %cmp.i.1 = icmp eq %list_head* %ld.2, null
+    br i1 %cmp.i.1, label %exit, label %it.block
+
+  it.block:                                         ; preds = %while.body.a
+    %info.i.1 = getelementptr inbounds %list_head, %list_head* %ld.2, i32 0, i32 1
+    %ld.3 = load %list_data*, %list_data** %info.i.1, align 4
+    %data16.i.13 = bitcast %list_data* %ld.3 to i16*
+    %ld.4 = load i16, i16* %data16.i.13, align 2
+    %xor.2 = xor i16 %ld.4, %b
+    %cmp.i.2 = icmp eq i16 %xor.2, 0
+    br i1 %cmp.i.2, label %exit, label %while.body.end
+
+  while.body.end:                                   ; preds = %it.block
+    %next.i.14 = bitcast %list_head* %ld.2 to %list_head**
+    %ld.5 = load %list_head*, %list_head** %next.i.14, align 4
+    %cmp.i.3 = icmp eq %list_head* %ld.5, null
+    br i1 %cmp.i.3, label %exit, label %while.begin
+
+  exit:                                             ; preds = %while.body.end, %it.block, %while.body.a, %while.begin
+    %res = phi i16 [ %ld.1, %while.begin ], [ %ld.1, %while.body.a ], [ %ld.4, %it.block ], [ %ld.4, %while.body.end ]
+    ret i16 %res
+  }
+
+...
+---
+name:            reg_reg_it_block
+tracksRegLiveness: true
+liveins:
+  - { reg: '$r0', virtual-reg: '' }
+  - { reg: '$r1', virtual-reg: '' }
+body:             |
+  ; CHECK-LABEL: name: reg_reg_it_block
+  ; CHECK: bb.0.entry:
+  ; CHECK:   successors: %bb.2(0x80000000)
+  ; CHECK:   liveins: $r0, $r1
+  ; CHECK:   t2B %bb.2, 14, $noreg
+  ; CHECK: bb.1.while.body.end:
+  ; CHECK:   successors: %bb.2(0x80000000)
+  ; CHECK:   liveins: $r0, $r1
+  ; CHECK:   renamable $r0 = tLDRi killed renamable $r0, 0, 14, $noreg :: (load 4 from %ir.next.i.14)
+  ; CHECK:   tCMPi8 renamable $r0, 0, 14, $noreg, implicit-def $cpsr
+  ; CHECK:   BUNDLE implicit-def dead $itstate, implicit killed $cpsr, implicit $r0 {
+  ; CHECK:     t2IT 0, 8, implicit-def $itstate
+  ; CHECK:     tBX_RET 0, killed $cpsr, implicit $r0, implicit internal killed $itstate
+  ; CHECK:   }
+  ; CHECK: bb.2.while.begin:
+  ; CHECK:   successors: %bb.4(0x04000000), %bb.3(0x7c000000)
+  ; CHECK:   liveins: $r0, $r1
+  ; CHECK:   renamable $r2 = tLDRi renamable $r0, 1, 14, $noreg :: (load 4 from %ir.info.i)
+  ; CHECK:   renamable $r2 = tLDRHi killed renamable $r2, 0, 14, $noreg :: (load 2 from %ir.data16.i1)
+  ; CHECK:   dead renamable $r2, $cpsr = tEOR killed renamable $r2, renamable $r1, 14, $noreg
+  ; CHECK:   t2Bcc %bb.4, 0, killed $cpsr
+  ; CHECK: bb.3.while.body.a:
+  ; CHECK:   successors: %bb.4(0x4207fef8), %bb.1(0x3df80108)
+  ; CHECK:   liveins: $r0, $r1
+  ; CHECK:   renamable $r0 = tLDRi killed renamable $r0, 0, 14, $noreg :: (load 4 from %ir.next.i2)
+  ; CHECK:   tCMPi8 renamable $r0, 0, 14, $noreg, implicit-def $cpsr
+  ; CHECK:   BUNDLE implicit-def dead $itstate, implicit-def dead $r2, implicit-def $cpsr, implicit $r0, implicit killed $cpsr, implicit $r1 {
+  ; CHECK:     t2IT 1, 30, implicit-def $itstate
+  ; CHECK:     renamable $r2 = tLDRi renamable $r0, 1, 1, $cpsr, implicit internal $itstate :: (load 4 from %ir.info.i.1)
+  ; CHECK:     renamable $r2 = tLDRHi internal killed renamable $r2, 0, 1, $cpsr, implicit internal killed $r2, implicit internal $itstate :: (load 2 from %ir.data16.i.13)
+  ; CHECK:     t2TEQrr internal killed renamable $r2, renamable $r1, 1, killed $cpsr, implicit-def $cpsr, implicit internal killed $itstate
+  ; CHECK:   }
+  ; CHECK:   t2Bcc %bb.1, 1, killed $cpsr
+  ; CHECK: bb.4.exit:
+  ; CHECK:   liveins: $r0
+  ; CHECK:   tBX_RET 14, $noreg, implicit killed $r0
+  bb.0.entry:
+    successors: %bb.1(0x80000000)
+    liveins: $r0, $r1
+
+    t2B %bb.1, 14, $noreg
+
+  bb.3.while.body.end:
+    successors: %bb.1(0x80000000)
+    liveins: $r0, $r1
+
+    renamable $r0 = tLDRi killed renamable $r0, 0, 14, $noreg :: (load 4 from %ir.next.i.14)
+    tCMPi8 renamable $r0, 0, 14, $noreg, implicit-def $cpsr
+    BUNDLE implicit-def dead $itstate, implicit killed $cpsr, implicit $r0 {
+      t2IT 0, 8, implicit-def $itstate
+      tBX_RET 0, killed $cpsr, implicit $r0, implicit internal killed $itstate
+    }
+
+  bb.1.while.begin:
+    successors: %bb.4(0x04000000), %bb.2(0x7c000000)
+    liveins: $r0, $r1
+
+    renamable $r2 = tLDRi renamable $r0, 1, 14, $noreg :: (load 4 from %ir.info.i)
+    renamable $r2 = tLDRHi killed renamable $r2, 0, 14, $noreg :: (load 2 from %ir.data16.i1)
+    dead renamable $r2, $cpsr = tEOR killed renamable $r2, renamable $r1, 14, $noreg
+    t2Bcc %bb.4, 0, killed $cpsr
+
+  bb.2.while.body.a:
+    successors: %bb.4(0x80000000), %bb.3(0x78200000)
+    liveins: $r0, $r1
+
+    renamable $r0 = tLDRi killed renamable $r0, 0, 14, $noreg :: (load 4 from %ir.next.i2)
+    tCMPi8 renamable $r0, 0, 14, $noreg, implicit-def $cpsr
+    BUNDLE implicit-def dead $itstate, implicit-def dead $r2, implicit-def $cpsr, implicit $r0, implicit killed $cpsr, implicit $r1 {
+      t2IT 1, 30, implicit-def $itstate
+      renamable $r2 = tLDRi renamable $r0, 1, 1, $cpsr, implicit internal $itstate :: (load 4 from %ir.info.i.1)
+      renamable $r2 = tLDRHi internal killed renamable $r2, 0, 1, $cpsr, implicit internal killed $r2, implicit internal $itstate :: (load 2 from %ir.data16.i.13)
+      t2TEQrr internal killed renamable $r2, renamable $r1, 1, killed $cpsr, implicit-def $cpsr, implicit internal killed $itstate
+    }
+    t2Bcc %bb.3, 1, killed $cpsr
+
+  bb.4.exit:
+    liveins: $r0
+
+    tBX_RET 14, $noreg, implicit killed $r0
+
+...
+---
+name:            op_not_killed
+tracksRegLiveness: true
+liveins:
+  - { reg: '$r0', virtual-reg: '' }
+  - { reg: '$r1', virtual-reg: '' }
+body:             |
+  ; CHECK-LABEL: name: op_not_killed
+  ; CHECK: bb.0.entry:
+  ; CHECK:   successors: %bb.1(0x80000000)
+  ; CHECK:   liveins: $r0, $r1
+  ; CHECK:   $r2 = tMOVr $r0, 14, $noreg
+  ; CHECK: bb.1.while.begin:
+  ; CHECK:   successors: %bb.5(0x04000000), %bb.2(0x7c000000)
+  ; CHECK:   liveins: $r1, $r2
+  ; CHECK:   renamable $r0 = tLDRi renamable $r2, 1, 14, $noreg :: (load 4 from %ir.info.i)
+  ; CHECK:   renamable $r0 = tLDRHi killed renamable $r0, 0, 14, $noreg :: (load 2 from %ir.data16.i1)
+  ; CHECK:   t2TEQrr renamable $r0, renamable $r1, 14, $noreg, implicit-def $cpsr
+  ; CHECK:   t2Bcc %bb.5, 0, killed $cpsr
+  ; CHECK: bb.2.while.body.a:
+  ; CHECK:   successors: %bb.5(0x04000000), %bb.3(0x7c000000)
+  ; CHECK:   liveins: $r0, $r1, $r2
+  ; CHECK:   renamable $r2 = tLDRi killed renamable $r2, 0, 14, $noreg :: (load 4 from %ir.next.i2)
+  ; CHECK:   tCMPi8 renamable $r2, 0, 14, $noreg, implicit-def $cpsr
+  ; CHECK:   t2Bcc %bb.5, 0, killed $cpsr
+  ; CHECK: bb.3.it.block:
+  ; CHECK:   successors: %bb.5(0x04000000), %bb.4(0x7c000000)
+  ; CHECK:   liveins: $r1, $r2
+  ; CHECK:   renamable $r0 = tLDRi renamable $r2, 1, 14, $noreg :: (load 4 from %ir.info.i.1)
+  ; CHECK:   renamable $r0 = tLDRHi killed renamable $r0, 0, 14, $noreg :: (load 2 from %ir.data16.i.13)
+  ; CHECK:   t2TEQrr renamable $r0, renamable $r1, 14, $noreg, implicit-def $cpsr
+  ; CHECK:   t2Bcc %bb.5, 0, killed $cpsr
+  ; CHECK: bb.4.while.body.end:
+  ; CHECK:   successors: %bb.5(0x04000000), %bb.1(0x7c000000)
+  ; CHECK:   liveins: $r0, $r1, $r2
+  ; CHECK:   renamable $r2 = tLDRi killed renamable $r2, 0, 14, $noreg :: (load 4 from %ir.next.i.14)
+  ; CHECK:   tCMPi8 renamable $r2, 0, 14, $noreg, implicit-def $cpsr
+  ; CHECK:   t2Bcc %bb.1, 1, killed $cpsr
+  ; CHECK: bb.5.exit:
+  ; CHECK:   liveins: $r0
+  ; CHECK:   tBX_RET 14, $noreg, implicit $r0
+  bb.0.entry:
+    successors: %bb.1(0x80000000)
+    liveins: $r0, $r1
+
+    $r2 = tMOVr $r0, 14, $noreg
+
+  bb.1.while.begin:
+    successors: %bb.5(0x04000000), %bb.2(0x7c000000)
+    liveins: $r1, $r2
+
+    renamable $r0 = t2LDRi12 renamable $r2, 4, 14, $noreg :: (load 4 from %ir.info.i)
+    renamable $r0 = t2LDRHi12 killed renamable $r0, 0, 14, $noreg :: (load 2 from %ir.data16.i1)
+    t2TEQrr renamable $r0, renamable $r1, 14, $noreg, implicit-def $cpsr
+    t2Bcc %bb.5, 0, killed $cpsr
+
+  bb.2.while.body.a:
+    successors: %bb.5(0x04000000), %bb.3(0x7c000000)
+    liveins: $r0, $r1, $r2
+
+    renamable $r2 = t2LDRi12 killed renamable $r2, 0, 14, $noreg :: (load 4 from %ir.next.i2)
+    t2CMPri renamable $r2, 0, 14, $noreg, implicit-def $cpsr
+    t2Bcc %bb.5, 0, killed $cpsr
+
+  bb.3.it.block:
+    successors: %bb.5(0x04000000), %bb.4(0x7c000000)
+    liveins: $r1, $r2
+
+    renamable $r0 = t2LDRi12 renamable $r2, 4, 14, $noreg :: (load 4 from %ir.info.i.1)
+    renamable $r0 = t2LDRHi12 killed renamable $r0, 0, 14, $noreg :: (load 2 from %ir.data16.i.13)
+    t2TEQrr renamable $r0, renamable $r1, 14, $noreg, implicit-def $cpsr
+    t2Bcc %bb.5, 0, killed $cpsr
+
+  bb.4.while.body.end:
+    successors: %bb.5(0x04000000), %bb.1(0x7c000000)
+    liveins: $r0, $r1, $r2
+
+    renamable $r2 = t2LDRi12 killed renamable $r2, 0, 14, $noreg :: (load 4 from %ir.next.i.14)
+    t2CMPri renamable $r2, 0, 14, $noreg, implicit-def $cpsr
+    t2Bcc %bb.1, 1, killed $cpsr
+
+  bb.5.exit:
+    liveins: $r0
+
+    tBX_RET 14, $noreg, implicit $r0
+
+...
diff --git a/test/CodeGen/Thumb2/thumb2-teq2.ll b/test/CodeGen/Thumb2/thumb2-teq2.ll
index 22bde879549170455a71b991c147716c0b7fc8a9..1f37ddebd0b1d019e8d0134052b93cc57510ee42 100644
--- a/test/CodeGen/Thumb2/thumb2-teq2.ll
+++ b/test/CodeGen/Thumb2/thumb2-teq2.ll
@@ -5,7 +5,7 @@
 
 define i32 @f2(i32 %a, i32 %b) {
 ; CHECK: f2
-; CHECK: teq.w {{.*}}, r1
+; CHECK: eors {{.*}}, r1
     %tmp = xor i32 %a, %b
     %tmp1 = icmp eq i32 %tmp, 0
     %ret = select i1 %tmp1, i32 42, i32 24
@@ -14,7 +14,7 @@ define i32 @f2(i32 %a, i32 %b) {
 
 define i32 @f4(i32 %a, i32 %b) {
 ; CHECK: f4
-; CHECK: teq.w  {{.*}}, r1
+; CHECK: eors  {{.*}}, r1
     %tmp = xor i32 %a, %b
     %tmp1 = icmp eq i32 0, %tmp
     %ret = select i1 %tmp1, i32 42, i32 24
diff --git a/test/CodeGen/WebAssembly/PR40172.ll b/test/CodeGen/WebAssembly/PR40172.ll
new file mode 100644
index 0000000000000000000000000000000000000000..e0ad5be8b219e5b63ebd881b848115e5e1d8c0c2
--- /dev/null
+++ b/test/CodeGen/WebAssembly/PR40172.ll
@@ -0,0 +1,31 @@
+; RUN: llc < %s -O0 -wasm-disable-explicit-locals -wasm-keep-registers | FileCheck %s
+
+; Regression test for bug 40172. The problem was that FastISel assumed
+; that CmpInst results did not need to be zero extended because
+; WebAssembly's compare instructions always return 0 or 1. But in this
+; test case FastISel falls back to DAG ISel, which combines away the
+; comparison, invalidating FastISel's assumption.
+
+target datalayout = "e-m:e-p:32:32-i64:64-n32:64-S128"
+target triple = "wasm32-unknown-unknown"
+
+; CHECK:  i32.sub $[[BASE:[0-9]+]]=,
+; CHECK:  local.copy $[[ARG:[0-9]+]]=, $0{{$}}
+; CHECK:  i32.const $[[A0:[0-9]+]]=, 1{{$}}
+; CHECK:  i32.and $[[A1:[0-9]+]]=, $[[ARG]], $[[A0]]{{$}}
+; CHECK:  i32.store8 8($[[BASE]]), $[[A1]]{{$}}
+
+define void @test(i8 %byte) {
+  %t = alloca { i8, i8 }, align 1
+  %x4 = and i8 %byte, 1
+  %x5 = icmp eq i8 %x4, 1
+  %x6 = and i8 %byte, 2
+  %x7 = icmp eq i8 %x6, 2
+  %x8 = bitcast { i8, i8 }* %t to i8*
+  %x9 = zext i1 %x5 to i8
+  store i8 %x9, i8* %x8, align 1
+  %x10 = getelementptr inbounds { i8, i8 }, { i8, i8 }* %t, i32 0, i32 1
+  %x11 = zext i1 %x7 to i8
+  store i8 %x11, i8* %x10, align 1
+  ret void
+}
diff --git a/test/CodeGen/WebAssembly/PR40267.ll b/test/CodeGen/WebAssembly/PR40267.ll
new file mode 100644
index 0000000000000000000000000000000000000000..5773d08da74da1ef0d8210acddf54d443ee40755
--- /dev/null
+++ b/test/CodeGen/WebAssembly/PR40267.ll
@@ -0,0 +1,22 @@
+; RUN: llc < %s -asm-verbose=false -verify-machineinstrs -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-keep-registers
+
+; Regression test for PR40267. Tests that invalid indices in
+; extract_vector_elt can be handled when vectors ops are split. Notice
+; that SIMD is not enabled for this test. Check only that llc does not
+; crash, since it would previously trigger an assertion.
+
+target datalayout = "e-m:e-p:32:32-i64:64-n32:64-S128"
+target triple = "wasm32-unknown-unknown"
+
+define void @foo() {
+  %L6 = load i32, i32* undef
+  br label %BB1
+
+BB1:                                              ; preds = %BB1, %0
+  %bj = select <4 x i1> <i1 true, i1 true, i1 false, i1 false>, <4 x i32> <i32 55, i32 21, i32 92, i32 68>, <4 x i32> <i32 51, i32 61, i32 62, i32 39>
+  %E1 = extractelement <4 x i32> %bj, i32 0
+  %E23 = extractelement <4 x i32> zeroinitializer, i32 %E1
+  %I33 = insertelement <4 x i32> undef, i32 %E23, i1 undef
+  store <4 x i32> %I33, <4 x i32>* undef
+  br label %BB1
+}
diff --git a/test/CodeGen/WebAssembly/add-prototypes.ll b/test/CodeGen/WebAssembly/add-prototypes.ll
index 583cadea03b7ff293338e4c1ac41f617908232a5..865b3d3f3b312a683b7c21fbfe7b14e50eebceba 100644
--- a/test/CodeGen/WebAssembly/add-prototypes.ll
+++ b/test/CodeGen/WebAssembly/add-prototypes.ll
@@ -17,7 +17,8 @@ define void @call_foo(i32 %a) {
 }
 
 ; CHECK-LABEL: @call_foo_ptr
-; CHECK: %call = call i64 @foo(i32 43)
+; CHECK: %1 = bitcast i64 (...)* bitcast (i64 (i32)* @foo to i64 (...)*) to i64 (i32)*
+; CHECK-NEXT: %call = call i64 %1(i32 43)
 define i64 @call_foo_ptr(i32 %a) {
   %1 = bitcast i64 (...)* @foo to i64 (i32)*
   %call = call i64 (i32) %1(i32 43)
@@ -25,7 +26,8 @@ define i64 @call_foo_ptr(i32 %a) {
 }
 
 ; CHECK-LABEL: @to_intptr_inst
-; CHECK: ret i8* bitcast (i64 (i32)* @foo to i8*)
+; CHECK: %1 = bitcast i64 (...)* bitcast (i64 (i32)* @foo to i64 (...)*) to i8*
+; CHECK-NEXT: ret i8* %1
 define i8* @to_intptr_inst() {
   %1 = bitcast i64 (...)* @foo to i8*
   ret i8* %1
@@ -37,8 +39,27 @@ define i8* @to_intptr_constexpr() {
   ret i8* bitcast (i64 (...)* @foo to i8*)
 }
 
-; CHECK: declare i64 @foo(i32)
-declare i64 @foo(...) #1
+; CHECK-LABEL: @null_compare
+; CHECK: br i1 icmp eq (i64 (...)* bitcast (i64 (i32)* @foo to i64 (...)*), i64 (...)* null), label %if.then, label %if.end
+define i8 @null_compare() {
+  br i1 icmp eq (i64 (...)* @foo, i64 (...)* null), label %if.then, label %if.end
+if.then:
+  ret i8 0
+if.end:
+  ret i8 1
+}
+
+; CHECK-LABEL: @as_paramater
+; CHECK: call void @func_param(i64 (...)* bitcast (i64 (i32)* @foo to i64 (...)*))
+define void @as_paramater() {
+  call void @func_param(i64 (...)* @foo)
+  ret void
+}
+
+declare void @func_param(i64 (...)*)
+
+; CHECK: declare extern_weak i64 @foo(i32)
+declare extern_weak i64 @foo(...) #1
 
 ; CHECK-NOT: attributes {{.*}} = { {{.*}}"no-prototype"{{.*}} }
 attributes #1 = { "no-prototype" }
diff --git a/test/CodeGen/WebAssembly/annotations.mir b/test/CodeGen/WebAssembly/annotations.mir
deleted file mode 100644
index 1ae2db8248471b9caa60d758b72d9f305765eab8..0000000000000000000000000000000000000000
--- a/test/CodeGen/WebAssembly/annotations.mir
+++ /dev/null
@@ -1,94 +0,0 @@
-# RUN: llc -mtriple=wasm32-unknown-unknown -start-after xray-instrumentation -wasm-keep-registers %s -o - | FileCheck %s
-
----
-# Tests if block/loop/try/catch/end instructions are correctly printed with
-# their annotations.
-
-# CHECK: test0:
-# CHECK:   block
-# CHECK:   try
-# CHECK:   br        0               # 0: down to label1
-# CHECK:   catch_all                 # catch0:
-# CHECK:   block
-# CHECK:   br_if     0, 1            # 0: down to label2
-# CHECK:   loop                      # label3:
-# CHECK:   br_if     0, 1            # 0: up to label3
-# CHECK:   end_loop
-# CHECK:   end_block                 # label2:
-# CHECK:   try
-# CHECK:   rethrow   0               # 0: down to catch1
-# CHECK:   catch_all                 # catch1:
-# CHECK:   block
-# CHECK:   try
-# CHECK:   br        0               # 0: down to label6
-# CHECK:   catch_all                 # catch2:
-# CHECK:   unreachable
-# CHECK:   end_try                   # label6:
-# CHECK:   end_block                 # label5:
-# CHECK:   rethrow   0               # 0: to caller
-# CHECK:   end_try                   # label4:
-# CHECK:   end_try                   # label1:
-# CHECK:   end_block                 # label0:
-
-name: test0
-liveins:
-  - { reg: '$arguments', reg: '$value_stack' }
-body: |
-  bb.0:
-    successors: %bb.7, %bb.1
-    BLOCK 64, implicit-def $value_stack, implicit $value_stack
-    TRY 64, implicit-def $value_stack, implicit $value_stack
-    BR 0, implicit-def $arguments
-
-  bb.1 (landing-pad):
-  ; predecessors: %bb.0
-    successors: %bb.2, %bb.3
-
-    CATCH_ALL implicit-def $arguments
-    BLOCK 64, implicit-def $value_stack, implicit $value_stack
-    BR_IF 0, 1, implicit-def $arguments, implicit-def $value_stack, implicit $value_stack
-
-  bb.2:
-  ; predecessors: %bb.1, %bb.2
-    successors: %bb.2, %bb.3
-
-    LOOP 64, implicit-def $value_stack, implicit $value_stack
-    BR_IF 0, 1, implicit-def $arguments
-
-  bb.3:
-  ; predecessors: %bb.1, %bb.2
-    successors: %bb.4
-
-    END_LOOP implicit-def $value_stack, implicit $value_stack
-    END_BLOCK implicit-def $value_stack, implicit $value_stack
-    TRY 64, implicit-def $value_stack, implicit $value_stack
-    RETHROW 0, implicit-def $arguments
-
-  bb.4 (landing-pad):
-  ; predecessors: %bb.3
-    successors: %bb.6, %bb.5
-
-    CATCH_ALL implicit-def $arguments
-    BLOCK 64, implicit-def $value_stack, implicit $value_stack
-    TRY 64, implicit-def $value_stack, implicit $value_stack
-    BR 0, implicit-def $arguments
-
-  bb.5 (landing-pad):
-  ; predecessors: %bb.4
-    CATCH_ALL implicit-def $arguments
-    UNREACHABLE implicit-def dead $arguments
-
-  bb.6:
-  ; predecessors: %bb.4
-    END_TRY implicit-def $value_stack, implicit $value_stack
-    END_BLOCK implicit-def $value_stack, implicit $value_stack
-    RETHROW 0, implicit-def $arguments
-
-  bb.7:
-  ; predecessors: %bb.0
-    END_TRY implicit-def $value_stack, implicit $value_stack
-    END_TRY implicit-def $value_stack, implicit $value_stack
-    END_BLOCK implicit-def $value_stack, implicit $value_stack
-    FALLTHROUGH_RETURN_VOID implicit-def dead $arguments
-    END_FUNCTION implicit-def $value_stack, implicit $value_stack
-...
diff --git a/test/CodeGen/WebAssembly/atomic-rmw.ll b/test/CodeGen/WebAssembly/atomic-rmw.ll
index d4e66801347d2e53a95416e030400ce08a2e12cf..27b3be09943df6822e7a5fbb7bcd02a2ae6bed06 100644
--- a/test/CodeGen/WebAssembly/atomic-rmw.ll
+++ b/test/CodeGen/WebAssembly/atomic-rmw.ll
@@ -276,7 +276,7 @@ define i64 @umin_i64(i64* %p, i64 %v) {
 
 ; CHECK-LABEL: add_sext_i8_i32:
 ; CHECK-NEXT: .functype add_sext_i8_i32 (i32, i32) -> (i32){{$}}
-; CHECK: i32.atomic.rmw8_u.add $push0=, 0($0), $1{{$}}
+; CHECK: i32.atomic.rmw8.add_u $push0=, 0($0), $1{{$}}
 ; CHECK-NEXT: i32.extend8_s $push1=, $pop0{{$}}
 ; CHECK-NEXT: return $pop1{{$}}
 define i32 @add_sext_i8_i32(i8* %p, i32 %v) {
@@ -288,7 +288,7 @@ define i32 @add_sext_i8_i32(i8* %p, i32 %v) {
 
 ; CHECK-LABEL: add_sext_i16_i32:
 ; CHECK-NEXT: .functype add_sext_i16_i32 (i32, i32) -> (i32){{$}}
-; CHECK: i32.atomic.rmw16_u.add $push0=, 0($0), $1{{$}}
+; CHECK: i32.atomic.rmw16.add_u $push0=, 0($0), $1{{$}}
 ; CHECK-NEXT: i32.extend16_s $push1=, $pop0{{$}}
 ; CHECK-NEXT: return $pop1{{$}}
 define i32 @add_sext_i16_i32(i16* %p, i32 %v) {
@@ -300,7 +300,7 @@ define i32 @add_sext_i16_i32(i16* %p, i32 %v) {
 
 ; CHECK-LABEL: add_sext_i8_i64:
 ; CHECK-NEXT: .functype add_sext_i8_i64 (i32, i64) -> (i64){{$}}
-; CHECK: i64.atomic.rmw8_u.add $push0=, 0($0), $1{{$}}
+; CHECK: i64.atomic.rmw8.add_u $push0=, 0($0), $1{{$}}
 ; CHECK-NEXT: i64.extend8_s $push1=, $pop0{{$}}
 ; CHECK-NEXT: return $pop1{{$}}
 define i64 @add_sext_i8_i64(i8* %p, i64 %v) {
@@ -312,7 +312,7 @@ define i64 @add_sext_i8_i64(i8* %p, i64 %v) {
 
 ; CHECK-LABEL: add_sext_i16_i64:
 ; CHECK-NEXT: .functype add_sext_i16_i64 (i32, i64) -> (i64){{$}}
-; CHECK: i64.atomic.rmw16_u.add $push0=, 0($0), $1{{$}}
+; CHECK: i64.atomic.rmw16.add_u $push0=, 0($0), $1{{$}}
 ; CHECK-NEXT: i64.extend16_s $push1=, $pop0{{$}}
 ; CHECK-NEXT: return $pop1{{$}}
 define i64 @add_sext_i16_i64(i16* %p, i64 %v) {
@@ -322,12 +322,12 @@ define i64 @add_sext_i16_i64(i16* %p, i64 %v) {
   ret i64 %e
 }
 
-; 32->64 sext rmw gets selected as i32.atomic.rmw.add, i64_extend_s/i32
+; 32->64 sext rmw gets selected as i32.atomic.rmw.add, i64.extend_i32_s
 ; CHECK-LABEL: add_sext_i32_i64:
 ; CHECK-NEXT: .functype add_sext_i32_i64 (i32, i64) -> (i64){{$}}
-; CHECK: i32.wrap/i64 $push0=, $1{{$}}
+; CHECK: i32.wrap_i64 $push0=, $1{{$}}
 ; CHECK: i32.atomic.rmw.add $push1=, 0($0), $pop0{{$}}
-; CHECK-NEXT: i64.extend_s/i32 $push2=, $pop1{{$}}
+; CHECK-NEXT: i64.extend_i32_s $push2=, $pop1{{$}}
 ; CHECK-NEXT: return $pop2{{$}}
 define i64 @add_sext_i32_i64(i32* %p, i64 %v) {
   %t = trunc i64 %v to i32
@@ -340,7 +340,7 @@ define i64 @add_sext_i32_i64(i32* %p, i64 %v) {
 
 ; CHECK-LABEL: sub_sext_i8_i32:
 ; CHECK-NEXT: .functype sub_sext_i8_i32 (i32, i32) -> (i32){{$}}
-; CHECK: i32.atomic.rmw8_u.sub $push0=, 0($0), $1{{$}}
+; CHECK: i32.atomic.rmw8.sub_u $push0=, 0($0), $1{{$}}
 ; CHECK-NEXT: i32.extend8_s $push1=, $pop0{{$}}
 ; CHECK-NEXT: return $pop1{{$}}
 define i32 @sub_sext_i8_i32(i8* %p, i32 %v) {
@@ -352,7 +352,7 @@ define i32 @sub_sext_i8_i32(i8* %p, i32 %v) {
 
 ; CHECK-LABEL: sub_sext_i16_i32:
 ; CHECK-NEXT: .functype sub_sext_i16_i32 (i32, i32) -> (i32){{$}}
-; CHECK: i32.atomic.rmw16_u.sub $push0=, 0($0), $1{{$}}
+; CHECK: i32.atomic.rmw16.sub_u $push0=, 0($0), $1{{$}}
 ; CHECK-NEXT: i32.extend16_s $push1=, $pop0{{$}}
 ; CHECK-NEXT: return $pop1{{$}}
 define i32 @sub_sext_i16_i32(i16* %p, i32 %v) {
@@ -364,7 +364,7 @@ define i32 @sub_sext_i16_i32(i16* %p, i32 %v) {
 
 ; CHECK-LABEL: sub_sext_i8_i64:
 ; CHECK-NEXT: .functype sub_sext_i8_i64 (i32, i64) -> (i64){{$}}
-; CHECK: i64.atomic.rmw8_u.sub $push0=, 0($0), $1{{$}}
+; CHECK: i64.atomic.rmw8.sub_u $push0=, 0($0), $1{{$}}
 ; CHECK-NEXT: i64.extend8_s $push1=, $pop0{{$}}
 ; CHECK-NEXT: return $pop1{{$}}
 define i64 @sub_sext_i8_i64(i8* %p, i64 %v) {
@@ -376,7 +376,7 @@ define i64 @sub_sext_i8_i64(i8* %p, i64 %v) {
 
 ; CHECK-LABEL: sub_sext_i16_i64:
 ; CHECK-NEXT: .functype sub_sext_i16_i64 (i32, i64) -> (i64){{$}}
-; CHECK: i64.atomic.rmw16_u.sub $push0=, 0($0), $1{{$}}
+; CHECK: i64.atomic.rmw16.sub_u $push0=, 0($0), $1{{$}}
 ; CHECK-NEXT: i64.extend16_s $push1=, $pop0{{$}}
 ; CHECK-NEXT: return $pop1{{$}}
 define i64 @sub_sext_i16_i64(i16* %p, i64 %v) {
@@ -386,12 +386,12 @@ define i64 @sub_sext_i16_i64(i16* %p, i64 %v) {
   ret i64 %e
 }
 
-; 32->64 sext rmw gets selected as i32.atomic.rmw.sub, i64_extend_s/i32
+; 32->64 sext rmw gets selected as i32.atomic.rmw.sub, i64.extend_i32_s
 ; CHECK-LABEL: sub_sext_i32_i64:
 ; CHECK-NEXT: .functype sub_sext_i32_i64 (i32, i64) -> (i64){{$}}
-; CHECK: i32.wrap/i64 $push0=, $1
+; CHECK: i32.wrap_i64 $push0=, $1
 ; CHECK: i32.atomic.rmw.sub $push1=, 0($0), $pop0{{$}}
-; CHECK-NEXT: i64.extend_s/i32 $push2=, $pop1{{$}}
+; CHECK-NEXT: i64.extend_i32_s $push2=, $pop1{{$}}
 ; CHECK-NEXT: return $pop2{{$}}
 define i64 @sub_sext_i32_i64(i32* %p, i64 %v) {
   %t = trunc i64 %v to i32
@@ -404,7 +404,7 @@ define i64 @sub_sext_i32_i64(i32* %p, i64 %v) {
 
 ; CHECK-LABEL: and_sext_i8_i32:
 ; CHECK-NEXT: .functype and_sext_i8_i32 (i32, i32) -> (i32){{$}}
-; CHECK: i32.atomic.rmw8_u.and $push0=, 0($0), $1{{$}}
+; CHECK: i32.atomic.rmw8.and_u $push0=, 0($0), $1{{$}}
 ; CHECK-NEXT: i32.extend8_s $push1=, $pop0{{$}}
 ; CHECK-NEXT: return $pop1{{$}}
 define i32 @and_sext_i8_i32(i8* %p, i32 %v) {
@@ -416,7 +416,7 @@ define i32 @and_sext_i8_i32(i8* %p, i32 %v) {
 
 ; CHECK-LABEL: and_sext_i16_i32:
 ; CHECK-NEXT: .functype and_sext_i16_i32 (i32, i32) -> (i32){{$}}
-; CHECK: i32.atomic.rmw16_u.and $push0=, 0($0), $1{{$}}
+; CHECK: i32.atomic.rmw16.and_u $push0=, 0($0), $1{{$}}
 ; CHECK-NEXT: i32.extend16_s $push1=, $pop0{{$}}
 ; CHECK-NEXT: return $pop1{{$}}
 define i32 @and_sext_i16_i32(i16* %p, i32 %v) {
@@ -428,7 +428,7 @@ define i32 @and_sext_i16_i32(i16* %p, i32 %v) {
 
 ; CHECK-LABEL: and_sext_i8_i64:
 ; CHECK-NEXT: .functype and_sext_i8_i64 (i32, i64) -> (i64){{$}}
-; CHECK: i64.atomic.rmw8_u.and $push0=, 0($0), $1{{$}}
+; CHECK: i64.atomic.rmw8.and_u $push0=, 0($0), $1{{$}}
 ; CHECK-NEXT: i64.extend8_s $push1=, $pop0{{$}}
 ; CHECK-NEXT: return $pop1{{$}}
 define i64 @and_sext_i8_i64(i8* %p, i64 %v) {
@@ -440,7 +440,7 @@ define i64 @and_sext_i8_i64(i8* %p, i64 %v) {
 
 ; CHECK-LABEL: and_sext_i16_i64:
 ; CHECK-NEXT: .functype and_sext_i16_i64 (i32, i64) -> (i64){{$}}
-; CHECK: i64.atomic.rmw16_u.and $push0=, 0($0), $1{{$}}
+; CHECK: i64.atomic.rmw16.and_u $push0=, 0($0), $1{{$}}
 ; CHECK-NEXT: i64.extend16_s $push1=, $pop0{{$}}
 ; CHECK-NEXT: return $pop1{{$}}
 define i64 @and_sext_i16_i64(i16* %p, i64 %v) {
@@ -450,12 +450,12 @@ define i64 @and_sext_i16_i64(i16* %p, i64 %v) {
   ret i64 %e
 }
 
-; 32->64 sext rmw gets selected as i32.atomic.rmw.and, i64_extend_s/i32
+; 32->64 sext rmw gets selected as i32.atomic.rmw.and, i64.extend_i32_s
 ; CHECK-LABEL: and_sext_i32_i64:
 ; CHECK-NEXT: .functype and_sext_i32_i64 (i32, i64) -> (i64){{$}}
-; CHECK: i32.wrap/i64 $push0=, $1{{$}}
+; CHECK: i32.wrap_i64 $push0=, $1{{$}}
 ; CHECK: i32.atomic.rmw.and $push1=, 0($0), $pop0{{$}}
-; CHECK-NEXT: i64.extend_s/i32 $push2=, $pop1{{$}}
+; CHECK-NEXT: i64.extend_i32_s $push2=, $pop1{{$}}
 ; CHECK-NEXT: return $pop2{{$}}
 define i64 @and_sext_i32_i64(i32* %p, i64 %v) {
   %t = trunc i64 %v to i32
@@ -468,7 +468,7 @@ define i64 @and_sext_i32_i64(i32* %p, i64 %v) {
 
 ; CHECK-LABEL: or_sext_i8_i32:
 ; CHECK-NEXT: .functype or_sext_i8_i32 (i32, i32) -> (i32){{$}}
-; CHECK: i32.atomic.rmw8_u.or $push0=, 0($0), $1{{$}}
+; CHECK: i32.atomic.rmw8.or_u $push0=, 0($0), $1{{$}}
 ; CHECK-NEXT: i32.extend8_s $push1=, $pop0{{$}}
 ; CHECK-NEXT: return $pop1{{$}}
 define i32 @or_sext_i8_i32(i8* %p, i32 %v) {
@@ -480,7 +480,7 @@ define i32 @or_sext_i8_i32(i8* %p, i32 %v) {
 
 ; CHECK-LABEL: or_sext_i16_i32:
 ; CHECK-NEXT: .functype or_sext_i16_i32 (i32, i32) -> (i32){{$}}
-; CHECK: i32.atomic.rmw16_u.or $push0=, 0($0), $1{{$}}
+; CHECK: i32.atomic.rmw16.or_u $push0=, 0($0), $1{{$}}
 ; CHECK-NEXT: i32.extend16_s $push1=, $pop0{{$}}
 ; CHECK-NEXT: return $pop1{{$}}
 define i32 @or_sext_i16_i32(i16* %p, i32 %v) {
@@ -492,7 +492,7 @@ define i32 @or_sext_i16_i32(i16* %p, i32 %v) {
 
 ; CHECK-LABEL: or_sext_i8_i64:
 ; CHECK-NEXT: .functype or_sext_i8_i64 (i32, i64) -> (i64){{$}}
-; CHECK: i64.atomic.rmw8_u.or $push0=, 0($0), $1{{$}}
+; CHECK: i64.atomic.rmw8.or_u $push0=, 0($0), $1{{$}}
 ; CHECK-NEXT: i64.extend8_s $push1=, $pop0{{$}}
 ; CHECK-NEXT: return $pop1{{$}}
 define i64 @or_sext_i8_i64(i8* %p, i64 %v) {
@@ -504,7 +504,7 @@ define i64 @or_sext_i8_i64(i8* %p, i64 %v) {
 
 ; CHECK-LABEL: or_sext_i16_i64:
 ; CHECK-NEXT: .functype or_sext_i16_i64 (i32, i64) -> (i64){{$}}
-; CHECK: i64.atomic.rmw16_u.or $push0=, 0($0), $1{{$}}
+; CHECK: i64.atomic.rmw16.or_u $push0=, 0($0), $1{{$}}
 ; CHECK-NEXT: i64.extend16_s $push1=, $pop0{{$}}
 ; CHECK-NEXT: return $pop1{{$}}
 define i64 @or_sext_i16_i64(i16* %p, i64 %v) {
@@ -514,12 +514,12 @@ define i64 @or_sext_i16_i64(i16* %p, i64 %v) {
   ret i64 %e
 }
 
-; 32->64 sext rmw gets selected as i32.atomic.rmw.or, i64_extend_s/i32
+; 32->64 sext rmw gets selected as i32.atomic.rmw.or, i64.extend_i32_s
 ; CHECK-LABEL: or_sext_i32_i64:
 ; CHECK-NEXT: .functype or_sext_i32_i64 (i32, i64) -> (i64){{$}}
-; CHECK: i32.wrap/i64 $push0=, $1{{$}}
+; CHECK: i32.wrap_i64 $push0=, $1{{$}}
 ; CHECK: i32.atomic.rmw.or $push1=, 0($0), $pop0{{$}}
-; CHECK-NEXT: i64.extend_s/i32 $push2=, $pop1{{$}}
+; CHECK-NEXT: i64.extend_i32_s $push2=, $pop1{{$}}
 ; CHECK-NEXT: return $pop2{{$}}
 define i64 @or_sext_i32_i64(i32* %p, i64 %v) {
   %t = trunc i64 %v to i32
@@ -532,7 +532,7 @@ define i64 @or_sext_i32_i64(i32* %p, i64 %v) {
 
 ; CHECK-LABEL: xor_sext_i8_i32:
 ; CHECK-NEXT: .functype xor_sext_i8_i32 (i32, i32) -> (i32){{$}}
-; CHECK: i32.atomic.rmw8_u.xor $push0=, 0($0), $1{{$}}
+; CHECK: i32.atomic.rmw8.xor_u $push0=, 0($0), $1{{$}}
 ; CHECK-NEXT: i32.extend8_s $push1=, $pop0{{$}}
 ; CHECK-NEXT: return $pop1{{$}}
 define i32 @xor_sext_i8_i32(i8* %p, i32 %v) {
@@ -544,7 +544,7 @@ define i32 @xor_sext_i8_i32(i8* %p, i32 %v) {
 
 ; CHECK-LABEL: xor_sext_i16_i32:
 ; CHECK-NEXT: .functype xor_sext_i16_i32 (i32, i32) -> (i32){{$}}
-; CHECK: i32.atomic.rmw16_u.xor $push0=, 0($0), $1{{$}}
+; CHECK: i32.atomic.rmw16.xor_u $push0=, 0($0), $1{{$}}
 ; CHECK-NEXT: i32.extend16_s $push1=, $pop0{{$}}
 ; CHECK-NEXT: return $pop1{{$}}
 define i32 @xor_sext_i16_i32(i16* %p, i32 %v) {
@@ -556,7 +556,7 @@ define i32 @xor_sext_i16_i32(i16* %p, i32 %v) {
 
 ; CHECK-LABEL: xor_sext_i8_i64:
 ; CHECK-NEXT: .functype xor_sext_i8_i64 (i32, i64) -> (i64){{$}}
-; CHECK: i64.atomic.rmw8_u.xor $push0=, 0($0), $1{{$}}
+; CHECK: i64.atomic.rmw8.xor_u $push0=, 0($0), $1{{$}}
 ; CHECK-NEXT: i64.extend8_s $push1=, $pop0{{$}}
 ; CHECK-NEXT: return $pop1{{$}}
 define i64 @xor_sext_i8_i64(i8* %p, i64 %v) {
@@ -568,7 +568,7 @@ define i64 @xor_sext_i8_i64(i8* %p, i64 %v) {
 
 ; CHECK-LABEL: xor_sext_i16_i64:
 ; CHECK-NEXT: .functype xor_sext_i16_i64 (i32, i64) -> (i64){{$}}
-; CHECK: i64.atomic.rmw16_u.xor $push0=, 0($0), $1{{$}}
+; CHECK: i64.atomic.rmw16.xor_u $push0=, 0($0), $1{{$}}
 ; CHECK-NEXT: i64.extend16_s $push1=, $pop0{{$}}
 ; CHECK-NEXT: return $pop1{{$}}
 define i64 @xor_sext_i16_i64(i16* %p, i64 %v) {
@@ -578,12 +578,12 @@ define i64 @xor_sext_i16_i64(i16* %p, i64 %v) {
   ret i64 %e
 }
 
-; 32->64 sext rmw gets selected as i32.atomic.rmw.xor, i64_extend_s/i32
+; 32->64 sext rmw gets selected as i32.atomic.rmw.xor, i64.extend_i32_s
 ; CHECK-LABEL: xor_sext_i32_i64:
 ; CHECK-NEXT: .functype xor_sext_i32_i64 (i32, i64) -> (i64){{$}}
-; CHECK: i32.wrap/i64 $push0=, $1{{$}}
+; CHECK: i32.wrap_i64 $push0=, $1{{$}}
 ; CHECK: i32.atomic.rmw.xor $push1=, 0($0), $pop0{{$}}
-; CHECK-NEXT: i64.extend_s/i32 $push2=, $pop1{{$}}
+; CHECK-NEXT: i64.extend_i32_s $push2=, $pop1{{$}}
 ; CHECK-NEXT: return $pop2{{$}}
 define i64 @xor_sext_i32_i64(i32* %p, i64 %v) {
   %t = trunc i64 %v to i32
@@ -596,7 +596,7 @@ define i64 @xor_sext_i32_i64(i32* %p, i64 %v) {
 
 ; CHECK-LABEL: xchg_sext_i8_i32:
 ; CHECK-NEXT: .functype xchg_sext_i8_i32 (i32, i32) -> (i32){{$}}
-; CHECK: i32.atomic.rmw8_u.xchg $push0=, 0($0), $1{{$}}
+; CHECK: i32.atomic.rmw8.xchg_u $push0=, 0($0), $1{{$}}
 ; CHECK-NEXT: i32.extend8_s $push1=, $pop0{{$}}
 ; CHECK-NEXT: return $pop1{{$}}
 define i32 @xchg_sext_i8_i32(i8* %p, i32 %v) {
@@ -608,7 +608,7 @@ define i32 @xchg_sext_i8_i32(i8* %p, i32 %v) {
 
 ; CHECK-LABEL: xchg_sext_i16_i32:
 ; CHECK-NEXT: .functype xchg_sext_i16_i32 (i32, i32) -> (i32){{$}}
-; CHECK: i32.atomic.rmw16_u.xchg $push0=, 0($0), $1{{$}}
+; CHECK: i32.atomic.rmw16.xchg_u $push0=, 0($0), $1{{$}}
 ; CHECK-NEXT: i32.extend16_s $push1=, $pop0{{$}}
 ; CHECK-NEXT: return $pop1{{$}}
 define i32 @xchg_sext_i16_i32(i16* %p, i32 %v) {
@@ -620,7 +620,7 @@ define i32 @xchg_sext_i16_i32(i16* %p, i32 %v) {
 
 ; CHECK-LABEL: xchg_sext_i8_i64:
 ; CHECK-NEXT: .functype xchg_sext_i8_i64 (i32, i64) -> (i64){{$}}
-; CHECK: i64.atomic.rmw8_u.xchg $push0=, 0($0), $1{{$}}
+; CHECK: i64.atomic.rmw8.xchg_u $push0=, 0($0), $1{{$}}
 ; CHECK-NEXT: i64.extend8_s $push1=, $pop0{{$}}
 ; CHECK-NEXT: return $pop1{{$}}
 define i64 @xchg_sext_i8_i64(i8* %p, i64 %v) {
@@ -632,7 +632,7 @@ define i64 @xchg_sext_i8_i64(i8* %p, i64 %v) {
 
 ; CHECK-LABEL: xchg_sext_i16_i64:
 ; CHECK-NEXT: .functype xchg_sext_i16_i64 (i32, i64) -> (i64){{$}}
-; CHECK: i64.atomic.rmw16_u.xchg $push0=, 0($0), $1{{$}}
+; CHECK: i64.atomic.rmw16.xchg_u $push0=, 0($0), $1{{$}}
 ; CHECK-NEXT: i64.extend16_s $push1=, $pop0{{$}}
 ; CHECK-NEXT: return $pop1{{$}}
 define i64 @xchg_sext_i16_i64(i16* %p, i64 %v) {
@@ -642,12 +642,12 @@ define i64 @xchg_sext_i16_i64(i16* %p, i64 %v) {
   ret i64 %e
 }
 
-; 32->64 sext rmw gets selected as i32.atomic.rmw.xchg, i64_extend_s/i32
+; 32->64 sext rmw gets selected as i32.atomic.rmw.xchg, i64.extend_i32_s
 ; CHECK-LABEL: xchg_sext_i32_i64:
 ; CHECK-NEXT: .functype xchg_sext_i32_i64 (i32, i64) -> (i64){{$}}
-; CHECK: i32.wrap/i64 $push0=, $1{{$}}
+; CHECK: i32.wrap_i64 $push0=, $1{{$}}
 ; CHECK: i32.atomic.rmw.xchg $push1=, 0($0), $pop0{{$}}
-; CHECK-NEXT: i64.extend_s/i32 $push2=, $pop1{{$}}
+; CHECK-NEXT: i64.extend_i32_s $push2=, $pop1{{$}}
 ; CHECK-NEXT: return $pop2{{$}}
 define i64 @xchg_sext_i32_i64(i32* %p, i64 %v) {
   %t = trunc i64 %v to i32
@@ -660,7 +660,7 @@ define i64 @xchg_sext_i32_i64(i32* %p, i64 %v) {
 
 ; CHECK-LABEL: cmpxchg_sext_i8_i32:
 ; CHECK-NEXT: .functype cmpxchg_sext_i8_i32 (i32, i32, i32) -> (i32){{$}}
-; CHECK: i32.atomic.rmw8_u.cmpxchg $push0=, 0($0), $1, $2{{$}}
+; CHECK: i32.atomic.rmw8.cmpxchg_u $push0=, 0($0), $1, $2{{$}}
 ; CHECK-NEXT: i32.extend8_s $push1=, $pop0{{$}}
 ; CHECK-NEXT: return $pop1{{$}}
 define i32 @cmpxchg_sext_i8_i32(i8* %p, i32 %exp, i32 %new) {
@@ -674,7 +674,7 @@ define i32 @cmpxchg_sext_i8_i32(i8* %p, i32 %exp, i32 %new) {
 
 ; CHECK-LABEL: cmpxchg_sext_i16_i32:
 ; CHECK-NEXT: .functype cmpxchg_sext_i16_i32 (i32, i32, i32) -> (i32){{$}}
-; CHECK: i32.atomic.rmw16_u.cmpxchg $push0=, 0($0), $1, $2{{$}}
+; CHECK: i32.atomic.rmw16.cmpxchg_u $push0=, 0($0), $1, $2{{$}}
 ; CHECK-NEXT: i32.extend16_s $push1=, $pop0{{$}}
 ; CHECK-NEXT: return $pop1{{$}}
 define i32 @cmpxchg_sext_i16_i32(i16* %p, i32 %exp, i32 %new) {
@@ -688,7 +688,7 @@ define i32 @cmpxchg_sext_i16_i32(i16* %p, i32 %exp, i32 %new) {
 
 ; CHECK-LABEL: cmpxchg_sext_i8_i64:
 ; CHECK-NEXT: .functype cmpxchg_sext_i8_i64 (i32, i64, i64) -> (i64){{$}}
-; CHECK: i64.atomic.rmw8_u.cmpxchg $push0=, 0($0), $1, $2{{$}}
+; CHECK: i64.atomic.rmw8.cmpxchg_u $push0=, 0($0), $1, $2{{$}}
 ; CHECK-NEXT: i64.extend8_s $push1=, $pop0{{$}}
 ; CHECK-NEXT: return $pop1{{$}}
 define i64 @cmpxchg_sext_i8_i64(i8* %p, i64 %exp, i64 %new) {
@@ -702,7 +702,7 @@ define i64 @cmpxchg_sext_i8_i64(i8* %p, i64 %exp, i64 %new) {
 
 ; CHECK-LABEL: cmpxchg_sext_i16_i64:
 ; CHECK-NEXT: .functype cmpxchg_sext_i16_i64 (i32, i64, i64) -> (i64){{$}}
-; CHECK: i64.atomic.rmw16_u.cmpxchg $push0=, 0($0), $1, $2{{$}}
+; CHECK: i64.atomic.rmw16.cmpxchg_u $push0=, 0($0), $1, $2{{$}}
 ; CHECK-NEXT: i64.extend16_s $push1=, $pop0{{$}}
 ; CHECK-NEXT: return $pop1{{$}}
 define i64 @cmpxchg_sext_i16_i64(i16* %p, i64 %exp, i64 %new) {
@@ -714,13 +714,13 @@ define i64 @cmpxchg_sext_i16_i64(i16* %p, i64 %exp, i64 %new) {
   ret i64 %e
 }
 
-; 32->64 sext rmw gets selected as i32.atomic.rmw.cmpxchg, i64_extend_s/i32
+; 32->64 sext rmw gets selected as i32.atomic.rmw.cmpxchg, i64.extend_i32_s
 ; CHECK-LABEL: cmpxchg_sext_i32_i64:
 ; CHECK-NEXT: .functype cmpxchg_sext_i32_i64 (i32, i64, i64) -> (i64){{$}}
-; CHECK: i32.wrap/i64 $push1=, $1{{$}}
-; CHECK-NEXT: i32.wrap/i64 $push0=, $2{{$}}
+; CHECK: i32.wrap_i64 $push1=, $1{{$}}
+; CHECK-NEXT: i32.wrap_i64 $push0=, $2{{$}}
 ; CHECK-NEXT: i32.atomic.rmw.cmpxchg $push2=, 0($0), $pop1, $pop0{{$}}
-; CHECK-NEXT: i64.extend_s/i32 $push3=, $pop2{{$}}
+; CHECK-NEXT: i64.extend_i32_s $push3=, $pop2{{$}}
 ; CHECK-NEXT: return $pop3{{$}}
 define i64 @cmpxchg_sext_i32_i64(i32* %p, i64 %exp, i64 %new) {
   %exp_t = trunc i64 %exp to i32
@@ -739,7 +739,7 @@ define i64 @cmpxchg_sext_i32_i64(i32* %p, i64 %exp, i64 %new) {
 ; CHECK-LABEL: nand_sext_i8_i32:
 ; CHECK-NEXT: .functype nand_sext_i8_i32 (i32, i32) -> (i32){{$}}
 ; CHECK: loop
-; CHECK: i32.atomic.rmw8_u.cmpxchg
+; CHECK: i32.atomic.rmw8.cmpxchg_u
 ; CHECK: i32.extend8_s
 define i32 @nand_sext_i8_i32(i8* %p, i32 %v) {
   %t = trunc i32 %v to i8
@@ -751,7 +751,7 @@ define i32 @nand_sext_i8_i32(i8* %p, i32 %v) {
 ; CHECK-LABEL: nand_sext_i16_i32:
 ; CHECK-NEXT: .functype nand_sext_i16_i32 (i32, i32) -> (i32){{$}}
 ; CHECK: loop
-; CHECK: i32.atomic.rmw16_u.cmpxchg
+; CHECK: i32.atomic.rmw16.cmpxchg_u
 ; CHECK: i32.extend16_s
 define i32 @nand_sext_i16_i32(i16* %p, i32 %v) {
   %t = trunc i32 %v to i16
@@ -760,12 +760,12 @@ define i32 @nand_sext_i16_i32(i16* %p, i32 %v) {
   ret i32 %e
 }
 
-; FIXME Currently this cannot make use of i64.atomic.rmw8_u.cmpxchg
+; FIXME Currently this cannot make use of i64.atomic.rmw8.cmpxchg_u
 ; CHECK-LABEL: nand_sext_i8_i64:
 ; CHECK-NEXT: .functype nand_sext_i8_i64 (i32, i64) -> (i64){{$}}
 ; CHECK: loop
-; CHECK: i32.atomic.rmw8_u.cmpxchg
-; CHECK: i64.extend_u/i32
+; CHECK: i32.atomic.rmw8.cmpxchg_u
+; CHECK: i64.extend_i32_u
 ; CHECK: i64.extend8_s
 define i64 @nand_sext_i8_i64(i8* %p, i64 %v) {
   %t = trunc i64 %v to i8
@@ -774,12 +774,12 @@ define i64 @nand_sext_i8_i64(i8* %p, i64 %v) {
   ret i64 %e
 }
 
-; FIXME Currently this cannot make use of i64.atomic.rmw16_u.cmpxchg
+; FIXME Currently this cannot make use of i64.atomic.rmw16.cmpxchg_u
 ; CHECK-LABEL: nand_sext_i16_i64:
 ; CHECK-NEXT: .functype nand_sext_i16_i64 (i32, i64) -> (i64){{$}}
 ; CHECK: loop
-; CHECK: i32.atomic.rmw16_u.cmpxchg
-; CHECK: i64.extend_u/i32
+; CHECK: i32.atomic.rmw16.cmpxchg_u
+; CHECK: i64.extend_i32_u
 ; CHECK: i64.extend16_s
 define i64 @nand_sext_i16_i64(i16* %p, i64 %v) {
   %t = trunc i64 %v to i16
@@ -788,12 +788,12 @@ define i64 @nand_sext_i16_i64(i16* %p, i64 %v) {
   ret i64 %e
 }
 
-; 32->64 sext rmw gets selected as i32.atomic.rmw.nand, i64_extend_s/i32
+; 32->64 sext rmw gets selected as i32.atomic.rmw.nand, i64.extend_i32_s
 ; CHECK-LABEL: nand_sext_i32_i64:
 ; CHECK-NEXT: .functype nand_sext_i32_i64 (i32, i64) -> (i64){{$}}
 ; CHECK: loop
 ; CHECK: i32.atomic.rmw.cmpxchg
-; CHECK: i64.extend_s/i32
+; CHECK: i64.extend_i32_s
 define i64 @nand_sext_i32_i64(i32* %p, i64 %v) {
   %t = trunc i64 %v to i32
   %old = atomicrmw nand i32* %p, i32 %t seq_cst
@@ -809,7 +809,7 @@ define i64 @nand_sext_i32_i64(i32* %p, i64 %v) {
 
 ; CHECK-LABEL: add_zext_i8_i32:
 ; CHECK-NEXT: .functype add_zext_i8_i32 (i32, i32) -> (i32){{$}}
-; CHECK: i32.atomic.rmw8_u.add $push0=, 0($0), $1{{$}}
+; CHECK: i32.atomic.rmw8.add_u $push0=, 0($0), $1{{$}}
 ; CHECK-NEXT: return $pop0{{$}}
 define i32 @add_zext_i8_i32(i8* %p, i32 %v) {
   %t = trunc i32 %v to i8
@@ -820,7 +820,7 @@ define i32 @add_zext_i8_i32(i8* %p, i32 %v) {
 
 ; CHECK-LABEL: add_zext_i16_i32:
 ; CHECK-NEXT: .functype add_zext_i16_i32 (i32, i32) -> (i32){{$}}
-; CHECK: i32.atomic.rmw16_u.add $push0=, 0($0), $1{{$}}
+; CHECK: i32.atomic.rmw16.add_u $push0=, 0($0), $1{{$}}
 ; CHECK-NEXT: return $pop0{{$}}
 define i32 @add_zext_i16_i32(i16* %p, i32 %v) {
   %t = trunc i32 %v to i16
@@ -831,7 +831,7 @@ define i32 @add_zext_i16_i32(i16* %p, i32 %v) {
 
 ; CHECK-LABEL: add_zext_i8_i64:
 ; CHECK-NEXT: .functype add_zext_i8_i64 (i32, i64) -> (i64){{$}}
-; CHECK: i64.atomic.rmw8_u.add $push0=, 0($0), $1{{$}}
+; CHECK: i64.atomic.rmw8.add_u $push0=, 0($0), $1{{$}}
 ; CHECK-NEXT: return $pop0{{$}}
 define i64 @add_zext_i8_i64(i8* %p, i64 %v) {
   %t = trunc i64 %v to i8
@@ -842,7 +842,7 @@ define i64 @add_zext_i8_i64(i8* %p, i64 %v) {
 
 ; CHECK-LABEL: add_zext_i16_i64:
 ; CHECK-NEXT: .functype add_zext_i16_i64 (i32, i64) -> (i64){{$}}
-; CHECK: i64.atomic.rmw16_u.add $push0=, 0($0), $1{{$}}
+; CHECK: i64.atomic.rmw16.add_u $push0=, 0($0), $1{{$}}
 ; CHECK-NEXT: return $pop0{{$}}
 define i64 @add_zext_i16_i64(i16* %p, i64 %v) {
   %t = trunc i64 %v to i16
@@ -853,7 +853,7 @@ define i64 @add_zext_i16_i64(i16* %p, i64 %v) {
 
 ; CHECK-LABEL: add_zext_i32_i64:
 ; CHECK-NEXT: .functype add_zext_i32_i64 (i32, i64) -> (i64){{$}}
-; CHECK: i64.atomic.rmw32_u.add $push0=, 0($0), $1{{$}}
+; CHECK: i64.atomic.rmw32.add_u $push0=, 0($0), $1{{$}}
 ; CHECK-NEXT: return $pop0{{$}}
 define i64 @add_zext_i32_i64(i32* %p, i64 %v) {
   %t = trunc i64 %v to i32
@@ -866,7 +866,7 @@ define i64 @add_zext_i32_i64(i32* %p, i64 %v) {
 
 ; CHECK-LABEL: sub_zext_i8_i32:
 ; CHECK-NEXT: .functype sub_zext_i8_i32 (i32, i32) -> (i32){{$}}
-; CHECK: i32.atomic.rmw8_u.sub $push0=, 0($0), $1{{$}}
+; CHECK: i32.atomic.rmw8.sub_u $push0=, 0($0), $1{{$}}
 ; CHECK-NEXT: return $pop0{{$}}
 define i32 @sub_zext_i8_i32(i8* %p, i32 %v) {
   %t = trunc i32 %v to i8
@@ -877,7 +877,7 @@ define i32 @sub_zext_i8_i32(i8* %p, i32 %v) {
 
 ; CHECK-LABEL: sub_zext_i16_i32:
 ; CHECK-NEXT: .functype sub_zext_i16_i32 (i32, i32) -> (i32){{$}}
-; CHECK: i32.atomic.rmw16_u.sub $push0=, 0($0), $1{{$}}
+; CHECK: i32.atomic.rmw16.sub_u $push0=, 0($0), $1{{$}}
 ; CHECK-NEXT: return $pop0{{$}}
 define i32 @sub_zext_i16_i32(i16* %p, i32 %v) {
   %t = trunc i32 %v to i16
@@ -888,7 +888,7 @@ define i32 @sub_zext_i16_i32(i16* %p, i32 %v) {
 
 ; CHECK-LABEL: sub_zext_i8_i64:
 ; CHECK-NEXT: .functype sub_zext_i8_i64 (i32, i64) -> (i64){{$}}
-; CHECK: i64.atomic.rmw8_u.sub $push0=, 0($0), $1{{$}}
+; CHECK: i64.atomic.rmw8.sub_u $push0=, 0($0), $1{{$}}
 ; CHECK-NEXT: return $pop0{{$}}
 define i64 @sub_zext_i8_i64(i8* %p, i64 %v) {
   %t = trunc i64 %v to i8
@@ -899,7 +899,7 @@ define i64 @sub_zext_i8_i64(i8* %p, i64 %v) {
 
 ; CHECK-LABEL: sub_zext_i16_i64:
 ; CHECK-NEXT: .functype sub_zext_i16_i64 (i32, i64) -> (i64){{$}}
-; CHECK: i64.atomic.rmw16_u.sub $push0=, 0($0), $1{{$}}
+; CHECK: i64.atomic.rmw16.sub_u $push0=, 0($0), $1{{$}}
 ; CHECK-NEXT: return $pop0{{$}}
 define i64 @sub_zext_i16_i64(i16* %p, i64 %v) {
   %t = trunc i64 %v to i16
@@ -910,7 +910,7 @@ define i64 @sub_zext_i16_i64(i16* %p, i64 %v) {
 
 ; CHECK-LABEL: sub_zext_i32_i64:
 ; CHECK-NEXT: .functype sub_zext_i32_i64 (i32, i64) -> (i64){{$}}
-; CHECK: i64.atomic.rmw32_u.sub $push0=, 0($0), $1{{$}}
+; CHECK: i64.atomic.rmw32.sub_u $push0=, 0($0), $1{{$}}
 ; CHECK-NEXT: return $pop0{{$}}
 define i64 @sub_zext_i32_i64(i32* %p, i64 %v) {
   %t = trunc i64 %v to i32
@@ -923,7 +923,7 @@ define i64 @sub_zext_i32_i64(i32* %p, i64 %v) {
 
 ; CHECK-LABEL: and_zext_i8_i32:
 ; CHECK-NEXT: .functype and_zext_i8_i32 (i32, i32) -> (i32){{$}}
-; CHECK: i32.atomic.rmw8_u.and $push0=, 0($0), $1{{$}}
+; CHECK: i32.atomic.rmw8.and_u $push0=, 0($0), $1{{$}}
 ; CHECK-NEXT: return $pop0{{$}}
 define i32 @and_zext_i8_i32(i8* %p, i32 %v) {
   %t = trunc i32 %v to i8
@@ -934,7 +934,7 @@ define i32 @and_zext_i8_i32(i8* %p, i32 %v) {
 
 ; CHECK-LABEL: and_zext_i16_i32:
 ; CHECK-NEXT: .functype and_zext_i16_i32 (i32, i32) -> (i32){{$}}
-; CHECK: i32.atomic.rmw16_u.and $push0=, 0($0), $1{{$}}
+; CHECK: i32.atomic.rmw16.and_u $push0=, 0($0), $1{{$}}
 ; CHECK-NEXT: return $pop0{{$}}
 define i32 @and_zext_i16_i32(i16* %p, i32 %v) {
   %t = trunc i32 %v to i16
@@ -945,7 +945,7 @@ define i32 @and_zext_i16_i32(i16* %p, i32 %v) {
 
 ; CHECK-LABEL: and_zext_i8_i64:
 ; CHECK-NEXT: .functype and_zext_i8_i64 (i32, i64) -> (i64){{$}}
-; CHECK: i64.atomic.rmw8_u.and $push0=, 0($0), $1{{$}}
+; CHECK: i64.atomic.rmw8.and_u $push0=, 0($0), $1{{$}}
 ; CHECK-NEXT: return $pop0{{$}}
 define i64 @and_zext_i8_i64(i8* %p, i64 %v) {
   %t = trunc i64 %v to i8
@@ -956,7 +956,7 @@ define i64 @and_zext_i8_i64(i8* %p, i64 %v) {
 
 ; CHECK-LABEL: and_zext_i16_i64:
 ; CHECK-NEXT: .functype and_zext_i16_i64 (i32, i64) -> (i64){{$}}
-; CHECK: i64.atomic.rmw16_u.and $push0=, 0($0), $1{{$}}
+; CHECK: i64.atomic.rmw16.and_u $push0=, 0($0), $1{{$}}
 ; CHECK-NEXT: return $pop0{{$}}
 define i64 @and_zext_i16_i64(i16* %p, i64 %v) {
   %t = trunc i64 %v to i16
@@ -967,7 +967,7 @@ define i64 @and_zext_i16_i64(i16* %p, i64 %v) {
 
 ; CHECK-LABEL: and_zext_i32_i64:
 ; CHECK-NEXT: .functype and_zext_i32_i64 (i32, i64) -> (i64){{$}}
-; CHECK: i64.atomic.rmw32_u.and $push0=, 0($0), $1{{$}}
+; CHECK: i64.atomic.rmw32.and_u $push0=, 0($0), $1{{$}}
 ; CHECK-NEXT: return $pop0{{$}}
 define i64 @and_zext_i32_i64(i32* %p, i64 %v) {
   %t = trunc i64 %v to i32
@@ -980,7 +980,7 @@ define i64 @and_zext_i32_i64(i32* %p, i64 %v) {
 
 ; CHECK-LABEL: or_zext_i8_i32:
 ; CHECK-NEXT: .functype or_zext_i8_i32 (i32, i32) -> (i32){{$}}
-; CHECK: i32.atomic.rmw8_u.or $push0=, 0($0), $1{{$}}
+; CHECK: i32.atomic.rmw8.or_u $push0=, 0($0), $1{{$}}
 ; CHECK-NEXT: return $pop0{{$}}
 define i32 @or_zext_i8_i32(i8* %p, i32 %v) {
   %t = trunc i32 %v to i8
@@ -991,7 +991,7 @@ define i32 @or_zext_i8_i32(i8* %p, i32 %v) {
 
 ; CHECK-LABEL: or_zext_i16_i32:
 ; CHECK-NEXT: .functype or_zext_i16_i32 (i32, i32) -> (i32){{$}}
-; CHECK: i32.atomic.rmw16_u.or $push0=, 0($0), $1{{$}}
+; CHECK: i32.atomic.rmw16.or_u $push0=, 0($0), $1{{$}}
 ; CHECK-NEXT: return $pop0{{$}}
 define i32 @or_zext_i16_i32(i16* %p, i32 %v) {
   %t = trunc i32 %v to i16
@@ -1002,7 +1002,7 @@ define i32 @or_zext_i16_i32(i16* %p, i32 %v) {
 
 ; CHECK-LABEL: or_zext_i8_i64:
 ; CHECK-NEXT: .functype or_zext_i8_i64 (i32, i64) -> (i64){{$}}
-; CHECK: i64.atomic.rmw8_u.or $push0=, 0($0), $1{{$}}
+; CHECK: i64.atomic.rmw8.or_u $push0=, 0($0), $1{{$}}
 ; CHECK-NEXT: return $pop0{{$}}
 define i64 @or_zext_i8_i64(i8* %p, i64 %v) {
   %t = trunc i64 %v to i8
@@ -1013,7 +1013,7 @@ define i64 @or_zext_i8_i64(i8* %p, i64 %v) {
 
 ; CHECK-LABEL: or_zext_i16_i64:
 ; CHECK-NEXT: .functype or_zext_i16_i64 (i32, i64) -> (i64){{$}}
-; CHECK: i64.atomic.rmw16_u.or $push0=, 0($0), $1{{$}}
+; CHECK: i64.atomic.rmw16.or_u $push0=, 0($0), $1{{$}}
 ; CHECK-NEXT: return $pop0{{$}}
 define i64 @or_zext_i16_i64(i16* %p, i64 %v) {
   %t = trunc i64 %v to i16
@@ -1024,7 +1024,7 @@ define i64 @or_zext_i16_i64(i16* %p, i64 %v) {
 
 ; CHECK-LABEL: or_zext_i32_i64:
 ; CHECK-NEXT: .functype or_zext_i32_i64 (i32, i64) -> (i64){{$}}
-; CHECK: i64.atomic.rmw32_u.or $push0=, 0($0), $1{{$}}
+; CHECK: i64.atomic.rmw32.or_u $push0=, 0($0), $1{{$}}
 ; CHECK-NEXT: return $pop0{{$}}
 define i64 @or_zext_i32_i64(i32* %p, i64 %v) {
   %t = trunc i64 %v to i32
@@ -1037,7 +1037,7 @@ define i64 @or_zext_i32_i64(i32* %p, i64 %v) {
 
 ; CHECK-LABEL: xor_zext_i8_i32:
 ; CHECK-NEXT: .functype xor_zext_i8_i32 (i32, i32) -> (i32){{$}}
-; CHECK: i32.atomic.rmw8_u.xor $push0=, 0($0), $1{{$}}
+; CHECK: i32.atomic.rmw8.xor_u $push0=, 0($0), $1{{$}}
 ; CHECK-NEXT: return $pop0{{$}}
 define i32 @xor_zext_i8_i32(i8* %p, i32 %v) {
   %t = trunc i32 %v to i8
@@ -1048,7 +1048,7 @@ define i32 @xor_zext_i8_i32(i8* %p, i32 %v) {
 
 ; CHECK-LABEL: xor_zext_i16_i32:
 ; CHECK-NEXT: .functype xor_zext_i16_i32 (i32, i32) -> (i32){{$}}
-; CHECK: i32.atomic.rmw16_u.xor $push0=, 0($0), $1{{$}}
+; CHECK: i32.atomic.rmw16.xor_u $push0=, 0($0), $1{{$}}
 ; CHECK-NEXT: return $pop0{{$}}
 define i32 @xor_zext_i16_i32(i16* %p, i32 %v) {
   %t = trunc i32 %v to i16
@@ -1059,7 +1059,7 @@ define i32 @xor_zext_i16_i32(i16* %p, i32 %v) {
 
 ; CHECK-LABEL: xor_zext_i8_i64:
 ; CHECK-NEXT: .functype xor_zext_i8_i64 (i32, i64) -> (i64){{$}}
-; CHECK: i64.atomic.rmw8_u.xor $push0=, 0($0), $1{{$}}
+; CHECK: i64.atomic.rmw8.xor_u $push0=, 0($0), $1{{$}}
 ; CHECK-NEXT: return $pop0{{$}}
 define i64 @xor_zext_i8_i64(i8* %p, i64 %v) {
   %t = trunc i64 %v to i8
@@ -1070,7 +1070,7 @@ define i64 @xor_zext_i8_i64(i8* %p, i64 %v) {
 
 ; CHECK-LABEL: xor_zext_i16_i64:
 ; CHECK-NEXT: .functype xor_zext_i16_i64 (i32, i64) -> (i64){{$}}
-; CHECK: i64.atomic.rmw16_u.xor $push0=, 0($0), $1{{$}}
+; CHECK: i64.atomic.rmw16.xor_u $push0=, 0($0), $1{{$}}
 ; CHECK-NEXT: return $pop0{{$}}
 define i64 @xor_zext_i16_i64(i16* %p, i64 %v) {
   %t = trunc i64 %v to i16
@@ -1081,7 +1081,7 @@ define i64 @xor_zext_i16_i64(i16* %p, i64 %v) {
 
 ; CHECK-LABEL: xor_zext_i32_i64:
 ; CHECK-NEXT: .functype xor_zext_i32_i64 (i32, i64) -> (i64){{$}}
-; CHECK: i64.atomic.rmw32_u.xor $push0=, 0($0), $1{{$}}
+; CHECK: i64.atomic.rmw32.xor_u $push0=, 0($0), $1{{$}}
 ; CHECK-NEXT: return $pop0{{$}}
 define i64 @xor_zext_i32_i64(i32* %p, i64 %v) {
   %t = trunc i64 %v to i32
@@ -1094,7 +1094,7 @@ define i64 @xor_zext_i32_i64(i32* %p, i64 %v) {
 
 ; CHECK-LABEL: xchg_zext_i8_i32:
 ; CHECK-NEXT: .functype xchg_zext_i8_i32 (i32, i32) -> (i32){{$}}
-; CHECK: i32.atomic.rmw8_u.xchg $push0=, 0($0), $1{{$}}
+; CHECK: i32.atomic.rmw8.xchg_u $push0=, 0($0), $1{{$}}
 ; CHECK-NEXT: return $pop0{{$}}
 define i32 @xchg_zext_i8_i32(i8* %p, i32 %v) {
   %t = trunc i32 %v to i8
@@ -1105,7 +1105,7 @@ define i32 @xchg_zext_i8_i32(i8* %p, i32 %v) {
 
 ; CHECK-LABEL: xchg_zext_i16_i32:
 ; CHECK-NEXT: .functype xchg_zext_i16_i32 (i32, i32) -> (i32){{$}}
-; CHECK: i32.atomic.rmw16_u.xchg $push0=, 0($0), $1{{$}}
+; CHECK: i32.atomic.rmw16.xchg_u $push0=, 0($0), $1{{$}}
 ; CHECK-NEXT: return $pop0{{$}}
 define i32 @xchg_zext_i16_i32(i16* %p, i32 %v) {
   %t = trunc i32 %v to i16
@@ -1116,7 +1116,7 @@ define i32 @xchg_zext_i16_i32(i16* %p, i32 %v) {
 
 ; CHECK-LABEL: xchg_zext_i8_i64:
 ; CHECK-NEXT: .functype xchg_zext_i8_i64 (i32, i64) -> (i64){{$}}
-; CHECK: i64.atomic.rmw8_u.xchg $push0=, 0($0), $1{{$}}
+; CHECK: i64.atomic.rmw8.xchg_u $push0=, 0($0), $1{{$}}
 ; CHECK-NEXT: return $pop0{{$}}
 define i64 @xchg_zext_i8_i64(i8* %p, i64 %v) {
   %t = trunc i64 %v to i8
@@ -1127,7 +1127,7 @@ define i64 @xchg_zext_i8_i64(i8* %p, i64 %v) {
 
 ; CHECK-LABEL: xchg_zext_i16_i64:
 ; CHECK-NEXT: .functype xchg_zext_i16_i64 (i32, i64) -> (i64){{$}}
-; CHECK: i64.atomic.rmw16_u.xchg $push0=, 0($0), $1{{$}}
+; CHECK: i64.atomic.rmw16.xchg_u $push0=, 0($0), $1{{$}}
 ; CHECK-NEXT: return $pop0{{$}}
 define i64 @xchg_zext_i16_i64(i16* %p, i64 %v) {
   %t = trunc i64 %v to i16
@@ -1138,7 +1138,7 @@ define i64 @xchg_zext_i16_i64(i16* %p, i64 %v) {
 
 ; CHECK-LABEL: xchg_zext_i32_i64:
 ; CHECK-NEXT: .functype xchg_zext_i32_i64 (i32, i64) -> (i64){{$}}
-; CHECK: i64.atomic.rmw32_u.xchg $push0=, 0($0), $1{{$}}
+; CHECK: i64.atomic.rmw32.xchg_u $push0=, 0($0), $1{{$}}
 ; CHECK-NEXT: return $pop0{{$}}
 define i64 @xchg_zext_i32_i64(i32* %p, i64 %v) {
   %t = trunc i64 %v to i32
@@ -1151,7 +1151,7 @@ define i64 @xchg_zext_i32_i64(i32* %p, i64 %v) {
 
 ; CHECK-LABEL: cmpxchg_zext_i8_i32:
 ; CHECK-NEXT: .functype cmpxchg_zext_i8_i32 (i32, i32, i32) -> (i32){{$}}
-; CHECK: i32.atomic.rmw8_u.cmpxchg $push0=, 0($0), $1, $2{{$}}
+; CHECK: i32.atomic.rmw8.cmpxchg_u $push0=, 0($0), $1, $2{{$}}
 ; CHECK-NEXT: return $pop0{{$}}
 define i32 @cmpxchg_zext_i8_i32(i8* %p, i32 %exp, i32 %new) {
   %exp_t = trunc i32 %exp to i8
@@ -1164,7 +1164,7 @@ define i32 @cmpxchg_zext_i8_i32(i8* %p, i32 %exp, i32 %new) {
 
 ; CHECK-LABEL: cmpxchg_zext_i16_i32:
 ; CHECK-NEXT: .functype cmpxchg_zext_i16_i32 (i32, i32, i32) -> (i32){{$}}
-; CHECK: i32.atomic.rmw16_u.cmpxchg $push0=, 0($0), $1, $2{{$}}
+; CHECK: i32.atomic.rmw16.cmpxchg_u $push0=, 0($0), $1, $2{{$}}
 ; CHECK-NEXT: return $pop0{{$}}
 define i32 @cmpxchg_zext_i16_i32(i16* %p, i32 %exp, i32 %new) {
   %exp_t = trunc i32 %exp to i16
@@ -1177,7 +1177,7 @@ define i32 @cmpxchg_zext_i16_i32(i16* %p, i32 %exp, i32 %new) {
 
 ; CHECK-LABEL: cmpxchg_zext_i8_i64:
 ; CHECK-NEXT: .functype cmpxchg_zext_i8_i64 (i32, i64, i64) -> (i64){{$}}
-; CHECK: i64.atomic.rmw8_u.cmpxchg $push0=, 0($0), $1, $2{{$}}
+; CHECK: i64.atomic.rmw8.cmpxchg_u $push0=, 0($0), $1, $2{{$}}
 ; CHECK-NEXT: return $pop0{{$}}
 define i64 @cmpxchg_zext_i8_i64(i8* %p, i64 %exp, i64 %new) {
   %exp_t = trunc i64 %exp to i8
@@ -1190,7 +1190,7 @@ define i64 @cmpxchg_zext_i8_i64(i8* %p, i64 %exp, i64 %new) {
 
 ; CHECK-LABEL: cmpxchg_zext_i16_i64:
 ; CHECK-NEXT: .functype cmpxchg_zext_i16_i64 (i32, i64, i64) -> (i64){{$}}
-; CHECK: i64.atomic.rmw16_u.cmpxchg $push0=, 0($0), $1, $2{{$}}
+; CHECK: i64.atomic.rmw16.cmpxchg_u $push0=, 0($0), $1, $2{{$}}
 ; CHECK-NEXT: return $pop0{{$}}
 define i64 @cmpxchg_zext_i16_i64(i16* %p, i64 %exp, i64 %new) {
   %exp_t = trunc i64 %exp to i16
@@ -1203,7 +1203,7 @@ define i64 @cmpxchg_zext_i16_i64(i16* %p, i64 %exp, i64 %new) {
 
 ; CHECK-LABEL: cmpxchg_zext_i32_i64:
 ; CHECK-NEXT: .functype cmpxchg_zext_i32_i64 (i32, i64, i64) -> (i64){{$}}
-; CHECK: i64.atomic.rmw32_u.cmpxchg $push0=, 0($0), $1, $2{{$}}
+; CHECK: i64.atomic.rmw32.cmpxchg_u $push0=, 0($0), $1, $2{{$}}
 ; CHECK-NEXT: return $pop0{{$}}
 define i64 @cmpxchg_zext_i32_i64(i32* %p, i64 %exp, i64 %new) {
   %exp_t = trunc i64 %exp to i32
@@ -1222,7 +1222,7 @@ define i64 @cmpxchg_zext_i32_i64(i32* %p, i64 %exp, i64 %new) {
 ; CHECK-LABEL: nand_zext_i8_i32:
 ; CHECK-NEXT: .functype nand_zext_i8_i32 (i32, i32) -> (i32){{$}}
 ; CHECK: loop
-; CHECK: i32.atomic.rmw8_u.cmpxchg
+; CHECK: i32.atomic.rmw8.cmpxchg_u
 define i32 @nand_zext_i8_i32(i8* %p, i32 %v) {
   %t = trunc i32 %v to i8
   %old = atomicrmw nand i8* %p, i8 %t seq_cst
@@ -1233,7 +1233,7 @@ define i32 @nand_zext_i8_i32(i8* %p, i32 %v) {
 ; CHECK-LABEL: nand_zext_i16_i32:
 ; CHECK-NEXT: .functype nand_zext_i16_i32 (i32, i32) -> (i32){{$}}
 ; CHECK: loop
-; CHECK: i32.atomic.rmw16_u.cmpxchg
+; CHECK: i32.atomic.rmw16.cmpxchg_u
 define i32 @nand_zext_i16_i32(i16* %p, i32 %v) {
   %t = trunc i32 %v to i16
   %old = atomicrmw nand i16* %p, i16 %t seq_cst
@@ -1241,12 +1241,12 @@ define i32 @nand_zext_i16_i32(i16* %p, i32 %v) {
   ret i32 %e
 }
 
-; FIXME Currently this cannot make use of i64.atomic.rmw8_u.cmpxchg
+; FIXME Currently this cannot make use of i64.atomic.rmw8.cmpxchg_u
 ; CHECK-LABEL: nand_zext_i8_i64:
 ; CHECK-NEXT: .functype nand_zext_i8_i64 (i32, i64) -> (i64){{$}}
 ; CHECK: loop
-; CHECK: i32.atomic.rmw8_u.cmpxchg
-; CHECK: i64.extend_u/i32
+; CHECK: i32.atomic.rmw8.cmpxchg_u
+; CHECK: i64.extend_i32_u
 define i64 @nand_zext_i8_i64(i8* %p, i64 %v) {
   %t = trunc i64 %v to i8
   %old = atomicrmw nand i8* %p, i8 %t seq_cst
@@ -1254,12 +1254,12 @@ define i64 @nand_zext_i8_i64(i8* %p, i64 %v) {
   ret i64 %e
 }
 
-; FIXME Currently this cannot make use of i64.atomic.rmw16_u.cmpxchg
+; FIXME Currently this cannot make use of i64.atomic.rmw16.cmpxchg_u
 ; CHECK-LABEL: nand_zext_i16_i64:
 ; CHECK-NEXT: .functype nand_zext_i16_i64 (i32, i64) -> (i64){{$}}
 ; CHECK: loop
-; CHECK: i32.atomic.rmw16_u.cmpxchg
-; CHECK: i64.extend_u/i32
+; CHECK: i32.atomic.rmw16.cmpxchg_u
+; CHECK: i64.extend_i32_u
 define i64 @nand_zext_i16_i64(i16* %p, i64 %v) {
   %t = trunc i64 %v to i16
   %old = atomicrmw nand i16* %p, i16 %t seq_cst
@@ -1267,12 +1267,12 @@ define i64 @nand_zext_i16_i64(i16* %p, i64 %v) {
   ret i64 %e
 }
 
-; FIXME Currently this cannot make use of i64.atomic.rmw32_u.cmpxchg
+; FIXME Currently this cannot make use of i64.atomic.rmw32.cmpxchg_u
 ; CHECK-LABEL: nand_zext_i32_i64:
 ; CHECK-NEXT: .functype nand_zext_i32_i64 (i32, i64) -> (i64){{$}}
 ; CHECK: loop
 ; CHECK: i32.atomic.rmw.cmpxchg
-; CHECK: i64.extend_u/i32
+; CHECK: i64.extend_i32_u
 define i64 @nand_zext_i32_i64(i32* %p, i64 %v) {
   %t = trunc i64 %v to i32
   %old = atomicrmw nand i32* %p, i32 %t seq_cst
diff --git a/test/CodeGen/WebAssembly/bulk-memory-intrinsics.ll b/test/CodeGen/WebAssembly/bulk-memory-intrinsics.ll
new file mode 100644
index 0000000000000000000000000000000000000000..dfb74b78f64fdb0ca2bb8cc7102a4706d4ee7524
--- /dev/null
+++ b/test/CodeGen/WebAssembly/bulk-memory-intrinsics.ll
@@ -0,0 +1,28 @@
+; RUN: llc < %s -asm-verbose=false -verify-machineinstrs -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-keep-registers -mattr=+bulk-memory | FileCheck %s
+
+; Test that bulk memory intrinsics lower correctly
+
+target datalayout = "e-m:e-p:32:32-i64:64-n32:64-S128"
+target triple = "wasm32-unknown-unknown"
+
+; CHECK-LABEL: memory_init:
+; CHECK-NEXT: .functype  memory_init (i32, i32, i32) -> ()
+; CHECK-NEXT: memory.init 3, 0, $0, $1, $2
+; CHECK-NEXT: return
+declare void @llvm.wasm.memory.init(i32, i32, i8*, i32, i32)
+define void @memory_init(i8* %dest, i32 %offset, i32 %size) {
+  call void @llvm.wasm.memory.init(
+    i32 3, i32 0, i8* %dest, i32 %offset, i32 %size
+  )
+  ret void
+}
+
+; CHECK-LABEL: data_drop:
+; CHECK-NEXT: .functype data_drop () -> ()
+; CHECK-NEXT: data.drop 3
+; CHECK-NEXT: return
+declare void @llvm.wasm.data.drop(i32)
+define void @data_drop() {
+  call void @llvm.wasm.data.drop(i32 3)
+  ret void
+}
diff --git a/test/CodeGen/WebAssembly/bulk-memory.ll b/test/CodeGen/WebAssembly/bulk-memory.ll
new file mode 100644
index 0000000000000000000000000000000000000000..68ce3e40fa3e4ca85b228e6d282737f93c416464
--- /dev/null
+++ b/test/CodeGen/WebAssembly/bulk-memory.ll
@@ -0,0 +1,97 @@
+; RUN: llc < %s -asm-verbose=false -verify-machineinstrs -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-keep-registers -mattr=+bulk-memory | FileCheck %s --check-prefixes CHECK,BULK-MEM
+; RUN: llc < %s -asm-verbose=false -verify-machineinstrs -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-keep-registers -mattr=-bulk-memory | FileCheck %s --check-prefixes CHECK,NO-BULK-MEM
+
+; Test that basic bulk memory codegen works correctly
+
+target datalayout = "e-m:e-p:32:32-i64:64-n32:64-S128"
+target triple = "wasm32-unknown-unknown"
+
+declare void @llvm.memcpy.p0i8.p0i8.i8(i8*, i8*, i8, i1)
+declare void @llvm.memcpy.p0i8.p0i8.i32(i8*, i8*, i32, i1)
+declare void @llvm.memcpy.p0i32.p0i32.i32(i32*, i32*, i32, i1)
+
+declare void @llvm.memmove.p0i8.p0i8.i8(i8*, i8*, i8, i1)
+declare void @llvm.memmove.p0i8.p0i8.i32(i8*, i8*, i32, i1)
+declare void @llvm.memmove.p0i32.p0i32.i32(i32*, i32*, i32, i1)
+
+; CHECK-LABEL: memcpy_i8:
+; NO-BULK-MEM-NOT: memory.copy
+; BULK-MEM-NEXT: .functype memcpy_i8 (i32, i32, i32) -> ()
+; BULK-MEM-NEXT: memory.copy 0, 0, $0, $1, $2
+; BULK-MEM-NEXT: return
+define void @memcpy_i8(i8* %dest, i8* %src, i8 zeroext %len) {
+  call void @llvm.memcpy.p0i8.p0i8.i8(i8* %dest, i8* %src, i8 %len, i1 0)
+  ret void
+}
+
+; CHECK-LABEL: memmove_i8:
+; NO-BULK-MEM-NOT: memory.copy
+; BULK-MEM-NEXT: .functype memmove_i8 (i32, i32, i32) -> ()
+; BULK-MEM-NEXT: memory.copy 0, 0, $0, $1, $2
+; BULK-MEM-NEXT: return
+define void @memmove_i8(i8* %dest, i8* %src, i8 zeroext %len) {
+  call void @llvm.memmove.p0i8.p0i8.i8(i8* %dest, i8* %src, i8 %len, i1 0)
+  ret void
+}
+
+; CHECK-LABEL: memcpy_i32:
+; NO-BULK-MEM-NOT: memory.copy
+; BULK-MEM-NEXT: .functype memcpy_i32 (i32, i32, i32) -> ()
+; BULK-MEM-NEXT: memory.copy 0, 0, $0, $1, $2
+; BULK-MEM-NEXT: return
+define void @memcpy_i32(i32* %dest, i32* %src, i32 %len) {
+  call void @llvm.memcpy.p0i32.p0i32.i32(i32* %dest, i32* %src, i32 %len, i1 0)
+  ret void
+}
+
+; CHECK-LABEL: memmove_i32:
+; NO-BULK-MEM-NOT: memory.copy
+; BULK-MEM-NEXT: .functype memmove_i32 (i32, i32, i32) -> ()
+; BULK-MEM-NEXT: memory.copy 0, 0, $0, $1, $2
+; BULK-MEM-NEXT: return
+define void @memmove_i32(i32* %dest, i32* %src, i32 %len) {
+  call void @llvm.memmove.p0i32.p0i32.i32(i32* %dest, i32* %src, i32 %len, i1 0)
+  ret void
+}
+
+; CHECK-LABEL: memcpy_1:
+; CHECK-NEXT: .functype memcpy_1 (i32, i32) -> ()
+; CHECK-NEXT: i32.load8_u $push[[L0:[0-9]+]]=, 0($1)
+; CHECK-NEXT: i32.store8 0($0), $pop[[L0]]
+; CHECK-NEXT: return
+define void @memcpy_1(i8* %dest, i8* %src) {
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* %dest, i8* %src, i32 1, i1 0)
+  ret void
+}
+
+; CHECK-LABEL: memmove_1:
+; CHECK-NEXT: .functype memmove_1 (i32, i32) -> ()
+; CHECK-NEXT: i32.load8_u $push[[L0:[0-9]+]]=, 0($1)
+; CHECK-NEXT: i32.store8 0($0), $pop[[L0]]
+; CHECK-NEXT: return
+define void @memmove_1(i8* %dest, i8* %src) {
+  call void @llvm.memmove.p0i8.p0i8.i32(i8* %dest, i8* %src, i32 1, i1 0)
+  ret void
+}
+
+; CHECK-LABEL: memcpy_1024:
+; NO-BULK-MEM-NOT: memory.copy
+; BULK-MEM-NEXT: .functype memcpy_1024 (i32, i32) -> ()
+; BULK-MEM-NEXT: i32.const $push[[L0:[0-9]+]]=, 1024
+; BULK-MEM-NEXT: memory.copy 0, 0, $0, $1, $pop[[L0]]
+; BULK-MEM-NEXT: return
+define void @memcpy_1024(i8* %dest, i8* %src) {
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* %dest, i8* %src, i32 1024, i1 0)
+  ret void
+}
+
+; CHECK-LABEL: memmove_1024:
+; NO-BULK-MEM-NOT: memory.copy
+; BULK-MEM-NEXT: .functype memmove_1024 (i32, i32) -> ()
+; BULK-MEM-NEXT: i32.const $push[[L0:[0-9]+]]=, 1024
+; BULK-MEM-NEXT: memory.copy 0, 0, $0, $1, $pop[[L0]]
+; BULK-MEM-NEXT: return
+define void @memmove_1024(i8* %dest, i8* %src) {
+  call void @llvm.memmove.p0i8.p0i8.i32(i8* %dest, i8* %src, i32 1024, i1 0)
+  ret void
+}
diff --git a/test/CodeGen/WebAssembly/byval.ll b/test/CodeGen/WebAssembly/byval.ll
index 0caa72015d7878da604077a0fc1fedba12bf6328..68f86e4814c33443d1a44060d9cddf629b75bca2 100644
--- a/test/CodeGen/WebAssembly/byval.ll
+++ b/test/CodeGen/WebAssembly/byval.ll
@@ -21,12 +21,12 @@ declare void @ext_byval_func_empty(%EmptyStruct* byval)
 define void @byval_arg(%SmallStruct* %ptr) {
  ; CHECK: .functype byval_arg (i32) -> ()
  ; Subtract 16 from SP (SP is 16-byte aligned)
- ; CHECK-NEXT: get_global $push[[L2:.+]]=, __stack_pointer@GLOBAL
+ ; CHECK-NEXT: global.get $push[[L2:.+]]=, __stack_pointer@GLOBAL
  ; CHECK-NEXT: i32.const $push[[L3:.+]]=, 16
  ; CHECK-NEXT: i32.sub $push[[L11:.+]]=, $pop[[L2]], $pop[[L3]]
  ; Ensure SP is stored back before the call
- ; CHECK-NEXT: tee_local $push[[L10:.+]]=, $[[SP:.+]]=, $pop[[L11]]{{$}}
- ; CHECK-NEXT: set_global __stack_pointer@GLOBAL, $pop[[L10]]{{$}}
+ ; CHECK-NEXT: local.tee $push[[L10:.+]]=, $[[SP:.+]]=, $pop[[L11]]{{$}}
+ ; CHECK-NEXT: global.set __stack_pointer@GLOBAL, $pop[[L10]]{{$}}
  ; Copy the SmallStruct argument to the stack (SP+12, original SP-4)
  ; CHECK-NEXT: i32.load $push[[L0:.+]]=, 0($0)
  ; CHECK-NEXT: i32.store 12($[[SP]]), $pop[[L0]]
@@ -38,7 +38,7 @@ define void @byval_arg(%SmallStruct* %ptr) {
  ; Restore the stack
  ; CHECK-NEXT: i32.const $push[[L6:.+]]=, 16
  ; CHECK-NEXT: i32.add $push[[L8:.+]]=, $[[SP]], $pop[[L6]]
- ; CHECK-NEXT: set_global __stack_pointer@GLOBAL, $pop[[L8]]
+ ; CHECK-NEXT: global.set __stack_pointer@GLOBAL, $pop[[L8]]
  ; CHECK-NEXT: return
  ret void
 }
@@ -49,8 +49,8 @@ define void @byval_arg_align8(%SmallStruct* %ptr) {
  ; Don't check the entire SP sequence, just enough to get the alignment.
  ; CHECK: i32.const $push[[L1:.+]]=, 16
  ; CHECK-NEXT: i32.sub $push[[L11:.+]]=, {{.+}}, $pop[[L1]]
- ; CHECK-NEXT: tee_local $push[[L10:.+]]=, $[[SP:.+]]=, $pop[[L11]]{{$}}
- ; CHECK-NEXT: set_global __stack_pointer@GLOBAL, $pop[[L10]]{{$}}
+ ; CHECK-NEXT: local.tee $push[[L10:.+]]=, $[[SP:.+]]=, $pop[[L11]]{{$}}
+ ; CHECK-NEXT: global.set __stack_pointer@GLOBAL, $pop[[L10]]{{$}}
  ; Copy the SmallStruct argument to the stack (SP+8, original SP-8)
  ; CHECK-NEXT: i32.load $push[[L0:.+]]=, 0($0){{$}}
  ; CHECK-NEXT: i32.store 8($[[SP]]), $pop[[L0]]{{$}}
@@ -68,8 +68,8 @@ define void @byval_arg_double(%AlignedStruct* %ptr) {
  ; Subtract 16 from SP (SP is 16-byte aligned)
  ; CHECK: i32.const $push[[L1:.+]]=, 16
  ; CHECK-NEXT: i32.sub $push[[L14:.+]]=, {{.+}}, $pop[[L1]]
- ; CHECK-NEXT: tee_local $push[[L13:.+]]=, $[[SP:.+]]=, $pop[[L14]]
- ; CHECK-NEXT: set_global __stack_pointer@GLOBAL, $pop[[L13]]
+ ; CHECK-NEXT: local.tee $push[[L13:.+]]=, $[[SP:.+]]=, $pop[[L14]]
+ ; CHECK-NEXT: global.set __stack_pointer@GLOBAL, $pop[[L13]]
  ; Copy the AlignedStruct argument to the stack (SP+0, original SP-16)
  ; Just check the last load/store pair of the memcpy
  ; CHECK: i64.load $push[[L4:.+]]=, 0($0)
@@ -107,14 +107,14 @@ define void @byval_empty_callee(%EmptyStruct* byval %ptr) {
 
 ; Call memcpy for "big" byvals.
 ; CHECK-LABEL: big_byval:
-; CHECK:      get_global $push[[L2:.+]]=, __stack_pointer@GLOBAL{{$}}
+; CHECK:      global.get $push[[L2:.+]]=, __stack_pointer@GLOBAL{{$}}
 ; CHECK-NEXT: i32.const $push[[L3:.+]]=, 131072
 ; CHECK-NEXT: i32.sub $push[[L11:.+]]=, $pop[[L2]], $pop[[L3]]
-; CHECK-NEXT: tee_local $push[[L10:.+]]=, $[[SP:.+]]=, $pop[[L11]]{{$}}
-; CHECK-NEXT: set_global __stack_pointer@GLOBAL, $pop[[L10]]{{$}}
+; CHECK-NEXT: local.tee $push[[L10:.+]]=, $[[SP:.+]]=, $pop[[L11]]{{$}}
+; CHECK-NEXT: global.set __stack_pointer@GLOBAL, $pop[[L10]]{{$}}
 ; CHECK-NEXT: i32.const $push[[L0:.+]]=, 131072
 ; CHECK-NEXT: i32.call       $push[[L11:.+]]=, memcpy@FUNCTION, $[[SP]], ${{.+}}, $pop{{.+}}
-; CHECK-NEXT: tee_local      $push[[L9:.+]]=, $[[SP:.+]]=, $pop[[L11]]{{$}}
+; CHECK-NEXT: local.tee      $push[[L9:.+]]=, $[[SP:.+]]=, $pop[[L11]]{{$}}
 ; CHECK-NEXT: call           big_byval_callee@FUNCTION,
 %big = type [131072 x i8]
 declare void @big_byval_callee(%big* byval align 1)
diff --git a/test/CodeGen/WebAssembly/call.ll b/test/CodeGen/WebAssembly/call.ll
index 3a849739e76ec6b18ed8443371c786b9c208711f..77f17c850edab48d1dbdcdb14c7826243de865a0 100644
--- a/test/CodeGen/WebAssembly/call.ll
+++ b/test/CodeGen/WebAssembly/call.ll
@@ -1,5 +1,5 @@
-; RUN: llc < %s -asm-verbose=false -disable-wasm-fallthrough-return-opt -wasm-keep-registers -wasm-temporary-workarounds=false -mattr=+sign-ext,+simd128 | FileCheck %s
-; RUN: llc < %s -asm-verbose=false -disable-wasm-fallthrough-return-opt -wasm-keep-registers -fast-isel -fast-isel-abort=1 -wasm-temporary-workarounds=false -mattr=+sign-ext,+simd128 | FileCheck %s
+; RUN: llc < %s -asm-verbose=false -disable-wasm-fallthrough-return-opt -wasm-keep-registers -mattr=+sign-ext,+simd128 | FileCheck %s
+; RUN: llc < %s -asm-verbose=false -disable-wasm-fallthrough-return-opt -wasm-keep-registers -fast-isel -fast-isel-abort=1 -mattr=+sign-ext,+simd128 | FileCheck %s
 
 ; Test that basic call operations assemble as expected.
 
@@ -71,7 +71,7 @@ define void @call_void_nullary() {
 
 ; CHECK-LABEL: call_i32_unary:
 ; CHECK-NEXT: .functype call_i32_unary (i32) -> (i32){{$}}
-; CHECK-NEXT: get_local $push[[L0:[0-9]+]]=, 0{{$}}
+; CHECK-NEXT: local.get $push[[L0:[0-9]+]]=, 0{{$}}
 ; CHECK-NEXT: {{^}} i32.call $push[[NUM:[0-9]+]]=, i32_unary@FUNCTION, $pop[[L0]]{{$}}
 ; CHECK-NEXT: return $pop[[NUM]]{{$}}
 define i32 @call_i32_unary(i32 %a) {
@@ -81,8 +81,8 @@ define i32 @call_i32_unary(i32 %a) {
 
 ; CHECK-LABEL: call_i32_binary:
 ; CHECK-NEXT: .functype call_i32_binary (i32, i32) -> (i32){{$}}
-; CHECK-NEXT: get_local $push[[L0:[0-9]+]]=, 0{{$}}
-; CHECK-NEXT: get_local $push[[L1:[0-9]+]]=, 1{{$}}
+; CHECK-NEXT: local.get $push[[L0:[0-9]+]]=, 0{{$}}
+; CHECK-NEXT: local.get $push[[L1:[0-9]+]]=, 1{{$}}
 ; CHECK-NEXT: {{^}} i32.call $push[[NUM:[0-9]+]]=, i32_binary@FUNCTION, $pop[[L0]], $pop[[L1]]{{$}}
 ; CHECK-NEXT: return $pop[[NUM]]{{$}}
 define i32 @call_i32_binary(i32 %a, i32 %b) {
@@ -92,7 +92,7 @@ define i32 @call_i32_binary(i32 %a, i32 %b) {
 
 ; CHECK-LABEL: call_indirect_void:
 ; CHECK-NEXT: .functype call_indirect_void (i32) -> (){{$}}
-; CHECK-NEXT: get_local $push[[L0:[0-9]+]]=, 0{{$}}
+; CHECK-NEXT: local.get $push[[L0:[0-9]+]]=, 0{{$}}
 ; CHECK-NEXT: {{^}} call_indirect $pop[[L0]]{{$}}
 ; CHECK-NEXT: return{{$}}
 define void @call_indirect_void(void ()* %callee) {
@@ -102,7 +102,7 @@ define void @call_indirect_void(void ()* %callee) {
 
 ; CHECK-LABEL: call_indirect_i32:
 ; CHECK-NEXT: .functype call_indirect_i32 (i32) -> (i32){{$}}
-; CHECK-NEXT: get_local $push[[L0:[0-9]+]]=, 0{{$}}
+; CHECK-NEXT: local.get $push[[L0:[0-9]+]]=, 0{{$}}
 ; CHECK-NEXT: {{^}} i32.call_indirect $push[[NUM:[0-9]+]]=, $pop[[L0]]{{$}}
 ; CHECK-NEXT: return $pop[[NUM]]{{$}}
 define i32 @call_indirect_i32(i32 ()* %callee) {
@@ -112,7 +112,7 @@ define i32 @call_indirect_i32(i32 ()* %callee) {
 
 ; CHECK-LABEL: call_indirect_i64:
 ; CHECK-NEXT: .functype call_indirect_i64 (i32) -> (i64){{$}}
-; CHECK-NEXT: get_local $push[[L0:[0-9]+]]=, 0{{$}}
+; CHECK-NEXT: local.get $push[[L0:[0-9]+]]=, 0{{$}}
 ; CHECK-NEXT: {{^}} i64.call_indirect $push[[NUM:[0-9]+]]=, $pop[[L0]]{{$}}
 ; CHECK-NEXT: return $pop[[NUM]]{{$}}
 define i64 @call_indirect_i64(i64 ()* %callee) {
@@ -122,7 +122,7 @@ define i64 @call_indirect_i64(i64 ()* %callee) {
 
 ; CHECK-LABEL: call_indirect_float:
 ; CHECK-NEXT: .functype call_indirect_float (i32) -> (f32){{$}}
-; CHECK-NEXT: get_local $push[[L0:[0-9]+]]=, 0{{$}}
+; CHECK-NEXT: local.get $push[[L0:[0-9]+]]=, 0{{$}}
 ; CHECK-NEXT: {{^}} f32.call_indirect $push[[NUM:[0-9]+]]=, $pop[[L0]]{{$}}
 ; CHECK-NEXT: return $pop[[NUM]]{{$}}
 define float @call_indirect_float(float ()* %callee) {
@@ -132,7 +132,7 @@ define float @call_indirect_float(float ()* %callee) {
 
 ; CHECK-LABEL: call_indirect_double:
 ; CHECK-NEXT: .functype call_indirect_double (i32) -> (f64){{$}}
-; CHECK-NEXT: get_local $push[[L0:[0-9]+]]=, 0{{$}}
+; CHECK-NEXT: local.get $push[[L0:[0-9]+]]=, 0{{$}}
 ; CHECK-NEXT: {{^}} f64.call_indirect $push[[NUM:[0-9]+]]=, $pop[[L0]]{{$}}
 ; CHECK-NEXT: return $pop[[NUM]]{{$}}
 define double @call_indirect_double(double ()* %callee) {
@@ -142,7 +142,7 @@ define double @call_indirect_double(double ()* %callee) {
 
 ; CHECK-LABEL: call_indirect_v128:
 ; CHECK-NEXT: .functype call_indirect_v128 (i32) -> (v128){{$}}
-; CHECK-NEXT: get_local $push[[L0:[0-9]+]]=, 0{{$}}
+; CHECK-NEXT: local.get $push[[L0:[0-9]+]]=, 0{{$}}
 ; CHECK-NEXT: {{^}} v128.call_indirect $push[[NUM:[0-9]+]]=, $pop[[L0]]{{$}}
 ; CHECK-NEXT: return $pop[[NUM]]{{$}}
 define <16 x i8> @call_indirect_v128(<16 x i8> ()* %callee) {
@@ -152,8 +152,8 @@ define <16 x i8> @call_indirect_v128(<16 x i8> ()* %callee) {
 
 ; CHECK-LABEL: call_indirect_arg:
 ; CHECK-NEXT: .functype call_indirect_arg (i32, i32) -> (){{$}}
-; CHECK-NEXT: get_local $push[[L0:[0-9]+]]=, 1{{$}}
-; CHECK-NEXT: get_local $push[[L1:[0-9]+]]=, 0{{$}}
+; CHECK-NEXT: local.get $push[[L0:[0-9]+]]=, 1{{$}}
+; CHECK-NEXT: local.get $push[[L1:[0-9]+]]=, 0{{$}}
 ; CHECK-NEXT: {{^}} call_indirect $pop[[L0]], $pop[[L1]]{{$}}
 ; CHECK-NEXT: return{{$}}
 define void @call_indirect_arg(void (i32)* %callee, i32 %arg) {
@@ -163,9 +163,9 @@ define void @call_indirect_arg(void (i32)* %callee, i32 %arg) {
 
 ; CHECK-LABEL: call_indirect_arg_2:
 ; CHECK-NEXT: .functype call_indirect_arg_2 (i32, i32, i32) -> (){{$}}
-; CHECK-NEXT: get_local $push[[L0:[0-9]+]]=, 1{{$}}
-; CHECK-NEXT: get_local $push[[L1:[0-9]+]]=, 2{{$}}
-; CHECK-NEXT: get_local $push[[L2:[0-9]+]]=, 0{{$}}
+; CHECK-NEXT: local.get $push[[L0:[0-9]+]]=, 1{{$}}
+; CHECK-NEXT: local.get $push[[L1:[0-9]+]]=, 2{{$}}
+; CHECK-NEXT: local.get $push[[L2:[0-9]+]]=, 0{{$}}
 ; CHECK-NEXT: {{^}} i32.call_indirect $push[[NUM:[0-9]+]]=, $pop[[L0]], $pop[[L1]], $pop[[L2]]{{$}}
 ; CHECK-NEXT: drop $pop[[NUM]]{{$}}
 ; CHECK-NEXT: return{{$}}
diff --git a/test/CodeGen/WebAssembly/cfg-stackify-dbg-skip.ll b/test/CodeGen/WebAssembly/cfg-stackify-dbg-skip.ll
index 9077d7a73d60661d933c7eeadb903dac07ed7607..5a7253493a19c832e774ca6cfce36e2f4c7e4492 100644
--- a/test/CodeGen/WebAssembly/cfg-stackify-dbg-skip.ll
+++ b/test/CodeGen/WebAssembly/cfg-stackify-dbg-skip.ll
@@ -7,7 +7,7 @@
 ; CHECK: body:
 ; CHECK: BLOCK
 ;                       <-- Stackified expression starts
-; CHECK-NEXT: GET_LOCAL_I64
+; CHECK-NEXT: LOCAL_GET_I64
 ; CHECK-NEXT: I32_WRAP_I64
 ; CHECK-NEXT: DBG_VALUE
 ;                       <-- BLOCK should NOT be placed here!
diff --git a/test/CodeGen/WebAssembly/cfg-stackify-eh.ll b/test/CodeGen/WebAssembly/cfg-stackify-eh.ll
index e3c34f87e13e7526f0429a05e09787660a2b4580..037730141b3142470d935d08ec5323fa62aba29b 100644
--- a/test/CodeGen/WebAssembly/cfg-stackify-eh.ll
+++ b/test/CodeGen/WebAssembly/cfg-stackify-eh.ll
@@ -7,21 +7,25 @@ target triple = "wasm32-unknown-unknown"
 @_ZTId = external constant i8*
 
 ; Simple test case with two catch clauses
+; void test0() {
+;   try {
+;     foo();
+;   } catch (int n) {
+;     bar();
+;   } catch (double d) {
+;   }
+; }
 
 ; CHECK-LABEL: test0
+; CHECK:   try
 ; CHECK:   call      foo@FUNCTION
-; CHECK: .LBB0_1:
-; CHECK:   i32.catch
+; CHECK:   catch     $[[EXCEPT_REF:[0-9]+]]=
+; CHECK:   block i32
+; CHECK:   br_on_exn 0, __cpp_exception@EVENT, $[[EXCEPT_REF]]
+; CHECK:   rethrow
+; CHECK:   end_block
 ; CHECK:   i32.call  $drop=, _Unwind_CallPersonality@FUNCTION
-; CHECK:   i32.call  $drop=, __cxa_begin_catch@FUNCTION
-; CHECK:   call      bar@FUNCTION
-; CHECK:   call      __cxa_end_catch@FUNCTION
-; CHECK: .LBB0_3:
-; CHECK:   i32.call  $drop=, __cxa_begin_catch@FUNCTION
-; CHECK:   call      __cxa_end_catch@FUNCTION
-; CHECK: .LBB0_5:
-; CHECK:   call      __cxa_rethrow@FUNCTION
-; CHECK: .LBB0_6:
+; CHECK:   end_try
 ; CHECK:   return
 define void @test0() personality i8* bitcast (i32 (...)* @__gxx_wasm_personality_v0 to i8*) {
 entry:
@@ -68,39 +72,45 @@ try.cont:                                         ; preds = %entry, %catch, %cat
 }
 
 ; Nested try-catches within a catch
+; void test1() {
+;   try {
+;     foo();
+;   } catch (int n) {
+;     try {
+;       foo();
+;     } catch (int n) {
+;       foo();
+;     }
+;   }
+; }
 
 ; CHECK-LABEL: test1
+; CHECK:   try
 ; CHECK:   call      foo@FUNCTION
-; CHECK: .LBB1_1:
-; CHECK:   i32.catch     $0=, 0
-; CHECK:   i32.call  $drop=, _Unwind_CallPersonality@FUNCTION, $0
-; CHECK:   i32.call  $drop=, __cxa_begin_catch@FUNCTION, $0
-; CHECK:   call      foo@FUNCTION
-; CHECK: .LBB1_3:
-; CHECK:   i32.catch     $0=, 0
-; CHECK:   i32.call  $drop=, _Unwind_CallPersonality@FUNCTION, $0
-; CHECK:   i32.call  $drop=, __cxa_begin_catch@FUNCTION, $0
+; CHECK:   catch
+; CHECK:   br_on_exn 0, __cpp_exception@EVENT
+; CHECK:   rethrow
+; CHECK:   i32.call  $drop=, _Unwind_CallPersonality@FUNCTION
+; CHECK:   try
 ; CHECK:   call      foo@FUNCTION
-; CHECK: .LBB1_5:
-; CHECK:   catch_all
-; CHECK:   call      __cxa_end_catch@FUNCTION
+; CHECK:   catch
+; CHECK:   br_on_exn   0, __cpp_exception@EVENT
 ; CHECK:   rethrow
-; CHECK: .LBB1_6:
-; CHECK:   call      __cxa_rethrow@FUNCTION
+; CHECK:   i32.call  $drop=, _Unwind_CallPersonality@FUNCTION
+; CHECK:   try
+; CHECK:   i32.call  $drop=, __cxa_begin_catch@FUNCTION
+; CHECK:   try
+; CHECK:   call      foo@FUNCTION
+; CHECK:   catch     $drop=
 ; CHECK:   rethrow
-; CHECK: .LBB1_7:
-; CHECK:   call      __cxa_end_catch@FUNCTION
-; CHECK: .LBB1_8:
-; CHECK:   catch_all
-; CHECK:   call      __cxa_end_catch@FUNCTION
-; CHECK: .LBB1_9:
-; CHECK:   call      __cxa_rethrow@FUNCTION
+; CHECK:   end_try
+; CHECK:   catch     $drop=
 ; CHECK:   rethrow
-; CHECK: .LBB1_10:
-; CHECK:   call      __cxa_end_catch@FUNCTION
-; CHECK: .LBB1_11:
+; CHECK:   end_try
+; CHECK:   end_try
+; CHECK:   end_try
 ; CHECK:   return
-define hidden void @test1() personality i8* bitcast (i32 (...)* @__gxx_wasm_personality_v0 to i8*) {
+define void @test1() personality i8* bitcast (i32 (...)* @__gxx_wasm_personality_v0 to i8*) {
 entry:
   invoke void @foo()
           to label %try.cont11 unwind label %catch.dispatch
@@ -175,30 +185,38 @@ unreachable:                                      ; preds = %rethrow5
 }
 
 ; Nested loop within a catch clause
+; void test2() {
+;   try {
+;     foo();
+;   } catch (...) {
+;     for (int i = 0; i < 50; i++)
+;       foo();
+;   }
+; }
 
 ; CHECK-LABEL: test2
+; CHECK:   try
 ; CHECK:   call      foo@FUNCTION
-; CHECK: .LBB2_1:
-; CHECK:   i32.catch
-; CHECK:   i32.call  $drop=, __cxa_begin_catch@FUNCTION
-; CHECK: .LBB2_2:
+; CHECK:   catch
+; CHECK:   br_on_exn   0, __cpp_exception@EVENT
+; CHECK:   rethrow
+; CHECK:   loop
+; CHECK:   try
 ; CHECK:   call      foo@FUNCTION
-; CHECK: .LBB2_4:
-; CHECK:   catch_all
+; CHECK:   catch     $drop=
+; CHECK:   try
 ; CHECK:   call      __cxa_end_catch@FUNCTION
-; CHECK: .LBB2_5:
-; CHECK:   i32.catch
-; CHECK:   call      __clang_call_terminate@FUNCTION
+; CHECK:   catch
+; CHECK:   br_on_exn   0, __cpp_exception@EVENT
+; CHECK:   call      __clang_call_terminate@FUNCTION, 0
 ; CHECK:   unreachable
-; CHECK: .LBB2_6:
-; CHECK:   catch_all
-; CHECK:   call      _ZSt9terminatev@FUNCTION
+; CHECK:   call      __clang_call_terminate@FUNCTION
 ; CHECK:   unreachable
-; CHECK: .LBB2_7:
+; CHECK:   end_try
 ; CHECK:   rethrow
-; CHECK: .LBB2_8:
-; CHECK:   call      __cxa_end_catch@FUNCTION
-; CHECK: .LBB2_10:
+; CHECK:   end_try
+; CHECK:   end_loop
+; CHECK:   end_try
 ; CHECK:   return
 define void @test2() personality i8* bitcast (i32 (...)* @__gxx_wasm_personality_v0 to i8*) {
 entry:
diff --git a/test/CodeGen/WebAssembly/cfg-stackify-eh.mir b/test/CodeGen/WebAssembly/cfg-stackify-eh.mir
deleted file mode 100644
index b67579087faefbd00df9f76463858d7bedf357bf..0000000000000000000000000000000000000000
--- a/test/CodeGen/WebAssembly/cfg-stackify-eh.mir
+++ /dev/null
@@ -1,322 +0,0 @@
-# RUN: llc -mtriple=wasm32-unknown-unknown -exception-model=wasm -run-pass wasm-cfg-stackify %s -o - | FileCheck %s
-
---- |
-  target datalayout = "e-m:e-p:32:32-i64:64-n32:64-S128"
-  target triple = "wasm32-unknown-unknown"
-
-  @__wasm_lpad_context = external global { i32, i8*, i32 }
-
-  declare void @may_throw()
-  ; Function Attrs: nounwind
-  declare void @dont_throw() #0
-  declare i8* @__cxa_begin_catch(i8*)
-  declare void @__cxa_end_catch()
-  declare void @__cxa_rethrow()
-  ; Function Attrs: nounwind
-  declare i32 @__gxx_wasm_personality_v0(...)
-  declare i32 @_Unwind_CallPersonality(i8*) #0
-
-  define void @test0() personality i8* bitcast (i32 (...)* @__gxx_wasm_personality_v0 to i8*) {
-    unreachable
-  }
-  define void @test1() personality i8* bitcast (i32 (...)* @__gxx_wasm_personality_v0 to i8*) {
-    unreachable
-  }
-  define void @test2() personality i8* bitcast (i32 (...)* @__gxx_wasm_personality_v0 to i8*) {
-    unreachable
-  }
-  define void @test3() personality i8* bitcast (i32 (...)* @__gxx_wasm_personality_v0 to i8*) {
-    unreachable
-  }
-
-  attributes #0 = { nounwind }
-
----
-# Simplest try-catch
-# try {
-#   may_throw();
-# } catch (...) {
-# }
-name: test0
-# CHECK-LABEL: name: test0
-liveins:
-  - { reg: '$arguments', reg: '$value_stack' }
-body: |
-  bb.0:
-    successors: %bb.2, %bb.1
-
-    CALL_VOID @may_throw, implicit-def dead $arguments, implicit $sp32, implicit $sp64
-    BR %bb.2, implicit-def $arguments
-  ; CHECK-LABEL: bb.0:
-    ; CHECK: TRY
-    ; CHECK-NEXT: CALL_VOID @may_throw
-
-  bb.1 (landing-pad):
-  ; predecessors: %bb.0
-    successors: %bb.2
-
-    %2:i32 = CATCH_I32 0, implicit-def dead $arguments, implicit-def $value_stack, implicit $value_stack
-    %3:i32 = CALL_I32 @__cxa_begin_catch, %2:i32, implicit-def dead $arguments, implicit $sp32, implicit $sp64, implicit-def $value_stack, implicit $value_stack
-    DROP_I32 killed %3:i32, implicit-def $arguments
-    CALL_VOID @__cxa_end_catch, implicit-def dead $arguments, implicit $sp32, implicit $sp64
-
-  bb.2:
-  ; predecessors: %bb.0, %bb.1
-
-    RETURN_VOID implicit-def dead $arguments
-  ; CHECK-LABEL: bb.2:
-    ; CHECK-NEXT: END_TRY
-    ; CHECK: RETURN_VOID
-...
----
-
-# Nested try-catch inside another catch
-#  try {
-#    may_throw();
-#  } catch (int n) {
-#    try {
-#      may_throw();
-#    } catch (int n) {
-#    }
-#  }
-name: test1
-# CHECK-LABEL: name: test1
-liveins:
-  - { reg: '$arguments', reg: '$value_stack' }
-body: |
-  bb.0:
-    successors: %bb.9, %bb.1
-
-    CALL_VOID @may_throw, implicit-def dead $arguments, implicit $sp32, implicit $sp64
-    BR %bb.9, implicit-def $arguments
-  ; CHECK-LABEL: bb.0:
-    ; CHECK: TRY
-    ; CHECK-NEXT: CALL_VOID @may_throw
-
-  bb.1 (landing-pad):
-  ; predecessors: %bb.0
-    successors: %bb.2, %bb.7
-
-    %30:i32 = CATCH_I32 0, implicit-def dead $arguments
-    SET_LOCAL_I32 0, %30:i32, implicit-def $arguments
-    %16:i32 = CONST_I32 0, implicit-def dead $arguments, implicit-def $value_stack, implicit $value_stack
-    %27:i32 = CONST_I32 0, implicit-def dead $arguments, implicit-def $value_stack, implicit $value_stack
-    STORE_I32 2, @__wasm_lpad_context + 4, %16:i32, %27:i32, implicit-def dead $arguments, implicit-def $value_stack, implicit $value_stack :: (store 4 into `i8** getelementptr inbounds ({ i32, i8*, i32 }, { i32, i8*, i32 }* @__wasm_lpad_context, i32 0, i32 1)`)
-    %26:i32 = CONST_I32 0, implicit-def dead $arguments, implicit-def $value_stack, implicit $value_stack
-    %25:i32 = CONST_I32 0, implicit-def dead $arguments, implicit-def $value_stack, implicit $value_stack
-    STORE_I32 2, @__wasm_lpad_context, %26:i32, %25:i32, implicit-def dead $arguments, implicit-def $value_stack, implicit $value_stack :: (store 4 into `i32* getelementptr inbounds ({ i32, i8*, i32 }, { i32, i8*, i32 }* @__wasm_lpad_context, i32 0, i32 0)`)
-    %32:i32 = GET_LOCAL_I32 0, implicit-def $arguments
-    %31:i32 = CALL_I32 @_Unwind_CallPersonality, %32:i32, implicit-def dead $arguments, implicit $sp32, implicit $sp64
-    DROP_I32 killed %31:i32, implicit-def $arguments
-    %24:i32 = CONST_I32 0, implicit-def dead $arguments, implicit-def $value_stack, implicit $value_stack
-    %17:i32 = LOAD_I32 2, @__wasm_lpad_context + 8, %24:i32, implicit-def dead $arguments, implicit-def $value_stack, implicit $value_stack :: (dereferenceable load 4 from `i32* getelementptr inbounds ({ i32, i8*, i32 }, { i32, i8*, i32 }* @__wasm_lpad_context, i32 0, i32 2)`)
-    %18:i32 = CONST_I32 1, implicit-def dead $arguments, implicit-def $value_stack, implicit $value_stack
-    %19:i32 = NE_I32 %17:i32, %18:i32, implicit-def dead $arguments, implicit-def $value_stack, implicit $value_stack
-    BR_IF %bb.7, %19:i32, implicit-def $arguments, implicit-def $value_stack, implicit $value_stack
-
-  bb.2:
-  ; predecessors: %bb.1
-    successors: %bb.8, %bb.3, %bb.6
-
-    %34:i32 = GET_LOCAL_I32 0, implicit-def $arguments
-    %33:i32 = CALL_I32 @__cxa_begin_catch, %34:i32, implicit-def dead $arguments, implicit $sp32, implicit $sp64
-    DROP_I32 killed %33:i32, implicit-def $arguments
-    CALL_VOID @may_throw, implicit-def dead $arguments, implicit $sp32, implicit $sp64
-    BR %bb.8, implicit-def $arguments
-  ; CHECK-LABEL: bb.2:
-    ; CHECK: DROP_I32
-    ; CHECK-NEXT: TRY
-    ; CHECK-NEXT: TRY
-    ; CHECK-NEXT: CALL_VOID @may_throw
-
-  bb.3 (landing-pad):
-  ; predecessors: %bb.2
-    successors: %bb.4, %bb.5
-
-    %35:i32 = CATCH_I32 0, implicit-def dead $arguments
-    SET_LOCAL_I32 0, %35:i32, implicit-def $arguments
-    %21:i32 = CONST_I32 0, implicit-def dead $arguments, implicit-def $value_stack, implicit $value_stack
-    %20:i32 = CONST_I32 1, implicit-def dead $arguments, implicit-def $value_stack, implicit $value_stack
-    STORE_I32 2, @__wasm_lpad_context, %21:i32, %20:i32, implicit-def dead $arguments, implicit-def $value_stack, implicit $value_stack :: (store 4 into `i32* getelementptr inbounds ({ i32, i8*, i32 }, { i32, i8*, i32 }* @__wasm_lpad_context, i32 0, i32 0)`)
-    %37:i32 = GET_LOCAL_I32 0, implicit-def $arguments
-    %36:i32 = CALL_I32 @_Unwind_CallPersonality, %37:i32, implicit-def dead $arguments, implicit $sp32, implicit $sp64
-    DROP_I32 killed %36:i32, implicit-def $arguments
-    %29:i32 = CONST_I32 0, implicit-def dead $arguments, implicit-def $value_stack, implicit $value_stack
-    %22:i32 = LOAD_I32 2, @__wasm_lpad_context + 8, %29:i32, implicit-def dead $arguments, implicit-def $value_stack, implicit $value_stack :: (dereferenceable load 4 from `i32* getelementptr inbounds ({ i32, i8*, i32 }, { i32, i8*, i32 }* @__wasm_lpad_context, i32 0, i32 2)`)
-    %28:i32 = CONST_I32 1, implicit-def dead $arguments, implicit-def $value_stack, implicit $value_stack
-    %23:i32 = NE_I32 %22:i32, %28:i32, implicit-def dead $arguments, implicit-def $value_stack, implicit $value_stack
-    BR_IF %bb.5, %23:i32, implicit-def $arguments, implicit-def $value_stack, implicit $value_stack
-
-  bb.4:
-  ; predecessors: %bb.3
-    successors: %bb.8
-
-    %39:i32 = GET_LOCAL_I32 0, implicit-def $arguments
-    %38:i32 = CALL_I32 @__cxa_begin_catch, %39:i32, implicit-def dead $arguments, implicit $sp32, implicit $sp64
-    DROP_I32 killed %38:i32, implicit-def $arguments
-    CALL_VOID @__cxa_end_catch, implicit-def dead $arguments, implicit $sp32, implicit $sp64
-    BR %bb.8, implicit-def $arguments
-
-  bb.5:
-  ; predecessors: %bb.3
-    successors: %bb.6
-
-    CALL_VOID @__cxa_rethrow, implicit-def dead $arguments, implicit $sp32, implicit $sp64
-    RETHROW %bb.6, implicit-def $arguments
-
-  bb.6 (landing-pad):
-  ; predecessors: %bb.2, %bb.5
-
-    CATCH_ALL implicit-def $arguments
-    CALL_VOID @__cxa_end_catch, implicit-def dead $arguments, implicit $sp32, implicit $sp64
-    RETHROW_TO_CALLER implicit-def $arguments
-  ; CHECK-LABEL: bb.6 (landing-pad):
-    ; CHECK-NEXT: END_TRY
-
-  bb.7:
-  ; predecessors: %bb.1
-
-    CALL_VOID @__cxa_rethrow, implicit-def dead $arguments, implicit $sp32, implicit $sp64
-    RETHROW_TO_CALLER implicit-def $arguments
-  ; CHECK-LABEL: bb.7:
-    ; CHECK-NEXT: END_TRY
-    ; CHECK: RETHROW 0
-
-  bb.8:
-  ; predecessors: %bb.2, %bb.4
-    successors: %bb.9
-
-    CALL_VOID @__cxa_end_catch, implicit-def dead $arguments, implicit $sp32, implicit $sp64
-
-  bb.9:
-  ; predecessors: %bb.0, %bb.8
-
-    RETURN_VOID implicit-def dead $arguments
-  ; CHECK-LABEL: bb.9:
-    ; CHECK-NEXT: END_TRY
-...
----
-
-# A loop within a try.
-#  try {
-#    for (int i = 0; i < n; ++i)
-#      may_throw();
-#  } catch (...) {
-#  }
-name: test2
-# CHECK-LABEL: name: test2
-liveins:
-  - { reg: '$arguments', reg: '$value_stack' }
-body: |
-  bb.0:
-    successors: %bb.1, %bb.4
-
-    %18:i32 = CONST_I32 0, implicit-def dead $arguments
-    SET_LOCAL_I32 1, %18:i32, implicit-def $arguments
-    %14:i32 = CONST_I32 0, implicit-def dead $arguments, implicit-def $value_stack, implicit $value_stack
-    %19:i32 = GET_LOCAL_I32 0, implicit-def $arguments
-    %9:i32 = GE_S_I32 %14:i32, %19:i32, implicit-def dead $arguments, implicit-def $value_stack, implicit $value_stack
-    BR_IF %bb.4, %9:i32, implicit-def $arguments
-
-  bb.1:
-  ; predecessors: %bb.0, %bb.3
-    successors: %bb.3, %bb.2
-
-    CALL_VOID @may_throw, implicit-def dead $arguments, implicit $sp32, implicit $sp64
-    BR %bb.3, implicit-def $arguments
-  ; CHECK-LABEL: bb.1:
-    ; CHECK: LOOP
-    ; CHECK: TRY
-    ; CHECK-NEXT: CALL_VOID @may_throw
-
-  bb.2 (landing-pad):
-  ; predecessors: %bb.1
-    successors: %bb.4
-
-    %11:i32 = CATCH_I32 0, implicit-def dead $arguments, implicit-def $value_stack, implicit $value_stack
-    %22:i32 = CALL_I32 @__cxa_begin_catch, %11:i32, implicit-def dead $arguments, implicit $sp32, implicit $sp64, implicit-def $value_stack, implicit $value_stack
-    DROP_I32 killed %22:i32, implicit-def $arguments
-    CALL_VOID @__cxa_end_catch, implicit-def dead $arguments, implicit $sp32, implicit $sp64
-    BR %bb.4, implicit-def $arguments
-
-  bb.3:
-  ; predecessors: %bb.1
-    successors: %bb.1, %bb.4
-
-    %20:i32 = GET_LOCAL_I32 1, implicit-def $arguments
-    %17:i32 = CONST_I32 1, implicit-def dead $arguments, implicit-def $value_stack, implicit $value_stack
-    %16:i32 = ADD_I32 %20:i32, %17:i32, implicit-def dead $arguments, implicit-def $value_stack, implicit $value_stack
-    %15:i32 = TEE_LOCAL_I32 1, %16:i32, implicit-def $arguments
-    %21:i32 = GET_LOCAL_I32 0, implicit-def $arguments
-    %10:i32 = GE_S_I32 %15:i32, %21:i32, implicit-def dead $arguments, implicit-def $value_stack, implicit $value_stack
-    BR_UNLESS %bb.1, %10:i32, implicit-def $arguments
-  ; CHECK-LABEL: bb.3:
-    ; CHECK: END_TRY
-
-  bb.4:
-  ; predecessors: %bb.2, %bb.0, %bb.3
-
-    RETURN_VOID implicit-def dead $arguments
-...
----
-
-# A loop within a catch
-#  try {
-#    may_throw();
-#  } catch (...) {
-#    for (int i = 0; i < n; ++i)
-#      dont_throw();
-#  }
-name: test3
-# CHECK-LABEL: name: test3
-liveins:
-  - { reg: '$arguments', reg: '$value_stack' }
-body: |
-  bb.0:
-    successors: %bb.4, %bb.1
-
-    CALL_VOID @may_throw, implicit-def dead $arguments, implicit $sp32, implicit $sp64
-    BR %bb.4, implicit-def $arguments
-  ; CHECK-LABEL: bb.0:
-    ; CHECK: TRY
-    ; CHECK-NEXT: CALL_VOID @may_throw
-
-  bb.1 (landing-pad):
-  ; predecessors: %bb.0
-    successors: %bb.2, %bb.3
-
-    %9:i32 = CATCH_I32 0, implicit-def dead $arguments, implicit-def $value_stack, implicit $value_stack
-    %18:i32 = CALL_I32 @__cxa_begin_catch, %9:i32, implicit-def dead $arguments, implicit $sp32, implicit $sp64, implicit-def $value_stack, implicit $value_stack
-    DROP_I32 killed %18:i32, implicit-def $arguments
-    %19:i32 = CONST_I32 0, implicit-def dead $arguments
-    SET_LOCAL_I32 1, %19:i32, implicit-def $arguments
-    %14:i32 = CONST_I32 0, implicit-def dead $arguments, implicit-def $value_stack, implicit $value_stack
-    %20:i32 = GET_LOCAL_I32 0, implicit-def $arguments
-    %10:i32 = GE_S_I32 %14:i32, %20:i32, implicit-def dead $arguments, implicit-def $value_stack, implicit $value_stack
-    BR_IF %bb.3, %10:i32, implicit-def $arguments, implicit-def $value_stack, implicit $value_stack
-
-  bb.2:
-  ; predecessors: %bb.1, %bb.2
-    successors: %bb.2, %bb.3
-
-    CALL_VOID @dont_throw, implicit-def dead $arguments, implicit $sp32, implicit $sp64
-    %21:i32 = GET_LOCAL_I32 1, implicit-def $arguments
-    %17:i32 = CONST_I32 1, implicit-def dead $arguments, implicit-def $value_stack, implicit $value_stack
-    %16:i32 = ADD_I32 %21:i32, %17:i32, implicit-def dead $arguments, implicit-def $value_stack, implicit $value_stack
-    %15:i32 = TEE_LOCAL_I32 1, %16:i32, implicit-def $arguments
-    %22:i32 = GET_LOCAL_I32 0, implicit-def $arguments
-    %11:i32 = GE_S_I32 %15:i32, %22:i32, implicit-def dead $arguments, implicit-def $value_stack, implicit $value_stack
-    BR_UNLESS %bb.2, %11:i32, implicit-def $arguments, implicit-def $value_stack, implicit $value_stack
-
-  bb.3:
-  ; predecessors: %bb.1, %bb.2
-    successors: %bb.4
-
-    CALL_VOID @__cxa_end_catch, implicit-def dead $arguments, implicit $sp32, implicit $sp64
-
-  bb.4:
-  ; predecessors: %bb.0, %bb.3
-
-    RETURN_VOID implicit-def dead $arguments
-  ; CHECK-LABEL: bb.4:
-    ; CHECK: END_TRY
diff --git a/test/CodeGen/WebAssembly/comparisons-f32.ll b/test/CodeGen/WebAssembly/comparisons-f32.ll
index 7ccfe5e8a97aaf8ff4fd0766934ad8c87552ec7e..e4753c4e4260921417a138f7abdba594e34a3862 100644
--- a/test/CodeGen/WebAssembly/comparisons-f32.ll
+++ b/test/CodeGen/WebAssembly/comparisons-f32.ll
@@ -8,11 +8,11 @@ target triple = "wasm32-unknown-unknown"
 
 ; CHECK-LABEL: ord_f32:
 ; CHECK-NEXT: .functype ord_f32 (f32, f32) -> (i32){{$}}
-; CHECK-NEXT: get_local $push[[L0:[0-9]+]]=, 0{{$}}
-; CHECK-NEXT: get_local $push[[L1:[0-9]+]]=, 0{{$}}
+; CHECK-NEXT: local.get $push[[L0:[0-9]+]]=, 0{{$}}
+; CHECK-NEXT: local.get $push[[L1:[0-9]+]]=, 0{{$}}
 ; CHECK-NEXT: f32.eq $push[[NUM0:[0-9]+]]=, $pop[[L0]], $pop[[L1]]{{$}}
-; CHECK-NEXT: get_local $push[[L2:[0-9]+]]=, 1{{$}}
-; CHECK-NEXT: get_local $push[[L3:[0-9]+]]=, 1{{$}}
+; CHECK-NEXT: local.get $push[[L2:[0-9]+]]=, 1{{$}}
+; CHECK-NEXT: local.get $push[[L3:[0-9]+]]=, 1{{$}}
 ; CHECK-NEXT: f32.eq $push[[NUM1:[0-9]+]]=, $pop[[L2]], $pop[[L3]]{{$}}
 ; CHECK-NEXT: i32.and $push[[NUM2:[0-9]+]]=, $pop[[NUM0]], $pop[[NUM1]]{{$}}
 ; CHECK-NEXT: return $pop[[NUM2]]{{$}}
@@ -24,11 +24,11 @@ define i32 @ord_f32(float %x, float %y) {
 
 ; CHECK-LABEL: uno_f32:
 ; CHECK-NEXT: .functype uno_f32 (f32, f32) -> (i32){{$}}
-; CHECK-NEXT: get_local $push[[L0:[0-9]+]]=, 0{{$}}
-; CHECK-NEXT: get_local $push[[L1:[0-9]+]]=, 0{{$}}
+; CHECK-NEXT: local.get $push[[L0:[0-9]+]]=, 0{{$}}
+; CHECK-NEXT: local.get $push[[L1:[0-9]+]]=, 0{{$}}
 ; CHECK-NEXT: f32.ne $push[[NUM0:[0-9]+]]=, $pop[[L0]], $pop[[L1]]{{$}}
-; CHECK-NEXT: get_local $push[[L2:[0-9]+]]=, 1{{$}}
-; CHECK-NEXT: get_local $push[[L3:[0-9]+]]=, 1{{$}}
+; CHECK-NEXT: local.get $push[[L2:[0-9]+]]=, 1{{$}}
+; CHECK-NEXT: local.get $push[[L3:[0-9]+]]=, 1{{$}}
 ; CHECK-NEXT: f32.ne $push[[NUM1:[0-9]+]]=, $pop[[L2]], $pop[[L3]]{{$}}
 ; CHECK-NEXT: i32.or $push[[NUM2:[0-9]+]]=, $pop[[NUM0]], $pop[[NUM1]]{{$}}
 ; CHECK-NEXT: return $pop[[NUM2]]{{$}}
@@ -40,8 +40,8 @@ define i32 @uno_f32(float %x, float %y) {
 
 ; CHECK-LABEL: oeq_f32:
 ; CHECK-NEXT: .functype oeq_f32 (f32, f32) -> (i32){{$}}
-; CHECK-NEXT: get_local $push[[L0:[0-9]+]]=, 0{{$}}
-; CHECK-NEXT: get_local $push[[L1:[0-9]+]]=, 1{{$}}
+; CHECK-NEXT: local.get $push[[L0:[0-9]+]]=, 0{{$}}
+; CHECK-NEXT: local.get $push[[L1:[0-9]+]]=, 1{{$}}
 ; CHECK-NEXT: f32.eq $push[[NUM:[0-9]+]]=, $pop[[L0]], $pop[[L1]]{{$}}
 ; CHECK-NEXT: return $pop[[NUM]]{{$}}
 define i32 @oeq_f32(float %x, float %y) {
@@ -100,14 +100,14 @@ define i32 @oge_f32(float %x, float %y) {
 
 ; CHECK-LABEL: ueq_f32:
 ; CHECK-NEXT: .functype ueq_f32 (f32, f32) -> (i32){{$}}
-; CHECK-NEXT: get_local $push[[L0:[0-9]+]]=, 0{{$}}
-; CHECK-NEXT: get_local $push[[L1:[0-9]+]]=, 1{{$}}
+; CHECK-NEXT: local.get $push[[L0:[0-9]+]]=, 0{{$}}
+; CHECK-NEXT: local.get $push[[L1:[0-9]+]]=, 1{{$}}
 ; CHECK-NEXT: f32.eq $push[[NUM0:[0-9]+]]=, $pop[[L0]], $pop[[L1]]{{$}}
-; CHECK-NEXT: get_local $push[[L2:[0-9]+]]=, 0{{$}}
-; CHECK-NEXT: get_local $push[[L3:[0-9]+]]=, 0{{$}}
+; CHECK-NEXT: local.get $push[[L2:[0-9]+]]=, 0{{$}}
+; CHECK-NEXT: local.get $push[[L3:[0-9]+]]=, 0{{$}}
 ; CHECK-NEXT: f32.ne $push[[NUM1:[0-9]+]]=, $pop[[L2]], $pop[[L3]]{{$}}
-; CHECK-NEXT: get_local $push[[L4:[0-9]+]]=, 1{{$}}
-; CHECK-NEXT: get_local $push[[L5:[0-9]+]]=, 1{{$}}
+; CHECK-NEXT: local.get $push[[L4:[0-9]+]]=, 1{{$}}
+; CHECK-NEXT: local.get $push[[L5:[0-9]+]]=, 1{{$}}
 ; CHECK-NEXT: f32.ne $push[[NUM2:[0-9]+]]=, $pop[[L4]], $pop[[L5]]{{$}}
 ; CHECK-NEXT: i32.or $push[[NUM3:[0-9]+]]=, $pop[[NUM1]], $pop[[NUM2]]{{$}}
 ; CHECK-NEXT: i32.or $push[[NUM4:[0-9]+]]=, $pop[[NUM0]], $pop[[NUM3]]{{$}}
@@ -120,14 +120,14 @@ define i32 @ueq_f32(float %x, float %y) {
 
 ; CHECK-LABEL: one_f32:
 ; CHECK-NEXT: .functype one_f32 (f32, f32) -> (i32){{$}}
-; CHECK-NEXT: get_local $push[[L0:[0-9]+]]=, 0{{$}}
-; CHECK-NEXT: get_local $push[[L1:[0-9]+]]=, 1{{$}}
+; CHECK-NEXT: local.get $push[[L0:[0-9]+]]=, 0{{$}}
+; CHECK-NEXT: local.get $push[[L1:[0-9]+]]=, 1{{$}}
 ; CHECK-NEXT: f32.ne $push[[NUM0:[0-9]+]]=, $pop[[L0]], $pop[[L1]]{{$}}
-; CHECK-NEXT: get_local $push[[L2:[0-9]+]]=, 0{{$}}
-; CHECK-NEXT: get_local $push[[L3:[0-9]+]]=, 0{{$}}
+; CHECK-NEXT: local.get $push[[L2:[0-9]+]]=, 0{{$}}
+; CHECK-NEXT: local.get $push[[L3:[0-9]+]]=, 0{{$}}
 ; CHECK-NEXT: f32.eq $push[[NUM1:[0-9]+]]=, $pop[[L2]], $pop[[L3]]{{$}}
-; CHECK-NEXT: get_local $push[[L4:[0-9]+]]=, 1{{$}}
-; CHECK-NEXT: get_local $push[[L5:[0-9]+]]=, 1{{$}}
+; CHECK-NEXT: local.get $push[[L4:[0-9]+]]=, 1{{$}}
+; CHECK-NEXT: local.get $push[[L5:[0-9]+]]=, 1{{$}}
 ; CHECK-NEXT: f32.eq $push[[NUM2:[0-9]+]]=, $pop[[L4]], $pop[[L5]]{{$}}
 ; CHECK-NEXT: i32.and $push[[NUM3:[0-9]+]]=, $pop[[NUM1]], $pop[[NUM2]]{{$}}
 ; CHECK-NEXT: i32.and $push[[NUM4:[0-9]+]]=, $pop[[NUM0]], $pop[[NUM3]]{{$}}
@@ -140,8 +140,8 @@ define i32 @one_f32(float %x, float %y) {
 
 ; CHECK-LABEL: ult_f32:
 ; CHECK-NEXT: .functype ult_f32 (f32, f32) -> (i32){{$}}
-; CHECK-NEXT: get_local $push[[L0:[0-9]+]]=, 0{{$}}
-; CHECK-NEXT: get_local $push[[L1:[0-9]+]]=, 1{{$}}
+; CHECK-NEXT: local.get $push[[L0:[0-9]+]]=, 0{{$}}
+; CHECK-NEXT: local.get $push[[L1:[0-9]+]]=, 1{{$}}
 ; CHECK-NEXT: f32.ge $push[[NUM0:[0-9]+]]=, $pop[[L0]], $pop[[L1]]{{$}}
 ; CHECK-NEXT: i32.const $push[[C0:[0-9]+]]=, 1
 ; CHECK-NEXT: i32.xor $push[[NUM2:[0-9]+]]=, $pop[[NUM0]], $pop[[C0]]{{$}}
@@ -154,8 +154,8 @@ define i32 @ult_f32(float %x, float %y) {
 
 ; CHECK-LABEL: ule_f32:
 ; CHECK-NEXT: .functype ule_f32 (f32, f32) -> (i32){{$}}
-; CHECK-NEXT: get_local $push[[L0:[0-9]+]]=, 0{{$}}
-; CHECK-NEXT: get_local $push[[L1:[0-9]+]]=, 1{{$}}
+; CHECK-NEXT: local.get $push[[L0:[0-9]+]]=, 0{{$}}
+; CHECK-NEXT: local.get $push[[L1:[0-9]+]]=, 1{{$}}
 ; CHECK-NEXT: f32.gt $push[[NUM0:[0-9]+]]=, $pop[[L0]], $pop[[L1]]{{$}}
 ; CHECK-NEXT: i32.const $push[[C0:[0-9]+]]=, 1
 ; CHECK-NEXT: i32.xor $push[[NUM2:[0-9]+]]=, $pop[[NUM0]], $pop[[C0]]{{$}}
@@ -168,8 +168,8 @@ define i32 @ule_f32(float %x, float %y) {
 
 ; CHECK-LABEL: ugt_f32:
 ; CHECK-NEXT: .functype ugt_f32 (f32, f32) -> (i32){{$}}
-; CHECK-NEXT: get_local $push[[L0:[0-9]+]]=, 0{{$}}
-; CHECK-NEXT: get_local $push[[L1:[0-9]+]]=, 1{{$}}
+; CHECK-NEXT: local.get $push[[L0:[0-9]+]]=, 0{{$}}
+; CHECK-NEXT: local.get $push[[L1:[0-9]+]]=, 1{{$}}
 ; CHECK-NEXT: f32.le $push[[NUM0:[0-9]+]]=, $pop[[L0]], $pop[[L1]]{{$}}
 ; CHECK-NEXT: i32.const $push[[C0:[0-9]+]]=, 1
 ; CHECK-NEXT: i32.xor $push[[NUM2:[0-9]+]]=, $pop[[NUM0]], $pop[[C0]]{{$}}
@@ -182,8 +182,8 @@ define i32 @ugt_f32(float %x, float %y) {
 
 ; CHECK-LABEL: uge_f32:
 ; CHECK-NEXT: .functype uge_f32 (f32, f32) -> (i32){{$}}
-; CHECK-NEXT: get_local $push[[L0:[0-9]+]]=, 0{{$}}
-; CHECK-NEXT: get_local $push[[L1:[0-9]+]]=, 1{{$}}
+; CHECK-NEXT: local.get $push[[L0:[0-9]+]]=, 0{{$}}
+; CHECK-NEXT: local.get $push[[L1:[0-9]+]]=, 1{{$}}
 ; CHECK-NEXT: f32.lt $push[[NUM0:[0-9]+]]=, $pop[[L0]], $pop[[L1]]{{$}}
 ; CHECK-NEXT: i32.const $push[[C0:[0-9]+]]=, 1
 ; CHECK-NEXT: i32.xor $push[[NUM2:[0-9]+]]=, $pop[[NUM0]], $pop[[C0]]{{$}}
diff --git a/test/CodeGen/WebAssembly/comparisons-f64.ll b/test/CodeGen/WebAssembly/comparisons-f64.ll
index 46b24db67c1c43ca7a39c8add80400ec394f4927..37150e8cded70f33bcd974d792f735832f7341ad 100644
--- a/test/CodeGen/WebAssembly/comparisons-f64.ll
+++ b/test/CodeGen/WebAssembly/comparisons-f64.ll
@@ -8,11 +8,11 @@ target triple = "wasm32-unknown-unknown"
 
 ; CHECK-LABEL: ord_f64:
 ; CHECK-NEXT: .functype ord_f64 (f64, f64) -> (i32){{$}}
-; CHECK-NEXT: get_local $push[[L0:[0-9]+]]=, 0{{$}}
-; CHECK-NEXT: get_local $push[[L1:[0-9]+]]=, 0{{$}}
+; CHECK-NEXT: local.get $push[[L0:[0-9]+]]=, 0{{$}}
+; CHECK-NEXT: local.get $push[[L1:[0-9]+]]=, 0{{$}}
 ; CHECK-NEXT: f64.eq $push[[NUM0:[0-9]+]]=, $pop[[L0]], $pop[[L1]]{{$}}
-; CHECK-NEXT: get_local $push[[L2:[0-9]+]]=, 1{{$}}
-; CHECK-NEXT: get_local $push[[L3:[0-9]+]]=, 1{{$}}
+; CHECK-NEXT: local.get $push[[L2:[0-9]+]]=, 1{{$}}
+; CHECK-NEXT: local.get $push[[L3:[0-9]+]]=, 1{{$}}
 ; CHECK-NEXT: f64.eq $push[[NUM1:[0-9]+]]=, $pop[[L2]], $pop[[L3]]{{$}}
 ; CHECK-NEXT: i32.and $push[[NUM2:[0-9]+]]=, $pop[[NUM0]], $pop[[NUM1]]{{$}}
 ; CHECK-NEXT: return $pop[[NUM2]]{{$}}
@@ -24,11 +24,11 @@ define i32 @ord_f64(double %x, double %y) {
 
 ; CHECK-LABEL: uno_f64:
 ; CHECK-NEXT: .functype uno_f64 (f64, f64) -> (i32){{$}}
-; CHECK-NEXT: get_local $push[[L0:[0-9]+]]=, 0{{$}}
-; CHECK-NEXT: get_local $push[[L1:[0-9]+]]=, 0{{$}}
+; CHECK-NEXT: local.get $push[[L0:[0-9]+]]=, 0{{$}}
+; CHECK-NEXT: local.get $push[[L1:[0-9]+]]=, 0{{$}}
 ; CHECK-NEXT: f64.ne $push[[NUM0:[0-9]+]]=, $pop[[L0]], $pop[[L1]]{{$}}
-; CHECK-NEXT: get_local $push[[L2:[0-9]+]]=, 1{{$}}
-; CHECK-NEXT: get_local $push[[L3:[0-9]+]]=, 1{{$}}
+; CHECK-NEXT: local.get $push[[L2:[0-9]+]]=, 1{{$}}
+; CHECK-NEXT: local.get $push[[L3:[0-9]+]]=, 1{{$}}
 ; CHECK-NEXT: f64.ne $push[[NUM1:[0-9]+]]=, $pop[[L2]], $pop[[L3]]{{$}}
 ; CHECK-NEXT: i32.or $push[[NUM2:[0-9]+]]=, $pop[[NUM0]], $pop[[NUM1]]{{$}}
 ; CHECK-NEXT: return $pop[[NUM2]]{{$}}
@@ -40,8 +40,8 @@ define i32 @uno_f64(double %x, double %y) {
 
 ; CHECK-LABEL: oeq_f64:
 ; CHECK-NEXT: .functype oeq_f64 (f64, f64) -> (i32){{$}}
-; CHECK-NEXT: get_local $push[[L0:[0-9]+]]=, 0{{$}}
-; CHECK-NEXT: get_local $push[[L1:[0-9]+]]=, 1{{$}}
+; CHECK-NEXT: local.get $push[[L0:[0-9]+]]=, 0{{$}}
+; CHECK-NEXT: local.get $push[[L1:[0-9]+]]=, 1{{$}}
 ; CHECK-NEXT: f64.eq $push[[NUM:[0-9]+]]=, $pop[[L0]], $pop[[L1]]{{$}}
 ; CHECK-NEXT: return $pop[[NUM]]{{$}}
 define i32 @oeq_f64(double %x, double %y) {
@@ -99,14 +99,14 @@ define i32 @oge_f64(double %x, double %y) {
 
 ; CHECK-LABEL: ueq_f64:
 ; CHECK-NEXT: .functype ueq_f64 (f64, f64) -> (i32){{$}}
-; CHECK-NEXT: get_local $push[[L0:[0-9]+]]=, 0{{$}}
-; CHECK-NEXT: get_local $push[[L1:[0-9]+]]=, 1{{$}}
+; CHECK-NEXT: local.get $push[[L0:[0-9]+]]=, 0{{$}}
+; CHECK-NEXT: local.get $push[[L1:[0-9]+]]=, 1{{$}}
 ; CHECK-NEXT: f64.eq $push[[NUM0:[0-9]+]]=, $pop[[L0]], $pop[[L1]]{{$}}
-; CHECK-NEXT: get_local $push[[L2:[0-9]+]]=, 0{{$}}
-; CHECK-NEXT: get_local $push[[L3:[0-9]+]]=, 0{{$}}
+; CHECK-NEXT: local.get $push[[L2:[0-9]+]]=, 0{{$}}
+; CHECK-NEXT: local.get $push[[L3:[0-9]+]]=, 0{{$}}
 ; CHECK-NEXT: f64.ne $push[[NUM1:[0-9]+]]=, $pop[[L2]], $pop[[L3]]{{$}}
-; CHECK-NEXT: get_local $push[[L4:[0-9]+]]=, 1{{$}}
-; CHECK-NEXT: get_local $push[[L5:[0-9]+]]=, 1{{$}}
+; CHECK-NEXT: local.get $push[[L4:[0-9]+]]=, 1{{$}}
+; CHECK-NEXT: local.get $push[[L5:[0-9]+]]=, 1{{$}}
 ; CHECK-NEXT: f64.ne $push[[NUM2:[0-9]+]]=, $pop[[L4]], $pop[[L5]]{{$}}
 ; CHECK-NEXT: i32.or $push[[NUM3:[0-9]+]]=, $pop[[NUM1]], $pop[[NUM2]]{{$}}
 ; CHECK-NEXT: i32.or $push[[NUM4:[0-9]+]]=, $pop[[NUM0]], $pop[[NUM3]]{{$}}
@@ -119,14 +119,14 @@ define i32 @ueq_f64(double %x, double %y) {
 
 ; CHECK-LABEL: one_f64:
 ; CHECK-NEXT: .functype one_f64 (f64, f64) -> (i32){{$}}
-; CHECK-NEXT: get_local $push[[L0:[0-9]+]]=, 0{{$}}
-; CHECK-NEXT: get_local $push[[L1:[0-9]+]]=, 1{{$}}
+; CHECK-NEXT: local.get $push[[L0:[0-9]+]]=, 0{{$}}
+; CHECK-NEXT: local.get $push[[L1:[0-9]+]]=, 1{{$}}
 ; CHECK-NEXT: f64.ne $push[[NUM0:[0-9]+]]=, $pop[[L0]], $pop[[L1]]{{$}}
-; CHECK-NEXT: get_local $push[[L2:[0-9]+]]=, 0{{$}}
-; CHECK-NEXT: get_local $push[[L3:[0-9]+]]=, 0{{$}}
+; CHECK-NEXT: local.get $push[[L2:[0-9]+]]=, 0{{$}}
+; CHECK-NEXT: local.get $push[[L3:[0-9]+]]=, 0{{$}}
 ; CHECK-NEXT: f64.eq $push[[NUM1:[0-9]+]]=, $pop[[L2]], $pop[[L3]]{{$}}
-; CHECK-NEXT: get_local $push[[L4:[0-9]+]]=, 1{{$}}
-; CHECK-NEXT: get_local $push[[L5:[0-9]+]]=, 1{{$}}
+; CHECK-NEXT: local.get $push[[L4:[0-9]+]]=, 1{{$}}
+; CHECK-NEXT: local.get $push[[L5:[0-9]+]]=, 1{{$}}
 ; CHECK-NEXT: f64.eq $push[[NUM2:[0-9]+]]=, $pop[[L4]], $pop[[L5]]{{$}}
 ; CHECK-NEXT: i32.and $push[[NUM3:[0-9]+]]=, $pop[[NUM1]], $pop[[NUM2]]{{$}}
 ; CHECK-NEXT: i32.and $push[[NUM4:[0-9]+]]=, $pop[[NUM0]], $pop[[NUM3]]{{$}}
@@ -139,8 +139,8 @@ define i32 @one_f64(double %x, double %y) {
 
 ; CHECK-LABEL: ult_f64:
 ; CHECK-NEXT: .functype ult_f64 (f64, f64) -> (i32){{$}}
-; CHECK-NEXT: get_local $push[[L0:[0-9]+]]=, 0{{$}}
-; CHECK-NEXT: get_local $push[[L1:[0-9]+]]=, 1{{$}}
+; CHECK-NEXT: local.get $push[[L0:[0-9]+]]=, 0{{$}}
+; CHECK-NEXT: local.get $push[[L1:[0-9]+]]=, 1{{$}}
 ; CHECK-NEXT: f64.ge $push[[NUM0:[0-9]+]]=, $pop[[L0]], $pop[[L1]]{{$}}
 ; CHECK-NEXT: i32.const $push[[C0:[0-9]+]]=, 1
 ; CHECK-NEXT: i32.xor $push[[NUM2:[0-9]+]]=, $pop[[NUM0]], $pop[[C0]]{{$}}
@@ -153,8 +153,8 @@ define i32 @ult_f64(double %x, double %y) {
 
 ; CHECK-LABEL: ule_f64:
 ; CHECK-NEXT: .functype ule_f64 (f64, f64) -> (i32){{$}}
-; CHECK-NEXT: get_local $push[[L0:[0-9]+]]=, 0{{$}}
-; CHECK-NEXT: get_local $push[[L1:[0-9]+]]=, 1{{$}}
+; CHECK-NEXT: local.get $push[[L0:[0-9]+]]=, 0{{$}}
+; CHECK-NEXT: local.get $push[[L1:[0-9]+]]=, 1{{$}}
 ; CHECK-NEXT: f64.gt $push[[NUM0:[0-9]+]]=, $pop[[L0]], $pop[[L1]]{{$}}
 ; CHECK-NEXT: i32.const $push[[C0:[0-9]+]]=, 1
 ; CHECK-NEXT: i32.xor $push[[NUM2:[0-9]+]]=, $pop[[NUM0]], $pop[[C0]]{{$}}
@@ -167,8 +167,8 @@ define i32 @ule_f64(double %x, double %y) {
 
 ; CHECK-LABEL: ugt_f64:
 ; CHECK-NEXT: .functype ugt_f64 (f64, f64) -> (i32){{$}}
-; CHECK-NEXT: get_local $push[[L0:[0-9]+]]=, 0{{$}}
-; CHECK-NEXT: get_local $push[[L1:[0-9]+]]=, 1{{$}}
+; CHECK-NEXT: local.get $push[[L0:[0-9]+]]=, 0{{$}}
+; CHECK-NEXT: local.get $push[[L1:[0-9]+]]=, 1{{$}}
 ; CHECK-NEXT: f64.le $push[[NUM0:[0-9]+]]=, $pop[[L0]], $pop[[L1]]{{$}}
 ; CHECK-NEXT: i32.const $push[[C0:[0-9]+]]=, 1
 ; CHECK-NEXT: i32.xor $push[[NUM2:[0-9]+]]=, $pop[[NUM0]], $pop[[C0]]{{$}}
@@ -181,8 +181,8 @@ define i32 @ugt_f64(double %x, double %y) {
 
 ; CHECK-LABEL: uge_f64:
 ; CHECK-NEXT: .functype uge_f64 (f64, f64) -> (i32){{$}}
-; CHECK-NEXT: get_local $push[[L0:[0-9]+]]=, 0{{$}}
-; CHECK-NEXT: get_local $push[[L1:[0-9]+]]=, 1{{$}}
+; CHECK-NEXT: local.get $push[[L0:[0-9]+]]=, 0{{$}}
+; CHECK-NEXT: local.get $push[[L1:[0-9]+]]=, 1{{$}}
 ; CHECK-NEXT: f64.lt $push[[NUM0:[0-9]+]]=, $pop[[L0]], $pop[[L1]]{{$}}
 ; CHECK-NEXT: i32.const $push[[C0:[0-9]+]]=, 1
 ; CHECK-NEXT: i32.xor $push[[NUM2:[0-9]+]]=, $pop[[NUM0]], $pop[[C0]]{{$}}
diff --git a/test/CodeGen/WebAssembly/comparisons-i32.ll b/test/CodeGen/WebAssembly/comparisons-i32.ll
index e7d4e1b04a4e1d159ff6a33d7238e7dc54db38ae..ed884e751b2cd1fe460bff45edbd7de4060bcbcc 100644
--- a/test/CodeGen/WebAssembly/comparisons-i32.ll
+++ b/test/CodeGen/WebAssembly/comparisons-i32.ll
@@ -1,5 +1,5 @@
-; RUN: llc < %s -asm-verbose=false -disable-wasm-fallthrough-return-opt -wasm-keep-registers | FileCheck %s
-; RUN: llc < %s -asm-verbose=false -disable-wasm-fallthrough-return-opt -wasm-keep-registers -fast-isel -fast-isel-abort=1 | FileCheck %s
+; RUN: llc < %s -asm-verbose=false -disable-wasm-fallthrough-return-opt -wasm-keep-registers | FileCheck %s --check-prefixes CHECK,SLOW
+; RUN: llc < %s -asm-verbose=false -disable-wasm-fallthrough-return-opt -wasm-keep-registers -fast-isel -fast-isel-abort=1 | FileCheck %s --check-prefixes CHECK,FAST
 
 ; Test that basic 32-bit integer comparison operations assemble as expected.
 
@@ -8,10 +8,13 @@ target triple = "wasm32-unknown-unknown"
 
 ; CHECK-LABEL: eq_i32:
 ; CHECK-NEXT: .functype eq_i32 (i32, i32) -> (i32){{$}}
-; CHECK-NEXT: get_local $push[[L0:[0-9]+]]=, 0{{$}}
-; CHECK-NEXT: get_local $push[[L1:[0-9]+]]=, 1{{$}}
-; CHECK-NEXT: i32.eq $push[[NUM:[0-9]+]]=, $pop[[L0]], $pop[[L1]]{{$}}
-; CHECK-NEXT: return $pop[[NUM]]{{$}}
+; CHECK-NEXT: local.get $push[[L0:[0-9]+]]=, 0{{$}}
+; CHECK-NEXT: local.get $push[[L1:[0-9]+]]=, 1{{$}}
+; CHECK-NEXT: i32.eq $push[[L2:[0-9]+]]=, $pop[[L0]], $pop[[L1]]{{$}}
+; SLOW-NEXT: return $pop[[L2]]{{$}}
+; FAST-NEXT:  i32.const $push[[L3:[0-9]+]]=, 1{{$}}
+; FAST-NEXT:  i32.and $push[[L4:[0-9]+]]=, $pop[[L2]], $pop[[L3]]{{$}}
+; FAST-NEXT:  return $pop[[L4]]{{$}}
 define i32 @eq_i32(i32 %x, i32 %y) {
   %a = icmp eq i32 %x, %y
   %b = zext i1 %a to i32
@@ -19,8 +22,11 @@ define i32 @eq_i32(i32 %x, i32 %y) {
 }
 
 ; CHECK-LABEL: ne_i32:
-; CHECK: i32.ne $push[[NUM:[0-9]+]]=, $pop{{[0-9]+}}, $pop{{[0-9]+}}{{$}}
-; CHECK-NEXT: return $pop[[NUM]]{{$}}
+; CHECK: i32.ne $push[[L0:[0-9]+]]=, $pop{{[0-9]+}}, $pop{{[0-9]+}}{{$}}
+; SLOW-NEXT: return $pop[[L0]]{{$}}
+; FAST-NEXT: i32.const $push[[L1:[0-9]+]]=, 1{{$}}
+; FAST-NEXT: i32.and $push[[L2:[0-9]+]]=, $pop[[L0]], $pop[[L1]]{{$}}
+; FAST-NEXT: return $pop[[L2]]{{$}}
 define i32 @ne_i32(i32 %x, i32 %y) {
   %a = icmp ne i32 %x, %y
   %b = zext i1 %a to i32
@@ -28,8 +34,11 @@ define i32 @ne_i32(i32 %x, i32 %y) {
 }
 
 ; CHECK-LABEL: slt_i32:
-; CHECK: i32.lt_s $push[[NUM:[0-9]+]]=, $pop{{[0-9]+}}, $pop{{[0-9]+}}{{$}}
-; CHECK-NEXT: return $pop[[NUM]]{{$}}
+; CHECK: i32.lt_s $push[[L0:[0-9]+]]=, $pop{{[0-9]+}}, $pop{{[0-9]+}}{{$}}
+; SLOW-NEXT: return $pop[[L0]]{{$}}
+; FAST-NEXT: i32.const $push[[L1:[0-9]+]]=, 1{{$}}
+; FAST-NEXT: i32.and $push[[L2:[0-9]+]]=, $pop[[L0]], $pop[[L1]]{{$}}
+; FAST-NEXT: return $pop[[L2]]{{$}}
 define i32 @slt_i32(i32 %x, i32 %y) {
   %a = icmp slt i32 %x, %y
   %b = zext i1 %a to i32
@@ -37,8 +46,11 @@ define i32 @slt_i32(i32 %x, i32 %y) {
 }
 
 ; CHECK-LABEL: sle_i32:
-; CHECK: i32.le_s $push[[NUM:[0-9]+]]=, $pop{{[0-9]+}}, $pop{{[0-9]+}}{{$}}
-; CHECK-NEXT: return $pop[[NUM]]{{$}}
+; CHECK: i32.le_s $push[[L0:[0-9]+]]=, $pop{{[0-9]+}}, $pop{{[0-9]+}}{{$}}
+; SLOW-NEXT: return $pop[[L0]]{{$}}
+; FAST-NEXT: i32.const $push[[L1:[0-9]+]]=, 1{{$}}
+; FAST-NEXT: i32.and $push[[L2:[0-9]+]]=, $pop[[L0]], $pop[[L1]]{{$}}
+; FAST-NEXT: return $pop[[L2]]{{$}}
 define i32 @sle_i32(i32 %x, i32 %y) {
   %a = icmp sle i32 %x, %y
   %b = zext i1 %a to i32
@@ -46,8 +58,11 @@ define i32 @sle_i32(i32 %x, i32 %y) {
 }
 
 ; CHECK-LABEL: ult_i32:
-; CHECK: i32.lt_u $push[[NUM:[0-9]+]]=, $pop{{[0-9]+}}, $pop{{[0-9]+}}{{$}}
-; CHECK-NEXT: return $pop[[NUM]]{{$}}
+; CHECK: i32.lt_u $push[[L0:[0-9]+]]=, $pop{{[0-9]+}}, $pop{{[0-9]+}}{{$}}
+; SLOW-NEXT: return $pop[[L0]]{{$}}
+; FAST-NEXT: i32.const $push[[L1:[0-9]+]]=, 1{{$}}
+; FAST-NEXT: i32.and $push[[L2:[0-9]+]]=, $pop[[L0]], $pop[[L1]]{{$}}
+; FAST-NEXT: return $pop[[L2]]{{$}}
 define i32 @ult_i32(i32 %x, i32 %y) {
   %a = icmp ult i32 %x, %y
   %b = zext i1 %a to i32
@@ -55,8 +70,11 @@ define i32 @ult_i32(i32 %x, i32 %y) {
 }
 
 ; CHECK-LABEL: ule_i32:
-; CHECK: i32.le_u $push[[NUM:[0-9]+]]=, $pop{{[0-9]+}}, $pop{{[0-9]+}}{{$}}
-; CHECK-NEXT: return $pop[[NUM]]{{$}}
+; CHECK: i32.le_u $push[[L0:[0-9]+]]=, $pop{{[0-9]+}}, $pop{{[0-9]+}}{{$}}
+; SLOW-NEXT: return $pop[[L0]]{{$}}
+; FAST-NEXT: i32.const $push[[L1:[0-9]+]]=, 1{{$}}
+; FAST-NEXT: i32.and $push[[L2:[0-9]+]]=, $pop[[L0]], $pop[[L1]]{{$}}
+; FAST-NEXT: return $pop[[L2]]{{$}}
 define i32 @ule_i32(i32 %x, i32 %y) {
   %a = icmp ule i32 %x, %y
   %b = zext i1 %a to i32
@@ -64,8 +82,11 @@ define i32 @ule_i32(i32 %x, i32 %y) {
 }
 
 ; CHECK-LABEL: sgt_i32:
-; CHECK: i32.gt_s $push[[NUM:[0-9]+]]=, $pop{{[0-9]+}}, $pop{{[0-9]+}}{{$}}
-; CHECK-NEXT: return $pop[[NUM]]{{$}}
+; CHECK: i32.gt_s $push[[L0:[0-9]+]]=, $pop{{[0-9]+}}, $pop{{[0-9]+}}{{$}}
+; SLOW-NEXT: return $pop[[L0]]{{$}}
+; FAST-NEXT: i32.const $push[[L1:[0-9]+]]=, 1{{$}}
+; FAST-NEXT: i32.and $push[[L2:[0-9]+]]=, $pop[[L0]], $pop[[L1]]{{$}}
+; FAST-NEXT: return $pop[[L2]]{{$}}
 define i32 @sgt_i32(i32 %x, i32 %y) {
   %a = icmp sgt i32 %x, %y
   %b = zext i1 %a to i32
@@ -73,8 +94,11 @@ define i32 @sgt_i32(i32 %x, i32 %y) {
 }
 
 ; CHECK-LABEL: sge_i32:
-; CHECK: i32.ge_s $push[[NUM:[0-9]+]]=, $pop{{[0-9]+}}, $pop{{[0-9]+}}{{$}}
-; CHECK-NEXT: return $pop[[NUM]]{{$}}
+; CHECK: i32.ge_s $push[[L0:[0-9]+]]=, $pop{{[0-9]+}}, $pop{{[0-9]+}}{{$}}
+; SLOW-NEXT: return $pop[[L0]]{{$}}
+; FAST-NEXT: i32.const $push[[L1:[0-9]+]]=, 1{{$}}
+; FAST-NEXT: i32.and $push[[L2:[0-9]+]]=, $pop[[L0]], $pop[[L1]]{{$}}
+; FAST-NEXT: return $pop[[L2]]{{$}}
 define i32 @sge_i32(i32 %x, i32 %y) {
   %a = icmp sge i32 %x, %y
   %b = zext i1 %a to i32
@@ -82,8 +106,11 @@ define i32 @sge_i32(i32 %x, i32 %y) {
 }
 
 ; CHECK-LABEL: ugt_i32:
-; CHECK: i32.gt_u $push[[NUM:[0-9]+]]=, $pop{{[0-9]+}}, $pop{{[0-9]+}}{{$}}
-; CHECK-NEXT: return $pop[[NUM]]{{$}}
+; CHECK: i32.gt_u $push[[L0:[0-9]+]]=, $pop{{[0-9]+}}, $pop{{[0-9]+}}{{$}}
+; SLOW-NEXT: return $pop[[L0]]{{$}}
+; FAST-NEXT: i32.const $push[[L1:[0-9]+]]=, 1{{$}}
+; FAST-NEXT: i32.and $push[[L2:[0-9]+]]=, $pop[[L0]], $pop[[L1]]{{$}}
+; FAST-NEXT: return $pop[[L2]]{{$}}
 define i32 @ugt_i32(i32 %x, i32 %y) {
   %a = icmp ugt i32 %x, %y
   %b = zext i1 %a to i32
@@ -91,8 +118,11 @@ define i32 @ugt_i32(i32 %x, i32 %y) {
 }
 
 ; CHECK-LABEL: uge_i32:
-; CHECK: i32.ge_u $push[[NUM:[0-9]+]]=, $pop{{[0-9]+}}, $pop{{[0-9]+}}{{$}}
-; CHECK-NEXT: return $pop[[NUM]]{{$}}
+; CHECK: i32.ge_u $push[[L0:[0-9]+]]=, $pop{{[0-9]+}}, $pop{{[0-9]+}}{{$}}
+; SLOW-NEXT: return $pop[[L0]]{{$}}
+; FAST-NEXT: i32.const $push[[L1:[0-9]+]]=, 1{{$}}
+; FAST-NEXT: i32.and $push[[L2:[0-9]+]]=, $pop[[L0]], $pop[[L1]]{{$}}
+; FAST-NEXT: return $pop[[L2]]{{$}}
 define i32 @uge_i32(i32 %x, i32 %y) {
   %a = icmp uge i32 %x, %y
   %b = zext i1 %a to i32
diff --git a/test/CodeGen/WebAssembly/comparisons-i64.ll b/test/CodeGen/WebAssembly/comparisons-i64.ll
index ffd460fe9abd14100ae479e2a134ac17ae3592e8..899cc063f5041aaf67fe202d3251cfe30261c09e 100644
--- a/test/CodeGen/WebAssembly/comparisons-i64.ll
+++ b/test/CodeGen/WebAssembly/comparisons-i64.ll
@@ -1,5 +1,5 @@
-; RUN: llc < %s -asm-verbose=false -disable-wasm-fallthrough-return-opt -wasm-keep-registers | FileCheck %s
-; RUN: llc < %s -asm-verbose=false -disable-wasm-fallthrough-return-opt -wasm-keep-registers -fast-isel -fast-isel-abort=1 | FileCheck %s
+; RUN: llc < %s -asm-verbose=false -disable-wasm-fallthrough-return-opt -wasm-keep-registers | FileCheck %s --check-prefixes CHECK,SLOW
+; RUN: llc < %s -asm-verbose=false -disable-wasm-fallthrough-return-opt -wasm-keep-registers -fast-isel -fast-isel-abort=1 | FileCheck %s --check-prefixes CHECK,FAST
 
 ; Test that basic 64-bit integer comparison operations assemble as expected.
 
@@ -8,10 +8,13 @@ target triple = "wasm32-unknown-unknown"
 
 ; CHECK-LABEL: eq_i64:
 ; CHECK-NEXT: .functype eq_i64 (i64, i64) -> (i32){{$}}
-; CHECK-NEXT: get_local $push[[L0:[0-9]+]]=, 0{{$}}
-; CHECK-NEXT: get_local $push[[L1:[0-9]+]]=, 1{{$}}
-; CHECK-NEXT: i64.eq $push[[NUM:[0-9]+]]=, $pop[[L0]], $pop[[L1]]{{$}}
-; CHECK-NEXT: return $pop[[NUM]]{{$}}
+; CHECK-NEXT: local.get $push[[L0:[0-9]+]]=, 0{{$}}
+; CHECK-NEXT: local.get $push[[L1:[0-9]+]]=, 1{{$}}
+; CHECK-NEXT: i64.eq $push[[L2:[0-9]+]]=, $pop[[L0]], $pop[[L1]]{{$}}
+; SLOW-NEXT:  return $pop[[L2]]{{$}}
+; FAST-NEXT:  i32.const $push[[L3:[0-9]+]]=, 1{{$}}
+; FAST-NEXT:  i32.and $push[[L4:[0-9]+]]=, $pop[[L2]], $pop[[L3]]{{$}}
+; FAST-NEXT:  return $pop[[L4]]{{$}}
 define i32 @eq_i64(i64 %x, i64 %y) {
   %a = icmp eq i64 %x, %y
   %b = zext i1 %a to i32
@@ -19,8 +22,11 @@ define i32 @eq_i64(i64 %x, i64 %y) {
 }
 
 ; CHECK-LABEL: ne_i64:
-; CHECK: i64.ne $push[[NUM:[0-9]+]]=, $pop{{[0-9]+}}, $pop{{[0-9]+}}{{$}}
-; CHECK-NEXT: return $pop[[NUM]]{{$}}
+; CHECK: i64.ne $push[[L0:[0-9]+]]=, $pop{{[0-9]+}}, $pop{{[0-9]+}}{{$}}
+; SLOW-NEXT: return $pop[[L0]]{{$}}
+; FAST-NEXT: i32.const $push[[L1:[0-9]+]]=, 1{{$}}
+; FAST-NEXT: i32.and $push[[L2:[0-9]+]]=, $pop[[L0]], $pop[[L1]]{{$}}
+; FAST-NEXT: return $pop[[L2]]{{$}}
 define i32 @ne_i64(i64 %x, i64 %y) {
   %a = icmp ne i64 %x, %y
   %b = zext i1 %a to i32
@@ -28,8 +34,11 @@ define i32 @ne_i64(i64 %x, i64 %y) {
 }
 
 ; CHECK-LABEL: slt_i64:
-; CHECK: i64.lt_s $push[[NUM:[0-9]+]]=, $pop{{[0-9]+}}, $pop{{[0-9]+}}{{$}}
-; CHECK-NEXT: return $pop[[NUM]]{{$}}
+; CHECK: i64.lt_s $push[[L0:[0-9]+]]=, $pop{{[0-9]+}}, $pop{{[0-9]+}}{{$}}
+; SLOW-NEXT: return $pop[[L0]]{{$}}
+; FAST-NEXT: i32.const $push[[L1:[0-9]+]]=, 1{{$}}
+; FAST-NEXT: i32.and $push[[L2:[0-9]+]]=, $pop[[L0]], $pop[[L1]]{{$}}
+; FAST-NEXT: return $pop[[L2]]{{$}}
 define i32 @slt_i64(i64 %x, i64 %y) {
   %a = icmp slt i64 %x, %y
   %b = zext i1 %a to i32
@@ -37,8 +46,11 @@ define i32 @slt_i64(i64 %x, i64 %y) {
 }
 
 ; CHECK-LABEL: sle_i64:
-; CHECK: i64.le_s $push[[NUM:[0-9]+]]=, $pop{{[0-9]+}}, $pop{{[0-9]+}}{{$}}
-; CHECK-NEXT: return $pop[[NUM]]{{$}}
+; CHECK: i64.le_s $push[[L0:[0-9]+]]=, $pop{{[0-9]+}}, $pop{{[0-9]+}}{{$}}
+; SLOW-NEXT: return $pop[[L0]]{{$}}
+; FAST-NEXT: i32.const $push[[L1:[0-9]+]]=, 1{{$}}
+; FAST-NEXT: i32.and $push[[L2:[0-9]+]]=, $pop[[L0]], $pop[[L1]]{{$}}
+; FAST-NEXT: return $pop[[L2]]{{$}}
 define i32 @sle_i64(i64 %x, i64 %y) {
   %a = icmp sle i64 %x, %y
   %b = zext i1 %a to i32
@@ -46,8 +58,11 @@ define i32 @sle_i64(i64 %x, i64 %y) {
 }
 
 ; CHECK-LABEL: ult_i64:
-; CHECK: i64.lt_u $push[[NUM:[0-9]+]]=, $pop{{[0-9]+}}, $pop{{[0-9]+}}{{$}}
-; CHECK-NEXT: return $pop[[NUM]]{{$}}
+; CHECK: i64.lt_u $push[[L0:[0-9]+]]=, $pop{{[0-9]+}}, $pop{{[0-9]+}}{{$}}
+; SLOW-NEXT: return $pop[[L0]]{{$}}
+; FAST-NEXT: i32.const $push[[L1:[0-9]+]]=, 1{{$}}
+; FAST-NEXT: i32.and $push[[L2:[0-9]+]]=, $pop[[L0]], $pop[[L1]]{{$}}
+; FAST-NEXT: return $pop[[L2]]{{$}}
 define i32 @ult_i64(i64 %x, i64 %y) {
   %a = icmp ult i64 %x, %y
   %b = zext i1 %a to i32
@@ -55,8 +70,11 @@ define i32 @ult_i64(i64 %x, i64 %y) {
 }
 
 ; CHECK-LABEL: ule_i64:
-; CHECK: i64.le_u $push[[NUM:[0-9]+]]=, $pop{{[0-9]+}}, $pop{{[0-9]+}}{{$}}
-; CHECK-NEXT: return $pop[[NUM]]{{$}}
+; CHECK: i64.le_u $push[[L0:[0-9]+]]=, $pop{{[0-9]+}}, $pop{{[0-9]+}}{{$}}
+; SLOW-NEXT: return $pop[[L0]]{{$}}
+; FAST-NEXT: i32.const $push[[L1:[0-9]+]]=, 1{{$}}
+; FAST-NEXT: i32.and $push[[L2:[0-9]+]]=, $pop[[L0]], $pop[[L1]]{{$}}
+; FAST-NEXT: return $pop[[L2]]{{$}}
 define i32 @ule_i64(i64 %x, i64 %y) {
   %a = icmp ule i64 %x, %y
   %b = zext i1 %a to i32
@@ -64,8 +82,11 @@ define i32 @ule_i64(i64 %x, i64 %y) {
 }
 
 ; CHECK-LABEL: sgt_i64:
-; CHECK: i64.gt_s $push[[NUM:[0-9]+]]=, $pop{{[0-9]+}}, $pop{{[0-9]+}}{{$}}
-; CHECK-NEXT: return $pop[[NUM]]{{$}}
+; CHECK: i64.gt_s $push[[L0:[0-9]+]]=, $pop{{[0-9]+}}, $pop{{[0-9]+}}{{$}}
+; SLOW-NEXT: return $pop[[L0]]{{$}}
+; FAST-NEXT: i32.const $push[[L1:[0-9]+]]=, 1{{$}}
+; FAST-NEXT: i32.and $push[[L2:[0-9]+]]=, $pop[[L0]], $pop[[L1]]{{$}}
+; FAST-NEXT: return $pop[[L2]]{{$}}
 define i32 @sgt_i64(i64 %x, i64 %y) {
   %a = icmp sgt i64 %x, %y
   %b = zext i1 %a to i32
@@ -73,8 +94,11 @@ define i32 @sgt_i64(i64 %x, i64 %y) {
 }
 
 ; CHECK-LABEL: sge_i64:
-; CHECK: i64.ge_s $push[[NUM:[0-9]+]]=, $pop{{[0-9]+}}, $pop{{[0-9]+}}{{$}}
-; CHECK-NEXT: return $pop[[NUM]]{{$}}
+; CHECK: i64.ge_s $push[[L0:[0-9]+]]=, $pop{{[0-9]+}}, $pop{{[0-9]+}}{{$}}
+; SLOW-NEXT: return $pop[[L0]]{{$}}
+; FAST-NEXT: i32.const $push[[L1:[0-9]+]]=, 1{{$}}
+; FAST-NEXT: i32.and $push[[L2:[0-9]+]]=, $pop[[L0]], $pop[[L1]]{{$}}
+; FAST-NEXT: return $pop[[L2]]{{$}}
 define i32 @sge_i64(i64 %x, i64 %y) {
   %a = icmp sge i64 %x, %y
   %b = zext i1 %a to i32
@@ -82,8 +106,11 @@ define i32 @sge_i64(i64 %x, i64 %y) {
 }
 
 ; CHECK-LABEL: ugt_i64:
-; CHECK: i64.gt_u $push[[NUM:[0-9]+]]=, $pop{{[0-9]+}}, $pop{{[0-9]+}}{{$}}
-; CHECK-NEXT: return $pop[[NUM]]{{$}}
+; CHECK: i64.gt_u $push[[L0:[0-9]+]]=, $pop{{[0-9]+}}, $pop{{[0-9]+}}{{$}}
+; SLOW-NEXT: return $pop[[L0]]{{$}}
+; FAST-NEXT: i32.const $push[[L1:[0-9]+]]=, 1{{$}}
+; FAST-NEXT: i32.and $push[[L2:[0-9]+]]=, $pop[[L0]], $pop[[L1]]{{$}}
+; FAST-NEXT: return $pop[[L2]]{{$}}
 define i32 @ugt_i64(i64 %x, i64 %y) {
   %a = icmp ugt i64 %x, %y
   %b = zext i1 %a to i32
@@ -91,8 +118,11 @@ define i32 @ugt_i64(i64 %x, i64 %y) {
 }
 
 ; CHECK-LABEL: uge_i64:
-; CHECK: i64.ge_u $push[[NUM:[0-9]+]]=, $pop{{[0-9]+}}, $pop{{[0-9]+}}{{$}}
-; CHECK-NEXT: return $pop[[NUM]]{{$}}
+; CHECK: i64.ge_u $push[[L0:[0-9]+]]=, $pop{{[0-9]+}}, $pop{{[0-9]+}}{{$}}
+; SLOW-NEXT: return $pop[[L0]]{{$}}
+; FAST-NEXT: i32.const $push[[L1:[0-9]+]]=, 1{{$}}
+; FAST-NEXT: i32.and $push[[L2:[0-9]+]]=, $pop[[L0]], $pop[[L1]]{{$}}
+; FAST-NEXT: return $pop[[L2]]{{$}}
 define i32 @uge_i64(i64 %x, i64 %y) {
   %a = icmp uge i64 %x, %y
   %b = zext i1 %a to i32
diff --git a/test/CodeGen/WebAssembly/conv-trap.ll b/test/CodeGen/WebAssembly/conv-trap.ll
index ca87f6b8dd94bd7522c13d72e908f6ee527092f1..aa589de7f369a8e5e700a3d291cba71dcab6d0ad 100644
--- a/test/CodeGen/WebAssembly/conv-trap.ll
+++ b/test/CodeGen/WebAssembly/conv-trap.ll
@@ -17,7 +17,7 @@ target triple = "wasm32-unknown-unknown"
 ; CHECK-NEXT: return $pop[[ALT]]{{$}}
 ; CHECK-NEXT: BB
 ; CHECK-NEXT: end_block
-; CHECK-NEXT: i32.trunc_s/f32 $push[[NUM:[0-9]+]]=, $0{{$}}
+; CHECK-NEXT: i32.trunc_f32_s $push[[NUM:[0-9]+]]=, $0{{$}}
 ; CHECK-NEXT: return $pop[[NUM]]{{$}}
 define i32 @i32_trunc_s_f32(float %x) {
   %a = fptosi float %x to i32
@@ -37,7 +37,7 @@ define i32 @i32_trunc_s_f32(float %x) {
 ; CHECK-NEXT: return $pop[[ALT]]{{$}}
 ; CHECK-NEXT: BB
 ; CHECK-NEXT: end_block
-; CHECK-NEXT: i32.trunc_u/f32 $push[[NUM:[0-9]+]]=, $0{{$}}
+; CHECK-NEXT: i32.trunc_f32_u $push[[NUM:[0-9]+]]=, $0{{$}}
 ; CHECK-NEXT: return $pop[[NUM]]{{$}}
 define i32 @i32_trunc_u_f32(float %x) {
   %a = fptoui float %x to i32
@@ -55,7 +55,7 @@ define i32 @i32_trunc_u_f32(float %x) {
 ; CHECK-NEXT: return $pop[[ALT]]{{$}}
 ; CHECK-NEXT: BB
 ; CHECK-NEXT: end_block
-; CHECK-NEXT: i32.trunc_s/f64 $push[[NUM:[0-9]+]]=, $0{{$}}
+; CHECK-NEXT: i32.trunc_f64_s $push[[NUM:[0-9]+]]=, $0{{$}}
 ; CHECK-NEXT: return $pop[[NUM]]{{$}}
 define i32 @i32_trunc_s_f64(double %x) {
   %a = fptosi double %x to i32
@@ -75,7 +75,7 @@ define i32 @i32_trunc_s_f64(double %x) {
 ; CHECK-NEXT: return $pop[[ALT]]{{$}}
 ; CHECK-NEXT: BB
 ; CHECK-NEXT: end_block
-; CHECK-NEXT: i32.trunc_u/f64 $push[[NUM:[0-9]+]]=, $0{{$}}
+; CHECK-NEXT: i32.trunc_f64_u $push[[NUM:[0-9]+]]=, $0{{$}}
 ; CHECK-NEXT: return $pop[[NUM]]{{$}}
 define i32 @i32_trunc_u_f64(double %x) {
   %a = fptoui double %x to i32
@@ -93,7 +93,7 @@ define i32 @i32_trunc_u_f64(double %x) {
 ; CHECK-NEXT: return $pop[[ALT]]{{$}}
 ; CHECK-NEXT: BB
 ; CHECK-NEXT: end_block
-; CHECK-NEXT: i64.trunc_s/f32 $push[[NUM:[0-9]+]]=, $0{{$}}
+; CHECK-NEXT: i64.trunc_f32_s $push[[NUM:[0-9]+]]=, $0{{$}}
 ; CHECK-NEXT: return $pop[[NUM]]{{$}}
 define i64 @i64_trunc_s_f32(float %x) {
   %a = fptosi float %x to i64
@@ -113,7 +113,7 @@ define i64 @i64_trunc_s_f32(float %x) {
 ; CHECK-NEXT: return $pop[[ALT]]{{$}}
 ; CHECK-NEXT: BB
 ; CHECK-NEXT: end_block
-; CHECK-NEXT: i64.trunc_u/f32 $push[[NUM:[0-9]+]]=, $0{{$}}
+; CHECK-NEXT: i64.trunc_f32_u $push[[NUM:[0-9]+]]=, $0{{$}}
 ; CHECK-NEXT: return $pop[[NUM]]{{$}}
 define i64 @i64_trunc_u_f32(float %x) {
   %a = fptoui float %x to i64
@@ -131,7 +131,7 @@ define i64 @i64_trunc_u_f32(float %x) {
 ; CHECK-NEXT: return $pop[[ALT]]{{$}}
 ; CHECK-NEXT: BB
 ; CHECK-NEXT: end_block
-; CHECK-NEXT: i64.trunc_s/f64 $push[[NUM:[0-9]+]]=, $0{{$}}
+; CHECK-NEXT: i64.trunc_f64_s $push[[NUM:[0-9]+]]=, $0{{$}}
 ; CHECK-NEXT: return $pop[[NUM]]{{$}}
 define i64 @i64_trunc_s_f64(double %x) {
   %a = fptosi double %x to i64
@@ -151,7 +151,7 @@ define i64 @i64_trunc_s_f64(double %x) {
 ; CHECK-NEXT: return $pop[[ALT]]{{$}}
 ; CHECK-NEXT: BB
 ; CHECK-NEXT: end_block
-; CHECK-NEXT: i64.trunc_u/f64 $push[[NUM:[0-9]+]]=, $0{{$}}
+; CHECK-NEXT: i64.trunc_f64_u $push[[NUM:[0-9]+]]=, $0{{$}}
 ; CHECK-NEXT: return $pop[[NUM]]{{$}}
 define i64 @i64_trunc_u_f64(double %x) {
   %a = fptoui double %x to i64
diff --git a/test/CodeGen/WebAssembly/conv.ll b/test/CodeGen/WebAssembly/conv.ll
index 8b729a67194bd5e54287a563679cc1a493f7f1b6..68f941546ce108dc888dff04f57830f3cb16dbdf 100644
--- a/test/CodeGen/WebAssembly/conv.ll
+++ b/test/CodeGen/WebAssembly/conv.ll
@@ -7,7 +7,7 @@ target triple = "wasm32-unknown-unknown"
 
 ; CHECK-LABEL: i32_wrap_i64:
 ; CHECK-NEXT: .functype i32_wrap_i64 (i64) -> (i32){{$}}
-; CHECK-NEXT: i32.wrap/i64 $push[[NUM:[0-9]+]]=, $0{{$}}
+; CHECK-NEXT: i32.wrap_i64 $push[[NUM:[0-9]+]]=, $0{{$}}
 ; CHECK-NEXT: return $pop[[NUM]]{{$}}
 define i32 @i32_wrap_i64(i64 %x) {
   %a = trunc i64 %x to i32
@@ -16,7 +16,7 @@ define i32 @i32_wrap_i64(i64 %x) {
 
 ; CHECK-LABEL: i64_extend_s_i32:
 ; CHECK-NEXT: .functype i64_extend_s_i32 (i32) -> (i64){{$}}
-; CHECK-NEXT: i64.extend_s/i32 $push[[NUM:[0-9]+]]=, $0{{$}}
+; CHECK-NEXT: i64.extend_i32_s $push[[NUM:[0-9]+]]=, $0{{$}}
 ; CHECK-NEXT: return $pop[[NUM]]{{$}}
 define i64 @i64_extend_s_i32(i32 %x) {
   %a = sext i32 %x to i64
@@ -25,7 +25,7 @@ define i64 @i64_extend_s_i32(i32 %x) {
 
 ; CHECK-LABEL: i64_extend_u_i32:
 ; CHECK-NEXT: .functype i64_extend_u_i32 (i32) -> (i64){{$}}
-; CHECK-NEXT: i64.extend_u/i32 $push[[NUM:[0-9]+]]=, $0{{$}}
+; CHECK-NEXT: i64.extend_i32_u $push[[NUM:[0-9]+]]=, $0{{$}}
 ; CHECK-NEXT: return $pop[[NUM]]{{$}}
 define i64 @i64_extend_u_i32(i32 %x) {
   %a = zext i32 %x to i64
@@ -34,7 +34,7 @@ define i64 @i64_extend_u_i32(i32 %x) {
 
 ; CHECK-LABEL: i32_trunc_s_f32:
 ; CHECK-NEXT: .functype i32_trunc_s_f32 (f32) -> (i32){{$}}
-; CHECK-NEXT: i32.trunc_s:sat/f32 $push[[NUM:[0-9]+]]=, $0{{$}}
+; CHECK-NEXT: i32.trunc_sat_f32_s $push[[NUM:[0-9]+]]=, $0{{$}}
 ; CHECK-NEXT: return $pop[[NUM]]{{$}}
 define i32 @i32_trunc_s_f32(float %x) {
   %a = fptosi float %x to i32
@@ -43,7 +43,7 @@ define i32 @i32_trunc_s_f32(float %x) {
 
 ; CHECK-LABEL: i32_trunc_sat_s_f32:
 ; CHECK-NEXT: .functype i32_trunc_sat_s_f32 (f32) -> (i32){{$}}
-; CHECK-NEXT: i32.trunc_s:sat/f32 $push[[NUM:[0-9]+]]=, $0{{$}}
+; CHECK-NEXT: i32.trunc_sat_f32_s $push[[NUM:[0-9]+]]=, $0{{$}}
 ; CHECK-NEXT: return $pop[[NUM]]{{$}}
 declare i32 @llvm.wasm.trunc.saturate.signed.i32.f32(float)
 define i32 @i32_trunc_sat_s_f32(float %x) {
@@ -53,7 +53,7 @@ define i32 @i32_trunc_sat_s_f32(float %x) {
 
 ; CHECK-LABEL: i32_trunc_u_f32:
 ; CHECK-NEXT: .functype i32_trunc_u_f32 (f32) -> (i32){{$}}
-; CHECK-NEXT: i32.trunc_u:sat/f32 $push[[NUM:[0-9]+]]=, $0{{$}}
+; CHECK-NEXT: i32.trunc_sat_f32_u $push[[NUM:[0-9]+]]=, $0{{$}}
 ; CHECK-NEXT: return $pop[[NUM]]{{$}}
 define i32 @i32_trunc_u_f32(float %x) {
   %a = fptoui float %x to i32
@@ -62,7 +62,7 @@ define i32 @i32_trunc_u_f32(float %x) {
 
 ; CHECK-LABEL: i32_trunc_sat_u_f32:
 ; CHECK-NEXT: .functype i32_trunc_sat_u_f32 (f32) -> (i32){{$}}
-; CHECK-NEXT: i32.trunc_u:sat/f32 $push[[NUM:[0-9]+]]=, $0{{$}}
+; CHECK-NEXT: i32.trunc_sat_f32_u $push[[NUM:[0-9]+]]=, $0{{$}}
 ; CHECK-NEXT: return $pop[[NUM]]{{$}}
 declare i32 @llvm.wasm.trunc.saturate.unsigned.i32.f32(float)
 define i32 @i32_trunc_sat_u_f32(float %x) {
@@ -72,7 +72,7 @@ define i32 @i32_trunc_sat_u_f32(float %x) {
 
 ; CHECK-LABEL: i32_trunc_s_f64:
 ; CHECK-NEXT: .functype i32_trunc_s_f64 (f64) -> (i32){{$}}
-; CHECK-NEXT: i32.trunc_s:sat/f64 $push[[NUM:[0-9]+]]=, $0{{$}}
+; CHECK-NEXT: i32.trunc_sat_f64_s $push[[NUM:[0-9]+]]=, $0{{$}}
 ; CHECK-NEXT: return $pop[[NUM]]{{$}}
 define i32 @i32_trunc_s_f64(double %x) {
   %a = fptosi double %x to i32
@@ -81,7 +81,7 @@ define i32 @i32_trunc_s_f64(double %x) {
 
 ; CHECK-LABEL: i32_trunc_sat_s_f64:
 ; CHECK-NEXT: .functype i32_trunc_sat_s_f64 (f64) -> (i32){{$}}
-; CHECK-NEXT: i32.trunc_s:sat/f64 $push[[NUM:[0-9]+]]=, $0{{$}}
+; CHECK-NEXT: i32.trunc_sat_f64_s $push[[NUM:[0-9]+]]=, $0{{$}}
 ; CHECK-NEXT: return $pop[[NUM]]{{$}}
 declare i32 @llvm.wasm.trunc.saturate.signed.i32.f64(double)
 define i32 @i32_trunc_sat_s_f64(double %x) {
@@ -91,7 +91,7 @@ define i32 @i32_trunc_sat_s_f64(double %x) {
 
 ; CHECK-LABEL: i32_trunc_u_f64:
 ; CHECK-NEXT: .functype i32_trunc_u_f64 (f64) -> (i32){{$}}
-; CHECK-NEXT: i32.trunc_u:sat/f64 $push[[NUM:[0-9]+]]=, $0{{$}}
+; CHECK-NEXT: i32.trunc_sat_f64_u $push[[NUM:[0-9]+]]=, $0{{$}}
 ; CHECK-NEXT: return $pop[[NUM]]{{$}}
 define i32 @i32_trunc_u_f64(double %x) {
   %a = fptoui double %x to i32
@@ -100,7 +100,7 @@ define i32 @i32_trunc_u_f64(double %x) {
 
 ; CHECK-LABEL: i32_trunc_sat_u_f64:
 ; CHECK-NEXT: .functype i32_trunc_sat_u_f64 (f64) -> (i32){{$}}
-; CHECK-NEXT: i32.trunc_u:sat/f64 $push[[NUM:[0-9]+]]=, $0{{$}}
+; CHECK-NEXT: i32.trunc_sat_f64_u $push[[NUM:[0-9]+]]=, $0{{$}}
 ; CHECK-NEXT: return $pop[[NUM]]{{$}}
 declare i32 @llvm.wasm.trunc.saturate.unsigned.i32.f64(double)
 define i32 @i32_trunc_sat_u_f64(double %x) {
@@ -110,7 +110,7 @@ define i32 @i32_trunc_sat_u_f64(double %x) {
 
 ; CHECK-LABEL: i64_trunc_s_f32:
 ; CHECK-NEXT: .functype i64_trunc_s_f32 (f32) -> (i64){{$}}
-; CHECK-NEXT: i64.trunc_s:sat/f32 $push[[NUM:[0-9]+]]=, $0{{$}}
+; CHECK-NEXT: i64.trunc_sat_f32_s $push[[NUM:[0-9]+]]=, $0{{$}}
 ; CHECK-NEXT: return $pop[[NUM]]{{$}}
 define i64 @i64_trunc_s_f32(float %x) {
   %a = fptosi float %x to i64
@@ -119,7 +119,7 @@ define i64 @i64_trunc_s_f32(float %x) {
 
 ; CHECK-LABEL: i64_trunc_sat_s_f32:
 ; CHECK-NEXT: .functype i64_trunc_sat_s_f32 (f32) -> (i64){{$}}
-; CHECK-NEXT: i64.trunc_s:sat/f32 $push[[NUM:[0-9]+]]=, $0{{$}}
+; CHECK-NEXT: i64.trunc_sat_f32_s $push[[NUM:[0-9]+]]=, $0{{$}}
 ; CHECK-NEXT: return $pop[[NUM]]{{$}}
 declare i64 @llvm.wasm.trunc.saturate.signed.i64.f32(float)
 define i64 @i64_trunc_sat_s_f32(float %x) {
@@ -129,7 +129,7 @@ define i64 @i64_trunc_sat_s_f32(float %x) {
 
 ; CHECK-LABEL: i64_trunc_u_f32:
 ; CHECK-NEXT: .functype i64_trunc_u_f32 (f32) -> (i64){{$}}
-; CHECK-NEXT: i64.trunc_u:sat/f32 $push[[NUM:[0-9]+]]=, $0{{$}}
+; CHECK-NEXT: i64.trunc_sat_f32_u $push[[NUM:[0-9]+]]=, $0{{$}}
 ; CHECK-NEXT: return $pop[[NUM]]{{$}}
 define i64 @i64_trunc_u_f32(float %x) {
   %a = fptoui float %x to i64
@@ -138,7 +138,7 @@ define i64 @i64_trunc_u_f32(float %x) {
 
 ; CHECK-LABEL: i64_trunc_sat_u_f32:
 ; CHECK-NEXT: .functype i64_trunc_sat_u_f32 (f32) -> (i64){{$}}
-; CHECK-NEXT: i64.trunc_u:sat/f32 $push[[NUM:[0-9]+]]=, $0{{$}}
+; CHECK-NEXT: i64.trunc_sat_f32_u $push[[NUM:[0-9]+]]=, $0{{$}}
 ; CHECK-NEXT: return $pop[[NUM]]{{$}}
 declare i64 @llvm.wasm.trunc.saturate.unsigned.i64.f32(float)
 define i64 @i64_trunc_sat_u_f32(float %x) {
@@ -148,7 +148,7 @@ define i64 @i64_trunc_sat_u_f32(float %x) {
 
 ; CHECK-LABEL: i64_trunc_s_f64:
 ; CHECK-NEXT: .functype i64_trunc_s_f64 (f64) -> (i64){{$}}
-; CHECK-NEXT: i64.trunc_s:sat/f64 $push[[NUM:[0-9]+]]=, $0{{$}}
+; CHECK-NEXT: i64.trunc_sat_f64_s $push[[NUM:[0-9]+]]=, $0{{$}}
 ; CHECK-NEXT: return $pop[[NUM]]{{$}}
 define i64 @i64_trunc_s_f64(double %x) {
   %a = fptosi double %x to i64
@@ -157,7 +157,7 @@ define i64 @i64_trunc_s_f64(double %x) {
 
 ; CHECK-LABEL: i64_trunc_sat_s_f64:
 ; CHECK-NEXT: .functype i64_trunc_sat_s_f64 (f64) -> (i64){{$}}
-; CHECK-NEXT: i64.trunc_s:sat/f64 $push[[NUM:[0-9]+]]=, $0{{$}}
+; CHECK-NEXT: i64.trunc_sat_f64_s $push[[NUM:[0-9]+]]=, $0{{$}}
 ; CHECK-NEXT: return $pop[[NUM]]{{$}}
 declare i64 @llvm.wasm.trunc.saturate.signed.i64.f64(double)
 define i64 @i64_trunc_sat_s_f64(double %x) {
@@ -167,7 +167,7 @@ define i64 @i64_trunc_sat_s_f64(double %x) {
 
 ; CHECK-LABEL: i64_trunc_u_f64:
 ; CHECK-NEXT: .functype i64_trunc_u_f64 (f64) -> (i64){{$}}
-; CHECK-NEXT: i64.trunc_u:sat/f64 $push[[NUM:[0-9]+]]=, $0{{$}}
+; CHECK-NEXT: i64.trunc_sat_f64_u $push[[NUM:[0-9]+]]=, $0{{$}}
 ; CHECK-NEXT: return $pop[[NUM]]{{$}}
 define i64 @i64_trunc_u_f64(double %x) {
   %a = fptoui double %x to i64
@@ -176,7 +176,7 @@ define i64 @i64_trunc_u_f64(double %x) {
 
 ; CHECK-LABEL: i64_trunc_sat_u_f64:
 ; CHECK-NEXT: .functype i64_trunc_sat_u_f64 (f64) -> (i64){{$}}
-; CHECK-NEXT: i64.trunc_u:sat/f64 $push[[NUM:[0-9]+]]=, $0{{$}}
+; CHECK-NEXT: i64.trunc_sat_f64_u $push[[NUM:[0-9]+]]=, $0{{$}}
 ; CHECK-NEXT: return $pop[[NUM]]{{$}}
 declare i64 @llvm.wasm.trunc.saturate.unsigned.i64.f64(double)
 define i64 @i64_trunc_sat_u_f64(double %x) {
@@ -186,7 +186,7 @@ define i64 @i64_trunc_sat_u_f64(double %x) {
 
 ; CHECK-LABEL: f32_convert_s_i32:
 ; CHECK-NEXT: .functype f32_convert_s_i32 (i32) -> (f32){{$}}
-; CHECK-NEXT: f32.convert_s/i32 $push[[NUM:[0-9]+]]=, $0{{$}}
+; CHECK-NEXT: f32.convert_i32_s $push[[NUM:[0-9]+]]=, $0{{$}}
 ; CHECK-NEXT: return $pop[[NUM]]{{$}}
 define float @f32_convert_s_i32(i32 %x) {
   %a = sitofp i32 %x to float
@@ -195,7 +195,7 @@ define float @f32_convert_s_i32(i32 %x) {
 
 ; CHECK-LABEL: f32_convert_u_i32:
 ; CHECK-NEXT: .functype f32_convert_u_i32 (i32) -> (f32){{$}}
-; CHECK-NEXT: f32.convert_u/i32 $push[[NUM:[0-9]+]]=, $0{{$}}
+; CHECK-NEXT: f32.convert_i32_u $push[[NUM:[0-9]+]]=, $0{{$}}
 ; CHECK-NEXT: return $pop[[NUM]]{{$}}
 define float @f32_convert_u_i32(i32 %x) {
   %a = uitofp i32 %x to float
@@ -204,7 +204,7 @@ define float @f32_convert_u_i32(i32 %x) {
 
 ; CHECK-LABEL: f64_convert_s_i32:
 ; CHECK-NEXT: .functype f64_convert_s_i32 (i32) -> (f64){{$}}
-; CHECK-NEXT: f64.convert_s/i32 $push[[NUM:[0-9]+]]=, $0{{$}}
+; CHECK-NEXT: f64.convert_i32_s $push[[NUM:[0-9]+]]=, $0{{$}}
 ; CHECK-NEXT: return $pop[[NUM]]{{$}}
 define double @f64_convert_s_i32(i32 %x) {
   %a = sitofp i32 %x to double
@@ -213,7 +213,7 @@ define double @f64_convert_s_i32(i32 %x) {
 
 ; CHECK-LABEL: f64_convert_u_i32:
 ; CHECK-NEXT: .functype f64_convert_u_i32 (i32) -> (f64){{$}}
-; CHECK-NEXT: f64.convert_u/i32 $push[[NUM:[0-9]+]]=, $0{{$}}
+; CHECK-NEXT: f64.convert_i32_u $push[[NUM:[0-9]+]]=, $0{{$}}
 ; CHECK-NEXT: return $pop[[NUM]]{{$}}
 define double @f64_convert_u_i32(i32 %x) {
   %a = uitofp i32 %x to double
@@ -222,7 +222,7 @@ define double @f64_convert_u_i32(i32 %x) {
 
 ; CHECK-LABEL: f32_convert_s_i64:
 ; CHECK-NEXT: .functype f32_convert_s_i64 (i64) -> (f32){{$}}
-; CHECK-NEXT: f32.convert_s/i64 $push[[NUM:[0-9]+]]=, $0{{$}}
+; CHECK-NEXT: f32.convert_i64_s $push[[NUM:[0-9]+]]=, $0{{$}}
 ; CHECK-NEXT: return $pop[[NUM]]{{$}}
 define float @f32_convert_s_i64(i64 %x) {
   %a = sitofp i64 %x to float
@@ -231,7 +231,7 @@ define float @f32_convert_s_i64(i64 %x) {
 
 ; CHECK-LABEL: f32_convert_u_i64:
 ; CHECK-NEXT: .functype f32_convert_u_i64 (i64) -> (f32){{$}}
-; CHECK-NEXT: f32.convert_u/i64 $push[[NUM:[0-9]+]]=, $0{{$}}
+; CHECK-NEXT: f32.convert_i64_u $push[[NUM:[0-9]+]]=, $0{{$}}
 ; CHECK-NEXT: return $pop[[NUM]]{{$}}
 define float @f32_convert_u_i64(i64 %x) {
   %a = uitofp i64 %x to float
@@ -240,7 +240,7 @@ define float @f32_convert_u_i64(i64 %x) {
 
 ; CHECK-LABEL: f64_convert_s_i64:
 ; CHECK-NEXT: .functype f64_convert_s_i64 (i64) -> (f64){{$}}
-; CHECK-NEXT: f64.convert_s/i64 $push[[NUM:[0-9]+]]=, $0{{$}}
+; CHECK-NEXT: f64.convert_i64_s $push[[NUM:[0-9]+]]=, $0{{$}}
 ; CHECK-NEXT: return $pop[[NUM]]{{$}}
 define double @f64_convert_s_i64(i64 %x) {
   %a = sitofp i64 %x to double
@@ -249,7 +249,7 @@ define double @f64_convert_s_i64(i64 %x) {
 
 ; CHECK-LABEL: f64_convert_u_i64:
 ; CHECK-NEXT: .functype f64_convert_u_i64 (i64) -> (f64){{$}}
-; CHECK-NEXT: f64.convert_u/i64 $push[[NUM:[0-9]+]]=, $0{{$}}
+; CHECK-NEXT: f64.convert_i64_u $push[[NUM:[0-9]+]]=, $0{{$}}
 ; CHECK-NEXT: return $pop[[NUM]]{{$}}
 define double @f64_convert_u_i64(i64 %x) {
   %a = uitofp i64 %x to double
@@ -258,7 +258,7 @@ define double @f64_convert_u_i64(i64 %x) {
 
 ; CHECK-LABEL: f64_promote_f32:
 ; CHECK-NEXT: .functype f64_promote_f32 (f32) -> (f64){{$}}
-; CHECK-NEXT: f64.promote/f32 $push[[NUM:[0-9]+]]=, $0{{$}}
+; CHECK-NEXT: f64.promote_f32 $push[[NUM:[0-9]+]]=, $0{{$}}
 ; CHECK-NEXT: return $pop[[NUM]]{{$}}
 define double @f64_promote_f32(float %x) {
   %a = fpext float %x to double
@@ -267,7 +267,7 @@ define double @f64_promote_f32(float %x) {
 
 ; CHECK-LABEL: f32_demote_f64:
 ; CHECK-NEXT: .functype f32_demote_f64 (f64) -> (f32){{$}}
-; CHECK-NEXT: f32.demote/f64 $push[[NUM:[0-9]+]]=, $0{{$}}
+; CHECK-NEXT: f32.demote_f64 $push[[NUM:[0-9]+]]=, $0{{$}}
 ; CHECK-NEXT: return $pop[[NUM]]{{$}}
 define float @f32_demote_f64(double %x) {
   %a = fptrunc double %x to float
@@ -278,7 +278,7 @@ define float @f32_demote_f64(double %x) {
 ; we need to patterm-match back to a specific instruction.
 
 ; CHECK-LABEL: anyext:
-; CHECK: i64.extend_u/i32 $push0=, $0{{$}}
+; CHECK: i64.extend_i32_u $push0=, $0{{$}}
 define i64 @anyext(i32 %x) {
     %y = sext i32 %x to i64
     %w = shl i64 %y, 32
@@ -286,28 +286,28 @@ define i64 @anyext(i32 %x) {
 }
 
 ; CHECK-LABEL: bitcast_i32_to_float:
-; CHECK: f32.reinterpret/i32   $push0=, $0{{$}}
+; CHECK: f32.reinterpret_i32   $push0=, $0{{$}}
 define float @bitcast_i32_to_float(i32 %a) {
   %t = bitcast i32 %a to float
   ret float %t
 }
 
 ; CHECK-LABEL: bitcast_float_to_i32:
-; CHECK: i32.reinterpret/f32   $push0=, $0{{$}}
+; CHECK: i32.reinterpret_f32   $push0=, $0{{$}}
 define i32 @bitcast_float_to_i32(float %a) {
   %t = bitcast float %a to i32
   ret i32 %t
 }
 
 ; CHECK-LABEL: bitcast_i64_to_double:
-; CHECK: f64.reinterpret/i64   $push0=, $0{{$}}
+; CHECK: f64.reinterpret_i64   $push0=, $0{{$}}
 define double @bitcast_i64_to_double(i64 %a) {
   %t = bitcast i64 %a to double
   ret double %t
 }
 
 ; CHECK-LABEL: bitcast_double_to_i64:
-; CHECK: i64.reinterpret/f64   $push0=, $0{{$}}
+; CHECK: i64.reinterpret_f64   $push0=, $0{{$}}
 define i64 @bitcast_double_to_i64(double %a) {
   %t = bitcast double %a to i64
   ret i64 %t
diff --git a/test/CodeGen/WebAssembly/copysign-casts.ll b/test/CodeGen/WebAssembly/copysign-casts.ll
index 381d880d62ba38d6134627a0322bfe22c76cd18e..51aeb86bfad89a06321ebff662fe909749dcdc34 100644
--- a/test/CodeGen/WebAssembly/copysign-casts.ll
+++ b/test/CodeGen/WebAssembly/copysign-casts.ll
@@ -10,7 +10,7 @@ declare double @copysign(double, double) nounwind readnone
 declare float @copysignf(float, float) nounwind readnone
 
 ; CHECK-LABEL: fold_promote:
-; CHECK: f64.promote/f32 $push0=, $pop{{[0-9]+}}{{$}}
+; CHECK: f64.promote_f32 $push0=, $pop{{[0-9]+}}{{$}}
 ; CHECK: f64.copysign    $push1=, $pop{{[0-9]+}}, $pop0{{$}}
 define double @fold_promote(double %a, float %b) {
   %c = fpext float %b to double
@@ -19,7 +19,7 @@ define double @fold_promote(double %a, float %b) {
 }
 
 ; CHECK-LABEL: fold_demote:{{$}}
-; CHECK: f32.demote/f64  $push0=, $pop{{[0-9]+}}{{$}}
+; CHECK: f32.demote_f64  $push0=, $pop{{[0-9]+}}{{$}}
 ; CHECK: f32.copysign    $push1=, $pop{{[0-9]+}}, $pop0{{$}}
 define float @fold_demote(float %a, double %b) {
   %c = fptrunc double %b to float
diff --git a/test/CodeGen/WebAssembly/custom-sections.ll b/test/CodeGen/WebAssembly/custom-sections.ll
index cf21b0af4fb12e2303d9b12770508c4e8402b54d..78fe3c5458fdb972cfa685f9d96f144798be9160 100644
--- a/test/CodeGen/WebAssembly/custom-sections.ll
+++ b/test/CodeGen/WebAssembly/custom-sections.ll
@@ -10,6 +10,16 @@ target triple = "wasm32-unknown-unknown"
 !2 = !{ !"green", !"qux" }
 !wasm.custom_sections = !{ !0, !1, !2 }
 
+!llvm.ident = !{!3}
+!3 = !{!"clang version 123"}
+
+!llvm.module.flags = !{!4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+
+!llvm.dbg.cu = !{!5}
+!5 = distinct !DICompileUnit(language: DW_LANG_C99, file: !6)
+!6 = !DIFile(filename: "test", directory: "testdir")
+
 ; CHECK:      .section	.custom_section.red,"",@
 ; CHECK-NEXT: .ascii	"foo"
 
@@ -18,3 +28,19 @@ target triple = "wasm32-unknown-unknown"
 
 ; CHECK:      .section	.custom_section.green,"",@
 ; CHECK-NEXT: .ascii	"qux"
+
+; CHECK:      .section	.custom_section.producers,"",@
+; CHECK-NEXT: .int8	2
+; CHECK-NEXT: .int8	8
+; CHECK-NEXT: .ascii	"language"
+; CHECK-NEXT: .int8	1
+; CHECK-NEXT: .int8	3
+; CHECK-NEXT: .ascii	"C99"
+; CHECK-NEXT: .int8	0
+; CHECK-NEXT: .int8	12
+; CHECK-NEXT: .ascii	"processed-by"
+; CHECK-NEXT: .int8	1
+; CHECK-NEXT: .int8	5
+; CHECK-NEXT: .ascii	"clang"
+; CHECK-NEXT: .int8	3
+; CHECK-NEXT: .ascii	"123"
diff --git a/test/CodeGen/WebAssembly/exception.ll b/test/CodeGen/WebAssembly/exception.ll
index b0365fc9d43c9766fe903bb3b3add4602d030953..e2a48ed47d6c2533a030aa94da05626d508b4f76 100644
--- a/test/CodeGen/WebAssembly/exception.ll
+++ b/test/CodeGen/WebAssembly/exception.ll
@@ -1,5 +1,5 @@
 ; RUN: not llc < %s -asm-verbose=false -disable-wasm-fallthrough-return-opt -wasm-keep-registers -exception-model=wasm
-; RUN: llc < %s -asm-verbose=false -disable-wasm-fallthrough-return-opt -wasm-keep-registers -exception-model=wasm -mattr=+exception-handling -verify-machineinstrs | FileCheck -allow-deprecated-dag-overlap %s
+; RUN: llc < %s -asm-verbose=false -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-keep-registers -exception-model=wasm -mattr=+exception-handling -verify-machineinstrs | FileCheck -allow-deprecated-dag-overlap %s
 ; RUN: llc < %s -disable-wasm-fallthrough-return-opt -wasm-keep-registers -exception-model=wasm -mattr=+exception-handling
 
 target datalayout = "e-m:e-p:32:32-i64:64-n32:64-S128"
@@ -9,30 +9,39 @@ target triple = "wasm32-unknown-unknown"
 
 @_ZTIi = external constant i8*
 
-declare void @llvm.wasm.throw(i32, i8*)
-
 ; CHECK-LABEL: test_throw:
-; CHECK:      get_local $push0=, 0
-; CHECK-NEXT: throw __cpp_exception@EVENT, $pop0
+; CHECK: throw __cpp_exception@EVENT, $0
 ; CHECK-NOT:  unreachable
 define void @test_throw(i8* %p) {
   call void @llvm.wasm.throw(i32 0, i8* %p)
   ret void
 }
 
+; CHECK-LABEL: test_rethrow:
+; CHECK:      rethrow
+; CHECK-NOT:  unreachable
+define void @test_rethrow(i8* %p) {
+  call void @llvm.wasm.rethrow()
+  ret void
+}
+
 ; CHECK-LABEL: test_catch_rethrow:
-; CHECK:   get_global  $push{{.+}}=, __stack_pointer@GLOBAL
+; CHECK:   global.get  ${{.+}}=, __stack_pointer@GLOBAL
 ; CHECK:   try
 ; CHECK:   call      foo@FUNCTION
-; CHECK:   i32.catch     $push{{.+}}=, 0
-; CHECK:   set_global  __stack_pointer@GLOBAL
+; CHECK:   catch     $[[EXCEPT_REF:[0-9]+]]=
+; CHECK:   global.set  __stack_pointer@GLOBAL
+; CHECK:   block i32
+; CHECK:   br_on_exn 0, __cpp_exception@EVENT, $[[EXCEPT_REF]]
+; CHECK:   rethrow
+; CHECK:   end_block
+; CHECK:   extract_exception $[[EXN:[0-9]+]]=
 ; CHECK-DAG:   i32.store  __wasm_lpad_context
 ; CHECK-DAG:   i32.store  __wasm_lpad_context+4
-; CHECK:   i32.call  $push{{.+}}=, _Unwind_CallPersonality@FUNCTION
-; CHECK:   i32.call  $push{{.+}}=, __cxa_begin_catch@FUNCTION
+; CHECK:   i32.call  $drop=, _Unwind_CallPersonality@FUNCTION, $[[EXN]]
+; CHECK:   i32.call  $drop=, __cxa_begin_catch@FUNCTION
 ; CHECK:   call      __cxa_end_catch@FUNCTION
 ; CHECK:   call      __cxa_rethrow@FUNCTION
-; CHECK-NEXT:   rethrow
 ; CHECK:   end_try
 define void @test_catch_rethrow() personality i8* bitcast (i32 (...)* @__gxx_wasm_personality_v0 to i8*) {
 entry:
@@ -66,9 +75,9 @@ try.cont:                                         ; preds = %entry, %catch
 ; CHECK-LABEL: test_cleanup:
 ; CHECK:   try
 ; CHECK:   call      foo@FUNCTION
-; CHECK:   catch_all
-; CHECK:   set_global  __stack_pointer@GLOBAL
-; CHECK:   i32.call  $push{{.+}}=, _ZN7CleanupD1Ev@FUNCTION
+; CHECK:   catch
+; CHECK:   global.set  __stack_pointer@GLOBAL
+; CHECK:   i32.call  $drop=, _ZN7CleanupD1Ev@FUNCTION
 ; CHECK:   rethrow
 ; CHECK:   end_try
 define void @test_cleanup() personality i8* bitcast (i32 (...)* @__gxx_wasm_personality_v0 to i8*) {
@@ -87,71 +96,51 @@ ehcleanup:                                        ; preds = %entry
   cleanupret from %0 unwind to caller
 }
 
-; - Tests multple terminate pads are merged into one
-; - Tests a catch_all terminate pad is created after a catch terminate pad
-
 ; CHECK-LABEL: test_terminatepad
-; CHECK:  i32.catch
-; CHECK:  call      __clang_call_terminate@FUNCTION
-; CHECK:  unreachable
-; CHECK:  catch_all
-; CHECK:  call      _ZSt9terminatev@FUNCTION
-; CHECK-NOT:  call      __clang_call_terminate@FUNCTION
-define hidden i32 @test_terminatepad() personality i8* bitcast (i32 (...)* @__gxx_wasm_personality_v0 to i8*) {
+; CHECK:   catch
+; CHECK:   block     i32
+; CHECK:   br_on_exn   0, __cpp_exception@EVENT
+; CHECK:   call      __clang_call_terminate@FUNCTION, 0
+; CHECK:   unreachable
+; CHECK:   end_block
+; CHECK:   extract_exception
+; CHECK:   call      __clang_call_terminate@FUNCTION
+; CHECK:   unreachable
+define void @test_terminatepad() personality i8* bitcast (i32 (...)* @__gxx_wasm_personality_v0 to i8*) {
 entry:
-  %c = alloca %struct.Cleanup, align 1
-  %c1 = alloca %struct.Cleanup, align 1
   invoke void @foo()
-          to label %invoke.cont unwind label %ehcleanup
-
-invoke.cont:                                      ; preds = %entry
-  %call = invoke %struct.Cleanup* @_ZN7CleanupD1Ev(%struct.Cleanup* %c1)
           to label %try.cont unwind label %catch.dispatch
 
-ehcleanup:                                        ; preds = %entry
-  %0 = cleanuppad within none []
-  %call4 = invoke %struct.Cleanup* @_ZN7CleanupD1Ev(%struct.Cleanup* %c1) [ "funclet"(token %0) ]
-          to label %invoke.cont3 unwind label %terminate
-
-invoke.cont3:                                     ; preds = %ehcleanup
-  cleanupret from %0 unwind label %catch.dispatch
-
-catch.dispatch:                                   ; preds = %invoke.cont3, %invoke.cont
-  %1 = catchswitch within none [label %catch.start] unwind label %ehcleanup7
+catch.dispatch:                                   ; preds = %entry
+  %0 = catchswitch within none [label %catch.start] unwind to caller
 
 catch.start:                                      ; preds = %catch.dispatch
-  %2 = catchpad within %1 [i8* null]
-  %3 = call i8* @llvm.wasm.get.exception(token %2)
-  %4 = call i32 @llvm.wasm.get.ehselector(token %2)
-  %5 = call i8* @__cxa_begin_catch(i8* %3) [ "funclet"(token %2) ]
-  invoke void @__cxa_end_catch() [ "funclet"(token %2) ]
-          to label %invoke.cont5 unwind label %ehcleanup7
+  %1 = catchpad within %0 [i8* null]
+  %2 = call i8* @llvm.wasm.get.exception(token %1)
+  %3 = call i32 @llvm.wasm.get.ehselector(token %1)
+  %4 = call i8* @__cxa_begin_catch(i8* %2) [ "funclet"(token %1) ]
+  invoke void @foo() [ "funclet"(token %1) ]
+          to label %invoke.cont1 unwind label %ehcleanup
 
-invoke.cont5:                                     ; preds = %catch.start
-  catchret from %2 to label %try.cont
+invoke.cont1:                                     ; preds = %catch.start
+  call void @__cxa_end_catch() [ "funclet"(token %1) ]
+  catchret from %1 to label %try.cont
 
-try.cont:                                         ; preds = %invoke.cont5, %invoke.cont
-  %call6 = call %struct.Cleanup* @_ZN7CleanupD1Ev(%struct.Cleanup* %c)
-  ret i32 0
+try.cont:                                         ; preds = %entry, %invoke.cont1
+  ret void
 
-ehcleanup7:                                       ; preds = %catch.start, %catch.dispatch
-  %6 = cleanuppad within none []
-  %call9 = invoke %struct.Cleanup* @_ZN7CleanupD1Ev(%struct.Cleanup* %c) [ "funclet"(token %6) ]
-          to label %invoke.cont8 unwind label %terminate10
+ehcleanup:                                        ; preds = %catch.start
+  %5 = cleanuppad within %1 []
+  invoke void @__cxa_end_catch() [ "funclet"(token %5) ]
+          to label %invoke.cont2 unwind label %terminate
 
-invoke.cont8:                                     ; preds = %ehcleanup7
-  cleanupret from %6 unwind to caller
+invoke.cont2:                                     ; preds = %ehcleanup
+  cleanupret from %5 unwind to caller
 
 terminate:                                        ; preds = %ehcleanup
-  %7 = cleanuppad within %0 []
-  %8 = call i8* @llvm.wasm.get.exception(token %7)
-  call void @__clang_call_terminate(i8* %8) [ "funclet"(token %7) ]
-  unreachable
-
-terminate10:                                      ; preds = %ehcleanup7
-  %9 = cleanuppad within %6 []
-  %10 = call i8* @llvm.wasm.get.exception(token %9)
-  call void @__clang_call_terminate(i8* %10) [ "funclet"(token %9) ]
+  %6 = cleanuppad within %5 []
+  %7 = call i8* @llvm.wasm.get.exception(token %6)
+  call void @__clang_call_terminate(i8* %7) [ "funclet"(token %6) ]
   unreachable
 }
 
@@ -164,18 +153,18 @@ terminate10:                                      ; preds = %ehcleanup7
 ; CHECK-LABEL: test_no_prolog_epilog_in_ehpad
 ; CHECK:  try
 ; CHECK:  call      foo@FUNCTION
-; CHECK:  i32.catch
-; CHECK-NOT:  get_global  $push{{.+}}=, __stack_pointer@GLOBAL
-; CHECK:  set_global  __stack_pointer@GLOBAL
+; CHECK:  catch
+; CHECK-NOT:  global.get  $push{{.+}}=, __stack_pointer@GLOBAL
+; CHECK:  global.set  __stack_pointer@GLOBAL
 ; CHECK:  try
 ; CHECK:  call      foo@FUNCTION
-; CHECK:  catch_all
-; CHECK-NOT:  get_global  $push{{.+}}=, __stack_pointer@GLOBAL
-; CHECK:  set_global  __stack_pointer@GLOBAL
+; CHECK:  catch
+; CHECK-NOT:  global.get  $push{{.+}}=, __stack_pointer@GLOBAL
+; CHECK:  global.set  __stack_pointer@GLOBAL
 ; CHECK:  call      __cxa_end_catch@FUNCTION
-; CHECK-NOT:  set_global  __stack_pointer@GLOBAL, $pop{{.+}}
+; CHECK-NOT:  global.set  __stack_pointer@GLOBAL, $pop{{.+}}
 ; CHECK:  end_try
-; CHECK-NOT:  set_global  __stack_pointer@GLOBAL, $pop{{.+}}
+; CHECK-NOT:  global.set  __stack_pointer@GLOBAL, $pop{{.+}}
 ; CHECK:  end_try
 define void @test_no_prolog_epilog_in_ehpad() personality i8* bitcast (i32 (...)* @__gxx_wasm_personality_v0 to i8*) {
 entry:
@@ -226,7 +215,7 @@ ehcleanup:                                        ; preds = %catch
 ; CHECK:  try
 ; CHECK:  call foo@FUNCTION
 ; CHECK:  end_try
-; CHECK-NOT:  set_global  __stack_pointer@GLOBAL
+; CHECK-NOT:  global.set  __stack_pointer@GLOBAL
 ; CHECK:  return
 define void @no_sp_writeback() personality i8* bitcast (i32 (...)* @__gxx_wasm_personality_v0 to i8*) {
 entry:
@@ -251,6 +240,8 @@ try.cont:                                         ; preds = %entry, %catch.start
 declare void @foo()
 declare void @bar(i32*)
 declare i32 @__gxx_wasm_personality_v0(...)
+declare void @llvm.wasm.throw(i32, i8*)
+declare void @llvm.wasm.rethrow()
 declare i8* @llvm.wasm.get.exception(token)
 declare i32 @llvm.wasm.get.ehselector(token)
 declare i32 @llvm.eh.typeid.for(i8*)
@@ -258,7 +249,6 @@ declare i8* @__cxa_begin_catch(i8*)
 declare void @__cxa_end_catch()
 declare void @__cxa_rethrow()
 declare void @__clang_call_terminate(i8*)
-declare void @_ZSt9terminatev()
 declare %struct.Cleanup* @_ZN7CleanupD1Ev(%struct.Cleanup* returned)
 
 ; CHECK: __cpp_exception:
diff --git a/test/CodeGen/WebAssembly/f16.ll b/test/CodeGen/WebAssembly/f16.ll
index 6ffedd3f310a5970427e0ccc5dd6f98fdaf47d1c..645a09a4f9f5550dbb2ea07b4c68fde5a118d749 100644
--- a/test/CodeGen/WebAssembly/f16.ll
+++ b/test/CodeGen/WebAssembly/f16.ll
@@ -8,7 +8,7 @@ target triple = "wasm32-unknown-unknown"
 
 ; CHECK-LABEL: demote:
 ; CHECK-NEXT: .functype demote (f32) -> (f32){{$}}
-; CHECK-NEXT: get_local	$push[[L0:[0-9]+]]=, 0{{$}}
+; CHECK-NEXT: local.get	$push[[L0:[0-9]+]]=, 0{{$}}
 ; CHECK-NEXT: i32.call	$push[[L1:[0-9]+]]=, __gnu_f2h_ieee@FUNCTION, $pop[[L0]]{{$}}
 ; CHECK-NEXT: f32.call	$push[[L2:[0-9]+]]=, __gnu_h2f_ieee@FUNCTION, $pop[[L1]]{{$}}
 ; CHECK-NEXT: return  	$pop[[L2]]{{$}}
@@ -19,7 +19,7 @@ define half @demote(float %f) {
 
 ; CHECK-LABEL: promote:
 ; CHECK-NEXT: .functype promote (f32) -> (f32){{$}}
-; CHECK-NEXT: get_local	$push0=, 0{{$}}
+; CHECK-NEXT: local.get	$push0=, 0{{$}}
 ; CHECK-NEXT: return  	$pop0{{$}}
 define float @promote(half %f) {
     %t = fpext half %f to float
diff --git a/test/CodeGen/WebAssembly/f32.ll b/test/CodeGen/WebAssembly/f32.ll
index fd89261f1727a5f1585b2f66b17f9a95e450cd40..45912912d814192d45fd92767dfea89b16989930 100644
--- a/test/CodeGen/WebAssembly/f32.ll
+++ b/test/CodeGen/WebAssembly/f32.ll
@@ -17,8 +17,8 @@ declare float @llvm.fma.f32(float, float, float)
 
 ; CHECK-LABEL: fadd32:
 ; CHECK-NEXT: .functype fadd32 (f32, f32) -> (f32){{$}}
-; CHECK-NEXT: get_local $push[[L0:[0-9]+]]=, 0{{$}}
-; CHECK-NEXT: get_local $push[[L1:[0-9]+]]=, 1{{$}}
+; CHECK-NEXT: local.get $push[[L0:[0-9]+]]=, 0{{$}}
+; CHECK-NEXT: local.get $push[[L1:[0-9]+]]=, 1{{$}}
 ; CHECK-NEXT: f32.add $push[[LR:[0-9]+]]=, $pop[[L0]], $pop[[L1]]{{$}}
 ; CHECK-NEXT: return $pop[[LR]]{{$}}
 define float @fadd32(float %x, float %y) {
diff --git a/test/CodeGen/WebAssembly/f64.ll b/test/CodeGen/WebAssembly/f64.ll
index 98a23b193359c13f4ecd1b9a675f450e1cad7cc1..4536e8d94f22fc935b58a35796206834d89184f2 100644
--- a/test/CodeGen/WebAssembly/f64.ll
+++ b/test/CodeGen/WebAssembly/f64.ll
@@ -17,8 +17,8 @@ declare double @llvm.fma.f64(double, double, double)
 
 ; CHECK-LABEL: fadd64:
 ; CHECK-NEXT: .functype fadd64 (f64, f64) -> (f64){{$}}
-; CHECK-NEXT: get_local $push[[L0:[0-9]+]]=, 0{{$}}
-; CHECK-NEXT: get_local $push[[L1:[0-9]+]]=, 1{{$}}
+; CHECK-NEXT: local.get $push[[L0:[0-9]+]]=, 0{{$}}
+; CHECK-NEXT: local.get $push[[L1:[0-9]+]]=, 1{{$}}
 ; CHECK-NEXT: f64.add $push[[LR:[0-9]+]]=, $pop[[L0]], $pop[[L1]]{{$}}
 ; CHECK-NEXT: return $pop[[LR]]{{$}}
 define double @fadd64(double %x, double %y) {
diff --git a/test/CodeGen/WebAssembly/fast-isel-i24.ll b/test/CodeGen/WebAssembly/fast-isel-i24.ll
index b0ca27674a1acce053c15f502fafa53be514c251..dddf2c4ff2dc6ddb7bcb4b2f48193014aeeafadb 100644
--- a/test/CodeGen/WebAssembly/fast-isel-i24.ll
+++ b/test/CodeGen/WebAssembly/fast-isel-i24.ll
@@ -9,8 +9,8 @@ target triple = "wasm32-unknown-unknown"
 
 ; CHECK-LABEL: add:
 ; CHECK-NEXT: .functype add (i32, i32) -> (i32){{$}}
-; CHECK-NEXT: get_local	$push2=, 0{{$}}
-; CHECK-NEXT: get_local	$push1=, 1{{$}}
+; CHECK-NEXT: local.get	$push2=, 0{{$}}
+; CHECK-NEXT: local.get	$push1=, 1{{$}}
 ; CHECK-NEXT: i32.add 	$push0=, $pop2, $pop1{{$}}
 ; CHECK-NEXT: end_function
 define i24 @add(i24 %x, i24 %y) {
diff --git a/test/CodeGen/WebAssembly/fast-isel-noreg.ll b/test/CodeGen/WebAssembly/fast-isel-noreg.ll
index d4c0af68dfe06f34d4fb1100c07769c3e3bbbc89..ce36fd0c0b8fbda7b4c76150d317745cb945f9d0 100644
--- a/test/CodeGen/WebAssembly/fast-isel-noreg.ll
+++ b/test/CodeGen/WebAssembly/fast-isel-noreg.ll
@@ -38,7 +38,11 @@ entry:
 ; CHECK: i32.const {{.*}}, 24
 ; CHECK: i32.shr_s
 ; CHECK: i32.const {{.*}}, 64
-; CHECK: br_if 0, $pop0
+; CHECK: i32.lt_s
+; CHECK: i32.const {{.*}}, 1
+; CHECK: i32.and
+; CHECK: i32.eqz
+; CHECK: br_if 0, $pop{{[0-9]+}}
 define hidden i32 @d() #0 {
 entry:
   %t = icmp slt i8 ptrtoint (void ()* @addr to i8), 64
@@ -53,7 +57,11 @@ b:
 ; CHECK: i32.const {{.*}}, 255
 ; CHECK: i32.and
 ; CHECK: i32.const {{.*}}, 64
-; CHECK: br_if 0, $pop0
+; CHECK: i32.lt_u
+; CHECK: i32.const {{.*}}, 1
+; CHECK: i32.and
+; CHECK: i32.eqz
+; CHECK: br_if 0, $pop{{[0-9]+}}
 define hidden i32 @e() #0 {
 entry:
   %t = icmp ult i8 ptrtoint (void ()* @addr to i8), 64
diff --git a/test/CodeGen/WebAssembly/fast-isel.ll b/test/CodeGen/WebAssembly/fast-isel.ll
index b03abc50ed00be49ae1b487748ca8f2f7d871b93..fa9bf247a40920b3b6c39f02e5bedf86575976c0 100644
--- a/test/CodeGen/WebAssembly/fast-isel.ll
+++ b/test/CodeGen/WebAssembly/fast-isel.ll
@@ -21,28 +21,28 @@ define double @immediate_f64() {
 }
 
 ; CHECK-LABEL: bitcast_i32_f32:
-; CHECK: i32.reinterpret/f32 $push{{[0-9]+}}=, $0{{$}}
+; CHECK: i32.reinterpret_f32 $push{{[0-9]+}}=, $0{{$}}
 define i32 @bitcast_i32_f32(float %x) {
   %y = bitcast float %x to i32
   ret i32 %y
 }
 
 ; CHECK-LABEL: bitcast_f32_i32:
-; CHECK: f32.reinterpret/i32 $push{{[0-9]+}}=, $0{{$}}
+; CHECK: f32.reinterpret_i32 $push{{[0-9]+}}=, $0{{$}}
 define float @bitcast_f32_i32(i32 %x) {
   %y = bitcast i32 %x to float
   ret float %y
 }
 
 ; CHECK-LABEL: bitcast_i64_f64:
-; CHECK: i64.reinterpret/f64 $push{{[0-9]+}}=, $0{{$}}
+; CHECK: i64.reinterpret_f64 $push{{[0-9]+}}=, $0{{$}}
 define i64 @bitcast_i64_f64(double %x) {
   %y = bitcast double %x to i64
   ret i64 %y
 }
 
 ; CHECK-LABEL: bitcast_f64_i64:
-; CHECK: f64.reinterpret/i64 $push{{[0-9]+}}=, $0{{$}}
+; CHECK: f64.reinterpret_i64 $push{{[0-9]+}}=, $0{{$}}
 define double @bitcast_f64_i64(i64 %x) {
   %y = bitcast i64 %x to double
   ret double %y
diff --git a/test/CodeGen/WebAssembly/function-bitcasts-varargs.ll b/test/CodeGen/WebAssembly/function-bitcasts-varargs.ll
index 515c5703d86c06c64249c83e5f6e7210826a60d6..b542276e068f6d7e9c043c3594db3264fdde18e0 100644
--- a/test/CodeGen/WebAssembly/function-bitcasts-varargs.ll
+++ b/test/CodeGen/WebAssembly/function-bitcasts-varargs.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -asm-verbose=false -wasm-temporary-workarounds=false -wasm-keep-registers | FileCheck %s
+; RUN: llc < %s -asm-verbose=false -wasm-keep-registers | FileCheck %s
 
 ; Test that function pointer casts casting away varargs are replaced with
 ; wrappers.
diff --git a/test/CodeGen/WebAssembly/function-bitcasts.ll b/test/CodeGen/WebAssembly/function-bitcasts.ll
index 93c09cc5dd2d418f9e75578355e861ebe4c2d0fe..813e8420ae54bb01a02691e282ba5fc6c30b3403 100644
--- a/test/CodeGen/WebAssembly/function-bitcasts.ll
+++ b/test/CodeGen/WebAssembly/function-bitcasts.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -asm-verbose=false -wasm-disable-explicit-locals -wasm-keep-registers -enable-emscripten-cxx-exceptions -wasm-temporary-workarounds=false | FileCheck %s
+; RUN: llc < %s -asm-verbose=false -wasm-disable-explicit-locals -wasm-keep-registers -enable-emscripten-cxx-exceptions | FileCheck %s
 
 ; Test that function pointer casts are replaced with wrappers.
 
@@ -79,7 +79,7 @@ entry:
 }
 
 ; CHECK-LABEL: test_varargs:
-; CHECK:      set_global
+; CHECK:      global.set
 ; CHECK:      i32.const   $push[[L3:[0-9]+]]=, 0{{$}}
 ; CHECK-NEXT: call        .Lvararg_bitcast@FUNCTION, $pop[[L3]]{{$}}
 ; CHECK-NEXT: i32.const   $push[[L4:[0-9]+]]=, 0{{$}}
@@ -199,5 +199,5 @@ end:
 ; CHECK-LABEL: .Lfoo1_bitcast:
 ; CHECK-NEXT: .functype .Lfoo1_bitcast () -> (i32)
 ; CHECK-NEXT: call        foo1@FUNCTION{{$}}
-; CHECK-NEXT: copy_local  $push0=, $0
+; CHECK-NEXT: local.copy  $push0=, $0
 ; CHECK-NEXT: end_function
diff --git a/test/CodeGen/WebAssembly/i32-load-store-alignment.ll b/test/CodeGen/WebAssembly/i32-load-store-alignment.ll
index 4fcc4254b69536e44522ea8ffd48b397d3de6f84..cff7fc7faf6d544e05060139a2a16f05df4451da 100644
--- a/test/CodeGen/WebAssembly/i32-load-store-alignment.ll
+++ b/test/CodeGen/WebAssembly/i32-load-store-alignment.ll
@@ -38,7 +38,7 @@ define i32 @ldi32_a4(i32 *%p) {
   ret i32 %v
 }
 
-; The default alignment in LLVM is the same as the defualt alignment in wasm.
+; The default alignment in LLVM is the same as the default alignment in wasm.
 
 ; CHECK-LABEL: ldi32:
 ; CHECK-NEXT: .functype ldi32 (i32) -> (i32){{$}}
@@ -142,7 +142,7 @@ define void @sti32_a4(i32 *%p, i32 %v) {
   ret void
 }
 
-; The default alignment in LLVM is the same as the defualt alignment in wasm.
+; The default alignment in LLVM is the same as the default alignment in wasm.
 
 ; CHECK-LABEL: sti32:
 ; CHECK-NEXT: .functype sti32 (i32, i32) -> (){{$}}
diff --git a/test/CodeGen/WebAssembly/i32.ll b/test/CodeGen/WebAssembly/i32.ll
index 352713779640be9c8f05065cd5c587a5d5fc52c9..897e4114f55656b999dd4aa5f6bb41342755c104 100644
--- a/test/CodeGen/WebAssembly/i32.ll
+++ b/test/CodeGen/WebAssembly/i32.ll
@@ -11,8 +11,8 @@ declare i32 @llvm.ctpop.i32(i32)
 
 ; CHECK-LABEL: add32:
 ; CHECK-NEXT: .functype add32 (i32, i32) -> (i32){{$}}
-; CHECK-NEXT: get_local $push[[L0:[0-9]+]]=, 0{{$}}
-; CHECK-NEXT: get_local $push[[L1:[0-9]+]]=, 1{{$}}
+; CHECK-NEXT: local.get $push[[L0:[0-9]+]]=, 0{{$}}
+; CHECK-NEXT: local.get $push[[L1:[0-9]+]]=, 1{{$}}
 ; CHECK-NEXT: i32.add $push0=, $pop[[L0]], $pop[[L1]]{{$}}
 ; CHECK-NEXT: return $pop0{{$}}
 define i32 @add32(i32 %x, i32 %y) {
@@ -22,8 +22,8 @@ define i32 @add32(i32 %x, i32 %y) {
 
 ; CHECK-LABEL: sub32:
 ; CHECK-NEXT: .functype sub32 (i32, i32) -> (i32){{$}}
-; CHECK-NEXT: get_local $push[[L0:[0-9]+]]=, 0{{$}}
-; CHECK-NEXT: get_local $push[[L1:[0-9]+]]=, 1{{$}}
+; CHECK-NEXT: local.get $push[[L0:[0-9]+]]=, 0{{$}}
+; CHECK-NEXT: local.get $push[[L1:[0-9]+]]=, 1{{$}}
 ; CHECK-NEXT: i32.sub $push0=, $pop[[L0]], $pop[[L1]]{{$}}
 ; CHECK-NEXT: return $pop0{{$}}
 define i32 @sub32(i32 %x, i32 %y) {
@@ -33,8 +33,8 @@ define i32 @sub32(i32 %x, i32 %y) {
 
 ; CHECK-LABEL: mul32:
 ; CHECK-NEXT: .functype mul32 (i32, i32) -> (i32){{$}}
-; CHECK-NEXT: get_local $push[[L0:[0-9]+]]=, 0{{$}}
-; CHECK-NEXT: get_local $push[[L1:[0-9]+]]=, 1{{$}}
+; CHECK-NEXT: local.get $push[[L0:[0-9]+]]=, 0{{$}}
+; CHECK-NEXT: local.get $push[[L1:[0-9]+]]=, 1{{$}}
 ; CHECK-NEXT: i32.mul $push0=, $pop[[L0]], $pop[[L1]]{{$}}
 ; CHECK-NEXT: return $pop0{{$}}
 define i32 @mul32(i32 %x, i32 %y) {
@@ -44,8 +44,8 @@ define i32 @mul32(i32 %x, i32 %y) {
 
 ; CHECK-LABEL: sdiv32:
 ; CHECK-NEXT: .functype sdiv32 (i32, i32) -> (i32){{$}}
-; CHECK-NEXT: get_local $push[[L0:[0-9]+]]=, 0{{$}}
-; CHECK-NEXT: get_local $push[[L1:[0-9]+]]=, 1{{$}}
+; CHECK-NEXT: local.get $push[[L0:[0-9]+]]=, 0{{$}}
+; CHECK-NEXT: local.get $push[[L1:[0-9]+]]=, 1{{$}}
 ; CHECK-NEXT: i32.div_s $push0=, $pop[[L0]], $pop[[L1]]{{$}}
 ; CHECK-NEXT: return $pop0{{$}}
 define i32 @sdiv32(i32 %x, i32 %y) {
@@ -55,8 +55,8 @@ define i32 @sdiv32(i32 %x, i32 %y) {
 
 ; CHECK-LABEL: udiv32:
 ; CHECK-NEXT: .functype udiv32 (i32, i32) -> (i32){{$}}
-; CHECK-NEXT: get_local $push[[L0:[0-9]+]]=, 0{{$}}
-; CHECK-NEXT: get_local $push[[L1:[0-9]+]]=, 1{{$}}
+; CHECK-NEXT: local.get $push[[L0:[0-9]+]]=, 0{{$}}
+; CHECK-NEXT: local.get $push[[L1:[0-9]+]]=, 1{{$}}
 ; CHECK-NEXT: i32.div_u $push0=, $pop[[L0]], $pop[[L1]]{{$}}
 ; CHECK-NEXT: return $pop0{{$}}
 define i32 @udiv32(i32 %x, i32 %y) {
@@ -66,8 +66,8 @@ define i32 @udiv32(i32 %x, i32 %y) {
 
 ; CHECK-LABEL: srem32:
 ; CHECK-NEXT: .functype srem32 (i32, i32) -> (i32){{$}}
-; CHECK-NEXT: get_local $push[[L0:[0-9]+]]=, 0{{$}}
-; CHECK-NEXT: get_local $push[[L1:[0-9]+]]=, 1{{$}}
+; CHECK-NEXT: local.get $push[[L0:[0-9]+]]=, 0{{$}}
+; CHECK-NEXT: local.get $push[[L1:[0-9]+]]=, 1{{$}}
 ; CHECK-NEXT: i32.rem_s $push0=, $pop[[L0]], $pop[[L1]]{{$}}
 ; CHECK-NEXT: return $pop0{{$}}
 define i32 @srem32(i32 %x, i32 %y) {
@@ -77,8 +77,8 @@ define i32 @srem32(i32 %x, i32 %y) {
 
 ; CHECK-LABEL: urem32:
 ; CHECK-NEXT: .functype urem32 (i32, i32) -> (i32){{$}}
-; CHECK-NEXT: get_local $push[[L0:[0-9]+]]=, 0{{$}}
-; CHECK-NEXT: get_local $push[[L1:[0-9]+]]=, 1{{$}}
+; CHECK-NEXT: local.get $push[[L0:[0-9]+]]=, 0{{$}}
+; CHECK-NEXT: local.get $push[[L1:[0-9]+]]=, 1{{$}}
 ; CHECK-NEXT: i32.rem_u $push0=, $pop[[L0]], $pop[[L1]]{{$}}
 ; CHECK-NEXT: return $pop0{{$}}
 define i32 @urem32(i32 %x, i32 %y) {
@@ -88,8 +88,8 @@ define i32 @urem32(i32 %x, i32 %y) {
 
 ; CHECK-LABEL: and32:
 ; CHECK-NEXT: .functype and32 (i32, i32) -> (i32){{$}}
-; CHECK-NEXT: get_local $push[[L0:[0-9]+]]=, 0{{$}}
-; CHECK-NEXT: get_local $push[[L1:[0-9]+]]=, 1{{$}}
+; CHECK-NEXT: local.get $push[[L0:[0-9]+]]=, 0{{$}}
+; CHECK-NEXT: local.get $push[[L1:[0-9]+]]=, 1{{$}}
 ; CHECK-NEXT: i32.and $push0=, $pop[[L0]], $pop[[L1]]{{$}}
 ; CHECK-NEXT: return $pop0{{$}}
 define i32 @and32(i32 %x, i32 %y) {
@@ -99,8 +99,8 @@ define i32 @and32(i32 %x, i32 %y) {
 
 ; CHECK-LABEL: or32:
 ; CHECK-NEXT: .functype or32 (i32, i32) -> (i32){{$}}
-; CHECK-NEXT: get_local $push[[L0:[0-9]+]]=, 0{{$}}
-; CHECK-NEXT: get_local $push[[L1:[0-9]+]]=, 1{{$}}
+; CHECK-NEXT: local.get $push[[L0:[0-9]+]]=, 0{{$}}
+; CHECK-NEXT: local.get $push[[L1:[0-9]+]]=, 1{{$}}
 ; CHECK-NEXT: i32.or $push0=, $pop[[L0]], $pop[[L1]]{{$}}
 ; CHECK-NEXT: return $pop0{{$}}
 define i32 @or32(i32 %x, i32 %y) {
@@ -110,8 +110,8 @@ define i32 @or32(i32 %x, i32 %y) {
 
 ; CHECK-LABEL: xor32:
 ; CHECK-NEXT: .functype xor32 (i32, i32) -> (i32){{$}}
-; CHECK-NEXT: get_local $push[[L0:[0-9]+]]=, 0{{$}}
-; CHECK-NEXT: get_local $push[[L1:[0-9]+]]=, 1{{$}}
+; CHECK-NEXT: local.get $push[[L0:[0-9]+]]=, 0{{$}}
+; CHECK-NEXT: local.get $push[[L1:[0-9]+]]=, 1{{$}}
 ; CHECK-NEXT: i32.xor $push0=, $pop[[L0]], $pop[[L1]]{{$}}
 ; CHECK-NEXT: return $pop0{{$}}
 define i32 @xor32(i32 %x, i32 %y) {
@@ -121,8 +121,8 @@ define i32 @xor32(i32 %x, i32 %y) {
 
 ; CHECK-LABEL: shl32:
 ; CHECK-NEXT: .functype shl32 (i32, i32) -> (i32){{$}}
-; CHECK-NEXT: get_local $push[[L0:[0-9]+]]=, 0{{$}}
-; CHECK-NEXT: get_local $push[[L1:[0-9]+]]=, 1{{$}}
+; CHECK-NEXT: local.get $push[[L0:[0-9]+]]=, 0{{$}}
+; CHECK-NEXT: local.get $push[[L1:[0-9]+]]=, 1{{$}}
 ; CHECK-NEXT: i32.shl $push0=, $pop[[L0]], $pop[[L1]]{{$}}
 ; CHECK-NEXT: return $pop0{{$}}
 define i32 @shl32(i32 %x, i32 %y) {
@@ -132,8 +132,8 @@ define i32 @shl32(i32 %x, i32 %y) {
 
 ; CHECK-LABEL: shr32:
 ; CHECK-NEXT: .functype shr32 (i32, i32) -> (i32){{$}}
-; CHECK-NEXT: get_local $push[[L0:[0-9]+]]=, 0{{$}}
-; CHECK-NEXT: get_local $push[[L1:[0-9]+]]=, 1{{$}}
+; CHECK-NEXT: local.get $push[[L0:[0-9]+]]=, 0{{$}}
+; CHECK-NEXT: local.get $push[[L1:[0-9]+]]=, 1{{$}}
 ; CHECK-NEXT: i32.shr_u $push0=, $pop[[L0]], $pop[[L1]]{{$}}
 ; CHECK-NEXT: return $pop0{{$}}
 define i32 @shr32(i32 %x, i32 %y) {
@@ -143,8 +143,8 @@ define i32 @shr32(i32 %x, i32 %y) {
 
 ; CHECK-LABEL: sar32:
 ; CHECK-NEXT: .functype sar32 (i32, i32) -> (i32){{$}}
-; CHECK-NEXT: get_local $push[[L0:[0-9]+]]=, 0{{$}}
-; CHECK-NEXT: get_local $push[[L1:[0-9]+]]=, 1{{$}}
+; CHECK-NEXT: local.get $push[[L0:[0-9]+]]=, 0{{$}}
+; CHECK-NEXT: local.get $push[[L1:[0-9]+]]=, 1{{$}}
 ; CHECK-NEXT: i32.shr_s $push0=, $pop[[L0]], $pop[[L1]]{{$}}
 ; CHECK-NEXT: return $pop0{{$}}
 define i32 @sar32(i32 %x, i32 %y) {
@@ -154,7 +154,7 @@ define i32 @sar32(i32 %x, i32 %y) {
 
 ; CHECK-LABEL: clz32:
 ; CHECK-NEXT: .functype clz32 (i32) -> (i32){{$}}
-; CHECK-NEXT: get_local $push[[L0:[0-9]+]]=, 0{{$}}
+; CHECK-NEXT: local.get $push[[L0:[0-9]+]]=, 0{{$}}
 ; CHECK-NEXT: i32.clz $push0=, $pop[[L0]]{{$}}
 ; CHECK-NEXT: return $pop0{{$}}
 define i32 @clz32(i32 %x) {
@@ -164,7 +164,7 @@ define i32 @clz32(i32 %x) {
 
 ; CHECK-LABEL: clz32_zero_undef:
 ; CHECK-NEXT: .functype clz32_zero_undef (i32) -> (i32){{$}}
-; CHECK-NEXT: get_local $push[[L0:[0-9]+]]=, 0{{$}}
+; CHECK-NEXT: local.get $push[[L0:[0-9]+]]=, 0{{$}}
 ; CHECK-NEXT: i32.clz $push0=, $pop[[L0]]{{$}}
 ; CHECK-NEXT: return $pop0{{$}}
 define i32 @clz32_zero_undef(i32 %x) {
@@ -174,7 +174,7 @@ define i32 @clz32_zero_undef(i32 %x) {
 
 ; CHECK-LABEL: ctz32:
 ; CHECK-NEXT: .functype ctz32 (i32) -> (i32){{$}}
-; CHECK-NEXT: get_local $push[[L0:[0-9]+]]=, 0{{$}}
+; CHECK-NEXT: local.get $push[[L0:[0-9]+]]=, 0{{$}}
 ; CHECK-NEXT: i32.ctz $push0=, $pop[[L0]]{{$}}
 ; CHECK-NEXT: return $pop0{{$}}
 define i32 @ctz32(i32 %x) {
@@ -184,7 +184,7 @@ define i32 @ctz32(i32 %x) {
 
 ; CHECK-LABEL: ctz32_zero_undef:
 ; CHECK-NEXT: .functype ctz32_zero_undef (i32) -> (i32){{$}}
-; CHECK-NEXT: get_local $push[[L0:[0-9]+]]=, 0{{$}}
+; CHECK-NEXT: local.get $push[[L0:[0-9]+]]=, 0{{$}}
 ; CHECK-NEXT: i32.ctz $push0=, $pop[[L0]]{{$}}
 ; CHECK-NEXT: return $pop0{{$}}
 define i32 @ctz32_zero_undef(i32 %x) {
@@ -194,7 +194,7 @@ define i32 @ctz32_zero_undef(i32 %x) {
 
 ; CHECK-LABEL: popcnt32:
 ; CHECK-NEXT: .functype popcnt32 (i32) -> (i32){{$}}
-; CHECK-NEXT: get_local $push[[L0:[0-9]+]]=, 0{{$}}
+; CHECK-NEXT: local.get $push[[L0:[0-9]+]]=, 0{{$}}
 ; CHECK-NEXT: i32.popcnt $push0=, $pop[[L0]]{{$}}
 ; CHECK-NEXT: return $pop0{{$}}
 define i32 @popcnt32(i32 %x) {
@@ -204,7 +204,7 @@ define i32 @popcnt32(i32 %x) {
 
 ; CHECK-LABEL: eqz32:
 ; CHECK-NEXT: .functype eqz32 (i32) -> (i32){{$}}
-; CHECK-NEXT: get_local $push[[L0:[0-9]+]]=, 0{{$}}
+; CHECK-NEXT: local.get $push[[L0:[0-9]+]]=, 0{{$}}
 ; CHECK-NEXT: i32.eqz $push0=, $pop[[L0]]{{$}}
 ; CHECK-NEXT: return $pop0{{$}}
 define i32 @eqz32(i32 %x) {
@@ -215,8 +215,8 @@ define i32 @eqz32(i32 %x) {
 
 ; CHECK-LABEL: rotl:
 ; CHECK-NEXT: .functype rotl (i32, i32) -> (i32){{$}}
-; CHECK-NEXT: get_local $push[[L0:[0-9]+]]=, 0{{$}}
-; CHECK-NEXT: get_local $push[[L1:[0-9]+]]=, 1{{$}}
+; CHECK-NEXT: local.get $push[[L0:[0-9]+]]=, 0{{$}}
+; CHECK-NEXT: local.get $push[[L1:[0-9]+]]=, 1{{$}}
 ; CHECK-NEXT: i32.rotl $push0=, $pop[[L0]], $pop[[L1]]
 ; CHECK-NEXT: return $pop0{{$}}
 define i32 @rotl(i32 %x, i32 %y) {
@@ -229,8 +229,8 @@ define i32 @rotl(i32 %x, i32 %y) {
 
 ; CHECK-LABEL: masked_rotl:
 ; CHECK-NEXT: .functype masked_rotl (i32, i32) -> (i32){{$}}
-; CHECK-NEXT: get_local $push[[L0:[0-9]+]]=, 0{{$}}
-; CHECK-NEXT: get_local $push[[L1:[0-9]+]]=, 1{{$}}
+; CHECK-NEXT: local.get $push[[L0:[0-9]+]]=, 0{{$}}
+; CHECK-NEXT: local.get $push[[L1:[0-9]+]]=, 1{{$}}
 ; CHECK-NEXT: i32.rotl $push0=, $pop[[L0]], $pop[[L1]]
 ; CHECK-NEXT: return $pop0{{$}}
 define i32 @masked_rotl(i32 %x, i32 %y) {
@@ -244,8 +244,8 @@ define i32 @masked_rotl(i32 %x, i32 %y) {
 
 ; CHECK-LABEL: rotr:
 ; CHECK-NEXT: .functype rotr (i32, i32) -> (i32){{$}}
-; CHECK-NEXT: get_local $push[[L0:[0-9]+]]=, 0{{$}}
-; CHECK-NEXT: get_local $push[[L1:[0-9]+]]=, 1{{$}}
+; CHECK-NEXT: local.get $push[[L0:[0-9]+]]=, 0{{$}}
+; CHECK-NEXT: local.get $push[[L1:[0-9]+]]=, 1{{$}}
 ; CHECK-NEXT: i32.rotr $push0=, $pop[[L0]], $pop[[L1]]
 ; CHECK-NEXT: return $pop0{{$}}
 define i32 @rotr(i32 %x, i32 %y) {
@@ -258,8 +258,8 @@ define i32 @rotr(i32 %x, i32 %y) {
 
 ; CHECK-LABEL: masked_rotr:
 ; CHECK-NEXT: .functype masked_rotr (i32, i32) -> (i32){{$}}
-; CHECK-NEXT: get_local $push[[L0:[0-9]+]]=, 0{{$}}
-; CHECK-NEXT: get_local $push[[L1:[0-9]+]]=, 1{{$}}
+; CHECK-NEXT: local.get $push[[L0:[0-9]+]]=, 0{{$}}
+; CHECK-NEXT: local.get $push[[L1:[0-9]+]]=, 1{{$}}
 ; CHECK-NEXT: i32.rotr $push0=, $pop[[L0]], $pop[[L1]]
 ; CHECK-NEXT: return $pop0{{$}}
 define i32 @masked_rotr(i32 %x, i32 %y) {
diff --git a/test/CodeGen/WebAssembly/i64-load-store-alignment.ll b/test/CodeGen/WebAssembly/i64-load-store-alignment.ll
index 32933c19fe34d93d826fa20247c3afda700f1cb6..5af843bcbaacb27aeb764786591c13affcc7ecf7 100644
--- a/test/CodeGen/WebAssembly/i64-load-store-alignment.ll
+++ b/test/CodeGen/WebAssembly/i64-load-store-alignment.ll
@@ -47,7 +47,7 @@ define i64 @ldi64_a8(i64 *%p) {
   ret i64 %v
 }
 
-; The default alignment in LLVM is the same as the defualt alignment in wasm.
+; The default alignment in LLVM is the same as the default alignment in wasm.
 
 ; CHECK-LABEL: ldi64:
 ; CHECK-NEXT: .functype ldi64 (i32) -> (i64){{$}}
@@ -205,7 +205,7 @@ define void @sti64_a8(i64 *%p, i64 %v) {
   ret void
 }
 
-; The default alignment in LLVM is the same as the defualt alignment in wasm.
+; The default alignment in LLVM is the same as the default alignment in wasm.
 
 ; CHECK-LABEL: sti64:
 ; CHECK-NEXT: .functype sti64 (i32, i64) -> (){{$}}
diff --git a/test/CodeGen/WebAssembly/i64.ll b/test/CodeGen/WebAssembly/i64.ll
index 72adf7a277183ecfce87fe08a4811819341dfd4c..9ac52f97afac27b0eb9fba24692c489bcd38b7fd 100644
--- a/test/CodeGen/WebAssembly/i64.ll
+++ b/test/CodeGen/WebAssembly/i64.ll
@@ -11,8 +11,8 @@ declare i64 @llvm.ctpop.i64(i64)
 
 ; CHECK-LABEL: add64:
 ; CHECK-NEXT: .functype add64 (i64, i64) -> (i64){{$}}
-; CHECK-NEXT: get_local $push[[L0:[0-9]+]]=, 0{{$}}
-; CHECK-NEXT: get_local $push[[L1:[0-9]+]]=, 1{{$}}
+; CHECK-NEXT: local.get $push[[L0:[0-9]+]]=, 0{{$}}
+; CHECK-NEXT: local.get $push[[L1:[0-9]+]]=, 1{{$}}
 ; CHECK-NEXT: i64.add $push0=, $pop[[L0]], $pop[[L1]]{{$}}
 ; CHECK-NEXT: return $pop0{{$}}
 define i64 @add64(i64 %x, i64 %y) {
@@ -22,8 +22,8 @@ define i64 @add64(i64 %x, i64 %y) {
 
 ; CHECK-LABEL: sub64:
 ; CHECK-NEXT: .functype sub64 (i64, i64) -> (i64){{$}}
-; CHECK-NEXT: get_local $push[[L0:[0-9]+]]=, 0{{$}}
-; CHECK-NEXT: get_local $push[[L1:[0-9]+]]=, 1{{$}}
+; CHECK-NEXT: local.get $push[[L0:[0-9]+]]=, 0{{$}}
+; CHECK-NEXT: local.get $push[[L1:[0-9]+]]=, 1{{$}}
 ; CHECK-NEXT: i64.sub $push0=, $pop[[L0]], $pop[[L1]]{{$}}
 ; CHECK-NEXT: return $pop0{{$}}
 define i64 @sub64(i64 %x, i64 %y) {
@@ -33,8 +33,8 @@ define i64 @sub64(i64 %x, i64 %y) {
 
 ; CHECK-LABEL: mul64:
 ; CHECK-NEXT: .functype mul64 (i64, i64) -> (i64){{$}}
-; CHECK-NEXT: get_local $push[[L0:[0-9]+]]=, 0{{$}}
-; CHECK-NEXT: get_local $push[[L1:[0-9]+]]=, 1{{$}}
+; CHECK-NEXT: local.get $push[[L0:[0-9]+]]=, 0{{$}}
+; CHECK-NEXT: local.get $push[[L1:[0-9]+]]=, 1{{$}}
 ; CHECK-NEXT: i64.mul $push0=, $pop[[L0]], $pop[[L1]]{{$}}
 ; CHECK-NEXT: return $pop0{{$}}
 define i64 @mul64(i64 %x, i64 %y) {
@@ -44,8 +44,8 @@ define i64 @mul64(i64 %x, i64 %y) {
 
 ; CHECK-LABEL: sdiv64:
 ; CHECK-NEXT: .functype sdiv64 (i64, i64) -> (i64){{$}}
-; CHECK-NEXT: get_local $push[[L0:[0-9]+]]=, 0{{$}}
-; CHECK-NEXT: get_local $push[[L1:[0-9]+]]=, 1{{$}}
+; CHECK-NEXT: local.get $push[[L0:[0-9]+]]=, 0{{$}}
+; CHECK-NEXT: local.get $push[[L1:[0-9]+]]=, 1{{$}}
 ; CHECK-NEXT: i64.div_s $push0=, $pop[[L0]], $pop[[L1]]{{$}}
 ; CHECK-NEXT: return $pop0{{$}}
 define i64 @sdiv64(i64 %x, i64 %y) {
@@ -55,8 +55,8 @@ define i64 @sdiv64(i64 %x, i64 %y) {
 
 ; CHECK-LABEL: udiv64:
 ; CHECK-NEXT: .functype udiv64 (i64, i64) -> (i64){{$}}
-; CHECK-NEXT: get_local $push[[L0:[0-9]+]]=, 0{{$}}
-; CHECK-NEXT: get_local $push[[L1:[0-9]+]]=, 1{{$}}
+; CHECK-NEXT: local.get $push[[L0:[0-9]+]]=, 0{{$}}
+; CHECK-NEXT: local.get $push[[L1:[0-9]+]]=, 1{{$}}
 ; CHECK-NEXT: i64.div_u $push0=, $pop[[L0]], $pop[[L1]]{{$}}
 ; CHECK-NEXT: return $pop0{{$}}
 define i64 @udiv64(i64 %x, i64 %y) {
@@ -66,8 +66,8 @@ define i64 @udiv64(i64 %x, i64 %y) {
 
 ; CHECK-LABEL: srem64:
 ; CHECK-NEXT: .functype srem64 (i64, i64) -> (i64){{$}}
-; CHECK-NEXT: get_local $push[[L0:[0-9]+]]=, 0{{$}}
-; CHECK-NEXT: get_local $push[[L1:[0-9]+]]=, 1{{$}}
+; CHECK-NEXT: local.get $push[[L0:[0-9]+]]=, 0{{$}}
+; CHECK-NEXT: local.get $push[[L1:[0-9]+]]=, 1{{$}}
 ; CHECK-NEXT: i64.rem_s $push0=, $pop[[L0]], $pop[[L1]]{{$}}
 ; CHECK-NEXT: return $pop0{{$}}
 define i64 @srem64(i64 %x, i64 %y) {
@@ -77,8 +77,8 @@ define i64 @srem64(i64 %x, i64 %y) {
 
 ; CHECK-LABEL: urem64:
 ; CHECK-NEXT: .functype urem64 (i64, i64) -> (i64){{$}}
-; CHECK-NEXT: get_local $push[[L0:[0-9]+]]=, 0{{$}}
-; CHECK-NEXT: get_local $push[[L1:[0-9]+]]=, 1{{$}}
+; CHECK-NEXT: local.get $push[[L0:[0-9]+]]=, 0{{$}}
+; CHECK-NEXT: local.get $push[[L1:[0-9]+]]=, 1{{$}}
 ; CHECK-NEXT: i64.rem_u $push0=, $pop[[L0]], $pop[[L1]]{{$}}
 ; CHECK-NEXT: return $pop0{{$}}
 define i64 @urem64(i64 %x, i64 %y) {
@@ -88,8 +88,8 @@ define i64 @urem64(i64 %x, i64 %y) {
 
 ; CHECK-LABEL: and64:
 ; CHECK-NEXT: .functype and64 (i64, i64) -> (i64){{$}}
-; CHECK-NEXT: get_local $push[[L0:[0-9]+]]=, 0{{$}}
-; CHECK-NEXT: get_local $push[[L1:[0-9]+]]=, 1{{$}}
+; CHECK-NEXT: local.get $push[[L0:[0-9]+]]=, 0{{$}}
+; CHECK-NEXT: local.get $push[[L1:[0-9]+]]=, 1{{$}}
 ; CHECK-NEXT: i64.and $push0=, $pop[[L0]], $pop[[L1]]{{$}}
 ; CHECK-NEXT: return $pop0{{$}}
 define i64 @and64(i64 %x, i64 %y) {
@@ -99,8 +99,8 @@ define i64 @and64(i64 %x, i64 %y) {
 
 ; CHECK-LABEL: or64:
 ; CHECK-NEXT: .functype or64 (i64, i64) -> (i64){{$}}
-; CHECK-NEXT: get_local $push[[L0:[0-9]+]]=, 0{{$}}
-; CHECK-NEXT: get_local $push[[L1:[0-9]+]]=, 1{{$}}
+; CHECK-NEXT: local.get $push[[L0:[0-9]+]]=, 0{{$}}
+; CHECK-NEXT: local.get $push[[L1:[0-9]+]]=, 1{{$}}
 ; CHECK-NEXT: i64.or $push0=, $pop[[L0]], $pop[[L1]]{{$}}
 ; CHECK-NEXT: return $pop0{{$}}
 define i64 @or64(i64 %x, i64 %y) {
@@ -110,8 +110,8 @@ define i64 @or64(i64 %x, i64 %y) {
 
 ; CHECK-LABEL: xor64:
 ; CHECK-NEXT: .functype xor64 (i64, i64) -> (i64){{$}}
-; CHECK-NEXT: get_local $push[[L0:[0-9]+]]=, 0{{$}}
-; CHECK-NEXT: get_local $push[[L1:[0-9]+]]=, 1{{$}}
+; CHECK-NEXT: local.get $push[[L0:[0-9]+]]=, 0{{$}}
+; CHECK-NEXT: local.get $push[[L1:[0-9]+]]=, 1{{$}}
 ; CHECK-NEXT: i64.xor $push0=, $pop[[L0]], $pop[[L1]]{{$}}
 ; CHECK-NEXT: return $pop0{{$}}
 define i64 @xor64(i64 %x, i64 %y) {
@@ -121,8 +121,8 @@ define i64 @xor64(i64 %x, i64 %y) {
 
 ; CHECK-LABEL: shl64:
 ; CHECK-NEXT: .functype shl64 (i64, i64) -> (i64){{$}}
-; CHECK-NEXT: get_local $push[[L0:[0-9]+]]=, 0{{$}}
-; CHECK-NEXT: get_local $push[[L1:[0-9]+]]=, 1{{$}}
+; CHECK-NEXT: local.get $push[[L0:[0-9]+]]=, 0{{$}}
+; CHECK-NEXT: local.get $push[[L1:[0-9]+]]=, 1{{$}}
 ; CHECK-NEXT: i64.shl $push0=, $pop[[L0]], $pop[[L1]]{{$}}
 ; CHECK-NEXT: return $pop0{{$}}
 define i64 @shl64(i64 %x, i64 %y) {
@@ -132,8 +132,8 @@ define i64 @shl64(i64 %x, i64 %y) {
 
 ; CHECK-LABEL: shr64:
 ; CHECK-NEXT: .functype shr64 (i64, i64) -> (i64){{$}}
-; CHECK-NEXT: get_local $push[[L0:[0-9]+]]=, 0{{$}}
-; CHECK-NEXT: get_local $push[[L1:[0-9]+]]=, 1{{$}}
+; CHECK-NEXT: local.get $push[[L0:[0-9]+]]=, 0{{$}}
+; CHECK-NEXT: local.get $push[[L1:[0-9]+]]=, 1{{$}}
 ; CHECK-NEXT: i64.shr_u $push0=, $pop[[L0]], $pop[[L1]]{{$}}
 ; CHECK-NEXT: return $pop0{{$}}
 define i64 @shr64(i64 %x, i64 %y) {
@@ -143,8 +143,8 @@ define i64 @shr64(i64 %x, i64 %y) {
 
 ; CHECK-LABEL: sar64:
 ; CHECK-NEXT: .functype sar64 (i64, i64) -> (i64){{$}}
-; CHECK-NEXT: get_local $push[[L0:[0-9]+]]=, 0{{$}}
-; CHECK-NEXT: get_local $push[[L1:[0-9]+]]=, 1{{$}}
+; CHECK-NEXT: local.get $push[[L0:[0-9]+]]=, 0{{$}}
+; CHECK-NEXT: local.get $push[[L1:[0-9]+]]=, 1{{$}}
 ; CHECK-NEXT: i64.shr_s $push0=, $pop[[L0]], $pop[[L1]]{{$}}
 ; CHECK-NEXT: return $pop0{{$}}
 define i64 @sar64(i64 %x, i64 %y) {
@@ -154,7 +154,7 @@ define i64 @sar64(i64 %x, i64 %y) {
 
 ; CHECK-LABEL: clz64:
 ; CHECK-NEXT: .functype clz64 (i64) -> (i64){{$}}
-; CHECK-NEXT: get_local $push[[L0:[0-9]+]]=, 0{{$}}
+; CHECK-NEXT: local.get $push[[L0:[0-9]+]]=, 0{{$}}
 ; CHECK-NEXT: i64.clz $push0=, $pop[[L0]]{{$}}
 ; CHECK-NEXT: return $pop0{{$}}
 define i64 @clz64(i64 %x) {
@@ -164,7 +164,7 @@ define i64 @clz64(i64 %x) {
 
 ; CHECK-LABEL: clz64_zero_undef:
 ; CHECK-NEXT: .functype clz64_zero_undef (i64) -> (i64){{$}}
-; CHECK-NEXT: get_local $push[[L0:[0-9]+]]=, 0{{$}}
+; CHECK-NEXT: local.get $push[[L0:[0-9]+]]=, 0{{$}}
 ; CHECK-NEXT: i64.clz $push0=, $pop[[L0]]{{$}}
 ; CHECK-NEXT: return $pop0{{$}}
 define i64 @clz64_zero_undef(i64 %x) {
@@ -174,7 +174,7 @@ define i64 @clz64_zero_undef(i64 %x) {
 
 ; CHECK-LABEL: ctz64:
 ; CHECK-NEXT: .functype ctz64 (i64) -> (i64){{$}}
-; CHECK-NEXT: get_local $push[[L0:[0-9]+]]=, 0{{$}}
+; CHECK-NEXT: local.get $push[[L0:[0-9]+]]=, 0{{$}}
 ; CHECK-NEXT: i64.ctz $push0=, $pop[[L0]]{{$}}
 ; CHECK-NEXT: return $pop0{{$}}
 define i64 @ctz64(i64 %x) {
@@ -184,7 +184,7 @@ define i64 @ctz64(i64 %x) {
 
 ; CHECK-LABEL: ctz64_zero_undef:
 ; CHECK-NEXT: .functype ctz64_zero_undef (i64) -> (i64){{$}}
-; CHECK-NEXT: get_local $push[[L0:[0-9]+]]=, 0{{$}}
+; CHECK-NEXT: local.get $push[[L0:[0-9]+]]=, 0{{$}}
 ; CHECK-NEXT: i64.ctz $push0=, $pop[[L0]]{{$}}
 ; CHECK-NEXT: return $pop0{{$}}
 define i64 @ctz64_zero_undef(i64 %x) {
@@ -194,7 +194,7 @@ define i64 @ctz64_zero_undef(i64 %x) {
 
 ; CHECK-LABEL: popcnt64:
 ; CHECK-NEXT: .functype popcnt64 (i64) -> (i64){{$}}
-; CHECK-NEXT: get_local $push[[L0:[0-9]+]]=, 0{{$}}
+; CHECK-NEXT: local.get $push[[L0:[0-9]+]]=, 0{{$}}
 ; CHECK-NEXT: i64.popcnt $push0=, $pop[[L0]]{{$}}
 ; CHECK-NEXT: return $pop0{{$}}
 define i64 @popcnt64(i64 %x) {
@@ -204,7 +204,7 @@ define i64 @popcnt64(i64 %x) {
 
 ; CHECK-LABEL: eqz64:
 ; CHECK-NEXT: .functype eqz64 (i64) -> (i32){{$}}
-; CHECK-NEXT: get_local $push[[L0:[0-9]+]]=, 0{{$}}
+; CHECK-NEXT: local.get $push[[L0:[0-9]+]]=, 0{{$}}
 ; CHECK-NEXT: i64.eqz $push0=, $pop[[L0]]{{$}}
 ; CHECK-NEXT: return $pop0{{$}}
 define i32 @eqz64(i64 %x) {
@@ -215,8 +215,8 @@ define i32 @eqz64(i64 %x) {
 
 ; CHECK-LABEL: rotl:
 ; CHECK-NEXT: .functype rotl (i64, i64) -> (i64){{$}}
-; CHECK-NEXT: get_local $push[[L0:[0-9]+]]=, 0{{$}}
-; CHECK-NEXT: get_local $push[[L1:[0-9]+]]=, 1{{$}}
+; CHECK-NEXT: local.get $push[[L0:[0-9]+]]=, 0{{$}}
+; CHECK-NEXT: local.get $push[[L1:[0-9]+]]=, 1{{$}}
 ; CHECK-NEXT: i64.rotl $push0=, $pop[[L0]], $pop[[L1]]
 ; CHECK-NEXT: return $pop0{{$}}
 define i64 @rotl(i64 %x, i64 %y) {
@@ -229,8 +229,8 @@ define i64 @rotl(i64 %x, i64 %y) {
 
 ; CHECK-LABEL: masked_rotl:
 ; CHECK-NEXT: .functype masked_rotl (i64, i64) -> (i64){{$}}
-; CHECK-NEXT: get_local $push[[L0:[0-9]+]]=, 0{{$}}
-; CHECK-NEXT: get_local $push[[L1:[0-9]+]]=, 1{{$}}
+; CHECK-NEXT: local.get $push[[L0:[0-9]+]]=, 0{{$}}
+; CHECK-NEXT: local.get $push[[L1:[0-9]+]]=, 1{{$}}
 ; CHECK-NEXT: i64.rotl $push0=, $pop[[L0]], $pop[[L1]]
 ; CHECK-NEXT: return $pop0{{$}}
 define i64 @masked_rotl(i64 %x, i64 %y) {
@@ -244,8 +244,8 @@ define i64 @masked_rotl(i64 %x, i64 %y) {
 
 ; CHECK-LABEL: rotr:
 ; CHECK-NEXT: .functype rotr (i64, i64) -> (i64){{$}}
-; CHECK-NEXT: get_local $push[[L0:[0-9]+]]=, 0{{$}}
-; CHECK-NEXT: get_local $push[[L1:[0-9]+]]=, 1{{$}}
+; CHECK-NEXT: local.get $push[[L0:[0-9]+]]=, 0{{$}}
+; CHECK-NEXT: local.get $push[[L1:[0-9]+]]=, 1{{$}}
 ; CHECK-NEXT: i64.rotr $push0=, $pop[[L0]], $pop[[L1]]
 ; CHECK-NEXT: return $pop0{{$}}
 define i64 @rotr(i64 %x, i64 %y) {
@@ -258,8 +258,8 @@ define i64 @rotr(i64 %x, i64 %y) {
 
 ; CHECK-LABEL: masked_rotr:
 ; CHECK-NEXT: .functype masked_rotr (i64, i64) -> (i64){{$}}
-; CHECK-NEXT: get_local $push[[L0:[0-9]+]]=, 0{{$}}
-; CHECK-NEXT: get_local $push[[L1:[0-9]+]]=, 1{{$}}
+; CHECK-NEXT: local.get $push[[L0:[0-9]+]]=, 0{{$}}
+; CHECK-NEXT: local.get $push[[L1:[0-9]+]]=, 1{{$}}
 ; CHECK-NEXT: i64.rotr $push0=, $pop[[L0]], $pop[[L1]]
 ; CHECK-NEXT: return $pop0{{$}}
 define i64 @masked_rotr(i64 %x, i64 %y) {
diff --git a/test/CodeGen/WebAssembly/implicit-def.ll b/test/CodeGen/WebAssembly/implicit-def.ll
index 8f7dcc8cee3e4ac8f4212aab8f197689c1f89ce3..702879c8ccfbc0fbe4f51aa3df0be34a5af72393 100644
--- a/test/CodeGen/WebAssembly/implicit-def.ll
+++ b/test/CodeGen/WebAssembly/implicit-def.ll
@@ -109,8 +109,8 @@ X:                                                ; preds = %0, C
 ; CHECK-LABEL: implicit_def_v4i32:
 ; CHECK: .LBB{{[0-9]+}}_4:{{$}}
 ; CHECK-NEXT: end_block{{$}}
-; CHECK-NEXT: v128.const $push[[R:[0-9]+]]=, 0, 0, 0, 0, 0, 0, 0, 0,
-; CHECK-SAME:                                0, 0, 0, 0, 0, 0, 0, 0{{$}}
+; CHECK-NEXT: i32.const $push[[L0:[0-9]+]]=, 0{{$}}
+; CHECK-NEXT: i32x4.splat $push[[R:[0-9]+]]=, $pop[[L0]]
 ; CHECK-NEXT: return $pop[[R]]{{$}}
 ; CHECK-NEXT: end_function{{$}}
 define <4 x i32> @implicit_def_v4i32() {
diff --git a/test/CodeGen/WebAssembly/import-module.ll b/test/CodeGen/WebAssembly/import-module.ll
index a8202a77acb5aeb56d2003a2ac875884f13772db..0cf0f2f25e0b9325e629ae5fcd96bdf9323351d8 100644
--- a/test/CodeGen/WebAssembly/import-module.ll
+++ b/test/CodeGen/WebAssembly/import-module.ll
@@ -12,8 +12,9 @@ define void @test() {
 declare void @foo() #0
 declare void @plain()
 
-attributes #0 = { "wasm-import-module"="bar" }
+attributes #0 = { "wasm-import-module"="bar" "wasm-import-name"="qux" }
 
 ; CHECK-NOT: .import_module plain
 ;     CHECK: .import_module foo, bar
+;     CHECK: .import_name foo, qux
 ; CHECK-NOT: .import_module plain
diff --git a/test/CodeGen/WebAssembly/inline-asm-roundtrip.ll b/test/CodeGen/WebAssembly/inline-asm-roundtrip.ll
index 037d26eb2c4f8a12e9169e7aa2e76f4f6e2fe132..2f79cc88df6e0a0f5b7995b88cf7f0b5c2fdbc89 100644
--- a/test/CodeGen/WebAssembly/inline-asm-roundtrip.ll
+++ b/test/CodeGen/WebAssembly/inline-asm-roundtrip.ll
@@ -9,9 +9,9 @@
 ;   int src = 1;
 ;   int dst;
 ;   asm ("i32.const\t2\n"
-;        "\tget_local\t%1\n"
+;        "\tlocal.get\t%1\n"
 ;        "\ti32.add\n"
-;        "\tset_local\t%0"
+;        "\tlocal.set\t%0"
 ;        : "=r" (dst)
 ;        : "r" (src));
 ;   return dst != 3;
@@ -24,18 +24,18 @@ target triple = "wasm32-unknown-unknown"
 ; CHECK-NEXT:	.functype main (i32, i32) -> (i32)
 ; CHECK-NEXT:	.local  	i32
 ; CHECK-NEXT:	i32.const	1
-; CHECK-NEXT:	set_local	[[SRC:[0-9]+]]
+; CHECK-NEXT:	local.set	[[SRC:[0-9]+]]
 ; CHECK-NEXT:	i32.const	2
-; CHECK-NEXT:	get_local	[[SRC]]
+; CHECK-NEXT:	local.get	[[SRC]]
 ; CHECK-NEXT:	i32.add
-; CHECK-NEXT:	set_local	[[DST:[0-9]+]]
-; CHECK-NEXT:	get_local	[[DST]]
+; CHECK-NEXT:	local.set	[[DST:[0-9]+]]
+; CHECK-NEXT:	local.get	[[DST]]
 ; CHECK-NEXT:	i32.const	3
 ; CHECK-NEXT:	i32.ne
 
 define i32 @main(i32 %argc, i8** nocapture readnone %argv) #0 {
 entry:
-  %0 = tail call i32 asm "i32.const\092\0A\09get_local\09$1\0A\09i32.add\0A\09set_local\09$0", "=r,r"(i32 1) #1
+  %0 = tail call i32 asm "i32.const\092\0A\09local.get\09$1\0A\09i32.add\0A\09local.set\09$0", "=r,r"(i32 1) #1
   %cmp = icmp ne i32 %0, 3
   %conv = zext i1 %cmp to i32
   ret i32 %conv
diff --git a/test/CodeGen/WebAssembly/inline-asm.ll b/test/CodeGen/WebAssembly/inline-asm.ll
index 657f9a6e399f05d7b6b55697901f4fea46d7e488..95c10a68b9c20e92e62a73b5777b19b00c951055 100644
--- a/test/CodeGen/WebAssembly/inline-asm.ll
+++ b/test/CodeGen/WebAssembly/inline-asm.ll
@@ -11,7 +11,7 @@ target triple = "wasm32-unknown-unknown"
 ; CHECK-NEXT: #APP{{$}}
 ; CHECK-NEXT: # 0 = aaa(0){{$}}
 ; CHECK-NEXT: #NO_APP{{$}}
-; CHECK-NEXT: get_local $push0=, 0{{$}}
+; CHECK-NEXT: local.get $push0=, 0{{$}}
 ; CHECK-NEXT: return $pop0{{$}}
 define i32 @foo(i32 %r) {
 entry:
@@ -25,7 +25,7 @@ entry:
 ; CHECK-NEXT: #APP{{$}}
 ; CHECK-NEXT: # 0 = ccc(42){{$}}
 ; CHECK-NEXT: #NO_APP{{$}}
-; CHECK-NEXT: get_local $push0=, 0{{$}}
+; CHECK-NEXT: local.get $push0=, 0{{$}}
 ; CHECK-NEXT: return $pop0{{$}}
 define i32 @imm() {
 entry:
@@ -38,7 +38,7 @@ entry:
 ; CHECK-NEXT: #APP{{$}}
 ; CHECK-NEXT: # 0 = aaa(0){{$}}
 ; CHECK-NEXT: #NO_APP{{$}}
-; CHECK-NEXT: get_local $push0=, 0{{$}}
+; CHECK-NEXT: local.get $push0=, 0{{$}}
 ; CHECK-NEXT: return $pop0{{$}}
 define i64 @foo_i64(i64 %r) {
 entry:
@@ -48,8 +48,8 @@ entry:
 
 ; CHECK-LABEL: X_i16:
 ; CHECK: foo 1{{$}}
-; CHECK: get_local $push[[S0:[0-9]+]]=, 0{{$}}
-; CHECK-NEXT: get_local $push[[S1:[0-9]+]]=, 1{{$}}
+; CHECK: local.get $push[[S0:[0-9]+]]=, 0{{$}}
+; CHECK-NEXT: local.get $push[[S1:[0-9]+]]=, 1{{$}}
 ; CHECK-NEXT: i32.store16 0($pop[[S0]]), $pop[[S1]]{{$}}
 define void @X_i16(i16 * %t) {
   call void asm sideeffect "foo $0", "=*X,~{dirflag},~{fpsr},~{flags},~{memory}"(i16* %t)
@@ -58,8 +58,8 @@ define void @X_i16(i16 * %t) {
 
 ; CHECK-LABEL: X_ptr:
 ; CHECK: foo 1{{$}}
-; CHECK: get_local $push[[S0:[0-9]+]]=, 0{{$}}
-; CHECK-NEXT: get_local $push[[S1:[0-9]+]]=, 1{{$}}
+; CHECK: local.get $push[[S0:[0-9]+]]=, 0{{$}}
+; CHECK-NEXT: local.get $push[[S1:[0-9]+]]=, 1{{$}}
 ; CHECK-NEXT: i32.store 0($pop[[S0]]), $pop[[S1]]{{$}}
 define void @X_ptr(i16 ** %t) {
   call void asm sideeffect "foo $0", "=*X,~{dirflag},~{fpsr},~{flags},~{memory}"(i16** %t)
@@ -83,11 +83,11 @@ define void @varname() {
 
 ; CHECK-LABEL: r_constraint
 ; CHECK:      i32.const $push[[S0:[0-9]+]]=, 0{{$}}
-; CHECK-NEXT: set_local [[L0:[0-9]+]], $pop[[S0]]{{$}}
+; CHECK-NEXT: local.set [[L0:[0-9]+]], $pop[[S0]]{{$}}
 ; CHECK-NEXT: i32.const $push[[S1:[0-9]+]]=, 37{{$}}
-; CHECK-NEXT: set_local [[L1:[0-9]+]], $pop[[S1]]{{$}}
+; CHECK-NEXT: local.set [[L1:[0-9]+]], $pop[[S1]]{{$}}
 ; CHECK:      foo [[L2:[0-9]+]], 1, [[L0]], [[L1]]{{$}}
-; CHECK:      get_local $push{{[0-9]+}}=, [[L2]]{{$}}
+; CHECK:      local.get $push{{[0-9]+}}=, [[L2]]{{$}}
 define hidden i32 @r_constraint(i32 %a, i32 %y) {
 entry:
   %z = bitcast i32 0 to i32
@@ -96,7 +96,7 @@ entry:
 }
 
 ; CHECK-LABEL: tied_operands
-; CHECK: get_local  $push0=, 0
+; CHECK: local.get  $push0=, 0
 ; CHECK: return    $pop0
 define i32 @tied_operands(i32 %var) {
 entry:
diff --git a/test/CodeGen/WebAssembly/irreducible-cfg-exceptions.ll b/test/CodeGen/WebAssembly/irreducible-cfg-exceptions.ll
new file mode 100644
index 0000000000000000000000000000000000000000..00fb6769bbd7b5e64c2743a2bbf57dbe4b00b42a
--- /dev/null
+++ b/test/CodeGen/WebAssembly/irreducible-cfg-exceptions.ll
@@ -0,0 +1,108 @@
+; RUN: llc < %s -asm-verbose=false -verify-machineinstrs -disable-block-placement -wasm-disable-explicit-locals -wasm-keep-registers -enable-emscripten-cxx-exceptions | FileCheck %s
+
+target datalayout = "e-m:e-p:32:32-i64:64-n32:64-S128"
+target triple = "wasm32-unknown-unknown"
+
+declare i32 @__gxx_personality_v0(...)
+
+; Check an interesting case of complex control flow due to exceptions CFG rewriting.
+; There should *not* be any irreducible control flow here.
+
+; CHECK-LABEL: crashy:
+; CHECK-NOT: br_table
+
+; Function Attrs: minsize noinline optsize
+define void @crashy() personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
+entry:
+  invoke void undef()
+          to label %invoke.cont unwind label %lpad
+
+invoke.cont: ; preds = %entry
+  invoke void undef()
+          to label %invoke.cont4 unwind label %lpad3
+
+invoke.cont4: ; preds = %invoke.cont
+  %call.i82 = invoke i8* undef()
+          to label %invoke.cont6 unwind label %lpad3
+
+invoke.cont6: ; preds = %invoke.cont4
+  invoke void undef()
+          to label %invoke.cont13 unwind label %lpad12
+
+invoke.cont13: ; preds = %invoke.cont6
+  br label %for.cond
+
+for.cond: ; preds = %for.cond.backedge, %invoke.cont13
+  br i1 undef, label %exit2, label %land.lhs
+
+land.lhs: ; preds = %for.cond
+  %call.i.i.i.i92 = invoke i32 undef()
+          to label %exit1 unwind label %lpad16.loopexit
+
+exit1: ; preds = %land.lhs
+  br label %exit2
+
+exit2: ; preds = %exit1, %for.cond
+  %call.i.i12.i.i93 = invoke i32 undef()
+          to label %exit3 unwind label %lpad16.loopexit
+
+exit3: ; preds = %exit2
+  invoke void undef()
+          to label %invoke.cont23 unwind label %lpad22
+
+invoke.cont23: ; preds = %exit3
+  invoke void undef()
+          to label %invoke.cont25 unwind label %lpad22
+
+invoke.cont25: ; preds = %invoke.cont23
+  %call.i.i137 = invoke i32 undef()
+          to label %invoke.cont29 unwind label %lpad16.loopexit
+
+lpad: ; preds = %entry
+  %0 = landingpad { i8*, i32 }
+          cleanup
+  unreachable
+
+lpad3: ; preds = %invoke.cont4, %invoke.cont
+  %1 = landingpad { i8*, i32 }
+          cleanup
+  unreachable
+
+lpad12: ; preds = %invoke.cont6
+  %2 = landingpad { i8*, i32 }
+          cleanup
+  resume { i8*, i32 } undef
+
+lpad16.loopexit: ; preds = %if.then, %invoke.cont29, %invoke.cont25, %exit2, %land.lhs
+  %lpad.loopexit = landingpad { i8*, i32 }
+          cleanup
+  unreachable
+
+lpad22: ; preds = %invoke.cont23, %exit3
+  %3 = landingpad { i8*, i32 }
+          cleanup
+  unreachable
+
+invoke.cont29: ; preds = %invoke.cont25
+  invoke void undef()
+          to label %invoke.cont33 unwind label %lpad16.loopexit
+
+invoke.cont33: ; preds = %invoke.cont29
+  br label %for.inc
+
+for.inc: ; preds = %invoke.cont33
+  %cmp.i.i141 = icmp eq i8* undef, undef
+  br i1 %cmp.i.i141, label %if.then, label %if.end.i.i146
+
+if.then: ; preds = %for.inc
+  %call.i.i148 = invoke i32 undef()
+          to label %for.cond.backedge unwind label %lpad16.loopexit
+
+for.cond.backedge: ; preds = %if.end.i.i146, %if.then
+  br label %for.cond
+
+if.end.i.i146: ; preds = %for.inc
+  call void undef()
+  br label %for.cond.backedge
+}
+
diff --git a/test/CodeGen/WebAssembly/irreducible-cfg-nested.ll b/test/CodeGen/WebAssembly/irreducible-cfg-nested.ll
new file mode 100644
index 0000000000000000000000000000000000000000..56777090ad515924a38238a461c4e256bca4a8db
--- /dev/null
+++ b/test/CodeGen/WebAssembly/irreducible-cfg-nested.ll
@@ -0,0 +1,63 @@
+; RUN: llc < %s -asm-verbose=false -verify-machineinstrs -disable-block-placement -wasm-disable-explicit-locals -wasm-keep-registers | FileCheck %s
+
+target datalayout = "e-m:e-p:32:32-i64:64-n32:64-S128"
+target triple = "wasm32-unknown-unknown"
+
+
+; Test an interesting pattern of nested irreducibility.
+; Just check we resolve all the irreducibility here (if not we'd crash).
+
+; CHECK-LABEL: tre_parse:
+
+define void @tre_parse() {
+entry:
+  br label %for.cond.outer
+
+for.cond.outer:                                   ; preds = %do.body14, %entry
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.cond.backedge, %for.cond.outer
+  %nbranch.0 = phi i32* [ null, %for.cond.outer ], [ %call188, %for.cond.backedge ]
+  switch i8 undef, label %if.else [
+    i8 40, label %do.body14
+    i8 41, label %if.then63
+  ]
+
+do.body14:                                        ; preds = %for.cond
+  br label %for.cond.outer
+
+if.then63:                                        ; preds = %for.cond
+  unreachable
+
+if.else:                                          ; preds = %for.cond
+  switch i8 undef, label %if.then84 [
+    i8 92, label %if.end101
+    i8 42, label %if.end101
+  ]
+
+if.then84:                                        ; preds = %if.else
+  switch i8 undef, label %cleanup.thread [
+    i8 43, label %if.end101
+    i8 63, label %if.end101
+    i8 123, label %if.end101
+  ]
+
+if.end101:                                        ; preds = %if.then84, %if.then84, %if.then84, %if.else, %if.else
+  unreachable
+
+cleanup.thread:                                   ; preds = %if.then84
+  %call188 = tail call i32* undef(i32* %nbranch.0)
+  switch i8 undef, label %for.cond.backedge [
+    i8 92, label %land.lhs.true208
+    i8 0, label %if.else252
+  ]
+
+land.lhs.true208:                                 ; preds = %cleanup.thread
+  unreachable
+
+for.cond.backedge:                                ; preds = %cleanup.thread
+  br label %for.cond
+
+if.else252:                                       ; preds = %cleanup.thread
+  unreachable
+}
diff --git a/test/CodeGen/WebAssembly/irreducible-cfg-nested2.ll b/test/CodeGen/WebAssembly/irreducible-cfg-nested2.ll
new file mode 100644
index 0000000000000000000000000000000000000000..3dc0cd16e8842927020e72f7d60c45c21d26dcb7
--- /dev/null
+++ b/test/CodeGen/WebAssembly/irreducible-cfg-nested2.ll
@@ -0,0 +1,39 @@
+; RUN: llc < %s -asm-verbose=false -verify-machineinstrs -disable-block-placement -wasm-disable-explicit-locals -wasm-keep-registers | FileCheck %s
+
+target datalayout = "e-m:e-p:32:32-i64:64-n32:64-S128"
+target triple = "wasm32-unknown-unknown"
+
+; Test an interesting pattern of nested irreducibility.
+; Just check we resolve all the irreducibility here (if not we'd crash).
+
+; CHECK-LABEL: func_2:
+
+; Function Attrs: noinline nounwind optnone
+define void @func_2() {
+entry:
+  br i1 undef, label %lbl_937, label %if.else787
+
+lbl_937:                                          ; preds = %for.body978, %entry
+  br label %if.end965
+
+if.else787:                                       ; preds = %entry
+  br label %if.end965
+
+if.end965:                                        ; preds = %if.else787, %lbl_937
+  br label %for.cond967
+
+for.cond967:                                      ; preds = %for.end1035, %if.end965
+  br label %for.cond975
+
+for.cond975:                                      ; preds = %if.end984, %for.cond967
+  br i1 undef, label %for.body978, label %for.end1035
+
+for.body978:                                      ; preds = %for.cond975
+  br i1 undef, label %lbl_937, label %if.end984
+
+if.end984:                                        ; preds = %for.body978
+  br label %for.cond975
+
+for.end1035:                                      ; preds = %for.cond975
+  br label %for.cond967
+}
diff --git a/test/CodeGen/WebAssembly/irreducible-cfg.ll b/test/CodeGen/WebAssembly/irreducible-cfg.ll
index d289d9f440629626231b4f6c20561e5a513216b3..8eb0a6f592220e2464bfc110776fcc9817ae3207 100644
--- a/test/CodeGen/WebAssembly/irreducible-cfg.ll
+++ b/test/CodeGen/WebAssembly/irreducible-cfg.ll
@@ -9,7 +9,7 @@ target triple = "wasm32-unknown-unknown"
 
 ; CHECK-LABEL: test0:
 ; CHECK: f64.load
-; CHECK: i32.const $[[REG:[^,]+]]=, 0{{$}}
+; CHECK: i32.const $[[REG:[^,]+]]=
 ; CHECK: br_table  $[[REG]],
 define void @test0(double* %arg, i32 %arg1, i32 %arg2, i32 %arg3) {
 bb:
@@ -50,7 +50,7 @@ bb19:
 
 ; CHECK-LABEL: test1:
 ; CHECK: f64.load
-; CHECK: i32.const $[[REG:[^,]+]]=, 0{{$}}
+; CHECK: i32.const $[[REG:[^,]+]]=
 ; CHECK: br_table  $[[REG]],
 define void @test1(double* %arg, i32 %arg1, i32 %arg2, i32 %arg3) {
 bb:
@@ -92,3 +92,128 @@ bb13:
 bb19:
   ret void
 }
+
+; A simple loop 2 blocks that are both entries.
+
+; CHECK-LABEL: test2:
+; CHECK: br_if
+; CHECK: i32.const $[[REG:[^,]+]]=
+; CHECK: br_table  $[[REG]],
+define internal i32 @test2(i32) noinline {
+entry:
+  br label %A0
+
+A0:
+  %a0a = tail call i32 @test2(i32 1)
+  %a0b = icmp eq i32 %a0a, 0
+  br i1 %a0b, label %A1, label %A2
+
+A1:
+  %a1a = tail call i32 @test2(i32 2)
+  %a1b = icmp eq i32 %a1a, 0
+  br i1 %a1b, label %A1, label %A2
+
+A2:
+  %a2a = tail call i32 @test2(i32 3)
+  %a2b = icmp eq i32 %a2a, 0
+  br i1 %a2b, label %A1, label %A2
+}
+
+; An interesting loop with inner loop and if-else structure too.
+
+; CHECK-LABEL: test3:
+; CHECK: br_if
+define void @test3(i32 %ws) {
+entry:
+  %ws.addr = alloca i32, align 4
+  store volatile i32 %ws, i32* %ws.addr, align 4
+  %0 = load volatile i32, i32* %ws.addr, align 4
+  %tobool = icmp ne i32 %0, 0
+  br i1 %tobool, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  br label %wynn
+
+if.end:                                           ; preds = %entry
+  %1 = load volatile i32, i32* %ws.addr, align 4
+  %tobool1 = icmp ne i32 %1, 0
+  br i1 %tobool1, label %if.end9, label %if.then2
+
+if.then2:                                         ; preds = %if.end
+  br label %for.cond
+
+for.cond:                                         ; preds = %wynn, %if.then7, %if.then2
+  %2 = load volatile i32, i32* %ws.addr, align 4
+  %tobool3 = icmp ne i32 %2, 0
+  br i1 %tobool3, label %if.then4, label %if.end5
+
+if.then4:                                         ; preds = %for.cond
+  br label %if.end5
+
+if.end5:                                          ; preds = %if.then4, %for.cond
+  %3 = load volatile i32, i32* %ws.addr, align 4
+  %tobool6 = icmp ne i32 %3, 0
+  br i1 %tobool6, label %if.then7, label %if.end8
+
+if.then7:                                         ; preds = %if.end5
+  br label %for.cond
+
+if.end8:                                          ; preds = %if.end5
+  br label %wynn
+
+wynn:                                             ; preds = %if.end8, %if.then
+  br label %for.cond
+
+if.end9:                                          ; preds = %if.end
+  ret void
+}
+
+; Multi-level irreducibility, after reducing in the main scope we must then
+; reduce in the inner loop that we just created.
+; CHECK: br_table
+; CHECK: br_table
+define void @pi_next() {
+entry:
+  br i1 undef, label %sw.bb5, label %return
+
+sw.bb5:                                           ; preds = %entry
+  br i1 undef, label %if.then.i49, label %if.else.i52
+
+if.then.i49:                                      ; preds = %sw.bb5
+  br label %for.inc197.i
+
+if.else.i52:                                      ; preds = %sw.bb5
+  br label %for.cond57.i
+
+for.cond57.i:                                     ; preds = %for.inc205.i, %if.else.i52
+  store i32 0, i32* undef, align 4
+  br label %for.cond65.i
+
+for.cond65.i:                                     ; preds = %for.inc201.i, %for.cond57.i
+  br i1 undef, label %for.body70.i, label %for.inc205.i
+
+for.body70.i:                                     ; preds = %for.cond65.i
+  br label %for.cond76.i
+
+for.cond76.i:                                     ; preds = %for.inc197.i, %for.body70.i
+  %0 = phi i32 [ %inc199.i, %for.inc197.i ], [ 0, %for.body70.i ]
+  %cmp81.i = icmp slt i32 %0, 0
+  br i1 %cmp81.i, label %for.body82.i, label %for.inc201.i
+
+for.body82.i:                                     ; preds = %for.cond76.i
+  br label %for.inc197.i
+
+for.inc197.i:                                     ; preds = %for.body82.i, %if.then.i49
+  %inc199.i = add nsw i32 undef, 1
+  br label %for.cond76.i
+
+for.inc201.i:                                     ; preds = %for.cond76.i
+  br label %for.cond65.i
+
+for.inc205.i:                                     ; preds = %for.cond65.i
+  br label %for.cond57.i
+
+return:                                           ; preds = %entry
+  ret void
+}
+
diff --git a/test/CodeGen/WebAssembly/legalize.ll b/test/CodeGen/WebAssembly/legalize.ll
index ea6dabf0b9c20e6c2f9b05dd704a78a2bb3e302f..24b842513ddae261b8693de648a1f86ba8a2875d 100644
--- a/test/CodeGen/WebAssembly/legalize.ll
+++ b/test/CodeGen/WebAssembly/legalize.ll
@@ -34,7 +34,7 @@ define i64 @sext_in_reg_i32_i64(i64 %a) {
 
 ; CHECK-LABEL: fpext_f32_f64:
 ; CHECK: f32.load $push0=, 0($0){{$}}
-; CHECK: f64.promote/f32 $push1=, $pop0{{$}}
+; CHECK: f64.promote_f32 $push1=, $pop0{{$}}
 ; CHECK: return $pop1{{$}}
 define double @fpext_f32_f64(float *%p) {
   %v = load float, float* %p
@@ -44,7 +44,7 @@ define double @fpext_f32_f64(float *%p) {
 
 ; CHECK-LABEL: fpconv_f64_f32:
 ; CHECK: f64.load $push0=, 0($0){{$}}
-; CHECK: f32.demote/f64 $push1=, $pop0{{$}}
+; CHECK: f32.demote_f64 $push1=, $pop0{{$}}
 ; CHECK: return $pop1{{$}}
 define float @fpconv_f64_f32(double *%p) {
   %v = load double, double* %p
diff --git a/test/CodeGen/WebAssembly/load-ext-atomic.ll b/test/CodeGen/WebAssembly/load-ext-atomic.ll
index 891c161393523158717ab15e4f9b06ab05f8e6c5..a6d7d4f6fa5bd9253693116ce8c27f2f7335cc36 100644
--- a/test/CodeGen/WebAssembly/load-ext-atomic.ll
+++ b/test/CodeGen/WebAssembly/load-ext-atomic.ll
@@ -84,7 +84,7 @@ define i64 @zext_i16_i64(i16 *%p) {
 
 ; CHECK-LABEL: sext_i32_i64:
 ; CHECK: i32.atomic.load $push0=, 0($0){{$}}
-; CHECK: i64.extend_s/i32 $push1=, $pop0{{$}}
+; CHECK: i64.extend_i32_s $push1=, $pop0{{$}}
 ; CHECK-NEXT: return $pop1{{$}}
 define i64 @sext_i32_i64(i32 *%p) {
   %v = load atomic i32, i32* %p seq_cst, align 4
diff --git a/test/CodeGen/WebAssembly/load.ll b/test/CodeGen/WebAssembly/load.ll
index 1878c1cae7e0c84f8b939d2d7fd777b0fcc039a6..b06bef42ecf1731c3eb41ffd8947bb56c33ea0bb 100644
--- a/test/CodeGen/WebAssembly/load.ll
+++ b/test/CodeGen/WebAssembly/load.ll
@@ -8,7 +8,7 @@ target triple = "wasm32-unknown-unknown"
 
 ; CHECK-LABEL: ldi32:
 ; CHECK-NEXT: .functype ldi32 (i32) -> (i32){{$}}
-; CHECK-NEXT: get_local $push[[L0:[0-9]+]]=, 0{{$}}
+; CHECK-NEXT: local.get $push[[L0:[0-9]+]]=, 0{{$}}
 ; CHECK-NEXT: i32.load $push[[NUM:[0-9]+]]=, 0($pop[[L0]]){{$}}
 ; CHECK-NEXT: return $pop[[NUM]]{{$}}
 define i32 @ldi32(i32 *%p) {
@@ -18,7 +18,7 @@ define i32 @ldi32(i32 *%p) {
 
 ; CHECK-LABEL: ldi64:
 ; CHECK-NEXT: .functype ldi64 (i32) -> (i64){{$}}
-; CHECK-NEXT: get_local $push[[L0:[0-9]+]]=, 0{{$}}
+; CHECK-NEXT: local.get $push[[L0:[0-9]+]]=, 0{{$}}
 ; CHECK-NEXT: i64.load $push[[NUM:[0-9]+]]=, 0($pop[[L0]]){{$}}
 ; CHECK-NEXT: return $pop[[NUM]]{{$}}
 define i64 @ldi64(i64 *%p) {
@@ -28,7 +28,7 @@ define i64 @ldi64(i64 *%p) {
 
 ; CHECK-LABEL: ldf32:
 ; CHECK-NEXT: .functype ldf32 (i32) -> (f32){{$}}
-; CHECK-NEXT: get_local $push[[L0:[0-9]+]]=, 0{{$}}
+; CHECK-NEXT: local.get $push[[L0:[0-9]+]]=, 0{{$}}
 ; CHECK-NEXT: f32.load $push[[NUM:[0-9]+]]=, 0($pop[[L0]]){{$}}
 ; CHECK-NEXT: return $pop[[NUM]]{{$}}
 define float @ldf32(float *%p) {
@@ -38,7 +38,7 @@ define float @ldf32(float *%p) {
 
 ; CHECK-LABEL: ldf64:
 ; CHECK-NEXT: .functype ldf64 (i32) -> (f64){{$}}
-; CHECK-NEXT: get_local $push[[L0:[0-9]+]]=, 0{{$}}
+; CHECK-NEXT: local.get $push[[L0:[0-9]+]]=, 0{{$}}
 ; CHECK-NEXT: f64.load $push[[NUM:[0-9]+]]=, 0($pop[[L0]]){{$}}
 ; CHECK-NEXT: return $pop[[NUM]]{{$}}
 define double @ldf64(double *%p) {
diff --git a/test/CodeGen/WebAssembly/main-declaration.ll b/test/CodeGen/WebAssembly/main-declaration.ll
index f9d68db2bae8e7add4ed03917b1bb97ce15eccfe..544f5588c5043cdb473632df5a05df8d02ad3927 100644
--- a/test/CodeGen/WebAssembly/main-declaration.ll
+++ b/test/CodeGen/WebAssembly/main-declaration.ll
@@ -1,20 +1,18 @@
-; RUN: llc < %s -asm-verbose=false -wasm-temporary-workarounds=false | FileCheck %s
+; RUN: llc < %s -asm-verbose=false | FileCheck %s
 
 ; Test main functions with alternate signatures.
 
 target datalayout = "e-m:e-p:32:32-i64:64-n32:64-S128"
 target triple = "wasm32-unknown-unknown"
 
-declare void @main()
+declare i32 @main()
 
-define void @foo() {
-  call void @main()
-  ret void
+define i32 @foo() {
+  %t = call i32 @main()
+  ret i32 %t
 }
 
-; CHECK-NOT:   __original_main
 ; CHECK-LABEL: foo:
-; CHECK-NEXT:    .functype foo () -> ()
-; CHECK-NEXT:    call main@FUNCTION
+; CHECK-NEXT:    .functype foo () -> (i32)
+; CHECK-NEXT:    call __original_main@FUNCTION
 ; CHECK-NEXT:    end_function
-; CHECK-NOT:   __original_main
diff --git a/test/CodeGen/WebAssembly/main-no-args.ll b/test/CodeGen/WebAssembly/main-no-args.ll
index 0bc46717d97be315d4102a38eeddde8141dfd739..97023e269454b6682dffb3540f42ff296ffe9a92 100644
--- a/test/CodeGen/WebAssembly/main-no-args.ll
+++ b/test/CodeGen/WebAssembly/main-no-args.ll
@@ -1,18 +1,19 @@
-; RUN: llc < %s -asm-verbose=false -wasm-temporary-workarounds=false | FileCheck %s
+; RUN: llc < %s -asm-verbose=false | FileCheck %s
 
 ; Test main functions with alternate signatures.
 
 target datalayout = "e-m:e-p:32:32-i64:64-n32:64-S128"
 target triple = "wasm32-unknown-unknown"
 
-define void @main() {
-  ret void
+define i32 @main() {
+  ret i32 0
 }
 
-; CHECK-LABEL: .L__original_main:
-; CHECK-NEXT: .functype .L__original_main () -> ()
+; CHECK-LABEL: __original_main:
+; CHECK-NEXT: .functype __original_main () -> (i32)
+; CHECK-NEXT: i32.const 0
 ; CHECK-NEXT: end_function
 
 ; CHECK-LABEL: main:
 ; CHECK-NEXT: .functype main (i32, i32) -> (i32)
-; CHECK:      call .L__original_main@FUNCTION
+; CHECK:      call __original_main@FUNCTION
diff --git a/test/CodeGen/WebAssembly/main-three-args.ll b/test/CodeGen/WebAssembly/main-three-args.ll
new file mode 100644
index 0000000000000000000000000000000000000000..77b3e5b8c30631a202390d20a2c476a260488537
--- /dev/null
+++ b/test/CodeGen/WebAssembly/main-three-args.ll
@@ -0,0 +1,16 @@
+; RUN: llc < %s -asm-verbose=false | FileCheck %s
+
+; Test that main function with a non-standard third argument is
+; not wrapped.
+
+target datalayout = "e-m:e-p:32:32-i64:64-n32:64-S128"
+target triple = "wasm32-unknown-unknown"
+
+define i32 @main(i32 %a, i8** %b, i8** %c) {
+  ret i32 0
+}
+
+; CHECK-LABEL: main:
+; CHECK-NEXT: .functype main (i32, i32, i32) -> (i32)
+
+; CHECK-NOT: __original_main:
diff --git a/test/CodeGen/WebAssembly/main-with-args.ll b/test/CodeGen/WebAssembly/main-with-args.ll
index d4a11ef14d46e51c9e8e04efda458b9fbd29b5ae..205cb133f8ca688a7df9506e49352994c9949e2c 100644
--- a/test/CodeGen/WebAssembly/main-with-args.ll
+++ b/test/CodeGen/WebAssembly/main-with-args.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -asm-verbose=false -wasm-temporary-workarounds=false | FileCheck %s
+; RUN: llc < %s -asm-verbose=false | FileCheck %s
 
 ; Test that main function with expected signature is not wrapped
 
diff --git a/test/CodeGen/WebAssembly/memory-addr32.ll b/test/CodeGen/WebAssembly/memory-addr32.ll
index 0eb886fb0796fb23257e2e656ca1a6a904c52d96..26e9d4819ac61d58381f0839b26a2c81ab091159 100644
--- a/test/CodeGen/WebAssembly/memory-addr32.ll
+++ b/test/CodeGen/WebAssembly/memory-addr32.ll
@@ -7,10 +7,6 @@ target triple = "wasm32-unknown-unknown"
 
 declare i32 @llvm.wasm.memory.size.i32(i32) nounwind readonly
 declare i32 @llvm.wasm.memory.grow.i32(i32, i32) nounwind
-declare i32 @llvm.wasm.mem.size.i32(i32) nounwind readonly
-declare i32 @llvm.wasm.mem.grow.i32(i32, i32) nounwind
-declare i32 @llvm.wasm.current.memory.i32() nounwind readonly
-declare i32 @llvm.wasm.grow.memory.i32(i32) nounwind
 
 ; CHECK-LABEL: memory_size:
 ; CHECK-NEXT: .functype memory_size () -> (i32){{$}}
@@ -29,39 +25,3 @@ define i32 @memory_grow(i32 %n) {
   %a = call i32 @llvm.wasm.memory.grow.i32(i32 0, i32 %n)
   ret i32 %a
 }
-
-; CHECK-LABEL: mem_size:
-; CHECK-NEXT: .functype mem_size () -> (i32){{$}}
-; CHECK-NEXT: mem.size $push0=, 0{{$}}
-; CHECK-NEXT: return $pop0{{$}}
-define i32 @mem_size() {
-  %a = call i32 @llvm.wasm.mem.size.i32(i32 0)
-  ret i32 %a
-}
-
-; CHECK-LABEL: mem_grow:
-; CHECK-NEXT: .functype mem_grow (i32) -> (i32){{$}}
-; CHECK: mem.grow $push0=, 0, $0{{$}}
-; CHECK-NEXT: return $pop0{{$}}
-define i32 @mem_grow(i32 %n) {
-  %a = call i32 @llvm.wasm.mem.grow.i32(i32 0, i32 %n)
-  ret i32 %a
-}
-
-; CHECK-LABEL: current_memory:
-; CHECK-NEXT: .functype current_memory () -> (i32){{$}}
-; CHECK-NEXT: current_memory $push0={{$}}
-; CHECK-NEXT: return $pop0{{$}}
-define i32 @current_memory() {
-  %a = call i32 @llvm.wasm.current.memory.i32()
-  ret i32 %a
-}
-
-; CHECK-LABEL: grow_memory:
-; CHECK-NEXT: .functype grow_memory (i32) -> (i32){{$}}
-; CHECK: grow_memory $push0=, $0{{$}}
-; CHECK-NEXT: return $pop0{{$}}
-define i32 @grow_memory(i32 %n) {
-  %a = call i32 @llvm.wasm.grow.memory.i32(i32 %n)
-  ret i32 %a
-}
diff --git a/test/CodeGen/WebAssembly/negative-base-reg.ll b/test/CodeGen/WebAssembly/negative-base-reg.ll
index 0be276f2f83ee700aeb076ff362dc66c6b4c2d30..7408e14fc42742daf92db7efbefd7f08514e401b 100644
--- a/test/CodeGen/WebAssembly/negative-base-reg.ll
+++ b/test/CodeGen/WebAssembly/negative-base-reg.ll
@@ -11,7 +11,7 @@ define hidden i32 @main() local_unnamed_addr #0 {
 ; If LSR stops selecting a negative base reg value, then this test will no
 ; longer be useful as written.
 ; CHECK:      i32.const $push[[L0:[0-9]+]]=, -128
-; CHECK-NEXT: set_local 0, $pop[[L0]]
+; CHECK-NEXT: local.set 0, $pop[[L0]]
 entry:
   br label %for.body
 
diff --git a/test/CodeGen/WebAssembly/offset-atomics.ll b/test/CodeGen/WebAssembly/offset-atomics.ll
index 6406a51e0f8fef483747c543fe6ab1798c8da3a9..6884b6a56ee7d5569a2a5d05ac20d8c79eaa9094 100644
--- a/test/CodeGen/WebAssembly/offset-atomics.ll
+++ b/test/CodeGen/WebAssembly/offset-atomics.ll
@@ -363,10 +363,10 @@ define i32 @load_i8_i32_s_with_folded_offset(i8* %p) {
   ret i32 %u
 }
 
-; 32->64 sext load gets selected as i32.atomic.load, i64_extend_s/i32
+; 32->64 sext load gets selected as i32.atomic.load, i64.extend_i32_s
 ; CHECK-LABEL: load_i32_i64_s_with_folded_offset:
 ; CHECK: i32.atomic.load $push0=, 24($0){{$}}
-; CHECK-NEXT: i64.extend_s/i32 $push1=, $pop0{{$}}
+; CHECK-NEXT: i64.extend_i32_s $push1=, $pop0{{$}}
 define i64 @load_i32_i64_s_with_folded_offset(i32* %p) {
   %q = ptrtoint i32* %p to i32
   %r = add nuw i32 %q, 24
@@ -832,7 +832,7 @@ define i64 @rmw_add_i64_with_unfolded_gep_offset(i64* %p, i64 %v) {
 ; Fold an offset into a sign-extending rmw.
 
 ; CHECK-LABEL: rmw_add_i8_i32_s_with_folded_offset:
-; CHECK: i32.atomic.rmw8_u.add $push0=, 24($0), $1{{$}}
+; CHECK: i32.atomic.rmw8.add_u $push0=, 24($0), $1{{$}}
 ; CHECK-NEXT: i32.extend8_s $push1=, $pop0
 define i32 @rmw_add_i8_i32_s_with_folded_offset(i8* %p, i32 %v) {
   %q = ptrtoint i8* %p to i32
@@ -844,11 +844,11 @@ define i32 @rmw_add_i8_i32_s_with_folded_offset(i8* %p, i32 %v) {
   ret i32 %u
 }
 
-; 32->64 sext rmw gets selected as i32.atomic.rmw.add, i64_extend_s/i32
+; 32->64 sext rmw gets selected as i32.atomic.rmw.add, i64.extend_i32_s
 ; CHECK-LABEL: rmw_add_i32_i64_s_with_folded_offset:
-; CHECK: i32.wrap/i64 $push0=, $1
+; CHECK: i32.wrap_i64 $push0=, $1
 ; CHECK-NEXT: i32.atomic.rmw.add $push1=, 24($0), $pop0{{$}}
-; CHECK-NEXT: i64.extend_s/i32 $push2=, $pop1{{$}}
+; CHECK-NEXT: i64.extend_i32_s $push2=, $pop1{{$}}
 define i64 @rmw_add_i32_i64_s_with_folded_offset(i32* %p, i64 %v) {
   %q = ptrtoint i32* %p to i32
   %r = add nuw i32 %q, 24
@@ -862,7 +862,7 @@ define i64 @rmw_add_i32_i64_s_with_folded_offset(i32* %p, i64 %v) {
 ; Fold a gep offset into a sign-extending rmw.
 
 ; CHECK-LABEL: rmw_add_i8_i32_s_with_folded_gep_offset:
-; CHECK: i32.atomic.rmw8_u.add $push0=, 24($0), $1{{$}}
+; CHECK: i32.atomic.rmw8.add_u $push0=, 24($0), $1{{$}}
 ; CHECK-NEXT: i32.extend8_s $push1=, $pop0
 define i32 @rmw_add_i8_i32_s_with_folded_gep_offset(i8* %p, i32 %v) {
   %s = getelementptr inbounds i8, i8* %p, i32 24
@@ -873,7 +873,7 @@ define i32 @rmw_add_i8_i32_s_with_folded_gep_offset(i8* %p, i32 %v) {
 }
 
 ; CHECK-LABEL: rmw_add_i16_i32_s_with_folded_gep_offset:
-; CHECK: i32.atomic.rmw16_u.add $push0=, 48($0), $1{{$}}
+; CHECK: i32.atomic.rmw16.add_u $push0=, 48($0), $1{{$}}
 ; CHECK-NEXT: i32.extend16_s $push1=, $pop0
 define i32 @rmw_add_i16_i32_s_with_folded_gep_offset(i16* %p, i32 %v) {
   %s = getelementptr inbounds i16, i16* %p, i32 24
@@ -884,7 +884,7 @@ define i32 @rmw_add_i16_i32_s_with_folded_gep_offset(i16* %p, i32 %v) {
 }
 
 ; CHECK-LABEL: rmw_add_i16_i64_s_with_folded_gep_offset:
-; CHECK: i64.atomic.rmw16_u.add $push0=, 48($0), $1{{$}}
+; CHECK: i64.atomic.rmw16.add_u $push0=, 48($0), $1{{$}}
 ; CHECK-NEXT: i64.extend16_s $push1=, $pop0
 define i64 @rmw_add_i16_i64_s_with_folded_gep_offset(i16* %p, i64 %v) {
   %s = getelementptr inbounds i16, i16* %p, i32 24
@@ -898,7 +898,7 @@ define i64 @rmw_add_i16_i64_s_with_folded_gep_offset(i16* %p, i64 %v) {
 ; an 'add' if the or'ed bits are known to be zero.
 
 ; CHECK-LABEL: rmw_add_i8_i32_s_with_folded_or_offset:
-; CHECK: i32.atomic.rmw8_u.add $push[[R1:[0-9]+]]=, 2($pop{{[0-9]+}}), $1{{$}}
+; CHECK: i32.atomic.rmw8.add_u $push[[R1:[0-9]+]]=, 2($pop{{[0-9]+}}), $1{{$}}
 ; CHECK-NEXT: i32.extend8_s $push{{[0-9]+}}=, $pop[[R1]]{{$}}
 define i32 @rmw_add_i8_i32_s_with_folded_or_offset(i32 %x, i32 %v) {
   %and = and i32 %x, -4
@@ -911,7 +911,7 @@ define i32 @rmw_add_i8_i32_s_with_folded_or_offset(i32 %x, i32 %v) {
 }
 
 ; CHECK-LABEL: rmw_add_i8_i64_s_with_folded_or_offset:
-; CHECK: i64.atomic.rmw8_u.add $push[[R1:[0-9]+]]=, 2($pop{{[0-9]+}}), $1{{$}}
+; CHECK: i64.atomic.rmw8.add_u $push[[R1:[0-9]+]]=, 2($pop{{[0-9]+}}), $1{{$}}
 ; CHECK-NEXT: i64.extend8_s $push{{[0-9]+}}=, $pop[[R1]]{{$}}
 define i64 @rmw_add_i8_i64_s_with_folded_or_offset(i32 %x, i64 %v) {
   %and = and i32 %x, -4
@@ -927,7 +927,7 @@ define i64 @rmw_add_i8_i64_s_with_folded_or_offset(i32 %x, i64 %v) {
 
 ; CHECK-LABEL: rmw_add_i16_i32_s_from_numeric_address
 ; CHECK: i32.const $push0=, 0{{$}}
-; CHECK: i32.atomic.rmw16_u.add $push1=, 42($pop0), $0{{$}}
+; CHECK: i32.atomic.rmw16.add_u $push1=, 42($pop0), $0{{$}}
 ; CHECK-NEXT: i32.extend16_s $push2=, $pop1
 define i32 @rmw_add_i16_i32_s_from_numeric_address(i32 %v) {
   %s = inttoptr i32 42 to i16*
@@ -939,7 +939,7 @@ define i32 @rmw_add_i16_i32_s_from_numeric_address(i32 %v) {
 
 ; CHECK-LABEL: rmw_add_i8_i32_s_from_global_address
 ; CHECK: i32.const $push0=, 0{{$}}
-; CHECK: i32.atomic.rmw8_u.add $push1=, gv8($pop0), $0{{$}}
+; CHECK: i32.atomic.rmw8.add_u $push1=, gv8($pop0), $0{{$}}
 ; CHECK-NEXT: i32.extend8_s $push2=, $pop1{{$}}
 define i32 @rmw_add_i8_i32_s_from_global_address(i32 %v) {
   %t = trunc i32 %v to i8
@@ -955,7 +955,7 @@ define i32 @rmw_add_i8_i32_s_from_global_address(i32 %v) {
 ; Fold an offset into a zero-extending rmw.
 
 ; CHECK-LABEL: rmw_add_i8_i32_z_with_folded_offset:
-; CHECK: i32.atomic.rmw8_u.add $push0=, 24($0), $1{{$}}
+; CHECK: i32.atomic.rmw8.add_u $push0=, 24($0), $1{{$}}
 define i32 @rmw_add_i8_i32_z_with_folded_offset(i8* %p, i32 %v) {
   %q = ptrtoint i8* %p to i32
   %r = add nuw i32 %q, 24
@@ -967,7 +967,7 @@ define i32 @rmw_add_i8_i32_z_with_folded_offset(i8* %p, i32 %v) {
 }
 
 ; CHECK-LABEL: rmw_add_i32_i64_z_with_folded_offset:
-; CHECK: i64.atomic.rmw32_u.add $push0=, 24($0), $1{{$}}
+; CHECK: i64.atomic.rmw32.add_u $push0=, 24($0), $1{{$}}
 define i64 @rmw_add_i32_i64_z_with_folded_offset(i32* %p, i64 %v) {
   %q = ptrtoint i32* %p to i32
   %r = add nuw i32 %q, 24
@@ -981,7 +981,7 @@ define i64 @rmw_add_i32_i64_z_with_folded_offset(i32* %p, i64 %v) {
 ; Fold a gep offset into a zero-extending rmw.
 
 ; CHECK-LABEL: rmw_add_i8_i32_z_with_folded_gep_offset:
-; CHECK: i32.atomic.rmw8_u.add $push0=, 24($0), $1{{$}}
+; CHECK: i32.atomic.rmw8.add_u $push0=, 24($0), $1{{$}}
 define i32 @rmw_add_i8_i32_z_with_folded_gep_offset(i8* %p, i32 %v) {
   %s = getelementptr inbounds i8, i8* %p, i32 24
   %t = trunc i32 %v to i8
@@ -991,7 +991,7 @@ define i32 @rmw_add_i8_i32_z_with_folded_gep_offset(i8* %p, i32 %v) {
 }
 
 ; CHECK-LABEL: rmw_add_i16_i32_z_with_folded_gep_offset:
-; CHECK: i32.atomic.rmw16_u.add $push0=, 48($0), $1{{$}}
+; CHECK: i32.atomic.rmw16.add_u $push0=, 48($0), $1{{$}}
 define i32 @rmw_add_i16_i32_z_with_folded_gep_offset(i16* %p, i32 %v) {
   %s = getelementptr inbounds i16, i16* %p, i32 24
   %t = trunc i32 %v to i16
@@ -1001,7 +1001,7 @@ define i32 @rmw_add_i16_i32_z_with_folded_gep_offset(i16* %p, i32 %v) {
 }
 
 ; CHECK-LABEL: rmw_add_i16_i64_z_with_folded_gep_offset:
-; CHECK: i64.atomic.rmw16_u.add $push0=, 48($0), $1{{$}}
+; CHECK: i64.atomic.rmw16.add_u $push0=, 48($0), $1{{$}}
 define i64 @rmw_add_i16_i64_z_with_folded_gep_offset(i16* %p, i64 %v) {
   %s = getelementptr inbounds i16, i16* %p, i32 24
   %t = trunc i64 %v to i16
@@ -1014,7 +1014,7 @@ define i64 @rmw_add_i16_i64_z_with_folded_gep_offset(i16* %p, i64 %v) {
 ; an 'add' if the or'ed bits are known to be zero.
 
 ; CHECK-LABEL: rmw_add_i8_i32_z_with_folded_or_offset:
-; CHECK: i32.atomic.rmw8_u.add $push[[R1:[0-9]+]]=, 2($pop{{[0-9]+}}), $1{{$}}
+; CHECK: i32.atomic.rmw8.add_u $push[[R1:[0-9]+]]=, 2($pop{{[0-9]+}}), $1{{$}}
 define i32 @rmw_add_i8_i32_z_with_folded_or_offset(i32 %x, i32 %v) {
   %and = and i32 %x, -4
   %t0 = inttoptr i32 %and to i8*
@@ -1026,7 +1026,7 @@ define i32 @rmw_add_i8_i32_z_with_folded_or_offset(i32 %x, i32 %v) {
 }
 
 ; CHECK-LABEL: rmw_add_i8_i64_z_with_folded_or_offset:
-; CHECK: i64.atomic.rmw8_u.add $push[[R1:[0-9]+]]=, 2($pop{{[0-9]+}}), $1{{$}}
+; CHECK: i64.atomic.rmw8.add_u $push[[R1:[0-9]+]]=, 2($pop{{[0-9]+}}), $1{{$}}
 define i64 @rmw_add_i8_i64_z_with_folded_or_offset(i32 %x, i64 %v) {
   %and = and i32 %x, -4
   %t0 = inttoptr i32 %and to i8*
@@ -1041,7 +1041,7 @@ define i64 @rmw_add_i8_i64_z_with_folded_or_offset(i32 %x, i64 %v) {
 
 ; CHECK-LABEL: rmw_add_i16_i32_z_from_numeric_address
 ; CHECK: i32.const $push0=, 0{{$}}
-; CHECK: i32.atomic.rmw16_u.add $push1=, 42($pop0), $0{{$}}
+; CHECK: i32.atomic.rmw16.add_u $push1=, 42($pop0), $0{{$}}
 define i32 @rmw_add_i16_i32_z_from_numeric_address(i32 %v) {
   %s = inttoptr i32 42 to i16*
   %t = trunc i32 %v to i16
@@ -1052,7 +1052,7 @@ define i32 @rmw_add_i16_i32_z_from_numeric_address(i32 %v) {
 
 ; CHECK-LABEL: rmw_add_i8_i32_z_from_global_address
 ; CHECK: i32.const $push0=, 0{{$}}
-; CHECK: i32.atomic.rmw8_u.add $push1=, gv8($pop0), $0{{$}}
+; CHECK: i32.atomic.rmw8.add_u $push1=, gv8($pop0), $0{{$}}
 define i32 @rmw_add_i8_i32_z_from_global_address(i32 %v) {
   %t = trunc i32 %v to i8
   %old = atomicrmw add i8* @gv8, i8 %t seq_cst
@@ -1063,7 +1063,7 @@ define i32 @rmw_add_i8_i32_z_from_global_address(i32 %v) {
 ; i8 return value should test anyext RMWs
 
 ; CHECK-LABEL: rmw_add_i8_i32_retvalue:
-; CHECK: i32.atomic.rmw8_u.add $push0=, 0($0), $1{{$}}
+; CHECK: i32.atomic.rmw8.add_u $push0=, 0($0), $1{{$}}
 ; CHECK-NEXT: return $pop0{{$}}
 define i8 @rmw_add_i8_i32_retvalue(i8 *%p, i32 %v) {
   %t = trunc i32 %v to i8
@@ -1261,7 +1261,7 @@ define i64 @cmpxchg_i64_with_unfolded_gep_offset(i64* %p, i64 %exp, i64 %new) {
 ; Fold an offset into a sign-extending rmw.
 
 ; CHECK-LABEL: cmpxchg_i8_i32_s_with_folded_offset:
-; CHECK: i32.atomic.rmw8_u.cmpxchg $push0=, 24($0), $1, $2{{$}}
+; CHECK: i32.atomic.rmw8.cmpxchg_u $push0=, 24($0), $1, $2{{$}}
 ; CHECK-NEXT: i32.extend8_s $push1=, $pop0
 define i32 @cmpxchg_i8_i32_s_with_folded_offset(i8* %p, i32 %exp, i32 %new) {
   %q = ptrtoint i8* %p to i32
@@ -1275,12 +1275,12 @@ define i32 @cmpxchg_i8_i32_s_with_folded_offset(i8* %p, i32 %exp, i32 %new) {
   ret i32 %u
 }
 
-; 32->64 sext rmw gets selected as i32.atomic.rmw.cmpxchg, i64_extend_s/i32
+; 32->64 sext rmw gets selected as i32.atomic.rmw.cmpxchg, i64.extend_i32_s
 ; CHECK-LABEL: cmpxchg_i32_i64_s_with_folded_offset:
-; CHECK: i32.wrap/i64 $push1=, $1
-; CHECK-NEXT: i32.wrap/i64 $push0=, $2
+; CHECK: i32.wrap_i64 $push1=, $1
+; CHECK-NEXT: i32.wrap_i64 $push0=, $2
 ; CHECK-NEXT: i32.atomic.rmw.cmpxchg $push2=, 24($0), $pop1, $pop0{{$}}
-; CHECK-NEXT: i64.extend_s/i32 $push3=, $pop2{{$}}
+; CHECK-NEXT: i64.extend_i32_s $push3=, $pop2{{$}}
 define i64 @cmpxchg_i32_i64_s_with_folded_offset(i32* %p, i64 %exp, i64 %new) {
   %q = ptrtoint i32* %p to i32
   %r = add nuw i32 %q, 24
@@ -1296,7 +1296,7 @@ define i64 @cmpxchg_i32_i64_s_with_folded_offset(i32* %p, i64 %exp, i64 %new) {
 ; Fold a gep offset into a sign-extending rmw.
 
 ; CHECK-LABEL: cmpxchg_i8_i32_s_with_folded_gep_offset:
-; CHECK: i32.atomic.rmw8_u.cmpxchg $push0=, 24($0), $1, $2{{$}}
+; CHECK: i32.atomic.rmw8.cmpxchg_u $push0=, 24($0), $1, $2{{$}}
 ; CHECK-NEXT: i32.extend8_s $push1=, $pop0
 define i32 @cmpxchg_i8_i32_s_with_folded_gep_offset(i8* %p, i32 %exp, i32 %new) {
   %s = getelementptr inbounds i8, i8* %p, i32 24
@@ -1309,7 +1309,7 @@ define i32 @cmpxchg_i8_i32_s_with_folded_gep_offset(i8* %p, i32 %exp, i32 %new)
 }
 
 ; CHECK-LABEL: cmpxchg_i16_i32_s_with_folded_gep_offset:
-; CHECK: i32.atomic.rmw16_u.cmpxchg $push0=, 48($0), $1, $2{{$}}
+; CHECK: i32.atomic.rmw16.cmpxchg_u $push0=, 48($0), $1, $2{{$}}
 ; CHECK-NEXT: i32.extend16_s $push1=, $pop0
 define i32 @cmpxchg_i16_i32_s_with_folded_gep_offset(i16* %p, i32 %exp, i32 %new) {
   %s = getelementptr inbounds i16, i16* %p, i32 24
@@ -1322,7 +1322,7 @@ define i32 @cmpxchg_i16_i32_s_with_folded_gep_offset(i16* %p, i32 %exp, i32 %new
 }
 
 ; CHECK-LABEL: cmpxchg_i16_i64_s_with_folded_gep_offset:
-; CHECK: i64.atomic.rmw16_u.cmpxchg $push0=, 48($0), $1, $2{{$}}
+; CHECK: i64.atomic.rmw16.cmpxchg_u $push0=, 48($0), $1, $2{{$}}
 ; CHECK-NEXT: i64.extend16_s $push1=, $pop0
 define i64 @cmpxchg_i16_i64_s_with_folded_gep_offset(i16* %p, i64 %exp, i64 %new) {
   %s = getelementptr inbounds i16, i16* %p, i32 24
@@ -1338,7 +1338,7 @@ define i64 @cmpxchg_i16_i64_s_with_folded_gep_offset(i16* %p, i64 %exp, i64 %new
 ; an 'add' if the or'ed bits are known to be zero.
 
 ; CHECK-LABEL: cmpxchg_i8_i32_s_with_folded_or_offset:
-; CHECK: i32.atomic.rmw8_u.cmpxchg $push[[R1:[0-9]+]]=, 2($pop{{[0-9]+}}), $1, $2{{$}}
+; CHECK: i32.atomic.rmw8.cmpxchg_u $push[[R1:[0-9]+]]=, 2($pop{{[0-9]+}}), $1, $2{{$}}
 ; CHECK-NEXT: i32.extend8_s $push{{[0-9]+}}=, $pop[[R1]]{{$}}
 define i32 @cmpxchg_i8_i32_s_with_folded_or_offset(i32 %x, i32 %exp, i32 %new) {
   %and = and i32 %x, -4
@@ -1353,7 +1353,7 @@ define i32 @cmpxchg_i8_i32_s_with_folded_or_offset(i32 %x, i32 %exp, i32 %new) {
 }
 
 ; CHECK-LABEL: cmpxchg_i8_i64_s_with_folded_or_offset:
-; CHECK: i64.atomic.rmw8_u.cmpxchg $push[[R1:[0-9]+]]=, 2($pop{{[0-9]+}}), $1, $2{{$}}
+; CHECK: i64.atomic.rmw8.cmpxchg_u $push[[R1:[0-9]+]]=, 2($pop{{[0-9]+}}), $1, $2{{$}}
 ; CHECK-NEXT: i64.extend8_s $push{{[0-9]+}}=, $pop[[R1]]{{$}}
 define i64 @cmpxchg_i8_i64_s_with_folded_or_offset(i32 %x, i64 %exp, i64 %new) {
   %and = and i32 %x, -4
@@ -1371,7 +1371,7 @@ define i64 @cmpxchg_i8_i64_s_with_folded_or_offset(i32 %x, i64 %exp, i64 %new) {
 
 ; CHECK-LABEL: cmpxchg_i16_i32_s_from_numeric_address
 ; CHECK: i32.const $push0=, 0{{$}}
-; CHECK: i32.atomic.rmw16_u.cmpxchg $push1=, 42($pop0), $0, $1{{$}}
+; CHECK: i32.atomic.rmw16.cmpxchg_u $push1=, 42($pop0), $0, $1{{$}}
 ; CHECK-NEXT: i32.extend16_s $push2=, $pop1
 define i32 @cmpxchg_i16_i32_s_from_numeric_address(i32 %exp, i32 %new) {
   %s = inttoptr i32 42 to i16*
@@ -1385,7 +1385,7 @@ define i32 @cmpxchg_i16_i32_s_from_numeric_address(i32 %exp, i32 %new) {
 
 ; CHECK-LABEL: cmpxchg_i8_i32_s_from_global_address
 ; CHECK: i32.const $push0=, 0{{$}}
-; CHECK: i32.atomic.rmw8_u.cmpxchg $push1=, gv8($pop0), $0, $1{{$}}
+; CHECK: i32.atomic.rmw8.cmpxchg_u $push1=, gv8($pop0), $0, $1{{$}}
 ; CHECK-NEXT: i32.extend8_s $push2=, $pop1{{$}}
 define i32 @cmpxchg_i8_i32_s_from_global_address(i32 %exp, i32 %new) {
   %exp_t = trunc i32 %exp to i8
@@ -1403,7 +1403,7 @@ define i32 @cmpxchg_i8_i32_s_from_global_address(i32 %exp, i32 %new) {
 ; Fold an offset into a sign-extending rmw.
 
 ; CHECK-LABEL: cmpxchg_i8_i32_z_with_folded_offset:
-; CHECK: i32.atomic.rmw8_u.cmpxchg $push0=, 24($0), $1, $2{{$}}
+; CHECK: i32.atomic.rmw8.cmpxchg_u $push0=, 24($0), $1, $2{{$}}
 define i32 @cmpxchg_i8_i32_z_with_folded_offset(i8* %p, i32 %exp, i32 %new) {
   %q = ptrtoint i8* %p to i32
   %r = add nuw i32 %q, 24
@@ -1417,7 +1417,7 @@ define i32 @cmpxchg_i8_i32_z_with_folded_offset(i8* %p, i32 %exp, i32 %new) {
 }
 
 ; CHECK-LABEL: cmpxchg_i32_i64_z_with_folded_offset:
-; CHECK: i64.atomic.rmw32_u.cmpxchg $push0=, 24($0), $1, $2{{$}}
+; CHECK: i64.atomic.rmw32.cmpxchg_u $push0=, 24($0), $1, $2{{$}}
 define i64 @cmpxchg_i32_i64_z_with_folded_offset(i32* %p, i64 %exp, i64 %new) {
   %q = ptrtoint i32* %p to i32
   %r = add nuw i32 %q, 24
@@ -1433,7 +1433,7 @@ define i64 @cmpxchg_i32_i64_z_with_folded_offset(i32* %p, i64 %exp, i64 %new) {
 ; Fold a gep offset into a sign-extending rmw.
 
 ; CHECK-LABEL: cmpxchg_i8_i32_z_with_folded_gep_offset:
-; CHECK: i32.atomic.rmw8_u.cmpxchg $push0=, 24($0), $1, $2{{$}}
+; CHECK: i32.atomic.rmw8.cmpxchg_u $push0=, 24($0), $1, $2{{$}}
 define i32 @cmpxchg_i8_i32_z_with_folded_gep_offset(i8* %p, i32 %exp, i32 %new) {
   %s = getelementptr inbounds i8, i8* %p, i32 24
   %exp_t = trunc i32 %exp to i8
@@ -1445,7 +1445,7 @@ define i32 @cmpxchg_i8_i32_z_with_folded_gep_offset(i8* %p, i32 %exp, i32 %new)
 }
 
 ; CHECK-LABEL: cmpxchg_i16_i32_z_with_folded_gep_offset:
-; CHECK: i32.atomic.rmw16_u.cmpxchg $push0=, 48($0), $1, $2{{$}}
+; CHECK: i32.atomic.rmw16.cmpxchg_u $push0=, 48($0), $1, $2{{$}}
 define i32 @cmpxchg_i16_i32_z_with_folded_gep_offset(i16* %p, i32 %exp, i32 %new) {
   %s = getelementptr inbounds i16, i16* %p, i32 24
   %exp_t = trunc i32 %exp to i16
@@ -1457,7 +1457,7 @@ define i32 @cmpxchg_i16_i32_z_with_folded_gep_offset(i16* %p, i32 %exp, i32 %new
 }
 
 ; CHECK-LABEL: cmpxchg_i16_i64_z_with_folded_gep_offset:
-; CHECK: i64.atomic.rmw16_u.cmpxchg $push0=, 48($0), $1, $2{{$}}
+; CHECK: i64.atomic.rmw16.cmpxchg_u $push0=, 48($0), $1, $2{{$}}
 define i64 @cmpxchg_i16_i64_z_with_folded_gep_offset(i16* %p, i64 %exp, i64 %new) {
   %s = getelementptr inbounds i16, i16* %p, i32 24
   %exp_t = trunc i64 %exp to i16
@@ -1472,7 +1472,7 @@ define i64 @cmpxchg_i16_i64_z_with_folded_gep_offset(i16* %p, i64 %exp, i64 %new
 ; an 'add' if the or'ed bits are known to be zero.
 
 ; CHECK-LABEL: cmpxchg_i8_i32_z_with_folded_or_offset:
-; CHECK: i32.atomic.rmw8_u.cmpxchg $push[[R1:[0-9]+]]=, 2($pop{{[0-9]+}}), $1, $2{{$}}
+; CHECK: i32.atomic.rmw8.cmpxchg_u $push[[R1:[0-9]+]]=, 2($pop{{[0-9]+}}), $1, $2{{$}}
 define i32 @cmpxchg_i8_i32_z_with_folded_or_offset(i32 %x, i32 %exp, i32 %new) {
   %and = and i32 %x, -4
   %t0 = inttoptr i32 %and to i8*
@@ -1486,7 +1486,7 @@ define i32 @cmpxchg_i8_i32_z_with_folded_or_offset(i32 %x, i32 %exp, i32 %new) {
 }
 
 ; CHECK-LABEL: cmpxchg_i8_i64_z_with_folded_or_offset:
-; CHECK: i64.atomic.rmw8_u.cmpxchg $push[[R1:[0-9]+]]=, 2($pop{{[0-9]+}}), $1, $2{{$}}
+; CHECK: i64.atomic.rmw8.cmpxchg_u $push[[R1:[0-9]+]]=, 2($pop{{[0-9]+}}), $1, $2{{$}}
 define i64 @cmpxchg_i8_i64_z_with_folded_or_offset(i32 %x, i64 %exp, i64 %new) {
   %and = and i32 %x, -4
   %t0 = inttoptr i32 %and to i8*
@@ -1503,7 +1503,7 @@ define i64 @cmpxchg_i8_i64_z_with_folded_or_offset(i32 %x, i64 %exp, i64 %new) {
 
 ; CHECK-LABEL: cmpxchg_i16_i32_z_from_numeric_address
 ; CHECK: i32.const $push0=, 0{{$}}
-; CHECK: i32.atomic.rmw16_u.cmpxchg $push1=, 42($pop0), $0, $1{{$}}
+; CHECK: i32.atomic.rmw16.cmpxchg_u $push1=, 42($pop0), $0, $1{{$}}
 define i32 @cmpxchg_i16_i32_z_from_numeric_address(i32 %exp, i32 %new) {
   %s = inttoptr i32 42 to i16*
   %exp_t = trunc i32 %exp to i16
@@ -1516,7 +1516,7 @@ define i32 @cmpxchg_i16_i32_z_from_numeric_address(i32 %exp, i32 %new) {
 
 ; CHECK-LABEL: cmpxchg_i8_i32_z_from_global_address
 ; CHECK: i32.const $push0=, 0{{$}}
-; CHECK: i32.atomic.rmw8_u.cmpxchg $push1=, gv8($pop0), $0, $1{{$}}
+; CHECK: i32.atomic.rmw8.cmpxchg_u $push1=, gv8($pop0), $0, $1{{$}}
 define i32 @cmpxchg_i8_i32_z_from_global_address(i32 %exp, i32 %new) {
   %exp_t = trunc i32 %exp to i8
   %new_t = trunc i32 %new to i8
diff --git a/test/CodeGen/WebAssembly/offset-fastisel.ll b/test/CodeGen/WebAssembly/offset-fastisel.ll
index c41757c479bf89fa93ea571980993f6dba826b1f..9f10a6af17d47edfca9973db2337bdb93f4368dc 100644
--- a/test/CodeGen/WebAssembly/offset-fastisel.ll
+++ b/test/CodeGen/WebAssembly/offset-fastisel.ll
@@ -16,10 +16,10 @@ define void @store_i8_with_variable_gep_offset(i8* %p, i32 %idx) {
 }
 
 ; CHECK-LABEL: store_i8_with_array_alloca_gep:
-; CHECK: get_global  $push[[L0:[0-9]+]]=, __stack_pointer
+; CHECK: global.get  $push[[L0:[0-9]+]]=, __stack_pointer
 ; CHECK: i32.const   $push[[L1:[0-9]+]]=, 32{{$}}
 ; CHECK: i32.sub     $push[[L2:[0-9]+]]=, $pop[[L0]], $pop[[L1]]{{$}}
-; CHECK: copy_local  $push[[L3:[0-9]+]]=, $pop[[L2]]
+; CHECK: local.copy  $push[[L3:[0-9]+]]=, $pop[[L2]]
 ; CHECK: i32.add     $push[[L4:[0-9]+]]=, $pop[[L3]], $0{{$}}
 ; CHECK: i32.const   $push[[L5:[0-9]+]]=, 0{{$}}
 ; CHECK: i32.store8  0($pop[[L4]]), $pop[[L5]]{{$}}
diff --git a/test/CodeGen/WebAssembly/phi.ll b/test/CodeGen/WebAssembly/phi.ll
index 9582b25d91576534577e1a439b41c1f6cfa8401d..7e47f5a1c675f9b0494e933543b6874594c511ae 100644
--- a/test/CodeGen/WebAssembly/phi.ll
+++ b/test/CodeGen/WebAssembly/phi.ll
@@ -27,9 +27,9 @@ done:
 
 ; CHECK-LABEL: test1:
 ; CHECK: .LBB1_1:
-; CHECK: copy_local $[[NUM0:[0-9]+]]=, $[[NUM1:[0-9]+]]{{$}}
-; CHECK: copy_local $[[NUM1]]=, $[[NUM2:[0-9]+]]{{$}}
-; CHECK: copy_local $[[NUM2]]=, $[[NUM0]]{{$}}
+; CHECK: local.copy $[[NUM0:[0-9]+]]=, $[[NUM1:[0-9]+]]{{$}}
+; CHECK: local.copy $[[NUM1]]=, $[[NUM2:[0-9]+]]{{$}}
+; CHECK: local.copy $[[NUM2]]=, $[[NUM0]]{{$}}
 define i32 @test1(i32 %n) {
 entry:
   br label %loop
diff --git a/test/CodeGen/WebAssembly/reg-stackify.ll b/test/CodeGen/WebAssembly/reg-stackify.ll
index 53a52a5f30a824a9d30288884e13d2657ef63015..6b4487bc66fc358e13dc163ea42cc59322fe2874 100644
--- a/test/CodeGen/WebAssembly/reg-stackify.ll
+++ b/test/CodeGen/WebAssembly/reg-stackify.ll
@@ -125,17 +125,17 @@ define i32 @no_sink_readonly_call(i32 %x, i32 %y, i32* %p) {
 ; NOREGS-LABEL: stack_uses:
 ; NOREGS: .functype stack_uses (i32, i32, i32, i32) -> (i32){{$}}
 ; NOREGS-NEXT: block {{$}}
-; NOREGS-NEXT: get_local 0{{$}}
+; NOREGS-NEXT: local.get 0{{$}}
 ; NOREGS-NEXT: i32.const   1{{$}}
 ; NOREGS-NEXT: i32.lt_s
-; NOREGS-NEXT: get_local 1{{$}}
+; NOREGS-NEXT: local.get 1{{$}}
 ; NOREGS-NEXT: i32.const   2{{$}}
 ; NOREGS-NEXT: i32.lt_s
 ; NOREGS-NEXT: i32.xor {{$}}
-; NOREGS-NEXT: get_local 2{{$}}
+; NOREGS-NEXT: local.get 2{{$}}
 ; NOREGS-NEXT: i32.const   1{{$}}
 ; NOREGS-NEXT: i32.lt_s
-; NOREGS-NEXT: get_local 3{{$}}
+; NOREGS-NEXT: local.get 3{{$}}
 ; NOREGS-NEXT: i32.const   2{{$}}
 ; NOREGS-NEXT: i32.lt_s
 ; NOREGS-NEXT: i32.xor {{$}}
@@ -166,13 +166,13 @@ false:
 }
 
 ; Test an interesting case where the load has multiple uses and cannot
-; be trivially stackified. However, it can be stackified with a tee_local.
+; be trivially stackified. However, it can be stackified with a local.tee.
 
 ; CHECK-LABEL: multiple_uses:
 ; CHECK: .functype multiple_uses (i32, i32, i32) -> (){{$}}
 ; CHECK-NEXT: block   {{$}}
 ; CHECK-NEXT: i32.load    $push[[NUM0:[0-9]+]]=, 0($2){{$}}
-; CHECK-NEXT: tee_local   $push[[NUM1:[0-9]+]]=, $3=, $pop[[NUM0]]{{$}}
+; CHECK-NEXT: local.tee   $push[[NUM1:[0-9]+]]=, $3=, $pop[[NUM0]]{{$}}
 ; CHECK-NEXT: i32.ge_u    $push[[NUM2:[0-9]+]]=, $pop[[NUM1]], $1{{$}}
 ; CHECK-NEXT: br_if       0, $pop[[NUM2]]{{$}}
 ; CHECK-NEXT: i32.lt_u    $push[[NUM3:[0-9]+]]=, $3, $0{{$}}
@@ -185,18 +185,18 @@ false:
 ; NOREGS: .functype multiple_uses (i32, i32, i32) -> (){{$}}
 ; NOREGS: .local i32{{$}}
 ; NOREGS-NEXT: block {{$}}
-; NOREGS-NEXT: get_local   2{{$}}
+; NOREGS-NEXT: local.get   2{{$}}
 ; NOREGS-NEXT: i32.load    0{{$}}
-; NOREGS-NEXT: tee_local   3{{$}}
-; NOREGS-NEXT: get_local   1{{$}}
+; NOREGS-NEXT: local.tee   3{{$}}
+; NOREGS-NEXT: local.get   1{{$}}
 ; NOREGS-NEXT: i32.ge_u
 ; NOREGS-NEXT: br_if       0{{$}}
-; NOREGS-NEXT: get_local   3{{$}}
-; NOREGS-NEXT: get_local   0{{$}}
+; NOREGS-NEXT: local.get   3{{$}}
+; NOREGS-NEXT: local.get   0{{$}}
 ; NOREGS-NEXT: i32.lt_u
 ; NOREGS-NEXT: br_if       0{{$}}
-; NOREGS-NEXT: get_local   2{{$}}
-; NOREGS-NEXT: get_local   3{{$}}
+; NOREGS-NEXT: local.get   2{{$}}
+; NOREGS-NEXT: local.get   3{{$}}
 ; NOREGS-NEXT: i32.store   0{{$}}
 ; NOREGS-NEXT: .LBB8_3:
 ; NOREGS-NEXT: end_block{{$}}
@@ -270,33 +270,33 @@ entry:
 ; CHECK-NEXT: return      $pop[[L14]]{{$}}
 ; NOREGS-LABEL: div_tree:
 ; NOREGS: .functype div_tree (i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32) -> (i32){{$}}
-; NOREGS-NEXT: get_local 0{{$}}
-; NOREGS-NEXT: get_local 1{{$}}
+; NOREGS-NEXT: local.get 0{{$}}
+; NOREGS-NEXT: local.get 1{{$}}
 ; NOREGS-NEXT: i32.div_s{{$}}
-; NOREGS-NEXT: get_local 2{{$}}
-; NOREGS-NEXT: get_local 3{{$}}
+; NOREGS-NEXT: local.get 2{{$}}
+; NOREGS-NEXT: local.get 3{{$}}
 ; NOREGS-NEXT: i32.div_s{{$}}
 ; NOREGS-NEXT: i32.div_s{{$}}
-; NOREGS-NEXT: get_local 4{{$}}
-; NOREGS-NEXT: get_local 5{{$}}
+; NOREGS-NEXT: local.get 4{{$}}
+; NOREGS-NEXT: local.get 5{{$}}
 ; NOREGS-NEXT: i32.div_s{{$}}
-; NOREGS-NEXT: get_local 6{{$}}
-; NOREGS-NEXT: get_local 7{{$}}
+; NOREGS-NEXT: local.get 6{{$}}
+; NOREGS-NEXT: local.get 7{{$}}
 ; NOREGS-NEXT: i32.div_s{{$}}
 ; NOREGS-NEXT: i32.div_s{{$}}
 ; NOREGS-NEXT: i32.div_s{{$}}
-; NOREGS-NEXT: get_local 8{{$}}
-; NOREGS-NEXT: get_local 9{{$}}
+; NOREGS-NEXT: local.get 8{{$}}
+; NOREGS-NEXT: local.get 9{{$}}
 ; NOREGS-NEXT: i32.div_s{{$}}
-; NOREGS-NEXT: get_local 10{{$}}
-; NOREGS-NEXT: get_local 11{{$}}
+; NOREGS-NEXT: local.get 10{{$}}
+; NOREGS-NEXT: local.get 11{{$}}
 ; NOREGS-NEXT: i32.div_s{{$}}
 ; NOREGS-NEXT: i32.div_s{{$}}
-; NOREGS-NEXT: get_local 12{{$}}
-; NOREGS-NEXT: get_local 13{{$}}
+; NOREGS-NEXT: local.get 12{{$}}
+; NOREGS-NEXT: local.get 13{{$}}
 ; NOREGS-NEXT: i32.div_s{{$}}
-; NOREGS-NEXT: get_local 14{{$}}
-; NOREGS-NEXT: get_local 15{{$}}
+; NOREGS-NEXT: local.get 14{{$}}
+; NOREGS-NEXT: local.get 15{{$}}
 ; NOREGS-NEXT: i32.div_s{{$}}
 ; NOREGS-NEXT: i32.div_s{{$}}
 ; NOREGS-NEXT: i32.div_s{{$}}
@@ -327,18 +327,18 @@ entry:
 ; CHECK-LABEL: simple_multiple_use:
 ; CHECK:       .functype simple_multiple_use (i32, i32) -> (){{$}}
 ; CHECK-NEXT:  i32.mul     $push[[NUM0:[0-9]+]]=, $1, $0{{$}}
-; CHECK-NEXT:  tee_local   $push[[NUM1:[0-9]+]]=, $[[NUM2:[0-9]+]]=, $pop[[NUM0]]{{$}}
+; CHECK-NEXT:  local.tee   $push[[NUM1:[0-9]+]]=, $[[NUM2:[0-9]+]]=, $pop[[NUM0]]{{$}}
 ; CHECK-NEXT:  call        use_a@FUNCTION, $pop[[NUM1]]{{$}}
 ; CHECK-NEXT:  call        use_b@FUNCTION, $[[NUM2]]{{$}}
 ; CHECK-NEXT:  return{{$}}
 ; NOREGS-LABEL: simple_multiple_use:
 ; NOREGS:       .functype simple_multiple_use (i32, i32) -> (){{$}}
-; NOREGS-NEXT:  get_local 1{{$}}
-; NOREGS-NEXT:  get_local 0{{$}}
+; NOREGS-NEXT:  local.get 1{{$}}
+; NOREGS-NEXT:  local.get 0{{$}}
 ; NOREGS-NEXT:  i32.mul
-; NOREGS-NEXT:  tee_local   1{{$}}
+; NOREGS-NEXT:  local.tee   1{{$}}
 ; NOREGS-NEXT:  call        use_a@FUNCTION{{$}}
-; NOREGS-NEXT:  get_local   1{{$}}
+; NOREGS-NEXT:  local.get   1{{$}}
 ; NOREGS-NEXT:  call        use_b@FUNCTION{{$}}
 ; NOREGS-NEXT:  return{{$}}
 declare void @use_a(i32)
@@ -355,16 +355,16 @@ define void @simple_multiple_use(i32 %x, i32 %y) {
 ; CHECK-LABEL: multiple_uses_in_same_insn:
 ; CHECK:       .functype multiple_uses_in_same_insn (i32, i32) -> (){{$}}
 ; CHECK-NEXT:  i32.mul     $push[[NUM0:[0-9]+]]=, $1, $0{{$}}
-; CHECK-NEXT:  tee_local   $push[[NUM1:[0-9]+]]=, $[[NUM2:[0-9]+]]=, $pop[[NUM0]]{{$}}
+; CHECK-NEXT:  local.tee   $push[[NUM1:[0-9]+]]=, $[[NUM2:[0-9]+]]=, $pop[[NUM0]]{{$}}
 ; CHECK-NEXT:  call        use_2@FUNCTION, $pop[[NUM1]], $[[NUM2]]{{$}}
 ; CHECK-NEXT:  return{{$}}
 ; NOREGS-LABEL: multiple_uses_in_same_insn:
 ; NOREGS:       .functype multiple_uses_in_same_insn (i32, i32) -> (){{$}}
-; NOREGS-NEXT:  get_local 1{{$}}
-; NOREGS-NEXT:  get_local 0{{$}}
+; NOREGS-NEXT:  local.get 1{{$}}
+; NOREGS-NEXT:  local.get 0{{$}}
 ; NOREGS-NEXT:  i32.mul
-; NOREGS-NEXT:  tee_local   1{{$}}
-; NOREGS-NEXT:  get_local   1{{$}}
+; NOREGS-NEXT:  local.tee   1{{$}}
+; NOREGS-NEXT:  local.get   1{{$}}
 ; NOREGS-NEXT:  call        use_2@FUNCTION{{$}}
 ; NOREGS-NEXT:  return{{$}}
 declare void @use_2(i32, i32)
@@ -405,7 +405,7 @@ define i32 @commute() {
 }
 
 ; Don't stackify a register when it would move a the def of the register past
-; an implicit get_local for the register.
+; an implicit local.get for the register.
 
 ; CHECK-LABEL: no_stackify_past_use:
 ; CHECK:      i32.call        $1=, callee@FUNCTION, $0
@@ -416,16 +416,16 @@ define i32 @commute() {
 ; CHECK-NEXT: i32.div_s       $push4=, $pop3, $1
 ; CHECK-NEXT: return          $pop4
 ; NOREGS-LABEL: no_stackify_past_use:
-; NOREGS:      get_local       0{{$}}
+; NOREGS:      local.get       0{{$}}
 ; NOREGS-NEXT: i32.call        callee@FUNCTION
-; NOREGS-NEXT: set_local       1{{$}}
-; NOREGS-NEXT: get_local       0{{$}}
+; NOREGS-NEXT: local.set       1{{$}}
+; NOREGS-NEXT: local.get       0{{$}}
 ; NOREGS-NEXT: i32.const       1
 ; NOREGS-NEXT: i32.add
 ; NOREGS-NEXT: i32.call        callee@FUNCTION
-; NOREGS-NEXT: get_local       1{{$}}
+; NOREGS-NEXT: local.get       1{{$}}
 ; NOREGS-NEXT: i32.sub
-; NOREGS-NEXT: get_local       1{{$}}
+; NOREGS-NEXT: local.get       1{{$}}
 ; NOREGS-NEXT: i32.div_s
 ; NOREGS-NEXT: return
 declare i32 @callee(i32)
@@ -443,7 +443,7 @@ define i32 @no_stackify_past_use(i32 %arg) {
 
 ; CHECK-LABEL: commute_to_fix_ordering:
 ; CHECK: i32.call        $push[[L0:.+]]=, callee@FUNCTION, $0
-; CHECK: tee_local       $push[[L1:.+]]=, $1=, $pop[[L0]]
+; CHECK: local.tee       $push[[L1:.+]]=, $1=, $pop[[L0]]
 ; CHECK: i32.const       $push0=, 1
 ; CHECK: i32.add         $push1=, $0, $pop0
 ; CHECK: i32.call        $push2=, callee@FUNCTION, $pop1
@@ -451,11 +451,11 @@ define i32 @no_stackify_past_use(i32 %arg) {
 ; CHECK: i32.mul         $push4=, $pop[[L1]], $pop3
 ; CHECK: return          $pop4
 ; NOREGS-LABEL: commute_to_fix_ordering:
-; NOREGS: get_local       0{{$}}
+; NOREGS: local.get       0{{$}}
 ; NOREGS: i32.call        callee@FUNCTION
-; NOREGS: tee_local       1
-; NOREGS: get_local       1{{$}}
-; NOREGS: get_local       0{{$}}
+; NOREGS: local.tee       1
+; NOREGS: local.get       1{{$}}
+; NOREGS: local.get       0{{$}}
 ; NOREGS: i32.const       1
 ; NOREGS: i32.add
 ; NOREGS: i32.call        callee@FUNCTION
@@ -475,12 +475,12 @@ define i32 @commute_to_fix_ordering(i32 %arg) {
 
 ; CHECK-LABEL: multiple_defs:
 ; CHECK:        f64.add         $push[[NUM0:[0-9]+]]=, ${{[0-9]+}}, $pop{{[0-9]+}}{{$}}
-; CHECK-NEXT:   tee_local       $push[[NUM1:[0-9]+]]=, $[[NUM2:[0-9]+]]=, $pop[[NUM0]]{{$}}
+; CHECK-NEXT:   local.tee       $push[[NUM1:[0-9]+]]=, $[[NUM2:[0-9]+]]=, $pop[[NUM0]]{{$}}
 ; CHECK-NEXT:   f64.select      $push{{[0-9]+}}=, $pop{{[0-9]+}}, $pop[[NUM1]], ${{[0-9]+}}{{$}}
 ; CHECK:        $[[NUM2]]=,
 ; NOREGS-LABEL: multiple_defs:
 ; NOREGS:        f64.add
-; NOREGS:        tee_local
+; NOREGS:        local.tee
 ; NOREGS:        f64.select
 define void @multiple_defs(i32 %arg, i32 %arg1, i1 %arg2, i1 %arg3, i1 %arg4) {
 bb:
@@ -602,12 +602,12 @@ define i32 @no_stackify_past_epilogue() {
 ; CHECK-LABEL: stackify_indvar:
 ; CHECK:             i32.const   $push[[L5:.+]]=, 1{{$}}
 ; CHECK-NEXT:        i32.add     $push[[L4:.+]]=, $[[R0:.+]], $pop[[L5]]{{$}}
-; CHECK-NEXT:        tee_local   $push[[L3:.+]]=, $[[R0]]=, $pop[[L4]]{{$}}
+; CHECK-NEXT:        local.tee   $push[[L3:.+]]=, $[[R0]]=, $pop[[L4]]{{$}}
 ; CHECK-NEXT:        i32.ne      $push[[L2:.+]]=, $0, $pop[[L3]]{{$}}
 ; NOREGS-LABEL: stackify_indvar:
 ; NOREGS:             i32.const   1{{$}}
 ; NOREGS-NEXT:        i32.add
-; NOREGS-NEXT:        tee_local   2{{$}}
+; NOREGS-NEXT:        local.tee   2{{$}}
 ; NOREGS-NEXT:        i32.ne
 define void @stackify_indvar(i32 %tmp, i32* %v) #0 {
 bb:
@@ -630,11 +630,11 @@ bb10:                                             ; preds = %bb9, %bb
 
 ; CHECK-LABEL: stackpointer_dependency:
 ; CHECK:      call {{.+}}, stackpointer_callee@FUNCTION,
-; CHECK-NEXT: set_global __stack_pointer@GLOBAL,
+; CHECK-NEXT: global.set __stack_pointer@GLOBAL,
 ; NOREGS-LABEL: stackpointer_dependency:
 ; NOREGS:      call stackpointer_callee@FUNCTION
-; NOREGS:      set_global __stack_pointer
-declare i32 @stackpointer_callee(i8* readnone, i8* readnone)
+; NOREGS:      global.set __stack_pointer
+declare i32 @stackpointer_callee(i8* readnone, i8* readnone) nounwind readnone
 declare i8* @llvm.frameaddress(i32)
 define i32 @stackpointer_dependency(i8* readnone) {
   %2 = tail call i8* @llvm.frameaddress(i32 0)
@@ -646,13 +646,13 @@ define i32 @stackpointer_dependency(i8* readnone) {
 
 ; CHECK-LABEL: call_indirect_stackify:
 ; CHECK: i32.load  $push[[L4:.+]]=, 0($0)
-; CHECK-NEXT: tee_local $push[[L3:.+]]=, $0=, $pop[[L4]]
+; CHECK-NEXT: local.tee $push[[L3:.+]]=, $0=, $pop[[L4]]
 ; CHECK-NEXT: i32.load  $push[[L0:.+]]=, 0($0)
 ; CHECK-NEXT: i32.load  $push[[L1:.+]]=, 0($pop[[L0]])
 ; CHECK-NEXT: i32.call_indirect $push{{.+}}=, $pop[[L3]], $1, $pop[[L1]]
 ; NOREGS-LABEL: call_indirect_stackify:
 ; NOREGS: i32.load  0
-; NOREGS-NEXT: tee_local 0
+; NOREGS-NEXT: local.tee 0
 ; NOREGS:      i32.load  0
 ; NOREGS-NEXT: i32.load  0
 ; NOREGS-NEXT: i32.call_indirect
diff --git a/test/CodeGen/WebAssembly/return-int32.ll b/test/CodeGen/WebAssembly/return-int32.ll
index 066006ebf29aab74c458f41c447c10eeccda321b..e25ca94d85e35b38b219b8f7cb9ab93ff98db931 100644
--- a/test/CodeGen/WebAssembly/return-int32.ll
+++ b/test/CodeGen/WebAssembly/return-int32.ll
@@ -6,7 +6,7 @@ target triple = "wasm32-unknown-unknown"
 
 ; CHECK-LABEL: return_i32:
 ; CHECK-NEXT:  .functype return_i32 (i32) -> (i32){{$}}
-; CHECK-NEXT:  get_local  $push0=, 0
+; CHECK-NEXT:  local.get  $push0=, 0
 ; CHECK-NEXT:  end_function{{$}}
 define i32 @return_i32(i32 %p) {
   ret i32 %p
diff --git a/test/CodeGen/WebAssembly/signext-inreg.ll b/test/CodeGen/WebAssembly/signext-inreg.ll
index b6e2c763740931c4123006f37f6c7294949bfece..3af90bef20f0ce9c6f6b63184d239e79334ff5df 100644
--- a/test/CodeGen/WebAssembly/signext-inreg.ll
+++ b/test/CodeGen/WebAssembly/signext-inreg.ll
@@ -30,7 +30,7 @@ define i32 @i32_extend16_s(i16 %x) {
 
 ; CHECK-LABEL: i64_extend8_s:
 ; CHECK-NEXT: .functype i64_extend8_s (i32) -> (i64){{$}}
-; CHECK-NEXT: i64.extend_u/i32 $push[[NUM1:[0-9]+]]=, $0{{$}}
+; CHECK-NEXT: i64.extend_i32_u $push[[NUM1:[0-9]+]]=, $0{{$}}
 ; CHECK-NEXT: i64.extend8_s $push[[NUM2:[0-9]+]]=, $pop[[NUM1]]{{$}}
 ; CHECK-NEXT: return $pop[[NUM2]]{{$}}
 
@@ -43,7 +43,7 @@ define i64 @i64_extend8_s(i8 %x) {
 
 ; CHECK-LABEL: i64_extend16_s:
 ; CHECK-NEXT: .functype i64_extend16_s (i32) -> (i64){{$}}
-; CHECK-NEXT: i64.extend_u/i32 $push[[NUM1:[0-9]+]]=, $0{{$}}
+; CHECK-NEXT: i64.extend_i32_u $push[[NUM1:[0-9]+]]=, $0{{$}}
 ; CHECK-NEXT: i64.extend16_s $push[[NUM2:[0-9]+]]=, $pop[[NUM1]]{{$}}
 ; CHECK-NEXT: return $pop[[NUM2]]{{$}}
 
@@ -57,7 +57,7 @@ define i64 @i64_extend16_s(i16 %x) {
 ; No SIGN_EXTEND_INREG is needed for 32->64 extension.
 ; CHECK-LABEL: i64_extend32_s:
 ; CHECK-NEXT: .functype i64_extend32_s (i32) -> (i64){{$}}
-; CHECK-NEXT: i64.extend_s/i32 $push[[NUM:[0-9]+]]=, $0{{$}}
+; CHECK-NEXT: i64.extend_i32_s $push[[NUM:[0-9]+]]=, $0{{$}}
 ; CHECK-NEXT: return $pop[[NUM]]{{$}}
 define i64 @i64_extend32_s(i32 %x) {
   %a = sext i32 %x to i64
diff --git a/test/CodeGen/WebAssembly/simd-arith.ll b/test/CodeGen/WebAssembly/simd-arith.ll
index 6d3d04c9e494db9597a2fdb5ea02971df0e47032..8d7f0205415b466c28d7cec0656bf4d8fcca4c67 100644
--- a/test/CodeGen/WebAssembly/simd-arith.ll
+++ b/test/CodeGen/WebAssembly/simd-arith.ll
@@ -1,13 +1,13 @@
-; RUN: llc < %s -asm-verbose=false -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-keep-registers -wasm-enable-unimplemented-simd -mattr=+simd128 | FileCheck %s --check-prefixes CHECK,SIMD128,SIMD128-SLOW
-; RUN: llc < %s -asm-verbose=false -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-keep-registers -wasm-enable-unimplemented-simd -mattr=+simd128 -fast-isel | FileCheck %s --check-prefixes CHECK,SIMD128,SIMD128-FAST
-; RUN: llc < %s -asm-verbose=false -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-keep-registers -mattr=+simd128 | FileCheck %s --check-prefixes CHECK,SIMD128-VM
-; RUN: llc < %s -asm-verbose=false -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-keep-registers -mattr=+simd128 -fast-isel | FileCheck %s --check-prefixes CHECK,SIMD128-VM
-; RUN: llc < %s -asm-verbose=false -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-keep-registers -mattr=-simd128 | FileCheck %s --check-prefixes CHECK,NO-SIMD128
-; RUN: llc < %s -asm-verbose=false -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-keep-registers -mattr=-simd128 -fast-isel | FileCheck %s --check-prefixes CHECK,NO-SIMD128
+; RUN: llc < %s -asm-verbose=false -verify-machineinstrs -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-keep-registers -mattr=+unimplemented-simd128 | FileCheck %s --check-prefixes CHECK,SIMD128,SIMD128-SLOW
+; RUN: llc < %s -asm-verbose=false -verify-machineinstrs -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-keep-registers -mattr=+unimplemented-simd128 -fast-isel | FileCheck %s --check-prefixes CHECK,SIMD128,SIMD128-FAST
+; RUN: llc < %s -asm-verbose=false -verify-machineinstrs -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-keep-registers -mattr=+simd128 | FileCheck %s --check-prefixes CHECK,SIMD128-VM
+; RUN: llc < %s -asm-verbose=false -verify-machineinstrs -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-keep-registers -mattr=+simd128 -fast-isel | FileCheck %s --check-prefixes CHECK,SIMD128-VM
+; RUN: llc < %s -asm-verbose=false -verify-machineinstrs -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-keep-registers | FileCheck %s --check-prefixes CHECK,NO-SIMD128
+; RUN: llc < %s -asm-verbose=false -verify-machineinstrs -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-keep-registers -fast-isel | FileCheck %s --check-prefixes CHECK,NO-SIMD128
 
 ; check that a non-test run (including explicit locals pass) at least finishes
-; RUN: llc < %s -O0 -wasm-enable-unimplemented-simd -mattr=+simd128,+sign-ext
-; RUN: llc < %s -O2 -wasm-enable-unimplemented-simd -mattr=+simd128,+sign-ext
+; RUN: llc < %s -O0 -mattr=+unimplemented-simd128
+; RUN: llc < %s -O2 -mattr=+unimplemented-simd128
 
 ; Test that basic SIMD128 arithmetic operations assemble as expected.
 
@@ -89,12 +89,16 @@ define <16 x i8> @shl_const_v16i8(<16 x i8> %v) {
 ; CHECK-LABEL: shl_vec_v16i8:
 ; NO-SIMD128-NOT: i8x16
 ; SIMD128-NEXT: .functype shl_vec_v16i8 (v128, v128) -> (v128){{$}}
-; SIMD128-NEXT: i8x16.extract_lane_u $push[[L0:[0-9]+]]=, $0, 0{{$}}
-; SIMD128-NEXT: i8x16.extract_lane_u $push[[L1:[0-9]+]]=, $1, 0{{$}}
+; SIMD128-NEXT: i8x16.extract_lane_s $push[[L0:[0-9]+]]=, $0, 0{{$}}
+; SIMD128-NEXT: i32.const $push[[M0:[0-9]+]]=, 7{{$}}
+; SIMD128-NEXT: i8x16.splat $push[[M1:[0-9]+]]=, $pop[[M0]]{{$}}
+; SIMD128-NEXT: v128.and $push[[M2:[0-9]+]]=, $1, $pop[[M1]]{{$}}
+; SIMD128-NEXT: local.tee $push[[M:[0-9]+]]=, $1=, $pop[[M2]]{{$}}
+; SIMD128-NEXT: i8x16.extract_lane_u $push[[L1:[0-9]+]]=, $pop[[M]], 0{{$}}
 ; SIMD128-NEXT: i32.shl $push[[L2:[0-9]+]]=, $pop[[L0]], $pop[[L1]]{{$}}
 ; SIMD128-NEXT: i8x16.splat $push[[L3:[0-9]+]]=, $pop[[L2]]{{$}}
 ; Skip 14 lanes
-; SIMD128:      i8x16.extract_lane_u $push[[L4:[0-9]+]]=, $0, 15{{$}}
+; SIMD128:      i8x16.extract_lane_s $push[[L4:[0-9]+]]=, $0, 15{{$}}
 ; SIMD128-NEXT: i8x16.extract_lane_u $push[[L5:[0-9]+]]=, $1, 15{{$}}
 ; SIMD128-NEXT: i32.shl $push[[L6:[0-9]+]]=, $pop[[L4]], $pop[[L5]]{{$}}
 ; SIMD128-NEXT: i8x16.replace_lane $push[[R:[0-9]+]]=, $pop[[L7:[0-9]+]], 15, $pop[[L6]]{{$}}
@@ -121,23 +125,19 @@ define <16 x i8> @shr_s_v16i8(<16 x i8> %v, i8 %x) {
 ; CHECK-LABEL: shr_s_vec_v16i8:
 ; NO-SIMD128-NOT: i8x16
 ; SIMD128-NEXT: .functype shr_s_vec_v16i8 (v128, v128) -> (v128){{$}}
-; SIMD128-NEXT: i8x16.extract_lane_u $push[[L0:[0-9]+]]=, $0, 0{{$}}
-; SIMD128-NEXT: i32.const $push[[L1:[0-9]+]]=, 24{{$}}
-; SIMD128-NEXT: i32.shl $push[[L2:[0-9]+]]=, $pop[[L0]], $pop[[L1]]{{$}}
-; SIMD128-NEXT: i32.const $push[[L3:[0-9]+]]=, 24{{$}}
-; SIMD128-NEXT: i32.shr_s $push[[L4:[0-9]+]]=, $pop[[L2]], $pop[[L3]]{{$}}
-; SIMD128-NEXT: i8x16.extract_lane_u $push[[L5:[0-9]+]]=, $1, 0{{$}}
-; SIMD128-NEXT: i32.shr_s $push[[L6:[0-9]+]]=, $pop[[L4]], $pop[[L5]]{{$}}
-; SIMD128-NEXT: i8x16.splat $push[[L7:[0-9]+]]=, $pop[[L6]]{{$}}
+; SIMD128-NEXT: i8x16.extract_lane_s $push[[L0:[0-9]+]]=, $0, 0{{$}}
+; SIMD128-NEXT: i32.const $push[[M0:[0-9]+]]=, 7{{$}}
+; SIMD128-NEXT: i8x16.splat $push[[M1:[0-9]+]]=, $pop[[M0]]{{$}}
+; SIMD128-NEXT: v128.and $push[[M2:[0-9]+]]=, $1, $pop[[M1]]{{$}}
+; SIMD128-NEXT: local.tee $push[[M:[0-9]+]]=, $1=, $pop[[M2]]{{$}}
+; SIMD128-NEXT: i8x16.extract_lane_u $push[[L1:[0-9]+]]=, $pop[[M]], 0{{$}}
+; SIMD128-NEXT: i32.shr_s $push[[L2:[0-9]+]]=, $pop[[L0]], $pop[[L1]]{{$}}
+; SIMD128-NEXT: i8x16.splat $push[[L3:[0-9]+]]=, $pop[[L2]]{{$}}
 ; Skip 14 lanes
-; SIMD128:      i8x16.extract_lane_u $push[[L7:[0-9]+]]=, $0, 15{{$}}
-; SIMD128-NEXT: i32.const $push[[L8:[0-9]+]]=, 24{{$}}
-; SIMD128-NEXT: i32.shl $push[[L9:[0-9]+]]=, $pop[[L7]], $pop[[L8]]{{$}}
-; SIMD128-NEXT: i32.const $push[[L10:[0-9]+]]=, 24{{$}}
-; SIMD128-NEXT: i32.shr_s $push[[L11:[0-9]+]]=, $pop[[L9]], $pop[[L10]]{{$}}
-; SIMD128-NEXT: i8x16.extract_lane_u $push[[L12:[0-9]+]]=, $1, 15{{$}}
-; SIMD128-NEXT: i32.shr_s $push[[L13:[0-9]+]]=, $pop[[L11]], $pop[[L12]]{{$}}
-; SIMD128-NEXT: i8x16.replace_lane $push[[R:[0-9]+]]=, $pop[[L14:[0-9]+]], 15, $pop[[L13]]{{$}}
+; SIMD128:      i8x16.extract_lane_s $push[[L0:[0-9]+]]=, $0, 15{{$}}
+; SIMD128-NEXT: i8x16.extract_lane_u $push[[L1:[0-9]+]]=, $1, 15{{$}}
+; SIMD128-NEXT: i32.shr_s $push[[L2:[0-9]+]]=, $pop[[L0]], $pop[[L1]]{{$}}
+; SIMD128-NEXT: i8x16.replace_lane $push[[R:[0-9]+]]=, $pop{{[0-9]+}}, 15, $pop[[L2]]{{$}}
 ; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <16 x i8> @shr_s_vec_v16i8(<16 x i8> %v, <16 x i8> %x) {
   %a = ashr <16 x i8> %v, %x
@@ -162,7 +162,11 @@ define <16 x i8> @shr_u_v16i8(<16 x i8> %v, i8 %x) {
 ; NO-SIMD128-NOT: i8x16
 ; SIMD128-NEXT: .functype shr_u_vec_v16i8 (v128, v128) -> (v128){{$}}
 ; SIMD128-NEXT: i8x16.extract_lane_u $push[[L0:[0-9]+]]=, $0, 0{{$}}
-; SIMD128-NEXT: i8x16.extract_lane_u $push[[L1:[0-9]+]]=, $1, 0{{$}}
+; SIMD128-NEXT: i32.const $push[[M0:[0-9]+]]=, 7{{$}}
+; SIMD128-NEXT: i8x16.splat $push[[M1:[0-9]+]]=, $pop[[M0]]{{$}}
+; SIMD128-NEXT: v128.and $push[[M2:[0-9]+]]=, $1, $pop[[M1]]{{$}}
+; SIMD128-NEXT: local.tee $push[[M:[0-9]+]]=, $1=, $pop[[M2]]{{$}}
+; SIMD128-NEXT: i8x16.extract_lane_u $push[[L1:[0-9]+]]=, $pop[[M]], 0{{$}}
 ; SIMD128-NEXT: i32.shr_u $push[[L2:[0-9]+]]=, $pop[[L0]], $pop[[L1]]{{$}}
 ; SIMD128-NEXT: i8x16.splat $push[[L3:[0-9]+]]=, $pop[[L2]]{{$}}
 ; Skip 14 lanes
@@ -311,12 +315,16 @@ define <8 x i16> @shl_const_v8i16(<8 x i16> %v) {
 ; CHECK-LABEL: shl_vec_v8i16:
 ; NO-SIMD128-NOT: i16x8
 ; SIMD128-NEXT: .functype shl_vec_v8i16 (v128, v128) -> (v128){{$}}
-; SIMD128-NEXT: i16x8.extract_lane_u $push[[L0:[0-9]+]]=, $0, 0{{$}}
-; SIMD128-NEXT: i16x8.extract_lane_u $push[[L1:[0-9]+]]=, $1, 0{{$}}
+; SIMD128-NEXT: i16x8.extract_lane_s $push[[L0:[0-9]+]]=, $0, 0{{$}}
+; SIMD128-NEXT: i32.const $push[[M0:[0-9]+]]=, 15{{$}}
+; SIMD128-NEXT: i16x8.splat $push[[M1:[0-9]+]]=, $pop[[M0]]{{$}}
+; SIMD128-NEXT: v128.and $push[[M2:[0-9]+]]=, $1, $pop[[M1]]{{$}}
+; SIMD128-NEXT: local.tee $push[[M:[0-9]+]]=, $1=, $pop[[M2]]{{$}}
+; SIMD128-NEXT: i16x8.extract_lane_u $push[[L1:[0-9]+]]=, $pop[[M]], 0{{$}}
 ; SIMD128-NEXT: i32.shl $push[[L2:[0-9]+]]=, $pop[[L0]], $pop[[L1]]{{$}}
 ; SIMD128-NEXT: i16x8.splat $push[[L3:[0-9]+]]=, $pop[[L2]]{{$}}
 ; Skip 6 lanes
-; SIMD128:      i16x8.extract_lane_u $push[[L4:[0-9]+]]=, $0, 7{{$}}
+; SIMD128:      i16x8.extract_lane_s $push[[L4:[0-9]+]]=, $0, 7{{$}}
 ; SIMD128-NEXT: i16x8.extract_lane_u $push[[L5:[0-9]+]]=, $1, 7{{$}}
 ; SIMD128-NEXT: i32.shl $push[[L6:[0-9]+]]=, $pop[[L4]], $pop[[L5]]{{$}}
 ; SIMD128-NEXT: i16x8.replace_lane $push[[R:[0-9]+]]=, $pop[[L7:[0-9]+]], 7, $pop[[L6]]{{$}}
@@ -342,23 +350,19 @@ define <8 x i16> @shr_s_v8i16(<8 x i16> %v, i16 %x) {
 ; CHECK-LABEL: shr_s_vec_v8i16:
 ; NO-SIMD128-NOT: i16x8
 ; SIMD128-NEXT: .functype shr_s_vec_v8i16 (v128, v128) -> (v128){{$}}
-; SIMD128-NEXT: i16x8.extract_lane_u $push[[L0:[0-9]+]]=, $0, 0{{$}}
-; SIMD128-NEXT: i32.const $push[[L1:[0-9]+]]=, 16{{$}}
-; SIMD128-NEXT: i32.shl $push[[L2:[0-9]+]]=, $pop[[L0]], $pop[[L1]]{{$}}
-; SIMD128-NEXT: i32.const $push[[L3:[0-9]+]]=, 16{{$}}
-; SIMD128-NEXT: i32.shr_s $push[[L4:[0-9]+]]=, $pop[[L2]], $pop[[L3]]{{$}}
-; SIMD128-NEXT: i16x8.extract_lane_u $push[[L5:[0-9]+]]=, $1, 0{{$}}
-; SIMD128-NEXT: i32.shr_s $push[[L6:[0-9]+]]=, $pop[[L4]], $pop[[L5]]{{$}}
-; SIMD128-NEXT: i16x8.splat $push[[L7:[0-9]+]]=, $pop[[L6]]{{$}}
+; SIMD128-NEXT: i16x8.extract_lane_s $push[[L0:[0-9]+]]=, $0, 0{{$}}
+; SIMD128-NEXT: i32.const $push[[M0:[0-9]+]]=, 15{{$}}
+; SIMD128-NEXT: i16x8.splat $push[[M1:[0-9]+]]=, $pop[[M0]]{{$}}
+; SIMD128-NEXT: v128.and $push[[M2:[0-9]+]]=, $1, $pop[[M1]]{{$}}
+; SIMD128-NEXT: local.tee $push[[M:[0-9]+]]=, $1=, $pop[[M2]]{{$}}
+; SIMD128-NEXT: i16x8.extract_lane_u $push[[L1:[0-9]+]]=, $pop[[M]], 0{{$}}
+; SIMD128-NEXT: i32.shr_s $push[[L2:[0-9]+]]=, $pop[[L0]], $pop[[L1]]{{$}}
+; SIMD128-NEXT: i16x8.splat $push[[L3:[0-9]+]]=, $pop[[L2]]{{$}}
 ; Skip 6 lanes
-; SIMD128:      i16x8.extract_lane_u $push[[L7:[0-9]+]]=, $0, 7{{$}}
-; SIMD128-NEXT: i32.const $push[[L8:[0-9]+]]=, 16{{$}}
-; SIMD128-NEXT: i32.shl $push[[L9:[0-9]+]]=, $pop[[L7]], $pop[[L8]]{{$}}
-; SIMD128-NEXT: i32.const $push[[L10:[0-9]+]]=, 16{{$}}
-; SIMD128-NEXT: i32.shr_s $push[[L11:[0-9]+]]=, $pop[[L9]], $pop[[L10]]{{$}}
-; SIMD128-NEXT: i16x8.extract_lane_u $push[[L12:[0-9]+]]=, $1, 7{{$}}
-; SIMD128-NEXT: i32.shr_s $push[[L13:[0-9]+]]=, $pop[[L11]], $pop[[L12]]{{$}}
-; SIMD128-NEXT: i16x8.replace_lane $push[[R:[0-9]+]]=, $pop[[L14:[0-9]+]], 7, $pop[[L13]]{{$}}
+; SIMD128:      i16x8.extract_lane_s $push[[L0:[0-9]+]]=, $0, 7{{$}}
+; SIMD128-NEXT: i16x8.extract_lane_u $push[[L1:[0-9]+]]=, $1, 7{{$}}
+; SIMD128-NEXT: i32.shr_s $push[[L2:[0-9]+]]=, $pop[[L0]], $pop[[L1]]{{$}}
+; SIMD128-NEXT: i16x8.replace_lane $push[[R:[0-9]+]]=, $pop{{[0-9]+}}, 7, $pop[[L2]]{{$}}
 ; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <8 x i16> @shr_s_vec_v8i16(<8 x i16> %v, <8 x i16> %x) {
   %a = ashr <8 x i16> %v, %x
@@ -382,7 +386,11 @@ define <8 x i16> @shr_u_v8i16(<8 x i16> %v, i16 %x) {
 ; NO-SIMD128-NOT: i16x8
 ; SIMD128-NEXT: .functype shr_u_vec_v8i16 (v128, v128) -> (v128){{$}}
 ; SIMD128-NEXT: i16x8.extract_lane_u $push[[L0:[0-9]+]]=, $0, 0{{$}}
-; SIMD128-NEXT: i16x8.extract_lane_u $push[[L1:[0-9]+]]=, $1, 0{{$}}
+; SIMD128-NEXT: i32.const $push[[M0:[0-9]+]]=, 15{{$}}
+; SIMD128-NEXT: i16x8.splat $push[[M1:[0-9]+]]=, $pop[[M0]]{{$}}
+; SIMD128-NEXT: v128.and $push[[M2:[0-9]+]]=, $1, $pop[[M1]]{{$}}
+; SIMD128-NEXT: local.tee $push[[M:[0-9]+]]=, $1=, $pop[[M2]]{{$}}
+; SIMD128-NEXT: i16x8.extract_lane_u $push[[L1:[0-9]+]]=, $pop[[M]], 0{{$}}
 ; SIMD128-NEXT: i32.shr_u $push[[L2:[0-9]+]]=, $pop[[L0]], $pop[[L1]]{{$}}
 ; SIMD128-NEXT: i16x8.splat $push[[L3:[0-9]+]]=, $pop[[L2]]{{$}}
 ; Skip 6 lanes
@@ -725,7 +733,7 @@ define <2 x i64> @shl_v2i64(<2 x i64> %v, i32 %x) {
 ; CHECK-LABEL: shl_nozext_v2i64:
 ; NO-SIMD128-NOT: i64x2
 ; SIMD128-NEXT: .functype shl_nozext_v2i64 (v128, i64) -> (v128){{$}}
-; SIMD128-NEXT: i32.wrap/i64 $push[[L0:[0-9]+]]=, $1{{$}}
+; SIMD128-NEXT: i32.wrap_i64 $push[[L0:[0-9]+]]=, $1{{$}}
 ; SIMD128-NEXT: i64x2.shl $push[[R:[0-9]+]]=, $0, $pop[[L0]]{{$}}
 ; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <2 x i64> @shl_nozext_v2i64(<2 x i64> %v, i64 %x) {
@@ -779,7 +787,7 @@ define <2 x i64> @shr_s_v2i64(<2 x i64> %v, i32 %x) {
 ; CHECK-LABEL: shr_s_nozext_v2i64:
 ; NO-SIMD128-NOT: i64x2
 ; SIMD128-NEXT: .functype shr_s_nozext_v2i64 (v128, i64) -> (v128){{$}}
-; SIMD128-NEXT: i32.wrap/i64 $push[[L0:[0-9]+]]=, $1{{$}}
+; SIMD128-NEXT: i32.wrap_i64 $push[[L0:[0-9]+]]=, $1{{$}}
 ; SIMD128-NEXT: i64x2.shr_s $push[[R:[0-9]+]]=, $0, $pop[[L0]]{{$}}
 ; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <2 x i64> @shr_s_nozext_v2i64(<2 x i64> %v, i64 %x) {
@@ -833,7 +841,7 @@ define <2 x i64> @shr_u_v2i64(<2 x i64> %v, i32 %x) {
 ; CHECK-LABEL: shr_u_nozext_v2i64:
 ; NO-SIMD128-NOT: i64x2
 ; SIMD128-NEXT: .functype shr_u_nozext_v2i64 (v128, i64) -> (v128){{$}}
-; SIMD128-NEXT: i32.wrap/i64 $push[[L0:[0-9]+]]=, $1{{$}}
+; SIMD128-NEXT: i32.wrap_i64 $push[[L0:[0-9]+]]=, $1{{$}}
 ; SIMD128-NEXT: i64x2.shr_u $push[[R:[0-9]+]]=, $0, $pop[[L0]]{{$}}
 ; SIMD128-NEXT: return $pop[[R]]{{$}}
 define <2 x i64> @shr_u_nozext_v2i64(<2 x i64> %v, i64 %x) {
@@ -1087,6 +1095,7 @@ define <4 x float> @sub_v4f32(<4 x float> %x, <4 x float> %y) {
 
 ; CHECK-LABEL: div_v4f32:
 ; NO-SIMD128-NOT: f32x4
+; SIMD128-VM-NOT: f32x4.div
 ; SIMD128-NEXT: .functype div_v4f32 (v128, v128) -> (v128){{$}}
 ; SIMD128-NEXT: f32x4.div $push[[R:[0-9]+]]=, $0, $1{{$}}
 ; SIMD128-NEXT: return $pop[[R]]{{$}}
@@ -1107,6 +1116,7 @@ define <4 x float> @mul_v4f32(<4 x float> %x, <4 x float> %y) {
 
 ; CHECK-LABEL: sqrt_v4f32:
 ; NO-SIMD128-NOT: f32x4
+; SIMD128-VM-NOT: f32x4.sqrt
 ; SIMD128-NEXT: .functype sqrt_v4f32 (v128) -> (v128){{$}}
 ; SIMD128-NEXT: f32x4.sqrt $push[[R:[0-9]+]]=, $0{{$}}
 ; SIMD128-NEXT: return $pop[[R]]{{$}}
diff --git a/test/CodeGen/WebAssembly/simd-bitcasts.ll b/test/CodeGen/WebAssembly/simd-bitcasts.ll
index b36f5190bccacb3857aa84b391e8006b72c3e4c3..5aab1c68ce281aff8db6f2113092431d58a60b16 100644
--- a/test/CodeGen/WebAssembly/simd-bitcasts.ll
+++ b/test/CodeGen/WebAssembly/simd-bitcasts.ll
@@ -1,6 +1,6 @@
-; RUN: llc < %s -asm-verbose=false -disable-wasm-fallthrough-return-opt -wasm-keep-registers -wasm-disable-explicit-locals -wasm-enable-unimplemented-simd -mattr=+simd128,+sign-ext | FileCheck %s --check-prefixes CHECK,SIMD128
-; RUN: llc < %s -asm-verbose=false -disable-wasm-fallthrough-return-opt -wasm-keep-registers -wasm-disable-explicit-locals -mattr=+simd128,+sign-ext | FileCheck %s --check-prefixes CHECK,SIMD128-VM
-; RUN: llc < %s -asm-verbose=false -disable-wasm-fallthrough-return-opt -wasm-keep-registers -wasm-disable-explicit-locals -mattr=-simd128,+sign-ext | FileCheck %s --check-prefixes CHECK,NO-SIMD128
+; RUN: llc < %s -asm-verbose=false -verify-machineinstrs -disable-wasm-fallthrough-return-opt -wasm-keep-registers -wasm-disable-explicit-locals -mattr=+unimplemented-simd128 | FileCheck %s --check-prefixes CHECK,SIMD128
+; RUN: llc < %s -asm-verbose=false -verify-machineinstrs -disable-wasm-fallthrough-return-opt -wasm-keep-registers -wasm-disable-explicit-locals -mattr=+simd128 | FileCheck %s --check-prefixes CHECK,SIMD128-VM
+; RUN: llc < %s -asm-verbose=false -verify-machineinstrs -disable-wasm-fallthrough-return-opt -wasm-keep-registers -wasm-disable-explicit-locals | FileCheck %s --check-prefixes CHECK,NO-SIMD128
 
 ; Test that bitcasts between vector types are lowered to zero instructions
 
diff --git a/test/CodeGen/WebAssembly/simd-build-pair.ll b/test/CodeGen/WebAssembly/simd-build-pair.ll
new file mode 100644
index 0000000000000000000000000000000000000000..a3b7ab8e8eca4de9e067ae854d3ce90b7bd6eacc
--- /dev/null
+++ b/test/CodeGen/WebAssembly/simd-build-pair.ll
@@ -0,0 +1,33 @@
+; RUN: llc < %s -asm-verbose=false -verify-machineinstrs -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-keep-registers -mattr=simd128 | FileCheck %s --check-prefixes CHECK
+
+target datalayout = "e-m:e-p:32:32-i64:64-n32:64-S128"
+target triple = "wasm32-unknown-unknown"
+
+; Test that BUILD_PAIR dag nodes are correctly lowered.
+; This code produces a selection DAG like the following:
+
+;    t0: ch = EntryToken
+;  t3: v4i32,ch = load<(load 16 from `<4 x i32>* undef`)> t0, undef:i32, undef:i32
+;        t30: i32 = extract_vector_elt t3, Constant:i32<2>
+;        t28: i32 = extract_vector_elt t3, Constant:i32<3>
+;      t24: i64 = build_pair t30, t28
+;    t8: ch = store<(store 8 into `i64* undef`, align 1)> t3:1, t24, undef:i32, undef:i32
+;  t9: ch = WebAssemblyISD::RETURN t8
+
+; CHECK:      i32x4.extract_lane
+; CHECK-NEXT: i64.extend_i32_u
+; CHECK-NEXT: i32x4.extract_lane
+; CHECK-NEXT: i64.extend_i32_u
+; CHECK-NEXT: i64.const {{.*}} 32
+; CHECK-NEXT: i64.shl
+; CHECK-NEXT: i64.or
+; CHECK-NEXT: i64.store
+define void @build_pair_i32s() {
+entry:
+  %0 = load <4 x i32>, <4 x i32>* undef, align 16
+  %shuffle.i184 = shufflevector <4 x i32> %0, <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+  %bc357 = bitcast <4 x i32> %shuffle.i184 to <2 x i64>
+  %1 = extractelement <2 x i64> %bc357, i32 0
+  store i64 %1, i64* undef, align 1
+  ret void
+}
diff --git a/test/CodeGen/WebAssembly/simd-build-vector.ll b/test/CodeGen/WebAssembly/simd-build-vector.ll
new file mode 100644
index 0000000000000000000000000000000000000000..41a320c92f702b8eda440b1e6b13b27f0dd0e1c4
--- /dev/null
+++ b/test/CodeGen/WebAssembly/simd-build-vector.ll
@@ -0,0 +1,127 @@
+; RUN: llc < %s -asm-verbose=false -verify-machineinstrs -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-keep-registers -mattr=+unimplemented-simd128 | FileCheck %s
+
+; Test that the logic to choose between v128.const vector
+; initialization and splat vector initialization and to optimize the
+; choice of splat value works correctly.
+
+target datalayout = "e-m:e-p:32:32-i64:64-n32:64-S128"
+target triple = "wasm32-unknown-unknown"
+
+; CHECK-LABEL: same_const_one_replaced_i8x16:
+; CHECK-NEXT:  .functype       same_const_one_replaced_i8x16 (i32) -> (v128)
+; CHECK-NEXT:  i32.const       $push[[L0:[0-9]+]]=, 42
+; CHECK-NEXT:  i16x8.splat     $push[[L1:[0-9]+]]=, $pop[[L0]]
+; CHECK-NEXT:  i16x8.replace_lane      $push[[L2:[0-9]+]]=, $pop[[L1]], 5, $0
+; CHECK-NEXT:  return          $pop[[L2]]
+define <8 x i16> @same_const_one_replaced_i8x16(i16 %x) {
+  %v = insertelement
+    <8 x i16> <i16 42, i16 42, i16 42, i16 42, i16 42, i16 42, i16 42, i16 42>,
+    i16 %x,
+    i32 5
+  ret <8 x i16> %v
+}
+
+; CHECK-LABEL: different_const_one_replaced_i8x16:
+; CHECK-NEXT:  .functype       different_const_one_replaced_i8x16 (i32) -> (v128)
+; CHECK-NEXT:  v128.const      $push[[L0:[0-9]+]]=, 1, -2, 3, -4, 5, 0, 7, -8
+; CHECK-NEXT:  i16x8.replace_lane      $push[[L1:[0-9]+]]=, $pop[[L0]], 5, $0
+; CHECK-NEXT:  return          $pop[[L1]]
+define <8 x i16> @different_const_one_replaced_i8x16(i16 %x) {
+  %v = insertelement
+    <8 x i16> <i16 1, i16 -2, i16 3, i16 -4, i16 5, i16 -6, i16 7, i16 -8>,
+    i16 %x,
+    i32 5
+  ret <8 x i16> %v
+}
+
+; CHECK-LABEL: same_const_one_replaced_f32x4:
+; CHECK-NEXT:  .functype       same_const_one_replaced_f32x4 (f32) -> (v128)
+; CHECK-NEXT:  f32.const       $push[[L0:[0-9]+]]=, 0x1.5p5
+; CHECK-NEXT:  f32x4.splat     $push[[L1:[0-9]+]]=, $pop[[L0]]
+; CHECK-NEXT:  f32x4.replace_lane      $push[[L2:[0-9]+]]=, $pop[[L1]], 2, $0
+; CHECK-NEXT:  return          $pop[[L2]]
+define <4 x float> @same_const_one_replaced_f32x4(float %x) {
+  %v = insertelement
+    <4 x float> <float 42., float 42., float 42., float 42.>,
+    float %x,
+    i32 2
+  ret <4 x float> %v
+}
+
+; CHECK-LABEL: different_const_one_replaced_f32x4:
+; CHECK-NEXT:  .functype       different_const_one_replaced_f32x4 (f32) -> (v128)
+; CHECK-NEXT:  v128.const      $push[[L0:[0-9]+]]=, 0x1p0, 0x1p1, 0x0p0, 0x1p2
+; CHECK-NEXT:  f32x4.replace_lane      $push[[L1:[0-9]+]]=, $pop[[L0]], 2, $0
+; CHECK-NEXT:  return          $pop[[L1]]
+define <4 x float> @different_const_one_replaced_f32x4(float %x) {
+  %v = insertelement
+    <4 x float> <float 1., float 2., float 3., float 4.>,
+    float %x,
+    i32 2
+  ret <4 x float> %v
+}
+
+; CHECK-LABEL: splat_common_const_i32x4:
+; CHECK-NEXT:  .functype       splat_common_const_i32x4 () -> (v128)
+; CHECK-NEXT:  i32.const       $push[[L0:[0-9]+]]=, 3
+; CHECK-NEXT:  i32x4.splat     $push[[L1:[0-9]+]]=, $pop[[L0]]
+; CHECK-NEXT:  i32.const       $push[[L2:[0-9]+]]=, 1
+; CHECK-NEXT:  i32x4.replace_lane      $push[[L3:[0-9]+]]=, $pop[[L1]], 3, $pop[[L2]]
+; CHECK-NEXT:  return          $pop[[L3]]
+define <4 x i32> @splat_common_const_i32x4() {
+  ret <4 x i32> <i32 undef, i32 3, i32 3, i32 1>
+}
+
+; CHECK-LABEL: splat_common_arg_i16x8:
+; CHECK-NEXT:  .functype       splat_common_arg_i16x8 (i32, i32, i32) -> (v128)
+; CHECK-NEXT:  i16x8.splat     $push[[L0:[0-9]+]]=, $2
+; CHECK-NEXT:  i16x8.replace_lane      $push[[L1:[0-9]+]]=, $pop[[L0]], 0, $1
+; CHECK-NEXT:  i16x8.replace_lane      $push[[L2:[0-9]+]]=, $pop[[L1]], 2, $0
+; CHECK-NEXT:  i16x8.replace_lane      $push[[L3:[0-9]+]]=, $pop[[L2]], 4, $1
+; CHECK-NEXT:  i16x8.replace_lane      $push[[L4:[0-9]+]]=, $pop[[L3]], 7, $1
+; CHECK-NEXT:  return          $pop[[L4]]
+define <8 x i16> @splat_common_arg_i16x8(i16 %a, i16 %b, i16 %c) {
+  %v0 = insertelement <8 x i16> undef, i16 %b, i32 0
+  %v1 = insertelement <8 x i16> %v0, i16 %c, i32 1
+  %v2 = insertelement <8 x i16> %v1, i16 %a, i32 2
+  %v3 = insertelement <8 x i16> %v2, i16 %c, i32 3
+  %v4 = insertelement <8 x i16> %v3, i16 %b, i32 4
+  %v5 = insertelement <8 x i16> %v4, i16 %c, i32 5
+  %v6 = insertelement <8 x i16> %v5, i16 %c, i32 6
+  %v7 = insertelement <8 x i16> %v6, i16 %b, i32 7
+  ret <8 x i16> %v7
+}
+
+; CHECK-LABEL: undef_const_insert_f32x4:
+; CHECK-NEXT:  .functype       undef_const_insert_f32x4 () -> (v128)
+; CHECK-NEXT:  f32.const       $push[[L0:[0-9]+]]=, 0x1.5p5
+; CHECK-NEXT:  f32x4.splat     $push[[L1:[0-9]+]]=, $pop[[L0]]
+; CHECK-NEXT:  return          $pop[[L1]]
+define <4 x float> @undef_const_insert_f32x4() {
+  %v = insertelement <4 x float> undef, float 42., i32 1
+  ret <4 x float> %v
+}
+
+; CHECK-LABEL: undef_arg_insert_i32x4:
+; CHECK-NEXT:  .functype       undef_arg_insert_i32x4 (i32) -> (v128)
+; CHECK-NEXT:  i32x4.splat     $push[[L0:[0-9]+]]=, $0
+; CHECK-NEXT:  return          $pop[[L0]]
+define <4 x i32> @undef_arg_insert_i32x4(i32 %x) {
+  %v = insertelement <4 x i32> undef, i32 %x, i32 3
+  ret <4 x i32> %v
+}
+
+; CHECK-LABEL: all_undef_i8x16:
+; CHECK-NEXT:  .functype       all_undef_i8x16 () -> (v128)
+; CHECK-NEXT:  return          $0
+define <16 x i8> @all_undef_i8x16() {
+  %v = insertelement <16 x i8> undef, i8 undef, i32 4
+  ret <16 x i8> %v
+}
+
+; CHECK-LABEL: all_undef_f64x2:
+; CHECK-NEXT:  .functype       all_undef_f64x2 () -> (v128)
+; CHECK-NEXT:  return          $0
+define <2 x double> @all_undef_f64x2() {
+  ret <2 x double> undef
+}
diff --git a/test/CodeGen/WebAssembly/simd-comparisons.ll b/test/CodeGen/WebAssembly/simd-comparisons.ll
index 5e4c51ab6aa9b4984ead2c3346deade1799537db..929385f92896c287b15a4299c70f30fe448623c2 100644
--- a/test/CodeGen/WebAssembly/simd-comparisons.ll
+++ b/test/CodeGen/WebAssembly/simd-comparisons.ll
@@ -1,6 +1,6 @@
-; RUN: llc < %s -asm-verbose=false -disable-wasm-fallthrough-return-opt -wasm-keep-registers -wasm-disable-explicit-locals -wasm-enable-unimplemented-simd -mattr=+simd128,+sign-ext | FileCheck %s --check-prefixes CHECK,SIMD128
-; RUN: llc < %s -asm-verbose=false -disable-wasm-fallthrough-return-opt -wasm-keep-registers -wasm-disable-explicit-locals -mattr=+simd128,+sign-ext | FileCheck %s --check-prefixes CHECK,SIMD128-VM
-; RUN: llc < %s -asm-verbose=false -disable-wasm-fallthrough-return-opt -wasm-keep-registers -wasm-disable-explicit-locals -mattr=-simd128,+sign-ext | FileCheck %s --check-prefixes CHECK,NO-SIMD128
+; RUN: llc < %s -asm-verbose=false -verify-machineinstrs -disable-wasm-fallthrough-return-opt -wasm-keep-registers -wasm-disable-explicit-locals -mattr=+unimplemented-simd128 | FileCheck %s --check-prefixes CHECK,SIMD128
+; RUN: llc < %s -asm-verbose=false -verify-machineinstrs -disable-wasm-fallthrough-return-opt -wasm-keep-registers -wasm-disable-explicit-locals -mattr=+simd128 | FileCheck %s --check-prefixes CHECK,SIMD128-VM
+; RUN: llc < %s -asm-verbose=false -verify-machineinstrs -disable-wasm-fallthrough-return-opt -wasm-keep-registers -wasm-disable-explicit-locals | FileCheck %s --check-prefixes CHECK,NO-SIMD128
 
 ; Test SIMD comparison operators
 
diff --git a/test/CodeGen/WebAssembly/simd-conversions.ll b/test/CodeGen/WebAssembly/simd-conversions.ll
index 9c3f80fa16ad3651c7a7dc5b677f132786680c5e..5437a9ab0a81dca83e9ca074b052a9ba0db99875 100644
--- a/test/CodeGen/WebAssembly/simd-conversions.ll
+++ b/test/CodeGen/WebAssembly/simd-conversions.ll
@@ -1,6 +1,6 @@
-; RUN: llc < %s -asm-verbose=false -wasm-keep-registers -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-enable-unimplemented-simd -mattr=+simd128,+sign-ext | FileCheck %s --check-prefixes CHECK,SIMD128
-; RUN: llc < %s -asm-verbose=false -wasm-keep-registers -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -mattr=+simd128,+sign-ext | FileCheck %s --check-prefixes CHECK,SIMD128-VM
-; RUN: llc < %s -asm-verbose=false -wasm-keep-registers -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -mattr=-simd128,+sign-ext | FileCheck %s --check-prefixes CHECK,NO-SIMD128
+; RUN: llc < %s -asm-verbose=false -verify-machineinstrs -wasm-keep-registers -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -mattr=+unimplemented-simd128 | FileCheck %s --check-prefixes CHECK,SIMD128
+; RUN: llc < %s -asm-verbose=false -verify-machineinstrs -wasm-keep-registers -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -mattr=+simd128 | FileCheck %s --check-prefixes CHECK,SIMD128-VM
+; RUN: llc < %s -asm-verbose=false -verify-machineinstrs -wasm-keep-registers -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals | FileCheck %s --check-prefixes CHECK,NO-SIMD128
 
 ; Test that vector float-to-int and int-to-float instructions lower correctly
 
@@ -10,7 +10,7 @@ target triple = "wasm32-unknown-unknown"
 ; CHECK-LABEL: convert_s_v4f32:
 ; NO-SIMD128-NOT: i32x4
 ; SIMD128-NEXT: .functype convert_s_v4f32 (v128) -> (v128){{$}}
-; SIMD128-NEXT: f32x4.convert_s/i32x4 $push[[R:[0-9]+]]=, $0
+; SIMD128-NEXT: f32x4.convert_i32x4_s $push[[R:[0-9]+]]=, $0
 ; SIMD128-NEXT: return $pop[[R]]
 define <4 x float> @convert_s_v4f32(<4 x i32> %x) {
   %a = sitofp <4 x i32> %x to <4 x float>
@@ -20,7 +20,7 @@ define <4 x float> @convert_s_v4f32(<4 x i32> %x) {
 ; CHECK-LABEL: convert_u_v4f32:
 ; NO-SIMD128-NOT: i32x4
 ; SIMD128-NEXT: .functype convert_u_v4f32 (v128) -> (v128){{$}}
-; SIMD128-NEXT: f32x4.convert_u/i32x4 $push[[R:[0-9]+]]=, $0
+; SIMD128-NEXT: f32x4.convert_i32x4_u $push[[R:[0-9]+]]=, $0
 ; SIMD128-NEXT: return $pop[[R]]
 define <4 x float> @convert_u_v4f32(<4 x i32> %x) {
   %a = uitofp <4 x i32> %x to <4 x float>
@@ -29,9 +29,9 @@ define <4 x float> @convert_u_v4f32(<4 x i32> %x) {
 
 ; CHECK-LABEL: convert_s_v2f64:
 ; NO-SIMD128-NOT: i64x2
-; SIMD128-VM-NOT: f64x2.convert_s/i64x2
+; SIMD128-VM-NOT: f64x2.convert_i64x2_s
 ; SIMD128-NEXT: .functype convert_s_v2f64 (v128) -> (v128){{$}}
-; SIMD128-NEXT: f64x2.convert_s/i64x2 $push[[R:[0-9]+]]=, $0
+; SIMD128-NEXT: f64x2.convert_i64x2_s $push[[R:[0-9]+]]=, $0
 ; SIMD128-NEXT: return $pop[[R]]
 define <2 x double> @convert_s_v2f64(<2 x i64> %x) {
   %a = sitofp <2 x i64> %x to <2 x double>
@@ -40,9 +40,9 @@ define <2 x double> @convert_s_v2f64(<2 x i64> %x) {
 
 ; CHECK-LABEL: convert_u_v2f64:
 ; NO-SIMD128-NOT: i64x2
-; SIMD128-VM-NOT: f64x2.convert_u/i64x2
+; SIMD128-VM-NOT: f64x2.convert_i64x2_u
 ; SIMD128-NEXT: .functype convert_u_v2f64 (v128) -> (v128){{$}}
-; SIMD128-NEXT: f64x2.convert_u/i64x2 $push[[R:[0-9]+]]=, $0
+; SIMD128-NEXT: f64x2.convert_i64x2_u $push[[R:[0-9]+]]=, $0
 ; SIMD128-NEXT: return $pop[[R]]
 define <2 x double> @convert_u_v2f64(<2 x i64> %x) {
   %a = uitofp <2 x i64> %x to <2 x double>
@@ -52,7 +52,7 @@ define <2 x double> @convert_u_v2f64(<2 x i64> %x) {
 ; CHECK-LABEL: trunc_sat_s_v4i32:
 ; NO-SIMD128-NOT: f32x4
 ; SIMD128-NEXT: .functype trunc_sat_s_v4i32 (v128) -> (v128){{$}}
-; SIMD128-NEXT: i32x4.trunc_sat_s/f32x4 $push[[R:[0-9]+]]=, $0
+; SIMD128-NEXT: i32x4.trunc_sat_f32x4_s $push[[R:[0-9]+]]=, $0
 ; SIMD128-NEXT: return $pop[[R]]
 define <4 x i32> @trunc_sat_s_v4i32(<4 x float> %x) {
   %a = fptosi <4 x float> %x to <4 x i32>
@@ -62,7 +62,7 @@ define <4 x i32> @trunc_sat_s_v4i32(<4 x float> %x) {
 ; CHECK-LABEL: trunc_sat_u_v4i32:
 ; NO-SIMD128-NOT: f32x4
 ; SIMD128-NEXT: .functype trunc_sat_u_v4i32 (v128) -> (v128){{$}}
-; SIMD128-NEXT: i32x4.trunc_sat_u/f32x4 $push[[R:[0-9]+]]=, $0
+; SIMD128-NEXT: i32x4.trunc_sat_f32x4_u $push[[R:[0-9]+]]=, $0
 ; SIMD128-NEXT: return $pop[[R]]
 define <4 x i32> @trunc_sat_u_v4i32(<4 x float> %x) {
   %a = fptoui <4 x float> %x to <4 x i32>
@@ -71,9 +71,9 @@ define <4 x i32> @trunc_sat_u_v4i32(<4 x float> %x) {
 
 ; CHECK-LABEL: trunc_sat_s_v2i64:
 ; NO-SIMD128-NOT: f64x2
-; SIMD128-VM-NOT: i64x2.trunc_sat_s/f64x2
+; SIMD128-VM-NOT: i64x2.trunc_sat_f64x2_s
 ; SIMD128-NEXT: .functype trunc_sat_s_v2i64 (v128) -> (v128){{$}}
-; SIMD128-NEXT: i64x2.trunc_sat_s/f64x2 $push[[R:[0-9]+]]=, $0
+; SIMD128-NEXT: i64x2.trunc_sat_f64x2_s $push[[R:[0-9]+]]=, $0
 ; SIMD128-NEXT: return $pop[[R]]
 define <2 x i64> @trunc_sat_s_v2i64(<2 x double> %x) {
   %a = fptosi <2 x double> %x to <2 x i64>
@@ -82,9 +82,9 @@ define <2 x i64> @trunc_sat_s_v2i64(<2 x double> %x) {
 
 ; CHECK-LABEL: trunc_sat_u_v2i64:
 ; NO-SIMD128-NOT: f64x2
-; SIMD128-VM-NOT: i64x2.trunc_sat_u/f64x2
+; SIMD128-VM-NOT: i64x2.trunc_sat_f64x2_u
 ; SIMD128-NEXT: .functype trunc_sat_u_v2i64 (v128) -> (v128){{$}}
-; SIMD128-NEXT: i64x2.trunc_sat_u/f64x2 $push[[R:[0-9]+]]=, $0
+; SIMD128-NEXT: i64x2.trunc_sat_f64x2_u $push[[R:[0-9]+]]=, $0
 ; SIMD128-NEXT: return $pop[[R]]
 define <2 x i64> @trunc_sat_u_v2i64(<2 x double> %x) {
   %a = fptoui <2 x double> %x to <2 x i64>
diff --git a/test/CodeGen/WebAssembly/simd-ext-load-trunc-store.ll b/test/CodeGen/WebAssembly/simd-ext-load-trunc-store.ll
index cabb96ab9cd604a8a2546284216962f2d5c902d1..39bcc1d0bde84e25adfff5e9973d8c9766b8431b 100644
--- a/test/CodeGen/WebAssembly/simd-ext-load-trunc-store.ll
+++ b/test/CodeGen/WebAssembly/simd-ext-load-trunc-store.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -asm-verbose=false -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-keep-registers -wasm-enable-unimplemented-simd -mattr=+simd128 | FileCheck %s
+; RUN: llc < %s -asm-verbose=false -verify-machineinstrs -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-keep-registers -mattr=+unimplemented-simd128 | FileCheck %s
 
 ; Check that store in memory with smaller lanes are loaded and stored
 ; as expected. This is a regression test for part of bug 39275.
diff --git a/test/CodeGen/WebAssembly/simd-intrinsics.ll b/test/CodeGen/WebAssembly/simd-intrinsics.ll
index 6245efd0a0b68a04fe5b1acdadaa7ebb41a6bb03..53c98d2722b383a622a38ccc7f4c4b0986ba9009 100644
--- a/test/CodeGen/WebAssembly/simd-intrinsics.ll
+++ b/test/CodeGen/WebAssembly/simd-intrinsics.ll
@@ -1,5 +1,5 @@
-; RUN: llc < %s -asm-verbose=false -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-keep-registers -wasm-enable-unimplemented-simd -mattr=+simd128 | FileCheck %s --check-prefixes CHECK,SIMD128
-; RUN: llc < %s -asm-verbose=false -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-keep-registers -wasm-enable-unimplemented-simd -mattr=+simd128 -fast-isel | FileCheck %s --check-prefixes CHECK,SIMD128
+; RUN: llc < %s -asm-verbose=false -verify-machineinstrs -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-keep-registers -mattr=+unimplemented-simd128 | FileCheck %s --check-prefixes CHECK,SIMD128
+; RUN: llc < %s -asm-verbose=false -verify-machineinstrs -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-keep-registers -mattr=+unimplemented-simd128 -fast-isel | FileCheck %s --check-prefixes CHECK,SIMD128
 
 ; Test that SIMD128 intrinsics lower as expected. These intrinsics are
 ; only expected to lower successfully if the simd128 attribute is
@@ -77,12 +77,12 @@ define i32 @all_v16i8(<16 x i8> %x) {
 
 ; CHECK-LABEL: bitselect_v16i8:
 ; SIMD128-NEXT: .functype bitselect_v16i8 (v128, v128, v128) -> (v128){{$}}
-; SIMD128-NEXT: v128.bitselect $push[[R:[0-9]+]]=, $1, $2, $0{{$}}
+; SIMD128-NEXT: v128.bitselect $push[[R:[0-9]+]]=, $0, $1, $2{{$}}
 ; SIMD128-NEXT: return $pop[[R]]{{$}}
 declare <16 x i8> @llvm.wasm.bitselect.v16i8(<16 x i8>, <16 x i8>, <16 x i8>)
-define <16 x i8> @bitselect_v16i8(<16 x i8> %c, <16 x i8> %v1, <16 x i8> %v2) {
+define <16 x i8> @bitselect_v16i8(<16 x i8> %v1, <16 x i8> %v2, <16 x i8> %c) {
   %a = call <16 x i8> @llvm.wasm.bitselect.v16i8(
-    <16 x i8> %c, <16 x i8> %v1, <16 x i8> %v2
+     <16 x i8> %v1, <16 x i8> %v2, <16 x i8> %c
   )
   ret <16 x i8> %a
 }
@@ -156,12 +156,12 @@ define i32 @all_v8i16(<8 x i16> %x) {
 
 ; CHECK-LABEL: bitselect_v8i16:
 ; SIMD128-NEXT: .functype bitselect_v8i16 (v128, v128, v128) -> (v128){{$}}
-; SIMD128-NEXT: v128.bitselect $push[[R:[0-9]+]]=, $1, $2, $0{{$}}
+; SIMD128-NEXT: v128.bitselect $push[[R:[0-9]+]]=, $0, $1, $2{{$}}
 ; SIMD128-NEXT: return $pop[[R]]{{$}}
 declare <8 x i16> @llvm.wasm.bitselect.v8i16(<8 x i16>, <8 x i16>, <8 x i16>)
-define <8 x i16> @bitselect_v8i16(<8 x i16> %c, <8 x i16> %v1, <8 x i16> %v2) {
+define <8 x i16> @bitselect_v8i16(<8 x i16> %v1, <8 x i16> %v2, <8 x i16> %c) {
   %a = call <8 x i16> @llvm.wasm.bitselect.v8i16(
-    <8 x i16> %c, <8 x i16> %v1, <8 x i16> %v2
+    <8 x i16> %v1, <8 x i16> %v2, <8 x i16> %c
   )
   ret <8 x i16> %a
 }
@@ -191,12 +191,12 @@ define i32 @all_v4i32(<4 x i32> %x) {
 
 ; CHECK-LABEL: bitselect_v4i32:
 ; SIMD128-NEXT: .functype bitselect_v4i32 (v128, v128, v128) -> (v128){{$}}
-; SIMD128-NEXT: v128.bitselect $push[[R:[0-9]+]]=, $1, $2, $0{{$}}
+; SIMD128-NEXT: v128.bitselect $push[[R:[0-9]+]]=, $0, $1, $2{{$}}
 ; SIMD128-NEXT: return $pop[[R]]{{$}}
 declare <4 x i32> @llvm.wasm.bitselect.v4i32(<4 x i32>, <4 x i32>, <4 x i32>)
-define <4 x i32> @bitselect_v4i32(<4 x i32> %c, <4 x i32> %v1, <4 x i32> %v2) {
+define <4 x i32> @bitselect_v4i32(<4 x i32> %v1, <4 x i32> %v2, <4 x i32> %c) {
   %a = call <4 x i32> @llvm.wasm.bitselect.v4i32(
-    <4 x i32> %c, <4 x i32> %v1, <4 x i32> %v2
+    <4 x i32> %v1, <4 x i32> %v2, <4 x i32> %c
   )
   ret <4 x i32> %a
 }
@@ -204,7 +204,7 @@ define <4 x i32> @bitselect_v4i32(<4 x i32> %c, <4 x i32> %v1, <4 x i32> %v2) {
 ; CHECK-LABEL: trunc_sat_s_v4i32:
 ; NO-SIMD128-NOT: f32x4
 ; SIMD128-NEXT: .functype trunc_sat_s_v4i32 (v128) -> (v128){{$}}
-; SIMD128-NEXT: i32x4.trunc_sat_s/f32x4 $push[[R:[0-9]+]]=, $0
+; SIMD128-NEXT: i32x4.trunc_sat_f32x4_s $push[[R:[0-9]+]]=, $0
 ; SIMD128-NEXT: return $pop[[R]]
 declare <4 x i32> @llvm.wasm.trunc.saturate.signed.v4i32.v4f32(<4 x float>)
 define <4 x i32> @trunc_sat_s_v4i32(<4 x float> %x) {
@@ -215,7 +215,7 @@ define <4 x i32> @trunc_sat_s_v4i32(<4 x float> %x) {
 ; CHECK-LABEL: trunc_sat_u_v4i32:
 ; NO-SIMD128-NOT: f32x4
 ; SIMD128-NEXT: .functype trunc_sat_u_v4i32 (v128) -> (v128){{$}}
-; SIMD128-NEXT: i32x4.trunc_sat_u/f32x4 $push[[R:[0-9]+]]=, $0
+; SIMD128-NEXT: i32x4.trunc_sat_f32x4_u $push[[R:[0-9]+]]=, $0
 ; SIMD128-NEXT: return $pop[[R]]
 declare <4 x i32> @llvm.wasm.trunc.saturate.unsigned.v4i32.v4f32(<4 x float>)
 define <4 x i32> @trunc_sat_u_v4i32(<4 x float> %x) {
@@ -248,12 +248,12 @@ define i32 @all_v2i64(<2 x i64> %x) {
 
 ; CHECK-LABEL: bitselect_v2i64:
 ; SIMD128-NEXT: .functype bitselect_v2i64 (v128, v128, v128) -> (v128){{$}}
-; SIMD128-NEXT: v128.bitselect $push[[R:[0-9]+]]=, $1, $2, $0{{$}}
+; SIMD128-NEXT: v128.bitselect $push[[R:[0-9]+]]=, $0, $1, $2{{$}}
 ; SIMD128-NEXT: return $pop[[R]]{{$}}
 declare <2 x i64> @llvm.wasm.bitselect.v2i64(<2 x i64>, <2 x i64>, <2 x i64>)
-define <2 x i64> @bitselect_v2i64(<2 x i64> %c, <2 x i64> %v1, <2 x i64> %v2) {
+define <2 x i64> @bitselect_v2i64(<2 x i64> %v1, <2 x i64> %v2, <2 x i64> %c) {
   %a = call <2 x i64> @llvm.wasm.bitselect.v2i64(
-    <2 x i64> %c, <2 x i64> %v1, <2 x i64> %v2
+    <2 x i64> %v1, <2 x i64> %v2, <2 x i64> %c
   )
   ret <2 x i64> %a
 }
@@ -261,7 +261,7 @@ define <2 x i64> @bitselect_v2i64(<2 x i64> %c, <2 x i64> %v1, <2 x i64> %v2) {
 ; CHECK-LABEL: trunc_sat_s_v2i64:
 ; NO-SIMD128-NOT: f32x4
 ; SIMD128-NEXT: .functype trunc_sat_s_v2i64 (v128) -> (v128){{$}}
-; SIMD128-NEXT: i64x2.trunc_sat_s/f64x2 $push[[R:[0-9]+]]=, $0
+; SIMD128-NEXT: i64x2.trunc_sat_f64x2_s $push[[R:[0-9]+]]=, $0
 ; SIMD128-NEXT: return $pop[[R]]
 declare <2 x i64> @llvm.wasm.trunc.saturate.signed.v2i64.v2f64(<2 x double>)
 define <2 x i64> @trunc_sat_s_v2i64(<2 x double> %x) {
@@ -272,7 +272,7 @@ define <2 x i64> @trunc_sat_s_v2i64(<2 x double> %x) {
 ; CHECK-LABEL: trunc_sat_u_v2i64:
 ; NO-SIMD128-NOT: f32x4
 ; SIMD128-NEXT: .functype trunc_sat_u_v2i64 (v128) -> (v128){{$}}
-; SIMD128-NEXT: i64x2.trunc_sat_u/f64x2 $push[[R:[0-9]+]]=, $0
+; SIMD128-NEXT: i64x2.trunc_sat_f64x2_u $push[[R:[0-9]+]]=, $0
 ; SIMD128-NEXT: return $pop[[R]]
 declare <2 x i64> @llvm.wasm.trunc.saturate.unsigned.v2i64.v2f64(<2 x double>)
 define <2 x i64> @trunc_sat_u_v2i64(<2 x double> %x) {
@@ -285,12 +285,12 @@ define <2 x i64> @trunc_sat_u_v2i64(<2 x double> %x) {
 ; ==============================================================================
 ; CHECK-LABEL: bitselect_v4f32:
 ; SIMD128-NEXT: .functype bitselect_v4f32 (v128, v128, v128) -> (v128){{$}}
-; SIMD128-NEXT: v128.bitselect $push[[R:[0-9]+]]=, $1, $2, $0{{$}}
+; SIMD128-NEXT: v128.bitselect $push[[R:[0-9]+]]=, $0, $1, $2{{$}}
 ; SIMD128-NEXT: return $pop[[R]]{{$}}
 declare <4 x float> @llvm.wasm.bitselect.v4f32(<4 x float>, <4 x float>, <4 x float>)
-define <4 x float> @bitselect_v4f32(<4 x float> %c, <4 x float> %v1, <4 x float> %v2) {
+define <4 x float> @bitselect_v4f32(<4 x float> %v1, <4 x float> %v2, <4 x float> %c) {
   %a = call <4 x float> @llvm.wasm.bitselect.v4f32(
-    <4 x float> %c, <4 x float> %v1, <4 x float> %v2
+     <4 x float> %v1, <4 x float> %v2, <4 x float> %c
   )
   ret <4 x float> %a
 }
@@ -300,12 +300,12 @@ define <4 x float> @bitselect_v4f32(<4 x float> %c, <4 x float> %v1, <4 x float>
 ; ==============================================================================
 ; CHECK-LABEL: bitselect_v2f64:
 ; SIMD128-NEXT: .functype bitselect_v2f64 (v128, v128, v128) -> (v128){{$}}
-; SIMD128-NEXT: v128.bitselect $push[[R:[0-9]+]]=, $1, $2, $0{{$}}
+; SIMD128-NEXT: v128.bitselect $push[[R:[0-9]+]]=, $0, $1, $2{{$}}
 ; SIMD128-NEXT: return $pop[[R]]{{$}}
 declare <2 x double> @llvm.wasm.bitselect.v2f64(<2 x double>, <2 x double>, <2 x double>)
-define <2 x double> @bitselect_v2f64(<2 x double> %c, <2 x double> %v1, <2 x double> %v2) {
+define <2 x double> @bitselect_v2f64(<2 x double> %v1, <2 x double> %v2, <2 x double> %c) {
   %a = call <2 x double> @llvm.wasm.bitselect.v2f64(
-    <2 x double> %c, <2 x double> %v1, <2 x double> %v2
+    <2 x double> %v1, <2 x double> %v2, <2 x double> %c
   )
   ret <2 x double> %a
 }
diff --git a/test/CodeGen/WebAssembly/simd-load-store-alignment.ll b/test/CodeGen/WebAssembly/simd-load-store-alignment.ll
index 50935b610985fac7f8a4330c85caa31ff09e4341..c19a7f616296429fbe5148743993f2043d750270 100644
--- a/test/CodeGen/WebAssembly/simd-load-store-alignment.ll
+++ b/test/CodeGen/WebAssembly/simd-load-store-alignment.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -asm-verbose=false -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-keep-registers -wasm-enable-unimplemented-simd -mattr=+simd128 | FileCheck %s
+; RUN: llc < %s -asm-verbose=false -verify-machineinstrs -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-keep-registers -mattr=+unimplemented-simd128 | FileCheck %s
 
 ; Test loads and stores with custom alignment values.
 
diff --git a/test/CodeGen/WebAssembly/simd-nested-shuffles.ll b/test/CodeGen/WebAssembly/simd-nested-shuffles.ll
index 51ba5a99be618dce515cdd57114ca43ba4a1757f..597ab58e879e00d6453e7068430fcb3bf17dcf6f 100644
--- a/test/CodeGen/WebAssembly/simd-nested-shuffles.ll
+++ b/test/CodeGen/WebAssembly/simd-nested-shuffles.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -mattr=+simd128 | FileCheck %s --check-prefixes CHECK
+; RUN: llc < %s -mattr=+simd128 -verify-machineinstrs | FileCheck %s --check-prefixes CHECK
 
 ; Check that shuffles maintain their type when being custom
 ; lowered. Regression test for bug 39275.
diff --git a/test/CodeGen/WebAssembly/simd-noopt.ll b/test/CodeGen/WebAssembly/simd-noopt.ll
new file mode 100644
index 0000000000000000000000000000000000000000..1ec259ccca73127afe4a57e91fe6d1945e9f1833
--- /dev/null
+++ b/test/CodeGen/WebAssembly/simd-noopt.ll
@@ -0,0 +1,20 @@
+; RUN: llc < %s -fast-isel -mattr=+simd128,+sign-ext -verify-machineinstrs
+
+;; Ensures fastisel produces valid code when storing and loading split
+;; up v2i64 values. Lowering away v2i64s is a temporary measure while
+;; V8 does not have support for i64x2.* operations, and is done when
+;; -wasm-enable-unimplemented-simd is not present. This is a
+;; regression test for a bug that crashed llc after fastisel produced
+;; machineinstrs that used registers that had never been defined.
+
+target datalayout = "e-m:e-p:32:32-i64:64-n32:64-S128"
+target triple = "wasm32-unknown-unknown"
+
+define i64 @foo(<2 x i64> %vec) {
+entry:
+  %vec.addr = alloca <2 x i64>, align 16
+  store <2 x i64> %vec, <2 x i64>* %vec.addr, align 16
+  %0 = load <2 x i64>, <2 x i64>* %vec.addr, align 16
+  %1 = extractelement <2 x i64> %0, i32 0
+  ret i64 %1
+}
diff --git a/test/CodeGen/WebAssembly/simd-offset.ll b/test/CodeGen/WebAssembly/simd-offset.ll
index 6bc165f28c1cd2423b015c37ed064a7880c8df63..61177ae924c918d58302e465e209ce2f5600f6a7 100644
--- a/test/CodeGen/WebAssembly/simd-offset.ll
+++ b/test/CodeGen/WebAssembly/simd-offset.ll
@@ -1,6 +1,6 @@
-; RUN: llc < %s -asm-verbose=false -disable-wasm-fallthrough-return-opt -wasm-keep-registers -wasm-disable-explicit-locals -wasm-enable-unimplemented-simd -mattr=+simd128,+sign-ext | FileCheck %s --check-prefixes CHECK,SIMD128
-; RUN: llc < %s -asm-verbose=false -disable-wasm-fallthrough-return-opt -wasm-keep-registers -wasm-disable-explicit-locals -mattr=+simd128,+sign-ext | FileCheck %s --check-prefixes CHECK,SIMD128-VM
-; RUN: llc < %s -asm-verbose=false -disable-wasm-fallthrough-return-opt -wasm-keep-registers -wasm-disable-explicit-locals -mattr=-simd128,+sign-ext | FileCheck %s --check-prefixes CHECK,NO-SIMD128
+; RUN: llc < %s -asm-verbose=false -verify-machineinstrs -disable-wasm-fallthrough-return-opt -wasm-keep-registers -wasm-disable-explicit-locals -mattr=+unimplemented-simd128 | FileCheck %s --check-prefixes CHECK,SIMD128
+; RUN: llc < %s -asm-verbose=false -verify-machineinstrs -disable-wasm-fallthrough-return-opt -wasm-keep-registers -wasm-disable-explicit-locals -mattr=+simd128 | FileCheck %s --check-prefixes CHECK,SIMD128-VM
+; RUN: llc < %s -asm-verbose=false -verify-machineinstrs -disable-wasm-fallthrough-return-opt -wasm-keep-registers -wasm-disable-explicit-locals | FileCheck %s --check-prefixes CHECK,NO-SIMD128
 
 ; Test SIMD loads and stores
 
diff --git a/test/CodeGen/WebAssembly/simd-scalar-to-vector.ll b/test/CodeGen/WebAssembly/simd-scalar-to-vector.ll
new file mode 100644
index 0000000000000000000000000000000000000000..b867bd6a31b90d5161de08b24ba56212fbebf8c2
--- /dev/null
+++ b/test/CodeGen/WebAssembly/simd-scalar-to-vector.ll
@@ -0,0 +1,43 @@
+; RUN: llc < %s -asm-verbose=false -verify-machineinstrs -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-keep-registers -mattr=+simd128 | FileCheck %s
+
+; Test that scalar_to_vector is lowered into a splat correctly.
+; This bugpoint-reduced code turns into the selection dag below.
+; TODO: find small test cases that produce scalar_to_vector dag nodes
+; to make this test more readable and comprehensive.
+
+;   t0: ch = EntryToken
+; t32: i32,ch = load<(load 4 from `<2 x i16>* undef`, align 1)> t0, undef:i32, undef:i32
+;   t33: v4i32 = scalar_to_vector t32
+; t34: v8i16 = bitcast t33
+;       t51: i32 = extract_vector_elt t34, Constant:i32<0>
+;   t52: ch = store<(store 2 into `<4 x i16>* undef`, align 1), trunc to i16> t32:1, t51, undef:i32, undef:i32
+;       t50: i32 = extract_vector_elt t34, Constant:i32<1>
+;     t53: ch = store<(store 2 into `<4 x i16>* undef` + 2, align 1), trunc to i16> t32:1, t50, undef:i32, undef:i32
+;       t49: i32 = extract_vector_elt t34, Constant:i32<2>
+;     t55: ch = store<(store 2 into `<4 x i16>* undef` + 4, align 1), trunc to i16> t32:1, t49, undef:i32, undef:i32
+;       t48: i32 = extract_vector_elt t34, Constant:i32<3>
+;     t57: ch = store<(store 2 into `<4 x i16>* undef` + 6, align 1), trunc to i16> t32:1, t48, undef:i32, undef:i32
+;   t58: ch = TokenFactor t52, t53, t55, t57
+; t24: ch = WebAssemblyISD::RETURN t58
+
+target datalayout = "e-m:e-p:32:32-i64:64-n32:64-S128"
+target triple = "wasm32-unknown-unknown"
+
+; CHECK-LABEL: foo:
+; CHECK: i32x4.splat
+define void @foo() {
+entry:
+  %a = load <2 x i16>, <2 x i16>* undef, align 1
+  %b = shufflevector <2 x i16> %a, <2 x i16> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %0 = bitcast <8 x i16> %b to <16 x i8>
+  %shuffle.i214 = shufflevector <16 x i8> %0, <16 x i8> <i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef>, <16 x i32> <i32 0, i32 16, i32 1, i32 17, i32 2, i32 18, i32 3, i32 19, i32 4, i32 20, i32 5, i32 21, i32 6, i32 22, i32 7, i32 23>
+  %1 = bitcast <16 x i8> %shuffle.i214 to <8 x i16>
+  %add82 = add <8 x i16> %1, zeroinitializer
+  %2 = select <8 x i1> undef, <8 x i16> undef, <8 x i16> %add82
+  %3 = bitcast <8 x i16> %2 to <16 x i8>
+  %shuffle.i204 = shufflevector <16 x i8> %3, <16 x i8> undef, <16 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14, i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
+  %4 = bitcast <16 x i8> %shuffle.i204 to <8 x i16>
+  %dst2.0.vec.extract = shufflevector <8 x i16> %4, <8 x i16> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  store <4 x i16> %dst2.0.vec.extract, <4 x i16>* undef, align 1
+  ret void
+}
diff --git a/test/CodeGen/WebAssembly/simd-select.ll b/test/CodeGen/WebAssembly/simd-select.ll
index 70a9149d49c4462508494c6a3c7861e0713bd023..c871f60e6454ce486788089962b18b9d18071ca4 100644
--- a/test/CodeGen/WebAssembly/simd-select.ll
+++ b/test/CodeGen/WebAssembly/simd-select.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -asm-verbose=false -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-keep-registers -wasm-enable-unimplemented-simd -mattr=+simd128,+sign-ext | FileCheck %s
+; RUN: llc < %s -asm-verbose=false -verify-machineinstrs -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-keep-registers -mattr=+unimplemented-simd128 | FileCheck %s
 
 ; Test that vector selects of various varieties lower correctly to bitselects.
 
diff --git a/test/CodeGen/WebAssembly/simd-sext-inreg.ll b/test/CodeGen/WebAssembly/simd-sext-inreg.ll
index 5fecb4d0bac44e4034de13acaa959972d298697c..0c375827c0f58d023fd03a0338349b5901c8140a 100644
--- a/test/CodeGen/WebAssembly/simd-sext-inreg.ll
+++ b/test/CodeGen/WebAssembly/simd-sext-inreg.ll
@@ -1,6 +1,6 @@
-; RUN: llc < %s -asm-verbose=false -wasm-keep-registers -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-enable-unimplemented-simd -mattr=+simd128,+sign-ext | FileCheck %s --check-prefixes CHECK,SIMD128
-; RUN: llc < %s -asm-verbose=false -wasm-keep-registers -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -mattr=+simd128,+sign-ext | FileCheck %s --check-prefixes CHECK,SIMD128-VM
-; RUN: llc < %s -asm-verbose=false -wasm-keep-registers -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -mattr=-simd128,+sign-ext | FileCheck %s --check-prefixes CHECK,NO-SIMD128
+; RUN: llc < %s -asm-verbose=false -verify-machineinstrs -wasm-keep-registers -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -mattr=+unimplemented-simd128 | FileCheck %s --check-prefixes CHECK,SIMD128
+; RUN: llc < %s -asm-verbose=false -verify-machineinstrs -wasm-keep-registers -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -mattr=+simd128 | FileCheck %s --check-prefixes CHECK,SIMD128-VM
+; RUN: llc < %s -asm-verbose=false -verify-machineinstrs -wasm-keep-registers -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals | FileCheck %s --check-prefixes CHECK,NO-SIMD128
 
 ; Test that vector sign extensions lower to shifts
 
diff --git a/test/CodeGen/WebAssembly/simd.ll b/test/CodeGen/WebAssembly/simd.ll
index 29b51f16694b00d0108c13a3f66cf75e5dadc8b8..1393f62e4597c02694c679551d1718546c4c79c4 100644
--- a/test/CodeGen/WebAssembly/simd.ll
+++ b/test/CodeGen/WebAssembly/simd.ll
@@ -1,6 +1,6 @@
-; RUN: llc < %s -asm-verbose=false -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-keep-registers -wasm-enable-unimplemented-simd -mattr=+simd128,+sign-ext | FileCheck %s --check-prefixes CHECK,SIMD128
-; RUN: llc < %s -asm-verbose=false -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-keep-registers -mattr=+simd128,+sign-ext | FileCheck %s --check-prefixes CHECK,SIMD128-VM
-; RUN: llc < %s -asm-verbose=false -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-keep-registers -mattr=-simd128,+sign-ext | FileCheck %s --check-prefixes CHECK,NO-SIMD128
+; RUN: llc < %s -asm-verbose=false -verify-machineinstrs -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-keep-registers -mattr=+unimplemented-simd128,+sign-ext | FileCheck %s --check-prefixes CHECK,SIMD128
+; RUN: llc < %s -asm-verbose=false -verify-machineinstrs -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-keep-registers -mattr=+simd128,+sign-ext | FileCheck %s --check-prefixes CHECK,SIMD128-VM
+; RUN: llc < %s -asm-verbose=false -verify-machineinstrs -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-keep-registers | FileCheck %s --check-prefixes CHECK,NO-SIMD128
 
 ; Test that basic SIMD128 vector manipulation operations assemble as expected.
 
@@ -12,6 +12,7 @@ target triple = "wasm32-unknown-unknown"
 ; ==============================================================================
 ; CHECK-LABEL: const_v16i8:
 ; NO-SIMD128-NOT: i8x16
+; SIMD128-VM-NOT: v128.const
 ; SIMD128-NEXT: .functype const_v16i8 () -> (v128){{$}}
 ; SIMD128-NEXT: v128.const $push[[R:[0-9]+]]=,
 ; SIMD128-SAME: 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
@@ -55,10 +56,10 @@ define i32 @extract_v16i8_s(<16 x i8> %v) {
 ; CHECK-LABEL: extract_var_v16i8_s:
 ; NO-SIMD128-NOT: i8x16
 ; SIMD128-NEXT: .functype extract_var_v16i8_s (v128, i32) -> (i32){{$}}
-; SIMD128-NEXT: get_global $push[[L0:[0-9]+]]=, __stack_pointer@GLOBAL
+; SIMD128-NEXT: global.get $push[[L0:[0-9]+]]=, __stack_pointer@GLOBAL
 ; SIMD128-NEXT: i32.const $push[[L1:[0-9]+]]=, 16
 ; SIMD128-NEXT: i32.sub $push[[L2:[0-9]+]]=, $pop[[L0]], $pop[[L1]]
-; SIMD128-NEXT: tee_local $push[[L3:[0-9]+]]=, $2=, $pop[[L2]]
+; SIMD128-NEXT: local.tee $push[[L3:[0-9]+]]=, $2=, $pop[[L2]]
 ; SIMD128-NEXT: v128.store 0($pop[[L3]]), $0
 ; SIMD128-NEXT: i32.const $push[[L4:[0-9]+]]=, 15
 ; SIMD128-NEXT: i32.and $push[[L5:[0-9]+]]=, $1, $pop[[L4]]
@@ -84,6 +85,7 @@ define i32 @extract_undef_v16i8_s(<16 x i8> %v) {
 
 ; CHECK-LABEL: extract_v16i8_u:
 ; NO-SIMD128-NOT: i8x16
+; SIMD128-VM-NOT: i8x16.extract_lane_u
 ; SIMD128-NEXT: .functype extract_v16i8_u (v128) -> (i32){{$}}
 ; SIMD128-NEXT: i8x16.extract_lane_u $push[[R:[0-9]+]]=, $0, 13{{$}}
 ; SIMD128-NEXT: return $pop[[R]]{{$}}
@@ -96,10 +98,10 @@ define i32 @extract_v16i8_u(<16 x i8> %v) {
 ; CHECK-LABEL: extract_var_v16i8_u:
 ; NO-SIMD128-NOT: i8x16
 ; SIMD128-NEXT: .functype extract_var_v16i8_u (v128, i32) -> (i32){{$}}
-; SIMD128-NEXT: get_global $push[[L0:[0-9]+]]=, __stack_pointer@GLOBAL{{$}}
+; SIMD128-NEXT: global.get $push[[L0:[0-9]+]]=, __stack_pointer@GLOBAL{{$}}
 ; SIMD128-NEXT: i32.const $push[[L1:[0-9]+]]=, 16{{$}}
 ; SIMD128-NEXT: i32.sub $push[[L2:[0-9]+]]=, $pop[[L0]], $pop[[L1]]{{$}}
-; SIMD128-NEXT: tee_local $push[[L3:[0-9]+]]=, $2=, $pop[[L2]]{{$}}
+; SIMD128-NEXT: local.tee $push[[L3:[0-9]+]]=, $2=, $pop[[L2]]{{$}}
 ; SIMD128-NEXT: v128.store 0($pop[[L3]]), $0{{$}}
 ; SIMD128-NEXT: i32.const $push[[L4:[0-9]+]]=, 15{{$}}
 ; SIMD128-NEXT: i32.and $push[[L5:[0-9]+]]=, $1, $pop[[L4]]{{$}}
@@ -114,6 +116,7 @@ define i32 @extract_var_v16i8_u(<16 x i8> %v, i32 %i) {
 
 ; CHECK-LABEL: extract_undef_v16i8_u:
 ; NO-SIMD128-NOT: i8x16
+; SIMD128-VM-NOT: i8x16.extract_lane_u
 ; SIMD128-NEXT: .functype extract_undef_v16i8_u (v128) -> (i32){{$}}
 ; SIMD128-NEXT: i8x16.extract_lane_u $push[[R:[0-9]+]]=, $0, 0{{$}}
 ; SIMD128-NEXT: return $pop[[R]]{{$}}
@@ -126,7 +129,7 @@ define i32 @extract_undef_v16i8_u(<16 x i8> %v) {
 ; CHECK-LABEL: extract_v16i8:
 ; NO-SIMD128-NOT: i8x16
 ; SIMD128-NEXT: .functype extract_v16i8 (v128) -> (i32){{$}}
-; SIMD128-NEXT: i8x16.extract_lane_u $push[[R:[0-9]+]]=, $0, 13{{$}}
+; SIMD128-NEXT: i8x16.extract_lane_s $push[[R:[0-9]+]]=, $0, 13{{$}}
 ; SIMD128-NEXT: return $pop[[R]]{{$}}
 define i8 @extract_v16i8(<16 x i8> %v) {
   %elem = extractelement <16 x i8> %v, i8 13
@@ -136,10 +139,10 @@ define i8 @extract_v16i8(<16 x i8> %v) {
 ; CHECK-LABEL: extract_var_v16i8:
 ; NO-SIMD128-NOT: i8x16
 ; SIMD128-NEXT: .functype extract_var_v16i8 (v128, i32) -> (i32){{$}}
-; SIMD128-NEXT: get_global $push[[L0:[0-9]+]]=, __stack_pointer@GLOBAL{{$}}
+; SIMD128-NEXT: global.get $push[[L0:[0-9]+]]=, __stack_pointer@GLOBAL{{$}}
 ; SIMD128-NEXT: i32.const $push[[L1:[0-9]+]]=, 16{{$}}
 ; SIMD128-NEXT: i32.sub $push[[L2:[0-9]+]]=, $pop[[L0]], $pop[[L1]]{{$}}
-; SIMD128-NEXT: tee_local $push[[L3:[0-9]+]]=, $2=, $pop[[L2]]{{$}}
+; SIMD128-NEXT: local.tee $push[[L3:[0-9]+]]=, $2=, $pop[[L2]]{{$}}
 ; SIMD128-NEXT: v128.store 0($pop[[L3]]), $0{{$}}
 ; SIMD128-NEXT: i32.const $push[[L4:[0-9]+]]=, 15{{$}}
 ; SIMD128-NEXT: i32.and $push[[L5:[0-9]+]]=, $1, $pop[[L4]]{{$}}
@@ -154,7 +157,7 @@ define i8 @extract_var_v16i8(<16 x i8> %v, i32 %i) {
 ; CHECK-LABEL: extract_undef_v16i8:
 ; NO-SIMD128-NOT: i8x16
 ; SIMD128-NEXT: .functype extract_undef_v16i8 (v128) -> (i32){{$}}
-; SIMD128-NEXT: i8x16.extract_lane_u $push[[R:[0-9]+]]=, $0, 0{{$}}
+; SIMD128-NEXT: i8x16.extract_lane_s $push[[R:[0-9]+]]=, $0, 0{{$}}
 ; SIMD128-NEXT: return $pop[[R]]{{$}}
 define i8 @extract_undef_v16i8(<16 x i8> %v) {
   %elem = extractelement <16 x i8> %v, i8 undef
@@ -174,10 +177,10 @@ define <16 x i8> @replace_v16i8(<16 x i8> %v, i8 %x) {
 ; CHECK-LABEL: replace_var_v16i8:
 ; NO-SIMD128-NOT: i8x16
 ; SIMD128-NEXT: .functype replace_var_v16i8 (v128, i32, i32) -> (v128){{$}}
-; SIMD128-NEXT: get_global $push[[L0:[0-9]+]]=, __stack_pointer@GLOBAL{{$}}
+; SIMD128-NEXT: global.get $push[[L0:[0-9]+]]=, __stack_pointer@GLOBAL{{$}}
 ; SIMD128-NEXT: i32.const $push[[L1:[0-9]+]]=, 16{{$}}
 ; SIMD128-NEXT: i32.sub $push[[L2:[0-9]+]]=, $pop[[L0]], $pop[[L1]]{{$}}
-; SIMD128-NEXT: tee_local $push[[L3:[0-9]+]]=, $3=, $pop[[L2]]{{$}}
+; SIMD128-NEXT: local.tee $push[[L3:[0-9]+]]=, $3=, $pop[[L2]]{{$}}
 ; SIMD128-NEXT: v128.store 0($pop[[L3]]), $0{{$}}
 ; SIMD128-NEXT: i32.const $push[[L4:[0-9]+]]=, 15{{$}}
 ; SIMD128-NEXT: i32.and $push[[L5:[0-9]+]]=, $1, $pop[[L4]]{{$}}
@@ -316,10 +319,10 @@ define i32 @extract_v8i16_s(<8 x i16> %v) {
 ; CHECK-LABEL: extract_var_v8i16_s:
 ; NO-SIMD128-NOT: i16x8
 ; SIMD128-NEXT: .functype extract_var_v8i16_s (v128, i32) -> (i32){{$}}
-; SIMD128-NEXT: get_global $push[[L0:[0-9]+]]=, __stack_pointer@GLOBAL{{$}}
+; SIMD128-NEXT: global.get $push[[L0:[0-9]+]]=, __stack_pointer@GLOBAL{{$}}
 ; SIMD128-NEXT: i32.const $push[[L1:[0-9]+]]=, 16{{$}}
 ; SIMD128-NEXT: i32.sub $push[[L2:[0-9]+]]=, $pop[[L0]], $pop[[L1]]{{$}}
-; SIMD128-NEXT: tee_local $push[[L3:[0-9]+]]=, $2=, $pop[[L2]]{{$}}
+; SIMD128-NEXT: local.tee $push[[L3:[0-9]+]]=, $2=, $pop[[L2]]{{$}}
 ; SIMD128-NEXT: v128.store 0($pop[[L3]]), $0{{$}}
 ; SIMD128-NEXT: i32.const $push[[L4:[0-9]+]]=, 7{{$}}
 ; SIMD128-NEXT: i32.and $push[[L5:[0-9]+]]=, $1, $pop[[L4]]{{$}}
@@ -347,6 +350,7 @@ define i32 @extract_undef_v8i16_s(<8 x i16> %v) {
 
 ; CHECK-LABEL: extract_v8i16_u:
 ; NO-SIMD128-NOT: i16x8
+; SIMD128-VM-NOT: i16x8.extract_lane_u
 ; SIMD128-NEXT: .functype extract_v8i16_u (v128) -> (i32){{$}}
 ; SIMD128-NEXT: i16x8.extract_lane_u $push[[R:[0-9]+]]=, $0, 5{{$}}
 ; SIMD128-NEXT: return $pop[[R]]{{$}}
@@ -359,10 +363,10 @@ define i32 @extract_v8i16_u(<8 x i16> %v) {
 ; CHECK-LABEL: extract_var_v8i16_u:
 ; NO-SIMD128-NOT: i16x8
 ; SIMD128-NEXT: .functype extract_var_v8i16_u (v128, i32) -> (i32){{$}}
-; SIMD128-NEXT: get_global $push[[L0:[0-9]+]]=, __stack_pointer@GLOBAL{{$}}
+; SIMD128-NEXT: global.get $push[[L0:[0-9]+]]=, __stack_pointer@GLOBAL{{$}}
 ; SIMD128-NEXT: i32.const $push[[L1:[0-9]+]]=, 16{{$}}
 ; SIMD128-NEXT: i32.sub $push[[L2:[0-9]+]]=, $pop[[L0]], $pop[[L1]]{{$}}
-; SIMD128-NEXT: tee_local $push[[L3:[0-9]+]]=, $2=, $pop[[L2]]{{$}}
+; SIMD128-NEXT: local.tee $push[[L3:[0-9]+]]=, $2=, $pop[[L2]]{{$}}
 ; SIMD128-NEXT: v128.store 0($pop[[L3]]), $0{{$}}
 ; SIMD128-NEXT: i32.const $push[[L4:[0-9]+]]=, 7{{$}}
 ; SIMD128-NEXT: i32.and $push[[L5:[0-9]+]]=, $1, $pop[[L4]]{{$}}
@@ -379,6 +383,7 @@ define i32 @extract_var_v8i16_u(<8 x i16> %v, i32 %i) {
 
 ; CHECK-LABEL: extract_undef_v8i16_u:
 ; NO-SIMD128-NOT: i16x8
+; SIMD128-VM-NOT: i16x8.extract_lane_u
 ; SIMD128-NEXT: .functype extract_undef_v8i16_u (v128) -> (i32){{$}}
 ; SIMD128-NEXT: i16x8.extract_lane_u $push[[R:[0-9]+]]=, $0, 0{{$}}
 ; SIMD128-NEXT: return $pop[[R]]{{$}}
@@ -391,7 +396,7 @@ define i32 @extract_undef_v8i16_u(<8 x i16> %v) {
 ; CHECK-LABEL: extract_v8i16:
 ; NO-SIMD128-NOT: i16x8
 ; SIMD128-NEXT: .functype extract_v8i16 (v128) -> (i32){{$}}
-; SIMD128-NEXT: i16x8.extract_lane_u $push[[R:[0-9]+]]=, $0, 5{{$}}
+; SIMD128-NEXT: i16x8.extract_lane_s $push[[R:[0-9]+]]=, $0, 5{{$}}
 ; SIMD128-NEXT: return $pop[[R]]{{$}}
 define i16 @extract_v8i16(<8 x i16> %v) {
   %elem = extractelement <8 x i16> %v, i16 5
@@ -401,10 +406,10 @@ define i16 @extract_v8i16(<8 x i16> %v) {
 ; CHECK-LABEL: extract_var_v8i16:
 ; NO-SIMD128-NOT: i16x8
 ; SIMD128-NEXT: .functype extract_var_v8i16 (v128, i32) -> (i32){{$}}
-; SIMD128-NEXT: get_global $push[[L0:[0-9]+]]=, __stack_pointer@GLOBAL{{$}}
+; SIMD128-NEXT: global.get $push[[L0:[0-9]+]]=, __stack_pointer@GLOBAL{{$}}
 ; SIMD128-NEXT: i32.const $push[[L1:[0-9]+]]=, 16{{$}}
 ; SIMD128-NEXT: i32.sub $push[[L2:[0-9]+]]=, $pop[[L0]], $pop[[L1]]{{$}}
-; SIMD128-NEXT: tee_local $push[[L3:[0-9]+]]=, $2=, $pop[[L2]]{{$}}
+; SIMD128-NEXT: local.tee $push[[L3:[0-9]+]]=, $2=, $pop[[L2]]{{$}}
 ; SIMD128-NEXT: v128.store 0($pop[[L3]]), $0{{$}}
 ; SIMD128-NEXT: i32.const $push[[L4:[0-9]+]]=, 7{{$}}
 ; SIMD128-NEXT: i32.and $push[[L5:[0-9]+]]=, $1, $pop[[L4]]{{$}}
@@ -421,7 +426,7 @@ define i16 @extract_var_v8i16(<8 x i16> %v, i32 %i) {
 ; CHECK-LABEL: extract_undef_v8i16:
 ; NO-SIMD128-NOT: i16x8
 ; SIMD128-NEXT: .functype extract_undef_v8i16 (v128) -> (i32){{$}}
-; SIMD128-NEXT: i16x8.extract_lane_u $push[[R:[0-9]+]]=, $0, 0{{$}}
+; SIMD128-NEXT: i16x8.extract_lane_s $push[[R:[0-9]+]]=, $0, 0{{$}}
 ; SIMD128-NEXT: return $pop[[R]]{{$}}
 define i16 @extract_undef_v8i16(<8 x i16> %v) {
   %elem = extractelement <8 x i16> %v, i16 undef
@@ -441,10 +446,10 @@ define <8 x i16> @replace_v8i16(<8 x i16> %v, i16 %x) {
 ; CHECK-LABEL: replace_var_v8i16:
 ; NO-SIMD128-NOT: i16x8
 ; SIMD128-NEXT: .functype replace_var_v8i16 (v128, i32, i32) -> (v128){{$}}
-; SIMD128-NEXT: get_global $push[[L0:[0-9]+]]=, __stack_pointer@GLOBAL{{$}}
+; SIMD128-NEXT: global.get $push[[L0:[0-9]+]]=, __stack_pointer@GLOBAL{{$}}
 ; SIMD128-NEXT: i32.const $push[[L1:[0-9]+]]=, 16{{$}}
 ; SIMD128-NEXT: i32.sub $push[[L2:[0-9]+]]=, $pop[[L0]], $pop[[L1]]{{$}}
-; SIMD128-NEXT: tee_local $push[[L3:[0-9]+]]=, $3=, $pop[[L2]]{{$}}
+; SIMD128-NEXT: local.tee $push[[L3:[0-9]+]]=, $3=, $pop[[L2]]{{$}}
 ; SIMD128-NEXT: v128.store 0($pop[[L3]]), $0{{$}}
 ; SIMD128-NEXT: i32.const $push[[L4:[0-9]+]]=, 7{{$}}
 ; SIMD128-NEXT: i32.and $push[[L5:[0-9]+]]=, $1, $pop[[L4]]{{$}}
@@ -562,10 +567,10 @@ define i32 @extract_v4i32(<4 x i32> %v) {
 ; CHECK-LABEL: extract_var_v4i32:
 ; NO-SIMD128-NOT: i32x4
 ; SIMD128-NEXT: .functype extract_var_v4i32 (v128, i32) -> (i32){{$}}
-; SIMD128-NEXT: get_global $push[[L0:[0-9]+]]=, __stack_pointer@GLOBAL{{$}}
+; SIMD128-NEXT: global.get $push[[L0:[0-9]+]]=, __stack_pointer@GLOBAL{{$}}
 ; SIMD128-NEXT: i32.const $push[[L1:[0-9]+]]=, 16{{$}}
 ; SIMD128-NEXT: i32.sub $push[[L2:[0-9]+]]=, $pop[[L0]], $pop[[L1]]{{$}}
-; SIMD128-NEXT: tee_local $push[[L3:[0-9]+]]=, $2=, $pop[[L2]]{{$}}
+; SIMD128-NEXT: local.tee $push[[L3:[0-9]+]]=, $2=, $pop[[L2]]{{$}}
 ; SIMD128-NEXT: v128.store 0($pop[[L3]]), $0{{$}}
 ; SIMD128-NEXT: i32.const $push[[L4:[0-9]+]]=, 3{{$}}
 ; SIMD128-NEXT: i32.and $push[[L5:[0-9]+]]=, $1, $pop[[L4]]{{$}}
@@ -602,10 +607,10 @@ define <4 x i32> @replace_v4i32(<4 x i32> %v, i32 %x) {
 ; CHECK-LABEL: replace_var_v4i32:
 ; NO-SIMD128-NOT: i32x4
 ; SIMD128-NEXT: .functype replace_var_v4i32 (v128, i32, i32) -> (v128){{$}}
-; SIMD128-NEXT: get_global $push[[L0:[0-9]+]]=, __stack_pointer@GLOBAL{{$}}
+; SIMD128-NEXT: global.get $push[[L0:[0-9]+]]=, __stack_pointer@GLOBAL{{$}}
 ; SIMD128-NEXT: i32.const $push[[L1:[0-9]+]]=, 16{{$}}
 ; SIMD128-NEXT: i32.sub $push[[L2:[0-9]+]]=, $pop[[L0]], $pop[[L1]]{{$}}
-; SIMD128-NEXT: tee_local $push[[L3:[0-9]+]]=, $3=, $pop[[L2]]{{$}}
+; SIMD128-NEXT: local.tee $push[[L3:[0-9]+]]=, $3=, $pop[[L2]]{{$}}
 ; SIMD128-NEXT: v128.store 0($pop[[L3]]), $0{{$}}
 ; SIMD128-NEXT: i32.const $push[[L4:[0-9]+]]=, 3{{$}}
 ; SIMD128-NEXT: i32.and $push[[L5:[0-9]+]]=, $1, $pop[[L4]]{{$}}
@@ -715,10 +720,10 @@ define i64 @extract_v2i64(<2 x i64> %v) {
 ; CHECK-LABEL: extract_var_v2i64:
 ; NO-SIMD128-NOT: i64x2
 ; SIMD128-NEXT: .functype extract_var_v2i64 (v128, i32) -> (i64){{$}}
-; SIMD128-NEXT: get_global $push[[L0:[0-9]+]]=, __stack_pointer@GLOBAL{{$}}
+; SIMD128-NEXT: global.get $push[[L0:[0-9]+]]=, __stack_pointer@GLOBAL{{$}}
 ; SIMD128-NEXT: i32.const $push[[L1:[0-9]+]]=, 16{{$}}
 ; SIMD128-NEXT: i32.sub $push[[L2:[0-9]+]]=, $pop[[L0]], $pop[[L1]]{{$}}
-; SIMD128-NEXT: tee_local $push[[L3:[0-9]+]]=, $2=, $pop[[L2]]{{$}}
+; SIMD128-NEXT: local.tee $push[[L3:[0-9]+]]=, $2=, $pop[[L2]]{{$}}
 ; SIMD128-NEXT: v128.store 0($pop[[L3]]), $0{{$}}
 ; SIMD128-NEXT: i32.const $push[[L2:[0-9]+]]=, 1{{$}}
 ; SIMD128-NEXT: i32.and $push[[L5:[0-9]+]]=, $1, $pop[[L2]]{{$}}
@@ -758,10 +763,10 @@ define <2 x i64> @replace_v2i64(<2 x i64> %v, i64 %x) {
 ; NO-SIMD128-NOT: i64x2
 ; SIMD128-VM-NOT: i64x2
 ; SIMD128-NEXT: .functype replace_var_v2i64 (v128, i32, i64) -> (v128){{$}}
-; SIMD128-NEXT: get_global $push[[L0:[0-9]+]]=, __stack_pointer@GLOBAL{{$}}
+; SIMD128-NEXT: global.get $push[[L0:[0-9]+]]=, __stack_pointer@GLOBAL{{$}}
 ; SIMD128-NEXT: i32.const $push[[L1:[0-9]+]]=, 16{{$}}
 ; SIMD128-NEXT: i32.sub $push[[L2:[0-9]+]]=, $pop[[L0]], $pop[[L1]]{{$}}
-; SIMD128-NEXT: tee_local $push[[L3:[0-9]+]]=, $3=, $pop[[L2]]{{$}}
+; SIMD128-NEXT: local.tee $push[[L3:[0-9]+]]=, $3=, $pop[[L2]]{{$}}
 ; SIMD128-NEXT: v128.store 0($pop[[L3]]), $0{{$}}
 ; SIMD128-NEXT: i32.const $push[[L2:[0-9]+]]=, 1{{$}}
 ; SIMD128-NEXT: i32.and $push[[L5:[0-9]+]]=, $1, $pop[[L2]]{{$}}
@@ -868,10 +873,10 @@ define float @extract_v4f32(<4 x float> %v) {
 ; CHECK-LABEL: extract_var_v4f32:
 ; NO-SIMD128-NOT: i64x2
 ; SIMD128-NEXT: .functype extract_var_v4f32 (v128, i32) -> (f32){{$}}
-; SIMD128-NEXT: get_global $push[[L0:[0-9]+]]=, __stack_pointer@GLOBAL{{$}}
+; SIMD128-NEXT: global.get $push[[L0:[0-9]+]]=, __stack_pointer@GLOBAL{{$}}
 ; SIMD128-NEXT: i32.const $push[[L1:[0-9]+]]=, 16{{$}}
 ; SIMD128-NEXT: i32.sub $push[[L2:[0-9]+]]=, $pop[[L0]], $pop[[L1]]{{$}}
-; SIMD128-NEXT: tee_local $push[[L3:[0-9]+]]=, $2=, $pop[[L2]]{{$}}
+; SIMD128-NEXT: local.tee $push[[L3:[0-9]+]]=, $2=, $pop[[L2]]{{$}}
 ; SIMD128-NEXT: v128.store 0($pop[[L3]]), $0{{$}}
 ; SIMD128-NEXT: i32.const $push[[L2:[0-9]+]]=, 3{{$}}
 ; SIMD128-NEXT: i32.and $push[[L5:[0-9]+]]=, $1, $pop[[L2]]{{$}}
@@ -908,10 +913,10 @@ define <4 x float> @replace_v4f32(<4 x float> %v, float %x) {
 ; CHECK-LABEL: replace_var_v4f32:
 ; NO-SIMD128-NOT: f32x4
 ; SIMD128-NEXT: .functype replace_var_v4f32 (v128, i32, f32) -> (v128){{$}}
-; SIMD128-NEXT: get_global $push[[L0:[0-9]+]]=, __stack_pointer@GLOBAL{{$}}
+; SIMD128-NEXT: global.get $push[[L0:[0-9]+]]=, __stack_pointer@GLOBAL{{$}}
 ; SIMD128-NEXT: i32.const $push[[L1:[0-9]+]]=, 16{{$}}
 ; SIMD128-NEXT: i32.sub $push[[L2:[0-9]+]]=, $pop[[L0]], $pop[[L1]]{{$}}
-; SIMD128-NEXT: tee_local $push[[L3:[0-9]+]]=, $3=, $pop[[L2]]{{$}}
+; SIMD128-NEXT: local.tee $push[[L3:[0-9]+]]=, $3=, $pop[[L2]]{{$}}
 ; SIMD128-NEXT: v128.store 0($pop[[L3]]), $0{{$}}
 ; SIMD128-NEXT: i32.const $push[[L2:[0-9]+]]=, 3{{$}}
 ; SIMD128-NEXT: i32.and $push[[L5:[0-9]+]]=, $1, $pop[[L2]]{{$}}
@@ -1020,10 +1025,10 @@ define double @extract_v2f64(<2 x double> %v) {
 ; CHECK-LABEL: extract_var_v2f64:
 ; NO-SIMD128-NOT: i62x2
 ; SIMD128-NEXT: .functype extract_var_v2f64 (v128, i32) -> (f64){{$}}
-; SIMD128-NEXT: get_global $push[[L0:[0-9]+]]=, __stack_pointer@GLOBAL{{$}}
+; SIMD128-NEXT: global.get $push[[L0:[0-9]+]]=, __stack_pointer@GLOBAL{{$}}
 ; SIMD128-NEXT: i32.const $push[[L1:[0-9]+]]=, 16{{$}}
 ; SIMD128-NEXT: i32.sub $push[[L2:[0-9]+]]=, $pop[[L0]], $pop[[L1]]{{$}}
-; SIMD128-NEXT: tee_local $push[[L3:[0-9]+]]=, $2=, $pop[[L2]]{{$}}
+; SIMD128-NEXT: local.tee $push[[L3:[0-9]+]]=, $2=, $pop[[L2]]{{$}}
 ; SIMD128-NEXT: v128.store 0($pop[[L3]]), $0{{$}}
 ; SIMD128-NEXT: i32.const $push[[L2:[0-9]+]]=, 1{{$}}
 ; SIMD128-NEXT: i32.and $push[[L5:[0-9]+]]=, $1, $pop[[L2]]{{$}}
@@ -1063,10 +1068,10 @@ define <2 x double> @replace_v2f64(<2 x double> %v, double %x) {
 ; NO-SIMD128-NOT: f64x2
 ; SIMD128-VM-NOT: f64x2
 ; SIMD128-NEXT: .functype replace_var_v2f64 (v128, i32, f64) -> (v128){{$}}
-; SIMD128-NEXT: get_global $push[[L0:[0-9]+]]=, __stack_pointer@GLOBAL{{$}}
+; SIMD128-NEXT: global.get $push[[L0:[0-9]+]]=, __stack_pointer@GLOBAL{{$}}
 ; SIMD128-NEXT: i32.const $push[[L1:[0-9]+]]=, 16{{$}}
 ; SIMD128-NEXT: i32.sub $push[[L2:[0-9]+]]=, $pop[[L0]], $pop[[L1]]{{$}}
-; SIMD128-NEXT: tee_local $push[[L3:[0-9]+]]=, $3=, $pop[[L2]]{{$}}
+; SIMD128-NEXT: local.tee $push[[L3:[0-9]+]]=, $3=, $pop[[L2]]{{$}}
 ; SIMD128-NEXT: v128.store 0($pop[[L3]]), $0{{$}}
 ; SIMD128-NEXT: i32.const $push[[L2:[0-9]+]]=, 1{{$}}
 ; SIMD128-NEXT: i32.and $push[[L5:[0-9]+]]=, $1, $pop[[L2]]{{$}}
diff --git a/test/CodeGen/WebAssembly/stack-alignment.ll b/test/CodeGen/WebAssembly/stack-alignment.ll
index 2c7380a62ce94cfe8dc559c7101e0c3875c9577a..a610db98c2742bfc91c8954c54439323299b7baf 100644
--- a/test/CodeGen/WebAssembly/stack-alignment.ll
+++ b/test/CodeGen/WebAssembly/stack-alignment.ll
@@ -6,18 +6,18 @@ target triple = "wasm32-unknown-unknown"
 declare void @somefunc(i32*)
 
 ; CHECK-LABEL: underalign:
-; CHECK:      get_global $push[[L1:.+]]=, __stack_pointer@GLOBAL{{$}}
+; CHECK:      global.get $push[[L1:.+]]=, __stack_pointer@GLOBAL{{$}}
 ; CHECK-NEXT: i32.const $push[[L2:.+]]=, 16
 ; CHECK-NEXT: i32.sub   $push[[L10:.+]]=, $pop[[L1]], $pop[[L2]]
-; CHECK-NEXT: tee_local $push{{.+}}=, [[SP:.+]], $pop[[L10]]
+; CHECK-NEXT: local.tee $push{{.+}}=, [[SP:.+]], $pop[[L10]]
 
-; CHECK:      get_local $push[[L3:.+]]=, [[SP]]{{$}}
+; CHECK:      local.get $push[[L3:.+]]=, [[SP]]{{$}}
 ; CHECK:      i32.add   $push[[underaligned:.+]]=, $pop[[L3]], $pop{{.+}}
 ; CHECK-NEXT: call      somefunc@FUNCTION, $pop[[underaligned]]
 
-; CHECK:      get_local $push[[M4:.+]]=, [[SP]]{{$}}
+; CHECK:      local.get $push[[M4:.+]]=, [[SP]]{{$}}
 ; CHECK:      i32.add   $push[[L5:.+]]=, $pop[[M4]], $pop{{.+}}
-; CHECK-NEXT: set_global __stack_pointer@GLOBAL, $pop[[L5]]
+; CHECK-NEXT: global.set __stack_pointer@GLOBAL, $pop[[L5]]
 define void @underalign() {
 entry:
   %underaligned = alloca i32, align 8
@@ -26,19 +26,19 @@ entry:
 }
 
 ; CHECK-LABEL: overalign:
-; CHECK:      get_global $push[[L10:.+]]=, __stack_pointer@GLOBAL{{$}}
-; CHECK-NEXT: tee_local  $push[[L9:.+]]=, [[BP:.+]], $pop[[L10]]
+; CHECK:      global.get $push[[L10:.+]]=, __stack_pointer@GLOBAL{{$}}
+; CHECK-NEXT: local.tee  $push[[L9:.+]]=, [[BP:.+]], $pop[[L10]]
 ; CHECK-NEXT: i32.const  $push[[L2:.+]]=, 32
 ; CHECK-NEXT: i32.sub    $push[[L8:.+]]=, $pop[[L9]], $pop[[L2]]
 ; CHECK-NEXT: i32.const  $push[[L3:.+]]=, -32
 ; CHECK-NEXT: i32.and    $push[[L7:.+]]=, $pop[[L8]], $pop[[L3]]
-; CHECK-NEXT: tee_local  $push{{.+}}=, [[SP:.+]], $pop[[L7]]
+; CHECK-NEXT: local.tee  $push{{.+}}=, [[SP:.+]], $pop[[L7]]
 
-; CHECK:      get_local  $push[[M5:.+]]=, [[SP]]{{$}}
+; CHECK:      local.get  $push[[M5:.+]]=, [[SP]]{{$}}
 ; CHECK:      call       somefunc@FUNCTION, $pop[[M5]]{{$}}
 
-; CHECK:      get_local  $push[[M6:.+]]=, [[BP]]{{$}}
-; CHECK-NEXT: set_global __stack_pointer@GLOBAL, $pop[[M6]]
+; CHECK:      local.get  $push[[M6:.+]]=, [[BP]]{{$}}
+; CHECK-NEXT: global.set __stack_pointer@GLOBAL, $pop[[M6]]
 define void @overalign() {
 entry:
   %overaligned = alloca i32, align 32
@@ -47,21 +47,21 @@ entry:
 }
 
 ; CHECK-LABEL: over_and_normal_align:
-; CHECK:      get_global $push[[L14:.+]]=, __stack_pointer@GLOBAL{{$}}
-; CHECK-NEXT: tee_local  $push[[L13:.+]]=, [[BP:.+]], $pop[[L14]]
+; CHECK:      global.get $push[[L14:.+]]=, __stack_pointer@GLOBAL{{$}}
+; CHECK-NEXT: local.tee  $push[[L13:.+]]=, [[BP:.+]], $pop[[L14]]
 ; CHECK:      i32.sub    $push[[L12:.+]]=, $pop[[L13]], $pop{{.+}}
 ; CHECK:      i32.and    $push[[L11:.+]]=, $pop[[L12]], $pop{{.+}}
-; CHECK-NEXT: tee_local  $push{{.+}}=, [[SP:.+]], $pop[[L11]]
+; CHECK-NEXT: local.tee  $push{{.+}}=, [[SP:.+]], $pop[[L11]]
 
-; CHECK:      get_local  $push[[M6:.+]]=, [[SP]]{{$}}
+; CHECK:      local.get  $push[[M6:.+]]=, [[SP]]{{$}}
 ; CHECK:      i32.add    $push[[L6:.+]]=, $pop[[M6]], $pop{{.+}}
 ; CHECK-NEXT: call       somefunc@FUNCTION, $pop[[L6]]
-; CHECK:      get_local  $push[[M7:.+]]=, [[SP]]{{$}}
+; CHECK:      local.get  $push[[M7:.+]]=, [[SP]]{{$}}
 ; CHECK:      i32.add    $push[[L8:.+]]=, $pop[[M7]], $pop{{.+}}
 ; CHECK-NEXT: call       somefunc@FUNCTION, $pop[[L8]]
 
-; CHECK:      get_local  $push[[L6:.+]]=, [[BP]]{{$}}
-; CHECK-NEXT: set_global __stack_pointer@GLOBAL, $pop[[L6]]
+; CHECK:      local.get  $push[[L6:.+]]=, [[BP]]{{$}}
+; CHECK-NEXT: global.set __stack_pointer@GLOBAL, $pop[[L6]]
 define void @over_and_normal_align() {
 entry:
   %over = alloca i32, align 32
@@ -72,16 +72,16 @@ entry:
 }
 
 ; CHECK-LABEL: dynamic_overalign:
-; CHECK:      get_global $push[[L18:.+]]=, __stack_pointer@GLOBAL{{$}}
-; CHECK-NEXT: tee_local  $push[[L17:.+]]=, [[SP:.+]], $pop[[L18]]
-; CHECK-NEXT: set_local  [[BP:.+]], $pop[[L17]]
-; CHECK:      tee_local  $push{{.+}}=, [[SP_2:.+]], $pop{{.+}}
+; CHECK:      global.get $push[[L18:.+]]=, __stack_pointer@GLOBAL{{$}}
+; CHECK-NEXT: local.tee  $push[[L17:.+]]=, [[SP:.+]], $pop[[L18]]
+; CHECK-NEXT: local.set  [[BP:.+]], $pop[[L17]]
+; CHECK:      local.tee  $push{{.+}}=, [[SP_2:.+]], $pop{{.+}}
 
-; CHECK:      get_local  $push[[M8:.+]]=, [[SP_2]]{{$}}
+; CHECK:      local.get  $push[[M8:.+]]=, [[SP_2]]{{$}}
 ; CHECK:      call       somefunc@FUNCTION, $pop[[M8]]
 
-; CHECK:      get_local  $push[[M9:.+]]=, [[BP]]{{$}}
-; CHECK-NEXT: set_global __stack_pointer@GLOBAL, $pop[[M9]]
+; CHECK:      local.get  $push[[M9:.+]]=, [[BP]]{{$}}
+; CHECK-NEXT: global.set __stack_pointer@GLOBAL, $pop[[M9]]
 define void @dynamic_overalign(i32 %num) {
 entry:
   %dynamic = alloca i32, i32 %num, align 32
@@ -90,22 +90,22 @@ entry:
 }
 
 ; CHECK-LABEL: overalign_and_dynamic:
-; CHECK:      get_global $push[[L21:.+]]=, __stack_pointer@GLOBAL{{$}}
-; CHECK-NEXT: tee_local  $push[[L20:.+]]=, [[BP:.+]], $pop[[L21]]
+; CHECK:      global.get $push[[L21:.+]]=, __stack_pointer@GLOBAL{{$}}
+; CHECK-NEXT: local.tee  $push[[L20:.+]]=, [[BP:.+]], $pop[[L21]]
 ; CHECK:      i32.sub    $push[[L19:.+]]=, $pop[[L20]], $pop{{.+}}
 ; CHECK:      i32.and    $push[[L18:.+]]=, $pop[[L19]], $pop{{.+}}
-; CHECK:      tee_local  $push{{.+}}=, [[FP:.+]], $pop[[L18]]
-; CHECK:      get_local  $push[[M10:.+]]=, [[FP]]{{$}}
+; CHECK:      local.tee  $push{{.+}}=, [[FP:.+]], $pop[[L18]]
+; CHECK:      local.get  $push[[M10:.+]]=, [[FP]]{{$}}
 ; CHECK:      i32.sub    $push[[L16:.+]]=, $pop[[M10]], $pop{{.+}}
-; CHECK-NEXT: tee_local  $push{{.+}}=, [[SP:.+]], $pop[[L16]]
+; CHECK-NEXT: local.tee  $push{{.+}}=, [[SP:.+]], $pop[[L16]]
 
-; CHECK:      get_local  $push[[over:.+]]=, [[FP]]
+; CHECK:      local.get  $push[[over:.+]]=, [[FP]]
 ; CHECK-NEXT: call       somefunc@FUNCTION, $pop[[over]]
-; CHECK:      get_local  $push[[another:.+]]=, [[SP]]
+; CHECK:      local.get  $push[[another:.+]]=, [[SP]]
 ; CHECK-NEXT: call       somefunc@FUNCTION, $pop[[another]]
 
-; CHECK:      get_local  $push[[M11:.+]]=, [[BP]]{{$}}
-; CHECK-NEXT: set_global __stack_pointer@GLOBAL, $pop[[M11]]
+; CHECK:      local.get  $push[[M11:.+]]=, [[BP]]{{$}}
+; CHECK-NEXT: global.set __stack_pointer@GLOBAL, $pop[[M11]]
 define void @overalign_and_dynamic(i32 %num) {
 entry:
   %over = alloca i32, align 32
@@ -116,27 +116,27 @@ entry:
 }
 
 ; CHECK-LABEL: overalign_static_and_dynamic:
-; CHECK:      get_global $push[[L26:.+]]=, __stack_pointer@GLOBAL{{$}}
-; CHECK-NEXT: tee_local  $push[[L25:.+]]=, [[BP:.+]], $pop[[L26]]
+; CHECK:      global.get $push[[L26:.+]]=, __stack_pointer@GLOBAL{{$}}
+; CHECK-NEXT: local.tee  $push[[L25:.+]]=, [[BP:.+]], $pop[[L26]]
 ; CHECK:      i32.sub    $push[[L24:.+]]=, $pop[[L25]], $pop{{.+}}
 ; CHECK:      i32.and    $push[[L23:.+]]=, $pop[[L24]], $pop{{.+}}
-; CHECK:      tee_local  $push{{.+}}=, [[FP:.+]], $pop[[L23]]
-; CHECK:      get_local  $push[[M12:.+]]=, [[FP]]{{$}}
+; CHECK:      local.tee  $push{{.+}}=, [[FP:.+]], $pop[[L23]]
+; CHECK:      local.get  $push[[M12:.+]]=, [[FP]]{{$}}
 ; CHECK:      i32.sub    $push[[L21:.+]]=, $pop[[M12]], $pop{{.+}}
-; CHECK-NEXT: tee_local  $push{{.+}}=, [[SP:.+]], $pop[[L21]]
+; CHECK-NEXT: local.tee  $push{{.+}}=, [[SP:.+]], $pop[[L21]]
 
-; CHECK:      get_local  $push[[L19:.+]]=, [[FP]]
-; CHECK:      tee_local  $push[[L18:.+]]=, [[FP_2:.+]], $pop[[L19]]
+; CHECK:      local.get  $push[[L19:.+]]=, [[FP]]
+; CHECK:      local.tee  $push[[L18:.+]]=, [[FP_2:.+]], $pop[[L19]]
 ; CHECK:      i32.add    $push[[over:.+]]=, $pop[[L18]], $pop{{.+}}
 ; CHECK-NEXT: call       somefunc@FUNCTION, $pop[[over]]
-; CHECK:      get_local  $push[[M12:.+]]=, [[SP]]
+; CHECK:      local.get  $push[[M12:.+]]=, [[SP]]
 ; CHECK:      call       somefunc@FUNCTION, $pop[[M12]]
-; CHECK:      get_local  $push[[M13:.+]]=, [[FP_2]]
+; CHECK:      local.get  $push[[M13:.+]]=, [[FP_2]]
 ; CHECK:      i32.add    $push[[static:.+]]=, $pop[[M13]], $pop{{.+}}
 ; CHECK-NEXT: call       somefunc@FUNCTION, $pop[[static]]
 
-; CHECK:      get_local  $push[[M14:.+]]=, [[BP]]{{$}}
-; CHECK-NEXT: set_global __stack_pointer@GLOBAL, $pop[[M14]]
+; CHECK:      local.get  $push[[M14:.+]]=, [[BP]]{{$}}
+; CHECK-NEXT: global.set __stack_pointer@GLOBAL, $pop[[M14]]
 define void @overalign_static_and_dynamic(i32 %num) {
 entry:
   %over = alloca i32, align 32
diff --git a/test/CodeGen/WebAssembly/store.ll b/test/CodeGen/WebAssembly/store.ll
index 9e528bdf6b934e1bdb3286ee5e971a00bc2d0f0a..c107d9a2d0b9b6351872573cfcfb4768f9d9a9ea 100644
--- a/test/CodeGen/WebAssembly/store.ll
+++ b/test/CodeGen/WebAssembly/store.ll
@@ -8,8 +8,8 @@ target triple = "wasm32-unknown-unknown"
 
 ; CHECK-LABEL: sti32:
 ; CHECK-NEXT: .functype sti32 (i32, i32) -> (){{$}}
-; CHECK-NEXT: get_local $push[[L0:[0-9]+]]=, 0{{$}}
-; CHECK-NEXT: get_local $push[[L1:[0-9]+]]=, 1{{$}}
+; CHECK-NEXT: local.get $push[[L0:[0-9]+]]=, 0{{$}}
+; CHECK-NEXT: local.get $push[[L1:[0-9]+]]=, 1{{$}}
 ; CHECK-NEXT: i32.store 0($pop[[L0]]), $pop[[L1]]{{$}}
 ; CHECK-NEXT: return{{$}}
 define void @sti32(i32 *%p, i32 %v) {
@@ -19,8 +19,8 @@ define void @sti32(i32 *%p, i32 %v) {
 
 ; CHECK-LABEL: sti64:
 ; CHECK-NEXT: .functype sti64 (i32, i64) -> (){{$}}
-; CHECK-NEXT: get_local $push[[L0:[0-9]+]]=, 0{{$}}
-; CHECK-NEXT: get_local $push[[L1:[0-9]+]]=, 1{{$}}
+; CHECK-NEXT: local.get $push[[L0:[0-9]+]]=, 0{{$}}
+; CHECK-NEXT: local.get $push[[L1:[0-9]+]]=, 1{{$}}
 ; CHECK-NEXT: i64.store 0($pop[[L0]]), $pop[[L1]]{{$}}
 ; CHECK-NEXT: return{{$}}
 define void @sti64(i64 *%p, i64 %v) {
@@ -30,8 +30,8 @@ define void @sti64(i64 *%p, i64 %v) {
 
 ; CHECK-LABEL: stf32:
 ; CHECK-NEXT: .functype stf32 (i32, f32) -> (){{$}}
-; CHECK-NEXT: get_local $push[[L0:[0-9]+]]=, 0{{$}}
-; CHECK-NEXT: get_local $push[[L1:[0-9]+]]=, 1{{$}}
+; CHECK-NEXT: local.get $push[[L0:[0-9]+]]=, 0{{$}}
+; CHECK-NEXT: local.get $push[[L1:[0-9]+]]=, 1{{$}}
 ; CHECK-NEXT: f32.store 0($pop[[L0]]), $pop[[L1]]{{$}}
 ; CHECK-NEXT: return{{$}}
 define void @stf32(float *%p, float %v) {
@@ -41,8 +41,8 @@ define void @stf32(float *%p, float %v) {
 
 ; CHECK-LABEL: stf64:
 ; CHECK-NEXT: .functype stf64 (i32, f64) -> (){{$}}
-; CHECK-NEXT: get_local $push[[L0:[0-9]+]]=, 0{{$}}
-; CHECK-NEXT: get_local $push[[L1:[0-9]+]]=, 1{{$}}
+; CHECK-NEXT: local.get $push[[L0:[0-9]+]]=, 0{{$}}
+; CHECK-NEXT: local.get $push[[L1:[0-9]+]]=, 1{{$}}
 ; CHECK-NEXT: f64.store 0($pop[[L0]]), $pop[[L1]]{{$}}
 ; CHECK-NEXT: return{{$}}
 define void @stf64(double *%p, double %v) {
diff --git a/test/CodeGen/WebAssembly/umulo-128-legalisation-lowering.ll b/test/CodeGen/WebAssembly/umulo-128-legalisation-lowering.ll
index 3cdb704d197ecf78ca6c9ef9c12c1651cd95867a..5ec021fdb99a4e06f0f5357a206715db7fb07df8 100644
--- a/test/CodeGen/WebAssembly/umulo-128-legalisation-lowering.ll
+++ b/test/CodeGen/WebAssembly/umulo-128-legalisation-lowering.ll
@@ -3,78 +3,78 @@
 
 define { i128, i8 } @muloti_test(i128 %l, i128 %r) unnamed_addr #0 {
 ; WASM32-LABEL: muloti_test
-; WASM32: get_global      $push18=, __stack_pointer@GLOBAL
+; WASM32: global.get      $push18=, __stack_pointer@GLOBAL
 ; WASM32: i32.const       $push19=, 48
 ; WASM32: i32.sub         $push40=, $pop18, $pop19
-; WASM32: tee_local       $push39=, 5, $pop40
-; WASM32: set_global      __stack_pointer@GLOBAL, $pop39
-; WASM32: get_local       $push41=, 5
+; WASM32: local.tee       $push39=, 5, $pop40
+; WASM32: global.set      __stack_pointer@GLOBAL, $pop39
+; WASM32: local.get       $push41=, 5
 ; WASM32: i32.const       $push22=, 32
 ; WASM32: i32.add         $push23=, $pop41, $pop22
-; WASM32: get_local       $push43=, 1
+; WASM32: local.get       $push43=, 1
 ; WASM32: i64.const       $push0=, 0
-; WASM32: get_local       $push42=, 3
+; WASM32: local.get       $push42=, 3
 ; WASM32: i64.const       $push38=, 0
 ; WASM32: call            __multi3@FUNCTION, $pop23, $pop43, $pop0, $pop42, $pop38
-; WASM32: get_local       $push44=, 5
+; WASM32: local.get       $push44=, 5
 ; WASM32: i32.const       $push24=, 16
 ; WASM32: i32.add         $push25=, $pop44, $pop24
-; WASM32: get_local       $push46=, 4
+; WASM32: local.get       $push46=, 4
 ; WASM32: i64.const       $push37=, 0
-; WASM32: get_local       $push45=, 1
+; WASM32: local.get       $push45=, 1
 ; WASM32: i64.const       $push36=, 0
 ; WASM32: call            __multi3@FUNCTION, $pop25, $pop46, $pop37, $pop45, $pop36
-; WASM32: get_local       $push49=, 5
-; WASM32: get_local       $push48=, 2
+; WASM32: local.get       $push49=, 5
+; WASM32: local.get       $push48=, 2
 ; WASM32: i64.const       $push35=, 0
-; WASM32: get_local       $push47=, 3
+; WASM32: local.get       $push47=, 3
 ; WASM32: i64.const       $push34=, 0
 ; WASM32: call            __multi3@FUNCTION, $pop49, $pop48, $pop35, $pop47, $pop34
-; WASM32: get_local       $push51=, 0
-; WASM32: get_local       $push50=, 5
+; WASM32: local.get       $push51=, 0
+; WASM32: local.get       $push50=, 5
 ; WASM32: i64.load        $push1=, 32($pop50)
 ; WASM32: i64.store       0($pop51), $pop1
-; WASM32: get_local       $push55=, 0
-; WASM32: get_local       $push52=, 5
+; WASM32: local.get       $push55=, 0
+; WASM32: local.get       $push52=, 5
 ; WASM32: i32.const       $push5=, 40
 ; WASM32: i32.add         $push6=, $pop52, $pop5
 ; WASM32: i64.load        $push33=, 0($pop6)
-; WASM32: tee_local       $push32=, 1, $pop33
-; WASM32: get_local       $push53=, 5
+; WASM32: local.tee       $push32=, 1, $pop33
+; WASM32: local.get       $push53=, 5
 ; WASM32: i64.load        $push3=, 0($pop53)
-; WASM32: get_local       $push54=, 5
+; WASM32: local.get       $push54=, 5
 ; WASM32: i64.load        $push2=, 16($pop54)
 ; WASM32: i64.add         $push4=, $pop3, $pop2
 ; WASM32: i64.add         $push31=, $pop32, $pop4
-; WASM32: tee_local       $push30=, 3, $pop31
+; WASM32: local.tee       $push30=, 3, $pop31
 ; WASM32: i64.store       8($pop55), $pop30
-; WASM32: get_local       $push62=, 0
-; WASM32: get_local       $push56=, 2
+; WASM32: local.get       $push62=, 0
+; WASM32: local.get       $push56=, 2
 ; WASM32: i64.const       $push29=, 0
 ; WASM32: i64.ne          $push8=, $pop56, $pop29
-; WASM32: get_local       $push57=, 4
+; WASM32: local.get       $push57=, 4
 ; WASM32: i64.const       $push28=, 0
 ; WASM32: i64.ne          $push7=, $pop57, $pop28
 ; WASM32: i32.and         $push9=, $pop8, $pop7
-; WASM32: get_local       $push58=, 5
+; WASM32: local.get       $push58=, 5
 ; WASM32: i64.load        $push10=, 8($pop58)
 ; WASM32: i64.const       $push27=, 0
 ; WASM32: i64.ne          $push11=, $pop10, $pop27
 ; WASM32: i32.or          $push12=, $pop9, $pop11
-; WASM32: get_local       $push59=, 5
+; WASM32: local.get       $push59=, 5
 ; WASM32: i64.load        $push13=, 24($pop59)
 ; WASM32: i64.const       $push26=, 0
 ; WASM32: i64.ne          $push14=, $pop13, $pop26
 ; WASM32: i32.or          $push15=, $pop12, $pop14
-; WASM32: get_local       $push61=, 3
-; WASM32: get_local       $push60=, 1
+; WASM32: local.get       $push61=, 3
+; WASM32: local.get       $push60=, 1
 ; WASM32: i64.lt_u        $push16=, $pop61, $pop60
 ; WASM32: i32.or          $push17=, $pop15, $pop16
 ; WASM32: i32.store8      16($pop62), $pop17
-; WASM32: get_local       $push63=, 5
+; WASM32: local.get       $push63=, 5
 ; WASM32: i32.const       $push20=, 48
 ; WASM32: i32.add         $push21=, $pop63, $pop20
-; WASM32: set_global      __stack_pointer@GLOBAL, $pop21
+; WASM32: global.set      __stack_pointer@GLOBAL, $pop21
 
 start:
   %0 = tail call { i128, i1 } @llvm.umul.with.overflow.i128(i128 %l, i128 %r) #2
diff --git a/test/CodeGen/WebAssembly/userstack.ll b/test/CodeGen/WebAssembly/userstack.ll
index 9b4f28324c11d79efaef7f3c5209b226b2d993fd..f26240d324c256716d55ad9ea7595a1bb4928e5b 100644
--- a/test/CodeGen/WebAssembly/userstack.ll
+++ b/test/CodeGen/WebAssembly/userstack.ll
@@ -10,37 +10,37 @@ declare void @ext_func_i32(i32* %ptr)
 ; Check that there is an extra local for the stack pointer.
 ; CHECK: .local i32{{$}}
 define void @alloca32() noredzone {
- ; CHECK-NEXT: get_global $push[[L2:.+]]=, __stack_pointer@GLOBAL{{$}}
+ ; CHECK-NEXT: global.get $push[[L2:.+]]=, __stack_pointer@GLOBAL{{$}}
  ; CHECK-NEXT: i32.const $push[[L3:.+]]=, 16
  ; CHECK-NEXT: i32.sub $push[[L9:.+]]=, $pop[[L2]], $pop[[L3]]
- ; CHECK-NEXT: tee_local $push[[L8:.+]]=, [[SP:.+]], $pop[[L9]]{{$}}
- ; CHECK-NEXT: set_global __stack_pointer@GLOBAL, $pop[[L8]]{{$}}
+ ; CHECK-NEXT: local.tee $push[[L8:.+]]=, [[SP:.+]], $pop[[L9]]{{$}}
+ ; CHECK-NEXT: global.set __stack_pointer@GLOBAL, $pop[[L8]]{{$}}
  %retval = alloca i32
- ; CHECK: get_local $push[[L4:.+]]=, [[SP]]{{$}}
+ ; CHECK: local.get $push[[L4:.+]]=, [[SP]]{{$}}
  ; CHECK: i32.const $push[[L0:.+]]=, 0
  ; CHECK: i32.store 12($pop[[L4]]), $pop[[L0]]
  store i32 0, i32* %retval
- ; CHECK: get_local $push[[L6:.+]]=, [[SP]]{{$}}
+ ; CHECK: local.get $push[[L6:.+]]=, [[SP]]{{$}}
  ; CHECK-NEXT: i32.const $push[[L5:.+]]=, 16
  ; CHECK-NEXT: i32.add $push[[L7:.+]]=, $pop[[L6]], $pop[[L5]]
- ; CHECK-NEXT: set_global __stack_pointer@GLOBAL, $pop[[L7]]
+ ; CHECK-NEXT: global.set __stack_pointer@GLOBAL, $pop[[L7]]
  ret void
 }
 
 ; CHECK-LABEL: alloca3264:
 ; CHECK: .local i32{{$}}
 define void @alloca3264() {
- ; CHECK: get_global $push[[L3:.+]]=, __stack_pointer@GLOBAL{{$}}
+ ; CHECK: global.get $push[[L3:.+]]=, __stack_pointer@GLOBAL{{$}}
  ; CHECK-NEXT: i32.const $push[[L4:.+]]=, 16
  ; CHECK-NEXT: i32.sub $push[[L6:.+]]=, $pop[[L3]], $pop[[L4]]
- ; CHECK-NEXT: tee_local $push[[L5:.+]]=, [[SP:.+]], $pop[[L6]]
+ ; CHECK-NEXT: local.tee $push[[L5:.+]]=, [[SP:.+]], $pop[[L6]]
  %r1 = alloca i32
  %r2 = alloca double
  store i32 0, i32* %r1
  store double 0.0, double* %r2
  ; CHECK-NEXT: i64.const $push[[L1:.+]]=, 0
  ; CHECK-NEXT: i64.store 0($pop[[L5]]), $pop[[L1]]
- ; CHECK-NEXT: get_local $push[[L2:.+]]=, [[SP]]{{$}}
+ ; CHECK-NEXT: local.get $push[[L2:.+]]=, [[SP]]{{$}}
  ; CHECK-NEXT: i32.const $push[[L0:.+]]=, 0
  ; CHECK-NEXT: i32.store 12($pop[[L2]]), $pop[[L0]]
  ; CHECK-NEXT: return
@@ -50,18 +50,18 @@ define void @alloca3264() {
 ; CHECK-LABEL: allocarray:
 ; CHECK: .local i32{{$}}
 define void @allocarray() {
- ; CHECK-NEXT: get_global $push[[L4:.+]]=, __stack_pointer@GLOBAL{{$}}
+ ; CHECK-NEXT: global.get $push[[L4:.+]]=, __stack_pointer@GLOBAL{{$}}
  ; CHECK-NEXT: i32.const $push[[L5:.+]]=, 144{{$}}
  ; CHECK-NEXT: i32.sub $push[[L12:.+]]=, $pop[[L4]], $pop[[L5]]
- ; CHECK-NEXT: tee_local $push[[L11:.+]]=, 0, $pop[[L12]]
- ; CHECK-NEXT: set_global __stack_pointer@GLOBAL, $pop[[L11]]
+ ; CHECK-NEXT: local.tee $push[[L11:.+]]=, 0, $pop[[L12]]
+ ; CHECK-NEXT: global.set __stack_pointer@GLOBAL, $pop[[L11]]
  %r = alloca [33 x i32]
 
  ; CHECK:      i32.const $push{{.+}}=, 24
  ; CHECK-NEXT: i32.add $push[[L3:.+]]=, $pop{{.+}}, $pop{{.+}}
  ; CHECK-NEXT: i32.const $push[[L1:.+]]=, 1{{$}}
  ; CHECK-NEXT: i32.store 0($pop[[L3]]), $pop[[L1]]{{$}}
- ; CHECK-NEXT: get_local $push[[L4:.+]]=, 0{{$}}
+ ; CHECK-NEXT: local.get $push[[L4:.+]]=, 0{{$}}
  ; CHECK-NEXT: i32.const $push[[L10:.+]]=, 1{{$}}
  ; CHECK-NEXT: i32.store 12($pop[[L4]]), $pop[[L10]]{{$}}
  %p = getelementptr [33 x i32], [33 x i32]* %r, i32 0, i32 0
@@ -69,10 +69,10 @@ define void @allocarray() {
  %p2 = getelementptr [33 x i32], [33 x i32]* %r, i32 0, i32 3
  store i32 1, i32* %p2
 
- ; CHECK-NEXT: get_local $push[[L2:.+]]=, [[SP]]{{$}}
+ ; CHECK-NEXT: local.get $push[[L2:.+]]=, [[SP]]{{$}}
  ; CHECK-NEXT: i32.const $push[[L7:.+]]=, 144
  ; CHECK-NEXT: i32.add $push[[L8:.+]]=, $pop[[L2]], $pop[[L7]]
- ; CHECK-NEXT: set_global __stack_pointer@GLOBAL, $pop[[L8]]
+ ; CHECK-NEXT: global.set __stack_pointer@GLOBAL, $pop[[L8]]
  ret void
 }
 
@@ -80,24 +80,24 @@ define void @allocarray() {
 define void @non_mem_use(i8** %addr) {
  ; CHECK: i32.const $push[[L2:.+]]=, 48
  ; CHECK-NEXT: i32.sub $push[[L12:.+]]=, {{.+}}, $pop[[L2]]
- ; CHECK-NEXT: tee_local $push[[L11:.+]]=, [[SP:.+]], $pop[[L12]]
- ; CHECK-NEXT: set_global __stack_pointer@GLOBAL, $pop[[L11]]
+ ; CHECK-NEXT: local.tee $push[[L11:.+]]=, [[SP:.+]], $pop[[L12]]
+ ; CHECK-NEXT: global.set __stack_pointer@GLOBAL, $pop[[L11]]
  %buf = alloca [27 x i8], align 16
  %r = alloca i64
  %r2 = alloca i64
  ; %r is at SP+8
- ; CHECK: get_local $push[[L3:.+]]=, [[SP]]
+ ; CHECK: local.get $push[[L3:.+]]=, [[SP]]
  ; CHECK: i32.const $push[[OFF:.+]]=, 8
  ; CHECK-NEXT: i32.add $push[[ARG1:.+]]=, $pop[[L3]], $pop[[OFF]]
  ; CHECK-NEXT: call ext_func@FUNCTION, $pop[[ARG1]]
  call void @ext_func(i64* %r)
  ; %r2 is at SP+0, no add needed
- ; CHECK: get_local $push[[L4:.+]]=, [[SP]]
+ ; CHECK: local.get $push[[L4:.+]]=, [[SP]]
  ; CHECK-NEXT: call ext_func@FUNCTION, $pop[[L4]]
  call void @ext_func(i64* %r2)
  ; Use as a value, but in a store
  ; %buf is at SP+16
- ; CHECK: get_local $push[[L5:.+]]=, [[SP]]
+ ; CHECK: local.get $push[[L5:.+]]=, [[SP]]
  ; CHECK: i32.const $push[[OFF:.+]]=, 16
  ; CHECK-NEXT: i32.add $push[[VAL:.+]]=, $pop[[L5]], $pop[[OFF]]
  ; CHECK-NEXT: i32.store 0($pop{{.+}}), $pop[[VAL]]
@@ -109,11 +109,11 @@ define void @non_mem_use(i8** %addr) {
 ; CHECK-LABEL: allocarray_inbounds:
 ; CHECK: .local i32{{$}}
 define void @allocarray_inbounds() {
- ; CHECK: get_global $push[[L3:.+]]=, __stack_pointer@GLOBAL{{$}}
+ ; CHECK: global.get $push[[L3:.+]]=, __stack_pointer@GLOBAL{{$}}
  ; CHECK-NEXT: i32.const $push[[L4:.+]]=, 32{{$}}
  ; CHECK-NEXT: i32.sub $push[[L11:.+]]=, $pop[[L3]], $pop[[L4]]
- ; CHECK-NEXT: tee_local $push[[L10:.+]]=, [[SP:.+]], $pop[[L11]]
- ; CHECK-NEXT: set_global __stack_pointer@GLOBAL, $pop[[L10]]{{$}}
+ ; CHECK-NEXT: local.tee $push[[L10:.+]]=, [[SP:.+]], $pop[[L11]]
+ ; CHECK-NEXT: global.set __stack_pointer@GLOBAL, $pop[[L10]]{{$}}
  %r = alloca [5 x i32]
  ; CHECK: i32.const $push[[L3:.+]]=, 1
  ; CHECK-DAG: i32.store 24(${{.+}}), $pop[[L3]]
@@ -127,35 +127,35 @@ define void @allocarray_inbounds() {
  ; CHECK: call ext_func
  ; CHECK: i32.const $push[[L5:.+]]=, 32{{$}}
  ; CHECK-NEXT: i32.add $push[[L7:.+]]=, ${{.+}}, $pop[[L5]]
- ; CHECK-NEXT: set_global __stack_pointer@GLOBAL, $pop[[L7]]
+ ; CHECK-NEXT: global.set __stack_pointer@GLOBAL, $pop[[L7]]
  ret void
 }
 
 ; CHECK-LABEL: dynamic_alloca:
 define void @dynamic_alloca(i32 %alloc) {
- ; CHECK: get_global $push[[L13:.+]]=, __stack_pointer@GLOBAL{{$}}
- ; CHECK-NEXT: tee_local $push[[L12:.+]]=, [[SP:.+]], $pop[[L13]]{{$}}
+ ; CHECK: global.get $push[[L13:.+]]=, __stack_pointer@GLOBAL{{$}}
+ ; CHECK-NEXT: local.tee $push[[L12:.+]]=, [[SP:.+]], $pop[[L13]]{{$}}
  ; Target independent codegen bumps the stack pointer.
  ; CHECK: i32.sub
  ; Check that SP is written back to memory after decrement
- ; CHECK: set_global __stack_pointer@GLOBAL,
+ ; CHECK: global.set __stack_pointer@GLOBAL,
  %r = alloca i32, i32 %alloc
  ; Target-independent codegen also calculates the store addr
  ; CHECK: call ext_func_i32@FUNCTION
  call void @ext_func_i32(i32* %r)
- ; CHECK: set_global __stack_pointer@GLOBAL, $pop{{.+}}
+ ; CHECK: global.set __stack_pointer@GLOBAL, $pop{{.+}}
  ret void
 }
 
 ; CHECK-LABEL: dynamic_alloca_redzone:
 define void @dynamic_alloca_redzone(i32 %alloc) {
- ; CHECK: get_global $push[[L13:.+]]=, __stack_pointer@GLOBAL{{$}}
- ; CHECK-NEXT: tee_local $push[[L12:.+]]=, [[SP:.+]], $pop[[L13]]{{$}}
+ ; CHECK: global.get $push[[L13:.+]]=, __stack_pointer@GLOBAL{{$}}
+ ; CHECK-NEXT: local.tee $push[[L12:.+]]=, [[SP:.+]], $pop[[L13]]{{$}}
  ; Target independent codegen bumps the stack pointer
  ; CHECK: i32.sub
  %r = alloca i32, i32 %alloc
- ; CHECK-NEXT: tee_local       $push[[L8:.+]]=, {{.+}}, $pop
- ; CHECK: get_local $push[[L7:.+]]=, 0{{$}}
+ ; CHECK-NEXT: local.tee       $push[[L8:.+]]=, {{.+}}, $pop
+ ; CHECK: local.get $push[[L7:.+]]=, 0{{$}}
  ; CHECK-NEXT: i32.const       $push[[L6:.+]]=, 0{{$}}
  ; CHECK-NEXT: i32.store       0($pop[[L7]]), $pop[[L6]]{{$}}
  store i32 0, i32* %r
@@ -166,15 +166,15 @@ define void @dynamic_alloca_redzone(i32 %alloc) {
 ; CHECK-LABEL: dynamic_static_alloca:
 define void @dynamic_static_alloca(i32 %alloc) noredzone {
  ; Decrement SP in the prolog by the static amount and writeback to memory.
- ; CHECK: get_global $push[[L11:.+]]=, __stack_pointer@GLOBAL{{$}}
+ ; CHECK: global.get $push[[L11:.+]]=, __stack_pointer@GLOBAL{{$}}
  ; CHECK-NEXT: i32.const $push[[L12:.+]]=, 16
  ; CHECK-NEXT: i32.sub $push[[L23:.+]]=, $pop[[L11]], $pop[[L12]]
- ; CHECK-NEXT: tee_local $push[[L22:.+]]=, [[SP:.+]], $pop[[L23]]
- ; CHECK-NEXT: set_global __stack_pointer@GLOBAL, $pop[[L22]]
+ ; CHECK-NEXT: local.tee $push[[L22:.+]]=, [[SP:.+]], $pop[[L23]]
+ ; CHECK-NEXT: global.set __stack_pointer@GLOBAL, $pop[[L22]]
 
  ; Alloc and write to a static alloca
- ; CHECK: get_local $push[[L21:.+]]=, [[SP:.+]]
- ; CHECK-NEXT: tee_local $push[[pushedFP:.+]]=, [[FP:.+]], $pop[[L21]]
+ ; CHECK: local.get $push[[L21:.+]]=, [[SP:.+]]
+ ; CHECK-NEXT: local.tee $push[[pushedFP:.+]]=, [[FP:.+]], $pop[[L21]]
  ; CHECK-NEXT: i32.const $push[[L0:.+]]=, 101
  ; CHECK-NEXT: i32.store [[static_offset:.+]]($pop[[pushedFP]]), $pop[[L0]]
  %static = alloca i32
@@ -182,19 +182,19 @@ define void @dynamic_static_alloca(i32 %alloc) noredzone {
 
  ; Decrement SP in the body by the dynamic amount.
  ; CHECK: i32.sub
- ; CHECK: tee_local $push[[L16:.+]]=, [[dynamic_local:.+]], $pop{{.+}}
- ; CHECK: tee_local $push[[L15:.+]]=, [[other:.+]], $pop[[L16]]{{$}}
- ; CHECK: set_global __stack_pointer@GLOBAL, $pop[[L15]]{{$}}
+ ; CHECK: local.tee $push[[L16:.+]]=, [[dynamic_local:.+]], $pop{{.+}}
+ ; CHECK: local.tee $push[[L15:.+]]=, [[other:.+]], $pop[[L16]]{{$}}
+ ; CHECK: global.set __stack_pointer@GLOBAL, $pop[[L15]]{{$}}
  %dynamic = alloca i32, i32 %alloc
 
  ; Ensure we don't modify the frame pointer after assigning it.
  ; CHECK-NOT: $[[FP]]=
 
  ; Ensure the static address doesn't change after modifying the stack pointer.
- ; CHECK: get_local $push[[L17:.+]]=, [[FP]]
+ ; CHECK: local.get $push[[L17:.+]]=, [[FP]]
  ; CHECK: i32.const $push[[L7:.+]]=, 102
  ; CHECK-NEXT: i32.store [[static_offset]]($pop[[L17]]), $pop[[L7]]
- ; CHECK-NEXT: get_local $push[[L9:.+]]=, [[dynamic_local]]{{$}}
+ ; CHECK-NEXT: local.get $push[[L9:.+]]=, [[dynamic_local]]{{$}}
  ; CHECK-NEXT: i32.const $push[[L8:.+]]=, 103
  ; CHECK-NEXT: i32.store 0($pop[[L9]]), $pop[[L8]]
  store volatile i32 102, i32* %static
@@ -202,20 +202,20 @@ define void @dynamic_static_alloca(i32 %alloc) noredzone {
 
  ; Decrement SP in the body by the dynamic amount.
  ; CHECK: i32.sub
- ; CHECK: tee_local $push{{.+}}=, [[dynamic2_local:.+]], $pop{{.+}}
+ ; CHECK: local.tee $push{{.+}}=, [[dynamic2_local:.+]], $pop{{.+}}
  %dynamic.2 = alloca i32, i32 %alloc
 
  ; CHECK-NOT: $[[FP]]=
 
  ; Ensure neither the static nor dynamic address changes after the second
  ; modification of the stack pointer.
- ; CHECK: get_local $push[[L22:.+]]=, [[FP]]
+ ; CHECK: local.get $push[[L22:.+]]=, [[FP]]
  ; CHECK: i32.const $push[[L9:.+]]=, 104
  ; CHECK-NEXT: i32.store [[static_offset]]($pop[[L22]]), $pop[[L9]]
- ; CHECK-NEXT: get_local $push[[L23:.+]]=, [[dynamic_local]]
+ ; CHECK-NEXT: local.get $push[[L23:.+]]=, [[dynamic_local]]
  ; CHECK-NEXT: i32.const $push[[L10:.+]]=, 105
  ; CHECK-NEXT: i32.store 0($pop[[L23]]), $pop[[L10]]
- ; CHECK-NEXT: get_local $push[[L23:.+]]=, [[dynamic2_local]]
+ ; CHECK-NEXT: local.get $push[[L23:.+]]=, [[dynamic2_local]]
  ; CHECK-NEXT: i32.const $push[[L11:.+]]=, 106
  ; CHECK-NEXT: i32.store 0($pop[[L23]]), $pop[[L11]]
  store volatile i32 104, i32* %static
@@ -223,10 +223,10 @@ define void @dynamic_static_alloca(i32 %alloc) noredzone {
  store volatile i32 106, i32* %dynamic.2
 
  ; Writeback to memory.
- ; CHECK: get_local $push[[L24:.+]]=, [[FP]]{{$}}
+ ; CHECK: local.get $push[[L24:.+]]=, [[FP]]{{$}}
  ; CHECK: i32.const $push[[L18:.+]]=, 16
  ; CHECK-NEXT: i32.add $push[[L19:.+]]=, $pop[[L24]], $pop[[L18]]
- ; CHECK-NEXT: set_global __stack_pointer@GLOBAL, $pop[[L19]]
+ ; CHECK-NEXT: global.set __stack_pointer@GLOBAL, $pop[[L19]]
  ret void
 }
 
@@ -235,17 +235,17 @@ declare void @llvm.stackrestore(i8*)
 
 ; CHECK-LABEL: llvm_stack_builtins:
 define void @llvm_stack_builtins(i32 %alloc) noredzone {
- ; CHECK: get_global $push[[L11:.+]]=, __stack_pointer@GLOBAL{{$}}
- ; CHECK-NEXT: tee_local $push[[L10:.+]]=, {{.+}}, $pop[[L11]]
- ; CHECK-NEXT: set_local [[STACK:.+]], $pop[[L10]]
+ ; CHECK: global.get $push[[L11:.+]]=, __stack_pointer@GLOBAL{{$}}
+ ; CHECK-NEXT: local.tee $push[[L10:.+]]=, {{.+}}, $pop[[L11]]
+ ; CHECK-NEXT: local.set [[STACK:.+]], $pop[[L10]]
  %stack = call i8* @llvm.stacksave()
 
  ; Ensure we don't reassign the stacksave local
- ; CHECK-NOT: set_local [[STACK]],
+ ; CHECK-NOT: local.set [[STACK]],
  %dynamic = alloca i32, i32 %alloc
 
- ; CHECK: get_local $push[[L12:.+]]=, [[STACK]]
- ; CHECK-NEXT: set_global __stack_pointer@GLOBAL, $pop[[L12]]
+ ; CHECK: local.get $push[[L12:.+]]=, [[STACK]]
+ ; CHECK-NEXT: global.set __stack_pointer@GLOBAL, $pop[[L12]]
  call void @llvm.stackrestore(i8* %stack)
 
  ret void
@@ -256,15 +256,15 @@ define void @llvm_stack_builtins(i32 %alloc) noredzone {
 ; moved after the stack pointer was updated for the dynamic alloca.
 ; CHECK-LABEL: dynamic_alloca_nouse:
 define void @dynamic_alloca_nouse(i32 %alloc) noredzone {
- ; CHECK: get_global $push[[L11:.+]]=, __stack_pointer@GLOBAL{{$}}
- ; CHECK-NEXT: tee_local $push[[L10:.+]]=, {{.+}}, $pop[[L11]]
- ; CHECK-NEXT: set_local [[FP:.+]], $pop[[L10]]
+ ; CHECK: global.get $push[[L11:.+]]=, __stack_pointer@GLOBAL{{$}}
+ ; CHECK-NEXT: local.tee $push[[L10:.+]]=, {{.+}}, $pop[[L11]]
+ ; CHECK-NEXT: local.set [[FP:.+]], $pop[[L10]]
  %dynamic = alloca i32, i32 %alloc
 
- ; CHECK-NOT: set_local [[FP]],
+ ; CHECK-NOT: local.set [[FP]],
 
- ; CHECK: get_local $push[[L12:.+]]=, [[FP]]
- ; CHECK-NEXT: set_global __stack_pointer@GLOBAL, $pop[[L12]]
+ ; CHECK: local.get $push[[L12:.+]]=, [[FP]]
+ ; CHECK-NEXT: global.set __stack_pointer@GLOBAL, $pop[[L12]]
  ret void
 }
 
@@ -278,12 +278,12 @@ entry:
  %addr = alloca i32
  ; CHECK: i32.const $push[[OFF:.+]]=, 12
  ; CHECK-NEXT: i32.add $push[[ADDR:.+]]=, $pop[[L3]], $pop[[OFF]]
- ; CHECK-NEXT: set_local [[COPY:.+]], $pop[[ADDR]]
+ ; CHECK-NEXT: local.set [[COPY:.+]], $pop[[ADDR]]
  br label %body
 body:
  %a = phi i32* [%addr, %entry], [%b, %body]
  store i32 1, i32* %a
- ; CHECK: get_local $push[[L12:.+]]=, [[COPY]]
+ ; CHECK: local.get $push[[L12:.+]]=, [[COPY]]
  ; CHECK: i32.store 0($pop[[L12]]),
  br i1 %cond, label %body, label %exit
 exit:
@@ -295,11 +295,11 @@ declare i8* @llvm.frameaddress(i32)
 
 ; Test __builtin_frame_address(0).
 ; CHECK-LABEL: frameaddress_0:
-; CHECK: get_global $push[[L3:.+]]=, __stack_pointer@GLOBAL{{$}}
-; CHECK-NEXT: tee_local $push[[L2:.+]]=, [[FP:.+]], $pop[[L3]]{{$}}
+; CHECK: global.get $push[[L3:.+]]=, __stack_pointer@GLOBAL{{$}}
+; CHECK-NEXT: local.tee $push[[L2:.+]]=, [[FP:.+]], $pop[[L3]]{{$}}
 ; CHECK-NEXT: call use_i8_star@FUNCTION, $pop[[L2]]
-; CHECK-NEXT: get_local $push[[L5:.+]]=, [[FP]]
-; CHECK-NEXT: set_global __stack_pointer@GLOBAL, $pop[[L5]]
+; CHECK-NEXT: local.get $push[[L5:.+]]=, [[FP]]
+; CHECK-NEXT: global.set __stack_pointer@GLOBAL, $pop[[L5]]
 define void @frameaddress_0() {
   %t = call i8* @llvm.frameaddress(i32 0)
   call void @use_i8_star(i8* %t)
@@ -320,7 +320,7 @@ define void @frameaddress_1() {
 
 ; Test a stack address passed to an inline asm.
 ; CHECK-LABEL: inline_asm:
-; CHECK:       get_global {{.+}}, __stack_pointer@GLOBAL{{$}}
+; CHECK:       global.get {{.+}}, __stack_pointer@GLOBAL{{$}}
 ; CHECK:       #APP
 ; CHECK-NEXT:  # %{{[0-9]+}}{{$}}
 ; CHECK-NEXT:  #NO_APP
diff --git a/test/CodeGen/WebAssembly/varargs.ll b/test/CodeGen/WebAssembly/varargs.ll
index 0ea29762f11c5b19f1206ff917c2ddcf1af3ec0c..1a73716c2a67cb6cde14cac78cc25b4ca3d971e0 100644
--- a/test/CodeGen/WebAssembly/varargs.ll
+++ b/test/CodeGen/WebAssembly/varargs.ll
@@ -52,7 +52,7 @@ entry:
 ; CHECK-LABEL: arg_i8:
 ; CHECK-NEXT: .functype arg_i8 (i32) -> (i32){{$}}
 ; CHECK-NEXT: i32.load   $push[[NUM0:[0-9]+]]=, 0($0){{$}}
-; CHECK-NEXT: tee_local  $push[[NUM1:[0-9]+]]=, $1=, $pop[[NUM0]]{{$}}
+; CHECK-NEXT: local.tee  $push[[NUM1:[0-9]+]]=, $1=, $pop[[NUM0]]{{$}}
 ; CHECK-NEXT: i32.const  $push[[NUM2:[0-9]+]]=, 4{{$}}
 ; CHECK-NEXT: i32.add    $push[[NUM3:[0-9]+]]=, $pop[[NUM1]], $pop[[NUM2]]{{$}}
 ; CHECK-NEXT: i32.store  0($0), $pop[[NUM3]]{{$}}
@@ -73,7 +73,7 @@ entry:
 ; CHECK-NEXT: i32.add    $push[[NUM2:[0-9]+]]=, $pop[[NUM0]], $pop[[NUM1]]{{$}}
 ; CHECK-NEXT: i32.const  $push[[NUM3:[0-9]+]]=, -4{{$}}
 ; CHECK-NEXT: i32.and    $push[[NUM4:[0-9]+]]=, $pop[[NUM2]], $pop[[NUM3]]{{$}}
-; CHECK-NEXT: tee_local  $push[[NUM5:[0-9]+]]=, $1=, $pop[[NUM4]]{{$}}
+; CHECK-NEXT: local.tee  $push[[NUM5:[0-9]+]]=, $1=, $pop[[NUM4]]{{$}}
 ; CHECK-NEXT: i32.const  $push[[NUM6:[0-9]+]]=, 4{{$}}
 ; CHECK-NEXT: i32.add    $push[[NUM7:[0-9]+]]=, $pop[[NUM5]], $pop[[NUM6]]{{$}}
 ; CHECK-NEXT: i32.store  0($0), $pop[[NUM7]]{{$}}
diff --git a/test/CodeGen/WebAssembly/wasmehprepare.ll b/test/CodeGen/WebAssembly/wasmehprepare.ll
index 6df7175b1ca7bd47709c9f47dc72bb7c5190a2da..09013a0a75414ca68fe1a9806a7277c407c08cd2 100644
--- a/test/CodeGen/WebAssembly/wasmehprepare.ll
+++ b/test/CodeGen/WebAssembly/wasmehprepare.ll
@@ -29,7 +29,7 @@ catch.start:                                      ; preds = %catch.dispatch
   br i1 %matches, label %catch, label %rethrow
 ; CHECK: catch.start:
 ; CHECK-NEXT:   %[[CATCHPAD:.*]] = catchpad
-; CHECK-NEXT:   %[[EXN:.*]] = call i8* @llvm.wasm.catch(i32 0)
+; CHECK-NEXT:   %[[EXN:.*]] = call i8* @llvm.wasm.extract.exception()
 ; CHECK-NEXT:   call void @llvm.wasm.landingpad.index(token %[[CATCHPAD]], i32 0)
 ; CHECK-NEXT:   store i32 0, i32* getelementptr inbounds ({ i32, i8*, i32 }, { i32, i8*, i32 }* @__wasm_lpad_context, i32 0, i32 0)
 ; CHECK-NEXT:   %[[LSDA:.*]] = call i8* @llvm.wasm.lsda()
@@ -76,7 +76,6 @@ catch.start:                                      ; preds = %catch.dispatch
   catchret from %1 to label %try.cont
 ; CHECK: catch.start:
 ; CHECK-NEXT:   catchpad within %0 [i8* null]
-; CHECK-NEXT:   call i8* @llvm.wasm.catch(i32 0)
 ; CHECK-NOT:   call void @llvm.wasm.landingpad.index
 ; CHECK-NOT:   store {{.*}} @__wasm_lpad_context
 ; CHECK-NOT:   call i8* @llvm.wasm.lsda()
@@ -178,7 +177,6 @@ ehcleanup:                                        ; preds = %rethrow5, %catch.di
   cleanupret from %12 unwind to caller
 ; CHECK: ehcleanup:
 ; CHECK-NEXT:   cleanuppad
-; CHECK-NOT:   call i8* @llvm.wasm.catch(i32 0)
 ; CHECK-NOT:   call void @llvm.wasm.landingpad.index
 ; CHECK-NOT:   store {{.*}} @__wasm_lpad_context
 ; CHECK-NOT:   call i8* @llvm.wasm.lsda()
@@ -191,7 +189,7 @@ unreachable:                                      ; preds = %rethrow5
 
 ; A cleanuppad with a call to __clang_call_terminate().
 ; A call to wasm.catch() should be generated after the cleanuppad.
-define hidden void @test3() personality i8* bitcast (i32 (...)* @__gxx_wasm_personality_v0 to i8*) {
+define void @test3() personality i8* bitcast (i32 (...)* @__gxx_wasm_personality_v0 to i8*) {
 ; CHECK-LABEL: @test3
 entry:
   invoke void @foo()
@@ -230,14 +228,14 @@ terminate:                                        ; preds = %ehcleanup
   unreachable
 ; CHECK: terminate:
 ; CHECK-NEXT: cleanuppad
-; CHECK-NEXT:   %[[EXN:.*]] = call i8* @llvm.wasm.catch(i32 0)
+; CHECK-NEXT:   %[[EXN:.*]] = call i8* @llvm.wasm.extract.exception
 ; CHECK-NEXT:   call void @__clang_call_terminate(i8* %[[EXN]])
 }
 
 ; PHI demotion test. Only the phi before catchswitch should be demoted; the phi
 ; before cleanuppad should NOT.
-define void @test5() personality i8* bitcast (i32 (...)* @__gxx_wasm_personality_v0 to i8*) {
-; CHECK-LABEL: @test5
+define void @test4() personality i8* bitcast (i32 (...)* @__gxx_wasm_personality_v0 to i8*) {
+; CHECK-LABEL: @test4
 entry:
   %c = alloca %struct.Cleanup, align 1
   invoke void @foo()
@@ -301,8 +299,8 @@ try.cont10:                                       ; preds = %invoke.cont3, %catc
 ; Tests if instructions after a call to @llvm.wasm.throw are deleted and the
 ; BB's dead children are deleted.
 
-; CHECK-LABEL: @test6
-define i32 @test6(i1 %b, i8* %p) {
+; CHECK-LABEL: @test5
+define i32 @test5(i1 %b, i8* %p) {
 entry:
   br i1 %b, label %bb.true, label %bb.false
 
@@ -326,6 +324,34 @@ merge:                                            ; preds = %bb.true.0, %bb.fals
   ret i32 0
 }
 
+; Tests if instructions after a call to @llvm.wasm.rethrow are deleted and the
+; BB's dead children are deleted.
+
+; CHECK-LABEL: @test6
+define i32 @test6(i1 %b, i8* %p) {
+entry:
+  br i1 %b, label %bb.true, label %bb.false
+
+; CHECK:      bb.true:
+; CHECK-NEXT:   call void @llvm.wasm.rethrow()
+; CHECK-NEXT:   unreachable
+bb.true:                                          ; preds = %entry
+  call void @llvm.wasm.rethrow()
+  br label %bb.true.0
+
+; CHECK-NOT:  bb.true.0
+bb.true.0:                                        ; preds = %bb.true
+  br label %merge
+
+; CHECK:      bb.false
+bb.false:                                         ; preds = %entry
+  br label %merge
+
+; CHECK:      merge
+merge:                                            ; preds = %bb.true.0, %bb.false
+  ret i32 0
+}
+
 declare void @foo()
 declare void @func(i32)
 declare %struct.Cleanup* @_ZN7CleanupD1Ev(%struct.Cleanup* returned)
@@ -334,12 +360,12 @@ declare i8* @llvm.wasm.get.exception(token)
 declare i32 @llvm.wasm.get.ehselector(token)
 declare i32 @llvm.eh.typeid.for(i8*)
 declare void @llvm.wasm.throw(i32, i8*)
+declare void @llvm.wasm.rethrow()
 declare i8* @__cxa_begin_catch(i8*)
 declare void @__cxa_end_catch()
 declare void @__cxa_rethrow()
 declare void @__clang_call_terminate(i8*)
 
-; CHECK-DAG: declare i8* @llvm.wasm.catch(i32)
 ; CHECK-DAG: declare void @llvm.wasm.landingpad.index(token, i32)
 ; CHECK-DAG: declare i8* @llvm.wasm.lsda()
 ; CHECK-DAG: declare i32 @_Unwind_CallPersonality(i8*)
diff --git a/test/CodeGen/WinEH/wineh-statenumbering.ll b/test/CodeGen/WinEH/wineh-statenumbering.ll
index d5c330bdf4ee71c1bd8476252acf38fcaabbcbb7..4f0b553b11136dc83cd30c8db3767b90924b7a5d 100644
--- a/test/CodeGen/WinEH/wineh-statenumbering.ll
+++ b/test/CodeGen/WinEH/wineh-statenumbering.ll
@@ -180,7 +180,7 @@ if.end:                                           ; preds = %if.then, %__except.
 define internal i32 @"\01?filt$0@0@required_state_store@@"() {
 entry:
   %0 = tail call i8* @llvm.frameaddress(i32 1)
-  %1 = tail call i8* @llvm.x86.seh.recoverfp(i8* bitcast (void (i1)* @required_state_store to i8*), i8* %0)
+  %1 = tail call i8* @llvm.eh.recoverfp(i8* bitcast (void (i1)* @required_state_store to i8*), i8* %0)
   %2 = tail call i8* @llvm.localrecover(i8* bitcast (void (i1)* @required_state_store to i8*), i8* %1, i32 0)
   %__exception_code = bitcast i8* %2 to i32*
   %3 = getelementptr inbounds i8, i8* %0, i32 -20
@@ -203,7 +203,7 @@ declare i32 @__CxxFrameHandler3(...)
 
 declare i8* @llvm.frameaddress(i32)
 
-declare i8* @llvm.x86.seh.recoverfp(i8*, i8*)
+declare i8* @llvm.eh.recoverfp(i8*, i8*)
 
 declare i8* @llvm.localrecover(i8*, i8*, i32)
 
diff --git a/test/CodeGen/X86/2007-04-17-LiveIntervalAssert.ll b/test/CodeGen/X86/2007-04-17-LiveIntervalAssert.ll
index 31c6b532d8c7cffd345724b6cb4e661637e1798a..3997cfba7f3e146cf4af190e305555c0c6b3e1e0 100644
--- a/test/CodeGen/X86/2007-04-17-LiveIntervalAssert.ll
+++ b/test/CodeGen/X86/2007-04-17-LiveIntervalAssert.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -mtriple=i686-apple-darwin -relocation-model=pic --disable-fp-elim
+; RUN: llc < %s -mtriple=i686-apple-darwin -relocation-model=pic --frame-pointer=all
 
 	%struct.FILE = type { i8*, i32, i32, i16, i16, %struct.__sbuf, i32, i8*, i32 (i8*)*, i32 (i8*, i8*, i32)*, i64 (i8*, i64, i32)*, i32 (i8*, i8*, i32)*, %struct.__sbuf, %struct.__sFILEX*, i32, [3 x i8], [1 x i8], %struct.__sbuf, i32, i64 }
 	%struct.__sFILEX = type opaque
diff --git a/test/CodeGen/X86/2008-03-10-RegAllocInfLoop.ll b/test/CodeGen/X86/2008-03-10-RegAllocInfLoop.ll
index 9fb325c1223b21b70f7bef97713f2a0c6a41f5b5..00423d6d6259f74a0fc1d936749fb2f6c9494864 100644
--- a/test/CodeGen/X86/2008-03-10-RegAllocInfLoop.ll
+++ b/test/CodeGen/X86/2008-03-10-RegAllocInfLoop.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -mtriple=i386-pc-linux-gnu -relocation-model=pic -disable-fp-elim
+; RUN: llc < %s -mtriple=i386-pc-linux-gnu -relocation-model=pic -frame-pointer=all
 ; PR2134
 
 declare fastcc i8* @w_addchar(i8*, i32*, i32*, i8 signext ) nounwind 
diff --git a/test/CodeGen/X86/2008-03-31-SpillerFoldingBug.ll b/test/CodeGen/X86/2008-03-31-SpillerFoldingBug.ll
index 3cc3b83a34055b37e6557b1b64d85ce194e7fdd1..fd0b3e75155c0f41c38f98c3c5f3bf7f4c07c327 100644
--- a/test/CodeGen/X86/2008-03-31-SpillerFoldingBug.ll
+++ b/test/CodeGen/X86/2008-03-31-SpillerFoldingBug.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -mtriple=i386-apple-darwin -relocation-model=pic -disable-fp-elim | grep add | grep 12 | not grep non_lazy_ptr
+; RUN: llc < %s -mtriple=i386-apple-darwin -relocation-model=pic -frame-pointer=all | grep add | grep 12 | not grep non_lazy_ptr
 ; Don't fold re-materialized load into a two address instruction
 
 	%"struct.Smarts::Runnable" = type { i32 (...)**, i32 }
diff --git a/test/CodeGen/X86/2008-04-15-LiveVariableBug.ll b/test/CodeGen/X86/2008-04-15-LiveVariableBug.ll
index b52659134c1bf3017bc518788c452af60cbaf899..175f1d98ff3d52a7373883d0abfdb18c51ffc7e8 100644
--- a/test/CodeGen/X86/2008-04-15-LiveVariableBug.ll
+++ b/test/CodeGen/X86/2008-04-15-LiveVariableBug.ll
@@ -1,5 +1,5 @@
 ; RUN: llc < %s -mtriple=x86_64-apple-darwin
-; RUN: llc < %s -mtriple=x86_64-apple-darwin -relocation-model=pic -disable-fp-elim -O0 -regalloc=fast
+; RUN: llc < %s -mtriple=x86_64-apple-darwin -relocation-model=pic -frame-pointer=all -O0 -regalloc=fast
 ; PR5534
 
 	%struct.CGPoint = type { double, double }
diff --git a/test/CodeGen/X86/2008-09-29-ReMatBug.ll b/test/CodeGen/X86/2008-09-29-ReMatBug.ll
index cc481a056c84820f4f8d2d52ec128b9ba1544f23..90f3ffd2a5f463c3cee749a297fe1527c79daf70 100644
--- a/test/CodeGen/X86/2008-09-29-ReMatBug.ll
+++ b/test/CodeGen/X86/2008-09-29-ReMatBug.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -mtriple=i386-apple-darwin -relocation-model=pic -disable-fp-elim
+; RUN: llc < %s -mtriple=i386-apple-darwin -relocation-model=pic -frame-pointer=all
 
 	%struct..0objc_selector = type opaque
 	%struct.NSString = type opaque
diff --git a/test/CodeGen/X86/2009-02-26-MachineLICMBug.ll b/test/CodeGen/X86/2009-02-26-MachineLICMBug.ll
index 25a7d057f66a805dd39ccb18965659e63dc7c68b..1e6e24c680051434264b04e280571a316963610b 100644
--- a/test/CodeGen/X86/2009-02-26-MachineLICMBug.ll
+++ b/test/CodeGen/X86/2009-02-26-MachineLICMBug.ll
@@ -20,22 +20,7 @@ define %struct.__vv* @t(%struct.Key* %desc, i64 %p) nounwind ssp {
 ; CHECK-NEXT:    movq %rdi, %rbx
 ; CHECK-NEXT:    orq $2097152, %r14 ## imm = 0x200000
 ; CHECK-NEXT:    andl $15728640, %r14d ## imm = 0xF00000
-; CHECK-NEXT:    jmp LBB0_1
 ; CHECK-NEXT:    .p2align 4, 0x90
-; CHECK-NEXT:  LBB0_3: ## %bb.i
-; CHECK-NEXT:    ## in Loop: Header=BB0_1 Depth=1
-; CHECK-NEXT:    movl 0, %eax
-; CHECK-NEXT:    xorps %xmm0, %xmm0
-; CHECK-NEXT:    cvtsi2ssq %rax, %xmm0
-; CHECK-NEXT:    movl 4, %eax
-; CHECK-NEXT:    xorps %xmm1, %xmm1
-; CHECK-NEXT:    cvtsi2ssq %rax, %xmm1
-; CHECK-NEXT:    movl 8, %eax
-; CHECK-NEXT:    xorps %xmm2, %xmm2
-; CHECK-NEXT:    cvtsi2ssq %rax, %xmm2
-; CHECK-NEXT:    insertps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[2,3]
-; CHECK-NEXT:    insertps {{.*#+}} xmm0 = xmm0[0,1],xmm2[0],xmm0[3]
-; CHECK-NEXT:    movaps %xmm0, 0
 ; CHECK-NEXT:  LBB0_1: ## %bb4
 ; CHECK-NEXT:    ## =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    xorl %eax, %eax
@@ -50,7 +35,21 @@ define %struct.__vv* @t(%struct.Key* %desc, i64 %p) nounwind ssp {
 ; CHECK-NEXT:    ## in Loop: Header=BB0_1 Depth=1
 ; CHECK-NEXT:    cmpq $1048576, %r14 ## imm = 0x100000
 ; CHECK-NEXT:    jne LBB0_1
-; CHECK-NEXT:    jmp LBB0_3
+; CHECK-NEXT:  ## %bb.3: ## %bb.i
+; CHECK-NEXT:    ## in Loop: Header=BB0_1 Depth=1
+; CHECK-NEXT:    movl 0, %eax
+; CHECK-NEXT:    xorps %xmm0, %xmm0
+; CHECK-NEXT:    cvtsi2ssq %rax, %xmm0
+; CHECK-NEXT:    movl 4, %eax
+; CHECK-NEXT:    xorps %xmm1, %xmm1
+; CHECK-NEXT:    cvtsi2ssq %rax, %xmm1
+; CHECK-NEXT:    movl 8, %eax
+; CHECK-NEXT:    xorps %xmm2, %xmm2
+; CHECK-NEXT:    cvtsi2ssq %rax, %xmm2
+; CHECK-NEXT:    insertps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[2,3]
+; CHECK-NEXT:    insertps {{.*#+}} xmm0 = xmm0[0,1],xmm2[0],xmm0[3]
+; CHECK-NEXT:    movaps %xmm0, 0
+; CHECK-NEXT:    jmp LBB0_1
 entry:
 	br label %bb4
 
diff --git a/test/CodeGen/X86/2009-04-16-SpillerUnfold.ll b/test/CodeGen/X86/2009-04-16-SpillerUnfold.ll
index 1d03a1b20a34b24caeefa819f6d2c762a9cc2b48..2d198032d5a26db96ac4a4d6b456daad9ffa52e2 100644
--- a/test/CodeGen/X86/2009-04-16-SpillerUnfold.ll
+++ b/test/CodeGen/X86/2009-04-16-SpillerUnfold.ll
@@ -1,5 +1,5 @@
 ; REQUIRES: asserts
-; RUN: llc < %s -mtriple=x86_64-apple-darwin10.0 -relocation-model=pic -disable-fp-elim -stats 2>&1 | FileCheck %s
+; RUN: llc < %s -mtriple=x86_64-apple-darwin10.0 -relocation-model=pic -frame-pointer=all -stats 2>&1 | FileCheck %s
 ; XFAIL: *
 ; 69408 removed the opportunity for this optimization to work
 
diff --git a/test/CodeGen/X86/2009-04-29-RegAllocAssert.ll b/test/CodeGen/X86/2009-04-29-RegAllocAssert.ll
index c291fede98ebf02137cf1a0a17bb63a78878b81a..80d4b0b79e810b4e136b60df849d815ca9379696 100644
--- a/test/CodeGen/X86/2009-04-29-RegAllocAssert.ll
+++ b/test/CodeGen/X86/2009-04-29-RegAllocAssert.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -mtriple=x86_64-apple-darwin10 -disable-fp-elim -relocation-model=pic
+; RUN: llc < %s -mtriple=x86_64-apple-darwin10 -frame-pointer=all -relocation-model=pic
 ; PR4099
 
 	%0 = type { [62 x %struct.Bitvec*] }		; type %0
diff --git a/test/CodeGen/X86/2009-06-02-RewriterBug.ll b/test/CodeGen/X86/2009-06-02-RewriterBug.ll
index 6ce7af632ee15eb1627b03d3a208236cf36aab00..f08533846fcb3555f74aabb0438991af4603d16d 100644
--- a/test/CodeGen/X86/2009-06-02-RewriterBug.ll
+++ b/test/CodeGen/X86/2009-06-02-RewriterBug.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -mtriple=x86_64-undermydesk-freebsd8.0 -relocation-model=pic -disable-fp-elim
+; RUN: llc < %s -mtriple=x86_64-undermydesk-freebsd8.0 -relocation-model=pic -frame-pointer=all
 ; PR4225
 
 define void @sha256_block1(i32* nocapture %arr, i8* nocapture %in, i64 %num) nounwind {
diff --git a/test/CodeGen/X86/2009-09-10-LoadFoldingBug.ll b/test/CodeGen/X86/2009-09-10-LoadFoldingBug.ll
index aa88576c148e9f3aa2949ef1e191766ed7049181..7d1a30003f9fe2fc29c66ec8396d8630282f294d 100644
--- a/test/CodeGen/X86/2009-09-10-LoadFoldingBug.ll
+++ b/test/CodeGen/X86/2009-09-10-LoadFoldingBug.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -mtriple=x86_64-apple-darwin10.0 -relocation-model=pic -disable-fp-elim | FileCheck %s
+; RUN: llc < %s -mtriple=x86_64-apple-darwin10.0 -relocation-model=pic -frame-pointer=all | FileCheck %s
 
 ; It's not legal to fold a load from 32-bit stack slot into a 64-bit
 ; instruction. If done, the instruction does a 64-bit load and that's not
diff --git a/test/CodeGen/X86/2009-10-19-EmergencySpill.ll b/test/CodeGen/X86/2009-10-19-EmergencySpill.ll
index ec73f5a2c82313c9212d4c8159dca77c03966e1a..2b0ecc37a002fa2d9b7bd8d4917cbfc3cb2cb929 100644
--- a/test/CodeGen/X86/2009-10-19-EmergencySpill.ll
+++ b/test/CodeGen/X86/2009-10-19-EmergencySpill.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -mtriple=x86_64-apple-darwin10 -disable-fp-elim
+; RUN: llc < %s -mtriple=x86_64-apple-darwin10 -frame-pointer=all
 ; rdar://7291624
 
 %union.RtreeCoord = type { float }
diff --git a/test/CodeGen/X86/2009-10-19-atomic-cmp-eflags.ll b/test/CodeGen/X86/2009-10-19-atomic-cmp-eflags.ll
index 157bbfff6e00720a75ef57fb9b9c2d4565d30fc2..bca1c0360dc0ab02be401b95a00087cbd09e13fd 100644
--- a/test/CodeGen/X86/2009-10-19-atomic-cmp-eflags.ll
+++ b/test/CodeGen/X86/2009-10-19-atomic-cmp-eflags.ll
@@ -1,4 +1,4 @@
-; RUN: llvm-as <%s | llc | FileCheck %s
+; RUN: llvm-as < %s | llc | FileCheck %s
 ; PR 5247
 ; check that cmp/test is not scheduled before the add
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128"
diff --git a/test/CodeGen/X86/2009-10-25-RewriterBug.ll b/test/CodeGen/X86/2009-10-25-RewriterBug.ll
index be18186463d48ca0dd875b97c66338ab7c5fe0d5..ad78f960056216237c5920babfe8f52e897d6372 100644
--- a/test/CodeGen/X86/2009-10-25-RewriterBug.ll
+++ b/test/CodeGen/X86/2009-10-25-RewriterBug.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -mtriple=x86_64-apple-darwin -relocation-model=pic -disable-fp-elim
+; RUN: llc < %s -mtriple=x86_64-apple-darwin -relocation-model=pic -frame-pointer=all
 
 %struct.DecRefPicMarking_t = type { i32, i32, i32, i32, i32, %struct.DecRefPicMarking_t* }
 %struct.FrameStore = type { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, %struct.StorablePicture*, %struct.StorablePicture*, %struct.StorablePicture* }
diff --git a/test/CodeGen/X86/2009-11-13-VirtRegRewriterBug.ll b/test/CodeGen/X86/2009-11-13-VirtRegRewriterBug.ll
index 5398eef3dd1733bb593140bcd8f7bc1bc95ccd81..3608e09a22b84ff732742675fe38bf394cc7864f 100644
--- a/test/CodeGen/X86/2009-11-13-VirtRegRewriterBug.ll
+++ b/test/CodeGen/X86/2009-11-13-VirtRegRewriterBug.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -mtriple=i386-apple-darwin -relocation-model=pic -disable-fp-elim
+; RUN: llc < %s -mtriple=i386-apple-darwin -relocation-model=pic -frame-pointer=all
 ; rdar://7394770
 
 %struct.JVTLib_100487 = type <{ i8 }>
diff --git a/test/CodeGen/X86/2010-01-19-OptExtBug.ll b/test/CodeGen/X86/2010-01-19-OptExtBug.ll
index def8dd31978bc13ac1daf9dd2cc77b5611273522..a3e3d3dbadd434d10952ba3a4d996070d100c21d 100644
--- a/test/CodeGen/X86/2010-01-19-OptExtBug.ll
+++ b/test/CodeGen/X86/2010-01-19-OptExtBug.ll
@@ -1,5 +1,5 @@
 ; REQUIRES: asserts
-; RUN: llc < %s -mtriple=x86_64-apple-darwin11 -relocation-model=pic -disable-fp-elim -stats 2>&1 | not grep ext-opt
+; RUN: llc < %s -mtriple=x86_64-apple-darwin11 -relocation-model=pic -frame-pointer=all -stats 2>&1 | not grep ext-opt
 
 define fastcc i8* @S_scan_str(i8* %start, i32 %keep_quoted, i32 %keep_delims) nounwind ssp {
 entry:
diff --git a/test/CodeGen/X86/2010-04-06-SSEDomainFixCrash.ll b/test/CodeGen/X86/2010-04-06-SSEDomainFixCrash.ll
index 41c318b62eab73c1ab73d8be553b58f9e3d1ac80..1f2fe320cc601a5f06f9c307f50210f25c2a84b7 100644
--- a/test/CodeGen/X86/2010-04-06-SSEDomainFixCrash.ll
+++ b/test/CodeGen/X86/2010-04-06-SSEDomainFixCrash.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -O3 -relocation-model=pic -disable-fp-elim -mcpu=nocona
+; RUN: llc < %s -O3 -relocation-model=pic -frame-pointer=all -mcpu=nocona
 ;
 ; This test case is reduced from Bullet. It crashes SSEDomainFix.
 ;
diff --git a/test/CodeGen/X86/2010-04-29-CoalescerCrash.ll b/test/CodeGen/X86/2010-04-29-CoalescerCrash.ll
index a22f38ae3baaa8bab11612ed46559d185745a0ce..7b34489537bb76f714f809905d6fb18506fc0f26 100644
--- a/test/CodeGen/X86/2010-04-29-CoalescerCrash.ll
+++ b/test/CodeGen/X86/2010-04-29-CoalescerCrash.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -relocation-model=pic -disable-fp-elim -verify-machineinstrs
+; RUN: llc < %s -relocation-model=pic -frame-pointer=all -verify-machineinstrs
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 target triple = "x86_64-unknown-linux-gnu"
 
diff --git a/test/CodeGen/X86/2010-04-30-LocalAlloc-LandingPad.ll b/test/CodeGen/X86/2010-04-30-LocalAlloc-LandingPad.ll
index 24abb719b0f960349b7b53a05ce465a0a66b3749..615a57274393a8667f8210be9a565921e819eb83 100644
--- a/test/CodeGen/X86/2010-04-30-LocalAlloc-LandingPad.ll
+++ b/test/CodeGen/X86/2010-04-30-LocalAlloc-LandingPad.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -O0 -regalloc=fast -relocation-model=pic -disable-fp-elim | FileCheck %s
+; RUN: llc < %s -O0 -regalloc=fast -relocation-model=pic -frame-pointer=all | FileCheck %s
 target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128-n8:16:32"
 target triple = "i386-apple-darwin10.0.0"
 
diff --git a/test/CodeGen/X86/2010-06-09-FastAllocRegisters.ll b/test/CodeGen/X86/2010-06-09-FastAllocRegisters.ll
index 7c7792ac65a0d98eed59ff99d68774c1d82adf0c..1557fcb509b6146b35a67218e07cac6edfbee8b3 100644
--- a/test/CodeGen/X86/2010-06-09-FastAllocRegisters.ll
+++ b/test/CodeGen/X86/2010-06-09-FastAllocRegisters.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -O0 -disable-fp-elim -relocation-model=pic
+; RUN: llc < %s -O0 -frame-pointer=all -relocation-model=pic
 ; PR7313
 ;
 ; The inline asm in this function clobbers almost all allocatable registers.
diff --git a/test/CodeGen/X86/2010-06-24-g-constraint-crash.ll b/test/CodeGen/X86/2010-06-24-g-constraint-crash.ll
index 905b34ff6f5cd1809e2353a025fa13c5c568eb87..91e6d18c951209daac750f7efd5a820ae221f67c 100644
--- a/test/CodeGen/X86/2010-06-24-g-constraint-crash.ll
+++ b/test/CodeGen/X86/2010-06-24-g-constraint-crash.ll
@@ -1,4 +1,4 @@
-; RUN: llc %s -mtriple=x86_64-apple-darwin10 -disable-fp-elim -o /dev/null
+; RUN: llc %s -mtriple=x86_64-apple-darwin10 -frame-pointer=all -o /dev/null
 ; Formerly crashed, rdar://8015842
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
diff --git a/test/CodeGen/X86/2010-06-25-CoalescerSubRegDefDead.ll b/test/CodeGen/X86/2010-06-25-CoalescerSubRegDefDead.ll
index dd7c3fa571cce0976efda3354931c5697fcc5c99..19e3d71324a246fe16970b22e51154b5ae70d476 100644
--- a/test/CodeGen/X86/2010-06-25-CoalescerSubRegDefDead.ll
+++ b/test/CodeGen/X86/2010-06-25-CoalescerSubRegDefDead.ll
@@ -1,4 +1,4 @@
-; RUN: llc -O1 -mtriple=x86_64-unknown-linux-gnu -mcpu=core2 -relocation-model=pic -disable-fp-elim < %s | FileCheck %s
+; RUN: llc -O1 -mtriple=x86_64-unknown-linux-gnu -mcpu=core2 -relocation-model=pic -frame-pointer=all < %s | FileCheck %s
 ; <rdar://problem/8124405>
 
 %struct.type = type { %struct.subtype*, i32, i8, i32, i8, i32, i32, i32, i32, i32, i8, i32, i32, i32, i32, i32, [256 x i32], i32, [257 x i32], [257 x i32], i32*, i16*, i8*, i32, i32, i32, i32, i32, [256 x i8], [16 x i8], [256 x i8], [4096 x i8], [16 x i32], [18002 x i8], [18002 x i8], [6 x [258 x i8]], [6 x [258 x i32]], [6 x [258 x i32]], [6 x [258 x i32]], [6 x i32], i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32*, i32*, i32* }
diff --git a/test/CodeGen/X86/2010-06-25-asm-RA-crash.ll b/test/CodeGen/X86/2010-06-25-asm-RA-crash.ll
index 8f5f083209b4bfb9970f69a4885c071a45100f63..44f6c73329de9fe3b7502739360595c06fc699dc 100644
--- a/test/CodeGen/X86/2010-06-25-asm-RA-crash.ll
+++ b/test/CodeGen/X86/2010-06-25-asm-RA-crash.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -disable-fp-elim -mtriple=i686-pc-mingw32 -no-integrated-as
+; RUN: llc < %s -frame-pointer=all -mtriple=i686-pc-mingw32 -no-integrated-as
 
 %struct.__SEH2Frame = type {}
 
diff --git a/test/CodeGen/X86/2011-06-12-FastAllocSpill.ll b/test/CodeGen/X86/2011-06-12-FastAllocSpill.ll
index 1285d20b8522c1cf3332313d681de7e82e018104..c438335ca54d44b5be8303efef0b3bc426578766 100644
--- a/test/CodeGen/X86/2011-06-12-FastAllocSpill.ll
+++ b/test/CodeGen/X86/2011-06-12-FastAllocSpill.ll
@@ -1,5 +1,5 @@
 ; REQUIRES: asserts
-; RUN: llc < %s -O0 -disable-fp-elim -relocation-model=pic -stats 2>&1 | FileCheck %s
+; RUN: llc < %s -O0 -frame-pointer=all -relocation-model=pic -stats 2>&1 | FileCheck %s
 ;
 ; This test should not cause any spilling with RAFast.
 ;
diff --git a/test/CodeGen/X86/2011-07-13-BadFrameIndexDisplacement.ll b/test/CodeGen/X86/2011-07-13-BadFrameIndexDisplacement.ll
index f1b1a7077bd68ee0167d8662bd95df784c4e1fe5..93aabb92d47a81d51635f27b0389540f52e0d941 100644
--- a/test/CodeGen/X86/2011-07-13-BadFrameIndexDisplacement.ll
+++ b/test/CodeGen/X86/2011-07-13-BadFrameIndexDisplacement.ll
@@ -1,4 +1,4 @@
-; RUN: llc -mtriple=x86_64-- < %s -disable-fp-elim | FileCheck %s
+; RUN: llc -mtriple=x86_64-- < %s -frame-pointer=all | FileCheck %s
 
 ; This test is checking that we don't crash and we don't incorrectly fold
 ; a large displacement and a frame index into a single lea.
diff --git a/test/CodeGen/X86/2012-01-10-UndefExceptionEdge.ll b/test/CodeGen/X86/2012-01-10-UndefExceptionEdge.ll
index 97a33893fa0fa20f07c387915916f829c7dac937..8f5d46cc2b01b937a3e2852d51652a2f051afacd 100644
--- a/test/CodeGen/X86/2012-01-10-UndefExceptionEdge.ll
+++ b/test/CodeGen/X86/2012-01-10-UndefExceptionEdge.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -disable-fp-elim
+; RUN: llc < %s -frame-pointer=all
 target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128-n8:16:32-S128"
 target triple = "i386-apple-macosx10.7"
 
diff --git a/test/CodeGen/X86/2012-02-29-CoalescerBug.ll b/test/CodeGen/X86/2012-02-29-CoalescerBug.ll
index bbeb2a022192f0f279f7efac1ca9f3f1e0f13b0a..1d0ce617fc2f7ce82d14449eb09b255599dd325c 100644
--- a/test/CodeGen/X86/2012-02-29-CoalescerBug.ll
+++ b/test/CodeGen/X86/2012-02-29-CoalescerBug.ll
@@ -1,4 +1,4 @@
-; RUN: llc -O1 <%s
+; RUN: llc -O1 < %s
 ; PR12138
 target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128-n8:16:32-S128"
 target triple = "i386-apple-macosx10.7.0"
diff --git a/test/CodeGen/X86/3dnow-schedule.ll b/test/CodeGen/X86/3dnow-schedule.ll
deleted file mode 100644
index eb317ebbb2a94552107709b54e1945d34cea5e1b..0000000000000000000000000000000000000000
--- a/test/CodeGen/X86/3dnow-schedule.ll
+++ /dev/null
@@ -1,394 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=x86-64 -mattr=+3dnowa | FileCheck %s --check-prefix=CHECK --check-prefix=GENERIC
-
-define void @test_femms() optsize {
-; CHECK-LABEL: test_femms:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    femms # sched: [31:10.33]
-; CHECK-NEXT:    retq # sched: [1:1.00]
-  call void @llvm.x86.mmx.femms()
-  ret void
-}
-declare void @llvm.x86.mmx.femms() nounwind readnone
-
-define i64 @test_pavgusb(x86_mmx %a0, x86_mmx %a1, x86_mmx* %a2) optsize {
-; CHECK-LABEL: test_pavgusb:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    pavgusb %mm1, %mm0 # sched: [3:1.00]
-; CHECK-NEXT:    pavgusb (%rdi), %mm0 # sched: [8:1.00]
-; CHECK-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; CHECK-NEXT:    retq # sched: [1:1.00]
-  %1 = call x86_mmx @llvm.x86.3dnow.pavgusb(x86_mmx %a0, x86_mmx %a1)
-  %2 = load x86_mmx, x86_mmx *%a2, align 8
-  %3 = call x86_mmx @llvm.x86.3dnow.pavgusb(x86_mmx %1, x86_mmx %2)
-  %4 = bitcast x86_mmx %3 to i64
-  ret i64 %4
-}
-declare x86_mmx @llvm.x86.3dnow.pavgusb(x86_mmx, x86_mmx) nounwind readnone
-
-define i64 @test_pf2id(x86_mmx* %a0) optsize {
-; CHECK-LABEL: test_pf2id:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    pf2id (%rdi), %mm0 # sched: [9:1.00]
-; CHECK-NEXT:    pf2id %mm0, %mm0 # sched: [3:1.00]
-; CHECK-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; CHECK-NEXT:    retq # sched: [1:1.00]
-  %1 = load x86_mmx, x86_mmx *%a0, align 8
-  %2 = call x86_mmx @llvm.x86.3dnow.pf2id(x86_mmx %1)
-  %3 = call x86_mmx @llvm.x86.3dnow.pf2id(x86_mmx %2)
-  %4 = bitcast x86_mmx %3 to i64
-  ret i64 %4
-}
-declare x86_mmx @llvm.x86.3dnow.pf2id(x86_mmx) nounwind readnone
-
-define i64 @test_pf2iw(x86_mmx* %a0) optsize {
-; CHECK-LABEL: test_pf2iw:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    pf2iw (%rdi), %mm0 # sched: [9:1.00]
-; CHECK-NEXT:    pf2iw %mm0, %mm0 # sched: [3:1.00]
-; CHECK-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; CHECK-NEXT:    retq # sched: [1:1.00]
-  %1 = load x86_mmx, x86_mmx *%a0, align 8
-  %2 = call x86_mmx @llvm.x86.3dnowa.pf2iw(x86_mmx %1)
-  %3 = call x86_mmx @llvm.x86.3dnowa.pf2iw(x86_mmx %2)
-  %4 = bitcast x86_mmx %3 to i64
-  ret i64 %4
-}
-declare x86_mmx @llvm.x86.3dnowa.pf2iw(x86_mmx) nounwind readnone
-
-define i64 @test_pfacc(x86_mmx %a0, x86_mmx %a1, x86_mmx* %a2) optsize {
-; CHECK-LABEL: test_pfacc:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    pfacc %mm1, %mm0 # sched: [3:1.00]
-; CHECK-NEXT:    pfacc (%rdi), %mm0 # sched: [9:1.00]
-; CHECK-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; CHECK-NEXT:    retq # sched: [1:1.00]
-  %1 = call x86_mmx @llvm.x86.3dnow.pfacc(x86_mmx %a0, x86_mmx %a1)
-  %2 = load x86_mmx, x86_mmx *%a2, align 8
-  %3 = call x86_mmx @llvm.x86.3dnow.pfacc(x86_mmx %1, x86_mmx %2)
-  %4 = bitcast x86_mmx %3 to i64
-  ret i64 %4
-}
-declare x86_mmx @llvm.x86.3dnow.pfacc(x86_mmx, x86_mmx) nounwind readnone
-
-define i64 @test_pfadd(x86_mmx %a0, x86_mmx %a1, x86_mmx* %a2) optsize {
-; CHECK-LABEL: test_pfadd:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    pfadd %mm1, %mm0 # sched: [3:1.00]
-; CHECK-NEXT:    pfadd (%rdi), %mm0 # sched: [9:1.00]
-; CHECK-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; CHECK-NEXT:    retq # sched: [1:1.00]
-  %1 = call x86_mmx @llvm.x86.3dnow.pfadd(x86_mmx %a0, x86_mmx %a1)
-  %2 = load x86_mmx, x86_mmx *%a2, align 8
-  %3 = call x86_mmx @llvm.x86.3dnow.pfadd(x86_mmx %1, x86_mmx %2)
-  %4 = bitcast x86_mmx %3 to i64
-  ret i64 %4
-}
-declare x86_mmx @llvm.x86.3dnow.pfadd(x86_mmx, x86_mmx) nounwind readnone
-
-define i64 @test_pfcmpeq(x86_mmx %a0, x86_mmx %a1, x86_mmx* %a2) optsize {
-; CHECK-LABEL: test_pfcmpeq:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    pfcmpeq %mm1, %mm0 # sched: [3:1.00]
-; CHECK-NEXT:    pfcmpeq (%rdi), %mm0 # sched: [9:1.00]
-; CHECK-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; CHECK-NEXT:    retq # sched: [1:1.00]
-  %1 = call x86_mmx @llvm.x86.3dnow.pfcmpeq(x86_mmx %a0, x86_mmx %a1)
-  %2 = load x86_mmx, x86_mmx *%a2, align 8
-  %3 = call x86_mmx @llvm.x86.3dnow.pfcmpeq(x86_mmx %1, x86_mmx %2)
-  %4 = bitcast x86_mmx %3 to i64
-  ret i64 %4
-}
-declare x86_mmx @llvm.x86.3dnow.pfcmpeq(x86_mmx, x86_mmx) nounwind readnone
-
-define i64 @test_pfcmpge(x86_mmx %a0, x86_mmx %a1, x86_mmx* %a2) optsize {
-; CHECK-LABEL: test_pfcmpge:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    pfcmpge %mm1, %mm0 # sched: [3:1.00]
-; CHECK-NEXT:    pfcmpge (%rdi), %mm0 # sched: [9:1.00]
-; CHECK-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; CHECK-NEXT:    retq # sched: [1:1.00]
-  %1 = call x86_mmx @llvm.x86.3dnow.pfcmpge(x86_mmx %a0, x86_mmx %a1)
-  %2 = load x86_mmx, x86_mmx *%a2, align 8
-  %3 = call x86_mmx @llvm.x86.3dnow.pfcmpge(x86_mmx %1, x86_mmx %2)
-  %4 = bitcast x86_mmx %3 to i64
-  ret i64 %4
-}
-declare x86_mmx @llvm.x86.3dnow.pfcmpge(x86_mmx, x86_mmx) nounwind readnone
-
-define i64 @test_pfcmpgt(x86_mmx %a0, x86_mmx %a1, x86_mmx* %a2) optsize {
-; CHECK-LABEL: test_pfcmpgt:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    pfcmpgt %mm1, %mm0 # sched: [3:1.00]
-; CHECK-NEXT:    pfcmpgt (%rdi), %mm0 # sched: [9:1.00]
-; CHECK-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; CHECK-NEXT:    retq # sched: [1:1.00]
-  %1 = call x86_mmx @llvm.x86.3dnow.pfcmpgt(x86_mmx %a0, x86_mmx %a1)
-  %2 = load x86_mmx, x86_mmx *%a2, align 8
-  %3 = call x86_mmx @llvm.x86.3dnow.pfcmpgt(x86_mmx %1, x86_mmx %2)
-  %4 = bitcast x86_mmx %3 to i64
-  ret i64 %4
-}
-declare x86_mmx @llvm.x86.3dnow.pfcmpgt(x86_mmx, x86_mmx) nounwind readnone
-
-define i64 @test_pfmax(x86_mmx %a0, x86_mmx %a1, x86_mmx* %a2) optsize {
-; CHECK-LABEL: test_pfmax:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    pfmax %mm1, %mm0 # sched: [3:1.00]
-; CHECK-NEXT:    pfmax (%rdi), %mm0 # sched: [9:1.00]
-; CHECK-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; CHECK-NEXT:    retq # sched: [1:1.00]
-  %1 = call x86_mmx @llvm.x86.3dnow.pfmax(x86_mmx %a0, x86_mmx %a1)
-  %2 = load x86_mmx, x86_mmx *%a2, align 8
-  %3 = call x86_mmx @llvm.x86.3dnow.pfmax(x86_mmx %1, x86_mmx %2)
-  %4 = bitcast x86_mmx %3 to i64
-  ret i64 %4
-}
-declare x86_mmx @llvm.x86.3dnow.pfmax(x86_mmx, x86_mmx) nounwind readnone
-
-define i64 @test_pfmin(x86_mmx %a0, x86_mmx %a1, x86_mmx* %a2) optsize {
-; CHECK-LABEL: test_pfmin:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    pfmin %mm1, %mm0 # sched: [3:1.00]
-; CHECK-NEXT:    pfmin (%rdi), %mm0 # sched: [9:1.00]
-; CHECK-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; CHECK-NEXT:    retq # sched: [1:1.00]
-  %1 = call x86_mmx @llvm.x86.3dnow.pfmin(x86_mmx %a0, x86_mmx %a1)
-  %2 = load x86_mmx, x86_mmx *%a2, align 8
-  %3 = call x86_mmx @llvm.x86.3dnow.pfmin(x86_mmx %1, x86_mmx %2)
-  %4 = bitcast x86_mmx %3 to i64
-  ret i64 %4
-}
-declare x86_mmx @llvm.x86.3dnow.pfmin(x86_mmx, x86_mmx) nounwind readnone
-
-define i64 @test_pfmul(x86_mmx %a0, x86_mmx %a1, x86_mmx* %a2) optsize {
-; CHECK-LABEL: test_pfmul:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    pfmul %mm1, %mm0 # sched: [3:1.00]
-; CHECK-NEXT:    pfmul (%rdi), %mm0 # sched: [9:1.00]
-; CHECK-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; CHECK-NEXT:    retq # sched: [1:1.00]
-  %1 = call x86_mmx @llvm.x86.3dnow.pfmul(x86_mmx %a0, x86_mmx %a1)
-  %2 = load x86_mmx, x86_mmx *%a2, align 8
-  %3 = call x86_mmx @llvm.x86.3dnow.pfmul(x86_mmx %1, x86_mmx %2)
-  %4 = bitcast x86_mmx %3 to i64
-  ret i64 %4
-}
-declare x86_mmx @llvm.x86.3dnow.pfmul(x86_mmx, x86_mmx) nounwind readnone
-
-define i64 @test_pfnacc(x86_mmx %a0, x86_mmx %a1, x86_mmx* %a2) optsize {
-; CHECK-LABEL: test_pfnacc:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    pfnacc %mm1, %mm0 # sched: [3:1.00]
-; CHECK-NEXT:    pfnacc (%rdi), %mm0 # sched: [9:1.00]
-; CHECK-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; CHECK-NEXT:    retq # sched: [1:1.00]
-  %1 = call x86_mmx @llvm.x86.3dnowa.pfnacc(x86_mmx %a0, x86_mmx %a1)
-  %2 = load x86_mmx, x86_mmx *%a2, align 8
-  %3 = call x86_mmx @llvm.x86.3dnowa.pfnacc(x86_mmx %1, x86_mmx %2)
-  %4 = bitcast x86_mmx %3 to i64
-  ret i64 %4
-}
-declare x86_mmx @llvm.x86.3dnowa.pfnacc(x86_mmx, x86_mmx) nounwind readnone
-
-define i64 @test_pfpnacc(x86_mmx %a0, x86_mmx %a1, x86_mmx* %a2) optsize {
-; CHECK-LABEL: test_pfpnacc:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    pfpnacc %mm1, %mm0 # sched: [3:1.00]
-; CHECK-NEXT:    pfpnacc (%rdi), %mm0 # sched: [9:1.00]
-; CHECK-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; CHECK-NEXT:    retq # sched: [1:1.00]
-  %1 = call x86_mmx @llvm.x86.3dnowa.pfpnacc(x86_mmx %a0, x86_mmx %a1)
-  %2 = load x86_mmx, x86_mmx *%a2, align 8
-  %3 = call x86_mmx @llvm.x86.3dnowa.pfpnacc(x86_mmx %1, x86_mmx %2)
-  %4 = bitcast x86_mmx %3 to i64
-  ret i64 %4
-}
-declare x86_mmx @llvm.x86.3dnowa.pfpnacc(x86_mmx, x86_mmx) nounwind readnone
-
-define i64 @test_pfrcp(x86_mmx* %a0) optsize {
-; CHECK-LABEL: test_pfrcp:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    pfrcp (%rdi), %mm0 # sched: [9:1.00]
-; CHECK-NEXT:    pfrcp %mm0, %mm0 # sched: [3:1.00]
-; CHECK-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; CHECK-NEXT:    retq # sched: [1:1.00]
-  %1 = load x86_mmx, x86_mmx *%a0, align 8
-  %2 = call x86_mmx @llvm.x86.3dnow.pfrcp(x86_mmx %1)
-  %3 = call x86_mmx @llvm.x86.3dnow.pfrcp(x86_mmx %2)
-  %4 = bitcast x86_mmx %3 to i64
-  ret i64 %4
-}
-declare x86_mmx @llvm.x86.3dnow.pfrcp(x86_mmx) nounwind readnone
-
-define i64 @test_pfrcpit1(x86_mmx %a0, x86_mmx %a1, x86_mmx* %a2) optsize {
-; CHECK-LABEL: test_pfrcpit1:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    pfrcpit1 %mm1, %mm0 # sched: [3:1.00]
-; CHECK-NEXT:    pfrcpit1 (%rdi), %mm0 # sched: [9:1.00]
-; CHECK-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; CHECK-NEXT:    retq # sched: [1:1.00]
-  %1 = call x86_mmx @llvm.x86.3dnow.pfrcpit1(x86_mmx %a0, x86_mmx %a1)
-  %2 = load x86_mmx, x86_mmx *%a2, align 8
-  %3 = call x86_mmx @llvm.x86.3dnow.pfrcpit1(x86_mmx %1, x86_mmx %2)
-  %4 = bitcast x86_mmx %3 to i64
-  ret i64 %4
-}
-declare x86_mmx @llvm.x86.3dnow.pfrcpit1(x86_mmx, x86_mmx) nounwind readnone
-
-define i64 @test_pfrcpit2(x86_mmx %a0, x86_mmx %a1, x86_mmx* %a2) optsize {
-; CHECK-LABEL: test_pfrcpit2:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    pfrcpit2 %mm1, %mm0 # sched: [3:1.00]
-; CHECK-NEXT:    pfrcpit2 (%rdi), %mm0 # sched: [9:1.00]
-; CHECK-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; CHECK-NEXT:    retq # sched: [1:1.00]
-  %1 = call x86_mmx @llvm.x86.3dnow.pfrcpit2(x86_mmx %a0, x86_mmx %a1)
-  %2 = load x86_mmx, x86_mmx *%a2, align 8
-  %3 = call x86_mmx @llvm.x86.3dnow.pfrcpit2(x86_mmx %1, x86_mmx %2)
-  %4 = bitcast x86_mmx %3 to i64
-  ret i64 %4
-}
-declare x86_mmx @llvm.x86.3dnow.pfrcpit2(x86_mmx, x86_mmx) nounwind readnone
-
-define i64 @test_pfrsqit1(x86_mmx %a0, x86_mmx %a1, x86_mmx* %a2) optsize {
-; CHECK-LABEL: test_pfrsqit1:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    pfrsqit1 %mm1, %mm0 # sched: [3:1.00]
-; CHECK-NEXT:    pfrsqit1 (%rdi), %mm0 # sched: [9:1.00]
-; CHECK-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; CHECK-NEXT:    retq # sched: [1:1.00]
-  %1 = call x86_mmx @llvm.x86.3dnow.pfrsqit1(x86_mmx %a0, x86_mmx %a1)
-  %2 = load x86_mmx, x86_mmx *%a2, align 8
-  %3 = call x86_mmx @llvm.x86.3dnow.pfrsqit1(x86_mmx %1, x86_mmx %2)
-  %4 = bitcast x86_mmx %3 to i64
-  ret i64 %4
-}
-declare x86_mmx @llvm.x86.3dnow.pfrsqit1(x86_mmx, x86_mmx) nounwind readnone
-
-define i64 @test_pfrsqrt(x86_mmx* %a0) optsize {
-; CHECK-LABEL: test_pfrsqrt:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    pfrsqrt (%rdi), %mm0 # sched: [9:1.00]
-; CHECK-NEXT:    pfrsqrt %mm0, %mm0 # sched: [3:1.00]
-; CHECK-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; CHECK-NEXT:    retq # sched: [1:1.00]
-  %1 = load x86_mmx, x86_mmx *%a0, align 8
-  %2 = call x86_mmx @llvm.x86.3dnow.pfrsqrt(x86_mmx %1)
-  %3 = call x86_mmx @llvm.x86.3dnow.pfrsqrt(x86_mmx %2)
-  %4 = bitcast x86_mmx %3 to i64
-  ret i64 %4
-}
-declare x86_mmx @llvm.x86.3dnow.pfrsqrt(x86_mmx) nounwind readnone
-
-define i64 @test_pfsub(x86_mmx %a0, x86_mmx %a1, x86_mmx* %a2) optsize {
-; CHECK-LABEL: test_pfsub:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    pfsub %mm1, %mm0 # sched: [3:1.00]
-; CHECK-NEXT:    pfsub (%rdi), %mm0 # sched: [9:1.00]
-; CHECK-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; CHECK-NEXT:    retq # sched: [1:1.00]
-  %1 = call x86_mmx @llvm.x86.3dnow.pfsub(x86_mmx %a0, x86_mmx %a1)
-  %2 = load x86_mmx, x86_mmx *%a2, align 8
-  %3 = call x86_mmx @llvm.x86.3dnow.pfsub(x86_mmx %1, x86_mmx %2)
-  %4 = bitcast x86_mmx %3 to i64
-  ret i64 %4
-}
-declare x86_mmx @llvm.x86.3dnow.pfsub(x86_mmx, x86_mmx) nounwind readnone
-
-define i64 @test_pfsubr(x86_mmx %a0, x86_mmx %a1, x86_mmx* %a2) optsize {
-; CHECK-LABEL: test_pfsubr:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    pfsubr %mm1, %mm0 # sched: [3:1.00]
-; CHECK-NEXT:    pfsubr (%rdi), %mm0 # sched: [9:1.00]
-; CHECK-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; CHECK-NEXT:    retq # sched: [1:1.00]
-  %1 = call x86_mmx @llvm.x86.3dnow.pfsubr(x86_mmx %a0, x86_mmx %a1)
-  %2 = load x86_mmx, x86_mmx *%a2, align 8
-  %3 = call x86_mmx @llvm.x86.3dnow.pfsubr(x86_mmx %1, x86_mmx %2)
-  %4 = bitcast x86_mmx %3 to i64
-  ret i64 %4
-}
-declare x86_mmx @llvm.x86.3dnow.pfsubr(x86_mmx, x86_mmx) nounwind readnone
-
-define i64 @test_pi2fd(x86_mmx* %a0) optsize {
-; CHECK-LABEL: test_pi2fd:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    pi2fd (%rdi), %mm0 # sched: [9:1.00]
-; CHECK-NEXT:    pi2fd %mm0, %mm0 # sched: [3:1.00]
-; CHECK-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; CHECK-NEXT:    retq # sched: [1:1.00]
-  %1 = load x86_mmx, x86_mmx *%a0, align 8
-  %2 = call x86_mmx @llvm.x86.3dnow.pi2fd(x86_mmx %1)
-  %3 = call x86_mmx @llvm.x86.3dnow.pi2fd(x86_mmx %2)
-  %4 = bitcast x86_mmx %3 to i64
-  ret i64 %4
-}
-declare x86_mmx @llvm.x86.3dnow.pi2fd(x86_mmx) nounwind readnone
-
-define i64 @test_pi2fw(x86_mmx* %a0) optsize {
-; CHECK-LABEL: test_pi2fw:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    pi2fw (%rdi), %mm0 # sched: [9:1.00]
-; CHECK-NEXT:    pi2fw %mm0, %mm0 # sched: [3:1.00]
-; CHECK-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; CHECK-NEXT:    retq # sched: [1:1.00]
-  %1 = load x86_mmx, x86_mmx *%a0, align 8
-  %2 = call x86_mmx @llvm.x86.3dnowa.pi2fw(x86_mmx %1)
-  %3 = call x86_mmx @llvm.x86.3dnowa.pi2fw(x86_mmx %2)
-  %4 = bitcast x86_mmx %3 to i64
-  ret i64 %4
-}
-declare x86_mmx @llvm.x86.3dnowa.pi2fw(x86_mmx) nounwind readnone
-
-define i64 @test_pmulhrw(x86_mmx %a0, x86_mmx %a1, x86_mmx* %a2) optsize {
-; CHECK-LABEL: test_pmulhrw:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    pmulhrw %mm1, %mm0 # sched: [5:1.00]
-; CHECK-NEXT:    pmulhrw (%rdi), %mm0 # sched: [10:1.00]
-; CHECK-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; CHECK-NEXT:    retq # sched: [1:1.00]
-  %1 = call x86_mmx @llvm.x86.3dnow.pmulhrw(x86_mmx %a0, x86_mmx %a1)
-  %2 = load x86_mmx, x86_mmx *%a2, align 8
-  %3 = call x86_mmx @llvm.x86.3dnow.pmulhrw(x86_mmx %1, x86_mmx %2)
-  %4 = bitcast x86_mmx %3 to i64
-  ret i64 %4
-}
-declare x86_mmx @llvm.x86.3dnow.pmulhrw(x86_mmx, x86_mmx) nounwind readnone
-
-define void @test_prefetch(i8* %a0) optsize {
-; CHECK-LABEL: test_prefetch:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    #APP
-; CHECK-NEXT:    prefetch (%rdi) # sched: [5:0.50]
-; CHECK-NEXT:    #NO_APP
-; CHECK-NEXT:    retq # sched: [1:1.00]
-  tail call void asm sideeffect "prefetch $0", "*m"(i8 *%a0) nounwind
-  ret void
-}
-
-define void @test_prefetchw(i8* %a0) optsize {
-; CHECK-LABEL: test_prefetchw:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    #APP
-; CHECK-NEXT:    prefetchw (%rdi) # sched: [5:0.50]
-; CHECK-NEXT:    #NO_APP
-; CHECK-NEXT:    retq # sched: [1:1.00]
-  tail call void asm sideeffect "prefetchw $0", "*m"(i8 *%a0) nounwind
-  ret void
-}
-
-define i64 @test_pswapd(x86_mmx* %a0) optsize {
-; CHECK-LABEL: test_pswapd:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    pswapd (%rdi), %mm0 # mm0 = mem[1,0] sched: [6:1.00]
-; CHECK-NEXT:    pswapd %mm0, %mm0 # mm0 = mm0[1,0] sched: [1:1.00]
-; CHECK-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; CHECK-NEXT:    retq # sched: [1:1.00]
-  %1 = load x86_mmx, x86_mmx *%a0, align 8
-  %2 = call x86_mmx @llvm.x86.3dnowa.pswapd(x86_mmx %1)
-  %3 = call x86_mmx @llvm.x86.3dnowa.pswapd(x86_mmx %2)
-  %4 = bitcast x86_mmx %3 to i64
-  ret i64 %4
-}
-declare x86_mmx @llvm.x86.3dnowa.pswapd(x86_mmx) nounwind readnone
diff --git a/test/CodeGen/X86/GlobalISel/ashr-scalar.ll b/test/CodeGen/X86/GlobalISel/ashr-scalar.ll
index df63071c4ee2e2b20c19131c77fd9e2edc5af542..a8496bf02cfaf1dde91901c9b8334a2d0835a4b0 100644
--- a/test/CodeGen/X86/GlobalISel/ashr-scalar.ll
+++ b/test/CodeGen/X86/GlobalISel/ashr-scalar.ll
@@ -6,7 +6,7 @@ define i64 @test_ashr_i64(i64 %arg1, i64 %arg2) {
 ; X64:       # %bb.0:
 ; X64-NEXT:    movq %rdi, %rax
 ; X64-NEXT:    movq %rsi, %rcx
-; X64-NEXT:    # kill: def $cl killed $rcx
+; X64-NEXT:    # kill: def $cl killed $cl killed $rcx
 ; X64-NEXT:    sarq %cl, %rax
 ; X64-NEXT:    retq
   %res = ashr i64 %arg1, %arg2
@@ -18,7 +18,6 @@ define i64 @test_ashr_i64_imm(i64 %arg1) {
 ; X64:       # %bb.0:
 ; X64-NEXT:    movq %rdi, %rax
 ; X64-NEXT:    movq $5, %rcx
-; X64-NEXT:    # kill: def $cl killed $rcx
 ; X64-NEXT:    sarq %cl, %rax
 ; X64-NEXT:    retq
   %res = ashr i64 %arg1, 5
@@ -30,7 +29,6 @@ define i64 @test_ashr_i64_imm1(i64 %arg1) {
 ; X64:       # %bb.0:
 ; X64-NEXT:    movq %rdi, %rax
 ; X64-NEXT:    movq $1, %rcx
-; X64-NEXT:    # kill: def $cl killed $rcx
 ; X64-NEXT:    sarq %cl, %rax
 ; X64-NEXT:    retq
   %res = ashr i64 %arg1, 1
@@ -42,7 +40,7 @@ define i32 @test_ashr_i32(i32 %arg1, i32 %arg2) {
 ; X64:       # %bb.0:
 ; X64-NEXT:    movl %edi, %eax
 ; X64-NEXT:    movl %esi, %ecx
-; X64-NEXT:    # kill: def $cl killed $ecx
+; X64-NEXT:    # kill: def $cl killed $cl killed $ecx
 ; X64-NEXT:    sarl %cl, %eax
 ; X64-NEXT:    retq
   %res = ashr i32 %arg1, %arg2
@@ -54,7 +52,6 @@ define i32 @test_ashr_i32_imm(i32 %arg1) {
 ; X64:       # %bb.0:
 ; X64-NEXT:    movl %edi, %eax
 ; X64-NEXT:    movl $5, %ecx
-; X64-NEXT:    # kill: def $cl killed $ecx
 ; X64-NEXT:    sarl %cl, %eax
 ; X64-NEXT:    retq
   %res = ashr i32 %arg1, 5
@@ -66,7 +63,6 @@ define i32 @test_ashr_i32_imm1(i32 %arg1) {
 ; X64:       # %bb.0:
 ; X64-NEXT:    movl %edi, %eax
 ; X64-NEXT:    movl $1, %ecx
-; X64-NEXT:    # kill: def $cl killed $ecx
 ; X64-NEXT:    sarl %cl, %eax
 ; X64-NEXT:    retq
   %res = ashr i32 %arg1, 1
@@ -78,8 +74,7 @@ define i16 @test_ashr_i16(i32 %arg1, i32 %arg2) {
 ; X64:       # %bb.0:
 ; X64-NEXT:    movl %edi, %eax
 ; X64-NEXT:    movl %esi, %ecx
-; X64-NEXT:    # kill: def $cx killed $cx killed $ecx
-; X64-NEXT:    # kill: def $cl killed $cx
+; X64-NEXT:    # kill: def $cl killed $cl killed $ecx
 ; X64-NEXT:    sarw %cl, %ax
 ; X64-NEXT:    # kill: def $ax killed $ax killed $eax
 ; X64-NEXT:    retq
@@ -94,7 +89,6 @@ define i16 @test_ashr_i16_imm(i32 %arg1) {
 ; X64:       # %bb.0:
 ; X64-NEXT:    movl %edi, %eax
 ; X64-NEXT:    movw $5, %cx
-; X64-NEXT:    # kill: def $cl killed $cx
 ; X64-NEXT:    sarw %cl, %ax
 ; X64-NEXT:    # kill: def $ax killed $ax killed $eax
 ; X64-NEXT:    retq
@@ -108,7 +102,6 @@ define i16 @test_ashr_i16_imm1(i32 %arg1) {
 ; X64:       # %bb.0:
 ; X64-NEXT:    movl %edi, %eax
 ; X64-NEXT:    movw $1, %cx
-; X64-NEXT:    # kill: def $cl killed $cx
 ; X64-NEXT:    sarw %cl, %ax
 ; X64-NEXT:    # kill: def $ax killed $ax killed $eax
 ; X64-NEXT:    retq
diff --git a/test/CodeGen/X86/GlobalISel/callingconv.ll b/test/CodeGen/X86/GlobalISel/callingconv.ll
index 1fe4318b7468afa9378bbda87483e2aac9205ebd..33e16893473c7daae34908bc9fad57d5da16715b 100644
--- a/test/CodeGen/X86/GlobalISel/callingconv.ll
+++ b/test/CodeGen/X86/GlobalISel/callingconv.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=i386-linux-gnu -mattr=+sse2  -global-isel -verify-machineinstrs < %s -o - | FileCheck %s --check-prefix=ALL --check-prefix=X32
-; RUN: llc -mtriple=x86_64-linux-gnu             -global-isel -verify-machineinstrs < %s -o - | FileCheck %s --check-prefix=ALL --check-prefix=X64
+; RUN: llc -enable-cse-in-irtranslator=0 -enable-cse-in-legalizer=0 -mtriple=i386-linux-gnu -mattr=+sse2  -global-isel -verify-machineinstrs < %s -o - | FileCheck %s --check-prefix=ALL --check-prefix=X32
+; RUN: llc -enable-cse-in-irtranslator=0 -enable-cse-in-legalizer=0 -mtriple=x86_64-linux-gnu             -global-isel -verify-machineinstrs < %s -o - | FileCheck %s --check-prefix=ALL --check-prefix=X64
 
 define i32 @test_ret_i32() {
 ; X32-LABEL: test_ret_i32:
diff --git a/test/CodeGen/X86/GlobalISel/ext-x86-64.ll b/test/CodeGen/X86/GlobalISel/ext-x86-64.ll
index a9cdcbeb566a3e22e04405bf9245984703295f65..498126fbcf3639adeac091999ddd5ac61ac4fc51 100644
--- a/test/CodeGen/X86/GlobalISel/ext-x86-64.ll
+++ b/test/CodeGen/X86/GlobalISel/ext-x86-64.ll
@@ -19,10 +19,8 @@ define i64 @test_sext_i8(i8 %val) {
 ; X64:       # %bb.0:
 ; X64-NEXT:    movl %edi, %eax
 ; X64-NEXT:    movq $56, %rcx
-; X64-NEXT:    # kill: def $cl killed $rcx
 ; X64-NEXT:    shlq %cl, %rax
 ; X64-NEXT:    movq $56, %rcx
-; X64-NEXT:    # kill: def $cl killed $rcx
 ; X64-NEXT:    sarq %cl, %rax
 ; X64-NEXT:    retq
   %r = sext i8 %val to i64
@@ -34,10 +32,8 @@ define i64 @test_sext_i16(i16 %val) {
 ; X64:       # %bb.0:
 ; X64-NEXT:    movl %edi, %eax
 ; X64-NEXT:    movq $48, %rcx
-; X64-NEXT:    # kill: def $cl killed $rcx
 ; X64-NEXT:    shlq %cl, %rax
 ; X64-NEXT:    movq $48, %rcx
-; X64-NEXT:    # kill: def $cl killed $rcx
 ; X64-NEXT:    sarq %cl, %rax
 ; X64-NEXT:    retq
   %r = sext i16 %val to i64
diff --git a/test/CodeGen/X86/GlobalISel/ext.ll b/test/CodeGen/X86/GlobalISel/ext.ll
index eaed4309b1291ebea1cad4bb114bceb0dffb5de5..a54656b55a6b5eed6eb2dfda894388ac8e2b5b48 100644
--- a/test/CodeGen/X86/GlobalISel/ext.ll
+++ b/test/CodeGen/X86/GlobalISel/ext.ll
@@ -90,10 +90,8 @@ define i32 @test_sext_i8(i8 %val) {
 ; X64:       # %bb.0:
 ; X64-NEXT:    movl %edi, %eax
 ; X64-NEXT:    movl $24, %ecx
-; X64-NEXT:    # kill: def $cl killed $ecx
 ; X64-NEXT:    shll %cl, %eax
 ; X64-NEXT:    movl $24, %ecx
-; X64-NEXT:    # kill: def $cl killed $ecx
 ; X64-NEXT:    sarl %cl, %eax
 ; X64-NEXT:    retq
 ;
@@ -110,10 +108,8 @@ define i32 @test_sext_i16(i16 %val) {
 ; X64:       # %bb.0:
 ; X64-NEXT:    movl %edi, %eax
 ; X64-NEXT:    movl $16, %ecx
-; X64-NEXT:    # kill: def $cl killed $ecx
 ; X64-NEXT:    shll %cl, %eax
 ; X64-NEXT:    movl $16, %ecx
-; X64-NEXT:    # kill: def $cl killed $ecx
 ; X64-NEXT:    sarl %cl, %eax
 ; X64-NEXT:    retq
 ;
diff --git a/test/CodeGen/X86/GlobalISel/gep.ll b/test/CodeGen/X86/GlobalISel/gep.ll
index 2b64fc4da482679609f481ab6fb72678b97784e8..82248161403b2daee3e75f58ea4c46f89a27202c 100644
--- a/test/CodeGen/X86/GlobalISel/gep.ll
+++ b/test/CodeGen/X86/GlobalISel/gep.ll
@@ -8,10 +8,8 @@ define i32* @test_gep_i8(i32 *%arr, i8 %ind) {
 ; X64_GISEL-NEXT:    # kill: def $esi killed $esi def $rsi
 ; X64_GISEL-NEXT:    movq $4, %rax
 ; X64_GISEL-NEXT:    movq $56, %rcx
-; X64_GISEL-NEXT:    # kill: def $cl killed $rcx
 ; X64_GISEL-NEXT:    shlq %cl, %rsi
 ; X64_GISEL-NEXT:    movq $56, %rcx
-; X64_GISEL-NEXT:    # kill: def $cl killed $rcx
 ; X64_GISEL-NEXT:    sarq %cl, %rsi
 ; X64_GISEL-NEXT:    imulq %rax, %rsi
 ; X64_GISEL-NEXT:    leaq (%rdi,%rsi), %rax
@@ -48,10 +46,8 @@ define i32* @test_gep_i16(i32 *%arr, i16 %ind) {
 ; X64_GISEL-NEXT:    # kill: def $esi killed $esi def $rsi
 ; X64_GISEL-NEXT:    movq $4, %rax
 ; X64_GISEL-NEXT:    movq $48, %rcx
-; X64_GISEL-NEXT:    # kill: def $cl killed $rcx
 ; X64_GISEL-NEXT:    shlq %cl, %rsi
 ; X64_GISEL-NEXT:    movq $48, %rcx
-; X64_GISEL-NEXT:    # kill: def $cl killed $rcx
 ; X64_GISEL-NEXT:    sarq %cl, %rsi
 ; X64_GISEL-NEXT:    imulq %rax, %rsi
 ; X64_GISEL-NEXT:    leaq (%rdi,%rsi), %rax
diff --git a/test/CodeGen/X86/GlobalISel/irtranslator-callingconv.ll b/test/CodeGen/X86/GlobalISel/irtranslator-callingconv.ll
index 5f08877b4a24c38d9322af66625baa60ff49ce53..37cd71f99ca5c6da596d100d351dc859a9a77806 100644
--- a/test/CodeGen/X86/GlobalISel/irtranslator-callingconv.ll
+++ b/test/CodeGen/X86/GlobalISel/irtranslator-callingconv.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-; RUN: llc -mtriple=i386-linux-gnu   -mattr=+sse2 -global-isel -stop-after=irtranslator < %s -o - | FileCheck %s --check-prefix=ALL --check-prefix=X32
-; RUN: llc -mtriple=x86_64-linux-gnu              -global-isel -stop-after=irtranslator < %s -o - | FileCheck %s --check-prefix=ALL --check-prefix=X64
+; RUN: llc -O0 -mtriple=i386-linux-gnu   -mattr=+sse2 -global-isel -stop-after=irtranslator < %s -o - | FileCheck %s --check-prefix=ALL --check-prefix=X32
+; RUN: llc -O0 -mtriple=x86_64-linux-gnu              -global-isel -stop-after=irtranslator < %s -o - | FileCheck %s --check-prefix=ALL --check-prefix=X64
 
 @a1_8bit = external global i8
 @a7_8bit = external global i8
@@ -10,21 +10,21 @@ define i8 @test_i8_args_8(i8 %arg1, i8 %arg2, i8 %arg3, i8 %arg4,
   ; X32-LABEL: name: test_i8_args_8
   ; X32: bb.1.entry:
   ; X32:   [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.7
-  ; X32:   [[LOAD:%[0-9]+]]:_(s8) = G_LOAD [[FRAME_INDEX]](p0) :: (invariant load 1 from %fixed-stack.7, align 0)
+  ; X32:   [[LOAD:%[0-9]+]]:_(s8) = G_LOAD [[FRAME_INDEX]](p0) :: (invariant load 1 from %fixed-stack.7)
   ; X32:   [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.6
-  ; X32:   [[LOAD1:%[0-9]+]]:_(s8) = G_LOAD [[FRAME_INDEX1]](p0) :: (invariant load 1 from %fixed-stack.6, align 0)
+  ; X32:   [[LOAD1:%[0-9]+]]:_(s8) = G_LOAD [[FRAME_INDEX1]](p0) :: (invariant load 1 from %fixed-stack.6)
   ; X32:   [[FRAME_INDEX2:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.5
-  ; X32:   [[LOAD2:%[0-9]+]]:_(s8) = G_LOAD [[FRAME_INDEX2]](p0) :: (invariant load 1 from %fixed-stack.5, align 0)
+  ; X32:   [[LOAD2:%[0-9]+]]:_(s8) = G_LOAD [[FRAME_INDEX2]](p0) :: (invariant load 1 from %fixed-stack.5)
   ; X32:   [[FRAME_INDEX3:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.4
-  ; X32:   [[LOAD3:%[0-9]+]]:_(s8) = G_LOAD [[FRAME_INDEX3]](p0) :: (invariant load 1 from %fixed-stack.4, align 0)
+  ; X32:   [[LOAD3:%[0-9]+]]:_(s8) = G_LOAD [[FRAME_INDEX3]](p0) :: (invariant load 1 from %fixed-stack.4)
   ; X32:   [[FRAME_INDEX4:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.3
-  ; X32:   [[LOAD4:%[0-9]+]]:_(s8) = G_LOAD [[FRAME_INDEX4]](p0) :: (invariant load 1 from %fixed-stack.3, align 0)
+  ; X32:   [[LOAD4:%[0-9]+]]:_(s8) = G_LOAD [[FRAME_INDEX4]](p0) :: (invariant load 1 from %fixed-stack.3)
   ; X32:   [[FRAME_INDEX5:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.2
-  ; X32:   [[LOAD5:%[0-9]+]]:_(s8) = G_LOAD [[FRAME_INDEX5]](p0) :: (invariant load 1 from %fixed-stack.2, align 0)
+  ; X32:   [[LOAD5:%[0-9]+]]:_(s8) = G_LOAD [[FRAME_INDEX5]](p0) :: (invariant load 1 from %fixed-stack.2)
   ; X32:   [[FRAME_INDEX6:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.1
-  ; X32:   [[LOAD6:%[0-9]+]]:_(s8) = G_LOAD [[FRAME_INDEX6]](p0) :: (invariant load 1 from %fixed-stack.1, align 0)
+  ; X32:   [[LOAD6:%[0-9]+]]:_(s8) = G_LOAD [[FRAME_INDEX6]](p0) :: (invariant load 1 from %fixed-stack.1)
   ; X32:   [[FRAME_INDEX7:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.0
-  ; X32:   [[LOAD7:%[0-9]+]]:_(s8) = G_LOAD [[FRAME_INDEX7]](p0) :: (invariant load 1 from %fixed-stack.0, align 0)
+  ; X32:   [[LOAD7:%[0-9]+]]:_(s8) = G_LOAD [[FRAME_INDEX7]](p0) :: (invariant load 1 from %fixed-stack.0)
   ; X32:   [[GV:%[0-9]+]]:_(p0) = G_GLOBAL_VALUE @a1_8bit
   ; X32:   [[GV1:%[0-9]+]]:_(p0) = G_GLOBAL_VALUE @a7_8bit
   ; X32:   [[GV2:%[0-9]+]]:_(p0) = G_GLOBAL_VALUE @a8_8bit
@@ -49,9 +49,9 @@ define i8 @test_i8_args_8(i8 %arg1, i8 %arg2, i8 %arg3, i8 %arg4,
   ; X64:   [[COPY5:%[0-9]+]]:_(s32) = COPY $r9d
   ; X64:   [[TRUNC5:%[0-9]+]]:_(s8) = G_TRUNC [[COPY5]](s32)
   ; X64:   [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.1
-  ; X64:   [[LOAD:%[0-9]+]]:_(s8) = G_LOAD [[FRAME_INDEX]](p0) :: (invariant load 1 from %fixed-stack.1, align 0)
+  ; X64:   [[LOAD:%[0-9]+]]:_(s8) = G_LOAD [[FRAME_INDEX]](p0) :: (invariant load 1 from %fixed-stack.1)
   ; X64:   [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.0
-  ; X64:   [[LOAD1:%[0-9]+]]:_(s8) = G_LOAD [[FRAME_INDEX1]](p0) :: (invariant load 1 from %fixed-stack.0, align 0)
+  ; X64:   [[LOAD1:%[0-9]+]]:_(s8) = G_LOAD [[FRAME_INDEX1]](p0) :: (invariant load 1 from %fixed-stack.0)
   ; X64:   [[GV:%[0-9]+]]:_(p0) = G_GLOBAL_VALUE @a1_8bit
   ; X64:   [[GV1:%[0-9]+]]:_(p0) = G_GLOBAL_VALUE @a7_8bit
   ; X64:   [[GV2:%[0-9]+]]:_(p0) = G_GLOBAL_VALUE @a8_8bit
@@ -76,21 +76,21 @@ define i32 @test_i32_args_8(i32 %arg1, i32 %arg2, i32 %arg3, i32 %arg4,
   ; X32-LABEL: name: test_i32_args_8
   ; X32: bb.1.entry:
   ; X32:   [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.7
-  ; X32:   [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX]](p0) :: (invariant load 4 from %fixed-stack.7, align 0)
+  ; X32:   [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX]](p0) :: (invariant load 4 from %fixed-stack.7, align 1)
   ; X32:   [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.6
-  ; X32:   [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX1]](p0) :: (invariant load 4 from %fixed-stack.6, align 0)
+  ; X32:   [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX1]](p0) :: (invariant load 4 from %fixed-stack.6, align 1)
   ; X32:   [[FRAME_INDEX2:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.5
-  ; X32:   [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX2]](p0) :: (invariant load 4 from %fixed-stack.5, align 0)
+  ; X32:   [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX2]](p0) :: (invariant load 4 from %fixed-stack.5, align 1)
   ; X32:   [[FRAME_INDEX3:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.4
-  ; X32:   [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX3]](p0) :: (invariant load 4 from %fixed-stack.4, align 0)
+  ; X32:   [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX3]](p0) :: (invariant load 4 from %fixed-stack.4, align 1)
   ; X32:   [[FRAME_INDEX4:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.3
-  ; X32:   [[LOAD4:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX4]](p0) :: (invariant load 4 from %fixed-stack.3, align 0)
+  ; X32:   [[LOAD4:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX4]](p0) :: (invariant load 4 from %fixed-stack.3, align 1)
   ; X32:   [[FRAME_INDEX5:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.2
-  ; X32:   [[LOAD5:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX5]](p0) :: (invariant load 4 from %fixed-stack.2, align 0)
+  ; X32:   [[LOAD5:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX5]](p0) :: (invariant load 4 from %fixed-stack.2, align 1)
   ; X32:   [[FRAME_INDEX6:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.1
-  ; X32:   [[LOAD6:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX6]](p0) :: (invariant load 4 from %fixed-stack.1, align 0)
+  ; X32:   [[LOAD6:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX6]](p0) :: (invariant load 4 from %fixed-stack.1, align 1)
   ; X32:   [[FRAME_INDEX7:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.0
-  ; X32:   [[LOAD7:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX7]](p0) :: (invariant load 4 from %fixed-stack.0, align 0)
+  ; X32:   [[LOAD7:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX7]](p0) :: (invariant load 4 from %fixed-stack.0, align 1)
   ; X32:   [[GV:%[0-9]+]]:_(p0) = G_GLOBAL_VALUE @a1_32bit
   ; X32:   [[GV1:%[0-9]+]]:_(p0) = G_GLOBAL_VALUE @a7_32bit
   ; X32:   [[GV2:%[0-9]+]]:_(p0) = G_GLOBAL_VALUE @a8_32bit
@@ -109,9 +109,9 @@ define i32 @test_i32_args_8(i32 %arg1, i32 %arg2, i32 %arg3, i32 %arg4,
   ; X64:   [[COPY4:%[0-9]+]]:_(s32) = COPY $r8d
   ; X64:   [[COPY5:%[0-9]+]]:_(s32) = COPY $r9d
   ; X64:   [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.1
-  ; X64:   [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX]](p0) :: (invariant load 4 from %fixed-stack.1, align 0)
+  ; X64:   [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX]](p0) :: (invariant load 4 from %fixed-stack.1, align 1)
   ; X64:   [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.0
-  ; X64:   [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX1]](p0) :: (invariant load 4 from %fixed-stack.0, align 0)
+  ; X64:   [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX1]](p0) :: (invariant load 4 from %fixed-stack.0, align 1)
   ; X64:   [[GV:%[0-9]+]]:_(p0) = G_GLOBAL_VALUE @a1_32bit
   ; X64:   [[GV1:%[0-9]+]]:_(p0) = G_GLOBAL_VALUE @a7_32bit
   ; X64:   [[GV2:%[0-9]+]]:_(p0) = G_GLOBAL_VALUE @a8_32bit
@@ -136,37 +136,37 @@ define i64 @test_i64_args_8(i64 %arg1, i64 %arg2, i64 %arg3, i64 %arg4,
   ; X32-LABEL: name: test_i64_args_8
   ; X32: bb.1.entry:
   ; X32:   [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.15
-  ; X32:   [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX]](p0) :: (invariant load 4 from %fixed-stack.15, align 0)
+  ; X32:   [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX]](p0) :: (invariant load 4 from %fixed-stack.15, align 1)
   ; X32:   [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.14
-  ; X32:   [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX1]](p0) :: (invariant load 4 from %fixed-stack.14, align 0)
+  ; X32:   [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX1]](p0) :: (invariant load 4 from %fixed-stack.14, align 1)
   ; X32:   [[FRAME_INDEX2:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.13
-  ; X32:   [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX2]](p0) :: (invariant load 4 from %fixed-stack.13, align 0)
+  ; X32:   [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX2]](p0) :: (invariant load 4 from %fixed-stack.13, align 1)
   ; X32:   [[FRAME_INDEX3:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.12
-  ; X32:   [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX3]](p0) :: (invariant load 4 from %fixed-stack.12, align 0)
+  ; X32:   [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX3]](p0) :: (invariant load 4 from %fixed-stack.12, align 1)
   ; X32:   [[FRAME_INDEX4:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.11
-  ; X32:   [[LOAD4:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX4]](p0) :: (invariant load 4 from %fixed-stack.11, align 0)
+  ; X32:   [[LOAD4:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX4]](p0) :: (invariant load 4 from %fixed-stack.11, align 1)
   ; X32:   [[FRAME_INDEX5:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.10
-  ; X32:   [[LOAD5:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX5]](p0) :: (invariant load 4 from %fixed-stack.10, align 0)
+  ; X32:   [[LOAD5:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX5]](p0) :: (invariant load 4 from %fixed-stack.10, align 1)
   ; X32:   [[FRAME_INDEX6:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.9
-  ; X32:   [[LOAD6:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX6]](p0) :: (invariant load 4 from %fixed-stack.9, align 0)
+  ; X32:   [[LOAD6:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX6]](p0) :: (invariant load 4 from %fixed-stack.9, align 1)
   ; X32:   [[FRAME_INDEX7:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.8
-  ; X32:   [[LOAD7:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX7]](p0) :: (invariant load 4 from %fixed-stack.8, align 0)
+  ; X32:   [[LOAD7:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX7]](p0) :: (invariant load 4 from %fixed-stack.8, align 1)
   ; X32:   [[FRAME_INDEX8:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.7
-  ; X32:   [[LOAD8:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX8]](p0) :: (invariant load 4 from %fixed-stack.7, align 0)
+  ; X32:   [[LOAD8:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX8]](p0) :: (invariant load 4 from %fixed-stack.7, align 1)
   ; X32:   [[FRAME_INDEX9:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.6
-  ; X32:   [[LOAD9:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX9]](p0) :: (invariant load 4 from %fixed-stack.6, align 0)
+  ; X32:   [[LOAD9:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX9]](p0) :: (invariant load 4 from %fixed-stack.6, align 1)
   ; X32:   [[FRAME_INDEX10:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.5
-  ; X32:   [[LOAD10:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX10]](p0) :: (invariant load 4 from %fixed-stack.5, align 0)
+  ; X32:   [[LOAD10:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX10]](p0) :: (invariant load 4 from %fixed-stack.5, align 1)
   ; X32:   [[FRAME_INDEX11:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.4
-  ; X32:   [[LOAD11:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX11]](p0) :: (invariant load 4 from %fixed-stack.4, align 0)
+  ; X32:   [[LOAD11:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX11]](p0) :: (invariant load 4 from %fixed-stack.4, align 1)
   ; X32:   [[FRAME_INDEX12:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.3
-  ; X32:   [[LOAD12:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX12]](p0) :: (invariant load 4 from %fixed-stack.3, align 0)
+  ; X32:   [[LOAD12:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX12]](p0) :: (invariant load 4 from %fixed-stack.3, align 1)
   ; X32:   [[FRAME_INDEX13:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.2
-  ; X32:   [[LOAD13:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX13]](p0) :: (invariant load 4 from %fixed-stack.2, align 0)
+  ; X32:   [[LOAD13:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX13]](p0) :: (invariant load 4 from %fixed-stack.2, align 1)
   ; X32:   [[FRAME_INDEX14:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.1
-  ; X32:   [[LOAD14:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX14]](p0) :: (invariant load 4 from %fixed-stack.1, align 0)
+  ; X32:   [[LOAD14:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX14]](p0) :: (invariant load 4 from %fixed-stack.1, align 1)
   ; X32:   [[FRAME_INDEX15:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.0
-  ; X32:   [[LOAD15:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX15]](p0) :: (invariant load 4 from %fixed-stack.0, align 0)
+  ; X32:   [[LOAD15:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX15]](p0) :: (invariant load 4 from %fixed-stack.0, align 1)
   ; X32:   [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD]](s32), [[LOAD1]](s32)
   ; X32:   [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD2]](s32), [[LOAD3]](s32)
   ; X32:   [[MV2:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[LOAD4]](s32), [[LOAD5]](s32)
@@ -195,9 +195,9 @@ define i64 @test_i64_args_8(i64 %arg1, i64 %arg2, i64 %arg3, i64 %arg4,
   ; X64:   [[COPY4:%[0-9]+]]:_(s64) = COPY $r8
   ; X64:   [[COPY5:%[0-9]+]]:_(s64) = COPY $r9
   ; X64:   [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.1
-  ; X64:   [[LOAD:%[0-9]+]]:_(s64) = G_LOAD [[FRAME_INDEX]](p0) :: (invariant load 8 from %fixed-stack.1, align 0)
+  ; X64:   [[LOAD:%[0-9]+]]:_(s64) = G_LOAD [[FRAME_INDEX]](p0) :: (invariant load 8 from %fixed-stack.1, align 1)
   ; X64:   [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.0
-  ; X64:   [[LOAD1:%[0-9]+]]:_(s64) = G_LOAD [[FRAME_INDEX1]](p0) :: (invariant load 8 from %fixed-stack.0, align 0)
+  ; X64:   [[LOAD1:%[0-9]+]]:_(s64) = G_LOAD [[FRAME_INDEX1]](p0) :: (invariant load 8 from %fixed-stack.0, align 1)
   ; X64:   [[GV:%[0-9]+]]:_(p0) = G_GLOBAL_VALUE @a1_64bit
   ; X64:   [[GV1:%[0-9]+]]:_(p0) = G_GLOBAL_VALUE @a7_64bit
   ; X64:   [[GV2:%[0-9]+]]:_(p0) = G_GLOBAL_VALUE @a8_64bit
@@ -219,9 +219,9 @@ define float @test_float_args(float %arg1, float %arg2) {
   ; X32-LABEL: name: test_float_args
   ; X32: bb.1 (%ir-block.0):
   ; X32:   [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.1
-  ; X32:   [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX]](p0) :: (invariant load 4 from %fixed-stack.1, align 0)
+  ; X32:   [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX]](p0) :: (invariant load 4 from %fixed-stack.1, align 1)
   ; X32:   [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.0
-  ; X32:   [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX1]](p0) :: (invariant load 4 from %fixed-stack.0, align 0)
+  ; X32:   [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX1]](p0) :: (invariant load 4 from %fixed-stack.0, align 1)
   ; X32:   [[ANYEXT:%[0-9]+]]:_(s80) = G_ANYEXT [[LOAD1]](s32)
   ; X32:   $fp0 = COPY [[ANYEXT]](s80)
   ; X32:   RET 0, implicit $fp0
@@ -242,9 +242,9 @@ define double @test_double_args(double %arg1, double %arg2) {
   ; X32-LABEL: name: test_double_args
   ; X32: bb.1 (%ir-block.0):
   ; X32:   [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.1
-  ; X32:   [[LOAD:%[0-9]+]]:_(s64) = G_LOAD [[FRAME_INDEX]](p0) :: (invariant load 8 from %fixed-stack.1, align 0)
+  ; X32:   [[LOAD:%[0-9]+]]:_(s64) = G_LOAD [[FRAME_INDEX]](p0) :: (invariant load 8 from %fixed-stack.1, align 1)
   ; X32:   [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.0
-  ; X32:   [[LOAD1:%[0-9]+]]:_(s64) = G_LOAD [[FRAME_INDEX1]](p0) :: (invariant load 8 from %fixed-stack.0, align 0)
+  ; X32:   [[LOAD1:%[0-9]+]]:_(s64) = G_LOAD [[FRAME_INDEX1]](p0) :: (invariant load 8 from %fixed-stack.0, align 1)
   ; X32:   [[ANYEXT:%[0-9]+]]:_(s80) = G_ANYEXT [[LOAD1]](s64)
   ; X32:   $fp0 = COPY [[ANYEXT]](s80)
   ; X32:   RET 0, implicit $fp0
@@ -319,7 +319,7 @@ define i32 * @test_memop_i32(i32 * %p1) {
   ; X32-LABEL: name: test_memop_i32
   ; X32: bb.1 (%ir-block.0):
   ; X32:   [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.0
-  ; X32:   [[LOAD:%[0-9]+]]:_(p0) = G_LOAD [[FRAME_INDEX]](p0) :: (invariant load 4 from %fixed-stack.0, align 0)
+  ; X32:   [[LOAD:%[0-9]+]]:_(p0) = G_LOAD [[FRAME_INDEX]](p0) :: (invariant load 4 from %fixed-stack.0, align 1)
   ; X32:   $eax = COPY [[LOAD]](p0)
   ; X32:   RET 0, implicit $eax
   ; X64-LABEL: name: test_memop_i32
@@ -354,18 +354,18 @@ define void @test_simple_arg(i32 %in0, i32 %in1) {
   ; X32-LABEL: name: test_simple_arg
   ; X32: bb.1 (%ir-block.0):
   ; X32:   [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.1
-  ; X32:   [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX]](p0) :: (invariant load 4 from %fixed-stack.1, align 0)
+  ; X32:   [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX]](p0) :: (invariant load 4 from %fixed-stack.1, align 1)
   ; X32:   [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.0
-  ; X32:   [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX1]](p0) :: (invariant load 4 from %fixed-stack.0, align 0)
+  ; X32:   [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX1]](p0) :: (invariant load 4 from %fixed-stack.0, align 1)
   ; X32:   ADJCALLSTACKDOWN32 8, 0, 0, implicit-def $esp, implicit-def $eflags, implicit-def $ssp, implicit $esp, implicit $ssp
   ; X32:   [[COPY:%[0-9]+]]:_(p0) = COPY $esp
   ; X32:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; X32:   [[GEP:%[0-9]+]]:_(p0) = G_GEP [[COPY]], [[C]](s32)
-  ; X32:   G_STORE [[LOAD1]](s32), [[GEP]](p0) :: (store 4 into stack, align 0)
+  ; X32:   G_STORE [[LOAD1]](s32), [[GEP]](p0) :: (store 4 into stack, align 1)
   ; X32:   [[COPY1:%[0-9]+]]:_(p0) = COPY $esp
   ; X32:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
   ; X32:   [[GEP1:%[0-9]+]]:_(p0) = G_GEP [[COPY1]], [[C1]](s32)
-  ; X32:   G_STORE [[LOAD]](s32), [[GEP1]](p0) :: (store 4 into stack + 4, align 0)
+  ; X32:   G_STORE [[LOAD]](s32), [[GEP1]](p0) :: (store 4 into stack + 4, align 1)
   ; X32:   CALLpcrel32 @simple_arg_callee, csr_32, implicit $esp, implicit $ssp
   ; X32:   ADJCALLSTACKUP32 8, 0, implicit-def $esp, implicit-def $eflags, implicit-def $ssp, implicit $esp, implicit $ssp
   ; X32:   RET 0
@@ -389,40 +389,40 @@ define void @test_simple_arg8_call(i32 %in0) {
   ; X32-LABEL: name: test_simple_arg8_call
   ; X32: bb.1 (%ir-block.0):
   ; X32:   [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.0
-  ; X32:   [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX]](p0) :: (invariant load 4 from %fixed-stack.0, align 0)
+  ; X32:   [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX]](p0) :: (invariant load 4 from %fixed-stack.0, align 1)
   ; X32:   ADJCALLSTACKDOWN32 32, 0, 0, implicit-def $esp, implicit-def $eflags, implicit-def $ssp, implicit $esp, implicit $ssp
   ; X32:   [[COPY:%[0-9]+]]:_(p0) = COPY $esp
   ; X32:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; X32:   [[GEP:%[0-9]+]]:_(p0) = G_GEP [[COPY]], [[C]](s32)
-  ; X32:   G_STORE [[LOAD]](s32), [[GEP]](p0) :: (store 4 into stack, align 0)
+  ; X32:   G_STORE [[LOAD]](s32), [[GEP]](p0) :: (store 4 into stack, align 1)
   ; X32:   [[COPY1:%[0-9]+]]:_(p0) = COPY $esp
   ; X32:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
   ; X32:   [[GEP1:%[0-9]+]]:_(p0) = G_GEP [[COPY1]], [[C1]](s32)
-  ; X32:   G_STORE [[LOAD]](s32), [[GEP1]](p0) :: (store 4 into stack + 4, align 0)
+  ; X32:   G_STORE [[LOAD]](s32), [[GEP1]](p0) :: (store 4 into stack + 4, align 1)
   ; X32:   [[COPY2:%[0-9]+]]:_(p0) = COPY $esp
   ; X32:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
   ; X32:   [[GEP2:%[0-9]+]]:_(p0) = G_GEP [[COPY2]], [[C2]](s32)
-  ; X32:   G_STORE [[LOAD]](s32), [[GEP2]](p0) :: (store 4 into stack + 8, align 0)
+  ; X32:   G_STORE [[LOAD]](s32), [[GEP2]](p0) :: (store 4 into stack + 8, align 1)
   ; X32:   [[COPY3:%[0-9]+]]:_(p0) = COPY $esp
   ; X32:   [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 12
   ; X32:   [[GEP3:%[0-9]+]]:_(p0) = G_GEP [[COPY3]], [[C3]](s32)
-  ; X32:   G_STORE [[LOAD]](s32), [[GEP3]](p0) :: (store 4 into stack + 12, align 0)
+  ; X32:   G_STORE [[LOAD]](s32), [[GEP3]](p0) :: (store 4 into stack + 12, align 1)
   ; X32:   [[COPY4:%[0-9]+]]:_(p0) = COPY $esp
   ; X32:   [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
   ; X32:   [[GEP4:%[0-9]+]]:_(p0) = G_GEP [[COPY4]], [[C4]](s32)
-  ; X32:   G_STORE [[LOAD]](s32), [[GEP4]](p0) :: (store 4 into stack + 16, align 0)
+  ; X32:   G_STORE [[LOAD]](s32), [[GEP4]](p0) :: (store 4 into stack + 16, align 1)
   ; X32:   [[COPY5:%[0-9]+]]:_(p0) = COPY $esp
   ; X32:   [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
   ; X32:   [[GEP5:%[0-9]+]]:_(p0) = G_GEP [[COPY5]], [[C5]](s32)
-  ; X32:   G_STORE [[LOAD]](s32), [[GEP5]](p0) :: (store 4 into stack + 20, align 0)
+  ; X32:   G_STORE [[LOAD]](s32), [[GEP5]](p0) :: (store 4 into stack + 20, align 1)
   ; X32:   [[COPY6:%[0-9]+]]:_(p0) = COPY $esp
   ; X32:   [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
   ; X32:   [[GEP6:%[0-9]+]]:_(p0) = G_GEP [[COPY6]], [[C6]](s32)
-  ; X32:   G_STORE [[LOAD]](s32), [[GEP6]](p0) :: (store 4 into stack + 24, align 0)
+  ; X32:   G_STORE [[LOAD]](s32), [[GEP6]](p0) :: (store 4 into stack + 24, align 1)
   ; X32:   [[COPY7:%[0-9]+]]:_(p0) = COPY $esp
   ; X32:   [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 28
   ; X32:   [[GEP7:%[0-9]+]]:_(p0) = G_GEP [[COPY7]], [[C7]](s32)
-  ; X32:   G_STORE [[LOAD]](s32), [[GEP7]](p0) :: (store 4 into stack + 28, align 0)
+  ; X32:   G_STORE [[LOAD]](s32), [[GEP7]](p0) :: (store 4 into stack + 28, align 1)
   ; X32:   CALLpcrel32 @simple_arg8_callee, csr_32, implicit $esp, implicit $ssp
   ; X32:   ADJCALLSTACKUP32 32, 0, implicit-def $esp, implicit-def $eflags, implicit-def $ssp, implicit $esp, implicit $ssp
   ; X32:   RET 0
@@ -440,11 +440,11 @@ define void @test_simple_arg8_call(i32 %in0) {
   ; X64:   [[COPY1:%[0-9]+]]:_(p0) = COPY $rsp
   ; X64:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
   ; X64:   [[GEP:%[0-9]+]]:_(p0) = G_GEP [[COPY1]], [[C]](s64)
-  ; X64:   G_STORE [[COPY]](s32), [[GEP]](p0) :: (store 4 into stack, align 0)
+  ; X64:   G_STORE [[COPY]](s32), [[GEP]](p0) :: (store 4 into stack, align 1)
   ; X64:   [[COPY2:%[0-9]+]]:_(p0) = COPY $rsp
   ; X64:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
   ; X64:   [[GEP1:%[0-9]+]]:_(p0) = G_GEP [[COPY2]], [[C1]](s64)
-  ; X64:   G_STORE [[COPY]](s32), [[GEP1]](p0) :: (store 4 into stack + 8, align 0)
+  ; X64:   G_STORE [[COPY]](s32), [[GEP1]](p0) :: (store 4 into stack + 8, align 1)
   ; X64:   CALL64pcrel32 @simple_arg8_callee, csr_64, implicit $rsp, implicit $ssp, implicit $edi, implicit $esi, implicit $edx, implicit $ecx, implicit $r8d, implicit $r9d
   ; X64:   ADJCALLSTACKUP64 16, 0, implicit-def $rsp, implicit-def $eflags, implicit-def $ssp, implicit $rsp, implicit $ssp
   ; X64:   RET 0
@@ -461,7 +461,7 @@ define i32 @test_simple_return_callee() {
   ; X32:   [[COPY:%[0-9]+]]:_(p0) = COPY $esp
   ; X32:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; X32:   [[GEP:%[0-9]+]]:_(p0) = G_GEP [[COPY]], [[C1]](s32)
-  ; X32:   G_STORE [[C]](s32), [[GEP]](p0) :: (store 4 into stack, align 0)
+  ; X32:   G_STORE [[C]](s32), [[GEP]](p0) :: (store 4 into stack, align 1)
   ; X32:   CALLpcrel32 @simple_return_callee, csr_32, implicit $esp, implicit $ssp, implicit-def $eax
   ; X32:   [[COPY1:%[0-9]+]]:_(s32) = COPY $eax
   ; X32:   ADJCALLSTACKUP32 4, 0, implicit-def $esp, implicit-def $eflags, implicit-def $ssp, implicit $esp, implicit $ssp
@@ -493,7 +493,7 @@ define <8 x i32> @test_split_return_callee(<8 x i32> %arg1, <8 x i32> %arg2) {
   ; X32:   [[COPY1:%[0-9]+]]:_(<4 x s32>) = COPY $xmm1
   ; X32:   [[COPY2:%[0-9]+]]:_(<4 x s32>) = COPY $xmm2
   ; X32:   [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.0
-  ; X32:   [[LOAD:%[0-9]+]]:_(<4 x s32>) = G_LOAD [[FRAME_INDEX]](p0) :: (invariant load 16 from %fixed-stack.0, align 0)
+  ; X32:   [[LOAD:%[0-9]+]]:_(<4 x s32>) = G_LOAD [[FRAME_INDEX]](p0) :: (invariant load 16 from %fixed-stack.0, align 1)
   ; X32:   [[MV:%[0-9]+]]:_(<8 x s32>) = G_CONCAT_VECTORS [[COPY]](<4 x s32>), [[COPY1]](<4 x s32>)
   ; X32:   [[MV1:%[0-9]+]]:_(<8 x s32>) = G_CONCAT_VECTORS [[COPY2]](<4 x s32>), [[LOAD]](<4 x s32>)
   ; X32:   ADJCALLSTACKDOWN32 0, 0, 0, implicit-def $esp, implicit-def $eflags, implicit-def $ssp, implicit $esp, implicit $ssp
@@ -542,7 +542,7 @@ define void @test_indirect_call(void()* %func) {
   ; X32-LABEL: name: test_indirect_call
   ; X32: bb.1 (%ir-block.0):
   ; X32:   [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.0
-  ; X32:   [[LOAD:%[0-9]+]]:gr32(p0) = G_LOAD [[FRAME_INDEX]](p0) :: (invariant load 4 from %fixed-stack.0, align 0)
+  ; X32:   [[LOAD:%[0-9]+]]:gr32(p0) = G_LOAD [[FRAME_INDEX]](p0) :: (invariant load 4 from %fixed-stack.0, align 1)
   ; X32:   ADJCALLSTACKDOWN32 0, 0, 0, implicit-def $esp, implicit-def $eflags, implicit-def $ssp, implicit $esp, implicit $ssp
   ; X32:   CALL32r [[LOAD]](p0), csr_32, implicit $esp, implicit $ssp
   ; X32:   ADJCALLSTACKUP32 0, 0, implicit-def $esp, implicit-def $eflags, implicit-def $ssp, implicit $esp, implicit $ssp
@@ -565,14 +565,14 @@ define void @test_abi_exts_call(i8* %addr) {
   ; X32-LABEL: name: test_abi_exts_call
   ; X32: bb.1 (%ir-block.0):
   ; X32:   [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.0
-  ; X32:   [[LOAD:%[0-9]+]]:_(p0) = G_LOAD [[FRAME_INDEX]](p0) :: (invariant load 4 from %fixed-stack.0, align 0)
+  ; X32:   [[LOAD:%[0-9]+]]:_(p0) = G_LOAD [[FRAME_INDEX]](p0) :: (invariant load 4 from %fixed-stack.0, align 1)
   ; X32:   [[LOAD1:%[0-9]+]]:_(s8) = G_LOAD [[LOAD]](p0) :: (load 1 from %ir.addr)
   ; X32:   ADJCALLSTACKDOWN32 4, 0, 0, implicit-def $esp, implicit-def $eflags, implicit-def $ssp, implicit $esp, implicit $ssp
   ; X32:   [[COPY:%[0-9]+]]:_(p0) = COPY $esp
   ; X32:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; X32:   [[GEP:%[0-9]+]]:_(p0) = G_GEP [[COPY]], [[C]](s32)
   ; X32:   [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[LOAD1]](s8)
-  ; X32:   G_STORE [[ANYEXT]](s32), [[GEP]](p0) :: (store 4 into stack, align 0)
+  ; X32:   G_STORE [[ANYEXT]](s32), [[GEP]](p0) :: (store 4 into stack, align 1)
   ; X32:   CALLpcrel32 @take_char, csr_32, implicit $esp, implicit $ssp
   ; X32:   ADJCALLSTACKUP32 4, 0, implicit-def $esp, implicit-def $eflags, implicit-def $ssp, implicit $esp, implicit $ssp
   ; X32:   ADJCALLSTACKDOWN32 4, 0, 0, implicit-def $esp, implicit-def $eflags, implicit-def $ssp, implicit $esp, implicit $ssp
@@ -580,7 +580,7 @@ define void @test_abi_exts_call(i8* %addr) {
   ; X32:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; X32:   [[GEP1:%[0-9]+]]:_(p0) = G_GEP [[COPY1]], [[C1]](s32)
   ; X32:   [[SEXT:%[0-9]+]]:_(s32) = G_SEXT [[LOAD1]](s8)
-  ; X32:   G_STORE [[SEXT]](s32), [[GEP1]](p0) :: (store 4 into stack, align 0)
+  ; X32:   G_STORE [[SEXT]](s32), [[GEP1]](p0) :: (store 4 into stack, align 1)
   ; X32:   CALLpcrel32 @take_char, csr_32, implicit $esp, implicit $ssp
   ; X32:   ADJCALLSTACKUP32 4, 0, implicit-def $esp, implicit-def $eflags, implicit-def $ssp, implicit $esp, implicit $ssp
   ; X32:   ADJCALLSTACKDOWN32 4, 0, 0, implicit-def $esp, implicit-def $eflags, implicit-def $ssp, implicit $esp, implicit $ssp
@@ -588,7 +588,7 @@ define void @test_abi_exts_call(i8* %addr) {
   ; X32:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; X32:   [[GEP2:%[0-9]+]]:_(p0) = G_GEP [[COPY2]], [[C2]](s32)
   ; X32:   [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[LOAD1]](s8)
-  ; X32:   G_STORE [[ZEXT]](s32), [[GEP2]](p0) :: (store 4 into stack, align 0)
+  ; X32:   G_STORE [[ZEXT]](s32), [[GEP2]](p0) :: (store 4 into stack, align 1)
   ; X32:   CALLpcrel32 @take_char, csr_32, implicit $esp, implicit $ssp
   ; X32:   ADJCALLSTACKUP32 4, 0, implicit-def $esp, implicit-def $eflags, implicit-def $ssp, implicit $esp, implicit $ssp
   ; X32:   RET 0
@@ -625,20 +625,20 @@ define void @test_variadic_call_1(i8** %addr_ptr, i32* %val_ptr) {
   ; X32-LABEL: name: test_variadic_call_1
   ; X32: bb.1 (%ir-block.0):
   ; X32:   [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.1
-  ; X32:   [[LOAD:%[0-9]+]]:_(p0) = G_LOAD [[FRAME_INDEX]](p0) :: (invariant load 4 from %fixed-stack.1, align 0)
+  ; X32:   [[LOAD:%[0-9]+]]:_(p0) = G_LOAD [[FRAME_INDEX]](p0) :: (invariant load 4 from %fixed-stack.1, align 1)
   ; X32:   [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.0
-  ; X32:   [[LOAD1:%[0-9]+]]:_(p0) = G_LOAD [[FRAME_INDEX1]](p0) :: (invariant load 4 from %fixed-stack.0, align 0)
+  ; X32:   [[LOAD1:%[0-9]+]]:_(p0) = G_LOAD [[FRAME_INDEX1]](p0) :: (invariant load 4 from %fixed-stack.0, align 1)
   ; X32:   [[LOAD2:%[0-9]+]]:_(p0) = G_LOAD [[LOAD]](p0) :: (load 4 from %ir.addr_ptr)
   ; X32:   [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[LOAD1]](p0) :: (load 4 from %ir.val_ptr)
   ; X32:   ADJCALLSTACKDOWN32 8, 0, 0, implicit-def $esp, implicit-def $eflags, implicit-def $ssp, implicit $esp, implicit $ssp
   ; X32:   [[COPY:%[0-9]+]]:_(p0) = COPY $esp
   ; X32:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; X32:   [[GEP:%[0-9]+]]:_(p0) = G_GEP [[COPY]], [[C]](s32)
-  ; X32:   G_STORE [[LOAD2]](p0), [[GEP]](p0) :: (store 4 into stack, align 0)
+  ; X32:   G_STORE [[LOAD2]](p0), [[GEP]](p0) :: (store 4 into stack, align 1)
   ; X32:   [[COPY1:%[0-9]+]]:_(p0) = COPY $esp
   ; X32:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
   ; X32:   [[GEP1:%[0-9]+]]:_(p0) = G_GEP [[COPY1]], [[C1]](s32)
-  ; X32:   G_STORE [[LOAD3]](s32), [[GEP1]](p0) :: (store 4 into stack + 4, align 0)
+  ; X32:   G_STORE [[LOAD3]](s32), [[GEP1]](p0) :: (store 4 into stack + 4, align 1)
   ; X32:   CALLpcrel32 @variadic_callee, csr_32, implicit $esp, implicit $ssp
   ; X32:   ADJCALLSTACKUP32 8, 0, implicit-def $esp, implicit-def $eflags, implicit-def $ssp, implicit $esp, implicit $ssp
   ; X32:   RET 0
@@ -666,20 +666,20 @@ define void @test_variadic_call_2(i8** %addr_ptr, double* %val_ptr) {
   ; X32-LABEL: name: test_variadic_call_2
   ; X32: bb.1 (%ir-block.0):
   ; X32:   [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.1
-  ; X32:   [[LOAD:%[0-9]+]]:_(p0) = G_LOAD [[FRAME_INDEX]](p0) :: (invariant load 4 from %fixed-stack.1, align 0)
+  ; X32:   [[LOAD:%[0-9]+]]:_(p0) = G_LOAD [[FRAME_INDEX]](p0) :: (invariant load 4 from %fixed-stack.1, align 1)
   ; X32:   [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.0
-  ; X32:   [[LOAD1:%[0-9]+]]:_(p0) = G_LOAD [[FRAME_INDEX1]](p0) :: (invariant load 4 from %fixed-stack.0, align 0)
+  ; X32:   [[LOAD1:%[0-9]+]]:_(p0) = G_LOAD [[FRAME_INDEX1]](p0) :: (invariant load 4 from %fixed-stack.0, align 1)
   ; X32:   [[LOAD2:%[0-9]+]]:_(p0) = G_LOAD [[LOAD]](p0) :: (load 4 from %ir.addr_ptr)
   ; X32:   [[LOAD3:%[0-9]+]]:_(s64) = G_LOAD [[LOAD1]](p0) :: (load 8 from %ir.val_ptr, align 4)
   ; X32:   ADJCALLSTACKDOWN32 12, 0, 0, implicit-def $esp, implicit-def $eflags, implicit-def $ssp, implicit $esp, implicit $ssp
   ; X32:   [[COPY:%[0-9]+]]:_(p0) = COPY $esp
   ; X32:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; X32:   [[GEP:%[0-9]+]]:_(p0) = G_GEP [[COPY]], [[C]](s32)
-  ; X32:   G_STORE [[LOAD2]](p0), [[GEP]](p0) :: (store 4 into stack, align 0)
+  ; X32:   G_STORE [[LOAD2]](p0), [[GEP]](p0) :: (store 4 into stack, align 1)
   ; X32:   [[COPY1:%[0-9]+]]:_(p0) = COPY $esp
   ; X32:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
   ; X32:   [[GEP1:%[0-9]+]]:_(p0) = G_GEP [[COPY1]], [[C1]](s32)
-  ; X32:   G_STORE [[LOAD3]](s64), [[GEP1]](p0) :: (store 8 into stack + 4, align 0)
+  ; X32:   G_STORE [[LOAD3]](s64), [[GEP1]](p0) :: (store 8 into stack + 4, align 1)
   ; X32:   CALLpcrel32 @variadic_callee, csr_32, implicit $esp, implicit $ssp
   ; X32:   ADJCALLSTACKUP32 12, 0, implicit-def $esp, implicit-def $eflags, implicit-def $ssp, implicit $esp, implicit $ssp
   ; X32:   RET 0
diff --git a/test/CodeGen/X86/GlobalISel/legalize-add.mir b/test/CodeGen/X86/GlobalISel/legalize-add.mir
index 922abcfaa9781eae981e2011b525f66970710890..ac8222001eb9cc1cfd51eb19280c98459647c05c 100644
--- a/test/CodeGen/X86/GlobalISel/legalize-add.mir
+++ b/test/CodeGen/X86/GlobalISel/legalize-add.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=x86_64-linux-gnu -run-pass=legalizer %s -o - | FileCheck %s --check-prefix=ALL --check-prefix=X64
-# RUN: llc -mtriple=i386-linux-gnu   -run-pass=legalizer %s -o - | FileCheck %s --check-prefix=ALL --check-prefix=X32
+# RUN: llc -O0 -mtriple=x86_64-linux-gnu -run-pass=legalizer %s -o - | FileCheck %s --check-prefix=ALL --check-prefix=X64
+# RUN: llc -O0 -mtriple=i386-linux-gnu   -run-pass=legalizer %s -o - | FileCheck %s --check-prefix=ALL --check-prefix=X32
 --- |
 
   define void @test_add_i1() { ret void}
diff --git a/test/CodeGen/X86/GlobalISel/legalize-and-scalar.mir b/test/CodeGen/X86/GlobalISel/legalize-and-scalar.mir
index 4b73fce680d4504427d41d8801f1be42b325cf52..abde7a28c14efaf8d005f29847edaa80584a5e2a 100644
--- a/test/CodeGen/X86/GlobalISel/legalize-and-scalar.mir
+++ b/test/CodeGen/X86/GlobalISel/legalize-and-scalar.mir
@@ -1,5 +1,5 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=x86_64-linux-gnu -run-pass=legalizer %s -o - | FileCheck %s
+# RUN: llc -O0 -mtriple=x86_64-linux-gnu -run-pass=legalizer %s -o - | FileCheck %s
 
 --- |
   define i1 @test_and_i1() {
diff --git a/test/CodeGen/X86/GlobalISel/legalize-ext-x86-64.mir b/test/CodeGen/X86/GlobalISel/legalize-ext-x86-64.mir
index 6c9af4d2fb2632bd938f84e223c03aaef0b4a65f..8685b058aeabebdbd8690c4188ab6c50fa97e8ce 100644
--- a/test/CodeGen/X86/GlobalISel/legalize-ext-x86-64.mir
+++ b/test/CodeGen/X86/GlobalISel/legalize-ext-x86-64.mir
@@ -1,5 +1,5 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=x86_64-linux-gnu -run-pass=legalizer %s -o - | FileCheck %s
+# RUN: llc -O0 -mtriple=x86_64-linux-gnu -run-pass=legalizer %s -o - | FileCheck %s
 
 --- |
   define i64 @test_sext_i1(i8 %a) {
@@ -79,8 +79,10 @@ body:             |
     ; CHECK: [[COPY:%[0-9]+]]:_(s8) = COPY $dil
     ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 63
     ; CHECK: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[COPY]](s8)
-    ; CHECK: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[ANYEXT]], [[C]]
-    ; CHECK: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[SHL]], [[C]]
+    ; CHECK: [[TRUNC:%[0-9]+]]:_(s8) = G_TRUNC [[C]](s64)
+    ; CHECK: [[SHL:%[0-9]+]]:_(s64) = G_SHL [[ANYEXT]], [[TRUNC]](s8)
+    ; CHECK: [[TRUNC1:%[0-9]+]]:_(s8) = G_TRUNC [[C]](s64)
+    ; CHECK: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[SHL]], [[TRUNC1]](s8)
     ; CHECK: $rax = COPY [[ASHR]](s64)
     ; CHECK: RET 0, implicit $rax
     %0(s8) = COPY $dil
diff --git a/test/CodeGen/AArch64/GlobalISel/legalize-fneg.mir b/test/CodeGen/X86/GlobalISel/legalize-fneg.mir
similarity index 60%
rename from test/CodeGen/AArch64/GlobalISel/legalize-fneg.mir
rename to test/CodeGen/X86/GlobalISel/legalize-fneg.mir
index 59de652d7446f839397afe6d7e6ded859400921c..dabe3acc93b4ea04282828bd4ee4c7e31937b669 100644
--- a/test/CodeGen/AArch64/GlobalISel/legalize-fneg.mir
+++ b/test/CodeGen/X86/GlobalISel/legalize-fneg.mir
@@ -2,8 +2,7 @@
 # RUN: llc -O0 -run-pass=legalizer %s -o - | FileCheck %s
 
 --- |
-  target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
-  target triple = "aarch64--"
+  target triple = "x86_64--"
   define void @test_fneg_f32() {
   entry:
     ret void
@@ -20,15 +19,15 @@ registers:
   - { id: 1, class: _ }
 body:             |
   bb.1:
-    liveins: $s0
+    liveins:
     ; CHECK-LABEL: name: test_fneg_f32
-    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $s0
+    ; CHECK: [[DEF:%[0-9]+]]:_(s32) = IMPLICIT_DEF
     ; CHECK: [[C:%[0-9]+]]:_(s32) = G_FCONSTANT float -0.000000e+00
-    ; CHECK: [[FSUB:%[0-9]+]]:_(s32) = G_FSUB [[C]], [[COPY]]
-    ; CHECK: $s0 = COPY [[FSUB]](s32)
-    %0(s32) = COPY $s0
+    ; CHECK: [[FSUB:%[0-9]+]]:_(s32) = G_FSUB [[C]], [[DEF]]
+    ; CHECK: $edi = COPY [[FSUB]](s32)
+    %0(s32) = IMPLICIT_DEF
     %1(s32) = G_FNEG %0
-    $s0 = COPY %1(s32)
+    $edi = COPY %1
 ...
 ---
 name:            test_fneg_f64
@@ -37,13 +36,13 @@ registers:
   - { id: 1, class: _ }
 body:             |
   bb.1:
-    liveins: $d0
+    liveins:
     ; CHECK-LABEL: name: test_fneg_f64
-    ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY $d0
+    ; CHECK: [[DEF:%[0-9]+]]:_(s64) = G_IMPLICIT_DEF
     ; CHECK: [[C:%[0-9]+]]:_(s64) = G_FCONSTANT double -0.000000e+00
-    ; CHECK: [[FSUB:%[0-9]+]]:_(s64) = G_FSUB [[C]], [[COPY]]
-    ; CHECK: $d0 = COPY [[FSUB]](s64)
-    %0(s64) = COPY $d0
+    ; CHECK: [[FSUB:%[0-9]+]]:_(s64) = G_FSUB [[C]], [[DEF]]
+    ; CHECK: $rdi = COPY [[FSUB]](s64)
+    %0(s64) = G_IMPLICIT_DEF
     %1(s64) = G_FNEG %0
-    $d0 = COPY %1(s64)
+    $rdi = COPY %1
 ...
diff --git a/test/CodeGen/X86/GlobalISel/legalize-memop-scalar-32.mir b/test/CodeGen/X86/GlobalISel/legalize-memop-scalar-32.mir
new file mode 100644
index 0000000000000000000000000000000000000000..7b08c1130ee8d6b2cbe6c0fad31f572bf8504e70
--- /dev/null
+++ b/test/CodeGen/X86/GlobalISel/legalize-memop-scalar-32.mir
@@ -0,0 +1,63 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -O0 -mtriple=i386-linux-gnu   -run-pass=legalizer %s -o - | FileCheck --check-prefix=X32 %s
+
+---
+name:            test_memop_s8tos32
+alignment:       4
+legalized:       false
+regBankSelected: false
+body:             |
+  bb.0:
+    ; X32-LABEL: name: test_memop_s8tos32
+    ; X32: [[DEF:%[0-9]+]]:_(p0) = IMPLICIT_DEF
+    ; X32: [[LOAD:%[0-9]+]]:_(s8) = G_LOAD [[DEF]](p0) :: (load 1)
+    ; X32: [[LOAD1:%[0-9]+]]:_(s8) = G_LOAD [[DEF]](p0) :: (load 1)
+    ; X32: [[LOAD2:%[0-9]+]]:_(s16) = G_LOAD [[DEF]](p0) :: (load 2)
+    ; X32: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[DEF]](p0) :: (load 4)
+    ; X32: [[LOAD4:%[0-9]+]]:_(p0) = G_LOAD [[DEF]](p0) :: (load 4)
+    ; X32: [[C:%[0-9]+]]:_(s8) = G_CONSTANT i8 1
+    ; X32: [[COPY:%[0-9]+]]:_(s8) = COPY [[LOAD]](s8)
+    ; X32: [[AND:%[0-9]+]]:_(s8) = G_AND [[COPY]], [[C]]
+    ; X32: G_STORE [[AND]](s8), [[DEF]](p0) :: (store 1)
+    ; X32: G_STORE [[LOAD1]](s8), [[DEF]](p0) :: (store 1)
+    ; X32: G_STORE [[LOAD2]](s16), [[DEF]](p0) :: (store 2)
+    ; X32: G_STORE [[LOAD3]](s32), [[DEF]](p0) :: (store 4)
+    ; X32: G_STORE [[LOAD4]](p0), [[DEF]](p0) :: (store 4)
+    %0:_(p0) = IMPLICIT_DEF
+    %9:_(s1) = G_LOAD %0 :: (load 1)
+    %1:_(s8) = G_LOAD %0 :: (load 1)
+    %2:_(s16) = G_LOAD %0 :: (load 2)
+    %3:_(s32) = G_LOAD %0 :: (load 4)
+    %4:_(p0) = G_LOAD %0 :: (load 4)
+
+    G_STORE %9, %0 :: (store 1)
+    G_STORE %1, %0 :: (store 1)
+    G_STORE %2, %0 :: (store 2)
+    G_STORE %3, %0 :: (store 4)
+    G_STORE %4, %0 :: (store 4)
+...
+---
+name:            test_memop_s64
+alignment:       4
+legalized:       false
+regBankSelected: false
+liveins:
+body:             |
+  bb.0:
+
+    ; X32-LABEL: name: test_memop_s64
+    ; X32: [[DEF:%[0-9]+]]:_(p0) = IMPLICIT_DEF
+    ; X32: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[DEF]](p0) :: (load 4, align 8)
+    ; X32: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+    ; X32: [[GEP:%[0-9]+]]:_(p0) = G_GEP [[DEF]], [[C]](s32)
+    ; X32: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[GEP]](p0) :: (load 4)
+    ; X32: G_STORE [[LOAD]](s32), [[DEF]](p0) :: (store 4, align 8)
+    ; X32: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+    ; X32: [[GEP1:%[0-9]+]]:_(p0) = G_GEP [[DEF]], [[C1]](s32)
+    ; X32: G_STORE [[LOAD1]](s32), [[GEP1]](p0) :: (store 4)
+    %0:_(p0) = IMPLICIT_DEF
+    %1:_(s64) = G_LOAD %0 :: (load 8)
+
+    G_STORE %1, %0 :: (store 8)
+
+...
diff --git a/test/CodeGen/X86/GlobalISel/legalize-memop-scalar-64.mir b/test/CodeGen/X86/GlobalISel/legalize-memop-scalar-64.mir
new file mode 100644
index 0000000000000000000000000000000000000000..c7de33ddb580940a162c295ef32446a30509fd9f
--- /dev/null
+++ b/test/CodeGen/X86/GlobalISel/legalize-memop-scalar-64.mir
@@ -0,0 +1,57 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -O0 -mtriple=x86_64-linux-gnu -run-pass=legalizer -o - %s | FileCheck -check-prefix=X64 %s
+
+---
+name:            test_memop_s8tos32
+alignment:       4
+legalized:       false
+regBankSelected: false
+body:             |
+  bb.0:
+    ; X64-LABEL: name: test_memop_s8tos32
+    ; X64: [[DEF:%[0-9]+]]:_(p0) = IMPLICIT_DEF
+    ; X64: [[LOAD:%[0-9]+]]:_(s8) = G_LOAD [[DEF]](p0) :: (load 1)
+    ; X64: [[LOAD1:%[0-9]+]]:_(s8) = G_LOAD [[DEF]](p0) :: (load 1)
+    ; X64: [[LOAD2:%[0-9]+]]:_(s16) = G_LOAD [[DEF]](p0) :: (load 2)
+    ; X64: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[DEF]](p0) :: (load 4)
+    ; X64: [[LOAD4:%[0-9]+]]:_(p0) = G_LOAD [[DEF]](p0) :: (load 4)
+    ; X64: [[C:%[0-9]+]]:_(s8) = G_CONSTANT i8 1
+    ; X64: [[COPY:%[0-9]+]]:_(s8) = COPY [[LOAD]](s8)
+    ; X64: [[AND:%[0-9]+]]:_(s8) = G_AND [[COPY]], [[C]]
+    ; X64: G_STORE [[AND]](s8), [[DEF]](p0) :: (store 1)
+    ; X64: G_STORE [[LOAD1]](s8), [[DEF]](p0) :: (store 1)
+    ; X64: G_STORE [[LOAD2]](s16), [[DEF]](p0) :: (store 2)
+    ; X64: G_STORE [[LOAD3]](s32), [[DEF]](p0) :: (store 4)
+    ; X64: G_STORE [[LOAD4]](p0), [[DEF]](p0) :: (store 4)
+    %0:_(p0) = IMPLICIT_DEF
+    %9:_(s1) = G_LOAD %0(p0) :: (load 1)
+    %1:_(s8) = G_LOAD %0(p0) :: (load 1)
+    %2:_(s16) = G_LOAD %0(p0) :: (load 2)
+    %3:_(s32) = G_LOAD %0(p0) :: (load 4)
+    %4:_(p0) = G_LOAD %0(p0) :: (load 4)
+
+    G_STORE %9, %0 :: (store 1)
+    G_STORE %1, %0 :: (store 1)
+    G_STORE %2, %0 :: (store 2)
+    G_STORE %3, %0 :: (store 4)
+    G_STORE %4, %0 :: (store 4)
+...
+---
+name:            test_memop_s64
+alignment:       4
+legalized:       false
+regBankSelected: false
+liveins:
+body:             |
+  bb.0:
+
+    ; X64-LABEL: name: test_memop_s64
+    ; X64: [[DEF:%[0-9]+]]:_(p0) = IMPLICIT_DEF
+    ; X64: [[LOAD:%[0-9]+]]:_(s64) = G_LOAD [[DEF]](p0) :: (load 8)
+    ; X64: G_STORE [[LOAD]](s64), [[DEF]](p0) :: (store 8)
+    %0:_(p0) = IMPLICIT_DEF
+    %1:_(s64) = G_LOAD %0 :: (load 8)
+
+    G_STORE %1, %0 :: (store 8)
+
+...
diff --git a/test/CodeGen/X86/GlobalISel/legalize-memop-scalar.mir b/test/CodeGen/X86/GlobalISel/legalize-memop-scalar.mir
deleted file mode 100644
index 3a5eac22e603aec7b8eeea38dd81ef98db34e9d1..0000000000000000000000000000000000000000
--- a/test/CodeGen/X86/GlobalISel/legalize-memop-scalar.mir
+++ /dev/null
@@ -1,112 +0,0 @@
-# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=x86_64-linux-gnu -run-pass=legalizer %s -o - | FileCheck %s --check-prefix=ALL --check-prefix=X64
-# RUN: llc -mtriple=i386-linux-gnu   -run-pass=legalizer %s -o - | FileCheck %s --check-prefix=ALL --check-prefix=X32
-
---- |
-  define void @test_memop_s8tos32() {
-    ret void
-  }
-
-  define void @test_memop_s64() {
-    ret void
-  }
-...
----
-name:            test_memop_s8tos32
-alignment:       4
-legalized:       false
-regBankSelected: false
-registers:
-  - { id: 0, class: _, preferred-register: '' }
-  - { id: 1, class: _, preferred-register: '' }
-  - { id: 2, class: _, preferred-register: '' }
-  - { id: 3, class: _, preferred-register: '' }
-  - { id: 4, class: _, preferred-register: '' }
-  - { id: 5, class: _, preferred-register: '' }
-  - { id: 6, class: _, preferred-register: '' }
-  - { id: 7, class: _, preferred-register: '' }
-  - { id: 8, class: _, preferred-register: '' }
-  - { id: 9, class: _, preferred-register: '' }
-  - { id: 10, class: _, preferred-register: '' }
-body:             |
-  bb.1 (%ir-block.0):
-    liveins: $rdi
-
-    ; X64-LABEL: name: test_memop_s8tos32
-    ; X64: [[DEF:%[0-9]+]]:_(p0) = IMPLICIT_DEF
-    ; X64: [[LOAD:%[0-9]+]]:_(s8) = G_LOAD [[DEF]](p0) :: (load 1)
-    ; X64: [[LOAD1:%[0-9]+]]:_(s8) = G_LOAD [[DEF]](p0) :: (load 1)
-    ; X64: [[LOAD2:%[0-9]+]]:_(s16) = G_LOAD [[DEF]](p0) :: (load 2)
-    ; X64: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[DEF]](p0) :: (load 4)
-    ; X64: [[LOAD4:%[0-9]+]]:_(p0) = G_LOAD [[DEF]](p0) :: (load 8)
-    ; X64: [[C:%[0-9]+]]:_(s8) = G_CONSTANT i8 1
-    ; X64: [[COPY:%[0-9]+]]:_(s8) = COPY [[LOAD]](s8)
-    ; X64: [[AND:%[0-9]+]]:_(s8) = G_AND [[COPY]], [[C]]
-    ; X64: G_STORE [[AND]](s8), [[DEF]](p0) :: (store 1)
-    ; X64: G_STORE [[LOAD1]](s8), [[DEF]](p0) :: (store 1)
-    ; X64: G_STORE [[LOAD2]](s16), [[DEF]](p0) :: (store 2)
-    ; X64: G_STORE [[LOAD3]](s32), [[DEF]](p0) :: (store 4)
-    ; X64: G_STORE [[LOAD4]](p0), [[DEF]](p0) :: (store 8)
-    ; X32-LABEL: name: test_memop_s8tos32
-    ; X32: [[DEF:%[0-9]+]]:_(p0) = IMPLICIT_DEF
-    ; X32: [[LOAD:%[0-9]+]]:_(s8) = G_LOAD [[DEF]](p0) :: (load 1)
-    ; X32: [[LOAD1:%[0-9]+]]:_(s8) = G_LOAD [[DEF]](p0) :: (load 1)
-    ; X32: [[LOAD2:%[0-9]+]]:_(s16) = G_LOAD [[DEF]](p0) :: (load 2)
-    ; X32: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[DEF]](p0) :: (load 4)
-    ; X32: [[LOAD4:%[0-9]+]]:_(p0) = G_LOAD [[DEF]](p0) :: (load 8)
-    ; X32: [[C:%[0-9]+]]:_(s8) = G_CONSTANT i8 1
-    ; X32: [[COPY:%[0-9]+]]:_(s8) = COPY [[LOAD]](s8)
-    ; X32: [[AND:%[0-9]+]]:_(s8) = G_AND [[COPY]], [[C]]
-    ; X32: G_STORE [[AND]](s8), [[DEF]](p0) :: (store 1)
-    ; X32: G_STORE [[LOAD1]](s8), [[DEF]](p0) :: (store 1)
-    ; X32: G_STORE [[LOAD2]](s16), [[DEF]](p0) :: (store 2)
-    ; X32: G_STORE [[LOAD3]](s32), [[DEF]](p0) :: (store 4)
-    ; X32: G_STORE [[LOAD4]](p0), [[DEF]](p0) :: (store 8)
-    %0(p0) = IMPLICIT_DEF
-    %9(s1) = G_LOAD %0(p0) :: (load 1)
-    %1(s8) = G_LOAD %0(p0) :: (load 1)
-    %2(s16) = G_LOAD %0(p0) :: (load 2)
-    %3(s32) = G_LOAD %0(p0) :: (load 4)
-    %4(p0) = G_LOAD %0(p0) :: (load 8)
-
-    G_STORE %9, %0 :: (store 1)
-    G_STORE %1, %0 :: (store 1)
-    G_STORE %2, %0 :: (store 2)
-    G_STORE %3, %0 :: (store 4)
-    G_STORE %4, %0 :: (store 8)
-...
----
-name:            test_memop_s64
-alignment:       4
-legalized:       false
-regBankSelected: false
-registers:
-  - { id: 0, class: _, preferred-register: '' }
-  - { id: 1, class: _, preferred-register: '' }
-  - { id: 2, class: _, preferred-register: '' }
-liveins:
-#
-body:             |
-  bb.1 (%ir-block.0):
-    liveins: $rdi
-
-    ; X64-LABEL: name: test_memop_s64
-    ; X64: [[DEF:%[0-9]+]]:_(p0) = IMPLICIT_DEF
-    ; X64: [[LOAD:%[0-9]+]]:_(s64) = G_LOAD [[DEF]](p0) :: (load 8)
-    ; X64: G_STORE [[LOAD]](s64), [[DEF]](p0) :: (store 8)
-    ; X32-LABEL: name: test_memop_s64
-    ; X32: [[DEF:%[0-9]+]]:_(p0) = IMPLICIT_DEF
-    ; X32: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[DEF]](p0) :: (load 4, align 8)
-    ; X32: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
-    ; X32: [[GEP:%[0-9]+]]:_(p0) = G_GEP [[DEF]], [[C]](s32)
-    ; X32: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[GEP]](p0) :: (load 4)
-    ; X32: G_STORE [[LOAD]](s32), [[DEF]](p0) :: (store 4, align 8)
-    ; X32: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
-    ; X32: [[GEP1:%[0-9]+]]:_(p0) = G_GEP [[DEF]], [[C1]](s32)
-    ; X32: G_STORE [[LOAD1]](s32), [[GEP1]](p0) :: (store 4)
-    %0(p0) = IMPLICIT_DEF
-    %1(s64) = G_LOAD %0(p0) :: (load 8)
-
-    G_STORE %1, %0 :: (store 8)
-
-...
diff --git a/test/CodeGen/X86/GlobalISel/legalize-mul-scalar.mir b/test/CodeGen/X86/GlobalISel/legalize-mul-scalar.mir
index 2a54cfb9921be8cc2ca904a2dd99f10003feb0b5..122c45212a80d927519c644d2c6c6895e472b4c1 100644
--- a/test/CodeGen/X86/GlobalISel/legalize-mul-scalar.mir
+++ b/test/CodeGen/X86/GlobalISel/legalize-mul-scalar.mir
@@ -1,5 +1,5 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=x86_64-linux-gnu -run-pass=legalizer %s -o - | FileCheck %s
+# RUN: llc -O0 -mtriple=x86_64-linux-gnu -run-pass=legalizer %s -o - | FileCheck %s
 
 --- |
   define void @test_mul_i1() { ret void}
diff --git a/test/CodeGen/X86/GlobalISel/legalize-or-scalar.mir b/test/CodeGen/X86/GlobalISel/legalize-or-scalar.mir
index ce31b03d0531e7d2cb2f374d7eb7a1c4dce1ef38..6da13f654f1e5f64ba1dbbb895fcf0d4fa32af63 100644
--- a/test/CodeGen/X86/GlobalISel/legalize-or-scalar.mir
+++ b/test/CodeGen/X86/GlobalISel/legalize-or-scalar.mir
@@ -1,5 +1,5 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=x86_64-linux-gnu -run-pass=legalizer %s -o - | FileCheck %s
+# RUN: llc -O0 -mtriple=x86_64-linux-gnu -run-pass=legalizer %s -o - | FileCheck %s
 
 --- |
   define i1 @test_or_i1() {
diff --git a/test/CodeGen/X86/GlobalISel/legalize-sub.mir b/test/CodeGen/X86/GlobalISel/legalize-sub.mir
index 4856140e9e0649cb71f6d1a845b151837a9a29d6..ff4af534baa11402cdcd667a4342f649cc72bde5 100644
--- a/test/CodeGen/X86/GlobalISel/legalize-sub.mir
+++ b/test/CodeGen/X86/GlobalISel/legalize-sub.mir
@@ -1,5 +1,5 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=x86_64-linux-gnu -run-pass=legalizer %s -o - | FileCheck %s
+# RUN: llc -O0 -mtriple=x86_64-linux-gnu -run-pass=legalizer %s -o - | FileCheck %s
 
 --- |
 
diff --git a/test/CodeGen/X86/GlobalISel/legalize-trunc.mir b/test/CodeGen/X86/GlobalISel/legalize-trunc.mir
index 758023176f074161d185f69d461f95c91af21a5f..091c4b7e65780e49dd06d93730da092c6e03ff5e 100644
--- a/test/CodeGen/X86/GlobalISel/legalize-trunc.mir
+++ b/test/CodeGen/X86/GlobalISel/legalize-trunc.mir
@@ -1,6 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=i386-linux-gnu   -run-pass=legalizer %s -o - | FileCheck %s --check-prefix=ALL --check-prefix=X32
-# RUN: llc -mtriple=x86_64-linux-gnu -run-pass=legalizer %s -o - | FileCheck %s --check-prefix=ALL --check-prefix=X64
+# RUN: llc -O0 -mtriple=i386-linux-gnu   -run-pass=legalizer %s -o - | FileCheck %s --check-prefix=ALL --check-prefix=X32
+# RUN: llc -O0 -mtriple=x86_64-linux-gnu -run-pass=legalizer %s -o - | FileCheck %s --check-prefix=ALL --check-prefix=X64
 --- |
   define void @trunc_check() {
     ret void
@@ -24,9 +24,9 @@ body:             |
     ; X32: [[AND:%[0-9]+]]:_(s8) = G_AND [[TRUNC]], [[C]]
     ; X32: G_STORE [[AND]](s8), [[DEF1]](p0) :: (store 1)
     ; X32: [[TRUNC1:%[0-9]+]]:_(s8) = G_TRUNC [[DEF]](s32)
-    ; X32: G_STORE [[TRUNC1]](s8), [[DEF1]](p0) :: (store 8)
+    ; X32: G_STORE [[TRUNC1]](s8), [[DEF1]](p0) :: (store 1)
     ; X32: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[DEF]](s32)
-    ; X32: G_STORE [[TRUNC2]](s16), [[DEF1]](p0) :: (store 16)
+    ; X32: G_STORE [[TRUNC2]](s16), [[DEF1]](p0) :: (store 2)
     ; X32: RET 0
     ; X64-LABEL: name: trunc_check
     ; X64: [[DEF:%[0-9]+]]:_(s32) = IMPLICIT_DEF
@@ -36,9 +36,9 @@ body:             |
     ; X64: [[AND:%[0-9]+]]:_(s8) = G_AND [[TRUNC]], [[C]]
     ; X64: G_STORE [[AND]](s8), [[DEF1]](p0) :: (store 1)
     ; X64: [[TRUNC1:%[0-9]+]]:_(s8) = G_TRUNC [[DEF]](s32)
-    ; X64: G_STORE [[TRUNC1]](s8), [[DEF1]](p0) :: (store 8)
+    ; X64: G_STORE [[TRUNC1]](s8), [[DEF1]](p0) :: (store 1)
     ; X64: [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[DEF]](s32)
-    ; X64: G_STORE [[TRUNC2]](s16), [[DEF1]](p0) :: (store 16)
+    ; X64: G_STORE [[TRUNC2]](s16), [[DEF1]](p0) :: (store 2)
     ; X64: RET 0
     %0(s32) = IMPLICIT_DEF
     %1(s1)  = G_TRUNC %0(s32)
@@ -46,10 +46,10 @@ body:             |
     G_STORE %1, %4 :: (store 1)
 
     %2(s8)  = G_TRUNC %0(s32)
-    G_STORE %2, %4 :: (store 8)
+    G_STORE %2, %4 :: (store 1)
 
     %3(s16) = G_TRUNC %0(s32)
-    G_STORE %3, %4 :: (store 16)
+    G_STORE %3, %4 :: (store 2)
     RET 0
 
 ...
diff --git a/test/CodeGen/X86/GlobalISel/legalize-xor-scalar.mir b/test/CodeGen/X86/GlobalISel/legalize-xor-scalar.mir
index 073deff405a1a80bf0d2cd826038bf1744fe9b52..13ac1e072b49a3e0517daf8e313f118ba8e0e241 100644
--- a/test/CodeGen/X86/GlobalISel/legalize-xor-scalar.mir
+++ b/test/CodeGen/X86/GlobalISel/legalize-xor-scalar.mir
@@ -1,5 +1,5 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=x86_64-linux-gnu -run-pass=legalizer %s -o - | FileCheck %s
+# RUN: llc -O0 -mtriple=x86_64-linux-gnu -run-pass=legalizer %s -o - | FileCheck %s
 
 --- |
   define i1 @test_xor_i1() {
diff --git a/test/CodeGen/X86/GlobalISel/lshr-scalar.ll b/test/CodeGen/X86/GlobalISel/lshr-scalar.ll
index f159248f4b14cd67c8572b962126985cecf2262d..9df4c1a4e800e1ed92f8ac9c831c0210947270ca 100644
--- a/test/CodeGen/X86/GlobalISel/lshr-scalar.ll
+++ b/test/CodeGen/X86/GlobalISel/lshr-scalar.ll
@@ -6,7 +6,7 @@ define i64 @test_lshr_i64(i64 %arg1, i64 %arg2) {
 ; X64:       # %bb.0:
 ; X64-NEXT:    movq %rdi, %rax
 ; X64-NEXT:    movq %rsi, %rcx
-; X64-NEXT:    # kill: def $cl killed $rcx
+; X64-NEXT:    # kill: def $cl killed $cl killed $rcx
 ; X64-NEXT:    shrq %cl, %rax
 ; X64-NEXT:    retq
   %res = lshr i64 %arg1, %arg2
@@ -18,7 +18,6 @@ define i64 @test_lshr_i64_imm(i64 %arg1) {
 ; X64:       # %bb.0:
 ; X64-NEXT:    movq %rdi, %rax
 ; X64-NEXT:    movq $5, %rcx
-; X64-NEXT:    # kill: def $cl killed $rcx
 ; X64-NEXT:    shrq %cl, %rax
 ; X64-NEXT:    retq
   %res = lshr i64 %arg1, 5
@@ -30,7 +29,6 @@ define i64 @test_lshr_i64_imm1(i64 %arg1) {
 ; X64:       # %bb.0:
 ; X64-NEXT:    movq %rdi, %rax
 ; X64-NEXT:    movq $1, %rcx
-; X64-NEXT:    # kill: def $cl killed $rcx
 ; X64-NEXT:    shrq %cl, %rax
 ; X64-NEXT:    retq
   %res = lshr i64 %arg1, 1
@@ -42,7 +40,7 @@ define i32 @test_lshr_i32(i32 %arg1, i32 %arg2) {
 ; X64:       # %bb.0:
 ; X64-NEXT:    movl %edi, %eax
 ; X64-NEXT:    movl %esi, %ecx
-; X64-NEXT:    # kill: def $cl killed $ecx
+; X64-NEXT:    # kill: def $cl killed $cl killed $ecx
 ; X64-NEXT:    shrl %cl, %eax
 ; X64-NEXT:    retq
   %res = lshr i32 %arg1, %arg2
@@ -54,7 +52,6 @@ define i32 @test_lshr_i32_imm(i32 %arg1) {
 ; X64:       # %bb.0:
 ; X64-NEXT:    movl %edi, %eax
 ; X64-NEXT:    movl $5, %ecx
-; X64-NEXT:    # kill: def $cl killed $ecx
 ; X64-NEXT:    shrl %cl, %eax
 ; X64-NEXT:    retq
   %res = lshr i32 %arg1, 5
@@ -66,7 +63,6 @@ define i32 @test_lshr_i32_imm1(i32 %arg1) {
 ; X64:       # %bb.0:
 ; X64-NEXT:    movl %edi, %eax
 ; X64-NEXT:    movl $1, %ecx
-; X64-NEXT:    # kill: def $cl killed $ecx
 ; X64-NEXT:    shrl %cl, %eax
 ; X64-NEXT:    retq
   %res = lshr i32 %arg1, 1
@@ -78,8 +74,7 @@ define i16 @test_lshr_i16(i32 %arg1, i32 %arg2) {
 ; X64:       # %bb.0:
 ; X64-NEXT:    movl %edi, %eax
 ; X64-NEXT:    movl %esi, %ecx
-; X64-NEXT:    # kill: def $cx killed $cx killed $ecx
-; X64-NEXT:    # kill: def $cl killed $cx
+; X64-NEXT:    # kill: def $cl killed $cl killed $ecx
 ; X64-NEXT:    shrw %cl, %ax
 ; X64-NEXT:    # kill: def $ax killed $ax killed $eax
 ; X64-NEXT:    retq
@@ -94,7 +89,6 @@ define i16 @test_lshr_i16_imm(i32 %arg1) {
 ; X64:       # %bb.0:
 ; X64-NEXT:    movl %edi, %eax
 ; X64-NEXT:    movw $5, %cx
-; X64-NEXT:    # kill: def $cl killed $cx
 ; X64-NEXT:    shrw %cl, %ax
 ; X64-NEXT:    # kill: def $ax killed $ax killed $eax
 ; X64-NEXT:    retq
@@ -108,7 +102,6 @@ define i16 @test_lshr_i16_imm1(i32 %arg1) {
 ; X64:       # %bb.0:
 ; X64-NEXT:    movl %edi, %eax
 ; X64-NEXT:    movw $1, %cx
-; X64-NEXT:    # kill: def $cl killed $cx
 ; X64-NEXT:    shrw %cl, %ax
 ; X64-NEXT:    # kill: def $ax killed $ax killed $eax
 ; X64-NEXT:    retq
diff --git a/test/CodeGen/X86/GlobalISel/select-GV-32.mir b/test/CodeGen/X86/GlobalISel/select-GV-32.mir
new file mode 100644
index 0000000000000000000000000000000000000000..d4fd6e29ab3adb302528ee8dc237613849fd8d64
--- /dev/null
+++ b/test/CodeGen/X86/GlobalISel/select-GV-32.mir
@@ -0,0 +1,84 @@
+# RUN: llc -mtriple=i386-linux-gnu -run-pass=instruction-select -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=CHECK --check-prefix=X32ALL --check-prefix=X32
+# RUN: llc -mtriple=x86_64-linux-gnux32 -run-pass=instruction-select -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=CHECK --check-prefix=X32ALL --check-prefix=X32ABI
+
+--- |
+
+  @g_int = global i32 0, align 4
+
+  define void @test_global_ptrv() {
+  entry:
+    store i32* @g_int, i32** undef
+    ret void
+  }
+
+  define i32 @test_global_valv() {
+  entry:
+    %0 = load i32, i32* @g_int, align 4
+    ret i32 %0
+  }
+
+...
+---
+name:            test_global_ptrv
+# CHECK-LABEL: name:  test_global_ptrv
+alignment:       4
+legalized:       true
+regBankSelected: true
+# X32:                 registers:
+# X32-NEXT:              - { id: 0, class: gr32, preferred-register: '' }
+# X32-NEXT:              - { id: 1, class: gr32, preferred-register: '' }
+#
+# X32ABI:              registers:
+# X32ABI-NEXT:           - { id: 0, class: low32_addr_access, preferred-register: '' }
+# X32ABI-NEXT:           - { id: 1, class: gr32, preferred-register: '' }
+registers:
+  - { id: 0, class: gpr, preferred-register: '' }
+  - { id: 1, class: gpr, preferred-register: '' }
+# X32:                     %0:gr32 = IMPLICIT_DEF
+# X32-NEXT:                %1:gr32 = LEA32r $noreg, 1, $noreg, @g_int, $noreg
+# X32-NEXT:                MOV32mr %0, 1, $noreg, 0, $noreg, %1 :: (store 4 into `i32** undef`)
+# X32-NEXT:                RET 0
+#
+# X32ABI:                  %0:low32_addr_access = IMPLICIT_DEF
+# X32ABI-NEXT:             %1:gr32 = LEA64_32r $noreg, 1, $noreg, @g_int, $noreg
+# X32ABI-NEXT:             MOV32mr %0, 1, $noreg, 0, $noreg, %1 :: (store 4 into `i32** undef`)
+# X32ABI-NEXT:             RET 0
+body:             |
+  bb.1.entry:
+    liveins: $rdi
+
+    %0(p0) = IMPLICIT_DEF
+    %1(p0) = G_GLOBAL_VALUE @g_int
+    G_STORE %1(p0), %0(p0) :: (store 4 into `i32** undef`)
+    RET 0
+
+...
+---
+name:            test_global_valv
+# CHECK-LABEL: name:  test_global_valv
+alignment:       4
+legalized:       true
+regBankSelected: true
+# X32ALL:              registers:
+# X32ALL-NEXT:           - { id: 0, class: gr32, preferred-register: '' }
+# X32ALL-NEXT:           - { id: 1, class: gr32, preferred-register: '' }
+registers:
+  - { id: 0, class: gpr, preferred-register: '' }
+  - { id: 1, class: gpr, preferred-register: '' }
+# X32:                     %1:gr32 = LEA32r $noreg, 1, $noreg, @g_int, $noreg
+# X32-NEXT:                %0:gr32 = MOV32rm %1, 1, $noreg, 0, $noreg :: (load 4 from @g_int)
+# X32-NEXT:                $eax = COPY %0
+# X32-NEXT:                RET 0, implicit $eax
+#
+# X32ABI:                  %1:gr32 = LEA64_32r $noreg, 1, $noreg, @g_int, $noreg
+# X32ABI-NEXT:             %0:gr32 = MOV32rm %1, 1, $noreg, 0, $noreg :: (load 4 from @g_int)
+# X32ABI-NEXT:             $eax = COPY %0
+# X32ABI-NEXT:             RET 0, implicit $eax
+body:             |
+  bb.1.entry:
+    %1(p0) = G_GLOBAL_VALUE @g_int
+    %0(s32) = G_LOAD %1(p0) :: (load 4 from @g_int)
+    $eax = COPY %0(s32)
+    RET 0, implicit $eax
+
+...
diff --git a/test/CodeGen/X86/GlobalISel/select-GV.mir b/test/CodeGen/X86/GlobalISel/select-GV-64.mir
similarity index 59%
rename from test/CodeGen/X86/GlobalISel/select-GV.mir
rename to test/CodeGen/X86/GlobalISel/select-GV-64.mir
index 33eb333b5f1e7ce43184078b2125aaea6daeab15..830251e37585aea1cfbcc9bfa75fddf924f7aa6d 100644
--- a/test/CodeGen/X86/GlobalISel/select-GV.mir
+++ b/test/CodeGen/X86/GlobalISel/select-GV-64.mir
@@ -1,7 +1,5 @@
 # RUN: llc -mtriple=x86_64-linux-gnu                          -run-pass=instruction-select -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=CHECK --check-prefix=X64ALL --check-prefix=X64
 # RUN: llc -mtriple=x86_64-apple-darwin -relocation-model=pic -run-pass=instruction-select -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=CHECK --check-prefix=X64ALL --check-prefix=X64_DARWIN_PIC
-# RUN: llc -mtriple=i386-linux-gnu                            -run-pass=instruction-select -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=CHECK --check-prefix=X32ALL --check-prefix=X32
-# RUN: llc -mtriple=x86_64-linux-gnux32                       -run-pass=instruction-select -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=CHECK --check-prefix=X32ALL --check-prefix=X32ABI
 
 --- |
 
@@ -30,13 +28,6 @@ regBankSelected: true
 # X64ALL-NEXT:           - { id: 0, class: gr64, preferred-register: '' }
 # X64ALL-NEXT:           - { id: 1, class: gr64, preferred-register: '' }
 #
-# X32:                 registers:
-# X32-NEXT:              - { id: 0, class: gr32, preferred-register: '' }
-# X32-NEXT:              - { id: 1, class: gr32, preferred-register: '' }
-#
-# X32ABI:              registers:
-# X32ABI-NEXT:           - { id: 0, class: low32_addr_access, preferred-register: '' }
-# X32ABI-NEXT:           - { id: 1, class: gr32, preferred-register: '' }
 registers:
   - { id: 0, class: gpr, preferred-register: '' }
   - { id: 1, class: gpr, preferred-register: '' }
@@ -50,15 +41,6 @@ registers:
 # X64_DARWIN_PIC-NEXT:     MOV64mr %0, 1, $noreg, 0, $noreg, %1 :: (store 8 into `i32** undef`)
 # X64_DARWIN_PIC-NEXT:     RET 0
 #
-# X32:                     %0:gr32 = IMPLICIT_DEF
-# X32-NEXT:                %1:gr32 = LEA32r $noreg, 1, $noreg, @g_int, $noreg
-# X32-NEXT:                MOV32mr %0, 1, $noreg, 0, $noreg, %1 :: (store 8 into `i32** undef`)
-# X32-NEXT:                RET 0
-#
-# X32ABI:                  %0:low32_addr_access = IMPLICIT_DEF
-# X32ABI-NEXT:             %1:gr32 = LEA64_32r $noreg, 1, $noreg, @g_int, $noreg
-# X32ABI-NEXT:             MOV32mr %0, 1, $noreg, 0, $noreg, %1 :: (store 8 into `i32** undef`)
-# X32ABI-NEXT:             RET 0
 body:             |
   bb.1.entry:
     liveins: $rdi
@@ -79,9 +61,6 @@ regBankSelected: true
 # X64ALL-NEXT:           - { id: 0, class: gr32, preferred-register: '' }
 # X64ALL-NEXT:           - { id: 1, class: gr64, preferred-register: '' }
 #
-# X32ALL:              registers:
-# X32ALL-NEXT:           - { id: 0, class: gr32, preferred-register: '' }
-# X32ALL-NEXT:           - { id: 1, class: gr32, preferred-register: '' }
 registers:
   - { id: 0, class: gpr, preferred-register: '' }
   - { id: 1, class: gpr, preferred-register: '' }
@@ -95,15 +74,6 @@ registers:
 # X64_DARWIN_PIC-NEXT:     $eax = COPY %0
 # X64_DARWIN_PIC-NEXT:     RET 0, implicit $eax
 #
-# X32:                     %1:gr32 = LEA32r $noreg, 1, $noreg, @g_int, $noreg
-# X32-NEXT:                %0:gr32 = MOV32rm %1, 1, $noreg, 0, $noreg :: (load 4 from @g_int)
-# X32-NEXT:                $eax = COPY %0
-# X32-NEXT:                RET 0, implicit $eax
-#
-# X32ABI:                  %1:gr32 = LEA64_32r $noreg, 1, $noreg, @g_int, $noreg
-# X32ABI-NEXT:             %0:gr32 = MOV32rm %1, 1, $noreg, 0, $noreg :: (load 4 from @g_int)
-# X32ABI-NEXT:             $eax = COPY %0
-# X32ABI-NEXT:             RET 0, implicit $eax
 body:             |
   bb.1.entry:
     %1(p0) = G_GLOBAL_VALUE @g_int
diff --git a/test/CodeGen/X86/GlobalISel/select-ashr-scalar.mir b/test/CodeGen/X86/GlobalISel/select-ashr-scalar.mir
index dd12adb76fa9317afa2096f163b28c7e602af05d..ae051f9908472f639c4aaf309b2a79407d35a59d 100644
--- a/test/CodeGen/X86/GlobalISel/select-ashr-scalar.mir
+++ b/test/CodeGen/X86/GlobalISel/select-ashr-scalar.mir
@@ -80,6 +80,7 @@ registers:
   - { id: 0, class: gpr, preferred-register: '' }
   - { id: 1, class: gpr, preferred-register: '' }
   - { id: 2, class: gpr, preferred-register: '' }
+  - { id: 3, class: gpr, preferred-register: '' }
 liveins:
 fixedStack:
 stack:
@@ -91,16 +92,17 @@ body:             |
     ; ALL-LABEL: name: test_ashr_i64
     ; ALL: liveins: $rdi, $rsi
     ; ALL: [[COPY:%[0-9]+]]:gr64 = COPY $rdi
-    ; ALL: [[COPY1:%[0-9]+]]:gr64 = COPY $rsi
-    ; ALL: $rcx = COPY [[COPY1]]
-    ; ALL: $cl = KILL killed $rcx
+    ; ALL: [[COPY1:%[0-9]+]]:gr64_with_sub_8bit = COPY $rsi
+    ; ALL: [[COPY2:%[0-9]+]]:gr8 = COPY [[COPY1]].sub_8bit
+    ; ALL: $cl = COPY [[COPY2]]
     ; ALL: [[SAR64rCL:%[0-9]+]]:gr64 = SAR64rCL [[COPY]], implicit-def $eflags, implicit $cl
     ; ALL: $rax = COPY [[SAR64rCL]]
     ; ALL: RET 0, implicit $rax
     %0(s64) = COPY $rdi
     %1(s64) = COPY $rsi
-    %2(s64) = G_ASHR %0, %1
-    $rax = COPY %2(s64)
+    %2(s8) = G_TRUNC %1
+    %3(s64) = G_ASHR %0, %2
+    $rax = COPY %3(s64)
     RET 0, implicit $rax
 
 ...
@@ -125,14 +127,11 @@ body:             |
     ; ALL-LABEL: name: test_ashr_i64_imm
     ; ALL: liveins: $rdi
     ; ALL: [[COPY:%[0-9]+]]:gr64 = COPY $rdi
-    ; ALL: [[MOV64ri32_:%[0-9]+]]:gr64 = MOV64ri32 5
-    ; ALL: $rcx = COPY [[MOV64ri32_]]
-    ; ALL: $cl = KILL killed $rcx
-    ; ALL: [[SAR64rCL:%[0-9]+]]:gr64 = SAR64rCL [[COPY]], implicit-def $eflags, implicit $cl
-    ; ALL: $rax = COPY [[SAR64rCL]]
+    ; ALL: [[SAR64ri:%[0-9]+]]:gr64 = SAR64ri [[COPY]], 5, implicit-def $eflags
+    ; ALL: $rax = COPY [[SAR64ri]]
     ; ALL: RET 0, implicit $rax
     %0(s64) = COPY $rdi
-    %1(s64) = G_CONSTANT i64 5
+    %1(s8) = G_CONSTANT i8 5
     %2(s64) = G_ASHR %0, %1
     $rax = COPY %2(s64)
     RET 0, implicit $rax
@@ -159,14 +158,11 @@ body:             |
     ; ALL-LABEL: name: test_ashr_i64_imm1
     ; ALL: liveins: $rdi
     ; ALL: [[COPY:%[0-9]+]]:gr64 = COPY $rdi
-    ; ALL: [[MOV64ri32_:%[0-9]+]]:gr64 = MOV64ri32 1
-    ; ALL: $rcx = COPY [[MOV64ri32_]]
-    ; ALL: $cl = KILL killed $rcx
-    ; ALL: [[SAR64rCL:%[0-9]+]]:gr64 = SAR64rCL [[COPY]], implicit-def $eflags, implicit $cl
-    ; ALL: $rax = COPY [[SAR64rCL]]
+    ; ALL: [[SAR64r1_:%[0-9]+]]:gr64 = SAR64r1 [[COPY]], implicit-def $eflags
+    ; ALL: $rax = COPY [[SAR64r1_]]
     ; ALL: RET 0, implicit $rax
     %0(s64) = COPY $rdi
-    %1(s64) = G_CONSTANT i64 1
+    %1(s8) = G_CONSTANT i8 1
     %2(s64) = G_ASHR %0, %1
     $rax = COPY %2(s64)
     RET 0, implicit $rax
@@ -182,6 +178,7 @@ registers:
   - { id: 0, class: gpr, preferred-register: '' }
   - { id: 1, class: gpr, preferred-register: '' }
   - { id: 2, class: gpr, preferred-register: '' }
+  - { id: 3, class: gpr, preferred-register: '' }
 liveins:
 fixedStack:
 stack:
@@ -194,15 +191,16 @@ body:             |
     ; ALL: liveins: $edi, $esi
     ; ALL: [[COPY:%[0-9]+]]:gr32 = COPY $edi
     ; ALL: [[COPY1:%[0-9]+]]:gr32 = COPY $esi
-    ; ALL: $ecx = COPY [[COPY1]]
-    ; ALL: $cl = KILL killed $ecx
+    ; ALL: [[COPY2:%[0-9]+]]:gr8 = COPY [[COPY1]].sub_8bit
+    ; ALL: $cl = COPY [[COPY2]]
     ; ALL: [[SAR32rCL:%[0-9]+]]:gr32 = SAR32rCL [[COPY]], implicit-def $eflags, implicit $cl
     ; ALL: $eax = COPY [[SAR32rCL]]
     ; ALL: RET 0, implicit $eax
     %0(s32) = COPY $edi
     %1(s32) = COPY $esi
-    %2(s32) = G_ASHR %0, %1
-    $eax = COPY %2(s32)
+    %2(s8) = G_TRUNC %1
+    %3(s32) = G_ASHR %0, %2
+    $eax = COPY %3(s32)
     RET 0, implicit $eax
 
 ...
@@ -227,14 +225,11 @@ body:             |
     ; ALL-LABEL: name: test_ashr_i32_imm
     ; ALL: liveins: $edi
     ; ALL: [[COPY:%[0-9]+]]:gr32 = COPY $edi
-    ; ALL: [[MOV32ri:%[0-9]+]]:gr32 = MOV32ri 5
-    ; ALL: $ecx = COPY [[MOV32ri]]
-    ; ALL: $cl = KILL killed $ecx
-    ; ALL: [[SAR32rCL:%[0-9]+]]:gr32 = SAR32rCL [[COPY]], implicit-def $eflags, implicit $cl
-    ; ALL: $eax = COPY [[SAR32rCL]]
+    ; ALL: [[SAR32ri:%[0-9]+]]:gr32 = SAR32ri [[COPY]], 5, implicit-def $eflags
+    ; ALL: $eax = COPY [[SAR32ri]]
     ; ALL: RET 0, implicit $eax
     %0(s32) = COPY $edi
-    %1(s32) = G_CONSTANT i32 5
+    %1(s8) = G_CONSTANT i8 5
     %2(s32) = G_ASHR %0, %1
     $eax = COPY %2(s32)
     RET 0, implicit $eax
@@ -261,14 +256,11 @@ body:             |
     ; ALL-LABEL: name: test_ashr_i32_imm1
     ; ALL: liveins: $edi
     ; ALL: [[COPY:%[0-9]+]]:gr32 = COPY $edi
-    ; ALL: [[MOV32ri:%[0-9]+]]:gr32 = MOV32ri 1
-    ; ALL: $ecx = COPY [[MOV32ri]]
-    ; ALL: $cl = KILL killed $ecx
-    ; ALL: [[SAR32rCL:%[0-9]+]]:gr32 = SAR32rCL [[COPY]], implicit-def $eflags, implicit $cl
-    ; ALL: $eax = COPY [[SAR32rCL]]
+    ; ALL: [[SAR32r1_:%[0-9]+]]:gr32 = SAR32r1 [[COPY]], implicit-def $eflags
+    ; ALL: $eax = COPY [[SAR32r1_]]
     ; ALL: RET 0, implicit $eax
     %0(s32) = COPY $edi
-    %1(s32) = G_CONSTANT i32 1
+    %1(s8) = G_CONSTANT i8 1
     %2(s32) = G_ASHR %0, %1
     $eax = COPY %2(s32)
     RET 0, implicit $eax
@@ -299,16 +291,15 @@ body:             |
     ; ALL: [[COPY:%[0-9]+]]:gr32 = COPY $edi
     ; ALL: [[COPY1:%[0-9]+]]:gr32 = COPY $esi
     ; ALL: [[COPY2:%[0-9]+]]:gr16 = COPY [[COPY]].sub_16bit
-    ; ALL: [[COPY3:%[0-9]+]]:gr16 = COPY [[COPY1]].sub_16bit
-    ; ALL: $cx = COPY [[COPY3]]
-    ; ALL: $cl = KILL killed $cx
+    ; ALL: [[COPY3:%[0-9]+]]:gr8 = COPY [[COPY1]].sub_8bit
+    ; ALL: $cl = COPY [[COPY3]]
     ; ALL: [[SAR16rCL:%[0-9]+]]:gr16 = SAR16rCL [[COPY2]], implicit-def $eflags, implicit $cl
     ; ALL: $ax = COPY [[SAR16rCL]]
     ; ALL: RET 0, implicit $ax
     %0(s32) = COPY $edi
     %1(s32) = COPY $esi
     %2(s16) = G_TRUNC %0(s32)
-    %3(s16) = G_TRUNC %1(s32)
+    %3(s8) = G_TRUNC %1(s32)
     %4(s16) = G_ASHR %2, %3
     $ax = COPY %4(s16)
     RET 0, implicit $ax
@@ -336,15 +327,12 @@ body:             |
     ; ALL-LABEL: name: test_ashr_i16_imm
     ; ALL: liveins: $edi
     ; ALL: [[COPY:%[0-9]+]]:gr32 = COPY $edi
-    ; ALL: [[MOV16ri:%[0-9]+]]:gr16 = MOV16ri 5
     ; ALL: [[COPY1:%[0-9]+]]:gr16 = COPY [[COPY]].sub_16bit
-    ; ALL: $cx = COPY [[MOV16ri]]
-    ; ALL: $cl = KILL killed $cx
-    ; ALL: [[SAR16rCL:%[0-9]+]]:gr16 = SAR16rCL [[COPY1]], implicit-def $eflags, implicit $cl
-    ; ALL: $ax = COPY [[SAR16rCL]]
+    ; ALL: [[SAR16ri:%[0-9]+]]:gr16 = SAR16ri [[COPY1]], 5, implicit-def $eflags
+    ; ALL: $ax = COPY [[SAR16ri]]
     ; ALL: RET 0, implicit $ax
     %0(s32) = COPY $edi
-    %2(s16) = G_CONSTANT i16 5
+    %2(s8) = G_CONSTANT i8 5
     %1(s16) = G_TRUNC %0(s32)
     %3(s16) = G_ASHR %1, %2
     $ax = COPY %3(s16)
@@ -373,15 +361,12 @@ body:             |
     ; ALL-LABEL: name: test_ashr_i16_imm1
     ; ALL: liveins: $edi
     ; ALL: [[COPY:%[0-9]+]]:gr32 = COPY $edi
-    ; ALL: [[MOV16ri:%[0-9]+]]:gr16 = MOV16ri 1
     ; ALL: [[COPY1:%[0-9]+]]:gr16 = COPY [[COPY]].sub_16bit
-    ; ALL: $cx = COPY [[MOV16ri]]
-    ; ALL: $cl = KILL killed $cx
-    ; ALL: [[SAR16rCL:%[0-9]+]]:gr16 = SAR16rCL [[COPY1]], implicit-def $eflags, implicit $cl
-    ; ALL: $ax = COPY [[SAR16rCL]]
+    ; ALL: [[SAR16r1_:%[0-9]+]]:gr16 = SAR16r1 [[COPY1]], implicit-def $eflags
+    ; ALL: $ax = COPY [[SAR16r1_]]
     ; ALL: RET 0, implicit $ax
     %0(s32) = COPY $edi
-    %2(s16) = G_CONSTANT i16 1
+    %2(s8) = G_CONSTANT i8 1
     %1(s16) = G_TRUNC %0(s32)
     %3(s16) = G_ASHR %1, %2
     $ax = COPY %3(s16)
diff --git a/test/CodeGen/X86/GlobalISel/select-lshr-scalar.mir b/test/CodeGen/X86/GlobalISel/select-lshr-scalar.mir
index c85d70b4b07c95c0645e980562bf6a6531a78a4c..c20243d0c28896de4f02511e096b937b925b25e9 100644
--- a/test/CodeGen/X86/GlobalISel/select-lshr-scalar.mir
+++ b/test/CodeGen/X86/GlobalISel/select-lshr-scalar.mir
@@ -80,6 +80,7 @@ registers:
   - { id: 0, class: gpr, preferred-register: '' }
   - { id: 1, class: gpr, preferred-register: '' }
   - { id: 2, class: gpr, preferred-register: '' }
+  - { id: 3, class: gpr, preferred-register: '' }
 liveins:
 fixedStack:
 stack:
@@ -91,16 +92,17 @@ body:             |
     ; ALL-LABEL: name: test_lshr_i64
     ; ALL: liveins: $rdi, $rsi
     ; ALL: [[COPY:%[0-9]+]]:gr64 = COPY $rdi
-    ; ALL: [[COPY1:%[0-9]+]]:gr64 = COPY $rsi
-    ; ALL: $rcx = COPY [[COPY1]]
-    ; ALL: $cl = KILL killed $rcx
+    ; ALL: [[COPY1:%[0-9]+]]:gr64_with_sub_8bit = COPY $rsi
+    ; ALL: [[COPY2:%[0-9]+]]:gr8 = COPY [[COPY1]].sub_8bit
+    ; ALL: $cl = COPY [[COPY2]]
     ; ALL: [[SHR64rCL:%[0-9]+]]:gr64 = SHR64rCL [[COPY]], implicit-def $eflags, implicit $cl
     ; ALL: $rax = COPY [[SHR64rCL]]
     ; ALL: RET 0, implicit $rax
     %0(s64) = COPY $rdi
     %1(s64) = COPY $rsi
-    %2(s64) = G_LSHR %0, %1
-    $rax = COPY %2(s64)
+    %2(s8) = G_TRUNC %1
+    %3(s64) = G_LSHR %0, %2
+    $rax = COPY %3(s64)
     RET 0, implicit $rax
 
 ...
@@ -125,14 +127,11 @@ body:             |
     ; ALL-LABEL: name: test_lshr_i64_imm
     ; ALL: liveins: $rdi
     ; ALL: [[COPY:%[0-9]+]]:gr64 = COPY $rdi
-    ; ALL: [[MOV64ri32_:%[0-9]+]]:gr64 = MOV64ri32 5
-    ; ALL: $rcx = COPY [[MOV64ri32_]]
-    ; ALL: $cl = KILL killed $rcx
-    ; ALL: [[SHR64rCL:%[0-9]+]]:gr64 = SHR64rCL [[COPY]], implicit-def $eflags, implicit $cl
-    ; ALL: $rax = COPY [[SHR64rCL]]
+    ; ALL: [[SHR64ri:%[0-9]+]]:gr64 = SHR64ri [[COPY]], 5, implicit-def $eflags
+    ; ALL: $rax = COPY [[SHR64ri]]
     ; ALL: RET 0, implicit $rax
     %0(s64) = COPY $rdi
-    %1(s64) = G_CONSTANT i64 5
+    %1(s8) = G_CONSTANT i8 5
     %2(s64) = G_LSHR %0, %1
     $rax = COPY %2(s64)
     RET 0, implicit $rax
@@ -159,14 +158,11 @@ body:             |
     ; ALL-LABEL: name: test_lshr_i64_imm1
     ; ALL: liveins: $rdi
     ; ALL: [[COPY:%[0-9]+]]:gr64 = COPY $rdi
-    ; ALL: [[MOV64ri32_:%[0-9]+]]:gr64 = MOV64ri32 1
-    ; ALL: $rcx = COPY [[MOV64ri32_]]
-    ; ALL: $cl = KILL killed $rcx
-    ; ALL: [[SHR64rCL:%[0-9]+]]:gr64 = SHR64rCL [[COPY]], implicit-def $eflags, implicit $cl
-    ; ALL: $rax = COPY [[SHR64rCL]]
+    ; ALL: [[SHR64r1_:%[0-9]+]]:gr64 = SHR64r1 [[COPY]], implicit-def $eflags
+    ; ALL: $rax = COPY [[SHR64r1_]]
     ; ALL: RET 0, implicit $rax
     %0(s64) = COPY $rdi
-    %1(s64) = G_CONSTANT i64 1
+    %1(s8) = G_CONSTANT i8 1
     %2(s64) = G_LSHR %0, %1
     $rax = COPY %2(s64)
     RET 0, implicit $rax
@@ -182,6 +178,7 @@ registers:
   - { id: 0, class: gpr, preferred-register: '' }
   - { id: 1, class: gpr, preferred-register: '' }
   - { id: 2, class: gpr, preferred-register: '' }
+  - { id: 3, class: gpr, preferred-register: '' }
 liveins:
 fixedStack:
 stack:
@@ -194,15 +191,16 @@ body:             |
     ; ALL: liveins: $edi, $esi
     ; ALL: [[COPY:%[0-9]+]]:gr32 = COPY $edi
     ; ALL: [[COPY1:%[0-9]+]]:gr32 = COPY $esi
-    ; ALL: $ecx = COPY [[COPY1]]
-    ; ALL: $cl = KILL killed $ecx
+    ; ALL: [[COPY2:%[0-9]+]]:gr8 = COPY [[COPY1]].sub_8bit
+    ; ALL: $cl = COPY [[COPY2]]
     ; ALL: [[SHR32rCL:%[0-9]+]]:gr32 = SHR32rCL [[COPY]], implicit-def $eflags, implicit $cl
     ; ALL: $eax = COPY [[SHR32rCL]]
     ; ALL: RET 0, implicit $eax
     %0(s32) = COPY $edi
     %1(s32) = COPY $esi
-    %2(s32) = G_LSHR %0, %1
-    $eax = COPY %2(s32)
+    %2(s8) = G_TRUNC %1
+    %3(s32) = G_LSHR %0, %2
+    $eax = COPY %3(s32)
     RET 0, implicit $eax
 
 ...
@@ -227,14 +225,11 @@ body:             |
     ; ALL-LABEL: name: test_lshr_i32_imm
     ; ALL: liveins: $edi
     ; ALL: [[COPY:%[0-9]+]]:gr32 = COPY $edi
-    ; ALL: [[MOV32ri:%[0-9]+]]:gr32 = MOV32ri 5
-    ; ALL: $ecx = COPY [[MOV32ri]]
-    ; ALL: $cl = KILL killed $ecx
-    ; ALL: [[SHR32rCL:%[0-9]+]]:gr32 = SHR32rCL [[COPY]], implicit-def $eflags, implicit $cl
-    ; ALL: $eax = COPY [[SHR32rCL]]
+    ; ALL: [[SHR32ri:%[0-9]+]]:gr32 = SHR32ri [[COPY]], 5, implicit-def $eflags
+    ; ALL: $eax = COPY [[SHR32ri]]
     ; ALL: RET 0, implicit $eax
     %0(s32) = COPY $edi
-    %1(s32) = G_CONSTANT i32 5
+    %1(s8) = G_CONSTANT i8 5
     %2(s32) = G_LSHR %0, %1
     $eax = COPY %2(s32)
     RET 0, implicit $eax
@@ -261,14 +256,11 @@ body:             |
     ; ALL-LABEL: name: test_lshr_i32_imm1
     ; ALL: liveins: $edi
     ; ALL: [[COPY:%[0-9]+]]:gr32 = COPY $edi
-    ; ALL: [[MOV32ri:%[0-9]+]]:gr32 = MOV32ri 1
-    ; ALL: $ecx = COPY [[MOV32ri]]
-    ; ALL: $cl = KILL killed $ecx
-    ; ALL: [[SHR32rCL:%[0-9]+]]:gr32 = SHR32rCL [[COPY]], implicit-def $eflags, implicit $cl
-    ; ALL: $eax = COPY [[SHR32rCL]]
+    ; ALL: [[SHR32r1_:%[0-9]+]]:gr32 = SHR32r1 [[COPY]], implicit-def $eflags
+    ; ALL: $eax = COPY [[SHR32r1_]]
     ; ALL: RET 0, implicit $eax
     %0(s32) = COPY $edi
-    %1(s32) = G_CONSTANT i32 1
+    %1(s8) = G_CONSTANT i8 1
     %2(s32) = G_LSHR %0, %1
     $eax = COPY %2(s32)
     RET 0, implicit $eax
@@ -299,16 +291,15 @@ body:             |
     ; ALL: [[COPY:%[0-9]+]]:gr32 = COPY $edi
     ; ALL: [[COPY1:%[0-9]+]]:gr32 = COPY $esi
     ; ALL: [[COPY2:%[0-9]+]]:gr16 = COPY [[COPY]].sub_16bit
-    ; ALL: [[COPY3:%[0-9]+]]:gr16 = COPY [[COPY1]].sub_16bit
-    ; ALL: $cx = COPY [[COPY3]]
-    ; ALL: $cl = KILL killed $cx
+    ; ALL: [[COPY3:%[0-9]+]]:gr8 = COPY [[COPY1]].sub_8bit
+    ; ALL: $cl = COPY [[COPY3]]
     ; ALL: [[SHR16rCL:%[0-9]+]]:gr16 = SHR16rCL [[COPY2]], implicit-def $eflags, implicit $cl
     ; ALL: $ax = COPY [[SHR16rCL]]
     ; ALL: RET 0, implicit $ax
     %0(s32) = COPY $edi
     %1(s32) = COPY $esi
     %2(s16) = G_TRUNC %0(s32)
-    %3(s16) = G_TRUNC %1(s32)
+    %3(s8) = G_TRUNC %1(s32)
     %4(s16) = G_LSHR %2, %3
     $ax = COPY %4(s16)
     RET 0, implicit $ax
@@ -336,15 +327,12 @@ body:             |
     ; ALL-LABEL: name: test_lshr_i16_imm
     ; ALL: liveins: $edi
     ; ALL: [[COPY:%[0-9]+]]:gr32 = COPY $edi
-    ; ALL: [[MOV16ri:%[0-9]+]]:gr16 = MOV16ri 5
     ; ALL: [[COPY1:%[0-9]+]]:gr16 = COPY [[COPY]].sub_16bit
-    ; ALL: $cx = COPY [[MOV16ri]]
-    ; ALL: $cl = KILL killed $cx
-    ; ALL: [[SHR16rCL:%[0-9]+]]:gr16 = SHR16rCL [[COPY1]], implicit-def $eflags, implicit $cl
-    ; ALL: $ax = COPY [[SHR16rCL]]
+    ; ALL: [[SHR16ri:%[0-9]+]]:gr16 = SHR16ri [[COPY1]], 5, implicit-def $eflags
+    ; ALL: $ax = COPY [[SHR16ri]]
     ; ALL: RET 0, implicit $ax
     %0(s32) = COPY $edi
-    %2(s16) = G_CONSTANT i16 5
+    %2(s8) = G_CONSTANT i8 5
     %1(s16) = G_TRUNC %0(s32)
     %3(s16) = G_LSHR %1, %2
     $ax = COPY %3(s16)
@@ -373,15 +361,12 @@ body:             |
     ; ALL-LABEL: name: test_lshr_i16_imm1
     ; ALL: liveins: $edi
     ; ALL: [[COPY:%[0-9]+]]:gr32 = COPY $edi
-    ; ALL: [[MOV16ri:%[0-9]+]]:gr16 = MOV16ri 1
     ; ALL: [[COPY1:%[0-9]+]]:gr16 = COPY [[COPY]].sub_16bit
-    ; ALL: $cx = COPY [[MOV16ri]]
-    ; ALL: $cl = KILL killed $cx
-    ; ALL: [[SHR16rCL:%[0-9]+]]:gr16 = SHR16rCL [[COPY1]], implicit-def $eflags, implicit $cl
-    ; ALL: $ax = COPY [[SHR16rCL]]
+    ; ALL: [[SHR16r1_:%[0-9]+]]:gr16 = SHR16r1 [[COPY1]], implicit-def $eflags
+    ; ALL: $ax = COPY [[SHR16r1_]]
     ; ALL: RET 0, implicit $ax
     %0(s32) = COPY $edi
-    %2(s16) = G_CONSTANT i16 1
+    %2(s8) = G_CONSTANT i8 1
     %1(s16) = G_TRUNC %0(s32)
     %3(s16) = G_LSHR %1, %2
     $ax = COPY %3(s16)
diff --git a/test/CodeGen/X86/GlobalISel/select-memop-scalar-x32.mir b/test/CodeGen/X86/GlobalISel/select-memop-scalar-x32.mir
index bf03d1d2caae4d9607a33e07cb49e54e6351b6aa..e940a462f07b028cfa7a30034967bb57b4e83e01 100644
--- a/test/CodeGen/X86/GlobalISel/select-memop-scalar-x32.mir
+++ b/test/CodeGen/X86/GlobalISel/select-memop-scalar-x32.mir
@@ -57,12 +57,12 @@ fixedStack:
 body:             |
   bb.1 (%ir-block.0):
     ; ALL-LABEL: name: test_load_i8
-    ; ALL: [[MOV32rm:%[0-9]+]]:gr32 = MOV32rm %fixed-stack.0, 1, $noreg, 0, $noreg :: (invariant load 4 from %fixed-stack.0, align 0)
+    ; ALL: [[MOV32rm:%[0-9]+]]:gr32 = MOV32rm %fixed-stack.0, 1, $noreg, 0, $noreg :: (invariant load 4 from %fixed-stack.0, align 16)
     ; ALL: [[MOV8rm:%[0-9]+]]:gr8 = MOV8rm [[MOV32rm]], 1, $noreg, 0, $noreg :: (load 1 from %ir.p1)
     ; ALL: $al = COPY [[MOV8rm]]
     ; ALL: RET 0, implicit $al
     %1(p0) = G_FRAME_INDEX %fixed-stack.0
-    %0(p0) = G_LOAD %1(p0) :: (invariant load 4 from %fixed-stack.0, align 0)
+    %0(p0) = G_LOAD %1(p0) :: (invariant load 4 from %fixed-stack.0, align 16)
     %2(s8) = G_LOAD %0(p0) :: (load 1 from %ir.p1)
     $al = COPY %2(s8)
     RET 0, implicit $al
@@ -82,12 +82,12 @@ fixedStack:
 body:             |
   bb.1 (%ir-block.0):
     ; ALL-LABEL: name: test_load_i16
-    ; ALL: [[MOV32rm:%[0-9]+]]:gr32 = MOV32rm %fixed-stack.0, 1, $noreg, 0, $noreg :: (invariant load 4 from %fixed-stack.0, align 0)
+    ; ALL: [[MOV32rm:%[0-9]+]]:gr32 = MOV32rm %fixed-stack.0, 1, $noreg, 0, $noreg :: (invariant load 4 from %fixed-stack.0, align 16)
     ; ALL: [[MOV16rm:%[0-9]+]]:gr16 = MOV16rm [[MOV32rm]], 1, $noreg, 0, $noreg :: (load 2 from %ir.p1)
     ; ALL: $ax = COPY [[MOV16rm]]
     ; ALL: RET 0, implicit $ax
     %1(p0) = G_FRAME_INDEX %fixed-stack.0
-    %0(p0) = G_LOAD %1(p0) :: (invariant load 4 from %fixed-stack.0, align 0)
+    %0(p0) = G_LOAD %1(p0) :: (invariant load 4 from %fixed-stack.0, align 16)
     %2(s16) = G_LOAD %0(p0) :: (load 2 from %ir.p1)
     $ax = COPY %2(s16)
     RET 0, implicit $ax
@@ -107,12 +107,12 @@ fixedStack:
 body:             |
   bb.1 (%ir-block.0):
     ; ALL-LABEL: name: test_load_i32
-    ; ALL: [[MOV32rm:%[0-9]+]]:gr32 = MOV32rm %fixed-stack.0, 1, $noreg, 0, $noreg :: (invariant load 4 from %fixed-stack.0, align 0)
+    ; ALL: [[MOV32rm:%[0-9]+]]:gr32 = MOV32rm %fixed-stack.0, 1, $noreg, 0, $noreg :: (invariant load 4 from %fixed-stack.0, align 16)
     ; ALL: [[MOV32rm1:%[0-9]+]]:gr32 = MOV32rm [[MOV32rm]], 1, $noreg, 0, $noreg :: (load 4 from %ir.p1)
     ; ALL: $eax = COPY [[MOV32rm1]]
     ; ALL: RET 0, implicit $eax
     %1(p0) = G_FRAME_INDEX %fixed-stack.0
-    %0(p0) = G_LOAD %1(p0) :: (invariant load 4 from %fixed-stack.0, align 0)
+    %0(p0) = G_LOAD %1(p0) :: (invariant load 4 from %fixed-stack.0, align 16)
     %2(s32) = G_LOAD %0(p0) :: (load 4 from %ir.p1)
     $eax = COPY %2(s32)
     RET 0, implicit $eax
@@ -134,15 +134,15 @@ fixedStack:
 body:             |
   bb.1 (%ir-block.0):
     ; ALL-LABEL: name: test_store_i8
-    ; ALL: [[MOV8rm:%[0-9]+]]:gr8 = MOV8rm %fixed-stack.0, 1, $noreg, 0, $noreg :: (invariant load 1 from %fixed-stack.0, align 0)
-    ; ALL: [[MOV32rm:%[0-9]+]]:gr32 = MOV32rm %fixed-stack.1, 1, $noreg, 0, $noreg :: (invariant load 4 from %fixed-stack.1, align 0)
+    ; ALL: [[MOV8rm:%[0-9]+]]:gr8 = MOV8rm %fixed-stack.0, 1, $noreg, 0, $noreg :: (invariant load 1 from %fixed-stack.0, align 16)
+    ; ALL: [[MOV32rm:%[0-9]+]]:gr32 = MOV32rm %fixed-stack.1, 1, $noreg, 0, $noreg :: (invariant load 4 from %fixed-stack.1)
     ; ALL: MOV8mr [[MOV32rm]], 1, $noreg, 0, $noreg, [[MOV8rm]] :: (store 1 into %ir.p1)
     ; ALL: $eax = COPY [[MOV32rm]]
     ; ALL: RET 0, implicit $eax
     %2(p0) = G_FRAME_INDEX %fixed-stack.1
-    %0(s8) = G_LOAD %2(p0) :: (invariant load 1 from %fixed-stack.1, align 0)
+    %0(s8) = G_LOAD %2(p0) :: (invariant load 1 from %fixed-stack.1, align 16)
     %3(p0) = G_FRAME_INDEX %fixed-stack.0
-    %1(p0) = G_LOAD %3(p0) :: (invariant load 4 from %fixed-stack.0, align 0)
+    %1(p0) = G_LOAD %3(p0) :: (invariant load 4 from %fixed-stack.0, align 4)
     G_STORE %0(s8), %1(p0) :: (store 1 into %ir.p1)
     $eax = COPY %1(p0)
     RET 0, implicit $eax
@@ -164,15 +164,15 @@ fixedStack:
 body:             |
   bb.1 (%ir-block.0):
     ; ALL-LABEL: name: test_store_i16
-    ; ALL: [[MOV16rm:%[0-9]+]]:gr16 = MOV16rm %fixed-stack.0, 1, $noreg, 0, $noreg :: (invariant load 2 from %fixed-stack.0, align 0)
-    ; ALL: [[MOV32rm:%[0-9]+]]:gr32 = MOV32rm %fixed-stack.1, 1, $noreg, 0, $noreg :: (invariant load 4 from %fixed-stack.1, align 0)
+    ; ALL: [[MOV16rm:%[0-9]+]]:gr16 = MOV16rm %fixed-stack.0, 1, $noreg, 0, $noreg :: (invariant load 2 from %fixed-stack.0, align 16)
+    ; ALL: [[MOV32rm:%[0-9]+]]:gr32 = MOV32rm %fixed-stack.1, 1, $noreg, 0, $noreg :: (invariant load 4 from %fixed-stack.1)
     ; ALL: MOV16mr [[MOV32rm]], 1, $noreg, 0, $noreg, [[MOV16rm]] :: (store 2 into %ir.p1)
     ; ALL: $eax = COPY [[MOV32rm]]
     ; ALL: RET 0, implicit $eax
     %2(p0) = G_FRAME_INDEX %fixed-stack.1
-    %0(s16) = G_LOAD %2(p0) :: (invariant load 2 from %fixed-stack.1, align 0)
+    %0(s16) = G_LOAD %2(p0) :: (invariant load 2 from %fixed-stack.1, align 16)
     %3(p0) = G_FRAME_INDEX %fixed-stack.0
-    %1(p0) = G_LOAD %3(p0) :: (invariant load 4 from %fixed-stack.0, align 0)
+    %1(p0) = G_LOAD %3(p0) :: (invariant load 4 from %fixed-stack.0, align 4)
     G_STORE %0(s16), %1(p0) :: (store 2 into %ir.p1)
     $eax = COPY %1(p0)
     RET 0, implicit $eax
@@ -194,15 +194,15 @@ fixedStack:
 body:             |
   bb.1 (%ir-block.0):
     ; ALL-LABEL: name: test_store_i32
-    ; ALL: [[MOV32rm:%[0-9]+]]:gr32 = MOV32rm %fixed-stack.0, 1, $noreg, 0, $noreg :: (invariant load 4 from %fixed-stack.0, align 0)
-    ; ALL: [[MOV32rm1:%[0-9]+]]:gr32 = MOV32rm %fixed-stack.1, 1, $noreg, 0, $noreg :: (invariant load 4 from %fixed-stack.1, align 0)
+    ; ALL: [[MOV32rm:%[0-9]+]]:gr32 = MOV32rm %fixed-stack.0, 1, $noreg, 0, $noreg :: (invariant load 4 from %fixed-stack.0, align 16)
+    ; ALL: [[MOV32rm1:%[0-9]+]]:gr32 = MOV32rm %fixed-stack.1, 1, $noreg, 0, $noreg :: (invariant load 4 from %fixed-stack.1)
     ; ALL: MOV32mr [[MOV32rm1]], 1, $noreg, 0, $noreg, [[MOV32rm]] :: (store 4 into %ir.p1)
     ; ALL: $eax = COPY [[MOV32rm1]]
     ; ALL: RET 0, implicit $eax
     %2(p0) = G_FRAME_INDEX %fixed-stack.1
-    %0(s32) = G_LOAD %2(p0) :: (invariant load 4 from %fixed-stack.1, align 0)
+    %0(s32) = G_LOAD %2(p0) :: (invariant load 4 from %fixed-stack.1, align 16)
     %3(p0) = G_FRAME_INDEX %fixed-stack.0
-    %1(p0) = G_LOAD %3(p0) :: (invariant load 4 from %fixed-stack.0, align 0)
+    %1(p0) = G_LOAD %3(p0) :: (invariant load 4 from %fixed-stack.0, align 4)
     G_STORE %0(s32), %1(p0) :: (store 4 into %ir.p1)
     $eax = COPY %1(p0)
     RET 0, implicit $eax
@@ -222,12 +222,12 @@ fixedStack:
 body:             |
   bb.1 (%ir-block.0):
     ; ALL-LABEL: name: test_load_ptr
-    ; ALL: [[MOV32rm:%[0-9]+]]:gr32 = MOV32rm %fixed-stack.0, 1, $noreg, 0, $noreg :: (invariant load 4 from %fixed-stack.0, align 0)
+    ; ALL: [[MOV32rm:%[0-9]+]]:gr32 = MOV32rm %fixed-stack.0, 1, $noreg, 0, $noreg :: (invariant load 4 from %fixed-stack.0, align 16)
     ; ALL: [[MOV32rm1:%[0-9]+]]:gr32 = MOV32rm [[MOV32rm]], 1, $noreg, 0, $noreg :: (load 4 from %ir.ptr1)
     ; ALL: $eax = COPY [[MOV32rm1]]
     ; ALL: RET 0, implicit $eax
     %1(p0) = G_FRAME_INDEX %fixed-stack.0
-    %0(p0) = G_LOAD %1(p0) :: (invariant load 4 from %fixed-stack.0, align 0)
+    %0(p0) = G_LOAD %1(p0) :: (invariant load 4 from %fixed-stack.0, align 16)
     %2(p0) = G_LOAD %0(p0) :: (load 4 from %ir.ptr1)
     $eax = COPY %2(p0)
     RET 0, implicit $eax
@@ -249,14 +249,14 @@ fixedStack:
 body:             |
   bb.1 (%ir-block.0):
     ; ALL-LABEL: name: test_store_ptr
-    ; ALL: [[MOV32rm:%[0-9]+]]:gr32 = MOV32rm %fixed-stack.0, 1, $noreg, 0, $noreg :: (invariant load 4 from %fixed-stack.0, align 0)
-    ; ALL: [[MOV32rm1:%[0-9]+]]:gr32 = MOV32rm %fixed-stack.1, 1, $noreg, 0, $noreg :: (invariant load 4 from %fixed-stack.1, align 0)
+    ; ALL: [[MOV32rm:%[0-9]+]]:gr32 = MOV32rm %fixed-stack.0, 1, $noreg, 0, $noreg :: (invariant load 4 from %fixed-stack.0, align 16)
+    ; ALL: [[MOV32rm1:%[0-9]+]]:gr32 = MOV32rm %fixed-stack.1, 1, $noreg, 0, $noreg :: (invariant load 4 from %fixed-stack.1)
     ; ALL: MOV32mr [[MOV32rm]], 1, $noreg, 0, $noreg, [[MOV32rm1]] :: (store 4 into %ir.ptr1)
     ; ALL: RET 0
     %2(p0) = G_FRAME_INDEX %fixed-stack.1
-    %0(p0) = G_LOAD %2(p0) :: (invariant load 4 from %fixed-stack.1, align 0)
+    %0(p0) = G_LOAD %2(p0) :: (invariant load 4 from %fixed-stack.1, align 16)
     %3(p0) = G_FRAME_INDEX %fixed-stack.0
-    %1(p0) = G_LOAD %3(p0) :: (invariant load 4 from %fixed-stack.0, align 0)
+    %1(p0) = G_LOAD %3(p0) :: (invariant load 4 from %fixed-stack.0, align 4)
     G_STORE %1(p0), %0(p0) :: (store 4 into %ir.ptr1)
     RET 0
 
diff --git a/test/CodeGen/X86/GlobalISel/select-shl-scalar.mir b/test/CodeGen/X86/GlobalISel/select-shl-scalar.mir
index 04a399f492e7ecbe63fc42ed315a0c56ff72ef3d..1c4dcf4a334969c8048dc9f5700f1659b1dd830f 100644
--- a/test/CodeGen/X86/GlobalISel/select-shl-scalar.mir
+++ b/test/CodeGen/X86/GlobalISel/select-shl-scalar.mir
@@ -81,6 +81,7 @@ registers:
   - { id: 0, class: gpr, preferred-register: '' }
   - { id: 1, class: gpr, preferred-register: '' }
   - { id: 2, class: gpr, preferred-register: '' }
+  - { id: 3, class: gpr, preferred-register: '' }
 liveins:
 fixedStack:
 stack:
@@ -92,16 +93,17 @@ body:             |
     ; ALL-LABEL: name: test_shl_i64
     ; ALL: liveins: $rdi, $rsi
     ; ALL: [[COPY:%[0-9]+]]:gr64 = COPY $rdi
-    ; ALL: [[COPY1:%[0-9]+]]:gr64 = COPY $rsi
-    ; ALL: $rcx = COPY [[COPY1]]
-    ; ALL: $cl = KILL killed $rcx
+    ; ALL: [[COPY1:%[0-9]+]]:gr64_with_sub_8bit = COPY $rsi
+    ; ALL: [[COPY2:%[0-9]+]]:gr8 = COPY [[COPY1]].sub_8bit
+    ; ALL: $cl = COPY [[COPY2]]
     ; ALL: [[SHL64rCL:%[0-9]+]]:gr64 = SHL64rCL [[COPY]], implicit-def $eflags, implicit $cl
     ; ALL: $rax = COPY [[SHL64rCL]]
     ; ALL: RET 0, implicit $rax
     %0(s64) = COPY $rdi
     %1(s64) = COPY $rsi
-    %2(s64) = G_SHL %0, %1
-    $rax = COPY %2(s64)
+    %2(s8) = G_TRUNC %1
+    %3(s64) = G_SHL %0, %2
+    $rax = COPY %3(s64)
     RET 0, implicit $rax
 
 ...
@@ -126,14 +128,11 @@ body:             |
     ; ALL-LABEL: name: test_shl_i64_imm
     ; ALL: liveins: $rdi
     ; ALL: [[COPY:%[0-9]+]]:gr64 = COPY $rdi
-    ; ALL: [[MOV64ri32_:%[0-9]+]]:gr64 = MOV64ri32 5
-    ; ALL: $rcx = COPY [[MOV64ri32_]]
-    ; ALL: $cl = KILL killed $rcx
-    ; ALL: [[SHL64rCL:%[0-9]+]]:gr64 = SHL64rCL [[COPY]], implicit-def $eflags, implicit $cl
-    ; ALL: $rax = COPY [[SHL64rCL]]
+    ; ALL: [[SHL64ri:%[0-9]+]]:gr64 = SHL64ri [[COPY]], 5, implicit-def $eflags
+    ; ALL: $rax = COPY [[SHL64ri]]
     ; ALL: RET 0, implicit $rax
     %0(s64) = COPY $rdi
-    %1(s64) = G_CONSTANT i64 5
+    %1(s8) = G_CONSTANT i8 5
     %2(s64) = G_SHL %0, %1
     $rax = COPY %2(s64)
     RET 0, implicit $rax
@@ -160,14 +159,11 @@ body:             |
     ; ALL-LABEL: name: test_shl_i64_imm1
     ; ALL: liveins: $rdi
     ; ALL: [[COPY:%[0-9]+]]:gr64 = COPY $rdi
-    ; ALL: [[MOV64ri32_:%[0-9]+]]:gr64 = MOV64ri32 1
-    ; ALL: $rcx = COPY [[MOV64ri32_]]
-    ; ALL: $cl = KILL killed $rcx
-    ; ALL: [[SHL64rCL:%[0-9]+]]:gr64 = SHL64rCL [[COPY]], implicit-def $eflags, implicit $cl
-    ; ALL: $rax = COPY [[SHL64rCL]]
+    ; ALL: [[ADD64rr:%[0-9]+]]:gr64 = ADD64rr [[COPY]], [[COPY]], implicit-def $eflags
+    ; ALL: $rax = COPY [[ADD64rr]]
     ; ALL: RET 0, implicit $rax
     %0(s64) = COPY $rdi
-    %1(s64) = G_CONSTANT i64 1
+    %1(s8) = G_CONSTANT i8 1
     %2(s64) = G_SHL %0, %1
     $rax = COPY %2(s64)
     RET 0, implicit $rax
@@ -183,6 +179,7 @@ registers:
   - { id: 0, class: gpr, preferred-register: '' }
   - { id: 1, class: gpr, preferred-register: '' }
   - { id: 2, class: gpr, preferred-register: '' }
+  - { id: 3, class: gpr, preferred-register: '' }
 liveins:
 fixedStack:
 stack:
@@ -195,15 +192,16 @@ body:             |
     ; ALL: liveins: $edi, $esi
     ; ALL: [[COPY:%[0-9]+]]:gr32 = COPY $edi
     ; ALL: [[COPY1:%[0-9]+]]:gr32 = COPY $esi
-    ; ALL: $ecx = COPY [[COPY1]]
-    ; ALL: $cl = KILL killed $ecx
+    ; ALL: [[COPY2:%[0-9]+]]:gr8 = COPY [[COPY1]].sub_8bit
+    ; ALL: $cl = COPY [[COPY2]]
     ; ALL: [[SHL32rCL:%[0-9]+]]:gr32 = SHL32rCL [[COPY]], implicit-def $eflags, implicit $cl
     ; ALL: $eax = COPY [[SHL32rCL]]
     ; ALL: RET 0, implicit $eax
     %0(s32) = COPY $edi
     %1(s32) = COPY $esi
-    %2(s32) = G_SHL %0, %1
-    $eax = COPY %2(s32)
+    %2(s8) = G_TRUNC %1
+    %3(s32) = G_SHL %0, %2
+    $eax = COPY %3(s32)
     RET 0, implicit $eax
 
 ...
@@ -228,14 +226,11 @@ body:             |
     ; ALL-LABEL: name: test_shl_i32_imm
     ; ALL: liveins: $edi
     ; ALL: [[COPY:%[0-9]+]]:gr32 = COPY $edi
-    ; ALL: [[MOV32ri:%[0-9]+]]:gr32 = MOV32ri 5
-    ; ALL: $ecx = COPY [[MOV32ri]]
-    ; ALL: $cl = KILL killed $ecx
-    ; ALL: [[SHL32rCL:%[0-9]+]]:gr32 = SHL32rCL [[COPY]], implicit-def $eflags, implicit $cl
-    ; ALL: $eax = COPY [[SHL32rCL]]
+    ; ALL: [[SHL32ri:%[0-9]+]]:gr32 = SHL32ri [[COPY]], 5, implicit-def $eflags
+    ; ALL: $eax = COPY [[SHL32ri]]
     ; ALL: RET 0, implicit $eax
     %0(s32) = COPY $edi
-    %1(s32) = G_CONSTANT i32 5
+    %1(s8) = G_CONSTANT i8 5
     %2(s32) = G_SHL %0, %1
     $eax = COPY %2(s32)
     RET 0, implicit $eax
@@ -262,14 +257,11 @@ body:             |
     ; ALL-LABEL: name: test_shl_i32_imm1
     ; ALL: liveins: $edi
     ; ALL: [[COPY:%[0-9]+]]:gr32 = COPY $edi
-    ; ALL: [[MOV32ri:%[0-9]+]]:gr32 = MOV32ri 1
-    ; ALL: $ecx = COPY [[MOV32ri]]
-    ; ALL: $cl = KILL killed $ecx
-    ; ALL: [[SHL32rCL:%[0-9]+]]:gr32 = SHL32rCL [[COPY]], implicit-def $eflags, implicit $cl
-    ; ALL: $eax = COPY [[SHL32rCL]]
+    ; ALL: [[ADD32rr:%[0-9]+]]:gr32 = ADD32rr [[COPY]], [[COPY]], implicit-def $eflags
+    ; ALL: $eax = COPY [[ADD32rr]]
     ; ALL: RET 0, implicit $eax
     %0(s32) = COPY $edi
-    %1(s32) = G_CONSTANT i32 1
+    %1(s8) = G_CONSTANT i8 1
     %2(s32) = G_SHL %0, %1
     $eax = COPY %2(s32)
     RET 0, implicit $eax
@@ -300,16 +292,15 @@ body:             |
     ; ALL: [[COPY:%[0-9]+]]:gr32 = COPY $edi
     ; ALL: [[COPY1:%[0-9]+]]:gr32 = COPY $esi
     ; ALL: [[COPY2:%[0-9]+]]:gr16 = COPY [[COPY]].sub_16bit
-    ; ALL: [[COPY3:%[0-9]+]]:gr16 = COPY [[COPY1]].sub_16bit
-    ; ALL: $cx = COPY [[COPY3]]
-    ; ALL: $cl = KILL killed $cx
+    ; ALL: [[COPY3:%[0-9]+]]:gr8 = COPY [[COPY1]].sub_8bit
+    ; ALL: $cl = COPY [[COPY3]]
     ; ALL: [[SHL16rCL:%[0-9]+]]:gr16 = SHL16rCL [[COPY2]], implicit-def $eflags, implicit $cl
     ; ALL: $ax = COPY [[SHL16rCL]]
     ; ALL: RET 0, implicit $ax
     %0(s32) = COPY $edi
     %1(s32) = COPY $esi
     %2(s16) = G_TRUNC %0(s32)
-    %3(s16) = G_TRUNC %1(s32)
+    %3(s8) = G_TRUNC %1(s32)
     %4(s16) = G_SHL %2, %3
     $ax = COPY %4(s16)
     RET 0, implicit $ax
@@ -337,15 +328,12 @@ body:             |
     ; ALL-LABEL: name: test_shl_i16_imm
     ; ALL: liveins: $edi
     ; ALL: [[COPY:%[0-9]+]]:gr32 = COPY $edi
-    ; ALL: [[MOV16ri:%[0-9]+]]:gr16 = MOV16ri 5
     ; ALL: [[COPY1:%[0-9]+]]:gr16 = COPY [[COPY]].sub_16bit
-    ; ALL: $cx = COPY [[MOV16ri]]
-    ; ALL: $cl = KILL killed $cx
-    ; ALL: [[SHL16rCL:%[0-9]+]]:gr16 = SHL16rCL [[COPY1]], implicit-def $eflags, implicit $cl
-    ; ALL: $ax = COPY [[SHL16rCL]]
+    ; ALL: [[SHL16ri:%[0-9]+]]:gr16 = SHL16ri [[COPY1]], 5, implicit-def $eflags
+    ; ALL: $ax = COPY [[SHL16ri]]
     ; ALL: RET 0, implicit $ax
     %0(s32) = COPY $edi
-    %2(s16) = G_CONSTANT i16 5
+    %2(s8) = G_CONSTANT i8 5
     %1(s16) = G_TRUNC %0(s32)
     %3(s16) = G_SHL %1, %2
     $ax = COPY %3(s16)
@@ -374,15 +362,12 @@ body:             |
     ; ALL-LABEL: name: test_shl_i16_imm1
     ; ALL: liveins: $edi
     ; ALL: [[COPY:%[0-9]+]]:gr32 = COPY $edi
-    ; ALL: [[MOV16ri:%[0-9]+]]:gr16 = MOV16ri 1
     ; ALL: [[COPY1:%[0-9]+]]:gr16 = COPY [[COPY]].sub_16bit
-    ; ALL: $cx = COPY [[MOV16ri]]
-    ; ALL: $cl = KILL killed $cx
-    ; ALL: [[SHL16rCL:%[0-9]+]]:gr16 = SHL16rCL [[COPY1]], implicit-def $eflags, implicit $cl
-    ; ALL: $ax = COPY [[SHL16rCL]]
+    ; ALL: [[ADD16rr:%[0-9]+]]:gr16 = ADD16rr [[COPY1]], [[COPY1]], implicit-def $eflags
+    ; ALL: $ax = COPY [[ADD16rr]]
     ; ALL: RET 0, implicit $ax
     %0(s32) = COPY $edi
-    %2(s16) = G_CONSTANT i16 1
+    %2(s8) = G_CONSTANT i8 1
     %1(s16) = G_TRUNC %0(s32)
     %3(s16) = G_SHL %1, %2
     $ax = COPY %3(s16)
diff --git a/test/CodeGen/X86/GlobalISel/shl-scalar.ll b/test/CodeGen/X86/GlobalISel/shl-scalar.ll
index 4f2de3450f191d5cb154fbdb0317d4a172d01072..e6625c9226e658d92159441f7c8a2bc5f1e7ab96 100644
--- a/test/CodeGen/X86/GlobalISel/shl-scalar.ll
+++ b/test/CodeGen/X86/GlobalISel/shl-scalar.ll
@@ -6,7 +6,7 @@ define i64 @test_shl_i64(i64 %arg1, i64 %arg2) {
 ; X64:       # %bb.0:
 ; X64-NEXT:    movq %rdi, %rax
 ; X64-NEXT:    movq %rsi, %rcx
-; X64-NEXT:    # kill: def $cl killed $rcx
+; X64-NEXT:    # kill: def $cl killed $cl killed $rcx
 ; X64-NEXT:    shlq %cl, %rax
 ; X64-NEXT:    retq
   %res = shl i64 %arg1, %arg2
@@ -18,7 +18,6 @@ define i64 @test_shl_i64_imm(i64 %arg1) {
 ; X64:       # %bb.0:
 ; X64-NEXT:    movq %rdi, %rax
 ; X64-NEXT:    movq $5, %rcx
-; X64-NEXT:    # kill: def $cl killed $rcx
 ; X64-NEXT:    shlq %cl, %rax
 ; X64-NEXT:    retq
   %res = shl i64 %arg1, 5
@@ -30,7 +29,6 @@ define i64 @test_shl_i64_imm1(i64 %arg1) {
 ; X64:       # %bb.0:
 ; X64-NEXT:    movq %rdi, %rax
 ; X64-NEXT:    movq $1, %rcx
-; X64-NEXT:    # kill: def $cl killed $rcx
 ; X64-NEXT:    shlq %cl, %rax
 ; X64-NEXT:    retq
   %res = shl i64 %arg1, 1
@@ -42,7 +40,7 @@ define i32 @test_shl_i32(i32 %arg1, i32 %arg2) {
 ; X64:       # %bb.0:
 ; X64-NEXT:    movl %edi, %eax
 ; X64-NEXT:    movl %esi, %ecx
-; X64-NEXT:    # kill: def $cl killed $ecx
+; X64-NEXT:    # kill: def $cl killed $cl killed $ecx
 ; X64-NEXT:    shll %cl, %eax
 ; X64-NEXT:    retq
   %res = shl i32 %arg1, %arg2
@@ -54,7 +52,6 @@ define i32 @test_shl_i32_imm(i32 %arg1) {
 ; X64:       # %bb.0:
 ; X64-NEXT:    movl %edi, %eax
 ; X64-NEXT:    movl $5, %ecx
-; X64-NEXT:    # kill: def $cl killed $ecx
 ; X64-NEXT:    shll %cl, %eax
 ; X64-NEXT:    retq
   %res = shl i32 %arg1, 5
@@ -66,7 +63,6 @@ define i32 @test_shl_i32_imm1(i32 %arg1) {
 ; X64:       # %bb.0:
 ; X64-NEXT:    movl %edi, %eax
 ; X64-NEXT:    movl $1, %ecx
-; X64-NEXT:    # kill: def $cl killed $ecx
 ; X64-NEXT:    shll %cl, %eax
 ; X64-NEXT:    retq
   %res = shl i32 %arg1, 1
@@ -78,8 +74,7 @@ define i16 @test_shl_i16(i32 %arg1, i32 %arg2) {
 ; X64:       # %bb.0:
 ; X64-NEXT:    movl %edi, %eax
 ; X64-NEXT:    movl %esi, %ecx
-; X64-NEXT:    # kill: def $cx killed $cx killed $ecx
-; X64-NEXT:    # kill: def $cl killed $cx
+; X64-NEXT:    # kill: def $cl killed $cl killed $ecx
 ; X64-NEXT:    shlw %cl, %ax
 ; X64-NEXT:    # kill: def $ax killed $ax killed $eax
 ; X64-NEXT:    retq
@@ -94,7 +89,6 @@ define i16 @test_shl_i16_imm(i32 %arg1) {
 ; X64:       # %bb.0:
 ; X64-NEXT:    movl %edi, %eax
 ; X64-NEXT:    movw $5, %cx
-; X64-NEXT:    # kill: def $cl killed $cx
 ; X64-NEXT:    shlw %cl, %ax
 ; X64-NEXT:    # kill: def $ax killed $ax killed $eax
 ; X64-NEXT:    retq
@@ -108,7 +102,6 @@ define i16 @test_shl_i16_imm1(i32 %arg1) {
 ; X64:       # %bb.0:
 ; X64-NEXT:    movl %edi, %eax
 ; X64-NEXT:    movw $1, %cx
-; X64-NEXT:    # kill: def $cl killed $cx
 ; X64-NEXT:    shlw %cl, %ax
 ; X64-NEXT:    # kill: def $ax killed $ax killed $eax
 ; X64-NEXT:    retq
diff --git a/test/CodeGen/X86/GlobalISel/x86-legalize-inttoptr.mir b/test/CodeGen/X86/GlobalISel/x86-legalize-inttoptr.mir
index 06759ae035aee9155d68ef2768ee69212ddb8e1d..be1e852248d52223d6eb86d2776a93f946d5b68f 100644
--- a/test/CodeGen/X86/GlobalISel/x86-legalize-inttoptr.mir
+++ b/test/CodeGen/X86/GlobalISel/x86-legalize-inttoptr.mir
@@ -26,12 +26,12 @@ body:             |
   bb.1.entry:
     ; CHECK-LABEL: name: inttoptr_p0_s32
     ; CHECK: [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.0
-    ; CHECK: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX]](p0) :: (invariant load 4 from %fixed-stack.0, align 0)
+    ; CHECK: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX]](p0) :: (invariant load 4 from %fixed-stack.0, align 16)
     ; CHECK: [[INTTOPTR:%[0-9]+]]:_(p0) = G_INTTOPTR [[LOAD]](s32)
     ; CHECK: $eax = COPY [[INTTOPTR]](p0)
     ; CHECK: RET 0, implicit $eax
     %1:_(p0) = G_FRAME_INDEX %fixed-stack.0
-    %0:_(s32) = G_LOAD %1(p0) :: (invariant load 4 from %fixed-stack.0, align 0)
+    %0:_(s32) = G_LOAD %1(p0) :: (invariant load 4 from %fixed-stack.0, align 16)
     %2:_(p0) = G_INTTOPTR %0(s32)
     $eax = COPY %2(p0)
     RET 0, implicit $eax
diff --git a/test/CodeGen/X86/GlobalISel/x86-legalize-ptrtoint.mir b/test/CodeGen/X86/GlobalISel/x86-legalize-ptrtoint.mir
index fec474d3bad09e66f6e8e9511ada2b39b78f19bb..60d876e32d97e4ae0498a3428fb97bda2c0b5838 100644
--- a/test/CodeGen/X86/GlobalISel/x86-legalize-ptrtoint.mir
+++ b/test/CodeGen/X86/GlobalISel/x86-legalize-ptrtoint.mir
@@ -45,13 +45,13 @@ body:             |
   bb.1.entry:
     ; CHECK-LABEL: name: ptrtoint_s1_p0
     ; CHECK: [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.0
-    ; CHECK: [[LOAD:%[0-9]+]]:_(p0) = G_LOAD [[FRAME_INDEX]](p0) :: (invariant load 4 from %fixed-stack.0, align 0)
+    ; CHECK: [[LOAD:%[0-9]+]]:_(p0) = G_LOAD [[FRAME_INDEX]](p0) :: (invariant load 4 from %fixed-stack.0, align 16)
     ; CHECK: [[PTRTOINT:%[0-9]+]]:_(s1) = G_PTRTOINT [[LOAD]](p0)
     ; CHECK: [[ANYEXT:%[0-9]+]]:_(s8) = G_ANYEXT [[PTRTOINT]](s1)
     ; CHECK: $al = COPY [[ANYEXT]](s8)
     ; CHECK: RET 0, implicit $al
     %1:_(p0) = G_FRAME_INDEX %fixed-stack.0
-    %0:_(p0) = G_LOAD %1(p0) :: (invariant load 4 from %fixed-stack.0, align 0)
+    %0:_(p0) = G_LOAD %1(p0) :: (invariant load 4 from %fixed-stack.0, align 16)
     %2:_(s1) = G_PTRTOINT %0(p0)
     %3:_(s8) = G_ANYEXT %2(s1)
     $al = COPY %3(s8)
@@ -74,12 +74,12 @@ body:             |
   bb.1.entry:
     ; CHECK-LABEL: name: ptrtoint_s8_p0
     ; CHECK: [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.0
-    ; CHECK: [[LOAD:%[0-9]+]]:_(p0) = G_LOAD [[FRAME_INDEX]](p0) :: (invariant load 4 from %fixed-stack.0, align 0)
+    ; CHECK: [[LOAD:%[0-9]+]]:_(p0) = G_LOAD [[FRAME_INDEX]](p0) :: (invariant load 4 from %fixed-stack.0, align 16)
     ; CHECK: [[PTRTOINT:%[0-9]+]]:_(s8) = G_PTRTOINT [[LOAD]](p0)
     ; CHECK: $al = COPY [[PTRTOINT]](s8)
     ; CHECK: RET 0, implicit $al
     %1:_(p0) = G_FRAME_INDEX %fixed-stack.0
-    %0:_(p0) = G_LOAD %1(p0) :: (invariant load 4 from %fixed-stack.0, align 0)
+    %0:_(p0) = G_LOAD %1(p0) :: (invariant load 4 from %fixed-stack.0, align 16)
     %2:_(s8) = G_PTRTOINT %0(p0)
     $al = COPY %2(s8)
     RET 0, implicit $al
@@ -101,12 +101,12 @@ body:             |
   bb.1.entry:
     ; CHECK-LABEL: name: ptrtoint_s16_p0
     ; CHECK: [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.0
-    ; CHECK: [[LOAD:%[0-9]+]]:_(p0) = G_LOAD [[FRAME_INDEX]](p0) :: (invariant load 4 from %fixed-stack.0, align 0)
+    ; CHECK: [[LOAD:%[0-9]+]]:_(p0) = G_LOAD [[FRAME_INDEX]](p0) :: (invariant load 4 from %fixed-stack.0, align 16)
     ; CHECK: [[PTRTOINT:%[0-9]+]]:_(s16) = G_PTRTOINT [[LOAD]](p0)
     ; CHECK: $ax = COPY [[PTRTOINT]](s16)
     ; CHECK: RET 0, implicit $ax
     %1:_(p0) = G_FRAME_INDEX %fixed-stack.0
-    %0:_(p0) = G_LOAD %1(p0) :: (invariant load 4 from %fixed-stack.0, align 0)
+    %0:_(p0) = G_LOAD %1(p0) :: (invariant load 4 from %fixed-stack.0, align 16)
     %2:_(s16) = G_PTRTOINT %0(p0)
     $ax = COPY %2(s16)
     RET 0, implicit $ax
@@ -128,12 +128,12 @@ body:             |
   bb.1.entry:
     ; CHECK-LABEL: name: ptrtoint_s32_p0
     ; CHECK: [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.0
-    ; CHECK: [[LOAD:%[0-9]+]]:_(p0) = G_LOAD [[FRAME_INDEX]](p0) :: (invariant load 4 from %fixed-stack.0, align 0)
+    ; CHECK: [[LOAD:%[0-9]+]]:_(p0) = G_LOAD [[FRAME_INDEX]](p0) :: (invariant load 4 from %fixed-stack.0, align 16)
     ; CHECK: [[PTRTOINT:%[0-9]+]]:_(s32) = G_PTRTOINT [[LOAD]](p0)
     ; CHECK: $eax = COPY [[PTRTOINT]](s32)
     ; CHECK: RET 0, implicit $eax
     %1:_(p0) = G_FRAME_INDEX %fixed-stack.0
-    %0:_(p0) = G_LOAD %1(p0) :: (invariant load 4 from %fixed-stack.0, align 0)
+    %0:_(p0) = G_LOAD %1(p0) :: (invariant load 4 from %fixed-stack.0, align 16)
     %2:_(s32) = G_PTRTOINT %0(p0)
     $eax = COPY %2(s32)
     RET 0, implicit $eax
diff --git a/test/CodeGen/X86/GlobalISel/x86-legalize-srem.mir b/test/CodeGen/X86/GlobalISel/x86-legalize-srem.mir
index b0f74cf9bc5f6667ab7a1c844112ea104f516ac5..777536a646b7e4138939a5ee9d70fe2da38843ea 100644
--- a/test/CodeGen/X86/GlobalISel/x86-legalize-srem.mir
+++ b/test/CodeGen/X86/GlobalISel/x86-legalize-srem.mir
@@ -70,16 +70,16 @@ body:             |
   bb.1 (%ir-block.0):
     ; CHECK-LABEL: name: test_srem_i8
     ; CHECK: [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.0
-    ; CHECK: [[LOAD:%[0-9]+]]:_(s8) = G_LOAD [[FRAME_INDEX]](p0) :: (invariant load 1 from %fixed-stack.0, align 0)
+    ; CHECK: [[LOAD:%[0-9]+]]:_(s8) = G_LOAD [[FRAME_INDEX]](p0) :: (invariant load 1 from %fixed-stack.0, align 16)
     ; CHECK: [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.1
-    ; CHECK: [[LOAD1:%[0-9]+]]:_(s8) = G_LOAD [[FRAME_INDEX1]](p0) :: (invariant load 1 from %fixed-stack.1, align 0)
+    ; CHECK: [[LOAD1:%[0-9]+]]:_(s8) = G_LOAD [[FRAME_INDEX1]](p0) :: (invariant load 1 from %fixed-stack.1, align 4)
     ; CHECK: [[SREM:%[0-9]+]]:_(s8) = G_SREM [[LOAD]], [[LOAD1]]
     ; CHECK: $al = COPY [[SREM]](s8)
     ; CHECK: RET 0, implicit $al
     %2:_(p0) = G_FRAME_INDEX %fixed-stack.1
-    %0:_(s8) = G_LOAD %2(p0) :: (invariant load 1 from %fixed-stack.1, align 0)
+    %0:_(s8) = G_LOAD %2(p0) :: (invariant load 1 from %fixed-stack.1, align 16)
     %3:_(p0) = G_FRAME_INDEX %fixed-stack.0
-    %1:_(s8) = G_LOAD %3(p0) :: (invariant load 1 from %fixed-stack.0, align 0)
+    %1:_(s8) = G_LOAD %3(p0) :: (invariant load 1 from %fixed-stack.0, align 4)
     %4:_(s8) = G_SREM %0, %1
     $al = COPY %4(s8)
     RET 0, implicit $al
@@ -132,16 +132,16 @@ body:             |
   bb.1 (%ir-block.0):
     ; CHECK-LABEL: name: test_srem_i16
     ; CHECK: [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.0
-    ; CHECK: [[LOAD:%[0-9]+]]:_(s16) = G_LOAD [[FRAME_INDEX]](p0) :: (invariant load 2 from %fixed-stack.0, align 0)
+    ; CHECK: [[LOAD:%[0-9]+]]:_(s16) = G_LOAD [[FRAME_INDEX]](p0) :: (invariant load 2 from %fixed-stack.0, align 16)
     ; CHECK: [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.1
-    ; CHECK: [[LOAD1:%[0-9]+]]:_(s16) = G_LOAD [[FRAME_INDEX1]](p0) :: (invariant load 2 from %fixed-stack.1, align 0)
+    ; CHECK: [[LOAD1:%[0-9]+]]:_(s16) = G_LOAD [[FRAME_INDEX1]](p0) :: (invariant load 2 from %fixed-stack.1, align 4)
     ; CHECK: [[SREM:%[0-9]+]]:_(s16) = G_SREM [[LOAD]], [[LOAD1]]
     ; CHECK: $ax = COPY [[SREM]](s16)
     ; CHECK: RET 0, implicit $ax
     %2:_(p0) = G_FRAME_INDEX %fixed-stack.1
-    %0:_(s16) = G_LOAD %2(p0) :: (invariant load 2 from %fixed-stack.1, align 0)
+    %0:_(s16) = G_LOAD %2(p0) :: (invariant load 2 from %fixed-stack.1, align 16)
     %3:_(p0) = G_FRAME_INDEX %fixed-stack.0
-    %1:_(s16) = G_LOAD %3(p0) :: (invariant load 2 from %fixed-stack.0, align 0)
+    %1:_(s16) = G_LOAD %3(p0) :: (invariant load 2 from %fixed-stack.0, align 4)
     %4:_(s16) = G_SREM %0, %1
     $ax = COPY %4(s16)
     RET 0, implicit $ax
@@ -194,16 +194,16 @@ body:             |
   bb.1 (%ir-block.0):
     ; CHECK-LABEL: name: test_srem_i32
     ; CHECK: [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.0
-    ; CHECK: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX]](p0) :: (invariant load 4 from %fixed-stack.0, align 0)
+    ; CHECK: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX]](p0) :: (invariant load 4 from %fixed-stack.0, align 16)
     ; CHECK: [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.1
-    ; CHECK: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX1]](p0) :: (invariant load 4 from %fixed-stack.1, align 0)
+    ; CHECK: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX1]](p0) :: (invariant load 4 from %fixed-stack.1)
     ; CHECK: [[SREM:%[0-9]+]]:_(s32) = G_SREM [[LOAD]], [[LOAD1]]
     ; CHECK: $eax = COPY [[SREM]](s32)
     ; CHECK: RET 0, implicit $eax
     %2:_(p0) = G_FRAME_INDEX %fixed-stack.1
-    %0:_(s32) = G_LOAD %2(p0) :: (invariant load 4 from %fixed-stack.1, align 0)
+    %0:_(s32) = G_LOAD %2(p0) :: (invariant load 4 from %fixed-stack.1, align 16)
     %3:_(p0) = G_FRAME_INDEX %fixed-stack.0
-    %1:_(s32) = G_LOAD %3(p0) :: (invariant load 4 from %fixed-stack.0, align 0)
+    %1:_(s32) = G_LOAD %3(p0) :: (invariant load 4 from %fixed-stack.0, align 4)
     %4:_(s32) = G_SREM %0, %1
     $eax = COPY %4(s32)
     RET 0, implicit $eax
diff --git a/test/CodeGen/X86/GlobalISel/x86-legalize-urem.mir b/test/CodeGen/X86/GlobalISel/x86-legalize-urem.mir
index 5fd01532066cfea322a5a617842ee42749537734..b9c46b44abbf5d937f770b6498089b2080b5790f 100644
--- a/test/CodeGen/X86/GlobalISel/x86-legalize-urem.mir
+++ b/test/CodeGen/X86/GlobalISel/x86-legalize-urem.mir
@@ -70,16 +70,16 @@ body:             |
   bb.1 (%ir-block.0):
     ; CHECK-LABEL: name: test_urem_i8
     ; CHECK: [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.0
-    ; CHECK: [[LOAD:%[0-9]+]]:_(s8) = G_LOAD [[FRAME_INDEX]](p0) :: (invariant load 1 from %fixed-stack.0, align 0)
+    ; CHECK: [[LOAD:%[0-9]+]]:_(s8) = G_LOAD [[FRAME_INDEX]](p0) :: (invariant load 1 from %fixed-stack.0, align 16)
     ; CHECK: [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.1
-    ; CHECK: [[LOAD1:%[0-9]+]]:_(s8) = G_LOAD [[FRAME_INDEX1]](p0) :: (invariant load 1 from %fixed-stack.1, align 0)
+    ; CHECK: [[LOAD1:%[0-9]+]]:_(s8) = G_LOAD [[FRAME_INDEX1]](p0) :: (invariant load 1 from %fixed-stack.1, align 4)
     ; CHECK: [[UREM:%[0-9]+]]:_(s8) = G_UREM [[LOAD]], [[LOAD1]]
     ; CHECK: $al = COPY [[UREM]](s8)
     ; CHECK: RET 0, implicit $al
     %2:_(p0) = G_FRAME_INDEX %fixed-stack.1
-    %0:_(s8) = G_LOAD %2(p0) :: (invariant load 1 from %fixed-stack.1, align 0)
+    %0:_(s8) = G_LOAD %2(p0) :: (invariant load 1 from %fixed-stack.1, align 16)
     %3:_(p0) = G_FRAME_INDEX %fixed-stack.0
-    %1:_(s8) = G_LOAD %3(p0) :: (invariant load 1 from %fixed-stack.0, align 0)
+    %1:_(s8) = G_LOAD %3(p0) :: (invariant load 1 from %fixed-stack.0, align 4)
     %4:_(s8) = G_UREM %0, %1
     $al = COPY %4(s8)
     RET 0, implicit $al
@@ -132,16 +132,16 @@ body:             |
   bb.1 (%ir-block.0):
     ; CHECK-LABEL: name: test_urem_i16
     ; CHECK: [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.0
-    ; CHECK: [[LOAD:%[0-9]+]]:_(s16) = G_LOAD [[FRAME_INDEX]](p0) :: (invariant load 2 from %fixed-stack.0, align 0)
+    ; CHECK: [[LOAD:%[0-9]+]]:_(s16) = G_LOAD [[FRAME_INDEX]](p0) :: (invariant load 2 from %fixed-stack.0, align 16)
     ; CHECK: [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.1
-    ; CHECK: [[LOAD1:%[0-9]+]]:_(s16) = G_LOAD [[FRAME_INDEX1]](p0) :: (invariant load 2 from %fixed-stack.1, align 0)
+    ; CHECK: [[LOAD1:%[0-9]+]]:_(s16) = G_LOAD [[FRAME_INDEX1]](p0) :: (invariant load 2 from %fixed-stack.1, align 4)
     ; CHECK: [[UREM:%[0-9]+]]:_(s16) = G_UREM [[LOAD]], [[LOAD1]]
     ; CHECK: $ax = COPY [[UREM]](s16)
     ; CHECK: RET 0, implicit $ax
     %2:_(p0) = G_FRAME_INDEX %fixed-stack.1
-    %0:_(s16) = G_LOAD %2(p0) :: (invariant load 2 from %fixed-stack.1, align 0)
+    %0:_(s16) = G_LOAD %2(p0) :: (invariant load 2 from %fixed-stack.1, align 16)
     %3:_(p0) = G_FRAME_INDEX %fixed-stack.0
-    %1:_(s16) = G_LOAD %3(p0) :: (invariant load 2 from %fixed-stack.0, align 0)
+    %1:_(s16) = G_LOAD %3(p0) :: (invariant load 2 from %fixed-stack.0, align 4)
     %4:_(s16) = G_UREM %0, %1
     $ax = COPY %4(s16)
     RET 0, implicit $ax
@@ -194,16 +194,16 @@ body:             |
   bb.1 (%ir-block.0):
     ; CHECK-LABEL: name: test_urem_i32
     ; CHECK: [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.0
-    ; CHECK: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX]](p0) :: (invariant load 4 from %fixed-stack.0, align 0)
+    ; CHECK: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX]](p0) :: (invariant load 4 from %fixed-stack.0, align 16)
     ; CHECK: [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.1
-    ; CHECK: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX1]](p0) :: (invariant load 4 from %fixed-stack.1, align 0)
+    ; CHECK: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX1]](p0) :: (invariant load 4 from %fixed-stack.1)
     ; CHECK: [[UREM:%[0-9]+]]:_(s32) = G_UREM [[LOAD]], [[LOAD1]]
     ; CHECK: $eax = COPY [[UREM]](s32)
     ; CHECK: RET 0, implicit $eax
     %2:_(p0) = G_FRAME_INDEX %fixed-stack.1
-    %0:_(s32) = G_LOAD %2(p0) :: (invariant load 4 from %fixed-stack.1, align 0)
+    %0:_(s32) = G_LOAD %2(p0) :: (invariant load 4 from %fixed-stack.1, align 16)
     %3:_(p0) = G_FRAME_INDEX %fixed-stack.0
-    %1:_(s32) = G_LOAD %3(p0) :: (invariant load 4 from %fixed-stack.0, align 0)
+    %1:_(s32) = G_LOAD %3(p0) :: (invariant load 4 from %fixed-stack.0, align 4)
     %4:_(s32) = G_UREM %0, %1
     $eax = COPY %4(s32)
     RET 0, implicit $eax
diff --git a/test/CodeGen/X86/GlobalISel/x86-select-inttoptr.mir b/test/CodeGen/X86/GlobalISel/x86-select-inttoptr.mir
index 521d1997fb30a6b08819d66fe4ba4f884db3547e..29cc1c4a102e52dffedbd841f58311455db84b3e 100644
--- a/test/CodeGen/X86/GlobalISel/x86-select-inttoptr.mir
+++ b/test/CodeGen/X86/GlobalISel/x86-select-inttoptr.mir
@@ -27,11 +27,11 @@ fixedStack:
 body:             |
   bb.1.entry:
     ; CHECK-LABEL: name: inttoptr_p0_s32
-    ; CHECK: [[MOV32rm:%[0-9]+]]:gr32 = MOV32rm %fixed-stack.0, 1, $noreg, 0, $noreg :: (invariant load 4 from %fixed-stack.0, align 0)
+    ; CHECK: [[MOV32rm:%[0-9]+]]:gr32 = MOV32rm %fixed-stack.0, 1, $noreg, 0, $noreg :: (invariant load 4 from %fixed-stack.0, align 16)
     ; CHECK: $eax = COPY [[MOV32rm]]
     ; CHECK: RET 0, implicit $eax
     %1:gpr(p0) = G_FRAME_INDEX %fixed-stack.0
-    %0:gpr(s32) = G_LOAD %1(p0) :: (invariant load 4 from %fixed-stack.0, align 0)
+    %0:gpr(s32) = G_LOAD %1(p0) :: (invariant load 4 from %fixed-stack.0, align 16)
     %2:gpr(p0) = G_INTTOPTR %0(s32)
     $eax = COPY %2(p0)
     RET 0, implicit $eax
diff --git a/test/CodeGen/X86/GlobalISel/x86-select-ptrtoint.mir b/test/CodeGen/X86/GlobalISel/x86-select-ptrtoint.mir
index 901633bf6fe3cf2c9867958bd620e7c2c47ead95..43e8619ddaa265dbb78f3d0474dfd1397060a9f0 100644
--- a/test/CodeGen/X86/GlobalISel/x86-select-ptrtoint.mir
+++ b/test/CodeGen/X86/GlobalISel/x86-select-ptrtoint.mir
@@ -46,12 +46,12 @@ fixedStack:
 body:             |
   bb.1.entry:
     ; CHECK-LABEL: name: ptrtoint_s1_p0
-    ; CHECK: [[MOV32rm:%[0-9]+]]:gr32_abcd = MOV32rm %fixed-stack.0, 1, $noreg, 0, $noreg :: (invariant load 4 from %fixed-stack.0, align 0)
+    ; CHECK: [[MOV32rm:%[0-9]+]]:gr32_abcd = MOV32rm %fixed-stack.0, 1, $noreg, 0, $noreg :: (invariant load 4 from %fixed-stack.0, align 16)
     ; CHECK: [[COPY:%[0-9]+]]:gr8 = COPY [[MOV32rm]].sub_8bit
     ; CHECK: $al = COPY [[COPY]]
     ; CHECK: RET 0, implicit $al
     %1:gpr(p0) = G_FRAME_INDEX %fixed-stack.0
-    %0:gpr(p0) = G_LOAD %1(p0) :: (invariant load 4 from %fixed-stack.0, align 0)
+    %0:gpr(p0) = G_LOAD %1(p0) :: (invariant load 4 from %fixed-stack.0, align 16)
     %2:gpr(s1) = G_PTRTOINT %0(p0)
     %3:gpr(s8) = G_ANYEXT %2(s1)
     $al = COPY %3(s8)
@@ -75,12 +75,12 @@ fixedStack:
 body:             |
   bb.1.entry:
     ; CHECK-LABEL: name: ptrtoint_s8_p0
-    ; CHECK: [[MOV32rm:%[0-9]+]]:gr32_abcd = MOV32rm %fixed-stack.0, 1, $noreg, 0, $noreg :: (invariant load 4 from %fixed-stack.0, align 0)
+    ; CHECK: [[MOV32rm:%[0-9]+]]:gr32_abcd = MOV32rm %fixed-stack.0, 1, $noreg, 0, $noreg :: (invariant load 4 from %fixed-stack.0, align 16)
     ; CHECK: [[COPY:%[0-9]+]]:gr8 = COPY [[MOV32rm]].sub_8bit
     ; CHECK: $al = COPY [[COPY]]
     ; CHECK: RET 0, implicit $al
     %1:gpr(p0) = G_FRAME_INDEX %fixed-stack.0
-    %0:gpr(p0) = G_LOAD %1(p0) :: (invariant load 4 from %fixed-stack.0, align 0)
+    %0:gpr(p0) = G_LOAD %1(p0) :: (invariant load 4 from %fixed-stack.0, align 16)
     %2:gpr(s8) = G_PTRTOINT %0(p0)
     $al = COPY %2(s8)
     RET 0, implicit $al
@@ -103,12 +103,12 @@ fixedStack:
 body:             |
   bb.1.entry:
     ; CHECK-LABEL: name: ptrtoint_s16_p0
-    ; CHECK: [[MOV32rm:%[0-9]+]]:gr32 = MOV32rm %fixed-stack.0, 1, $noreg, 0, $noreg :: (invariant load 4 from %fixed-stack.0, align 0)
+    ; CHECK: [[MOV32rm:%[0-9]+]]:gr32 = MOV32rm %fixed-stack.0, 1, $noreg, 0, $noreg :: (invariant load 4 from %fixed-stack.0, align 16)
     ; CHECK: [[COPY:%[0-9]+]]:gr16 = COPY [[MOV32rm]].sub_16bit
     ; CHECK: $ax = COPY [[COPY]]
     ; CHECK: RET 0, implicit $ax
     %1:gpr(p0) = G_FRAME_INDEX %fixed-stack.0
-    %0:gpr(p0) = G_LOAD %1(p0) :: (invariant load 4 from %fixed-stack.0, align 0)
+    %0:gpr(p0) = G_LOAD %1(p0) :: (invariant load 4 from %fixed-stack.0, align 16)
     %2:gpr(s16) = G_PTRTOINT %0(p0)
     $ax = COPY %2(s16)
     RET 0, implicit $ax
@@ -131,11 +131,11 @@ fixedStack:
 body:             |
   bb.1.entry:
     ; CHECK-LABEL: name: ptrtoint_s32_p0
-    ; CHECK: [[MOV32rm:%[0-9]+]]:gr32 = MOV32rm %fixed-stack.0, 1, $noreg, 0, $noreg :: (invariant load 4 from %fixed-stack.0, align 0)
+    ; CHECK: [[MOV32rm:%[0-9]+]]:gr32 = MOV32rm %fixed-stack.0, 1, $noreg, 0, $noreg :: (invariant load 4 from %fixed-stack.0, align 16)
     ; CHECK: $eax = COPY [[MOV32rm]]
     ; CHECK: RET 0, implicit $eax
     %1:gpr(p0) = G_FRAME_INDEX %fixed-stack.0
-    %0:gpr(p0) = G_LOAD %1(p0) :: (invariant load 4 from %fixed-stack.0, align 0)
+    %0:gpr(p0) = G_LOAD %1(p0) :: (invariant load 4 from %fixed-stack.0, align 16)
     %2:gpr(s32) = G_PTRTOINT %0(p0)
     $eax = COPY %2(s32)
     RET 0, implicit $eax
diff --git a/test/CodeGen/X86/GlobalISel/x86-select-srem.mir b/test/CodeGen/X86/GlobalISel/x86-select-srem.mir
index 29600a9dd2dab539cec1c7d68202d12972c610e3..50ebc9faaeb958e827dc8b601723104f657160dd 100644
--- a/test/CodeGen/X86/GlobalISel/x86-select-srem.mir
+++ b/test/CodeGen/X86/GlobalISel/x86-select-srem.mir
@@ -69,17 +69,17 @@ constants:
 body:             |
   bb.1 (%ir-block.0):
     ; CHECK-LABEL: name: test_srem_i8
-    ; CHECK: [[MOV8rm:%[0-9]+]]:gr8 = MOV8rm %fixed-stack.0, 1, $noreg, 0, $noreg :: (invariant load 1 from %fixed-stack.0, align 0)
-    ; CHECK: [[MOV8rm1:%[0-9]+]]:gr8 = MOV8rm %fixed-stack.1, 1, $noreg, 0, $noreg :: (invariant load 1 from %fixed-stack.1, align 0)
+    ; CHECK: [[MOV8rm:%[0-9]+]]:gr8 = MOV8rm %fixed-stack.0, 1, $noreg, 0, $noreg :: (invariant load 1 from %fixed-stack.0, align 16)
+    ; CHECK: [[MOV8rm1:%[0-9]+]]:gr8 = MOV8rm %fixed-stack.1, 1, $noreg, 0, $noreg :: (invariant load 1 from %fixed-stack.1, align 4)
     ; CHECK: $ax = MOVSX16rr8 [[MOV8rm]]
     ; CHECK: IDIV8r [[MOV8rm1]], implicit-def $al, implicit-def $ah, implicit-def $eflags, implicit $ax
     ; CHECK: [[COPY:%[0-9]+]]:gr8 = COPY $ah
     ; CHECK: $al = COPY [[COPY]]
     ; CHECK: RET 0, implicit $al
     %2:gpr(p0) = G_FRAME_INDEX %fixed-stack.1
-    %0:gpr(s8) = G_LOAD %2(p0) :: (invariant load 1 from %fixed-stack.1, align 0)
+    %0:gpr(s8) = G_LOAD %2(p0) :: (invariant load 1 from %fixed-stack.1, align 16)
     %3:gpr(p0) = G_FRAME_INDEX %fixed-stack.0
-    %1:gpr(s8) = G_LOAD %3(p0) :: (invariant load 1 from %fixed-stack.0, align 0)
+    %1:gpr(s8) = G_LOAD %3(p0) :: (invariant load 1 from %fixed-stack.0, align 4)
     %4:gpr(s8) = G_SREM %0, %1
     $al = COPY %4(s8)
     RET 0, implicit $al
@@ -131,8 +131,8 @@ constants:
 body:             |
   bb.1 (%ir-block.0):
     ; CHECK-LABEL: name: test_srem_i16
-    ; CHECK: [[MOV16rm:%[0-9]+]]:gr16 = MOV16rm %fixed-stack.0, 1, $noreg, 0, $noreg :: (invariant load 2 from %fixed-stack.0, align 0)
-    ; CHECK: [[MOV16rm1:%[0-9]+]]:gr16 = MOV16rm %fixed-stack.1, 1, $noreg, 0, $noreg :: (invariant load 2 from %fixed-stack.1, align 0)
+    ; CHECK: [[MOV16rm:%[0-9]+]]:gr16 = MOV16rm %fixed-stack.0, 1, $noreg, 0, $noreg :: (invariant load 2 from %fixed-stack.0, align 16)
+    ; CHECK: [[MOV16rm1:%[0-9]+]]:gr16 = MOV16rm %fixed-stack.1, 1, $noreg, 0, $noreg :: (invariant load 2 from %fixed-stack.1, align 4)
     ; CHECK: $ax = COPY [[MOV16rm]]
     ; CHECK: CWD implicit-def $ax, implicit-def $dx, implicit $ax
     ; CHECK: IDIV16r [[MOV16rm1]], implicit-def $ax, implicit-def $dx, implicit-def $eflags, implicit $ax, implicit $dx
@@ -140,9 +140,9 @@ body:             |
     ; CHECK: $ax = COPY [[COPY]]
     ; CHECK: RET 0, implicit $ax
     %2:gpr(p0) = G_FRAME_INDEX %fixed-stack.1
-    %0:gpr(s16) = G_LOAD %2(p0) :: (invariant load 2 from %fixed-stack.1, align 0)
+    %0:gpr(s16) = G_LOAD %2(p0) :: (invariant load 2 from %fixed-stack.1, align 16)
     %3:gpr(p0) = G_FRAME_INDEX %fixed-stack.0
-    %1:gpr(s16) = G_LOAD %3(p0) :: (invariant load 2 from %fixed-stack.0, align 0)
+    %1:gpr(s16) = G_LOAD %3(p0) :: (invariant load 2 from %fixed-stack.0, align 4)
     %4:gpr(s16) = G_SREM %0, %1
     $ax = COPY %4(s16)
     RET 0, implicit $ax
@@ -194,8 +194,8 @@ constants:
 body:             |
   bb.1 (%ir-block.0):
     ; CHECK-LABEL: name: test_srem_i32
-    ; CHECK: [[MOV32rm:%[0-9]+]]:gr32 = MOV32rm %fixed-stack.0, 1, $noreg, 0, $noreg :: (invariant load 4 from %fixed-stack.0, align 0)
-    ; CHECK: [[MOV32rm1:%[0-9]+]]:gr32 = MOV32rm %fixed-stack.1, 1, $noreg, 0, $noreg :: (invariant load 4 from %fixed-stack.1, align 0)
+    ; CHECK: [[MOV32rm:%[0-9]+]]:gr32 = MOV32rm %fixed-stack.0, 1, $noreg, 0, $noreg :: (invariant load 4 from %fixed-stack.0, align 16)
+    ; CHECK: [[MOV32rm1:%[0-9]+]]:gr32 = MOV32rm %fixed-stack.1, 1, $noreg, 0, $noreg :: (invariant load 4 from %fixed-stack.1)
     ; CHECK: $eax = COPY [[MOV32rm]]
     ; CHECK: CDQ implicit-def $eax, implicit-def $edx, implicit $eax
     ; CHECK: IDIV32r [[MOV32rm1]], implicit-def $eax, implicit-def $edx, implicit-def $eflags, implicit $eax, implicit $edx
@@ -203,9 +203,9 @@ body:             |
     ; CHECK: $eax = COPY [[COPY]]
     ; CHECK: RET 0, implicit $eax
     %2:gpr(p0) = G_FRAME_INDEX %fixed-stack.1
-    %0:gpr(s32) = G_LOAD %2(p0) :: (invariant load 4 from %fixed-stack.1, align 0)
+    %0:gpr(s32) = G_LOAD %2(p0) :: (invariant load 4 from %fixed-stack.1, align 16)
     %3:gpr(p0) = G_FRAME_INDEX %fixed-stack.0
-    %1:gpr(s32) = G_LOAD %3(p0) :: (invariant load 4 from %fixed-stack.0, align 0)
+    %1:gpr(s32) = G_LOAD %3(p0) :: (invariant load 4 from %fixed-stack.0, align 4)
     %4:gpr(s32) = G_SREM %0, %1
     $eax = COPY %4(s32)
     RET 0, implicit $eax
diff --git a/test/CodeGen/X86/GlobalISel/x86-select-udiv.mir b/test/CodeGen/X86/GlobalISel/x86-select-udiv.mir
index 4b8766160c7811df784d72e0e4d3eb0ee583d380..afbbab73f51ca195db8e90db34ec97651cb6c3b4 100644
--- a/test/CodeGen/X86/GlobalISel/x86-select-udiv.mir
+++ b/test/CodeGen/X86/GlobalISel/x86-select-udiv.mir
@@ -69,17 +69,17 @@ constants:
 body:             |
   bb.1 (%ir-block.0):
     ; CHECK-LABEL: name: test_udiv_i8
-    ; CHECK: [[MOV8rm:%[0-9]+]]:gr8 = MOV8rm %fixed-stack.0, 1, $noreg, 0, $noreg :: (invariant load 1 from %fixed-stack.0, align 0)
-    ; CHECK: [[MOV8rm1:%[0-9]+]]:gr8 = MOV8rm %fixed-stack.1, 1, $noreg, 0, $noreg :: (invariant load 1 from %fixed-stack.1, align 0)
+    ; CHECK: [[MOV8rm:%[0-9]+]]:gr8 = MOV8rm %fixed-stack.0, 1, $noreg, 0, $noreg :: (invariant load 1 from %fixed-stack.0, align 16)
+    ; CHECK: [[MOV8rm1:%[0-9]+]]:gr8 = MOV8rm %fixed-stack.1, 1, $noreg, 0, $noreg :: (invariant load 1 from %fixed-stack.1, align 4)
     ; CHECK: $ax = MOVZX16rr8 [[MOV8rm]]
     ; CHECK: DIV8r [[MOV8rm1]], implicit-def $al, implicit-def $ah, implicit-def $eflags, implicit $ax
     ; CHECK: [[COPY:%[0-9]+]]:gr8 = COPY $al
     ; CHECK: $al = COPY [[COPY]]
     ; CHECK: RET 0, implicit $al
     %2:gpr(p0) = G_FRAME_INDEX %fixed-stack.1
-    %0:gpr(s8) = G_LOAD %2(p0) :: (invariant load 1 from %fixed-stack.1, align 0)
+    %0:gpr(s8) = G_LOAD %2(p0) :: (invariant load 1 from %fixed-stack.1, align 16)
     %3:gpr(p0) = G_FRAME_INDEX %fixed-stack.0
-    %1:gpr(s8) = G_LOAD %3(p0) :: (invariant load 1 from %fixed-stack.0, align 0)
+    %1:gpr(s8) = G_LOAD %3(p0) :: (invariant load 1 from %fixed-stack.0, align 4)
     %4:gpr(s8) = G_UDIV %0, %1
     $al = COPY %4(s8)
     RET 0, implicit $al
@@ -131,8 +131,8 @@ constants:
 body:             |
   bb.1 (%ir-block.0):
     ; CHECK-LABEL: name: test_udiv_i16
-    ; CHECK: [[MOV16rm:%[0-9]+]]:gr16 = MOV16rm %fixed-stack.0, 1, $noreg, 0, $noreg :: (invariant load 2 from %fixed-stack.0, align 0)
-    ; CHECK: [[MOV16rm1:%[0-9]+]]:gr16 = MOV16rm %fixed-stack.1, 1, $noreg, 0, $noreg :: (invariant load 2 from %fixed-stack.1, align 0)
+    ; CHECK: [[MOV16rm:%[0-9]+]]:gr16 = MOV16rm %fixed-stack.0, 1, $noreg, 0, $noreg :: (invariant load 2 from %fixed-stack.0, align 16)
+    ; CHECK: [[MOV16rm1:%[0-9]+]]:gr16 = MOV16rm %fixed-stack.1, 1, $noreg, 0, $noreg :: (invariant load 2 from %fixed-stack.1, align 4)
     ; CHECK: $ax = COPY [[MOV16rm]]
     ; CHECK: [[MOV32r0_:%[0-9]+]]:gr32 = MOV32r0 implicit-def $eflags
     ; CHECK: $dx = COPY [[MOV32r0_]].sub_16bit
@@ -141,9 +141,9 @@ body:             |
     ; CHECK: $ax = COPY [[COPY]]
     ; CHECK: RET 0, implicit $ax
     %2:gpr(p0) = G_FRAME_INDEX %fixed-stack.1
-    %0:gpr(s16) = G_LOAD %2(p0) :: (invariant load 2 from %fixed-stack.1, align 0)
+    %0:gpr(s16) = G_LOAD %2(p0) :: (invariant load 2 from %fixed-stack.1, align 16)
     %3:gpr(p0) = G_FRAME_INDEX %fixed-stack.0
-    %1:gpr(s16) = G_LOAD %3(p0) :: (invariant load 2 from %fixed-stack.0, align 0)
+    %1:gpr(s16) = G_LOAD %3(p0) :: (invariant load 2 from %fixed-stack.0, align 4)
     %4:gpr(s16) = G_UDIV %0, %1
     $ax = COPY %4(s16)
     RET 0, implicit $ax
@@ -195,8 +195,8 @@ constants:
 body:             |
   bb.1 (%ir-block.0):
     ; CHECK-LABEL: name: test_udiv_i32
-    ; CHECK: [[MOV32rm:%[0-9]+]]:gr32 = MOV32rm %fixed-stack.0, 1, $noreg, 0, $noreg :: (invariant load 4 from %fixed-stack.0, align 0)
-    ; CHECK: [[MOV32rm1:%[0-9]+]]:gr32 = MOV32rm %fixed-stack.1, 1, $noreg, 0, $noreg :: (invariant load 4 from %fixed-stack.1, align 0)
+    ; CHECK: [[MOV32rm:%[0-9]+]]:gr32 = MOV32rm %fixed-stack.0, 1, $noreg, 0, $noreg :: (invariant load 4 from %fixed-stack.0)
+    ; CHECK: [[MOV32rm1:%[0-9]+]]:gr32 = MOV32rm %fixed-stack.1, 1, $noreg, 0, $noreg :: (invariant load 4 from %fixed-stack.1, align 16)
     ; CHECK: $eax = COPY [[MOV32rm]]
     ; CHECK: [[MOV32r0_:%[0-9]+]]:gr32 = MOV32r0 implicit-def $eflags
     ; CHECK: $edx = COPY [[MOV32r0_]]
@@ -205,9 +205,9 @@ body:             |
     ; CHECK: $eax = COPY [[COPY]]
     ; CHECK: RET 0, implicit $eax
     %2:gpr(p0) = G_FRAME_INDEX %fixed-stack.1
-    %0:gpr(s32) = G_LOAD %2(p0) :: (invariant load 4 from %fixed-stack.1, align 0)
+    %0:gpr(s32) = G_LOAD %2(p0) :: (invariant load 4 from %fixed-stack.1, align 4)
     %3:gpr(p0) = G_FRAME_INDEX %fixed-stack.0
-    %1:gpr(s32) = G_LOAD %3(p0) :: (invariant load 4 from %fixed-stack.0, align 0)
+    %1:gpr(s32) = G_LOAD %3(p0) :: (invariant load 4 from %fixed-stack.0, align 16)
     %4:gpr(s32) = G_UDIV %0, %1
     $eax = COPY %4(s32)
     RET 0, implicit $eax
diff --git a/test/CodeGen/X86/GlobalISel/x86-select-urem.mir b/test/CodeGen/X86/GlobalISel/x86-select-urem.mir
index 08f7df4cae3c3abfd522a826b16c8acf16be55bd..1ca33657d7d4c98a092fd55178080a8c8d4dabc5 100644
--- a/test/CodeGen/X86/GlobalISel/x86-select-urem.mir
+++ b/test/CodeGen/X86/GlobalISel/x86-select-urem.mir
@@ -69,17 +69,17 @@ constants:
 body:             |
   bb.1 (%ir-block.0):
     ; CHECK-LABEL: name: test_urem_i8
-    ; CHECK: [[MOV8rm:%[0-9]+]]:gr8 = MOV8rm %fixed-stack.0, 1, $noreg, 0, $noreg :: (invariant load 1 from %fixed-stack.0, align 0)
-    ; CHECK: [[MOV8rm1:%[0-9]+]]:gr8 = MOV8rm %fixed-stack.1, 1, $noreg, 0, $noreg :: (invariant load 1 from %fixed-stack.1, align 0)
+    ; CHECK: [[MOV8rm:%[0-9]+]]:gr8 = MOV8rm %fixed-stack.0, 1, $noreg, 0, $noreg :: (invariant load 1 from %fixed-stack.0, align 16)
+    ; CHECK: [[MOV8rm1:%[0-9]+]]:gr8 = MOV8rm %fixed-stack.1, 1, $noreg, 0, $noreg :: (invariant load 1 from %fixed-stack.1, align 4)
     ; CHECK: $ax = MOVZX16rr8 [[MOV8rm]]
     ; CHECK: DIV8r [[MOV8rm1]], implicit-def $al, implicit-def $ah, implicit-def $eflags, implicit $ax
     ; CHECK: [[COPY:%[0-9]+]]:gr8 = COPY $ah
     ; CHECK: $al = COPY [[COPY]]
     ; CHECK: RET 0, implicit $al
     %2:gpr(p0) = G_FRAME_INDEX %fixed-stack.1
-    %0:gpr(s8) = G_LOAD %2(p0) :: (invariant load 1 from %fixed-stack.1, align 0)
+    %0:gpr(s8) = G_LOAD %2(p0) :: (invariant load 1 from %fixed-stack.1, align 16)
     %3:gpr(p0) = G_FRAME_INDEX %fixed-stack.0
-    %1:gpr(s8) = G_LOAD %3(p0) :: (invariant load 1 from %fixed-stack.0, align 0)
+    %1:gpr(s8) = G_LOAD %3(p0) :: (invariant load 1 from %fixed-stack.0, align 4)
     %4:gpr(s8) = G_UREM %0, %1
     $al = COPY %4(s8)
     RET 0, implicit $al
@@ -131,8 +131,8 @@ constants:
 body:             |
   bb.1 (%ir-block.0):
     ; CHECK-LABEL: name: test_urem_i16
-    ; CHECK: [[MOV16rm:%[0-9]+]]:gr16 = MOV16rm %fixed-stack.0, 1, $noreg, 0, $noreg :: (invariant load 2 from %fixed-stack.0, align 0)
-    ; CHECK: [[MOV16rm1:%[0-9]+]]:gr16 = MOV16rm %fixed-stack.1, 1, $noreg, 0, $noreg :: (invariant load 2 from %fixed-stack.1, align 0)
+    ; CHECK: [[MOV16rm:%[0-9]+]]:gr16 = MOV16rm %fixed-stack.0, 1, $noreg, 0, $noreg :: (invariant load 2 from %fixed-stack.0, align 16)
+    ; CHECK: [[MOV16rm1:%[0-9]+]]:gr16 = MOV16rm %fixed-stack.1, 1, $noreg, 0, $noreg :: (invariant load 2 from %fixed-stack.1, align 4)
     ; CHECK: $ax = COPY [[MOV16rm]]
     ; CHECK: [[MOV32r0_:%[0-9]+]]:gr32 = MOV32r0 implicit-def $eflags
     ; CHECK: $dx = COPY [[MOV32r0_]].sub_16bit
@@ -141,9 +141,9 @@ body:             |
     ; CHECK: $ax = COPY [[COPY]]
     ; CHECK: RET 0, implicit $ax
     %2:gpr(p0) = G_FRAME_INDEX %fixed-stack.1
-    %0:gpr(s16) = G_LOAD %2(p0) :: (invariant load 2 from %fixed-stack.1, align 0)
+    %0:gpr(s16) = G_LOAD %2(p0) :: (invariant load 2 from %fixed-stack.1, align 16)
     %3:gpr(p0) = G_FRAME_INDEX %fixed-stack.0
-    %1:gpr(s16) = G_LOAD %3(p0) :: (invariant load 2 from %fixed-stack.0, align 0)
+    %1:gpr(s16) = G_LOAD %3(p0) :: (invariant load 2 from %fixed-stack.0, align 4)
     %4:gpr(s16) = G_UREM %0, %1
     $ax = COPY %4(s16)
     RET 0, implicit $ax
@@ -195,8 +195,8 @@ constants:
 body:             |
   bb.1 (%ir-block.0):
     ; CHECK-LABEL: name: test_urem_i32
-    ; CHECK: [[MOV32rm:%[0-9]+]]:gr32 = MOV32rm %fixed-stack.0, 1, $noreg, 0, $noreg :: (invariant load 4 from %fixed-stack.0, align 0)
-    ; CHECK: [[MOV32rm1:%[0-9]+]]:gr32 = MOV32rm %fixed-stack.1, 1, $noreg, 0, $noreg :: (invariant load 4 from %fixed-stack.1, align 0)
+    ; CHECK: [[MOV32rm:%[0-9]+]]:gr32 = MOV32rm %fixed-stack.0, 1, $noreg, 0, $noreg :: (invariant load 4 from %fixed-stack.0, align 16)
+    ; CHECK: [[MOV32rm1:%[0-9]+]]:gr32 = MOV32rm %fixed-stack.1, 1, $noreg, 0, $noreg :: (invariant load 4 from %fixed-stack.1)
     ; CHECK: $eax = COPY [[MOV32rm]]
     ; CHECK: [[MOV32r0_:%[0-9]+]]:gr32 = MOV32r0 implicit-def $eflags
     ; CHECK: $edx = COPY [[MOV32r0_]]
@@ -205,9 +205,9 @@ body:             |
     ; CHECK: $eax = COPY [[COPY]]
     ; CHECK: RET 0, implicit $eax
     %2:gpr(p0) = G_FRAME_INDEX %fixed-stack.1
-    %0:gpr(s32) = G_LOAD %2(p0) :: (invariant load 4 from %fixed-stack.1, align 0)
+    %0:gpr(s32) = G_LOAD %2(p0) :: (invariant load 4 from %fixed-stack.1, align 16)
     %3:gpr(p0) = G_FRAME_INDEX %fixed-stack.0
-    %1:gpr(s32) = G_LOAD %3(p0) :: (invariant load 4 from %fixed-stack.0, align 0)
+    %1:gpr(s32) = G_LOAD %3(p0) :: (invariant load 4 from %fixed-stack.0, align 4)
     %4:gpr(s32) = G_UREM %0, %1
     $eax = COPY %4(s32)
     RET 0, implicit $eax
diff --git a/test/CodeGen/X86/GlobalISel/x86_64-fallback.ll b/test/CodeGen/X86/GlobalISel/x86_64-fallback.ll
index 92bd661286c988e38bbed66154116c5f1eb320a3..8083af34736d2fadff19febfbf359e1bf003c7a6 100644
--- a/test/CodeGen/X86/GlobalISel/x86_64-fallback.ll
+++ b/test/CodeGen/X86/GlobalISel/x86_64-fallback.ll
@@ -8,7 +8,7 @@
 ; the fallback path.
 
 ; Check that we fallback on invoke translation failures.
-; FALLBACK-WITH-REPORT-ERR: remark: <unknown>:0:0: unable to legalize instruction: G_STORE %1:_(s80), %0:_(p0) :: (store 10 into %ir.ptr, align 16) (in function: test_x86_fp80_dump)
+; FALLBACK-WITH-REPORT-ERR: remark: <unknown>:0:0: unable to legalize instruction: %1:_(s80) = G_FCONSTANT x86_fp80 0xK4002A000000000000000
 ; FALLBACK-WITH-REPORT-ERR: warning: Instruction selection used fallback path for test_x86_fp80_dump
 ; FALLBACK-WITH-REPORT-OUT-LABEL: test_x86_fp80_dump:
 define void @test_x86_fp80_dump(x86_fp80* %ptr){
diff --git a/test/CodeGen/X86/GlobalISel/x86_64-legalize-sitofp.mir b/test/CodeGen/X86/GlobalISel/x86_64-legalize-sitofp.mir
index a8be320de786fbfcf0cb612ed74f4bada2bec18f..0df0cdc2fc0953f7023c03a11b7e5419ea3a9711 100644
--- a/test/CodeGen/X86/GlobalISel/x86_64-legalize-sitofp.mir
+++ b/test/CodeGen/X86/GlobalISel/x86_64-legalize-sitofp.mir
@@ -1,5 +1,5 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=x86_64-linux-gnu -global-isel -run-pass=legalizer -verify-machineinstrs %s -o - | FileCheck %s
+# RUN: llc -O0 -mtriple=x86_64-linux-gnu -global-isel -run-pass=legalizer -verify-machineinstrs %s -o - | FileCheck %s
 
 --- |
   ; ModuleID = 'sitofp.ll'
@@ -69,7 +69,7 @@
   !llvm.ident = !{!1}
 
   !0 = !{i32 1, !"wchar_size", i32 4}
-  !1 = !{!"clang version 7.0.0 (http://llvm.org/git/clang.git a05f37359b23be7c068e19968c8f106edf6f2b34) (http://llvm.org/git/llvm.git d693de1fee74d455e20f96006aac50317ca1da6b)"}
+  !1 = !{!"clang version 7.0.0"}
 
 ...
 ---
@@ -90,8 +90,10 @@ body:             |
     ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $edi
     ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
     ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
-    ; CHECK: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY1]], [[C]]
-    ; CHECK: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C]]
+    ; CHECK: [[TRUNC:%[0-9]+]]:_(s8) = G_TRUNC [[C]](s32)
+    ; CHECK: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY1]], [[TRUNC]](s8)
+    ; CHECK: [[TRUNC1:%[0-9]+]]:_(s8) = G_TRUNC [[C]](s32)
+    ; CHECK: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[TRUNC1]](s8)
     ; CHECK: [[SITOFP:%[0-9]+]]:_(s32) = G_SITOFP [[ASHR]](s32)
     ; CHECK: [[ANYEXT:%[0-9]+]]:_(s128) = G_ANYEXT [[SITOFP]](s32)
     ; CHECK: $xmm0 = COPY [[ANYEXT]](s128)
@@ -122,8 +124,10 @@ body:             |
     ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $edi
     ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
     ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
-    ; CHECK: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY1]], [[C]]
-    ; CHECK: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C]]
+    ; CHECK: [[TRUNC:%[0-9]+]]:_(s8) = G_TRUNC [[C]](s32)
+    ; CHECK: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY1]], [[TRUNC]](s8)
+    ; CHECK: [[TRUNC1:%[0-9]+]]:_(s8) = G_TRUNC [[C]](s32)
+    ; CHECK: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[TRUNC1]](s8)
     ; CHECK: [[SITOFP:%[0-9]+]]:_(s32) = G_SITOFP [[ASHR]](s32)
     ; CHECK: [[ANYEXT:%[0-9]+]]:_(s128) = G_ANYEXT [[SITOFP]](s32)
     ; CHECK: $xmm0 = COPY [[ANYEXT]](s128)
@@ -206,8 +210,10 @@ body:             |
     ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $edi
     ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
     ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
-    ; CHECK: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY1]], [[C]]
-    ; CHECK: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C]]
+    ; CHECK: [[TRUNC:%[0-9]+]]:_(s8) = G_TRUNC [[C]](s32)
+    ; CHECK: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY1]], [[TRUNC]](s8)
+    ; CHECK: [[TRUNC1:%[0-9]+]]:_(s8) = G_TRUNC [[C]](s32)
+    ; CHECK: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[TRUNC1]](s8)
     ; CHECK: [[SITOFP:%[0-9]+]]:_(s64) = G_SITOFP [[ASHR]](s32)
     ; CHECK: [[ANYEXT:%[0-9]+]]:_(s128) = G_ANYEXT [[SITOFP]](s64)
     ; CHECK: $xmm0 = COPY [[ANYEXT]](s128)
@@ -238,8 +244,10 @@ body:             |
     ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $edi
     ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
     ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
-    ; CHECK: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY1]], [[C]]
-    ; CHECK: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[C]]
+    ; CHECK: [[TRUNC:%[0-9]+]]:_(s8) = G_TRUNC [[C]](s32)
+    ; CHECK: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY1]], [[TRUNC]](s8)
+    ; CHECK: [[TRUNC1:%[0-9]+]]:_(s8) = G_TRUNC [[C]](s32)
+    ; CHECK: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SHL]], [[TRUNC1]](s8)
     ; CHECK: [[SITOFP:%[0-9]+]]:_(s64) = G_SITOFP [[ASHR]](s32)
     ; CHECK: [[ANYEXT:%[0-9]+]]:_(s128) = G_ANYEXT [[SITOFP]](s64)
     ; CHECK: $xmm0 = COPY [[ANYEXT]](s128)
diff --git a/test/CodeGen/X86/GlobalISel/x86_64-select-sitofp.mir b/test/CodeGen/X86/GlobalISel/x86_64-select-sitofp.mir
index 896c2f4b396d6a60a1900b33c2e5b45187c19b3b..eeb976cae380278d8098b5a014c0cfab5ccec6c6 100644
--- a/test/CodeGen/X86/GlobalISel/x86_64-select-sitofp.mir
+++ b/test/CodeGen/X86/GlobalISel/x86_64-select-sitofp.mir
@@ -41,7 +41,7 @@
   !llvm.ident = !{!1}
 
   !0 = !{i32 1, !"wchar_size", i32 4}
-  !1 = !{!"clang version 7.0.0 (http://llvm.org/git/clang.git a05f37359b23be7c068e19968c8f106edf6f2b34) (http://llvm.org/git/llvm.git d693de1fee74d455e20f96006aac50317ca1da6b)"}
+  !1 = !{!"clang version 7.0.0"}
 
 ...
 ---
diff --git a/test/CodeGen/X86/abi-isel.ll b/test/CodeGen/X86/abi-isel.ll
index 62b8a8c7a5b4902bd82a0decd7cc8c78a8ce26b0..967aac4bc9b59ed19b7d6c48caa82e175ba39280 100644
--- a/test/CodeGen/X86/abi-isel.ll
+++ b/test/CodeGen/X86/abi-isel.ll
@@ -1,8 +1,9 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; NOTE: Use the --no_x86_scrub_rip additional argument to keep the rip address math.
 
 ; RUN: llc < %s -mcpu=generic -mtriple=x86_64-unknown-linux-gnu -relocation-model=static -code-model=small -pre-RA-sched=list-ilp | FileCheck %s -check-prefix=LINUX-64-STATIC
 ; RUN: llc < %s -mcpu=generic -mtriple=i686-unknown-linux-gnu -relocation-model=static -code-model=small -pre-RA-sched=list-ilp | FileCheck %s -check-prefix=LINUX-32-STATIC
-; RUN: llc < %s -mcpu=generic -mtriple=i686-unknown-linux-gnu -relocation-model=static -code-model=small -pre-RA-sched=list-ilp | FileCheck %s -check-prefix=LINUX-32-PIC
+; RUN: llc < %s -mcpu=generic -mtriple=i686-unknown-linux-gnu -relocation-model=pic -code-model=small -pre-RA-sched=list-ilp | FileCheck %s -check-prefix=LINUX-32-PIC
 ; RUN: llc < %s -mcpu=generic -mtriple=x86_64-unknown-linux-gnu -relocation-model=pic -code-model=small -pre-RA-sched=list-ilp | FileCheck %s -check-prefix=LINUX-64-PIC
 
 ; RUN: llc < %s -mcpu=generic -mtriple=i686-apple-darwin -relocation-model=static -code-model=small -pre-RA-sched=list-ilp | FileCheck %s -check-prefix=DARWIN-32-STATIC
@@ -47,8 +48,15 @@ define void @foo00() nounwind {
 ;
 ; LINUX-32-PIC-LABEL: foo00:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl src, %eax
-; LINUX-32-PIC-NEXT:    movl %eax, dst
+; LINUX-32-PIC-NEXT:    calll .L0$pb
+; LINUX-32-PIC-NEXT:  .L0$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp0:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp0-.L0$pb), %eax
+; LINUX-32-PIC-NEXT:    movl src@GOT(%eax), %ecx
+; LINUX-32-PIC-NEXT:    movl (%ecx), %ecx
+; LINUX-32-PIC-NEXT:    movl dst@GOT(%eax), %eax
+; LINUX-32-PIC-NEXT:    movl %ecx, (%eax)
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: foo00:
@@ -130,8 +138,15 @@ define void @fxo00() nounwind {
 ;
 ; LINUX-32-PIC-LABEL: fxo00:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl xsrc, %eax
-; LINUX-32-PIC-NEXT:    movl %eax, xdst
+; LINUX-32-PIC-NEXT:    calll .L1$pb
+; LINUX-32-PIC-NEXT:  .L1$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp1:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp1-.L1$pb), %eax
+; LINUX-32-PIC-NEXT:    movl xsrc@GOT(%eax), %ecx
+; LINUX-32-PIC-NEXT:    movl (%ecx), %ecx
+; LINUX-32-PIC-NEXT:    movl xdst@GOT(%eax), %eax
+; LINUX-32-PIC-NEXT:    movl %ecx, (%eax)
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: fxo00:
@@ -211,7 +226,14 @@ define void @foo01() nounwind {
 ;
 ; LINUX-32-PIC-LABEL: foo01:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl $dst, ptr
+; LINUX-32-PIC-NEXT:    calll .L2$pb
+; LINUX-32-PIC-NEXT:  .L2$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp2:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp2-.L2$pb), %eax
+; LINUX-32-PIC-NEXT:    movl dst@GOT(%eax), %ecx
+; LINUX-32-PIC-NEXT:    movl ptr@GOT(%eax), %eax
+; LINUX-32-PIC-NEXT:    movl %ecx, (%eax)
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: foo01:
@@ -282,7 +304,14 @@ define void @fxo01() nounwind {
 ;
 ; LINUX-32-PIC-LABEL: fxo01:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl $xdst, ptr
+; LINUX-32-PIC-NEXT:    calll .L3$pb
+; LINUX-32-PIC-NEXT:  .L3$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp3:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp3-.L3$pb), %eax
+; LINUX-32-PIC-NEXT:    movl xdst@GOT(%eax), %ecx
+; LINUX-32-PIC-NEXT:    movl ptr@GOT(%eax), %eax
+; LINUX-32-PIC-NEXT:    movl %ecx, (%eax)
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: fxo01:
@@ -357,9 +386,16 @@ define void @foo02() nounwind {
 ;
 ; LINUX-32-PIC-LABEL: foo02:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl src, %eax
-; LINUX-32-PIC-NEXT:    movl ptr, %ecx
-; LINUX-32-PIC-NEXT:    movl %eax, (%ecx)
+; LINUX-32-PIC-NEXT:    calll .L4$pb
+; LINUX-32-PIC-NEXT:  .L4$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp4:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp4-.L4$pb), %eax
+; LINUX-32-PIC-NEXT:    movl src@GOT(%eax), %ecx
+; LINUX-32-PIC-NEXT:    movl (%ecx), %ecx
+; LINUX-32-PIC-NEXT:    movl ptr@GOT(%eax), %eax
+; LINUX-32-PIC-NEXT:    movl (%eax), %eax
+; LINUX-32-PIC-NEXT:    movl %ecx, (%eax)
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: foo02:
@@ -450,9 +486,16 @@ define void @fxo02() nounwind {
 ;
 ; LINUX-32-PIC-LABEL: fxo02:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl xsrc, %eax
-; LINUX-32-PIC-NEXT:    movl ptr, %ecx
-; LINUX-32-PIC-NEXT:    movl %eax, (%ecx)
+; LINUX-32-PIC-NEXT:    calll .L5$pb
+; LINUX-32-PIC-NEXT:  .L5$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp5:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp5-.L5$pb), %eax
+; LINUX-32-PIC-NEXT:    movl xsrc@GOT(%eax), %ecx
+; LINUX-32-PIC-NEXT:    movl (%ecx), %ecx
+; LINUX-32-PIC-NEXT:    movl ptr@GOT(%eax), %eax
+; LINUX-32-PIC-NEXT:    movl (%eax), %eax
+; LINUX-32-PIC-NEXT:    movl %ecx, (%eax)
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: fxo02:
@@ -541,8 +584,15 @@ define void @foo03() nounwind {
 ;
 ; LINUX-32-PIC-LABEL: foo03:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl dsrc, %eax
-; LINUX-32-PIC-NEXT:    movl %eax, ddst
+; LINUX-32-PIC-NEXT:    calll .L6$pb
+; LINUX-32-PIC-NEXT:  .L6$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp6:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp6-.L6$pb), %eax
+; LINUX-32-PIC-NEXT:    movl dsrc@GOT(%eax), %ecx
+; LINUX-32-PIC-NEXT:    movl (%ecx), %ecx
+; LINUX-32-PIC-NEXT:    movl ddst@GOT(%eax), %eax
+; LINUX-32-PIC-NEXT:    movl %ecx, (%eax)
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: foo03:
@@ -611,7 +661,14 @@ define void @foo04() nounwind {
 ;
 ; LINUX-32-PIC-LABEL: foo04:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl $ddst, dptr
+; LINUX-32-PIC-NEXT:    calll .L7$pb
+; LINUX-32-PIC-NEXT:  .L7$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp7:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp7-.L7$pb), %eax
+; LINUX-32-PIC-NEXT:    movl ddst@GOT(%eax), %ecx
+; LINUX-32-PIC-NEXT:    movl dptr@GOT(%eax), %eax
+; LINUX-32-PIC-NEXT:    movl %ecx, (%eax)
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: foo04:
@@ -680,9 +737,16 @@ define void @foo05() nounwind {
 ;
 ; LINUX-32-PIC-LABEL: foo05:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl dsrc, %eax
-; LINUX-32-PIC-NEXT:    movl dptr, %ecx
-; LINUX-32-PIC-NEXT:    movl %eax, (%ecx)
+; LINUX-32-PIC-NEXT:    calll .L8$pb
+; LINUX-32-PIC-NEXT:  .L8$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp8:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp8-.L8$pb), %eax
+; LINUX-32-PIC-NEXT:    movl dsrc@GOT(%eax), %ecx
+; LINUX-32-PIC-NEXT:    movl (%ecx), %ecx
+; LINUX-32-PIC-NEXT:    movl dptr@GOT(%eax), %eax
+; LINUX-32-PIC-NEXT:    movl (%eax), %eax
+; LINUX-32-PIC-NEXT:    movl %ecx, (%eax)
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: foo05:
@@ -761,8 +825,13 @@ define void @foo06() nounwind {
 ;
 ; LINUX-32-PIC-LABEL: foo06:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl lsrc, %eax
-; LINUX-32-PIC-NEXT:    movl %eax, ldst
+; LINUX-32-PIC-NEXT:    calll .L9$pb
+; LINUX-32-PIC-NEXT:  .L9$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp9:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp9-.L9$pb), %eax
+; LINUX-32-PIC-NEXT:    movl lsrc@GOTOFF(%eax), %ecx
+; LINUX-32-PIC-NEXT:    movl %ecx, ldst@GOTOFF(%eax)
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: foo06:
@@ -829,7 +898,13 @@ define void @foo07() nounwind {
 ;
 ; LINUX-32-PIC-LABEL: foo07:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl $ldst, lptr
+; LINUX-32-PIC-NEXT:    calll .L10$pb
+; LINUX-32-PIC-NEXT:  .L10$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp10:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp10-.L10$pb), %eax
+; LINUX-32-PIC-NEXT:    leal ldst@GOTOFF(%eax), %ecx
+; LINUX-32-PIC-NEXT:    movl %ecx, lptr@GOTOFF(%eax)
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: foo07:
@@ -897,9 +972,14 @@ define void @foo08() nounwind {
 ;
 ; LINUX-32-PIC-LABEL: foo08:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl lsrc, %eax
-; LINUX-32-PIC-NEXT:    movl lptr, %ecx
-; LINUX-32-PIC-NEXT:    movl %eax, (%ecx)
+; LINUX-32-PIC-NEXT:    calll .L11$pb
+; LINUX-32-PIC-NEXT:  .L11$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp11:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp11-.L11$pb), %eax
+; LINUX-32-PIC-NEXT:    movl lsrc@GOTOFF(%eax), %ecx
+; LINUX-32-PIC-NEXT:    movl lptr@GOTOFF(%eax), %eax
+; LINUX-32-PIC-NEXT:    movl %ecx, (%eax)
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: foo08:
@@ -976,8 +1056,15 @@ define void @qux00() nounwind {
 ;
 ; LINUX-32-PIC-LABEL: qux00:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl src+64, %eax
-; LINUX-32-PIC-NEXT:    movl %eax, dst+64
+; LINUX-32-PIC-NEXT:    calll .L12$pb
+; LINUX-32-PIC-NEXT:  .L12$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp12:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp12-.L12$pb), %eax
+; LINUX-32-PIC-NEXT:    movl src@GOT(%eax), %ecx
+; LINUX-32-PIC-NEXT:    movl 64(%ecx), %ecx
+; LINUX-32-PIC-NEXT:    movl dst@GOT(%eax), %eax
+; LINUX-32-PIC-NEXT:    movl %ecx, 64(%eax)
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: qux00:
@@ -1058,8 +1145,15 @@ define void @qxx00() nounwind {
 ;
 ; LINUX-32-PIC-LABEL: qxx00:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl xsrc+64, %eax
-; LINUX-32-PIC-NEXT:    movl %eax, xdst+64
+; LINUX-32-PIC-NEXT:    calll .L13$pb
+; LINUX-32-PIC-NEXT:  .L13$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp13:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp13-.L13$pb), %eax
+; LINUX-32-PIC-NEXT:    movl xsrc@GOT(%eax), %ecx
+; LINUX-32-PIC-NEXT:    movl 64(%ecx), %ecx
+; LINUX-32-PIC-NEXT:    movl xdst@GOT(%eax), %eax
+; LINUX-32-PIC-NEXT:    movl %ecx, 64(%eax)
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: qxx00:
@@ -1138,7 +1232,15 @@ define void @qux01() nounwind {
 ;
 ; LINUX-32-PIC-LABEL: qux01:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl $dst+64, ptr
+; LINUX-32-PIC-NEXT:    calll .L14$pb
+; LINUX-32-PIC-NEXT:  .L14$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp14:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp14-.L14$pb), %eax
+; LINUX-32-PIC-NEXT:    movl dst@GOT(%eax), %ecx
+; LINUX-32-PIC-NEXT:    addl $64, %ecx
+; LINUX-32-PIC-NEXT:    movl ptr@GOT(%eax), %eax
+; LINUX-32-PIC-NEXT:    movl %ecx, (%eax)
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: qux01:
@@ -1215,7 +1317,15 @@ define void @qxx01() nounwind {
 ;
 ; LINUX-32-PIC-LABEL: qxx01:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl $xdst+64, ptr
+; LINUX-32-PIC-NEXT:    calll .L15$pb
+; LINUX-32-PIC-NEXT:  .L15$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp15:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp15-.L15$pb), %eax
+; LINUX-32-PIC-NEXT:    movl xdst@GOT(%eax), %ecx
+; LINUX-32-PIC-NEXT:    addl $64, %ecx
+; LINUX-32-PIC-NEXT:    movl ptr@GOT(%eax), %eax
+; LINUX-32-PIC-NEXT:    movl %ecx, (%eax)
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: qxx01:
@@ -1296,9 +1406,16 @@ define void @qux02() nounwind {
 ;
 ; LINUX-32-PIC-LABEL: qux02:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl src+64, %eax
-; LINUX-32-PIC-NEXT:    movl ptr, %ecx
-; LINUX-32-PIC-NEXT:    movl %eax, 64(%ecx)
+; LINUX-32-PIC-NEXT:    calll .L16$pb
+; LINUX-32-PIC-NEXT:  .L16$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp16:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp16-.L16$pb), %eax
+; LINUX-32-PIC-NEXT:    movl src@GOT(%eax), %ecx
+; LINUX-32-PIC-NEXT:    movl 64(%ecx), %ecx
+; LINUX-32-PIC-NEXT:    movl ptr@GOT(%eax), %eax
+; LINUX-32-PIC-NEXT:    movl (%eax), %eax
+; LINUX-32-PIC-NEXT:    movl %ecx, 64(%eax)
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: qux02:
@@ -1390,9 +1507,16 @@ define void @qxx02() nounwind {
 ;
 ; LINUX-32-PIC-LABEL: qxx02:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl xsrc+64, %eax
-; LINUX-32-PIC-NEXT:    movl ptr, %ecx
-; LINUX-32-PIC-NEXT:    movl %eax, 64(%ecx)
+; LINUX-32-PIC-NEXT:    calll .L17$pb
+; LINUX-32-PIC-NEXT:  .L17$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp17:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp17-.L17$pb), %eax
+; LINUX-32-PIC-NEXT:    movl xsrc@GOT(%eax), %ecx
+; LINUX-32-PIC-NEXT:    movl 64(%ecx), %ecx
+; LINUX-32-PIC-NEXT:    movl ptr@GOT(%eax), %eax
+; LINUX-32-PIC-NEXT:    movl (%eax), %eax
+; LINUX-32-PIC-NEXT:    movl %ecx, 64(%eax)
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: qxx02:
@@ -1482,8 +1606,15 @@ define void @qux03() nounwind {
 ;
 ; LINUX-32-PIC-LABEL: qux03:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl dsrc+64, %eax
-; LINUX-32-PIC-NEXT:    movl %eax, ddst+64
+; LINUX-32-PIC-NEXT:    calll .L18$pb
+; LINUX-32-PIC-NEXT:  .L18$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp18:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp18-.L18$pb), %eax
+; LINUX-32-PIC-NEXT:    movl dsrc@GOT(%eax), %ecx
+; LINUX-32-PIC-NEXT:    movl 64(%ecx), %ecx
+; LINUX-32-PIC-NEXT:    movl ddst@GOT(%eax), %eax
+; LINUX-32-PIC-NEXT:    movl %ecx, 64(%eax)
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: qux03:
@@ -1552,7 +1683,15 @@ define void @qux04() nounwind {
 ;
 ; LINUX-32-PIC-LABEL: qux04:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl $ddst+64, dptr
+; LINUX-32-PIC-NEXT:    calll .L19$pb
+; LINUX-32-PIC-NEXT:  .L19$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp19:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp19-.L19$pb), %eax
+; LINUX-32-PIC-NEXT:    movl ddst@GOT(%eax), %ecx
+; LINUX-32-PIC-NEXT:    addl $64, %ecx
+; LINUX-32-PIC-NEXT:    movl dptr@GOT(%eax), %eax
+; LINUX-32-PIC-NEXT:    movl %ecx, (%eax)
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: qux04:
@@ -1622,9 +1761,16 @@ define void @qux05() nounwind {
 ;
 ; LINUX-32-PIC-LABEL: qux05:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl dsrc+64, %eax
-; LINUX-32-PIC-NEXT:    movl dptr, %ecx
-; LINUX-32-PIC-NEXT:    movl %eax, 64(%ecx)
+; LINUX-32-PIC-NEXT:    calll .L20$pb
+; LINUX-32-PIC-NEXT:  .L20$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp20:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp20-.L20$pb), %eax
+; LINUX-32-PIC-NEXT:    movl dsrc@GOT(%eax), %ecx
+; LINUX-32-PIC-NEXT:    movl 64(%ecx), %ecx
+; LINUX-32-PIC-NEXT:    movl dptr@GOT(%eax), %eax
+; LINUX-32-PIC-NEXT:    movl (%eax), %eax
+; LINUX-32-PIC-NEXT:    movl %ecx, 64(%eax)
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: qux05:
@@ -1704,8 +1850,13 @@ define void @qux06() nounwind {
 ;
 ; LINUX-32-PIC-LABEL: qux06:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl lsrc+64, %eax
-; LINUX-32-PIC-NEXT:    movl %eax, ldst+64
+; LINUX-32-PIC-NEXT:    calll .L21$pb
+; LINUX-32-PIC-NEXT:  .L21$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp21:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp21-.L21$pb), %eax
+; LINUX-32-PIC-NEXT:    movl lsrc@GOTOFF+64(%eax), %ecx
+; LINUX-32-PIC-NEXT:    movl %ecx, ldst@GOTOFF+64(%eax)
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: qux06:
@@ -1772,7 +1923,13 @@ define void @qux07() nounwind {
 ;
 ; LINUX-32-PIC-LABEL: qux07:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl $ldst+64, lptr
+; LINUX-32-PIC-NEXT:    calll .L22$pb
+; LINUX-32-PIC-NEXT:  .L22$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp22:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp22-.L22$pb), %eax
+; LINUX-32-PIC-NEXT:    leal ldst@GOTOFF+64(%eax), %ecx
+; LINUX-32-PIC-NEXT:    movl %ecx, lptr@GOTOFF(%eax)
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: qux07:
@@ -1840,9 +1997,14 @@ define void @qux08() nounwind {
 ;
 ; LINUX-32-PIC-LABEL: qux08:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl lsrc+64, %eax
-; LINUX-32-PIC-NEXT:    movl lptr, %ecx
-; LINUX-32-PIC-NEXT:    movl %eax, 64(%ecx)
+; LINUX-32-PIC-NEXT:    calll .L23$pb
+; LINUX-32-PIC-NEXT:  .L23$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp23:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp23-.L23$pb), %eax
+; LINUX-32-PIC-NEXT:    movl lsrc@GOTOFF+64(%eax), %ecx
+; LINUX-32-PIC-NEXT:    movl lptr@GOTOFF(%eax), %eax
+; LINUX-32-PIC-NEXT:    movl %ecx, 64(%eax)
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: qux08:
@@ -1921,9 +2083,16 @@ define void @ind00(i64 %i) nounwind {
 ;
 ; LINUX-32-PIC-LABEL: ind00:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; LINUX-32-PIC-NEXT:    movl src(,%eax,4), %ecx
-; LINUX-32-PIC-NEXT:    movl %ecx, dst(,%eax,4)
+; LINUX-32-PIC-NEXT:    calll .L24$pb
+; LINUX-32-PIC-NEXT:  .L24$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp24:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp24-.L24$pb), %eax
+; LINUX-32-PIC-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; LINUX-32-PIC-NEXT:    movl src@GOT(%eax), %edx
+; LINUX-32-PIC-NEXT:    movl (%edx,%ecx,4), %edx
+; LINUX-32-PIC-NEXT:    movl dst@GOT(%eax), %eax
+; LINUX-32-PIC-NEXT:    movl %edx, (%eax,%ecx,4)
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: ind00:
@@ -2010,9 +2179,16 @@ define void @ixd00(i64 %i) nounwind {
 ;
 ; LINUX-32-PIC-LABEL: ixd00:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; LINUX-32-PIC-NEXT:    movl xsrc(,%eax,4), %ecx
-; LINUX-32-PIC-NEXT:    movl %ecx, xdst(,%eax,4)
+; LINUX-32-PIC-NEXT:    calll .L25$pb
+; LINUX-32-PIC-NEXT:  .L25$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp25:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp25-.L25$pb), %eax
+; LINUX-32-PIC-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; LINUX-32-PIC-NEXT:    movl xsrc@GOT(%eax), %edx
+; LINUX-32-PIC-NEXT:    movl (%edx,%ecx,4), %edx
+; LINUX-32-PIC-NEXT:    movl xdst@GOT(%eax), %eax
+; LINUX-32-PIC-NEXT:    movl %edx, (%eax,%ecx,4)
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: ixd00:
@@ -2099,9 +2275,16 @@ define void @ind01(i64 %i) nounwind {
 ;
 ; LINUX-32-PIC-LABEL: ind01:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; LINUX-32-PIC-NEXT:    leal dst(,%eax,4), %eax
-; LINUX-32-PIC-NEXT:    movl %eax, ptr
+; LINUX-32-PIC-NEXT:    calll .L26$pb
+; LINUX-32-PIC-NEXT:  .L26$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp26:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp26-.L26$pb), %eax
+; LINUX-32-PIC-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; LINUX-32-PIC-NEXT:    shll $2, %ecx
+; LINUX-32-PIC-NEXT:    addl dst@GOT(%eax), %ecx
+; LINUX-32-PIC-NEXT:    movl ptr@GOT(%eax), %eax
+; LINUX-32-PIC-NEXT:    movl %ecx, (%eax)
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: ind01:
@@ -2186,9 +2369,16 @@ define void @ixd01(i64 %i) nounwind {
 ;
 ; LINUX-32-PIC-LABEL: ixd01:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; LINUX-32-PIC-NEXT:    leal xdst(,%eax,4), %eax
-; LINUX-32-PIC-NEXT:    movl %eax, ptr
+; LINUX-32-PIC-NEXT:    calll .L27$pb
+; LINUX-32-PIC-NEXT:  .L27$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp27:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp27-.L27$pb), %eax
+; LINUX-32-PIC-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; LINUX-32-PIC-NEXT:    shll $2, %ecx
+; LINUX-32-PIC-NEXT:    addl xdst@GOT(%eax), %ecx
+; LINUX-32-PIC-NEXT:    movl ptr@GOT(%eax), %eax
+; LINUX-32-PIC-NEXT:    movl %ecx, (%eax)
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: ixd01:
@@ -2275,10 +2465,17 @@ define void @ind02(i64 %i) nounwind {
 ;
 ; LINUX-32-PIC-LABEL: ind02:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; LINUX-32-PIC-NEXT:    movl src(,%eax,4), %ecx
-; LINUX-32-PIC-NEXT:    movl ptr, %edx
-; LINUX-32-PIC-NEXT:    movl %ecx, (%edx,%eax,4)
+; LINUX-32-PIC-NEXT:    calll .L28$pb
+; LINUX-32-PIC-NEXT:  .L28$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp28:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp28-.L28$pb), %eax
+; LINUX-32-PIC-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; LINUX-32-PIC-NEXT:    movl src@GOT(%eax), %edx
+; LINUX-32-PIC-NEXT:    movl (%edx,%ecx,4), %edx
+; LINUX-32-PIC-NEXT:    movl ptr@GOT(%eax), %eax
+; LINUX-32-PIC-NEXT:    movl (%eax), %eax
+; LINUX-32-PIC-NEXT:    movl %edx, (%eax,%ecx,4)
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: ind02:
@@ -2375,10 +2572,17 @@ define void @ixd02(i64 %i) nounwind {
 ;
 ; LINUX-32-PIC-LABEL: ixd02:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; LINUX-32-PIC-NEXT:    movl xsrc(,%eax,4), %ecx
-; LINUX-32-PIC-NEXT:    movl ptr, %edx
-; LINUX-32-PIC-NEXT:    movl %ecx, (%edx,%eax,4)
+; LINUX-32-PIC-NEXT:    calll .L29$pb
+; LINUX-32-PIC-NEXT:  .L29$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp29:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp29-.L29$pb), %eax
+; LINUX-32-PIC-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; LINUX-32-PIC-NEXT:    movl xsrc@GOT(%eax), %edx
+; LINUX-32-PIC-NEXT:    movl (%edx,%ecx,4), %edx
+; LINUX-32-PIC-NEXT:    movl ptr@GOT(%eax), %eax
+; LINUX-32-PIC-NEXT:    movl (%eax), %eax
+; LINUX-32-PIC-NEXT:    movl %edx, (%eax,%ecx,4)
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: ixd02:
@@ -2473,9 +2677,16 @@ define void @ind03(i64 %i) nounwind {
 ;
 ; LINUX-32-PIC-LABEL: ind03:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; LINUX-32-PIC-NEXT:    movl dsrc(,%eax,4), %ecx
-; LINUX-32-PIC-NEXT:    movl %ecx, ddst(,%eax,4)
+; LINUX-32-PIC-NEXT:    calll .L30$pb
+; LINUX-32-PIC-NEXT:  .L30$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp30:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp30-.L30$pb), %eax
+; LINUX-32-PIC-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; LINUX-32-PIC-NEXT:    movl dsrc@GOT(%eax), %edx
+; LINUX-32-PIC-NEXT:    movl (%edx,%ecx,4), %edx
+; LINUX-32-PIC-NEXT:    movl ddst@GOT(%eax), %eax
+; LINUX-32-PIC-NEXT:    movl %edx, (%eax,%ecx,4)
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: ind03:
@@ -2558,9 +2769,16 @@ define void @ind04(i64 %i) nounwind {
 ;
 ; LINUX-32-PIC-LABEL: ind04:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; LINUX-32-PIC-NEXT:    leal ddst(,%eax,4), %eax
-; LINUX-32-PIC-NEXT:    movl %eax, dptr
+; LINUX-32-PIC-NEXT:    calll .L31$pb
+; LINUX-32-PIC-NEXT:  .L31$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp31:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp31-.L31$pb), %eax
+; LINUX-32-PIC-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; LINUX-32-PIC-NEXT:    shll $2, %ecx
+; LINUX-32-PIC-NEXT:    addl ddst@GOT(%eax), %ecx
+; LINUX-32-PIC-NEXT:    movl dptr@GOT(%eax), %eax
+; LINUX-32-PIC-NEXT:    movl %ecx, (%eax)
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: ind04:
@@ -2640,10 +2858,17 @@ define void @ind05(i64 %i) nounwind {
 ;
 ; LINUX-32-PIC-LABEL: ind05:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; LINUX-32-PIC-NEXT:    movl dsrc(,%eax,4), %ecx
-; LINUX-32-PIC-NEXT:    movl dptr, %edx
-; LINUX-32-PIC-NEXT:    movl %ecx, (%edx,%eax,4)
+; LINUX-32-PIC-NEXT:    calll .L32$pb
+; LINUX-32-PIC-NEXT:  .L32$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp32:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp32-.L32$pb), %eax
+; LINUX-32-PIC-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; LINUX-32-PIC-NEXT:    movl dsrc@GOT(%eax), %edx
+; LINUX-32-PIC-NEXT:    movl (%edx,%ecx,4), %edx
+; LINUX-32-PIC-NEXT:    movl dptr@GOT(%eax), %eax
+; LINUX-32-PIC-NEXT:    movl (%eax), %eax
+; LINUX-32-PIC-NEXT:    movl %edx, (%eax,%ecx,4)
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: ind05:
@@ -2731,9 +2956,14 @@ define void @ind06(i64 %i) nounwind {
 ;
 ; LINUX-32-PIC-LABEL: ind06:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; LINUX-32-PIC-NEXT:    movl lsrc(,%eax,4), %ecx
-; LINUX-32-PIC-NEXT:    movl %ecx, ldst(,%eax,4)
+; LINUX-32-PIC-NEXT:    calll .L33$pb
+; LINUX-32-PIC-NEXT:  .L33$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp33:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp33-.L33$pb), %eax
+; LINUX-32-PIC-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; LINUX-32-PIC-NEXT:    movl lsrc@GOTOFF(%eax,%ecx,4), %edx
+; LINUX-32-PIC-NEXT:    movl %edx, ldst@GOTOFF(%eax,%ecx,4)
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: ind06:
@@ -2816,9 +3046,14 @@ define void @ind07(i64 %i) nounwind {
 ;
 ; LINUX-32-PIC-LABEL: ind07:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; LINUX-32-PIC-NEXT:    leal ldst(,%eax,4), %eax
-; LINUX-32-PIC-NEXT:    movl %eax, lptr
+; LINUX-32-PIC-NEXT:    calll .L34$pb
+; LINUX-32-PIC-NEXT:  .L34$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp34:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp34-.L34$pb), %eax
+; LINUX-32-PIC-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; LINUX-32-PIC-NEXT:    leal ldst@GOTOFF(%eax,%ecx,4), %ecx
+; LINUX-32-PIC-NEXT:    movl %ecx, lptr@GOTOFF(%eax)
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: ind07:
@@ -2897,10 +3132,15 @@ define void @ind08(i64 %i) nounwind {
 ;
 ; LINUX-32-PIC-LABEL: ind08:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; LINUX-32-PIC-NEXT:    movl lsrc(,%eax,4), %ecx
-; LINUX-32-PIC-NEXT:    movl lptr, %edx
-; LINUX-32-PIC-NEXT:    movl %ecx, (%edx,%eax,4)
+; LINUX-32-PIC-NEXT:    calll .L35$pb
+; LINUX-32-PIC-NEXT:  .L35$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp35:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp35-.L35$pb), %eax
+; LINUX-32-PIC-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; LINUX-32-PIC-NEXT:    movl lsrc@GOTOFF(%eax,%ecx,4), %edx
+; LINUX-32-PIC-NEXT:    movl lptr@GOTOFF(%eax), %eax
+; LINUX-32-PIC-NEXT:    movl %edx, (%eax,%ecx,4)
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: ind08:
@@ -2987,9 +3227,16 @@ define void @off00(i64 %i) nounwind {
 ;
 ; LINUX-32-PIC-LABEL: off00:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; LINUX-32-PIC-NEXT:    movl src+64(,%eax,4), %ecx
-; LINUX-32-PIC-NEXT:    movl %ecx, dst+64(,%eax,4)
+; LINUX-32-PIC-NEXT:    calll .L36$pb
+; LINUX-32-PIC-NEXT:  .L36$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp36:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp36-.L36$pb), %eax
+; LINUX-32-PIC-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; LINUX-32-PIC-NEXT:    movl src@GOT(%eax), %edx
+; LINUX-32-PIC-NEXT:    movl 64(%edx,%ecx,4), %edx
+; LINUX-32-PIC-NEXT:    movl dst@GOT(%eax), %eax
+; LINUX-32-PIC-NEXT:    movl %edx, 64(%eax,%ecx,4)
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: off00:
@@ -3077,9 +3324,16 @@ define void @oxf00(i64 %i) nounwind {
 ;
 ; LINUX-32-PIC-LABEL: oxf00:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; LINUX-32-PIC-NEXT:    movl xsrc+64(,%eax,4), %ecx
-; LINUX-32-PIC-NEXT:    movl %ecx, xdst+64(,%eax,4)
+; LINUX-32-PIC-NEXT:    calll .L37$pb
+; LINUX-32-PIC-NEXT:  .L37$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp37:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp37-.L37$pb), %eax
+; LINUX-32-PIC-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; LINUX-32-PIC-NEXT:    movl xsrc@GOT(%eax), %edx
+; LINUX-32-PIC-NEXT:    movl 64(%edx,%ecx,4), %edx
+; LINUX-32-PIC-NEXT:    movl xdst@GOT(%eax), %eax
+; LINUX-32-PIC-NEXT:    movl %edx, 64(%eax,%ecx,4)
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: oxf00:
@@ -3167,9 +3421,16 @@ define void @off01(i64 %i) nounwind {
 ;
 ; LINUX-32-PIC-LABEL: off01:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; LINUX-32-PIC-NEXT:    leal dst+64(,%eax,4), %eax
-; LINUX-32-PIC-NEXT:    movl %eax, ptr
+; LINUX-32-PIC-NEXT:    calll .L38$pb
+; LINUX-32-PIC-NEXT:  .L38$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp38:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp38-.L38$pb), %eax
+; LINUX-32-PIC-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; LINUX-32-PIC-NEXT:    movl dst@GOT(%eax), %edx
+; LINUX-32-PIC-NEXT:    leal 64(%edx,%ecx,4), %ecx
+; LINUX-32-PIC-NEXT:    movl ptr@GOT(%eax), %eax
+; LINUX-32-PIC-NEXT:    movl %ecx, (%eax)
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: off01:
@@ -3255,9 +3516,16 @@ define void @oxf01(i64 %i) nounwind {
 ;
 ; LINUX-32-PIC-LABEL: oxf01:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; LINUX-32-PIC-NEXT:    leal xdst+64(,%eax,4), %eax
-; LINUX-32-PIC-NEXT:    movl %eax, ptr
+; LINUX-32-PIC-NEXT:    calll .L39$pb
+; LINUX-32-PIC-NEXT:  .L39$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp39:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp39-.L39$pb), %eax
+; LINUX-32-PIC-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; LINUX-32-PIC-NEXT:    movl xdst@GOT(%eax), %edx
+; LINUX-32-PIC-NEXT:    leal 64(%edx,%ecx,4), %ecx
+; LINUX-32-PIC-NEXT:    movl ptr@GOT(%eax), %eax
+; LINUX-32-PIC-NEXT:    movl %ecx, (%eax)
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: oxf01:
@@ -3345,10 +3613,17 @@ define void @off02(i64 %i) nounwind {
 ;
 ; LINUX-32-PIC-LABEL: off02:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; LINUX-32-PIC-NEXT:    movl src+64(,%eax,4), %ecx
-; LINUX-32-PIC-NEXT:    movl ptr, %edx
-; LINUX-32-PIC-NEXT:    movl %ecx, 64(%edx,%eax,4)
+; LINUX-32-PIC-NEXT:    calll .L40$pb
+; LINUX-32-PIC-NEXT:  .L40$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp40:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp40-.L40$pb), %eax
+; LINUX-32-PIC-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; LINUX-32-PIC-NEXT:    movl src@GOT(%eax), %edx
+; LINUX-32-PIC-NEXT:    movl 64(%edx,%ecx,4), %edx
+; LINUX-32-PIC-NEXT:    movl ptr@GOT(%eax), %eax
+; LINUX-32-PIC-NEXT:    movl (%eax), %eax
+; LINUX-32-PIC-NEXT:    movl %edx, 64(%eax,%ecx,4)
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: off02:
@@ -3446,10 +3721,17 @@ define void @oxf02(i64 %i) nounwind {
 ;
 ; LINUX-32-PIC-LABEL: oxf02:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; LINUX-32-PIC-NEXT:    movl xsrc+64(,%eax,4), %ecx
-; LINUX-32-PIC-NEXT:    movl ptr, %edx
-; LINUX-32-PIC-NEXT:    movl %ecx, 64(%edx,%eax,4)
+; LINUX-32-PIC-NEXT:    calll .L41$pb
+; LINUX-32-PIC-NEXT:  .L41$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp41:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp41-.L41$pb), %eax
+; LINUX-32-PIC-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; LINUX-32-PIC-NEXT:    movl xsrc@GOT(%eax), %edx
+; LINUX-32-PIC-NEXT:    movl 64(%edx,%ecx,4), %edx
+; LINUX-32-PIC-NEXT:    movl ptr@GOT(%eax), %eax
+; LINUX-32-PIC-NEXT:    movl (%eax), %eax
+; LINUX-32-PIC-NEXT:    movl %edx, 64(%eax,%ecx,4)
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: oxf02:
@@ -3545,9 +3827,16 @@ define void @off03(i64 %i) nounwind {
 ;
 ; LINUX-32-PIC-LABEL: off03:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; LINUX-32-PIC-NEXT:    movl dsrc+64(,%eax,4), %ecx
-; LINUX-32-PIC-NEXT:    movl %ecx, ddst+64(,%eax,4)
+; LINUX-32-PIC-NEXT:    calll .L42$pb
+; LINUX-32-PIC-NEXT:  .L42$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp42:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp42-.L42$pb), %eax
+; LINUX-32-PIC-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; LINUX-32-PIC-NEXT:    movl dsrc@GOT(%eax), %edx
+; LINUX-32-PIC-NEXT:    movl 64(%edx,%ecx,4), %edx
+; LINUX-32-PIC-NEXT:    movl ddst@GOT(%eax), %eax
+; LINUX-32-PIC-NEXT:    movl %edx, 64(%eax,%ecx,4)
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: off03:
@@ -3631,9 +3920,16 @@ define void @off04(i64 %i) nounwind {
 ;
 ; LINUX-32-PIC-LABEL: off04:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; LINUX-32-PIC-NEXT:    leal ddst+64(,%eax,4), %eax
-; LINUX-32-PIC-NEXT:    movl %eax, dptr
+; LINUX-32-PIC-NEXT:    calll .L43$pb
+; LINUX-32-PIC-NEXT:  .L43$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp43:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp43-.L43$pb), %eax
+; LINUX-32-PIC-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; LINUX-32-PIC-NEXT:    movl ddst@GOT(%eax), %edx
+; LINUX-32-PIC-NEXT:    leal 64(%edx,%ecx,4), %ecx
+; LINUX-32-PIC-NEXT:    movl dptr@GOT(%eax), %eax
+; LINUX-32-PIC-NEXT:    movl %ecx, (%eax)
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: off04:
@@ -3714,10 +4010,17 @@ define void @off05(i64 %i) nounwind {
 ;
 ; LINUX-32-PIC-LABEL: off05:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; LINUX-32-PIC-NEXT:    movl dsrc+64(,%eax,4), %ecx
-; LINUX-32-PIC-NEXT:    movl dptr, %edx
-; LINUX-32-PIC-NEXT:    movl %ecx, 64(%edx,%eax,4)
+; LINUX-32-PIC-NEXT:    calll .L44$pb
+; LINUX-32-PIC-NEXT:  .L44$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp44:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp44-.L44$pb), %eax
+; LINUX-32-PIC-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; LINUX-32-PIC-NEXT:    movl dsrc@GOT(%eax), %edx
+; LINUX-32-PIC-NEXT:    movl 64(%edx,%ecx,4), %edx
+; LINUX-32-PIC-NEXT:    movl dptr@GOT(%eax), %eax
+; LINUX-32-PIC-NEXT:    movl (%eax), %eax
+; LINUX-32-PIC-NEXT:    movl %edx, 64(%eax,%ecx,4)
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: off05:
@@ -3806,9 +4109,14 @@ define void @off06(i64 %i) nounwind {
 ;
 ; LINUX-32-PIC-LABEL: off06:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; LINUX-32-PIC-NEXT:    movl lsrc+64(,%eax,4), %ecx
-; LINUX-32-PIC-NEXT:    movl %ecx, ldst+64(,%eax,4)
+; LINUX-32-PIC-NEXT:    calll .L45$pb
+; LINUX-32-PIC-NEXT:  .L45$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp45:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp45-.L45$pb), %eax
+; LINUX-32-PIC-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; LINUX-32-PIC-NEXT:    movl lsrc@GOTOFF+64(%eax,%ecx,4), %edx
+; LINUX-32-PIC-NEXT:    movl %edx, ldst@GOTOFF+64(%eax,%ecx,4)
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: off06:
@@ -3892,9 +4200,14 @@ define void @off07(i64 %i) nounwind {
 ;
 ; LINUX-32-PIC-LABEL: off07:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; LINUX-32-PIC-NEXT:    leal ldst+64(,%eax,4), %eax
-; LINUX-32-PIC-NEXT:    movl %eax, lptr
+; LINUX-32-PIC-NEXT:    calll .L46$pb
+; LINUX-32-PIC-NEXT:  .L46$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp46:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp46-.L46$pb), %eax
+; LINUX-32-PIC-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; LINUX-32-PIC-NEXT:    leal ldst@GOTOFF+64(%eax,%ecx,4), %ecx
+; LINUX-32-PIC-NEXT:    movl %ecx, lptr@GOTOFF(%eax)
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: off07:
@@ -3974,10 +4287,15 @@ define void @off08(i64 %i) nounwind {
 ;
 ; LINUX-32-PIC-LABEL: off08:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; LINUX-32-PIC-NEXT:    movl lsrc+64(,%eax,4), %ecx
-; LINUX-32-PIC-NEXT:    movl lptr, %edx
-; LINUX-32-PIC-NEXT:    movl %ecx, 64(%edx,%eax,4)
+; LINUX-32-PIC-NEXT:    calll .L47$pb
+; LINUX-32-PIC-NEXT:  .L47$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp47:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp47-.L47$pb), %eax
+; LINUX-32-PIC-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; LINUX-32-PIC-NEXT:    movl lsrc@GOTOFF+64(%eax,%ecx,4), %edx
+; LINUX-32-PIC-NEXT:    movl lptr@GOTOFF(%eax), %eax
+; LINUX-32-PIC-NEXT:    movl %edx, 64(%eax,%ecx,4)
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: off08:
@@ -4064,8 +4382,15 @@ define void @moo00(i64 %i) nounwind {
 ;
 ; LINUX-32-PIC-LABEL: moo00:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl src+262144, %eax
-; LINUX-32-PIC-NEXT:    movl %eax, dst+262144
+; LINUX-32-PIC-NEXT:    calll .L48$pb
+; LINUX-32-PIC-NEXT:  .L48$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp48:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp48-.L48$pb), %eax
+; LINUX-32-PIC-NEXT:    movl src@GOT(%eax), %ecx
+; LINUX-32-PIC-NEXT:    movl 262144(%ecx), %ecx
+; LINUX-32-PIC-NEXT:    movl dst@GOT(%eax), %eax
+; LINUX-32-PIC-NEXT:    movl %ecx, 262144(%eax)
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: moo00:
@@ -4144,7 +4469,15 @@ define void @moo01(i64 %i) nounwind {
 ;
 ; LINUX-32-PIC-LABEL: moo01:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl $dst+262144, ptr
+; LINUX-32-PIC-NEXT:    calll .L49$pb
+; LINUX-32-PIC-NEXT:  .L49$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp49:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp49-.L49$pb), %eax
+; LINUX-32-PIC-NEXT:    movl $262144, %ecx # imm = 0x40000
+; LINUX-32-PIC-NEXT:    addl dst@GOT(%eax), %ecx
+; LINUX-32-PIC-NEXT:    movl ptr@GOT(%eax), %eax
+; LINUX-32-PIC-NEXT:    movl %ecx, (%eax)
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: moo01:
@@ -4225,9 +4558,16 @@ define void @moo02(i64 %i) nounwind {
 ;
 ; LINUX-32-PIC-LABEL: moo02:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl src+262144, %eax
-; LINUX-32-PIC-NEXT:    movl ptr, %ecx
-; LINUX-32-PIC-NEXT:    movl %eax, 262144(%ecx)
+; LINUX-32-PIC-NEXT:    calll .L50$pb
+; LINUX-32-PIC-NEXT:  .L50$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp50:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp50-.L50$pb), %eax
+; LINUX-32-PIC-NEXT:    movl src@GOT(%eax), %ecx
+; LINUX-32-PIC-NEXT:    movl 262144(%ecx), %ecx
+; LINUX-32-PIC-NEXT:    movl ptr@GOT(%eax), %eax
+; LINUX-32-PIC-NEXT:    movl (%eax), %eax
+; LINUX-32-PIC-NEXT:    movl %ecx, 262144(%eax)
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: moo02:
@@ -4317,8 +4657,15 @@ define void @moo03(i64 %i) nounwind {
 ;
 ; LINUX-32-PIC-LABEL: moo03:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl dsrc+262144, %eax
-; LINUX-32-PIC-NEXT:    movl %eax, ddst+262144
+; LINUX-32-PIC-NEXT:    calll .L51$pb
+; LINUX-32-PIC-NEXT:  .L51$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp51:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp51-.L51$pb), %eax
+; LINUX-32-PIC-NEXT:    movl dsrc@GOT(%eax), %ecx
+; LINUX-32-PIC-NEXT:    movl 262144(%ecx), %ecx
+; LINUX-32-PIC-NEXT:    movl ddst@GOT(%eax), %eax
+; LINUX-32-PIC-NEXT:    movl %ecx, 262144(%eax)
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: moo03:
@@ -4387,7 +4734,15 @@ define void @moo04(i64 %i) nounwind {
 ;
 ; LINUX-32-PIC-LABEL: moo04:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl $ddst+262144, dptr
+; LINUX-32-PIC-NEXT:    calll .L52$pb
+; LINUX-32-PIC-NEXT:  .L52$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp52:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp52-.L52$pb), %eax
+; LINUX-32-PIC-NEXT:    movl $262144, %ecx # imm = 0x40000
+; LINUX-32-PIC-NEXT:    addl ddst@GOT(%eax), %ecx
+; LINUX-32-PIC-NEXT:    movl dptr@GOT(%eax), %eax
+; LINUX-32-PIC-NEXT:    movl %ecx, (%eax)
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: moo04:
@@ -4457,9 +4812,16 @@ define void @moo05(i64 %i) nounwind {
 ;
 ; LINUX-32-PIC-LABEL: moo05:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl dsrc+262144, %eax
-; LINUX-32-PIC-NEXT:    movl dptr, %ecx
-; LINUX-32-PIC-NEXT:    movl %eax, 262144(%ecx)
+; LINUX-32-PIC-NEXT:    calll .L53$pb
+; LINUX-32-PIC-NEXT:  .L53$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp53:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp53-.L53$pb), %eax
+; LINUX-32-PIC-NEXT:    movl dsrc@GOT(%eax), %ecx
+; LINUX-32-PIC-NEXT:    movl 262144(%ecx), %ecx
+; LINUX-32-PIC-NEXT:    movl dptr@GOT(%eax), %eax
+; LINUX-32-PIC-NEXT:    movl (%eax), %eax
+; LINUX-32-PIC-NEXT:    movl %ecx, 262144(%eax)
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: moo05:
@@ -4539,8 +4901,13 @@ define void @moo06(i64 %i) nounwind {
 ;
 ; LINUX-32-PIC-LABEL: moo06:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl lsrc+262144, %eax
-; LINUX-32-PIC-NEXT:    movl %eax, ldst+262144
+; LINUX-32-PIC-NEXT:    calll .L54$pb
+; LINUX-32-PIC-NEXT:  .L54$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp54:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp54-.L54$pb), %eax
+; LINUX-32-PIC-NEXT:    movl lsrc@GOTOFF+262144(%eax), %ecx
+; LINUX-32-PIC-NEXT:    movl %ecx, ldst@GOTOFF+262144(%eax)
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: moo06:
@@ -4607,7 +4974,13 @@ define void @moo07(i64 %i) nounwind {
 ;
 ; LINUX-32-PIC-LABEL: moo07:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl $ldst+262144, lptr
+; LINUX-32-PIC-NEXT:    calll .L55$pb
+; LINUX-32-PIC-NEXT:  .L55$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp55:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp55-.L55$pb), %eax
+; LINUX-32-PIC-NEXT:    leal ldst@GOTOFF+262144(%eax), %ecx
+; LINUX-32-PIC-NEXT:    movl %ecx, lptr@GOTOFF(%eax)
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: moo07:
@@ -4675,9 +5048,14 @@ define void @moo08(i64 %i) nounwind {
 ;
 ; LINUX-32-PIC-LABEL: moo08:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl lsrc+262144, %eax
-; LINUX-32-PIC-NEXT:    movl lptr, %ecx
-; LINUX-32-PIC-NEXT:    movl %eax, 262144(%ecx)
+; LINUX-32-PIC-NEXT:    calll .L56$pb
+; LINUX-32-PIC-NEXT:  .L56$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp56:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp56-.L56$pb), %eax
+; LINUX-32-PIC-NEXT:    movl lsrc@GOTOFF+262144(%eax), %ecx
+; LINUX-32-PIC-NEXT:    movl lptr@GOTOFF(%eax), %eax
+; LINUX-32-PIC-NEXT:    movl %ecx, 262144(%eax)
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: moo08:
@@ -4756,9 +5134,16 @@ define void @big00(i64 %i) nounwind {
 ;
 ; LINUX-32-PIC-LABEL: big00:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; LINUX-32-PIC-NEXT:    movl src+262144(,%eax,4), %ecx
-; LINUX-32-PIC-NEXT:    movl %ecx, dst+262144(,%eax,4)
+; LINUX-32-PIC-NEXT:    calll .L57$pb
+; LINUX-32-PIC-NEXT:  .L57$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp57:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp57-.L57$pb), %eax
+; LINUX-32-PIC-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; LINUX-32-PIC-NEXT:    movl src@GOT(%eax), %edx
+; LINUX-32-PIC-NEXT:    movl 262144(%edx,%ecx,4), %edx
+; LINUX-32-PIC-NEXT:    movl dst@GOT(%eax), %eax
+; LINUX-32-PIC-NEXT:    movl %edx, 262144(%eax,%ecx,4)
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: big00:
@@ -4846,9 +5231,16 @@ define void @big01(i64 %i) nounwind {
 ;
 ; LINUX-32-PIC-LABEL: big01:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; LINUX-32-PIC-NEXT:    leal dst+262144(,%eax,4), %eax
-; LINUX-32-PIC-NEXT:    movl %eax, ptr
+; LINUX-32-PIC-NEXT:    calll .L58$pb
+; LINUX-32-PIC-NEXT:  .L58$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp58:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp58-.L58$pb), %eax
+; LINUX-32-PIC-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; LINUX-32-PIC-NEXT:    movl dst@GOT(%eax), %edx
+; LINUX-32-PIC-NEXT:    leal 262144(%edx,%ecx,4), %ecx
+; LINUX-32-PIC-NEXT:    movl ptr@GOT(%eax), %eax
+; LINUX-32-PIC-NEXT:    movl %ecx, (%eax)
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: big01:
@@ -4936,10 +5328,17 @@ define void @big02(i64 %i) nounwind {
 ;
 ; LINUX-32-PIC-LABEL: big02:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; LINUX-32-PIC-NEXT:    movl src+262144(,%eax,4), %ecx
-; LINUX-32-PIC-NEXT:    movl ptr, %edx
-; LINUX-32-PIC-NEXT:    movl %ecx, 262144(%edx,%eax,4)
+; LINUX-32-PIC-NEXT:    calll .L59$pb
+; LINUX-32-PIC-NEXT:  .L59$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp59:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp59-.L59$pb), %eax
+; LINUX-32-PIC-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; LINUX-32-PIC-NEXT:    movl src@GOT(%eax), %edx
+; LINUX-32-PIC-NEXT:    movl 262144(%edx,%ecx,4), %edx
+; LINUX-32-PIC-NEXT:    movl ptr@GOT(%eax), %eax
+; LINUX-32-PIC-NEXT:    movl (%eax), %eax
+; LINUX-32-PIC-NEXT:    movl %edx, 262144(%eax,%ecx,4)
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: big02:
@@ -5035,9 +5434,16 @@ define void @big03(i64 %i) nounwind {
 ;
 ; LINUX-32-PIC-LABEL: big03:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; LINUX-32-PIC-NEXT:    movl dsrc+262144(,%eax,4), %ecx
-; LINUX-32-PIC-NEXT:    movl %ecx, ddst+262144(,%eax,4)
+; LINUX-32-PIC-NEXT:    calll .L60$pb
+; LINUX-32-PIC-NEXT:  .L60$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp60:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp60-.L60$pb), %eax
+; LINUX-32-PIC-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; LINUX-32-PIC-NEXT:    movl dsrc@GOT(%eax), %edx
+; LINUX-32-PIC-NEXT:    movl 262144(%edx,%ecx,4), %edx
+; LINUX-32-PIC-NEXT:    movl ddst@GOT(%eax), %eax
+; LINUX-32-PIC-NEXT:    movl %edx, 262144(%eax,%ecx,4)
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: big03:
@@ -5121,9 +5527,16 @@ define void @big04(i64 %i) nounwind {
 ;
 ; LINUX-32-PIC-LABEL: big04:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; LINUX-32-PIC-NEXT:    leal ddst+262144(,%eax,4), %eax
-; LINUX-32-PIC-NEXT:    movl %eax, dptr
+; LINUX-32-PIC-NEXT:    calll .L61$pb
+; LINUX-32-PIC-NEXT:  .L61$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp61:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp61-.L61$pb), %eax
+; LINUX-32-PIC-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; LINUX-32-PIC-NEXT:    movl ddst@GOT(%eax), %edx
+; LINUX-32-PIC-NEXT:    leal 262144(%edx,%ecx,4), %ecx
+; LINUX-32-PIC-NEXT:    movl dptr@GOT(%eax), %eax
+; LINUX-32-PIC-NEXT:    movl %ecx, (%eax)
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: big04:
@@ -5204,10 +5617,17 @@ define void @big05(i64 %i) nounwind {
 ;
 ; LINUX-32-PIC-LABEL: big05:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; LINUX-32-PIC-NEXT:    movl dsrc+262144(,%eax,4), %ecx
-; LINUX-32-PIC-NEXT:    movl dptr, %edx
-; LINUX-32-PIC-NEXT:    movl %ecx, 262144(%edx,%eax,4)
+; LINUX-32-PIC-NEXT:    calll .L62$pb
+; LINUX-32-PIC-NEXT:  .L62$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp62:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp62-.L62$pb), %eax
+; LINUX-32-PIC-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; LINUX-32-PIC-NEXT:    movl dsrc@GOT(%eax), %edx
+; LINUX-32-PIC-NEXT:    movl 262144(%edx,%ecx,4), %edx
+; LINUX-32-PIC-NEXT:    movl dptr@GOT(%eax), %eax
+; LINUX-32-PIC-NEXT:    movl (%eax), %eax
+; LINUX-32-PIC-NEXT:    movl %edx, 262144(%eax,%ecx,4)
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: big05:
@@ -5296,9 +5716,14 @@ define void @big06(i64 %i) nounwind {
 ;
 ; LINUX-32-PIC-LABEL: big06:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; LINUX-32-PIC-NEXT:    movl lsrc+262144(,%eax,4), %ecx
-; LINUX-32-PIC-NEXT:    movl %ecx, ldst+262144(,%eax,4)
+; LINUX-32-PIC-NEXT:    calll .L63$pb
+; LINUX-32-PIC-NEXT:  .L63$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp63:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp63-.L63$pb), %eax
+; LINUX-32-PIC-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; LINUX-32-PIC-NEXT:    movl lsrc@GOTOFF+262144(%eax,%ecx,4), %edx
+; LINUX-32-PIC-NEXT:    movl %edx, ldst@GOTOFF+262144(%eax,%ecx,4)
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: big06:
@@ -5382,9 +5807,14 @@ define void @big07(i64 %i) nounwind {
 ;
 ; LINUX-32-PIC-LABEL: big07:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; LINUX-32-PIC-NEXT:    leal ldst+262144(,%eax,4), %eax
-; LINUX-32-PIC-NEXT:    movl %eax, lptr
+; LINUX-32-PIC-NEXT:    calll .L64$pb
+; LINUX-32-PIC-NEXT:  .L64$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp64:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp64-.L64$pb), %eax
+; LINUX-32-PIC-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; LINUX-32-PIC-NEXT:    leal ldst@GOTOFF+262144(%eax,%ecx,4), %ecx
+; LINUX-32-PIC-NEXT:    movl %ecx, lptr@GOTOFF(%eax)
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: big07:
@@ -5464,10 +5894,15 @@ define void @big08(i64 %i) nounwind {
 ;
 ; LINUX-32-PIC-LABEL: big08:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; LINUX-32-PIC-NEXT:    movl lsrc+262144(,%eax,4), %ecx
-; LINUX-32-PIC-NEXT:    movl lptr, %edx
-; LINUX-32-PIC-NEXT:    movl %ecx, 262144(%edx,%eax,4)
+; LINUX-32-PIC-NEXT:    calll .L65$pb
+; LINUX-32-PIC-NEXT:  .L65$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp65:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp65-.L65$pb), %eax
+; LINUX-32-PIC-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; LINUX-32-PIC-NEXT:    movl lsrc@GOTOFF+262144(%eax,%ecx,4), %edx
+; LINUX-32-PIC-NEXT:    movl lptr@GOTOFF(%eax), %eax
+; LINUX-32-PIC-NEXT:    movl %edx, 262144(%eax,%ecx,4)
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: big08:
@@ -5552,7 +5987,12 @@ define i8* @bar00() nounwind {
 ;
 ; LINUX-32-PIC-LABEL: bar00:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl $src, %eax
+; LINUX-32-PIC-NEXT:    calll .L66$pb
+; LINUX-32-PIC-NEXT:  .L66$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp66:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp66-.L66$pb), %eax
+; LINUX-32-PIC-NEXT:    movl src@GOT(%eax), %eax
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: bar00:
@@ -5610,7 +6050,12 @@ define i8* @bxr00() nounwind {
 ;
 ; LINUX-32-PIC-LABEL: bxr00:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl $xsrc, %eax
+; LINUX-32-PIC-NEXT:    calll .L67$pb
+; LINUX-32-PIC-NEXT:  .L67$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp67:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp67-.L67$pb), %eax
+; LINUX-32-PIC-NEXT:    movl xsrc@GOT(%eax), %eax
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: bxr00:
@@ -5668,7 +6113,12 @@ define i8* @bar01() nounwind {
 ;
 ; LINUX-32-PIC-LABEL: bar01:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl $dst, %eax
+; LINUX-32-PIC-NEXT:    calll .L68$pb
+; LINUX-32-PIC-NEXT:  .L68$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp68:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp68-.L68$pb), %eax
+; LINUX-32-PIC-NEXT:    movl dst@GOT(%eax), %eax
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: bar01:
@@ -5726,7 +6176,12 @@ define i8* @bxr01() nounwind {
 ;
 ; LINUX-32-PIC-LABEL: bxr01:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl $xdst, %eax
+; LINUX-32-PIC-NEXT:    calll .L69$pb
+; LINUX-32-PIC-NEXT:  .L69$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp69:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp69-.L69$pb), %eax
+; LINUX-32-PIC-NEXT:    movl xdst@GOT(%eax), %eax
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: bxr01:
@@ -5784,7 +6239,12 @@ define i8* @bar02() nounwind {
 ;
 ; LINUX-32-PIC-LABEL: bar02:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl $ptr, %eax
+; LINUX-32-PIC-NEXT:    calll .L70$pb
+; LINUX-32-PIC-NEXT:  .L70$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp70:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp70-.L70$pb), %eax
+; LINUX-32-PIC-NEXT:    movl ptr@GOT(%eax), %eax
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: bar02:
@@ -5842,7 +6302,12 @@ define i8* @bar03() nounwind {
 ;
 ; LINUX-32-PIC-LABEL: bar03:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl $dsrc, %eax
+; LINUX-32-PIC-NEXT:    calll .L71$pb
+; LINUX-32-PIC-NEXT:  .L71$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp71:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp71-.L71$pb), %eax
+; LINUX-32-PIC-NEXT:    movl dsrc@GOT(%eax), %eax
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: bar03:
@@ -5900,7 +6365,12 @@ define i8* @bar04() nounwind {
 ;
 ; LINUX-32-PIC-LABEL: bar04:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl $ddst, %eax
+; LINUX-32-PIC-NEXT:    calll .L72$pb
+; LINUX-32-PIC-NEXT:  .L72$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp72:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp72-.L72$pb), %eax
+; LINUX-32-PIC-NEXT:    movl ddst@GOT(%eax), %eax
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: bar04:
@@ -5958,7 +6428,12 @@ define i8* @bar05() nounwind {
 ;
 ; LINUX-32-PIC-LABEL: bar05:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl $dptr, %eax
+; LINUX-32-PIC-NEXT:    calll .L73$pb
+; LINUX-32-PIC-NEXT:  .L73$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp73:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp73-.L73$pb), %eax
+; LINUX-32-PIC-NEXT:    movl dptr@GOT(%eax), %eax
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: bar05:
@@ -6016,7 +6491,12 @@ define i8* @bar06() nounwind {
 ;
 ; LINUX-32-PIC-LABEL: bar06:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl $lsrc, %eax
+; LINUX-32-PIC-NEXT:    calll .L74$pb
+; LINUX-32-PIC-NEXT:  .L74$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp74:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp74-.L74$pb), %eax
+; LINUX-32-PIC-NEXT:    leal lsrc@GOTOFF(%eax), %eax
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: bar06:
@@ -6074,7 +6554,12 @@ define i8* @bar07() nounwind {
 ;
 ; LINUX-32-PIC-LABEL: bar07:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl $ldst, %eax
+; LINUX-32-PIC-NEXT:    calll .L75$pb
+; LINUX-32-PIC-NEXT:  .L75$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp75:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp75-.L75$pb), %eax
+; LINUX-32-PIC-NEXT:    leal ldst@GOTOFF(%eax), %eax
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: bar07:
@@ -6132,7 +6617,12 @@ define i8* @bar08() nounwind {
 ;
 ; LINUX-32-PIC-LABEL: bar08:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl $lptr, %eax
+; LINUX-32-PIC-NEXT:    calll .L76$pb
+; LINUX-32-PIC-NEXT:  .L76$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp76:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp76-.L76$pb), %eax
+; LINUX-32-PIC-NEXT:    leal lptr@GOTOFF(%eax), %eax
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: bar08:
@@ -6190,7 +6680,12 @@ define i8* @har00() nounwind {
 ;
 ; LINUX-32-PIC-LABEL: har00:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl $src, %eax
+; LINUX-32-PIC-NEXT:    calll .L77$pb
+; LINUX-32-PIC-NEXT:  .L77$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp77:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp77-.L77$pb), %eax
+; LINUX-32-PIC-NEXT:    movl src@GOT(%eax), %eax
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: har00:
@@ -6248,7 +6743,12 @@ define i8* @hxr00() nounwind {
 ;
 ; LINUX-32-PIC-LABEL: hxr00:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl $xsrc, %eax
+; LINUX-32-PIC-NEXT:    calll .L78$pb
+; LINUX-32-PIC-NEXT:  .L78$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp78:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp78-.L78$pb), %eax
+; LINUX-32-PIC-NEXT:    movl xsrc@GOT(%eax), %eax
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: hxr00:
@@ -6306,7 +6806,12 @@ define i8* @har01() nounwind {
 ;
 ; LINUX-32-PIC-LABEL: har01:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl $dst, %eax
+; LINUX-32-PIC-NEXT:    calll .L79$pb
+; LINUX-32-PIC-NEXT:  .L79$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp79:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp79-.L79$pb), %eax
+; LINUX-32-PIC-NEXT:    movl dst@GOT(%eax), %eax
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: har01:
@@ -6364,7 +6869,12 @@ define i8* @hxr01() nounwind {
 ;
 ; LINUX-32-PIC-LABEL: hxr01:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl $xdst, %eax
+; LINUX-32-PIC-NEXT:    calll .L80$pb
+; LINUX-32-PIC-NEXT:  .L80$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp80:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp80-.L80$pb), %eax
+; LINUX-32-PIC-NEXT:    movl xdst@GOT(%eax), %eax
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: hxr01:
@@ -6422,7 +6932,13 @@ define i8* @har02() nounwind {
 ;
 ; LINUX-32-PIC-LABEL: har02:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl ptr, %eax
+; LINUX-32-PIC-NEXT:    calll .L81$pb
+; LINUX-32-PIC-NEXT:  .L81$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp81:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp81-.L81$pb), %eax
+; LINUX-32-PIC-NEXT:    movl ptr@GOT(%eax), %eax
+; LINUX-32-PIC-NEXT:    movl (%eax), %eax
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: har02:
@@ -6488,7 +7004,12 @@ define i8* @har03() nounwind {
 ;
 ; LINUX-32-PIC-LABEL: har03:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl $dsrc, %eax
+; LINUX-32-PIC-NEXT:    calll .L82$pb
+; LINUX-32-PIC-NEXT:  .L82$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp82:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp82-.L82$pb), %eax
+; LINUX-32-PIC-NEXT:    movl dsrc@GOT(%eax), %eax
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: har03:
@@ -6546,7 +7067,12 @@ define i8* @har04() nounwind {
 ;
 ; LINUX-32-PIC-LABEL: har04:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl $ddst, %eax
+; LINUX-32-PIC-NEXT:    calll .L83$pb
+; LINUX-32-PIC-NEXT:  .L83$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp83:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp83-.L83$pb), %eax
+; LINUX-32-PIC-NEXT:    movl ddst@GOT(%eax), %eax
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: har04:
@@ -6604,7 +7130,13 @@ define i8* @har05() nounwind {
 ;
 ; LINUX-32-PIC-LABEL: har05:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl dptr, %eax
+; LINUX-32-PIC-NEXT:    calll .L84$pb
+; LINUX-32-PIC-NEXT:  .L84$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp84:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp84-.L84$pb), %eax
+; LINUX-32-PIC-NEXT:    movl dptr@GOT(%eax), %eax
+; LINUX-32-PIC-NEXT:    movl (%eax), %eax
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: har05:
@@ -6665,7 +7197,12 @@ define i8* @har06() nounwind {
 ;
 ; LINUX-32-PIC-LABEL: har06:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl $lsrc, %eax
+; LINUX-32-PIC-NEXT:    calll .L85$pb
+; LINUX-32-PIC-NEXT:  .L85$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp85:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp85-.L85$pb), %eax
+; LINUX-32-PIC-NEXT:    leal lsrc@GOTOFF(%eax), %eax
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: har06:
@@ -6723,7 +7260,12 @@ define i8* @har07() nounwind {
 ;
 ; LINUX-32-PIC-LABEL: har07:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl $ldst, %eax
+; LINUX-32-PIC-NEXT:    calll .L86$pb
+; LINUX-32-PIC-NEXT:  .L86$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp86:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp86-.L86$pb), %eax
+; LINUX-32-PIC-NEXT:    leal ldst@GOTOFF(%eax), %eax
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: har07:
@@ -6781,7 +7323,12 @@ define i8* @har08() nounwind {
 ;
 ; LINUX-32-PIC-LABEL: har08:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl lptr, %eax
+; LINUX-32-PIC-NEXT:    calll .L87$pb
+; LINUX-32-PIC-NEXT:  .L87$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp87:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp87-.L87$pb), %eax
+; LINUX-32-PIC-NEXT:    movl lptr@GOTOFF(%eax), %eax
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: har08:
@@ -6841,7 +7388,13 @@ define i8* @bat00() nounwind {
 ;
 ; LINUX-32-PIC-LABEL: bat00:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl $src+64, %eax
+; LINUX-32-PIC-NEXT:    calll .L88$pb
+; LINUX-32-PIC-NEXT:  .L88$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp88:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp88-.L88$pb), %eax
+; LINUX-32-PIC-NEXT:    movl src@GOT(%eax), %eax
+; LINUX-32-PIC-NEXT:    addl $64, %eax
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: bat00:
@@ -6905,7 +7458,13 @@ define i8* @bxt00() nounwind {
 ;
 ; LINUX-32-PIC-LABEL: bxt00:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl $xsrc+64, %eax
+; LINUX-32-PIC-NEXT:    calll .L89$pb
+; LINUX-32-PIC-NEXT:  .L89$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp89:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp89-.L89$pb), %eax
+; LINUX-32-PIC-NEXT:    movl xsrc@GOT(%eax), %eax
+; LINUX-32-PIC-NEXT:    addl $64, %eax
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: bxt00:
@@ -6969,7 +7528,13 @@ define i8* @bat01() nounwind {
 ;
 ; LINUX-32-PIC-LABEL: bat01:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl $dst+64, %eax
+; LINUX-32-PIC-NEXT:    calll .L90$pb
+; LINUX-32-PIC-NEXT:  .L90$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp90:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp90-.L90$pb), %eax
+; LINUX-32-PIC-NEXT:    movl dst@GOT(%eax), %eax
+; LINUX-32-PIC-NEXT:    addl $64, %eax
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: bat01:
@@ -7033,7 +7598,13 @@ define i8* @bxt01() nounwind {
 ;
 ; LINUX-32-PIC-LABEL: bxt01:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl $xdst+64, %eax
+; LINUX-32-PIC-NEXT:    calll .L91$pb
+; LINUX-32-PIC-NEXT:  .L91$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp91:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp91-.L91$pb), %eax
+; LINUX-32-PIC-NEXT:    movl xdst@GOT(%eax), %eax
+; LINUX-32-PIC-NEXT:    addl $64, %eax
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: bxt01:
@@ -7099,7 +7670,13 @@ define i8* @bat02() nounwind {
 ;
 ; LINUX-32-PIC-LABEL: bat02:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl ptr, %eax
+; LINUX-32-PIC-NEXT:    calll .L92$pb
+; LINUX-32-PIC-NEXT:  .L92$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp92:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp92-.L92$pb), %eax
+; LINUX-32-PIC-NEXT:    movl ptr@GOT(%eax), %eax
+; LINUX-32-PIC-NEXT:    movl (%eax), %eax
 ; LINUX-32-PIC-NEXT:    addl $64, %eax
 ; LINUX-32-PIC-NEXT:    retl
 ;
@@ -7174,7 +7751,13 @@ define i8* @bat03() nounwind {
 ;
 ; LINUX-32-PIC-LABEL: bat03:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl $dsrc+64, %eax
+; LINUX-32-PIC-NEXT:    calll .L93$pb
+; LINUX-32-PIC-NEXT:  .L93$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp93:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp93-.L93$pb), %eax
+; LINUX-32-PIC-NEXT:    movl dsrc@GOT(%eax), %eax
+; LINUX-32-PIC-NEXT:    addl $64, %eax
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: bat03:
@@ -7233,7 +7816,13 @@ define i8* @bat04() nounwind {
 ;
 ; LINUX-32-PIC-LABEL: bat04:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl $ddst+64, %eax
+; LINUX-32-PIC-NEXT:    calll .L94$pb
+; LINUX-32-PIC-NEXT:  .L94$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp94:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp94-.L94$pb), %eax
+; LINUX-32-PIC-NEXT:    movl ddst@GOT(%eax), %eax
+; LINUX-32-PIC-NEXT:    addl $64, %eax
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: bat04:
@@ -7294,7 +7883,13 @@ define i8* @bat05() nounwind {
 ;
 ; LINUX-32-PIC-LABEL: bat05:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl dptr, %eax
+; LINUX-32-PIC-NEXT:    calll .L95$pb
+; LINUX-32-PIC-NEXT:  .L95$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp95:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp95-.L95$pb), %eax
+; LINUX-32-PIC-NEXT:    movl dptr@GOT(%eax), %eax
+; LINUX-32-PIC-NEXT:    movl (%eax), %eax
 ; LINUX-32-PIC-NEXT:    addl $64, %eax
 ; LINUX-32-PIC-NEXT:    retl
 ;
@@ -7364,7 +7959,12 @@ define i8* @bat06() nounwind {
 ;
 ; LINUX-32-PIC-LABEL: bat06:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl $lsrc+64, %eax
+; LINUX-32-PIC-NEXT:    calll .L96$pb
+; LINUX-32-PIC-NEXT:  .L96$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp96:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp96-.L96$pb), %eax
+; LINUX-32-PIC-NEXT:    leal lsrc@GOTOFF+64(%eax), %eax
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: bat06:
@@ -7422,7 +8022,12 @@ define i8* @bat07() nounwind {
 ;
 ; LINUX-32-PIC-LABEL: bat07:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl $ldst+64, %eax
+; LINUX-32-PIC-NEXT:    calll .L97$pb
+; LINUX-32-PIC-NEXT:  .L97$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp97:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp97-.L97$pb), %eax
+; LINUX-32-PIC-NEXT:    leal ldst@GOTOFF+64(%eax), %eax
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: bat07:
@@ -7482,7 +8087,12 @@ define i8* @bat08() nounwind {
 ;
 ; LINUX-32-PIC-LABEL: bat08:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl lptr, %eax
+; LINUX-32-PIC-NEXT:    calll .L98$pb
+; LINUX-32-PIC-NEXT:  .L98$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp98:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp98-.L98$pb), %eax
+; LINUX-32-PIC-NEXT:    movl lptr@GOTOFF(%eax), %eax
 ; LINUX-32-PIC-NEXT:    addl $64, %eax
 ; LINUX-32-PIC-NEXT:    retl
 ;
@@ -7551,7 +8161,13 @@ define i8* @bam00() nounwind {
 ;
 ; LINUX-32-PIC-LABEL: bam00:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl $src+262144, %eax
+; LINUX-32-PIC-NEXT:    calll .L99$pb
+; LINUX-32-PIC-NEXT:  .L99$pb:
+; LINUX-32-PIC-NEXT:    popl %ecx
+; LINUX-32-PIC-NEXT:  .Ltmp99:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp99-.L99$pb), %ecx
+; LINUX-32-PIC-NEXT:    movl $262144, %eax # imm = 0x40000
+; LINUX-32-PIC-NEXT:    addl src@GOT(%ecx), %eax
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: bam00:
@@ -7615,7 +8231,13 @@ define i8* @bam01() nounwind {
 ;
 ; LINUX-32-PIC-LABEL: bam01:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl $dst+262144, %eax
+; LINUX-32-PIC-NEXT:    calll .L100$pb
+; LINUX-32-PIC-NEXT:  .L100$pb:
+; LINUX-32-PIC-NEXT:    popl %ecx
+; LINUX-32-PIC-NEXT:  .Ltmp100:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp100-.L100$pb), %ecx
+; LINUX-32-PIC-NEXT:    movl $262144, %eax # imm = 0x40000
+; LINUX-32-PIC-NEXT:    addl dst@GOT(%ecx), %eax
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: bam01:
@@ -7679,7 +8301,13 @@ define i8* @bxm01() nounwind {
 ;
 ; LINUX-32-PIC-LABEL: bxm01:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl $xdst+262144, %eax
+; LINUX-32-PIC-NEXT:    calll .L101$pb
+; LINUX-32-PIC-NEXT:  .L101$pb:
+; LINUX-32-PIC-NEXT:    popl %ecx
+; LINUX-32-PIC-NEXT:  .Ltmp101:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp101-.L101$pb), %ecx
+; LINUX-32-PIC-NEXT:    movl $262144, %eax # imm = 0x40000
+; LINUX-32-PIC-NEXT:    addl xdst@GOT(%ecx), %eax
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: bxm01:
@@ -7745,13 +8373,19 @@ define i8* @bam02() nounwind {
 ;
 ; LINUX-32-PIC-LABEL: bam02:
 ; LINUX-32-PIC:       # %bb.0: # %entry
+; LINUX-32-PIC-NEXT:    calll .L102$pb
+; LINUX-32-PIC-NEXT:  .L102$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp102:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp102-.L102$pb), %eax
+; LINUX-32-PIC-NEXT:    movl ptr@GOT(%eax), %ecx
 ; LINUX-32-PIC-NEXT:    movl $262144, %eax # imm = 0x40000
-; LINUX-32-PIC-NEXT:    addl ptr, %eax
+; LINUX-32-PIC-NEXT:    addl (%ecx), %eax
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: bam02:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    movq ptr@{{.*}}(%rip), %rcx
+; LINUX-64-PIC-NEXT:    movq ptr@GOTPCREL(%rip), %rcx
 ; LINUX-64-PIC-NEXT:    movl $262144, %eax # imm = 0x40000
 ; LINUX-64-PIC-NEXT:    addq (%rcx), %rax
 ; LINUX-64-PIC-NEXT:    retq
@@ -7781,21 +8415,21 @@ define i8* @bam02() nounwind {
 ;
 ; DARWIN-64-STATIC-LABEL: bam02:
 ; DARWIN-64-STATIC:       ## %bb.0: ## %entry
-; DARWIN-64-STATIC-NEXT:    movq _ptr@{{.*}}(%rip), %rcx
+; DARWIN-64-STATIC-NEXT:    movq _ptr@GOTPCREL(%rip), %rcx
 ; DARWIN-64-STATIC-NEXT:    movl $262144, %eax ## imm = 0x40000
 ; DARWIN-64-STATIC-NEXT:    addq (%rcx), %rax
 ; DARWIN-64-STATIC-NEXT:    retq
 ;
 ; DARWIN-64-DYNAMIC-LABEL: bam02:
 ; DARWIN-64-DYNAMIC:       ## %bb.0: ## %entry
-; DARWIN-64-DYNAMIC-NEXT:    movq _ptr@{{.*}}(%rip), %rcx
+; DARWIN-64-DYNAMIC-NEXT:    movq _ptr@GOTPCREL(%rip), %rcx
 ; DARWIN-64-DYNAMIC-NEXT:    movl $262144, %eax ## imm = 0x40000
 ; DARWIN-64-DYNAMIC-NEXT:    addq (%rcx), %rax
 ; DARWIN-64-DYNAMIC-NEXT:    retq
 ;
 ; DARWIN-64-PIC-LABEL: bam02:
 ; DARWIN-64-PIC:       ## %bb.0: ## %entry
-; DARWIN-64-PIC-NEXT:    movq _ptr@{{.*}}(%rip), %rcx
+; DARWIN-64-PIC-NEXT:    movq _ptr@GOTPCREL(%rip), %rcx
 ; DARWIN-64-PIC-NEXT:    movl $262144, %eax ## imm = 0x40000
 ; DARWIN-64-PIC-NEXT:    addq (%rcx), %rax
 ; DARWIN-64-PIC-NEXT:    retq
@@ -7820,7 +8454,13 @@ define i8* @bam03() nounwind {
 ;
 ; LINUX-32-PIC-LABEL: bam03:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl $dsrc+262144, %eax
+; LINUX-32-PIC-NEXT:    calll .L103$pb
+; LINUX-32-PIC-NEXT:  .L103$pb:
+; LINUX-32-PIC-NEXT:    popl %ecx
+; LINUX-32-PIC-NEXT:  .Ltmp103:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp103-.L103$pb), %ecx
+; LINUX-32-PIC-NEXT:    movl $262144, %eax # imm = 0x40000
+; LINUX-32-PIC-NEXT:    addl dsrc@GOT(%ecx), %eax
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: bam03:
@@ -7879,7 +8519,13 @@ define i8* @bam04() nounwind {
 ;
 ; LINUX-32-PIC-LABEL: bam04:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl $ddst+262144, %eax
+; LINUX-32-PIC-NEXT:    calll .L104$pb
+; LINUX-32-PIC-NEXT:  .L104$pb:
+; LINUX-32-PIC-NEXT:    popl %ecx
+; LINUX-32-PIC-NEXT:  .Ltmp104:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp104-.L104$pb), %ecx
+; LINUX-32-PIC-NEXT:    movl $262144, %eax # imm = 0x40000
+; LINUX-32-PIC-NEXT:    addl ddst@GOT(%ecx), %eax
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: bam04:
@@ -7940,13 +8586,19 @@ define i8* @bam05() nounwind {
 ;
 ; LINUX-32-PIC-LABEL: bam05:
 ; LINUX-32-PIC:       # %bb.0: # %entry
+; LINUX-32-PIC-NEXT:    calll .L105$pb
+; LINUX-32-PIC-NEXT:  .L105$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp105:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp105-.L105$pb), %eax
+; LINUX-32-PIC-NEXT:    movl dptr@GOT(%eax), %ecx
 ; LINUX-32-PIC-NEXT:    movl $262144, %eax # imm = 0x40000
-; LINUX-32-PIC-NEXT:    addl dptr, %eax
+; LINUX-32-PIC-NEXT:    addl (%ecx), %eax
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: bam05:
 ; LINUX-64-PIC:       # %bb.0: # %entry
-; LINUX-64-PIC-NEXT:    movq dptr@{{.*}}(%rip), %rcx
+; LINUX-64-PIC-NEXT:    movq dptr@GOTPCREL(%rip), %rcx
 ; LINUX-64-PIC-NEXT:    movl $262144, %eax # imm = 0x40000
 ; LINUX-64-PIC-NEXT:    addq (%rcx), %rax
 ; LINUX-64-PIC-NEXT:    retq
@@ -8010,7 +8662,12 @@ define i8* @bam06() nounwind {
 ;
 ; LINUX-32-PIC-LABEL: bam06:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl $lsrc+262144, %eax
+; LINUX-32-PIC-NEXT:    calll .L106$pb
+; LINUX-32-PIC-NEXT:  .L106$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp106:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp106-.L106$pb), %eax
+; LINUX-32-PIC-NEXT:    leal lsrc@GOTOFF+262144(%eax), %eax
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: bam06:
@@ -8068,7 +8725,12 @@ define i8* @bam07() nounwind {
 ;
 ; LINUX-32-PIC-LABEL: bam07:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl $ldst+262144, %eax
+; LINUX-32-PIC-NEXT:    calll .L107$pb
+; LINUX-32-PIC-NEXT:  .L107$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp107:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp107-.L107$pb), %eax
+; LINUX-32-PIC-NEXT:    leal ldst@GOTOFF+262144(%eax), %eax
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: bam07:
@@ -8128,8 +8790,13 @@ define i8* @bam08() nounwind {
 ;
 ; LINUX-32-PIC-LABEL: bam08:
 ; LINUX-32-PIC:       # %bb.0: # %entry
+; LINUX-32-PIC-NEXT:    calll .L108$pb
+; LINUX-32-PIC-NEXT:  .L108$pb:
+; LINUX-32-PIC-NEXT:    popl %ecx
+; LINUX-32-PIC-NEXT:  .Ltmp108:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp108-.L108$pb), %ecx
 ; LINUX-32-PIC-NEXT:    movl $262144, %eax # imm = 0x40000
-; LINUX-32-PIC-NEXT:    addl lptr, %eax
+; LINUX-32-PIC-NEXT:    addl lptr@GOTOFF(%ecx), %eax
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: bam08:
@@ -8198,8 +8865,14 @@ define i8* @cat00(i64 %i) nounwind {
 ;
 ; LINUX-32-PIC-LABEL: cat00:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; LINUX-32-PIC-NEXT:    leal src+64(,%eax,4), %eax
+; LINUX-32-PIC-NEXT:    calll .L109$pb
+; LINUX-32-PIC-NEXT:  .L109$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp109:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp109-.L109$pb), %eax
+; LINUX-32-PIC-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; LINUX-32-PIC-NEXT:    movl src@GOT(%eax), %eax
+; LINUX-32-PIC-NEXT:    leal 64(%eax,%ecx,4), %eax
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: cat00:
@@ -8270,8 +8943,14 @@ define i8* @cxt00(i64 %i) nounwind {
 ;
 ; LINUX-32-PIC-LABEL: cxt00:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; LINUX-32-PIC-NEXT:    leal xsrc+64(,%eax,4), %eax
+; LINUX-32-PIC-NEXT:    calll .L110$pb
+; LINUX-32-PIC-NEXT:  .L110$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp110:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp110-.L110$pb), %eax
+; LINUX-32-PIC-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; LINUX-32-PIC-NEXT:    movl xsrc@GOT(%eax), %eax
+; LINUX-32-PIC-NEXT:    leal 64(%eax,%ecx,4), %eax
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: cxt00:
@@ -8342,8 +9021,14 @@ define i8* @cat01(i64 %i) nounwind {
 ;
 ; LINUX-32-PIC-LABEL: cat01:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; LINUX-32-PIC-NEXT:    leal dst+64(,%eax,4), %eax
+; LINUX-32-PIC-NEXT:    calll .L111$pb
+; LINUX-32-PIC-NEXT:  .L111$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp111:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp111-.L111$pb), %eax
+; LINUX-32-PIC-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; LINUX-32-PIC-NEXT:    movl dst@GOT(%eax), %eax
+; LINUX-32-PIC-NEXT:    leal 64(%eax,%ecx,4), %eax
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: cat01:
@@ -8414,8 +9099,14 @@ define i8* @cxt01(i64 %i) nounwind {
 ;
 ; LINUX-32-PIC-LABEL: cxt01:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; LINUX-32-PIC-NEXT:    leal xdst+64(,%eax,4), %eax
+; LINUX-32-PIC-NEXT:    calll .L112$pb
+; LINUX-32-PIC-NEXT:  .L112$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp112:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp112-.L112$pb), %eax
+; LINUX-32-PIC-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; LINUX-32-PIC-NEXT:    movl xdst@GOT(%eax), %eax
+; LINUX-32-PIC-NEXT:    leal 64(%eax,%ecx,4), %eax
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: cxt01:
@@ -8488,9 +9179,15 @@ define i8* @cat02(i64 %i) nounwind {
 ;
 ; LINUX-32-PIC-LABEL: cat02:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; LINUX-32-PIC-NEXT:    movl ptr, %ecx
-; LINUX-32-PIC-NEXT:    leal 64(%ecx,%eax,4), %eax
+; LINUX-32-PIC-NEXT:    calll .L113$pb
+; LINUX-32-PIC-NEXT:  .L113$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp113:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp113-.L113$pb), %eax
+; LINUX-32-PIC-NEXT:    movl ptr@GOT(%eax), %eax
+; LINUX-32-PIC-NEXT:    movl (%eax), %eax
+; LINUX-32-PIC-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; LINUX-32-PIC-NEXT:    leal 64(%eax,%ecx,4), %eax
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: cat02:
@@ -8569,8 +9266,14 @@ define i8* @cat03(i64 %i) nounwind {
 ;
 ; LINUX-32-PIC-LABEL: cat03:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; LINUX-32-PIC-NEXT:    leal dsrc+64(,%eax,4), %eax
+; LINUX-32-PIC-NEXT:    calll .L114$pb
+; LINUX-32-PIC-NEXT:  .L114$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp114:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp114-.L114$pb), %eax
+; LINUX-32-PIC-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; LINUX-32-PIC-NEXT:    movl dsrc@GOT(%eax), %eax
+; LINUX-32-PIC-NEXT:    leal 64(%eax,%ecx,4), %eax
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: cat03:
@@ -8639,8 +9342,14 @@ define i8* @cat04(i64 %i) nounwind {
 ;
 ; LINUX-32-PIC-LABEL: cat04:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; LINUX-32-PIC-NEXT:    leal ddst+64(,%eax,4), %eax
+; LINUX-32-PIC-NEXT:    calll .L115$pb
+; LINUX-32-PIC-NEXT:  .L115$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp115:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp115-.L115$pb), %eax
+; LINUX-32-PIC-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; LINUX-32-PIC-NEXT:    movl ddst@GOT(%eax), %eax
+; LINUX-32-PIC-NEXT:    leal 64(%eax,%ecx,4), %eax
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: cat04:
@@ -8711,9 +9420,15 @@ define i8* @cat05(i64 %i) nounwind {
 ;
 ; LINUX-32-PIC-LABEL: cat05:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; LINUX-32-PIC-NEXT:    movl dptr, %ecx
-; LINUX-32-PIC-NEXT:    leal 64(%ecx,%eax,4), %eax
+; LINUX-32-PIC-NEXT:    calll .L116$pb
+; LINUX-32-PIC-NEXT:  .L116$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp116:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp116-.L116$pb), %eax
+; LINUX-32-PIC-NEXT:    movl dptr@GOT(%eax), %eax
+; LINUX-32-PIC-NEXT:    movl (%eax), %eax
+; LINUX-32-PIC-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; LINUX-32-PIC-NEXT:    leal 64(%eax,%ecx,4), %eax
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: cat05:
@@ -8787,8 +9502,13 @@ define i8* @cat06(i64 %i) nounwind {
 ;
 ; LINUX-32-PIC-LABEL: cat06:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; LINUX-32-PIC-NEXT:    leal lsrc+64(,%eax,4), %eax
+; LINUX-32-PIC-NEXT:    calll .L117$pb
+; LINUX-32-PIC-NEXT:  .L117$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp117:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp117-.L117$pb), %eax
+; LINUX-32-PIC-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; LINUX-32-PIC-NEXT:    leal lsrc@GOTOFF+64(%eax,%ecx,4), %eax
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: cat06:
@@ -8857,8 +9577,13 @@ define i8* @cat07(i64 %i) nounwind {
 ;
 ; LINUX-32-PIC-LABEL: cat07:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; LINUX-32-PIC-NEXT:    leal ldst+64(,%eax,4), %eax
+; LINUX-32-PIC-NEXT:    calll .L118$pb
+; LINUX-32-PIC-NEXT:  .L118$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp118:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp118-.L118$pb), %eax
+; LINUX-32-PIC-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; LINUX-32-PIC-NEXT:    leal ldst@GOTOFF+64(%eax,%ecx,4), %eax
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: cat07:
@@ -8929,9 +9654,14 @@ define i8* @cat08(i64 %i) nounwind {
 ;
 ; LINUX-32-PIC-LABEL: cat08:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; LINUX-32-PIC-NEXT:    movl lptr, %ecx
-; LINUX-32-PIC-NEXT:    leal 64(%ecx,%eax,4), %eax
+; LINUX-32-PIC-NEXT:    calll .L119$pb
+; LINUX-32-PIC-NEXT:  .L119$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp119:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp119-.L119$pb), %eax
+; LINUX-32-PIC-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; LINUX-32-PIC-NEXT:    movl lptr@GOTOFF(%eax), %eax
+; LINUX-32-PIC-NEXT:    leal 64(%eax,%ecx,4), %eax
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: cat08:
@@ -9004,8 +9734,14 @@ define i8* @cam00(i64 %i) nounwind {
 ;
 ; LINUX-32-PIC-LABEL: cam00:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; LINUX-32-PIC-NEXT:    leal src+262144(,%eax,4), %eax
+; LINUX-32-PIC-NEXT:    calll .L120$pb
+; LINUX-32-PIC-NEXT:  .L120$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp120:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp120-.L120$pb), %eax
+; LINUX-32-PIC-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; LINUX-32-PIC-NEXT:    movl src@GOT(%eax), %eax
+; LINUX-32-PIC-NEXT:    leal 262144(%eax,%ecx,4), %eax
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: cam00:
@@ -9076,8 +9812,14 @@ define i8* @cxm00(i64 %i) nounwind {
 ;
 ; LINUX-32-PIC-LABEL: cxm00:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; LINUX-32-PIC-NEXT:    leal xsrc+262144(,%eax,4), %eax
+; LINUX-32-PIC-NEXT:    calll .L121$pb
+; LINUX-32-PIC-NEXT:  .L121$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp121:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp121-.L121$pb), %eax
+; LINUX-32-PIC-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; LINUX-32-PIC-NEXT:    movl xsrc@GOT(%eax), %eax
+; LINUX-32-PIC-NEXT:    leal 262144(%eax,%ecx,4), %eax
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: cxm00:
@@ -9148,8 +9890,14 @@ define i8* @cam01(i64 %i) nounwind {
 ;
 ; LINUX-32-PIC-LABEL: cam01:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; LINUX-32-PIC-NEXT:    leal dst+262144(,%eax,4), %eax
+; LINUX-32-PIC-NEXT:    calll .L122$pb
+; LINUX-32-PIC-NEXT:  .L122$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp122:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp122-.L122$pb), %eax
+; LINUX-32-PIC-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; LINUX-32-PIC-NEXT:    movl dst@GOT(%eax), %eax
+; LINUX-32-PIC-NEXT:    leal 262144(%eax,%ecx,4), %eax
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: cam01:
@@ -9220,8 +9968,14 @@ define i8* @cxm01(i64 %i) nounwind {
 ;
 ; LINUX-32-PIC-LABEL: cxm01:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; LINUX-32-PIC-NEXT:    leal xdst+262144(,%eax,4), %eax
+; LINUX-32-PIC-NEXT:    calll .L123$pb
+; LINUX-32-PIC-NEXT:  .L123$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp123:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp123-.L123$pb), %eax
+; LINUX-32-PIC-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; LINUX-32-PIC-NEXT:    movl xdst@GOT(%eax), %eax
+; LINUX-32-PIC-NEXT:    leal 262144(%eax,%ecx,4), %eax
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: cxm01:
@@ -9294,9 +10048,15 @@ define i8* @cam02(i64 %i) nounwind {
 ;
 ; LINUX-32-PIC-LABEL: cam02:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; LINUX-32-PIC-NEXT:    movl ptr, %ecx
-; LINUX-32-PIC-NEXT:    leal 262144(%ecx,%eax,4), %eax
+; LINUX-32-PIC-NEXT:    calll .L124$pb
+; LINUX-32-PIC-NEXT:  .L124$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp124:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp124-.L124$pb), %eax
+; LINUX-32-PIC-NEXT:    movl ptr@GOT(%eax), %eax
+; LINUX-32-PIC-NEXT:    movl (%eax), %eax
+; LINUX-32-PIC-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; LINUX-32-PIC-NEXT:    leal 262144(%eax,%ecx,4), %eax
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: cam02:
@@ -9375,8 +10135,14 @@ define i8* @cam03(i64 %i) nounwind {
 ;
 ; LINUX-32-PIC-LABEL: cam03:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; LINUX-32-PIC-NEXT:    leal dsrc+262144(,%eax,4), %eax
+; LINUX-32-PIC-NEXT:    calll .L125$pb
+; LINUX-32-PIC-NEXT:  .L125$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp125:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp125-.L125$pb), %eax
+; LINUX-32-PIC-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; LINUX-32-PIC-NEXT:    movl dsrc@GOT(%eax), %eax
+; LINUX-32-PIC-NEXT:    leal 262144(%eax,%ecx,4), %eax
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: cam03:
@@ -9445,8 +10211,14 @@ define i8* @cam04(i64 %i) nounwind {
 ;
 ; LINUX-32-PIC-LABEL: cam04:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; LINUX-32-PIC-NEXT:    leal ddst+262144(,%eax,4), %eax
+; LINUX-32-PIC-NEXT:    calll .L126$pb
+; LINUX-32-PIC-NEXT:  .L126$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp126:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp126-.L126$pb), %eax
+; LINUX-32-PIC-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; LINUX-32-PIC-NEXT:    movl ddst@GOT(%eax), %eax
+; LINUX-32-PIC-NEXT:    leal 262144(%eax,%ecx,4), %eax
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: cam04:
@@ -9517,9 +10289,15 @@ define i8* @cam05(i64 %i) nounwind {
 ;
 ; LINUX-32-PIC-LABEL: cam05:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; LINUX-32-PIC-NEXT:    movl dptr, %ecx
-; LINUX-32-PIC-NEXT:    leal 262144(%ecx,%eax,4), %eax
+; LINUX-32-PIC-NEXT:    calll .L127$pb
+; LINUX-32-PIC-NEXT:  .L127$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp127:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp127-.L127$pb), %eax
+; LINUX-32-PIC-NEXT:    movl dptr@GOT(%eax), %eax
+; LINUX-32-PIC-NEXT:    movl (%eax), %eax
+; LINUX-32-PIC-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; LINUX-32-PIC-NEXT:    leal 262144(%eax,%ecx,4), %eax
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: cam05:
@@ -9593,8 +10371,13 @@ define i8* @cam06(i64 %i) nounwind {
 ;
 ; LINUX-32-PIC-LABEL: cam06:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; LINUX-32-PIC-NEXT:    leal lsrc+262144(,%eax,4), %eax
+; LINUX-32-PIC-NEXT:    calll .L128$pb
+; LINUX-32-PIC-NEXT:  .L128$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp128:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp128-.L128$pb), %eax
+; LINUX-32-PIC-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; LINUX-32-PIC-NEXT:    leal lsrc@GOTOFF+262144(%eax,%ecx,4), %eax
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: cam06:
@@ -9663,8 +10446,13 @@ define i8* @cam07(i64 %i) nounwind {
 ;
 ; LINUX-32-PIC-LABEL: cam07:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; LINUX-32-PIC-NEXT:    leal ldst+262144(,%eax,4), %eax
+; LINUX-32-PIC-NEXT:    calll .L129$pb
+; LINUX-32-PIC-NEXT:  .L129$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp129:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp129-.L129$pb), %eax
+; LINUX-32-PIC-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; LINUX-32-PIC-NEXT:    leal ldst@GOTOFF+262144(%eax,%ecx,4), %eax
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: cam07:
@@ -9735,9 +10523,14 @@ define i8* @cam08(i64 %i) nounwind {
 ;
 ; LINUX-32-PIC-LABEL: cam08:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; LINUX-32-PIC-NEXT:    movl lptr, %ecx
-; LINUX-32-PIC-NEXT:    leal 262144(%ecx,%eax,4), %eax
+; LINUX-32-PIC-NEXT:    calll .L130$pb
+; LINUX-32-PIC-NEXT:  .L130$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp130:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp130-.L130$pb), %eax
+; LINUX-32-PIC-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; LINUX-32-PIC-NEXT:    movl lptr@GOTOFF(%eax), %eax
+; LINUX-32-PIC-NEXT:    leal 262144(%eax,%ecx,4), %eax
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: cam08:
@@ -9825,15 +10618,22 @@ define void @lcallee() nounwind {
 ;
 ; LINUX-32-PIC-LABEL: lcallee:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    subl $12, %esp
-; LINUX-32-PIC-NEXT:    calll x
-; LINUX-32-PIC-NEXT:    calll x
-; LINUX-32-PIC-NEXT:    calll x
-; LINUX-32-PIC-NEXT:    calll x
-; LINUX-32-PIC-NEXT:    calll x
-; LINUX-32-PIC-NEXT:    calll x
-; LINUX-32-PIC-NEXT:    calll x
-; LINUX-32-PIC-NEXT:    addl $12, %esp
+; LINUX-32-PIC-NEXT:    pushl %ebx
+; LINUX-32-PIC-NEXT:    subl $8, %esp
+; LINUX-32-PIC-NEXT:    calll .L131$pb
+; LINUX-32-PIC-NEXT:  .L131$pb:
+; LINUX-32-PIC-NEXT:    popl %ebx
+; LINUX-32-PIC-NEXT:  .Ltmp131:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp131-.L131$pb), %ebx
+; LINUX-32-PIC-NEXT:    calll x@PLT
+; LINUX-32-PIC-NEXT:    calll x@PLT
+; LINUX-32-PIC-NEXT:    calll x@PLT
+; LINUX-32-PIC-NEXT:    calll x@PLT
+; LINUX-32-PIC-NEXT:    calll x@PLT
+; LINUX-32-PIC-NEXT:    calll x@PLT
+; LINUX-32-PIC-NEXT:    calll x@PLT
+; LINUX-32-PIC-NEXT:    addl $8, %esp
+; LINUX-32-PIC-NEXT:    popl %ebx
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: lcallee:
@@ -9969,15 +10769,22 @@ define internal void @dcallee() nounwind {
 ;
 ; LINUX-32-PIC-LABEL: dcallee:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    subl $12, %esp
-; LINUX-32-PIC-NEXT:    calll y
-; LINUX-32-PIC-NEXT:    calll y
-; LINUX-32-PIC-NEXT:    calll y
-; LINUX-32-PIC-NEXT:    calll y
-; LINUX-32-PIC-NEXT:    calll y
-; LINUX-32-PIC-NEXT:    calll y
-; LINUX-32-PIC-NEXT:    calll y
-; LINUX-32-PIC-NEXT:    addl $12, %esp
+; LINUX-32-PIC-NEXT:    pushl %ebx
+; LINUX-32-PIC-NEXT:    subl $8, %esp
+; LINUX-32-PIC-NEXT:    calll .L132$pb
+; LINUX-32-PIC-NEXT:  .L132$pb:
+; LINUX-32-PIC-NEXT:    popl %ebx
+; LINUX-32-PIC-NEXT:  .Ltmp132:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp132-.L132$pb), %ebx
+; LINUX-32-PIC-NEXT:    calll y@PLT
+; LINUX-32-PIC-NEXT:    calll y@PLT
+; LINUX-32-PIC-NEXT:    calll y@PLT
+; LINUX-32-PIC-NEXT:    calll y@PLT
+; LINUX-32-PIC-NEXT:    calll y@PLT
+; LINUX-32-PIC-NEXT:    calll y@PLT
+; LINUX-32-PIC-NEXT:    calll y@PLT
+; LINUX-32-PIC-NEXT:    addl $8, %esp
+; LINUX-32-PIC-NEXT:    popl %ebx
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: dcallee:
@@ -10097,7 +10904,12 @@ define void ()* @address() nounwind {
 ;
 ; LINUX-32-PIC-LABEL: address:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl $callee, %eax
+; LINUX-32-PIC-NEXT:    calll .L133$pb
+; LINUX-32-PIC-NEXT:  .L133$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp133:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp133-.L133$pb), %eax
+; LINUX-32-PIC-NEXT:    movl callee@GOT(%eax), %eax
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: address:
@@ -10157,7 +10969,12 @@ define void ()* @laddress() nounwind {
 ;
 ; LINUX-32-PIC-LABEL: laddress:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl $lcallee, %eax
+; LINUX-32-PIC-NEXT:    calll .L134$pb
+; LINUX-32-PIC-NEXT:  .L134$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp134:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp134-.L134$pb), %eax
+; LINUX-32-PIC-NEXT:    movl lcallee@GOT(%eax), %eax
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: laddress:
@@ -10215,7 +11032,12 @@ define void ()* @daddress() nounwind {
 ;
 ; LINUX-32-PIC-LABEL: daddress:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    movl $dcallee, %eax
+; LINUX-32-PIC-NEXT:    calll .L135$pb
+; LINUX-32-PIC-NEXT:  .L135$pb:
+; LINUX-32-PIC-NEXT:    popl %eax
+; LINUX-32-PIC-NEXT:  .Ltmp135:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp135-.L135$pb), %eax
+; LINUX-32-PIC-NEXT:    leal dcallee@GOTOFF(%eax), %eax
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: daddress:
@@ -10279,10 +11101,17 @@ define void @caller() nounwind {
 ;
 ; LINUX-32-PIC-LABEL: caller:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    subl $12, %esp
-; LINUX-32-PIC-NEXT:    calll callee
-; LINUX-32-PIC-NEXT:    calll callee
-; LINUX-32-PIC-NEXT:    addl $12, %esp
+; LINUX-32-PIC-NEXT:    pushl %ebx
+; LINUX-32-PIC-NEXT:    subl $8, %esp
+; LINUX-32-PIC-NEXT:    calll .L136$pb
+; LINUX-32-PIC-NEXT:  .L136$pb:
+; LINUX-32-PIC-NEXT:    popl %ebx
+; LINUX-32-PIC-NEXT:  .Ltmp136:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp136-.L136$pb), %ebx
+; LINUX-32-PIC-NEXT:    calll callee@PLT
+; LINUX-32-PIC-NEXT:    calll callee@PLT
+; LINUX-32-PIC-NEXT:    addl $8, %esp
+; LINUX-32-PIC-NEXT:    popl %ebx
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: caller:
@@ -10366,10 +11195,17 @@ define void @dcaller() nounwind {
 ;
 ; LINUX-32-PIC-LABEL: dcaller:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    subl $12, %esp
+; LINUX-32-PIC-NEXT:    pushl %ebx
+; LINUX-32-PIC-NEXT:    subl $8, %esp
+; LINUX-32-PIC-NEXT:    calll .L137$pb
+; LINUX-32-PIC-NEXT:  .L137$pb:
+; LINUX-32-PIC-NEXT:    popl %ebx
+; LINUX-32-PIC-NEXT:  .Ltmp137:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp137-.L137$pb), %ebx
 ; LINUX-32-PIC-NEXT:    calll dcallee
 ; LINUX-32-PIC-NEXT:    calll dcallee
-; LINUX-32-PIC-NEXT:    addl $12, %esp
+; LINUX-32-PIC-NEXT:    addl $8, %esp
+; LINUX-32-PIC-NEXT:    popl %ebx
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: dcaller:
@@ -10453,10 +11289,17 @@ define void @lcaller() nounwind {
 ;
 ; LINUX-32-PIC-LABEL: lcaller:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    subl $12, %esp
-; LINUX-32-PIC-NEXT:    calll lcallee
-; LINUX-32-PIC-NEXT:    calll lcallee
-; LINUX-32-PIC-NEXT:    addl $12, %esp
+; LINUX-32-PIC-NEXT:    pushl %ebx
+; LINUX-32-PIC-NEXT:    subl $8, %esp
+; LINUX-32-PIC-NEXT:    calll .L138$pb
+; LINUX-32-PIC-NEXT:  .L138$pb:
+; LINUX-32-PIC-NEXT:    popl %ebx
+; LINUX-32-PIC-NEXT:  .Ltmp138:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp138-.L138$pb), %ebx
+; LINUX-32-PIC-NEXT:    calll lcallee@PLT
+; LINUX-32-PIC-NEXT:    calll lcallee@PLT
+; LINUX-32-PIC-NEXT:    addl $8, %esp
+; LINUX-32-PIC-NEXT:    popl %ebx
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: lcaller:
@@ -10538,9 +11381,16 @@ define void @tailcaller() nounwind {
 ;
 ; LINUX-32-PIC-LABEL: tailcaller:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    subl $12, %esp
-; LINUX-32-PIC-NEXT:    calll callee
-; LINUX-32-PIC-NEXT:    addl $12, %esp
+; LINUX-32-PIC-NEXT:    pushl %ebx
+; LINUX-32-PIC-NEXT:    subl $8, %esp
+; LINUX-32-PIC-NEXT:    calll .L139$pb
+; LINUX-32-PIC-NEXT:  .L139$pb:
+; LINUX-32-PIC-NEXT:    popl %ebx
+; LINUX-32-PIC-NEXT:  .Ltmp139:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp139-.L139$pb), %ebx
+; LINUX-32-PIC-NEXT:    calll callee@PLT
+; LINUX-32-PIC-NEXT:    addl $8, %esp
+; LINUX-32-PIC-NEXT:    popl %ebx
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: tailcaller:
@@ -10614,9 +11464,16 @@ define void @dtailcaller() nounwind {
 ;
 ; LINUX-32-PIC-LABEL: dtailcaller:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    subl $12, %esp
+; LINUX-32-PIC-NEXT:    pushl %ebx
+; LINUX-32-PIC-NEXT:    subl $8, %esp
+; LINUX-32-PIC-NEXT:    calll .L140$pb
+; LINUX-32-PIC-NEXT:  .L140$pb:
+; LINUX-32-PIC-NEXT:    popl %ebx
+; LINUX-32-PIC-NEXT:  .Ltmp140:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp140-.L140$pb), %ebx
 ; LINUX-32-PIC-NEXT:    calll dcallee
-; LINUX-32-PIC-NEXT:    addl $12, %esp
+; LINUX-32-PIC-NEXT:    addl $8, %esp
+; LINUX-32-PIC-NEXT:    popl %ebx
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: dtailcaller:
@@ -10690,9 +11547,16 @@ define void @ltailcaller() nounwind {
 ;
 ; LINUX-32-PIC-LABEL: ltailcaller:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    subl $12, %esp
-; LINUX-32-PIC-NEXT:    calll lcallee
-; LINUX-32-PIC-NEXT:    addl $12, %esp
+; LINUX-32-PIC-NEXT:    pushl %ebx
+; LINUX-32-PIC-NEXT:    subl $8, %esp
+; LINUX-32-PIC-NEXT:    calll .L141$pb
+; LINUX-32-PIC-NEXT:  .L141$pb:
+; LINUX-32-PIC-NEXT:    popl %ebx
+; LINUX-32-PIC-NEXT:  .Ltmp141:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp141-.L141$pb), %ebx
+; LINUX-32-PIC-NEXT:    calll lcallee@PLT
+; LINUX-32-PIC-NEXT:    addl $8, %esp
+; LINUX-32-PIC-NEXT:    popl %ebx
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: ltailcaller:
@@ -10768,10 +11632,20 @@ define void @icaller() nounwind {
 ;
 ; LINUX-32-PIC-LABEL: icaller:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    subl $12, %esp
-; LINUX-32-PIC-NEXT:    calll *ifunc
-; LINUX-32-PIC-NEXT:    calll *ifunc
-; LINUX-32-PIC-NEXT:    addl $12, %esp
+; LINUX-32-PIC-NEXT:    pushl %ebx
+; LINUX-32-PIC-NEXT:    pushl %esi
+; LINUX-32-PIC-NEXT:    pushl %eax
+; LINUX-32-PIC-NEXT:    calll .L142$pb
+; LINUX-32-PIC-NEXT:  .L142$pb:
+; LINUX-32-PIC-NEXT:    popl %ebx
+; LINUX-32-PIC-NEXT:  .Ltmp142:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp142-.L142$pb), %ebx
+; LINUX-32-PIC-NEXT:    movl ifunc@GOT(%ebx), %esi
+; LINUX-32-PIC-NEXT:    calll *(%esi)
+; LINUX-32-PIC-NEXT:    calll *(%esi)
+; LINUX-32-PIC-NEXT:    addl $4, %esp
+; LINUX-32-PIC-NEXT:    popl %esi
+; LINUX-32-PIC-NEXT:    popl %ebx
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: icaller:
@@ -10870,10 +11744,20 @@ define void @dicaller() nounwind {
 ;
 ; LINUX-32-PIC-LABEL: dicaller:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    subl $12, %esp
-; LINUX-32-PIC-NEXT:    calll *difunc
-; LINUX-32-PIC-NEXT:    calll *difunc
-; LINUX-32-PIC-NEXT:    addl $12, %esp
+; LINUX-32-PIC-NEXT:    pushl %ebx
+; LINUX-32-PIC-NEXT:    pushl %esi
+; LINUX-32-PIC-NEXT:    pushl %eax
+; LINUX-32-PIC-NEXT:    calll .L143$pb
+; LINUX-32-PIC-NEXT:  .L143$pb:
+; LINUX-32-PIC-NEXT:    popl %ebx
+; LINUX-32-PIC-NEXT:  .Ltmp143:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp143-.L143$pb), %ebx
+; LINUX-32-PIC-NEXT:    movl difunc@GOT(%ebx), %esi
+; LINUX-32-PIC-NEXT:    calll *(%esi)
+; LINUX-32-PIC-NEXT:    calll *(%esi)
+; LINUX-32-PIC-NEXT:    addl $4, %esp
+; LINUX-32-PIC-NEXT:    popl %esi
+; LINUX-32-PIC-NEXT:    popl %ebx
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: dicaller:
@@ -10965,10 +11849,17 @@ define void @licaller() nounwind {
 ;
 ; LINUX-32-PIC-LABEL: licaller:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    subl $12, %esp
-; LINUX-32-PIC-NEXT:    calll *lifunc
-; LINUX-32-PIC-NEXT:    calll *lifunc
-; LINUX-32-PIC-NEXT:    addl $12, %esp
+; LINUX-32-PIC-NEXT:    pushl %ebx
+; LINUX-32-PIC-NEXT:    subl $8, %esp
+; LINUX-32-PIC-NEXT:    calll .L144$pb
+; LINUX-32-PIC-NEXT:  .L144$pb:
+; LINUX-32-PIC-NEXT:    popl %ebx
+; LINUX-32-PIC-NEXT:  .Ltmp144:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp144-.L144$pb), %ebx
+; LINUX-32-PIC-NEXT:    calll *lifunc@GOTOFF(%ebx)
+; LINUX-32-PIC-NEXT:    calll *lifunc@GOTOFF(%ebx)
+; LINUX-32-PIC-NEXT:    addl $8, %esp
+; LINUX-32-PIC-NEXT:    popl %ebx
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: licaller:
@@ -11059,10 +11950,20 @@ define void @itailcaller() nounwind {
 ;
 ; LINUX-32-PIC-LABEL: itailcaller:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    subl $12, %esp
-; LINUX-32-PIC-NEXT:    calll *ifunc
-; LINUX-32-PIC-NEXT:    calll *ifunc
-; LINUX-32-PIC-NEXT:    addl $12, %esp
+; LINUX-32-PIC-NEXT:    pushl %ebx
+; LINUX-32-PIC-NEXT:    pushl %esi
+; LINUX-32-PIC-NEXT:    pushl %eax
+; LINUX-32-PIC-NEXT:    calll .L145$pb
+; LINUX-32-PIC-NEXT:  .L145$pb:
+; LINUX-32-PIC-NEXT:    popl %ebx
+; LINUX-32-PIC-NEXT:  .Ltmp145:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp145-.L145$pb), %ebx
+; LINUX-32-PIC-NEXT:    movl ifunc@GOT(%ebx), %esi
+; LINUX-32-PIC-NEXT:    calll *(%esi)
+; LINUX-32-PIC-NEXT:    calll *(%esi)
+; LINUX-32-PIC-NEXT:    addl $4, %esp
+; LINUX-32-PIC-NEXT:    popl %esi
+; LINUX-32-PIC-NEXT:    popl %ebx
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: itailcaller:
@@ -11159,9 +12060,17 @@ define void @ditailcaller() nounwind {
 ;
 ; LINUX-32-PIC-LABEL: ditailcaller:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    subl $12, %esp
-; LINUX-32-PIC-NEXT:    calll *difunc
-; LINUX-32-PIC-NEXT:    addl $12, %esp
+; LINUX-32-PIC-NEXT:    pushl %ebx
+; LINUX-32-PIC-NEXT:    subl $8, %esp
+; LINUX-32-PIC-NEXT:    calll .L146$pb
+; LINUX-32-PIC-NEXT:  .L146$pb:
+; LINUX-32-PIC-NEXT:    popl %ebx
+; LINUX-32-PIC-NEXT:  .Ltmp146:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp146-.L146$pb), %ebx
+; LINUX-32-PIC-NEXT:    movl difunc@GOT(%ebx), %eax
+; LINUX-32-PIC-NEXT:    calll *(%eax)
+; LINUX-32-PIC-NEXT:    addl $8, %esp
+; LINUX-32-PIC-NEXT:    popl %ebx
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: ditailcaller:
@@ -11240,9 +12149,16 @@ define void @litailcaller() nounwind {
 ;
 ; LINUX-32-PIC-LABEL: litailcaller:
 ; LINUX-32-PIC:       # %bb.0: # %entry
-; LINUX-32-PIC-NEXT:    subl $12, %esp
-; LINUX-32-PIC-NEXT:    calll *lifunc
-; LINUX-32-PIC-NEXT:    addl $12, %esp
+; LINUX-32-PIC-NEXT:    pushl %ebx
+; LINUX-32-PIC-NEXT:    subl $8, %esp
+; LINUX-32-PIC-NEXT:    calll .L147$pb
+; LINUX-32-PIC-NEXT:  .L147$pb:
+; LINUX-32-PIC-NEXT:    popl %ebx
+; LINUX-32-PIC-NEXT:  .Ltmp147:
+; LINUX-32-PIC-NEXT:    addl $_GLOBAL_OFFSET_TABLE_+(.Ltmp147-.L147$pb), %ebx
+; LINUX-32-PIC-NEXT:    calll *lifunc@GOTOFF(%ebx)
+; LINUX-32-PIC-NEXT:    addl $8, %esp
+; LINUX-32-PIC-NEXT:    popl %ebx
 ; LINUX-32-PIC-NEXT:    retl
 ;
 ; LINUX-64-PIC-LABEL: litailcaller:
diff --git a/test/CodeGen/X86/addr-mode-matcher-2.ll b/test/CodeGen/X86/addr-mode-matcher-2.ll
new file mode 100644
index 0000000000000000000000000000000000000000..1b32ba40c66989eff8becf34f172404f73e542e2
--- /dev/null
+++ b/test/CodeGen/X86/addr-mode-matcher-2.ll
@@ -0,0 +1,73 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-unknown | FileCheck %s --check-prefix=X86
+; RUN: llc < %s -mtriple=x86_64-unknown | FileCheck %s --check-prefix=X64
+
+; Test coverage for matchAddressRecursively's MUL handling
+
+; Based off:
+; struct A {
+;   int m_ints[5];
+;   int m_bar();
+; };
+; struct {
+;   A* m_data;
+; } c;
+; void foo(bool b, int i) {
+;   if (b)
+;     return;
+;   int j = c.m_data[i + 1].m_bar();
+;   foo(false, j);
+; }
+
+%struct.A = type { [5 x i32] }
+
+define void @foo(i1 zeroext, i32) nounwind {
+; X86-LABEL: foo:
+; X86:       # %bb.0:
+; X86-NEXT:    movb {{[0-9]+}}(%esp), %al
+; X86-NEXT:    testb %al, %al
+; X86-NEXT:    je .LBB0_1
+; X86-NEXT:  # %bb.3:
+; X86-NEXT:    retl
+; X86-NEXT:  .LBB0_1: # %.preheader
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    .p2align 4, 0x90
+; X86-NEXT:  .LBB0_2: # =>This Inner Loop Header: Depth=1
+; X86-NEXT:    leal (%eax,%eax,4), %eax
+; X86-NEXT:    leal 20(,%eax,4), %eax
+; X86-NEXT:    pushl %eax
+; X86-NEXT:    calll bar
+; X86-NEXT:    addl $4, %esp
+; X86-NEXT:    jmp .LBB0_2
+;
+; X64-LABEL: foo:
+; X64:       # %bb.0:
+; X64-NEXT:    pushq %rax
+; X64-NEXT:    testl %edi, %edi
+; X64-NEXT:    je .LBB0_1
+; X64-NEXT:  # %bb.3:
+; X64-NEXT:    popq %rax
+; X64-NEXT:    retq
+; X64-NEXT:  .LBB0_1: # %.preheader
+; X64-NEXT:    movl %esi, %eax
+; X64-NEXT:    .p2align 4, 0x90
+; X64-NEXT:  .LBB0_2: # =>This Inner Loop Header: Depth=1
+; X64-NEXT:    incl %eax
+; X64-NEXT:    cltq
+; X64-NEXT:    shlq $2, %rax
+; X64-NEXT:    leaq (%rax,%rax,4), %rdi
+; X64-NEXT:    callq bar
+; X64-NEXT:    jmp .LBB0_2
+  br i1 %0, label %9, label %3
+
+  %4 = phi i32 [ %8, %3 ], [ %1, %2 ]
+  %5 = add nsw i32 %4, 1
+  %6 = sext i32 %5 to i64
+  %7 = getelementptr inbounds %struct.A, %struct.A* null, i64 %6
+  %8 = tail call i32 @bar(%struct.A* %7)
+  br label %3
+
+  ret void
+}
+
+declare i32 @bar(%struct.A*)
diff --git a/test/CodeGen/X86/addr-of-ret-addr.ll b/test/CodeGen/X86/addr-of-ret-addr.ll
index 67ebb7f9d78d9786acce4050d9dd6d110dfa0819..848368938d04ab60d4232b4b02467ee9b6ea4df9 100644
--- a/test/CodeGen/X86/addr-of-ret-addr.ll
+++ b/test/CodeGen/X86/addr-of-ret-addr.ll
@@ -1,5 +1,5 @@
-; RUN: llc < %s -disable-fp-elim -mtriple=i686-- | FileCheck %s --check-prefix=CHECK-X86
-; RUN: llc < %s -disable-fp-elim -mtriple=x86_64-- | FileCheck %s --check-prefix=CHECK-X64
+; RUN: llc < %s -frame-pointer=all -mtriple=i686-- | FileCheck %s --check-prefix=CHECK-X86
+; RUN: llc < %s -frame-pointer=all -mtriple=x86_64-- | FileCheck %s --check-prefix=CHECK-X64
 
 define i8* @f() nounwind readnone optsize {
 entry:
diff --git a/test/CodeGen/X86/adx-intrinsics-upgrade.ll b/test/CodeGen/X86/adx-intrinsics-upgrade.ll
index 34f8ff8725958e0fe2cf20d84eac2e3c2dec7bc1..762314a5b5546f4728c4e1ed8764f6ae4e92cb7f 100644
--- a/test/CodeGen/X86/adx-intrinsics-upgrade.ll
+++ b/test/CodeGen/X86/adx-intrinsics-upgrade.ll
@@ -108,9 +108,7 @@ define i32 @load_crash(i64* nocapture readonly %a, i64* nocapture readonly %b, i
 define void @allzeros() {
 ; CHECK-LABEL: allzeros:
 ; CHECK:       ## %bb.0: ## %entry
-; CHECK-NEXT:    xorl %eax, %eax ## encoding: [0x31,0xc0]
-; CHECK-NEXT:    addq $0, %rax ## encoding: [0x48,0x83,0xc0,0x00]
-; CHECK-NEXT:    movq %rax, 0 ## encoding: [0x48,0x89,0x04,0x25,0x00,0x00,0x00,0x00]
+; CHECK-NEXT:    movq $0, 0 ## encoding: [0x48,0xc7,0x04,0x25,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00]
 ; CHECK-NEXT:    retq ## encoding: [0xc3]
 entry:
   %0 = tail call i8 @llvm.x86.addcarryx.u64(i8 0, i64 0, i64 0, i8* null)
diff --git a/test/CodeGen/X86/adx-intrinsics.ll b/test/CodeGen/X86/adx-intrinsics.ll
index d37be0653289bbd36773f4941d48cc6b61771495..6c1c2739c64dcead5b678c32b169665a4cca2b06 100644
--- a/test/CodeGen/X86/adx-intrinsics.ll
+++ b/test/CodeGen/X86/adx-intrinsics.ll
@@ -134,9 +134,7 @@ define i32 @load_crash(i64* nocapture readonly %a, i64* nocapture readonly %b, i
 define void @allzeros() {
 ; CHECK-LABEL: allzeros:
 ; CHECK:       ## %bb.0: ## %entry
-; CHECK-NEXT:    xorl %eax, %eax ## encoding: [0x31,0xc0]
-; CHECK-NEXT:    addq $0, %rax ## encoding: [0x48,0x83,0xc0,0x00]
-; CHECK-NEXT:    movq %rax, 0 ## encoding: [0x48,0x89,0x04,0x25,0x00,0x00,0x00,0x00]
+; CHECK-NEXT:    movq $0, 0 ## encoding: [0x48,0xc7,0x04,0x25,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00]
 ; CHECK-NEXT:    retq ## encoding: [0xc3]
 entry:
   %0 = call { i8, i64 } @llvm.x86.addcarry.64(i8 0, i64 0, i64 0)
diff --git a/test/CodeGen/X86/adx-schedule.ll b/test/CodeGen/X86/adx-schedule.ll
deleted file mode 100644
index e6c65b2d21495e154f329db85652f6726832ef3c..0000000000000000000000000000000000000000
--- a/test/CodeGen/X86/adx-schedule.ll
+++ /dev/null
@@ -1,114 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=x86-64 -mattr=+adx | FileCheck %s --check-prefix=CHECK --check-prefix=GENERIC
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=broadwell | FileCheck %s --check-prefix=CHECK --check-prefix=BROADWELL
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=skylake | FileCheck %s --check-prefix=CHECK --check-prefix=SKYLAKE --check-prefix=SKL
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=skx     | FileCheck %s --check-prefix=CHECK --check-prefix=SKYLAKE --check-prefix=SKX
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=knl     | FileCheck %s --check-prefix=CHECK --check-prefix=KNL
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=znver1  | FileCheck %s --check-prefix=CHECK --check-prefix=ZNVER1
-
-define void @test_adcx(i32 %a0, i32* %a1, i64 %a2, i64* %a3) optsize {
-; GENERIC-LABEL: test_adcx:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    adcxl %edi, %edi # sched: [2:0.67]
-; GENERIC-NEXT:    adcxq %rdx, %rdx # sched: [2:0.67]
-; GENERIC-NEXT:    adcxl (%rsi), %edi # sched: [7:0.67]
-; GENERIC-NEXT:    adcxq (%rcx), %rdx # sched: [7:0.67]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BROADWELL-LABEL: test_adcx:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    adcxl %edi, %edi # sched: [1:0.50]
-; BROADWELL-NEXT:    adcxq %rdx, %rdx # sched: [1:0.50]
-; BROADWELL-NEXT:    adcxl (%rsi), %edi # sched: [6:0.50]
-; BROADWELL-NEXT:    adcxq (%rcx), %rdx # sched: [6:0.50]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_adcx:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    adcxl %edi, %edi # sched: [1:0.50]
-; SKYLAKE-NEXT:    adcxq %rdx, %rdx # sched: [1:0.50]
-; SKYLAKE-NEXT:    adcxl (%rsi), %edi # sched: [6:0.50]
-; SKYLAKE-NEXT:    adcxq (%rcx), %rdx # sched: [6:0.50]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; KNL-LABEL: test_adcx:
-; KNL:       # %bb.0:
-; KNL-NEXT:    #APP
-; KNL-NEXT:    adcxl %edi, %edi # sched: [2:0.50]
-; KNL-NEXT:    adcxq %rdx, %rdx # sched: [2:0.50]
-; KNL-NEXT:    adcxl (%rsi), %edi # sched: [7:0.50]
-; KNL-NEXT:    adcxq (%rcx), %rdx # sched: [7:0.50]
-; KNL-NEXT:    #NO_APP
-; KNL-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_adcx:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    adcxl %edi, %edi # sched: [1:0.25]
-; ZNVER1-NEXT:    adcxq %rdx, %rdx # sched: [1:0.25]
-; ZNVER1-NEXT:    adcxl (%rsi), %edi # sched: [5:0.50]
-; ZNVER1-NEXT:    adcxq (%rcx), %rdx # sched: [5:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  tail call void asm "adcx $0, $0 \0A\09 adcx $2, $2 \0A\09 adcx $1, $0 \0A\09 adcx $3, $2", "r,*m,r,*m"(i32 %a0, i32* %a1, i64 %a2, i64* %a3) nounwind
-  ret void
-}
-define void @test_adox(i32 %a0, i32* %a1, i64 %a2, i64* %a3) optsize {
-; GENERIC-LABEL: test_adox:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    adoxl %edi, %edi # sched: [2:0.67]
-; GENERIC-NEXT:    adoxq %rdx, %rdx # sched: [2:0.67]
-; GENERIC-NEXT:    adoxl (%rsi), %edi # sched: [7:0.67]
-; GENERIC-NEXT:    adoxq (%rcx), %rdx # sched: [7:0.67]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BROADWELL-LABEL: test_adox:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    adoxl %edi, %edi # sched: [1:0.50]
-; BROADWELL-NEXT:    adoxq %rdx, %rdx # sched: [1:0.50]
-; BROADWELL-NEXT:    adoxl (%rsi), %edi # sched: [6:0.50]
-; BROADWELL-NEXT:    adoxq (%rcx), %rdx # sched: [6:0.50]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_adox:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    adoxl %edi, %edi # sched: [1:0.50]
-; SKYLAKE-NEXT:    adoxq %rdx, %rdx # sched: [1:0.50]
-; SKYLAKE-NEXT:    adoxl (%rsi), %edi # sched: [6:0.50]
-; SKYLAKE-NEXT:    adoxq (%rcx), %rdx # sched: [6:0.50]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; KNL-LABEL: test_adox:
-; KNL:       # %bb.0:
-; KNL-NEXT:    #APP
-; KNL-NEXT:    adoxl %edi, %edi # sched: [2:0.50]
-; KNL-NEXT:    adoxq %rdx, %rdx # sched: [2:0.50]
-; KNL-NEXT:    adoxl (%rsi), %edi # sched: [7:0.50]
-; KNL-NEXT:    adoxq (%rcx), %rdx # sched: [7:0.50]
-; KNL-NEXT:    #NO_APP
-; KNL-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_adox:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    adoxl %edi, %edi # sched: [1:0.25]
-; ZNVER1-NEXT:    adoxq %rdx, %rdx # sched: [1:0.25]
-; ZNVER1-NEXT:    adoxl (%rsi), %edi # sched: [5:0.50]
-; ZNVER1-NEXT:    adoxq (%rcx), %rdx # sched: [5:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  tail call void asm "adox $0, $0 \0A\09 adox $2, $2 \0A\09 adox $1, $0 \0A\09 adox $3, $2", "r,*m,r,*m"(i32 %a0, i32* %a1, i64 %a2, i64* %a3) nounwind
-  ret void
-}
diff --git a/test/CodeGen/X86/aes-schedule.ll b/test/CodeGen/X86/aes-schedule.ll
deleted file mode 100644
index c622899ca09168b6e95bdb34646035fcbee38045..0000000000000000000000000000000000000000
--- a/test/CodeGen/X86/aes-schedule.ll
+++ /dev/null
@@ -1,751 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=x86-64 -mattr=+aes | FileCheck %s --check-prefix=CHECK --check-prefix=GENERIC
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=slm -mattr=+aes | FileCheck %s --check-prefix=CHECK --check-prefix=SLM
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=goldmont -mattr=+aes | FileCheck %s --check-prefix=CHECK --check-prefix=SLM
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=sandybridge -mattr=+aes,-avx  | FileCheck %s --check-prefixes=CHECK,SANDY-SSE
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=sandybridge -mattr=+aes,-avx2 | FileCheck %s --check-prefixes=CHECK,SANDY
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=ivybridge -mattr=+aes,-avx  | FileCheck %s --check-prefixes=CHECK,SANDY-SSE
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=ivybridge -mattr=+aes,-avx2 | FileCheck %s --check-prefixes=CHECK,SANDY
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=haswell -mattr=+aes,-avx  | FileCheck %s --check-prefixes=CHECK,HASWELL-SSE
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=haswell -mattr=+aes,-avx2 | FileCheck %s --check-prefixes=CHECK,HASWELL
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=broadwell -mattr=+aes,-avx  | FileCheck %s --check-prefixes=CHECK,BROADWELL-SSE
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=broadwell -mattr=+aes,-avx2 | FileCheck %s --check-prefixes=CHECK,BROADWELL
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=skylake -mattr=+aes,-avx  | FileCheck %s --check-prefixes=CHECK,SKYLAKE-SSE
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=skylake -mattr=+aes,-avx2 | FileCheck %s --check-prefixes=CHECK,SKYLAKE
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=skx -mattr=+aes,-avx  | FileCheck %s --check-prefixes=CHECK,SKX-SSE
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=skx -mattr=+aes,-avx2 | FileCheck %s --check-prefixes=CHECK,SKX
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=bdver2 -mattr=+aes,-avx  | FileCheck %s --check-prefixes=CHECK,BDVER2-SSE
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=bdver2 -mattr=+aes,-avx2 | FileCheck %s --check-prefixes=CHECK,BDVER2
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=btver2 -mattr=+aes,-avx  | FileCheck %s --check-prefixes=CHECK,BTVER2-SSE
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=btver2 -mattr=+aes,-avx2 | FileCheck %s --check-prefixes=CHECK,BTVER2
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=znver1 -mattr=+aes,-avx  | FileCheck %s --check-prefixes=CHECK,ZNVER1-SSE
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=znver1 -mattr=+aes,-avx2 | FileCheck %s --check-prefixes=CHECK,ZNVER1
-
-define <2 x i64> @test_aesdec(<2 x i64> %a0, <2 x i64> %a1, <2 x i64> *%a2) {
-; GENERIC-LABEL: test_aesdec:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    aesdec %xmm1, %xmm0 # sched: [7:1.00]
-; GENERIC-NEXT:    aesdec (%rdi), %xmm0 # sched: [13:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SLM-LABEL: test_aesdec:
-; SLM:       # %bb.0:
-; SLM-NEXT:    aesdec %xmm1, %xmm0 # sched: [8:5.00]
-; SLM-NEXT:    aesdec (%rdi), %xmm0 # sched: [8:5.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_aesdec:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    aesdec %xmm1, %xmm0 # sched: [7:1.00]
-; SANDY-SSE-NEXT:    aesdec (%rdi), %xmm0 # sched: [13:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_aesdec:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vaesdec %xmm1, %xmm0, %xmm0 # sched: [7:1.00]
-; SANDY-NEXT:    vaesdec (%rdi), %xmm0, %xmm0 # sched: [13:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_aesdec:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    aesdec %xmm1, %xmm0 # sched: [7:1.00]
-; HASWELL-SSE-NEXT:    aesdec (%rdi), %xmm0 # sched: [13:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_aesdec:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vaesdec %xmm1, %xmm0, %xmm0 # sched: [7:1.00]
-; HASWELL-NEXT:    vaesdec (%rdi), %xmm0, %xmm0 # sched: [13:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_aesdec:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    aesdec %xmm1, %xmm0 # sched: [7:1.00]
-; BROADWELL-SSE-NEXT:    aesdec (%rdi), %xmm0 # sched: [12:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_aesdec:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vaesdec %xmm1, %xmm0, %xmm0 # sched: [7:1.00]
-; BROADWELL-NEXT:    vaesdec (%rdi), %xmm0, %xmm0 # sched: [12:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_aesdec:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    aesdec %xmm1, %xmm0 # sched: [4:1.00]
-; SKYLAKE-SSE-NEXT:    aesdec (%rdi), %xmm0 # sched: [10:1.00]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_aesdec:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vaesdec %xmm1, %xmm0, %xmm0 # sched: [4:1.00]
-; SKYLAKE-NEXT:    vaesdec (%rdi), %xmm0, %xmm0 # sched: [10:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_aesdec:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    aesdec %xmm1, %xmm0 # sched: [4:1.00]
-; SKX-SSE-NEXT:    aesdec (%rdi), %xmm0 # sched: [10:1.00]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_aesdec:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vaesdec %xmm1, %xmm0, %xmm0 # sched: [4:1.00]
-; SKX-NEXT:    vaesdec (%rdi), %xmm0, %xmm0 # sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_aesdec:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    aesdec %xmm1, %xmm0 # sched: [9:1.00]
-; BDVER2-SSE-NEXT:    aesdec (%rdi), %xmm0 # sched: [14:1.00]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_aesdec:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vaesdec %xmm1, %xmm0, %xmm0 # sched: [9:1.00]
-; BDVER2-NEXT:    vaesdec (%rdi), %xmm0, %xmm0 # sched: [14:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_aesdec:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    aesdec %xmm1, %xmm0 # sched: [3:1.00]
-; BTVER2-SSE-NEXT:    aesdec (%rdi), %xmm0 # sched: [8:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_aesdec:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vaesdec %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; BTVER2-NEXT:    vaesdec (%rdi), %xmm0, %xmm0 # sched: [8:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_aesdec:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    aesdec %xmm1, %xmm0 # sched: [4:0.50]
-; ZNVER1-SSE-NEXT:    aesdec (%rdi), %xmm0 # sched: [11:0.50]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_aesdec:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vaesdec %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; ZNVER1-NEXT:    vaesdec (%rdi), %xmm0, %xmm0 # sched: [11:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = load <2 x i64>, <2 x i64> *%a2, align 16
-  %2 = call <2 x i64> @llvm.x86.aesni.aesdec(<2 x i64> %a0, <2 x i64> %a1)
-  %3 = call <2 x i64> @llvm.x86.aesni.aesdec(<2 x i64> %2, <2 x i64> %1)
-  ret <2 x i64> %3
-}
-declare <2 x i64> @llvm.x86.aesni.aesdec(<2 x i64>, <2 x i64>)
-
-define <2 x i64> @test_aesdeclast(<2 x i64> %a0, <2 x i64> %a1, <2 x i64> *%a2) {
-; GENERIC-LABEL: test_aesdeclast:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    aesdeclast %xmm1, %xmm0 # sched: [7:1.00]
-; GENERIC-NEXT:    aesdeclast (%rdi), %xmm0 # sched: [13:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SLM-LABEL: test_aesdeclast:
-; SLM:       # %bb.0:
-; SLM-NEXT:    aesdeclast %xmm1, %xmm0 # sched: [8:5.00]
-; SLM-NEXT:    aesdeclast (%rdi), %xmm0 # sched: [8:5.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_aesdeclast:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    aesdeclast %xmm1, %xmm0 # sched: [7:1.00]
-; SANDY-SSE-NEXT:    aesdeclast (%rdi), %xmm0 # sched: [13:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_aesdeclast:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vaesdeclast %xmm1, %xmm0, %xmm0 # sched: [7:1.00]
-; SANDY-NEXT:    vaesdeclast (%rdi), %xmm0, %xmm0 # sched: [13:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_aesdeclast:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    aesdeclast %xmm1, %xmm0 # sched: [7:1.00]
-; HASWELL-SSE-NEXT:    aesdeclast (%rdi), %xmm0 # sched: [13:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_aesdeclast:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vaesdeclast %xmm1, %xmm0, %xmm0 # sched: [7:1.00]
-; HASWELL-NEXT:    vaesdeclast (%rdi), %xmm0, %xmm0 # sched: [13:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_aesdeclast:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    aesdeclast %xmm1, %xmm0 # sched: [7:1.00]
-; BROADWELL-SSE-NEXT:    aesdeclast (%rdi), %xmm0 # sched: [12:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_aesdeclast:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vaesdeclast %xmm1, %xmm0, %xmm0 # sched: [7:1.00]
-; BROADWELL-NEXT:    vaesdeclast (%rdi), %xmm0, %xmm0 # sched: [12:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_aesdeclast:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    aesdeclast %xmm1, %xmm0 # sched: [4:1.00]
-; SKYLAKE-SSE-NEXT:    aesdeclast (%rdi), %xmm0 # sched: [10:1.00]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_aesdeclast:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vaesdeclast %xmm1, %xmm0, %xmm0 # sched: [4:1.00]
-; SKYLAKE-NEXT:    vaesdeclast (%rdi), %xmm0, %xmm0 # sched: [10:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_aesdeclast:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    aesdeclast %xmm1, %xmm0 # sched: [4:1.00]
-; SKX-SSE-NEXT:    aesdeclast (%rdi), %xmm0 # sched: [10:1.00]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_aesdeclast:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vaesdeclast %xmm1, %xmm0, %xmm0 # sched: [4:1.00]
-; SKX-NEXT:    vaesdeclast (%rdi), %xmm0, %xmm0 # sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_aesdeclast:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    aesdeclast %xmm1, %xmm0 # sched: [9:1.00]
-; BDVER2-SSE-NEXT:    aesdeclast (%rdi), %xmm0 # sched: [14:1.00]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_aesdeclast:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vaesdeclast %xmm1, %xmm0, %xmm0 # sched: [9:1.00]
-; BDVER2-NEXT:    vaesdeclast (%rdi), %xmm0, %xmm0 # sched: [14:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_aesdeclast:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    aesdeclast %xmm1, %xmm0 # sched: [3:1.00]
-; BTVER2-SSE-NEXT:    aesdeclast (%rdi), %xmm0 # sched: [8:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_aesdeclast:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vaesdeclast %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; BTVER2-NEXT:    vaesdeclast (%rdi), %xmm0, %xmm0 # sched: [8:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_aesdeclast:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    aesdeclast %xmm1, %xmm0 # sched: [4:0.50]
-; ZNVER1-SSE-NEXT:    aesdeclast (%rdi), %xmm0 # sched: [11:0.50]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_aesdeclast:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vaesdeclast %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; ZNVER1-NEXT:    vaesdeclast (%rdi), %xmm0, %xmm0 # sched: [11:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = load <2 x i64>, <2 x i64> *%a2, align 16
-  %2 = call <2 x i64> @llvm.x86.aesni.aesdeclast(<2 x i64> %a0, <2 x i64> %a1)
-  %3 = call <2 x i64> @llvm.x86.aesni.aesdeclast(<2 x i64> %2, <2 x i64> %1)
-  ret <2 x i64> %3
-}
-declare <2 x i64> @llvm.x86.aesni.aesdeclast(<2 x i64>, <2 x i64>)
-
-define <2 x i64> @test_aesenc(<2 x i64> %a0, <2 x i64> %a1, <2 x i64> *%a2) {
-; GENERIC-LABEL: test_aesenc:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    aesenc %xmm1, %xmm0 # sched: [7:1.00]
-; GENERIC-NEXT:    aesenc (%rdi), %xmm0 # sched: [13:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SLM-LABEL: test_aesenc:
-; SLM:       # %bb.0:
-; SLM-NEXT:    aesenc %xmm1, %xmm0 # sched: [8:5.00]
-; SLM-NEXT:    aesenc (%rdi), %xmm0 # sched: [8:5.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_aesenc:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    aesenc %xmm1, %xmm0 # sched: [7:1.00]
-; SANDY-SSE-NEXT:    aesenc (%rdi), %xmm0 # sched: [13:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_aesenc:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vaesenc %xmm1, %xmm0, %xmm0 # sched: [7:1.00]
-; SANDY-NEXT:    vaesenc (%rdi), %xmm0, %xmm0 # sched: [13:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_aesenc:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    aesenc %xmm1, %xmm0 # sched: [7:1.00]
-; HASWELL-SSE-NEXT:    aesenc (%rdi), %xmm0 # sched: [13:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_aesenc:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vaesenc %xmm1, %xmm0, %xmm0 # sched: [7:1.00]
-; HASWELL-NEXT:    vaesenc (%rdi), %xmm0, %xmm0 # sched: [13:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_aesenc:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    aesenc %xmm1, %xmm0 # sched: [7:1.00]
-; BROADWELL-SSE-NEXT:    aesenc (%rdi), %xmm0 # sched: [12:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_aesenc:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vaesenc %xmm1, %xmm0, %xmm0 # sched: [7:1.00]
-; BROADWELL-NEXT:    vaesenc (%rdi), %xmm0, %xmm0 # sched: [12:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_aesenc:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    aesenc %xmm1, %xmm0 # sched: [4:1.00]
-; SKYLAKE-SSE-NEXT:    aesenc (%rdi), %xmm0 # sched: [10:1.00]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_aesenc:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vaesenc %xmm1, %xmm0, %xmm0 # sched: [4:1.00]
-; SKYLAKE-NEXT:    vaesenc (%rdi), %xmm0, %xmm0 # sched: [10:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_aesenc:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    aesenc %xmm1, %xmm0 # sched: [4:1.00]
-; SKX-SSE-NEXT:    aesenc (%rdi), %xmm0 # sched: [10:1.00]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_aesenc:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vaesenc %xmm1, %xmm0, %xmm0 # sched: [4:1.00]
-; SKX-NEXT:    vaesenc (%rdi), %xmm0, %xmm0 # sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_aesenc:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    aesenc %xmm1, %xmm0 # sched: [9:1.00]
-; BDVER2-SSE-NEXT:    aesenc (%rdi), %xmm0 # sched: [14:1.00]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_aesenc:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vaesenc %xmm1, %xmm0, %xmm0 # sched: [9:1.00]
-; BDVER2-NEXT:    vaesenc (%rdi), %xmm0, %xmm0 # sched: [14:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_aesenc:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    aesenc %xmm1, %xmm0 # sched: [3:1.00]
-; BTVER2-SSE-NEXT:    aesenc (%rdi), %xmm0 # sched: [8:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_aesenc:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vaesenc %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; BTVER2-NEXT:    vaesenc (%rdi), %xmm0, %xmm0 # sched: [8:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_aesenc:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    aesenc %xmm1, %xmm0 # sched: [4:0.50]
-; ZNVER1-SSE-NEXT:    aesenc (%rdi), %xmm0 # sched: [11:0.50]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_aesenc:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vaesenc %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; ZNVER1-NEXT:    vaesenc (%rdi), %xmm0, %xmm0 # sched: [11:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = load <2 x i64>, <2 x i64> *%a2, align 16
-  %2 = call <2 x i64> @llvm.x86.aesni.aesenc(<2 x i64> %a0, <2 x i64> %a1)
-  %3 = call <2 x i64> @llvm.x86.aesni.aesenc(<2 x i64> %2, <2 x i64> %1)
-  ret <2 x i64> %3
-}
-declare <2 x i64> @llvm.x86.aesni.aesenc(<2 x i64>, <2 x i64>)
-
-define <2 x i64> @test_aesenclast(<2 x i64> %a0, <2 x i64> %a1, <2 x i64> *%a2) {
-; GENERIC-LABEL: test_aesenclast:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    aesenclast %xmm1, %xmm0 # sched: [7:1.00]
-; GENERIC-NEXT:    aesenclast (%rdi), %xmm0 # sched: [13:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SLM-LABEL: test_aesenclast:
-; SLM:       # %bb.0:
-; SLM-NEXT:    aesenclast %xmm1, %xmm0 # sched: [8:5.00]
-; SLM-NEXT:    aesenclast (%rdi), %xmm0 # sched: [8:5.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_aesenclast:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    aesenclast %xmm1, %xmm0 # sched: [7:1.00]
-; SANDY-SSE-NEXT:    aesenclast (%rdi), %xmm0 # sched: [13:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_aesenclast:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vaesenclast %xmm1, %xmm0, %xmm0 # sched: [7:1.00]
-; SANDY-NEXT:    vaesenclast (%rdi), %xmm0, %xmm0 # sched: [13:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_aesenclast:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    aesenclast %xmm1, %xmm0 # sched: [7:1.00]
-; HASWELL-SSE-NEXT:    aesenclast (%rdi), %xmm0 # sched: [13:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_aesenclast:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vaesenclast %xmm1, %xmm0, %xmm0 # sched: [7:1.00]
-; HASWELL-NEXT:    vaesenclast (%rdi), %xmm0, %xmm0 # sched: [13:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_aesenclast:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    aesenclast %xmm1, %xmm0 # sched: [7:1.00]
-; BROADWELL-SSE-NEXT:    aesenclast (%rdi), %xmm0 # sched: [12:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_aesenclast:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vaesenclast %xmm1, %xmm0, %xmm0 # sched: [7:1.00]
-; BROADWELL-NEXT:    vaesenclast (%rdi), %xmm0, %xmm0 # sched: [12:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_aesenclast:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    aesenclast %xmm1, %xmm0 # sched: [4:1.00]
-; SKYLAKE-SSE-NEXT:    aesenclast (%rdi), %xmm0 # sched: [10:1.00]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_aesenclast:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vaesenclast %xmm1, %xmm0, %xmm0 # sched: [4:1.00]
-; SKYLAKE-NEXT:    vaesenclast (%rdi), %xmm0, %xmm0 # sched: [10:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_aesenclast:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    aesenclast %xmm1, %xmm0 # sched: [4:1.00]
-; SKX-SSE-NEXT:    aesenclast (%rdi), %xmm0 # sched: [10:1.00]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_aesenclast:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vaesenclast %xmm1, %xmm0, %xmm0 # sched: [4:1.00]
-; SKX-NEXT:    vaesenclast (%rdi), %xmm0, %xmm0 # sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_aesenclast:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    aesenclast %xmm1, %xmm0 # sched: [9:1.00]
-; BDVER2-SSE-NEXT:    aesenclast (%rdi), %xmm0 # sched: [14:1.00]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_aesenclast:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vaesenclast %xmm1, %xmm0, %xmm0 # sched: [9:1.00]
-; BDVER2-NEXT:    vaesenclast (%rdi), %xmm0, %xmm0 # sched: [14:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_aesenclast:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    aesenclast %xmm1, %xmm0 # sched: [3:1.00]
-; BTVER2-SSE-NEXT:    aesenclast (%rdi), %xmm0 # sched: [8:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_aesenclast:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vaesenclast %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; BTVER2-NEXT:    vaesenclast (%rdi), %xmm0, %xmm0 # sched: [8:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_aesenclast:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    aesenclast %xmm1, %xmm0 # sched: [4:0.50]
-; ZNVER1-SSE-NEXT:    aesenclast (%rdi), %xmm0 # sched: [11:0.50]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_aesenclast:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vaesenclast %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; ZNVER1-NEXT:    vaesenclast (%rdi), %xmm0, %xmm0 # sched: [11:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = load <2 x i64>, <2 x i64> *%a2, align 16
-  %2 = call <2 x i64> @llvm.x86.aesni.aesenclast(<2 x i64> %a0, <2 x i64> %a1)
-  %3 = call <2 x i64> @llvm.x86.aesni.aesenclast(<2 x i64> %2, <2 x i64> %1)
-  ret <2 x i64> %3
-}
-declare <2 x i64> @llvm.x86.aesni.aesenclast(<2 x i64>, <2 x i64>)
-
-define <2 x i64> @test_aesimc(<2 x i64> %a0, <2 x i64> *%a1) {
-; GENERIC-LABEL: test_aesimc:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    aesimc %xmm0, %xmm1 # sched: [12:2.00]
-; GENERIC-NEXT:    aesimc (%rdi), %xmm0 # sched: [18:2.00]
-; GENERIC-NEXT:    por %xmm1, %xmm0 # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SLM-LABEL: test_aesimc:
-; SLM:       # %bb.0:
-; SLM-NEXT:    aesimc %xmm0, %xmm1 # sched: [8:5.00]
-; SLM-NEXT:    aesimc (%rdi), %xmm0 # sched: [8:5.00]
-; SLM-NEXT:    por %xmm1, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_aesimc:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    aesimc %xmm0, %xmm1 # sched: [12:2.00]
-; SANDY-SSE-NEXT:    aesimc (%rdi), %xmm0 # sched: [18:2.00]
-; SANDY-SSE-NEXT:    por %xmm1, %xmm0 # sched: [1:0.33]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_aesimc:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vaesimc %xmm0, %xmm0 # sched: [12:2.00]
-; SANDY-NEXT:    vaesimc (%rdi), %xmm1 # sched: [18:2.00]
-; SANDY-NEXT:    vpor %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_aesimc:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    aesimc %xmm0, %xmm1 # sched: [14:2.00]
-; HASWELL-SSE-NEXT:    aesimc (%rdi), %xmm0 # sched: [20:2.00]
-; HASWELL-SSE-NEXT:    por %xmm1, %xmm0 # sched: [1:0.33]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_aesimc:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vaesimc %xmm0, %xmm0 # sched: [14:2.00]
-; HASWELL-NEXT:    vaesimc (%rdi), %xmm1 # sched: [20:2.00]
-; HASWELL-NEXT:    vpor %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_aesimc:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    aesimc (%rdi), %xmm1 # sched: [19:2.00]
-; BROADWELL-SSE-NEXT:    aesimc %xmm0, %xmm0 # sched: [14:2.00]
-; BROADWELL-SSE-NEXT:    por %xmm0, %xmm1 # sched: [1:0.33]
-; BROADWELL-SSE-NEXT:    movdqa %xmm1, %xmm0 # sched: [1:0.33]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_aesimc:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vaesimc (%rdi), %xmm1 # sched: [19:2.00]
-; BROADWELL-NEXT:    vaesimc %xmm0, %xmm0 # sched: [14:2.00]
-; BROADWELL-NEXT:    vpor %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_aesimc:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    aesimc %xmm0, %xmm1 # sched: [8:2.00]
-; SKYLAKE-SSE-NEXT:    aesimc (%rdi), %xmm0 # sched: [14:2.00]
-; SKYLAKE-SSE-NEXT:    por %xmm1, %xmm0 # sched: [1:0.33]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_aesimc:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vaesimc %xmm0, %xmm0 # sched: [8:2.00]
-; SKYLAKE-NEXT:    vaesimc (%rdi), %xmm1 # sched: [14:2.00]
-; SKYLAKE-NEXT:    vpor %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_aesimc:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    aesimc %xmm0, %xmm1 # sched: [8:2.00]
-; SKX-SSE-NEXT:    aesimc (%rdi), %xmm0 # sched: [14:2.00]
-; SKX-SSE-NEXT:    por %xmm1, %xmm0 # sched: [1:0.33]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_aesimc:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vaesimc %xmm0, %xmm0 # sched: [8:2.00]
-; SKX-NEXT:    vaesimc (%rdi), %xmm1 # sched: [14:2.00]
-; SKX-NEXT:    vpor %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_aesimc:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    aesimc %xmm0, %xmm1 # sched: [5:1.00]
-; BDVER2-SSE-NEXT:    aesimc (%rdi), %xmm0 # sched: [10:1.00]
-; BDVER2-SSE-NEXT:    por %xmm1, %xmm0 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_aesimc:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vaesimc (%rdi), %xmm1 # sched: [10:1.00]
-; BDVER2-NEXT:    vaesimc %xmm0, %xmm0 # sched: [5:1.00]
-; BDVER2-NEXT:    vpor %xmm1, %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_aesimc:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    aesimc %xmm0, %xmm1 # sched: [2:1.00]
-; BTVER2-SSE-NEXT:    aesimc (%rdi), %xmm0 # sched: [7:1.00]
-; BTVER2-SSE-NEXT:    por %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_aesimc:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vaesimc (%rdi), %xmm1 # sched: [7:1.00]
-; BTVER2-NEXT:    vaesimc %xmm0, %xmm0 # sched: [2:1.00]
-; BTVER2-NEXT:    vpor %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_aesimc:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    aesimc %xmm0, %xmm1 # sched: [4:0.50]
-; ZNVER1-SSE-NEXT:    aesimc (%rdi), %xmm0 # sched: [11:0.50]
-; ZNVER1-SSE-NEXT:    por %xmm1, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_aesimc:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vaesimc (%rdi), %xmm1 # sched: [11:0.50]
-; ZNVER1-NEXT:    vaesimc %xmm0, %xmm0 # sched: [4:0.50]
-; ZNVER1-NEXT:    vpor %xmm1, %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = load <2 x i64>, <2 x i64> *%a1, align 16
-  %2 = call <2 x i64> @llvm.x86.aesni.aesimc(<2 x i64> %a0)
-  %3 = call <2 x i64> @llvm.x86.aesni.aesimc(<2 x i64> %1)
-  %4 = or <2 x i64> %2, %3
-  ret <2 x i64> %4
-}
-declare <2 x i64> @llvm.x86.aesni.aesimc(<2 x i64>)
-
-define <2 x i64> @test_aeskeygenassist(<2 x i64> %a0, <2 x i64> *%a1) {
-; GENERIC-LABEL: test_aeskeygenassist:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    aeskeygenassist $7, %xmm0, %xmm1 # sched: [8:3.67]
-; GENERIC-NEXT:    aeskeygenassist $7, (%rdi), %xmm0 # sched: [8:3.33]
-; GENERIC-NEXT:    por %xmm1, %xmm0 # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SLM-LABEL: test_aeskeygenassist:
-; SLM:       # %bb.0:
-; SLM-NEXT:    aeskeygenassist $7, %xmm0, %xmm1 # sched: [8:5.00]
-; SLM-NEXT:    aeskeygenassist $7, (%rdi), %xmm0 # sched: [8:5.00]
-; SLM-NEXT:    por %xmm1, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_aeskeygenassist:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    aeskeygenassist $7, %xmm0, %xmm1 # sched: [8:3.67]
-; SANDY-SSE-NEXT:    aeskeygenassist $7, (%rdi), %xmm0 # sched: [8:3.33]
-; SANDY-SSE-NEXT:    por %xmm1, %xmm0 # sched: [1:0.33]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_aeskeygenassist:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vaeskeygenassist $7, %xmm0, %xmm0 # sched: [8:3.67]
-; SANDY-NEXT:    vaeskeygenassist $7, (%rdi), %xmm1 # sched: [8:3.33]
-; SANDY-NEXT:    vpor %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_aeskeygenassist:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    aeskeygenassist $7, %xmm0, %xmm1 # sched: [29:7.00]
-; HASWELL-SSE-NEXT:    aeskeygenassist $7, (%rdi), %xmm0 # sched: [34:7.00]
-; HASWELL-SSE-NEXT:    por %xmm1, %xmm0 # sched: [1:0.33]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_aeskeygenassist:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vaeskeygenassist $7, %xmm0, %xmm0 # sched: [29:7.00]
-; HASWELL-NEXT:    vaeskeygenassist $7, (%rdi), %xmm1 # sched: [34:7.00]
-; HASWELL-NEXT:    vpor %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_aeskeygenassist:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    aeskeygenassist $7, %xmm0, %xmm1 # sched: [29:7.00]
-; BROADWELL-SSE-NEXT:    aeskeygenassist $7, (%rdi), %xmm0 # sched: [33:7.00]
-; BROADWELL-SSE-NEXT:    por %xmm1, %xmm0 # sched: [1:0.33]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_aeskeygenassist:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vaeskeygenassist $7, %xmm0, %xmm0 # sched: [29:7.00]
-; BROADWELL-NEXT:    vaeskeygenassist $7, (%rdi), %xmm1 # sched: [33:7.00]
-; BROADWELL-NEXT:    vpor %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_aeskeygenassist:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    aeskeygenassist $7, %xmm0, %xmm1 # sched: [20:6.00]
-; SKYLAKE-SSE-NEXT:    aeskeygenassist $7, (%rdi), %xmm0 # sched: [25:6.00]
-; SKYLAKE-SSE-NEXT:    por %xmm1, %xmm0 # sched: [1:0.33]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_aeskeygenassist:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vaeskeygenassist $7, %xmm0, %xmm0 # sched: [20:6.00]
-; SKYLAKE-NEXT:    vaeskeygenassist $7, (%rdi), %xmm1 # sched: [25:6.00]
-; SKYLAKE-NEXT:    vpor %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_aeskeygenassist:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    aeskeygenassist $7, %xmm0, %xmm1 # sched: [20:6.00]
-; SKX-SSE-NEXT:    aeskeygenassist $7, (%rdi), %xmm0 # sched: [25:6.00]
-; SKX-SSE-NEXT:    por %xmm1, %xmm0 # sched: [1:0.33]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_aeskeygenassist:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vaeskeygenassist $7, %xmm0, %xmm0 # sched: [20:6.00]
-; SKX-NEXT:    vaeskeygenassist $7, (%rdi), %xmm1 # sched: [25:6.00]
-; SKX-NEXT:    vpor %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_aeskeygenassist:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    aeskeygenassist $7, %xmm0, %xmm1 # sched: [5:1.00]
-; BDVER2-SSE-NEXT:    aeskeygenassist $7, (%rdi), %xmm0 # sched: [10:1.00]
-; BDVER2-SSE-NEXT:    por %xmm1, %xmm0 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_aeskeygenassist:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vaeskeygenassist $7, (%rdi), %xmm1 # sched: [10:1.00]
-; BDVER2-NEXT:    vaeskeygenassist $7, %xmm0, %xmm0 # sched: [5:1.00]
-; BDVER2-NEXT:    vpor %xmm1, %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_aeskeygenassist:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    aeskeygenassist $7, %xmm0, %xmm1 # sched: [2:1.00]
-; BTVER2-SSE-NEXT:    aeskeygenassist $7, (%rdi), %xmm0 # sched: [7:1.00]
-; BTVER2-SSE-NEXT:    por %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_aeskeygenassist:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vaeskeygenassist $7, (%rdi), %xmm1 # sched: [7:1.00]
-; BTVER2-NEXT:    vaeskeygenassist $7, %xmm0, %xmm0 # sched: [2:1.00]
-; BTVER2-NEXT:    vpor %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_aeskeygenassist:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    aeskeygenassist $7, %xmm0, %xmm1 # sched: [4:0.50]
-; ZNVER1-SSE-NEXT:    aeskeygenassist $7, (%rdi), %xmm0 # sched: [11:0.50]
-; ZNVER1-SSE-NEXT:    por %xmm1, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_aeskeygenassist:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vaeskeygenassist $7, (%rdi), %xmm1 # sched: [11:0.50]
-; ZNVER1-NEXT:    vaeskeygenassist $7, %xmm0, %xmm0 # sched: [4:0.50]
-; ZNVER1-NEXT:    vpor %xmm1, %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = load <2 x i64>, <2 x i64> *%a1, align 16
-  %2 = call <2 x i64> @llvm.x86.aesni.aeskeygenassist(<2 x i64> %a0, i8 7)
-  %3 = call <2 x i64> @llvm.x86.aesni.aeskeygenassist(<2 x i64> %1, i8 7)
-  %4 = or <2 x i64> %2, %3
-  ret <2 x i64> %4
-}
-declare <2 x i64> @llvm.x86.aesni.aeskeygenassist(<2 x i64>, i8)
diff --git a/test/CodeGen/X86/and-load-fold.ll b/test/CodeGen/X86/and-load-fold.ll
index 4c49df1c5bc15cbf6f08f7088951d0d13befccca..367ef2a344a61bbd8dd2ebceb468440390ea23b9 100644
--- a/test/CodeGen/X86/and-load-fold.ll
+++ b/test/CodeGen/X86/and-load-fold.ll
@@ -7,10 +7,8 @@ define i8 @foo(<4 x i8>* %V) {
 ; CHECK-LABEL: foo:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; CHECK-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; CHECK-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
-; CHECK-NEXT:    pand {{.*}}(%rip), %xmm0
-; CHECK-NEXT:    pextrw $4, %xmm0, %eax
+; CHECK-NEXT:    pextrw $1, %xmm0, %eax
+; CHECK-NEXT:    andb $95, %al
 ; CHECK-NEXT:    # kill: def $al killed $al killed $eax
 ; CHECK-NEXT:    retq
   %Vp = bitcast <4 x i8>* %V to <3 x i8>*
diff --git a/test/CodeGen/X86/and-su.ll b/test/CodeGen/X86/and-su.ll
index 55bfa8def44f2e4af62504eeb50fc12acde36191..de384368bfca5fe71fdb1c65eba409ea82466064 100644
--- a/test/CodeGen/X86/and-su.ll
+++ b/test/CodeGen/X86/and-su.ll
@@ -49,7 +49,7 @@ define fastcc double @bar(i32 %hash, double %x, double %y) nounwind {
 ; CHECK-NEXT:    fchs
 ; CHECK-NEXT:    fxch %st(1)
 ; CHECK-NEXT:  .LBB1_5: # %bb16
-; CHECK-NEXT:    faddp %st(1)
+; CHECK-NEXT:    faddp %st, %st(1)
 ; CHECK-NEXT:    movl %ebp, %esp
 ; CHECK-NEXT:    popl %ebp
 ; CHECK-NEXT:    retl
diff --git a/test/CodeGen/X86/anyregcc.ll b/test/CodeGen/X86/anyregcc.ll
index b75774ab12c077a2a473f7021d311dc7d9dd777b..7cf6d71b78cfb2a0b05074cee66e5dbc6da89540 100644
--- a/test/CodeGen/X86/anyregcc.ll
+++ b/test/CodeGen/X86/anyregcc.ll
@@ -1,6 +1,6 @@
-; RUN: llc < %s -mtriple=x86_64-apple-darwin                  -disable-fp-elim | FileCheck %s
-; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=corei7     -disable-fp-elim | FileCheck --check-prefix=SSE %s
-; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=corei7-avx -disable-fp-elim | FileCheck --check-prefix=AVX %s
+; RUN: llc < %s -mtriple=x86_64-apple-darwin                  -frame-pointer=all | FileCheck %s
+; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=corei7     -frame-pointer=all | FileCheck --check-prefix=SSE %s
+; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=corei7-avx -frame-pointer=all | FileCheck --check-prefix=AVX %s
 
 
 ; Stackmap Header: no constants - 6 callsites
diff --git a/test/CodeGen/X86/atom-sched.ll b/test/CodeGen/X86/atom-sched.ll
index 19249ab684ac7c24caf5034ab713c03188feb156..3600a905b62b6cb125d647f01ad2e30694aed52d 100644
--- a/test/CodeGen/X86/atom-sched.ll
+++ b/test/CodeGen/X86/atom-sched.ll
@@ -1,7 +1,7 @@
-; RUN: llc <%s -O2 -mcpu=atom -mtriple=i686-- -relocation-model=static | FileCheck -check-prefix=atom %s
-; RUN: llc <%s -O2 -mcpu=slm -mtriple=i686-- -relocation-model=static | FileCheck -check-prefix=slm %s
-; RUN: llc <%s -O2 -mcpu=goldmont -mtriple=i686-- -relocation-model=static | FileCheck -check-prefix=slm %s
-; RUN: llc <%s -O2 -mcpu=core2 -mtriple=i686-- -relocation-model=static | FileCheck %s
+; RUN: llc < %s -O2 -mcpu=atom -mtriple=i686-- -relocation-model=static | FileCheck -check-prefix=atom %s
+; RUN: llc < %s -O2 -mcpu=slm -mtriple=i686-- -relocation-model=static | FileCheck -check-prefix=slm %s
+; RUN: llc < %s -O2 -mcpu=goldmont -mtriple=i686-- -relocation-model=static | FileCheck -check-prefix=slm %s
+; RUN: llc < %s -O2 -mcpu=core2 -mtriple=i686-- -relocation-model=static | FileCheck %s
 ;
 
 @a = common global i32 0, align 4
diff --git a/test/CodeGen/X86/atomic-add.ll b/test/CodeGen/X86/atomic-add.ll
new file mode 100644
index 0000000000000000000000000000000000000000..f438f05ea2f002f4cc5a067d8a449af0d68ac1df
--- /dev/null
+++ b/test/CodeGen/X86/atomic-add.ll
@@ -0,0 +1,375 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-- -verify-machineinstrs | FileCheck %s
+; RUN: llc < %s -mtriple=x86_64-- -mattr=slow-incdec -verify-machineinstrs | FileCheck %s --check-prefix SLOW_INC
+
+; rdar://7103704
+
+define void @sub1(i32* nocapture %p, i32 %v) nounwind ssp {
+; CHECK-LABEL: sub1:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lock subl %esi, (%rdi)
+; CHECK-NEXT:    retq
+;
+; SLOW_INC-LABEL: sub1:
+; SLOW_INC:       # %bb.0: # %entry
+; SLOW_INC-NEXT:    lock subl %esi, (%rdi)
+; SLOW_INC-NEXT:    retq
+entry:
+  %0 = atomicrmw sub i32* %p, i32 %v monotonic
+  ret void
+}
+
+define void @inc4(i64* nocapture %p) nounwind ssp {
+; CHECK-LABEL: inc4:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lock incq (%rdi)
+; CHECK-NEXT:    retq
+;
+; SLOW_INC-LABEL: inc4:
+; SLOW_INC:       # %bb.0: # %entry
+; SLOW_INC-NEXT:    lock addq $1, (%rdi)
+; SLOW_INC-NEXT:    retq
+entry:
+  %0 = atomicrmw add i64* %p, i64 1 monotonic
+  ret void
+}
+
+define void @add8(i64* nocapture %p) nounwind ssp {
+; CHECK-LABEL: add8:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lock addq $2, (%rdi)
+; CHECK-NEXT:    retq
+;
+; SLOW_INC-LABEL: add8:
+; SLOW_INC:       # %bb.0: # %entry
+; SLOW_INC-NEXT:    lock addq $2, (%rdi)
+; SLOW_INC-NEXT:    retq
+entry:
+  %0 = atomicrmw add i64* %p, i64 2 monotonic
+  ret void
+}
+
+define void @add4(i64* nocapture %p, i32 %v) nounwind ssp {
+; CHECK-LABEL: add4:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movslq %esi, %rax
+; CHECK-NEXT:    lock addq %rax, (%rdi)
+; CHECK-NEXT:    retq
+;
+; SLOW_INC-LABEL: add4:
+; SLOW_INC:       # %bb.0: # %entry
+; SLOW_INC-NEXT:    movslq %esi, %rax
+; SLOW_INC-NEXT:    lock addq %rax, (%rdi)
+; SLOW_INC-NEXT:    retq
+entry:
+  %0 = sext i32 %v to i64		; <i64> [#uses=1]
+  %1 = atomicrmw add i64* %p, i64 %0 monotonic
+  ret void
+}
+
+define void @inc3(i8* nocapture %p) nounwind ssp {
+; CHECK-LABEL: inc3:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lock incb (%rdi)
+; CHECK-NEXT:    retq
+;
+; SLOW_INC-LABEL: inc3:
+; SLOW_INC:       # %bb.0: # %entry
+; SLOW_INC-NEXT:    lock addb $1, (%rdi)
+; SLOW_INC-NEXT:    retq
+entry:
+  %0 = atomicrmw add i8* %p, i8 1 monotonic
+  ret void
+}
+
+define void @add7(i8* nocapture %p) nounwind ssp {
+; CHECK-LABEL: add7:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lock addb $2, (%rdi)
+; CHECK-NEXT:    retq
+;
+; SLOW_INC-LABEL: add7:
+; SLOW_INC:       # %bb.0: # %entry
+; SLOW_INC-NEXT:    lock addb $2, (%rdi)
+; SLOW_INC-NEXT:    retq
+entry:
+  %0 = atomicrmw add i8* %p, i8 2 monotonic
+  ret void
+}
+
+define void @add3(i8* nocapture %p, i32 %v) nounwind ssp {
+; CHECK-LABEL: add3:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lock addb %sil, (%rdi)
+; CHECK-NEXT:    retq
+;
+; SLOW_INC-LABEL: add3:
+; SLOW_INC:       # %bb.0: # %entry
+; SLOW_INC-NEXT:    lock addb %sil, (%rdi)
+; SLOW_INC-NEXT:    retq
+entry:
+  %0 = trunc i32 %v to i8		; <i8> [#uses=1]
+  %1 = atomicrmw add i8* %p, i8 %0 monotonic
+  ret void
+}
+
+define void @inc2(i16* nocapture %p) nounwind ssp {
+; CHECK-LABEL: inc2:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lock incw (%rdi)
+; CHECK-NEXT:    retq
+;
+; SLOW_INC-LABEL: inc2:
+; SLOW_INC:       # %bb.0: # %entry
+; SLOW_INC-NEXT:    lock addw $1, (%rdi)
+; SLOW_INC-NEXT:    retq
+entry:
+  %0 = atomicrmw add i16* %p, i16 1 monotonic
+  ret void
+}
+
+define void @add6(i16* nocapture %p) nounwind ssp {
+; CHECK-LABEL: add6:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lock addw $2, (%rdi)
+; CHECK-NEXT:    retq
+;
+; SLOW_INC-LABEL: add6:
+; SLOW_INC:       # %bb.0: # %entry
+; SLOW_INC-NEXT:    lock addw $2, (%rdi)
+; SLOW_INC-NEXT:    retq
+entry:
+  %0 = atomicrmw add i16* %p, i16 2 monotonic
+  ret void
+}
+
+define void @add2(i16* nocapture %p, i32 %v) nounwind ssp {
+; CHECK-LABEL: add2:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lock addw %si, (%rdi)
+; CHECK-NEXT:    retq
+;
+; SLOW_INC-LABEL: add2:
+; SLOW_INC:       # %bb.0: # %entry
+; SLOW_INC-NEXT:    lock addw %si, (%rdi)
+; SLOW_INC-NEXT:    retq
+entry:
+	%0 = trunc i32 %v to i16		; <i16> [#uses=1]
+  %1 = atomicrmw add i16* %p, i16 %0 monotonic
+  ret void
+}
+
+define void @inc1(i32* nocapture %p) nounwind ssp {
+; CHECK-LABEL: inc1:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lock incl (%rdi)
+; CHECK-NEXT:    retq
+;
+; SLOW_INC-LABEL: inc1:
+; SLOW_INC:       # %bb.0: # %entry
+; SLOW_INC-NEXT:    lock addl $1, (%rdi)
+; SLOW_INC-NEXT:    retq
+entry:
+  %0 = atomicrmw add i32* %p, i32 1 monotonic
+  ret void
+}
+
+define void @add5(i32* nocapture %p) nounwind ssp {
+; CHECK-LABEL: add5:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lock addl $2, (%rdi)
+; CHECK-NEXT:    retq
+;
+; SLOW_INC-LABEL: add5:
+; SLOW_INC:       # %bb.0: # %entry
+; SLOW_INC-NEXT:    lock addl $2, (%rdi)
+; SLOW_INC-NEXT:    retq
+entry:
+  %0 = atomicrmw add i32* %p, i32 2 monotonic
+  ret void
+}
+
+define void @add1(i32* nocapture %p, i32 %v) nounwind ssp {
+; CHECK-LABEL: add1:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lock addl %esi, (%rdi)
+; CHECK-NEXT:    retq
+;
+; SLOW_INC-LABEL: add1:
+; SLOW_INC:       # %bb.0: # %entry
+; SLOW_INC-NEXT:    lock addl %esi, (%rdi)
+; SLOW_INC-NEXT:    retq
+entry:
+  %0 = atomicrmw add i32* %p, i32 %v monotonic
+  ret void
+}
+
+define void @dec4(i64* nocapture %p) nounwind ssp {
+; CHECK-LABEL: dec4:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lock decq (%rdi)
+; CHECK-NEXT:    retq
+;
+; SLOW_INC-LABEL: dec4:
+; SLOW_INC:       # %bb.0: # %entry
+; SLOW_INC-NEXT:    lock subq $1, (%rdi)
+; SLOW_INC-NEXT:    retq
+entry:
+  %0 = atomicrmw sub i64* %p, i64 1 monotonic
+  ret void
+}
+
+define void @sub8(i64* nocapture %p) nounwind ssp {
+; CHECK-LABEL: sub8:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lock subq $2, (%rdi)
+; CHECK-NEXT:    retq
+;
+; SLOW_INC-LABEL: sub8:
+; SLOW_INC:       # %bb.0: # %entry
+; SLOW_INC-NEXT:    lock subq $2, (%rdi)
+; SLOW_INC-NEXT:    retq
+entry:
+  %0 = atomicrmw sub i64* %p, i64 2 monotonic
+  ret void
+}
+
+define void @sub4(i64* nocapture %p, i32 %v) nounwind ssp {
+; CHECK-LABEL: sub4:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movslq %esi, %rax
+; CHECK-NEXT:    lock subq %rax, (%rdi)
+; CHECK-NEXT:    retq
+;
+; SLOW_INC-LABEL: sub4:
+; SLOW_INC:       # %bb.0: # %entry
+; SLOW_INC-NEXT:    movslq %esi, %rax
+; SLOW_INC-NEXT:    lock subq %rax, (%rdi)
+; SLOW_INC-NEXT:    retq
+entry:
+	%0 = sext i32 %v to i64		; <i64> [#uses=1]
+  %1 = atomicrmw sub i64* %p, i64 %0 monotonic
+  ret void
+}
+
+define void @dec3(i8* nocapture %p) nounwind ssp {
+; CHECK-LABEL: dec3:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lock decb (%rdi)
+; CHECK-NEXT:    retq
+;
+; SLOW_INC-LABEL: dec3:
+; SLOW_INC:       # %bb.0: # %entry
+; SLOW_INC-NEXT:    lock subb $1, (%rdi)
+; SLOW_INC-NEXT:    retq
+entry:
+  %0 = atomicrmw sub i8* %p, i8 1 monotonic
+  ret void
+}
+
+define void @sub7(i8* nocapture %p) nounwind ssp {
+; CHECK-LABEL: sub7:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lock subb $2, (%rdi)
+; CHECK-NEXT:    retq
+;
+; SLOW_INC-LABEL: sub7:
+; SLOW_INC:       # %bb.0: # %entry
+; SLOW_INC-NEXT:    lock subb $2, (%rdi)
+; SLOW_INC-NEXT:    retq
+entry:
+  %0 = atomicrmw sub i8* %p, i8 2 monotonic
+  ret void
+}
+
+define void @sub3(i8* nocapture %p, i32 %v) nounwind ssp {
+; CHECK-LABEL: sub3:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lock subb %sil, (%rdi)
+; CHECK-NEXT:    retq
+;
+; SLOW_INC-LABEL: sub3:
+; SLOW_INC:       # %bb.0: # %entry
+; SLOW_INC-NEXT:    lock subb %sil, (%rdi)
+; SLOW_INC-NEXT:    retq
+entry:
+	%0 = trunc i32 %v to i8		; <i8> [#uses=1]
+  %1 = atomicrmw sub i8* %p, i8 %0 monotonic
+  ret void
+}
+
+define void @dec2(i16* nocapture %p) nounwind ssp {
+; CHECK-LABEL: dec2:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lock decw (%rdi)
+; CHECK-NEXT:    retq
+;
+; SLOW_INC-LABEL: dec2:
+; SLOW_INC:       # %bb.0: # %entry
+; SLOW_INC-NEXT:    lock subw $1, (%rdi)
+; SLOW_INC-NEXT:    retq
+entry:
+  %0 = atomicrmw sub i16* %p, i16 1 monotonic
+  ret void
+}
+
+define void @sub6(i16* nocapture %p) nounwind ssp {
+; CHECK-LABEL: sub6:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lock subw $2, (%rdi)
+; CHECK-NEXT:    retq
+;
+; SLOW_INC-LABEL: sub6:
+; SLOW_INC:       # %bb.0: # %entry
+; SLOW_INC-NEXT:    lock subw $2, (%rdi)
+; SLOW_INC-NEXT:    retq
+entry:
+  %0 = atomicrmw sub i16* %p, i16 2 monotonic
+  ret void
+}
+
+define void @sub2(i16* nocapture %p, i32 %v) nounwind ssp {
+; CHECK-LABEL: sub2:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lock subw %si, (%rdi)
+; CHECK-NEXT:    retq
+;
+; SLOW_INC-LABEL: sub2:
+; SLOW_INC:       # %bb.0: # %entry
+; SLOW_INC-NEXT:    lock subw %si, (%rdi)
+; SLOW_INC-NEXT:    retq
+entry:
+	%0 = trunc i32 %v to i16		; <i16> [#uses=1]
+  %1 = atomicrmw sub i16* %p, i16 %0 monotonic
+  ret void
+}
+
+define void @dec1(i32* nocapture %p) nounwind ssp {
+; CHECK-LABEL: dec1:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lock decl (%rdi)
+; CHECK-NEXT:    retq
+;
+; SLOW_INC-LABEL: dec1:
+; SLOW_INC:       # %bb.0: # %entry
+; SLOW_INC-NEXT:    lock subl $1, (%rdi)
+; SLOW_INC-NEXT:    retq
+entry:
+  %0 = atomicrmw sub i32* %p, i32 1 monotonic
+  ret void
+}
+
+define void @sub5(i32* nocapture %p) nounwind ssp {
+; CHECK-LABEL: sub5:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lock subl $2, (%rdi)
+; CHECK-NEXT:    retq
+;
+; SLOW_INC-LABEL: sub5:
+; SLOW_INC:       # %bb.0: # %entry
+; SLOW_INC-NEXT:    lock subl $2, (%rdi)
+; SLOW_INC-NEXT:    retq
+entry:
+  %0 = atomicrmw sub i32* %p, i32 2 monotonic
+  ret void
+}
diff --git a/test/CodeGen/X86/atomic_idempotent.ll b/test/CodeGen/X86/atomic-idempotent.ll
similarity index 100%
rename from test/CodeGen/X86/atomic_idempotent.ll
rename to test/CodeGen/X86/atomic-idempotent.ll
diff --git a/test/CodeGen/X86/atomic_mi.ll b/test/CodeGen/X86/atomic-mi.ll
similarity index 100%
rename from test/CodeGen/X86/atomic_mi.ll
rename to test/CodeGen/X86/atomic-mi.ll
diff --git a/test/CodeGen/X86/atomic-monotonic.ll b/test/CodeGen/X86/atomic-monotonic.ll
new file mode 100644
index 0000000000000000000000000000000000000000..a66d79053a10ff29d8591b431664042497e7991c
--- /dev/null
+++ b/test/CodeGen/X86/atomic-monotonic.ll
@@ -0,0 +1,118 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -O0 < %s -mtriple=x86_64-linux-generic -verify-machineinstrs -mattr=sse2 | FileCheck --check-prefix=CHECK-O0 %s
+; RUN: llc -O3 < %s -mtriple=x86_64-linux-generic -verify-machineinstrs -mattr=sse2 | FileCheck --check-prefix=CHECK-O3 %s
+
+define i8 @load_i8(i8* %ptr) {
+; CHECK-O0-LABEL: load_i8:
+; CHECK-O0:       # %bb.0:
+; CHECK-O0-NEXT:    movb (%rdi), %al
+; CHECK-O0-NEXT:    retq
+;
+; CHECK-O3-LABEL: load_i8:
+; CHECK-O3:       # %bb.0:
+; CHECK-O3-NEXT:    movb (%rdi), %al
+; CHECK-O3-NEXT:    retq
+  %v = load atomic i8, i8* %ptr monotonic, align 1
+  ret i8 %v
+}
+
+define void @store_i8(i8* %ptr, i8 %v) {
+; CHECK-O0-LABEL: store_i8:
+; CHECK-O0:       # %bb.0:
+; CHECK-O0-NEXT:    movb %sil, %al
+; CHECK-O0-NEXT:    movb %al, (%rdi)
+; CHECK-O0-NEXT:    retq
+;
+; CHECK-O3-LABEL: store_i8:
+; CHECK-O3:       # %bb.0:
+; CHECK-O3-NEXT:    movb %sil, (%rdi)
+; CHECK-O3-NEXT:    retq
+  store atomic i8 %v, i8* %ptr monotonic, align 1
+  ret void
+}
+
+define i16 @load_i16(i16* %ptr) {
+; CHECK-O0-LABEL: load_i16:
+; CHECK-O0:       # %bb.0:
+; CHECK-O0-NEXT:    movw (%rdi), %ax
+; CHECK-O0-NEXT:    retq
+;
+; CHECK-O3-LABEL: load_i16:
+; CHECK-O3:       # %bb.0:
+; CHECK-O3-NEXT:    movzwl (%rdi), %eax
+; CHECK-O3-NEXT:    retq
+  %v = load atomic i16, i16* %ptr monotonic, align 2
+  ret i16 %v
+}
+
+
+define void @store_i16(i16* %ptr, i16 %v) {
+; CHECK-O0-LABEL: store_i16:
+; CHECK-O0:       # %bb.0:
+; CHECK-O0-NEXT:    movw %si, %ax
+; CHECK-O0-NEXT:    movw %ax, (%rdi)
+; CHECK-O0-NEXT:    retq
+;
+; CHECK-O3-LABEL: store_i16:
+; CHECK-O3:       # %bb.0:
+; CHECK-O3-NEXT:    movw %si, (%rdi)
+; CHECK-O3-NEXT:    retq
+  store atomic i16 %v, i16* %ptr monotonic, align 2
+  ret void
+}
+
+define i32 @load_i32(i32* %ptr) {
+; CHECK-O0-LABEL: load_i32:
+; CHECK-O0:       # %bb.0:
+; CHECK-O0-NEXT:    movl (%rdi), %eax
+; CHECK-O0-NEXT:    retq
+;
+; CHECK-O3-LABEL: load_i32:
+; CHECK-O3:       # %bb.0:
+; CHECK-O3-NEXT:    movl (%rdi), %eax
+; CHECK-O3-NEXT:    retq
+  %v = load atomic i32, i32* %ptr monotonic, align 4
+  ret i32 %v
+}
+
+define void @store_i32(i32* %ptr, i32 %v) {
+; CHECK-O0-LABEL: store_i32:
+; CHECK-O0:       # %bb.0:
+; CHECK-O0-NEXT:    movl %esi, (%rdi)
+; CHECK-O0-NEXT:    retq
+;
+; CHECK-O3-LABEL: store_i32:
+; CHECK-O3:       # %bb.0:
+; CHECK-O3-NEXT:    movl %esi, (%rdi)
+; CHECK-O3-NEXT:    retq
+  store atomic i32 %v, i32* %ptr monotonic, align 4
+  ret void
+}
+
+define i64 @load_i64(i64* %ptr) {
+; CHECK-O0-LABEL: load_i64:
+; CHECK-O0:       # %bb.0:
+; CHECK-O0-NEXT:    movq (%rdi), %rax
+; CHECK-O0-NEXT:    retq
+;
+; CHECK-O3-LABEL: load_i64:
+; CHECK-O3:       # %bb.0:
+; CHECK-O3-NEXT:    movq (%rdi), %rax
+; CHECK-O3-NEXT:    retq
+  %v = load atomic i64, i64* %ptr monotonic, align 8
+  ret i64 %v
+}
+
+define void @store_i64(i64* %ptr, i64 %v) {
+; CHECK-O0-LABEL: store_i64:
+; CHECK-O0:       # %bb.0:
+; CHECK-O0-NEXT:    movq %rsi, (%rdi)
+; CHECK-O0-NEXT:    retq
+;
+; CHECK-O3-LABEL: store_i64:
+; CHECK-O3:       # %bb.0:
+; CHECK-O3-NEXT:    movq %rsi, (%rdi)
+; CHECK-O3-NEXT:    retq
+  store atomic i64 %v, i64* %ptr monotonic, align 8
+  ret void
+}
diff --git a/test/CodeGen/X86/atomic_op.ll b/test/CodeGen/X86/atomic-op.ll
similarity index 100%
rename from test/CodeGen/X86/atomic_op.ll
rename to test/CodeGen/X86/atomic-op.ll
diff --git a/test/CodeGen/X86/atomic-unordered.ll b/test/CodeGen/X86/atomic-unordered.ll
new file mode 100644
index 0000000000000000000000000000000000000000..61c44262ae9e731a195bf74f570e696e6af3c660
--- /dev/null
+++ b/test/CodeGen/X86/atomic-unordered.ll
@@ -0,0 +1,2138 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -O0 < %s -mtriple=x86_64-linux-generic -verify-machineinstrs -mattr=sse2 | FileCheck --check-prefix=CHECK-O0 %s
+; RUN: llc -O3 < %s -mtriple=x86_64-linux-generic -verify-machineinstrs -mattr=sse2 | FileCheck --check-prefix=CHECK-O3 %s
+
+define i8 @load_i8(i8* %ptr) {
+; CHECK-O0-LABEL: load_i8:
+; CHECK-O0:       # %bb.0:
+; CHECK-O0-NEXT:    movb (%rdi), %al
+; CHECK-O0-NEXT:    retq
+;
+; CHECK-O3-LABEL: load_i8:
+; CHECK-O3:       # %bb.0:
+; CHECK-O3-NEXT:    movb (%rdi), %al
+; CHECK-O3-NEXT:    retq
+  %v = load atomic i8, i8* %ptr unordered, align 1
+  ret i8 %v
+}
+
+define void @store_i8(i8* %ptr, i8 %v) {
+; CHECK-O0-LABEL: store_i8:
+; CHECK-O0:       # %bb.0:
+; CHECK-O0-NEXT:    movb %sil, %al
+; CHECK-O0-NEXT:    movb %al, (%rdi)
+; CHECK-O0-NEXT:    retq
+;
+; CHECK-O3-LABEL: store_i8:
+; CHECK-O3:       # %bb.0:
+; CHECK-O3-NEXT:    movb %sil, (%rdi)
+; CHECK-O3-NEXT:    retq
+  store atomic i8 %v, i8* %ptr unordered, align 1
+  ret void
+}
+
+define i16 @load_i16(i16* %ptr) {
+; CHECK-O0-LABEL: load_i16:
+; CHECK-O0:       # %bb.0:
+; CHECK-O0-NEXT:    movw (%rdi), %ax
+; CHECK-O0-NEXT:    retq
+;
+; CHECK-O3-LABEL: load_i16:
+; CHECK-O3:       # %bb.0:
+; CHECK-O3-NEXT:    movzwl (%rdi), %eax
+; CHECK-O3-NEXT:    retq
+  %v = load atomic i16, i16* %ptr unordered, align 2
+  ret i16 %v
+}
+
+
+define void @store_i16(i16* %ptr, i16 %v) {
+; CHECK-O0-LABEL: store_i16:
+; CHECK-O0:       # %bb.0:
+; CHECK-O0-NEXT:    movw %si, %ax
+; CHECK-O0-NEXT:    movw %ax, (%rdi)
+; CHECK-O0-NEXT:    retq
+;
+; CHECK-O3-LABEL: store_i16:
+; CHECK-O3:       # %bb.0:
+; CHECK-O3-NEXT:    movw %si, (%rdi)
+; CHECK-O3-NEXT:    retq
+  store atomic i16 %v, i16* %ptr unordered, align 2
+  ret void
+}
+
+define i32 @load_i32(i32* %ptr) {
+; CHECK-O0-LABEL: load_i32:
+; CHECK-O0:       # %bb.0:
+; CHECK-O0-NEXT:    movl (%rdi), %eax
+; CHECK-O0-NEXT:    retq
+;
+; CHECK-O3-LABEL: load_i32:
+; CHECK-O3:       # %bb.0:
+; CHECK-O3-NEXT:    movl (%rdi), %eax
+; CHECK-O3-NEXT:    retq
+  %v = load atomic i32, i32* %ptr unordered, align 4
+  ret i32 %v
+}
+
+define void @store_i32(i32* %ptr, i32 %v) {
+; CHECK-O0-LABEL: store_i32:
+; CHECK-O0:       # %bb.0:
+; CHECK-O0-NEXT:    movl %esi, (%rdi)
+; CHECK-O0-NEXT:    retq
+;
+; CHECK-O3-LABEL: store_i32:
+; CHECK-O3:       # %bb.0:
+; CHECK-O3-NEXT:    movl %esi, (%rdi)
+; CHECK-O3-NEXT:    retq
+  store atomic i32 %v, i32* %ptr unordered, align 4
+  ret void
+}
+
+define i64 @load_i64(i64* %ptr) {
+; CHECK-O0-LABEL: load_i64:
+; CHECK-O0:       # %bb.0:
+; CHECK-O0-NEXT:    movq (%rdi), %rax
+; CHECK-O0-NEXT:    retq
+;
+; CHECK-O3-LABEL: load_i64:
+; CHECK-O3:       # %bb.0:
+; CHECK-O3-NEXT:    movq (%rdi), %rax
+; CHECK-O3-NEXT:    retq
+  %v = load atomic i64, i64* %ptr unordered, align 8
+  ret i64 %v
+}
+
+define void @store_i64(i64* %ptr, i64 %v) {
+; CHECK-O0-LABEL: store_i64:
+; CHECK-O0:       # %bb.0:
+; CHECK-O0-NEXT:    movq %rsi, (%rdi)
+; CHECK-O0-NEXT:    retq
+;
+; CHECK-O3-LABEL: store_i64:
+; CHECK-O3:       # %bb.0:
+; CHECK-O3-NEXT:    movq %rsi, (%rdi)
+; CHECK-O3-NEXT:    retq
+  store atomic i64 %v, i64* %ptr unordered, align 8
+  ret void
+}
+
+;; The tests in the rest of this file are intended to show transforms which we
+;; either *can't* do for legality, or don't currently implement.  The later
+;; are noted carefully where relevant.
+
+;; Start w/some clearly illegal ones.
+
+; Must use a full width op, not a byte op
+define void @narrow_writeback_or(i64* %ptr) {
+; CHECK-O0-LABEL: narrow_writeback_or:
+; CHECK-O0:       # %bb.0:
+; CHECK-O0-NEXT:    movq (%rdi), %rax
+; CHECK-O0-NEXT:    orq $7, %rax
+; CHECK-O0-NEXT:    movq %rax, (%rdi)
+; CHECK-O0-NEXT:    retq
+;
+; CHECK-O3-LABEL: narrow_writeback_or:
+; CHECK-O3:       # %bb.0:
+; CHECK-O3-NEXT:    orq $7, (%rdi)
+; CHECK-O3-NEXT:    retq
+  %v = load atomic i64, i64* %ptr unordered, align 8
+  %v.new = or i64 %v, 7
+  store atomic i64 %v.new, i64* %ptr unordered, align 8
+  ret void
+}
+
+; Must use a full width op, not a byte op
+define void @narrow_writeback_and(i64* %ptr) {
+; CHECK-O0-LABEL: narrow_writeback_and:
+; CHECK-O0:       # %bb.0:
+; CHECK-O0-NEXT:    movq (%rdi), %rax
+; CHECK-O0-NEXT:    movl %eax, %ecx
+; CHECK-O0-NEXT:    andl $-256, %ecx
+; CHECK-O0-NEXT:    movl %ecx, %eax
+; CHECK-O0-NEXT:    movq %rax, (%rdi)
+; CHECK-O0-NEXT:    retq
+;
+; CHECK-O3-LABEL: narrow_writeback_and:
+; CHECK-O3:       # %bb.0:
+; CHECK-O3-NEXT:    movl $4294967040, %eax # imm = 0xFFFFFF00
+; CHECK-O3-NEXT:    andq %rax, (%rdi)
+; CHECK-O3-NEXT:    retq
+  %v = load atomic i64, i64* %ptr unordered, align 8
+  %v.new = and i64 %v, 4294967040 ;; 0xFFFF_FF00
+  store atomic i64 %v.new, i64* %ptr unordered, align 8
+  ret void
+}
+
+; Must use a full width op, not a byte op
+define void @narrow_writeback_xor(i64* %ptr) {
+; CHECK-O0-LABEL: narrow_writeback_xor:
+; CHECK-O0:       # %bb.0:
+; CHECK-O0-NEXT:    movq (%rdi), %rax
+; CHECK-O0-NEXT:    xorq $7, %rax
+; CHECK-O0-NEXT:    movq %rax, (%rdi)
+; CHECK-O0-NEXT:    retq
+;
+; CHECK-O3-LABEL: narrow_writeback_xor:
+; CHECK-O3:       # %bb.0:
+; CHECK-O3-NEXT:    xorq $7, (%rdi)
+; CHECK-O3-NEXT:    retq
+  %v = load atomic i64, i64* %ptr unordered, align 8
+  %v.new = xor i64 %v, 7
+  store atomic i64 %v.new, i64* %ptr unordered, align 8
+  ret void
+}
+
+;; Next batch of tests are exercising cases where store widening would
+;; improve codegeneration.  Note that widening is only legal if the
+;; resulting type would be atomic.  Each tests has a well aligned, and
+;; unaligned variant to ensure we get correct codegen here.
+;;
+;; Note: It's not a legality issue, but there's a gotcha here to be aware
+;; of.  Once we widen a pair of atomic stores, we loose the information
+;; that the original atomicity requirement was half the width.  Given that,
+;; we can't then split the load again.  This challenges our usual iterative
+;; approach to incremental improvement.
+
+; Legal if wider type is also atomic (TODO)
+define void @widen_store(i32* %p0, i32 %v1, i32 %v2) {
+; CHECK-O0-LABEL: widen_store:
+; CHECK-O0:       # %bb.0:
+; CHECK-O0-NEXT:    movl %esi, (%rdi)
+; CHECK-O0-NEXT:    movl %edx, 4(%rdi)
+; CHECK-O0-NEXT:    retq
+;
+; CHECK-O3-LABEL: widen_store:
+; CHECK-O3:       # %bb.0:
+; CHECK-O3-NEXT:    movl %esi, (%rdi)
+; CHECK-O3-NEXT:    movl %edx, 4(%rdi)
+; CHECK-O3-NEXT:    retq
+  %p1 = getelementptr i32, i32* %p0, i64 1
+  store atomic i32 %v1, i32* %p0 unordered, align 8
+  store atomic i32 %v2, i32* %p1 unordered, align 4
+  ret void
+}
+
+; This one is *NOT* legal to widen.  With weaker alignment,
+; the wider type might cross a cache line and violate the
+; atomicity requirement.
+define void @widen_store_unaligned(i32* %p0, i32 %v1, i32 %v2) {
+; CHECK-O0-LABEL: widen_store_unaligned:
+; CHECK-O0:       # %bb.0:
+; CHECK-O0-NEXT:    movl %esi, (%rdi)
+; CHECK-O0-NEXT:    movl %edx, 4(%rdi)
+; CHECK-O0-NEXT:    retq
+;
+; CHECK-O3-LABEL: widen_store_unaligned:
+; CHECK-O3:       # %bb.0:
+; CHECK-O3-NEXT:    movl %esi, (%rdi)
+; CHECK-O3-NEXT:    movl %edx, 4(%rdi)
+; CHECK-O3-NEXT:    retq
+  %p1 = getelementptr i32, i32* %p0, i64 1
+  store atomic i32 %v1, i32* %p0 unordered, align 4
+  store atomic i32 %v2, i32* %p1 unordered, align 4
+  ret void
+}
+
+; Legal if wider type is also atomic (TODO)
+define void @widen_broadcast(i32* %p0, i32 %v) {
+; CHECK-O0-LABEL: widen_broadcast:
+; CHECK-O0:       # %bb.0:
+; CHECK-O0-NEXT:    movl %esi, (%rdi)
+; CHECK-O0-NEXT:    movl %esi, 4(%rdi)
+; CHECK-O0-NEXT:    retq
+;
+; CHECK-O3-LABEL: widen_broadcast:
+; CHECK-O3:       # %bb.0:
+; CHECK-O3-NEXT:    movl %esi, (%rdi)
+; CHECK-O3-NEXT:    movl %esi, 4(%rdi)
+; CHECK-O3-NEXT:    retq
+  %p1 = getelementptr i32, i32* %p0, i64 1
+  store atomic i32 %v, i32* %p0 unordered, align 8
+  store atomic i32 %v, i32* %p1 unordered, align 4
+  ret void
+}
+
+; Not legal to widen due to alignment restriction
+define void @widen_broadcast_unaligned(i32* %p0, i32 %v) {
+; CHECK-O0-LABEL: widen_broadcast_unaligned:
+; CHECK-O0:       # %bb.0:
+; CHECK-O0-NEXT:    movl %esi, (%rdi)
+; CHECK-O0-NEXT:    movl %esi, 4(%rdi)
+; CHECK-O0-NEXT:    retq
+;
+; CHECK-O3-LABEL: widen_broadcast_unaligned:
+; CHECK-O3:       # %bb.0:
+; CHECK-O3-NEXT:    movl %esi, (%rdi)
+; CHECK-O3-NEXT:    movl %esi, 4(%rdi)
+; CHECK-O3-NEXT:    retq
+  %p1 = getelementptr i32, i32* %p0, i64 1
+  store atomic i32 %v, i32* %p0 unordered, align 4
+  store atomic i32 %v, i32* %p1 unordered, align 4
+  ret void
+}
+
+; Legal if wider type is also atomic (TODO)
+define void @vec_store(i32* %p0, <2 x i32> %vec) {
+; CHECK-O0-LABEL: vec_store:
+; CHECK-O0:       # %bb.0:
+; CHECK-O0-NEXT:    movd %xmm0, %eax
+; CHECK-O0-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; CHECK-O0-NEXT:    movd %xmm0, %ecx
+; CHECK-O0-NEXT:    movl %eax, (%rdi)
+; CHECK-O0-NEXT:    movl %ecx, 4(%rdi)
+; CHECK-O0-NEXT:    retq
+;
+; CHECK-O3-LABEL: vec_store:
+; CHECK-O3:       # %bb.0:
+; CHECK-O3-NEXT:    movd %xmm0, %eax
+; CHECK-O3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; CHECK-O3-NEXT:    movd %xmm0, %ecx
+; CHECK-O3-NEXT:    movl %eax, (%rdi)
+; CHECK-O3-NEXT:    movl %ecx, 4(%rdi)
+; CHECK-O3-NEXT:    retq
+  %v1 = extractelement <2 x i32> %vec, i32 0
+  %v2 = extractelement <2 x i32> %vec, i32 1
+  %p1 = getelementptr i32, i32* %p0, i64 1
+  store atomic i32 %v1, i32* %p0 unordered, align 8
+  store atomic i32 %v2, i32* %p1 unordered, align 4
+  ret void
+}
+
+; Not legal to widen due to alignment restriction
+define void @vec_store_unaligned(i32* %p0, <2 x i32> %vec) {
+; CHECK-O0-LABEL: vec_store_unaligned:
+; CHECK-O0:       # %bb.0:
+; CHECK-O0-NEXT:    movd %xmm0, %eax
+; CHECK-O0-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; CHECK-O0-NEXT:    movd %xmm0, %ecx
+; CHECK-O0-NEXT:    movl %eax, (%rdi)
+; CHECK-O0-NEXT:    movl %ecx, 4(%rdi)
+; CHECK-O0-NEXT:    retq
+;
+; CHECK-O3-LABEL: vec_store_unaligned:
+; CHECK-O3:       # %bb.0:
+; CHECK-O3-NEXT:    movd %xmm0, %eax
+; CHECK-O3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; CHECK-O3-NEXT:    movd %xmm0, %ecx
+; CHECK-O3-NEXT:    movl %eax, (%rdi)
+; CHECK-O3-NEXT:    movl %ecx, 4(%rdi)
+; CHECK-O3-NEXT:    retq
+  %v1 = extractelement <2 x i32> %vec, i32 0
+  %v2 = extractelement <2 x i32> %vec, i32 1
+  %p1 = getelementptr i32, i32* %p0, i64 1
+  store atomic i32 %v1, i32* %p0 unordered, align 4
+  store atomic i32 %v2, i32* %p1 unordered, align 4
+  ret void
+}
+
+
+
+; Legal if wider type is also atomic (TODO)
+; Also, can avoid register move from xmm to eax (TODO)
+define void @widen_broadcast2(i32* %p0, <2 x i32> %vec) {
+; CHECK-O0-LABEL: widen_broadcast2:
+; CHECK-O0:       # %bb.0:
+; CHECK-O0-NEXT:    movd %xmm0, %eax
+; CHECK-O0-NEXT:    movl %eax, (%rdi)
+; CHECK-O0-NEXT:    movl %eax, 4(%rdi)
+; CHECK-O0-NEXT:    retq
+;
+; CHECK-O3-LABEL: widen_broadcast2:
+; CHECK-O3:       # %bb.0:
+; CHECK-O3-NEXT:    movd %xmm0, %eax
+; CHECK-O3-NEXT:    movl %eax, (%rdi)
+; CHECK-O3-NEXT:    movl %eax, 4(%rdi)
+; CHECK-O3-NEXT:    retq
+  %v1 = extractelement <2 x i32> %vec, i32 0
+  %p1 = getelementptr i32, i32* %p0, i64 1
+  store atomic i32 %v1, i32* %p0 unordered, align 8
+  store atomic i32 %v1, i32* %p1 unordered, align 4
+  ret void
+}
+
+; Not legal to widen due to alignment restriction
+define void @widen_broadcast2_unaligned(i32* %p0, <2 x i32> %vec) {
+; CHECK-O0-LABEL: widen_broadcast2_unaligned:
+; CHECK-O0:       # %bb.0:
+; CHECK-O0-NEXT:    movd %xmm0, %eax
+; CHECK-O0-NEXT:    movl %eax, (%rdi)
+; CHECK-O0-NEXT:    movl %eax, 4(%rdi)
+; CHECK-O0-NEXT:    retq
+;
+; CHECK-O3-LABEL: widen_broadcast2_unaligned:
+; CHECK-O3:       # %bb.0:
+; CHECK-O3-NEXT:    movd %xmm0, %eax
+; CHECK-O3-NEXT:    movl %eax, (%rdi)
+; CHECK-O3-NEXT:    movl %eax, 4(%rdi)
+; CHECK-O3-NEXT:    retq
+  %v1 = extractelement <2 x i32> %vec, i32 0
+  %p1 = getelementptr i32, i32* %p0, i64 1
+  store atomic i32 %v1, i32* %p0 unordered, align 4
+  store atomic i32 %v1, i32* %p1 unordered, align 4
+  ret void
+}
+
+; Legal if wider type is also atomic (TODO)
+define void @widen_zero_init(i32* %p0, i32 %v1, i32 %v2) {
+; CHECK-O0-LABEL: widen_zero_init:
+; CHECK-O0:       # %bb.0:
+; CHECK-O0-NEXT:    movl $0, (%rdi)
+; CHECK-O0-NEXT:    movl $0, 4(%rdi)
+; CHECK-O0-NEXT:    movl %esi, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-O0-NEXT:    movl %edx, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-O0-NEXT:    retq
+;
+; CHECK-O3-LABEL: widen_zero_init:
+; CHECK-O3:       # %bb.0:
+; CHECK-O3-NEXT:    movl $0, (%rdi)
+; CHECK-O3-NEXT:    movl $0, 4(%rdi)
+; CHECK-O3-NEXT:    retq
+  %p1 = getelementptr i32, i32* %p0, i64 1
+  store atomic i32 0, i32* %p0 unordered, align 8
+  store atomic i32 0, i32* %p1 unordered, align 4
+  ret void
+}
+
+; Not legal to widen due to alignment restriction
+define void @widen_zero_init_unaligned(i32* %p0, i32 %v1, i32 %v2) {
+; CHECK-O0-LABEL: widen_zero_init_unaligned:
+; CHECK-O0:       # %bb.0:
+; CHECK-O0-NEXT:    movl $0, (%rdi)
+; CHECK-O0-NEXT:    movl $0, 4(%rdi)
+; CHECK-O0-NEXT:    movl %esi, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-O0-NEXT:    movl %edx, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-O0-NEXT:    retq
+;
+; CHECK-O3-LABEL: widen_zero_init_unaligned:
+; CHECK-O3:       # %bb.0:
+; CHECK-O3-NEXT:    movl $0, (%rdi)
+; CHECK-O3-NEXT:    movl $0, 4(%rdi)
+; CHECK-O3-NEXT:    retq
+  %p1 = getelementptr i32, i32* %p0, i64 1
+  store atomic i32 0, i32* %p0 unordered, align 4
+  store atomic i32 0, i32* %p1 unordered, align 4
+  ret void
+}
+
+;; The next batch of tests are stressing load folding. Folding is legal
+;; on x86, so these are simply checking optimization quality.
+
+; Legal, as expected
+define i64 @load_fold_add1(i64* %p) {
+; CHECK-O0-LABEL: load_fold_add1:
+; CHECK-O0:       # %bb.0:
+; CHECK-O0-NEXT:    movq (%rdi), %rdi
+; CHECK-O0-NEXT:    addq $15, %rdi
+; CHECK-O0-NEXT:    movq %rdi, %rax
+; CHECK-O0-NEXT:    retq
+;
+; CHECK-O3-LABEL: load_fold_add1:
+; CHECK-O3:       # %bb.0:
+; CHECK-O3-NEXT:    movq (%rdi), %rax
+; CHECK-O3-NEXT:    addq $15, %rax
+; CHECK-O3-NEXT:    retq
+  %v = load atomic i64, i64* %p unordered, align 8
+  %ret = add i64 %v, 15
+  ret i64 %ret
+}
+
+; Legal, as expected
+define i64 @load_fold_add2(i64* %p, i64 %v2) {
+; CHECK-O0-LABEL: load_fold_add2:
+; CHECK-O0:       # %bb.0:
+; CHECK-O0-NEXT:    addq (%rdi), %rsi
+; CHECK-O0-NEXT:    movq %rsi, %rax
+; CHECK-O0-NEXT:    retq
+;
+; CHECK-O3-LABEL: load_fold_add2:
+; CHECK-O3:       # %bb.0:
+; CHECK-O3-NEXT:    movq (%rdi), %rax
+; CHECK-O3-NEXT:    addq %rsi, %rax
+; CHECK-O3-NEXT:    retq
+  %v = load atomic i64, i64* %p unordered, align 8
+  %ret = add i64 %v, %v2
+  ret i64 %ret
+}
+
+; Legal to fold (TODO)
+define i64 @load_fold_add3(i64* %p1, i64* %p2) {
+; CHECK-O0-LABEL: load_fold_add3:
+; CHECK-O0:       # %bb.0:
+; CHECK-O0-NEXT:    movq (%rdi), %rdi
+; CHECK-O0-NEXT:    addq (%rsi), %rdi
+; CHECK-O0-NEXT:    movq %rdi, %rax
+; CHECK-O0-NEXT:    retq
+;
+; CHECK-O3-LABEL: load_fold_add3:
+; CHECK-O3:       # %bb.0:
+; CHECK-O3-NEXT:    movq (%rdi), %rcx
+; CHECK-O3-NEXT:    movq (%rsi), %rax
+; CHECK-O3-NEXT:    addq %rcx, %rax
+; CHECK-O3-NEXT:    retq
+  %v = load atomic i64, i64* %p1 unordered, align 8
+  %v2 = load atomic i64, i64* %p2 unordered, align 8
+  %ret = add i64 %v, %v2
+  ret i64 %ret
+}
+
+; Legal, as expected
+define i64 @load_fold_sub1(i64* %p) {
+; CHECK-O0-LABEL: load_fold_sub1:
+; CHECK-O0:       # %bb.0:
+; CHECK-O0-NEXT:    movq (%rdi), %rdi
+; CHECK-O0-NEXT:    subq $15, %rdi
+; CHECK-O0-NEXT:    movq %rdi, %rax
+; CHECK-O0-NEXT:    retq
+;
+; CHECK-O3-LABEL: load_fold_sub1:
+; CHECK-O3:       # %bb.0:
+; CHECK-O3-NEXT:    movq (%rdi), %rax
+; CHECK-O3-NEXT:    addq $-15, %rax
+; CHECK-O3-NEXT:    retq
+  %v = load atomic i64, i64* %p unordered, align 8
+  %ret = sub i64 %v, 15
+  ret i64 %ret
+}
+
+; Legal, as expected
+define i64 @load_fold_sub2(i64* %p, i64 %v2) {
+; CHECK-O0-LABEL: load_fold_sub2:
+; CHECK-O0:       # %bb.0:
+; CHECK-O0-NEXT:    movq (%rdi), %rdi
+; CHECK-O0-NEXT:    subq %rsi, %rdi
+; CHECK-O0-NEXT:    movq %rdi, %rax
+; CHECK-O0-NEXT:    retq
+;
+; CHECK-O3-LABEL: load_fold_sub2:
+; CHECK-O3:       # %bb.0:
+; CHECK-O3-NEXT:    movq (%rdi), %rax
+; CHECK-O3-NEXT:    subq %rsi, %rax
+; CHECK-O3-NEXT:    retq
+  %v = load atomic i64, i64* %p unordered, align 8
+  %ret = sub i64 %v, %v2
+  ret i64 %ret
+}
+
+; Legal to fold (TODO)
+define i64 @load_fold_sub3(i64* %p1, i64* %p2) {
+; CHECK-O0-LABEL: load_fold_sub3:
+; CHECK-O0:       # %bb.0:
+; CHECK-O0-NEXT:    movq (%rdi), %rdi
+; CHECK-O0-NEXT:    subq (%rsi), %rdi
+; CHECK-O0-NEXT:    movq %rdi, %rax
+; CHECK-O0-NEXT:    retq
+;
+; CHECK-O3-LABEL: load_fold_sub3:
+; CHECK-O3:       # %bb.0:
+; CHECK-O3-NEXT:    movq (%rdi), %rax
+; CHECK-O3-NEXT:    movq (%rsi), %rcx
+; CHECK-O3-NEXT:    subq %rcx, %rax
+; CHECK-O3-NEXT:    retq
+  %v = load atomic i64, i64* %p1 unordered, align 8
+  %v2 = load atomic i64, i64* %p2 unordered, align 8
+  %ret = sub i64 %v, %v2
+  ret i64 %ret
+}
+
+; Legal, as expected
+define i64 @load_fold_mul1(i64* %p) {
+; CHECK-O0-LABEL: load_fold_mul1:
+; CHECK-O0:       # %bb.0:
+; CHECK-O0-NEXT:    imulq $15, (%rdi), %rax
+; CHECK-O0-NEXT:    retq
+;
+; CHECK-O3-LABEL: load_fold_mul1:
+; CHECK-O3:       # %bb.0:
+; CHECK-O3-NEXT:    movq (%rdi), %rax
+; CHECK-O3-NEXT:    leaq (%rax,%rax,4), %rax
+; CHECK-O3-NEXT:    leaq (%rax,%rax,2), %rax
+; CHECK-O3-NEXT:    retq
+  %v = load atomic i64, i64* %p unordered, align 8
+  %ret = mul i64 %v, 15
+  ret i64 %ret
+}
+
+; Legal, O0 is better than O3 codegen (TODO)
+define i64 @load_fold_mul2(i64* %p, i64 %v2) {
+; CHECK-O0-LABEL: load_fold_mul2:
+; CHECK-O0:       # %bb.0:
+; CHECK-O0-NEXT:    imulq (%rdi), %rsi
+; CHECK-O0-NEXT:    movq %rsi, %rax
+; CHECK-O0-NEXT:    retq
+;
+; CHECK-O3-LABEL: load_fold_mul2:
+; CHECK-O3:       # %bb.0:
+; CHECK-O3-NEXT:    movq (%rdi), %rax
+; CHECK-O3-NEXT:    imulq %rsi, %rax
+; CHECK-O3-NEXT:    retq
+  %v = load atomic i64, i64* %p unordered, align 8
+  %ret = mul i64 %v, %v2
+  ret i64 %ret
+}
+
+; Legal to fold (TODO)
+define i64 @load_fold_mul3(i64* %p1, i64* %p2) {
+; CHECK-O0-LABEL: load_fold_mul3:
+; CHECK-O0:       # %bb.0:
+; CHECK-O0-NEXT:    movq (%rdi), %rdi
+; CHECK-O0-NEXT:    imulq (%rsi), %rdi
+; CHECK-O0-NEXT:    movq %rdi, %rax
+; CHECK-O0-NEXT:    retq
+;
+; CHECK-O3-LABEL: load_fold_mul3:
+; CHECK-O3:       # %bb.0:
+; CHECK-O3-NEXT:    movq (%rdi), %rcx
+; CHECK-O3-NEXT:    movq (%rsi), %rax
+; CHECK-O3-NEXT:    imulq %rcx, %rax
+; CHECK-O3-NEXT:    retq
+  %v = load atomic i64, i64* %p1 unordered, align 8
+  %v2 = load atomic i64, i64* %p2 unordered, align 8
+  %ret = mul i64 %v, %v2
+  ret i64 %ret
+}
+
+; Legal to fold (TODO)
+define i64 @load_fold_sdiv1(i64* %p) {
+; CHECK-O0-LABEL: load_fold_sdiv1:
+; CHECK-O0:       # %bb.0:
+; CHECK-O0-NEXT:    movq (%rdi), %rax
+; CHECK-O0-NEXT:    cqto
+; CHECK-O0-NEXT:    movl $15, %edi
+; CHECK-O0-NEXT:    idivq %rdi
+; CHECK-O0-NEXT:    retq
+;
+; CHECK-O3-LABEL: load_fold_sdiv1:
+; CHECK-O3:       # %bb.0:
+; CHECK-O3-NEXT:    movq (%rdi), %rcx
+; CHECK-O3-NEXT:    movabsq $-8608480567731124087, %rdx # imm = 0x8888888888888889
+; CHECK-O3-NEXT:    movq %rcx, %rax
+; CHECK-O3-NEXT:    imulq %rdx
+; CHECK-O3-NEXT:    addq %rcx, %rdx
+; CHECK-O3-NEXT:    movq %rdx, %rax
+; CHECK-O3-NEXT:    shrq $63, %rax
+; CHECK-O3-NEXT:    sarq $3, %rdx
+; CHECK-O3-NEXT:    leaq (%rdx,%rax), %rax
+; CHECK-O3-NEXT:    retq
+  %v = load atomic i64, i64* %p unordered, align 8
+  %ret = sdiv i64 %v, 15
+  ret i64 %ret
+}
+
+; Legal to fold (TODO)
+define i64 @load_fold_sdiv2(i64* %p, i64 %v2) {
+; CHECK-O0-LABEL: load_fold_sdiv2:
+; CHECK-O0:       # %bb.0:
+; CHECK-O0-NEXT:    movq (%rdi), %rax
+; CHECK-O0-NEXT:    cqto
+; CHECK-O0-NEXT:    idivq %rsi
+; CHECK-O0-NEXT:    retq
+;
+; CHECK-O3-LABEL: load_fold_sdiv2:
+; CHECK-O3:       # %bb.0:
+; CHECK-O3-NEXT:    movq (%rdi), %rax
+; CHECK-O3-NEXT:    cqto
+; CHECK-O3-NEXT:    idivq %rsi
+; CHECK-O3-NEXT:    retq
+  %v = load atomic i64, i64* %p unordered, align 8
+  %ret = sdiv i64 %v, %v2
+  ret i64 %ret
+}
+
+; Legal to fold (TODO)
+define i64 @load_fold_sdiv3(i64* %p1, i64* %p2) {
+; CHECK-O0-LABEL: load_fold_sdiv3:
+; CHECK-O0:       # %bb.0:
+; CHECK-O0-NEXT:    movq (%rdi), %rax
+; CHECK-O0-NEXT:    cqto
+; CHECK-O0-NEXT:    idivq (%rsi)
+; CHECK-O0-NEXT:    retq
+;
+; CHECK-O3-LABEL: load_fold_sdiv3:
+; CHECK-O3:       # %bb.0:
+; CHECK-O3-NEXT:    movq (%rdi), %rax
+; CHECK-O3-NEXT:    movq (%rsi), %rcx
+; CHECK-O3-NEXT:    cqto
+; CHECK-O3-NEXT:    idivq %rcx
+; CHECK-O3-NEXT:    retq
+  %v = load atomic i64, i64* %p1 unordered, align 8
+  %v2 = load atomic i64, i64* %p2 unordered, align 8
+  %ret = sdiv i64 %v, %v2
+  ret i64 %ret
+}
+
+; Legal to fold (TODO)
+define i64 @load_fold_udiv1(i64* %p) {
+; CHECK-O0-LABEL: load_fold_udiv1:
+; CHECK-O0:       # %bb.0:
+; CHECK-O0-NEXT:    movq (%rdi), %rax
+; CHECK-O0-NEXT:    xorl %ecx, %ecx
+; CHECK-O0-NEXT:    movl %ecx, %edx
+; CHECK-O0-NEXT:    movl $15, %edi
+; CHECK-O0-NEXT:    divq %rdi
+; CHECK-O0-NEXT:    retq
+;
+; CHECK-O3-LABEL: load_fold_udiv1:
+; CHECK-O3:       # %bb.0:
+; CHECK-O3-NEXT:    movq (%rdi), %rax
+; CHECK-O3-NEXT:    movabsq $-8608480567731124087, %rcx # imm = 0x8888888888888889
+; CHECK-O3-NEXT:    mulq %rcx
+; CHECK-O3-NEXT:    movq %rdx, %rax
+; CHECK-O3-NEXT:    shrq $3, %rax
+; CHECK-O3-NEXT:    retq
+  %v = load atomic i64, i64* %p unordered, align 8
+  %ret = udiv i64 %v, 15
+  ret i64 %ret
+}
+
+; Legal, as expected
+define i64 @load_fold_udiv2(i64* %p, i64 %v2) {
+; CHECK-O0-LABEL: load_fold_udiv2:
+; CHECK-O0:       # %bb.0:
+; CHECK-O0-NEXT:    movq (%rdi), %rax
+; CHECK-O0-NEXT:    xorl %ecx, %ecx
+; CHECK-O0-NEXT:    movl %ecx, %edx
+; CHECK-O0-NEXT:    divq %rsi
+; CHECK-O0-NEXT:    retq
+;
+; CHECK-O3-LABEL: load_fold_udiv2:
+; CHECK-O3:       # %bb.0:
+; CHECK-O3-NEXT:    movq (%rdi), %rax
+; CHECK-O3-NEXT:    xorl %edx, %edx
+; CHECK-O3-NEXT:    divq %rsi
+; CHECK-O3-NEXT:    retq
+  %v = load atomic i64, i64* %p unordered, align 8
+  %ret = udiv i64 %v, %v2
+  ret i64 %ret
+}
+
+; Legal to fold (TODO)
+define i64 @load_fold_udiv3(i64* %p1, i64* %p2) {
+; CHECK-O0-LABEL: load_fold_udiv3:
+; CHECK-O0:       # %bb.0:
+; CHECK-O0-NEXT:    movq (%rdi), %rax
+; CHECK-O0-NEXT:    xorl %ecx, %ecx
+; CHECK-O0-NEXT:    movl %ecx, %edx
+; CHECK-O0-NEXT:    divq (%rsi)
+; CHECK-O0-NEXT:    retq
+;
+; CHECK-O3-LABEL: load_fold_udiv3:
+; CHECK-O3:       # %bb.0:
+; CHECK-O3-NEXT:    movq (%rdi), %rax
+; CHECK-O3-NEXT:    movq (%rsi), %rcx
+; CHECK-O3-NEXT:    xorl %edx, %edx
+; CHECK-O3-NEXT:    divq %rcx
+; CHECK-O3-NEXT:    retq
+  %v = load atomic i64, i64* %p1 unordered, align 8
+  %v2 = load atomic i64, i64* %p2 unordered, align 8
+  %ret = udiv i64 %v, %v2
+  ret i64 %ret
+}
+
+; Legal to fold (TODO)
+define i64 @load_fold_srem1(i64* %p) {
+; CHECK-O0-LABEL: load_fold_srem1:
+; CHECK-O0:       # %bb.0:
+; CHECK-O0-NEXT:    movq (%rdi), %rax
+; CHECK-O0-NEXT:    cqto
+; CHECK-O0-NEXT:    movl $15, %edi
+; CHECK-O0-NEXT:    idivq %rdi
+; CHECK-O0-NEXT:    movq %rdx, %rax
+; CHECK-O0-NEXT:    retq
+;
+; CHECK-O3-LABEL: load_fold_srem1:
+; CHECK-O3:       # %bb.0:
+; CHECK-O3-NEXT:    movq (%rdi), %rcx
+; CHECK-O3-NEXT:    movabsq $-8608480567731124087, %rdx # imm = 0x8888888888888889
+; CHECK-O3-NEXT:    movq %rcx, %rax
+; CHECK-O3-NEXT:    imulq %rdx
+; CHECK-O3-NEXT:    addq %rcx, %rdx
+; CHECK-O3-NEXT:    movq %rdx, %rax
+; CHECK-O3-NEXT:    shrq $63, %rax
+; CHECK-O3-NEXT:    sarq $3, %rdx
+; CHECK-O3-NEXT:    addq %rax, %rdx
+; CHECK-O3-NEXT:    leaq (%rdx,%rdx,4), %rax
+; CHECK-O3-NEXT:    leaq (%rax,%rax,2), %rax
+; CHECK-O3-NEXT:    subq %rax, %rcx
+; CHECK-O3-NEXT:    movq %rcx, %rax
+; CHECK-O3-NEXT:    retq
+  %v = load atomic i64, i64* %p unordered, align 8
+  %ret = srem i64 %v, 15
+  ret i64 %ret
+}
+
+; Legal, as expected
+define i64 @load_fold_srem2(i64* %p, i64 %v2) {
+; CHECK-O0-LABEL: load_fold_srem2:
+; CHECK-O0:       # %bb.0:
+; CHECK-O0-NEXT:    movq (%rdi), %rax
+; CHECK-O0-NEXT:    cqto
+; CHECK-O0-NEXT:    idivq %rsi
+; CHECK-O0-NEXT:    movq %rdx, %rax
+; CHECK-O0-NEXT:    retq
+;
+; CHECK-O3-LABEL: load_fold_srem2:
+; CHECK-O3:       # %bb.0:
+; CHECK-O3-NEXT:    movq (%rdi), %rax
+; CHECK-O3-NEXT:    cqto
+; CHECK-O3-NEXT:    idivq %rsi
+; CHECK-O3-NEXT:    movq %rdx, %rax
+; CHECK-O3-NEXT:    retq
+  %v = load atomic i64, i64* %p unordered, align 8
+  %ret = srem i64 %v, %v2
+  ret i64 %ret
+}
+
+; Legal to fold (TODO)
+define i64 @load_fold_srem3(i64* %p1, i64* %p2) {
+; CHECK-O0-LABEL: load_fold_srem3:
+; CHECK-O0:       # %bb.0:
+; CHECK-O0-NEXT:    movq (%rdi), %rax
+; CHECK-O0-NEXT:    cqto
+; CHECK-O0-NEXT:    idivq (%rsi)
+; CHECK-O0-NEXT:    movq %rdx, %rax
+; CHECK-O0-NEXT:    retq
+;
+; CHECK-O3-LABEL: load_fold_srem3:
+; CHECK-O3:       # %bb.0:
+; CHECK-O3-NEXT:    movq (%rdi), %rax
+; CHECK-O3-NEXT:    movq (%rsi), %rcx
+; CHECK-O3-NEXT:    cqto
+; CHECK-O3-NEXT:    idivq %rcx
+; CHECK-O3-NEXT:    movq %rdx, %rax
+; CHECK-O3-NEXT:    retq
+  %v = load atomic i64, i64* %p1 unordered, align 8
+  %v2 = load atomic i64, i64* %p2 unordered, align 8
+  %ret = srem i64 %v, %v2
+  ret i64 %ret
+}
+
+; Legal to fold (TODO)
+define i64 @load_fold_urem1(i64* %p) {
+; CHECK-O0-LABEL: load_fold_urem1:
+; CHECK-O0:       # %bb.0:
+; CHECK-O0-NEXT:    movq (%rdi), %rax
+; CHECK-O0-NEXT:    xorl %ecx, %ecx
+; CHECK-O0-NEXT:    movl %ecx, %edx
+; CHECK-O0-NEXT:    movl $15, %edi
+; CHECK-O0-NEXT:    divq %rdi
+; CHECK-O0-NEXT:    movq %rdx, %rax
+; CHECK-O0-NEXT:    retq
+;
+; CHECK-O3-LABEL: load_fold_urem1:
+; CHECK-O3:       # %bb.0:
+; CHECK-O3-NEXT:    movq (%rdi), %rcx
+; CHECK-O3-NEXT:    movabsq $-8608480567731124087, %rdx # imm = 0x8888888888888889
+; CHECK-O3-NEXT:    movq %rcx, %rax
+; CHECK-O3-NEXT:    mulq %rdx
+; CHECK-O3-NEXT:    shrq $3, %rdx
+; CHECK-O3-NEXT:    leaq (%rdx,%rdx,4), %rax
+; CHECK-O3-NEXT:    leaq (%rax,%rax,2), %rax
+; CHECK-O3-NEXT:    subq %rax, %rcx
+; CHECK-O3-NEXT:    movq %rcx, %rax
+; CHECK-O3-NEXT:    retq
+  %v = load atomic i64, i64* %p unordered, align 8
+  %ret = urem i64 %v, 15
+  ret i64 %ret
+}
+
+; Legal, as expected
+define i64 @load_fold_urem2(i64* %p, i64 %v2) {
+; CHECK-O0-LABEL: load_fold_urem2:
+; CHECK-O0:       # %bb.0:
+; CHECK-O0-NEXT:    movq (%rdi), %rax
+; CHECK-O0-NEXT:    xorl %ecx, %ecx
+; CHECK-O0-NEXT:    movl %ecx, %edx
+; CHECK-O0-NEXT:    divq %rsi
+; CHECK-O0-NEXT:    movq %rdx, %rax
+; CHECK-O0-NEXT:    retq
+;
+; CHECK-O3-LABEL: load_fold_urem2:
+; CHECK-O3:       # %bb.0:
+; CHECK-O3-NEXT:    movq (%rdi), %rax
+; CHECK-O3-NEXT:    xorl %edx, %edx
+; CHECK-O3-NEXT:    divq %rsi
+; CHECK-O3-NEXT:    movq %rdx, %rax
+; CHECK-O3-NEXT:    retq
+  %v = load atomic i64, i64* %p unordered, align 8
+  %ret = urem i64 %v, %v2
+  ret i64 %ret
+}
+
+; Legal to fold (TODO)
+define i64 @load_fold_urem3(i64* %p1, i64* %p2) {
+; CHECK-O0-LABEL: load_fold_urem3:
+; CHECK-O0:       # %bb.0:
+; CHECK-O0-NEXT:    movq (%rdi), %rax
+; CHECK-O0-NEXT:    xorl %ecx, %ecx
+; CHECK-O0-NEXT:    movl %ecx, %edx
+; CHECK-O0-NEXT:    divq (%rsi)
+; CHECK-O0-NEXT:    movq %rdx, %rax
+; CHECK-O0-NEXT:    retq
+;
+; CHECK-O3-LABEL: load_fold_urem3:
+; CHECK-O3:       # %bb.0:
+; CHECK-O3-NEXT:    movq (%rdi), %rax
+; CHECK-O3-NEXT:    movq (%rsi), %rcx
+; CHECK-O3-NEXT:    xorl %edx, %edx
+; CHECK-O3-NEXT:    divq %rcx
+; CHECK-O3-NEXT:    movq %rdx, %rax
+; CHECK-O3-NEXT:    retq
+  %v = load atomic i64, i64* %p1 unordered, align 8
+  %v2 = load atomic i64, i64* %p2 unordered, align 8
+  %ret = urem i64 %v, %v2
+  ret i64 %ret
+}
+
+; Legal, as expected
+define i64 @load_fold_shl1(i64* %p) {
+; CHECK-O0-LABEL: load_fold_shl1:
+; CHECK-O0:       # %bb.0:
+; CHECK-O0-NEXT:    movq (%rdi), %rdi
+; CHECK-O0-NEXT:    shlq $15, %rdi
+; CHECK-O0-NEXT:    movq %rdi, %rax
+; CHECK-O0-NEXT:    retq
+;
+; CHECK-O3-LABEL: load_fold_shl1:
+; CHECK-O3:       # %bb.0:
+; CHECK-O3-NEXT:    movq (%rdi), %rax
+; CHECK-O3-NEXT:    shlq $15, %rax
+; CHECK-O3-NEXT:    retq
+  %v = load atomic i64, i64* %p unordered, align 8
+  %ret = shl i64 %v, 15
+  ret i64 %ret
+}
+
+; Legal to fold (TODO)
+define i64 @load_fold_shl2(i64* %p, i64 %v2) {
+; CHECK-O0-LABEL: load_fold_shl2:
+; CHECK-O0:       # %bb.0:
+; CHECK-O0-NEXT:    movq (%rdi), %rdi
+; CHECK-O0-NEXT:    movq %rsi, %rcx
+; CHECK-O0-NEXT:    # kill: def $cl killed $rcx
+; CHECK-O0-NEXT:    shlq %cl, %rdi
+; CHECK-O0-NEXT:    movq %rdi, %rax
+; CHECK-O0-NEXT:    retq
+;
+; CHECK-O3-LABEL: load_fold_shl2:
+; CHECK-O3:       # %bb.0:
+; CHECK-O3-NEXT:    movq %rsi, %rcx
+; CHECK-O3-NEXT:    movq (%rdi), %rax
+; CHECK-O3-NEXT:    # kill: def $cl killed $cl killed $rcx
+; CHECK-O3-NEXT:    shlq %cl, %rax
+; CHECK-O3-NEXT:    retq
+  %v = load atomic i64, i64* %p unordered, align 8
+  %ret = shl i64 %v, %v2
+  ret i64 %ret
+}
+
+; Legal to fold (TODO)
+define i64 @load_fold_shl3(i64* %p1, i64* %p2) {
+; CHECK-O0-LABEL: load_fold_shl3:
+; CHECK-O0:       # %bb.0:
+; CHECK-O0-NEXT:    movq (%rdi), %rdi
+; CHECK-O0-NEXT:    movq (%rsi), %rcx
+; CHECK-O0-NEXT:    # kill: def $cl killed $rcx
+; CHECK-O0-NEXT:    shlq %cl, %rdi
+; CHECK-O0-NEXT:    movq %rdi, %rax
+; CHECK-O0-NEXT:    retq
+;
+; CHECK-O3-LABEL: load_fold_shl3:
+; CHECK-O3:       # %bb.0:
+; CHECK-O3-NEXT:    movq (%rdi), %rax
+; CHECK-O3-NEXT:    movq (%rsi), %rcx
+; CHECK-O3-NEXT:    # kill: def $cl killed $cl killed $rcx
+; CHECK-O3-NEXT:    shlq %cl, %rax
+; CHECK-O3-NEXT:    retq
+  %v = load atomic i64, i64* %p1 unordered, align 8
+  %v2 = load atomic i64, i64* %p2 unordered, align 8
+  %ret = shl i64 %v, %v2
+  ret i64 %ret
+}
+
+; Legal, as expected
+define i64 @load_fold_lshr1(i64* %p) {
+; CHECK-O0-LABEL: load_fold_lshr1:
+; CHECK-O0:       # %bb.0:
+; CHECK-O0-NEXT:    movq (%rdi), %rdi
+; CHECK-O0-NEXT:    shrq $15, %rdi
+; CHECK-O0-NEXT:    movq %rdi, %rax
+; CHECK-O0-NEXT:    retq
+;
+; CHECK-O3-LABEL: load_fold_lshr1:
+; CHECK-O3:       # %bb.0:
+; CHECK-O3-NEXT:    movq (%rdi), %rax
+; CHECK-O3-NEXT:    shrq $15, %rax
+; CHECK-O3-NEXT:    retq
+  %v = load atomic i64, i64* %p unordered, align 8
+  %ret = lshr i64 %v, 15
+  ret i64 %ret
+}
+
+; Legal to fold (TODO)
+define i64 @load_fold_lshr2(i64* %p, i64 %v2) {
+; CHECK-O0-LABEL: load_fold_lshr2:
+; CHECK-O0:       # %bb.0:
+; CHECK-O0-NEXT:    movq (%rdi), %rdi
+; CHECK-O0-NEXT:    movq %rsi, %rcx
+; CHECK-O0-NEXT:    # kill: def $cl killed $rcx
+; CHECK-O0-NEXT:    shrq %cl, %rdi
+; CHECK-O0-NEXT:    movq %rdi, %rax
+; CHECK-O0-NEXT:    retq
+;
+; CHECK-O3-LABEL: load_fold_lshr2:
+; CHECK-O3:       # %bb.0:
+; CHECK-O3-NEXT:    movq %rsi, %rcx
+; CHECK-O3-NEXT:    movq (%rdi), %rax
+; CHECK-O3-NEXT:    # kill: def $cl killed $cl killed $rcx
+; CHECK-O3-NEXT:    shrq %cl, %rax
+; CHECK-O3-NEXT:    retq
+  %v = load atomic i64, i64* %p unordered, align 8
+  %ret = lshr i64 %v, %v2
+  ret i64 %ret
+}
+
+; Legal to fold (TODO)
+define i64 @load_fold_lshr3(i64* %p1, i64* %p2) {
+; CHECK-O0-LABEL: load_fold_lshr3:
+; CHECK-O0:       # %bb.0:
+; CHECK-O0-NEXT:    movq (%rdi), %rdi
+; CHECK-O0-NEXT:    movq (%rsi), %rcx
+; CHECK-O0-NEXT:    # kill: def $cl killed $rcx
+; CHECK-O0-NEXT:    shrq %cl, %rdi
+; CHECK-O0-NEXT:    movq %rdi, %rax
+; CHECK-O0-NEXT:    retq
+;
+; CHECK-O3-LABEL: load_fold_lshr3:
+; CHECK-O3:       # %bb.0:
+; CHECK-O3-NEXT:    movq (%rdi), %rax
+; CHECK-O3-NEXT:    movq (%rsi), %rcx
+; CHECK-O3-NEXT:    # kill: def $cl killed $cl killed $rcx
+; CHECK-O3-NEXT:    shrq %cl, %rax
+; CHECK-O3-NEXT:    retq
+  %v = load atomic i64, i64* %p1 unordered, align 8
+  %v2 = load atomic i64, i64* %p2 unordered, align 8
+  %ret = lshr i64 %v, %v2
+  ret i64 %ret
+}
+
+; Legal, as expected
+define i64 @load_fold_ashr1(i64* %p) {
+; CHECK-O0-LABEL: load_fold_ashr1:
+; CHECK-O0:       # %bb.0:
+; CHECK-O0-NEXT:    movq (%rdi), %rdi
+; CHECK-O0-NEXT:    sarq $15, %rdi
+; CHECK-O0-NEXT:    movq %rdi, %rax
+; CHECK-O0-NEXT:    retq
+;
+; CHECK-O3-LABEL: load_fold_ashr1:
+; CHECK-O3:       # %bb.0:
+; CHECK-O3-NEXT:    movq (%rdi), %rax
+; CHECK-O3-NEXT:    sarq $15, %rax
+; CHECK-O3-NEXT:    retq
+  %v = load atomic i64, i64* %p unordered, align 8
+  %ret = ashr i64 %v, 15
+  ret i64 %ret
+}
+
+; Legal to fold (TODO)
+define i64 @load_fold_ashr2(i64* %p, i64 %v2) {
+; CHECK-O0-LABEL: load_fold_ashr2:
+; CHECK-O0:       # %bb.0:
+; CHECK-O0-NEXT:    movq (%rdi), %rdi
+; CHECK-O0-NEXT:    movq %rsi, %rcx
+; CHECK-O0-NEXT:    # kill: def $cl killed $rcx
+; CHECK-O0-NEXT:    sarq %cl, %rdi
+; CHECK-O0-NEXT:    movq %rdi, %rax
+; CHECK-O0-NEXT:    retq
+;
+; CHECK-O3-LABEL: load_fold_ashr2:
+; CHECK-O3:       # %bb.0:
+; CHECK-O3-NEXT:    movq %rsi, %rcx
+; CHECK-O3-NEXT:    movq (%rdi), %rax
+; CHECK-O3-NEXT:    # kill: def $cl killed $cl killed $rcx
+; CHECK-O3-NEXT:    sarq %cl, %rax
+; CHECK-O3-NEXT:    retq
+  %v = load atomic i64, i64* %p unordered, align 8
+  %ret = ashr i64 %v, %v2
+  ret i64 %ret
+}
+
+; Legal to fold (TODO)
+define i64 @load_fold_ashr3(i64* %p1, i64* %p2) {
+; CHECK-O0-LABEL: load_fold_ashr3:
+; CHECK-O0:       # %bb.0:
+; CHECK-O0-NEXT:    movq (%rdi), %rdi
+; CHECK-O0-NEXT:    movq (%rsi), %rcx
+; CHECK-O0-NEXT:    # kill: def $cl killed $rcx
+; CHECK-O0-NEXT:    sarq %cl, %rdi
+; CHECK-O0-NEXT:    movq %rdi, %rax
+; CHECK-O0-NEXT:    retq
+;
+; CHECK-O3-LABEL: load_fold_ashr3:
+; CHECK-O3:       # %bb.0:
+; CHECK-O3-NEXT:    movq (%rdi), %rax
+; CHECK-O3-NEXT:    movq (%rsi), %rcx
+; CHECK-O3-NEXT:    # kill: def $cl killed $cl killed $rcx
+; CHECK-O3-NEXT:    sarq %cl, %rax
+; CHECK-O3-NEXT:    retq
+  %v = load atomic i64, i64* %p1 unordered, align 8
+  %v2 = load atomic i64, i64* %p2 unordered, align 8
+  %ret = ashr i64 %v, %v2
+  ret i64 %ret
+}
+
+; Legal, as expected
+define i64 @load_fold_and1(i64* %p) {
+; CHECK-O0-LABEL: load_fold_and1:
+; CHECK-O0:       # %bb.0:
+; CHECK-O0-NEXT:    movq (%rdi), %rdi
+; CHECK-O0-NEXT:    andq $15, %rdi
+; CHECK-O0-NEXT:    movq %rdi, %rax
+; CHECK-O0-NEXT:    retq
+;
+; CHECK-O3-LABEL: load_fold_and1:
+; CHECK-O3:       # %bb.0:
+; CHECK-O3-NEXT:    movq (%rdi), %rax
+; CHECK-O3-NEXT:    andl $15, %eax
+; CHECK-O3-NEXT:    retq
+  %v = load atomic i64, i64* %p unordered, align 8
+  %ret = and i64 %v, 15
+  ret i64 %ret
+}
+
+; Legal, as expected
+define i64 @load_fold_and2(i64* %p, i64 %v2) {
+; CHECK-O0-LABEL: load_fold_and2:
+; CHECK-O0:       # %bb.0:
+; CHECK-O0-NEXT:    andq (%rdi), %rsi
+; CHECK-O0-NEXT:    movq %rsi, %rax
+; CHECK-O0-NEXT:    retq
+;
+; CHECK-O3-LABEL: load_fold_and2:
+; CHECK-O3:       # %bb.0:
+; CHECK-O3-NEXT:    movq (%rdi), %rax
+; CHECK-O3-NEXT:    andq %rsi, %rax
+; CHECK-O3-NEXT:    retq
+  %v = load atomic i64, i64* %p unordered, align 8
+  %ret = and i64 %v, %v2
+  ret i64 %ret
+}
+
+; Legal to fold (TODO)
+define i64 @load_fold_and3(i64* %p1, i64* %p2) {
+; CHECK-O0-LABEL: load_fold_and3:
+; CHECK-O0:       # %bb.0:
+; CHECK-O0-NEXT:    movq (%rdi), %rdi
+; CHECK-O0-NEXT:    andq (%rsi), %rdi
+; CHECK-O0-NEXT:    movq %rdi, %rax
+; CHECK-O0-NEXT:    retq
+;
+; CHECK-O3-LABEL: load_fold_and3:
+; CHECK-O3:       # %bb.0:
+; CHECK-O3-NEXT:    movq (%rdi), %rcx
+; CHECK-O3-NEXT:    movq (%rsi), %rax
+; CHECK-O3-NEXT:    andq %rcx, %rax
+; CHECK-O3-NEXT:    retq
+  %v = load atomic i64, i64* %p1 unordered, align 8
+  %v2 = load atomic i64, i64* %p2 unordered, align 8
+  %ret = and i64 %v, %v2
+  ret i64 %ret
+}
+
+; Legal, as expected
+define i64 @load_fold_or1(i64* %p) {
+; CHECK-O0-LABEL: load_fold_or1:
+; CHECK-O0:       # %bb.0:
+; CHECK-O0-NEXT:    movq (%rdi), %rdi
+; CHECK-O0-NEXT:    orq $15, %rdi
+; CHECK-O0-NEXT:    movq %rdi, %rax
+; CHECK-O0-NEXT:    retq
+;
+; CHECK-O3-LABEL: load_fold_or1:
+; CHECK-O3:       # %bb.0:
+; CHECK-O3-NEXT:    movq (%rdi), %rax
+; CHECK-O3-NEXT:    orq $15, %rax
+; CHECK-O3-NEXT:    retq
+  %v = load atomic i64, i64* %p unordered, align 8
+  %ret = or i64 %v, 15
+  ret i64 %ret
+}
+
+; Legal, as expected
+define i64 @load_fold_or2(i64* %p, i64 %v2) {
+; CHECK-O0-LABEL: load_fold_or2:
+; CHECK-O0:       # %bb.0:
+; CHECK-O0-NEXT:    orq (%rdi), %rsi
+; CHECK-O0-NEXT:    movq %rsi, %rax
+; CHECK-O0-NEXT:    retq
+;
+; CHECK-O3-LABEL: load_fold_or2:
+; CHECK-O3:       # %bb.0:
+; CHECK-O3-NEXT:    movq (%rdi), %rax
+; CHECK-O3-NEXT:    orq %rsi, %rax
+; CHECK-O3-NEXT:    retq
+  %v = load atomic i64, i64* %p unordered, align 8
+  %ret = or i64 %v, %v2
+  ret i64 %ret
+}
+
+; Legal to fold (TODO)
+define i64 @load_fold_or3(i64* %p1, i64* %p2) {
+; CHECK-O0-LABEL: load_fold_or3:
+; CHECK-O0:       # %bb.0:
+; CHECK-O0-NEXT:    movq (%rdi), %rdi
+; CHECK-O0-NEXT:    orq (%rsi), %rdi
+; CHECK-O0-NEXT:    movq %rdi, %rax
+; CHECK-O0-NEXT:    retq
+;
+; CHECK-O3-LABEL: load_fold_or3:
+; CHECK-O3:       # %bb.0:
+; CHECK-O3-NEXT:    movq (%rdi), %rcx
+; CHECK-O3-NEXT:    movq (%rsi), %rax
+; CHECK-O3-NEXT:    orq %rcx, %rax
+; CHECK-O3-NEXT:    retq
+  %v = load atomic i64, i64* %p1 unordered, align 8
+  %v2 = load atomic i64, i64* %p2 unordered, align 8
+  %ret = or i64 %v, %v2
+  ret i64 %ret
+}
+
+; Legal, as expected
+define i64 @load_fold_xor1(i64* %p) {
+; CHECK-O0-LABEL: load_fold_xor1:
+; CHECK-O0:       # %bb.0:
+; CHECK-O0-NEXT:    movq (%rdi), %rdi
+; CHECK-O0-NEXT:    xorq $15, %rdi
+; CHECK-O0-NEXT:    movq %rdi, %rax
+; CHECK-O0-NEXT:    retq
+;
+; CHECK-O3-LABEL: load_fold_xor1:
+; CHECK-O3:       # %bb.0:
+; CHECK-O3-NEXT:    movq (%rdi), %rax
+; CHECK-O3-NEXT:    xorq $15, %rax
+; CHECK-O3-NEXT:    retq
+  %v = load atomic i64, i64* %p unordered, align 8
+  %ret = xor i64 %v, 15
+  ret i64 %ret
+}
+
+; Legal, as expected
+define i64 @load_fold_xor2(i64* %p, i64 %v2) {
+; CHECK-O0-LABEL: load_fold_xor2:
+; CHECK-O0:       # %bb.0:
+; CHECK-O0-NEXT:    xorq (%rdi), %rsi
+; CHECK-O0-NEXT:    movq %rsi, %rax
+; CHECK-O0-NEXT:    retq
+;
+; CHECK-O3-LABEL: load_fold_xor2:
+; CHECK-O3:       # %bb.0:
+; CHECK-O3-NEXT:    movq (%rdi), %rax
+; CHECK-O3-NEXT:    xorq %rsi, %rax
+; CHECK-O3-NEXT:    retq
+  %v = load atomic i64, i64* %p unordered, align 8
+  %ret = xor i64 %v, %v2
+  ret i64 %ret
+}
+
+; Legal to fold (TODO)
+define i64 @load_fold_xor3(i64* %p1, i64* %p2) {
+; CHECK-O0-LABEL: load_fold_xor3:
+; CHECK-O0:       # %bb.0:
+; CHECK-O0-NEXT:    movq (%rdi), %rdi
+; CHECK-O0-NEXT:    xorq (%rsi), %rdi
+; CHECK-O0-NEXT:    movq %rdi, %rax
+; CHECK-O0-NEXT:    retq
+;
+; CHECK-O3-LABEL: load_fold_xor3:
+; CHECK-O3:       # %bb.0:
+; CHECK-O3-NEXT:    movq (%rdi), %rcx
+; CHECK-O3-NEXT:    movq (%rsi), %rax
+; CHECK-O3-NEXT:    xorq %rcx, %rax
+; CHECK-O3-NEXT:    retq
+  %v = load atomic i64, i64* %p1 unordered, align 8
+  %v2 = load atomic i64, i64* %p2 unordered, align 8
+  %ret = xor i64 %v, %v2
+  ret i64 %ret
+}
+
+; Legal to fold (TODO)
+define i1 @load_fold_icmp1(i64* %p) {
+; CHECK-O0-LABEL: load_fold_icmp1:
+; CHECK-O0:       # %bb.0:
+; CHECK-O0-NEXT:    movq (%rdi), %rdi
+; CHECK-O0-NEXT:    subq $15, %rdi
+; CHECK-O0-NEXT:    sete %al
+; CHECK-O0-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-O0-NEXT:    retq
+;
+; CHECK-O3-LABEL: load_fold_icmp1:
+; CHECK-O3:       # %bb.0:
+; CHECK-O3-NEXT:    movq (%rdi), %rax
+; CHECK-O3-NEXT:    cmpq $15, %rax
+; CHECK-O3-NEXT:    sete %al
+; CHECK-O3-NEXT:    retq
+  %v = load atomic i64, i64* %p unordered, align 8
+  %ret = icmp eq i64 %v, 15
+  ret i1 %ret
+}
+
+; Legal to fold (TODO)
+define i1 @load_fold_icmp2(i64* %p, i64 %v2) {
+; CHECK-O0-LABEL: load_fold_icmp2:
+; CHECK-O0:       # %bb.0:
+; CHECK-O0-NEXT:    movq (%rdi), %rdi
+; CHECK-O0-NEXT:    subq %rsi, %rdi
+; CHECK-O0-NEXT:    sete %al
+; CHECK-O0-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-O0-NEXT:    retq
+;
+; CHECK-O3-LABEL: load_fold_icmp2:
+; CHECK-O3:       # %bb.0:
+; CHECK-O3-NEXT:    movq (%rdi), %rax
+; CHECK-O3-NEXT:    cmpq %rsi, %rax
+; CHECK-O3-NEXT:    sete %al
+; CHECK-O3-NEXT:    retq
+  %v = load atomic i64, i64* %p unordered, align 8
+  %ret = icmp eq i64 %v, %v2
+  ret i1 %ret
+}
+
+; Legal to fold (TODO)
+define i1 @load_fold_icmp3(i64* %p1, i64* %p2) {
+; CHECK-O0-LABEL: load_fold_icmp3:
+; CHECK-O0:       # %bb.0:
+; CHECK-O0-NEXT:    movq (%rdi), %rdi
+; CHECK-O0-NEXT:    movq (%rsi), %rsi
+; CHECK-O0-NEXT:    subq %rsi, %rdi
+; CHECK-O0-NEXT:    sete %al
+; CHECK-O0-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-O0-NEXT:    retq
+;
+; CHECK-O3-LABEL: load_fold_icmp3:
+; CHECK-O3:       # %bb.0:
+; CHECK-O3-NEXT:    movq (%rdi), %rax
+; CHECK-O3-NEXT:    movq (%rsi), %rcx
+; CHECK-O3-NEXT:    cmpq %rcx, %rax
+; CHECK-O3-NEXT:    sete %al
+; CHECK-O3-NEXT:    retq
+  %v = load atomic i64, i64* %p1 unordered, align 8
+  %v2 = load atomic i64, i64* %p2 unordered, align 8
+  %ret = icmp eq i64 %v, %v2
+  ret i1 %ret
+}
+
+
+;; The next batch of tests check for read-modify-write patterns
+;; Legally, it's okay to use a memory operand here as long as the operand
+;; is well aligned (i.e. doesn't cross a cache line boundary).  We are
+;; required not to narrow the store though!
+
+; Legal, as expected
+define void @rmw_fold_add1(i64* %p, i64 %v) {
+; CHECK-O0-LABEL: rmw_fold_add1:
+; CHECK-O0:       # %bb.0:
+; CHECK-O0-NEXT:    movq (%rdi), %rax
+; CHECK-O0-NEXT:    addq $15, %rax
+; CHECK-O0-NEXT:    movq %rax, (%rdi)
+; CHECK-O0-NEXT:    movq %rsi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-O0-NEXT:    retq
+;
+; CHECK-O3-LABEL: rmw_fold_add1:
+; CHECK-O3:       # %bb.0:
+; CHECK-O3-NEXT:    addq $15, (%rdi)
+; CHECK-O3-NEXT:    retq
+  %prev = load atomic i64, i64* %p unordered, align 8
+  %val = add i64 %prev, 15
+  store atomic i64 %val, i64* %p unordered, align 8
+  ret void
+}
+
+; Legal, as expected
+define void @rmw_fold_add2(i64* %p, i64 %v) {
+; CHECK-O0-LABEL: rmw_fold_add2:
+; CHECK-O0:       # %bb.0:
+; CHECK-O0-NEXT:    movq (%rdi), %rax
+; CHECK-O0-NEXT:    addq %rsi, %rax
+; CHECK-O0-NEXT:    movq %rax, (%rdi)
+; CHECK-O0-NEXT:    retq
+;
+; CHECK-O3-LABEL: rmw_fold_add2:
+; CHECK-O3:       # %bb.0:
+; CHECK-O3-NEXT:    addq %rsi, (%rdi)
+; CHECK-O3-NEXT:    retq
+  %prev = load atomic i64, i64* %p unordered, align 8
+  %val = add i64 %prev, %v
+  store atomic i64 %val, i64* %p unordered, align 8
+  ret void
+}
+
+; Legal, as expected
+define void @rmw_fold_sub1(i64* %p, i64 %v) {
+; CHECK-O0-LABEL: rmw_fold_sub1:
+; CHECK-O0:       # %bb.0:
+; CHECK-O0-NEXT:    movq (%rdi), %rax
+; CHECK-O0-NEXT:    addq $-15, %rax
+; CHECK-O0-NEXT:    movq %rax, (%rdi)
+; CHECK-O0-NEXT:    movq %rsi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-O0-NEXT:    retq
+;
+; CHECK-O3-LABEL: rmw_fold_sub1:
+; CHECK-O3:       # %bb.0:
+; CHECK-O3-NEXT:    addq $-15, (%rdi)
+; CHECK-O3-NEXT:    retq
+  %prev = load atomic i64, i64* %p unordered, align 8
+  %val = sub i64 %prev, 15
+  store atomic i64 %val, i64* %p unordered, align 8
+  ret void
+}
+
+; Legal, as expected
+define void @rmw_fold_sub2(i64* %p, i64 %v) {
+; CHECK-O0-LABEL: rmw_fold_sub2:
+; CHECK-O0:       # %bb.0:
+; CHECK-O0-NEXT:    movq (%rdi), %rax
+; CHECK-O0-NEXT:    subq %rsi, %rax
+; CHECK-O0-NEXT:    movq %rax, (%rdi)
+; CHECK-O0-NEXT:    retq
+;
+; CHECK-O3-LABEL: rmw_fold_sub2:
+; CHECK-O3:       # %bb.0:
+; CHECK-O3-NEXT:    subq %rsi, (%rdi)
+; CHECK-O3-NEXT:    retq
+  %prev = load atomic i64, i64* %p unordered, align 8
+  %val = sub i64 %prev, %v
+  store atomic i64 %val, i64* %p unordered, align 8
+  ret void
+}
+
+; Legal, as expected
+define void @rmw_fold_mul1(i64* %p, i64 %v) {
+; CHECK-O0-LABEL: rmw_fold_mul1:
+; CHECK-O0:       # %bb.0:
+; CHECK-O0-NEXT:    movq (%rdi), %rax
+; CHECK-O0-NEXT:    leaq (%rax,%rax,4), %rax
+; CHECK-O0-NEXT:    leaq (%rax,%rax,2), %rax
+; CHECK-O0-NEXT:    movq %rax, (%rdi)
+; CHECK-O0-NEXT:    movq %rsi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-O0-NEXT:    retq
+;
+; CHECK-O3-LABEL: rmw_fold_mul1:
+; CHECK-O3:       # %bb.0:
+; CHECK-O3-NEXT:    movq (%rdi), %rax
+; CHECK-O3-NEXT:    leaq (%rax,%rax,4), %rax
+; CHECK-O3-NEXT:    leaq (%rax,%rax,2), %rax
+; CHECK-O3-NEXT:    movq %rax, (%rdi)
+; CHECK-O3-NEXT:    retq
+  %prev = load atomic i64, i64* %p unordered, align 8
+  %val = mul i64 %prev, 15
+  store atomic i64 %val, i64* %p unordered, align 8
+  ret void
+}
+
+; Legal to fold (TODO)
+define void @rmw_fold_mul2(i64* %p, i64 %v) {
+; CHECK-O0-LABEL: rmw_fold_mul2:
+; CHECK-O0:       # %bb.0:
+; CHECK-O0-NEXT:    movq (%rdi), %rax
+; CHECK-O0-NEXT:    imulq %rsi, %rax
+; CHECK-O0-NEXT:    movq %rax, (%rdi)
+; CHECK-O0-NEXT:    retq
+;
+; CHECK-O3-LABEL: rmw_fold_mul2:
+; CHECK-O3:       # %bb.0:
+; CHECK-O3-NEXT:    movq (%rdi), %rax
+; CHECK-O3-NEXT:    imulq %rsi, %rax
+; CHECK-O3-NEXT:    movq %rax, (%rdi)
+; CHECK-O3-NEXT:    retq
+  %prev = load atomic i64, i64* %p unordered, align 8
+  %val = mul i64 %prev, %v
+  store atomic i64 %val, i64* %p unordered, align 8
+  ret void
+}
+
+; Legal, as expected
+define void @rmw_fold_sdiv1(i64* %p, i64 %v) {
+; CHECK-O0-LABEL: rmw_fold_sdiv1:
+; CHECK-O0:       # %bb.0:
+; CHECK-O0-NEXT:    movq (%rdi), %rax
+; CHECK-O0-NEXT:    movabsq $-8608480567731124087, %rcx # imm = 0x8888888888888889
+; CHECK-O0-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-O0-NEXT:    imulq %rcx
+; CHECK-O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
+; CHECK-O0-NEXT:    addq %rax, %rdx
+; CHECK-O0-NEXT:    movq %rdx, %rcx
+; CHECK-O0-NEXT:    shrq $63, %rcx
+; CHECK-O0-NEXT:    sarq $3, %rdx
+; CHECK-O0-NEXT:    addq %rcx, %rdx
+; CHECK-O0-NEXT:    movq %rdx, (%rdi)
+; CHECK-O0-NEXT:    movq %rsi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-O0-NEXT:    retq
+;
+; CHECK-O3-LABEL: rmw_fold_sdiv1:
+; CHECK-O3:       # %bb.0:
+; CHECK-O3-NEXT:    movq (%rdi), %rcx
+; CHECK-O3-NEXT:    movabsq $-8608480567731124087, %rdx # imm = 0x8888888888888889
+; CHECK-O3-NEXT:    movq %rcx, %rax
+; CHECK-O3-NEXT:    imulq %rdx
+; CHECK-O3-NEXT:    addq %rcx, %rdx
+; CHECK-O3-NEXT:    movq %rdx, %rax
+; CHECK-O3-NEXT:    shrq $63, %rax
+; CHECK-O3-NEXT:    sarq $3, %rdx
+; CHECK-O3-NEXT:    addq %rax, %rdx
+; CHECK-O3-NEXT:    movq %rdx, (%rdi)
+; CHECK-O3-NEXT:    retq
+  %prev = load atomic i64, i64* %p unordered, align 8
+  %val = sdiv i64 %prev, 15
+  store atomic i64 %val, i64* %p unordered, align 8
+  ret void
+}
+
+; Legal, as expected
+define void @rmw_fold_sdiv2(i64* %p, i64 %v) {
+; CHECK-O0-LABEL: rmw_fold_sdiv2:
+; CHECK-O0:       # %bb.0:
+; CHECK-O0-NEXT:    movq (%rdi), %rax
+; CHECK-O0-NEXT:    cqto
+; CHECK-O0-NEXT:    idivq %rsi
+; CHECK-O0-NEXT:    movq %rax, (%rdi)
+; CHECK-O0-NEXT:    retq
+;
+; CHECK-O3-LABEL: rmw_fold_sdiv2:
+; CHECK-O3:       # %bb.0:
+; CHECK-O3-NEXT:    movq (%rdi), %rax
+; CHECK-O3-NEXT:    cqto
+; CHECK-O3-NEXT:    idivq %rsi
+; CHECK-O3-NEXT:    movq %rax, (%rdi)
+; CHECK-O3-NEXT:    retq
+  %prev = load atomic i64, i64* %p unordered, align 8
+  %val = sdiv i64 %prev, %v
+  store atomic i64 %val, i64* %p unordered, align 8
+  ret void
+}
+
+; Legal, as expected
+define void @rmw_fold_udiv1(i64* %p, i64 %v) {
+; CHECK-O0-LABEL: rmw_fold_udiv1:
+; CHECK-O0:       # %bb.0:
+; CHECK-O0-NEXT:    movq (%rdi), %rax
+; CHECK-O0-NEXT:    movabsq $-8608480567731124087, %rcx # imm = 0x8888888888888889
+; CHECK-O0-NEXT:    mulq %rcx
+; CHECK-O0-NEXT:    shrq $3, %rdx
+; CHECK-O0-NEXT:    movq %rdx, (%rdi)
+; CHECK-O0-NEXT:    movq %rsi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-O0-NEXT:    retq
+;
+; CHECK-O3-LABEL: rmw_fold_udiv1:
+; CHECK-O3:       # %bb.0:
+; CHECK-O3-NEXT:    movq (%rdi), %rax
+; CHECK-O3-NEXT:    movabsq $-8608480567731124087, %rcx # imm = 0x8888888888888889
+; CHECK-O3-NEXT:    mulq %rcx
+; CHECK-O3-NEXT:    shrq $3, %rdx
+; CHECK-O3-NEXT:    movq %rdx, (%rdi)
+; CHECK-O3-NEXT:    retq
+  %prev = load atomic i64, i64* %p unordered, align 8
+  %val = udiv i64 %prev, 15
+  store atomic i64 %val, i64* %p unordered, align 8
+  ret void
+}
+
+; Legal, as expected
+define void @rmw_fold_udiv2(i64* %p, i64 %v) {
+; CHECK-O0-LABEL: rmw_fold_udiv2:
+; CHECK-O0:       # %bb.0:
+; CHECK-O0-NEXT:    movq (%rdi), %rax
+; CHECK-O0-NEXT:    xorl %ecx, %ecx
+; CHECK-O0-NEXT:    movl %ecx, %edx
+; CHECK-O0-NEXT:    divq %rsi
+; CHECK-O0-NEXT:    movq %rax, (%rdi)
+; CHECK-O0-NEXT:    retq
+;
+; CHECK-O3-LABEL: rmw_fold_udiv2:
+; CHECK-O3:       # %bb.0:
+; CHECK-O3-NEXT:    movq (%rdi), %rax
+; CHECK-O3-NEXT:    xorl %edx, %edx
+; CHECK-O3-NEXT:    divq %rsi
+; CHECK-O3-NEXT:    movq %rax, (%rdi)
+; CHECK-O3-NEXT:    retq
+  %prev = load atomic i64, i64* %p unordered, align 8
+  %val = udiv i64 %prev, %v
+  store atomic i64 %val, i64* %p unordered, align 8
+  ret void
+}
+
+; Legal, as expected
+define void @rmw_fold_srem1(i64* %p, i64 %v) {
+; CHECK-O0-LABEL: rmw_fold_srem1:
+; CHECK-O0:       # %bb.0:
+; CHECK-O0-NEXT:    movq (%rdi), %rax
+; CHECK-O0-NEXT:    movabsq $-8608480567731124087, %rcx # imm = 0x8888888888888889
+; CHECK-O0-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-O0-NEXT:    imulq %rcx
+; CHECK-O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
+; CHECK-O0-NEXT:    addq %rax, %rdx
+; CHECK-O0-NEXT:    movq %rdx, %rcx
+; CHECK-O0-NEXT:    shrq $63, %rcx
+; CHECK-O0-NEXT:    sarq $3, %rdx
+; CHECK-O0-NEXT:    addq %rcx, %rdx
+; CHECK-O0-NEXT:    leaq (%rdx,%rdx,4), %rcx
+; CHECK-O0-NEXT:    leaq (%rcx,%rcx,2), %rcx
+; CHECK-O0-NEXT:    subq %rcx, %rax
+; CHECK-O0-NEXT:    movq %rax, (%rdi)
+; CHECK-O0-NEXT:    movq %rsi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-O0-NEXT:    retq
+;
+; CHECK-O3-LABEL: rmw_fold_srem1:
+; CHECK-O3:       # %bb.0:
+; CHECK-O3-NEXT:    movq (%rdi), %rcx
+; CHECK-O3-NEXT:    movabsq $-8608480567731124087, %rdx # imm = 0x8888888888888889
+; CHECK-O3-NEXT:    movq %rcx, %rax
+; CHECK-O3-NEXT:    imulq %rdx
+; CHECK-O3-NEXT:    addq %rcx, %rdx
+; CHECK-O3-NEXT:    movq %rdx, %rax
+; CHECK-O3-NEXT:    shrq $63, %rax
+; CHECK-O3-NEXT:    sarq $3, %rdx
+; CHECK-O3-NEXT:    addq %rax, %rdx
+; CHECK-O3-NEXT:    leaq (%rdx,%rdx,4), %rax
+; CHECK-O3-NEXT:    leaq (%rax,%rax,2), %rax
+; CHECK-O3-NEXT:    subq %rax, %rcx
+; CHECK-O3-NEXT:    movq %rcx, (%rdi)
+; CHECK-O3-NEXT:    retq
+  %prev = load atomic i64, i64* %p unordered, align 8
+  %val = srem i64 %prev, 15
+  store atomic i64 %val, i64* %p unordered, align 8
+  ret void
+}
+
+; Legal, as expected
+define void @rmw_fold_srem2(i64* %p, i64 %v) {
+; CHECK-O0-LABEL: rmw_fold_srem2:
+; CHECK-O0:       # %bb.0:
+; CHECK-O0-NEXT:    movq (%rdi), %rax
+; CHECK-O0-NEXT:    cqto
+; CHECK-O0-NEXT:    idivq %rsi
+; CHECK-O0-NEXT:    movq %rdx, (%rdi)
+; CHECK-O0-NEXT:    retq
+;
+; CHECK-O3-LABEL: rmw_fold_srem2:
+; CHECK-O3:       # %bb.0:
+; CHECK-O3-NEXT:    movq (%rdi), %rax
+; CHECK-O3-NEXT:    cqto
+; CHECK-O3-NEXT:    idivq %rsi
+; CHECK-O3-NEXT:    movq %rdx, (%rdi)
+; CHECK-O3-NEXT:    retq
+  %prev = load atomic i64, i64* %p unordered, align 8
+  %val = srem i64 %prev, %v
+  store atomic i64 %val, i64* %p unordered, align 8
+  ret void
+}
+
+; Legal, as expected
+define void @rmw_fold_urem1(i64* %p, i64 %v) {
+; CHECK-O0-LABEL: rmw_fold_urem1:
+; CHECK-O0:       # %bb.0:
+; CHECK-O0-NEXT:    movq (%rdi), %rax
+; CHECK-O0-NEXT:    movabsq $-8608480567731124087, %rcx # imm = 0x8888888888888889
+; CHECK-O0-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-O0-NEXT:    mulq %rcx
+; CHECK-O0-NEXT:    shrq $3, %rdx
+; CHECK-O0-NEXT:    leaq (%rdx,%rdx,4), %rax
+; CHECK-O0-NEXT:    leaq (%rax,%rax,2), %rax
+; CHECK-O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
+; CHECK-O0-NEXT:    subq %rax, %rcx
+; CHECK-O0-NEXT:    movq %rcx, (%rdi)
+; CHECK-O0-NEXT:    movq %rsi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-O0-NEXT:    retq
+;
+; CHECK-O3-LABEL: rmw_fold_urem1:
+; CHECK-O3:       # %bb.0:
+; CHECK-O3-NEXT:    movq (%rdi), %rcx
+; CHECK-O3-NEXT:    movabsq $-8608480567731124087, %rdx # imm = 0x8888888888888889
+; CHECK-O3-NEXT:    movq %rcx, %rax
+; CHECK-O3-NEXT:    mulq %rdx
+; CHECK-O3-NEXT:    shrq $3, %rdx
+; CHECK-O3-NEXT:    leaq (%rdx,%rdx,4), %rax
+; CHECK-O3-NEXT:    leaq (%rax,%rax,2), %rax
+; CHECK-O3-NEXT:    subq %rax, %rcx
+; CHECK-O3-NEXT:    movq %rcx, (%rdi)
+; CHECK-O3-NEXT:    retq
+  %prev = load atomic i64, i64* %p unordered, align 8
+  %val = urem i64 %prev, 15
+  store atomic i64 %val, i64* %p unordered, align 8
+  ret void
+}
+
+; Legal, as expected
+define void @rmw_fold_urem2(i64* %p, i64 %v) {
+; CHECK-O0-LABEL: rmw_fold_urem2:
+; CHECK-O0:       # %bb.0:
+; CHECK-O0-NEXT:    movq (%rdi), %rax
+; CHECK-O0-NEXT:    xorl %ecx, %ecx
+; CHECK-O0-NEXT:    movl %ecx, %edx
+; CHECK-O0-NEXT:    divq %rsi
+; CHECK-O0-NEXT:    movq %rdx, (%rdi)
+; CHECK-O0-NEXT:    retq
+;
+; CHECK-O3-LABEL: rmw_fold_urem2:
+; CHECK-O3:       # %bb.0:
+; CHECK-O3-NEXT:    movq (%rdi), %rax
+; CHECK-O3-NEXT:    xorl %edx, %edx
+; CHECK-O3-NEXT:    divq %rsi
+; CHECK-O3-NEXT:    movq %rdx, (%rdi)
+; CHECK-O3-NEXT:    retq
+  %prev = load atomic i64, i64* %p unordered, align 8
+  %val = urem i64 %prev, %v
+  store atomic i64 %val, i64* %p unordered, align 8
+  ret void
+}
+
+; Legal to fold (TODO)
+define void @rmw_fold_shl1(i64* %p, i64 %v) {
+; CHECK-O0-LABEL: rmw_fold_shl1:
+; CHECK-O0:       # %bb.0:
+; CHECK-O0-NEXT:    movq (%rdi), %rax
+; CHECK-O0-NEXT:    shlq $15, %rax
+; CHECK-O0-NEXT:    movq %rax, (%rdi)
+; CHECK-O0-NEXT:    movq %rsi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-O0-NEXT:    retq
+;
+; CHECK-O3-LABEL: rmw_fold_shl1:
+; CHECK-O3:       # %bb.0:
+; CHECK-O3-NEXT:    movq (%rdi), %rax
+; CHECK-O3-NEXT:    shlq $15, %rax
+; CHECK-O3-NEXT:    movq %rax, (%rdi)
+; CHECK-O3-NEXT:    retq
+  %prev = load atomic i64, i64* %p unordered, align 8
+  %val = shl i64 %prev, 15
+  store atomic i64 %val, i64* %p unordered, align 8
+  ret void
+}
+
+; Legal to fold (TODO)
+define void @rmw_fold_shl2(i64* %p, i64 %v) {
+; CHECK-O0-LABEL: rmw_fold_shl2:
+; CHECK-O0:       # %bb.0:
+; CHECK-O0-NEXT:    movq (%rdi), %rax
+; CHECK-O0-NEXT:    movb %sil, %cl
+; CHECK-O0-NEXT:    shlq %cl, %rax
+; CHECK-O0-NEXT:    movq %rax, (%rdi)
+; CHECK-O0-NEXT:    retq
+;
+; CHECK-O3-LABEL: rmw_fold_shl2:
+; CHECK-O3:       # %bb.0:
+; CHECK-O3-NEXT:    movq %rsi, %rcx
+; CHECK-O3-NEXT:    movq (%rdi), %rax
+; CHECK-O3-NEXT:    # kill: def $cl killed $cl killed $rcx
+; CHECK-O3-NEXT:    shlq %cl, %rax
+; CHECK-O3-NEXT:    movq %rax, (%rdi)
+; CHECK-O3-NEXT:    retq
+  %prev = load atomic i64, i64* %p unordered, align 8
+  %val = shl i64 %prev, %v
+  store atomic i64 %val, i64* %p unordered, align 8
+  ret void
+}
+
+; Legal to fold (TODO)
+define void @rmw_fold_lshr1(i64* %p, i64 %v) {
+; CHECK-O0-LABEL: rmw_fold_lshr1:
+; CHECK-O0:       # %bb.0:
+; CHECK-O0-NEXT:    movq (%rdi), %rax
+; CHECK-O0-NEXT:    shrq $15, %rax
+; CHECK-O0-NEXT:    movq %rax, (%rdi)
+; CHECK-O0-NEXT:    movq %rsi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-O0-NEXT:    retq
+;
+; CHECK-O3-LABEL: rmw_fold_lshr1:
+; CHECK-O3:       # %bb.0:
+; CHECK-O3-NEXT:    movq (%rdi), %rax
+; CHECK-O3-NEXT:    shrq $15, %rax
+; CHECK-O3-NEXT:    movq %rax, (%rdi)
+; CHECK-O3-NEXT:    retq
+  %prev = load atomic i64, i64* %p unordered, align 8
+  %val = lshr i64 %prev, 15
+  store atomic i64 %val, i64* %p unordered, align 8
+  ret void
+}
+
+; Legal to fold (TODO)
+define void @rmw_fold_lshr2(i64* %p, i64 %v) {
+; CHECK-O0-LABEL: rmw_fold_lshr2:
+; CHECK-O0:       # %bb.0:
+; CHECK-O0-NEXT:    movq (%rdi), %rax
+; CHECK-O0-NEXT:    movb %sil, %cl
+; CHECK-O0-NEXT:    shrq %cl, %rax
+; CHECK-O0-NEXT:    movq %rax, (%rdi)
+; CHECK-O0-NEXT:    retq
+;
+; CHECK-O3-LABEL: rmw_fold_lshr2:
+; CHECK-O3:       # %bb.0:
+; CHECK-O3-NEXT:    movq %rsi, %rcx
+; CHECK-O3-NEXT:    movq (%rdi), %rax
+; CHECK-O3-NEXT:    # kill: def $cl killed $cl killed $rcx
+; CHECK-O3-NEXT:    shrq %cl, %rax
+; CHECK-O3-NEXT:    movq %rax, (%rdi)
+; CHECK-O3-NEXT:    retq
+  %prev = load atomic i64, i64* %p unordered, align 8
+  %val = lshr i64 %prev, %v
+  store atomic i64 %val, i64* %p unordered, align 8
+  ret void
+}
+
+; Legal to fold (TODO)
+define void @rmw_fold_ashr1(i64* %p, i64 %v) {
+; CHECK-O0-LABEL: rmw_fold_ashr1:
+; CHECK-O0:       # %bb.0:
+; CHECK-O0-NEXT:    movq (%rdi), %rax
+; CHECK-O0-NEXT:    sarq $15, %rax
+; CHECK-O0-NEXT:    movq %rax, (%rdi)
+; CHECK-O0-NEXT:    movq %rsi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-O0-NEXT:    retq
+;
+; CHECK-O3-LABEL: rmw_fold_ashr1:
+; CHECK-O3:       # %bb.0:
+; CHECK-O3-NEXT:    movq (%rdi), %rax
+; CHECK-O3-NEXT:    sarq $15, %rax
+; CHECK-O3-NEXT:    movq %rax, (%rdi)
+; CHECK-O3-NEXT:    retq
+  %prev = load atomic i64, i64* %p unordered, align 8
+  %val = ashr i64 %prev, 15
+  store atomic i64 %val, i64* %p unordered, align 8
+  ret void
+}
+
+; Legal to fold (TODO)
+define void @rmw_fold_ashr2(i64* %p, i64 %v) {
+; CHECK-O0-LABEL: rmw_fold_ashr2:
+; CHECK-O0:       # %bb.0:
+; CHECK-O0-NEXT:    movq (%rdi), %rax
+; CHECK-O0-NEXT:    movb %sil, %cl
+; CHECK-O0-NEXT:    sarq %cl, %rax
+; CHECK-O0-NEXT:    movq %rax, (%rdi)
+; CHECK-O0-NEXT:    retq
+;
+; CHECK-O3-LABEL: rmw_fold_ashr2:
+; CHECK-O3:       # %bb.0:
+; CHECK-O3-NEXT:    movq %rsi, %rcx
+; CHECK-O3-NEXT:    movq (%rdi), %rax
+; CHECK-O3-NEXT:    # kill: def $cl killed $cl killed $rcx
+; CHECK-O3-NEXT:    sarq %cl, %rax
+; CHECK-O3-NEXT:    movq %rax, (%rdi)
+; CHECK-O3-NEXT:    retq
+  %prev = load atomic i64, i64* %p unordered, align 8
+  %val = ashr i64 %prev, %v
+  store atomic i64 %val, i64* %p unordered, align 8
+  ret void
+}
+
+; Legal, as expected
+define void @rmw_fold_and1(i64* %p, i64 %v) {
+; CHECK-O0-LABEL: rmw_fold_and1:
+; CHECK-O0:       # %bb.0:
+; CHECK-O0-NEXT:    movq (%rdi), %rax
+; CHECK-O0-NEXT:    movl %eax, %ecx
+; CHECK-O0-NEXT:    andl $15, %ecx
+; CHECK-O0-NEXT:    movl %ecx, %eax
+; CHECK-O0-NEXT:    movq %rax, (%rdi)
+; CHECK-O0-NEXT:    movq %rsi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-O0-NEXT:    retq
+;
+; CHECK-O3-LABEL: rmw_fold_and1:
+; CHECK-O3:       # %bb.0:
+; CHECK-O3-NEXT:    andq $15, (%rdi)
+; CHECK-O3-NEXT:    retq
+  %prev = load atomic i64, i64* %p unordered, align 8
+  %val = and i64 %prev, 15
+  store atomic i64 %val, i64* %p unordered, align 8
+  ret void
+}
+
+; Legal, as expected
+define void @rmw_fold_and2(i64* %p, i64 %v) {
+; CHECK-O0-LABEL: rmw_fold_and2:
+; CHECK-O0:       # %bb.0:
+; CHECK-O0-NEXT:    movq (%rdi), %rax
+; CHECK-O0-NEXT:    andq %rsi, %rax
+; CHECK-O0-NEXT:    movq %rax, (%rdi)
+; CHECK-O0-NEXT:    retq
+;
+; CHECK-O3-LABEL: rmw_fold_and2:
+; CHECK-O3:       # %bb.0:
+; CHECK-O3-NEXT:    andq %rsi, (%rdi)
+; CHECK-O3-NEXT:    retq
+  %prev = load atomic i64, i64* %p unordered, align 8
+  %val = and i64 %prev, %v
+  store atomic i64 %val, i64* %p unordered, align 8
+  ret void
+}
+
+; Legal, as expected
+define void @rmw_fold_or1(i64* %p, i64 %v) {
+; CHECK-O0-LABEL: rmw_fold_or1:
+; CHECK-O0:       # %bb.0:
+; CHECK-O0-NEXT:    movq (%rdi), %rax
+; CHECK-O0-NEXT:    orq $15, %rax
+; CHECK-O0-NEXT:    movq %rax, (%rdi)
+; CHECK-O0-NEXT:    movq %rsi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-O0-NEXT:    retq
+;
+; CHECK-O3-LABEL: rmw_fold_or1:
+; CHECK-O3:       # %bb.0:
+; CHECK-O3-NEXT:    orq $15, (%rdi)
+; CHECK-O3-NEXT:    retq
+  %prev = load atomic i64, i64* %p unordered, align 8
+  %val = or i64 %prev, 15
+  store atomic i64 %val, i64* %p unordered, align 8
+  ret void
+}
+
+; Legal, as expected
+define void @rmw_fold_or2(i64* %p, i64 %v) {
+; CHECK-O0-LABEL: rmw_fold_or2:
+; CHECK-O0:       # %bb.0:
+; CHECK-O0-NEXT:    movq (%rdi), %rax
+; CHECK-O0-NEXT:    orq %rsi, %rax
+; CHECK-O0-NEXT:    movq %rax, (%rdi)
+; CHECK-O0-NEXT:    retq
+;
+; CHECK-O3-LABEL: rmw_fold_or2:
+; CHECK-O3:       # %bb.0:
+; CHECK-O3-NEXT:    orq %rsi, (%rdi)
+; CHECK-O3-NEXT:    retq
+  %prev = load atomic i64, i64* %p unordered, align 8
+  %val = or i64 %prev, %v
+  store atomic i64 %val, i64* %p unordered, align 8
+  ret void
+}
+
+; Legal, as expected
+define void @rmw_fold_xor1(i64* %p, i64 %v) {
+; CHECK-O0-LABEL: rmw_fold_xor1:
+; CHECK-O0:       # %bb.0:
+; CHECK-O0-NEXT:    movq (%rdi), %rax
+; CHECK-O0-NEXT:    xorq $15, %rax
+; CHECK-O0-NEXT:    movq %rax, (%rdi)
+; CHECK-O0-NEXT:    movq %rsi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-O0-NEXT:    retq
+;
+; CHECK-O3-LABEL: rmw_fold_xor1:
+; CHECK-O3:       # %bb.0:
+; CHECK-O3-NEXT:    xorq $15, (%rdi)
+; CHECK-O3-NEXT:    retq
+  %prev = load atomic i64, i64* %p unordered, align 8
+  %val = xor i64 %prev, 15
+  store atomic i64 %val, i64* %p unordered, align 8
+  ret void
+}
+
+; Legal, as expected
+define void @rmw_fold_xor2(i64* %p, i64 %v) {
+; CHECK-O0-LABEL: rmw_fold_xor2:
+; CHECK-O0:       # %bb.0:
+; CHECK-O0-NEXT:    movq (%rdi), %rax
+; CHECK-O0-NEXT:    xorq %rsi, %rax
+; CHECK-O0-NEXT:    movq %rax, (%rdi)
+; CHECK-O0-NEXT:    retq
+;
+; CHECK-O3-LABEL: rmw_fold_xor2:
+; CHECK-O3:       # %bb.0:
+; CHECK-O3-NEXT:    xorq %rsi, (%rdi)
+; CHECK-O3-NEXT:    retq
+  %prev = load atomic i64, i64* %p unordered, align 8
+  %val = xor i64 %prev, %v
+  store atomic i64 %val, i64* %p unordered, align 8
+  ret void
+}
+
+;; The next batch test truncations, in combination w/operations which could
+;; be folded against the memory operation.
+
+; Legal to reduce the load width (TODO)
+define i32 @fold_trunc(i64* %p) {
+; CHECK-O0-LABEL: fold_trunc:
+; CHECK-O0:       # %bb.0:
+; CHECK-O0-NEXT:    movq (%rdi), %rdi
+; CHECK-O0-NEXT:    movl %edi, %eax
+; CHECK-O0-NEXT:    retq
+;
+; CHECK-O3-LABEL: fold_trunc:
+; CHECK-O3:       # %bb.0:
+; CHECK-O3-NEXT:    movq (%rdi), %rax
+; CHECK-O3-NEXT:    # kill: def $eax killed $eax killed $rax
+; CHECK-O3-NEXT:    retq
+  %v = load atomic i64, i64* %p unordered, align 8
+  %ret = trunc i64 %v to i32
+  ret i32 %ret
+}
+
+; Legal to reduce the load width and fold the load (TODO)
+define i32 @fold_trunc_add(i64* %p, i32 %v2) {
+; CHECK-O0-LABEL: fold_trunc_add:
+; CHECK-O0:       # %bb.0:
+; CHECK-O0-NEXT:    movq (%rdi), %rdi
+; CHECK-O0-NEXT:    movl %edi, %eax
+; CHECK-O0-NEXT:    addl %esi, %eax
+; CHECK-O0-NEXT:    retq
+;
+; CHECK-O3-LABEL: fold_trunc_add:
+; CHECK-O3:       # %bb.0:
+; CHECK-O3-NEXT:    movq (%rdi), %rax
+; CHECK-O3-NEXT:    addl %esi, %eax
+; CHECK-O3-NEXT:    # kill: def $eax killed $eax killed $rax
+; CHECK-O3-NEXT:    retq
+  %v = load atomic i64, i64* %p unordered, align 8
+  %trunc = trunc i64 %v to i32
+  %ret = add i32 %trunc, %v2
+  ret i32 %ret
+}
+
+; Legal to reduce the load width and fold the load (TODO)
+define i32 @fold_trunc_and(i64* %p, i32 %v2) {
+; CHECK-O0-LABEL: fold_trunc_and:
+; CHECK-O0:       # %bb.0:
+; CHECK-O0-NEXT:    movq (%rdi), %rdi
+; CHECK-O0-NEXT:    movl %edi, %eax
+; CHECK-O0-NEXT:    andl %esi, %eax
+; CHECK-O0-NEXT:    retq
+;
+; CHECK-O3-LABEL: fold_trunc_and:
+; CHECK-O3:       # %bb.0:
+; CHECK-O3-NEXT:    movq (%rdi), %rax
+; CHECK-O3-NEXT:    andl %esi, %eax
+; CHECK-O3-NEXT:    # kill: def $eax killed $eax killed $rax
+; CHECK-O3-NEXT:    retq
+  %v = load atomic i64, i64* %p unordered, align 8
+  %trunc = trunc i64 %v to i32
+  %ret = and i32 %trunc, %v2
+  ret i32 %ret
+}
+
+; Legal to reduce the load width and fold the load (TODO)
+define i32 @fold_trunc_or(i64* %p, i32 %v2) {
+; CHECK-O0-LABEL: fold_trunc_or:
+; CHECK-O0:       # %bb.0:
+; CHECK-O0-NEXT:    movq (%rdi), %rdi
+; CHECK-O0-NEXT:    movl %edi, %eax
+; CHECK-O0-NEXT:    orl %esi, %eax
+; CHECK-O0-NEXT:    retq
+;
+; CHECK-O3-LABEL: fold_trunc_or:
+; CHECK-O3:       # %bb.0:
+; CHECK-O3-NEXT:    movq (%rdi), %rax
+; CHECK-O3-NEXT:    orl %esi, %eax
+; CHECK-O3-NEXT:    # kill: def $eax killed $eax killed $rax
+; CHECK-O3-NEXT:    retq
+  %v = load atomic i64, i64* %p unordered, align 8
+  %trunc = trunc i64 %v to i32
+  %ret = or i32 %trunc, %v2
+  ret i32 %ret
+}
+
+; It's tempting to split the wide load into two smaller byte loads
+; to reduce memory traffic, but this would be illegal for a atomic load
+define i32 @split_load(i64* %p) {
+; CHECK-O0-LABEL: split_load:
+; CHECK-O0:       # %bb.0:
+; CHECK-O0-NEXT:    movq (%rdi), %rdi
+; CHECK-O0-NEXT:    movb %dil, %al
+; CHECK-O0-NEXT:    shrq $32, %rdi
+; CHECK-O0-NEXT:    movb %dil, %cl
+; CHECK-O0-NEXT:    orb %cl, %al
+; CHECK-O0-NEXT:    movzbl %al, %eax
+; CHECK-O0-NEXT:    retq
+;
+; CHECK-O3-LABEL: split_load:
+; CHECK-O3:       # %bb.0:
+; CHECK-O3-NEXT:    movq (%rdi), %rax
+; CHECK-O3-NEXT:    movq %rax, %rcx
+; CHECK-O3-NEXT:    shrq $32, %rcx
+; CHECK-O3-NEXT:    orl %eax, %ecx
+; CHECK-O3-NEXT:    movzbl %cl, %eax
+; CHECK-O3-NEXT:    retq
+  %v = load atomic i64, i64* %p unordered, align 8
+  %b1 = trunc i64 %v to i8
+  %v.shift = lshr i64 %v, 32
+  %b2 = trunc i64 %v.shift to i8
+  %or = or i8 %b1, %b2
+  %ret = zext i8 %or to i32
+  ret i32 %ret
+}
+
+;; A collection of simple memory forwarding tests.  Nothing particular
+;; interesting semantic wise, just demonstrating obvious missed transforms.
+
+; Legal to forward and fold (TODO)
+define i64 @load_forwarding(i64* %p) {
+; CHECK-O0-LABEL: load_forwarding:
+; CHECK-O0:       # %bb.0:
+; CHECK-O0-NEXT:    movq (%rdi), %rax
+; CHECK-O0-NEXT:    orq (%rdi), %rax
+; CHECK-O0-NEXT:    retq
+;
+; CHECK-O3-LABEL: load_forwarding:
+; CHECK-O3:       # %bb.0:
+; CHECK-O3-NEXT:    movq (%rdi), %rcx
+; CHECK-O3-NEXT:    movq (%rdi), %rax
+; CHECK-O3-NEXT:    orq %rcx, %rax
+; CHECK-O3-NEXT:    retq
+  %v = load atomic i64, i64* %p unordered, align 8
+  %v2 = load atomic i64, i64* %p unordered, align 8
+  %ret = or i64 %v, %v2
+  ret i64 %ret
+}
+
+; Legal to forward (TODO)
+define i64 @store_forward(i64* %p, i64 %v) {
+; CHECK-O0-LABEL: store_forward:
+; CHECK-O0:       # %bb.0:
+; CHECK-O0-NEXT:    movq %rsi, (%rdi)
+; CHECK-O0-NEXT:    movq (%rdi), %rax
+; CHECK-O0-NEXT:    retq
+;
+; CHECK-O3-LABEL: store_forward:
+; CHECK-O3:       # %bb.0:
+; CHECK-O3-NEXT:    movq %rsi, (%rdi)
+; CHECK-O3-NEXT:    movq (%rdi), %rax
+; CHECK-O3-NEXT:    retq
+  store atomic i64 %v, i64* %p unordered, align 8
+  %ret = load atomic i64, i64* %p unordered, align 8
+  ret i64 %ret
+}
+
+; Legal to kill (TODO)
+define void @dead_writeback(i64* %p) {
+; CHECK-O0-LABEL: dead_writeback:
+; CHECK-O0:       # %bb.0:
+; CHECK-O0-NEXT:    movq (%rdi), %rax
+; CHECK-O0-NEXT:    movq %rax, (%rdi)
+; CHECK-O0-NEXT:    retq
+;
+; CHECK-O3-LABEL: dead_writeback:
+; CHECK-O3:       # %bb.0:
+; CHECK-O3-NEXT:    movq (%rdi), %rax
+; CHECK-O3-NEXT:    movq %rax, (%rdi)
+; CHECK-O3-NEXT:    retq
+  %v = load atomic i64, i64* %p unordered, align 8
+  store atomic i64 %v, i64* %p unordered, align 8
+  ret void
+}
+
+; Legal to kill (TODO)
+define void @dead_store(i64* %p, i64 %v) {
+; CHECK-O0-LABEL: dead_store:
+; CHECK-O0:       # %bb.0:
+; CHECK-O0-NEXT:    movq $0, (%rdi)
+; CHECK-O0-NEXT:    movq %rsi, (%rdi)
+; CHECK-O0-NEXT:    retq
+;
+; CHECK-O3-LABEL: dead_store:
+; CHECK-O3:       # %bb.0:
+; CHECK-O3-NEXT:    movq $0, (%rdi)
+; CHECK-O3-NEXT:    movq %rsi, (%rdi)
+; CHECK-O3-NEXT:    retq
+  store atomic i64 0, i64* %p unordered, align 8
+  store atomic i64 %v, i64* %p unordered, align 8
+  ret void
+}
+
+
+
diff --git a/test/CodeGen/X86/atomic128.ll b/test/CodeGen/X86/atomic128.ll
index 76475b3bcf7108ed2d41d696d69ff4c8e0922f23..c3fb5ab9125c268b2403562cc981bc717084b736 100644
--- a/test/CodeGen/X86/atomic128.ll
+++ b/test/CodeGen/X86/atomic128.ll
@@ -360,3 +360,27 @@ define void @atomic_store_relaxed(i128* %p, i128 %in) {
    store atomic i128 %in, i128* %p unordered, align 16
    ret void
 }
+
+
+@fsc128 = external global fp128
+
+define void @atomic_fetch_swapf128(fp128 %x) nounwind {
+; CHECK-LABEL: atomic_fetch_swapf128:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    movq %rsi, %rcx
+; CHECK-NEXT:    movq %rdi, %rbx
+; CHECK-NEXT:    movq _fsc128@{{.*}}(%rip), %rsi
+; CHECK-NEXT:    movq (%rsi), %rax
+; CHECK-NEXT:    movq 8(%rsi), %rdx
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  LBB14_1: ## %atomicrmw.start
+; CHECK-NEXT:    ## =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    lock cmpxchg16b (%rsi)
+; CHECK-NEXT:    jne LBB14_1
+; CHECK-NEXT:  ## %bb.2: ## %atomicrmw.end
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    retq
+  %t1 = atomicrmw xchg fp128* @fsc128, fp128 %x acquire
+  ret void
+}
diff --git a/test/CodeGen/X86/atomic16.ll b/test/CodeGen/X86/atomic16.ll
index 90716cc3984fba6e693de58f4bf56aabc3ecf3c6..c96160f9a8f3600f1c68e56a4ca0f63174ada704 100644
--- a/test/CodeGen/X86/atomic16.ll
+++ b/test/CodeGen/X86/atomic16.ll
@@ -2,6 +2,7 @@
 ; RUN: llc < %s -O0 -mtriple=i386-unknown-unknown -mcpu=corei7 -verify-machineinstrs | FileCheck %s --check-prefix X32
 
 @sc16 = external global i16
+@fsc16 = external global half
 
 define void @atomic_fetch_add16() nounwind {
 ; X64-LABEL:   atomic_fetch_add16
@@ -273,3 +274,14 @@ define void @atomic_fetch_swap16(i16 %x) nounwind {
 ; X64:       ret
 ; X32:       ret
 }
+
+define void @atomic_fetch_swapf16(half %x) nounwind {
+  %t1 = atomicrmw xchg half* @fsc16, half %x acquire
+; X64-NOT:   lock
+; X64:       xchgw
+; X32-NOT:   lock
+; X32:       xchgw
+  ret void
+; X64:       ret
+; X32:       ret
+}
diff --git a/test/CodeGen/X86/atomic32.ll b/test/CodeGen/X86/atomic32.ll
index 519b169c0f9e424fff90253bc8bddcbc579b864e..3a8038a2d76b6512df51ac01f24bcb5ff6e94753 100644
--- a/test/CodeGen/X86/atomic32.ll
+++ b/test/CodeGen/X86/atomic32.ll
@@ -4,6 +4,7 @@
 ; RUN: llc < %s -O0 -mtriple=i686-unknown-unknown -mcpu=corei7 -mattr=-cmov,-sse -verify-machineinstrs | FileCheck %s --check-prefixes=X86,X86-NOCMOV
 
 @sc32 = external global i32
+@fsc32 = external global float
 
 define void @atomic_fetch_add32() nounwind {
 ; X64-LABEL: atomic_fetch_add32:
@@ -708,3 +709,35 @@ define void @atomic_fetch_swap32(i32 %x) nounwind {
   %t1 = atomicrmw xchg i32* @sc32, i32 %x acquire
   ret void
 }
+
+define void @atomic_fetch_swapf32(float %x) nounwind {
+; X64-LABEL: atomic_fetch_swapf32:
+; X64:       # %bb.0:
+; X64-NEXT:    movd %xmm0, %eax
+; X64-NEXT:    xchgl %eax, {{.*}}(%rip)
+; X64-NEXT:    movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; X64-NEXT:    retq
+;
+; X86-CMOV-LABEL: atomic_fetch_swapf32:
+; X86-CMOV:       # %bb.0:
+; X86-CMOV-NEXT:    pushl %eax
+; X86-CMOV-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86-CMOV-NEXT:    movd %xmm0, %eax
+; X86-CMOV-NEXT:    xchgl %eax, fsc32
+; X86-CMOV-NEXT:    movl %eax, (%esp) # 4-byte Spill
+; X86-CMOV-NEXT:    popl %eax
+; X86-CMOV-NEXT:    retl
+;
+; X86-NOCMOV-LABEL: atomic_fetch_swapf32:
+; X86-NOCMOV:       # %bb.0:
+; X86-NOCMOV-NEXT:    subl $8, %esp
+; X86-NOCMOV-NEXT:    flds {{[0-9]+}}(%esp)
+; X86-NOCMOV-NEXT:    fstps {{[0-9]+}}(%esp)
+; X86-NOCMOV-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NOCMOV-NEXT:    xchgl %eax, fsc32
+; X86-NOCMOV-NEXT:    movl %eax, (%esp) # 4-byte Spill
+; X86-NOCMOV-NEXT:    addl $8, %esp
+; X86-NOCMOV-NEXT:    retl
+  %t1 = atomicrmw xchg float* @fsc32, float %x acquire
+  ret void
+}
diff --git a/test/CodeGen/X86/atomic64.ll b/test/CodeGen/X86/atomic64.ll
index 3df34af9bd778f4954e5ce32449d6cf97618c5e2..104d159565e4f268af9168e957ffbfbf21cd51dd 100644
--- a/test/CodeGen/X86/atomic64.ll
+++ b/test/CodeGen/X86/atomic64.ll
@@ -1,6 +1,7 @@
 ; RUN: llc < %s -O0 -mtriple=x86_64-- -mcpu=corei7 -verify-machineinstrs | FileCheck %s --check-prefix X64
 
 @sc64 = external global i64
+@fsc64 = external global double
 
 define void @atomic_fetch_add64() nounwind {
 ; X64-LABEL:   atomic_fetch_add64:
@@ -233,3 +234,16 @@ define void @atomic_fetch_swap64(i64 %x) nounwind {
 ; X64:       ret
 ; X32:       ret
 }
+
+define void @atomic_fetch_swapf64(double %x) nounwind {
+; X64-LABEL:   atomic_fetch_swapf64:
+; X32-LABEL:   atomic_fetch_swapf64:
+  %t1 = atomicrmw xchg double* @fsc64, double %x acquire
+; X64-NOT:   lock
+; X64:       xchgq
+; X32:       lock
+; X32:       xchg8b
+  ret void
+; X64:       ret
+; X32:       ret
+}
diff --git a/test/CodeGen/X86/atomic_add.ll b/test/CodeGen/X86/atomic_add.ll
deleted file mode 100644
index 86d950abc07d618500ec53cd060b30c6da850f3a..0000000000000000000000000000000000000000
--- a/test/CodeGen/X86/atomic_add.ll
+++ /dev/null
@@ -1,219 +0,0 @@
-; RUN: llc < %s -mtriple=x86_64-- -verify-machineinstrs | FileCheck %s
-; RUN: llc < %s -mtriple=x86_64-- -mattr=slow-incdec -verify-machineinstrs | FileCheck %s --check-prefix SLOW_INC
-
-; rdar://7103704
-
-define void @sub1(i32* nocapture %p, i32 %v) nounwind ssp {
-entry:
-; CHECK-LABEL: sub1:
-; CHECK: subl
-  %0 = atomicrmw sub i32* %p, i32 %v monotonic
-  ret void
-}
-
-define void @inc4(i64* nocapture %p) nounwind ssp {
-entry:
-; CHECK-LABEL: inc4:
-; CHECK: incq
-; SLOW_INC-LABEL: inc4:
-; SLOW_INC-NOT: incq
-  %0 = atomicrmw add i64* %p, i64 1 monotonic
-  ret void
-}
-
-define void @add8(i64* nocapture %p) nounwind ssp {
-entry:
-; CHECK-LABEL: add8:
-; CHECK: addq $2
-  %0 = atomicrmw add i64* %p, i64 2 monotonic
-  ret void
-}
-
-define void @add4(i64* nocapture %p, i32 %v) nounwind ssp {
-entry:
-; CHECK-LABEL: add4:
-; CHECK: addq
-  %0 = sext i32 %v to i64		; <i64> [#uses=1]
-  %1 = atomicrmw add i64* %p, i64 %0 monotonic
-  ret void
-}
-
-define void @inc3(i8* nocapture %p) nounwind ssp {
-entry:
-; CHECK-LABEL: inc3:
-; CHECK: incb
-; SLOW_INC-LABEL: inc3:
-; SLOW_INC-NOT: incb
-  %0 = atomicrmw add i8* %p, i8 1 monotonic
-  ret void
-}
-
-define void @add7(i8* nocapture %p) nounwind ssp {
-entry:
-; CHECK-LABEL: add7:
-; CHECK: addb $2
-  %0 = atomicrmw add i8* %p, i8 2 monotonic
-  ret void
-}
-
-define void @add3(i8* nocapture %p, i32 %v) nounwind ssp {
-entry:
-; CHECK-LABEL: add3:
-; CHECK: addb
-  %0 = trunc i32 %v to i8		; <i8> [#uses=1]
-  %1 = atomicrmw add i8* %p, i8 %0 monotonic
-  ret void
-}
-
-define void @inc2(i16* nocapture %p) nounwind ssp {
-entry:
-; CHECK-LABEL: inc2:
-; CHECK: incw
-; SLOW_INC-LABEL: inc2:
-; SLOW_INC-NOT: incw
-  %0 = atomicrmw add i16* %p, i16 1 monotonic
-  ret void
-}
-
-define void @add6(i16* nocapture %p) nounwind ssp {
-entry:
-; CHECK-LABEL: add6:
-; CHECK: addw $2
-  %0 = atomicrmw add i16* %p, i16 2 monotonic
-  ret void
-}
-
-define void @add2(i16* nocapture %p, i32 %v) nounwind ssp {
-entry:
-; CHECK-LABEL: add2:
-; CHECK: addw
-	%0 = trunc i32 %v to i16		; <i16> [#uses=1]
-  %1 = atomicrmw add i16* %p, i16 %0 monotonic
-  ret void
-}
-
-define void @inc1(i32* nocapture %p) nounwind ssp {
-entry:
-; CHECK-LABEL: inc1:
-; CHECK: incl
-; SLOW_INC-LABEL: inc1:
-; SLOW_INC-NOT: incl
-  %0 = atomicrmw add i32* %p, i32 1 monotonic
-  ret void
-}
-
-define void @add5(i32* nocapture %p) nounwind ssp {
-entry:
-; CHECK-LABEL: add5:
-; CHECK: addl $2
-  %0 = atomicrmw add i32* %p, i32 2 monotonic
-  ret void
-}
-
-define void @add1(i32* nocapture %p, i32 %v) nounwind ssp {
-entry:
-; CHECK-LABEL: add1:
-; CHECK: addl
-  %0 = atomicrmw add i32* %p, i32 %v monotonic
-  ret void
-}
-
-define void @dec4(i64* nocapture %p) nounwind ssp {
-entry:
-; CHECK-LABEL: dec4:
-; CHECK: decq
-; SLOW_INC-LABEL: dec4:
-; SLOW_INC-NOT: decq
-  %0 = atomicrmw sub i64* %p, i64 1 monotonic
-  ret void
-}
-
-define void @sub8(i64* nocapture %p) nounwind ssp {
-entry:
-; CHECK-LABEL: sub8:
-; CHECK: subq $2
-  %0 = atomicrmw sub i64* %p, i64 2 monotonic
-  ret void
-}
-
-define void @sub4(i64* nocapture %p, i32 %v) nounwind ssp {
-entry:
-; CHECK-LABEL: sub4:
-; CHECK: subq
-	%0 = sext i32 %v to i64		; <i64> [#uses=1]
-  %1 = atomicrmw sub i64* %p, i64 %0 monotonic
-  ret void
-}
-
-define void @dec3(i8* nocapture %p) nounwind ssp {
-entry:
-; CHECK-LABEL: dec3:
-; CHECK: decb
-; SLOW_INC-LABEL: dec3:
-; SLOW_INC-NOT: decb
-  %0 = atomicrmw sub i8* %p, i8 1 monotonic
-  ret void
-}
-
-define void @sub7(i8* nocapture %p) nounwind ssp {
-entry:
-; CHECK-LABEL: sub7:
-; CHECK: subb $2
-  %0 = atomicrmw sub i8* %p, i8 2 monotonic
-  ret void
-}
-
-define void @sub3(i8* nocapture %p, i32 %v) nounwind ssp {
-entry:
-; CHECK-LABEL: sub3:
-; CHECK: subb
-	%0 = trunc i32 %v to i8		; <i8> [#uses=1]
-  %1 = atomicrmw sub i8* %p, i8 %0 monotonic
-  ret void
-}
-
-define void @dec2(i16* nocapture %p) nounwind ssp {
-entry:
-; CHECK-LABEL: dec2:
-; CHECK: decw
-; SLOW_INC-LABEL: dec2:
-; SLOW_INC-NOT: decw
-  %0 = atomicrmw sub i16* %p, i16 1 monotonic
-  ret void
-}
-
-define void @sub6(i16* nocapture %p) nounwind ssp {
-entry:
-; CHECK-LABEL: sub6:
-; CHECK: subw $2
-  %0 = atomicrmw sub i16* %p, i16 2 monotonic
-  ret void
-}
-
-define void @sub2(i16* nocapture %p, i32 %v) nounwind ssp {
-entry:
-; CHECK-LABEL: sub2:
-; CHECK-NOT: negl
-; CHECK: subw
-	%0 = trunc i32 %v to i16		; <i16> [#uses=1]
-  %1 = atomicrmw sub i16* %p, i16 %0 monotonic
-  ret void
-}
-
-define void @dec1(i32* nocapture %p) nounwind ssp {
-entry:
-; CHECK-LABEL: dec1:
-; CHECK: decl
-; SLOW_INC-LABEL: dec1:
-; SLOW_INC-NOT: decl
-  %0 = atomicrmw sub i32* %p, i32 1 monotonic
-  ret void
-}
-
-define void @sub5(i32* nocapture %p) nounwind ssp {
-entry:
-; CHECK-LABEL: sub5:
-; CHECK: subl $2
-  %0 = atomicrmw sub i32* %p, i32 2 monotonic
-  ret void
-}
diff --git a/test/CodeGen/X86/avg.ll b/test/CodeGen/X86/avg.ll
index 93b7c3350e0998d2bf1eb63ff6a115a5ad73adff..f9678b92f666ccab45697cbe8ed75a20de3df72a 100644
--- a/test/CodeGen/X86/avg.ll
+++ b/test/CodeGen/X86/avg.ll
@@ -268,115 +268,112 @@ define void @avg_v48i8(<48 x i8>* %a, <48 x i8>* %b) nounwind {
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vmovdqa (%rdi), %xmm0
 ; AVX1-NEXT:    vmovdqa 16(%rdi), %xmm1
-; AVX1-NEXT:    vmovdqa 32(%rdi), %xmm2
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[2,3,0,1]
-; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm5 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[3,3,0,1]
-; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm7 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[1,1,2,3]
-; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
+; AVX1-NEXT:    vmovdqa 32(%rdi), %xmm4
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[2,3,0,1]
+; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm5 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[3,3,0,1]
+; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm6 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,2,3]
+; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm7 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm0[2,3,0,1]
-; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm14 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero,xmm4[2],zero,zero,zero,xmm4[3],zero,zero,zero
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[3,3,0,1]
-; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm13 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
-; AVX1-NEXT:    vmovdqa (%rdi), %xmm6
-; AVX1-NEXT:    vmovdqa 32(%rdi), %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm6[1,1,2,3]
-; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm12 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero,xmm4[2],zero,zero,zero,xmm4[3],zero,zero,zero
-; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm11 = xmm6[0],zero,zero,zero,xmm6[1],zero,zero,zero,xmm6[2],zero,zero,zero,xmm6[3],zero,zero,zero
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm2[2,3,0,1]
-; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm15 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero,xmm4[2],zero,zero,zero,xmm4[3],zero,zero,zero
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[3,3,0,1]
-; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero
-; AVX1-NEXT:    vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm0[1,1,2,3]
-; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm2 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero,xmm4[2],zero,zero,zero,xmm4[3],zero,zero,zero
-; AVX1-NEXT:    vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm15 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[3,3,0,1]
+; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm10 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm14 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
+; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm12 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm4[2,3,0,1]
+; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
+; AVX1-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm4[3,3,0,1]
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
 ; AVX1-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm4[1,1,2,3]
+; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
+; AVX1-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero,xmm4[2],zero,zero,zero,xmm4[3],zero,zero,zero
+; AVX1-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; AVX1-NEXT:    vmovdqa (%rsi), %xmm0
 ; AVX1-NEXT:    vmovdqa 16(%rsi), %xmm4
-; AVX1-NEXT:    vmovdqa 32(%rsi), %xmm6
+; AVX1-NEXT:    vmovdqa 32(%rsi), %xmm3
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm4[2,3,0,1]
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero
-; AVX1-NEXT:    vpaddd %xmm2, %xmm5, %xmm10
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm4[3,3,0,1]
-; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero
-; AVX1-NEXT:    vpaddd %xmm2, %xmm7, %xmm9
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm7 = xmm4[1,1,2,3]
-; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm7 = xmm7[0],zero,zero,zero,xmm7[1],zero,zero,zero,xmm7[2],zero,zero,zero,xmm7[3],zero,zero,zero
-; AVX1-NEXT:    vpaddd %xmm7, %xmm3, %xmm8
+; AVX1-NEXT:    vpaddd %xmm2, %xmm5, %xmm13
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm4[3,3,0,1]
+; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm5 = xmm5[0],zero,zero,zero,xmm5[1],zero,zero,zero,xmm5[2],zero,zero,zero,xmm5[3],zero,zero,zero
+; AVX1-NEXT:    vpaddd %xmm5, %xmm6, %xmm11
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm6 = xmm4[1,1,2,3]
+; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm6 = xmm6[0],zero,zero,zero,xmm6[1],zero,zero,zero,xmm6[2],zero,zero,zero,xmm6[3],zero,zero,zero
+; AVX1-NEXT:    vpaddd %xmm6, %xmm7, %xmm9
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero,xmm4[2],zero,zero,zero,xmm4[3],zero,zero,zero
-; AVX1-NEXT:    vpaddd %xmm4, %xmm1, %xmm1
+; AVX1-NEXT:    vpaddd %xmm4, %xmm1, %xmm8
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm0[2,3,0,1]
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero,xmm4[2],zero,zero,zero,xmm4[3],zero,zero,zero
-; AVX1-NEXT:    vpaddd %xmm4, %xmm14, %xmm14
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[3,3,0,1]
-; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
-; AVX1-NEXT:    vpaddd %xmm0, %xmm13, %xmm13
-; AVX1-NEXT:    vmovdqa (%rsi), %xmm7
-; AVX1-NEXT:    vmovdqa 32(%rsi), %xmm4
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm7[1,1,2,3]
-; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm5 = xmm5[0],zero,zero,zero,xmm5[1],zero,zero,zero,xmm5[2],zero,zero,zero,xmm5[3],zero,zero,zero
-; AVX1-NEXT:    vpaddd %xmm5, %xmm12, %xmm12
+; AVX1-NEXT:    vpaddd %xmm4, %xmm15, %xmm15
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm7 = xmm0[3,3,0,1]
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm7 = xmm7[0],zero,zero,zero,xmm7[1],zero,zero,zero,xmm7[2],zero,zero,zero,xmm7[3],zero,zero,zero
-; AVX1-NEXT:    vpaddd %xmm7, %xmm11, %xmm7
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm6[2,3,0,1]
+; AVX1-NEXT:    vpaddd %xmm7, %xmm10, %xmm7
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[1,1,2,3]
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero
-; AVX1-NEXT:    vpaddd %xmm2, %xmm15, %xmm2
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm6 = xmm6[3,3,0,1]
+; AVX1-NEXT:    vpaddd %xmm2, %xmm14, %xmm14
+; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
+; AVX1-NEXT:    vpaddd %xmm0, %xmm12, %xmm12
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm3[2,3,0,1]
+; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm5 = xmm5[0],zero,zero,zero,xmm5[1],zero,zero,zero,xmm5[2],zero,zero,zero,xmm5[3],zero,zero,zero
+; AVX1-NEXT:    vpaddd {{[-0-9]+}}(%r{{[sb]}}p), %xmm5, %xmm5 # 16-byte Folded Reload
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm6 = xmm3[3,3,0,1]
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm6 = xmm6[0],zero,zero,zero,xmm6[1],zero,zero,zero,xmm6[2],zero,zero,zero,xmm6[3],zero,zero,zero
 ; AVX1-NEXT:    vpaddd {{[-0-9]+}}(%r{{[sb]}}p), %xmm6, %xmm6 # 16-byte Folded Reload
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm4[1,1,2,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm3[1,1,2,3]
+; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
+; AVX1-NEXT:    vpaddd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm1 # 16-byte Folded Reload
 ; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
 ; AVX1-NEXT:    vpaddd {{[-0-9]+}}(%r{{[sb]}}p), %xmm3, %xmm3 # 16-byte Folded Reload
-; AVX1-NEXT:    vpmovzxbd {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero,xmm4[2],zero,zero,zero,xmm4[3],zero,zero,zero
-; AVX1-NEXT:    vpaddd {{[-0-9]+}}(%r{{[sb]}}p), %xmm4, %xmm4 # 16-byte Folded Reload
-; AVX1-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
-; AVX1-NEXT:    vpsubd %xmm0, %xmm10, %xmm10
-; AVX1-NEXT:    vpsubd %xmm0, %xmm9, %xmm9
-; AVX1-NEXT:    vpsubd %xmm0, %xmm8, %xmm8
-; AVX1-NEXT:    vpsubd %xmm0, %xmm1, %xmm11
-; AVX1-NEXT:    vpsubd %xmm0, %xmm14, %xmm14
-; AVX1-NEXT:    vpsubd %xmm0, %xmm13, %xmm5
-; AVX1-NEXT:    vpsubd %xmm0, %xmm12, %xmm1
-; AVX1-NEXT:    vpsubd %xmm0, %xmm7, %xmm7
-; AVX1-NEXT:    vpsubd %xmm0, %xmm2, %xmm2
-; AVX1-NEXT:    vpsubd %xmm0, %xmm6, %xmm6
-; AVX1-NEXT:    vpsubd %xmm0, %xmm3, %xmm3
-; AVX1-NEXT:    vpsubd %xmm0, %xmm4, %xmm0
-; AVX1-NEXT:    vpsrld $1, %xmm0, %xmm0
+; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
+; AVX1-NEXT:    vpsubd %xmm4, %xmm13, %xmm10
+; AVX1-NEXT:    vpsubd %xmm4, %xmm11, %xmm11
+; AVX1-NEXT:    vpsubd %xmm4, %xmm9, %xmm9
+; AVX1-NEXT:    vpsubd %xmm4, %xmm8, %xmm8
+; AVX1-NEXT:    vpsubd %xmm4, %xmm15, %xmm13
+; AVX1-NEXT:    vpsubd %xmm4, %xmm7, %xmm7
+; AVX1-NEXT:    vpsubd %xmm4, %xmm14, %xmm0
+; AVX1-NEXT:    vpsubd %xmm4, %xmm12, %xmm2
+; AVX1-NEXT:    vpsubd %xmm4, %xmm5, %xmm5
+; AVX1-NEXT:    vpsubd %xmm4, %xmm6, %xmm6
+; AVX1-NEXT:    vpsubd %xmm4, %xmm1, %xmm1
+; AVX1-NEXT:    vpsubd %xmm4, %xmm3, %xmm3
 ; AVX1-NEXT:    vpsrld $1, %xmm3, %xmm3
-; AVX1-NEXT:    vpackusdw %xmm3, %xmm0, %xmm0
-; AVX1-NEXT:    vpsrld $1, %xmm6, %xmm3
-; AVX1-NEXT:    vpsrld $1, %xmm2, %xmm2
-; AVX1-NEXT:    vpackusdw %xmm3, %xmm2, %xmm2
-; AVX1-NEXT:    vpsrld $1, %xmm7, %xmm3
 ; AVX1-NEXT:    vpsrld $1, %xmm1, %xmm1
 ; AVX1-NEXT:    vpackusdw %xmm1, %xmm3, %xmm1
-; AVX1-NEXT:    vpsrld $1, %xmm5, %xmm3
-; AVX1-NEXT:    vpsrld $1, %xmm14, %xmm4
+; AVX1-NEXT:    vpsrld $1, %xmm6, %xmm3
+; AVX1-NEXT:    vpsrld $1, %xmm5, %xmm4
 ; AVX1-NEXT:    vpackusdw %xmm3, %xmm4, %xmm3
-; AVX1-NEXT:    vpsrld $1, %xmm11, %xmm4
-; AVX1-NEXT:    vpsrld $1, %xmm8, %xmm5
-; AVX1-NEXT:    vpackusdw %xmm5, %xmm4, %xmm4
+; AVX1-NEXT:    vpsrld $1, %xmm2, %xmm2
+; AVX1-NEXT:    vpsrld $1, %xmm0, %xmm0
+; AVX1-NEXT:    vpackusdw %xmm0, %xmm2, %xmm0
+; AVX1-NEXT:    vpsrld $1, %xmm7, %xmm2
+; AVX1-NEXT:    vpsrld $1, %xmm13, %xmm4
+; AVX1-NEXT:    vpackusdw %xmm2, %xmm4, %xmm2
+; AVX1-NEXT:    vpsrld $1, %xmm8, %xmm4
 ; AVX1-NEXT:    vpsrld $1, %xmm9, %xmm5
+; AVX1-NEXT:    vpackusdw %xmm5, %xmm4, %xmm4
+; AVX1-NEXT:    vpsrld $1, %xmm11, %xmm5
 ; AVX1-NEXT:    vpsrld $1, %xmm10, %xmm6
 ; AVX1-NEXT:    vpackusdw %xmm5, %xmm6, %xmm5
 ; AVX1-NEXT:    vmovdqa {{.*#+}} xmm6 = [255,255,255,255,255,255,255,255]
 ; AVX1-NEXT:    vpand %xmm6, %xmm5, %xmm5
 ; AVX1-NEXT:    vpand %xmm6, %xmm4, %xmm4
 ; AVX1-NEXT:    vpackuswb %xmm5, %xmm4, %xmm4
-; AVX1-NEXT:    vpand %xmm6, %xmm3, %xmm3
-; AVX1-NEXT:    vpand %xmm6, %xmm1, %xmm1
-; AVX1-NEXT:    vpackuswb %xmm3, %xmm1, %xmm1
-; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm1, %ymm1
 ; AVX1-NEXT:    vpand %xmm6, %xmm2, %xmm2
 ; AVX1-NEXT:    vpand %xmm6, %xmm0, %xmm0
 ; AVX1-NEXT:    vpackuswb %xmm2, %xmm0, %xmm0
-; AVX1-NEXT:    vmovdqu %xmm0, (%rax)
-; AVX1-NEXT:    vmovups %ymm1, (%rax)
+; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm0, %ymm0
+; AVX1-NEXT:    vpand %xmm6, %xmm3, %xmm2
+; AVX1-NEXT:    vpand %xmm6, %xmm1, %xmm1
+; AVX1-NEXT:    vpackuswb %xmm2, %xmm1, %xmm1
+; AVX1-NEXT:    vmovdqu %xmm1, (%rax)
+; AVX1-NEXT:    vmovups %ymm0, (%rax)
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
 ;
@@ -385,38 +382,42 @@ define void @avg_v48i8(<48 x i8>* %a, <48 x i8>* %b) nounwind {
 ; AVX2-NEXT:    vpmovzxbd {{.*#+}} ymm0 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero
 ; AVX2-NEXT:    vpbroadcastq 24(%rdi), %ymm1
 ; AVX2-NEXT:    vpmovzxbd {{.*#+}} ymm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero
-; AVX2-NEXT:    vpmovzxbd {{.*#+}} ymm2 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = mem[2,3,0,1]
+; AVX2-NEXT:    vmovdqa (%rdi), %xmm2
+; AVX2-NEXT:    vmovdqa 32(%rdi), %xmm3
+; AVX2-NEXT:    vpmovzxbd {{.*#+}} ymm4 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero,xmm2[4],zero,zero,zero,xmm2[5],zero,zero,zero,xmm2[6],zero,zero,zero,xmm2[7],zero,zero,zero
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,3,0,1]
+; AVX2-NEXT:    vpmovzxbd {{.*#+}} ymm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero,xmm2[4],zero,zero,zero,xmm2[5],zero,zero,zero,xmm2[6],zero,zero,zero,xmm2[7],zero,zero,zero
+; AVX2-NEXT:    vpmovzxbd {{.*#+}} ymm5 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero,xmm3[4],zero,zero,zero,xmm3[5],zero,zero,zero,xmm3[6],zero,zero,zero,xmm3[7],zero,zero,zero
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[2,3,0,1]
 ; AVX2-NEXT:    vpmovzxbd {{.*#+}} ymm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero,xmm3[4],zero,zero,zero,xmm3[5],zero,zero,zero,xmm3[6],zero,zero,zero,xmm3[7],zero,zero,zero
-; AVX2-NEXT:    vpmovzxbd {{.*#+}} ymm4 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm5 = mem[2,3,0,1]
-; AVX2-NEXT:    vpmovzxbd {{.*#+}} ymm5 = xmm5[0],zero,zero,zero,xmm5[1],zero,zero,zero,xmm5[2],zero,zero,zero,xmm5[3],zero,zero,zero,xmm5[4],zero,zero,zero,xmm5[5],zero,zero,zero,xmm5[6],zero,zero,zero,xmm5[7],zero,zero,zero
 ; AVX2-NEXT:    vpmovzxbd {{.*#+}} ymm6 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero
 ; AVX2-NEXT:    vpaddd %ymm6, %ymm0, %ymm0
 ; AVX2-NEXT:    vpbroadcastq 24(%rsi), %ymm6
 ; AVX2-NEXT:    vpmovzxbd {{.*#+}} ymm6 = xmm6[0],zero,zero,zero,xmm6[1],zero,zero,zero,xmm6[2],zero,zero,zero,xmm6[3],zero,zero,zero,xmm6[4],zero,zero,zero,xmm6[5],zero,zero,zero,xmm6[6],zero,zero,zero,xmm6[7],zero,zero,zero
 ; AVX2-NEXT:    vpaddd %ymm6, %ymm1, %ymm1
-; AVX2-NEXT:    vpmovzxbd {{.*#+}} ymm6 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero
+; AVX2-NEXT:    vmovdqa (%rsi), %xmm6
+; AVX2-NEXT:    vmovdqa 32(%rsi), %xmm7
+; AVX2-NEXT:    vpmovzxbd {{.*#+}} ymm8 = xmm6[0],zero,zero,zero,xmm6[1],zero,zero,zero,xmm6[2],zero,zero,zero,xmm6[3],zero,zero,zero,xmm6[4],zero,zero,zero,xmm6[5],zero,zero,zero,xmm6[6],zero,zero,zero,xmm6[7],zero,zero,zero
+; AVX2-NEXT:    vpaddd %ymm8, %ymm4, %ymm4
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm6 = xmm6[2,3,0,1]
+; AVX2-NEXT:    vpmovzxbd {{.*#+}} ymm6 = xmm6[0],zero,zero,zero,xmm6[1],zero,zero,zero,xmm6[2],zero,zero,zero,xmm6[3],zero,zero,zero,xmm6[4],zero,zero,zero,xmm6[5],zero,zero,zero,xmm6[6],zero,zero,zero,xmm6[7],zero,zero,zero
 ; AVX2-NEXT:    vpaddd %ymm6, %ymm2, %ymm2
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm6 = mem[2,3,0,1]
+; AVX2-NEXT:    vpmovzxbd {{.*#+}} ymm6 = xmm7[0],zero,zero,zero,xmm7[1],zero,zero,zero,xmm7[2],zero,zero,zero,xmm7[3],zero,zero,zero,xmm7[4],zero,zero,zero,xmm7[5],zero,zero,zero,xmm7[6],zero,zero,zero,xmm7[7],zero,zero,zero
+; AVX2-NEXT:    vpaddd %ymm6, %ymm5, %ymm5
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm6 = xmm7[2,3,0,1]
 ; AVX2-NEXT:    vpmovzxbd {{.*#+}} ymm6 = xmm6[0],zero,zero,zero,xmm6[1],zero,zero,zero,xmm6[2],zero,zero,zero,xmm6[3],zero,zero,zero,xmm6[4],zero,zero,zero,xmm6[5],zero,zero,zero,xmm6[6],zero,zero,zero,xmm6[7],zero,zero,zero
 ; AVX2-NEXT:    vpaddd %ymm6, %ymm3, %ymm3
-; AVX2-NEXT:    vpmovzxbd {{.*#+}} ymm6 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero
-; AVX2-NEXT:    vpaddd %ymm6, %ymm4, %ymm4
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm6 = mem[2,3,0,1]
-; AVX2-NEXT:    vpmovzxbd {{.*#+}} ymm6 = xmm6[0],zero,zero,zero,xmm6[1],zero,zero,zero,xmm6[2],zero,zero,zero,xmm6[3],zero,zero,zero,xmm6[4],zero,zero,zero,xmm6[5],zero,zero,zero,xmm6[6],zero,zero,zero,xmm6[7],zero,zero,zero
-; AVX2-NEXT:    vpaddd %ymm6, %ymm5, %ymm5
 ; AVX2-NEXT:    vpcmpeqd %ymm6, %ymm6, %ymm6
 ; AVX2-NEXT:    vpsubd %ymm6, %ymm0, %ymm7
 ; AVX2-NEXT:    vpsubd %ymm6, %ymm1, %ymm1
-; AVX2-NEXT:    vpsubd %ymm6, %ymm2, %ymm2
-; AVX2-NEXT:    vpsubd %ymm6, %ymm3, %ymm3
 ; AVX2-NEXT:    vpsubd %ymm6, %ymm4, %ymm4
-; AVX2-NEXT:    vpsubd %ymm6, %ymm5, %ymm0
+; AVX2-NEXT:    vpsubd %ymm6, %ymm2, %ymm2
+; AVX2-NEXT:    vpsubd %ymm6, %ymm5, %ymm5
+; AVX2-NEXT:    vpsubd %ymm6, %ymm3, %ymm0
 ; AVX2-NEXT:    vpsrld $1, %ymm0, %ymm0
-; AVX2-NEXT:    vpsrld $1, %ymm4, %ymm4
-; AVX2-NEXT:    vpsrld $1, %ymm3, %ymm3
+; AVX2-NEXT:    vpsrld $1, %ymm5, %ymm3
 ; AVX2-NEXT:    vpsrld $1, %ymm2, %ymm2
+; AVX2-NEXT:    vpsrld $1, %ymm4, %ymm4
 ; AVX2-NEXT:    vpsrld $1, %ymm1, %ymm1
 ; AVX2-NEXT:    vpsrld $1, %ymm7, %ymm5
 ; AVX2-NEXT:    vextracti128 $1, %ymm5, %xmm6
@@ -427,16 +428,16 @@ define void @avg_v48i8(<48 x i8>* %a, <48 x i8>* %b) nounwind {
 ; AVX2-NEXT:    vpackusdw %xmm7, %xmm1, %xmm1
 ; AVX2-NEXT:    vpand %xmm6, %xmm1, %xmm1
 ; AVX2-NEXT:    vpackuswb %xmm1, %xmm5, %xmm1
+; AVX2-NEXT:    vextracti128 $1, %ymm4, %xmm5
+; AVX2-NEXT:    vpackusdw %xmm5, %xmm4, %xmm4
+; AVX2-NEXT:    vpand %xmm6, %xmm4, %xmm4
 ; AVX2-NEXT:    vextracti128 $1, %ymm2, %xmm5
 ; AVX2-NEXT:    vpackusdw %xmm5, %xmm2, %xmm2
 ; AVX2-NEXT:    vpand %xmm6, %xmm2, %xmm2
-; AVX2-NEXT:    vextracti128 $1, %ymm3, %xmm5
-; AVX2-NEXT:    vpackusdw %xmm5, %xmm3, %xmm3
-; AVX2-NEXT:    vpand %xmm6, %xmm3, %xmm3
-; AVX2-NEXT:    vpackuswb %xmm3, %xmm2, %xmm2
+; AVX2-NEXT:    vpackuswb %xmm2, %xmm4, %xmm2
 ; AVX2-NEXT:    vinserti128 $1, %xmm1, %ymm2, %ymm1
-; AVX2-NEXT:    vextracti128 $1, %ymm4, %xmm2
-; AVX2-NEXT:    vpackusdw %xmm2, %xmm4, %xmm2
+; AVX2-NEXT:    vextracti128 $1, %ymm3, %xmm2
+; AVX2-NEXT:    vpackusdw %xmm2, %xmm3, %xmm2
 ; AVX2-NEXT:    vpand %xmm6, %xmm2, %xmm2
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm3
 ; AVX2-NEXT:    vpackusdw %xmm3, %xmm0, %xmm0
diff --git a/test/CodeGen/X86/avx-cmp.ll b/test/CodeGen/X86/avx-cmp.ll
index 534a00050ad293b523fb98ae7eea466d5232d9bf..c789ec4d77fbec5df18d11deb768925de8b9f154 100644
--- a/test/CodeGen/X86/avx-cmp.ll
+++ b/test/CodeGen/X86/avx-cmp.ll
@@ -35,11 +35,7 @@ define void @render() nounwind {
 ; CHECK-NEXT:  # %bb.1: # %for.cond5.preheader
 ; CHECK-NEXT:    xorl %ebx, %ebx
 ; CHECK-NEXT:    movb $1, %bpl
-; CHECK-NEXT:    jmp .LBB2_2
 ; CHECK-NEXT:    .p2align 4, 0x90
-; CHECK-NEXT:  .LBB2_5: # %if.then
-; CHECK-NEXT:    # in Loop: Header=BB2_2 Depth=1
-; CHECK-NEXT:    callq scale
 ; CHECK-NEXT:  .LBB2_2: # %for.cond5
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    testb %bl, %bl
@@ -52,7 +48,10 @@ define void @render() nounwind {
 ; CHECK-NEXT:    # in Loop: Header=BB2_2 Depth=1
 ; CHECK-NEXT:    vucomisd {{\.LCPI.*}}, %xmm0
 ; CHECK-NEXT:    jne .LBB2_5
-; CHECK-NEXT:    jp .LBB2_5
+; CHECK-NEXT:    jnp .LBB2_2
+; CHECK-NEXT:  .LBB2_5: # %if.then
+; CHECK-NEXT:   # in Loop: Header=BB2_2 Depth=1
+; CHECK-NEXT:    callq   scale
 ; CHECK-NEXT:    jmp .LBB2_2
 ; CHECK-NEXT:  .LBB2_6: # %for.end52
 ; CHECK-NEXT:    addq $8, %rsp
diff --git a/test/CodeGen/X86/avx-intrinsics-fast-isel.ll b/test/CodeGen/X86/avx-intrinsics-fast-isel.ll
index 84b3b007310c7b34a22e08fbab2620bb291e9c48..1e6d0e8766b305b86e9b1e3d4c14f05dc0d2bd97 100644
--- a/test/CodeGen/X86/avx-intrinsics-fast-isel.ll
+++ b/test/CodeGen/X86/avx-intrinsics-fast-isel.ll
@@ -1956,12 +1956,9 @@ define <4 x i64> @test_mm256_set1_epi32(i32 %a0) nounwind {
 define <4 x i64> @test_mm256_set1_epi64x(i64 %a0) nounwind {
 ; X86-LABEL: test_mm256_set1_epi64x:
 ; X86:       # %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT:    vmovd %ecx, %xmm0
-; X86-NEXT:    vpinsrd $1, %eax, %xmm0, %xmm0
-; X86-NEXT:    vpinsrd $2, %ecx, %xmm0, %xmm0
-; X86-NEXT:    vpinsrd $3, %eax, %xmm0, %xmm0
+; X86-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86-NEXT:    vpinsrd $1, {{[0-9]+}}(%esp), %xmm0, %xmm0
+; X86-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,1,0,1]
 ; X86-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
 ; X86-NEXT:    retl
 ;
diff --git a/test/CodeGen/X86/avx-schedule.ll b/test/CodeGen/X86/avx-schedule.ll
deleted file mode 100644
index e1db9bab294cd1db1ab5d52519bd99915a4d7adc..0000000000000000000000000000000000000000
--- a/test/CodeGen/X86/avx-schedule.ll
+++ /dev/null
@@ -1,6120 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=x86-64 -mattr=+avx | FileCheck %s --check-prefix=CHECK --check-prefix=GENERIC
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=sandybridge -mattr=-avx2 | FileCheck %s --check-prefix=CHECK --check-prefix=SANDY
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=ivybridge -mattr=-avx2 | FileCheck %s --check-prefix=CHECK --check-prefix=SANDY
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=haswell -mattr=-avx2 | FileCheck %s --check-prefix=CHECK --check-prefix=HASWELL
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=broadwell -mattr=-avx2 | FileCheck %s --check-prefix=CHECK --check-prefix=BROADWELL
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=skylake -mattr=-avx2 | FileCheck %s --check-prefix=CHECK --check-prefix=SKYLAKE
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=skx -mattr=-avx2 | FileCheck %s --check-prefix=CHECK --check-prefix=SKX
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=bdver2 -mattr=-avx2 | FileCheck %s --check-prefix=CHECK --check-prefix=BDVER2
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=btver2 -mattr=-avx2 | FileCheck %s --check-prefix=CHECK --check-prefix=BTVER2
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=znver1 -mattr=-avx2 | FileCheck %s --check-prefix=CHECK --check-prefix=ZNVER1
-
-define <4 x double> @test_addpd(<4 x double> %a0, <4 x double> %a1, <4 x double> *%a2) {
-; GENERIC-LABEL: test_addpd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vaddpd %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; GENERIC-NEXT:    vaddpd (%rdi), %ymm0, %ymm0 # sched: [10:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_addpd:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vaddpd %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; SANDY-NEXT:    vaddpd (%rdi), %ymm0, %ymm0 # sched: [10:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_addpd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vaddpd %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; HASWELL-NEXT:    vaddpd (%rdi), %ymm0, %ymm0 # sched: [10:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_addpd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vaddpd %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    vaddpd (%rdi), %ymm0, %ymm0 # sched: [9:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_addpd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vaddpd %ymm1, %ymm0, %ymm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    vaddpd (%rdi), %ymm0, %ymm0 # sched: [11:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_addpd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vaddpd %ymm1, %ymm0, %ymm0 # sched: [4:0.50]
-; SKX-NEXT:    vaddpd (%rdi), %ymm0, %ymm0 # sched: [11:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_addpd:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vaddpd %ymm1, %ymm0, %ymm0 # sched: [5:2.00]
-; BDVER2-NEXT:    vaddpd (%rdi), %ymm0, %ymm0 # sched: [10:2.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_addpd:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vaddpd %ymm1, %ymm0, %ymm0 # sched: [3:2.00]
-; BTVER2-NEXT:    vaddpd (%rdi), %ymm0, %ymm0 # sched: [8:2.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_addpd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vaddpd %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    vaddpd (%rdi), %ymm0, %ymm0 # sched: [10:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = fadd <4 x double> %a0, %a1
-  %2 = load <4 x double>, <4 x double> *%a2, align 32
-  %3 = fadd <4 x double> %1, %2
-  ret <4 x double> %3
-}
-
-define <8 x float> @test_addps(<8 x float> %a0, <8 x float> %a1, <8 x float> *%a2) {
-; GENERIC-LABEL: test_addps:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vaddps %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; GENERIC-NEXT:    vaddps (%rdi), %ymm0, %ymm0 # sched: [10:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_addps:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vaddps %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; SANDY-NEXT:    vaddps (%rdi), %ymm0, %ymm0 # sched: [10:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_addps:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vaddps %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; HASWELL-NEXT:    vaddps (%rdi), %ymm0, %ymm0 # sched: [10:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_addps:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vaddps %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    vaddps (%rdi), %ymm0, %ymm0 # sched: [9:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_addps:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vaddps %ymm1, %ymm0, %ymm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    vaddps (%rdi), %ymm0, %ymm0 # sched: [11:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_addps:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vaddps %ymm1, %ymm0, %ymm0 # sched: [4:0.50]
-; SKX-NEXT:    vaddps (%rdi), %ymm0, %ymm0 # sched: [11:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_addps:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vaddps %ymm1, %ymm0, %ymm0 # sched: [5:2.00]
-; BDVER2-NEXT:    vaddps (%rdi), %ymm0, %ymm0 # sched: [10:2.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_addps:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vaddps %ymm1, %ymm0, %ymm0 # sched: [3:2.00]
-; BTVER2-NEXT:    vaddps (%rdi), %ymm0, %ymm0 # sched: [8:2.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_addps:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vaddps %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    vaddps (%rdi), %ymm0, %ymm0 # sched: [10:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = fadd <8 x float> %a0, %a1
-  %2 = load <8 x float>, <8 x float> *%a2, align 32
-  %3 = fadd <8 x float> %1, %2
-  ret <8 x float> %3
-}
-
-define <4 x double> @test_addsubpd(<4 x double> %a0, <4 x double> %a1, <4 x double> *%a2) {
-; GENERIC-LABEL: test_addsubpd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vaddsubpd %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; GENERIC-NEXT:    vaddsubpd (%rdi), %ymm0, %ymm0 # sched: [10:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_addsubpd:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vaddsubpd %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; SANDY-NEXT:    vaddsubpd (%rdi), %ymm0, %ymm0 # sched: [10:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_addsubpd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vaddsubpd %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; HASWELL-NEXT:    vaddsubpd (%rdi), %ymm0, %ymm0 # sched: [10:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_addsubpd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vaddsubpd %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    vaddsubpd (%rdi), %ymm0, %ymm0 # sched: [9:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_addsubpd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vaddsubpd %ymm1, %ymm0, %ymm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    vaddsubpd (%rdi), %ymm0, %ymm0 # sched: [11:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_addsubpd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vaddsubpd %ymm1, %ymm0, %ymm0 # sched: [4:0.50]
-; SKX-NEXT:    vaddsubpd (%rdi), %ymm0, %ymm0 # sched: [11:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_addsubpd:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vaddsubpd %ymm1, %ymm0, %ymm0 # sched: [5:2.00]
-; BDVER2-NEXT:    vaddsubpd (%rdi), %ymm0, %ymm0 # sched: [10:2.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_addsubpd:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vaddsubpd %ymm1, %ymm0, %ymm0 # sched: [3:2.00]
-; BTVER2-NEXT:    vaddsubpd (%rdi), %ymm0, %ymm0 # sched: [8:2.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_addsubpd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vaddsubpd %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    vaddsubpd (%rdi), %ymm0, %ymm0 # sched: [10:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <4 x double> @llvm.x86.avx.addsub.pd.256(<4 x double> %a0, <4 x double> %a1)
-  %2 = load <4 x double>, <4 x double> *%a2, align 32
-  %3 = call <4 x double> @llvm.x86.avx.addsub.pd.256(<4 x double> %1, <4 x double> %2)
-  ret <4 x double> %3
-}
-declare <4 x double> @llvm.x86.avx.addsub.pd.256(<4 x double>, <4 x double>) nounwind readnone
-
-define <8 x float> @test_addsubps(<8 x float> %a0, <8 x float> %a1, <8 x float> *%a2) {
-; GENERIC-LABEL: test_addsubps:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vaddsubps %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; GENERIC-NEXT:    vaddsubps (%rdi), %ymm0, %ymm0 # sched: [10:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_addsubps:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vaddsubps %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; SANDY-NEXT:    vaddsubps (%rdi), %ymm0, %ymm0 # sched: [10:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_addsubps:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vaddsubps %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; HASWELL-NEXT:    vaddsubps (%rdi), %ymm0, %ymm0 # sched: [10:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_addsubps:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vaddsubps %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    vaddsubps (%rdi), %ymm0, %ymm0 # sched: [9:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_addsubps:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vaddsubps %ymm1, %ymm0, %ymm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    vaddsubps (%rdi), %ymm0, %ymm0 # sched: [11:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_addsubps:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vaddsubps %ymm1, %ymm0, %ymm0 # sched: [4:0.50]
-; SKX-NEXT:    vaddsubps (%rdi), %ymm0, %ymm0 # sched: [11:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_addsubps:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vaddsubps %ymm1, %ymm0, %ymm0 # sched: [5:2.00]
-; BDVER2-NEXT:    vaddsubps (%rdi), %ymm0, %ymm0 # sched: [10:2.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_addsubps:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vaddsubps %ymm1, %ymm0, %ymm0 # sched: [3:2.00]
-; BTVER2-NEXT:    vaddsubps (%rdi), %ymm0, %ymm0 # sched: [8:2.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_addsubps:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vaddsubps %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    vaddsubps (%rdi), %ymm0, %ymm0 # sched: [10:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <8 x float> @llvm.x86.avx.addsub.ps.256(<8 x float> %a0, <8 x float> %a1)
-  %2 = load <8 x float>, <8 x float> *%a2, align 32
-  %3 = call <8 x float> @llvm.x86.avx.addsub.ps.256(<8 x float> %1, <8 x float> %2)
-  ret <8 x float> %3
-}
-declare <8 x float> @llvm.x86.avx.addsub.ps.256(<8 x float>, <8 x float>) nounwind readnone
-
-define <4 x double> @test_andnotpd(<4 x double> %a0, <4 x double> %a1, <4 x double> *%a2) {
-; GENERIC-LABEL: test_andnotpd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vandnpd %ymm1, %ymm0, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vandnpd (%rdi), %ymm0, %ymm0 # sched: [8:1.00]
-; GENERIC-NEXT:    vaddpd %ymm0, %ymm1, %ymm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_andnotpd:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vandnpd %ymm1, %ymm0, %ymm0 # sched: [1:1.00]
-; SANDY-NEXT:    vandnpd (%rdi), %ymm0, %ymm0 # sched: [8:1.00]
-; SANDY-NEXT:    vaddpd %ymm0, %ymm1, %ymm0 # sched: [3:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_andnotpd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vandnpd %ymm1, %ymm0, %ymm0 # sched: [1:1.00]
-; HASWELL-NEXT:    vandnpd (%rdi), %ymm0, %ymm0 # sched: [8:1.00]
-; HASWELL-NEXT:    vaddpd %ymm0, %ymm1, %ymm0 # sched: [3:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_andnotpd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vandnpd %ymm1, %ymm0, %ymm0 # sched: [1:1.00]
-; BROADWELL-NEXT:    vandnpd (%rdi), %ymm0, %ymm0 # sched: [7:1.00]
-; BROADWELL-NEXT:    vaddpd %ymm0, %ymm1, %ymm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_andnotpd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vandnpd %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    vandnpd (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKYLAKE-NEXT:    vaddpd %ymm0, %ymm1, %ymm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_andnotpd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vandnpd %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    vandnpd (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKX-NEXT:    vaddpd %ymm0, %ymm1, %ymm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_andnotpd:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vandnpd %ymm1, %ymm0, %ymm0 # sched: [2:1.00]
-; BDVER2-NEXT:    vandnpd (%rdi), %ymm0, %ymm0 # sched: [7:1.00]
-; BDVER2-NEXT:    vaddpd %ymm0, %ymm1, %ymm0 # sched: [5:2.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_andnotpd:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vandnpd %ymm1, %ymm0, %ymm0 # sched: [1:1.00]
-; BTVER2-NEXT:    vandnpd (%rdi), %ymm0, %ymm0 # sched: [6:2.00]
-; BTVER2-NEXT:    vaddpd %ymm0, %ymm1, %ymm0 # sched: [3:2.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_andnotpd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vandnpd %ymm1, %ymm0, %ymm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vandnpd (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    vaddpd %ymm0, %ymm1, %ymm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = bitcast <4 x double> %a0 to <4 x i64>
-  %2 = bitcast <4 x double> %a1 to <4 x i64>
-  %3 = xor <4 x i64> %1, <i64 -1, i64 -1, i64 -1, i64 -1>
-  %4 = and <4 x i64> %3, %2
-  %5 = load <4 x double>, <4 x double> *%a2, align 32
-  %6 = bitcast <4 x double> %5 to <4 x i64>
-  %7 = xor <4 x i64> %4, <i64 -1, i64 -1, i64 -1, i64 -1>
-  %8 = and <4 x i64> %6, %7
-  %9 = bitcast <4 x i64> %8 to <4 x double>
-  %10 = fadd <4 x double> %a1, %9
-  ret <4 x double> %10
-}
-
-define <8 x float> @test_andnotps(<8 x float> %a0, <8 x float> %a1, <8 x float> *%a2) {
-; GENERIC-LABEL: test_andnotps:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vandnps %ymm1, %ymm0, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vandnps (%rdi), %ymm0, %ymm0 # sched: [8:1.00]
-; GENERIC-NEXT:    vaddps %ymm0, %ymm1, %ymm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_andnotps:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vandnps %ymm1, %ymm0, %ymm0 # sched: [1:1.00]
-; SANDY-NEXT:    vandnps (%rdi), %ymm0, %ymm0 # sched: [8:1.00]
-; SANDY-NEXT:    vaddps %ymm0, %ymm1, %ymm0 # sched: [3:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_andnotps:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vandnps %ymm1, %ymm0, %ymm0 # sched: [1:1.00]
-; HASWELL-NEXT:    vandnps (%rdi), %ymm0, %ymm0 # sched: [8:1.00]
-; HASWELL-NEXT:    vaddps %ymm0, %ymm1, %ymm0 # sched: [3:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_andnotps:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vandnps %ymm1, %ymm0, %ymm0 # sched: [1:1.00]
-; BROADWELL-NEXT:    vandnps (%rdi), %ymm0, %ymm0 # sched: [7:1.00]
-; BROADWELL-NEXT:    vaddps %ymm0, %ymm1, %ymm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_andnotps:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vandnps %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    vandnps (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKYLAKE-NEXT:    vaddps %ymm0, %ymm1, %ymm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_andnotps:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vandnps %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    vandnps (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKX-NEXT:    vaddps %ymm0, %ymm1, %ymm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_andnotps:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vandnps %ymm1, %ymm0, %ymm0 # sched: [2:1.00]
-; BDVER2-NEXT:    vandnps (%rdi), %ymm0, %ymm0 # sched: [7:1.00]
-; BDVER2-NEXT:    vaddps %ymm0, %ymm1, %ymm0 # sched: [5:2.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_andnotps:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vandnps %ymm1, %ymm0, %ymm0 # sched: [1:1.00]
-; BTVER2-NEXT:    vandnps (%rdi), %ymm0, %ymm0 # sched: [6:2.00]
-; BTVER2-NEXT:    vaddps %ymm0, %ymm1, %ymm0 # sched: [3:2.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_andnotps:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vandnps %ymm1, %ymm0, %ymm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vandnps (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    vaddps %ymm0, %ymm1, %ymm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = bitcast <8 x float> %a0 to <4 x i64>
-  %2 = bitcast <8 x float> %a1 to <4 x i64>
-  %3 = xor <4 x i64> %1, <i64 -1, i64 -1, i64 -1, i64 -1>
-  %4 = and <4 x i64> %3, %2
-  %5 = load <8 x float>, <8 x float> *%a2, align 32
-  %6 = bitcast <8 x float> %5 to <4 x i64>
-  %7 = xor <4 x i64> %4, <i64 -1, i64 -1, i64 -1, i64 -1>
-  %8 = and <4 x i64> %6, %7
-  %9 = bitcast <4 x i64> %8 to <8 x float>
-  %10 = fadd <8 x float> %a1, %9
-  ret <8 x float> %10
-}
-
-define <4 x double> @test_andpd(<4 x double> %a0, <4 x double> %a1, <4 x double> *%a2) {
-; GENERIC-LABEL: test_andpd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vandpd %ymm1, %ymm0, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vandpd (%rdi), %ymm0, %ymm0 # sched: [8:1.00]
-; GENERIC-NEXT:    vaddpd %ymm0, %ymm1, %ymm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_andpd:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vandpd %ymm1, %ymm0, %ymm0 # sched: [1:1.00]
-; SANDY-NEXT:    vandpd (%rdi), %ymm0, %ymm0 # sched: [8:1.00]
-; SANDY-NEXT:    vaddpd %ymm0, %ymm1, %ymm0 # sched: [3:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_andpd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vandpd %ymm1, %ymm0, %ymm0 # sched: [1:1.00]
-; HASWELL-NEXT:    vandpd (%rdi), %ymm0, %ymm0 # sched: [8:1.00]
-; HASWELL-NEXT:    vaddpd %ymm0, %ymm1, %ymm0 # sched: [3:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_andpd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vandpd %ymm1, %ymm0, %ymm0 # sched: [1:1.00]
-; BROADWELL-NEXT:    vandpd (%rdi), %ymm0, %ymm0 # sched: [7:1.00]
-; BROADWELL-NEXT:    vaddpd %ymm0, %ymm1, %ymm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_andpd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vandpd %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    vandpd (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKYLAKE-NEXT:    vaddpd %ymm0, %ymm1, %ymm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_andpd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vandpd %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    vandpd (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKX-NEXT:    vaddpd %ymm0, %ymm1, %ymm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_andpd:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vandpd %ymm1, %ymm0, %ymm0 # sched: [2:1.00]
-; BDVER2-NEXT:    vandpd (%rdi), %ymm0, %ymm0 # sched: [7:1.00]
-; BDVER2-NEXT:    vaddpd %ymm0, %ymm1, %ymm0 # sched: [5:2.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_andpd:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vandpd %ymm1, %ymm0, %ymm0 # sched: [1:1.00]
-; BTVER2-NEXT:    vandpd (%rdi), %ymm0, %ymm0 # sched: [6:2.00]
-; BTVER2-NEXT:    vaddpd %ymm0, %ymm1, %ymm0 # sched: [3:2.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_andpd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vandpd %ymm1, %ymm0, %ymm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vandpd (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    vaddpd %ymm0, %ymm1, %ymm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = bitcast <4 x double> %a0 to <4 x i64>
-  %2 = bitcast <4 x double> %a1 to <4 x i64>
-  %3 = and <4 x i64> %1, %2
-  %4 = load <4 x double>, <4 x double> *%a2, align 32
-  %5 = bitcast <4 x double> %4 to <4 x i64>
-  %6 = and <4 x i64> %3, %5
-  %7 = bitcast <4 x i64> %6 to <4 x double>
-  %8 = fadd <4 x double> %a1, %7
-  ret <4 x double> %8
-}
-
-define <8 x float> @test_andps(<8 x float> %a0, <8 x float> %a1, <8 x float> *%a2) {
-; GENERIC-LABEL: test_andps:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vandps %ymm1, %ymm0, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vandps (%rdi), %ymm0, %ymm0 # sched: [8:1.00]
-; GENERIC-NEXT:    vaddps %ymm0, %ymm1, %ymm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_andps:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vandps %ymm1, %ymm0, %ymm0 # sched: [1:1.00]
-; SANDY-NEXT:    vandps (%rdi), %ymm0, %ymm0 # sched: [8:1.00]
-; SANDY-NEXT:    vaddps %ymm0, %ymm1, %ymm0 # sched: [3:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_andps:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vandps %ymm1, %ymm0, %ymm0 # sched: [1:1.00]
-; HASWELL-NEXT:    vandps (%rdi), %ymm0, %ymm0 # sched: [8:1.00]
-; HASWELL-NEXT:    vaddps %ymm0, %ymm1, %ymm0 # sched: [3:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_andps:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vandps %ymm1, %ymm0, %ymm0 # sched: [1:1.00]
-; BROADWELL-NEXT:    vandps (%rdi), %ymm0, %ymm0 # sched: [7:1.00]
-; BROADWELL-NEXT:    vaddps %ymm0, %ymm1, %ymm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_andps:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vandps %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    vandps (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKYLAKE-NEXT:    vaddps %ymm0, %ymm1, %ymm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_andps:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vandps %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    vandps (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKX-NEXT:    vaddps %ymm0, %ymm1, %ymm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_andps:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vandps %ymm1, %ymm0, %ymm0 # sched: [2:1.00]
-; BDVER2-NEXT:    vandps (%rdi), %ymm0, %ymm0 # sched: [7:1.00]
-; BDVER2-NEXT:    vaddps %ymm0, %ymm1, %ymm0 # sched: [5:2.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_andps:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vandps %ymm1, %ymm0, %ymm0 # sched: [1:1.00]
-; BTVER2-NEXT:    vandps (%rdi), %ymm0, %ymm0 # sched: [6:2.00]
-; BTVER2-NEXT:    vaddps %ymm0, %ymm1, %ymm0 # sched: [3:2.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_andps:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vandps %ymm1, %ymm0, %ymm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vandps (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    vaddps %ymm0, %ymm1, %ymm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = bitcast <8 x float> %a0 to <4 x i64>
-  %2 = bitcast <8 x float> %a1 to <4 x i64>
-  %3 = and <4 x i64> %1, %2
-  %4 = load <8 x float>, <8 x float> *%a2, align 32
-  %5 = bitcast <8 x float> %4 to <4 x i64>
-  %6 = and <4 x i64> %3, %5
-  %7 = bitcast <4 x i64> %6 to <8 x float>
-  %8 = fadd <8 x float> %a1, %7
-  ret <8 x float> %8
-}
-
-define <4 x double> @test_blendpd(<4 x double> %a0, <4 x double> %a1, <4 x double> *%a2) {
-; GENERIC-LABEL: test_blendpd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vblendpd {{.*#+}} ymm0 = ymm0[0],ymm1[1,2],ymm0[3] sched: [1:0.50]
-; GENERIC-NEXT:    vblendpd {{.*#+}} ymm1 = ymm0[0,1],mem[2,3] sched: [8:0.50]
-; GENERIC-NEXT:    vaddpd %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_blendpd:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vblendpd {{.*#+}} ymm0 = ymm0[0],ymm1[1,2],ymm0[3] sched: [1:0.50]
-; SANDY-NEXT:    vblendpd {{.*#+}} ymm1 = ymm0[0,1],mem[2,3] sched: [8:0.50]
-; SANDY-NEXT:    vaddpd %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_blendpd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vblendpd {{.*#+}} ymm0 = ymm0[0],ymm1[1,2],ymm0[3] sched: [1:0.33]
-; HASWELL-NEXT:    vblendpd {{.*#+}} ymm1 = ymm0[0,1],mem[2,3] sched: [8:0.50]
-; HASWELL-NEXT:    vaddpd %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_blendpd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vblendpd {{.*#+}} ymm0 = ymm0[0],ymm1[1,2],ymm0[3] sched: [1:0.33]
-; BROADWELL-NEXT:    vblendpd {{.*#+}} ymm1 = ymm0[0,1],mem[2,3] sched: [7:0.50]
-; BROADWELL-NEXT:    vaddpd %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_blendpd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vblendpd {{.*#+}} ymm0 = ymm0[0],ymm1[1,2],ymm0[3] sched: [1:0.33]
-; SKYLAKE-NEXT:    vblendpd {{.*#+}} ymm1 = ymm0[0,1],mem[2,3] sched: [8:0.50]
-; SKYLAKE-NEXT:    vaddpd %ymm1, %ymm0, %ymm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_blendpd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vblendpd {{.*#+}} ymm0 = ymm0[0],ymm1[1,2],ymm0[3] sched: [1:0.33]
-; SKX-NEXT:    vblendpd {{.*#+}} ymm1 = ymm0[0,1],mem[2,3] sched: [8:0.50]
-; SKX-NEXT:    vaddpd %ymm1, %ymm0, %ymm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_blendpd:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vblendpd {{.*#+}} ymm0 = ymm0[0],ymm1[1,2],ymm0[3] sched: [2:1.00]
-; BDVER2-NEXT:    vblendpd {{.*#+}} ymm1 = ymm0[0,1],mem[2,3] sched: [7:1.00]
-; BDVER2-NEXT:    vaddpd %ymm1, %ymm0, %ymm0 # sched: [5:2.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_blendpd:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vblendpd {{.*#+}} ymm0 = ymm0[0],ymm1[1,2],ymm0[3] sched: [1:1.00]
-; BTVER2-NEXT:    vblendpd {{.*#+}} ymm1 = ymm0[0,1],mem[2,3] sched: [6:2.00]
-; BTVER2-NEXT:    vaddpd %ymm1, %ymm0, %ymm0 # sched: [3:2.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_blendpd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vblendpd {{.*#+}} ymm0 = ymm0[0],ymm1[1,2],ymm0[3] sched: [1:0.50]
-; ZNVER1-NEXT:    vblendpd {{.*#+}} ymm1 = ymm0[0,1],mem[2,3] sched: [8:0.50]
-; ZNVER1-NEXT:    vaddpd %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = shufflevector <4 x double> %a0, <4 x double> %a1, <4 x i32> <i32 0, i32 5, i32 6, i32 3>
-  %2 = load <4 x double>, <4 x double> *%a2, align 32
-  %3 = shufflevector <4 x double> %1, <4 x double> %2, <4 x i32> <i32 0, i32 1, i32 6, i32 7>
-  %4 = fadd <4 x double> %1, %3
-  ret <4 x double> %4
-}
-
-define <8 x float> @test_blendps(<8 x float> %a0, <8 x float> %a1, <8 x float> *%a2) {
-; GENERIC-LABEL: test_blendps:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm1[1,2],ymm0[3,4,5,6,7] sched: [1:0.50]
-; GENERIC-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1],mem[2],ymm1[3],mem[4,5,6],ymm1[7] sched: [8:0.50]
-; GENERIC-NEXT:    vaddps %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_blendps:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm1[1,2],ymm0[3,4,5,6,7] sched: [1:0.50]
-; SANDY-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1],mem[2],ymm1[3],mem[4,5,6],ymm1[7] sched: [8:0.50]
-; SANDY-NEXT:    vaddps %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_blendps:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm1[1,2],ymm0[3,4,5,6,7] sched: [1:0.33]
-; HASWELL-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1],mem[2],ymm1[3],mem[4,5,6],ymm1[7] sched: [8:0.50]
-; HASWELL-NEXT:    vaddps %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_blendps:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm1[1,2],ymm0[3,4,5,6,7] sched: [1:0.33]
-; BROADWELL-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1],mem[2],ymm1[3],mem[4,5,6],ymm1[7] sched: [7:0.50]
-; BROADWELL-NEXT:    vaddps %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_blendps:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm1[1,2],ymm0[3,4,5,6,7] sched: [1:0.33]
-; SKYLAKE-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1],mem[2],ymm1[3],mem[4,5,6],ymm1[7] sched: [8:0.50]
-; SKYLAKE-NEXT:    vaddps %ymm1, %ymm0, %ymm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_blendps:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm1[1,2],ymm0[3,4,5,6,7] sched: [1:0.33]
-; SKX-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1],mem[2],ymm1[3],mem[4,5,6],ymm1[7] sched: [8:0.50]
-; SKX-NEXT:    vaddps %ymm1, %ymm0, %ymm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_blendps:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm1[1,2],ymm0[3,4,5,6,7] sched: [2:1.00]
-; BDVER2-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1],mem[2],ymm1[3],mem[4,5,6],ymm1[7] sched: [7:1.00]
-; BDVER2-NEXT:    vaddps %ymm1, %ymm0, %ymm0 # sched: [5:2.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_blendps:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm1[1,2],ymm0[3,4,5,6,7] sched: [1:1.00]
-; BTVER2-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1],mem[2],ymm1[3],mem[4,5,6],ymm1[7] sched: [6:2.00]
-; BTVER2-NEXT:    vaddps %ymm1, %ymm0, %ymm0 # sched: [3:2.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_blendps:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm1[1,2],ymm0[3,4,5,6,7] sched: [1:0.50]
-; ZNVER1-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1],mem[2],ymm1[3],mem[4,5,6],ymm1[7] sched: [8:0.50]
-; ZNVER1-NEXT:    vaddps %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = shufflevector <8 x float> %a0, <8 x float> %a1, <8 x i32> <i32 0, i32 9, i32 10, i32 3, i32 4, i32 5, i32 6, i32 7>
-  %2 = load <8 x float>, <8 x float> *%a2, align 32
-  %3 = shufflevector <8 x float> %a1, <8 x float> %2, <8 x i32> <i32 0, i32 1, i32 10, i32 3, i32 12, i32 13, i32 14, i32 7>
-  %4 = fadd <8 x float> %1, %3
-  ret <8 x float> %4
-}
-
-define <4 x double> @test_blendvpd(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2, <4 x double> *%a3) {
-; GENERIC-LABEL: test_blendvpd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vblendvpd %ymm2, %ymm1, %ymm0, %ymm0 # sched: [2:1.00]
-; GENERIC-NEXT:    vblendvpd %ymm2, (%rdi), %ymm0, %ymm0 # sched: [9:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_blendvpd:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vblendvpd %ymm2, %ymm1, %ymm0, %ymm0 # sched: [2:1.00]
-; SANDY-NEXT:    vblendvpd %ymm2, (%rdi), %ymm0, %ymm0 # sched: [9:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_blendvpd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vblendvpd %ymm2, %ymm1, %ymm0, %ymm0 # sched: [2:2.00]
-; HASWELL-NEXT:    vblendvpd %ymm2, (%rdi), %ymm0, %ymm0 # sched: [9:2.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_blendvpd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vblendvpd %ymm2, %ymm1, %ymm0, %ymm0 # sched: [2:2.00]
-; BROADWELL-NEXT:    vblendvpd %ymm2, (%rdi), %ymm0, %ymm0 # sched: [8:2.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_blendvpd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vblendvpd %ymm2, %ymm1, %ymm0, %ymm0 # sched: [2:0.67]
-; SKYLAKE-NEXT:    vblendvpd %ymm2, (%rdi), %ymm0, %ymm0 # sched: [9:0.67]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_blendvpd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vblendvpd %ymm2, %ymm1, %ymm0, %ymm0 # sched: [2:0.67]
-; SKX-NEXT:    vblendvpd %ymm2, (%rdi), %ymm0, %ymm0 # sched: [9:0.67]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_blendvpd:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vblendvpd %ymm2, %ymm1, %ymm0, %ymm0 # sched: [2:3.00]
-; BDVER2-NEXT:    vblendvpd %ymm2, (%rdi), %ymm0, %ymm0 # sched: [7:3.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_blendvpd:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vblendvpd %ymm2, %ymm1, %ymm0, %ymm0 # sched: [3:3.00]
-; BTVER2-NEXT:    vblendvpd %ymm2, (%rdi), %ymm0, %ymm0 # sched: [8:3.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_blendvpd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vblendvpd %ymm2, %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; ZNVER1-NEXT:    vblendvpd %ymm2, (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <4 x double> @llvm.x86.avx.blendv.pd.256(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2)
-  %2 = load <4 x double>, <4 x double> *%a3, align 32
-  %3 = call <4 x double> @llvm.x86.avx.blendv.pd.256(<4 x double> %1, <4 x double> %2, <4 x double> %a2)
-  ret <4 x double> %3
-}
-declare <4 x double> @llvm.x86.avx.blendv.pd.256(<4 x double>, <4 x double>, <4 x double>) nounwind readnone
-
-define <8 x float> @test_blendvps(<8 x float> %a0, <8 x float> %a1, <8 x float> %a2, <8 x float> *%a3) {
-; GENERIC-LABEL: test_blendvps:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vblendvps %ymm2, %ymm1, %ymm0, %ymm0 # sched: [2:1.00]
-; GENERIC-NEXT:    vblendvps %ymm2, (%rdi), %ymm0, %ymm0 # sched: [9:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_blendvps:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vblendvps %ymm2, %ymm1, %ymm0, %ymm0 # sched: [2:1.00]
-; SANDY-NEXT:    vblendvps %ymm2, (%rdi), %ymm0, %ymm0 # sched: [9:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_blendvps:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vblendvps %ymm2, %ymm1, %ymm0, %ymm0 # sched: [2:2.00]
-; HASWELL-NEXT:    vblendvps %ymm2, (%rdi), %ymm0, %ymm0 # sched: [9:2.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_blendvps:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vblendvps %ymm2, %ymm1, %ymm0, %ymm0 # sched: [2:2.00]
-; BROADWELL-NEXT:    vblendvps %ymm2, (%rdi), %ymm0, %ymm0 # sched: [8:2.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_blendvps:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vblendvps %ymm2, %ymm1, %ymm0, %ymm0 # sched: [2:0.67]
-; SKYLAKE-NEXT:    vblendvps %ymm2, (%rdi), %ymm0, %ymm0 # sched: [9:0.67]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_blendvps:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vblendvps %ymm2, %ymm1, %ymm0, %ymm0 # sched: [2:0.67]
-; SKX-NEXT:    vblendvps %ymm2, (%rdi), %ymm0, %ymm0 # sched: [9:0.67]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_blendvps:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vblendvps %ymm2, %ymm1, %ymm0, %ymm0 # sched: [2:3.00]
-; BDVER2-NEXT:    vblendvps %ymm2, (%rdi), %ymm0, %ymm0 # sched: [7:3.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_blendvps:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vblendvps %ymm2, %ymm1, %ymm0, %ymm0 # sched: [3:3.00]
-; BTVER2-NEXT:    vblendvps %ymm2, (%rdi), %ymm0, %ymm0 # sched: [8:3.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_blendvps:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vblendvps %ymm2, %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; ZNVER1-NEXT:    vblendvps %ymm2, (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <8 x float> @llvm.x86.avx.blendv.ps.256(<8 x float> %a0, <8 x float> %a1, <8 x float> %a2)
-  %2 = load <8 x float>, <8 x float> *%a3, align 32
-  %3 = call <8 x float> @llvm.x86.avx.blendv.ps.256(<8 x float> %1, <8 x float> %2, <8 x float> %a2)
-  ret <8 x float> %3
-}
-declare <8 x float> @llvm.x86.avx.blendv.ps.256(<8 x float>, <8 x float>, <8 x float>) nounwind readnone
-
-define <8 x float> @test_broadcastf128(<4 x float> *%a0) {
-; GENERIC-LABEL: test_broadcastf128:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vbroadcastf128 {{.*#+}} ymm0 = mem[0,1,0,1] sched: [7:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_broadcastf128:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vbroadcastf128 {{.*#+}} ymm0 = mem[0,1,0,1] sched: [7:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_broadcastf128:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vbroadcastf128 {{.*#+}} ymm0 = mem[0,1,0,1] sched: [7:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_broadcastf128:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vbroadcastf128 {{.*#+}} ymm0 = mem[0,1,0,1] sched: [6:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_broadcastf128:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vbroadcastf128 {{.*#+}} ymm0 = mem[0,1,0,1] sched: [7:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_broadcastf128:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vbroadcastf128 {{.*#+}} ymm0 = mem[0,1,0,1] sched: [7:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_broadcastf128:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vbroadcastf128 {{.*#+}} ymm0 = mem[0,1,0,1] sched: [7:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_broadcastf128:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vbroadcastf128 {{.*#+}} ymm0 = mem[0,1,0,1] sched: [6:2.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_broadcastf128:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vbroadcastf128 {{.*#+}} ymm0 = mem[0,1,0,1] sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = load <4 x float>, <4 x float> *%a0, align 32
-  %2 = shufflevector <4 x float> %1, <4 x float> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
-  ret <8 x float> %2
-}
-
-define <4 x double> @test_broadcastsd_ymm(double *%a0) {
-; GENERIC-LABEL: test_broadcastsd_ymm:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vbroadcastsd (%rdi), %ymm0 # sched: [7:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_broadcastsd_ymm:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vbroadcastsd (%rdi), %ymm0 # sched: [7:0.50]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_broadcastsd_ymm:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vbroadcastsd (%rdi), %ymm0 # sched: [7:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_broadcastsd_ymm:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vbroadcastsd (%rdi), %ymm0 # sched: [6:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_broadcastsd_ymm:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vbroadcastsd (%rdi), %ymm0 # sched: [7:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_broadcastsd_ymm:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vbroadcastsd (%rdi), %ymm0 # sched: [7:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_broadcastsd_ymm:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vbroadcastsd (%rdi), %ymm0 # sched: [6:2.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_broadcastsd_ymm:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vbroadcastsd (%rdi), %ymm0 # sched: [6:2.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_broadcastsd_ymm:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vbroadcastsd (%rdi), %ymm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = load double, double *%a0, align 8
-  %2 = insertelement <4 x double> undef, double %1, i32 0
-  %3 = shufflevector <4 x double> %2, <4 x double> undef, <4 x i32> zeroinitializer
-  ret <4 x double> %3
-}
-
-define <4 x float> @test_broadcastss(float *%a0) {
-; GENERIC-LABEL: test_broadcastss:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vbroadcastss (%rdi), %xmm0 # sched: [6:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_broadcastss:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vbroadcastss (%rdi), %xmm0 # sched: [6:0.50]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_broadcastss:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vbroadcastss (%rdi), %xmm0 # sched: [6:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_broadcastss:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vbroadcastss (%rdi), %xmm0 # sched: [5:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_broadcastss:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vbroadcastss (%rdi), %xmm0 # sched: [6:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_broadcastss:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vbroadcastss (%rdi), %xmm0 # sched: [6:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_broadcastss:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vbroadcastss (%rdi), %xmm0 # sched: [7:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_broadcastss:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vbroadcastss (%rdi), %xmm0 # sched: [6:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_broadcastss:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vbroadcastss (%rdi), %xmm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = load float, float *%a0, align 4
-  %2 = insertelement <4 x float> undef, float %1, i32 0
-  %3 = shufflevector <4 x float> %2, <4 x float> undef, <4 x i32> zeroinitializer
-  ret <4 x float> %3
-}
-
-define <8 x float> @test_broadcastss_ymm(float *%a0) {
-; GENERIC-LABEL: test_broadcastss_ymm:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vbroadcastss (%rdi), %ymm0 # sched: [7:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_broadcastss_ymm:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vbroadcastss (%rdi), %ymm0 # sched: [7:0.50]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_broadcastss_ymm:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vbroadcastss (%rdi), %ymm0 # sched: [7:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_broadcastss_ymm:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vbroadcastss (%rdi), %ymm0 # sched: [6:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_broadcastss_ymm:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vbroadcastss (%rdi), %ymm0 # sched: [7:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_broadcastss_ymm:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vbroadcastss (%rdi), %ymm0 # sched: [7:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_broadcastss_ymm:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vbroadcastss (%rdi), %ymm0 # sched: [6:2.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_broadcastss_ymm:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vbroadcastss (%rdi), %ymm0 # sched: [6:2.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_broadcastss_ymm:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vbroadcastss (%rdi), %ymm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = load float, float *%a0, align 4
-  %2 = insertelement <8 x float> undef, float %1, i32 0
-  %3 = shufflevector <8 x float> %2, <8 x float> undef, <8 x i32> zeroinitializer
-  ret <8 x float> %3
-}
-
-define <4 x double> @test_cmppd(<4 x double> %a0, <4 x double> %a1, <4 x double> *%a2) {
-; GENERIC-LABEL: test_cmppd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vcmpeqpd %ymm1, %ymm0, %ymm1 # sched: [3:1.00]
-; GENERIC-NEXT:    vcmpeqpd (%rdi), %ymm0, %ymm0 # sched: [10:1.00]
-; GENERIC-NEXT:    vorpd %ymm0, %ymm1, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_cmppd:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vcmpeqpd %ymm1, %ymm0, %ymm1 # sched: [3:1.00]
-; SANDY-NEXT:    vcmpeqpd (%rdi), %ymm0, %ymm0 # sched: [10:1.00]
-; SANDY-NEXT:    vorpd %ymm0, %ymm1, %ymm0 # sched: [1:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_cmppd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vcmpeqpd %ymm1, %ymm0, %ymm1 # sched: [3:1.00]
-; HASWELL-NEXT:    vcmpeqpd (%rdi), %ymm0, %ymm0 # sched: [10:1.00]
-; HASWELL-NEXT:    vorpd %ymm0, %ymm1, %ymm0 # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_cmppd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vcmpeqpd %ymm1, %ymm0, %ymm1 # sched: [3:1.00]
-; BROADWELL-NEXT:    vcmpeqpd (%rdi), %ymm0, %ymm0 # sched: [9:1.00]
-; BROADWELL-NEXT:    vorpd %ymm0, %ymm1, %ymm0 # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_cmppd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vcmpeqpd %ymm1, %ymm0, %ymm1 # sched: [4:0.50]
-; SKYLAKE-NEXT:    vcmpeqpd (%rdi), %ymm0, %ymm0 # sched: [11:0.50]
-; SKYLAKE-NEXT:    vorpd %ymm0, %ymm1, %ymm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_cmppd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vcmpeqpd %ymm1, %ymm0, %ymm1 # sched: [4:0.50]
-; SKX-NEXT:    vcmpeqpd (%rdi), %ymm0, %ymm0 # sched: [11:0.50]
-; SKX-NEXT:    vorpd %ymm0, %ymm1, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_cmppd:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vcmpeqpd %ymm1, %ymm0, %ymm1 # sched: [2:2.00]
-; BDVER2-NEXT:    vcmpeqpd (%rdi), %ymm0, %ymm0 # sched: [7:2.00]
-; BDVER2-NEXT:    vorpd %ymm0, %ymm1, %ymm0 # sched: [2:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_cmppd:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vcmpeqpd %ymm1, %ymm0, %ymm1 # sched: [2:2.00]
-; BTVER2-NEXT:    vcmpeqpd (%rdi), %ymm0, %ymm0 # sched: [7:2.00]
-; BTVER2-NEXT:    vorpd %ymm0, %ymm1, %ymm0 # sched: [1:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_cmppd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vcmpeqpd %ymm1, %ymm0, %ymm1 # sched: [3:1.00]
-; ZNVER1-NEXT:    vcmpeqpd (%rdi), %ymm0, %ymm0 # sched: [10:1.00]
-; ZNVER1-NEXT:    vorpd %ymm0, %ymm1, %ymm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = fcmp oeq <4 x double> %a0, %a1
-  %2 = load <4 x double>, <4 x double> *%a2, align 32
-  %3 = fcmp oeq <4 x double> %a0, %2
-  %4 = sext <4 x i1> %1 to <4 x i64>
-  %5 = sext <4 x i1> %3 to <4 x i64>
-  %6 = or <4 x i64> %4, %5
-  %7 = bitcast <4 x i64> %6 to <4 x double>
-  ret <4 x double> %7
-}
-
-define <8 x float> @test_cmpps(<8 x float> %a0, <8 x float> %a1, <8 x float> *%a2) {
-; GENERIC-LABEL: test_cmpps:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vcmpeqps %ymm1, %ymm0, %ymm1 # sched: [3:1.00]
-; GENERIC-NEXT:    vcmpeqps (%rdi), %ymm0, %ymm0 # sched: [10:1.00]
-; GENERIC-NEXT:    vorps %ymm0, %ymm1, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_cmpps:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vcmpeqps %ymm1, %ymm0, %ymm1 # sched: [3:1.00]
-; SANDY-NEXT:    vcmpeqps (%rdi), %ymm0, %ymm0 # sched: [10:1.00]
-; SANDY-NEXT:    vorps %ymm0, %ymm1, %ymm0 # sched: [1:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_cmpps:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vcmpeqps %ymm1, %ymm0, %ymm1 # sched: [3:1.00]
-; HASWELL-NEXT:    vcmpeqps (%rdi), %ymm0, %ymm0 # sched: [10:1.00]
-; HASWELL-NEXT:    vorps %ymm0, %ymm1, %ymm0 # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_cmpps:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vcmpeqps %ymm1, %ymm0, %ymm1 # sched: [3:1.00]
-; BROADWELL-NEXT:    vcmpeqps (%rdi), %ymm0, %ymm0 # sched: [9:1.00]
-; BROADWELL-NEXT:    vorps %ymm0, %ymm1, %ymm0 # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_cmpps:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vcmpeqps %ymm1, %ymm0, %ymm1 # sched: [4:0.50]
-; SKYLAKE-NEXT:    vcmpeqps (%rdi), %ymm0, %ymm0 # sched: [11:0.50]
-; SKYLAKE-NEXT:    vorps %ymm0, %ymm1, %ymm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_cmpps:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vcmpeqps %ymm1, %ymm0, %ymm1 # sched: [4:0.50]
-; SKX-NEXT:    vcmpeqps (%rdi), %ymm0, %ymm0 # sched: [11:0.50]
-; SKX-NEXT:    vorps %ymm0, %ymm1, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_cmpps:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vcmpeqps %ymm1, %ymm0, %ymm1 # sched: [2:2.00]
-; BDVER2-NEXT:    vcmpeqps (%rdi), %ymm0, %ymm0 # sched: [7:2.00]
-; BDVER2-NEXT:    vorps %ymm0, %ymm1, %ymm0 # sched: [2:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_cmpps:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vcmpeqps %ymm1, %ymm0, %ymm1 # sched: [2:2.00]
-; BTVER2-NEXT:    vcmpeqps (%rdi), %ymm0, %ymm0 # sched: [7:2.00]
-; BTVER2-NEXT:    vorps %ymm0, %ymm1, %ymm0 # sched: [1:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_cmpps:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vcmpeqps %ymm1, %ymm0, %ymm1 # sched: [3:1.00]
-; ZNVER1-NEXT:    vcmpeqps (%rdi), %ymm0, %ymm0 # sched: [10:1.00]
-; ZNVER1-NEXT:    vorps %ymm0, %ymm1, %ymm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = fcmp oeq <8 x float> %a0, %a1
-  %2 = load <8 x float>, <8 x float> *%a2, align 32
-  %3 = fcmp oeq <8 x float> %a0, %2
-  %4 = sext <8 x i1> %1 to <8 x i32>
-  %5 = sext <8 x i1> %3 to <8 x i32>
-  %6 = or <8 x i32> %4, %5
-  %7 = bitcast <8 x i32> %6 to <8 x float>
-  ret <8 x float> %7
-}
-
-define <4 x double> @test_cvtdq2pd(<4 x i32> %a0, <4 x i32> *%a1) {
-; GENERIC-LABEL: test_cvtdq2pd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vcvtdq2pd %xmm0, %ymm0 # sched: [4:1.00]
-; GENERIC-NEXT:    vcvtdq2pd (%rdi), %ymm1 # sched: [10:1.00]
-; GENERIC-NEXT:    vaddpd %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_cvtdq2pd:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vcvtdq2pd %xmm0, %ymm0 # sched: [4:1.00]
-; SANDY-NEXT:    vcvtdq2pd (%rdi), %ymm1 # sched: [10:1.00]
-; SANDY-NEXT:    vaddpd %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_cvtdq2pd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vcvtdq2pd %xmm0, %ymm0 # sched: [6:1.00]
-; HASWELL-NEXT:    vcvtdq2pd (%rdi), %ymm1 # sched: [12:1.00]
-; HASWELL-NEXT:    vaddpd %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_cvtdq2pd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vcvtdq2pd (%rdi), %ymm1 # sched: [11:1.00]
-; BROADWELL-NEXT:    vcvtdq2pd %xmm0, %ymm0 # sched: [6:1.00]
-; BROADWELL-NEXT:    vaddpd %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_cvtdq2pd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vcvtdq2pd %xmm0, %ymm0 # sched: [7:1.00]
-; SKYLAKE-NEXT:    vcvtdq2pd (%rdi), %ymm1 # sched: [13:1.00]
-; SKYLAKE-NEXT:    vaddpd %ymm1, %ymm0, %ymm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_cvtdq2pd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vcvtdq2pd %xmm0, %ymm0 # sched: [7:1.00]
-; SKX-NEXT:    vcvtdq2pd (%rdi), %ymm1 # sched: [13:1.00]
-; SKX-NEXT:    vaddpd %ymm1, %ymm0, %ymm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_cvtdq2pd:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vcvtdq2pd (%rdi), %ymm1 # sched: [13:2.00]
-; BDVER2-NEXT:    vcvtdq2pd %xmm0, %ymm0 # sched: [8:2.00]
-; BDVER2-NEXT:    vaddpd %ymm1, %ymm0, %ymm0 # sched: [5:2.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_cvtdq2pd:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vcvtdq2pd (%rdi), %ymm1 # sched: [8:2.00]
-; BTVER2-NEXT:    vcvtdq2pd %xmm0, %ymm0 # sched: [3:2.00]
-; BTVER2-NEXT:    vaddpd %ymm1, %ymm0, %ymm0 # sched: [3:2.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_cvtdq2pd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vcvtdq2pd (%rdi), %ymm1 # sched: [12:1.00]
-; ZNVER1-NEXT:    vcvtdq2pd %xmm0, %ymm0 # sched: [5:1.00]
-; ZNVER1-NEXT:    vaddpd %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = sitofp <4 x i32> %a0 to <4 x double>
-  %2 = load <4 x i32>, <4 x i32> *%a1, align 16
-  %3 = sitofp <4 x i32> %2 to <4 x double>
-  %4 = fadd <4 x double> %1, %3
-  ret <4 x double> %4
-}
-
-define <8 x float> @test_cvtdq2ps(<8 x i32> %a0, <8 x i32> *%a1) {
-; GENERIC-LABEL: test_cvtdq2ps:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vcvtdq2ps %ymm0, %ymm0 # sched: [3:1.00]
-; GENERIC-NEXT:    vcvtdq2ps (%rdi), %ymm1 # sched: [10:1.00]
-; GENERIC-NEXT:    vaddps %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_cvtdq2ps:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vcvtdq2ps %ymm0, %ymm0 # sched: [3:1.00]
-; SANDY-NEXT:    vmovaps (%rdi), %xmm1 # sched: [6:0.50]
-; SANDY-NEXT:    vinsertf128 $1, 16(%rdi), %ymm1, %ymm1 # sched: [7:0.50]
-; SANDY-NEXT:    vcvtdq2ps %ymm1, %ymm1 # sched: [3:1.00]
-; SANDY-NEXT:    vaddps %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_cvtdq2ps:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vcvtdq2ps %ymm0, %ymm0 # sched: [3:1.00]
-; HASWELL-NEXT:    vcvtdq2ps (%rdi), %ymm1 # sched: [10:1.00]
-; HASWELL-NEXT:    vaddps %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_cvtdq2ps:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vcvtdq2ps %ymm0, %ymm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    vcvtdq2ps (%rdi), %ymm1 # sched: [9:1.00]
-; BROADWELL-NEXT:    vaddps %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_cvtdq2ps:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vcvtdq2ps %ymm0, %ymm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    vcvtdq2ps (%rdi), %ymm1 # sched: [11:0.50]
-; SKYLAKE-NEXT:    vaddps %ymm1, %ymm0, %ymm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_cvtdq2ps:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vcvtdq2ps %ymm0, %ymm0 # sched: [4:0.50]
-; SKX-NEXT:    vcvtdq2ps (%rdi), %ymm1 # sched: [11:0.50]
-; SKX-NEXT:    vaddps %ymm1, %ymm0, %ymm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_cvtdq2ps:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vcvtdq2ps (%rdi), %ymm1 # sched: [9:2.00]
-; BDVER2-NEXT:    vcvtdq2ps %ymm0, %ymm0 # sched: [4:2.00]
-; BDVER2-NEXT:    vaddps %ymm1, %ymm0, %ymm0 # sched: [5:2.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_cvtdq2ps:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vcvtdq2ps (%rdi), %ymm1 # sched: [8:2.00]
-; BTVER2-NEXT:    vcvtdq2ps %ymm0, %ymm0 # sched: [3:2.00]
-; BTVER2-NEXT:    vaddps %ymm1, %ymm0, %ymm0 # sched: [3:2.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_cvtdq2ps:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vcvtdq2ps (%rdi), %ymm1 # sched: [12:1.00]
-; ZNVER1-NEXT:    vcvtdq2ps %ymm0, %ymm0 # sched: [5:1.00]
-; ZNVER1-NEXT:    vaddps %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = sitofp <8 x i32> %a0 to <8 x float>
-  %2 = load <8 x i32>, <8 x i32> *%a1, align 16
-  %3 = sitofp <8 x i32> %2 to <8 x float>
-  %4 = fadd <8 x float> %1, %3
-  ret <8 x float> %4
-}
-
-define <8 x i32> @test_cvtpd2dq(<4 x double> %a0, <4 x double> *%a1) {
-; GENERIC-LABEL: test_cvtpd2dq:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vcvtpd2dq %ymm0, %xmm0 # sched: [4:1.00]
-; GENERIC-NEXT:    vcvtpd2dqy (%rdi), %xmm1 # sched: [11:1.00]
-; GENERIC-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_cvtpd2dq:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vcvtpd2dq %ymm0, %xmm0 # sched: [4:1.00]
-; SANDY-NEXT:    vcvtpd2dqy (%rdi), %xmm1 # sched: [11:1.00]
-; SANDY-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0 # sched: [1:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_cvtpd2dq:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vcvtpd2dq %ymm0, %xmm0 # sched: [6:1.00]
-; HASWELL-NEXT:    vcvtpd2dqy (%rdi), %xmm1 # sched: [8:1.00]
-; HASWELL-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0 # sched: [3:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_cvtpd2dq:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vcvtpd2dq %ymm0, %xmm0 # sched: [6:1.00]
-; BROADWELL-NEXT:    vcvtpd2dqy (%rdi), %xmm1 # sched: [8:1.00]
-; BROADWELL-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_cvtpd2dq:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vcvtpd2dq %ymm0, %xmm0 # sched: [7:1.00]
-; SKYLAKE-NEXT:    vcvtpd2dqy (%rdi), %xmm1 # sched: [8:1.00]
-; SKYLAKE-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0 # sched: [3:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_cvtpd2dq:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vcvtpd2dq %ymm0, %xmm0 # sched: [7:1.00]
-; SKX-NEXT:    vcvtpd2dqy (%rdi), %xmm1 # sched: [8:0.50]
-; SKX-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0 # sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_cvtpd2dq:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vcvtpd2dqy (%rdi), %xmm1 # sched: [13:2.00]
-; BDVER2-NEXT:    vcvtpd2dq %ymm0, %xmm0 # sched: [8:2.00]
-; BDVER2-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0 # sched: [2:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_cvtpd2dq:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vcvtpd2dqy (%rdi), %xmm1 # sched: [11:2.00]
-; BTVER2-NEXT:    vcvtpd2dq %ymm0, %xmm0 # sched: [6:2.00]
-; BTVER2-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0 # sched: [1:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_cvtpd2dq:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vcvtpd2dqy (%rdi), %xmm1 # sched: [12:1.00]
-; ZNVER1-NEXT:    vcvtpd2dq %ymm0, %xmm0 # sched: [5:1.00]
-; ZNVER1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0 # sched: [2:0.67]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <4 x i32> @llvm.x86.avx.cvt.pd2dq.256(<4 x double> %a0)
-  %2 = load <4 x double>, <4 x double> *%a1, align 32
-  %3 = call <4 x i32> @llvm.x86.avx.cvt.pd2dq.256(<4 x double> %2)
-  %4 = shufflevector <4 x i32> %1, <4 x i32> %3, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-  ret <8 x i32> %4
-}
-declare <4 x i32> @llvm.x86.avx.cvt.pd2dq.256(<4 x double>) nounwind readnone
-
-define <8 x i32> @test_cvttpd2dq(<4 x double> %a0, <4 x double> *%a1) {
-; GENERIC-LABEL: test_cvttpd2dq:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vcvttpd2dq %ymm0, %xmm0 # sched: [4:1.00]
-; GENERIC-NEXT:    vcvttpd2dqy (%rdi), %xmm1 # sched: [11:1.00]
-; GENERIC-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_cvttpd2dq:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vcvttpd2dq %ymm0, %xmm0 # sched: [4:1.00]
-; SANDY-NEXT:    vcvttpd2dqy (%rdi), %xmm1 # sched: [11:1.00]
-; SANDY-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0 # sched: [1:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_cvttpd2dq:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vcvttpd2dq %ymm0, %xmm0 # sched: [6:1.00]
-; HASWELL-NEXT:    vcvttpd2dqy (%rdi), %xmm1 # sched: [8:1.00]
-; HASWELL-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0 # sched: [3:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_cvttpd2dq:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vcvttpd2dq %ymm0, %xmm0 # sched: [6:1.00]
-; BROADWELL-NEXT:    vcvttpd2dqy (%rdi), %xmm1 # sched: [8:1.00]
-; BROADWELL-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_cvttpd2dq:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vcvttpd2dq %ymm0, %xmm0 # sched: [7:1.00]
-; SKYLAKE-NEXT:    vcvttpd2dqy (%rdi), %xmm1 # sched: [8:1.00]
-; SKYLAKE-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0 # sched: [3:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_cvttpd2dq:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vcvttpd2dq %ymm0, %xmm0 # sched: [7:1.00]
-; SKX-NEXT:    vcvttpd2dqy (%rdi), %xmm1 # sched: [8:0.50]
-; SKX-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0 # sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_cvttpd2dq:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vcvttpd2dqy (%rdi), %xmm1 # sched: [13:2.00]
-; BDVER2-NEXT:    vcvttpd2dq %ymm0, %xmm0 # sched: [8:2.00]
-; BDVER2-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0 # sched: [2:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_cvttpd2dq:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vcvttpd2dqy (%rdi), %xmm1 # sched: [11:2.00]
-; BTVER2-NEXT:    vcvttpd2dq %ymm0, %xmm0 # sched: [6:2.00]
-; BTVER2-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0 # sched: [1:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_cvttpd2dq:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vcvttpd2dqy (%rdi), %xmm1 # sched: [12:1.00]
-; ZNVER1-NEXT:    vcvttpd2dq %ymm0, %xmm0 # sched: [5:1.00]
-; ZNVER1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0 # sched: [2:0.67]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = fptosi <4 x double> %a0 to <4 x i32>
-  %2 = load <4 x double>, <4 x double> *%a1, align 32
-  %3 = fptosi <4 x double> %2 to <4 x i32>
-  %4 = shufflevector <4 x i32> %1, <4 x i32> %3, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-  ret <8 x i32> %4
-}
-
-define <8 x float> @test_cvtpd2ps(<4 x double> %a0, <4 x double> *%a1) {
-; GENERIC-LABEL: test_cvtpd2ps:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vcvtpd2ps %ymm0, %xmm0 # sched: [4:1.00]
-; GENERIC-NEXT:    vcvtpd2psy (%rdi), %xmm1 # sched: [11:1.00]
-; GENERIC-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_cvtpd2ps:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vcvtpd2ps %ymm0, %xmm0 # sched: [4:1.00]
-; SANDY-NEXT:    vcvtpd2psy (%rdi), %xmm1 # sched: [11:1.00]
-; SANDY-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0 # sched: [1:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_cvtpd2ps:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vcvtpd2ps %ymm0, %xmm0 # sched: [6:1.00]
-; HASWELL-NEXT:    vcvtpd2psy (%rdi), %xmm1 # sched: [8:1.00]
-; HASWELL-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0 # sched: [3:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_cvtpd2ps:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vcvtpd2ps %ymm0, %xmm0 # sched: [6:1.00]
-; BROADWELL-NEXT:    vcvtpd2psy (%rdi), %xmm1 # sched: [8:1.00]
-; BROADWELL-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_cvtpd2ps:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vcvtpd2ps %ymm0, %xmm0 # sched: [7:1.00]
-; SKYLAKE-NEXT:    vcvtpd2psy (%rdi), %xmm1 # sched: [8:1.00]
-; SKYLAKE-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0 # sched: [3:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_cvtpd2ps:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vcvtpd2ps %ymm0, %xmm0 # sched: [7:1.00]
-; SKX-NEXT:    vcvtpd2psy (%rdi), %xmm1 # sched: [8:1.00]
-; SKX-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0 # sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_cvtpd2ps:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vcvtpd2psy (%rdi), %xmm1 # sched: [13:2.00]
-; BDVER2-NEXT:    vcvtpd2ps %ymm0, %xmm0 # sched: [8:2.00]
-; BDVER2-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0 # sched: [2:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_cvtpd2ps:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vcvtpd2psy (%rdi), %xmm1 # sched: [11:2.00]
-; BTVER2-NEXT:    vcvtpd2ps %ymm0, %xmm0 # sched: [6:2.00]
-; BTVER2-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0 # sched: [1:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_cvtpd2ps:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vcvtpd2psy (%rdi), %xmm1 # sched: [11:1.00]
-; ZNVER1-NEXT:    vcvtpd2ps %ymm0, %xmm0 # sched: [5:1.00]
-; ZNVER1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0 # sched: [2:0.67]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = fptrunc <4 x double> %a0 to <4 x float>
-  %2 = load <4 x double>, <4 x double> *%a1, align 32
-  %3 = fptrunc <4 x double> %2 to <4 x float>
-  %4 = shufflevector <4 x float> %1, <4 x float> %3, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-  ret <8 x float> %4
-}
-
-define <8 x i32> @test_cvtps2dq(<8 x float> %a0, <8 x float> *%a1) {
-; GENERIC-LABEL: test_cvtps2dq:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vcvtps2dq %ymm0, %ymm0 # sched: [3:1.00]
-; GENERIC-NEXT:    vcvtps2dq (%rdi), %ymm1 # sched: [10:1.00]
-; GENERIC-NEXT:    vorpd %ymm1, %ymm0, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_cvtps2dq:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vcvtps2dq %ymm0, %ymm0 # sched: [3:1.00]
-; SANDY-NEXT:    vcvtps2dq (%rdi), %ymm1 # sched: [10:1.00]
-; SANDY-NEXT:    vorpd %ymm1, %ymm0, %ymm0 # sched: [1:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_cvtps2dq:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vcvtps2dq %ymm0, %ymm0 # sched: [3:1.00]
-; HASWELL-NEXT:    vcvtps2dq (%rdi), %ymm1 # sched: [10:1.00]
-; HASWELL-NEXT:    vorpd %ymm1, %ymm0, %ymm0 # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_cvtps2dq:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vcvtps2dq %ymm0, %ymm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    vcvtps2dq (%rdi), %ymm1 # sched: [9:1.00]
-; BROADWELL-NEXT:    vorpd %ymm1, %ymm0, %ymm0 # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_cvtps2dq:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vcvtps2dq %ymm0, %ymm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    vcvtps2dq (%rdi), %ymm1 # sched: [11:0.50]
-; SKYLAKE-NEXT:    vorpd %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_cvtps2dq:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vcvtps2dq %ymm0, %ymm0 # sched: [4:0.50]
-; SKX-NEXT:    vcvtps2dq (%rdi), %ymm1 # sched: [11:0.50]
-; SKX-NEXT:    vorpd %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_cvtps2dq:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vcvtps2dq (%rdi), %ymm1 # sched: [9:2.00]
-; BDVER2-NEXT:    vcvtps2dq %ymm0, %ymm0 # sched: [4:2.00]
-; BDVER2-NEXT:    vorpd %ymm1, %ymm0, %ymm0 # sched: [2:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_cvtps2dq:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vcvtps2dq (%rdi), %ymm1 # sched: [8:2.00]
-; BTVER2-NEXT:    vcvtps2dq %ymm0, %ymm0 # sched: [3:2.00]
-; BTVER2-NEXT:    vorpd %ymm1, %ymm0, %ymm0 # sched: [1:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_cvtps2dq:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vcvtps2dq (%rdi), %ymm1 # sched: [12:1.00]
-; ZNVER1-NEXT:    vcvtps2dq %ymm0, %ymm0 # sched: [5:1.00]
-; ZNVER1-NEXT:    vorpd %ymm1, %ymm0, %ymm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <8 x i32> @llvm.x86.avx.cvt.ps2dq.256(<8 x float> %a0)
-  %2 = load <8 x float>, <8 x float> *%a1, align 32
-  %3 = call <8 x i32> @llvm.x86.avx.cvt.ps2dq.256(<8 x float> %2)
-  %4 = or <8 x i32> %1, %3
-  ret <8 x i32> %4
-}
-declare <8 x i32> @llvm.x86.avx.cvt.ps2dq.256(<8 x float>) nounwind readnone
-
-define <8 x i32> @test_cvttps2dq(<8 x float> %a0, <8 x float> *%a1) {
-; GENERIC-LABEL: test_cvttps2dq:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vcvttps2dq %ymm0, %ymm0 # sched: [3:1.00]
-; GENERIC-NEXT:    vcvttps2dq (%rdi), %ymm1 # sched: [10:1.00]
-; GENERIC-NEXT:    vorps %ymm1, %ymm0, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_cvttps2dq:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vcvttps2dq %ymm0, %ymm0 # sched: [3:1.00]
-; SANDY-NEXT:    vcvttps2dq (%rdi), %ymm1 # sched: [10:1.00]
-; SANDY-NEXT:    vorps %ymm1, %ymm0, %ymm0 # sched: [1:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_cvttps2dq:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vcvttps2dq %ymm0, %ymm0 # sched: [3:1.00]
-; HASWELL-NEXT:    vcvttps2dq (%rdi), %ymm1 # sched: [10:1.00]
-; HASWELL-NEXT:    vorps %ymm1, %ymm0, %ymm0 # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_cvttps2dq:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vcvttps2dq %ymm0, %ymm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    vcvttps2dq (%rdi), %ymm1 # sched: [9:1.00]
-; BROADWELL-NEXT:    vorps %ymm1, %ymm0, %ymm0 # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_cvttps2dq:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vcvttps2dq %ymm0, %ymm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    vcvttps2dq (%rdi), %ymm1 # sched: [11:0.50]
-; SKYLAKE-NEXT:    vorps %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_cvttps2dq:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vcvttps2dq %ymm0, %ymm0 # sched: [3:0.50]
-; SKX-NEXT:    vcvttps2dq (%rdi), %ymm1 # sched: [11:0.50]
-; SKX-NEXT:    vorps %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_cvttps2dq:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vcvttps2dq (%rdi), %ymm1 # sched: [9:2.00]
-; BDVER2-NEXT:    vcvttps2dq %ymm0, %ymm0 # sched: [4:2.00]
-; BDVER2-NEXT:    vorps %ymm1, %ymm0, %ymm0 # sched: [2:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_cvttps2dq:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vcvttps2dq (%rdi), %ymm1 # sched: [8:2.00]
-; BTVER2-NEXT:    vcvttps2dq %ymm0, %ymm0 # sched: [3:2.00]
-; BTVER2-NEXT:    vorps %ymm1, %ymm0, %ymm0 # sched: [1:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_cvttps2dq:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vcvttps2dq (%rdi), %ymm1 # sched: [12:1.00]
-; ZNVER1-NEXT:    vcvttps2dq %ymm0, %ymm0 # sched: [5:1.00]
-; ZNVER1-NEXT:    vorps %ymm1, %ymm0, %ymm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = fptosi <8 x float> %a0 to <8 x i32>
-  %2 = load <8 x float>, <8 x float> *%a1, align 32
-  %3 = fptosi <8 x float> %2 to <8 x i32>
-  %4 = or <8 x i32> %1, %3
-  ret <8 x i32> %4
-}
-
-define <4 x double> @test_divpd(<4 x double> %a0, <4 x double> %a1, <4 x double> *%a2) {
-; GENERIC-LABEL: test_divpd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vdivpd %ymm1, %ymm0, %ymm0 # sched: [45:44.00]
-; GENERIC-NEXT:    vdivpd (%rdi), %ymm0, %ymm0 # sched: [52:44.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_divpd:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vdivpd %ymm1, %ymm0, %ymm0 # sched: [45:44.00]
-; SANDY-NEXT:    vdivpd (%rdi), %ymm0, %ymm0 # sched: [52:44.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_divpd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vdivpd %ymm1, %ymm0, %ymm0 # sched: [35:28.00]
-; HASWELL-NEXT:    vdivpd (%rdi), %ymm0, %ymm0 # sched: [42:28.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_divpd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vdivpd %ymm1, %ymm0, %ymm0 # sched: [23:16.00]
-; BROADWELL-NEXT:    vdivpd (%rdi), %ymm0, %ymm0 # sched: [29:16.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_divpd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vdivpd %ymm1, %ymm0, %ymm0 # sched: [14:5.00]
-; SKYLAKE-NEXT:    vdivpd (%rdi), %ymm0, %ymm0 # sched: [21:8.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_divpd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vdivpd %ymm1, %ymm0, %ymm0 # sched: [14:5.00]
-; SKX-NEXT:    vdivpd (%rdi), %ymm0, %ymm0 # sched: [21:8.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_divpd:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vdivpd %ymm1, %ymm0, %ymm0 # sched: [9:19.00]
-; BDVER2-NEXT:    vdivpd (%rdi), %ymm0, %ymm0 # sched: [14:19.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_divpd:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vdivpd %ymm1, %ymm0, %ymm0 # sched: [38:38.00]
-; BTVER2-NEXT:    vdivpd (%rdi), %ymm0, %ymm0 # sched: [43:38.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_divpd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vdivpd %ymm1, %ymm0, %ymm0 # sched: [15:15.00]
-; ZNVER1-NEXT:    vdivpd (%rdi), %ymm0, %ymm0 # sched: [22:22.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = fdiv <4 x double> %a0, %a1
-  %2 = load <4 x double>, <4 x double> *%a2, align 32
-  %3 = fdiv <4 x double> %1, %2
-  ret <4 x double> %3
-}
-
-define <8 x float> @test_divps(<8 x float> %a0, <8 x float> %a1, <8 x float> *%a2) {
-; GENERIC-LABEL: test_divps:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vdivps %ymm1, %ymm0, %ymm0 # sched: [29:28.00]
-; GENERIC-NEXT:    vdivps (%rdi), %ymm0, %ymm0 # sched: [36:28.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_divps:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vdivps %ymm1, %ymm0, %ymm0 # sched: [29:28.00]
-; SANDY-NEXT:    vdivps (%rdi), %ymm0, %ymm0 # sched: [36:28.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_divps:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vdivps %ymm1, %ymm0, %ymm0 # sched: [21:14.00]
-; HASWELL-NEXT:    vdivps (%rdi), %ymm0, %ymm0 # sched: [28:14.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_divps:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vdivps %ymm1, %ymm0, %ymm0 # sched: [17:10.00]
-; BROADWELL-NEXT:    vdivps (%rdi), %ymm0, %ymm0 # sched: [23:10.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_divps:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vdivps %ymm1, %ymm0, %ymm0 # sched: [11:5.00]
-; SKYLAKE-NEXT:    vdivps (%rdi), %ymm0, %ymm0 # sched: [18:5.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_divps:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vdivps %ymm1, %ymm0, %ymm0 # sched: [11:5.00]
-; SKX-NEXT:    vdivps (%rdi), %ymm0, %ymm0 # sched: [18:5.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_divps:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vdivps %ymm1, %ymm0, %ymm0 # sched: [9:19.00]
-; BDVER2-NEXT:    vdivps (%rdi), %ymm0, %ymm0 # sched: [14:19.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_divps:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vdivps %ymm1, %ymm0, %ymm0 # sched: [38:38.00]
-; BTVER2-NEXT:    vdivps (%rdi), %ymm0, %ymm0 # sched: [43:38.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_divps:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vdivps %ymm1, %ymm0, %ymm0 # sched: [12:12.00]
-; ZNVER1-NEXT:    vdivps (%rdi), %ymm0, %ymm0 # sched: [19:19.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = fdiv <8 x float> %a0, %a1
-  %2 = load <8 x float>, <8 x float> *%a2, align 32
-  %3 = fdiv <8 x float> %1, %2
-  ret <8 x float> %3
-}
-
-define <8 x float> @test_dpps(<8 x float> %a0, <8 x float> %a1, <8 x float> *%a2) {
-; GENERIC-LABEL: test_dpps:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vdpps $7, %ymm1, %ymm0, %ymm0 # sched: [12:2.00]
-; GENERIC-NEXT:    vdpps $7, (%rdi), %ymm0, %ymm0 # sched: [19:2.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_dpps:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vdpps $7, %ymm1, %ymm0, %ymm0 # sched: [12:2.00]
-; SANDY-NEXT:    vdpps $7, (%rdi), %ymm0, %ymm0 # sched: [19:2.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_dpps:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vdpps $7, %ymm1, %ymm0, %ymm0 # sched: [14:2.00]
-; HASWELL-NEXT:    vdpps $7, (%rdi), %ymm0, %ymm0 # sched: [21:2.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_dpps:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vdpps $7, %ymm1, %ymm0, %ymm0 # sched: [14:2.00]
-; BROADWELL-NEXT:    vdpps $7, (%rdi), %ymm0, %ymm0 # sched: [20:2.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_dpps:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vdpps $7, %ymm1, %ymm0, %ymm0 # sched: [13:1.50]
-; SKYLAKE-NEXT:    vdpps $7, (%rdi), %ymm0, %ymm0 # sched: [20:1.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_dpps:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vdpps $7, %ymm1, %ymm0, %ymm0 # sched: [13:1.33]
-; SKX-NEXT:    vdpps $7, (%rdi), %ymm0, %ymm0 # sched: [20:1.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_dpps:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vdpps $7, %ymm1, %ymm0, %ymm0 # sched: [27:3.00]
-; BDVER2-NEXT:    vdpps $7, (%rdi), %ymm0, %ymm0 # sched: [32:3.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_dpps:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vdpps $7, %ymm1, %ymm0, %ymm0 # sched: [12:6.00]
-; BTVER2-NEXT:    vdpps $7, (%rdi), %ymm0, %ymm0 # sched: [17:6.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_dpps:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vdpps $7, %ymm1, %ymm0, %ymm0 # sched: [100:0.25]
-; ZNVER1-NEXT:    vdpps $7, (%rdi), %ymm0, %ymm0 # sched: [100:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <8 x float> @llvm.x86.avx.dp.ps.256(<8 x float> %a0, <8 x float> %a1, i8 7)
-  %2 = load <8 x float>, <8 x float> *%a2, align 32
-  %3 = call <8 x float> @llvm.x86.avx.dp.ps.256(<8 x float> %1, <8 x float> %2, i8 7)
-  ret <8 x float> %3
-}
-declare <8 x float> @llvm.x86.avx.dp.ps.256(<8 x float>, <8 x float>, i8) nounwind readnone
-
-define <4 x float> @test_extractf128(<8 x float> %a0, <8 x float> %a1, <4 x float> *%a2) {
-; GENERIC-LABEL: test_extractf128:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vextractf128 $1, %ymm0, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vextractf128 $1, %ymm1, (%rdi) # sched: [1:1.00]
-; GENERIC-NEXT:    vzeroupper # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_extractf128:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vextractf128 $1, %ymm0, %xmm0 # sched: [1:1.00]
-; SANDY-NEXT:    vextractf128 $1, %ymm1, (%rdi) # sched: [1:1.00]
-; SANDY-NEXT:    vzeroupper # sched: [1:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_extractf128:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vextractf128 $1, %ymm0, %xmm0 # sched: [3:1.00]
-; HASWELL-NEXT:    vextractf128 $1, %ymm1, (%rdi) # sched: [1:1.00]
-; HASWELL-NEXT:    vzeroupper # sched: [0:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_extractf128:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vextractf128 $1, %ymm0, %xmm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    vextractf128 $1, %ymm1, (%rdi) # sched: [1:1.00]
-; BROADWELL-NEXT:    vzeroupper # sched: [0:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_extractf128:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vextractf128 $1, %ymm0, %xmm0 # sched: [3:1.00]
-; SKYLAKE-NEXT:    vextractf128 $1, %ymm1, (%rdi) # sched: [1:1.00]
-; SKYLAKE-NEXT:    vzeroupper # sched: [0:0.67]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_extractf128:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vextractf128 $1, %ymm0, %xmm0 # sched: [3:1.00]
-; SKX-NEXT:    vextractf128 $1, %ymm1, (%rdi) # sched: [1:1.00]
-; SKX-NEXT:    vzeroupper # sched: [0:0.67]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_extractf128:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vextractf128 $1, %ymm0, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    vextractf128 $1, %ymm1, (%rdi) # sched: [7:0.50]
-; BDVER2-NEXT:    vzeroupper # sched: [46:4.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_extractf128:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vextractf128 $1, %ymm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    vextractf128 $1, %ymm1, (%rdi) # sched: [1:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_extractf128:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vextractf128 $1, %ymm0, %xmm0 # sched: [1:0.33]
-; ZNVER1-NEXT:    vextractf128 $1, %ymm1, (%rdi) # sched: [8:0.50]
-; ZNVER1-NEXT:    vzeroupper # sched: [100:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = shufflevector <8 x float> %a0, <8 x float> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-  %2 = shufflevector <8 x float> %a1, <8 x float> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-  store <4 x float> %2, <4 x float> *%a2
-  ret <4 x float> %1
-}
-
-define <4 x double> @test_haddpd(<4 x double> %a0, <4 x double> %a1, <4 x double> *%a2) {
-; GENERIC-LABEL: test_haddpd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vhaddpd %ymm1, %ymm0, %ymm0 # sched: [5:2.00]
-; GENERIC-NEXT:    vhaddpd (%rdi), %ymm0, %ymm0 # sched: [12:2.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_haddpd:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vhaddpd %ymm1, %ymm0, %ymm0 # sched: [5:2.00]
-; SANDY-NEXT:    vhaddpd (%rdi), %ymm0, %ymm0 # sched: [12:2.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_haddpd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vhaddpd %ymm1, %ymm0, %ymm0 # sched: [5:2.00]
-; HASWELL-NEXT:    vhaddpd (%rdi), %ymm0, %ymm0 # sched: [12:2.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_haddpd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vhaddpd %ymm1, %ymm0, %ymm0 # sched: [5:2.00]
-; BROADWELL-NEXT:    vhaddpd (%rdi), %ymm0, %ymm0 # sched: [11:2.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_haddpd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vhaddpd %ymm1, %ymm0, %ymm0 # sched: [6:2.00]
-; SKYLAKE-NEXT:    vhaddpd (%rdi), %ymm0, %ymm0 # sched: [13:2.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_haddpd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vhaddpd %ymm1, %ymm0, %ymm0 # sched: [6:2.00]
-; SKX-NEXT:    vhaddpd (%rdi), %ymm0, %ymm0 # sched: [13:2.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_haddpd:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vhaddpd %ymm1, %ymm0, %ymm0 # sched: [11:2.00]
-; BDVER2-NEXT:    vhaddpd (%rdi), %ymm0, %ymm0 # sched: [16:2.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_haddpd:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vhaddpd %ymm1, %ymm0, %ymm0 # sched: [3:2.00]
-; BTVER2-NEXT:    vhaddpd (%rdi), %ymm0, %ymm0 # sched: [8:2.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_haddpd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vhaddpd %ymm1, %ymm0, %ymm0 # sched: [100:0.25]
-; ZNVER1-NEXT:    vhaddpd (%rdi), %ymm0, %ymm0 # sched: [100:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <4 x double> @llvm.x86.avx.hadd.pd.256(<4 x double> %a0, <4 x double> %a1)
-  %2 = load <4 x double>, <4 x double> *%a2, align 32
-  %3 = call <4 x double> @llvm.x86.avx.hadd.pd.256(<4 x double> %1, <4 x double> %2)
-  ret <4 x double> %3
-}
-declare <4 x double> @llvm.x86.avx.hadd.pd.256(<4 x double>, <4 x double>) nounwind readnone
-
-define <8 x float> @test_haddps(<8 x float> %a0, <8 x float> %a1, <8 x float> *%a2) {
-; GENERIC-LABEL: test_haddps:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vhaddps %ymm1, %ymm0, %ymm0 # sched: [5:2.00]
-; GENERIC-NEXT:    vhaddps (%rdi), %ymm0, %ymm0 # sched: [12:2.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_haddps:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vhaddps %ymm1, %ymm0, %ymm0 # sched: [5:2.00]
-; SANDY-NEXT:    vhaddps (%rdi), %ymm0, %ymm0 # sched: [12:2.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_haddps:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vhaddps %ymm1, %ymm0, %ymm0 # sched: [5:2.00]
-; HASWELL-NEXT:    vhaddps (%rdi), %ymm0, %ymm0 # sched: [12:2.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_haddps:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vhaddps %ymm1, %ymm0, %ymm0 # sched: [5:2.00]
-; BROADWELL-NEXT:    vhaddps (%rdi), %ymm0, %ymm0 # sched: [11:2.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_haddps:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vhaddps %ymm1, %ymm0, %ymm0 # sched: [6:2.00]
-; SKYLAKE-NEXT:    vhaddps (%rdi), %ymm0, %ymm0 # sched: [13:2.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_haddps:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vhaddps %ymm1, %ymm0, %ymm0 # sched: [6:2.00]
-; SKX-NEXT:    vhaddps (%rdi), %ymm0, %ymm0 # sched: [13:2.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_haddps:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vhaddps %ymm1, %ymm0, %ymm0 # sched: [11:2.00]
-; BDVER2-NEXT:    vhaddps (%rdi), %ymm0, %ymm0 # sched: [16:2.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_haddps:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vhaddps %ymm1, %ymm0, %ymm0 # sched: [3:2.00]
-; BTVER2-NEXT:    vhaddps (%rdi), %ymm0, %ymm0 # sched: [8:2.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_haddps:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vhaddps %ymm1, %ymm0, %ymm0 # sched: [100:0.25]
-; ZNVER1-NEXT:    vhaddps (%rdi), %ymm0, %ymm0 # sched: [100:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <8 x float> @llvm.x86.avx.hadd.ps.256(<8 x float> %a0, <8 x float> %a1)
-  %2 = load <8 x float>, <8 x float> *%a2, align 32
-  %3 = call <8 x float> @llvm.x86.avx.hadd.ps.256(<8 x float> %1, <8 x float> %2)
-  ret <8 x float> %3
-}
-declare <8 x float> @llvm.x86.avx.hadd.ps.256(<8 x float>, <8 x float>) nounwind readnone
-
-define <4 x double> @test_hsubpd(<4 x double> %a0, <4 x double> %a1, <4 x double> *%a2) {
-; GENERIC-LABEL: test_hsubpd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vhsubpd %ymm1, %ymm0, %ymm0 # sched: [5:2.00]
-; GENERIC-NEXT:    vhsubpd (%rdi), %ymm0, %ymm0 # sched: [12:2.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_hsubpd:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vhsubpd %ymm1, %ymm0, %ymm0 # sched: [5:2.00]
-; SANDY-NEXT:    vhsubpd (%rdi), %ymm0, %ymm0 # sched: [12:2.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_hsubpd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vhsubpd %ymm1, %ymm0, %ymm0 # sched: [5:2.00]
-; HASWELL-NEXT:    vhsubpd (%rdi), %ymm0, %ymm0 # sched: [12:2.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_hsubpd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vhsubpd %ymm1, %ymm0, %ymm0 # sched: [5:2.00]
-; BROADWELL-NEXT:    vhsubpd (%rdi), %ymm0, %ymm0 # sched: [11:2.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_hsubpd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vhsubpd %ymm1, %ymm0, %ymm0 # sched: [6:2.00]
-; SKYLAKE-NEXT:    vhsubpd (%rdi), %ymm0, %ymm0 # sched: [13:2.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_hsubpd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vhsubpd %ymm1, %ymm0, %ymm0 # sched: [6:2.00]
-; SKX-NEXT:    vhsubpd (%rdi), %ymm0, %ymm0 # sched: [13:2.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_hsubpd:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vhsubpd %ymm1, %ymm0, %ymm0 # sched: [11:2.00]
-; BDVER2-NEXT:    vhsubpd (%rdi), %ymm0, %ymm0 # sched: [16:2.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_hsubpd:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vhsubpd %ymm1, %ymm0, %ymm0 # sched: [3:2.00]
-; BTVER2-NEXT:    vhsubpd (%rdi), %ymm0, %ymm0 # sched: [8:2.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_hsubpd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vhsubpd %ymm1, %ymm0, %ymm0 # sched: [100:0.25]
-; ZNVER1-NEXT:    vhsubpd (%rdi), %ymm0, %ymm0 # sched: [100:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <4 x double> @llvm.x86.avx.hsub.pd.256(<4 x double> %a0, <4 x double> %a1)
-  %2 = load <4 x double>, <4 x double> *%a2, align 32
-  %3 = call <4 x double> @llvm.x86.avx.hsub.pd.256(<4 x double> %1, <4 x double> %2)
-  ret <4 x double> %3
-}
-declare <4 x double> @llvm.x86.avx.hsub.pd.256(<4 x double>, <4 x double>) nounwind readnone
-
-define <8 x float> @test_hsubps(<8 x float> %a0, <8 x float> %a1, <8 x float> *%a2) {
-; GENERIC-LABEL: test_hsubps:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vhsubps %ymm1, %ymm0, %ymm0 # sched: [5:2.00]
-; GENERIC-NEXT:    vhsubps (%rdi), %ymm0, %ymm0 # sched: [12:2.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_hsubps:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vhsubps %ymm1, %ymm0, %ymm0 # sched: [5:2.00]
-; SANDY-NEXT:    vhsubps (%rdi), %ymm0, %ymm0 # sched: [12:2.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_hsubps:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vhsubps %ymm1, %ymm0, %ymm0 # sched: [5:2.00]
-; HASWELL-NEXT:    vhsubps (%rdi), %ymm0, %ymm0 # sched: [12:2.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_hsubps:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vhsubps %ymm1, %ymm0, %ymm0 # sched: [5:2.00]
-; BROADWELL-NEXT:    vhsubps (%rdi), %ymm0, %ymm0 # sched: [11:2.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_hsubps:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vhsubps %ymm1, %ymm0, %ymm0 # sched: [6:2.00]
-; SKYLAKE-NEXT:    vhsubps (%rdi), %ymm0, %ymm0 # sched: [13:2.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_hsubps:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vhsubps %ymm1, %ymm0, %ymm0 # sched: [6:2.00]
-; SKX-NEXT:    vhsubps (%rdi), %ymm0, %ymm0 # sched: [13:2.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_hsubps:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vhsubps %ymm1, %ymm0, %ymm0 # sched: [11:2.00]
-; BDVER2-NEXT:    vhsubps (%rdi), %ymm0, %ymm0 # sched: [16:2.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_hsubps:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vhsubps %ymm1, %ymm0, %ymm0 # sched: [3:2.00]
-; BTVER2-NEXT:    vhsubps (%rdi), %ymm0, %ymm0 # sched: [8:2.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_hsubps:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vhsubps %ymm1, %ymm0, %ymm0 # sched: [100:0.25]
-; ZNVER1-NEXT:    vhsubps (%rdi), %ymm0, %ymm0 # sched: [100:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <8 x float> @llvm.x86.avx.hsub.ps.256(<8 x float> %a0, <8 x float> %a1)
-  %2 = load <8 x float>, <8 x float> *%a2, align 32
-  %3 = call <8 x float> @llvm.x86.avx.hsub.ps.256(<8 x float> %1, <8 x float> %2)
-  ret <8 x float> %3
-}
-declare <8 x float> @llvm.x86.avx.hsub.ps.256(<8 x float>, <8 x float>) nounwind readnone
-
-define <8 x float> @test_insertf128(<8 x float> %a0, <4 x float> %a1, <4 x float> *%a2) {
-; GENERIC-LABEL: test_insertf128:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm1 # sched: [1:1.00]
-; GENERIC-NEXT:    vinsertf128 $1, (%rdi), %ymm0, %ymm0 # sched: [7:0.50]
-; GENERIC-NEXT:    vaddps %ymm0, %ymm1, %ymm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_insertf128:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm1 # sched: [1:1.00]
-; SANDY-NEXT:    vinsertf128 $1, (%rdi), %ymm0, %ymm0 # sched: [7:0.50]
-; SANDY-NEXT:    vaddps %ymm0, %ymm1, %ymm0 # sched: [3:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_insertf128:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm1 # sched: [3:1.00]
-; HASWELL-NEXT:    vinsertf128 $1, (%rdi), %ymm0, %ymm0 # sched: [7:0.50]
-; HASWELL-NEXT:    vaddps %ymm0, %ymm1, %ymm0 # sched: [3:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_insertf128:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm1 # sched: [3:1.00]
-; BROADWELL-NEXT:    vinsertf128 $1, (%rdi), %ymm0, %ymm0 # sched: [6:0.50]
-; BROADWELL-NEXT:    vaddps %ymm0, %ymm1, %ymm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_insertf128:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm1 # sched: [3:1.00]
-; SKYLAKE-NEXT:    vinsertf128 $1, (%rdi), %ymm0, %ymm0 # sched: [7:0.50]
-; SKYLAKE-NEXT:    vaddps %ymm0, %ymm1, %ymm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_insertf128:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm1 # sched: [3:1.00]
-; SKX-NEXT:    vinsertf128 $1, (%rdi), %ymm0, %ymm0 # sched: [7:0.50]
-; SKX-NEXT:    vaddps %ymm0, %ymm1, %ymm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_insertf128:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm1 # sched: [2:0.50]
-; BDVER2-NEXT:    vinsertf128 $1, (%rdi), %ymm0, %ymm0 # sched: [7:0.50]
-; BDVER2-NEXT:    vaddps %ymm0, %ymm1, %ymm0 # sched: [5:2.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_insertf128:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm1 # sched: [1:1.00]
-; BTVER2-NEXT:    vinsertf128 $1, (%rdi), %ymm0, %ymm0 # sched: [6:1.00]
-; BTVER2-NEXT:    vaddps %ymm0, %ymm1, %ymm0 # sched: [3:2.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_insertf128:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm1 # sched: [2:0.67]
-; ZNVER1-NEXT:    vinsertf128 $1, (%rdi), %ymm0, %ymm0 # sched: [9:0.67]
-; ZNVER1-NEXT:    vaddps %ymm0, %ymm1, %ymm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = shufflevector <4 x float> %a1, <4 x float> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-  %2 = shufflevector <8 x float> %a0, <8 x float> %1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
-  %3 = load <4 x float>, <4 x float> *%a2, align 16
-  %4 = shufflevector <4 x float> %3, <4 x float> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-  %5 = shufflevector <8 x float> %a0, <8 x float> %4, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
-  %6 = fadd <8 x float> %2, %5
-  ret <8 x float> %6
-}
-
-define <32 x i8> @test_lddqu(i8* %a0) {
-; GENERIC-LABEL: test_lddqu:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vlddqu (%rdi), %ymm0 # sched: [7:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_lddqu:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vlddqu (%rdi), %ymm0 # sched: [7:0.50]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_lddqu:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vlddqu (%rdi), %ymm0 # sched: [7:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_lddqu:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vlddqu (%rdi), %ymm0 # sched: [6:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_lddqu:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vlddqu (%rdi), %ymm0 # sched: [7:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_lddqu:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vlddqu (%rdi), %ymm0 # sched: [7:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_lddqu:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vlddqu (%rdi), %ymm0 # sched: [5:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_lddqu:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vlddqu (%rdi), %ymm0 # sched: [5:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_lddqu:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vlddqu (%rdi), %ymm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <32 x i8> @llvm.x86.avx.ldu.dq.256(i8* %a0)
-  ret <32 x i8> %1
-}
-declare <32 x i8> @llvm.x86.avx.ldu.dq.256(i8*) nounwind readonly
-
-define <2 x double> @test_maskmovpd(i8* %a0, <2 x i64> %a1, <2 x double> %a2) {
-; GENERIC-LABEL: test_maskmovpd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmaskmovpd (%rdi), %xmm0, %xmm2 # sched: [8:1.00]
-; GENERIC-NEXT:    vmaskmovpd %xmm1, %xmm0, (%rdi) # sched: [5:1.00]
-; GENERIC-NEXT:    vmovapd %xmm2, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_maskmovpd:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vmaskmovpd (%rdi), %xmm0, %xmm2 # sched: [8:1.00]
-; SANDY-NEXT:    vmaskmovpd %xmm1, %xmm0, (%rdi) # sched: [5:1.00]
-; SANDY-NEXT:    vmovapd %xmm2, %xmm0 # sched: [1:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_maskmovpd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vmaskmovpd (%rdi), %xmm0, %xmm2 # sched: [8:2.00]
-; HASWELL-NEXT:    vmaskmovpd %xmm1, %xmm0, (%rdi) # sched: [5:1.00]
-; HASWELL-NEXT:    vmovapd %xmm2, %xmm0 # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_maskmovpd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vmaskmovpd (%rdi), %xmm0, %xmm2 # sched: [7:2.00]
-; BROADWELL-NEXT:    vmaskmovpd %xmm1, %xmm0, (%rdi) # sched: [5:1.00]
-; BROADWELL-NEXT:    vmovapd %xmm2, %xmm0 # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_maskmovpd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vmaskmovpd (%rdi), %xmm0, %xmm2 # sched: [7:0.50]
-; SKYLAKE-NEXT:    vmaskmovpd %xmm1, %xmm0, (%rdi) # sched: [2:1.00]
-; SKYLAKE-NEXT:    vmovapd %xmm2, %xmm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_maskmovpd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmaskmovpd (%rdi), %xmm0, %xmm2 # sched: [7:0.50]
-; SKX-NEXT:    vmaskmovpd %xmm1, %xmm0, (%rdi) # sched: [2:1.00]
-; SKX-NEXT:    vmovapd %xmm2, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_maskmovpd:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vmaskmovpd (%rdi), %xmm0, %xmm2 # sched: [6:1.00]
-; BDVER2-NEXT:    vmaskmovpd %xmm1, %xmm0, (%rdi) # sched: [6:2.00]
-; BDVER2-NEXT:    vmovapd %xmm2, %xmm0 # sched: [1:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_maskmovpd:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vmaskmovpd (%rdi), %xmm0, %xmm2 # sched: [6:1.00]
-; BTVER2-NEXT:    vmaskmovpd %xmm1, %xmm0, (%rdi) # sched: [6:2.00]
-; BTVER2-NEXT:    vmovapd %xmm2, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_maskmovpd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vmaskmovpd (%rdi), %xmm0, %xmm2 # sched: [8:0.50]
-; ZNVER1-NEXT:    vmaskmovpd %xmm1, %xmm0, (%rdi) # sched: [4:0.50]
-; ZNVER1-NEXT:    vmovapd %xmm2, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <2 x double> @llvm.x86.avx.maskload.pd(i8* %a0, <2 x i64> %a1)
-  call void @llvm.x86.avx.maskstore.pd(i8* %a0, <2 x i64> %a1, <2 x double> %a2)
-  ret <2 x double> %1
-}
-declare <2 x double> @llvm.x86.avx.maskload.pd(i8*, <2 x i64>) nounwind readonly
-declare void @llvm.x86.avx.maskstore.pd(i8*, <2 x i64>, <2 x double>) nounwind
-
-define <4 x double> @test_maskmovpd_ymm(i8* %a0, <4 x i64> %a1, <4 x double> %a2) {
-; GENERIC-LABEL: test_maskmovpd_ymm:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmaskmovpd (%rdi), %ymm0, %ymm2 # sched: [9:1.00]
-; GENERIC-NEXT:    vmaskmovpd %ymm1, %ymm0, (%rdi) # sched: [5:1.00]
-; GENERIC-NEXT:    vmovapd %ymm2, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_maskmovpd_ymm:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vmaskmovpd (%rdi), %ymm0, %ymm2 # sched: [9:1.00]
-; SANDY-NEXT:    vmaskmovpd %ymm1, %ymm0, (%rdi) # sched: [5:1.00]
-; SANDY-NEXT:    vmovapd %ymm2, %ymm0 # sched: [1:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_maskmovpd_ymm:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vmaskmovpd (%rdi), %ymm0, %ymm2 # sched: [9:2.00]
-; HASWELL-NEXT:    vmaskmovpd %ymm1, %ymm0, (%rdi) # sched: [5:1.00]
-; HASWELL-NEXT:    vmovapd %ymm2, %ymm0 # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_maskmovpd_ymm:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vmaskmovpd (%rdi), %ymm0, %ymm2 # sched: [8:2.00]
-; BROADWELL-NEXT:    vmaskmovpd %ymm1, %ymm0, (%rdi) # sched: [5:1.00]
-; BROADWELL-NEXT:    vmovapd %ymm2, %ymm0 # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_maskmovpd_ymm:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vmaskmovpd (%rdi), %ymm0, %ymm2 # sched: [8:0.50]
-; SKYLAKE-NEXT:    vmaskmovpd %ymm1, %ymm0, (%rdi) # sched: [2:1.00]
-; SKYLAKE-NEXT:    vmovapd %ymm2, %ymm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_maskmovpd_ymm:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmaskmovpd (%rdi), %ymm0, %ymm2 # sched: [8:0.50]
-; SKX-NEXT:    vmaskmovpd %ymm1, %ymm0, (%rdi) # sched: [2:1.00]
-; SKX-NEXT:    vmovapd %ymm2, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_maskmovpd_ymm:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vmaskmovpd (%rdi), %ymm0, %ymm2 # sched: [6:2.00]
-; BDVER2-NEXT:    vmaskmovpd %ymm1, %ymm0, (%rdi) # sched: [6:2.00]
-; BDVER2-NEXT:    vmovapd %ymm2, %ymm0 # sched: [2:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_maskmovpd_ymm:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vmaskmovpd (%rdi), %ymm0, %ymm2 # sched: [6:2.00]
-; BTVER2-NEXT:    vmaskmovpd %ymm1, %ymm0, (%rdi) # sched: [6:2.00]
-; BTVER2-NEXT:    vmovapd %ymm2, %ymm0 # sched: [1:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_maskmovpd_ymm:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vmaskmovpd (%rdi), %ymm0, %ymm2 # sched: [8:1.00]
-; ZNVER1-NEXT:    vmaskmovpd %ymm1, %ymm0, (%rdi) # sched: [5:1.00]
-; ZNVER1-NEXT:    vmovapd %ymm2, %ymm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <4 x double> @llvm.x86.avx.maskload.pd.256(i8* %a0, <4 x i64> %a1)
-  call void @llvm.x86.avx.maskstore.pd.256(i8* %a0, <4 x i64> %a1, <4 x double> %a2)
-  ret <4 x double> %1
-}
-declare <4 x double> @llvm.x86.avx.maskload.pd.256(i8*, <4 x i64>) nounwind readonly
-declare void @llvm.x86.avx.maskstore.pd.256(i8*, <4 x i64>, <4 x double>) nounwind
-
-define <4 x float> @test_maskmovps(i8* %a0, <4 x i32> %a1, <4 x float> %a2) {
-; GENERIC-LABEL: test_maskmovps:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmaskmovps (%rdi), %xmm0, %xmm2 # sched: [8:1.00]
-; GENERIC-NEXT:    vmaskmovps %xmm1, %xmm0, (%rdi) # sched: [5:1.00]
-; GENERIC-NEXT:    vmovaps %xmm2, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_maskmovps:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vmaskmovps (%rdi), %xmm0, %xmm2 # sched: [8:1.00]
-; SANDY-NEXT:    vmaskmovps %xmm1, %xmm0, (%rdi) # sched: [5:1.00]
-; SANDY-NEXT:    vmovaps %xmm2, %xmm0 # sched: [1:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_maskmovps:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vmaskmovps (%rdi), %xmm0, %xmm2 # sched: [8:2.00]
-; HASWELL-NEXT:    vmaskmovps %xmm1, %xmm0, (%rdi) # sched: [5:1.00]
-; HASWELL-NEXT:    vmovaps %xmm2, %xmm0 # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_maskmovps:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vmaskmovps (%rdi), %xmm0, %xmm2 # sched: [7:2.00]
-; BROADWELL-NEXT:    vmaskmovps %xmm1, %xmm0, (%rdi) # sched: [5:1.00]
-; BROADWELL-NEXT:    vmovaps %xmm2, %xmm0 # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_maskmovps:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vmaskmovps (%rdi), %xmm0, %xmm2 # sched: [7:0.50]
-; SKYLAKE-NEXT:    vmaskmovps %xmm1, %xmm0, (%rdi) # sched: [2:1.00]
-; SKYLAKE-NEXT:    vmovaps %xmm2, %xmm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_maskmovps:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmaskmovps (%rdi), %xmm0, %xmm2 # sched: [7:0.50]
-; SKX-NEXT:    vmaskmovps %xmm1, %xmm0, (%rdi) # sched: [2:1.00]
-; SKX-NEXT:    vmovaps %xmm2, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_maskmovps:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vmaskmovps (%rdi), %xmm0, %xmm2 # sched: [6:1.00]
-; BDVER2-NEXT:    vmaskmovps %xmm1, %xmm0, (%rdi) # sched: [6:2.00]
-; BDVER2-NEXT:    vmovaps %xmm2, %xmm0 # sched: [1:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_maskmovps:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vmaskmovps (%rdi), %xmm0, %xmm2 # sched: [6:1.00]
-; BTVER2-NEXT:    vmaskmovps %xmm1, %xmm0, (%rdi) # sched: [6:2.00]
-; BTVER2-NEXT:    vmovaps %xmm2, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_maskmovps:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vmaskmovps (%rdi), %xmm0, %xmm2 # sched: [8:0.50]
-; ZNVER1-NEXT:    vmaskmovps %xmm1, %xmm0, (%rdi) # sched: [4:0.50]
-; ZNVER1-NEXT:    vmovaps %xmm2, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <4 x float> @llvm.x86.avx.maskload.ps(i8* %a0, <4 x i32> %a1)
-  call void @llvm.x86.avx.maskstore.ps(i8* %a0, <4 x i32> %a1, <4 x float> %a2)
-  ret <4 x float> %1
-}
-declare <4 x float> @llvm.x86.avx.maskload.ps(i8*, <4 x i32>) nounwind readonly
-declare void @llvm.x86.avx.maskstore.ps(i8*, <4 x i32>, <4 x float>) nounwind
-
-define <8 x float> @test_maskmovps_ymm(i8* %a0, <8 x i32> %a1, <8 x float> %a2) {
-; GENERIC-LABEL: test_maskmovps_ymm:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmaskmovps (%rdi), %ymm0, %ymm2 # sched: [9:1.00]
-; GENERIC-NEXT:    vmaskmovps %ymm1, %ymm0, (%rdi) # sched: [5:1.00]
-; GENERIC-NEXT:    vmovaps %ymm2, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_maskmovps_ymm:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vmaskmovps (%rdi), %ymm0, %ymm2 # sched: [9:1.00]
-; SANDY-NEXT:    vmaskmovps %ymm1, %ymm0, (%rdi) # sched: [5:1.00]
-; SANDY-NEXT:    vmovaps %ymm2, %ymm0 # sched: [1:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_maskmovps_ymm:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vmaskmovps (%rdi), %ymm0, %ymm2 # sched: [9:2.00]
-; HASWELL-NEXT:    vmaskmovps %ymm1, %ymm0, (%rdi) # sched: [5:1.00]
-; HASWELL-NEXT:    vmovaps %ymm2, %ymm0 # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_maskmovps_ymm:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vmaskmovps (%rdi), %ymm0, %ymm2 # sched: [8:2.00]
-; BROADWELL-NEXT:    vmaskmovps %ymm1, %ymm0, (%rdi) # sched: [5:1.00]
-; BROADWELL-NEXT:    vmovaps %ymm2, %ymm0 # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_maskmovps_ymm:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vmaskmovps (%rdi), %ymm0, %ymm2 # sched: [8:0.50]
-; SKYLAKE-NEXT:    vmaskmovps %ymm1, %ymm0, (%rdi) # sched: [2:1.00]
-; SKYLAKE-NEXT:    vmovaps %ymm2, %ymm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_maskmovps_ymm:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmaskmovps (%rdi), %ymm0, %ymm2 # sched: [8:0.50]
-; SKX-NEXT:    vmaskmovps %ymm1, %ymm0, (%rdi) # sched: [2:1.00]
-; SKX-NEXT:    vmovaps %ymm2, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_maskmovps_ymm:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vmaskmovps (%rdi), %ymm0, %ymm2 # sched: [6:2.00]
-; BDVER2-NEXT:    vmaskmovps %ymm1, %ymm0, (%rdi) # sched: [6:2.00]
-; BDVER2-NEXT:    vmovaps %ymm2, %ymm0 # sched: [2:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_maskmovps_ymm:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vmaskmovps (%rdi), %ymm0, %ymm2 # sched: [6:2.00]
-; BTVER2-NEXT:    vmaskmovps %ymm1, %ymm0, (%rdi) # sched: [6:2.00]
-; BTVER2-NEXT:    vmovaps %ymm2, %ymm0 # sched: [1:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_maskmovps_ymm:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vmaskmovps (%rdi), %ymm0, %ymm2 # sched: [8:1.00]
-; ZNVER1-NEXT:    vmaskmovps %ymm1, %ymm0, (%rdi) # sched: [5:1.00]
-; ZNVER1-NEXT:    vmovaps %ymm2, %ymm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <8 x float> @llvm.x86.avx.maskload.ps.256(i8* %a0, <8 x i32> %a1)
-  call void @llvm.x86.avx.maskstore.ps.256(i8* %a0, <8 x i32> %a1, <8 x float> %a2)
-  ret <8 x float> %1
-}
-declare <8 x float> @llvm.x86.avx.maskload.ps.256(i8*, <8 x i32>) nounwind readonly
-declare void @llvm.x86.avx.maskstore.ps.256(i8*, <8 x i32>, <8 x float>) nounwind
-
-define <4 x double> @test_maxpd(<4 x double> %a0, <4 x double> %a1, <4 x double> *%a2) {
-; GENERIC-LABEL: test_maxpd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmaxpd %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; GENERIC-NEXT:    vmaxpd (%rdi), %ymm0, %ymm0 # sched: [10:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_maxpd:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vmaxpd %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; SANDY-NEXT:    vmaxpd (%rdi), %ymm0, %ymm0 # sched: [10:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_maxpd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vmaxpd %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; HASWELL-NEXT:    vmaxpd (%rdi), %ymm0, %ymm0 # sched: [10:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_maxpd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vmaxpd %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    vmaxpd (%rdi), %ymm0, %ymm0 # sched: [9:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_maxpd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vmaxpd %ymm1, %ymm0, %ymm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    vmaxpd (%rdi), %ymm0, %ymm0 # sched: [11:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_maxpd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmaxpd %ymm1, %ymm0, %ymm0 # sched: [4:0.50]
-; SKX-NEXT:    vmaxpd (%rdi), %ymm0, %ymm0 # sched: [11:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_maxpd:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vmaxpd %ymm1, %ymm0, %ymm0 # sched: [2:2.00]
-; BDVER2-NEXT:    vmaxpd (%rdi), %ymm0, %ymm0 # sched: [7:2.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_maxpd:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vmaxpd %ymm1, %ymm0, %ymm0 # sched: [2:2.00]
-; BTVER2-NEXT:    vmaxpd (%rdi), %ymm0, %ymm0 # sched: [7:2.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_maxpd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vmaxpd %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    vmaxpd (%rdi), %ymm0, %ymm0 # sched: [10:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <4 x double> @llvm.x86.avx.max.pd.256(<4 x double> %a0, <4 x double> %a1)
-  %2 = load <4 x double>, <4 x double> *%a2, align 32
-  %3 = call <4 x double> @llvm.x86.avx.max.pd.256(<4 x double> %1, <4 x double> %2)
-  ret <4 x double> %3
-}
-declare <4 x double> @llvm.x86.avx.max.pd.256(<4 x double>, <4 x double>) nounwind readnone
-
-define <8 x float> @test_maxps(<8 x float> %a0, <8 x float> %a1, <8 x float> *%a2) {
-; GENERIC-LABEL: test_maxps:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmaxps %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; GENERIC-NEXT:    vmaxps (%rdi), %ymm0, %ymm0 # sched: [10:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_maxps:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vmaxps %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; SANDY-NEXT:    vmaxps (%rdi), %ymm0, %ymm0 # sched: [10:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_maxps:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vmaxps %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; HASWELL-NEXT:    vmaxps (%rdi), %ymm0, %ymm0 # sched: [10:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_maxps:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vmaxps %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    vmaxps (%rdi), %ymm0, %ymm0 # sched: [9:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_maxps:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vmaxps %ymm1, %ymm0, %ymm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    vmaxps (%rdi), %ymm0, %ymm0 # sched: [11:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_maxps:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmaxps %ymm1, %ymm0, %ymm0 # sched: [4:0.50]
-; SKX-NEXT:    vmaxps (%rdi), %ymm0, %ymm0 # sched: [11:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_maxps:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vmaxps %ymm1, %ymm0, %ymm0 # sched: [2:2.00]
-; BDVER2-NEXT:    vmaxps (%rdi), %ymm0, %ymm0 # sched: [7:2.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_maxps:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vmaxps %ymm1, %ymm0, %ymm0 # sched: [2:2.00]
-; BTVER2-NEXT:    vmaxps (%rdi), %ymm0, %ymm0 # sched: [7:2.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_maxps:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vmaxps %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    vmaxps (%rdi), %ymm0, %ymm0 # sched: [10:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <8 x float> @llvm.x86.avx.max.ps.256(<8 x float> %a0, <8 x float> %a1)
-  %2 = load <8 x float>, <8 x float> *%a2, align 32
-  %3 = call <8 x float> @llvm.x86.avx.max.ps.256(<8 x float> %1, <8 x float> %2)
-  ret <8 x float> %3
-}
-declare <8 x float> @llvm.x86.avx.max.ps.256(<8 x float>, <8 x float>) nounwind readnone
-
-define <4 x double> @test_minpd(<4 x double> %a0, <4 x double> %a1, <4 x double> *%a2) {
-; GENERIC-LABEL: test_minpd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vminpd %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; GENERIC-NEXT:    vminpd (%rdi), %ymm0, %ymm0 # sched: [10:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_minpd:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vminpd %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; SANDY-NEXT:    vminpd (%rdi), %ymm0, %ymm0 # sched: [10:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_minpd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vminpd %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; HASWELL-NEXT:    vminpd (%rdi), %ymm0, %ymm0 # sched: [10:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_minpd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vminpd %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    vminpd (%rdi), %ymm0, %ymm0 # sched: [9:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_minpd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vminpd %ymm1, %ymm0, %ymm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    vminpd (%rdi), %ymm0, %ymm0 # sched: [11:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_minpd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vminpd %ymm1, %ymm0, %ymm0 # sched: [4:0.50]
-; SKX-NEXT:    vminpd (%rdi), %ymm0, %ymm0 # sched: [11:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_minpd:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vminpd %ymm1, %ymm0, %ymm0 # sched: [2:2.00]
-; BDVER2-NEXT:    vminpd (%rdi), %ymm0, %ymm0 # sched: [7:2.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_minpd:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vminpd %ymm1, %ymm0, %ymm0 # sched: [2:2.00]
-; BTVER2-NEXT:    vminpd (%rdi), %ymm0, %ymm0 # sched: [7:2.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_minpd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vminpd %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    vminpd (%rdi), %ymm0, %ymm0 # sched: [10:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <4 x double> @llvm.x86.avx.min.pd.256(<4 x double> %a0, <4 x double> %a1)
-  %2 = load <4 x double>, <4 x double> *%a2, align 32
-  %3 = call <4 x double> @llvm.x86.avx.min.pd.256(<4 x double> %1, <4 x double> %2)
-  ret <4 x double> %3
-}
-declare <4 x double> @llvm.x86.avx.min.pd.256(<4 x double>, <4 x double>) nounwind readnone
-
-define <8 x float> @test_minps(<8 x float> %a0, <8 x float> %a1, <8 x float> *%a2) {
-; GENERIC-LABEL: test_minps:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vminps %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; GENERIC-NEXT:    vminps (%rdi), %ymm0, %ymm0 # sched: [10:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_minps:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vminps %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; SANDY-NEXT:    vminps (%rdi), %ymm0, %ymm0 # sched: [10:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_minps:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vminps %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; HASWELL-NEXT:    vminps (%rdi), %ymm0, %ymm0 # sched: [10:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_minps:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vminps %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    vminps (%rdi), %ymm0, %ymm0 # sched: [9:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_minps:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vminps %ymm1, %ymm0, %ymm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    vminps (%rdi), %ymm0, %ymm0 # sched: [11:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_minps:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vminps %ymm1, %ymm0, %ymm0 # sched: [4:0.50]
-; SKX-NEXT:    vminps (%rdi), %ymm0, %ymm0 # sched: [11:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_minps:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vminps %ymm1, %ymm0, %ymm0 # sched: [2:2.00]
-; BDVER2-NEXT:    vminps (%rdi), %ymm0, %ymm0 # sched: [7:2.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_minps:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vminps %ymm1, %ymm0, %ymm0 # sched: [2:2.00]
-; BTVER2-NEXT:    vminps (%rdi), %ymm0, %ymm0 # sched: [7:2.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_minps:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vminps %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    vminps (%rdi), %ymm0, %ymm0 # sched: [10:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <8 x float> @llvm.x86.avx.min.ps.256(<8 x float> %a0, <8 x float> %a1)
-  %2 = load <8 x float>, <8 x float> *%a2, align 32
-  %3 = call <8 x float> @llvm.x86.avx.min.ps.256(<8 x float> %1, <8 x float> %2)
-  ret <8 x float> %3
-}
-declare <8 x float> @llvm.x86.avx.min.ps.256(<8 x float>, <8 x float>) nounwind readnone
-
-define <4 x double> @test_movapd(<4 x double> *%a0, <4 x double> *%a1) {
-; GENERIC-LABEL: test_movapd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovapd (%rdi), %ymm0 # sched: [7:0.50]
-; GENERIC-NEXT:    vaddpd %ymm0, %ymm0, %ymm0 # sched: [3:1.00]
-; GENERIC-NEXT:    vmovapd %ymm0, (%rsi) # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_movapd:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vmovapd (%rdi), %ymm0 # sched: [7:0.50]
-; SANDY-NEXT:    vaddpd %ymm0, %ymm0, %ymm0 # sched: [3:1.00]
-; SANDY-NEXT:    vmovapd %ymm0, (%rsi) # sched: [1:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_movapd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vmovapd (%rdi), %ymm0 # sched: [7:0.50]
-; HASWELL-NEXT:    vaddpd %ymm0, %ymm0, %ymm0 # sched: [3:1.00]
-; HASWELL-NEXT:    vmovapd %ymm0, (%rsi) # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_movapd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vmovapd (%rdi), %ymm0 # sched: [6:0.50]
-; BROADWELL-NEXT:    vaddpd %ymm0, %ymm0, %ymm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    vmovapd %ymm0, (%rsi) # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_movapd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vmovapd (%rdi), %ymm0 # sched: [7:0.50]
-; SKYLAKE-NEXT:    vaddpd %ymm0, %ymm0, %ymm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    vmovapd %ymm0, (%rsi) # sched: [1:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_movapd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovapd (%rdi), %ymm0 # sched: [7:0.50]
-; SKX-NEXT:    vaddpd %ymm0, %ymm0, %ymm0 # sched: [4:0.50]
-; SKX-NEXT:    vmovapd %ymm0, (%rsi) # sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_movapd:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vmovapd (%rdi), %ymm0 # sched: [5:0.50]
-; BDVER2-NEXT:    vaddpd %ymm0, %ymm0, %ymm0 # sched: [5:2.00]
-; BDVER2-NEXT:    vmovapd %ymm0, (%rsi) # sched: [1:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_movapd:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vmovapd (%rdi), %ymm0 # sched: [5:1.00]
-; BTVER2-NEXT:    vaddpd %ymm0, %ymm0, %ymm0 # sched: [3:2.00]
-; BTVER2-NEXT:    vmovapd %ymm0, (%rsi) # sched: [1:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_movapd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vmovapd (%rdi), %ymm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    vaddpd %ymm0, %ymm0, %ymm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    vmovapd %ymm0, (%rsi) # sched: [1:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = load <4 x double>, <4 x double> *%a0, align 32
-  %2 = fadd <4 x double> %1, %1
-  store <4 x double> %2, <4 x double> *%a1, align 32
-  ret <4 x double> %2
-}
-
-define <8 x float> @test_movaps(<8 x float> *%a0, <8 x float> *%a1) {
-; GENERIC-LABEL: test_movaps:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovaps (%rdi), %ymm0 # sched: [7:0.50]
-; GENERIC-NEXT:    vaddps %ymm0, %ymm0, %ymm0 # sched: [3:1.00]
-; GENERIC-NEXT:    vmovaps %ymm0, (%rsi) # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_movaps:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vmovaps (%rdi), %ymm0 # sched: [7:0.50]
-; SANDY-NEXT:    vaddps %ymm0, %ymm0, %ymm0 # sched: [3:1.00]
-; SANDY-NEXT:    vmovaps %ymm0, (%rsi) # sched: [1:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_movaps:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vmovaps (%rdi), %ymm0 # sched: [7:0.50]
-; HASWELL-NEXT:    vaddps %ymm0, %ymm0, %ymm0 # sched: [3:1.00]
-; HASWELL-NEXT:    vmovaps %ymm0, (%rsi) # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_movaps:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vmovaps (%rdi), %ymm0 # sched: [6:0.50]
-; BROADWELL-NEXT:    vaddps %ymm0, %ymm0, %ymm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    vmovaps %ymm0, (%rsi) # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_movaps:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vmovaps (%rdi), %ymm0 # sched: [7:0.50]
-; SKYLAKE-NEXT:    vaddps %ymm0, %ymm0, %ymm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    vmovaps %ymm0, (%rsi) # sched: [1:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_movaps:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovaps (%rdi), %ymm0 # sched: [7:0.50]
-; SKX-NEXT:    vaddps %ymm0, %ymm0, %ymm0 # sched: [4:0.50]
-; SKX-NEXT:    vmovaps %ymm0, (%rsi) # sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_movaps:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vmovaps (%rdi), %ymm0 # sched: [5:0.50]
-; BDVER2-NEXT:    vaddps %ymm0, %ymm0, %ymm0 # sched: [5:2.00]
-; BDVER2-NEXT:    vmovaps %ymm0, (%rsi) # sched: [1:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_movaps:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vmovaps (%rdi), %ymm0 # sched: [5:1.00]
-; BTVER2-NEXT:    vaddps %ymm0, %ymm0, %ymm0 # sched: [3:2.00]
-; BTVER2-NEXT:    vmovaps %ymm0, (%rsi) # sched: [1:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_movaps:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vmovaps (%rdi), %ymm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    vaddps %ymm0, %ymm0, %ymm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    vmovaps %ymm0, (%rsi) # sched: [1:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = load <8 x float>, <8 x float> *%a0, align 32
-  %2 = fadd <8 x float> %1, %1
-  store <8 x float> %2, <8 x float> *%a1, align 32
-  ret <8 x float> %2
-}
-
-define <4 x double> @test_movddup(<4 x double> %a0, <4 x double> *%a1) {
-; GENERIC-LABEL: test_movddup:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovddup {{.*#+}} ymm0 = ymm0[0,0,2,2] sched: [1:1.00]
-; GENERIC-NEXT:    vmovddup {{.*#+}} ymm1 = mem[0,0,2,2] sched: [7:0.50]
-; GENERIC-NEXT:    vaddpd %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_movddup:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vmovddup {{.*#+}} ymm0 = ymm0[0,0,2,2] sched: [1:1.00]
-; SANDY-NEXT:    vmovddup {{.*#+}} ymm1 = mem[0,0,2,2] sched: [7:0.50]
-; SANDY-NEXT:    vaddpd %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_movddup:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vmovddup {{.*#+}} ymm0 = ymm0[0,0,2,2] sched: [1:1.00]
-; HASWELL-NEXT:    vmovddup {{.*#+}} ymm1 = mem[0,0,2,2] sched: [7:0.50]
-; HASWELL-NEXT:    vaddpd %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_movddup:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vmovddup {{.*#+}} ymm0 = ymm0[0,0,2,2] sched: [1:1.00]
-; BROADWELL-NEXT:    vmovddup {{.*#+}} ymm1 = mem[0,0,2,2] sched: [6:0.50]
-; BROADWELL-NEXT:    vaddpd %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_movddup:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vmovddup {{.*#+}} ymm0 = ymm0[0,0,2,2] sched: [1:1.00]
-; SKYLAKE-NEXT:    vmovddup {{.*#+}} ymm1 = mem[0,0,2,2] sched: [7:0.50]
-; SKYLAKE-NEXT:    vaddpd %ymm1, %ymm0, %ymm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_movddup:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovddup {{.*#+}} ymm0 = ymm0[0,0,2,2] sched: [1:1.00]
-; SKX-NEXT:    vmovddup {{.*#+}} ymm1 = mem[0,0,2,2] sched: [7:0.50]
-; SKX-NEXT:    vaddpd %ymm1, %ymm0, %ymm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_movddup:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vmovddup {{.*#+}} ymm1 = mem[0,0,2,2] sched: [7:1.00]
-; BDVER2-NEXT:    vmovddup {{.*#+}} ymm0 = ymm0[0,0,2,2] sched: [2:1.00]
-; BDVER2-NEXT:    vaddpd %ymm1, %ymm0, %ymm0 # sched: [5:2.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_movddup:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vmovddup {{.*#+}} ymm1 = mem[0,0,2,2] sched: [6:2.00]
-; BTVER2-NEXT:    vmovddup {{.*#+}} ymm0 = ymm0[0,0,2,2] sched: [1:1.00]
-; BTVER2-NEXT:    vaddpd %ymm1, %ymm0, %ymm0 # sched: [3:2.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_movddup:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vmovddup {{.*#+}} ymm1 = mem[0,0,2,2] sched: [8:0.50]
-; ZNVER1-NEXT:    vmovddup {{.*#+}} ymm0 = ymm0[0,0,2,2] sched: [1:0.50]
-; ZNVER1-NEXT:    vaddpd %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = shufflevector <4 x double> %a0, <4 x double> undef, <4 x i32> <i32 0, i32 0, i32 2, i32 2>
-  %2 = load <4 x double>, <4 x double> *%a1, align 32
-  %3 = shufflevector <4 x double> %2, <4 x double> undef, <4 x i32> <i32 0, i32 0, i32 2, i32 2>
-  %4 = fadd <4 x double> %1, %3
-  ret <4 x double> %4
-}
-
-define i32 @test_movmskpd(<4 x double> %a0) {
-; GENERIC-LABEL: test_movmskpd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovmskpd %ymm0, %eax # sched: [2:1.00]
-; GENERIC-NEXT:    vzeroupper # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_movmskpd:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vmovmskpd %ymm0, %eax # sched: [2:1.00]
-; SANDY-NEXT:    vzeroupper # sched: [1:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_movmskpd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vmovmskpd %ymm0, %eax # sched: [3:1.00]
-; HASWELL-NEXT:    vzeroupper # sched: [0:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_movmskpd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vmovmskpd %ymm0, %eax # sched: [3:1.00]
-; BROADWELL-NEXT:    vzeroupper # sched: [0:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_movmskpd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vmovmskpd %ymm0, %eax # sched: [2:1.00]
-; SKYLAKE-NEXT:    vzeroupper # sched: [0:0.67]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_movmskpd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovmskpd %ymm0, %eax # sched: [2:1.00]
-; SKX-NEXT:    vzeroupper # sched: [0:0.67]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_movmskpd:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vmovmskpd %ymm0, %eax # sched: [10:1.00]
-; BDVER2-NEXT:    vzeroupper # sched: [46:4.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_movmskpd:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vmovmskpd %ymm0, %eax # sched: [3:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_movmskpd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vmovmskpd %ymm0, %eax # sched: [1:1.00]
-; ZNVER1-NEXT:    vzeroupper # sched: [100:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call i32 @llvm.x86.avx.movmsk.pd.256(<4 x double> %a0)
-  ret i32 %1
-}
-declare i32 @llvm.x86.avx.movmsk.pd.256(<4 x double>) nounwind readnone
-
-define i32 @test_movmskps(<8 x float> %a0) {
-; GENERIC-LABEL: test_movmskps:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovmskps %ymm0, %eax # sched: [2:1.00]
-; GENERIC-NEXT:    vzeroupper # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_movmskps:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vmovmskps %ymm0, %eax # sched: [2:1.00]
-; SANDY-NEXT:    vzeroupper # sched: [1:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_movmskps:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vmovmskps %ymm0, %eax # sched: [3:1.00]
-; HASWELL-NEXT:    vzeroupper # sched: [0:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_movmskps:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vmovmskps %ymm0, %eax # sched: [3:1.00]
-; BROADWELL-NEXT:    vzeroupper # sched: [0:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_movmskps:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vmovmskps %ymm0, %eax # sched: [2:1.00]
-; SKYLAKE-NEXT:    vzeroupper # sched: [0:0.67]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_movmskps:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovmskps %ymm0, %eax # sched: [2:1.00]
-; SKX-NEXT:    vzeroupper # sched: [0:0.67]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_movmskps:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vmovmskps %ymm0, %eax # sched: [10:1.00]
-; BDVER2-NEXT:    vzeroupper # sched: [46:4.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_movmskps:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vmovmskps %ymm0, %eax # sched: [3:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_movmskps:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vmovmskps %ymm0, %eax # sched: [1:1.00]
-; ZNVER1-NEXT:    vzeroupper # sched: [100:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call i32 @llvm.x86.avx.movmsk.ps.256(<8 x float> %a0)
-  ret i32 %1
-}
-declare i32 @llvm.x86.avx.movmsk.ps.256(<8 x float>) nounwind readnone
-
-define void @test_movntdq(<4 x i64> %a0, <4 x i64> *%a1) {
-; GENERIC-LABEL: test_movntdq:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    vmovntdq %ymm0, (%rdi) # sched: [1:1.00]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    vzeroupper # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_movntdq:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    vmovntdq %ymm0, (%rdi) # sched: [1:1.00]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    vzeroupper # sched: [1:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_movntdq:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    vmovntdq %ymm0, (%rdi) # sched: [1:1.00]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    vzeroupper # sched: [0:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_movntdq:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    vmovntdq %ymm0, (%rdi) # sched: [1:1.00]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    vzeroupper # sched: [0:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_movntdq:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    vmovntdq %ymm0, (%rdi) # sched: [1:1.00]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    vzeroupper # sched: [0:0.67]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_movntdq:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    vmovntdq %ymm0, (%rdi) # sched: [1:1.00]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    vzeroupper # sched: [0:0.67]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_movntdq:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    vmovntdq %ymm0, (%rdi) # sched: [2:2.00]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    vzeroupper # sched: [46:4.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_movntdq:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    vmovntdq %ymm0, (%rdi) # sched: [2:2.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_movntdq:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    vmovntdq %ymm0, (%rdi) # sched: [1:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    vzeroupper # sched: [100:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  call void asm sideeffect "vmovntdq $0, $1", "x,*m"(<4 x i64> %a0, <4 x i64> *%a1)
-  ret void
-}
-
-define <4 x double> @test_movntpd(<4 x double> %a0, <4 x double> *%a1) {
-; GENERIC-LABEL: test_movntpd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vaddpd %ymm0, %ymm0, %ymm0 # sched: [3:1.00]
-; GENERIC-NEXT:    vmovntpd %ymm0, (%rdi) # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_movntpd:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vaddpd %ymm0, %ymm0, %ymm0 # sched: [3:1.00]
-; SANDY-NEXT:    vmovntpd %ymm0, (%rdi) # sched: [1:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_movntpd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vaddpd %ymm0, %ymm0, %ymm0 # sched: [3:1.00]
-; HASWELL-NEXT:    vmovntpd %ymm0, (%rdi) # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_movntpd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vaddpd %ymm0, %ymm0, %ymm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    vmovntpd %ymm0, (%rdi) # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_movntpd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vaddpd %ymm0, %ymm0, %ymm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    vmovntpd %ymm0, (%rdi) # sched: [1:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_movntpd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vaddpd %ymm0, %ymm0, %ymm0 # sched: [4:0.50]
-; SKX-NEXT:    vmovntpd %ymm0, (%rdi) # sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_movntpd:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vaddpd %ymm0, %ymm0, %ymm0 # sched: [5:2.00]
-; BDVER2-NEXT:    vmovntpd %ymm0, (%rdi) # sched: [3:2.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_movntpd:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vaddpd %ymm0, %ymm0, %ymm0 # sched: [3:2.00]
-; BTVER2-NEXT:    vmovntpd %ymm0, (%rdi) # sched: [3:2.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_movntpd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vaddpd %ymm0, %ymm0, %ymm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    vmovntpd %ymm0, (%rdi) # sched: [1:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = fadd <4 x double> %a0, %a0
-  store <4 x double> %1, <4 x double> *%a1, align 32, !nontemporal !0
-  ret <4 x double> %1
-}
-
-define <8 x float> @test_movntps(<8 x float> %a0, <8 x float> *%a1) {
-; GENERIC-LABEL: test_movntps:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vaddps %ymm0, %ymm0, %ymm0 # sched: [3:1.00]
-; GENERIC-NEXT:    vmovntps %ymm0, (%rdi) # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_movntps:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vaddps %ymm0, %ymm0, %ymm0 # sched: [3:1.00]
-; SANDY-NEXT:    vmovntps %ymm0, (%rdi) # sched: [1:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_movntps:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vaddps %ymm0, %ymm0, %ymm0 # sched: [3:1.00]
-; HASWELL-NEXT:    vmovntps %ymm0, (%rdi) # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_movntps:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vaddps %ymm0, %ymm0, %ymm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    vmovntps %ymm0, (%rdi) # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_movntps:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vaddps %ymm0, %ymm0, %ymm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    vmovntps %ymm0, (%rdi) # sched: [1:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_movntps:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vaddps %ymm0, %ymm0, %ymm0 # sched: [4:0.50]
-; SKX-NEXT:    vmovntps %ymm0, (%rdi) # sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_movntps:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vaddps %ymm0, %ymm0, %ymm0 # sched: [5:2.00]
-; BDVER2-NEXT:    vmovntps %ymm0, (%rdi) # sched: [3:2.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_movntps:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vaddps %ymm0, %ymm0, %ymm0 # sched: [3:2.00]
-; BTVER2-NEXT:    vmovntps %ymm0, (%rdi) # sched: [3:2.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_movntps:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vaddps %ymm0, %ymm0, %ymm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    vmovntps %ymm0, (%rdi) # sched: [1:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = fadd <8 x float> %a0, %a0
-  store <8 x float> %1, <8 x float> *%a1, align 32, !nontemporal !0
-  ret <8 x float> %1
-}
-
-define <8 x float> @test_movshdup(<8 x float> %a0, <8 x float> *%a1) {
-; GENERIC-LABEL: test_movshdup:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovshdup {{.*#+}} ymm0 = ymm0[1,1,3,3,5,5,7,7] sched: [1:1.00]
-; GENERIC-NEXT:    vmovshdup {{.*#+}} ymm1 = mem[1,1,3,3,5,5,7,7] sched: [7:0.50]
-; GENERIC-NEXT:    vaddps %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_movshdup:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vmovshdup {{.*#+}} ymm0 = ymm0[1,1,3,3,5,5,7,7] sched: [1:1.00]
-; SANDY-NEXT:    vmovshdup {{.*#+}} ymm1 = mem[1,1,3,3,5,5,7,7] sched: [7:0.50]
-; SANDY-NEXT:    vaddps %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_movshdup:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vmovshdup {{.*#+}} ymm0 = ymm0[1,1,3,3,5,5,7,7] sched: [1:1.00]
-; HASWELL-NEXT:    vmovshdup {{.*#+}} ymm1 = mem[1,1,3,3,5,5,7,7] sched: [7:0.50]
-; HASWELL-NEXT:    vaddps %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_movshdup:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vmovshdup {{.*#+}} ymm0 = ymm0[1,1,3,3,5,5,7,7] sched: [1:1.00]
-; BROADWELL-NEXT:    vmovshdup {{.*#+}} ymm1 = mem[1,1,3,3,5,5,7,7] sched: [6:0.50]
-; BROADWELL-NEXT:    vaddps %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_movshdup:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vmovshdup {{.*#+}} ymm0 = ymm0[1,1,3,3,5,5,7,7] sched: [1:1.00]
-; SKYLAKE-NEXT:    vmovshdup {{.*#+}} ymm1 = mem[1,1,3,3,5,5,7,7] sched: [7:0.50]
-; SKYLAKE-NEXT:    vaddps %ymm1, %ymm0, %ymm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_movshdup:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovshdup {{.*#+}} ymm0 = ymm0[1,1,3,3,5,5,7,7] sched: [1:1.00]
-; SKX-NEXT:    vmovshdup {{.*#+}} ymm1 = mem[1,1,3,3,5,5,7,7] sched: [7:0.50]
-; SKX-NEXT:    vaddps %ymm1, %ymm0, %ymm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_movshdup:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vmovshdup {{.*#+}} ymm1 = mem[1,1,3,3,5,5,7,7] sched: [7:1.00]
-; BDVER2-NEXT:    vmovshdup {{.*#+}} ymm0 = ymm0[1,1,3,3,5,5,7,7] sched: [2:1.00]
-; BDVER2-NEXT:    vaddps %ymm1, %ymm0, %ymm0 # sched: [5:2.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_movshdup:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vmovshdup {{.*#+}} ymm1 = mem[1,1,3,3,5,5,7,7] sched: [6:2.00]
-; BTVER2-NEXT:    vmovshdup {{.*#+}} ymm0 = ymm0[1,1,3,3,5,5,7,7] sched: [1:1.00]
-; BTVER2-NEXT:    vaddps %ymm1, %ymm0, %ymm0 # sched: [3:2.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_movshdup:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vmovshdup {{.*#+}} ymm1 = mem[1,1,3,3,5,5,7,7] sched: [8:0.50]
-; ZNVER1-NEXT:    vmovshdup {{.*#+}} ymm0 = ymm0[1,1,3,3,5,5,7,7] sched: [1:0.50]
-; ZNVER1-NEXT:    vaddps %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = shufflevector <8 x float> %a0, <8 x float> undef, <8 x i32> <i32 1, i32 1, i32 3, i32 3, i32 5, i32 5, i32 7, i32 7>
-  %2 = load <8 x float>, <8 x float> *%a1, align 32
-  %3 = shufflevector <8 x float> %2, <8 x float> undef, <8 x i32> <i32 1, i32 1, i32 3, i32 3, i32 5, i32 5, i32 7, i32 7>
-  %4 = fadd <8 x float> %1, %3
-  ret <8 x float> %4
-}
-
-define <8 x float> @test_movsldup(<8 x float> %a0, <8 x float> *%a1) {
-; GENERIC-LABEL: test_movsldup:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovsldup {{.*#+}} ymm0 = ymm0[0,0,2,2,4,4,6,6] sched: [1:1.00]
-; GENERIC-NEXT:    vmovsldup {{.*#+}} ymm1 = mem[0,0,2,2,4,4,6,6] sched: [7:0.50]
-; GENERIC-NEXT:    vaddps %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_movsldup:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vmovsldup {{.*#+}} ymm0 = ymm0[0,0,2,2,4,4,6,6] sched: [1:1.00]
-; SANDY-NEXT:    vmovsldup {{.*#+}} ymm1 = mem[0,0,2,2,4,4,6,6] sched: [7:0.50]
-; SANDY-NEXT:    vaddps %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_movsldup:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vmovsldup {{.*#+}} ymm0 = ymm0[0,0,2,2,4,4,6,6] sched: [1:1.00]
-; HASWELL-NEXT:    vmovsldup {{.*#+}} ymm1 = mem[0,0,2,2,4,4,6,6] sched: [7:0.50]
-; HASWELL-NEXT:    vaddps %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_movsldup:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vmovsldup {{.*#+}} ymm0 = ymm0[0,0,2,2,4,4,6,6] sched: [1:1.00]
-; BROADWELL-NEXT:    vmovsldup {{.*#+}} ymm1 = mem[0,0,2,2,4,4,6,6] sched: [6:0.50]
-; BROADWELL-NEXT:    vaddps %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_movsldup:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vmovsldup {{.*#+}} ymm0 = ymm0[0,0,2,2,4,4,6,6] sched: [1:1.00]
-; SKYLAKE-NEXT:    vmovsldup {{.*#+}} ymm1 = mem[0,0,2,2,4,4,6,6] sched: [7:0.50]
-; SKYLAKE-NEXT:    vaddps %ymm1, %ymm0, %ymm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_movsldup:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovsldup {{.*#+}} ymm0 = ymm0[0,0,2,2,4,4,6,6] sched: [1:1.00]
-; SKX-NEXT:    vmovsldup {{.*#+}} ymm1 = mem[0,0,2,2,4,4,6,6] sched: [7:0.50]
-; SKX-NEXT:    vaddps %ymm1, %ymm0, %ymm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_movsldup:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vmovsldup {{.*#+}} ymm1 = mem[0,0,2,2,4,4,6,6] sched: [7:1.00]
-; BDVER2-NEXT:    vmovsldup {{.*#+}} ymm0 = ymm0[0,0,2,2,4,4,6,6] sched: [2:1.00]
-; BDVER2-NEXT:    vaddps %ymm1, %ymm0, %ymm0 # sched: [5:2.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_movsldup:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vmovsldup {{.*#+}} ymm1 = mem[0,0,2,2,4,4,6,6] sched: [6:2.00]
-; BTVER2-NEXT:    vmovsldup {{.*#+}} ymm0 = ymm0[0,0,2,2,4,4,6,6] sched: [1:1.00]
-; BTVER2-NEXT:    vaddps %ymm1, %ymm0, %ymm0 # sched: [3:2.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_movsldup:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vmovsldup {{.*#+}} ymm1 = mem[0,0,2,2,4,4,6,6] sched: [8:0.50]
-; ZNVER1-NEXT:    vmovsldup {{.*#+}} ymm0 = ymm0[0,0,2,2,4,4,6,6] sched: [1:0.50]
-; ZNVER1-NEXT:    vaddps %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = shufflevector <8 x float> %a0, <8 x float> undef, <8 x i32> <i32 0, i32 0, i32 2, i32 2, i32 4, i32 4, i32 6, i32 6>
-  %2 = load <8 x float>, <8 x float> *%a1, align 32
-  %3 = shufflevector <8 x float> %2, <8 x float> undef, <8 x i32> <i32 0, i32 0, i32 2, i32 2, i32 4, i32 4, i32 6, i32 6>
-  %4 = fadd <8 x float> %1, %3
-  ret <8 x float> %4
-}
-
-define <4 x double> @test_movupd(<4 x double> *%a0, <4 x double> *%a1) {
-; GENERIC-LABEL: test_movupd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovupd (%rdi), %ymm0 # sched: [7:0.50]
-; GENERIC-NEXT:    vaddpd %ymm0, %ymm0, %ymm0 # sched: [3:1.00]
-; GENERIC-NEXT:    vmovupd %ymm0, (%rsi) # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_movupd:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vmovups (%rdi), %xmm0 # sched: [6:0.50]
-; SANDY-NEXT:    vinsertf128 $1, 16(%rdi), %ymm0, %ymm0 # sched: [7:0.50]
-; SANDY-NEXT:    vaddpd %ymm0, %ymm0, %ymm0 # sched: [3:1.00]
-; SANDY-NEXT:    vextractf128 $1, %ymm0, 16(%rsi) # sched: [1:1.00]
-; SANDY-NEXT:    vmovupd %xmm0, (%rsi) # sched: [1:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_movupd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vmovupd (%rdi), %ymm0 # sched: [7:0.50]
-; HASWELL-NEXT:    vaddpd %ymm0, %ymm0, %ymm0 # sched: [3:1.00]
-; HASWELL-NEXT:    vmovupd %ymm0, (%rsi) # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_movupd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vmovupd (%rdi), %ymm0 # sched: [6:0.50]
-; BROADWELL-NEXT:    vaddpd %ymm0, %ymm0, %ymm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    vmovupd %ymm0, (%rsi) # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_movupd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vmovupd (%rdi), %ymm0 # sched: [7:0.50]
-; SKYLAKE-NEXT:    vaddpd %ymm0, %ymm0, %ymm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    vmovupd %ymm0, (%rsi) # sched: [1:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_movupd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovupd (%rdi), %ymm0 # sched: [7:0.50]
-; SKX-NEXT:    vaddpd %ymm0, %ymm0, %ymm0 # sched: [4:0.50]
-; SKX-NEXT:    vmovupd %ymm0, (%rsi) # sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_movupd:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vmovupd (%rdi), %ymm0 # sched: [5:0.50]
-; BDVER2-NEXT:    vaddpd %ymm0, %ymm0, %ymm0 # sched: [5:2.00]
-; BDVER2-NEXT:    vmovupd %ymm0, (%rsi) # sched: [1:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_movupd:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vmovupd (%rdi), %ymm0 # sched: [5:1.00]
-; BTVER2-NEXT:    vaddpd %ymm0, %ymm0, %ymm0 # sched: [3:2.00]
-; BTVER2-NEXT:    vmovupd %ymm0, (%rsi) # sched: [1:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_movupd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vmovupd (%rdi), %ymm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    vaddpd %ymm0, %ymm0, %ymm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    vmovupd %ymm0, (%rsi) # sched: [1:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = load <4 x double>, <4 x double> *%a0, align 1
-  %2 = fadd <4 x double> %1, %1
-  store <4 x double> %2, <4 x double> *%a1, align 1
-  ret <4 x double> %2
-}
-
-define <8 x float> @test_movups(<8 x float> *%a0, <8 x float> *%a1) {
-; GENERIC-LABEL: test_movups:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovups (%rdi), %ymm0 # sched: [7:0.50]
-; GENERIC-NEXT:    vaddps %ymm0, %ymm0, %ymm0 # sched: [3:1.00]
-; GENERIC-NEXT:    vmovups %ymm0, (%rsi) # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_movups:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vmovups (%rdi), %xmm0 # sched: [6:0.50]
-; SANDY-NEXT:    vinsertf128 $1, 16(%rdi), %ymm0, %ymm0 # sched: [7:0.50]
-; SANDY-NEXT:    vaddps %ymm0, %ymm0, %ymm0 # sched: [3:1.00]
-; SANDY-NEXT:    vextractf128 $1, %ymm0, 16(%rsi) # sched: [1:1.00]
-; SANDY-NEXT:    vmovups %xmm0, (%rsi) # sched: [1:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_movups:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vmovups (%rdi), %ymm0 # sched: [7:0.50]
-; HASWELL-NEXT:    vaddps %ymm0, %ymm0, %ymm0 # sched: [3:1.00]
-; HASWELL-NEXT:    vmovups %ymm0, (%rsi) # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_movups:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vmovups (%rdi), %ymm0 # sched: [6:0.50]
-; BROADWELL-NEXT:    vaddps %ymm0, %ymm0, %ymm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    vmovups %ymm0, (%rsi) # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_movups:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vmovups (%rdi), %ymm0 # sched: [7:0.50]
-; SKYLAKE-NEXT:    vaddps %ymm0, %ymm0, %ymm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    vmovups %ymm0, (%rsi) # sched: [1:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_movups:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovups (%rdi), %ymm0 # sched: [7:0.50]
-; SKX-NEXT:    vaddps %ymm0, %ymm0, %ymm0 # sched: [4:0.50]
-; SKX-NEXT:    vmovups %ymm0, (%rsi) # sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_movups:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vmovups (%rdi), %ymm0 # sched: [5:0.50]
-; BDVER2-NEXT:    vaddps %ymm0, %ymm0, %ymm0 # sched: [5:2.00]
-; BDVER2-NEXT:    vmovups %ymm0, (%rsi) # sched: [1:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_movups:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vmovups (%rdi), %ymm0 # sched: [5:1.00]
-; BTVER2-NEXT:    vaddps %ymm0, %ymm0, %ymm0 # sched: [3:2.00]
-; BTVER2-NEXT:    vmovups %ymm0, (%rsi) # sched: [1:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_movups:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vmovups (%rdi), %ymm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    vaddps %ymm0, %ymm0, %ymm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    vmovups %ymm0, (%rsi) # sched: [1:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = load <8 x float>, <8 x float> *%a0, align 1
-  %2 = fadd <8 x float> %1, %1
-  store <8 x float> %2, <8 x float> *%a1, align 1
-  ret <8 x float> %2
-}
-
-define <4 x double> @test_mulpd(<4 x double> %a0, <4 x double> %a1, <4 x double> *%a2) {
-; GENERIC-LABEL: test_mulpd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmulpd %ymm1, %ymm0, %ymm0 # sched: [5:1.00]
-; GENERIC-NEXT:    vmulpd (%rdi), %ymm0, %ymm0 # sched: [12:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_mulpd:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vmulpd %ymm1, %ymm0, %ymm0 # sched: [5:1.00]
-; SANDY-NEXT:    vmulpd (%rdi), %ymm0, %ymm0 # sched: [12:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_mulpd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vmulpd %ymm1, %ymm0, %ymm0 # sched: [5:0.50]
-; HASWELL-NEXT:    vmulpd (%rdi), %ymm0, %ymm0 # sched: [12:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_mulpd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vmulpd %ymm1, %ymm0, %ymm0 # sched: [3:0.50]
-; BROADWELL-NEXT:    vmulpd (%rdi), %ymm0, %ymm0 # sched: [9:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_mulpd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vmulpd %ymm1, %ymm0, %ymm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    vmulpd (%rdi), %ymm0, %ymm0 # sched: [11:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_mulpd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmulpd %ymm1, %ymm0, %ymm0 # sched: [4:0.50]
-; SKX-NEXT:    vmulpd (%rdi), %ymm0, %ymm0 # sched: [11:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_mulpd:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vmulpd %ymm1, %ymm0, %ymm0 # sched: [5:2.00]
-; BDVER2-NEXT:    vmulpd (%rdi), %ymm0, %ymm0 # sched: [10:2.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_mulpd:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vmulpd %ymm1, %ymm0, %ymm0 # sched: [4:4.00]
-; BTVER2-NEXT:    vmulpd (%rdi), %ymm0, %ymm0 # sched: [9:4.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_mulpd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vmulpd %ymm1, %ymm0, %ymm0 # sched: [4:0.50]
-; ZNVER1-NEXT:    vmulpd (%rdi), %ymm0, %ymm0 # sched: [11:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = fmul <4 x double> %a0, %a1
-  %2 = load <4 x double>, <4 x double> *%a2, align 32
-  %3 = fmul <4 x double> %1, %2
-  ret <4 x double> %3
-}
-
-define <8 x float> @test_mulps(<8 x float> %a0, <8 x float> %a1, <8 x float> *%a2) {
-; GENERIC-LABEL: test_mulps:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmulps %ymm1, %ymm0, %ymm0 # sched: [5:1.00]
-; GENERIC-NEXT:    vmulps (%rdi), %ymm0, %ymm0 # sched: [12:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_mulps:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vmulps %ymm1, %ymm0, %ymm0 # sched: [5:1.00]
-; SANDY-NEXT:    vmulps (%rdi), %ymm0, %ymm0 # sched: [12:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_mulps:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vmulps %ymm1, %ymm0, %ymm0 # sched: [5:0.50]
-; HASWELL-NEXT:    vmulps (%rdi), %ymm0, %ymm0 # sched: [12:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_mulps:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vmulps %ymm1, %ymm0, %ymm0 # sched: [3:0.50]
-; BROADWELL-NEXT:    vmulps (%rdi), %ymm0, %ymm0 # sched: [9:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_mulps:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vmulps %ymm1, %ymm0, %ymm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    vmulps (%rdi), %ymm0, %ymm0 # sched: [11:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_mulps:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmulps %ymm1, %ymm0, %ymm0 # sched: [4:0.50]
-; SKX-NEXT:    vmulps (%rdi), %ymm0, %ymm0 # sched: [11:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_mulps:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vmulps %ymm1, %ymm0, %ymm0 # sched: [5:2.00]
-; BDVER2-NEXT:    vmulps (%rdi), %ymm0, %ymm0 # sched: [10:2.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_mulps:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vmulps %ymm1, %ymm0, %ymm0 # sched: [2:2.00]
-; BTVER2-NEXT:    vmulps (%rdi), %ymm0, %ymm0 # sched: [7:2.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_mulps:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vmulps %ymm1, %ymm0, %ymm0 # sched: [4:0.50]
-; ZNVER1-NEXT:    vmulps (%rdi), %ymm0, %ymm0 # sched: [11:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = fmul <8 x float> %a0, %a1
-  %2 = load <8 x float>, <8 x float> *%a2, align 32
-  %3 = fmul <8 x float> %1, %2
-  ret <8 x float> %3
-}
-
-define <4 x double> @orpd(<4 x double> %a0, <4 x double> %a1, <4 x double> *%a2) {
-; GENERIC-LABEL: orpd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vorpd %ymm1, %ymm0, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vorpd (%rdi), %ymm0, %ymm0 # sched: [8:1.00]
-; GENERIC-NEXT:    vaddpd %ymm0, %ymm1, %ymm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: orpd:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vorpd %ymm1, %ymm0, %ymm0 # sched: [1:1.00]
-; SANDY-NEXT:    vorpd (%rdi), %ymm0, %ymm0 # sched: [8:1.00]
-; SANDY-NEXT:    vaddpd %ymm0, %ymm1, %ymm0 # sched: [3:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: orpd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vorpd %ymm1, %ymm0, %ymm0 # sched: [1:1.00]
-; HASWELL-NEXT:    vorpd (%rdi), %ymm0, %ymm0 # sched: [8:1.00]
-; HASWELL-NEXT:    vaddpd %ymm0, %ymm1, %ymm0 # sched: [3:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: orpd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vorpd %ymm1, %ymm0, %ymm0 # sched: [1:1.00]
-; BROADWELL-NEXT:    vorpd (%rdi), %ymm0, %ymm0 # sched: [7:1.00]
-; BROADWELL-NEXT:    vaddpd %ymm0, %ymm1, %ymm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: orpd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vorpd %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    vorpd (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKYLAKE-NEXT:    vaddpd %ymm0, %ymm1, %ymm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: orpd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vorpd %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    vorpd (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKX-NEXT:    vaddpd %ymm0, %ymm1, %ymm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: orpd:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vorpd %ymm1, %ymm0, %ymm0 # sched: [2:1.00]
-; BDVER2-NEXT:    vorpd (%rdi), %ymm0, %ymm0 # sched: [7:1.00]
-; BDVER2-NEXT:    vaddpd %ymm0, %ymm1, %ymm0 # sched: [5:2.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: orpd:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vorpd %ymm1, %ymm0, %ymm0 # sched: [1:1.00]
-; BTVER2-NEXT:    vorpd (%rdi), %ymm0, %ymm0 # sched: [6:2.00]
-; BTVER2-NEXT:    vaddpd %ymm0, %ymm1, %ymm0 # sched: [3:2.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: orpd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vorpd %ymm1, %ymm0, %ymm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vorpd (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    vaddpd %ymm0, %ymm1, %ymm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = bitcast <4 x double> %a0 to <4 x i64>
-  %2 = bitcast <4 x double> %a1 to <4 x i64>
-  %3 = or <4 x i64> %1, %2
-  %4 = load <4 x double>, <4 x double> *%a2, align 32
-  %5 = bitcast <4 x double> %4 to <4 x i64>
-  %6 = or <4 x i64> %3, %5
-  %7 = bitcast <4 x i64> %6 to <4 x double>
-  %8 = fadd <4 x double> %a1, %7
-  ret <4 x double> %8
-}
-
-define <8 x float> @test_orps(<8 x float> %a0, <8 x float> %a1, <8 x float> *%a2) {
-; GENERIC-LABEL: test_orps:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vorps %ymm1, %ymm0, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vorps (%rdi), %ymm0, %ymm0 # sched: [8:1.00]
-; GENERIC-NEXT:    vaddps %ymm0, %ymm1, %ymm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_orps:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vorps %ymm1, %ymm0, %ymm0 # sched: [1:1.00]
-; SANDY-NEXT:    vorps (%rdi), %ymm0, %ymm0 # sched: [8:1.00]
-; SANDY-NEXT:    vaddps %ymm0, %ymm1, %ymm0 # sched: [3:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_orps:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vorps %ymm1, %ymm0, %ymm0 # sched: [1:1.00]
-; HASWELL-NEXT:    vorps (%rdi), %ymm0, %ymm0 # sched: [8:1.00]
-; HASWELL-NEXT:    vaddps %ymm0, %ymm1, %ymm0 # sched: [3:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_orps:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vorps %ymm1, %ymm0, %ymm0 # sched: [1:1.00]
-; BROADWELL-NEXT:    vorps (%rdi), %ymm0, %ymm0 # sched: [7:1.00]
-; BROADWELL-NEXT:    vaddps %ymm0, %ymm1, %ymm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_orps:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vorps %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    vorps (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKYLAKE-NEXT:    vaddps %ymm0, %ymm1, %ymm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_orps:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vorps %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    vorps (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKX-NEXT:    vaddps %ymm0, %ymm1, %ymm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_orps:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vorps %ymm1, %ymm0, %ymm0 # sched: [2:1.00]
-; BDVER2-NEXT:    vorps (%rdi), %ymm0, %ymm0 # sched: [7:1.00]
-; BDVER2-NEXT:    vaddps %ymm0, %ymm1, %ymm0 # sched: [5:2.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_orps:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vorps %ymm1, %ymm0, %ymm0 # sched: [1:1.00]
-; BTVER2-NEXT:    vorps (%rdi), %ymm0, %ymm0 # sched: [6:2.00]
-; BTVER2-NEXT:    vaddps %ymm0, %ymm1, %ymm0 # sched: [3:2.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_orps:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vorps %ymm1, %ymm0, %ymm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vorps (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    vaddps %ymm0, %ymm1, %ymm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = bitcast <8 x float> %a0 to <4 x i64>
-  %2 = bitcast <8 x float> %a1 to <4 x i64>
-  %3 = or <4 x i64> %1, %2
-  %4 = load <8 x float>, <8 x float> *%a2, align 32
-  %5 = bitcast <8 x float> %4 to <4 x i64>
-  %6 = or <4 x i64> %3, %5
-  %7 = bitcast <4 x i64> %6 to <8 x float>
-  %8 = fadd <8 x float> %a1, %7
-  ret <8 x float> %8
-}
-
-define <4 x double> @test_perm2f128(<4 x double> %a0, <4 x double> %a1, <4 x double> *%a2) {
-; GENERIC-LABEL: test_perm2f128:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vperm2f128 {{.*#+}} ymm1 = ymm0[2,3],ymm1[0,1] sched: [1:1.00]
-; GENERIC-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm0[2,3],mem[0,1] sched: [8:1.00]
-; GENERIC-NEXT:    vaddpd %ymm0, %ymm1, %ymm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_perm2f128:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vperm2f128 {{.*#+}} ymm1 = ymm0[2,3],ymm1[0,1] sched: [1:1.00]
-; SANDY-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm0[2,3],mem[0,1] sched: [8:1.00]
-; SANDY-NEXT:    vaddpd %ymm0, %ymm1, %ymm0 # sched: [3:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_perm2f128:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vperm2f128 {{.*#+}} ymm1 = ymm0[2,3],ymm1[0,1] sched: [3:1.00]
-; HASWELL-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm0[2,3],mem[0,1] sched: [10:1.00]
-; HASWELL-NEXT:    vaddpd %ymm0, %ymm1, %ymm0 # sched: [3:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_perm2f128:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vperm2f128 {{.*#+}} ymm1 = ymm0[2,3],ymm1[0,1] sched: [3:1.00]
-; BROADWELL-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm0[2,3],mem[0,1] sched: [9:1.00]
-; BROADWELL-NEXT:    vaddpd %ymm0, %ymm1, %ymm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_perm2f128:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vperm2f128 {{.*#+}} ymm1 = ymm0[2,3],ymm1[0,1] sched: [3:1.00]
-; SKYLAKE-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm0[2,3],mem[0,1] sched: [10:1.00]
-; SKYLAKE-NEXT:    vaddpd %ymm0, %ymm1, %ymm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_perm2f128:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vperm2f128 {{.*#+}} ymm1 = ymm0[2,3],ymm1[0,1] sched: [3:1.00]
-; SKX-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm0[2,3],mem[0,1] sched: [10:1.00]
-; SKX-NEXT:    vaddpd %ymm0, %ymm1, %ymm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_perm2f128:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vperm2f128 {{.*#+}} ymm1 = ymm0[2,3],ymm1[0,1] sched: [4:0.50]
-; BDVER2-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm0[2,3],mem[0,1] sched: [8:0.50]
-; BDVER2-NEXT:    vaddpd %ymm0, %ymm1, %ymm0 # sched: [5:2.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_perm2f128:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vperm2f128 {{.*#+}} ymm1 = ymm0[2,3],ymm1[0,1] sched: [1:1.00]
-; BTVER2-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm0[2,3],mem[0,1] sched: [6:1.00]
-; BTVER2-NEXT:    vaddpd %ymm0, %ymm1, %ymm0 # sched: [3:2.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_perm2f128:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vperm2f128 {{.*#+}} ymm1 = ymm0[2,3],ymm1[0,1] sched: [100:0.25]
-; ZNVER1-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm0[2,3],mem[0,1] sched: [100:0.25]
-; ZNVER1-NEXT:    vaddpd %ymm0, %ymm1, %ymm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = shufflevector <4 x double> %a0, <4 x double> %a1, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
-  %2 = load <4 x double>, <4 x double> *%a2, align 32
-  %3 = shufflevector <4 x double> %a0, <4 x double> %2, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
-  %4 = fadd <4 x double> %1, %3
-  ret <4 x double> %4
-}
-
-define <2 x double> @test_permilpd(<2 x double> %a0, <2 x double> *%a1) {
-; GENERIC-LABEL: test_permilpd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0] sched: [1:1.00]
-; GENERIC-NEXT:    vpermilpd {{.*#+}} xmm1 = mem[1,0] sched: [7:1.00]
-; GENERIC-NEXT:    vaddpd %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_permilpd:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0] sched: [1:1.00]
-; SANDY-NEXT:    vpermilpd {{.*#+}} xmm1 = mem[1,0] sched: [7:1.00]
-; SANDY-NEXT:    vaddpd %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_permilpd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0] sched: [1:1.00]
-; HASWELL-NEXT:    vpermilpd {{.*#+}} xmm1 = mem[1,0] sched: [7:1.00]
-; HASWELL-NEXT:    vaddpd %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_permilpd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0] sched: [1:1.00]
-; BROADWELL-NEXT:    vpermilpd {{.*#+}} xmm1 = mem[1,0] sched: [6:1.00]
-; BROADWELL-NEXT:    vaddpd %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_permilpd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0] sched: [1:1.00]
-; SKYLAKE-NEXT:    vpermilpd {{.*#+}} xmm1 = mem[1,0] sched: [7:1.00]
-; SKYLAKE-NEXT:    vaddpd %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_permilpd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0] sched: [1:1.00]
-; SKX-NEXT:    vpermilpd {{.*#+}} xmm1 = mem[1,0] sched: [7:1.00]
-; SKX-NEXT:    vaddpd %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_permilpd:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vpermilpd {{.*#+}} xmm1 = mem[1,0] sched: [7:0.50]
-; BDVER2-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0] sched: [2:0.50]
-; BDVER2-NEXT:    vaddpd %xmm1, %xmm0, %xmm0 # sched: [5:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_permilpd:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vpermilpd {{.*#+}} xmm1 = mem[1,0] sched: [6:1.00]
-; BTVER2-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0] sched: [1:0.50]
-; BTVER2-NEXT:    vaddpd %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_permilpd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpermilpd {{.*#+}} xmm1 = mem[1,0] sched: [8:0.50]
-; ZNVER1-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0] sched: [1:0.50]
-; ZNVER1-NEXT:    vaddpd %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = shufflevector <2 x double> %a0, <2 x double> undef, <2 x i32> <i32 1, i32 0>
-  %2 = load <2 x double>, <2 x double> *%a1, align 16
-  %3 = shufflevector <2 x double> %2, <2 x double> undef, <2 x i32> <i32 1, i32 0>
-  %4 = fadd <2 x double> %1, %3
-  ret <2 x double> %4
-}
-
-define <4 x double> @test_permilpd_ymm(<4 x double> %a0, <4 x double> *%a1) {
-; GENERIC-LABEL: test_permilpd_ymm:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpermilpd {{.*#+}} ymm0 = ymm0[1,0,2,3] sched: [1:1.00]
-; GENERIC-NEXT:    vpermilpd {{.*#+}} ymm1 = mem[1,0,2,3] sched: [8:1.00]
-; GENERIC-NEXT:    vaddpd %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_permilpd_ymm:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vpermilpd {{.*#+}} ymm0 = ymm0[1,0,2,3] sched: [1:1.00]
-; SANDY-NEXT:    vpermilpd {{.*#+}} ymm1 = mem[1,0,2,3] sched: [8:1.00]
-; SANDY-NEXT:    vaddpd %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_permilpd_ymm:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpermilpd {{.*#+}} ymm0 = ymm0[1,0,2,3] sched: [1:1.00]
-; HASWELL-NEXT:    vpermilpd {{.*#+}} ymm1 = mem[1,0,2,3] sched: [8:1.00]
-; HASWELL-NEXT:    vaddpd %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_permilpd_ymm:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpermilpd {{.*#+}} ymm0 = ymm0[1,0,2,3] sched: [1:1.00]
-; BROADWELL-NEXT:    vpermilpd {{.*#+}} ymm1 = mem[1,0,2,3] sched: [7:1.00]
-; BROADWELL-NEXT:    vaddpd %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_permilpd_ymm:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpermilpd {{.*#+}} ymm0 = ymm0[1,0,2,3] sched: [1:1.00]
-; SKYLAKE-NEXT:    vpermilpd {{.*#+}} ymm1 = mem[1,0,2,3] sched: [8:1.00]
-; SKYLAKE-NEXT:    vaddpd %ymm1, %ymm0, %ymm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_permilpd_ymm:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpermilpd {{.*#+}} ymm0 = ymm0[1,0,2,3] sched: [1:1.00]
-; SKX-NEXT:    vpermilpd {{.*#+}} ymm1 = mem[1,0,2,3] sched: [8:1.00]
-; SKX-NEXT:    vaddpd %ymm1, %ymm0, %ymm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_permilpd_ymm:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vpermilpd {{.*#+}} ymm1 = mem[1,0,2,3] sched: [7:1.00]
-; BDVER2-NEXT:    vpermilpd {{.*#+}} ymm0 = ymm0[1,0,2,3] sched: [2:1.00]
-; BDVER2-NEXT:    vaddpd %ymm1, %ymm0, %ymm0 # sched: [5:2.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_permilpd_ymm:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vpermilpd {{.*#+}} ymm1 = mem[1,0,2,3] sched: [6:2.00]
-; BTVER2-NEXT:    vpermilpd {{.*#+}} ymm0 = ymm0[1,0,2,3] sched: [1:1.00]
-; BTVER2-NEXT:    vaddpd %ymm1, %ymm0, %ymm0 # sched: [3:2.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_permilpd_ymm:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpermilpd {{.*#+}} ymm1 = mem[1,0,2,3] sched: [8:0.50]
-; ZNVER1-NEXT:    vpermilpd {{.*#+}} ymm0 = ymm0[1,0,2,3] sched: [1:0.50]
-; ZNVER1-NEXT:    vaddpd %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = shufflevector <4 x double> %a0, <4 x double> undef, <4 x i32> <i32 1, i32 0, i32 2, i32 3>
-  %2 = load <4 x double>, <4 x double> *%a1, align 32
-  %3 = shufflevector <4 x double> %2, <4 x double> undef, <4 x i32> <i32 1, i32 0, i32 2, i32 3>
-  %4 = fadd <4 x double> %1, %3
-  ret <4 x double> %4
-}
-
-define <4 x float> @test_permilps(<4 x float> %a0, <4 x float> *%a1) {
-; GENERIC-LABEL: test_permilps:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,2,1,0] sched: [1:1.00]
-; GENERIC-NEXT:    vpermilps {{.*#+}} xmm1 = mem[3,2,1,0] sched: [7:1.00]
-; GENERIC-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_permilps:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,2,1,0] sched: [1:1.00]
-; SANDY-NEXT:    vpermilps {{.*#+}} xmm1 = mem[3,2,1,0] sched: [7:1.00]
-; SANDY-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_permilps:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,2,1,0] sched: [1:1.00]
-; HASWELL-NEXT:    vpermilps {{.*#+}} xmm1 = mem[3,2,1,0] sched: [7:1.00]
-; HASWELL-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_permilps:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,2,1,0] sched: [1:1.00]
-; BROADWELL-NEXT:    vpermilps {{.*#+}} xmm1 = mem[3,2,1,0] sched: [6:1.00]
-; BROADWELL-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_permilps:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,2,1,0] sched: [1:1.00]
-; SKYLAKE-NEXT:    vpermilps {{.*#+}} xmm1 = mem[3,2,1,0] sched: [7:1.00]
-; SKYLAKE-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_permilps:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,2,1,0] sched: [1:1.00]
-; SKX-NEXT:    vpermilps {{.*#+}} xmm1 = mem[3,2,1,0] sched: [7:1.00]
-; SKX-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_permilps:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vpermilps {{.*#+}} xmm1 = mem[3,2,1,0] sched: [7:0.50]
-; BDVER2-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,2,1,0] sched: [2:0.50]
-; BDVER2-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [5:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_permilps:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vpermilps {{.*#+}} xmm1 = mem[3,2,1,0] sched: [6:1.00]
-; BTVER2-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,2,1,0] sched: [1:0.50]
-; BTVER2-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_permilps:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpermilps {{.*#+}} xmm1 = mem[3,2,1,0] sched: [8:0.50]
-; ZNVER1-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,2,1,0] sched: [1:0.50]
-; ZNVER1-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = shufflevector <4 x float> %a0, <4 x float> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
-  %2 = load <4 x float>, <4 x float> *%a1, align 16
-  %3 = shufflevector <4 x float> %2, <4 x float> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
-  %4 = fadd <4 x float> %1, %3
-  ret <4 x float> %4
-}
-
-define <8 x float> @test_permilps_ymm(<8 x float> %a0, <8 x float> *%a1) {
-; GENERIC-LABEL: test_permilps_ymm:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpermilps {{.*#+}} ymm0 = ymm0[3,2,1,0,7,6,5,4] sched: [1:1.00]
-; GENERIC-NEXT:    vpermilps {{.*#+}} ymm1 = mem[3,2,1,0,7,6,5,4] sched: [8:1.00]
-; GENERIC-NEXT:    vaddps %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_permilps_ymm:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vpermilps {{.*#+}} ymm0 = ymm0[3,2,1,0,7,6,5,4] sched: [1:1.00]
-; SANDY-NEXT:    vpermilps {{.*#+}} ymm1 = mem[3,2,1,0,7,6,5,4] sched: [8:1.00]
-; SANDY-NEXT:    vaddps %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_permilps_ymm:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpermilps {{.*#+}} ymm0 = ymm0[3,2,1,0,7,6,5,4] sched: [1:1.00]
-; HASWELL-NEXT:    vpermilps {{.*#+}} ymm1 = mem[3,2,1,0,7,6,5,4] sched: [8:1.00]
-; HASWELL-NEXT:    vaddps %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_permilps_ymm:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpermilps {{.*#+}} ymm0 = ymm0[3,2,1,0,7,6,5,4] sched: [1:1.00]
-; BROADWELL-NEXT:    vpermilps {{.*#+}} ymm1 = mem[3,2,1,0,7,6,5,4] sched: [7:1.00]
-; BROADWELL-NEXT:    vaddps %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_permilps_ymm:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpermilps {{.*#+}} ymm0 = ymm0[3,2,1,0,7,6,5,4] sched: [1:1.00]
-; SKYLAKE-NEXT:    vpermilps {{.*#+}} ymm1 = mem[3,2,1,0,7,6,5,4] sched: [8:1.00]
-; SKYLAKE-NEXT:    vaddps %ymm1, %ymm0, %ymm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_permilps_ymm:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpermilps {{.*#+}} ymm0 = ymm0[3,2,1,0,7,6,5,4] sched: [1:1.00]
-; SKX-NEXT:    vpermilps {{.*#+}} ymm1 = mem[3,2,1,0,7,6,5,4] sched: [8:1.00]
-; SKX-NEXT:    vaddps %ymm1, %ymm0, %ymm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_permilps_ymm:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vpermilps {{.*#+}} ymm1 = mem[3,2,1,0,7,6,5,4] sched: [7:1.00]
-; BDVER2-NEXT:    vpermilps {{.*#+}} ymm0 = ymm0[3,2,1,0,7,6,5,4] sched: [2:1.00]
-; BDVER2-NEXT:    vaddps %ymm1, %ymm0, %ymm0 # sched: [5:2.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_permilps_ymm:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vpermilps {{.*#+}} ymm1 = mem[3,2,1,0,7,6,5,4] sched: [6:2.00]
-; BTVER2-NEXT:    vpermilps {{.*#+}} ymm0 = ymm0[3,2,1,0,7,6,5,4] sched: [1:1.00]
-; BTVER2-NEXT:    vaddps %ymm1, %ymm0, %ymm0 # sched: [3:2.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_permilps_ymm:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpermilps {{.*#+}} ymm1 = mem[3,2,1,0,7,6,5,4] sched: [8:0.50]
-; ZNVER1-NEXT:    vpermilps {{.*#+}} ymm0 = ymm0[3,2,1,0,7,6,5,4] sched: [1:0.50]
-; ZNVER1-NEXT:    vaddps %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = shufflevector <8 x float> %a0, <8 x float> undef, <8 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4>
-  %2 = load <8 x float>, <8 x float> *%a1, align 32
-  %3 = shufflevector <8 x float> %2, <8 x float> undef, <8 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4>
-  %4 = fadd <8 x float> %1, %3
-  ret <8 x float> %4
-}
-
-define <2 x double> @test_permilvarpd(<2 x double> %a0, <2 x i64> %a1, <2 x i64> *%a2) {
-; GENERIC-LABEL: test_permilvarpd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpermilpd %xmm1, %xmm0, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vpermilpd (%rdi), %xmm0, %xmm0 # sched: [7:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_permilvarpd:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vpermilpd %xmm1, %xmm0, %xmm0 # sched: [1:1.00]
-; SANDY-NEXT:    vpermilpd (%rdi), %xmm0, %xmm0 # sched: [7:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_permilvarpd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpermilpd %xmm1, %xmm0, %xmm0 # sched: [1:1.00]
-; HASWELL-NEXT:    vpermilpd (%rdi), %xmm0, %xmm0 # sched: [7:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_permilvarpd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpermilpd %xmm1, %xmm0, %xmm0 # sched: [1:1.00]
-; BROADWELL-NEXT:    vpermilpd (%rdi), %xmm0, %xmm0 # sched: [6:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_permilvarpd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpermilpd %xmm1, %xmm0, %xmm0 # sched: [1:1.00]
-; SKYLAKE-NEXT:    vpermilpd (%rdi), %xmm0, %xmm0 # sched: [7:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_permilvarpd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpermilpd %xmm1, %xmm0, %xmm0 # sched: [1:1.00]
-; SKX-NEXT:    vpermilpd (%rdi), %xmm0, %xmm0 # sched: [7:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_permilvarpd:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vpermilpd %xmm1, %xmm0, %xmm0 # sched: [3:2.00]
-; BDVER2-NEXT:    vpermilpd (%rdi), %xmm0, %xmm0 # sched: [8:2.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_permilvarpd:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vpermilpd %xmm1, %xmm0, %xmm0 # sched: [2:2.00]
-; BTVER2-NEXT:    vpermilpd (%rdi), %xmm0, %xmm0 # sched: [7:2.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_permilvarpd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpermilpd %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; ZNVER1-NEXT:    vpermilpd (%rdi), %xmm0, %xmm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <2 x double> @llvm.x86.avx.vpermilvar.pd(<2 x double> %a0, <2 x i64> %a1)
-  %2 = load <2 x i64>, <2 x i64> *%a2, align 16
-  %3 = call <2 x double> @llvm.x86.avx.vpermilvar.pd(<2 x double> %1, <2 x i64> %2)
-  ret <2 x double> %3
-}
-declare <2 x double> @llvm.x86.avx.vpermilvar.pd(<2 x double>, <2 x i64>) nounwind readnone
-
-define <4 x double> @test_permilvarpd_ymm(<4 x double> %a0, <4 x i64> %a1, <4 x i64> *%a2) {
-; GENERIC-LABEL: test_permilvarpd_ymm:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpermilpd %ymm1, %ymm0, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vpermilpd (%rdi), %ymm0, %ymm0 # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_permilvarpd_ymm:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vpermilpd %ymm1, %ymm0, %ymm0 # sched: [1:1.00]
-; SANDY-NEXT:    vpermilpd (%rdi), %ymm0, %ymm0 # sched: [8:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_permilvarpd_ymm:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpermilpd %ymm1, %ymm0, %ymm0 # sched: [1:1.00]
-; HASWELL-NEXT:    vpermilpd (%rdi), %ymm0, %ymm0 # sched: [8:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_permilvarpd_ymm:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpermilpd %ymm1, %ymm0, %ymm0 # sched: [1:1.00]
-; BROADWELL-NEXT:    vpermilpd (%rdi), %ymm0, %ymm0 # sched: [7:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_permilvarpd_ymm:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpermilpd %ymm1, %ymm0, %ymm0 # sched: [1:1.00]
-; SKYLAKE-NEXT:    vpermilpd (%rdi), %ymm0, %ymm0 # sched: [8:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_permilvarpd_ymm:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpermilpd %ymm1, %ymm0, %ymm0 # sched: [1:1.00]
-; SKX-NEXT:    vpermilpd (%rdi), %ymm0, %ymm0 # sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_permilvarpd_ymm:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vpermilpd %ymm1, %ymm0, %ymm0 # sched: [3:3.00]
-; BDVER2-NEXT:    vpermilpd (%rdi), %ymm0, %ymm0 # sched: [8:3.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_permilvarpd_ymm:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vpermilpd %ymm1, %ymm0, %ymm0 # sched: [3:3.00]
-; BTVER2-NEXT:    vpermilpd (%rdi), %ymm0, %ymm0 # sched: [8:3.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_permilvarpd_ymm:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpermilpd %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; ZNVER1-NEXT:    vpermilpd (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <4 x double> @llvm.x86.avx.vpermilvar.pd.256(<4 x double> %a0, <4 x i64> %a1)
-  %2 = load <4 x i64>, <4 x i64> *%a2, align 32
-  %3 = call <4 x double> @llvm.x86.avx.vpermilvar.pd.256(<4 x double> %1, <4 x i64> %2)
-  ret <4 x double> %3
-}
-declare <4 x double> @llvm.x86.avx.vpermilvar.pd.256(<4 x double>, <4 x i64>) nounwind readnone
-
-define <4 x float> @test_permilvarps(<4 x float> %a0, <4 x i32> %a1, <4 x i32> *%a2) {
-; GENERIC-LABEL: test_permilvarps:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpermilps %xmm1, %xmm0, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vpermilps (%rdi), %xmm0, %xmm0 # sched: [7:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_permilvarps:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vpermilps %xmm1, %xmm0, %xmm0 # sched: [1:1.00]
-; SANDY-NEXT:    vpermilps (%rdi), %xmm0, %xmm0 # sched: [7:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_permilvarps:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpermilps %xmm1, %xmm0, %xmm0 # sched: [1:1.00]
-; HASWELL-NEXT:    vpermilps (%rdi), %xmm0, %xmm0 # sched: [7:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_permilvarps:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpermilps %xmm1, %xmm0, %xmm0 # sched: [1:1.00]
-; BROADWELL-NEXT:    vpermilps (%rdi), %xmm0, %xmm0 # sched: [6:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_permilvarps:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpermilps %xmm1, %xmm0, %xmm0 # sched: [1:1.00]
-; SKYLAKE-NEXT:    vpermilps (%rdi), %xmm0, %xmm0 # sched: [7:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_permilvarps:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpermilps %xmm1, %xmm0, %xmm0 # sched: [1:1.00]
-; SKX-NEXT:    vpermilps (%rdi), %xmm0, %xmm0 # sched: [7:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_permilvarps:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vpermilps %xmm1, %xmm0, %xmm0 # sched: [3:2.00]
-; BDVER2-NEXT:    vpermilps (%rdi), %xmm0, %xmm0 # sched: [8:2.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_permilvarps:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vpermilps %xmm1, %xmm0, %xmm0 # sched: [2:2.00]
-; BTVER2-NEXT:    vpermilps (%rdi), %xmm0, %xmm0 # sched: [7:2.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_permilvarps:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpermilps %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; ZNVER1-NEXT:    vpermilps (%rdi), %xmm0, %xmm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <4 x float> @llvm.x86.avx.vpermilvar.ps(<4 x float> %a0, <4 x i32> %a1)
-  %2 = load <4 x i32>, <4 x i32> *%a2, align 16
-  %3 = call <4 x float> @llvm.x86.avx.vpermilvar.ps(<4 x float> %1, <4 x i32> %2)
-  ret <4 x float> %3
-}
-declare <4 x float> @llvm.x86.avx.vpermilvar.ps(<4 x float>, <4 x i32>) nounwind readnone
-
-define <8 x float> @test_permilvarps_ymm(<8 x float> %a0, <8 x i32> %a1, <8 x i32> *%a2) {
-; GENERIC-LABEL: test_permilvarps_ymm:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpermilps %ymm1, %ymm0, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vpermilps (%rdi), %ymm0, %ymm0 # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_permilvarps_ymm:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vpermilps %ymm1, %ymm0, %ymm0 # sched: [1:1.00]
-; SANDY-NEXT:    vpermilps (%rdi), %ymm0, %ymm0 # sched: [8:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_permilvarps_ymm:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpermilps %ymm1, %ymm0, %ymm0 # sched: [1:1.00]
-; HASWELL-NEXT:    vpermilps (%rdi), %ymm0, %ymm0 # sched: [8:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_permilvarps_ymm:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpermilps %ymm1, %ymm0, %ymm0 # sched: [1:1.00]
-; BROADWELL-NEXT:    vpermilps (%rdi), %ymm0, %ymm0 # sched: [7:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_permilvarps_ymm:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpermilps %ymm1, %ymm0, %ymm0 # sched: [1:1.00]
-; SKYLAKE-NEXT:    vpermilps (%rdi), %ymm0, %ymm0 # sched: [8:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_permilvarps_ymm:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpermilps %ymm1, %ymm0, %ymm0 # sched: [1:1.00]
-; SKX-NEXT:    vpermilps (%rdi), %ymm0, %ymm0 # sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_permilvarps_ymm:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vpermilps %ymm1, %ymm0, %ymm0 # sched: [3:3.00]
-; BDVER2-NEXT:    vpermilps (%rdi), %ymm0, %ymm0 # sched: [8:3.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_permilvarps_ymm:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vpermilps %ymm1, %ymm0, %ymm0 # sched: [3:3.00]
-; BTVER2-NEXT:    vpermilps (%rdi), %ymm0, %ymm0 # sched: [8:3.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_permilvarps_ymm:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpermilps %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; ZNVER1-NEXT:    vpermilps (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <8 x float> @llvm.x86.avx.vpermilvar.ps.256(<8 x float> %a0, <8 x i32> %a1)
-  %2 = load <8 x i32>, <8 x i32> *%a2, align 32
-  %3 = call <8 x float> @llvm.x86.avx.vpermilvar.ps.256(<8 x float> %1, <8 x i32> %2)
-  ret <8 x float> %3
-}
-declare <8 x float> @llvm.x86.avx.vpermilvar.ps.256(<8 x float>, <8 x i32>) nounwind readnone
-
-define <8 x float> @test_rcpps(<8 x float> %a0, <8 x float> *%a1) {
-; GENERIC-LABEL: test_rcpps:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vrcpps (%rdi), %ymm1 # sched: [14:2.00]
-; GENERIC-NEXT:    vrcpps %ymm0, %ymm0 # sched: [7:2.00]
-; GENERIC-NEXT:    vaddps %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_rcpps:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vrcpps (%rdi), %ymm1 # sched: [14:2.00]
-; SANDY-NEXT:    vrcpps %ymm0, %ymm0 # sched: [7:2.00]
-; SANDY-NEXT:    vaddps %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_rcpps:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vrcpps (%rdi), %ymm1 # sched: [18:2.00]
-; HASWELL-NEXT:    vrcpps %ymm0, %ymm0 # sched: [11:2.00]
-; HASWELL-NEXT:    vaddps %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_rcpps:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vrcpps %ymm0, %ymm0 # sched: [11:2.00]
-; BROADWELL-NEXT:    vrcpps (%rdi), %ymm1 # sched: [17:2.00]
-; BROADWELL-NEXT:    vaddps %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_rcpps:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vrcpps %ymm0, %ymm0 # sched: [4:1.00]
-; SKYLAKE-NEXT:    vrcpps (%rdi), %ymm1 # sched: [11:1.00]
-; SKYLAKE-NEXT:    vaddps %ymm1, %ymm0, %ymm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_rcpps:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vrcpps %ymm0, %ymm0 # sched: [4:1.00]
-; SKX-NEXT:    vrcpps (%rdi), %ymm1 # sched: [11:1.00]
-; SKX-NEXT:    vaddps %ymm1, %ymm0, %ymm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_rcpps:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vrcpps (%rdi), %ymm1 # sched: [10:2.00]
-; BDVER2-NEXT:    vrcpps %ymm0, %ymm0 # sched: [5:2.00]
-; BDVER2-NEXT:    vaddps %ymm1, %ymm0, %ymm0 # sched: [5:2.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_rcpps:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vrcpps (%rdi), %ymm1 # sched: [7:2.00]
-; BTVER2-NEXT:    vrcpps %ymm0, %ymm0 # sched: [2:2.00]
-; BTVER2-NEXT:    vaddps %ymm1, %ymm0, %ymm0 # sched: [3:2.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_rcpps:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vrcpps (%rdi), %ymm1 # sched: [12:0.50]
-; ZNVER1-NEXT:    vrcpps %ymm0, %ymm0 # sched: [5:0.50]
-; ZNVER1-NEXT:    vaddps %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <8 x float> @llvm.x86.avx.rcp.ps.256(<8 x float> %a0)
-  %2 = load <8 x float>, <8 x float> *%a1, align 32
-  %3 = call <8 x float> @llvm.x86.avx.rcp.ps.256(<8 x float> %2)
-  %4 = fadd <8 x float> %1, %3
-  ret <8 x float> %4
-}
-declare <8 x float> @llvm.x86.avx.rcp.ps.256(<8 x float>) nounwind readnone
-
-define <4 x double> @test_roundpd(<4 x double> %a0, <4 x double> *%a1) {
-; GENERIC-LABEL: test_roundpd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vroundpd $7, %ymm0, %ymm0 # sched: [3:1.00]
-; GENERIC-NEXT:    vroundpd $7, (%rdi), %ymm1 # sched: [10:1.00]
-; GENERIC-NEXT:    vaddpd %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_roundpd:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vroundpd $7, %ymm0, %ymm0 # sched: [3:1.00]
-; SANDY-NEXT:    vroundpd $7, (%rdi), %ymm1 # sched: [10:1.00]
-; SANDY-NEXT:    vaddpd %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_roundpd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vroundpd $7, %ymm0, %ymm0 # sched: [6:0.50]
-; HASWELL-NEXT:    vroundpd $7, (%rdi), %ymm1 # sched: [13:2.00]
-; HASWELL-NEXT:    vaddpd %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_roundpd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vroundpd $7, (%rdi), %ymm1 # sched: [12:2.00]
-; BROADWELL-NEXT:    vroundpd $7, %ymm0, %ymm0 # sched: [6:0.50]
-; BROADWELL-NEXT:    vaddpd %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_roundpd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vroundpd $7, %ymm0, %ymm0 # sched: [8:1.00]
-; SKYLAKE-NEXT:    vroundpd $7, (%rdi), %ymm1 # sched: [15:1.00]
-; SKYLAKE-NEXT:    vaddpd %ymm1, %ymm0, %ymm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_roundpd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vroundpd $7, %ymm0, %ymm0 # sched: [8:1.00]
-; SKX-NEXT:    vroundpd $7, (%rdi), %ymm1 # sched: [15:1.00]
-; SKX-NEXT:    vaddpd %ymm1, %ymm0, %ymm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_roundpd:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vroundpd $7, (%rdi), %ymm1 # sched: [9:2.00]
-; BDVER2-NEXT:    vroundpd $7, %ymm0, %ymm0 # sched: [4:2.00]
-; BDVER2-NEXT:    vaddpd %ymm1, %ymm0, %ymm0 # sched: [5:2.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_roundpd:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vroundpd $7, (%rdi), %ymm1 # sched: [8:2.00]
-; BTVER2-NEXT:    vroundpd $7, %ymm0, %ymm0 # sched: [3:2.00]
-; BTVER2-NEXT:    vaddpd %ymm1, %ymm0, %ymm0 # sched: [3:2.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_roundpd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vroundpd $7, (%rdi), %ymm1 # sched: [11:1.00]
-; ZNVER1-NEXT:    vroundpd $7, %ymm0, %ymm0 # sched: [4:1.00]
-; ZNVER1-NEXT:    vaddpd %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <4 x double> @llvm.x86.avx.round.pd.256(<4 x double> %a0, i32 7)
-  %2 = load <4 x double>, <4 x double> *%a1, align 32
-  %3 = call <4 x double> @llvm.x86.avx.round.pd.256(<4 x double> %2, i32 7)
-  %4 = fadd <4 x double> %1, %3
-  ret <4 x double> %4
-}
-declare <4 x double> @llvm.x86.avx.round.pd.256(<4 x double>, i32) nounwind readnone
-
-define <8 x float> @test_roundps(<8 x float> %a0, <8 x float> *%a1) {
-; GENERIC-LABEL: test_roundps:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vroundps $7, %ymm0, %ymm0 # sched: [3:1.00]
-; GENERIC-NEXT:    vroundps $7, (%rdi), %ymm1 # sched: [10:1.00]
-; GENERIC-NEXT:    vaddps %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_roundps:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vroundps $7, %ymm0, %ymm0 # sched: [3:1.00]
-; SANDY-NEXT:    vroundps $7, (%rdi), %ymm1 # sched: [10:1.00]
-; SANDY-NEXT:    vaddps %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_roundps:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vroundps $7, %ymm0, %ymm0 # sched: [6:0.50]
-; HASWELL-NEXT:    vroundps $7, (%rdi), %ymm1 # sched: [13:2.00]
-; HASWELL-NEXT:    vaddps %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_roundps:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vroundps $7, (%rdi), %ymm1 # sched: [12:2.00]
-; BROADWELL-NEXT:    vroundps $7, %ymm0, %ymm0 # sched: [6:0.50]
-; BROADWELL-NEXT:    vaddps %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_roundps:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vroundps $7, %ymm0, %ymm0 # sched: [8:1.00]
-; SKYLAKE-NEXT:    vroundps $7, (%rdi), %ymm1 # sched: [15:1.00]
-; SKYLAKE-NEXT:    vaddps %ymm1, %ymm0, %ymm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_roundps:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vroundps $7, %ymm0, %ymm0 # sched: [8:1.00]
-; SKX-NEXT:    vroundps $7, (%rdi), %ymm1 # sched: [15:1.00]
-; SKX-NEXT:    vaddps %ymm1, %ymm0, %ymm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_roundps:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vroundps $7, (%rdi), %ymm1 # sched: [9:2.00]
-; BDVER2-NEXT:    vroundps $7, %ymm0, %ymm0 # sched: [4:2.00]
-; BDVER2-NEXT:    vaddps %ymm1, %ymm0, %ymm0 # sched: [5:2.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_roundps:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vroundps $7, (%rdi), %ymm1 # sched: [8:2.00]
-; BTVER2-NEXT:    vroundps $7, %ymm0, %ymm0 # sched: [3:2.00]
-; BTVER2-NEXT:    vaddps %ymm1, %ymm0, %ymm0 # sched: [3:2.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_roundps:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vroundps $7, (%rdi), %ymm1 # sched: [11:1.00]
-; ZNVER1-NEXT:    vroundps $7, %ymm0, %ymm0 # sched: [4:1.00]
-; ZNVER1-NEXT:    vaddps %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <8 x float> @llvm.x86.avx.round.ps.256(<8 x float> %a0, i32 7)
-  %2 = load <8 x float>, <8 x float> *%a1, align 32
-  %3 = call <8 x float> @llvm.x86.avx.round.ps.256(<8 x float> %2, i32 7)
-  %4 = fadd <8 x float> %1, %3
-  ret <8 x float> %4
-}
-declare <8 x float> @llvm.x86.avx.round.ps.256(<8 x float>, i32) nounwind readnone
-
-define <8 x float> @test_rsqrtps(<8 x float> %a0, <8 x float> *%a1) {
-; GENERIC-LABEL: test_rsqrtps:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vrsqrtps (%rdi), %ymm1 # sched: [14:2.00]
-; GENERIC-NEXT:    vrsqrtps %ymm0, %ymm0 # sched: [7:2.00]
-; GENERIC-NEXT:    vaddps %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_rsqrtps:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vrsqrtps (%rdi), %ymm1 # sched: [14:2.00]
-; SANDY-NEXT:    vrsqrtps %ymm0, %ymm0 # sched: [7:2.00]
-; SANDY-NEXT:    vaddps %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_rsqrtps:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vrsqrtps (%rdi), %ymm1 # sched: [18:2.00]
-; HASWELL-NEXT:    vrsqrtps %ymm0, %ymm0 # sched: [11:2.00]
-; HASWELL-NEXT:    vaddps %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_rsqrtps:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vrsqrtps %ymm0, %ymm0 # sched: [11:2.00]
-; BROADWELL-NEXT:    vrsqrtps (%rdi), %ymm1 # sched: [17:2.00]
-; BROADWELL-NEXT:    vaddps %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_rsqrtps:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vrsqrtps %ymm0, %ymm0 # sched: [4:1.00]
-; SKYLAKE-NEXT:    vrsqrtps (%rdi), %ymm1 # sched: [11:1.00]
-; SKYLAKE-NEXT:    vaddps %ymm1, %ymm0, %ymm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_rsqrtps:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vrsqrtps %ymm0, %ymm0 # sched: [4:1.00]
-; SKX-NEXT:    vrsqrtps (%rdi), %ymm1 # sched: [11:1.00]
-; SKX-NEXT:    vaddps %ymm1, %ymm0, %ymm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_rsqrtps:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vrsqrtps (%rdi), %ymm1 # sched: [10:2.00]
-; BDVER2-NEXT:    vrsqrtps %ymm0, %ymm0 # sched: [5:2.00]
-; BDVER2-NEXT:    vaddps %ymm1, %ymm0, %ymm0 # sched: [5:2.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_rsqrtps:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vrsqrtps (%rdi), %ymm1 # sched: [7:2.00]
-; BTVER2-NEXT:    vrsqrtps %ymm0, %ymm0 # sched: [2:2.00]
-; BTVER2-NEXT:    vaddps %ymm1, %ymm0, %ymm0 # sched: [3:2.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_rsqrtps:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vrsqrtps (%rdi), %ymm1 # sched: [12:0.50]
-; ZNVER1-NEXT:    vrsqrtps %ymm0, %ymm0 # sched: [5:1.00]
-; ZNVER1-NEXT:    vaddps %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <8 x float> @llvm.x86.avx.rsqrt.ps.256(<8 x float> %a0)
-  %2 = load <8 x float>, <8 x float> *%a1, align 32
-  %3 = call <8 x float> @llvm.x86.avx.rsqrt.ps.256(<8 x float> %2)
-  %4 = fadd <8 x float> %1, %3
-  ret <8 x float> %4
-}
-declare <8 x float> @llvm.x86.avx.rsqrt.ps.256(<8 x float>) nounwind readnone
-
-define <4 x double> @test_shufpd(<4 x double> %a0, <4 x double> %a1, <4 x double> *%a2) {
-; GENERIC-LABEL: test_shufpd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vshufpd {{.*#+}} ymm0 = ymm0[1],ymm1[0],ymm0[2],ymm1[3] sched: [1:1.00]
-; GENERIC-NEXT:    vshufpd {{.*#+}} ymm1 = ymm1[1],mem[0],ymm1[2],mem[3] sched: [8:1.00]
-; GENERIC-NEXT:    vaddpd %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_shufpd:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vshufpd {{.*#+}} ymm0 = ymm0[1],ymm1[0],ymm0[2],ymm1[3] sched: [1:1.00]
-; SANDY-NEXT:    vshufpd {{.*#+}} ymm1 = ymm1[1],mem[0],ymm1[2],mem[3] sched: [8:1.00]
-; SANDY-NEXT:    vaddpd %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_shufpd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vshufpd {{.*#+}} ymm0 = ymm0[1],ymm1[0],ymm0[2],ymm1[3] sched: [1:1.00]
-; HASWELL-NEXT:    vshufpd {{.*#+}} ymm1 = ymm1[1],mem[0],ymm1[2],mem[3] sched: [8:1.00]
-; HASWELL-NEXT:    vaddpd %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_shufpd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vshufpd {{.*#+}} ymm0 = ymm0[1],ymm1[0],ymm0[2],ymm1[3] sched: [1:1.00]
-; BROADWELL-NEXT:    vshufpd {{.*#+}} ymm1 = ymm1[1],mem[0],ymm1[2],mem[3] sched: [7:1.00]
-; BROADWELL-NEXT:    vaddpd %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_shufpd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vshufpd {{.*#+}} ymm0 = ymm0[1],ymm1[0],ymm0[2],ymm1[3] sched: [1:1.00]
-; SKYLAKE-NEXT:    vshufpd {{.*#+}} ymm1 = ymm1[1],mem[0],ymm1[2],mem[3] sched: [8:1.00]
-; SKYLAKE-NEXT:    vaddpd %ymm1, %ymm0, %ymm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_shufpd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vshufpd {{.*#+}} ymm0 = ymm0[1],ymm1[0],ymm0[2],ymm1[3] sched: [1:1.00]
-; SKX-NEXT:    vshufpd {{.*#+}} ymm1 = ymm1[1],mem[0],ymm1[2],mem[3] sched: [8:1.00]
-; SKX-NEXT:    vaddpd %ymm1, %ymm0, %ymm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_shufpd:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vshufpd {{.*#+}} ymm0 = ymm0[1],ymm1[0],ymm0[2],ymm1[3] sched: [2:1.00]
-; BDVER2-NEXT:    vshufpd {{.*#+}} ymm1 = ymm1[1],mem[0],ymm1[2],mem[3] sched: [7:1.00]
-; BDVER2-NEXT:    vaddpd %ymm1, %ymm0, %ymm0 # sched: [5:2.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_shufpd:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vshufpd {{.*#+}} ymm0 = ymm0[1],ymm1[0],ymm0[2],ymm1[3] sched: [1:1.00]
-; BTVER2-NEXT:    vshufpd {{.*#+}} ymm1 = ymm1[1],mem[0],ymm1[2],mem[3] sched: [6:2.00]
-; BTVER2-NEXT:    vaddpd %ymm1, %ymm0, %ymm0 # sched: [3:2.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_shufpd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vshufpd {{.*#+}} ymm0 = ymm0[1],ymm1[0],ymm0[2],ymm1[3] sched: [1:0.50]
-; ZNVER1-NEXT:    vshufpd {{.*#+}} ymm1 = ymm1[1],mem[0],ymm1[2],mem[3] sched: [8:0.50]
-; ZNVER1-NEXT:    vaddpd %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = shufflevector <4 x double> %a0, <4 x double> %a1, <4 x i32> <i32 1, i32 4, i32 2, i32 7>
-  %2 = load <4 x double>, <4 x double> *%a2, align 32
-  %3 = shufflevector <4 x double> %a1, <4 x double> %2, <4 x i32> <i32 1, i32 4, i32 2, i32 7>
-  %4 = fadd <4 x double> %1, %3
-  ret <4 x double> %4
-}
-
-define <8 x float> @test_shufps(<8 x float> %a0, <8 x float> %a1, <8 x float> *%a2) nounwind {
-; GENERIC-LABEL: test_shufps:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[0,0],ymm1[0,0],ymm0[4,4],ymm1[4,4] sched: [1:1.00]
-; GENERIC-NEXT:    vshufps {{.*#+}} ymm1 = ymm1[0,3],mem[0,0],ymm1[4,7],mem[4,4] sched: [8:1.00]
-; GENERIC-NEXT:    vaddps %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_shufps:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[0,0],ymm1[0,0],ymm0[4,4],ymm1[4,4] sched: [1:1.00]
-; SANDY-NEXT:    vshufps {{.*#+}} ymm1 = ymm1[0,3],mem[0,0],ymm1[4,7],mem[4,4] sched: [8:1.00]
-; SANDY-NEXT:    vaddps %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_shufps:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[0,0],ymm1[0,0],ymm0[4,4],ymm1[4,4] sched: [1:1.00]
-; HASWELL-NEXT:    vshufps {{.*#+}} ymm1 = ymm1[0,3],mem[0,0],ymm1[4,7],mem[4,4] sched: [8:1.00]
-; HASWELL-NEXT:    vaddps %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_shufps:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[0,0],ymm1[0,0],ymm0[4,4],ymm1[4,4] sched: [1:1.00]
-; BROADWELL-NEXT:    vshufps {{.*#+}} ymm1 = ymm1[0,3],mem[0,0],ymm1[4,7],mem[4,4] sched: [7:1.00]
-; BROADWELL-NEXT:    vaddps %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_shufps:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[0,0],ymm1[0,0],ymm0[4,4],ymm1[4,4] sched: [1:1.00]
-; SKYLAKE-NEXT:    vshufps {{.*#+}} ymm1 = ymm1[0,3],mem[0,0],ymm1[4,7],mem[4,4] sched: [8:1.00]
-; SKYLAKE-NEXT:    vaddps %ymm1, %ymm0, %ymm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_shufps:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[0,0],ymm1[0,0],ymm0[4,4],ymm1[4,4] sched: [1:1.00]
-; SKX-NEXT:    vshufps {{.*#+}} ymm1 = ymm1[0,3],mem[0,0],ymm1[4,7],mem[4,4] sched: [8:1.00]
-; SKX-NEXT:    vaddps %ymm1, %ymm0, %ymm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_shufps:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[0,0],ymm1[0,0],ymm0[4,4],ymm1[4,4] sched: [2:1.00]
-; BDVER2-NEXT:    vshufps {{.*#+}} ymm1 = ymm1[0,3],mem[0,0],ymm1[4,7],mem[4,4] sched: [7:1.00]
-; BDVER2-NEXT:    vaddps %ymm1, %ymm0, %ymm0 # sched: [5:2.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_shufps:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[0,0],ymm1[0,0],ymm0[4,4],ymm1[4,4] sched: [1:1.00]
-; BTVER2-NEXT:    vshufps {{.*#+}} ymm1 = ymm1[0,3],mem[0,0],ymm1[4,7],mem[4,4] sched: [6:2.00]
-; BTVER2-NEXT:    vaddps %ymm1, %ymm0, %ymm0 # sched: [3:2.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_shufps:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[0,0],ymm1[0,0],ymm0[4,4],ymm1[4,4] sched: [1:0.50]
-; ZNVER1-NEXT:    vshufps {{.*#+}} ymm1 = ymm1[0,3],mem[0,0],ymm1[4,7],mem[4,4] sched: [8:0.50]
-; ZNVER1-NEXT:    vaddps %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = shufflevector <8 x float> %a0, <8 x float> %a1, <8 x i32> <i32 0, i32 0, i32 8, i32 8, i32 4, i32 4, i32 12, i32 12>
-  %2 = load <8 x float>, <8 x float> *%a2, align 32
-  %3 = shufflevector <8 x float> %a1, <8 x float> %2, <8 x i32> <i32 0, i32 3, i32 8, i32 8, i32 4, i32 7, i32 12, i32 12>
-  %4 = fadd <8 x float> %1, %3
-  ret <8 x float> %4
-}
-
-define <4 x double> @test_sqrtpd(<4 x double> %a0, <4 x double> *%a1) {
-; GENERIC-LABEL: test_sqrtpd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vsqrtpd (%rdi), %ymm1 # sched: [52:44.00]
-; GENERIC-NEXT:    vsqrtpd %ymm0, %ymm0 # sched: [45:44.00]
-; GENERIC-NEXT:    vaddpd %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_sqrtpd:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vsqrtpd (%rdi), %ymm1 # sched: [52:44.00]
-; SANDY-NEXT:    vsqrtpd %ymm0, %ymm0 # sched: [45:44.00]
-; SANDY-NEXT:    vaddpd %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_sqrtpd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vsqrtpd (%rdi), %ymm1 # sched: [42:28.00]
-; HASWELL-NEXT:    vsqrtpd %ymm0, %ymm0 # sched: [35:28.00]
-; HASWELL-NEXT:    vaddpd %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_sqrtpd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vsqrtpd %ymm0, %ymm0 # sched: [29:28.00]
-; BROADWELL-NEXT:    vsqrtpd (%rdi), %ymm1 # sched: [35:28.00]
-; BROADWELL-NEXT:    vaddpd %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_sqrtpd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vsqrtpd %ymm0, %ymm0 # sched: [18:12.00]
-; SKYLAKE-NEXT:    vsqrtpd (%rdi), %ymm1 # sched: [25:12.00]
-; SKYLAKE-NEXT:    vaddpd %ymm1, %ymm0, %ymm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_sqrtpd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vsqrtpd %ymm0, %ymm0 # sched: [18:12.00]
-; SKX-NEXT:    vsqrtpd (%rdi), %ymm1 # sched: [25:12.00]
-; SKX-NEXT:    vaddpd %ymm1, %ymm0, %ymm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_sqrtpd:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vsqrtpd (%rdi), %ymm1 # sched: [14:27.00]
-; BDVER2-NEXT:    vsqrtpd %ymm0, %ymm0 # sched: [9:27.00]
-; BDVER2-NEXT:    vaddpd %ymm1, %ymm0, %ymm0 # sched: [5:2.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_sqrtpd:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vsqrtpd (%rdi), %ymm1 # sched: [59:54.00]
-; BTVER2-NEXT:    vsqrtpd %ymm0, %ymm0 # sched: [54:54.00]
-; BTVER2-NEXT:    vaddpd %ymm1, %ymm0, %ymm0 # sched: [3:2.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_sqrtpd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vsqrtpd (%rdi), %ymm1 # sched: [47:40.00]
-; ZNVER1-NEXT:    vsqrtpd %ymm0, %ymm0 # sched: [40:40.00]
-; ZNVER1-NEXT:    vaddpd %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <4 x double> @llvm.x86.avx.sqrt.pd.256(<4 x double> %a0)
-  %2 = load <4 x double>, <4 x double> *%a1, align 32
-  %3 = call <4 x double> @llvm.x86.avx.sqrt.pd.256(<4 x double> %2)
-  %4 = fadd <4 x double> %1, %3
-  ret <4 x double> %4
-}
-declare <4 x double> @llvm.x86.avx.sqrt.pd.256(<4 x double>) nounwind readnone
-
-define <8 x float> @test_sqrtps(<8 x float> %a0, <8 x float> *%a1) {
-; GENERIC-LABEL: test_sqrtps:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vsqrtps (%rdi), %ymm1 # sched: [36:28.00]
-; GENERIC-NEXT:    vsqrtps %ymm0, %ymm0 # sched: [29:28.00]
-; GENERIC-NEXT:    vaddps %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_sqrtps:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vsqrtps (%rdi), %ymm1 # sched: [36:28.00]
-; SANDY-NEXT:    vsqrtps %ymm0, %ymm0 # sched: [29:28.00]
-; SANDY-NEXT:    vaddps %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_sqrtps:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vsqrtps (%rdi), %ymm1 # sched: [28:14.00]
-; HASWELL-NEXT:    vsqrtps %ymm0, %ymm0 # sched: [21:14.00]
-; HASWELL-NEXT:    vaddps %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_sqrtps:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vsqrtps %ymm0, %ymm0 # sched: [21:14.00]
-; BROADWELL-NEXT:    vsqrtps (%rdi), %ymm1 # sched: [27:14.00]
-; BROADWELL-NEXT:    vaddps %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_sqrtps:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vsqrtps %ymm0, %ymm0 # sched: [12:6.00]
-; SKYLAKE-NEXT:    vsqrtps (%rdi), %ymm1 # sched: [19:6.00]
-; SKYLAKE-NEXT:    vaddps %ymm1, %ymm0, %ymm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_sqrtps:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vsqrtps %ymm0, %ymm0 # sched: [12:6.00]
-; SKX-NEXT:    vsqrtps (%rdi), %ymm1 # sched: [19:6.00]
-; SKX-NEXT:    vaddps %ymm1, %ymm0, %ymm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_sqrtps:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vsqrtps (%rdi), %ymm1 # sched: [14:21.00]
-; BDVER2-NEXT:    vsqrtps %ymm0, %ymm0 # sched: [9:21.00]
-; BDVER2-NEXT:    vaddps %ymm1, %ymm0, %ymm0 # sched: [5:2.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_sqrtps:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vsqrtps (%rdi), %ymm1 # sched: [47:42.00]
-; BTVER2-NEXT:    vsqrtps %ymm0, %ymm0 # sched: [42:42.00]
-; BTVER2-NEXT:    vaddps %ymm1, %ymm0, %ymm0 # sched: [3:2.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_sqrtps:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vsqrtps (%rdi), %ymm1 # sched: [35:28.00]
-; ZNVER1-NEXT:    vsqrtps %ymm0, %ymm0 # sched: [28:28.00]
-; ZNVER1-NEXT:    vaddps %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <8 x float> @llvm.x86.avx.sqrt.ps.256(<8 x float> %a0)
-  %2 = load <8 x float>, <8 x float> *%a1, align 32
-  %3 = call <8 x float> @llvm.x86.avx.sqrt.ps.256(<8 x float> %2)
-  %4 = fadd <8 x float> %1, %3
-  ret <8 x float> %4
-}
-declare <8 x float> @llvm.x86.avx.sqrt.ps.256(<8 x float>) nounwind readnone
-
-define <4 x double> @test_subpd(<4 x double> %a0, <4 x double> %a1, <4 x double> *%a2) {
-; GENERIC-LABEL: test_subpd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vsubpd %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; GENERIC-NEXT:    vsubpd (%rdi), %ymm0, %ymm0 # sched: [10:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_subpd:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vsubpd %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; SANDY-NEXT:    vsubpd (%rdi), %ymm0, %ymm0 # sched: [10:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_subpd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vsubpd %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; HASWELL-NEXT:    vsubpd (%rdi), %ymm0, %ymm0 # sched: [10:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_subpd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vsubpd %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    vsubpd (%rdi), %ymm0, %ymm0 # sched: [9:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_subpd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vsubpd %ymm1, %ymm0, %ymm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    vsubpd (%rdi), %ymm0, %ymm0 # sched: [11:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_subpd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vsubpd %ymm1, %ymm0, %ymm0 # sched: [4:0.50]
-; SKX-NEXT:    vsubpd (%rdi), %ymm0, %ymm0 # sched: [11:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_subpd:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vsubpd %ymm1, %ymm0, %ymm0 # sched: [5:2.00]
-; BDVER2-NEXT:    vsubpd (%rdi), %ymm0, %ymm0 # sched: [10:2.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_subpd:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vsubpd %ymm1, %ymm0, %ymm0 # sched: [3:2.00]
-; BTVER2-NEXT:    vsubpd (%rdi), %ymm0, %ymm0 # sched: [8:2.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_subpd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vsubpd %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    vsubpd (%rdi), %ymm0, %ymm0 # sched: [10:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = fsub <4 x double> %a0, %a1
-  %2 = load <4 x double>, <4 x double> *%a2, align 32
-  %3 = fsub <4 x double> %1, %2
-  ret <4 x double> %3
-}
-
-define <8 x float> @test_subps(<8 x float> %a0, <8 x float> %a1, <8 x float> *%a2) {
-; GENERIC-LABEL: test_subps:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vsubps %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; GENERIC-NEXT:    vsubps (%rdi), %ymm0, %ymm0 # sched: [10:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_subps:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vsubps %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; SANDY-NEXT:    vsubps (%rdi), %ymm0, %ymm0 # sched: [10:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_subps:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vsubps %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; HASWELL-NEXT:    vsubps (%rdi), %ymm0, %ymm0 # sched: [10:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_subps:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vsubps %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    vsubps (%rdi), %ymm0, %ymm0 # sched: [9:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_subps:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vsubps %ymm1, %ymm0, %ymm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    vsubps (%rdi), %ymm0, %ymm0 # sched: [11:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_subps:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vsubps %ymm1, %ymm0, %ymm0 # sched: [4:0.50]
-; SKX-NEXT:    vsubps (%rdi), %ymm0, %ymm0 # sched: [11:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_subps:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vsubps %ymm1, %ymm0, %ymm0 # sched: [5:2.00]
-; BDVER2-NEXT:    vsubps (%rdi), %ymm0, %ymm0 # sched: [10:2.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_subps:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vsubps %ymm1, %ymm0, %ymm0 # sched: [3:2.00]
-; BTVER2-NEXT:    vsubps (%rdi), %ymm0, %ymm0 # sched: [8:2.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_subps:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vsubps %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    vsubps (%rdi), %ymm0, %ymm0 # sched: [10:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = fsub <8 x float> %a0, %a1
-  %2 = load <8 x float>, <8 x float> *%a2, align 32
-  %3 = fsub <8 x float> %1, %2
-  ret <8 x float> %3
-}
-
-define i32 @test_testpd(<2 x double> %a0, <2 x double> %a1, <2 x double> *%a2) {
-; GENERIC-LABEL: test_testpd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    xorl %eax, %eax # sched: [0:0.25]
-; GENERIC-NEXT:    vtestpd %xmm1, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    setb %al # sched: [1:0.50]
-; GENERIC-NEXT:    vtestpd (%rdi), %xmm0 # sched: [7:1.00]
-; GENERIC-NEXT:    adcl $0, %eax # sched: [2:0.67]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_testpd:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    xorl %eax, %eax # sched: [0:0.25]
-; SANDY-NEXT:    vtestpd %xmm1, %xmm0 # sched: [1:1.00]
-; SANDY-NEXT:    setb %al # sched: [1:0.50]
-; SANDY-NEXT:    vtestpd (%rdi), %xmm0 # sched: [7:1.00]
-; SANDY-NEXT:    adcl $0, %eax # sched: [2:0.67]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_testpd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    xorl %eax, %eax # sched: [1:0.25]
-; HASWELL-NEXT:    vtestpd %xmm1, %xmm0 # sched: [1:1.00]
-; HASWELL-NEXT:    setb %al # sched: [1:0.50]
-; HASWELL-NEXT:    vtestpd (%rdi), %xmm0 # sched: [7:1.00]
-; HASWELL-NEXT:    adcl $0, %eax # sched: [2:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_testpd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    xorl %eax, %eax # sched: [1:0.25]
-; BROADWELL-NEXT:    vtestpd %xmm1, %xmm0 # sched: [1:1.00]
-; BROADWELL-NEXT:    setb %al # sched: [1:0.50]
-; BROADWELL-NEXT:    vtestpd (%rdi), %xmm0 # sched: [6:1.00]
-; BROADWELL-NEXT:    adcl $0, %eax # sched: [1:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_testpd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    xorl %eax, %eax # sched: [1:0.25]
-; SKYLAKE-NEXT:    vtestpd %xmm1, %xmm0 # sched: [2:1.00]
-; SKYLAKE-NEXT:    setb %al # sched: [1:0.50]
-; SKYLAKE-NEXT:    vtestpd (%rdi), %xmm0 # sched: [8:1.00]
-; SKYLAKE-NEXT:    adcl $0, %eax # sched: [1:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_testpd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    xorl %eax, %eax # sched: [1:0.25]
-; SKX-NEXT:    vtestpd %xmm1, %xmm0 # sched: [2:1.00]
-; SKX-NEXT:    setb %al # sched: [1:0.50]
-; SKX-NEXT:    vtestpd (%rdi), %xmm0 # sched: [8:1.00]
-; SKX-NEXT:    adcl $0, %eax # sched: [1:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_testpd:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    xorl %eax, %eax # sched: [0:0.25]
-; BDVER2-NEXT:    vtestpd %xmm1, %xmm0 # sched: [1:1.00]
-; BDVER2-NEXT:    setb %al # sched: [1:0.50]
-; BDVER2-NEXT:    vtestpd (%rdi), %xmm0 # sched: [6:1.00]
-; BDVER2-NEXT:    adcl $0, %eax # sched: [1:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_testpd:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    xorl %eax, %eax # sched: [0:0.50]
-; BTVER2-NEXT:    vtestpd %xmm1, %xmm0 # sched: [3:1.00]
-; BTVER2-NEXT:    setb %al # sched: [1:0.50]
-; BTVER2-NEXT:    vtestpd (%rdi), %xmm0 # sched: [8:1.00]
-; BTVER2-NEXT:    adcl $0, %eax # sched: [1:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_testpd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    xorl %eax, %eax # sched: [1:0.25]
-; ZNVER1-NEXT:    vtestpd %xmm1, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    setb %al # sched: [1:0.25]
-; ZNVER1-NEXT:    vtestpd (%rdi), %xmm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    adcl $0, %eax # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call i32 @llvm.x86.avx.vtestc.pd(<2 x double> %a0, <2 x double> %a1)
-  %2 = load <2 x double>, <2 x double> *%a2, align 16
-  %3 = call i32 @llvm.x86.avx.vtestc.pd(<2 x double> %a0, <2 x double> %2)
-  %4 = add i32 %1, %3
-  ret i32 %4
-}
-declare i32 @llvm.x86.avx.vtestc.pd(<2 x double>, <2 x double>) nounwind readnone
-
-define i32 @test_testpd_ymm(<4 x double> %a0, <4 x double> %a1, <4 x double> *%a2) {
-; GENERIC-LABEL: test_testpd_ymm:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    xorl %eax, %eax # sched: [0:0.25]
-; GENERIC-NEXT:    vtestpd %ymm1, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    setb %al # sched: [1:0.50]
-; GENERIC-NEXT:    vtestpd (%rdi), %ymm0 # sched: [8:1.00]
-; GENERIC-NEXT:    adcl $0, %eax # sched: [2:0.67]
-; GENERIC-NEXT:    vzeroupper # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_testpd_ymm:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    xorl %eax, %eax # sched: [0:0.25]
-; SANDY-NEXT:    vtestpd %ymm1, %ymm0 # sched: [1:1.00]
-; SANDY-NEXT:    setb %al # sched: [1:0.50]
-; SANDY-NEXT:    vtestpd (%rdi), %ymm0 # sched: [8:1.00]
-; SANDY-NEXT:    adcl $0, %eax # sched: [2:0.67]
-; SANDY-NEXT:    vzeroupper # sched: [1:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_testpd_ymm:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    xorl %eax, %eax # sched: [1:0.25]
-; HASWELL-NEXT:    vtestpd %ymm1, %ymm0 # sched: [1:1.00]
-; HASWELL-NEXT:    setb %al # sched: [1:0.50]
-; HASWELL-NEXT:    vtestpd (%rdi), %ymm0 # sched: [8:1.00]
-; HASWELL-NEXT:    adcl $0, %eax # sched: [2:0.50]
-; HASWELL-NEXT:    vzeroupper # sched: [0:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_testpd_ymm:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    xorl %eax, %eax # sched: [1:0.25]
-; BROADWELL-NEXT:    vtestpd %ymm1, %ymm0 # sched: [1:1.00]
-; BROADWELL-NEXT:    setb %al # sched: [1:0.50]
-; BROADWELL-NEXT:    vtestpd (%rdi), %ymm0 # sched: [7:1.00]
-; BROADWELL-NEXT:    adcl $0, %eax # sched: [1:0.50]
-; BROADWELL-NEXT:    vzeroupper # sched: [0:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_testpd_ymm:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    xorl %eax, %eax # sched: [1:0.25]
-; SKYLAKE-NEXT:    vtestpd %ymm1, %ymm0 # sched: [2:1.00]
-; SKYLAKE-NEXT:    setb %al # sched: [1:0.50]
-; SKYLAKE-NEXT:    vtestpd (%rdi), %ymm0 # sched: [9:1.00]
-; SKYLAKE-NEXT:    adcl $0, %eax # sched: [1:0.50]
-; SKYLAKE-NEXT:    vzeroupper # sched: [0:0.67]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_testpd_ymm:
-; SKX:       # %bb.0:
-; SKX-NEXT:    xorl %eax, %eax # sched: [1:0.25]
-; SKX-NEXT:    vtestpd %ymm1, %ymm0 # sched: [2:1.00]
-; SKX-NEXT:    setb %al # sched: [1:0.50]
-; SKX-NEXT:    vtestpd (%rdi), %ymm0 # sched: [9:1.00]
-; SKX-NEXT:    adcl $0, %eax # sched: [1:0.50]
-; SKX-NEXT:    vzeroupper # sched: [0:0.67]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_testpd_ymm:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    xorl %eax, %eax # sched: [0:0.25]
-; BDVER2-NEXT:    vtestpd %ymm1, %ymm0 # sched: [1:1.00]
-; BDVER2-NEXT:    setb %al # sched: [1:0.50]
-; BDVER2-NEXT:    vtestpd (%rdi), %ymm0 # sched: [6:1.00]
-; BDVER2-NEXT:    adcl $0, %eax # sched: [1:1.00]
-; BDVER2-NEXT:    vzeroupper # sched: [46:4.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_testpd_ymm:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    xorl %eax, %eax # sched: [0:0.50]
-; BTVER2-NEXT:    vtestpd %ymm1, %ymm0 # sched: [4:2.00]
-; BTVER2-NEXT:    setb %al # sched: [1:0.50]
-; BTVER2-NEXT:    vtestpd (%rdi), %ymm0 # sched: [9:2.00]
-; BTVER2-NEXT:    adcl $0, %eax # sched: [1:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_testpd_ymm:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    xorl %eax, %eax # sched: [1:0.25]
-; ZNVER1-NEXT:    vtestpd %ymm1, %ymm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    setb %al # sched: [1:0.25]
-; ZNVER1-NEXT:    vtestpd (%rdi), %ymm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    adcl $0, %eax # sched: [1:0.25]
-; ZNVER1-NEXT:    vzeroupper # sched: [100:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call i32 @llvm.x86.avx.vtestc.pd.256(<4 x double> %a0, <4 x double> %a1)
-  %2 = load <4 x double>, <4 x double> *%a2, align 32
-  %3 = call i32 @llvm.x86.avx.vtestc.pd.256(<4 x double> %a0, <4 x double> %2)
-  %4 = add i32 %1, %3
-  ret i32 %4
-}
-declare i32 @llvm.x86.avx.vtestc.pd.256(<4 x double>, <4 x double>) nounwind readnone
-
-define i32 @test_testps(<4 x float> %a0, <4 x float> %a1, <4 x float> *%a2) {
-; GENERIC-LABEL: test_testps:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    xorl %eax, %eax # sched: [0:0.25]
-; GENERIC-NEXT:    vtestps %xmm1, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    setb %al # sched: [1:0.50]
-; GENERIC-NEXT:    vtestps (%rdi), %xmm0 # sched: [7:1.00]
-; GENERIC-NEXT:    adcl $0, %eax # sched: [2:0.67]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_testps:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    xorl %eax, %eax # sched: [0:0.25]
-; SANDY-NEXT:    vtestps %xmm1, %xmm0 # sched: [1:1.00]
-; SANDY-NEXT:    setb %al # sched: [1:0.50]
-; SANDY-NEXT:    vtestps (%rdi), %xmm0 # sched: [7:1.00]
-; SANDY-NEXT:    adcl $0, %eax # sched: [2:0.67]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_testps:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    xorl %eax, %eax # sched: [1:0.25]
-; HASWELL-NEXT:    vtestps %xmm1, %xmm0 # sched: [1:1.00]
-; HASWELL-NEXT:    setb %al # sched: [1:0.50]
-; HASWELL-NEXT:    vtestps (%rdi), %xmm0 # sched: [7:1.00]
-; HASWELL-NEXT:    adcl $0, %eax # sched: [2:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_testps:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    xorl %eax, %eax # sched: [1:0.25]
-; BROADWELL-NEXT:    vtestps %xmm1, %xmm0 # sched: [1:1.00]
-; BROADWELL-NEXT:    setb %al # sched: [1:0.50]
-; BROADWELL-NEXT:    vtestps (%rdi), %xmm0 # sched: [6:1.00]
-; BROADWELL-NEXT:    adcl $0, %eax # sched: [1:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_testps:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    xorl %eax, %eax # sched: [1:0.25]
-; SKYLAKE-NEXT:    vtestps %xmm1, %xmm0 # sched: [2:1.00]
-; SKYLAKE-NEXT:    setb %al # sched: [1:0.50]
-; SKYLAKE-NEXT:    vtestps (%rdi), %xmm0 # sched: [8:1.00]
-; SKYLAKE-NEXT:    adcl $0, %eax # sched: [1:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_testps:
-; SKX:       # %bb.0:
-; SKX-NEXT:    xorl %eax, %eax # sched: [1:0.25]
-; SKX-NEXT:    vtestps %xmm1, %xmm0 # sched: [2:1.00]
-; SKX-NEXT:    setb %al # sched: [1:0.50]
-; SKX-NEXT:    vtestps (%rdi), %xmm0 # sched: [8:1.00]
-; SKX-NEXT:    adcl $0, %eax # sched: [1:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_testps:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    xorl %eax, %eax # sched: [0:0.25]
-; BDVER2-NEXT:    vtestps %xmm1, %xmm0 # sched: [1:1.00]
-; BDVER2-NEXT:    setb %al # sched: [1:0.50]
-; BDVER2-NEXT:    vtestps (%rdi), %xmm0 # sched: [6:1.00]
-; BDVER2-NEXT:    adcl $0, %eax # sched: [1:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_testps:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    xorl %eax, %eax # sched: [0:0.50]
-; BTVER2-NEXT:    vtestps %xmm1, %xmm0 # sched: [3:1.00]
-; BTVER2-NEXT:    setb %al # sched: [1:0.50]
-; BTVER2-NEXT:    vtestps (%rdi), %xmm0 # sched: [8:1.00]
-; BTVER2-NEXT:    adcl $0, %eax # sched: [1:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_testps:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    xorl %eax, %eax # sched: [1:0.25]
-; ZNVER1-NEXT:    vtestps %xmm1, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    setb %al # sched: [1:0.25]
-; ZNVER1-NEXT:    vtestps (%rdi), %xmm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    adcl $0, %eax # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call i32 @llvm.x86.avx.vtestc.ps(<4 x float> %a0, <4 x float> %a1)
-  %2 = load <4 x float>, <4 x float> *%a2, align 16
-  %3 = call i32 @llvm.x86.avx.vtestc.ps(<4 x float> %a0, <4 x float> %2)
-  %4 = add i32 %1, %3
-  ret i32 %4
-}
-declare i32 @llvm.x86.avx.vtestc.ps(<4 x float>, <4 x float>) nounwind readnone
-
-define i32 @test_testps_ymm(<8 x float> %a0, <8 x float> %a1, <8 x float> *%a2) {
-; GENERIC-LABEL: test_testps_ymm:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    xorl %eax, %eax # sched: [0:0.25]
-; GENERIC-NEXT:    vtestps %ymm1, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    setb %al # sched: [1:0.50]
-; GENERIC-NEXT:    vtestps (%rdi), %ymm0 # sched: [8:1.00]
-; GENERIC-NEXT:    adcl $0, %eax # sched: [2:0.67]
-; GENERIC-NEXT:    vzeroupper # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_testps_ymm:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    xorl %eax, %eax # sched: [0:0.25]
-; SANDY-NEXT:    vtestps %ymm1, %ymm0 # sched: [1:1.00]
-; SANDY-NEXT:    setb %al # sched: [1:0.50]
-; SANDY-NEXT:    vtestps (%rdi), %ymm0 # sched: [8:1.00]
-; SANDY-NEXT:    adcl $0, %eax # sched: [2:0.67]
-; SANDY-NEXT:    vzeroupper # sched: [1:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_testps_ymm:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    xorl %eax, %eax # sched: [1:0.25]
-; HASWELL-NEXT:    vtestps %ymm1, %ymm0 # sched: [1:1.00]
-; HASWELL-NEXT:    setb %al # sched: [1:0.50]
-; HASWELL-NEXT:    vtestps (%rdi), %ymm0 # sched: [8:1.00]
-; HASWELL-NEXT:    adcl $0, %eax # sched: [2:0.50]
-; HASWELL-NEXT:    vzeroupper # sched: [0:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_testps_ymm:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    xorl %eax, %eax # sched: [1:0.25]
-; BROADWELL-NEXT:    vtestps %ymm1, %ymm0 # sched: [1:1.00]
-; BROADWELL-NEXT:    setb %al # sched: [1:0.50]
-; BROADWELL-NEXT:    vtestps (%rdi), %ymm0 # sched: [7:1.00]
-; BROADWELL-NEXT:    adcl $0, %eax # sched: [1:0.50]
-; BROADWELL-NEXT:    vzeroupper # sched: [0:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_testps_ymm:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    xorl %eax, %eax # sched: [1:0.25]
-; SKYLAKE-NEXT:    vtestps %ymm1, %ymm0 # sched: [2:1.00]
-; SKYLAKE-NEXT:    setb %al # sched: [1:0.50]
-; SKYLAKE-NEXT:    vtestps (%rdi), %ymm0 # sched: [9:1.00]
-; SKYLAKE-NEXT:    adcl $0, %eax # sched: [1:0.50]
-; SKYLAKE-NEXT:    vzeroupper # sched: [0:0.67]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_testps_ymm:
-; SKX:       # %bb.0:
-; SKX-NEXT:    xorl %eax, %eax # sched: [1:0.25]
-; SKX-NEXT:    vtestps %ymm1, %ymm0 # sched: [2:1.00]
-; SKX-NEXT:    setb %al # sched: [1:0.50]
-; SKX-NEXT:    vtestps (%rdi), %ymm0 # sched: [9:1.00]
-; SKX-NEXT:    adcl $0, %eax # sched: [1:0.50]
-; SKX-NEXT:    vzeroupper # sched: [0:0.67]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_testps_ymm:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    xorl %eax, %eax # sched: [0:0.25]
-; BDVER2-NEXT:    vtestps %ymm1, %ymm0 # sched: [1:1.00]
-; BDVER2-NEXT:    setb %al # sched: [1:0.50]
-; BDVER2-NEXT:    vtestps (%rdi), %ymm0 # sched: [6:1.00]
-; BDVER2-NEXT:    adcl $0, %eax # sched: [1:1.00]
-; BDVER2-NEXT:    vzeroupper # sched: [46:4.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_testps_ymm:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    xorl %eax, %eax # sched: [0:0.50]
-; BTVER2-NEXT:    vtestps %ymm1, %ymm0 # sched: [4:2.00]
-; BTVER2-NEXT:    setb %al # sched: [1:0.50]
-; BTVER2-NEXT:    vtestps (%rdi), %ymm0 # sched: [9:2.00]
-; BTVER2-NEXT:    adcl $0, %eax # sched: [1:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_testps_ymm:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    xorl %eax, %eax # sched: [1:0.25]
-; ZNVER1-NEXT:    vtestps %ymm1, %ymm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    setb %al # sched: [1:0.25]
-; ZNVER1-NEXT:    vtestps (%rdi), %ymm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    adcl $0, %eax # sched: [1:0.25]
-; ZNVER1-NEXT:    vzeroupper # sched: [100:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call i32 @llvm.x86.avx.vtestc.ps.256(<8 x float> %a0, <8 x float> %a1)
-  %2 = load <8 x float>, <8 x float> *%a2, align 32
-  %3 = call i32 @llvm.x86.avx.vtestc.ps.256(<8 x float> %a0, <8 x float> %2)
-  %4 = add i32 %1, %3
-  ret i32 %4
-}
-declare i32 @llvm.x86.avx.vtestc.ps.256(<8 x float>, <8 x float>) nounwind readnone
-
-define <4 x double> @test_unpckhpd(<4 x double> %a0, <4 x double> %a1, <4 x double> *%a2) {
-; GENERIC-LABEL: test_unpckhpd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vunpckhpd {{.*#+}} ymm0 = ymm0[1],ymm1[1],ymm0[3],ymm1[3] sched: [1:1.00]
-; GENERIC-NEXT:    vunpckhpd {{.*#+}} ymm1 = ymm1[1],mem[1],ymm1[3],mem[3] sched: [8:1.00]
-; GENERIC-NEXT:    vaddpd %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_unpckhpd:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vunpckhpd {{.*#+}} ymm0 = ymm0[1],ymm1[1],ymm0[3],ymm1[3] sched: [1:1.00]
-; SANDY-NEXT:    vunpckhpd {{.*#+}} ymm1 = ymm1[1],mem[1],ymm1[3],mem[3] sched: [8:1.00]
-; SANDY-NEXT:    vaddpd %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_unpckhpd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vunpckhpd {{.*#+}} ymm0 = ymm0[1],ymm1[1],ymm0[3],ymm1[3] sched: [1:1.00]
-; HASWELL-NEXT:    vunpckhpd {{.*#+}} ymm1 = ymm1[1],mem[1],ymm1[3],mem[3] sched: [8:1.00]
-; HASWELL-NEXT:    vaddpd %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_unpckhpd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vunpckhpd {{.*#+}} ymm0 = ymm0[1],ymm1[1],ymm0[3],ymm1[3] sched: [1:1.00]
-; BROADWELL-NEXT:    vunpckhpd {{.*#+}} ymm1 = ymm1[1],mem[1],ymm1[3],mem[3] sched: [7:1.00]
-; BROADWELL-NEXT:    vaddpd %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_unpckhpd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vunpckhpd {{.*#+}} ymm0 = ymm0[1],ymm1[1],ymm0[3],ymm1[3] sched: [1:1.00]
-; SKYLAKE-NEXT:    vunpckhpd {{.*#+}} ymm1 = ymm1[1],mem[1],ymm1[3],mem[3] sched: [8:1.00]
-; SKYLAKE-NEXT:    vaddpd %ymm1, %ymm0, %ymm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_unpckhpd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vunpckhpd {{.*#+}} ymm0 = ymm0[1],ymm1[1],ymm0[3],ymm1[3] sched: [1:1.00]
-; SKX-NEXT:    vunpckhpd {{.*#+}} ymm1 = ymm1[1],mem[1],ymm1[3],mem[3] sched: [8:1.00]
-; SKX-NEXT:    vaddpd %ymm1, %ymm0, %ymm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_unpckhpd:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vunpckhpd {{.*#+}} ymm0 = ymm0[1],ymm1[1],ymm0[3],ymm1[3] sched: [2:1.00]
-; BDVER2-NEXT:    vunpckhpd {{.*#+}} ymm1 = ymm1[1],mem[1],ymm1[3],mem[3] sched: [7:1.00]
-; BDVER2-NEXT:    vaddpd %ymm1, %ymm0, %ymm0 # sched: [5:2.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_unpckhpd:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vunpckhpd {{.*#+}} ymm0 = ymm0[1],ymm1[1],ymm0[3],ymm1[3] sched: [1:1.00]
-; BTVER2-NEXT:    vunpckhpd {{.*#+}} ymm1 = ymm1[1],mem[1],ymm1[3],mem[3] sched: [6:2.00]
-; BTVER2-NEXT:    vaddpd %ymm1, %ymm0, %ymm0 # sched: [3:2.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_unpckhpd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vunpckhpd {{.*#+}} ymm0 = ymm0[1],ymm1[1],ymm0[3],ymm1[3] sched: [1:0.50]
-; ZNVER1-NEXT:    vunpckhpd {{.*#+}} ymm1 = ymm1[1],mem[1],ymm1[3],mem[3] sched: [8:0.50]
-; ZNVER1-NEXT:    vaddpd %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = shufflevector <4 x double> %a0, <4 x double> %a1, <4 x i32> <i32 1, i32 5, i32 3, i32 7>
-  %2 = load <4 x double>, <4 x double> *%a2, align 32
-  %3 = shufflevector <4 x double> %a1, <4 x double> %2, <4 x i32> <i32 1, i32 5, i32 3, i32 7>
-  %4 = fadd <4 x double> %1, %3
-  ret <4 x double> %4
-}
-
-define <8 x float> @test_unpckhps(<8 x float> %a0, <8 x float> %a1, <8 x float> *%a2) nounwind {
-; GENERIC-LABEL: test_unpckhps:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vunpckhps {{.*#+}} ymm0 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7] sched: [1:1.00]
-; GENERIC-NEXT:    vunpckhps {{.*#+}} ymm0 = ymm0[2],mem[2],ymm0[3],mem[3],ymm0[6],mem[6],ymm0[7],mem[7] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_unpckhps:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vunpckhps {{.*#+}} ymm0 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7] sched: [1:1.00]
-; SANDY-NEXT:    vunpckhps {{.*#+}} ymm0 = ymm0[2],mem[2],ymm0[3],mem[3],ymm0[6],mem[6],ymm0[7],mem[7] sched: [8:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_unpckhps:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vunpckhps {{.*#+}} ymm0 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7] sched: [1:1.00]
-; HASWELL-NEXT:    vunpckhps {{.*#+}} ymm0 = ymm0[2],mem[2],ymm0[3],mem[3],ymm0[6],mem[6],ymm0[7],mem[7] sched: [8:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_unpckhps:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vunpckhps {{.*#+}} ymm0 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7] sched: [1:1.00]
-; BROADWELL-NEXT:    vunpckhps {{.*#+}} ymm0 = ymm0[2],mem[2],ymm0[3],mem[3],ymm0[6],mem[6],ymm0[7],mem[7] sched: [7:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_unpckhps:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vunpckhps {{.*#+}} ymm0 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7] sched: [1:1.00]
-; SKYLAKE-NEXT:    vunpckhps {{.*#+}} ymm0 = ymm0[2],mem[2],ymm0[3],mem[3],ymm0[6],mem[6],ymm0[7],mem[7] sched: [8:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_unpckhps:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vunpckhps {{.*#+}} ymm0 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7] sched: [1:1.00]
-; SKX-NEXT:    vunpckhps {{.*#+}} ymm0 = ymm0[2],mem[2],ymm0[3],mem[3],ymm0[6],mem[6],ymm0[7],mem[7] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_unpckhps:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vunpckhps {{.*#+}} ymm0 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7] sched: [2:1.00]
-; BDVER2-NEXT:    vunpckhps {{.*#+}} ymm0 = ymm0[2],mem[2],ymm0[3],mem[3],ymm0[6],mem[6],ymm0[7],mem[7] sched: [7:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_unpckhps:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vunpckhps {{.*#+}} ymm0 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7] sched: [1:1.00]
-; BTVER2-NEXT:    vunpckhps {{.*#+}} ymm0 = ymm0[2],mem[2],ymm0[3],mem[3],ymm0[6],mem[6],ymm0[7],mem[7] sched: [6:2.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_unpckhps:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vunpckhps {{.*#+}} ymm0 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7] sched: [1:0.50]
-; ZNVER1-NEXT:    vunpckhps {{.*#+}} ymm0 = ymm0[2],mem[2],ymm0[3],mem[3],ymm0[6],mem[6],ymm0[7],mem[7] sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = shufflevector <8 x float> %a0, <8 x float> %a1, <8 x i32> <i32 2, i32 10, i32 3, i32 11, i32 6, i32 14, i32 7, i32 15>
-  %2 = load <8 x float>, <8 x float> *%a2, align 32
-  %3 = shufflevector <8 x float> %1, <8 x float> %2, <8 x i32> <i32 2, i32 10, i32 3, i32 11, i32 6, i32 14, i32 7, i32 15>
-  ret <8 x float> %3
-}
-
-define <4 x double> @test_unpcklpd(<4 x double> %a0, <4 x double> %a1, <4 x double> *%a2) {
-; GENERIC-LABEL: test_unpcklpd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vunpcklpd {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[2],ymm1[2] sched: [1:1.00]
-; GENERIC-NEXT:    vunpcklpd {{.*#+}} ymm1 = ymm1[0],mem[0],ymm1[2],mem[2] sched: [8:1.00]
-; GENERIC-NEXT:    vaddpd %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_unpcklpd:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vunpcklpd {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[2],ymm1[2] sched: [1:1.00]
-; SANDY-NEXT:    vunpcklpd {{.*#+}} ymm1 = ymm1[0],mem[0],ymm1[2],mem[2] sched: [8:1.00]
-; SANDY-NEXT:    vaddpd %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_unpcklpd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vunpcklpd {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[2],ymm1[2] sched: [1:1.00]
-; HASWELL-NEXT:    vunpcklpd {{.*#+}} ymm1 = ymm1[0],mem[0],ymm1[2],mem[2] sched: [8:1.00]
-; HASWELL-NEXT:    vaddpd %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_unpcklpd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vunpcklpd {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[2],ymm1[2] sched: [1:1.00]
-; BROADWELL-NEXT:    vunpcklpd {{.*#+}} ymm1 = ymm1[0],mem[0],ymm1[2],mem[2] sched: [7:1.00]
-; BROADWELL-NEXT:    vaddpd %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_unpcklpd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vunpcklpd {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[2],ymm1[2] sched: [1:1.00]
-; SKYLAKE-NEXT:    vunpcklpd {{.*#+}} ymm1 = ymm1[0],mem[0],ymm1[2],mem[2] sched: [8:1.00]
-; SKYLAKE-NEXT:    vaddpd %ymm1, %ymm0, %ymm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_unpcklpd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vunpcklpd {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[2],ymm1[2] sched: [1:1.00]
-; SKX-NEXT:    vunpcklpd {{.*#+}} ymm1 = ymm1[0],mem[0],ymm1[2],mem[2] sched: [8:1.00]
-; SKX-NEXT:    vaddpd %ymm1, %ymm0, %ymm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_unpcklpd:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vunpcklpd {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[2],ymm1[2] sched: [2:1.00]
-; BDVER2-NEXT:    vunpcklpd {{.*#+}} ymm1 = ymm1[0],mem[0],ymm1[2],mem[2] sched: [7:1.00]
-; BDVER2-NEXT:    vaddpd %ymm1, %ymm0, %ymm0 # sched: [5:2.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_unpcklpd:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vunpcklpd {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[2],ymm1[2] sched: [1:1.00]
-; BTVER2-NEXT:    vunpcklpd {{.*#+}} ymm1 = ymm1[0],mem[0],ymm1[2],mem[2] sched: [6:2.00]
-; BTVER2-NEXT:    vaddpd %ymm1, %ymm0, %ymm0 # sched: [3:2.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_unpcklpd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vunpcklpd {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[2],ymm1[2] sched: [1:0.50]
-; ZNVER1-NEXT:    vunpcklpd {{.*#+}} ymm1 = ymm1[0],mem[0],ymm1[2],mem[2] sched: [8:0.50]
-; ZNVER1-NEXT:    vaddpd %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = shufflevector <4 x double> %a0, <4 x double> %a1, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
-  %2 = load <4 x double>, <4 x double> *%a2, align 32
-  %3 = shufflevector <4 x double> %a1, <4 x double> %2, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
-  %4 = fadd <4 x double> %1, %3
-  ret <4 x double> %4
-}
-
-define <8 x float> @test_unpcklps(<8 x float> %a0, <8 x float> %a1, <8 x float> *%a2) nounwind {
-; GENERIC-LABEL: test_unpcklps:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vunpcklps {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5] sched: [1:1.00]
-; GENERIC-NEXT:    vunpcklps {{.*#+}} ymm0 = ymm0[0],mem[0],ymm0[1],mem[1],ymm0[4],mem[4],ymm0[5],mem[5] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_unpcklps:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vunpcklps {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5] sched: [1:1.00]
-; SANDY-NEXT:    vunpcklps {{.*#+}} ymm0 = ymm0[0],mem[0],ymm0[1],mem[1],ymm0[4],mem[4],ymm0[5],mem[5] sched: [8:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_unpcklps:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vunpcklps {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5] sched: [1:1.00]
-; HASWELL-NEXT:    vunpcklps {{.*#+}} ymm0 = ymm0[0],mem[0],ymm0[1],mem[1],ymm0[4],mem[4],ymm0[5],mem[5] sched: [8:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_unpcklps:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vunpcklps {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5] sched: [1:1.00]
-; BROADWELL-NEXT:    vunpcklps {{.*#+}} ymm0 = ymm0[0],mem[0],ymm0[1],mem[1],ymm0[4],mem[4],ymm0[5],mem[5] sched: [7:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_unpcklps:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vunpcklps {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5] sched: [1:1.00]
-; SKYLAKE-NEXT:    vunpcklps {{.*#+}} ymm0 = ymm0[0],mem[0],ymm0[1],mem[1],ymm0[4],mem[4],ymm0[5],mem[5] sched: [8:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_unpcklps:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vunpcklps {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5] sched: [1:1.00]
-; SKX-NEXT:    vunpcklps {{.*#+}} ymm0 = ymm0[0],mem[0],ymm0[1],mem[1],ymm0[4],mem[4],ymm0[5],mem[5] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_unpcklps:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vunpcklps {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5] sched: [2:1.00]
-; BDVER2-NEXT:    vunpcklps {{.*#+}} ymm0 = ymm0[0],mem[0],ymm0[1],mem[1],ymm0[4],mem[4],ymm0[5],mem[5] sched: [7:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_unpcklps:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vunpcklps {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5] sched: [1:1.00]
-; BTVER2-NEXT:    vunpcklps {{.*#+}} ymm0 = ymm0[0],mem[0],ymm0[1],mem[1],ymm0[4],mem[4],ymm0[5],mem[5] sched: [6:2.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_unpcklps:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vunpcklps {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5] sched: [1:0.50]
-; ZNVER1-NEXT:    vunpcklps {{.*#+}} ymm0 = ymm0[0],mem[0],ymm0[1],mem[1],ymm0[4],mem[4],ymm0[5],mem[5] sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = shufflevector <8 x float> %a0, <8 x float> %a1, <8 x i32> <i32 0, i32 8, i32 1, i32 9, i32 4, i32 12, i32 5, i32 13>
-  %2 = load <8 x float>, <8 x float> *%a2, align 32
-  %3 = shufflevector <8 x float> %1, <8 x float> %2, <8 x i32> <i32 0, i32 8, i32 1, i32 9, i32 4, i32 12, i32 5, i32 13>
-  ret <8 x float> %3
-}
-
-define <4 x double> @test_xorpd(<4 x double> %a0, <4 x double> %a1, <4 x double> *%a2) {
-; GENERIC-LABEL: test_xorpd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vxorpd %ymm1, %ymm0, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vxorpd (%rdi), %ymm0, %ymm0 # sched: [8:1.00]
-; GENERIC-NEXT:    vaddpd %ymm0, %ymm1, %ymm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_xorpd:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vxorpd %ymm1, %ymm0, %ymm0 # sched: [1:1.00]
-; SANDY-NEXT:    vxorpd (%rdi), %ymm0, %ymm0 # sched: [8:1.00]
-; SANDY-NEXT:    vaddpd %ymm0, %ymm1, %ymm0 # sched: [3:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_xorpd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vxorpd %ymm1, %ymm0, %ymm0 # sched: [1:1.00]
-; HASWELL-NEXT:    vxorpd (%rdi), %ymm0, %ymm0 # sched: [8:1.00]
-; HASWELL-NEXT:    vaddpd %ymm0, %ymm1, %ymm0 # sched: [3:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_xorpd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vxorpd %ymm1, %ymm0, %ymm0 # sched: [1:1.00]
-; BROADWELL-NEXT:    vxorpd (%rdi), %ymm0, %ymm0 # sched: [7:1.00]
-; BROADWELL-NEXT:    vaddpd %ymm0, %ymm1, %ymm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_xorpd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vxorpd %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    vxorpd (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKYLAKE-NEXT:    vaddpd %ymm0, %ymm1, %ymm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_xorpd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vxorpd %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    vxorpd (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKX-NEXT:    vaddpd %ymm0, %ymm1, %ymm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_xorpd:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vxorpd %ymm1, %ymm0, %ymm0 # sched: [2:1.00]
-; BDVER2-NEXT:    vxorpd (%rdi), %ymm0, %ymm0 # sched: [7:1.00]
-; BDVER2-NEXT:    vaddpd %ymm0, %ymm1, %ymm0 # sched: [5:2.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_xorpd:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vxorpd %ymm1, %ymm0, %ymm0 # sched: [1:1.00]
-; BTVER2-NEXT:    vxorpd (%rdi), %ymm0, %ymm0 # sched: [6:2.00]
-; BTVER2-NEXT:    vaddpd %ymm0, %ymm1, %ymm0 # sched: [3:2.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_xorpd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vxorpd %ymm1, %ymm0, %ymm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vxorpd (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    vaddpd %ymm0, %ymm1, %ymm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = bitcast <4 x double> %a0 to <4 x i64>
-  %2 = bitcast <4 x double> %a1 to <4 x i64>
-  %3 = xor <4 x i64> %1, %2
-  %4 = load <4 x double>, <4 x double> *%a2, align 32
-  %5 = bitcast <4 x double> %4 to <4 x i64>
-  %6 = xor <4 x i64> %3, %5
-  %7 = bitcast <4 x i64> %6 to <4 x double>
-  %8 = fadd <4 x double> %a1, %7
-  ret <4 x double> %8
-}
-
-define <8 x float> @test_xorps(<8 x float> %a0, <8 x float> %a1, <8 x float> *%a2) {
-; GENERIC-LABEL: test_xorps:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vxorps %ymm1, %ymm0, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vxorps (%rdi), %ymm0, %ymm0 # sched: [8:1.00]
-; GENERIC-NEXT:    vaddps %ymm0, %ymm1, %ymm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_xorps:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vxorps %ymm1, %ymm0, %ymm0 # sched: [1:1.00]
-; SANDY-NEXT:    vxorps (%rdi), %ymm0, %ymm0 # sched: [8:1.00]
-; SANDY-NEXT:    vaddps %ymm0, %ymm1, %ymm0 # sched: [3:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_xorps:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vxorps %ymm1, %ymm0, %ymm0 # sched: [1:1.00]
-; HASWELL-NEXT:    vxorps (%rdi), %ymm0, %ymm0 # sched: [8:1.00]
-; HASWELL-NEXT:    vaddps %ymm0, %ymm1, %ymm0 # sched: [3:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_xorps:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vxorps %ymm1, %ymm0, %ymm0 # sched: [1:1.00]
-; BROADWELL-NEXT:    vxorps (%rdi), %ymm0, %ymm0 # sched: [7:1.00]
-; BROADWELL-NEXT:    vaddps %ymm0, %ymm1, %ymm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_xorps:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vxorps %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    vxorps (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKYLAKE-NEXT:    vaddps %ymm0, %ymm1, %ymm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_xorps:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vxorps %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    vxorps (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKX-NEXT:    vaddps %ymm0, %ymm1, %ymm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_xorps:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vxorps %ymm1, %ymm0, %ymm0 # sched: [2:1.00]
-; BDVER2-NEXT:    vxorps (%rdi), %ymm0, %ymm0 # sched: [7:1.00]
-; BDVER2-NEXT:    vaddps %ymm0, %ymm1, %ymm0 # sched: [5:2.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_xorps:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vxorps %ymm1, %ymm0, %ymm0 # sched: [1:1.00]
-; BTVER2-NEXT:    vxorps (%rdi), %ymm0, %ymm0 # sched: [6:2.00]
-; BTVER2-NEXT:    vaddps %ymm0, %ymm1, %ymm0 # sched: [3:2.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_xorps:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vxorps %ymm1, %ymm0, %ymm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vxorps (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    vaddps %ymm0, %ymm1, %ymm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = bitcast <8 x float> %a0 to <4 x i64>
-  %2 = bitcast <8 x float> %a1 to <4 x i64>
-  %3 = xor <4 x i64> %1, %2
-  %4 = load <8 x float>, <8 x float> *%a2, align 32
-  %5 = bitcast <8 x float> %4 to <4 x i64>
-  %6 = xor <4 x i64> %3, %5
-  %7 = bitcast <4 x i64> %6 to <8 x float>
-  %8 = fadd <8 x float> %a1, %7
-  ret <8 x float> %8
-}
-
-define void @test_zeroall() {
-; GENERIC-LABEL: test_zeroall:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vzeroall # sched: [9:2.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_zeroall:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vzeroall # sched: [9:2.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_zeroall:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vzeroall # sched: [8:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_zeroall:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vzeroall # sched: [8:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_zeroall:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vzeroall # sched: [16:4.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_zeroall:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vzeroall # sched: [12:5.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_zeroall:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vzeroall # sched: [90:8.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_zeroall:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vzeroall # sched: [90:36.50]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_zeroall:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vzeroall # sched: [100:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  call void @llvm.x86.avx.vzeroall()
-  ret void
-}
-declare void @llvm.x86.avx.vzeroall() nounwind
-
-define void @test_zeroupper() {
-; GENERIC-LABEL: test_zeroupper:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vzeroupper # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_zeroupper:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vzeroupper # sched: [1:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_zeroupper:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vzeroupper # sched: [0:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_zeroupper:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vzeroupper # sched: [0:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_zeroupper:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vzeroupper # sched: [0:0.67]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_zeroupper:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vzeroupper # sched: [0:0.67]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_zeroupper:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vzeroupper # sched: [46:4.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_zeroupper:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vzeroupper # sched: [46:18.50]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_zeroupper:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vzeroupper # sched: [100:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  call void @llvm.x86.avx.vzeroupper()
-  ret void
-}
-
-define void @test_avx256_zero_idioms() {
-; GENERIC-LABEL: test_avx256_zero_idioms:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    vxorps %ymm0, %ymm0, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vxorpd %ymm1, %ymm1, %ymm1 # sched: [1:1.00]
-; GENERIC-NEXT:    vandnps %ymm2, %ymm2, %ymm2 # sched: [1:1.00]
-; GENERIC-NEXT:    vandnpd %ymm3, %ymm3, %ymm3 # sched: [1:1.00]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_avx256_zero_idioms:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    vxorps %ymm0, %ymm0, %ymm0 # sched: [1:1.00]
-; SANDY-NEXT:    vxorpd %ymm1, %ymm1, %ymm1 # sched: [1:1.00]
-; SANDY-NEXT:    vandnps %ymm2, %ymm2, %ymm2 # sched: [1:1.00]
-; SANDY-NEXT:    vandnpd %ymm3, %ymm3, %ymm3 # sched: [1:1.00]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_avx256_zero_idioms:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    vxorps %ymm0, %ymm0, %ymm0 # sched: [1:1.00]
-; HASWELL-NEXT:    vxorpd %ymm1, %ymm1, %ymm1 # sched: [1:1.00]
-; HASWELL-NEXT:    vandnps %ymm2, %ymm2, %ymm2 # sched: [1:1.00]
-; HASWELL-NEXT:    vandnpd %ymm3, %ymm3, %ymm3 # sched: [1:1.00]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_avx256_zero_idioms:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    vxorps %ymm0, %ymm0, %ymm0 # sched: [1:1.00]
-; BROADWELL-NEXT:    vxorpd %ymm1, %ymm1, %ymm1 # sched: [1:1.00]
-; BROADWELL-NEXT:    vandnps %ymm2, %ymm2, %ymm2 # sched: [1:1.00]
-; BROADWELL-NEXT:    vandnpd %ymm3, %ymm3, %ymm3 # sched: [1:1.00]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_avx256_zero_idioms:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    vxorps %ymm0, %ymm0, %ymm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    vxorpd %ymm1, %ymm1, %ymm1 # sched: [1:0.33]
-; SKYLAKE-NEXT:    vandnps %ymm2, %ymm2, %ymm2 # sched: [1:0.33]
-; SKYLAKE-NEXT:    vandnpd %ymm3, %ymm3, %ymm3 # sched: [1:0.33]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_avx256_zero_idioms:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    vxorps %ymm0, %ymm0, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    vxorpd %ymm1, %ymm1, %ymm1 # sched: [1:0.33]
-; SKX-NEXT:    vandnps %ymm2, %ymm2, %ymm2 # sched: [1:0.33]
-; SKX-NEXT:    vandnpd %ymm3, %ymm3, %ymm3 # sched: [1:0.33]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_avx256_zero_idioms:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    vxorps %ymm0, %ymm0, %ymm0 # sched: [2:1.00]
-; BDVER2-NEXT:    vxorpd %ymm1, %ymm1, %ymm1 # sched: [2:1.00]
-; BDVER2-NEXT:    vandnps %ymm2, %ymm2, %ymm2 # sched: [2:1.00]
-; BDVER2-NEXT:    vandnpd %ymm3, %ymm3, %ymm3 # sched: [2:1.00]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_avx256_zero_idioms:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    vxorps %ymm0, %ymm0, %ymm0 # sched: [1:0.50]
-; BTVER2-NEXT:    vxorpd %ymm1, %ymm1, %ymm1 # sched: [1:0.50]
-; BTVER2-NEXT:    vandnps %ymm2, %ymm2, %ymm2 # sched: [1:0.50]
-; BTVER2-NEXT:    vandnpd %ymm3, %ymm3, %ymm3 # sched: [1:0.50]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_avx256_zero_idioms:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    vxorps %ymm0, %ymm0, %ymm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vxorpd %ymm1, %ymm1, %ymm1 # sched: [1:0.25]
-; ZNVER1-NEXT:    vandnps %ymm2, %ymm2, %ymm2 # sched: [1:0.25]
-; ZNVER1-NEXT:    vandnpd %ymm3, %ymm3, %ymm3 # sched: [1:0.25]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  call void asm sideeffect "vxorps %ymm0, %ymm0, %ymm0\0Avxorpd %ymm1, %ymm1, %ymm1\0Avandnps %ymm2, %ymm2, %ymm2\0Avandnpd %ymm3, %ymm3, %ymm3", ""()
-  ret void
-}
-declare void @llvm.x86.avx.vzeroupper() nounwind
-
-!0 = !{i32 1}
diff --git a/test/CodeGen/X86/avx-vbroadcast.ll b/test/CodeGen/X86/avx-vbroadcast.ll
index b136c72366e9a4536fd38e7d7a96f4827b99feda..9346c944eb8be9f966dd2bb52e0b5e14a45e984e 100644
--- a/test/CodeGen/X86/avx-vbroadcast.ll
+++ b/test/CodeGen/X86/avx-vbroadcast.ll
@@ -6,12 +6,8 @@ define <4 x i64> @A(i64* %ptr) nounwind uwtable readnone ssp {
 ; X32-LABEL: A:
 ; X32:       ## %bb.0: ## %entry
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    movl (%eax), %ecx
-; X32-NEXT:    movl 4(%eax), %eax
-; X32-NEXT:    vmovd %ecx, %xmm0
-; X32-NEXT:    vpinsrd $1, %eax, %xmm0, %xmm0
-; X32-NEXT:    vpinsrd $2, %ecx, %xmm0, %xmm0
-; X32-NEXT:    vpinsrd $3, %eax, %xmm0, %xmm0
+; X32-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; X32-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,1,0,1]
 ; X32-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
 ; X32-NEXT:    retl
 ;
@@ -31,25 +27,27 @@ entry:
 define <4 x i64> @A2(i64* %ptr, i64* %ptr2) nounwind uwtable readnone ssp {
 ; X32-LABEL: A2:
 ; X32:       ## %bb.0: ## %entry
+; X32-NEXT:    pushl %esi
+; X32-NEXT:    .cfi_def_cfa_offset 8
+; X32-NEXT:    .cfi_offset %esi, -8
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X32-NEXT:    movl (%ecx), %edx
-; X32-NEXT:    movl 4(%ecx), %ecx
-; X32-NEXT:    movl %ecx, 4(%eax)
+; X32-NEXT:    movl 4(%ecx), %esi
+; X32-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
 ; X32-NEXT:    movl %edx, (%eax)
-; X32-NEXT:    vmovd %edx, %xmm0
-; X32-NEXT:    vpinsrd $1, %ecx, %xmm0, %xmm0
-; X32-NEXT:    vpinsrd $2, %edx, %xmm0, %xmm0
-; X32-NEXT:    vpinsrd $3, %ecx, %xmm0, %xmm0
+; X32-NEXT:    movl %esi, 4(%eax)
+; X32-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,1,0,1]
 ; X32-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
+; X32-NEXT:    popl %esi
 ; X32-NEXT:    retl
 ;
 ; X64-LABEL: A2:
 ; X64:       ## %bb.0: ## %entry
 ; X64-NEXT:    movq (%rdi), %rax
-; X64-NEXT:    vmovq %rax, %xmm0
 ; X64-NEXT:    movq %rax, (%rsi)
-; X64-NEXT:    vmovddup {{.*#+}} xmm0 = xmm0[0,0]
+; X64-NEXT:    vmovq %rax, %xmm0
+; X64-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,1,0,1]
 ; X64-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
 ; X64-NEXT:    retq
 entry:
@@ -112,8 +110,8 @@ define <8 x i32> @B3(i32* %ptr, i32* %ptr2) nounwind uwtable readnone ssp {
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X32-NEXT:    movl (%ecx), %ecx
-; X32-NEXT:    vmovd %ecx, %xmm0
 ; X32-NEXT:    movl %ecx, (%eax)
+; X32-NEXT:    vmovd %ecx, %xmm0
 ; X32-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; X32-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
 ; X32-NEXT:    retl
@@ -121,8 +119,8 @@ define <8 x i32> @B3(i32* %ptr, i32* %ptr2) nounwind uwtable readnone ssp {
 ; X64-LABEL: B3:
 ; X64:       ## %bb.0: ## %entry
 ; X64-NEXT:    movl (%rdi), %eax
-; X64-NEXT:    vmovd %eax, %xmm0
 ; X64-NEXT:    movl %eax, (%rsi)
+; X64-NEXT:    vmovd %eax, %xmm0
 ; X64-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
 ; X64-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
 ; X64-NEXT:    retq
@@ -592,18 +590,13 @@ define <2 x i64> @G(i64* %ptr) nounwind uwtable readnone ssp {
 ; X32-LABEL: G:
 ; X32:       ## %bb.0: ## %entry
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    movl (%eax), %ecx
-; X32-NEXT:    movl 4(%eax), %eax
-; X32-NEXT:    vmovd %ecx, %xmm0
-; X32-NEXT:    vpinsrd $1, %eax, %xmm0, %xmm0
-; X32-NEXT:    vpinsrd $2, %ecx, %xmm0, %xmm0
-; X32-NEXT:    vpinsrd $3, %eax, %xmm0, %xmm0
+; X32-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; X32-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,1,0,1]
 ; X32-NEXT:    retl
 ;
 ; X64-LABEL: G:
 ; X64:       ## %bb.0: ## %entry
-; X64-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
-; X64-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,1,0,1]
+; X64-NEXT:    vmovddup {{.*#+}} xmm0 = mem[0,0]
 ; X64-NEXT:    retq
 entry:
   %q = load i64, i64* %ptr, align 8
@@ -615,16 +608,18 @@ entry:
 define <2 x i64> @G2(i64* %ptr, i64* %ptr2) nounwind uwtable readnone ssp {
 ; X32-LABEL: G2:
 ; X32:       ## %bb.0: ## %entry
+; X32-NEXT:    pushl %esi
+; X32-NEXT:    .cfi_def_cfa_offset 8
+; X32-NEXT:    .cfi_offset %esi, -8
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X32-NEXT:    movl (%ecx), %edx
-; X32-NEXT:    movl 4(%ecx), %ecx
-; X32-NEXT:    movl %ecx, 4(%eax)
+; X32-NEXT:    movl 4(%ecx), %esi
+; X32-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
 ; X32-NEXT:    movl %edx, (%eax)
-; X32-NEXT:    vmovd %edx, %xmm0
-; X32-NEXT:    vpinsrd $1, %ecx, %xmm0, %xmm0
-; X32-NEXT:    vpinsrd $2, %edx, %xmm0, %xmm0
-; X32-NEXT:    vpinsrd $3, %ecx, %xmm0, %xmm0
+; X32-NEXT:    movl %esi, 4(%eax)
+; X32-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,1,0,1]
+; X32-NEXT:    popl %esi
 ; X32-NEXT:    retl
 ;
 ; X64-LABEL: G2:
@@ -879,11 +874,11 @@ define float @broadcast_lifetime() nounwind {
 ; X32-NEXT:    movl %esi, (%esp)
 ; X32-NEXT:    calll _gfunc
 ; X32-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; X32-NEXT:    vmovss %xmm0, {{[0-9]+}}(%esp) ## 4-byte Spill
+; X32-NEXT:    vmovss %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) ## 4-byte Spill
 ; X32-NEXT:    movl %esi, (%esp)
 ; X32-NEXT:    calll _gfunc
 ; X32-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; X32-NEXT:    vsubss {{[0-9]+}}(%esp), %xmm0, %xmm0 ## 4-byte Folded Reload
+; X32-NEXT:    vsubss {{[-0-9]+}}(%e{{[sb]}}p), %xmm0, %xmm0 ## 4-byte Folded Reload
 ; X32-NEXT:    vmovss %xmm0, {{[0-9]+}}(%esp)
 ; X32-NEXT:    flds {{[0-9]+}}(%esp)
 ; X32-NEXT:    addl $40, %esp
@@ -896,11 +891,11 @@ define float @broadcast_lifetime() nounwind {
 ; X64-NEXT:    leaq {{[0-9]+}}(%rsp), %rdi
 ; X64-NEXT:    callq _gfunc
 ; X64-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; X64-NEXT:    vmovss %xmm0, {{[0-9]+}}(%rsp) ## 4-byte Spill
+; X64-NEXT:    vmovss %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 4-byte Spill
 ; X64-NEXT:    leaq {{[0-9]+}}(%rsp), %rdi
 ; X64-NEXT:    callq _gfunc
 ; X64-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; X64-NEXT:    vsubss {{[0-9]+}}(%rsp), %xmm0, %xmm0 ## 4-byte Folded Reload
+; X64-NEXT:    vsubss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 ## 4-byte Folded Reload
 ; X64-NEXT:    addq $40, %rsp
 ; X64-NEXT:    retq
   %1 = alloca <4 x float>, align 16
diff --git a/test/CodeGen/X86/avx2-gather.ll b/test/CodeGen/X86/avx2-gather.ll
index a7da2fcc91d0467f12d503002190cbb6aa90191b..23a8c9ef8268c092cd8b4839da93d818d185bb00 100644
--- a/test/CodeGen/X86/avx2-gather.ll
+++ b/test/CodeGen/X86/avx2-gather.ll
@@ -145,3 +145,28 @@ define <2 x double> @test_mm_i32gather_pd(double *%a0, <2 x i64> %a1) {
   %res = call <2 x double> @llvm.x86.avx2.gather.d.pd(<2 x double> zeroinitializer, i8* %arg0, <4 x i32> %arg1, <2 x double> %mask, i8 2)
   ret <2 x double> %res
 }
+
+@x = global [1024 x float] zeroinitializer, align 16
+
+define <4 x float> @gather_global(<4 x i64>, i32* nocapture readnone) {
+; X32-LABEL: gather_global:
+; X32:       # %bb.0:
+; X32-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
+; X32-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; X32-NEXT:    vgatherqps %xmm2, x(,%ymm0,4), %xmm1
+; X32-NEXT:    vmovaps %xmm1, %xmm0
+; X32-NEXT:    vzeroupper
+; X32-NEXT:    retl
+;
+; X64-LABEL: gather_global:
+; X64:       # %bb.0:
+; X64-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
+; X64-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; X64-NEXT:    vgatherqps %xmm2, x(,%ymm0,4), %xmm1
+; X64-NEXT:    vmovaps %xmm1, %xmm0
+; X64-NEXT:    vzeroupper
+; X64-NEXT:    retq
+  %3 = tail call <4 x float> @llvm.x86.avx2.gather.q.ps.256(<4 x float> zeroinitializer, i8* bitcast ([1024 x float]* @x to i8*), <4 x i64> %0, <4 x float> <float 0xFFFFFFFFE0000000, float 0xFFFFFFFFE0000000, float 0xFFFFFFFFE0000000, float 0xFFFFFFFFE0000000>, i8 4)
+  ret <4 x float> %3
+}
+declare <4 x float> @llvm.x86.avx2.gather.q.ps.256(<4 x float>, i8*, <4 x i64>, <4 x float>, i8)
diff --git a/test/CodeGen/X86/avx2-intrinsics-fast-isel.ll b/test/CodeGen/X86/avx2-intrinsics-fast-isel.ll
index e0c7a5e673b8ccbab8453dc07c172f33a71ed330..8720c86edffff908d0f9c90f327a84098817fa89 100644
--- a/test/CodeGen/X86/avx2-intrinsics-fast-isel.ll
+++ b/test/CodeGen/X86/avx2-intrinsics-fast-isel.ll
@@ -98,11 +98,11 @@ define <4 x i64> @test_mm256_adds_epi8(<4 x i64> %a0, <4 x i64> %a1) {
 ; CHECK-NEXT:    ret{{[l|q]}}
   %arg0 = bitcast <4 x i64> %a0 to <32 x i8>
   %arg1 = bitcast <4 x i64> %a1 to <32 x i8>
-  %res = call <32 x i8> @llvm.x86.avx2.padds.b(<32 x i8> %arg0, <32 x i8> %arg1)
+  %res = call <32 x i8> @llvm.sadd.sat.v32i8(<32 x i8> %arg0, <32 x i8> %arg1)
   %bc = bitcast <32 x i8> %res to <4 x i64>
   ret <4 x i64> %bc
 }
-declare <32 x i8> @llvm.x86.avx2.padds.b(<32 x i8>, <32 x i8>) nounwind readnone
+declare <32 x i8> @llvm.sadd.sat.v32i8(<32 x i8>, <32 x i8>) nounwind readnone
 
 define <4 x i64> @test_mm256_adds_epi16(<4 x i64> %a0, <4 x i64> %a1) {
 ; CHECK-LABEL: test_mm256_adds_epi16:
@@ -111,11 +111,11 @@ define <4 x i64> @test_mm256_adds_epi16(<4 x i64> %a0, <4 x i64> %a1) {
 ; CHECK-NEXT:    ret{{[l|q]}}
   %arg0 = bitcast <4 x i64> %a0 to <16 x i16>
   %arg1 = bitcast <4 x i64> %a1 to <16 x i16>
-  %res = call <16 x i16> @llvm.x86.avx2.padds.w(<16 x i16> %arg0, <16 x i16> %arg1)
+  %res = call <16 x i16> @llvm.sadd.sat.v16i16(<16 x i16> %arg0, <16 x i16> %arg1)
   %bc = bitcast <16 x i16> %res to <4 x i64>
   ret <4 x i64> %bc
 }
-declare <16 x i16> @llvm.x86.avx2.padds.w(<16 x i16>, <16 x i16>) nounwind readnone
+declare <16 x i16> @llvm.sadd.sat.v16i16(<16 x i16>, <16 x i16>) nounwind readnone
 
 define <4 x i64> @test_mm256_adds_epu8(<4 x i64> %a0, <4 x i64> %a1) {
 ; CHECK-LABEL: test_mm256_adds_epu8:
@@ -124,13 +124,11 @@ define <4 x i64> @test_mm256_adds_epu8(<4 x i64> %a0, <4 x i64> %a1) {
 ; CHECK-NEXT:    ret{{[l|q]}}
   %arg0 = bitcast <4 x i64> %a0 to <32 x i8>
   %arg1 = bitcast <4 x i64> %a1 to <32 x i8>
-  %1 = add <32 x i8> %arg0, %arg1
-  %2 = icmp ugt <32 x i8> %arg0, %1
-  %3 = select <32 x i1> %2, <32 x i8> <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>, <32 x i8> %1
-  %bc = bitcast <32 x i8> %3 to <4 x i64>
+  %res = call <32 x i8> @llvm.uadd.sat.v32i8(<32 x i8> %arg0, <32 x i8> %arg1)
+  %bc = bitcast <32 x i8> %res to <4 x i64>
   ret <4 x i64> %bc
 }
-
+declare <32 x i8> @llvm.uadd.sat.v32i8(<32 x i8>, <32 x i8>)
 
 define <4 x i64> @test_mm256_adds_epu16(<4 x i64> %a0, <4 x i64> %a1) {
 ; CHECK-LABEL: test_mm256_adds_epu16:
@@ -139,12 +137,11 @@ define <4 x i64> @test_mm256_adds_epu16(<4 x i64> %a0, <4 x i64> %a1) {
 ; CHECK-NEXT:    ret{{[l|q]}}
   %arg0 = bitcast <4 x i64> %a0 to <16 x i16>
   %arg1 = bitcast <4 x i64> %a1 to <16 x i16>
-  %1 = add <16 x i16> %arg0, %arg1
-  %2 = icmp ugt <16 x i16> %arg0, %1
-  %3 = select <16 x i1> %2, <16 x i16> <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>, <16 x i16> %1
-  %bc = bitcast <16 x i16> %3 to <4 x i64>
+  %res = call <16 x i16> @llvm.uadd.sat.v16i16(<16 x i16> %arg0, <16 x i16> %arg1)
+  %bc = bitcast <16 x i16> %res to <4 x i64>
   ret <4 x i64> %bc
 }
+declare <16 x i16> @llvm.uadd.sat.v16i16(<16 x i16>, <16 x i16>)
 
 define <4 x i64> @test_mm256_alignr_epi8(<4 x i64> %a0, <4 x i64> %a1) {
 ; CHECK-LABEL: test_mm256_alignr_epi8:
@@ -322,7 +319,7 @@ define <4 x i64> @test_mm256_broadcastd_epi32(<4 x i64> %a0) {
 define <2 x i64> @test_mm_broadcastq_epi64(<2 x i64> %a0) {
 ; CHECK-LABEL: test_mm_broadcastq_epi64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vpbroadcastq %xmm0, %xmm0
+; CHECK-NEXT:    vmovddup {{.*#+}} xmm0 = xmm0[0,0]
 ; CHECK-NEXT:    ret{{[l|q]}}
   %res = shufflevector <2 x i64> %a0, <2 x i64> undef, <2 x i32> zeroinitializer
   ret <2 x i64> %res
@@ -1826,12 +1823,6 @@ declare <16 x i16> @llvm.x86.avx2.mpsadbw(<32 x i8>, <32 x i8>, i8) nounwind rea
 define <4 x i64> @test_mm256_mul_epi32(<4 x i64> %a0, <4 x i64> %a1) {
 ; CHECK-LABEL: test_mm256_mul_epi32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vpsllq $32, %ymm0, %ymm2
-; CHECK-NEXT:    vpsrad $31, %ymm2, %ymm2
-; CHECK-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0],ymm2[1],ymm0[2],ymm2[3],ymm0[4],ymm2[5],ymm0[6],ymm2[7]
-; CHECK-NEXT:    vpsllq $32, %ymm1, %ymm2
-; CHECK-NEXT:    vpsrad $31, %ymm2, %ymm2
-; CHECK-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0],ymm2[1],ymm1[2],ymm2[3],ymm1[4],ymm2[5],ymm1[6],ymm2[7]
 ; CHECK-NEXT:    vpmuldq %ymm1, %ymm0, %ymm0
 ; CHECK-NEXT:    ret{{[l|q]}}
   %A = shl <4 x i64> %a0, <i64 32, i64 32, i64 32, i64 32>
@@ -1846,9 +1837,6 @@ declare <4 x i64> @llvm.x86.avx2.pmul.dq(<8 x i32>, <8 x i32>) nounwind readnone
 define <4 x i64> @test_mm256_mul_epu32(<4 x i64> %a0, <4 x i64> %a1) {
 ; CHECK-LABEL: test_mm256_mul_epu32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; CHECK-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0],ymm2[1],ymm0[2],ymm2[3],ymm0[4],ymm2[5],ymm0[6],ymm2[7]
-; CHECK-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0],ymm2[1],ymm1[2],ymm2[3],ymm1[4],ymm2[5],ymm1[6],ymm2[7]
 ; CHECK-NEXT:    vpmuludq %ymm1, %ymm0, %ymm0
 ; CHECK-NEXT:    ret{{[l|q]}}
   %A = and <4 x i64> %a0, <i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295>
@@ -2530,11 +2518,11 @@ define <4 x i64> @test_mm256_subs_epi8(<4 x i64> %a0, <4 x i64> %a1) {
 ; CHECK-NEXT:    ret{{[l|q]}}
   %arg0 = bitcast <4 x i64> %a0 to <32 x i8>
   %arg1 = bitcast <4 x i64> %a1 to <32 x i8>
-  %res = call <32 x i8> @llvm.x86.avx2.psubs.b(<32 x i8> %arg0, <32 x i8> %arg1)
+  %res = call <32 x i8> @llvm.ssub.sat.v32i8(<32 x i8> %arg0, <32 x i8> %arg1)
   %bc = bitcast <32 x i8> %res to <4 x i64>
   ret <4 x i64> %bc
 }
-declare <32 x i8> @llvm.x86.avx2.psubs.b(<32 x i8>, <32 x i8>) nounwind readnone
+declare <32 x i8> @llvm.ssub.sat.v32i8(<32 x i8>, <32 x i8>) nounwind readnone
 
 define <4 x i64> @test_mm256_subs_epi16(<4 x i64> %a0, <4 x i64> %a1) {
 ; CHECK-LABEL: test_mm256_subs_epi16:
@@ -2543,42 +2531,37 @@ define <4 x i64> @test_mm256_subs_epi16(<4 x i64> %a0, <4 x i64> %a1) {
 ; CHECK-NEXT:    ret{{[l|q]}}
   %arg0 = bitcast <4 x i64> %a0 to <16 x i16>
   %arg1 = bitcast <4 x i64> %a1 to <16 x i16>
-  %res = call <16 x i16> @llvm.x86.avx2.psubs.w(<16 x i16> %arg0, <16 x i16> %arg1)
+  %res = call <16 x i16> @llvm.ssub.sat.v16i16(<16 x i16> %arg0, <16 x i16> %arg1)
   %bc = bitcast <16 x i16> %res to <4 x i64>
   ret <4 x i64> %bc
 }
-declare <16 x i16> @llvm.x86.avx2.psubs.w(<16 x i16>, <16 x i16>) nounwind readnone
+declare <16 x i16> @llvm.ssub.sat.v16i16(<16 x i16>, <16 x i16>) nounwind readnone
 
 define <4 x i64> @test_mm256_subs_epu8(<4 x i64> %a0, <4 x i64> %a1) {
 ; CHECK-LABEL: test_mm256_subs_epu8:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vpmaxub %ymm1, %ymm0, %ymm0
-; CHECK-NEXT:    vpsubb %ymm1, %ymm0, %ymm0
+; CHECK-NEXT:    vpsubusb %ymm1, %ymm0, %ymm0
 ; CHECK-NEXT:    ret{{[l|q]}}
   %arg0 = bitcast <4 x i64> %a0 to <32 x i8>
   %arg1 = bitcast <4 x i64> %a1 to <32 x i8>
-  %cmp = icmp ugt <32 x i8> %arg0, %arg1
-  %sel = select <32 x i1> %cmp, <32 x i8> %arg0, <32 x i8> %arg1
-  %sub = sub <32 x i8> %sel, %arg1
-  %bc = bitcast <32 x i8> %sub to <4 x i64>
+  %res = call <32 x i8> @llvm.usub.sat.v32i8(<32 x i8> %arg0, <32 x i8> %arg1)
+  %bc = bitcast <32 x i8> %res to <4 x i64>
   ret <4 x i64> %bc
 }
-
+declare <32 x i8> @llvm.usub.sat.v32i8(<32 x i8>, <32 x i8>)
 
 define <4 x i64> @test_mm256_subs_epu16(<4 x i64> %a0, <4 x i64> %a1) {
 ; CHECK-LABEL: test_mm256_subs_epu16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vpmaxuw %ymm1, %ymm0, %ymm0
-; CHECK-NEXT:    vpsubw %ymm1, %ymm0, %ymm0
+; CHECK-NEXT:    vpsubusw %ymm1, %ymm0, %ymm0
 ; CHECK-NEXT:    ret{{[l|q]}}
   %arg0 = bitcast <4 x i64> %a0 to <16 x i16>
   %arg1 = bitcast <4 x i64> %a1 to <16 x i16>
-  %cmp = icmp ugt <16 x i16> %arg0, %arg1
-  %sel = select <16 x i1> %cmp, <16 x i16> %arg0, <16 x i16> %arg1
-  %sub = sub <16 x i16> %sel, %arg1
-  %bc = bitcast <16 x i16> %sub to <4 x i64>
+  %res = call <16 x i16> @llvm.usub.sat.v16i16(<16 x i16> %arg0, <16 x i16> %arg1)
+  %bc = bitcast <16 x i16> %res to <4 x i64>
   ret <4 x i64> %bc
 }
+declare <16 x i16> @llvm.usub.sat.v16i16(<16 x i16>, <16 x i16>)
 
 define <4 x i64> @test_mm256_unpackhi_epi8(<4 x i64> %a0, <4 x i64> %a1) nounwind {
 ; CHECK-LABEL: test_mm256_unpackhi_epi8:
diff --git a/test/CodeGen/X86/avx2-intrinsics-x86-upgrade.ll b/test/CodeGen/X86/avx2-intrinsics-x86-upgrade.ll
index 81e10a5d2427a619ff74ee9dac25ea3653f54edf..b340a66def96520b905a0d6a0db9e86206002826 100644
--- a/test/CodeGen/X86/avx2-intrinsics-x86-upgrade.ll
+++ b/test/CodeGen/X86/avx2-intrinsics-x86-upgrade.ll
@@ -234,7 +234,7 @@ declare <8 x i32> @llvm.x86.avx2.pbroadcastd.256(<4 x i32>) nounwind readonly
 define <2 x i64> @test_x86_avx2_pbroadcastq_128(<2 x i64> %a0) {
 ; CHECK-LABEL: test_x86_avx2_pbroadcastq_128:
 ; CHECK:       ## %bb.0:
-; CHECK-NEXT:    vpbroadcastq %xmm0, %xmm0
+; CHECK-NEXT:    vmovddup {{.*#+}} xmm0 = xmm0[0,0]
 ; CHECK-NEXT:    ret{{[l|q]}}
   %res = call <2 x i64> @llvm.x86.avx2.pbroadcastq.128(<2 x i64> %a0)
   ret <2 x i64> %res
@@ -614,6 +614,28 @@ define <4 x i64> @test_x86_avx2_pmul_dq(<8 x i32> %a0, <8 x i32> %a1) {
 declare <4 x i64> @llvm.x86.avx2.pmul.dq(<8 x i32>, <8 x i32>) nounwind readnone
 
 
+define <32 x i8> @test_x86_avx2_padds_b(<32 x i8> %a0, <32 x i8> %a1) {
+; CHECK-LABEL: test_x86_avx2_padds_b:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    vpaddsb %ymm1, %ymm0, %ymm0
+; CHECK-NEXT:    ret{{[l|q]}}
+  %res = call <32 x i8> @llvm.sadd.sat.v32i8(<32 x i8> %a0, <32 x i8> %a1) ; <<32 x i8>> [#uses=1]
+  ret <32 x i8> %res
+}
+declare <32 x i8> @llvm.sadd.sat.v32i8(<32 x i8>, <32 x i8>) nounwind readnone
+
+
+define <16 x i16> @test_x86_avx2_padds_w(<16 x i16> %a0, <16 x i16> %a1) {
+; CHECK-LABEL: test_x86_avx2_padds_w:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    vpaddsw %ymm1, %ymm0, %ymm0
+; CHECK-NEXT:    ret{{[l|q]}}
+  %res = call <16 x i16> @llvm.sadd.sat.v16i16(<16 x i16> %a0, <16 x i16> %a1) ; <<16 x i16>> [#uses=1]
+  ret <16 x i16> %res
+}
+declare <16 x i16> @llvm.sadd.sat.v16i16(<16 x i16>, <16 x i16>) nounwind readnone
+
+
 define <32 x i8> @test_x86_avx2_paddus_b(<32 x i8> %a0, <32 x i8> %a1) {
 ; CHECK-LABEL: test_x86_avx2_paddus_b:
 ; CHECK:       ## %bb.0:
@@ -636,6 +658,28 @@ define <16 x i16> @test_x86_avx2_paddus_w(<16 x i16> %a0, <16 x i16> %a1) {
 declare <16 x i16> @llvm.x86.avx2.paddus.w(<16 x i16>, <16 x i16>) nounwind readnone
 
 
+define <32 x i8> @test_x86_avx2_psubs_b(<32 x i8> %a0, <32 x i8> %a1) {
+; CHECK-LABEL: test_x86_avx2_psubs_b:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    vpsubsb %ymm1, %ymm0, %ymm0
+; CHECK-NEXT:    ret{{[l|q]}}
+  %res = call <32 x i8> @llvm.ssub.sat.v32i8(<32 x i8> %a0, <32 x i8> %a1) ; <<32 x i8>> [#uses=1]
+  ret <32 x i8> %res
+}
+declare <32 x i8> @llvm.ssub.sat.v32i8(<32 x i8>, <32 x i8>) nounwind readnone
+
+
+define <16 x i16> @test_x86_avx2_psubs_w(<16 x i16> %a0, <16 x i16> %a1) {
+; CHECK-LABEL: test_x86_avx2_psubs_w:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    vpsubsw %ymm1, %ymm0, %ymm0
+; CHECK-NEXT:    ret{{[l|q]}}
+  %res = call <16 x i16> @llvm.ssub.sat.v16i16(<16 x i16> %a0, <16 x i16> %a1) ; <<16 x i16>> [#uses=1]
+  ret <16 x i16> %res
+}
+declare <16 x i16> @llvm.ssub.sat.v16i16(<16 x i16>, <16 x i16>) nounwind readnone
+
+
 define <32 x i8> @test_x86_avx2_psubus_b(<32 x i8> %a0, <32 x i8> %a1) {
 ; CHECK-LABEL: test_x86_avx2_psubus_b:
 ; CHECK:       ## %bb.0:
diff --git a/test/CodeGen/X86/avx2-intrinsics-x86.ll b/test/CodeGen/X86/avx2-intrinsics-x86.ll
index 101448e22acbeeb3523446437e23ef9feb7462c7..de20c07d41399a8cbbee5a99299d5d1984fd0651 100644
--- a/test/CodeGen/X86/avx2-intrinsics-x86.ll
+++ b/test/CodeGen/X86/avx2-intrinsics-x86.ll
@@ -1,19 +1,19 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -disable-peephole -mtriple=i686-apple-darwin -mattr=avx2 -show-mc-encoding | FileCheck %s --check-prefix=CHECK --check-prefix=AVX2 --check-prefix=X86 --check-prefix=X86-AVX
-; RUN: llc < %s -disable-peephole -mtriple=i686-apple-darwin -mattr=+avx512f,+avx512bw,+avx512vl,+avx512dq -show-mc-encoding | FileCheck %s --check-prefix=CHECK --check-prefix=AVX512VL --check-prefix=X86 --check-prefix=X86-AVX512VL
-; RUN: llc < %s -disable-peephole -mtriple=x86_64-apple-darwin -mattr=avx2 -show-mc-encoding | FileCheck %s --check-prefix=CHECK --check-prefix=AVX2 --check-prefix=X64 --check-prefix=X64-AVX
-; RUN: llc < %s -disable-peephole -mtriple=x86_64-apple-darwin -mattr=+avx512f,+avx512bw,+avx512vl,+avx512dq -show-mc-encoding | FileCheck %s --check-prefix=CHECK --check-prefix=AVX512VL --check-prefix=X64 --check-prefix=X64-AVX512VL
+; RUN: llc < %s -disable-peephole -mtriple=i686-unknown-unknown -mattr=avx2 -show-mc-encoding | FileCheck %s --check-prefix=CHECK --check-prefix=AVX2 --check-prefix=X86 --check-prefix=X86-AVX
+; RUN: llc < %s -disable-peephole -mtriple=i686-unknown-unknown -mattr=+avx512f,+avx512bw,+avx512vl,+avx512dq -show-mc-encoding | FileCheck %s --check-prefix=CHECK --check-prefix=AVX512VL --check-prefix=X86 --check-prefix=X86-AVX512VL
+; RUN: llc < %s -disable-peephole -mtriple=x86_64-unknown-unknown -mattr=avx2 -show-mc-encoding | FileCheck %s --check-prefix=CHECK --check-prefix=AVX2 --check-prefix=X64 --check-prefix=X64-AVX
+; RUN: llc < %s -disable-peephole -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512bw,+avx512vl,+avx512dq -show-mc-encoding | FileCheck %s --check-prefix=CHECK --check-prefix=AVX512VL --check-prefix=X64 --check-prefix=X64-AVX512VL
 
 define <16 x i16> @test_x86_avx2_packssdw(<8 x i32> %a0, <8 x i32> %a1) {
 ; AVX2-LABEL: test_x86_avx2_packssdw:
-; AVX2:       ## %bb.0:
-; AVX2-NEXT:    vpackssdw %ymm1, %ymm0, %ymm0 ## encoding: [0xc5,0xfd,0x6b,0xc1]
-; AVX2-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpackssdw %ymm1, %ymm0, %ymm0 # encoding: [0xc5,0xfd,0x6b,0xc1]
+; AVX2-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
 ;
 ; AVX512VL-LABEL: test_x86_avx2_packssdw:
-; AVX512VL:       ## %bb.0:
-; AVX512VL-NEXT:    vpackssdw %ymm1, %ymm0, %ymm0 ## EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6b,0xc1]
-; AVX512VL-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpackssdw %ymm1, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6b,0xc1]
+; AVX512VL-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
   %res = call <16 x i16> @llvm.x86.avx2.packssdw(<8 x i32> %a0, <8 x i32> %a1) ; <<16 x i16>> [#uses=1]
   ret <16 x i16> %res
 }
@@ -22,32 +22,32 @@ declare <16 x i16> @llvm.x86.avx2.packssdw(<8 x i32>, <8 x i32>) nounwind readno
 
 define <16 x i16> @test_x86_avx2_packssdw_fold() {
 ; X86-AVX-LABEL: test_x86_avx2_packssdw_fold:
-; X86-AVX:       ## %bb.0:
+; X86-AVX:       # %bb.0:
 ; X86-AVX-NEXT:    vmovaps {{.*#+}} ymm0 = [0,0,0,0,255,32767,32767,65535,0,0,0,0,32769,32768,0,65280]
-; X86-AVX-NEXT:    ## encoding: [0xc5,0xfc,0x28,0x05,A,A,A,A]
-; X86-AVX-NEXT:    ## fixup A - offset: 4, value: LCPI1_0, kind: FK_Data_4
-; X86-AVX-NEXT:    retl ## encoding: [0xc3]
+; X86-AVX-NEXT:    # encoding: [0xc5,0xfc,0x28,0x05,A,A,A,A]
+; X86-AVX-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}, kind: FK_Data_4
+; X86-AVX-NEXT:    retl # encoding: [0xc3]
 ;
 ; X86-AVX512VL-LABEL: test_x86_avx2_packssdw_fold:
-; X86-AVX512VL:       ## %bb.0:
-; X86-AVX512VL-NEXT:    vmovaps LCPI1_0, %ymm0 ## EVEX TO VEX Compression ymm0 = [0,0,0,0,255,32767,32767,65535,0,0,0,0,32769,32768,0,65280]
-; X86-AVX512VL-NEXT:    ## encoding: [0xc5,0xfc,0x28,0x05,A,A,A,A]
-; X86-AVX512VL-NEXT:    ## fixup A - offset: 4, value: LCPI1_0, kind: FK_Data_4
-; X86-AVX512VL-NEXT:    retl ## encoding: [0xc3]
+; X86-AVX512VL:       # %bb.0:
+; X86-AVX512VL-NEXT:    vmovaps {{\.LCPI.*}}, %ymm0 # EVEX TO VEX Compression ymm0 = [0,0,0,0,255,32767,32767,65535,0,0,0,0,32769,32768,0,65280]
+; X86-AVX512VL-NEXT:    # encoding: [0xc5,0xfc,0x28,0x05,A,A,A,A]
+; X86-AVX512VL-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}, kind: FK_Data_4
+; X86-AVX512VL-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-AVX-LABEL: test_x86_avx2_packssdw_fold:
-; X64-AVX:       ## %bb.0:
+; X64-AVX:       # %bb.0:
 ; X64-AVX-NEXT:    vmovaps {{.*#+}} ymm0 = [0,0,0,0,255,32767,32767,65535,0,0,0,0,32769,32768,0,65280]
-; X64-AVX-NEXT:    ## encoding: [0xc5,0xfc,0x28,0x05,A,A,A,A]
-; X64-AVX-NEXT:    ## fixup A - offset: 4, value: LCPI1_0-4, kind: reloc_riprel_4byte
-; X64-AVX-NEXT:    retq ## encoding: [0xc3]
+; X64-AVX-NEXT:    # encoding: [0xc5,0xfc,0x28,0x05,A,A,A,A]
+; X64-AVX-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX-NEXT:    retq # encoding: [0xc3]
 ;
 ; X64-AVX512VL-LABEL: test_x86_avx2_packssdw_fold:
-; X64-AVX512VL:       ## %bb.0:
-; X64-AVX512VL-NEXT:    vmovaps {{.*}}(%rip), %ymm0 ## EVEX TO VEX Compression ymm0 = [0,0,0,0,255,32767,32767,65535,0,0,0,0,32769,32768,0,65280]
-; X64-AVX512VL-NEXT:    ## encoding: [0xc5,0xfc,0x28,0x05,A,A,A,A]
-; X64-AVX512VL-NEXT:    ## fixup A - offset: 4, value: LCPI1_0-4, kind: reloc_riprel_4byte
-; X64-AVX512VL-NEXT:    retq ## encoding: [0xc3]
+; X64-AVX512VL:       # %bb.0:
+; X64-AVX512VL-NEXT:    vmovaps {{.*}}(%rip), %ymm0 # EVEX TO VEX Compression ymm0 = [0,0,0,0,255,32767,32767,65535,0,0,0,0,32769,32768,0,65280]
+; X64-AVX512VL-NEXT:    # encoding: [0xc5,0xfc,0x28,0x05,A,A,A,A]
+; X64-AVX512VL-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX512VL-NEXT:    retq # encoding: [0xc3]
   %res = call <16 x i16> @llvm.x86.avx2.packssdw(<8 x i32> zeroinitializer, <8 x i32> <i32 255, i32 32767, i32 65535, i32 -1, i32 -32767, i32 -65535, i32 0, i32 -256>)
   ret <16 x i16> %res
 }
@@ -55,14 +55,14 @@ define <16 x i16> @test_x86_avx2_packssdw_fold() {
 
 define <32 x i8> @test_x86_avx2_packsswb(<16 x i16> %a0, <16 x i16> %a1) {
 ; AVX2-LABEL: test_x86_avx2_packsswb:
-; AVX2:       ## %bb.0:
-; AVX2-NEXT:    vpacksswb %ymm1, %ymm0, %ymm0 ## encoding: [0xc5,0xfd,0x63,0xc1]
-; AVX2-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpacksswb %ymm1, %ymm0, %ymm0 # encoding: [0xc5,0xfd,0x63,0xc1]
+; AVX2-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
 ;
 ; AVX512VL-LABEL: test_x86_avx2_packsswb:
-; AVX512VL:       ## %bb.0:
-; AVX512VL-NEXT:    vpacksswb %ymm1, %ymm0, %ymm0 ## EVEX TO VEX Compression encoding: [0xc5,0xfd,0x63,0xc1]
-; AVX512VL-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpacksswb %ymm1, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x63,0xc1]
+; AVX512VL-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
   %res = call <32 x i8> @llvm.x86.avx2.packsswb(<16 x i16> %a0, <16 x i16> %a1) ; <<32 x i8>> [#uses=1]
   ret <32 x i8> %res
 }
@@ -71,32 +71,32 @@ declare <32 x i8> @llvm.x86.avx2.packsswb(<16 x i16>, <16 x i16>) nounwind readn
 
 define <32 x i8> @test_x86_avx2_packsswb_fold() {
 ; X86-AVX-LABEL: test_x86_avx2_packsswb_fold:
-; X86-AVX:       ## %bb.0:
+; X86-AVX:       # %bb.0:
 ; X86-AVX-NEXT:    vmovaps {{.*#+}} ymm0 = [0,127,127,255,255,128,128,128,0,0,0,0,0,0,0,0,0,127,127,255,255,128,128,128,0,0,0,0,0,0,0,0]
-; X86-AVX-NEXT:    ## encoding: [0xc5,0xfc,0x28,0x05,A,A,A,A]
-; X86-AVX-NEXT:    ## fixup A - offset: 4, value: LCPI3_0, kind: FK_Data_4
-; X86-AVX-NEXT:    retl ## encoding: [0xc3]
+; X86-AVX-NEXT:    # encoding: [0xc5,0xfc,0x28,0x05,A,A,A,A]
+; X86-AVX-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}, kind: FK_Data_4
+; X86-AVX-NEXT:    retl # encoding: [0xc3]
 ;
 ; X86-AVX512VL-LABEL: test_x86_avx2_packsswb_fold:
-; X86-AVX512VL:       ## %bb.0:
-; X86-AVX512VL-NEXT:    vmovaps LCPI3_0, %ymm0 ## EVEX TO VEX Compression ymm0 = [0,127,127,255,255,128,128,128,0,0,0,0,0,0,0,0,0,127,127,255,255,128,128,128,0,0,0,0,0,0,0,0]
-; X86-AVX512VL-NEXT:    ## encoding: [0xc5,0xfc,0x28,0x05,A,A,A,A]
-; X86-AVX512VL-NEXT:    ## fixup A - offset: 4, value: LCPI3_0, kind: FK_Data_4
-; X86-AVX512VL-NEXT:    retl ## encoding: [0xc3]
+; X86-AVX512VL:       # %bb.0:
+; X86-AVX512VL-NEXT:    vmovaps {{\.LCPI.*}}, %ymm0 # EVEX TO VEX Compression ymm0 = [0,127,127,255,255,128,128,128,0,0,0,0,0,0,0,0,0,127,127,255,255,128,128,128,0,0,0,0,0,0,0,0]
+; X86-AVX512VL-NEXT:    # encoding: [0xc5,0xfc,0x28,0x05,A,A,A,A]
+; X86-AVX512VL-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}, kind: FK_Data_4
+; X86-AVX512VL-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-AVX-LABEL: test_x86_avx2_packsswb_fold:
-; X64-AVX:       ## %bb.0:
+; X64-AVX:       # %bb.0:
 ; X64-AVX-NEXT:    vmovaps {{.*#+}} ymm0 = [0,127,127,255,255,128,128,128,0,0,0,0,0,0,0,0,0,127,127,255,255,128,128,128,0,0,0,0,0,0,0,0]
-; X64-AVX-NEXT:    ## encoding: [0xc5,0xfc,0x28,0x05,A,A,A,A]
-; X64-AVX-NEXT:    ## fixup A - offset: 4, value: LCPI3_0-4, kind: reloc_riprel_4byte
-; X64-AVX-NEXT:    retq ## encoding: [0xc3]
+; X64-AVX-NEXT:    # encoding: [0xc5,0xfc,0x28,0x05,A,A,A,A]
+; X64-AVX-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX-NEXT:    retq # encoding: [0xc3]
 ;
 ; X64-AVX512VL-LABEL: test_x86_avx2_packsswb_fold:
-; X64-AVX512VL:       ## %bb.0:
-; X64-AVX512VL-NEXT:    vmovaps {{.*}}(%rip), %ymm0 ## EVEX TO VEX Compression ymm0 = [0,127,127,255,255,128,128,128,0,0,0,0,0,0,0,0,0,127,127,255,255,128,128,128,0,0,0,0,0,0,0,0]
-; X64-AVX512VL-NEXT:    ## encoding: [0xc5,0xfc,0x28,0x05,A,A,A,A]
-; X64-AVX512VL-NEXT:    ## fixup A - offset: 4, value: LCPI3_0-4, kind: reloc_riprel_4byte
-; X64-AVX512VL-NEXT:    retq ## encoding: [0xc3]
+; X64-AVX512VL:       # %bb.0:
+; X64-AVX512VL-NEXT:    vmovaps {{.*}}(%rip), %ymm0 # EVEX TO VEX Compression ymm0 = [0,127,127,255,255,128,128,128,0,0,0,0,0,0,0,0,0,127,127,255,255,128,128,128,0,0,0,0,0,0,0,0]
+; X64-AVX512VL-NEXT:    # encoding: [0xc5,0xfc,0x28,0x05,A,A,A,A]
+; X64-AVX512VL-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX512VL-NEXT:    retq # encoding: [0xc3]
   %res = call <32 x i8> @llvm.x86.avx2.packsswb(<16 x i16> <i16 0, i16 255, i16 256, i16 65535, i16 -1, i16 -255, i16 -256, i16 -32678, i16 0, i16 255, i16 256, i16 65535, i16 -1, i16 -255, i16 -256, i16 -32678>, <16 x i16> zeroinitializer)
   ret <32 x i8> %res
 }
@@ -104,14 +104,14 @@ define <32 x i8> @test_x86_avx2_packsswb_fold() {
 
 define <32 x i8> @test_x86_avx2_packuswb(<16 x i16> %a0, <16 x i16> %a1) {
 ; AVX2-LABEL: test_x86_avx2_packuswb:
-; AVX2:       ## %bb.0:
-; AVX2-NEXT:    vpackuswb %ymm1, %ymm0, %ymm0 ## encoding: [0xc5,0xfd,0x67,0xc1]
-; AVX2-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpackuswb %ymm1, %ymm0, %ymm0 # encoding: [0xc5,0xfd,0x67,0xc1]
+; AVX2-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
 ;
 ; AVX512VL-LABEL: test_x86_avx2_packuswb:
-; AVX512VL:       ## %bb.0:
-; AVX512VL-NEXT:    vpackuswb %ymm1, %ymm0, %ymm0 ## EVEX TO VEX Compression encoding: [0xc5,0xfd,0x67,0xc1]
-; AVX512VL-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpackuswb %ymm1, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x67,0xc1]
+; AVX512VL-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
   %res = call <32 x i8> @llvm.x86.avx2.packuswb(<16 x i16> %a0, <16 x i16> %a1) ; <<32 x i8>> [#uses=1]
   ret <32 x i8> %res
 }
@@ -120,79 +120,47 @@ declare <32 x i8> @llvm.x86.avx2.packuswb(<16 x i16>, <16 x i16>) nounwind readn
 
 define <32 x i8> @test_x86_avx2_packuswb_fold() {
 ; X86-AVX-LABEL: test_x86_avx2_packuswb_fold:
-; X86-AVX:       ## %bb.0:
+; X86-AVX:       # %bb.0:
 ; X86-AVX-NEXT:    vmovaps {{.*#+}} ymm0 = [0,255,255,0,0,0,0,0,0,0,0,0,0,0,0,0,0,255,255,0,0,0,0,0,0,0,0,0,0,0,0,0]
-; X86-AVX-NEXT:    ## encoding: [0xc5,0xfc,0x28,0x05,A,A,A,A]
-; X86-AVX-NEXT:    ## fixup A - offset: 4, value: LCPI5_0, kind: FK_Data_4
-; X86-AVX-NEXT:    retl ## encoding: [0xc3]
+; X86-AVX-NEXT:    # encoding: [0xc5,0xfc,0x28,0x05,A,A,A,A]
+; X86-AVX-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}, kind: FK_Data_4
+; X86-AVX-NEXT:    retl # encoding: [0xc3]
 ;
 ; X86-AVX512VL-LABEL: test_x86_avx2_packuswb_fold:
-; X86-AVX512VL:       ## %bb.0:
-; X86-AVX512VL-NEXT:    vmovaps LCPI5_0, %ymm0 ## EVEX TO VEX Compression ymm0 = [0,255,255,0,0,0,0,0,0,0,0,0,0,0,0,0,0,255,255,0,0,0,0,0,0,0,0,0,0,0,0,0]
-; X86-AVX512VL-NEXT:    ## encoding: [0xc5,0xfc,0x28,0x05,A,A,A,A]
-; X86-AVX512VL-NEXT:    ## fixup A - offset: 4, value: LCPI5_0, kind: FK_Data_4
-; X86-AVX512VL-NEXT:    retl ## encoding: [0xc3]
+; X86-AVX512VL:       # %bb.0:
+; X86-AVX512VL-NEXT:    vmovaps {{\.LCPI.*}}, %ymm0 # EVEX TO VEX Compression ymm0 = [0,255,255,0,0,0,0,0,0,0,0,0,0,0,0,0,0,255,255,0,0,0,0,0,0,0,0,0,0,0,0,0]
+; X86-AVX512VL-NEXT:    # encoding: [0xc5,0xfc,0x28,0x05,A,A,A,A]
+; X86-AVX512VL-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}, kind: FK_Data_4
+; X86-AVX512VL-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-AVX-LABEL: test_x86_avx2_packuswb_fold:
-; X64-AVX:       ## %bb.0:
+; X64-AVX:       # %bb.0:
 ; X64-AVX-NEXT:    vmovaps {{.*#+}} ymm0 = [0,255,255,0,0,0,0,0,0,0,0,0,0,0,0,0,0,255,255,0,0,0,0,0,0,0,0,0,0,0,0,0]
-; X64-AVX-NEXT:    ## encoding: [0xc5,0xfc,0x28,0x05,A,A,A,A]
-; X64-AVX-NEXT:    ## fixup A - offset: 4, value: LCPI5_0-4, kind: reloc_riprel_4byte
-; X64-AVX-NEXT:    retq ## encoding: [0xc3]
+; X64-AVX-NEXT:    # encoding: [0xc5,0xfc,0x28,0x05,A,A,A,A]
+; X64-AVX-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX-NEXT:    retq # encoding: [0xc3]
 ;
 ; X64-AVX512VL-LABEL: test_x86_avx2_packuswb_fold:
-; X64-AVX512VL:       ## %bb.0:
-; X64-AVX512VL-NEXT:    vmovaps {{.*}}(%rip), %ymm0 ## EVEX TO VEX Compression ymm0 = [0,255,255,0,0,0,0,0,0,0,0,0,0,0,0,0,0,255,255,0,0,0,0,0,0,0,0,0,0,0,0,0]
-; X64-AVX512VL-NEXT:    ## encoding: [0xc5,0xfc,0x28,0x05,A,A,A,A]
-; X64-AVX512VL-NEXT:    ## fixup A - offset: 4, value: LCPI5_0-4, kind: reloc_riprel_4byte
-; X64-AVX512VL-NEXT:    retq ## encoding: [0xc3]
+; X64-AVX512VL:       # %bb.0:
+; X64-AVX512VL-NEXT:    vmovaps {{.*}}(%rip), %ymm0 # EVEX TO VEX Compression ymm0 = [0,255,255,0,0,0,0,0,0,0,0,0,0,0,0,0,0,255,255,0,0,0,0,0,0,0,0,0,0,0,0,0]
+; X64-AVX512VL-NEXT:    # encoding: [0xc5,0xfc,0x28,0x05,A,A,A,A]
+; X64-AVX512VL-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX512VL-NEXT:    retq # encoding: [0xc3]
   %res = call <32 x i8> @llvm.x86.avx2.packuswb(<16 x i16> <i16 0, i16 255, i16 256, i16 65535, i16 -1, i16 -255, i16 -256, i16 -32678, i16 0, i16 255, i16 256, i16 65535, i16 -1, i16 -255, i16 -256, i16 -32678>, <16 x i16> zeroinitializer)
   ret <32 x i8> %res
 }
 
 
-define <32 x i8> @test_x86_avx2_padds_b(<32 x i8> %a0, <32 x i8> %a1) {
-; AVX2-LABEL: test_x86_avx2_padds_b:
-; AVX2:       ## %bb.0:
-; AVX2-NEXT:    vpaddsb %ymm1, %ymm0, %ymm0 ## encoding: [0xc5,0xfd,0xec,0xc1]
-; AVX2-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
-;
-; AVX512VL-LABEL: test_x86_avx2_padds_b:
-; AVX512VL:       ## %bb.0:
-; AVX512VL-NEXT:    vpaddsb %ymm1, %ymm0, %ymm0 ## EVEX TO VEX Compression encoding: [0xc5,0xfd,0xec,0xc1]
-; AVX512VL-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
-  %res = call <32 x i8> @llvm.x86.avx2.padds.b(<32 x i8> %a0, <32 x i8> %a1) ; <<32 x i8>> [#uses=1]
-  ret <32 x i8> %res
-}
-declare <32 x i8> @llvm.x86.avx2.padds.b(<32 x i8>, <32 x i8>) nounwind readnone
-
-
-define <16 x i16> @test_x86_avx2_padds_w(<16 x i16> %a0, <16 x i16> %a1) {
-; AVX2-LABEL: test_x86_avx2_padds_w:
-; AVX2:       ## %bb.0:
-; AVX2-NEXT:    vpaddsw %ymm1, %ymm0, %ymm0 ## encoding: [0xc5,0xfd,0xed,0xc1]
-; AVX2-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
-;
-; AVX512VL-LABEL: test_x86_avx2_padds_w:
-; AVX512VL:       ## %bb.0:
-; AVX512VL-NEXT:    vpaddsw %ymm1, %ymm0, %ymm0 ## EVEX TO VEX Compression encoding: [0xc5,0xfd,0xed,0xc1]
-; AVX512VL-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
-  %res = call <16 x i16> @llvm.x86.avx2.padds.w(<16 x i16> %a0, <16 x i16> %a1) ; <<16 x i16>> [#uses=1]
-  ret <16 x i16> %res
-}
-declare <16 x i16> @llvm.x86.avx2.padds.w(<16 x i16>, <16 x i16>) nounwind readnone
-
-
 define <8 x i32> @test_x86_avx2_pmadd_wd(<16 x i16> %a0, <16 x i16> %a1) {
 ; AVX2-LABEL: test_x86_avx2_pmadd_wd:
-; AVX2:       ## %bb.0:
-; AVX2-NEXT:    vpmaddwd %ymm1, %ymm0, %ymm0 ## encoding: [0xc5,0xfd,0xf5,0xc1]
-; AVX2-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpmaddwd %ymm1, %ymm0, %ymm0 # encoding: [0xc5,0xfd,0xf5,0xc1]
+; AVX2-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
 ;
 ; AVX512VL-LABEL: test_x86_avx2_pmadd_wd:
-; AVX512VL:       ## %bb.0:
-; AVX512VL-NEXT:    vpmaddwd %ymm1, %ymm0, %ymm0 ## EVEX TO VEX Compression encoding: [0xc5,0xfd,0xf5,0xc1]
-; AVX512VL-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpmaddwd %ymm1, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xf5,0xc1]
+; AVX512VL-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
   %res = call <8 x i32> @llvm.x86.avx2.pmadd.wd(<16 x i16> %a0, <16 x i16> %a1) ; <<8 x i32>> [#uses=1]
   ret <8 x i32> %res
 }
@@ -201,14 +169,14 @@ declare <8 x i32> @llvm.x86.avx2.pmadd.wd(<16 x i16>, <16 x i16>) nounwind readn
 
 define <16 x i16> @test_x86_avx2_pmaxs_w(<16 x i16> %a0, <16 x i16> %a1) {
 ; AVX2-LABEL: test_x86_avx2_pmaxs_w:
-; AVX2:       ## %bb.0:
-; AVX2-NEXT:    vpmaxsw %ymm1, %ymm0, %ymm0 ## encoding: [0xc5,0xfd,0xee,0xc1]
-; AVX2-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpmaxsw %ymm1, %ymm0, %ymm0 # encoding: [0xc5,0xfd,0xee,0xc1]
+; AVX2-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
 ;
 ; AVX512VL-LABEL: test_x86_avx2_pmaxs_w:
-; AVX512VL:       ## %bb.0:
-; AVX512VL-NEXT:    vpmaxsw %ymm1, %ymm0, %ymm0 ## EVEX TO VEX Compression encoding: [0xc5,0xfd,0xee,0xc1]
-; AVX512VL-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpmaxsw %ymm1, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xee,0xc1]
+; AVX512VL-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
   %res = call <16 x i16> @llvm.x86.avx2.pmaxs.w(<16 x i16> %a0, <16 x i16> %a1) ; <<16 x i16>> [#uses=1]
   ret <16 x i16> %res
 }
@@ -217,14 +185,14 @@ declare <16 x i16> @llvm.x86.avx2.pmaxs.w(<16 x i16>, <16 x i16>) nounwind readn
 
 define <32 x i8> @test_x86_avx2_pmaxu_b(<32 x i8> %a0, <32 x i8> %a1) {
 ; AVX2-LABEL: test_x86_avx2_pmaxu_b:
-; AVX2:       ## %bb.0:
-; AVX2-NEXT:    vpmaxub %ymm1, %ymm0, %ymm0 ## encoding: [0xc5,0xfd,0xde,0xc1]
-; AVX2-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpmaxub %ymm1, %ymm0, %ymm0 # encoding: [0xc5,0xfd,0xde,0xc1]
+; AVX2-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
 ;
 ; AVX512VL-LABEL: test_x86_avx2_pmaxu_b:
-; AVX512VL:       ## %bb.0:
-; AVX512VL-NEXT:    vpmaxub %ymm1, %ymm0, %ymm0 ## EVEX TO VEX Compression encoding: [0xc5,0xfd,0xde,0xc1]
-; AVX512VL-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpmaxub %ymm1, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xde,0xc1]
+; AVX512VL-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
   %res = call <32 x i8> @llvm.x86.avx2.pmaxu.b(<32 x i8> %a0, <32 x i8> %a1) ; <<32 x i8>> [#uses=1]
   ret <32 x i8> %res
 }
@@ -233,14 +201,14 @@ declare <32 x i8> @llvm.x86.avx2.pmaxu.b(<32 x i8>, <32 x i8>) nounwind readnone
 
 define <16 x i16> @test_x86_avx2_pmins_w(<16 x i16> %a0, <16 x i16> %a1) {
 ; AVX2-LABEL: test_x86_avx2_pmins_w:
-; AVX2:       ## %bb.0:
-; AVX2-NEXT:    vpminsw %ymm1, %ymm0, %ymm0 ## encoding: [0xc5,0xfd,0xea,0xc1]
-; AVX2-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpminsw %ymm1, %ymm0, %ymm0 # encoding: [0xc5,0xfd,0xea,0xc1]
+; AVX2-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
 ;
 ; AVX512VL-LABEL: test_x86_avx2_pmins_w:
-; AVX512VL:       ## %bb.0:
-; AVX512VL-NEXT:    vpminsw %ymm1, %ymm0, %ymm0 ## EVEX TO VEX Compression encoding: [0xc5,0xfd,0xea,0xc1]
-; AVX512VL-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpminsw %ymm1, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xea,0xc1]
+; AVX512VL-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
   %res = call <16 x i16> @llvm.x86.avx2.pmins.w(<16 x i16> %a0, <16 x i16> %a1) ; <<16 x i16>> [#uses=1]
   ret <16 x i16> %res
 }
@@ -249,14 +217,14 @@ declare <16 x i16> @llvm.x86.avx2.pmins.w(<16 x i16>, <16 x i16>) nounwind readn
 
 define <32 x i8> @test_x86_avx2_pminu_b(<32 x i8> %a0, <32 x i8> %a1) {
 ; AVX2-LABEL: test_x86_avx2_pminu_b:
-; AVX2:       ## %bb.0:
-; AVX2-NEXT:    vpminub %ymm1, %ymm0, %ymm0 ## encoding: [0xc5,0xfd,0xda,0xc1]
-; AVX2-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpminub %ymm1, %ymm0, %ymm0 # encoding: [0xc5,0xfd,0xda,0xc1]
+; AVX2-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
 ;
 ; AVX512VL-LABEL: test_x86_avx2_pminu_b:
-; AVX512VL:       ## %bb.0:
-; AVX512VL-NEXT:    vpminub %ymm1, %ymm0, %ymm0 ## EVEX TO VEX Compression encoding: [0xc5,0xfd,0xda,0xc1]
-; AVX512VL-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpminub %ymm1, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xda,0xc1]
+; AVX512VL-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
   %res = call <32 x i8> @llvm.x86.avx2.pminu.b(<32 x i8> %a0, <32 x i8> %a1) ; <<32 x i8>> [#uses=1]
   ret <32 x i8> %res
 }
@@ -265,10 +233,10 @@ declare <32 x i8> @llvm.x86.avx2.pminu.b(<32 x i8>, <32 x i8>) nounwind readnone
 
 define i32 @test_x86_avx2_pmovmskb(<32 x i8> %a0) {
 ; CHECK-LABEL: test_x86_avx2_pmovmskb:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    vpmovmskb %ymm0, %eax ## encoding: [0xc5,0xfd,0xd7,0xc0]
-; CHECK-NEXT:    vzeroupper ## encoding: [0xc5,0xf8,0x77]
-; CHECK-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpmovmskb %ymm0, %eax # encoding: [0xc5,0xfd,0xd7,0xc0]
+; CHECK-NEXT:    vzeroupper # encoding: [0xc5,0xf8,0x77]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
   %res = call i32 @llvm.x86.avx2.pmovmskb(<32 x i8> %a0) ; <i32> [#uses=1]
   ret i32 %res
 }
@@ -277,14 +245,14 @@ declare i32 @llvm.x86.avx2.pmovmskb(<32 x i8>) nounwind readnone
 
 define <16 x i16> @test_x86_avx2_pmulh_w(<16 x i16> %a0, <16 x i16> %a1) {
 ; AVX2-LABEL: test_x86_avx2_pmulh_w:
-; AVX2:       ## %bb.0:
-; AVX2-NEXT:    vpmulhw %ymm1, %ymm0, %ymm0 ## encoding: [0xc5,0xfd,0xe5,0xc1]
-; AVX2-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpmulhw %ymm1, %ymm0, %ymm0 # encoding: [0xc5,0xfd,0xe5,0xc1]
+; AVX2-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
 ;
 ; AVX512VL-LABEL: test_x86_avx2_pmulh_w:
-; AVX512VL:       ## %bb.0:
-; AVX512VL-NEXT:    vpmulhw %ymm1, %ymm0, %ymm0 ## EVEX TO VEX Compression encoding: [0xc5,0xfd,0xe5,0xc1]
-; AVX512VL-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpmulhw %ymm1, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xe5,0xc1]
+; AVX512VL-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
   %res = call <16 x i16> @llvm.x86.avx2.pmulh.w(<16 x i16> %a0, <16 x i16> %a1) ; <<16 x i16>> [#uses=1]
   ret <16 x i16> %res
 }
@@ -293,14 +261,14 @@ declare <16 x i16> @llvm.x86.avx2.pmulh.w(<16 x i16>, <16 x i16>) nounwind readn
 
 define <16 x i16> @test_x86_avx2_pmulhu_w(<16 x i16> %a0, <16 x i16> %a1) {
 ; AVX2-LABEL: test_x86_avx2_pmulhu_w:
-; AVX2:       ## %bb.0:
-; AVX2-NEXT:    vpmulhuw %ymm1, %ymm0, %ymm0 ## encoding: [0xc5,0xfd,0xe4,0xc1]
-; AVX2-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpmulhuw %ymm1, %ymm0, %ymm0 # encoding: [0xc5,0xfd,0xe4,0xc1]
+; AVX2-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
 ;
 ; AVX512VL-LABEL: test_x86_avx2_pmulhu_w:
-; AVX512VL:       ## %bb.0:
-; AVX512VL-NEXT:    vpmulhuw %ymm1, %ymm0, %ymm0 ## EVEX TO VEX Compression encoding: [0xc5,0xfd,0xe4,0xc1]
-; AVX512VL-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpmulhuw %ymm1, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xe4,0xc1]
+; AVX512VL-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
   %res = call <16 x i16> @llvm.x86.avx2.pmulhu.w(<16 x i16> %a0, <16 x i16> %a1) ; <<16 x i16>> [#uses=1]
   ret <16 x i16> %res
 }
@@ -309,14 +277,14 @@ declare <16 x i16> @llvm.x86.avx2.pmulhu.w(<16 x i16>, <16 x i16>) nounwind read
 
 define <4 x i64> @test_x86_avx2_psad_bw(<32 x i8> %a0, <32 x i8> %a1) {
 ; AVX2-LABEL: test_x86_avx2_psad_bw:
-; AVX2:       ## %bb.0:
-; AVX2-NEXT:    vpsadbw %ymm1, %ymm0, %ymm0 ## encoding: [0xc5,0xfd,0xf6,0xc1]
-; AVX2-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpsadbw %ymm1, %ymm0, %ymm0 # encoding: [0xc5,0xfd,0xf6,0xc1]
+; AVX2-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
 ;
 ; AVX512VL-LABEL: test_x86_avx2_psad_bw:
-; AVX512VL:       ## %bb.0:
-; AVX512VL-NEXT:    vpsadbw %ymm1, %ymm0, %ymm0 ## EVEX TO VEX Compression encoding: [0xc5,0xfd,0xf6,0xc1]
-; AVX512VL-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpsadbw %ymm1, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xf6,0xc1]
+; AVX512VL-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
   %res = call <4 x i64> @llvm.x86.avx2.psad.bw(<32 x i8> %a0, <32 x i8> %a1) ; <<4 x i64>> [#uses=1]
   ret <4 x i64> %res
 }
@@ -325,14 +293,14 @@ declare <4 x i64> @llvm.x86.avx2.psad.bw(<32 x i8>, <32 x i8>) nounwind readnone
 
 define <8 x i32> @test_x86_avx2_psll_d(<8 x i32> %a0, <4 x i32> %a1) {
 ; AVX2-LABEL: test_x86_avx2_psll_d:
-; AVX2:       ## %bb.0:
-; AVX2-NEXT:    vpslld %xmm1, %ymm0, %ymm0 ## encoding: [0xc5,0xfd,0xf2,0xc1]
-; AVX2-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpslld %xmm1, %ymm0, %ymm0 # encoding: [0xc5,0xfd,0xf2,0xc1]
+; AVX2-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
 ;
 ; AVX512VL-LABEL: test_x86_avx2_psll_d:
-; AVX512VL:       ## %bb.0:
-; AVX512VL-NEXT:    vpslld %xmm1, %ymm0, %ymm0 ## EVEX TO VEX Compression encoding: [0xc5,0xfd,0xf2,0xc1]
-; AVX512VL-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpslld %xmm1, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xf2,0xc1]
+; AVX512VL-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
   %res = call <8 x i32> @llvm.x86.avx2.psll.d(<8 x i32> %a0, <4 x i32> %a1) ; <<8 x i32>> [#uses=1]
   ret <8 x i32> %res
 }
@@ -341,14 +309,14 @@ declare <8 x i32> @llvm.x86.avx2.psll.d(<8 x i32>, <4 x i32>) nounwind readnone
 
 define <4 x i64> @test_x86_avx2_psll_q(<4 x i64> %a0, <2 x i64> %a1) {
 ; AVX2-LABEL: test_x86_avx2_psll_q:
-; AVX2:       ## %bb.0:
-; AVX2-NEXT:    vpsllq %xmm1, %ymm0, %ymm0 ## encoding: [0xc5,0xfd,0xf3,0xc1]
-; AVX2-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpsllq %xmm1, %ymm0, %ymm0 # encoding: [0xc5,0xfd,0xf3,0xc1]
+; AVX2-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
 ;
 ; AVX512VL-LABEL: test_x86_avx2_psll_q:
-; AVX512VL:       ## %bb.0:
-; AVX512VL-NEXT:    vpsllq %xmm1, %ymm0, %ymm0 ## EVEX TO VEX Compression encoding: [0xc5,0xfd,0xf3,0xc1]
-; AVX512VL-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpsllq %xmm1, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xf3,0xc1]
+; AVX512VL-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
   %res = call <4 x i64> @llvm.x86.avx2.psll.q(<4 x i64> %a0, <2 x i64> %a1) ; <<4 x i64>> [#uses=1]
   ret <4 x i64> %res
 }
@@ -357,14 +325,14 @@ declare <4 x i64> @llvm.x86.avx2.psll.q(<4 x i64>, <2 x i64>) nounwind readnone
 
 define <16 x i16> @test_x86_avx2_psll_w(<16 x i16> %a0, <8 x i16> %a1) {
 ; AVX2-LABEL: test_x86_avx2_psll_w:
-; AVX2:       ## %bb.0:
-; AVX2-NEXT:    vpsllw %xmm1, %ymm0, %ymm0 ## encoding: [0xc5,0xfd,0xf1,0xc1]
-; AVX2-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpsllw %xmm1, %ymm0, %ymm0 # encoding: [0xc5,0xfd,0xf1,0xc1]
+; AVX2-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
 ;
 ; AVX512VL-LABEL: test_x86_avx2_psll_w:
-; AVX512VL:       ## %bb.0:
-; AVX512VL-NEXT:    vpsllw %xmm1, %ymm0, %ymm0 ## EVEX TO VEX Compression encoding: [0xc5,0xfd,0xf1,0xc1]
-; AVX512VL-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpsllw %xmm1, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xf1,0xc1]
+; AVX512VL-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
   %res = call <16 x i16> @llvm.x86.avx2.psll.w(<16 x i16> %a0, <8 x i16> %a1) ; <<16 x i16>> [#uses=1]
   ret <16 x i16> %res
 }
@@ -373,14 +341,14 @@ declare <16 x i16> @llvm.x86.avx2.psll.w(<16 x i16>, <8 x i16>) nounwind readnon
 
 define <8 x i32> @test_x86_avx2_pslli_d(<8 x i32> %a0) {
 ; AVX2-LABEL: test_x86_avx2_pslli_d:
-; AVX2:       ## %bb.0:
-; AVX2-NEXT:    vpslld $7, %ymm0, %ymm0 ## encoding: [0xc5,0xfd,0x72,0xf0,0x07]
-; AVX2-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpslld $7, %ymm0, %ymm0 # encoding: [0xc5,0xfd,0x72,0xf0,0x07]
+; AVX2-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
 ;
 ; AVX512VL-LABEL: test_x86_avx2_pslli_d:
-; AVX512VL:       ## %bb.0:
-; AVX512VL-NEXT:    vpslld $7, %ymm0, %ymm0 ## EVEX TO VEX Compression encoding: [0xc5,0xfd,0x72,0xf0,0x07]
-; AVX512VL-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpslld $7, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x72,0xf0,0x07]
+; AVX512VL-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
   %res = call <8 x i32> @llvm.x86.avx2.pslli.d(<8 x i32> %a0, i32 7) ; <<8 x i32>> [#uses=1]
   ret <8 x i32> %res
 }
@@ -389,14 +357,14 @@ declare <8 x i32> @llvm.x86.avx2.pslli.d(<8 x i32>, i32) nounwind readnone
 
 define <4 x i64> @test_x86_avx2_pslli_q(<4 x i64> %a0) {
 ; AVX2-LABEL: test_x86_avx2_pslli_q:
-; AVX2:       ## %bb.0:
-; AVX2-NEXT:    vpsllq $7, %ymm0, %ymm0 ## encoding: [0xc5,0xfd,0x73,0xf0,0x07]
-; AVX2-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpsllq $7, %ymm0, %ymm0 # encoding: [0xc5,0xfd,0x73,0xf0,0x07]
+; AVX2-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
 ;
 ; AVX512VL-LABEL: test_x86_avx2_pslli_q:
-; AVX512VL:       ## %bb.0:
-; AVX512VL-NEXT:    vpsllq $7, %ymm0, %ymm0 ## EVEX TO VEX Compression encoding: [0xc5,0xfd,0x73,0xf0,0x07]
-; AVX512VL-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpsllq $7, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x73,0xf0,0x07]
+; AVX512VL-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
   %res = call <4 x i64> @llvm.x86.avx2.pslli.q(<4 x i64> %a0, i32 7) ; <<4 x i64>> [#uses=1]
   ret <4 x i64> %res
 }
@@ -405,14 +373,14 @@ declare <4 x i64> @llvm.x86.avx2.pslli.q(<4 x i64>, i32) nounwind readnone
 
 define <16 x i16> @test_x86_avx2_pslli_w(<16 x i16> %a0) {
 ; AVX2-LABEL: test_x86_avx2_pslli_w:
-; AVX2:       ## %bb.0:
-; AVX2-NEXT:    vpsllw $7, %ymm0, %ymm0 ## encoding: [0xc5,0xfd,0x71,0xf0,0x07]
-; AVX2-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpsllw $7, %ymm0, %ymm0 # encoding: [0xc5,0xfd,0x71,0xf0,0x07]
+; AVX2-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
 ;
 ; AVX512VL-LABEL: test_x86_avx2_pslli_w:
-; AVX512VL:       ## %bb.0:
-; AVX512VL-NEXT:    vpsllw $7, %ymm0, %ymm0 ## EVEX TO VEX Compression encoding: [0xc5,0xfd,0x71,0xf0,0x07]
-; AVX512VL-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpsllw $7, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x71,0xf0,0x07]
+; AVX512VL-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
   %res = call <16 x i16> @llvm.x86.avx2.pslli.w(<16 x i16> %a0, i32 7) ; <<16 x i16>> [#uses=1]
   ret <16 x i16> %res
 }
@@ -421,14 +389,14 @@ declare <16 x i16> @llvm.x86.avx2.pslli.w(<16 x i16>, i32) nounwind readnone
 
 define <8 x i32> @test_x86_avx2_psra_d(<8 x i32> %a0, <4 x i32> %a1) {
 ; AVX2-LABEL: test_x86_avx2_psra_d:
-; AVX2:       ## %bb.0:
-; AVX2-NEXT:    vpsrad %xmm1, %ymm0, %ymm0 ## encoding: [0xc5,0xfd,0xe2,0xc1]
-; AVX2-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpsrad %xmm1, %ymm0, %ymm0 # encoding: [0xc5,0xfd,0xe2,0xc1]
+; AVX2-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
 ;
 ; AVX512VL-LABEL: test_x86_avx2_psra_d:
-; AVX512VL:       ## %bb.0:
-; AVX512VL-NEXT:    vpsrad %xmm1, %ymm0, %ymm0 ## EVEX TO VEX Compression encoding: [0xc5,0xfd,0xe2,0xc1]
-; AVX512VL-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpsrad %xmm1, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xe2,0xc1]
+; AVX512VL-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
   %res = call <8 x i32> @llvm.x86.avx2.psra.d(<8 x i32> %a0, <4 x i32> %a1) ; <<8 x i32>> [#uses=1]
   ret <8 x i32> %res
 }
@@ -437,14 +405,14 @@ declare <8 x i32> @llvm.x86.avx2.psra.d(<8 x i32>, <4 x i32>) nounwind readnone
 
 define <16 x i16> @test_x86_avx2_psra_w(<16 x i16> %a0, <8 x i16> %a1) {
 ; AVX2-LABEL: test_x86_avx2_psra_w:
-; AVX2:       ## %bb.0:
-; AVX2-NEXT:    vpsraw %xmm1, %ymm0, %ymm0 ## encoding: [0xc5,0xfd,0xe1,0xc1]
-; AVX2-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpsraw %xmm1, %ymm0, %ymm0 # encoding: [0xc5,0xfd,0xe1,0xc1]
+; AVX2-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
 ;
 ; AVX512VL-LABEL: test_x86_avx2_psra_w:
-; AVX512VL:       ## %bb.0:
-; AVX512VL-NEXT:    vpsraw %xmm1, %ymm0, %ymm0 ## EVEX TO VEX Compression encoding: [0xc5,0xfd,0xe1,0xc1]
-; AVX512VL-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpsraw %xmm1, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xe1,0xc1]
+; AVX512VL-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
   %res = call <16 x i16> @llvm.x86.avx2.psra.w(<16 x i16> %a0, <8 x i16> %a1) ; <<16 x i16>> [#uses=1]
   ret <16 x i16> %res
 }
@@ -453,14 +421,14 @@ declare <16 x i16> @llvm.x86.avx2.psra.w(<16 x i16>, <8 x i16>) nounwind readnon
 
 define <8 x i32> @test_x86_avx2_psrai_d(<8 x i32> %a0) {
 ; AVX2-LABEL: test_x86_avx2_psrai_d:
-; AVX2:       ## %bb.0:
-; AVX2-NEXT:    vpsrad $7, %ymm0, %ymm0 ## encoding: [0xc5,0xfd,0x72,0xe0,0x07]
-; AVX2-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpsrad $7, %ymm0, %ymm0 # encoding: [0xc5,0xfd,0x72,0xe0,0x07]
+; AVX2-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
 ;
 ; AVX512VL-LABEL: test_x86_avx2_psrai_d:
-; AVX512VL:       ## %bb.0:
-; AVX512VL-NEXT:    vpsrad $7, %ymm0, %ymm0 ## EVEX TO VEX Compression encoding: [0xc5,0xfd,0x72,0xe0,0x07]
-; AVX512VL-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpsrad $7, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x72,0xe0,0x07]
+; AVX512VL-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
   %res = call <8 x i32> @llvm.x86.avx2.psrai.d(<8 x i32> %a0, i32 7) ; <<8 x i32>> [#uses=1]
   ret <8 x i32> %res
 }
@@ -469,14 +437,14 @@ declare <8 x i32> @llvm.x86.avx2.psrai.d(<8 x i32>, i32) nounwind readnone
 
 define <16 x i16> @test_x86_avx2_psrai_w(<16 x i16> %a0) {
 ; AVX2-LABEL: test_x86_avx2_psrai_w:
-; AVX2:       ## %bb.0:
-; AVX2-NEXT:    vpsraw $7, %ymm0, %ymm0 ## encoding: [0xc5,0xfd,0x71,0xe0,0x07]
-; AVX2-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpsraw $7, %ymm0, %ymm0 # encoding: [0xc5,0xfd,0x71,0xe0,0x07]
+; AVX2-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
 ;
 ; AVX512VL-LABEL: test_x86_avx2_psrai_w:
-; AVX512VL:       ## %bb.0:
-; AVX512VL-NEXT:    vpsraw $7, %ymm0, %ymm0 ## EVEX TO VEX Compression encoding: [0xc5,0xfd,0x71,0xe0,0x07]
-; AVX512VL-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpsraw $7, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x71,0xe0,0x07]
+; AVX512VL-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
   %res = call <16 x i16> @llvm.x86.avx2.psrai.w(<16 x i16> %a0, i32 7) ; <<16 x i16>> [#uses=1]
   ret <16 x i16> %res
 }
@@ -485,14 +453,14 @@ declare <16 x i16> @llvm.x86.avx2.psrai.w(<16 x i16>, i32) nounwind readnone
 
 define <8 x i32> @test_x86_avx2_psrl_d(<8 x i32> %a0, <4 x i32> %a1) {
 ; AVX2-LABEL: test_x86_avx2_psrl_d:
-; AVX2:       ## %bb.0:
-; AVX2-NEXT:    vpsrld %xmm1, %ymm0, %ymm0 ## encoding: [0xc5,0xfd,0xd2,0xc1]
-; AVX2-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpsrld %xmm1, %ymm0, %ymm0 # encoding: [0xc5,0xfd,0xd2,0xc1]
+; AVX2-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
 ;
 ; AVX512VL-LABEL: test_x86_avx2_psrl_d:
-; AVX512VL:       ## %bb.0:
-; AVX512VL-NEXT:    vpsrld %xmm1, %ymm0, %ymm0 ## EVEX TO VEX Compression encoding: [0xc5,0xfd,0xd2,0xc1]
-; AVX512VL-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpsrld %xmm1, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xd2,0xc1]
+; AVX512VL-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
   %res = call <8 x i32> @llvm.x86.avx2.psrl.d(<8 x i32> %a0, <4 x i32> %a1) ; <<8 x i32>> [#uses=1]
   ret <8 x i32> %res
 }
@@ -501,14 +469,14 @@ declare <8 x i32> @llvm.x86.avx2.psrl.d(<8 x i32>, <4 x i32>) nounwind readnone
 
 define <4 x i64> @test_x86_avx2_psrl_q(<4 x i64> %a0, <2 x i64> %a1) {
 ; AVX2-LABEL: test_x86_avx2_psrl_q:
-; AVX2:       ## %bb.0:
-; AVX2-NEXT:    vpsrlq %xmm1, %ymm0, %ymm0 ## encoding: [0xc5,0xfd,0xd3,0xc1]
-; AVX2-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpsrlq %xmm1, %ymm0, %ymm0 # encoding: [0xc5,0xfd,0xd3,0xc1]
+; AVX2-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
 ;
 ; AVX512VL-LABEL: test_x86_avx2_psrl_q:
-; AVX512VL:       ## %bb.0:
-; AVX512VL-NEXT:    vpsrlq %xmm1, %ymm0, %ymm0 ## EVEX TO VEX Compression encoding: [0xc5,0xfd,0xd3,0xc1]
-; AVX512VL-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpsrlq %xmm1, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xd3,0xc1]
+; AVX512VL-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
   %res = call <4 x i64> @llvm.x86.avx2.psrl.q(<4 x i64> %a0, <2 x i64> %a1) ; <<4 x i64>> [#uses=1]
   ret <4 x i64> %res
 }
@@ -517,14 +485,14 @@ declare <4 x i64> @llvm.x86.avx2.psrl.q(<4 x i64>, <2 x i64>) nounwind readnone
 
 define <16 x i16> @test_x86_avx2_psrl_w(<16 x i16> %a0, <8 x i16> %a1) {
 ; AVX2-LABEL: test_x86_avx2_psrl_w:
-; AVX2:       ## %bb.0:
-; AVX2-NEXT:    vpsrlw %xmm1, %ymm0, %ymm0 ## encoding: [0xc5,0xfd,0xd1,0xc1]
-; AVX2-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpsrlw %xmm1, %ymm0, %ymm0 # encoding: [0xc5,0xfd,0xd1,0xc1]
+; AVX2-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
 ;
 ; AVX512VL-LABEL: test_x86_avx2_psrl_w:
-; AVX512VL:       ## %bb.0:
-; AVX512VL-NEXT:    vpsrlw %xmm1, %ymm0, %ymm0 ## EVEX TO VEX Compression encoding: [0xc5,0xfd,0xd1,0xc1]
-; AVX512VL-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpsrlw %xmm1, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xd1,0xc1]
+; AVX512VL-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
   %res = call <16 x i16> @llvm.x86.avx2.psrl.w(<16 x i16> %a0, <8 x i16> %a1) ; <<16 x i16>> [#uses=1]
   ret <16 x i16> %res
 }
@@ -533,26 +501,26 @@ declare <16 x i16> @llvm.x86.avx2.psrl.w(<16 x i16>, <8 x i16>) nounwind readnon
 
 define <16 x i16> @test_x86_avx2_psrl_w_load(<16 x i16> %a0, <8 x i16>* %p) {
 ; X86-AVX-LABEL: test_x86_avx2_psrl_w_load:
-; X86-AVX:       ## %bb.0:
-; X86-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax ## encoding: [0x8b,0x44,0x24,0x04]
-; X86-AVX-NEXT:    vpsrlw (%eax), %ymm0, %ymm0 ## encoding: [0xc5,0xfd,0xd1,0x00]
-; X86-AVX-NEXT:    retl ## encoding: [0xc3]
+; X86-AVX:       # %bb.0:
+; X86-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-AVX-NEXT:    vpsrlw (%eax), %ymm0, %ymm0 # encoding: [0xc5,0xfd,0xd1,0x00]
+; X86-AVX-NEXT:    retl # encoding: [0xc3]
 ;
 ; X86-AVX512VL-LABEL: test_x86_avx2_psrl_w_load:
-; X86-AVX512VL:       ## %bb.0:
-; X86-AVX512VL-NEXT:    movl {{[0-9]+}}(%esp), %eax ## encoding: [0x8b,0x44,0x24,0x04]
-; X86-AVX512VL-NEXT:    vpsrlw (%eax), %ymm0, %ymm0 ## EVEX TO VEX Compression encoding: [0xc5,0xfd,0xd1,0x00]
-; X86-AVX512VL-NEXT:    retl ## encoding: [0xc3]
+; X86-AVX512VL:       # %bb.0:
+; X86-AVX512VL-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-AVX512VL-NEXT:    vpsrlw (%eax), %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xd1,0x00]
+; X86-AVX512VL-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-AVX-LABEL: test_x86_avx2_psrl_w_load:
-; X64-AVX:       ## %bb.0:
-; X64-AVX-NEXT:    vpsrlw (%rdi), %ymm0, %ymm0 ## encoding: [0xc5,0xfd,0xd1,0x07]
-; X64-AVX-NEXT:    retq ## encoding: [0xc3]
+; X64-AVX:       # %bb.0:
+; X64-AVX-NEXT:    vpsrlw (%rdi), %ymm0, %ymm0 # encoding: [0xc5,0xfd,0xd1,0x07]
+; X64-AVX-NEXT:    retq # encoding: [0xc3]
 ;
 ; X64-AVX512VL-LABEL: test_x86_avx2_psrl_w_load:
-; X64-AVX512VL:       ## %bb.0:
-; X64-AVX512VL-NEXT:    vpsrlw (%rdi), %ymm0, %ymm0 ## EVEX TO VEX Compression encoding: [0xc5,0xfd,0xd1,0x07]
-; X64-AVX512VL-NEXT:    retq ## encoding: [0xc3]
+; X64-AVX512VL:       # %bb.0:
+; X64-AVX512VL-NEXT:    vpsrlw (%rdi), %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xd1,0x07]
+; X64-AVX512VL-NEXT:    retq # encoding: [0xc3]
   %a1 = load <8 x i16>, <8 x i16>* %p
   %res = call <16 x i16> @llvm.x86.avx2.psrl.w(<16 x i16> %a0, <8 x i16> %a1) ; <<16 x i16>> [#uses=1]
   ret <16 x i16> %res
@@ -561,14 +529,14 @@ define <16 x i16> @test_x86_avx2_psrl_w_load(<16 x i16> %a0, <8 x i16>* %p) {
 
 define <8 x i32> @test_x86_avx2_psrli_d(<8 x i32> %a0) {
 ; AVX2-LABEL: test_x86_avx2_psrli_d:
-; AVX2:       ## %bb.0:
-; AVX2-NEXT:    vpsrld $7, %ymm0, %ymm0 ## encoding: [0xc5,0xfd,0x72,0xd0,0x07]
-; AVX2-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpsrld $7, %ymm0, %ymm0 # encoding: [0xc5,0xfd,0x72,0xd0,0x07]
+; AVX2-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
 ;
 ; AVX512VL-LABEL: test_x86_avx2_psrli_d:
-; AVX512VL:       ## %bb.0:
-; AVX512VL-NEXT:    vpsrld $7, %ymm0, %ymm0 ## EVEX TO VEX Compression encoding: [0xc5,0xfd,0x72,0xd0,0x07]
-; AVX512VL-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpsrld $7, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x72,0xd0,0x07]
+; AVX512VL-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
   %res = call <8 x i32> @llvm.x86.avx2.psrli.d(<8 x i32> %a0, i32 7) ; <<8 x i32>> [#uses=1]
   ret <8 x i32> %res
 }
@@ -577,14 +545,14 @@ declare <8 x i32> @llvm.x86.avx2.psrli.d(<8 x i32>, i32) nounwind readnone
 
 define <4 x i64> @test_x86_avx2_psrli_q(<4 x i64> %a0) {
 ; AVX2-LABEL: test_x86_avx2_psrli_q:
-; AVX2:       ## %bb.0:
-; AVX2-NEXT:    vpsrlq $7, %ymm0, %ymm0 ## encoding: [0xc5,0xfd,0x73,0xd0,0x07]
-; AVX2-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpsrlq $7, %ymm0, %ymm0 # encoding: [0xc5,0xfd,0x73,0xd0,0x07]
+; AVX2-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
 ;
 ; AVX512VL-LABEL: test_x86_avx2_psrli_q:
-; AVX512VL:       ## %bb.0:
-; AVX512VL-NEXT:    vpsrlq $7, %ymm0, %ymm0 ## EVEX TO VEX Compression encoding: [0xc5,0xfd,0x73,0xd0,0x07]
-; AVX512VL-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpsrlq $7, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x73,0xd0,0x07]
+; AVX512VL-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
   %res = call <4 x i64> @llvm.x86.avx2.psrli.q(<4 x i64> %a0, i32 7) ; <<4 x i64>> [#uses=1]
   ret <4 x i64> %res
 }
@@ -593,56 +561,25 @@ declare <4 x i64> @llvm.x86.avx2.psrli.q(<4 x i64>, i32) nounwind readnone
 
 define <16 x i16> @test_x86_avx2_psrli_w(<16 x i16> %a0) {
 ; AVX2-LABEL: test_x86_avx2_psrli_w:
-; AVX2:       ## %bb.0:
-; AVX2-NEXT:    vpsrlw $7, %ymm0, %ymm0 ## encoding: [0xc5,0xfd,0x71,0xd0,0x07]
-; AVX2-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpsrlw $7, %ymm0, %ymm0 # encoding: [0xc5,0xfd,0x71,0xd0,0x07]
+; AVX2-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
 ;
 ; AVX512VL-LABEL: test_x86_avx2_psrli_w:
-; AVX512VL:       ## %bb.0:
-; AVX512VL-NEXT:    vpsrlw $7, %ymm0, %ymm0 ## EVEX TO VEX Compression encoding: [0xc5,0xfd,0x71,0xd0,0x07]
-; AVX512VL-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpsrlw $7, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x71,0xd0,0x07]
+; AVX512VL-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
   %res = call <16 x i16> @llvm.x86.avx2.psrli.w(<16 x i16> %a0, i32 7) ; <<16 x i16>> [#uses=1]
   ret <16 x i16> %res
 }
 declare <16 x i16> @llvm.x86.avx2.psrli.w(<16 x i16>, i32) nounwind readnone
 
 
-define <32 x i8> @test_x86_avx2_psubs_b(<32 x i8> %a0, <32 x i8> %a1) {
-; AVX2-LABEL: test_x86_avx2_psubs_b:
-; AVX2:       ## %bb.0:
-; AVX2-NEXT:    vpsubsb %ymm1, %ymm0, %ymm0 ## encoding: [0xc5,0xfd,0xe8,0xc1]
-; AVX2-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
-;
-; AVX512VL-LABEL: test_x86_avx2_psubs_b:
-; AVX512VL:       ## %bb.0:
-; AVX512VL-NEXT:    vpsubsb %ymm1, %ymm0, %ymm0 ## EVEX TO VEX Compression encoding: [0xc5,0xfd,0xe8,0xc1]
-; AVX512VL-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
-  %res = call <32 x i8> @llvm.x86.avx2.psubs.b(<32 x i8> %a0, <32 x i8> %a1) ; <<32 x i8>> [#uses=1]
-  ret <32 x i8> %res
-}
-declare <32 x i8> @llvm.x86.avx2.psubs.b(<32 x i8>, <32 x i8>) nounwind readnone
-
-
-define <16 x i16> @test_x86_avx2_psubs_w(<16 x i16> %a0, <16 x i16> %a1) {
-; AVX2-LABEL: test_x86_avx2_psubs_w:
-; AVX2:       ## %bb.0:
-; AVX2-NEXT:    vpsubsw %ymm1, %ymm0, %ymm0 ## encoding: [0xc5,0xfd,0xe9,0xc1]
-; AVX2-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
-;
-; AVX512VL-LABEL: test_x86_avx2_psubs_w:
-; AVX512VL:       ## %bb.0:
-; AVX512VL-NEXT:    vpsubsw %ymm1, %ymm0, %ymm0 ## EVEX TO VEX Compression encoding: [0xc5,0xfd,0xe9,0xc1]
-; AVX512VL-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
-  %res = call <16 x i16> @llvm.x86.avx2.psubs.w(<16 x i16> %a0, <16 x i16> %a1) ; <<16 x i16>> [#uses=1]
-  ret <16 x i16> %res
-}
-declare <16 x i16> @llvm.x86.avx2.psubs.w(<16 x i16>, <16 x i16>) nounwind readnone
-
 define <8 x i32> @test_x86_avx2_phadd_d(<8 x i32> %a0, <8 x i32> %a1) {
 ; CHECK-LABEL: test_x86_avx2_phadd_d:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    vphaddd %ymm1, %ymm0, %ymm0 ## encoding: [0xc4,0xe2,0x7d,0x02,0xc1]
-; CHECK-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vphaddd %ymm1, %ymm0, %ymm0 # encoding: [0xc4,0xe2,0x7d,0x02,0xc1]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
   %res = call <8 x i32> @llvm.x86.avx2.phadd.d(<8 x i32> %a0, <8 x i32> %a1) ; <<8 x i32>> [#uses=1]
   ret <8 x i32> %res
 }
@@ -651,9 +588,9 @@ declare <8 x i32> @llvm.x86.avx2.phadd.d(<8 x i32>, <8 x i32>) nounwind readnone
 
 define <16 x i16> @test_x86_avx2_phadd_sw(<16 x i16> %a0, <16 x i16> %a1) {
 ; CHECK-LABEL: test_x86_avx2_phadd_sw:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    vphaddsw %ymm1, %ymm0, %ymm0 ## encoding: [0xc4,0xe2,0x7d,0x03,0xc1]
-; CHECK-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vphaddsw %ymm1, %ymm0, %ymm0 # encoding: [0xc4,0xe2,0x7d,0x03,0xc1]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
   %res = call <16 x i16> @llvm.x86.avx2.phadd.sw(<16 x i16> %a0, <16 x i16> %a1) ; <<16 x i16>> [#uses=1]
   ret <16 x i16> %res
 }
@@ -662,9 +599,9 @@ declare <16 x i16> @llvm.x86.avx2.phadd.sw(<16 x i16>, <16 x i16>) nounwind read
 
 define <16 x i16> @test_x86_avx2_phadd_w(<16 x i16> %a0, <16 x i16> %a1) {
 ; CHECK-LABEL: test_x86_avx2_phadd_w:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    vphaddw %ymm1, %ymm0, %ymm0 ## encoding: [0xc4,0xe2,0x7d,0x01,0xc1]
-; CHECK-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vphaddw %ymm1, %ymm0, %ymm0 # encoding: [0xc4,0xe2,0x7d,0x01,0xc1]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
   %res = call <16 x i16> @llvm.x86.avx2.phadd.w(<16 x i16> %a0, <16 x i16> %a1) ; <<16 x i16>> [#uses=1]
   ret <16 x i16> %res
 }
@@ -673,9 +610,9 @@ declare <16 x i16> @llvm.x86.avx2.phadd.w(<16 x i16>, <16 x i16>) nounwind readn
 
 define <8 x i32> @test_x86_avx2_phsub_d(<8 x i32> %a0, <8 x i32> %a1) {
 ; CHECK-LABEL: test_x86_avx2_phsub_d:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    vphsubd %ymm1, %ymm0, %ymm0 ## encoding: [0xc4,0xe2,0x7d,0x06,0xc1]
-; CHECK-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vphsubd %ymm1, %ymm0, %ymm0 # encoding: [0xc4,0xe2,0x7d,0x06,0xc1]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
   %res = call <8 x i32> @llvm.x86.avx2.phsub.d(<8 x i32> %a0, <8 x i32> %a1) ; <<8 x i32>> [#uses=1]
   ret <8 x i32> %res
 }
@@ -684,9 +621,9 @@ declare <8 x i32> @llvm.x86.avx2.phsub.d(<8 x i32>, <8 x i32>) nounwind readnone
 
 define <16 x i16> @test_x86_avx2_phsub_sw(<16 x i16> %a0, <16 x i16> %a1) {
 ; CHECK-LABEL: test_x86_avx2_phsub_sw:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    vphsubsw %ymm1, %ymm0, %ymm0 ## encoding: [0xc4,0xe2,0x7d,0x07,0xc1]
-; CHECK-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vphsubsw %ymm1, %ymm0, %ymm0 # encoding: [0xc4,0xe2,0x7d,0x07,0xc1]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
   %res = call <16 x i16> @llvm.x86.avx2.phsub.sw(<16 x i16> %a0, <16 x i16> %a1) ; <<16 x i16>> [#uses=1]
   ret <16 x i16> %res
 }
@@ -695,9 +632,9 @@ declare <16 x i16> @llvm.x86.avx2.phsub.sw(<16 x i16>, <16 x i16>) nounwind read
 
 define <16 x i16> @test_x86_avx2_phsub_w(<16 x i16> %a0, <16 x i16> %a1) {
 ; CHECK-LABEL: test_x86_avx2_phsub_w:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    vphsubw %ymm1, %ymm0, %ymm0 ## encoding: [0xc4,0xe2,0x7d,0x05,0xc1]
-; CHECK-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vphsubw %ymm1, %ymm0, %ymm0 # encoding: [0xc4,0xe2,0x7d,0x05,0xc1]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
   %res = call <16 x i16> @llvm.x86.avx2.phsub.w(<16 x i16> %a0, <16 x i16> %a1) ; <<16 x i16>> [#uses=1]
   ret <16 x i16> %res
 }
@@ -706,14 +643,14 @@ declare <16 x i16> @llvm.x86.avx2.phsub.w(<16 x i16>, <16 x i16>) nounwind readn
 
 define <16 x i16> @test_x86_avx2_pmadd_ub_sw(<32 x i8> %a0, <32 x i8> %a1) {
 ; AVX2-LABEL: test_x86_avx2_pmadd_ub_sw:
-; AVX2:       ## %bb.0:
-; AVX2-NEXT:    vpmaddubsw %ymm1, %ymm0, %ymm0 ## encoding: [0xc4,0xe2,0x7d,0x04,0xc1]
-; AVX2-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpmaddubsw %ymm1, %ymm0, %ymm0 # encoding: [0xc4,0xe2,0x7d,0x04,0xc1]
+; AVX2-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
 ;
 ; AVX512VL-LABEL: test_x86_avx2_pmadd_ub_sw:
-; AVX512VL:       ## %bb.0:
-; AVX512VL-NEXT:    vpmaddubsw %ymm1, %ymm0, %ymm0 ## EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7d,0x04,0xc1]
-; AVX512VL-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpmaddubsw %ymm1, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7d,0x04,0xc1]
+; AVX512VL-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
   %res = call <16 x i16> @llvm.x86.avx2.pmadd.ub.sw(<32 x i8> %a0, <32 x i8> %a1) ; <<16 x i16>> [#uses=1]
   ret <16 x i16> %res
 }
@@ -722,30 +659,30 @@ declare <16 x i16> @llvm.x86.avx2.pmadd.ub.sw(<32 x i8>, <32 x i8>) nounwind rea
 ; Make sure we don't commute this operation.
 define <16 x i16> @test_x86_avx2_pmadd_ub_sw_load_op0(<32 x i8>* %ptr, <32 x i8> %a1) {
 ; X86-AVX-LABEL: test_x86_avx2_pmadd_ub_sw_load_op0:
-; X86-AVX:       ## %bb.0:
-; X86-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax ## encoding: [0x8b,0x44,0x24,0x04]
-; X86-AVX-NEXT:    vmovdqa (%eax), %ymm1 ## encoding: [0xc5,0xfd,0x6f,0x08]
-; X86-AVX-NEXT:    vpmaddubsw %ymm0, %ymm1, %ymm0 ## encoding: [0xc4,0xe2,0x75,0x04,0xc0]
-; X86-AVX-NEXT:    retl ## encoding: [0xc3]
+; X86-AVX:       # %bb.0:
+; X86-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-AVX-NEXT:    vmovdqa (%eax), %ymm1 # encoding: [0xc5,0xfd,0x6f,0x08]
+; X86-AVX-NEXT:    vpmaddubsw %ymm0, %ymm1, %ymm0 # encoding: [0xc4,0xe2,0x75,0x04,0xc0]
+; X86-AVX-NEXT:    retl # encoding: [0xc3]
 ;
 ; X86-AVX512VL-LABEL: test_x86_avx2_pmadd_ub_sw_load_op0:
-; X86-AVX512VL:       ## %bb.0:
-; X86-AVX512VL-NEXT:    movl {{[0-9]+}}(%esp), %eax ## encoding: [0x8b,0x44,0x24,0x04]
-; X86-AVX512VL-NEXT:    vmovdqa (%eax), %ymm1 ## EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0x08]
-; X86-AVX512VL-NEXT:    vpmaddubsw %ymm0, %ymm1, %ymm0 ## EVEX TO VEX Compression encoding: [0xc4,0xe2,0x75,0x04,0xc0]
-; X86-AVX512VL-NEXT:    retl ## encoding: [0xc3]
+; X86-AVX512VL:       # %bb.0:
+; X86-AVX512VL-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-AVX512VL-NEXT:    vmovdqa (%eax), %ymm1 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0x08]
+; X86-AVX512VL-NEXT:    vpmaddubsw %ymm0, %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x75,0x04,0xc0]
+; X86-AVX512VL-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-AVX-LABEL: test_x86_avx2_pmadd_ub_sw_load_op0:
-; X64-AVX:       ## %bb.0:
-; X64-AVX-NEXT:    vmovdqa (%rdi), %ymm1 ## encoding: [0xc5,0xfd,0x6f,0x0f]
-; X64-AVX-NEXT:    vpmaddubsw %ymm0, %ymm1, %ymm0 ## encoding: [0xc4,0xe2,0x75,0x04,0xc0]
-; X64-AVX-NEXT:    retq ## encoding: [0xc3]
+; X64-AVX:       # %bb.0:
+; X64-AVX-NEXT:    vmovdqa (%rdi), %ymm1 # encoding: [0xc5,0xfd,0x6f,0x0f]
+; X64-AVX-NEXT:    vpmaddubsw %ymm0, %ymm1, %ymm0 # encoding: [0xc4,0xe2,0x75,0x04,0xc0]
+; X64-AVX-NEXT:    retq # encoding: [0xc3]
 ;
 ; X64-AVX512VL-LABEL: test_x86_avx2_pmadd_ub_sw_load_op0:
-; X64-AVX512VL:       ## %bb.0:
-; X64-AVX512VL-NEXT:    vmovdqa (%rdi), %ymm1 ## EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0x0f]
-; X64-AVX512VL-NEXT:    vpmaddubsw %ymm0, %ymm1, %ymm0 ## EVEX TO VEX Compression encoding: [0xc4,0xe2,0x75,0x04,0xc0]
-; X64-AVX512VL-NEXT:    retq ## encoding: [0xc3]
+; X64-AVX512VL:       # %bb.0:
+; X64-AVX512VL-NEXT:    vmovdqa (%rdi), %ymm1 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0x0f]
+; X64-AVX512VL-NEXT:    vpmaddubsw %ymm0, %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x75,0x04,0xc0]
+; X64-AVX512VL-NEXT:    retq # encoding: [0xc3]
   %a0 = load <32 x i8>, <32 x i8>* %ptr
   %res = call <16 x i16> @llvm.x86.avx2.pmadd.ub.sw(<32 x i8> %a0, <32 x i8> %a1) ; <<16 x i16>> [#uses=1]
   ret <16 x i16> %res
@@ -753,14 +690,14 @@ define <16 x i16> @test_x86_avx2_pmadd_ub_sw_load_op0(<32 x i8>* %ptr, <32 x i8>
 
 define <16 x i16> @test_x86_avx2_pmul_hr_sw(<16 x i16> %a0, <16 x i16> %a1) {
 ; AVX2-LABEL: test_x86_avx2_pmul_hr_sw:
-; AVX2:       ## %bb.0:
-; AVX2-NEXT:    vpmulhrsw %ymm1, %ymm0, %ymm0 ## encoding: [0xc4,0xe2,0x7d,0x0b,0xc1]
-; AVX2-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpmulhrsw %ymm1, %ymm0, %ymm0 # encoding: [0xc4,0xe2,0x7d,0x0b,0xc1]
+; AVX2-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
 ;
 ; AVX512VL-LABEL: test_x86_avx2_pmul_hr_sw:
-; AVX512VL:       ## %bb.0:
-; AVX512VL-NEXT:    vpmulhrsw %ymm1, %ymm0, %ymm0 ## EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7d,0x0b,0xc1]
-; AVX512VL-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpmulhrsw %ymm1, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7d,0x0b,0xc1]
+; AVX512VL-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
   %res = call <16 x i16> @llvm.x86.avx2.pmul.hr.sw(<16 x i16> %a0, <16 x i16> %a1) ; <<16 x i16>> [#uses=1]
   ret <16 x i16> %res
 }
@@ -769,14 +706,14 @@ declare <16 x i16> @llvm.x86.avx2.pmul.hr.sw(<16 x i16>, <16 x i16>) nounwind re
 
 define <32 x i8> @test_x86_avx2_pshuf_b(<32 x i8> %a0, <32 x i8> %a1) {
 ; AVX2-LABEL: test_x86_avx2_pshuf_b:
-; AVX2:       ## %bb.0:
-; AVX2-NEXT:    vpshufb %ymm1, %ymm0, %ymm0 ## encoding: [0xc4,0xe2,0x7d,0x00,0xc1]
-; AVX2-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpshufb %ymm1, %ymm0, %ymm0 # encoding: [0xc4,0xe2,0x7d,0x00,0xc1]
+; AVX2-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
 ;
 ; AVX512VL-LABEL: test_x86_avx2_pshuf_b:
-; AVX512VL:       ## %bb.0:
-; AVX512VL-NEXT:    vpshufb %ymm1, %ymm0, %ymm0 ## EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7d,0x00,0xc1]
-; AVX512VL-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpshufb %ymm1, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7d,0x00,0xc1]
+; AVX512VL-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
   %res = call <32 x i8> @llvm.x86.avx2.pshuf.b(<32 x i8> %a0, <32 x i8> %a1) ; <<16 x i8>> [#uses=1]
   ret <32 x i8> %res
 }
@@ -785,9 +722,9 @@ declare <32 x i8> @llvm.x86.avx2.pshuf.b(<32 x i8>, <32 x i8>) nounwind readnone
 
 define <32 x i8> @test_x86_avx2_psign_b(<32 x i8> %a0, <32 x i8> %a1) {
 ; CHECK-LABEL: test_x86_avx2_psign_b:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    vpsignb %ymm1, %ymm0, %ymm0 ## encoding: [0xc4,0xe2,0x7d,0x08,0xc1]
-; CHECK-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpsignb %ymm1, %ymm0, %ymm0 # encoding: [0xc4,0xe2,0x7d,0x08,0xc1]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
   %res = call <32 x i8> @llvm.x86.avx2.psign.b(<32 x i8> %a0, <32 x i8> %a1) ; <<32 x i8>> [#uses=1]
   ret <32 x i8> %res
 }
@@ -796,9 +733,9 @@ declare <32 x i8> @llvm.x86.avx2.psign.b(<32 x i8>, <32 x i8>) nounwind readnone
 
 define <8 x i32> @test_x86_avx2_psign_d(<8 x i32> %a0, <8 x i32> %a1) {
 ; CHECK-LABEL: test_x86_avx2_psign_d:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    vpsignd %ymm1, %ymm0, %ymm0 ## encoding: [0xc4,0xe2,0x7d,0x0a,0xc1]
-; CHECK-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpsignd %ymm1, %ymm0, %ymm0 # encoding: [0xc4,0xe2,0x7d,0x0a,0xc1]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
   %res = call <8 x i32> @llvm.x86.avx2.psign.d(<8 x i32> %a0, <8 x i32> %a1) ; <<4 x i32>> [#uses=1]
   ret <8 x i32> %res
 }
@@ -807,9 +744,9 @@ declare <8 x i32> @llvm.x86.avx2.psign.d(<8 x i32>, <8 x i32>) nounwind readnone
 
 define <16 x i16> @test_x86_avx2_psign_w(<16 x i16> %a0, <16 x i16> %a1) {
 ; CHECK-LABEL: test_x86_avx2_psign_w:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    vpsignw %ymm1, %ymm0, %ymm0 ## encoding: [0xc4,0xe2,0x7d,0x09,0xc1]
-; CHECK-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpsignw %ymm1, %ymm0, %ymm0 # encoding: [0xc4,0xe2,0x7d,0x09,0xc1]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
   %res = call <16 x i16> @llvm.x86.avx2.psign.w(<16 x i16> %a0, <16 x i16> %a1) ; <<16 x i16>> [#uses=1]
   ret <16 x i16> %res
 }
@@ -818,9 +755,9 @@ declare <16 x i16> @llvm.x86.avx2.psign.w(<16 x i16>, <16 x i16>) nounwind readn
 
 define <16 x i16> @test_x86_avx2_mpsadbw(<32 x i8> %a0, <32 x i8> %a1) {
 ; CHECK-LABEL: test_x86_avx2_mpsadbw:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    vmpsadbw $7, %ymm1, %ymm0, %ymm0 ## encoding: [0xc4,0xe3,0x7d,0x42,0xc1,0x07]
-; CHECK-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmpsadbw $7, %ymm1, %ymm0, %ymm0 # encoding: [0xc4,0xe3,0x7d,0x42,0xc1,0x07]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
   %res = call <16 x i16> @llvm.x86.avx2.mpsadbw(<32 x i8> %a0, <32 x i8> %a1, i8 7) ; <<16 x i16>> [#uses=1]
   ret <16 x i16> %res
 }
@@ -829,14 +766,14 @@ declare <16 x i16> @llvm.x86.avx2.mpsadbw(<32 x i8>, <32 x i8>, i8) nounwind rea
 
 define <16 x i16> @test_x86_avx2_packusdw(<8 x i32> %a0, <8 x i32> %a1) {
 ; AVX2-LABEL: test_x86_avx2_packusdw:
-; AVX2:       ## %bb.0:
-; AVX2-NEXT:    vpackusdw %ymm1, %ymm0, %ymm0 ## encoding: [0xc4,0xe2,0x7d,0x2b,0xc1]
-; AVX2-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpackusdw %ymm1, %ymm0, %ymm0 # encoding: [0xc4,0xe2,0x7d,0x2b,0xc1]
+; AVX2-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
 ;
 ; AVX512VL-LABEL: test_x86_avx2_packusdw:
-; AVX512VL:       ## %bb.0:
-; AVX512VL-NEXT:    vpackusdw %ymm1, %ymm0, %ymm0 ## EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7d,0x2b,0xc1]
-; AVX512VL-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpackusdw %ymm1, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7d,0x2b,0xc1]
+; AVX512VL-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
   %res = call <16 x i16> @llvm.x86.avx2.packusdw(<8 x i32> %a0, <8 x i32> %a1) ; <<16 x i16>> [#uses=1]
   ret <16 x i16> %res
 }
@@ -845,32 +782,32 @@ declare <16 x i16> @llvm.x86.avx2.packusdw(<8 x i32>, <8 x i32>) nounwind readno
 
 define <16 x i16> @test_x86_avx2_packusdw_fold() {
 ; X86-AVX-LABEL: test_x86_avx2_packusdw_fold:
-; X86-AVX:       ## %bb.0:
+; X86-AVX:       # %bb.0:
 ; X86-AVX-NEXT:    vmovaps {{.*#+}} ymm0 = [0,0,0,0,255,32767,65535,0,0,0,0,0,0,0,0,0]
-; X86-AVX-NEXT:    ## encoding: [0xc5,0xfc,0x28,0x05,A,A,A,A]
-; X86-AVX-NEXT:    ## fixup A - offset: 4, value: LCPI51_0, kind: FK_Data_4
-; X86-AVX-NEXT:    retl ## encoding: [0xc3]
+; X86-AVX-NEXT:    # encoding: [0xc5,0xfc,0x28,0x05,A,A,A,A]
+; X86-AVX-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}, kind: FK_Data_4
+; X86-AVX-NEXT:    retl # encoding: [0xc3]
 ;
 ; X86-AVX512VL-LABEL: test_x86_avx2_packusdw_fold:
-; X86-AVX512VL:       ## %bb.0:
-; X86-AVX512VL-NEXT:    vmovaps LCPI51_0, %ymm0 ## EVEX TO VEX Compression ymm0 = [0,0,0,0,255,32767,65535,0,0,0,0,0,0,0,0,0]
-; X86-AVX512VL-NEXT:    ## encoding: [0xc5,0xfc,0x28,0x05,A,A,A,A]
-; X86-AVX512VL-NEXT:    ## fixup A - offset: 4, value: LCPI51_0, kind: FK_Data_4
-; X86-AVX512VL-NEXT:    retl ## encoding: [0xc3]
+; X86-AVX512VL:       # %bb.0:
+; X86-AVX512VL-NEXT:    vmovaps {{\.LCPI.*}}, %ymm0 # EVEX TO VEX Compression ymm0 = [0,0,0,0,255,32767,65535,0,0,0,0,0,0,0,0,0]
+; X86-AVX512VL-NEXT:    # encoding: [0xc5,0xfc,0x28,0x05,A,A,A,A]
+; X86-AVX512VL-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}, kind: FK_Data_4
+; X86-AVX512VL-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-AVX-LABEL: test_x86_avx2_packusdw_fold:
-; X64-AVX:       ## %bb.0:
+; X64-AVX:       # %bb.0:
 ; X64-AVX-NEXT:    vmovaps {{.*#+}} ymm0 = [0,0,0,0,255,32767,65535,0,0,0,0,0,0,0,0,0]
-; X64-AVX-NEXT:    ## encoding: [0xc5,0xfc,0x28,0x05,A,A,A,A]
-; X64-AVX-NEXT:    ## fixup A - offset: 4, value: LCPI51_0-4, kind: reloc_riprel_4byte
-; X64-AVX-NEXT:    retq ## encoding: [0xc3]
+; X64-AVX-NEXT:    # encoding: [0xc5,0xfc,0x28,0x05,A,A,A,A]
+; X64-AVX-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX-NEXT:    retq # encoding: [0xc3]
 ;
 ; X64-AVX512VL-LABEL: test_x86_avx2_packusdw_fold:
-; X64-AVX512VL:       ## %bb.0:
-; X64-AVX512VL-NEXT:    vmovaps {{.*}}(%rip), %ymm0 ## EVEX TO VEX Compression ymm0 = [0,0,0,0,255,32767,65535,0,0,0,0,0,0,0,0,0]
-; X64-AVX512VL-NEXT:    ## encoding: [0xc5,0xfc,0x28,0x05,A,A,A,A]
-; X64-AVX512VL-NEXT:    ## fixup A - offset: 4, value: LCPI51_0-4, kind: reloc_riprel_4byte
-; X64-AVX512VL-NEXT:    retq ## encoding: [0xc3]
+; X64-AVX512VL:       # %bb.0:
+; X64-AVX512VL-NEXT:    vmovaps {{.*}}(%rip), %ymm0 # EVEX TO VEX Compression ymm0 = [0,0,0,0,255,32767,65535,0,0,0,0,0,0,0,0,0]
+; X64-AVX512VL-NEXT:    # encoding: [0xc5,0xfc,0x28,0x05,A,A,A,A]
+; X64-AVX512VL-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX512VL-NEXT:    retq # encoding: [0xc3]
   %res = call <16 x i16> @llvm.x86.avx2.packusdw(<8 x i32> zeroinitializer, <8 x i32> <i32 255, i32 32767, i32 65535, i32 -1, i32 -32767, i32 -65535, i32 0, i32 -256>)
   ret <16 x i16> %res
 }
@@ -878,9 +815,9 @@ define <16 x i16> @test_x86_avx2_packusdw_fold() {
 
 define <32 x i8> @test_x86_avx2_pblendvb(<32 x i8> %a0, <32 x i8> %a1, <32 x i8> %a2) {
 ; CHECK-LABEL: test_x86_avx2_pblendvb:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    vpblendvb %ymm2, %ymm1, %ymm0, %ymm0 ## encoding: [0xc4,0xe3,0x7d,0x4c,0xc1,0x20]
-; CHECK-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpblendvb %ymm2, %ymm1, %ymm0, %ymm0 # encoding: [0xc4,0xe3,0x7d,0x4c,0xc1,0x20]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
   %res = call <32 x i8> @llvm.x86.avx2.pblendvb(<32 x i8> %a0, <32 x i8> %a1, <32 x i8> %a2) ; <<32 x i8>> [#uses=1]
   ret <32 x i8> %res
 }
@@ -889,10 +826,10 @@ declare <32 x i8> @llvm.x86.avx2.pblendvb(<32 x i8>, <32 x i8>, <32 x i8>) nounw
 
 define <16 x i16> @test_x86_avx2_pblendw(<16 x i16> %a0, <16 x i16> %a1) {
 ; CHECK-LABEL: test_x86_avx2_pblendw:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    vpblendw $7, %ymm1, %ymm0, %ymm0 ## encoding: [0xc4,0xe3,0x7d,0x0e,0xc1,0x07]
-; CHECK-NEXT:    ## ymm0 = ymm1[0,1,2],ymm0[3,4,5,6,7],ymm1[8,9,10],ymm0[11,12,13,14,15]
-; CHECK-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpblendw $7, %ymm1, %ymm0, %ymm0 # encoding: [0xc4,0xe3,0x7d,0x0e,0xc1,0x07]
+; CHECK-NEXT:    # ymm0 = ymm1[0,1,2],ymm0[3,4,5,6,7],ymm1[8,9,10],ymm0[11,12,13,14,15]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
   %res = call <16 x i16> @llvm.x86.avx2.pblendw(<16 x i16> %a0, <16 x i16> %a1, i8 7) ; <<16 x i16>> [#uses=1]
   ret <16 x i16> %res
 }
@@ -901,14 +838,14 @@ declare <16 x i16> @llvm.x86.avx2.pblendw(<16 x i16>, <16 x i16>, i8) nounwind r
 
 define <32 x i8> @test_x86_avx2_pmaxsb(<32 x i8> %a0, <32 x i8> %a1) {
 ; AVX2-LABEL: test_x86_avx2_pmaxsb:
-; AVX2:       ## %bb.0:
-; AVX2-NEXT:    vpmaxsb %ymm1, %ymm0, %ymm0 ## encoding: [0xc4,0xe2,0x7d,0x3c,0xc1]
-; AVX2-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpmaxsb %ymm1, %ymm0, %ymm0 # encoding: [0xc4,0xe2,0x7d,0x3c,0xc1]
+; AVX2-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
 ;
 ; AVX512VL-LABEL: test_x86_avx2_pmaxsb:
-; AVX512VL:       ## %bb.0:
-; AVX512VL-NEXT:    vpmaxsb %ymm1, %ymm0, %ymm0 ## EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7d,0x3c,0xc1]
-; AVX512VL-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpmaxsb %ymm1, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7d,0x3c,0xc1]
+; AVX512VL-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
   %res = call <32 x i8> @llvm.x86.avx2.pmaxs.b(<32 x i8> %a0, <32 x i8> %a1) ; <<32 x i8>> [#uses=1]
   ret <32 x i8> %res
 }
@@ -917,14 +854,14 @@ declare <32 x i8> @llvm.x86.avx2.pmaxs.b(<32 x i8>, <32 x i8>) nounwind readnone
 
 define <8 x i32> @test_x86_avx2_pmaxsd(<8 x i32> %a0, <8 x i32> %a1) {
 ; AVX2-LABEL: test_x86_avx2_pmaxsd:
-; AVX2:       ## %bb.0:
-; AVX2-NEXT:    vpmaxsd %ymm1, %ymm0, %ymm0 ## encoding: [0xc4,0xe2,0x7d,0x3d,0xc1]
-; AVX2-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpmaxsd %ymm1, %ymm0, %ymm0 # encoding: [0xc4,0xe2,0x7d,0x3d,0xc1]
+; AVX2-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
 ;
 ; AVX512VL-LABEL: test_x86_avx2_pmaxsd:
-; AVX512VL:       ## %bb.0:
-; AVX512VL-NEXT:    vpmaxsd %ymm1, %ymm0, %ymm0 ## EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7d,0x3d,0xc1]
-; AVX512VL-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpmaxsd %ymm1, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7d,0x3d,0xc1]
+; AVX512VL-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
   %res = call <8 x i32> @llvm.x86.avx2.pmaxs.d(<8 x i32> %a0, <8 x i32> %a1) ; <<8 x i32>> [#uses=1]
   ret <8 x i32> %res
 }
@@ -933,14 +870,14 @@ declare <8 x i32> @llvm.x86.avx2.pmaxs.d(<8 x i32>, <8 x i32>) nounwind readnone
 
 define <8 x i32> @test_x86_avx2_pmaxud(<8 x i32> %a0, <8 x i32> %a1) {
 ; AVX2-LABEL: test_x86_avx2_pmaxud:
-; AVX2:       ## %bb.0:
-; AVX2-NEXT:    vpmaxud %ymm1, %ymm0, %ymm0 ## encoding: [0xc4,0xe2,0x7d,0x3f,0xc1]
-; AVX2-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpmaxud %ymm1, %ymm0, %ymm0 # encoding: [0xc4,0xe2,0x7d,0x3f,0xc1]
+; AVX2-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
 ;
 ; AVX512VL-LABEL: test_x86_avx2_pmaxud:
-; AVX512VL:       ## %bb.0:
-; AVX512VL-NEXT:    vpmaxud %ymm1, %ymm0, %ymm0 ## EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7d,0x3f,0xc1]
-; AVX512VL-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpmaxud %ymm1, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7d,0x3f,0xc1]
+; AVX512VL-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
   %res = call <8 x i32> @llvm.x86.avx2.pmaxu.d(<8 x i32> %a0, <8 x i32> %a1) ; <<8 x i32>> [#uses=1]
   ret <8 x i32> %res
 }
@@ -949,14 +886,14 @@ declare <8 x i32> @llvm.x86.avx2.pmaxu.d(<8 x i32>, <8 x i32>) nounwind readnone
 
 define <16 x i16> @test_x86_avx2_pmaxuw(<16 x i16> %a0, <16 x i16> %a1) {
 ; AVX2-LABEL: test_x86_avx2_pmaxuw:
-; AVX2:       ## %bb.0:
-; AVX2-NEXT:    vpmaxuw %ymm1, %ymm0, %ymm0 ## encoding: [0xc4,0xe2,0x7d,0x3e,0xc1]
-; AVX2-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpmaxuw %ymm1, %ymm0, %ymm0 # encoding: [0xc4,0xe2,0x7d,0x3e,0xc1]
+; AVX2-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
 ;
 ; AVX512VL-LABEL: test_x86_avx2_pmaxuw:
-; AVX512VL:       ## %bb.0:
-; AVX512VL-NEXT:    vpmaxuw %ymm1, %ymm0, %ymm0 ## EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7d,0x3e,0xc1]
-; AVX512VL-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpmaxuw %ymm1, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7d,0x3e,0xc1]
+; AVX512VL-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
   %res = call <16 x i16> @llvm.x86.avx2.pmaxu.w(<16 x i16> %a0, <16 x i16> %a1) ; <<16 x i16>> [#uses=1]
   ret <16 x i16> %res
 }
@@ -965,14 +902,14 @@ declare <16 x i16> @llvm.x86.avx2.pmaxu.w(<16 x i16>, <16 x i16>) nounwind readn
 
 define <32 x i8> @test_x86_avx2_pminsb(<32 x i8> %a0, <32 x i8> %a1) {
 ; AVX2-LABEL: test_x86_avx2_pminsb:
-; AVX2:       ## %bb.0:
-; AVX2-NEXT:    vpminsb %ymm1, %ymm0, %ymm0 ## encoding: [0xc4,0xe2,0x7d,0x38,0xc1]
-; AVX2-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpminsb %ymm1, %ymm0, %ymm0 # encoding: [0xc4,0xe2,0x7d,0x38,0xc1]
+; AVX2-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
 ;
 ; AVX512VL-LABEL: test_x86_avx2_pminsb:
-; AVX512VL:       ## %bb.0:
-; AVX512VL-NEXT:    vpminsb %ymm1, %ymm0, %ymm0 ## EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7d,0x38,0xc1]
-; AVX512VL-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpminsb %ymm1, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7d,0x38,0xc1]
+; AVX512VL-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
   %res = call <32 x i8> @llvm.x86.avx2.pmins.b(<32 x i8> %a0, <32 x i8> %a1) ; <<32 x i8>> [#uses=1]
   ret <32 x i8> %res
 }
@@ -981,14 +918,14 @@ declare <32 x i8> @llvm.x86.avx2.pmins.b(<32 x i8>, <32 x i8>) nounwind readnone
 
 define <8 x i32> @test_x86_avx2_pminsd(<8 x i32> %a0, <8 x i32> %a1) {
 ; AVX2-LABEL: test_x86_avx2_pminsd:
-; AVX2:       ## %bb.0:
-; AVX2-NEXT:    vpminsd %ymm1, %ymm0, %ymm0 ## encoding: [0xc4,0xe2,0x7d,0x39,0xc1]
-; AVX2-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpminsd %ymm1, %ymm0, %ymm0 # encoding: [0xc4,0xe2,0x7d,0x39,0xc1]
+; AVX2-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
 ;
 ; AVX512VL-LABEL: test_x86_avx2_pminsd:
-; AVX512VL:       ## %bb.0:
-; AVX512VL-NEXT:    vpminsd %ymm1, %ymm0, %ymm0 ## EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7d,0x39,0xc1]
-; AVX512VL-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpminsd %ymm1, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7d,0x39,0xc1]
+; AVX512VL-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
   %res = call <8 x i32> @llvm.x86.avx2.pmins.d(<8 x i32> %a0, <8 x i32> %a1) ; <<8 x i32>> [#uses=1]
   ret <8 x i32> %res
 }
@@ -997,14 +934,14 @@ declare <8 x i32> @llvm.x86.avx2.pmins.d(<8 x i32>, <8 x i32>) nounwind readnone
 
 define <8 x i32> @test_x86_avx2_pminud(<8 x i32> %a0, <8 x i32> %a1) {
 ; AVX2-LABEL: test_x86_avx2_pminud:
-; AVX2:       ## %bb.0:
-; AVX2-NEXT:    vpminud %ymm1, %ymm0, %ymm0 ## encoding: [0xc4,0xe2,0x7d,0x3b,0xc1]
-; AVX2-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpminud %ymm1, %ymm0, %ymm0 # encoding: [0xc4,0xe2,0x7d,0x3b,0xc1]
+; AVX2-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
 ;
 ; AVX512VL-LABEL: test_x86_avx2_pminud:
-; AVX512VL:       ## %bb.0:
-; AVX512VL-NEXT:    vpminud %ymm1, %ymm0, %ymm0 ## EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7d,0x3b,0xc1]
-; AVX512VL-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpminud %ymm1, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7d,0x3b,0xc1]
+; AVX512VL-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
   %res = call <8 x i32> @llvm.x86.avx2.pminu.d(<8 x i32> %a0, <8 x i32> %a1) ; <<8 x i32>> [#uses=1]
   ret <8 x i32> %res
 }
@@ -1013,14 +950,14 @@ declare <8 x i32> @llvm.x86.avx2.pminu.d(<8 x i32>, <8 x i32>) nounwind readnone
 
 define <16 x i16> @test_x86_avx2_pminuw(<16 x i16> %a0, <16 x i16> %a1) {
 ; AVX2-LABEL: test_x86_avx2_pminuw:
-; AVX2:       ## %bb.0:
-; AVX2-NEXT:    vpminuw %ymm1, %ymm0, %ymm0 ## encoding: [0xc4,0xe2,0x7d,0x3a,0xc1]
-; AVX2-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpminuw %ymm1, %ymm0, %ymm0 # encoding: [0xc4,0xe2,0x7d,0x3a,0xc1]
+; AVX2-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
 ;
 ; AVX512VL-LABEL: test_x86_avx2_pminuw:
-; AVX512VL:       ## %bb.0:
-; AVX512VL-NEXT:    vpminuw %ymm1, %ymm0, %ymm0 ## EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7d,0x3a,0xc1]
-; AVX512VL-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpminuw %ymm1, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7d,0x3a,0xc1]
+; AVX512VL-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
   %res = call <16 x i16> @llvm.x86.avx2.pminu.w(<16 x i16> %a0, <16 x i16> %a1) ; <<16 x i16>> [#uses=1]
   ret <16 x i16> %res
 }
@@ -1029,10 +966,10 @@ declare <16 x i16> @llvm.x86.avx2.pminu.w(<16 x i16>, <16 x i16>) nounwind readn
 
 define <4 x i32> @test_x86_avx2_pblendd_128(<4 x i32> %a0, <4 x i32> %a1) {
 ; CHECK-LABEL: test_x86_avx2_pblendd_128:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    vblendps $8, %xmm0, %xmm1, %xmm0 ## encoding: [0xc4,0xe3,0x71,0x0c,0xc0,0x08]
-; CHECK-NEXT:    ## xmm0 = xmm1[0,1,2],xmm0[3]
-; CHECK-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vblendps $8, %xmm0, %xmm1, %xmm0 # encoding: [0xc4,0xe3,0x71,0x0c,0xc0,0x08]
+; CHECK-NEXT:    # xmm0 = xmm1[0,1,2],xmm0[3]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
   %res = call <4 x i32> @llvm.x86.avx2.pblendd.128(<4 x i32> %a0, <4 x i32> %a1, i8 7) ; <<4 x i32>> [#uses=1]
   ret <4 x i32> %res
 }
@@ -1041,10 +978,10 @@ declare <4 x i32> @llvm.x86.avx2.pblendd.128(<4 x i32>, <4 x i32>, i8) nounwind
 
 define <8 x i32> @test_x86_avx2_pblendd_256(<8 x i32> %a0, <8 x i32> %a1) {
 ; CHECK-LABEL: test_x86_avx2_pblendd_256:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    vblendps $7, %ymm1, %ymm0, %ymm0 ## encoding: [0xc4,0xe3,0x7d,0x0c,0xc1,0x07]
-; CHECK-NEXT:    ## ymm0 = ymm1[0,1,2],ymm0[3,4,5,6,7]
-; CHECK-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vblendps $7, %ymm1, %ymm0, %ymm0 # encoding: [0xc4,0xe3,0x7d,0x0c,0xc1,0x07]
+; CHECK-NEXT:    # ymm0 = ymm1[0,1,2],ymm0[3,4,5,6,7]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
   %res = call <8 x i32> @llvm.x86.avx2.pblendd.256(<8 x i32> %a0, <8 x i32> %a1, i8 7) ; <<8 x i32>> [#uses=1]
   ret <8 x i32> %res
 }
@@ -1056,14 +993,14 @@ declare <8 x i32> @llvm.x86.avx2.pblendd.256(<8 x i32>, <8 x i32>, i8) nounwind
 ; the instruction.
 define <8 x i32> @test_x86_avx2_permd(<8 x i32> %a0, <8 x i32> %a1) {
 ; AVX2-LABEL: test_x86_avx2_permd:
-; AVX2:       ## %bb.0:
-; AVX2-NEXT:    vpermps %ymm0, %ymm1, %ymm0 ## encoding: [0xc4,0xe2,0x75,0x16,0xc0]
-; AVX2-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpermps %ymm0, %ymm1, %ymm0 # encoding: [0xc4,0xe2,0x75,0x16,0xc0]
+; AVX2-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
 ;
 ; AVX512VL-LABEL: test_x86_avx2_permd:
-; AVX512VL:       ## %bb.0:
-; AVX512VL-NEXT:    vpermps %ymm0, %ymm1, %ymm0 ## EVEX TO VEX Compression encoding: [0xc4,0xe2,0x75,0x16,0xc0]
-; AVX512VL-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpermps %ymm0, %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x75,0x16,0xc0]
+; AVX512VL-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
   %res = call <8 x i32> @llvm.x86.avx2.permd(<8 x i32> %a0, <8 x i32> %a1) ; <<8 x i32>> [#uses=1]
   ret <8 x i32> %res
 }
@@ -1075,14 +1012,14 @@ declare <8 x i32> @llvm.x86.avx2.permd(<8 x i32>, <8 x i32>) nounwind readonly
 ; the instruction.
 define <8 x float> @test_x86_avx2_permps(<8 x float> %a0, <8 x i32> %a1) {
 ; AVX2-LABEL: test_x86_avx2_permps:
-; AVX2:       ## %bb.0:
-; AVX2-NEXT:    vpermps %ymm0, %ymm1, %ymm0 ## encoding: [0xc4,0xe2,0x75,0x16,0xc0]
-; AVX2-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpermps %ymm0, %ymm1, %ymm0 # encoding: [0xc4,0xe2,0x75,0x16,0xc0]
+; AVX2-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
 ;
 ; AVX512VL-LABEL: test_x86_avx2_permps:
-; AVX512VL:       ## %bb.0:
-; AVX512VL-NEXT:    vpermps %ymm0, %ymm1, %ymm0 ## EVEX TO VEX Compression encoding: [0xc4,0xe2,0x75,0x16,0xc0]
-; AVX512VL-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpermps %ymm0, %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x75,0x16,0xc0]
+; AVX512VL-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
   %res = call <8 x float> @llvm.x86.avx2.permps(<8 x float> %a0, <8 x i32> %a1) ; <<8 x float>> [#uses=1]
   ret <8 x float> %res
 }
@@ -1091,15 +1028,15 @@ declare <8 x float> @llvm.x86.avx2.permps(<8 x float>, <8 x i32>) nounwind reado
 
 define <2 x i64> @test_x86_avx2_maskload_q(i8* %a0, <2 x i64> %a1) {
 ; X86-LABEL: test_x86_avx2_maskload_q:
-; X86:       ## %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax ## encoding: [0x8b,0x44,0x24,0x04]
-; X86-NEXT:    vpmaskmovq (%eax), %xmm0, %xmm0 ## encoding: [0xc4,0xe2,0xf9,0x8c,0x00]
-; X86-NEXT:    retl ## encoding: [0xc3]
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-NEXT:    vpmaskmovq (%eax), %xmm0, %xmm0 # encoding: [0xc4,0xe2,0xf9,0x8c,0x00]
+; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_x86_avx2_maskload_q:
-; X64:       ## %bb.0:
-; X64-NEXT:    vpmaskmovq (%rdi), %xmm0, %xmm0 ## encoding: [0xc4,0xe2,0xf9,0x8c,0x07]
-; X64-NEXT:    retq ## encoding: [0xc3]
+; X64:       # %bb.0:
+; X64-NEXT:    vpmaskmovq (%rdi), %xmm0, %xmm0 # encoding: [0xc4,0xe2,0xf9,0x8c,0x07]
+; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <2 x i64> @llvm.x86.avx2.maskload.q(i8* %a0, <2 x i64> %a1) ; <<2 x i64>> [#uses=1]
   ret <2 x i64> %res
 }
@@ -1108,15 +1045,15 @@ declare <2 x i64> @llvm.x86.avx2.maskload.q(i8*, <2 x i64>) nounwind readonly
 
 define <4 x i64> @test_x86_avx2_maskload_q_256(i8* %a0, <4 x i64> %a1) {
 ; X86-LABEL: test_x86_avx2_maskload_q_256:
-; X86:       ## %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax ## encoding: [0x8b,0x44,0x24,0x04]
-; X86-NEXT:    vpmaskmovq (%eax), %ymm0, %ymm0 ## encoding: [0xc4,0xe2,0xfd,0x8c,0x00]
-; X86-NEXT:    retl ## encoding: [0xc3]
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-NEXT:    vpmaskmovq (%eax), %ymm0, %ymm0 # encoding: [0xc4,0xe2,0xfd,0x8c,0x00]
+; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_x86_avx2_maskload_q_256:
-; X64:       ## %bb.0:
-; X64-NEXT:    vpmaskmovq (%rdi), %ymm0, %ymm0 ## encoding: [0xc4,0xe2,0xfd,0x8c,0x07]
-; X64-NEXT:    retq ## encoding: [0xc3]
+; X64:       # %bb.0:
+; X64-NEXT:    vpmaskmovq (%rdi), %ymm0, %ymm0 # encoding: [0xc4,0xe2,0xfd,0x8c,0x07]
+; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <4 x i64> @llvm.x86.avx2.maskload.q.256(i8* %a0, <4 x i64> %a1) ; <<4 x i64>> [#uses=1]
   ret <4 x i64> %res
 }
@@ -1125,15 +1062,15 @@ declare <4 x i64> @llvm.x86.avx2.maskload.q.256(i8*, <4 x i64>) nounwind readonl
 
 define <4 x i32> @test_x86_avx2_maskload_d(i8* %a0, <4 x i32> %a1) {
 ; X86-LABEL: test_x86_avx2_maskload_d:
-; X86:       ## %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax ## encoding: [0x8b,0x44,0x24,0x04]
-; X86-NEXT:    vpmaskmovd (%eax), %xmm0, %xmm0 ## encoding: [0xc4,0xe2,0x79,0x8c,0x00]
-; X86-NEXT:    retl ## encoding: [0xc3]
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-NEXT:    vpmaskmovd (%eax), %xmm0, %xmm0 # encoding: [0xc4,0xe2,0x79,0x8c,0x00]
+; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_x86_avx2_maskload_d:
-; X64:       ## %bb.0:
-; X64-NEXT:    vpmaskmovd (%rdi), %xmm0, %xmm0 ## encoding: [0xc4,0xe2,0x79,0x8c,0x07]
-; X64-NEXT:    retq ## encoding: [0xc3]
+; X64:       # %bb.0:
+; X64-NEXT:    vpmaskmovd (%rdi), %xmm0, %xmm0 # encoding: [0xc4,0xe2,0x79,0x8c,0x07]
+; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <4 x i32> @llvm.x86.avx2.maskload.d(i8* %a0, <4 x i32> %a1) ; <<4 x i32>> [#uses=1]
   ret <4 x i32> %res
 }
@@ -1142,15 +1079,15 @@ declare <4 x i32> @llvm.x86.avx2.maskload.d(i8*, <4 x i32>) nounwind readonly
 
 define <8 x i32> @test_x86_avx2_maskload_d_256(i8* %a0, <8 x i32> %a1) {
 ; X86-LABEL: test_x86_avx2_maskload_d_256:
-; X86:       ## %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax ## encoding: [0x8b,0x44,0x24,0x04]
-; X86-NEXT:    vpmaskmovd (%eax), %ymm0, %ymm0 ## encoding: [0xc4,0xe2,0x7d,0x8c,0x00]
-; X86-NEXT:    retl ## encoding: [0xc3]
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-NEXT:    vpmaskmovd (%eax), %ymm0, %ymm0 # encoding: [0xc4,0xe2,0x7d,0x8c,0x00]
+; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_x86_avx2_maskload_d_256:
-; X64:       ## %bb.0:
-; X64-NEXT:    vpmaskmovd (%rdi), %ymm0, %ymm0 ## encoding: [0xc4,0xe2,0x7d,0x8c,0x07]
-; X64-NEXT:    retq ## encoding: [0xc3]
+; X64:       # %bb.0:
+; X64-NEXT:    vpmaskmovd (%rdi), %ymm0, %ymm0 # encoding: [0xc4,0xe2,0x7d,0x8c,0x07]
+; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <8 x i32> @llvm.x86.avx2.maskload.d.256(i8* %a0, <8 x i32> %a1) ; <<8 x i32>> [#uses=1]
   ret <8 x i32> %res
 }
@@ -1159,15 +1096,15 @@ declare <8 x i32> @llvm.x86.avx2.maskload.d.256(i8*, <8 x i32>) nounwind readonl
 
 define void @test_x86_avx2_maskstore_q(i8* %a0, <2 x i64> %a1, <2 x i64> %a2) {
 ; X86-LABEL: test_x86_avx2_maskstore_q:
-; X86:       ## %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax ## encoding: [0x8b,0x44,0x24,0x04]
-; X86-NEXT:    vpmaskmovq %xmm1, %xmm0, (%eax) ## encoding: [0xc4,0xe2,0xf9,0x8e,0x08]
-; X86-NEXT:    retl ## encoding: [0xc3]
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-NEXT:    vpmaskmovq %xmm1, %xmm0, (%eax) # encoding: [0xc4,0xe2,0xf9,0x8e,0x08]
+; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_x86_avx2_maskstore_q:
-; X64:       ## %bb.0:
-; X64-NEXT:    vpmaskmovq %xmm1, %xmm0, (%rdi) ## encoding: [0xc4,0xe2,0xf9,0x8e,0x0f]
-; X64-NEXT:    retq ## encoding: [0xc3]
+; X64:       # %bb.0:
+; X64-NEXT:    vpmaskmovq %xmm1, %xmm0, (%rdi) # encoding: [0xc4,0xe2,0xf9,0x8e,0x0f]
+; X64-NEXT:    retq # encoding: [0xc3]
   call void @llvm.x86.avx2.maskstore.q(i8* %a0, <2 x i64> %a1, <2 x i64> %a2)
   ret void
 }
@@ -1176,17 +1113,17 @@ declare void @llvm.x86.avx2.maskstore.q(i8*, <2 x i64>, <2 x i64>) nounwind
 
 define void @test_x86_avx2_maskstore_q_256(i8* %a0, <4 x i64> %a1, <4 x i64> %a2) {
 ; X86-LABEL: test_x86_avx2_maskstore_q_256:
-; X86:       ## %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax ## encoding: [0x8b,0x44,0x24,0x04]
-; X86-NEXT:    vpmaskmovq %ymm1, %ymm0, (%eax) ## encoding: [0xc4,0xe2,0xfd,0x8e,0x08]
-; X86-NEXT:    vzeroupper ## encoding: [0xc5,0xf8,0x77]
-; X86-NEXT:    retl ## encoding: [0xc3]
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-NEXT:    vpmaskmovq %ymm1, %ymm0, (%eax) # encoding: [0xc4,0xe2,0xfd,0x8e,0x08]
+; X86-NEXT:    vzeroupper # encoding: [0xc5,0xf8,0x77]
+; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_x86_avx2_maskstore_q_256:
-; X64:       ## %bb.0:
-; X64-NEXT:    vpmaskmovq %ymm1, %ymm0, (%rdi) ## encoding: [0xc4,0xe2,0xfd,0x8e,0x0f]
-; X64-NEXT:    vzeroupper ## encoding: [0xc5,0xf8,0x77]
-; X64-NEXT:    retq ## encoding: [0xc3]
+; X64:       # %bb.0:
+; X64-NEXT:    vpmaskmovq %ymm1, %ymm0, (%rdi) # encoding: [0xc4,0xe2,0xfd,0x8e,0x0f]
+; X64-NEXT:    vzeroupper # encoding: [0xc5,0xf8,0x77]
+; X64-NEXT:    retq # encoding: [0xc3]
   call void @llvm.x86.avx2.maskstore.q.256(i8* %a0, <4 x i64> %a1, <4 x i64> %a2)
   ret void
 }
@@ -1195,15 +1132,15 @@ declare void @llvm.x86.avx2.maskstore.q.256(i8*, <4 x i64>, <4 x i64>) nounwind
 
 define void @test_x86_avx2_maskstore_d(i8* %a0, <4 x i32> %a1, <4 x i32> %a2) {
 ; X86-LABEL: test_x86_avx2_maskstore_d:
-; X86:       ## %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax ## encoding: [0x8b,0x44,0x24,0x04]
-; X86-NEXT:    vpmaskmovd %xmm1, %xmm0, (%eax) ## encoding: [0xc4,0xe2,0x79,0x8e,0x08]
-; X86-NEXT:    retl ## encoding: [0xc3]
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-NEXT:    vpmaskmovd %xmm1, %xmm0, (%eax) # encoding: [0xc4,0xe2,0x79,0x8e,0x08]
+; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_x86_avx2_maskstore_d:
-; X64:       ## %bb.0:
-; X64-NEXT:    vpmaskmovd %xmm1, %xmm0, (%rdi) ## encoding: [0xc4,0xe2,0x79,0x8e,0x0f]
-; X64-NEXT:    retq ## encoding: [0xc3]
+; X64:       # %bb.0:
+; X64-NEXT:    vpmaskmovd %xmm1, %xmm0, (%rdi) # encoding: [0xc4,0xe2,0x79,0x8e,0x0f]
+; X64-NEXT:    retq # encoding: [0xc3]
   call void @llvm.x86.avx2.maskstore.d(i8* %a0, <4 x i32> %a1, <4 x i32> %a2)
   ret void
 }
@@ -1212,17 +1149,17 @@ declare void @llvm.x86.avx2.maskstore.d(i8*, <4 x i32>, <4 x i32>) nounwind
 
 define void @test_x86_avx2_maskstore_d_256(i8* %a0, <8 x i32> %a1, <8 x i32> %a2) {
 ; X86-LABEL: test_x86_avx2_maskstore_d_256:
-; X86:       ## %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax ## encoding: [0x8b,0x44,0x24,0x04]
-; X86-NEXT:    vpmaskmovd %ymm1, %ymm0, (%eax) ## encoding: [0xc4,0xe2,0x7d,0x8e,0x08]
-; X86-NEXT:    vzeroupper ## encoding: [0xc5,0xf8,0x77]
-; X86-NEXT:    retl ## encoding: [0xc3]
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-NEXT:    vpmaskmovd %ymm1, %ymm0, (%eax) # encoding: [0xc4,0xe2,0x7d,0x8e,0x08]
+; X86-NEXT:    vzeroupper # encoding: [0xc5,0xf8,0x77]
+; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_x86_avx2_maskstore_d_256:
-; X64:       ## %bb.0:
-; X64-NEXT:    vpmaskmovd %ymm1, %ymm0, (%rdi) ## encoding: [0xc4,0xe2,0x7d,0x8e,0x0f]
-; X64-NEXT:    vzeroupper ## encoding: [0xc5,0xf8,0x77]
-; X64-NEXT:    retq ## encoding: [0xc3]
+; X64:       # %bb.0:
+; X64-NEXT:    vpmaskmovd %ymm1, %ymm0, (%rdi) # encoding: [0xc4,0xe2,0x7d,0x8e,0x0f]
+; X64-NEXT:    vzeroupper # encoding: [0xc5,0xf8,0x77]
+; X64-NEXT:    retq # encoding: [0xc3]
   call void @llvm.x86.avx2.maskstore.d.256(i8* %a0, <8 x i32> %a1, <8 x i32> %a2)
   ret void
 }
@@ -1231,182 +1168,602 @@ declare void @llvm.x86.avx2.maskstore.d.256(i8*, <8 x i32>, <8 x i32>) nounwind
 
 define <4 x i32> @test_x86_avx2_psllv_d(<4 x i32> %a0, <4 x i32> %a1) {
 ; AVX2-LABEL: test_x86_avx2_psllv_d:
-; AVX2:       ## %bb.0:
-; AVX2-NEXT:    vpsllvd %xmm1, %xmm0, %xmm0 ## encoding: [0xc4,0xe2,0x79,0x47,0xc1]
-; AVX2-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpsllvd %xmm1, %xmm0, %xmm0 # encoding: [0xc4,0xe2,0x79,0x47,0xc1]
+; AVX2-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
 ;
 ; AVX512VL-LABEL: test_x86_avx2_psllv_d:
-; AVX512VL:       ## %bb.0:
-; AVX512VL-NEXT:    vpsllvd %xmm1, %xmm0, %xmm0 ## EVEX TO VEX Compression encoding: [0xc4,0xe2,0x79,0x47,0xc1]
-; AVX512VL-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpsllvd %xmm1, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x79,0x47,0xc1]
+; AVX512VL-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
   %res = call <4 x i32> @llvm.x86.avx2.psllv.d(<4 x i32> %a0, <4 x i32> %a1) ; <<4 x i32>> [#uses=1]
   ret <4 x i32> %res
 }
+
+define <4 x i32> @test_x86_avx2_psllv_d_const() {
+; X86-AVX-LABEL: test_x86_avx2_psllv_d_const:
+; X86-AVX:       # %bb.0:
+; X86-AVX-NEXT:    vmovdqa {{.*#+}} xmm0 = [2,9,0,4294967295]
+; X86-AVX-NEXT:    # encoding: [0xc5,0xf9,0x6f,0x05,A,A,A,A]
+; X86-AVX-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}, kind: FK_Data_4
+; X86-AVX-NEXT:    vpsllvd {{\.LCPI.*}}, %xmm0, %xmm0 # encoding: [0xc4,0xe2,0x79,0x47,0x05,A,A,A,A]
+; X86-AVX-NEXT:    # fixup A - offset: 5, value: {{\.LCPI.*}}, kind: FK_Data_4
+; X86-AVX-NEXT:    vmovdqa {{.*#+}} xmm1 = [1,1,1,4294967295]
+; X86-AVX-NEXT:    # encoding: [0xc5,0xf9,0x6f,0x0d,A,A,A,A]
+; X86-AVX-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}, kind: FK_Data_4
+; X86-AVX-NEXT:    vpsllvd %xmm1, %xmm1, %xmm1 # encoding: [0xc4,0xe2,0x71,0x47,0xc9]
+; X86-AVX-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xfe,0xc1]
+; X86-AVX-NEXT:    retl # encoding: [0xc3]
+;
+; X86-AVX512VL-LABEL: test_x86_avx2_psllv_d_const:
+; X86-AVX512VL:       # %bb.0:
+; X86-AVX512VL-NEXT:    vmovdqa {{\.LCPI.*}}, %xmm0 # EVEX TO VEX Compression xmm0 = [2,9,0,4294967295]
+; X86-AVX512VL-NEXT:    # encoding: [0xc5,0xf9,0x6f,0x05,A,A,A,A]
+; X86-AVX512VL-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}, kind: FK_Data_4
+; X86-AVX512VL-NEXT:    vpsllvd {{\.LCPI.*}}, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x79,0x47,0x05,A,A,A,A]
+; X86-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.LCPI.*}}, kind: FK_Data_4
+; X86-AVX512VL-NEXT:    vmovdqa {{\.LCPI.*}}, %xmm1 # EVEX TO VEX Compression xmm1 = [1,1,1,4294967295]
+; X86-AVX512VL-NEXT:    # encoding: [0xc5,0xf9,0x6f,0x0d,A,A,A,A]
+; X86-AVX512VL-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}, kind: FK_Data_4
+; X86-AVX512VL-NEXT:    vpsllvd %xmm1, %xmm1, %xmm1 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x71,0x47,0xc9]
+; X86-AVX512VL-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfe,0xc1]
+; X86-AVX512VL-NEXT:    retl # encoding: [0xc3]
+;
+; X64-AVX-LABEL: test_x86_avx2_psllv_d_const:
+; X64-AVX:       # %bb.0:
+; X64-AVX-NEXT:    vmovdqa {{.*#+}} xmm0 = [2,9,0,4294967295]
+; X64-AVX-NEXT:    # encoding: [0xc5,0xf9,0x6f,0x05,A,A,A,A]
+; X64-AVX-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX-NEXT:    vpsllvd {{.*}}(%rip), %xmm0, %xmm0 # encoding: [0xc4,0xe2,0x79,0x47,0x05,A,A,A,A]
+; X64-AVX-NEXT:    # fixup A - offset: 5, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX-NEXT:    vmovdqa {{.*#+}} xmm1 = [1,1,1,4294967295]
+; X64-AVX-NEXT:    # encoding: [0xc5,0xf9,0x6f,0x0d,A,A,A,A]
+; X64-AVX-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX-NEXT:    vpsllvd %xmm1, %xmm1, %xmm1 # encoding: [0xc4,0xe2,0x71,0x47,0xc9]
+; X64-AVX-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xfe,0xc1]
+; X64-AVX-NEXT:    retq # encoding: [0xc3]
+;
+; X64-AVX512VL-LABEL: test_x86_avx2_psllv_d_const:
+; X64-AVX512VL:       # %bb.0:
+; X64-AVX512VL-NEXT:    vmovdqa {{.*}}(%rip), %xmm0 # EVEX TO VEX Compression xmm0 = [2,9,0,4294967295]
+; X64-AVX512VL-NEXT:    # encoding: [0xc5,0xf9,0x6f,0x05,A,A,A,A]
+; X64-AVX512VL-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX512VL-NEXT:    vpsllvd {{.*}}(%rip), %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x79,0x47,0x05,A,A,A,A]
+; X64-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX512VL-NEXT:    vmovdqa {{.*}}(%rip), %xmm1 # EVEX TO VEX Compression xmm1 = [1,1,1,4294967295]
+; X64-AVX512VL-NEXT:    # encoding: [0xc5,0xf9,0x6f,0x0d,A,A,A,A]
+; X64-AVX512VL-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX512VL-NEXT:    vpsllvd %xmm1, %xmm1, %xmm1 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x71,0x47,0xc9]
+; X64-AVX512VL-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfe,0xc1]
+; X64-AVX512VL-NEXT:    retq # encoding: [0xc3]
+  %res0 = call <4 x i32> @llvm.x86.avx2.psllv.d(<4 x i32> <i32 2, i32 9, i32 0, i32 -1>, <4 x i32> <i32 1, i32 0, i32 33, i32 -1>)
+  %res1 = call <4 x i32> @llvm.x86.avx2.psllv.d(<4 x i32> <i32 1, i32 1, i32 1, i32 -1>, <4 x i32> <i32 1, i32 1, i32 1,  i32 -1>)
+  %res2 = add <4 x i32> %res0, %res1
+  ret <4 x i32> %res2
+}
 declare <4 x i32> @llvm.x86.avx2.psllv.d(<4 x i32>, <4 x i32>) nounwind readnone
 
 
 define <8 x i32> @test_x86_avx2_psllv_d_256(<8 x i32> %a0, <8 x i32> %a1) {
 ; AVX2-LABEL: test_x86_avx2_psllv_d_256:
-; AVX2:       ## %bb.0:
-; AVX2-NEXT:    vpsllvd %ymm1, %ymm0, %ymm0 ## encoding: [0xc4,0xe2,0x7d,0x47,0xc1]
-; AVX2-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpsllvd %ymm1, %ymm0, %ymm0 # encoding: [0xc4,0xe2,0x7d,0x47,0xc1]
+; AVX2-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
 ;
 ; AVX512VL-LABEL: test_x86_avx2_psllv_d_256:
-; AVX512VL:       ## %bb.0:
-; AVX512VL-NEXT:    vpsllvd %ymm1, %ymm0, %ymm0 ## EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7d,0x47,0xc1]
-; AVX512VL-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpsllvd %ymm1, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7d,0x47,0xc1]
+; AVX512VL-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
   %res = call <8 x i32> @llvm.x86.avx2.psllv.d.256(<8 x i32> %a0, <8 x i32> %a1) ; <<8 x i32>> [#uses=1]
   ret <8 x i32> %res
 }
+
+define <8 x i32> @test_x86_avx2_psllv_d_256_const() {
+; X86-AVX-LABEL: test_x86_avx2_psllv_d_256_const:
+; X86-AVX:       # %bb.0:
+; X86-AVX-NEXT:    vmovdqa {{.*#+}} ymm0 = [2,9,0,4294967295,3,7,4294967295,0]
+; X86-AVX-NEXT:    # encoding: [0xc5,0xfd,0x6f,0x05,A,A,A,A]
+; X86-AVX-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}, kind: FK_Data_4
+; X86-AVX-NEXT:    vpsllvd {{\.LCPI.*}}, %ymm0, %ymm0 # encoding: [0xc4,0xe2,0x7d,0x47,0x05,A,A,A,A]
+; X86-AVX-NEXT:    # fixup A - offset: 5, value: {{\.LCPI.*}}, kind: FK_Data_4
+; X86-AVX-NEXT:    vmovdqa {{.*#+}} ymm1 = [4,4,4,4,4,4,4,4294967295]
+; X86-AVX-NEXT:    # encoding: [0xc5,0xfd,0x6f,0x0d,A,A,A,A]
+; X86-AVX-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}, kind: FK_Data_4
+; X86-AVX-NEXT:    vpsllvd {{\.LCPI.*}}, %ymm1, %ymm1 # encoding: [0xc4,0xe2,0x75,0x47,0x0d,A,A,A,A]
+; X86-AVX-NEXT:    # fixup A - offset: 5, value: {{\.LCPI.*}}, kind: FK_Data_4
+; X86-AVX-NEXT:    vpaddd %ymm1, %ymm0, %ymm0 # encoding: [0xc5,0xfd,0xfe,0xc1]
+; X86-AVX-NEXT:    retl # encoding: [0xc3]
+;
+; X86-AVX512VL-LABEL: test_x86_avx2_psllv_d_256_const:
+; X86-AVX512VL:       # %bb.0:
+; X86-AVX512VL-NEXT:    vmovdqa {{\.LCPI.*}}, %ymm0 # EVEX TO VEX Compression ymm0 = [2,9,0,4294967295,3,7,4294967295,0]
+; X86-AVX512VL-NEXT:    # encoding: [0xc5,0xfd,0x6f,0x05,A,A,A,A]
+; X86-AVX512VL-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}, kind: FK_Data_4
+; X86-AVX512VL-NEXT:    vpsllvd {{\.LCPI.*}}, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7d,0x47,0x05,A,A,A,A]
+; X86-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.LCPI.*}}, kind: FK_Data_4
+; X86-AVX512VL-NEXT:    vmovdqa {{\.LCPI.*}}, %ymm1 # EVEX TO VEX Compression ymm1 = [4,4,4,4,4,4,4,4294967295]
+; X86-AVX512VL-NEXT:    # encoding: [0xc5,0xfd,0x6f,0x0d,A,A,A,A]
+; X86-AVX512VL-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}, kind: FK_Data_4
+; X86-AVX512VL-NEXT:    vpsllvd {{\.LCPI.*}}, %ymm1, %ymm1 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x75,0x47,0x0d,A,A,A,A]
+; X86-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.LCPI.*}}, kind: FK_Data_4
+; X86-AVX512VL-NEXT:    vpaddd %ymm1, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xfe,0xc1]
+; X86-AVX512VL-NEXT:    retl # encoding: [0xc3]
+;
+; X64-AVX-LABEL: test_x86_avx2_psllv_d_256_const:
+; X64-AVX:       # %bb.0:
+; X64-AVX-NEXT:    vmovdqa {{.*#+}} ymm0 = [2,9,0,4294967295,3,7,4294967295,0]
+; X64-AVX-NEXT:    # encoding: [0xc5,0xfd,0x6f,0x05,A,A,A,A]
+; X64-AVX-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX-NEXT:    vpsllvd {{.*}}(%rip), %ymm0, %ymm0 # encoding: [0xc4,0xe2,0x7d,0x47,0x05,A,A,A,A]
+; X64-AVX-NEXT:    # fixup A - offset: 5, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX-NEXT:    vmovdqa {{.*#+}} ymm1 = [4,4,4,4,4,4,4,4294967295]
+; X64-AVX-NEXT:    # encoding: [0xc5,0xfd,0x6f,0x0d,A,A,A,A]
+; X64-AVX-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX-NEXT:    vpsllvd {{.*}}(%rip), %ymm1, %ymm1 # encoding: [0xc4,0xe2,0x75,0x47,0x0d,A,A,A,A]
+; X64-AVX-NEXT:    # fixup A - offset: 5, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX-NEXT:    vpaddd %ymm1, %ymm0, %ymm0 # encoding: [0xc5,0xfd,0xfe,0xc1]
+; X64-AVX-NEXT:    retq # encoding: [0xc3]
+;
+; X64-AVX512VL-LABEL: test_x86_avx2_psllv_d_256_const:
+; X64-AVX512VL:       # %bb.0:
+; X64-AVX512VL-NEXT:    vmovdqa {{.*}}(%rip), %ymm0 # EVEX TO VEX Compression ymm0 = [2,9,0,4294967295,3,7,4294967295,0]
+; X64-AVX512VL-NEXT:    # encoding: [0xc5,0xfd,0x6f,0x05,A,A,A,A]
+; X64-AVX512VL-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX512VL-NEXT:    vpsllvd {{.*}}(%rip), %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7d,0x47,0x05,A,A,A,A]
+; X64-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX512VL-NEXT:    vmovdqa {{.*}}(%rip), %ymm1 # EVEX TO VEX Compression ymm1 = [4,4,4,4,4,4,4,4294967295]
+; X64-AVX512VL-NEXT:    # encoding: [0xc5,0xfd,0x6f,0x0d,A,A,A,A]
+; X64-AVX512VL-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX512VL-NEXT:    vpsllvd {{.*}}(%rip), %ymm1, %ymm1 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x75,0x47,0x0d,A,A,A,A]
+; X64-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX512VL-NEXT:    vpaddd %ymm1, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xfe,0xc1]
+; X64-AVX512VL-NEXT:    retq # encoding: [0xc3]
+  %res0 = call <8 x i32> @llvm.x86.avx2.psllv.d.256(<8 x i32> <i32 2, i32 9, i32 0, i32 -1, i32 3, i32 7, i32 -1, i32 0>, <8 x i32> <i32 1, i32 0, i32 33, i32 -1,i32 2, i32 0, i32 34, i32 -2>)
+  %res1 = call <8 x i32> @llvm.x86.avx2.psllv.d.256(<8 x i32> <i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 -1>, <8 x i32> <i32 1, i32 1, i32 1,  i32 1, i32 1, i32 1, i32 1, i32 -1>)
+  %res2 = add <8 x i32> %res0, %res1
+  ret <8 x i32> %res2
+}
 declare <8 x i32> @llvm.x86.avx2.psllv.d.256(<8 x i32>, <8 x i32>) nounwind readnone
 
 
 define <2 x i64> @test_x86_avx2_psllv_q(<2 x i64> %a0, <2 x i64> %a1) {
 ; AVX2-LABEL: test_x86_avx2_psllv_q:
-; AVX2:       ## %bb.0:
-; AVX2-NEXT:    vpsllvq %xmm1, %xmm0, %xmm0 ## encoding: [0xc4,0xe2,0xf9,0x47,0xc1]
-; AVX2-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpsllvq %xmm1, %xmm0, %xmm0 # encoding: [0xc4,0xe2,0xf9,0x47,0xc1]
+; AVX2-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
 ;
 ; AVX512VL-LABEL: test_x86_avx2_psllv_q:
-; AVX512VL:       ## %bb.0:
-; AVX512VL-NEXT:    vpsllvq %xmm1, %xmm0, %xmm0 ## EVEX TO VEX Compression encoding: [0xc4,0xe2,0xf9,0x47,0xc1]
-; AVX512VL-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpsllvq %xmm1, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0xf9,0x47,0xc1]
+; AVX512VL-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
   %res = call <2 x i64> @llvm.x86.avx2.psllv.q(<2 x i64> %a0, <2 x i64> %a1) ; <<2 x i64>> [#uses=1]
   ret <2 x i64> %res
 }
+define <2 x i64> @test_x86_avx2_psllv_q_const() {
+; X86-AVX-LABEL: test_x86_avx2_psllv_q_const:
+; X86-AVX:       # %bb.0:
+; X86-AVX-NEXT:    vmovdqa {{.*#+}} xmm0 = [4,0,4294967295,4294967295]
+; X86-AVX-NEXT:    # encoding: [0xc5,0xf9,0x6f,0x05,A,A,A,A]
+; X86-AVX-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}, kind: FK_Data_4
+; X86-AVX-NEXT:    vpsllvq {{\.LCPI.*}}, %xmm0, %xmm0 # encoding: [0xc4,0xe2,0xf9,0x47,0x05,A,A,A,A]
+; X86-AVX-NEXT:    # fixup A - offset: 5, value: {{\.LCPI.*}}, kind: FK_Data_4
+; X86-AVX-NEXT:    retl # encoding: [0xc3]
+;
+; X86-AVX512VL-LABEL: test_x86_avx2_psllv_q_const:
+; X86-AVX512VL:       # %bb.0:
+; X86-AVX512VL-NEXT:    vmovdqa {{\.LCPI.*}}, %xmm0 # EVEX TO VEX Compression xmm0 = [4,0,4294967295,4294967295]
+; X86-AVX512VL-NEXT:    # encoding: [0xc5,0xf9,0x6f,0x05,A,A,A,A]
+; X86-AVX512VL-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}, kind: FK_Data_4
+; X86-AVX512VL-NEXT:    vpsllvq {{\.LCPI.*}}, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0xf9,0x47,0x05,A,A,A,A]
+; X86-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.LCPI.*}}, kind: FK_Data_4
+; X86-AVX512VL-NEXT:    retl # encoding: [0xc3]
+;
+; X64-AVX-LABEL: test_x86_avx2_psllv_q_const:
+; X64-AVX:       # %bb.0:
+; X64-AVX-NEXT:    vmovdqa {{.*#+}} xmm0 = [4,18446744073709551615]
+; X64-AVX-NEXT:    # encoding: [0xc5,0xf9,0x6f,0x05,A,A,A,A]
+; X64-AVX-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX-NEXT:    vpsllvq {{.*}}(%rip), %xmm0, %xmm0 # encoding: [0xc4,0xe2,0xf9,0x47,0x05,A,A,A,A]
+; X64-AVX-NEXT:    # fixup A - offset: 5, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX-NEXT:    retq # encoding: [0xc3]
+;
+; X64-AVX512VL-LABEL: test_x86_avx2_psllv_q_const:
+; X64-AVX512VL:       # %bb.0:
+; X64-AVX512VL-NEXT:    vmovdqa {{.*}}(%rip), %xmm0 # EVEX TO VEX Compression xmm0 = [4,18446744073709551615]
+; X64-AVX512VL-NEXT:    # encoding: [0xc5,0xf9,0x6f,0x05,A,A,A,A]
+; X64-AVX512VL-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX512VL-NEXT:    vpsllvq {{.*}}(%rip), %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0xf9,0x47,0x05,A,A,A,A]
+; X64-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX512VL-NEXT:    retq # encoding: [0xc3]
+  %res = call <2 x i64> @llvm.x86.avx2.psllv.q(<2 x i64> <i64 4, i64 -1>, <2 x i64> <i64 1, i64 -1>)
+  ret <2 x i64> %res
+}
 declare <2 x i64> @llvm.x86.avx2.psllv.q(<2 x i64>, <2 x i64>) nounwind readnone
 
 
 define <4 x i64> @test_x86_avx2_psllv_q_256(<4 x i64> %a0, <4 x i64> %a1) {
 ; AVX2-LABEL: test_x86_avx2_psllv_q_256:
-; AVX2:       ## %bb.0:
-; AVX2-NEXT:    vpsllvq %ymm1, %ymm0, %ymm0 ## encoding: [0xc4,0xe2,0xfd,0x47,0xc1]
-; AVX2-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpsllvq %ymm1, %ymm0, %ymm0 # encoding: [0xc4,0xe2,0xfd,0x47,0xc1]
+; AVX2-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
 ;
 ; AVX512VL-LABEL: test_x86_avx2_psllv_q_256:
-; AVX512VL:       ## %bb.0:
-; AVX512VL-NEXT:    vpsllvq %ymm1, %ymm0, %ymm0 ## EVEX TO VEX Compression encoding: [0xc4,0xe2,0xfd,0x47,0xc1]
-; AVX512VL-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpsllvq %ymm1, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0xfd,0x47,0xc1]
+; AVX512VL-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
   %res = call <4 x i64> @llvm.x86.avx2.psllv.q.256(<4 x i64> %a0, <4 x i64> %a1) ; <<4 x i64>> [#uses=1]
   ret <4 x i64> %res
 }
+
+define <4 x i64> @test_x86_avx2_psllv_q_256_const() {
+; X86-AVX-LABEL: test_x86_avx2_psllv_q_256_const:
+; X86-AVX:       # %bb.0:
+; X86-AVX-NEXT:    vmovdqa {{.*#+}} ymm0 = [4,0,4,0,4,0,4294967295,4294967295]
+; X86-AVX-NEXT:    # encoding: [0xc5,0xfd,0x6f,0x05,A,A,A,A]
+; X86-AVX-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}, kind: FK_Data_4
+; X86-AVX-NEXT:    vpsllvq {{\.LCPI.*}}, %ymm0, %ymm0 # encoding: [0xc4,0xe2,0xfd,0x47,0x05,A,A,A,A]
+; X86-AVX-NEXT:    # fixup A - offset: 5, value: {{\.LCPI.*}}, kind: FK_Data_4
+; X86-AVX-NEXT:    retl # encoding: [0xc3]
+;
+; X86-AVX512VL-LABEL: test_x86_avx2_psllv_q_256_const:
+; X86-AVX512VL:       # %bb.0:
+; X86-AVX512VL-NEXT:    vmovdqa {{\.LCPI.*}}, %ymm0 # EVEX TO VEX Compression ymm0 = [4,0,4,0,4,0,4294967295,4294967295]
+; X86-AVX512VL-NEXT:    # encoding: [0xc5,0xfd,0x6f,0x05,A,A,A,A]
+; X86-AVX512VL-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}, kind: FK_Data_4
+; X86-AVX512VL-NEXT:    vpsllvq {{\.LCPI.*}}, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0xfd,0x47,0x05,A,A,A,A]
+; X86-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.LCPI.*}}, kind: FK_Data_4
+; X86-AVX512VL-NEXT:    retl # encoding: [0xc3]
+;
+; X64-AVX-LABEL: test_x86_avx2_psllv_q_256_const:
+; X64-AVX:       # %bb.0:
+; X64-AVX-NEXT:    vmovdqa {{.*#+}} ymm0 = [4,4,4,18446744073709551615]
+; X64-AVX-NEXT:    # encoding: [0xc5,0xfd,0x6f,0x05,A,A,A,A]
+; X64-AVX-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX-NEXT:    vpsllvq {{.*}}(%rip), %ymm0, %ymm0 # encoding: [0xc4,0xe2,0xfd,0x47,0x05,A,A,A,A]
+; X64-AVX-NEXT:    # fixup A - offset: 5, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX-NEXT:    retq # encoding: [0xc3]
+;
+; X64-AVX512VL-LABEL: test_x86_avx2_psllv_q_256_const:
+; X64-AVX512VL:       # %bb.0:
+; X64-AVX512VL-NEXT:    vmovdqa {{.*}}(%rip), %ymm0 # EVEX TO VEX Compression ymm0 = [4,4,4,18446744073709551615]
+; X64-AVX512VL-NEXT:    # encoding: [0xc5,0xfd,0x6f,0x05,A,A,A,A]
+; X64-AVX512VL-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX512VL-NEXT:    vpsllvq {{.*}}(%rip), %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0xfd,0x47,0x05,A,A,A,A]
+; X64-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX512VL-NEXT:    retq # encoding: [0xc3]
+  %res = call <4 x i64> @llvm.x86.avx2.psllv.q.256(<4 x i64> <i64 4, i64 4, i64 4, i64 -1>, <4 x i64> <i64 1, i64 1, i64 1, i64 -1>)
+  ret <4 x i64> %res
+}
 declare <4 x i64> @llvm.x86.avx2.psllv.q.256(<4 x i64>, <4 x i64>) nounwind readnone
 
 
 define <4 x i32> @test_x86_avx2_psrlv_d(<4 x i32> %a0, <4 x i32> %a1) {
 ; AVX2-LABEL: test_x86_avx2_psrlv_d:
-; AVX2:       ## %bb.0:
-; AVX2-NEXT:    vpsrlvd %xmm1, %xmm0, %xmm0 ## encoding: [0xc4,0xe2,0x79,0x45,0xc1]
-; AVX2-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpsrlvd %xmm1, %xmm0, %xmm0 # encoding: [0xc4,0xe2,0x79,0x45,0xc1]
+; AVX2-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
 ;
 ; AVX512VL-LABEL: test_x86_avx2_psrlv_d:
-; AVX512VL:       ## %bb.0:
-; AVX512VL-NEXT:    vpsrlvd %xmm1, %xmm0, %xmm0 ## EVEX TO VEX Compression encoding: [0xc4,0xe2,0x79,0x45,0xc1]
-; AVX512VL-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpsrlvd %xmm1, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x79,0x45,0xc1]
+; AVX512VL-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
   %res = call <4 x i32> @llvm.x86.avx2.psrlv.d(<4 x i32> %a0, <4 x i32> %a1) ; <<4 x i32>> [#uses=1]
   ret <4 x i32> %res
 }
+
+define <4 x i32> @test_x86_avx2_psrlv_d_const() {
+; X86-AVX-LABEL: test_x86_avx2_psrlv_d_const:
+; X86-AVX:       # %bb.0:
+; X86-AVX-NEXT:    vmovdqa {{.*#+}} xmm0 = [2,9,0,4294967295]
+; X86-AVX-NEXT:    # encoding: [0xc5,0xf9,0x6f,0x05,A,A,A,A]
+; X86-AVX-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}, kind: FK_Data_4
+; X86-AVX-NEXT:    vpsrlvd {{\.LCPI.*}}, %xmm0, %xmm0 # encoding: [0xc4,0xe2,0x79,0x45,0x05,A,A,A,A]
+; X86-AVX-NEXT:    # fixup A - offset: 5, value: {{\.LCPI.*}}, kind: FK_Data_4
+; X86-AVX-NEXT:    vmovdqa {{.*#+}} xmm1 = [4,4,4,4294967295]
+; X86-AVX-NEXT:    # encoding: [0xc5,0xf9,0x6f,0x0d,A,A,A,A]
+; X86-AVX-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}, kind: FK_Data_4
+; X86-AVX-NEXT:    vpsrlvd {{\.LCPI.*}}, %xmm1, %xmm1 # encoding: [0xc4,0xe2,0x71,0x45,0x0d,A,A,A,A]
+; X86-AVX-NEXT:    # fixup A - offset: 5, value: {{\.LCPI.*}}, kind: FK_Data_4
+; X86-AVX-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xfe,0xc1]
+; X86-AVX-NEXT:    retl # encoding: [0xc3]
+;
+; X86-AVX512VL-LABEL: test_x86_avx2_psrlv_d_const:
+; X86-AVX512VL:       # %bb.0:
+; X86-AVX512VL-NEXT:    vmovdqa {{\.LCPI.*}}, %xmm0 # EVEX TO VEX Compression xmm0 = [2,9,0,4294967295]
+; X86-AVX512VL-NEXT:    # encoding: [0xc5,0xf9,0x6f,0x05,A,A,A,A]
+; X86-AVX512VL-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}, kind: FK_Data_4
+; X86-AVX512VL-NEXT:    vpsrlvd {{\.LCPI.*}}, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x79,0x45,0x05,A,A,A,A]
+; X86-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.LCPI.*}}, kind: FK_Data_4
+; X86-AVX512VL-NEXT:    vmovdqa {{\.LCPI.*}}, %xmm1 # EVEX TO VEX Compression xmm1 = [4,4,4,4294967295]
+; X86-AVX512VL-NEXT:    # encoding: [0xc5,0xf9,0x6f,0x0d,A,A,A,A]
+; X86-AVX512VL-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}, kind: FK_Data_4
+; X86-AVX512VL-NEXT:    vpsrlvd {{\.LCPI.*}}, %xmm1, %xmm1 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x71,0x45,0x0d,A,A,A,A]
+; X86-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.LCPI.*}}, kind: FK_Data_4
+; X86-AVX512VL-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfe,0xc1]
+; X86-AVX512VL-NEXT:    retl # encoding: [0xc3]
+;
+; X64-AVX-LABEL: test_x86_avx2_psrlv_d_const:
+; X64-AVX:       # %bb.0:
+; X64-AVX-NEXT:    vmovdqa {{.*#+}} xmm0 = [2,9,0,4294967295]
+; X64-AVX-NEXT:    # encoding: [0xc5,0xf9,0x6f,0x05,A,A,A,A]
+; X64-AVX-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX-NEXT:    vpsrlvd {{.*}}(%rip), %xmm0, %xmm0 # encoding: [0xc4,0xe2,0x79,0x45,0x05,A,A,A,A]
+; X64-AVX-NEXT:    # fixup A - offset: 5, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX-NEXT:    vmovdqa {{.*#+}} xmm1 = [4,4,4,4294967295]
+; X64-AVX-NEXT:    # encoding: [0xc5,0xf9,0x6f,0x0d,A,A,A,A]
+; X64-AVX-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX-NEXT:    vpsrlvd {{.*}}(%rip), %xmm1, %xmm1 # encoding: [0xc4,0xe2,0x71,0x45,0x0d,A,A,A,A]
+; X64-AVX-NEXT:    # fixup A - offset: 5, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xfe,0xc1]
+; X64-AVX-NEXT:    retq # encoding: [0xc3]
+;
+; X64-AVX512VL-LABEL: test_x86_avx2_psrlv_d_const:
+; X64-AVX512VL:       # %bb.0:
+; X64-AVX512VL-NEXT:    vmovdqa {{.*}}(%rip), %xmm0 # EVEX TO VEX Compression xmm0 = [2,9,0,4294967295]
+; X64-AVX512VL-NEXT:    # encoding: [0xc5,0xf9,0x6f,0x05,A,A,A,A]
+; X64-AVX512VL-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX512VL-NEXT:    vpsrlvd {{.*}}(%rip), %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x79,0x45,0x05,A,A,A,A]
+; X64-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX512VL-NEXT:    vmovdqa {{.*}}(%rip), %xmm1 # EVEX TO VEX Compression xmm1 = [4,4,4,4294967295]
+; X64-AVX512VL-NEXT:    # encoding: [0xc5,0xf9,0x6f,0x0d,A,A,A,A]
+; X64-AVX512VL-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX512VL-NEXT:    vpsrlvd {{.*}}(%rip), %xmm1, %xmm1 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x71,0x45,0x0d,A,A,A,A]
+; X64-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX512VL-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfe,0xc1]
+; X64-AVX512VL-NEXT:    retq # encoding: [0xc3]
+  %res0 = call <4 x i32> @llvm.x86.avx2.psrlv.d(<4 x i32> <i32 2, i32 9, i32 0, i32 -1>, <4 x i32> <i32 1, i32 0, i32 33, i32 -1>)
+  %res1 = call <4 x i32> @llvm.x86.avx2.psrlv.d(<4 x i32> <i32 4, i32 4, i32 4, i32 -1>, <4 x i32> <i32 1, i32 1, i32 1,  i32 -1>)
+  %res2 = add <4 x i32> %res0, %res1
+  ret <4 x i32> %res2
+}
 declare <4 x i32> @llvm.x86.avx2.psrlv.d(<4 x i32>, <4 x i32>) nounwind readnone
 
 
 define <8 x i32> @test_x86_avx2_psrlv_d_256(<8 x i32> %a0, <8 x i32> %a1) {
 ; AVX2-LABEL: test_x86_avx2_psrlv_d_256:
-; AVX2:       ## %bb.0:
-; AVX2-NEXT:    vpsrlvd %ymm1, %ymm0, %ymm0 ## encoding: [0xc4,0xe2,0x7d,0x45,0xc1]
-; AVX2-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpsrlvd %ymm1, %ymm0, %ymm0 # encoding: [0xc4,0xe2,0x7d,0x45,0xc1]
+; AVX2-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
 ;
 ; AVX512VL-LABEL: test_x86_avx2_psrlv_d_256:
-; AVX512VL:       ## %bb.0:
-; AVX512VL-NEXT:    vpsrlvd %ymm1, %ymm0, %ymm0 ## EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7d,0x45,0xc1]
-; AVX512VL-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpsrlvd %ymm1, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7d,0x45,0xc1]
+; AVX512VL-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
   %res = call <8 x i32> @llvm.x86.avx2.psrlv.d.256(<8 x i32> %a0, <8 x i32> %a1) ; <<8 x i32>> [#uses=1]
   ret <8 x i32> %res
 }
+
+define <8 x i32> @test_x86_avx2_psrlv_d_256_const() {
+; X86-AVX-LABEL: test_x86_avx2_psrlv_d_256_const:
+; X86-AVX:       # %bb.0:
+; X86-AVX-NEXT:    vmovdqa {{.*#+}} ymm0 = [2,9,0,4294967295,3,7,4294967295,0]
+; X86-AVX-NEXT:    # encoding: [0xc5,0xfd,0x6f,0x05,A,A,A,A]
+; X86-AVX-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}, kind: FK_Data_4
+; X86-AVX-NEXT:    vpsrlvd {{\.LCPI.*}}, %ymm0, %ymm0 # encoding: [0xc4,0xe2,0x7d,0x45,0x05,A,A,A,A]
+; X86-AVX-NEXT:    # fixup A - offset: 5, value: {{\.LCPI.*}}, kind: FK_Data_4
+; X86-AVX-NEXT:    vmovdqa {{.*#+}} ymm1 = [4,4,4,4,4,4,4,4294967295]
+; X86-AVX-NEXT:    # encoding: [0xc5,0xfd,0x6f,0x0d,A,A,A,A]
+; X86-AVX-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}, kind: FK_Data_4
+; X86-AVX-NEXT:    vpsrlvd {{\.LCPI.*}}, %ymm1, %ymm1 # encoding: [0xc4,0xe2,0x75,0x45,0x0d,A,A,A,A]
+; X86-AVX-NEXT:    # fixup A - offset: 5, value: {{\.LCPI.*}}, kind: FK_Data_4
+; X86-AVX-NEXT:    vpaddd %ymm1, %ymm0, %ymm0 # encoding: [0xc5,0xfd,0xfe,0xc1]
+; X86-AVX-NEXT:    retl # encoding: [0xc3]
+;
+; X86-AVX512VL-LABEL: test_x86_avx2_psrlv_d_256_const:
+; X86-AVX512VL:       # %bb.0:
+; X86-AVX512VL-NEXT:    vmovdqa {{\.LCPI.*}}, %ymm0 # EVEX TO VEX Compression ymm0 = [2,9,0,4294967295,3,7,4294967295,0]
+; X86-AVX512VL-NEXT:    # encoding: [0xc5,0xfd,0x6f,0x05,A,A,A,A]
+; X86-AVX512VL-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}, kind: FK_Data_4
+; X86-AVX512VL-NEXT:    vpsrlvd {{\.LCPI.*}}, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7d,0x45,0x05,A,A,A,A]
+; X86-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.LCPI.*}}, kind: FK_Data_4
+; X86-AVX512VL-NEXT:    vmovdqa {{\.LCPI.*}}, %ymm1 # EVEX TO VEX Compression ymm1 = [4,4,4,4,4,4,4,4294967295]
+; X86-AVX512VL-NEXT:    # encoding: [0xc5,0xfd,0x6f,0x0d,A,A,A,A]
+; X86-AVX512VL-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}, kind: FK_Data_4
+; X86-AVX512VL-NEXT:    vpsrlvd {{\.LCPI.*}}, %ymm1, %ymm1 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x75,0x45,0x0d,A,A,A,A]
+; X86-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.LCPI.*}}, kind: FK_Data_4
+; X86-AVX512VL-NEXT:    vpaddd %ymm1, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xfe,0xc1]
+; X86-AVX512VL-NEXT:    retl # encoding: [0xc3]
+;
+; X64-AVX-LABEL: test_x86_avx2_psrlv_d_256_const:
+; X64-AVX:       # %bb.0:
+; X64-AVX-NEXT:    vmovdqa {{.*#+}} ymm0 = [2,9,0,4294967295,3,7,4294967295,0]
+; X64-AVX-NEXT:    # encoding: [0xc5,0xfd,0x6f,0x05,A,A,A,A]
+; X64-AVX-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX-NEXT:    vpsrlvd {{.*}}(%rip), %ymm0, %ymm0 # encoding: [0xc4,0xe2,0x7d,0x45,0x05,A,A,A,A]
+; X64-AVX-NEXT:    # fixup A - offset: 5, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX-NEXT:    vmovdqa {{.*#+}} ymm1 = [4,4,4,4,4,4,4,4294967295]
+; X64-AVX-NEXT:    # encoding: [0xc5,0xfd,0x6f,0x0d,A,A,A,A]
+; X64-AVX-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX-NEXT:    vpsrlvd {{.*}}(%rip), %ymm1, %ymm1 # encoding: [0xc4,0xe2,0x75,0x45,0x0d,A,A,A,A]
+; X64-AVX-NEXT:    # fixup A - offset: 5, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX-NEXT:    vpaddd %ymm1, %ymm0, %ymm0 # encoding: [0xc5,0xfd,0xfe,0xc1]
+; X64-AVX-NEXT:    retq # encoding: [0xc3]
+;
+; X64-AVX512VL-LABEL: test_x86_avx2_psrlv_d_256_const:
+; X64-AVX512VL:       # %bb.0:
+; X64-AVX512VL-NEXT:    vmovdqa {{.*}}(%rip), %ymm0 # EVEX TO VEX Compression ymm0 = [2,9,0,4294967295,3,7,4294967295,0]
+; X64-AVX512VL-NEXT:    # encoding: [0xc5,0xfd,0x6f,0x05,A,A,A,A]
+; X64-AVX512VL-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX512VL-NEXT:    vpsrlvd {{.*}}(%rip), %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7d,0x45,0x05,A,A,A,A]
+; X64-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX512VL-NEXT:    vmovdqa {{.*}}(%rip), %ymm1 # EVEX TO VEX Compression ymm1 = [4,4,4,4,4,4,4,4294967295]
+; X64-AVX512VL-NEXT:    # encoding: [0xc5,0xfd,0x6f,0x0d,A,A,A,A]
+; X64-AVX512VL-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX512VL-NEXT:    vpsrlvd {{.*}}(%rip), %ymm1, %ymm1 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x75,0x45,0x0d,A,A,A,A]
+; X64-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX512VL-NEXT:    vpaddd %ymm1, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xfe,0xc1]
+; X64-AVX512VL-NEXT:    retq # encoding: [0xc3]
+  %res0 = call <8 x i32> @llvm.x86.avx2.psrlv.d.256(<8 x i32> <i32 2, i32 9, i32 0, i32 -1, i32 3, i32 7, i32 -1, i32 0>, <8 x i32> <i32 1, i32 0, i32 33, i32 -1,i32 2, i32 0, i32 34, i32 -2>)
+  %res1 = call <8 x i32> @llvm.x86.avx2.psrlv.d.256(<8 x i32> <i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 -1>, <8 x i32> <i32 1, i32 1, i32 1,  i32 1, i32 1, i32 1, i32 1, i32 -1>)
+  %res2 = add <8 x i32> %res0, %res1
+  ret <8 x i32> %res2
+}
 declare <8 x i32> @llvm.x86.avx2.psrlv.d.256(<8 x i32>, <8 x i32>) nounwind readnone
 
 
 define <2 x i64> @test_x86_avx2_psrlv_q(<2 x i64> %a0, <2 x i64> %a1) {
 ; AVX2-LABEL: test_x86_avx2_psrlv_q:
-; AVX2:       ## %bb.0:
-; AVX2-NEXT:    vpsrlvq %xmm1, %xmm0, %xmm0 ## encoding: [0xc4,0xe2,0xf9,0x45,0xc1]
-; AVX2-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpsrlvq %xmm1, %xmm0, %xmm0 # encoding: [0xc4,0xe2,0xf9,0x45,0xc1]
+; AVX2-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
 ;
 ; AVX512VL-LABEL: test_x86_avx2_psrlv_q:
-; AVX512VL:       ## %bb.0:
-; AVX512VL-NEXT:    vpsrlvq %xmm1, %xmm0, %xmm0 ## EVEX TO VEX Compression encoding: [0xc4,0xe2,0xf9,0x45,0xc1]
-; AVX512VL-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpsrlvq %xmm1, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0xf9,0x45,0xc1]
+; AVX512VL-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
   %res = call <2 x i64> @llvm.x86.avx2.psrlv.q(<2 x i64> %a0, <2 x i64> %a1) ; <<2 x i64>> [#uses=1]
   ret <2 x i64> %res
 }
+
+define <2 x i64> @test_x86_avx2_psrlv_q_const() {
+; X86-AVX-LABEL: test_x86_avx2_psrlv_q_const:
+; X86-AVX:       # %bb.0:
+; X86-AVX-NEXT:    vmovdqa {{.*#+}} xmm0 = [4,0,4,0]
+; X86-AVX-NEXT:    # encoding: [0xc5,0xf9,0x6f,0x05,A,A,A,A]
+; X86-AVX-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}, kind: FK_Data_4
+; X86-AVX-NEXT:    vpsrlvq {{\.LCPI.*}}, %xmm0, %xmm0 # encoding: [0xc4,0xe2,0xf9,0x45,0x05,A,A,A,A]
+; X86-AVX-NEXT:    # fixup A - offset: 5, value: {{\.LCPI.*}}, kind: FK_Data_4
+; X86-AVX-NEXT:    retl # encoding: [0xc3]
+;
+; X86-AVX512VL-LABEL: test_x86_avx2_psrlv_q_const:
+; X86-AVX512VL:       # %bb.0:
+; X86-AVX512VL-NEXT:    vmovdqa {{\.LCPI.*}}, %xmm0 # EVEX TO VEX Compression xmm0 = [4,0,4,0]
+; X86-AVX512VL-NEXT:    # encoding: [0xc5,0xf9,0x6f,0x05,A,A,A,A]
+; X86-AVX512VL-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}, kind: FK_Data_4
+; X86-AVX512VL-NEXT:    vpsrlvq {{\.LCPI.*}}, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0xf9,0x45,0x05,A,A,A,A]
+; X86-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.LCPI.*}}, kind: FK_Data_4
+; X86-AVX512VL-NEXT:    retl # encoding: [0xc3]
+;
+; X64-AVX-LABEL: test_x86_avx2_psrlv_q_const:
+; X64-AVX:       # %bb.0:
+; X64-AVX-NEXT:    vmovdqa {{.*#+}} xmm0 = [4,4]
+; X64-AVX-NEXT:    # encoding: [0xc5,0xf9,0x6f,0x05,A,A,A,A]
+; X64-AVX-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX-NEXT:    vpsrlvq {{.*}}(%rip), %xmm0, %xmm0 # encoding: [0xc4,0xe2,0xf9,0x45,0x05,A,A,A,A]
+; X64-AVX-NEXT:    # fixup A - offset: 5, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX-NEXT:    retq # encoding: [0xc3]
+;
+; X64-AVX512VL-LABEL: test_x86_avx2_psrlv_q_const:
+; X64-AVX512VL:       # %bb.0:
+; X64-AVX512VL-NEXT:    vmovdqa {{.*}}(%rip), %xmm0 # EVEX TO VEX Compression xmm0 = [4,4]
+; X64-AVX512VL-NEXT:    # encoding: [0xc5,0xf9,0x6f,0x05,A,A,A,A]
+; X64-AVX512VL-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX512VL-NEXT:    vpsrlvq {{.*}}(%rip), %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0xf9,0x45,0x05,A,A,A,A]
+; X64-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX512VL-NEXT:    retq # encoding: [0xc3]
+  %res = call <2 x i64> @llvm.x86.avx2.psrlv.q(<2 x i64> <i64 4, i64 4>, <2 x i64> <i64 1, i64 -1>)
+  ret <2 x i64> %res
+}
 declare <2 x i64> @llvm.x86.avx2.psrlv.q(<2 x i64>, <2 x i64>) nounwind readnone
 
 
 define <4 x i64> @test_x86_avx2_psrlv_q_256(<4 x i64> %a0, <4 x i64> %a1) {
 ; AVX2-LABEL: test_x86_avx2_psrlv_q_256:
-; AVX2:       ## %bb.0:
-; AVX2-NEXT:    vpsrlvq %ymm1, %ymm0, %ymm0 ## encoding: [0xc4,0xe2,0xfd,0x45,0xc1]
-; AVX2-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpsrlvq %ymm1, %ymm0, %ymm0 # encoding: [0xc4,0xe2,0xfd,0x45,0xc1]
+; AVX2-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
 ;
 ; AVX512VL-LABEL: test_x86_avx2_psrlv_q_256:
-; AVX512VL:       ## %bb.0:
-; AVX512VL-NEXT:    vpsrlvq %ymm1, %ymm0, %ymm0 ## EVEX TO VEX Compression encoding: [0xc4,0xe2,0xfd,0x45,0xc1]
-; AVX512VL-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpsrlvq %ymm1, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0xfd,0x45,0xc1]
+; AVX512VL-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
   %res = call <4 x i64> @llvm.x86.avx2.psrlv.q.256(<4 x i64> %a0, <4 x i64> %a1) ; <<4 x i64>> [#uses=1]
   ret <4 x i64> %res
 }
+
+
+define <4 x i64> @test_x86_avx2_psrlv_q_256_const() {
+; X86-AVX-LABEL: test_x86_avx2_psrlv_q_256_const:
+; X86-AVX:       # %bb.0:
+; X86-AVX-NEXT:    vmovdqa {{.*#+}} ymm0 = [4,0,4,0,4,0,4,0]
+; X86-AVX-NEXT:    # encoding: [0xc5,0xfd,0x6f,0x05,A,A,A,A]
+; X86-AVX-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}, kind: FK_Data_4
+; X86-AVX-NEXT:    vpsrlvq {{\.LCPI.*}}, %ymm0, %ymm0 # encoding: [0xc4,0xe2,0xfd,0x45,0x05,A,A,A,A]
+; X86-AVX-NEXT:    # fixup A - offset: 5, value: {{\.LCPI.*}}, kind: FK_Data_4
+; X86-AVX-NEXT:    retl # encoding: [0xc3]
+;
+; X86-AVX512VL-LABEL: test_x86_avx2_psrlv_q_256_const:
+; X86-AVX512VL:       # %bb.0:
+; X86-AVX512VL-NEXT:    vmovdqa {{\.LCPI.*}}, %ymm0 # EVEX TO VEX Compression ymm0 = [4,0,4,0,4,0,4,0]
+; X86-AVX512VL-NEXT:    # encoding: [0xc5,0xfd,0x6f,0x05,A,A,A,A]
+; X86-AVX512VL-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}, kind: FK_Data_4
+; X86-AVX512VL-NEXT:    vpsrlvq {{\.LCPI.*}}, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0xfd,0x45,0x05,A,A,A,A]
+; X86-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.LCPI.*}}, kind: FK_Data_4
+; X86-AVX512VL-NEXT:    retl # encoding: [0xc3]
+;
+; X64-AVX-LABEL: test_x86_avx2_psrlv_q_256_const:
+; X64-AVX:       # %bb.0:
+; X64-AVX-NEXT:    vpbroadcastq {{.*#+}} ymm0 = [4,4,4,4]
+; X64-AVX-NEXT:    # encoding: [0xc4,0xe2,0x7d,0x59,0x05,A,A,A,A]
+; X64-AVX-NEXT:    # fixup A - offset: 5, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX-NEXT:    vpsrlvq {{.*}}(%rip), %ymm0, %ymm0 # encoding: [0xc4,0xe2,0xfd,0x45,0x05,A,A,A,A]
+; X64-AVX-NEXT:    # fixup A - offset: 5, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX-NEXT:    retq # encoding: [0xc3]
+;
+; X64-AVX512VL-LABEL: test_x86_avx2_psrlv_q_256_const:
+; X64-AVX512VL:       # %bb.0:
+; X64-AVX512VL-NEXT:    vpbroadcastq {{.*}}(%rip), %ymm0 # EVEX TO VEX Compression ymm0 = [4,4,4,4]
+; X64-AVX512VL-NEXT:    # encoding: [0xc4,0xe2,0x7d,0x59,0x05,A,A,A,A]
+; X64-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX512VL-NEXT:    vpsrlvq {{.*}}(%rip), %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0xfd,0x45,0x05,A,A,A,A]
+; X64-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX512VL-NEXT:    retq # encoding: [0xc3]
+  %res = call <4 x i64> @llvm.x86.avx2.psrlv.q.256(<4 x i64> <i64 4, i64 4, i64 4, i64 4>, <4 x i64> <i64 1, i64 1, i64 1, i64 -1>)
+  ret <4 x i64> %res
+}
 declare <4 x i64> @llvm.x86.avx2.psrlv.q.256(<4 x i64>, <4 x i64>) nounwind readnone
 
 
 define <4 x i32> @test_x86_avx2_psrav_d(<4 x i32> %a0, <4 x i32> %a1) {
 ; AVX2-LABEL: test_x86_avx2_psrav_d:
-; AVX2:       ## %bb.0:
-; AVX2-NEXT:    vpsravd %xmm1, %xmm0, %xmm0 ## encoding: [0xc4,0xe2,0x79,0x46,0xc1]
-; AVX2-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpsravd %xmm1, %xmm0, %xmm0 # encoding: [0xc4,0xe2,0x79,0x46,0xc1]
+; AVX2-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
 ;
 ; AVX512VL-LABEL: test_x86_avx2_psrav_d:
-; AVX512VL:       ## %bb.0:
-; AVX512VL-NEXT:    vpsravd %xmm1, %xmm0, %xmm0 ## EVEX TO VEX Compression encoding: [0xc4,0xe2,0x79,0x46,0xc1]
-; AVX512VL-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpsravd %xmm1, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x79,0x46,0xc1]
+; AVX512VL-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
   %res = call <4 x i32> @llvm.x86.avx2.psrav.d(<4 x i32> %a0, <4 x i32> %a1) ; <<4 x i32>> [#uses=1]
   ret <4 x i32> %res
 }
 
-define <4 x i32> @test_x86_avx2_psrav_d_const(<4 x i32> %a0, <4 x i32> %a1) {
+define <4 x i32> @test_x86_avx2_psrav_d_const() {
 ; X86-AVX-LABEL: test_x86_avx2_psrav_d_const:
-; X86-AVX:       ## %bb.0:
+; X86-AVX:       # %bb.0:
 ; X86-AVX-NEXT:    vmovdqa {{.*#+}} xmm0 = [2,9,4294967284,23]
-; X86-AVX-NEXT:    ## encoding: [0xc5,0xf9,0x6f,0x05,A,A,A,A]
-; X86-AVX-NEXT:    ## fixup A - offset: 4, value: LCPI83_0, kind: FK_Data_4
-; X86-AVX-NEXT:    vpsravd LCPI83_1, %xmm0, %xmm0 ## encoding: [0xc4,0xe2,0x79,0x46,0x05,A,A,A,A]
-; X86-AVX-NEXT:    ## fixup A - offset: 5, value: LCPI83_1, kind: FK_Data_4
-; X86-AVX-NEXT:    retl ## encoding: [0xc3]
+; X86-AVX-NEXT:    # encoding: [0xc5,0xf9,0x6f,0x05,A,A,A,A]
+; X86-AVX-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}, kind: FK_Data_4
+; X86-AVX-NEXT:    vpsravd {{\.LCPI.*}}, %xmm0, %xmm0 # encoding: [0xc4,0xe2,0x79,0x46,0x05,A,A,A,A]
+; X86-AVX-NEXT:    # fixup A - offset: 5, value: {{\.LCPI.*}}, kind: FK_Data_4
+; X86-AVX-NEXT:    retl # encoding: [0xc3]
 ;
 ; X86-AVX512VL-LABEL: test_x86_avx2_psrav_d_const:
-; X86-AVX512VL:       ## %bb.0:
-; X86-AVX512VL-NEXT:    vmovdqa LCPI83_0, %xmm0 ## EVEX TO VEX Compression xmm0 = [2,9,4294967284,23]
-; X86-AVX512VL-NEXT:    ## encoding: [0xc5,0xf9,0x6f,0x05,A,A,A,A]
-; X86-AVX512VL-NEXT:    ## fixup A - offset: 4, value: LCPI83_0, kind: FK_Data_4
-; X86-AVX512VL-NEXT:    vpsravd LCPI83_1, %xmm0, %xmm0 ## EVEX TO VEX Compression encoding: [0xc4,0xe2,0x79,0x46,0x05,A,A,A,A]
-; X86-AVX512VL-NEXT:    ## fixup A - offset: 5, value: LCPI83_1, kind: FK_Data_4
-; X86-AVX512VL-NEXT:    retl ## encoding: [0xc3]
+; X86-AVX512VL:       # %bb.0:
+; X86-AVX512VL-NEXT:    vmovdqa {{\.LCPI.*}}, %xmm0 # EVEX TO VEX Compression xmm0 = [2,9,4294967284,23]
+; X86-AVX512VL-NEXT:    # encoding: [0xc5,0xf9,0x6f,0x05,A,A,A,A]
+; X86-AVX512VL-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}, kind: FK_Data_4
+; X86-AVX512VL-NEXT:    vpsravd {{\.LCPI.*}}, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x79,0x46,0x05,A,A,A,A]
+; X86-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.LCPI.*}}, kind: FK_Data_4
+; X86-AVX512VL-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-AVX-LABEL: test_x86_avx2_psrav_d_const:
-; X64-AVX:       ## %bb.0:
+; X64-AVX:       # %bb.0:
 ; X64-AVX-NEXT:    vmovdqa {{.*#+}} xmm0 = [2,9,4294967284,23]
-; X64-AVX-NEXT:    ## encoding: [0xc5,0xf9,0x6f,0x05,A,A,A,A]
-; X64-AVX-NEXT:    ## fixup A - offset: 4, value: LCPI83_0-4, kind: reloc_riprel_4byte
-; X64-AVX-NEXT:    vpsravd {{.*}}(%rip), %xmm0, %xmm0 ## encoding: [0xc4,0xe2,0x79,0x46,0x05,A,A,A,A]
-; X64-AVX-NEXT:    ## fixup A - offset: 5, value: LCPI83_1-4, kind: reloc_riprel_4byte
-; X64-AVX-NEXT:    retq ## encoding: [0xc3]
+; X64-AVX-NEXT:    # encoding: [0xc5,0xf9,0x6f,0x05,A,A,A,A]
+; X64-AVX-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX-NEXT:    vpsravd {{.*}}(%rip), %xmm0, %xmm0 # encoding: [0xc4,0xe2,0x79,0x46,0x05,A,A,A,A]
+; X64-AVX-NEXT:    # fixup A - offset: 5, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX-NEXT:    retq # encoding: [0xc3]
 ;
 ; X64-AVX512VL-LABEL: test_x86_avx2_psrav_d_const:
-; X64-AVX512VL:       ## %bb.0:
-; X64-AVX512VL-NEXT:    vmovdqa {{.*}}(%rip), %xmm0 ## EVEX TO VEX Compression xmm0 = [2,9,4294967284,23]
-; X64-AVX512VL-NEXT:    ## encoding: [0xc5,0xf9,0x6f,0x05,A,A,A,A]
-; X64-AVX512VL-NEXT:    ## fixup A - offset: 4, value: LCPI83_0-4, kind: reloc_riprel_4byte
-; X64-AVX512VL-NEXT:    vpsravd {{.*}}(%rip), %xmm0, %xmm0 ## EVEX TO VEX Compression encoding: [0xc4,0xe2,0x79,0x46,0x05,A,A,A,A]
-; X64-AVX512VL-NEXT:    ## fixup A - offset: 5, value: LCPI83_1-4, kind: reloc_riprel_4byte
-; X64-AVX512VL-NEXT:    retq ## encoding: [0xc3]
+; X64-AVX512VL:       # %bb.0:
+; X64-AVX512VL-NEXT:    vmovdqa {{.*}}(%rip), %xmm0 # EVEX TO VEX Compression xmm0 = [2,9,4294967284,23]
+; X64-AVX512VL-NEXT:    # encoding: [0xc5,0xf9,0x6f,0x05,A,A,A,A]
+; X64-AVX512VL-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX512VL-NEXT:    vpsravd {{.*}}(%rip), %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x79,0x46,0x05,A,A,A,A]
+; X64-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX512VL-NEXT:    retq # encoding: [0xc3]
   %res = call <4 x i32> @llvm.x86.avx2.psrav.d(<4 x i32> <i32 2, i32 9, i32 -12, i32 23>, <4 x i32> <i32 1, i32 18, i32 35, i32 52>)
   ret <4 x i32> %res
 }
@@ -1414,54 +1771,54 @@ declare <4 x i32> @llvm.x86.avx2.psrav.d(<4 x i32>, <4 x i32>) nounwind readnone
 
 define <8 x i32> @test_x86_avx2_psrav_d_256(<8 x i32> %a0, <8 x i32> %a1) {
 ; AVX2-LABEL: test_x86_avx2_psrav_d_256:
-; AVX2:       ## %bb.0:
-; AVX2-NEXT:    vpsravd %ymm1, %ymm0, %ymm0 ## encoding: [0xc4,0xe2,0x7d,0x46,0xc1]
-; AVX2-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpsravd %ymm1, %ymm0, %ymm0 # encoding: [0xc4,0xe2,0x7d,0x46,0xc1]
+; AVX2-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
 ;
 ; AVX512VL-LABEL: test_x86_avx2_psrav_d_256:
-; AVX512VL:       ## %bb.0:
-; AVX512VL-NEXT:    vpsravd %ymm1, %ymm0, %ymm0 ## EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7d,0x46,0xc1]
-; AVX512VL-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpsravd %ymm1, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7d,0x46,0xc1]
+; AVX512VL-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
   %res = call <8 x i32> @llvm.x86.avx2.psrav.d.256(<8 x i32> %a0, <8 x i32> %a1) ; <<8 x i32>> [#uses=1]
   ret <8 x i32> %res
 }
 
-define <8 x i32> @test_x86_avx2_psrav_d_256_const(<8 x i32> %a0, <8 x i32> %a1) {
+define <8 x i32> @test_x86_avx2_psrav_d_256_const() {
 ; X86-AVX-LABEL: test_x86_avx2_psrav_d_256_const:
-; X86-AVX:       ## %bb.0:
+; X86-AVX:       # %bb.0:
 ; X86-AVX-NEXT:    vmovdqa {{.*#+}} ymm0 = [2,9,4294967284,23,4294967270,37,4294967256,51]
-; X86-AVX-NEXT:    ## encoding: [0xc5,0xfd,0x6f,0x05,A,A,A,A]
-; X86-AVX-NEXT:    ## fixup A - offset: 4, value: LCPI85_0, kind: FK_Data_4
-; X86-AVX-NEXT:    vpsravd LCPI85_1, %ymm0, %ymm0 ## encoding: [0xc4,0xe2,0x7d,0x46,0x05,A,A,A,A]
-; X86-AVX-NEXT:    ## fixup A - offset: 5, value: LCPI85_1, kind: FK_Data_4
-; X86-AVX-NEXT:    retl ## encoding: [0xc3]
+; X86-AVX-NEXT:    # encoding: [0xc5,0xfd,0x6f,0x05,A,A,A,A]
+; X86-AVX-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}, kind: FK_Data_4
+; X86-AVX-NEXT:    vpsravd {{\.LCPI.*}}, %ymm0, %ymm0 # encoding: [0xc4,0xe2,0x7d,0x46,0x05,A,A,A,A]
+; X86-AVX-NEXT:    # fixup A - offset: 5, value: {{\.LCPI.*}}, kind: FK_Data_4
+; X86-AVX-NEXT:    retl # encoding: [0xc3]
 ;
 ; X86-AVX512VL-LABEL: test_x86_avx2_psrav_d_256_const:
-; X86-AVX512VL:       ## %bb.0:
-; X86-AVX512VL-NEXT:    vmovdqa LCPI85_0, %ymm0 ## EVEX TO VEX Compression ymm0 = [2,9,4294967284,23,4294967270,37,4294967256,51]
-; X86-AVX512VL-NEXT:    ## encoding: [0xc5,0xfd,0x6f,0x05,A,A,A,A]
-; X86-AVX512VL-NEXT:    ## fixup A - offset: 4, value: LCPI85_0, kind: FK_Data_4
-; X86-AVX512VL-NEXT:    vpsravd LCPI85_1, %ymm0, %ymm0 ## EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7d,0x46,0x05,A,A,A,A]
-; X86-AVX512VL-NEXT:    ## fixup A - offset: 5, value: LCPI85_1, kind: FK_Data_4
-; X86-AVX512VL-NEXT:    retl ## encoding: [0xc3]
+; X86-AVX512VL:       # %bb.0:
+; X86-AVX512VL-NEXT:    vmovdqa {{\.LCPI.*}}, %ymm0 # EVEX TO VEX Compression ymm0 = [2,9,4294967284,23,4294967270,37,4294967256,51]
+; X86-AVX512VL-NEXT:    # encoding: [0xc5,0xfd,0x6f,0x05,A,A,A,A]
+; X86-AVX512VL-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}, kind: FK_Data_4
+; X86-AVX512VL-NEXT:    vpsravd {{\.LCPI.*}}, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7d,0x46,0x05,A,A,A,A]
+; X86-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.LCPI.*}}, kind: FK_Data_4
+; X86-AVX512VL-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-AVX-LABEL: test_x86_avx2_psrav_d_256_const:
-; X64-AVX:       ## %bb.0:
+; X64-AVX:       # %bb.0:
 ; X64-AVX-NEXT:    vmovdqa {{.*#+}} ymm0 = [2,9,4294967284,23,4294967270,37,4294967256,51]
-; X64-AVX-NEXT:    ## encoding: [0xc5,0xfd,0x6f,0x05,A,A,A,A]
-; X64-AVX-NEXT:    ## fixup A - offset: 4, value: LCPI85_0-4, kind: reloc_riprel_4byte
-; X64-AVX-NEXT:    vpsravd {{.*}}(%rip), %ymm0, %ymm0 ## encoding: [0xc4,0xe2,0x7d,0x46,0x05,A,A,A,A]
-; X64-AVX-NEXT:    ## fixup A - offset: 5, value: LCPI85_1-4, kind: reloc_riprel_4byte
-; X64-AVX-NEXT:    retq ## encoding: [0xc3]
+; X64-AVX-NEXT:    # encoding: [0xc5,0xfd,0x6f,0x05,A,A,A,A]
+; X64-AVX-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX-NEXT:    vpsravd {{.*}}(%rip), %ymm0, %ymm0 # encoding: [0xc4,0xe2,0x7d,0x46,0x05,A,A,A,A]
+; X64-AVX-NEXT:    # fixup A - offset: 5, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX-NEXT:    retq # encoding: [0xc3]
 ;
 ; X64-AVX512VL-LABEL: test_x86_avx2_psrav_d_256_const:
-; X64-AVX512VL:       ## %bb.0:
-; X64-AVX512VL-NEXT:    vmovdqa {{.*}}(%rip), %ymm0 ## EVEX TO VEX Compression ymm0 = [2,9,4294967284,23,4294967270,37,4294967256,51]
-; X64-AVX512VL-NEXT:    ## encoding: [0xc5,0xfd,0x6f,0x05,A,A,A,A]
-; X64-AVX512VL-NEXT:    ## fixup A - offset: 4, value: LCPI85_0-4, kind: reloc_riprel_4byte
-; X64-AVX512VL-NEXT:    vpsravd {{.*}}(%rip), %ymm0, %ymm0 ## EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7d,0x46,0x05,A,A,A,A]
-; X64-AVX512VL-NEXT:    ## fixup A - offset: 5, value: LCPI85_1-4, kind: reloc_riprel_4byte
-; X64-AVX512VL-NEXT:    retq ## encoding: [0xc3]
+; X64-AVX512VL:       # %bb.0:
+; X64-AVX512VL-NEXT:    vmovdqa {{.*}}(%rip), %ymm0 # EVEX TO VEX Compression ymm0 = [2,9,4294967284,23,4294967270,37,4294967256,51]
+; X64-AVX512VL-NEXT:    # encoding: [0xc5,0xfd,0x6f,0x05,A,A,A,A]
+; X64-AVX512VL-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX512VL-NEXT:    vpsravd {{.*}}(%rip), %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7d,0x46,0x05,A,A,A,A]
+; X64-AVX512VL-NEXT:    # fixup A - offset: 5, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-AVX512VL-NEXT:    retq # encoding: [0xc3]
   %res = call <8 x i32> @llvm.x86.avx2.psrav.d.256(<8 x i32> <i32 2, i32 9, i32 -12, i32 23, i32 -26, i32 37, i32 -40, i32 51>, <8 x i32> <i32 1, i32 18, i32 35, i32 52, i32 69, i32 15, i32 32, i32 49>)
   ret <8 x i32> %res
 }
@@ -1469,15 +1826,15 @@ declare <8 x i32> @llvm.x86.avx2.psrav.d.256(<8 x i32>, <8 x i32>) nounwind read
 
 define <2 x double> @test_x86_avx2_gather_d_pd(<2 x double> %a0, i8* %a1, <4 x i32> %idx, <2 x double> %mask) {
 ; X86-LABEL: test_x86_avx2_gather_d_pd:
-; X86:       ## %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax ## encoding: [0x8b,0x44,0x24,0x04]
-; X86-NEXT:    vgatherdpd %xmm2, (%eax,%xmm1,2), %xmm0 ## encoding: [0xc4,0xe2,0xe9,0x92,0x04,0x48]
-; X86-NEXT:    retl ## encoding: [0xc3]
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-NEXT:    vgatherdpd %xmm2, (%eax,%xmm1,2), %xmm0 # encoding: [0xc4,0xe2,0xe9,0x92,0x04,0x48]
+; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_x86_avx2_gather_d_pd:
-; X64:       ## %bb.0:
-; X64-NEXT:    vgatherdpd %xmm2, (%rdi,%xmm1,2), %xmm0 ## encoding: [0xc4,0xe2,0xe9,0x92,0x04,0x4f]
-; X64-NEXT:    retq ## encoding: [0xc3]
+; X64:       # %bb.0:
+; X64-NEXT:    vgatherdpd %xmm2, (%rdi,%xmm1,2), %xmm0 # encoding: [0xc4,0xe2,0xe9,0x92,0x04,0x4f]
+; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <2 x double> @llvm.x86.avx2.gather.d.pd(<2 x double> %a0,
                             i8* %a1, <4 x i32> %idx, <2 x double> %mask, i8 2) ;
   ret <2 x double> %res
@@ -1487,15 +1844,15 @@ declare <2 x double> @llvm.x86.avx2.gather.d.pd(<2 x double>, i8*,
 
 define <4 x double> @test_x86_avx2_gather_d_pd_256(<4 x double> %a0, i8* %a1, <4 x i32> %idx, <4 x double> %mask) {
 ; X86-LABEL: test_x86_avx2_gather_d_pd_256:
-; X86:       ## %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax ## encoding: [0x8b,0x44,0x24,0x04]
-; X86-NEXT:    vgatherdpd %ymm2, (%eax,%xmm1,2), %ymm0 ## encoding: [0xc4,0xe2,0xed,0x92,0x04,0x48]
-; X86-NEXT:    retl ## encoding: [0xc3]
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-NEXT:    vgatherdpd %ymm2, (%eax,%xmm1,2), %ymm0 # encoding: [0xc4,0xe2,0xed,0x92,0x04,0x48]
+; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_x86_avx2_gather_d_pd_256:
-; X64:       ## %bb.0:
-; X64-NEXT:    vgatherdpd %ymm2, (%rdi,%xmm1,2), %ymm0 ## encoding: [0xc4,0xe2,0xed,0x92,0x04,0x4f]
-; X64-NEXT:    retq ## encoding: [0xc3]
+; X64:       # %bb.0:
+; X64-NEXT:    vgatherdpd %ymm2, (%rdi,%xmm1,2), %ymm0 # encoding: [0xc4,0xe2,0xed,0x92,0x04,0x4f]
+; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <4 x double> @llvm.x86.avx2.gather.d.pd.256(<4 x double> %a0,
                             i8* %a1, <4 x i32> %idx, <4 x double> %mask, i8 2) ;
   ret <4 x double> %res
@@ -1505,15 +1862,15 @@ declare <4 x double> @llvm.x86.avx2.gather.d.pd.256(<4 x double>, i8*,
 
 define <2 x double> @test_x86_avx2_gather_q_pd(<2 x double> %a0, i8* %a1, <2 x i64> %idx, <2 x double> %mask) {
 ; X86-LABEL: test_x86_avx2_gather_q_pd:
-; X86:       ## %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax ## encoding: [0x8b,0x44,0x24,0x04]
-; X86-NEXT:    vgatherqpd %xmm2, (%eax,%xmm1,2), %xmm0 ## encoding: [0xc4,0xe2,0xe9,0x93,0x04,0x48]
-; X86-NEXT:    retl ## encoding: [0xc3]
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-NEXT:    vgatherqpd %xmm2, (%eax,%xmm1,2), %xmm0 # encoding: [0xc4,0xe2,0xe9,0x93,0x04,0x48]
+; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_x86_avx2_gather_q_pd:
-; X64:       ## %bb.0:
-; X64-NEXT:    vgatherqpd %xmm2, (%rdi,%xmm1,2), %xmm0 ## encoding: [0xc4,0xe2,0xe9,0x93,0x04,0x4f]
-; X64-NEXT:    retq ## encoding: [0xc3]
+; X64:       # %bb.0:
+; X64-NEXT:    vgatherqpd %xmm2, (%rdi,%xmm1,2), %xmm0 # encoding: [0xc4,0xe2,0xe9,0x93,0x04,0x4f]
+; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <2 x double> @llvm.x86.avx2.gather.q.pd(<2 x double> %a0,
                             i8* %a1, <2 x i64> %idx, <2 x double> %mask, i8 2) ;
   ret <2 x double> %res
@@ -1523,15 +1880,15 @@ declare <2 x double> @llvm.x86.avx2.gather.q.pd(<2 x double>, i8*,
 
 define <4 x double> @test_x86_avx2_gather_q_pd_256(<4 x double> %a0, i8* %a1, <4 x i64> %idx, <4 x double> %mask) {
 ; X86-LABEL: test_x86_avx2_gather_q_pd_256:
-; X86:       ## %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax ## encoding: [0x8b,0x44,0x24,0x04]
-; X86-NEXT:    vgatherqpd %ymm2, (%eax,%ymm1,2), %ymm0 ## encoding: [0xc4,0xe2,0xed,0x93,0x04,0x48]
-; X86-NEXT:    retl ## encoding: [0xc3]
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-NEXT:    vgatherqpd %ymm2, (%eax,%ymm1,2), %ymm0 # encoding: [0xc4,0xe2,0xed,0x93,0x04,0x48]
+; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_x86_avx2_gather_q_pd_256:
-; X64:       ## %bb.0:
-; X64-NEXT:    vgatherqpd %ymm2, (%rdi,%ymm1,2), %ymm0 ## encoding: [0xc4,0xe2,0xed,0x93,0x04,0x4f]
-; X64-NEXT:    retq ## encoding: [0xc3]
+; X64:       # %bb.0:
+; X64-NEXT:    vgatherqpd %ymm2, (%rdi,%ymm1,2), %ymm0 # encoding: [0xc4,0xe2,0xed,0x93,0x04,0x4f]
+; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <4 x double> @llvm.x86.avx2.gather.q.pd.256(<4 x double> %a0,
                             i8* %a1, <4 x i64> %idx, <4 x double> %mask, i8 2) ;
   ret <4 x double> %res
@@ -1541,15 +1898,15 @@ declare <4 x double> @llvm.x86.avx2.gather.q.pd.256(<4 x double>, i8*,
 
 define <4 x float> @test_x86_avx2_gather_d_ps(<4 x float> %a0, i8* %a1, <4 x i32> %idx, <4 x float> %mask) {
 ; X86-LABEL: test_x86_avx2_gather_d_ps:
-; X86:       ## %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax ## encoding: [0x8b,0x44,0x24,0x04]
-; X86-NEXT:    vgatherdps %xmm2, (%eax,%xmm1,2), %xmm0 ## encoding: [0xc4,0xe2,0x69,0x92,0x04,0x48]
-; X86-NEXT:    retl ## encoding: [0xc3]
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-NEXT:    vgatherdps %xmm2, (%eax,%xmm1,2), %xmm0 # encoding: [0xc4,0xe2,0x69,0x92,0x04,0x48]
+; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_x86_avx2_gather_d_ps:
-; X64:       ## %bb.0:
-; X64-NEXT:    vgatherdps %xmm2, (%rdi,%xmm1,2), %xmm0 ## encoding: [0xc4,0xe2,0x69,0x92,0x04,0x4f]
-; X64-NEXT:    retq ## encoding: [0xc3]
+; X64:       # %bb.0:
+; X64-NEXT:    vgatherdps %xmm2, (%rdi,%xmm1,2), %xmm0 # encoding: [0xc4,0xe2,0x69,0x92,0x04,0x4f]
+; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <4 x float> @llvm.x86.avx2.gather.d.ps(<4 x float> %a0,
                             i8* %a1, <4 x i32> %idx, <4 x float> %mask, i8 2) ;
   ret <4 x float> %res
@@ -1559,15 +1916,15 @@ declare <4 x float> @llvm.x86.avx2.gather.d.ps(<4 x float>, i8*,
 
 define <8 x float> @test_x86_avx2_gather_d_ps_256(<8 x float> %a0, i8* %a1, <8 x i32> %idx, <8 x float> %mask) {
 ; X86-LABEL: test_x86_avx2_gather_d_ps_256:
-; X86:       ## %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax ## encoding: [0x8b,0x44,0x24,0x04]
-; X86-NEXT:    vgatherdps %ymm2, (%eax,%ymm1,2), %ymm0 ## encoding: [0xc4,0xe2,0x6d,0x92,0x04,0x48]
-; X86-NEXT:    retl ## encoding: [0xc3]
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-NEXT:    vgatherdps %ymm2, (%eax,%ymm1,2), %ymm0 # encoding: [0xc4,0xe2,0x6d,0x92,0x04,0x48]
+; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_x86_avx2_gather_d_ps_256:
-; X64:       ## %bb.0:
-; X64-NEXT:    vgatherdps %ymm2, (%rdi,%ymm1,2), %ymm0 ## encoding: [0xc4,0xe2,0x6d,0x92,0x04,0x4f]
-; X64-NEXT:    retq ## encoding: [0xc3]
+; X64:       # %bb.0:
+; X64-NEXT:    vgatherdps %ymm2, (%rdi,%ymm1,2), %ymm0 # encoding: [0xc4,0xe2,0x6d,0x92,0x04,0x4f]
+; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <8 x float> @llvm.x86.avx2.gather.d.ps.256(<8 x float> %a0,
                             i8* %a1, <8 x i32> %idx, <8 x float> %mask, i8 2) ;
   ret <8 x float> %res
@@ -1577,15 +1934,15 @@ declare <8 x float> @llvm.x86.avx2.gather.d.ps.256(<8 x float>, i8*,
 
 define <4 x float> @test_x86_avx2_gather_q_ps(<4 x float> %a0, i8* %a1, <2 x i64> %idx, <4 x float> %mask) {
 ; X86-LABEL: test_x86_avx2_gather_q_ps:
-; X86:       ## %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax ## encoding: [0x8b,0x44,0x24,0x04]
-; X86-NEXT:    vgatherqps %xmm2, (%eax,%xmm1,2), %xmm0 ## encoding: [0xc4,0xe2,0x69,0x93,0x04,0x48]
-; X86-NEXT:    retl ## encoding: [0xc3]
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-NEXT:    vgatherqps %xmm2, (%eax,%xmm1,2), %xmm0 # encoding: [0xc4,0xe2,0x69,0x93,0x04,0x48]
+; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_x86_avx2_gather_q_ps:
-; X64:       ## %bb.0:
-; X64-NEXT:    vgatherqps %xmm2, (%rdi,%xmm1,2), %xmm0 ## encoding: [0xc4,0xe2,0x69,0x93,0x04,0x4f]
-; X64-NEXT:    retq ## encoding: [0xc3]
+; X64:       # %bb.0:
+; X64-NEXT:    vgatherqps %xmm2, (%rdi,%xmm1,2), %xmm0 # encoding: [0xc4,0xe2,0x69,0x93,0x04,0x4f]
+; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <4 x float> @llvm.x86.avx2.gather.q.ps(<4 x float> %a0,
                             i8* %a1, <2 x i64> %idx, <4 x float> %mask, i8 2) ;
   ret <4 x float> %res
@@ -1595,17 +1952,17 @@ declare <4 x float> @llvm.x86.avx2.gather.q.ps(<4 x float>, i8*,
 
 define <4 x float> @test_x86_avx2_gather_q_ps_256(<4 x float> %a0, i8* %a1, <4 x i64> %idx, <4 x float> %mask) {
 ; X86-LABEL: test_x86_avx2_gather_q_ps_256:
-; X86:       ## %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax ## encoding: [0x8b,0x44,0x24,0x04]
-; X86-NEXT:    vgatherqps %xmm2, (%eax,%ymm1,2), %xmm0 ## encoding: [0xc4,0xe2,0x6d,0x93,0x04,0x48]
-; X86-NEXT:    vzeroupper ## encoding: [0xc5,0xf8,0x77]
-; X86-NEXT:    retl ## encoding: [0xc3]
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-NEXT:    vgatherqps %xmm2, (%eax,%ymm1,2), %xmm0 # encoding: [0xc4,0xe2,0x6d,0x93,0x04,0x48]
+; X86-NEXT:    vzeroupper # encoding: [0xc5,0xf8,0x77]
+; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_x86_avx2_gather_q_ps_256:
-; X64:       ## %bb.0:
-; X64-NEXT:    vgatherqps %xmm2, (%rdi,%ymm1,2), %xmm0 ## encoding: [0xc4,0xe2,0x6d,0x93,0x04,0x4f]
-; X64-NEXT:    vzeroupper ## encoding: [0xc5,0xf8,0x77]
-; X64-NEXT:    retq ## encoding: [0xc3]
+; X64:       # %bb.0:
+; X64-NEXT:    vgatherqps %xmm2, (%rdi,%ymm1,2), %xmm0 # encoding: [0xc4,0xe2,0x6d,0x93,0x04,0x4f]
+; X64-NEXT:    vzeroupper # encoding: [0xc5,0xf8,0x77]
+; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <4 x float> @llvm.x86.avx2.gather.q.ps.256(<4 x float> %a0,
                             i8* %a1, <4 x i64> %idx, <4 x float> %mask, i8 2) ;
   ret <4 x float> %res
@@ -1615,15 +1972,15 @@ declare <4 x float> @llvm.x86.avx2.gather.q.ps.256(<4 x float>, i8*,
 
 define <2 x i64> @test_x86_avx2_gather_d_q(<2 x i64> %a0, i8* %a1, <4 x i32> %idx, <2 x i64> %mask) {
 ; X86-LABEL: test_x86_avx2_gather_d_q:
-; X86:       ## %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax ## encoding: [0x8b,0x44,0x24,0x04]
-; X86-NEXT:    vpgatherdq %xmm2, (%eax,%xmm1,2), %xmm0 ## encoding: [0xc4,0xe2,0xe9,0x90,0x04,0x48]
-; X86-NEXT:    retl ## encoding: [0xc3]
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-NEXT:    vpgatherdq %xmm2, (%eax,%xmm1,2), %xmm0 # encoding: [0xc4,0xe2,0xe9,0x90,0x04,0x48]
+; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_x86_avx2_gather_d_q:
-; X64:       ## %bb.0:
-; X64-NEXT:    vpgatherdq %xmm2, (%rdi,%xmm1,2), %xmm0 ## encoding: [0xc4,0xe2,0xe9,0x90,0x04,0x4f]
-; X64-NEXT:    retq ## encoding: [0xc3]
+; X64:       # %bb.0:
+; X64-NEXT:    vpgatherdq %xmm2, (%rdi,%xmm1,2), %xmm0 # encoding: [0xc4,0xe2,0xe9,0x90,0x04,0x4f]
+; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <2 x i64> @llvm.x86.avx2.gather.d.q(<2 x i64> %a0,
                             i8* %a1, <4 x i32> %idx, <2 x i64> %mask, i8 2) ;
   ret <2 x i64> %res
@@ -1633,15 +1990,15 @@ declare <2 x i64> @llvm.x86.avx2.gather.d.q(<2 x i64>, i8*,
 
 define <4 x i64> @test_x86_avx2_gather_d_q_256(<4 x i64> %a0, i8* %a1, <4 x i32> %idx, <4 x i64> %mask) {
 ; X86-LABEL: test_x86_avx2_gather_d_q_256:
-; X86:       ## %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax ## encoding: [0x8b,0x44,0x24,0x04]
-; X86-NEXT:    vpgatherdq %ymm2, (%eax,%xmm1,2), %ymm0 ## encoding: [0xc4,0xe2,0xed,0x90,0x04,0x48]
-; X86-NEXT:    retl ## encoding: [0xc3]
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-NEXT:    vpgatherdq %ymm2, (%eax,%xmm1,2), %ymm0 # encoding: [0xc4,0xe2,0xed,0x90,0x04,0x48]
+; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_x86_avx2_gather_d_q_256:
-; X64:       ## %bb.0:
-; X64-NEXT:    vpgatherdq %ymm2, (%rdi,%xmm1,2), %ymm0 ## encoding: [0xc4,0xe2,0xed,0x90,0x04,0x4f]
-; X64-NEXT:    retq ## encoding: [0xc3]
+; X64:       # %bb.0:
+; X64-NEXT:    vpgatherdq %ymm2, (%rdi,%xmm1,2), %ymm0 # encoding: [0xc4,0xe2,0xed,0x90,0x04,0x4f]
+; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <4 x i64> @llvm.x86.avx2.gather.d.q.256(<4 x i64> %a0,
                             i8* %a1, <4 x i32> %idx, <4 x i64> %mask, i8 2) ;
   ret <4 x i64> %res
@@ -1651,15 +2008,15 @@ declare <4 x i64> @llvm.x86.avx2.gather.d.q.256(<4 x i64>, i8*,
 
 define <2 x i64> @test_x86_avx2_gather_q_q(<2 x i64> %a0, i8* %a1, <2 x i64> %idx, <2 x i64> %mask) {
 ; X86-LABEL: test_x86_avx2_gather_q_q:
-; X86:       ## %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax ## encoding: [0x8b,0x44,0x24,0x04]
-; X86-NEXT:    vpgatherqq %xmm2, (%eax,%xmm1,2), %xmm0 ## encoding: [0xc4,0xe2,0xe9,0x91,0x04,0x48]
-; X86-NEXT:    retl ## encoding: [0xc3]
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-NEXT:    vpgatherqq %xmm2, (%eax,%xmm1,2), %xmm0 # encoding: [0xc4,0xe2,0xe9,0x91,0x04,0x48]
+; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_x86_avx2_gather_q_q:
-; X64:       ## %bb.0:
-; X64-NEXT:    vpgatherqq %xmm2, (%rdi,%xmm1,2), %xmm0 ## encoding: [0xc4,0xe2,0xe9,0x91,0x04,0x4f]
-; X64-NEXT:    retq ## encoding: [0xc3]
+; X64:       # %bb.0:
+; X64-NEXT:    vpgatherqq %xmm2, (%rdi,%xmm1,2), %xmm0 # encoding: [0xc4,0xe2,0xe9,0x91,0x04,0x4f]
+; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <2 x i64> @llvm.x86.avx2.gather.q.q(<2 x i64> %a0,
                             i8* %a1, <2 x i64> %idx, <2 x i64> %mask, i8 2) ;
   ret <2 x i64> %res
@@ -1669,15 +2026,15 @@ declare <2 x i64> @llvm.x86.avx2.gather.q.q(<2 x i64>, i8*,
 
 define <4 x i64> @test_x86_avx2_gather_q_q_256(<4 x i64> %a0, i8* %a1, <4 x i64> %idx, <4 x i64> %mask) {
 ; X86-LABEL: test_x86_avx2_gather_q_q_256:
-; X86:       ## %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax ## encoding: [0x8b,0x44,0x24,0x04]
-; X86-NEXT:    vpgatherqq %ymm2, (%eax,%ymm1,2), %ymm0 ## encoding: [0xc4,0xe2,0xed,0x91,0x04,0x48]
-; X86-NEXT:    retl ## encoding: [0xc3]
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-NEXT:    vpgatherqq %ymm2, (%eax,%ymm1,2), %ymm0 # encoding: [0xc4,0xe2,0xed,0x91,0x04,0x48]
+; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_x86_avx2_gather_q_q_256:
-; X64:       ## %bb.0:
-; X64-NEXT:    vpgatherqq %ymm2, (%rdi,%ymm1,2), %ymm0 ## encoding: [0xc4,0xe2,0xed,0x91,0x04,0x4f]
-; X64-NEXT:    retq ## encoding: [0xc3]
+; X64:       # %bb.0:
+; X64-NEXT:    vpgatherqq %ymm2, (%rdi,%ymm1,2), %ymm0 # encoding: [0xc4,0xe2,0xed,0x91,0x04,0x4f]
+; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <4 x i64> @llvm.x86.avx2.gather.q.q.256(<4 x i64> %a0,
                             i8* %a1, <4 x i64> %idx, <4 x i64> %mask, i8 2) ;
   ret <4 x i64> %res
@@ -1687,15 +2044,15 @@ declare <4 x i64> @llvm.x86.avx2.gather.q.q.256(<4 x i64>, i8*,
 
 define <4 x i32> @test_x86_avx2_gather_d_d(<4 x i32> %a0, i8* %a1, <4 x i32> %idx, <4 x i32> %mask) {
 ; X86-LABEL: test_x86_avx2_gather_d_d:
-; X86:       ## %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax ## encoding: [0x8b,0x44,0x24,0x04]
-; X86-NEXT:    vpgatherdd %xmm2, (%eax,%xmm1,2), %xmm0 ## encoding: [0xc4,0xe2,0x69,0x90,0x04,0x48]
-; X86-NEXT:    retl ## encoding: [0xc3]
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-NEXT:    vpgatherdd %xmm2, (%eax,%xmm1,2), %xmm0 # encoding: [0xc4,0xe2,0x69,0x90,0x04,0x48]
+; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_x86_avx2_gather_d_d:
-; X64:       ## %bb.0:
-; X64-NEXT:    vpgatherdd %xmm2, (%rdi,%xmm1,2), %xmm0 ## encoding: [0xc4,0xe2,0x69,0x90,0x04,0x4f]
-; X64-NEXT:    retq ## encoding: [0xc3]
+; X64:       # %bb.0:
+; X64-NEXT:    vpgatherdd %xmm2, (%rdi,%xmm1,2), %xmm0 # encoding: [0xc4,0xe2,0x69,0x90,0x04,0x4f]
+; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <4 x i32> @llvm.x86.avx2.gather.d.d(<4 x i32> %a0,
                             i8* %a1, <4 x i32> %idx, <4 x i32> %mask, i8 2) ;
   ret <4 x i32> %res
@@ -1705,15 +2062,15 @@ declare <4 x i32> @llvm.x86.avx2.gather.d.d(<4 x i32>, i8*,
 
 define <8 x i32> @test_x86_avx2_gather_d_d_256(<8 x i32> %a0, i8* %a1, <8 x i32> %idx, <8 x i32> %mask) {
 ; X86-LABEL: test_x86_avx2_gather_d_d_256:
-; X86:       ## %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax ## encoding: [0x8b,0x44,0x24,0x04]
-; X86-NEXT:    vpgatherdd %ymm2, (%eax,%ymm1,2), %ymm0 ## encoding: [0xc4,0xe2,0x6d,0x90,0x04,0x48]
-; X86-NEXT:    retl ## encoding: [0xc3]
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-NEXT:    vpgatherdd %ymm2, (%eax,%ymm1,2), %ymm0 # encoding: [0xc4,0xe2,0x6d,0x90,0x04,0x48]
+; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_x86_avx2_gather_d_d_256:
-; X64:       ## %bb.0:
-; X64-NEXT:    vpgatherdd %ymm2, (%rdi,%ymm1,2), %ymm0 ## encoding: [0xc4,0xe2,0x6d,0x90,0x04,0x4f]
-; X64-NEXT:    retq ## encoding: [0xc3]
+; X64:       # %bb.0:
+; X64-NEXT:    vpgatherdd %ymm2, (%rdi,%ymm1,2), %ymm0 # encoding: [0xc4,0xe2,0x6d,0x90,0x04,0x4f]
+; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <8 x i32> @llvm.x86.avx2.gather.d.d.256(<8 x i32> %a0,
                             i8* %a1, <8 x i32> %idx, <8 x i32> %mask, i8 2) ;
   ret <8 x i32> %res
@@ -1723,15 +2080,15 @@ declare <8 x i32> @llvm.x86.avx2.gather.d.d.256(<8 x i32>, i8*,
 
 define <4 x i32> @test_x86_avx2_gather_q_d(<4 x i32> %a0, i8* %a1, <2 x i64> %idx, <4 x i32> %mask) {
 ; X86-LABEL: test_x86_avx2_gather_q_d:
-; X86:       ## %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax ## encoding: [0x8b,0x44,0x24,0x04]
-; X86-NEXT:    vpgatherqd %xmm2, (%eax,%xmm1,2), %xmm0 ## encoding: [0xc4,0xe2,0x69,0x91,0x04,0x48]
-; X86-NEXT:    retl ## encoding: [0xc3]
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-NEXT:    vpgatherqd %xmm2, (%eax,%xmm1,2), %xmm0 # encoding: [0xc4,0xe2,0x69,0x91,0x04,0x48]
+; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_x86_avx2_gather_q_d:
-; X64:       ## %bb.0:
-; X64-NEXT:    vpgatherqd %xmm2, (%rdi,%xmm1,2), %xmm0 ## encoding: [0xc4,0xe2,0x69,0x91,0x04,0x4f]
-; X64-NEXT:    retq ## encoding: [0xc3]
+; X64:       # %bb.0:
+; X64-NEXT:    vpgatherqd %xmm2, (%rdi,%xmm1,2), %xmm0 # encoding: [0xc4,0xe2,0x69,0x91,0x04,0x4f]
+; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <4 x i32> @llvm.x86.avx2.gather.q.d(<4 x i32> %a0,
                             i8* %a1, <2 x i64> %idx, <4 x i32> %mask, i8 2) ;
   ret <4 x i32> %res
@@ -1741,17 +2098,17 @@ declare <4 x i32> @llvm.x86.avx2.gather.q.d(<4 x i32>, i8*,
 
 define <4 x i32> @test_x86_avx2_gather_q_d_256(<4 x i32> %a0, i8* %a1, <4 x i64> %idx, <4 x i32> %mask) {
 ; X86-LABEL: test_x86_avx2_gather_q_d_256:
-; X86:       ## %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax ## encoding: [0x8b,0x44,0x24,0x04]
-; X86-NEXT:    vpgatherqd %xmm2, (%eax,%ymm1,2), %xmm0 ## encoding: [0xc4,0xe2,0x6d,0x91,0x04,0x48]
-; X86-NEXT:    vzeroupper ## encoding: [0xc5,0xf8,0x77]
-; X86-NEXT:    retl ## encoding: [0xc3]
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-NEXT:    vpgatherqd %xmm2, (%eax,%ymm1,2), %xmm0 # encoding: [0xc4,0xe2,0x6d,0x91,0x04,0x48]
+; X86-NEXT:    vzeroupper # encoding: [0xc5,0xf8,0x77]
+; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_x86_avx2_gather_q_d_256:
-; X64:       ## %bb.0:
-; X64-NEXT:    vpgatherqd %xmm2, (%rdi,%ymm1,2), %xmm0 ## encoding: [0xc4,0xe2,0x6d,0x91,0x04,0x4f]
-; X64-NEXT:    vzeroupper ## encoding: [0xc5,0xf8,0x77]
-; X64-NEXT:    retq ## encoding: [0xc3]
+; X64:       # %bb.0:
+; X64-NEXT:    vpgatherqd %xmm2, (%rdi,%ymm1,2), %xmm0 # encoding: [0xc4,0xe2,0x6d,0x91,0x04,0x4f]
+; X64-NEXT:    vzeroupper # encoding: [0xc5,0xf8,0x77]
+; X64-NEXT:    retq # encoding: [0xc3]
   %res = call <4 x i32> @llvm.x86.avx2.gather.q.d.256(<4 x i32> %a0,
                             i8* %a1, <4 x i64> %idx, <4 x i32> %mask, i8 2) ;
   ret <4 x i32> %res
@@ -1763,36 +2120,36 @@ declare <4 x i32> @llvm.x86.avx2.gather.q.d.256(<4 x i32>, i8*,
 define <8 x float>  @test_gather_mask(<8 x float> %a0, float* %a, <8 x i32> %idx, <8 x float> %mask, float* nocapture %out) {
 ;; gather with mask
 ; X86-AVX-LABEL: test_gather_mask:
-; X86-AVX:       ## %bb.0:
-; X86-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax ## encoding: [0x8b,0x44,0x24,0x08]
-; X86-AVX-NEXT:    movl {{[0-9]+}}(%esp), %ecx ## encoding: [0x8b,0x4c,0x24,0x04]
-; X86-AVX-NEXT:    vmovaps %ymm2, %ymm3 ## encoding: [0xc5,0xfc,0x28,0xda]
-; X86-AVX-NEXT:    vgatherdps %ymm3, (%ecx,%ymm1,4), %ymm0 ## encoding: [0xc4,0xe2,0x65,0x92,0x04,0x89]
-; X86-AVX-NEXT:    vmovups %ymm2, (%eax) ## encoding: [0xc5,0xfc,0x11,0x10]
-; X86-AVX-NEXT:    retl ## encoding: [0xc3]
+; X86-AVX:       # %bb.0:
+; X86-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x08]
+; X86-AVX-NEXT:    movl {{[0-9]+}}(%esp), %ecx # encoding: [0x8b,0x4c,0x24,0x04]
+; X86-AVX-NEXT:    vmovaps %ymm2, %ymm3 # encoding: [0xc5,0xfc,0x28,0xda]
+; X86-AVX-NEXT:    vgatherdps %ymm3, (%ecx,%ymm1,4), %ymm0 # encoding: [0xc4,0xe2,0x65,0x92,0x04,0x89]
+; X86-AVX-NEXT:    vmovups %ymm2, (%eax) # encoding: [0xc5,0xfc,0x11,0x10]
+; X86-AVX-NEXT:    retl # encoding: [0xc3]
 ;
 ; X86-AVX512VL-LABEL: test_gather_mask:
-; X86-AVX512VL:       ## %bb.0:
-; X86-AVX512VL-NEXT:    movl {{[0-9]+}}(%esp), %eax ## encoding: [0x8b,0x44,0x24,0x08]
-; X86-AVX512VL-NEXT:    movl {{[0-9]+}}(%esp), %ecx ## encoding: [0x8b,0x4c,0x24,0x04]
-; X86-AVX512VL-NEXT:    vmovaps %ymm2, %ymm3 ## EVEX TO VEX Compression encoding: [0xc5,0xfc,0x28,0xda]
-; X86-AVX512VL-NEXT:    vgatherdps %ymm3, (%ecx,%ymm1,4), %ymm0 ## encoding: [0xc4,0xe2,0x65,0x92,0x04,0x89]
-; X86-AVX512VL-NEXT:    vmovups %ymm2, (%eax) ## EVEX TO VEX Compression encoding: [0xc5,0xfc,0x11,0x10]
-; X86-AVX512VL-NEXT:    retl ## encoding: [0xc3]
+; X86-AVX512VL:       # %bb.0:
+; X86-AVX512VL-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x08]
+; X86-AVX512VL-NEXT:    movl {{[0-9]+}}(%esp), %ecx # encoding: [0x8b,0x4c,0x24,0x04]
+; X86-AVX512VL-NEXT:    vmovaps %ymm2, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfc,0x28,0xda]
+; X86-AVX512VL-NEXT:    vgatherdps %ymm3, (%ecx,%ymm1,4), %ymm0 # encoding: [0xc4,0xe2,0x65,0x92,0x04,0x89]
+; X86-AVX512VL-NEXT:    vmovups %ymm2, (%eax) # EVEX TO VEX Compression encoding: [0xc5,0xfc,0x11,0x10]
+; X86-AVX512VL-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-AVX-LABEL: test_gather_mask:
-; X64-AVX:       ## %bb.0:
-; X64-AVX-NEXT:    vmovaps %ymm2, %ymm3 ## encoding: [0xc5,0xfc,0x28,0xda]
-; X64-AVX-NEXT:    vgatherdps %ymm3, (%rdi,%ymm1,4), %ymm0 ## encoding: [0xc4,0xe2,0x65,0x92,0x04,0x8f]
-; X64-AVX-NEXT:    vmovups %ymm2, (%rsi) ## encoding: [0xc5,0xfc,0x11,0x16]
-; X64-AVX-NEXT:    retq ## encoding: [0xc3]
+; X64-AVX:       # %bb.0:
+; X64-AVX-NEXT:    vmovaps %ymm2, %ymm3 # encoding: [0xc5,0xfc,0x28,0xda]
+; X64-AVX-NEXT:    vgatherdps %ymm3, (%rdi,%ymm1,4), %ymm0 # encoding: [0xc4,0xe2,0x65,0x92,0x04,0x8f]
+; X64-AVX-NEXT:    vmovups %ymm2, (%rsi) # encoding: [0xc5,0xfc,0x11,0x16]
+; X64-AVX-NEXT:    retq # encoding: [0xc3]
 ;
 ; X64-AVX512VL-LABEL: test_gather_mask:
-; X64-AVX512VL:       ## %bb.0:
-; X64-AVX512VL-NEXT:    vmovaps %ymm2, %ymm3 ## EVEX TO VEX Compression encoding: [0xc5,0xfc,0x28,0xda]
-; X64-AVX512VL-NEXT:    vgatherdps %ymm3, (%rdi,%ymm1,4), %ymm0 ## encoding: [0xc4,0xe2,0x65,0x92,0x04,0x8f]
-; X64-AVX512VL-NEXT:    vmovups %ymm2, (%rsi) ## EVEX TO VEX Compression encoding: [0xc5,0xfc,0x11,0x16]
-; X64-AVX512VL-NEXT:    retq ## encoding: [0xc3]
+; X64-AVX512VL:       # %bb.0:
+; X64-AVX512VL-NEXT:    vmovaps %ymm2, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfc,0x28,0xda]
+; X64-AVX512VL-NEXT:    vgatherdps %ymm3, (%rdi,%ymm1,4), %ymm0 # encoding: [0xc4,0xe2,0x65,0x92,0x04,0x8f]
+; X64-AVX512VL-NEXT:    vmovups %ymm2, (%rsi) # EVEX TO VEX Compression encoding: [0xc5,0xfc,0x11,0x16]
+; X64-AVX512VL-NEXT:    retq # encoding: [0xc3]
   %a_i8 = bitcast float* %a to i8*
   %res = call <8 x float> @llvm.x86.avx2.gather.d.ps.256(<8 x float> %a0,
                            i8* %a_i8, <8 x i32> %idx, <8 x float> %mask, i8 4) ;
diff --git a/test/CodeGen/X86/avx2-logic.ll b/test/CodeGen/X86/avx2-logic.ll
index 8f2207fafef9462377d78bcde8c673f6fdbc3483..f7b62eabb57356aa98b69b69c9332c11556054dd 100644
--- a/test/CodeGen/X86/avx2-logic.ll
+++ b/test/CodeGen/X86/avx2-logic.ll
@@ -91,14 +91,12 @@ define <32 x i8> @vpblendvb(<32 x i1> %cond, <32 x i8> %x, <32 x i8> %y) {
 ; X32-LABEL: vpblendvb:
 ; X32:       # %bb.0:
 ; X32-NEXT:    vpsllw $7, %ymm0, %ymm0
-; X32-NEXT:    vpand {{\.LCPI.*}}, %ymm0, %ymm0
 ; X32-NEXT:    vpblendvb %ymm0, %ymm1, %ymm2, %ymm0
 ; X32-NEXT:    retl
 ;
 ; X64-LABEL: vpblendvb:
 ; X64:       # %bb.0:
 ; X64-NEXT:    vpsllw $7, %ymm0, %ymm0
-; X64-NEXT:    vpand {{.*}}(%rip), %ymm0, %ymm0
 ; X64-NEXT:    vpblendvb %ymm0, %ymm1, %ymm2, %ymm0
 ; X64-NEXT:    retq
   %min = select <32 x i1> %cond, <32 x i8> %x, <32 x i8> %y
diff --git a/test/CodeGen/X86/avx2-schedule.ll b/test/CodeGen/X86/avx2-schedule.ll
deleted file mode 100644
index 2901827f401cf65f6098f275a26b55bf675c1e03..0000000000000000000000000000000000000000
--- a/test/CodeGen/X86/avx2-schedule.ll
+++ /dev/null
@@ -1,7111 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=x86-64 -mattr=+avx2 | FileCheck %s --check-prefix=CHECK --check-prefix=GENERIC
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=haswell -mattr=-avx512f | FileCheck %s --check-prefix=CHECK --check-prefix=HASWELL
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=broadwell -mattr=-avx512f | FileCheck %s --check-prefix=CHECK --check-prefix=BROADWELL
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=skylake -mattr=-avx512f | FileCheck %s --check-prefix=CHECK --check-prefix=SKYLAKE
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=skx -mattr=-avx512f | FileCheck %s --check-prefix=CHECK --check-prefix=SKX
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=znver1 -mattr=-avx512f | FileCheck %s --check-prefix=CHECK --check-prefix=ZNVER1
-
-define <8 x i32> @test_broadcasti128(<8 x i32> %a0, <4 x i32> *%a1) {
-; GENERIC-LABEL: test_broadcasti128:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vbroadcasti128 {{.*#+}} ymm1 = mem[0,1,0,1] sched: [6:1.00]
-; GENERIC-NEXT:    vpaddd %ymm0, %ymm1, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_broadcasti128:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vbroadcasti128 {{.*#+}} ymm1 = mem[0,1,0,1] sched: [7:0.50]
-; HASWELL-NEXT:    vpaddd %ymm0, %ymm1, %ymm0 # sched: [1:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_broadcasti128:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vbroadcasti128 {{.*#+}} ymm1 = mem[0,1,0,1] sched: [6:0.50]
-; BROADWELL-NEXT:    vpaddd %ymm0, %ymm1, %ymm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_broadcasti128:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vbroadcasti128 {{.*#+}} ymm1 = mem[0,1,0,1] sched: [7:0.50]
-; SKYLAKE-NEXT:    vpaddd %ymm0, %ymm1, %ymm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_broadcasti128:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vbroadcasti128 {{.*#+}} ymm1 = mem[0,1,0,1] sched: [7:0.50]
-; SKX-NEXT:    vpaddd %ymm0, %ymm1, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_broadcasti128:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vbroadcasti128 {{.*#+}} ymm1 = mem[0,1,0,1] sched: [8:0.50]
-; ZNVER1-NEXT:    vpaddd %ymm0, %ymm1, %ymm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = load <4 x i32>, <4 x i32> *%a1, align 16
-  %2 = shufflevector <4 x i32> %1, <4 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
-  %3 = add <8 x i32> %2, %a0
-  ret <8 x i32> %3
-}
-
-define <4 x double> @test_broadcastsd_ymm(<2 x double> %a0) {
-; GENERIC-LABEL: test_broadcastsd_ymm:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vbroadcastsd %xmm0, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vaddpd %ymm0, %ymm0, %ymm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_broadcastsd_ymm:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vbroadcastsd %xmm0, %ymm0 # sched: [3:1.00]
-; HASWELL-NEXT:    vaddpd %ymm0, %ymm0, %ymm0 # sched: [3:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_broadcastsd_ymm:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vbroadcastsd %xmm0, %ymm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    vaddpd %ymm0, %ymm0, %ymm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_broadcastsd_ymm:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vbroadcastsd %xmm0, %ymm0 # sched: [3:1.00]
-; SKYLAKE-NEXT:    vaddpd %ymm0, %ymm0, %ymm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_broadcastsd_ymm:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vbroadcastsd %xmm0, %ymm0 # sched: [3:1.00]
-; SKX-NEXT:    vaddpd %ymm0, %ymm0, %ymm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_broadcastsd_ymm:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vbroadcastsd %xmm0, %ymm0 # sched: [100:0.25]
-; ZNVER1-NEXT:    vaddpd %ymm0, %ymm0, %ymm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = shufflevector <2 x double> %a0, <2 x double> undef, <4 x i32> zeroinitializer
-  %2 = fadd <4 x double> %1, %1
-  ret <4 x double> %2
-}
-
-define <4 x float> @test_broadcastss(<4 x float> %a0) {
-; GENERIC-LABEL: test_broadcastss:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vbroadcastss %xmm0, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vaddps %xmm0, %xmm0, %xmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_broadcastss:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vbroadcastss %xmm0, %xmm0 # sched: [1:1.00]
-; HASWELL-NEXT:    vaddps %xmm0, %xmm0, %xmm0 # sched: [3:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_broadcastss:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vbroadcastss %xmm0, %xmm0 # sched: [1:1.00]
-; BROADWELL-NEXT:    vaddps %xmm0, %xmm0, %xmm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_broadcastss:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vbroadcastss %xmm0, %xmm0 # sched: [1:1.00]
-; SKYLAKE-NEXT:    vaddps %xmm0, %xmm0, %xmm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_broadcastss:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vbroadcastss %xmm0, %xmm0 # sched: [1:1.00]
-; SKX-NEXT:    vaddps %xmm0, %xmm0, %xmm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_broadcastss:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vbroadcastss %xmm0, %xmm0 # sched: [1:0.50]
-; ZNVER1-NEXT:    vaddps %xmm0, %xmm0, %xmm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = shufflevector <4 x float> %a0, <4 x float> undef, <4 x i32> zeroinitializer
-  %2 = fadd <4 x float> %1, %1
-  ret <4 x float> %2
-}
-
-define <8 x float> @test_broadcastss_ymm(<4 x float> %a0) {
-; GENERIC-LABEL: test_broadcastss_ymm:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vbroadcastss %xmm0, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vaddps %ymm0, %ymm0, %ymm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_broadcastss_ymm:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vbroadcastss %xmm0, %ymm0 # sched: [3:1.00]
-; HASWELL-NEXT:    vaddps %ymm0, %ymm0, %ymm0 # sched: [3:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_broadcastss_ymm:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vbroadcastss %xmm0, %ymm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    vaddps %ymm0, %ymm0, %ymm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_broadcastss_ymm:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vbroadcastss %xmm0, %ymm0 # sched: [3:1.00]
-; SKYLAKE-NEXT:    vaddps %ymm0, %ymm0, %ymm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_broadcastss_ymm:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vbroadcastss %xmm0, %ymm0 # sched: [3:1.00]
-; SKX-NEXT:    vaddps %ymm0, %ymm0, %ymm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_broadcastss_ymm:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vbroadcastss %xmm0, %ymm0 # sched: [100:0.25]
-; ZNVER1-NEXT:    vaddps %ymm0, %ymm0, %ymm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = shufflevector <4 x float> %a0, <4 x float> undef, <8 x i32> zeroinitializer
-  %2 = fadd <8 x float> %1, %1
-  ret <8 x float> %2
-}
-
-define <4 x i32> @test_extracti128(<8 x i16> %a0, <4 x i32> *%a1) {
-; GENERIC-LABEL: test_extracti128:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpmovzxwd {{.*#+}} ymm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero sched: [1:1.00]
-; GENERIC-NEXT:    vextracti128 $1, %ymm1, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vextracti128 $1, %ymm1, (%rdi) # sched: [1:1.00]
-; GENERIC-NEXT:    vzeroupper # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_extracti128:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpmovzxwd {{.*#+}} ymm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero sched: [3:1.00]
-; HASWELL-NEXT:    vextracti128 $1, %ymm1, %xmm0 # sched: [3:1.00]
-; HASWELL-NEXT:    vextracti128 $1, %ymm1, (%rdi) # sched: [1:1.00]
-; HASWELL-NEXT:    vzeroupper # sched: [0:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_extracti128:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpmovzxwd {{.*#+}} ymm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero sched: [3:1.00]
-; BROADWELL-NEXT:    vextracti128 $1, %ymm1, %xmm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    vextracti128 $1, %ymm1, (%rdi) # sched: [1:1.00]
-; BROADWELL-NEXT:    vzeroupper # sched: [0:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_extracti128:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpmovzxwd {{.*#+}} ymm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero sched: [3:1.00]
-; SKYLAKE-NEXT:    vextracti128 $1, %ymm1, %xmm0 # sched: [3:1.00]
-; SKYLAKE-NEXT:    vextracti128 $1, %ymm1, (%rdi) # sched: [1:1.00]
-; SKYLAKE-NEXT:    vzeroupper # sched: [0:0.67]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_extracti128:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpmovzxwd {{.*#+}} ymm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero sched: [3:1.00]
-; SKX-NEXT:    vextracti128 $1, %ymm1, %xmm0 # sched: [3:1.00]
-; SKX-NEXT:    vextracti128 $1, %ymm1, (%rdi) # sched: [1:1.00]
-; SKX-NEXT:    vzeroupper # sched: [0:0.67]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_extracti128:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpmovzxwd {{.*#+}} ymm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero sched: [1:0.50]
-; ZNVER1-NEXT:    vextracti128 $1, %ymm1, %xmm0 # sched: [2:0.25]
-; ZNVER1-NEXT:    vextracti128 $1, %ymm1, (%rdi) # sched: [1:0.50]
-; ZNVER1-NEXT:    vzeroupper # sched: [100:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %z = zext <8 x i16> %a0 to <8 x i32>
-  %ext = shufflevector <8 x i32> %z, <8 x i32> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
-  store <4 x i32> %ext, <4 x i32> *%a1
-  ret <4 x i32> %ext
-}
-
-define <2 x double> @test_gatherdpd(<2 x double> %a0, i8* %a1, <4 x i32> %a2, <2 x double> %a3) {
-; GENERIC-LABEL: test_gatherdpd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vgatherdpd %xmm2, (%rdi,%xmm1,2), %xmm0 # sched: [5:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_gatherdpd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vgatherdpd %xmm2, (%rdi,%xmm1,2), %xmm0 # sched: [26:2.67]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_gatherdpd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vgatherdpd %xmm2, (%rdi,%xmm1,2), %xmm0 # sched: [25:3.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_gatherdpd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vgatherdpd %xmm2, (%rdi,%xmm1,2), %xmm0 # sched: [22:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_gatherdpd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vgatherdpd %xmm2, (%rdi,%xmm1,2), %xmm0 # sched: [22:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_gatherdpd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vgatherdpd %xmm2, (%rdi,%xmm1,2), %xmm0 # sched: [100:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <2 x double> @llvm.x86.avx2.gather.d.pd(<2 x double> %a0, i8* %a1, <4 x i32> %a2, <2 x double> %a3, i8 2)
-  ret <2 x double> %1
-}
-declare <2 x double> @llvm.x86.avx2.gather.d.pd(<2 x double>, i8*, <4 x i32>, <2 x double>, i8) nounwind readonly
-
-define <4 x double> @test_gatherdpd_ymm(<4 x double> %a0, i8* %a1, <4 x i32> %a2, <4 x double> %a3) {
-; GENERIC-LABEL: test_gatherdpd_ymm:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vgatherdpd %ymm2, (%rdi,%xmm1,8), %ymm0 # sched: [5:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_gatherdpd_ymm:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vgatherdpd %ymm2, (%rdi,%xmm1,8), %ymm0 # sched: [27:4.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_gatherdpd_ymm:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vgatherdpd %ymm2, (%rdi,%xmm1,8), %ymm0 # sched: [26:5.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_gatherdpd_ymm:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vgatherdpd %ymm2, (%rdi,%xmm1,8), %ymm0 # sched: [25:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_gatherdpd_ymm:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vgatherdpd %ymm2, (%rdi,%xmm1,8), %ymm0 # sched: [25:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_gatherdpd_ymm:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vgatherdpd %ymm2, (%rdi,%xmm1,8), %ymm0 # sched: [100:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <4 x double> @llvm.x86.avx2.gather.d.pd.256(<4 x double> %a0, i8* %a1, <4 x i32> %a2, <4 x double> %a3, i8 8)
-  ret <4 x double> %1
-}
-declare <4 x double> @llvm.x86.avx2.gather.d.pd.256(<4 x double>, i8*, <4 x i32>, <4 x double>, i8) nounwind readonly
-
-define <4 x float> @test_gatherdps(<4 x float> %a0, i8* %a1, <4 x i32> %a2, <4 x float> %a3) {
-; GENERIC-LABEL: test_gatherdps:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vgatherdps %xmm2, (%rdi,%xmm1,2), %xmm0 # sched: [5:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_gatherdps:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vgatherdps %xmm2, (%rdi,%xmm1,2), %xmm0 # sched: [25:3.67]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_gatherdps:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vgatherdps %xmm2, (%rdi,%xmm1,2), %xmm0 # sched: [25:3.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_gatherdps:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vgatherdps %xmm2, (%rdi,%xmm1,2), %xmm0 # sched: [22:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_gatherdps:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vgatherdps %xmm2, (%rdi,%xmm1,2), %xmm0 # sched: [22:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_gatherdps:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vgatherdps %xmm2, (%rdi,%xmm1,2), %xmm0 # sched: [100:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <4 x float> @llvm.x86.avx2.gather.d.ps(<4 x float> %a0, i8* %a1, <4 x i32> %a2, <4 x float> %a3, i8 2)
-  ret <4 x float> %1
-}
-declare <4 x float> @llvm.x86.avx2.gather.d.ps(<4 x float>, i8*, <4 x i32>, <4 x float>, i8) nounwind readonly
-
-define <8 x float> @test_gatherdps_ymm(<8 x float> %a0, i8* %a1, <8 x i32> %a2, <8 x float> %a3) {
-; GENERIC-LABEL: test_gatherdps_ymm:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vgatherdps %ymm2, (%rdi,%ymm1,4), %ymm0 # sched: [5:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_gatherdps_ymm:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vgatherdps %ymm2, (%rdi,%ymm1,4), %ymm0 # sched: [27:6.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_gatherdps_ymm:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vgatherdps %ymm2, (%rdi,%ymm1,4), %ymm0 # sched: [26:4.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_gatherdps_ymm:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vgatherdps %ymm2, (%rdi,%ymm1,4), %ymm0 # sched: [25:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_gatherdps_ymm:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vgatherdps %ymm2, (%rdi,%ymm1,4), %ymm0 # sched: [25:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_gatherdps_ymm:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vgatherdps %ymm2, (%rdi,%ymm1,4), %ymm0 # sched: [100:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <8 x float> @llvm.x86.avx2.gather.d.ps.256(<8 x float> %a0, i8* %a1, <8 x i32> %a2, <8 x float> %a3, i8 4)
-  ret <8 x float> %1
-}
-declare <8 x float> @llvm.x86.avx2.gather.d.ps.256(<8 x float>, i8*, <8 x i32>, <8 x float>, i8) nounwind readonly
-
-define <2 x double> @test_gatherqpd(<2 x double> %a0, i8* %a1, <2 x i64> %a2, <2 x double> %a3) {
-; GENERIC-LABEL: test_gatherqpd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vgatherqpd %xmm2, (%rdi,%xmm1,2), %xmm0 # sched: [5:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_gatherqpd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vgatherqpd %xmm2, (%rdi,%xmm1,2), %xmm0 # sched: [23:3.33]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_gatherqpd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vgatherqpd %xmm2, (%rdi,%xmm1,2), %xmm0 # sched: [22:3.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_gatherqpd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vgatherqpd %xmm2, (%rdi,%xmm1,2), %xmm0 # sched: [22:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_gatherqpd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vgatherqpd %xmm2, (%rdi,%xmm1,2), %xmm0 # sched: [22:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_gatherqpd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vgatherqpd %xmm2, (%rdi,%xmm1,2), %xmm0 # sched: [100:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <2 x double> @llvm.x86.avx2.gather.q.pd(<2 x double> %a0, i8* %a1, <2 x i64> %a2, <2 x double> %a3, i8 2)
-  ret <2 x double> %1
-}
-declare <2 x double> @llvm.x86.avx2.gather.q.pd(<2 x double>, i8*, <2 x i64>, <2 x double>, i8) nounwind readonly
-
-define <4 x double> @test_gatherqpd_ymm(<4 x double> %a0, i8* %a1, <4 x i64> %a2, <4 x double> %a3) {
-; GENERIC-LABEL: test_gatherqpd_ymm:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vgatherqpd %ymm2, (%rdi,%ymm1,8), %ymm0 # sched: [5:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_gatherqpd_ymm:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vgatherqpd %ymm2, (%rdi,%ymm1,8), %ymm0 # sched: [24:5.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_gatherqpd_ymm:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vgatherqpd %ymm2, (%rdi,%ymm1,8), %ymm0 # sched: [23:3.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_gatherqpd_ymm:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vgatherqpd %ymm2, (%rdi,%ymm1,8), %ymm0 # sched: [25:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_gatherqpd_ymm:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vgatherqpd %ymm2, (%rdi,%ymm1,8), %ymm0 # sched: [25:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_gatherqpd_ymm:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vgatherqpd %ymm2, (%rdi,%ymm1,8), %ymm0 # sched: [100:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <4 x double> @llvm.x86.avx2.gather.q.pd.256(<4 x double> %a0, i8* %a1, <4 x i64> %a2, <4 x double> %a3, i8 8)
-  ret <4 x double> %1
-}
-declare <4 x double> @llvm.x86.avx2.gather.q.pd.256(<4 x double>, i8*, <4 x i64>, <4 x double>, i8) nounwind readonly
-
-define <4 x float> @test_gatherqps(<4 x float> %a0, i8* %a1, <2 x i64> %a2, <4 x float> %a3) {
-; GENERIC-LABEL: test_gatherqps:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vgatherqps %xmm2, (%rdi,%xmm1,2), %xmm0 # sched: [5:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_gatherqps:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vgatherqps %xmm2, (%rdi,%xmm1,2), %xmm0 # sched: [25:3.67]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_gatherqps:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vgatherqps %xmm2, (%rdi,%xmm1,2), %xmm0 # sched: [27:5.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_gatherqps:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vgatherqps %xmm2, (%rdi,%xmm1,2), %xmm0 # sched: [22:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_gatherqps:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vgatherqps %xmm2, (%rdi,%xmm1,2), %xmm0 # sched: [22:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_gatherqps:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vgatherqps %xmm2, (%rdi,%xmm1,2), %xmm0 # sched: [100:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <4 x float> @llvm.x86.avx2.gather.q.ps(<4 x float> %a0, i8* %a1, <2 x i64> %a2, <4 x float> %a3, i8 2)
-  ret <4 x float> %1
-}
-declare <4 x float> @llvm.x86.avx2.gather.q.ps(<4 x float>, i8*, <2 x i64>, <4 x float>, i8) nounwind readonly
-
-define <4 x float> @test_gatherqps_ymm(<4 x float> %a0, i8* %a1, <4 x i64> %a2, <4 x float> %a3) {
-; GENERIC-LABEL: test_gatherqps_ymm:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vgatherqps %xmm2, (%rdi,%ymm1,4), %xmm0 # sched: [5:0.50]
-; GENERIC-NEXT:    vzeroupper # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_gatherqps_ymm:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vgatherqps %xmm2, (%rdi,%ymm1,4), %xmm0 # sched: [28:3.67]
-; HASWELL-NEXT:    vzeroupper # sched: [0:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_gatherqps_ymm:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vgatherqps %xmm2, (%rdi,%ymm1,4), %xmm0 # sched: [24:5.00]
-; BROADWELL-NEXT:    vzeroupper # sched: [0:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_gatherqps_ymm:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vgatherqps %xmm2, (%rdi,%ymm1,4), %xmm0 # sched: [25:1.00]
-; SKYLAKE-NEXT:    vzeroupper # sched: [0:0.67]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_gatherqps_ymm:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vgatherqps %xmm2, (%rdi,%ymm1,4), %xmm0 # sched: [25:1.00]
-; SKX-NEXT:    vzeroupper # sched: [0:0.67]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_gatherqps_ymm:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vgatherqps %xmm2, (%rdi,%ymm1,4), %xmm0 # sched: [100:0.25]
-; ZNVER1-NEXT:    vzeroupper # sched: [100:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <4 x float> @llvm.x86.avx2.gather.q.ps.256(<4 x float> %a0, i8* %a1, <4 x i64> %a2, <4 x float> %a3, i8 4)
-  ret <4 x float> %1
-}
-declare <4 x float> @llvm.x86.avx2.gather.q.ps.256(<4 x float>, i8*, <4 x i64>, <4 x float>, i8) nounwind readonly
-
-define <8 x i32> @test_inserti128(<8 x i32> %a0, <4 x i32> %a1, <4 x i32> *%a2) {
-; GENERIC-LABEL: test_inserti128:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1 # sched: [1:1.00]
-; GENERIC-NEXT:    vinserti128 $1, (%rdi), %ymm0, %ymm0 # sched: [8:1.00]
-; GENERIC-NEXT:    vpaddd %ymm0, %ymm1, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_inserti128:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1 # sched: [3:1.00]
-; HASWELL-NEXT:    vinserti128 $1, (%rdi), %ymm0, %ymm0 # sched: [7:0.50]
-; HASWELL-NEXT:    vpaddd %ymm0, %ymm1, %ymm0 # sched: [1:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_inserti128:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1 # sched: [3:1.00]
-; BROADWELL-NEXT:    vinserti128 $1, (%rdi), %ymm0, %ymm0 # sched: [6:0.50]
-; BROADWELL-NEXT:    vpaddd %ymm0, %ymm1, %ymm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_inserti128:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1 # sched: [3:1.00]
-; SKYLAKE-NEXT:    vinserti128 $1, (%rdi), %ymm0, %ymm0 # sched: [7:0.50]
-; SKYLAKE-NEXT:    vpaddd %ymm0, %ymm1, %ymm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_inserti128:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1 # sched: [3:1.00]
-; SKX-NEXT:    vinserti128 $1, (%rdi), %ymm0, %ymm0 # sched: [7:0.50]
-; SKX-NEXT:    vpaddd %ymm0, %ymm1, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_inserti128:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1 # sched: [2:0.25]
-; ZNVER1-NEXT:    vinserti128 $1, (%rdi), %ymm0, %ymm0 # sched: [9:0.50]
-; ZNVER1-NEXT:    vpaddd %ymm0, %ymm1, %ymm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = shufflevector <4 x i32> %a1, <4 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-  %2 = shufflevector <8 x i32> %a0, <8 x i32> %1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
-  %3 = load <4 x i32>, <4 x i32> *%a2, align 16
-  %4 = shufflevector <4 x i32> %3, <4 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
-  %5 = shufflevector <8 x i32> %a0, <8 x i32> %4, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
-  %6 = add <8 x i32> %2, %5
-  ret <8 x i32> %6
-}
-
-define <4 x i64> @test_movntdqa(i8* %a0) {
-; GENERIC-LABEL: test_movntdqa:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovntdqa (%rdi), %ymm0 # sched: [7:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_movntdqa:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vmovntdqa (%rdi), %ymm0 # sched: [7:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_movntdqa:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vmovntdqa (%rdi), %ymm0 # sched: [6:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_movntdqa:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vmovntdqa (%rdi), %ymm0 # sched: [7:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_movntdqa:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovntdqa (%rdi), %ymm0 # sched: [7:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_movntdqa:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vmovntdqa (%rdi), %ymm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <4 x i64> @llvm.x86.avx2.movntdqa(i8* %a0)
-  ret <4 x i64> %1
-}
-declare <4 x i64> @llvm.x86.avx2.movntdqa(i8*) nounwind readonly
-
-define <16 x i16> @test_mpsadbw(<32 x i8> %a0, <32 x i8> %a1, <32 x i8> *%a2) {
-; GENERIC-LABEL: test_mpsadbw:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmpsadbw $7, %ymm1, %ymm0, %ymm0 # sched: [7:1.00]
-; GENERIC-NEXT:    vmpsadbw $7, (%rdi), %ymm0, %ymm0 # sched: [14:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_mpsadbw:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vmpsadbw $7, %ymm1, %ymm0, %ymm0 # sched: [7:2.00]
-; HASWELL-NEXT:    vmpsadbw $7, (%rdi), %ymm0, %ymm0 # sched: [14:2.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_mpsadbw:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vmpsadbw $7, %ymm1, %ymm0, %ymm0 # sched: [7:2.00]
-; BROADWELL-NEXT:    vmpsadbw $7, (%rdi), %ymm0, %ymm0 # sched: [13:2.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_mpsadbw:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vmpsadbw $7, %ymm1, %ymm0, %ymm0 # sched: [4:2.00]
-; SKYLAKE-NEXT:    vmpsadbw $7, (%rdi), %ymm0, %ymm0 # sched: [11:2.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_mpsadbw:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmpsadbw $7, %ymm1, %ymm0, %ymm0 # sched: [4:2.00]
-; SKX-NEXT:    vmpsadbw $7, (%rdi), %ymm0, %ymm0 # sched: [11:2.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_mpsadbw:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vmpsadbw $7, %ymm1, %ymm0, %ymm0 # sched: [100:0.25]
-; ZNVER1-NEXT:    vmpsadbw $7, (%rdi), %ymm0, %ymm0 # sched: [100:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <16 x i16> @llvm.x86.avx2.mpsadbw(<32 x i8> %a0, <32 x i8> %a1, i8 7)
-  %2 = bitcast <16 x i16> %1 to <32 x i8>
-  %3 = load <32 x i8>, <32 x i8> *%a2, align 32
-  %4 = call <16 x i16> @llvm.x86.avx2.mpsadbw(<32 x i8> %2, <32 x i8> %3, i8 7)
-  ret <16 x i16> %4
-}
-declare <16 x i16> @llvm.x86.avx2.mpsadbw(<32 x i8>, <32 x i8>, i8) nounwind readnone
-
-define <32 x i8> @test_pabsb(<32 x i8> %a0, <32 x i8> *%a1) {
-; GENERIC-LABEL: test_pabsb:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpabsb %ymm0, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    vpabsb (%rdi), %ymm1 # sched: [8:0.50]
-; GENERIC-NEXT:    vpor %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_pabsb:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpabsb %ymm0, %ymm0 # sched: [1:0.50]
-; HASWELL-NEXT:    vpabsb (%rdi), %ymm1 # sched: [8:0.50]
-; HASWELL-NEXT:    vpor %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pabsb:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpabsb %ymm0, %ymm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    vpabsb (%rdi), %ymm1 # sched: [7:0.50]
-; BROADWELL-NEXT:    vpor %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pabsb:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpabsb %ymm0, %ymm0 # sched: [1:0.50]
-; SKYLAKE-NEXT:    vpabsb (%rdi), %ymm1 # sched: [8:0.50]
-; SKYLAKE-NEXT:    vpor %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pabsb:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpabsb %ymm0, %ymm0 # sched: [1:0.50]
-; SKX-NEXT:    vpabsb (%rdi), %ymm1 # sched: [8:0.50]
-; SKX-NEXT:    vpor %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_pabsb:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpabsb (%rdi), %ymm1 # sched: [8:0.50]
-; ZNVER1-NEXT:    vpabsb %ymm0, %ymm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vpor %ymm1, %ymm0, %ymm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <32 x i8> @llvm.x86.avx2.pabs.b(<32 x i8> %a0)
-  %2 = load <32 x i8>, <32 x i8> *%a1, align 32
-  %3 = call <32 x i8> @llvm.x86.avx2.pabs.b(<32 x i8> %2)
-  %4 = or <32 x i8> %1, %3
-  ret <32 x i8> %4
-}
-declare <32 x i8> @llvm.x86.avx2.pabs.b(<32 x i8>) nounwind readnone
-
-define <8 x i32> @test_pabsd(<8 x i32> %a0, <8 x i32> *%a1) {
-; GENERIC-LABEL: test_pabsd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpabsd %ymm0, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    vpabsd (%rdi), %ymm1 # sched: [8:0.50]
-; GENERIC-NEXT:    vpor %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_pabsd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpabsd %ymm0, %ymm0 # sched: [1:0.50]
-; HASWELL-NEXT:    vpabsd (%rdi), %ymm1 # sched: [8:0.50]
-; HASWELL-NEXT:    vpor %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pabsd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpabsd %ymm0, %ymm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    vpabsd (%rdi), %ymm1 # sched: [7:0.50]
-; BROADWELL-NEXT:    vpor %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pabsd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpabsd %ymm0, %ymm0 # sched: [1:0.50]
-; SKYLAKE-NEXT:    vpabsd (%rdi), %ymm1 # sched: [8:0.50]
-; SKYLAKE-NEXT:    vpor %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pabsd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpabsd %ymm0, %ymm0 # sched: [1:0.50]
-; SKX-NEXT:    vpabsd (%rdi), %ymm1 # sched: [8:0.50]
-; SKX-NEXT:    vpor %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_pabsd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpabsd (%rdi), %ymm1 # sched: [8:0.50]
-; ZNVER1-NEXT:    vpabsd %ymm0, %ymm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vpor %ymm1, %ymm0, %ymm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <8 x i32> @llvm.x86.avx2.pabs.d(<8 x i32> %a0)
-  %2 = load <8 x i32>, <8 x i32> *%a1, align 32
-  %3 = call <8 x i32> @llvm.x86.avx2.pabs.d(<8 x i32> %2)
-  %4 = or <8 x i32> %1, %3
-  ret <8 x i32> %4
-}
-declare <8 x i32> @llvm.x86.avx2.pabs.d(<8 x i32>) nounwind readnone
-
-define <16 x i16> @test_pabsw(<16 x i16> %a0, <16 x i16> *%a1) {
-; GENERIC-LABEL: test_pabsw:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpabsw %ymm0, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    vpabsw (%rdi), %ymm1 # sched: [8:0.50]
-; GENERIC-NEXT:    vpor %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_pabsw:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpabsw %ymm0, %ymm0 # sched: [1:0.50]
-; HASWELL-NEXT:    vpabsw (%rdi), %ymm1 # sched: [8:0.50]
-; HASWELL-NEXT:    vpor %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pabsw:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpabsw %ymm0, %ymm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    vpabsw (%rdi), %ymm1 # sched: [7:0.50]
-; BROADWELL-NEXT:    vpor %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pabsw:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpabsw %ymm0, %ymm0 # sched: [1:0.50]
-; SKYLAKE-NEXT:    vpabsw (%rdi), %ymm1 # sched: [8:0.50]
-; SKYLAKE-NEXT:    vpor %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pabsw:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpabsw %ymm0, %ymm0 # sched: [1:0.50]
-; SKX-NEXT:    vpabsw (%rdi), %ymm1 # sched: [8:0.50]
-; SKX-NEXT:    vpor %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_pabsw:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpabsw (%rdi), %ymm1 # sched: [8:0.50]
-; ZNVER1-NEXT:    vpabsw %ymm0, %ymm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vpor %ymm1, %ymm0, %ymm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <16 x i16> @llvm.x86.avx2.pabs.w(<16 x i16> %a0)
-  %2 = load <16 x i16>, <16 x i16> *%a1, align 32
-  %3 = call <16 x i16> @llvm.x86.avx2.pabs.w(<16 x i16> %2)
-  %4 = or <16 x i16> %1, %3
-  ret <16 x i16> %4
-}
-declare <16 x i16> @llvm.x86.avx2.pabs.w(<16 x i16>) nounwind readnone
-
-define <16 x i16> @test_packssdw(<8 x i32> %a0, <8 x i32> %a1, <8 x i32> *%a2) {
-; GENERIC-LABEL: test_packssdw:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpackssdw %ymm1, %ymm0, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vpackssdw (%rdi), %ymm0, %ymm0 # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_packssdw:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpackssdw %ymm1, %ymm0, %ymm0 # sched: [1:1.00]
-; HASWELL-NEXT:    vpackssdw (%rdi), %ymm0, %ymm0 # sched: [8:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_packssdw:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpackssdw %ymm1, %ymm0, %ymm0 # sched: [1:1.00]
-; BROADWELL-NEXT:    vpackssdw (%rdi), %ymm0, %ymm0 # sched: [7:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_packssdw:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpackssdw %ymm1, %ymm0, %ymm0 # sched: [1:1.00]
-; SKYLAKE-NEXT:    vpackssdw (%rdi), %ymm0, %ymm0 # sched: [8:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_packssdw:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpackssdw %ymm1, %ymm0, %ymm0 # sched: [1:1.00]
-; SKX-NEXT:    vpackssdw (%rdi), %ymm0, %ymm0 # sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_packssdw:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpackssdw %ymm1, %ymm0, %ymm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vpackssdw (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <16 x i16> @llvm.x86.avx2.packssdw(<8 x i32> %a0, <8 x i32> %a1)
-  %2 = bitcast <16 x i16> %1 to <8 x i32>
-  %3 = load <8 x i32>, <8 x i32> *%a2, align 32
-  %4 = call <16 x i16> @llvm.x86.avx2.packssdw(<8 x i32> %2, <8 x i32> %3)
-  ret <16 x i16> %4
-}
-declare <16 x i16> @llvm.x86.avx2.packssdw(<8 x i32>, <8 x i32>) nounwind readnone
-
-define <32 x i8> @test_packsswb(<16 x i16> %a0, <16 x i16> %a1, <16 x i16> *%a2) {
-; GENERIC-LABEL: test_packsswb:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpacksswb %ymm1, %ymm0, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vpacksswb (%rdi), %ymm0, %ymm0 # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_packsswb:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpacksswb %ymm1, %ymm0, %ymm0 # sched: [1:1.00]
-; HASWELL-NEXT:    vpacksswb (%rdi), %ymm0, %ymm0 # sched: [8:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_packsswb:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpacksswb %ymm1, %ymm0, %ymm0 # sched: [1:1.00]
-; BROADWELL-NEXT:    vpacksswb (%rdi), %ymm0, %ymm0 # sched: [7:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_packsswb:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpacksswb %ymm1, %ymm0, %ymm0 # sched: [1:1.00]
-; SKYLAKE-NEXT:    vpacksswb (%rdi), %ymm0, %ymm0 # sched: [8:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_packsswb:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpacksswb %ymm1, %ymm0, %ymm0 # sched: [1:1.00]
-; SKX-NEXT:    vpacksswb (%rdi), %ymm0, %ymm0 # sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_packsswb:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpacksswb %ymm1, %ymm0, %ymm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vpacksswb (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <32 x i8> @llvm.x86.avx2.packsswb(<16 x i16> %a0, <16 x i16> %a1)
-  %2 = bitcast <32 x i8> %1 to <16 x i16>
-  %3 = load <16 x i16>, <16 x i16> *%a2, align 32
-  %4 = call <32 x i8> @llvm.x86.avx2.packsswb(<16 x i16> %2, <16 x i16> %3)
-  ret <32 x i8> %4
-}
-declare <32 x i8> @llvm.x86.avx2.packsswb(<16 x i16>, <16 x i16>) nounwind readnone
-
-define <16 x i16> @test_packusdw(<8 x i32> %a0, <8 x i32> %a1, <8 x i32> *%a2) {
-; GENERIC-LABEL: test_packusdw:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpackusdw %ymm1, %ymm0, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vpackusdw (%rdi), %ymm0, %ymm0 # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_packusdw:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpackusdw %ymm1, %ymm0, %ymm0 # sched: [1:1.00]
-; HASWELL-NEXT:    vpackusdw (%rdi), %ymm0, %ymm0 # sched: [8:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_packusdw:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpackusdw %ymm1, %ymm0, %ymm0 # sched: [1:1.00]
-; BROADWELL-NEXT:    vpackusdw (%rdi), %ymm0, %ymm0 # sched: [7:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_packusdw:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpackusdw %ymm1, %ymm0, %ymm0 # sched: [1:1.00]
-; SKYLAKE-NEXT:    vpackusdw (%rdi), %ymm0, %ymm0 # sched: [8:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_packusdw:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpackusdw %ymm1, %ymm0, %ymm0 # sched: [1:1.00]
-; SKX-NEXT:    vpackusdw (%rdi), %ymm0, %ymm0 # sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_packusdw:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpackusdw %ymm1, %ymm0, %ymm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vpackusdw (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <16 x i16> @llvm.x86.avx2.packusdw(<8 x i32> %a0, <8 x i32> %a1)
-  %2 = bitcast <16 x i16> %1 to <8 x i32>
-  %3 = load <8 x i32>, <8 x i32> *%a2, align 32
-  %4 = call <16 x i16> @llvm.x86.avx2.packusdw(<8 x i32> %2, <8 x i32> %3)
-  ret <16 x i16> %4
-}
-declare <16 x i16> @llvm.x86.avx2.packusdw(<8 x i32>, <8 x i32>) nounwind readnone
-
-define <32 x i8> @test_packuswb(<16 x i16> %a0, <16 x i16> %a1, <16 x i16> *%a2) {
-; GENERIC-LABEL: test_packuswb:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpackuswb %ymm1, %ymm0, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vpackuswb (%rdi), %ymm0, %ymm0 # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_packuswb:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpackuswb %ymm1, %ymm0, %ymm0 # sched: [1:1.00]
-; HASWELL-NEXT:    vpackuswb (%rdi), %ymm0, %ymm0 # sched: [8:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_packuswb:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpackuswb %ymm1, %ymm0, %ymm0 # sched: [1:1.00]
-; BROADWELL-NEXT:    vpackuswb (%rdi), %ymm0, %ymm0 # sched: [7:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_packuswb:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpackuswb %ymm1, %ymm0, %ymm0 # sched: [1:1.00]
-; SKYLAKE-NEXT:    vpackuswb (%rdi), %ymm0, %ymm0 # sched: [8:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_packuswb:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpackuswb %ymm1, %ymm0, %ymm0 # sched: [1:1.00]
-; SKX-NEXT:    vpackuswb (%rdi), %ymm0, %ymm0 # sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_packuswb:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpackuswb %ymm1, %ymm0, %ymm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vpackuswb (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <32 x i8> @llvm.x86.avx2.packuswb(<16 x i16> %a0, <16 x i16> %a1)
-  %2 = bitcast <32 x i8> %1 to <16 x i16>
-  %3 = load <16 x i16>, <16 x i16> *%a2, align 32
-  %4 = call <32 x i8> @llvm.x86.avx2.packuswb(<16 x i16> %2, <16 x i16> %3)
-  ret <32 x i8> %4
-}
-declare <32 x i8> @llvm.x86.avx2.packuswb(<16 x i16>, <16 x i16>) nounwind readnone
-
-define <32 x i8> @test_paddb(<32 x i8> %a0, <32 x i8> %a1, <32 x i8> *%a2) {
-; GENERIC-LABEL: test_paddb:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpaddb %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    vpaddb (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_paddb:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpaddb %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; HASWELL-NEXT:    vpaddb (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_paddb:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpaddb %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    vpaddb (%rdi), %ymm0, %ymm0 # sched: [7:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_paddb:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpaddb %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    vpaddb (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_paddb:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpaddb %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    vpaddb (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_paddb:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpaddb %ymm1, %ymm0, %ymm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vpaddb (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = add <32 x i8> %a0, %a1
-  %2 = load <32 x i8>, <32 x i8> *%a2, align 32
-  %3 = add <32 x i8> %1, %2
-  ret <32 x i8> %3
-}
-
-define <8 x i32> @test_paddd(<8 x i32> %a0, <8 x i32> %a1, <8 x i32> *%a2) {
-; GENERIC-LABEL: test_paddd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpaddd %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    vpaddd (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_paddd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpaddd %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; HASWELL-NEXT:    vpaddd (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_paddd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpaddd %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    vpaddd (%rdi), %ymm0, %ymm0 # sched: [7:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_paddd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpaddd %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    vpaddd (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_paddd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpaddd %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    vpaddd (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_paddd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpaddd %ymm1, %ymm0, %ymm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vpaddd (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = add <8 x i32> %a0, %a1
-  %2 = load <8 x i32>, <8 x i32> *%a2, align 32
-  %3 = add <8 x i32> %1, %2
-  ret <8 x i32> %3
-}
-
-define <4 x i64> @test_paddq(<4 x i64> %a0, <4 x i64> %a1, <4 x i64> *%a2) {
-; GENERIC-LABEL: test_paddq:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpaddq %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    vpaddq (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_paddq:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpaddq %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; HASWELL-NEXT:    vpaddq (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_paddq:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpaddq %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    vpaddq (%rdi), %ymm0, %ymm0 # sched: [7:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_paddq:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpaddq %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    vpaddq (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_paddq:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpaddq %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    vpaddq (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_paddq:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpaddq %ymm1, %ymm0, %ymm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vpaddq (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = add <4 x i64> %a0, %a1
-  %2 = load <4 x i64>, <4 x i64> *%a2, align 32
-  %3 = add <4 x i64> %1, %2
-  ret <4 x i64> %3
-}
-
-define <32 x i8> @test_paddsb(<32 x i8> %a0, <32 x i8> %a1, <32 x i8> *%a2) {
-; GENERIC-LABEL: test_paddsb:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpaddsb %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    vpaddsb (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_paddsb:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpaddsb %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; HASWELL-NEXT:    vpaddsb (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_paddsb:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpaddsb %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    vpaddsb (%rdi), %ymm0, %ymm0 # sched: [7:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_paddsb:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpaddsb %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; SKYLAKE-NEXT:    vpaddsb (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_paddsb:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpaddsb %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; SKX-NEXT:    vpaddsb (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_paddsb:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpaddsb %ymm1, %ymm0, %ymm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vpaddsb (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <32 x i8> @llvm.x86.avx2.padds.b(<32 x i8> %a0, <32 x i8> %a1)
-  %2 = load <32 x i8>, <32 x i8> *%a2, align 32
-  %3 = call <32 x i8> @llvm.x86.avx2.padds.b(<32 x i8> %1, <32 x i8> %2)
-  ret <32 x i8> %3
-}
-declare <32 x i8> @llvm.x86.avx2.padds.b(<32 x i8>, <32 x i8>) nounwind readnone
-
-define <16 x i16> @test_paddsw(<16 x i16> %a0, <16 x i16> %a1, <16 x i16> *%a2) {
-; GENERIC-LABEL: test_paddsw:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpaddsw %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    vpaddsw (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_paddsw:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpaddsw %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; HASWELL-NEXT:    vpaddsw (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_paddsw:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpaddsw %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    vpaddsw (%rdi), %ymm0, %ymm0 # sched: [7:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_paddsw:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpaddsw %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; SKYLAKE-NEXT:    vpaddsw (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_paddsw:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpaddsw %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; SKX-NEXT:    vpaddsw (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_paddsw:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpaddsw %ymm1, %ymm0, %ymm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vpaddsw (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <16 x i16> @llvm.x86.avx2.padds.w(<16 x i16> %a0, <16 x i16> %a1)
-  %2 = load <16 x i16>, <16 x i16> *%a2, align 32
-  %3 = call <16 x i16> @llvm.x86.avx2.padds.w(<16 x i16> %1, <16 x i16> %2)
-  ret <16 x i16> %3
-}
-declare <16 x i16> @llvm.x86.avx2.padds.w(<16 x i16>, <16 x i16>) nounwind readnone
-
-define <32 x i8> @test_paddusb(<32 x i8> %a0, <32 x i8> %a1, <32 x i8> *%a2) {
-; GENERIC-LABEL: test_paddusb:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpaddusb %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    vpaddusb (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_paddusb:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpaddusb %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; HASWELL-NEXT:    vpaddusb (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_paddusb:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpaddusb %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    vpaddusb (%rdi), %ymm0, %ymm0 # sched: [7:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_paddusb:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpaddusb %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; SKYLAKE-NEXT:    vpaddusb (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_paddusb:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpaddusb %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; SKX-NEXT:    vpaddusb (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_paddusb:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpaddusb %ymm1, %ymm0, %ymm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vpaddusb (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <32 x i8> @llvm.x86.avx2.paddus.b(<32 x i8> %a0, <32 x i8> %a1)
-  %2 = load <32 x i8>, <32 x i8> *%a2, align 32
-  %3 = call <32 x i8> @llvm.x86.avx2.paddus.b(<32 x i8> %1, <32 x i8> %2)
-  ret <32 x i8> %3
-}
-declare <32 x i8> @llvm.x86.avx2.paddus.b(<32 x i8>, <32 x i8>) nounwind readnone
-
-define <16 x i16> @test_paddusw(<16 x i16> %a0, <16 x i16> %a1, <16 x i16> *%a2) {
-; GENERIC-LABEL: test_paddusw:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpaddusw %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    vpaddusw (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_paddusw:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpaddusw %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; HASWELL-NEXT:    vpaddusw (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_paddusw:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpaddusw %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    vpaddusw (%rdi), %ymm0, %ymm0 # sched: [7:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_paddusw:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpaddusw %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; SKYLAKE-NEXT:    vpaddusw (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_paddusw:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpaddusw %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; SKX-NEXT:    vpaddusw (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_paddusw:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpaddusw %ymm1, %ymm0, %ymm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vpaddusw (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <16 x i16> @llvm.x86.avx2.paddus.w(<16 x i16> %a0, <16 x i16> %a1)
-  %2 = load <16 x i16>, <16 x i16> *%a2, align 32
-  %3 = call <16 x i16> @llvm.x86.avx2.paddus.w(<16 x i16> %1, <16 x i16> %2)
-  ret <16 x i16> %3
-}
-declare <16 x i16> @llvm.x86.avx2.paddus.w(<16 x i16>, <16 x i16>) nounwind readnone
-
-define <16 x i16> @test_paddw(<16 x i16> %a0, <16 x i16> %a1, <16 x i16> *%a2) {
-; GENERIC-LABEL: test_paddw:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpaddw %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    vpaddw (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_paddw:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpaddw %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; HASWELL-NEXT:    vpaddw (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_paddw:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpaddw %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    vpaddw (%rdi), %ymm0, %ymm0 # sched: [7:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_paddw:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpaddw %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    vpaddw (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_paddw:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpaddw %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    vpaddw (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_paddw:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpaddw %ymm1, %ymm0, %ymm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vpaddw (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = add <16 x i16> %a0, %a1
-  %2 = load <16 x i16>, <16 x i16> *%a2, align 32
-  %3 = add <16 x i16> %1, %2
-  ret <16 x i16> %3
-}
-
-define <32 x i8> @test_palignr(<32 x i8> %a0, <32 x i8> %a1, <32 x i8> *%a2) {
-; GENERIC-LABEL: test_palignr:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpalignr {{.*#+}} ymm1 = ymm1[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15],ymm0[0],ymm1[17,18,19,20,21,22,23,24,25,26,27,28,29,30,31],ymm0[16] sched: [1:1.00]
-; GENERIC-NEXT:    vpalignr {{.*#+}} ymm0 = ymm0[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15],ymm1[0],ymm0[17,18,19,20,21,22,23,24,25,26,27,28,29,30,31],ymm1[16] sched: [1:1.00]
-; GENERIC-NEXT:    vpaddb %ymm0, %ymm1, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_palignr:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpalignr {{.*#+}} ymm1 = ymm1[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15],ymm0[0],ymm1[17,18,19,20,21,22,23,24,25,26,27,28,29,30,31],ymm0[16] sched: [1:1.00]
-; HASWELL-NEXT:    vpalignr {{.*#+}} ymm0 = ymm0[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15],ymm1[0],ymm0[17,18,19,20,21,22,23,24,25,26,27,28,29,30,31],ymm1[16] sched: [1:1.00]
-; HASWELL-NEXT:    vpaddb %ymm0, %ymm1, %ymm0 # sched: [1:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_palignr:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpalignr {{.*#+}} ymm1 = ymm1[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15],ymm0[0],ymm1[17,18,19,20,21,22,23,24,25,26,27,28,29,30,31],ymm0[16] sched: [1:1.00]
-; BROADWELL-NEXT:    vpalignr {{.*#+}} ymm0 = ymm0[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15],ymm1[0],ymm0[17,18,19,20,21,22,23,24,25,26,27,28,29,30,31],ymm1[16] sched: [1:1.00]
-; BROADWELL-NEXT:    vpaddb %ymm0, %ymm1, %ymm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_palignr:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpalignr {{.*#+}} ymm1 = ymm1[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15],ymm0[0],ymm1[17,18,19,20,21,22,23,24,25,26,27,28,29,30,31],ymm0[16] sched: [1:1.00]
-; SKYLAKE-NEXT:    vpalignr {{.*#+}} ymm0 = ymm0[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15],ymm1[0],ymm0[17,18,19,20,21,22,23,24,25,26,27,28,29,30,31],ymm1[16] sched: [1:1.00]
-; SKYLAKE-NEXT:    vpaddb %ymm0, %ymm1, %ymm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_palignr:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpalignr {{.*#+}} ymm1 = ymm1[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15],ymm0[0],ymm1[17,18,19,20,21,22,23,24,25,26,27,28,29,30,31],ymm0[16] sched: [1:1.00]
-; SKX-NEXT:    vpalignr {{.*#+}} ymm0 = ymm0[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15],ymm1[0],ymm0[17,18,19,20,21,22,23,24,25,26,27,28,29,30,31],ymm1[16] sched: [1:1.00]
-; SKX-NEXT:    vpaddb %ymm0, %ymm1, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_palignr:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpalignr {{.*#+}} ymm1 = ymm1[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15],ymm0[0],ymm1[17,18,19,20,21,22,23,24,25,26,27,28,29,30,31],ymm0[16] sched: [1:0.25]
-; ZNVER1-NEXT:    vpalignr {{.*#+}} ymm0 = ymm0[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15],ymm1[0],ymm0[17,18,19,20,21,22,23,24,25,26,27,28,29,30,31],ymm1[16] sched: [1:0.25]
-; ZNVER1-NEXT:    vpaddb %ymm0, %ymm1, %ymm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = shufflevector <32 x i8> %a1, <32 x i8> %a0, <32 x i32> <i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 32, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 48>
-  %2 = load <32 x i8>, <32 x i8> *%a2, align 32
-  %3 = shufflevector <32 x i8> %a0, <32 x i8> %1, <32 x i32> <i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 32, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 48>
-  %4 = add <32 x i8> %1, %3
-  ret <32 x i8> %4
-}
-
-define <4 x i64> @test_pand(<4 x i64> %a0, <4 x i64> %a1, <4 x i64> *%a2) {
-; GENERIC-LABEL: test_pand:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpand %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; GENERIC-NEXT:    vpand (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; GENERIC-NEXT:    vpaddq %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_pand:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpand %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; HASWELL-NEXT:    vpand (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; HASWELL-NEXT:    vpaddq %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pand:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpand %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; BROADWELL-NEXT:    vpand (%rdi), %ymm0, %ymm0 # sched: [7:0.50]
-; BROADWELL-NEXT:    vpaddq %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pand:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpand %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    vpand (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKYLAKE-NEXT:    vpaddq %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pand:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpand %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    vpand (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKX-NEXT:    vpaddq %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_pand:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpand %ymm1, %ymm0, %ymm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vpand (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    vpaddq %ymm1, %ymm0, %ymm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = and <4 x i64> %a0, %a1
-  %2 = load <4 x i64>, <4 x i64> *%a2, align 32
-  %3 = and <4 x i64> %1, %2
-  %4 = add <4 x i64> %3, %a1
-  ret <4 x i64> %4
-}
-
-define <4 x i64> @test_pandn(<4 x i64> %a0, <4 x i64> %a1, <4 x i64> *%a2) {
-; GENERIC-LABEL: test_pandn:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpandn %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; GENERIC-NEXT:    vpandn (%rdi), %ymm0, %ymm1 # sched: [8:0.50]
-; GENERIC-NEXT:    vpaddq %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_pandn:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpandn %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; HASWELL-NEXT:    vpandn (%rdi), %ymm0, %ymm1 # sched: [8:0.50]
-; HASWELL-NEXT:    vpaddq %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pandn:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpandn %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; BROADWELL-NEXT:    vpandn (%rdi), %ymm0, %ymm1 # sched: [7:0.50]
-; BROADWELL-NEXT:    vpaddq %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pandn:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpandn %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    vpandn (%rdi), %ymm0, %ymm1 # sched: [8:0.50]
-; SKYLAKE-NEXT:    vpaddq %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pandn:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpandn %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    vpandn (%rdi), %ymm0, %ymm1 # sched: [8:0.50]
-; SKX-NEXT:    vpaddq %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_pandn:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpandn %ymm1, %ymm0, %ymm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vpandn (%rdi), %ymm0, %ymm1 # sched: [8:0.50]
-; ZNVER1-NEXT:    vpaddq %ymm1, %ymm0, %ymm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = xor <4 x i64> %a0, <i64 -1, i64 -1, i64 -1, i64 -1>
-  %2 = and <4 x i64> %a1, %1
-  %3 = load <4 x i64>, <4 x i64> *%a2, align 32
-  %4 = xor <4 x i64> %2, <i64 -1, i64 -1, i64 -1, i64 -1>
-  %5 = and <4 x i64> %3, %4
-  %6 = add <4 x i64> %2, %5
-  ret <4 x i64> %6
-}
-
-define <32 x i8> @test_pavgb(<32 x i8> %a0, <32 x i8> %a1, <32 x i8> *%a2) {
-; GENERIC-LABEL: test_pavgb:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpavgb %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    vpavgb (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_pavgb:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpavgb %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; HASWELL-NEXT:    vpavgb (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pavgb:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpavgb %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    vpavgb (%rdi), %ymm0, %ymm0 # sched: [7:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pavgb:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpavgb %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; SKYLAKE-NEXT:    vpavgb (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pavgb:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpavgb %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; SKX-NEXT:    vpavgb (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_pavgb:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpavgb %ymm1, %ymm0, %ymm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vpavgb (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = zext <32 x i8> %a0 to <32 x i16>
-  %2 = zext <32 x i8> %a1 to <32 x i16>
-  %3 = add <32 x i16> %1, %2
-  %4 = add <32 x i16> %3, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
-  %5 = lshr <32 x i16> %4, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
-  %6 = trunc <32 x i16> %5 to <32 x i8>
-  %7 = load <32 x i8>, <32 x i8> *%a2, align 32
-  %8 = zext <32 x i8> %6 to <32 x i16>
-  %9 = zext <32 x i8> %7 to <32 x i16>
-  %10 = add <32 x i16> %8, %9
-  %11 = add <32 x i16> %10, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
-  %12 = lshr <32 x i16> %11, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
-  %13 = trunc <32 x i16> %12 to <32 x i8>
-  ret <32 x i8> %13
-}
-
-define <16 x i16> @test_pavgw(<16 x i16> %a0, <16 x i16> %a1, <16 x i16> *%a2) {
-; GENERIC-LABEL: test_pavgw:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpavgw %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    vpavgw (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_pavgw:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpavgw %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; HASWELL-NEXT:    vpavgw (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pavgw:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpavgw %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    vpavgw (%rdi), %ymm0, %ymm0 # sched: [7:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pavgw:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpavgw %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; SKYLAKE-NEXT:    vpavgw (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pavgw:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpavgw %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; SKX-NEXT:    vpavgw (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_pavgw:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpavgw %ymm1, %ymm0, %ymm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vpavgw (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = zext <16 x i16> %a0 to <16 x i32>
-  %2 = zext <16 x i16> %a1 to <16 x i32>
-  %3 = add <16 x i32> %1, %2
-  %4 = add <16 x i32> %3, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
-  %5 = lshr <16 x i32> %4, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
-  %6 = trunc <16 x i32> %5 to <16 x i16>
-  %7 = load <16 x i16>, <16 x i16> *%a2, align 32
-  %8 = zext <16 x i16> %6 to <16 x i32>
-  %9 = zext <16 x i16> %7 to <16 x i32>
-  %10 = add <16 x i32> %8, %9
-  %11 = add <16 x i32> %10, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
-  %12 = lshr <16 x i32> %11, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
-  %13 = trunc <16 x i32> %12 to <16 x i16>
-  ret <16 x i16> %13
-}
-
-define <4 x i32> @test_pblendd(<4 x i32> %a0, <4 x i32> %a1, <4 x i32> *%a2) {
-; GENERIC-LABEL: test_pblendd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpblendd {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[3] sched: [1:0.50]
-; GENERIC-NEXT:    vpblendd {{.*#+}} xmm1 = mem[0],xmm1[1],mem[2],xmm1[3] sched: [7:0.50]
-; GENERIC-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_pblendd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpblendd {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[3] sched: [1:0.33]
-; HASWELL-NEXT:    vpblendd {{.*#+}} xmm1 = mem[0],xmm1[1],mem[2],xmm1[3] sched: [7:0.50]
-; HASWELL-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pblendd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpblendd {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[3] sched: [1:0.33]
-; BROADWELL-NEXT:    vpblendd {{.*#+}} xmm1 = mem[0],xmm1[1],mem[2],xmm1[3] sched: [6:0.50]
-; BROADWELL-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pblendd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpblendd {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[3] sched: [1:0.33]
-; SKYLAKE-NEXT:    vpblendd {{.*#+}} xmm1 = mem[0],xmm1[1],mem[2],xmm1[3] sched: [7:0.50]
-; SKYLAKE-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pblendd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpblendd {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[3] sched: [1:0.33]
-; SKX-NEXT:    vpblendd {{.*#+}} xmm1 = mem[0],xmm1[1],mem[2],xmm1[3] sched: [7:0.50]
-; SKX-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_pblendd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpblendd {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[3] sched: [1:0.50]
-; ZNVER1-NEXT:    vpblendd {{.*#+}} xmm1 = mem[0],xmm1[1],mem[2],xmm1[3] sched: [8:1.00]
-; ZNVER1-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = shufflevector <4 x i32> %a0, <4 x i32> %a1, <4 x i32> <i32 4, i32 5, i32 6, i32 3>
-  %2 = load <4 x i32>, <4 x i32> *%a2, align 16
-  %3 = shufflevector <4 x i32> %a1, <4 x i32> %2, <4 x i32> <i32 4, i32 1, i32 6, i32 3>
-  %4 = add <4 x i32> %1, %3
-  ret <4 x i32> %4
-}
-
-define <8 x i32> @test_pblendd_ymm(<8 x i32> %a0, <8 x i32> %a1, <8 x i32> *%a2) {
-; GENERIC-LABEL: test_pblendd_ymm:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0,1,2],ymm0[3,4,5,6],ymm1[7] sched: [1:0.50]
-; GENERIC-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0],mem[1,2],ymm1[3,4,5,6,7] sched: [8:0.50]
-; GENERIC-NEXT:    vpaddd %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_pblendd_ymm:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0,1,2],ymm0[3,4,5,6],ymm1[7] sched: [1:0.33]
-; HASWELL-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0],mem[1,2],ymm1[3,4,5,6,7] sched: [8:0.50]
-; HASWELL-NEXT:    vpaddd %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pblendd_ymm:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0,1,2],ymm0[3,4,5,6],ymm1[7] sched: [1:0.33]
-; BROADWELL-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0],mem[1,2],ymm1[3,4,5,6,7] sched: [7:0.50]
-; BROADWELL-NEXT:    vpaddd %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pblendd_ymm:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0,1,2],ymm0[3,4,5,6],ymm1[7] sched: [1:0.33]
-; SKYLAKE-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0],mem[1,2],ymm1[3,4,5,6,7] sched: [8:0.50]
-; SKYLAKE-NEXT:    vpaddd %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pblendd_ymm:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0,1,2],ymm0[3,4,5,6],ymm1[7] sched: [1:0.33]
-; SKX-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0],mem[1,2],ymm1[3,4,5,6,7] sched: [8:0.50]
-; SKX-NEXT:    vpaddd %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_pblendd_ymm:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0,1,2],ymm0[3,4,5,6],ymm1[7] sched: [1:0.50]
-; ZNVER1-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0],mem[1,2],ymm1[3,4,5,6,7] sched: [9:1.50]
-; ZNVER1-NEXT:    vpaddd %ymm1, %ymm0, %ymm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = shufflevector <8 x i32> %a0, <8 x i32> %a1, <8 x i32> <i32 8, i32 9, i32 10, i32 3, i32 4, i32 5, i32 6, i32 15>
-  %2 = load <8 x i32>, <8 x i32> *%a2, align 32
-  %3 = shufflevector <8 x i32> %a1, <8 x i32> %2, <8 x i32> <i32 0, i32 9, i32 10, i32 3, i32 4, i32 5, i32 6, i32 7>
-  %4 = add <8 x i32> %1, %3
-  ret <8 x i32> %4
-}
-
-define <32 x i8> @test_pblendvb(<32 x i8> %a0, <32 x i8> %a1, <32 x i8> %a2, <32 x i8> *%a3, <32 x i8> %a4) {
-; GENERIC-LABEL: test_pblendvb:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpblendvb %ymm2, %ymm1, %ymm0, %ymm0 # sched: [2:1.00]
-; GENERIC-NEXT:    vpblendvb %ymm3, (%rdi), %ymm0, %ymm0 # sched: [9:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_pblendvb:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpblendvb %ymm2, %ymm1, %ymm0, %ymm0 # sched: [2:2.00]
-; HASWELL-NEXT:    vpblendvb %ymm3, (%rdi), %ymm0, %ymm0 # sched: [9:2.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pblendvb:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpblendvb %ymm2, %ymm1, %ymm0, %ymm0 # sched: [2:2.00]
-; BROADWELL-NEXT:    vpblendvb %ymm3, (%rdi), %ymm0, %ymm0 # sched: [8:2.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pblendvb:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpblendvb %ymm2, %ymm1, %ymm0, %ymm0 # sched: [2:0.67]
-; SKYLAKE-NEXT:    vpblendvb %ymm3, (%rdi), %ymm0, %ymm0 # sched: [8:0.67]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pblendvb:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpblendvb %ymm2, %ymm1, %ymm0, %ymm0 # sched: [2:0.67]
-; SKX-NEXT:    vpblendvb %ymm3, (%rdi), %ymm0, %ymm0 # sched: [8:0.67]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_pblendvb:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpblendvb %ymm2, %ymm1, %ymm0, %ymm0 # sched: [1:1.00]
-; ZNVER1-NEXT:    vpblendvb %ymm3, (%rdi), %ymm0, %ymm0 # sched: [8:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <32 x i8> @llvm.x86.avx2.pblendvb(<32 x i8> %a0, <32 x i8> %a1, <32 x i8> %a2)
-  %2 = load <32 x i8>, <32 x i8> *%a3, align 32
-  %3 = call <32 x i8> @llvm.x86.avx2.pblendvb(<32 x i8> %1, <32 x i8> %2, <32 x i8> %a4)
-  ret <32 x i8> %3
-}
-declare <32 x i8> @llvm.x86.avx2.pblendvb(<32 x i8>, <32 x i8>, <32 x i8>) nounwind readnone
-
-define <16 x i16> @test_pblendw(<16 x i16> %a0, <16 x i16> %a1, <16 x i16> *%a2) {
-; GENERIC-LABEL: test_pblendw:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3,4],ymm0[5,6,7,8,9],ymm1[10,11,12],ymm0[13,14,15] sched: [1:0.50]
-; GENERIC-NEXT:    vpblendw {{.*#+}} ymm1 = mem[0],ymm1[1],mem[2],ymm1[3],mem[4],ymm1[5],mem[6],ymm1[7],mem[8],ymm1[9],mem[10],ymm1[11],mem[12],ymm1[13],mem[14],ymm1[15] sched: [8:0.50]
-; GENERIC-NEXT:    vpaddw %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_pblendw:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3,4],ymm0[5,6,7,8,9],ymm1[10,11,12],ymm0[13,14,15] sched: [1:1.00]
-; HASWELL-NEXT:    vpblendw {{.*#+}} ymm1 = mem[0],ymm1[1],mem[2],ymm1[3],mem[4],ymm1[5],mem[6],ymm1[7],mem[8],ymm1[9],mem[10],ymm1[11],mem[12],ymm1[13],mem[14],ymm1[15] sched: [8:1.00]
-; HASWELL-NEXT:    vpaddw %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pblendw:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3,4],ymm0[5,6,7,8,9],ymm1[10,11,12],ymm0[13,14,15] sched: [1:1.00]
-; BROADWELL-NEXT:    vpblendw {{.*#+}} ymm1 = mem[0],ymm1[1],mem[2],ymm1[3],mem[4],ymm1[5],mem[6],ymm1[7],mem[8],ymm1[9],mem[10],ymm1[11],mem[12],ymm1[13],mem[14],ymm1[15] sched: [7:1.00]
-; BROADWELL-NEXT:    vpaddw %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pblendw:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3,4],ymm0[5,6,7,8,9],ymm1[10,11,12],ymm0[13,14,15] sched: [1:1.00]
-; SKYLAKE-NEXT:    vpblendw {{.*#+}} ymm1 = mem[0],ymm1[1],mem[2],ymm1[3],mem[4],ymm1[5],mem[6],ymm1[7],mem[8],ymm1[9],mem[10],ymm1[11],mem[12],ymm1[13],mem[14],ymm1[15] sched: [8:1.00]
-; SKYLAKE-NEXT:    vpaddw %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pblendw:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3,4],ymm0[5,6,7,8,9],ymm1[10,11,12],ymm0[13,14,15] sched: [1:1.00]
-; SKX-NEXT:    vpblendw {{.*#+}} ymm1 = mem[0],ymm1[1],mem[2],ymm1[3],mem[4],ymm1[5],mem[6],ymm1[7],mem[8],ymm1[9],mem[10],ymm1[11],mem[12],ymm1[13],mem[14],ymm1[15] sched: [8:1.00]
-; SKX-NEXT:    vpaddw %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_pblendw:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3,4],ymm0[5,6,7,8,9],ymm1[10,11,12],ymm0[13,14,15] sched: [2:0.33]
-; ZNVER1-NEXT:    vpblendw {{.*#+}} ymm1 = mem[0],ymm1[1],mem[2],ymm1[3],mem[4],ymm1[5],mem[6],ymm1[7],mem[8],ymm1[9],mem[10],ymm1[11],mem[12],ymm1[13],mem[14],ymm1[15] sched: [9:0.50]
-; ZNVER1-NEXT:    vpaddw %ymm1, %ymm0, %ymm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = shufflevector <16 x i16> %a0, <16 x i16> %a1, <16 x i32> <i32 0, i32 1, i32 18, i32 19, i32 20, i32 5, i32 6, i32 7, i32 8, i32 9, i32 26, i32 27, i32 28, i32 13, i32 14, i32 15>
-  %2 = load <16 x i16>, <16 x i16> *%a2, align 32
-  %3 = shufflevector <16 x i16> %a1, <16 x i16> %2, <16 x i32> <i32 16, i32 1, i32 18, i32 3, i32 20, i32 5, i32 22, i32 7, i32 24, i32 9, i32 26, i32 11, i32 28, i32 13, i32 30, i32 15>
-  %4 = add <16 x i16> %1, %3
-  ret <16 x i16> %4
-}
-
-define <16 x i8> @test_pbroadcastb(<16 x i8> %a0, <16 x i8> *%a1) {
-; GENERIC-LABEL: test_pbroadcastb:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpbroadcastb %xmm0, %xmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    vpbroadcastb (%rdi), %xmm1 # sched: [7:0.50]
-; GENERIC-NEXT:    vpaddb %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_pbroadcastb:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpbroadcastb %xmm0, %xmm0 # sched: [3:1.00]
-; HASWELL-NEXT:    vpbroadcastb (%rdi), %xmm1 # sched: [9:1.00]
-; HASWELL-NEXT:    vpaddb %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pbroadcastb:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpbroadcastb (%rdi), %xmm1 # sched: [9:1.00]
-; BROADWELL-NEXT:    vpbroadcastb %xmm0, %xmm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    vpaddb %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pbroadcastb:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpbroadcastb %xmm0, %xmm0 # sched: [3:1.00]
-; SKYLAKE-NEXT:    vpbroadcastb (%rdi), %xmm1 # sched: [7:1.00]
-; SKYLAKE-NEXT:    vpaddb %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pbroadcastb:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpbroadcastb %xmm0, %xmm0 # sched: [3:1.00]
-; SKX-NEXT:    vpbroadcastb (%rdi), %xmm1 # sched: [7:1.00]
-; SKX-NEXT:    vpaddb %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_pbroadcastb:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpbroadcastb (%rdi), %xmm1 # sched: [8:1.00]
-; ZNVER1-NEXT:    vpbroadcastb %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vpaddb %xmm1, %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = shufflevector <16 x i8> %a0, <16 x i8> undef, <16 x i32> zeroinitializer
-  %2 = load <16 x i8>, <16 x i8> *%a1, align 16
-  %3 = shufflevector <16 x i8> %2, <16 x i8> undef, <16 x i32> zeroinitializer
-  %4 = add <16 x i8> %1, %3
-  ret <16 x i8> %4
-}
-
-define <32 x i8> @test_pbroadcastb_ymm(<32 x i8> %a0, <32 x i8> *%a1) {
-; GENERIC-LABEL: test_pbroadcastb_ymm:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpbroadcastb %xmm0, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vpbroadcastb (%rdi), %ymm1 # sched: [7:0.50]
-; GENERIC-NEXT:    vpaddb %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_pbroadcastb_ymm:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpbroadcastb %xmm0, %ymm0 # sched: [3:1.00]
-; HASWELL-NEXT:    vpbroadcastb (%rdi), %ymm1 # sched: [9:1.00]
-; HASWELL-NEXT:    vpaddb %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pbroadcastb_ymm:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpbroadcastb (%rdi), %ymm1 # sched: [9:1.00]
-; BROADWELL-NEXT:    vpbroadcastb %xmm0, %ymm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    vpaddb %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pbroadcastb_ymm:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpbroadcastb %xmm0, %ymm0 # sched: [3:1.00]
-; SKYLAKE-NEXT:    vpbroadcastb (%rdi), %ymm1 # sched: [8:1.00]
-; SKYLAKE-NEXT:    vpaddb %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pbroadcastb_ymm:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpbroadcastb %xmm0, %ymm0 # sched: [3:1.00]
-; SKX-NEXT:    vpbroadcastb (%rdi), %ymm1 # sched: [8:1.00]
-; SKX-NEXT:    vpaddb %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_pbroadcastb_ymm:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpbroadcastb (%rdi), %ymm1 # sched: [8:2.00]
-; ZNVER1-NEXT:    vpbroadcastb %xmm0, %ymm0 # sched: [2:0.25]
-; ZNVER1-NEXT:    vpaddb %ymm1, %ymm0, %ymm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = shufflevector <32 x i8> %a0, <32 x i8> undef, <32 x i32> zeroinitializer
-  %2 = load <32 x i8>, <32 x i8> *%a1, align 32
-  %3 = shufflevector <32 x i8> %2, <32 x i8> undef, <32 x i32> zeroinitializer
-  %4 = add <32 x i8> %1, %3
-  ret <32 x i8> %4
-}
-
-define <4 x i32> @test_pbroadcastd(<4 x i32> %a0, <4 x i32> *%a1) {
-; GENERIC-LABEL: test_pbroadcastd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpbroadcastd %xmm0, %xmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    vpbroadcastd (%rdi), %xmm1 # sched: [7:0.50]
-; GENERIC-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_pbroadcastd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpbroadcastd %xmm0, %xmm0 # sched: [1:1.00]
-; HASWELL-NEXT:    vpbroadcastd (%rdi), %xmm1 # sched: [6:0.50]
-; HASWELL-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pbroadcastd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpbroadcastd %xmm0, %xmm0 # sched: [1:1.00]
-; BROADWELL-NEXT:    vpbroadcastd (%rdi), %xmm1 # sched: [5:0.50]
-; BROADWELL-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pbroadcastd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpbroadcastd %xmm0, %xmm0 # sched: [1:1.00]
-; SKYLAKE-NEXT:    vpbroadcastd (%rdi), %xmm1 # sched: [6:0.50]
-; SKYLAKE-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pbroadcastd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpbroadcastd %xmm0, %xmm0 # sched: [1:1.00]
-; SKX-NEXT:    vpbroadcastd (%rdi), %xmm1 # sched: [6:0.50]
-; SKX-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_pbroadcastd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpbroadcastd (%rdi), %xmm1 # sched: [8:0.50]
-; ZNVER1-NEXT:    vpbroadcastd %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = shufflevector <4 x i32> %a0, <4 x i32> undef, <4 x i32> zeroinitializer
-  %2 = load <4 x i32>, <4 x i32> *%a1, align 16
-  %3 = shufflevector <4 x i32> %2, <4 x i32> undef, <4 x i32> zeroinitializer
-  %4 = add <4 x i32> %1, %3
-  ret <4 x i32> %4
-}
-
-define <8 x i32> @test_pbroadcastd_ymm(<8 x i32> %a0, <8 x i32> *%a1) {
-; GENERIC-LABEL: test_pbroadcastd_ymm:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpbroadcastd %xmm0, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vpbroadcastd (%rdi), %ymm1 # sched: [7:0.50]
-; GENERIC-NEXT:    vpaddd %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_pbroadcastd_ymm:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpbroadcastd %xmm0, %ymm0 # sched: [3:1.00]
-; HASWELL-NEXT:    vpbroadcastd (%rdi), %ymm1 # sched: [7:0.50]
-; HASWELL-NEXT:    vpaddd %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pbroadcastd_ymm:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpbroadcastd %xmm0, %ymm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    vpbroadcastd (%rdi), %ymm1 # sched: [6:0.50]
-; BROADWELL-NEXT:    vpaddd %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pbroadcastd_ymm:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpbroadcastd %xmm0, %ymm0 # sched: [3:1.00]
-; SKYLAKE-NEXT:    vpbroadcastd (%rdi), %ymm1 # sched: [7:0.50]
-; SKYLAKE-NEXT:    vpaddd %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pbroadcastd_ymm:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpbroadcastd %xmm0, %ymm0 # sched: [3:1.00]
-; SKX-NEXT:    vpbroadcastd (%rdi), %ymm1 # sched: [7:0.50]
-; SKX-NEXT:    vpaddd %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_pbroadcastd_ymm:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpbroadcastd (%rdi), %ymm1 # sched: [8:0.50]
-; ZNVER1-NEXT:    vpbroadcastd %xmm0, %ymm0 # sched: [2:0.25]
-; ZNVER1-NEXT:    vpaddd %ymm1, %ymm0, %ymm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = shufflevector <8 x i32> %a0, <8 x i32> undef, <8 x i32> zeroinitializer
-  %2 = load <8 x i32>, <8 x i32> *%a1, align 32
-  %3 = shufflevector <8 x i32> %2, <8 x i32> undef, <8 x i32> zeroinitializer
-  %4 = add <8 x i32> %1, %3
-  ret <8 x i32> %4
-}
-
-define <2 x i64> @test_pbroadcastq(<2 x i64> %a0, <2 x i64> *%a1) {
-; GENERIC-LABEL: test_pbroadcastq:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpbroadcastq %xmm0, %xmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    vpbroadcastq (%rdi), %xmm1 # sched: [7:0.50]
-; GENERIC-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_pbroadcastq:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpbroadcastq %xmm0, %xmm0 # sched: [1:1.00]
-; HASWELL-NEXT:    vpbroadcastq (%rdi), %xmm1 # sched: [6:0.50]
-; HASWELL-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pbroadcastq:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpbroadcastq %xmm0, %xmm0 # sched: [1:1.00]
-; BROADWELL-NEXT:    vpbroadcastq (%rdi), %xmm1 # sched: [5:0.50]
-; BROADWELL-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pbroadcastq:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpbroadcastq %xmm0, %xmm0 # sched: [1:1.00]
-; SKYLAKE-NEXT:    vpbroadcastq (%rdi), %xmm1 # sched: [6:0.50]
-; SKYLAKE-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pbroadcastq:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpbroadcastq %xmm0, %xmm0 # sched: [1:1.00]
-; SKX-NEXT:    vpbroadcastq (%rdi), %xmm1 # sched: [6:0.50]
-; SKX-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_pbroadcastq:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpbroadcastq (%rdi), %xmm1 # sched: [8:0.50]
-; ZNVER1-NEXT:    vpbroadcastq %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = shufflevector <2 x i64> %a0, <2 x i64> undef, <2 x i32> zeroinitializer
-  %2 = load <2 x i64>, <2 x i64> *%a1, align 16
-  %3 = shufflevector <2 x i64> %2, <2 x i64> undef, <2 x i32> zeroinitializer
-  %4 = add <2 x i64> %1, %3
-  ret <2 x i64> %4
-}
-
-define <4 x i64> @test_pbroadcastq_ymm(<4 x i64> %a0, <4 x i64> *%a1) {
-; GENERIC-LABEL: test_pbroadcastq_ymm:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpbroadcastq %xmm0, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vpbroadcastq (%rdi), %ymm1 # sched: [7:0.50]
-; GENERIC-NEXT:    vpaddq %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_pbroadcastq_ymm:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpbroadcastq %xmm0, %ymm0 # sched: [3:1.00]
-; HASWELL-NEXT:    vpbroadcastq (%rdi), %ymm1 # sched: [7:0.50]
-; HASWELL-NEXT:    vpaddq %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pbroadcastq_ymm:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpbroadcastq %xmm0, %ymm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    vpbroadcastq (%rdi), %ymm1 # sched: [6:0.50]
-; BROADWELL-NEXT:    vpaddq %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pbroadcastq_ymm:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpbroadcastq %xmm0, %ymm0 # sched: [3:1.00]
-; SKYLAKE-NEXT:    vpbroadcastq (%rdi), %ymm1 # sched: [7:0.50]
-; SKYLAKE-NEXT:    vpaddq %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pbroadcastq_ymm:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpbroadcastq %xmm0, %ymm0 # sched: [3:1.00]
-; SKX-NEXT:    vpbroadcastq (%rdi), %ymm1 # sched: [7:0.50]
-; SKX-NEXT:    vpaddq %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_pbroadcastq_ymm:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpbroadcastq (%rdi), %ymm1 # sched: [8:0.50]
-; ZNVER1-NEXT:    vpbroadcastq %xmm0, %ymm0 # sched: [2:0.25]
-; ZNVER1-NEXT:    vpaddq %ymm1, %ymm0, %ymm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = shufflevector <4 x i64> %a0, <4 x i64> undef, <4 x i32> zeroinitializer
-  %2 = load <4 x i64>, <4 x i64> *%a1, align 32
-  %3 = shufflevector <4 x i64> %2, <4 x i64> undef, <4 x i32> zeroinitializer
-  %4 = add <4 x i64> %1, %3
-  ret <4 x i64> %4
-}
-
-define <8 x i16> @test_pbroadcastw(<8 x i16> %a0, <8 x i16> *%a1) {
-; GENERIC-LABEL: test_pbroadcastw:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpbroadcastw %xmm0, %xmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    vpbroadcastw (%rdi), %xmm1 # sched: [7:0.50]
-; GENERIC-NEXT:    vpaddw %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_pbroadcastw:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpbroadcastw %xmm0, %xmm0 # sched: [3:1.00]
-; HASWELL-NEXT:    vpbroadcastw (%rdi), %xmm1 # sched: [9:1.00]
-; HASWELL-NEXT:    vpaddw %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pbroadcastw:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpbroadcastw (%rdi), %xmm1 # sched: [9:1.00]
-; BROADWELL-NEXT:    vpbroadcastw %xmm0, %xmm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    vpaddw %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pbroadcastw:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpbroadcastw %xmm0, %xmm0 # sched: [3:1.00]
-; SKYLAKE-NEXT:    vpbroadcastw (%rdi), %xmm1 # sched: [7:1.00]
-; SKYLAKE-NEXT:    vpaddw %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pbroadcastw:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpbroadcastw %xmm0, %xmm0 # sched: [3:1.00]
-; SKX-NEXT:    vpbroadcastw (%rdi), %xmm1 # sched: [7:1.00]
-; SKX-NEXT:    vpaddw %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_pbroadcastw:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpbroadcastw (%rdi), %xmm1 # sched: [8:1.00]
-; ZNVER1-NEXT:    vpbroadcastw %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vpaddw %xmm1, %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = shufflevector <8 x i16> %a0, <8 x i16> undef, <8 x i32> zeroinitializer
-  %2 = load <8 x i16>, <8 x i16> *%a1, align 16
-  %3 = shufflevector <8 x i16> %2, <8 x i16> undef, <8 x i32> zeroinitializer
-  %4 = add <8 x i16> %1, %3
-  ret <8 x i16> %4
-}
-
-define <16 x i16> @test_pbroadcastw_ymm(<16 x i16> %a0, <16 x i16> *%a1) {
-; GENERIC-LABEL: test_pbroadcastw_ymm:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpbroadcastw %xmm0, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vpbroadcastw (%rdi), %ymm1 # sched: [7:0.50]
-; GENERIC-NEXT:    vpaddw %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_pbroadcastw_ymm:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpbroadcastw %xmm0, %ymm0 # sched: [3:1.00]
-; HASWELL-NEXT:    vpbroadcastw (%rdi), %ymm1 # sched: [9:1.00]
-; HASWELL-NEXT:    vpaddw %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pbroadcastw_ymm:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpbroadcastw (%rdi), %ymm1 # sched: [9:1.00]
-; BROADWELL-NEXT:    vpbroadcastw %xmm0, %ymm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    vpaddw %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pbroadcastw_ymm:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpbroadcastw %xmm0, %ymm0 # sched: [3:1.00]
-; SKYLAKE-NEXT:    vpbroadcastw (%rdi), %ymm1 # sched: [8:1.00]
-; SKYLAKE-NEXT:    vpaddw %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pbroadcastw_ymm:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpbroadcastw %xmm0, %ymm0 # sched: [3:1.00]
-; SKX-NEXT:    vpbroadcastw (%rdi), %ymm1 # sched: [8:1.00]
-; SKX-NEXT:    vpaddw %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_pbroadcastw_ymm:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpbroadcastw (%rdi), %ymm1 # sched: [8:2.00]
-; ZNVER1-NEXT:    vpbroadcastw %xmm0, %ymm0 # sched: [2:0.25]
-; ZNVER1-NEXT:    vpaddw %ymm1, %ymm0, %ymm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = shufflevector <16 x i16> %a0, <16 x i16> undef, <16 x i32> zeroinitializer
-  %2 = load <16 x i16>, <16 x i16> *%a1, align 32
-  %3 = shufflevector <16 x i16> %2, <16 x i16> undef, <16 x i32> zeroinitializer
-  %4 = add <16 x i16> %1, %3
-  ret <16 x i16> %4
-}
-
-define <32 x i8> @test_pcmpeqb(<32 x i8> %a0, <32 x i8> %a1, <32 x i8> *%a2) {
-; GENERIC-LABEL: test_pcmpeqb:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpcmpeqb %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    vpcmpeqb (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_pcmpeqb:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpcmpeqb %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; HASWELL-NEXT:    vpcmpeqb (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pcmpeqb:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpcmpeqb %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    vpcmpeqb (%rdi), %ymm0, %ymm0 # sched: [7:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pcmpeqb:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpcmpeqb %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; SKYLAKE-NEXT:    vpcmpeqb (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pcmpeqb:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpcmpeqb %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; SKX-NEXT:    vpcmpeqb (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_pcmpeqb:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpcmpeqb %ymm1, %ymm0, %ymm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vpcmpeqb (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = icmp eq <32 x i8> %a0, %a1
-  %2 = sext <32 x i1> %1 to <32 x i8>
-  %3 = load <32 x i8>, <32 x i8> *%a2, align 32
-  %4 = icmp eq <32 x i8> %2, %3
-  %5 = sext <32 x i1> %4 to <32 x i8>
-  ret <32 x i8> %5
-}
-
-define <8 x i32> @test_pcmpeqd(<8 x i32> %a0, <8 x i32> %a1, <8 x i32> *%a2) {
-; GENERIC-LABEL: test_pcmpeqd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpcmpeqd %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    vpcmpeqd (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_pcmpeqd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpcmpeqd %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; HASWELL-NEXT:    vpcmpeqd (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pcmpeqd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpcmpeqd %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    vpcmpeqd (%rdi), %ymm0, %ymm0 # sched: [7:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pcmpeqd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpcmpeqd %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; SKYLAKE-NEXT:    vpcmpeqd (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pcmpeqd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpcmpeqd %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; SKX-NEXT:    vpcmpeqd (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_pcmpeqd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpcmpeqd %ymm1, %ymm0, %ymm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vpcmpeqd (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = icmp eq <8 x i32> %a0, %a1
-  %2 = sext <8 x i1> %1 to <8 x i32>
-  %3 = load <8 x i32>, <8 x i32> *%a2, align 32
-  %4 = icmp eq <8 x i32> %2, %3
-  %5 = sext <8 x i1> %4 to <8 x i32>
-  ret <8 x i32> %5
-}
-
-define <4 x i64> @test_pcmpeqq(<4 x i64> %a0, <4 x i64> %a1, <4 x i64> *%a2) {
-; GENERIC-LABEL: test_pcmpeqq:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpcmpeqq %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    vpcmpeqq (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_pcmpeqq:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpcmpeqq %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; HASWELL-NEXT:    vpcmpeqq (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pcmpeqq:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpcmpeqq %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    vpcmpeqq (%rdi), %ymm0, %ymm0 # sched: [7:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pcmpeqq:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpcmpeqq %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; SKYLAKE-NEXT:    vpcmpeqq (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pcmpeqq:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpcmpeqq %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; SKX-NEXT:    vpcmpeqq (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_pcmpeqq:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpcmpeqq %ymm1, %ymm0, %ymm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vpcmpeqq (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = icmp eq <4 x i64> %a0, %a1
-  %2 = sext <4 x i1> %1 to <4 x i64>
-  %3 = load <4 x i64>, <4 x i64> *%a2, align 32
-  %4 = icmp eq <4 x i64> %2, %3
-  %5 = sext <4 x i1> %4 to <4 x i64>
-  ret <4 x i64> %5
-}
-
-define <16 x i16> @test_pcmpeqw(<16 x i16> %a0, <16 x i16> %a1, <16 x i16> *%a2) {
-; GENERIC-LABEL: test_pcmpeqw:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpcmpeqw %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    vpcmpeqw (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_pcmpeqw:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpcmpeqw %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; HASWELL-NEXT:    vpcmpeqw (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pcmpeqw:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpcmpeqw %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    vpcmpeqw (%rdi), %ymm0, %ymm0 # sched: [7:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pcmpeqw:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpcmpeqw %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; SKYLAKE-NEXT:    vpcmpeqw (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pcmpeqw:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpcmpeqw %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; SKX-NEXT:    vpcmpeqw (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_pcmpeqw:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpcmpeqw %ymm1, %ymm0, %ymm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vpcmpeqw (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = icmp eq <16 x i16> %a0, %a1
-  %2 = sext <16 x i1> %1 to <16 x i16>
-  %3 = load <16 x i16>, <16 x i16> *%a2, align 32
-  %4 = icmp eq <16 x i16> %2, %3
-  %5 = sext <16 x i1> %4 to <16 x i16>
-  ret <16 x i16> %5
-}
-
-define <32 x i8> @test_pcmpgtb(<32 x i8> %a0, <32 x i8> %a1, <32 x i8> *%a2) {
-; GENERIC-LABEL: test_pcmpgtb:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpcmpgtb %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    vpcmpgtb (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_pcmpgtb:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpcmpgtb %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; HASWELL-NEXT:    vpcmpgtb (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pcmpgtb:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpcmpgtb %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    vpcmpgtb (%rdi), %ymm0, %ymm0 # sched: [7:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pcmpgtb:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpcmpgtb %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; SKYLAKE-NEXT:    vpcmpgtb (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pcmpgtb:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpcmpgtb %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; SKX-NEXT:    vpcmpgtb (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_pcmpgtb:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpcmpgtb %ymm1, %ymm0, %ymm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vpcmpgtb (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = icmp sgt <32 x i8> %a0, %a1
-  %2 = sext <32 x i1> %1 to <32 x i8>
-  %3 = load <32 x i8>, <32 x i8> *%a2, align 32
-  %4 = icmp sgt <32 x i8> %2, %3
-  %5 = sext <32 x i1> %4 to <32 x i8>
-  ret <32 x i8> %5
-}
-
-define <8 x i32> @test_pcmpgtd(<8 x i32> %a0, <8 x i32> %a1, <8 x i32> *%a2) {
-; GENERIC-LABEL: test_pcmpgtd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpcmpgtd %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    vpcmpgtd (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_pcmpgtd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpcmpgtd %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; HASWELL-NEXT:    vpcmpgtd (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pcmpgtd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpcmpgtd %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    vpcmpgtd (%rdi), %ymm0, %ymm0 # sched: [7:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pcmpgtd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpcmpgtd %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; SKYLAKE-NEXT:    vpcmpgtd (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pcmpgtd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpcmpgtd %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; SKX-NEXT:    vpcmpgtd (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_pcmpgtd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpcmpgtd %ymm1, %ymm0, %ymm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vpcmpgtd (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = icmp sgt <8 x i32> %a0, %a1
-  %2 = sext <8 x i1> %1 to <8 x i32>
-  %3 = load <8 x i32>, <8 x i32> *%a2, align 32
-  %4 = icmp sgt <8 x i32> %2, %3
-  %5 = sext <8 x i1> %4 to <8 x i32>
-  ret <8 x i32> %5
-}
-
-define <4 x i64> @test_pcmpgtq(<4 x i64> %a0, <4 x i64> %a1, <4 x i64> *%a2) {
-; GENERIC-LABEL: test_pcmpgtq:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpcmpgtq %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    vpcmpgtq (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_pcmpgtq:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpcmpgtq %ymm1, %ymm0, %ymm0 # sched: [5:1.00]
-; HASWELL-NEXT:    vpcmpgtq (%rdi), %ymm0, %ymm0 # sched: [12:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pcmpgtq:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpcmpgtq %ymm1, %ymm0, %ymm0 # sched: [5:1.00]
-; BROADWELL-NEXT:    vpcmpgtq (%rdi), %ymm0, %ymm0 # sched: [11:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pcmpgtq:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpcmpgtq %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; SKYLAKE-NEXT:    vpcmpgtq (%rdi), %ymm0, %ymm0 # sched: [10:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pcmpgtq:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpcmpgtq %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; SKX-NEXT:    vpcmpgtq (%rdi), %ymm0, %ymm0 # sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_pcmpgtq:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpcmpgtq %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; ZNVER1-NEXT:    vpcmpgtq (%rdi), %ymm0, %ymm0 # sched: [8:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = icmp sgt <4 x i64> %a0, %a1
-  %2 = sext <4 x i1> %1 to <4 x i64>
-  %3 = load <4 x i64>, <4 x i64> *%a2, align 32
-  %4 = icmp sgt <4 x i64> %2, %3
-  %5 = sext <4 x i1> %4 to <4 x i64>
-  ret <4 x i64> %5
-}
-
-define <16 x i16> @test_pcmpgtw(<16 x i16> %a0, <16 x i16> %a1, <16 x i16> *%a2) {
-; GENERIC-LABEL: test_pcmpgtw:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpcmpgtw %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    vpcmpgtw (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_pcmpgtw:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpcmpgtw %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; HASWELL-NEXT:    vpcmpgtw (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pcmpgtw:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpcmpgtw %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    vpcmpgtw (%rdi), %ymm0, %ymm0 # sched: [7:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pcmpgtw:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpcmpgtw %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; SKYLAKE-NEXT:    vpcmpgtw (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pcmpgtw:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpcmpgtw %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; SKX-NEXT:    vpcmpgtw (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_pcmpgtw:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpcmpgtw %ymm1, %ymm0, %ymm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vpcmpgtw (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = icmp sgt <16 x i16> %a0, %a1
-  %2 = sext <16 x i1> %1 to <16 x i16>
-  %3 = load <16 x i16>, <16 x i16> *%a2, align 32
-  %4 = icmp sgt <16 x i16> %2, %3
-  %5 = sext <16 x i1> %4 to <16 x i16>
-  ret <16 x i16> %5
-}
-
-define <4 x i64> @test_perm2i128(<4 x i64> %a0, <4 x i64> %a1, <4 x i64> *%a2) {
-; GENERIC-LABEL: test_perm2i128:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vperm2i128 {{.*#+}} ymm1 = ymm0[2,3],ymm1[0,1] sched: [1:1.00]
-; GENERIC-NEXT:    vperm2i128 {{.*#+}} ymm0 = ymm0[2,3],mem[0,1] sched: [8:1.00]
-; GENERIC-NEXT:    vpaddq %ymm0, %ymm1, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_perm2i128:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vperm2i128 {{.*#+}} ymm1 = ymm0[2,3],ymm1[0,1] sched: [3:1.00]
-; HASWELL-NEXT:    vperm2i128 {{.*#+}} ymm0 = ymm0[2,3],mem[0,1] sched: [10:1.00]
-; HASWELL-NEXT:    vpaddq %ymm0, %ymm1, %ymm0 # sched: [1:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_perm2i128:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vperm2i128 {{.*#+}} ymm1 = ymm0[2,3],ymm1[0,1] sched: [3:1.00]
-; BROADWELL-NEXT:    vperm2i128 {{.*#+}} ymm0 = ymm0[2,3],mem[0,1] sched: [9:1.00]
-; BROADWELL-NEXT:    vpaddq %ymm0, %ymm1, %ymm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_perm2i128:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vperm2i128 {{.*#+}} ymm1 = ymm0[2,3],ymm1[0,1] sched: [3:1.00]
-; SKYLAKE-NEXT:    vperm2i128 {{.*#+}} ymm0 = ymm0[2,3],mem[0,1] sched: [10:1.00]
-; SKYLAKE-NEXT:    vpaddq %ymm0, %ymm1, %ymm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_perm2i128:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vperm2i128 {{.*#+}} ymm1 = ymm0[2,3],ymm1[0,1] sched: [3:1.00]
-; SKX-NEXT:    vperm2i128 {{.*#+}} ymm0 = ymm0[2,3],mem[0,1] sched: [10:1.00]
-; SKX-NEXT:    vpaddq %ymm0, %ymm1, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_perm2i128:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vperm2i128 {{.*#+}} ymm1 = ymm0[2,3],ymm1[0,1] sched: [2:0.25]
-; ZNVER1-NEXT:    vperm2i128 {{.*#+}} ymm0 = ymm0[2,3],mem[0,1] sched: [9:0.50]
-; ZNVER1-NEXT:    vpaddq %ymm0, %ymm1, %ymm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = shufflevector <4 x i64> %a0, <4 x i64> %a1, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
-  %2 = load <4 x i64>, <4 x i64> *%a2, align 32
-  %3 = shufflevector <4 x i64> %a0, <4 x i64> %2, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
-  %4 = add <4 x i64> %1, %3
-  ret <4 x i64> %4
-}
-
-define <8 x i32> @test_permd(<8 x i32> %a0, <8 x i32> %a1, <8 x i32> *%a2) {
-; GENERIC-LABEL: test_permd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpermd %ymm1, %ymm0, %ymm1 # sched: [1:1.00]
-; GENERIC-NEXT:    vpermd (%rdi), %ymm0, %ymm0 # sched: [8:1.00]
-; GENERIC-NEXT:    vpaddd %ymm0, %ymm1, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_permd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpermd %ymm1, %ymm0, %ymm1 # sched: [3:1.00]
-; HASWELL-NEXT:    vpermd (%rdi), %ymm0, %ymm0 # sched: [10:1.00]
-; HASWELL-NEXT:    vpaddd %ymm0, %ymm1, %ymm0 # sched: [1:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_permd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpermd %ymm1, %ymm0, %ymm1 # sched: [3:1.00]
-; BROADWELL-NEXT:    vpermd (%rdi), %ymm0, %ymm0 # sched: [9:1.00]
-; BROADWELL-NEXT:    vpaddd %ymm0, %ymm1, %ymm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_permd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpermd %ymm1, %ymm0, %ymm1 # sched: [3:1.00]
-; SKYLAKE-NEXT:    vpermd (%rdi), %ymm0, %ymm0 # sched: [10:1.00]
-; SKYLAKE-NEXT:    vpaddd %ymm0, %ymm1, %ymm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_permd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpermd %ymm1, %ymm0, %ymm1 # sched: [3:1.00]
-; SKX-NEXT:    vpermd (%rdi), %ymm0, %ymm0 # sched: [10:1.00]
-; SKX-NEXT:    vpaddd %ymm0, %ymm1, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_permd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpermd %ymm1, %ymm0, %ymm1 # sched: [2:0.25]
-; ZNVER1-NEXT:    vpermd (%rdi), %ymm0, %ymm0 # sched: [9:0.50]
-; ZNVER1-NEXT:    vpaddd %ymm0, %ymm1, %ymm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <8 x i32> @llvm.x86.avx2.permd(<8 x i32> %a1, <8 x i32> %a0)
-  %2 = load <8 x i32>, <8 x i32> *%a2, align 32
-  %3 = call <8 x i32> @llvm.x86.avx2.permd(<8 x i32> %2, <8 x i32> %a0)
-  %4 = add <8 x i32> %1, %3
-  ret <8 x i32> %4
-}
-declare <8 x i32> @llvm.x86.avx2.permd(<8 x i32>, <8 x i32>) nounwind readonly
-
-define <4 x double> @test_permpd(<4 x double> %a0, <4 x double> *%a1) {
-; GENERIC-LABEL: test_permpd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[3,2,2,3] sched: [1:1.00]
-; GENERIC-NEXT:    vpermpd {{.*#+}} ymm1 = mem[0,2,2,3] sched: [8:1.00]
-; GENERIC-NEXT:    vaddpd %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_permpd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[3,2,2,3] sched: [3:1.00]
-; HASWELL-NEXT:    vpermpd {{.*#+}} ymm1 = mem[0,2,2,3] sched: [10:1.00]
-; HASWELL-NEXT:    vaddpd %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_permpd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[3,2,2,3] sched: [3:1.00]
-; BROADWELL-NEXT:    vpermpd {{.*#+}} ymm1 = mem[0,2,2,3] sched: [9:1.00]
-; BROADWELL-NEXT:    vaddpd %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_permpd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[3,2,2,3] sched: [3:1.00]
-; SKYLAKE-NEXT:    vpermpd {{.*#+}} ymm1 = mem[0,2,2,3] sched: [10:1.00]
-; SKYLAKE-NEXT:    vaddpd %ymm1, %ymm0, %ymm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_permpd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[3,2,2,3] sched: [3:1.00]
-; SKX-NEXT:    vpermpd {{.*#+}} ymm1 = mem[0,2,2,3] sched: [10:1.00]
-; SKX-NEXT:    vaddpd %ymm1, %ymm0, %ymm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_permpd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpermpd {{.*#+}} ymm1 = mem[0,2,2,3] sched: [107:0.50]
-; ZNVER1-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[3,2,2,3] sched: [100:0.25]
-; ZNVER1-NEXT:    vaddpd %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = shufflevector <4 x double> %a0, <4 x double> undef, <4 x i32> <i32 3, i32 2, i32 2, i32 3>
-  %2 = load <4 x double>, <4 x double> *%a1, align 32
-  %3 = shufflevector <4 x double> %2, <4 x double> undef, <4 x i32> <i32 0, i32 2, i32 2, i32 3>
-  %4 = fadd <4 x double> %1, %3
-  ret <4 x double> %4
-}
-
-define <8 x float> @test_permps(<8 x i32> %a0, <8 x float> %a1, <8 x float> *%a2) {
-; GENERIC-LABEL: test_permps:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpermps %ymm1, %ymm0, %ymm1 # sched: [1:1.00]
-; GENERIC-NEXT:    vpermps (%rdi), %ymm0, %ymm0 # sched: [8:1.00]
-; GENERIC-NEXT:    vaddps %ymm0, %ymm1, %ymm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_permps:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpermps %ymm1, %ymm0, %ymm1 # sched: [3:1.00]
-; HASWELL-NEXT:    vpermps (%rdi), %ymm0, %ymm0 # sched: [10:1.00]
-; HASWELL-NEXT:    vaddps %ymm0, %ymm1, %ymm0 # sched: [3:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_permps:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpermps %ymm1, %ymm0, %ymm1 # sched: [3:1.00]
-; BROADWELL-NEXT:    vpermps (%rdi), %ymm0, %ymm0 # sched: [9:1.00]
-; BROADWELL-NEXT:    vaddps %ymm0, %ymm1, %ymm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_permps:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpermps %ymm1, %ymm0, %ymm1 # sched: [3:1.00]
-; SKYLAKE-NEXT:    vpermps (%rdi), %ymm0, %ymm0 # sched: [10:1.00]
-; SKYLAKE-NEXT:    vaddps %ymm0, %ymm1, %ymm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_permps:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpermps %ymm1, %ymm0, %ymm1 # sched: [3:1.00]
-; SKX-NEXT:    vpermps (%rdi), %ymm0, %ymm0 # sched: [10:1.00]
-; SKX-NEXT:    vaddps %ymm0, %ymm1, %ymm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_permps:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpermps %ymm1, %ymm0, %ymm1 # sched: [100:0.25]
-; ZNVER1-NEXT:    vpermps (%rdi), %ymm0, %ymm0 # sched: [107:0.50]
-; ZNVER1-NEXT:    vaddps %ymm0, %ymm1, %ymm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <8 x float> @llvm.x86.avx2.permps(<8 x float> %a1, <8 x i32> %a0)
-  %2 = load <8 x float>, <8 x float> *%a2, align 32
-  %3 = call <8 x float> @llvm.x86.avx2.permps(<8 x float> %2, <8 x i32> %a0)
-  %4 = fadd <8 x float> %1, %3
-  ret <8 x float> %4
-}
-declare <8 x float> @llvm.x86.avx2.permps(<8 x float>, <8 x i32>) nounwind readonly
-
-define <4 x i64> @test_permq(<4 x i64> %a0, <4 x i64> *%a1) {
-; GENERIC-LABEL: test_permq:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[3,2,2,3] sched: [1:1.00]
-; GENERIC-NEXT:    vpermq {{.*#+}} ymm1 = mem[0,2,2,3] sched: [8:1.00]
-; GENERIC-NEXT:    vpaddq %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_permq:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[3,2,2,3] sched: [3:1.00]
-; HASWELL-NEXT:    vpermq {{.*#+}} ymm1 = mem[0,2,2,3] sched: [10:1.00]
-; HASWELL-NEXT:    vpaddq %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_permq:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[3,2,2,3] sched: [3:1.00]
-; BROADWELL-NEXT:    vpermq {{.*#+}} ymm1 = mem[0,2,2,3] sched: [9:1.00]
-; BROADWELL-NEXT:    vpaddq %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_permq:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[3,2,2,3] sched: [3:1.00]
-; SKYLAKE-NEXT:    vpermq {{.*#+}} ymm1 = mem[0,2,2,3] sched: [10:1.00]
-; SKYLAKE-NEXT:    vpaddq %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_permq:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[3,2,2,3] sched: [3:1.00]
-; SKX-NEXT:    vpermq {{.*#+}} ymm1 = mem[0,2,2,3] sched: [10:1.00]
-; SKX-NEXT:    vpaddq %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_permq:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpermq {{.*#+}} ymm1 = mem[0,2,2,3] sched: [9:0.50]
-; ZNVER1-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[3,2,2,3] sched: [2:0.25]
-; ZNVER1-NEXT:    vpaddq %ymm1, %ymm0, %ymm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = shufflevector <4 x i64> %a0, <4 x i64> undef, <4 x i32> <i32 3, i32 2, i32 2, i32 3>
-  %2 = load <4 x i64>, <4 x i64> *%a1, align 32
-  %3 = shufflevector <4 x i64> %2, <4 x i64> undef, <4 x i32> <i32 0, i32 2, i32 2, i32 3>
-  %4 = add <4 x i64> %1, %3
-  ret <4 x i64> %4
-}
-
-define <4 x i32> @test_pgatherdd(<4 x i32> %a0, i8* %a1, <4 x i32> %a2, <4 x i32> %a3) {
-; GENERIC-LABEL: test_pgatherdd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpgatherdd %xmm2, (%rdi,%xmm1,2), %xmm0 # sched: [5:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_pgatherdd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpgatherdd %xmm2, (%rdi,%xmm1,2), %xmm0 # sched: [26:2.67]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pgatherdd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpgatherdd %xmm2, (%rdi,%xmm1,2), %xmm0 # sched: [5:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pgatherdd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpgatherdd %xmm2, (%rdi,%xmm1,2), %xmm0 # sched: [22:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pgatherdd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpgatherdd %xmm2, (%rdi,%xmm1,2), %xmm0 # sched: [22:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_pgatherdd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpgatherdd %xmm2, (%rdi,%xmm1,2), %xmm0 # sched: [100:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <4 x i32> @llvm.x86.avx2.gather.d.d(<4 x i32> %a0, i8* %a1, <4 x i32> %a2, <4 x i32> %a3, i8 2)
-  ret <4 x i32> %1
-}
-declare <4 x i32> @llvm.x86.avx2.gather.d.d(<4 x i32>, i8*, <4 x i32>, <4 x i32>, i8) nounwind readonly
-
-define <8 x i32> @test_pgatherdd_ymm(<8 x i32> %a0, i8* %a1, <8 x i32> %a2, <8 x i32> %a3) {
-; GENERIC-LABEL: test_pgatherdd_ymm:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpgatherdd %ymm2, (%rdi,%ymm1,2), %ymm0 # sched: [5:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_pgatherdd_ymm:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpgatherdd %ymm2, (%rdi,%ymm1,2), %ymm0 # sched: [27:6.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pgatherdd_ymm:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpgatherdd %ymm2, (%rdi,%ymm1,2), %ymm0 # sched: [5:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pgatherdd_ymm:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpgatherdd %ymm2, (%rdi,%ymm1,2), %ymm0 # sched: [25:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pgatherdd_ymm:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpgatherdd %ymm2, (%rdi,%ymm1,2), %ymm0 # sched: [25:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_pgatherdd_ymm:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpgatherdd %ymm2, (%rdi,%ymm1,2), %ymm0 # sched: [100:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <8 x i32> @llvm.x86.avx2.gather.d.d.256(<8 x i32> %a0, i8* %a1, <8 x i32> %a2, <8 x i32> %a3, i8 2)
-  ret <8 x i32> %1
-}
-declare <8 x i32> @llvm.x86.avx2.gather.d.d.256(<8 x i32>, i8*, <8 x i32>, <8 x i32>, i8) nounwind readonly
-
-define <2 x i64> @test_pgatherdq(<2 x i64> %a0, i8* %a1, <4 x i32> %a2, <2 x i64> %a3) {
-; GENERIC-LABEL: test_pgatherdq:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpgatherdq %xmm2, (%rdi,%xmm1,2), %xmm0 # sched: [5:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_pgatherdq:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpgatherdq %xmm2, (%rdi,%xmm1,2), %xmm0 # sched: [26:2.67]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pgatherdq:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpgatherdq %xmm2, (%rdi,%xmm1,2), %xmm0 # sched: [5:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pgatherdq:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpgatherdq %xmm2, (%rdi,%xmm1,2), %xmm0 # sched: [22:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pgatherdq:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpgatherdq %xmm2, (%rdi,%xmm1,2), %xmm0 # sched: [22:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_pgatherdq:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpgatherdq %xmm2, (%rdi,%xmm1,2), %xmm0 # sched: [100:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <2 x i64> @llvm.x86.avx2.gather.d.q(<2 x i64> %a0, i8* %a1, <4 x i32> %a2, <2 x i64> %a3, i8 2)
-  ret <2 x i64> %1
-}
-declare <2 x i64> @llvm.x86.avx2.gather.d.q(<2 x i64>, i8*, <4 x i32>, <2 x i64>, i8) nounwind readonly
-
-define <4 x i64> @test_pgatherdq_ymm(<4 x i64> %a0, i8* %a1, <4 x i32> %a2, <4 x i64> %a3) {
-; GENERIC-LABEL: test_pgatherdq_ymm:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpgatherdq %ymm2, (%rdi,%xmm1,2), %ymm0 # sched: [5:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_pgatherdq_ymm:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpgatherdq %ymm2, (%rdi,%xmm1,2), %ymm0 # sched: [27:4.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pgatherdq_ymm:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpgatherdq %ymm2, (%rdi,%xmm1,2), %ymm0 # sched: [5:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pgatherdq_ymm:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpgatherdq %ymm2, (%rdi,%xmm1,2), %ymm0 # sched: [25:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pgatherdq_ymm:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpgatherdq %ymm2, (%rdi,%xmm1,2), %ymm0 # sched: [25:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_pgatherdq_ymm:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpgatherdq %ymm2, (%rdi,%xmm1,2), %ymm0 # sched: [100:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <4 x i64> @llvm.x86.avx2.gather.d.q.256(<4 x i64> %a0, i8* %a1, <4 x i32> %a2, <4 x i64> %a3, i8 2)
-  ret <4 x i64> %1
-}
-declare <4 x i64> @llvm.x86.avx2.gather.d.q.256(<4 x i64>, i8*, <4 x i32>, <4 x i64>, i8) nounwind readonly
-
-define <4 x i32> @test_pgatherqd(<4 x i32> %a0, i8* %a1, <2 x i64> %a2, <4 x i32> %a3) {
-; GENERIC-LABEL: test_pgatherqd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpgatherqd %xmm2, (%rdi,%xmm1,2), %xmm0 # sched: [5:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_pgatherqd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpgatherqd %xmm2, (%rdi,%xmm1,2), %xmm0 # sched: [25:5.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pgatherqd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpgatherqd %xmm2, (%rdi,%xmm1,2), %xmm0 # sched: [5:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pgatherqd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpgatherqd %xmm2, (%rdi,%xmm1,2), %xmm0 # sched: [22:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pgatherqd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpgatherqd %xmm2, (%rdi,%xmm1,2), %xmm0 # sched: [22:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_pgatherqd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpgatherqd %xmm2, (%rdi,%xmm1,2), %xmm0 # sched: [100:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <4 x i32> @llvm.x86.avx2.gather.q.d(<4 x i32> %a0, i8* %a1, <2 x i64> %a2, <4 x i32> %a3, i8 2)
-  ret <4 x i32> %1
-}
-declare <4 x i32> @llvm.x86.avx2.gather.q.d(<4 x i32>, i8*, <2 x i64>, <4 x i32>, i8) nounwind readonly
-
-define <4 x i32> @test_pgatherqd_ymm(<4 x i32> %a0, i8* %a1, <4 x i64> %a2, <4 x i32> %a3) {
-; GENERIC-LABEL: test_pgatherqd_ymm:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpgatherqd %xmm2, (%rdi,%ymm1,2), %xmm0 # sched: [5:0.50]
-; GENERIC-NEXT:    vzeroupper # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_pgatherqd_ymm:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpgatherqd %xmm2, (%rdi,%ymm1,2), %xmm0 # sched: [28:5.00]
-; HASWELL-NEXT:    vzeroupper # sched: [0:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pgatherqd_ymm:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpgatherqd %xmm2, (%rdi,%ymm1,2), %xmm0 # sched: [5:0.50]
-; BROADWELL-NEXT:    vzeroupper # sched: [0:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pgatherqd_ymm:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpgatherqd %xmm2, (%rdi,%ymm1,2), %xmm0 # sched: [25:1.00]
-; SKYLAKE-NEXT:    vzeroupper # sched: [0:0.67]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pgatherqd_ymm:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpgatherqd %xmm2, (%rdi,%ymm1,2), %xmm0 # sched: [25:1.00]
-; SKX-NEXT:    vzeroupper # sched: [0:0.67]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_pgatherqd_ymm:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpgatherqd %xmm2, (%rdi,%ymm1,2), %xmm0 # sched: [100:0.25]
-; ZNVER1-NEXT:    vzeroupper # sched: [100:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <4 x i32> @llvm.x86.avx2.gather.q.d.256(<4 x i32> %a0, i8* %a1, <4 x i64> %a2, <4 x i32> %a3, i8 2)
-  ret <4 x i32> %1
-}
-declare <4 x i32> @llvm.x86.avx2.gather.q.d.256(<4 x i32>, i8*, <4 x i64>, <4 x i32>, i8) nounwind readonly
-
-define <2 x i64> @test_pgatherqq(<2 x i64> %a0, i8 *%a1, <2 x i64> %a2, <2 x i64> %a3) {
-; GENERIC-LABEL: test_pgatherqq:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpgatherqq %xmm2, (%rdi,%xmm1,2), %xmm0 # sched: [5:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_pgatherqq:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpgatherqq %xmm2, (%rdi,%xmm1,2), %xmm0 # sched: [23:3.33]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pgatherqq:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpgatherqq %xmm2, (%rdi,%xmm1,2), %xmm0 # sched: [5:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pgatherqq:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpgatherqq %xmm2, (%rdi,%xmm1,2), %xmm0 # sched: [22:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pgatherqq:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpgatherqq %xmm2, (%rdi,%xmm1,2), %xmm0 # sched: [22:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_pgatherqq:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpgatherqq %xmm2, (%rdi,%xmm1,2), %xmm0 # sched: [100:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <2 x i64> @llvm.x86.avx2.gather.q.q(<2 x i64> %a0, i8* %a1, <2 x i64> %a2, <2 x i64> %a3, i8 2)
-  ret <2 x i64> %1
-}
-declare <2 x i64> @llvm.x86.avx2.gather.q.q(<2 x i64>, i8*, <2 x i64>, <2 x i64>, i8) nounwind readonly
-
-define <4 x i64> @test_pgatherqq_ymm(<4 x i64> %a0, i8 *%a1, <4 x i64> %a2, <4 x i64> %a3) {
-; GENERIC-LABEL: test_pgatherqq_ymm:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpgatherqq %ymm2, (%rdi,%ymm1,2), %ymm0 # sched: [5:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_pgatherqq_ymm:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpgatherqq %ymm2, (%rdi,%ymm1,2), %ymm0 # sched: [24:5.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pgatherqq_ymm:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpgatherqq %ymm2, (%rdi,%ymm1,2), %ymm0 # sched: [5:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pgatherqq_ymm:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpgatherqq %ymm2, (%rdi,%ymm1,2), %ymm0 # sched: [25:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pgatherqq_ymm:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpgatherqq %ymm2, (%rdi,%ymm1,2), %ymm0 # sched: [25:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_pgatherqq_ymm:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpgatherqq %ymm2, (%rdi,%ymm1,2), %ymm0 # sched: [100:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <4 x i64> @llvm.x86.avx2.gather.q.q.256(<4 x i64> %a0, i8* %a1, <4 x i64> %a2, <4 x i64> %a3, i8 2)
-  ret <4 x i64> %1
-}
-declare <4 x i64> @llvm.x86.avx2.gather.q.q.256(<4 x i64>, i8*, <4 x i64>, <4 x i64>, i8) nounwind readonly
-
-define <8 x i32> @test_phaddd(<8 x i32> %a0, <8 x i32> %a1, <8 x i32> *%a2) {
-; GENERIC-LABEL: test_phaddd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vphaddd %ymm1, %ymm0, %ymm0 # sched: [3:1.50]
-; GENERIC-NEXT:    vphaddd (%rdi), %ymm0, %ymm0 # sched: [10:1.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_phaddd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vphaddd %ymm1, %ymm0, %ymm0 # sched: [3:2.00]
-; HASWELL-NEXT:    vphaddd (%rdi), %ymm0, %ymm0 # sched: [10:2.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_phaddd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vphaddd %ymm1, %ymm0, %ymm0 # sched: [3:2.00]
-; BROADWELL-NEXT:    vphaddd (%rdi), %ymm0, %ymm0 # sched: [9:2.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_phaddd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vphaddd %ymm1, %ymm0, %ymm0 # sched: [3:2.00]
-; SKYLAKE-NEXT:    vphaddd (%rdi), %ymm0, %ymm0 # sched: [10:2.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_phaddd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vphaddd %ymm1, %ymm0, %ymm0 # sched: [3:2.00]
-; SKX-NEXT:    vphaddd (%rdi), %ymm0, %ymm0 # sched: [10:2.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_phaddd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vphaddd %ymm1, %ymm0, %ymm0 # sched: [100:0.25]
-; ZNVER1-NEXT:    vphaddd (%rdi), %ymm0, %ymm0 # sched: [100:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <8 x i32> @llvm.x86.avx2.phadd.d(<8 x i32> %a0, <8 x i32> %a1)
-  %2 = load <8 x i32>, <8 x i32> *%a2, align 32
-  %3 = call <8 x i32> @llvm.x86.avx2.phadd.d(<8 x i32> %1, <8 x i32> %2)
-  ret <8 x i32> %3
-}
-declare <8 x i32> @llvm.x86.avx2.phadd.d(<8 x i32>, <8 x i32>) nounwind readnone
-
-define <16 x i16> @test_phaddsw(<16 x i16> %a0, <16 x i16> %a1, <16 x i16> *%a2) {
-; GENERIC-LABEL: test_phaddsw:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vphaddsw %ymm1, %ymm0, %ymm0 # sched: [3:1.50]
-; GENERIC-NEXT:    vphaddsw (%rdi), %ymm0, %ymm0 # sched: [10:1.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_phaddsw:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vphaddsw %ymm1, %ymm0, %ymm0 # sched: [3:2.00]
-; HASWELL-NEXT:    vphaddsw (%rdi), %ymm0, %ymm0 # sched: [10:2.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_phaddsw:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vphaddsw %ymm1, %ymm0, %ymm0 # sched: [3:2.00]
-; BROADWELL-NEXT:    vphaddsw (%rdi), %ymm0, %ymm0 # sched: [9:2.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_phaddsw:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vphaddsw %ymm1, %ymm0, %ymm0 # sched: [3:2.00]
-; SKYLAKE-NEXT:    vphaddsw (%rdi), %ymm0, %ymm0 # sched: [10:2.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_phaddsw:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vphaddsw %ymm1, %ymm0, %ymm0 # sched: [3:2.00]
-; SKX-NEXT:    vphaddsw (%rdi), %ymm0, %ymm0 # sched: [10:2.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_phaddsw:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vphaddsw %ymm1, %ymm0, %ymm0 # sched: [100:0.25]
-; ZNVER1-NEXT:    vphaddsw (%rdi), %ymm0, %ymm0 # sched: [100:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <16 x i16> @llvm.x86.avx2.phadd.sw(<16 x i16> %a0, <16 x i16> %a1)
-  %2 = load <16 x i16>, <16 x i16> *%a2, align 32
-  %3 = call <16 x i16> @llvm.x86.avx2.phadd.sw(<16 x i16> %1, <16 x i16> %2)
-  ret <16 x i16> %3
-}
-declare <16 x i16> @llvm.x86.avx2.phadd.sw(<16 x i16>, <16 x i16>) nounwind readnone
-
-define <16 x i16> @test_phaddw(<16 x i16> %a0, <16 x i16> %a1, <16 x i16> *%a2) {
-; GENERIC-LABEL: test_phaddw:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vphaddw %ymm1, %ymm0, %ymm0 # sched: [3:1.50]
-; GENERIC-NEXT:    vphaddw (%rdi), %ymm0, %ymm0 # sched: [10:1.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_phaddw:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vphaddw %ymm1, %ymm0, %ymm0 # sched: [3:2.00]
-; HASWELL-NEXT:    vphaddw (%rdi), %ymm0, %ymm0 # sched: [10:2.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_phaddw:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vphaddw %ymm1, %ymm0, %ymm0 # sched: [3:2.00]
-; BROADWELL-NEXT:    vphaddw (%rdi), %ymm0, %ymm0 # sched: [9:2.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_phaddw:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vphaddw %ymm1, %ymm0, %ymm0 # sched: [3:2.00]
-; SKYLAKE-NEXT:    vphaddw (%rdi), %ymm0, %ymm0 # sched: [10:2.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_phaddw:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vphaddw %ymm1, %ymm0, %ymm0 # sched: [3:2.00]
-; SKX-NEXT:    vphaddw (%rdi), %ymm0, %ymm0 # sched: [10:2.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_phaddw:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vphaddw %ymm1, %ymm0, %ymm0 # sched: [100:0.25]
-; ZNVER1-NEXT:    vphaddw (%rdi), %ymm0, %ymm0 # sched: [100:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <16 x i16> @llvm.x86.avx2.phadd.w(<16 x i16> %a0, <16 x i16> %a1)
-  %2 = load <16 x i16>, <16 x i16> *%a2, align 32
-  %3 = call <16 x i16> @llvm.x86.avx2.phadd.w(<16 x i16> %1, <16 x i16> %2)
-  ret <16 x i16> %3
-}
-declare <16 x i16> @llvm.x86.avx2.phadd.w(<16 x i16>, <16 x i16>) nounwind readnone
-
-define <8 x i32> @test_phsubd(<8 x i32> %a0, <8 x i32> %a1, <8 x i32> *%a2) {
-; GENERIC-LABEL: test_phsubd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vphsubd %ymm1, %ymm0, %ymm0 # sched: [3:1.50]
-; GENERIC-NEXT:    vphsubd (%rdi), %ymm0, %ymm0 # sched: [10:1.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_phsubd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vphsubd %ymm1, %ymm0, %ymm0 # sched: [3:2.00]
-; HASWELL-NEXT:    vphsubd (%rdi), %ymm0, %ymm0 # sched: [10:2.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_phsubd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vphsubd %ymm1, %ymm0, %ymm0 # sched: [3:2.00]
-; BROADWELL-NEXT:    vphsubd (%rdi), %ymm0, %ymm0 # sched: [9:2.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_phsubd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vphsubd %ymm1, %ymm0, %ymm0 # sched: [3:2.00]
-; SKYLAKE-NEXT:    vphsubd (%rdi), %ymm0, %ymm0 # sched: [10:2.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_phsubd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vphsubd %ymm1, %ymm0, %ymm0 # sched: [3:2.00]
-; SKX-NEXT:    vphsubd (%rdi), %ymm0, %ymm0 # sched: [10:2.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_phsubd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vphsubd %ymm1, %ymm0, %ymm0 # sched: [100:0.25]
-; ZNVER1-NEXT:    vphsubd (%rdi), %ymm0, %ymm0 # sched: [100:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <8 x i32> @llvm.x86.avx2.phsub.d(<8 x i32> %a0, <8 x i32> %a1)
-  %2 = load <8 x i32>, <8 x i32> *%a2, align 32
-  %3 = call <8 x i32> @llvm.x86.avx2.phsub.d(<8 x i32> %1, <8 x i32> %2)
-  ret <8 x i32> %3
-}
-declare <8 x i32> @llvm.x86.avx2.phsub.d(<8 x i32>, <8 x i32>) nounwind readnone
-
-define <16 x i16> @test_phsubsw(<16 x i16> %a0, <16 x i16> %a1, <16 x i16> *%a2) {
-; GENERIC-LABEL: test_phsubsw:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vphsubsw %ymm1, %ymm0, %ymm0 # sched: [3:1.50]
-; GENERIC-NEXT:    vphsubsw (%rdi), %ymm0, %ymm0 # sched: [10:1.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_phsubsw:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vphsubsw %ymm1, %ymm0, %ymm0 # sched: [3:2.00]
-; HASWELL-NEXT:    vphsubsw (%rdi), %ymm0, %ymm0 # sched: [10:2.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_phsubsw:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vphsubsw %ymm1, %ymm0, %ymm0 # sched: [3:2.00]
-; BROADWELL-NEXT:    vphsubsw (%rdi), %ymm0, %ymm0 # sched: [9:2.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_phsubsw:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vphsubsw %ymm1, %ymm0, %ymm0 # sched: [3:2.00]
-; SKYLAKE-NEXT:    vphsubsw (%rdi), %ymm0, %ymm0 # sched: [10:2.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_phsubsw:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vphsubsw %ymm1, %ymm0, %ymm0 # sched: [3:2.00]
-; SKX-NEXT:    vphsubsw (%rdi), %ymm0, %ymm0 # sched: [10:2.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_phsubsw:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vphsubsw %ymm1, %ymm0, %ymm0 # sched: [100:0.25]
-; ZNVER1-NEXT:    vphsubsw (%rdi), %ymm0, %ymm0 # sched: [100:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <16 x i16> @llvm.x86.avx2.phsub.sw(<16 x i16> %a0, <16 x i16> %a1)
-  %2 = load <16 x i16>, <16 x i16> *%a2, align 32
-  %3 = call <16 x i16> @llvm.x86.avx2.phsub.sw(<16 x i16> %1, <16 x i16> %2)
-  ret <16 x i16> %3
-}
-declare <16 x i16> @llvm.x86.avx2.phsub.sw(<16 x i16>, <16 x i16>) nounwind readnone
-
-define <16 x i16> @test_phsubw(<16 x i16> %a0, <16 x i16> %a1, <16 x i16> *%a2) {
-; GENERIC-LABEL: test_phsubw:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vphsubw %ymm1, %ymm0, %ymm0 # sched: [3:1.50]
-; GENERIC-NEXT:    vphsubw (%rdi), %ymm0, %ymm0 # sched: [10:1.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_phsubw:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vphsubw %ymm1, %ymm0, %ymm0 # sched: [3:2.00]
-; HASWELL-NEXT:    vphsubw (%rdi), %ymm0, %ymm0 # sched: [10:2.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_phsubw:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vphsubw %ymm1, %ymm0, %ymm0 # sched: [3:2.00]
-; BROADWELL-NEXT:    vphsubw (%rdi), %ymm0, %ymm0 # sched: [9:2.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_phsubw:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vphsubw %ymm1, %ymm0, %ymm0 # sched: [3:2.00]
-; SKYLAKE-NEXT:    vphsubw (%rdi), %ymm0, %ymm0 # sched: [10:2.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_phsubw:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vphsubw %ymm1, %ymm0, %ymm0 # sched: [3:2.00]
-; SKX-NEXT:    vphsubw (%rdi), %ymm0, %ymm0 # sched: [10:2.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_phsubw:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vphsubw %ymm1, %ymm0, %ymm0 # sched: [100:0.25]
-; ZNVER1-NEXT:    vphsubw (%rdi), %ymm0, %ymm0 # sched: [100:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <16 x i16> @llvm.x86.avx2.phsub.w(<16 x i16> %a0, <16 x i16> %a1)
-  %2 = load <16 x i16>, <16 x i16> *%a2, align 32
-  %3 = call <16 x i16> @llvm.x86.avx2.phsub.w(<16 x i16> %1, <16 x i16> %2)
-  ret <16 x i16> %3
-}
-declare <16 x i16> @llvm.x86.avx2.phsub.w(<16 x i16>, <16 x i16>) nounwind readnone
-
-define <16 x i16> @test_pmaddubsw(<32 x i8> %a0, <32 x i8> %a1, <32 x i8> *%a2) {
-; GENERIC-LABEL: test_pmaddubsw:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpmaddubsw %ymm1, %ymm0, %ymm0 # sched: [5:1.00]
-; GENERIC-NEXT:    vpmaddubsw (%rdi), %ymm0, %ymm0 # sched: [12:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_pmaddubsw:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpmaddubsw %ymm1, %ymm0, %ymm0 # sched: [5:1.00]
-; HASWELL-NEXT:    vpmaddubsw (%rdi), %ymm0, %ymm0 # sched: [12:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pmaddubsw:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpmaddubsw %ymm1, %ymm0, %ymm0 # sched: [5:1.00]
-; BROADWELL-NEXT:    vpmaddubsw (%rdi), %ymm0, %ymm0 # sched: [11:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pmaddubsw:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpmaddubsw %ymm1, %ymm0, %ymm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    vpmaddubsw (%rdi), %ymm0, %ymm0 # sched: [11:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pmaddubsw:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpmaddubsw %ymm1, %ymm0, %ymm0 # sched: [4:0.50]
-; SKX-NEXT:    vpmaddubsw (%rdi), %ymm0, %ymm0 # sched: [11:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_pmaddubsw:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpmaddubsw %ymm1, %ymm0, %ymm0 # sched: [4:1.00]
-; ZNVER1-NEXT:    vpmaddubsw (%rdi), %ymm0, %ymm0 # sched: [11:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <16 x i16> @llvm.x86.avx2.pmadd.ub.sw(<32 x i8> %a0, <32 x i8> %a1)
-  %2 = bitcast <16 x i16> %1 to <32 x i8>
-  %3 = load <32 x i8>, <32 x i8> *%a2, align 32
-  %4 = call <16 x i16> @llvm.x86.avx2.pmadd.ub.sw(<32 x i8> %2, <32 x i8> %3)
-  ret <16 x i16> %4
-}
-declare <16 x i16> @llvm.x86.avx2.pmadd.ub.sw(<32 x i8>, <32 x i8>) nounwind readnone
-
-define <8 x i32> @test_pmaddwd(<16 x i16> %a0, <16 x i16> %a1, <16 x i16> *%a2) {
-; GENERIC-LABEL: test_pmaddwd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpmaddwd %ymm1, %ymm0, %ymm0 # sched: [5:1.00]
-; GENERIC-NEXT:    vpmaddwd (%rdi), %ymm0, %ymm0 # sched: [12:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_pmaddwd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpmaddwd %ymm1, %ymm0, %ymm0 # sched: [5:1.00]
-; HASWELL-NEXT:    vpmaddwd (%rdi), %ymm0, %ymm0 # sched: [12:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pmaddwd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpmaddwd %ymm1, %ymm0, %ymm0 # sched: [5:1.00]
-; BROADWELL-NEXT:    vpmaddwd (%rdi), %ymm0, %ymm0 # sched: [11:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pmaddwd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpmaddwd %ymm1, %ymm0, %ymm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    vpmaddwd (%rdi), %ymm0, %ymm0 # sched: [11:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pmaddwd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpmaddwd %ymm1, %ymm0, %ymm0 # sched: [4:0.50]
-; SKX-NEXT:    vpmaddwd (%rdi), %ymm0, %ymm0 # sched: [11:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_pmaddwd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpmaddwd %ymm1, %ymm0, %ymm0 # sched: [4:1.00]
-; ZNVER1-NEXT:    vpmaddwd (%rdi), %ymm0, %ymm0 # sched: [11:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <8 x i32> @llvm.x86.avx2.pmadd.wd(<16 x i16> %a0, <16 x i16> %a1)
-  %2 = bitcast <8 x i32> %1 to <16 x i16>
-  %3 = load <16 x i16>, <16 x i16> *%a2, align 32
-  %4 = call <8 x i32> @llvm.x86.avx2.pmadd.wd(<16 x i16> %2, <16 x i16> %3)
-  ret <8 x i32> %4
-}
-declare <8 x i32> @llvm.x86.avx2.pmadd.wd(<16 x i16>, <16 x i16>) nounwind readnone
-
-define <4 x i32> @test_pmaskmovd(i8* %a0, <4 x i32> %a1, <4 x i32> %a2) {
-; GENERIC-LABEL: test_pmaskmovd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpmaskmovd (%rdi), %xmm0, %xmm2 # sched: [8:1.00]
-; GENERIC-NEXT:    vpmaskmovd %xmm1, %xmm0, (%rdi) # sched: [5:1.00]
-; GENERIC-NEXT:    vmovdqa %xmm2, %xmm0 # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_pmaskmovd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpmaskmovd (%rdi), %xmm0, %xmm2 # sched: [8:2.00]
-; HASWELL-NEXT:    vpmaskmovd %xmm1, %xmm0, (%rdi) # sched: [5:1.00]
-; HASWELL-NEXT:    vmovdqa %xmm2, %xmm0 # sched: [1:0.33]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pmaskmovd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpmaskmovd (%rdi), %xmm0, %xmm2 # sched: [7:2.00]
-; BROADWELL-NEXT:    vpmaskmovd %xmm1, %xmm0, (%rdi) # sched: [5:1.00]
-; BROADWELL-NEXT:    vmovdqa %xmm2, %xmm0 # sched: [1:0.33]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pmaskmovd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpmaskmovd (%rdi), %xmm0, %xmm2 # sched: [7:0.50]
-; SKYLAKE-NEXT:    vpmaskmovd %xmm1, %xmm0, (%rdi) # sched: [2:1.00]
-; SKYLAKE-NEXT:    vmovdqa %xmm2, %xmm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pmaskmovd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpmaskmovd (%rdi), %xmm0, %xmm2 # sched: [7:0.50]
-; SKX-NEXT:    vpmaskmovd %xmm1, %xmm0, (%rdi) # sched: [2:1.00]
-; SKX-NEXT:    vmovdqa %xmm2, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_pmaskmovd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpmaskmovd (%rdi), %xmm0, %xmm2 # sched: [100:0.25]
-; ZNVER1-NEXT:    vpmaskmovd %xmm1, %xmm0, (%rdi) # sched: [100:0.25]
-; ZNVER1-NEXT:    vmovdqa %xmm2, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <4 x i32> @llvm.x86.avx2.maskload.d(i8* %a0, <4 x i32> %a1)
-  call void @llvm.x86.avx2.maskstore.d(i8* %a0, <4 x i32> %a1, <4 x i32> %a2)
-  ret <4 x i32> %1
-}
-declare <4 x i32> @llvm.x86.avx2.maskload.d(i8*, <4 x i32>) nounwind readonly
-declare void @llvm.x86.avx2.maskstore.d(i8*, <4 x i32>, <4 x i32>) nounwind
-
-define <8 x i32> @test_pmaskmovd_ymm(i8* %a0, <8 x i32> %a1, <8 x i32> %a2) {
-; GENERIC-LABEL: test_pmaskmovd_ymm:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpmaskmovd (%rdi), %ymm0, %ymm2 # sched: [9:1.00]
-; GENERIC-NEXT:    vpmaskmovd %ymm1, %ymm0, (%rdi) # sched: [5:1.00]
-; GENERIC-NEXT:    vmovdqa %ymm2, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_pmaskmovd_ymm:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpmaskmovd (%rdi), %ymm0, %ymm2 # sched: [9:2.00]
-; HASWELL-NEXT:    vpmaskmovd %ymm1, %ymm0, (%rdi) # sched: [5:1.00]
-; HASWELL-NEXT:    vmovdqa %ymm2, %ymm0 # sched: [1:0.33]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pmaskmovd_ymm:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpmaskmovd (%rdi), %ymm0, %ymm2 # sched: [8:2.00]
-; BROADWELL-NEXT:    vpmaskmovd %ymm1, %ymm0, (%rdi) # sched: [5:1.00]
-; BROADWELL-NEXT:    vmovdqa %ymm2, %ymm0 # sched: [1:0.33]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pmaskmovd_ymm:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpmaskmovd (%rdi), %ymm0, %ymm2 # sched: [8:0.50]
-; SKYLAKE-NEXT:    vpmaskmovd %ymm1, %ymm0, (%rdi) # sched: [2:1.00]
-; SKYLAKE-NEXT:    vmovdqa %ymm2, %ymm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pmaskmovd_ymm:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpmaskmovd (%rdi), %ymm0, %ymm2 # sched: [8:0.50]
-; SKX-NEXT:    vpmaskmovd %ymm1, %ymm0, (%rdi) # sched: [2:1.00]
-; SKX-NEXT:    vmovdqa %ymm2, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_pmaskmovd_ymm:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpmaskmovd (%rdi), %ymm0, %ymm2 # sched: [100:0.25]
-; ZNVER1-NEXT:    vpmaskmovd %ymm1, %ymm0, (%rdi) # sched: [100:0.25]
-; ZNVER1-NEXT:    vmovdqa %ymm2, %ymm0 # sched: [2:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <8 x i32> @llvm.x86.avx2.maskload.d.256(i8* %a0, <8 x i32> %a1)
-  call void @llvm.x86.avx2.maskstore.d.256(i8* %a0, <8 x i32> %a1, <8 x i32> %a2)
-  ret <8 x i32> %1
-}
-declare <8 x i32> @llvm.x86.avx2.maskload.d.256(i8*, <8 x i32>) nounwind readonly
-declare void @llvm.x86.avx2.maskstore.d.256(i8*, <8 x i32>, <8 x i32>) nounwind
-
-define <2 x i64> @test_pmaskmovq(i8* %a0, <2 x i64> %a1, <2 x i64> %a2) {
-; GENERIC-LABEL: test_pmaskmovq:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpmaskmovq (%rdi), %xmm0, %xmm2 # sched: [8:1.00]
-; GENERIC-NEXT:    vpmaskmovq %xmm1, %xmm0, (%rdi) # sched: [5:1.00]
-; GENERIC-NEXT:    vmovdqa %xmm2, %xmm0 # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_pmaskmovq:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpmaskmovq (%rdi), %xmm0, %xmm2 # sched: [8:2.00]
-; HASWELL-NEXT:    vpmaskmovq %xmm1, %xmm0, (%rdi) # sched: [5:1.00]
-; HASWELL-NEXT:    vmovdqa %xmm2, %xmm0 # sched: [1:0.33]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pmaskmovq:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpmaskmovq (%rdi), %xmm0, %xmm2 # sched: [7:2.00]
-; BROADWELL-NEXT:    vpmaskmovq %xmm1, %xmm0, (%rdi) # sched: [5:1.00]
-; BROADWELL-NEXT:    vmovdqa %xmm2, %xmm0 # sched: [1:0.33]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pmaskmovq:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpmaskmovq (%rdi), %xmm0, %xmm2 # sched: [7:0.50]
-; SKYLAKE-NEXT:    vpmaskmovq %xmm1, %xmm0, (%rdi) # sched: [2:1.00]
-; SKYLAKE-NEXT:    vmovdqa %xmm2, %xmm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pmaskmovq:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpmaskmovq (%rdi), %xmm0, %xmm2 # sched: [7:0.50]
-; SKX-NEXT:    vpmaskmovq %xmm1, %xmm0, (%rdi) # sched: [2:1.00]
-; SKX-NEXT:    vmovdqa %xmm2, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_pmaskmovq:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpmaskmovq (%rdi), %xmm0, %xmm2 # sched: [8:1.00]
-; ZNVER1-NEXT:    vpmaskmovq %xmm1, %xmm0, (%rdi) # sched: [100:0.25]
-; ZNVER1-NEXT:    vmovdqa %xmm2, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <2 x i64> @llvm.x86.avx2.maskload.q(i8* %a0, <2 x i64> %a1)
-  call void @llvm.x86.avx2.maskstore.q(i8* %a0, <2 x i64> %a1, <2 x i64> %a2)
-  ret <2 x i64> %1
-}
-declare <2 x i64> @llvm.x86.avx2.maskload.q(i8*, <2 x i64>) nounwind readonly
-declare void @llvm.x86.avx2.maskstore.q(i8*, <2 x i64>, <2 x i64>) nounwind
-
-define <4 x i64> @test_pmaskmovq_ymm(i8* %a0, <4 x i64> %a1, <4 x i64> %a2) {
-; GENERIC-LABEL: test_pmaskmovq_ymm:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpmaskmovq (%rdi), %ymm0, %ymm2 # sched: [9:1.00]
-; GENERIC-NEXT:    vpmaskmovq %ymm1, %ymm0, (%rdi) # sched: [5:1.00]
-; GENERIC-NEXT:    vmovdqa %ymm2, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_pmaskmovq_ymm:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpmaskmovq (%rdi), %ymm0, %ymm2 # sched: [9:2.00]
-; HASWELL-NEXT:    vpmaskmovq %ymm1, %ymm0, (%rdi) # sched: [5:1.00]
-; HASWELL-NEXT:    vmovdqa %ymm2, %ymm0 # sched: [1:0.33]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pmaskmovq_ymm:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpmaskmovq (%rdi), %ymm0, %ymm2 # sched: [8:2.00]
-; BROADWELL-NEXT:    vpmaskmovq %ymm1, %ymm0, (%rdi) # sched: [5:1.00]
-; BROADWELL-NEXT:    vmovdqa %ymm2, %ymm0 # sched: [1:0.33]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pmaskmovq_ymm:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpmaskmovq (%rdi), %ymm0, %ymm2 # sched: [8:0.50]
-; SKYLAKE-NEXT:    vpmaskmovq %ymm1, %ymm0, (%rdi) # sched: [2:1.00]
-; SKYLAKE-NEXT:    vmovdqa %ymm2, %ymm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pmaskmovq_ymm:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpmaskmovq (%rdi), %ymm0, %ymm2 # sched: [8:0.50]
-; SKX-NEXT:    vpmaskmovq %ymm1, %ymm0, (%rdi) # sched: [2:1.00]
-; SKX-NEXT:    vmovdqa %ymm2, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_pmaskmovq_ymm:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpmaskmovq (%rdi), %ymm0, %ymm2 # sched: [9:1.50]
-; ZNVER1-NEXT:    vpmaskmovq %ymm1, %ymm0, (%rdi) # sched: [100:0.25]
-; ZNVER1-NEXT:    vmovdqa %ymm2, %ymm0 # sched: [2:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <4 x i64> @llvm.x86.avx2.maskload.q.256(i8* %a0, <4 x i64> %a1)
-  call void @llvm.x86.avx2.maskstore.q.256(i8* %a0, <4 x i64> %a1, <4 x i64> %a2)
-  ret <4 x i64> %1
-}
-declare <4 x i64> @llvm.x86.avx2.maskload.q.256(i8*, <4 x i64>) nounwind readonly
-declare void @llvm.x86.avx2.maskstore.q.256(i8*, <4 x i64>, <4 x i64>) nounwind
-
-define <32 x i8> @test_pmaxsb(<32 x i8> %a0, <32 x i8> %a1, <32 x i8> *%a2) {
-; GENERIC-LABEL: test_pmaxsb:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpmaxsb %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    vpmaxsb (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_pmaxsb:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpmaxsb %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; HASWELL-NEXT:    vpmaxsb (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pmaxsb:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpmaxsb %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    vpmaxsb (%rdi), %ymm0, %ymm0 # sched: [7:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pmaxsb:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpmaxsb %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; SKYLAKE-NEXT:    vpmaxsb (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pmaxsb:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpmaxsb %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; SKX-NEXT:    vpmaxsb (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_pmaxsb:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpmaxsb %ymm1, %ymm0, %ymm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vpmaxsb (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <32 x i8> @llvm.x86.avx2.pmaxs.b(<32 x i8> %a0, <32 x i8> %a1)
-  %2 = load <32 x i8>, <32 x i8> *%a2, align 32
-  %3 = call <32 x i8> @llvm.x86.avx2.pmaxs.b(<32 x i8> %1, <32 x i8> %2)
-  ret <32 x i8> %3
-}
-declare <32 x i8> @llvm.x86.avx2.pmaxs.b(<32 x i8>, <32 x i8>) nounwind readnone
-
-define <8 x i32> @test_pmaxsd(<8 x i32> %a0, <8 x i32> %a1, <8 x i32> *%a2) {
-; GENERIC-LABEL: test_pmaxsd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpmaxsd %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    vpmaxsd (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_pmaxsd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpmaxsd %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; HASWELL-NEXT:    vpmaxsd (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pmaxsd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpmaxsd %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    vpmaxsd (%rdi), %ymm0, %ymm0 # sched: [7:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pmaxsd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpmaxsd %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; SKYLAKE-NEXT:    vpmaxsd (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pmaxsd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpmaxsd %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; SKX-NEXT:    vpmaxsd (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_pmaxsd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpmaxsd %ymm1, %ymm0, %ymm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vpmaxsd (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <8 x i32> @llvm.x86.avx2.pmaxs.d(<8 x i32> %a0, <8 x i32> %a1)
-  %2 = load <8 x i32>, <8 x i32> *%a2, align 32
-  %3 = call <8 x i32> @llvm.x86.avx2.pmaxs.d(<8 x i32> %1, <8 x i32> %2)
-  ret <8 x i32> %3
-}
-declare <8 x i32> @llvm.x86.avx2.pmaxs.d(<8 x i32>, <8 x i32>) nounwind readnone
-
-define <16 x i16> @test_pmaxsw(<16 x i16> %a0, <16 x i16> %a1, <16 x i16> *%a2) {
-; GENERIC-LABEL: test_pmaxsw:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpmaxsw %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    vpmaxsw (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_pmaxsw:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpmaxsw %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; HASWELL-NEXT:    vpmaxsw (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pmaxsw:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpmaxsw %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    vpmaxsw (%rdi), %ymm0, %ymm0 # sched: [7:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pmaxsw:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpmaxsw %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; SKYLAKE-NEXT:    vpmaxsw (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pmaxsw:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpmaxsw %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; SKX-NEXT:    vpmaxsw (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_pmaxsw:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpmaxsw %ymm1, %ymm0, %ymm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vpmaxsw (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <16 x i16> @llvm.x86.avx2.pmaxs.w(<16 x i16> %a0, <16 x i16> %a1)
-  %2 = load <16 x i16>, <16 x i16> *%a2, align 32
-  %3 = call <16 x i16> @llvm.x86.avx2.pmaxs.w(<16 x i16> %1, <16 x i16> %2)
-  ret <16 x i16> %3
-}
-declare <16 x i16> @llvm.x86.avx2.pmaxs.w(<16 x i16>, <16 x i16>) nounwind readnone
-
-define <32 x i8> @test_pmaxub(<32 x i8> %a0, <32 x i8> %a1, <32 x i8> *%a2) {
-; GENERIC-LABEL: test_pmaxub:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpmaxub %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    vpmaxub (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_pmaxub:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpmaxub %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; HASWELL-NEXT:    vpmaxub (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pmaxub:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpmaxub %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    vpmaxub (%rdi), %ymm0, %ymm0 # sched: [7:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pmaxub:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpmaxub %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; SKYLAKE-NEXT:    vpmaxub (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pmaxub:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpmaxub %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; SKX-NEXT:    vpmaxub (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_pmaxub:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpmaxub %ymm1, %ymm0, %ymm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vpmaxub (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <32 x i8> @llvm.x86.avx2.pmaxu.b(<32 x i8> %a0, <32 x i8> %a1)
-  %2 = load <32 x i8>, <32 x i8> *%a2, align 32
-  %3 = call <32 x i8> @llvm.x86.avx2.pmaxu.b(<32 x i8> %1, <32 x i8> %2)
-  ret <32 x i8> %3
-}
-declare <32 x i8> @llvm.x86.avx2.pmaxu.b(<32 x i8>, <32 x i8>) nounwind readnone
-
-define <8 x i32> @test_pmaxud(<8 x i32> %a0, <8 x i32> %a1, <8 x i32> *%a2) {
-; GENERIC-LABEL: test_pmaxud:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpmaxud %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    vpmaxud (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_pmaxud:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpmaxud %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; HASWELL-NEXT:    vpmaxud (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pmaxud:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpmaxud %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    vpmaxud (%rdi), %ymm0, %ymm0 # sched: [7:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pmaxud:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpmaxud %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; SKYLAKE-NEXT:    vpmaxud (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pmaxud:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpmaxud %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; SKX-NEXT:    vpmaxud (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_pmaxud:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpmaxud %ymm1, %ymm0, %ymm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vpmaxud (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <8 x i32> @llvm.x86.avx2.pmaxu.d(<8 x i32> %a0, <8 x i32> %a1)
-  %2 = load <8 x i32>, <8 x i32> *%a2, align 32
-  %3 = call <8 x i32> @llvm.x86.avx2.pmaxu.d(<8 x i32> %1, <8 x i32> %2)
-  ret <8 x i32> %3
-}
-declare <8 x i32> @llvm.x86.avx2.pmaxu.d(<8 x i32>, <8 x i32>) nounwind readnone
-
-define <16 x i16> @test_pmaxuw(<16 x i16> %a0, <16 x i16> %a1, <16 x i16> *%a2) {
-; GENERIC-LABEL: test_pmaxuw:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpmaxuw %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    vpmaxuw (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_pmaxuw:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpmaxuw %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; HASWELL-NEXT:    vpmaxuw (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pmaxuw:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpmaxuw %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    vpmaxuw (%rdi), %ymm0, %ymm0 # sched: [7:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pmaxuw:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpmaxuw %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; SKYLAKE-NEXT:    vpmaxuw (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pmaxuw:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpmaxuw %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; SKX-NEXT:    vpmaxuw (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_pmaxuw:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpmaxuw %ymm1, %ymm0, %ymm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vpmaxuw (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <16 x i16> @llvm.x86.avx2.pmaxu.w(<16 x i16> %a0, <16 x i16> %a1)
-  %2 = load <16 x i16>, <16 x i16> *%a2, align 32
-  %3 = call <16 x i16> @llvm.x86.avx2.pmaxu.w(<16 x i16> %1, <16 x i16> %2)
-  ret <16 x i16> %3
-}
-declare <16 x i16> @llvm.x86.avx2.pmaxu.w(<16 x i16>, <16 x i16>) nounwind readnone
-
-define <32 x i8> @test_pminsb(<32 x i8> %a0, <32 x i8> %a1, <32 x i8> *%a2) {
-; GENERIC-LABEL: test_pminsb:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpminsb %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    vpminsb (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_pminsb:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpminsb %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; HASWELL-NEXT:    vpminsb (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pminsb:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpminsb %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    vpminsb (%rdi), %ymm0, %ymm0 # sched: [7:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pminsb:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpminsb %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; SKYLAKE-NEXT:    vpminsb (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pminsb:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpminsb %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; SKX-NEXT:    vpminsb (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_pminsb:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpminsb %ymm1, %ymm0, %ymm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vpminsb (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <32 x i8> @llvm.x86.avx2.pmins.b(<32 x i8> %a0, <32 x i8> %a1)
-  %2 = load <32 x i8>, <32 x i8> *%a2, align 32
-  %3 = call <32 x i8> @llvm.x86.avx2.pmins.b(<32 x i8> %1, <32 x i8> %2)
-  ret <32 x i8> %3
-}
-declare <32 x i8> @llvm.x86.avx2.pmins.b(<32 x i8>, <32 x i8>) nounwind readnone
-
-define <8 x i32> @test_pminsd(<8 x i32> %a0, <8 x i32> %a1, <8 x i32> *%a2) {
-; GENERIC-LABEL: test_pminsd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpminsd %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    vpminsd (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_pminsd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpminsd %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; HASWELL-NEXT:    vpminsd (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pminsd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpminsd %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    vpminsd (%rdi), %ymm0, %ymm0 # sched: [7:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pminsd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpminsd %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; SKYLAKE-NEXT:    vpminsd (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pminsd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpminsd %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; SKX-NEXT:    vpminsd (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_pminsd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpminsd %ymm1, %ymm0, %ymm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vpminsd (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <8 x i32> @llvm.x86.avx2.pmins.d(<8 x i32> %a0, <8 x i32> %a1)
-  %2 = load <8 x i32>, <8 x i32> *%a2, align 32
-  %3 = call <8 x i32> @llvm.x86.avx2.pmins.d(<8 x i32> %1, <8 x i32> %2)
-  ret <8 x i32> %3
-}
-declare <8 x i32> @llvm.x86.avx2.pmins.d(<8 x i32>, <8 x i32>) nounwind readnone
-
-define <16 x i16> @test_pminsw(<16 x i16> %a0, <16 x i16> %a1, <16 x i16> *%a2) {
-; GENERIC-LABEL: test_pminsw:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpminsw %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    vpminsw (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_pminsw:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpminsw %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; HASWELL-NEXT:    vpminsw (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pminsw:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpminsw %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    vpminsw (%rdi), %ymm0, %ymm0 # sched: [7:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pminsw:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpminsw %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; SKYLAKE-NEXT:    vpminsw (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pminsw:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpminsw %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; SKX-NEXT:    vpminsw (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_pminsw:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpminsw %ymm1, %ymm0, %ymm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vpminsw (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <16 x i16> @llvm.x86.avx2.pmins.w(<16 x i16> %a0, <16 x i16> %a1)
-  %2 = load <16 x i16>, <16 x i16> *%a2, align 32
-  %3 = call <16 x i16> @llvm.x86.avx2.pmins.w(<16 x i16> %1, <16 x i16> %2)
-  ret <16 x i16> %3
-}
-declare <16 x i16> @llvm.x86.avx2.pmins.w(<16 x i16>, <16 x i16>) nounwind readnone
-
-define <32 x i8> @test_pminub(<32 x i8> %a0, <32 x i8> %a1, <32 x i8> *%a2) {
-; GENERIC-LABEL: test_pminub:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpminub %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    vpminub (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_pminub:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpminub %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; HASWELL-NEXT:    vpminub (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pminub:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpminub %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    vpminub (%rdi), %ymm0, %ymm0 # sched: [7:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pminub:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpminub %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; SKYLAKE-NEXT:    vpminub (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pminub:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpminub %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; SKX-NEXT:    vpminub (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_pminub:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpminub %ymm1, %ymm0, %ymm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vpminub (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <32 x i8> @llvm.x86.avx2.pminu.b(<32 x i8> %a0, <32 x i8> %a1)
-  %2 = load <32 x i8>, <32 x i8> *%a2, align 32
-  %3 = call <32 x i8> @llvm.x86.avx2.pminu.b(<32 x i8> %1, <32 x i8> %2)
-  ret <32 x i8> %3
-}
-declare <32 x i8> @llvm.x86.avx2.pminu.b(<32 x i8>, <32 x i8>) nounwind readnone
-
-define <8 x i32> @test_pminud(<8 x i32> %a0, <8 x i32> %a1, <8 x i32> *%a2) {
-; GENERIC-LABEL: test_pminud:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpminud %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    vpminud (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_pminud:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpminud %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; HASWELL-NEXT:    vpminud (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pminud:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpminud %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    vpminud (%rdi), %ymm0, %ymm0 # sched: [7:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pminud:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpminud %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; SKYLAKE-NEXT:    vpminud (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pminud:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpminud %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; SKX-NEXT:    vpminud (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_pminud:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpminud %ymm1, %ymm0, %ymm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vpminud (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <8 x i32> @llvm.x86.avx2.pminu.d(<8 x i32> %a0, <8 x i32> %a1)
-  %2 = load <8 x i32>, <8 x i32> *%a2, align 32
-  %3 = call <8 x i32> @llvm.x86.avx2.pminu.d(<8 x i32> %1, <8 x i32> %2)
-  ret <8 x i32> %3
-}
-declare <8 x i32> @llvm.x86.avx2.pminu.d(<8 x i32>, <8 x i32>) nounwind readnone
-
-define <16 x i16> @test_pminuw(<16 x i16> %a0, <16 x i16> %a1, <16 x i16> *%a2) {
-; GENERIC-LABEL: test_pminuw:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpminuw %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    vpminuw (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_pminuw:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpminuw %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; HASWELL-NEXT:    vpminuw (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pminuw:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpminuw %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    vpminuw (%rdi), %ymm0, %ymm0 # sched: [7:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pminuw:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpminuw %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; SKYLAKE-NEXT:    vpminuw (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pminuw:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpminuw %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; SKX-NEXT:    vpminuw (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_pminuw:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpminuw %ymm1, %ymm0, %ymm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vpminuw (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <16 x i16> @llvm.x86.avx2.pminu.w(<16 x i16> %a0, <16 x i16> %a1)
-  %2 = load <16 x i16>, <16 x i16> *%a2, align 32
-  %3 = call <16 x i16> @llvm.x86.avx2.pminu.w(<16 x i16> %1, <16 x i16> %2)
-  ret <16 x i16> %3
-}
-declare <16 x i16> @llvm.x86.avx2.pminu.w(<16 x i16>, <16 x i16>) nounwind readnone
-
-define i32 @test_pmovmskb(<32 x i8> %a0) {
-; GENERIC-LABEL: test_pmovmskb:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpmovmskb %ymm0, %eax # sched: [2:1.00]
-; GENERIC-NEXT:    vzeroupper # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_pmovmskb:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpmovmskb %ymm0, %eax # sched: [3:1.00]
-; HASWELL-NEXT:    vzeroupper # sched: [0:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pmovmskb:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpmovmskb %ymm0, %eax # sched: [3:1.00]
-; BROADWELL-NEXT:    vzeroupper # sched: [0:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pmovmskb:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpmovmskb %ymm0, %eax # sched: [2:1.00]
-; SKYLAKE-NEXT:    vzeroupper # sched: [0:0.67]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pmovmskb:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpmovmskb %ymm0, %eax # sched: [2:1.00]
-; SKX-NEXT:    vzeroupper # sched: [0:0.67]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_pmovmskb:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpmovmskb %ymm0, %eax # sched: [2:2.00]
-; ZNVER1-NEXT:    vzeroupper # sched: [100:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call i32 @llvm.x86.avx2.pmovmskb(<32 x i8> %a0)
-  ret i32 %1
-}
-declare i32 @llvm.x86.avx2.pmovmskb(<32 x i8>) nounwind readnone
-
-define <8 x i32> @test_pmovsxbd(<16 x i8> %a0, <16 x i8> *%a1) {
-; GENERIC-LABEL: test_pmovsxbd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpmovsxbd %xmm0, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vpmovsxbd (%rdi), %ymm1 # sched: [8:1.00]
-; GENERIC-NEXT:    vpaddd %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_pmovsxbd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpmovsxbd %xmm0, %ymm0 # sched: [3:1.00]
-; HASWELL-NEXT:    vpmovsxbd (%rdi), %ymm1 # sched: [8:1.00]
-; HASWELL-NEXT:    vpaddd %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pmovsxbd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpmovsxbd %xmm0, %ymm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    vpmovsxbd (%rdi), %ymm1 # sched: [8:1.00]
-; BROADWELL-NEXT:    vpaddd %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pmovsxbd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpmovsxbd %xmm0, %ymm0 # sched: [3:1.00]
-; SKYLAKE-NEXT:    vpmovsxbd (%rdi), %ymm1 # sched: [8:1.00]
-; SKYLAKE-NEXT:    vpaddd %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pmovsxbd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpmovsxbd %xmm0, %ymm0 # sched: [3:1.00]
-; SKX-NEXT:    vpmovsxbd (%rdi), %ymm1 # sched: [8:1.00]
-; SKX-NEXT:    vpaddd %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_pmovsxbd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpmovsxbd (%rdi), %ymm1 # sched: [8:0.50]
-; ZNVER1-NEXT:    vpmovsxbd %xmm0, %ymm0 # sched: [1:0.50]
-; ZNVER1-NEXT:    vpaddd %ymm1, %ymm0, %ymm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = shufflevector <16 x i8> %a0, <16 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-  %2 = sext <8 x i8> %1 to <8 x i32>
-  %3 = load <16 x i8>, <16 x i8> *%a1, align 16
-  %4 = shufflevector <16 x i8> %3, <16 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-  %5 = sext <8 x i8> %4 to <8 x i32>
-  %6 = add <8 x i32> %2, %5
-  ret <8 x i32> %6
-}
-
-define <4 x i64> @test_pmovsxbq(<16 x i8> %a0, <16 x i8> *%a1) {
-; GENERIC-LABEL: test_pmovsxbq:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpmovsxbq %xmm0, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vpmovsxbq (%rdi), %ymm1 # sched: [8:1.00]
-; GENERIC-NEXT:    vpaddq %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_pmovsxbq:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpmovsxbq %xmm0, %ymm0 # sched: [3:1.00]
-; HASWELL-NEXT:    vpmovsxbq (%rdi), %ymm1 # sched: [8:1.00]
-; HASWELL-NEXT:    vpaddq %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pmovsxbq:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpmovsxbq %xmm0, %ymm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    vpmovsxbq (%rdi), %ymm1 # sched: [8:1.00]
-; BROADWELL-NEXT:    vpaddq %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pmovsxbq:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpmovsxbq %xmm0, %ymm0 # sched: [3:1.00]
-; SKYLAKE-NEXT:    vpmovsxbq (%rdi), %ymm1 # sched: [8:1.00]
-; SKYLAKE-NEXT:    vpaddq %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pmovsxbq:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpmovsxbq %xmm0, %ymm0 # sched: [3:1.00]
-; SKX-NEXT:    vpmovsxbq (%rdi), %ymm1 # sched: [8:1.00]
-; SKX-NEXT:    vpaddq %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_pmovsxbq:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpmovsxbq (%rdi), %ymm1 # sched: [8:0.50]
-; ZNVER1-NEXT:    vpmovsxbq %xmm0, %ymm0 # sched: [1:0.50]
-; ZNVER1-NEXT:    vpaddq %ymm1, %ymm0, %ymm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = shufflevector <16 x i8> %a0, <16 x i8> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-  %2 = sext <4 x i8> %1 to <4 x i64>
-  %3 = load <16 x i8>, <16 x i8> *%a1, align 16
-  %4 = shufflevector <16 x i8> %3, <16 x i8> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-  %5 = sext <4 x i8> %4 to <4 x i64>
-  %6 = add <4 x i64> %2, %5
-  ret <4 x i64> %6
-}
-
-define <16 x i16> @test_pmovsxbw(<16 x i8> %a0, <16 x i8> *%a1) {
-; GENERIC-LABEL: test_pmovsxbw:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpmovsxbw %xmm0, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vpmovsxbw (%rdi), %ymm1 # sched: [8:1.00]
-; GENERIC-NEXT:    vpaddw %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_pmovsxbw:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpmovsxbw %xmm0, %ymm0 # sched: [3:1.00]
-; HASWELL-NEXT:    vpmovsxbw (%rdi), %ymm1 # sched: [9:1.00]
-; HASWELL-NEXT:    vpaddw %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pmovsxbw:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpmovsxbw %xmm0, %ymm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    vpmovsxbw (%rdi), %ymm1 # sched: [8:1.00]
-; BROADWELL-NEXT:    vpaddw %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pmovsxbw:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpmovsxbw %xmm0, %ymm0 # sched: [3:1.00]
-; SKYLAKE-NEXT:    vpmovsxbw (%rdi), %ymm1 # sched: [9:1.00]
-; SKYLAKE-NEXT:    vpaddw %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pmovsxbw:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpmovsxbw %xmm0, %ymm0 # sched: [3:1.00]
-; SKX-NEXT:    vpmovsxbw (%rdi), %ymm1 # sched: [9:1.00]
-; SKX-NEXT:    vpaddw %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_pmovsxbw:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpmovsxbw (%rdi), %ymm1 # sched: [8:0.50]
-; ZNVER1-NEXT:    vpmovsxbw %xmm0, %ymm0 # sched: [1:0.50]
-; ZNVER1-NEXT:    vpaddw %ymm1, %ymm0, %ymm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = sext <16 x i8> %a0 to <16 x i16>
-  %2 = load <16 x i8>, <16 x i8> *%a1, align 16
-  %3 = sext <16 x i8> %2 to <16 x i16>
-  %4 = add <16 x i16> %1, %3
-  ret <16 x i16> %4
-}
-
-define <4 x i64> @test_pmovsxdq(<4 x i32> %a0, <4 x i32> *%a1) {
-; GENERIC-LABEL: test_pmovsxdq:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpmovsxdq %xmm0, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vpmovsxdq (%rdi), %ymm1 # sched: [8:1.00]
-; GENERIC-NEXT:    vpaddq %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_pmovsxdq:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpmovsxdq %xmm0, %ymm0 # sched: [3:1.00]
-; HASWELL-NEXT:    vpmovsxdq (%rdi), %ymm1 # sched: [9:1.00]
-; HASWELL-NEXT:    vpaddq %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pmovsxdq:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpmovsxdq %xmm0, %ymm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    vpmovsxdq (%rdi), %ymm1 # sched: [8:1.00]
-; BROADWELL-NEXT:    vpaddq %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pmovsxdq:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpmovsxdq %xmm0, %ymm0 # sched: [3:1.00]
-; SKYLAKE-NEXT:    vpmovsxdq (%rdi), %ymm1 # sched: [9:1.00]
-; SKYLAKE-NEXT:    vpaddq %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pmovsxdq:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpmovsxdq %xmm0, %ymm0 # sched: [3:1.00]
-; SKX-NEXT:    vpmovsxdq (%rdi), %ymm1 # sched: [9:1.00]
-; SKX-NEXT:    vpaddq %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_pmovsxdq:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpmovsxdq (%rdi), %ymm1 # sched: [8:0.50]
-; ZNVER1-NEXT:    vpmovsxdq %xmm0, %ymm0 # sched: [1:0.50]
-; ZNVER1-NEXT:    vpaddq %ymm1, %ymm0, %ymm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = sext <4 x i32> %a0 to <4 x i64>
-  %2 = load <4 x i32>, <4 x i32> *%a1, align 16
-  %3 = sext <4 x i32> %2 to <4 x i64>
-  %4 = add <4 x i64> %1, %3
-  ret <4 x i64> %4
-}
-
-define <8 x i32> @test_pmovsxwd(<8 x i16> %a0, <8 x i16> *%a1) {
-; GENERIC-LABEL: test_pmovsxwd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpmovsxwd %xmm0, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vpmovsxwd (%rdi), %ymm1 # sched: [8:1.00]
-; GENERIC-NEXT:    vpaddd %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_pmovsxwd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpmovsxwd %xmm0, %ymm0 # sched: [3:1.00]
-; HASWELL-NEXT:    vpmovsxwd (%rdi), %ymm1 # sched: [9:1.00]
-; HASWELL-NEXT:    vpaddd %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pmovsxwd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpmovsxwd %xmm0, %ymm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    vpmovsxwd (%rdi), %ymm1 # sched: [8:1.00]
-; BROADWELL-NEXT:    vpaddd %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pmovsxwd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpmovsxwd %xmm0, %ymm0 # sched: [3:1.00]
-; SKYLAKE-NEXT:    vpmovsxwd (%rdi), %ymm1 # sched: [9:1.00]
-; SKYLAKE-NEXT:    vpaddd %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pmovsxwd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpmovsxwd %xmm0, %ymm0 # sched: [3:1.00]
-; SKX-NEXT:    vpmovsxwd (%rdi), %ymm1 # sched: [9:1.00]
-; SKX-NEXT:    vpaddd %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_pmovsxwd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpmovsxwd (%rdi), %ymm1 # sched: [8:0.50]
-; ZNVER1-NEXT:    vpmovsxwd %xmm0, %ymm0 # sched: [1:0.50]
-; ZNVER1-NEXT:    vpaddd %ymm1, %ymm0, %ymm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = sext <8 x i16> %a0 to <8 x i32>
-  %2 = load <8 x i16>, <8 x i16> *%a1, align 16
-  %3 = sext <8 x i16> %2 to <8 x i32>
-  %4 = add <8 x i32> %1, %3
-  ret <8 x i32> %4
-}
-
-define <4 x i64> @test_pmovsxwq(<8 x i16> %a0, <8 x i16> *%a1) {
-; GENERIC-LABEL: test_pmovsxwq:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpmovsxwq %xmm0, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vpmovsxwq (%rdi), %ymm1 # sched: [8:1.00]
-; GENERIC-NEXT:    vpaddq %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_pmovsxwq:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpmovsxwq %xmm0, %ymm0 # sched: [3:1.00]
-; HASWELL-NEXT:    vpmovsxwq (%rdi), %ymm1 # sched: [8:1.00]
-; HASWELL-NEXT:    vpaddq %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pmovsxwq:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpmovsxwq %xmm0, %ymm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    vpmovsxwq (%rdi), %ymm1 # sched: [8:1.00]
-; BROADWELL-NEXT:    vpaddq %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pmovsxwq:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpmovsxwq %xmm0, %ymm0 # sched: [3:1.00]
-; SKYLAKE-NEXT:    vpmovsxwq (%rdi), %ymm1 # sched: [8:1.00]
-; SKYLAKE-NEXT:    vpaddq %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pmovsxwq:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpmovsxwq %xmm0, %ymm0 # sched: [3:1.00]
-; SKX-NEXT:    vpmovsxwq (%rdi), %ymm1 # sched: [8:1.00]
-; SKX-NEXT:    vpaddq %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_pmovsxwq:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpmovsxwq (%rdi), %ymm1 # sched: [8:0.50]
-; ZNVER1-NEXT:    vpmovsxwq %xmm0, %ymm0 # sched: [1:0.50]
-; ZNVER1-NEXT:    vpaddq %ymm1, %ymm0, %ymm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = shufflevector <8 x i16> %a0, <8 x i16> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-  %2 = sext <4 x i16> %1 to <4 x i64>
-  %3 = load <8 x i16>, <8 x i16> *%a1, align 16
-  %4 = shufflevector <8 x i16> %3, <8 x i16> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-  %5 = sext <4 x i16> %4 to <4 x i64>
-  %6 = add <4 x i64> %2, %5
-  ret <4 x i64> %6
-}
-
-define <8 x i32> @test_pmovzxbd(<16 x i8> %a0, <16 x i8> *%a1) {
-; GENERIC-LABEL: test_pmovzxbd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpmovzxbd {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero sched: [1:1.00]
-; GENERIC-NEXT:    vpmovzxbd {{.*#+}} ymm1 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero sched: [8:1.00]
-; GENERIC-NEXT:    vpaddd %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_pmovzxbd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpmovzxbd {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero sched: [3:1.00]
-; HASWELL-NEXT:    vpmovzxbd {{.*#+}} ymm1 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero sched: [10:1.00]
-; HASWELL-NEXT:    vpaddd %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pmovzxbd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpmovzxbd {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero sched: [3:1.00]
-; BROADWELL-NEXT:    vpmovzxbd {{.*#+}} ymm1 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero sched: [9:1.00]
-; BROADWELL-NEXT:    vpaddd %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pmovzxbd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpmovzxbd {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero sched: [3:1.00]
-; SKYLAKE-NEXT:    vpmovzxbd {{.*#+}} ymm1 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero sched: [10:1.00]
-; SKYLAKE-NEXT:    vpaddd %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pmovzxbd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpmovzxbd {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero sched: [3:1.00]
-; SKX-NEXT:    vpmovzxbd {{.*#+}} ymm1 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero sched: [10:1.00]
-; SKX-NEXT:    vpaddd %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_pmovzxbd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpmovzxbd {{.*#+}} ymm1 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero sched: [8:0.50]
-; ZNVER1-NEXT:    vpmovzxbd {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero sched: [1:0.50]
-; ZNVER1-NEXT:    vpaddd %ymm1, %ymm0, %ymm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = shufflevector <16 x i8> %a0, <16 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-  %2 = zext <8 x i8> %1 to <8 x i32>
-  %3 = load <16 x i8>, <16 x i8> *%a1, align 16
-  %4 = shufflevector <16 x i8> %3, <16 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-  %5 = zext <8 x i8> %4 to <8 x i32>
-  %6 = add <8 x i32> %2, %5
-  ret <8 x i32> %6
-}
-
-define <4 x i64> @test_pmovzxbq(<16 x i8> %a0, <16 x i8> *%a1) {
-; GENERIC-LABEL: test_pmovzxbq:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpmovzxbq {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero,xmm0[2],zero,zero,zero,zero,zero,zero,zero,xmm0[3],zero,zero,zero,zero,zero,zero,zero sched: [1:1.00]
-; GENERIC-NEXT:    vpmovzxbq {{.*#+}} ymm1 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero,mem[2],zero,zero,zero,zero,zero,zero,zero,mem[3],zero,zero,zero,zero,zero,zero,zero sched: [8:1.00]
-; GENERIC-NEXT:    vpaddq %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_pmovzxbq:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpmovzxbq {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero,xmm0[2],zero,zero,zero,zero,zero,zero,zero,xmm0[3],zero,zero,zero,zero,zero,zero,zero sched: [3:1.00]
-; HASWELL-NEXT:    vpmovzxbq {{.*#+}} ymm1 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero,mem[2],zero,zero,zero,zero,zero,zero,zero,mem[3],zero,zero,zero,zero,zero,zero,zero sched: [10:1.00]
-; HASWELL-NEXT:    vpaddq %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pmovzxbq:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpmovzxbq {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero,xmm0[2],zero,zero,zero,zero,zero,zero,zero,xmm0[3],zero,zero,zero,zero,zero,zero,zero sched: [3:1.00]
-; BROADWELL-NEXT:    vpmovzxbq {{.*#+}} ymm1 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero,mem[2],zero,zero,zero,zero,zero,zero,zero,mem[3],zero,zero,zero,zero,zero,zero,zero sched: [9:1.00]
-; BROADWELL-NEXT:    vpaddq %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pmovzxbq:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpmovzxbq {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero,xmm0[2],zero,zero,zero,zero,zero,zero,zero,xmm0[3],zero,zero,zero,zero,zero,zero,zero sched: [3:1.00]
-; SKYLAKE-NEXT:    vpmovzxbq {{.*#+}} ymm1 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero,mem[2],zero,zero,zero,zero,zero,zero,zero,mem[3],zero,zero,zero,zero,zero,zero,zero sched: [10:1.00]
-; SKYLAKE-NEXT:    vpaddq %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pmovzxbq:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpmovzxbq {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero,xmm0[2],zero,zero,zero,zero,zero,zero,zero,xmm0[3],zero,zero,zero,zero,zero,zero,zero sched: [3:1.00]
-; SKX-NEXT:    vpmovzxbq {{.*#+}} ymm1 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero,mem[2],zero,zero,zero,zero,zero,zero,zero,mem[3],zero,zero,zero,zero,zero,zero,zero sched: [10:1.00]
-; SKX-NEXT:    vpaddq %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_pmovzxbq:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpmovzxbq {{.*#+}} ymm1 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero,mem[2],zero,zero,zero,zero,zero,zero,zero,mem[3],zero,zero,zero,zero,zero,zero,zero sched: [8:0.50]
-; ZNVER1-NEXT:    vpmovzxbq {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero,xmm0[2],zero,zero,zero,zero,zero,zero,zero,xmm0[3],zero,zero,zero,zero,zero,zero,zero sched: [1:0.50]
-; ZNVER1-NEXT:    vpaddq %ymm1, %ymm0, %ymm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = shufflevector <16 x i8> %a0, <16 x i8> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-  %2 = zext <4 x i8> %1 to <4 x i64>
-  %3 = load <16 x i8>, <16 x i8> *%a1, align 16
-  %4 = shufflevector <16 x i8> %3, <16 x i8> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-  %5 = zext <4 x i8> %4 to <4 x i64>
-  %6 = add <4 x i64> %2, %5
-  ret <4 x i64> %6
-}
-
-define <16 x i16> @test_pmovzxbw(<16 x i8> %a0, <16 x i8> *%a1) {
-; GENERIC-LABEL: test_pmovzxbw:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero sched: [1:1.00]
-; GENERIC-NEXT:    vpmovzxbw {{.*#+}} ymm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero,mem[8],zero,mem[9],zero,mem[10],zero,mem[11],zero,mem[12],zero,mem[13],zero,mem[14],zero,mem[15],zero sched: [8:1.00]
-; GENERIC-NEXT:    vpaddw %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_pmovzxbw:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero sched: [3:1.00]
-; HASWELL-NEXT:    vpmovzxbw {{.*#+}} ymm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero,mem[8],zero,mem[9],zero,mem[10],zero,mem[11],zero,mem[12],zero,mem[13],zero,mem[14],zero,mem[15],zero sched: [10:1.00]
-; HASWELL-NEXT:    vpaddw %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pmovzxbw:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero sched: [3:1.00]
-; BROADWELL-NEXT:    vpmovzxbw {{.*#+}} ymm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero,mem[8],zero,mem[9],zero,mem[10],zero,mem[11],zero,mem[12],zero,mem[13],zero,mem[14],zero,mem[15],zero sched: [9:1.00]
-; BROADWELL-NEXT:    vpaddw %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pmovzxbw:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero sched: [3:1.00]
-; SKYLAKE-NEXT:    vpmovzxbw {{.*#+}} ymm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero,mem[8],zero,mem[9],zero,mem[10],zero,mem[11],zero,mem[12],zero,mem[13],zero,mem[14],zero,mem[15],zero sched: [10:1.00]
-; SKYLAKE-NEXT:    vpaddw %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pmovzxbw:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero sched: [3:1.00]
-; SKX-NEXT:    vpmovzxbw {{.*#+}} ymm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero,mem[8],zero,mem[9],zero,mem[10],zero,mem[11],zero,mem[12],zero,mem[13],zero,mem[14],zero,mem[15],zero sched: [10:1.00]
-; SKX-NEXT:    vpaddw %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_pmovzxbw:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpmovzxbw {{.*#+}} ymm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero,mem[8],zero,mem[9],zero,mem[10],zero,mem[11],zero,mem[12],zero,mem[13],zero,mem[14],zero,mem[15],zero sched: [8:0.50]
-; ZNVER1-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero sched: [1:0.50]
-; ZNVER1-NEXT:    vpaddw %ymm1, %ymm0, %ymm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = zext <16 x i8> %a0 to <16 x i16>
-  %2 = load <16 x i8>, <16 x i8> *%a1, align 16
-  %3 = zext <16 x i8> %2 to <16 x i16>
-  %4 = add <16 x i16> %1, %3
-  ret <16 x i16> %4
-}
-
-define <4 x i64> @test_pmovzxdq(<4 x i32> %a0, <4 x i32> *%a1) {
-; GENERIC-LABEL: test_pmovzxdq:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpmovzxdq {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero sched: [1:1.00]
-; GENERIC-NEXT:    vpmovzxdq {{.*#+}} ymm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero sched: [8:1.00]
-; GENERIC-NEXT:    vpaddq %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_pmovzxdq:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpmovzxdq {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero sched: [3:1.00]
-; HASWELL-NEXT:    vpmovzxdq {{.*#+}} ymm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero sched: [10:1.00]
-; HASWELL-NEXT:    vpaddq %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pmovzxdq:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpmovzxdq {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero sched: [3:1.00]
-; BROADWELL-NEXT:    vpmovzxdq {{.*#+}} ymm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero sched: [9:1.00]
-; BROADWELL-NEXT:    vpaddq %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pmovzxdq:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpmovzxdq {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero sched: [3:1.00]
-; SKYLAKE-NEXT:    vpmovzxdq {{.*#+}} ymm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero sched: [10:1.00]
-; SKYLAKE-NEXT:    vpaddq %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pmovzxdq:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpmovzxdq {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero sched: [3:1.00]
-; SKX-NEXT:    vpmovzxdq {{.*#+}} ymm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero sched: [10:1.00]
-; SKX-NEXT:    vpaddq %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_pmovzxdq:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpmovzxdq {{.*#+}} ymm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero sched: [8:0.50]
-; ZNVER1-NEXT:    vpmovzxdq {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero sched: [1:0.50]
-; ZNVER1-NEXT:    vpaddq %ymm1, %ymm0, %ymm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = zext <4 x i32> %a0 to <4 x i64>
-  %2 = load <4 x i32>, <4 x i32> *%a1, align 16
-  %3 = zext <4 x i32> %2 to <4 x i64>
-  %4 = add <4 x i64> %1, %3
-  ret <4 x i64> %4
-}
-
-define <8 x i32> @test_pmovzxwd(<8 x i16> %a0, <8 x i16> *%a1) {
-; GENERIC-LABEL: test_pmovzxwd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero sched: [1:1.00]
-; GENERIC-NEXT:    vpmovzxwd {{.*#+}} ymm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero sched: [8:1.00]
-; GENERIC-NEXT:    vpaddd %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_pmovzxwd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero sched: [3:1.00]
-; HASWELL-NEXT:    vpmovzxwd {{.*#+}} ymm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero sched: [9:1.00]
-; HASWELL-NEXT:    vpaddd %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pmovzxwd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero sched: [3:1.00]
-; BROADWELL-NEXT:    vpmovzxwd {{.*#+}} ymm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero sched: [8:1.00]
-; BROADWELL-NEXT:    vpaddd %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pmovzxwd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero sched: [3:1.00]
-; SKYLAKE-NEXT:    vpmovzxwd {{.*#+}} ymm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero sched: [9:1.00]
-; SKYLAKE-NEXT:    vpaddd %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pmovzxwd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero sched: [3:1.00]
-; SKX-NEXT:    vpmovzxwd {{.*#+}} ymm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero sched: [9:1.00]
-; SKX-NEXT:    vpaddd %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_pmovzxwd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpmovzxwd {{.*#+}} ymm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero sched: [8:0.50]
-; ZNVER1-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero sched: [1:0.50]
-; ZNVER1-NEXT:    vpaddd %ymm1, %ymm0, %ymm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = zext <8 x i16> %a0 to <8 x i32>
-  %2 = load <8 x i16>, <8 x i16> *%a1, align 16
-  %3 = zext <8 x i16> %2 to <8 x i32>
-  %4 = add <8 x i32> %1, %3
-  ret <8 x i32> %4
-}
-
-define <4 x i64> @test_pmovzxwq(<8 x i16> %a0, <8 x i16> *%a1) {
-; GENERIC-LABEL: test_pmovzxwq:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpmovzxwq {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero sched: [1:1.00]
-; GENERIC-NEXT:    vpmovzxwq {{.*#+}} ymm1 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero sched: [8:1.00]
-; GENERIC-NEXT:    vpaddq %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_pmovzxwq:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpmovzxwq {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero sched: [3:1.00]
-; HASWELL-NEXT:    vpmovzxwq {{.*#+}} ymm1 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero sched: [10:1.00]
-; HASWELL-NEXT:    vpaddq %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pmovzxwq:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpmovzxwq {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero sched: [3:1.00]
-; BROADWELL-NEXT:    vpmovzxwq {{.*#+}} ymm1 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero sched: [9:1.00]
-; BROADWELL-NEXT:    vpaddq %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pmovzxwq:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpmovzxwq {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero sched: [3:1.00]
-; SKYLAKE-NEXT:    vpmovzxwq {{.*#+}} ymm1 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero sched: [10:1.00]
-; SKYLAKE-NEXT:    vpaddq %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pmovzxwq:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpmovzxwq {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero sched: [3:1.00]
-; SKX-NEXT:    vpmovzxwq {{.*#+}} ymm1 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero sched: [10:1.00]
-; SKX-NEXT:    vpaddq %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_pmovzxwq:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpmovzxwq {{.*#+}} ymm1 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero sched: [8:0.50]
-; ZNVER1-NEXT:    vpmovzxwq {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero sched: [1:0.50]
-; ZNVER1-NEXT:    vpaddq %ymm1, %ymm0, %ymm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = shufflevector <8 x i16> %a0, <8 x i16> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-  %2 = zext <4 x i16> %1 to <4 x i64>
-  %3 = load <8 x i16>, <8 x i16> *%a1, align 16
-  %4 = shufflevector <8 x i16> %3, <8 x i16> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-  %5 = zext <4 x i16> %4 to <4 x i64>
-  %6 = add <4 x i64> %2, %5
-  ret <4 x i64> %6
-}
-
-define <4 x i64> @test_pmuldq(<8 x i32> %a0, <8 x i32> %a1, <8 x i32> %a2, <8 x i32> *%a3) {
-; GENERIC-LABEL: test_pmuldq:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpmuldq %ymm1, %ymm0, %ymm0 # sched: [5:1.00]
-; GENERIC-NEXT:    vpmuldq (%rdi), %ymm2, %ymm1 # sched: [12:1.00]
-; GENERIC-NEXT:    vpor %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_pmuldq:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpmuldq %ymm1, %ymm0, %ymm0 # sched: [5:1.00]
-; HASWELL-NEXT:    vpmuldq (%rdi), %ymm2, %ymm1 # sched: [12:1.00]
-; HASWELL-NEXT:    vpor %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pmuldq:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpmuldq %ymm1, %ymm0, %ymm0 # sched: [5:1.00]
-; BROADWELL-NEXT:    vpmuldq (%rdi), %ymm2, %ymm1 # sched: [11:1.00]
-; BROADWELL-NEXT:    vpor %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pmuldq:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpmuldq %ymm1, %ymm0, %ymm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    vpmuldq (%rdi), %ymm2, %ymm1 # sched: [11:0.50]
-; SKYLAKE-NEXT:    vpor %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pmuldq:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpmuldq %ymm1, %ymm0, %ymm0 # sched: [4:0.50]
-; SKX-NEXT:    vpmuldq (%rdi), %ymm2, %ymm1 # sched: [11:0.50]
-; SKX-NEXT:    vpor %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_pmuldq:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpmuldq (%rdi), %ymm2, %ymm2 # sched: [11:1.00]
-; ZNVER1-NEXT:    vpmuldq %ymm1, %ymm0, %ymm0 # sched: [4:1.00]
-; ZNVER1-NEXT:    vpor %ymm2, %ymm0, %ymm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <4 x i64> @llvm.x86.avx2.pmul.dq(<8 x i32> %a0, <8 x i32> %a1)
-  %2 = load <8 x i32>, <8 x i32> *%a3, align 32
-  %3 = call <4 x i64> @llvm.x86.avx2.pmul.dq(<8 x i32> %a2, <8 x i32> %2)
-  %4 = or <4 x i64> %1, %3
-  ret <4 x i64> %4
-}
-declare <4 x i64> @llvm.x86.avx2.pmul.dq(<8 x i32>, <8 x i32>) nounwind readnone
-
-define <16 x i16> @test_pmulhrsw(<16 x i16> %a0, <16 x i16> %a1, <16 x i16> *%a2) {
-; GENERIC-LABEL: test_pmulhrsw:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpmulhrsw %ymm1, %ymm0, %ymm0 # sched: [5:1.00]
-; GENERIC-NEXT:    vpmulhrsw (%rdi), %ymm0, %ymm0 # sched: [12:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_pmulhrsw:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpmulhrsw %ymm1, %ymm0, %ymm0 # sched: [5:1.00]
-; HASWELL-NEXT:    vpmulhrsw (%rdi), %ymm0, %ymm0 # sched: [12:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pmulhrsw:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpmulhrsw %ymm1, %ymm0, %ymm0 # sched: [5:1.00]
-; BROADWELL-NEXT:    vpmulhrsw (%rdi), %ymm0, %ymm0 # sched: [11:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pmulhrsw:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpmulhrsw %ymm1, %ymm0, %ymm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    vpmulhrsw (%rdi), %ymm0, %ymm0 # sched: [11:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pmulhrsw:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpmulhrsw %ymm1, %ymm0, %ymm0 # sched: [4:0.50]
-; SKX-NEXT:    vpmulhrsw (%rdi), %ymm0, %ymm0 # sched: [11:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_pmulhrsw:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpmulhrsw %ymm1, %ymm0, %ymm0 # sched: [4:1.00]
-; ZNVER1-NEXT:    vpmulhrsw (%rdi), %ymm0, %ymm0 # sched: [11:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <16 x i16> @llvm.x86.avx2.pmul.hr.sw(<16 x i16> %a0, <16 x i16> %a1)
-  %2 = load <16 x i16>, <16 x i16> *%a2, align 32
-  %3 = call <16 x i16> @llvm.x86.avx2.pmul.hr.sw(<16 x i16> %1, <16 x i16> %2)
-  ret <16 x i16> %3
-}
-declare <16 x i16> @llvm.x86.avx2.pmul.hr.sw(<16 x i16>, <16 x i16>) nounwind readnone
-
-define <16 x i16> @test_pmulhuw(<16 x i16> %a0, <16 x i16> %a1, <16 x i16> *%a2) {
-; GENERIC-LABEL: test_pmulhuw:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpmulhuw %ymm1, %ymm0, %ymm0 # sched: [5:1.00]
-; GENERIC-NEXT:    vpmulhuw (%rdi), %ymm0, %ymm0 # sched: [12:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_pmulhuw:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpmulhuw %ymm1, %ymm0, %ymm0 # sched: [5:1.00]
-; HASWELL-NEXT:    vpmulhuw (%rdi), %ymm0, %ymm0 # sched: [12:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pmulhuw:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpmulhuw %ymm1, %ymm0, %ymm0 # sched: [5:1.00]
-; BROADWELL-NEXT:    vpmulhuw (%rdi), %ymm0, %ymm0 # sched: [11:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pmulhuw:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpmulhuw %ymm1, %ymm0, %ymm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    vpmulhuw (%rdi), %ymm0, %ymm0 # sched: [11:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pmulhuw:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpmulhuw %ymm1, %ymm0, %ymm0 # sched: [4:0.50]
-; SKX-NEXT:    vpmulhuw (%rdi), %ymm0, %ymm0 # sched: [11:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_pmulhuw:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpmulhuw %ymm1, %ymm0, %ymm0 # sched: [4:1.00]
-; ZNVER1-NEXT:    vpmulhuw (%rdi), %ymm0, %ymm0 # sched: [11:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <16 x i16> @llvm.x86.avx2.pmulhu.w(<16 x i16> %a0, <16 x i16> %a1)
-  %2 = load <16 x i16>, <16 x i16> *%a2, align 32
-  %3 = call <16 x i16> @llvm.x86.avx2.pmulhu.w(<16 x i16> %1, <16 x i16> %2)
-  ret <16 x i16> %3
-}
-declare <16 x i16> @llvm.x86.avx2.pmulhu.w(<16 x i16>, <16 x i16>) nounwind readnone
-
-define <16 x i16> @test_pmulhw(<16 x i16> %a0, <16 x i16> %a1, <16 x i16> *%a2) {
-; GENERIC-LABEL: test_pmulhw:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpmulhw %ymm1, %ymm0, %ymm0 # sched: [5:1.00]
-; GENERIC-NEXT:    vpmulhw (%rdi), %ymm0, %ymm0 # sched: [12:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_pmulhw:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpmulhw %ymm1, %ymm0, %ymm0 # sched: [5:1.00]
-; HASWELL-NEXT:    vpmulhw (%rdi), %ymm0, %ymm0 # sched: [12:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pmulhw:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpmulhw %ymm1, %ymm0, %ymm0 # sched: [5:1.00]
-; BROADWELL-NEXT:    vpmulhw (%rdi), %ymm0, %ymm0 # sched: [11:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pmulhw:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpmulhw %ymm1, %ymm0, %ymm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    vpmulhw (%rdi), %ymm0, %ymm0 # sched: [11:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pmulhw:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpmulhw %ymm1, %ymm0, %ymm0 # sched: [4:0.50]
-; SKX-NEXT:    vpmulhw (%rdi), %ymm0, %ymm0 # sched: [11:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_pmulhw:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpmulhw %ymm1, %ymm0, %ymm0 # sched: [4:1.00]
-; ZNVER1-NEXT:    vpmulhw (%rdi), %ymm0, %ymm0 # sched: [11:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <16 x i16> @llvm.x86.avx2.pmulh.w(<16 x i16> %a0, <16 x i16> %a1)
-  %2 = load <16 x i16>, <16 x i16> *%a2, align 32
-  %3 = call <16 x i16> @llvm.x86.avx2.pmulh.w(<16 x i16> %1, <16 x i16> %2)
-  ret <16 x i16> %3
-}
-declare <16 x i16> @llvm.x86.avx2.pmulh.w(<16 x i16>, <16 x i16>) nounwind readnone
-
-define <8 x i32> @test_pmulld(<8 x i32> %a0, <8 x i32> %a1, <8 x i32> *%a2) {
-; GENERIC-LABEL: test_pmulld:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpmulld %ymm1, %ymm0, %ymm0 # sched: [5:1.00]
-; GENERIC-NEXT:    vpmulld (%rdi), %ymm0, %ymm0 # sched: [12:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_pmulld:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpmulld %ymm1, %ymm0, %ymm0 # sched: [10:2.00]
-; HASWELL-NEXT:    vpmulld (%rdi), %ymm0, %ymm0 # sched: [17:2.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pmulld:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpmulld %ymm1, %ymm0, %ymm0 # sched: [10:2.00]
-; BROADWELL-NEXT:    vpmulld (%rdi), %ymm0, %ymm0 # sched: [16:2.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pmulld:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpmulld %ymm1, %ymm0, %ymm0 # sched: [10:1.00]
-; SKYLAKE-NEXT:    vpmulld (%rdi), %ymm0, %ymm0 # sched: [17:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pmulld:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpmulld %ymm1, %ymm0, %ymm0 # sched: [10:1.00]
-; SKX-NEXT:    vpmulld (%rdi), %ymm0, %ymm0 # sched: [17:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_pmulld:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpmulld %ymm1, %ymm0, %ymm0 # sched: [5:2.00]
-; ZNVER1-NEXT:    vpmulld (%rdi), %ymm0, %ymm0 # sched: [12:2.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = mul <8 x i32> %a0, %a1
-  %2 = load <8 x i32>, <8 x i32> *%a2, align 32
-  %3 = mul <8 x i32> %1, %2
-  ret <8 x i32> %3
-}
-
-define <16 x i16> @test_pmullw(<16 x i16> %a0, <16 x i16> %a1, <16 x i16> *%a2) {
-; GENERIC-LABEL: test_pmullw:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpmullw %ymm1, %ymm0, %ymm0 # sched: [5:1.00]
-; GENERIC-NEXT:    vpmullw (%rdi), %ymm0, %ymm0 # sched: [12:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_pmullw:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpmullw %ymm1, %ymm0, %ymm0 # sched: [5:1.00]
-; HASWELL-NEXT:    vpmullw (%rdi), %ymm0, %ymm0 # sched: [12:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pmullw:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpmullw %ymm1, %ymm0, %ymm0 # sched: [5:1.00]
-; BROADWELL-NEXT:    vpmullw (%rdi), %ymm0, %ymm0 # sched: [11:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pmullw:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpmullw %ymm1, %ymm0, %ymm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    vpmullw (%rdi), %ymm0, %ymm0 # sched: [11:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pmullw:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpmullw %ymm1, %ymm0, %ymm0 # sched: [4:0.50]
-; SKX-NEXT:    vpmullw (%rdi), %ymm0, %ymm0 # sched: [11:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_pmullw:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpmullw %ymm1, %ymm0, %ymm0 # sched: [4:1.00]
-; ZNVER1-NEXT:    vpmullw (%rdi), %ymm0, %ymm0 # sched: [11:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = mul <16 x i16> %a0, %a1
-  %2 = load <16 x i16>, <16 x i16> *%a2, align 32
-  %3 = mul <16 x i16> %1, %2
-  ret <16 x i16> %3
-}
-
-define <4 x i64> @test_pmuludq(<8 x i32> %a0, <8 x i32> %a1, <8 x i32> *%a2) {
-; GENERIC-LABEL: test_pmuludq:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpmuludq %ymm1, %ymm0, %ymm0 # sched: [5:1.00]
-; GENERIC-NEXT:    vpmuludq (%rdi), %ymm0, %ymm0 # sched: [12:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_pmuludq:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpmuludq %ymm1, %ymm0, %ymm0 # sched: [5:1.00]
-; HASWELL-NEXT:    vpmuludq (%rdi), %ymm0, %ymm0 # sched: [12:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pmuludq:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpmuludq %ymm1, %ymm0, %ymm0 # sched: [5:1.00]
-; BROADWELL-NEXT:    vpmuludq (%rdi), %ymm0, %ymm0 # sched: [11:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pmuludq:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpmuludq %ymm1, %ymm0, %ymm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    vpmuludq (%rdi), %ymm0, %ymm0 # sched: [11:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pmuludq:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpmuludq %ymm1, %ymm0, %ymm0 # sched: [4:0.50]
-; SKX-NEXT:    vpmuludq (%rdi), %ymm0, %ymm0 # sched: [11:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_pmuludq:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpmuludq %ymm1, %ymm0, %ymm0 # sched: [4:1.00]
-; ZNVER1-NEXT:    vpmuludq (%rdi), %ymm0, %ymm0 # sched: [11:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <4 x i64> @llvm.x86.avx2.pmulu.dq(<8 x i32> %a0, <8 x i32> %a1)
-  %2 = bitcast <4 x i64> %1 to <8 x i32>
-  %3 = load <8 x i32>, <8 x i32> *%a2, align 32
-  %4 = call <4 x i64> @llvm.x86.avx2.pmulu.dq(<8 x i32> %2, <8 x i32> %3)
-  ret <4 x i64> %4
-}
-declare <4 x i64> @llvm.x86.avx2.pmulu.dq(<8 x i32>, <8 x i32>) nounwind readnone
-
-define <4 x i64> @test_por(<4 x i64> %a0, <4 x i64> %a1, <4 x i64> *%a2) {
-; GENERIC-LABEL: test_por:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpor %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; GENERIC-NEXT:    vpor (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; GENERIC-NEXT:    vpaddq %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_por:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpor %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; HASWELL-NEXT:    vpor (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; HASWELL-NEXT:    vpaddq %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_por:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpor %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; BROADWELL-NEXT:    vpor (%rdi), %ymm0, %ymm0 # sched: [7:0.50]
-; BROADWELL-NEXT:    vpaddq %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_por:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpor %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    vpor (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKYLAKE-NEXT:    vpaddq %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_por:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpor %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    vpor (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKX-NEXT:    vpaddq %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_por:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpor %ymm1, %ymm0, %ymm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vpor (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    vpaddq %ymm1, %ymm0, %ymm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = or <4 x i64> %a0, %a1
-  %2 = load <4 x i64>, <4 x i64> *%a2, align 32
-  %3 = or <4 x i64> %1, %2
-  %4 = add <4 x i64> %3, %a1
-  ret <4 x i64> %4
-}
-
-define <4 x i64> @test_psadbw(<32 x i8> %a0, <32 x i8> %a1, <32 x i8> *%a2) {
-; GENERIC-LABEL: test_psadbw:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpsadbw %ymm1, %ymm0, %ymm0 # sched: [5:1.00]
-; GENERIC-NEXT:    vpsadbw (%rdi), %ymm0, %ymm0 # sched: [12:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_psadbw:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpsadbw %ymm1, %ymm0, %ymm0 # sched: [5:1.00]
-; HASWELL-NEXT:    vpsadbw (%rdi), %ymm0, %ymm0 # sched: [12:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_psadbw:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpsadbw %ymm1, %ymm0, %ymm0 # sched: [5:1.00]
-; BROADWELL-NEXT:    vpsadbw (%rdi), %ymm0, %ymm0 # sched: [11:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_psadbw:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpsadbw %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; SKYLAKE-NEXT:    vpsadbw (%rdi), %ymm0, %ymm0 # sched: [10:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_psadbw:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpsadbw %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; SKX-NEXT:    vpsadbw (%rdi), %ymm0, %ymm0 # sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_psadbw:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpsadbw %ymm1, %ymm0, %ymm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    vpsadbw (%rdi), %ymm0, %ymm0 # sched: [10:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <4 x i64> @llvm.x86.avx2.psad.bw(<32 x i8> %a0, <32 x i8> %a1)
-  %2 = bitcast <4 x i64> %1 to <32 x i8>
-  %3 = load <32 x i8>, <32 x i8> *%a2, align 32
-  %4 = call <4 x i64> @llvm.x86.avx2.psad.bw(<32 x i8> %2, <32 x i8> %3)
-  ret <4 x i64> %4
-}
-declare <4 x i64> @llvm.x86.avx2.psad.bw(<32 x i8>, <32 x i8>) nounwind readnone
-
-define <32 x i8> @test_pshufb(<32 x i8> %a0, <32 x i8> %a1, <32 x i8> *%a2) {
-; GENERIC-LABEL: test_pshufb:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpshufb %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    vpshufb (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_pshufb:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpshufb %ymm1, %ymm0, %ymm0 # sched: [1:1.00]
-; HASWELL-NEXT:    vpshufb (%rdi), %ymm0, %ymm0 # sched: [8:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pshufb:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpshufb %ymm1, %ymm0, %ymm0 # sched: [1:1.00]
-; BROADWELL-NEXT:    vpshufb (%rdi), %ymm0, %ymm0 # sched: [7:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pshufb:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpshufb %ymm1, %ymm0, %ymm0 # sched: [1:1.00]
-; SKYLAKE-NEXT:    vpshufb (%rdi), %ymm0, %ymm0 # sched: [8:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pshufb:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpshufb %ymm1, %ymm0, %ymm0 # sched: [1:1.00]
-; SKX-NEXT:    vpshufb (%rdi), %ymm0, %ymm0 # sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_pshufb:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpshufb %ymm1, %ymm0, %ymm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vpshufb (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <32 x i8> @llvm.x86.avx2.pshuf.b(<32 x i8> %a0, <32 x i8> %a1)
-  %2 = load <32 x i8>, <32 x i8> *%a2, align 32
-  %3 = call <32 x i8> @llvm.x86.avx2.pshuf.b(<32 x i8> %1, <32 x i8> %2)
-  ret <32 x i8> %3
-}
-declare <32 x i8> @llvm.x86.avx2.pshuf.b(<32 x i8>, <32 x i8>) nounwind readnone
-
-define <8 x i32> @test_pshufd(<8 x i32> %a0, <8 x i32> *%a1) {
-; GENERIC-LABEL: test_pshufd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpshufd {{.*#+}} ymm0 = ymm0[3,2,1,0,7,6,5,4] sched: [1:1.00]
-; GENERIC-NEXT:    vpshufd {{.*#+}} ymm1 = mem[1,0,3,2,5,4,7,6] sched: [8:1.00]
-; GENERIC-NEXT:    vpaddd %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_pshufd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpshufd {{.*#+}} ymm0 = ymm0[3,2,1,0,7,6,5,4] sched: [1:1.00]
-; HASWELL-NEXT:    vpshufd {{.*#+}} ymm1 = mem[1,0,3,2,5,4,7,6] sched: [8:1.00]
-; HASWELL-NEXT:    vpaddd %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pshufd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpshufd {{.*#+}} ymm0 = ymm0[3,2,1,0,7,6,5,4] sched: [1:1.00]
-; BROADWELL-NEXT:    vpshufd {{.*#+}} ymm1 = mem[1,0,3,2,5,4,7,6] sched: [7:1.00]
-; BROADWELL-NEXT:    vpaddd %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pshufd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpshufd {{.*#+}} ymm0 = ymm0[3,2,1,0,7,6,5,4] sched: [1:1.00]
-; SKYLAKE-NEXT:    vpshufd {{.*#+}} ymm1 = mem[1,0,3,2,5,4,7,6] sched: [8:1.00]
-; SKYLAKE-NEXT:    vpaddd %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pshufd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpshufd {{.*#+}} ymm0 = ymm0[3,2,1,0,7,6,5,4] sched: [1:1.00]
-; SKX-NEXT:    vpshufd {{.*#+}} ymm1 = mem[1,0,3,2,5,4,7,6] sched: [8:1.00]
-; SKX-NEXT:    vpaddd %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_pshufd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpshufd {{.*#+}} ymm1 = mem[1,0,3,2,5,4,7,6] sched: [8:0.50]
-; ZNVER1-NEXT:    vpshufd {{.*#+}} ymm0 = ymm0[3,2,1,0,7,6,5,4] sched: [1:0.25]
-; ZNVER1-NEXT:    vpaddd %ymm1, %ymm0, %ymm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = shufflevector <8 x i32> %a0, <8 x i32> undef, <8 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4>
-  %2 = load <8 x i32>, <8 x i32> *%a1, align 32
-  %3 = shufflevector <8 x i32> %2, <8 x i32> undef, <8 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6>
-  %4 = add <8 x i32> %1, %3
-  ret <8 x i32> %4
-}
-
-define <16 x i16> @test_pshufhw(<16 x i16> %a0, <16 x i16> *%a1) {
-; GENERIC-LABEL: test_pshufhw:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpshufhw {{.*#+}} ymm0 = ymm0[0,1,2,3,7,6,5,4,8,9,10,11,15,14,13,12] sched: [1:1.00]
-; GENERIC-NEXT:    vpshufhw {{.*#+}} ymm1 = mem[0,1,2,3,5,4,7,6,8,9,10,11,13,12,15,14] sched: [8:1.00]
-; GENERIC-NEXT:    vpor %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_pshufhw:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpshufhw {{.*#+}} ymm0 = ymm0[0,1,2,3,7,6,5,4,8,9,10,11,15,14,13,12] sched: [1:1.00]
-; HASWELL-NEXT:    vpshufhw {{.*#+}} ymm1 = mem[0,1,2,3,5,4,7,6,8,9,10,11,13,12,15,14] sched: [8:1.00]
-; HASWELL-NEXT:    vpor %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pshufhw:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpshufhw {{.*#+}} ymm0 = ymm0[0,1,2,3,7,6,5,4,8,9,10,11,15,14,13,12] sched: [1:1.00]
-; BROADWELL-NEXT:    vpshufhw {{.*#+}} ymm1 = mem[0,1,2,3,5,4,7,6,8,9,10,11,13,12,15,14] sched: [7:1.00]
-; BROADWELL-NEXT:    vpor %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pshufhw:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpshufhw {{.*#+}} ymm0 = ymm0[0,1,2,3,7,6,5,4,8,9,10,11,15,14,13,12] sched: [1:1.00]
-; SKYLAKE-NEXT:    vpshufhw {{.*#+}} ymm1 = mem[0,1,2,3,5,4,7,6,8,9,10,11,13,12,15,14] sched: [8:1.00]
-; SKYLAKE-NEXT:    vpor %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pshufhw:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpshufhw {{.*#+}} ymm0 = ymm0[0,1,2,3,7,6,5,4,8,9,10,11,15,14,13,12] sched: [1:1.00]
-; SKX-NEXT:    vpshufhw {{.*#+}} ymm1 = mem[0,1,2,3,5,4,7,6,8,9,10,11,13,12,15,14] sched: [8:1.00]
-; SKX-NEXT:    vpor %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_pshufhw:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpshufhw {{.*#+}} ymm1 = mem[0,1,2,3,5,4,7,6,8,9,10,11,13,12,15,14] sched: [8:0.50]
-; ZNVER1-NEXT:    vpshufhw {{.*#+}} ymm0 = ymm0[0,1,2,3,7,6,5,4,8,9,10,11,15,14,13,12] sched: [1:0.25]
-; ZNVER1-NEXT:    vpor %ymm1, %ymm0, %ymm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = shufflevector <16 x i16> %a0, <16 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 7, i32 6, i32 5, i32 4, i32 8, i32 9, i32 10, i32 11, i32 15, i32 14, i32 13, i32 12>
-  %2 = load <16 x i16>, <16 x i16> *%a1, align 32
-  %3 = shufflevector <16 x i16> %2, <16 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 5, i32 4, i32 7, i32 6, i32 8, i32 9, i32 10, i32 11, i32 13, i32 12, i32 15, i32 14>
-  %4 = or <16 x i16> %1, %3
-  ret <16 x i16> %4
-}
-
-define <16 x i16> @test_pshuflw(<16 x i16> %a0, <16 x i16> *%a1) {
-; GENERIC-LABEL: test_pshuflw:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpshuflw {{.*#+}} ymm0 = ymm0[3,2,1,0,4,5,6,7,11,10,9,8,12,13,14,15] sched: [1:1.00]
-; GENERIC-NEXT:    vpshuflw {{.*#+}} ymm1 = mem[1,0,3,2,4,5,6,7,9,8,11,10,12,13,14,15] sched: [8:1.00]
-; GENERIC-NEXT:    vpor %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_pshuflw:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpshuflw {{.*#+}} ymm0 = ymm0[3,2,1,0,4,5,6,7,11,10,9,8,12,13,14,15] sched: [1:1.00]
-; HASWELL-NEXT:    vpshuflw {{.*#+}} ymm1 = mem[1,0,3,2,4,5,6,7,9,8,11,10,12,13,14,15] sched: [8:1.00]
-; HASWELL-NEXT:    vpor %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pshuflw:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpshuflw {{.*#+}} ymm0 = ymm0[3,2,1,0,4,5,6,7,11,10,9,8,12,13,14,15] sched: [1:1.00]
-; BROADWELL-NEXT:    vpshuflw {{.*#+}} ymm1 = mem[1,0,3,2,4,5,6,7,9,8,11,10,12,13,14,15] sched: [7:1.00]
-; BROADWELL-NEXT:    vpor %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pshuflw:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpshuflw {{.*#+}} ymm0 = ymm0[3,2,1,0,4,5,6,7,11,10,9,8,12,13,14,15] sched: [1:1.00]
-; SKYLAKE-NEXT:    vpshuflw {{.*#+}} ymm1 = mem[1,0,3,2,4,5,6,7,9,8,11,10,12,13,14,15] sched: [8:1.00]
-; SKYLAKE-NEXT:    vpor %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pshuflw:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpshuflw {{.*#+}} ymm0 = ymm0[3,2,1,0,4,5,6,7,11,10,9,8,12,13,14,15] sched: [1:1.00]
-; SKX-NEXT:    vpshuflw {{.*#+}} ymm1 = mem[1,0,3,2,4,5,6,7,9,8,11,10,12,13,14,15] sched: [8:1.00]
-; SKX-NEXT:    vpor %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_pshuflw:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpshuflw {{.*#+}} ymm1 = mem[1,0,3,2,4,5,6,7,9,8,11,10,12,13,14,15] sched: [8:0.50]
-; ZNVER1-NEXT:    vpshuflw {{.*#+}} ymm0 = ymm0[3,2,1,0,4,5,6,7,11,10,9,8,12,13,14,15] sched: [1:0.25]
-; ZNVER1-NEXT:    vpor %ymm1, %ymm0, %ymm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = shufflevector <16 x i16> %a0, <16 x i16> undef, <16 x i32> <i32 3, i32 2, i32 1, i32 0, i32 4, i32 5, i32 6, i32 7, i32 11, i32 10, i32 9, i32 8, i32 12, i32 13, i32 14, i32 15>
-  %2 = load <16 x i16>, <16 x i16> *%a1, align 32
-  %3 = shufflevector <16 x i16> %2, <16 x i16> undef, <16 x i32> <i32 1, i32 0, i32 3, i32 2, i32 4, i32 5, i32 6, i32 7, i32 9, i32 8, i32 11, i32 10, i32 12, i32 13, i32 14, i32 15>
-  %4 = or <16 x i16> %1, %3
-  ret <16 x i16> %4
-}
-
-define <32 x i8> @test_psignb(<32 x i8> %a0, <32 x i8> %a1, <32 x i8> *%a2) {
-; GENERIC-LABEL: test_psignb:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpsignb %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    vpsignb (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_psignb:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpsignb %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; HASWELL-NEXT:    vpsignb (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_psignb:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpsignb %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    vpsignb (%rdi), %ymm0, %ymm0 # sched: [7:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_psignb:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpsignb %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; SKYLAKE-NEXT:    vpsignb (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_psignb:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpsignb %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; SKX-NEXT:    vpsignb (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_psignb:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpsignb %ymm1, %ymm0, %ymm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vpsignb (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <32 x i8> @llvm.x86.avx2.psign.b(<32 x i8> %a0, <32 x i8> %a1)
-  %2 = load <32 x i8>, <32 x i8> *%a2, align 32
-  %3 = call <32 x i8> @llvm.x86.avx2.psign.b(<32 x i8> %1, <32 x i8> %2)
-  ret <32 x i8> %3
-}
-declare <32 x i8> @llvm.x86.avx2.psign.b(<32 x i8>, <32 x i8>) nounwind readnone
-
-define <8 x i32> @test_psignd(<8 x i32> %a0, <8 x i32> %a1, <8 x i32> *%a2) {
-; GENERIC-LABEL: test_psignd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpsignd %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    vpsignd (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_psignd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpsignd %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; HASWELL-NEXT:    vpsignd (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_psignd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpsignd %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    vpsignd (%rdi), %ymm0, %ymm0 # sched: [7:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_psignd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpsignd %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; SKYLAKE-NEXT:    vpsignd (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_psignd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpsignd %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; SKX-NEXT:    vpsignd (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_psignd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpsignd %ymm1, %ymm0, %ymm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vpsignd (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <8 x i32> @llvm.x86.avx2.psign.d(<8 x i32> %a0, <8 x i32> %a1)
-  %2 = load <8 x i32>, <8 x i32> *%a2, align 32
-  %3 = call <8 x i32> @llvm.x86.avx2.psign.d(<8 x i32> %1, <8 x i32> %2)
-  ret <8 x i32> %3
-}
-declare <8 x i32> @llvm.x86.avx2.psign.d(<8 x i32>, <8 x i32>) nounwind readnone
-
-define <16 x i16> @test_psignw(<16 x i16> %a0, <16 x i16> %a1, <16 x i16> *%a2) {
-; GENERIC-LABEL: test_psignw:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpsignw %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    vpsignw (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_psignw:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpsignw %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; HASWELL-NEXT:    vpsignw (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_psignw:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpsignw %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    vpsignw (%rdi), %ymm0, %ymm0 # sched: [7:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_psignw:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpsignw %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; SKYLAKE-NEXT:    vpsignw (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_psignw:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpsignw %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; SKX-NEXT:    vpsignw (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_psignw:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpsignw %ymm1, %ymm0, %ymm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vpsignw (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <16 x i16> @llvm.x86.avx2.psign.w(<16 x i16> %a0, <16 x i16> %a1)
-  %2 = load <16 x i16>, <16 x i16> *%a2, align 32
-  %3 = call <16 x i16> @llvm.x86.avx2.psign.w(<16 x i16> %1, <16 x i16> %2)
-  ret <16 x i16> %3
-}
-declare <16 x i16> @llvm.x86.avx2.psign.w(<16 x i16>, <16 x i16>) nounwind readnone
-
-define <8 x i32> @test_pslld(<8 x i32> %a0, <4 x i32> %a1, <4 x i32> *%a2) {
-; GENERIC-LABEL: test_pslld:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpslld %xmm1, %ymm0, %ymm0 # sched: [4:1.00]
-; GENERIC-NEXT:    vpslld (%rdi), %ymm0, %ymm0 # sched: [11:1.00]
-; GENERIC-NEXT:    vpslld $2, %ymm0, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_pslld:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpslld %xmm1, %ymm0, %ymm0 # sched: [4:1.00]
-; HASWELL-NEXT:    vpslld (%rdi), %ymm0, %ymm0 # sched: [8:1.00]
-; HASWELL-NEXT:    vpslld $2, %ymm0, %ymm0 # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pslld:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpslld %xmm1, %ymm0, %ymm0 # sched: [4:1.00]
-; BROADWELL-NEXT:    vpslld (%rdi), %ymm0, %ymm0 # sched: [7:1.00]
-; BROADWELL-NEXT:    vpslld $2, %ymm0, %ymm0 # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pslld:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpslld %xmm1, %ymm0, %ymm0 # sched: [4:1.00]
-; SKYLAKE-NEXT:    vpslld (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKYLAKE-NEXT:    vpslld $2, %ymm0, %ymm0 # sched: [1:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pslld:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpslld %xmm1, %ymm0, %ymm0 # sched: [4:1.00]
-; SKX-NEXT:    vpslld (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKX-NEXT:    vpslld $2, %ymm0, %ymm0 # sched: [1:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_pslld:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpslld %xmm1, %ymm0, %ymm0 # sched: [2:1.00]
-; ZNVER1-NEXT:    vpslld (%rdi), %ymm0, %ymm0 # sched: [9:1.00]
-; ZNVER1-NEXT:    vpslld $2, %ymm0, %ymm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <8 x i32> @llvm.x86.avx2.psll.d(<8 x i32> %a0, <4 x i32> %a1)
-  %2 = load <4 x i32>, <4 x i32> *%a2, align 16
-  %3 = call <8 x i32> @llvm.x86.avx2.psll.d(<8 x i32> %1, <4 x i32> %2)
-  %4 = shl <8 x i32> %3, <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
-  ret <8 x i32> %4
-}
-declare <8 x i32> @llvm.x86.avx2.psll.d(<8 x i32>, <4 x i32>) nounwind readnone
-
-define <32 x i8> @test_pslldq(<32 x i8> %a0) {
-; GENERIC-LABEL: test_pslldq:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpslldq {{.*#+}} ymm0 = zero,zero,zero,ymm0[0,1,2,3,4,5,6,7,8,9,10,11,12],zero,zero,zero,ymm0[16,17,18,19,20,21,22,23,24,25,26,27,28] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_pslldq:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpslldq {{.*#+}} ymm0 = zero,zero,zero,ymm0[0,1,2,3,4,5,6,7,8,9,10,11,12],zero,zero,zero,ymm0[16,17,18,19,20,21,22,23,24,25,26,27,28] sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pslldq:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpslldq {{.*#+}} ymm0 = zero,zero,zero,ymm0[0,1,2,3,4,5,6,7,8,9,10,11,12],zero,zero,zero,ymm0[16,17,18,19,20,21,22,23,24,25,26,27,28] sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pslldq:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpslldq {{.*#+}} ymm0 = zero,zero,zero,ymm0[0,1,2,3,4,5,6,7,8,9,10,11,12],zero,zero,zero,ymm0[16,17,18,19,20,21,22,23,24,25,26,27,28] sched: [1:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pslldq:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpslldq {{.*#+}} ymm0 = zero,zero,zero,ymm0[0,1,2,3,4,5,6,7,8,9,10,11,12],zero,zero,zero,ymm0[16,17,18,19,20,21,22,23,24,25,26,27,28] sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_pslldq:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpslldq {{.*#+}} ymm0 = zero,zero,zero,ymm0[0,1,2,3,4,5,6,7,8,9,10,11,12],zero,zero,zero,ymm0[16,17,18,19,20,21,22,23,24,25,26,27,28] sched: [2:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = shufflevector <32 x i8> zeroinitializer, <32 x i8> %a0, <32 x i32> <i32 13, i32 14, i32 15, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 29, i32 30, i32 31, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60>
-  ret <32 x i8> %1
-}
-
-define <4 x i64> @test_psllq(<4 x i64> %a0, <2 x i64> %a1, <2 x i64> *%a2) {
-; GENERIC-LABEL: test_psllq:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpsllq %xmm1, %ymm0, %ymm0 # sched: [4:1.00]
-; GENERIC-NEXT:    vpsllq (%rdi), %ymm0, %ymm0 # sched: [11:1.00]
-; GENERIC-NEXT:    vpsllq $2, %ymm0, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_psllq:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpsllq %xmm1, %ymm0, %ymm0 # sched: [4:1.00]
-; HASWELL-NEXT:    vpsllq (%rdi), %ymm0, %ymm0 # sched: [8:1.00]
-; HASWELL-NEXT:    vpsllq $2, %ymm0, %ymm0 # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_psllq:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpsllq %xmm1, %ymm0, %ymm0 # sched: [4:1.00]
-; BROADWELL-NEXT:    vpsllq (%rdi), %ymm0, %ymm0 # sched: [7:1.00]
-; BROADWELL-NEXT:    vpsllq $2, %ymm0, %ymm0 # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_psllq:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpsllq %xmm1, %ymm0, %ymm0 # sched: [4:1.00]
-; SKYLAKE-NEXT:    vpsllq (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKYLAKE-NEXT:    vpsllq $2, %ymm0, %ymm0 # sched: [1:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_psllq:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpsllq %xmm1, %ymm0, %ymm0 # sched: [4:1.00]
-; SKX-NEXT:    vpsllq (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKX-NEXT:    vpsllq $2, %ymm0, %ymm0 # sched: [1:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_psllq:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpsllq %xmm1, %ymm0, %ymm0 # sched: [2:1.00]
-; ZNVER1-NEXT:    vpsllq (%rdi), %ymm0, %ymm0 # sched: [9:1.00]
-; ZNVER1-NEXT:    vpsllq $2, %ymm0, %ymm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <4 x i64> @llvm.x86.avx2.psll.q(<4 x i64> %a0, <2 x i64> %a1)
-  %2 = load <2 x i64>, <2 x i64> *%a2, align 16
-  %3 = call <4 x i64> @llvm.x86.avx2.psll.q(<4 x i64> %1, <2 x i64> %2)
-  %4 = shl <4 x i64> %3, <i64 2, i64 2, i64 2, i64 2>
-  ret <4 x i64> %4
-}
-declare <4 x i64> @llvm.x86.avx2.psll.q(<4 x i64>, <2 x i64>) nounwind readnone
-
-define <4 x i32> @test_psllvd(<4 x i32> %a0, <4 x i32> %a1, <4 x i32> *%a2) {
-; GENERIC-LABEL: test_psllvd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpsllvd %xmm1, %xmm0, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vpsllvd (%rdi), %xmm0, %xmm0 # sched: [7:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_psllvd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpsllvd %xmm1, %xmm0, %xmm0 # sched: [3:2.00]
-; HASWELL-NEXT:    vpsllvd (%rdi), %xmm0, %xmm0 # sched: [9:2.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_psllvd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpsllvd %xmm1, %xmm0, %xmm0 # sched: [3:2.00]
-; BROADWELL-NEXT:    vpsllvd (%rdi), %xmm0, %xmm0 # sched: [8:2.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_psllvd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpsllvd %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SKYLAKE-NEXT:    vpsllvd (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_psllvd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpsllvd %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SKX-NEXT:    vpsllvd (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_psllvd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpsllvd %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; ZNVER1-NEXT:    vpsllvd (%rdi), %xmm0, %xmm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <4 x i32> @llvm.x86.avx2.psllv.d(<4 x i32> %a0, <4 x i32> %a1)
-  %2 = load <4 x i32>, <4 x i32> *%a2, align 16
-  %3 = call <4 x i32> @llvm.x86.avx2.psllv.d(<4 x i32> %1, <4 x i32> %2)
-  ret <4 x i32> %3
-}
-declare <4 x i32> @llvm.x86.avx2.psllv.d(<4 x i32>, <4 x i32>) nounwind readnone
-
-define <8 x i32> @test_psllvd_ymm(<8 x i32> %a0, <8 x i32> %a1, <8 x i32> *%a2) {
-; GENERIC-LABEL: test_psllvd_ymm:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpsllvd %ymm1, %ymm0, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vpsllvd (%rdi), %ymm0, %ymm0 # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_psllvd_ymm:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpsllvd %ymm1, %ymm0, %ymm0 # sched: [3:2.00]
-; HASWELL-NEXT:    vpsllvd (%rdi), %ymm0, %ymm0 # sched: [10:2.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_psllvd_ymm:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpsllvd %ymm1, %ymm0, %ymm0 # sched: [3:2.00]
-; BROADWELL-NEXT:    vpsllvd (%rdi), %ymm0, %ymm0 # sched: [9:2.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_psllvd_ymm:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpsllvd %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; SKYLAKE-NEXT:    vpsllvd (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_psllvd_ymm:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpsllvd %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; SKX-NEXT:    vpsllvd (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_psllvd_ymm:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpsllvd %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; ZNVER1-NEXT:    vpsllvd (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <8 x i32> @llvm.x86.avx2.psllv.d.256(<8 x i32> %a0, <8 x i32> %a1)
-  %2 = load <8 x i32>, <8 x i32> *%a2, align 32
-  %3 = call <8 x i32> @llvm.x86.avx2.psllv.d.256(<8 x i32> %1, <8 x i32> %2)
-  ret <8 x i32> %3
-}
-declare <8 x i32> @llvm.x86.avx2.psllv.d.256(<8 x i32>, <8 x i32>) nounwind readnone
-
-define <2 x i64> @test_psllvq(<2 x i64> %a0, <2 x i64> %a1, <2 x i64> *%a2) {
-; GENERIC-LABEL: test_psllvq:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpsllvq %xmm1, %xmm0, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vpsllvq (%rdi), %xmm0, %xmm0 # sched: [7:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_psllvq:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpsllvq %xmm1, %xmm0, %xmm0 # sched: [1:1.00]
-; HASWELL-NEXT:    vpsllvq (%rdi), %xmm0, %xmm0 # sched: [7:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_psllvq:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpsllvq %xmm1, %xmm0, %xmm0 # sched: [1:1.00]
-; BROADWELL-NEXT:    vpsllvq (%rdi), %xmm0, %xmm0 # sched: [6:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_psllvq:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpsllvq %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SKYLAKE-NEXT:    vpsllvq (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_psllvq:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpsllvq %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SKX-NEXT:    vpsllvq (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_psllvq:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpsllvq %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; ZNVER1-NEXT:    vpsllvq (%rdi), %xmm0, %xmm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <2 x i64> @llvm.x86.avx2.psllv.q(<2 x i64> %a0, <2 x i64> %a1)
-  %2 = load <2 x i64>, <2 x i64> *%a2, align 16
-  %3 = call <2 x i64> @llvm.x86.avx2.psllv.q(<2 x i64> %1, <2 x i64> %2)
-  ret <2 x i64> %3
-}
-declare <2 x i64> @llvm.x86.avx2.psllv.q(<2 x i64>, <2 x i64>) nounwind readnone
-
-define <4 x i64> @test_psllvq_ymm(<4 x i64> %a0, <4 x i64> %a1, <4 x i64> *%a2) {
-; GENERIC-LABEL: test_psllvq_ymm:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpsllvq %ymm1, %ymm0, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vpsllvq (%rdi), %ymm0, %ymm0 # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_psllvq_ymm:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpsllvq %ymm1, %ymm0, %ymm0 # sched: [1:1.00]
-; HASWELL-NEXT:    vpsllvq (%rdi), %ymm0, %ymm0 # sched: [8:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_psllvq_ymm:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpsllvq %ymm1, %ymm0, %ymm0 # sched: [1:1.00]
-; BROADWELL-NEXT:    vpsllvq (%rdi), %ymm0, %ymm0 # sched: [7:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_psllvq_ymm:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpsllvq %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; SKYLAKE-NEXT:    vpsllvq (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_psllvq_ymm:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpsllvq %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; SKX-NEXT:    vpsllvq (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_psllvq_ymm:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpsllvq %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; ZNVER1-NEXT:    vpsllvq (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <4 x i64> @llvm.x86.avx2.psllv.q.256(<4 x i64> %a0, <4 x i64> %a1)
-  %2 = load <4 x i64>, <4 x i64> *%a2, align 32
-  %3 = call <4 x i64> @llvm.x86.avx2.psllv.q.256(<4 x i64> %1, <4 x i64> %2)
-  ret <4 x i64> %3
-}
-declare <4 x i64> @llvm.x86.avx2.psllv.q.256(<4 x i64>, <4 x i64>) nounwind readnone
-
-define <16 x i16> @test_psllw(<16 x i16> %a0, <8 x i16> %a1, <8 x i16> *%a2) {
-; GENERIC-LABEL: test_psllw:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpsllw %xmm1, %ymm0, %ymm0 # sched: [4:1.00]
-; GENERIC-NEXT:    vpsllw (%rdi), %ymm0, %ymm0 # sched: [11:1.00]
-; GENERIC-NEXT:    vpsllw $2, %ymm0, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_psllw:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpsllw %xmm1, %ymm0, %ymm0 # sched: [4:1.00]
-; HASWELL-NEXT:    vpsllw (%rdi), %ymm0, %ymm0 # sched: [8:1.00]
-; HASWELL-NEXT:    vpsllw $2, %ymm0, %ymm0 # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_psllw:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpsllw %xmm1, %ymm0, %ymm0 # sched: [4:1.00]
-; BROADWELL-NEXT:    vpsllw (%rdi), %ymm0, %ymm0 # sched: [7:1.00]
-; BROADWELL-NEXT:    vpsllw $2, %ymm0, %ymm0 # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_psllw:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpsllw %xmm1, %ymm0, %ymm0 # sched: [4:1.00]
-; SKYLAKE-NEXT:    vpsllw (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKYLAKE-NEXT:    vpsllw $2, %ymm0, %ymm0 # sched: [1:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_psllw:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpsllw %xmm1, %ymm0, %ymm0 # sched: [4:1.00]
-; SKX-NEXT:    vpsllw (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKX-NEXT:    vpsllw $2, %ymm0, %ymm0 # sched: [1:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_psllw:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpsllw %xmm1, %ymm0, %ymm0 # sched: [2:1.00]
-; ZNVER1-NEXT:    vpsllw (%rdi), %ymm0, %ymm0 # sched: [9:1.00]
-; ZNVER1-NEXT:    vpsllw $2, %ymm0, %ymm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <16 x i16> @llvm.x86.avx2.psll.w(<16 x i16> %a0, <8 x i16> %a1)
-  %2 = load <8 x i16>, <8 x i16> *%a2, align 16
-  %3 = call <16 x i16> @llvm.x86.avx2.psll.w(<16 x i16> %1, <8 x i16> %2)
-  %4 = shl <16 x i16> %3, <i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2>
-  ret <16 x i16> %4
-}
-declare <16 x i16> @llvm.x86.avx2.psll.w(<16 x i16>, <8 x i16>) nounwind readnone
-
-define <8 x i32> @test_psrad(<8 x i32> %a0, <4 x i32> %a1, <4 x i32> *%a2) {
-; GENERIC-LABEL: test_psrad:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpsrad %xmm1, %ymm0, %ymm0 # sched: [4:1.00]
-; GENERIC-NEXT:    vpsrad (%rdi), %ymm0, %ymm0 # sched: [11:1.00]
-; GENERIC-NEXT:    vpsrad $2, %ymm0, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_psrad:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpsrad %xmm1, %ymm0, %ymm0 # sched: [4:1.00]
-; HASWELL-NEXT:    vpsrad (%rdi), %ymm0, %ymm0 # sched: [8:1.00]
-; HASWELL-NEXT:    vpsrad $2, %ymm0, %ymm0 # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_psrad:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpsrad %xmm1, %ymm0, %ymm0 # sched: [4:1.00]
-; BROADWELL-NEXT:    vpsrad (%rdi), %ymm0, %ymm0 # sched: [7:1.00]
-; BROADWELL-NEXT:    vpsrad $2, %ymm0, %ymm0 # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_psrad:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpsrad %xmm1, %ymm0, %ymm0 # sched: [4:1.00]
-; SKYLAKE-NEXT:    vpsrad (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKYLAKE-NEXT:    vpsrad $2, %ymm0, %ymm0 # sched: [1:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_psrad:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpsrad %xmm1, %ymm0, %ymm0 # sched: [4:1.00]
-; SKX-NEXT:    vpsrad (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKX-NEXT:    vpsrad $2, %ymm0, %ymm0 # sched: [1:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_psrad:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpsrad %xmm1, %ymm0, %ymm0 # sched: [2:1.00]
-; ZNVER1-NEXT:    vpsrad (%rdi), %ymm0, %ymm0 # sched: [9:1.00]
-; ZNVER1-NEXT:    vpsrad $2, %ymm0, %ymm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <8 x i32> @llvm.x86.avx2.psra.d(<8 x i32> %a0, <4 x i32> %a1)
-  %2 = load <4 x i32>, <4 x i32> *%a2, align 16
-  %3 = call <8 x i32> @llvm.x86.avx2.psra.d(<8 x i32> %1, <4 x i32> %2)
-  %4 = ashr <8 x i32> %3, <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
-  ret <8 x i32> %4
-}
-declare <8 x i32> @llvm.x86.avx2.psra.d(<8 x i32>, <4 x i32>) nounwind readnone
-
-define <4 x i32> @test_psravd(<4 x i32> %a0, <4 x i32> %a1, <4 x i32> *%a2) {
-; GENERIC-LABEL: test_psravd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpsravd %xmm1, %xmm0, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vpsravd (%rdi), %xmm0, %xmm0 # sched: [7:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_psravd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpsravd %xmm1, %xmm0, %xmm0 # sched: [3:2.00]
-; HASWELL-NEXT:    vpsravd (%rdi), %xmm0, %xmm0 # sched: [9:2.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_psravd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpsravd %xmm1, %xmm0, %xmm0 # sched: [3:2.00]
-; BROADWELL-NEXT:    vpsravd (%rdi), %xmm0, %xmm0 # sched: [8:2.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_psravd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpsravd %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SKYLAKE-NEXT:    vpsravd (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_psravd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpsravd %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SKX-NEXT:    vpsravd (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_psravd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpsravd %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; ZNVER1-NEXT:    vpsravd (%rdi), %xmm0, %xmm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <4 x i32> @llvm.x86.avx2.psrav.d(<4 x i32> %a0, <4 x i32> %a1)
-  %2 = load <4 x i32>, <4 x i32> *%a2, align 16
-  %3 = call <4 x i32> @llvm.x86.avx2.psrav.d(<4 x i32> %1, <4 x i32> %2)
-  ret <4 x i32> %3
-}
-declare <4 x i32> @llvm.x86.avx2.psrav.d(<4 x i32>, <4 x i32>) nounwind readnone
-
-define <8 x i32> @test_psravd_ymm(<8 x i32> %a0, <8 x i32> %a1, <8 x i32> *%a2) {
-; GENERIC-LABEL: test_psravd_ymm:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpsravd %ymm1, %ymm0, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vpsravd (%rdi), %ymm0, %ymm0 # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_psravd_ymm:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpsravd %ymm1, %ymm0, %ymm0 # sched: [3:2.00]
-; HASWELL-NEXT:    vpsravd (%rdi), %ymm0, %ymm0 # sched: [10:2.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_psravd_ymm:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpsravd %ymm1, %ymm0, %ymm0 # sched: [3:2.00]
-; BROADWELL-NEXT:    vpsravd (%rdi), %ymm0, %ymm0 # sched: [9:2.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_psravd_ymm:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpsravd %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; SKYLAKE-NEXT:    vpsravd (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_psravd_ymm:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpsravd %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; SKX-NEXT:    vpsravd (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_psravd_ymm:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpsravd %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; ZNVER1-NEXT:    vpsravd (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <8 x i32> @llvm.x86.avx2.psrav.d.256(<8 x i32> %a0, <8 x i32> %a1)
-  %2 = load <8 x i32>, <8 x i32> *%a2, align 32
-  %3 = call <8 x i32> @llvm.x86.avx2.psrav.d.256(<8 x i32> %1, <8 x i32> %2)
-  ret <8 x i32> %3
-}
-declare <8 x i32> @llvm.x86.avx2.psrav.d.256(<8 x i32>, <8 x i32>) nounwind readnone
-
-define <16 x i16> @test_psraw(<16 x i16> %a0, <8 x i16> %a1, <8 x i16> *%a2) {
-; GENERIC-LABEL: test_psraw:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpsraw %xmm1, %ymm0, %ymm0 # sched: [4:1.00]
-; GENERIC-NEXT:    vpsraw (%rdi), %ymm0, %ymm0 # sched: [11:1.00]
-; GENERIC-NEXT:    vpsraw $2, %ymm0, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_psraw:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpsraw %xmm1, %ymm0, %ymm0 # sched: [4:1.00]
-; HASWELL-NEXT:    vpsraw (%rdi), %ymm0, %ymm0 # sched: [8:1.00]
-; HASWELL-NEXT:    vpsraw $2, %ymm0, %ymm0 # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_psraw:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpsraw %xmm1, %ymm0, %ymm0 # sched: [4:1.00]
-; BROADWELL-NEXT:    vpsraw (%rdi), %ymm0, %ymm0 # sched: [7:1.00]
-; BROADWELL-NEXT:    vpsraw $2, %ymm0, %ymm0 # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_psraw:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpsraw %xmm1, %ymm0, %ymm0 # sched: [4:1.00]
-; SKYLAKE-NEXT:    vpsraw (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKYLAKE-NEXT:    vpsraw $2, %ymm0, %ymm0 # sched: [1:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_psraw:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpsraw %xmm1, %ymm0, %ymm0 # sched: [4:1.00]
-; SKX-NEXT:    vpsraw (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKX-NEXT:    vpsraw $2, %ymm0, %ymm0 # sched: [1:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_psraw:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpsraw %xmm1, %ymm0, %ymm0 # sched: [2:1.00]
-; ZNVER1-NEXT:    vpsraw (%rdi), %ymm0, %ymm0 # sched: [9:1.00]
-; ZNVER1-NEXT:    vpsraw $2, %ymm0, %ymm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <16 x i16> @llvm.x86.avx2.psra.w(<16 x i16> %a0, <8 x i16> %a1)
-  %2 = load <8 x i16>, <8 x i16> *%a2, align 16
-  %3 = call <16 x i16> @llvm.x86.avx2.psra.w(<16 x i16> %1, <8 x i16> %2)
-  %4 = ashr <16 x i16> %3, <i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2>
-  ret <16 x i16> %4
-}
-declare <16 x i16> @llvm.x86.avx2.psra.w(<16 x i16>, <8 x i16>) nounwind readnone
-
-define <8 x i32> @test_psrld(<8 x i32> %a0, <4 x i32> %a1, <4 x i32> *%a2) {
-; GENERIC-LABEL: test_psrld:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpsrld %xmm1, %ymm0, %ymm0 # sched: [4:1.00]
-; GENERIC-NEXT:    vpsrld (%rdi), %ymm0, %ymm0 # sched: [11:1.00]
-; GENERIC-NEXT:    vpsrld $2, %ymm0, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_psrld:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpsrld %xmm1, %ymm0, %ymm0 # sched: [4:1.00]
-; HASWELL-NEXT:    vpsrld (%rdi), %ymm0, %ymm0 # sched: [8:1.00]
-; HASWELL-NEXT:    vpsrld $2, %ymm0, %ymm0 # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_psrld:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpsrld %xmm1, %ymm0, %ymm0 # sched: [4:1.00]
-; BROADWELL-NEXT:    vpsrld (%rdi), %ymm0, %ymm0 # sched: [7:1.00]
-; BROADWELL-NEXT:    vpsrld $2, %ymm0, %ymm0 # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_psrld:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpsrld %xmm1, %ymm0, %ymm0 # sched: [4:1.00]
-; SKYLAKE-NEXT:    vpsrld (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKYLAKE-NEXT:    vpsrld $2, %ymm0, %ymm0 # sched: [1:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_psrld:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpsrld %xmm1, %ymm0, %ymm0 # sched: [4:1.00]
-; SKX-NEXT:    vpsrld (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKX-NEXT:    vpsrld $2, %ymm0, %ymm0 # sched: [1:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_psrld:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpsrld %xmm1, %ymm0, %ymm0 # sched: [2:1.00]
-; ZNVER1-NEXT:    vpsrld (%rdi), %ymm0, %ymm0 # sched: [9:1.00]
-; ZNVER1-NEXT:    vpsrld $2, %ymm0, %ymm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <8 x i32> @llvm.x86.avx2.psrl.d(<8 x i32> %a0, <4 x i32> %a1)
-  %2 = load <4 x i32>, <4 x i32> *%a2, align 16
-  %3 = call <8 x i32> @llvm.x86.avx2.psrl.d(<8 x i32> %1, <4 x i32> %2)
-  %4 = lshr <8 x i32> %3, <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
-  ret <8 x i32> %4
-}
-declare <8 x i32> @llvm.x86.avx2.psrl.d(<8 x i32>, <4 x i32>) nounwind readnone
-
-define <32 x i8> @test_psrldq(<32 x i8> %a0) {
-; GENERIC-LABEL: test_psrldq:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpsrldq {{.*#+}} ymm0 = ymm0[3,4,5,6,7,8,9,10,11,12,13,14,15],zero,zero,zero,ymm0[19,20,21,22,23,24,25,26,27,28,29,30,31],zero,zero,zero sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_psrldq:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpsrldq {{.*#+}} ymm0 = ymm0[3,4,5,6,7,8,9,10,11,12,13,14,15],zero,zero,zero,ymm0[19,20,21,22,23,24,25,26,27,28,29,30,31],zero,zero,zero sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_psrldq:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpsrldq {{.*#+}} ymm0 = ymm0[3,4,5,6,7,8,9,10,11,12,13,14,15],zero,zero,zero,ymm0[19,20,21,22,23,24,25,26,27,28,29,30,31],zero,zero,zero sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_psrldq:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpsrldq {{.*#+}} ymm0 = ymm0[3,4,5,6,7,8,9,10,11,12,13,14,15],zero,zero,zero,ymm0[19,20,21,22,23,24,25,26,27,28,29,30,31],zero,zero,zero sched: [1:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_psrldq:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpsrldq {{.*#+}} ymm0 = ymm0[3,4,5,6,7,8,9,10,11,12,13,14,15],zero,zero,zero,ymm0[19,20,21,22,23,24,25,26,27,28,29,30,31],zero,zero,zero sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_psrldq:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpsrldq {{.*#+}} ymm0 = ymm0[3,4,5,6,7,8,9,10,11,12,13,14,15],zero,zero,zero,ymm0[19,20,21,22,23,24,25,26,27,28,29,30,31],zero,zero,zero sched: [2:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = shufflevector <32 x i8> %a0, <32 x i8> zeroinitializer, <32 x i32> <i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 32, i32 33, i32 34, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 48, i32 49, i32 50>
-  ret <32 x i8> %1
-}
-
-define <4 x i64> @test_psrlq(<4 x i64> %a0, <2 x i64> %a1, <2 x i64> *%a2) {
-; GENERIC-LABEL: test_psrlq:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpsrlq %xmm1, %ymm0, %ymm0 # sched: [4:1.00]
-; GENERIC-NEXT:    vpsrlq (%rdi), %ymm0, %ymm0 # sched: [11:1.00]
-; GENERIC-NEXT:    vpsrlq $2, %ymm0, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_psrlq:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpsrlq %xmm1, %ymm0, %ymm0 # sched: [4:1.00]
-; HASWELL-NEXT:    vpsrlq (%rdi), %ymm0, %ymm0 # sched: [8:1.00]
-; HASWELL-NEXT:    vpsrlq $2, %ymm0, %ymm0 # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_psrlq:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpsrlq %xmm1, %ymm0, %ymm0 # sched: [4:1.00]
-; BROADWELL-NEXT:    vpsrlq (%rdi), %ymm0, %ymm0 # sched: [7:1.00]
-; BROADWELL-NEXT:    vpsrlq $2, %ymm0, %ymm0 # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_psrlq:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpsrlq %xmm1, %ymm0, %ymm0 # sched: [4:1.00]
-; SKYLAKE-NEXT:    vpsrlq (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKYLAKE-NEXT:    vpsrlq $2, %ymm0, %ymm0 # sched: [1:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_psrlq:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpsrlq %xmm1, %ymm0, %ymm0 # sched: [4:1.00]
-; SKX-NEXT:    vpsrlq (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKX-NEXT:    vpsrlq $2, %ymm0, %ymm0 # sched: [1:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_psrlq:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpsrlq %xmm1, %ymm0, %ymm0 # sched: [2:1.00]
-; ZNVER1-NEXT:    vpsrlq (%rdi), %ymm0, %ymm0 # sched: [9:1.00]
-; ZNVER1-NEXT:    vpsrlq $2, %ymm0, %ymm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <4 x i64> @llvm.x86.avx2.psrl.q(<4 x i64> %a0, <2 x i64> %a1)
-  %2 = load <2 x i64>, <2 x i64> *%a2, align 16
-  %3 = call <4 x i64> @llvm.x86.avx2.psrl.q(<4 x i64> %1, <2 x i64> %2)
-  %4 = lshr <4 x i64> %3, <i64 2, i64 2, i64 2, i64 2>
-  ret <4 x i64> %4
-}
-declare <4 x i64> @llvm.x86.avx2.psrl.q(<4 x i64>, <2 x i64>) nounwind readnone
-
-define <4 x i32> @test_psrlvd(<4 x i32> %a0, <4 x i32> %a1, <4 x i32> *%a2) {
-; GENERIC-LABEL: test_psrlvd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpsrlvd %xmm1, %xmm0, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vpsrlvd (%rdi), %xmm0, %xmm0 # sched: [7:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_psrlvd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpsrlvd %xmm1, %xmm0, %xmm0 # sched: [3:2.00]
-; HASWELL-NEXT:    vpsrlvd (%rdi), %xmm0, %xmm0 # sched: [9:2.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_psrlvd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpsrlvd %xmm1, %xmm0, %xmm0 # sched: [3:2.00]
-; BROADWELL-NEXT:    vpsrlvd (%rdi), %xmm0, %xmm0 # sched: [8:2.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_psrlvd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpsrlvd %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SKYLAKE-NEXT:    vpsrlvd (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_psrlvd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpsrlvd %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SKX-NEXT:    vpsrlvd (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_psrlvd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpsrlvd %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; ZNVER1-NEXT:    vpsrlvd (%rdi), %xmm0, %xmm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <4 x i32> @llvm.x86.avx2.psrlv.d(<4 x i32> %a0, <4 x i32> %a1)
-  %2 = load <4 x i32>, <4 x i32> *%a2, align 16
-  %3 = call <4 x i32> @llvm.x86.avx2.psrlv.d(<4 x i32> %1, <4 x i32> %2)
-  ret <4 x i32> %3
-}
-declare <4 x i32> @llvm.x86.avx2.psrlv.d(<4 x i32>, <4 x i32>) nounwind readnone
-
-define <8 x i32> @test_psrlvd_ymm(<8 x i32> %a0, <8 x i32> %a1, <8 x i32> *%a2) {
-; GENERIC-LABEL: test_psrlvd_ymm:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpsrlvd %ymm1, %ymm0, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vpsrlvd (%rdi), %ymm0, %ymm0 # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_psrlvd_ymm:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpsrlvd %ymm1, %ymm0, %ymm0 # sched: [3:2.00]
-; HASWELL-NEXT:    vpsrlvd (%rdi), %ymm0, %ymm0 # sched: [10:2.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_psrlvd_ymm:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpsrlvd %ymm1, %ymm0, %ymm0 # sched: [3:2.00]
-; BROADWELL-NEXT:    vpsrlvd (%rdi), %ymm0, %ymm0 # sched: [9:2.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_psrlvd_ymm:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpsrlvd %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; SKYLAKE-NEXT:    vpsrlvd (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_psrlvd_ymm:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpsrlvd %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; SKX-NEXT:    vpsrlvd (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_psrlvd_ymm:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpsrlvd %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; ZNVER1-NEXT:    vpsrlvd (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <8 x i32> @llvm.x86.avx2.psrlv.d.256(<8 x i32> %a0, <8 x i32> %a1)
-  %2 = load <8 x i32>, <8 x i32> *%a2, align 32
-  %3 = call <8 x i32> @llvm.x86.avx2.psrlv.d.256(<8 x i32> %1, <8 x i32> %2)
-  ret <8 x i32> %3
-}
-declare <8 x i32> @llvm.x86.avx2.psrlv.d.256(<8 x i32>, <8 x i32>) nounwind readnone
-
-define <2 x i64> @test_psrlvq(<2 x i64> %a0, <2 x i64> %a1, <2 x i64> *%a2) {
-; GENERIC-LABEL: test_psrlvq:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpsrlvq %xmm1, %xmm0, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vpsrlvq (%rdi), %xmm0, %xmm0 # sched: [7:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_psrlvq:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpsrlvq %xmm1, %xmm0, %xmm0 # sched: [1:1.00]
-; HASWELL-NEXT:    vpsrlvq (%rdi), %xmm0, %xmm0 # sched: [7:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_psrlvq:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpsrlvq %xmm1, %xmm0, %xmm0 # sched: [1:1.00]
-; BROADWELL-NEXT:    vpsrlvq (%rdi), %xmm0, %xmm0 # sched: [6:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_psrlvq:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpsrlvq %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SKYLAKE-NEXT:    vpsrlvq (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_psrlvq:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpsrlvq %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SKX-NEXT:    vpsrlvq (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_psrlvq:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpsrlvq %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; ZNVER1-NEXT:    vpsrlvq (%rdi), %xmm0, %xmm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <2 x i64> @llvm.x86.avx2.psrlv.q(<2 x i64> %a0, <2 x i64> %a1)
-  %2 = load <2 x i64>, <2 x i64> *%a2, align 16
-  %3 = call <2 x i64> @llvm.x86.avx2.psrlv.q(<2 x i64> %1, <2 x i64> %2)
-  ret <2 x i64> %3
-}
-declare <2 x i64> @llvm.x86.avx2.psrlv.q(<2 x i64>, <2 x i64>) nounwind readnone
-
-define <4 x i64> @test_psrlvq_ymm(<4 x i64> %a0, <4 x i64> %a1, <4 x i64> *%a2) {
-; GENERIC-LABEL: test_psrlvq_ymm:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpsrlvq %ymm1, %ymm0, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vpsrlvq (%rdi), %ymm0, %ymm0 # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_psrlvq_ymm:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpsrlvq %ymm1, %ymm0, %ymm0 # sched: [1:1.00]
-; HASWELL-NEXT:    vpsrlvq (%rdi), %ymm0, %ymm0 # sched: [8:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_psrlvq_ymm:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpsrlvq %ymm1, %ymm0, %ymm0 # sched: [1:1.00]
-; BROADWELL-NEXT:    vpsrlvq (%rdi), %ymm0, %ymm0 # sched: [7:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_psrlvq_ymm:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpsrlvq %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; SKYLAKE-NEXT:    vpsrlvq (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_psrlvq_ymm:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpsrlvq %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; SKX-NEXT:    vpsrlvq (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_psrlvq_ymm:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpsrlvq %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; ZNVER1-NEXT:    vpsrlvq (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <4 x i64> @llvm.x86.avx2.psrlv.q.256(<4 x i64> %a0, <4 x i64> %a1)
-  %2 = load <4 x i64>, <4 x i64> *%a2, align 32
-  %3 = call <4 x i64> @llvm.x86.avx2.psrlv.q.256(<4 x i64> %1, <4 x i64> %2)
-  ret <4 x i64> %3
-}
-declare <4 x i64> @llvm.x86.avx2.psrlv.q.256(<4 x i64>, <4 x i64>) nounwind readnone
-
-define <16 x i16> @test_psrlw(<16 x i16> %a0, <8 x i16> %a1, <8 x i16> *%a2) {
-; GENERIC-LABEL: test_psrlw:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpsrlw %xmm1, %ymm0, %ymm0 # sched: [4:1.00]
-; GENERIC-NEXT:    vpsrlw (%rdi), %ymm0, %ymm0 # sched: [11:1.00]
-; GENERIC-NEXT:    vpsrlw $2, %ymm0, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_psrlw:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpsrlw %xmm1, %ymm0, %ymm0 # sched: [4:1.00]
-; HASWELL-NEXT:    vpsrlw (%rdi), %ymm0, %ymm0 # sched: [8:1.00]
-; HASWELL-NEXT:    vpsrlw $2, %ymm0, %ymm0 # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_psrlw:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpsrlw %xmm1, %ymm0, %ymm0 # sched: [4:1.00]
-; BROADWELL-NEXT:    vpsrlw (%rdi), %ymm0, %ymm0 # sched: [7:1.00]
-; BROADWELL-NEXT:    vpsrlw $2, %ymm0, %ymm0 # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_psrlw:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpsrlw %xmm1, %ymm0, %ymm0 # sched: [4:1.00]
-; SKYLAKE-NEXT:    vpsrlw (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKYLAKE-NEXT:    vpsrlw $2, %ymm0, %ymm0 # sched: [1:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_psrlw:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpsrlw %xmm1, %ymm0, %ymm0 # sched: [4:1.00]
-; SKX-NEXT:    vpsrlw (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKX-NEXT:    vpsrlw $2, %ymm0, %ymm0 # sched: [1:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_psrlw:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpsrlw %xmm1, %ymm0, %ymm0 # sched: [2:1.00]
-; ZNVER1-NEXT:    vpsrlw (%rdi), %ymm0, %ymm0 # sched: [9:1.00]
-; ZNVER1-NEXT:    vpsrlw $2, %ymm0, %ymm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <16 x i16> @llvm.x86.avx2.psrl.w(<16 x i16> %a0, <8 x i16> %a1)
-  %2 = load <8 x i16>, <8 x i16> *%a2, align 16
-  %3 = call <16 x i16> @llvm.x86.avx2.psrl.w(<16 x i16> %1, <8 x i16> %2)
-  %4 = lshr <16 x i16> %3, <i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2, i16 2>
-  ret <16 x i16> %4
-}
-declare <16 x i16> @llvm.x86.avx2.psrl.w(<16 x i16>, <8 x i16>) nounwind readnone
-
-define <32 x i8> @test_psubb(<32 x i8> %a0, <32 x i8> %a1, <32 x i8> *%a2) {
-; GENERIC-LABEL: test_psubb:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpsubb %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    vpsubb (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_psubb:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpsubb %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; HASWELL-NEXT:    vpsubb (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_psubb:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpsubb %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    vpsubb (%rdi), %ymm0, %ymm0 # sched: [7:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_psubb:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpsubb %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    vpsubb (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_psubb:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpsubb %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    vpsubb (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_psubb:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpsubb %ymm1, %ymm0, %ymm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vpsubb (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = sub <32 x i8> %a0, %a1
-  %2 = load <32 x i8>, <32 x i8> *%a2, align 32
-  %3 = sub <32 x i8> %1, %2
-  ret <32 x i8> %3
-}
-
-define <8 x i32> @test_psubd(<8 x i32> %a0, <8 x i32> %a1, <8 x i32> *%a2) {
-; GENERIC-LABEL: test_psubd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpsubd %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    vpsubd (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_psubd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpsubd %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; HASWELL-NEXT:    vpsubd (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_psubd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpsubd %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    vpsubd (%rdi), %ymm0, %ymm0 # sched: [7:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_psubd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpsubd %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    vpsubd (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_psubd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpsubd %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    vpsubd (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_psubd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpsubd %ymm1, %ymm0, %ymm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vpsubd (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = sub <8 x i32> %a0, %a1
-  %2 = load <8 x i32>, <8 x i32> *%a2, align 32
-  %3 = sub <8 x i32> %1, %2
-  ret <8 x i32> %3
-}
-
-define <4 x i64> @test_psubq(<4 x i64> %a0, <4 x i64> %a1, <4 x i64> *%a2) {
-; GENERIC-LABEL: test_psubq:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpsubq %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    vpsubq (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_psubq:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpsubq %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; HASWELL-NEXT:    vpsubq (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_psubq:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpsubq %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    vpsubq (%rdi), %ymm0, %ymm0 # sched: [7:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_psubq:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpsubq %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    vpsubq (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_psubq:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpsubq %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    vpsubq (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_psubq:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpsubq %ymm1, %ymm0, %ymm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vpsubq (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = sub <4 x i64> %a0, %a1
-  %2 = load <4 x i64>, <4 x i64> *%a2, align 32
-  %3 = sub <4 x i64> %1, %2
-  ret <4 x i64> %3
-}
-
-define <32 x i8> @test_psubsb(<32 x i8> %a0, <32 x i8> %a1, <32 x i8> *%a2) {
-; GENERIC-LABEL: test_psubsb:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpsubsb %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    vpsubsb (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_psubsb:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpsubsb %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; HASWELL-NEXT:    vpsubsb (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_psubsb:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpsubsb %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    vpsubsb (%rdi), %ymm0, %ymm0 # sched: [7:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_psubsb:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpsubsb %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; SKYLAKE-NEXT:    vpsubsb (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_psubsb:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpsubsb %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; SKX-NEXT:    vpsubsb (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_psubsb:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpsubsb %ymm1, %ymm0, %ymm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vpsubsb (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <32 x i8> @llvm.x86.avx2.psubs.b(<32 x i8> %a0, <32 x i8> %a1)
-  %2 = load <32 x i8>, <32 x i8> *%a2, align 32
-  %3 = call <32 x i8> @llvm.x86.avx2.psubs.b(<32 x i8> %1, <32 x i8> %2)
-  ret <32 x i8> %3
-}
-declare <32 x i8> @llvm.x86.avx2.psubs.b(<32 x i8>, <32 x i8>) nounwind readnone
-
-define <16 x i16> @test_psubsw(<16 x i16> %a0, <16 x i16> %a1, <16 x i16> *%a2) {
-; GENERIC-LABEL: test_psubsw:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpsubsw %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    vpsubsw (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_psubsw:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpsubsw %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; HASWELL-NEXT:    vpsubsw (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_psubsw:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpsubsw %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    vpsubsw (%rdi), %ymm0, %ymm0 # sched: [7:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_psubsw:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpsubsw %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; SKYLAKE-NEXT:    vpsubsw (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_psubsw:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpsubsw %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; SKX-NEXT:    vpsubsw (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_psubsw:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpsubsw %ymm1, %ymm0, %ymm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vpsubsw (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <16 x i16> @llvm.x86.avx2.psubs.w(<16 x i16> %a0, <16 x i16> %a1)
-  %2 = load <16 x i16>, <16 x i16> *%a2, align 32
-  %3 = call <16 x i16> @llvm.x86.avx2.psubs.w(<16 x i16> %1, <16 x i16> %2)
-  ret <16 x i16> %3
-}
-declare <16 x i16> @llvm.x86.avx2.psubs.w(<16 x i16>, <16 x i16>) nounwind readnone
-
-define <32 x i8> @test_psubusb(<32 x i8> %a0, <32 x i8> %a1, <32 x i8> *%a2) {
-; GENERIC-LABEL: test_psubusb:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpsubusb %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    vpsubusb (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_psubusb:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpsubusb %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; HASWELL-NEXT:    vpsubusb (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_psubusb:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpsubusb %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    vpsubusb (%rdi), %ymm0, %ymm0 # sched: [7:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_psubusb:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpsubusb %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; SKYLAKE-NEXT:    vpsubusb (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_psubusb:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpsubusb %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; SKX-NEXT:    vpsubusb (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_psubusb:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpsubusb %ymm1, %ymm0, %ymm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vpsubusb (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <32 x i8> @llvm.x86.avx2.psubus.b(<32 x i8> %a0, <32 x i8> %a1)
-  %2 = load <32 x i8>, <32 x i8> *%a2, align 32
-  %3 = call <32 x i8> @llvm.x86.avx2.psubus.b(<32 x i8> %1, <32 x i8> %2)
-  ret <32 x i8> %3
-}
-declare <32 x i8> @llvm.x86.avx2.psubus.b(<32 x i8>, <32 x i8>) nounwind readnone
-
-define <16 x i16> @test_psubusw(<16 x i16> %a0, <16 x i16> %a1, <16 x i16> *%a2) {
-; GENERIC-LABEL: test_psubusw:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpsubusw %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    vpsubusw (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_psubusw:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpsubusw %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; HASWELL-NEXT:    vpsubusw (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_psubusw:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpsubusw %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    vpsubusw (%rdi), %ymm0, %ymm0 # sched: [7:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_psubusw:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpsubusw %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; SKYLAKE-NEXT:    vpsubusw (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_psubusw:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpsubusw %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; SKX-NEXT:    vpsubusw (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_psubusw:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpsubusw %ymm1, %ymm0, %ymm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vpsubusw (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <16 x i16> @llvm.x86.avx2.psubus.w(<16 x i16> %a0, <16 x i16> %a1)
-  %2 = load <16 x i16>, <16 x i16> *%a2, align 32
-  %3 = call <16 x i16> @llvm.x86.avx2.psubus.w(<16 x i16> %1, <16 x i16> %2)
-  ret <16 x i16> %3
-}
-declare <16 x i16> @llvm.x86.avx2.psubus.w(<16 x i16>, <16 x i16>) nounwind readnone
-
-define <16 x i16> @test_psubw(<16 x i16> %a0, <16 x i16> %a1, <16 x i16> *%a2) {
-; GENERIC-LABEL: test_psubw:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpsubw %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    vpsubw (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_psubw:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpsubw %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; HASWELL-NEXT:    vpsubw (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_psubw:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpsubw %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    vpsubw (%rdi), %ymm0, %ymm0 # sched: [7:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_psubw:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpsubw %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    vpsubw (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_psubw:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpsubw %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    vpsubw (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_psubw:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpsubw %ymm1, %ymm0, %ymm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vpsubw (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = sub <16 x i16> %a0, %a1
-  %2 = load <16 x i16>, <16 x i16> *%a2, align 32
-  %3 = sub <16 x i16> %1, %2
-  ret <16 x i16> %3
-}
-
-define <32 x i8> @test_punpckhbw(<32 x i8> %a0, <32 x i8> %a1, <32 x i8> *%a2) {
-; GENERIC-LABEL: test_punpckhbw:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpunpckhbw {{.*#+}} ymm0 = ymm0[8],ymm1[8],ymm0[9],ymm1[9],ymm0[10],ymm1[10],ymm0[11],ymm1[11],ymm0[12],ymm1[12],ymm0[13],ymm1[13],ymm0[14],ymm1[14],ymm0[15],ymm1[15],ymm0[24],ymm1[24],ymm0[25],ymm1[25],ymm0[26],ymm1[26],ymm0[27],ymm1[27],ymm0[28],ymm1[28],ymm0[29],ymm1[29],ymm0[30],ymm1[30],ymm0[31],ymm1[31] sched: [1:1.00]
-; GENERIC-NEXT:    vpunpckhbw {{.*#+}} ymm0 = ymm0[8],mem[8],ymm0[9],mem[9],ymm0[10],mem[10],ymm0[11],mem[11],ymm0[12],mem[12],ymm0[13],mem[13],ymm0[14],mem[14],ymm0[15],mem[15],ymm0[24],mem[24],ymm0[25],mem[25],ymm0[26],mem[26],ymm0[27],mem[27],ymm0[28],mem[28],ymm0[29],mem[29],ymm0[30],mem[30],ymm0[31],mem[31] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_punpckhbw:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpunpckhbw {{.*#+}} ymm0 = ymm0[8],ymm1[8],ymm0[9],ymm1[9],ymm0[10],ymm1[10],ymm0[11],ymm1[11],ymm0[12],ymm1[12],ymm0[13],ymm1[13],ymm0[14],ymm1[14],ymm0[15],ymm1[15],ymm0[24],ymm1[24],ymm0[25],ymm1[25],ymm0[26],ymm1[26],ymm0[27],ymm1[27],ymm0[28],ymm1[28],ymm0[29],ymm1[29],ymm0[30],ymm1[30],ymm0[31],ymm1[31] sched: [1:1.00]
-; HASWELL-NEXT:    vpunpckhbw {{.*#+}} ymm0 = ymm0[8],mem[8],ymm0[9],mem[9],ymm0[10],mem[10],ymm0[11],mem[11],ymm0[12],mem[12],ymm0[13],mem[13],ymm0[14],mem[14],ymm0[15],mem[15],ymm0[24],mem[24],ymm0[25],mem[25],ymm0[26],mem[26],ymm0[27],mem[27],ymm0[28],mem[28],ymm0[29],mem[29],ymm0[30],mem[30],ymm0[31],mem[31] sched: [8:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_punpckhbw:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpunpckhbw {{.*#+}} ymm0 = ymm0[8],ymm1[8],ymm0[9],ymm1[9],ymm0[10],ymm1[10],ymm0[11],ymm1[11],ymm0[12],ymm1[12],ymm0[13],ymm1[13],ymm0[14],ymm1[14],ymm0[15],ymm1[15],ymm0[24],ymm1[24],ymm0[25],ymm1[25],ymm0[26],ymm1[26],ymm0[27],ymm1[27],ymm0[28],ymm1[28],ymm0[29],ymm1[29],ymm0[30],ymm1[30],ymm0[31],ymm1[31] sched: [1:1.00]
-; BROADWELL-NEXT:    vpunpckhbw {{.*#+}} ymm0 = ymm0[8],mem[8],ymm0[9],mem[9],ymm0[10],mem[10],ymm0[11],mem[11],ymm0[12],mem[12],ymm0[13],mem[13],ymm0[14],mem[14],ymm0[15],mem[15],ymm0[24],mem[24],ymm0[25],mem[25],ymm0[26],mem[26],ymm0[27],mem[27],ymm0[28],mem[28],ymm0[29],mem[29],ymm0[30],mem[30],ymm0[31],mem[31] sched: [7:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_punpckhbw:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpunpckhbw {{.*#+}} ymm0 = ymm0[8],ymm1[8],ymm0[9],ymm1[9],ymm0[10],ymm1[10],ymm0[11],ymm1[11],ymm0[12],ymm1[12],ymm0[13],ymm1[13],ymm0[14],ymm1[14],ymm0[15],ymm1[15],ymm0[24],ymm1[24],ymm0[25],ymm1[25],ymm0[26],ymm1[26],ymm0[27],ymm1[27],ymm0[28],ymm1[28],ymm0[29],ymm1[29],ymm0[30],ymm1[30],ymm0[31],ymm1[31] sched: [1:1.00]
-; SKYLAKE-NEXT:    vpunpckhbw {{.*#+}} ymm0 = ymm0[8],mem[8],ymm0[9],mem[9],ymm0[10],mem[10],ymm0[11],mem[11],ymm0[12],mem[12],ymm0[13],mem[13],ymm0[14],mem[14],ymm0[15],mem[15],ymm0[24],mem[24],ymm0[25],mem[25],ymm0[26],mem[26],ymm0[27],mem[27],ymm0[28],mem[28],ymm0[29],mem[29],ymm0[30],mem[30],ymm0[31],mem[31] sched: [8:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_punpckhbw:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpunpckhbw {{.*#+}} ymm0 = ymm0[8],ymm1[8],ymm0[9],ymm1[9],ymm0[10],ymm1[10],ymm0[11],ymm1[11],ymm0[12],ymm1[12],ymm0[13],ymm1[13],ymm0[14],ymm1[14],ymm0[15],ymm1[15],ymm0[24],ymm1[24],ymm0[25],ymm1[25],ymm0[26],ymm1[26],ymm0[27],ymm1[27],ymm0[28],ymm1[28],ymm0[29],ymm1[29],ymm0[30],ymm1[30],ymm0[31],ymm1[31] sched: [1:1.00]
-; SKX-NEXT:    vpunpckhbw {{.*#+}} ymm0 = ymm0[8],mem[8],ymm0[9],mem[9],ymm0[10],mem[10],ymm0[11],mem[11],ymm0[12],mem[12],ymm0[13],mem[13],ymm0[14],mem[14],ymm0[15],mem[15],ymm0[24],mem[24],ymm0[25],mem[25],ymm0[26],mem[26],ymm0[27],mem[27],ymm0[28],mem[28],ymm0[29],mem[29],ymm0[30],mem[30],ymm0[31],mem[31] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_punpckhbw:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpunpckhbw {{.*#+}} ymm0 = ymm0[8],ymm1[8],ymm0[9],ymm1[9],ymm0[10],ymm1[10],ymm0[11],ymm1[11],ymm0[12],ymm1[12],ymm0[13],ymm1[13],ymm0[14],ymm1[14],ymm0[15],ymm1[15],ymm0[24],ymm1[24],ymm0[25],ymm1[25],ymm0[26],ymm1[26],ymm0[27],ymm1[27],ymm0[28],ymm1[28],ymm0[29],ymm1[29],ymm0[30],ymm1[30],ymm0[31],ymm1[31] sched: [1:0.25]
-; ZNVER1-NEXT:    vpunpckhbw {{.*#+}} ymm0 = ymm0[8],mem[8],ymm0[9],mem[9],ymm0[10],mem[10],ymm0[11],mem[11],ymm0[12],mem[12],ymm0[13],mem[13],ymm0[14],mem[14],ymm0[15],mem[15],ymm0[24],mem[24],ymm0[25],mem[25],ymm0[26],mem[26],ymm0[27],mem[27],ymm0[28],mem[28],ymm0[29],mem[29],ymm0[30],mem[30],ymm0[31],mem[31] sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = shufflevector <32 x i8> %a0, <32 x i8> %a1, <32 x i32> <i32 8, i32 40, i32 9, i32 41, i32 10, i32 42, i32 11, i32 43, i32 12, i32 44, i32 13, i32 45, i32 14, i32 46, i32 15, i32 47, i32 24, i32 56, i32 25, i32 57, i32 26, i32 58, i32 27, i32 59, i32 28, i32 60, i32 29, i32 61, i32 30, i32 62, i32 31, i32 63>
-  %2 = load <32 x i8>, <32 x i8> *%a2, align 32
-  %3 = shufflevector <32 x i8> %1, <32 x i8> %2, <32 x i32> <i32 8, i32 40, i32 9, i32 41, i32 10, i32 42, i32 11, i32 43, i32 12, i32 44, i32 13, i32 45, i32 14, i32 46, i32 15, i32 47, i32 24, i32 56, i32 25, i32 57, i32 26, i32 58, i32 27, i32 59, i32 28, i32 60, i32 29, i32 61, i32 30, i32 62, i32 31, i32 63>
-  ret <32 x i8> %3
-}
-
-define <8 x i32> @test_punpckhdq(<8 x i32> %a0, <8 x i32> %a1, <8 x i32> *%a2) {
-; GENERIC-LABEL: test_punpckhdq:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpunpckhdq {{.*#+}} ymm0 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7] sched: [1:1.00]
-; GENERIC-NEXT:    vpunpckhdq {{.*#+}} ymm0 = ymm0[2],mem[2],ymm0[3],mem[3],ymm0[6],mem[6],ymm0[7],mem[7] sched: [8:1.00]
-; GENERIC-NEXT:    vpcmpeqd %ymm1, %ymm1, %ymm1 # sched: [1:0.50]
-; GENERIC-NEXT:    vpsubd %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_punpckhdq:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpunpckhdq {{.*#+}} ymm0 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7] sched: [1:1.00]
-; HASWELL-NEXT:    vpunpckhdq {{.*#+}} ymm0 = ymm0[2],mem[2],ymm0[3],mem[3],ymm0[6],mem[6],ymm0[7],mem[7] sched: [8:1.00]
-; HASWELL-NEXT:    vpcmpeqd %ymm1, %ymm1, %ymm1 # sched: [1:0.50]
-; HASWELL-NEXT:    vpsubd %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_punpckhdq:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpunpckhdq {{.*#+}} ymm0 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7] sched: [1:1.00]
-; BROADWELL-NEXT:    vpunpckhdq {{.*#+}} ymm0 = ymm0[2],mem[2],ymm0[3],mem[3],ymm0[6],mem[6],ymm0[7],mem[7] sched: [7:1.00]
-; BROADWELL-NEXT:    vpcmpeqd %ymm1, %ymm1, %ymm1 # sched: [1:0.50]
-; BROADWELL-NEXT:    vpsubd %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_punpckhdq:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpunpckhdq {{.*#+}} ymm0 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7] sched: [1:1.00]
-; SKYLAKE-NEXT:    vpunpckhdq {{.*#+}} ymm0 = ymm0[2],mem[2],ymm0[3],mem[3],ymm0[6],mem[6],ymm0[7],mem[7] sched: [8:1.00]
-; SKYLAKE-NEXT:    vpcmpeqd %ymm1, %ymm1, %ymm1 # sched: [1:0.50]
-; SKYLAKE-NEXT:    vpsubd %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_punpckhdq:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpunpckhdq {{.*#+}} ymm0 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7] sched: [1:1.00]
-; SKX-NEXT:    vpunpckhdq {{.*#+}} ymm0 = ymm0[2],mem[2],ymm0[3],mem[3],ymm0[6],mem[6],ymm0[7],mem[7] sched: [8:1.00]
-; SKX-NEXT:    vpcmpeqd %ymm1, %ymm1, %ymm1 # sched: [1:0.50]
-; SKX-NEXT:    vpsubd %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_punpckhdq:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpunpckhdq {{.*#+}} ymm0 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7] sched: [1:0.25]
-; ZNVER1-NEXT:    vpunpckhdq {{.*#+}} ymm0 = ymm0[2],mem[2],ymm0[3],mem[3],ymm0[6],mem[6],ymm0[7],mem[7] sched: [8:0.50]
-; ZNVER1-NEXT:    vpcmpeqd %ymm1, %ymm1, %ymm1 # sched: [1:0.25]
-; ZNVER1-NEXT:    vpsubd %ymm1, %ymm0, %ymm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = shufflevector <8 x i32> %a0, <8 x i32> %a1, <8 x i32> <i32 2, i32 10, i32 3, i32 11, i32 6, i32 14, i32 7, i32 15>
-  %2 = load <8 x i32>, <8 x i32> *%a2, align 32
-  %3 = shufflevector <8 x i32> %1, <8 x i32> %2, <8 x i32> <i32 2, i32 10, i32 3, i32 11, i32 6, i32 14, i32 7, i32 15>
-  %4 = add <8 x i32> %3, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
-  ret <8 x i32> %4
-}
-
-define <4 x i64> @test_punpckhqdq(<4 x i64> %a0, <4 x i64> %a1, <4 x i64> *%a2) {
-; GENERIC-LABEL: test_punpckhqdq:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpunpckhqdq {{.*#+}} ymm1 = ymm0[1],ymm1[1],ymm0[3],ymm1[3] sched: [1:1.00]
-; GENERIC-NEXT:    vpunpckhqdq {{.*#+}} ymm0 = ymm0[1],mem[1],ymm0[3],mem[3] sched: [8:1.00]
-; GENERIC-NEXT:    vpaddq %ymm0, %ymm1, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_punpckhqdq:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpunpckhqdq {{.*#+}} ymm1 = ymm0[1],ymm1[1],ymm0[3],ymm1[3] sched: [1:1.00]
-; HASWELL-NEXT:    vpunpckhqdq {{.*#+}} ymm0 = ymm0[1],mem[1],ymm0[3],mem[3] sched: [8:1.00]
-; HASWELL-NEXT:    vpaddq %ymm0, %ymm1, %ymm0 # sched: [1:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_punpckhqdq:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpunpckhqdq {{.*#+}} ymm1 = ymm0[1],ymm1[1],ymm0[3],ymm1[3] sched: [1:1.00]
-; BROADWELL-NEXT:    vpunpckhqdq {{.*#+}} ymm0 = ymm0[1],mem[1],ymm0[3],mem[3] sched: [7:1.00]
-; BROADWELL-NEXT:    vpaddq %ymm0, %ymm1, %ymm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_punpckhqdq:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpunpckhqdq {{.*#+}} ymm1 = ymm0[1],ymm1[1],ymm0[3],ymm1[3] sched: [1:1.00]
-; SKYLAKE-NEXT:    vpunpckhqdq {{.*#+}} ymm0 = ymm0[1],mem[1],ymm0[3],mem[3] sched: [8:1.00]
-; SKYLAKE-NEXT:    vpaddq %ymm0, %ymm1, %ymm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_punpckhqdq:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpunpckhqdq {{.*#+}} ymm1 = ymm0[1],ymm1[1],ymm0[3],ymm1[3] sched: [1:1.00]
-; SKX-NEXT:    vpunpckhqdq {{.*#+}} ymm0 = ymm0[1],mem[1],ymm0[3],mem[3] sched: [8:1.00]
-; SKX-NEXT:    vpaddq %ymm0, %ymm1, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_punpckhqdq:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpunpckhqdq {{.*#+}} ymm1 = ymm0[1],ymm1[1],ymm0[3],ymm1[3] sched: [1:0.25]
-; ZNVER1-NEXT:    vpunpckhqdq {{.*#+}} ymm0 = ymm0[1],mem[1],ymm0[3],mem[3] sched: [8:0.50]
-; ZNVER1-NEXT:    vpaddq %ymm0, %ymm1, %ymm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = shufflevector <4 x i64> %a0, <4 x i64> %a1, <4 x i32> <i32 1, i32 5, i32 3, i32 7>
-  %2 = load <4 x i64>, <4 x i64> *%a2, align 32
-  %3 = shufflevector <4 x i64> %a0, <4 x i64> %2, <4 x i32> <i32 1, i32 5, i32 3, i32 7>
-  %4 = add <4 x i64> %1, %3
-  ret <4 x i64> %4
-}
-
-define <16 x i16> @test_punpckhwd(<16 x i16> %a0, <16 x i16> %a1, <16 x i16> *%a2) {
-; GENERIC-LABEL: test_punpckhwd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpunpckhwd {{.*#+}} ymm0 = ymm0[4],ymm1[4],ymm0[5],ymm1[5],ymm0[6],ymm1[6],ymm0[7],ymm1[7],ymm0[12],ymm1[12],ymm0[13],ymm1[13],ymm0[14],ymm1[14],ymm0[15],ymm1[15] sched: [1:1.00]
-; GENERIC-NEXT:    vpunpckhwd {{.*#+}} ymm0 = ymm0[4],mem[4],ymm0[5],mem[5],ymm0[6],mem[6],ymm0[7],mem[7],ymm0[12],mem[12],ymm0[13],mem[13],ymm0[14],mem[14],ymm0[15],mem[15] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_punpckhwd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpunpckhwd {{.*#+}} ymm0 = ymm0[4],ymm1[4],ymm0[5],ymm1[5],ymm0[6],ymm1[6],ymm0[7],ymm1[7],ymm0[12],ymm1[12],ymm0[13],ymm1[13],ymm0[14],ymm1[14],ymm0[15],ymm1[15] sched: [1:1.00]
-; HASWELL-NEXT:    vpunpckhwd {{.*#+}} ymm0 = ymm0[4],mem[4],ymm0[5],mem[5],ymm0[6],mem[6],ymm0[7],mem[7],ymm0[12],mem[12],ymm0[13],mem[13],ymm0[14],mem[14],ymm0[15],mem[15] sched: [8:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_punpckhwd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpunpckhwd {{.*#+}} ymm0 = ymm0[4],ymm1[4],ymm0[5],ymm1[5],ymm0[6],ymm1[6],ymm0[7],ymm1[7],ymm0[12],ymm1[12],ymm0[13],ymm1[13],ymm0[14],ymm1[14],ymm0[15],ymm1[15] sched: [1:1.00]
-; BROADWELL-NEXT:    vpunpckhwd {{.*#+}} ymm0 = ymm0[4],mem[4],ymm0[5],mem[5],ymm0[6],mem[6],ymm0[7],mem[7],ymm0[12],mem[12],ymm0[13],mem[13],ymm0[14],mem[14],ymm0[15],mem[15] sched: [7:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_punpckhwd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpunpckhwd {{.*#+}} ymm0 = ymm0[4],ymm1[4],ymm0[5],ymm1[5],ymm0[6],ymm1[6],ymm0[7],ymm1[7],ymm0[12],ymm1[12],ymm0[13],ymm1[13],ymm0[14],ymm1[14],ymm0[15],ymm1[15] sched: [1:1.00]
-; SKYLAKE-NEXT:    vpunpckhwd {{.*#+}} ymm0 = ymm0[4],mem[4],ymm0[5],mem[5],ymm0[6],mem[6],ymm0[7],mem[7],ymm0[12],mem[12],ymm0[13],mem[13],ymm0[14],mem[14],ymm0[15],mem[15] sched: [8:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_punpckhwd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpunpckhwd {{.*#+}} ymm0 = ymm0[4],ymm1[4],ymm0[5],ymm1[5],ymm0[6],ymm1[6],ymm0[7],ymm1[7],ymm0[12],ymm1[12],ymm0[13],ymm1[13],ymm0[14],ymm1[14],ymm0[15],ymm1[15] sched: [1:1.00]
-; SKX-NEXT:    vpunpckhwd {{.*#+}} ymm0 = ymm0[4],mem[4],ymm0[5],mem[5],ymm0[6],mem[6],ymm0[7],mem[7],ymm0[12],mem[12],ymm0[13],mem[13],ymm0[14],mem[14],ymm0[15],mem[15] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_punpckhwd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpunpckhwd {{.*#+}} ymm0 = ymm0[4],ymm1[4],ymm0[5],ymm1[5],ymm0[6],ymm1[6],ymm0[7],ymm1[7],ymm0[12],ymm1[12],ymm0[13],ymm1[13],ymm0[14],ymm1[14],ymm0[15],ymm1[15] sched: [1:0.25]
-; ZNVER1-NEXT:    vpunpckhwd {{.*#+}} ymm0 = ymm0[4],mem[4],ymm0[5],mem[5],ymm0[6],mem[6],ymm0[7],mem[7],ymm0[12],mem[12],ymm0[13],mem[13],ymm0[14],mem[14],ymm0[15],mem[15] sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = shufflevector <16 x i16> %a0, <16 x i16> %a1, <16 x i32> <i32 4, i32 20, i32 5, i32 21, i32 6, i32 22, i32 7, i32 23, i32 12, i32 28, i32 13, i32 29, i32 14, i32 30, i32 15, i32 31>
-  %2 = load <16 x i16>, <16 x i16> *%a2, align 32
-  %3 = shufflevector <16 x i16> %1, <16 x i16> %2, <16 x i32> <i32 4, i32 20, i32 5, i32 21, i32 6, i32 22, i32 7, i32 23, i32 12, i32 28, i32 13, i32 29, i32 14, i32 30, i32 15, i32 31>
-  ret <16 x i16> %3
-}
-
-define <32 x i8> @test_punpcklbw(<32 x i8> %a0, <32 x i8> %a1, <32 x i8> *%a2) {
-; GENERIC-LABEL: test_punpcklbw:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[4],ymm1[4],ymm0[5],ymm1[5],ymm0[6],ymm1[6],ymm0[7],ymm1[7],ymm0[16],ymm1[16],ymm0[17],ymm1[17],ymm0[18],ymm1[18],ymm0[19],ymm1[19],ymm0[20],ymm1[20],ymm0[21],ymm1[21],ymm0[22],ymm1[22],ymm0[23],ymm1[23] sched: [1:1.00]
-; GENERIC-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0],mem[0],ymm0[1],mem[1],ymm0[2],mem[2],ymm0[3],mem[3],ymm0[4],mem[4],ymm0[5],mem[5],ymm0[6],mem[6],ymm0[7],mem[7],ymm0[16],mem[16],ymm0[17],mem[17],ymm0[18],mem[18],ymm0[19],mem[19],ymm0[20],mem[20],ymm0[21],mem[21],ymm0[22],mem[22],ymm0[23],mem[23] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_punpcklbw:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[4],ymm1[4],ymm0[5],ymm1[5],ymm0[6],ymm1[6],ymm0[7],ymm1[7],ymm0[16],ymm1[16],ymm0[17],ymm1[17],ymm0[18],ymm1[18],ymm0[19],ymm1[19],ymm0[20],ymm1[20],ymm0[21],ymm1[21],ymm0[22],ymm1[22],ymm0[23],ymm1[23] sched: [1:1.00]
-; HASWELL-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0],mem[0],ymm0[1],mem[1],ymm0[2],mem[2],ymm0[3],mem[3],ymm0[4],mem[4],ymm0[5],mem[5],ymm0[6],mem[6],ymm0[7],mem[7],ymm0[16],mem[16],ymm0[17],mem[17],ymm0[18],mem[18],ymm0[19],mem[19],ymm0[20],mem[20],ymm0[21],mem[21],ymm0[22],mem[22],ymm0[23],mem[23] sched: [8:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_punpcklbw:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[4],ymm1[4],ymm0[5],ymm1[5],ymm0[6],ymm1[6],ymm0[7],ymm1[7],ymm0[16],ymm1[16],ymm0[17],ymm1[17],ymm0[18],ymm1[18],ymm0[19],ymm1[19],ymm0[20],ymm1[20],ymm0[21],ymm1[21],ymm0[22],ymm1[22],ymm0[23],ymm1[23] sched: [1:1.00]
-; BROADWELL-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0],mem[0],ymm0[1],mem[1],ymm0[2],mem[2],ymm0[3],mem[3],ymm0[4],mem[4],ymm0[5],mem[5],ymm0[6],mem[6],ymm0[7],mem[7],ymm0[16],mem[16],ymm0[17],mem[17],ymm0[18],mem[18],ymm0[19],mem[19],ymm0[20],mem[20],ymm0[21],mem[21],ymm0[22],mem[22],ymm0[23],mem[23] sched: [7:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_punpcklbw:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[4],ymm1[4],ymm0[5],ymm1[5],ymm0[6],ymm1[6],ymm0[7],ymm1[7],ymm0[16],ymm1[16],ymm0[17],ymm1[17],ymm0[18],ymm1[18],ymm0[19],ymm1[19],ymm0[20],ymm1[20],ymm0[21],ymm1[21],ymm0[22],ymm1[22],ymm0[23],ymm1[23] sched: [1:1.00]
-; SKYLAKE-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0],mem[0],ymm0[1],mem[1],ymm0[2],mem[2],ymm0[3],mem[3],ymm0[4],mem[4],ymm0[5],mem[5],ymm0[6],mem[6],ymm0[7],mem[7],ymm0[16],mem[16],ymm0[17],mem[17],ymm0[18],mem[18],ymm0[19],mem[19],ymm0[20],mem[20],ymm0[21],mem[21],ymm0[22],mem[22],ymm0[23],mem[23] sched: [8:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_punpcklbw:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[4],ymm1[4],ymm0[5],ymm1[5],ymm0[6],ymm1[6],ymm0[7],ymm1[7],ymm0[16],ymm1[16],ymm0[17],ymm1[17],ymm0[18],ymm1[18],ymm0[19],ymm1[19],ymm0[20],ymm1[20],ymm0[21],ymm1[21],ymm0[22],ymm1[22],ymm0[23],ymm1[23] sched: [1:1.00]
-; SKX-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0],mem[0],ymm0[1],mem[1],ymm0[2],mem[2],ymm0[3],mem[3],ymm0[4],mem[4],ymm0[5],mem[5],ymm0[6],mem[6],ymm0[7],mem[7],ymm0[16],mem[16],ymm0[17],mem[17],ymm0[18],mem[18],ymm0[19],mem[19],ymm0[20],mem[20],ymm0[21],mem[21],ymm0[22],mem[22],ymm0[23],mem[23] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_punpcklbw:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[4],ymm1[4],ymm0[5],ymm1[5],ymm0[6],ymm1[6],ymm0[7],ymm1[7],ymm0[16],ymm1[16],ymm0[17],ymm1[17],ymm0[18],ymm1[18],ymm0[19],ymm1[19],ymm0[20],ymm1[20],ymm0[21],ymm1[21],ymm0[22],ymm1[22],ymm0[23],ymm1[23] sched: [1:0.25]
-; ZNVER1-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0],mem[0],ymm0[1],mem[1],ymm0[2],mem[2],ymm0[3],mem[3],ymm0[4],mem[4],ymm0[5],mem[5],ymm0[6],mem[6],ymm0[7],mem[7],ymm0[16],mem[16],ymm0[17],mem[17],ymm0[18],mem[18],ymm0[19],mem[19],ymm0[20],mem[20],ymm0[21],mem[21],ymm0[22],mem[22],ymm0[23],mem[23] sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = shufflevector <32 x i8> %a0, <32 x i8> %a1, <32 x i32> <i32 0, i32 32, i32 1, i32 33, i32 2, i32 34, i32 3, i32 35, i32 4, i32 36, i32 5, i32 37, i32 6, i32 38, i32 7, i32 39, i32 16, i32 48, i32 17, i32 49, i32 18, i32 50, i32 19, i32 51, i32 20, i32 52, i32 21, i32 53, i32 22, i32 54, i32 23, i32 55>
-  %2 = load <32 x i8>, <32 x i8> *%a2, align 32
-  %3 = shufflevector <32 x i8> %1, <32 x i8> %2, <32 x i32> <i32 0, i32 32, i32 1, i32 33, i32 2, i32 34, i32 3, i32 35, i32 4, i32 36, i32 5, i32 37, i32 6, i32 38, i32 7, i32 39, i32 16, i32 48, i32 17, i32 49, i32 18, i32 50, i32 19, i32 51, i32 20, i32 52, i32 21, i32 53, i32 22, i32 54, i32 23, i32 55>
-  ret <32 x i8> %3
-}
-
-define <8 x i32> @test_punpckldq(<8 x i32> %a0, <8 x i32> %a1, <8 x i32> *%a2) {
-; GENERIC-LABEL: test_punpckldq:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5] sched: [1:1.00]
-; GENERIC-NEXT:    vpunpckldq {{.*#+}} ymm0 = ymm0[0],mem[0],ymm0[1],mem[1],ymm0[4],mem[4],ymm0[5],mem[5] sched: [8:1.00]
-; GENERIC-NEXT:    vpcmpeqd %ymm1, %ymm1, %ymm1 # sched: [1:0.50]
-; GENERIC-NEXT:    vpsubd %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_punpckldq:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5] sched: [1:1.00]
-; HASWELL-NEXT:    vpunpckldq {{.*#+}} ymm0 = ymm0[0],mem[0],ymm0[1],mem[1],ymm0[4],mem[4],ymm0[5],mem[5] sched: [8:1.00]
-; HASWELL-NEXT:    vpcmpeqd %ymm1, %ymm1, %ymm1 # sched: [1:0.50]
-; HASWELL-NEXT:    vpsubd %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_punpckldq:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5] sched: [1:1.00]
-; BROADWELL-NEXT:    vpunpckldq {{.*#+}} ymm0 = ymm0[0],mem[0],ymm0[1],mem[1],ymm0[4],mem[4],ymm0[5],mem[5] sched: [7:1.00]
-; BROADWELL-NEXT:    vpcmpeqd %ymm1, %ymm1, %ymm1 # sched: [1:0.50]
-; BROADWELL-NEXT:    vpsubd %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_punpckldq:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5] sched: [1:1.00]
-; SKYLAKE-NEXT:    vpunpckldq {{.*#+}} ymm0 = ymm0[0],mem[0],ymm0[1],mem[1],ymm0[4],mem[4],ymm0[5],mem[5] sched: [8:1.00]
-; SKYLAKE-NEXT:    vpcmpeqd %ymm1, %ymm1, %ymm1 # sched: [1:0.50]
-; SKYLAKE-NEXT:    vpsubd %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_punpckldq:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5] sched: [1:1.00]
-; SKX-NEXT:    vpunpckldq {{.*#+}} ymm0 = ymm0[0],mem[0],ymm0[1],mem[1],ymm0[4],mem[4],ymm0[5],mem[5] sched: [8:1.00]
-; SKX-NEXT:    vpcmpeqd %ymm1, %ymm1, %ymm1 # sched: [1:0.50]
-; SKX-NEXT:    vpsubd %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_punpckldq:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5] sched: [1:0.25]
-; ZNVER1-NEXT:    vpunpckldq {{.*#+}} ymm0 = ymm0[0],mem[0],ymm0[1],mem[1],ymm0[4],mem[4],ymm0[5],mem[5] sched: [8:0.50]
-; ZNVER1-NEXT:    vpcmpeqd %ymm1, %ymm1, %ymm1 # sched: [1:0.25]
-; ZNVER1-NEXT:    vpsubd %ymm1, %ymm0, %ymm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = shufflevector <8 x i32> %a0, <8 x i32> %a1, <8 x i32> <i32 0, i32 8, i32 1, i32 9, i32 4, i32 12, i32 5, i32 13>
-  %2 = load <8 x i32>, <8 x i32> *%a2, align 32
-  %3 = shufflevector <8 x i32> %1, <8 x i32> %2, <8 x i32> <i32 0, i32 8, i32 1, i32 9, i32 4, i32 12, i32 5, i32 13>
-  %4 = add <8 x i32> %3, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
-  ret <8 x i32> %4
-}
-
-define <4 x i64> @test_punpcklqdq(<4 x i64> %a0, <4 x i64> %a1, <4 x i64> *%a2) {
-; GENERIC-LABEL: test_punpcklqdq:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpunpcklqdq {{.*#+}} ymm1 = ymm0[0],ymm1[0],ymm0[2],ymm1[2] sched: [1:1.00]
-; GENERIC-NEXT:    vpunpcklqdq {{.*#+}} ymm0 = ymm0[0],mem[0],ymm0[2],mem[2] sched: [8:1.00]
-; GENERIC-NEXT:    vpaddq %ymm0, %ymm1, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_punpcklqdq:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpunpcklqdq {{.*#+}} ymm1 = ymm0[0],ymm1[0],ymm0[2],ymm1[2] sched: [1:1.00]
-; HASWELL-NEXT:    vpunpcklqdq {{.*#+}} ymm0 = ymm0[0],mem[0],ymm0[2],mem[2] sched: [8:1.00]
-; HASWELL-NEXT:    vpaddq %ymm0, %ymm1, %ymm0 # sched: [1:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_punpcklqdq:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpunpcklqdq {{.*#+}} ymm1 = ymm0[0],ymm1[0],ymm0[2],ymm1[2] sched: [1:1.00]
-; BROADWELL-NEXT:    vpunpcklqdq {{.*#+}} ymm0 = ymm0[0],mem[0],ymm0[2],mem[2] sched: [7:1.00]
-; BROADWELL-NEXT:    vpaddq %ymm0, %ymm1, %ymm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_punpcklqdq:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpunpcklqdq {{.*#+}} ymm1 = ymm0[0],ymm1[0],ymm0[2],ymm1[2] sched: [1:1.00]
-; SKYLAKE-NEXT:    vpunpcklqdq {{.*#+}} ymm0 = ymm0[0],mem[0],ymm0[2],mem[2] sched: [8:1.00]
-; SKYLAKE-NEXT:    vpaddq %ymm0, %ymm1, %ymm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_punpcklqdq:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpunpcklqdq {{.*#+}} ymm1 = ymm0[0],ymm1[0],ymm0[2],ymm1[2] sched: [1:1.00]
-; SKX-NEXT:    vpunpcklqdq {{.*#+}} ymm0 = ymm0[0],mem[0],ymm0[2],mem[2] sched: [8:1.00]
-; SKX-NEXT:    vpaddq %ymm0, %ymm1, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_punpcklqdq:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpunpcklqdq {{.*#+}} ymm1 = ymm0[0],ymm1[0],ymm0[2],ymm1[2] sched: [1:0.25]
-; ZNVER1-NEXT:    vpunpcklqdq {{.*#+}} ymm0 = ymm0[0],mem[0],ymm0[2],mem[2] sched: [8:0.50]
-; ZNVER1-NEXT:    vpaddq %ymm0, %ymm1, %ymm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = shufflevector <4 x i64> %a0, <4 x i64> %a1, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
-  %2 = load <4 x i64>, <4 x i64> *%a2, align 32
-  %3 = shufflevector <4 x i64> %a0, <4 x i64> %2, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
-  %4 = add <4 x i64> %1, %3
-  ret <4 x i64> %4
-}
-
-define <16 x i16> @test_punpcklwd(<16 x i16> %a0, <16 x i16> %a1, <16 x i16> *%a2) {
-; GENERIC-LABEL: test_punpcklwd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[8],ymm1[8],ymm0[9],ymm1[9],ymm0[10],ymm1[10],ymm0[11],ymm1[11] sched: [1:1.00]
-; GENERIC-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm0[0],mem[0],ymm0[1],mem[1],ymm0[2],mem[2],ymm0[3],mem[3],ymm0[8],mem[8],ymm0[9],mem[9],ymm0[10],mem[10],ymm0[11],mem[11] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_punpcklwd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[8],ymm1[8],ymm0[9],ymm1[9],ymm0[10],ymm1[10],ymm0[11],ymm1[11] sched: [1:1.00]
-; HASWELL-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm0[0],mem[0],ymm0[1],mem[1],ymm0[2],mem[2],ymm0[3],mem[3],ymm0[8],mem[8],ymm0[9],mem[9],ymm0[10],mem[10],ymm0[11],mem[11] sched: [8:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_punpcklwd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[8],ymm1[8],ymm0[9],ymm1[9],ymm0[10],ymm1[10],ymm0[11],ymm1[11] sched: [1:1.00]
-; BROADWELL-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm0[0],mem[0],ymm0[1],mem[1],ymm0[2],mem[2],ymm0[3],mem[3],ymm0[8],mem[8],ymm0[9],mem[9],ymm0[10],mem[10],ymm0[11],mem[11] sched: [7:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_punpcklwd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[8],ymm1[8],ymm0[9],ymm1[9],ymm0[10],ymm1[10],ymm0[11],ymm1[11] sched: [1:1.00]
-; SKYLAKE-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm0[0],mem[0],ymm0[1],mem[1],ymm0[2],mem[2],ymm0[3],mem[3],ymm0[8],mem[8],ymm0[9],mem[9],ymm0[10],mem[10],ymm0[11],mem[11] sched: [8:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_punpcklwd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[8],ymm1[8],ymm0[9],ymm1[9],ymm0[10],ymm1[10],ymm0[11],ymm1[11] sched: [1:1.00]
-; SKX-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm0[0],mem[0],ymm0[1],mem[1],ymm0[2],mem[2],ymm0[3],mem[3],ymm0[8],mem[8],ymm0[9],mem[9],ymm0[10],mem[10],ymm0[11],mem[11] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_punpcklwd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[8],ymm1[8],ymm0[9],ymm1[9],ymm0[10],ymm1[10],ymm0[11],ymm1[11] sched: [1:0.25]
-; ZNVER1-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm0[0],mem[0],ymm0[1],mem[1],ymm0[2],mem[2],ymm0[3],mem[3],ymm0[8],mem[8],ymm0[9],mem[9],ymm0[10],mem[10],ymm0[11],mem[11] sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = shufflevector <16 x i16> %a0, <16 x i16> %a1, <16 x i32> <i32 0, i32 16, i32 1, i32 17, i32 2, i32 18, i32 3, i32 19, i32 8, i32 24, i32 9, i32 25, i32 10, i32 26, i32 11, i32 27>
-  %2 = load <16 x i16>, <16 x i16> *%a2, align 32
-  %3 = shufflevector <16 x i16> %1, <16 x i16> %2, <16 x i32> <i32 0, i32 16, i32 1, i32 17, i32 2, i32 18, i32 3, i32 19, i32 8, i32 24, i32 9, i32 25, i32 10, i32 26, i32 11, i32 27>
-  ret <16 x i16> %3
-}
-
-define <4 x i64> @test_pxor(<4 x i64> %a0, <4 x i64> %a1, <4 x i64> *%a2) {
-; GENERIC-LABEL: test_pxor:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpxor %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; GENERIC-NEXT:    vpxor (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; GENERIC-NEXT:    vpaddq %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_pxor:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpxor %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; HASWELL-NEXT:    vpxor (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; HASWELL-NEXT:    vpaddq %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pxor:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpxor %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; BROADWELL-NEXT:    vpxor (%rdi), %ymm0, %ymm0 # sched: [7:0.50]
-; BROADWELL-NEXT:    vpaddq %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pxor:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpxor %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    vpxor (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKYLAKE-NEXT:    vpaddq %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pxor:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpxor %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    vpxor (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; SKX-NEXT:    vpaddq %ymm1, %ymm0, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_pxor:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpxor %ymm1, %ymm0, %ymm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vpxor (%rdi), %ymm0, %ymm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    vpaddq %ymm1, %ymm0, %ymm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = xor <4 x i64> %a0, %a1
-  %2 = load <4 x i64>, <4 x i64> *%a2, align 32
-  %3 = xor <4 x i64> %1, %2
-  %4 = add <4 x i64> %3, %a1
-  ret <4 x i64> %4
-}
-
-!0 = !{i32 1}
diff --git a/test/CodeGen/X86/avx2-vbroadcast.ll b/test/CodeGen/X86/avx2-vbroadcast.ll
index b333e9109bdcdbec8a84f3337a8c38efbbf9a683..165f9071ffa0e2cab43288ce2a3782d333de7b8f 100644
--- a/test/CodeGen/X86/avx2-vbroadcast.ll
+++ b/test/CodeGen/X86/avx2-vbroadcast.ll
@@ -189,12 +189,12 @@ define <2 x i64> @Q64(i64* %ptr) nounwind uwtable readnone ssp {
 ; X32-LABEL: Q64:
 ; X32:       ## %bb.0: ## %entry
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    vpbroadcastq (%eax), %xmm0
+; X32-NEXT:    vmovddup {{.*#+}} xmm0 = mem[0,0]
 ; X32-NEXT:    retl
 ;
 ; X64-LABEL: Q64:
 ; X64:       ## %bb.0: ## %entry
-; X64-NEXT:    vpbroadcastq (%rdi), %xmm0
+; X64-NEXT:    vmovddup {{.*#+}} xmm0 = mem[0,0]
 ; X64-NEXT:    retq
 entry:
   %q = load i64, i64* %ptr, align 4
@@ -233,7 +233,7 @@ define <8 x i16> @broadcast_mem_v4i16_v8i16(<4 x i16>* %ptr) {
 ;
 ; X64-LABEL: broadcast_mem_v4i16_v8i16:
 ; X64:       ## %bb.0:
-; X64-NEXT:    vpbroadcastq (%rdi), %xmm0
+; X64-NEXT:    vmovddup {{.*#+}} xmm0 = mem[0,0]
 ; X64-NEXT:    retq
   %load = load <4 x i16>, <4 x i16>* %ptr
   %shuf = shufflevector <4 x i16> %load, <4 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
@@ -244,8 +244,7 @@ define <16 x i16> @broadcast_mem_v4i16_v16i16(<4 x i16>* %ptr) {
 ; X32-LABEL: broadcast_mem_v4i16_v16i16:
 ; X32:       ## %bb.0:
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
-; X32-NEXT:    vbroadcastsd %xmm0, %ymm0
+; X32-NEXT:    vbroadcastsd (%eax), %ymm0
 ; X32-NEXT:    retl
 ;
 ; X64-LABEL: broadcast_mem_v4i16_v16i16:
@@ -472,7 +471,7 @@ define <2 x i64> @load_splat_2i64_2i64_1111(<2 x i64>* %ptr) nounwind uwtable re
 ;
 ; X64-LABEL: load_splat_2i64_2i64_1111:
 ; X64:       ## %bb.0: ## %entry
-; X64-NEXT:    vpbroadcastq 8(%rdi), %xmm0
+; X64-NEXT:    vmovddup {{.*#+}} xmm0 = mem[0,0]
 ; X64-NEXT:    retq
 entry:
   %ld = load <2 x i64>, <2 x i64>* %ptr
@@ -866,12 +865,12 @@ define   <4 x i64> @_inreg4xi64(<4 x i64> %a) {
 define   <2 x i64> @_inreg2xi64(<2 x i64> %a) {
 ; X32-LABEL: _inreg2xi64:
 ; X32:       ## %bb.0:
-; X32-NEXT:    vpbroadcastq %xmm0, %xmm0
+; X32-NEXT:    vmovddup {{.*#+}} xmm0 = xmm0[0,0]
 ; X32-NEXT:    retl
 ;
 ; X64-LABEL: _inreg2xi64:
 ; X64:       ## %bb.0:
-; X64-NEXT:    vpbroadcastq %xmm0, %xmm0
+; X64-NEXT:    vmovddup {{.*#+}} xmm0 = xmm0[0,0]
 ; X64-NEXT:    retq
   %b = shufflevector <2 x i64> %a, <2 x i64> undef, <2 x i32> zeroinitializer
   ret <2 x i64> %b
@@ -1328,9 +1327,9 @@ define void @isel_crash_2q(i64* %cV_R.addr) {
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X32-NEXT:    vxorps %xmm0, %xmm0, %xmm0
 ; X32-NEXT:    vmovaps %xmm0, (%esp)
-; X32-NEXT:    vpbroadcastq (%eax), %xmm1
+; X32-NEXT:    vmovddup {{.*#+}} xmm1 = mem[0,0]
 ; X32-NEXT:    vmovaps %xmm0, {{[0-9]+}}(%esp)
-; X32-NEXT:    vmovdqa %xmm1, {{[0-9]+}}(%esp)
+; X32-NEXT:    vmovaps %xmm1, {{[0-9]+}}(%esp)
 ; X32-NEXT:    addl $60, %esp
 ; X32-NEXT:    retl
 ;
@@ -1338,9 +1337,9 @@ define void @isel_crash_2q(i64* %cV_R.addr) {
 ; X64:       ## %bb.0: ## %entry
 ; X64-NEXT:    vxorps %xmm0, %xmm0, %xmm0
 ; X64-NEXT:    vmovaps %xmm0, -{{[0-9]+}}(%rsp)
-; X64-NEXT:    vpbroadcastq (%rdi), %xmm1
+; X64-NEXT:    vmovddup {{.*#+}} xmm1 = mem[0,0]
 ; X64-NEXT:    vmovaps %xmm0, -{{[0-9]+}}(%rsp)
-; X64-NEXT:    vmovdqa %xmm1, -{{[0-9]+}}(%rsp)
+; X64-NEXT:    vmovaps %xmm1, -{{[0-9]+}}(%rsp)
 ; X64-NEXT:    retq
 entry:
   %__a.addr.i = alloca <2 x i64>, align 16
diff --git a/test/CodeGen/X86/avx512-cvt-widen.ll b/test/CodeGen/X86/avx512-cvt-widen.ll
index 849a814e640dac1e014a959b49ce6eec56fc2ec1..01eba7912f940ba5322dfc5cd972edbb9ae49677 100644
--- a/test/CodeGen/X86/avx512-cvt-widen.ll
+++ b/test/CodeGen/X86/avx512-cvt-widen.ll
@@ -1909,8 +1909,6 @@ define <8 x float> @ubto8f32(<8 x i32> %a) {
 define <8 x double> @ubto8f64(<8 x i32> %a) {
 ; ALL-LABEL: ubto8f64:
 ; ALL:       # %bb.0:
-; ALL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; ALL-NEXT:    vpcmpgtd %ymm0, %ymm1, %ymm0
 ; ALL-NEXT:    vpsrld $31, %ymm0, %ymm0
 ; ALL-NEXT:    vcvtdq2pd %ymm0, %zmm0
 ; ALL-NEXT:    retq
@@ -1942,8 +1940,6 @@ define <4 x float> @ubto4f32(<4 x i32> %a) {
 define <4 x double> @ubto4f64(<4 x i32> %a) {
 ; ALL-LABEL: ubto4f64:
 ; ALL:       # %bb.0:
-; ALL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; ALL-NEXT:    vpcmpgtd %xmm0, %xmm1, %xmm0
 ; ALL-NEXT:    vpsrld $31, %xmm0, %xmm0
 ; ALL-NEXT:    vcvtdq2pd %xmm0, %ymm0
 ; ALL-NEXT:    retq
diff --git a/test/CodeGen/X86/avx512-cvt.ll b/test/CodeGen/X86/avx512-cvt.ll
index 264b1ea841fd3105f2b94ed84b306e5edbff2c52..170e5a30798c063bc574643ea7c6d8ff664ab4ab 100644
--- a/test/CodeGen/X86/avx512-cvt.ll
+++ b/test/CodeGen/X86/avx512-cvt.ll
@@ -1897,8 +1897,6 @@ define <8 x float> @ubto8f32(<8 x i32> %a) {
 define <8 x double> @ubto8f64(<8 x i32> %a) {
 ; ALL-LABEL: ubto8f64:
 ; ALL:       # %bb.0:
-; ALL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; ALL-NEXT:    vpcmpgtd %ymm0, %ymm1, %ymm0
 ; ALL-NEXT:    vpsrld $31, %ymm0, %ymm0
 ; ALL-NEXT:    vcvtdq2pd %ymm0, %zmm0
 ; ALL-NEXT:    retq
@@ -1930,8 +1928,6 @@ define <4 x float> @ubto4f32(<4 x i32> %a) {
 define <4 x double> @ubto4f64(<4 x i32> %a) {
 ; ALL-LABEL: ubto4f64:
 ; ALL:       # %bb.0:
-; ALL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; ALL-NEXT:    vpcmpgtd %xmm0, %xmm1, %xmm0
 ; ALL-NEXT:    vpsrld $31, %xmm0, %xmm0
 ; ALL-NEXT:    vcvtdq2pd %xmm0, %ymm0
 ; ALL-NEXT:    retq
diff --git a/test/CodeGen/X86/avx512-ext.ll b/test/CodeGen/X86/avx512-ext.ll
index 2381180af763a234b18745a907758110f6126037..072e3c8bdae52b3f673aa191dfb90735cac6f30a 100644
--- a/test/CodeGen/X86/avx512-ext.ll
+++ b/test/CodeGen/X86/avx512-ext.ll
@@ -1644,33 +1644,12 @@ define   <8 x i64> @zext_8i1_to_8xi64(i8 %b) {
 }
 
 define i16 @trunc_16i8_to_16i1(<16 x i8> %a) {
-; KNL-LABEL: trunc_16i8_to_16i1:
-; KNL:       # %bb.0:
-; KNL-NEXT:    vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
-; KNL-NEXT:    vpslld $31, %zmm0, %zmm0
-; KNL-NEXT:    vptestmd %zmm0, %zmm0, %k0
-; KNL-NEXT:    kmovw %k0, %eax
-; KNL-NEXT:    # kill: def $ax killed $ax killed $eax
-; KNL-NEXT:    vzeroupper
-; KNL-NEXT:    retq
-;
-; SKX-LABEL: trunc_16i8_to_16i1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpsllw $7, %xmm0, %xmm0
-; SKX-NEXT:    vpmovb2m %xmm0, %k0
-; SKX-NEXT:    kmovd %k0, %eax
-; SKX-NEXT:    # kill: def $ax killed $ax killed $eax
-; SKX-NEXT:    retq
-;
-; AVX512DQNOBW-LABEL: trunc_16i8_to_16i1:
-; AVX512DQNOBW:       # %bb.0:
-; AVX512DQNOBW-NEXT:    vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
-; AVX512DQNOBW-NEXT:    vpslld $31, %zmm0, %zmm0
-; AVX512DQNOBW-NEXT:    vpmovd2m %zmm0, %k0
-; AVX512DQNOBW-NEXT:    kmovw %k0, %eax
-; AVX512DQNOBW-NEXT:    # kill: def $ax killed $ax killed $eax
-; AVX512DQNOBW-NEXT:    vzeroupper
-; AVX512DQNOBW-NEXT:    retq
+; ALL-LABEL: trunc_16i8_to_16i1:
+; ALL:       # %bb.0:
+; ALL-NEXT:    vpsllw $7, %xmm0, %xmm0
+; ALL-NEXT:    vpmovmskb %xmm0, %eax
+; ALL-NEXT:    # kill: def $ax killed $ax killed $eax
+; ALL-NEXT:    retq
   %mask_b = trunc <16 x i8>%a to <16 x i1>
   %mask = bitcast <16 x i1> %mask_b to i16
   ret i16 %mask
diff --git a/test/CodeGen/X86/avx512-extract-subvector-load-store.ll b/test/CodeGen/X86/avx512-extract-subvector-load-store.ll
index e38ee562d578b4eb8b81fd05159524b36658cd4c..df80e08280d49c896d538ef4bb755a087715715f 100644
--- a/test/CodeGen/X86/avx512-extract-subvector-load-store.ll
+++ b/test/CodeGen/X86/avx512-extract-subvector-load-store.ll
@@ -15,8 +15,7 @@ define void @load_v8i1_broadcast_4_v2i1(<8 x i1>* %a0,<2 x double> %a1,<2 x doub
 ;
 ; AVX512NOTDQ-LABEL: load_v8i1_broadcast_4_v2i1:
 ; AVX512NOTDQ:       # %bb.0:
-; AVX512NOTDQ-NEXT:    movzbl 4(%rdi), %eax
-; AVX512NOTDQ-NEXT:    kmovd %eax, %k1
+; AVX512NOTDQ-NEXT:    kmovw 4(%rdi), %k1
 ; AVX512NOTDQ-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
 ; AVX512NOTDQ-NEXT:    vmovdqa64 %xmm2, %xmm2 {%k1} {z}
 ; AVX512NOTDQ-NEXT:    vpbroadcastq %xmm2, %xmm2
@@ -43,8 +42,7 @@ define void @load_v8i1_broadcast_7_v2i1(<8 x i1>* %a0,<2 x double> %a1,<2 x doub
 ;
 ; AVX512NOTDQ-LABEL: load_v8i1_broadcast_7_v2i1:
 ; AVX512NOTDQ:       # %bb.0:
-; AVX512NOTDQ-NEXT:    movzbl 6(%rdi), %eax
-; AVX512NOTDQ-NEXT:    kmovd %eax, %k1
+; AVX512NOTDQ-NEXT:    kmovw 6(%rdi), %k1
 ; AVX512NOTDQ-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
 ; AVX512NOTDQ-NEXT:    vmovdqa64 %xmm2, %xmm2 {%k1} {z}
 ; AVX512NOTDQ-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,3,2,3]
@@ -71,8 +69,7 @@ define void @load_v16i1_broadcast_8_v2i1(<16 x i1>* %a0,<2 x double> %a1,<2 x do
 ;
 ; AVX512NOTDQ-LABEL: load_v16i1_broadcast_8_v2i1:
 ; AVX512NOTDQ:       # %bb.0:
-; AVX512NOTDQ-NEXT:    movzbl 8(%rdi), %eax
-; AVX512NOTDQ-NEXT:    kmovd %eax, %k1
+; AVX512NOTDQ-NEXT:    kmovw 8(%rdi), %k1
 ; AVX512NOTDQ-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
 ; AVX512NOTDQ-NEXT:    vmovdqa64 %xmm2, %xmm2 {%k1} {z}
 ; AVX512NOTDQ-NEXT:    vpbroadcastq %xmm2, %xmm2
@@ -99,8 +96,7 @@ define void @load_v16i1_broadcast_8_v4i1(<16 x i1>* %a0,<4 x float> %a1,<4 x flo
 ;
 ; AVX512NOTDQ-LABEL: load_v16i1_broadcast_8_v4i1:
 ; AVX512NOTDQ:       # %bb.0:
-; AVX512NOTDQ-NEXT:    movzbl 8(%rdi), %eax
-; AVX512NOTDQ-NEXT:    kmovd %eax, %k1
+; AVX512NOTDQ-NEXT:    kmovw 8(%rdi), %k1
 ; AVX512NOTDQ-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
 ; AVX512NOTDQ-NEXT:    vmovdqa32 %xmm2, %xmm2 {%k1} {z}
 ; AVX512NOTDQ-NEXT:    vpbroadcastd %xmm2, %xmm2
@@ -127,8 +123,7 @@ define void @load_v16i1_broadcast_15_v2i1(<16 x i1>* %a0,<2 x double> %a1,<2 x d
 ;
 ; AVX512NOTDQ-LABEL: load_v16i1_broadcast_15_v2i1:
 ; AVX512NOTDQ:       # %bb.0:
-; AVX512NOTDQ-NEXT:    movzbl 14(%rdi), %eax
-; AVX512NOTDQ-NEXT:    kmovd %eax, %k1
+; AVX512NOTDQ-NEXT:    kmovw 14(%rdi), %k1
 ; AVX512NOTDQ-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
 ; AVX512NOTDQ-NEXT:    vmovdqa64 %xmm2, %xmm2 {%k1} {z}
 ; AVX512NOTDQ-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,3,2,3]
@@ -155,8 +150,7 @@ define void @load_v16i1_broadcast_15_v4i1(<16 x i1>* %a0,<4 x float> %a1,<4 x fl
 ;
 ; AVX512NOTDQ-LABEL: load_v16i1_broadcast_15_v4i1:
 ; AVX512NOTDQ:       # %bb.0:
-; AVX512NOTDQ-NEXT:    movzbl 12(%rdi), %eax
-; AVX512NOTDQ-NEXT:    kmovd %eax, %k1
+; AVX512NOTDQ-NEXT:    kmovw 12(%rdi), %k1
 ; AVX512NOTDQ-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
 ; AVX512NOTDQ-NEXT:    vmovdqa32 %xmm2, %xmm2 {%k1} {z}
 ; AVX512NOTDQ-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[3,3,3,3]
@@ -183,8 +177,7 @@ define void @load_v32i1_broadcast_16_v2i1(<32 x i1>* %a0,<2 x double> %a1,<2 x d
 ;
 ; AVX512NOTDQ-LABEL: load_v32i1_broadcast_16_v2i1:
 ; AVX512NOTDQ:       # %bb.0:
-; AVX512NOTDQ-NEXT:    movzbl 16(%rdi), %eax
-; AVX512NOTDQ-NEXT:    kmovd %eax, %k1
+; AVX512NOTDQ-NEXT:    kmovw 16(%rdi), %k1
 ; AVX512NOTDQ-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
 ; AVX512NOTDQ-NEXT:    vmovdqa64 %xmm2, %xmm2 {%k1} {z}
 ; AVX512NOTDQ-NEXT:    vpbroadcastq %xmm2, %xmm2
@@ -211,8 +204,7 @@ define void @load_v32i1_broadcast_16_v4i1(<32 x i1>* %a0,<4 x float> %a1,<4 x fl
 ;
 ; AVX512NOTDQ-LABEL: load_v32i1_broadcast_16_v4i1:
 ; AVX512NOTDQ:       # %bb.0:
-; AVX512NOTDQ-NEXT:    movzbl 16(%rdi), %eax
-; AVX512NOTDQ-NEXT:    kmovd %eax, %k1
+; AVX512NOTDQ-NEXT:    kmovw 16(%rdi), %k1
 ; AVX512NOTDQ-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
 ; AVX512NOTDQ-NEXT:    vmovdqa32 %xmm2, %xmm2 {%k1} {z}
 ; AVX512NOTDQ-NEXT:    vpbroadcastd %xmm2, %xmm2
@@ -240,8 +232,7 @@ define void @load_v32i1_broadcast_16_v8i1(<32 x i1>* %a0,<8 x float> %a1,<8 x fl
 ;
 ; AVX512NOTDQ-LABEL: load_v32i1_broadcast_16_v8i1:
 ; AVX512NOTDQ:       # %bb.0:
-; AVX512NOTDQ-NEXT:    movzbl 16(%rdi), %eax
-; AVX512NOTDQ-NEXT:    kmovd %eax, %k1
+; AVX512NOTDQ-NEXT:    kmovw 16(%rdi), %k1
 ; AVX512NOTDQ-NEXT:    vpcmpeqd %ymm2, %ymm2, %ymm2
 ; AVX512NOTDQ-NEXT:    vmovdqa32 %ymm2, %ymm2 {%k1} {z}
 ; AVX512NOTDQ-NEXT:    vpbroadcastd %xmm2, %ymm2
@@ -269,8 +260,7 @@ define void @load_v32i1_broadcast_31_v2i1(<32 x i1>* %a0,<2 x double> %a1,<2 x d
 ;
 ; AVX512NOTDQ-LABEL: load_v32i1_broadcast_31_v2i1:
 ; AVX512NOTDQ:       # %bb.0:
-; AVX512NOTDQ-NEXT:    movzbl 30(%rdi), %eax
-; AVX512NOTDQ-NEXT:    kmovd %eax, %k1
+; AVX512NOTDQ-NEXT:    kmovw 30(%rdi), %k1
 ; AVX512NOTDQ-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
 ; AVX512NOTDQ-NEXT:    vmovdqa64 %xmm2, %xmm2 {%k1} {z}
 ; AVX512NOTDQ-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,3,2,3]
@@ -297,8 +287,7 @@ define void @load_v32i1_broadcast_31_v4i1(<32 x i1>* %a0,<4 x float> %a1,<4 x fl
 ;
 ; AVX512NOTDQ-LABEL: load_v32i1_broadcast_31_v4i1:
 ; AVX512NOTDQ:       # %bb.0:
-; AVX512NOTDQ-NEXT:    movzbl 28(%rdi), %eax
-; AVX512NOTDQ-NEXT:    kmovd %eax, %k1
+; AVX512NOTDQ-NEXT:    kmovw 28(%rdi), %k1
 ; AVX512NOTDQ-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
 ; AVX512NOTDQ-NEXT:    vmovdqa32 %xmm2, %xmm2 {%k1} {z}
 ; AVX512NOTDQ-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[3,3,3,3]
@@ -327,8 +316,7 @@ define void @load_v32i1_broadcast_31_v8i1(<32 x i1>* %a0,<8 x float> %a1,<8 x fl
 ;
 ; AVX512NOTDQ-LABEL: load_v32i1_broadcast_31_v8i1:
 ; AVX512NOTDQ:       # %bb.0:
-; AVX512NOTDQ-NEXT:    movzbl 24(%rdi), %eax
-; AVX512NOTDQ-NEXT:    kmovd %eax, %k1
+; AVX512NOTDQ-NEXT:    kmovw 24(%rdi), %k1
 ; AVX512NOTDQ-NEXT:    vpcmpeqd %ymm2, %ymm2, %ymm2
 ; AVX512NOTDQ-NEXT:    vmovdqa32 %ymm2, %ymm2 {%k1} {z}
 ; AVX512NOTDQ-NEXT:    vpbroadcastd {{.*#+}} ymm3 = [7,7,7,7,7,7,7,7]
@@ -357,8 +345,7 @@ define void @load_v64i1_broadcast_32_v2i1(<64 x i1>* %a0,<2 x double> %a1,<2 x d
 ;
 ; AVX512NOTDQ-LABEL: load_v64i1_broadcast_32_v2i1:
 ; AVX512NOTDQ:       # %bb.0:
-; AVX512NOTDQ-NEXT:    movzbl 32(%rdi), %eax
-; AVX512NOTDQ-NEXT:    kmovd %eax, %k1
+; AVX512NOTDQ-NEXT:    kmovw 32(%rdi), %k1
 ; AVX512NOTDQ-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
 ; AVX512NOTDQ-NEXT:    vmovdqa64 %xmm2, %xmm2 {%k1} {z}
 ; AVX512NOTDQ-NEXT:    vpbroadcastq %xmm2, %xmm2
@@ -385,8 +372,7 @@ define void @load_v64i1_broadcast_32_v4i1(<64 x i1>* %a0,<4 x float> %a1,<4 x fl
 ;
 ; AVX512NOTDQ-LABEL: load_v64i1_broadcast_32_v4i1:
 ; AVX512NOTDQ:       # %bb.0:
-; AVX512NOTDQ-NEXT:    movzbl 32(%rdi), %eax
-; AVX512NOTDQ-NEXT:    kmovd %eax, %k1
+; AVX512NOTDQ-NEXT:    kmovw 32(%rdi), %k1
 ; AVX512NOTDQ-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
 ; AVX512NOTDQ-NEXT:    vmovdqa32 %xmm2, %xmm2 {%k1} {z}
 ; AVX512NOTDQ-NEXT:    vpbroadcastd %xmm2, %xmm2
@@ -414,8 +400,7 @@ define void @load_v64i1_broadcast_32_v8i1(<64 x i1>* %a0,<8 x float> %a1,<8 x fl
 ;
 ; AVX512NOTDQ-LABEL: load_v64i1_broadcast_32_v8i1:
 ; AVX512NOTDQ:       # %bb.0:
-; AVX512NOTDQ-NEXT:    movzbl 32(%rdi), %eax
-; AVX512NOTDQ-NEXT:    kmovd %eax, %k1
+; AVX512NOTDQ-NEXT:    kmovw 32(%rdi), %k1
 ; AVX512NOTDQ-NEXT:    vpcmpeqd %ymm2, %ymm2, %ymm2
 ; AVX512NOTDQ-NEXT:    vmovdqa32 %ymm2, %ymm2 {%k1} {z}
 ; AVX512NOTDQ-NEXT:    vpbroadcastd %xmm2, %ymm2
@@ -471,8 +456,7 @@ define void @load_v64i1_broadcast_63_v2i1(<64 x i1>* %a0,<2 x double> %a1,<2 x d
 ;
 ; AVX512NOTDQ-LABEL: load_v64i1_broadcast_63_v2i1:
 ; AVX512NOTDQ:       # %bb.0:
-; AVX512NOTDQ-NEXT:    movzbl 62(%rdi), %eax
-; AVX512NOTDQ-NEXT:    kmovd %eax, %k1
+; AVX512NOTDQ-NEXT:    kmovw 62(%rdi), %k1
 ; AVX512NOTDQ-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
 ; AVX512NOTDQ-NEXT:    vmovdqa64 %xmm2, %xmm2 {%k1} {z}
 ; AVX512NOTDQ-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,3,2,3]
@@ -499,8 +483,7 @@ define void @load_v64i1_broadcast_63_v4i1(<64 x i1>* %a0,<4 x float> %a1,<4 x fl
 ;
 ; AVX512NOTDQ-LABEL: load_v64i1_broadcast_63_v4i1:
 ; AVX512NOTDQ:       # %bb.0:
-; AVX512NOTDQ-NEXT:    movzbl 60(%rdi), %eax
-; AVX512NOTDQ-NEXT:    kmovd %eax, %k1
+; AVX512NOTDQ-NEXT:    kmovw 60(%rdi), %k1
 ; AVX512NOTDQ-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
 ; AVX512NOTDQ-NEXT:    vmovdqa32 %xmm2, %xmm2 {%k1} {z}
 ; AVX512NOTDQ-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[3,3,3,3]
@@ -529,8 +512,7 @@ define void @load_v64i1_broadcast_63_v8i1(<64 x i1>* %a0,<8 x float> %a1,<8 x fl
 ;
 ; AVX512NOTDQ-LABEL: load_v64i1_broadcast_63_v8i1:
 ; AVX512NOTDQ:       # %bb.0:
-; AVX512NOTDQ-NEXT:    movzbl 56(%rdi), %eax
-; AVX512NOTDQ-NEXT:    kmovd %eax, %k1
+; AVX512NOTDQ-NEXT:    kmovw 56(%rdi), %k1
 ; AVX512NOTDQ-NEXT:    vpcmpeqd %ymm2, %ymm2, %ymm2
 ; AVX512NOTDQ-NEXT:    vmovdqa32 %ymm2, %ymm2 {%k1} {z}
 ; AVX512NOTDQ-NEXT:    vpbroadcastd {{.*#+}} ymm3 = [7,7,7,7,7,7,7,7]
@@ -690,8 +672,7 @@ define void @load_v8i1_broadcast_4_v2i1_store(<8 x i1>* %a0,<2 x i1>* %a1) {
 ;
 ; AVX512NOTDQ-LABEL: load_v8i1_broadcast_4_v2i1_store:
 ; AVX512NOTDQ:       # %bb.0:
-; AVX512NOTDQ-NEXT:    movzbl 4(%rdi), %eax
-; AVX512NOTDQ-NEXT:    kmovd %eax, %k1
+; AVX512NOTDQ-NEXT:    kmovw 4(%rdi), %k1
 ; AVX512NOTDQ-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
 ; AVX512NOTDQ-NEXT:    vmovdqa64 %xmm0, %xmm0 {%k1} {z}
 ; AVX512NOTDQ-NEXT:    vpbroadcastq %xmm0, %xmm0
@@ -733,8 +714,7 @@ define void @load_v8i1_broadcast_7_v2i1_store(<8 x i1>* %a0,<2 x i1>* %a1) {
 ;
 ; AVX512NOTDQ-LABEL: load_v8i1_broadcast_7_v2i1_store:
 ; AVX512NOTDQ:       # %bb.0:
-; AVX512NOTDQ-NEXT:    movzbl 6(%rdi), %eax
-; AVX512NOTDQ-NEXT:    kmovd %eax, %k1
+; AVX512NOTDQ-NEXT:    kmovw 6(%rdi), %k1
 ; AVX512NOTDQ-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
 ; AVX512NOTDQ-NEXT:    vmovdqa64 %xmm0, %xmm0 {%k1} {z}
 ; AVX512NOTDQ-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
@@ -776,8 +756,7 @@ define void @load_v16i1_broadcast_8_v2i1_store(<16 x i1>* %a0,<2 x i1>* %a1) {
 ;
 ; AVX512NOTDQ-LABEL: load_v16i1_broadcast_8_v2i1_store:
 ; AVX512NOTDQ:       # %bb.0:
-; AVX512NOTDQ-NEXT:    movzbl 8(%rdi), %eax
-; AVX512NOTDQ-NEXT:    kmovd %eax, %k1
+; AVX512NOTDQ-NEXT:    kmovw 8(%rdi), %k1
 ; AVX512NOTDQ-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
 ; AVX512NOTDQ-NEXT:    vmovdqa64 %xmm0, %xmm0 {%k1} {z}
 ; AVX512NOTDQ-NEXT:    vpbroadcastq %xmm0, %xmm0
@@ -802,8 +781,7 @@ define void @load_v16i1_broadcast_8_v4i1_store(<16 x i1>* %a0,<4 x i1>* %a1) {
 ;
 ; AVX512NOTDQ-LABEL: load_v16i1_broadcast_8_v4i1_store:
 ; AVX512NOTDQ:       # %bb.0:
-; AVX512NOTDQ-NEXT:    movzbl 8(%rdi), %eax
-; AVX512NOTDQ-NEXT:    kmovd %eax, %k1
+; AVX512NOTDQ-NEXT:    kmovw 8(%rdi), %k1
 ; AVX512NOTDQ-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
 ; AVX512NOTDQ-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
 ; AVX512NOTDQ-NEXT:    vpbroadcastd %xmm0, %xmm0
@@ -845,8 +823,7 @@ define void @load_v16i1_broadcast_15_v2i1_store(<16 x i1>* %a0,<2 x i1>* %a1) {
 ;
 ; AVX512NOTDQ-LABEL: load_v16i1_broadcast_15_v2i1_store:
 ; AVX512NOTDQ:       # %bb.0:
-; AVX512NOTDQ-NEXT:    movzbl 14(%rdi), %eax
-; AVX512NOTDQ-NEXT:    kmovd %eax, %k1
+; AVX512NOTDQ-NEXT:    kmovw 14(%rdi), %k1
 ; AVX512NOTDQ-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
 ; AVX512NOTDQ-NEXT:    vmovdqa64 %xmm0, %xmm0 {%k1} {z}
 ; AVX512NOTDQ-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
@@ -871,8 +848,7 @@ define void @load_v16i1_broadcast_15_v4i1_store(<16 x i1>* %a0,<4 x i1>* %a1) {
 ;
 ; AVX512NOTDQ-LABEL: load_v16i1_broadcast_15_v4i1_store:
 ; AVX512NOTDQ:       # %bb.0:
-; AVX512NOTDQ-NEXT:    movzbl 12(%rdi), %eax
-; AVX512NOTDQ-NEXT:    kmovd %eax, %k1
+; AVX512NOTDQ-NEXT:    kmovw 12(%rdi), %k1
 ; AVX512NOTDQ-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
 ; AVX512NOTDQ-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
 ; AVX512NOTDQ-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
@@ -914,8 +890,7 @@ define void @load_v32i1_broadcast_16_v2i1_store(<32 x i1>* %a0,<2 x i1>* %a1) {
 ;
 ; AVX512NOTDQ-LABEL: load_v32i1_broadcast_16_v2i1_store:
 ; AVX512NOTDQ:       # %bb.0:
-; AVX512NOTDQ-NEXT:    movzbl 16(%rdi), %eax
-; AVX512NOTDQ-NEXT:    kmovd %eax, %k1
+; AVX512NOTDQ-NEXT:    kmovw 16(%rdi), %k1
 ; AVX512NOTDQ-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
 ; AVX512NOTDQ-NEXT:    vmovdqa64 %xmm0, %xmm0 {%k1} {z}
 ; AVX512NOTDQ-NEXT:    vpbroadcastq %xmm0, %xmm0
@@ -940,8 +915,7 @@ define void @load_v32i1_broadcast_16_v4i1_store(<32 x i1>* %a0,<4 x i1>* %a1) {
 ;
 ; AVX512NOTDQ-LABEL: load_v32i1_broadcast_16_v4i1_store:
 ; AVX512NOTDQ:       # %bb.0:
-; AVX512NOTDQ-NEXT:    movzbl 16(%rdi), %eax
-; AVX512NOTDQ-NEXT:    kmovd %eax, %k1
+; AVX512NOTDQ-NEXT:    kmovw 16(%rdi), %k1
 ; AVX512NOTDQ-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
 ; AVX512NOTDQ-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
 ; AVX512NOTDQ-NEXT:    vpbroadcastd %xmm0, %xmm0
@@ -967,8 +941,7 @@ define void @load_v32i1_broadcast_16_v8i1_store(<32 x i1>* %a0,<8 x i1>* %a1) {
 ;
 ; AVX512NOTDQ-LABEL: load_v32i1_broadcast_16_v8i1_store:
 ; AVX512NOTDQ:       # %bb.0:
-; AVX512NOTDQ-NEXT:    movzbl 16(%rdi), %eax
-; AVX512NOTDQ-NEXT:    kmovd %eax, %k1
+; AVX512NOTDQ-NEXT:    kmovw 16(%rdi), %k1
 ; AVX512NOTDQ-NEXT:    vpcmpeqd %ymm0, %ymm0, %ymm0
 ; AVX512NOTDQ-NEXT:    vmovdqa32 %ymm0, %ymm0 {%k1} {z}
 ; AVX512NOTDQ-NEXT:    vpbroadcastd %xmm0, %ymm0
@@ -1011,8 +984,7 @@ define void @load_v32i1_broadcast_31_v2i1_store(<32 x i1>* %a0,<2 x i1>* %a1) {
 ;
 ; AVX512NOTDQ-LABEL: load_v32i1_broadcast_31_v2i1_store:
 ; AVX512NOTDQ:       # %bb.0:
-; AVX512NOTDQ-NEXT:    movzbl 30(%rdi), %eax
-; AVX512NOTDQ-NEXT:    kmovd %eax, %k1
+; AVX512NOTDQ-NEXT:    kmovw 30(%rdi), %k1
 ; AVX512NOTDQ-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
 ; AVX512NOTDQ-NEXT:    vmovdqa64 %xmm0, %xmm0 {%k1} {z}
 ; AVX512NOTDQ-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
@@ -1037,8 +1009,7 @@ define void @load_v32i1_broadcast_31_v4i1_store(<32 x i1>* %a0,<4 x i1>* %a1) {
 ;
 ; AVX512NOTDQ-LABEL: load_v32i1_broadcast_31_v4i1_store:
 ; AVX512NOTDQ:       # %bb.0:
-; AVX512NOTDQ-NEXT:    movzbl 28(%rdi), %eax
-; AVX512NOTDQ-NEXT:    kmovd %eax, %k1
+; AVX512NOTDQ-NEXT:    kmovw 28(%rdi), %k1
 ; AVX512NOTDQ-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
 ; AVX512NOTDQ-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
 ; AVX512NOTDQ-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
@@ -1065,8 +1036,7 @@ define void @load_v32i1_broadcast_31_v8i1_store(<32 x i1>* %a0,<8 x i1>* %a1) {
 ;
 ; AVX512NOTDQ-LABEL: load_v32i1_broadcast_31_v8i1_store:
 ; AVX512NOTDQ:       # %bb.0:
-; AVX512NOTDQ-NEXT:    movzbl 24(%rdi), %eax
-; AVX512NOTDQ-NEXT:    kmovd %eax, %k1
+; AVX512NOTDQ-NEXT:    kmovw 24(%rdi), %k1
 ; AVX512NOTDQ-NEXT:    vpcmpeqd %ymm0, %ymm0, %ymm0
 ; AVX512NOTDQ-NEXT:    vmovdqa32 %ymm0, %ymm0 {%k1} {z}
 ; AVX512NOTDQ-NEXT:    vpbroadcastd {{.*#+}} ymm1 = [7,7,7,7,7,7,7,7]
@@ -1110,8 +1080,7 @@ define void @load_v64i1_broadcast_32_v2i1_store(<64 x i1>* %a0,<2 x i1>* %a1) {
 ;
 ; AVX512NOTDQ-LABEL: load_v64i1_broadcast_32_v2i1_store:
 ; AVX512NOTDQ:       # %bb.0:
-; AVX512NOTDQ-NEXT:    movzbl 32(%rdi), %eax
-; AVX512NOTDQ-NEXT:    kmovd %eax, %k1
+; AVX512NOTDQ-NEXT:    kmovw 32(%rdi), %k1
 ; AVX512NOTDQ-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
 ; AVX512NOTDQ-NEXT:    vmovdqa64 %xmm0, %xmm0 {%k1} {z}
 ; AVX512NOTDQ-NEXT:    vpbroadcastq %xmm0, %xmm0
@@ -1136,8 +1105,7 @@ define void @load_v64i1_broadcast_32_v4i1_store(<64 x i1>* %a0,<4 x i1>* %a1) {
 ;
 ; AVX512NOTDQ-LABEL: load_v64i1_broadcast_32_v4i1_store:
 ; AVX512NOTDQ:       # %bb.0:
-; AVX512NOTDQ-NEXT:    movzbl 32(%rdi), %eax
-; AVX512NOTDQ-NEXT:    kmovd %eax, %k1
+; AVX512NOTDQ-NEXT:    kmovw 32(%rdi), %k1
 ; AVX512NOTDQ-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
 ; AVX512NOTDQ-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
 ; AVX512NOTDQ-NEXT:    vpbroadcastd %xmm0, %xmm0
@@ -1163,8 +1131,7 @@ define void @load_v64i1_broadcast_32_v8i1_store(<64 x i1>* %a0,<8 x i1>* %a1) {
 ;
 ; AVX512NOTDQ-LABEL: load_v64i1_broadcast_32_v8i1_store:
 ; AVX512NOTDQ:       # %bb.0:
-; AVX512NOTDQ-NEXT:    movzbl 32(%rdi), %eax
-; AVX512NOTDQ-NEXT:    kmovd %eax, %k1
+; AVX512NOTDQ-NEXT:    kmovw 32(%rdi), %k1
 ; AVX512NOTDQ-NEXT:    vpcmpeqd %ymm0, %ymm0, %ymm0
 ; AVX512NOTDQ-NEXT:    vmovdqa32 %ymm0, %ymm0 {%k1} {z}
 ; AVX512NOTDQ-NEXT:    vpbroadcastd %xmm0, %ymm0
@@ -1232,8 +1199,7 @@ define void @load_v64i1_broadcast_63_v2i1_store(<64 x i1>* %a0,<2 x i1>* %a1) {
 ;
 ; AVX512NOTDQ-LABEL: load_v64i1_broadcast_63_v2i1_store:
 ; AVX512NOTDQ:       # %bb.0:
-; AVX512NOTDQ-NEXT:    movzbl 62(%rdi), %eax
-; AVX512NOTDQ-NEXT:    kmovd %eax, %k1
+; AVX512NOTDQ-NEXT:    kmovw 62(%rdi), %k1
 ; AVX512NOTDQ-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
 ; AVX512NOTDQ-NEXT:    vmovdqa64 %xmm0, %xmm0 {%k1} {z}
 ; AVX512NOTDQ-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
@@ -1258,8 +1224,7 @@ define void @load_v64i1_broadcast_63_v4i1_store(<64 x i1>* %a0,<4 x i1>* %a1) {
 ;
 ; AVX512NOTDQ-LABEL: load_v64i1_broadcast_63_v4i1_store:
 ; AVX512NOTDQ:       # %bb.0:
-; AVX512NOTDQ-NEXT:    movzbl 60(%rdi), %eax
-; AVX512NOTDQ-NEXT:    kmovd %eax, %k1
+; AVX512NOTDQ-NEXT:    kmovw 60(%rdi), %k1
 ; AVX512NOTDQ-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
 ; AVX512NOTDQ-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
 ; AVX512NOTDQ-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
@@ -1286,8 +1251,7 @@ define void @load_v64i1_broadcast_63_v8i1_store(<64 x i1>* %a0,<8 x i1>* %a1) {
 ;
 ; AVX512NOTDQ-LABEL: load_v64i1_broadcast_63_v8i1_store:
 ; AVX512NOTDQ:       # %bb.0:
-; AVX512NOTDQ-NEXT:    movzbl 56(%rdi), %eax
-; AVX512NOTDQ-NEXT:    kmovd %eax, %k1
+; AVX512NOTDQ-NEXT:    kmovw 56(%rdi), %k1
 ; AVX512NOTDQ-NEXT:    vpcmpeqd %ymm0, %ymm0, %ymm0
 ; AVX512NOTDQ-NEXT:    vmovdqa32 %ymm0, %ymm0 {%k1} {z}
 ; AVX512NOTDQ-NEXT:    vpbroadcastd {{.*#+}} ymm1 = [7,7,7,7,7,7,7,7]
diff --git a/test/CodeGen/X86/avx512-gather-scatter-intrin-deprecated.ll b/test/CodeGen/X86/avx512-gather-scatter-intrin-deprecated.ll
new file mode 100644
index 0000000000000000000000000000000000000000..9502ec95d0928750d9f0e02d07116fd59c2bd3b8
--- /dev/null
+++ b/test/CodeGen/X86/avx512-gather-scatter-intrin-deprecated.ll
@@ -0,0 +1,875 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-apple-darwin -mattr=+avx512f,+avx512bw,+avx512vl,+avx512dq | FileCheck %s
+
+declare <16 x float> @llvm.x86.avx512.gather.dps.512 (<16 x float>, i8*, <16 x i32>, i16, i32)
+declare void @llvm.x86.avx512.scatter.dps.512 (i8*, i16, <16 x i32>, <16 x float>, i32)
+declare <8 x double> @llvm.x86.avx512.gather.dpd.512 (<8 x double>, i8*, <8 x i32>, i8, i32)
+declare void @llvm.x86.avx512.scatter.dpd.512 (i8*, i8, <8 x i32>, <8 x double>, i32)
+
+declare <8 x float> @llvm.x86.avx512.gather.qps.512 (<8 x float>, i8*, <8 x i64>, i8, i32)
+declare void @llvm.x86.avx512.scatter.qps.512 (i8*, i8, <8 x i64>, <8 x float>, i32)
+declare <8 x double> @llvm.x86.avx512.gather.qpd.512 (<8 x double>, i8*, <8 x i64>, i8, i32)
+declare void @llvm.x86.avx512.scatter.qpd.512 (i8*, i8, <8 x i64>, <8 x double>, i32)
+
+define void @gather_mask_dps(<16 x i32> %ind, <16 x float> %src, i16 %mask, i8* %base, i8* %stbuf)  {
+; CHECK-LABEL: gather_mask_dps:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    kmovq %k1, %k2
+; CHECK-NEXT:    vgatherdps (%rsi,%zmm0,4), %zmm1 {%k2}
+; CHECK-NEXT:    vpaddd {{.*}}(%rip), %zmm0, %zmm0
+; CHECK-NEXT:    vscatterdps %zmm1, (%rdx,%zmm0,4) {%k1}
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+  %x = call <16 x float> @llvm.x86.avx512.gather.dps.512 (<16 x float> %src, i8* %base, <16 x i32>%ind, i16 %mask, i32 4)
+  %ind2 = add <16 x i32> %ind, <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
+  call void @llvm.x86.avx512.scatter.dps.512 (i8* %stbuf, i16 %mask, <16 x i32>%ind2, <16 x float> %x, i32 4)
+  ret void
+}
+
+define void @gather_mask_dpd(<8 x i32> %ind, <8 x double> %src, i8 %mask, i8* %base, i8* %stbuf)  {
+; CHECK-LABEL: gather_mask_dpd:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    kmovq %k1, %k2
+; CHECK-NEXT:    vgatherdpd (%rsi,%ymm0,4), %zmm1 {%k2}
+; CHECK-NEXT:    vpaddd {{.*}}(%rip), %ymm0, %ymm0
+; CHECK-NEXT:    vscatterdpd %zmm1, (%rdx,%ymm0,4) {%k1}
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+  %x = call <8 x double> @llvm.x86.avx512.gather.dpd.512 (<8 x double> %src, i8* %base, <8 x i32>%ind, i8 %mask, i32 4)
+  %ind2 = add <8 x i32> %ind, <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
+  call void @llvm.x86.avx512.scatter.dpd.512 (i8* %stbuf, i8 %mask, <8 x i32>%ind2, <8 x double> %x, i32 4)
+  ret void
+}
+
+define void @gather_mask_qps(<8 x i64> %ind, <8 x float> %src, i8 %mask, i8* %base, i8* %stbuf)  {
+; CHECK-LABEL: gather_mask_qps:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    kmovq %k1, %k2
+; CHECK-NEXT:    vgatherqps (%rsi,%zmm0,4), %ymm1 {%k2}
+; CHECK-NEXT:    vpaddq {{.*}}(%rip), %zmm0, %zmm0
+; CHECK-NEXT:    vscatterqps %ymm1, (%rdx,%zmm0,4) {%k1}
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+  %x = call <8 x float> @llvm.x86.avx512.gather.qps.512 (<8 x float> %src, i8* %base, <8 x i64>%ind, i8 %mask, i32 4)
+  %ind2 = add <8 x i64> %ind, <i64 0, i64 1, i64 2, i64 3, i64 0, i64 1, i64 2, i64 3>
+  call void @llvm.x86.avx512.scatter.qps.512 (i8* %stbuf, i8 %mask, <8 x i64>%ind2, <8 x float> %x, i32 4)
+  ret void
+}
+
+define void @gather_mask_qpd(<8 x i64> %ind, <8 x double> %src, i8 %mask, i8* %base, i8* %stbuf)  {
+; CHECK-LABEL: gather_mask_qpd:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    kmovq %k1, %k2
+; CHECK-NEXT:    vgatherqpd (%rsi,%zmm0,4), %zmm1 {%k2}
+; CHECK-NEXT:    vpaddq {{.*}}(%rip), %zmm0, %zmm0
+; CHECK-NEXT:    vscatterqpd %zmm1, (%rdx,%zmm0,4) {%k1}
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+  %x = call <8 x double> @llvm.x86.avx512.gather.qpd.512 (<8 x double> %src, i8* %base, <8 x i64>%ind, i8 %mask, i32 4)
+  %ind2 = add <8 x i64> %ind, <i64 0, i64 1, i64 2, i64 3, i64 0, i64 1, i64 2, i64 3>
+  call void @llvm.x86.avx512.scatter.qpd.512 (i8* %stbuf, i8 %mask, <8 x i64>%ind2, <8 x double> %x, i32 4)
+  ret void
+}
+;;
+;; Integer Gather/Scatter
+;;
+declare <16 x i32> @llvm.x86.avx512.gather.dpi.512 (<16 x i32>, i8*, <16 x i32>, i16, i32)
+declare void @llvm.x86.avx512.scatter.dpi.512 (i8*, i16, <16 x i32>, <16 x i32>, i32)
+declare <8 x i64> @llvm.x86.avx512.gather.dpq.512 (<8 x i64>, i8*, <8 x i32>, i8, i32)
+declare void @llvm.x86.avx512.scatter.dpq.512 (i8*, i8, <8 x i32>, <8 x i64>, i32)
+
+declare <8 x i32> @llvm.x86.avx512.gather.qpi.512 (<8 x i32>, i8*, <8 x i64>, i8, i32)
+declare void @llvm.x86.avx512.scatter.qpi.512 (i8*, i8, <8 x i64>, <8 x i32>, i32)
+declare <8 x i64> @llvm.x86.avx512.gather.qpq.512 (<8 x i64>, i8*, <8 x i64>, i8, i32)
+declare void @llvm.x86.avx512.scatter.qpq.512 (i8*, i8, <8 x i64>, <8 x i64>, i32)
+
+define void @gather_mask_dd(<16 x i32> %ind, <16 x i32> %src, i16 %mask, i8* %base, i8* %stbuf)  {
+; CHECK-LABEL: gather_mask_dd:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    kmovq %k1, %k2
+; CHECK-NEXT:    vpgatherdd (%rsi,%zmm0,4), %zmm1 {%k2}
+; CHECK-NEXT:    vpaddd {{.*}}(%rip), %zmm0, %zmm0
+; CHECK-NEXT:    vpscatterdd %zmm1, (%rdx,%zmm0,4) {%k1}
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+  %x = call <16 x i32> @llvm.x86.avx512.gather.dpi.512 (<16 x i32> %src, i8* %base, <16 x i32>%ind, i16 %mask, i32 4)
+  %ind2 = add <16 x i32> %ind, <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
+  call void @llvm.x86.avx512.scatter.dpi.512 (i8* %stbuf, i16 %mask, <16 x i32>%ind2, <16 x i32> %x, i32 4)
+  ret void
+}
+
+define void @gather_mask_qd(<8 x i64> %ind, <8 x i32> %src, i8 %mask, i8* %base, i8* %stbuf)  {
+; CHECK-LABEL: gather_mask_qd:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    kmovq %k1, %k2
+; CHECK-NEXT:    vpgatherqd (%rsi,%zmm0,4), %ymm1 {%k2}
+; CHECK-NEXT:    vpaddq {{.*}}(%rip), %zmm0, %zmm0
+; CHECK-NEXT:    vpscatterqd %ymm1, (%rdx,%zmm0,4) {%k1}
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+  %x = call <8 x i32> @llvm.x86.avx512.gather.qpi.512 (<8 x i32> %src, i8* %base, <8 x i64>%ind, i8 %mask, i32 4)
+  %ind2 = add <8 x i64> %ind, <i64 0, i64 1, i64 2, i64 3, i64 0, i64 1, i64 2, i64 3>
+  call void @llvm.x86.avx512.scatter.qpi.512 (i8* %stbuf, i8 %mask, <8 x i64>%ind2, <8 x i32> %x, i32 4)
+  ret void
+}
+
+define void @gather_mask_qq(<8 x i64> %ind, <8 x i64> %src, i8 %mask, i8* %base, i8* %stbuf)  {
+; CHECK-LABEL: gather_mask_qq:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    kmovq %k1, %k2
+; CHECK-NEXT:    vpgatherqq (%rsi,%zmm0,4), %zmm1 {%k2}
+; CHECK-NEXT:    vpaddq {{.*}}(%rip), %zmm0, %zmm0
+; CHECK-NEXT:    vpscatterqq %zmm1, (%rdx,%zmm0,4) {%k1}
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+  %x = call <8 x i64> @llvm.x86.avx512.gather.qpq.512 (<8 x i64> %src, i8* %base, <8 x i64>%ind, i8 %mask, i32 4)
+  %ind2 = add <8 x i64> %ind, <i64 0, i64 1, i64 2, i64 3, i64 0, i64 1, i64 2, i64 3>
+  call void @llvm.x86.avx512.scatter.qpq.512 (i8* %stbuf, i8 %mask, <8 x i64>%ind2, <8 x i64> %x, i32 4)
+  ret void
+}
+
+define void @gather_mask_dq(<8 x i32> %ind, <8 x i64> %src, i8 %mask, i8* %base, i8* %stbuf)  {
+; CHECK-LABEL: gather_mask_dq:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    kmovq %k1, %k2
+; CHECK-NEXT:    vpgatherdq (%rsi,%ymm0,4), %zmm1 {%k2}
+; CHECK-NEXT:    vpaddd {{.*}}(%rip), %ymm0, %ymm0
+; CHECK-NEXT:    vpscatterdq %zmm1, (%rdx,%ymm0,4) {%k1}
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+  %x = call <8 x i64> @llvm.x86.avx512.gather.dpq.512 (<8 x i64> %src, i8* %base, <8 x i32>%ind, i8 %mask, i32 4)
+  %ind2 = add <8 x i32> %ind, <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
+  call void @llvm.x86.avx512.scatter.dpq.512 (i8* %stbuf, i8 %mask, <8 x i32>%ind2, <8 x i64> %x, i32 4)
+  ret void
+}
+
+define void @gather_mask_dpd_execdomain(<8 x i32> %ind, <8 x double> %src, i8 %mask, i8* %base, <8 x double>* %stbuf)  {
+; CHECK-LABEL: gather_mask_dpd_execdomain:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vgatherdpd (%rsi,%ymm0,4), %zmm1 {%k1}
+; CHECK-NEXT:    vmovapd %zmm1, (%rdx)
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+  %x = call <8 x double> @llvm.x86.avx512.gather.dpd.512 (<8 x double> %src, i8* %base, <8 x i32>%ind, i8 %mask, i32 4)
+  store <8 x double> %x, <8 x double>* %stbuf
+  ret void
+}
+
+define void @gather_mask_qpd_execdomain(<8 x i64> %ind, <8 x double> %src, i8 %mask, i8* %base, <8 x double>* %stbuf)  {
+; CHECK-LABEL: gather_mask_qpd_execdomain:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vgatherqpd (%rsi,%zmm0,4), %zmm1 {%k1}
+; CHECK-NEXT:    vmovapd %zmm1, (%rdx)
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+  %x = call <8 x double> @llvm.x86.avx512.gather.qpd.512 (<8 x double> %src, i8* %base, <8 x i64>%ind, i8 %mask, i32 4)
+  store <8 x double> %x, <8 x double>* %stbuf
+  ret void
+}
+
+define <16 x float> @gather_mask_dps_execdomain(<16 x i32> %ind, <16 x float> %src, i16 %mask, i8* %base)  {
+; CHECK-LABEL: gather_mask_dps_execdomain:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vgatherdps (%rsi,%zmm0,4), %zmm1 {%k1}
+; CHECK-NEXT:    vmovaps %zmm1, %zmm0
+; CHECK-NEXT:    retq
+  %res = call <16 x float> @llvm.x86.avx512.gather.dps.512 (<16 x float> %src, i8* %base, <16 x i32>%ind, i16 %mask, i32 4)
+  ret <16 x float> %res;
+}
+
+define <8 x float> @gather_mask_qps_execdomain(<8 x i64> %ind, <8 x float> %src, i8 %mask, i8* %base)  {
+; CHECK-LABEL: gather_mask_qps_execdomain:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    kmovd %edi, %k1
+; CHECK-NEXT:    vgatherqps (%rsi,%zmm0,4), %ymm1 {%k1}
+; CHECK-NEXT:    vmovaps %ymm1, %ymm0
+; CHECK-NEXT:    retq
+  %res = call <8 x float> @llvm.x86.avx512.gather.qps.512 (<8 x float> %src, i8* %base, <8 x i64>%ind, i8 %mask, i32 4)
+  ret <8 x float> %res;
+}
+
+define void @scatter_mask_dpd_execdomain(<8 x i32> %ind, <8 x double>* %src, i8 %mask, i8* %base, i8* %stbuf)  {
+; CHECK-LABEL: scatter_mask_dpd_execdomain:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    kmovd %esi, %k1
+; CHECK-NEXT:    vmovapd (%rdi), %zmm1
+; CHECK-NEXT:    vscatterdpd %zmm1, (%rcx,%ymm0,4) {%k1}
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+  %x = load <8 x double>, <8 x double>* %src, align 64
+  call void @llvm.x86.avx512.scatter.dpd.512 (i8* %stbuf, i8 %mask, <8 x i32>%ind, <8 x double> %x, i32 4)
+  ret void
+}
+
+define void @scatter_mask_qpd_execdomain(<8 x i64> %ind, <8 x double>* %src, i8 %mask, i8* %base, i8* %stbuf)  {
+; CHECK-LABEL: scatter_mask_qpd_execdomain:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    kmovd %esi, %k1
+; CHECK-NEXT:    vmovapd (%rdi), %zmm1
+; CHECK-NEXT:    vscatterqpd %zmm1, (%rcx,%zmm0,4) {%k1}
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+  %x = load <8 x double>, <8 x double>* %src, align 64
+  call void @llvm.x86.avx512.scatter.qpd.512 (i8* %stbuf, i8 %mask, <8 x i64>%ind, <8 x double> %x, i32 4)
+  ret void
+}
+
+define void @scatter_mask_dps_execdomain(<16 x i32> %ind, <16 x float>* %src, i16 %mask, i8* %base, i8* %stbuf)  {
+; CHECK-LABEL: scatter_mask_dps_execdomain:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    kmovd %esi, %k1
+; CHECK-NEXT:    vmovaps (%rdi), %zmm1
+; CHECK-NEXT:    vscatterdps %zmm1, (%rcx,%zmm0,4) {%k1}
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+  %x = load <16 x float>, <16 x float>* %src, align 64
+  call void @llvm.x86.avx512.scatter.dps.512 (i8* %stbuf, i16 %mask, <16 x i32>%ind, <16 x float> %x, i32 4)
+  ret void
+}
+
+define void @scatter_mask_qps_execdomain(<8 x i64> %ind, <8 x float>* %src, i8 %mask, i8* %base, i8* %stbuf)  {
+; CHECK-LABEL: scatter_mask_qps_execdomain:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    kmovd %esi, %k1
+; CHECK-NEXT:    vmovaps (%rdi), %ymm1
+; CHECK-NEXT:    vscatterqps %ymm1, (%rcx,%zmm0,4) {%k1}
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+  %x = load <8 x float>, <8 x float>* %src, align 32
+  call void @llvm.x86.avx512.scatter.qps.512 (i8* %stbuf, i8 %mask, <8 x i64>%ind, <8 x float> %x, i32 4)
+  ret void
+}
+
+define void @gather_qps(<8 x i64> %ind, <8 x float> %src, i8* %base, i8* %stbuf)  {
+; CHECK-LABEL: gather_qps:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; CHECK-NEXT:    kxnorw %k0, %k0, %k1
+; CHECK-NEXT:    kxnorw %k0, %k0, %k2
+; CHECK-NEXT:    vgatherqps (%rdi,%zmm0,4), %ymm1 {%k2}
+; CHECK-NEXT:    vpaddq {{.*}}(%rip), %zmm0, %zmm0
+; CHECK-NEXT:    vscatterqps %ymm1, (%rsi,%zmm0,4) {%k1}
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+  %x = call <8 x float> @llvm.x86.avx512.gather.qps.512 (<8 x float> %src, i8* %base, <8 x i64>%ind, i8 -1, i32 4)
+  %ind2 = add <8 x i64> %ind, <i64 0, i64 1, i64 2, i64 3, i64 0, i64 1, i64 2, i64 3>
+  call void @llvm.x86.avx512.scatter.qps.512 (i8* %stbuf, i8 -1, <8 x i64>%ind2, <8 x float> %x, i32 4)
+  ret void
+}
+
+declare  void @llvm.x86.avx512.gatherpf.qps.512(i8, <8 x i64>, i8* , i32, i32);
+declare  void @llvm.x86.avx512.scatterpf.qps.512(i8, <8 x i64>, i8* , i32, i32);
+define void @prefetch(<8 x i64> %ind, i8* %base) {
+; CHECK-LABEL: prefetch:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    kxnorw %k0, %k0, %k1
+; CHECK-NEXT:    vgatherpf0qps (%rdi,%zmm0,4) {%k1}
+; CHECK-NEXT:    kxorw %k0, %k0, %k1
+; CHECK-NEXT:    vgatherpf1qps (%rdi,%zmm0,4) {%k1}
+; CHECK-NEXT:    movb $1, %al
+; CHECK-NEXT:    kmovd %eax, %k1
+; CHECK-NEXT:    vscatterpf0qps (%rdi,%zmm0,2) {%k1}
+; CHECK-NEXT:    movb $120, %al
+; CHECK-NEXT:    kmovd %eax, %k1
+; CHECK-NEXT:    vscatterpf1qps (%rdi,%zmm0,2) {%k1}
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+  call void @llvm.x86.avx512.gatherpf.qps.512(i8 -1, <8 x i64> %ind, i8* %base, i32 4, i32 3)
+  call void @llvm.x86.avx512.gatherpf.qps.512(i8 0, <8 x i64> %ind, i8* %base, i32 4, i32 2)
+  call void @llvm.x86.avx512.scatterpf.qps.512(i8 1, <8 x i64> %ind, i8* %base, i32 2, i32 3)
+  call void @llvm.x86.avx512.scatterpf.qps.512(i8 120, <8 x i64> %ind, i8* %base, i32 2, i32 2)
+  ret void
+}
+
+declare <2 x double> @llvm.x86.avx512.gather3div2.df(<2 x double>, i8*, <2 x i64>, i8, i32)
+
+define <2 x double>@test_int_x86_avx512_gather3div2_df(<2 x double> %x0, i8* %x1, <2 x i64> %x2, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_gather3div2_df:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    kmovd %esi, %k1
+; CHECK-NEXT:    vgatherqpd (%rdi,%xmm1,4), %xmm0 {%k1}
+; CHECK-NEXT:    kxnorw %k0, %k0, %k1
+; CHECK-NEXT:    vxorpd %xmm2, %xmm2, %xmm2
+; CHECK-NEXT:    vgatherqpd (%rdi,%xmm1,2), %xmm2 {%k1}
+; CHECK-NEXT:    vaddpd %xmm2, %xmm0, %xmm0
+; CHECK-NEXT:    retq
+  %res = call <2 x double> @llvm.x86.avx512.gather3div2.df(<2 x double> %x0, i8* %x1, <2 x i64> %x2, i8 %x3, i32 4)
+  %res1 = call <2 x double> @llvm.x86.avx512.gather3div2.df(<2 x double> %x0, i8* %x1, <2 x i64> %x2, i8 -1, i32 2)
+  %res2 = fadd <2 x double> %res, %res1
+  ret <2 x double> %res2
+}
+
+declare <2 x i64> @llvm.x86.avx512.gather3div2.di(<2 x i64>, i8*, <2 x i64>, i8, i32)
+
+define <2 x i64>@test_int_x86_avx512_gather3div2_di(<2 x i64> %x0, i8* %x1, <2 x i64> %x2, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_gather3div2_di:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    kmovd %esi, %k1
+; CHECK-NEXT:    vpgatherqq (%rdi,%xmm1,8), %xmm0 {%k1}
+; CHECK-NEXT:    vpaddq %xmm0, %xmm0, %xmm0
+; CHECK-NEXT:    retq
+  %res = call <2 x i64> @llvm.x86.avx512.gather3div2.di(<2 x i64> %x0, i8* %x1, <2 x i64> %x2, i8 %x3, i32 8)
+  %res1 = call <2 x i64> @llvm.x86.avx512.gather3div2.di(<2 x i64> %x0, i8* %x1, <2 x i64> %x2, i8 %x3, i32 8)
+  %res2 = add <2 x i64> %res, %res1
+  ret <2 x i64> %res2
+}
+
+declare <4 x double> @llvm.x86.avx512.gather3div4.df(<4 x double>, i8*, <4 x i64>, i8, i32)
+
+define <4 x double>@test_int_x86_avx512_gather3div4_df(<4 x double> %x0, i8* %x1, <4 x i64> %x2, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_gather3div4_df:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    kmovd %esi, %k1
+; CHECK-NEXT:    vgatherqpd (%rdi,%ymm1,4), %ymm0 {%k1}
+; CHECK-NEXT:    kxnorw %k0, %k0, %k1
+; CHECK-NEXT:    vxorpd %xmm2, %xmm2, %xmm2
+; CHECK-NEXT:    vgatherqpd (%rdi,%ymm1,2), %ymm2 {%k1}
+; CHECK-NEXT:    vaddpd %ymm2, %ymm0, %ymm0
+; CHECK-NEXT:    retq
+  %res = call <4 x double> @llvm.x86.avx512.gather3div4.df(<4 x double> %x0, i8* %x1, <4 x i64> %x2, i8 %x3, i32 4)
+  %res1 = call <4 x double> @llvm.x86.avx512.gather3div4.df(<4 x double> %x0, i8* %x1, <4 x i64> %x2, i8 -1, i32 2)
+  %res2 = fadd <4 x double> %res, %res1
+  ret <4 x double> %res2
+}
+
+declare <4 x i64> @llvm.x86.avx512.gather3div4.di(<4 x i64>, i8*, <4 x i64>, i8, i32)
+
+define <4 x i64>@test_int_x86_avx512_gather3div4_di(<4 x i64> %x0, i8* %x1, <4 x i64> %x2, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_gather3div4_di:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    kmovd %esi, %k1
+; CHECK-NEXT:    vpgatherqq (%rdi,%ymm1,8), %ymm0 {%k1}
+; CHECK-NEXT:    kxnorw %k0, %k0, %k1
+; CHECK-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; CHECK-NEXT:    vpgatherqq (%rdi,%ymm1,8), %ymm2 {%k1}
+; CHECK-NEXT:    vpaddq %ymm2, %ymm0, %ymm0
+; CHECK-NEXT:    retq
+  %res = call <4 x i64> @llvm.x86.avx512.gather3div4.di(<4 x i64> %x0, i8* %x1, <4 x i64> %x2, i8 %x3, i32 8)
+  %res1 = call <4 x i64> @llvm.x86.avx512.gather3div4.di(<4 x i64> %x0, i8* %x1, <4 x i64> %x2, i8 -1, i32 8)
+  %res2 = add <4 x i64> %res, %res1
+  ret <4 x i64> %res2
+}
+
+declare <4 x float> @llvm.x86.avx512.gather3div4.sf(<4 x float>, i8*, <2 x i64>, i8, i32)
+
+define <4 x float>@test_int_x86_avx512_gather3div4_sf(<4 x float> %x0, i8* %x1, <2 x i64> %x2, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_gather3div4_sf:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    kmovd %esi, %k1
+; CHECK-NEXT:    vgatherqps (%rdi,%xmm1,4), %xmm0 {%k1}
+; CHECK-NEXT:    kxnorw %k0, %k0, %k1
+; CHECK-NEXT:    vxorps %xmm2, %xmm2, %xmm2
+; CHECK-NEXT:    vgatherqps (%rdi,%xmm1,2), %xmm2 {%k1}
+; CHECK-NEXT:    vaddps %xmm2, %xmm0, %xmm0
+; CHECK-NEXT:    retq
+  %res = call <4 x float> @llvm.x86.avx512.gather3div4.sf(<4 x float> %x0, i8* %x1, <2 x i64> %x2, i8 %x3, i32 4)
+  %res1 = call <4 x float> @llvm.x86.avx512.gather3div4.sf(<4 x float> %x0, i8* %x1, <2 x i64> %x2, i8 -1, i32 2)
+  %res2 = fadd <4 x float> %res, %res1
+  ret <4 x float> %res2
+}
+
+declare <4 x i32> @llvm.x86.avx512.gather3div4.si(<4 x i32>, i8*, <2 x i64>, i8, i32)
+
+define <4 x i32>@test_int_x86_avx512_gather3div4_si(<4 x i32> %x0, i8* %x1, <2 x i64> %x2, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_gather3div4_si:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    kmovd %esi, %k1
+; CHECK-NEXT:    kxnorw %k0, %k0, %k2
+; CHECK-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; CHECK-NEXT:    vpgatherqd (%rdi,%xmm1,4), %xmm2 {%k2}
+; CHECK-NEXT:    vpgatherqd (%rdi,%xmm1,4), %xmm0 {%k1}
+; CHECK-NEXT:    vpaddd %xmm0, %xmm2, %xmm0
+; CHECK-NEXT:    retq
+  %res = call <4 x i32> @llvm.x86.avx512.gather3div4.si(<4 x i32> %x0, i8* %x1, <2 x i64> %x2, i8 -1, i32 4)
+  %res1 = call <4 x i32> @llvm.x86.avx512.gather3div4.si(<4 x i32> %x0, i8* %x1, <2 x i64> %x2, i8 %x3, i32 4)
+  %res2 = add <4 x i32> %res, %res1
+  ret <4 x i32> %res2
+}
+
+declare <4 x float> @llvm.x86.avx512.gather3div8.sf(<4 x float>, i8*, <4 x i64>, i8, i32)
+
+define <4 x float>@test_int_x86_avx512_gather3div8_sf(<4 x float> %x0, i8* %x1, <4 x i64> %x2, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_gather3div8_sf:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    kmovd %esi, %k1
+; CHECK-NEXT:    vgatherqps (%rdi,%ymm1,4), %xmm0 {%k1}
+; CHECK-NEXT:    kxnorw %k0, %k0, %k1
+; CHECK-NEXT:    vxorps %xmm2, %xmm2, %xmm2
+; CHECK-NEXT:    vgatherqps (%rdi,%ymm1,2), %xmm2 {%k1}
+; CHECK-NEXT:    vaddps %xmm2, %xmm0, %xmm0
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+  %res = call <4 x float> @llvm.x86.avx512.gather3div8.sf(<4 x float> %x0, i8* %x1, <4 x i64> %x2, i8 %x3, i32 4)
+  %res1 = call <4 x float> @llvm.x86.avx512.gather3div8.sf(<4 x float> %x0, i8* %x1, <4 x i64> %x2, i8 -1, i32 2)
+  %res2 = fadd <4 x float> %res, %res1
+  ret <4 x float> %res2
+}
+
+declare <4 x i32> @llvm.x86.avx512.gather3div8.si(<4 x i32>, i8*, <4 x i64>, i8, i32)
+
+define <4 x i32>@test_int_x86_avx512_gather3div8_si(<4 x i32> %x0, i8* %x1, <4 x i64> %x2, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_gather3div8_si:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    kmovd %esi, %k1
+; CHECK-NEXT:    vmovdqa %xmm0, %xmm2
+; CHECK-NEXT:    kmovq %k1, %k2
+; CHECK-NEXT:    vpgatherqd (%rdi,%ymm1,4), %xmm2 {%k2}
+; CHECK-NEXT:    vpgatherqd (%rdi,%ymm1,2), %xmm0 {%k1}
+; CHECK-NEXT:    vpaddd %xmm0, %xmm2, %xmm0
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+  %res = call <4 x i32> @llvm.x86.avx512.gather3div8.si(<4 x i32> %x0, i8* %x1, <4 x i64> %x2, i8 %x3, i32 4)
+  %res1 = call <4 x i32> @llvm.x86.avx512.gather3div8.si(<4 x i32> %x0, i8* %x1, <4 x i64> %x2, i8 %x3, i32 2)
+  %res2 = add <4 x i32> %res, %res1
+  ret <4 x i32> %res2
+}
+
+declare <2 x double> @llvm.x86.avx512.gather3siv2.df(<2 x double>, i8*, <4 x i32>, i8, i32)
+
+define <2 x double>@test_int_x86_avx512_gather3siv2_df(<2 x double> %x0, i8* %x1, <4 x i32> %x2, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_gather3siv2_df:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    kmovd %esi, %k1
+; CHECK-NEXT:    vgatherdpd (%rdi,%xmm1,4), %xmm0 {%k1}
+; CHECK-NEXT:    kxnorw %k0, %k0, %k1
+; CHECK-NEXT:    vxorpd %xmm2, %xmm2, %xmm2
+; CHECK-NEXT:    vgatherdpd (%rdi,%xmm1,2), %xmm2 {%k1}
+; CHECK-NEXT:    vaddpd %xmm2, %xmm0, %xmm0
+; CHECK-NEXT:    retq
+  %res = call <2 x double> @llvm.x86.avx512.gather3siv2.df(<2 x double> %x0, i8* %x1, <4 x i32> %x2, i8 %x3, i32 4)
+  %res1 = call <2 x double> @llvm.x86.avx512.gather3siv2.df(<2 x double> %x0, i8* %x1, <4 x i32> %x2, i8 -1, i32 2)
+  %res2 = fadd <2 x double> %res, %res1
+  ret <2 x double> %res2
+}
+
+declare <2 x i64> @llvm.x86.avx512.gather3siv2.di(<2 x i64>, i8*, <4 x i32>, i8, i32)
+
+define <2 x i64>@test_int_x86_avx512_gather3siv2_di(<2 x i64> %x0, i8* %x1, <4 x i32> %x2, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_gather3siv2_di:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    kmovd %esi, %k1
+; CHECK-NEXT:    vpgatherdq (%rdi,%xmm1,8), %xmm0 {%k1}
+; CHECK-NEXT:    vpaddq %xmm0, %xmm0, %xmm0
+; CHECK-NEXT:    retq
+  %res = call <2 x i64> @llvm.x86.avx512.gather3siv2.di(<2 x i64> %x0, i8* %x1, <4 x i32> %x2, i8 %x3, i32 8)
+  %res1 = call <2 x i64> @llvm.x86.avx512.gather3siv2.di(<2 x i64> %x0, i8* %x1, <4 x i32> %x2, i8 %x3, i32 8)
+  %res2 = add <2 x i64> %res, %res1
+  ret <2 x i64> %res2
+}
+
+declare <4 x double> @llvm.x86.avx512.gather3siv4.df(<4 x double>, i8*, <4 x i32>, i8, i32)
+
+define <4 x double>@test_int_x86_avx512_gather3siv4_df(<4 x double> %x0, i8* %x1, <4 x i32> %x2, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_gather3siv4_df:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    kmovd %esi, %k1
+; CHECK-NEXT:    vgatherdpd (%rdi,%xmm1,4), %ymm0 {%k1}
+; CHECK-NEXT:    kxnorw %k0, %k0, %k1
+; CHECK-NEXT:    vxorpd %xmm2, %xmm2, %xmm2
+; CHECK-NEXT:    vgatherdpd (%rdi,%xmm1,2), %ymm2 {%k1}
+; CHECK-NEXT:    vaddpd %ymm2, %ymm0, %ymm0
+; CHECK-NEXT:    retq
+  %res = call <4 x double> @llvm.x86.avx512.gather3siv4.df(<4 x double> %x0, i8* %x1, <4 x i32> %x2, i8 %x3, i32 4)
+  %res1 = call <4 x double> @llvm.x86.avx512.gather3siv4.df(<4 x double> %x0, i8* %x1, <4 x i32> %x2, i8 -1, i32 2)
+  %res2 = fadd <4 x double> %res, %res1
+  ret <4 x double> %res2
+}
+
+declare <4 x i64> @llvm.x86.avx512.gather3siv4.di(<4 x i64>, i8*, <4 x i32>, i8, i32)
+
+define <4 x i64>@test_int_x86_avx512_gather3siv4_di(<4 x i64> %x0, i8* %x1, <4 x i32> %x2, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_gather3siv4_di:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    kmovd %esi, %k1
+; CHECK-NEXT:    vpgatherdq (%rdi,%xmm1,8), %ymm0 {%k1}
+; CHECK-NEXT:    vpaddq %ymm0, %ymm0, %ymm0
+; CHECK-NEXT:    retq
+  %res = call <4 x i64> @llvm.x86.avx512.gather3siv4.di(<4 x i64> %x0, i8* %x1, <4 x i32> %x2, i8 %x3, i32 8)
+  %res1 = call <4 x i64> @llvm.x86.avx512.gather3siv4.di(<4 x i64> %x0, i8* %x1, <4 x i32> %x2, i8 %x3, i32 8)
+  %res2 = add <4 x i64> %res, %res1
+  ret <4 x i64> %res2
+}
+
+declare <4 x float> @llvm.x86.avx512.gather3siv4.sf(<4 x float>, i8*, <4 x i32>, i8, i32)
+
+define <4 x float>@test_int_x86_avx512_gather3siv4_sf(<4 x float> %x0, i8* %x1, <4 x i32> %x2, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_gather3siv4_sf:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    kmovd %esi, %k1
+; CHECK-NEXT:    vgatherdps (%rdi,%xmm1,4), %xmm0 {%k1}
+; CHECK-NEXT:    kxnorw %k0, %k0, %k1
+; CHECK-NEXT:    vxorps %xmm2, %xmm2, %xmm2
+; CHECK-NEXT:    vgatherdps (%rdi,%xmm1,2), %xmm2 {%k1}
+; CHECK-NEXT:    vaddps %xmm2, %xmm0, %xmm0
+; CHECK-NEXT:    retq
+  %res = call <4 x float> @llvm.x86.avx512.gather3siv4.sf(<4 x float> %x0, i8* %x1, <4 x i32> %x2, i8 %x3, i32 4)
+  %res1 = call <4 x float> @llvm.x86.avx512.gather3siv4.sf(<4 x float> %x0, i8* %x1, <4 x i32> %x2, i8 -1, i32 2)
+  %res2 = fadd <4 x float> %res, %res1
+  ret <4 x float> %res2
+}
+
+declare <4 x i32> @llvm.x86.avx512.gather3siv4.si(<4 x i32>, i8*, <4 x i32>, i8, i32)
+
+define <4 x i32>@test_int_x86_avx512_gather3siv4_si(<4 x i32> %x0, i8* %x1, <4 x i32> %x2, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_gather3siv4_si:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    kmovd %esi, %k1
+; CHECK-NEXT:    kxnorw %k0, %k0, %k2
+; CHECK-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; CHECK-NEXT:    vpgatherdd (%rdi,%xmm1,4), %xmm2 {%k2}
+; CHECK-NEXT:    vpgatherdd (%rdi,%xmm1,2), %xmm0 {%k1}
+; CHECK-NEXT:    vpaddd %xmm0, %xmm2, %xmm0
+; CHECK-NEXT:    retq
+  %res = call <4 x i32> @llvm.x86.avx512.gather3siv4.si(<4 x i32> %x0, i8* %x1, <4 x i32> %x2, i8 -1, i32 4)
+  %res1 = call <4 x i32> @llvm.x86.avx512.gather3siv4.si(<4 x i32> %x0, i8* %x1, <4 x i32> %x2, i8 %x3, i32 2)
+  %res2 = add <4 x i32> %res, %res1
+  ret <4 x i32> %res2
+}
+
+declare <8 x float> @llvm.x86.avx512.gather3siv8.sf(<8 x float>, i8*, <8 x i32>, i8, i32)
+
+define <8 x float>@test_int_x86_avx512_gather3siv8_sf(<8 x float> %x0, i8* %x1, <8 x i32> %x2, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_gather3siv8_sf:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    kmovd %esi, %k1
+; CHECK-NEXT:    vgatherdps (%rdi,%ymm1,4), %ymm0 {%k1}
+; CHECK-NEXT:    kxnorw %k0, %k0, %k1
+; CHECK-NEXT:    vxorps %xmm2, %xmm2, %xmm2
+; CHECK-NEXT:    vgatherdps (%rdi,%ymm1,2), %ymm2 {%k1}
+; CHECK-NEXT:    vaddps %ymm2, %ymm0, %ymm0
+; CHECK-NEXT:    retq
+  %res = call <8 x float> @llvm.x86.avx512.gather3siv8.sf(<8 x float> %x0, i8* %x1, <8 x i32> %x2, i8 %x3, i32 4)
+  %res1 = call <8 x float> @llvm.x86.avx512.gather3siv8.sf(<8 x float> %x0, i8* %x1, <8 x i32> %x2, i8 -1, i32 2)
+  %res2 = fadd <8 x float> %res, %res1
+  ret <8 x float> %res2
+}
+
+declare <8 x i32> @llvm.x86.avx512.gather3siv8.si(<8 x i32>, i8*, <8 x i32>, i8, i32)
+
+define <8 x i32>@test_int_x86_avx512_gather3siv8_si(<8 x i32> %x0, i8* %x1, <8 x i32> %x2, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_gather3siv8_si:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    kmovd %esi, %k1
+; CHECK-NEXT:    vmovdqa %ymm0, %ymm2
+; CHECK-NEXT:    kmovq %k1, %k2
+; CHECK-NEXT:    vpgatherdd (%rdi,%ymm1,4), %ymm2 {%k2}
+; CHECK-NEXT:    vpgatherdd (%rdi,%ymm1,2), %ymm0 {%k1}
+; CHECK-NEXT:    vpaddd %ymm0, %ymm2, %ymm0
+; CHECK-NEXT:    retq
+  %res = call <8 x i32> @llvm.x86.avx512.gather3siv8.si(<8 x i32> %x0, i8* %x1, <8 x i32> %x2, i8 %x3, i32 4)
+  %res1 = call <8 x i32> @llvm.x86.avx512.gather3siv8.si(<8 x i32> %x0, i8* %x1, <8 x i32> %x2, i8 %x3, i32 2)
+  %res2 = add <8 x i32> %res, %res1
+  ret <8 x i32> %res2
+}
+
+declare void @llvm.x86.avx512.scatterdiv2.df(i8*, i8, <2 x i64>, <2 x double>, i32)
+
+define void@test_int_x86_avx512_scatterdiv2_df(i8* %x0, i8 %x1, <2 x i64> %x2, <2 x double> %x3) {
+; CHECK-LABEL: test_int_x86_avx512_scatterdiv2_df:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    kmovd %esi, %k1
+; CHECK-NEXT:    kxnorw %k0, %k0, %k2
+; CHECK-NEXT:    vscatterqpd %xmm1, (%rdi,%xmm0,2) {%k2}
+; CHECK-NEXT:    vscatterqpd %xmm1, (%rdi,%xmm0,4) {%k1}
+; CHECK-NEXT:    retq
+  call void @llvm.x86.avx512.scatterdiv2.df(i8* %x0, i8 -1, <2 x i64> %x2, <2 x double> %x3, i32 2)
+  call void @llvm.x86.avx512.scatterdiv2.df(i8* %x0, i8 %x1, <2 x i64> %x2, <2 x double> %x3, i32 4)
+  ret void
+}
+
+declare void @llvm.x86.avx512.scatterdiv2.di(i8*, i8, <2 x i64>, <2 x i64>, i32)
+
+define void@test_int_x86_avx512_scatterdiv2_di(i8* %x0, i8 %x1, <2 x i64> %x2, <2 x i64> %x3) {
+; CHECK-LABEL: test_int_x86_avx512_scatterdiv2_di:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    kmovd %esi, %k1
+; CHECK-NEXT:    vpscatterqq %xmm1, (%rdi,%xmm0,2) {%k1}
+; CHECK-NEXT:    kxnorw %k0, %k0, %k1
+; CHECK-NEXT:    vpscatterqq %xmm1, (%rdi,%xmm0,4) {%k1}
+; CHECK-NEXT:    retq
+  call void @llvm.x86.avx512.scatterdiv2.di(i8* %x0, i8 %x1, <2 x i64> %x2, <2 x i64> %x3, i32 2)
+  call void @llvm.x86.avx512.scatterdiv2.di(i8* %x0, i8 -1, <2 x i64> %x2, <2 x i64> %x3, i32 4)
+  ret void
+}
+
+declare void @llvm.x86.avx512.scatterdiv4.df(i8*, i8, <4 x i64>, <4 x double>, i32)
+
+define void@test_int_x86_avx512_scatterdiv4_df(i8* %x0, i8 %x1, <4 x i64> %x2, <4 x double> %x3) {
+; CHECK-LABEL: test_int_x86_avx512_scatterdiv4_df:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    kmovd %esi, %k1
+; CHECK-NEXT:    vscatterqpd %ymm1, (%rdi,%ymm0,2) {%k1}
+; CHECK-NEXT:    kxnorw %k0, %k0, %k1
+; CHECK-NEXT:    vscatterqpd %ymm1, (%rdi,%ymm0,4) {%k1}
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+  call void @llvm.x86.avx512.scatterdiv4.df(i8* %x0, i8 %x1, <4 x i64> %x2, <4 x double> %x3, i32 2)
+  call void @llvm.x86.avx512.scatterdiv4.df(i8* %x0, i8 -1, <4 x i64> %x2, <4 x double> %x3, i32 4)
+  ret void
+}
+
+declare void @llvm.x86.avx512.scatterdiv4.di(i8*, i8, <4 x i64>, <4 x i64>, i32)
+
+define void@test_int_x86_avx512_scatterdiv4_di(i8* %x0, i8 %x1, <4 x i64> %x2, <4 x i64> %x3) {
+; CHECK-LABEL: test_int_x86_avx512_scatterdiv4_di:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    kmovd %esi, %k1
+; CHECK-NEXT:    vpscatterqq %ymm1, (%rdi,%ymm0,2) {%k1}
+; CHECK-NEXT:    kxnorw %k0, %k0, %k1
+; CHECK-NEXT:    vpscatterqq %ymm1, (%rdi,%ymm0,4) {%k1}
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+  call void @llvm.x86.avx512.scatterdiv4.di(i8* %x0, i8 %x1, <4 x i64> %x2, <4 x i64> %x3, i32 2)
+  call void @llvm.x86.avx512.scatterdiv4.di(i8* %x0, i8 -1, <4 x i64> %x2, <4 x i64> %x3, i32 4)
+  ret void
+}
+
+declare void @llvm.x86.avx512.scatterdiv4.sf(i8*, i8, <2 x i64>, <4 x float>, i32)
+
+define void@test_int_x86_avx512_scatterdiv4_sf(i8* %x0, i8 %x1, <2 x i64> %x2, <4 x float> %x3) {
+; CHECK-LABEL: test_int_x86_avx512_scatterdiv4_sf:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    kmovd %esi, %k1
+; CHECK-NEXT:    vscatterqps %xmm1, (%rdi,%xmm0,2) {%k1}
+; CHECK-NEXT:    kxnorw %k0, %k0, %k1
+; CHECK-NEXT:    vscatterqps %xmm1, (%rdi,%xmm0,4) {%k1}
+; CHECK-NEXT:    retq
+  call void @llvm.x86.avx512.scatterdiv4.sf(i8* %x0, i8 %x1, <2 x i64> %x2, <4 x float> %x3, i32 2)
+  call void @llvm.x86.avx512.scatterdiv4.sf(i8* %x0, i8 -1, <2 x i64> %x2, <4 x float> %x3, i32 4)
+  ret void
+}
+
+declare void @llvm.x86.avx512.scatterdiv4.si(i8*, i8, <2 x i64>, <4 x i32>, i32)
+
+define void@test_int_x86_avx512_scatterdiv4_si(i8* %x0, i8 %x1, <2 x i64> %x2, <4 x i32> %x3) {
+; CHECK-LABEL: test_int_x86_avx512_scatterdiv4_si:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    kmovd %esi, %k1
+; CHECK-NEXT:    kxnorw %k0, %k0, %k2
+; CHECK-NEXT:    vpscatterqd %xmm1, (%rdi,%xmm0,2) {%k2}
+; CHECK-NEXT:    vpscatterqd %xmm1, (%rdi,%xmm0,4) {%k1}
+; CHECK-NEXT:    retq
+  call void @llvm.x86.avx512.scatterdiv4.si(i8* %x0, i8 -1, <2 x i64> %x2, <4 x i32> %x3, i32 2)
+  call void @llvm.x86.avx512.scatterdiv4.si(i8* %x0, i8 %x1, <2 x i64> %x2, <4 x i32> %x3, i32 4)
+  ret void
+}
+
+declare void @llvm.x86.avx512.scatterdiv8.sf(i8*, i8, <4 x i64>, <4 x float>, i32)
+
+define void@test_int_x86_avx512_scatterdiv8_sf(i8* %x0, i8 %x1, <4 x i64> %x2, <4 x float> %x3) {
+; CHECK-LABEL: test_int_x86_avx512_scatterdiv8_sf:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    kmovd %esi, %k1
+; CHECK-NEXT:    vscatterqps %xmm1, (%rdi,%ymm0,2) {%k1}
+; CHECK-NEXT:    kxnorw %k0, %k0, %k1
+; CHECK-NEXT:    vscatterqps %xmm1, (%rdi,%ymm0,4) {%k1}
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+  call void @llvm.x86.avx512.scatterdiv8.sf(i8* %x0, i8 %x1, <4 x i64> %x2, <4 x float> %x3, i32 2)
+  call void @llvm.x86.avx512.scatterdiv8.sf(i8* %x0, i8 -1, <4 x i64> %x2, <4 x float> %x3, i32 4)
+  ret void
+}
+
+declare void @llvm.x86.avx512.scatterdiv8.si(i8*, i8, <4 x i64>, <4 x i32>, i32)
+
+define void@test_int_x86_avx512_scatterdiv8_si(i8* %x0, i8 %x1, <4 x i64> %x2, <4 x i32> %x3) {
+; CHECK-LABEL: test_int_x86_avx512_scatterdiv8_si:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    kmovd %esi, %k1
+; CHECK-NEXT:    vpscatterqd %xmm1, (%rdi,%ymm0,2) {%k1}
+; CHECK-NEXT:    kxnorw %k0, %k0, %k1
+; CHECK-NEXT:    vpscatterqd %xmm1, (%rdi,%ymm0,4) {%k1}
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+  call void @llvm.x86.avx512.scatterdiv8.si(i8* %x0, i8 %x1, <4 x i64> %x2, <4 x i32> %x3, i32 2)
+  call void @llvm.x86.avx512.scatterdiv8.si(i8* %x0, i8 -1, <4 x i64> %x2, <4 x i32> %x3, i32 4)
+  ret void
+}
+
+declare void @llvm.x86.avx512.scattersiv2.df(i8*, i8, <4 x i32>, <2 x double>, i32)
+
+define void@test_int_x86_avx512_scattersiv2_df(i8* %x0, i8 %x1, <4 x i32> %x2, <2 x double> %x3) {
+; CHECK-LABEL: test_int_x86_avx512_scattersiv2_df:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    kmovd %esi, %k1
+; CHECK-NEXT:    kxnorw %k0, %k0, %k2
+; CHECK-NEXT:    vscatterdpd %xmm1, (%rdi,%xmm0,2) {%k2}
+; CHECK-NEXT:    vscatterdpd %xmm1, (%rdi,%xmm0,4) {%k1}
+; CHECK-NEXT:    retq
+  call void @llvm.x86.avx512.scattersiv2.df(i8* %x0, i8 -1, <4 x i32> %x2, <2 x double> %x3, i32 2)
+  call void @llvm.x86.avx512.scattersiv2.df(i8* %x0, i8 %x1, <4 x i32> %x2, <2 x double> %x3, i32 4)
+  ret void
+}
+
+declare void @llvm.x86.avx512.scattersiv2.di(i8*, i8, <4 x i32>, <2 x i64>, i32)
+
+define void@test_int_x86_avx512_scattersiv2_di(i8* %x0, i8 %x1, <4 x i32> %x2, <2 x i64> %x3) {
+; CHECK-LABEL: test_int_x86_avx512_scattersiv2_di:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    kmovd %esi, %k1
+; CHECK-NEXT:    kxnorw %k0, %k0, %k2
+; CHECK-NEXT:    vpscatterdq %xmm1, (%rdi,%xmm0,2) {%k2}
+; CHECK-NEXT:    vpscatterdq %xmm1, (%rdi,%xmm0,4) {%k1}
+; CHECK-NEXT:    retq
+  call void @llvm.x86.avx512.scattersiv2.di(i8* %x0, i8 -1, <4 x i32> %x2, <2 x i64> %x3, i32 2)
+  call void @llvm.x86.avx512.scattersiv2.di(i8* %x0, i8 %x1, <4 x i32> %x2, <2 x i64> %x3, i32 4)
+  ret void
+}
+
+declare void @llvm.x86.avx512.scattersiv4.df(i8*, i8, <4 x i32>, <4 x double>, i32)
+
+define void@test_int_x86_avx512_scattersiv4_df(i8* %x0, i8 %x1, <4 x i32> %x2, <4 x double> %x3) {
+; CHECK-LABEL: test_int_x86_avx512_scattersiv4_df:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    kmovd %esi, %k1
+; CHECK-NEXT:    vscatterdpd %ymm1, (%rdi,%xmm0,2) {%k1}
+; CHECK-NEXT:    kxnorw %k0, %k0, %k1
+; CHECK-NEXT:    vscatterdpd %ymm1, (%rdi,%xmm0,4) {%k1}
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+  call void @llvm.x86.avx512.scattersiv4.df(i8* %x0, i8 %x1, <4 x i32> %x2, <4 x double> %x3, i32 2)
+  call void @llvm.x86.avx512.scattersiv4.df(i8* %x0, i8 -1, <4 x i32> %x2, <4 x double> %x3, i32 4)
+  ret void
+}
+
+declare void @llvm.x86.avx512.scattersiv4.di(i8*, i8, <4 x i32>, <4 x i64>, i32)
+
+define void@test_int_x86_avx512_scattersiv4_di(i8* %x0, i8 %x1, <4 x i32> %x2, <4 x i64> %x3) {
+; CHECK-LABEL: test_int_x86_avx512_scattersiv4_di:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    kmovd %esi, %k1
+; CHECK-NEXT:    kxnorw %k0, %k0, %k2
+; CHECK-NEXT:    vpscatterdq %ymm1, (%rdi,%xmm0,2) {%k2}
+; CHECK-NEXT:    vpscatterdq %ymm1, (%rdi,%xmm0,4) {%k1}
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+  call void @llvm.x86.avx512.scattersiv4.di(i8* %x0, i8 -1, <4 x i32> %x2, <4 x i64> %x3, i32 2)
+  call void @llvm.x86.avx512.scattersiv4.di(i8* %x0, i8 %x1, <4 x i32> %x2, <4 x i64> %x3, i32 4)
+  ret void
+}
+
+declare void @llvm.x86.avx512.scattersiv4.sf(i8*, i8, <4 x i32>, <4 x float>, i32)
+
+define void@test_int_x86_avx512_scattersiv4_sf(i8* %x0, i8 %x1, <4 x i32> %x2, <4 x float> %x3) {
+; CHECK-LABEL: test_int_x86_avx512_scattersiv4_sf:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    kmovd %esi, %k1
+; CHECK-NEXT:    vscatterdps %xmm1, (%rdi,%xmm0,2) {%k1}
+; CHECK-NEXT:    kxnorw %k0, %k0, %k1
+; CHECK-NEXT:    vscatterdps %xmm1, (%rdi,%xmm0,4) {%k1}
+; CHECK-NEXT:    retq
+  call void @llvm.x86.avx512.scattersiv4.sf(i8* %x0, i8 %x1, <4 x i32> %x2, <4 x float> %x3, i32 2)
+  call void @llvm.x86.avx512.scattersiv4.sf(i8* %x0, i8 -1, <4 x i32> %x2, <4 x float> %x3, i32 4)
+  ret void
+}
+
+declare void @llvm.x86.avx512.scattersiv4.si(i8*, i8, <4 x i32>, <4 x i32>, i32)
+
+define void@test_int_x86_avx512_scattersiv4_si(i8* %x0, i8 %x1, <4 x i32> %x2, <4 x i32> %x3) {
+; CHECK-LABEL: test_int_x86_avx512_scattersiv4_si:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    kmovd %esi, %k1
+; CHECK-NEXT:    vpscatterdd %xmm1, (%rdi,%xmm0,2) {%k1}
+; CHECK-NEXT:    kxnorw %k0, %k0, %k1
+; CHECK-NEXT:    vpscatterdd %xmm1, (%rdi,%xmm0,4) {%k1}
+; CHECK-NEXT:    retq
+  call void @llvm.x86.avx512.scattersiv4.si(i8* %x0, i8 %x1, <4 x i32> %x2, <4 x i32> %x3, i32 2)
+  call void @llvm.x86.avx512.scattersiv4.si(i8* %x0, i8 -1, <4 x i32> %x2, <4 x i32> %x3, i32 4)
+  ret void
+}
+
+declare void @llvm.x86.avx512.scattersiv8.sf(i8*, i8, <8 x i32>, <8 x float>, i32)
+
+define void@test_int_x86_avx512_scattersiv8_sf(i8* %x0, i8 %x1, <8 x i32> %x2, <8 x float> %x3) {
+; CHECK-LABEL: test_int_x86_avx512_scattersiv8_sf:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    kmovd %esi, %k1
+; CHECK-NEXT:    vscatterdps %ymm1, (%rdi,%ymm0,2) {%k1}
+; CHECK-NEXT:    kxnorw %k0, %k0, %k1
+; CHECK-NEXT:    vscatterdps %ymm1, (%rdi,%ymm0,4) {%k1}
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+  call void @llvm.x86.avx512.scattersiv8.sf(i8* %x0, i8 %x1, <8 x i32> %x2, <8 x float> %x3, i32 2)
+  call void @llvm.x86.avx512.scattersiv8.sf(i8* %x0, i8 -1, <8 x i32> %x2, <8 x float> %x3, i32 4)
+  ret void
+}
+
+declare void @llvm.x86.avx512.scattersiv8.si(i8*, i8, <8 x i32>, <8 x i32>, i32)
+
+define void@test_int_x86_avx512_scattersiv8_si(i8* %x0, i8 %x1, <8 x i32> %x2, <8 x i32> %x3) {
+; CHECK-LABEL: test_int_x86_avx512_scattersiv8_si:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    kmovd %esi, %k1
+; CHECK-NEXT:    vpscatterdd %ymm1, (%rdi,%ymm0,2) {%k1}
+; CHECK-NEXT:    kxnorw %k0, %k0, %k1
+; CHECK-NEXT:    vpscatterdd %ymm1, (%rdi,%ymm0,4) {%k1}
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+  call void @llvm.x86.avx512.scattersiv8.si(i8* %x0, i8 %x1, <8 x i32> %x2, <8 x i32> %x3, i32 2)
+  call void @llvm.x86.avx512.scattersiv8.si(i8* %x0, i8 -1, <8 x i32> %x2, <8 x i32> %x3, i32 4)
+  ret void
+}
+
+define void @scatter_mask_test(i8* %x0, <8 x i32> %x2, <8 x i32> %x3) {
+; CHECK-LABEL: scatter_mask_test:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    kxnorw %k0, %k0, %k1
+; CHECK-NEXT:    vpscatterdd %ymm1, (%rdi,%ymm0,2) {%k1}
+; CHECK-NEXT:    kxorw %k0, %k0, %k1
+; CHECK-NEXT:    vpscatterdd %ymm1, (%rdi,%ymm0,4) {%k1}
+; CHECK-NEXT:    movb $1, %al
+; CHECK-NEXT:    kmovd %eax, %k1
+; CHECK-NEXT:    vpscatterdd %ymm1, (%rdi,%ymm0,2) {%k1}
+; CHECK-NEXT:    movb $96, %al
+; CHECK-NEXT:    kmovd %eax, %k1
+; CHECK-NEXT:    vpscatterdd %ymm1, (%rdi,%ymm0,4) {%k1}
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+  call void @llvm.x86.avx512.scattersiv8.si(i8* %x0, i8 -1, <8 x i32> %x2, <8 x i32> %x3, i32 2)
+  call void @llvm.x86.avx512.scattersiv8.si(i8* %x0, i8 0, <8 x i32> %x2, <8 x i32> %x3, i32 4)
+  call void @llvm.x86.avx512.scattersiv8.si(i8* %x0, i8 1, <8 x i32> %x2, <8 x i32> %x3, i32 2)
+  call void @llvm.x86.avx512.scattersiv8.si(i8* %x0, i8 96, <8 x i32> %x2, <8 x i32> %x3, i32 4)
+  ret void
+}
+
+define <16 x float> @gather_mask_test(<16 x i32> %ind, <16 x float> %src, i8* %base)  {
+; CHECK-LABEL: gather_mask_test:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    kxnorw %k0, %k0, %k1
+; CHECK-NEXT:    vxorps %xmm2, %xmm2, %xmm2
+; CHECK-NEXT:    vgatherdps (%rdi,%zmm0,4), %zmm2 {%k1}
+; CHECK-NEXT:    kxorw %k0, %k0, %k1
+; CHECK-NEXT:    vmovaps %zmm1, %zmm3
+; CHECK-NEXT:    vgatherdps (%rdi,%zmm0,4), %zmm3 {%k1}
+; CHECK-NEXT:    vaddps %zmm3, %zmm2, %zmm2
+; CHECK-NEXT:    movw $1, %ax
+; CHECK-NEXT:    kmovd %eax, %k1
+; CHECK-NEXT:    vmovaps %zmm1, %zmm3
+; CHECK-NEXT:    vgatherdps (%rdi,%zmm0,4), %zmm3 {%k1}
+; CHECK-NEXT:    movw $220, %ax
+; CHECK-NEXT:    kmovd %eax, %k1
+; CHECK-NEXT:    vgatherdps (%rdi,%zmm0,4), %zmm1 {%k1}
+; CHECK-NEXT:    vaddps %zmm3, %zmm1, %zmm0
+; CHECK-NEXT:    vaddps %zmm2, %zmm0, %zmm0
+; CHECK-NEXT:    retq
+  %res = call <16 x float> @llvm.x86.avx512.gather.dps.512 (<16 x float> %src, i8* %base, <16 x i32>%ind, i16 -1, i32 4)
+  %res1 = call <16 x float> @llvm.x86.avx512.gather.dps.512 (<16 x float> %src, i8* %base, <16 x i32>%ind, i16 0, i32 4)
+  %res2 = call <16 x float> @llvm.x86.avx512.gather.dps.512 (<16 x float> %src, i8* %base, <16 x i32>%ind, i16 1, i32 4)
+  %res3 = call <16 x float> @llvm.x86.avx512.gather.dps.512 (<16 x float> %src, i8* %base, <16 x i32>%ind, i16 220, i32 4)
+
+  %res4 = fadd <16 x float> %res, %res1
+  %res5 = fadd <16 x float> %res3, %res2
+  %res6 = fadd <16 x float> %res5, %res4
+  ret <16 x float> %res6
+}
diff --git a/test/CodeGen/X86/avx512-gather-scatter-intrin.ll b/test/CodeGen/X86/avx512-gather-scatter-intrin.ll
index 9502ec95d0928750d9f0e02d07116fd59c2bd3b8..376788cb79bf3248b16faf9b6969161dee41a478 100644
--- a/test/CodeGen/X86/avx512-gather-scatter-intrin.ll
+++ b/test/CodeGen/X86/avx512-gather-scatter-intrin.ll
@@ -1,19 +1,9 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-apple-darwin -mattr=+avx512f,+avx512bw,+avx512vl,+avx512dq | FileCheck %s
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512bw,+avx512vl,+avx512dq | FileCheck %s
 
-declare <16 x float> @llvm.x86.avx512.gather.dps.512 (<16 x float>, i8*, <16 x i32>, i16, i32)
-declare void @llvm.x86.avx512.scatter.dps.512 (i8*, i16, <16 x i32>, <16 x float>, i32)
-declare <8 x double> @llvm.x86.avx512.gather.dpd.512 (<8 x double>, i8*, <8 x i32>, i8, i32)
-declare void @llvm.x86.avx512.scatter.dpd.512 (i8*, i8, <8 x i32>, <8 x double>, i32)
-
-declare <8 x float> @llvm.x86.avx512.gather.qps.512 (<8 x float>, i8*, <8 x i64>, i8, i32)
-declare void @llvm.x86.avx512.scatter.qps.512 (i8*, i8, <8 x i64>, <8 x float>, i32)
-declare <8 x double> @llvm.x86.avx512.gather.qpd.512 (<8 x double>, i8*, <8 x i64>, i8, i32)
-declare void @llvm.x86.avx512.scatter.qpd.512 (i8*, i8, <8 x i64>, <8 x double>, i32)
-
-define void @gather_mask_dps(<16 x i32> %ind, <16 x float> %src, i16 %mask, i8* %base, i8* %stbuf)  {
+define void @gather_mask_dps(<16 x i32> %ind, <16 x float> %src, i16 %mask, i8* %base, i8* %stbuf) {
 ; CHECK-LABEL: gather_mask_dps:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    kmovd %edi, %k1
 ; CHECK-NEXT:    kmovq %k1, %k2
 ; CHECK-NEXT:    vgatherdps (%rsi,%zmm0,4), %zmm1 {%k2}
@@ -21,15 +11,16 @@ define void @gather_mask_dps(<16 x i32> %ind, <16 x float> %src, i16 %mask, i8*
 ; CHECK-NEXT:    vscatterdps %zmm1, (%rdx,%zmm0,4) {%k1}
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retq
-  %x = call <16 x float> @llvm.x86.avx512.gather.dps.512 (<16 x float> %src, i8* %base, <16 x i32>%ind, i16 %mask, i32 4)
+  %1 = bitcast i16 %mask to <16 x i1>
+  %x = call <16 x float> @llvm.x86.avx512.mask.gather.dps.512(<16 x float> %src, i8* %base, <16 x i32> %ind, <16 x i1> %1, i32 4)
   %ind2 = add <16 x i32> %ind, <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
-  call void @llvm.x86.avx512.scatter.dps.512 (i8* %stbuf, i16 %mask, <16 x i32>%ind2, <16 x float> %x, i32 4)
+  call void @llvm.x86.avx512.mask.scatter.dps.512(i8* %stbuf, <16 x i1> %1, <16 x i32> %ind2, <16 x float> %x, i32 4)
   ret void
 }
 
-define void @gather_mask_dpd(<8 x i32> %ind, <8 x double> %src, i8 %mask, i8* %base, i8* %stbuf)  {
+define void @gather_mask_dpd(<8 x i32> %ind, <8 x double> %src, i8 %mask, i8* %base, i8* %stbuf) {
 ; CHECK-LABEL: gather_mask_dpd:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    kmovd %edi, %k1
 ; CHECK-NEXT:    kmovq %k1, %k2
 ; CHECK-NEXT:    vgatherdpd (%rsi,%ymm0,4), %zmm1 {%k2}
@@ -37,15 +28,16 @@ define void @gather_mask_dpd(<8 x i32> %ind, <8 x double> %src, i8 %mask, i8* %b
 ; CHECK-NEXT:    vscatterdpd %zmm1, (%rdx,%ymm0,4) {%k1}
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retq
-  %x = call <8 x double> @llvm.x86.avx512.gather.dpd.512 (<8 x double> %src, i8* %base, <8 x i32>%ind, i8 %mask, i32 4)
+  %1 = bitcast i8 %mask to <8 x i1>
+  %x = call <8 x double> @llvm.x86.avx512.mask.gather.dpd.512(<8 x double> %src, i8* %base, <8 x i32> %ind, <8 x i1> %1, i32 4)
   %ind2 = add <8 x i32> %ind, <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
-  call void @llvm.x86.avx512.scatter.dpd.512 (i8* %stbuf, i8 %mask, <8 x i32>%ind2, <8 x double> %x, i32 4)
+  call void @llvm.x86.avx512.mask.scatter.dpd.512(i8* %stbuf, <8 x i1> %1, <8 x i32> %ind2, <8 x double> %x, i32 4)
   ret void
 }
 
-define void @gather_mask_qps(<8 x i64> %ind, <8 x float> %src, i8 %mask, i8* %base, i8* %stbuf)  {
+define void @gather_mask_qps(<8 x i64> %ind, <8 x float> %src, i8 %mask, i8* %base, i8* %stbuf) {
 ; CHECK-LABEL: gather_mask_qps:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    kmovd %edi, %k1
 ; CHECK-NEXT:    kmovq %k1, %k2
 ; CHECK-NEXT:    vgatherqps (%rsi,%zmm0,4), %ymm1 {%k2}
@@ -53,15 +45,16 @@ define void @gather_mask_qps(<8 x i64> %ind, <8 x float> %src, i8 %mask, i8* %ba
 ; CHECK-NEXT:    vscatterqps %ymm1, (%rdx,%zmm0,4) {%k1}
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retq
-  %x = call <8 x float> @llvm.x86.avx512.gather.qps.512 (<8 x float> %src, i8* %base, <8 x i64>%ind, i8 %mask, i32 4)
+  %1 = bitcast i8 %mask to <8 x i1>
+  %x = call <8 x float> @llvm.x86.avx512.mask.gather.qps.512(<8 x float> %src, i8* %base, <8 x i64> %ind, <8 x i1> %1, i32 4)
   %ind2 = add <8 x i64> %ind, <i64 0, i64 1, i64 2, i64 3, i64 0, i64 1, i64 2, i64 3>
-  call void @llvm.x86.avx512.scatter.qps.512 (i8* %stbuf, i8 %mask, <8 x i64>%ind2, <8 x float> %x, i32 4)
+  call void @llvm.x86.avx512.mask.scatter.qps.512(i8* %stbuf, <8 x i1> %1, <8 x i64> %ind2, <8 x float> %x, i32 4)
   ret void
 }
 
-define void @gather_mask_qpd(<8 x i64> %ind, <8 x double> %src, i8 %mask, i8* %base, i8* %stbuf)  {
+define void @gather_mask_qpd(<8 x i64> %ind, <8 x double> %src, i8 %mask, i8* %base, i8* %stbuf) {
 ; CHECK-LABEL: gather_mask_qpd:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    kmovd %edi, %k1
 ; CHECK-NEXT:    kmovq %k1, %k2
 ; CHECK-NEXT:    vgatherqpd (%rsi,%zmm0,4), %zmm1 {%k2}
@@ -69,27 +62,19 @@ define void @gather_mask_qpd(<8 x i64> %ind, <8 x double> %src, i8 %mask, i8* %b
 ; CHECK-NEXT:    vscatterqpd %zmm1, (%rdx,%zmm0,4) {%k1}
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retq
-  %x = call <8 x double> @llvm.x86.avx512.gather.qpd.512 (<8 x double> %src, i8* %base, <8 x i64>%ind, i8 %mask, i32 4)
+  %1 = bitcast i8 %mask to <8 x i1>
+  %x = call <8 x double> @llvm.x86.avx512.mask.gather.qpd.512(<8 x double> %src, i8* %base, <8 x i64> %ind, <8 x i1> %1, i32 4)
   %ind2 = add <8 x i64> %ind, <i64 0, i64 1, i64 2, i64 3, i64 0, i64 1, i64 2, i64 3>
-  call void @llvm.x86.avx512.scatter.qpd.512 (i8* %stbuf, i8 %mask, <8 x i64>%ind2, <8 x double> %x, i32 4)
+  call void @llvm.x86.avx512.mask.scatter.qpd.512(i8* %stbuf, <8 x i1> %1, <8 x i64> %ind2, <8 x double> %x, i32 4)
   ret void
 }
 ;;
 ;; Integer Gather/Scatter
 ;;
-declare <16 x i32> @llvm.x86.avx512.gather.dpi.512 (<16 x i32>, i8*, <16 x i32>, i16, i32)
-declare void @llvm.x86.avx512.scatter.dpi.512 (i8*, i16, <16 x i32>, <16 x i32>, i32)
-declare <8 x i64> @llvm.x86.avx512.gather.dpq.512 (<8 x i64>, i8*, <8 x i32>, i8, i32)
-declare void @llvm.x86.avx512.scatter.dpq.512 (i8*, i8, <8 x i32>, <8 x i64>, i32)
 
-declare <8 x i32> @llvm.x86.avx512.gather.qpi.512 (<8 x i32>, i8*, <8 x i64>, i8, i32)
-declare void @llvm.x86.avx512.scatter.qpi.512 (i8*, i8, <8 x i64>, <8 x i32>, i32)
-declare <8 x i64> @llvm.x86.avx512.gather.qpq.512 (<8 x i64>, i8*, <8 x i64>, i8, i32)
-declare void @llvm.x86.avx512.scatter.qpq.512 (i8*, i8, <8 x i64>, <8 x i64>, i32)
-
-define void @gather_mask_dd(<16 x i32> %ind, <16 x i32> %src, i16 %mask, i8* %base, i8* %stbuf)  {
+define void @gather_mask_dd(<16 x i32> %ind, <16 x i32> %src, i16 %mask, i8* %base, i8* %stbuf) {
 ; CHECK-LABEL: gather_mask_dd:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    kmovd %edi, %k1
 ; CHECK-NEXT:    kmovq %k1, %k2
 ; CHECK-NEXT:    vpgatherdd (%rsi,%zmm0,4), %zmm1 {%k2}
@@ -97,15 +82,16 @@ define void @gather_mask_dd(<16 x i32> %ind, <16 x i32> %src, i16 %mask, i8* %ba
 ; CHECK-NEXT:    vpscatterdd %zmm1, (%rdx,%zmm0,4) {%k1}
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retq
-  %x = call <16 x i32> @llvm.x86.avx512.gather.dpi.512 (<16 x i32> %src, i8* %base, <16 x i32>%ind, i16 %mask, i32 4)
+  %1 = bitcast i16 %mask to <16 x i1>
+  %x = call <16 x i32> @llvm.x86.avx512.mask.gather.dpi.512(<16 x i32> %src, i8* %base, <16 x i32> %ind, <16 x i1> %1, i32 4)
   %ind2 = add <16 x i32> %ind, <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
-  call void @llvm.x86.avx512.scatter.dpi.512 (i8* %stbuf, i16 %mask, <16 x i32>%ind2, <16 x i32> %x, i32 4)
+  call void @llvm.x86.avx512.mask.scatter.dpi.512(i8* %stbuf, <16 x i1> %1, <16 x i32> %ind2, <16 x i32> %x, i32 4)
   ret void
 }
 
-define void @gather_mask_qd(<8 x i64> %ind, <8 x i32> %src, i8 %mask, i8* %base, i8* %stbuf)  {
+define void @gather_mask_qd(<8 x i64> %ind, <8 x i32> %src, i8 %mask, i8* %base, i8* %stbuf) {
 ; CHECK-LABEL: gather_mask_qd:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    kmovd %edi, %k1
 ; CHECK-NEXT:    kmovq %k1, %k2
 ; CHECK-NEXT:    vpgatherqd (%rsi,%zmm0,4), %ymm1 {%k2}
@@ -113,15 +99,16 @@ define void @gather_mask_qd(<8 x i64> %ind, <8 x i32> %src, i8 %mask, i8* %base,
 ; CHECK-NEXT:    vpscatterqd %ymm1, (%rdx,%zmm0,4) {%k1}
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retq
-  %x = call <8 x i32> @llvm.x86.avx512.gather.qpi.512 (<8 x i32> %src, i8* %base, <8 x i64>%ind, i8 %mask, i32 4)
+  %1 = bitcast i8 %mask to <8 x i1>
+  %x = call <8 x i32> @llvm.x86.avx512.mask.gather.qpi.512(<8 x i32> %src, i8* %base, <8 x i64> %ind, <8 x i1> %1, i32 4)
   %ind2 = add <8 x i64> %ind, <i64 0, i64 1, i64 2, i64 3, i64 0, i64 1, i64 2, i64 3>
-  call void @llvm.x86.avx512.scatter.qpi.512 (i8* %stbuf, i8 %mask, <8 x i64>%ind2, <8 x i32> %x, i32 4)
+  call void @llvm.x86.avx512.mask.scatter.qpi.512(i8* %stbuf, <8 x i1> %1, <8 x i64> %ind2, <8 x i32> %x, i32 4)
   ret void
 }
 
-define void @gather_mask_qq(<8 x i64> %ind, <8 x i64> %src, i8 %mask, i8* %base, i8* %stbuf)  {
+define void @gather_mask_qq(<8 x i64> %ind, <8 x i64> %src, i8 %mask, i8* %base, i8* %stbuf) {
 ; CHECK-LABEL: gather_mask_qq:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    kmovd %edi, %k1
 ; CHECK-NEXT:    kmovq %k1, %k2
 ; CHECK-NEXT:    vpgatherqq (%rsi,%zmm0,4), %zmm1 {%k2}
@@ -129,15 +116,16 @@ define void @gather_mask_qq(<8 x i64> %ind, <8 x i64> %src, i8 %mask, i8* %base,
 ; CHECK-NEXT:    vpscatterqq %zmm1, (%rdx,%zmm0,4) {%k1}
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retq
-  %x = call <8 x i64> @llvm.x86.avx512.gather.qpq.512 (<8 x i64> %src, i8* %base, <8 x i64>%ind, i8 %mask, i32 4)
+  %1 = bitcast i8 %mask to <8 x i1>
+  %x = call <8 x i64> @llvm.x86.avx512.mask.gather.qpq.512(<8 x i64> %src, i8* %base, <8 x i64> %ind, <8 x i1> %1, i32 4)
   %ind2 = add <8 x i64> %ind, <i64 0, i64 1, i64 2, i64 3, i64 0, i64 1, i64 2, i64 3>
-  call void @llvm.x86.avx512.scatter.qpq.512 (i8* %stbuf, i8 %mask, <8 x i64>%ind2, <8 x i64> %x, i32 4)
+  call void @llvm.x86.avx512.mask.scatter.qpq.512(i8* %stbuf, <8 x i1> %1, <8 x i64> %ind2, <8 x i64> %x, i32 4)
   ret void
 }
 
-define void @gather_mask_dq(<8 x i32> %ind, <8 x i64> %src, i8 %mask, i8* %base, i8* %stbuf)  {
+define void @gather_mask_dq(<8 x i32> %ind, <8 x i64> %src, i8 %mask, i8* %base, i8* %stbuf) {
 ; CHECK-LABEL: gather_mask_dq:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    kmovd %edi, %k1
 ; CHECK-NEXT:    kmovq %k1, %k2
 ; CHECK-NEXT:    vpgatherdq (%rsi,%ymm0,4), %zmm1 {%k2}
@@ -145,126 +133,135 @@ define void @gather_mask_dq(<8 x i32> %ind, <8 x i64> %src, i8 %mask, i8* %base,
 ; CHECK-NEXT:    vpscatterdq %zmm1, (%rdx,%ymm0,4) {%k1}
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retq
-  %x = call <8 x i64> @llvm.x86.avx512.gather.dpq.512 (<8 x i64> %src, i8* %base, <8 x i32>%ind, i8 %mask, i32 4)
+  %1 = bitcast i8 %mask to <8 x i1>
+  %x = call <8 x i64> @llvm.x86.avx512.mask.gather.dpq.512(<8 x i64> %src, i8* %base, <8 x i32> %ind, <8 x i1> %1, i32 4)
   %ind2 = add <8 x i32> %ind, <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
-  call void @llvm.x86.avx512.scatter.dpq.512 (i8* %stbuf, i8 %mask, <8 x i32>%ind2, <8 x i64> %x, i32 4)
+  call void @llvm.x86.avx512.mask.scatter.dpq.512(i8* %stbuf, <8 x i1> %1, <8 x i32> %ind2, <8 x i64> %x, i32 4)
   ret void
 }
 
-define void @gather_mask_dpd_execdomain(<8 x i32> %ind, <8 x double> %src, i8 %mask, i8* %base, <8 x double>* %stbuf)  {
+define void @gather_mask_dpd_execdomain(<8 x i32> %ind, <8 x double> %src, i8 %mask, i8* %base, <8 x double>* %stbuf) {
 ; CHECK-LABEL: gather_mask_dpd_execdomain:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    kmovd %edi, %k1
 ; CHECK-NEXT:    vgatherdpd (%rsi,%ymm0,4), %zmm1 {%k1}
 ; CHECK-NEXT:    vmovapd %zmm1, (%rdx)
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retq
-  %x = call <8 x double> @llvm.x86.avx512.gather.dpd.512 (<8 x double> %src, i8* %base, <8 x i32>%ind, i8 %mask, i32 4)
+  %1 = bitcast i8 %mask to <8 x i1>
+  %x = call <8 x double> @llvm.x86.avx512.mask.gather.dpd.512(<8 x double> %src, i8* %base, <8 x i32> %ind, <8 x i1> %1, i32 4)
   store <8 x double> %x, <8 x double>* %stbuf
   ret void
 }
 
-define void @gather_mask_qpd_execdomain(<8 x i64> %ind, <8 x double> %src, i8 %mask, i8* %base, <8 x double>* %stbuf)  {
+define void @gather_mask_qpd_execdomain(<8 x i64> %ind, <8 x double> %src, i8 %mask, i8* %base, <8 x double>* %stbuf) {
 ; CHECK-LABEL: gather_mask_qpd_execdomain:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    kmovd %edi, %k1
 ; CHECK-NEXT:    vgatherqpd (%rsi,%zmm0,4), %zmm1 {%k1}
 ; CHECK-NEXT:    vmovapd %zmm1, (%rdx)
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retq
-  %x = call <8 x double> @llvm.x86.avx512.gather.qpd.512 (<8 x double> %src, i8* %base, <8 x i64>%ind, i8 %mask, i32 4)
+  %1 = bitcast i8 %mask to <8 x i1>
+  %x = call <8 x double> @llvm.x86.avx512.mask.gather.qpd.512(<8 x double> %src, i8* %base, <8 x i64> %ind, <8 x i1> %1, i32 4)
   store <8 x double> %x, <8 x double>* %stbuf
   ret void
 }
 
-define <16 x float> @gather_mask_dps_execdomain(<16 x i32> %ind, <16 x float> %src, i16 %mask, i8* %base)  {
+define <16 x float> @gather_mask_dps_execdomain(<16 x i32> %ind, <16 x float> %src, i16 %mask, i8* %base) {
 ; CHECK-LABEL: gather_mask_dps_execdomain:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    kmovd %edi, %k1
 ; CHECK-NEXT:    vgatherdps (%rsi,%zmm0,4), %zmm1 {%k1}
 ; CHECK-NEXT:    vmovaps %zmm1, %zmm0
 ; CHECK-NEXT:    retq
-  %res = call <16 x float> @llvm.x86.avx512.gather.dps.512 (<16 x float> %src, i8* %base, <16 x i32>%ind, i16 %mask, i32 4)
-  ret <16 x float> %res;
+  %1 = bitcast i16 %mask to <16 x i1>
+  %res = call <16 x float> @llvm.x86.avx512.mask.gather.dps.512(<16 x float> %src, i8* %base, <16 x i32> %ind, <16 x i1> %1, i32 4)
+  ret <16 x float> %res
 }
 
-define <8 x float> @gather_mask_qps_execdomain(<8 x i64> %ind, <8 x float> %src, i8 %mask, i8* %base)  {
+define <8 x float> @gather_mask_qps_execdomain(<8 x i64> %ind, <8 x float> %src, i8 %mask, i8* %base) {
 ; CHECK-LABEL: gather_mask_qps_execdomain:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    kmovd %edi, %k1
 ; CHECK-NEXT:    vgatherqps (%rsi,%zmm0,4), %ymm1 {%k1}
 ; CHECK-NEXT:    vmovaps %ymm1, %ymm0
 ; CHECK-NEXT:    retq
-  %res = call <8 x float> @llvm.x86.avx512.gather.qps.512 (<8 x float> %src, i8* %base, <8 x i64>%ind, i8 %mask, i32 4)
-  ret <8 x float> %res;
+  %1 = bitcast i8 %mask to <8 x i1>
+  %res = call <8 x float> @llvm.x86.avx512.mask.gather.qps.512(<8 x float> %src, i8* %base, <8 x i64> %ind, <8 x i1> %1, i32 4)
+  ret <8 x float> %res
 }
 
-define void @scatter_mask_dpd_execdomain(<8 x i32> %ind, <8 x double>* %src, i8 %mask, i8* %base, i8* %stbuf)  {
+define void @scatter_mask_dpd_execdomain(<8 x i32> %ind, <8 x double>* %src, i8 %mask, i8* %base, i8* %stbuf) {
 ; CHECK-LABEL: scatter_mask_dpd_execdomain:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    kmovd %esi, %k1
 ; CHECK-NEXT:    vmovapd (%rdi), %zmm1
 ; CHECK-NEXT:    vscatterdpd %zmm1, (%rcx,%ymm0,4) {%k1}
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retq
+  %1 = bitcast i8 %mask to <8 x i1>
   %x = load <8 x double>, <8 x double>* %src, align 64
-  call void @llvm.x86.avx512.scatter.dpd.512 (i8* %stbuf, i8 %mask, <8 x i32>%ind, <8 x double> %x, i32 4)
+  call void @llvm.x86.avx512.mask.scatter.dpd.512(i8* %stbuf, <8 x i1> %1, <8 x i32>%ind, <8 x double> %x, i32 4)
   ret void
 }
 
 define void @scatter_mask_qpd_execdomain(<8 x i64> %ind, <8 x double>* %src, i8 %mask, i8* %base, i8* %stbuf)  {
 ; CHECK-LABEL: scatter_mask_qpd_execdomain:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    kmovd %esi, %k1
 ; CHECK-NEXT:    vmovapd (%rdi), %zmm1
 ; CHECK-NEXT:    vscatterqpd %zmm1, (%rcx,%zmm0,4) {%k1}
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retq
+  %1 = bitcast i8 %mask to <8 x i1>
   %x = load <8 x double>, <8 x double>* %src, align 64
-  call void @llvm.x86.avx512.scatter.qpd.512 (i8* %stbuf, i8 %mask, <8 x i64>%ind, <8 x double> %x, i32 4)
+  call void @llvm.x86.avx512.mask.scatter.qpd.512(i8* %stbuf, <8 x i1> %1, <8 x i64>%ind, <8 x double> %x, i32 4)
   ret void
 }
 
 define void @scatter_mask_dps_execdomain(<16 x i32> %ind, <16 x float>* %src, i16 %mask, i8* %base, i8* %stbuf)  {
 ; CHECK-LABEL: scatter_mask_dps_execdomain:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    kmovd %esi, %k1
 ; CHECK-NEXT:    vmovaps (%rdi), %zmm1
 ; CHECK-NEXT:    vscatterdps %zmm1, (%rcx,%zmm0,4) {%k1}
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retq
+  %1 = bitcast i16 %mask to <16 x i1>
   %x = load <16 x float>, <16 x float>* %src, align 64
-  call void @llvm.x86.avx512.scatter.dps.512 (i8* %stbuf, i16 %mask, <16 x i32>%ind, <16 x float> %x, i32 4)
+  call void @llvm.x86.avx512.mask.scatter.dps.512(i8* %stbuf, <16 x i1> %1, <16 x i32>%ind, <16 x float> %x, i32 4)
   ret void
 }
 
 define void @scatter_mask_qps_execdomain(<8 x i64> %ind, <8 x float>* %src, i8 %mask, i8* %base, i8* %stbuf)  {
 ; CHECK-LABEL: scatter_mask_qps_execdomain:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    kmovd %esi, %k1
 ; CHECK-NEXT:    vmovaps (%rdi), %ymm1
 ; CHECK-NEXT:    vscatterqps %ymm1, (%rcx,%zmm0,4) {%k1}
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retq
+  %1 = bitcast i8 %mask to <8 x i1>
   %x = load <8 x float>, <8 x float>* %src, align 32
-  call void @llvm.x86.avx512.scatter.qps.512 (i8* %stbuf, i8 %mask, <8 x i64>%ind, <8 x float> %x, i32 4)
+  call void @llvm.x86.avx512.mask.scatter.qps.512(i8* %stbuf, <8 x i1> %1, <8 x i64>%ind, <8 x float> %x, i32 4)
   ret void
 }
 
-define void @gather_qps(<8 x i64> %ind, <8 x float> %src, i8* %base, i8* %stbuf)  {
+define void @gather_qps(<8 x i64> %ind, <8 x float> %src, i8* %base, i8* %stbuf) {
 ; CHECK-LABEL: gather_qps:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    kxnorw %k0, %k0, %k1
+; CHECK-NEXT:    vxorps %xmm1, %xmm1, %xmm1
 ; CHECK-NEXT:    kxnorw %k0, %k0, %k2
 ; CHECK-NEXT:    vgatherqps (%rdi,%zmm0,4), %ymm1 {%k2}
 ; CHECK-NEXT:    vpaddq {{.*}}(%rip), %zmm0, %zmm0
 ; CHECK-NEXT:    vscatterqps %ymm1, (%rsi,%zmm0,4) {%k1}
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retq
-  %x = call <8 x float> @llvm.x86.avx512.gather.qps.512 (<8 x float> %src, i8* %base, <8 x i64>%ind, i8 -1, i32 4)
+  %x = call <8 x float> @llvm.x86.avx512.mask.gather.qps.512(<8 x float> %src, i8* %base, <8 x i64> %ind, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, i32 4)
   %ind2 = add <8 x i64> %ind, <i64 0, i64 1, i64 2, i64 3, i64 0, i64 1, i64 2, i64 3>
-  call void @llvm.x86.avx512.scatter.qps.512 (i8* %stbuf, i8 -1, <8 x i64>%ind2, <8 x float> %x, i32 4)
+  call void @llvm.x86.avx512.mask.scatter.qps.512(i8* %stbuf, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <8 x i64> %ind2, <8 x float> %x, i32 4)
   ret void
 }
 
@@ -272,7 +269,7 @@ declare  void @llvm.x86.avx512.gatherpf.qps.512(i8, <8 x i64>, i8* , i32, i32);
 declare  void @llvm.x86.avx512.scatterpf.qps.512(i8, <8 x i64>, i8* , i32, i32);
 define void @prefetch(<8 x i64> %ind, i8* %base) {
 ; CHECK-LABEL: prefetch:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    kxnorw %k0, %k0, %k1
 ; CHECK-NEXT:    vgatherpf0qps (%rdi,%zmm0,4) {%k1}
 ; CHECK-NEXT:    kxorw %k0, %k0, %k1
@@ -292,11 +289,9 @@ define void @prefetch(<8 x i64> %ind, i8* %base) {
   ret void
 }
 
-declare <2 x double> @llvm.x86.avx512.gather3div2.df(<2 x double>, i8*, <2 x i64>, i8, i32)
-
-define <2 x double>@test_int_x86_avx512_gather3div2_df(<2 x double> %x0, i8* %x1, <2 x i64> %x2, i8 %x3) {
-; CHECK-LABEL: test_int_x86_avx512_gather3div2_df:
-; CHECK:       ## %bb.0:
+define <2 x double> @test_int_x86_avx512_mask_gather3div2_df(<2 x double> %x0, i8* %x1, <2 x i64> %x2, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_mask_gather3div2_df:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    kmovd %esi, %k1
 ; CHECK-NEXT:    vgatherqpd (%rdi,%xmm1,4), %xmm0 {%k1}
 ; CHECK-NEXT:    kxnorw %k0, %k0, %k1
@@ -304,32 +299,34 @@ define <2 x double>@test_int_x86_avx512_gather3div2_df(<2 x double> %x0, i8* %x1
 ; CHECK-NEXT:    vgatherqpd (%rdi,%xmm1,2), %xmm2 {%k1}
 ; CHECK-NEXT:    vaddpd %xmm2, %xmm0, %xmm0
 ; CHECK-NEXT:    retq
-  %res = call <2 x double> @llvm.x86.avx512.gather3div2.df(<2 x double> %x0, i8* %x1, <2 x i64> %x2, i8 %x3, i32 4)
-  %res1 = call <2 x double> @llvm.x86.avx512.gather3div2.df(<2 x double> %x0, i8* %x1, <2 x i64> %x2, i8 -1, i32 2)
+  %1 = bitcast i8 %x3 to <8 x i1>
+  %extract = shufflevector <8 x i1> %1, <8 x i1> %1, <2 x i32> <i32 0, i32 1>
+  %res = call <2 x double> @llvm.x86.avx512.mask.gather3div2.df(<2 x double> %x0, i8* %x1, <2 x i64> %x2, <2 x i1> %extract, i32 4)
+  %res1 = call <2 x double> @llvm.x86.avx512.mask.gather3div2.df(<2 x double> %x0, i8* %x1, <2 x i64> %x2, <2 x i1> <i1 true, i1 true>, i32 2)
   %res2 = fadd <2 x double> %res, %res1
   ret <2 x double> %res2
 }
 
-declare <2 x i64> @llvm.x86.avx512.gather3div2.di(<2 x i64>, i8*, <2 x i64>, i8, i32)
-
-define <2 x i64>@test_int_x86_avx512_gather3div2_di(<2 x i64> %x0, i8* %x1, <2 x i64> %x2, i8 %x3) {
-; CHECK-LABEL: test_int_x86_avx512_gather3div2_di:
-; CHECK:       ## %bb.0:
+define <2 x i64> @test_int_x86_avx512_mask_gather3div2_di(<2 x i64> %x0, i8* %x1, <2 x i64> %x2, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_mask_gather3div2_di:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    kmovd %esi, %k1
 ; CHECK-NEXT:    vpgatherqq (%rdi,%xmm1,8), %xmm0 {%k1}
 ; CHECK-NEXT:    vpaddq %xmm0, %xmm0, %xmm0
 ; CHECK-NEXT:    retq
-  %res = call <2 x i64> @llvm.x86.avx512.gather3div2.di(<2 x i64> %x0, i8* %x1, <2 x i64> %x2, i8 %x3, i32 8)
-  %res1 = call <2 x i64> @llvm.x86.avx512.gather3div2.di(<2 x i64> %x0, i8* %x1, <2 x i64> %x2, i8 %x3, i32 8)
+  %1 = bitcast i8 %x3 to <8 x i1>
+  %extract1 = shufflevector <8 x i1> %1, <8 x i1> %1, <2 x i32> <i32 0, i32 1>
+  %res = call <2 x i64> @llvm.x86.avx512.mask.gather3div2.di(<2 x i64> %x0, i8* %x1, <2 x i64> %x2, <2 x i1> %extract1, i32 8)
+  %2 = bitcast i8 %x3 to <8 x i1>
+  %extract = shufflevector <8 x i1> %2, <8 x i1> %2, <2 x i32> <i32 0, i32 1>
+  %res1 = call <2 x i64> @llvm.x86.avx512.mask.gather3div2.di(<2 x i64> %x0, i8* %x1, <2 x i64> %x2, <2 x i1> %extract, i32 8)
   %res2 = add <2 x i64> %res, %res1
   ret <2 x i64> %res2
 }
 
-declare <4 x double> @llvm.x86.avx512.gather3div4.df(<4 x double>, i8*, <4 x i64>, i8, i32)
-
-define <4 x double>@test_int_x86_avx512_gather3div4_df(<4 x double> %x0, i8* %x1, <4 x i64> %x2, i8 %x3) {
-; CHECK-LABEL: test_int_x86_avx512_gather3div4_df:
-; CHECK:       ## %bb.0:
+define <4 x double> @test_int_x86_avx512_mask_gather3div4_df(<4 x double> %x0, i8* %x1, <4 x i64> %x2, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_mask_gather3div4_df:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    kmovd %esi, %k1
 ; CHECK-NEXT:    vgatherqpd (%rdi,%ymm1,4), %ymm0 {%k1}
 ; CHECK-NEXT:    kxnorw %k0, %k0, %k1
@@ -337,17 +334,17 @@ define <4 x double>@test_int_x86_avx512_gather3div4_df(<4 x double> %x0, i8* %x1
 ; CHECK-NEXT:    vgatherqpd (%rdi,%ymm1,2), %ymm2 {%k1}
 ; CHECK-NEXT:    vaddpd %ymm2, %ymm0, %ymm0
 ; CHECK-NEXT:    retq
-  %res = call <4 x double> @llvm.x86.avx512.gather3div4.df(<4 x double> %x0, i8* %x1, <4 x i64> %x2, i8 %x3, i32 4)
-  %res1 = call <4 x double> @llvm.x86.avx512.gather3div4.df(<4 x double> %x0, i8* %x1, <4 x i64> %x2, i8 -1, i32 2)
+  %1 = bitcast i8 %x3 to <8 x i1>
+  %extract = shufflevector <8 x i1> %1, <8 x i1> %1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %res = call <4 x double> @llvm.x86.avx512.mask.gather3div4.df(<4 x double> %x0, i8* %x1, <4 x i64> %x2, <4 x i1> %extract, i32 4)
+  %res1 = call <4 x double> @llvm.x86.avx512.mask.gather3div4.df(<4 x double> %x0, i8* %x1, <4 x i64> %x2, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, i32 2)
   %res2 = fadd <4 x double> %res, %res1
   ret <4 x double> %res2
 }
 
-declare <4 x i64> @llvm.x86.avx512.gather3div4.di(<4 x i64>, i8*, <4 x i64>, i8, i32)
-
-define <4 x i64>@test_int_x86_avx512_gather3div4_di(<4 x i64> %x0, i8* %x1, <4 x i64> %x2, i8 %x3) {
-; CHECK-LABEL: test_int_x86_avx512_gather3div4_di:
-; CHECK:       ## %bb.0:
+define <4 x i64> @test_int_x86_avx512_mask_gather3div4_di(<4 x i64> %x0, i8* %x1, <4 x i64> %x2, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_mask_gather3div4_di:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    kmovd %esi, %k1
 ; CHECK-NEXT:    vpgatherqq (%rdi,%ymm1,8), %ymm0 {%k1}
 ; CHECK-NEXT:    kxnorw %k0, %k0, %k1
@@ -355,17 +352,17 @@ define <4 x i64>@test_int_x86_avx512_gather3div4_di(<4 x i64> %x0, i8* %x1, <4 x
 ; CHECK-NEXT:    vpgatherqq (%rdi,%ymm1,8), %ymm2 {%k1}
 ; CHECK-NEXT:    vpaddq %ymm2, %ymm0, %ymm0
 ; CHECK-NEXT:    retq
-  %res = call <4 x i64> @llvm.x86.avx512.gather3div4.di(<4 x i64> %x0, i8* %x1, <4 x i64> %x2, i8 %x3, i32 8)
-  %res1 = call <4 x i64> @llvm.x86.avx512.gather3div4.di(<4 x i64> %x0, i8* %x1, <4 x i64> %x2, i8 -1, i32 8)
+  %1 = bitcast i8 %x3 to <8 x i1>
+  %extract = shufflevector <8 x i1> %1, <8 x i1> %1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %res = call <4 x i64> @llvm.x86.avx512.mask.gather3div4.di(<4 x i64> %x0, i8* %x1, <4 x i64> %x2, <4 x i1> %extract, i32 8)
+  %res1 = call <4 x i64> @llvm.x86.avx512.mask.gather3div4.di(<4 x i64> %x0, i8* %x1, <4 x i64> %x2, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, i32 8)
   %res2 = add <4 x i64> %res, %res1
   ret <4 x i64> %res2
 }
 
-declare <4 x float> @llvm.x86.avx512.gather3div4.sf(<4 x float>, i8*, <2 x i64>, i8, i32)
-
-define <4 x float>@test_int_x86_avx512_gather3div4_sf(<4 x float> %x0, i8* %x1, <2 x i64> %x2, i8 %x3) {
-; CHECK-LABEL: test_int_x86_avx512_gather3div4_sf:
-; CHECK:       ## %bb.0:
+define <4 x float> @test_int_x86_avx512_mask_gather3div4_sf(<4 x float> %x0, i8* %x1, <2 x i64> %x2, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_mask_gather3div4_sf:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    kmovd %esi, %k1
 ; CHECK-NEXT:    vgatherqps (%rdi,%xmm1,4), %xmm0 {%k1}
 ; CHECK-NEXT:    kxnorw %k0, %k0, %k1
@@ -373,35 +370,35 @@ define <4 x float>@test_int_x86_avx512_gather3div4_sf(<4 x float> %x0, i8* %x1,
 ; CHECK-NEXT:    vgatherqps (%rdi,%xmm1,2), %xmm2 {%k1}
 ; CHECK-NEXT:    vaddps %xmm2, %xmm0, %xmm0
 ; CHECK-NEXT:    retq
-  %res = call <4 x float> @llvm.x86.avx512.gather3div4.sf(<4 x float> %x0, i8* %x1, <2 x i64> %x2, i8 %x3, i32 4)
-  %res1 = call <4 x float> @llvm.x86.avx512.gather3div4.sf(<4 x float> %x0, i8* %x1, <2 x i64> %x2, i8 -1, i32 2)
+  %1 = bitcast i8 %x3 to <8 x i1>
+  %extract = shufflevector <8 x i1> %1, <8 x i1> %1, <2 x i32> <i32 0, i32 1>
+  %res = call <4 x float> @llvm.x86.avx512.mask.gather3div4.sf(<4 x float> %x0, i8* %x1, <2 x i64> %x2, <2 x i1> %extract, i32 4)
+  %res1 = call <4 x float> @llvm.x86.avx512.mask.gather3div4.sf(<4 x float> %x0, i8* %x1, <2 x i64> %x2, <2 x i1> <i1 true, i1 true>, i32 2)
   %res2 = fadd <4 x float> %res, %res1
   ret <4 x float> %res2
 }
 
-declare <4 x i32> @llvm.x86.avx512.gather3div4.si(<4 x i32>, i8*, <2 x i64>, i8, i32)
-
-define <4 x i32>@test_int_x86_avx512_gather3div4_si(<4 x i32> %x0, i8* %x1, <2 x i64> %x2, i8 %x3) {
-; CHECK-LABEL: test_int_x86_avx512_gather3div4_si:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovd %esi, %k1
-; CHECK-NEXT:    kxnorw %k0, %k0, %k2
+define <4 x i32> @test_int_x86_avx512_mask_gather3div4_si(<4 x i32> %x0, i8* %x1, <2 x i64> %x2, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_mask_gather3div4_si:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    kxnorw %k0, %k0, %k1
 ; CHECK-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; CHECK-NEXT:    vpgatherqd (%rdi,%xmm1,4), %xmm2 {%k2}
+; CHECK-NEXT:    vpgatherqd (%rdi,%xmm1,4), %xmm2 {%k1}
+; CHECK-NEXT:    kmovd %esi, %k1
 ; CHECK-NEXT:    vpgatherqd (%rdi,%xmm1,4), %xmm0 {%k1}
 ; CHECK-NEXT:    vpaddd %xmm0, %xmm2, %xmm0
 ; CHECK-NEXT:    retq
-  %res = call <4 x i32> @llvm.x86.avx512.gather3div4.si(<4 x i32> %x0, i8* %x1, <2 x i64> %x2, i8 -1, i32 4)
-  %res1 = call <4 x i32> @llvm.x86.avx512.gather3div4.si(<4 x i32> %x0, i8* %x1, <2 x i64> %x2, i8 %x3, i32 4)
+  %res = call <4 x i32> @llvm.x86.avx512.mask.gather3div4.si(<4 x i32> %x0, i8* %x1, <2 x i64> %x2, <2 x i1> <i1 true, i1 true>, i32 4)
+  %1 = bitcast i8 %x3 to <8 x i1>
+  %extract = shufflevector <8 x i1> %1, <8 x i1> %1, <2 x i32> <i32 0, i32 1>
+  %res1 = call <4 x i32> @llvm.x86.avx512.mask.gather3div4.si(<4 x i32> %x0, i8* %x1, <2 x i64> %x2, <2 x i1> %extract, i32 4)
   %res2 = add <4 x i32> %res, %res1
   ret <4 x i32> %res2
 }
 
-declare <4 x float> @llvm.x86.avx512.gather3div8.sf(<4 x float>, i8*, <4 x i64>, i8, i32)
-
-define <4 x float>@test_int_x86_avx512_gather3div8_sf(<4 x float> %x0, i8* %x1, <4 x i64> %x2, i8 %x3) {
-; CHECK-LABEL: test_int_x86_avx512_gather3div8_sf:
-; CHECK:       ## %bb.0:
+define <4 x float> @test_int_x86_avx512_mask_gather3div8_sf(<4 x float> %x0, i8* %x1, <4 x i64> %x2, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_mask_gather3div8_sf:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    kmovd %esi, %k1
 ; CHECK-NEXT:    vgatherqps (%rdi,%ymm1,4), %xmm0 {%k1}
 ; CHECK-NEXT:    kxnorw %k0, %k0, %k1
@@ -410,17 +407,17 @@ define <4 x float>@test_int_x86_avx512_gather3div8_sf(<4 x float> %x0, i8* %x1,
 ; CHECK-NEXT:    vaddps %xmm2, %xmm0, %xmm0
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retq
-  %res = call <4 x float> @llvm.x86.avx512.gather3div8.sf(<4 x float> %x0, i8* %x1, <4 x i64> %x2, i8 %x3, i32 4)
-  %res1 = call <4 x float> @llvm.x86.avx512.gather3div8.sf(<4 x float> %x0, i8* %x1, <4 x i64> %x2, i8 -1, i32 2)
+  %1 = bitcast i8 %x3 to <8 x i1>
+  %extract = shufflevector <8 x i1> %1, <8 x i1> %1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %res = call <4 x float> @llvm.x86.avx512.mask.gather3div8.sf(<4 x float> %x0, i8* %x1, <4 x i64> %x2, <4 x i1> %extract, i32 4)
+  %res1 = call <4 x float> @llvm.x86.avx512.mask.gather3div8.sf(<4 x float> %x0, i8* %x1, <4 x i64> %x2, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, i32 2)
   %res2 = fadd <4 x float> %res, %res1
   ret <4 x float> %res2
 }
 
-declare <4 x i32> @llvm.x86.avx512.gather3div8.si(<4 x i32>, i8*, <4 x i64>, i8, i32)
-
-define <4 x i32>@test_int_x86_avx512_gather3div8_si(<4 x i32> %x0, i8* %x1, <4 x i64> %x2, i8 %x3) {
-; CHECK-LABEL: test_int_x86_avx512_gather3div8_si:
-; CHECK:       ## %bb.0:
+define <4 x i32> @test_int_x86_avx512_mask_gather3div8_si(<4 x i32> %x0, i8* %x1, <4 x i64> %x2, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_mask_gather3div8_si:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    kmovd %esi, %k1
 ; CHECK-NEXT:    vmovdqa %xmm0, %xmm2
 ; CHECK-NEXT:    kmovq %k1, %k2
@@ -429,17 +426,19 @@ define <4 x i32>@test_int_x86_avx512_gather3div8_si(<4 x i32> %x0, i8* %x1, <4 x
 ; CHECK-NEXT:    vpaddd %xmm0, %xmm2, %xmm0
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retq
-  %res = call <4 x i32> @llvm.x86.avx512.gather3div8.si(<4 x i32> %x0, i8* %x1, <4 x i64> %x2, i8 %x3, i32 4)
-  %res1 = call <4 x i32> @llvm.x86.avx512.gather3div8.si(<4 x i32> %x0, i8* %x1, <4 x i64> %x2, i8 %x3, i32 2)
+  %1 = bitcast i8 %x3 to <8 x i1>
+  %extract1 = shufflevector <8 x i1> %1, <8 x i1> %1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %res = call <4 x i32> @llvm.x86.avx512.mask.gather3div8.si(<4 x i32> %x0, i8* %x1, <4 x i64> %x2, <4 x i1> %extract1, i32 4)
+  %2 = bitcast i8 %x3 to <8 x i1>
+  %extract = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %res1 = call <4 x i32> @llvm.x86.avx512.mask.gather3div8.si(<4 x i32> %x0, i8* %x1, <4 x i64> %x2, <4 x i1> %extract, i32 2)
   %res2 = add <4 x i32> %res, %res1
   ret <4 x i32> %res2
 }
 
-declare <2 x double> @llvm.x86.avx512.gather3siv2.df(<2 x double>, i8*, <4 x i32>, i8, i32)
-
-define <2 x double>@test_int_x86_avx512_gather3siv2_df(<2 x double> %x0, i8* %x1, <4 x i32> %x2, i8 %x3) {
-; CHECK-LABEL: test_int_x86_avx512_gather3siv2_df:
-; CHECK:       ## %bb.0:
+define <2 x double> @test_int_x86_avx512_mask_gather3siv2_df(<2 x double> %x0, i8* %x1, <4 x i32> %x2, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_mask_gather3siv2_df:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    kmovd %esi, %k1
 ; CHECK-NEXT:    vgatherdpd (%rdi,%xmm1,4), %xmm0 {%k1}
 ; CHECK-NEXT:    kxnorw %k0, %k0, %k1
@@ -447,32 +446,34 @@ define <2 x double>@test_int_x86_avx512_gather3siv2_df(<2 x double> %x0, i8* %x1
 ; CHECK-NEXT:    vgatherdpd (%rdi,%xmm1,2), %xmm2 {%k1}
 ; CHECK-NEXT:    vaddpd %xmm2, %xmm0, %xmm0
 ; CHECK-NEXT:    retq
-  %res = call <2 x double> @llvm.x86.avx512.gather3siv2.df(<2 x double> %x0, i8* %x1, <4 x i32> %x2, i8 %x3, i32 4)
-  %res1 = call <2 x double> @llvm.x86.avx512.gather3siv2.df(<2 x double> %x0, i8* %x1, <4 x i32> %x2, i8 -1, i32 2)
+  %1 = bitcast i8 %x3 to <8 x i1>
+  %extract = shufflevector <8 x i1> %1, <8 x i1> %1, <2 x i32> <i32 0, i32 1>
+  %res = call <2 x double> @llvm.x86.avx512.mask.gather3siv2.df(<2 x double> %x0, i8* %x1, <4 x i32> %x2, <2 x i1> %extract, i32 4)
+  %res1 = call <2 x double> @llvm.x86.avx512.mask.gather3siv2.df(<2 x double> %x0, i8* %x1, <4 x i32> %x2, <2 x i1> <i1 true, i1 true>, i32 2)
   %res2 = fadd <2 x double> %res, %res1
   ret <2 x double> %res2
 }
 
-declare <2 x i64> @llvm.x86.avx512.gather3siv2.di(<2 x i64>, i8*, <4 x i32>, i8, i32)
-
-define <2 x i64>@test_int_x86_avx512_gather3siv2_di(<2 x i64> %x0, i8* %x1, <4 x i32> %x2, i8 %x3) {
-; CHECK-LABEL: test_int_x86_avx512_gather3siv2_di:
-; CHECK:       ## %bb.0:
+define <2 x i64> @test_int_x86_avx512_mask_gather3siv2_di(<2 x i64> %x0, i8* %x1, <4 x i32> %x2, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_mask_gather3siv2_di:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    kmovd %esi, %k1
 ; CHECK-NEXT:    vpgatherdq (%rdi,%xmm1,8), %xmm0 {%k1}
 ; CHECK-NEXT:    vpaddq %xmm0, %xmm0, %xmm0
 ; CHECK-NEXT:    retq
-  %res = call <2 x i64> @llvm.x86.avx512.gather3siv2.di(<2 x i64> %x0, i8* %x1, <4 x i32> %x2, i8 %x3, i32 8)
-  %res1 = call <2 x i64> @llvm.x86.avx512.gather3siv2.di(<2 x i64> %x0, i8* %x1, <4 x i32> %x2, i8 %x3, i32 8)
+  %1 = bitcast i8 %x3 to <8 x i1>
+  %extract1 = shufflevector <8 x i1> %1, <8 x i1> %1, <2 x i32> <i32 0, i32 1>
+  %res = call <2 x i64> @llvm.x86.avx512.mask.gather3siv2.di(<2 x i64> %x0, i8* %x1, <4 x i32> %x2, <2 x i1> %extract1, i32 8)
+  %2 = bitcast i8 %x3 to <8 x i1>
+  %extract = shufflevector <8 x i1> %2, <8 x i1> %2, <2 x i32> <i32 0, i32 1>
+  %res1 = call <2 x i64> @llvm.x86.avx512.mask.gather3siv2.di(<2 x i64> %x0, i8* %x1, <4 x i32> %x2, <2 x i1> %extract, i32 8)
   %res2 = add <2 x i64> %res, %res1
   ret <2 x i64> %res2
 }
 
-declare <4 x double> @llvm.x86.avx512.gather3siv4.df(<4 x double>, i8*, <4 x i32>, i8, i32)
-
-define <4 x double>@test_int_x86_avx512_gather3siv4_df(<4 x double> %x0, i8* %x1, <4 x i32> %x2, i8 %x3) {
-; CHECK-LABEL: test_int_x86_avx512_gather3siv4_df:
-; CHECK:       ## %bb.0:
+define <4 x double> @test_int_x86_avx512_mask_gather3siv4_df(<4 x double> %x0, i8* %x1, <4 x i32> %x2, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_mask_gather3siv4_df:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    kmovd %esi, %k1
 ; CHECK-NEXT:    vgatherdpd (%rdi,%xmm1,4), %ymm0 {%k1}
 ; CHECK-NEXT:    kxnorw %k0, %k0, %k1
@@ -480,32 +481,34 @@ define <4 x double>@test_int_x86_avx512_gather3siv4_df(<4 x double> %x0, i8* %x1
 ; CHECK-NEXT:    vgatherdpd (%rdi,%xmm1,2), %ymm2 {%k1}
 ; CHECK-NEXT:    vaddpd %ymm2, %ymm0, %ymm0
 ; CHECK-NEXT:    retq
-  %res = call <4 x double> @llvm.x86.avx512.gather3siv4.df(<4 x double> %x0, i8* %x1, <4 x i32> %x2, i8 %x3, i32 4)
-  %res1 = call <4 x double> @llvm.x86.avx512.gather3siv4.df(<4 x double> %x0, i8* %x1, <4 x i32> %x2, i8 -1, i32 2)
+  %1 = bitcast i8 %x3 to <8 x i1>
+  %extract = shufflevector <8 x i1> %1, <8 x i1> %1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %res = call <4 x double> @llvm.x86.avx512.mask.gather3siv4.df(<4 x double> %x0, i8* %x1, <4 x i32> %x2, <4 x i1> %extract, i32 4)
+  %res1 = call <4 x double> @llvm.x86.avx512.mask.gather3siv4.df(<4 x double> %x0, i8* %x1, <4 x i32> %x2, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, i32 2)
   %res2 = fadd <4 x double> %res, %res1
   ret <4 x double> %res2
 }
 
-declare <4 x i64> @llvm.x86.avx512.gather3siv4.di(<4 x i64>, i8*, <4 x i32>, i8, i32)
-
-define <4 x i64>@test_int_x86_avx512_gather3siv4_di(<4 x i64> %x0, i8* %x1, <4 x i32> %x2, i8 %x3) {
-; CHECK-LABEL: test_int_x86_avx512_gather3siv4_di:
-; CHECK:       ## %bb.0:
+define <4 x i64> @test_int_x86_avx512_mask_gather3siv4_di(<4 x i64> %x0, i8* %x1, <4 x i32> %x2, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_mask_gather3siv4_di:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    kmovd %esi, %k1
 ; CHECK-NEXT:    vpgatherdq (%rdi,%xmm1,8), %ymm0 {%k1}
 ; CHECK-NEXT:    vpaddq %ymm0, %ymm0, %ymm0
 ; CHECK-NEXT:    retq
-  %res = call <4 x i64> @llvm.x86.avx512.gather3siv4.di(<4 x i64> %x0, i8* %x1, <4 x i32> %x2, i8 %x3, i32 8)
-  %res1 = call <4 x i64> @llvm.x86.avx512.gather3siv4.di(<4 x i64> %x0, i8* %x1, <4 x i32> %x2, i8 %x3, i32 8)
+  %1 = bitcast i8 %x3 to <8 x i1>
+  %extract1 = shufflevector <8 x i1> %1, <8 x i1> %1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %res = call <4 x i64> @llvm.x86.avx512.mask.gather3siv4.di(<4 x i64> %x0, i8* %x1, <4 x i32> %x2, <4 x i1> %extract1, i32 8)
+  %2 = bitcast i8 %x3 to <8 x i1>
+  %extract = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %res1 = call <4 x i64> @llvm.x86.avx512.mask.gather3siv4.di(<4 x i64> %x0, i8* %x1, <4 x i32> %x2, <4 x i1> %extract, i32 8)
   %res2 = add <4 x i64> %res, %res1
   ret <4 x i64> %res2
 }
 
-declare <4 x float> @llvm.x86.avx512.gather3siv4.sf(<4 x float>, i8*, <4 x i32>, i8, i32)
-
-define <4 x float>@test_int_x86_avx512_gather3siv4_sf(<4 x float> %x0, i8* %x1, <4 x i32> %x2, i8 %x3) {
-; CHECK-LABEL: test_int_x86_avx512_gather3siv4_sf:
-; CHECK:       ## %bb.0:
+define <4 x float> @test_int_x86_avx512_mask_gather3siv4_sf(<4 x float> %x0, i8* %x1, <4 x i32> %x2, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_mask_gather3siv4_sf:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    kmovd %esi, %k1
 ; CHECK-NEXT:    vgatherdps (%rdi,%xmm1,4), %xmm0 {%k1}
 ; CHECK-NEXT:    kxnorw %k0, %k0, %k1
@@ -513,35 +516,35 @@ define <4 x float>@test_int_x86_avx512_gather3siv4_sf(<4 x float> %x0, i8* %x1,
 ; CHECK-NEXT:    vgatherdps (%rdi,%xmm1,2), %xmm2 {%k1}
 ; CHECK-NEXT:    vaddps %xmm2, %xmm0, %xmm0
 ; CHECK-NEXT:    retq
-  %res = call <4 x float> @llvm.x86.avx512.gather3siv4.sf(<4 x float> %x0, i8* %x1, <4 x i32> %x2, i8 %x3, i32 4)
-  %res1 = call <4 x float> @llvm.x86.avx512.gather3siv4.sf(<4 x float> %x0, i8* %x1, <4 x i32> %x2, i8 -1, i32 2)
+  %1 = bitcast i8 %x3 to <8 x i1>
+  %extract = shufflevector <8 x i1> %1, <8 x i1> %1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %res = call <4 x float> @llvm.x86.avx512.mask.gather3siv4.sf(<4 x float> %x0, i8* %x1, <4 x i32> %x2, <4 x i1> %extract, i32 4)
+  %res1 = call <4 x float> @llvm.x86.avx512.mask.gather3siv4.sf(<4 x float> %x0, i8* %x1, <4 x i32> %x2, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, i32 2)
   %res2 = fadd <4 x float> %res, %res1
   ret <4 x float> %res2
 }
 
-declare <4 x i32> @llvm.x86.avx512.gather3siv4.si(<4 x i32>, i8*, <4 x i32>, i8, i32)
-
-define <4 x i32>@test_int_x86_avx512_gather3siv4_si(<4 x i32> %x0, i8* %x1, <4 x i32> %x2, i8 %x3) {
-; CHECK-LABEL: test_int_x86_avx512_gather3siv4_si:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovd %esi, %k1
-; CHECK-NEXT:    kxnorw %k0, %k0, %k2
+define <4 x i32> @test_int_x86_avx512_mask_gather3siv4_si(<4 x i32> %x0, i8* %x1, <4 x i32> %x2, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_mask_gather3siv4_si:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    kxnorw %k0, %k0, %k1
 ; CHECK-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; CHECK-NEXT:    vpgatherdd (%rdi,%xmm1,4), %xmm2 {%k2}
+; CHECK-NEXT:    vpgatherdd (%rdi,%xmm1,4), %xmm2 {%k1}
+; CHECK-NEXT:    kmovd %esi, %k1
 ; CHECK-NEXT:    vpgatherdd (%rdi,%xmm1,2), %xmm0 {%k1}
 ; CHECK-NEXT:    vpaddd %xmm0, %xmm2, %xmm0
 ; CHECK-NEXT:    retq
-  %res = call <4 x i32> @llvm.x86.avx512.gather3siv4.si(<4 x i32> %x0, i8* %x1, <4 x i32> %x2, i8 -1, i32 4)
-  %res1 = call <4 x i32> @llvm.x86.avx512.gather3siv4.si(<4 x i32> %x0, i8* %x1, <4 x i32> %x2, i8 %x3, i32 2)
+  %res = call <4 x i32> @llvm.x86.avx512.mask.gather3siv4.si(<4 x i32> %x0, i8* %x1, <4 x i32> %x2, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, i32 4)
+  %1 = bitcast i8 %x3 to <8 x i1>
+  %extract = shufflevector <8 x i1> %1, <8 x i1> %1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %res1 = call <4 x i32> @llvm.x86.avx512.mask.gather3siv4.si(<4 x i32> %x0, i8* %x1, <4 x i32> %x2, <4 x i1> %extract, i32 2)
   %res2 = add <4 x i32> %res, %res1
   ret <4 x i32> %res2
 }
 
-declare <8 x float> @llvm.x86.avx512.gather3siv8.sf(<8 x float>, i8*, <8 x i32>, i8, i32)
-
-define <8 x float>@test_int_x86_avx512_gather3siv8_sf(<8 x float> %x0, i8* %x1, <8 x i32> %x2, i8 %x3) {
-; CHECK-LABEL: test_int_x86_avx512_gather3siv8_sf:
-; CHECK:       ## %bb.0:
+define <8 x float> @test_int_x86_avx512_mask_gather3siv8_sf(<8 x float> %x0, i8* %x1, <8 x i32> %x2, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_mask_gather3siv8_sf:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    kmovd %esi, %k1
 ; CHECK-NEXT:    vgatherdps (%rdi,%ymm1,4), %ymm0 {%k1}
 ; CHECK-NEXT:    kxnorw %k0, %k0, %k1
@@ -549,17 +552,16 @@ define <8 x float>@test_int_x86_avx512_gather3siv8_sf(<8 x float> %x0, i8* %x1,
 ; CHECK-NEXT:    vgatherdps (%rdi,%ymm1,2), %ymm2 {%k1}
 ; CHECK-NEXT:    vaddps %ymm2, %ymm0, %ymm0
 ; CHECK-NEXT:    retq
-  %res = call <8 x float> @llvm.x86.avx512.gather3siv8.sf(<8 x float> %x0, i8* %x1, <8 x i32> %x2, i8 %x3, i32 4)
-  %res1 = call <8 x float> @llvm.x86.avx512.gather3siv8.sf(<8 x float> %x0, i8* %x1, <8 x i32> %x2, i8 -1, i32 2)
+  %1 = bitcast i8 %x3 to <8 x i1>
+  %res = call <8 x float> @llvm.x86.avx512.mask.gather3siv8.sf(<8 x float> %x0, i8* %x1, <8 x i32> %x2, <8 x i1> %1, i32 4)
+  %res1 = call <8 x float> @llvm.x86.avx512.mask.gather3siv8.sf(<8 x float> %x0, i8* %x1, <8 x i32> %x2, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, i32 2)
   %res2 = fadd <8 x float> %res, %res1
   ret <8 x float> %res2
 }
 
-declare <8 x i32> @llvm.x86.avx512.gather3siv8.si(<8 x i32>, i8*, <8 x i32>, i8, i32)
-
-define <8 x i32>@test_int_x86_avx512_gather3siv8_si(<8 x i32> %x0, i8* %x1, <8 x i32> %x2, i8 %x3) {
-; CHECK-LABEL: test_int_x86_avx512_gather3siv8_si:
-; CHECK:       ## %bb.0:
+define <8 x i32> @test_int_x86_avx512_mask_gather3siv8_si(<8 x i32> %x0, i8* %x1, <8 x i32> %x2, i8 %x3) {
+; CHECK-LABEL: test_int_x86_avx512_mask_gather3siv8_si:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    kmovd %esi, %k1
 ; CHECK-NEXT:    vmovdqa %ymm0, %ymm2
 ; CHECK-NEXT:    kmovq %k1, %k2
@@ -567,263 +569,263 @@ define <8 x i32>@test_int_x86_avx512_gather3siv8_si(<8 x i32> %x0, i8* %x1, <8 x
 ; CHECK-NEXT:    vpgatherdd (%rdi,%ymm1,2), %ymm0 {%k1}
 ; CHECK-NEXT:    vpaddd %ymm0, %ymm2, %ymm0
 ; CHECK-NEXT:    retq
-  %res = call <8 x i32> @llvm.x86.avx512.gather3siv8.si(<8 x i32> %x0, i8* %x1, <8 x i32> %x2, i8 %x3, i32 4)
-  %res1 = call <8 x i32> @llvm.x86.avx512.gather3siv8.si(<8 x i32> %x0, i8* %x1, <8 x i32> %x2, i8 %x3, i32 2)
+  %1 = bitcast i8 %x3 to <8 x i1>
+  %res = call <8 x i32> @llvm.x86.avx512.mask.gather3siv8.si(<8 x i32> %x0, i8* %x1, <8 x i32> %x2, <8 x i1> %1, i32 4)
+  %2 = bitcast i8 %x3 to <8 x i1>
+  %res1 = call <8 x i32> @llvm.x86.avx512.mask.gather3siv8.si(<8 x i32> %x0, i8* %x1, <8 x i32> %x2, <8 x i1> %2, i32 2)
   %res2 = add <8 x i32> %res, %res1
   ret <8 x i32> %res2
 }
 
-declare void @llvm.x86.avx512.scatterdiv2.df(i8*, i8, <2 x i64>, <2 x double>, i32)
-
 define void@test_int_x86_avx512_scatterdiv2_df(i8* %x0, i8 %x1, <2 x i64> %x2, <2 x double> %x3) {
 ; CHECK-LABEL: test_int_x86_avx512_scatterdiv2_df:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    kmovd %esi, %k1
 ; CHECK-NEXT:    kxnorw %k0, %k0, %k2
 ; CHECK-NEXT:    vscatterqpd %xmm1, (%rdi,%xmm0,2) {%k2}
 ; CHECK-NEXT:    vscatterqpd %xmm1, (%rdi,%xmm0,4) {%k1}
 ; CHECK-NEXT:    retq
-  call void @llvm.x86.avx512.scatterdiv2.df(i8* %x0, i8 -1, <2 x i64> %x2, <2 x double> %x3, i32 2)
-  call void @llvm.x86.avx512.scatterdiv2.df(i8* %x0, i8 %x1, <2 x i64> %x2, <2 x double> %x3, i32 4)
+  %1 = bitcast i8 %x1 to <8 x i1>
+  %2 = shufflevector <8 x i1> %1, <8 x i1> undef, <2 x i32> <i32 0, i32 1>
+  call void @llvm.x86.avx512.mask.scatterdiv2.df(i8* %x0, <2 x i1> <i1 true, i1 true>, <2 x i64> %x2, <2 x double> %x3, i32 2)
+  call void @llvm.x86.avx512.mask.scatterdiv2.df(i8* %x0, <2 x i1> %2, <2 x i64> %x2, <2 x double> %x3, i32 4)
   ret void
 }
 
-declare void @llvm.x86.avx512.scatterdiv2.di(i8*, i8, <2 x i64>, <2 x i64>, i32)
-
 define void@test_int_x86_avx512_scatterdiv2_di(i8* %x0, i8 %x1, <2 x i64> %x2, <2 x i64> %x3) {
 ; CHECK-LABEL: test_int_x86_avx512_scatterdiv2_di:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    kmovd %esi, %k1
 ; CHECK-NEXT:    vpscatterqq %xmm1, (%rdi,%xmm0,2) {%k1}
 ; CHECK-NEXT:    kxnorw %k0, %k0, %k1
 ; CHECK-NEXT:    vpscatterqq %xmm1, (%rdi,%xmm0,4) {%k1}
 ; CHECK-NEXT:    retq
-  call void @llvm.x86.avx512.scatterdiv2.di(i8* %x0, i8 %x1, <2 x i64> %x2, <2 x i64> %x3, i32 2)
-  call void @llvm.x86.avx512.scatterdiv2.di(i8* %x0, i8 -1, <2 x i64> %x2, <2 x i64> %x3, i32 4)
+  %1 = bitcast i8 %x1 to <8 x i1>
+  %2 = shufflevector <8 x i1> %1, <8 x i1> undef, <2 x i32> <i32 0, i32 1>
+  call void @llvm.x86.avx512.mask.scatterdiv2.di(i8* %x0, <2 x i1> %2, <2 x i64> %x2, <2 x i64> %x3, i32 2)
+  call void @llvm.x86.avx512.mask.scatterdiv2.di(i8* %x0, <2 x i1> <i1 true, i1 true>, <2 x i64> %x2, <2 x i64> %x3, i32 4)
   ret void
 }
 
-declare void @llvm.x86.avx512.scatterdiv4.df(i8*, i8, <4 x i64>, <4 x double>, i32)
-
 define void@test_int_x86_avx512_scatterdiv4_df(i8* %x0, i8 %x1, <4 x i64> %x2, <4 x double> %x3) {
 ; CHECK-LABEL: test_int_x86_avx512_scatterdiv4_df:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    kmovd %esi, %k1
 ; CHECK-NEXT:    vscatterqpd %ymm1, (%rdi,%ymm0,2) {%k1}
 ; CHECK-NEXT:    kxnorw %k0, %k0, %k1
 ; CHECK-NEXT:    vscatterqpd %ymm1, (%rdi,%ymm0,4) {%k1}
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retq
-  call void @llvm.x86.avx512.scatterdiv4.df(i8* %x0, i8 %x1, <4 x i64> %x2, <4 x double> %x3, i32 2)
-  call void @llvm.x86.avx512.scatterdiv4.df(i8* %x0, i8 -1, <4 x i64> %x2, <4 x double> %x3, i32 4)
+  %1 = bitcast i8 %x1 to <8 x i1>
+  %2 = shufflevector <8 x i1> %1, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  call void @llvm.x86.avx512.mask.scatterdiv4.df(i8* %x0, <4 x i1> %2, <4 x i64> %x2, <4 x double> %x3, i32 2)
+  call void @llvm.x86.avx512.mask.scatterdiv4.df(i8* %x0, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i64> %x2, <4 x double> %x3, i32 4)
   ret void
 }
 
-declare void @llvm.x86.avx512.scatterdiv4.di(i8*, i8, <4 x i64>, <4 x i64>, i32)
-
 define void@test_int_x86_avx512_scatterdiv4_di(i8* %x0, i8 %x1, <4 x i64> %x2, <4 x i64> %x3) {
 ; CHECK-LABEL: test_int_x86_avx512_scatterdiv4_di:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    kmovd %esi, %k1
 ; CHECK-NEXT:    vpscatterqq %ymm1, (%rdi,%ymm0,2) {%k1}
 ; CHECK-NEXT:    kxnorw %k0, %k0, %k1
 ; CHECK-NEXT:    vpscatterqq %ymm1, (%rdi,%ymm0,4) {%k1}
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retq
-  call void @llvm.x86.avx512.scatterdiv4.di(i8* %x0, i8 %x1, <4 x i64> %x2, <4 x i64> %x3, i32 2)
-  call void @llvm.x86.avx512.scatterdiv4.di(i8* %x0, i8 -1, <4 x i64> %x2, <4 x i64> %x3, i32 4)
+  %1 = bitcast i8 %x1 to <8 x i1>
+  %2 = shufflevector <8 x i1> %1, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  call void @llvm.x86.avx512.mask.scatterdiv4.di(i8* %x0, <4 x i1> %2, <4 x i64> %x2, <4 x i64> %x3, i32 2)
+  call void @llvm.x86.avx512.mask.scatterdiv4.di(i8* %x0, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i64> %x2, <4 x i64> %x3, i32 4)
   ret void
 }
 
-declare void @llvm.x86.avx512.scatterdiv4.sf(i8*, i8, <2 x i64>, <4 x float>, i32)
-
 define void@test_int_x86_avx512_scatterdiv4_sf(i8* %x0, i8 %x1, <2 x i64> %x2, <4 x float> %x3) {
 ; CHECK-LABEL: test_int_x86_avx512_scatterdiv4_sf:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    kmovd %esi, %k1
 ; CHECK-NEXT:    vscatterqps %xmm1, (%rdi,%xmm0,2) {%k1}
 ; CHECK-NEXT:    kxnorw %k0, %k0, %k1
 ; CHECK-NEXT:    vscatterqps %xmm1, (%rdi,%xmm0,4) {%k1}
 ; CHECK-NEXT:    retq
-  call void @llvm.x86.avx512.scatterdiv4.sf(i8* %x0, i8 %x1, <2 x i64> %x2, <4 x float> %x3, i32 2)
-  call void @llvm.x86.avx512.scatterdiv4.sf(i8* %x0, i8 -1, <2 x i64> %x2, <4 x float> %x3, i32 4)
+  %1 = bitcast i8 %x1 to <8 x i1>
+  %2 = shufflevector <8 x i1> %1, <8 x i1> undef, <2 x i32> <i32 0, i32 1>
+  call void @llvm.x86.avx512.mask.scatterdiv4.sf(i8* %x0, <2 x i1> %2, <2 x i64> %x2, <4 x float> %x3, i32 2)
+  call void @llvm.x86.avx512.mask.scatterdiv4.sf(i8* %x0, <2 x i1> <i1 true, i1 true>, <2 x i64> %x2, <4 x float> %x3, i32 4)
   ret void
 }
 
-declare void @llvm.x86.avx512.scatterdiv4.si(i8*, i8, <2 x i64>, <4 x i32>, i32)
-
 define void@test_int_x86_avx512_scatterdiv4_si(i8* %x0, i8 %x1, <2 x i64> %x2, <4 x i32> %x3) {
 ; CHECK-LABEL: test_int_x86_avx512_scatterdiv4_si:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    kmovd %esi, %k1
 ; CHECK-NEXT:    kxnorw %k0, %k0, %k2
 ; CHECK-NEXT:    vpscatterqd %xmm1, (%rdi,%xmm0,2) {%k2}
 ; CHECK-NEXT:    vpscatterqd %xmm1, (%rdi,%xmm0,4) {%k1}
 ; CHECK-NEXT:    retq
-  call void @llvm.x86.avx512.scatterdiv4.si(i8* %x0, i8 -1, <2 x i64> %x2, <4 x i32> %x3, i32 2)
-  call void @llvm.x86.avx512.scatterdiv4.si(i8* %x0, i8 %x1, <2 x i64> %x2, <4 x i32> %x3, i32 4)
+  %1 = bitcast i8 %x1 to <8 x i1>
+  %2 = shufflevector <8 x i1> %1, <8 x i1> undef, <2 x i32> <i32 0, i32 1>
+  call void @llvm.x86.avx512.mask.scatterdiv4.si(i8* %x0, <2 x i1> <i1 true, i1 true>, <2 x i64> %x2, <4 x i32> %x3, i32 2)
+  call void @llvm.x86.avx512.mask.scatterdiv4.si(i8* %x0, <2 x i1> %2, <2 x i64> %x2, <4 x i32> %x3, i32 4)
   ret void
 }
 
-declare void @llvm.x86.avx512.scatterdiv8.sf(i8*, i8, <4 x i64>, <4 x float>, i32)
-
 define void@test_int_x86_avx512_scatterdiv8_sf(i8* %x0, i8 %x1, <4 x i64> %x2, <4 x float> %x3) {
 ; CHECK-LABEL: test_int_x86_avx512_scatterdiv8_sf:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    kmovd %esi, %k1
 ; CHECK-NEXT:    vscatterqps %xmm1, (%rdi,%ymm0,2) {%k1}
 ; CHECK-NEXT:    kxnorw %k0, %k0, %k1
 ; CHECK-NEXT:    vscatterqps %xmm1, (%rdi,%ymm0,4) {%k1}
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retq
-  call void @llvm.x86.avx512.scatterdiv8.sf(i8* %x0, i8 %x1, <4 x i64> %x2, <4 x float> %x3, i32 2)
-  call void @llvm.x86.avx512.scatterdiv8.sf(i8* %x0, i8 -1, <4 x i64> %x2, <4 x float> %x3, i32 4)
+  %1 = bitcast i8 %x1 to <8 x i1>
+  %2 = shufflevector <8 x i1> %1, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  call void @llvm.x86.avx512.mask.scatterdiv8.sf(i8* %x0, <4 x i1> %2, <4 x i64> %x2, <4 x float> %x3, i32 2)
+  call void @llvm.x86.avx512.mask.scatterdiv8.sf(i8* %x0, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i64> %x2, <4 x float> %x3, i32 4)
   ret void
 }
 
-declare void @llvm.x86.avx512.scatterdiv8.si(i8*, i8, <4 x i64>, <4 x i32>, i32)
-
 define void@test_int_x86_avx512_scatterdiv8_si(i8* %x0, i8 %x1, <4 x i64> %x2, <4 x i32> %x3) {
 ; CHECK-LABEL: test_int_x86_avx512_scatterdiv8_si:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    kmovd %esi, %k1
 ; CHECK-NEXT:    vpscatterqd %xmm1, (%rdi,%ymm0,2) {%k1}
 ; CHECK-NEXT:    kxnorw %k0, %k0, %k1
 ; CHECK-NEXT:    vpscatterqd %xmm1, (%rdi,%ymm0,4) {%k1}
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retq
-  call void @llvm.x86.avx512.scatterdiv8.si(i8* %x0, i8 %x1, <4 x i64> %x2, <4 x i32> %x3, i32 2)
-  call void @llvm.x86.avx512.scatterdiv8.si(i8* %x0, i8 -1, <4 x i64> %x2, <4 x i32> %x3, i32 4)
+  %1 = bitcast i8 %x1 to <8 x i1>
+  %2 = shufflevector <8 x i1> %1, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  call void @llvm.x86.avx512.mask.scatterdiv8.si(i8* %x0, <4 x i1> %2, <4 x i64> %x2, <4 x i32> %x3, i32 2)
+  call void @llvm.x86.avx512.mask.scatterdiv8.si(i8* %x0, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i64> %x2, <4 x i32> %x3, i32 4)
   ret void
 }
 
-declare void @llvm.x86.avx512.scattersiv2.df(i8*, i8, <4 x i32>, <2 x double>, i32)
-
 define void@test_int_x86_avx512_scattersiv2_df(i8* %x0, i8 %x1, <4 x i32> %x2, <2 x double> %x3) {
 ; CHECK-LABEL: test_int_x86_avx512_scattersiv2_df:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    kmovd %esi, %k1
 ; CHECK-NEXT:    kxnorw %k0, %k0, %k2
 ; CHECK-NEXT:    vscatterdpd %xmm1, (%rdi,%xmm0,2) {%k2}
 ; CHECK-NEXT:    vscatterdpd %xmm1, (%rdi,%xmm0,4) {%k1}
 ; CHECK-NEXT:    retq
-  call void @llvm.x86.avx512.scattersiv2.df(i8* %x0, i8 -1, <4 x i32> %x2, <2 x double> %x3, i32 2)
-  call void @llvm.x86.avx512.scattersiv2.df(i8* %x0, i8 %x1, <4 x i32> %x2, <2 x double> %x3, i32 4)
+  %1 = bitcast i8 %x1 to <8 x i1>
+  %2 = shufflevector <8 x i1> %1, <8 x i1> undef, <2 x i32> <i32 0, i32 1>
+  call void @llvm.x86.avx512.mask.scattersiv2.df(i8* %x0, <2 x i1> <i1 true, i1 true>, <4 x i32> %x2, <2 x double> %x3, i32 2)
+  call void @llvm.x86.avx512.mask.scattersiv2.df(i8* %x0, <2 x i1> %2, <4 x i32> %x2, <2 x double> %x3, i32 4)
   ret void
 }
 
-declare void @llvm.x86.avx512.scattersiv2.di(i8*, i8, <4 x i32>, <2 x i64>, i32)
-
 define void@test_int_x86_avx512_scattersiv2_di(i8* %x0, i8 %x1, <4 x i32> %x2, <2 x i64> %x3) {
 ; CHECK-LABEL: test_int_x86_avx512_scattersiv2_di:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    kmovd %esi, %k1
 ; CHECK-NEXT:    kxnorw %k0, %k0, %k2
 ; CHECK-NEXT:    vpscatterdq %xmm1, (%rdi,%xmm0,2) {%k2}
 ; CHECK-NEXT:    vpscatterdq %xmm1, (%rdi,%xmm0,4) {%k1}
 ; CHECK-NEXT:    retq
-  call void @llvm.x86.avx512.scattersiv2.di(i8* %x0, i8 -1, <4 x i32> %x2, <2 x i64> %x3, i32 2)
-  call void @llvm.x86.avx512.scattersiv2.di(i8* %x0, i8 %x1, <4 x i32> %x2, <2 x i64> %x3, i32 4)
+  %1 = bitcast i8 %x1 to <8 x i1>
+  %2 = shufflevector <8 x i1> %1, <8 x i1> undef, <2 x i32> <i32 0, i32 1>
+  call void @llvm.x86.avx512.mask.scattersiv2.di(i8* %x0, <2 x i1> <i1 true, i1 true>, <4 x i32> %x2, <2 x i64> %x3, i32 2)
+  call void @llvm.x86.avx512.mask.scattersiv2.di(i8* %x0, <2 x i1> %2, <4 x i32> %x2, <2 x i64> %x3, i32 4)
   ret void
 }
 
-declare void @llvm.x86.avx512.scattersiv4.df(i8*, i8, <4 x i32>, <4 x double>, i32)
-
 define void@test_int_x86_avx512_scattersiv4_df(i8* %x0, i8 %x1, <4 x i32> %x2, <4 x double> %x3) {
 ; CHECK-LABEL: test_int_x86_avx512_scattersiv4_df:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    kmovd %esi, %k1
 ; CHECK-NEXT:    vscatterdpd %ymm1, (%rdi,%xmm0,2) {%k1}
 ; CHECK-NEXT:    kxnorw %k0, %k0, %k1
 ; CHECK-NEXT:    vscatterdpd %ymm1, (%rdi,%xmm0,4) {%k1}
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retq
-  call void @llvm.x86.avx512.scattersiv4.df(i8* %x0, i8 %x1, <4 x i32> %x2, <4 x double> %x3, i32 2)
-  call void @llvm.x86.avx512.scattersiv4.df(i8* %x0, i8 -1, <4 x i32> %x2, <4 x double> %x3, i32 4)
+  %1 = bitcast i8 %x1 to <8 x i1>
+  %2 = shufflevector <8 x i1> %1, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  call void @llvm.x86.avx512.mask.scattersiv4.df(i8* %x0, <4 x i1> %2, <4 x i32> %x2, <4 x double> %x3, i32 2)
+  call void @llvm.x86.avx512.mask.scattersiv4.df(i8* %x0, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> %x2, <4 x double> %x3, i32 4)
   ret void
 }
 
-declare void @llvm.x86.avx512.scattersiv4.di(i8*, i8, <4 x i32>, <4 x i64>, i32)
-
 define void@test_int_x86_avx512_scattersiv4_di(i8* %x0, i8 %x1, <4 x i32> %x2, <4 x i64> %x3) {
 ; CHECK-LABEL: test_int_x86_avx512_scattersiv4_di:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    kmovd %esi, %k1
 ; CHECK-NEXT:    kxnorw %k0, %k0, %k2
 ; CHECK-NEXT:    vpscatterdq %ymm1, (%rdi,%xmm0,2) {%k2}
 ; CHECK-NEXT:    vpscatterdq %ymm1, (%rdi,%xmm0,4) {%k1}
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retq
-  call void @llvm.x86.avx512.scattersiv4.di(i8* %x0, i8 -1, <4 x i32> %x2, <4 x i64> %x3, i32 2)
-  call void @llvm.x86.avx512.scattersiv4.di(i8* %x0, i8 %x1, <4 x i32> %x2, <4 x i64> %x3, i32 4)
+  %1 = bitcast i8 %x1 to <8 x i1>
+  %2 = shufflevector <8 x i1> %1, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  call void @llvm.x86.avx512.mask.scattersiv4.di(i8* %x0, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> %x2, <4 x i64> %x3, i32 2)
+  call void @llvm.x86.avx512.mask.scattersiv4.di(i8* %x0, <4 x i1> %2, <4 x i32> %x2, <4 x i64> %x3, i32 4)
   ret void
 }
 
-declare void @llvm.x86.avx512.scattersiv4.sf(i8*, i8, <4 x i32>, <4 x float>, i32)
-
 define void@test_int_x86_avx512_scattersiv4_sf(i8* %x0, i8 %x1, <4 x i32> %x2, <4 x float> %x3) {
 ; CHECK-LABEL: test_int_x86_avx512_scattersiv4_sf:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    kmovd %esi, %k1
 ; CHECK-NEXT:    vscatterdps %xmm1, (%rdi,%xmm0,2) {%k1}
 ; CHECK-NEXT:    kxnorw %k0, %k0, %k1
 ; CHECK-NEXT:    vscatterdps %xmm1, (%rdi,%xmm0,4) {%k1}
 ; CHECK-NEXT:    retq
-  call void @llvm.x86.avx512.scattersiv4.sf(i8* %x0, i8 %x1, <4 x i32> %x2, <4 x float> %x3, i32 2)
-  call void @llvm.x86.avx512.scattersiv4.sf(i8* %x0, i8 -1, <4 x i32> %x2, <4 x float> %x3, i32 4)
+  %1 = bitcast i8 %x1 to <8 x i1>
+  %2 = shufflevector <8 x i1> %1, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  call void @llvm.x86.avx512.mask.scattersiv4.sf(i8* %x0, <4 x i1> %2, <4 x i32> %x2, <4 x float> %x3, i32 2)
+  call void @llvm.x86.avx512.mask.scattersiv4.sf(i8* %x0, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> %x2, <4 x float> %x3, i32 4)
   ret void
 }
 
-declare void @llvm.x86.avx512.scattersiv4.si(i8*, i8, <4 x i32>, <4 x i32>, i32)
-
 define void@test_int_x86_avx512_scattersiv4_si(i8* %x0, i8 %x1, <4 x i32> %x2, <4 x i32> %x3) {
 ; CHECK-LABEL: test_int_x86_avx512_scattersiv4_si:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    kmovd %esi, %k1
 ; CHECK-NEXT:    vpscatterdd %xmm1, (%rdi,%xmm0,2) {%k1}
 ; CHECK-NEXT:    kxnorw %k0, %k0, %k1
 ; CHECK-NEXT:    vpscatterdd %xmm1, (%rdi,%xmm0,4) {%k1}
 ; CHECK-NEXT:    retq
-  call void @llvm.x86.avx512.scattersiv4.si(i8* %x0, i8 %x1, <4 x i32> %x2, <4 x i32> %x3, i32 2)
-  call void @llvm.x86.avx512.scattersiv4.si(i8* %x0, i8 -1, <4 x i32> %x2, <4 x i32> %x3, i32 4)
+  %1 = bitcast i8 %x1 to <8 x i1>
+  %2 = shufflevector <8 x i1> %1, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  call void @llvm.x86.avx512.mask.scattersiv4.si(i8* %x0, <4 x i1> %2, <4 x i32> %x2, <4 x i32> %x3, i32 2)
+  call void @llvm.x86.avx512.mask.scattersiv4.si(i8* %x0, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> %x2, <4 x i32> %x3, i32 4)
   ret void
 }
 
-declare void @llvm.x86.avx512.scattersiv8.sf(i8*, i8, <8 x i32>, <8 x float>, i32)
-
 define void@test_int_x86_avx512_scattersiv8_sf(i8* %x0, i8 %x1, <8 x i32> %x2, <8 x float> %x3) {
 ; CHECK-LABEL: test_int_x86_avx512_scattersiv8_sf:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    kmovd %esi, %k1
 ; CHECK-NEXT:    vscatterdps %ymm1, (%rdi,%ymm0,2) {%k1}
 ; CHECK-NEXT:    kxnorw %k0, %k0, %k1
 ; CHECK-NEXT:    vscatterdps %ymm1, (%rdi,%ymm0,4) {%k1}
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retq
-  call void @llvm.x86.avx512.scattersiv8.sf(i8* %x0, i8 %x1, <8 x i32> %x2, <8 x float> %x3, i32 2)
-  call void @llvm.x86.avx512.scattersiv8.sf(i8* %x0, i8 -1, <8 x i32> %x2, <8 x float> %x3, i32 4)
+  %1 = bitcast i8 %x1 to <8 x i1>
+  call void @llvm.x86.avx512.mask.scattersiv8.sf(i8* %x0, <8 x i1> %1, <8 x i32> %x2, <8 x float> %x3, i32 2)
+  call void @llvm.x86.avx512.mask.scattersiv8.sf(i8* %x0, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <8 x i32> %x2, <8 x float> %x3, i32 4)
   ret void
 }
 
-declare void @llvm.x86.avx512.scattersiv8.si(i8*, i8, <8 x i32>, <8 x i32>, i32)
-
 define void@test_int_x86_avx512_scattersiv8_si(i8* %x0, i8 %x1, <8 x i32> %x2, <8 x i32> %x3) {
 ; CHECK-LABEL: test_int_x86_avx512_scattersiv8_si:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    kmovd %esi, %k1
 ; CHECK-NEXT:    vpscatterdd %ymm1, (%rdi,%ymm0,2) {%k1}
 ; CHECK-NEXT:    kxnorw %k0, %k0, %k1
 ; CHECK-NEXT:    vpscatterdd %ymm1, (%rdi,%ymm0,4) {%k1}
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retq
-  call void @llvm.x86.avx512.scattersiv8.si(i8* %x0, i8 %x1, <8 x i32> %x2, <8 x i32> %x3, i32 2)
-  call void @llvm.x86.avx512.scattersiv8.si(i8* %x0, i8 -1, <8 x i32> %x2, <8 x i32> %x3, i32 4)
+  %1 = bitcast i8 %x1 to <8 x i1>
+  call void @llvm.x86.avx512.mask.scattersiv8.si(i8* %x0, <8 x i1> %1, <8 x i32> %x2, <8 x i32> %x3, i32 2)
+  call void @llvm.x86.avx512.mask.scattersiv8.si(i8* %x0, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <8 x i32> %x2, <8 x i32> %x3, i32 4)
   ret void
 }
 
 define void @scatter_mask_test(i8* %x0, <8 x i32> %x2, <8 x i32> %x3) {
 ; CHECK-LABEL: scatter_mask_test:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    kxnorw %k0, %k0, %k1
 ; CHECK-NEXT:    vpscatterdd %ymm1, (%rdi,%ymm0,2) {%k1}
 ; CHECK-NEXT:    kxorw %k0, %k0, %k1
@@ -836,16 +838,16 @@ define void @scatter_mask_test(i8* %x0, <8 x i32> %x2, <8 x i32> %x3) {
 ; CHECK-NEXT:    vpscatterdd %ymm1, (%rdi,%ymm0,4) {%k1}
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retq
-  call void @llvm.x86.avx512.scattersiv8.si(i8* %x0, i8 -1, <8 x i32> %x2, <8 x i32> %x3, i32 2)
-  call void @llvm.x86.avx512.scattersiv8.si(i8* %x0, i8 0, <8 x i32> %x2, <8 x i32> %x3, i32 4)
-  call void @llvm.x86.avx512.scattersiv8.si(i8* %x0, i8 1, <8 x i32> %x2, <8 x i32> %x3, i32 2)
-  call void @llvm.x86.avx512.scattersiv8.si(i8* %x0, i8 96, <8 x i32> %x2, <8 x i32> %x3, i32 4)
+  call void @llvm.x86.avx512.mask.scattersiv8.si(i8* %x0, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <8 x i32> %x2, <8 x i32> %x3, i32 2)
+  call void @llvm.x86.avx512.mask.scattersiv8.si(i8* %x0, <8 x i1> zeroinitializer, <8 x i32> %x2, <8 x i32> %x3, i32 4)
+  call void @llvm.x86.avx512.mask.scattersiv8.si(i8* %x0, <8 x i1> bitcast (<1 x i8> <i8 1> to <8 x i1>), <8 x i32> %x2, <8 x i32> %x3, i32 2)
+  call void @llvm.x86.avx512.mask.scattersiv8.si(i8* %x0, <8 x i1> bitcast (<1 x i8> <i8 96> to <8 x i1>), <8 x i32> %x2, <8 x i32> %x3, i32 4)
   ret void
 }
 
-define <16 x float> @gather_mask_test(<16 x i32> %ind, <16 x float> %src, i8* %base)  {
+define <16 x float> @gather_mask_test(<16 x i32> %ind, <16 x float> %src, i8* %base) {
 ; CHECK-LABEL: gather_mask_test:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    kxnorw %k0, %k0, %k1
 ; CHECK-NEXT:    vxorps %xmm2, %xmm2, %xmm2
 ; CHECK-NEXT:    vgatherdps (%rdi,%zmm0,4), %zmm2 {%k1}
@@ -863,13 +865,76 @@ define <16 x float> @gather_mask_test(<16 x i32> %ind, <16 x float> %src, i8* %b
 ; CHECK-NEXT:    vaddps %zmm3, %zmm1, %zmm0
 ; CHECK-NEXT:    vaddps %zmm2, %zmm0, %zmm0
 ; CHECK-NEXT:    retq
-  %res = call <16 x float> @llvm.x86.avx512.gather.dps.512 (<16 x float> %src, i8* %base, <16 x i32>%ind, i16 -1, i32 4)
-  %res1 = call <16 x float> @llvm.x86.avx512.gather.dps.512 (<16 x float> %src, i8* %base, <16 x i32>%ind, i16 0, i32 4)
-  %res2 = call <16 x float> @llvm.x86.avx512.gather.dps.512 (<16 x float> %src, i8* %base, <16 x i32>%ind, i16 1, i32 4)
-  %res3 = call <16 x float> @llvm.x86.avx512.gather.dps.512 (<16 x float> %src, i8* %base, <16 x i32>%ind, i16 220, i32 4)
-
+  %res = call <16 x float> @llvm.x86.avx512.mask.gather.dps.512(<16 x float> %src, i8* %base, <16 x i32> %ind, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, i32 4)
+  %res1 = call <16 x float> @llvm.x86.avx512.mask.gather.dps.512(<16 x float> %src, i8* %base, <16 x i32> %ind, <16 x i1> zeroinitializer, i32 4)
+  %res2 = call <16 x float> @llvm.x86.avx512.mask.gather.dps.512(<16 x float> %src, i8* %base, <16 x i32> %ind, <16 x i1> bitcast (<1 x i16> <i16 1> to <16 x i1>), i32 4)
+  %res3 = call <16 x float> @llvm.x86.avx512.mask.gather.dps.512(<16 x float> %src, i8* %base, <16 x i32> %ind, <16 x i1> bitcast (<1 x i16> <i16 220> to <16 x i1>), i32 4)
   %res4 = fadd <16 x float> %res, %res1
   %res5 = fadd <16 x float> %res3, %res2
   %res6 = fadd <16 x float> %res5, %res4
   ret <16 x float> %res6
 }
+
+@x = global [1024 x float] zeroinitializer, align 16
+
+define <8 x float> @gather_global(<8 x i64>, i32* nocapture readnone) {
+; CHECK-LABEL: gather_global:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    kxnorw %k0, %k0, %k1
+; CHECK-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; CHECK-NEXT:    vgatherqps x(,%zmm0,4), %ymm1 {%k1}
+; CHECK-NEXT:    vmovaps %ymm1, %ymm0
+; CHECK-NEXT:    retq
+  %3 = tail call <8 x float> @llvm.x86.avx512.mask.gather.qps.512(<8 x float> zeroinitializer, i8* bitcast ([1024 x float]* @x to i8*), <8 x i64> %0, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, i32 4)
+  ret <8 x float> %3
+}
+
+declare <16 x float> @llvm.x86.avx512.mask.gather.dps.512(<16 x float>, i8*, <16 x i32>, <16 x i1>, i32)
+declare <8 x double> @llvm.x86.avx512.mask.gather.dpd.512(<8 x double>, i8*, <8 x i32>, <8 x i1>, i32)
+declare <8 x float> @llvm.x86.avx512.mask.gather.qps.512(<8 x float>, i8*, <8 x i64>, <8 x i1>, i32)
+declare <8 x double> @llvm.x86.avx512.mask.gather.qpd.512(<8 x double>, i8*, <8 x i64>, <8 x i1>, i32)
+declare <16 x i32> @llvm.x86.avx512.mask.gather.dpi.512(<16 x i32>, i8*, <16 x i32>, <16 x i1>, i32)
+declare <8 x i64> @llvm.x86.avx512.mask.gather.dpq.512(<8 x i64>, i8*, <8 x i32>, <8 x i1>, i32)
+declare <8 x i32> @llvm.x86.avx512.mask.gather.qpi.512(<8 x i32>, i8*, <8 x i64>, <8 x i1>, i32)
+declare <8 x i64> @llvm.x86.avx512.mask.gather.qpq.512(<8 x i64>, i8*, <8 x i64>, <8 x i1>, i32)
+declare <2 x double> @llvm.x86.avx512.mask.gather3div2.df(<2 x double>, i8*, <2 x i64>, <2 x i1>, i32)
+declare <2 x i64> @llvm.x86.avx512.mask.gather3div2.di(<2 x i64>, i8*, <2 x i64>, <2 x i1>, i32)
+declare <4 x double> @llvm.x86.avx512.mask.gather3div4.df(<4 x double>, i8*, <4 x i64>, <4 x i1>, i32)
+declare <4 x i64> @llvm.x86.avx512.mask.gather3div4.di(<4 x i64>, i8*, <4 x i64>, <4 x i1>, i32)
+declare <4 x float> @llvm.x86.avx512.mask.gather3div4.sf(<4 x float>, i8*, <2 x i64>, <2 x i1>, i32)
+declare <4 x i32> @llvm.x86.avx512.mask.gather3div4.si(<4 x i32>, i8*, <2 x i64>, <2 x i1>, i32)
+declare <4 x float> @llvm.x86.avx512.mask.gather3div8.sf(<4 x float>, i8*, <4 x i64>, <4 x i1>, i32)
+declare <4 x i32> @llvm.x86.avx512.mask.gather3div8.si(<4 x i32>, i8*, <4 x i64>, <4 x i1>, i32)
+declare <2 x double> @llvm.x86.avx512.mask.gather3siv2.df(<2 x double>, i8*, <4 x i32>, <2 x i1>, i32)
+declare <2 x i64> @llvm.x86.avx512.mask.gather3siv2.di(<2 x i64>, i8*, <4 x i32>, <2 x i1>, i32)
+declare <4 x double> @llvm.x86.avx512.mask.gather3siv4.df(<4 x double>, i8*, <4 x i32>, <4 x i1>, i32)
+declare <4 x i64> @llvm.x86.avx512.mask.gather3siv4.di(<4 x i64>, i8*, <4 x i32>, <4 x i1>, i32)
+declare <4 x float> @llvm.x86.avx512.mask.gather3siv4.sf(<4 x float>, i8*, <4 x i32>, <4 x i1>, i32)
+declare <4 x i32> @llvm.x86.avx512.mask.gather3siv4.si(<4 x i32>, i8*, <4 x i32>, <4 x i1>, i32)
+declare <8 x float> @llvm.x86.avx512.mask.gather3siv8.sf(<8 x float>, i8*, <8 x i32>, <8 x i1>, i32)
+declare <8 x i32> @llvm.x86.avx512.mask.gather3siv8.si(<8 x i32>, i8*, <8 x i32>, <8 x i1>, i32)
+declare void @llvm.x86.avx512.mask.scatter.dps.512(i8*, <16 x i1>, <16 x i32>, <16 x float>, i32)
+declare void @llvm.x86.avx512.mask.scatter.dpd.512(i8*, <8 x i1>, <8 x i32>, <8 x double>, i32)
+declare void @llvm.x86.avx512.mask.scatter.qps.512(i8*, <8 x i1>, <8 x i64>, <8 x float>, i32)
+declare void @llvm.x86.avx512.mask.scatter.qpd.512(i8*, <8 x i1>, <8 x i64>, <8 x double>, i32)
+declare void @llvm.x86.avx512.mask.scatter.dpi.512(i8*, <16 x i1>, <16 x i32>, <16 x i32>, i32)
+declare void @llvm.x86.avx512.mask.scatter.dpq.512(i8*, <8 x i1>, <8 x i32>, <8 x i64>, i32)
+declare void @llvm.x86.avx512.mask.scatter.qpi.512(i8*, <8 x i1>, <8 x i64>, <8 x i32>, i32)
+declare void @llvm.x86.avx512.mask.scatter.qpq.512(i8*, <8 x i1>, <8 x i64>, <8 x i64>, i32)
+declare void @llvm.x86.avx512.mask.scatterdiv2.df(i8*, <2 x i1>, <2 x i64>, <2 x double>, i32)
+declare void @llvm.x86.avx512.mask.scatterdiv2.di(i8*, <2 x i1>, <2 x i64>, <2 x i64>, i32)
+declare void @llvm.x86.avx512.mask.scatterdiv4.df(i8*, <4 x i1>, <4 x i64>, <4 x double>, i32)
+declare void @llvm.x86.avx512.mask.scatterdiv4.di(i8*, <4 x i1>, <4 x i64>, <4 x i64>, i32)
+declare void @llvm.x86.avx512.mask.scatterdiv4.sf(i8*, <2 x i1>, <2 x i64>, <4 x float>, i32)
+declare void @llvm.x86.avx512.mask.scatterdiv4.si(i8*, <2 x i1>, <2 x i64>, <4 x i32>, i32)
+declare void @llvm.x86.avx512.mask.scatterdiv8.sf(i8*, <4 x i1>, <4 x i64>, <4 x float>, i32)
+declare void @llvm.x86.avx512.mask.scatterdiv8.si(i8*, <4 x i1>, <4 x i64>, <4 x i32>, i32)
+declare void @llvm.x86.avx512.mask.scattersiv2.df(i8*, <2 x i1>, <4 x i32>, <2 x double>, i32)
+declare void @llvm.x86.avx512.mask.scattersiv2.di(i8*, <2 x i1>, <4 x i32>, <2 x i64>, i32)
+declare void @llvm.x86.avx512.mask.scattersiv4.df(i8*, <4 x i1>, <4 x i32>, <4 x double>, i32)
+declare void @llvm.x86.avx512.mask.scattersiv4.di(i8*, <4 x i1>, <4 x i32>, <4 x i64>, i32)
+declare void @llvm.x86.avx512.mask.scattersiv4.sf(i8*, <4 x i1>, <4 x i32>, <4 x float>, i32)
+declare void @llvm.x86.avx512.mask.scattersiv4.si(i8*, <4 x i1>, <4 x i32>, <4 x i32>, i32)
+declare void @llvm.x86.avx512.mask.scattersiv8.sf(i8*, <8 x i1>, <8 x i32>, <8 x float>, i32)
+declare void @llvm.x86.avx512.mask.scattersiv8.si(i8*, <8 x i1>, <8 x i32>, <8 x i32>, i32)
+
diff --git a/test/CodeGen/X86/avx512-hadd-hsub.ll b/test/CodeGen/X86/avx512-hadd-hsub.ll
index 00063521c6d13745d92343b4f9edc804893699ce..32d5299a88aa90f22221de2c8ac5a467981b470c 100644
--- a/test/CodeGen/X86/avx512-hadd-hsub.ll
+++ b/test/CodeGen/X86/avx512-hadd-hsub.ll
@@ -6,7 +6,7 @@ define i32 @hadd_16(<16 x i32> %x225) {
 ; KNL-LABEL: hadd_16:
 ; KNL:       # %bb.0:
 ; KNL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; KNL-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
+; KNL-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; KNL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; KNL-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; KNL-NEXT:    vmovd %xmm0, %eax
@@ -15,7 +15,7 @@ define i32 @hadd_16(<16 x i32> %x225) {
 ; SKX-LABEL: hadd_16:
 ; SKX:       # %bb.0:
 ; SKX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; SKX-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
+; SKX-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; SKX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; SKX-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; SKX-NEXT:    vmovd %xmm0, %eax
@@ -33,7 +33,7 @@ define i32 @hsub_16(<16 x i32> %x225) {
 ; KNL-LABEL: hsub_16:
 ; KNL:       # %bb.0:
 ; KNL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; KNL-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
+; KNL-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; KNL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; KNL-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
 ; KNL-NEXT:    vmovd %xmm0, %eax
@@ -42,7 +42,7 @@ define i32 @hsub_16(<16 x i32> %x225) {
 ; SKX-LABEL: hsub_16:
 ; SKX:       # %bb.0:
 ; SKX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; SKX-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
+; SKX-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; SKX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; SKX-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
 ; SKX-NEXT:    vmovd %xmm0, %eax
@@ -60,7 +60,7 @@ define float @fhadd_16(<16 x float> %x225) {
 ; KNL-LABEL: fhadd_16:
 ; KNL:       # %bb.0:
 ; KNL-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
-; KNL-NEXT:    vaddps %zmm1, %zmm0, %zmm0
+; KNL-NEXT:    vaddps %xmm1, %xmm0, %xmm0
 ; KNL-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
 ; KNL-NEXT:    vaddps %xmm1, %xmm0, %xmm0
 ; KNL-NEXT:    retq
@@ -68,7 +68,7 @@ define float @fhadd_16(<16 x float> %x225) {
 ; SKX-LABEL: fhadd_16:
 ; SKX:       # %bb.0:
 ; SKX-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
-; SKX-NEXT:    vaddps %zmm1, %zmm0, %zmm0
+; SKX-NEXT:    vaddps %xmm1, %xmm0, %xmm0
 ; SKX-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
 ; SKX-NEXT:    vaddps %xmm1, %xmm0, %xmm0
 ; SKX-NEXT:    vzeroupper
@@ -85,7 +85,7 @@ define float @fhsub_16(<16 x float> %x225) {
 ; KNL-LABEL: fhsub_16:
 ; KNL:       # %bb.0:
 ; KNL-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
-; KNL-NEXT:    vaddps %zmm1, %zmm0, %zmm0
+; KNL-NEXT:    vaddps %xmm1, %xmm0, %xmm0
 ; KNL-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
 ; KNL-NEXT:    vsubps %xmm1, %xmm0, %xmm0
 ; KNL-NEXT:    retq
@@ -93,7 +93,7 @@ define float @fhsub_16(<16 x float> %x225) {
 ; SKX-LABEL: fhsub_16:
 ; SKX:       # %bb.0:
 ; SKX-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
-; SKX-NEXT:    vaddps %zmm1, %zmm0, %zmm0
+; SKX-NEXT:    vaddps %xmm1, %xmm0, %xmm0
 ; SKX-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
 ; SKX-NEXT:    vsubps %xmm1, %xmm0, %xmm0
 ; SKX-NEXT:    vzeroupper
@@ -109,16 +109,12 @@ define float @fhsub_16(<16 x float> %x225) {
 define <16 x i32> @hadd_16_3(<16 x i32> %x225, <16 x i32> %x227) {
 ; KNL-LABEL: hadd_16_3:
 ; KNL:       # %bb.0:
-; KNL-NEXT:    vshufps {{.*#+}} ymm2 = ymm0[0,2],ymm1[0,2],ymm0[4,6],ymm1[4,6]
-; KNL-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[1,3],ymm1[1,3],ymm0[5,7],ymm1[5,7]
-; KNL-NEXT:    vpaddd %zmm0, %zmm2, %zmm0
+; KNL-NEXT:    vphaddd %ymm1, %ymm0, %ymm0
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: hadd_16_3:
 ; SKX:       # %bb.0:
-; SKX-NEXT:    vshufps {{.*#+}} ymm2 = ymm0[0,2],ymm1[0,2],ymm0[4,6],ymm1[4,6]
-; SKX-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[1,3],ymm1[1,3],ymm0[5,7],ymm1[5,7]
-; SKX-NEXT:    vpaddd %zmm0, %zmm2, %zmm0
+; SKX-NEXT:    vphaddd %ymm1, %ymm0, %ymm0
 ; SKX-NEXT:    retq
   %x226 = shufflevector <16 x i32> %x225, <16 x i32> %x227, <16 x i32> <i32 0, i32 2, i32 16, i32 18
 , i32 4, i32 6, i32 20, i32 22, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
@@ -132,16 +128,12 @@ define <16 x i32> @hadd_16_3(<16 x i32> %x225, <16 x i32> %x227) {
 define <16 x float> @fhadd_16_3(<16 x float> %x225, <16 x float> %x227) {
 ; KNL-LABEL: fhadd_16_3:
 ; KNL:       # %bb.0:
-; KNL-NEXT:    vshufps {{.*#+}} ymm2 = ymm0[0,2],ymm1[0,2],ymm0[4,6],ymm1[4,6]
-; KNL-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[1,3],ymm1[1,3],ymm0[5,7],ymm1[5,7]
-; KNL-NEXT:    vaddps %zmm0, %zmm2, %zmm0
+; KNL-NEXT:    vhaddps %ymm1, %ymm0, %ymm0
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: fhadd_16_3:
 ; SKX:       # %bb.0:
-; SKX-NEXT:    vshufps {{.*#+}} ymm2 = ymm0[0,2],ymm1[0,2],ymm0[4,6],ymm1[4,6]
-; SKX-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[1,3],ymm1[1,3],ymm0[5,7],ymm1[5,7]
-; SKX-NEXT:    vaddps %zmm0, %zmm2, %zmm0
+; SKX-NEXT:    vhaddps %ymm1, %ymm0, %ymm0
 ; SKX-NEXT:    retq
   %x226 = shufflevector <16 x float> %x225, <16 x float> %x227, <16 x i32> <i32 0, i32 2, i32 16, i32 18
 , i32 4, i32 6, i32 20, i32 22, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
@@ -154,16 +146,12 @@ define <16 x float> @fhadd_16_3(<16 x float> %x225, <16 x float> %x227) {
 define <8 x double> @fhadd_16_4(<8 x double> %x225, <8 x double> %x227) {
 ; KNL-LABEL: fhadd_16_4:
 ; KNL:       # %bb.0:
-; KNL-NEXT:    vunpcklpd {{.*#+}} ymm2 = ymm0[0],ymm1[0],ymm0[2],ymm1[2]
-; KNL-NEXT:    vunpckhpd {{.*#+}} ymm0 = ymm0[1],ymm1[1],ymm0[3],ymm1[3]
-; KNL-NEXT:    vaddpd %zmm0, %zmm2, %zmm0
+; KNL-NEXT:    vhaddpd %ymm1, %ymm0, %ymm0
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: fhadd_16_4:
 ; SKX:       # %bb.0:
-; SKX-NEXT:    vunpcklpd {{.*#+}} ymm2 = ymm0[0],ymm1[0],ymm0[2],ymm1[2]
-; SKX-NEXT:    vunpckhpd {{.*#+}} ymm0 = ymm0[1],ymm1[1],ymm0[3],ymm1[3]
-; SKX-NEXT:    vaddpd %zmm0, %zmm2, %zmm0
+; SKX-NEXT:    vhaddpd %ymm1, %ymm0, %ymm0
 ; SKX-NEXT:    retq
   %x226 = shufflevector <8 x double> %x225, <8 x double> %x227, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 undef, i32 undef, i32 undef, i32 undef>
   %x228 = shufflevector <8 x double> %x225, <8 x double> %x227, <8 x i32> <i32 1, i32 9, i32 3, i32 11, i32 undef ,i32 undef, i32 undef, i32 undef>
@@ -174,16 +162,12 @@ define <8 x double> @fhadd_16_4(<8 x double> %x225, <8 x double> %x227) {
 define <4 x double> @fadd_noundef_low(<8 x double> %x225, <8 x double> %x227) {
 ; KNL-LABEL: fadd_noundef_low:
 ; KNL:       # %bb.0:
-; KNL-NEXT:    vunpcklpd {{.*#+}} ymm2 = ymm0[0],ymm1[0],ymm0[2],ymm1[2]
-; KNL-NEXT:    vunpckhpd {{.*#+}} ymm0 = ymm0[1],ymm1[1],ymm0[3],ymm1[3]
-; KNL-NEXT:    vaddpd %ymm0, %ymm2, %ymm0
+; KNL-NEXT:    vhaddpd %ymm1, %ymm0, %ymm0
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: fadd_noundef_low:
 ; SKX:       # %bb.0:
-; SKX-NEXT:    vunpcklpd {{.*#+}} ymm2 = ymm0[0],ymm1[0],ymm0[2],ymm1[2]
-; SKX-NEXT:    vunpckhpd {{.*#+}} ymm0 = ymm0[1],ymm1[1],ymm0[3],ymm1[3]
-; SKX-NEXT:    vaddpd %ymm0, %ymm2, %ymm0
+; SKX-NEXT:    vhaddpd %ymm1, %ymm0, %ymm0
 ; SKX-NEXT:    retq
   %x226 = shufflevector <8 x double> %x225, <8 x double> %x227, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
   %x228 = shufflevector <8 x double> %x225, <8 x double> %x227, <8 x i32> <i32 1, i32 9, i32 3, i32 11, i32 5 ,i32 13, i32 7, i32 15>
@@ -221,16 +205,12 @@ define <4 x double> @fadd_noundef_high(<8 x double> %x225, <8 x double> %x227) {
 define <8 x i32> @hadd_16_3_sv(<16 x i32> %x225, <16 x i32> %x227) {
 ; KNL-LABEL: hadd_16_3_sv:
 ; KNL:       # %bb.0:
-; KNL-NEXT:    vshufps {{.*#+}} ymm2 = ymm0[0,2],ymm1[0,2],ymm0[4,6],ymm1[4,6]
-; KNL-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[1,3],ymm1[1,3],ymm0[5,7],ymm1[5,7]
-; KNL-NEXT:    vpaddd %ymm0, %ymm2, %ymm0
+; KNL-NEXT:    vphaddd %ymm1, %ymm0, %ymm0
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: hadd_16_3_sv:
 ; SKX:       # %bb.0:
-; SKX-NEXT:    vshufps {{.*#+}} ymm2 = ymm0[0,2],ymm1[0,2],ymm0[4,6],ymm1[4,6]
-; SKX-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[1,3],ymm1[1,3],ymm0[5,7],ymm1[5,7]
-; SKX-NEXT:    vpaddd %ymm0, %ymm2, %ymm0
+; SKX-NEXT:    vphaddd %ymm1, %ymm0, %ymm0
 ; SKX-NEXT:    retq
   %x226 = shufflevector <16 x i32> %x225, <16 x i32> %x227, <16 x i32> <i32 0, i32 2, i32 16, i32 18
 , i32 4, i32 6, i32 20, i32 22, i32 8, i32 10, i32 24, i32 26, i32 12, i32 14, i32 28, i32 30>
diff --git a/test/CodeGen/X86/avx512-i1test.ll b/test/CodeGen/X86/avx512-i1test.ll
index 7cf86fe747faccc2f3215f66acad0a845f415309..108a90814a44c04e2d36e49d2264f4b611422007 100644
--- a/test/CodeGen/X86/avx512-i1test.ll
+++ b/test/CodeGen/X86/avx512-i1test.ll
@@ -15,16 +15,15 @@ define void @func() {
 ; CHECK-NEXT:    retq
 ; CHECK-NEXT:  .LBB0_1: # %bb56
 ; CHECK-NEXT:    xorl %eax, %eax
-; CHECK-NEXT:    jmp .LBB0_2
 ; CHECK-NEXT:    .p2align 4, 0x90
-; CHECK-NEXT:  .LBB0_3: # %bb35
-; CHECK-NEXT:    # in Loop: Header=BB0_2 Depth=1
-; CHECK-NEXT:    testb %al, %al
 ; CHECK-NEXT:  .LBB0_2: # %bb33
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    testb %al, %al
 ; CHECK-NEXT:    jne .LBB0_2
-; CHECK-NEXT:    jmp .LBB0_3
+; CHECK-NEXT:  # %bb.3: # %bb35
+; CHECK-NEXT:    # in Loop: Header=BB0_2 Depth=1
+; CHECK-NEXT:    testb %al, %al
+; CHECK-NEXT:    jmp .LBB0_2
 bb1:
   br i1 undef, label %L_10, label %L_10
 
diff --git a/test/CodeGen/X86/avx512-insert-extract.ll b/test/CodeGen/X86/avx512-insert-extract.ll
index 05542ac1c97d29e343d72d17e8d4caf6d41f021c..3d194477f9a31699c52bf8828f34aa4b27c4fc76 100644
--- a/test/CodeGen/X86/avx512-insert-extract.ll
+++ b/test/CodeGen/X86/avx512-insert-extract.ll
@@ -270,7 +270,7 @@ define i64 @test14(<8 x i64>%a, <8 x i64>%b, i64 %a1, i64 %b1) {
 ; SKX:       ## %bb.0:
 ; SKX-NEXT:    movq %rdi, %rax
 ; SKX-NEXT:    vpcmpgtq %zmm0, %zmm1, %k0
-; SKX-NEXT:    kshiftrw $4, %k0, %k0
+; SKX-NEXT:    kshiftrb $4, %k0, %k0
 ; SKX-NEXT:    kmovd %k0, %ecx
 ; SKX-NEXT:    testb $1, %cl
 ; SKX-NEXT:    cmoveq %rsi, %rax
@@ -976,7 +976,7 @@ define zeroext i8 @test_extractelement_v4i1(<4 x i32> %a, <4 x i32> %b) {
 ; SKX-LABEL: test_extractelement_v4i1:
 ; SKX:       ## %bb.0:
 ; SKX-NEXT:    vpcmpnleud %xmm1, %xmm0, %k0
-; SKX-NEXT:    kshiftrw $3, %k0, %k0
+; SKX-NEXT:    kshiftrb $3, %k0, %k0
 ; SKX-NEXT:    kmovd %k0, %eax
 ; SKX-NEXT:    andl $1, %eax
 ; SKX-NEXT:    retq
diff --git a/test/CodeGen/X86/avx512-intrinsics-fast-isel.ll b/test/CodeGen/X86/avx512-intrinsics-fast-isel.ll
index e04d8e3194420f4c13768b4de240bda0d2678aa5..6a6fbe333aaf66004e1113cf995b9937e4f948fc 100644
--- a/test/CodeGen/X86/avx512-intrinsics-fast-isel.ll
+++ b/test/CodeGen/X86/avx512-intrinsics-fast-isel.ll
@@ -169,7 +169,8 @@ entry:
 define <16 x float> @test_mm512_mask_shuffle_f32x4(<16 x float> %__W, i16 zeroext %__U, <16 x float> %__A, <16 x float> %__B) {
 ; X86-LABEL: test_mm512_mask_shuffle_f32x4:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vshuff32x4 {{.*#+}} zmm0 {%k1} = zmm1[0,1,2,3,4,5,6,7],zmm2[0,1,2,3,0,1,2,3]
 ; X86-NEXT:    retl
 ;
@@ -188,7 +189,8 @@ entry:
 define <16 x float> @test_mm512_maskz_shuffle_f32x4(i16 zeroext %__U, <16 x float> %__A, <16 x float> %__B) {
 ; X86-LABEL: test_mm512_maskz_shuffle_f32x4:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vshuff32x4 {{.*#+}} zmm0 {%k1} {z} = zmm0[0,1,2,3,4,5,6,7],zmm1[0,1,2,3,0,1,2,3]
 ; X86-NEXT:    retl
 ;
@@ -267,7 +269,8 @@ entry:
 define <8 x i64> @test_mm512_mask_shuffle_i32x4(<8 x i64> %__W, i16 zeroext %__U, <8 x i64> %__A, <8 x i64> %__B) local_unnamed_addr #0 {
 ; X86-LABEL: test_mm512_mask_shuffle_i32x4:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vshufi32x4 {{.*#+}} zmm0 {%k1} = zmm1[0,1,2,3,4,5,6,7],zmm2[0,1,2,3,0,1,2,3]
 ; X86-NEXT:    retl
 ;
@@ -289,7 +292,8 @@ entry:
 define <8 x i64> @test_mm512_maskz_shuffle_i32x4(i16 zeroext %__U, <8 x i64> %__A, <8 x i64> %__B) local_unnamed_addr #0 {
 ; X86-LABEL: test_mm512_maskz_shuffle_i32x4:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vshufi32x4 {{.*#+}} zmm0 {%k1} {z} = zmm0[0,1,2,3,4,5,6,7],zmm1[0,1,2,3,0,1,2,3]
 ; X86-NEXT:    retl
 ;
@@ -377,7 +381,8 @@ entry:
 define zeroext i16 @test_mm512_mask_testn_epi32_mask(i16 zeroext %__U, <8 x i64> %__A, <8 x i64> %__B) {
 ; X86-LABEL: test_mm512_mask_testn_epi32_mask:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vptestnmd %zmm0, %zmm1, %k0 {%k1}
 ; X86-NEXT:    kmovw %k0, %eax
 ; X86-NEXT:    movzwl %ax, %eax
@@ -448,7 +453,8 @@ entry:
 define zeroext i16 @test_mm512_mask_test_epi32_mask(i16 zeroext %__U, <8 x i64> %__A, <8 x i64> %__B) {
 ; X86-LABEL: test_mm512_mask_test_epi32_mask:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vptestmd %zmm0, %zmm1, %k0 {%k1}
 ; X86-NEXT:    kmovw %k0, %eax
 ; X86-NEXT:    movzwl %ax, %eax
@@ -505,7 +511,8 @@ define <8 x i64> @test_mm512_mask_set1_epi32(<8 x i64> %__O, i16 zeroext %__M, i
 ; X86-LABEL: test_mm512_mask_set1_epi32:
 ; X86:       # %bb.0: # %entry
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    kmovw %ecx, %k1
 ; X86-NEXT:    vpbroadcastd %eax, %zmm0 {%k1}
 ; X86-NEXT:    retl
 ;
@@ -528,7 +535,8 @@ define <8 x i64> @test_mm512_maskz_set1_epi32(i16 zeroext %__M, i32 %__A)  {
 ; X86-LABEL: test_mm512_maskz_set1_epi32:
 ; X86:       # %bb.0: # %entry
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    kmovw %ecx, %k1
 ; X86-NEXT:    vpbroadcastd %eax, %zmm0 {%k1} {z}
 ; X86-NEXT:    retl
 ;
@@ -607,7 +615,8 @@ define <8 x i64> @test_mm512_broadcastd_epi32(<2 x i64> %a0) {
 define <8 x i64> @test_mm512_mask_broadcastd_epi32(<8 x i64> %a0, i16 %a1, <2 x i64> %a2) {
 ; X86-LABEL: test_mm512_mask_broadcastd_epi32:
 ; X86:       # %bb.0:
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vpbroadcastd %xmm1, %zmm0 {%k1}
 ; X86-NEXT:    retl
 ;
@@ -628,7 +637,8 @@ define <8 x i64> @test_mm512_mask_broadcastd_epi32(<8 x i64> %a0, i16 %a1, <2 x
 define <8 x i64> @test_mm512_maskz_broadcastd_epi32(i16 %a0, <2 x i64> %a1) {
 ; X86-LABEL: test_mm512_maskz_broadcastd_epi32:
 ; X86:       # %bb.0:
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vpbroadcastd %xmm0, %zmm0 {%k1} {z}
 ; X86-NEXT:    retl
 ;
@@ -751,7 +761,8 @@ define <16 x float> @test_mm512_broadcastss_ps(<4 x float> %a0) {
 define <16 x float> @test_mm512_mask_broadcastss_ps(<16 x float> %a0, i16 %a1, <4 x float> %a2) {
 ; X86-LABEL: test_mm512_mask_broadcastss_ps:
 ; X86:       # %bb.0:
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vbroadcastss %xmm1, %zmm0 {%k1}
 ; X86-NEXT:    retl
 ;
@@ -769,7 +780,8 @@ define <16 x float> @test_mm512_mask_broadcastss_ps(<16 x float> %a0, i16 %a1, <
 define <16 x float> @test_mm512_maskz_broadcastss_ps(i16 %a0, <4 x float> %a1) {
 ; X86-LABEL: test_mm512_maskz_broadcastss_ps:
 ; X86:       # %bb.0:
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vbroadcastss %xmm0, %zmm0 {%k1} {z}
 ; X86-NEXT:    retl
 ;
@@ -843,7 +855,8 @@ define <16 x float> @test_mm512_movehdup_ps(<16 x float> %a0) {
 define <16 x float> @test_mm512_mask_movehdup_ps(<16 x float> %a0, i16 %a1, <16 x float> %a2) {
 ; X86-LABEL: test_mm512_mask_movehdup_ps:
 ; X86:       # %bb.0:
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vmovshdup {{.*#+}} zmm0 {%k1} = zmm1[1,1,3,3,5,5,7,7,9,9,11,11,13,13,15,15]
 ; X86-NEXT:    retl
 ;
@@ -861,7 +874,8 @@ define <16 x float> @test_mm512_mask_movehdup_ps(<16 x float> %a0, i16 %a1, <16
 define <16 x float> @test_mm512_maskz_movehdup_ps(i16 %a0, <16 x float> %a1) {
 ; X86-LABEL: test_mm512_maskz_movehdup_ps:
 ; X86:       # %bb.0:
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vmovshdup {{.*#+}} zmm0 {%k1} {z} = zmm0[1,1,3,3,5,5,7,7,9,9,11,11,13,13,15,15]
 ; X86-NEXT:    retl
 ;
@@ -888,7 +902,8 @@ define <16 x float> @test_mm512_moveldup_ps(<16 x float> %a0) {
 define <16 x float> @test_mm512_mask_moveldup_ps(<16 x float> %a0, i16 %a1, <16 x float> %a2) {
 ; X86-LABEL: test_mm512_mask_moveldup_ps:
 ; X86:       # %bb.0:
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vmovsldup {{.*#+}} zmm0 {%k1} = zmm1[0,0,2,2,4,4,6,6,8,8,10,10,12,12,14,14]
 ; X86-NEXT:    retl
 ;
@@ -906,7 +921,8 @@ define <16 x float> @test_mm512_mask_moveldup_ps(<16 x float> %a0, i16 %a1, <16
 define <16 x float> @test_mm512_maskz_moveldup_ps(i16 %a0, <16 x float> %a1) {
 ; X86-LABEL: test_mm512_maskz_moveldup_ps:
 ; X86:       # %bb.0:
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vmovsldup {{.*#+}} zmm0 {%k1} {z} = zmm0[0,0,2,2,4,4,6,6,8,8,10,10,12,12,14,14]
 ; X86-NEXT:    retl
 ;
@@ -980,7 +996,8 @@ define <16 x float> @test_mm512_permute_ps(<16 x float> %a0) {
 define <16 x float> @test_mm512_mask_permute_ps(<16 x float> %a0, i16 %a1, <16 x float> %a2) {
 ; X86-LABEL: test_mm512_mask_permute_ps:
 ; X86:       # %bb.0:
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vpermilps {{.*#+}} zmm0 {%k1} = zmm1[2,0,0,0,6,4,4,4,10,8,8,8,14,12,12,12]
 ; X86-NEXT:    retl
 ;
@@ -998,7 +1015,8 @@ define <16 x float> @test_mm512_mask_permute_ps(<16 x float> %a0, i16 %a1, <16 x
 define <16 x float> @test_mm512_maskz_permute_ps(i16 %a0, <16 x float> %a1) {
 ; X86-LABEL: test_mm512_maskz_permute_ps:
 ; X86:       # %bb.0:
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vpermilps {{.*#+}} zmm0 {%k1} {z} = zmm0[2,0,0,0,6,4,4,4,10,8,8,8,14,12,12,12]
 ; X86-NEXT:    retl
 ;
@@ -1121,7 +1139,8 @@ define <8 x i64> @test_mm512_shuffle_epi32(<8 x i64> %a0) {
 define <8 x i64> @test_mm512_mask_shuffle_epi32(<8 x i64> %a0, i16 %a1, <8 x i64> %a2) {
 ; X86-LABEL: test_mm512_mask_shuffle_epi32:
 ; X86:       # %bb.0:
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vpshufd {{.*#+}} zmm0 {%k1} = zmm1[1,0,0,0,5,4,4,4,9,8,8,8,13,12,12,12]
 ; X86-NEXT:    retl
 ;
@@ -1142,7 +1161,8 @@ define <8 x i64> @test_mm512_mask_shuffle_epi32(<8 x i64> %a0, i16 %a1, <8 x i64
 define <8 x i64> @test_mm512_maskz_shuffle_epi32(i16 %a0, <8 x i64> %a1) {
 ; X86-LABEL: test_mm512_maskz_shuffle_epi32:
 ; X86:       # %bb.0:
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vpshufd {{.*#+}} zmm0 {%k1} {z} = zmm0[1,0,0,0,5,4,4,4,9,8,8,8,13,12,12,12]
 ; X86-NEXT:    retl
 ;
@@ -1221,7 +1241,8 @@ define <8 x i64> @test_mm512_unpackhi_epi32(<8 x i64> %a0, <8 x i64> %a1) {
 define <8 x i64> @test_mm512_mask_unpackhi_epi32(<8 x i64> %a0, i16 %a1, <8 x i64> %a2, <8 x i64> %a3) {
 ; X86-LABEL: test_mm512_mask_unpackhi_epi32:
 ; X86:       # %bb.0:
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vpunpckhdq {{.*#+}} zmm0 {%k1} = zmm1[2],zmm2[2],zmm1[3],zmm2[3],zmm1[6],zmm2[6],zmm1[7],zmm2[7],zmm1[10],zmm2[10],zmm1[11],zmm2[11],zmm1[14],zmm2[14],zmm1[15],zmm2[15]
 ; X86-NEXT:    retl
 ;
@@ -1243,7 +1264,8 @@ define <8 x i64> @test_mm512_mask_unpackhi_epi32(<8 x i64> %a0, i16 %a1, <8 x i6
 define <8 x i64> @test_mm512_maskz_unpackhi_epi32(i16 %a0, <8 x i64> %a1, <8 x i64> %a2) {
 ; X86-LABEL: test_mm512_maskz_unpackhi_epi32:
 ; X86:       # %bb.0:
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vpunpckhdq {{.*#+}} zmm0 {%k1} {z} = zmm0[2],zmm1[2],zmm0[3],zmm1[3],zmm0[6],zmm1[6],zmm0[7],zmm1[7],zmm0[10],zmm1[10],zmm0[11],zmm1[11],zmm0[14],zmm1[14],zmm0[15],zmm1[15]
 ; X86-NEXT:    retl
 ;
@@ -1367,7 +1389,8 @@ define <16 x float> @test_mm512_unpackhi_ps(<16 x float> %a0, <16 x float> %a1)
 define <16 x float> @test_mm512_mask_unpackhi_ps(<16 x float> %a0, i16 %a1, <16 x float> %a2, <16 x float> %a3) {
 ; X86-LABEL: test_mm512_mask_unpackhi_ps:
 ; X86:       # %bb.0:
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vunpckhps {{.*#+}} zmm0 {%k1} = zmm1[2],zmm2[2],zmm1[3],zmm2[3],zmm1[6],zmm2[6],zmm1[7],zmm2[7],zmm1[10],zmm2[10],zmm1[11],zmm2[11],zmm1[14],zmm2[14],zmm1[15],zmm2[15]
 ; X86-NEXT:    retl
 ;
@@ -1385,7 +1408,8 @@ define <16 x float> @test_mm512_mask_unpackhi_ps(<16 x float> %a0, i16 %a1, <16
 define <16 x float> @test_mm512_maskz_unpackhi_ps(i16 %a0, <16 x float> %a1, <16 x float> %a2) {
 ; X86-LABEL: test_mm512_maskz_unpackhi_ps:
 ; X86:       # %bb.0:
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vunpckhps {{.*#+}} zmm0 {%k1} {z} = zmm0[2],zmm1[2],zmm0[3],zmm1[3],zmm0[6],zmm1[6],zmm0[7],zmm1[7],zmm0[10],zmm1[10],zmm0[11],zmm1[11],zmm0[14],zmm1[14],zmm0[15],zmm1[15]
 ; X86-NEXT:    retl
 ;
@@ -1415,7 +1439,8 @@ define <8 x i64> @test_mm512_unpacklo_epi32(<8 x i64> %a0, <8 x i64> %a1) {
 define <8 x i64> @test_mm512_mask_unpacklo_epi32(<8 x i64> %a0, i16 %a1, <8 x i64> %a2, <8 x i64> %a3) {
 ; X86-LABEL: test_mm512_mask_unpacklo_epi32:
 ; X86:       # %bb.0:
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vpunpckldq {{.*#+}} zmm0 {%k1} = zmm1[0],zmm2[0],zmm1[1],zmm2[1],zmm1[4],zmm2[4],zmm1[5],zmm2[5],zmm1[8],zmm2[8],zmm1[9],zmm2[9],zmm1[12],zmm2[12],zmm1[13],zmm2[13]
 ; X86-NEXT:    retl
 ;
@@ -1437,7 +1462,8 @@ define <8 x i64> @test_mm512_mask_unpacklo_epi32(<8 x i64> %a0, i16 %a1, <8 x i6
 define <8 x i64> @test_mm512_maskz_unpacklo_epi32(i16 %a0, <8 x i64> %a1, <8 x i64> %a2) {
 ; X86-LABEL: test_mm512_maskz_unpacklo_epi32:
 ; X86:       # %bb.0:
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vpunpckldq {{.*#+}} zmm0 {%k1} {z} = zmm0[0],zmm1[0],zmm0[1],zmm1[1],zmm0[4],zmm1[4],zmm0[5],zmm1[5],zmm0[8],zmm1[8],zmm0[9],zmm1[9],zmm0[12],zmm1[12],zmm0[13],zmm1[13]
 ; X86-NEXT:    retl
 ;
@@ -1561,7 +1587,8 @@ define <16 x float> @test_mm512_unpacklo_ps(<16 x float> %a0, <16 x float> %a1)
 define <16 x float> @test_mm512_mask_unpacklo_ps(<16 x float> %a0, i16 %a1, <16 x float> %a2, <16 x float> %a3) {
 ; X86-LABEL: test_mm512_mask_unpacklo_ps:
 ; X86:       # %bb.0:
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vunpcklps {{.*#+}} zmm0 {%k1} = zmm1[0],zmm2[0],zmm1[1],zmm2[1],zmm1[4],zmm2[4],zmm1[5],zmm2[5],zmm1[8],zmm2[8],zmm1[9],zmm2[9],zmm1[12],zmm2[12],zmm1[13],zmm2[13]
 ; X86-NEXT:    retl
 ;
@@ -1579,7 +1606,8 @@ define <16 x float> @test_mm512_mask_unpacklo_ps(<16 x float> %a0, i16 %a1, <16
 define <16 x float> @test_mm512_maskz_unpacklo_ps(i16 %a0, <16 x float> %a1, <16 x float> %a2) {
 ; X86-LABEL: test_mm512_maskz_unpacklo_ps:
 ; X86:       # %bb.0:
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vunpcklps {{.*#+}} zmm0 {%k1} {z} = zmm0[0],zmm1[0],zmm0[1],zmm1[1],zmm0[4],zmm1[4],zmm0[5],zmm1[5],zmm0[8],zmm1[8],zmm0[9],zmm1[9],zmm0[12],zmm1[12],zmm0[13],zmm1[13]
 ; X86-NEXT:    retl
 ;
@@ -1651,10 +1679,6 @@ define <8 x i64> @test_mm512_zextsi256_si512(<4 x i64> %a0) nounwind {
 define <8 x i64> @test_mm512_mul_epi32(<8 x i64> %__A, <8 x i64> %__B) nounwind {
 ; CHECK-LABEL: test_mm512_mul_epi32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vpsllq $32, %zmm0, %zmm0
-; CHECK-NEXT:    vpsraq $32, %zmm0, %zmm0
-; CHECK-NEXT:    vpsllq $32, %zmm1, %zmm1
-; CHECK-NEXT:    vpsraq $32, %zmm1, %zmm1
 ; CHECK-NEXT:    vpmuldq %zmm0, %zmm1, %zmm0
 ; CHECK-NEXT:    ret{{[l|q]}}
   %tmp = shl <8 x i64> %__A, <i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32, i64 32>
@@ -1718,11 +1742,6 @@ entry:
 define <8 x i64> @test_mm512_mul_epu32(<8 x i64> %__A, <8 x i64> %__B) nounwind {
 ; CHECK-LABEL: test_mm512_mul_epu32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    movw $-21846, %ax # imm = 0xAAAA
-; CHECK-NEXT:    kmovw %eax, %k0
-; CHECK-NEXT:    knotw %k0, %k1
-; CHECK-NEXT:    vmovdqa32 %zmm0, %zmm0 {%k1} {z}
-; CHECK-NEXT:    vmovdqa32 %zmm1, %zmm1 {%k1} {z}
 ; CHECK-NEXT:    vpmuludq %zmm0, %zmm1, %zmm0
 ; CHECK-NEXT:    ret{{[l|q]}}
   %tmp = and <8 x i64> %__A, <i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295>
@@ -1987,7 +2006,8 @@ entry:
 define <2 x i64> @test_mm512_mask_cvtepi32_epi8(<2 x i64> %__O, i16 zeroext %__M, <8 x i64> %__A) {
 ; X86-LABEL: test_mm512_mask_cvtepi32_epi8:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vpmovdb %zmm1, %xmm0 {%k1}
 ; X86-NEXT:    vzeroupper
 ; X86-NEXT:    retl
@@ -2009,7 +2029,8 @@ entry:
 define <2 x i64> @test_mm512_maskz_cvtepi32_epi8(i16 zeroext %__M, <8 x i64> %__A) {
 ; X86-LABEL: test_mm512_maskz_cvtepi32_epi8:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vpmovdb %zmm0, %xmm0 {%k1} {z}
 ; X86-NEXT:    vzeroupper
 ; X86-NEXT:    retl
@@ -2158,7 +2179,8 @@ declare <16 x i32> @llvm.x86.avx512.pternlog.d.512(<16 x i32>, <16 x i32>, <16 x
 define <8 x i64> @test_mm512_mask_ternarylogic_epi32(<8 x i64> %__A, i16 zeroext %__U, <8 x i64> %__B, <8 x i64> %__C) {
 ; X86-LABEL: test_mm512_mask_ternarylogic_epi32:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vpternlogd $4, %zmm2, %zmm1, %zmm0 {%k1}
 ; X86-NEXT:    retl
 ;
@@ -2181,7 +2203,8 @@ entry:
 define <8 x i64> @test_mm512_maskz_ternarylogic_epi32(i16 zeroext %__U, <8 x i64> %__A, <8 x i64> %__B, <8 x i64> %__C) {
 ; X86-LABEL: test_mm512_maskz_ternarylogic_epi32:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vpternlogd $4, %zmm2, %zmm1, %zmm0 {%k1} {z}
 ; X86-NEXT:    retl
 ;
@@ -2258,7 +2281,8 @@ declare <16 x i32> @llvm.x86.avx512.vpermi2var.d.512(<16 x i32>, <16 x i32>, <16
 define <8 x i64> @test_mm512_mask2_permutex2var_epi32(<8 x i64> %__A, <8 x i64> %__I, i16 zeroext %__U, <8 x i64> %__B) {
 ; X86-LABEL: test_mm512_mask2_permutex2var_epi32:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vpermi2d %zmm2, %zmm0, %zmm1 {%k1}
 ; X86-NEXT:    vmovdqa64 %zmm1, %zmm0
 ; X86-NEXT:    retl
@@ -2310,7 +2334,8 @@ declare <16 x float> @llvm.x86.avx512.vpermi2var.ps.512(<16 x float>, <16 x i32>
 define <16 x float> @test_mm512_mask2_permutex2var_ps(<16 x float> %__A, <8 x i64> %__I, i16 zeroext %__U, <16 x float> %__B) {
 ; X86-LABEL: test_mm512_mask2_permutex2var_ps:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vpermi2ps %zmm2, %zmm0, %zmm1 {%k1}
 ; X86-NEXT:    vmovaps %zmm1, %zmm0
 ; X86-NEXT:    retl
@@ -2371,7 +2396,8 @@ entry:
 define <8 x i64> @test_mm512_maskz_permutex2var_epi32(i16 zeroext %__U, <8 x i64> %__A, <8 x i64> %__I, <8 x i64> %__B) {
 ; X86-LABEL: test_mm512_maskz_permutex2var_epi32:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vpermt2d %zmm2, %zmm1, %zmm0 {%k1} {z}
 ; X86-NEXT:    retl
 ;
@@ -2394,7 +2420,8 @@ entry:
 define <8 x i64> @test_mm512_mask_permutex2var_epi32(<8 x i64> %__A, i16 zeroext %__U, <8 x i64> %__I, <8 x i64> %__B) {
 ; X86-LABEL: test_mm512_mask_permutex2var_epi32:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vpermt2d %zmm2, %zmm1, %zmm0 {%k1}
 ; X86-NEXT:    retl
 ;
@@ -2478,7 +2505,8 @@ entry:
 define <16 x float> @test_mm512_mask_permutex2var_ps(<16 x float> %__A, i16 zeroext %__U, <8 x i64> %__I, <16 x float> %__B) {
 ; X86-LABEL: test_mm512_mask_permutex2var_ps:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vpermt2ps %zmm2, %zmm1, %zmm0 {%k1}
 ; X86-NEXT:    retl
 ;
@@ -2498,7 +2526,8 @@ entry:
 define <16 x float> @test_mm512_maskz_permutex2var_ps(i16 zeroext %__U, <16 x float> %__A, <8 x i64> %__I, <16 x float> %__B) {
 ; X86-LABEL: test_mm512_maskz_permutex2var_ps:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vpermt2ps %zmm2, %zmm1, %zmm0 {%k1} {z}
 ; X86-NEXT:    retl
 ;
@@ -3436,7 +3465,8 @@ declare <16 x float> @llvm.x86.avx512.vfmadd.ps.512(<16 x float>, <16 x float>,
 define <16 x float> @test_mm512_mask_fmadd_round_ps(<16 x float> %__A, i16 zeroext %__U, <16 x float> %__B, <16 x float> %__C) {
 ; X86-LABEL: test_mm512_mask_fmadd_round_ps:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vfmadd132ps {rn-sae}, %zmm1, %zmm2, %zmm0 {%k1}
 ; X86-NEXT:    retl
 ;
@@ -3455,7 +3485,8 @@ entry:
 define <16 x float> @test_mm512_mask3_fmadd_round_ps(<16 x float> %__A, <16 x float> %__B, <16 x float> %__C, i16 zeroext %__U) {
 ; X86-LABEL: test_mm512_mask3_fmadd_round_ps:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vfmadd231ps {rn-sae}, %zmm1, %zmm0, %zmm2 {%k1}
 ; X86-NEXT:    vmovaps %zmm2, %zmm0
 ; X86-NEXT:    retl
@@ -3476,7 +3507,8 @@ entry:
 define <16 x float> @test_mm512_maskz_fmadd_round_ps(i16 zeroext %__U, <16 x float> %__A, <16 x float> %__B, <16 x float> %__C) {
 ; X86-LABEL: test_mm512_maskz_fmadd_round_ps:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vfmadd213ps {rn-sae}, %zmm2, %zmm1, %zmm0 {%k1} {z}
 ; X86-NEXT:    retl
 ;
@@ -3513,7 +3545,8 @@ entry:
 define <16 x float> @test_mm512_mask_fmsub_round_ps(<16 x float> %__A, i16 zeroext %__U, <16 x float> %__B, <16 x float> %__C) {
 ; X86-LABEL: test_mm512_mask_fmsub_round_ps:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vfmsub132ps {rn-sae}, %zmm1, %zmm2, %zmm0 {%k1}
 ; X86-NEXT:    retl
 ;
@@ -3533,7 +3566,8 @@ entry:
 define <16 x float> @test_mm512_maskz_fmsub_round_ps(i16 zeroext %__U, <16 x float> %__A, <16 x float> %__B, <16 x float> %__C) {
 ; X86-LABEL: test_mm512_maskz_fmsub_round_ps:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vfmsub213ps {rn-sae}, %zmm2, %zmm1, %zmm0 {%k1} {z}
 ; X86-NEXT:    retl
 ;
@@ -3571,7 +3605,8 @@ entry:
 define <16 x float> @test_mm512_mask3_fnmadd_round_ps(<16 x float> %__A, <16 x float> %__B, <16 x float> %__C, i16 zeroext %__U) {
 ; X86-LABEL: test_mm512_mask3_fnmadd_round_ps:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vfnmadd231ps {rn-sae}, %zmm1, %zmm0, %zmm2 {%k1}
 ; X86-NEXT:    vmovaps %zmm2, %zmm0
 ; X86-NEXT:    retl
@@ -3593,7 +3628,8 @@ entry:
 define <16 x float> @test_mm512_maskz_fnmadd_round_ps(i16 zeroext %__U, <16 x float> %__A, <16 x float> %__B, <16 x float> %__C) {
 ; X86-LABEL: test_mm512_maskz_fnmadd_round_ps:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vfnmadd213ps {rn-sae}, %zmm2, %zmm1, %zmm0 {%k1} {z}
 ; X86-NEXT:    retl
 ;
@@ -3628,7 +3664,8 @@ entry:
 define <16 x float> @test_mm512_maskz_fnmsub_round_ps(i16 zeroext %__U, <16 x float> %__A, <16 x float> %__B, <16 x float> %__C) {
 ; X86-LABEL: test_mm512_maskz_fnmsub_round_ps:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vfnmsub213ps {rn-sae}, %zmm2, %zmm1, %zmm0 {%k1} {z}
 ; X86-NEXT:    retl
 ;
@@ -3659,7 +3696,8 @@ entry:
 define <16 x float> @test_mm512_mask_fmadd_ps(<16 x float> %__A, i16 zeroext %__U, <16 x float> %__B, <16 x float> %__C) {
 ; X86-LABEL: test_mm512_mask_fmadd_ps:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vfmadd132ps {{.*#+}} zmm0 = (zmm0 * zmm1) + zmm2
 ; X86-NEXT:    retl
 ;
@@ -3678,7 +3716,8 @@ entry:
 define <16 x float> @test_mm512_mask3_fmadd_ps(<16 x float> %__A, <16 x float> %__B, <16 x float> %__C, i16 zeroext %__U) {
 ; X86-LABEL: test_mm512_mask3_fmadd_ps:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vfmadd231ps {{.*#+}} zmm2 = (zmm0 * zmm1) + zmm2
 ; X86-NEXT:    vmovaps %zmm2, %zmm0
 ; X86-NEXT:    retl
@@ -3699,7 +3738,8 @@ entry:
 define <16 x float> @test_mm512_maskz_fmadd_ps(i16 zeroext %__U, <16 x float> %__A, <16 x float> %__B, <16 x float> %__C) {
 ; X86-LABEL: test_mm512_maskz_fmadd_ps:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vfmadd213ps {{.*#+}} zmm0 = (zmm1 * zmm0) + zmm2
 ; X86-NEXT:    retl
 ;
@@ -3736,7 +3776,8 @@ entry:
 define <16 x float> @test_mm512_mask_fmsub_ps(<16 x float> %__A, i16 zeroext %__U, <16 x float> %__B, <16 x float> %__C) {
 ; X86-LABEL: test_mm512_mask_fmsub_ps:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vfmsub132ps {{.*#+}} zmm0 = (zmm0 * zmm1) - zmm2
 ; X86-NEXT:    retl
 ;
@@ -3756,7 +3797,8 @@ entry:
 define <16 x float> @test_mm512_maskz_fmsub_ps(i16 zeroext %__U, <16 x float> %__A, <16 x float> %__B, <16 x float> %__C) {
 ; X86-LABEL: test_mm512_maskz_fmsub_ps:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vfmsub213ps {{.*#+}} zmm0 = (zmm1 * zmm0) - zmm2
 ; X86-NEXT:    retl
 ;
@@ -3794,7 +3836,8 @@ entry:
 define <16 x float> @test_mm512_mask3_fnmadd_ps(<16 x float> %__A, <16 x float> %__B, <16 x float> %__C, i16 zeroext %__U) {
 ; X86-LABEL: test_mm512_mask3_fnmadd_ps:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vfnmadd231ps {{.*#+}} zmm2 = -(zmm0 * zmm1) + zmm2
 ; X86-NEXT:    vmovaps %zmm2, %zmm0
 ; X86-NEXT:    retl
@@ -3816,7 +3859,8 @@ entry:
 define <16 x float> @test_mm512_maskz_fnmadd_ps(i16 zeroext %__U, <16 x float> %__A, <16 x float> %__B, <16 x float> %__C) {
 ; X86-LABEL: test_mm512_maskz_fnmadd_ps:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vfnmadd213ps {{.*#+}} zmm0 = -(zmm1 * zmm0) + zmm2
 ; X86-NEXT:    retl
 ;
@@ -3851,7 +3895,8 @@ entry:
 define <16 x float> @test_mm512_maskz_fnmsub_ps(i16 zeroext %__U, <16 x float> %__A, <16 x float> %__B, <16 x float> %__C) {
 ; X86-LABEL: test_mm512_maskz_fnmsub_ps:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vfnmsub213ps {{.*#+}} zmm0 = -(zmm1 * zmm0) - zmm2
 ; X86-NEXT:    retl
 ;
@@ -4161,7 +4206,8 @@ declare <16 x float> @llvm.x86.avx512.vfmaddsub.ps.512(<16 x float>, <16 x float
 define <16 x float> @test_mm512_mask_fmaddsub_round_ps(<16 x float> %__A, i16 zeroext %__U, <16 x float> %__B, <16 x float> %__C) {
 ; X86-LABEL: test_mm512_mask_fmaddsub_round_ps:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vfmaddsub132ps {rn-sae}, %zmm1, %zmm2, %zmm0 {%k1}
 ; X86-NEXT:    retl
 ;
@@ -4180,7 +4226,8 @@ entry:
 define <16 x float> @test_mm512_mask3_fmaddsub_round_ps(<16 x float> %__A, <16 x float> %__B, <16 x float> %__C, i16 zeroext %__U) {
 ; X86-LABEL: test_mm512_mask3_fmaddsub_round_ps:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vfmaddsub231ps {rn-sae}, %zmm1, %zmm0, %zmm2 {%k1}
 ; X86-NEXT:    vmovaps %zmm2, %zmm0
 ; X86-NEXT:    retl
@@ -4201,7 +4248,8 @@ entry:
 define <16 x float> @test_mm512_maskz_fmaddsub_round_ps(i16 zeroext %__U, <16 x float> %__A, <16 x float> %__B, <16 x float> %__C) {
 ; X86-LABEL: test_mm512_maskz_fmaddsub_round_ps:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vfmaddsub213ps {rn-sae}, %zmm2, %zmm1, %zmm0 {%k1} {z}
 ; X86-NEXT:    retl
 ;
@@ -4238,7 +4286,8 @@ entry:
 define <16 x float> @test_mm512_mask_fmsubadd_round_ps(<16 x float> %__A, i16 zeroext %__U, <16 x float> %__B, <16 x float> %__C) {
 ; X86-LABEL: test_mm512_mask_fmsubadd_round_ps:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vfmsubadd132ps {rn-sae}, %zmm1, %zmm2, %zmm0 {%k1}
 ; X86-NEXT:    retl
 ;
@@ -4258,7 +4307,8 @@ entry:
 define <16 x float> @test_mm512_maskz_fmsubadd_round_ps(i16 zeroext %__U, <16 x float> %__A, <16 x float> %__B, <16 x float> %__C) {
 ; X86-LABEL: test_mm512_maskz_fmsubadd_round_ps:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vfmsubadd213ps {rn-sae}, %zmm2, %zmm1, %zmm0 {%k1} {z}
 ; X86-NEXT:    retl
 ;
@@ -4291,7 +4341,8 @@ entry:
 define <16 x float> @test_mm512_mask_fmaddsub_ps(<16 x float> %__A, i16 zeroext %__U, <16 x float> %__B, <16 x float> %__C) {
 ; X86-LABEL: test_mm512_mask_fmaddsub_ps:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vfmaddsub132ps {{.*#+}} zmm0 = (zmm0 * zmm1) +/- zmm2
 ; X86-NEXT:    retl
 ;
@@ -4313,7 +4364,8 @@ entry:
 define <16 x float> @test_mm512_mask3_fmaddsub_ps(<16 x float> %__A, <16 x float> %__B, <16 x float> %__C, i16 zeroext %__U) {
 ; X86-LABEL: test_mm512_mask3_fmaddsub_ps:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vfmaddsub231ps {{.*#+}} zmm2 = (zmm0 * zmm1) +/- zmm2
 ; X86-NEXT:    vmovaps %zmm2, %zmm0
 ; X86-NEXT:    retl
@@ -4337,7 +4389,8 @@ entry:
 define <16 x float> @test_mm512_maskz_fmaddsub_ps(i16 zeroext %__U, <16 x float> %__A, <16 x float> %__B, <16 x float> %__C) {
 ; X86-LABEL: test_mm512_maskz_fmaddsub_ps:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vfmaddsub213ps {{.*#+}} zmm0 = (zmm1 * zmm0) +/- zmm2
 ; X86-NEXT:    retl
 ;
@@ -4372,7 +4425,8 @@ entry:
 define <16 x float> @test_mm512_mask_fmsubadd_ps(<16 x float> %__A, i16 zeroext %__U, <16 x float> %__B, <16 x float> %__C) {
 ; X86-LABEL: test_mm512_mask_fmsubadd_ps:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vfmsubadd132ps {{.*#+}} zmm0 = (zmm0 * zmm1) -/+ zmm2
 ; X86-NEXT:    retl
 ;
@@ -4394,7 +4448,8 @@ entry:
 define <16 x float> @test_mm512_maskz_fmsubadd_ps(i16 zeroext %__U, <16 x float> %__A, <16 x float> %__B, <16 x float> %__C) {
 ; X86-LABEL: test_mm512_maskz_fmsubadd_ps:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vfmsubadd213ps {{.*#+}} zmm0 = (zmm1 * zmm0) -/+ zmm2
 ; X86-NEXT:    retl
 ;
@@ -4462,7 +4517,8 @@ entry:
 define <16 x float> @test_mm512_mask3_fmsub_round_ps(<16 x float> %__A, <16 x float> %__B, <16 x float> %__C, i16 zeroext %__U) {
 ; X86-LABEL: test_mm512_mask3_fmsub_round_ps:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vfmsub231ps {rn-sae}, %zmm1, %zmm0, %zmm2 {%k1}
 ; X86-NEXT:    vmovaps %zmm2, %zmm0
 ; X86-NEXT:    retl
@@ -4484,7 +4540,8 @@ entry:
 define <16 x float> @test_mm512_mask3_fmsub_ps(<16 x float> %__A, <16 x float> %__B, <16 x float> %__C, i16 zeroext %__U) {
 ; X86-LABEL: test_mm512_mask3_fmsub_ps:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vfmsub231ps {{.*#+}} zmm2 = (zmm0 * zmm1) - zmm2
 ; X86-NEXT:    vmovaps %zmm2, %zmm0
 ; X86-NEXT:    retl
@@ -4554,7 +4611,8 @@ entry:
 define <16 x float> @test_mm512_mask3_fmsubadd_round_ps(<16 x float> %__A, <16 x float> %__B, <16 x float> %__C, i16 zeroext %__U) {
 ; X86-LABEL: test_mm512_mask3_fmsubadd_round_ps:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vfmsubadd231ps {rn-sae}, %zmm1, %zmm0, %zmm2 {%k1}
 ; X86-NEXT:    vmovaps %zmm2, %zmm0
 ; X86-NEXT:    retl
@@ -4576,7 +4634,8 @@ entry:
 define <16 x float> @test_mm512_mask3_fmsubadd_ps(<16 x float> %__A, <16 x float> %__B, <16 x float> %__C, i16 zeroext %__U) {
 ; X86-LABEL: test_mm512_mask3_fmsubadd_ps:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vfmsubadd231ps {{.*#+}} zmm2 = (zmm0 * zmm1) -/+ zmm2
 ; X86-NEXT:    vmovaps %zmm2, %zmm0
 ; X86-NEXT:    retl
@@ -4642,7 +4701,8 @@ entry:
 define <16 x float> @test_mm512_mask_fnmadd_round_ps(<16 x float> %__A, i16 zeroext %__U, <16 x float> %__B, <16 x float> %__C) {
 ; X86-LABEL: test_mm512_mask_fnmadd_round_ps:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vfnmadd132ps {rn-sae}, %zmm1, %zmm2, %zmm0 {%k1}
 ; X86-NEXT:    retl
 ;
@@ -4662,7 +4722,8 @@ entry:
 define <16 x float> @test_mm512_mask_fnmadd_ps(<16 x float> %__A, i16 zeroext %__U, <16 x float> %__B, <16 x float> %__C) {
 ; X86-LABEL: test_mm512_mask_fnmadd_ps:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vfnmadd132ps {{.*#+}} zmm0 = -(zmm0 * zmm1) + zmm2
 ; X86-NEXT:    retl
 ;
@@ -4774,7 +4835,8 @@ entry:
 define <16 x float> @test_mm512_mask_fnmsub_round_ps(<16 x float> %__A, i16 zeroext %__U, <16 x float> %__B, <16 x float> %__C) {
 ; X86-LABEL: test_mm512_mask_fnmsub_round_ps:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vfnmsub132ps {rn-sae}, %zmm1, %zmm2, %zmm0 {%k1}
 ; X86-NEXT:    retl
 ;
@@ -4795,7 +4857,8 @@ entry:
 define <16 x float> @test_mm512_mask3_fnmsub_round_ps(<16 x float> %__A, <16 x float> %__B, <16 x float> %__C, i16 zeroext %__U) {
 ; X86-LABEL: test_mm512_mask3_fnmsub_round_ps:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vfnmsub231ps {rn-sae}, %zmm1, %zmm0, %zmm2 {%k1}
 ; X86-NEXT:    vmovaps %zmm2, %zmm0
 ; X86-NEXT:    retl
@@ -4818,7 +4881,8 @@ entry:
 define <16 x float> @test_mm512_mask_fnmsub_ps(<16 x float> %__A, i16 zeroext %__U, <16 x float> %__B, <16 x float> %__C) {
 ; X86-LABEL: test_mm512_mask_fnmsub_ps:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vfnmsub132ps {{.*#+}} zmm0 = -(zmm0 * zmm1) - zmm2
 ; X86-NEXT:    retl
 ;
@@ -4839,7 +4903,8 @@ entry:
 define <16 x float> @test_mm512_mask3_fnmsub_ps(<16 x float> %__A, <16 x float> %__B, <16 x float> %__C, i16 zeroext %__U) {
 ; X86-LABEL: test_mm512_mask3_fnmsub_ps:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vfnmsub231ps {{.*#+}} zmm2 = -(zmm0 * zmm1) - zmm2
 ; X86-NEXT:    vmovaps %zmm2, %zmm0
 ; X86-NEXT:    retl
@@ -6261,7 +6326,8 @@ define <8 x i64> @test_mm512_mask_expandloadu_epi32(<8 x i64> %__W, i16 zeroext
 ; X86-LABEL: test_mm512_mask_expandloadu_epi32:
 ; X86:       # %bb.0: # %entry
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    kmovw %ecx, %k1
 ; X86-NEXT:    vpexpandd (%eax), %zmm0 {%k1}
 ; X86-NEXT:    retl
 ;
@@ -6283,7 +6349,8 @@ define <8 x i64> @test_mm512_maskz_expandloadu_epi32(i16 zeroext %__U, i8* reado
 ; X86-LABEL: test_mm512_maskz_expandloadu_epi32:
 ; X86:       # %bb.0: # %entry
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    kmovw %ecx, %k1
 ; X86-NEXT:    vpexpandd (%eax), %zmm0 {%k1} {z}
 ; X86-NEXT:    retl
 ;
@@ -6304,7 +6371,8 @@ define <16 x float> @test_mm512_mask_expandloadu_ps(<16 x float> %__W, i16 zeroe
 ; X86-LABEL: test_mm512_mask_expandloadu_ps:
 ; X86:       # %bb.0: # %entry
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    kmovw %ecx, %k1
 ; X86-NEXT:    vexpandps (%eax), %zmm0 {%k1}
 ; X86-NEXT:    retl
 ;
@@ -6324,7 +6392,8 @@ define <16 x float> @test_mm512_maskz_expandloadu_ps(i16 zeroext %__U, i8* reado
 ; X86-LABEL: test_mm512_maskz_expandloadu_ps:
 ; X86:       # %bb.0: # %entry
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    kmovw %ecx, %k1
 ; X86-NEXT:    vexpandps (%eax), %zmm0 {%k1} {z}
 ; X86-NEXT:    retl
 ;
@@ -6389,9 +6458,10 @@ entry:
 define void @test_mm512_mask_compressstoreu_ps(i8* %__P, i16 zeroext %__U, <16 x float> %__A) {
 ; X86-LABEL: test_mm512_mask_compressstoreu_ps:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    vcompressps %zmm0, (%eax) {%k1}
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vcompressps %zmm0, (%ecx) {%k1}
 ; X86-NEXT:    vzeroupper
 ; X86-NEXT:    retl
 ;
@@ -6411,9 +6481,10 @@ entry:
 define void @test_mm512_mask_compressstoreu_epi32(i8* %__P, i16 zeroext %__U, <8 x i64> %__A) {
 ; X86-LABEL: test_mm512_mask_compressstoreu_epi32:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    vpcompressd %zmm0, (%eax) {%k1}
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vpcompressd %zmm0, (%ecx) {%k1}
 ; X86-NEXT:    vzeroupper
 ; X86-NEXT:    retl
 ;
@@ -6982,7 +7053,8 @@ entry:
 define i32 @test_mm512_mask_reduce_add_epi32(i16 zeroext %__M, <8 x i64> %__W) {
 ; X86-LABEL: test_mm512_mask_reduce_add_epi32:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vmovdqa32 %zmm0, %zmm0 {%k1} {z}
 ; X86-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; X86-NEXT:    vpaddd %ymm1, %ymm0, %ymm0
@@ -7038,7 +7110,8 @@ entry:
 define i32 @test_mm512_mask_reduce_mul_epi32(i16 zeroext %__M, <8 x i64> %__W) {
 ; X86-LABEL: test_mm512_mask_reduce_mul_epi32:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vpbroadcastd {{.*#+}} zmm1 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
 ; X86-NEXT:    vmovdqa32 %zmm0, %zmm1 {%k1}
 ; X86-NEXT:    vextracti64x4 $1, %zmm1, %ymm0
@@ -7096,7 +7169,8 @@ entry:
 define i32 @test_mm512_mask_reduce_and_epi32(i16 zeroext %__M, <8 x i64> %__W) {
 ; X86-LABEL: test_mm512_mask_reduce_and_epi32:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vpternlogd $255, %zmm1, %zmm1, %zmm1
 ; X86-NEXT:    vmovdqa32 %zmm0, %zmm1 {%k1}
 ; X86-NEXT:    vextracti64x4 $1, %zmm1, %ymm0
@@ -7150,7 +7224,8 @@ entry:
 define i32 @test_mm512_mask_reduce_or_epi32(i16 zeroext %__M, <8 x i64> %__W) {
 ; X86-LABEL: test_mm512_mask_reduce_or_epi32:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vmovdqa32 %zmm0, %zmm0 {%k1} {z}
 ; X86-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; X86-NEXT:    vpor %ymm1, %ymm0, %ymm0
@@ -7508,7 +7583,8 @@ define float @test_mm512_mask_reduce_add_ps(i16 zeroext %__M, <16 x float> %__W)
 ; X86:       # %bb.0: # %entry
 ; X86-NEXT:    pushl %eax
 ; X86-NEXT:    .cfi_def_cfa_offset 8
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vmovaps %zmm0, %zmm0 {%k1} {z}
 ; X86-NEXT:    vextractf64x4 $1, %zmm0, %ymm1
 ; X86-NEXT:    vaddps %ymm1, %ymm0, %ymm0
@@ -7564,7 +7640,8 @@ define float @test_mm512_mask_reduce_mul_ps(i16 zeroext %__M, <16 x float> %__W)
 ; X86:       # %bb.0: # %entry
 ; X86-NEXT:    pushl %eax
 ; X86-NEXT:    .cfi_def_cfa_offset 8
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vbroadcastss {{.*#+}} zmm1 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
 ; X86-NEXT:    vmovaps %zmm0, %zmm1 {%k1}
 ; X86-NEXT:    vextractf64x4 $1, %zmm1, %ymm0
@@ -8424,7 +8501,8 @@ entry:
 define i32 @test_mm512_mask_reduce_max_epi32(i16 zeroext %__M, <8 x i64> %__W) {
 ; X86-LABEL: test_mm512_mask_reduce_max_epi32:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vpbroadcastd {{.*#+}} zmm1 = [2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648]
 ; X86-NEXT:    vmovdqa32 %zmm0, %zmm1 {%k1}
 ; X86-NEXT:    vextracti64x4 $1, %zmm1, %ymm0
@@ -8486,7 +8564,8 @@ entry:
 define i32 @test_mm512_mask_reduce_max_epu32(i16 zeroext %__M, <8 x i64> %__W) {
 ; X86-LABEL: test_mm512_mask_reduce_max_epu32:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vmovdqa32 %zmm0, %zmm0 {%k1} {z}
 ; X86-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; X86-NEXT:    vpmaxud %ymm1, %ymm0, %ymm0
@@ -8548,7 +8627,8 @@ define float @test_mm512_mask_reduce_max_ps(i16 zeroext %__M, <16 x float> %__W)
 ; X86:       # %bb.0: # %entry
 ; X86-NEXT:    pushl %eax
 ; X86-NEXT:    .cfi_def_cfa_offset 8
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vbroadcastss {{.*#+}} zmm1 = [-Inf,-Inf,-Inf,-Inf,-Inf,-Inf,-Inf,-Inf,-Inf,-Inf,-Inf,-Inf,-Inf,-Inf,-Inf,-Inf]
 ; X86-NEXT:    vmovaps %zmm0, %zmm1 {%k1}
 ; X86-NEXT:    vextractf64x4 $1, %zmm1, %ymm0
@@ -8604,7 +8684,8 @@ entry:
 define i32 @test_mm512_mask_reduce_min_epi32(i16 zeroext %__M, <8 x i64> %__W) {
 ; X86-LABEL: test_mm512_mask_reduce_min_epi32:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vpbroadcastd {{.*#+}} zmm1 = [2147483647,2147483647,2147483647,2147483647,2147483647,2147483647,2147483647,2147483647,2147483647,2147483647,2147483647,2147483647,2147483647,2147483647,2147483647,2147483647]
 ; X86-NEXT:    vmovdqa32 %zmm0, %zmm1 {%k1}
 ; X86-NEXT:    vextracti64x4 $1, %zmm1, %ymm0
@@ -8666,7 +8747,8 @@ entry:
 define i32 @test_mm512_mask_reduce_min_epu32(i16 zeroext %__M, <8 x i64> %__W) {
 ; X86-LABEL: test_mm512_mask_reduce_min_epu32:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vpternlogd $255, %zmm1, %zmm1, %zmm1
 ; X86-NEXT:    vmovdqa32 %zmm0, %zmm1 {%k1}
 ; X86-NEXT:    vextracti64x4 $1, %zmm1, %ymm0
@@ -8730,7 +8812,8 @@ define float @test_mm512_mask_reduce_min_ps(i16 zeroext %__M, <16 x float> %__W)
 ; X86:       # %bb.0: # %entry
 ; X86-NEXT:    pushl %eax
 ; X86-NEXT:    .cfi_def_cfa_offset 8
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vbroadcastss {{.*#+}} zmm1 = [+Inf,+Inf,+Inf,+Inf,+Inf,+Inf,+Inf,+Inf,+Inf,+Inf,+Inf,+Inf,+Inf,+Inf,+Inf,+Inf]
 ; X86-NEXT:    vmovaps %zmm0, %zmm1 {%k1}
 ; X86-NEXT:    vextractf64x4 $1, %zmm1, %ymm0
@@ -8826,7 +8909,8 @@ entry:
 define <16 x float> @test_mm512_mask_max_ps(<16 x float> %__W, i16 zeroext %__U, <16 x float> %__A, <16 x float> %__B) {
 ; X86-LABEL: test_mm512_mask_max_ps:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vmaxps %zmm2, %zmm1, %zmm0 {%k1}
 ; X86-NEXT:    retl
 ;
@@ -8897,7 +8981,8 @@ entry:
 define <16 x float> @test_mm512_maskz_max_ps(i16 zeroext %__U, <16 x float> %__A, <16 x float> %__B) {
 ; X86-LABEL: test_mm512_maskz_max_ps:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vmaxps %zmm1, %zmm0, %zmm0 {%k1} {z}
 ; X86-NEXT:    retl
 ;
@@ -8916,7 +9001,8 @@ entry:
 define <16 x float> @test_mm512_mask_max_round_ps(<16 x float> %__W, i16 zeroext %__U, <16 x float> %__A, <16 x float> %__B) {
 ; X86-LABEL: test_mm512_mask_max_round_ps:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vmaxps %zmm2, %zmm1, %zmm0 {%k1}
 ; X86-NEXT:    retl
 ;
@@ -8937,7 +9023,8 @@ declare <16 x float> @llvm.x86.avx512.max.ps.512(<16 x float>, <16 x float>, i32
 define <16 x float> @test_mm512_maskz_max_round_ps(i16 zeroext %__U, <16 x float> %__A, <16 x float> %__B) {
 ; X86-LABEL: test_mm512_maskz_max_round_ps:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vmaxps %zmm1, %zmm0, %zmm0 {%k1} {z}
 ; X86-NEXT:    retl
 ;
@@ -9058,7 +9145,8 @@ entry:
 define <16 x float> @test_mm512_mask_min_ps(<16 x float> %__W, i16 zeroext %__U, <16 x float> %__A, <16 x float> %__B) {
 ; X86-LABEL: test_mm512_mask_min_ps:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vminps %zmm2, %zmm1, %zmm0 {%k1}
 ; X86-NEXT:    retl
 ;
@@ -9077,7 +9165,8 @@ entry:
 define <16 x float> @test_mm512_maskz_min_ps(i16 zeroext %__U, <16 x float> %__A, <16 x float> %__B) {
 ; X86-LABEL: test_mm512_maskz_min_ps:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vminps %zmm1, %zmm0, %zmm0 {%k1} {z}
 ; X86-NEXT:    retl
 ;
@@ -9096,7 +9185,8 @@ entry:
 define <16 x float> @test_mm512_mask_min_round_ps(<16 x float> %__W, i16 zeroext %__U, <16 x float> %__A, <16 x float> %__B) {
 ; X86-LABEL: test_mm512_mask_min_round_ps:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vminps %zmm2, %zmm1, %zmm0 {%k1}
 ; X86-NEXT:    retl
 ;
@@ -9117,7 +9207,8 @@ declare <16 x float> @llvm.x86.avx512.min.ps.512(<16 x float>, <16 x float>, i32
 define <16 x float> @test_mm512_maskz_min_round_ps(i16 zeroext %__U, <16 x float> %__A, <16 x float> %__B) {
 ; X86-LABEL: test_mm512_maskz_min_round_ps:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vminps %zmm1, %zmm0, %zmm0 {%k1} {z}
 ; X86-NEXT:    retl
 ;
@@ -9258,7 +9349,8 @@ entry:
 define <16 x float> @test_mm512_mask_sqrt_ps(<16 x float> %__W, i16 zeroext %__U, <16 x float> %__A) {
 ; X86-LABEL: test_mm512_mask_sqrt_ps:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vsqrtps %zmm1, %zmm0 {%k1}
 ; X86-NEXT:    retl
 ;
@@ -9277,7 +9369,8 @@ entry:
 define <16 x float> @test_mm512_maskz_sqrt_ps(i16 zeroext %__U, <16 x float> %__A) {
 ; X86-LABEL: test_mm512_maskz_sqrt_ps:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vsqrtps %zmm0, %zmm0 {%k1} {z}
 ; X86-NEXT:    retl
 ;
@@ -9296,7 +9389,8 @@ entry:
 define <16 x float> @test_mm512_mask_sqrt_round_ps(<16 x float> %__W, i16 zeroext %__U, <16 x float> %__A) {
 ; X86-LABEL: test_mm512_mask_sqrt_round_ps:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vsqrtps {rn-sae}, %zmm1, %zmm0 {%k1}
 ; X86-NEXT:    retl
 ;
@@ -9317,7 +9411,8 @@ declare <16 x float> @llvm.x86.avx512.sqrt.ps.512(<16 x float>, i32)
 define <16 x float> @test_mm512_maskz_sqrt_round_ps(i16 zeroext %__U, <16 x float> %__A) {
 ; X86-LABEL: test_mm512_maskz_sqrt_round_ps:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vsqrtps {rn-sae}, %zmm0, %zmm0 {%k1} {z}
 ; X86-NEXT:    retl
 ;
@@ -9350,17 +9445,16 @@ define <8 x i64> @test_mm512_rol_epi32(<8 x i64> %__A) local_unnamed_addr #0 {
 ; CHECK-NEXT:    ret{{[l|q]}}
 entry:
   %0 = bitcast <8 x i64> %__A to <16 x i32>
-  %1 = tail call <16 x i32> @llvm.x86.avx512.prol.d.512(<16 x i32> %0, i32 5)
+  %1 = tail call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %0, <16 x i32> %0, <16 x i32> <i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5>)
   %2 = bitcast <16 x i32> %1 to <8 x i64>
   ret <8 x i64> %2
 }
 
-declare <16 x i32> @llvm.x86.avx512.prol.d.512(<16 x i32>, i32) #1
-
 define <8 x i64> @test_mm512_mask_rol_epi32(<8 x i64> %__W, i16 zeroext %__U, <8 x i64> %__A) {
 ; X86-LABEL: test_mm512_mask_rol_epi32:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vprold $5, %zmm1, %zmm0 {%k1}
 ; X86-NEXT:    retl
 ;
@@ -9371,7 +9465,7 @@ define <8 x i64> @test_mm512_mask_rol_epi32(<8 x i64> %__W, i16 zeroext %__U, <8
 ; X64-NEXT:    retq
 entry:
   %0 = bitcast <8 x i64> %__A to <16 x i32>
-  %1 = tail call <16 x i32> @llvm.x86.avx512.prol.d.512(<16 x i32> %0, i32 5)
+  %1 = tail call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %0, <16 x i32> %0, <16 x i32> <i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5>)
   %2 = bitcast <8 x i64> %__W to <16 x i32>
   %3 = bitcast i16 %__U to <16 x i1>
   %4 = select <16 x i1> %3, <16 x i32> %1, <16 x i32> %2
@@ -9382,7 +9476,8 @@ entry:
 define <8 x i64> @test_mm512_maskz_rol_epi32(i16 zeroext %__U, <8 x i64> %__A) {
 ; X86-LABEL: test_mm512_maskz_rol_epi32:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vprold $5, %zmm0, %zmm0 {%k1} {z}
 ; X86-NEXT:    retl
 ;
@@ -9393,7 +9488,7 @@ define <8 x i64> @test_mm512_maskz_rol_epi32(i16 zeroext %__U, <8 x i64> %__A) {
 ; X64-NEXT:    retq
 entry:
   %0 = bitcast <8 x i64> %__A to <16 x i32>
-  %1 = tail call <16 x i32> @llvm.x86.avx512.prol.d.512(<16 x i32> %0, i32 5)
+  %1 = tail call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %0, <16 x i32> %0, <16 x i32> <i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5>)
   %2 = bitcast i16 %__U to <16 x i1>
   %3 = select <16 x i1> %2, <16 x i32> %1, <16 x i32> zeroinitializer
   %4 = bitcast <16 x i32> %3 to <8 x i64>
@@ -9406,12 +9501,10 @@ define <8 x i64> @test_mm512_rol_epi64(<8 x i64> %__A) {
 ; CHECK-NEXT:    vprolq $5, %zmm0, %zmm0
 ; CHECK-NEXT:    ret{{[l|q]}}
 entry:
-  %0 = tail call <8 x i64> @llvm.x86.avx512.prol.q.512(<8 x i64> %__A, i32 5)
+  %0 = tail call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %__A, <8 x i64> %__A, <8 x i64> <i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5>)
   ret <8 x i64> %0
 }
 
-declare <8 x i64> @llvm.x86.avx512.prol.q.512(<8 x i64>, i32) #1
-
 define <8 x i64> @test_mm512_mask_rol_epi64(<8 x i64> %__W, i8 zeroext %__U, <8 x i64> %__A) {
 ; X86-LABEL: test_mm512_mask_rol_epi64:
 ; X86:       # %bb.0: # %entry
@@ -9426,7 +9519,7 @@ define <8 x i64> @test_mm512_mask_rol_epi64(<8 x i64> %__W, i8 zeroext %__U, <8
 ; X64-NEXT:    vprolq $5, %zmm1, %zmm0 {%k1}
 ; X64-NEXT:    retq
 entry:
-  %0 = tail call <8 x i64> @llvm.x86.avx512.prol.q.512(<8 x i64> %__A, i32 5)
+  %0 = tail call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %__A, <8 x i64> %__A, <8 x i64> <i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5>)
   %1 = bitcast i8 %__U to <8 x i1>
   %2 = select <8 x i1> %1, <8 x i64> %0, <8 x i64> %__W
   ret <8 x i64> %2
@@ -9446,7 +9539,7 @@ define <8 x i64> @test_mm512_maskz_rol_epi64(i8 zeroext %__U, <8 x i64> %__A) {
 ; X64-NEXT:    vprolq $5, %zmm0, %zmm0 {%k1} {z}
 ; X64-NEXT:    retq
 entry:
-  %0 = tail call <8 x i64> @llvm.x86.avx512.prol.q.512(<8 x i64> %__A, i32 5)
+  %0 = tail call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %__A, <8 x i64> %__A, <8 x i64> <i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5>)
   %1 = bitcast i8 %__U to <8 x i1>
   %2 = select <8 x i1> %1, <8 x i64> %0, <8 x i64> zeroinitializer
   ret <8 x i64> %2
@@ -9460,7 +9553,7 @@ define <8 x i64> @test_mm512_rolv_epi32(<8 x i64> %__A, <8 x i64> %__B) {
 entry:
   %0 = bitcast <8 x i64> %__A to <16 x i32>
   %1 = bitcast <8 x i64> %__B to <16 x i32>
-  %2 = tail call <16 x i32> @llvm.x86.avx512.prolv.d.512(<16 x i32> %0, <16 x i32> %1)
+  %2 = tail call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %0, <16 x i32> %0, <16 x i32> %1)
   %3 = bitcast <16 x i32> %2 to <8 x i64>
   ret <8 x i64> %3
 }
@@ -9468,7 +9561,8 @@ entry:
 define <8 x i64> @test_mm512_mask_rolv_epi32(<8 x i64> %__W, i16 zeroext %__U, <8 x i64> %__A, <8 x i64> %__B) {
 ; X86-LABEL: test_mm512_mask_rolv_epi32:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vprolvd %zmm2, %zmm1, %zmm0 {%k1}
 ; X86-NEXT:    retl
 ;
@@ -9480,7 +9574,7 @@ define <8 x i64> @test_mm512_mask_rolv_epi32(<8 x i64> %__W, i16 zeroext %__U, <
 entry:
   %0 = bitcast <8 x i64> %__A to <16 x i32>
   %1 = bitcast <8 x i64> %__B to <16 x i32>
-  %2 = tail call <16 x i32> @llvm.x86.avx512.prolv.d.512(<16 x i32> %0, <16 x i32> %1)
+  %2 = tail call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %0, <16 x i32> %0, <16 x i32> %1)
   %3 = bitcast <8 x i64> %__W to <16 x i32>
   %4 = bitcast i16 %__U to <16 x i1>
   %5 = select <16 x i1> %4, <16 x i32> %2, <16 x i32> %3
@@ -9491,7 +9585,8 @@ entry:
 define <8 x i64> @test_mm512_maskz_rolv_epi32(i16 zeroext %__U, <8 x i64> %__A, <8 x i64> %__B) {
 ; X86-LABEL: test_mm512_maskz_rolv_epi32:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vprolvd %zmm1, %zmm0, %zmm0 {%k1} {z}
 ; X86-NEXT:    retl
 ;
@@ -9503,7 +9598,7 @@ define <8 x i64> @test_mm512_maskz_rolv_epi32(i16 zeroext %__U, <8 x i64> %__A,
 entry:
   %0 = bitcast <8 x i64> %__A to <16 x i32>
   %1 = bitcast <8 x i64> %__B to <16 x i32>
-  %2 = tail call <16 x i32> @llvm.x86.avx512.prolv.d.512(<16 x i32> %0, <16 x i32> %1)
+  %2 = tail call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %0, <16 x i32> %0, <16 x i32> %1)
   %3 = bitcast i16 %__U to <16 x i1>
   %4 = select <16 x i1> %3, <16 x i32> %2, <16 x i32> zeroinitializer
   %5 = bitcast <16 x i32> %4 to <8 x i64>
@@ -9516,7 +9611,7 @@ define <8 x i64> @test_mm512_rolv_epi64(<8 x i64> %__A, <8 x i64> %__B) {
 ; CHECK-NEXT:    vprolvq %zmm1, %zmm0, %zmm0
 ; CHECK-NEXT:    ret{{[l|q]}}
 entry:
-  %0 = tail call <8 x i64> @llvm.x86.avx512.prolv.q.512(<8 x i64> %__A, <8 x i64> %__B)
+  %0 = tail call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %__A, <8 x i64> %__A, <8 x i64> %__B)
   ret <8 x i64> %0
 }
 
@@ -9534,7 +9629,7 @@ define <8 x i64> @test_mm512_mask_rolv_epi64(<8 x i64> %__W, i8 zeroext %__U, <8
 ; X64-NEXT:    vprolvq %zmm2, %zmm1, %zmm0 {%k1}
 ; X64-NEXT:    retq
 entry:
-  %0 = tail call <8 x i64> @llvm.x86.avx512.prolv.q.512(<8 x i64> %__A, <8 x i64> %__B)
+  %0 = tail call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %__A, <8 x i64> %__A, <8 x i64> %__B)
   %1 = bitcast i8 %__U to <8 x i1>
   %2 = select <8 x i1> %1, <8 x i64> %0, <8 x i64> %__W
   ret <8 x i64> %2
@@ -9554,7 +9649,7 @@ define <8 x i64> @test_mm512_maskz_rolv_epi64(i8 zeroext %__U, <8 x i64> %__A, <
 ; X64-NEXT:    vprolvq %zmm1, %zmm0, %zmm0 {%k1} {z}
 ; X64-NEXT:    retq
 entry:
-  %0 = tail call <8 x i64> @llvm.x86.avx512.prolv.q.512(<8 x i64> %__A, <8 x i64> %__B)
+  %0 = tail call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %__A, <8 x i64> %__A, <8 x i64> %__B)
   %1 = bitcast i8 %__U to <8 x i1>
   %2 = select <8 x i1> %1, <8 x i64> %0, <8 x i64> zeroinitializer
   ret <8 x i64> %2
@@ -9567,17 +9662,17 @@ define <8 x i64> @test_mm512_ror_epi32(<8 x i64> %__A) {
 ; CHECK-NEXT:    ret{{[l|q]}}
 entry:
   %0 = bitcast <8 x i64> %__A to <16 x i32>
-  %1 = tail call <16 x i32> @llvm.x86.avx512.pror.d.512(<16 x i32> %0, i32 5)
+  %1 = tail call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %0, <16 x i32> %0, <16 x i32> <i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5>)
   %2 = bitcast <16 x i32> %1 to <8 x i64>
   ret <8 x i64> %2
 }
 
-declare <16 x i32> @llvm.x86.avx512.pror.d.512(<16 x i32>, i32) #1
 
 define <8 x i64> @test_mm512_mask_ror_epi32(<8 x i64> %__W, i16 zeroext %__U, <8 x i64> %__A) {
 ; X86-LABEL: test_mm512_mask_ror_epi32:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vprord $5, %zmm1, %zmm0 {%k1}
 ; X86-NEXT:    retl
 ;
@@ -9588,7 +9683,7 @@ define <8 x i64> @test_mm512_mask_ror_epi32(<8 x i64> %__W, i16 zeroext %__U, <8
 ; X64-NEXT:    retq
 entry:
   %0 = bitcast <8 x i64> %__A to <16 x i32>
-  %1 = tail call <16 x i32> @llvm.x86.avx512.pror.d.512(<16 x i32> %0, i32 5)
+  %1 = tail call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %0, <16 x i32> %0, <16 x i32> <i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5>)
   %2 = bitcast <8 x i64> %__W to <16 x i32>
   %3 = bitcast i16 %__U to <16 x i1>
   %4 = select <16 x i1> %3, <16 x i32> %1, <16 x i32> %2
@@ -9599,7 +9694,8 @@ entry:
 define <8 x i64> @test_mm512_maskz_ror_epi32(i16 zeroext %__U, <8 x i64> %__A) {
 ; X86-LABEL: test_mm512_maskz_ror_epi32:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vprord $5, %zmm0, %zmm0 {%k1} {z}
 ; X86-NEXT:    retl
 ;
@@ -9610,7 +9706,7 @@ define <8 x i64> @test_mm512_maskz_ror_epi32(i16 zeroext %__U, <8 x i64> %__A) {
 ; X64-NEXT:    retq
 entry:
   %0 = bitcast <8 x i64> %__A to <16 x i32>
-  %1 = tail call <16 x i32> @llvm.x86.avx512.pror.d.512(<16 x i32> %0, i32 5)
+  %1 = tail call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %0, <16 x i32> %0, <16 x i32> <i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5>)
   %2 = bitcast i16 %__U to <16 x i1>
   %3 = select <16 x i1> %2, <16 x i32> %1, <16 x i32> zeroinitializer
   %4 = bitcast <16 x i32> %3 to <8 x i64>
@@ -9623,12 +9719,10 @@ define <8 x i64> @test_mm512_ror_epi64(<8 x i64> %__A) {
 ; CHECK-NEXT:    vprorq $5, %zmm0, %zmm0
 ; CHECK-NEXT:    ret{{[l|q]}}
 entry:
-  %0 = tail call <8 x i64> @llvm.x86.avx512.pror.q.512(<8 x i64> %__A, i32 5)
+  %0 = tail call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %__A, <8 x i64> %__A, <8 x i64> <i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5>)
   ret <8 x i64> %0
 }
 
-declare <8 x i64> @llvm.x86.avx512.pror.q.512(<8 x i64>, i32) #1
-
 define <8 x i64> @test_mm512_mask_ror_epi64(<8 x i64> %__W, i8 zeroext %__U, <8 x i64> %__A) {
 ; X86-LABEL: test_mm512_mask_ror_epi64:
 ; X86:       # %bb.0: # %entry
@@ -9643,7 +9737,7 @@ define <8 x i64> @test_mm512_mask_ror_epi64(<8 x i64> %__W, i8 zeroext %__U, <8
 ; X64-NEXT:    vprorq $5, %zmm1, %zmm0 {%k1}
 ; X64-NEXT:    retq
 entry:
-  %0 = tail call <8 x i64> @llvm.x86.avx512.pror.q.512(<8 x i64> %__A, i32 5)
+  %0 = tail call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %__A, <8 x i64> %__A, <8 x i64> <i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5>)
   %1 = bitcast i8 %__U to <8 x i1>
   %2 = select <8 x i1> %1, <8 x i64> %0, <8 x i64> %__W
   ret <8 x i64> %2
@@ -9663,7 +9757,7 @@ define <8 x i64> @test_mm512_maskz_ror_epi64(i8 zeroext %__U, <8 x i64> %__A) {
 ; X64-NEXT:    vprorq $5, %zmm0, %zmm0 {%k1} {z}
 ; X64-NEXT:    retq
 entry:
-  %0 = tail call <8 x i64> @llvm.x86.avx512.pror.q.512(<8 x i64> %__A, i32 5)
+  %0 = tail call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %__A, <8 x i64> %__A, <8 x i64> <i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5, i64 5>)
   %1 = bitcast i8 %__U to <8 x i1>
   %2 = select <8 x i1> %1, <8 x i64> %0, <8 x i64> zeroinitializer
   ret <8 x i64> %2
@@ -9677,7 +9771,7 @@ define <8 x i64> @test_mm512_rorv_epi32(<8 x i64> %__A, <8 x i64> %__B) {
 entry:
   %0 = bitcast <8 x i64> %__A to <16 x i32>
   %1 = bitcast <8 x i64> %__B to <16 x i32>
-  %2 = tail call <16 x i32> @llvm.x86.avx512.prorv.d.512(<16 x i32> %0, <16 x i32> %1)
+  %2 = tail call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %0, <16 x i32> %0, <16 x i32> %1)
   %3 = bitcast <16 x i32> %2 to <8 x i64>
   ret <8 x i64> %3
 }
@@ -9685,7 +9779,8 @@ entry:
 define <8 x i64> @test_mm512_mask_rorv_epi32(<8 x i64> %__W, i16 zeroext %__U, <8 x i64> %__A, <8 x i64> %__B) {
 ; X86-LABEL: test_mm512_mask_rorv_epi32:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vprorvd %zmm2, %zmm1, %zmm0 {%k1}
 ; X86-NEXT:    retl
 ;
@@ -9697,7 +9792,7 @@ define <8 x i64> @test_mm512_mask_rorv_epi32(<8 x i64> %__W, i16 zeroext %__U, <
 entry:
   %0 = bitcast <8 x i64> %__A to <16 x i32>
   %1 = bitcast <8 x i64> %__B to <16 x i32>
-  %2 = tail call <16 x i32> @llvm.x86.avx512.prorv.d.512(<16 x i32> %0, <16 x i32> %1)
+  %2 = tail call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %0, <16 x i32> %0, <16 x i32> %1)
   %3 = bitcast <8 x i64> %__W to <16 x i32>
   %4 = bitcast i16 %__U to <16 x i1>
   %5 = select <16 x i1> %4, <16 x i32> %2, <16 x i32> %3
@@ -9708,7 +9803,8 @@ entry:
 define <8 x i64> @test_mm512_maskz_rorv_epi32(i16 zeroext %__U, <8 x i64> %__A, <8 x i64> %__B) {
 ; X86-LABEL: test_mm512_maskz_rorv_epi32:
 ; X86:       # %bb.0: # %entry
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vprorvd %zmm1, %zmm0, %zmm0 {%k1} {z}
 ; X86-NEXT:    retl
 ;
@@ -9720,7 +9816,7 @@ define <8 x i64> @test_mm512_maskz_rorv_epi32(i16 zeroext %__U, <8 x i64> %__A,
 entry:
   %0 = bitcast <8 x i64> %__A to <16 x i32>
   %1 = bitcast <8 x i64> %__B to <16 x i32>
-  %2 = tail call <16 x i32> @llvm.x86.avx512.prorv.d.512(<16 x i32> %0, <16 x i32> %1)
+  %2 = tail call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %0, <16 x i32> %0, <16 x i32> %1)
   %3 = bitcast i16 %__U to <16 x i1>
   %4 = select <16 x i1> %3, <16 x i32> %2, <16 x i32> zeroinitializer
   %5 = bitcast <16 x i32> %4 to <8 x i64>
@@ -9733,7 +9829,7 @@ define <8 x i64> @test_mm512_rorv_epi64(<8 x i64> %__A, <8 x i64> %__B) {
 ; CHECK-NEXT:    vprorvq %zmm1, %zmm0, %zmm0
 ; CHECK-NEXT:    ret{{[l|q]}}
 entry:
-  %0 = tail call <8 x i64> @llvm.x86.avx512.prorv.q.512(<8 x i64> %__A, <8 x i64> %__B)
+  %0 = tail call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %__A, <8 x i64> %__A, <8 x i64> %__B)
   ret <8 x i64> %0
 }
 
@@ -9751,7 +9847,7 @@ define <8 x i64> @test_mm512_mask_rorv_epi64(<8 x i64> %__W, i8 zeroext %__U, <8
 ; X64-NEXT:    vprorvq %zmm2, %zmm1, %zmm0 {%k1}
 ; X64-NEXT:    retq
 entry:
-  %0 = tail call <8 x i64> @llvm.x86.avx512.prorv.q.512(<8 x i64> %__A, <8 x i64> %__B)
+  %0 = tail call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %__A, <8 x i64> %__A, <8 x i64> %__B)
   %1 = bitcast i8 %__U to <8 x i1>
   %2 = select <8 x i1> %1, <8 x i64> %0, <8 x i64> %__W
   ret <8 x i64> %2
@@ -9771,7 +9867,7 @@ define <8 x i64> @test_mm512_maskz_rorv_epi64(i8 zeroext %__U, <8 x i64> %__A, <
 ; X64-NEXT:    vprorvq %zmm1, %zmm0, %zmm0 {%k1} {z}
 ; X64-NEXT:    retq
 entry:
-  %0 = tail call <8 x i64> @llvm.x86.avx512.prorv.q.512(<8 x i64> %__A, <8 x i64> %__B)
+  %0 = tail call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %__A, <8 x i64> %__A, <8 x i64> %__B)
   %1 = bitcast i8 %__U to <8 x i1>
   %2 = select <8 x i1> %1, <8 x i64> %0, <8 x i64> zeroinitializer
   ret <8 x i64> %2
@@ -9799,10 +9895,11 @@ declare <8 x float> @llvm.x86.avx.min.ps.256(<8 x float>, <8 x float>)
 declare <4 x float> @llvm.x86.sse.min.ps(<4 x float>, <4 x float>)
 declare <8 x double> @llvm.sqrt.v8f64(<8 x double>)
 declare <16 x float> @llvm.sqrt.v16f32(<16 x float>)
-declare <16 x i32> @llvm.x86.avx512.prolv.d.512(<16 x i32>, <16 x i32>)
-declare <8 x i64> @llvm.x86.avx512.prolv.q.512(<8 x i64>, <8 x i64>)
-declare <16 x i32> @llvm.x86.avx512.prorv.d.512(<16 x i32>, <16 x i32>)
-declare <8 x i64> @llvm.x86.avx512.prorv.q.512(<8 x i64>, <8 x i64>)
+
+declare <16 x i32> @llvm.fshl.v16i32(<16 x i32>, <16 x i32>, <16 x i32>)
+declare <8 x i64> @llvm.fshl.v8i64(<8 x i64>, <8 x i64>, <8 x i64>)
+declare <16 x i32> @llvm.fshr.v16i32(<16 x i32>, <16 x i32>, <16 x i32>)
+declare <8 x i64> @llvm.fshr.v8i64(<8 x i64>, <8 x i64>, <8 x i64>)
 
 !0 = !{i32 1}
 
diff --git a/test/CodeGen/X86/avx512-intrinsics-upgrade.ll b/test/CodeGen/X86/avx512-intrinsics-upgrade.ll
index 26e8636df8f8a4526cf6849354ba91531a8607bb..b3d8e14696e2c29f7fd75b1aa72d51e73b0d37c6 100644
--- a/test/CodeGen/X86/avx512-intrinsics-upgrade.ll
+++ b/test/CodeGen/X86/avx512-intrinsics-upgrade.ll
@@ -7,10 +7,8 @@ declare i16 @llvm.x86.avx512.kunpck.bw(i16, i16) nounwind readnone
 define i16 @unpckbw_test(i16 %a0, i16 %a1) {
 ; X86-LABEL: unpckbw_test:
 ; X86:       ## %bb.0:
-; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax ## encoding: [0x0f,0xb6,0x44,0x24,0x04]
-; X86-NEXT:    kmovw %eax, %k0 ## encoding: [0xc5,0xf8,0x92,0xc0]
-; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax ## encoding: [0x0f,0xb6,0x44,0x24,0x08]
-; X86-NEXT:    kmovw %eax, %k1 ## encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k0 ## encoding: [0xc5,0xf8,0x90,0x44,0x24,0x04]
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 ## encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x08]
 ; X86-NEXT:    kunpckbw %k1, %k0, %k0 ## encoding: [0xc5,0xfd,0x4b,0xc1]
 ; X86-NEXT:    kmovw %k0, %eax ## encoding: [0xc5,0xf8,0x93,0xc0]
 ; X86-NEXT:    ## kill: def $ax killed $ax killed $eax
@@ -3457,6 +3455,282 @@ define <8 x i64>@test_int_x86_avx512_mask_pmovsxw_q_512(<8 x i16> %x0, <8 x i64>
   ret <8 x i64> %res4
 }
 
+declare <16 x i32> @llvm.x86.avx512.prolv.d.512(<16 x i32>, <16 x i32>)
+
+define <16 x i32>@test_int_x86_avx512_prolv_d_512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 %x3) {
+; X86-LABEL: test_int_x86_avx512_prolv_d_512:
+; X86:       ## %bb.0:
+; X86-NEXT:    vprolvd %zmm1, %zmm0, %zmm3 ## encoding: [0x62,0xf2,0x7d,0x48,0x15,0xd9]
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 ## encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
+; X86-NEXT:    vprolvd %zmm1, %zmm0, %zmm2 {%k1} ## encoding: [0x62,0xf2,0x7d,0x49,0x15,0xd1]
+; X86-NEXT:    vprolvd %zmm1, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf2,0x7d,0xc9,0x15,0xc1]
+; X86-NEXT:    vpaddd %zmm3, %zmm0, %zmm0 ## encoding: [0x62,0xf1,0x7d,0x48,0xfe,0xc3]
+; X86-NEXT:    vpaddd %zmm0, %zmm2, %zmm0 ## encoding: [0x62,0xf1,0x6d,0x48,0xfe,0xc0]
+; X86-NEXT:    retl ## encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_prolv_d_512:
+; X64:       ## %bb.0:
+; X64-NEXT:    vprolvd %zmm1, %zmm0, %zmm3 ## encoding: [0x62,0xf2,0x7d,0x48,0x15,0xd9]
+; X64-NEXT:    kmovw %edi, %k1 ## encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vprolvd %zmm1, %zmm0, %zmm2 {%k1} ## encoding: [0x62,0xf2,0x7d,0x49,0x15,0xd1]
+; X64-NEXT:    vprolvd %zmm1, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf2,0x7d,0xc9,0x15,0xc1]
+; X64-NEXT:    vpaddd %zmm3, %zmm0, %zmm0 ## encoding: [0x62,0xf1,0x7d,0x48,0xfe,0xc3]
+; X64-NEXT:    vpaddd %zmm0, %zmm2, %zmm0 ## encoding: [0x62,0xf1,0x6d,0x48,0xfe,0xc0]
+; X64-NEXT:    retq ## encoding: [0xc3]
+  %1 = call <16 x i32> @llvm.x86.avx512.prolv.d.512(<16 x i32> %x0, <16 x i32> %x1)
+  %2 = bitcast i16 %x3 to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x i32> %1, <16 x i32> %x2
+  %4 = call <16 x i32> @llvm.x86.avx512.prolv.d.512(<16 x i32> %x0, <16 x i32> %x1)
+  %5 = bitcast i16 %x3 to <16 x i1>
+  %6 = select <16 x i1> %5, <16 x i32> %4, <16 x i32> zeroinitializer
+  %7 = call <16 x i32> @llvm.x86.avx512.prolv.d.512(<16 x i32> %x0, <16 x i32> %x1)
+  %res3 = add <16 x i32> %3, %6
+  %res4 = add <16 x i32> %res3, %7
+  ret <16 x i32> %res4
+}
+
+declare <8 x i64> @llvm.x86.avx512.prolv.q.512(<8 x i64>, <8 x i64>)
+
+define <8 x i64>@test_int_x86_avx512_prolv_q_512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_prolv_q_512:
+; X86:       ## %bb.0:
+; X86-NEXT:    vprolvq %zmm1, %zmm0, %zmm3 ## encoding: [0x62,0xf2,0xfd,0x48,0x15,0xd9]
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax ## encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 ## encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vprolvq %zmm1, %zmm0, %zmm2 {%k1} ## encoding: [0x62,0xf2,0xfd,0x49,0x15,0xd1]
+; X86-NEXT:    vprolvq %zmm1, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf2,0xfd,0xc9,0x15,0xc1]
+; X86-NEXT:    vpaddq %zmm3, %zmm0, %zmm0 ## encoding: [0x62,0xf1,0xfd,0x48,0xd4,0xc3]
+; X86-NEXT:    vpaddq %zmm0, %zmm2, %zmm0 ## encoding: [0x62,0xf1,0xed,0x48,0xd4,0xc0]
+; X86-NEXT:    retl ## encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_prolv_q_512:
+; X64:       ## %bb.0:
+; X64-NEXT:    vprolvq %zmm1, %zmm0, %zmm3 ## encoding: [0x62,0xf2,0xfd,0x48,0x15,0xd9]
+; X64-NEXT:    kmovw %edi, %k1 ## encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vprolvq %zmm1, %zmm0, %zmm2 {%k1} ## encoding: [0x62,0xf2,0xfd,0x49,0x15,0xd1]
+; X64-NEXT:    vprolvq %zmm1, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf2,0xfd,0xc9,0x15,0xc1]
+; X64-NEXT:    vpaddq %zmm3, %zmm0, %zmm0 ## encoding: [0x62,0xf1,0xfd,0x48,0xd4,0xc3]
+; X64-NEXT:    vpaddq %zmm0, %zmm2, %zmm0 ## encoding: [0x62,0xf1,0xed,0x48,0xd4,0xc0]
+; X64-NEXT:    retq ## encoding: [0xc3]
+  %1 = call <8 x i64> @llvm.x86.avx512.prolv.q.512(<8 x i64> %x0, <8 x i64> %x1)
+  %2 = bitcast i8 %x3 to <8 x i1>
+  %3 = select <8 x i1> %2, <8 x i64> %1, <8 x i64> %x2
+  %4 = call <8 x i64> @llvm.x86.avx512.prolv.q.512(<8 x i64> %x0, <8 x i64> %x1)
+  %5 = bitcast i8 %x3 to <8 x i1>
+  %6 = select <8 x i1> %5, <8 x i64> %4, <8 x i64> zeroinitializer
+  %7 = call <8 x i64> @llvm.x86.avx512.prolv.q.512(<8 x i64> %x0, <8 x i64> %x1)
+  %res3 = add <8 x i64> %3, %6
+  %res4 = add <8 x i64> %res3, %7
+  ret <8 x i64> %res4
+}
+
+declare <16 x i32> @llvm.x86.avx512.prorv.d.512(<16 x i32>, <16 x i32>)
+
+define <16 x i32>@test_int_x86_avx512_prorv_d_512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 %x3) {
+; X86-LABEL: test_int_x86_avx512_prorv_d_512:
+; X86:       ## %bb.0:
+; X86-NEXT:    vprorvd %zmm1, %zmm0, %zmm3 ## encoding: [0x62,0xf2,0x7d,0x48,0x14,0xd9]
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 ## encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
+; X86-NEXT:    vprorvd %zmm1, %zmm0, %zmm2 {%k1} ## encoding: [0x62,0xf2,0x7d,0x49,0x14,0xd1]
+; X86-NEXT:    vprorvd %zmm1, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf2,0x7d,0xc9,0x14,0xc1]
+; X86-NEXT:    vpaddd %zmm3, %zmm0, %zmm0 ## encoding: [0x62,0xf1,0x7d,0x48,0xfe,0xc3]
+; X86-NEXT:    vpaddd %zmm0, %zmm2, %zmm0 ## encoding: [0x62,0xf1,0x6d,0x48,0xfe,0xc0]
+; X86-NEXT:    retl ## encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_prorv_d_512:
+; X64:       ## %bb.0:
+; X64-NEXT:    vprorvd %zmm1, %zmm0, %zmm3 ## encoding: [0x62,0xf2,0x7d,0x48,0x14,0xd9]
+; X64-NEXT:    kmovw %edi, %k1 ## encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vprorvd %zmm1, %zmm0, %zmm2 {%k1} ## encoding: [0x62,0xf2,0x7d,0x49,0x14,0xd1]
+; X64-NEXT:    vprorvd %zmm1, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf2,0x7d,0xc9,0x14,0xc1]
+; X64-NEXT:    vpaddd %zmm3, %zmm0, %zmm0 ## encoding: [0x62,0xf1,0x7d,0x48,0xfe,0xc3]
+; X64-NEXT:    vpaddd %zmm0, %zmm2, %zmm0 ## encoding: [0x62,0xf1,0x6d,0x48,0xfe,0xc0]
+; X64-NEXT:    retq ## encoding: [0xc3]
+  %1 = call <16 x i32> @llvm.x86.avx512.prorv.d.512(<16 x i32> %x0, <16 x i32> %x1)
+  %2 = bitcast i16 %x3 to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x i32> %1, <16 x i32> %x2
+  %4 = call <16 x i32> @llvm.x86.avx512.prorv.d.512(<16 x i32> %x0, <16 x i32> %x1)
+  %5 = bitcast i16 %x3 to <16 x i1>
+  %6 = select <16 x i1> %5, <16 x i32> %4, <16 x i32> zeroinitializer
+  %7 = call <16 x i32> @llvm.x86.avx512.prorv.d.512(<16 x i32> %x0, <16 x i32> %x1)
+  %res3 = add <16 x i32> %3, %6
+  %res4 = add <16 x i32> %res3, %7
+  ret <16 x i32> %res4
+}
+
+declare <8 x i64> @llvm.x86.avx512.prorv.q.512(<8 x i64>, <8 x i64>)
+
+define <8 x i64>@test_int_x86_avx512_prorv_q_512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_prorv_q_512:
+; X86:       ## %bb.0:
+; X86-NEXT:    vprorvq %zmm1, %zmm0, %zmm3 ## encoding: [0x62,0xf2,0xfd,0x48,0x14,0xd9]
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax ## encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 ## encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vprorvq %zmm1, %zmm0, %zmm2 {%k1} ## encoding: [0x62,0xf2,0xfd,0x49,0x14,0xd1]
+; X86-NEXT:    vprorvq %zmm1, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf2,0xfd,0xc9,0x14,0xc1]
+; X86-NEXT:    vpaddq %zmm3, %zmm0, %zmm0 ## encoding: [0x62,0xf1,0xfd,0x48,0xd4,0xc3]
+; X86-NEXT:    vpaddq %zmm0, %zmm2, %zmm0 ## encoding: [0x62,0xf1,0xed,0x48,0xd4,0xc0]
+; X86-NEXT:    retl ## encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_prorv_q_512:
+; X64:       ## %bb.0:
+; X64-NEXT:    vprorvq %zmm1, %zmm0, %zmm3 ## encoding: [0x62,0xf2,0xfd,0x48,0x14,0xd9]
+; X64-NEXT:    kmovw %edi, %k1 ## encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vprorvq %zmm1, %zmm0, %zmm2 {%k1} ## encoding: [0x62,0xf2,0xfd,0x49,0x14,0xd1]
+; X64-NEXT:    vprorvq %zmm1, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf2,0xfd,0xc9,0x14,0xc1]
+; X64-NEXT:    vpaddq %zmm3, %zmm0, %zmm0 ## encoding: [0x62,0xf1,0xfd,0x48,0xd4,0xc3]
+; X64-NEXT:    vpaddq %zmm0, %zmm2, %zmm0 ## encoding: [0x62,0xf1,0xed,0x48,0xd4,0xc0]
+; X64-NEXT:    retq ## encoding: [0xc3]
+  %1 = call <8 x i64> @llvm.x86.avx512.prorv.q.512(<8 x i64> %x0, <8 x i64> %x1)
+  %2 = bitcast i8 %x3 to <8 x i1>
+  %3 = select <8 x i1> %2, <8 x i64> %1, <8 x i64> %x2
+  %4 = call <8 x i64> @llvm.x86.avx512.prorv.q.512(<8 x i64> %x0, <8 x i64> %x1)
+  %5 = bitcast i8 %x3 to <8 x i1>
+  %6 = select <8 x i1> %5, <8 x i64> %4, <8 x i64> zeroinitializer
+  %7 = call <8 x i64> @llvm.x86.avx512.prorv.q.512(<8 x i64> %x0, <8 x i64> %x1)
+  %res3 = add <8 x i64> %3, %6
+  %res4 = add <8 x i64> %res3, %7
+  ret <8 x i64> %res4
+}
+
+declare <16 x i32> @llvm.x86.avx512.prol.d.512(<16 x i32>, i32)
+
+define <16 x i32>@test_int_x86_avx512_prol_d_512(<16 x i32> %x0, i32 %x1, <16 x i32> %x2, i16 %x3) {
+; X86-LABEL: test_int_x86_avx512_prol_d_512:
+; X86:       ## %bb.0:
+; X86-NEXT:    vprold $3, %zmm0, %zmm2 ## encoding: [0x62,0xf1,0x6d,0x48,0x72,0xc8,0x03]
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 ## encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x08]
+; X86-NEXT:    vprold $3, %zmm0, %zmm1 {%k1} ## encoding: [0x62,0xf1,0x75,0x49,0x72,0xc8,0x03]
+; X86-NEXT:    vprold $3, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0xc9,0x72,0xc8,0x03]
+; X86-NEXT:    vpaddd %zmm2, %zmm0, %zmm0 ## encoding: [0x62,0xf1,0x7d,0x48,0xfe,0xc2]
+; X86-NEXT:    vpaddd %zmm0, %zmm1, %zmm0 ## encoding: [0x62,0xf1,0x75,0x48,0xfe,0xc0]
+; X86-NEXT:    retl ## encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_prol_d_512:
+; X64:       ## %bb.0:
+; X64-NEXT:    vprold $3, %zmm0, %zmm2 ## encoding: [0x62,0xf1,0x6d,0x48,0x72,0xc8,0x03]
+; X64-NEXT:    kmovw %esi, %k1 ## encoding: [0xc5,0xf8,0x92,0xce]
+; X64-NEXT:    vprold $3, %zmm0, %zmm1 {%k1} ## encoding: [0x62,0xf1,0x75,0x49,0x72,0xc8,0x03]
+; X64-NEXT:    vprold $3, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0xc9,0x72,0xc8,0x03]
+; X64-NEXT:    vpaddd %zmm2, %zmm0, %zmm0 ## encoding: [0x62,0xf1,0x7d,0x48,0xfe,0xc2]
+; X64-NEXT:    vpaddd %zmm0, %zmm1, %zmm0 ## encoding: [0x62,0xf1,0x75,0x48,0xfe,0xc0]
+; X64-NEXT:    retq ## encoding: [0xc3]
+  %1 = call <16 x i32> @llvm.x86.avx512.prol.d.512(<16 x i32> %x0, i32 3)
+  %2 = bitcast i16 %x3 to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x i32> %1, <16 x i32> %x2
+  %4 = call <16 x i32> @llvm.x86.avx512.prol.d.512(<16 x i32> %x0, i32 3)
+  %5 = bitcast i16 %x3 to <16 x i1>
+  %6 = select <16 x i1> %5, <16 x i32> %4, <16 x i32> zeroinitializer
+  %7 = call <16 x i32> @llvm.x86.avx512.prol.d.512(<16 x i32> %x0, i32 3)
+  %res3 = add <16 x i32> %3, %6
+  %res4 = add <16 x i32> %res3, %7
+  ret <16 x i32> %res4
+}
+
+declare <8 x i64> @llvm.x86.avx512.prol.q.512(<8 x i64>, i32)
+
+define <8 x i64>@test_int_x86_avx512_prol_q_512(<8 x i64> %x0, i32 %x1, <8 x i64> %x2, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_prol_q_512:
+; X86:       ## %bb.0:
+; X86-NEXT:    vprolq $3, %zmm0, %zmm2 ## encoding: [0x62,0xf1,0xed,0x48,0x72,0xc8,0x03]
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax ## encoding: [0x0f,0xb6,0x44,0x24,0x08]
+; X86-NEXT:    kmovw %eax, %k1 ## encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vprolq $3, %zmm0, %zmm1 {%k1} ## encoding: [0x62,0xf1,0xf5,0x49,0x72,0xc8,0x03]
+; X86-NEXT:    vprolq $3, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf1,0xfd,0xc9,0x72,0xc8,0x03]
+; X86-NEXT:    vpaddq %zmm2, %zmm0, %zmm0 ## encoding: [0x62,0xf1,0xfd,0x48,0xd4,0xc2]
+; X86-NEXT:    vpaddq %zmm0, %zmm1, %zmm0 ## encoding: [0x62,0xf1,0xf5,0x48,0xd4,0xc0]
+; X86-NEXT:    retl ## encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_prol_q_512:
+; X64:       ## %bb.0:
+; X64-NEXT:    vprolq $3, %zmm0, %zmm2 ## encoding: [0x62,0xf1,0xed,0x48,0x72,0xc8,0x03]
+; X64-NEXT:    kmovw %esi, %k1 ## encoding: [0xc5,0xf8,0x92,0xce]
+; X64-NEXT:    vprolq $3, %zmm0, %zmm1 {%k1} ## encoding: [0x62,0xf1,0xf5,0x49,0x72,0xc8,0x03]
+; X64-NEXT:    vprolq $3, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf1,0xfd,0xc9,0x72,0xc8,0x03]
+; X64-NEXT:    vpaddq %zmm2, %zmm0, %zmm0 ## encoding: [0x62,0xf1,0xfd,0x48,0xd4,0xc2]
+; X64-NEXT:    vpaddq %zmm0, %zmm1, %zmm0 ## encoding: [0x62,0xf1,0xf5,0x48,0xd4,0xc0]
+; X64-NEXT:    retq ## encoding: [0xc3]
+  %1 = call <8 x i64> @llvm.x86.avx512.prol.q.512(<8 x i64> %x0, i32 3)
+  %2 = bitcast i8 %x3 to <8 x i1>
+  %3 = select <8 x i1> %2, <8 x i64> %1, <8 x i64> %x2
+  %4 = call <8 x i64> @llvm.x86.avx512.prol.q.512(<8 x i64> %x0, i32 3)
+  %5 = bitcast i8 %x3 to <8 x i1>
+  %6 = select <8 x i1> %5, <8 x i64> %4, <8 x i64> zeroinitializer
+  %7 = call <8 x i64> @llvm.x86.avx512.prol.q.512(<8 x i64> %x0, i32 3)
+  %res3 = add <8 x i64> %3, %6
+  %res4 = add <8 x i64> %res3, %7
+  ret <8 x i64> %res4
+}
+
+declare <16 x i32> @llvm.x86.avx512.pror.d.512(<16 x i32>, i32)
+
+define <16 x i32>@test_int_x86_avx512_pror_d_512(<16 x i32> %x0, i32 %x1, <16 x i32> %x2, i16 %x3) {
+; X86-LABEL: test_int_x86_avx512_pror_d_512:
+; X86:       ## %bb.0:
+; X86-NEXT:    vprord $3, %zmm0, %zmm2 ## encoding: [0x62,0xf1,0x6d,0x48,0x72,0xc0,0x03]
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 ## encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x08]
+; X86-NEXT:    vprord $3, %zmm0, %zmm1 {%k1} ## encoding: [0x62,0xf1,0x75,0x49,0x72,0xc0,0x03]
+; X86-NEXT:    vprord $3, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0xc9,0x72,0xc0,0x03]
+; X86-NEXT:    vpaddd %zmm2, %zmm0, %zmm0 ## encoding: [0x62,0xf1,0x7d,0x48,0xfe,0xc2]
+; X86-NEXT:    vpaddd %zmm0, %zmm1, %zmm0 ## encoding: [0x62,0xf1,0x75,0x48,0xfe,0xc0]
+; X86-NEXT:    retl ## encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_pror_d_512:
+; X64:       ## %bb.0:
+; X64-NEXT:    vprord $3, %zmm0, %zmm2 ## encoding: [0x62,0xf1,0x6d,0x48,0x72,0xc0,0x03]
+; X64-NEXT:    kmovw %esi, %k1 ## encoding: [0xc5,0xf8,0x92,0xce]
+; X64-NEXT:    vprord $3, %zmm0, %zmm1 {%k1} ## encoding: [0x62,0xf1,0x75,0x49,0x72,0xc0,0x03]
+; X64-NEXT:    vprord $3, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0xc9,0x72,0xc0,0x03]
+; X64-NEXT:    vpaddd %zmm2, %zmm0, %zmm0 ## encoding: [0x62,0xf1,0x7d,0x48,0xfe,0xc2]
+; X64-NEXT:    vpaddd %zmm0, %zmm1, %zmm0 ## encoding: [0x62,0xf1,0x75,0x48,0xfe,0xc0]
+; X64-NEXT:    retq ## encoding: [0xc3]
+  %1 = call <16 x i32> @llvm.x86.avx512.pror.d.512(<16 x i32> %x0, i32 3)
+  %2 = bitcast i16 %x3 to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x i32> %1, <16 x i32> %x2
+  %4 = call <16 x i32> @llvm.x86.avx512.pror.d.512(<16 x i32> %x0, i32 3)
+  %5 = bitcast i16 %x3 to <16 x i1>
+  %6 = select <16 x i1> %5, <16 x i32> %4, <16 x i32> zeroinitializer
+  %7 = call <16 x i32> @llvm.x86.avx512.pror.d.512(<16 x i32> %x0, i32 3)
+  %res3 = add <16 x i32> %3, %6
+  %res4 = add <16 x i32> %res3, %7
+  ret <16 x i32> %res4
+}
+
+declare <8 x i64> @llvm.x86.avx512.pror.q.512(<8 x i64>, i32)
+
+define <8 x i64>@test_int_x86_avx512_pror_q_512(<8 x i64> %x0, i32 %x1, <8 x i64> %x2, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_pror_q_512:
+; X86:       ## %bb.0:
+; X86-NEXT:    vprorq $3, %zmm0, %zmm2 ## encoding: [0x62,0xf1,0xed,0x48,0x72,0xc0,0x03]
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax ## encoding: [0x0f,0xb6,0x44,0x24,0x08]
+; X86-NEXT:    kmovw %eax, %k1 ## encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vprorq $3, %zmm0, %zmm1 {%k1} ## encoding: [0x62,0xf1,0xf5,0x49,0x72,0xc0,0x03]
+; X86-NEXT:    vprorq $3, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf1,0xfd,0xc9,0x72,0xc0,0x03]
+; X86-NEXT:    vpaddq %zmm2, %zmm0, %zmm0 ## encoding: [0x62,0xf1,0xfd,0x48,0xd4,0xc2]
+; X86-NEXT:    vpaddq %zmm0, %zmm1, %zmm0 ## encoding: [0x62,0xf1,0xf5,0x48,0xd4,0xc0]
+; X86-NEXT:    retl ## encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_pror_q_512:
+; X64:       ## %bb.0:
+; X64-NEXT:    vprorq $3, %zmm0, %zmm2 ## encoding: [0x62,0xf1,0xed,0x48,0x72,0xc0,0x03]
+; X64-NEXT:    kmovw %esi, %k1 ## encoding: [0xc5,0xf8,0x92,0xce]
+; X64-NEXT:    vprorq $3, %zmm0, %zmm1 {%k1} ## encoding: [0x62,0xf1,0xf5,0x49,0x72,0xc0,0x03]
+; X64-NEXT:    vprorq $3, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf1,0xfd,0xc9,0x72,0xc0,0x03]
+; X64-NEXT:    vpaddq %zmm2, %zmm0, %zmm0 ## encoding: [0x62,0xf1,0xfd,0x48,0xd4,0xc2]
+; X64-NEXT:    vpaddq %zmm0, %zmm1, %zmm0 ## encoding: [0x62,0xf1,0xf5,0x48,0xd4,0xc0]
+; X64-NEXT:    retq ## encoding: [0xc3]
+  %1 = call <8 x i64> @llvm.x86.avx512.pror.q.512(<8 x i64> %x0, i32 3)
+  %2 = bitcast i8 %x3 to <8 x i1>
+  %3 = select <8 x i1> %2, <8 x i64> %1, <8 x i64> %x2
+  %4 = call <8 x i64> @llvm.x86.avx512.pror.q.512(<8 x i64> %x0, i32 3)
+  %5 = bitcast i8 %x3 to <8 x i1>
+  %6 = select <8 x i1> %5, <8 x i64> %4, <8 x i64> zeroinitializer
+  %7 = call <8 x i64> @llvm.x86.avx512.pror.q.512(<8 x i64> %x0, i32 3)
+  %res3 = add <8 x i64> %3, %6
+  %res4 = add <8 x i64> %res3, %7
+  ret <8 x i64> %res4
+}
+
 declare <8 x i64> @llvm.x86.avx512.mask.psrl.qi.512(<8 x i64>, i32, <8 x i64>, i8)
 
 define <8 x i64>@test_int_x86_avx512_mask_psrl_qi_512(<8 x i64> %x0, i32 %x1, <8 x i64> %x2, i8 %x3) {
@@ -4380,14 +4654,14 @@ define <16 x float>@test_int_x86_avx512_mask_vpermilvar_ps_512_constant_pool(<16
 ; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 ## encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
 ; X86-NEXT:    vpermilps {{.*#+}} zmm2 {%k1} = zmm0[2,3,0,1,7,6,5,4,9,8,11,10,12,13,14,15]
 ; X86-NEXT:    ## encoding: [0x62,0xf2,0x7d,0x49,0x0c,0x15,A,A,A,A]
-; X86-NEXT:    ## fixup A - offset: 6, value: LCPI203_0, kind: FK_Data_4
+; X86-NEXT:    ## fixup A - offset: 6, value: LCPI211_0, kind: FK_Data_4
 ; X86-NEXT:    vpermilps {{.*#+}} zmm1 {%k1} {z} = zmm0[1,0,3,2,4,5,6,7,9,8,11,10,12,13,14,15]
 ; X86-NEXT:    ## encoding: [0x62,0xf2,0x7d,0xc9,0x0c,0x0d,A,A,A,A]
-; X86-NEXT:    ## fixup A - offset: 6, value: LCPI203_1, kind: FK_Data_4
+; X86-NEXT:    ## fixup A - offset: 6, value: LCPI211_1, kind: FK_Data_4
 ; X86-NEXT:    vaddps %zmm1, %zmm2, %zmm1 ## encoding: [0x62,0xf1,0x6c,0x48,0x58,0xc9]
 ; X86-NEXT:    vpermilps {{.*#+}} zmm0 = zmm0[1,0,3,2,4,5,6,7,10,11,8,9,14,15,13,12]
 ; X86-NEXT:    ## encoding: [0x62,0xf2,0x7d,0x48,0x0c,0x05,A,A,A,A]
-; X86-NEXT:    ## fixup A - offset: 6, value: LCPI203_2, kind: FK_Data_4
+; X86-NEXT:    ## fixup A - offset: 6, value: LCPI211_2, kind: FK_Data_4
 ; X86-NEXT:    vaddps %zmm1, %zmm0, %zmm0 ## encoding: [0x62,0xf1,0x7c,0x48,0x58,0xc1]
 ; X86-NEXT:    retl ## encoding: [0xc3]
 ;
@@ -4396,14 +4670,14 @@ define <16 x float>@test_int_x86_avx512_mask_vpermilvar_ps_512_constant_pool(<16
 ; X64-NEXT:    kmovw %edi, %k1 ## encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vpermilps {{.*#+}} zmm2 {%k1} = zmm0[2,3,0,1,7,6,5,4,9,8,11,10,12,13,14,15]
 ; X64-NEXT:    ## encoding: [0x62,0xf2,0x7d,0x49,0x0c,0x15,A,A,A,A]
-; X64-NEXT:    ## fixup A - offset: 6, value: LCPI203_0-4, kind: reloc_riprel_4byte
+; X64-NEXT:    ## fixup A - offset: 6, value: LCPI211_0-4, kind: reloc_riprel_4byte
 ; X64-NEXT:    vpermilps {{.*#+}} zmm1 {%k1} {z} = zmm0[1,0,3,2,4,5,6,7,9,8,11,10,12,13,14,15]
 ; X64-NEXT:    ## encoding: [0x62,0xf2,0x7d,0xc9,0x0c,0x0d,A,A,A,A]
-; X64-NEXT:    ## fixup A - offset: 6, value: LCPI203_1-4, kind: reloc_riprel_4byte
+; X64-NEXT:    ## fixup A - offset: 6, value: LCPI211_1-4, kind: reloc_riprel_4byte
 ; X64-NEXT:    vaddps %zmm1, %zmm2, %zmm1 ## encoding: [0x62,0xf1,0x6c,0x48,0x58,0xc9]
 ; X64-NEXT:    vpermilps {{.*#+}} zmm0 = zmm0[1,0,3,2,4,5,6,7,10,11,8,9,14,15,13,12]
 ; X64-NEXT:    ## encoding: [0x62,0xf2,0x7d,0x48,0x0c,0x05,A,A,A,A]
-; X64-NEXT:    ## fixup A - offset: 6, value: LCPI203_2-4, kind: reloc_riprel_4byte
+; X64-NEXT:    ## fixup A - offset: 6, value: LCPI211_2-4, kind: reloc_riprel_4byte
 ; X64-NEXT:    vaddps %zmm1, %zmm0, %zmm0 ## encoding: [0x62,0xf1,0x7c,0x48,0x58,0xc1]
 ; X64-NEXT:    retq ## encoding: [0xc3]
   %res = call <16 x float> @llvm.x86.avx512.mask.vpermilvar.ps.512(<16 x float> %x0, <16 x i32> <i32 2, i32 3, i32 0, i32 1, i32 3, i32 2, i32 1, i32 0, i32 1, i32 0, i32 3, i32 2, i32 0, i32 1, i32 2, i32 3>, <16 x float> %x2, i16 %x3)
@@ -9429,3 +9703,442 @@ define <4 x float>@test_int_x86_avx512_maskz_vfmadd_ss_rm(<4 x float> %x0, <4 x
   %res = call <4 x float> @llvm.x86.avx512.maskz.vfmadd.ss(<4 x float> %x0, <4 x float> %x1, <4 x float> %vecinit.i, i8 0, i32 4)
   ret < 4 x float> %res
 }
+
+declare <8 x i32> @llvm.x86.avx512.mask.pmov.qd.512(<8 x i64>, <8 x i32>, i8)
+
+define <8 x i32>@test_int_x86_avx512_mask_pmov_qd_512(<8 x i64> %x0, <8 x i32> %x1, i8 %x2) {
+; X86-LABEL: test_int_x86_avx512_mask_pmov_qd_512:
+; X86:       ## %bb.0:
+; X86-NEXT:    vpmovqd %zmm0, %ymm2 ## encoding: [0x62,0xf2,0x7e,0x48,0x35,0xc2]
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax ## encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 ## encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vpmovqd %zmm0, %ymm1 {%k1} ## encoding: [0x62,0xf2,0x7e,0x49,0x35,0xc1]
+; X86-NEXT:    vpmovqd %zmm0, %ymm0 {%k1} {z} ## encoding: [0x62,0xf2,0x7e,0xc9,0x35,0xc0]
+; X86-NEXT:    vpaddd %ymm0, %ymm1, %ymm0 ## encoding: [0xc5,0xf5,0xfe,0xc0]
+; X86-NEXT:    vpaddd %ymm0, %ymm2, %ymm0 ## encoding: [0xc5,0xed,0xfe,0xc0]
+; X86-NEXT:    retl ## encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_mask_pmov_qd_512:
+; X64:       ## %bb.0:
+; X64-NEXT:    vpmovqd %zmm0, %ymm2 ## encoding: [0x62,0xf2,0x7e,0x48,0x35,0xc2]
+; X64-NEXT:    kmovw %edi, %k1 ## encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vpmovqd %zmm0, %ymm1 {%k1} ## encoding: [0x62,0xf2,0x7e,0x49,0x35,0xc1]
+; X64-NEXT:    vpmovqd %zmm0, %ymm0 {%k1} {z} ## encoding: [0x62,0xf2,0x7e,0xc9,0x35,0xc0]
+; X64-NEXT:    vpaddd %ymm0, %ymm1, %ymm0 ## encoding: [0xc5,0xf5,0xfe,0xc0]
+; X64-NEXT:    vpaddd %ymm0, %ymm2, %ymm0 ## encoding: [0xc5,0xed,0xfe,0xc0]
+; X64-NEXT:    retq ## encoding: [0xc3]
+    %res0 = call <8 x i32> @llvm.x86.avx512.mask.pmov.qd.512(<8 x i64> %x0, <8 x i32> %x1, i8 -1)
+    %res1 = call <8 x i32> @llvm.x86.avx512.mask.pmov.qd.512(<8 x i64> %x0, <8 x i32> %x1, i8 %x2)
+    %res2 = call <8 x i32> @llvm.x86.avx512.mask.pmov.qd.512(<8 x i64> %x0, <8 x i32> zeroinitializer, i8 %x2)
+    %res3 = add <8 x i32> %res0, %res1
+    %res4 = add <8 x i32> %res3, %res2
+    ret <8 x i32> %res4
+}
+
+declare <16 x float> @llvm.x86.avx512.mask.cvtdq2ps.512(<16 x i32>, <16 x float>, i16, i32)
+
+define <16 x float> @test_int_x86_avx512_mask_cvt_dq2ps_512(<16 x i32> %x0, <16 x float> %x1, i16 %x2) {
+; X86-LABEL: test_int_x86_avx512_mask_cvt_dq2ps_512:
+; X86:       ## %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 ## encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
+; X86-NEXT:    vcvtdq2ps %zmm0, %zmm1 {%k1} ## encoding: [0x62,0xf1,0x7c,0x49,0x5b,0xc8]
+; X86-NEXT:    vcvtdq2ps {rn-sae}, %zmm0, %zmm0 ## encoding: [0x62,0xf1,0x7c,0x18,0x5b,0xc0]
+; X86-NEXT:    vaddps %zmm0, %zmm1, %zmm0 ## encoding: [0x62,0xf1,0x74,0x48,0x58,0xc0]
+; X86-NEXT:    retl ## encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_mask_cvt_dq2ps_512:
+; X64:       ## %bb.0:
+; X64-NEXT:    kmovw %edi, %k1 ## encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vcvtdq2ps %zmm0, %zmm1 {%k1} ## encoding: [0x62,0xf1,0x7c,0x49,0x5b,0xc8]
+; X64-NEXT:    vcvtdq2ps {rn-sae}, %zmm0, %zmm0 ## encoding: [0x62,0xf1,0x7c,0x18,0x5b,0xc0]
+; X64-NEXT:    vaddps %zmm0, %zmm1, %zmm0 ## encoding: [0x62,0xf1,0x74,0x48,0x58,0xc0]
+; X64-NEXT:    retq ## encoding: [0xc3]
+  %res = call <16 x float> @llvm.x86.avx512.mask.cvtdq2ps.512(<16 x i32> %x0, <16 x float> %x1, i16 %x2, i32 4)
+  %res1 = call <16 x float> @llvm.x86.avx512.mask.cvtdq2ps.512(<16 x i32> %x0, <16 x float> %x1, i16 -1, i32 0)
+  %res2 = fadd <16 x float> %res, %res1
+  ret <16 x float> %res2
+}
+
+declare <16 x float> @llvm.x86.avx512.mask.cvtudq2ps.512(<16 x i32>, <16 x float>, i16, i32)
+
+define <16 x float> @test_int_x86_avx512_mask_cvt_udq2ps_512(<16 x i32> %x0, <16 x float> %x1, i16 %x2) {
+; X86-LABEL: test_int_x86_avx512_mask_cvt_udq2ps_512:
+; X86:       ## %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 ## encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
+; X86-NEXT:    vcvtudq2ps %zmm0, %zmm1 {%k1} ## encoding: [0x62,0xf1,0x7f,0x49,0x7a,0xc8]
+; X86-NEXT:    vcvtudq2ps {rn-sae}, %zmm0, %zmm0 ## encoding: [0x62,0xf1,0x7f,0x18,0x7a,0xc0]
+; X86-NEXT:    vaddps %zmm0, %zmm1, %zmm0 ## encoding: [0x62,0xf1,0x74,0x48,0x58,0xc0]
+; X86-NEXT:    retl ## encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_mask_cvt_udq2ps_512:
+; X64:       ## %bb.0:
+; X64-NEXT:    kmovw %edi, %k1 ## encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vcvtudq2ps %zmm0, %zmm1 {%k1} ## encoding: [0x62,0xf1,0x7f,0x49,0x7a,0xc8]
+; X64-NEXT:    vcvtudq2ps {rn-sae}, %zmm0, %zmm0 ## encoding: [0x62,0xf1,0x7f,0x18,0x7a,0xc0]
+; X64-NEXT:    vaddps %zmm0, %zmm1, %zmm0 ## encoding: [0x62,0xf1,0x74,0x48,0x58,0xc0]
+; X64-NEXT:    retq ## encoding: [0xc3]
+  %res = call <16 x float> @llvm.x86.avx512.mask.cvtudq2ps.512(<16 x i32> %x0, <16 x float> %x1, i16 %x2, i32 4)
+  %res1 = call <16 x float> @llvm.x86.avx512.mask.cvtudq2ps.512(<16 x i32> %x0, <16 x float> %x1, i16 -1, i32 0)
+  %res2 = fadd <16 x float> %res, %res1
+  ret <16 x float> %res2
+}
+
+define <8 x double> @test_mask_compress_pd_512(<8 x double> %data, <8 x double> %passthru, i8 %mask) {
+; X86-LABEL: test_mask_compress_pd_512:
+; X86:       ## %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax ## encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 ## encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vcompresspd %zmm0, %zmm1 {%k1} ## encoding: [0x62,0xf2,0xfd,0x49,0x8a,0xc1]
+; X86-NEXT:    vmovdqa64 %zmm1, %zmm0 ## encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc1]
+; X86-NEXT:    retl ## encoding: [0xc3]
+;
+; X64-LABEL: test_mask_compress_pd_512:
+; X64:       ## %bb.0:
+; X64-NEXT:    kmovw %edi, %k1 ## encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vcompresspd %zmm0, %zmm1 {%k1} ## encoding: [0x62,0xf2,0xfd,0x49,0x8a,0xc1]
+; X64-NEXT:    vmovdqa64 %zmm1, %zmm0 ## encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc1]
+; X64-NEXT:    retq ## encoding: [0xc3]
+  %res = call <8 x double> @llvm.x86.avx512.mask.compress.pd.512(<8 x double> %data, <8 x double> %passthru, i8 %mask)
+  ret <8 x double> %res
+}
+
+define <8 x double> @test_maskz_compress_pd_512(<8 x double> %data, i8 %mask) {
+; X86-LABEL: test_maskz_compress_pd_512:
+; X86:       ## %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax ## encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 ## encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vcompresspd %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf2,0xfd,0xc9,0x8a,0xc0]
+; X86-NEXT:    retl ## encoding: [0xc3]
+;
+; X64-LABEL: test_maskz_compress_pd_512:
+; X64:       ## %bb.0:
+; X64-NEXT:    kmovw %edi, %k1 ## encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vcompresspd %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf2,0xfd,0xc9,0x8a,0xc0]
+; X64-NEXT:    retq ## encoding: [0xc3]
+  %res = call <8 x double> @llvm.x86.avx512.mask.compress.pd.512(<8 x double> %data, <8 x double> zeroinitializer, i8 %mask)
+  ret <8 x double> %res
+}
+
+define <8 x double> @test_compress_pd_512(<8 x double> %data) {
+; CHECK-LABEL: test_compress_pd_512:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+  %res = call <8 x double> @llvm.x86.avx512.mask.compress.pd.512(<8 x double> %data, <8 x double> undef, i8 -1)
+  ret <8 x double> %res
+}
+
+declare <8 x double> @llvm.x86.avx512.mask.compress.pd.512(<8 x double> %data, <8 x double> %src0, i8 %mask)
+
+define <16 x float> @test_mask_compress_ps_512(<16 x float> %data, <16 x float> %passthru, i16 %mask) {
+; X86-LABEL: test_mask_compress_ps_512:
+; X86:       ## %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 ## encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
+; X86-NEXT:    vcompressps %zmm0, %zmm1 {%k1} ## encoding: [0x62,0xf2,0x7d,0x49,0x8a,0xc1]
+; X86-NEXT:    vmovdqa64 %zmm1, %zmm0 ## encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc1]
+; X86-NEXT:    retl ## encoding: [0xc3]
+;
+; X64-LABEL: test_mask_compress_ps_512:
+; X64:       ## %bb.0:
+; X64-NEXT:    kmovw %edi, %k1 ## encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vcompressps %zmm0, %zmm1 {%k1} ## encoding: [0x62,0xf2,0x7d,0x49,0x8a,0xc1]
+; X64-NEXT:    vmovdqa64 %zmm1, %zmm0 ## encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc1]
+; X64-NEXT:    retq ## encoding: [0xc3]
+  %res = call <16 x float> @llvm.x86.avx512.mask.compress.ps.512(<16 x float> %data, <16 x float> %passthru, i16 %mask)
+  ret <16 x float> %res
+}
+
+define <16 x float> @test_maskz_compress_ps_512(<16 x float> %data, i16 %mask) {
+; X86-LABEL: test_maskz_compress_ps_512:
+; X86:       ## %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 ## encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
+; X86-NEXT:    vcompressps %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf2,0x7d,0xc9,0x8a,0xc0]
+; X86-NEXT:    retl ## encoding: [0xc3]
+;
+; X64-LABEL: test_maskz_compress_ps_512:
+; X64:       ## %bb.0:
+; X64-NEXT:    kmovw %edi, %k1 ## encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vcompressps %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf2,0x7d,0xc9,0x8a,0xc0]
+; X64-NEXT:    retq ## encoding: [0xc3]
+  %res = call <16 x float> @llvm.x86.avx512.mask.compress.ps.512(<16 x float> %data, <16 x float> zeroinitializer, i16 %mask)
+  ret <16 x float> %res
+}
+
+define <16 x float> @test_compress_ps_512(<16 x float> %data) {
+; CHECK-LABEL: test_compress_ps_512:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+  %res = call <16 x float> @llvm.x86.avx512.mask.compress.ps.512(<16 x float> %data, <16 x float> undef, i16 -1)
+  ret <16 x float> %res
+}
+
+declare <16 x float> @llvm.x86.avx512.mask.compress.ps.512(<16 x float> %data, <16 x float> %src0, i16 %mask)
+
+define <8 x i64> @test_mask_compress_q_512(<8 x i64> %data, <8 x i64> %passthru, i8 %mask) {
+; X86-LABEL: test_mask_compress_q_512:
+; X86:       ## %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax ## encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 ## encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vpcompressq %zmm0, %zmm1 {%k1} ## encoding: [0x62,0xf2,0xfd,0x49,0x8b,0xc1]
+; X86-NEXT:    vmovdqa64 %zmm1, %zmm0 ## encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc1]
+; X86-NEXT:    retl ## encoding: [0xc3]
+;
+; X64-LABEL: test_mask_compress_q_512:
+; X64:       ## %bb.0:
+; X64-NEXT:    kmovw %edi, %k1 ## encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vpcompressq %zmm0, %zmm1 {%k1} ## encoding: [0x62,0xf2,0xfd,0x49,0x8b,0xc1]
+; X64-NEXT:    vmovdqa64 %zmm1, %zmm0 ## encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc1]
+; X64-NEXT:    retq ## encoding: [0xc3]
+  %res = call <8 x i64> @llvm.x86.avx512.mask.compress.q.512(<8 x i64> %data, <8 x i64> %passthru, i8 %mask)
+  ret <8 x i64> %res
+}
+
+define <8 x i64> @test_maskz_compress_q_512(<8 x i64> %data, i8 %mask) {
+; X86-LABEL: test_maskz_compress_q_512:
+; X86:       ## %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax ## encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 ## encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vpcompressq %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf2,0xfd,0xc9,0x8b,0xc0]
+; X86-NEXT:    retl ## encoding: [0xc3]
+;
+; X64-LABEL: test_maskz_compress_q_512:
+; X64:       ## %bb.0:
+; X64-NEXT:    kmovw %edi, %k1 ## encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vpcompressq %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf2,0xfd,0xc9,0x8b,0xc0]
+; X64-NEXT:    retq ## encoding: [0xc3]
+  %res = call <8 x i64> @llvm.x86.avx512.mask.compress.q.512(<8 x i64> %data, <8 x i64> zeroinitializer, i8 %mask)
+  ret <8 x i64> %res
+}
+
+define <8 x i64> @test_compress_q_512(<8 x i64> %data) {
+; CHECK-LABEL: test_compress_q_512:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+  %res = call <8 x i64> @llvm.x86.avx512.mask.compress.q.512(<8 x i64> %data, <8 x i64> undef, i8 -1)
+  ret <8 x i64> %res
+}
+
+declare <8 x i64> @llvm.x86.avx512.mask.compress.q.512(<8 x i64> %data, <8 x i64> %src0, i8 %mask)
+
+define <16 x i32> @test_mask_compress_d_512(<16 x i32> %data, <16 x i32> %passthru, i16 %mask) {
+; X86-LABEL: test_mask_compress_d_512:
+; X86:       ## %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 ## encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
+; X86-NEXT:    vpcompressd %zmm0, %zmm1 {%k1} ## encoding: [0x62,0xf2,0x7d,0x49,0x8b,0xc1]
+; X86-NEXT:    vmovdqa64 %zmm1, %zmm0 ## encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc1]
+; X86-NEXT:    retl ## encoding: [0xc3]
+;
+; X64-LABEL: test_mask_compress_d_512:
+; X64:       ## %bb.0:
+; X64-NEXT:    kmovw %edi, %k1 ## encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vpcompressd %zmm0, %zmm1 {%k1} ## encoding: [0x62,0xf2,0x7d,0x49,0x8b,0xc1]
+; X64-NEXT:    vmovdqa64 %zmm1, %zmm0 ## encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc1]
+; X64-NEXT:    retq ## encoding: [0xc3]
+  %res = call <16 x i32> @llvm.x86.avx512.mask.compress.d.512(<16 x i32> %data, <16 x i32> %passthru, i16 %mask)
+  ret <16 x i32> %res
+}
+
+define <16 x i32> @test_maskz_compress_d_512(<16 x i32> %data, i16 %mask) {
+; X86-LABEL: test_maskz_compress_d_512:
+; X86:       ## %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 ## encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
+; X86-NEXT:    vpcompressd %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf2,0x7d,0xc9,0x8b,0xc0]
+; X86-NEXT:    retl ## encoding: [0xc3]
+;
+; X64-LABEL: test_maskz_compress_d_512:
+; X64:       ## %bb.0:
+; X64-NEXT:    kmovw %edi, %k1 ## encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vpcompressd %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf2,0x7d,0xc9,0x8b,0xc0]
+; X64-NEXT:    retq ## encoding: [0xc3]
+  %res = call <16 x i32> @llvm.x86.avx512.mask.compress.d.512(<16 x i32> %data, <16 x i32> zeroinitializer, i16 %mask)
+  ret <16 x i32> %res
+}
+
+define <16 x i32> @test_compress_d_512(<16 x i32> %data) {
+; CHECK-LABEL: test_compress_d_512:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+  %res = call <16 x i32> @llvm.x86.avx512.mask.compress.d.512(<16 x i32> %data, <16 x i32> undef, i16 -1)
+  ret <16 x i32> %res
+}
+
+declare <16 x i32> @llvm.x86.avx512.mask.compress.d.512(<16 x i32> %data, <16 x i32> %src0, i16 %mask)
+
+define <8 x double> @test_expand_pd_512(<8 x double> %data) {
+; CHECK-LABEL: test_expand_pd_512:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+  %res = call <8 x double> @llvm.x86.avx512.mask.expand.pd.512(<8 x double> %data, <8 x double> undef, i8 -1)
+  ret <8 x double> %res
+}
+
+define <8 x double> @test_mask_expand_pd_512(<8 x double> %data, <8 x double> %passthru, i8 %mask) {
+; X86-LABEL: test_mask_expand_pd_512:
+; X86:       ## %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax ## encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 ## encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vexpandpd %zmm0, %zmm1 {%k1} ## encoding: [0x62,0xf2,0xfd,0x49,0x88,0xc8]
+; X86-NEXT:    vmovdqa64 %zmm1, %zmm0 ## encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc1]
+; X86-NEXT:    retl ## encoding: [0xc3]
+;
+; X64-LABEL: test_mask_expand_pd_512:
+; X64:       ## %bb.0:
+; X64-NEXT:    kmovw %edi, %k1 ## encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vexpandpd %zmm0, %zmm1 {%k1} ## encoding: [0x62,0xf2,0xfd,0x49,0x88,0xc8]
+; X64-NEXT:    vmovdqa64 %zmm1, %zmm0 ## encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc1]
+; X64-NEXT:    retq ## encoding: [0xc3]
+  %res = call <8 x double> @llvm.x86.avx512.mask.expand.pd.512(<8 x double> %data, <8 x double> %passthru, i8 %mask)
+  ret <8 x double> %res
+}
+
+define <8 x double> @test_maskz_expand_pd_512(<8 x double> %data, i8 %mask) {
+; X86-LABEL: test_maskz_expand_pd_512:
+; X86:       ## %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax ## encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 ## encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vexpandpd %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf2,0xfd,0xc9,0x88,0xc0]
+; X86-NEXT:    retl ## encoding: [0xc3]
+;
+; X64-LABEL: test_maskz_expand_pd_512:
+; X64:       ## %bb.0:
+; X64-NEXT:    kmovw %edi, %k1 ## encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vexpandpd %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf2,0xfd,0xc9,0x88,0xc0]
+; X64-NEXT:    retq ## encoding: [0xc3]
+  %res = call <8 x double> @llvm.x86.avx512.mask.expand.pd.512(<8 x double> %data, <8 x double> zeroinitializer, i8 %mask)
+  ret <8 x double> %res
+}
+
+declare <8 x double> @llvm.x86.avx512.mask.expand.pd.512(<8 x double> %data, <8 x double> %src0, i8 %mask)
+
+define <16 x float> @test_expand_ps_512(<16 x float> %data) {
+; CHECK-LABEL: test_expand_ps_512:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+  %res = call <16 x float> @llvm.x86.avx512.mask.expand.ps.512(<16 x float> %data, <16 x float> undef, i16 -1)
+  ret <16 x float> %res
+}
+
+define <16 x float> @test_mask_expand_ps_512(<16 x float> %data, <16 x float> %passthru, i16 %mask) {
+; X86-LABEL: test_mask_expand_ps_512:
+; X86:       ## %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 ## encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
+; X86-NEXT:    vexpandps %zmm0, %zmm1 {%k1} ## encoding: [0x62,0xf2,0x7d,0x49,0x88,0xc8]
+; X86-NEXT:    vmovdqa64 %zmm1, %zmm0 ## encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc1]
+; X86-NEXT:    retl ## encoding: [0xc3]
+;
+; X64-LABEL: test_mask_expand_ps_512:
+; X64:       ## %bb.0:
+; X64-NEXT:    kmovw %edi, %k1 ## encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vexpandps %zmm0, %zmm1 {%k1} ## encoding: [0x62,0xf2,0x7d,0x49,0x88,0xc8]
+; X64-NEXT:    vmovdqa64 %zmm1, %zmm0 ## encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc1]
+; X64-NEXT:    retq ## encoding: [0xc3]
+  %res = call <16 x float> @llvm.x86.avx512.mask.expand.ps.512(<16 x float> %data, <16 x float> %passthru, i16 %mask)
+  ret <16 x float> %res
+}
+
+define <16 x float> @test_maskz_expand_ps_512(<16 x float> %data, i16 %mask) {
+; X86-LABEL: test_maskz_expand_ps_512:
+; X86:       ## %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 ## encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
+; X86-NEXT:    vexpandps %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf2,0x7d,0xc9,0x88,0xc0]
+; X86-NEXT:    retl ## encoding: [0xc3]
+;
+; X64-LABEL: test_maskz_expand_ps_512:
+; X64:       ## %bb.0:
+; X64-NEXT:    kmovw %edi, %k1 ## encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vexpandps %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf2,0x7d,0xc9,0x88,0xc0]
+; X64-NEXT:    retq ## encoding: [0xc3]
+  %res = call <16 x float> @llvm.x86.avx512.mask.expand.ps.512(<16 x float> %data, <16 x float> zeroinitializer, i16 %mask)
+  ret <16 x float> %res
+}
+
+declare <16 x float> @llvm.x86.avx512.mask.expand.ps.512(<16 x float> %data, <16 x float> %src0, i16 %mask)
+
+define <8 x i64> @test_expand_q_512(<8 x i64> %data) {
+; CHECK-LABEL: test_expand_q_512:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+  %res = call <8 x i64> @llvm.x86.avx512.mask.expand.q.512(<8 x i64> %data, <8 x i64> undef, i8 -1)
+  ret <8 x i64> %res
+}
+
+define <8 x i64> @test_mask_expand_q_512(<8 x i64> %data, <8 x i64> %passthru, i8 %mask) {
+; X86-LABEL: test_mask_expand_q_512:
+; X86:       ## %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax ## encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 ## encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vpexpandq %zmm0, %zmm1 {%k1} ## encoding: [0x62,0xf2,0xfd,0x49,0x89,0xc8]
+; X86-NEXT:    vmovdqa64 %zmm1, %zmm0 ## encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc1]
+; X86-NEXT:    retl ## encoding: [0xc3]
+;
+; X64-LABEL: test_mask_expand_q_512:
+; X64:       ## %bb.0:
+; X64-NEXT:    kmovw %edi, %k1 ## encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vpexpandq %zmm0, %zmm1 {%k1} ## encoding: [0x62,0xf2,0xfd,0x49,0x89,0xc8]
+; X64-NEXT:    vmovdqa64 %zmm1, %zmm0 ## encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc1]
+; X64-NEXT:    retq ## encoding: [0xc3]
+  %res = call <8 x i64> @llvm.x86.avx512.mask.expand.q.512(<8 x i64> %data, <8 x i64> %passthru, i8 %mask)
+  ret <8 x i64> %res
+}
+
+define <8 x i64> @test_maskz_expand_q_512(<8 x i64> %data, i8 %mask) {
+; X86-LABEL: test_maskz_expand_q_512:
+; X86:       ## %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax ## encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 ## encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vpexpandq %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf2,0xfd,0xc9,0x89,0xc0]
+; X86-NEXT:    retl ## encoding: [0xc3]
+;
+; X64-LABEL: test_maskz_expand_q_512:
+; X64:       ## %bb.0:
+; X64-NEXT:    kmovw %edi, %k1 ## encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vpexpandq %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf2,0xfd,0xc9,0x89,0xc0]
+; X64-NEXT:    retq ## encoding: [0xc3]
+  %res = call <8 x i64> @llvm.x86.avx512.mask.expand.q.512(<8 x i64> %data, <8 x i64> zeroinitializer, i8 %mask)
+  ret <8 x i64> %res
+}
+
+declare <8 x i64> @llvm.x86.avx512.mask.expand.q.512(<8 x i64> %data, <8 x i64> %src0, i8 %mask)
+
+define <16 x i32> @test_expand_d_512(<16 x i32> %data) {
+; CHECK-LABEL: test_expand_d_512:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+  %res = call <16 x i32> @llvm.x86.avx512.mask.expand.d.512(<16 x i32> %data, <16 x i32> undef, i16 -1)
+  ret <16 x i32> %res
+}
+
+define <16 x i32> @test_mask_expand_d_512(<16 x i32> %data, <16 x i32> %passthru, i16 %mask) {
+; X86-LABEL: test_mask_expand_d_512:
+; X86:       ## %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 ## encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
+; X86-NEXT:    vpexpandd %zmm0, %zmm1 {%k1} ## encoding: [0x62,0xf2,0x7d,0x49,0x89,0xc8]
+; X86-NEXT:    vmovdqa64 %zmm1, %zmm0 ## encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc1]
+; X86-NEXT:    retl ## encoding: [0xc3]
+;
+; X64-LABEL: test_mask_expand_d_512:
+; X64:       ## %bb.0:
+; X64-NEXT:    kmovw %edi, %k1 ## encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vpexpandd %zmm0, %zmm1 {%k1} ## encoding: [0x62,0xf2,0x7d,0x49,0x89,0xc8]
+; X64-NEXT:    vmovdqa64 %zmm1, %zmm0 ## encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc1]
+; X64-NEXT:    retq ## encoding: [0xc3]
+  %res = call <16 x i32> @llvm.x86.avx512.mask.expand.d.512(<16 x i32> %data, <16 x i32> %passthru, i16 %mask)
+  ret <16 x i32> %res
+}
+
+define <16 x i32> @test_maskz_expand_d_512(<16 x i32> %data, i16 %mask) {
+; X86-LABEL: test_maskz_expand_d_512:
+; X86:       ## %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 ## encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
+; X86-NEXT:    vpexpandd %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf2,0x7d,0xc9,0x89,0xc0]
+; X86-NEXT:    retl ## encoding: [0xc3]
+;
+; X64-LABEL: test_maskz_expand_d_512:
+; X64:       ## %bb.0:
+; X64-NEXT:    kmovw %edi, %k1 ## encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vpexpandd %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf2,0x7d,0xc9,0x89,0xc0]
+; X64-NEXT:    retq ## encoding: [0xc3]
+  %res = call <16 x i32> @llvm.x86.avx512.mask.expand.d.512(<16 x i32> %data, <16 x i32> zeroinitializer, i16 %mask)
+  ret <16 x i32> %res
+}
+
+declare <16 x i32> @llvm.x86.avx512.mask.expand.d.512(<16 x i32> %data, <16 x i32> %src0, i16 %mask)
diff --git a/test/CodeGen/X86/avx512-intrinsics-x86_64.ll b/test/CodeGen/X86/avx512-intrinsics-x86_64.ll
new file mode 100644
index 0000000000000000000000000000000000000000..196b3e243436fd1695f0ef48beed2a328079ad21
--- /dev/null
+++ b/test/CodeGen/X86/avx512-intrinsics-x86_64.ll
@@ -0,0 +1,249 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -disable-peephole -mtriple=x86_64-apple-darwin -mattr=+avx512f | FileCheck %s
+
+
+define i64 @test_x86_sse2_cvtsd2si64(<2 x double> %a0) {
+; CHECK-LABEL: test_x86_sse2_cvtsd2si64:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    vcvtsd2si %xmm0, %rax
+; CHECK-NEXT:    retq
+  %res = call i64 @llvm.x86.sse2.cvtsd2si64(<2 x double> %a0) ; <i64> [#uses=1]
+  ret i64 %res
+}
+declare i64 @llvm.x86.sse2.cvtsd2si64(<2 x double>) nounwind readnone
+
+define <2 x double> @test_x86_sse2_cvtsi642sd(<2 x double> %a0, i64 %a1) {
+; CHECK-LABEL: test_x86_sse2_cvtsi642sd:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    vcvtsi2sdq %rdi, %xmm0, %xmm0
+; CHECK-NEXT:    retq
+  %res = call <2 x double> @llvm.x86.sse2.cvtsi642sd(<2 x double> %a0, i64 %a1) ; <<2 x double>> [#uses=1]
+  ret <2 x double> %res
+}
+declare <2 x double> @llvm.x86.sse2.cvtsi642sd(<2 x double>, i64) nounwind readnone
+
+define i64 @test_x86_avx512_cvttsd2si64(<2 x double> %a0) {
+; CHECK-LABEL: test_x86_avx512_cvttsd2si64:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    vcvttsd2si %xmm0, %rcx
+; CHECK-NEXT:    vcvttsd2si {sae}, %xmm0, %rax
+; CHECK-NEXT:    addq %rcx, %rax
+; CHECK-NEXT:    retq
+  %res0 = call i64 @llvm.x86.avx512.cvttsd2si64(<2 x double> %a0, i32 4) ;
+  %res1 = call i64 @llvm.x86.avx512.cvttsd2si64(<2 x double> %a0, i32 8) ;
+  %res2 = add i64 %res0, %res1
+  ret i64 %res2
+}
+declare i64 @llvm.x86.avx512.cvttsd2si64(<2 x double>, i32) nounwind readnone
+
+define i64 @test_x86_avx512_cvttsd2usi64(<2 x double> %a0) {
+; CHECK-LABEL: test_x86_avx512_cvttsd2usi64:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    vcvttsd2usi %xmm0, %rcx
+; CHECK-NEXT:    vcvttsd2usi {sae}, %xmm0, %rax
+; CHECK-NEXT:    addq %rcx, %rax
+; CHECK-NEXT:    retq
+  %res0 = call i64 @llvm.x86.avx512.cvttsd2usi64(<2 x double> %a0, i32 4) ;
+  %res1 = call i64 @llvm.x86.avx512.cvttsd2usi64(<2 x double> %a0, i32 8) ;
+  %res2 = add i64 %res0, %res1
+  ret i64 %res2
+}
+declare i64 @llvm.x86.avx512.cvttsd2usi64(<2 x double>, i32) nounwind readnone
+
+define i64 @test_x86_sse_cvtss2si64(<4 x float> %a0) {
+; CHECK-LABEL: test_x86_sse_cvtss2si64:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    vcvtss2si %xmm0, %rax
+; CHECK-NEXT:    retq
+  %res = call i64 @llvm.x86.sse.cvtss2si64(<4 x float> %a0) ; <i64> [#uses=1]
+  ret i64 %res
+}
+declare i64 @llvm.x86.sse.cvtss2si64(<4 x float>) nounwind readnone
+
+
+define <4 x float> @test_x86_sse_cvtsi642ss(<4 x float> %a0, i64 %a1) {
+; CHECK-LABEL: test_x86_sse_cvtsi642ss:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    vcvtsi2ssq %rdi, %xmm0, %xmm0
+; CHECK-NEXT:    retq
+  %res = call <4 x float> @llvm.x86.sse.cvtsi642ss(<4 x float> %a0, i64 %a1) ; <<4 x float>> [#uses=1]
+  ret <4 x float> %res
+}
+declare <4 x float> @llvm.x86.sse.cvtsi642ss(<4 x float>, i64) nounwind readnone
+
+
+define i64 @test_x86_avx512_cvttss2si64(<4 x float> %a0) {
+; CHECK-LABEL: test_x86_avx512_cvttss2si64:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    vcvttss2si %xmm0, %rcx
+; CHECK-NEXT:    vcvttss2si {sae}, %xmm0, %rax
+; CHECK-NEXT:    addq %rcx, %rax
+; CHECK-NEXT:    retq
+  %res0 = call i64 @llvm.x86.avx512.cvttss2si64(<4 x float> %a0, i32 4) ;
+  %res1 = call i64 @llvm.x86.avx512.cvttss2si64(<4 x float> %a0, i32 8) ;
+  %res2 = add i64 %res0, %res1
+  ret i64 %res2
+}
+declare i64 @llvm.x86.avx512.cvttss2si64(<4 x float>, i32) nounwind readnone
+
+define i32 @test_x86_avx512_cvttss2usi(<4 x float> %a0) {
+; CHECK-LABEL: test_x86_avx512_cvttss2usi:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    vcvttss2usi {sae}, %xmm0, %ecx
+; CHECK-NEXT:    vcvttss2usi %xmm0, %eax
+; CHECK-NEXT:    addl %ecx, %eax
+; CHECK-NEXT:    retq
+  %res0 = call i32 @llvm.x86.avx512.cvttss2usi(<4 x float> %a0, i32 8) ;
+  %res1 = call i32 @llvm.x86.avx512.cvttss2usi(<4 x float> %a0, i32 4) ;
+  %res2 = add i32 %res0, %res1
+  ret i32 %res2
+}
+declare i32 @llvm.x86.avx512.cvttss2usi(<4 x float>, i32) nounwind readnone
+
+define i64 @test_x86_avx512_cvttss2usi64(<4 x float> %a0) {
+; CHECK-LABEL: test_x86_avx512_cvttss2usi64:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    vcvttss2usi %xmm0, %rcx
+; CHECK-NEXT:    vcvttss2usi {sae}, %xmm0, %rax
+; CHECK-NEXT:    addq %rcx, %rax
+; CHECK-NEXT:    retq
+  %res0 = call i64 @llvm.x86.avx512.cvttss2usi64(<4 x float> %a0, i32 4) ;
+  %res1 = call i64 @llvm.x86.avx512.cvttss2usi64(<4 x float> %a0, i32 8) ;
+  %res2 = add i64 %res0, %res1
+  ret i64 %res2
+}
+declare i64 @llvm.x86.avx512.cvttss2usi64(<4 x float>, i32) nounwind readnone
+
+define i64 @test_x86_avx512_cvtsd2usi64(<2 x double> %a0) {
+; CHECK-LABEL: test_x86_avx512_cvtsd2usi64:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    vcvtsd2usi %xmm0, %rax
+; CHECK-NEXT:    vcvtsd2usi {rz-sae}, %xmm0, %rcx
+; CHECK-NEXT:    addq %rax, %rcx
+; CHECK-NEXT:    vcvtsd2usi {rd-sae}, %xmm0, %rax
+; CHECK-NEXT:    addq %rcx, %rax
+; CHECK-NEXT:    retq
+
+  %res = call i64 @llvm.x86.avx512.vcvtsd2usi64(<2 x double> %a0, i32 4)
+  %res1 = call i64 @llvm.x86.avx512.vcvtsd2usi64(<2 x double> %a0, i32 3)
+  %res2 = call i64 @llvm.x86.avx512.vcvtsd2usi64(<2 x double> %a0, i32 1)
+  %res3 = add i64 %res, %res1
+  %res4 = add i64 %res3, %res2
+  ret i64 %res4
+}
+declare i64 @llvm.x86.avx512.vcvtsd2usi64(<2 x double>, i32) nounwind readnone
+
+define i64 @test_x86_avx512_cvtsd2si64(<2 x double> %a0) {
+; CHECK-LABEL: test_x86_avx512_cvtsd2si64:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    vcvtsd2si %xmm0, %rax
+; CHECK-NEXT:    vcvtsd2si {rz-sae}, %xmm0, %rcx
+; CHECK-NEXT:    addq %rax, %rcx
+; CHECK-NEXT:    vcvtsd2si {rd-sae}, %xmm0, %rax
+; CHECK-NEXT:    addq %rcx, %rax
+; CHECK-NEXT:    retq
+
+  %res = call i64 @llvm.x86.avx512.vcvtsd2si64(<2 x double> %a0, i32 4)
+  %res1 = call i64 @llvm.x86.avx512.vcvtsd2si64(<2 x double> %a0, i32 3)
+  %res2 = call i64 @llvm.x86.avx512.vcvtsd2si64(<2 x double> %a0, i32 1)
+  %res3 = add i64 %res, %res1
+  %res4 = add i64 %res3, %res2
+  ret i64 %res4
+}
+declare i64 @llvm.x86.avx512.vcvtsd2si64(<2 x double>, i32) nounwind readnone
+
+define i64 @test_x86_avx512_cvtss2usi64(<4 x float> %a0) {
+; CHECK-LABEL: test_x86_avx512_cvtss2usi64:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    vcvtss2usi %xmm0, %rax
+; CHECK-NEXT:    vcvtss2usi {rz-sae}, %xmm0, %rcx
+; CHECK-NEXT:    addq %rax, %rcx
+; CHECK-NEXT:    vcvtss2usi {rd-sae}, %xmm0, %rax
+; CHECK-NEXT:    addq %rcx, %rax
+; CHECK-NEXT:    retq
+
+  %res = call i64 @llvm.x86.avx512.vcvtss2usi64(<4 x float> %a0, i32 4)
+  %res1 = call i64 @llvm.x86.avx512.vcvtss2usi64(<4 x float> %a0, i32 3)
+  %res2 = call i64 @llvm.x86.avx512.vcvtss2usi64(<4 x float> %a0, i32 1)
+  %res3 = add i64 %res, %res1
+  %res4 = add i64 %res3, %res2
+  ret i64 %res4
+}
+declare i64 @llvm.x86.avx512.vcvtss2usi64(<4 x float>, i32) nounwind readnone
+
+define i64 @test_x86_avx512_cvtss2si64(<4 x float> %a0) {
+; CHECK-LABEL: test_x86_avx512_cvtss2si64:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    vcvtss2si %xmm0, %rax
+; CHECK-NEXT:    vcvtss2si {rz-sae}, %xmm0, %rcx
+; CHECK-NEXT:    addq %rax, %rcx
+; CHECK-NEXT:    vcvtss2si {rd-sae}, %xmm0, %rax
+; CHECK-NEXT:    addq %rcx, %rax
+; CHECK-NEXT:    retq
+
+  %res = call i64 @llvm.x86.avx512.vcvtss2si64(<4 x float> %a0, i32 4)
+  %res1 = call i64 @llvm.x86.avx512.vcvtss2si64(<4 x float> %a0, i32 3)
+  %res2 = call i64 @llvm.x86.avx512.vcvtss2si64(<4 x float> %a0, i32 1)
+  %res3 = add i64 %res, %res1
+  %res4 = add i64 %res3, %res2
+  ret i64 %res4
+}
+declare i64 @llvm.x86.avx512.vcvtss2si64(<4 x float>, i32) nounwind readnone
+
+define <2 x double> @test_x86_avx512_cvtsi2sd64(<2 x double> %a, i64 %b) {
+; CHECK-LABEL: test_x86_avx512_cvtsi2sd64:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    vcvtsi2sdq %rdi, {rz-sae}, %xmm0, %xmm0
+; CHECK-NEXT:    retq
+  %res = call <2 x double> @llvm.x86.avx512.cvtsi2sd64(<2 x double> %a, i64 %b, i32 3) ; <<<2 x double>> [#uses=1]
+  ret <2 x double> %res
+}
+declare <2 x double> @llvm.x86.avx512.cvtsi2sd64(<2 x double>, i64, i32) nounwind readnone
+
+define <4 x float> @test_x86_avx512_cvtsi2ss64(<4 x float> %a, i64 %b) {
+; CHECK-LABEL: test_x86_avx512_cvtsi2ss64:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    vcvtsi2ssq %rdi, {rz-sae}, %xmm0, %xmm0
+; CHECK-NEXT:    retq
+  %res = call <4 x float> @llvm.x86.avx512.cvtsi2ss64(<4 x float> %a, i64 %b, i32 3) ; <<<4 x float>> [#uses=1]
+  ret <4 x float> %res
+}
+declare <4 x float> @llvm.x86.avx512.cvtsi2ss64(<4 x float>, i64, i32) nounwind readnone
+
+define <4 x float> @_mm_cvt_roundu64_ss (<4 x float> %a, i64 %b) {
+; CHECK-LABEL: _mm_cvt_roundu64_ss:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    vcvtusi2ssq %rdi, {rd-sae}, %xmm0, %xmm0
+; CHECK-NEXT:    retq
+  %res = call <4 x float> @llvm.x86.avx512.cvtusi642ss(<4 x float> %a, i64 %b, i32 1) ; <<<4 x float>> [#uses=1]
+  ret <4 x float> %res
+}
+
+define <4 x float> @_mm_cvtu64_ss(<4 x float> %a, i64 %b) {
+; CHECK-LABEL: _mm_cvtu64_ss:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    vcvtusi2ssq %rdi, %xmm0, %xmm0
+; CHECK-NEXT:    retq
+  %res = call <4 x float> @llvm.x86.avx512.cvtusi642ss(<4 x float> %a, i64 %b, i32 4) ; <<<4 x float>> [#uses=1]
+  ret <4 x float> %res
+}
+declare <4 x float> @llvm.x86.avx512.cvtusi642ss(<4 x float>, i64, i32) nounwind readnone
+
+define <2 x double> @test_x86_avx512_mm_cvtu64_sd(<2 x double> %a, i64 %b) {
+; CHECK-LABEL: test_x86_avx512_mm_cvtu64_sd:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    vcvtusi2sdq %rdi, {rd-sae}, %xmm0, %xmm0
+; CHECK-NEXT:    retq
+  %res = call <2 x double> @llvm.x86.avx512.cvtusi642sd(<2 x double> %a, i64 %b, i32 1) ; <<<2 x double>> [#uses=1]
+  ret <2 x double> %res
+}
+
+define <2 x double> @test_x86_avx512__mm_cvt_roundu64_sd(<2 x double> %a, i64 %b) {
+; CHECK-LABEL: test_x86_avx512__mm_cvt_roundu64_sd:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    vcvtusi2sdq %rdi, %xmm0, %xmm0
+; CHECK-NEXT:    retq
+  %res = call <2 x double> @llvm.x86.avx512.cvtusi642sd(<2 x double> %a, i64 %b, i32 4) ; <<<2 x double>> [#uses=1]
+  ret <2 x double> %res
+}
+declare <2 x double> @llvm.x86.avx512.cvtusi642sd(<2 x double>, i64, i32) nounwind readnone
diff --git a/test/CodeGen/X86/avx512-intrinsics.ll b/test/CodeGen/X86/avx512-intrinsics.ll
index 53a076087fe3cc723c272dd13e7d2e32a8c5ed56..3e3da21d56999d9d00a8cebd05f400f85aa595f6 100644
--- a/test/CodeGen/X86/avx512-intrinsics.ll
+++ b/test/CodeGen/X86/avx512-intrinsics.ll
@@ -1,260 +1,373 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -disable-peephole -mtriple=x86_64-apple-darwin -mattr=+avx512f | FileCheck %s
+; RUN: llc < %s -disable-peephole -mtriple=x86_64-unknown-unknown -mattr=+avx512f | FileCheck %s --check-prefixes=CHECK,X64
+; RUN: llc < %s -disable-peephole -mtriple=i686-unknown-unknown -mattr=+avx512f | FileCheck %s --check-prefixes=CHECK,X86
 
 
 define <8 x double> @test_mask_compress_pd_512(<8 x double> %data, <8 x double> %passthru, i8 %mask) {
-; CHECK-LABEL: test_mask_compress_pd_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vcompresspd %zmm0, %zmm1 {%k1}
-; CHECK-NEXT:    vmovdqa64 %zmm1, %zmm0
-; CHECK-NEXT:    retq
-  %res = call <8 x double> @llvm.x86.avx512.mask.compress.pd.512(<8 x double> %data, <8 x double> %passthru, i8 %mask)
-  ret <8 x double> %res
+; X64-LABEL: test_mask_compress_pd_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vcompresspd %zmm0, %zmm1 {%k1}
+; X64-NEXT:    vmovdqa64 %zmm1, %zmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_mask_compress_pd_512:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vcompresspd %zmm0, %zmm1 {%k1}
+; X86-NEXT:    vmovdqa64 %zmm1, %zmm0
+; X86-NEXT:    retl
+  %1 = bitcast i8 %mask to <8 x i1>
+  %2 = call <8 x double> @llvm.x86.avx512.mask.compress.v8f64(<8 x double> %data, <8 x double> %passthru, <8 x i1> %1)
+  ret <8 x double> %2
 }
 
 define <8 x double> @test_maskz_compress_pd_512(<8 x double> %data, i8 %mask) {
-; CHECK-LABEL: test_maskz_compress_pd_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vcompresspd %zmm0, %zmm0 {%k1} {z}
-; CHECK-NEXT:    retq
-  %res = call <8 x double> @llvm.x86.avx512.mask.compress.pd.512(<8 x double> %data, <8 x double> zeroinitializer, i8 %mask)
-  ret <8 x double> %res
+; X64-LABEL: test_maskz_compress_pd_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vcompresspd %zmm0, %zmm0 {%k1} {z}
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_maskz_compress_pd_512:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vcompresspd %zmm0, %zmm0 {%k1} {z}
+; X86-NEXT:    retl
+  %1 = bitcast i8 %mask to <8 x i1>
+  %2 = call <8 x double> @llvm.x86.avx512.mask.compress.v8f64(<8 x double> %data, <8 x double> zeroinitializer, <8 x i1> %1)
+  ret <8 x double> %2
 }
 
 define <8 x double> @test_compress_pd_512(<8 x double> %data) {
 ; CHECK-LABEL: test_compress_pd_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    retq
-  %res = call <8 x double> @llvm.x86.avx512.mask.compress.pd.512(<8 x double> %data, <8 x double> undef, i8 -1)
-  ret <8 x double> %res
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ret{{[l|q]}}
+  %1 = call <8 x double> @llvm.x86.avx512.mask.compress.v8f64(<8 x double> %data, <8 x double> undef, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>)
+  ret <8 x double> %1
 }
 
-declare <8 x double> @llvm.x86.avx512.mask.compress.pd.512(<8 x double> %data, <8 x double> %src0, i8 %mask)
-
 define <16 x float> @test_mask_compress_ps_512(<16 x float> %data, <16 x float> %passthru, i16 %mask) {
-; CHECK-LABEL: test_mask_compress_ps_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vcompressps %zmm0, %zmm1 {%k1}
-; CHECK-NEXT:    vmovdqa64 %zmm1, %zmm0
-; CHECK-NEXT:    retq
-  %res = call <16 x float> @llvm.x86.avx512.mask.compress.ps.512(<16 x float> %data, <16 x float> %passthru, i16 %mask)
-  ret <16 x float> %res
+; X64-LABEL: test_mask_compress_ps_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vcompressps %zmm0, %zmm1 {%k1}
+; X64-NEXT:    vmovdqa64 %zmm1, %zmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_mask_compress_ps_512:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vcompressps %zmm0, %zmm1 {%k1}
+; X86-NEXT:    vmovdqa64 %zmm1, %zmm0
+; X86-NEXT:    retl
+  %1 = bitcast i16 %mask to <16 x i1>
+  %2 = call <16 x float> @llvm.x86.avx512.mask.compress.v16f32(<16 x float> %data, <16 x float> %passthru, <16 x i1> %1)
+  ret <16 x float> %2
 }
 
 define <16 x float> @test_maskz_compress_ps_512(<16 x float> %data, i16 %mask) {
-; CHECK-LABEL: test_maskz_compress_ps_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vcompressps %zmm0, %zmm0 {%k1} {z}
-; CHECK-NEXT:    retq
-  %res = call <16 x float> @llvm.x86.avx512.mask.compress.ps.512(<16 x float> %data, <16 x float> zeroinitializer, i16 %mask)
-  ret <16 x float> %res
+; X64-LABEL: test_maskz_compress_ps_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vcompressps %zmm0, %zmm0 {%k1} {z}
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_maskz_compress_ps_512:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vcompressps %zmm0, %zmm0 {%k1} {z}
+; X86-NEXT:    retl
+  %1 = bitcast i16 %mask to <16 x i1>
+  %2 = call <16 x float> @llvm.x86.avx512.mask.compress.v16f32(<16 x float> %data, <16 x float> zeroinitializer, <16 x i1> %1)
+  ret <16 x float> %2
 }
 
 define <16 x float> @test_compress_ps_512(<16 x float> %data) {
 ; CHECK-LABEL: test_compress_ps_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    retq
-  %res = call <16 x float> @llvm.x86.avx512.mask.compress.ps.512(<16 x float> %data, <16 x float> undef, i16 -1)
-  ret <16 x float> %res
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ret{{[l|q]}}
+  %1 = call <16 x float> @llvm.x86.avx512.mask.compress.v16f32(<16 x float> %data, <16 x float> undef, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>)
+  ret <16 x float> %1
 }
 
-declare <16 x float> @llvm.x86.avx512.mask.compress.ps.512(<16 x float> %data, <16 x float> %src0, i16 %mask)
-
 define <8 x i64> @test_mask_compress_q_512(<8 x i64> %data, <8 x i64> %passthru, i8 %mask) {
-; CHECK-LABEL: test_mask_compress_q_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vpcompressq %zmm0, %zmm1 {%k1}
-; CHECK-NEXT:    vmovdqa64 %zmm1, %zmm0
-; CHECK-NEXT:    retq
-  %res = call <8 x i64> @llvm.x86.avx512.mask.compress.q.512(<8 x i64> %data, <8 x i64> %passthru, i8 %mask)
-  ret <8 x i64> %res
+; X64-LABEL: test_mask_compress_q_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vpcompressq %zmm0, %zmm1 {%k1}
+; X64-NEXT:    vmovdqa64 %zmm1, %zmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_mask_compress_q_512:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vpcompressq %zmm0, %zmm1 {%k1}
+; X86-NEXT:    vmovdqa64 %zmm1, %zmm0
+; X86-NEXT:    retl
+  %1 = bitcast i8 %mask to <8 x i1>
+  %2 = call <8 x i64> @llvm.x86.avx512.mask.compress.v8i64(<8 x i64> %data, <8 x i64> %passthru, <8 x i1> %1)
+  ret <8 x i64> %2
 }
 
 define <8 x i64> @test_maskz_compress_q_512(<8 x i64> %data, i8 %mask) {
-; CHECK-LABEL: test_maskz_compress_q_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vpcompressq %zmm0, %zmm0 {%k1} {z}
-; CHECK-NEXT:    retq
-  %res = call <8 x i64> @llvm.x86.avx512.mask.compress.q.512(<8 x i64> %data, <8 x i64> zeroinitializer, i8 %mask)
-  ret <8 x i64> %res
+; X64-LABEL: test_maskz_compress_q_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vpcompressq %zmm0, %zmm0 {%k1} {z}
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_maskz_compress_q_512:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vpcompressq %zmm0, %zmm0 {%k1} {z}
+; X86-NEXT:    retl
+  %1 = bitcast i8 %mask to <8 x i1>
+  %2 = call <8 x i64> @llvm.x86.avx512.mask.compress.v8i64(<8 x i64> %data, <8 x i64> zeroinitializer, <8 x i1> %1)
+  ret <8 x i64> %2
 }
 
 define <8 x i64> @test_compress_q_512(<8 x i64> %data) {
 ; CHECK-LABEL: test_compress_q_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    retq
-  %res = call <8 x i64> @llvm.x86.avx512.mask.compress.q.512(<8 x i64> %data, <8 x i64> undef, i8 -1)
-  ret <8 x i64> %res
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ret{{[l|q]}}
+  %1 = call <8 x i64> @llvm.x86.avx512.mask.compress.v8i64(<8 x i64> %data, <8 x i64> undef, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>)
+  ret <8 x i64> %1
 }
 
-declare <8 x i64> @llvm.x86.avx512.mask.compress.q.512(<8 x i64> %data, <8 x i64> %src0, i8 %mask)
-
 define <16 x i32> @test_mask_compress_d_512(<16 x i32> %data, <16 x i32> %passthru, i16 %mask) {
-; CHECK-LABEL: test_mask_compress_d_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vpcompressd %zmm0, %zmm1 {%k1}
-; CHECK-NEXT:    vmovdqa64 %zmm1, %zmm0
-; CHECK-NEXT:    retq
-  %res = call <16 x i32> @llvm.x86.avx512.mask.compress.d.512(<16 x i32> %data, <16 x i32> %passthru, i16 %mask)
-  ret <16 x i32> %res
+; X64-LABEL: test_mask_compress_d_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vpcompressd %zmm0, %zmm1 {%k1}
+; X64-NEXT:    vmovdqa64 %zmm1, %zmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_mask_compress_d_512:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vpcompressd %zmm0, %zmm1 {%k1}
+; X86-NEXT:    vmovdqa64 %zmm1, %zmm0
+; X86-NEXT:    retl
+  %1 = bitcast i16 %mask to <16 x i1>
+  %2 = call <16 x i32> @llvm.x86.avx512.mask.compress.v16i32(<16 x i32> %data, <16 x i32> %passthru, <16 x i1> %1)
+  ret <16 x i32> %2
 }
 
 define <16 x i32> @test_maskz_compress_d_512(<16 x i32> %data, i16 %mask) {
-; CHECK-LABEL: test_maskz_compress_d_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vpcompressd %zmm0, %zmm0 {%k1} {z}
-; CHECK-NEXT:    retq
-  %res = call <16 x i32> @llvm.x86.avx512.mask.compress.d.512(<16 x i32> %data, <16 x i32> zeroinitializer, i16 %mask)
-  ret <16 x i32> %res
+; X64-LABEL: test_maskz_compress_d_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vpcompressd %zmm0, %zmm0 {%k1} {z}
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_maskz_compress_d_512:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vpcompressd %zmm0, %zmm0 {%k1} {z}
+; X86-NEXT:    retl
+  %1 = bitcast i16 %mask to <16 x i1>
+  %2 = call <16 x i32> @llvm.x86.avx512.mask.compress.v16i32(<16 x i32> %data, <16 x i32> zeroinitializer, <16 x i1> %1)
+  ret <16 x i32> %2
 }
 
 define <16 x i32> @test_compress_d_512(<16 x i32> %data) {
 ; CHECK-LABEL: test_compress_d_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    retq
-  %res = call <16 x i32> @llvm.x86.avx512.mask.compress.d.512(<16 x i32> %data, <16 x i32> undef, i16 -1)
-  ret <16 x i32> %res
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ret{{[l|q]}}
+  %1 = call <16 x i32> @llvm.x86.avx512.mask.compress.v16i32(<16 x i32> %data, <16 x i32> undef, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>)
+  ret <16 x i32> %1
 }
 
-declare <16 x i32> @llvm.x86.avx512.mask.compress.d.512(<16 x i32> %data, <16 x i32> %src0, i16 %mask)
-
 define <8 x double> @test_expand_pd_512(<8 x double> %data) {
 ; CHECK-LABEL: test_expand_pd_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    retq
-  %res = call <8 x double> @llvm.x86.avx512.mask.expand.pd.512(<8 x double> %data, <8 x double> undef, i8 -1)
-  ret <8 x double> %res
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ret{{[l|q]}}
+  %1 = call <8 x double> @llvm.x86.avx512.mask.expand.v8f64(<8 x double> %data, <8 x double> undef, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>)
+  ret <8 x double> %1
 }
 
 define <8 x double> @test_mask_expand_pd_512(<8 x double> %data, <8 x double> %passthru, i8 %mask) {
-; CHECK-LABEL: test_mask_expand_pd_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vexpandpd %zmm0, %zmm1 {%k1}
-; CHECK-NEXT:    vmovdqa64 %zmm1, %zmm0
-; CHECK-NEXT:    retq
-  %res = call <8 x double> @llvm.x86.avx512.mask.expand.pd.512(<8 x double> %data, <8 x double> %passthru, i8 %mask)
-  ret <8 x double> %res
+; X64-LABEL: test_mask_expand_pd_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vexpandpd %zmm0, %zmm1 {%k1}
+; X64-NEXT:    vmovdqa64 %zmm1, %zmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_mask_expand_pd_512:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vexpandpd %zmm0, %zmm1 {%k1}
+; X86-NEXT:    vmovdqa64 %zmm1, %zmm0
+; X86-NEXT:    retl
+  %1 = bitcast i8 %mask to <8 x i1>
+  %2 = call <8 x double> @llvm.x86.avx512.mask.expand.v8f64(<8 x double> %data, <8 x double> %passthru, <8 x i1> %1)
+  ret <8 x double> %2
 }
 
 define <8 x double> @test_maskz_expand_pd_512(<8 x double> %data, i8 %mask) {
-; CHECK-LABEL: test_maskz_expand_pd_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vexpandpd %zmm0, %zmm0 {%k1} {z}
-; CHECK-NEXT:    retq
-  %res = call <8 x double> @llvm.x86.avx512.mask.expand.pd.512(<8 x double> %data, <8 x double> zeroinitializer, i8 %mask)
-  ret <8 x double> %res
+; X64-LABEL: test_maskz_expand_pd_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vexpandpd %zmm0, %zmm0 {%k1} {z}
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_maskz_expand_pd_512:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vexpandpd %zmm0, %zmm0 {%k1} {z}
+; X86-NEXT:    retl
+  %1 = bitcast i8 %mask to <8 x i1>
+  %2 = call <8 x double> @llvm.x86.avx512.mask.expand.v8f64(<8 x double> %data, <8 x double> zeroinitializer, <8 x i1> %1)
+  ret <8 x double> %2
 }
 
-declare <8 x double> @llvm.x86.avx512.mask.expand.pd.512(<8 x double> %data, <8 x double> %src0, i8 %mask)
-
 define <16 x float> @test_expand_ps_512(<16 x float> %data) {
 ; CHECK-LABEL: test_expand_ps_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    retq
-  %res = call <16 x float> @llvm.x86.avx512.mask.expand.ps.512(<16 x float> %data, <16 x float> undef, i16 -1)
-  ret <16 x float> %res
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ret{{[l|q]}}
+  %1 = call <16 x float> @llvm.x86.avx512.mask.expand.v16f32(<16 x float> %data, <16 x float> undef, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>)
+  ret <16 x float> %1
 }
 
 define <16 x float> @test_mask_expand_ps_512(<16 x float> %data, <16 x float> %passthru, i16 %mask) {
-; CHECK-LABEL: test_mask_expand_ps_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vexpandps %zmm0, %zmm1 {%k1}
-; CHECK-NEXT:    vmovdqa64 %zmm1, %zmm0
-; CHECK-NEXT:    retq
-  %res = call <16 x float> @llvm.x86.avx512.mask.expand.ps.512(<16 x float> %data, <16 x float> %passthru, i16 %mask)
-  ret <16 x float> %res
+; X64-LABEL: test_mask_expand_ps_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vexpandps %zmm0, %zmm1 {%k1}
+; X64-NEXT:    vmovdqa64 %zmm1, %zmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_mask_expand_ps_512:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vexpandps %zmm0, %zmm1 {%k1}
+; X86-NEXT:    vmovdqa64 %zmm1, %zmm0
+; X86-NEXT:    retl
+  %1 = bitcast i16 %mask to <16 x i1>
+  %2 = call <16 x float> @llvm.x86.avx512.mask.expand.v16f32(<16 x float> %data, <16 x float> %passthru, <16 x i1> %1)
+  ret <16 x float> %2
 }
 
 define <16 x float> @test_maskz_expand_ps_512(<16 x float> %data, i16 %mask) {
-; CHECK-LABEL: test_maskz_expand_ps_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vexpandps %zmm0, %zmm0 {%k1} {z}
-; CHECK-NEXT:    retq
-  %res = call <16 x float> @llvm.x86.avx512.mask.expand.ps.512(<16 x float> %data, <16 x float> zeroinitializer, i16 %mask)
-  ret <16 x float> %res
+; X64-LABEL: test_maskz_expand_ps_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vexpandps %zmm0, %zmm0 {%k1} {z}
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_maskz_expand_ps_512:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vexpandps %zmm0, %zmm0 {%k1} {z}
+; X86-NEXT:    retl
+  %1 = bitcast i16 %mask to <16 x i1>
+  %2 = call <16 x float> @llvm.x86.avx512.mask.expand.v16f32(<16 x float> %data, <16 x float> zeroinitializer, <16 x i1> %1)
+  ret <16 x float> %2
 }
 
-declare <16 x float> @llvm.x86.avx512.mask.expand.ps.512(<16 x float> %data, <16 x float> %src0, i16 %mask)
-
 define <8 x i64> @test_expand_q_512(<8 x i64> %data) {
 ; CHECK-LABEL: test_expand_q_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    retq
-  %res = call <8 x i64> @llvm.x86.avx512.mask.expand.q.512(<8 x i64> %data, <8 x i64> undef, i8 -1)
-  ret <8 x i64> %res
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ret{{[l|q]}}
+  %1 = call <8 x i64> @llvm.x86.avx512.mask.expand.v8i64(<8 x i64> %data, <8 x i64> undef, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>)
+  ret <8 x i64> %1
 }
 
 define <8 x i64> @test_mask_expand_q_512(<8 x i64> %data, <8 x i64> %passthru, i8 %mask) {
-; CHECK-LABEL: test_mask_expand_q_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vpexpandq %zmm0, %zmm1 {%k1}
-; CHECK-NEXT:    vmovdqa64 %zmm1, %zmm0
-; CHECK-NEXT:    retq
-  %res = call <8 x i64> @llvm.x86.avx512.mask.expand.q.512(<8 x i64> %data, <8 x i64> %passthru, i8 %mask)
-  ret <8 x i64> %res
+; X64-LABEL: test_mask_expand_q_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vpexpandq %zmm0, %zmm1 {%k1}
+; X64-NEXT:    vmovdqa64 %zmm1, %zmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_mask_expand_q_512:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vpexpandq %zmm0, %zmm1 {%k1}
+; X86-NEXT:    vmovdqa64 %zmm1, %zmm0
+; X86-NEXT:    retl
+  %1 = bitcast i8 %mask to <8 x i1>
+  %2 = call <8 x i64> @llvm.x86.avx512.mask.expand.v8i64(<8 x i64> %data, <8 x i64> %passthru, <8 x i1> %1)
+  ret <8 x i64> %2
 }
 
 define <8 x i64> @test_maskz_expand_q_512(<8 x i64> %data, i8 %mask) {
-; CHECK-LABEL: test_maskz_expand_q_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vpexpandq %zmm0, %zmm0 {%k1} {z}
-; CHECK-NEXT:    retq
-  %res = call <8 x i64> @llvm.x86.avx512.mask.expand.q.512(<8 x i64> %data, <8 x i64> zeroinitializer, i8 %mask)
-  ret <8 x i64> %res
+; X64-LABEL: test_maskz_expand_q_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vpexpandq %zmm0, %zmm0 {%k1} {z}
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_maskz_expand_q_512:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vpexpandq %zmm0, %zmm0 {%k1} {z}
+; X86-NEXT:    retl
+  %1 = bitcast i8 %mask to <8 x i1>
+  %2 = call <8 x i64> @llvm.x86.avx512.mask.expand.v8i64(<8 x i64> %data, <8 x i64> zeroinitializer, <8 x i1> %1)
+  ret <8 x i64> %2
 }
 
-declare <8 x i64> @llvm.x86.avx512.mask.expand.q.512(<8 x i64> %data, <8 x i64> %src0, i8 %mask)
-
 define <16 x i32> @test_expand_d_512(<16 x i32> %data) {
 ; CHECK-LABEL: test_expand_d_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    retq
-  %res = call <16 x i32> @llvm.x86.avx512.mask.expand.d.512(<16 x i32> %data, <16 x i32> undef, i16 -1)
-  ret <16 x i32> %res
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ret{{[l|q]}}
+  %1 = call <16 x i32> @llvm.x86.avx512.mask.expand.v16i32(<16 x i32> %data, <16 x i32> undef, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>)
+  ret <16 x i32> %1
 }
 
 define <16 x i32> @test_mask_expand_d_512(<16 x i32> %data, <16 x i32> %passthru, i16 %mask) {
-; CHECK-LABEL: test_mask_expand_d_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vpexpandd %zmm0, %zmm1 {%k1}
-; CHECK-NEXT:    vmovdqa64 %zmm1, %zmm0
-; CHECK-NEXT:    retq
-  %res = call <16 x i32> @llvm.x86.avx512.mask.expand.d.512(<16 x i32> %data, <16 x i32> %passthru, i16 %mask)
-  ret <16 x i32> %res
+; X64-LABEL: test_mask_expand_d_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vpexpandd %zmm0, %zmm1 {%k1}
+; X64-NEXT:    vmovdqa64 %zmm1, %zmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_mask_expand_d_512:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vpexpandd %zmm0, %zmm1 {%k1}
+; X86-NEXT:    vmovdqa64 %zmm1, %zmm0
+; X86-NEXT:    retl
+  %1 = bitcast i16 %mask to <16 x i1>
+  %2 = call <16 x i32> @llvm.x86.avx512.mask.expand.v16i32(<16 x i32> %data, <16 x i32> %passthru, <16 x i1> %1)
+  ret <16 x i32> %2
 }
 
 define <16 x i32> @test_maskz_expand_d_512(<16 x i32> %data, i16 %mask) {
-; CHECK-LABEL: test_maskz_expand_d_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vpexpandd %zmm0, %zmm0 {%k1} {z}
-; CHECK-NEXT:    retq
-  %res = call <16 x i32> @llvm.x86.avx512.mask.expand.d.512(<16 x i32> %data, <16 x i32> zeroinitializer, i16 %mask)
-  ret <16 x i32> %res
+; X64-LABEL: test_maskz_expand_d_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vpexpandd %zmm0, %zmm0 {%k1} {z}
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_maskz_expand_d_512:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vpexpandd %zmm0, %zmm0 {%k1} {z}
+; X86-NEXT:    retl
+  %1 = bitcast i16 %mask to <16 x i1>
+  %2 = call <16 x i32> @llvm.x86.avx512.mask.expand.v16i32(<16 x i32> %data, <16 x i32> zeroinitializer, <16 x i1> %1)
+  ret <16 x i32> %2
 }
 
-declare <16 x i32> @llvm.x86.avx512.mask.expand.d.512(<16 x i32> %data, <16 x i32> %src0, i16 %mask)
-
 define <16 x float> @test_rcp_ps_512(<16 x float> %a0) {
 ; CHECK-LABEL: test_rcp_ps_512:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vrcp14ps %zmm0, %zmm0
-; CHECK-NEXT:    retq
+; CHECK-NEXT:    ret{{[l|q]}}
   %res = call <16 x float> @llvm.x86.avx512.rcp14.ps.512(<16 x float> %a0, <16 x float> zeroinitializer, i16 -1) ; <<16 x float>> [#uses=1]
   ret <16 x float> %res
 }
@@ -262,9 +375,9 @@ declare <16 x float> @llvm.x86.avx512.rcp14.ps.512(<16 x float>, <16 x float>, i
 
 define <8 x double> @test_rcp_pd_512(<8 x double> %a0) {
 ; CHECK-LABEL: test_rcp_pd_512:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vrcp14pd %zmm0, %zmm0
-; CHECK-NEXT:    retq
+; CHECK-NEXT:    ret{{[l|q]}}
   %res = call <8 x double> @llvm.x86.avx512.rcp14.pd.512(<8 x double> %a0, <8 x double> zeroinitializer, i8 -1) ; <<8 x double>> [#uses=1]
   ret <8 x double> %res
 }
@@ -274,42 +387,66 @@ declare <2 x double> @llvm.x86.avx512.mask.rndscale.sd(<2 x double>, <2 x double
 
 define <2 x double> @test_rndscale_sd(<2 x double> %a, <2 x double> %b) {
 ; CHECK-LABEL: test_rndscale_sd:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vroundsd $11, %xmm1, %xmm0, %xmm0
-; CHECK-NEXT:    retq
+; CHECK-NEXT:    ret{{[l|q]}}
   %res = call <2 x double> @llvm.x86.avx512.mask.rndscale.sd(<2 x double> %a, <2 x double> %b, <2 x double> undef, i8 -1, i32 11, i32 4)
   ret <2 x double>%res
 }
 
 define <2 x double> @test_rndscale_sd_mask(<2 x double> %a, <2 x double> %b, <2 x double> %c, i8 %mask) {
-; CHECK-LABEL: test_rndscale_sd_mask:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vrndscalesd $11, %xmm1, %xmm0, %xmm2 {%k1}
-; CHECK-NEXT:    vmovapd %xmm2, %xmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_rndscale_sd_mask:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vrndscalesd $11, %xmm1, %xmm0, %xmm2 {%k1}
+; X64-NEXT:    vmovapd %xmm2, %xmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_rndscale_sd_mask:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vrndscalesd $11, %xmm1, %xmm0, %xmm2 {%k1}
+; X86-NEXT:    vmovapd %xmm2, %xmm0
+; X86-NEXT:    retl
   %res = call <2 x double> @llvm.x86.avx512.mask.rndscale.sd(<2 x double> %a, <2 x double> %b, <2 x double> %c, i8 %mask, i32 11, i32 4)
   ret <2 x double>%res
 }
 
 define <2 x double> @test_rndscale_sd_mask_load(<2 x double> %a, <2 x double>* %bptr, <2 x double> %c, i8 %mask) {
-; CHECK-LABEL: test_rndscale_sd_mask_load:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %esi, %k1
-; CHECK-NEXT:    vrndscalesd $11, (%rdi), %xmm0, %xmm1 {%k1}
-; CHECK-NEXT:    vmovapd %xmm1, %xmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_rndscale_sd_mask_load:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %esi, %k1
+; X64-NEXT:    vrndscalesd $11, (%rdi), %xmm0, %xmm1 {%k1}
+; X64-NEXT:    vmovapd %xmm1, %xmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_rndscale_sd_mask_load:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    vrndscalesd $11, (%eax), %xmm0, %xmm1 {%k1}
+; X86-NEXT:    vmovapd %xmm1, %xmm0
+; X86-NEXT:    retl
   %b = load <2 x double>, <2 x double>* %bptr
   %res = call <2 x double> @llvm.x86.avx512.mask.rndscale.sd(<2 x double> %a, <2 x double> %b, <2 x double> %c, i8 %mask, i32 11, i32 4)
   ret <2 x double>%res
 }
 
 define <2 x double> @test_rndscale_sd_maskz(<2 x double> %a, <2 x double> %b, i8 %mask) {
-; CHECK-LABEL: test_rndscale_sd_maskz:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vrndscalesd $11, %xmm1, %xmm0, %xmm0 {%k1} {z}
-; CHECK-NEXT:    retq
+; X64-LABEL: test_rndscale_sd_maskz:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vrndscalesd $11, %xmm1, %xmm0, %xmm0 {%k1} {z}
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_rndscale_sd_maskz:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vrndscalesd $11, %xmm1, %xmm0, %xmm0 {%k1} {z}
+; X86-NEXT:    retl
   %res = call <2 x double> @llvm.x86.avx512.mask.rndscale.sd(<2 x double> %a, <2 x double> %b, <2 x double> zeroinitializer, i8 %mask, i32 11, i32 4)
   ret <2 x double>%res
 }
@@ -318,40 +455,61 @@ declare <4 x float> @llvm.x86.avx512.mask.rndscale.ss(<4 x float>, <4 x float>,
 
 define <4 x float> @test_rndscale_ss(<4 x float> %a, <4 x float> %b) {
 ; CHECK-LABEL: test_rndscale_ss:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vroundss $11, %xmm1, %xmm0, %xmm0
-; CHECK-NEXT:    retq
+; CHECK-NEXT:    ret{{[l|q]}}
   %res = call <4 x float> @llvm.x86.avx512.mask.rndscale.ss(<4 x float> %a, <4 x float> %b, <4 x float> undef, i8 -1, i32 11, i32 4)
   ret <4 x float>%res
 }
 
 define <4 x float> @test_rndscale_ss_load(<4 x float> %a, <4 x float>* %bptr) {
-; CHECK-LABEL: test_rndscale_ss_load:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    vroundss $11, (%rdi), %xmm0, %xmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_rndscale_ss_load:
+; X64:       # %bb.0:
+; X64-NEXT:    vroundss $11, (%rdi), %xmm0, %xmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_rndscale_ss_load:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    vroundss $11, (%eax), %xmm0, %xmm0
+; X86-NEXT:    retl
   %b = load <4 x float>, <4 x float>* %bptr
   %res = call <4 x float> @llvm.x86.avx512.mask.rndscale.ss(<4 x float> %a, <4 x float> %b, <4 x float> undef, i8 -1, i32 11, i32 4)
   ret <4 x float>%res
 }
 
 define <4 x float> @test_rndscale_ss_mask(<4 x float> %a, <4 x float> %b, <4 x float> %c, i8 %mask) {
-; CHECK-LABEL: test_rndscale_ss_mask:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vrndscaless $11, %xmm1, %xmm0, %xmm2 {%k1}
-; CHECK-NEXT:    vmovaps %xmm2, %xmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_rndscale_ss_mask:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vrndscaless $11, %xmm1, %xmm0, %xmm2 {%k1}
+; X64-NEXT:    vmovaps %xmm2, %xmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_rndscale_ss_mask:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vrndscaless $11, %xmm1, %xmm0, %xmm2 {%k1}
+; X86-NEXT:    vmovaps %xmm2, %xmm0
+; X86-NEXT:    retl
   %res = call <4 x float> @llvm.x86.avx512.mask.rndscale.ss(<4 x float> %a, <4 x float> %b, <4 x float> %c, i8 %mask, i32 11, i32 4)
   ret <4 x float>%res
 }
 
 define <4 x float> @test_rndscale_ss_maskz(<4 x float> %a, <4 x float> %b, i8 %mask) {
-; CHECK-LABEL: test_rndscale_ss_maskz:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vrndscaless $11, %xmm1, %xmm0, %xmm0 {%k1} {z}
-; CHECK-NEXT:    retq
+; X64-LABEL: test_rndscale_ss_maskz:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vrndscaless $11, %xmm1, %xmm0, %xmm0 {%k1} {z}
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_rndscale_ss_maskz:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vrndscaless $11, %xmm1, %xmm0, %xmm0 {%k1} {z}
+; X86-NEXT:    retl
   %res = call <4 x float> @llvm.x86.avx512.mask.rndscale.ss(<4 x float> %a, <4 x float> %b, <4 x float> zeroinitializer, i8 %mask, i32 11, i32 4)
   ret <4 x float>%res
 }
@@ -360,9 +518,9 @@ declare <8 x double> @llvm.x86.avx512.mask.rndscale.pd.512(<8 x double>, i32, <8
 
 define <8 x double> @test7(<8 x double> %a) {
 ; CHECK-LABEL: test7:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vrndscalepd $11, %zmm0, %zmm0
-; CHECK-NEXT:    retq
+; CHECK-NEXT:    ret{{[l|q]}}
   %res = call <8 x double> @llvm.x86.avx512.mask.rndscale.pd.512(<8 x double> %a, i32 11, <8 x double> %a, i8 -1, i32 4)
   ret <8 x double>%res
 }
@@ -371,18 +529,18 @@ declare <16 x float> @llvm.x86.avx512.mask.rndscale.ps.512(<16 x float>, i32, <1
 
 define <16 x float> @test8(<16 x float> %a) {
 ; CHECK-LABEL: test8:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vrndscaleps $11, %zmm0, %zmm0
-; CHECK-NEXT:    retq
+; CHECK-NEXT:    ret{{[l|q]}}
   %res = call <16 x float> @llvm.x86.avx512.mask.rndscale.ps.512(<16 x float> %a, i32 11, <16 x float> %a, i16 -1, i32 4)
   ret <16 x float>%res
 }
 
 define <16 x float> @test_rsqrt_ps_512(<16 x float> %a0) {
 ; CHECK-LABEL: test_rsqrt_ps_512:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vrsqrt14ps %zmm0, %zmm0
-; CHECK-NEXT:    retq
+; CHECK-NEXT:    ret{{[l|q]}}
   %res = call <16 x float> @llvm.x86.avx512.rsqrt14.ps.512(<16 x float> %a0, <16 x float> zeroinitializer, i16 -1) ; <<16 x float>> [#uses=1]
   ret <16 x float> %res
 }
@@ -390,20 +548,28 @@ declare <16 x float> @llvm.x86.avx512.rsqrt14.ps.512(<16 x float>, <16 x float>,
 
 define <8 x double> @test_sqrt_pd_512(<8 x double> %a0) {
 ; CHECK-LABEL: test_sqrt_pd_512:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsqrtpd %zmm0, %zmm0
-; CHECK-NEXT:    retq
+; CHECK-NEXT:    ret{{[l|q]}}
   %1 = call <8 x double> @llvm.sqrt.v8f64(<8 x double> %a0)
   ret <8 x double> %1
 }
 
 define <8 x double> @test_mask_sqrt_pd_512(<8 x double> %a0, <8 x double> %passthru, i8 %mask) {
-; CHECK-LABEL: test_mask_sqrt_pd_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vsqrtpd %zmm0, %zmm1 {%k1}
-; CHECK-NEXT:    vmovapd %zmm1, %zmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_mask_sqrt_pd_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vsqrtpd %zmm0, %zmm1 {%k1}
+; X64-NEXT:    vmovapd %zmm1, %zmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_mask_sqrt_pd_512:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vsqrtpd %zmm0, %zmm1 {%k1}
+; X86-NEXT:    vmovapd %zmm1, %zmm0
+; X86-NEXT:    retl
   %1 = call <8 x double> @llvm.sqrt.v8f64(<8 x double> %a0)
   %2 = bitcast i8 %mask to <8 x i1>
   %3 = select <8 x i1> %2, <8 x double> %1, <8 x double> %passthru
@@ -411,11 +577,18 @@ define <8 x double> @test_mask_sqrt_pd_512(<8 x double> %a0, <8 x double> %passt
 }
 
 define <8 x double> @test_maskz_sqrt_pd_512(<8 x double> %a0, i8 %mask) {
-; CHECK-LABEL: test_maskz_sqrt_pd_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vsqrtpd %zmm0, %zmm0 {%k1} {z}
-; CHECK-NEXT:    retq
+; X64-LABEL: test_maskz_sqrt_pd_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vsqrtpd %zmm0, %zmm0 {%k1} {z}
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_maskz_sqrt_pd_512:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vsqrtpd %zmm0, %zmm0 {%k1} {z}
+; X86-NEXT:    retl
   %1 = call <8 x double> @llvm.sqrt.v8f64(<8 x double> %a0)
   %2 = bitcast i8 %mask to <8 x i1>
   %3 = select <8 x i1> %2, <8 x double> %1, <8 x double> zeroinitializer
@@ -425,20 +598,28 @@ declare <8 x double> @llvm.sqrt.v8f64(<8 x double>)
 
 define <8 x double> @test_sqrt_round_pd_512(<8 x double> %a0) {
 ; CHECK-LABEL: test_sqrt_round_pd_512:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsqrtpd {rz-sae}, %zmm0, %zmm0
-; CHECK-NEXT:    retq
+; CHECK-NEXT:    ret{{[l|q]}}
   %1 = call <8 x double> @llvm.x86.avx512.sqrt.pd.512(<8 x double> %a0, i32 11)
   ret <8 x double> %1
 }
 
 define <8 x double> @test_mask_sqrt_round_pd_512(<8 x double> %a0, <8 x double> %passthru, i8 %mask) {
-; CHECK-LABEL: test_mask_sqrt_round_pd_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vsqrtpd {rz-sae}, %zmm0, %zmm1 {%k1}
-; CHECK-NEXT:    vmovapd %zmm1, %zmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_mask_sqrt_round_pd_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vsqrtpd {rz-sae}, %zmm0, %zmm1 {%k1}
+; X64-NEXT:    vmovapd %zmm1, %zmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_mask_sqrt_round_pd_512:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vsqrtpd {rz-sae}, %zmm0, %zmm1 {%k1}
+; X86-NEXT:    vmovapd %zmm1, %zmm0
+; X86-NEXT:    retl
   %1 = call <8 x double> @llvm.x86.avx512.sqrt.pd.512(<8 x double> %a0, i32 11)
   %2 = bitcast i8 %mask to <8 x i1>
   %3 = select <8 x i1> %2, <8 x double> %1, <8 x double> %passthru
@@ -446,11 +627,18 @@ define <8 x double> @test_mask_sqrt_round_pd_512(<8 x double> %a0, <8 x double>
 }
 
 define <8 x double> @test_maskz_sqrt_round_pd_512(<8 x double> %a0, i8 %mask) {
-; CHECK-LABEL: test_maskz_sqrt_round_pd_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vsqrtpd {rz-sae}, %zmm0, %zmm0 {%k1} {z}
-; CHECK-NEXT:    retq
+; X64-LABEL: test_maskz_sqrt_round_pd_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vsqrtpd {rz-sae}, %zmm0, %zmm0 {%k1} {z}
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_maskz_sqrt_round_pd_512:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vsqrtpd {rz-sae}, %zmm0, %zmm0 {%k1} {z}
+; X86-NEXT:    retl
   %1 = call <8 x double> @llvm.x86.avx512.sqrt.pd.512(<8 x double> %a0, i32 11)
   %2 = bitcast i8 %mask to <8 x i1>
   %3 = select <8 x i1> %2, <8 x double> %1, <8 x double> zeroinitializer
@@ -460,20 +648,27 @@ declare <8 x double> @llvm.x86.avx512.sqrt.pd.512(<8 x double>, i32) nounwind re
 
 define <16 x float> @test_sqrt_ps_512(<16 x float> %a0) {
 ; CHECK-LABEL: test_sqrt_ps_512:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsqrtps %zmm0, %zmm0
-; CHECK-NEXT:    retq
+; CHECK-NEXT:    ret{{[l|q]}}
   %1 = call <16 x float> @llvm.sqrt.v16f32(<16 x float> %a0)
   ret <16 x float> %1
 }
 
 define <16 x float> @test_mask_sqrt_ps_512(<16 x float> %a0, <16 x float> %passthru, i16 %mask) {
-; CHECK-LABEL: test_mask_sqrt_ps_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vsqrtps %zmm0, %zmm1 {%k1}
-; CHECK-NEXT:    vmovaps %zmm1, %zmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_mask_sqrt_ps_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vsqrtps %zmm0, %zmm1 {%k1}
+; X64-NEXT:    vmovaps %zmm1, %zmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_mask_sqrt_ps_512:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vsqrtps %zmm0, %zmm1 {%k1}
+; X86-NEXT:    vmovaps %zmm1, %zmm0
+; X86-NEXT:    retl
   %1 = call <16 x float> @llvm.sqrt.v16f32(<16 x float> %a0)
   %2 = bitcast i16 %mask to <16 x i1>
   %3 = select <16 x i1> %2, <16 x float> %1, <16 x float> %passthru
@@ -481,11 +676,17 @@ define <16 x float> @test_mask_sqrt_ps_512(<16 x float> %a0, <16 x float> %passt
 }
 
 define <16 x float> @test_maskz_sqrt_ps_512(<16 x float> %a0, i16 %mask) {
-; CHECK-LABEL: test_maskz_sqrt_ps_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vsqrtps %zmm0, %zmm0 {%k1} {z}
-; CHECK-NEXT:    retq
+; X64-LABEL: test_maskz_sqrt_ps_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vsqrtps %zmm0, %zmm0 {%k1} {z}
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_maskz_sqrt_ps_512:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vsqrtps %zmm0, %zmm0 {%k1} {z}
+; X86-NEXT:    retl
   %1 = call <16 x float> @llvm.sqrt.v16f32(<16 x float> %a0)
   %2 = bitcast i16 %mask to <16 x i1>
   %3 = select <16 x i1> %2, <16 x float> %1, <16 x float> zeroinitializer
@@ -495,20 +696,27 @@ declare <16 x float> @llvm.sqrt.v16f32(<16 x float>)
 
 define <16 x float> @test_sqrt_round_ps_512(<16 x float> %a0) {
 ; CHECK-LABEL: test_sqrt_round_ps_512:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsqrtps {rz-sae}, %zmm0, %zmm0
-; CHECK-NEXT:    retq
+; CHECK-NEXT:    ret{{[l|q]}}
   %1 = call <16 x float> @llvm.x86.avx512.sqrt.ps.512(<16 x float> %a0, i32 11)
   ret <16 x float> %1
 }
 
 define <16 x float> @test_mask_sqrt_round_ps_512(<16 x float> %a0, <16 x float> %passthru, i16 %mask) {
-; CHECK-LABEL: test_mask_sqrt_round_ps_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vsqrtps {rz-sae}, %zmm0, %zmm1 {%k1}
-; CHECK-NEXT:    vmovaps %zmm1, %zmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_mask_sqrt_round_ps_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vsqrtps {rz-sae}, %zmm0, %zmm1 {%k1}
+; X64-NEXT:    vmovaps %zmm1, %zmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_mask_sqrt_round_ps_512:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vsqrtps {rz-sae}, %zmm0, %zmm1 {%k1}
+; X86-NEXT:    vmovaps %zmm1, %zmm0
+; X86-NEXT:    retl
   %1 = call <16 x float> @llvm.x86.avx512.sqrt.ps.512(<16 x float> %a0, i32 11)
   %2 = bitcast i16 %mask to <16 x i1>
   %3 = select <16 x i1> %2, <16 x float> %1, <16 x float> %passthru
@@ -516,11 +724,17 @@ define <16 x float> @test_mask_sqrt_round_ps_512(<16 x float> %a0, <16 x float>
 }
 
 define <16 x float> @test_maskz_sqrt_round_ps_512(<16 x float> %a0, i16 %mask) {
-; CHECK-LABEL: test_maskz_sqrt_round_ps_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vsqrtps {rz-sae}, %zmm0, %zmm0 {%k1} {z}
-; CHECK-NEXT:    retq
+; X64-LABEL: test_maskz_sqrt_round_ps_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vsqrtps {rz-sae}, %zmm0, %zmm0 {%k1} {z}
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_maskz_sqrt_round_ps_512:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vsqrtps {rz-sae}, %zmm0, %zmm0 {%k1} {z}
+; X86-NEXT:    retl
   %1 = call <16 x float> @llvm.x86.avx512.sqrt.ps.512(<16 x float> %a0, i32 11)
   %2 = bitcast i16 %mask to <16 x i1>
   %3 = select <16 x i1> %2, <16 x float> %1, <16 x float> zeroinitializer
@@ -530,17 +744,17 @@ declare <16 x float> @llvm.x86.avx512.sqrt.ps.512(<16 x float>, i32) nounwind re
 
 define <8 x double> @test_getexp_pd_512(<8 x double> %a0) {
 ; CHECK-LABEL: test_getexp_pd_512:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vgetexppd %zmm0, %zmm0
-; CHECK-NEXT:    retq
+; CHECK-NEXT:    ret{{[l|q]}}
   %res = call <8 x double> @llvm.x86.avx512.mask.getexp.pd.512(<8 x double> %a0,  <8 x double> zeroinitializer, i8 -1, i32 4)
   ret <8 x double> %res
 }
 define <8 x double> @test_getexp_round_pd_512(<8 x double> %a0) {
 ; CHECK-LABEL: test_getexp_round_pd_512:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vgetexppd {sae}, %zmm0, %zmm0
-; CHECK-NEXT:    retq
+; CHECK-NEXT:    ret{{[l|q]}}
   %res = call <8 x double> @llvm.x86.avx512.mask.getexp.pd.512(<8 x double> %a0,  <8 x double> zeroinitializer, i8 -1, i32 8)
   ret <8 x double> %res
 }
@@ -548,18 +762,18 @@ declare <8 x double> @llvm.x86.avx512.mask.getexp.pd.512(<8 x double>, <8 x doub
 
 define <16 x float> @test_getexp_ps_512(<16 x float> %a0) {
 ; CHECK-LABEL: test_getexp_ps_512:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vgetexpps %zmm0, %zmm0
-; CHECK-NEXT:    retq
+; CHECK-NEXT:    ret{{[l|q]}}
   %res = call <16 x float> @llvm.x86.avx512.mask.getexp.ps.512(<16 x float> %a0, <16 x float> zeroinitializer, i16 -1, i32 4)
   ret <16 x float> %res
 }
 
 define <16 x float> @test_getexp_round_ps_512(<16 x float> %a0) {
 ; CHECK-LABEL: test_getexp_round_ps_512:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vgetexpps {sae}, %zmm0, %zmm0
-; CHECK-NEXT:    retq
+; CHECK-NEXT:    ret{{[l|q]}}
   %res = call <16 x float> @llvm.x86.avx512.mask.getexp.ps.512(<16 x float> %a0, <16 x float> zeroinitializer, i16 -1, i32 8)
   ret <16 x float> %res
 }
@@ -568,18 +782,32 @@ declare <16 x float> @llvm.x86.avx512.mask.getexp.ps.512(<16 x float>, <16 x flo
 declare <4 x float> @llvm.x86.avx512.mask.sqrt.ss(<4 x float>, <4 x float>, <4 x float>, i8, i32) nounwind readnone
 
 define <4 x float> @test_sqrt_ss(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, i8 %mask) {
-; CHECK-LABEL: test_sqrt_ss:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vmovaps %xmm2, %xmm3
-; CHECK-NEXT:    vsqrtss %xmm1, %xmm0, %xmm3 {%k1}
-; CHECK-NEXT:    vsqrtss {rd-sae}, %xmm1, %xmm0, %xmm2 {%k1}
-; CHECK-NEXT:    vaddps %xmm2, %xmm3, %xmm2
-; CHECK-NEXT:    vsqrtss {ru-sae}, %xmm1, %xmm0, %xmm3 {%k1} {z}
-; CHECK-NEXT:    vsqrtss {rz-sae}, %xmm1, %xmm0, %xmm0
-; CHECK-NEXT:    vaddps %xmm0, %xmm3, %xmm0
-; CHECK-NEXT:    vaddps %xmm0, %xmm2, %xmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_sqrt_ss:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vmovaps %xmm2, %xmm3
+; X64-NEXT:    vsqrtss %xmm1, %xmm0, %xmm3 {%k1}
+; X64-NEXT:    vsqrtss {rd-sae}, %xmm1, %xmm0, %xmm2 {%k1}
+; X64-NEXT:    vaddps %xmm2, %xmm3, %xmm2
+; X64-NEXT:    vsqrtss {ru-sae}, %xmm1, %xmm0, %xmm3 {%k1} {z}
+; X64-NEXT:    vsqrtss {rz-sae}, %xmm1, %xmm0, %xmm0
+; X64-NEXT:    vaddps %xmm0, %xmm3, %xmm0
+; X64-NEXT:    vaddps %xmm0, %xmm2, %xmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_sqrt_ss:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vmovaps %xmm2, %xmm3
+; X86-NEXT:    vsqrtss %xmm1, %xmm0, %xmm3 {%k1}
+; X86-NEXT:    vsqrtss {rd-sae}, %xmm1, %xmm0, %xmm2 {%k1}
+; X86-NEXT:    vaddps %xmm2, %xmm3, %xmm2
+; X86-NEXT:    vsqrtss {ru-sae}, %xmm1, %xmm0, %xmm3 {%k1} {z}
+; X86-NEXT:    vsqrtss {rz-sae}, %xmm1, %xmm0, %xmm0
+; X86-NEXT:    vaddps %xmm0, %xmm3, %xmm0
+; X86-NEXT:    vaddps %xmm0, %xmm2, %xmm0
+; X86-NEXT:    retl
   %res0 = call <4 x float> @llvm.x86.avx512.mask.sqrt.ss(<4 x float>%a0, <4 x float> %a1, <4 x float> %a2, i8 %mask, i32 4)
   %res1 = call <4 x float> @llvm.x86.avx512.mask.sqrt.ss(<4 x float>%a0, <4 x float> %a1, <4 x float> %a2, i8 %mask, i32 1)
   %res2 = call <4 x float> @llvm.x86.avx512.mask.sqrt.ss(<4 x float>%a0, <4 x float> %a1, <4 x float> zeroinitializer, i8 %mask, i32 2)
@@ -594,18 +822,32 @@ define <4 x float> @test_sqrt_ss(<4 x float> %a0, <4 x float> %a1, <4 x float> %
 declare <2 x double> @llvm.x86.avx512.mask.sqrt.sd(<2 x double>, <2 x double>, <2 x double>, i8, i32) nounwind readnone
 
 define <2 x double> @test_sqrt_sd(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, i8 %mask) {
-; CHECK-LABEL: test_sqrt_sd:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vmovapd %xmm2, %xmm3
-; CHECK-NEXT:    vsqrtsd %xmm1, %xmm0, %xmm3 {%k1}
-; CHECK-NEXT:    vsqrtsd {rd-sae}, %xmm1, %xmm0, %xmm2 {%k1}
-; CHECK-NEXT:    vaddpd %xmm2, %xmm3, %xmm2
-; CHECK-NEXT:    vsqrtsd {ru-sae}, %xmm1, %xmm0, %xmm3 {%k1} {z}
-; CHECK-NEXT:    vsqrtsd {rz-sae}, %xmm1, %xmm0, %xmm0
-; CHECK-NEXT:    vaddpd %xmm0, %xmm3, %xmm0
-; CHECK-NEXT:    vaddpd %xmm0, %xmm2, %xmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_sqrt_sd:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vmovapd %xmm2, %xmm3
+; X64-NEXT:    vsqrtsd %xmm1, %xmm0, %xmm3 {%k1}
+; X64-NEXT:    vsqrtsd {rd-sae}, %xmm1, %xmm0, %xmm2 {%k1}
+; X64-NEXT:    vaddpd %xmm2, %xmm3, %xmm2
+; X64-NEXT:    vsqrtsd {ru-sae}, %xmm1, %xmm0, %xmm3 {%k1} {z}
+; X64-NEXT:    vsqrtsd {rz-sae}, %xmm1, %xmm0, %xmm0
+; X64-NEXT:    vaddpd %xmm0, %xmm3, %xmm0
+; X64-NEXT:    vaddpd %xmm0, %xmm2, %xmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_sqrt_sd:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vmovapd %xmm2, %xmm3
+; X86-NEXT:    vsqrtsd %xmm1, %xmm0, %xmm3 {%k1}
+; X86-NEXT:    vsqrtsd {rd-sae}, %xmm1, %xmm0, %xmm2 {%k1}
+; X86-NEXT:    vaddpd %xmm2, %xmm3, %xmm2
+; X86-NEXT:    vsqrtsd {ru-sae}, %xmm1, %xmm0, %xmm3 {%k1} {z}
+; X86-NEXT:    vsqrtsd {rz-sae}, %xmm1, %xmm0, %xmm0
+; X86-NEXT:    vaddpd %xmm0, %xmm3, %xmm0
+; X86-NEXT:    vaddpd %xmm0, %xmm2, %xmm0
+; X86-NEXT:    retl
   %res0 = call <2 x double> @llvm.x86.avx512.mask.sqrt.sd(<2 x double>%a0, <2 x double> %a1, <2 x double> %a2, i8 %mask, i32 4)
   %res1 = call <2 x double> @llvm.x86.avx512.mask.sqrt.sd(<2 x double>%a0, <2 x double> %a1, <2 x double> %a2, i8 %mask, i32 1)
   %res2 = call <2 x double> @llvm.x86.avx512.mask.sqrt.sd(<2 x double>%a0, <2 x double> %a1, <2 x double> zeroinitializer, i8 %mask, i32 2)
@@ -617,47 +859,13 @@ define <2 x double> @test_sqrt_sd(<2 x double> %a0, <2 x double> %a1, <2 x doubl
   ret <2 x double> %res
 }
 
-define i64 @test_x86_sse2_cvtsd2si64(<2 x double> %a0) {
-; CHECK-LABEL: test_x86_sse2_cvtsd2si64:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    vcvtsd2si %xmm0, %rax
-; CHECK-NEXT:    retq
-  %res = call i64 @llvm.x86.sse2.cvtsd2si64(<2 x double> %a0) ; <i64> [#uses=1]
-  ret i64 %res
-}
-declare i64 @llvm.x86.sse2.cvtsd2si64(<2 x double>) nounwind readnone
-
-define <2 x double> @test_x86_sse2_cvtsi642sd(<2 x double> %a0, i64 %a1) {
-; CHECK-LABEL: test_x86_sse2_cvtsi642sd:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    vcvtsi2sdq %rdi, %xmm0, %xmm0
-; CHECK-NEXT:    retq
-  %res = call <2 x double> @llvm.x86.sse2.cvtsi642sd(<2 x double> %a0, i64 %a1) ; <<2 x double>> [#uses=1]
-  ret <2 x double> %res
-}
-declare <2 x double> @llvm.x86.sse2.cvtsi642sd(<2 x double>, i64) nounwind readnone
-
-define i64 @test_x86_avx512_cvttsd2si64(<2 x double> %a0) {
-; CHECK-LABEL: test_x86_avx512_cvttsd2si64:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    vcvttsd2si %xmm0, %rcx
-; CHECK-NEXT:    vcvttsd2si {sae}, %xmm0, %rax
-; CHECK-NEXT:    addq %rcx, %rax
-; CHECK-NEXT:    retq
-  %res0 = call i64 @llvm.x86.avx512.cvttsd2si64(<2 x double> %a0, i32 4) ;
-  %res1 = call i64 @llvm.x86.avx512.cvttsd2si64(<2 x double> %a0, i32 8) ;
-  %res2 = add i64 %res0, %res1
-  ret i64 %res2
-}
-declare i64 @llvm.x86.avx512.cvttsd2si64(<2 x double>, i32) nounwind readnone
-
 define i32 @test_x86_avx512_cvttsd2usi(<2 x double> %a0) {
 ; CHECK-LABEL: test_x86_avx512_cvttsd2usi:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vcvttsd2usi %xmm0, %ecx
 ; CHECK-NEXT:    vcvttsd2usi {sae}, %xmm0, %eax
 ; CHECK-NEXT:    addl %ecx, %eax
-; CHECK-NEXT:    retq
+; CHECK-NEXT:    ret{{[l|q]}}
   %res0 = call i32 @llvm.x86.avx512.cvttsd2usi(<2 x double> %a0, i32 4) ;
   %res1 = call i32 @llvm.x86.avx512.cvttsd2usi(<2 x double> %a0, i32 8) ;
   %res2 = add i32 %res0, %res1
@@ -667,11 +875,11 @@ declare i32 @llvm.x86.avx512.cvttsd2usi(<2 x double>, i32) nounwind readnone
 
 define i32 @test_x86_avx512_cvttsd2si(<2 x double> %a0) {
 ; CHECK-LABEL: test_x86_avx512_cvttsd2si:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vcvttsd2si %xmm0, %ecx
 ; CHECK-NEXT:    vcvttsd2si {sae}, %xmm0, %eax
 ; CHECK-NEXT:    addl %ecx, %eax
-; CHECK-NEXT:    retq
+; CHECK-NEXT:    ret{{[l|q]}}
   %res0 = call i32 @llvm.x86.avx512.cvttsd2si(<2 x double> %a0, i32 4) ;
   %res1 = call i32 @llvm.x86.avx512.cvttsd2si(<2 x double> %a0, i32 8) ;
   %res2 = add i32 %res0, %res1
@@ -679,51 +887,13 @@ define i32 @test_x86_avx512_cvttsd2si(<2 x double> %a0) {
 }
 declare i32 @llvm.x86.avx512.cvttsd2si(<2 x double>, i32) nounwind readnone
 
-
-
-define i64 @test_x86_avx512_cvttsd2usi64(<2 x double> %a0) {
-; CHECK-LABEL: test_x86_avx512_cvttsd2usi64:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    vcvttsd2usi %xmm0, %rcx
-; CHECK-NEXT:    vcvttsd2usi {sae}, %xmm0, %rax
-; CHECK-NEXT:    addq %rcx, %rax
-; CHECK-NEXT:    retq
-  %res0 = call i64 @llvm.x86.avx512.cvttsd2usi64(<2 x double> %a0, i32 4) ;
-  %res1 = call i64 @llvm.x86.avx512.cvttsd2usi64(<2 x double> %a0, i32 8) ;
-  %res2 = add i64 %res0, %res1
-  ret i64 %res2
-}
-declare i64 @llvm.x86.avx512.cvttsd2usi64(<2 x double>, i32) nounwind readnone
-
-define i64 @test_x86_sse_cvtss2si64(<4 x float> %a0) {
-; CHECK-LABEL: test_x86_sse_cvtss2si64:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    vcvtss2si %xmm0, %rax
-; CHECK-NEXT:    retq
-  %res = call i64 @llvm.x86.sse.cvtss2si64(<4 x float> %a0) ; <i64> [#uses=1]
-  ret i64 %res
-}
-declare i64 @llvm.x86.sse.cvtss2si64(<4 x float>) nounwind readnone
-
-
-define <4 x float> @test_x86_sse_cvtsi642ss(<4 x float> %a0, i64 %a1) {
-; CHECK-LABEL: test_x86_sse_cvtsi642ss:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    vcvtsi2ssq %rdi, %xmm0, %xmm0
-; CHECK-NEXT:    retq
-  %res = call <4 x float> @llvm.x86.sse.cvtsi642ss(<4 x float> %a0, i64 %a1) ; <<4 x float>> [#uses=1]
-  ret <4 x float> %res
-}
-declare <4 x float> @llvm.x86.sse.cvtsi642ss(<4 x float>, i64) nounwind readnone
-
-
 define i32 @test_x86_avx512_cvttss2si(<4 x float> %a0) {
 ; CHECK-LABEL: test_x86_avx512_cvttss2si:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vcvttss2si {sae}, %xmm0, %ecx
 ; CHECK-NEXT:    vcvttss2si %xmm0, %eax
 ; CHECK-NEXT:    addl %ecx, %eax
-; CHECK-NEXT:    retq
+; CHECK-NEXT:    ret{{[l|q]}}
   %res0 = call i32 @llvm.x86.avx512.cvttss2si(<4 x float> %a0, i32 8) ;
   %res1 = call i32 @llvm.x86.avx512.cvttss2si(<4 x float> %a0, i32 4) ;
   %res2 = add i32 %res0, %res1
@@ -732,36 +902,28 @@ define i32 @test_x86_avx512_cvttss2si(<4 x float> %a0) {
 declare i32 @llvm.x86.avx512.cvttss2si(<4 x float>, i32) nounwind readnone
 
 define i32 @test_x86_avx512_cvttss2si_load(<4 x float>* %a0) {
-; CHECK-LABEL: test_x86_avx512_cvttss2si_load:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    vcvttss2si (%rdi), %eax
-; CHECK-NEXT:    retq
+; X64-LABEL: test_x86_avx512_cvttss2si_load:
+; X64:       # %bb.0:
+; X64-NEXT:    vcvttss2si (%rdi), %eax
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_x86_avx512_cvttss2si_load:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    vcvttss2si (%eax), %eax
+; X86-NEXT:    retl
   %a1 = load <4 x float>, <4 x float>* %a0
   %res = call i32 @llvm.x86.avx512.cvttss2si(<4 x float> %a1, i32 4) ;
   ret i32 %res
 }
 
-define i64 @test_x86_avx512_cvttss2si64(<4 x float> %a0) {
-; CHECK-LABEL: test_x86_avx512_cvttss2si64:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    vcvttss2si %xmm0, %rcx
-; CHECK-NEXT:    vcvttss2si {sae}, %xmm0, %rax
-; CHECK-NEXT:    addq %rcx, %rax
-; CHECK-NEXT:    retq
-  %res0 = call i64 @llvm.x86.avx512.cvttss2si64(<4 x float> %a0, i32 4) ;
-  %res1 = call i64 @llvm.x86.avx512.cvttss2si64(<4 x float> %a0, i32 8) ;
-  %res2 = add i64 %res0, %res1
-  ret i64 %res2
-}
-declare i64 @llvm.x86.avx512.cvttss2si64(<4 x float>, i32) nounwind readnone
-
 define i32 @test_x86_avx512_cvttss2usi(<4 x float> %a0) {
 ; CHECK-LABEL: test_x86_avx512_cvttss2usi:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vcvttss2usi {sae}, %xmm0, %ecx
 ; CHECK-NEXT:    vcvttss2usi %xmm0, %eax
 ; CHECK-NEXT:    addl %ecx, %eax
-; CHECK-NEXT:    retq
+; CHECK-NEXT:    ret{{[l|q]}}
   %res0 = call i32 @llvm.x86.avx512.cvttss2usi(<4 x float> %a0, i32 8) ;
   %res1 = call i32 @llvm.x86.avx512.cvttss2usi(<4 x float> %a0, i32 4) ;
   %res2 = add i32 %res0, %res1
@@ -769,105 +931,15 @@ define i32 @test_x86_avx512_cvttss2usi(<4 x float> %a0) {
 }
 declare i32 @llvm.x86.avx512.cvttss2usi(<4 x float>, i32) nounwind readnone
 
-define i64 @test_x86_avx512_cvttss2usi64(<4 x float> %a0) {
-; CHECK-LABEL: test_x86_avx512_cvttss2usi64:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    vcvttss2usi %xmm0, %rcx
-; CHECK-NEXT:    vcvttss2usi {sae}, %xmm0, %rax
-; CHECK-NEXT:    addq %rcx, %rax
-; CHECK-NEXT:    retq
-  %res0 = call i64 @llvm.x86.avx512.cvttss2usi64(<4 x float> %a0, i32 4) ;
-  %res1 = call i64 @llvm.x86.avx512.cvttss2usi64(<4 x float> %a0, i32 8) ;
-  %res2 = add i64 %res0, %res1
-  ret i64 %res2
-}
-declare i64 @llvm.x86.avx512.cvttss2usi64(<4 x float>, i32) nounwind readnone
-
-define i64 @test_x86_avx512_cvtsd2usi64(<2 x double> %a0) {
-; CHECK-LABEL: test_x86_avx512_cvtsd2usi64:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    vcvtsd2usi %xmm0, %rax
-; CHECK-NEXT:    vcvtsd2usi {rz-sae}, %xmm0, %rcx
-; CHECK-NEXT:    addq %rax, %rcx
-; CHECK-NEXT:    vcvtsd2usi {rd-sae}, %xmm0, %rax
-; CHECK-NEXT:    addq %rcx, %rax
-; CHECK-NEXT:    retq
-
-  %res = call i64 @llvm.x86.avx512.vcvtsd2usi64(<2 x double> %a0, i32 4)
-  %res1 = call i64 @llvm.x86.avx512.vcvtsd2usi64(<2 x double> %a0, i32 3)
-  %res2 = call i64 @llvm.x86.avx512.vcvtsd2usi64(<2 x double> %a0, i32 1)
-  %res3 = add i64 %res, %res1
-  %res4 = add i64 %res3, %res2
-  ret i64 %res4
-}
-declare i64 @llvm.x86.avx512.vcvtsd2usi64(<2 x double>, i32) nounwind readnone
-
-define i64 @test_x86_avx512_cvtsd2si64(<2 x double> %a0) {
-; CHECK-LABEL: test_x86_avx512_cvtsd2si64:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    vcvtsd2si %xmm0, %rax
-; CHECK-NEXT:    vcvtsd2si {rz-sae}, %xmm0, %rcx
-; CHECK-NEXT:    addq %rax, %rcx
-; CHECK-NEXT:    vcvtsd2si {rd-sae}, %xmm0, %rax
-; CHECK-NEXT:    addq %rcx, %rax
-; CHECK-NEXT:    retq
-
-  %res = call i64 @llvm.x86.avx512.vcvtsd2si64(<2 x double> %a0, i32 4)
-  %res1 = call i64 @llvm.x86.avx512.vcvtsd2si64(<2 x double> %a0, i32 3)
-  %res2 = call i64 @llvm.x86.avx512.vcvtsd2si64(<2 x double> %a0, i32 1)
-  %res3 = add i64 %res, %res1
-  %res4 = add i64 %res3, %res2
-  ret i64 %res4
-}
-declare i64 @llvm.x86.avx512.vcvtsd2si64(<2 x double>, i32) nounwind readnone
-
-define i64 @test_x86_avx512_cvtss2usi64(<4 x float> %a0) {
-; CHECK-LABEL: test_x86_avx512_cvtss2usi64:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    vcvtss2usi %xmm0, %rax
-; CHECK-NEXT:    vcvtss2usi {rz-sae}, %xmm0, %rcx
-; CHECK-NEXT:    addq %rax, %rcx
-; CHECK-NEXT:    vcvtss2usi {rd-sae}, %xmm0, %rax
-; CHECK-NEXT:    addq %rcx, %rax
-; CHECK-NEXT:    retq
-
-  %res = call i64 @llvm.x86.avx512.vcvtss2usi64(<4 x float> %a0, i32 4)
-  %res1 = call i64 @llvm.x86.avx512.vcvtss2usi64(<4 x float> %a0, i32 3)
-  %res2 = call i64 @llvm.x86.avx512.vcvtss2usi64(<4 x float> %a0, i32 1)
-  %res3 = add i64 %res, %res1
-  %res4 = add i64 %res3, %res2
-  ret i64 %res4
-}
-declare i64 @llvm.x86.avx512.vcvtss2usi64(<4 x float>, i32) nounwind readnone
-
-define i64 @test_x86_avx512_cvtss2si64(<4 x float> %a0) {
-; CHECK-LABEL: test_x86_avx512_cvtss2si64:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    vcvtss2si %xmm0, %rax
-; CHECK-NEXT:    vcvtss2si {rz-sae}, %xmm0, %rcx
-; CHECK-NEXT:    addq %rax, %rcx
-; CHECK-NEXT:    vcvtss2si {rd-sae}, %xmm0, %rax
-; CHECK-NEXT:    addq %rcx, %rax
-; CHECK-NEXT:    retq
-
-  %res = call i64 @llvm.x86.avx512.vcvtss2si64(<4 x float> %a0, i32 4)
-  %res1 = call i64 @llvm.x86.avx512.vcvtss2si64(<4 x float> %a0, i32 3)
-  %res2 = call i64 @llvm.x86.avx512.vcvtss2si64(<4 x float> %a0, i32 1)
-  %res3 = add i64 %res, %res1
-  %res4 = add i64 %res3, %res2
-  ret i64 %res4
-}
-declare i64 @llvm.x86.avx512.vcvtss2si64(<4 x float>, i32) nounwind readnone
-
 define i32 @test_x86_avx512_cvtsd2usi32(<2 x double> %a0) {
 ; CHECK-LABEL: test_x86_avx512_cvtsd2usi32:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vcvtsd2usi %xmm0, %eax
 ; CHECK-NEXT:    vcvtsd2usi {rz-sae}, %xmm0, %ecx
 ; CHECK-NEXT:    addl %eax, %ecx
 ; CHECK-NEXT:    vcvtsd2usi {rd-sae}, %xmm0, %eax
 ; CHECK-NEXT:    addl %ecx, %eax
-; CHECK-NEXT:    retq
+; CHECK-NEXT:    ret{{[l|q]}}
 
   %res = call i32 @llvm.x86.avx512.vcvtsd2usi32(<2 x double> %a0, i32 4)
   %res1 = call i32 @llvm.x86.avx512.vcvtsd2usi32(<2 x double> %a0, i32 3)
@@ -880,13 +952,13 @@ declare i32 @llvm.x86.avx512.vcvtsd2usi32(<2 x double>, i32) nounwind readnone
 
 define i32 @test_x86_avx512_cvtsd2si32(<2 x double> %a0) {
 ; CHECK-LABEL: test_x86_avx512_cvtsd2si32:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vcvtsd2si %xmm0, %eax
 ; CHECK-NEXT:    vcvtsd2si {rz-sae}, %xmm0, %ecx
 ; CHECK-NEXT:    addl %eax, %ecx
 ; CHECK-NEXT:    vcvtsd2si {rd-sae}, %xmm0, %eax
 ; CHECK-NEXT:    addl %ecx, %eax
-; CHECK-NEXT:    retq
+; CHECK-NEXT:    ret{{[l|q]}}
 
   %res = call i32 @llvm.x86.avx512.vcvtsd2si32(<2 x double> %a0, i32 4)
   %res1 = call i32 @llvm.x86.avx512.vcvtsd2si32(<2 x double> %a0, i32 3)
@@ -899,13 +971,13 @@ declare i32 @llvm.x86.avx512.vcvtsd2si32(<2 x double>, i32) nounwind readnone
 
 define i32 @test_x86_avx512_cvtss2usi32(<4 x float> %a0) {
 ; CHECK-LABEL: test_x86_avx512_cvtss2usi32:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vcvtss2usi %xmm0, %eax
 ; CHECK-NEXT:    vcvtss2usi {rz-sae}, %xmm0, %ecx
 ; CHECK-NEXT:    addl %eax, %ecx
 ; CHECK-NEXT:    vcvtss2usi {rd-sae}, %xmm0, %eax
 ; CHECK-NEXT:    addl %ecx, %eax
-; CHECK-NEXT:    retq
+; CHECK-NEXT:    ret{{[l|q]}}
 
   %res = call i32 @llvm.x86.avx512.vcvtss2usi32(<4 x float> %a0, i32 4)
   %res1 = call i32 @llvm.x86.avx512.vcvtss2usi32(<4 x float> %a0, i32 3)
@@ -918,13 +990,13 @@ declare i32 @llvm.x86.avx512.vcvtss2usi32(<4 x float>, i32) nounwind readnone
 
 define i32 @test_x86_avx512_cvtss2si32(<4 x float> %a0) {
 ; CHECK-LABEL: test_x86_avx512_cvtss2si32:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vcvtss2si %xmm0, %eax
 ; CHECK-NEXT:    vcvtss2si {rz-sae}, %xmm0, %ecx
 ; CHECK-NEXT:    addl %eax, %ecx
 ; CHECK-NEXT:    vcvtss2si {rd-sae}, %xmm0, %eax
 ; CHECK-NEXT:    addl %ecx, %eax
-; CHECK-NEXT:    retq
+; CHECK-NEXT:    ret{{[l|q]}}
 
   %res = call i32 @llvm.x86.avx512.vcvtss2si32(<4 x float> %a0, i32 4)
   %res1 = call i32 @llvm.x86.avx512.vcvtss2si32(<4 x float> %a0, i32 3)
@@ -937,49 +1009,68 @@ declare i32 @llvm.x86.avx512.vcvtss2si32(<4 x float>, i32) nounwind readnone
 
 define <16 x float> @test_x86_vcvtph2ps_512(<16 x i16> %a0) {
 ; CHECK-LABEL: test_x86_vcvtph2ps_512:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vcvtph2ps %ymm0, %zmm0
-; CHECK-NEXT:    retq
+; CHECK-NEXT:    ret{{[l|q]}}
   %res = call <16 x float> @llvm.x86.avx512.mask.vcvtph2ps.512(<16 x i16> %a0, <16 x float> zeroinitializer, i16 -1, i32 4)
   ret <16 x float> %res
 }
 
 define <16 x float> @test_x86_vcvtph2ps_512_sae(<16 x i16> %a0) {
 ; CHECK-LABEL: test_x86_vcvtph2ps_512_sae:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vcvtph2ps {sae}, %ymm0, %zmm0
-; CHECK-NEXT:    retq
+; CHECK-NEXT:    ret{{[l|q]}}
   %res = call <16 x float> @llvm.x86.avx512.mask.vcvtph2ps.512(<16 x i16> %a0, <16 x float> zeroinitializer, i16 -1, i32 8)
   ret <16 x float> %res
 }
 
 define <16 x float> @test_x86_vcvtph2ps_512_rrk(<16 x i16> %a0,<16 x float> %a1, i16 %mask) {
-; CHECK-LABEL: test_x86_vcvtph2ps_512_rrk:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vcvtph2ps %ymm0, %zmm1 {%k1}
-; CHECK-NEXT:    vmovaps %zmm1, %zmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_x86_vcvtph2ps_512_rrk:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vcvtph2ps %ymm0, %zmm1 {%k1}
+; X64-NEXT:    vmovaps %zmm1, %zmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_x86_vcvtph2ps_512_rrk:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vcvtph2ps %ymm0, %zmm1 {%k1}
+; X86-NEXT:    vmovaps %zmm1, %zmm0
+; X86-NEXT:    retl
   %res = call <16 x float> @llvm.x86.avx512.mask.vcvtph2ps.512(<16 x i16> %a0, <16 x float> %a1, i16 %mask, i32 4)
   ret <16 x float> %res
 }
 
 define <16 x float> @test_x86_vcvtph2ps_512_sae_rrkz(<16 x i16> %a0, i16 %mask) {
-; CHECK-LABEL: test_x86_vcvtph2ps_512_sae_rrkz:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vcvtph2ps {sae}, %ymm0, %zmm0 {%k1} {z}
-; CHECK-NEXT:    retq
+; X64-LABEL: test_x86_vcvtph2ps_512_sae_rrkz:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vcvtph2ps {sae}, %ymm0, %zmm0 {%k1} {z}
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_x86_vcvtph2ps_512_sae_rrkz:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vcvtph2ps {sae}, %ymm0, %zmm0 {%k1} {z}
+; X86-NEXT:    retl
   %res = call <16 x float> @llvm.x86.avx512.mask.vcvtph2ps.512(<16 x i16> %a0, <16 x float> zeroinitializer, i16 %mask, i32 8)
   ret <16 x float> %res
 }
 
 define <16 x float> @test_x86_vcvtph2ps_512_rrkz(<16 x i16> %a0, i16 %mask) {
-; CHECK-LABEL: test_x86_vcvtph2ps_512_rrkz:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vcvtph2ps %ymm0, %zmm0 {%k1} {z}
-; CHECK-NEXT:    retq
+; X64-LABEL: test_x86_vcvtph2ps_512_rrkz:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vcvtph2ps %ymm0, %zmm0 {%k1} {z}
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_x86_vcvtph2ps_512_rrkz:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vcvtph2ps %ymm0, %zmm0 {%k1} {z}
+; X86-NEXT:    retl
   %res = call <16 x float> @llvm.x86.avx512.mask.vcvtph2ps.512(<16 x i16> %a0, <16 x float> zeroinitializer, i16 %mask, i32 4)
   ret <16 x float> %res
 }
@@ -987,15 +1078,26 @@ define <16 x float> @test_x86_vcvtph2ps_512_rrkz(<16 x i16> %a0, i16 %mask) {
 declare <16 x float> @llvm.x86.avx512.mask.vcvtph2ps.512(<16 x i16>, <16 x float>, i16, i32) nounwind readonly
 
 define <16 x i16> @test_x86_vcvtps2ph_256(<16 x float> %a0, <16 x i16> %src, i16 %mask, <16 x i16> * %dst) {
-; CHECK-LABEL: test_x86_vcvtps2ph_256:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vcvtps2ph $2, %zmm0, %ymm1 {%k1}
-; CHECK-NEXT:    vcvtps2ph $2, %zmm0, %ymm2 {%k1} {z}
-; CHECK-NEXT:    vpaddw %ymm1, %ymm2, %ymm1
-; CHECK-NEXT:    vcvtps2ph $2, %zmm0, (%rsi)
-; CHECK-NEXT:    vmovdqa %ymm1, %ymm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_x86_vcvtps2ph_256:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vcvtps2ph $2, %zmm0, %ymm2 {%k1} {z}
+; X64-NEXT:    vcvtps2ph $2, %zmm0, %ymm1 {%k1}
+; X64-NEXT:    vpaddw %ymm1, %ymm2, %ymm1
+; X64-NEXT:    vcvtps2ph $2, %zmm0, (%rsi)
+; X64-NEXT:    vmovdqa %ymm1, %ymm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_x86_vcvtps2ph_256:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vcvtps2ph $2, %zmm0, %ymm2 {%k1} {z}
+; X86-NEXT:    vcvtps2ph $2, %zmm0, %ymm1 {%k1}
+; X86-NEXT:    vpaddw %ymm1, %ymm2, %ymm1
+; X86-NEXT:    vcvtps2ph $2, %zmm0, (%eax)
+; X86-NEXT:    vmovdqa %ymm1, %ymm0
+; X86-NEXT:    retl
   %res1 = call <16 x i16> @llvm.x86.avx512.mask.vcvtps2ph.512(<16 x float> %a0, i32 2, <16 x i16> zeroinitializer, i16 -1)
   %res2 = call <16 x i16> @llvm.x86.avx512.mask.vcvtps2ph.512(<16 x float> %a0, i32 2, <16 x i16> zeroinitializer, i16 %mask)
   %res3 = call <16 x i16> @llvm.x86.avx512.mask.vcvtps2ph.512(<16 x float> %a0, i32 2, <16 x i16> %src, i16 %mask)
@@ -1008,12 +1110,12 @@ declare <16 x i16> @llvm.x86.avx512.mask.vcvtps2ph.512(<16 x float>, i32, <16 x
 
 define i16 @test_cmpps(<16 x float> %a, <16 x float> %b) {
 ; CHECK-LABEL: test_cmpps:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vcmpleps {sae}, %zmm1, %zmm0, %k0
 ; CHECK-NEXT:    kmovw %k0, %eax
-; CHECK-NEXT:    ## kill: def $ax killed $ax killed $eax
+; CHECK-NEXT:    # kill: def $ax killed $ax killed $eax
 ; CHECK-NEXT:    vzeroupper
-; CHECK-NEXT:    retq
+; CHECK-NEXT:    ret{{[l|q]}}
   %res = call <16 x i1> @llvm.x86.avx512.cmp.ps.512(<16 x float> %a, <16 x float> %b, i32 2, i32 8)
   %1 = bitcast <16 x i1> %res to i16
   ret i16 %1
@@ -1022,12 +1124,12 @@ declare <16 x i1> @llvm.x86.avx512.cmp.ps.512(<16 x float>, <16 x float>, i32, i
 
 define i8 @test_cmppd(<8 x double> %a, <8 x double> %b) {
 ; CHECK-LABEL: test_cmppd:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vcmpneqpd %zmm1, %zmm0, %k0
 ; CHECK-NEXT:    kmovw %k0, %eax
-; CHECK-NEXT:    ## kill: def $al killed $al killed $eax
+; CHECK-NEXT:    # kill: def $al killed $al killed $eax
 ; CHECK-NEXT:    vzeroupper
-; CHECK-NEXT:    retq
+; CHECK-NEXT:    ret{{[l|q]}}
   %res = call <8 x i1> @llvm.x86.avx512.cmp.pd.512(<8 x double> %a, <8 x double> %b, i32 4, i32 4)
   %1 = bitcast <8 x i1> %res to i8
   ret i8 %1
@@ -1039,9 +1141,9 @@ declare <8 x i1> @llvm.x86.avx512.cmp.pd.512(<8 x double>, <8 x double>, i32, i3
  ; fp min - max
 define <8 x double> @test_vmaxpd(<8 x double> %a0, <8 x double> %a1) {
 ; CHECK-LABEL: test_vmaxpd:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vmaxpd %zmm1, %zmm0, %zmm0
-; CHECK-NEXT:    retq
+; CHECK-NEXT:    ret{{[l|q]}}
   %1 = call <8 x double> @llvm.x86.avx512.max.pd.512(<8 x double> %a0, <8 x double> %a1, i32 4)
   ret <8 x double> %1
 }
@@ -1049,20 +1151,28 @@ declare <8 x double> @llvm.x86.avx512.max.pd.512(<8 x double>, <8 x double>, i32
 
 define <8 x double> @test_vminpd(<8 x double> %a0, <8 x double> %a1) {
 ; CHECK-LABEL: test_vminpd:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vminpd %zmm1, %zmm0, %zmm0
-; CHECK-NEXT:    retq
+; CHECK-NEXT:    ret{{[l|q]}}
   %1 = call <8 x double> @llvm.x86.avx512.min.pd.512(<8 x double> %a0, <8 x double> %a1, i32 4)
   ret <8 x double> %1
 }
 declare <8 x double> @llvm.x86.avx512.min.pd.512(<8 x double>, <8 x double>, i32)
 
 define void @test_mask_store_ss(i8* %ptr, <4 x float> %data, i8 %mask) {
-; CHECK-LABEL: test_mask_store_ss:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %esi, %k1
-; CHECK-NEXT:    vmovss %xmm0, (%rdi) {%k1}
-; CHECK-NEXT:    retq
+; X64-LABEL: test_mask_store_ss:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %esi, %k1
+; X64-NEXT:    vmovss %xmm0, (%rdi) {%k1}
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_mask_store_ss:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movb {{[0-9]+}}(%esp), %cl
+; X86-NEXT:    kmovw %ecx, %k1
+; X86-NEXT:    vmovss %xmm0, (%eax) {%k1}
+; X86-NEXT:    retl
   %1 = and i8 %mask, 1
   %2 = bitcast i8* %ptr to <4 x float>*
   %3 = bitcast i8 %1 to <8 x i1>
@@ -1079,83 +1189,89 @@ declare <8 x double> @llvm.x86.avx512.mul.pd.512(<8 x double>, <8 x double>, i32
 
 define <16 x float> @test_vsubps_rn(<16 x float> %a0, <16 x float> %a1) {
 ; CHECK-LABEL: test_vsubps_rn:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsubps {rn-sae}, %zmm1, %zmm0, %zmm0
-; CHECK-NEXT:    retq
+; CHECK-NEXT:    ret{{[l|q]}}
   %1 = call <16 x float> @llvm.x86.avx512.sub.ps.512(<16 x float> %a0, <16 x float> %a1, i32 0)
   ret <16 x float> %1
 }
 
 define <16 x float> @test_vsubps_rd(<16 x float> %a0, <16 x float> %a1) {
 ; CHECK-LABEL: test_vsubps_rd:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsubps {rd-sae}, %zmm1, %zmm0, %zmm0
-; CHECK-NEXT:    retq
+; CHECK-NEXT:    ret{{[l|q]}}
   %1 = call <16 x float> @llvm.x86.avx512.sub.ps.512(<16 x float> %a0, <16 x float> %a1, i32 1)
   ret <16 x float> %1
 }
 
 define <16 x float> @test_vsubps_ru(<16 x float> %a0, <16 x float> %a1) {
 ; CHECK-LABEL: test_vsubps_ru:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsubps {ru-sae}, %zmm1, %zmm0, %zmm0
-; CHECK-NEXT:    retq
+; CHECK-NEXT:    ret{{[l|q]}}
   %1 = call <16 x float> @llvm.x86.avx512.sub.ps.512(<16 x float> %a0, <16 x float> %a1, i32 2)
   ret <16 x float> %1
 }
 
 define <16 x float> @test_vsubps_rz(<16 x float> %a0, <16 x float> %a1) {
 ; CHECK-LABEL: test_vsubps_rz:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsubps {rz-sae}, %zmm1, %zmm0, %zmm0
-; CHECK-NEXT:    retq
+; CHECK-NEXT:    ret{{[l|q]}}
   %1 = call <16 x float> @llvm.x86.avx512.sub.ps.512(<16 x float> %a0, <16 x float> %a1, i32 3)
   ret <16 x float> %1
 }
 
 define <16 x float> @test_vmulps_rn(<16 x float> %a0, <16 x float> %a1) {
 ; CHECK-LABEL: test_vmulps_rn:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vmulps {rn-sae}, %zmm1, %zmm0, %zmm0
-; CHECK-NEXT:    retq
+; CHECK-NEXT:    ret{{[l|q]}}
   %1 = call <16 x float> @llvm.x86.avx512.mul.ps.512(<16 x float> %a0, <16 x float> %a1, i32 0)
   ret <16 x float> %1
 }
 
 define <16 x float> @test_vmulps_rd(<16 x float> %a0, <16 x float> %a1) {
 ; CHECK-LABEL: test_vmulps_rd:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vmulps {rd-sae}, %zmm1, %zmm0, %zmm0
-; CHECK-NEXT:    retq
+; CHECK-NEXT:    ret{{[l|q]}}
   %1 = call <16 x float> @llvm.x86.avx512.mul.ps.512(<16 x float> %a0, <16 x float> %a1, i32 1)
   ret <16 x float> %1
 }
 
 define <16 x float> @test_vmulps_ru(<16 x float> %a0, <16 x float> %a1) {
 ; CHECK-LABEL: test_vmulps_ru:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vmulps {ru-sae}, %zmm1, %zmm0, %zmm0
-; CHECK-NEXT:    retq
+; CHECK-NEXT:    ret{{[l|q]}}
   %1 = call <16 x float> @llvm.x86.avx512.mul.ps.512(<16 x float> %a0, <16 x float> %a1, i32 2)
   ret <16 x float> %1
 }
 
 define <16 x float> @test_vmulps_rz(<16 x float> %a0, <16 x float> %a1) {
 ; CHECK-LABEL: test_vmulps_rz:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vmulps {rz-sae}, %zmm1, %zmm0, %zmm0
-; CHECK-NEXT:    retq
+; CHECK-NEXT:    ret{{[l|q]}}
   %1 = call <16 x float> @llvm.x86.avx512.mul.ps.512(<16 x float> %a0, <16 x float> %a1, i32 3)
   ret <16 x float> %1
 }
 
 ;; mask float
 define <16 x float> @test_vmulps_mask_rn(<16 x float> %a0, <16 x float> %a1, i16 %mask) {
-; CHECK-LABEL: test_vmulps_mask_rn:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vmulps {rn-sae}, %zmm1, %zmm0, %zmm0 {%k1} {z}
-; CHECK-NEXT:    retq
+; X64-LABEL: test_vmulps_mask_rn:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vmulps {rn-sae}, %zmm1, %zmm0, %zmm0 {%k1} {z}
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_vmulps_mask_rn:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vmulps {rn-sae}, %zmm1, %zmm0, %zmm0 {%k1} {z}
+; X86-NEXT:    retl
   %1 = call <16 x float> @llvm.x86.avx512.mul.ps.512(<16 x float> %a0, <16 x float> %a1, i32 0)
   %2 = bitcast i16 %mask to <16 x i1>
   %3 = select <16 x i1> %2, <16 x float> %1, <16 x float> zeroinitializer
@@ -1163,11 +1279,17 @@ define <16 x float> @test_vmulps_mask_rn(<16 x float> %a0, <16 x float> %a1, i16
 }
 
 define <16 x float> @test_vmulps_mask_rd(<16 x float> %a0, <16 x float> %a1, i16 %mask) {
-; CHECK-LABEL: test_vmulps_mask_rd:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vmulps {rd-sae}, %zmm1, %zmm0, %zmm0 {%k1} {z}
-; CHECK-NEXT:    retq
+; X64-LABEL: test_vmulps_mask_rd:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vmulps {rd-sae}, %zmm1, %zmm0, %zmm0 {%k1} {z}
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_vmulps_mask_rd:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vmulps {rd-sae}, %zmm1, %zmm0, %zmm0 {%k1} {z}
+; X86-NEXT:    retl
   %1 = call <16 x float> @llvm.x86.avx512.mul.ps.512(<16 x float> %a0, <16 x float> %a1, i32 1)
   %2 = bitcast i16 %mask to <16 x i1>
   %3 = select <16 x i1> %2, <16 x float> %1, <16 x float> zeroinitializer
@@ -1175,11 +1297,17 @@ define <16 x float> @test_vmulps_mask_rd(<16 x float> %a0, <16 x float> %a1, i16
 }
 
 define <16 x float> @test_vmulps_mask_ru(<16 x float> %a0, <16 x float> %a1, i16 %mask) {
-; CHECK-LABEL: test_vmulps_mask_ru:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vmulps {ru-sae}, %zmm1, %zmm0, %zmm0 {%k1} {z}
-; CHECK-NEXT:    retq
+; X64-LABEL: test_vmulps_mask_ru:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vmulps {ru-sae}, %zmm1, %zmm0, %zmm0 {%k1} {z}
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_vmulps_mask_ru:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vmulps {ru-sae}, %zmm1, %zmm0, %zmm0 {%k1} {z}
+; X86-NEXT:    retl
   %1 = call <16 x float> @llvm.x86.avx512.mul.ps.512(<16 x float> %a0, <16 x float> %a1, i32 2)
   %2 = bitcast i16 %mask to <16 x i1>
   %3 = select <16 x i1> %2, <16 x float> %1, <16 x float> zeroinitializer
@@ -1187,11 +1315,17 @@ define <16 x float> @test_vmulps_mask_ru(<16 x float> %a0, <16 x float> %a1, i16
 }
 
 define <16 x float> @test_vmulps_mask_rz(<16 x float> %a0, <16 x float> %a1, i16 %mask) {
-; CHECK-LABEL: test_vmulps_mask_rz:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vmulps {rz-sae}, %zmm1, %zmm0, %zmm0 {%k1} {z}
-; CHECK-NEXT:    retq
+; X64-LABEL: test_vmulps_mask_rz:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vmulps {rz-sae}, %zmm1, %zmm0, %zmm0 {%k1} {z}
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_vmulps_mask_rz:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vmulps {rz-sae}, %zmm1, %zmm0, %zmm0 {%k1} {z}
+; X86-NEXT:    retl
   %1 = call <16 x float> @llvm.x86.avx512.mul.ps.512(<16 x float> %a0, <16 x float> %a1, i32 3)
   %2 = bitcast i16 %mask to <16 x i1>
   %3 = select <16 x i1> %2, <16 x float> %1, <16 x float> zeroinitializer
@@ -1200,12 +1334,19 @@ define <16 x float> @test_vmulps_mask_rz(<16 x float> %a0, <16 x float> %a1, i16
 
 ;; With Passthru value
 define <16 x float> @test_vmulps_mask_passthru_rn(<16 x float> %a0, <16 x float> %a1, <16 x float> %passthru, i16 %mask) {
-; CHECK-LABEL: test_vmulps_mask_passthru_rn:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vmulps {rn-sae}, %zmm1, %zmm0, %zmm2 {%k1}
-; CHECK-NEXT:    vmovaps %zmm2, %zmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_vmulps_mask_passthru_rn:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vmulps {rn-sae}, %zmm1, %zmm0, %zmm2 {%k1}
+; X64-NEXT:    vmovaps %zmm2, %zmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_vmulps_mask_passthru_rn:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vmulps {rn-sae}, %zmm1, %zmm0, %zmm2 {%k1}
+; X86-NEXT:    vmovaps %zmm2, %zmm0
+; X86-NEXT:    retl
   %1 = call <16 x float> @llvm.x86.avx512.mul.ps.512(<16 x float> %a0, <16 x float> %a1, i32 0)
   %2 = bitcast i16 %mask to <16 x i1>
   %3 = select <16 x i1> %2, <16 x float> %1, <16 x float> %passthru
@@ -1213,12 +1354,19 @@ define <16 x float> @test_vmulps_mask_passthru_rn(<16 x float> %a0, <16 x float>
 }
 
 define <16 x float> @test_vmulps_mask_passthru_rd(<16 x float> %a0, <16 x float> %a1, <16 x float> %passthru, i16 %mask) {
-; CHECK-LABEL: test_vmulps_mask_passthru_rd:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vmulps {rd-sae}, %zmm1, %zmm0, %zmm2 {%k1}
-; CHECK-NEXT:    vmovaps %zmm2, %zmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_vmulps_mask_passthru_rd:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vmulps {rd-sae}, %zmm1, %zmm0, %zmm2 {%k1}
+; X64-NEXT:    vmovaps %zmm2, %zmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_vmulps_mask_passthru_rd:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vmulps {rd-sae}, %zmm1, %zmm0, %zmm2 {%k1}
+; X86-NEXT:    vmovaps %zmm2, %zmm0
+; X86-NEXT:    retl
   %1 = call <16 x float> @llvm.x86.avx512.mul.ps.512(<16 x float> %a0, <16 x float> %a1, i32 1)
   %2 = bitcast i16 %mask to <16 x i1>
   %3 = select <16 x i1> %2, <16 x float> %1, <16 x float> %passthru
@@ -1226,12 +1374,19 @@ define <16 x float> @test_vmulps_mask_passthru_rd(<16 x float> %a0, <16 x float>
 }
 
 define <16 x float> @test_vmulps_mask_passthru_ru(<16 x float> %a0, <16 x float> %a1, <16 x float> %passthru, i16 %mask) {
-; CHECK-LABEL: test_vmulps_mask_passthru_ru:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vmulps {ru-sae}, %zmm1, %zmm0, %zmm2 {%k1}
-; CHECK-NEXT:    vmovaps %zmm2, %zmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_vmulps_mask_passthru_ru:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vmulps {ru-sae}, %zmm1, %zmm0, %zmm2 {%k1}
+; X64-NEXT:    vmovaps %zmm2, %zmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_vmulps_mask_passthru_ru:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vmulps {ru-sae}, %zmm1, %zmm0, %zmm2 {%k1}
+; X86-NEXT:    vmovaps %zmm2, %zmm0
+; X86-NEXT:    retl
   %1 = call <16 x float> @llvm.x86.avx512.mul.ps.512(<16 x float> %a0, <16 x float> %a1, i32 2)
   %2 = bitcast i16 %mask to <16 x i1>
   %3 = select <16 x i1> %2, <16 x float> %1, <16 x float> %passthru
@@ -1239,12 +1394,19 @@ define <16 x float> @test_vmulps_mask_passthru_ru(<16 x float> %a0, <16 x float>
 }
 
 define <16 x float> @test_vmulps_mask_passthru_rz(<16 x float> %a0, <16 x float> %a1, <16 x float> %passthru, i16 %mask) {
-; CHECK-LABEL: test_vmulps_mask_passthru_rz:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vmulps {rz-sae}, %zmm1, %zmm0, %zmm2 {%k1}
-; CHECK-NEXT:    vmovaps %zmm2, %zmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_vmulps_mask_passthru_rz:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vmulps {rz-sae}, %zmm1, %zmm0, %zmm2 {%k1}
+; X64-NEXT:    vmovaps %zmm2, %zmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_vmulps_mask_passthru_rz:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vmulps {rz-sae}, %zmm1, %zmm0, %zmm2 {%k1}
+; X86-NEXT:    vmovaps %zmm2, %zmm0
+; X86-NEXT:    retl
   %1 = call <16 x float> @llvm.x86.avx512.mul.ps.512(<16 x float> %a0, <16 x float> %a1, i32 3)
   %2 = bitcast i16 %mask to <16 x i1>
   %3 = select <16 x i1> %2, <16 x float> %1, <16 x float> %passthru
@@ -1253,11 +1415,18 @@ define <16 x float> @test_vmulps_mask_passthru_rz(<16 x float> %a0, <16 x float>
 
 ;; mask double
 define <8 x double> @test_vmulpd_mask_rn(<8 x double> %a0, <8 x double> %a1, i8 %mask) {
-; CHECK-LABEL: test_vmulpd_mask_rn:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vmulpd {rn-sae}, %zmm1, %zmm0, %zmm0 {%k1} {z}
-; CHECK-NEXT:    retq
+; X64-LABEL: test_vmulpd_mask_rn:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vmulpd {rn-sae}, %zmm1, %zmm0, %zmm0 {%k1} {z}
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_vmulpd_mask_rn:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vmulpd {rn-sae}, %zmm1, %zmm0, %zmm0 {%k1} {z}
+; X86-NEXT:    retl
   %1 = call <8 x double> @llvm.x86.avx512.mul.pd.512(<8 x double> %a0, <8 x double> %a1, i32 0)
   %2 = bitcast i8 %mask to <8 x i1>
   %3 = select <8 x i1> %2, <8 x double> %1, <8 x double> zeroinitializer
@@ -1265,11 +1434,18 @@ define <8 x double> @test_vmulpd_mask_rn(<8 x double> %a0, <8 x double> %a1, i8
 }
 
 define <8 x double> @test_vmulpd_mask_rd(<8 x double> %a0, <8 x double> %a1, i8 %mask) {
-; CHECK-LABEL: test_vmulpd_mask_rd:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vmulpd {rd-sae}, %zmm1, %zmm0, %zmm0 {%k1} {z}
-; CHECK-NEXT:    retq
+; X64-LABEL: test_vmulpd_mask_rd:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vmulpd {rd-sae}, %zmm1, %zmm0, %zmm0 {%k1} {z}
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_vmulpd_mask_rd:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vmulpd {rd-sae}, %zmm1, %zmm0, %zmm0 {%k1} {z}
+; X86-NEXT:    retl
   %1 = call <8 x double> @llvm.x86.avx512.mul.pd.512(<8 x double> %a0, <8 x double> %a1, i32 1)
   %2 = bitcast i8 %mask to <8 x i1>
   %3 = select <8 x i1> %2, <8 x double> %1, <8 x double> zeroinitializer
@@ -1277,11 +1453,18 @@ define <8 x double> @test_vmulpd_mask_rd(<8 x double> %a0, <8 x double> %a1, i8
 }
 
 define <8 x double> @test_vmulpd_mask_ru(<8 x double> %a0, <8 x double> %a1, i8 %mask) {
-; CHECK-LABEL: test_vmulpd_mask_ru:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vmulpd {ru-sae}, %zmm1, %zmm0, %zmm0 {%k1} {z}
-; CHECK-NEXT:    retq
+; X64-LABEL: test_vmulpd_mask_ru:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vmulpd {ru-sae}, %zmm1, %zmm0, %zmm0 {%k1} {z}
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_vmulpd_mask_ru:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vmulpd {ru-sae}, %zmm1, %zmm0, %zmm0 {%k1} {z}
+; X86-NEXT:    retl
   %1 = call <8 x double> @llvm.x86.avx512.mul.pd.512(<8 x double> %a0, <8 x double> %a1, i32 2)
   %2 = bitcast i8 %mask to <8 x i1>
   %3 = select <8 x i1> %2, <8 x double> %1, <8 x double> zeroinitializer
@@ -1289,11 +1472,18 @@ define <8 x double> @test_vmulpd_mask_ru(<8 x double> %a0, <8 x double> %a1, i8
 }
 
 define <8 x double> @test_vmulpd_mask_rz(<8 x double> %a0, <8 x double> %a1, i8 %mask) {
-; CHECK-LABEL: test_vmulpd_mask_rz:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vmulpd {rz-sae}, %zmm1, %zmm0, %zmm0 {%k1} {z}
-; CHECK-NEXT:    retq
+; X64-LABEL: test_vmulpd_mask_rz:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vmulpd {rz-sae}, %zmm1, %zmm0, %zmm0 {%k1} {z}
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_vmulpd_mask_rz:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vmulpd {rz-sae}, %zmm1, %zmm0, %zmm0 {%k1} {z}
+; X86-NEXT:    retl
   %1 = call <8 x double> @llvm.x86.avx512.mul.pd.512(<8 x double> %a0, <8 x double> %a1, i32 3)
   %2 = bitcast i8 %mask to <8 x i1>
   %3 = select <8 x i1> %2, <8 x double> %1, <8 x double> zeroinitializer
@@ -1301,11 +1491,17 @@ define <8 x double> @test_vmulpd_mask_rz(<8 x double> %a0, <8 x double> %a1, i8
 }
 
 define <16 x float> @test_mm512_maskz_add_round_ps_rn_sae(<16 x float> %a0, <16 x float> %a1, i16 %mask) {
-; CHECK-LABEL: test_mm512_maskz_add_round_ps_rn_sae:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vaddps {rn-sae}, %zmm1, %zmm0, %zmm0 {%k1} {z}
-; CHECK-NEXT:    retq
+; X64-LABEL: test_mm512_maskz_add_round_ps_rn_sae:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vaddps {rn-sae}, %zmm1, %zmm0, %zmm0 {%k1} {z}
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_mm512_maskz_add_round_ps_rn_sae:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vaddps {rn-sae}, %zmm1, %zmm0, %zmm0 {%k1} {z}
+; X86-NEXT:    retl
   %1 = call <16 x float> @llvm.x86.avx512.add.ps.512(<16 x float> %a0, <16 x float> %a1, i32 0)
   %2 = bitcast i16 %mask to <16 x i1>
   %3 = select <16 x i1> %2, <16 x float> %1, <16 x float> zeroinitializer
@@ -1313,11 +1509,17 @@ define <16 x float> @test_mm512_maskz_add_round_ps_rn_sae(<16 x float> %a0, <16
 }
 
 define <16 x float> @test_mm512_maskz_add_round_ps_rd_sae(<16 x float> %a0, <16 x float> %a1, i16 %mask) {
-; CHECK-LABEL: test_mm512_maskz_add_round_ps_rd_sae:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vaddps {rd-sae}, %zmm1, %zmm0, %zmm0 {%k1} {z}
-; CHECK-NEXT:    retq
+; X64-LABEL: test_mm512_maskz_add_round_ps_rd_sae:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vaddps {rd-sae}, %zmm1, %zmm0, %zmm0 {%k1} {z}
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_mm512_maskz_add_round_ps_rd_sae:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vaddps {rd-sae}, %zmm1, %zmm0, %zmm0 {%k1} {z}
+; X86-NEXT:    retl
   %1 = call <16 x float> @llvm.x86.avx512.add.ps.512(<16 x float> %a0, <16 x float> %a1, i32 1)
   %2 = bitcast i16 %mask to <16 x i1>
   %3 = select <16 x i1> %2, <16 x float> %1, <16 x float> zeroinitializer
@@ -1325,11 +1527,17 @@ define <16 x float> @test_mm512_maskz_add_round_ps_rd_sae(<16 x float> %a0, <16
 }
 
 define <16 x float> @test_mm512_maskz_add_round_ps_ru_sae(<16 x float> %a0, <16 x float> %a1, i16 %mask) {
-; CHECK-LABEL: test_mm512_maskz_add_round_ps_ru_sae:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vaddps {ru-sae}, %zmm1, %zmm0, %zmm0 {%k1} {z}
-; CHECK-NEXT:    retq
+; X64-LABEL: test_mm512_maskz_add_round_ps_ru_sae:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vaddps {ru-sae}, %zmm1, %zmm0, %zmm0 {%k1} {z}
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_mm512_maskz_add_round_ps_ru_sae:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vaddps {ru-sae}, %zmm1, %zmm0, %zmm0 {%k1} {z}
+; X86-NEXT:    retl
   %1 = call <16 x float> @llvm.x86.avx512.add.ps.512(<16 x float> %a0, <16 x float> %a1, i32 2)
   %2 = bitcast i16 %mask to <16 x i1>
   %3 = select <16 x i1> %2, <16 x float> %1, <16 x float> zeroinitializer
@@ -1337,11 +1545,17 @@ define <16 x float> @test_mm512_maskz_add_round_ps_ru_sae(<16 x float> %a0, <16
 }
 
 define <16 x float> @test_mm512_maskz_add_round_ps_rz_sae(<16 x float> %a0, <16 x float> %a1, i16 %mask) {
-; CHECK-LABEL: test_mm512_maskz_add_round_ps_rz_sae:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vaddps {rz-sae}, %zmm1, %zmm0, %zmm0 {%k1} {z}
-; CHECK-NEXT:    retq
+; X64-LABEL: test_mm512_maskz_add_round_ps_rz_sae:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vaddps {rz-sae}, %zmm1, %zmm0, %zmm0 {%k1} {z}
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_mm512_maskz_add_round_ps_rz_sae:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vaddps {rz-sae}, %zmm1, %zmm0, %zmm0 {%k1} {z}
+; X86-NEXT:    retl
   %1 = call <16 x float> @llvm.x86.avx512.add.ps.512(<16 x float> %a0, <16 x float> %a1, i32 3)
   %2 = bitcast i16 %mask to <16 x i1>
   %3 = select <16 x i1> %2, <16 x float> %1, <16 x float> zeroinitializer
@@ -1349,11 +1563,17 @@ define <16 x float> @test_mm512_maskz_add_round_ps_rz_sae(<16 x float> %a0, <16
 }
 
 define <16 x float> @test_mm512_maskz_add_round_ps_current(<16 x float> %a0, <16 x float> %a1, i16 %mask) {
-; CHECK-LABEL: test_mm512_maskz_add_round_ps_current:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vaddps %zmm1, %zmm0, %zmm0 {%k1} {z}
-; CHECK-NEXT:    retq
+; X64-LABEL: test_mm512_maskz_add_round_ps_current:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vaddps %zmm1, %zmm0, %zmm0 {%k1} {z}
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_mm512_maskz_add_round_ps_current:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vaddps %zmm1, %zmm0, %zmm0 {%k1} {z}
+; X86-NEXT:    retl
   %1 = call <16 x float> @llvm.x86.avx512.add.ps.512(<16 x float> %a0, <16 x float> %a1, i32 4)
   %2 = bitcast i16 %mask to <16 x i1>
   %3 = select <16 x i1> %2, <16 x float> %1, <16 x float> zeroinitializer
@@ -1361,12 +1581,19 @@ define <16 x float> @test_mm512_maskz_add_round_ps_current(<16 x float> %a0, <16
 }
 
 define <16 x float> @test_mm512_mask_add_round_ps_rn_sae(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask) {
-; CHECK-LABEL: test_mm512_mask_add_round_ps_rn_sae:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vaddps {rn-sae}, %zmm1, %zmm0, %zmm2 {%k1}
-; CHECK-NEXT:    vmovaps %zmm2, %zmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_mm512_mask_add_round_ps_rn_sae:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vaddps {rn-sae}, %zmm1, %zmm0, %zmm2 {%k1}
+; X64-NEXT:    vmovaps %zmm2, %zmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_mm512_mask_add_round_ps_rn_sae:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vaddps {rn-sae}, %zmm1, %zmm0, %zmm2 {%k1}
+; X86-NEXT:    vmovaps %zmm2, %zmm0
+; X86-NEXT:    retl
   %1 = call <16 x float> @llvm.x86.avx512.add.ps.512(<16 x float> %a0, <16 x float> %a1, i32 0)
   %2 = bitcast i16 %mask to <16 x i1>
   %3 = select <16 x i1> %2, <16 x float> %1, <16 x float> %src
@@ -1374,12 +1601,19 @@ define <16 x float> @test_mm512_mask_add_round_ps_rn_sae(<16 x float> %a0, <16 x
 }
 
 define <16 x float> @test_mm512_mask_add_round_ps_rd_sae(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask) {
-; CHECK-LABEL: test_mm512_mask_add_round_ps_rd_sae:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vaddps {rd-sae}, %zmm1, %zmm0, %zmm2 {%k1}
-; CHECK-NEXT:    vmovaps %zmm2, %zmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_mm512_mask_add_round_ps_rd_sae:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vaddps {rd-sae}, %zmm1, %zmm0, %zmm2 {%k1}
+; X64-NEXT:    vmovaps %zmm2, %zmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_mm512_mask_add_round_ps_rd_sae:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vaddps {rd-sae}, %zmm1, %zmm0, %zmm2 {%k1}
+; X86-NEXT:    vmovaps %zmm2, %zmm0
+; X86-NEXT:    retl
   %1 = call <16 x float> @llvm.x86.avx512.add.ps.512(<16 x float> %a0, <16 x float> %a1, i32 1)
   %2 = bitcast i16 %mask to <16 x i1>
   %3 = select <16 x i1> %2, <16 x float> %1, <16 x float> %src
@@ -1387,12 +1621,19 @@ define <16 x float> @test_mm512_mask_add_round_ps_rd_sae(<16 x float> %a0, <16 x
 }
 
 define <16 x float> @test_mm512_mask_add_round_ps_ru_sae(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask) {
-; CHECK-LABEL: test_mm512_mask_add_round_ps_ru_sae:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vaddps {ru-sae}, %zmm1, %zmm0, %zmm2 {%k1}
-; CHECK-NEXT:    vmovaps %zmm2, %zmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_mm512_mask_add_round_ps_ru_sae:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vaddps {ru-sae}, %zmm1, %zmm0, %zmm2 {%k1}
+; X64-NEXT:    vmovaps %zmm2, %zmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_mm512_mask_add_round_ps_ru_sae:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vaddps {ru-sae}, %zmm1, %zmm0, %zmm2 {%k1}
+; X86-NEXT:    vmovaps %zmm2, %zmm0
+; X86-NEXT:    retl
   %1 = call <16 x float> @llvm.x86.avx512.add.ps.512(<16 x float> %a0, <16 x float> %a1, i32 2)
   %2 = bitcast i16 %mask to <16 x i1>
   %3 = select <16 x i1> %2, <16 x float> %1, <16 x float> %src
@@ -1400,12 +1641,19 @@ define <16 x float> @test_mm512_mask_add_round_ps_ru_sae(<16 x float> %a0, <16 x
 }
 
 define <16 x float> @test_mm512_mask_add_round_ps_rz_sae(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask) {
-; CHECK-LABEL: test_mm512_mask_add_round_ps_rz_sae:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vaddps {rz-sae}, %zmm1, %zmm0, %zmm2 {%k1}
-; CHECK-NEXT:    vmovaps %zmm2, %zmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_mm512_mask_add_round_ps_rz_sae:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vaddps {rz-sae}, %zmm1, %zmm0, %zmm2 {%k1}
+; X64-NEXT:    vmovaps %zmm2, %zmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_mm512_mask_add_round_ps_rz_sae:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vaddps {rz-sae}, %zmm1, %zmm0, %zmm2 {%k1}
+; X86-NEXT:    vmovaps %zmm2, %zmm0
+; X86-NEXT:    retl
   %1 = call <16 x float> @llvm.x86.avx512.add.ps.512(<16 x float> %a0, <16 x float> %a1, i32 3)
   %2 = bitcast i16 %mask to <16 x i1>
   %3 = select <16 x i1> %2, <16 x float> %1, <16 x float> %src
@@ -1413,12 +1661,19 @@ define <16 x float> @test_mm512_mask_add_round_ps_rz_sae(<16 x float> %a0, <16 x
 }
 
 define <16 x float> @test_mm512_mask_add_round_ps_current(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask) {
-; CHECK-LABEL: test_mm512_mask_add_round_ps_current:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vaddps %zmm1, %zmm0, %zmm2 {%k1}
-; CHECK-NEXT:    vmovaps %zmm2, %zmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_mm512_mask_add_round_ps_current:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vaddps %zmm1, %zmm0, %zmm2 {%k1}
+; X64-NEXT:    vmovaps %zmm2, %zmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_mm512_mask_add_round_ps_current:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vaddps %zmm1, %zmm0, %zmm2 {%k1}
+; X86-NEXT:    vmovaps %zmm2, %zmm0
+; X86-NEXT:    retl
   %1 = call <16 x float> @llvm.x86.avx512.add.ps.512(<16 x float> %a0, <16 x float> %a1, i32 4)
   %2 = bitcast i16 %mask to <16 x i1>
   %3 = select <16 x i1> %2, <16 x float> %1, <16 x float> %src
@@ -1427,57 +1682,64 @@ define <16 x float> @test_mm512_mask_add_round_ps_current(<16 x float> %a0, <16
 
 define <16 x float> @test_mm512_add_round_ps_rn_sae(<16 x float> %a0, <16 x float> %a1, i16 %mask) {
 ; CHECK-LABEL: test_mm512_add_round_ps_rn_sae:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vaddps {rn-sae}, %zmm1, %zmm0, %zmm0
-; CHECK-NEXT:    retq
+; CHECK-NEXT:    ret{{[l|q]}}
   %1 = call <16 x float> @llvm.x86.avx512.add.ps.512(<16 x float> %a0, <16 x float> %a1, i32 0)
   ret <16 x float> %1
 }
 
 define <16 x float> @test_mm512_add_round_ps_rd_sae(<16 x float> %a0, <16 x float> %a1, i16 %mask) {
 ; CHECK-LABEL: test_mm512_add_round_ps_rd_sae:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vaddps {rd-sae}, %zmm1, %zmm0, %zmm0
-; CHECK-NEXT:    retq
+; CHECK-NEXT:    ret{{[l|q]}}
   %1 = call <16 x float> @llvm.x86.avx512.add.ps.512(<16 x float> %a0, <16 x float> %a1, i32 1)
   ret <16 x float> %1
 }
 
 define <16 x float> @test_mm512_add_round_ps_ru_sae(<16 x float> %a0, <16 x float> %a1, i16 %mask) {
 ; CHECK-LABEL: test_mm512_add_round_ps_ru_sae:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vaddps {ru-sae}, %zmm1, %zmm0, %zmm0
-; CHECK-NEXT:    retq
+; CHECK-NEXT:    ret{{[l|q]}}
   %1 = call <16 x float> @llvm.x86.avx512.add.ps.512(<16 x float> %a0, <16 x float> %a1, i32 2)
   ret <16 x float> %1
 }
 
 define <16 x float> @test_mm512_add_round_ps_rz_sae(<16 x float> %a0, <16 x float> %a1, i16 %mask) {
 ; CHECK-LABEL: test_mm512_add_round_ps_rz_sae:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vaddps {rz-sae}, %zmm1, %zmm0, %zmm0
-; CHECK-NEXT:    retq
+; CHECK-NEXT:    ret{{[l|q]}}
   %1 = call <16 x float> @llvm.x86.avx512.add.ps.512(<16 x float> %a0, <16 x float> %a1, i32 3)
   ret <16 x float> %1
 }
 
 define <16 x float> @test_mm512_add_round_ps_current(<16 x float> %a0, <16 x float> %a1, i16 %mask) {
 ; CHECK-LABEL: test_mm512_add_round_ps_current:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vaddps %zmm1, %zmm0, %zmm0
-; CHECK-NEXT:    retq
+; CHECK-NEXT:    ret{{[l|q]}}
   %1 = call <16 x float> @llvm.x86.avx512.add.ps.512(<16 x float> %a0, <16 x float> %a1, i32 4)
   ret <16 x float> %1
 }
 declare <16 x float> @llvm.x86.avx512.add.ps.512(<16 x float>, <16 x float>, i32)
 
 define <16 x float> @test_mm512_mask_sub_round_ps_rn_sae(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask) {
-; CHECK-LABEL: test_mm512_mask_sub_round_ps_rn_sae:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vsubps {rn-sae}, %zmm1, %zmm0, %zmm2 {%k1}
-; CHECK-NEXT:    vmovaps %zmm2, %zmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_mm512_mask_sub_round_ps_rn_sae:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vsubps {rn-sae}, %zmm1, %zmm0, %zmm2 {%k1}
+; X64-NEXT:    vmovaps %zmm2, %zmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_mm512_mask_sub_round_ps_rn_sae:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vsubps {rn-sae}, %zmm1, %zmm0, %zmm2 {%k1}
+; X86-NEXT:    vmovaps %zmm2, %zmm0
+; X86-NEXT:    retl
   %1 = call <16 x float> @llvm.x86.avx512.sub.ps.512(<16 x float> %a0, <16 x float> %a1, i32 0)
   %2 = bitcast i16 %mask to <16 x i1>
   %3 = select <16 x i1> %2, <16 x float> %1, <16 x float> %src
@@ -1485,12 +1747,19 @@ define <16 x float> @test_mm512_mask_sub_round_ps_rn_sae(<16 x float> %a0, <16 x
 }
 
 define <16 x float> @test_mm512_mask_sub_round_ps_rd_sae(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask) {
-; CHECK-LABEL: test_mm512_mask_sub_round_ps_rd_sae:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vsubps {rd-sae}, %zmm1, %zmm0, %zmm2 {%k1}
-; CHECK-NEXT:    vmovaps %zmm2, %zmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_mm512_mask_sub_round_ps_rd_sae:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vsubps {rd-sae}, %zmm1, %zmm0, %zmm2 {%k1}
+; X64-NEXT:    vmovaps %zmm2, %zmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_mm512_mask_sub_round_ps_rd_sae:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vsubps {rd-sae}, %zmm1, %zmm0, %zmm2 {%k1}
+; X86-NEXT:    vmovaps %zmm2, %zmm0
+; X86-NEXT:    retl
   %1 = call <16 x float> @llvm.x86.avx512.sub.ps.512(<16 x float> %a0, <16 x float> %a1, i32 1)
   %2 = bitcast i16 %mask to <16 x i1>
   %3 = select <16 x i1> %2, <16 x float> %1, <16 x float> %src
@@ -1498,12 +1767,19 @@ define <16 x float> @test_mm512_mask_sub_round_ps_rd_sae(<16 x float> %a0, <16 x
 }
 
 define <16 x float> @test_mm512_mask_sub_round_ps_ru_sae(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask) {
-; CHECK-LABEL: test_mm512_mask_sub_round_ps_ru_sae:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vsubps {ru-sae}, %zmm1, %zmm0, %zmm2 {%k1}
-; CHECK-NEXT:    vmovaps %zmm2, %zmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_mm512_mask_sub_round_ps_ru_sae:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vsubps {ru-sae}, %zmm1, %zmm0, %zmm2 {%k1}
+; X64-NEXT:    vmovaps %zmm2, %zmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_mm512_mask_sub_round_ps_ru_sae:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vsubps {ru-sae}, %zmm1, %zmm0, %zmm2 {%k1}
+; X86-NEXT:    vmovaps %zmm2, %zmm0
+; X86-NEXT:    retl
   %1 = call <16 x float> @llvm.x86.avx512.sub.ps.512(<16 x float> %a0, <16 x float> %a1, i32 2)
   %2 = bitcast i16 %mask to <16 x i1>
   %3 = select <16 x i1> %2, <16 x float> %1, <16 x float> %src
@@ -1511,12 +1787,19 @@ define <16 x float> @test_mm512_mask_sub_round_ps_ru_sae(<16 x float> %a0, <16 x
 }
 
 define <16 x float> @test_mm512_mask_sub_round_ps_rz_sae(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask) {
-; CHECK-LABEL: test_mm512_mask_sub_round_ps_rz_sae:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vsubps {rz-sae}, %zmm1, %zmm0, %zmm2 {%k1}
-; CHECK-NEXT:    vmovaps %zmm2, %zmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_mm512_mask_sub_round_ps_rz_sae:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vsubps {rz-sae}, %zmm1, %zmm0, %zmm2 {%k1}
+; X64-NEXT:    vmovaps %zmm2, %zmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_mm512_mask_sub_round_ps_rz_sae:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vsubps {rz-sae}, %zmm1, %zmm0, %zmm2 {%k1}
+; X86-NEXT:    vmovaps %zmm2, %zmm0
+; X86-NEXT:    retl
   %1 = call <16 x float> @llvm.x86.avx512.sub.ps.512(<16 x float> %a0, <16 x float> %a1, i32 3)
   %2 = bitcast i16 %mask to <16 x i1>
   %3 = select <16 x i1> %2, <16 x float> %1, <16 x float> %src
@@ -1524,12 +1807,19 @@ define <16 x float> @test_mm512_mask_sub_round_ps_rz_sae(<16 x float> %a0, <16 x
 }
 
 define <16 x float> @test_mm512_mask_sub_round_ps_current(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask) {
-; CHECK-LABEL: test_mm512_mask_sub_round_ps_current:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vsubps %zmm1, %zmm0, %zmm2 {%k1}
-; CHECK-NEXT:    vmovaps %zmm2, %zmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_mm512_mask_sub_round_ps_current:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vsubps %zmm1, %zmm0, %zmm2 {%k1}
+; X64-NEXT:    vmovaps %zmm2, %zmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_mm512_mask_sub_round_ps_current:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vsubps %zmm1, %zmm0, %zmm2 {%k1}
+; X86-NEXT:    vmovaps %zmm2, %zmm0
+; X86-NEXT:    retl
   %1 = call <16 x float> @llvm.x86.avx512.sub.ps.512(<16 x float> %a0, <16 x float> %a1, i32 4)
   %2 = bitcast i16 %mask to <16 x i1>
   %3 = select <16 x i1> %2, <16 x float> %1, <16 x float> %src
@@ -1538,55 +1828,61 @@ define <16 x float> @test_mm512_mask_sub_round_ps_current(<16 x float> %a0, <16
 
 define <16 x float> @test_mm512_sub_round_ps_rn_sae(<16 x float> %a0, <16 x float> %a1, i16 %mask) {
 ; CHECK-LABEL: test_mm512_sub_round_ps_rn_sae:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsubps {rn-sae}, %zmm1, %zmm0, %zmm0
-; CHECK-NEXT:    retq
+; CHECK-NEXT:    ret{{[l|q]}}
   %1 = call <16 x float> @llvm.x86.avx512.sub.ps.512(<16 x float> %a0, <16 x float> %a1, i32 0)
   ret <16 x float> %1
 }
 
 define <16 x float> @test_mm512_sub_round_ps_rd_sae(<16 x float> %a0, <16 x float> %a1, i16 %mask) {
 ; CHECK-LABEL: test_mm512_sub_round_ps_rd_sae:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsubps {rd-sae}, %zmm1, %zmm0, %zmm0
-; CHECK-NEXT:    retq
+; CHECK-NEXT:    ret{{[l|q]}}
   %1 = call <16 x float> @llvm.x86.avx512.sub.ps.512(<16 x float> %a0, <16 x float> %a1, i32 1)
   ret <16 x float> %1
 }
 
 define <16 x float> @test_mm512_sub_round_ps_ru_sae(<16 x float> %a0, <16 x float> %a1, i16 %mask) {
 ; CHECK-LABEL: test_mm512_sub_round_ps_ru_sae:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsubps {ru-sae}, %zmm1, %zmm0, %zmm0
-; CHECK-NEXT:    retq
+; CHECK-NEXT:    ret{{[l|q]}}
   %1 = call <16 x float> @llvm.x86.avx512.sub.ps.512(<16 x float> %a0, <16 x float> %a1, i32 2)
   ret <16 x float> %1
 }
 
 define <16 x float> @test_mm512_sub_round_ps_rz_sae(<16 x float> %a0, <16 x float> %a1, i16 %mask) {
 ; CHECK-LABEL: test_mm512_sub_round_ps_rz_sae:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsubps {rz-sae}, %zmm1, %zmm0, %zmm0
-; CHECK-NEXT:    retq
+; CHECK-NEXT:    ret{{[l|q]}}
   %1 = call <16 x float> @llvm.x86.avx512.sub.ps.512(<16 x float> %a0, <16 x float> %a1, i32 3)
   ret <16 x float> %1
 }
 
 define <16 x float> @test_mm512_sub_round_ps_current(<16 x float> %a0, <16 x float> %a1, i16 %mask) {
 ; CHECK-LABEL: test_mm512_sub_round_ps_current:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsubps %zmm1, %zmm0, %zmm0
-; CHECK-NEXT:    retq
+; CHECK-NEXT:    ret{{[l|q]}}
   %1 = call <16 x float> @llvm.x86.avx512.sub.ps.512(<16 x float> %a0, <16 x float> %a1, i32 4)
   ret <16 x float> %1
 }
 
 define <16 x float> @test_mm512_maskz_div_round_ps_rn_sae(<16 x float> %a0, <16 x float> %a1, i16 %mask) {
-; CHECK-LABEL: test_mm512_maskz_div_round_ps_rn_sae:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vdivps {rn-sae}, %zmm1, %zmm0, %zmm0 {%k1} {z}
-; CHECK-NEXT:    retq
+; X64-LABEL: test_mm512_maskz_div_round_ps_rn_sae:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vdivps {rn-sae}, %zmm1, %zmm0, %zmm0 {%k1} {z}
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_mm512_maskz_div_round_ps_rn_sae:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vdivps {rn-sae}, %zmm1, %zmm0, %zmm0 {%k1} {z}
+; X86-NEXT:    retl
   %1 = call <16 x float> @llvm.x86.avx512.div.ps.512(<16 x float> %a0, <16 x float> %a1, i32 0)
   %2 = bitcast i16 %mask to <16 x i1>
   %3 = select <16 x i1> %2, <16 x float> %1, <16 x float> zeroinitializer
@@ -1594,11 +1890,17 @@ define <16 x float> @test_mm512_maskz_div_round_ps_rn_sae(<16 x float> %a0, <16
 }
 
 define <16 x float> @test_mm512_maskz_div_round_ps_rd_sae(<16 x float> %a0, <16 x float> %a1, i16 %mask) {
-; CHECK-LABEL: test_mm512_maskz_div_round_ps_rd_sae:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vdivps {rd-sae}, %zmm1, %zmm0, %zmm0 {%k1} {z}
-; CHECK-NEXT:    retq
+; X64-LABEL: test_mm512_maskz_div_round_ps_rd_sae:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vdivps {rd-sae}, %zmm1, %zmm0, %zmm0 {%k1} {z}
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_mm512_maskz_div_round_ps_rd_sae:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vdivps {rd-sae}, %zmm1, %zmm0, %zmm0 {%k1} {z}
+; X86-NEXT:    retl
   %1 = call <16 x float> @llvm.x86.avx512.div.ps.512(<16 x float> %a0, <16 x float> %a1, i32 1)
   %2 = bitcast i16 %mask to <16 x i1>
   %3 = select <16 x i1> %2, <16 x float> %1, <16 x float> zeroinitializer
@@ -1606,11 +1908,17 @@ define <16 x float> @test_mm512_maskz_div_round_ps_rd_sae(<16 x float> %a0, <16
 }
 
 define <16 x float> @test_mm512_maskz_div_round_ps_ru_sae(<16 x float> %a0, <16 x float> %a1, i16 %mask) {
-; CHECK-LABEL: test_mm512_maskz_div_round_ps_ru_sae:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vdivps {ru-sae}, %zmm1, %zmm0, %zmm0 {%k1} {z}
-; CHECK-NEXT:    retq
+; X64-LABEL: test_mm512_maskz_div_round_ps_ru_sae:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vdivps {ru-sae}, %zmm1, %zmm0, %zmm0 {%k1} {z}
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_mm512_maskz_div_round_ps_ru_sae:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vdivps {ru-sae}, %zmm1, %zmm0, %zmm0 {%k1} {z}
+; X86-NEXT:    retl
   %1 = call <16 x float> @llvm.x86.avx512.div.ps.512(<16 x float> %a0, <16 x float> %a1, i32 2)
   %2 = bitcast i16 %mask to <16 x i1>
   %3 = select <16 x i1> %2, <16 x float> %1, <16 x float> zeroinitializer
@@ -1618,11 +1926,17 @@ define <16 x float> @test_mm512_maskz_div_round_ps_ru_sae(<16 x float> %a0, <16
 }
 
 define <16 x float> @test_mm512_maskz_div_round_ps_rz_sae(<16 x float> %a0, <16 x float> %a1, i16 %mask) {
-; CHECK-LABEL: test_mm512_maskz_div_round_ps_rz_sae:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vdivps {rz-sae}, %zmm1, %zmm0, %zmm0 {%k1} {z}
-; CHECK-NEXT:    retq
+; X64-LABEL: test_mm512_maskz_div_round_ps_rz_sae:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vdivps {rz-sae}, %zmm1, %zmm0, %zmm0 {%k1} {z}
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_mm512_maskz_div_round_ps_rz_sae:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vdivps {rz-sae}, %zmm1, %zmm0, %zmm0 {%k1} {z}
+; X86-NEXT:    retl
   %1 = call <16 x float> @llvm.x86.avx512.div.ps.512(<16 x float> %a0, <16 x float> %a1, i32 3)
   %2 = bitcast i16 %mask to <16 x i1>
   %3 = select <16 x i1> %2, <16 x float> %1, <16 x float> zeroinitializer
@@ -1630,11 +1944,17 @@ define <16 x float> @test_mm512_maskz_div_round_ps_rz_sae(<16 x float> %a0, <16
 }
 
 define <16 x float> @test_mm512_maskz_div_round_ps_current(<16 x float> %a0, <16 x float> %a1, i16 %mask) {
-; CHECK-LABEL: test_mm512_maskz_div_round_ps_current:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vdivps %zmm1, %zmm0, %zmm0 {%k1} {z}
-; CHECK-NEXT:    retq
+; X64-LABEL: test_mm512_maskz_div_round_ps_current:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vdivps %zmm1, %zmm0, %zmm0 {%k1} {z}
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_mm512_maskz_div_round_ps_current:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vdivps %zmm1, %zmm0, %zmm0 {%k1} {z}
+; X86-NEXT:    retl
   %1 = call <16 x float> @llvm.x86.avx512.div.ps.512(<16 x float> %a0, <16 x float> %a1, i32 4)
   %2 = bitcast i16 %mask to <16 x i1>
   %3 = select <16 x i1> %2, <16 x float> %1, <16 x float> zeroinitializer
@@ -1642,12 +1962,19 @@ define <16 x float> @test_mm512_maskz_div_round_ps_current(<16 x float> %a0, <16
 }
 
 define <16 x float> @test_mm512_mask_div_round_ps_rn_sae(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask) {
-; CHECK-LABEL: test_mm512_mask_div_round_ps_rn_sae:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vdivps {rn-sae}, %zmm1, %zmm0, %zmm2 {%k1}
-; CHECK-NEXT:    vmovaps %zmm2, %zmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_mm512_mask_div_round_ps_rn_sae:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vdivps {rn-sae}, %zmm1, %zmm0, %zmm2 {%k1}
+; X64-NEXT:    vmovaps %zmm2, %zmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_mm512_mask_div_round_ps_rn_sae:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vdivps {rn-sae}, %zmm1, %zmm0, %zmm2 {%k1}
+; X86-NEXT:    vmovaps %zmm2, %zmm0
+; X86-NEXT:    retl
   %1 = call <16 x float> @llvm.x86.avx512.div.ps.512(<16 x float> %a0, <16 x float> %a1, i32 0)
   %2 = bitcast i16 %mask to <16 x i1>
   %3 = select <16 x i1> %2, <16 x float> %1, <16 x float> %src
@@ -1655,12 +1982,19 @@ define <16 x float> @test_mm512_mask_div_round_ps_rn_sae(<16 x float> %a0, <16 x
 }
 
 define <16 x float> @test_mm512_mask_div_round_ps_rd_sae(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask) {
-; CHECK-LABEL: test_mm512_mask_div_round_ps_rd_sae:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vdivps {rd-sae}, %zmm1, %zmm0, %zmm2 {%k1}
-; CHECK-NEXT:    vmovaps %zmm2, %zmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_mm512_mask_div_round_ps_rd_sae:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vdivps {rd-sae}, %zmm1, %zmm0, %zmm2 {%k1}
+; X64-NEXT:    vmovaps %zmm2, %zmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_mm512_mask_div_round_ps_rd_sae:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vdivps {rd-sae}, %zmm1, %zmm0, %zmm2 {%k1}
+; X86-NEXT:    vmovaps %zmm2, %zmm0
+; X86-NEXT:    retl
   %1 = call <16 x float> @llvm.x86.avx512.div.ps.512(<16 x float> %a0, <16 x float> %a1, i32 1)
   %2 = bitcast i16 %mask to <16 x i1>
   %3 = select <16 x i1> %2, <16 x float> %1, <16 x float> %src
@@ -1668,12 +2002,19 @@ define <16 x float> @test_mm512_mask_div_round_ps_rd_sae(<16 x float> %a0, <16 x
 }
 
 define <16 x float> @test_mm512_mask_div_round_ps_ru_sae(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask) {
-; CHECK-LABEL: test_mm512_mask_div_round_ps_ru_sae:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vdivps {ru-sae}, %zmm1, %zmm0, %zmm2 {%k1}
-; CHECK-NEXT:    vmovaps %zmm2, %zmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_mm512_mask_div_round_ps_ru_sae:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vdivps {ru-sae}, %zmm1, %zmm0, %zmm2 {%k1}
+; X64-NEXT:    vmovaps %zmm2, %zmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_mm512_mask_div_round_ps_ru_sae:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vdivps {ru-sae}, %zmm1, %zmm0, %zmm2 {%k1}
+; X86-NEXT:    vmovaps %zmm2, %zmm0
+; X86-NEXT:    retl
   %1 = call <16 x float> @llvm.x86.avx512.div.ps.512(<16 x float> %a0, <16 x float> %a1, i32 2)
   %2 = bitcast i16 %mask to <16 x i1>
   %3 = select <16 x i1> %2, <16 x float> %1, <16 x float> %src
@@ -1681,12 +2022,19 @@ define <16 x float> @test_mm512_mask_div_round_ps_ru_sae(<16 x float> %a0, <16 x
 }
 
 define <16 x float> @test_mm512_mask_div_round_ps_rz_sae(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask) {
-; CHECK-LABEL: test_mm512_mask_div_round_ps_rz_sae:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vdivps {rz-sae}, %zmm1, %zmm0, %zmm2 {%k1}
-; CHECK-NEXT:    vmovaps %zmm2, %zmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_mm512_mask_div_round_ps_rz_sae:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vdivps {rz-sae}, %zmm1, %zmm0, %zmm2 {%k1}
+; X64-NEXT:    vmovaps %zmm2, %zmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_mm512_mask_div_round_ps_rz_sae:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vdivps {rz-sae}, %zmm1, %zmm0, %zmm2 {%k1}
+; X86-NEXT:    vmovaps %zmm2, %zmm0
+; X86-NEXT:    retl
   %1 = call <16 x float> @llvm.x86.avx512.div.ps.512(<16 x float> %a0, <16 x float> %a1, i32 3)
   %2 = bitcast i16 %mask to <16 x i1>
   %3 = select <16 x i1> %2, <16 x float> %1, <16 x float> %src
@@ -1694,12 +2042,19 @@ define <16 x float> @test_mm512_mask_div_round_ps_rz_sae(<16 x float> %a0, <16 x
 }
 
 define <16 x float> @test_mm512_mask_div_round_ps_current(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask) {
-; CHECK-LABEL: test_mm512_mask_div_round_ps_current:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vdivps %zmm1, %zmm0, %zmm2 {%k1}
-; CHECK-NEXT:    vmovaps %zmm2, %zmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_mm512_mask_div_round_ps_current:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vdivps %zmm1, %zmm0, %zmm2 {%k1}
+; X64-NEXT:    vmovaps %zmm2, %zmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_mm512_mask_div_round_ps_current:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vdivps %zmm1, %zmm0, %zmm2 {%k1}
+; X86-NEXT:    vmovaps %zmm2, %zmm0
+; X86-NEXT:    retl
   %1 = call <16 x float> @llvm.x86.avx512.div.ps.512(<16 x float> %a0, <16 x float> %a1, i32 4)
   %2 = bitcast i16 %mask to <16 x i1>
   %3 = select <16 x i1> %2, <16 x float> %1, <16 x float> %src
@@ -1708,56 +2063,62 @@ define <16 x float> @test_mm512_mask_div_round_ps_current(<16 x float> %a0, <16
 
 define <16 x float> @test_mm512_div_round_ps_rn_sae(<16 x float> %a0, <16 x float> %a1, i16 %mask) {
 ; CHECK-LABEL: test_mm512_div_round_ps_rn_sae:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vdivps {rn-sae}, %zmm1, %zmm0, %zmm0
-; CHECK-NEXT:    retq
+; CHECK-NEXT:    ret{{[l|q]}}
   %1 = call <16 x float> @llvm.x86.avx512.div.ps.512(<16 x float> %a0, <16 x float> %a1, i32 0)
   ret <16 x float> %1
 }
 
 define <16 x float> @test_mm512_div_round_ps_rd_sae(<16 x float> %a0, <16 x float> %a1, i16 %mask) {
 ; CHECK-LABEL: test_mm512_div_round_ps_rd_sae:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vdivps {rd-sae}, %zmm1, %zmm0, %zmm0
-; CHECK-NEXT:    retq
+; CHECK-NEXT:    ret{{[l|q]}}
   %1 = call <16 x float> @llvm.x86.avx512.div.ps.512(<16 x float> %a0, <16 x float> %a1, i32 1)
   ret <16 x float> %1
 }
 
 define <16 x float> @test_mm512_div_round_ps_ru_sae(<16 x float> %a0, <16 x float> %a1, i16 %mask) {
 ; CHECK-LABEL: test_mm512_div_round_ps_ru_sae:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vdivps {ru-sae}, %zmm1, %zmm0, %zmm0
-; CHECK-NEXT:    retq
+; CHECK-NEXT:    ret{{[l|q]}}
   %1 = call <16 x float> @llvm.x86.avx512.div.ps.512(<16 x float> %a0, <16 x float> %a1, i32 2)
   ret <16 x float> %1
 }
 
 define <16 x float> @test_mm512_div_round_ps_rz_sae(<16 x float> %a0, <16 x float> %a1, i16 %mask) {
 ; CHECK-LABEL: test_mm512_div_round_ps_rz_sae:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vdivps {rz-sae}, %zmm1, %zmm0, %zmm0
-; CHECK-NEXT:    retq
+; CHECK-NEXT:    ret{{[l|q]}}
   %1 = call <16 x float> @llvm.x86.avx512.div.ps.512(<16 x float> %a0, <16 x float> %a1, i32 3)
   ret <16 x float> %1
 }
 
 define <16 x float> @test_mm512_div_round_ps_current(<16 x float> %a0, <16 x float> %a1, i16 %mask) {
 ; CHECK-LABEL: test_mm512_div_round_ps_current:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vdivps %zmm1, %zmm0, %zmm0
-; CHECK-NEXT:    retq
+; CHECK-NEXT:    ret{{[l|q]}}
   %1 = call <16 x float> @llvm.x86.avx512.div.ps.512(<16 x float> %a0, <16 x float> %a1, i32 4)
   ret <16 x float> %1
 }
 declare <16 x float> @llvm.x86.avx512.div.ps.512(<16 x float>, <16 x float>, i32)
 
 define <16 x float> @test_mm512_maskz_min_round_ps_sae(<16 x float> %a0, <16 x float> %a1, i16 %mask) {
-; CHECK-LABEL: test_mm512_maskz_min_round_ps_sae:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vminps {sae}, %zmm1, %zmm0, %zmm0 {%k1} {z}
-; CHECK-NEXT:    retq
+; X64-LABEL: test_mm512_maskz_min_round_ps_sae:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vminps {sae}, %zmm1, %zmm0, %zmm0 {%k1} {z}
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_mm512_maskz_min_round_ps_sae:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vminps {sae}, %zmm1, %zmm0, %zmm0 {%k1} {z}
+; X86-NEXT:    retl
   %1 = call <16 x float> @llvm.x86.avx512.min.ps.512(<16 x float> %a0, <16 x float> %a1, i32 8)
   %2 = bitcast i16 %mask to <16 x i1>
   %3 = select <16 x i1> %2, <16 x float> %1, <16 x float> zeroinitializer
@@ -1765,11 +2126,17 @@ define <16 x float> @test_mm512_maskz_min_round_ps_sae(<16 x float> %a0, <16 x f
 }
 
 define <16 x float> @test_mm512_maskz_min_round_ps_current(<16 x float> %a0, <16 x float> %a1, i16 %mask) {
-; CHECK-LABEL: test_mm512_maskz_min_round_ps_current:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vminps %zmm1, %zmm0, %zmm0 {%k1} {z}
-; CHECK-NEXT:    retq
+; X64-LABEL: test_mm512_maskz_min_round_ps_current:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vminps %zmm1, %zmm0, %zmm0 {%k1} {z}
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_mm512_maskz_min_round_ps_current:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vminps %zmm1, %zmm0, %zmm0 {%k1} {z}
+; X86-NEXT:    retl
   %1 = call <16 x float> @llvm.x86.avx512.min.ps.512(<16 x float> %a0, <16 x float> %a1, i32 4)
   %2 = bitcast i16 %mask to <16 x i1>
   %3 = select <16 x i1> %2, <16 x float> %1, <16 x float> zeroinitializer
@@ -1777,12 +2144,19 @@ define <16 x float> @test_mm512_maskz_min_round_ps_current(<16 x float> %a0, <16
 }
 
 define <16 x float> @test_mm512_mask_min_round_ps_sae(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask) {
-; CHECK-LABEL: test_mm512_mask_min_round_ps_sae:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vminps {sae}, %zmm1, %zmm0, %zmm2 {%k1}
-; CHECK-NEXT:    vmovaps %zmm2, %zmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_mm512_mask_min_round_ps_sae:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vminps {sae}, %zmm1, %zmm0, %zmm2 {%k1}
+; X64-NEXT:    vmovaps %zmm2, %zmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_mm512_mask_min_round_ps_sae:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vminps {sae}, %zmm1, %zmm0, %zmm2 {%k1}
+; X86-NEXT:    vmovaps %zmm2, %zmm0
+; X86-NEXT:    retl
   %1 = call <16 x float> @llvm.x86.avx512.min.ps.512(<16 x float> %a0, <16 x float> %a1, i32 8)
   %2 = bitcast i16 %mask to <16 x i1>
   %3 = select <16 x i1> %2, <16 x float> %1, <16 x float> %src
@@ -1790,12 +2164,19 @@ define <16 x float> @test_mm512_mask_min_round_ps_sae(<16 x float> %a0, <16 x fl
 }
 
 define <16 x float> @test_mm512_mask_min_round_ps_current(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask) {
-; CHECK-LABEL: test_mm512_mask_min_round_ps_current:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vminps %zmm1, %zmm0, %zmm2 {%k1}
-; CHECK-NEXT:    vmovaps %zmm2, %zmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_mm512_mask_min_round_ps_current:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vminps %zmm1, %zmm0, %zmm2 {%k1}
+; X64-NEXT:    vmovaps %zmm2, %zmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_mm512_mask_min_round_ps_current:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vminps %zmm1, %zmm0, %zmm2 {%k1}
+; X86-NEXT:    vmovaps %zmm2, %zmm0
+; X86-NEXT:    retl
   %1 = call <16 x float> @llvm.x86.avx512.min.ps.512(<16 x float> %a0, <16 x float> %a1, i32 4)
   %2 = bitcast i16 %mask to <16 x i1>
   %3 = select <16 x i1> %2, <16 x float> %1, <16 x float> %src
@@ -1804,29 +2185,35 @@ define <16 x float> @test_mm512_mask_min_round_ps_current(<16 x float> %a0, <16
 
 define <16 x float> @test_mm512_min_round_ps_sae(<16 x float> %a0, <16 x float> %a1, i16 %mask) {
 ; CHECK-LABEL: test_mm512_min_round_ps_sae:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vminps {sae}, %zmm1, %zmm0, %zmm0
-; CHECK-NEXT:    retq
+; CHECK-NEXT:    ret{{[l|q]}}
   %1 = call <16 x float> @llvm.x86.avx512.min.ps.512(<16 x float> %a0, <16 x float> %a1, i32 8)
   ret <16 x float> %1
 }
 
 define <16 x float> @test_mm512_min_round_ps_current(<16 x float> %a0, <16 x float> %a1, i16 %mask) {
 ; CHECK-LABEL: test_mm512_min_round_ps_current:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vminps %zmm1, %zmm0, %zmm0
-; CHECK-NEXT:    retq
+; CHECK-NEXT:    ret{{[l|q]}}
   %1 = call <16 x float> @llvm.x86.avx512.min.ps.512(<16 x float> %a0, <16 x float> %a1, i32 4)
   ret <16 x float> %1
 }
 declare <16 x float> @llvm.x86.avx512.min.ps.512(<16 x float>, <16 x float>, i32)
 
 define <16 x float> @test_mm512_maskz_max_round_ps_sae(<16 x float> %a0, <16 x float> %a1, i16 %mask) {
-; CHECK-LABEL: test_mm512_maskz_max_round_ps_sae:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vmaxps {sae}, %zmm1, %zmm0, %zmm0 {%k1} {z}
-; CHECK-NEXT:    retq
+; X64-LABEL: test_mm512_maskz_max_round_ps_sae:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vmaxps {sae}, %zmm1, %zmm0, %zmm0 {%k1} {z}
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_mm512_maskz_max_round_ps_sae:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vmaxps {sae}, %zmm1, %zmm0, %zmm0 {%k1} {z}
+; X86-NEXT:    retl
   %1 = call <16 x float> @llvm.x86.avx512.max.ps.512(<16 x float> %a0, <16 x float> %a1, i32 8)
   %2 = bitcast i16 %mask to <16 x i1>
   %3 = select <16 x i1> %2, <16 x float> %1, <16 x float> zeroinitializer
@@ -1834,11 +2221,17 @@ define <16 x float> @test_mm512_maskz_max_round_ps_sae(<16 x float> %a0, <16 x f
 }
 
 define <16 x float> @test_mm512_maskz_max_round_ps_current(<16 x float> %a0, <16 x float> %a1, i16 %mask) {
-; CHECK-LABEL: test_mm512_maskz_max_round_ps_current:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vmaxps %zmm1, %zmm0, %zmm0 {%k1} {z}
-; CHECK-NEXT:    retq
+; X64-LABEL: test_mm512_maskz_max_round_ps_current:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vmaxps %zmm1, %zmm0, %zmm0 {%k1} {z}
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_mm512_maskz_max_round_ps_current:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vmaxps %zmm1, %zmm0, %zmm0 {%k1} {z}
+; X86-NEXT:    retl
   %1 = call <16 x float> @llvm.x86.avx512.max.ps.512(<16 x float> %a0, <16 x float> %a1, i32 4)
   %2 = bitcast i16 %mask to <16 x i1>
   %3 = select <16 x i1> %2, <16 x float> %1, <16 x float> zeroinitializer
@@ -1846,12 +2239,19 @@ define <16 x float> @test_mm512_maskz_max_round_ps_current(<16 x float> %a0, <16
 }
 
 define <16 x float> @test_mm512_mask_max_round_ps_sae(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask) {
-; CHECK-LABEL: test_mm512_mask_max_round_ps_sae:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vmaxps {sae}, %zmm1, %zmm0, %zmm2 {%k1}
-; CHECK-NEXT:    vmovaps %zmm2, %zmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_mm512_mask_max_round_ps_sae:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vmaxps {sae}, %zmm1, %zmm0, %zmm2 {%k1}
+; X64-NEXT:    vmovaps %zmm2, %zmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_mm512_mask_max_round_ps_sae:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vmaxps {sae}, %zmm1, %zmm0, %zmm2 {%k1}
+; X86-NEXT:    vmovaps %zmm2, %zmm0
+; X86-NEXT:    retl
   %1 = call <16 x float> @llvm.x86.avx512.max.ps.512(<16 x float> %a0, <16 x float> %a1, i32 8)
   %2 = bitcast i16 %mask to <16 x i1>
   %3 = select <16 x i1> %2, <16 x float> %1, <16 x float> %src
@@ -1859,12 +2259,19 @@ define <16 x float> @test_mm512_mask_max_round_ps_sae(<16 x float> %a0, <16 x fl
 }
 
 define <16 x float> @test_mm512_mask_max_round_ps_current(<16 x float> %a0, <16 x float> %a1, <16 x float> %src, i16 %mask) {
-; CHECK-LABEL: test_mm512_mask_max_round_ps_current:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vmaxps %zmm1, %zmm0, %zmm2 {%k1}
-; CHECK-NEXT:    vmovaps %zmm2, %zmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_mm512_mask_max_round_ps_current:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vmaxps %zmm1, %zmm0, %zmm2 {%k1}
+; X64-NEXT:    vmovaps %zmm2, %zmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_mm512_mask_max_round_ps_current:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vmaxps %zmm1, %zmm0, %zmm2 {%k1}
+; X86-NEXT:    vmovaps %zmm2, %zmm0
+; X86-NEXT:    retl
   %1 = call <16 x float> @llvm.x86.avx512.max.ps.512(<16 x float> %a0, <16 x float> %a1, i32 4)
   %2 = bitcast i16 %mask to <16 x i1>
   %3 = select <16 x i1> %2, <16 x float> %1, <16 x float> %src
@@ -1873,18 +2280,18 @@ define <16 x float> @test_mm512_mask_max_round_ps_current(<16 x float> %a0, <16
 
 define <16 x float> @test_mm512_max_round_ps_sae(<16 x float> %a0, <16 x float> %a1, i16 %mask) {
 ; CHECK-LABEL: test_mm512_max_round_ps_sae:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vmaxps {sae}, %zmm1, %zmm0, %zmm0
-; CHECK-NEXT:    retq
+; CHECK-NEXT:    ret{{[l|q]}}
   %1 = call <16 x float> @llvm.x86.avx512.max.ps.512(<16 x float> %a0, <16 x float> %a1, i32 8)
   ret <16 x float> %1
 }
 
 define <16 x float> @test_mm512_max_round_ps_current(<16 x float> %a0, <16 x float> %a1, i16 %mask) {
 ; CHECK-LABEL: test_mm512_max_round_ps_current:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vmaxps %zmm1, %zmm0, %zmm0
-; CHECK-NEXT:    retq
+; CHECK-NEXT:    ret{{[l|q]}}
   %1 = call <16 x float> @llvm.x86.avx512.max.ps.512(<16 x float> %a0, <16 x float> %a1, i32 4)
   ret <16 x float> %1
 }
@@ -1893,86 +2300,142 @@ declare <16 x float> @llvm.x86.avx512.max.ps.512(<16 x float>, <16 x float>, i32
 declare <4 x float> @llvm.x86.avx512.mask.add.ss.round(<4 x float>, <4 x float>, <4 x float>, i8, i32) nounwind readnone
 
 define <4 x float> @test_mask_add_ss_rn(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, i8 %mask) {
-; CHECK-LABEL: test_mask_add_ss_rn:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vaddss {rn-sae}, %xmm1, %xmm0, %xmm2 {%k1}
-; CHECK-NEXT:    vmovaps %xmm2, %xmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_mask_add_ss_rn:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vaddss {rn-sae}, %xmm1, %xmm0, %xmm2 {%k1}
+; X64-NEXT:    vmovaps %xmm2, %xmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_mask_add_ss_rn:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vaddss {rn-sae}, %xmm1, %xmm0, %xmm2 {%k1}
+; X86-NEXT:    vmovaps %xmm2, %xmm0
+; X86-NEXT:    retl
   %res = call <4 x float> @llvm.x86.avx512.mask.add.ss.round(<4 x float>%a0, <4 x float> %a1, <4 x float> %a2, i8 %mask, i32 0)
   ret <4 x float> %res
 }
 
 define <4 x float> @test_mask_add_ss_rd(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, i8 %mask) {
-; CHECK-LABEL: test_mask_add_ss_rd:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vaddss {rd-sae}, %xmm1, %xmm0, %xmm2 {%k1}
-; CHECK-NEXT:    vmovaps %xmm2, %xmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_mask_add_ss_rd:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vaddss {rd-sae}, %xmm1, %xmm0, %xmm2 {%k1}
+; X64-NEXT:    vmovaps %xmm2, %xmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_mask_add_ss_rd:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vaddss {rd-sae}, %xmm1, %xmm0, %xmm2 {%k1}
+; X86-NEXT:    vmovaps %xmm2, %xmm0
+; X86-NEXT:    retl
   %res = call <4 x float> @llvm.x86.avx512.mask.add.ss.round(<4 x float>%a0, <4 x float> %a1, <4 x float> %a2, i8 %mask, i32 1)
   ret <4 x float> %res
 }
 
 define <4 x float> @test_mask_add_ss_ru(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, i8 %mask) {
-; CHECK-LABEL: test_mask_add_ss_ru:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vaddss {ru-sae}, %xmm1, %xmm0, %xmm2 {%k1}
-; CHECK-NEXT:    vmovaps %xmm2, %xmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_mask_add_ss_ru:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vaddss {ru-sae}, %xmm1, %xmm0, %xmm2 {%k1}
+; X64-NEXT:    vmovaps %xmm2, %xmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_mask_add_ss_ru:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vaddss {ru-sae}, %xmm1, %xmm0, %xmm2 {%k1}
+; X86-NEXT:    vmovaps %xmm2, %xmm0
+; X86-NEXT:    retl
   %res = call <4 x float> @llvm.x86.avx512.mask.add.ss.round(<4 x float>%a0, <4 x float> %a1, <4 x float> %a2, i8 %mask, i32 2)
   ret <4 x float> %res
 }
 
 define <4 x float> @test_mask_add_ss_rz(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, i8 %mask) {
-; CHECK-LABEL: test_mask_add_ss_rz:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vaddss {rz-sae}, %xmm1, %xmm0, %xmm2 {%k1}
-; CHECK-NEXT:    vmovaps %xmm2, %xmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_mask_add_ss_rz:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vaddss {rz-sae}, %xmm1, %xmm0, %xmm2 {%k1}
+; X64-NEXT:    vmovaps %xmm2, %xmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_mask_add_ss_rz:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vaddss {rz-sae}, %xmm1, %xmm0, %xmm2 {%k1}
+; X86-NEXT:    vmovaps %xmm2, %xmm0
+; X86-NEXT:    retl
   %res = call <4 x float> @llvm.x86.avx512.mask.add.ss.round(<4 x float>%a0, <4 x float> %a1, <4 x float> %a2, i8 %mask, i32 3)
   ret <4 x float> %res
 }
 
 define <4 x float> @test_mask_add_ss_current(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, i8 %mask) {
-; CHECK-LABEL: test_mask_add_ss_current:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vaddss %xmm1, %xmm0, %xmm2 {%k1}
-; CHECK-NEXT:    vmovaps %xmm2, %xmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_mask_add_ss_current:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vaddss %xmm1, %xmm0, %xmm2 {%k1}
+; X64-NEXT:    vmovaps %xmm2, %xmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_mask_add_ss_current:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vaddss %xmm1, %xmm0, %xmm2 {%k1}
+; X86-NEXT:    vmovaps %xmm2, %xmm0
+; X86-NEXT:    retl
   %res = call <4 x float> @llvm.x86.avx512.mask.add.ss.round(<4 x float>%a0, <4 x float> %a1, <4 x float> %a2, i8 %mask, i32 4)
   ret <4 x float> %res
 }
 
 define <4 x float> @test_maskz_add_ss_rn(<4 x float> %a0, <4 x float> %a1, i8 %mask) {
-; CHECK-LABEL: test_maskz_add_ss_rn:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vaddss {rn-sae}, %xmm1, %xmm0, %xmm0 {%k1} {z}
-; CHECK-NEXT:    retq
+; X64-LABEL: test_maskz_add_ss_rn:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vaddss {rn-sae}, %xmm1, %xmm0, %xmm0 {%k1} {z}
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_maskz_add_ss_rn:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vaddss {rn-sae}, %xmm1, %xmm0, %xmm0 {%k1} {z}
+; X86-NEXT:    retl
   %res = call <4 x float> @llvm.x86.avx512.mask.add.ss.round(<4 x float>%a0, <4 x float> %a1, <4 x float> zeroinitializer, i8 %mask, i32 0)
   ret <4 x float> %res
 }
 
 define <4 x float> @test_add_ss_rn(<4 x float> %a0, <4 x float> %a1) {
 ; CHECK-LABEL: test_add_ss_rn:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vaddss {rn-sae}, %xmm1, %xmm0, %xmm0
-; CHECK-NEXT:    retq
+; CHECK-NEXT:    ret{{[l|q]}}
   %res = call <4 x float> @llvm.x86.avx512.mask.add.ss.round(<4 x float>%a0, <4 x float> %a1, <4 x float> zeroinitializer, i8 -1, i32 0)
   ret <4 x float> %res
 }
 
 define <4 x float> @test_mask_add_ss_current_memfold(<4 x float> %a0, float* %a1, <4 x float> %a2, i8 %mask) {
-; CHECK-LABEL: test_mask_add_ss_current_memfold:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %esi, %k1
-; CHECK-NEXT:    vaddss (%rdi), %xmm0, %xmm1 {%k1}
-; CHECK-NEXT:    vmovaps %xmm1, %xmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_mask_add_ss_current_memfold:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %esi, %k1
+; X64-NEXT:    vaddss (%rdi), %xmm0, %xmm1 {%k1}
+; X64-NEXT:    vmovaps %xmm1, %xmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_mask_add_ss_current_memfold:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    vaddss (%eax), %xmm0, %xmm1 {%k1}
+; X86-NEXT:    vmovaps %xmm1, %xmm0
+; X86-NEXT:    retl
   %a1.val = load float, float* %a1
   %a1v0 = insertelement <4 x float> undef, float %a1.val, i32 0
   %a1v1 = insertelement <4 x float> %a1v0, float 0.000000e+00, i32 1
@@ -1983,11 +2446,19 @@ define <4 x float> @test_mask_add_ss_current_memfold(<4 x float> %a0, float* %a1
 }
 
 define <4 x float> @test_maskz_add_ss_current_memfold(<4 x float> %a0, float* %a1, i8 %mask) {
-; CHECK-LABEL: test_maskz_add_ss_current_memfold:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %esi, %k1
-; CHECK-NEXT:    vaddss (%rdi), %xmm0, %xmm0 {%k1} {z}
-; CHECK-NEXT:    retq
+; X64-LABEL: test_maskz_add_ss_current_memfold:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %esi, %k1
+; X64-NEXT:    vaddss (%rdi), %xmm0, %xmm0 {%k1} {z}
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_maskz_add_ss_current_memfold:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    vaddss (%eax), %xmm0, %xmm0 {%k1} {z}
+; X86-NEXT:    retl
   %a1.val = load float, float* %a1
   %a1v0 = insertelement <4 x float> undef, float %a1.val, i32 0
   %a1v1 = insertelement <4 x float> %a1v0, float 0.000000e+00, i32 1
@@ -2000,86 +2471,142 @@ define <4 x float> @test_maskz_add_ss_current_memfold(<4 x float> %a0, float* %a
 declare <2 x double> @llvm.x86.avx512.mask.add.sd.round(<2 x double>, <2 x double>, <2 x double>, i8, i32) nounwind readnone
 
 define <2 x double> @test_mask_add_sd_rn(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, i8 %mask) {
-; CHECK-LABEL: test_mask_add_sd_rn:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vaddsd {rn-sae}, %xmm1, %xmm0, %xmm2 {%k1}
-; CHECK-NEXT:    vmovapd %xmm2, %xmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_mask_add_sd_rn:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vaddsd {rn-sae}, %xmm1, %xmm0, %xmm2 {%k1}
+; X64-NEXT:    vmovapd %xmm2, %xmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_mask_add_sd_rn:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vaddsd {rn-sae}, %xmm1, %xmm0, %xmm2 {%k1}
+; X86-NEXT:    vmovapd %xmm2, %xmm0
+; X86-NEXT:    retl
   %res = call <2 x double> @llvm.x86.avx512.mask.add.sd.round(<2 x double>%a0, <2 x double> %a1, <2 x double> %a2, i8 %mask, i32 0)
   ret <2 x double> %res
 }
 
 define <2 x double> @test_mask_add_sd_rd(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, i8 %mask) {
-; CHECK-LABEL: test_mask_add_sd_rd:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vaddsd {rd-sae}, %xmm1, %xmm0, %xmm2 {%k1}
-; CHECK-NEXT:    vmovapd %xmm2, %xmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_mask_add_sd_rd:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vaddsd {rd-sae}, %xmm1, %xmm0, %xmm2 {%k1}
+; X64-NEXT:    vmovapd %xmm2, %xmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_mask_add_sd_rd:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vaddsd {rd-sae}, %xmm1, %xmm0, %xmm2 {%k1}
+; X86-NEXT:    vmovapd %xmm2, %xmm0
+; X86-NEXT:    retl
   %res = call <2 x double> @llvm.x86.avx512.mask.add.sd.round(<2 x double>%a0, <2 x double> %a1, <2 x double> %a2, i8 %mask, i32 1)
   ret <2 x double> %res
 }
 
 define <2 x double> @test_mask_add_sd_ru(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, i8 %mask) {
-; CHECK-LABEL: test_mask_add_sd_ru:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vaddsd {ru-sae}, %xmm1, %xmm0, %xmm2 {%k1}
-; CHECK-NEXT:    vmovapd %xmm2, %xmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_mask_add_sd_ru:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vaddsd {ru-sae}, %xmm1, %xmm0, %xmm2 {%k1}
+; X64-NEXT:    vmovapd %xmm2, %xmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_mask_add_sd_ru:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vaddsd {ru-sae}, %xmm1, %xmm0, %xmm2 {%k1}
+; X86-NEXT:    vmovapd %xmm2, %xmm0
+; X86-NEXT:    retl
   %res = call <2 x double> @llvm.x86.avx512.mask.add.sd.round(<2 x double>%a0, <2 x double> %a1, <2 x double> %a2, i8 %mask, i32 2)
   ret <2 x double> %res
 }
 
 define <2 x double> @test_mask_add_sd_rz(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, i8 %mask) {
-; CHECK-LABEL: test_mask_add_sd_rz:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vaddsd {rz-sae}, %xmm1, %xmm0, %xmm2 {%k1}
-; CHECK-NEXT:    vmovapd %xmm2, %xmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_mask_add_sd_rz:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vaddsd {rz-sae}, %xmm1, %xmm0, %xmm2 {%k1}
+; X64-NEXT:    vmovapd %xmm2, %xmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_mask_add_sd_rz:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vaddsd {rz-sae}, %xmm1, %xmm0, %xmm2 {%k1}
+; X86-NEXT:    vmovapd %xmm2, %xmm0
+; X86-NEXT:    retl
   %res = call <2 x double> @llvm.x86.avx512.mask.add.sd.round(<2 x double>%a0, <2 x double> %a1, <2 x double> %a2, i8 %mask, i32 3)
   ret <2 x double> %res
 }
 
 define <2 x double> @test_mask_add_sd_current(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, i8 %mask) {
-; CHECK-LABEL: test_mask_add_sd_current:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vaddsd %xmm1, %xmm0, %xmm2 {%k1}
-; CHECK-NEXT:    vmovapd %xmm2, %xmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_mask_add_sd_current:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vaddsd %xmm1, %xmm0, %xmm2 {%k1}
+; X64-NEXT:    vmovapd %xmm2, %xmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_mask_add_sd_current:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vaddsd %xmm1, %xmm0, %xmm2 {%k1}
+; X86-NEXT:    vmovapd %xmm2, %xmm0
+; X86-NEXT:    retl
   %res = call <2 x double> @llvm.x86.avx512.mask.add.sd.round(<2 x double>%a0, <2 x double> %a1, <2 x double> %a2, i8 %mask, i32 4)
   ret <2 x double> %res
 }
 
 define <2 x double> @test_maskz_add_sd_rn(<2 x double> %a0, <2 x double> %a1, i8 %mask) {
-; CHECK-LABEL: test_maskz_add_sd_rn:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vaddsd {rn-sae}, %xmm1, %xmm0, %xmm0 {%k1} {z}
-; CHECK-NEXT:    retq
+; X64-LABEL: test_maskz_add_sd_rn:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vaddsd {rn-sae}, %xmm1, %xmm0, %xmm0 {%k1} {z}
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_maskz_add_sd_rn:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vaddsd {rn-sae}, %xmm1, %xmm0, %xmm0 {%k1} {z}
+; X86-NEXT:    retl
   %res = call <2 x double> @llvm.x86.avx512.mask.add.sd.round(<2 x double>%a0, <2 x double> %a1, <2 x double> zeroinitializer, i8 %mask, i32 0)
   ret <2 x double> %res
 }
 
 define <2 x double> @test_add_sd_rn(<2 x double> %a0, <2 x double> %a1) {
 ; CHECK-LABEL: test_add_sd_rn:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vaddsd {rn-sae}, %xmm1, %xmm0, %xmm0
-; CHECK-NEXT:    retq
+; CHECK-NEXT:    ret{{[l|q]}}
   %res = call <2 x double> @llvm.x86.avx512.mask.add.sd.round(<2 x double>%a0, <2 x double> %a1, <2 x double> zeroinitializer, i8 -1, i32 0)
   ret <2 x double> %res
 }
 
 define <2 x double> @test_mask_add_sd_current_memfold(<2 x double> %a0, double* %a1, <2 x double> %a2, i8 %mask) {
-; CHECK-LABEL: test_mask_add_sd_current_memfold:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %esi, %k1
-; CHECK-NEXT:    vaddsd (%rdi), %xmm0, %xmm1 {%k1}
-; CHECK-NEXT:    vmovapd %xmm1, %xmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_mask_add_sd_current_memfold:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %esi, %k1
+; X64-NEXT:    vaddsd (%rdi), %xmm0, %xmm1 {%k1}
+; X64-NEXT:    vmovapd %xmm1, %xmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_mask_add_sd_current_memfold:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    vaddsd (%eax), %xmm0, %xmm1 {%k1}
+; X86-NEXT:    vmovapd %xmm1, %xmm0
+; X86-NEXT:    retl
   %a1.val = load double, double* %a1
   %a1v0 = insertelement <2 x double> undef, double %a1.val, i32 0
   %a1v = insertelement <2 x double> %a1v0, double 0.000000e+00, i32 1
@@ -2088,11 +2615,19 @@ define <2 x double> @test_mask_add_sd_current_memfold(<2 x double> %a0, double*
 }
 
 define <2 x double> @test_maskz_add_sd_current_memfold(<2 x double> %a0, double* %a1, i8 %mask) {
-; CHECK-LABEL: test_maskz_add_sd_current_memfold:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %esi, %k1
-; CHECK-NEXT:    vaddsd (%rdi), %xmm0, %xmm0 {%k1} {z}
-; CHECK-NEXT:    retq
+; X64-LABEL: test_maskz_add_sd_current_memfold:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %esi, %k1
+; X64-NEXT:    vaddsd (%rdi), %xmm0, %xmm0 {%k1} {z}
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_maskz_add_sd_current_memfold:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    vaddsd (%eax), %xmm0, %xmm0 {%k1} {z}
+; X86-NEXT:    retl
   %a1.val = load double, double* %a1
   %a1v0 = insertelement <2 x double> undef, double %a1.val, i32 0
   %a1v = insertelement <2 x double> %a1v0, double 0.000000e+00, i32 1
@@ -2103,72 +2638,111 @@ define <2 x double> @test_maskz_add_sd_current_memfold(<2 x double> %a0, double*
 declare <4 x float> @llvm.x86.avx512.mask.max.ss.round(<4 x float>, <4 x float>, <4 x float>, i8, i32) nounwind readnone
 
 define <4 x float> @test_mask_max_ss_sae(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, i8 %mask) {
-; CHECK-LABEL: test_mask_max_ss_sae:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vmaxss {sae}, %xmm1, %xmm0, %xmm2 {%k1}
-; CHECK-NEXT:    vmovaps %xmm2, %xmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_mask_max_ss_sae:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vmaxss {sae}, %xmm1, %xmm0, %xmm2 {%k1}
+; X64-NEXT:    vmovaps %xmm2, %xmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_mask_max_ss_sae:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vmaxss {sae}, %xmm1, %xmm0, %xmm2 {%k1}
+; X86-NEXT:    vmovaps %xmm2, %xmm0
+; X86-NEXT:    retl
   %res = call <4 x float> @llvm.x86.avx512.mask.max.ss.round(<4 x float>%a0, <4 x float> %a1, <4 x float> %a2, i8 %mask, i32 8)
   ret <4 x float> %res
 }
 
 define <4 x float> @test_maskz_max_ss_sae(<4 x float> %a0, <4 x float> %a1, i8 %mask) {
-; CHECK-LABEL: test_maskz_max_ss_sae:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vmaxss {sae}, %xmm1, %xmm0, %xmm0 {%k1} {z}
-; CHECK-NEXT:    retq
+; X64-LABEL: test_maskz_max_ss_sae:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vmaxss {sae}, %xmm1, %xmm0, %xmm0 {%k1} {z}
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_maskz_max_ss_sae:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vmaxss {sae}, %xmm1, %xmm0, %xmm0 {%k1} {z}
+; X86-NEXT:    retl
   %res = call <4 x float> @llvm.x86.avx512.mask.max.ss.round(<4 x float>%a0, <4 x float> %a1, <4 x float> zeroinitializer, i8 %mask, i32 8)
   ret <4 x float> %res
 }
 
 define <4 x float> @test_max_ss_sae(<4 x float> %a0, <4 x float> %a1) {
 ; CHECK-LABEL: test_max_ss_sae:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vmaxss {sae}, %xmm1, %xmm0, %xmm0
-; CHECK-NEXT:    retq
+; CHECK-NEXT:    ret{{[l|q]}}
   %res = call <4 x float> @llvm.x86.avx512.mask.max.ss.round(<4 x float>%a0, <4 x float> %a1, <4 x float> zeroinitializer, i8 -1, i32 8)
   ret <4 x float> %res
 }
 
 define <4 x float> @test_mask_max_ss(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, i8 %mask) {
-; CHECK-LABEL: test_mask_max_ss:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vmaxss %xmm1, %xmm0, %xmm2 {%k1}
-; CHECK-NEXT:    vmovaps %xmm2, %xmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_mask_max_ss:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vmaxss %xmm1, %xmm0, %xmm2 {%k1}
+; X64-NEXT:    vmovaps %xmm2, %xmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_mask_max_ss:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vmaxss %xmm1, %xmm0, %xmm2 {%k1}
+; X86-NEXT:    vmovaps %xmm2, %xmm0
+; X86-NEXT:    retl
   %res = call <4 x float> @llvm.x86.avx512.mask.max.ss.round(<4 x float>%a0, <4 x float> %a1, <4 x float> %a2, i8 %mask, i32 4)
   ret <4 x float> %res
 }
 
 define <4 x float> @test_maskz_max_ss(<4 x float> %a0, <4 x float> %a1, i8 %mask) {
-; CHECK-LABEL: test_maskz_max_ss:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vmaxss %xmm1, %xmm0, %xmm0 {%k1} {z}
-; CHECK-NEXT:    retq
+; X64-LABEL: test_maskz_max_ss:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vmaxss %xmm1, %xmm0, %xmm0 {%k1} {z}
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_maskz_max_ss:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vmaxss %xmm1, %xmm0, %xmm0 {%k1} {z}
+; X86-NEXT:    retl
   %res = call <4 x float> @llvm.x86.avx512.mask.max.ss.round(<4 x float>%a0, <4 x float> %a1, <4 x float> zeroinitializer, i8 %mask, i32 4)
   ret <4 x float> %res
 }
 
 define <4 x float> @test_max_ss(<4 x float> %a0, <4 x float> %a1) {
 ; CHECK-LABEL: test_max_ss:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vmaxss %xmm1, %xmm0, %xmm0
-; CHECK-NEXT:    retq
+; CHECK-NEXT:    ret{{[l|q]}}
   %res = call <4 x float> @llvm.x86.avx512.mask.max.ss.round(<4 x float>%a0, <4 x float> %a1, <4 x float> zeroinitializer, i8 -1, i32 4)
   ret <4 x float> %res
 }
 
 define <4 x float> @test_mask_max_ss_memfold(<4 x float> %a0, float* %a1, <4 x float> %a2, i8 %mask) {
-; CHECK-LABEL: test_mask_max_ss_memfold:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %esi, %k1
-; CHECK-NEXT:    vmaxss (%rdi), %xmm0, %xmm1 {%k1}
-; CHECK-NEXT:    vmovaps %xmm1, %xmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_mask_max_ss_memfold:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %esi, %k1
+; X64-NEXT:    vmaxss (%rdi), %xmm0, %xmm1 {%k1}
+; X64-NEXT:    vmovaps %xmm1, %xmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_mask_max_ss_memfold:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    vmaxss (%eax), %xmm0, %xmm1 {%k1}
+; X86-NEXT:    vmovaps %xmm1, %xmm0
+; X86-NEXT:    retl
   %a1.val = load float, float* %a1
   %a1v0 = insertelement <4 x float> undef, float %a1.val, i32 0
   %a1v1 = insertelement <4 x float> %a1v0, float 0.000000e+00, i32 1
@@ -2179,11 +2753,19 @@ define <4 x float> @test_mask_max_ss_memfold(<4 x float> %a0, float* %a1, <4 x f
 }
 
 define <4 x float> @test_maskz_max_ss_memfold(<4 x float> %a0, float* %a1, i8 %mask) {
-; CHECK-LABEL: test_maskz_max_ss_memfold:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %esi, %k1
-; CHECK-NEXT:    vmaxss (%rdi), %xmm0, %xmm0 {%k1} {z}
-; CHECK-NEXT:    retq
+; X64-LABEL: test_maskz_max_ss_memfold:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %esi, %k1
+; X64-NEXT:    vmaxss (%rdi), %xmm0, %xmm0 {%k1} {z}
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_maskz_max_ss_memfold:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    vmaxss (%eax), %xmm0, %xmm0 {%k1} {z}
+; X86-NEXT:    retl
   %a1.val = load float, float* %a1
   %a1v0 = insertelement <4 x float> undef, float %a1.val, i32 0
   %a1v1 = insertelement <4 x float> %a1v0, float 0.000000e+00, i32 1
@@ -2195,72 +2777,111 @@ define <4 x float> @test_maskz_max_ss_memfold(<4 x float> %a0, float* %a1, i8 %m
 declare <2 x double> @llvm.x86.avx512.mask.max.sd.round(<2 x double>, <2 x double>, <2 x double>, i8, i32) nounwind readnone
 
 define <2 x double> @test_mask_max_sd_sae(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, i8 %mask) {
-; CHECK-LABEL: test_mask_max_sd_sae:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vmaxsd {sae}, %xmm1, %xmm0, %xmm2 {%k1}
-; CHECK-NEXT:    vmovapd %xmm2, %xmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_mask_max_sd_sae:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vmaxsd {sae}, %xmm1, %xmm0, %xmm2 {%k1}
+; X64-NEXT:    vmovapd %xmm2, %xmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_mask_max_sd_sae:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vmaxsd {sae}, %xmm1, %xmm0, %xmm2 {%k1}
+; X86-NEXT:    vmovapd %xmm2, %xmm0
+; X86-NEXT:    retl
   %res = call <2 x double> @llvm.x86.avx512.mask.max.sd.round(<2 x double>%a0, <2 x double> %a1, <2 x double> %a2, i8 %mask, i32 8)
   ret <2 x double> %res
 }
 
 define <2 x double> @test_maskz_max_sd_sae(<2 x double> %a0, <2 x double> %a1, i8 %mask) {
-; CHECK-LABEL: test_maskz_max_sd_sae:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vmaxsd {sae}, %xmm1, %xmm0, %xmm0 {%k1} {z}
-; CHECK-NEXT:    retq
+; X64-LABEL: test_maskz_max_sd_sae:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vmaxsd {sae}, %xmm1, %xmm0, %xmm0 {%k1} {z}
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_maskz_max_sd_sae:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vmaxsd {sae}, %xmm1, %xmm0, %xmm0 {%k1} {z}
+; X86-NEXT:    retl
   %res = call <2 x double> @llvm.x86.avx512.mask.max.sd.round(<2 x double>%a0, <2 x double> %a1, <2 x double> zeroinitializer, i8 %mask, i32 8)
   ret <2 x double> %res
 }
 
 define <2 x double> @test_max_sd_sae(<2 x double> %a0, <2 x double> %a1) {
 ; CHECK-LABEL: test_max_sd_sae:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vmaxsd {sae}, %xmm1, %xmm0, %xmm0
-; CHECK-NEXT:    retq
+; CHECK-NEXT:    ret{{[l|q]}}
   %res = call <2 x double> @llvm.x86.avx512.mask.max.sd.round(<2 x double>%a0, <2 x double> %a1, <2 x double> zeroinitializer, i8 -1, i32 8)
   ret <2 x double> %res
 }
 
 define <2 x double> @test_mask_max_sd(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, i8 %mask) {
-; CHECK-LABEL: test_mask_max_sd:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vmaxsd %xmm1, %xmm0, %xmm2 {%k1}
-; CHECK-NEXT:    vmovapd %xmm2, %xmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_mask_max_sd:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vmaxsd %xmm1, %xmm0, %xmm2 {%k1}
+; X64-NEXT:    vmovapd %xmm2, %xmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_mask_max_sd:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vmaxsd %xmm1, %xmm0, %xmm2 {%k1}
+; X86-NEXT:    vmovapd %xmm2, %xmm0
+; X86-NEXT:    retl
   %res = call <2 x double> @llvm.x86.avx512.mask.max.sd.round(<2 x double>%a0, <2 x double> %a1, <2 x double> %a2, i8 %mask, i32 4)
   ret <2 x double> %res
 }
 
 define <2 x double> @test_maskz_max_sd(<2 x double> %a0, <2 x double> %a1, i8 %mask) {
-; CHECK-LABEL: test_maskz_max_sd:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vmaxsd %xmm1, %xmm0, %xmm0 {%k1} {z}
-; CHECK-NEXT:    retq
+; X64-LABEL: test_maskz_max_sd:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vmaxsd %xmm1, %xmm0, %xmm0 {%k1} {z}
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_maskz_max_sd:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vmaxsd %xmm1, %xmm0, %xmm0 {%k1} {z}
+; X86-NEXT:    retl
   %res = call <2 x double> @llvm.x86.avx512.mask.max.sd.round(<2 x double>%a0, <2 x double> %a1, <2 x double> zeroinitializer, i8 %mask, i32 4)
   ret <2 x double> %res
 }
 
 define <2 x double> @test_max_sd(<2 x double> %a0, <2 x double> %a1) {
 ; CHECK-LABEL: test_max_sd:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vmaxsd %xmm1, %xmm0, %xmm0
-; CHECK-NEXT:    retq
+; CHECK-NEXT:    ret{{[l|q]}}
   %res = call <2 x double> @llvm.x86.avx512.mask.max.sd.round(<2 x double>%a0, <2 x double> %a1, <2 x double> zeroinitializer, i8 -1, i32 4)
   ret <2 x double> %res
 }
 
 define <2 x double> @test_mask_max_sd_memfold(<2 x double> %a0, double* %a1, <2 x double> %a2, i8 %mask) {
-; CHECK-LABEL: test_mask_max_sd_memfold:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %esi, %k1
-; CHECK-NEXT:    vmaxsd (%rdi), %xmm0, %xmm1 {%k1}
-; CHECK-NEXT:    vmovapd %xmm1, %xmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_mask_max_sd_memfold:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %esi, %k1
+; X64-NEXT:    vmaxsd (%rdi), %xmm0, %xmm1 {%k1}
+; X64-NEXT:    vmovapd %xmm1, %xmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_mask_max_sd_memfold:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    vmaxsd (%eax), %xmm0, %xmm1 {%k1}
+; X86-NEXT:    vmovapd %xmm1, %xmm0
+; X86-NEXT:    retl
   %a1.val = load double, double* %a1
   %a1v0 = insertelement <2 x double> undef, double %a1.val, i32 0
   %a1v = insertelement <2 x double> %a1v0, double 0.000000e+00, i32 1
@@ -2269,11 +2890,19 @@ define <2 x double> @test_mask_max_sd_memfold(<2 x double> %a0, double* %a1, <2
 }
 
 define <2 x double> @test_maskz_max_sd_memfold(<2 x double> %a0, double* %a1, i8 %mask) {
-; CHECK-LABEL: test_maskz_max_sd_memfold:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %esi, %k1
-; CHECK-NEXT:    vmaxsd (%rdi), %xmm0, %xmm0 {%k1} {z}
-; CHECK-NEXT:    retq
+; X64-LABEL: test_maskz_max_sd_memfold:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %esi, %k1
+; X64-NEXT:    vmaxsd (%rdi), %xmm0, %xmm0 {%k1} {z}
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_maskz_max_sd_memfold:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    vmaxsd (%eax), %xmm0, %xmm0 {%k1} {z}
+; X86-NEXT:    retl
   %a1.val = load double, double* %a1
   %a1v0 = insertelement <2 x double> undef, double %a1.val, i32 0
   %a1v = insertelement <2 x double> %a1v0, double 0.000000e+00, i32 1
@@ -2281,133 +2910,107 @@ define <2 x double> @test_maskz_max_sd_memfold(<2 x double> %a0, double* %a1, i8
   ret <2 x double> %res
 }
 
-define <2 x double> @test_x86_avx512_cvtsi2sd64(<2 x double> %a, i64 %b) {
-; CHECK-LABEL: test_x86_avx512_cvtsi2sd64:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    vcvtsi2sdq %rdi, {rz-sae}, %xmm0, %xmm0
-; CHECK-NEXT:    retq
-  %res = call <2 x double> @llvm.x86.avx512.cvtsi2sd64(<2 x double> %a, i64 %b, i32 3) ; <<<2 x double>> [#uses=1]
-  ret <2 x double> %res
-}
-declare <2 x double> @llvm.x86.avx512.cvtsi2sd64(<2 x double>, i64, i32) nounwind readnone
-
 define <4 x float> @test_x86_avx512_cvtsi2ss32(<4 x float> %a, i32 %b) {
-; CHECK-LABEL: test_x86_avx512_cvtsi2ss32:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    vcvtsi2ssl %edi, {rz-sae}, %xmm0, %xmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_x86_avx512_cvtsi2ss32:
+; X64:       # %bb.0:
+; X64-NEXT:    vcvtsi2ssl %edi, {rz-sae}, %xmm0, %xmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_x86_avx512_cvtsi2ss32:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    vcvtsi2ssl %eax, {rz-sae}, %xmm0, %xmm0
+; X86-NEXT:    retl
   %res = call <4 x float> @llvm.x86.avx512.cvtsi2ss32(<4 x float> %a, i32 %b, i32 3) ; <<<4 x float>> [#uses=1]
   ret <4 x float> %res
 }
 declare <4 x float> @llvm.x86.avx512.cvtsi2ss32(<4 x float>, i32, i32) nounwind readnone
 
-define <4 x float> @test_x86_avx512_cvtsi2ss64(<4 x float> %a, i64 %b) {
-; CHECK-LABEL: test_x86_avx512_cvtsi2ss64:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    vcvtsi2ssq %rdi, {rz-sae}, %xmm0, %xmm0
-; CHECK-NEXT:    retq
-  %res = call <4 x float> @llvm.x86.avx512.cvtsi2ss64(<4 x float> %a, i64 %b, i32 3) ; <<<4 x float>> [#uses=1]
-  ret <4 x float> %res
-}
-declare <4 x float> @llvm.x86.avx512.cvtsi2ss64(<4 x float>, i64, i32) nounwind readnone
-
-define <4 x float> @test_x86_avx512__mm_cvt_roundu32_ss (<4 x float> %a, i32 %b)
-; CHECK-LABEL: test_x86_avx512__mm_cvt_roundu32_ss:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    vcvtusi2ssl %edi, {rd-sae}, %xmm0, %xmm0
-; CHECK-NEXT:    retq
-{
+define <4 x float> @test_x86_avx512__mm_cvt_roundu32_ss (<4 x float> %a, i32 %b) {
+; X64-LABEL: test_x86_avx512__mm_cvt_roundu32_ss:
+; X64:       # %bb.0:
+; X64-NEXT:    vcvtusi2ssl %edi, {rd-sae}, %xmm0, %xmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_x86_avx512__mm_cvt_roundu32_ss:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    vcvtusi2ssl %eax, {rd-sae}, %xmm0, %xmm0
+; X86-NEXT:    retl
   %res = call <4 x float> @llvm.x86.avx512.cvtusi2ss(<4 x float> %a, i32 %b, i32 1) ; <<<4 x float>> [#uses=1]
   ret <4 x float> %res
 }
 
-define <4 x float> @test_x86_avx512__mm_cvt_roundu32_ss_mem(<4 x float> %a, i32* %ptr)
-; CHECK-LABEL: test_x86_avx512__mm_cvt_roundu32_ss_mem:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    movl (%rdi), %eax
-; CHECK-NEXT:    vcvtusi2ssl %eax, {rd-sae}, %xmm0, %xmm0
-; CHECK-NEXT:    retq
-{
+define <4 x float> @test_x86_avx512__mm_cvt_roundu32_ss_mem(<4 x float> %a, i32* %ptr) {
+; X64-LABEL: test_x86_avx512__mm_cvt_roundu32_ss_mem:
+; X64:       # %bb.0:
+; X64-NEXT:    movl (%rdi), %eax
+; X64-NEXT:    vcvtusi2ssl %eax, {rd-sae}, %xmm0, %xmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_x86_avx512__mm_cvt_roundu32_ss_mem:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl (%eax), %eax
+; X86-NEXT:    vcvtusi2ssl %eax, {rd-sae}, %xmm0, %xmm0
+; X86-NEXT:    retl
   %b = load i32, i32* %ptr
   %res = call <4 x float> @llvm.x86.avx512.cvtusi2ss(<4 x float> %a, i32 %b, i32 1) ; <<<4 x float>> [#uses=1]
   ret <4 x float> %res
 }
 
-define <4 x float> @test_x86_avx512__mm_cvtu32_ss(<4 x float> %a, i32 %b)
-; CHECK-LABEL: test_x86_avx512__mm_cvtu32_ss:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    vcvtusi2ssl %edi, %xmm0, %xmm0
-; CHECK-NEXT:    retq
-{
+define <4 x float> @test_x86_avx512__mm_cvtu32_ss(<4 x float> %a, i32 %b) {
+; X64-LABEL: test_x86_avx512__mm_cvtu32_ss:
+; X64:       # %bb.0:
+; X64-NEXT:    vcvtusi2ssl %edi, %xmm0, %xmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_x86_avx512__mm_cvtu32_ss:
+; X86:       # %bb.0:
+; X86-NEXT:    vcvtusi2ssl {{[0-9]+}}(%esp), %xmm0, %xmm0
+; X86-NEXT:    retl
   %res = call <4 x float> @llvm.x86.avx512.cvtusi2ss(<4 x float> %a, i32 %b, i32 4) ; <<<4 x float>> [#uses=1]
   ret <4 x float> %res
 }
 
-define <4 x float> @test_x86_avx512__mm_cvtu32_ss_mem(<4 x float> %a, i32* %ptr)
-; CHECK-LABEL: test_x86_avx512__mm_cvtu32_ss_mem:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    vcvtusi2ssl (%rdi), %xmm0, %xmm0
-; CHECK-NEXT:    retq
-{
+define <4 x float> @test_x86_avx512__mm_cvtu32_ss_mem(<4 x float> %a, i32* %ptr) {
+; X64-LABEL: test_x86_avx512__mm_cvtu32_ss_mem:
+; X64:       # %bb.0:
+; X64-NEXT:    vcvtusi2ssl (%rdi), %xmm0, %xmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_x86_avx512__mm_cvtu32_ss_mem:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    vcvtusi2ssl (%eax), %xmm0, %xmm0
+; X86-NEXT:    retl
   %b = load i32, i32* %ptr
   %res = call <4 x float> @llvm.x86.avx512.cvtusi2ss(<4 x float> %a, i32 %b, i32 4) ; <<<4 x float>> [#uses=1]
   ret <4 x float> %res
 }
 declare <4 x float> @llvm.x86.avx512.cvtusi2ss(<4 x float>, i32, i32) nounwind readnone
 
-define <4 x float> @_mm_cvt_roundu64_ss (<4 x float> %a, i64 %b)
-; CHECK-LABEL: _mm_cvt_roundu64_ss:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    vcvtusi2ssq %rdi, {rd-sae}, %xmm0, %xmm0
-; CHECK-NEXT:    retq
-{
-  %res = call <4 x float> @llvm.x86.avx512.cvtusi642ss(<4 x float> %a, i64 %b, i32 1) ; <<<4 x float>> [#uses=1]
-  ret <4 x float> %res
-}
-
-define <4 x float> @_mm_cvtu64_ss(<4 x float> %a, i64 %b)
-; CHECK-LABEL: _mm_cvtu64_ss:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    vcvtusi2ssq %rdi, %xmm0, %xmm0
-; CHECK-NEXT:    retq
-{
-  %res = call <4 x float> @llvm.x86.avx512.cvtusi642ss(<4 x float> %a, i64 %b, i32 4) ; <<<4 x float>> [#uses=1]
-  ret <4 x float> %res
-}
-declare <4 x float> @llvm.x86.avx512.cvtusi642ss(<4 x float>, i64, i32) nounwind readnone
-
-define <2 x double> @test_x86_avx512_mm_cvtu64_sd(<2 x double> %a, i64 %b)
-; CHECK-LABEL: test_x86_avx512_mm_cvtu64_sd:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    vcvtusi2sdq %rdi, {rd-sae}, %xmm0, %xmm0
-; CHECK-NEXT:    retq
-{
-  %res = call <2 x double> @llvm.x86.avx512.cvtusi642sd(<2 x double> %a, i64 %b, i32 1) ; <<<2 x double>> [#uses=1]
-  ret <2 x double> %res
-}
-
-define <2 x double> @test_x86_avx512__mm_cvt_roundu64_sd(<2 x double> %a, i64 %b)
-; CHECK-LABEL: test_x86_avx512__mm_cvt_roundu64_sd:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    vcvtusi2sdq %rdi, %xmm0, %xmm0
-; CHECK-NEXT:    retq
-{
-  %res = call <2 x double> @llvm.x86.avx512.cvtusi642sd(<2 x double> %a, i64 %b, i32 4) ; <<<2 x double>> [#uses=1]
-  ret <2 x double> %res
-}
-declare <2 x double> @llvm.x86.avx512.cvtusi642sd(<2 x double>, i64, i32) nounwind readnone
-
 declare <16 x i32> @llvm.x86.avx512.vpermi2var.d.512(<16 x i32>, <16 x i32>, <16 x i32>)
 
 define <16 x i32>@test_int_x86_avx512_mask_vpermi2var_d_512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32>* %x2p, <16 x i32> %x4, i16 %x3) {
-; CHECK-LABEL: test_int_x86_avx512_mask_vpermi2var_d_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %esi, %k1
-; CHECK-NEXT:    vmovdqa64 %zmm1, %zmm3
-; CHECK-NEXT:    vpermi2d (%rdi), %zmm0, %zmm3 {%k1}
-; CHECK-NEXT:    vpermt2d %zmm2, %zmm1, %zmm0
-; CHECK-NEXT:    vpaddd %zmm0, %zmm3, %zmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_int_x86_avx512_mask_vpermi2var_d_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %esi, %k1
+; X64-NEXT:    vmovdqa64 %zmm1, %zmm3
+; X64-NEXT:    vpermi2d (%rdi), %zmm0, %zmm3 {%k1}
+; X64-NEXT:    vpermt2d %zmm2, %zmm1, %zmm0
+; X64-NEXT:    vpaddd %zmm0, %zmm3, %zmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_int_x86_avx512_mask_vpermi2var_d_512:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vmovdqa64 %zmm1, %zmm3
+; X86-NEXT:    vpermi2d (%eax), %zmm0, %zmm3 {%k1}
+; X86-NEXT:    vpermt2d %zmm2, %zmm1, %zmm0
+; X86-NEXT:    vpaddd %zmm0, %zmm3, %zmm0
+; X86-NEXT:    retl
   %x2 = load <16 x i32>, <16 x i32>* %x2p
   %1 = call <16 x i32> @llvm.x86.avx512.vpermi2var.d.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2)
   %2 = bitcast i16 %x3 to <16 x i1>
@@ -2420,14 +3023,24 @@ define <16 x i32>@test_int_x86_avx512_mask_vpermi2var_d_512(<16 x i32> %x0, <16
 declare <8 x double> @llvm.x86.avx512.vpermi2var.pd.512(<8 x double>, <8 x i64>, <8 x double>)
 
 define <8 x double>@test_int_x86_avx512_mask_vpermi2var_pd_512(<8 x double> %x0, <8 x i64> %x1, <8 x double> %x2, i8 %x3) {
-; CHECK-LABEL: test_int_x86_avx512_mask_vpermi2var_pd_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    vmovapd %zmm0, %zmm3
-; CHECK-NEXT:    vpermt2pd %zmm2, %zmm1, %zmm3
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vpermi2pd %zmm2, %zmm0, %zmm1 {%k1}
-; CHECK-NEXT:    vaddpd %zmm3, %zmm1, %zmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_int_x86_avx512_mask_vpermi2var_pd_512:
+; X64:       # %bb.0:
+; X64-NEXT:    vmovapd %zmm0, %zmm3
+; X64-NEXT:    vpermt2pd %zmm2, %zmm1, %zmm3
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vpermi2pd %zmm2, %zmm0, %zmm1 {%k1}
+; X64-NEXT:    vaddpd %zmm3, %zmm1, %zmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_int_x86_avx512_mask_vpermi2var_pd_512:
+; X86:       # %bb.0:
+; X86-NEXT:    vmovapd %zmm0, %zmm3
+; X86-NEXT:    vpermt2pd %zmm2, %zmm1, %zmm3
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vpermi2pd %zmm2, %zmm0, %zmm1 {%k1}
+; X86-NEXT:    vaddpd %zmm3, %zmm1, %zmm0
+; X86-NEXT:    retl
   %1 = call <8 x double> @llvm.x86.avx512.vpermi2var.pd.512(<8 x double> %x0, <8 x i64> %x1, <8 x double> %x2)
   %2 = bitcast <8 x i64> %x1 to <8 x double>
   %3 = bitcast i8 %x3 to <8 x i1>
@@ -2441,14 +3054,23 @@ define <8 x double>@test_int_x86_avx512_mask_vpermi2var_pd_512(<8 x double> %x0,
 declare <16 x float> @llvm.x86.avx512.vpermi2var.ps.512(<16 x float>, <16 x i32>, <16 x float>)
 
 define <16 x float>@test_int_x86_avx512_mask_vpermi2var_ps_512(<16 x float> %x0, <16 x i32> %x1, <16 x float> %x2, i16 %x3) {
-; CHECK-LABEL: test_int_x86_avx512_mask_vpermi2var_ps_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    vmovaps %zmm0, %zmm3
-; CHECK-NEXT:    vpermt2ps %zmm2, %zmm1, %zmm3
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vpermi2ps %zmm2, %zmm0, %zmm1 {%k1}
-; CHECK-NEXT:    vaddps %zmm3, %zmm1, %zmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_int_x86_avx512_mask_vpermi2var_ps_512:
+; X64:       # %bb.0:
+; X64-NEXT:    vmovaps %zmm0, %zmm3
+; X64-NEXT:    vpermt2ps %zmm2, %zmm1, %zmm3
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vpermi2ps %zmm2, %zmm0, %zmm1 {%k1}
+; X64-NEXT:    vaddps %zmm3, %zmm1, %zmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_int_x86_avx512_mask_vpermi2var_ps_512:
+; X86:       # %bb.0:
+; X86-NEXT:    vmovaps %zmm0, %zmm3
+; X86-NEXT:    vpermt2ps %zmm2, %zmm1, %zmm3
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vpermi2ps %zmm2, %zmm0, %zmm1 {%k1}
+; X86-NEXT:    vaddps %zmm3, %zmm1, %zmm0
+; X86-NEXT:    retl
   %1 = call <16 x float> @llvm.x86.avx512.vpermi2var.ps.512(<16 x float> %x0, <16 x i32> %x1, <16 x float> %x2)
   %2 = bitcast <16 x i32> %x1 to <16 x float>
   %3 = bitcast i16 %x3 to <16 x i1>
@@ -2462,14 +3084,24 @@ define <16 x float>@test_int_x86_avx512_mask_vpermi2var_ps_512(<16 x float> %x0,
 declare <8 x i64> @llvm.x86.avx512.vpermi2var.q.512(<8 x i64>, <8 x i64>, <8 x i64>)
 
 define <8 x i64>@test_int_x86_avx512_mask_vpermi2var_q_512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 %x3) {
-; CHECK-LABEL: test_int_x86_avx512_mask_vpermi2var_q_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    vmovdqa64 %zmm0, %zmm3
-; CHECK-NEXT:    vpermt2q %zmm2, %zmm1, %zmm3
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vpermi2q %zmm2, %zmm0, %zmm1 {%k1}
-; CHECK-NEXT:    vpaddq %zmm3, %zmm1, %zmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_int_x86_avx512_mask_vpermi2var_q_512:
+; X64:       # %bb.0:
+; X64-NEXT:    vmovdqa64 %zmm0, %zmm3
+; X64-NEXT:    vpermt2q %zmm2, %zmm1, %zmm3
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vpermi2q %zmm2, %zmm0, %zmm1 {%k1}
+; X64-NEXT:    vpaddq %zmm3, %zmm1, %zmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_int_x86_avx512_mask_vpermi2var_q_512:
+; X86:       # %bb.0:
+; X86-NEXT:    vmovdqa64 %zmm0, %zmm3
+; X86-NEXT:    vpermt2q %zmm2, %zmm1, %zmm3
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vpermi2q %zmm2, %zmm0, %zmm1 {%k1}
+; X86-NEXT:    vpaddq %zmm3, %zmm1, %zmm0
+; X86-NEXT:    retl
   %1 = call <8 x i64> @llvm.x86.avx512.vpermi2var.q.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2)
   %2 = bitcast i8 %x3 to <8 x i1>
   %3 = select <8 x i1> %2, <8 x i64> %1, <8 x i64> %x1
@@ -2479,14 +3111,24 @@ define <8 x i64>@test_int_x86_avx512_mask_vpermi2var_q_512(<8 x i64> %x0, <8 x i
 }
 
 define <16 x i32>@test_int_x86_avx512_maskz_vpermt2var_d_512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32>* %x2p, i16 %x3) {
-; CHECK-LABEL: test_int_x86_avx512_maskz_vpermt2var_d_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %esi, %k1
-; CHECK-NEXT:    vmovdqa64 %zmm1, %zmm2
-; CHECK-NEXT:    vpermt2d (%rdi), %zmm0, %zmm2 {%k1} {z}
-; CHECK-NEXT:    vpermt2d %zmm1, %zmm0, %zmm1
-; CHECK-NEXT:    vpaddd %zmm1, %zmm2, %zmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_int_x86_avx512_maskz_vpermt2var_d_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %esi, %k1
+; X64-NEXT:    vmovdqa64 %zmm1, %zmm2
+; X64-NEXT:    vpermt2d (%rdi), %zmm0, %zmm2 {%k1} {z}
+; X64-NEXT:    vpermt2d %zmm1, %zmm0, %zmm1
+; X64-NEXT:    vpaddd %zmm1, %zmm2, %zmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_int_x86_avx512_maskz_vpermt2var_d_512:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vmovdqa64 %zmm1, %zmm2
+; X86-NEXT:    vpermt2d (%eax), %zmm0, %zmm2 {%k1} {z}
+; X86-NEXT:    vpermt2d %zmm1, %zmm0, %zmm1
+; X86-NEXT:    vpaddd %zmm1, %zmm2, %zmm0
+; X86-NEXT:    retl
   %x2 = load <16 x i32>, <16 x i32>* %x2p
   %1 = call <16 x i32> @llvm.x86.avx512.vpermi2var.d.512(<16 x i32> %x1, <16 x i32> %x0, <16 x i32> %x2)
   %2 = bitcast i16 %x3 to <16 x i1>
@@ -2497,14 +3139,25 @@ define <16 x i32>@test_int_x86_avx512_maskz_vpermt2var_d_512(<16 x i32> %x0, <16
 }
 
 define <8 x double>@test_int_x86_avx512_maskz_vpermt2var_pd_512(<8 x i64> %x0, <8 x double> %x1, double* %x2ptr, i8 %x3) {
-; CHECK-LABEL: test_int_x86_avx512_maskz_vpermt2var_pd_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %esi, %k1
-; CHECK-NEXT:    vmovapd %zmm1, %zmm2
-; CHECK-NEXT:    vpermt2pd (%rdi){1to8}, %zmm0, %zmm2 {%k1} {z}
-; CHECK-NEXT:    vpermt2pd %zmm1, %zmm0, %zmm1
-; CHECK-NEXT:    vaddpd %zmm1, %zmm2, %zmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_int_x86_avx512_maskz_vpermt2var_pd_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %esi, %k1
+; X64-NEXT:    vmovapd %zmm1, %zmm2
+; X64-NEXT:    vpermt2pd (%rdi){1to8}, %zmm0, %zmm2 {%k1} {z}
+; X64-NEXT:    vpermt2pd %zmm1, %zmm0, %zmm1
+; X64-NEXT:    vaddpd %zmm1, %zmm2, %zmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_int_x86_avx512_maskz_vpermt2var_pd_512:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    kmovw %ecx, %k1
+; X86-NEXT:    vmovapd %zmm1, %zmm2
+; X86-NEXT:    vpermt2pd (%eax){1to8}, %zmm0, %zmm2 {%k1} {z}
+; X86-NEXT:    vpermt2pd %zmm1, %zmm0, %zmm1
+; X86-NEXT:    vaddpd %zmm1, %zmm2, %zmm0
+; X86-NEXT:    retl
   %x2s = load double, double* %x2ptr
   %x2ins = insertelement <8 x double> undef, double %x2s, i32 0
   %x2 = shufflevector <8 x double> %x2ins, <8 x double> undef, <8 x i32> zeroinitializer
@@ -2517,14 +3170,23 @@ define <8 x double>@test_int_x86_avx512_maskz_vpermt2var_pd_512(<8 x i64> %x0, <
 }
 
 define <16 x float>@test_int_x86_avx512_maskz_vpermt2var_ps_512(<16 x i32> %x0, <16 x float> %x1, <16 x float> %x2, i16 %x3) {
-; CHECK-LABEL: test_int_x86_avx512_maskz_vpermt2var_ps_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    vmovaps %zmm1, %zmm3
-; CHECK-NEXT:    vpermt2ps %zmm2, %zmm0, %zmm3
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vpermt2ps %zmm2, %zmm0, %zmm1 {%k1} {z}
-; CHECK-NEXT:    vaddps %zmm3, %zmm1, %zmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_int_x86_avx512_maskz_vpermt2var_ps_512:
+; X64:       # %bb.0:
+; X64-NEXT:    vmovaps %zmm1, %zmm3
+; X64-NEXT:    vpermt2ps %zmm2, %zmm0, %zmm3
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vpermt2ps %zmm2, %zmm0, %zmm1 {%k1} {z}
+; X64-NEXT:    vaddps %zmm3, %zmm1, %zmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_int_x86_avx512_maskz_vpermt2var_ps_512:
+; X86:       # %bb.0:
+; X86-NEXT:    vmovaps %zmm1, %zmm3
+; X86-NEXT:    vpermt2ps %zmm2, %zmm0, %zmm3
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vpermt2ps %zmm2, %zmm0, %zmm1 {%k1} {z}
+; X86-NEXT:    vaddps %zmm3, %zmm1, %zmm0
+; X86-NEXT:    retl
   %1 = call <16 x float> @llvm.x86.avx512.vpermi2var.ps.512(<16 x float> %x1, <16 x i32> %x0, <16 x float> %x2)
   %2 = bitcast i16 %x3 to <16 x i1>
   %3 = select <16 x i1> %2, <16 x float> %1, <16 x float> zeroinitializer
@@ -2534,14 +3196,24 @@ define <16 x float>@test_int_x86_avx512_maskz_vpermt2var_ps_512(<16 x i32> %x0,
 }
 
 define <8 x i64>@test_int_x86_avx512_maskz_vpermt2var_q_512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 %x3) {
-; CHECK-LABEL: test_int_x86_avx512_maskz_vpermt2var_q_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    vmovdqa64 %zmm1, %zmm3
-; CHECK-NEXT:    vpermt2q %zmm2, %zmm0, %zmm3
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vpermt2q %zmm2, %zmm0, %zmm1 {%k1} {z}
-; CHECK-NEXT:    vpaddq %zmm3, %zmm1, %zmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_int_x86_avx512_maskz_vpermt2var_q_512:
+; X64:       # %bb.0:
+; X64-NEXT:    vmovdqa64 %zmm1, %zmm3
+; X64-NEXT:    vpermt2q %zmm2, %zmm0, %zmm3
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vpermt2q %zmm2, %zmm0, %zmm1 {%k1} {z}
+; X64-NEXT:    vpaddq %zmm3, %zmm1, %zmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_int_x86_avx512_maskz_vpermt2var_q_512:
+; X86:       # %bb.0:
+; X86-NEXT:    vmovdqa64 %zmm1, %zmm3
+; X86-NEXT:    vpermt2q %zmm2, %zmm0, %zmm3
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vpermt2q %zmm2, %zmm0, %zmm1 {%k1} {z}
+; X86-NEXT:    vpaddq %zmm3, %zmm1, %zmm0
+; X86-NEXT:    retl
   %1 = call <8 x i64> @llvm.x86.avx512.vpermi2var.q.512(<8 x i64> %x1, <8 x i64> %x0, <8 x i64> %x2)
   %2 = bitcast i8 %x3 to <8 x i1>
   %3 = select <8 x i1> %2, <8 x i64> %1, <8 x i64> zeroinitializer
@@ -2551,14 +3223,23 @@ define <8 x i64>@test_int_x86_avx512_maskz_vpermt2var_q_512(<8 x i64> %x0, <8 x
 }
 
 define <16 x i32>@test_int_x86_avx512_mask_vpermt2var_d_512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 %x3) {
-; CHECK-LABEL: test_int_x86_avx512_mask_vpermt2var_d_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    vmovdqa64 %zmm1, %zmm3
-; CHECK-NEXT:    vpermt2d %zmm2, %zmm0, %zmm3
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vpermt2d %zmm2, %zmm0, %zmm1 {%k1}
-; CHECK-NEXT:    vpaddd %zmm3, %zmm1, %zmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_int_x86_avx512_mask_vpermt2var_d_512:
+; X64:       # %bb.0:
+; X64-NEXT:    vmovdqa64 %zmm1, %zmm3
+; X64-NEXT:    vpermt2d %zmm2, %zmm0, %zmm3
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vpermt2d %zmm2, %zmm0, %zmm1 {%k1}
+; X64-NEXT:    vpaddd %zmm3, %zmm1, %zmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_int_x86_avx512_mask_vpermt2var_d_512:
+; X86:       # %bb.0:
+; X86-NEXT:    vmovdqa64 %zmm1, %zmm3
+; X86-NEXT:    vpermt2d %zmm2, %zmm0, %zmm3
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vpermt2d %zmm2, %zmm0, %zmm1 {%k1}
+; X86-NEXT:    vpaddd %zmm3, %zmm1, %zmm0
+; X86-NEXT:    retl
   %1 = call <16 x i32> @llvm.x86.avx512.vpermi2var.d.512(<16 x i32> %x1, <16 x i32> %x0, <16 x i32> %x2)
   %2 = bitcast i16 %x3 to <16 x i1>
   %3 = select <16 x i1> %2, <16 x i32> %1, <16 x i32> %x1
@@ -2569,13 +3250,22 @@ define <16 x i32>@test_int_x86_avx512_mask_vpermt2var_d_512(<16 x i32> %x0, <16
 
 declare <8 x double> @llvm.x86.avx512.mask.scalef.pd.512(<8 x double>, <8 x double>, <8 x double>, i8, i32)
 define <8 x double>@test_int_x86_avx512_mask_scalef_pd_512(<8 x double> %x0, <8 x double> %x1, <8 x double> %x2, i8 %x3) {
-; CHECK-LABEL: test_int_x86_avx512_mask_scalef_pd_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vscalefpd {rz-sae}, %zmm1, %zmm0, %zmm2 {%k1}
-; CHECK-NEXT:    vscalefpd {rn-sae}, %zmm1, %zmm0, %zmm0
-; CHECK-NEXT:    vaddpd %zmm0, %zmm2, %zmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_int_x86_avx512_mask_scalef_pd_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vscalefpd {rz-sae}, %zmm1, %zmm0, %zmm2 {%k1}
+; X64-NEXT:    vscalefpd {rn-sae}, %zmm1, %zmm0, %zmm0
+; X64-NEXT:    vaddpd %zmm0, %zmm2, %zmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_int_x86_avx512_mask_scalef_pd_512:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vscalefpd {rz-sae}, %zmm1, %zmm0, %zmm2 {%k1}
+; X86-NEXT:    vscalefpd {rn-sae}, %zmm1, %zmm0, %zmm0
+; X86-NEXT:    vaddpd %zmm0, %zmm2, %zmm0
+; X86-NEXT:    retl
   %res = call <8 x double> @llvm.x86.avx512.mask.scalef.pd.512(<8 x double> %x0, <8 x double> %x1, <8 x double> %x2, i8 %x3, i32 3)
   %res1 = call <8 x double> @llvm.x86.avx512.mask.scalef.pd.512(<8 x double> %x0, <8 x double> %x1, <8 x double> %x2, i8 -1, i32 0)
   %res2 = fadd <8 x double> %res, %res1
@@ -2584,13 +3274,21 @@ define <8 x double>@test_int_x86_avx512_mask_scalef_pd_512(<8 x double> %x0, <8
 
 declare <16 x float> @llvm.x86.avx512.mask.scalef.ps.512(<16 x float>, <16 x float>, <16 x float>, i16, i32)
 define <16 x float>@test_int_x86_avx512_mask_scalef_ps_512(<16 x float> %x0, <16 x float> %x1, <16 x float> %x2, i16 %x3) {
-; CHECK-LABEL: test_int_x86_avx512_mask_scalef_ps_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vscalefps {ru-sae}, %zmm1, %zmm0, %zmm2 {%k1}
-; CHECK-NEXT:    vscalefps {rn-sae}, %zmm1, %zmm0, %zmm0
-; CHECK-NEXT:    vaddps %zmm0, %zmm2, %zmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_int_x86_avx512_mask_scalef_ps_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vscalefps {ru-sae}, %zmm1, %zmm0, %zmm2 {%k1}
+; X64-NEXT:    vscalefps {rn-sae}, %zmm1, %zmm0, %zmm0
+; X64-NEXT:    vaddps %zmm0, %zmm2, %zmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_int_x86_avx512_mask_scalef_ps_512:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vscalefps {ru-sae}, %zmm1, %zmm0, %zmm2 {%k1}
+; X86-NEXT:    vscalefps {rn-sae}, %zmm1, %zmm0, %zmm0
+; X86-NEXT:    vaddps %zmm0, %zmm2, %zmm0
+; X86-NEXT:    retl
   %res = call <16 x float> @llvm.x86.avx512.mask.scalef.ps.512(<16 x float> %x0, <16 x float> %x1, <16 x float> %x2, i16 %x3, i32 2)
   %res1 = call <16 x float> @llvm.x86.avx512.mask.scalef.ps.512(<16 x float> %x0, <16 x float> %x1, <16 x float> %x2, i16 -1, i32 0)
   %res2 = fadd <16 x float> %res, %res1
@@ -2600,16 +3298,28 @@ define <16 x float>@test_int_x86_avx512_mask_scalef_ps_512(<16 x float> %x0, <16
 declare <16 x i8> @llvm.x86.avx512.mask.pmov.qb.512(<8 x i64>, <16 x i8>, i8)
 
 define <16 x i8>@test_int_x86_avx512_mask_pmov_qb_512(<8 x i64> %x0, <16 x i8> %x1, i8 %x2) {
-; CHECK-LABEL: test_int_x86_avx512_mask_pmov_qb_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vpmovqb %zmm0, %xmm2 {%k1} {z}
-; CHECK-NEXT:    vpmovqb %zmm0, %xmm1 {%k1}
-; CHECK-NEXT:    vpaddb %xmm2, %xmm1, %xmm1
-; CHECK-NEXT:    vpmovqb %zmm0, %xmm0
-; CHECK-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
-; CHECK-NEXT:    vzeroupper
-; CHECK-NEXT:    retq
+; X64-LABEL: test_int_x86_avx512_mask_pmov_qb_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vpmovqb %zmm0, %xmm2
+; X64-NEXT:    vpmovqb %zmm0, %xmm1 {%k1}
+; X64-NEXT:    vpmovqb %zmm0, %xmm0 {%k1} {z}
+; X64-NEXT:    vpaddb %xmm0, %xmm1, %xmm0
+; X64-NEXT:    vpaddb %xmm0, %xmm2, %xmm0
+; X64-NEXT:    vzeroupper
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_int_x86_avx512_mask_pmov_qb_512:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vpmovqb %zmm0, %xmm2
+; X86-NEXT:    vpmovqb %zmm0, %xmm1 {%k1}
+; X86-NEXT:    vpmovqb %zmm0, %xmm0 {%k1} {z}
+; X86-NEXT:    vpaddb %xmm0, %xmm1, %xmm0
+; X86-NEXT:    vpaddb %xmm0, %xmm2, %xmm0
+; X86-NEXT:    vzeroupper
+; X86-NEXT:    retl
     %res0 = call <16 x i8> @llvm.x86.avx512.mask.pmov.qb.512(<8 x i64> %x0, <16 x i8> %x1, i8 -1)
     %res1 = call <16 x i8> @llvm.x86.avx512.mask.pmov.qb.512(<8 x i64> %x0, <16 x i8> %x1, i8 %x2)
     %res2 = call <16 x i8> @llvm.x86.avx512.mask.pmov.qb.512(<8 x i64> %x0, <16 x i8> zeroinitializer, i8 %x2)
@@ -2621,13 +3331,23 @@ define <16 x i8>@test_int_x86_avx512_mask_pmov_qb_512(<8 x i64> %x0, <16 x i8> %
 declare void @llvm.x86.avx512.mask.pmov.qb.mem.512(i8* %ptr, <8 x i64>, i8)
 
 define void @test_int_x86_avx512_mask_pmov_qb_mem_512(i8* %ptr, <8 x i64> %x1, i8 %x2) {
-; CHECK-LABEL: test_int_x86_avx512_mask_pmov_qb_mem_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %esi, %k1
-; CHECK-NEXT:    vpmovqb %zmm0, (%rdi)
-; CHECK-NEXT:    vpmovqb %zmm0, (%rdi) {%k1}
-; CHECK-NEXT:    vzeroupper
-; CHECK-NEXT:    retq
+; X64-LABEL: test_int_x86_avx512_mask_pmov_qb_mem_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %esi, %k1
+; X64-NEXT:    vpmovqb %zmm0, (%rdi)
+; X64-NEXT:    vpmovqb %zmm0, (%rdi) {%k1}
+; X64-NEXT:    vzeroupper
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_int_x86_avx512_mask_pmov_qb_mem_512:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    vpmovqb %zmm0, (%eax)
+; X86-NEXT:    vpmovqb %zmm0, (%eax) {%k1}
+; X86-NEXT:    vzeroupper
+; X86-NEXT:    retl
     call void @llvm.x86.avx512.mask.pmov.qb.mem.512(i8* %ptr, <8 x i64> %x1, i8 -1)
     call void @llvm.x86.avx512.mask.pmov.qb.mem.512(i8* %ptr, <8 x i64> %x1, i8 %x2)
     ret void
@@ -2636,16 +3356,28 @@ define void @test_int_x86_avx512_mask_pmov_qb_mem_512(i8* %ptr, <8 x i64> %x1, i
 declare <16 x i8> @llvm.x86.avx512.mask.pmovs.qb.512(<8 x i64>, <16 x i8>, i8)
 
 define <16 x i8>@test_int_x86_avx512_mask_pmovs_qb_512(<8 x i64> %x0, <16 x i8> %x1, i8 %x2) {
-; CHECK-LABEL: test_int_x86_avx512_mask_pmovs_qb_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vpmovsqb %zmm0, %xmm2 {%k1} {z}
-; CHECK-NEXT:    vpmovsqb %zmm0, %xmm1 {%k1}
-; CHECK-NEXT:    vpaddb %xmm2, %xmm1, %xmm1
-; CHECK-NEXT:    vpmovsqb %zmm0, %xmm0
-; CHECK-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
-; CHECK-NEXT:    vzeroupper
-; CHECK-NEXT:    retq
+; X64-LABEL: test_int_x86_avx512_mask_pmovs_qb_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vpmovsqb %zmm0, %xmm2
+; X64-NEXT:    vpmovsqb %zmm0, %xmm1 {%k1}
+; X64-NEXT:    vpmovsqb %zmm0, %xmm0 {%k1} {z}
+; X64-NEXT:    vpaddb %xmm0, %xmm1, %xmm0
+; X64-NEXT:    vpaddb %xmm0, %xmm2, %xmm0
+; X64-NEXT:    vzeroupper
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_int_x86_avx512_mask_pmovs_qb_512:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vpmovsqb %zmm0, %xmm2
+; X86-NEXT:    vpmovsqb %zmm0, %xmm1 {%k1}
+; X86-NEXT:    vpmovsqb %zmm0, %xmm0 {%k1} {z}
+; X86-NEXT:    vpaddb %xmm0, %xmm1, %xmm0
+; X86-NEXT:    vpaddb %xmm0, %xmm2, %xmm0
+; X86-NEXT:    vzeroupper
+; X86-NEXT:    retl
     %res0 = call <16 x i8> @llvm.x86.avx512.mask.pmovs.qb.512(<8 x i64> %x0, <16 x i8> %x1, i8 -1)
     %res1 = call <16 x i8> @llvm.x86.avx512.mask.pmovs.qb.512(<8 x i64> %x0, <16 x i8> %x1, i8 %x2)
     %res2 = call <16 x i8> @llvm.x86.avx512.mask.pmovs.qb.512(<8 x i64> %x0, <16 x i8> zeroinitializer, i8 %x2)
@@ -2657,13 +3389,23 @@ define <16 x i8>@test_int_x86_avx512_mask_pmovs_qb_512(<8 x i64> %x0, <16 x i8>
 declare void @llvm.x86.avx512.mask.pmovs.qb.mem.512(i8* %ptr, <8 x i64>, i8)
 
 define void @test_int_x86_avx512_mask_pmovs_qb_mem_512(i8* %ptr, <8 x i64> %x1, i8 %x2) {
-; CHECK-LABEL: test_int_x86_avx512_mask_pmovs_qb_mem_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %esi, %k1
-; CHECK-NEXT:    vpmovsqb %zmm0, (%rdi)
-; CHECK-NEXT:    vpmovsqb %zmm0, (%rdi) {%k1}
-; CHECK-NEXT:    vzeroupper
-; CHECK-NEXT:    retq
+; X64-LABEL: test_int_x86_avx512_mask_pmovs_qb_mem_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %esi, %k1
+; X64-NEXT:    vpmovsqb %zmm0, (%rdi)
+; X64-NEXT:    vpmovsqb %zmm0, (%rdi) {%k1}
+; X64-NEXT:    vzeroupper
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_int_x86_avx512_mask_pmovs_qb_mem_512:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    vpmovsqb %zmm0, (%eax)
+; X86-NEXT:    vpmovsqb %zmm0, (%eax) {%k1}
+; X86-NEXT:    vzeroupper
+; X86-NEXT:    retl
     call void @llvm.x86.avx512.mask.pmovs.qb.mem.512(i8* %ptr, <8 x i64> %x1, i8 -1)
     call void @llvm.x86.avx512.mask.pmovs.qb.mem.512(i8* %ptr, <8 x i64> %x1, i8 %x2)
     ret void
@@ -2672,16 +3414,28 @@ define void @test_int_x86_avx512_mask_pmovs_qb_mem_512(i8* %ptr, <8 x i64> %x1,
 declare <16 x i8> @llvm.x86.avx512.mask.pmovus.qb.512(<8 x i64>, <16 x i8>, i8)
 
 define <16 x i8>@test_int_x86_avx512_mask_pmovus_qb_512(<8 x i64> %x0, <16 x i8> %x1, i8 %x2) {
-; CHECK-LABEL: test_int_x86_avx512_mask_pmovus_qb_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vpmovusqb %zmm0, %xmm2 {%k1} {z}
-; CHECK-NEXT:    vpmovusqb %zmm0, %xmm1 {%k1}
-; CHECK-NEXT:    vpaddb %xmm2, %xmm1, %xmm1
-; CHECK-NEXT:    vpmovusqb %zmm0, %xmm0
-; CHECK-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
-; CHECK-NEXT:    vzeroupper
-; CHECK-NEXT:    retq
+; X64-LABEL: test_int_x86_avx512_mask_pmovus_qb_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vpmovusqb %zmm0, %xmm2
+; X64-NEXT:    vpmovusqb %zmm0, %xmm1 {%k1}
+; X64-NEXT:    vpmovusqb %zmm0, %xmm0 {%k1} {z}
+; X64-NEXT:    vpaddb %xmm0, %xmm1, %xmm0
+; X64-NEXT:    vpaddb %xmm0, %xmm2, %xmm0
+; X64-NEXT:    vzeroupper
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_int_x86_avx512_mask_pmovus_qb_512:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vpmovusqb %zmm0, %xmm2
+; X86-NEXT:    vpmovusqb %zmm0, %xmm1 {%k1}
+; X86-NEXT:    vpmovusqb %zmm0, %xmm0 {%k1} {z}
+; X86-NEXT:    vpaddb %xmm0, %xmm1, %xmm0
+; X86-NEXT:    vpaddb %xmm0, %xmm2, %xmm0
+; X86-NEXT:    vzeroupper
+; X86-NEXT:    retl
     %res0 = call <16 x i8> @llvm.x86.avx512.mask.pmovus.qb.512(<8 x i64> %x0, <16 x i8> %x1, i8 -1)
     %res1 = call <16 x i8> @llvm.x86.avx512.mask.pmovus.qb.512(<8 x i64> %x0, <16 x i8> %x1, i8 %x2)
     %res2 = call <16 x i8> @llvm.x86.avx512.mask.pmovus.qb.512(<8 x i64> %x0, <16 x i8> zeroinitializer, i8 %x2)
@@ -2693,13 +3447,23 @@ define <16 x i8>@test_int_x86_avx512_mask_pmovus_qb_512(<8 x i64> %x0, <16 x i8>
 declare void @llvm.x86.avx512.mask.pmovus.qb.mem.512(i8* %ptr, <8 x i64>, i8)
 
 define void @test_int_x86_avx512_mask_pmovus_qb_mem_512(i8* %ptr, <8 x i64> %x1, i8 %x2) {
-; CHECK-LABEL: test_int_x86_avx512_mask_pmovus_qb_mem_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %esi, %k1
-; CHECK-NEXT:    vpmovusqb %zmm0, (%rdi)
-; CHECK-NEXT:    vpmovusqb %zmm0, (%rdi) {%k1}
-; CHECK-NEXT:    vzeroupper
-; CHECK-NEXT:    retq
+; X64-LABEL: test_int_x86_avx512_mask_pmovus_qb_mem_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %esi, %k1
+; X64-NEXT:    vpmovusqb %zmm0, (%rdi)
+; X64-NEXT:    vpmovusqb %zmm0, (%rdi) {%k1}
+; X64-NEXT:    vzeroupper
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_int_x86_avx512_mask_pmovus_qb_mem_512:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    vpmovusqb %zmm0, (%eax)
+; X86-NEXT:    vpmovusqb %zmm0, (%eax) {%k1}
+; X86-NEXT:    vzeroupper
+; X86-NEXT:    retl
     call void @llvm.x86.avx512.mask.pmovus.qb.mem.512(i8* %ptr, <8 x i64> %x1, i8 -1)
     call void @llvm.x86.avx512.mask.pmovus.qb.mem.512(i8* %ptr, <8 x i64> %x1, i8 %x2)
     ret void
@@ -2708,16 +3472,28 @@ define void @test_int_x86_avx512_mask_pmovus_qb_mem_512(i8* %ptr, <8 x i64> %x1,
 declare <8 x i16> @llvm.x86.avx512.mask.pmov.qw.512(<8 x i64>, <8 x i16>, i8)
 
 define <8 x i16>@test_int_x86_avx512_mask_pmov_qw_512(<8 x i64> %x0, <8 x i16> %x1, i8 %x2) {
-; CHECK-LABEL: test_int_x86_avx512_mask_pmov_qw_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vpmovqw %zmm0, %xmm2 {%k1} {z}
-; CHECK-NEXT:    vpmovqw %zmm0, %xmm1 {%k1}
-; CHECK-NEXT:    vpaddw %xmm2, %xmm1, %xmm1
-; CHECK-NEXT:    vpmovqw %zmm0, %xmm0
-; CHECK-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
-; CHECK-NEXT:    vzeroupper
-; CHECK-NEXT:    retq
+; X64-LABEL: test_int_x86_avx512_mask_pmov_qw_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vpmovqw %zmm0, %xmm2
+; X64-NEXT:    vpmovqw %zmm0, %xmm1 {%k1}
+; X64-NEXT:    vpmovqw %zmm0, %xmm0 {%k1} {z}
+; X64-NEXT:    vpaddw %xmm0, %xmm1, %xmm0
+; X64-NEXT:    vpaddw %xmm0, %xmm2, %xmm0
+; X64-NEXT:    vzeroupper
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_int_x86_avx512_mask_pmov_qw_512:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vpmovqw %zmm0, %xmm2
+; X86-NEXT:    vpmovqw %zmm0, %xmm1 {%k1}
+; X86-NEXT:    vpmovqw %zmm0, %xmm0 {%k1} {z}
+; X86-NEXT:    vpaddw %xmm0, %xmm1, %xmm0
+; X86-NEXT:    vpaddw %xmm0, %xmm2, %xmm0
+; X86-NEXT:    vzeroupper
+; X86-NEXT:    retl
     %res0 = call <8 x i16> @llvm.x86.avx512.mask.pmov.qw.512(<8 x i64> %x0, <8 x i16> %x1, i8 -1)
     %res1 = call <8 x i16> @llvm.x86.avx512.mask.pmov.qw.512(<8 x i64> %x0, <8 x i16> %x1, i8 %x2)
     %res2 = call <8 x i16> @llvm.x86.avx512.mask.pmov.qw.512(<8 x i64> %x0, <8 x i16> zeroinitializer, i8 %x2)
@@ -2729,13 +3505,23 @@ define <8 x i16>@test_int_x86_avx512_mask_pmov_qw_512(<8 x i64> %x0, <8 x i16> %
 declare void @llvm.x86.avx512.mask.pmov.qw.mem.512(i8* %ptr, <8 x i64>, i8)
 
 define void @test_int_x86_avx512_mask_pmov_qw_mem_512(i8* %ptr, <8 x i64> %x1, i8 %x2) {
-; CHECK-LABEL: test_int_x86_avx512_mask_pmov_qw_mem_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %esi, %k1
-; CHECK-NEXT:    vpmovqw %zmm0, (%rdi)
-; CHECK-NEXT:    vpmovqw %zmm0, (%rdi) {%k1}
-; CHECK-NEXT:    vzeroupper
-; CHECK-NEXT:    retq
+; X64-LABEL: test_int_x86_avx512_mask_pmov_qw_mem_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %esi, %k1
+; X64-NEXT:    vpmovqw %zmm0, (%rdi)
+; X64-NEXT:    vpmovqw %zmm0, (%rdi) {%k1}
+; X64-NEXT:    vzeroupper
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_int_x86_avx512_mask_pmov_qw_mem_512:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    vpmovqw %zmm0, (%eax)
+; X86-NEXT:    vpmovqw %zmm0, (%eax) {%k1}
+; X86-NEXT:    vzeroupper
+; X86-NEXT:    retl
     call void @llvm.x86.avx512.mask.pmov.qw.mem.512(i8* %ptr, <8 x i64> %x1, i8 -1)
     call void @llvm.x86.avx512.mask.pmov.qw.mem.512(i8* %ptr, <8 x i64> %x1, i8 %x2)
     ret void
@@ -2744,16 +3530,28 @@ define void @test_int_x86_avx512_mask_pmov_qw_mem_512(i8* %ptr, <8 x i64> %x1, i
 declare <8 x i16> @llvm.x86.avx512.mask.pmovs.qw.512(<8 x i64>, <8 x i16>, i8)
 
 define <8 x i16>@test_int_x86_avx512_mask_pmovs_qw_512(<8 x i64> %x0, <8 x i16> %x1, i8 %x2) {
-; CHECK-LABEL: test_int_x86_avx512_mask_pmovs_qw_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vpmovsqw %zmm0, %xmm2 {%k1} {z}
-; CHECK-NEXT:    vpmovsqw %zmm0, %xmm1 {%k1}
-; CHECK-NEXT:    vpaddw %xmm2, %xmm1, %xmm1
-; CHECK-NEXT:    vpmovsqw %zmm0, %xmm0
-; CHECK-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
-; CHECK-NEXT:    vzeroupper
-; CHECK-NEXT:    retq
+; X64-LABEL: test_int_x86_avx512_mask_pmovs_qw_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vpmovsqw %zmm0, %xmm2
+; X64-NEXT:    vpmovsqw %zmm0, %xmm1 {%k1}
+; X64-NEXT:    vpmovsqw %zmm0, %xmm0 {%k1} {z}
+; X64-NEXT:    vpaddw %xmm0, %xmm1, %xmm0
+; X64-NEXT:    vpaddw %xmm0, %xmm2, %xmm0
+; X64-NEXT:    vzeroupper
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_int_x86_avx512_mask_pmovs_qw_512:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vpmovsqw %zmm0, %xmm2
+; X86-NEXT:    vpmovsqw %zmm0, %xmm1 {%k1}
+; X86-NEXT:    vpmovsqw %zmm0, %xmm0 {%k1} {z}
+; X86-NEXT:    vpaddw %xmm0, %xmm1, %xmm0
+; X86-NEXT:    vpaddw %xmm0, %xmm2, %xmm0
+; X86-NEXT:    vzeroupper
+; X86-NEXT:    retl
     %res0 = call <8 x i16> @llvm.x86.avx512.mask.pmovs.qw.512(<8 x i64> %x0, <8 x i16> %x1, i8 -1)
     %res1 = call <8 x i16> @llvm.x86.avx512.mask.pmovs.qw.512(<8 x i64> %x0, <8 x i16> %x1, i8 %x2)
     %res2 = call <8 x i16> @llvm.x86.avx512.mask.pmovs.qw.512(<8 x i64> %x0, <8 x i16> zeroinitializer, i8 %x2)
@@ -2765,13 +3563,23 @@ define <8 x i16>@test_int_x86_avx512_mask_pmovs_qw_512(<8 x i64> %x0, <8 x i16>
 declare void @llvm.x86.avx512.mask.pmovs.qw.mem.512(i8* %ptr, <8 x i64>, i8)
 
 define void @test_int_x86_avx512_mask_pmovs_qw_mem_512(i8* %ptr, <8 x i64> %x1, i8 %x2) {
-; CHECK-LABEL: test_int_x86_avx512_mask_pmovs_qw_mem_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %esi, %k1
-; CHECK-NEXT:    vpmovsqw %zmm0, (%rdi)
-; CHECK-NEXT:    vpmovsqw %zmm0, (%rdi) {%k1}
-; CHECK-NEXT:    vzeroupper
-; CHECK-NEXT:    retq
+; X64-LABEL: test_int_x86_avx512_mask_pmovs_qw_mem_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %esi, %k1
+; X64-NEXT:    vpmovsqw %zmm0, (%rdi)
+; X64-NEXT:    vpmovsqw %zmm0, (%rdi) {%k1}
+; X64-NEXT:    vzeroupper
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_int_x86_avx512_mask_pmovs_qw_mem_512:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    vpmovsqw %zmm0, (%eax)
+; X86-NEXT:    vpmovsqw %zmm0, (%eax) {%k1}
+; X86-NEXT:    vzeroupper
+; X86-NEXT:    retl
     call void @llvm.x86.avx512.mask.pmovs.qw.mem.512(i8* %ptr, <8 x i64> %x1, i8 -1)
     call void @llvm.x86.avx512.mask.pmovs.qw.mem.512(i8* %ptr, <8 x i64> %x1, i8 %x2)
     ret void
@@ -2780,16 +3588,28 @@ define void @test_int_x86_avx512_mask_pmovs_qw_mem_512(i8* %ptr, <8 x i64> %x1,
 declare <8 x i16> @llvm.x86.avx512.mask.pmovus.qw.512(<8 x i64>, <8 x i16>, i8)
 
 define <8 x i16>@test_int_x86_avx512_mask_pmovus_qw_512(<8 x i64> %x0, <8 x i16> %x1, i8 %x2) {
-; CHECK-LABEL: test_int_x86_avx512_mask_pmovus_qw_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vpmovusqw %zmm0, %xmm2 {%k1} {z}
-; CHECK-NEXT:    vpmovusqw %zmm0, %xmm1 {%k1}
-; CHECK-NEXT:    vpaddw %xmm2, %xmm1, %xmm1
-; CHECK-NEXT:    vpmovusqw %zmm0, %xmm0
-; CHECK-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
-; CHECK-NEXT:    vzeroupper
-; CHECK-NEXT:    retq
+; X64-LABEL: test_int_x86_avx512_mask_pmovus_qw_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vpmovusqw %zmm0, %xmm2
+; X64-NEXT:    vpmovusqw %zmm0, %xmm1 {%k1}
+; X64-NEXT:    vpmovusqw %zmm0, %xmm0 {%k1} {z}
+; X64-NEXT:    vpaddw %xmm0, %xmm1, %xmm0
+; X64-NEXT:    vpaddw %xmm0, %xmm2, %xmm0
+; X64-NEXT:    vzeroupper
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_int_x86_avx512_mask_pmovus_qw_512:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vpmovusqw %zmm0, %xmm2
+; X86-NEXT:    vpmovusqw %zmm0, %xmm1 {%k1}
+; X86-NEXT:    vpmovusqw %zmm0, %xmm0 {%k1} {z}
+; X86-NEXT:    vpaddw %xmm0, %xmm1, %xmm0
+; X86-NEXT:    vpaddw %xmm0, %xmm2, %xmm0
+; X86-NEXT:    vzeroupper
+; X86-NEXT:    retl
     %res0 = call <8 x i16> @llvm.x86.avx512.mask.pmovus.qw.512(<8 x i64> %x0, <8 x i16> %x1, i8 -1)
     %res1 = call <8 x i16> @llvm.x86.avx512.mask.pmovus.qw.512(<8 x i64> %x0, <8 x i16> %x1, i8 %x2)
     %res2 = call <8 x i16> @llvm.x86.avx512.mask.pmovus.qw.512(<8 x i64> %x0, <8 x i16> zeroinitializer, i8 %x2)
@@ -2801,48 +3621,81 @@ define <8 x i16>@test_int_x86_avx512_mask_pmovus_qw_512(<8 x i64> %x0, <8 x i16>
 declare void @llvm.x86.avx512.mask.pmovus.qw.mem.512(i8* %ptr, <8 x i64>, i8)
 
 define void @test_int_x86_avx512_mask_pmovus_qw_mem_512(i8* %ptr, <8 x i64> %x1, i8 %x2) {
-; CHECK-LABEL: test_int_x86_avx512_mask_pmovus_qw_mem_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %esi, %k1
-; CHECK-NEXT:    vpmovusqw %zmm0, (%rdi)
-; CHECK-NEXT:    vpmovusqw %zmm0, (%rdi) {%k1}
-; CHECK-NEXT:    vzeroupper
-; CHECK-NEXT:    retq
+; X64-LABEL: test_int_x86_avx512_mask_pmovus_qw_mem_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %esi, %k1
+; X64-NEXT:    vpmovusqw %zmm0, (%rdi)
+; X64-NEXT:    vpmovusqw %zmm0, (%rdi) {%k1}
+; X64-NEXT:    vzeroupper
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_int_x86_avx512_mask_pmovus_qw_mem_512:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    vpmovusqw %zmm0, (%eax)
+; X86-NEXT:    vpmovusqw %zmm0, (%eax) {%k1}
+; X86-NEXT:    vzeroupper
+; X86-NEXT:    retl
     call void @llvm.x86.avx512.mask.pmovus.qw.mem.512(i8* %ptr, <8 x i64> %x1, i8 -1)
     call void @llvm.x86.avx512.mask.pmovus.qw.mem.512(i8* %ptr, <8 x i64> %x1, i8 %x2)
     ret void
 }
 
-declare <8 x i32> @llvm.x86.avx512.mask.pmov.qd.512(<8 x i64>, <8 x i32>, i8)
-
 define <8 x i32>@test_int_x86_avx512_mask_pmov_qd_512(<8 x i64> %x0, <8 x i32> %x1, i8 %x2) {
-; CHECK-LABEL: test_int_x86_avx512_mask_pmov_qd_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vpmovqd %zmm0, %ymm2 {%k1} {z}
-; CHECK-NEXT:    vpmovqd %zmm0, %ymm1 {%k1}
-; CHECK-NEXT:    vpaddd %ymm2, %ymm1, %ymm1
-; CHECK-NEXT:    vpmovqd %zmm0, %ymm0
-; CHECK-NEXT:    vpaddd %ymm1, %ymm0, %ymm0
-; CHECK-NEXT:    retq
-    %res0 = call <8 x i32> @llvm.x86.avx512.mask.pmov.qd.512(<8 x i64> %x0, <8 x i32> %x1, i8 -1)
-    %res1 = call <8 x i32> @llvm.x86.avx512.mask.pmov.qd.512(<8 x i64> %x0, <8 x i32> %x1, i8 %x2)
-    %res2 = call <8 x i32> @llvm.x86.avx512.mask.pmov.qd.512(<8 x i64> %x0, <8 x i32> zeroinitializer, i8 %x2)
-    %res3 = add <8 x i32> %res0, %res1
-    %res4 = add <8 x i32> %res3, %res2
-    ret <8 x i32> %res4
+; X64-LABEL: test_int_x86_avx512_mask_pmov_qd_512:
+; X64:       # %bb.0:
+; X64-NEXT:    vpmovqd %zmm0, %ymm2
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vpmovqd %zmm0, %ymm1 {%k1}
+; X64-NEXT:    vpmovqd %zmm0, %ymm0 {%k1} {z}
+; X64-NEXT:    vpaddd %ymm0, %ymm1, %ymm0
+; X64-NEXT:    vpaddd %ymm0, %ymm2, %ymm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_int_x86_avx512_mask_pmov_qd_512:
+; X86:       # %bb.0:
+; X86-NEXT:    vpmovqd %zmm0, %ymm2
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vpmovqd %zmm0, %ymm1 {%k1}
+; X86-NEXT:    vpmovqd %zmm0, %ymm0 {%k1} {z}
+; X86-NEXT:    vpaddd %ymm0, %ymm1, %ymm0
+; X86-NEXT:    vpaddd %ymm0, %ymm2, %ymm0
+; X86-NEXT:    retl
+  %1 = trunc <8 x i64> %x0 to <8 x i32>
+  %2 = trunc <8 x i64> %x0 to <8 x i32>
+  %3 = bitcast i8 %x2 to <8 x i1>
+  %4 = select <8 x i1> %3, <8 x i32> %2, <8 x i32> %x1
+  %5 = trunc <8 x i64> %x0 to <8 x i32>
+  %6 = bitcast i8 %x2 to <8 x i1>
+  %7 = select <8 x i1> %6, <8 x i32> %5, <8 x i32> zeroinitializer
+  %res3 = add <8 x i32> %1, %4
+  %res4 = add <8 x i32> %res3, %7
+  ret <8 x i32> %res4
 }
 
 declare void @llvm.x86.avx512.mask.pmov.qd.mem.512(i8* %ptr, <8 x i64>, i8)
 
 define void @test_int_x86_avx512_mask_pmov_qd_mem_512(i8* %ptr, <8 x i64> %x1, i8 %x2) {
-; CHECK-LABEL: test_int_x86_avx512_mask_pmov_qd_mem_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %esi, %k1
-; CHECK-NEXT:    vpmovqd %zmm0, (%rdi)
-; CHECK-NEXT:    vpmovqd %zmm0, (%rdi) {%k1}
-; CHECK-NEXT:    vzeroupper
-; CHECK-NEXT:    retq
+; X64-LABEL: test_int_x86_avx512_mask_pmov_qd_mem_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %esi, %k1
+; X64-NEXT:    vpmovqd %zmm0, (%rdi)
+; X64-NEXT:    vpmovqd %zmm0, (%rdi) {%k1}
+; X64-NEXT:    vzeroupper
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_int_x86_avx512_mask_pmov_qd_mem_512:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    vpmovqd %zmm0, (%eax)
+; X86-NEXT:    vpmovqd %zmm0, (%eax) {%k1}
+; X86-NEXT:    vzeroupper
+; X86-NEXT:    retl
     call void @llvm.x86.avx512.mask.pmov.qd.mem.512(i8* %ptr, <8 x i64> %x1, i8 -1)
     call void @llvm.x86.avx512.mask.pmov.qd.mem.512(i8* %ptr, <8 x i64> %x1, i8 %x2)
     ret void
@@ -2851,15 +3704,26 @@ define void @test_int_x86_avx512_mask_pmov_qd_mem_512(i8* %ptr, <8 x i64> %x1, i
 declare <8 x i32> @llvm.x86.avx512.mask.pmovs.qd.512(<8 x i64>, <8 x i32>, i8)
 
 define <8 x i32>@test_int_x86_avx512_mask_pmovs_qd_512(<8 x i64> %x0, <8 x i32> %x1, i8 %x2) {
-; CHECK-LABEL: test_int_x86_avx512_mask_pmovs_qd_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vpmovsqd %zmm0, %ymm2 {%k1} {z}
-; CHECK-NEXT:    vpmovsqd %zmm0, %ymm1 {%k1}
-; CHECK-NEXT:    vpaddd %ymm2, %ymm1, %ymm1
-; CHECK-NEXT:    vpmovsqd %zmm0, %ymm0
-; CHECK-NEXT:    vpaddd %ymm1, %ymm0, %ymm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_int_x86_avx512_mask_pmovs_qd_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vpmovsqd %zmm0, %ymm2 {%k1} {z}
+; X64-NEXT:    vpmovsqd %zmm0, %ymm1 {%k1}
+; X64-NEXT:    vpaddd %ymm2, %ymm1, %ymm1
+; X64-NEXT:    vpmovsqd %zmm0, %ymm0
+; X64-NEXT:    vpaddd %ymm1, %ymm0, %ymm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_int_x86_avx512_mask_pmovs_qd_512:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vpmovsqd %zmm0, %ymm1 {%k1}
+; X86-NEXT:    vpmovsqd %zmm0, %ymm2 {%k1} {z}
+; X86-NEXT:    vpaddd %ymm2, %ymm1, %ymm1
+; X86-NEXT:    vpmovsqd %zmm0, %ymm0
+; X86-NEXT:    vpaddd %ymm1, %ymm0, %ymm0
+; X86-NEXT:    retl
     %res0 = call <8 x i32> @llvm.x86.avx512.mask.pmovs.qd.512(<8 x i64> %x0, <8 x i32> %x1, i8 -1)
     %res1 = call <8 x i32> @llvm.x86.avx512.mask.pmovs.qd.512(<8 x i64> %x0, <8 x i32> %x1, i8 %x2)
     %res2 = call <8 x i32> @llvm.x86.avx512.mask.pmovs.qd.512(<8 x i64> %x0, <8 x i32> zeroinitializer, i8 %x2)
@@ -2871,13 +3735,23 @@ define <8 x i32>@test_int_x86_avx512_mask_pmovs_qd_512(<8 x i64> %x0, <8 x i32>
 declare void @llvm.x86.avx512.mask.pmovs.qd.mem.512(i8* %ptr, <8 x i64>, i8)
 
 define void @test_int_x86_avx512_mask_pmovs_qd_mem_512(i8* %ptr, <8 x i64> %x1, i8 %x2) {
-; CHECK-LABEL: test_int_x86_avx512_mask_pmovs_qd_mem_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %esi, %k1
-; CHECK-NEXT:    vpmovsqd %zmm0, (%rdi)
-; CHECK-NEXT:    vpmovsqd %zmm0, (%rdi) {%k1}
-; CHECK-NEXT:    vzeroupper
-; CHECK-NEXT:    retq
+; X64-LABEL: test_int_x86_avx512_mask_pmovs_qd_mem_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %esi, %k1
+; X64-NEXT:    vpmovsqd %zmm0, (%rdi)
+; X64-NEXT:    vpmovsqd %zmm0, (%rdi) {%k1}
+; X64-NEXT:    vzeroupper
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_int_x86_avx512_mask_pmovs_qd_mem_512:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    vpmovsqd %zmm0, (%eax)
+; X86-NEXT:    vpmovsqd %zmm0, (%eax) {%k1}
+; X86-NEXT:    vzeroupper
+; X86-NEXT:    retl
     call void @llvm.x86.avx512.mask.pmovs.qd.mem.512(i8* %ptr, <8 x i64> %x1, i8 -1)
     call void @llvm.x86.avx512.mask.pmovs.qd.mem.512(i8* %ptr, <8 x i64> %x1, i8 %x2)
     ret void
@@ -2886,15 +3760,26 @@ define void @test_int_x86_avx512_mask_pmovs_qd_mem_512(i8* %ptr, <8 x i64> %x1,
 declare <8 x i32> @llvm.x86.avx512.mask.pmovus.qd.512(<8 x i64>, <8 x i32>, i8)
 
 define <8 x i32>@test_int_x86_avx512_mask_pmovus_qd_512(<8 x i64> %x0, <8 x i32> %x1, i8 %x2) {
-; CHECK-LABEL: test_int_x86_avx512_mask_pmovus_qd_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vpmovusqd %zmm0, %ymm2 {%k1} {z}
-; CHECK-NEXT:    vpmovusqd %zmm0, %ymm1 {%k1}
-; CHECK-NEXT:    vpaddd %ymm2, %ymm1, %ymm1
-; CHECK-NEXT:    vpmovusqd %zmm0, %ymm0
-; CHECK-NEXT:    vpaddd %ymm1, %ymm0, %ymm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_int_x86_avx512_mask_pmovus_qd_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vpmovusqd %zmm0, %ymm2 {%k1} {z}
+; X64-NEXT:    vpmovusqd %zmm0, %ymm1 {%k1}
+; X64-NEXT:    vpaddd %ymm2, %ymm1, %ymm1
+; X64-NEXT:    vpmovusqd %zmm0, %ymm0
+; X64-NEXT:    vpaddd %ymm1, %ymm0, %ymm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_int_x86_avx512_mask_pmovus_qd_512:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vpmovusqd %zmm0, %ymm1 {%k1}
+; X86-NEXT:    vpmovusqd %zmm0, %ymm2 {%k1} {z}
+; X86-NEXT:    vpaddd %ymm2, %ymm1, %ymm1
+; X86-NEXT:    vpmovusqd %zmm0, %ymm0
+; X86-NEXT:    vpaddd %ymm1, %ymm0, %ymm0
+; X86-NEXT:    retl
     %res0 = call <8 x i32> @llvm.x86.avx512.mask.pmovus.qd.512(<8 x i64> %x0, <8 x i32> %x1, i8 -1)
     %res1 = call <8 x i32> @llvm.x86.avx512.mask.pmovus.qd.512(<8 x i64> %x0, <8 x i32> %x1, i8 %x2)
     %res2 = call <8 x i32> @llvm.x86.avx512.mask.pmovus.qd.512(<8 x i64> %x0, <8 x i32> zeroinitializer, i8 %x2)
@@ -2906,13 +3791,23 @@ define <8 x i32>@test_int_x86_avx512_mask_pmovus_qd_512(<8 x i64> %x0, <8 x i32>
 declare void @llvm.x86.avx512.mask.pmovus.qd.mem.512(i8* %ptr, <8 x i64>, i8)
 
 define void @test_int_x86_avx512_mask_pmovus_qd_mem_512(i8* %ptr, <8 x i64> %x1, i8 %x2) {
-; CHECK-LABEL: test_int_x86_avx512_mask_pmovus_qd_mem_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %esi, %k1
-; CHECK-NEXT:    vpmovusqd %zmm0, (%rdi)
-; CHECK-NEXT:    vpmovusqd %zmm0, (%rdi) {%k1}
-; CHECK-NEXT:    vzeroupper
-; CHECK-NEXT:    retq
+; X64-LABEL: test_int_x86_avx512_mask_pmovus_qd_mem_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %esi, %k1
+; X64-NEXT:    vpmovusqd %zmm0, (%rdi)
+; X64-NEXT:    vpmovusqd %zmm0, (%rdi) {%k1}
+; X64-NEXT:    vzeroupper
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_int_x86_avx512_mask_pmovus_qd_mem_512:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    vpmovusqd %zmm0, (%eax)
+; X86-NEXT:    vpmovusqd %zmm0, (%eax) {%k1}
+; X86-NEXT:    vzeroupper
+; X86-NEXT:    retl
     call void @llvm.x86.avx512.mask.pmovus.qd.mem.512(i8* %ptr, <8 x i64> %x1, i8 -1)
     call void @llvm.x86.avx512.mask.pmovus.qd.mem.512(i8* %ptr, <8 x i64> %x1, i8 %x2)
     ret void
@@ -2921,16 +3816,27 @@ define void @test_int_x86_avx512_mask_pmovus_qd_mem_512(i8* %ptr, <8 x i64> %x1,
 declare <16 x i8> @llvm.x86.avx512.mask.pmov.db.512(<16 x i32>, <16 x i8>, i16)
 
 define <16 x i8>@test_int_x86_avx512_mask_pmov_db_512(<16 x i32> %x0, <16 x i8> %x1, i16 %x2) {
-; CHECK-LABEL: test_int_x86_avx512_mask_pmov_db_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vpmovdb %zmm0, %xmm2 {%k1} {z}
-; CHECK-NEXT:    vpmovdb %zmm0, %xmm1 {%k1}
-; CHECK-NEXT:    vpaddb %xmm2, %xmm1, %xmm1
-; CHECK-NEXT:    vpmovdb %zmm0, %xmm0
-; CHECK-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
-; CHECK-NEXT:    vzeroupper
-; CHECK-NEXT:    retq
+; X64-LABEL: test_int_x86_avx512_mask_pmov_db_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vpmovdb %zmm0, %xmm2
+; X64-NEXT:    vpmovdb %zmm0, %xmm1 {%k1}
+; X64-NEXT:    vpmovdb %zmm0, %xmm0 {%k1} {z}
+; X64-NEXT:    vpaddb %xmm0, %xmm1, %xmm0
+; X64-NEXT:    vpaddb %xmm0, %xmm2, %xmm0
+; X64-NEXT:    vzeroupper
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_int_x86_avx512_mask_pmov_db_512:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vpmovdb %zmm0, %xmm2
+; X86-NEXT:    vpmovdb %zmm0, %xmm1 {%k1}
+; X86-NEXT:    vpmovdb %zmm0, %xmm0 {%k1} {z}
+; X86-NEXT:    vpaddb %xmm0, %xmm1, %xmm0
+; X86-NEXT:    vpaddb %xmm0, %xmm2, %xmm0
+; X86-NEXT:    vzeroupper
+; X86-NEXT:    retl
     %res0 = call <16 x i8> @llvm.x86.avx512.mask.pmov.db.512(<16 x i32> %x0, <16 x i8> %x1, i16 -1)
     %res1 = call <16 x i8> @llvm.x86.avx512.mask.pmov.db.512(<16 x i32> %x0, <16 x i8> %x1, i16 %x2)
     %res2 = call <16 x i8> @llvm.x86.avx512.mask.pmov.db.512(<16 x i32> %x0, <16 x i8> zeroinitializer, i16 %x2)
@@ -2942,13 +3848,22 @@ define <16 x i8>@test_int_x86_avx512_mask_pmov_db_512(<16 x i32> %x0, <16 x i8>
 declare void @llvm.x86.avx512.mask.pmov.db.mem.512(i8* %ptr, <16 x i32>, i16)
 
 define void @test_int_x86_avx512_mask_pmov_db_mem_512(i8* %ptr, <16 x i32> %x1, i16 %x2) {
-; CHECK-LABEL: test_int_x86_avx512_mask_pmov_db_mem_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %esi, %k1
-; CHECK-NEXT:    vpmovdb %zmm0, (%rdi)
-; CHECK-NEXT:    vpmovdb %zmm0, (%rdi) {%k1}
-; CHECK-NEXT:    vzeroupper
-; CHECK-NEXT:    retq
+; X64-LABEL: test_int_x86_avx512_mask_pmov_db_mem_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %esi, %k1
+; X64-NEXT:    vpmovdb %zmm0, (%rdi)
+; X64-NEXT:    vpmovdb %zmm0, (%rdi) {%k1}
+; X64-NEXT:    vzeroupper
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_int_x86_avx512_mask_pmov_db_mem_512:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    vpmovdb %zmm0, (%eax)
+; X86-NEXT:    vpmovdb %zmm0, (%eax) {%k1}
+; X86-NEXT:    vzeroupper
+; X86-NEXT:    retl
     call void @llvm.x86.avx512.mask.pmov.db.mem.512(i8* %ptr, <16 x i32> %x1, i16 -1)
     call void @llvm.x86.avx512.mask.pmov.db.mem.512(i8* %ptr, <16 x i32> %x1, i16 %x2)
     ret void
@@ -2957,16 +3872,27 @@ define void @test_int_x86_avx512_mask_pmov_db_mem_512(i8* %ptr, <16 x i32> %x1,
 declare <16 x i8> @llvm.x86.avx512.mask.pmovs.db.512(<16 x i32>, <16 x i8>, i16)
 
 define <16 x i8>@test_int_x86_avx512_mask_pmovs_db_512(<16 x i32> %x0, <16 x i8> %x1, i16 %x2) {
-; CHECK-LABEL: test_int_x86_avx512_mask_pmovs_db_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vpmovsdb %zmm0, %xmm2 {%k1} {z}
-; CHECK-NEXT:    vpmovsdb %zmm0, %xmm1 {%k1}
-; CHECK-NEXT:    vpaddb %xmm2, %xmm1, %xmm1
-; CHECK-NEXT:    vpmovsdb %zmm0, %xmm0
-; CHECK-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
-; CHECK-NEXT:    vzeroupper
-; CHECK-NEXT:    retq
+; X64-LABEL: test_int_x86_avx512_mask_pmovs_db_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vpmovsdb %zmm0, %xmm2
+; X64-NEXT:    vpmovsdb %zmm0, %xmm1 {%k1}
+; X64-NEXT:    vpmovsdb %zmm0, %xmm0 {%k1} {z}
+; X64-NEXT:    vpaddb %xmm0, %xmm1, %xmm0
+; X64-NEXT:    vpaddb %xmm0, %xmm2, %xmm0
+; X64-NEXT:    vzeroupper
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_int_x86_avx512_mask_pmovs_db_512:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vpmovsdb %zmm0, %xmm2
+; X86-NEXT:    vpmovsdb %zmm0, %xmm1 {%k1}
+; X86-NEXT:    vpmovsdb %zmm0, %xmm0 {%k1} {z}
+; X86-NEXT:    vpaddb %xmm0, %xmm1, %xmm0
+; X86-NEXT:    vpaddb %xmm0, %xmm2, %xmm0
+; X86-NEXT:    vzeroupper
+; X86-NEXT:    retl
     %res0 = call <16 x i8> @llvm.x86.avx512.mask.pmovs.db.512(<16 x i32> %x0, <16 x i8> %x1, i16 -1)
     %res1 = call <16 x i8> @llvm.x86.avx512.mask.pmovs.db.512(<16 x i32> %x0, <16 x i8> %x1, i16 %x2)
     %res2 = call <16 x i8> @llvm.x86.avx512.mask.pmovs.db.512(<16 x i32> %x0, <16 x i8> zeroinitializer, i16 %x2)
@@ -2978,13 +3904,22 @@ define <16 x i8>@test_int_x86_avx512_mask_pmovs_db_512(<16 x i32> %x0, <16 x i8>
 declare void @llvm.x86.avx512.mask.pmovs.db.mem.512(i8* %ptr, <16 x i32>, i16)
 
 define void @test_int_x86_avx512_mask_pmovs_db_mem_512(i8* %ptr, <16 x i32> %x1, i16 %x2) {
-; CHECK-LABEL: test_int_x86_avx512_mask_pmovs_db_mem_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %esi, %k1
-; CHECK-NEXT:    vpmovsdb %zmm0, (%rdi)
-; CHECK-NEXT:    vpmovsdb %zmm0, (%rdi) {%k1}
-; CHECK-NEXT:    vzeroupper
-; CHECK-NEXT:    retq
+; X64-LABEL: test_int_x86_avx512_mask_pmovs_db_mem_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %esi, %k1
+; X64-NEXT:    vpmovsdb %zmm0, (%rdi)
+; X64-NEXT:    vpmovsdb %zmm0, (%rdi) {%k1}
+; X64-NEXT:    vzeroupper
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_int_x86_avx512_mask_pmovs_db_mem_512:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    vpmovsdb %zmm0, (%eax)
+; X86-NEXT:    vpmovsdb %zmm0, (%eax) {%k1}
+; X86-NEXT:    vzeroupper
+; X86-NEXT:    retl
     call void @llvm.x86.avx512.mask.pmovs.db.mem.512(i8* %ptr, <16 x i32> %x1, i16 -1)
     call void @llvm.x86.avx512.mask.pmovs.db.mem.512(i8* %ptr, <16 x i32> %x1, i16 %x2)
     ret void
@@ -2993,16 +3928,27 @@ define void @test_int_x86_avx512_mask_pmovs_db_mem_512(i8* %ptr, <16 x i32> %x1,
 declare <16 x i8> @llvm.x86.avx512.mask.pmovus.db.512(<16 x i32>, <16 x i8>, i16)
 
 define <16 x i8>@test_int_x86_avx512_mask_pmovus_db_512(<16 x i32> %x0, <16 x i8> %x1, i16 %x2) {
-; CHECK-LABEL: test_int_x86_avx512_mask_pmovus_db_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vpmovusdb %zmm0, %xmm2 {%k1} {z}
-; CHECK-NEXT:    vpmovusdb %zmm0, %xmm1 {%k1}
-; CHECK-NEXT:    vpaddb %xmm2, %xmm1, %xmm1
-; CHECK-NEXT:    vpmovusdb %zmm0, %xmm0
-; CHECK-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
-; CHECK-NEXT:    vzeroupper
-; CHECK-NEXT:    retq
+; X64-LABEL: test_int_x86_avx512_mask_pmovus_db_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vpmovusdb %zmm0, %xmm2
+; X64-NEXT:    vpmovusdb %zmm0, %xmm1 {%k1}
+; X64-NEXT:    vpmovusdb %zmm0, %xmm0 {%k1} {z}
+; X64-NEXT:    vpaddb %xmm0, %xmm1, %xmm0
+; X64-NEXT:    vpaddb %xmm0, %xmm2, %xmm0
+; X64-NEXT:    vzeroupper
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_int_x86_avx512_mask_pmovus_db_512:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vpmovusdb %zmm0, %xmm2
+; X86-NEXT:    vpmovusdb %zmm0, %xmm1 {%k1}
+; X86-NEXT:    vpmovusdb %zmm0, %xmm0 {%k1} {z}
+; X86-NEXT:    vpaddb %xmm0, %xmm1, %xmm0
+; X86-NEXT:    vpaddb %xmm0, %xmm2, %xmm0
+; X86-NEXT:    vzeroupper
+; X86-NEXT:    retl
     %res0 = call <16 x i8> @llvm.x86.avx512.mask.pmovus.db.512(<16 x i32> %x0, <16 x i8> %x1, i16 -1)
     %res1 = call <16 x i8> @llvm.x86.avx512.mask.pmovus.db.512(<16 x i32> %x0, <16 x i8> %x1, i16 %x2)
     %res2 = call <16 x i8> @llvm.x86.avx512.mask.pmovus.db.512(<16 x i32> %x0, <16 x i8> zeroinitializer, i16 %x2)
@@ -3014,13 +3960,22 @@ define <16 x i8>@test_int_x86_avx512_mask_pmovus_db_512(<16 x i32> %x0, <16 x i8
 declare void @llvm.x86.avx512.mask.pmovus.db.mem.512(i8* %ptr, <16 x i32>, i16)
 
 define void @test_int_x86_avx512_mask_pmovus_db_mem_512(i8* %ptr, <16 x i32> %x1, i16 %x2) {
-; CHECK-LABEL: test_int_x86_avx512_mask_pmovus_db_mem_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %esi, %k1
-; CHECK-NEXT:    vpmovusdb %zmm0, (%rdi)
-; CHECK-NEXT:    vpmovusdb %zmm0, (%rdi) {%k1}
-; CHECK-NEXT:    vzeroupper
-; CHECK-NEXT:    retq
+; X64-LABEL: test_int_x86_avx512_mask_pmovus_db_mem_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %esi, %k1
+; X64-NEXT:    vpmovusdb %zmm0, (%rdi)
+; X64-NEXT:    vpmovusdb %zmm0, (%rdi) {%k1}
+; X64-NEXT:    vzeroupper
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_int_x86_avx512_mask_pmovus_db_mem_512:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    vpmovusdb %zmm0, (%eax)
+; X86-NEXT:    vpmovusdb %zmm0, (%eax) {%k1}
+; X86-NEXT:    vzeroupper
+; X86-NEXT:    retl
     call void @llvm.x86.avx512.mask.pmovus.db.mem.512(i8* %ptr, <16 x i32> %x1, i16 -1)
     call void @llvm.x86.avx512.mask.pmovus.db.mem.512(i8* %ptr, <16 x i32> %x1, i16 %x2)
     ret void
@@ -3029,15 +3984,25 @@ define void @test_int_x86_avx512_mask_pmovus_db_mem_512(i8* %ptr, <16 x i32> %x1
 declare <16 x i16> @llvm.x86.avx512.mask.pmov.dw.512(<16 x i32>, <16 x i16>, i16)
 
 define <16 x i16>@test_int_x86_avx512_mask_pmov_dw_512(<16 x i32> %x0, <16 x i16> %x1, i16 %x2) {
-; CHECK-LABEL: test_int_x86_avx512_mask_pmov_dw_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vpmovdw %zmm0, %ymm2 {%k1} {z}
-; CHECK-NEXT:    vpmovdw %zmm0, %ymm1 {%k1}
-; CHECK-NEXT:    vpaddw %ymm2, %ymm1, %ymm1
-; CHECK-NEXT:    vpmovdw %zmm0, %ymm0
-; CHECK-NEXT:    vpaddw %ymm1, %ymm0, %ymm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_int_x86_avx512_mask_pmov_dw_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vpmovdw %zmm0, %ymm2
+; X64-NEXT:    vpmovdw %zmm0, %ymm1 {%k1}
+; X64-NEXT:    vpmovdw %zmm0, %ymm0 {%k1} {z}
+; X64-NEXT:    vpaddw %ymm0, %ymm1, %ymm0
+; X64-NEXT:    vpaddw %ymm0, %ymm2, %ymm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_int_x86_avx512_mask_pmov_dw_512:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vpmovdw %zmm0, %ymm2
+; X86-NEXT:    vpmovdw %zmm0, %ymm1 {%k1}
+; X86-NEXT:    vpmovdw %zmm0, %ymm0 {%k1} {z}
+; X86-NEXT:    vpaddw %ymm0, %ymm1, %ymm0
+; X86-NEXT:    vpaddw %ymm0, %ymm2, %ymm0
+; X86-NEXT:    retl
     %res0 = call <16 x i16> @llvm.x86.avx512.mask.pmov.dw.512(<16 x i32> %x0, <16 x i16> %x1, i16 -1)
     %res1 = call <16 x i16> @llvm.x86.avx512.mask.pmov.dw.512(<16 x i32> %x0, <16 x i16> %x1, i16 %x2)
     %res2 = call <16 x i16> @llvm.x86.avx512.mask.pmov.dw.512(<16 x i32> %x0, <16 x i16> zeroinitializer, i16 %x2)
@@ -3049,13 +4014,22 @@ define <16 x i16>@test_int_x86_avx512_mask_pmov_dw_512(<16 x i32> %x0, <16 x i16
 declare void @llvm.x86.avx512.mask.pmov.dw.mem.512(i8* %ptr, <16 x i32>, i16)
 
 define void @test_int_x86_avx512_mask_pmov_dw_mem_512(i8* %ptr, <16 x i32> %x1, i16 %x2) {
-; CHECK-LABEL: test_int_x86_avx512_mask_pmov_dw_mem_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %esi, %k1
-; CHECK-NEXT:    vpmovdw %zmm0, (%rdi)
-; CHECK-NEXT:    vpmovdw %zmm0, (%rdi) {%k1}
-; CHECK-NEXT:    vzeroupper
-; CHECK-NEXT:    retq
+; X64-LABEL: test_int_x86_avx512_mask_pmov_dw_mem_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %esi, %k1
+; X64-NEXT:    vpmovdw %zmm0, (%rdi)
+; X64-NEXT:    vpmovdw %zmm0, (%rdi) {%k1}
+; X64-NEXT:    vzeroupper
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_int_x86_avx512_mask_pmov_dw_mem_512:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    vpmovdw %zmm0, (%eax)
+; X86-NEXT:    vpmovdw %zmm0, (%eax) {%k1}
+; X86-NEXT:    vzeroupper
+; X86-NEXT:    retl
     call void @llvm.x86.avx512.mask.pmov.dw.mem.512(i8* %ptr, <16 x i32> %x1, i16 -1)
     call void @llvm.x86.avx512.mask.pmov.dw.mem.512(i8* %ptr, <16 x i32> %x1, i16 %x2)
     ret void
@@ -3064,15 +4038,25 @@ define void @test_int_x86_avx512_mask_pmov_dw_mem_512(i8* %ptr, <16 x i32> %x1,
 declare <16 x i16> @llvm.x86.avx512.mask.pmovs.dw.512(<16 x i32>, <16 x i16>, i16)
 
 define <16 x i16>@test_int_x86_avx512_mask_pmovs_dw_512(<16 x i32> %x0, <16 x i16> %x1, i16 %x2) {
-; CHECK-LABEL: test_int_x86_avx512_mask_pmovs_dw_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vpmovsdw %zmm0, %ymm2 {%k1} {z}
-; CHECK-NEXT:    vpmovsdw %zmm0, %ymm1 {%k1}
-; CHECK-NEXT:    vpaddw %ymm2, %ymm1, %ymm1
-; CHECK-NEXT:    vpmovsdw %zmm0, %ymm0
-; CHECK-NEXT:    vpaddw %ymm1, %ymm0, %ymm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_int_x86_avx512_mask_pmovs_dw_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vpmovsdw %zmm0, %ymm2
+; X64-NEXT:    vpmovsdw %zmm0, %ymm1 {%k1}
+; X64-NEXT:    vpmovsdw %zmm0, %ymm0 {%k1} {z}
+; X64-NEXT:    vpaddw %ymm0, %ymm1, %ymm0
+; X64-NEXT:    vpaddw %ymm0, %ymm2, %ymm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_int_x86_avx512_mask_pmovs_dw_512:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vpmovsdw %zmm0, %ymm2
+; X86-NEXT:    vpmovsdw %zmm0, %ymm1 {%k1}
+; X86-NEXT:    vpmovsdw %zmm0, %ymm0 {%k1} {z}
+; X86-NEXT:    vpaddw %ymm0, %ymm1, %ymm0
+; X86-NEXT:    vpaddw %ymm0, %ymm2, %ymm0
+; X86-NEXT:    retl
     %res0 = call <16 x i16> @llvm.x86.avx512.mask.pmovs.dw.512(<16 x i32> %x0, <16 x i16> %x1, i16 -1)
     %res1 = call <16 x i16> @llvm.x86.avx512.mask.pmovs.dw.512(<16 x i32> %x0, <16 x i16> %x1, i16 %x2)
     %res2 = call <16 x i16> @llvm.x86.avx512.mask.pmovs.dw.512(<16 x i32> %x0, <16 x i16> zeroinitializer, i16 %x2)
@@ -3084,13 +4068,22 @@ define <16 x i16>@test_int_x86_avx512_mask_pmovs_dw_512(<16 x i32> %x0, <16 x i1
 declare void @llvm.x86.avx512.mask.pmovs.dw.mem.512(i8* %ptr, <16 x i32>, i16)
 
 define void @test_int_x86_avx512_mask_pmovs_dw_mem_512(i8* %ptr, <16 x i32> %x1, i16 %x2) {
-; CHECK-LABEL: test_int_x86_avx512_mask_pmovs_dw_mem_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %esi, %k1
-; CHECK-NEXT:    vpmovsdw %zmm0, (%rdi)
-; CHECK-NEXT:    vpmovsdw %zmm0, (%rdi) {%k1}
-; CHECK-NEXT:    vzeroupper
-; CHECK-NEXT:    retq
+; X64-LABEL: test_int_x86_avx512_mask_pmovs_dw_mem_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %esi, %k1
+; X64-NEXT:    vpmovsdw %zmm0, (%rdi)
+; X64-NEXT:    vpmovsdw %zmm0, (%rdi) {%k1}
+; X64-NEXT:    vzeroupper
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_int_x86_avx512_mask_pmovs_dw_mem_512:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    vpmovsdw %zmm0, (%eax)
+; X86-NEXT:    vpmovsdw %zmm0, (%eax) {%k1}
+; X86-NEXT:    vzeroupper
+; X86-NEXT:    retl
     call void @llvm.x86.avx512.mask.pmovs.dw.mem.512(i8* %ptr, <16 x i32> %x1, i16 -1)
     call void @llvm.x86.avx512.mask.pmovs.dw.mem.512(i8* %ptr, <16 x i32> %x1, i16 %x2)
     ret void
@@ -3099,15 +4092,25 @@ define void @test_int_x86_avx512_mask_pmovs_dw_mem_512(i8* %ptr, <16 x i32> %x1,
 declare <16 x i16> @llvm.x86.avx512.mask.pmovus.dw.512(<16 x i32>, <16 x i16>, i16)
 
 define <16 x i16>@test_int_x86_avx512_mask_pmovus_dw_512(<16 x i32> %x0, <16 x i16> %x1, i16 %x2) {
-; CHECK-LABEL: test_int_x86_avx512_mask_pmovus_dw_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vpmovusdw %zmm0, %ymm2 {%k1} {z}
-; CHECK-NEXT:    vpmovusdw %zmm0, %ymm1 {%k1}
-; CHECK-NEXT:    vpaddw %ymm2, %ymm1, %ymm1
-; CHECK-NEXT:    vpmovusdw %zmm0, %ymm0
-; CHECK-NEXT:    vpaddw %ymm1, %ymm0, %ymm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_int_x86_avx512_mask_pmovus_dw_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vpmovusdw %zmm0, %ymm2
+; X64-NEXT:    vpmovusdw %zmm0, %ymm1 {%k1}
+; X64-NEXT:    vpmovusdw %zmm0, %ymm0 {%k1} {z}
+; X64-NEXT:    vpaddw %ymm0, %ymm1, %ymm0
+; X64-NEXT:    vpaddw %ymm0, %ymm2, %ymm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_int_x86_avx512_mask_pmovus_dw_512:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vpmovusdw %zmm0, %ymm2
+; X86-NEXT:    vpmovusdw %zmm0, %ymm1 {%k1}
+; X86-NEXT:    vpmovusdw %zmm0, %ymm0 {%k1} {z}
+; X86-NEXT:    vpaddw %ymm0, %ymm1, %ymm0
+; X86-NEXT:    vpaddw %ymm0, %ymm2, %ymm0
+; X86-NEXT:    retl
     %res0 = call <16 x i16> @llvm.x86.avx512.mask.pmovus.dw.512(<16 x i32> %x0, <16 x i16> %x1, i16 -1)
     %res1 = call <16 x i16> @llvm.x86.avx512.mask.pmovus.dw.512(<16 x i32> %x0, <16 x i16> %x1, i16 %x2)
     %res2 = call <16 x i16> @llvm.x86.avx512.mask.pmovus.dw.512(<16 x i32> %x0, <16 x i16> zeroinitializer, i16 %x2)
@@ -3119,44 +4122,72 @@ define <16 x i16>@test_int_x86_avx512_mask_pmovus_dw_512(<16 x i32> %x0, <16 x i
 declare void @llvm.x86.avx512.mask.pmovus.dw.mem.512(i8* %ptr, <16 x i32>, i16)
 
 define void @test_int_x86_avx512_mask_pmovus_dw_mem_512(i8* %ptr, <16 x i32> %x1, i16 %x2) {
-; CHECK-LABEL: test_int_x86_avx512_mask_pmovus_dw_mem_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %esi, %k1
-; CHECK-NEXT:    vpmovusdw %zmm0, (%rdi)
-; CHECK-NEXT:    vpmovusdw %zmm0, (%rdi) {%k1}
-; CHECK-NEXT:    vzeroupper
-; CHECK-NEXT:    retq
+; X64-LABEL: test_int_x86_avx512_mask_pmovus_dw_mem_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %esi, %k1
+; X64-NEXT:    vpmovusdw %zmm0, (%rdi)
+; X64-NEXT:    vpmovusdw %zmm0, (%rdi) {%k1}
+; X64-NEXT:    vzeroupper
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_int_x86_avx512_mask_pmovus_dw_mem_512:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    vpmovusdw %zmm0, (%eax)
+; X86-NEXT:    vpmovusdw %zmm0, (%eax) {%k1}
+; X86-NEXT:    vzeroupper
+; X86-NEXT:    retl
     call void @llvm.x86.avx512.mask.pmovus.dw.mem.512(i8* %ptr, <16 x i32> %x1, i16 -1)
     call void @llvm.x86.avx512.mask.pmovus.dw.mem.512(i8* %ptr, <16 x i32> %x1, i16 %x2)
     ret void
 }
 
-declare <16 x float> @llvm.x86.avx512.mask.cvtdq2ps.512(<16 x i32>, <16 x float>, i16, i32)
+declare <16 x float> @llvm.x86.avx512.sitofp.round.v16f32.v16i32(<16 x i32>, i32)
 
 define <16 x float>@test_int_x86_avx512_mask_cvt_dq2ps_512(<16 x i32> %x0, <16 x float> %x1, i16 %x2) {
-; CHECK-LABEL: test_int_x86_avx512_mask_cvt_dq2ps_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vcvtdq2ps %zmm0, %zmm1 {%k1}
-; CHECK-NEXT:    vcvtdq2ps {rn-sae}, %zmm0, %zmm0
-; CHECK-NEXT:    vaddps %zmm0, %zmm1, %zmm0
-; CHECK-NEXT:    retq
-  %res = call <16 x float> @llvm.x86.avx512.mask.cvtdq2ps.512(<16 x i32> %x0, <16 x float> %x1, i16 %x2, i32 4)
-  %res1 = call <16 x float> @llvm.x86.avx512.mask.cvtdq2ps.512(<16 x i32> %x0, <16 x float> %x1, i16 -1, i32 0)
-  %res2 = fadd <16 x float> %res, %res1
+; X64-LABEL: test_int_x86_avx512_mask_cvt_dq2ps_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vcvtdq2ps %zmm0, %zmm1 {%k1}
+; X64-NEXT:    vcvtdq2ps {rn-sae}, %zmm0, %zmm0
+; X64-NEXT:    vaddps %zmm0, %zmm1, %zmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_int_x86_avx512_mask_cvt_dq2ps_512:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vcvtdq2ps %zmm0, %zmm1 {%k1}
+; X86-NEXT:    vcvtdq2ps {rn-sae}, %zmm0, %zmm0
+; X86-NEXT:    vaddps %zmm0, %zmm1, %zmm0
+; X86-NEXT:    retl
+  %cvt = sitofp <16 x i32> %x0 to <16 x float>
+  %1 = bitcast i16 %x2 to <16 x i1>
+  %2 = select <16 x i1> %1, <16 x float> %cvt, <16 x float> %x1
+  %3 = call <16 x float> @llvm.x86.avx512.sitofp.round.v16f32.v16i32(<16 x i32> %x0, i32 0)
+  %res2 = fadd <16 x float> %2, %3
   ret <16 x float> %res2
 }
 
 declare <8 x i32> @llvm.x86.avx512.mask.cvtpd2dq.512(<8 x double>, <8 x i32>, i8, i32)
 
 define <8 x i32>@test_int_x86_avx512_mask_cvt_pd2dq_512(<8 x double> %x0, <8 x i32> %x1, i8 %x2) {
-; CHECK-LABEL: test_int_x86_avx512_mask_cvt_pd2dq_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vcvtpd2dq %zmm0, %ymm1 {%k1}
-; CHECK-NEXT:    vcvtpd2dq {rn-sae}, %zmm0, %ymm0
-; CHECK-NEXT:    vpaddd %ymm0, %ymm1, %ymm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_int_x86_avx512_mask_cvt_pd2dq_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vcvtpd2dq %zmm0, %ymm1 {%k1}
+; X64-NEXT:    vcvtpd2dq {rn-sae}, %zmm0, %ymm0
+; X64-NEXT:    vpaddd %ymm0, %ymm1, %ymm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_int_x86_avx512_mask_cvt_pd2dq_512:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vcvtpd2dq %zmm0, %ymm1 {%k1}
+; X86-NEXT:    vcvtpd2dq {rn-sae}, %zmm0, %ymm0
+; X86-NEXT:    vpaddd %ymm0, %ymm1, %ymm0
+; X86-NEXT:    retl
   %res = call <8 x i32> @llvm.x86.avx512.mask.cvtpd2dq.512(<8 x double> %x0, <8 x i32> %x1, i8 %x2, i32 4)
   %res1 = call <8 x i32> @llvm.x86.avx512.mask.cvtpd2dq.512(<8 x double> %x0, <8 x i32> %x1, i8 -1, i32 0)
   %res2 = add <8 x i32> %res, %res1
@@ -3166,13 +4197,22 @@ define <8 x i32>@test_int_x86_avx512_mask_cvt_pd2dq_512(<8 x double> %x0, <8 x i
 declare <8 x float> @llvm.x86.avx512.mask.cvtpd2ps.512(<8 x double>, <8 x float>, i8, i32)
 
 define <8 x float>@test_int_x86_avx512_mask_cvt_pd2ps_512(<8 x double> %x0, <8 x float> %x1, i8 %x2) {
-; CHECK-LABEL: test_int_x86_avx512_mask_cvt_pd2ps_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vcvtpd2ps %zmm0, %ymm1 {%k1}
-; CHECK-NEXT:    vcvtpd2ps {ru-sae}, %zmm0, %ymm0
-; CHECK-NEXT:    vaddps %ymm0, %ymm1, %ymm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_int_x86_avx512_mask_cvt_pd2ps_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vcvtpd2ps %zmm0, %ymm1 {%k1}
+; X64-NEXT:    vcvtpd2ps {ru-sae}, %zmm0, %ymm0
+; X64-NEXT:    vaddps %ymm0, %ymm1, %ymm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_int_x86_avx512_mask_cvt_pd2ps_512:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vcvtpd2ps %zmm0, %ymm1 {%k1}
+; X86-NEXT:    vcvtpd2ps {ru-sae}, %zmm0, %ymm0
+; X86-NEXT:    vaddps %ymm0, %ymm1, %ymm0
+; X86-NEXT:    retl
   %res = call <8 x float> @llvm.x86.avx512.mask.cvtpd2ps.512(<8 x double> %x0, <8 x float> %x1, i8 %x2, i32 4)
   %res1 = call <8 x float> @llvm.x86.avx512.mask.cvtpd2ps.512(<8 x double> %x0, <8 x float> %x1, i8 -1, i32 2)
   %res2 = fadd <8 x float> %res, %res1
@@ -3182,13 +4222,22 @@ define <8 x float>@test_int_x86_avx512_mask_cvt_pd2ps_512(<8 x double> %x0, <8 x
 declare <8 x i32> @llvm.x86.avx512.mask.cvtpd2udq.512(<8 x double>, <8 x i32>, i8, i32)
 
 define <8 x i32>@test_int_x86_avx512_mask_cvt_pd2udq_512(<8 x double> %x0, <8 x i32> %x1, i8 %x2) {
-; CHECK-LABEL: test_int_x86_avx512_mask_cvt_pd2udq_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vcvtpd2udq {ru-sae}, %zmm0, %ymm1 {%k1}
-; CHECK-NEXT:    vcvtpd2udq {rn-sae}, %zmm0, %ymm0
-; CHECK-NEXT:    vpaddd %ymm0, %ymm1, %ymm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_int_x86_avx512_mask_cvt_pd2udq_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vcvtpd2udq {ru-sae}, %zmm0, %ymm1 {%k1}
+; X64-NEXT:    vcvtpd2udq {rn-sae}, %zmm0, %ymm0
+; X64-NEXT:    vpaddd %ymm0, %ymm1, %ymm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_int_x86_avx512_mask_cvt_pd2udq_512:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vcvtpd2udq {ru-sae}, %zmm0, %ymm1 {%k1}
+; X86-NEXT:    vcvtpd2udq {rn-sae}, %zmm0, %ymm0
+; X86-NEXT:    vpaddd %ymm0, %ymm1, %ymm0
+; X86-NEXT:    retl
   %res = call <8 x i32> @llvm.x86.avx512.mask.cvtpd2udq.512(<8 x double> %x0, <8 x i32> %x1, i8 %x2, i32 2)
   %res1 = call <8 x i32> @llvm.x86.avx512.mask.cvtpd2udq.512(<8 x double> %x0, <8 x i32> %x1, i8 -1, i32 0)
   %res2 = add <8 x i32> %res, %res1
@@ -3198,13 +4247,21 @@ define <8 x i32>@test_int_x86_avx512_mask_cvt_pd2udq_512(<8 x double> %x0, <8 x
 declare <16 x i32> @llvm.x86.avx512.mask.cvtps2dq.512(<16 x float>, <16 x i32>, i16, i32)
 
 define <16 x i32>@test_int_x86_avx512_mask_cvt_ps2dq_512(<16 x float> %x0, <16 x i32> %x1, i16 %x2) {
-; CHECK-LABEL: test_int_x86_avx512_mask_cvt_ps2dq_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vcvtps2dq {ru-sae}, %zmm0, %zmm1 {%k1}
-; CHECK-NEXT:    vcvtps2dq {rn-sae}, %zmm0, %zmm0
-; CHECK-NEXT:    vpaddd %zmm0, %zmm1, %zmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_int_x86_avx512_mask_cvt_ps2dq_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vcvtps2dq {ru-sae}, %zmm0, %zmm1 {%k1}
+; X64-NEXT:    vcvtps2dq {rn-sae}, %zmm0, %zmm0
+; X64-NEXT:    vpaddd %zmm0, %zmm1, %zmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_int_x86_avx512_mask_cvt_ps2dq_512:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vcvtps2dq {ru-sae}, %zmm0, %zmm1 {%k1}
+; X86-NEXT:    vcvtps2dq {rn-sae}, %zmm0, %zmm0
+; X86-NEXT:    vpaddd %zmm0, %zmm1, %zmm0
+; X86-NEXT:    retl
   %res = call <16 x i32> @llvm.x86.avx512.mask.cvtps2dq.512(<16 x float> %x0, <16 x i32> %x1, i16 %x2, i32 2)
   %res1 = call <16 x i32> @llvm.x86.avx512.mask.cvtps2dq.512(<16 x float> %x0, <16 x i32> %x1, i16 -1, i32 0)
   %res2 = add <16 x i32> %res, %res1
@@ -3214,13 +4271,22 @@ define <16 x i32>@test_int_x86_avx512_mask_cvt_ps2dq_512(<16 x float> %x0, <16 x
 declare <8 x double> @llvm.x86.avx512.mask.cvtps2pd.512(<8 x float>, <8 x double>, i8, i32)
 
 define <8 x double>@test_int_x86_avx512_mask_cvt_ps2pd_512(<8 x float> %x0, <8 x double> %x1, i8 %x2) {
-; CHECK-LABEL: test_int_x86_avx512_mask_cvt_ps2pd_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vcvtps2pd %ymm0, %zmm1 {%k1}
-; CHECK-NEXT:    vcvtps2pd {sae}, %ymm0, %zmm0
-; CHECK-NEXT:    vaddpd %zmm0, %zmm1, %zmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_int_x86_avx512_mask_cvt_ps2pd_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vcvtps2pd %ymm0, %zmm1 {%k1}
+; X64-NEXT:    vcvtps2pd {sae}, %ymm0, %zmm0
+; X64-NEXT:    vaddpd %zmm0, %zmm1, %zmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_int_x86_avx512_mask_cvt_ps2pd_512:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vcvtps2pd %ymm0, %zmm1 {%k1}
+; X86-NEXT:    vcvtps2pd {sae}, %ymm0, %zmm0
+; X86-NEXT:    vaddpd %zmm0, %zmm1, %zmm0
+; X86-NEXT:    retl
   %res = call <8 x double> @llvm.x86.avx512.mask.cvtps2pd.512(<8 x float> %x0, <8 x double> %x1, i8 %x2, i32 4)
   %res1 = call <8 x double> @llvm.x86.avx512.mask.cvtps2pd.512(<8 x float> %x0, <8 x double> %x1, i8 -1, i32 8)
   %res2 = fadd <8 x double> %res, %res1
@@ -3230,13 +4296,21 @@ define <8 x double>@test_int_x86_avx512_mask_cvt_ps2pd_512(<8 x float> %x0, <8 x
 declare <16 x i32> @llvm.x86.avx512.mask.cvtps2udq.512(<16 x float>, <16 x i32>, i16, i32)
 
 define <16 x i32>@test_int_x86_avx512_mask_cvt_ps2udq_512(<16 x float> %x0, <16 x i32> %x1, i16 %x2) {
-; CHECK-LABEL: test_int_x86_avx512_mask_cvt_ps2udq_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vcvtps2udq {ru-sae}, %zmm0, %zmm1 {%k1}
-; CHECK-NEXT:    vcvtps2udq {rn-sae}, %zmm0, %zmm0
-; CHECK-NEXT:    vpaddd %zmm0, %zmm1, %zmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_int_x86_avx512_mask_cvt_ps2udq_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vcvtps2udq {ru-sae}, %zmm0, %zmm1 {%k1}
+; X64-NEXT:    vcvtps2udq {rn-sae}, %zmm0, %zmm0
+; X64-NEXT:    vpaddd %zmm0, %zmm1, %zmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_int_x86_avx512_mask_cvt_ps2udq_512:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vcvtps2udq {ru-sae}, %zmm0, %zmm1 {%k1}
+; X86-NEXT:    vcvtps2udq {rn-sae}, %zmm0, %zmm0
+; X86-NEXT:    vpaddd %zmm0, %zmm1, %zmm0
+; X86-NEXT:    retl
   %res = call <16 x i32> @llvm.x86.avx512.mask.cvtps2udq.512(<16 x float> %x0, <16 x i32> %x1, i16 %x2, i32 2)
   %res1 = call <16 x i32> @llvm.x86.avx512.mask.cvtps2udq.512(<16 x float> %x0, <16 x i32> %x1, i16 -1, i32 0)
   %res2 = add <16 x i32> %res, %res1
@@ -3246,45 +4320,73 @@ define <16 x i32>@test_int_x86_avx512_mask_cvt_ps2udq_512(<16 x float> %x0, <16
 declare <8 x i32> @llvm.x86.avx512.mask.cvttpd2dq.512(<8 x double>, <8 x i32>, i8, i32)
 
 define <8 x i32>@test_int_x86_avx512_mask_cvtt_pd2dq_512(<8 x double> %x0, <8 x i32> %x1, i8 %x2) {
-; CHECK-LABEL: test_int_x86_avx512_mask_cvtt_pd2dq_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vcvttpd2dq %zmm0, %ymm1 {%k1}
-; CHECK-NEXT:    vcvttpd2dq {sae}, %zmm0, %ymm0
-; CHECK-NEXT:    vpaddd %ymm0, %ymm1, %ymm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_int_x86_avx512_mask_cvtt_pd2dq_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vcvttpd2dq %zmm0, %ymm1 {%k1}
+; X64-NEXT:    vcvttpd2dq {sae}, %zmm0, %ymm0
+; X64-NEXT:    vpaddd %ymm0, %ymm1, %ymm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_int_x86_avx512_mask_cvtt_pd2dq_512:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vcvttpd2dq %zmm0, %ymm1 {%k1}
+; X86-NEXT:    vcvttpd2dq {sae}, %zmm0, %ymm0
+; X86-NEXT:    vpaddd %ymm0, %ymm1, %ymm0
+; X86-NEXT:    retl
   %res = call <8 x i32> @llvm.x86.avx512.mask.cvttpd2dq.512(<8 x double> %x0, <8 x i32> %x1, i8 %x2, i32 4)
   %res1 = call <8 x i32> @llvm.x86.avx512.mask.cvttpd2dq.512(<8 x double> %x0, <8 x i32> %x1, i8 -1, i32 8)
   %res2 = add <8 x i32> %res, %res1
   ret <8 x i32> %res2
 }
 
-declare <16 x float> @llvm.x86.avx512.mask.cvtudq2ps.512(<16 x i32>, <16 x float>, i16, i32)
+declare <16 x float> @llvm.x86.avx512.uitofp.round.v16f32.v16i32(<16 x i32>, i32)
 
 define <16 x float>@test_int_x86_avx512_mask_cvt_udq2ps_512(<16 x i32> %x0, <16 x float> %x1, i16 %x2) {
-; CHECK-LABEL: test_int_x86_avx512_mask_cvt_udq2ps_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vcvtudq2ps %zmm0, %zmm1 {%k1}
-; CHECK-NEXT:    vcvtudq2ps {rn-sae}, %zmm0, %zmm0
-; CHECK-NEXT:    vaddps %zmm0, %zmm1, %zmm0
-; CHECK-NEXT:    retq
-  %res = call <16 x float> @llvm.x86.avx512.mask.cvtudq2ps.512(<16 x i32> %x0, <16 x float> %x1, i16 %x2, i32 4)
-  %res1 = call <16 x float> @llvm.x86.avx512.mask.cvtudq2ps.512(<16 x i32> %x0, <16 x float> %x1, i16 -1, i32 0)
-  %res2 = fadd <16 x float> %res, %res1
+; X64-LABEL: test_int_x86_avx512_mask_cvt_udq2ps_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vcvtudq2ps %zmm0, %zmm1 {%k1}
+; X64-NEXT:    vcvtudq2ps {rn-sae}, %zmm0, %zmm0
+; X64-NEXT:    vaddps %zmm0, %zmm1, %zmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_int_x86_avx512_mask_cvt_udq2ps_512:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vcvtudq2ps %zmm0, %zmm1 {%k1}
+; X86-NEXT:    vcvtudq2ps {rn-sae}, %zmm0, %zmm0
+; X86-NEXT:    vaddps %zmm0, %zmm1, %zmm0
+; X86-NEXT:    retl
+  %cvt = uitofp <16 x i32> %x0 to <16 x float>
+  %1 = bitcast i16 %x2 to <16 x i1>
+  %2 = select <16 x i1> %1, <16 x float> %cvt, <16 x float> %x1
+  %3 = call <16 x float> @llvm.x86.avx512.uitofp.round.v16f32.v16i32(<16 x i32> %x0, i32 0)
+  %res2 = fadd <16 x float> %2, %3
   ret <16 x float> %res2
 }
 
 declare <8 x i32> @llvm.x86.avx512.mask.cvttpd2udq.512(<8 x double>, <8 x i32>, i8, i32)
 
 define <8 x i32>@test_int_x86_avx512_mask_cvtt_pd2udq_512(<8 x double> %x0, <8 x i32> %x1, i8 %x2) {
-; CHECK-LABEL: test_int_x86_avx512_mask_cvtt_pd2udq_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vcvttpd2udq %zmm0, %ymm1 {%k1}
-; CHECK-NEXT:    vcvttpd2udq {sae}, %zmm0, %ymm0
-; CHECK-NEXT:    vpaddd %ymm0, %ymm1, %ymm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_int_x86_avx512_mask_cvtt_pd2udq_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vcvttpd2udq %zmm0, %ymm1 {%k1}
+; X64-NEXT:    vcvttpd2udq {sae}, %zmm0, %ymm0
+; X64-NEXT:    vpaddd %ymm0, %ymm1, %ymm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_int_x86_avx512_mask_cvtt_pd2udq_512:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vcvttpd2udq %zmm0, %ymm1 {%k1}
+; X86-NEXT:    vcvttpd2udq {sae}, %zmm0, %ymm0
+; X86-NEXT:    vpaddd %ymm0, %ymm1, %ymm0
+; X86-NEXT:    retl
   %res = call <8 x i32> @llvm.x86.avx512.mask.cvttpd2udq.512(<8 x double> %x0, <8 x i32> %x1, i8 %x2, i32 4)
   %res1 = call <8 x i32> @llvm.x86.avx512.mask.cvttpd2udq.512(<8 x double> %x0, <8 x i32> %x1, i8 -1, i32 8)
   %res2 = add <8 x i32> %res, %res1
@@ -3294,13 +4396,21 @@ define <8 x i32>@test_int_x86_avx512_mask_cvtt_pd2udq_512(<8 x double> %x0, <8 x
 declare <16 x i32> @llvm.x86.avx512.mask.cvttps2dq.512(<16 x float>, <16 x i32>, i16, i32)
 
 define <16 x i32>@test_int_x86_avx512_mask_cvtt_ps2dq_512(<16 x float> %x0, <16 x i32> %x1, i16 %x2) {
-; CHECK-LABEL: test_int_x86_avx512_mask_cvtt_ps2dq_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vcvttps2dq %zmm0, %zmm1 {%k1}
-; CHECK-NEXT:    vcvttps2dq {sae}, %zmm0, %zmm0
-; CHECK-NEXT:    vpaddd %zmm0, %zmm1, %zmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_int_x86_avx512_mask_cvtt_ps2dq_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vcvttps2dq %zmm0, %zmm1 {%k1}
+; X64-NEXT:    vcvttps2dq {sae}, %zmm0, %zmm0
+; X64-NEXT:    vpaddd %zmm0, %zmm1, %zmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_int_x86_avx512_mask_cvtt_ps2dq_512:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vcvttps2dq %zmm0, %zmm1 {%k1}
+; X86-NEXT:    vcvttps2dq {sae}, %zmm0, %zmm0
+; X86-NEXT:    vpaddd %zmm0, %zmm1, %zmm0
+; X86-NEXT:    retl
   %res = call <16 x i32> @llvm.x86.avx512.mask.cvttps2dq.512(<16 x float> %x0, <16 x i32> %x1, i16 %x2, i32 4)
   %res1 = call <16 x i32> @llvm.x86.avx512.mask.cvttps2dq.512(<16 x float> %x0, <16 x i32> %x1, i16 -1, i32 8)
   %res2 = add <16 x i32> %res, %res1
@@ -3310,13 +4420,21 @@ define <16 x i32>@test_int_x86_avx512_mask_cvtt_ps2dq_512(<16 x float> %x0, <16
 declare <16 x i32> @llvm.x86.avx512.mask.cvttps2udq.512(<16 x float>, <16 x i32>, i16, i32)
 
 define <16 x i32>@test_int_x86_avx512_mask_cvtt_ps2udq_512(<16 x float> %x0, <16 x i32> %x1, i16 %x2) {
-; CHECK-LABEL: test_int_x86_avx512_mask_cvtt_ps2udq_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vcvttps2udq %zmm0, %zmm1 {%k1}
-; CHECK-NEXT:    vcvttps2udq {sae}, %zmm0, %zmm0
-; CHECK-NEXT:    vpaddd %zmm0, %zmm1, %zmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_int_x86_avx512_mask_cvtt_ps2udq_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vcvttps2udq %zmm0, %zmm1 {%k1}
+; X64-NEXT:    vcvttps2udq {sae}, %zmm0, %zmm0
+; X64-NEXT:    vpaddd %zmm0, %zmm1, %zmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_int_x86_avx512_mask_cvtt_ps2udq_512:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vcvttps2udq %zmm0, %zmm1 {%k1}
+; X86-NEXT:    vcvttps2udq {sae}, %zmm0, %zmm0
+; X86-NEXT:    vpaddd %zmm0, %zmm1, %zmm0
+; X86-NEXT:    retl
   %res = call <16 x i32> @llvm.x86.avx512.mask.cvttps2udq.512(<16 x float> %x0, <16 x i32> %x1, i16 %x2, i32 4)
   %res1 = call <16 x i32> @llvm.x86.avx512.mask.cvttps2udq.512(<16 x float> %x0, <16 x i32> %x1, i16 -1, i32 8)
   %res2 = add <16 x i32> %res, %res1
@@ -3326,18 +4444,32 @@ define <16 x i32>@test_int_x86_avx512_mask_cvtt_ps2udq_512(<16 x float> %x0, <16
 declare <4 x float> @llvm.x86.avx512.mask.getexp.ss(<4 x float>, <4 x float>, <4 x float>, i8, i32) nounwind readnone
 
 define <4 x float> @test_getexp_ss(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, i8 %mask) {
-; CHECK-LABEL: test_getexp_ss:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vmovaps %xmm2, %xmm3
-; CHECK-NEXT:    vgetexpss %xmm1, %xmm0, %xmm3 {%k1}
-; CHECK-NEXT:    vgetexpss {sae}, %xmm1, %xmm0, %xmm4 {%k1} {z}
-; CHECK-NEXT:    vgetexpss {sae}, %xmm1, %xmm0, %xmm5
-; CHECK-NEXT:    vaddps %xmm5, %xmm4, %xmm4
-; CHECK-NEXT:    vgetexpss {sae}, %xmm1, %xmm0, %xmm2 {%k1}
-; CHECK-NEXT:    vaddps %xmm2, %xmm3, %xmm0
-; CHECK-NEXT:    vaddps %xmm4, %xmm0, %xmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_getexp_ss:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vmovaps %xmm2, %xmm3
+; X64-NEXT:    vgetexpss %xmm1, %xmm0, %xmm3 {%k1}
+; X64-NEXT:    vgetexpss {sae}, %xmm1, %xmm0, %xmm4 {%k1} {z}
+; X64-NEXT:    vgetexpss {sae}, %xmm1, %xmm0, %xmm5
+; X64-NEXT:    vaddps %xmm5, %xmm4, %xmm4
+; X64-NEXT:    vgetexpss {sae}, %xmm1, %xmm0, %xmm2 {%k1}
+; X64-NEXT:    vaddps %xmm2, %xmm3, %xmm0
+; X64-NEXT:    vaddps %xmm4, %xmm0, %xmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_getexp_ss:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vmovaps %xmm2, %xmm3
+; X86-NEXT:    vgetexpss %xmm1, %xmm0, %xmm3 {%k1}
+; X86-NEXT:    vgetexpss {sae}, %xmm1, %xmm0, %xmm2 {%k1}
+; X86-NEXT:    vaddps %xmm2, %xmm3, %xmm2
+; X86-NEXT:    vgetexpss {sae}, %xmm1, %xmm0, %xmm3 {%k1} {z}
+; X86-NEXT:    vgetexpss {sae}, %xmm1, %xmm0, %xmm0
+; X86-NEXT:    vaddps %xmm0, %xmm3, %xmm0
+; X86-NEXT:    vaddps %xmm0, %xmm2, %xmm0
+; X86-NEXT:    retl
   %res0 = call <4 x float> @llvm.x86.avx512.mask.getexp.ss(<4 x float>%a0, <4 x float> %a1, <4 x float> %a2, i8 %mask, i32 4)
   %res1 = call <4 x float> @llvm.x86.avx512.mask.getexp.ss(<4 x float>%a0, <4 x float> %a1, <4 x float> %a2, i8 %mask, i32 8)
   %res2 = call <4 x float> @llvm.x86.avx512.mask.getexp.ss(<4 x float>%a0, <4 x float> %a1, <4 x float> zeroinitializer, i8 %mask, i32 8)
@@ -3352,18 +4484,32 @@ define <4 x float> @test_getexp_ss(<4 x float> %a0, <4 x float> %a1, <4 x float>
 declare <2 x double> @llvm.x86.avx512.mask.getexp.sd(<2 x double>, <2 x double>, <2 x double>, i8, i32) nounwind readnone
 
 define <2 x double> @test_getexp_sd(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, i8 %mask) {
-; CHECK-LABEL: test_getexp_sd:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vgetexpsd %xmm1, %xmm0, %xmm3
-; CHECK-NEXT:    vmovapd %xmm2, %xmm4
-; CHECK-NEXT:    vgetexpsd %xmm1, %xmm0, %xmm4 {%k1}
-; CHECK-NEXT:    vgetexpsd {sae}, %xmm1, %xmm0, %xmm5 {%k1} {z}
-; CHECK-NEXT:    vaddpd %xmm3, %xmm5, %xmm3
-; CHECK-NEXT:    vgetexpsd {sae}, %xmm1, %xmm0, %xmm2 {%k1}
-; CHECK-NEXT:    vaddpd %xmm2, %xmm4, %xmm0
-; CHECK-NEXT:    vaddpd %xmm3, %xmm0, %xmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_getexp_sd:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vgetexpsd %xmm1, %xmm0, %xmm3
+; X64-NEXT:    vmovapd %xmm2, %xmm4
+; X64-NEXT:    vgetexpsd %xmm1, %xmm0, %xmm4 {%k1}
+; X64-NEXT:    vgetexpsd {sae}, %xmm1, %xmm0, %xmm5 {%k1} {z}
+; X64-NEXT:    vaddpd %xmm3, %xmm5, %xmm3
+; X64-NEXT:    vgetexpsd {sae}, %xmm1, %xmm0, %xmm2 {%k1}
+; X64-NEXT:    vaddpd %xmm2, %xmm4, %xmm0
+; X64-NEXT:    vaddpd %xmm3, %xmm0, %xmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_getexp_sd:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vmovapd %xmm2, %xmm3
+; X86-NEXT:    vgetexpsd {sae}, %xmm1, %xmm0, %xmm3 {%k1}
+; X86-NEXT:    vgetexpsd {sae}, %xmm1, %xmm0, %xmm4 {%k1} {z}
+; X86-NEXT:    vgetexpsd %xmm1, %xmm0, %xmm2 {%k1}
+; X86-NEXT:    vaddpd %xmm3, %xmm2, %xmm2
+; X86-NEXT:    vgetexpsd %xmm1, %xmm0, %xmm0
+; X86-NEXT:    vaddpd %xmm0, %xmm4, %xmm0
+; X86-NEXT:    vaddpd %xmm0, %xmm2, %xmm0
+; X86-NEXT:    retl
   %res0 = call <2 x double> @llvm.x86.avx512.mask.getexp.sd(<2 x double>%a0, <2 x double> %a1, <2 x double> %a2, i8 %mask, i32 4)
   %res1 = call <2 x double> @llvm.x86.avx512.mask.getexp.sd(<2 x double>%a0, <2 x double> %a1, <2 x double> %a2, i8 %mask, i32 8)
   %res2 = call <2 x double> @llvm.x86.avx512.mask.getexp.sd(<2 x double>%a0, <2 x double> %a1, <2 x double> zeroinitializer, i8 %mask, i32 8)
@@ -3378,35 +4524,67 @@ define <2 x double> @test_getexp_sd(<2 x double> %a0, <2 x double> %a1, <2 x dou
 declare i8 @llvm.x86.avx512.mask.cmp.sd(<2 x double>, <2 x double>, i32, i8, i32)
 
 define i8@test_int_x86_avx512_mask_cmp_sd(<2 x double> %x0, <2 x double> %x1, i8 %x3, i32 %x4) {
-; CHECK-LABEL: test_int_x86_avx512_mask_cmp_sd:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vcmpnltsd {sae}, %xmm1, %xmm0, %k0 {%k1}
-; CHECK-NEXT:    kmovw %k0, %eax
-; CHECK-NEXT:    ## kill: def $al killed $al killed $eax
-; CHECK-NEXT:    retq
+; X64-LABEL: test_int_x86_avx512_mask_cmp_sd:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vcmpnltsd {sae}, %xmm1, %xmm0, %k0 {%k1}
+; X64-NEXT:    kmovw %k0, %eax
+; X64-NEXT:    # kill: def $al killed $al killed $eax
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_int_x86_avx512_mask_cmp_sd:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vcmpnltsd {sae}, %xmm1, %xmm0, %k0 {%k1}
+; X86-NEXT:    kmovw %k0, %eax
+; X86-NEXT:    # kill: def $al killed $al killed $eax
+; X86-NEXT:    retl
 
   %res4 = call i8 @llvm.x86.avx512.mask.cmp.sd(<2 x double> %x0, <2 x double> %x1, i32 5, i8 %x3, i32 8)
   ret i8 %res4
 }
 
 define i8@test_int_x86_avx512_mask_cmp_sd_all(<2 x double> %x0, <2 x double> %x1, i8 %x3, i32 %x4) {
-; CHECK-LABEL: test_int_x86_avx512_mask_cmp_sd_all:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vcmplesd %xmm1, %xmm0, %k0
-; CHECK-NEXT:    kmovw %k0, %ecx
-; CHECK-NEXT:    vcmpunordsd {sae}, %xmm1, %xmm0, %k0
-; CHECK-NEXT:    kmovw %k0, %edx
-; CHECK-NEXT:    vcmpneqsd %xmm1, %xmm0, %k0 {%k1}
-; CHECK-NEXT:    kmovw %k0, %esi
-; CHECK-NEXT:    vcmpnltsd {sae}, %xmm1, %xmm0, %k0 {%k1}
-; CHECK-NEXT:    kmovw %k0, %eax
-; CHECK-NEXT:    orb %sil, %al
-; CHECK-NEXT:    orb %dl, %al
-; CHECK-NEXT:    orb %cl, %al
-; CHECK-NEXT:    ## kill: def $al killed $al killed $eax
-; CHECK-NEXT:    retq
+; X64-LABEL: test_int_x86_avx512_mask_cmp_sd_all:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vcmplesd %xmm1, %xmm0, %k0
+; X64-NEXT:    kmovw %k0, %ecx
+; X64-NEXT:    vcmpunordsd {sae}, %xmm1, %xmm0, %k0
+; X64-NEXT:    kmovw %k0, %edx
+; X64-NEXT:    vcmpneqsd %xmm1, %xmm0, %k0 {%k1}
+; X64-NEXT:    kmovw %k0, %esi
+; X64-NEXT:    vcmpnltsd {sae}, %xmm1, %xmm0, %k0 {%k1}
+; X64-NEXT:    kmovw %k0, %eax
+; X64-NEXT:    orb %sil, %al
+; X64-NEXT:    orb %dl, %al
+; X64-NEXT:    orb %cl, %al
+; X64-NEXT:    # kill: def $al killed $al killed $eax
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_int_x86_avx512_mask_cmp_sd_all:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %ebx
+; X86-NEXT:    .cfi_def_cfa_offset 8
+; X86-NEXT:    .cfi_offset %ebx, -8
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vcmplesd %xmm1, %xmm0, %k0
+; X86-NEXT:    kmovw %k0, %ecx
+; X86-NEXT:    vcmpunordsd {sae}, %xmm1, %xmm0, %k0
+; X86-NEXT:    kmovw %k0, %edx
+; X86-NEXT:    vcmpneqsd %xmm1, %xmm0, %k0 {%k1}
+; X86-NEXT:    kmovw %k0, %ebx
+; X86-NEXT:    vcmpnltsd {sae}, %xmm1, %xmm0, %k0 {%k1}
+; X86-NEXT:    kmovw %k0, %eax
+; X86-NEXT:    orb %bl, %al
+; X86-NEXT:    orb %dl, %al
+; X86-NEXT:    orb %cl, %al
+; X86-NEXT:    # kill: def $al killed $al killed $eax
+; X86-NEXT:    popl %ebx
+; X86-NEXT:    .cfi_def_cfa_offset 4
+; X86-NEXT:    retl
 
   %res1 = call i8 @llvm.x86.avx512.mask.cmp.sd(<2 x double> %x0, <2 x double> %x1, i32 2, i8 -1, i32 4)
   %res2 = call i8 @llvm.x86.avx512.mask.cmp.sd(<2 x double> %x0, <2 x double> %x1, i32 3, i8 -1, i32 8)
@@ -3422,13 +4600,22 @@ define i8@test_int_x86_avx512_mask_cmp_sd_all(<2 x double> %x0, <2 x double> %x1
 declare i8 @llvm.x86.avx512.mask.cmp.ss(<4 x float>, <4 x float>, i32, i8, i32)
 
 define i8@test_int_x86_avx512_mask_cmp_ss(<4 x float> %x0, <4 x float> %x1, i8 %x3, i32 %x4) {
-; CHECK-LABEL: test_int_x86_avx512_mask_cmp_ss:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vcmpunordss %xmm1, %xmm0, %k0 {%k1}
-; CHECK-NEXT:    kmovw %k0, %eax
-; CHECK-NEXT:    ## kill: def $al killed $al killed $eax
-; CHECK-NEXT:    retq
+; X64-LABEL: test_int_x86_avx512_mask_cmp_ss:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vcmpunordss %xmm1, %xmm0, %k0 {%k1}
+; X64-NEXT:    kmovw %k0, %eax
+; X64-NEXT:    # kill: def $al killed $al killed $eax
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_int_x86_avx512_mask_cmp_ss:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vcmpunordss %xmm1, %xmm0, %k0 {%k1}
+; X86-NEXT:    kmovw %k0, %eax
+; X86-NEXT:    # kill: def $al killed $al killed $eax
+; X86-NEXT:    retl
 
   %res2 = call i8 @llvm.x86.avx512.mask.cmp.ss(<4 x float> %x0, <4 x float> %x1, i32 3, i8 %x3, i32 4)
   ret i8 %res2
@@ -3436,22 +4623,45 @@ define i8@test_int_x86_avx512_mask_cmp_ss(<4 x float> %x0, <4 x float> %x1, i8 %
 
 
 define i8@test_int_x86_avx512_mask_cmp_ss_all(<4 x float> %x0, <4 x float> %x1, i8 %x3, i32 %x4) {
-; CHECK-LABEL: test_int_x86_avx512_mask_cmp_ss_all:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vcmpless %xmm1, %xmm0, %k0
-; CHECK-NEXT:    kmovw %k0, %ecx
-; CHECK-NEXT:    vcmpunordss {sae}, %xmm1, %xmm0, %k0
-; CHECK-NEXT:    kmovw %k0, %edx
-; CHECK-NEXT:    vcmpneqss %xmm1, %xmm0, %k0 {%k1}
-; CHECK-NEXT:    kmovw %k0, %esi
-; CHECK-NEXT:    vcmpnltss {sae}, %xmm1, %xmm0, %k0 {%k1}
-; CHECK-NEXT:    kmovw %k0, %eax
-; CHECK-NEXT:    andb %sil, %al
-; CHECK-NEXT:    andb %dl, %al
-; CHECK-NEXT:    andb %cl, %al
-; CHECK-NEXT:    ## kill: def $al killed $al killed $eax
-; CHECK-NEXT:    retq
+; X64-LABEL: test_int_x86_avx512_mask_cmp_ss_all:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vcmpless %xmm1, %xmm0, %k0
+; X64-NEXT:    kmovw %k0, %ecx
+; X64-NEXT:    vcmpunordss {sae}, %xmm1, %xmm0, %k0
+; X64-NEXT:    kmovw %k0, %edx
+; X64-NEXT:    vcmpneqss %xmm1, %xmm0, %k0 {%k1}
+; X64-NEXT:    kmovw %k0, %esi
+; X64-NEXT:    vcmpnltss {sae}, %xmm1, %xmm0, %k0 {%k1}
+; X64-NEXT:    kmovw %k0, %eax
+; X64-NEXT:    andb %sil, %al
+; X64-NEXT:    andb %dl, %al
+; X64-NEXT:    andb %cl, %al
+; X64-NEXT:    # kill: def $al killed $al killed $eax
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_int_x86_avx512_mask_cmp_ss_all:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %ebx
+; X86-NEXT:    .cfi_def_cfa_offset 8
+; X86-NEXT:    .cfi_offset %ebx, -8
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vcmpless %xmm1, %xmm0, %k0
+; X86-NEXT:    kmovw %k0, %ecx
+; X86-NEXT:    vcmpunordss {sae}, %xmm1, %xmm0, %k0
+; X86-NEXT:    kmovw %k0, %edx
+; X86-NEXT:    vcmpneqss %xmm1, %xmm0, %k0 {%k1}
+; X86-NEXT:    kmovw %k0, %ebx
+; X86-NEXT:    vcmpnltss {sae}, %xmm1, %xmm0, %k0 {%k1}
+; X86-NEXT:    kmovw %k0, %eax
+; X86-NEXT:    andb %bl, %al
+; X86-NEXT:    andb %dl, %al
+; X86-NEXT:    andb %cl, %al
+; X86-NEXT:    # kill: def $al killed $al killed $eax
+; X86-NEXT:    popl %ebx
+; X86-NEXT:    .cfi_def_cfa_offset 4
+; X86-NEXT:    retl
   %res1 = call i8 @llvm.x86.avx512.mask.cmp.ss(<4 x float> %x0, <4 x float> %x1, i32 2, i8 -1, i32 4)
   %res2 = call i8 @llvm.x86.avx512.mask.cmp.ss(<4 x float> %x0, <4 x float> %x1, i32 3, i8 -1, i32 8)
   %res3 = call i8 @llvm.x86.avx512.mask.cmp.ss(<4 x float> %x0, <4 x float> %x1, i32 4, i8 %x3, i32 4)
@@ -3466,13 +4676,22 @@ define i8@test_int_x86_avx512_mask_cmp_ss_all(<4 x float> %x0, <4 x float> %x1,
 declare <8 x double> @llvm.x86.avx512.mask.getmant.pd.512(<8 x double>, i32, <8 x double>, i8, i32)
 
 define <8 x double>@test_int_x86_avx512_mask_getmant_pd_512(<8 x double> %x0, <8 x double> %x2, i8 %x3) {
-; CHECK-LABEL: test_int_x86_avx512_mask_getmant_pd_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vgetmantpd $11, %zmm0, %zmm1 {%k1}
-; CHECK-NEXT:    vgetmantpd $11, {sae}, %zmm0, %zmm0
-; CHECK-NEXT:    vaddpd %zmm0, %zmm1, %zmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_int_x86_avx512_mask_getmant_pd_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vgetmantpd $11, %zmm0, %zmm1 {%k1}
+; X64-NEXT:    vgetmantpd $11, {sae}, %zmm0, %zmm0
+; X64-NEXT:    vaddpd %zmm0, %zmm1, %zmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_int_x86_avx512_mask_getmant_pd_512:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vgetmantpd $11, %zmm0, %zmm1 {%k1}
+; X86-NEXT:    vgetmantpd $11, {sae}, %zmm0, %zmm0
+; X86-NEXT:    vaddpd %zmm0, %zmm1, %zmm0
+; X86-NEXT:    retl
   %res = call <8 x double> @llvm.x86.avx512.mask.getmant.pd.512(<8 x double> %x0, i32 11, <8 x double> %x2, i8 %x3, i32 4)
   %res1 = call <8 x double> @llvm.x86.avx512.mask.getmant.pd.512(<8 x double> %x0, i32 11, <8 x double> %x2, i8 -1, i32 8)
   %res2 = fadd <8 x double> %res, %res1
@@ -3482,13 +4701,21 @@ define <8 x double>@test_int_x86_avx512_mask_getmant_pd_512(<8 x double> %x0, <8
 declare <16 x float> @llvm.x86.avx512.mask.getmant.ps.512(<16 x float>, i32, <16 x float>, i16, i32)
 
 define <16 x float>@test_int_x86_avx512_mask_getmant_ps_512(<16 x float> %x0, <16 x float> %x2, i16 %x3) {
-; CHECK-LABEL: test_int_x86_avx512_mask_getmant_ps_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vgetmantps $11, %zmm0, %zmm1 {%k1}
-; CHECK-NEXT:    vgetmantps $11, {sae}, %zmm0, %zmm0
-; CHECK-NEXT:    vaddps %zmm0, %zmm1, %zmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_int_x86_avx512_mask_getmant_ps_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vgetmantps $11, %zmm0, %zmm1 {%k1}
+; X64-NEXT:    vgetmantps $11, {sae}, %zmm0, %zmm0
+; X64-NEXT:    vaddps %zmm0, %zmm1, %zmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_int_x86_avx512_mask_getmant_ps_512:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vgetmantps $11, %zmm0, %zmm1 {%k1}
+; X86-NEXT:    vgetmantps $11, {sae}, %zmm0, %zmm0
+; X86-NEXT:    vaddps %zmm0, %zmm1, %zmm0
+; X86-NEXT:    retl
   %res = call <16 x float> @llvm.x86.avx512.mask.getmant.ps.512(<16 x float> %x0, i32 11, <16 x float> %x2, i16 %x3, i32 4)
   %res1 = call <16 x float> @llvm.x86.avx512.mask.getmant.ps.512(<16 x float> %x0, i32 11, <16 x float> %x2, i16 -1, i32 8)
   %res2 = fadd <16 x float> %res, %res1
@@ -3498,18 +4725,32 @@ define <16 x float>@test_int_x86_avx512_mask_getmant_ps_512(<16 x float> %x0, <1
 declare <2 x double> @llvm.x86.avx512.mask.getmant.sd(<2 x double>, <2 x double>, i32, <2 x double>, i8, i32)
 
 define <2 x double>@test_int_x86_avx512_mask_getmant_sd(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3) {
-; CHECK-LABEL: test_int_x86_avx512_mask_getmant_sd:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vgetmantsd $11, %xmm1, %xmm0, %xmm3
-; CHECK-NEXT:    vmovapd %xmm2, %xmm4
-; CHECK-NEXT:    vgetmantsd $11, %xmm1, %xmm0, %xmm4 {%k1}
-; CHECK-NEXT:    vgetmantsd $11, %xmm1, %xmm0, %xmm5 {%k1} {z}
-; CHECK-NEXT:    vaddpd %xmm5, %xmm4, %xmm4
-; CHECK-NEXT:    vgetmantsd $11, {sae}, %xmm1, %xmm0, %xmm2 {%k1}
-; CHECK-NEXT:    vaddpd %xmm3, %xmm2, %xmm0
-; CHECK-NEXT:    vaddpd %xmm0, %xmm4, %xmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_int_x86_avx512_mask_getmant_sd:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vgetmantsd $11, %xmm1, %xmm0, %xmm3
+; X64-NEXT:    vmovapd %xmm2, %xmm4
+; X64-NEXT:    vgetmantsd $11, %xmm1, %xmm0, %xmm4 {%k1}
+; X64-NEXT:    vgetmantsd $11, %xmm1, %xmm0, %xmm5 {%k1} {z}
+; X64-NEXT:    vaddpd %xmm5, %xmm4, %xmm4
+; X64-NEXT:    vgetmantsd $11, {sae}, %xmm1, %xmm0, %xmm2 {%k1}
+; X64-NEXT:    vaddpd %xmm3, %xmm2, %xmm0
+; X64-NEXT:    vaddpd %xmm0, %xmm4, %xmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_int_x86_avx512_mask_getmant_sd:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vmovapd %xmm2, %xmm3
+; X86-NEXT:    vgetmantsd $11, %xmm1, %xmm0, %xmm3 {%k1}
+; X86-NEXT:    vgetmantsd $11, %xmm1, %xmm0, %xmm4 {%k1} {z}
+; X86-NEXT:    vaddpd %xmm4, %xmm3, %xmm3
+; X86-NEXT:    vgetmantsd $11, %xmm1, %xmm0, %xmm4
+; X86-NEXT:    vgetmantsd $11, {sae}, %xmm1, %xmm0, %xmm2 {%k1}
+; X86-NEXT:    vaddpd %xmm4, %xmm2, %xmm0
+; X86-NEXT:    vaddpd %xmm0, %xmm3, %xmm0
+; X86-NEXT:    retl
   %res  = call <2 x double> @llvm.x86.avx512.mask.getmant.sd(<2 x double> %x0, <2 x double> %x1, i32 11, <2 x double> %x2, i8 %x3, i32 4)
   %res1 = call <2 x double> @llvm.x86.avx512.mask.getmant.sd(<2 x double> %x0, <2 x double> %x1, i32 11, <2 x double> zeroinitializer, i8 %x3, i32 4)
   %res2 = call <2 x double> @llvm.x86.avx512.mask.getmant.sd(<2 x double> %x0, <2 x double> %x1, i32 11, <2 x double> %x2, i8 %x3, i32 8)
@@ -3523,17 +4764,30 @@ define <2 x double>@test_int_x86_avx512_mask_getmant_sd(<2 x double> %x0, <2 x d
 declare <4 x float> @llvm.x86.avx512.mask.getmant.ss(<4 x float>, <4 x float>, i32, <4 x float>, i8, i32)
 
 define <4 x float>@test_int_x86_avx512_mask_getmant_ss(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3) {
-; CHECK-LABEL: test_int_x86_avx512_mask_getmant_ss:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vgetmantss $11, %xmm1, %xmm0, %xmm3
-; CHECK-NEXT:    vgetmantss $11, %xmm1, %xmm0, %xmm2 {%k1}
-; CHECK-NEXT:    vgetmantss $11, %xmm1, %xmm0, %xmm4 {%k1} {z}
-; CHECK-NEXT:    vaddps %xmm4, %xmm2, %xmm2
-; CHECK-NEXT:    vgetmantss $11, {sae}, %xmm1, %xmm0, %xmm0
-; CHECK-NEXT:    vaddps %xmm3, %xmm0, %xmm0
-; CHECK-NEXT:    vaddps %xmm0, %xmm2, %xmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_int_x86_avx512_mask_getmant_ss:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vgetmantss $11, %xmm1, %xmm0, %xmm3
+; X64-NEXT:    vgetmantss $11, %xmm1, %xmm0, %xmm2 {%k1}
+; X64-NEXT:    vgetmantss $11, %xmm1, %xmm0, %xmm4 {%k1} {z}
+; X64-NEXT:    vaddps %xmm4, %xmm2, %xmm2
+; X64-NEXT:    vgetmantss $11, {sae}, %xmm1, %xmm0, %xmm0
+; X64-NEXT:    vaddps %xmm3, %xmm0, %xmm0
+; X64-NEXT:    vaddps %xmm0, %xmm2, %xmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_int_x86_avx512_mask_getmant_ss:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vgetmantss $11, %xmm1, %xmm0, %xmm2 {%k1}
+; X86-NEXT:    vgetmantss $11, %xmm1, %xmm0, %xmm3 {%k1} {z}
+; X86-NEXT:    vaddps %xmm3, %xmm2, %xmm2
+; X86-NEXT:    vgetmantss $11, %xmm1, %xmm0, %xmm3
+; X86-NEXT:    vgetmantss $11, {sae}, %xmm1, %xmm0, %xmm0
+; X86-NEXT:    vaddps %xmm3, %xmm0, %xmm0
+; X86-NEXT:    vaddps %xmm0, %xmm2, %xmm0
+; X86-NEXT:    retl
   %res  = call <4 x float> @llvm.x86.avx512.mask.getmant.ss(<4 x float> %x0, <4 x float> %x1, i32 11, <4 x float> %x2, i8 %x3, i32 4)
   %res1 = call <4 x float> @llvm.x86.avx512.mask.getmant.ss(<4 x float> %x0, <4 x float> %x1, i32 11, <4 x float> zeroinitializer, i8 %x3, i32 4)
   %res2 = call <4 x float> @llvm.x86.avx512.mask.getmant.ss(<4 x float> %x0, <4 x float> %x1, i32 11, <4 x float> %x2, i8 -1, i32 8)
@@ -3548,20 +4802,28 @@ declare <8 x double> @llvm.x86.avx512.vpermilvar.pd.512(<8 x double>, <8 x i64>)
 
 define <8 x double>@test_int_x86_avx512_vpermilvar_pd_512(<8 x double> %x0, <8 x i64> %x1) {
 ; CHECK-LABEL: test_int_x86_avx512_vpermilvar_pd_512:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vpermilpd %zmm1, %zmm0, %zmm0
-; CHECK-NEXT:    retq
+; CHECK-NEXT:    ret{{[l|q]}}
   %res = call <8 x double> @llvm.x86.avx512.vpermilvar.pd.512(<8 x double> %x0, <8 x i64> %x1)
   ret <8 x double> %res
 }
 
 define <8 x double>@test_int_x86_avx512_vpermilvar_pd_512_mask(<8 x double> %x0, <8 x i64> %x1, <8 x double> %x2, i8 %mask) {
-; CHECK-LABEL: test_int_x86_avx512_vpermilvar_pd_512_mask:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vpermilpd %zmm1, %zmm0, %zmm2 {%k1}
-; CHECK-NEXT:    vmovapd %zmm2, %zmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_int_x86_avx512_vpermilvar_pd_512_mask:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vpermilpd %zmm1, %zmm0, %zmm2 {%k1}
+; X64-NEXT:    vmovapd %zmm2, %zmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_int_x86_avx512_vpermilvar_pd_512_mask:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vpermilpd %zmm1, %zmm0, %zmm2 {%k1}
+; X86-NEXT:    vmovapd %zmm2, %zmm0
+; X86-NEXT:    retl
   %res = call <8 x double> @llvm.x86.avx512.vpermilvar.pd.512(<8 x double> %x0, <8 x i64> %x1)
   %mask.cast = bitcast i8 %mask to <8 x i1>
   %res2 = select <8 x i1> %mask.cast, <8 x double> %res, <8 x double> %x2
@@ -3569,11 +4831,18 @@ define <8 x double>@test_int_x86_avx512_vpermilvar_pd_512_mask(<8 x double> %x0,
 }
 
 define <8 x double>@test_int_x86_avx512_vpermilvar_pd_512_maskz(<8 x double> %x0, <8 x i64> %x1, i8 %mask) {
-; CHECK-LABEL: test_int_x86_avx512_vpermilvar_pd_512_maskz:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vpermilpd %zmm1, %zmm0, %zmm0 {%k1} {z}
-; CHECK-NEXT:    retq
+; X64-LABEL: test_int_x86_avx512_vpermilvar_pd_512_maskz:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vpermilpd %zmm1, %zmm0, %zmm0 {%k1} {z}
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_int_x86_avx512_vpermilvar_pd_512_maskz:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vpermilpd %zmm1, %zmm0, %zmm0 {%k1} {z}
+; X86-NEXT:    retl
   %res = call <8 x double> @llvm.x86.avx512.vpermilvar.pd.512(<8 x double> %x0, <8 x i64> %x1)
   %mask.cast = bitcast i8 %mask to <8 x i1>
   %res2 = select <8 x i1> %mask.cast, <8 x double> %res, <8 x double> zeroinitializer
@@ -3584,20 +4853,27 @@ declare <16 x float> @llvm.x86.avx512.vpermilvar.ps.512(<16 x float>, <16 x i32>
 
 define <16 x float>@test_int_x86_avx512_vpermilvar_ps_512(<16 x float> %x0, <16 x i32> %x1) {
 ; CHECK-LABEL: test_int_x86_avx512_vpermilvar_ps_512:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vpermilps %zmm1, %zmm0, %zmm0
-; CHECK-NEXT:    retq
+; CHECK-NEXT:    ret{{[l|q]}}
   %res = call <16 x float> @llvm.x86.avx512.vpermilvar.ps.512(<16 x float> %x0, <16 x i32> %x1)
   ret <16 x float> %res
 }
 
 define <16 x float>@test_int_x86_avx512_vpermilvar_ps_512_mask(<16 x float> %x0, <16 x i32> %x1, <16 x float> %x2, i16 %mask) {
-; CHECK-LABEL: test_int_x86_avx512_vpermilvar_ps_512_mask:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vpermilps %zmm1, %zmm0, %zmm2 {%k1}
-; CHECK-NEXT:    vmovaps %zmm2, %zmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_int_x86_avx512_vpermilvar_ps_512_mask:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vpermilps %zmm1, %zmm0, %zmm2 {%k1}
+; X64-NEXT:    vmovaps %zmm2, %zmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_int_x86_avx512_vpermilvar_ps_512_mask:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vpermilps %zmm1, %zmm0, %zmm2 {%k1}
+; X86-NEXT:    vmovaps %zmm2, %zmm0
+; X86-NEXT:    retl
   %res = call <16 x float> @llvm.x86.avx512.vpermilvar.ps.512(<16 x float> %x0, <16 x i32> %x1)
   %mask.cast = bitcast i16 %mask to <16 x i1>
   %res2 = select <16 x i1> %mask.cast, <16 x float> %res, <16 x float> %x2
@@ -3605,11 +4881,17 @@ define <16 x float>@test_int_x86_avx512_vpermilvar_ps_512_mask(<16 x float> %x0,
 }
 
 define <16 x float>@test_int_x86_avx512_vpermilvar_ps_512_maskz(<16 x float> %x0, <16 x i32> %x1, i16 %mask) {
-; CHECK-LABEL: test_int_x86_avx512_vpermilvar_ps_512_maskz:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vpermilps %zmm1, %zmm0, %zmm0 {%k1} {z}
-; CHECK-NEXT:    retq
+; X64-LABEL: test_int_x86_avx512_vpermilvar_ps_512_maskz:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vpermilps %zmm1, %zmm0, %zmm0 {%k1} {z}
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_int_x86_avx512_vpermilvar_ps_512_maskz:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vpermilps %zmm1, %zmm0, %zmm0 {%k1} {z}
+; X86-NEXT:    retl
   %res = call <16 x float> @llvm.x86.avx512.vpermilvar.ps.512(<16 x float> %x0, <16 x i32> %x1)
   %mask.cast = bitcast i16 %mask to <16 x i1>
   %res2 = select <16 x i1> %mask.cast, <16 x float> %res, <16 x float> zeroinitializer
@@ -3619,20 +4901,27 @@ define <16 x float>@test_int_x86_avx512_vpermilvar_ps_512_maskz(<16 x float> %x0
 ; Test case to make sure we can print shuffle decode comments for constant pool loads.
 define <16 x float>@test_int_x86_avx512_vpermilvar_ps_512_constant_pool(<16 x float> %x0, <16 x i32> %x1) {
 ; CHECK-LABEL: test_int_x86_avx512_vpermilvar_ps_512_constant_pool:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vpermilps {{.*#+}} zmm0 = zmm0[1,0,3,2,4,5,6,7,10,11,8,9,14,15,13,12]
-; CHECK-NEXT:    retq
+; CHECK-NEXT:    ret{{[l|q]}}
   %res = call <16 x float> @llvm.x86.avx512.vpermilvar.ps.512(<16 x float> %x0, <16 x i32> <i32 1, i32 0, i32 3, i32 2, i32 0, i32 1, i32 2, i32 3, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 1, i32 0>)
   ret <16 x float> %res
 }
 
 define <16 x float>@test_int_x86_avx512_vpermilvar_ps_512_constant_pool_mask(<16 x float> %x0, <16 x i32> %x1, <16 x float> %x2, i16 %mask) {
-; CHECK-LABEL: test_int_x86_avx512_vpermilvar_ps_512_constant_pool_mask:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vpermilps {{.*#+}} zmm2 {%k1} = zmm0[1,0,3,2,4,5,6,7,10,11,8,9,14,15,13,12]
-; CHECK-NEXT:    vmovaps %zmm2, %zmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_int_x86_avx512_vpermilvar_ps_512_constant_pool_mask:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vpermilps {{.*#+}} zmm2 {%k1} = zmm0[1,0,3,2,4,5,6,7,10,11,8,9,14,15,13,12]
+; X64-NEXT:    vmovaps %zmm2, %zmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_int_x86_avx512_vpermilvar_ps_512_constant_pool_mask:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vpermilps {{.*#+}} zmm2 {%k1} = zmm0[1,0,3,2,4,5,6,7,10,11,8,9,14,15,13,12]
+; X86-NEXT:    vmovaps %zmm2, %zmm0
+; X86-NEXT:    retl
   %res = call <16 x float> @llvm.x86.avx512.vpermilvar.ps.512(<16 x float> %x0, <16 x i32> <i32 1, i32 0, i32 3, i32 2, i32 0, i32 1, i32 2, i32 3, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 1, i32 0>)
   %mask.cast = bitcast i16 %mask to <16 x i1>
   %res2 = select <16 x i1> %mask.cast, <16 x float> %res, <16 x float> %x2
@@ -3640,11 +4929,17 @@ define <16 x float>@test_int_x86_avx512_vpermilvar_ps_512_constant_pool_mask(<16
 }
 
 define <16 x float>@test_int_x86_avx512_vpermilvar_ps_512_constant_pool_maskz(<16 x float> %x0, <16 x i32> %x1, i16 %mask) {
-; CHECK-LABEL: test_int_x86_avx512_vpermilvar_ps_512_constant_pool_maskz:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vpermilps {{.*#+}} zmm0 {%k1} {z} = zmm0[1,0,3,2,4,5,6,7,10,11,8,9,14,15,13,12]
-; CHECK-NEXT:    retq
+; X64-LABEL: test_int_x86_avx512_vpermilvar_ps_512_constant_pool_maskz:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vpermilps {{.*#+}} zmm0 {%k1} {z} = zmm0[1,0,3,2,4,5,6,7,10,11,8,9,14,15,13,12]
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_int_x86_avx512_vpermilvar_ps_512_constant_pool_maskz:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vpermilps {{.*#+}} zmm0 {%k1} {z} = zmm0[1,0,3,2,4,5,6,7,10,11,8,9,14,15,13,12]
+; X86-NEXT:    retl
   %res = call <16 x float> @llvm.x86.avx512.vpermilvar.ps.512(<16 x float> %x0, <16 x i32> <i32 1, i32 0, i32 3, i32 2, i32 0, i32 1, i32 2, i32 3, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 1, i32 0>)
   %mask.cast = bitcast i16 %mask to <16 x i1>
   %res2 = select <16 x i1> %mask.cast, <16 x float> %res, <16 x float> zeroinitializer
@@ -3654,13 +4949,22 @@ define <16 x float>@test_int_x86_avx512_vpermilvar_ps_512_constant_pool_maskz(<1
 declare <2 x double> @llvm.x86.avx512.mask.cvtss2sd.round(<2 x double>, <4 x float>, <2 x double>, i8, i32)
 
 define <2 x double>@test_int_x86_avx512_mask_cvt_ss2sd_round(<2 x double> %x0,<4 x float> %x1, <2 x double> %x2, i8 %x3) {
-; CHECK-LABEL: test_int_x86_avx512_mask_cvt_ss2sd_round:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vcvtss2sd %xmm1, %xmm0, %xmm2 {%k1}
-; CHECK-NEXT:    vcvtss2sd {sae}, %xmm1, %xmm0, %xmm0
-; CHECK-NEXT:    vaddpd %xmm0, %xmm2, %xmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_int_x86_avx512_mask_cvt_ss2sd_round:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vcvtss2sd %xmm1, %xmm0, %xmm2 {%k1}
+; X64-NEXT:    vcvtss2sd {sae}, %xmm1, %xmm0, %xmm0
+; X64-NEXT:    vaddpd %xmm0, %xmm2, %xmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_int_x86_avx512_mask_cvt_ss2sd_round:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vcvtss2sd %xmm1, %xmm0, %xmm2 {%k1}
+; X86-NEXT:    vcvtss2sd {sae}, %xmm1, %xmm0, %xmm0
+; X86-NEXT:    vaddpd %xmm0, %xmm2, %xmm0
+; X86-NEXT:    retl
   %res = call <2 x double> @llvm.x86.avx512.mask.cvtss2sd.round(<2 x double> %x0, <4 x float> %x1, <2 x double> %x2, i8 %x3, i32 4)
   %res1 = call <2 x double> @llvm.x86.avx512.mask.cvtss2sd.round(<2 x double> %x0, <4 x float> %x1, <2 x double> %x2, i8 -1, i32 8)
   %res2 = fadd <2 x double> %res, %res1
@@ -3670,13 +4974,22 @@ define <2 x double>@test_int_x86_avx512_mask_cvt_ss2sd_round(<2 x double> %x0,<4
 declare <4 x float> @llvm.x86.avx512.mask.cvtsd2ss.round(<4 x float>, <2 x double>, <4 x float>, i8, i32)
 
 define <4 x float>@test_int_x86_avx512_mask_cvt_sd2ss_round(<4 x float> %x0,<2 x double> %x1, <4 x float> %x2, i8 %x3) {
-; CHECK-LABEL: test_int_x86_avx512_mask_cvt_sd2ss_round:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vcvtsd2ss {rz-sae}, %xmm1, %xmm0, %xmm2 {%k1}
-; CHECK-NEXT:    vcvtsd2ss {rn-sae}, %xmm1, %xmm0, %xmm0
-; CHECK-NEXT:    vaddps %xmm0, %xmm2, %xmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_int_x86_avx512_mask_cvt_sd2ss_round:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vcvtsd2ss {rz-sae}, %xmm1, %xmm0, %xmm2 {%k1}
+; X64-NEXT:    vcvtsd2ss {rn-sae}, %xmm1, %xmm0, %xmm0
+; X64-NEXT:    vaddps %xmm0, %xmm2, %xmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_int_x86_avx512_mask_cvt_sd2ss_round:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vcvtsd2ss {rz-sae}, %xmm1, %xmm0, %xmm2 {%k1}
+; X86-NEXT:    vcvtsd2ss {rn-sae}, %xmm1, %xmm0, %xmm0
+; X86-NEXT:    vaddps %xmm0, %xmm2, %xmm0
+; X86-NEXT:    retl
   %res = call <4 x float> @llvm.x86.avx512.mask.cvtsd2ss.round(<4 x float> %x0, <2 x double> %x1, <4 x float> %x2, i8 %x3, i32 3)
   %res1 = call <4 x float> @llvm.x86.avx512.mask.cvtsd2ss.round(<4 x float> %x0, <2 x double> %x1, <4 x float> %x2, i8 -1, i32 8)
   %res2 = fadd <4 x float> %res, %res1
@@ -3686,14 +4999,23 @@ define <4 x float>@test_int_x86_avx512_mask_cvt_sd2ss_round(<4 x float> %x0,<2 x
 declare <16 x i32> @llvm.x86.avx512.pternlog.d.512(<16 x i32>, <16 x i32>, <16 x i32>, i32)
 
 define <16 x i32>@test_int_x86_avx512_mask_pternlog_d_512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 %x4) {
-; CHECK-LABEL: test_int_x86_avx512_mask_pternlog_d_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    vmovdqa64 %zmm0, %zmm3
-; CHECK-NEXT:    vpternlogd $33, %zmm2, %zmm1, %zmm3
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vpternlogd $33, %zmm2, %zmm1, %zmm0 {%k1}
-; CHECK-NEXT:    vpaddd %zmm3, %zmm0, %zmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_int_x86_avx512_mask_pternlog_d_512:
+; X64:       # %bb.0:
+; X64-NEXT:    vmovdqa64 %zmm0, %zmm3
+; X64-NEXT:    vpternlogd $33, %zmm2, %zmm1, %zmm3
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vpternlogd $33, %zmm2, %zmm1, %zmm0 {%k1}
+; X64-NEXT:    vpaddd %zmm3, %zmm0, %zmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_int_x86_avx512_mask_pternlog_d_512:
+; X86:       # %bb.0:
+; X86-NEXT:    vmovdqa64 %zmm0, %zmm3
+; X86-NEXT:    vpternlogd $33, %zmm2, %zmm1, %zmm3
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vpternlogd $33, %zmm2, %zmm1, %zmm0 {%k1}
+; X86-NEXT:    vpaddd %zmm3, %zmm0, %zmm0
+; X86-NEXT:    retl
   %1 = call <16 x i32> @llvm.x86.avx512.pternlog.d.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i32 33)
   %2 = bitcast i16 %x4 to <16 x i1>
   %3 = select <16 x i1> %2, <16 x i32> %1, <16 x i32> %x0
@@ -3703,14 +5025,23 @@ define <16 x i32>@test_int_x86_avx512_mask_pternlog_d_512(<16 x i32> %x0, <16 x
 }
 
 define <16 x i32>@test_int_x86_avx512_maskz_pternlog_d_512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 %x4) {
-; CHECK-LABEL: test_int_x86_avx512_maskz_pternlog_d_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    vmovdqa64 %zmm0, %zmm3
-; CHECK-NEXT:    vpternlogd $33, %zmm2, %zmm1, %zmm3
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vpternlogd $33, %zmm2, %zmm1, %zmm0 {%k1} {z}
-; CHECK-NEXT:    vpaddd %zmm3, %zmm0, %zmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_int_x86_avx512_maskz_pternlog_d_512:
+; X64:       # %bb.0:
+; X64-NEXT:    vmovdqa64 %zmm0, %zmm3
+; X64-NEXT:    vpternlogd $33, %zmm2, %zmm1, %zmm3
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vpternlogd $33, %zmm2, %zmm1, %zmm0 {%k1} {z}
+; X64-NEXT:    vpaddd %zmm3, %zmm0, %zmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_int_x86_avx512_maskz_pternlog_d_512:
+; X86:       # %bb.0:
+; X86-NEXT:    vmovdqa64 %zmm0, %zmm3
+; X86-NEXT:    vpternlogd $33, %zmm2, %zmm1, %zmm3
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vpternlogd $33, %zmm2, %zmm1, %zmm0 {%k1} {z}
+; X86-NEXT:    vpaddd %zmm3, %zmm0, %zmm0
+; X86-NEXT:    retl
   %1 = call <16 x i32> @llvm.x86.avx512.pternlog.d.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i32 33)
   %2 = bitcast i16 %x4 to <16 x i1>
   %3 = select <16 x i1> %2, <16 x i32> %1, <16 x i32> zeroinitializer
@@ -3722,14 +5053,24 @@ define <16 x i32>@test_int_x86_avx512_maskz_pternlog_d_512(<16 x i32> %x0, <16 x
 declare <8 x i64> @llvm.x86.avx512.pternlog.q.512(<8 x i64>, <8 x i64>, <8 x i64>, i32)
 
 define <8 x i64>@test_int_x86_avx512_mask_pternlog_q_512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 %x4) {
-; CHECK-LABEL: test_int_x86_avx512_mask_pternlog_q_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    vmovdqa64 %zmm0, %zmm3
-; CHECK-NEXT:    vpternlogq $33, %zmm2, %zmm1, %zmm3
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vpternlogq $33, %zmm2, %zmm1, %zmm0 {%k1}
-; CHECK-NEXT:    vpaddq %zmm3, %zmm0, %zmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_int_x86_avx512_mask_pternlog_q_512:
+; X64:       # %bb.0:
+; X64-NEXT:    vmovdqa64 %zmm0, %zmm3
+; X64-NEXT:    vpternlogq $33, %zmm2, %zmm1, %zmm3
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vpternlogq $33, %zmm2, %zmm1, %zmm0 {%k1}
+; X64-NEXT:    vpaddq %zmm3, %zmm0, %zmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_int_x86_avx512_mask_pternlog_q_512:
+; X86:       # %bb.0:
+; X86-NEXT:    vmovdqa64 %zmm0, %zmm3
+; X86-NEXT:    vpternlogq $33, %zmm2, %zmm1, %zmm3
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vpternlogq $33, %zmm2, %zmm1, %zmm0 {%k1}
+; X86-NEXT:    vpaddq %zmm3, %zmm0, %zmm0
+; X86-NEXT:    retl
   %1 = call <8 x i64> @llvm.x86.avx512.pternlog.q.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i32 33)
   %2 = bitcast i8 %x4 to <8 x i1>
   %3 = select <8 x i1> %2, <8 x i64> %1, <8 x i64> %x0
@@ -3739,14 +5080,24 @@ define <8 x i64>@test_int_x86_avx512_mask_pternlog_q_512(<8 x i64> %x0, <8 x i64
 }
 
 define <8 x i64>@test_int_x86_avx512_maskz_pternlog_q_512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 %x4) {
-; CHECK-LABEL: test_int_x86_avx512_maskz_pternlog_q_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    vmovdqa64 %zmm0, %zmm3
-; CHECK-NEXT:    vpternlogq $33, %zmm2, %zmm1, %zmm3
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vpternlogq $33, %zmm2, %zmm1, %zmm0 {%k1} {z}
-; CHECK-NEXT:    vpaddq %zmm3, %zmm0, %zmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_int_x86_avx512_maskz_pternlog_q_512:
+; X64:       # %bb.0:
+; X64-NEXT:    vmovdqa64 %zmm0, %zmm3
+; X64-NEXT:    vpternlogq $33, %zmm2, %zmm1, %zmm3
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vpternlogq $33, %zmm2, %zmm1, %zmm0 {%k1} {z}
+; X64-NEXT:    vpaddq %zmm3, %zmm0, %zmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_int_x86_avx512_maskz_pternlog_q_512:
+; X86:       # %bb.0:
+; X86-NEXT:    vmovdqa64 %zmm0, %zmm3
+; X86-NEXT:    vpternlogq $33, %zmm2, %zmm1, %zmm3
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vpternlogq $33, %zmm2, %zmm1, %zmm0 {%k1} {z}
+; X86-NEXT:    vpaddq %zmm3, %zmm0, %zmm0
+; X86-NEXT:    retl
   %1 = call <8 x i64> @llvm.x86.avx512.pternlog.q.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i32 33)
   %2 = bitcast i8 %x4 to <8 x i1>
   %3 = select <8 x i1> %2, <8 x i64> %1, <8 x i64> zeroinitializer
@@ -3757,80 +5108,80 @@ define <8 x i64>@test_int_x86_avx512_maskz_pternlog_q_512(<8 x i64> %x0, <8 x i6
 
 define i32 @test_x86_avx512_comi_sd_eq_sae(<2 x double> %a0, <2 x double> %a1) {
 ; CHECK-LABEL: test_x86_avx512_comi_sd_eq_sae:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vcmpeqsd {sae}, %xmm1, %xmm0, %k0
 ; CHECK-NEXT:    kmovw %k0, %eax
-; CHECK-NEXT:    retq
+; CHECK-NEXT:    ret{{[l|q]}}
   %res = call i32 @llvm.x86.avx512.vcomi.sd(<2 x double> %a0, <2 x double> %a1, i32 0, i32 8)
   ret i32 %res
 }
 
 define i32 @test_x86_avx512_ucomi_sd_eq_sae(<2 x double> %a0, <2 x double> %a1) {
 ; CHECK-LABEL: test_x86_avx512_ucomi_sd_eq_sae:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vcmpeq_uqsd {sae}, %xmm1, %xmm0, %k0
 ; CHECK-NEXT:    kmovw %k0, %eax
-; CHECK-NEXT:    retq
+; CHECK-NEXT:    ret{{[l|q]}}
   %res = call i32 @llvm.x86.avx512.vcomi.sd(<2 x double> %a0, <2 x double> %a1, i32 8, i32 8)
   ret i32 %res
 }
 
 define i32 @test_x86_avx512_comi_sd_eq(<2 x double> %a0, <2 x double> %a1) {
 ; CHECK-LABEL: test_x86_avx512_comi_sd_eq:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vcmpeqsd %xmm1, %xmm0, %k0
 ; CHECK-NEXT:    kmovw %k0, %eax
-; CHECK-NEXT:    retq
+; CHECK-NEXT:    ret{{[l|q]}}
   %res = call i32 @llvm.x86.avx512.vcomi.sd(<2 x double> %a0, <2 x double> %a1, i32 0, i32 4)
   ret i32 %res
 }
 
 define i32 @test_x86_avx512_ucomi_sd_eq(<2 x double> %a0, <2 x double> %a1) {
 ; CHECK-LABEL: test_x86_avx512_ucomi_sd_eq:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vcmpeq_uqsd %xmm1, %xmm0, %k0
 ; CHECK-NEXT:    kmovw %k0, %eax
-; CHECK-NEXT:    retq
+; CHECK-NEXT:    ret{{[l|q]}}
   %res = call i32 @llvm.x86.avx512.vcomi.sd(<2 x double> %a0, <2 x double> %a1, i32 8, i32 4)
   ret i32 %res
 }
 
 define i32 @test_x86_avx512_comi_sd_lt_sae(<2 x double> %a0, <2 x double> %a1) {
 ; CHECK-LABEL: test_x86_avx512_comi_sd_lt_sae:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vcmpltsd {sae}, %xmm1, %xmm0, %k0
 ; CHECK-NEXT:    kmovw %k0, %eax
-; CHECK-NEXT:    retq
+; CHECK-NEXT:    ret{{[l|q]}}
   %res = call i32 @llvm.x86.avx512.vcomi.sd(<2 x double> %a0, <2 x double> %a1, i32 1, i32 8)
   ret i32 %res
 }
 
 define i32 @test_x86_avx512_ucomi_sd_lt_sae(<2 x double> %a0, <2 x double> %a1) {
 ; CHECK-LABEL: test_x86_avx512_ucomi_sd_lt_sae:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vcmpngesd {sae}, %xmm1, %xmm0, %k0
 ; CHECK-NEXT:    kmovw %k0, %eax
-; CHECK-NEXT:    retq
+; CHECK-NEXT:    ret{{[l|q]}}
   %res = call i32 @llvm.x86.avx512.vcomi.sd(<2 x double> %a0, <2 x double> %a1, i32 9, i32 8)
   ret i32 %res
 }
 
 define i32 @test_x86_avx512_comi_sd_lt(<2 x double> %a0, <2 x double> %a1) {
 ; CHECK-LABEL: test_x86_avx512_comi_sd_lt:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vcmpltsd %xmm1, %xmm0, %k0
 ; CHECK-NEXT:    kmovw %k0, %eax
-; CHECK-NEXT:    retq
+; CHECK-NEXT:    ret{{[l|q]}}
   %res = call i32 @llvm.x86.avx512.vcomi.sd(<2 x double> %a0, <2 x double> %a1, i32 1, i32 4)
   ret i32 %res
 }
 
 define i32 @test_x86_avx512_ucomi_sd_lt(<2 x double> %a0, <2 x double> %a1) {
 ; CHECK-LABEL: test_x86_avx512_ucomi_sd_lt:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vcmpngesd %xmm1, %xmm0, %k0
 ; CHECK-NEXT:    kmovw %k0, %eax
-; CHECK-NEXT:    retq
+; CHECK-NEXT:    ret{{[l|q]}}
   %res = call i32 @llvm.x86.avx512.vcomi.sd(<2 x double> %a0, <2 x double> %a1, i32 9, i32 4)
   ret i32 %res
 }
@@ -3839,220 +5190,39 @@ declare i32 @llvm.x86.avx512.vcomi.sd(<2 x double>, <2 x double>, i32, i32)
 
 define i32 @test_x86_avx512_ucomi_ss_lt(<4 x float> %a0, <4 x float> %a1) {
 ; CHECK-LABEL: test_x86_avx512_ucomi_ss_lt:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vcmpngess %xmm1, %xmm0, %k0
 ; CHECK-NEXT:    kmovw %k0, %eax
-; CHECK-NEXT:    retq
+; CHECK-NEXT:    ret{{[l|q]}}
   %res = call i32 @llvm.x86.avx512.vcomi.ss(<4 x float> %a0, <4 x float> %a1, i32 9, i32 4)
   ret i32 %res
 }
 
 declare i32 @llvm.x86.avx512.vcomi.ss(<4 x float>, <4 x float>, i32, i32)
 
-declare <16 x i32> @llvm.x86.avx512.prolv.d.512(<16 x i32>, <16 x i32>)
-
-define <16 x i32>@test_int_x86_avx512_mask_prolv_d_512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 %x3) {
-; CHECK-LABEL: test_int_x86_avx512_mask_prolv_d_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    vprolvd %zmm1, %zmm0, %zmm3
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vprolvd %zmm1, %zmm0, %zmm2 {%k1}
-; CHECK-NEXT:    vprolvd %zmm1, %zmm0, %zmm0 {%k1} {z}
-; CHECK-NEXT:    vpaddd %zmm3, %zmm0, %zmm0
-; CHECK-NEXT:    vpaddd %zmm0, %zmm2, %zmm0
-; CHECK-NEXT:    retq
-  %1 = call <16 x i32> @llvm.x86.avx512.prolv.d.512(<16 x i32> %x0, <16 x i32> %x1)
-  %2 = bitcast i16 %x3 to <16 x i1>
-  %3 = select <16 x i1> %2, <16 x i32> %1, <16 x i32> %x2
-  %4 = call <16 x i32> @llvm.x86.avx512.prolv.d.512(<16 x i32> %x0, <16 x i32> %x1)
-  %5 = bitcast i16 %x3 to <16 x i1>
-  %6 = select <16 x i1> %5, <16 x i32> %4, <16 x i32> zeroinitializer
-  %7 = call <16 x i32> @llvm.x86.avx512.prolv.d.512(<16 x i32> %x0, <16 x i32> %x1)
-  %res3 = add <16 x i32> %3, %6
-  %res4 = add <16 x i32> %res3, %7
-  ret <16 x i32> %res4
-}
-
-declare <8 x i64> @llvm.x86.avx512.prolv.q.512(<8 x i64>, <8 x i64>)
-
-define <8 x i64>@test_int_x86_avx512_mask_prolv_q_512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 %x3) {
-; CHECK-LABEL: test_int_x86_avx512_mask_prolv_q_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    vprolvq %zmm1, %zmm0, %zmm3
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vprolvq %zmm1, %zmm0, %zmm2 {%k1}
-; CHECK-NEXT:    vprolvq %zmm1, %zmm0, %zmm0 {%k1} {z}
-; CHECK-NEXT:    vpaddq %zmm3, %zmm0, %zmm0
-; CHECK-NEXT:    vpaddq %zmm0, %zmm2, %zmm0
-; CHECK-NEXT:    retq
-  %1 = call <8 x i64> @llvm.x86.avx512.prolv.q.512(<8 x i64> %x0, <8 x i64> %x1)
-  %2 = bitcast i8 %x3 to <8 x i1>
-  %3 = select <8 x i1> %2, <8 x i64> %1, <8 x i64> %x2
-  %4 = call <8 x i64> @llvm.x86.avx512.prolv.q.512(<8 x i64> %x0, <8 x i64> %x1)
-  %5 = bitcast i8 %x3 to <8 x i1>
-  %6 = select <8 x i1> %5, <8 x i64> %4, <8 x i64> zeroinitializer
-  %7 = call <8 x i64> @llvm.x86.avx512.prolv.q.512(<8 x i64> %x0, <8 x i64> %x1)
-  %res3 = add <8 x i64> %3, %6
-  %res4 = add <8 x i64> %res3, %7
-  ret <8 x i64> %res4
-}
-
-declare <16 x i32> @llvm.x86.avx512.prorv.d.512(<16 x i32>, <16 x i32>)
-
-define <16 x i32>@test_int_x86_avx512_mask_prorv_d_512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 %x3) {
-; CHECK-LABEL: test_int_x86_avx512_mask_prorv_d_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    vprorvd %zmm1, %zmm0, %zmm3
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vprorvd %zmm1, %zmm0, %zmm2 {%k1}
-; CHECK-NEXT:    vprorvd %zmm1, %zmm0, %zmm0 {%k1} {z}
-; CHECK-NEXT:    vpaddd %zmm3, %zmm0, %zmm0
-; CHECK-NEXT:    vpaddd %zmm0, %zmm2, %zmm0
-; CHECK-NEXT:    retq
-  %1 = call <16 x i32> @llvm.x86.avx512.prorv.d.512(<16 x i32> %x0, <16 x i32> %x1)
-  %2 = bitcast i16 %x3 to <16 x i1>
-  %3 = select <16 x i1> %2, <16 x i32> %1, <16 x i32> %x2
-  %4 = call <16 x i32> @llvm.x86.avx512.prorv.d.512(<16 x i32> %x0, <16 x i32> %x1)
-  %5 = bitcast i16 %x3 to <16 x i1>
-  %6 = select <16 x i1> %5, <16 x i32> %4, <16 x i32> zeroinitializer
-  %7 = call <16 x i32> @llvm.x86.avx512.prorv.d.512(<16 x i32> %x0, <16 x i32> %x1)
-  %res3 = add <16 x i32> %3, %6
-  %res4 = add <16 x i32> %res3, %7
-  ret <16 x i32> %res4
-}
-
-declare <8 x i64> @llvm.x86.avx512.prorv.q.512(<8 x i64>, <8 x i64>)
-
-define <8 x i64>@test_int_x86_avx512_mask_prorv_q_512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 %x3) {
-; CHECK-LABEL: test_int_x86_avx512_mask_prorv_q_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    vprorvq %zmm1, %zmm0, %zmm3
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vprorvq %zmm1, %zmm0, %zmm2 {%k1}
-; CHECK-NEXT:    vprorvq %zmm1, %zmm0, %zmm0 {%k1} {z}
-; CHECK-NEXT:    vpaddq %zmm3, %zmm0, %zmm0
-; CHECK-NEXT:    vpaddq %zmm0, %zmm2, %zmm0
-; CHECK-NEXT:    retq
-  %1 = call <8 x i64> @llvm.x86.avx512.prorv.q.512(<8 x i64> %x0, <8 x i64> %x1)
-  %2 = bitcast i8 %x3 to <8 x i1>
-  %3 = select <8 x i1> %2, <8 x i64> %1, <8 x i64> %x2
-  %4 = call <8 x i64> @llvm.x86.avx512.prorv.q.512(<8 x i64> %x0, <8 x i64> %x1)
-  %5 = bitcast i8 %x3 to <8 x i1>
-  %6 = select <8 x i1> %5, <8 x i64> %4, <8 x i64> zeroinitializer
-  %7 = call <8 x i64> @llvm.x86.avx512.prorv.q.512(<8 x i64> %x0, <8 x i64> %x1)
-  %res3 = add <8 x i64> %3, %6
-  %res4 = add <8 x i64> %res3, %7
-  ret <8 x i64> %res4
-}
-
-declare <16 x i32> @llvm.x86.avx512.prol.d.512(<16 x i32>, i32)
-
-define <16 x i32>@test_int_x86_avx512_mask_prol_d_512(<16 x i32> %x0, i32 %x1, <16 x i32> %x2, i16 %x3) {
-; CHECK-LABEL: test_int_x86_avx512_mask_prol_d_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    vprold $3, %zmm0, %zmm2
-; CHECK-NEXT:    kmovw %esi, %k1
-; CHECK-NEXT:    vprold $3, %zmm0, %zmm1 {%k1}
-; CHECK-NEXT:    vprold $3, %zmm0, %zmm0 {%k1} {z}
-; CHECK-NEXT:    vpaddd %zmm2, %zmm0, %zmm0
-; CHECK-NEXT:    vpaddd %zmm0, %zmm1, %zmm0
-; CHECK-NEXT:    retq
-  %1 = call <16 x i32> @llvm.x86.avx512.prol.d.512(<16 x i32> %x0, i32 3)
-  %2 = bitcast i16 %x3 to <16 x i1>
-  %3 = select <16 x i1> %2, <16 x i32> %1, <16 x i32> %x2
-  %4 = call <16 x i32> @llvm.x86.avx512.prol.d.512(<16 x i32> %x0, i32 3)
-  %5 = bitcast i16 %x3 to <16 x i1>
-  %6 = select <16 x i1> %5, <16 x i32> %4, <16 x i32> zeroinitializer
-  %7 = call <16 x i32> @llvm.x86.avx512.prol.d.512(<16 x i32> %x0, i32 3)
-  %res3 = add <16 x i32> %3, %6
-  %res4 = add <16 x i32> %res3, %7
-  ret <16 x i32> %res4
-}
-
-declare <8 x i64> @llvm.x86.avx512.prol.q.512(<8 x i64>, i32)
-
-define <8 x i64>@test_int_x86_avx512_mask_prol_q_512(<8 x i64> %x0, i32 %x1, <8 x i64> %x2, i8 %x3) {
-; CHECK-LABEL: test_int_x86_avx512_mask_prol_q_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    vprolq $3, %zmm0, %zmm2
-; CHECK-NEXT:    kmovw %esi, %k1
-; CHECK-NEXT:    vprolq $3, %zmm0, %zmm1 {%k1}
-; CHECK-NEXT:    vprolq $3, %zmm0, %zmm0 {%k1} {z}
-; CHECK-NEXT:    vpaddq %zmm2, %zmm0, %zmm0
-; CHECK-NEXT:    vpaddq %zmm0, %zmm1, %zmm0
-; CHECK-NEXT:    retq
-  %1 = call <8 x i64> @llvm.x86.avx512.prol.q.512(<8 x i64> %x0, i32 3)
-  %2 = bitcast i8 %x3 to <8 x i1>
-  %3 = select <8 x i1> %2, <8 x i64> %1, <8 x i64> %x2
-  %4 = call <8 x i64> @llvm.x86.avx512.prol.q.512(<8 x i64> %x0, i32 3)
-  %5 = bitcast i8 %x3 to <8 x i1>
-  %6 = select <8 x i1> %5, <8 x i64> %4, <8 x i64> zeroinitializer
-  %7 = call <8 x i64> @llvm.x86.avx512.prol.q.512(<8 x i64> %x0, i32 3)
-  %res3 = add <8 x i64> %3, %6
-  %res4 = add <8 x i64> %res3, %7
-  ret <8 x i64> %res4
-}
-
-declare <16 x i32> @llvm.x86.avx512.pror.d.512(<16 x i32>, i32)
-
-define <16 x i32>@test_int_x86_avx512_mask_pror_d_512(<16 x i32> %x0, i32 %x1, <16 x i32> %x2, i16 %x3) {
-; CHECK-LABEL: test_int_x86_avx512_mask_pror_d_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    vprord $3, %zmm0, %zmm2
-; CHECK-NEXT:    kmovw %esi, %k1
-; CHECK-NEXT:    vprord $3, %zmm0, %zmm1 {%k1}
-; CHECK-NEXT:    vprord $3, %zmm0, %zmm0 {%k1} {z}
-; CHECK-NEXT:    vpaddd %zmm2, %zmm0, %zmm0
-; CHECK-NEXT:    vpaddd %zmm0, %zmm1, %zmm0
-; CHECK-NEXT:    retq
-  %1 = call <16 x i32> @llvm.x86.avx512.pror.d.512(<16 x i32> %x0, i32 3)
-  %2 = bitcast i16 %x3 to <16 x i1>
-  %3 = select <16 x i1> %2, <16 x i32> %1, <16 x i32> %x2
-  %4 = call <16 x i32> @llvm.x86.avx512.pror.d.512(<16 x i32> %x0, i32 3)
-  %5 = bitcast i16 %x3 to <16 x i1>
-  %6 = select <16 x i1> %5, <16 x i32> %4, <16 x i32> zeroinitializer
-  %7 = call <16 x i32> @llvm.x86.avx512.pror.d.512(<16 x i32> %x0, i32 3)
-  %res3 = add <16 x i32> %3, %6
-  %res4 = add <16 x i32> %res3, %7
-  ret <16 x i32> %res4
-}
-
-declare <8 x i64> @llvm.x86.avx512.pror.q.512(<8 x i64>, i32)
-
-define <8 x i64>@test_int_x86_avx512_mask_pror_q_512(<8 x i64> %x0, i32 %x1, <8 x i64> %x2, i8 %x3) {
-; CHECK-LABEL: test_int_x86_avx512_mask_pror_q_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    vprorq $3, %zmm0, %zmm2
-; CHECK-NEXT:    kmovw %esi, %k1
-; CHECK-NEXT:    vprorq $3, %zmm0, %zmm1 {%k1}
-; CHECK-NEXT:    vprorq $3, %zmm0, %zmm0 {%k1} {z}
-; CHECK-NEXT:    vpaddq %zmm2, %zmm0, %zmm0
-; CHECK-NEXT:    vpaddq %zmm0, %zmm1, %zmm0
-; CHECK-NEXT:    retq
-  %1 = call <8 x i64> @llvm.x86.avx512.pror.q.512(<8 x i64> %x0, i32 3)
-  %2 = bitcast i8 %x3 to <8 x i1>
-  %3 = select <8 x i1> %2, <8 x i64> %1, <8 x i64> %x2
-  %4 = call <8 x i64> @llvm.x86.avx512.pror.q.512(<8 x i64> %x0, i32 3)
-  %5 = bitcast i8 %x3 to <8 x i1>
-  %6 = select <8 x i1> %5, <8 x i64> %4, <8 x i64> zeroinitializer
-  %7 = call <8 x i64> @llvm.x86.avx512.pror.q.512(<8 x i64> %x0, i32 3)
-  %res3 = add <8 x i64> %3, %6
-  %res4 = add <8 x i64> %res3, %7
-  ret <8 x i64> %res4
-}
-
 declare <8 x double> @llvm.x86.avx512.permvar.df.512(<8 x double>, <8 x i64>)
 
 define <8 x double>@test_int_x86_avx512_mask_permvar_df_512(<8 x double> %x0, <8 x i64> %x1, <8 x double> %x2, i8 %x3) {
-; CHECK-LABEL: test_int_x86_avx512_mask_permvar_df_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    vpermpd %zmm0, %zmm1, %zmm3
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vpermpd %zmm0, %zmm1, %zmm2 {%k1}
-; CHECK-NEXT:    vpermpd %zmm0, %zmm1, %zmm0 {%k1} {z}
-; CHECK-NEXT:    vaddpd %zmm0, %zmm2, %zmm0
-; CHECK-NEXT:    vaddpd %zmm3, %zmm0, %zmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_int_x86_avx512_mask_permvar_df_512:
+; X64:       # %bb.0:
+; X64-NEXT:    vpermpd %zmm0, %zmm1, %zmm3
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vpermpd %zmm0, %zmm1, %zmm2 {%k1}
+; X64-NEXT:    vpermpd %zmm0, %zmm1, %zmm0 {%k1} {z}
+; X64-NEXT:    vaddpd %zmm0, %zmm2, %zmm0
+; X64-NEXT:    vaddpd %zmm3, %zmm0, %zmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_int_x86_avx512_mask_permvar_df_512:
+; X86:       # %bb.0:
+; X86-NEXT:    vpermpd %zmm0, %zmm1, %zmm3
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vpermpd %zmm0, %zmm1, %zmm2 {%k1}
+; X86-NEXT:    vpermpd %zmm0, %zmm1, %zmm0 {%k1} {z}
+; X86-NEXT:    vaddpd %zmm0, %zmm2, %zmm0
+; X86-NEXT:    vaddpd %zmm3, %zmm0, %zmm0
+; X86-NEXT:    retl
   %1 = call <8 x double> @llvm.x86.avx512.permvar.df.512(<8 x double> %x0, <8 x i64> %x1)
   %2 = bitcast i8 %x3 to <8 x i1>
   %3 = select <8 x i1> %2, <8 x double> %1, <8 x double> %x2
@@ -4068,15 +5238,26 @@ define <8 x double>@test_int_x86_avx512_mask_permvar_df_512(<8 x double> %x0, <8
 declare <8 x i64> @llvm.x86.avx512.permvar.di.512(<8 x i64>, <8 x i64>)
 
 define <8 x i64>@test_int_x86_avx512_mask_permvar_di_512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 %x3) {
-; CHECK-LABEL: test_int_x86_avx512_mask_permvar_di_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    vpermq %zmm0, %zmm1, %zmm3
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vpermq %zmm0, %zmm1, %zmm2 {%k1}
-; CHECK-NEXT:    vpermq %zmm0, %zmm1, %zmm0 {%k1} {z}
-; CHECK-NEXT:    vpaddq %zmm3, %zmm0, %zmm0
-; CHECK-NEXT:    vpaddq %zmm0, %zmm2, %zmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_int_x86_avx512_mask_permvar_di_512:
+; X64:       # %bb.0:
+; X64-NEXT:    vpermq %zmm0, %zmm1, %zmm3
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vpermq %zmm0, %zmm1, %zmm2 {%k1}
+; X64-NEXT:    vpermq %zmm0, %zmm1, %zmm0 {%k1} {z}
+; X64-NEXT:    vpaddq %zmm3, %zmm0, %zmm0
+; X64-NEXT:    vpaddq %zmm0, %zmm2, %zmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_int_x86_avx512_mask_permvar_di_512:
+; X86:       # %bb.0:
+; X86-NEXT:    vpermq %zmm0, %zmm1, %zmm3
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vpermq %zmm0, %zmm1, %zmm2 {%k1}
+; X86-NEXT:    vpermq %zmm0, %zmm1, %zmm0 {%k1} {z}
+; X86-NEXT:    vpaddq %zmm3, %zmm0, %zmm0
+; X86-NEXT:    vpaddq %zmm0, %zmm2, %zmm0
+; X86-NEXT:    retl
   %1 = call <8 x i64> @llvm.x86.avx512.permvar.di.512(<8 x i64> %x0, <8 x i64> %x1)
   %2 = bitcast i8 %x3 to <8 x i1>
   %3 = select <8 x i1> %2, <8 x i64> %1, <8 x i64> %x2
@@ -4092,15 +5273,25 @@ define <8 x i64>@test_int_x86_avx512_mask_permvar_di_512(<8 x i64> %x0, <8 x i64
 declare <16 x float> @llvm.x86.avx512.permvar.sf.512(<16 x float>, <16 x i32>)
 
 define <16 x float>@test_int_x86_avx512_mask_permvar_sf_512(<16 x float> %x0, <16 x i32> %x1, <16 x float> %x2, i16 %x3) {
-; CHECK-LABEL: test_int_x86_avx512_mask_permvar_sf_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    vpermps %zmm0, %zmm1, %zmm3
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vpermps %zmm0, %zmm1, %zmm2 {%k1}
-; CHECK-NEXT:    vpermps %zmm0, %zmm1, %zmm0 {%k1} {z}
-; CHECK-NEXT:    vaddps %zmm0, %zmm2, %zmm0
-; CHECK-NEXT:    vaddps %zmm3, %zmm0, %zmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_int_x86_avx512_mask_permvar_sf_512:
+; X64:       # %bb.0:
+; X64-NEXT:    vpermps %zmm0, %zmm1, %zmm3
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vpermps %zmm0, %zmm1, %zmm2 {%k1}
+; X64-NEXT:    vpermps %zmm0, %zmm1, %zmm0 {%k1} {z}
+; X64-NEXT:    vaddps %zmm0, %zmm2, %zmm0
+; X64-NEXT:    vaddps %zmm3, %zmm0, %zmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_int_x86_avx512_mask_permvar_sf_512:
+; X86:       # %bb.0:
+; X86-NEXT:    vpermps %zmm0, %zmm1, %zmm3
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vpermps %zmm0, %zmm1, %zmm2 {%k1}
+; X86-NEXT:    vpermps %zmm0, %zmm1, %zmm0 {%k1} {z}
+; X86-NEXT:    vaddps %zmm0, %zmm2, %zmm0
+; X86-NEXT:    vaddps %zmm3, %zmm0, %zmm0
+; X86-NEXT:    retl
   %1 = call <16 x float> @llvm.x86.avx512.permvar.sf.512(<16 x float> %x0, <16 x i32> %x1)
   %2 = bitcast i16 %x3 to <16 x i1>
   %3 = select <16 x i1> %2, <16 x float> %1, <16 x float> %x2
@@ -4116,15 +5307,25 @@ define <16 x float>@test_int_x86_avx512_mask_permvar_sf_512(<16 x float> %x0, <1
 declare <16 x i32> @llvm.x86.avx512.permvar.si.512(<16 x i32>, <16 x i32>)
 
 define <16 x i32>@test_int_x86_avx512_mask_permvar_si_512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 %x3) {
-; CHECK-LABEL: test_int_x86_avx512_mask_permvar_si_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    vpermd %zmm0, %zmm1, %zmm3
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vpermd %zmm0, %zmm1, %zmm2 {%k1}
-; CHECK-NEXT:    vpermd %zmm0, %zmm1, %zmm0 {%k1} {z}
-; CHECK-NEXT:    vpaddd %zmm3, %zmm0, %zmm0
-; CHECK-NEXT:    vpaddd %zmm0, %zmm2, %zmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_int_x86_avx512_mask_permvar_si_512:
+; X64:       # %bb.0:
+; X64-NEXT:    vpermd %zmm0, %zmm1, %zmm3
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vpermd %zmm0, %zmm1, %zmm2 {%k1}
+; X64-NEXT:    vpermd %zmm0, %zmm1, %zmm0 {%k1} {z}
+; X64-NEXT:    vpaddd %zmm3, %zmm0, %zmm0
+; X64-NEXT:    vpaddd %zmm0, %zmm2, %zmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_int_x86_avx512_mask_permvar_si_512:
+; X86:       # %bb.0:
+; X86-NEXT:    vpermd %zmm0, %zmm1, %zmm3
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vpermd %zmm0, %zmm1, %zmm2 {%k1}
+; X86-NEXT:    vpermd %zmm0, %zmm1, %zmm0 {%k1} {z}
+; X86-NEXT:    vpaddd %zmm3, %zmm0, %zmm0
+; X86-NEXT:    vpaddd %zmm0, %zmm2, %zmm0
+; X86-NEXT:    retl
   %1 = call <16 x i32> @llvm.x86.avx512.permvar.si.512(<16 x i32> %x0, <16 x i32> %x1)
   %2 = bitcast i16 %x3 to <16 x i1>
   %3 = select <16 x i1> %2, <16 x i32> %1, <16 x i32> %x2
@@ -4140,17 +5341,30 @@ define <16 x i32>@test_int_x86_avx512_mask_permvar_si_512(<16 x i32> %x0, <16 x
 declare <8 x double> @llvm.x86.avx512.mask.fixupimm.pd.512(<8 x double>, <8 x double>, <8 x i64>, i32, i8, i32)
 
 define <8 x double>@test_int_x86_avx512_mask_fixupimm_pd_512(<8 x double> %x0, <8 x double> %x1, <8 x i64> %x2, i8 %x4) {
-; CHECK-LABEL: test_int_x86_avx512_mask_fixupimm_pd_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vmovapd %zmm0, %zmm3
-; CHECK-NEXT:    vfixupimmpd $4, %zmm2, %zmm1, %zmm3 {%k1}
-; CHECK-NEXT:    vxorpd %xmm4, %xmm4, %xmm4
-; CHECK-NEXT:    vfixupimmpd $5, %zmm2, %zmm1, %zmm4 {%k1} {z}
-; CHECK-NEXT:    vaddpd %zmm4, %zmm3, %zmm3
-; CHECK-NEXT:    vfixupimmpd $3, {sae}, %zmm2, %zmm1, %zmm0
-; CHECK-NEXT:    vaddpd %zmm0, %zmm3, %zmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_int_x86_avx512_mask_fixupimm_pd_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vmovapd %zmm0, %zmm3
+; X64-NEXT:    vfixupimmpd $4, %zmm2, %zmm1, %zmm3 {%k1}
+; X64-NEXT:    vxorpd %xmm4, %xmm4, %xmm4
+; X64-NEXT:    vfixupimmpd $5, %zmm2, %zmm1, %zmm4 {%k1} {z}
+; X64-NEXT:    vaddpd %zmm4, %zmm3, %zmm3
+; X64-NEXT:    vfixupimmpd $3, {sae}, %zmm2, %zmm1, %zmm0
+; X64-NEXT:    vaddpd %zmm0, %zmm3, %zmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_int_x86_avx512_mask_fixupimm_pd_512:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vmovapd %zmm0, %zmm3
+; X86-NEXT:    vfixupimmpd $4, %zmm2, %zmm1, %zmm3 {%k1}
+; X86-NEXT:    vxorpd %xmm4, %xmm4, %xmm4
+; X86-NEXT:    vfixupimmpd $5, %zmm2, %zmm1, %zmm4 {%k1} {z}
+; X86-NEXT:    vaddpd %zmm4, %zmm3, %zmm3
+; X86-NEXT:    vfixupimmpd $3, {sae}, %zmm2, %zmm1, %zmm0
+; X86-NEXT:    vaddpd %zmm0, %zmm3, %zmm0
+; X86-NEXT:    retl
   %res = call <8 x double> @llvm.x86.avx512.mask.fixupimm.pd.512(<8 x double> %x0, <8 x double> %x1, <8 x i64> %x2, i32 4, i8 %x4, i32 4)
   %res1 = call <8 x double> @llvm.x86.avx512.mask.fixupimm.pd.512(<8 x double> zeroinitializer, <8 x double> %x1, <8 x i64> %x2, i32 5, i8 %x4, i32 4)
   %res2 = call <8 x double> @llvm.x86.avx512.mask.fixupimm.pd.512(<8 x double> %x0, <8 x double> %x1, <8 x i64> %x2, i32 3, i8 -1, i32 8)
@@ -4160,10 +5374,16 @@ define <8 x double>@test_int_x86_avx512_mask_fixupimm_pd_512(<8 x double> %x0, <
 }
 
 define <8 x double>@test_int_x86_avx512_mask_fixupimm_pd_512_load(<8 x double> %x0, <8 x double> %x1, <8 x i64>* %x2ptr) {
-; CHECK-LABEL: test_int_x86_avx512_mask_fixupimm_pd_512_load:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    vfixupimmpd $3, (%rdi), %zmm1, %zmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_int_x86_avx512_mask_fixupimm_pd_512_load:
+; X64:       # %bb.0:
+; X64-NEXT:    vfixupimmpd $3, (%rdi), %zmm1, %zmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_int_x86_avx512_mask_fixupimm_pd_512_load:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    vfixupimmpd $3, (%eax), %zmm1, %zmm0
+; X86-NEXT:    retl
   %x2 = load <8 x i64>, <8 x i64>* %x2ptr
   %res = call <8 x double> @llvm.x86.avx512.mask.fixupimm.pd.512(<8 x double> %x0, <8 x double> %x1, <8 x i64> %x2, i32 3, i8 -1, i32 4)
   ret <8 x double> %res
@@ -4172,18 +5392,32 @@ define <8 x double>@test_int_x86_avx512_mask_fixupimm_pd_512_load(<8 x double> %
 declare <8 x double> @llvm.x86.avx512.maskz.fixupimm.pd.512(<8 x double>, <8 x double>, <8 x i64>, i32, i8, i32)
 
 define <8 x double>@test_int_x86_avx512_maskz_fixupimm_pd_512(<8 x double> %x0, <8 x double> %x1, <8 x i64> %x2, i8 %x4) {
-; CHECK-LABEL: test_int_x86_avx512_maskz_fixupimm_pd_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vmovapd %zmm0, %zmm3
-; CHECK-NEXT:    vfixupimmpd $3, %zmm2, %zmm1, %zmm3 {%k1} {z}
-; CHECK-NEXT:    vxorpd %xmm4, %xmm4, %xmm4
-; CHECK-NEXT:    vmovapd %zmm0, %zmm5
-; CHECK-NEXT:    vfixupimmpd $5, %zmm4, %zmm1, %zmm5 {%k1} {z}
-; CHECK-NEXT:    vaddpd %zmm5, %zmm3, %zmm3
-; CHECK-NEXT:    vfixupimmpd $2, {sae}, %zmm2, %zmm1, %zmm0
-; CHECK-NEXT:    vaddpd %zmm0, %zmm3, %zmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_int_x86_avx512_maskz_fixupimm_pd_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vmovapd %zmm0, %zmm3
+; X64-NEXT:    vfixupimmpd $3, %zmm2, %zmm1, %zmm3 {%k1} {z}
+; X64-NEXT:    vxorpd %xmm4, %xmm4, %xmm4
+; X64-NEXT:    vmovapd %zmm0, %zmm5
+; X64-NEXT:    vfixupimmpd $5, %zmm4, %zmm1, %zmm5 {%k1} {z}
+; X64-NEXT:    vaddpd %zmm5, %zmm3, %zmm3
+; X64-NEXT:    vfixupimmpd $2, {sae}, %zmm2, %zmm1, %zmm0
+; X64-NEXT:    vaddpd %zmm0, %zmm3, %zmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_int_x86_avx512_maskz_fixupimm_pd_512:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vmovapd %zmm0, %zmm3
+; X86-NEXT:    vfixupimmpd $3, %zmm2, %zmm1, %zmm3 {%k1} {z}
+; X86-NEXT:    vxorpd %xmm4, %xmm4, %xmm4
+; X86-NEXT:    vmovapd %zmm0, %zmm5
+; X86-NEXT:    vfixupimmpd $5, %zmm4, %zmm1, %zmm5 {%k1} {z}
+; X86-NEXT:    vaddpd %zmm5, %zmm3, %zmm3
+; X86-NEXT:    vfixupimmpd $2, {sae}, %zmm2, %zmm1, %zmm0
+; X86-NEXT:    vaddpd %zmm0, %zmm3, %zmm0
+; X86-NEXT:    retl
   %res = call <8 x double> @llvm.x86.avx512.maskz.fixupimm.pd.512(<8 x double> %x0, <8 x double> %x1, <8 x i64> %x2, i32 3, i8 %x4, i32 4)
   %res1 = call <8 x double> @llvm.x86.avx512.maskz.fixupimm.pd.512(<8 x double> %x0, <8 x double> %x1, <8 x i64> zeroinitializer, i32 5, i8 %x4, i32 4)
   %res2 = call <8 x double> @llvm.x86.avx512.maskz.fixupimm.pd.512(<8 x double> %x0, <8 x double> %x1, <8 x i64> %x2, i32 2, i8 -1, i32 8)
@@ -4195,18 +5429,32 @@ define <8 x double>@test_int_x86_avx512_maskz_fixupimm_pd_512(<8 x double> %x0,
 declare <4 x float> @llvm.x86.avx512.mask.fixupimm.ss(<4 x float>, <4 x float>, <4 x i32>, i32, i8, i32)
 
 define <4 x float>@test_int_x86_avx512_mask_fixupimm_ss(<4 x float> %x0, <4 x float> %x1, <4 x i32> %x2, i8 %x4) {
-; CHECK-LABEL: test_int_x86_avx512_mask_fixupimm_ss:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vmovaps %xmm0, %xmm3
-; CHECK-NEXT:    vfixupimmss $5, %xmm2, %xmm1, %xmm3 {%k1}
-; CHECK-NEXT:    vxorps %xmm4, %xmm4, %xmm4
-; CHECK-NEXT:    vmovaps %xmm0, %xmm5
-; CHECK-NEXT:    vfixupimmss $5, %xmm4, %xmm1, %xmm5 {%k1}
-; CHECK-NEXT:    vaddps %xmm5, %xmm3, %xmm3
-; CHECK-NEXT:    vfixupimmss $5, {sae}, %xmm2, %xmm1, %xmm0
-; CHECK-NEXT:    vaddps %xmm0, %xmm3, %xmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_int_x86_avx512_mask_fixupimm_ss:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vmovaps %xmm0, %xmm3
+; X64-NEXT:    vfixupimmss $5, %xmm2, %xmm1, %xmm3 {%k1}
+; X64-NEXT:    vxorps %xmm4, %xmm4, %xmm4
+; X64-NEXT:    vmovaps %xmm0, %xmm5
+; X64-NEXT:    vfixupimmss $5, %xmm4, %xmm1, %xmm5 {%k1}
+; X64-NEXT:    vaddps %xmm5, %xmm3, %xmm3
+; X64-NEXT:    vfixupimmss $5, {sae}, %xmm2, %xmm1, %xmm0
+; X64-NEXT:    vaddps %xmm0, %xmm3, %xmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_int_x86_avx512_mask_fixupimm_ss:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vmovaps %xmm0, %xmm3
+; X86-NEXT:    vfixupimmss $5, %xmm2, %xmm1, %xmm3 {%k1}
+; X86-NEXT:    vxorps %xmm4, %xmm4, %xmm4
+; X86-NEXT:    vmovaps %xmm0, %xmm5
+; X86-NEXT:    vfixupimmss $5, %xmm4, %xmm1, %xmm5 {%k1}
+; X86-NEXT:    vaddps %xmm5, %xmm3, %xmm3
+; X86-NEXT:    vfixupimmss $5, {sae}, %xmm2, %xmm1, %xmm0
+; X86-NEXT:    vaddps %xmm0, %xmm3, %xmm0
+; X86-NEXT:    retl
   %res = call <4 x float> @llvm.x86.avx512.mask.fixupimm.ss(<4 x float> %x0, <4 x float> %x1, <4 x i32> %x2, i32 5, i8 %x4, i32 4)
   %res1 = call <4 x float> @llvm.x86.avx512.mask.fixupimm.ss(<4 x float> %x0, <4 x float> %x1, <4 x i32> zeroinitializer, i32 5, i8 %x4, i32 4)
   %res2 = call <4 x float> @llvm.x86.avx512.mask.fixupimm.ss(<4 x float> %x0, <4 x float> %x1, <4 x i32> %x2, i32 5, i8 -1, i32 8)
@@ -4218,18 +5466,32 @@ define <4 x float>@test_int_x86_avx512_mask_fixupimm_ss(<4 x float> %x0, <4 x fl
 declare <4 x float> @llvm.x86.avx512.maskz.fixupimm.ss(<4 x float>, <4 x float>, <4 x i32>, i32, i8, i32)
 
 define <4 x float>@test_int_x86_avx512_maskz_fixupimm_ss(<4 x float> %x0, <4 x float> %x1, <4 x i32> %x2, i8 %x4) {
-; CHECK-LABEL: test_int_x86_avx512_maskz_fixupimm_ss:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vmovaps %xmm0, %xmm3
-; CHECK-NEXT:    vfixupimmss $5, %xmm2, %xmm1, %xmm3
-; CHECK-NEXT:    vmovaps %xmm0, %xmm4
-; CHECK-NEXT:    vfixupimmss $5, %xmm2, %xmm1, %xmm4 {%k1} {z}
-; CHECK-NEXT:    vxorps %xmm2, %xmm2, %xmm2
-; CHECK-NEXT:    vfixupimmss $5, {sae}, %xmm2, %xmm1, %xmm0 {%k1} {z}
-; CHECK-NEXT:    vaddps %xmm0, %xmm4, %xmm0
-; CHECK-NEXT:    vaddps %xmm3, %xmm0, %xmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_int_x86_avx512_maskz_fixupimm_ss:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vmovaps %xmm0, %xmm3
+; X64-NEXT:    vfixupimmss $5, %xmm2, %xmm1, %xmm3
+; X64-NEXT:    vmovaps %xmm0, %xmm4
+; X64-NEXT:    vfixupimmss $5, %xmm2, %xmm1, %xmm4 {%k1} {z}
+; X64-NEXT:    vxorps %xmm2, %xmm2, %xmm2
+; X64-NEXT:    vfixupimmss $5, {sae}, %xmm2, %xmm1, %xmm0 {%k1} {z}
+; X64-NEXT:    vaddps %xmm0, %xmm4, %xmm0
+; X64-NEXT:    vaddps %xmm3, %xmm0, %xmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_int_x86_avx512_maskz_fixupimm_ss:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vmovaps %xmm0, %xmm3
+; X86-NEXT:    vfixupimmss $5, %xmm2, %xmm1, %xmm3 {%k1} {z}
+; X86-NEXT:    vmovaps %xmm0, %xmm4
+; X86-NEXT:    vfixupimmss $5, %xmm2, %xmm1, %xmm4
+; X86-NEXT:    vxorps %xmm2, %xmm2, %xmm2
+; X86-NEXT:    vfixupimmss $5, {sae}, %xmm2, %xmm1, %xmm0 {%k1} {z}
+; X86-NEXT:    vaddps %xmm0, %xmm3, %xmm0
+; X86-NEXT:    vaddps %xmm4, %xmm0, %xmm0
+; X86-NEXT:    retl
   %res = call <4 x float> @llvm.x86.avx512.maskz.fixupimm.ss(<4 x float> %x0, <4 x float> %x1, <4 x i32> %x2, i32 5, i8 %x4, i32 4)
   %res1 = call <4 x float> @llvm.x86.avx512.maskz.fixupimm.ss(<4 x float> %x0, <4 x float> %x1, <4 x i32> zeroinitializer, i32 5, i8 %x4, i32 8)
   %res2 = call <4 x float> @llvm.x86.avx512.maskz.fixupimm.ss(<4 x float> %x0, <4 x float> %x1, <4 x i32> %x2, i32 5, i8 -1, i32 4)
@@ -4241,18 +5503,31 @@ define <4 x float>@test_int_x86_avx512_maskz_fixupimm_ss(<4 x float> %x0, <4 x f
 declare <16 x float> @llvm.x86.avx512.mask.fixupimm.ps.512(<16 x float>, <16 x float>, <16 x i32>, i32, i16, i32)
 
 define <16 x float>@test_int_x86_avx512_mask_fixupimm_ps_512(<16 x float> %x0, <16 x float> %x1, <16 x i32> %x2, i16 %x4) {
-; CHECK-LABEL: test_int_x86_avx512_mask_fixupimm_ps_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vmovaps %zmm0, %zmm3
-; CHECK-NEXT:    vfixupimmps $5, %zmm2, %zmm1, %zmm3 {%k1}
-; CHECK-NEXT:    vxorps %xmm4, %xmm4, %xmm4
-; CHECK-NEXT:    vmovaps %zmm0, %zmm5
-; CHECK-NEXT:    vfixupimmps $5, %zmm4, %zmm1, %zmm5 {%k1}
-; CHECK-NEXT:    vaddps %zmm5, %zmm3, %zmm3
-; CHECK-NEXT:    vfixupimmps $5, {sae}, %zmm2, %zmm1, %zmm0
-; CHECK-NEXT:    vaddps %zmm0, %zmm3, %zmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_int_x86_avx512_mask_fixupimm_ps_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vmovaps %zmm0, %zmm3
+; X64-NEXT:    vfixupimmps $5, %zmm2, %zmm1, %zmm3 {%k1}
+; X64-NEXT:    vxorps %xmm4, %xmm4, %xmm4
+; X64-NEXT:    vmovaps %zmm0, %zmm5
+; X64-NEXT:    vfixupimmps $5, %zmm4, %zmm1, %zmm5 {%k1}
+; X64-NEXT:    vaddps %zmm5, %zmm3, %zmm3
+; X64-NEXT:    vfixupimmps $5, {sae}, %zmm2, %zmm1, %zmm0
+; X64-NEXT:    vaddps %zmm0, %zmm3, %zmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_int_x86_avx512_mask_fixupimm_ps_512:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vmovaps %zmm0, %zmm3
+; X86-NEXT:    vfixupimmps $5, %zmm2, %zmm1, %zmm3 {%k1}
+; X86-NEXT:    vxorps %xmm4, %xmm4, %xmm4
+; X86-NEXT:    vmovaps %zmm0, %zmm5
+; X86-NEXT:    vfixupimmps $5, %zmm4, %zmm1, %zmm5 {%k1}
+; X86-NEXT:    vaddps %zmm5, %zmm3, %zmm3
+; X86-NEXT:    vfixupimmps $5, {sae}, %zmm2, %zmm1, %zmm0
+; X86-NEXT:    vaddps %zmm0, %zmm3, %zmm0
+; X86-NEXT:    retl
   %res = call <16 x float> @llvm.x86.avx512.mask.fixupimm.ps.512(<16 x float> %x0, <16 x float> %x1, <16 x i32> %x2, i32 5, i16 %x4, i32 4)
   %res1 = call <16 x float> @llvm.x86.avx512.mask.fixupimm.ps.512(<16 x float> %x0, <16 x float> %x1, <16 x i32> zeroinitializer, i32 5, i16 %x4, i32 4)
   %res2 = call <16 x float> @llvm.x86.avx512.mask.fixupimm.ps.512(<16 x float> %x0, <16 x float> %x1, <16 x i32> %x2, i32 5, i16 -1, i32 8)
@@ -4262,10 +5537,16 @@ define <16 x float>@test_int_x86_avx512_mask_fixupimm_ps_512(<16 x float> %x0, <
 }
 
 define <16 x float>@test_int_x86_avx512_mask_fixupimm_ps_512_load(<16 x float> %x0, <16 x float> %x1, <16 x i32>* %x2ptr) {
-; CHECK-LABEL: test_int_x86_avx512_mask_fixupimm_ps_512_load:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    vfixupimmps $5, (%rdi), %zmm1, %zmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_int_x86_avx512_mask_fixupimm_ps_512_load:
+; X64:       # %bb.0:
+; X64-NEXT:    vfixupimmps $5, (%rdi), %zmm1, %zmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_int_x86_avx512_mask_fixupimm_ps_512_load:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    vfixupimmps $5, (%eax), %zmm1, %zmm0
+; X86-NEXT:    retl
   %x2 = load <16 x i32>, <16 x i32>* %x2ptr
   %res = call <16 x float> @llvm.x86.avx512.mask.fixupimm.ps.512(<16 x float> %x0, <16 x float> %x1, <16 x i32> %x2, i32 5, i16 -1, i32 4)
   ret <16 x float> %res
@@ -4274,18 +5555,31 @@ define <16 x float>@test_int_x86_avx512_mask_fixupimm_ps_512_load(<16 x float> %
 declare <16 x float> @llvm.x86.avx512.maskz.fixupimm.ps.512(<16 x float>, <16 x float>, <16 x i32>, i32, i16, i32)
 
 define <16 x float>@test_int_x86_avx512_maskz_fixupimm_ps_512(<16 x float> %x0, <16 x float> %x1, <16 x i32> %x2, i16 %x4) {
-; CHECK-LABEL: test_int_x86_avx512_maskz_fixupimm_ps_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vmovaps %zmm0, %zmm3
-; CHECK-NEXT:    vfixupimmps $5, %zmm2, %zmm1, %zmm3
-; CHECK-NEXT:    vmovaps %zmm0, %zmm4
-; CHECK-NEXT:    vfixupimmps $5, %zmm2, %zmm1, %zmm4 {%k1} {z}
-; CHECK-NEXT:    vxorps %xmm2, %xmm2, %xmm2
-; CHECK-NEXT:    vfixupimmps $5, {sae}, %zmm2, %zmm1, %zmm0 {%k1} {z}
-; CHECK-NEXT:    vaddps %zmm0, %zmm4, %zmm0
-; CHECK-NEXT:    vaddps %zmm3, %zmm0, %zmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_int_x86_avx512_maskz_fixupimm_ps_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vmovaps %zmm0, %zmm3
+; X64-NEXT:    vfixupimmps $5, %zmm2, %zmm1, %zmm3
+; X64-NEXT:    vmovaps %zmm0, %zmm4
+; X64-NEXT:    vfixupimmps $5, %zmm2, %zmm1, %zmm4 {%k1} {z}
+; X64-NEXT:    vxorps %xmm2, %xmm2, %xmm2
+; X64-NEXT:    vfixupimmps $5, {sae}, %zmm2, %zmm1, %zmm0 {%k1} {z}
+; X64-NEXT:    vaddps %zmm0, %zmm4, %zmm0
+; X64-NEXT:    vaddps %zmm3, %zmm0, %zmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_int_x86_avx512_maskz_fixupimm_ps_512:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vmovaps %zmm0, %zmm3
+; X86-NEXT:    vfixupimmps $5, %zmm2, %zmm1, %zmm3 {%k1} {z}
+; X86-NEXT:    vmovaps %zmm0, %zmm4
+; X86-NEXT:    vfixupimmps $5, %zmm2, %zmm1, %zmm4
+; X86-NEXT:    vxorps %xmm2, %xmm2, %xmm2
+; X86-NEXT:    vfixupimmps $5, {sae}, %zmm2, %zmm1, %zmm0 {%k1} {z}
+; X86-NEXT:    vaddps %zmm0, %zmm3, %zmm0
+; X86-NEXT:    vaddps %zmm4, %zmm0, %zmm0
+; X86-NEXT:    retl
   %res = call <16 x float> @llvm.x86.avx512.maskz.fixupimm.ps.512(<16 x float> %x0, <16 x float> %x1, <16 x i32> %x2, i32 5, i16 %x4, i32 4)
   %res1 = call <16 x float> @llvm.x86.avx512.maskz.fixupimm.ps.512(<16 x float> %x0, <16 x float> %x1, <16 x i32> zeroinitializer, i32 5, i16 %x4, i32 8)
   %res2 = call <16 x float> @llvm.x86.avx512.maskz.fixupimm.ps.512(<16 x float> %x0, <16 x float> %x1, <16 x i32> %x2, i32 5, i16 -1, i32 4)
@@ -4297,18 +5591,32 @@ define <16 x float>@test_int_x86_avx512_maskz_fixupimm_ps_512(<16 x float> %x0,
 declare <2 x double> @llvm.x86.avx512.mask.fixupimm.sd(<2 x double>, <2 x double>, <2 x i64>, i32, i8, i32)
 
 define <2 x double>@test_int_x86_avx512_mask_fixupimm_sd(<2 x double> %x0, <2 x double> %x1, <2 x i64> %x2, i8 %x4) {
-; CHECK-LABEL: test_int_x86_avx512_mask_fixupimm_sd:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vmovapd %xmm0, %xmm3
-; CHECK-NEXT:    vfixupimmsd $5, %xmm2, %xmm1, %xmm3
-; CHECK-NEXT:    vmovapd %xmm0, %xmm4
-; CHECK-NEXT:    vfixupimmsd $5, %xmm2, %xmm1, %xmm4 {%k1}
-; CHECK-NEXT:    vxorpd %xmm2, %xmm2, %xmm2
-; CHECK-NEXT:    vfixupimmsd $5, {sae}, %xmm2, %xmm1, %xmm0 {%k1}
-; CHECK-NEXT:    vaddpd %xmm0, %xmm4, %xmm0
-; CHECK-NEXT:    vaddpd %xmm3, %xmm0, %xmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_int_x86_avx512_mask_fixupimm_sd:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vmovapd %xmm0, %xmm3
+; X64-NEXT:    vfixupimmsd $5, %xmm2, %xmm1, %xmm3
+; X64-NEXT:    vmovapd %xmm0, %xmm4
+; X64-NEXT:    vfixupimmsd $5, %xmm2, %xmm1, %xmm4 {%k1}
+; X64-NEXT:    vxorpd %xmm2, %xmm2, %xmm2
+; X64-NEXT:    vfixupimmsd $5, {sae}, %xmm2, %xmm1, %xmm0 {%k1}
+; X64-NEXT:    vaddpd %xmm0, %xmm4, %xmm0
+; X64-NEXT:    vaddpd %xmm3, %xmm0, %xmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_int_x86_avx512_mask_fixupimm_sd:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vmovapd %xmm0, %xmm3
+; X86-NEXT:    vfixupimmsd $5, %xmm2, %xmm1, %xmm3 {%k1}
+; X86-NEXT:    vmovapd %xmm0, %xmm4
+; X86-NEXT:    vfixupimmsd $5, %xmm2, %xmm1, %xmm4
+; X86-NEXT:    vxorpd %xmm2, %xmm2, %xmm2
+; X86-NEXT:    vfixupimmsd $5, {sae}, %xmm2, %xmm1, %xmm0 {%k1}
+; X86-NEXT:    vaddpd %xmm0, %xmm3, %xmm0
+; X86-NEXT:    vaddpd %xmm4, %xmm0, %xmm0
+; X86-NEXT:    retl
   %res = call <2 x double> @llvm.x86.avx512.mask.fixupimm.sd(<2 x double> %x0, <2 x double> %x1, <2 x i64> %x2, i32 5, i8 %x4, i32 4)
   %res1 = call <2 x double> @llvm.x86.avx512.mask.fixupimm.sd(<2 x double> %x0, <2 x double> %x1, <2 x i64> zeroinitializer, i32 5, i8 %x4, i32 8)
   %res2 = call <2 x double> @llvm.x86.avx512.mask.fixupimm.sd(<2 x double> %x0, <2 x double> %x1, <2 x i64> %x2, i32 5, i8 -1, i32 4)
@@ -4320,18 +5628,32 @@ define <2 x double>@test_int_x86_avx512_mask_fixupimm_sd(<2 x double> %x0, <2 x
 declare <2 x double> @llvm.x86.avx512.maskz.fixupimm.sd(<2 x double>, <2 x double>, <2 x i64>, i32, i8, i32)
 
 define <2 x double>@test_int_x86_avx512_maskz_fixupimm_sd(<2 x double> %x0, <2 x double> %x1, <2 x i64> %x2, i8 %x4) {
-; CHECK-LABEL: test_int_x86_avx512_maskz_fixupimm_sd:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vmovapd %xmm0, %xmm3
-; CHECK-NEXT:    vfixupimmsd $5, %xmm2, %xmm1, %xmm3 {%k1} {z}
-; CHECK-NEXT:    vxorpd %xmm4, %xmm4, %xmm4
-; CHECK-NEXT:    vmovapd %xmm0, %xmm5
-; CHECK-NEXT:    vfixupimmsd $5, {sae}, %xmm4, %xmm1, %xmm5 {%k1} {z}
-; CHECK-NEXT:    vaddpd %xmm5, %xmm3, %xmm3
-; CHECK-NEXT:    vfixupimmsd $5, {sae}, %xmm2, %xmm1, %xmm0 {%k1} {z}
-; CHECK-NEXT:    vaddpd %xmm0, %xmm3, %xmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_int_x86_avx512_maskz_fixupimm_sd:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vmovapd %xmm0, %xmm3
+; X64-NEXT:    vfixupimmsd $5, %xmm2, %xmm1, %xmm3 {%k1} {z}
+; X64-NEXT:    vxorpd %xmm4, %xmm4, %xmm4
+; X64-NEXT:    vmovapd %xmm0, %xmm5
+; X64-NEXT:    vfixupimmsd $5, {sae}, %xmm4, %xmm1, %xmm5 {%k1} {z}
+; X64-NEXT:    vaddpd %xmm5, %xmm3, %xmm3
+; X64-NEXT:    vfixupimmsd $5, {sae}, %xmm2, %xmm1, %xmm0 {%k1} {z}
+; X64-NEXT:    vaddpd %xmm0, %xmm3, %xmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_int_x86_avx512_maskz_fixupimm_sd:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vmovapd %xmm0, %xmm3
+; X86-NEXT:    vfixupimmsd $5, %xmm2, %xmm1, %xmm3 {%k1} {z}
+; X86-NEXT:    vxorpd %xmm4, %xmm4, %xmm4
+; X86-NEXT:    vmovapd %xmm0, %xmm5
+; X86-NEXT:    vfixupimmsd $5, {sae}, %xmm4, %xmm1, %xmm5 {%k1} {z}
+; X86-NEXT:    vaddpd %xmm5, %xmm3, %xmm3
+; X86-NEXT:    vfixupimmsd $5, {sae}, %xmm2, %xmm1, %xmm0 {%k1} {z}
+; X86-NEXT:    vaddpd %xmm0, %xmm3, %xmm0
+; X86-NEXT:    retl
   %res = call <2 x double> @llvm.x86.avx512.maskz.fixupimm.sd(<2 x double> %x0, <2 x double> %x1, <2 x i64> %x2, i32 5, i8 %x4, i32 4)
   %res1 = call <2 x double> @llvm.x86.avx512.maskz.fixupimm.sd(<2 x double> %x0, <2 x double> %x1, <2 x i64> zeroinitializer, i32 5, i8 %x4, i32 8)
   %res2 = call <2 x double> @llvm.x86.avx512.maskz.fixupimm.sd(<2 x double> %x0, <2 x double> %x1, <2 x i64> %x2, i32 5, i8 %x4, i32 8)
@@ -4344,20 +5666,36 @@ declare double @llvm.fma.f64(double, double, double) #1
 declare double @llvm.x86.avx512.vfmadd.f64(double, double, double, i32) #0
 
 define <2 x double>@test_int_x86_avx512_mask_vfmadd_sd(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3,i32 %x4 ){
-; CHECK-LABEL: test_int_x86_avx512_mask_vfmadd_sd:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    vmovapd %xmm0, %xmm3
-; CHECK-NEXT:    vfmadd213sd {{.*#+}} xmm3 = (xmm1 * xmm3) + xmm2
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vmovapd %xmm0, %xmm4
-; CHECK-NEXT:    vfmadd213sd {{.*#+}} xmm4 = (xmm1 * xmm4) + xmm2
-; CHECK-NEXT:    vaddpd %xmm4, %xmm3, %xmm3
-; CHECK-NEXT:    vmovapd %xmm0, %xmm4
-; CHECK-NEXT:    vfmadd213sd {rz-sae}, %xmm2, %xmm1, %xmm4
-; CHECK-NEXT:    vfmadd213sd {rz-sae}, %xmm2, %xmm1, %xmm0 {%k1}
-; CHECK-NEXT:    vaddpd %xmm0, %xmm4, %xmm0
-; CHECK-NEXT:    vaddpd %xmm0, %xmm3, %xmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_int_x86_avx512_mask_vfmadd_sd:
+; X64:       # %bb.0:
+; X64-NEXT:    vmovapd %xmm0, %xmm3
+; X64-NEXT:    vfmadd213sd {{.*#+}} xmm3 = (xmm1 * xmm3) + xmm2
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vmovapd %xmm0, %xmm4
+; X64-NEXT:    vfmadd213sd {{.*#+}} xmm4 = (xmm1 * xmm4) + xmm2
+; X64-NEXT:    vaddpd %xmm4, %xmm3, %xmm3
+; X64-NEXT:    vmovapd %xmm0, %xmm4
+; X64-NEXT:    vfmadd213sd {rz-sae}, %xmm2, %xmm1, %xmm4
+; X64-NEXT:    vfmadd213sd {rz-sae}, %xmm2, %xmm1, %xmm0 {%k1}
+; X64-NEXT:    vaddpd %xmm0, %xmm4, %xmm0
+; X64-NEXT:    vaddpd %xmm0, %xmm3, %xmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_int_x86_avx512_mask_vfmadd_sd:
+; X86:       # %bb.0:
+; X86-NEXT:    movb {{[0-9]+}}(%esp), %al
+; X86-NEXT:    vmovapd %xmm0, %xmm3
+; X86-NEXT:    vfmadd213sd {{.*#+}} xmm3 = (xmm1 * xmm3) + xmm2
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vmovapd %xmm0, %xmm4
+; X86-NEXT:    vfmadd213sd {{.*#+}} xmm4 = (xmm1 * xmm4) + xmm2
+; X86-NEXT:    vaddpd %xmm4, %xmm3, %xmm3
+; X86-NEXT:    vmovapd %xmm0, %xmm4
+; X86-NEXT:    vfmadd213sd {rz-sae}, %xmm2, %xmm1, %xmm4
+; X86-NEXT:    vfmadd213sd {rz-sae}, %xmm2, %xmm1, %xmm0 {%k1}
+; X86-NEXT:    vaddpd %xmm0, %xmm4, %xmm0
+; X86-NEXT:    vaddpd %xmm0, %xmm3, %xmm0
+; X86-NEXT:    retl
   %1 = extractelement <2 x double> %x0, i64 0
   %2 = extractelement <2 x double> %x1, i64 0
   %3 = extractelement <2 x double> %x2, i64 0
@@ -4391,20 +5729,36 @@ define <2 x double>@test_int_x86_avx512_mask_vfmadd_sd(<2 x double> %x0, <2 x do
 }
 
 define <4 x float>@test_int_x86_avx512_mask_vfmadd_ss(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3,i32 %x4 ){
-; CHECK-LABEL: test_int_x86_avx512_mask_vfmadd_ss:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    vmovaps %xmm0, %xmm3
-; CHECK-NEXT:    vfmadd213ss {{.*#+}} xmm3 = (xmm1 * xmm3) + xmm2
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vmovaps %xmm0, %xmm4
-; CHECK-NEXT:    vfmadd213ss {{.*#+}} xmm4 = (xmm1 * xmm4) + xmm2
-; CHECK-NEXT:    vaddps %xmm4, %xmm3, %xmm3
-; CHECK-NEXT:    vmovaps %xmm0, %xmm4
-; CHECK-NEXT:    vfmadd213ss {rz-sae}, %xmm2, %xmm1, %xmm4
-; CHECK-NEXT:    vfmadd213ss {rz-sae}, %xmm2, %xmm1, %xmm0 {%k1}
-; CHECK-NEXT:    vaddps %xmm0, %xmm4, %xmm0
-; CHECK-NEXT:    vaddps %xmm0, %xmm3, %xmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_int_x86_avx512_mask_vfmadd_ss:
+; X64:       # %bb.0:
+; X64-NEXT:    vmovaps %xmm0, %xmm3
+; X64-NEXT:    vfmadd213ss {{.*#+}} xmm3 = (xmm1 * xmm3) + xmm2
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vmovaps %xmm0, %xmm4
+; X64-NEXT:    vfmadd213ss {{.*#+}} xmm4 = (xmm1 * xmm4) + xmm2
+; X64-NEXT:    vaddps %xmm4, %xmm3, %xmm3
+; X64-NEXT:    vmovaps %xmm0, %xmm4
+; X64-NEXT:    vfmadd213ss {rz-sae}, %xmm2, %xmm1, %xmm4
+; X64-NEXT:    vfmadd213ss {rz-sae}, %xmm2, %xmm1, %xmm0 {%k1}
+; X64-NEXT:    vaddps %xmm0, %xmm4, %xmm0
+; X64-NEXT:    vaddps %xmm0, %xmm3, %xmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_int_x86_avx512_mask_vfmadd_ss:
+; X86:       # %bb.0:
+; X86-NEXT:    movb {{[0-9]+}}(%esp), %al
+; X86-NEXT:    vmovaps %xmm0, %xmm3
+; X86-NEXT:    vfmadd213ss {{.*#+}} xmm3 = (xmm1 * xmm3) + xmm2
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vmovaps %xmm0, %xmm4
+; X86-NEXT:    vfmadd213ss {{.*#+}} xmm4 = (xmm1 * xmm4) + xmm2
+; X86-NEXT:    vaddps %xmm4, %xmm3, %xmm3
+; X86-NEXT:    vmovaps %xmm0, %xmm4
+; X86-NEXT:    vfmadd213ss {rz-sae}, %xmm2, %xmm1, %xmm4
+; X86-NEXT:    vfmadd213ss {rz-sae}, %xmm2, %xmm1, %xmm0 {%k1}
+; X86-NEXT:    vaddps %xmm0, %xmm4, %xmm0
+; X86-NEXT:    vaddps %xmm0, %xmm3, %xmm0
+; X86-NEXT:    retl
   %1 = extractelement <4 x float> %x0, i64 0
   %2 = extractelement <4 x float> %x1, i64 0
   %3 = extractelement <4 x float> %x2, i64 0
@@ -4438,14 +5792,24 @@ define <4 x float>@test_int_x86_avx512_mask_vfmadd_ss(<4 x float> %x0, <4 x floa
 }
 
 define <2 x double>@test_int_x86_avx512_maskz_vfmadd_sd(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3,i32 %x4 ){
-; CHECK-LABEL: test_int_x86_avx512_maskz_vfmadd_sd:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vmovapd %xmm0, %xmm3
-; CHECK-NEXT:    vfmadd213sd {{.*#+}} xmm3 = (xmm1 * xmm3) + xmm2
-; CHECK-NEXT:    vfmadd213sd {rz-sae}, %xmm2, %xmm1, %xmm0 {%k1} {z}
-; CHECK-NEXT:    vaddpd %xmm0, %xmm3, %xmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_int_x86_avx512_maskz_vfmadd_sd:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vmovapd %xmm0, %xmm3
+; X64-NEXT:    vfmadd213sd {{.*#+}} xmm3 = (xmm1 * xmm3) + xmm2
+; X64-NEXT:    vfmadd213sd {rz-sae}, %xmm2, %xmm1, %xmm0 {%k1} {z}
+; X64-NEXT:    vaddpd %xmm0, %xmm3, %xmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_int_x86_avx512_maskz_vfmadd_sd:
+; X86:       # %bb.0:
+; X86-NEXT:    movb {{[0-9]+}}(%esp), %al
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vmovapd %xmm0, %xmm3
+; X86-NEXT:    vfmadd213sd {{.*#+}} xmm3 = (xmm1 * xmm3) + xmm2
+; X86-NEXT:    vfmadd213sd {rz-sae}, %xmm2, %xmm1, %xmm0 {%k1} {z}
+; X86-NEXT:    vaddpd %xmm0, %xmm3, %xmm0
+; X86-NEXT:    retl
   %1 = extractelement <2 x double> %x0, i64 0
   %2 = extractelement <2 x double> %x1, i64 0
   %3 = extractelement <2 x double> %x2, i64 0
@@ -4470,11 +5834,18 @@ declare float @llvm.fma.f32(float, float, float) #1
 declare float @llvm.x86.avx512.vfmadd.f32(float, float, float, i32) #0
 
 define <4 x float>@test_int_x86_avx512_maskz_vfmadd_ss(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3,i32 %x4 ){
-; CHECK-LABEL: test_int_x86_avx512_maskz_vfmadd_ss:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vfmadd213ss {{.*#+}} xmm0 = (xmm1 * xmm0) + xmm2
-; CHECK-NEXT:    retq
+; X64-LABEL: test_int_x86_avx512_maskz_vfmadd_ss:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vfmadd213ss {{.*#+}} xmm0 = (xmm1 * xmm0) + xmm2
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_int_x86_avx512_maskz_vfmadd_ss:
+; X86:       # %bb.0:
+; X86-NEXT:    movb {{[0-9]+}}(%esp), %al
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vfmadd213ss {{.*#+}} xmm0 = (xmm1 * xmm0) + xmm2
+; X86-NEXT:    retl
   %1 = extractelement <4 x float> %x0, i64 0
   %2 = extractelement <4 x float> %x1, i64 0
   %3 = extractelement <4 x float> %x2, i64 0
@@ -4496,20 +5867,36 @@ define <4 x float>@test_int_x86_avx512_maskz_vfmadd_ss(<4 x float> %x0, <4 x flo
 }
 
 define <2 x double>@test_int_x86_avx512_mask3_vfmadd_sd(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3,i32 %x4 ){
-; CHECK-LABEL: test_int_x86_avx512_mask3_vfmadd_sd:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    vmovapd %xmm2, %xmm3
-; CHECK-NEXT:    vfmadd231sd {{.*#+}} xmm3 = (xmm0 * xmm1) + xmm3
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vmovapd %xmm2, %xmm4
-; CHECK-NEXT:    vfmadd231sd {{.*#+}} xmm4 = (xmm0 * xmm1) + xmm4
-; CHECK-NEXT:    vaddpd %xmm4, %xmm3, %xmm3
-; CHECK-NEXT:    vmovapd %xmm2, %xmm4
-; CHECK-NEXT:    vfmadd231sd {rz-sae}, %xmm1, %xmm0, %xmm4
-; CHECK-NEXT:    vfmadd231sd {rz-sae}, %xmm1, %xmm0, %xmm2 {%k1}
-; CHECK-NEXT:    vaddpd %xmm2, %xmm4, %xmm0
-; CHECK-NEXT:    vaddpd %xmm0, %xmm3, %xmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_int_x86_avx512_mask3_vfmadd_sd:
+; X64:       # %bb.0:
+; X64-NEXT:    vmovapd %xmm2, %xmm3
+; X64-NEXT:    vfmadd231sd {{.*#+}} xmm3 = (xmm0 * xmm1) + xmm3
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vmovapd %xmm2, %xmm4
+; X64-NEXT:    vfmadd231sd {{.*#+}} xmm4 = (xmm0 * xmm1) + xmm4
+; X64-NEXT:    vaddpd %xmm4, %xmm3, %xmm3
+; X64-NEXT:    vmovapd %xmm2, %xmm4
+; X64-NEXT:    vfmadd231sd {rz-sae}, %xmm1, %xmm0, %xmm4
+; X64-NEXT:    vfmadd231sd {rz-sae}, %xmm1, %xmm0, %xmm2 {%k1}
+; X64-NEXT:    vaddpd %xmm2, %xmm4, %xmm0
+; X64-NEXT:    vaddpd %xmm0, %xmm3, %xmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_int_x86_avx512_mask3_vfmadd_sd:
+; X86:       # %bb.0:
+; X86-NEXT:    movb {{[0-9]+}}(%esp), %al
+; X86-NEXT:    vmovapd %xmm2, %xmm3
+; X86-NEXT:    vfmadd231sd {{.*#+}} xmm3 = (xmm0 * xmm1) + xmm3
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vmovapd %xmm2, %xmm4
+; X86-NEXT:    vfmadd231sd {{.*#+}} xmm4 = (xmm0 * xmm1) + xmm4
+; X86-NEXT:    vaddpd %xmm4, %xmm3, %xmm3
+; X86-NEXT:    vmovapd %xmm2, %xmm4
+; X86-NEXT:    vfmadd231sd {rz-sae}, %xmm1, %xmm0, %xmm4
+; X86-NEXT:    vfmadd231sd {rz-sae}, %xmm1, %xmm0, %xmm2 {%k1}
+; X86-NEXT:    vaddpd %xmm2, %xmm4, %xmm0
+; X86-NEXT:    vaddpd %xmm0, %xmm3, %xmm0
+; X86-NEXT:    retl
   %1 = extractelement <2 x double> %x0, i64 0
   %2 = extractelement <2 x double> %x1, i64 0
   %3 = extractelement <2 x double> %x2, i64 0
@@ -4543,20 +5930,36 @@ define <2 x double>@test_int_x86_avx512_mask3_vfmadd_sd(<2 x double> %x0, <2 x d
 }
 
 define <4 x float>@test_int_x86_avx512_mask3_vfmadd_ss(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3,i32 %x4 ){
-; CHECK-LABEL: test_int_x86_avx512_mask3_vfmadd_ss:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    vmovaps %xmm2, %xmm3
-; CHECK-NEXT:    vfmadd231ss {{.*#+}} xmm3 = (xmm0 * xmm1) + xmm3
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vmovaps %xmm2, %xmm4
-; CHECK-NEXT:    vfmadd231ss {{.*#+}} xmm4 = (xmm0 * xmm1) + xmm4
-; CHECK-NEXT:    vaddps %xmm4, %xmm3, %xmm3
-; CHECK-NEXT:    vmovaps %xmm2, %xmm4
-; CHECK-NEXT:    vfmadd231ss {rz-sae}, %xmm1, %xmm0, %xmm4
-; CHECK-NEXT:    vfmadd231ss {rz-sae}, %xmm1, %xmm0, %xmm2 {%k1}
-; CHECK-NEXT:    vaddps %xmm2, %xmm4, %xmm0
-; CHECK-NEXT:    vaddps %xmm0, %xmm3, %xmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_int_x86_avx512_mask3_vfmadd_ss:
+; X64:       # %bb.0:
+; X64-NEXT:    vmovaps %xmm2, %xmm3
+; X64-NEXT:    vfmadd231ss {{.*#+}} xmm3 = (xmm0 * xmm1) + xmm3
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vmovaps %xmm2, %xmm4
+; X64-NEXT:    vfmadd231ss {{.*#+}} xmm4 = (xmm0 * xmm1) + xmm4
+; X64-NEXT:    vaddps %xmm4, %xmm3, %xmm3
+; X64-NEXT:    vmovaps %xmm2, %xmm4
+; X64-NEXT:    vfmadd231ss {rz-sae}, %xmm1, %xmm0, %xmm4
+; X64-NEXT:    vfmadd231ss {rz-sae}, %xmm1, %xmm0, %xmm2 {%k1}
+; X64-NEXT:    vaddps %xmm2, %xmm4, %xmm0
+; X64-NEXT:    vaddps %xmm0, %xmm3, %xmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_int_x86_avx512_mask3_vfmadd_ss:
+; X86:       # %bb.0:
+; X86-NEXT:    movb {{[0-9]+}}(%esp), %al
+; X86-NEXT:    vmovaps %xmm2, %xmm3
+; X86-NEXT:    vfmadd231ss {{.*#+}} xmm3 = (xmm0 * xmm1) + xmm3
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vmovaps %xmm2, %xmm4
+; X86-NEXT:    vfmadd231ss {{.*#+}} xmm4 = (xmm0 * xmm1) + xmm4
+; X86-NEXT:    vaddps %xmm4, %xmm3, %xmm3
+; X86-NEXT:    vmovaps %xmm2, %xmm4
+; X86-NEXT:    vfmadd231ss {rz-sae}, %xmm1, %xmm0, %xmm4
+; X86-NEXT:    vfmadd231ss {rz-sae}, %xmm1, %xmm0, %xmm2 {%k1}
+; X86-NEXT:    vaddps %xmm2, %xmm4, %xmm0
+; X86-NEXT:    vaddps %xmm0, %xmm3, %xmm0
+; X86-NEXT:    retl
   %1 = extractelement <4 x float> %x0, i64 0
   %2 = extractelement <4 x float> %x1, i64 0
   %3 = extractelement <4 x float> %x2, i64 0
@@ -4590,15 +5993,28 @@ define <4 x float>@test_int_x86_avx512_mask3_vfmadd_ss(<4 x float> %x0, <4 x flo
 }
 
 define void @fmadd_ss_mask_memfold(float* %a, float* %b, i8 %c) {
-; CHECK-LABEL: fmadd_ss_mask_memfold:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; CHECK-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; CHECK-NEXT:    vfmadd213ss {{.*#+}} xmm1 = (xmm0 * xmm1) + xmm0
-; CHECK-NEXT:    kmovw %edx, %k1
-; CHECK-NEXT:    vmovss %xmm1, %xmm0, %xmm0 {%k1}
-; CHECK-NEXT:    vmovss %xmm0, (%rdi)
-; CHECK-NEXT:    retq
+; X64-LABEL: fmadd_ss_mask_memfold:
+; X64:       # %bb.0:
+; X64-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X64-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; X64-NEXT:    vfmadd213ss {{.*#+}} xmm1 = (xmm0 * xmm1) + xmm0
+; X64-NEXT:    kmovw %edx, %k1
+; X64-NEXT:    vmovss %xmm1, %xmm0, %xmm0 {%k1}
+; X64-NEXT:    vmovss %xmm0, (%rdi)
+; X64-NEXT:    retq
+;
+; X86-LABEL: fmadd_ss_mask_memfold:
+; X86:       # %bb.0:
+; X86-NEXT:    movb {{[0-9]+}}(%esp), %al
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; X86-NEXT:    vfmadd213ss {{.*#+}} xmm1 = (xmm0 * xmm1) + xmm0
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vmovss %xmm1, %xmm0, %xmm0 {%k1}
+; X86-NEXT:    vmovss %xmm0, (%edx)
+; X86-NEXT:    retl
   %a.val = load float, float* %a
   %av0 = insertelement <4 x float> undef, float %a.val, i32 0
   %av1 = insertelement <4 x float> %av0, float 0.000000e+00, i32 1
@@ -4624,14 +6040,26 @@ define void @fmadd_ss_mask_memfold(float* %a, float* %b, i8 %c) {
 }
 
 define void @fmadd_ss_maskz_memfold(float* %a, float* %b, i8 %c) {
-; CHECK-LABEL: fmadd_ss_maskz_memfold:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; CHECK-NEXT:    vfmadd231ss {{.*#+}} xmm0 = (xmm0 * mem) + xmm0
-; CHECK-NEXT:    kmovw %edx, %k1
-; CHECK-NEXT:    vmovss %xmm0, %xmm0, %xmm0 {%k1} {z}
-; CHECK-NEXT:    vmovss %xmm0, (%rdi)
-; CHECK-NEXT:    retq
+; X64-LABEL: fmadd_ss_maskz_memfold:
+; X64:       # %bb.0:
+; X64-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X64-NEXT:    vfmadd231ss {{.*#+}} xmm0 = (xmm0 * mem) + xmm0
+; X64-NEXT:    kmovw %edx, %k1
+; X64-NEXT:    vmovss %xmm0, %xmm0, %xmm0 {%k1} {z}
+; X64-NEXT:    vmovss %xmm0, (%rdi)
+; X64-NEXT:    retq
+;
+; X86-LABEL: fmadd_ss_maskz_memfold:
+; X86:       # %bb.0:
+; X86-NEXT:    movb {{[0-9]+}}(%esp), %al
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86-NEXT:    vfmadd231ss {{.*#+}} xmm0 = (xmm0 * mem) + xmm0
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vmovss %xmm0, %xmm0, %xmm0 {%k1} {z}
+; X86-NEXT:    vmovss %xmm0, (%edx)
+; X86-NEXT:    retl
   %a.val = load float, float* %a
   %av0 = insertelement <4 x float> undef, float %a.val, i32 0
   %av1 = insertelement <4 x float> %av0, float 0.000000e+00, i32 1
@@ -4657,15 +6085,28 @@ define void @fmadd_ss_maskz_memfold(float* %a, float* %b, i8 %c) {
 }
 
 define void @fmadd_sd_mask_memfold(double* %a, double* %b, i8 %c) {
-; CHECK-LABEL: fmadd_sd_mask_memfold:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
-; CHECK-NEXT:    vmovsd {{.*#+}} xmm1 = mem[0],zero
-; CHECK-NEXT:    vfmadd213sd {{.*#+}} xmm1 = (xmm0 * xmm1) + xmm0
-; CHECK-NEXT:    kmovw %edx, %k1
-; CHECK-NEXT:    vmovsd %xmm1, %xmm0, %xmm0 {%k1}
-; CHECK-NEXT:    vmovsd %xmm0, (%rdi)
-; CHECK-NEXT:    retq
+; X64-LABEL: fmadd_sd_mask_memfold:
+; X64:       # %bb.0:
+; X64-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; X64-NEXT:    vmovsd {{.*#+}} xmm1 = mem[0],zero
+; X64-NEXT:    vfmadd213sd {{.*#+}} xmm1 = (xmm0 * xmm1) + xmm0
+; X64-NEXT:    kmovw %edx, %k1
+; X64-NEXT:    vmovsd %xmm1, %xmm0, %xmm0 {%k1}
+; X64-NEXT:    vmovsd %xmm0, (%rdi)
+; X64-NEXT:    retq
+;
+; X86-LABEL: fmadd_sd_mask_memfold:
+; X86:       # %bb.0:
+; X86-NEXT:    movb {{[0-9]+}}(%esp), %al
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; X86-NEXT:    vmovsd {{.*#+}} xmm1 = mem[0],zero
+; X86-NEXT:    vfmadd213sd {{.*#+}} xmm1 = (xmm0 * xmm1) + xmm0
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vmovsd %xmm1, %xmm0, %xmm0 {%k1}
+; X86-NEXT:    vmovsd %xmm0, (%edx)
+; X86-NEXT:    retl
   %a.val = load double, double* %a
   %av0 = insertelement <2 x double> undef, double %a.val, i32 0
   %av = insertelement <2 x double> %av0, double 0.000000e+00, i32 1
@@ -4687,14 +6128,26 @@ define void @fmadd_sd_mask_memfold(double* %a, double* %b, i8 %c) {
 }
 
 define void @fmadd_sd_maskz_memfold(double* %a, double* %b, i8 %c) {
-; CHECK-LABEL: fmadd_sd_maskz_memfold:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
-; CHECK-NEXT:    vfmadd231sd {{.*#+}} xmm0 = (xmm0 * mem) + xmm0
-; CHECK-NEXT:    kmovw %edx, %k1
-; CHECK-NEXT:    vmovsd %xmm0, %xmm0, %xmm0 {%k1} {z}
-; CHECK-NEXT:    vmovsd %xmm0, (%rdi)
-; CHECK-NEXT:    retq
+; X64-LABEL: fmadd_sd_maskz_memfold:
+; X64:       # %bb.0:
+; X64-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; X64-NEXT:    vfmadd231sd {{.*#+}} xmm0 = (xmm0 * mem) + xmm0
+; X64-NEXT:    kmovw %edx, %k1
+; X64-NEXT:    vmovsd %xmm0, %xmm0, %xmm0 {%k1} {z}
+; X64-NEXT:    vmovsd %xmm0, (%rdi)
+; X64-NEXT:    retq
+;
+; X86-LABEL: fmadd_sd_maskz_memfold:
+; X86:       # %bb.0:
+; X86-NEXT:    movb {{[0-9]+}}(%esp), %al
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; X86-NEXT:    vfmadd231sd {{.*#+}} xmm0 = (xmm0 * mem) + xmm0
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vmovsd %xmm0, %xmm0, %xmm0 {%k1} {z}
+; X86-NEXT:    vmovsd %xmm0, (%edx)
+; X86-NEXT:    retl
   %a.val = load double, double* %a
   %av0 = insertelement <2 x double> undef, double %a.val, i32 0
   %av = insertelement <2 x double> %av0, double 0.000000e+00, i32 1
@@ -4716,20 +6169,36 @@ define void @fmadd_sd_maskz_memfold(double* %a, double* %b, i8 %c) {
 }
 
 define <2 x double>@test_int_x86_avx512_mask3_vfmsub_sd(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3,i32 %x4 ){
-; CHECK-LABEL: test_int_x86_avx512_mask3_vfmsub_sd:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    vmovapd %xmm2, %xmm3
-; CHECK-NEXT:    vfmsub231sd {{.*#+}} xmm3 = (xmm0 * xmm1) - xmm3
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vmovapd %xmm2, %xmm4
-; CHECK-NEXT:    vfmsub231sd {{.*#+}} xmm4 = (xmm0 * xmm1) - xmm4
-; CHECK-NEXT:    vaddpd %xmm4, %xmm3, %xmm3
-; CHECK-NEXT:    vmovapd %xmm2, %xmm4
-; CHECK-NEXT:    vfmsub231sd {rz-sae}, %xmm1, %xmm0, %xmm4
-; CHECK-NEXT:    vfmsub231sd {rz-sae}, %xmm1, %xmm0, %xmm2 {%k1}
-; CHECK-NEXT:    vaddpd %xmm2, %xmm4, %xmm0
-; CHECK-NEXT:    vaddpd %xmm0, %xmm3, %xmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_int_x86_avx512_mask3_vfmsub_sd:
+; X64:       # %bb.0:
+; X64-NEXT:    vmovapd %xmm2, %xmm3
+; X64-NEXT:    vfmsub231sd {{.*#+}} xmm3 = (xmm0 * xmm1) - xmm3
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vmovapd %xmm2, %xmm4
+; X64-NEXT:    vfmsub231sd {{.*#+}} xmm4 = (xmm0 * xmm1) - xmm4
+; X64-NEXT:    vaddpd %xmm4, %xmm3, %xmm3
+; X64-NEXT:    vmovapd %xmm2, %xmm4
+; X64-NEXT:    vfmsub231sd {rz-sae}, %xmm1, %xmm0, %xmm4
+; X64-NEXT:    vfmsub231sd {rz-sae}, %xmm1, %xmm0, %xmm2 {%k1}
+; X64-NEXT:    vaddpd %xmm2, %xmm4, %xmm0
+; X64-NEXT:    vaddpd %xmm0, %xmm3, %xmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_int_x86_avx512_mask3_vfmsub_sd:
+; X86:       # %bb.0:
+; X86-NEXT:    movb {{[0-9]+}}(%esp), %al
+; X86-NEXT:    vmovapd %xmm2, %xmm3
+; X86-NEXT:    vfmsub231sd {{.*#+}} xmm3 = (xmm0 * xmm1) - xmm3
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vmovapd %xmm2, %xmm4
+; X86-NEXT:    vfmsub231sd {{.*#+}} xmm4 = (xmm0 * xmm1) - xmm4
+; X86-NEXT:    vaddpd %xmm4, %xmm3, %xmm3
+; X86-NEXT:    vmovapd %xmm2, %xmm4
+; X86-NEXT:    vfmsub231sd {rz-sae}, %xmm1, %xmm0, %xmm4
+; X86-NEXT:    vfmsub231sd {rz-sae}, %xmm1, %xmm0, %xmm2 {%k1}
+; X86-NEXT:    vaddpd %xmm2, %xmm4, %xmm0
+; X86-NEXT:    vaddpd %xmm0, %xmm3, %xmm0
+; X86-NEXT:    retl
   %1 = fsub <2 x double> <double -0.000000e+00, double -0.000000e+00>, %x2
   %2 = extractelement <2 x double> %x0, i64 0
   %3 = extractelement <2 x double> %x1, i64 0
@@ -4771,20 +6240,36 @@ define <2 x double>@test_int_x86_avx512_mask3_vfmsub_sd(<2 x double> %x0, <2 x d
 }
 
 define <4 x float>@test_int_x86_avx512_mask3_vfmsub_ss(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3,i32 %x4 ){
-; CHECK-LABEL: test_int_x86_avx512_mask3_vfmsub_ss:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    vmovaps %xmm2, %xmm3
-; CHECK-NEXT:    vfmsub231ss {{.*#+}} xmm3 = (xmm0 * xmm1) - xmm3
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vmovaps %xmm2, %xmm4
-; CHECK-NEXT:    vfmsub231ss {{.*#+}} xmm4 = (xmm0 * xmm1) - xmm4
-; CHECK-NEXT:    vaddps %xmm4, %xmm3, %xmm3
-; CHECK-NEXT:    vmovaps %xmm2, %xmm4
-; CHECK-NEXT:    vfmsub231ss {rz-sae}, %xmm1, %xmm0, %xmm4
-; CHECK-NEXT:    vfmsub231ss {rz-sae}, %xmm1, %xmm0, %xmm2 {%k1}
-; CHECK-NEXT:    vaddps %xmm2, %xmm4, %xmm0
-; CHECK-NEXT:    vaddps %xmm0, %xmm3, %xmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_int_x86_avx512_mask3_vfmsub_ss:
+; X64:       # %bb.0:
+; X64-NEXT:    vmovaps %xmm2, %xmm3
+; X64-NEXT:    vfmsub231ss {{.*#+}} xmm3 = (xmm0 * xmm1) - xmm3
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vmovaps %xmm2, %xmm4
+; X64-NEXT:    vfmsub231ss {{.*#+}} xmm4 = (xmm0 * xmm1) - xmm4
+; X64-NEXT:    vaddps %xmm4, %xmm3, %xmm3
+; X64-NEXT:    vmovaps %xmm2, %xmm4
+; X64-NEXT:    vfmsub231ss {rz-sae}, %xmm1, %xmm0, %xmm4
+; X64-NEXT:    vfmsub231ss {rz-sae}, %xmm1, %xmm0, %xmm2 {%k1}
+; X64-NEXT:    vaddps %xmm2, %xmm4, %xmm0
+; X64-NEXT:    vaddps %xmm0, %xmm3, %xmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_int_x86_avx512_mask3_vfmsub_ss:
+; X86:       # %bb.0:
+; X86-NEXT:    movb {{[0-9]+}}(%esp), %al
+; X86-NEXT:    vmovaps %xmm2, %xmm3
+; X86-NEXT:    vfmsub231ss {{.*#+}} xmm3 = (xmm0 * xmm1) - xmm3
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vmovaps %xmm2, %xmm4
+; X86-NEXT:    vfmsub231ss {{.*#+}} xmm4 = (xmm0 * xmm1) - xmm4
+; X86-NEXT:    vaddps %xmm4, %xmm3, %xmm3
+; X86-NEXT:    vmovaps %xmm2, %xmm4
+; X86-NEXT:    vfmsub231ss {rz-sae}, %xmm1, %xmm0, %xmm4
+; X86-NEXT:    vfmsub231ss {rz-sae}, %xmm1, %xmm0, %xmm2 {%k1}
+; X86-NEXT:    vaddps %xmm2, %xmm4, %xmm0
+; X86-NEXT:    vaddps %xmm0, %xmm3, %xmm0
+; X86-NEXT:    retl
   %1 = fsub <4 x float> <float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00>, %x2
   %2 = extractelement <4 x float> %x0, i64 0
   %3 = extractelement <4 x float> %x1, i64 0
@@ -4826,20 +6311,36 @@ define <4 x float>@test_int_x86_avx512_mask3_vfmsub_ss(<4 x float> %x0, <4 x flo
 }
 
 define <2 x double>@test_int_x86_avx512_mask3_vfnmsub_sd(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3,i32 %x4 ){
-; CHECK-LABEL: test_int_x86_avx512_mask3_vfnmsub_sd:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    vmovapd %xmm2, %xmm3
-; CHECK-NEXT:    vfnmsub231sd {{.*#+}} xmm3 = -(xmm0 * xmm1) - xmm3
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vmovapd %xmm2, %xmm4
-; CHECK-NEXT:    vfnmsub231sd {{.*#+}} xmm4 = -(xmm0 * xmm1) - xmm4
-; CHECK-NEXT:    vaddpd %xmm4, %xmm3, %xmm3
-; CHECK-NEXT:    vmovapd %xmm2, %xmm4
-; CHECK-NEXT:    vfnmsub231sd {rz-sae}, %xmm1, %xmm0, %xmm4
-; CHECK-NEXT:    vfnmsub231sd {rz-sae}, %xmm1, %xmm0, %xmm2 {%k1}
-; CHECK-NEXT:    vaddpd %xmm2, %xmm4, %xmm0
-; CHECK-NEXT:    vaddpd %xmm0, %xmm3, %xmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_int_x86_avx512_mask3_vfnmsub_sd:
+; X64:       # %bb.0:
+; X64-NEXT:    vmovapd %xmm2, %xmm3
+; X64-NEXT:    vfnmsub231sd {{.*#+}} xmm3 = -(xmm0 * xmm1) - xmm3
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vmovapd %xmm2, %xmm4
+; X64-NEXT:    vfnmsub231sd {{.*#+}} xmm4 = -(xmm0 * xmm1) - xmm4
+; X64-NEXT:    vaddpd %xmm4, %xmm3, %xmm3
+; X64-NEXT:    vmovapd %xmm2, %xmm4
+; X64-NEXT:    vfnmsub231sd {rz-sae}, %xmm1, %xmm0, %xmm4
+; X64-NEXT:    vfnmsub231sd {rz-sae}, %xmm1, %xmm0, %xmm2 {%k1}
+; X64-NEXT:    vaddpd %xmm2, %xmm4, %xmm0
+; X64-NEXT:    vaddpd %xmm0, %xmm3, %xmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_int_x86_avx512_mask3_vfnmsub_sd:
+; X86:       # %bb.0:
+; X86-NEXT:    movb {{[0-9]+}}(%esp), %al
+; X86-NEXT:    vmovapd %xmm2, %xmm3
+; X86-NEXT:    vfnmsub231sd {{.*#+}} xmm3 = -(xmm0 * xmm1) - xmm3
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vmovapd %xmm2, %xmm4
+; X86-NEXT:    vfnmsub231sd {{.*#+}} xmm4 = -(xmm0 * xmm1) - xmm4
+; X86-NEXT:    vaddpd %xmm4, %xmm3, %xmm3
+; X86-NEXT:    vmovapd %xmm2, %xmm4
+; X86-NEXT:    vfnmsub231sd {rz-sae}, %xmm1, %xmm0, %xmm4
+; X86-NEXT:    vfnmsub231sd {rz-sae}, %xmm1, %xmm0, %xmm2 {%k1}
+; X86-NEXT:    vaddpd %xmm2, %xmm4, %xmm0
+; X86-NEXT:    vaddpd %xmm0, %xmm3, %xmm0
+; X86-NEXT:    retl
   %1 = fsub <2 x double> <double -0.000000e+00, double -0.000000e+00>, %x0
   %2 = fsub <2 x double> <double -0.000000e+00, double -0.000000e+00>, %x2
   %3 = extractelement <2 x double> %1, i64 0
@@ -4885,20 +6386,36 @@ define <2 x double>@test_int_x86_avx512_mask3_vfnmsub_sd(<2 x double> %x0, <2 x
 }
 
 define <4 x float>@test_int_x86_avx512_mask3_vfnmsub_ss(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3,i32 %x4 ){
-; CHECK-LABEL: test_int_x86_avx512_mask3_vfnmsub_ss:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    vmovaps %xmm2, %xmm3
-; CHECK-NEXT:    vfnmsub231ss {{.*#+}} xmm3 = -(xmm0 * xmm1) - xmm3
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vmovaps %xmm2, %xmm4
-; CHECK-NEXT:    vfnmsub231ss {{.*#+}} xmm4 = -(xmm0 * xmm1) - xmm4
-; CHECK-NEXT:    vaddps %xmm4, %xmm3, %xmm3
-; CHECK-NEXT:    vmovaps %xmm2, %xmm4
-; CHECK-NEXT:    vfnmsub231ss {rz-sae}, %xmm1, %xmm0, %xmm4
-; CHECK-NEXT:    vfnmsub231ss {rz-sae}, %xmm1, %xmm0, %xmm2 {%k1}
-; CHECK-NEXT:    vaddps %xmm2, %xmm4, %xmm0
-; CHECK-NEXT:    vaddps %xmm0, %xmm3, %xmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_int_x86_avx512_mask3_vfnmsub_ss:
+; X64:       # %bb.0:
+; X64-NEXT:    vmovaps %xmm2, %xmm3
+; X64-NEXT:    vfnmsub231ss {{.*#+}} xmm3 = -(xmm0 * xmm1) - xmm3
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vmovaps %xmm2, %xmm4
+; X64-NEXT:    vfnmsub231ss {{.*#+}} xmm4 = -(xmm0 * xmm1) - xmm4
+; X64-NEXT:    vaddps %xmm4, %xmm3, %xmm3
+; X64-NEXT:    vmovaps %xmm2, %xmm4
+; X64-NEXT:    vfnmsub231ss {rz-sae}, %xmm1, %xmm0, %xmm4
+; X64-NEXT:    vfnmsub231ss {rz-sae}, %xmm1, %xmm0, %xmm2 {%k1}
+; X64-NEXT:    vaddps %xmm2, %xmm4, %xmm0
+; X64-NEXT:    vaddps %xmm0, %xmm3, %xmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_int_x86_avx512_mask3_vfnmsub_ss:
+; X86:       # %bb.0:
+; X86-NEXT:    movb {{[0-9]+}}(%esp), %al
+; X86-NEXT:    vmovaps %xmm2, %xmm3
+; X86-NEXT:    vfnmsub231ss {{.*#+}} xmm3 = -(xmm0 * xmm1) - xmm3
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vmovaps %xmm2, %xmm4
+; X86-NEXT:    vfnmsub231ss {{.*#+}} xmm4 = -(xmm0 * xmm1) - xmm4
+; X86-NEXT:    vaddps %xmm4, %xmm3, %xmm3
+; X86-NEXT:    vmovaps %xmm2, %xmm4
+; X86-NEXT:    vfnmsub231ss {rz-sae}, %xmm1, %xmm0, %xmm4
+; X86-NEXT:    vfnmsub231ss {rz-sae}, %xmm1, %xmm0, %xmm2 {%k1}
+; X86-NEXT:    vaddps %xmm2, %xmm4, %xmm0
+; X86-NEXT:    vaddps %xmm0, %xmm3, %xmm0
+; X86-NEXT:    retl
   %1 = fsub <4 x float> <float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00>, %x0
   %2 = fsub <4 x float> <float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00>, %x2
   %3 = extractelement <4 x float> %1, i64 0
@@ -4944,12 +6461,21 @@ define <4 x float>@test_int_x86_avx512_mask3_vfnmsub_ss(<4 x float> %x0, <4 x fl
 }
 
 define <4 x float>@test_int_x86_avx512_mask3_vfmadd_ss_rm(<4 x float> %x0, <4 x float> %x1, float *%ptr_b ,i8 %x3,i32 %x4) {
-; CHECK-LABEL: test_int_x86_avx512_mask3_vfmadd_ss_rm:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %esi, %k1
-; CHECK-NEXT:    vfmadd231ss {{.*#+}} xmm1 = (xmm0 * mem) + xmm1
-; CHECK-NEXT:    vmovaps %xmm1, %xmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_int_x86_avx512_mask3_vfmadd_ss_rm:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %esi, %k1
+; X64-NEXT:    vfmadd231ss {{.*#+}} xmm1 = (xmm0 * mem) + xmm1
+; X64-NEXT:    vmovaps %xmm1, %xmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_int_x86_avx512_mask3_vfmadd_ss_rm:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movb {{[0-9]+}}(%esp), %cl
+; X86-NEXT:    kmovw %ecx, %k1
+; X86-NEXT:    vfmadd231ss {{.*#+}} xmm1 = (xmm0 * mem) + xmm1
+; X86-NEXT:    vmovaps %xmm1, %xmm0
+; X86-NEXT:    retl
   %q = load float, float* %ptr_b
   %vecinit.i = insertelement <4 x float> undef, float %q, i32 0
   %1 = extractelement <4 x float> %x0, i64 0
@@ -4964,11 +6490,19 @@ define <4 x float>@test_int_x86_avx512_mask3_vfmadd_ss_rm(<4 x float> %x0, <4 x
 }
 
 define <4 x float>@test_int_x86_avx512_mask_vfmadd_ss_rm(<4 x float> %x0, <4 x float> %x1,float *%ptr_b ,i8 %x3,i32 %x4) {
-; CHECK-LABEL: test_int_x86_avx512_mask_vfmadd_ss_rm:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %esi, %k1
-; CHECK-NEXT:    vfmadd132ss {{.*#+}} xmm0 = (xmm0 * mem) + xmm1
-; CHECK-NEXT:    retq
+; X64-LABEL: test_int_x86_avx512_mask_vfmadd_ss_rm:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %esi, %k1
+; X64-NEXT:    vfmadd132ss {{.*#+}} xmm0 = (xmm0 * mem) + xmm1
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_int_x86_avx512_mask_vfmadd_ss_rm:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movb {{[0-9]+}}(%esp), %cl
+; X86-NEXT:    kmovw %ecx, %k1
+; X86-NEXT:    vfmadd132ss {{.*#+}} xmm0 = (xmm0 * mem) + xmm1
+; X86-NEXT:    retl
   %q = load float, float* %ptr_b
   %vecinit.i = insertelement <4 x float> undef, float %q, i32 0
   %1 = extractelement <4 x float> %x0, i64 0
@@ -4985,10 +6519,10 @@ define <4 x float>@test_int_x86_avx512_mask_vfmadd_ss_rm(<4 x float> %x0, <4 x f
 
 define <4 x float>@test_int_x86_avx512_maskz_vfmadd_ss_rm(<4 x float> %x0, <4 x float> %x1,float *%ptr_b ,i8 %x3,i32 %x4) {
 ; CHECK-LABEL: test_int_x86_avx512_maskz_vfmadd_ss_rm:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vxorps %xmm1, %xmm1, %xmm1
 ; CHECK-NEXT:    vblendps {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
-; CHECK-NEXT:    retq
+; CHECK-NEXT:    ret{{[l|q]}}
   %q = load float, float* %ptr_b
   %vecinit.i = insertelement <4 x float> undef, float %q, i32 0
   %1 = extractelement <4 x float> %x0, i64 0
@@ -5002,30 +6536,43 @@ define <4 x float>@test_int_x86_avx512_maskz_vfmadd_ss_rm(<4 x float> %x0, <4 x
 
 define <16 x i32> @test_x86_avx512_psll_d_512(<16 x i32> %a0, <4 x i32> %a1) {
 ; CHECK-LABEL: test_x86_avx512_psll_d_512:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vpslld %xmm1, %zmm0, %zmm0
-; CHECK-NEXT:    retq
+; CHECK-NEXT:    ret{{[l|q]}}
   %res = call <16 x i32> @llvm.x86.avx512.psll.d.512(<16 x i32> %a0, <4 x i32> %a1) ; <<16 x i32>> [#uses=1]
   ret <16 x i32> %res
 }
 define <16 x i32> @test_x86_avx512_mask_psll_d_512(<16 x i32> %a0, <4 x i32> %a1, <16 x i32> %passthru, i16 %mask) {
-; CHECK-LABEL: test_x86_avx512_mask_psll_d_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vpslld %xmm1, %zmm0, %zmm2 {%k1}
-; CHECK-NEXT:    vmovdqa64 %zmm2, %zmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_x86_avx512_mask_psll_d_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vpslld %xmm1, %zmm0, %zmm2 {%k1}
+; X64-NEXT:    vmovdqa64 %zmm2, %zmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_x86_avx512_mask_psll_d_512:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vpslld %xmm1, %zmm0, %zmm2 {%k1}
+; X86-NEXT:    vmovdqa64 %zmm2, %zmm0
+; X86-NEXT:    retl
   %res = call <16 x i32> @llvm.x86.avx512.psll.d.512(<16 x i32> %a0, <4 x i32> %a1) ; <<16 x i32>> [#uses=1]
   %mask.cast = bitcast i16 %mask to <16 x i1>
   %res2 = select <16 x i1> %mask.cast, <16 x i32> %res, <16 x i32> %passthru
   ret <16 x i32> %res2
 }
 define <16 x i32> @test_x86_avx512_maskz_psll_d_512(<16 x i32> %a0, <4 x i32> %a1, i16 %mask) {
-; CHECK-LABEL: test_x86_avx512_maskz_psll_d_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vpslld %xmm1, %zmm0, %zmm0 {%k1} {z}
-; CHECK-NEXT:    retq
+; X64-LABEL: test_x86_avx512_maskz_psll_d_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vpslld %xmm1, %zmm0, %zmm0 {%k1} {z}
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_x86_avx512_maskz_psll_d_512:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vpslld %xmm1, %zmm0, %zmm0 {%k1} {z}
+; X86-NEXT:    retl
   %res = call <16 x i32> @llvm.x86.avx512.psll.d.512(<16 x i32> %a0, <4 x i32> %a1) ; <<16 x i32>> [#uses=1]
   %mask.cast = bitcast i16 %mask to <16 x i1>
   %res2 = select <16 x i1> %mask.cast, <16 x i32> %res, <16 x i32> zeroinitializer
@@ -5036,30 +6583,45 @@ declare <16 x i32> @llvm.x86.avx512.psll.d.512(<16 x i32>, <4 x i32>) nounwind r
 
 define <8 x i64> @test_x86_avx512_psll_q_512(<8 x i64> %a0, <2 x i64> %a1) {
 ; CHECK-LABEL: test_x86_avx512_psll_q_512:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vpsllq %xmm1, %zmm0, %zmm0
-; CHECK-NEXT:    retq
+; CHECK-NEXT:    ret{{[l|q]}}
   %res = call <8 x i64> @llvm.x86.avx512.psll.q.512(<8 x i64> %a0, <2 x i64> %a1) ; <<8 x i64>> [#uses=1]
   ret <8 x i64> %res
 }
 define <8 x i64> @test_x86_avx512_mask_psll_q_512(<8 x i64> %a0, <2 x i64> %a1, <8 x i64> %passthru, i8 %mask) {
-; CHECK-LABEL: test_x86_avx512_mask_psll_q_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vpsllq %xmm1, %zmm0, %zmm2 {%k1}
-; CHECK-NEXT:    vmovdqa64 %zmm2, %zmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_x86_avx512_mask_psll_q_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vpsllq %xmm1, %zmm0, %zmm2 {%k1}
+; X64-NEXT:    vmovdqa64 %zmm2, %zmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_x86_avx512_mask_psll_q_512:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vpsllq %xmm1, %zmm0, %zmm2 {%k1}
+; X86-NEXT:    vmovdqa64 %zmm2, %zmm0
+; X86-NEXT:    retl
   %res = call <8 x i64> @llvm.x86.avx512.psll.q.512(<8 x i64> %a0, <2 x i64> %a1) ; <<8 x i64>> [#uses=1]
   %mask.cast = bitcast i8 %mask to <8 x i1>
   %res2 = select <8 x i1> %mask.cast, <8 x i64> %res, <8 x i64> %passthru
   ret <8 x i64> %res2
 }
 define <8 x i64> @test_x86_avx512_maskz_psll_q_512(<8 x i64> %a0, <2 x i64> %a1, i8 %mask) {
-; CHECK-LABEL: test_x86_avx512_maskz_psll_q_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vpsllq %xmm1, %zmm0, %zmm0 {%k1} {z}
-; CHECK-NEXT:    retq
+; X64-LABEL: test_x86_avx512_maskz_psll_q_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vpsllq %xmm1, %zmm0, %zmm0 {%k1} {z}
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_x86_avx512_maskz_psll_q_512:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vpsllq %xmm1, %zmm0, %zmm0 {%k1} {z}
+; X86-NEXT:    retl
   %res = call <8 x i64> @llvm.x86.avx512.psll.q.512(<8 x i64> %a0, <2 x i64> %a1) ; <<8 x i64>> [#uses=1]
   %mask.cast = bitcast i8 %mask to <8 x i1>
   %res2 = select <8 x i1> %mask.cast, <8 x i64> %res, <8 x i64> zeroinitializer
@@ -5070,30 +6632,43 @@ declare <8 x i64> @llvm.x86.avx512.psll.q.512(<8 x i64>, <2 x i64>) nounwind rea
 
 define <16 x i32> @test_x86_avx512_pslli_d_512(<16 x i32> %a0) {
 ; CHECK-LABEL: test_x86_avx512_pslli_d_512:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vpslld $7, %zmm0, %zmm0
-; CHECK-NEXT:    retq
+; CHECK-NEXT:    ret{{[l|q]}}
   %res = call <16 x i32> @llvm.x86.avx512.pslli.d.512(<16 x i32> %a0, i32 7) ; <<16 x i32>> [#uses=1]
   ret <16 x i32> %res
 }
 define <16 x i32> @test_x86_avx512_mask_pslli_d_512(<16 x i32> %a0, <16 x i32> %passthru, i16 %mask) {
-; CHECK-LABEL: test_x86_avx512_mask_pslli_d_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vpslld $7, %zmm0, %zmm1 {%k1}
-; CHECK-NEXT:    vmovdqa64 %zmm1, %zmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_x86_avx512_mask_pslli_d_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vpslld $7, %zmm0, %zmm1 {%k1}
+; X64-NEXT:    vmovdqa64 %zmm1, %zmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_x86_avx512_mask_pslli_d_512:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vpslld $7, %zmm0, %zmm1 {%k1}
+; X86-NEXT:    vmovdqa64 %zmm1, %zmm0
+; X86-NEXT:    retl
   %res = call <16 x i32> @llvm.x86.avx512.pslli.d.512(<16 x i32> %a0, i32 7) ; <<16 x i32>> [#uses=1]
   %mask.cast = bitcast i16 %mask to <16 x i1>
   %res2 = select <16 x i1> %mask.cast, <16 x i32> %res, <16 x i32> %passthru
   ret <16 x i32> %res2
 }
 define <16 x i32> @test_x86_avx512_maskz_pslli_d_512(<16 x i32> %a0, i16 %mask) {
-; CHECK-LABEL: test_x86_avx512_maskz_pslli_d_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vpslld $7, %zmm0, %zmm0 {%k1} {z}
-; CHECK-NEXT:    retq
+; X64-LABEL: test_x86_avx512_maskz_pslli_d_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vpslld $7, %zmm0, %zmm0 {%k1} {z}
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_x86_avx512_maskz_pslli_d_512:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vpslld $7, %zmm0, %zmm0 {%k1} {z}
+; X86-NEXT:    retl
   %res = call <16 x i32> @llvm.x86.avx512.pslli.d.512(<16 x i32> %a0, i32 7) ; <<16 x i32>> [#uses=1]
   %mask.cast = bitcast i16 %mask to <16 x i1>
   %res2 = select <16 x i1> %mask.cast, <16 x i32> %res, <16 x i32> zeroinitializer
@@ -5104,30 +6679,45 @@ declare <16 x i32> @llvm.x86.avx512.pslli.d.512(<16 x i32>, i32) nounwind readno
 
 define <8 x i64> @test_x86_avx512_pslli_q_512(<8 x i64> %a0) {
 ; CHECK-LABEL: test_x86_avx512_pslli_q_512:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vpsllq $7, %zmm0, %zmm0
-; CHECK-NEXT:    retq
+; CHECK-NEXT:    ret{{[l|q]}}
   %res = call <8 x i64> @llvm.x86.avx512.pslli.q.512(<8 x i64> %a0, i32 7) ; <<8 x i64>> [#uses=1]
   ret <8 x i64> %res
 }
 define <8 x i64> @test_x86_avx512_mask_pslli_q_512(<8 x i64> %a0, <8 x i64> %passthru, i8 %mask) {
-; CHECK-LABEL: test_x86_avx512_mask_pslli_q_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vpsllq $7, %zmm0, %zmm1 {%k1}
-; CHECK-NEXT:    vmovdqa64 %zmm1, %zmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_x86_avx512_mask_pslli_q_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vpsllq $7, %zmm0, %zmm1 {%k1}
+; X64-NEXT:    vmovdqa64 %zmm1, %zmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_x86_avx512_mask_pslli_q_512:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vpsllq $7, %zmm0, %zmm1 {%k1}
+; X86-NEXT:    vmovdqa64 %zmm1, %zmm0
+; X86-NEXT:    retl
   %res = call <8 x i64> @llvm.x86.avx512.pslli.q.512(<8 x i64> %a0, i32 7) ; <<8 x i64>> [#uses=1]
   %mask.cast = bitcast i8 %mask to <8 x i1>
   %res2 = select <8 x i1> %mask.cast, <8 x i64> %res, <8 x i64> %passthru
   ret <8 x i64> %res2
 }
 define <8 x i64> @test_x86_avx512_maskz_pslli_q_512(<8 x i64> %a0, <8 x i64> %passthru, i8 %mask) {
-; CHECK-LABEL: test_x86_avx512_maskz_pslli_q_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vpsllq $7, %zmm0, %zmm0 {%k1} {z}
-; CHECK-NEXT:    retq
+; X64-LABEL: test_x86_avx512_maskz_pslli_q_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vpsllq $7, %zmm0, %zmm0 {%k1} {z}
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_x86_avx512_maskz_pslli_q_512:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vpsllq $7, %zmm0, %zmm0 {%k1} {z}
+; X86-NEXT:    retl
   %res = call <8 x i64> @llvm.x86.avx512.pslli.q.512(<8 x i64> %a0, i32 7) ; <<8 x i64>> [#uses=1]
   %mask.cast = bitcast i8 %mask to <8 x i1>
   %res2 = select <8 x i1> %mask.cast, <8 x i64> %res, <8 x i64> zeroinitializer
@@ -5138,30 +6728,45 @@ declare <8 x i64> @llvm.x86.avx512.pslli.q.512(<8 x i64>, i32) nounwind readnone
 
 define <8 x i64> @test_x86_avx512_psra_q_512(<8 x i64> %a0, <2 x i64> %a1) {
 ; CHECK-LABEL: test_x86_avx512_psra_q_512:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vpsraq %xmm1, %zmm0, %zmm0
-; CHECK-NEXT:    retq
+; CHECK-NEXT:    ret{{[l|q]}}
   %res = call <8 x i64> @llvm.x86.avx512.psra.q.512(<8 x i64> %a0, <2 x i64> %a1) ; <<8 x i64>> [#uses=1]
   ret <8 x i64> %res
 }
 define <8 x i64> @test_x86_avx512_mask_psra_q_512(<8 x i64> %a0, <2 x i64> %a1, <8 x i64> %passthru, i8 %mask) {
-; CHECK-LABEL: test_x86_avx512_mask_psra_q_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vpsraq %xmm1, %zmm0, %zmm2 {%k1}
-; CHECK-NEXT:    vmovdqa64 %zmm2, %zmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_x86_avx512_mask_psra_q_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vpsraq %xmm1, %zmm0, %zmm2 {%k1}
+; X64-NEXT:    vmovdqa64 %zmm2, %zmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_x86_avx512_mask_psra_q_512:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vpsraq %xmm1, %zmm0, %zmm2 {%k1}
+; X86-NEXT:    vmovdqa64 %zmm2, %zmm0
+; X86-NEXT:    retl
   %res = call <8 x i64> @llvm.x86.avx512.psra.q.512(<8 x i64> %a0, <2 x i64> %a1) ; <<8 x i64>> [#uses=1]
   %mask.cast = bitcast i8 %mask to <8 x i1>
   %res2 = select <8 x i1> %mask.cast, <8 x i64> %res, <8 x i64> %passthru
   ret <8 x i64> %res2
 }
 define <8 x i64> @test_x86_avx512_maskz_psra_q_512(<8 x i64> %a0, <2 x i64> %a1, i8 %mask) {
-; CHECK-LABEL: test_x86_avx512_maskz_psra_q_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vpsraq %xmm1, %zmm0, %zmm0 {%k1} {z}
-; CHECK-NEXT:    retq
+; X64-LABEL: test_x86_avx512_maskz_psra_q_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vpsraq %xmm1, %zmm0, %zmm0 {%k1} {z}
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_x86_avx512_maskz_psra_q_512:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vpsraq %xmm1, %zmm0, %zmm0 {%k1} {z}
+; X86-NEXT:    retl
   %res = call <8 x i64> @llvm.x86.avx512.psra.q.512(<8 x i64> %a0, <2 x i64> %a1) ; <<8 x i64>> [#uses=1]
   %mask.cast = bitcast i8 %mask to <8 x i1>
   %res2 = select <8 x i1> %mask.cast, <8 x i64> %res, <8 x i64> zeroinitializer
@@ -5172,30 +6777,43 @@ declare <8 x i64> @llvm.x86.avx512.psra.q.512(<8 x i64>, <2 x i64>) nounwind rea
 
 define <16 x i32> @test_x86_avx512_psra_d_512(<16 x i32> %a0, <4 x i32> %a1) {
 ; CHECK-LABEL: test_x86_avx512_psra_d_512:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vpsrad %xmm1, %zmm0, %zmm0
-; CHECK-NEXT:    retq
+; CHECK-NEXT:    ret{{[l|q]}}
   %res = call <16 x i32> @llvm.x86.avx512.psra.d.512(<16 x i32> %a0, <4 x i32> %a1) ; <<16 x i32>> [#uses=1]
   ret <16 x i32> %res
 }
 define <16 x i32> @test_x86_avx512_mask_psra_d_512(<16 x i32> %a0, <4 x i32> %a1, <16 x i32> %passthru, i16 %mask) {
-; CHECK-LABEL: test_x86_avx512_mask_psra_d_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vpsrad %xmm1, %zmm0, %zmm2 {%k1}
-; CHECK-NEXT:    vmovdqa64 %zmm2, %zmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_x86_avx512_mask_psra_d_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vpsrad %xmm1, %zmm0, %zmm2 {%k1}
+; X64-NEXT:    vmovdqa64 %zmm2, %zmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_x86_avx512_mask_psra_d_512:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vpsrad %xmm1, %zmm0, %zmm2 {%k1}
+; X86-NEXT:    vmovdqa64 %zmm2, %zmm0
+; X86-NEXT:    retl
   %res = call <16 x i32> @llvm.x86.avx512.psra.d.512(<16 x i32> %a0, <4 x i32> %a1) ; <<16 x i32>> [#uses=1]
   %mask.cast = bitcast i16 %mask to <16 x i1>
   %res2 = select <16 x i1> %mask.cast, <16 x i32> %res, <16 x i32> %passthru
   ret <16 x i32> %res2
 }
 define <16 x i32> @test_x86_avx512_maskz_psra_d_512(<16 x i32> %a0, <4 x i32> %a1, i16 %mask) {
-; CHECK-LABEL: test_x86_avx512_maskz_psra_d_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vpsrad %xmm1, %zmm0, %zmm0 {%k1} {z}
-; CHECK-NEXT:    retq
+; X64-LABEL: test_x86_avx512_maskz_psra_d_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vpsrad %xmm1, %zmm0, %zmm0 {%k1} {z}
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_x86_avx512_maskz_psra_d_512:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vpsrad %xmm1, %zmm0, %zmm0 {%k1} {z}
+; X86-NEXT:    retl
   %res = call <16 x i32> @llvm.x86.avx512.psra.d.512(<16 x i32> %a0, <4 x i32> %a1) ; <<16 x i32>> [#uses=1]
   %mask.cast = bitcast i16 %mask to <16 x i1>
   %res2 = select <16 x i1> %mask.cast, <16 x i32> %res, <16 x i32> zeroinitializer
@@ -5207,30 +6825,45 @@ declare <16 x i32> @llvm.x86.avx512.psra.d.512(<16 x i32>, <4 x i32>) nounwind r
 
 define <8 x i64> @test_x86_avx512_psrai_q_512(<8 x i64> %a0) {
 ; CHECK-LABEL: test_x86_avx512_psrai_q_512:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vpsraq $7, %zmm0, %zmm0
-; CHECK-NEXT:    retq
+; CHECK-NEXT:    ret{{[l|q]}}
   %res = call <8 x i64> @llvm.x86.avx512.psrai.q.512(<8 x i64> %a0, i32 7) ; <<8 x i64>> [#uses=1]
   ret <8 x i64> %res
 }
 define <8 x i64> @test_x86_avx512_mask_psrai_q_512(<8 x i64> %a0, <8 x i64> %passthru, i8 %mask) {
-; CHECK-LABEL: test_x86_avx512_mask_psrai_q_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vpsraq $7, %zmm0, %zmm1 {%k1}
-; CHECK-NEXT:    vmovdqa64 %zmm1, %zmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_x86_avx512_mask_psrai_q_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vpsraq $7, %zmm0, %zmm1 {%k1}
+; X64-NEXT:    vmovdqa64 %zmm1, %zmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_x86_avx512_mask_psrai_q_512:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vpsraq $7, %zmm0, %zmm1 {%k1}
+; X86-NEXT:    vmovdqa64 %zmm1, %zmm0
+; X86-NEXT:    retl
   %res = call <8 x i64> @llvm.x86.avx512.psrai.q.512(<8 x i64> %a0, i32 7) ; <<8 x i64>> [#uses=1]
   %mask.cast = bitcast i8 %mask to <8 x i1>
   %res2 = select <8 x i1> %mask.cast, <8 x i64> %res, <8 x i64> %passthru
   ret <8 x i64> %res2
 }
 define <8 x i64> @test_x86_avx512_maskz_psrai_q_512(<8 x i64> %a0, i8 %mask) {
-; CHECK-LABEL: test_x86_avx512_maskz_psrai_q_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vpsraq $7, %zmm0, %zmm0 {%k1} {z}
-; CHECK-NEXT:    retq
+; X64-LABEL: test_x86_avx512_maskz_psrai_q_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vpsraq $7, %zmm0, %zmm0 {%k1} {z}
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_x86_avx512_maskz_psrai_q_512:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vpsraq $7, %zmm0, %zmm0 {%k1} {z}
+; X86-NEXT:    retl
   %res = call <8 x i64> @llvm.x86.avx512.psrai.q.512(<8 x i64> %a0, i32 7) ; <<8 x i64>> [#uses=1]
   %mask.cast = bitcast i8 %mask to <8 x i1>
   %res2 = select <8 x i1> %mask.cast, <8 x i64> %res, <8 x i64> zeroinitializer
@@ -5241,30 +6874,43 @@ declare <8 x i64> @llvm.x86.avx512.psrai.q.512(<8 x i64>, i32) nounwind readnone
 
 define <16 x i32> @test_x86_avx512_psrai_d_512(<16 x i32> %a0) {
 ; CHECK-LABEL: test_x86_avx512_psrai_d_512:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vpsrad $7, %zmm0, %zmm0
-; CHECK-NEXT:    retq
+; CHECK-NEXT:    ret{{[l|q]}}
   %res = call <16 x i32> @llvm.x86.avx512.psrai.d.512(<16 x i32> %a0, i32 7) ; <<16 x i32>> [#uses=1]
   ret <16 x i32> %res
 }
 define <16 x i32> @test_x86_avx512_mask_psrai_d_512(<16 x i32> %a0, <16 x i32> %passthru, i16 %mask) {
-; CHECK-LABEL: test_x86_avx512_mask_psrai_d_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vpsrad $7, %zmm0, %zmm1 {%k1}
-; CHECK-NEXT:    vmovdqa64 %zmm1, %zmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_x86_avx512_mask_psrai_d_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vpsrad $7, %zmm0, %zmm1 {%k1}
+; X64-NEXT:    vmovdqa64 %zmm1, %zmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_x86_avx512_mask_psrai_d_512:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vpsrad $7, %zmm0, %zmm1 {%k1}
+; X86-NEXT:    vmovdqa64 %zmm1, %zmm0
+; X86-NEXT:    retl
   %res = call <16 x i32> @llvm.x86.avx512.psrai.d.512(<16 x i32> %a0, i32 7) ; <<16 x i32>> [#uses=1]
   %mask.cast = bitcast i16 %mask to <16 x i1>
   %res2 = select <16 x i1> %mask.cast, <16 x i32> %res, <16 x i32> %passthru
   ret <16 x i32> %res2
 }
 define <16 x i32> @test_x86_avx512_maskz_psrai_d_512(<16 x i32> %a0, i16 %mask) {
-; CHECK-LABEL: test_x86_avx512_maskz_psrai_d_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vpsrad $7, %zmm0, %zmm0 {%k1} {z}
-; CHECK-NEXT:    retq
+; X64-LABEL: test_x86_avx512_maskz_psrai_d_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vpsrad $7, %zmm0, %zmm0 {%k1} {z}
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_x86_avx512_maskz_psrai_d_512:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vpsrad $7, %zmm0, %zmm0 {%k1} {z}
+; X86-NEXT:    retl
   %res = call <16 x i32> @llvm.x86.avx512.psrai.d.512(<16 x i32> %a0, i32 7) ; <<16 x i32>> [#uses=1]
   %mask.cast = bitcast i16 %mask to <16 x i1>
   %res2 = select <16 x i1> %mask.cast, <16 x i32> %res, <16 x i32> zeroinitializer
@@ -5276,30 +6922,43 @@ declare <16 x i32> @llvm.x86.avx512.psrai.d.512(<16 x i32>, i32) nounwind readno
 
 define <16 x i32> @test_x86_avx512_psrl_d_512(<16 x i32> %a0, <4 x i32> %a1) {
 ; CHECK-LABEL: test_x86_avx512_psrl_d_512:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vpsrld %xmm1, %zmm0, %zmm0
-; CHECK-NEXT:    retq
+; CHECK-NEXT:    ret{{[l|q]}}
   %res = call <16 x i32> @llvm.x86.avx512.psrl.d.512(<16 x i32> %a0, <4 x i32> %a1) ; <<16 x i32>> [#uses=1]
   ret <16 x i32> %res
 }
 define <16 x i32> @test_x86_avx512_mask_psrl_d_512(<16 x i32> %a0, <4 x i32> %a1, <16 x i32> %passthru, i16 %mask) {
-; CHECK-LABEL: test_x86_avx512_mask_psrl_d_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vpsrld %xmm1, %zmm0, %zmm2 {%k1}
-; CHECK-NEXT:    vmovdqa64 %zmm2, %zmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_x86_avx512_mask_psrl_d_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vpsrld %xmm1, %zmm0, %zmm2 {%k1}
+; X64-NEXT:    vmovdqa64 %zmm2, %zmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_x86_avx512_mask_psrl_d_512:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vpsrld %xmm1, %zmm0, %zmm2 {%k1}
+; X86-NEXT:    vmovdqa64 %zmm2, %zmm0
+; X86-NEXT:    retl
   %res = call <16 x i32> @llvm.x86.avx512.psrl.d.512(<16 x i32> %a0, <4 x i32> %a1) ; <<16 x i32>> [#uses=1]
   %mask.cast = bitcast i16 %mask to <16 x i1>
   %res2 = select <16 x i1> %mask.cast, <16 x i32> %res, <16 x i32> %passthru
   ret <16 x i32> %res2
 }
 define <16 x i32> @test_x86_avx512_maskz_psrl_d_512(<16 x i32> %a0, <4 x i32> %a1, i16 %mask) {
-; CHECK-LABEL: test_x86_avx512_maskz_psrl_d_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vpsrld %xmm1, %zmm0, %zmm0 {%k1} {z}
-; CHECK-NEXT:    retq
+; X64-LABEL: test_x86_avx512_maskz_psrl_d_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vpsrld %xmm1, %zmm0, %zmm0 {%k1} {z}
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_x86_avx512_maskz_psrl_d_512:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vpsrld %xmm1, %zmm0, %zmm0 {%k1} {z}
+; X86-NEXT:    retl
   %res = call <16 x i32> @llvm.x86.avx512.psrl.d.512(<16 x i32> %a0, <4 x i32> %a1) ; <<16 x i32>> [#uses=1]
   %mask.cast = bitcast i16 %mask to <16 x i1>
   %res2 = select <16 x i1> %mask.cast, <16 x i32> %res, <16 x i32> zeroinitializer
@@ -5310,30 +6969,45 @@ declare <16 x i32> @llvm.x86.avx512.psrl.d.512(<16 x i32>, <4 x i32>) nounwind r
 
 define <8 x i64> @test_x86_avx512_psrl_q_512(<8 x i64> %a0, <2 x i64> %a1) {
 ; CHECK-LABEL: test_x86_avx512_psrl_q_512:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vpsrlq %xmm1, %zmm0, %zmm0
-; CHECK-NEXT:    retq
+; CHECK-NEXT:    ret{{[l|q]}}
   %res = call <8 x i64> @llvm.x86.avx512.psrl.q.512(<8 x i64> %a0, <2 x i64> %a1) ; <<8 x i64>> [#uses=1]
   ret <8 x i64> %res
 }
 define <8 x i64> @test_x86_avx512_mask_psrl_q_512(<8 x i64> %a0, <2 x i64> %a1, <8 x i64> %passthru, i8 %mask) {
-; CHECK-LABEL: test_x86_avx512_mask_psrl_q_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vpsrlq %xmm1, %zmm0, %zmm2 {%k1}
-; CHECK-NEXT:    vmovdqa64 %zmm2, %zmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_x86_avx512_mask_psrl_q_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vpsrlq %xmm1, %zmm0, %zmm2 {%k1}
+; X64-NEXT:    vmovdqa64 %zmm2, %zmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_x86_avx512_mask_psrl_q_512:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vpsrlq %xmm1, %zmm0, %zmm2 {%k1}
+; X86-NEXT:    vmovdqa64 %zmm2, %zmm0
+; X86-NEXT:    retl
   %res = call <8 x i64> @llvm.x86.avx512.psrl.q.512(<8 x i64> %a0, <2 x i64> %a1) ; <<8 x i64>> [#uses=1]
   %mask.cast = bitcast i8 %mask to <8 x i1>
   %res2 = select <8 x i1> %mask.cast, <8 x i64> %res, <8 x i64> %passthru
   ret <8 x i64> %res2
 }
 define <8 x i64> @test_x86_avx512_maskz_psrl_q_512(<8 x i64> %a0, <2 x i64> %a1, i8 %mask) {
-; CHECK-LABEL: test_x86_avx512_maskz_psrl_q_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vpsrlq %xmm1, %zmm0, %zmm0 {%k1} {z}
-; CHECK-NEXT:    retq
+; X64-LABEL: test_x86_avx512_maskz_psrl_q_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vpsrlq %xmm1, %zmm0, %zmm0 {%k1} {z}
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_x86_avx512_maskz_psrl_q_512:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vpsrlq %xmm1, %zmm0, %zmm0 {%k1} {z}
+; X86-NEXT:    retl
   %res = call <8 x i64> @llvm.x86.avx512.psrl.q.512(<8 x i64> %a0, <2 x i64> %a1) ; <<8 x i64>> [#uses=1]
   %mask.cast = bitcast i8 %mask to <8 x i1>
   %res2 = select <8 x i1> %mask.cast, <8 x i64> %res, <8 x i64> zeroinitializer
@@ -5344,30 +7018,43 @@ declare <8 x i64> @llvm.x86.avx512.psrl.q.512(<8 x i64>, <2 x i64>) nounwind rea
 
 define <16 x i32> @test_x86_avx512_psrli_d_512(<16 x i32> %a0) {
 ; CHECK-LABEL: test_x86_avx512_psrli_d_512:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vpsrld $7, %zmm0, %zmm0
-; CHECK-NEXT:    retq
+; CHECK-NEXT:    ret{{[l|q]}}
   %res = call <16 x i32> @llvm.x86.avx512.psrli.d.512(<16 x i32> %a0, i32 7) ; <<16 x i32>> [#uses=1]
   ret <16 x i32> %res
 }
 define <16 x i32> @test_x86_avx512_mask_psrli_d_512(<16 x i32> %a0, <16 x i32> %passthru, i16 %mask) {
-; CHECK-LABEL: test_x86_avx512_mask_psrli_d_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vpsrld $7, %zmm0, %zmm1 {%k1}
-; CHECK-NEXT:    vmovdqa64 %zmm1, %zmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_x86_avx512_mask_psrli_d_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vpsrld $7, %zmm0, %zmm1 {%k1}
+; X64-NEXT:    vmovdqa64 %zmm1, %zmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_x86_avx512_mask_psrli_d_512:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vpsrld $7, %zmm0, %zmm1 {%k1}
+; X86-NEXT:    vmovdqa64 %zmm1, %zmm0
+; X86-NEXT:    retl
   %res = call <16 x i32> @llvm.x86.avx512.psrli.d.512(<16 x i32> %a0, i32 7) ; <<16 x i32>> [#uses=1]
   %mask.cast = bitcast i16 %mask to <16 x i1>
   %res2 = select <16 x i1> %mask.cast, <16 x i32> %res, <16 x i32> %passthru
   ret <16 x i32> %res2
 }
 define <16 x i32> @test_x86_avx512_maskz_psrli_d_512(<16 x i32> %a0, i16 %mask) {
-; CHECK-LABEL: test_x86_avx512_maskz_psrli_d_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vpsrld $7, %zmm0, %zmm0 {%k1} {z}
-; CHECK-NEXT:    retq
+; X64-LABEL: test_x86_avx512_maskz_psrli_d_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vpsrld $7, %zmm0, %zmm0 {%k1} {z}
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_x86_avx512_maskz_psrli_d_512:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vpsrld $7, %zmm0, %zmm0 {%k1} {z}
+; X86-NEXT:    retl
   %res = call <16 x i32> @llvm.x86.avx512.psrli.d.512(<16 x i32> %a0, i32 7) ; <<16 x i32>> [#uses=1]
   %mask.cast = bitcast i16 %mask to <16 x i1>
   %res2 = select <16 x i1> %mask.cast, <16 x i32> %res, <16 x i32> zeroinitializer
@@ -5378,30 +7065,45 @@ declare <16 x i32> @llvm.x86.avx512.psrli.d.512(<16 x i32>, i32) nounwind readno
 
 define <8 x i64> @test_x86_avx512_psrli_q_512(<8 x i64> %a0) {
 ; CHECK-LABEL: test_x86_avx512_psrli_q_512:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vpsrlq $7, %zmm0, %zmm0
-; CHECK-NEXT:    retq
+; CHECK-NEXT:    ret{{[l|q]}}
   %res = call <8 x i64> @llvm.x86.avx512.psrli.q.512(<8 x i64> %a0, i32 7) ; <<8 x i64>> [#uses=1]
   ret <8 x i64> %res
 }
 define <8 x i64> @test_x86_avx512_mask_psrli_q_512(<8 x i64> %a0, <8 x i64> %passthru, i8 %mask) {
-; CHECK-LABEL: test_x86_avx512_mask_psrli_q_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vpsrlq $7, %zmm0, %zmm1 {%k1}
-; CHECK-NEXT:    vmovdqa64 %zmm1, %zmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_x86_avx512_mask_psrli_q_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vpsrlq $7, %zmm0, %zmm1 {%k1}
+; X64-NEXT:    vmovdqa64 %zmm1, %zmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_x86_avx512_mask_psrli_q_512:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vpsrlq $7, %zmm0, %zmm1 {%k1}
+; X86-NEXT:    vmovdqa64 %zmm1, %zmm0
+; X86-NEXT:    retl
   %res = call <8 x i64> @llvm.x86.avx512.psrli.q.512(<8 x i64> %a0, i32 7) ; <<8 x i64>> [#uses=1]
   %mask.cast = bitcast i8 %mask to <8 x i1>
   %res2 = select <8 x i1> %mask.cast, <8 x i64> %res, <8 x i64> %passthru
   ret <8 x i64> %res2
 }
 define <8 x i64> @test_x86_avx512_maskz_psrli_q_512(<8 x i64> %a0, i8 %mask) {
-; CHECK-LABEL: test_x86_avx512_maskz_psrli_q_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vpsrlq $7, %zmm0, %zmm0 {%k1} {z}
-; CHECK-NEXT:    retq
+; X64-LABEL: test_x86_avx512_maskz_psrli_q_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vpsrlq $7, %zmm0, %zmm0 {%k1} {z}
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_x86_avx512_maskz_psrli_q_512:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vpsrlq $7, %zmm0, %zmm0 {%k1} {z}
+; X86-NEXT:    retl
   %res = call <8 x i64> @llvm.x86.avx512.psrli.q.512(<8 x i64> %a0, i32 7) ; <<8 x i64>> [#uses=1]
   %mask.cast = bitcast i8 %mask to <8 x i1>
   %res2 = select <8 x i1> %mask.cast, <8 x i64> %res, <8 x i64> zeroinitializer
@@ -5411,20 +7113,51 @@ declare <8 x i64> @llvm.x86.avx512.psrli.q.512(<8 x i64>, i32) nounwind readnone
 
 define <16 x i32> @test_x86_avx512_psllv_d_512(<16 x i32> %a0, <16 x i32> %a1) {
 ; CHECK-LABEL: test_x86_avx512_psllv_d_512:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vpsllvd %zmm1, %zmm0, %zmm0
-; CHECK-NEXT:    retq
+; CHECK-NEXT:    ret{{[l|q]}}
   %res = call <16 x i32> @llvm.x86.avx512.psllv.d.512(<16 x i32> %a0, <16 x i32> %a1)
   ret <16 x i32> %res
 }
 
+define <16 x i32> @test_x86_avx512_psllv_d_512_const() {
+; X64-LABEL: test_x86_avx512_psllv_d_512_const:
+; X64:       # %bb.0:
+; X64-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [2,9,0,4294967295,3,7,4294967295,0,4,5,4294967294,0,5,3,4294967293,0]
+; X64-NEXT:    vpsllvd {{.*}}(%rip), %zmm0, %zmm0
+; X64-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4294967295]
+; X64-NEXT:    vpsllvd {{.*}}(%rip), %zmm1, %zmm1
+; X64-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_x86_avx512_psllv_d_512_const:
+; X86:       # %bb.0:
+; X86-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [2,9,0,4294967295,3,7,4294967295,0,4,5,4294967294,0,5,3,4294967293,0]
+; X86-NEXT:    vpsllvd {{\.LCPI.*}}, %zmm0, %zmm0
+; X86-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4294967295]
+; X86-NEXT:    vpsllvd {{\.LCPI.*}}, %zmm1, %zmm1
+; X86-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
+; X86-NEXT:    retl
+  %res0 = call <16 x i32> @llvm.x86.avx512.psllv.d.512(<16 x i32> <i32 2, i32 9, i32 0, i32 -1, i32 3, i32 7, i32 -1, i32 0, i32 4, i32 5, i32 -2, i32 0, i32 5, i32 3, i32 -3, i32 0>, <16 x i32> <i32 1, i32 0, i32 33, i32 -1,i32 2, i32 0, i32 34, i32 -2, i32 3, i32 0, i32 35, i32 -1, i32 4, i32 0, i32 36, i32 -3>)
+  %res1 = call <16 x i32> @llvm.x86.avx512.psllv.d.512(<16 x i32> <i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 -1>, <16 x i32> <i32 1, i32 1, i32 1,  i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1,  i32 1, i32 1, i32 1, i32 1, i32 -1>)
+  %res2 = add <16 x i32> %res0, %res1
+  ret <16 x i32> %res2
+}
+
 define <16 x i32> @test_x86_avx512_mask_psllv_d_512(<16 x i32> %a0, <16 x i32> %a1, <16 x i32> %a2, i16 %mask) {
-; CHECK-LABEL: test_x86_avx512_mask_psllv_d_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vpsllvd %zmm1, %zmm0, %zmm2 {%k1}
-; CHECK-NEXT:    vmovdqa64 %zmm2, %zmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_x86_avx512_mask_psllv_d_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vpsllvd %zmm1, %zmm0, %zmm2 {%k1}
+; X64-NEXT:    vmovdqa64 %zmm2, %zmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_x86_avx512_mask_psllv_d_512:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vpsllvd %zmm1, %zmm0, %zmm2 {%k1}
+; X86-NEXT:    vmovdqa64 %zmm2, %zmm0
+; X86-NEXT:    retl
   %res = call <16 x i32> @llvm.x86.avx512.psllv.d.512(<16 x i32> %a0, <16 x i32> %a1)
   %mask.cast = bitcast i16 %mask to <16 x i1>
   %res2 = select <16 x i1> %mask.cast, <16 x i32> %res, <16 x i32> %a2
@@ -5432,11 +7165,17 @@ define <16 x i32> @test_x86_avx512_mask_psllv_d_512(<16 x i32> %a0, <16 x i32> %
 }
 
 define <16 x i32> @test_x86_avx512_maskz_psllv_d_512(<16 x i32> %a0, <16 x i32> %a1, i16 %mask) {
-; CHECK-LABEL: test_x86_avx512_maskz_psllv_d_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vpsllvd %zmm1, %zmm0, %zmm0 {%k1} {z}
-; CHECK-NEXT:    retq
+; X64-LABEL: test_x86_avx512_maskz_psllv_d_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vpsllvd %zmm1, %zmm0, %zmm0 {%k1} {z}
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_x86_avx512_maskz_psllv_d_512:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vpsllvd %zmm1, %zmm0, %zmm0 {%k1} {z}
+; X86-NEXT:    retl
   %res = call <16 x i32> @llvm.x86.avx512.psllv.d.512(<16 x i32> %a0, <16 x i32> %a1)
   %mask.cast = bitcast i16 %mask to <16 x i1>
   %res2 = select <16 x i1> %mask.cast, <16 x i32> %res, <16 x i32> zeroinitializer
@@ -5447,20 +7186,52 @@ declare <16 x i32> @llvm.x86.avx512.psllv.d.512(<16 x i32>, <16 x i32>) nounwind
 
 define <8 x i64> @test_x86_avx512_psllv_q_512(<8 x i64> %a0, <8 x i64> %a1) {
 ; CHECK-LABEL: test_x86_avx512_psllv_q_512:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vpsllvq %zmm1, %zmm0, %zmm0
-; CHECK-NEXT:    retq
+; CHECK-NEXT:    ret{{[l|q]}}
   %res = call <8 x i64> @llvm.x86.avx512.psllv.q.512(<8 x i64> %a0, <8 x i64> %a1)
   ret <8 x i64> %res
 }
 
+define <8 x i64> @test_x86_avx512_psllv_q_512_const() {
+; X64-LABEL: test_x86_avx512_psllv_q_512_const:
+; X64:       # %bb.0:
+; X64-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [2,9,0,18446744073709551615,3,7,18446744073709551615,0]
+; X64-NEXT:    vpsllvq {{.*}}(%rip), %zmm0, %zmm0
+; X64-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [4,4,4,4,4,4,4,18446744073709551615]
+; X64-NEXT:    vpsllvq {{.*}}(%rip), %zmm1, %zmm1
+; X64-NEXT:    vpaddq %zmm1, %zmm0, %zmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_x86_avx512_psllv_q_512_const:
+; X86:       # %bb.0:
+; X86-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [2,0,9,0,0,0,4294967295,4294967295,3,0,7,0,4294967295,4294967295,0,0]
+; X86-NEXT:    vpsllvq {{\.LCPI.*}}, %zmm0, %zmm0
+; X86-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [4,0,4,0,4,0,4,0,4,0,4,0,4,0,4294967295,4294967295]
+; X86-NEXT:    vpsllvq {{\.LCPI.*}}, %zmm1, %zmm1
+; X86-NEXT:    vpaddq %zmm1, %zmm0, %zmm0
+; X86-NEXT:    retl
+  %res0 = call <8 x i64> @llvm.x86.avx512.psllv.q.512(<8 x i64> <i64 2, i64 9, i64 0, i64 -1, i64 3, i64 7, i64 -1, i64 0>, <8 x i64> <i64 1, i64 0, i64 33, i64 -1,i64 2, i64 0, i64 34, i64 -2>)
+  %res1 = call <8 x i64> @llvm.x86.avx512.psllv.q.512(<8 x i64> <i64 4, i64 4, i64 4, i64 4, i64 4, i64 4, i64 4, i64 -1>, <8 x i64> <i64 1, i64 1, i64 1, i64 1,  i64 1, i64 1, i64 1, i64 -1>)
+  %res2 = add <8 x i64> %res0, %res1
+  ret <8 x i64> %res2
+}
+
 define <8 x i64> @test_x86_avx512_mask_psllv_q_512(<8 x i64> %a0, <8 x i64> %a1, <8 x i64> %a2, i8 %mask) {
-; CHECK-LABEL: test_x86_avx512_mask_psllv_q_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vpsllvq %zmm1, %zmm0, %zmm2 {%k1}
-; CHECK-NEXT:    vmovdqa64 %zmm2, %zmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_x86_avx512_mask_psllv_q_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vpsllvq %zmm1, %zmm0, %zmm2 {%k1}
+; X64-NEXT:    vmovdqa64 %zmm2, %zmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_x86_avx512_mask_psllv_q_512:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vpsllvq %zmm1, %zmm0, %zmm2 {%k1}
+; X86-NEXT:    vmovdqa64 %zmm2, %zmm0
+; X86-NEXT:    retl
   %res = call <8 x i64> @llvm.x86.avx512.psllv.q.512(<8 x i64> %a0, <8 x i64> %a1)
   %mask.cast = bitcast i8 %mask to <8 x i1>
   %res2 = select <8 x i1> %mask.cast, <8 x i64> %res, <8 x i64> %a2
@@ -5468,11 +7239,18 @@ define <8 x i64> @test_x86_avx512_mask_psllv_q_512(<8 x i64> %a0, <8 x i64> %a1,
 }
 
 define <8 x i64> @test_x86_avx512_maskz_psllv_q_512(<8 x i64> %a0, <8 x i64> %a1, i8 %mask) {
-; CHECK-LABEL: test_x86_avx512_maskz_psllv_q_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vpsllvq %zmm1, %zmm0, %zmm0 {%k1} {z}
-; CHECK-NEXT:    retq
+; X64-LABEL: test_x86_avx512_maskz_psllv_q_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vpsllvq %zmm1, %zmm0, %zmm0 {%k1} {z}
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_x86_avx512_maskz_psllv_q_512:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vpsllvq %zmm1, %zmm0, %zmm0 {%k1} {z}
+; X86-NEXT:    retl
   %res = call <8 x i64> @llvm.x86.avx512.psllv.q.512(<8 x i64> %a0, <8 x i64> %a1)
   %mask.cast = bitcast i8 %mask to <8 x i1>
   %res2 = select <8 x i1> %mask.cast, <8 x i64> %res, <8 x i64> zeroinitializer
@@ -5483,20 +7261,27 @@ declare <8 x i64> @llvm.x86.avx512.psllv.q.512(<8 x i64>, <8 x i64>) nounwind re
 
 define <16 x i32> @test_x86_avx512_psrav_d_512(<16 x i32> %a0, <16 x i32> %a1) {
 ; CHECK-LABEL: test_x86_avx512_psrav_d_512:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vpsravd %zmm1, %zmm0, %zmm0
-; CHECK-NEXT:    retq
+; CHECK-NEXT:    ret{{[l|q]}}
   %res = call <16 x i32> @llvm.x86.avx512.psrav.d.512(<16 x i32> %a0, <16 x i32> %a1)
   ret <16 x i32> %res
 }
 
 define <16 x i32> @test_x86_avx512_mask_psrav_d_512(<16 x i32> %a0, <16 x i32> %a1, <16 x i32> %a2, i16 %mask) {
-; CHECK-LABEL: test_x86_avx512_mask_psrav_d_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vpsravd %zmm1, %zmm0, %zmm2 {%k1}
-; CHECK-NEXT:    vmovdqa64 %zmm2, %zmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_x86_avx512_mask_psrav_d_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vpsravd %zmm1, %zmm0, %zmm2 {%k1}
+; X64-NEXT:    vmovdqa64 %zmm2, %zmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_x86_avx512_mask_psrav_d_512:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vpsravd %zmm1, %zmm0, %zmm2 {%k1}
+; X86-NEXT:    vmovdqa64 %zmm2, %zmm0
+; X86-NEXT:    retl
   %res = call <16 x i32> @llvm.x86.avx512.psrav.d.512(<16 x i32> %a0, <16 x i32> %a1)
   %mask.cast = bitcast i16 %mask to <16 x i1>
   %res2 = select <16 x i1> %mask.cast, <16 x i32> %res, <16 x i32> %a2
@@ -5504,11 +7289,17 @@ define <16 x i32> @test_x86_avx512_mask_psrav_d_512(<16 x i32> %a0, <16 x i32> %
 }
 
 define <16 x i32> @test_x86_avx512_maskz_psrav_d_512(<16 x i32> %a0, <16 x i32> %a1, i16 %mask) {
-; CHECK-LABEL: test_x86_avx512_maskz_psrav_d_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vpsravd %zmm1, %zmm0, %zmm0 {%k1} {z}
-; CHECK-NEXT:    retq
+; X64-LABEL: test_x86_avx512_maskz_psrav_d_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vpsravd %zmm1, %zmm0, %zmm0 {%k1} {z}
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_x86_avx512_maskz_psrav_d_512:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vpsravd %zmm1, %zmm0, %zmm0 {%k1} {z}
+; X86-NEXT:    retl
   %res = call <16 x i32> @llvm.x86.avx512.psrav.d.512(<16 x i32> %a0, <16 x i32> %a1)
   %mask.cast = bitcast i16 %mask to <16 x i1>
   %res2 = select <16 x i1> %mask.cast, <16 x i32> %res, <16 x i32> zeroinitializer
@@ -5519,20 +7310,28 @@ declare <16 x i32> @llvm.x86.avx512.psrav.d.512(<16 x i32>, <16 x i32>) nounwind
 
 define <8 x i64> @test_x86_avx512_psrav_q_512(<8 x i64> %a0, <8 x i64> %a1) {
 ; CHECK-LABEL: test_x86_avx512_psrav_q_512:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vpsravq %zmm1, %zmm0, %zmm0
-; CHECK-NEXT:    retq
+; CHECK-NEXT:    ret{{[l|q]}}
   %res = call <8 x i64> @llvm.x86.avx512.psrav.q.512(<8 x i64> %a0, <8 x i64> %a1)
   ret <8 x i64> %res
 }
 
 define <8 x i64> @test_x86_avx512_mask_psrav_q_512(<8 x i64> %a0, <8 x i64> %a1, <8 x i64> %a2, i8 %mask) {
-; CHECK-LABEL: test_x86_avx512_mask_psrav_q_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vpsravq %zmm1, %zmm0, %zmm2 {%k1}
-; CHECK-NEXT:    vmovdqa64 %zmm2, %zmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_x86_avx512_mask_psrav_q_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vpsravq %zmm1, %zmm0, %zmm2 {%k1}
+; X64-NEXT:    vmovdqa64 %zmm2, %zmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_x86_avx512_mask_psrav_q_512:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vpsravq %zmm1, %zmm0, %zmm2 {%k1}
+; X86-NEXT:    vmovdqa64 %zmm2, %zmm0
+; X86-NEXT:    retl
   %res = call <8 x i64> @llvm.x86.avx512.psrav.q.512(<8 x i64> %a0, <8 x i64> %a1)
   %mask.cast = bitcast i8 %mask to <8 x i1>
   %res2 = select <8 x i1> %mask.cast, <8 x i64> %res, <8 x i64> %a2
@@ -5540,11 +7339,18 @@ define <8 x i64> @test_x86_avx512_mask_psrav_q_512(<8 x i64> %a0, <8 x i64> %a1,
 }
 
 define <8 x i64> @test_x86_avx512_maskz_psrav_q_512(<8 x i64> %a0, <8 x i64> %a1, i8 %mask) {
-; CHECK-LABEL: test_x86_avx512_maskz_psrav_q_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vpsravq %zmm1, %zmm0, %zmm0 {%k1} {z}
-; CHECK-NEXT:    retq
+; X64-LABEL: test_x86_avx512_maskz_psrav_q_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vpsravq %zmm1, %zmm0, %zmm0 {%k1} {z}
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_x86_avx512_maskz_psrav_q_512:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vpsravq %zmm1, %zmm0, %zmm0 {%k1} {z}
+; X86-NEXT:    retl
   %res = call <8 x i64> @llvm.x86.avx512.psrav.q.512(<8 x i64> %a0, <8 x i64> %a1)
   %mask.cast = bitcast i8 %mask to <8 x i1>
   %res2 = select <8 x i1> %mask.cast, <8 x i64> %res, <8 x i64> zeroinitializer
@@ -5555,20 +7361,51 @@ declare <8 x i64> @llvm.x86.avx512.psrav.q.512(<8 x i64>, <8 x i64>) nounwind re
 
 define <16 x i32> @test_x86_avx512_psrlv_d_512(<16 x i32> %a0, <16 x i32> %a1) {
 ; CHECK-LABEL: test_x86_avx512_psrlv_d_512:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vpsrlvd %zmm1, %zmm0, %zmm0
-; CHECK-NEXT:    retq
+; CHECK-NEXT:    ret{{[l|q]}}
   %res = call <16 x i32> @llvm.x86.avx512.psrlv.d.512(<16 x i32> %a0, <16 x i32> %a1)
   ret <16 x i32> %res
 }
 
+define <16 x i32> @test_x86_avx512_psrlv_d_512_const() {
+; X64-LABEL: test_x86_avx512_psrlv_d_512_const:
+; X64:       # %bb.0:
+; X64-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [2,9,0,4294967295,3,7,4294967295,0,4,5,4294967294,0,5,3,4294967293,0]
+; X64-NEXT:    vpsrlvd {{.*}}(%rip), %zmm0, %zmm0
+; X64-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4294967295]
+; X64-NEXT:    vpsrlvd {{.*}}(%rip), %zmm1, %zmm1
+; X64-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_x86_avx512_psrlv_d_512_const:
+; X86:       # %bb.0:
+; X86-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [2,9,0,4294967295,3,7,4294967295,0,4,5,4294967294,0,5,3,4294967293,0]
+; X86-NEXT:    vpsrlvd {{\.LCPI.*}}, %zmm0, %zmm0
+; X86-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4294967295]
+; X86-NEXT:    vpsrlvd {{\.LCPI.*}}, %zmm1, %zmm1
+; X86-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
+; X86-NEXT:    retl
+  %res0 = call <16 x i32> @llvm.x86.avx512.psrlv.d.512(<16 x i32> <i32 2, i32 9, i32 0, i32 -1, i32 3, i32 7, i32 -1, i32 0, i32 4, i32 5, i32 -2, i32 0, i32 5, i32 3, i32 -3, i32 0>, <16 x i32> <i32 1, i32 0, i32 33, i32 -1,i32 2, i32 0, i32 34, i32 -2, i32 3, i32 0, i32 35, i32 -1, i32 4, i32 0, i32 36, i32 -3>)
+  %res1 = call <16 x i32> @llvm.x86.avx512.psrlv.d.512(<16 x i32> <i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 -1>, <16 x i32> <i32 1, i32 1, i32 1,  i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1,  i32 1, i32 1, i32 1, i32 1, i32 -1  >)
+  %res2 = add <16 x i32> %res0, %res1
+  ret <16 x i32> %res2
+}
+
 define <16 x i32> @test_x86_avx512_mask_psrlv_d_512(<16 x i32> %a0, <16 x i32> %a1, <16 x i32> %a2, i16 %mask) {
-; CHECK-LABEL: test_x86_avx512_mask_psrlv_d_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vpsrlvd %zmm1, %zmm0, %zmm2 {%k1}
-; CHECK-NEXT:    vmovdqa64 %zmm2, %zmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_x86_avx512_mask_psrlv_d_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vpsrlvd %zmm1, %zmm0, %zmm2 {%k1}
+; X64-NEXT:    vmovdqa64 %zmm2, %zmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_x86_avx512_mask_psrlv_d_512:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vpsrlvd %zmm1, %zmm0, %zmm2 {%k1}
+; X86-NEXT:    vmovdqa64 %zmm2, %zmm0
+; X86-NEXT:    retl
   %res = call <16 x i32> @llvm.x86.avx512.psrlv.d.512(<16 x i32> %a0, <16 x i32> %a1)
   %mask.cast = bitcast i16 %mask to <16 x i1>
   %res2 = select <16 x i1> %mask.cast, <16 x i32> %res, <16 x i32> %a2
@@ -5576,11 +7413,17 @@ define <16 x i32> @test_x86_avx512_mask_psrlv_d_512(<16 x i32> %a0, <16 x i32> %
 }
 
 define <16 x i32> @test_x86_avx512_maskz_psrlv_d_512(<16 x i32> %a0, <16 x i32> %a1, i16 %mask) {
-; CHECK-LABEL: test_x86_avx512_maskz_psrlv_d_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vpsrlvd %zmm1, %zmm0, %zmm0 {%k1} {z}
-; CHECK-NEXT:    retq
+; X64-LABEL: test_x86_avx512_maskz_psrlv_d_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vpsrlvd %zmm1, %zmm0, %zmm0 {%k1} {z}
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_x86_avx512_maskz_psrlv_d_512:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vpsrlvd %zmm1, %zmm0, %zmm0 {%k1} {z}
+; X86-NEXT:    retl
   %res = call <16 x i32> @llvm.x86.avx512.psrlv.d.512(<16 x i32> %a0, <16 x i32> %a1)
   %mask.cast = bitcast i16 %mask to <16 x i1>
   %res2 = select <16 x i1> %mask.cast, <16 x i32> %res, <16 x i32> zeroinitializer
@@ -5591,20 +7434,52 @@ declare <16 x i32> @llvm.x86.avx512.psrlv.d.512(<16 x i32>, <16 x i32>) nounwind
 
 define <8 x i64> @test_x86_avx512_psrlv_q_512(<8 x i64> %a0, <8 x i64> %a1) {
 ; CHECK-LABEL: test_x86_avx512_psrlv_q_512:
-; CHECK:       ## %bb.0:
+; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vpsrlvq %zmm1, %zmm0, %zmm0
-; CHECK-NEXT:    retq
+; CHECK-NEXT:    ret{{[l|q]}}
   %res = call <8 x i64> @llvm.x86.avx512.psrlv.q.512(<8 x i64> %a0, <8 x i64> %a1)
   ret <8 x i64> %res
 }
 
+define <8 x i64> @test_x86_avx512_psrlv_q_512_const() {
+; X64-LABEL: test_x86_avx512_psrlv_q_512_const:
+; X64:       # %bb.0:
+; X64-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [2,9,0,18446744073709551615,3,7,18446744073709551615,0]
+; X64-NEXT:    vpsrlvq {{.*}}(%rip), %zmm0, %zmm0
+; X64-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [4,4,4,4,4,4,4,18446744073709551615]
+; X64-NEXT:    vpsrlvq {{.*}}(%rip), %zmm1, %zmm1
+; X64-NEXT:    vpaddq %zmm1, %zmm0, %zmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_x86_avx512_psrlv_q_512_const:
+; X86:       # %bb.0:
+; X86-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [2,0,9,0,0,0,4294967295,4294967295,3,0,7,0,4294967295,4294967295,0,0]
+; X86-NEXT:    vpsrlvq {{\.LCPI.*}}, %zmm0, %zmm0
+; X86-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [4,0,4,0,4,0,4,0,4,0,4,0,4,0,4294967295,4294967295]
+; X86-NEXT:    vpsrlvq {{\.LCPI.*}}, %zmm1, %zmm1
+; X86-NEXT:    vpaddq %zmm1, %zmm0, %zmm0
+; X86-NEXT:    retl
+  %res0 = call <8 x i64> @llvm.x86.avx512.psrlv.q.512(<8 x i64> <i64 2, i64 9, i64 0, i64 -1, i64 3, i64 7, i64 -1, i64 0>, <8 x i64> <i64 1, i64 0, i64 33, i64 -1,i64 2, i64 0, i64 34, i64 -2>)
+  %res1 = call <8 x i64> @llvm.x86.avx512.psrlv.q.512(<8 x i64> <i64 4, i64 4, i64 4, i64 4, i64 4, i64 4, i64 4, i64 -1>, <8 x i64> <i64 1, i64 1, i64 1, i64 1,  i64 1, i64 1, i64 1, i64 -1>)
+  %res2 = add <8 x i64> %res0, %res1
+  ret <8 x i64> %res2
+}
+
 define <8 x i64> @test_x86_avx512_mask_psrlv_q_512(<8 x i64> %a0, <8 x i64> %a1, <8 x i64> %a2, i8 %mask) {
-; CHECK-LABEL: test_x86_avx512_mask_psrlv_q_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vpsrlvq %zmm1, %zmm0, %zmm2 {%k1}
-; CHECK-NEXT:    vmovdqa64 %zmm2, %zmm0
-; CHECK-NEXT:    retq
+; X64-LABEL: test_x86_avx512_mask_psrlv_q_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vpsrlvq %zmm1, %zmm0, %zmm2 {%k1}
+; X64-NEXT:    vmovdqa64 %zmm2, %zmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_x86_avx512_mask_psrlv_q_512:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vpsrlvq %zmm1, %zmm0, %zmm2 {%k1}
+; X86-NEXT:    vmovdqa64 %zmm2, %zmm0
+; X86-NEXT:    retl
   %res = call <8 x i64> @llvm.x86.avx512.psrlv.q.512(<8 x i64> %a0, <8 x i64> %a1)
   %mask.cast = bitcast i8 %mask to <8 x i1>
   %res2 = select <8 x i1> %mask.cast, <8 x i64> %res, <8 x i64> %a2
@@ -5612,11 +7487,18 @@ define <8 x i64> @test_x86_avx512_mask_psrlv_q_512(<8 x i64> %a0, <8 x i64> %a1,
 }
 
 define <8 x i64> @test_x86_avx512_maskz_psrlv_q_512(<8 x i64> %a0, <8 x i64> %a1, i8 %mask) {
-; CHECK-LABEL: test_x86_avx512_maskz_psrlv_q_512:
-; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovw %edi, %k1
-; CHECK-NEXT:    vpsrlvq %zmm1, %zmm0, %zmm0 {%k1} {z}
-; CHECK-NEXT:    retq
+; X64-LABEL: test_x86_avx512_maskz_psrlv_q_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vpsrlvq %zmm1, %zmm0, %zmm0 {%k1} {z}
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_x86_avx512_maskz_psrlv_q_512:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vpsrlvq %zmm1, %zmm0, %zmm0 {%k1} {z}
+; X86-NEXT:    retl
   %res = call <8 x i64> @llvm.x86.avx512.psrlv.q.512(<8 x i64> %a0, <8 x i64> %a1)
   %mask.cast = bitcast i8 %mask to <8 x i1>
   %res2 = select <8 x i1> %mask.cast, <8 x i64> %res, <8 x i64> zeroinitializer
@@ -5626,19 +7508,45 @@ define <8 x i64> @test_x86_avx512_maskz_psrlv_q_512(<8 x i64> %a0, <8 x i64> %a1
 declare <8 x i64> @llvm.x86.avx512.psrlv.q.512(<8 x i64>, <8 x i64>) nounwind readnone
 
 define <16 x float> @bad_mask_transition(<8 x double> %a, <8 x double> %b, <8 x double> %c, <8 x double> %d, <16 x float> %e, <16 x float> %f) {
-; CHECK-LABEL: bad_mask_transition:
-; CHECK:       ## %bb.0: ## %entry
-; CHECK-NEXT:    vcmplt_oqpd %zmm1, %zmm0, %k0
-; CHECK-NEXT:    kmovw %k0, %eax
-; CHECK-NEXT:    vcmplt_oqpd %zmm3, %zmm2, %k0
-; CHECK-NEXT:    kmovw %k0, %ecx
-; CHECK-NEXT:    movzbl %al, %eax
-; CHECK-NEXT:    movzbl %cl, %ecx
-; CHECK-NEXT:    kmovw %eax, %k0
-; CHECK-NEXT:    kmovw %ecx, %k1
-; CHECK-NEXT:    kunpckbw %k0, %k1, %k1
-; CHECK-NEXT:    vblendmps %zmm5, %zmm4, %zmm0 {%k1}
-; CHECK-NEXT:    retq
+; X64-LABEL: bad_mask_transition:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    vcmplt_oqpd %zmm1, %zmm0, %k0
+; X64-NEXT:    kmovw %k0, %eax
+; X64-NEXT:    vcmplt_oqpd %zmm3, %zmm2, %k0
+; X64-NEXT:    kmovw %k0, %ecx
+; X64-NEXT:    movzbl %al, %eax
+; X64-NEXT:    movzbl %cl, %ecx
+; X64-NEXT:    kmovw %eax, %k0
+; X64-NEXT:    kmovw %ecx, %k1
+; X64-NEXT:    kunpckbw %k0, %k1, %k1
+; X64-NEXT:    vblendmps %zmm5, %zmm4, %zmm0 {%k1}
+; X64-NEXT:    retq
+;
+; X86-LABEL: bad_mask_transition:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    pushl %ebp
+; X86-NEXT:    .cfi_def_cfa_offset 8
+; X86-NEXT:    .cfi_offset %ebp, -8
+; X86-NEXT:    movl %esp, %ebp
+; X86-NEXT:    .cfi_def_cfa_register %ebp
+; X86-NEXT:    andl $-64, %esp
+; X86-NEXT:    subl $64, %esp
+; X86-NEXT:    vmovaps 72(%ebp), %zmm3
+; X86-NEXT:    vcmplt_oqpd %zmm1, %zmm0, %k0
+; X86-NEXT:    kmovw %k0, %eax
+; X86-NEXT:    vcmplt_oqpd 8(%ebp), %zmm2, %k0
+; X86-NEXT:    kmovw %k0, %ecx
+; X86-NEXT:    movzbl %al, %eax
+; X86-NEXT:    movzbl %cl, %ecx
+; X86-NEXT:    kmovw %eax, %k0
+; X86-NEXT:    kmovw %ecx, %k1
+; X86-NEXT:    kunpckbw %k0, %k1, %k1
+; X86-NEXT:    vmovaps 136(%ebp), %zmm3 {%k1}
+; X86-NEXT:    vmovaps %zmm3, %zmm0
+; X86-NEXT:    movl %ebp, %esp
+; X86-NEXT:    popl %ebp
+; X86-NEXT:    .cfi_def_cfa %esp, 4
+; X86-NEXT:    retl
 entry:
   %0 = call <8 x i1> @llvm.x86.avx512.cmp.pd.512(<8 x double> %a, <8 x double> %b, i32 17, i32 4)
   %1 = bitcast <8 x i1> %0 to i8
@@ -5656,14 +7564,35 @@ entry:
 }
 
 define <16 x float> @bad_mask_transition_2(<8 x double> %a, <8 x double> %b, <8 x double> %c, <8 x double> %d, <16 x float> %e, <16 x float> %f) {
-; CHECK-LABEL: bad_mask_transition_2:
-; CHECK:       ## %bb.0: ## %entry
-; CHECK-NEXT:    vcmplt_oqpd %zmm1, %zmm0, %k0
-; CHECK-NEXT:    kmovw %k0, %eax
-; CHECK-NEXT:    movzbl %al, %eax
-; CHECK-NEXT:    kmovw %eax, %k1
-; CHECK-NEXT:    vblendmps %zmm5, %zmm4, %zmm0 {%k1}
-; CHECK-NEXT:    retq
+; X64-LABEL: bad_mask_transition_2:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    vcmplt_oqpd %zmm1, %zmm0, %k0
+; X64-NEXT:    kmovw %k0, %eax
+; X64-NEXT:    movzbl %al, %eax
+; X64-NEXT:    kmovw %eax, %k1
+; X64-NEXT:    vblendmps %zmm5, %zmm4, %zmm0 {%k1}
+; X64-NEXT:    retq
+;
+; X86-LABEL: bad_mask_transition_2:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    pushl %ebp
+; X86-NEXT:    .cfi_def_cfa_offset 8
+; X86-NEXT:    .cfi_offset %ebp, -8
+; X86-NEXT:    movl %esp, %ebp
+; X86-NEXT:    .cfi_def_cfa_register %ebp
+; X86-NEXT:    andl $-64, %esp
+; X86-NEXT:    subl $64, %esp
+; X86-NEXT:    vmovaps 72(%ebp), %zmm2
+; X86-NEXT:    vcmplt_oqpd %zmm1, %zmm0, %k0
+; X86-NEXT:    kmovw %k0, %eax
+; X86-NEXT:    movzbl %al, %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vmovaps 136(%ebp), %zmm2 {%k1}
+; X86-NEXT:    vmovaps %zmm2, %zmm0
+; X86-NEXT:    movl %ebp, %esp
+; X86-NEXT:    popl %ebp
+; X86-NEXT:    .cfi_def_cfa %esp, 4
+; X86-NEXT:    retl
 entry:
   %0 = call <8 x i1> @llvm.x86.avx512.cmp.pd.512(<8 x double> %a, <8 x double> %b, i32 17, i32 4)
   %1 = bitcast <8 x i1> %0 to i8
@@ -5672,3 +7601,12 @@ entry:
   %3 = select <16 x i1> %2, <16 x float> %f, <16 x float> %e
   ret <16 x float> %3
 }
+
+declare <8 x double> @llvm.x86.avx512.mask.compress.v8f64(<8 x double>, <8 x double>, <8 x i1>)
+declare <16 x float> @llvm.x86.avx512.mask.compress.v16f32(<16 x float>, <16 x float>, <16 x i1>)
+declare <8 x i64> @llvm.x86.avx512.mask.compress.v8i64(<8 x i64>, <8 x i64>, <8 x i1>)
+declare <16 x i32> @llvm.x86.avx512.mask.compress.v16i32(<16 x i32>, <16 x i32>, <16 x i1>)
+declare <8 x double> @llvm.x86.avx512.mask.expand.v8f64(<8 x double>, <8 x double>, <8 x i1>)
+declare <16 x float> @llvm.x86.avx512.mask.expand.v16f32(<16 x float>, <16 x float>, <16 x i1>)
+declare <8 x i64> @llvm.x86.avx512.mask.expand.v8i64(<8 x i64>, <8 x i64>, <8 x i1>)
+declare <16 x i32> @llvm.x86.avx512.mask.expand.v16i32(<16 x i32>, <16 x i32>, <16 x i1>)
diff --git a/test/CodeGen/X86/avx512-mask-op.ll b/test/CodeGen/X86/avx512-mask-op.ll
index 5996de5455d75a4c67c48da309584f9ac1ffc721..1110e02e5874c44439e4effd97bb35e0a5cbab20 100644
--- a/test/CodeGen/X86/avx512-mask-op.ll
+++ b/test/CodeGen/X86/avx512-mask-op.ll
@@ -1278,19 +1278,19 @@ define <64 x i8> @test17(i64 %x, i32 %y, i32 %z) {
 define <8 x i1> @test18(i8 %a, i16 %y) {
 ; KNL-LABEL: test18:
 ; KNL:       ## %bb.0:
-; KNL-NEXT:    kmovw %edi, %k1
-; KNL-NEXT:    kmovw %esi, %k2
-; KNL-NEXT:    kshiftrw $8, %k2, %k0
-; KNL-NEXT:    kshiftrw $9, %k2, %k2
-; KNL-NEXT:    kshiftrw $6, %k1, %k3
-; KNL-NEXT:    kxorw %k2, %k3, %k2
-; KNL-NEXT:    kshiftlw $15, %k2, %k2
-; KNL-NEXT:    kshiftrw $9, %k2, %k2
-; KNL-NEXT:    kxorw %k2, %k1, %k1
-; KNL-NEXT:    kshiftlw $9, %k1, %k1
+; KNL-NEXT:    kmovw %edi, %k0
+; KNL-NEXT:    kmovw %esi, %k1
+; KNL-NEXT:    kshiftrw $8, %k1, %k2
 ; KNL-NEXT:    kshiftrw $9, %k1, %k1
-; KNL-NEXT:    kshiftlw $7, %k0, %k0
-; KNL-NEXT:    korw %k0, %k1, %k1
+; KNL-NEXT:    kshiftrw $6, %k0, %k3
+; KNL-NEXT:    kxorw %k1, %k3, %k1
+; KNL-NEXT:    kshiftlw $15, %k1, %k1
+; KNL-NEXT:    kshiftrw $9, %k1, %k1
+; KNL-NEXT:    kxorw %k1, %k0, %k0
+; KNL-NEXT:    kshiftlw $9, %k0, %k0
+; KNL-NEXT:    kshiftrw $9, %k0, %k0
+; KNL-NEXT:    kshiftlw $7, %k2, %k1
+; KNL-NEXT:    korw %k1, %k0, %k1
 ; KNL-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
 ; KNL-NEXT:    vpmovdw %zmm0, %ymm0
 ; KNL-NEXT:    ## kill: def $xmm0 killed $xmm0 killed $ymm0
@@ -1299,37 +1299,37 @@ define <8 x i1> @test18(i8 %a, i16 %y) {
 ;
 ; SKX-LABEL: test18:
 ; SKX:       ## %bb.0:
-; SKX-NEXT:    kmovd %edi, %k1
-; SKX-NEXT:    kmovd %esi, %k2
-; SKX-NEXT:    kshiftrw $8, %k2, %k0
-; SKX-NEXT:    kshiftrw $9, %k2, %k2
-; SKX-NEXT:    kshiftrb $6, %k1, %k3
-; SKX-NEXT:    kxorb %k2, %k3, %k2
-; SKX-NEXT:    kshiftlb $7, %k2, %k2
-; SKX-NEXT:    kshiftrb $1, %k2, %k2
-; SKX-NEXT:    kxorb %k2, %k1, %k1
-; SKX-NEXT:    kshiftlb $1, %k1, %k1
+; SKX-NEXT:    kmovd %edi, %k0
+; SKX-NEXT:    kmovd %esi, %k1
+; SKX-NEXT:    kshiftrw $8, %k1, %k2
+; SKX-NEXT:    kshiftrw $9, %k1, %k1
+; SKX-NEXT:    kshiftrb $6, %k0, %k3
+; SKX-NEXT:    kxorb %k1, %k3, %k1
+; SKX-NEXT:    kshiftlb $7, %k1, %k1
 ; SKX-NEXT:    kshiftrb $1, %k1, %k1
-; SKX-NEXT:    kshiftlb $7, %k0, %k0
-; SKX-NEXT:    korb %k0, %k1, %k0
+; SKX-NEXT:    kxorb %k1, %k0, %k0
+; SKX-NEXT:    kshiftlb $1, %k0, %k0
+; SKX-NEXT:    kshiftrb $1, %k0, %k0
+; SKX-NEXT:    kshiftlb $7, %k2, %k1
+; SKX-NEXT:    korb %k1, %k0, %k0
 ; SKX-NEXT:    vpmovm2w %k0, %xmm0
 ; SKX-NEXT:    retq
 ;
 ; AVX512BW-LABEL: test18:
 ; AVX512BW:       ## %bb.0:
-; AVX512BW-NEXT:    kmovd %edi, %k1
-; AVX512BW-NEXT:    kmovd %esi, %k2
-; AVX512BW-NEXT:    kshiftrw $8, %k2, %k0
-; AVX512BW-NEXT:    kshiftrw $9, %k2, %k2
-; AVX512BW-NEXT:    kshiftrw $6, %k1, %k3
-; AVX512BW-NEXT:    kxorw %k2, %k3, %k2
-; AVX512BW-NEXT:    kshiftlw $15, %k2, %k2
-; AVX512BW-NEXT:    kshiftrw $9, %k2, %k2
-; AVX512BW-NEXT:    kxorw %k2, %k1, %k1
-; AVX512BW-NEXT:    kshiftlw $9, %k1, %k1
+; AVX512BW-NEXT:    kmovd %edi, %k0
+; AVX512BW-NEXT:    kmovd %esi, %k1
+; AVX512BW-NEXT:    kshiftrw $8, %k1, %k2
+; AVX512BW-NEXT:    kshiftrw $9, %k1, %k1
+; AVX512BW-NEXT:    kshiftrw $6, %k0, %k3
+; AVX512BW-NEXT:    kxorw %k1, %k3, %k1
+; AVX512BW-NEXT:    kshiftlw $15, %k1, %k1
 ; AVX512BW-NEXT:    kshiftrw $9, %k1, %k1
-; AVX512BW-NEXT:    kshiftlw $7, %k0, %k0
-; AVX512BW-NEXT:    korw %k0, %k1, %k0
+; AVX512BW-NEXT:    kxorw %k1, %k0, %k0
+; AVX512BW-NEXT:    kshiftlw $9, %k0, %k0
+; AVX512BW-NEXT:    kshiftrw $9, %k0, %k0
+; AVX512BW-NEXT:    kshiftlw $7, %k2, %k1
+; AVX512BW-NEXT:    korw %k1, %k0, %k0
 ; AVX512BW-NEXT:    vpmovm2w %k0, %zmm0
 ; AVX512BW-NEXT:    ## kill: def $xmm0 killed $xmm0 killed $zmm0
 ; AVX512BW-NEXT:    vzeroupper
@@ -1337,19 +1337,19 @@ define <8 x i1> @test18(i8 %a, i16 %y) {
 ;
 ; AVX512DQ-LABEL: test18:
 ; AVX512DQ:       ## %bb.0:
-; AVX512DQ-NEXT:    kmovw %edi, %k1
-; AVX512DQ-NEXT:    kmovw %esi, %k2
-; AVX512DQ-NEXT:    kshiftrw $8, %k2, %k0
-; AVX512DQ-NEXT:    kshiftrw $9, %k2, %k2
-; AVX512DQ-NEXT:    kshiftrb $6, %k1, %k3
-; AVX512DQ-NEXT:    kxorb %k2, %k3, %k2
-; AVX512DQ-NEXT:    kshiftlb $7, %k2, %k2
-; AVX512DQ-NEXT:    kshiftrb $1, %k2, %k2
-; AVX512DQ-NEXT:    kxorb %k2, %k1, %k1
-; AVX512DQ-NEXT:    kshiftlb $1, %k1, %k1
+; AVX512DQ-NEXT:    kmovw %edi, %k0
+; AVX512DQ-NEXT:    kmovw %esi, %k1
+; AVX512DQ-NEXT:    kshiftrw $8, %k1, %k2
+; AVX512DQ-NEXT:    kshiftrw $9, %k1, %k1
+; AVX512DQ-NEXT:    kshiftrb $6, %k0, %k3
+; AVX512DQ-NEXT:    kxorb %k1, %k3, %k1
+; AVX512DQ-NEXT:    kshiftlb $7, %k1, %k1
 ; AVX512DQ-NEXT:    kshiftrb $1, %k1, %k1
-; AVX512DQ-NEXT:    kshiftlb $7, %k0, %k0
-; AVX512DQ-NEXT:    korb %k0, %k1, %k0
+; AVX512DQ-NEXT:    kxorb %k1, %k0, %k0
+; AVX512DQ-NEXT:    kshiftlb $1, %k0, %k0
+; AVX512DQ-NEXT:    kshiftrb $1, %k0, %k0
+; AVX512DQ-NEXT:    kshiftlb $7, %k2, %k1
+; AVX512DQ-NEXT:    korb %k1, %k0, %k0
 ; AVX512DQ-NEXT:    vpmovm2d %k0, %zmm0
 ; AVX512DQ-NEXT:    vpmovdw %zmm0, %ymm0
 ; AVX512DQ-NEXT:    ## kill: def $xmm0 killed $xmm0 killed $ymm0
@@ -1360,16 +1360,16 @@ define <8 x i1> @test18(i8 %a, i16 %y) {
 ; X86:       ## %bb.0:
 ; X86-NEXT:    kmovb {{[0-9]+}}(%esp), %k0
 ; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
-; X86-NEXT:    kshiftrw $9, %k1, %k2
-; X86-NEXT:    kshiftrw $8, %k1, %k1
-; X86-NEXT:    kshiftlb $7, %k1, %k1
+; X86-NEXT:    kshiftrw $8, %k1, %k2
+; X86-NEXT:    kshiftrw $9, %k1, %k1
 ; X86-NEXT:    kshiftrb $6, %k0, %k3
-; X86-NEXT:    kxorb %k2, %k3, %k2
-; X86-NEXT:    kshiftlb $7, %k2, %k2
-; X86-NEXT:    kshiftrb $1, %k2, %k2
-; X86-NEXT:    kxorb %k2, %k0, %k0
+; X86-NEXT:    kxorb %k1, %k3, %k1
+; X86-NEXT:    kshiftlb $7, %k1, %k1
+; X86-NEXT:    kshiftrb $1, %k1, %k1
+; X86-NEXT:    kxorb %k1, %k0, %k0
 ; X86-NEXT:    kshiftlb $1, %k0, %k0
 ; X86-NEXT:    kshiftrb $1, %k0, %k0
+; X86-NEXT:    kshiftlb $7, %k2, %k1
 ; X86-NEXT:    korb %k1, %k0, %k0
 ; X86-NEXT:    vpmovm2w %k0, %xmm0
 ; X86-NEXT:    retl
@@ -2197,8 +2197,7 @@ End:
 define <8 x i64> @load_8i1(<8 x i1>* %a) {
 ; KNL-LABEL: load_8i1:
 ; KNL:       ## %bb.0:
-; KNL-NEXT:    movzbl (%rdi), %eax
-; KNL-NEXT:    kmovw %eax, %k1
+; KNL-NEXT:    kmovw (%rdi), %k1
 ; KNL-NEXT:    vpternlogq $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
 ; KNL-NEXT:    retq
 ;
@@ -2210,8 +2209,7 @@ define <8 x i64> @load_8i1(<8 x i1>* %a) {
 ;
 ; AVX512BW-LABEL: load_8i1:
 ; AVX512BW:       ## %bb.0:
-; AVX512BW-NEXT:    movzbl (%rdi), %eax
-; AVX512BW-NEXT:    kmovd %eax, %k1
+; AVX512BW-NEXT:    kmovw (%rdi), %k1
 ; AVX512BW-NEXT:    vpternlogq $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
 ; AVX512BW-NEXT:    retq
 ;
@@ -2271,8 +2269,7 @@ define <16 x i32> @load_16i1(<16 x i1>* %a) {
 define <2 x i16> @load_2i1(<2 x i1>* %a) {
 ; KNL-LABEL: load_2i1:
 ; KNL:       ## %bb.0:
-; KNL-NEXT:    movzbl (%rdi), %eax
-; KNL-NEXT:    kmovw %eax, %k1
+; KNL-NEXT:    kmovw (%rdi), %k1
 ; KNL-NEXT:    vpternlogq $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
 ; KNL-NEXT:    ## kill: def $xmm0 killed $xmm0 killed $zmm0
 ; KNL-NEXT:    vzeroupper
@@ -2286,8 +2283,7 @@ define <2 x i16> @load_2i1(<2 x i1>* %a) {
 ;
 ; AVX512BW-LABEL: load_2i1:
 ; AVX512BW:       ## %bb.0:
-; AVX512BW-NEXT:    movzbl (%rdi), %eax
-; AVX512BW-NEXT:    kmovd %eax, %k1
+; AVX512BW-NEXT:    kmovw (%rdi), %k1
 ; AVX512BW-NEXT:    vpternlogq $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
 ; AVX512BW-NEXT:    ## kill: def $xmm0 killed $xmm0 killed $zmm0
 ; AVX512BW-NEXT:    vzeroupper
@@ -2315,8 +2311,7 @@ define <2 x i16> @load_2i1(<2 x i1>* %a) {
 define <4 x i16> @load_4i1(<4 x i1>* %a) {
 ; KNL-LABEL: load_4i1:
 ; KNL:       ## %bb.0:
-; KNL-NEXT:    movzbl (%rdi), %eax
-; KNL-NEXT:    kmovw %eax, %k1
+; KNL-NEXT:    kmovw (%rdi), %k1
 ; KNL-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
 ; KNL-NEXT:    ## kill: def $xmm0 killed $xmm0 killed $zmm0
 ; KNL-NEXT:    vzeroupper
@@ -2330,8 +2325,7 @@ define <4 x i16> @load_4i1(<4 x i1>* %a) {
 ;
 ; AVX512BW-LABEL: load_4i1:
 ; AVX512BW:       ## %bb.0:
-; AVX512BW-NEXT:    movzbl (%rdi), %eax
-; AVX512BW-NEXT:    kmovd %eax, %k1
+; AVX512BW-NEXT:    kmovw (%rdi), %k1
 ; AVX512BW-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
 ; AVX512BW-NEXT:    ## kill: def $xmm0 killed $xmm0 killed $zmm0
 ; AVX512BW-NEXT:    vzeroupper
@@ -3479,3 +3473,784 @@ define void @mask_not_cast(i8*, <8 x i64>, <8 x i64>, <8 x i64>, <8 x i64>) {
   ret void
 }
 declare void @llvm.masked.store.v16i32.p0v16i32(<16 x i32>, <16 x i32>*, i32, <16 x i1>)
+
+define void @ktest_3(<8 x i32> %w, <8 x i32> %x, <8 x i32> %y, <8 x i32> %z) {
+; KNL-LABEL: ktest_3:
+; KNL:       ## %bb.0:
+; KNL-NEXT:    pushq %rax
+; KNL-NEXT:    .cfi_def_cfa_offset 16
+; KNL-NEXT:    ## kill: def $ymm3 killed $ymm3 def $zmm3
+; KNL-NEXT:    ## kill: def $ymm2 killed $ymm2 def $zmm2
+; KNL-NEXT:    ## kill: def $ymm1 killed $ymm1 def $zmm1
+; KNL-NEXT:    ## kill: def $ymm0 killed $ymm0 def $zmm0
+; KNL-NEXT:    vptestnmd %zmm0, %zmm0, %k0
+; KNL-NEXT:    vptestnmd %zmm1, %zmm1, %k1
+; KNL-NEXT:    vptestnmd %zmm2, %zmm2, %k2
+; KNL-NEXT:    vptestnmd %zmm3, %zmm3, %k3
+; KNL-NEXT:    korw %k1, %k0, %k0
+; KNL-NEXT:    korw %k3, %k2, %k1
+; KNL-NEXT:    kandw %k1, %k0, %k0
+; KNL-NEXT:    kmovw %k0, %eax
+; KNL-NEXT:    testb %al, %al
+; KNL-NEXT:    je LBB71_1
+; KNL-NEXT:  ## %bb.2: ## %exit
+; KNL-NEXT:    popq %rax
+; KNL-NEXT:    vzeroupper
+; KNL-NEXT:    retq
+; KNL-NEXT:  LBB71_1: ## %bar
+; KNL-NEXT:    vzeroupper
+; KNL-NEXT:    callq _foo
+; KNL-NEXT:    popq %rax
+; KNL-NEXT:    retq
+;
+; SKX-LABEL: ktest_3:
+; SKX:       ## %bb.0:
+; SKX-NEXT:    pushq %rax
+; SKX-NEXT:    .cfi_def_cfa_offset 16
+; SKX-NEXT:    vptestnmd %ymm0, %ymm0, %k0
+; SKX-NEXT:    vptestnmd %ymm1, %ymm1, %k1
+; SKX-NEXT:    korb %k1, %k0, %k0
+; SKX-NEXT:    vptestnmd %ymm2, %ymm2, %k1
+; SKX-NEXT:    vptestnmd %ymm3, %ymm3, %k2
+; SKX-NEXT:    korb %k2, %k1, %k1
+; SKX-NEXT:    ktestb %k1, %k0
+; SKX-NEXT:    je LBB71_1
+; SKX-NEXT:  ## %bb.2: ## %exit
+; SKX-NEXT:    popq %rax
+; SKX-NEXT:    vzeroupper
+; SKX-NEXT:    retq
+; SKX-NEXT:  LBB71_1: ## %bar
+; SKX-NEXT:    vzeroupper
+; SKX-NEXT:    callq _foo
+; SKX-NEXT:    popq %rax
+; SKX-NEXT:    retq
+;
+; AVX512BW-LABEL: ktest_3:
+; AVX512BW:       ## %bb.0:
+; AVX512BW-NEXT:    pushq %rax
+; AVX512BW-NEXT:    .cfi_def_cfa_offset 16
+; AVX512BW-NEXT:    ## kill: def $ymm3 killed $ymm3 def $zmm3
+; AVX512BW-NEXT:    ## kill: def $ymm2 killed $ymm2 def $zmm2
+; AVX512BW-NEXT:    ## kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512BW-NEXT:    ## kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512BW-NEXT:    vptestnmd %zmm0, %zmm0, %k0
+; AVX512BW-NEXT:    vptestnmd %zmm1, %zmm1, %k1
+; AVX512BW-NEXT:    vptestnmd %zmm2, %zmm2, %k2
+; AVX512BW-NEXT:    vptestnmd %zmm3, %zmm3, %k3
+; AVX512BW-NEXT:    korw %k1, %k0, %k0
+; AVX512BW-NEXT:    korw %k3, %k2, %k1
+; AVX512BW-NEXT:    kandw %k1, %k0, %k0
+; AVX512BW-NEXT:    kmovd %k0, %eax
+; AVX512BW-NEXT:    testb %al, %al
+; AVX512BW-NEXT:    je LBB71_1
+; AVX512BW-NEXT:  ## %bb.2: ## %exit
+; AVX512BW-NEXT:    popq %rax
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+; AVX512BW-NEXT:  LBB71_1: ## %bar
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    callq _foo
+; AVX512BW-NEXT:    popq %rax
+; AVX512BW-NEXT:    retq
+;
+; AVX512DQ-LABEL: ktest_3:
+; AVX512DQ:       ## %bb.0:
+; AVX512DQ-NEXT:    pushq %rax
+; AVX512DQ-NEXT:    .cfi_def_cfa_offset 16
+; AVX512DQ-NEXT:    ## kill: def $ymm3 killed $ymm3 def $zmm3
+; AVX512DQ-NEXT:    ## kill: def $ymm2 killed $ymm2 def $zmm2
+; AVX512DQ-NEXT:    ## kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512DQ-NEXT:    ## kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512DQ-NEXT:    vptestnmd %zmm0, %zmm0, %k0
+; AVX512DQ-NEXT:    vptestnmd %zmm1, %zmm1, %k1
+; AVX512DQ-NEXT:    vptestnmd %zmm2, %zmm2, %k2
+; AVX512DQ-NEXT:    vptestnmd %zmm3, %zmm3, %k3
+; AVX512DQ-NEXT:    korb %k1, %k0, %k0
+; AVX512DQ-NEXT:    korb %k3, %k2, %k1
+; AVX512DQ-NEXT:    ktestb %k1, %k0
+; AVX512DQ-NEXT:    je LBB71_1
+; AVX512DQ-NEXT:  ## %bb.2: ## %exit
+; AVX512DQ-NEXT:    popq %rax
+; AVX512DQ-NEXT:    vzeroupper
+; AVX512DQ-NEXT:    retq
+; AVX512DQ-NEXT:  LBB71_1: ## %bar
+; AVX512DQ-NEXT:    vzeroupper
+; AVX512DQ-NEXT:    callq _foo
+; AVX512DQ-NEXT:    popq %rax
+; AVX512DQ-NEXT:    retq
+;
+; X86-LABEL: ktest_3:
+; X86:       ## %bb.0:
+; X86-NEXT:    subl $12, %esp
+; X86-NEXT:    .cfi_def_cfa_offset 16
+; X86-NEXT:    vptestnmd %ymm0, %ymm0, %k0
+; X86-NEXT:    vptestnmd %ymm1, %ymm1, %k1
+; X86-NEXT:    korb %k1, %k0, %k0
+; X86-NEXT:    vptestnmd %ymm2, %ymm2, %k1
+; X86-NEXT:    vptestnmd %ymm3, %ymm3, %k2
+; X86-NEXT:    korb %k2, %k1, %k1
+; X86-NEXT:    ktestb %k1, %k0
+; X86-NEXT:    je LBB71_1
+; X86-NEXT:  ## %bb.2: ## %exit
+; X86-NEXT:    addl $12, %esp
+; X86-NEXT:    vzeroupper
+; X86-NEXT:    retl
+; X86-NEXT:  LBB71_1: ## %bar
+; X86-NEXT:    vzeroupper
+; X86-NEXT:    calll _foo
+; X86-NEXT:    addl $12, %esp
+; X86-NEXT:    retl
+  %a = icmp eq <8 x i32> %w, zeroinitializer
+  %b = icmp eq <8 x i32> %x, zeroinitializer
+  %c = icmp eq <8 x i32> %y, zeroinitializer
+  %d = icmp eq <8 x i32> %z, zeroinitializer
+  %e = or <8 x i1> %a, %b
+  %f = or <8 x i1> %c, %d
+  %g = and <8 x i1> %e, %f
+  %h = bitcast <8 x i1> %g to i8
+  %i = icmp eq i8 %h, 0
+  br i1 %i, label %bar, label %exit
+
+bar:
+  call void @foo()
+  br label %exit
+
+exit:
+  ret void
+}
+
+define void @ktest_4(<8 x i64> %w, <8 x i64> %x, <8 x i64> %y, <8 x i64> %z) {
+; KNL-LABEL: ktest_4:
+; KNL:       ## %bb.0:
+; KNL-NEXT:    pushq %rax
+; KNL-NEXT:    .cfi_def_cfa_offset 16
+; KNL-NEXT:    vptestnmq %zmm0, %zmm0, %k0
+; KNL-NEXT:    vptestnmq %zmm1, %zmm1, %k1
+; KNL-NEXT:    vptestnmq %zmm2, %zmm2, %k2
+; KNL-NEXT:    vptestnmq %zmm3, %zmm3, %k3
+; KNL-NEXT:    korw %k1, %k0, %k0
+; KNL-NEXT:    korw %k3, %k2, %k1
+; KNL-NEXT:    kandw %k1, %k0, %k0
+; KNL-NEXT:    kmovw %k0, %eax
+; KNL-NEXT:    testb %al, %al
+; KNL-NEXT:    je LBB72_1
+; KNL-NEXT:  ## %bb.2: ## %exit
+; KNL-NEXT:    popq %rax
+; KNL-NEXT:    vzeroupper
+; KNL-NEXT:    retq
+; KNL-NEXT:  LBB72_1: ## %bar
+; KNL-NEXT:    vzeroupper
+; KNL-NEXT:    callq _foo
+; KNL-NEXT:    popq %rax
+; KNL-NEXT:    retq
+;
+; SKX-LABEL: ktest_4:
+; SKX:       ## %bb.0:
+; SKX-NEXT:    pushq %rax
+; SKX-NEXT:    .cfi_def_cfa_offset 16
+; SKX-NEXT:    vptestnmq %zmm0, %zmm0, %k0
+; SKX-NEXT:    vptestnmq %zmm1, %zmm1, %k1
+; SKX-NEXT:    korb %k1, %k0, %k0
+; SKX-NEXT:    vptestnmq %zmm2, %zmm2, %k1
+; SKX-NEXT:    vptestnmq %zmm3, %zmm3, %k2
+; SKX-NEXT:    korb %k2, %k1, %k1
+; SKX-NEXT:    ktestb %k1, %k0
+; SKX-NEXT:    je LBB72_1
+; SKX-NEXT:  ## %bb.2: ## %exit
+; SKX-NEXT:    popq %rax
+; SKX-NEXT:    vzeroupper
+; SKX-NEXT:    retq
+; SKX-NEXT:  LBB72_1: ## %bar
+; SKX-NEXT:    vzeroupper
+; SKX-NEXT:    callq _foo
+; SKX-NEXT:    popq %rax
+; SKX-NEXT:    retq
+;
+; AVX512BW-LABEL: ktest_4:
+; AVX512BW:       ## %bb.0:
+; AVX512BW-NEXT:    pushq %rax
+; AVX512BW-NEXT:    .cfi_def_cfa_offset 16
+; AVX512BW-NEXT:    vptestnmq %zmm0, %zmm0, %k0
+; AVX512BW-NEXT:    vptestnmq %zmm1, %zmm1, %k1
+; AVX512BW-NEXT:    vptestnmq %zmm2, %zmm2, %k2
+; AVX512BW-NEXT:    vptestnmq %zmm3, %zmm3, %k3
+; AVX512BW-NEXT:    korw %k1, %k0, %k0
+; AVX512BW-NEXT:    korw %k3, %k2, %k1
+; AVX512BW-NEXT:    kandw %k1, %k0, %k0
+; AVX512BW-NEXT:    kmovd %k0, %eax
+; AVX512BW-NEXT:    testb %al, %al
+; AVX512BW-NEXT:    je LBB72_1
+; AVX512BW-NEXT:  ## %bb.2: ## %exit
+; AVX512BW-NEXT:    popq %rax
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+; AVX512BW-NEXT:  LBB72_1: ## %bar
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    callq _foo
+; AVX512BW-NEXT:    popq %rax
+; AVX512BW-NEXT:    retq
+;
+; AVX512DQ-LABEL: ktest_4:
+; AVX512DQ:       ## %bb.0:
+; AVX512DQ-NEXT:    pushq %rax
+; AVX512DQ-NEXT:    .cfi_def_cfa_offset 16
+; AVX512DQ-NEXT:    vptestnmq %zmm0, %zmm0, %k0
+; AVX512DQ-NEXT:    vptestnmq %zmm1, %zmm1, %k1
+; AVX512DQ-NEXT:    korb %k1, %k0, %k0
+; AVX512DQ-NEXT:    vptestnmq %zmm2, %zmm2, %k1
+; AVX512DQ-NEXT:    vptestnmq %zmm3, %zmm3, %k2
+; AVX512DQ-NEXT:    korb %k2, %k1, %k1
+; AVX512DQ-NEXT:    ktestb %k1, %k0
+; AVX512DQ-NEXT:    je LBB72_1
+; AVX512DQ-NEXT:  ## %bb.2: ## %exit
+; AVX512DQ-NEXT:    popq %rax
+; AVX512DQ-NEXT:    vzeroupper
+; AVX512DQ-NEXT:    retq
+; AVX512DQ-NEXT:  LBB72_1: ## %bar
+; AVX512DQ-NEXT:    vzeroupper
+; AVX512DQ-NEXT:    callq _foo
+; AVX512DQ-NEXT:    popq %rax
+; AVX512DQ-NEXT:    retq
+;
+; X86-LABEL: ktest_4:
+; X86:       ## %bb.0:
+; X86-NEXT:    subl $12, %esp
+; X86-NEXT:    .cfi_def_cfa_offset 16
+; X86-NEXT:    vptestnmq %zmm0, %zmm0, %k0
+; X86-NEXT:    vptestnmq %zmm1, %zmm1, %k1
+; X86-NEXT:    korb %k1, %k0, %k0
+; X86-NEXT:    vptestnmq %zmm2, %zmm2, %k1
+; X86-NEXT:    vptestnmq %zmm3, %zmm3, %k2
+; X86-NEXT:    korb %k2, %k1, %k1
+; X86-NEXT:    ktestb %k1, %k0
+; X86-NEXT:    je LBB72_1
+; X86-NEXT:  ## %bb.2: ## %exit
+; X86-NEXT:    addl $12, %esp
+; X86-NEXT:    vzeroupper
+; X86-NEXT:    retl
+; X86-NEXT:  LBB72_1: ## %bar
+; X86-NEXT:    vzeroupper
+; X86-NEXT:    calll _foo
+; X86-NEXT:    addl $12, %esp
+; X86-NEXT:    retl
+  %a = icmp eq <8 x i64> %w, zeroinitializer
+  %b = icmp eq <8 x i64> %x, zeroinitializer
+  %c = icmp eq <8 x i64> %y, zeroinitializer
+  %d = icmp eq <8 x i64> %z, zeroinitializer
+  %e = or <8 x i1> %a, %b
+  %f = or <8 x i1> %c, %d
+  %g = and <8 x i1> %e, %f
+  %h = bitcast <8 x i1> %g to i8
+  %i = icmp eq i8 %h, 0
+  br i1 %i, label %bar, label %exit
+
+bar:
+  call void @foo()
+  br label %exit
+
+exit:
+  ret void
+}
+
+define void @ktest_5(<16 x i32> %w, <16 x i32> %x, <16 x i32> %y, <16 x i32> %z) {
+; KNL-LABEL: ktest_5:
+; KNL:       ## %bb.0:
+; KNL-NEXT:    pushq %rax
+; KNL-NEXT:    .cfi_def_cfa_offset 16
+; KNL-NEXT:    vptestnmd %zmm0, %zmm0, %k0
+; KNL-NEXT:    vptestnmd %zmm1, %zmm1, %k1
+; KNL-NEXT:    korw %k1, %k0, %k0
+; KNL-NEXT:    vptestnmd %zmm2, %zmm2, %k1
+; KNL-NEXT:    vptestnmd %zmm3, %zmm3, %k2
+; KNL-NEXT:    korw %k2, %k1, %k1
+; KNL-NEXT:    kandw %k1, %k0, %k0
+; KNL-NEXT:    kortestw %k0, %k0
+; KNL-NEXT:    je LBB73_1
+; KNL-NEXT:  ## %bb.2: ## %exit
+; KNL-NEXT:    popq %rax
+; KNL-NEXT:    vzeroupper
+; KNL-NEXT:    retq
+; KNL-NEXT:  LBB73_1: ## %bar
+; KNL-NEXT:    vzeroupper
+; KNL-NEXT:    callq _foo
+; KNL-NEXT:    popq %rax
+; KNL-NEXT:    retq
+;
+; SKX-LABEL: ktest_5:
+; SKX:       ## %bb.0:
+; SKX-NEXT:    pushq %rax
+; SKX-NEXT:    .cfi_def_cfa_offset 16
+; SKX-NEXT:    vptestnmd %zmm0, %zmm0, %k0
+; SKX-NEXT:    vptestnmd %zmm1, %zmm1, %k1
+; SKX-NEXT:    korw %k1, %k0, %k0
+; SKX-NEXT:    vptestnmd %zmm2, %zmm2, %k1
+; SKX-NEXT:    vptestnmd %zmm3, %zmm3, %k2
+; SKX-NEXT:    korw %k2, %k1, %k1
+; SKX-NEXT:    ktestw %k1, %k0
+; SKX-NEXT:    je LBB73_1
+; SKX-NEXT:  ## %bb.2: ## %exit
+; SKX-NEXT:    popq %rax
+; SKX-NEXT:    vzeroupper
+; SKX-NEXT:    retq
+; SKX-NEXT:  LBB73_1: ## %bar
+; SKX-NEXT:    vzeroupper
+; SKX-NEXT:    callq _foo
+; SKX-NEXT:    popq %rax
+; SKX-NEXT:    retq
+;
+; AVX512BW-LABEL: ktest_5:
+; AVX512BW:       ## %bb.0:
+; AVX512BW-NEXT:    pushq %rax
+; AVX512BW-NEXT:    .cfi_def_cfa_offset 16
+; AVX512BW-NEXT:    vptestnmd %zmm0, %zmm0, %k0
+; AVX512BW-NEXT:    vptestnmd %zmm1, %zmm1, %k1
+; AVX512BW-NEXT:    korw %k1, %k0, %k0
+; AVX512BW-NEXT:    vptestnmd %zmm2, %zmm2, %k1
+; AVX512BW-NEXT:    vptestnmd %zmm3, %zmm3, %k2
+; AVX512BW-NEXT:    korw %k2, %k1, %k1
+; AVX512BW-NEXT:    kandw %k1, %k0, %k0
+; AVX512BW-NEXT:    kortestw %k0, %k0
+; AVX512BW-NEXT:    je LBB73_1
+; AVX512BW-NEXT:  ## %bb.2: ## %exit
+; AVX512BW-NEXT:    popq %rax
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+; AVX512BW-NEXT:  LBB73_1: ## %bar
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    callq _foo
+; AVX512BW-NEXT:    popq %rax
+; AVX512BW-NEXT:    retq
+;
+; AVX512DQ-LABEL: ktest_5:
+; AVX512DQ:       ## %bb.0:
+; AVX512DQ-NEXT:    pushq %rax
+; AVX512DQ-NEXT:    .cfi_def_cfa_offset 16
+; AVX512DQ-NEXT:    vptestnmd %zmm0, %zmm0, %k0
+; AVX512DQ-NEXT:    vptestnmd %zmm1, %zmm1, %k1
+; AVX512DQ-NEXT:    korw %k1, %k0, %k0
+; AVX512DQ-NEXT:    vptestnmd %zmm2, %zmm2, %k1
+; AVX512DQ-NEXT:    vptestnmd %zmm3, %zmm3, %k2
+; AVX512DQ-NEXT:    korw %k2, %k1, %k1
+; AVX512DQ-NEXT:    ktestw %k1, %k0
+; AVX512DQ-NEXT:    je LBB73_1
+; AVX512DQ-NEXT:  ## %bb.2: ## %exit
+; AVX512DQ-NEXT:    popq %rax
+; AVX512DQ-NEXT:    vzeroupper
+; AVX512DQ-NEXT:    retq
+; AVX512DQ-NEXT:  LBB73_1: ## %bar
+; AVX512DQ-NEXT:    vzeroupper
+; AVX512DQ-NEXT:    callq _foo
+; AVX512DQ-NEXT:    popq %rax
+; AVX512DQ-NEXT:    retq
+;
+; X86-LABEL: ktest_5:
+; X86:       ## %bb.0:
+; X86-NEXT:    subl $12, %esp
+; X86-NEXT:    .cfi_def_cfa_offset 16
+; X86-NEXT:    vptestnmd %zmm0, %zmm0, %k0
+; X86-NEXT:    vptestnmd %zmm1, %zmm1, %k1
+; X86-NEXT:    korw %k1, %k0, %k0
+; X86-NEXT:    vptestnmd %zmm2, %zmm2, %k1
+; X86-NEXT:    vptestnmd %zmm3, %zmm3, %k2
+; X86-NEXT:    korw %k2, %k1, %k1
+; X86-NEXT:    ktestw %k1, %k0
+; X86-NEXT:    je LBB73_1
+; X86-NEXT:  ## %bb.2: ## %exit
+; X86-NEXT:    addl $12, %esp
+; X86-NEXT:    vzeroupper
+; X86-NEXT:    retl
+; X86-NEXT:  LBB73_1: ## %bar
+; X86-NEXT:    vzeroupper
+; X86-NEXT:    calll _foo
+; X86-NEXT:    addl $12, %esp
+; X86-NEXT:    retl
+  %a = icmp eq <16 x i32> %w, zeroinitializer
+  %b = icmp eq <16 x i32> %x, zeroinitializer
+  %c = icmp eq <16 x i32> %y, zeroinitializer
+  %d = icmp eq <16 x i32> %z, zeroinitializer
+  %e = or <16 x i1> %a, %b
+  %f = or <16 x i1> %c, %d
+  %g = and <16 x i1> %e, %f
+  %h = bitcast <16 x i1> %g to i16
+  %i = icmp eq i16 %h, 0
+  br i1 %i, label %bar, label %exit
+
+bar:
+  call void @foo()
+  br label %exit
+
+exit:
+  ret void
+}
+
+define void @ktest_6(<32 x i16> %w, <32 x i16> %x, <32 x i16> %y, <32 x i16> %z) {
+; KNL-LABEL: ktest_6:
+; KNL:       ## %bb.0:
+; KNL-NEXT:    pushq %rax
+; KNL-NEXT:    .cfi_def_cfa_offset 16
+; KNL-NEXT:    vpxor %xmm8, %xmm8, %xmm8
+; KNL-NEXT:    vpcmpeqw %ymm8, %ymm0, %ymm0
+; KNL-NEXT:    vpcmpeqw %ymm8, %ymm1, %ymm1
+; KNL-NEXT:    vpcmpeqw %ymm8, %ymm2, %ymm2
+; KNL-NEXT:    vpor %ymm2, %ymm0, %ymm0
+; KNL-NEXT:    vpcmpeqw %ymm8, %ymm3, %ymm2
+; KNL-NEXT:    vpor %ymm2, %ymm1, %ymm1
+; KNL-NEXT:    vpcmpeqw %ymm8, %ymm4, %ymm2
+; KNL-NEXT:    vpcmpeqw %ymm8, %ymm5, %ymm3
+; KNL-NEXT:    vpcmpeqw %ymm8, %ymm6, %ymm4
+; KNL-NEXT:    vpor %ymm4, %ymm2, %ymm2
+; KNL-NEXT:    vpand %ymm2, %ymm0, %ymm0
+; KNL-NEXT:    vpcmpeqw %ymm8, %ymm7, %ymm2
+; KNL-NEXT:    vpor %ymm2, %ymm3, %ymm2
+; KNL-NEXT:    vpand %ymm2, %ymm1, %ymm1
+; KNL-NEXT:    vpmovsxwd %ymm0, %zmm0
+; KNL-NEXT:    vptestmd %zmm0, %zmm0, %k0
+; KNL-NEXT:    kmovw %k0, %eax
+; KNL-NEXT:    vpmovsxwd %ymm1, %zmm0
+; KNL-NEXT:    vptestmd %zmm0, %zmm0, %k0
+; KNL-NEXT:    kmovw %k0, %ecx
+; KNL-NEXT:    shll $16, %ecx
+; KNL-NEXT:    orl %eax, %ecx
+; KNL-NEXT:    je LBB74_1
+; KNL-NEXT:  ## %bb.2: ## %exit
+; KNL-NEXT:    popq %rax
+; KNL-NEXT:    vzeroupper
+; KNL-NEXT:    retq
+; KNL-NEXT:  LBB74_1: ## %bar
+; KNL-NEXT:    vzeroupper
+; KNL-NEXT:    callq _foo
+; KNL-NEXT:    popq %rax
+; KNL-NEXT:    retq
+;
+; SKX-LABEL: ktest_6:
+; SKX:       ## %bb.0:
+; SKX-NEXT:    pushq %rax
+; SKX-NEXT:    .cfi_def_cfa_offset 16
+; SKX-NEXT:    vptestnmw %zmm0, %zmm0, %k0
+; SKX-NEXT:    vptestnmw %zmm1, %zmm1, %k1
+; SKX-NEXT:    kord %k1, %k0, %k0
+; SKX-NEXT:    vptestnmw %zmm2, %zmm2, %k1
+; SKX-NEXT:    vptestnmw %zmm3, %zmm3, %k2
+; SKX-NEXT:    kord %k2, %k1, %k1
+; SKX-NEXT:    ktestd %k1, %k0
+; SKX-NEXT:    je LBB74_1
+; SKX-NEXT:  ## %bb.2: ## %exit
+; SKX-NEXT:    popq %rax
+; SKX-NEXT:    vzeroupper
+; SKX-NEXT:    retq
+; SKX-NEXT:  LBB74_1: ## %bar
+; SKX-NEXT:    vzeroupper
+; SKX-NEXT:    callq _foo
+; SKX-NEXT:    popq %rax
+; SKX-NEXT:    retq
+;
+; AVX512BW-LABEL: ktest_6:
+; AVX512BW:       ## %bb.0:
+; AVX512BW-NEXT:    pushq %rax
+; AVX512BW-NEXT:    .cfi_def_cfa_offset 16
+; AVX512BW-NEXT:    vptestnmw %zmm0, %zmm0, %k0
+; AVX512BW-NEXT:    vptestnmw %zmm1, %zmm1, %k1
+; AVX512BW-NEXT:    kord %k1, %k0, %k0
+; AVX512BW-NEXT:    vptestnmw %zmm2, %zmm2, %k1
+; AVX512BW-NEXT:    vptestnmw %zmm3, %zmm3, %k2
+; AVX512BW-NEXT:    kord %k2, %k1, %k1
+; AVX512BW-NEXT:    ktestd %k1, %k0
+; AVX512BW-NEXT:    je LBB74_1
+; AVX512BW-NEXT:  ## %bb.2: ## %exit
+; AVX512BW-NEXT:    popq %rax
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+; AVX512BW-NEXT:  LBB74_1: ## %bar
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    callq _foo
+; AVX512BW-NEXT:    popq %rax
+; AVX512BW-NEXT:    retq
+;
+; AVX512DQ-LABEL: ktest_6:
+; AVX512DQ:       ## %bb.0:
+; AVX512DQ-NEXT:    pushq %rax
+; AVX512DQ-NEXT:    .cfi_def_cfa_offset 16
+; AVX512DQ-NEXT:    vpxor %xmm8, %xmm8, %xmm8
+; AVX512DQ-NEXT:    vpcmpeqw %ymm8, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vpcmpeqw %ymm8, %ymm1, %ymm1
+; AVX512DQ-NEXT:    vpcmpeqw %ymm8, %ymm2, %ymm2
+; AVX512DQ-NEXT:    vpor %ymm2, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vpcmpeqw %ymm8, %ymm3, %ymm2
+; AVX512DQ-NEXT:    vpor %ymm2, %ymm1, %ymm1
+; AVX512DQ-NEXT:    vpcmpeqw %ymm8, %ymm4, %ymm2
+; AVX512DQ-NEXT:    vpcmpeqw %ymm8, %ymm5, %ymm3
+; AVX512DQ-NEXT:    vpcmpeqw %ymm8, %ymm6, %ymm4
+; AVX512DQ-NEXT:    vpor %ymm4, %ymm2, %ymm2
+; AVX512DQ-NEXT:    vpand %ymm2, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vpcmpeqw %ymm8, %ymm7, %ymm2
+; AVX512DQ-NEXT:    vpor %ymm2, %ymm3, %ymm2
+; AVX512DQ-NEXT:    vpand %ymm2, %ymm1, %ymm1
+; AVX512DQ-NEXT:    vpmovsxwd %ymm0, %zmm0
+; AVX512DQ-NEXT:    vpmovd2m %zmm0, %k0
+; AVX512DQ-NEXT:    kmovw %k0, %eax
+; AVX512DQ-NEXT:    vpmovsxwd %ymm1, %zmm0
+; AVX512DQ-NEXT:    vpmovd2m %zmm0, %k0
+; AVX512DQ-NEXT:    kmovw %k0, %ecx
+; AVX512DQ-NEXT:    shll $16, %ecx
+; AVX512DQ-NEXT:    orl %eax, %ecx
+; AVX512DQ-NEXT:    je LBB74_1
+; AVX512DQ-NEXT:  ## %bb.2: ## %exit
+; AVX512DQ-NEXT:    popq %rax
+; AVX512DQ-NEXT:    vzeroupper
+; AVX512DQ-NEXT:    retq
+; AVX512DQ-NEXT:  LBB74_1: ## %bar
+; AVX512DQ-NEXT:    vzeroupper
+; AVX512DQ-NEXT:    callq _foo
+; AVX512DQ-NEXT:    popq %rax
+; AVX512DQ-NEXT:    retq
+;
+; X86-LABEL: ktest_6:
+; X86:       ## %bb.0:
+; X86-NEXT:    subl $12, %esp
+; X86-NEXT:    .cfi_def_cfa_offset 16
+; X86-NEXT:    vptestnmw %zmm0, %zmm0, %k0
+; X86-NEXT:    vptestnmw %zmm1, %zmm1, %k1
+; X86-NEXT:    kord %k1, %k0, %k0
+; X86-NEXT:    vptestnmw %zmm2, %zmm2, %k1
+; X86-NEXT:    vptestnmw %zmm3, %zmm3, %k2
+; X86-NEXT:    kord %k2, %k1, %k1
+; X86-NEXT:    ktestd %k1, %k0
+; X86-NEXT:    je LBB74_1
+; X86-NEXT:  ## %bb.2: ## %exit
+; X86-NEXT:    addl $12, %esp
+; X86-NEXT:    vzeroupper
+; X86-NEXT:    retl
+; X86-NEXT:  LBB74_1: ## %bar
+; X86-NEXT:    vzeroupper
+; X86-NEXT:    calll _foo
+; X86-NEXT:    addl $12, %esp
+; X86-NEXT:    retl
+  %a = icmp eq <32 x i16> %w, zeroinitializer
+  %b = icmp eq <32 x i16> %x, zeroinitializer
+  %c = icmp eq <32 x i16> %y, zeroinitializer
+  %d = icmp eq <32 x i16> %z, zeroinitializer
+  %e = or <32 x i1> %a, %b
+  %f = or <32 x i1> %c, %d
+  %g = and <32 x i1> %e, %f
+  %h = bitcast <32 x i1> %g to i32
+  %i = icmp eq i32 %h, 0
+  br i1 %i, label %bar, label %exit
+
+bar:
+  call void @foo()
+  br label %exit
+
+exit:
+  ret void
+}
+
+define void @ktest_7(<64 x i8> %w, <64 x i8> %x, <64 x i8> %y, <64 x i8> %z) {
+; KNL-LABEL: ktest_7:
+; KNL:       ## %bb.0:
+; KNL-NEXT:    pushq %rax
+; KNL-NEXT:    .cfi_def_cfa_offset 16
+; KNL-NEXT:    vpxor %xmm8, %xmm8, %xmm8
+; KNL-NEXT:    vpcmpeqb %ymm8, %ymm0, %ymm9
+; KNL-NEXT:    vextracti128 $1, %ymm9, %xmm0
+; KNL-NEXT:    vpcmpeqb %ymm8, %ymm1, %ymm10
+; KNL-NEXT:    vextracti128 $1, %ymm10, %xmm1
+; KNL-NEXT:    vpcmpeqb %ymm8, %ymm2, %ymm11
+; KNL-NEXT:    vextracti128 $1, %ymm11, %xmm2
+; KNL-NEXT:    vpor %xmm2, %xmm0, %xmm13
+; KNL-NEXT:    vpcmpeqb %ymm8, %ymm3, %ymm2
+; KNL-NEXT:    vextracti128 $1, %ymm2, %xmm3
+; KNL-NEXT:    vpor %xmm3, %xmm1, %xmm12
+; KNL-NEXT:    vpcmpeqb %ymm8, %ymm4, %ymm3
+; KNL-NEXT:    vextracti128 $1, %ymm3, %xmm4
+; KNL-NEXT:    vpcmpeqb %ymm8, %ymm5, %ymm5
+; KNL-NEXT:    vextracti128 $1, %ymm5, %xmm1
+; KNL-NEXT:    vpcmpeqb %ymm8, %ymm6, %ymm6
+; KNL-NEXT:    vextracti128 $1, %ymm6, %xmm0
+; KNL-NEXT:    vpor %xmm0, %xmm4, %xmm0
+; KNL-NEXT:    vpand %xmm0, %xmm13, %xmm0
+; KNL-NEXT:    vpcmpeqb %ymm8, %ymm7, %ymm4
+; KNL-NEXT:    vextracti128 $1, %ymm4, %xmm7
+; KNL-NEXT:    vpor %xmm7, %xmm1, %xmm1
+; KNL-NEXT:    vpand %xmm1, %xmm12, %xmm1
+; KNL-NEXT:    vpor %xmm2, %xmm10, %xmm2
+; KNL-NEXT:    vpor %xmm11, %xmm9, %xmm7
+; KNL-NEXT:    vpor %xmm4, %xmm5, %xmm4
+; KNL-NEXT:    vpand %xmm4, %xmm2, %xmm2
+; KNL-NEXT:    vpor %xmm6, %xmm3, %xmm3
+; KNL-NEXT:    vpand %xmm3, %xmm7, %xmm3
+; KNL-NEXT:    vpmovsxbd %xmm3, %zmm3
+; KNL-NEXT:    vptestmd %zmm3, %zmm3, %k0
+; KNL-NEXT:    kmovw %k0, %eax
+; KNL-NEXT:    vpmovsxbd %xmm0, %zmm0
+; KNL-NEXT:    vptestmd %zmm0, %zmm0, %k0
+; KNL-NEXT:    kmovw %k0, %ecx
+; KNL-NEXT:    shll $16, %ecx
+; KNL-NEXT:    orl %eax, %ecx
+; KNL-NEXT:    vpmovsxbd %xmm2, %zmm0
+; KNL-NEXT:    vptestmd %zmm0, %zmm0, %k0
+; KNL-NEXT:    kmovw %k0, %eax
+; KNL-NEXT:    vpmovsxbd %xmm1, %zmm0
+; KNL-NEXT:    vptestmd %zmm0, %zmm0, %k0
+; KNL-NEXT:    kmovw %k0, %edx
+; KNL-NEXT:    shll $16, %edx
+; KNL-NEXT:    orl %eax, %edx
+; KNL-NEXT:    shlq $32, %rdx
+; KNL-NEXT:    orq %rcx, %rdx
+; KNL-NEXT:    je LBB75_1
+; KNL-NEXT:  ## %bb.2: ## %exit
+; KNL-NEXT:    popq %rax
+; KNL-NEXT:    vzeroupper
+; KNL-NEXT:    retq
+; KNL-NEXT:  LBB75_1: ## %bar
+; KNL-NEXT:    vzeroupper
+; KNL-NEXT:    callq _foo
+; KNL-NEXT:    popq %rax
+; KNL-NEXT:    retq
+;
+; SKX-LABEL: ktest_7:
+; SKX:       ## %bb.0:
+; SKX-NEXT:    pushq %rax
+; SKX-NEXT:    .cfi_def_cfa_offset 16
+; SKX-NEXT:    vptestnmb %zmm0, %zmm0, %k0
+; SKX-NEXT:    vptestnmb %zmm1, %zmm1, %k1
+; SKX-NEXT:    korq %k1, %k0, %k0
+; SKX-NEXT:    vptestnmb %zmm2, %zmm2, %k1
+; SKX-NEXT:    vptestnmb %zmm3, %zmm3, %k2
+; SKX-NEXT:    korq %k2, %k1, %k1
+; SKX-NEXT:    ktestq %k1, %k0
+; SKX-NEXT:    je LBB75_1
+; SKX-NEXT:  ## %bb.2: ## %exit
+; SKX-NEXT:    popq %rax
+; SKX-NEXT:    vzeroupper
+; SKX-NEXT:    retq
+; SKX-NEXT:  LBB75_1: ## %bar
+; SKX-NEXT:    vzeroupper
+; SKX-NEXT:    callq _foo
+; SKX-NEXT:    popq %rax
+; SKX-NEXT:    retq
+;
+; AVX512BW-LABEL: ktest_7:
+; AVX512BW:       ## %bb.0:
+; AVX512BW-NEXT:    pushq %rax
+; AVX512BW-NEXT:    .cfi_def_cfa_offset 16
+; AVX512BW-NEXT:    vptestnmb %zmm0, %zmm0, %k0
+; AVX512BW-NEXT:    vptestnmb %zmm1, %zmm1, %k1
+; AVX512BW-NEXT:    korq %k1, %k0, %k0
+; AVX512BW-NEXT:    vptestnmb %zmm2, %zmm2, %k1
+; AVX512BW-NEXT:    vptestnmb %zmm3, %zmm3, %k2
+; AVX512BW-NEXT:    korq %k2, %k1, %k1
+; AVX512BW-NEXT:    ktestq %k1, %k0
+; AVX512BW-NEXT:    je LBB75_1
+; AVX512BW-NEXT:  ## %bb.2: ## %exit
+; AVX512BW-NEXT:    popq %rax
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+; AVX512BW-NEXT:  LBB75_1: ## %bar
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    callq _foo
+; AVX512BW-NEXT:    popq %rax
+; AVX512BW-NEXT:    retq
+;
+; AVX512DQ-LABEL: ktest_7:
+; AVX512DQ:       ## %bb.0:
+; AVX512DQ-NEXT:    pushq %rax
+; AVX512DQ-NEXT:    .cfi_def_cfa_offset 16
+; AVX512DQ-NEXT:    vpxor %xmm8, %xmm8, %xmm8
+; AVX512DQ-NEXT:    vpcmpeqb %ymm8, %ymm0, %ymm9
+; AVX512DQ-NEXT:    vextracti128 $1, %ymm9, %xmm0
+; AVX512DQ-NEXT:    vpcmpeqb %ymm8, %ymm1, %ymm10
+; AVX512DQ-NEXT:    vextracti128 $1, %ymm10, %xmm1
+; AVX512DQ-NEXT:    vpcmpeqb %ymm8, %ymm2, %ymm11
+; AVX512DQ-NEXT:    vextracti128 $1, %ymm11, %xmm2
+; AVX512DQ-NEXT:    vpor %xmm2, %xmm0, %xmm13
+; AVX512DQ-NEXT:    vpcmpeqb %ymm8, %ymm3, %ymm2
+; AVX512DQ-NEXT:    vextracti128 $1, %ymm2, %xmm3
+; AVX512DQ-NEXT:    vpor %xmm3, %xmm1, %xmm12
+; AVX512DQ-NEXT:    vpcmpeqb %ymm8, %ymm4, %ymm3
+; AVX512DQ-NEXT:    vextracti128 $1, %ymm3, %xmm4
+; AVX512DQ-NEXT:    vpcmpeqb %ymm8, %ymm5, %ymm5
+; AVX512DQ-NEXT:    vextracti128 $1, %ymm5, %xmm1
+; AVX512DQ-NEXT:    vpcmpeqb %ymm8, %ymm6, %ymm6
+; AVX512DQ-NEXT:    vextracti128 $1, %ymm6, %xmm0
+; AVX512DQ-NEXT:    vpor %xmm0, %xmm4, %xmm0
+; AVX512DQ-NEXT:    vpand %xmm0, %xmm13, %xmm0
+; AVX512DQ-NEXT:    vpcmpeqb %ymm8, %ymm7, %ymm4
+; AVX512DQ-NEXT:    vextracti128 $1, %ymm4, %xmm7
+; AVX512DQ-NEXT:    vpor %xmm7, %xmm1, %xmm1
+; AVX512DQ-NEXT:    vpand %xmm1, %xmm12, %xmm1
+; AVX512DQ-NEXT:    vpor %xmm2, %xmm10, %xmm2
+; AVX512DQ-NEXT:    vpor %xmm11, %xmm9, %xmm7
+; AVX512DQ-NEXT:    vpor %xmm4, %xmm5, %xmm4
+; AVX512DQ-NEXT:    vpand %xmm4, %xmm2, %xmm2
+; AVX512DQ-NEXT:    vpor %xmm6, %xmm3, %xmm3
+; AVX512DQ-NEXT:    vpand %xmm3, %xmm7, %xmm3
+; AVX512DQ-NEXT:    vpmovsxbd %xmm3, %zmm3
+; AVX512DQ-NEXT:    vpmovd2m %zmm3, %k0
+; AVX512DQ-NEXT:    kmovw %k0, %eax
+; AVX512DQ-NEXT:    vpmovsxbd %xmm0, %zmm0
+; AVX512DQ-NEXT:    vpmovd2m %zmm0, %k0
+; AVX512DQ-NEXT:    kmovw %k0, %ecx
+; AVX512DQ-NEXT:    shll $16, %ecx
+; AVX512DQ-NEXT:    orl %eax, %ecx
+; AVX512DQ-NEXT:    vpmovsxbd %xmm2, %zmm0
+; AVX512DQ-NEXT:    vpmovd2m %zmm0, %k0
+; AVX512DQ-NEXT:    kmovw %k0, %eax
+; AVX512DQ-NEXT:    vpmovsxbd %xmm1, %zmm0
+; AVX512DQ-NEXT:    vpmovd2m %zmm0, %k0
+; AVX512DQ-NEXT:    kmovw %k0, %edx
+; AVX512DQ-NEXT:    shll $16, %edx
+; AVX512DQ-NEXT:    orl %eax, %edx
+; AVX512DQ-NEXT:    shlq $32, %rdx
+; AVX512DQ-NEXT:    orq %rcx, %rdx
+; AVX512DQ-NEXT:    je LBB75_1
+; AVX512DQ-NEXT:  ## %bb.2: ## %exit
+; AVX512DQ-NEXT:    popq %rax
+; AVX512DQ-NEXT:    vzeroupper
+; AVX512DQ-NEXT:    retq
+; AVX512DQ-NEXT:  LBB75_1: ## %bar
+; AVX512DQ-NEXT:    vzeroupper
+; AVX512DQ-NEXT:    callq _foo
+; AVX512DQ-NEXT:    popq %rax
+; AVX512DQ-NEXT:    retq
+;
+; X86-LABEL: ktest_7:
+; X86:       ## %bb.0:
+; X86-NEXT:    subl $12, %esp
+; X86-NEXT:    .cfi_def_cfa_offset 16
+; X86-NEXT:    vptestnmb %zmm0, %zmm0, %k0
+; X86-NEXT:    vptestnmb %zmm1, %zmm1, %k1
+; X86-NEXT:    korq %k1, %k0, %k0
+; X86-NEXT:    vptestnmb %zmm2, %zmm2, %k1
+; X86-NEXT:    vptestnmb %zmm3, %zmm3, %k2
+; X86-NEXT:    korq %k2, %k1, %k1
+; X86-NEXT:    kandq %k1, %k0, %k0
+; X86-NEXT:    kshiftrq $32, %k0, %k1
+; X86-NEXT:    kortestd %k1, %k0
+; X86-NEXT:    je LBB75_1
+; X86-NEXT:  ## %bb.2: ## %exit
+; X86-NEXT:    addl $12, %esp
+; X86-NEXT:    vzeroupper
+; X86-NEXT:    retl
+; X86-NEXT:  LBB75_1: ## %bar
+; X86-NEXT:    vzeroupper
+; X86-NEXT:    calll _foo
+; X86-NEXT:    addl $12, %esp
+; X86-NEXT:    retl
+  %a = icmp eq <64 x i8> %w, zeroinitializer
+  %b = icmp eq <64 x i8> %x, zeroinitializer
+  %c = icmp eq <64 x i8> %y, zeroinitializer
+  %d = icmp eq <64 x i8> %z, zeroinitializer
+  %e = or <64 x i1> %a, %b
+  %f = or <64 x i1> %c, %d
+  %g = and <64 x i1> %e, %f
+  %h = bitcast <64 x i1> %g to i64
+  %i = icmp eq i64 %h, 0
+  br i1 %i, label %bar, label %exit
+
+bar:
+  call void @foo()
+  br label %exit
+
+exit:
+  ret void
+}
diff --git a/test/CodeGen/X86/avx512-regcall-NoMask.ll b/test/CodeGen/X86/avx512-regcall-NoMask.ll
index 1136a3a50693e8b6a9f695cca549f8d0a91828ec..985860166a3f23611c1d6afa34f0fb1ba333441d 100644
--- a/test/CodeGen/X86/avx512-regcall-NoMask.ll
+++ b/test/CodeGen/X86/avx512-regcall-NoMask.ll
@@ -508,17 +508,17 @@ define x86_regcallcc double @test_CallargRetDouble(double %a)  {
 define x86_regcallcc x86_fp80 @test_argRetf80(x86_fp80 %a0) nounwind {
 ; X32-LABEL: test_argRetf80:
 ; X32:       # %bb.0:
-; X32-NEXT:    fadd %st(0), %st(0)
+; X32-NEXT:    fadd %st, %st(0)
 ; X32-NEXT:    retl
 ;
 ; WIN64-LABEL: test_argRetf80:
 ; WIN64:       # %bb.0:
-; WIN64-NEXT:    fadd %st(0), %st(0)
+; WIN64-NEXT:    fadd %st, %st(0)
 ; WIN64-NEXT:    retq
 ;
 ; LINUXOSX64-LABEL: test_argRetf80:
 ; LINUXOSX64:       # %bb.0:
-; LINUXOSX64-NEXT:    fadd %st(0), %st(0)
+; LINUXOSX64-NEXT:    fadd %st, %st(0)
 ; LINUXOSX64-NEXT:    retq
   %r0 = fadd x86_fp80 %a0, %a0
   ret x86_fp80 %r0
@@ -529,9 +529,9 @@ define x86_regcallcc x86_fp80 @test_CallargRetf80(x86_fp80 %a)  {
 ; X32-LABEL: test_CallargRetf80:
 ; X32:       # %bb.0:
 ; X32-NEXT:    pushl %esp
-; X32-NEXT:    fadd %st(0), %st(0)
+; X32-NEXT:    fadd %st, %st(0)
 ; X32-NEXT:    calll _test_argRetf80
-; X32-NEXT:    fadd %st(0), %st(0)
+; X32-NEXT:    fadd %st, %st(0)
 ; X32-NEXT:    popl %esp
 ; X32-NEXT:    retl
 ;
@@ -540,9 +540,9 @@ define x86_regcallcc x86_fp80 @test_CallargRetf80(x86_fp80 %a)  {
 ; WIN64-NEXT:    pushq %rsp
 ; WIN64-NEXT:    .seh_pushreg 4
 ; WIN64-NEXT:    .seh_endprologue
-; WIN64-NEXT:    fadd %st(0), %st(0)
+; WIN64-NEXT:    fadd %st, %st(0)
 ; WIN64-NEXT:    callq test_argRetf80
-; WIN64-NEXT:    fadd %st(0), %st(0)
+; WIN64-NEXT:    fadd %st, %st(0)
 ; WIN64-NEXT:    popq %rsp
 ; WIN64-NEXT:    retq
 ; WIN64-NEXT:    .seh_handlerdata
@@ -554,9 +554,9 @@ define x86_regcallcc x86_fp80 @test_CallargRetf80(x86_fp80 %a)  {
 ; LINUXOSX64-NEXT:    pushq %rsp
 ; LINUXOSX64-NEXT:    .cfi_def_cfa_offset 16
 ; LINUXOSX64-NEXT:    .cfi_offset %rsp, -16
-; LINUXOSX64-NEXT:    fadd %st(0), %st(0)
+; LINUXOSX64-NEXT:    fadd %st, %st(0)
 ; LINUXOSX64-NEXT:    callq test_argRetf80
-; LINUXOSX64-NEXT:    fadd %st(0), %st(0)
+; LINUXOSX64-NEXT:    fadd %st, %st(0)
 ; LINUXOSX64-NEXT:    popq %rsp
 ; LINUXOSX64-NEXT:    .cfi_def_cfa_offset 8
 ; LINUXOSX64-NEXT:    retq
diff --git a/test/CodeGen/X86/avx512-schedule.ll b/test/CodeGen/X86/avx512-schedule.ll
deleted file mode 100755
index 5c44d8679b99c19b3a1f0a3225451ed149c6f69d..0000000000000000000000000000000000000000
--- a/test/CodeGen/X86/avx512-schedule.ll
+++ /dev/null
@@ -1,8764 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=x86-64 -mattr=+avx512f,+avx512dq,+avx512bw,+avx512vl | FileCheck %s --check-prefix=GENERIC
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=skx | FileCheck %s --check-prefix=SKX
-
-; This test is an assembly of avx512 instructions to check their scheduling
-
-define <8 x double> @addpd512(<8 x double> %y, <8 x double> %x) {
-; GENERIC-LABEL: addpd512:
-; GENERIC:       # %bb.0: # %entry
-; GENERIC-NEXT:    vaddpd %zmm0, %zmm1, %zmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: addpd512:
-; SKX:       # %bb.0: # %entry
-; SKX-NEXT:    vaddpd %zmm0, %zmm1, %zmm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-entry:
-  %add.i = fadd <8 x double> %x, %y
-  ret <8 x double> %add.i
-}
-
-define <8 x double> @addpd512fold(<8 x double> %y) {
-; GENERIC-LABEL: addpd512fold:
-; GENERIC:       # %bb.0: # %entry
-; GENERIC-NEXT:    vaddpd {{.*}}(%rip), %zmm0, %zmm0 # sched: [10:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: addpd512fold:
-; SKX:       # %bb.0: # %entry
-; SKX-NEXT:    vaddpd {{.*}}(%rip), %zmm0, %zmm0 # sched: [11:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-entry:
-  %add.i = fadd <8 x double> %y, <double 4.500000e+00, double 3.400000e+00, double 2.300000e+00, double 1.200000e+00, double 4.500000e+00, double 3.800000e+00, double 2.300000e+00, double 1.200000e+00>
-  ret <8 x double> %add.i
-}
-
-define <16 x float> @addps512(<16 x float> %y, <16 x float> %x) {
-; GENERIC-LABEL: addps512:
-; GENERIC:       # %bb.0: # %entry
-; GENERIC-NEXT:    vaddps %zmm0, %zmm1, %zmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: addps512:
-; SKX:       # %bb.0: # %entry
-; SKX-NEXT:    vaddps %zmm0, %zmm1, %zmm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-entry:
-  %add.i = fadd <16 x float> %x, %y
-  ret <16 x float> %add.i
-}
-
-define <16 x float> @addps512fold(<16 x float> %y) {
-; GENERIC-LABEL: addps512fold:
-; GENERIC:       # %bb.0: # %entry
-; GENERIC-NEXT:    vaddps {{.*}}(%rip), %zmm0, %zmm0 # sched: [10:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: addps512fold:
-; SKX:       # %bb.0: # %entry
-; SKX-NEXT:    vaddps {{.*}}(%rip), %zmm0, %zmm0 # sched: [11:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-entry:
-  %add.i = fadd <16 x float> %y, <float 4.500000e+00, float 0x400B333340000000, float 0x4002666660000000, float 0x3FF3333340000000, float 4.500000e+00, float 0x400B333340000000, float 0x4002666660000000, float 0x3FF3333340000000, float 4.500000e+00, float 0x400B333340000000, float 0x4002666660000000, float 4.500000e+00, float 4.500000e+00, float 0x400B333340000000,  float 0x4002666660000000, float 0x3FF3333340000000>
-  ret <16 x float> %add.i
-}
-
-define <8 x double> @subpd512(<8 x double> %y, <8 x double> %x) {
-; GENERIC-LABEL: subpd512:
-; GENERIC:       # %bb.0: # %entry
-; GENERIC-NEXT:    vsubpd %zmm0, %zmm1, %zmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: subpd512:
-; SKX:       # %bb.0: # %entry
-; SKX-NEXT:    vsubpd %zmm0, %zmm1, %zmm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-entry:
-  %sub.i = fsub <8 x double> %x, %y
-  ret <8 x double> %sub.i
-}
-
-define <8 x double> @subpd512fold(<8 x double> %y, <8 x double>* %x) {
-; GENERIC-LABEL: subpd512fold:
-; GENERIC:       # %bb.0: # %entry
-; GENERIC-NEXT:    vsubpd (%rdi), %zmm0, %zmm0 # sched: [10:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: subpd512fold:
-; SKX:       # %bb.0: # %entry
-; SKX-NEXT:    vsubpd (%rdi), %zmm0, %zmm0 # sched: [11:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-entry:
-  %tmp2 = load <8 x double>, <8 x double>* %x, align 8
-  %sub.i = fsub <8 x double> %y, %tmp2
-  ret <8 x double> %sub.i
-}
-
-define <16 x float> @subps512(<16 x float> %y, <16 x float> %x) {
-; GENERIC-LABEL: subps512:
-; GENERIC:       # %bb.0: # %entry
-; GENERIC-NEXT:    vsubps %zmm0, %zmm1, %zmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: subps512:
-; SKX:       # %bb.0: # %entry
-; SKX-NEXT:    vsubps %zmm0, %zmm1, %zmm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-entry:
-  %sub.i = fsub <16 x float> %x, %y
-  ret <16 x float> %sub.i
-}
-
-define <16 x float> @subps512fold(<16 x float> %y, <16 x float>* %x) {
-; GENERIC-LABEL: subps512fold:
-; GENERIC:       # %bb.0: # %entry
-; GENERIC-NEXT:    vsubps (%rdi), %zmm0, %zmm0 # sched: [10:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: subps512fold:
-; SKX:       # %bb.0: # %entry
-; SKX-NEXT:    vsubps (%rdi), %zmm0, %zmm0 # sched: [11:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-entry:
-  %tmp2 = load <16 x float>, <16 x float>* %x, align 4
-  %sub.i = fsub <16 x float> %y, %tmp2
-  ret <16 x float> %sub.i
-}
-
-define <8 x i64> @imulq512(<8 x i64> %y, <8 x i64> %x) {
-; GENERIC-LABEL: imulq512:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpmullq %zmm0, %zmm1, %zmm0 # sched: [5:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: imulq512:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpmullq %zmm0, %zmm1, %zmm0 # sched: [12:1.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %z = mul <8 x i64>%x, %y
-  ret <8 x i64>%z
-}
-
-define <4 x i64> @imulq256(<4 x i64> %y, <4 x i64> %x) {
-; GENERIC-LABEL: imulq256:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpmullq %ymm0, %ymm1, %ymm0 # sched: [5:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: imulq256:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpmullq %ymm0, %ymm1, %ymm0 # sched: [12:1.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %z = mul <4 x i64>%x, %y
-  ret <4 x i64>%z
-}
-
-define <2 x i64> @imulq128(<2 x i64> %y, <2 x i64> %x) {
-; GENERIC-LABEL: imulq128:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpmullq %xmm0, %xmm1, %xmm0 # sched: [5:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: imulq128:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpmullq %xmm0, %xmm1, %xmm0 # sched: [12:1.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %z = mul <2 x i64>%x, %y
-  ret <2 x i64>%z
-}
-
-define <8 x double> @mulpd512(<8 x double> %y, <8 x double> %x) {
-; GENERIC-LABEL: mulpd512:
-; GENERIC:       # %bb.0: # %entry
-; GENERIC-NEXT:    vmulpd %zmm0, %zmm1, %zmm0 # sched: [5:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: mulpd512:
-; SKX:       # %bb.0: # %entry
-; SKX-NEXT:    vmulpd %zmm0, %zmm1, %zmm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-entry:
-  %mul.i = fmul <8 x double> %x, %y
-  ret <8 x double> %mul.i
-}
-
-define <8 x double> @mulpd512fold(<8 x double> %y) {
-; GENERIC-LABEL: mulpd512fold:
-; GENERIC:       # %bb.0: # %entry
-; GENERIC-NEXT:    vmulpd {{.*}}(%rip), %zmm0, %zmm0 # sched: [12:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: mulpd512fold:
-; SKX:       # %bb.0: # %entry
-; SKX-NEXT:    vmulpd {{.*}}(%rip), %zmm0, %zmm0 # sched: [11:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-entry:
-  %mul.i = fmul <8 x double> %y, <double 4.500000e+00, double 3.400000e+00, double 2.300000e+00, double 1.200000e+00, double 4.500000e+00, double 3.400000e+00, double 2.300000e+00, double 1.200000e+00>
-  ret <8 x double> %mul.i
-}
-
-define <16 x float> @mulps512(<16 x float> %y, <16 x float> %x) {
-; GENERIC-LABEL: mulps512:
-; GENERIC:       # %bb.0: # %entry
-; GENERIC-NEXT:    vmulps %zmm0, %zmm1, %zmm0 # sched: [5:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: mulps512:
-; SKX:       # %bb.0: # %entry
-; SKX-NEXT:    vmulps %zmm0, %zmm1, %zmm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-entry:
-  %mul.i = fmul <16 x float> %x, %y
-  ret <16 x float> %mul.i
-}
-
-define <16 x float> @mulps512fold(<16 x float> %y) {
-; GENERIC-LABEL: mulps512fold:
-; GENERIC:       # %bb.0: # %entry
-; GENERIC-NEXT:    vmulps {{.*}}(%rip), %zmm0, %zmm0 # sched: [12:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: mulps512fold:
-; SKX:       # %bb.0: # %entry
-; SKX-NEXT:    vmulps {{.*}}(%rip), %zmm0, %zmm0 # sched: [11:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-entry:
-  %mul.i = fmul <16 x float> %y, <float 4.500000e+00, float 0x400B333340000000, float 0x4002666660000000, float 0x3FF3333340000000, float 4.500000e+00, float 0x400B333340000000, float 0x4002666660000000, float 0x3FF3333340000000, float 4.500000e+00, float 0x400B333340000000, float 0x4002666660000000, float 0x3FF3333340000000, float 4.500000e+00, float 0x400B333340000000, float 0x4002666660000000, float 0x3FF3333340000000>
-  ret <16 x float> %mul.i
-}
-
-define <8 x double> @divpd512(<8 x double> %y, <8 x double> %x) {
-; GENERIC-LABEL: divpd512:
-; GENERIC:       # %bb.0: # %entry
-; GENERIC-NEXT:    vdivpd %zmm0, %zmm1, %zmm0 # sched: [45:44.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: divpd512:
-; SKX:       # %bb.0: # %entry
-; SKX-NEXT:    vdivpd %zmm0, %zmm1, %zmm0 # sched: [23:16.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-entry:
-  %div.i = fdiv <8 x double> %x, %y
-  ret <8 x double> %div.i
-}
-
-define <8 x double> @divpd512fold(<8 x double> %y) {
-; GENERIC-LABEL: divpd512fold:
-; GENERIC:       # %bb.0: # %entry
-; GENERIC-NEXT:    vdivpd {{.*}}(%rip), %zmm0, %zmm0 # sched: [52:44.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: divpd512fold:
-; SKX:       # %bb.0: # %entry
-; SKX-NEXT:    vdivpd {{.*}}(%rip), %zmm0, %zmm0 # sched: [30:16.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-entry:
-  %div.i = fdiv <8 x double> %y, <double 4.500000e+00, double 3.400000e+00, double 2.300000e+00, double 1.200000e+00, double 4.500000e+00, double 3.400000e+00, double 2.300000e+00, double 1.200000e+00>
-  ret <8 x double> %div.i
-}
-
-define <16 x float> @divps512(<16 x float> %y, <16 x float> %x) {
-; GENERIC-LABEL: divps512:
-; GENERIC:       # %bb.0: # %entry
-; GENERIC-NEXT:    vdivps %zmm0, %zmm1, %zmm0 # sched: [29:28.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: divps512:
-; SKX:       # %bb.0: # %entry
-; SKX-NEXT:    vdivps %zmm0, %zmm1, %zmm0 # sched: [18:10.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-entry:
-  %div.i = fdiv <16 x float> %x, %y
-  ret <16 x float> %div.i
-}
-
-define <16 x float> @divps512fold(<16 x float> %y) {
-; GENERIC-LABEL: divps512fold:
-; GENERIC:       # %bb.0: # %entry
-; GENERIC-NEXT:    vdivps {{.*}}(%rip), %zmm0, %zmm0 # sched: [36:28.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: divps512fold:
-; SKX:       # %bb.0: # %entry
-; SKX-NEXT:    vdivps {{.*}}(%rip), %zmm0, %zmm0 # sched: [25:10.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-entry:
-  %div.i = fdiv <16 x float> %y, <float 4.500000e+00, float 0x400B333340000000, float 0x4002666660000000, float 0x3FF3333340000000, float 4.500000e+00, float 4.500000e+00, float 0x4002666660000000, float 0x3FF3333340000000, float 4.500000e+00, float 0x400B333340000000, float 0x4002666660000000, float 0x3FF3333340000000, float 4.500000e+00, float 4.500000e+00, float 0x4002666660000000, float 0x3FF3333340000000>
-  ret <16 x float> %div.i
-}
-
-define <8 x i64> @vpaddq_test(<8 x i64> %i, <8 x i64> %j) nounwind readnone {
-; GENERIC-LABEL: vpaddq_test:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpaddq %zmm1, %zmm0, %zmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: vpaddq_test:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpaddq %zmm1, %zmm0, %zmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %x = add <8 x i64> %i, %j
-  ret <8 x i64> %x
-}
-
-define <8 x i64> @vpaddq_fold_test(<8 x i64> %i, <8 x i64>* %j) nounwind {
-; GENERIC-LABEL: vpaddq_fold_test:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpaddq (%rdi), %zmm0, %zmm0 # sched: [8:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: vpaddq_fold_test:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpaddq (%rdi), %zmm0, %zmm0 # sched: [8:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %tmp = load <8 x i64>, <8 x i64>* %j, align 4
-  %x = add <8 x i64> %i, %tmp
-  ret <8 x i64> %x
-}
-
-define <8 x i64> @vpaddq_broadcast_test(<8 x i64> %i) nounwind {
-; GENERIC-LABEL: vpaddq_broadcast_test:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpaddq {{.*}}(%rip){1to8}, %zmm0, %zmm0 # sched: [8:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: vpaddq_broadcast_test:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpaddq {{.*}}(%rip){1to8}, %zmm0, %zmm0 # sched: [8:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %x = add <8 x i64> %i, <i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2>
-  ret <8 x i64> %x
-}
-
-define <8 x i64> @vpaddq_broadcast2_test(<8 x i64> %i, i64* %j) nounwind {
-; GENERIC-LABEL: vpaddq_broadcast2_test:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpaddq (%rdi){1to8}, %zmm0, %zmm0 # sched: [8:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: vpaddq_broadcast2_test:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpaddq (%rdi){1to8}, %zmm0, %zmm0 # sched: [8:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %tmp = load i64, i64* %j
-  %j.0 = insertelement <8 x i64> undef, i64 %tmp, i32 0
-  %j.1 = insertelement <8 x i64> %j.0, i64 %tmp, i32 1
-  %j.2 = insertelement <8 x i64> %j.1, i64 %tmp, i32 2
-  %j.3 = insertelement <8 x i64> %j.2, i64 %tmp, i32 3
-  %j.4 = insertelement <8 x i64> %j.3, i64 %tmp, i32 4
-  %j.5 = insertelement <8 x i64> %j.4, i64 %tmp, i32 5
-  %j.6 = insertelement <8 x i64> %j.5, i64 %tmp, i32 6
-  %j.7 = insertelement <8 x i64> %j.6, i64 %tmp, i32 7
-  %x = add <8 x i64> %i, %j.7
-  ret <8 x i64> %x
-}
-
-define <16 x i32> @vpaddd_test(<16 x i32> %i, <16 x i32> %j) nounwind readnone {
-; GENERIC-LABEL: vpaddd_test:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpaddd %zmm1, %zmm0, %zmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: vpaddd_test:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpaddd %zmm1, %zmm0, %zmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %x = add <16 x i32> %i, %j
-  ret <16 x i32> %x
-}
-
-define <16 x i32> @vpaddd_fold_test(<16 x i32> %i, <16 x i32>* %j) nounwind {
-; GENERIC-LABEL: vpaddd_fold_test:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpaddd (%rdi), %zmm0, %zmm0 # sched: [8:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: vpaddd_fold_test:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpaddd (%rdi), %zmm0, %zmm0 # sched: [8:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %tmp = load <16 x i32>, <16 x i32>* %j, align 4
-  %x = add <16 x i32> %i, %tmp
-  ret <16 x i32> %x
-}
-
-define <16 x i32> @vpaddd_broadcast_test(<16 x i32> %i) nounwind {
-; GENERIC-LABEL: vpaddd_broadcast_test:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpaddd {{.*}}(%rip){1to16}, %zmm0, %zmm0 # sched: [8:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: vpaddd_broadcast_test:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpaddd {{.*}}(%rip){1to16}, %zmm0, %zmm0 # sched: [8:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %x = add <16 x i32> %i, <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
-  ret <16 x i32> %x
-}
-
-define <16 x i32> @vpaddd_mask_test(<16 x i32> %i, <16 x i32> %j, <16 x i32> %mask1) nounwind readnone {
-; GENERIC-LABEL: vpaddd_mask_test:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestmd %zmm2, %zmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpaddd %zmm1, %zmm0, %zmm0 {%k1} # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: vpaddd_mask_test:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestmd %zmm2, %zmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpaddd %zmm1, %zmm0, %zmm0 {%k1} # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %mask = icmp ne <16 x i32> %mask1, zeroinitializer
-  %x = add <16 x i32> %i, %j
-  %r = select <16 x i1> %mask, <16 x i32> %x, <16 x i32> %i
-  ret <16 x i32> %r
-}
-
-define <16 x i32> @vpaddd_maskz_test(<16 x i32> %i, <16 x i32> %j, <16 x i32> %mask1) nounwind readnone {
-; GENERIC-LABEL: vpaddd_maskz_test:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestmd %zmm2, %zmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpaddd %zmm1, %zmm0, %zmm0 {%k1} {z} # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: vpaddd_maskz_test:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestmd %zmm2, %zmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpaddd %zmm1, %zmm0, %zmm0 {%k1} {z} # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %mask = icmp ne <16 x i32> %mask1, zeroinitializer
-  %x = add <16 x i32> %i, %j
-  %r = select <16 x i1> %mask, <16 x i32> %x, <16 x i32> zeroinitializer
-  ret <16 x i32> %r
-}
-
-define <16 x i32> @vpaddd_mask_fold_test(<16 x i32> %i, <16 x i32>* %j.ptr, <16 x i32> %mask1) nounwind readnone {
-; GENERIC-LABEL: vpaddd_mask_fold_test:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestmd %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpaddd (%rdi), %zmm0, %zmm0 {%k1} # sched: [8:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: vpaddd_mask_fold_test:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestmd %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpaddd (%rdi), %zmm0, %zmm0 {%k1} # sched: [8:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %mask = icmp ne <16 x i32> %mask1, zeroinitializer
-  %j = load <16 x i32>, <16 x i32>* %j.ptr
-  %x = add <16 x i32> %i, %j
-  %r = select <16 x i1> %mask, <16 x i32> %x, <16 x i32> %i
-  ret <16 x i32> %r
-}
-
-define <16 x i32> @vpaddd_mask_broadcast_test(<16 x i32> %i, <16 x i32> %mask1) nounwind readnone {
-; GENERIC-LABEL: vpaddd_mask_broadcast_test:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestmd %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpaddd {{.*}}(%rip){1to16}, %zmm0, %zmm0 {%k1} # sched: [8:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: vpaddd_mask_broadcast_test:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestmd %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpaddd {{.*}}(%rip){1to16}, %zmm0, %zmm0 {%k1} # sched: [8:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %mask = icmp ne <16 x i32> %mask1, zeroinitializer
-  %x = add <16 x i32> %i, <i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4>
-  %r = select <16 x i1> %mask, <16 x i32> %x, <16 x i32> %i
-  ret <16 x i32> %r
-}
-
-define <16 x i32> @vpaddd_maskz_fold_test(<16 x i32> %i, <16 x i32>* %j.ptr, <16 x i32> %mask1) nounwind readnone {
-; GENERIC-LABEL: vpaddd_maskz_fold_test:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestmd %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpaddd (%rdi), %zmm0, %zmm0 {%k1} {z} # sched: [8:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: vpaddd_maskz_fold_test:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestmd %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpaddd (%rdi), %zmm0, %zmm0 {%k1} {z} # sched: [8:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %mask = icmp ne <16 x i32> %mask1, zeroinitializer
-  %j = load <16 x i32>, <16 x i32>* %j.ptr
-  %x = add <16 x i32> %i, %j
-  %r = select <16 x i1> %mask, <16 x i32> %x, <16 x i32> zeroinitializer
-  ret <16 x i32> %r
-}
-
-define <16 x i32> @vpaddd_maskz_broadcast_test(<16 x i32> %i, <16 x i32> %mask1) nounwind readnone {
-; GENERIC-LABEL: vpaddd_maskz_broadcast_test:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestmd %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpaddd {{.*}}(%rip){1to16}, %zmm0, %zmm0 {%k1} {z} # sched: [8:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: vpaddd_maskz_broadcast_test:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestmd %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpaddd {{.*}}(%rip){1to16}, %zmm0, %zmm0 {%k1} {z} # sched: [8:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %mask = icmp ne <16 x i32> %mask1, zeroinitializer
-  %x = add <16 x i32> %i, <i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5>
-  %r = select <16 x i1> %mask, <16 x i32> %x, <16 x i32> zeroinitializer
-  ret <16 x i32> %r
-}
-
-define <8 x i64> @vpsubq_test(<8 x i64> %i, <8 x i64> %j) nounwind readnone {
-; GENERIC-LABEL: vpsubq_test:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpsubq %zmm1, %zmm0, %zmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: vpsubq_test:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpsubq %zmm1, %zmm0, %zmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %x = sub <8 x i64> %i, %j
-  ret <8 x i64> %x
-}
-
-define <16 x i32> @vpsubd_test(<16 x i32> %i, <16 x i32> %j) nounwind readnone {
-; GENERIC-LABEL: vpsubd_test:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpsubd %zmm1, %zmm0, %zmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: vpsubd_test:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpsubd %zmm1, %zmm0, %zmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %x = sub <16 x i32> %i, %j
-  ret <16 x i32> %x
-}
-
-define <16 x i32> @vpmulld_test(<16 x i32> %i, <16 x i32> %j) {
-; GENERIC-LABEL: vpmulld_test:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpmulld %zmm1, %zmm0, %zmm0 # sched: [5:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: vpmulld_test:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpmulld %zmm1, %zmm0, %zmm0 # sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %x = mul <16 x i32> %i, %j
-  ret <16 x i32> %x
-}
-
-declare float @sqrtf(float) readnone
-define float @sqrtA(float %a) nounwind uwtable readnone ssp {
-; GENERIC-LABEL: sqrtA:
-; GENERIC:       # %bb.0: # %entry
-; GENERIC-NEXT:    vsqrtss %xmm0, %xmm0, %xmm0 # sched: [14:14.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: sqrtA:
-; SKX:       # %bb.0: # %entry
-; SKX-NEXT:    vsqrtss %xmm0, %xmm0, %xmm0 # sched: [12:3.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-entry:
-  %conv1 = tail call float @sqrtf(float %a) nounwind readnone
-  ret float %conv1
-}
-
-declare double @sqrt(double) readnone
-define double @sqrtB(double %a) nounwind uwtable readnone ssp {
-; GENERIC-LABEL: sqrtB:
-; GENERIC:       # %bb.0: # %entry
-; GENERIC-NEXT:    vsqrtsd %xmm0, %xmm0, %xmm0 # sched: [21:21.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: sqrtB:
-; SKX:       # %bb.0: # %entry
-; SKX-NEXT:    vsqrtsd %xmm0, %xmm0, %xmm0 # sched: [18:6.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-entry:
-  %call = tail call double @sqrt(double %a) nounwind readnone
-  ret double %call
-}
-
-declare float @llvm.sqrt.f32(float)
-define float @sqrtC(float %a) nounwind {
-; GENERIC-LABEL: sqrtC:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vsqrtss %xmm0, %xmm0, %xmm0 # sched: [14:14.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: sqrtC:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vsqrtss %xmm0, %xmm0, %xmm0 # sched: [12:3.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %b = call float @llvm.sqrt.f32(float %a)
-  ret float %b
-}
-
-declare <16 x float> @llvm.sqrt.v16f32(<16 x float>)
-define <16 x float> @sqrtD(<16 x float> %a) nounwind {
-; GENERIC-LABEL: sqrtD:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vsqrtps %zmm0, %zmm0 # sched: [29:28.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: sqrtD:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vsqrtps %zmm0, %zmm0 # sched: [20:12.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %b = call <16 x float> @llvm.sqrt.v16f32(<16 x float> %a)
-  ret <16 x float> %b
-}
-
-declare <8 x double> @llvm.sqrt.v8f64(<8 x double>)
-define <8 x double> @sqrtE(<8 x double> %a) nounwind {
-; GENERIC-LABEL: sqrtE:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vsqrtpd %zmm0, %zmm0 # sched: [45:44.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: sqrtE:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vsqrtpd %zmm0, %zmm0 # sched: [32:24.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %b = call <8 x double> @llvm.sqrt.v8f64(<8 x double> %a)
-  ret <8 x double> %b
-}
-
-define <16 x float> @fadd_broadcast(<16 x float> %a) nounwind {
-; GENERIC-LABEL: fadd_broadcast:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vaddps {{.*}}(%rip){1to16}, %zmm0, %zmm0 # sched: [10:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: fadd_broadcast:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vaddps {{.*}}(%rip){1to16}, %zmm0, %zmm0 # sched: [11:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %b = fadd <16 x float> %a, <float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000>
-  ret <16 x float> %b
-}
-
-define <8 x i64> @addq_broadcast(<8 x i64> %a) nounwind {
-; GENERIC-LABEL: addq_broadcast:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpaddq {{.*}}(%rip){1to8}, %zmm0, %zmm0 # sched: [8:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: addq_broadcast:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpaddq {{.*}}(%rip){1to8}, %zmm0, %zmm0 # sched: [8:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %b = add <8 x i64> %a, <i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2>
-  ret <8 x i64> %b
-}
-
-define <8 x i64> @orq_broadcast(<8 x i64> %a) nounwind {
-; GENERIC-LABEL: orq_broadcast:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vorpd {{.*}}(%rip){1to8}, %zmm0, %zmm0 # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: orq_broadcast:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vorpd {{.*}}(%rip){1to8}, %zmm0, %zmm0 # sched: [8:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %b = or <8 x i64> %a, <i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2, i64 2>
-  ret <8 x i64> %b
-}
-
-define <16 x i32> @andd512fold(<16 x i32> %y, <16 x i32>* %x) {
-; GENERIC-LABEL: andd512fold:
-; GENERIC:       # %bb.0: # %entry
-; GENERIC-NEXT:    vandps (%rdi), %zmm0, %zmm0 # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: andd512fold:
-; SKX:       # %bb.0: # %entry
-; SKX-NEXT:    vandps (%rdi), %zmm0, %zmm0 # sched: [8:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-entry:
-  %a = load <16 x i32>, <16 x i32>* %x, align 4
-  %b = and <16 x i32> %y, %a
-  ret <16 x i32> %b
-}
-
-define <8 x i64> @andqbrst(<8 x i64> %p1, i64* %ap) {
-; GENERIC-LABEL: andqbrst:
-; GENERIC:       # %bb.0: # %entry
-; GENERIC-NEXT:    vandpd (%rdi){1to8}, %zmm0, %zmm0 # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: andqbrst:
-; SKX:       # %bb.0: # %entry
-; SKX-NEXT:    vandpd (%rdi){1to8}, %zmm0, %zmm0 # sched: [8:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-entry:
-  %a = load i64, i64* %ap, align 8
-  %b = insertelement <8 x i64> undef, i64 %a, i32 0
-  %c = shufflevector <8 x i64> %b, <8 x i64> undef, <8 x i32> zeroinitializer
-  %d = and <8 x i64> %p1, %c
-  ret <8 x i64>%d
-}
-
-define <16 x float> @test_mask_vaddps(<16 x float> %dst, <16 x float> %i,
-; GENERIC-LABEL: test_mask_vaddps:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestmd %zmm3, %zmm3, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vaddps %zmm2, %zmm1, %zmm0 {%k1} # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_mask_vaddps:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestmd %zmm3, %zmm3, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vaddps %zmm2, %zmm1, %zmm0 {%k1} # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-                                     <16 x float> %j, <16 x i32> %mask1)
-                                     nounwind readnone {
-  %mask = icmp ne <16 x i32> %mask1, zeroinitializer
-  %x = fadd <16 x float> %i, %j
-  %r = select <16 x i1> %mask, <16 x float> %x, <16 x float> %dst
-  ret <16 x float> %r
-}
-
-define <16 x float> @test_mask_vmulps(<16 x float> %dst, <16 x float> %i, <16 x float> %j, <16 x i32> %mask1) nounwind readnone {
-; GENERIC-LABEL: test_mask_vmulps:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestmd %zmm3, %zmm3, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vmulps %zmm2, %zmm1, %zmm0 {%k1} # sched: [5:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_mask_vmulps:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestmd %zmm3, %zmm3, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vmulps %zmm2, %zmm1, %zmm0 {%k1} # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %mask = icmp ne <16 x i32> %mask1, zeroinitializer
-  %x = fmul <16 x float> %i, %j
-  %r = select <16 x i1> %mask, <16 x float> %x, <16 x float> %dst
-  ret <16 x float> %r
-}
-
-define <16 x float> @test_mask_vminps(<16 x float> %dst, <16 x float> %i, <16 x float> %j, <16 x i32> %mask1) nounwind readnone {
-; GENERIC-LABEL: test_mask_vminps:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestmd %zmm3, %zmm3, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vminps %zmm2, %zmm1, %zmm0 {%k1} # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_mask_vminps:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestmd %zmm3, %zmm3, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vminps %zmm2, %zmm1, %zmm0 {%k1} # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %mask = icmp ne <16 x i32> %mask1, zeroinitializer
-  %cmp_res = fcmp olt <16 x float> %i, %j
-  %min = select <16 x i1> %cmp_res, <16 x float> %i, <16 x float> %j
-  %r = select <16 x i1> %mask, <16 x float> %min, <16 x float> %dst
-  ret <16 x float> %r
-}
-
-define <8 x double> @test_mask_vminpd(<8 x double> %dst, <8 x double> %i, <8 x double> %j, <8 x i32> %mask1) nounwind readnone {
-; GENERIC-LABEL: test_mask_vminpd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestmd %ymm3, %ymm3, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vminpd %zmm2, %zmm1, %zmm0 {%k1} # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_mask_vminpd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestmd %ymm3, %ymm3, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vminpd %zmm2, %zmm1, %zmm0 {%k1} # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %mask = icmp ne <8 x i32> %mask1, zeroinitializer
-  %cmp_res = fcmp olt <8 x double> %i, %j
-  %min = select <8 x i1> %cmp_res, <8 x double> %i, <8 x double> %j
-  %r = select <8 x i1> %mask, <8 x double> %min, <8 x double> %dst
-  ret <8 x double> %r
-}
-
-define <16 x float> @test_mask_vmaxps(<16 x float> %dst, <16 x float> %i, <16 x float> %j, <16 x i32> %mask1) nounwind readnone {
-; GENERIC-LABEL: test_mask_vmaxps:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestmd %zmm3, %zmm3, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vmaxps %zmm2, %zmm1, %zmm0 {%k1} # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_mask_vmaxps:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestmd %zmm3, %zmm3, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vmaxps %zmm2, %zmm1, %zmm0 {%k1} # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %mask = icmp ne <16 x i32> %mask1, zeroinitializer
-  %cmp_res = fcmp ogt <16 x float> %i, %j
-  %max = select <16 x i1> %cmp_res, <16 x float> %i, <16 x float> %j
-  %r = select <16 x i1> %mask, <16 x float> %max, <16 x float> %dst
-  ret <16 x float> %r
-}
-
-define <8 x double> @test_mask_vmaxpd(<8 x double> %dst, <8 x double> %i, <8 x double> %j, <8 x i32> %mask1) nounwind readnone {
-; GENERIC-LABEL: test_mask_vmaxpd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestmd %ymm3, %ymm3, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vmaxpd %zmm2, %zmm1, %zmm0 {%k1} # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_mask_vmaxpd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestmd %ymm3, %ymm3, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vmaxpd %zmm2, %zmm1, %zmm0 {%k1} # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %mask = icmp ne <8 x i32> %mask1, zeroinitializer
-  %cmp_res = fcmp ogt <8 x double> %i, %j
-  %max = select <8 x i1> %cmp_res, <8 x double> %i, <8 x double> %j
-  %r = select <8 x i1> %mask, <8 x double> %max, <8 x double> %dst
-  ret <8 x double> %r
-}
-
-define <16 x float> @test_mask_vsubps(<16 x float> %dst, <16 x float> %i, <16 x float> %j, <16 x i32> %mask1) nounwind readnone {
-; GENERIC-LABEL: test_mask_vsubps:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestmd %zmm3, %zmm3, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vsubps %zmm2, %zmm1, %zmm0 {%k1} # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_mask_vsubps:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestmd %zmm3, %zmm3, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vsubps %zmm2, %zmm1, %zmm0 {%k1} # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %mask = icmp ne <16 x i32> %mask1, zeroinitializer
-  %x = fsub <16 x float> %i, %j
-  %r = select <16 x i1> %mask, <16 x float> %x, <16 x float> %dst
-  ret <16 x float> %r
-}
-
-define <16 x float> @test_mask_vdivps(<16 x float> %dst, <16 x float> %i, <16 x float> %j, <16 x i32> %mask1) nounwind readnone {
-; GENERIC-LABEL: test_mask_vdivps:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestmd %zmm3, %zmm3, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vdivps %zmm2, %zmm1, %zmm0 {%k1} # sched: [29:28.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_mask_vdivps:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestmd %zmm3, %zmm3, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vdivps %zmm2, %zmm1, %zmm0 {%k1} # sched: [18:10.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %mask = icmp ne <16 x i32> %mask1, zeroinitializer
-  %x = fdiv <16 x float> %i, %j
-  %r = select <16 x i1> %mask, <16 x float> %x, <16 x float> %dst
-  ret <16 x float> %r
-}
-
-define <8 x double> @test_mask_vaddpd(<8 x double> %dst, <8 x double> %i, <8 x double> %j, <8 x i64> %mask1) nounwind readnone {
-; GENERIC-LABEL: test_mask_vaddpd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestmq %zmm3, %zmm3, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vaddpd %zmm2, %zmm1, %zmm0 {%k1} # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_mask_vaddpd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestmq %zmm3, %zmm3, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vaddpd %zmm2, %zmm1, %zmm0 {%k1} # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %mask = icmp ne <8 x i64> %mask1, zeroinitializer
-  %x = fadd <8 x double> %i, %j
-  %r = select <8 x i1> %mask, <8 x double> %x, <8 x double> %dst
-  ret <8 x double> %r
-}
-
-define <8 x double> @test_maskz_vaddpd(<8 x double> %i, <8 x double> %j, <8 x i64> %mask1) nounwind readnone {
-; GENERIC-LABEL: test_maskz_vaddpd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestmq %zmm2, %zmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vaddpd %zmm1, %zmm0, %zmm0 {%k1} {z} # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_maskz_vaddpd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestmq %zmm2, %zmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vaddpd %zmm1, %zmm0, %zmm0 {%k1} {z} # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %mask = icmp ne <8 x i64> %mask1, zeroinitializer
-  %x = fadd <8 x double> %i, %j
-  %r = select <8 x i1> %mask, <8 x double> %x, <8 x double> zeroinitializer
-  ret <8 x double> %r
-}
-
-define <8 x double> @test_mask_fold_vaddpd(<8 x double> %dst, <8 x double> %i, <8 x double>* %j,  <8 x i64> %mask1) nounwind {
-; GENERIC-LABEL: test_mask_fold_vaddpd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestmq %zmm2, %zmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vaddpd (%rdi), %zmm1, %zmm0 {%k1} # sched: [10:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_mask_fold_vaddpd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestmq %zmm2, %zmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vaddpd (%rdi), %zmm1, %zmm0 {%k1} # sched: [11:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %mask = icmp ne <8 x i64> %mask1, zeroinitializer
-  %tmp = load <8 x double>, <8 x double>* %j, align 8
-  %x = fadd <8 x double> %i, %tmp
-  %r = select <8 x i1> %mask, <8 x double> %x, <8 x double> %dst
-  ret <8 x double> %r
-}
-
-define <8 x double> @test_maskz_fold_vaddpd(<8 x double> %i, <8 x double>* %j, <8 x i64> %mask1) nounwind {
-; GENERIC-LABEL: test_maskz_fold_vaddpd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestmq %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vaddpd (%rdi), %zmm0, %zmm0 {%k1} {z} # sched: [10:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_maskz_fold_vaddpd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestmq %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vaddpd (%rdi), %zmm0, %zmm0 {%k1} {z} # sched: [11:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %mask = icmp ne <8 x i64> %mask1, zeroinitializer
-  %tmp = load <8 x double>, <8 x double>* %j, align 8
-  %x = fadd <8 x double> %i, %tmp
-  %r = select <8 x i1> %mask, <8 x double> %x, <8 x double> zeroinitializer
-  ret <8 x double> %r
-}
-
-define <8 x double> @test_broadcast_vaddpd(<8 x double> %i, double* %j) nounwind {
-; GENERIC-LABEL: test_broadcast_vaddpd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vaddpd (%rdi){1to8}, %zmm0, %zmm0 # sched: [10:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_broadcast_vaddpd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vaddpd (%rdi){1to8}, %zmm0, %zmm0 # sched: [11:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %tmp = load double, double* %j
-  %b = insertelement <8 x double> undef, double %tmp, i32 0
-  %c = shufflevector <8 x double> %b, <8 x double> undef,
-                     <8 x i32> zeroinitializer
-  %x = fadd <8 x double> %c, %i
-  ret <8 x double> %x
-}
-
-define <8 x double> @test_mask_broadcast_vaddpd(<8 x double> %dst, <8 x double> %i, double* %j, <8 x i64> %mask1) nounwind {
-; GENERIC-LABEL: test_mask_broadcast_vaddpd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovapd %zmm1, %zmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vptestmq %zmm2, %zmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vaddpd (%rdi){1to8}, %zmm1, %zmm0 {%k1} # sched: [10:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_mask_broadcast_vaddpd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovapd %zmm1, %zmm0 # sched: [1:0.33]
-; SKX-NEXT:    vptestmq %zmm2, %zmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vaddpd (%rdi){1to8}, %zmm1, %zmm0 {%k1} # sched: [11:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %mask = icmp ne <8 x i64> %mask1, zeroinitializer
-  %tmp = load double, double* %j
-  %b = insertelement <8 x double> undef, double %tmp, i32 0
-  %c = shufflevector <8 x double> %b, <8 x double> undef,
-                     <8 x i32> zeroinitializer
-  %x = fadd <8 x double> %c, %i
-  %r = select <8 x i1> %mask, <8 x double> %x, <8 x double> %i
-  ret <8 x double> %r
-}
-
-define <8 x double> @test_maskz_broadcast_vaddpd(<8 x double> %i, double* %j,
-; GENERIC-LABEL: test_maskz_broadcast_vaddpd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestmq %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vaddpd (%rdi){1to8}, %zmm0, %zmm0 {%k1} {z} # sched: [10:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_maskz_broadcast_vaddpd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestmq %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vaddpd (%rdi){1to8}, %zmm0, %zmm0 {%k1} {z} # sched: [11:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-                                       <8 x i64> %mask1) nounwind {
-  %mask = icmp ne <8 x i64> %mask1, zeroinitializer
-  %tmp = load double, double* %j
-  %b = insertelement <8 x double> undef, double %tmp, i32 0
-  %c = shufflevector <8 x double> %b, <8 x double> undef,
-                     <8 x i32> zeroinitializer
-  %x = fadd <8 x double> %c, %i
-  %r = select <8 x i1> %mask, <8 x double> %x, <8 x double> zeroinitializer
-  ret <8 x double> %r
-}
-
-define <16 x float>  @test_fxor(<16 x float> %a) {
-; GENERIC-LABEL: test_fxor:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vxorps {{.*}}(%rip){1to16}, %zmm0, %zmm0 # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_fxor:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vxorps {{.*}}(%rip){1to16}, %zmm0, %zmm0 # sched: [8:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-
-  %res = fsub <16 x float> <float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00>, %a
-  ret <16 x float>%res
-}
-
-define <8 x float>  @test_fxor_8f32(<8 x float> %a) {
-; GENERIC-LABEL: test_fxor_8f32:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vxorps {{.*}}(%rip){1to8}, %ymm0, %ymm0 # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_fxor_8f32:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vxorps {{.*}}(%rip){1to8}, %ymm0, %ymm0 # sched: [8:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %res = fsub <8 x float> <float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00>, %a
-  ret <8 x float>%res
-}
-
-define <8 x double> @fabs_v8f64(<8 x double> %p)
-; GENERIC-LABEL: fabs_v8f64:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vandpd {{.*}}(%rip){1to8}, %zmm0, %zmm0 # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: fabs_v8f64:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vandpd {{.*}}(%rip){1to8}, %zmm0, %zmm0 # sched: [8:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-{
-  %t = call <8 x double> @llvm.fabs.v8f64(<8 x double> %p)
-  ret <8 x double> %t
-}
-declare <8 x double> @llvm.fabs.v8f64(<8 x double> %p)
-
-define <16 x float> @fabs_v16f32(<16 x float> %p)
-; GENERIC-LABEL: fabs_v16f32:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vandps {{.*}}(%rip){1to16}, %zmm0, %zmm0 # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: fabs_v16f32:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vandps {{.*}}(%rip){1to16}, %zmm0, %zmm0 # sched: [8:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-{
-  %t = call <16 x float> @llvm.fabs.v16f32(<16 x float> %p)
-  ret <16 x float> %t
-}
-declare <16 x float> @llvm.fabs.v16f32(<16 x float> %p)
-
-define double @test1(double %a, double %b) nounwind {
-; GENERIC-LABEL: test1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vucomisd %xmm1, %xmm0 # sched: [2:1.00]
-; GENERIC-NEXT:    jne .LBB64_1 # sched: [1:1.00]
-; GENERIC-NEXT:    jnp .LBB64_2 # sched: [1:1.00]
-; GENERIC-NEXT:  .LBB64_1: # %l1
-; GENERIC-NEXT:    vsubsd %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-; GENERIC-NEXT:  .LBB64_2: # %l2
-; GENERIC-NEXT:    vaddsd %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vucomisd %xmm1, %xmm0 # sched: [2:1.00]
-; SKX-NEXT:    jne .LBB64_1 # sched: [1:0.50]
-; SKX-NEXT:    jnp .LBB64_2 # sched: [1:0.50]
-; SKX-NEXT:  .LBB64_1: # %l1
-; SKX-NEXT:    vsubsd %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-; SKX-NEXT:  .LBB64_2: # %l2
-; SKX-NEXT:    vaddsd %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %tobool = fcmp une double %a, %b
-  br i1 %tobool, label %l1, label %l2
-
-l1:
-  %c = fsub double %a, %b
-  ret double %c
-l2:
-  %c1 = fadd double %a, %b
-  ret double %c1
-}
-
-define float @test2(float %a, float %b) nounwind {
-; GENERIC-LABEL: test2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vucomiss %xmm0, %xmm1 # sched: [2:1.00]
-; GENERIC-NEXT:    jbe .LBB65_2 # sched: [1:1.00]
-; GENERIC-NEXT:  # %bb.1: # %l1
-; GENERIC-NEXT:    vsubss %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-; GENERIC-NEXT:  .LBB65_2: # %l2
-; GENERIC-NEXT:    vaddss %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vucomiss %xmm0, %xmm1 # sched: [2:1.00]
-; SKX-NEXT:    jbe .LBB65_2 # sched: [1:0.50]
-; SKX-NEXT:  # %bb.1: # %l1
-; SKX-NEXT:    vsubss %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-; SKX-NEXT:  .LBB65_2: # %l2
-; SKX-NEXT:    vaddss %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %tobool = fcmp olt float %a, %b
-  br i1 %tobool, label %l1, label %l2
-
-l1:
-  %c = fsub float %a, %b
-  ret float %c
-l2:
-  %c1 = fadd float %a, %b
-  ret float %c1
-}
-
-define i32 @test3(float %a, float %b) {
-; GENERIC-LABEL: test3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vcmpeqss %xmm1, %xmm0, %k0 # sched: [3:1.00]
-; GENERIC-NEXT:    kmovw %k0, %eax # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vcmpeqss %xmm1, %xmm0, %k0 # sched: [3:1.00]
-; SKX-NEXT:    kmovw %k0, %eax # sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-
-  %cmp10.i = fcmp oeq float %a, %b
-  %conv11.i = zext i1 %cmp10.i to i32
-  ret i32 %conv11.i
-}
-
-define float @test5(float %p) #0 {
-; GENERIC-LABEL: test5:
-; GENERIC:       # %bb.0: # %entry
-; GENERIC-NEXT:    vxorps %xmm1, %xmm1, %xmm1 # sched: [0:0.25]
-; GENERIC-NEXT:    vucomiss %xmm1, %xmm0 # sched: [2:1.00]
-; GENERIC-NEXT:    jne .LBB67_1 # sched: [1:1.00]
-; GENERIC-NEXT:    jp .LBB67_1 # sched: [1:1.00]
-; GENERIC-NEXT:  # %bb.2: # %return
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-; GENERIC-NEXT:  .LBB67_1: # %if.end
-; GENERIC-NEXT:    vcmpltss %xmm0, %xmm1, %k1 # sched: [3:1.00]
-; GENERIC-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero sched: [6:0.50]
-; GENERIC-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero sched: [6:0.50]
-; GENERIC-NEXT:    vmovss %xmm1, %xmm0, %xmm0 {%k1} # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test5:
-; SKX:       # %bb.0: # %entry
-; SKX-NEXT:    vxorps %xmm1, %xmm1, %xmm1 # sched: [1:0.33]
-; SKX-NEXT:    vucomiss %xmm1, %xmm0 # sched: [2:1.00]
-; SKX-NEXT:    jne .LBB67_1 # sched: [1:0.50]
-; SKX-NEXT:    jp .LBB67_1 # sched: [1:0.50]
-; SKX-NEXT:  # %bb.2: # %return
-; SKX-NEXT:    retq # sched: [7:1.00]
-; SKX-NEXT:  .LBB67_1: # %if.end
-; SKX-NEXT:    vcmpltss %xmm0, %xmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero sched: [5:0.50]
-; SKX-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero sched: [5:0.50]
-; SKX-NEXT:    vmovss %xmm1, %xmm0, %xmm0 {%k1} # sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-entry:
-  %cmp = fcmp oeq float %p, 0.000000e+00
-  br i1 %cmp, label %return, label %if.end
-
-if.end:                                           ; preds = %entry
-  %cmp1 = fcmp ogt float %p, 0.000000e+00
-  %cond = select i1 %cmp1, float 1.000000e+00, float -1.000000e+00
-  br label %return
-
-return:                                           ; preds = %if.end, %entry
-  %retval.0 = phi float [ %cond, %if.end ], [ %p, %entry ]
-  ret float %retval.0
-}
-
-define i32 @test6(i32 %a, i32 %b) {
-; GENERIC-LABEL: test6:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    xorl %eax, %eax # sched: [0:0.25]
-; GENERIC-NEXT:    cmpl %esi, %edi # sched: [1:0.33]
-; GENERIC-NEXT:    sete %al # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test6:
-; SKX:       # %bb.0:
-; SKX-NEXT:    xorl %eax, %eax # sched: [1:0.25]
-; SKX-NEXT:    cmpl %esi, %edi # sched: [1:0.25]
-; SKX-NEXT:    sete %al # sched: [1:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %cmp = icmp eq i32 %a, %b
-  %res = zext i1 %cmp to i32
-  ret i32 %res
-}
-
-define i32 @test7(double %x, double %y) #2 {
-; GENERIC-LABEL: test7:
-; GENERIC:       # %bb.0: # %entry
-; GENERIC-NEXT:    xorl %eax, %eax # sched: [0:0.25]
-; GENERIC-NEXT:    vucomisd %xmm1, %xmm0 # sched: [2:1.00]
-; GENERIC-NEXT:    setne %al # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test7:
-; SKX:       # %bb.0: # %entry
-; SKX-NEXT:    xorl %eax, %eax # sched: [1:0.25]
-; SKX-NEXT:    vucomisd %xmm1, %xmm0 # sched: [2:1.00]
-; SKX-NEXT:    setne %al # sched: [1:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-entry:
-  %0 = fcmp one double %x, %y
-  %or = zext i1 %0 to i32
-  ret i32 %or
-}
-
-define i32 @test8(i32 %a1, i32 %a2, i32 %a3) {
-; GENERIC-LABEL: test8:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    xorl $-2147483648, %esi # imm = 0x80000000
-; GENERIC-NEXT:    # sched: [1:0.33]
-; GENERIC-NEXT:    testl %edx, %edx # sched: [1:0.33]
-; GENERIC-NEXT:    movl $1, %eax # sched: [1:0.33]
-; GENERIC-NEXT:    cmovel %eax, %edx # sched: [2:0.67]
-; GENERIC-NEXT:    notl %edi # sched: [1:0.33]
-; GENERIC-NEXT:    orl %edi, %esi # sched: [1:0.33]
-; GENERIC-NEXT:    cmovnel %edx, %eax # sched: [2:0.67]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test8:
-; SKX:       # %bb.0:
-; SKX-NEXT:    notl %edi # sched: [1:0.25]
-; SKX-NEXT:    xorl $-2147483648, %esi # imm = 0x80000000
-; SKX-NEXT:    # sched: [1:0.25]
-; SKX-NEXT:    testl %edx, %edx # sched: [1:0.25]
-; SKX-NEXT:    movl $1, %eax # sched: [1:0.25]
-; SKX-NEXT:    cmovel %eax, %edx # sched: [1:0.50]
-; SKX-NEXT:    orl %edi, %esi # sched: [1:0.25]
-; SKX-NEXT:    cmovnel %edx, %eax # sched: [1:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %tmp1 = icmp eq i32 %a1, -1
-  %tmp2 = icmp eq i32 %a2, -2147483648
-  %tmp3 = and i1 %tmp1, %tmp2
-  %tmp4 = icmp eq i32 %a3, 0
-  %tmp5 = or i1 %tmp3, %tmp4
-  %res = select i1 %tmp5, i32 1, i32 %a3
-  ret i32 %res
-}
-
-define i32 @test9(i64 %a) {
-; GENERIC-LABEL: test9:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    testb $1, %dil # sched: [1:0.33]
-; GENERIC-NEXT:    jne .LBB71_2 # sched: [1:1.00]
-; GENERIC-NEXT:  # %bb.1: # %A
-; GENERIC-NEXT:    movl $6, %eax # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-; GENERIC-NEXT:  .LBB71_2: # %B
-; GENERIC-NEXT:    movl $7, %eax # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test9:
-; SKX:       # %bb.0:
-; SKX-NEXT:    testb $1, %dil # sched: [1:0.25]
-; SKX-NEXT:    jne .LBB71_2 # sched: [1:0.50]
-; SKX-NEXT:  # %bb.1: # %A
-; SKX-NEXT:    movl $6, %eax # sched: [1:0.25]
-; SKX-NEXT:    retq # sched: [7:1.00]
-; SKX-NEXT:  .LBB71_2: # %B
-; SKX-NEXT:    movl $7, %eax # sched: [1:0.25]
-; SKX-NEXT:    retq # sched: [7:1.00]
- %b = and i64 %a, 1
- %cmp10.i = icmp eq i64 %b, 0
- br i1 %cmp10.i, label %A, label %B
-A:
- ret i32 6
-B:
- ret i32 7
-}
-
-define i32 @test10(i64 %b, i64 %c, i1 %d) {
-; GENERIC-LABEL: test10:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    movl %edx, %eax # sched: [1:0.33]
-; GENERIC-NEXT:    andb $1, %al # sched: [1:0.33]
-; GENERIC-NEXT:    cmpq %rsi, %rdi # sched: [1:0.33]
-; GENERIC-NEXT:    sete %cl # sched: [1:0.50]
-; GENERIC-NEXT:    orb %dl, %cl # sched: [1:0.33]
-; GENERIC-NEXT:    andb $1, %cl # sched: [1:0.33]
-; GENERIC-NEXT:    cmpb %cl, %al # sched: [1:0.33]
-; GENERIC-NEXT:    je .LBB72_1 # sched: [1:1.00]
-; GENERIC-NEXT:  # %bb.2: # %if.end.i
-; GENERIC-NEXT:    movl $6, %eax # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-; GENERIC-NEXT:  .LBB72_1: # %if.then.i
-; GENERIC-NEXT:    movl $5, %eax # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test10:
-; SKX:       # %bb.0:
-; SKX-NEXT:    movl %edx, %eax # sched: [1:0.25]
-; SKX-NEXT:    andb $1, %al # sched: [1:0.25]
-; SKX-NEXT:    cmpq %rsi, %rdi # sched: [1:0.25]
-; SKX-NEXT:    sete %cl # sched: [1:0.50]
-; SKX-NEXT:    orb %dl, %cl # sched: [1:0.25]
-; SKX-NEXT:    andb $1, %cl # sched: [1:0.25]
-; SKX-NEXT:    cmpb %cl, %al # sched: [1:0.25]
-; SKX-NEXT:    je .LBB72_1 # sched: [1:0.50]
-; SKX-NEXT:  # %bb.2: # %if.end.i
-; SKX-NEXT:    movl $6, %eax # sched: [1:0.25]
-; SKX-NEXT:    retq # sched: [7:1.00]
-; SKX-NEXT:  .LBB72_1: # %if.then.i
-; SKX-NEXT:    movl $5, %eax # sched: [1:0.25]
-; SKX-NEXT:    retq # sched: [7:1.00]
-
-  %cmp8.i = icmp eq i64 %b, %c
-  %or1 = or i1 %d, %cmp8.i
-  %xor1 = xor i1 %d, %or1
-  br i1 %xor1, label %if.end.i, label %if.then.i
-
-if.then.i:
- ret i32 5
-
-if.end.i:
-  ret i32 6
-}
-
-define <16 x float> @sitof32(<16 x i32> %a) nounwind {
-; GENERIC-LABEL: sitof32:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vcvtdq2ps %zmm0, %zmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: sitof32:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vcvtdq2ps %zmm0, %zmm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %b = sitofp <16 x i32> %a to <16 x float>
-  ret <16 x float> %b
-}
-
-define <8 x double> @sltof864(<8 x i64> %a) {
-; GENERIC-LABEL: sltof864:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vcvtqq2pd %zmm0, %zmm0 # sched: [4:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: sltof864:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vcvtqq2pd %zmm0, %zmm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %b = sitofp <8 x i64> %a to <8 x double>
-  ret <8 x double> %b
-}
-
-define <4 x double> @slto4f64(<4 x i64> %a) {
-; GENERIC-LABEL: slto4f64:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vcvtqq2pd %ymm0, %ymm0 # sched: [4:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: slto4f64:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vcvtqq2pd %ymm0, %ymm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %b = sitofp <4 x i64> %a to <4 x double>
-  ret <4 x double> %b
-}
-
-define <2 x double> @slto2f64(<2 x i64> %a) {
-; GENERIC-LABEL: slto2f64:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vcvtqq2pd %xmm0, %xmm0 # sched: [4:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: slto2f64:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vcvtqq2pd %xmm0, %xmm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %b = sitofp <2 x i64> %a to <2 x double>
-  ret <2 x double> %b
-}
-
-define <2 x float> @sltof2f32(<2 x i64> %a) {
-; GENERIC-LABEL: sltof2f32:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vcvtqq2ps %xmm0, %xmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: sltof2f32:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vcvtqq2ps %xmm0, %xmm0 # sched: [5:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %b = sitofp <2 x i64> %a to <2 x float>
-  ret <2 x float>%b
-}
-
-define <4 x float> @slto4f32_mem(<4 x i64>* %a) {
-; GENERIC-LABEL: slto4f32_mem:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vcvtqq2psy (%rdi), %xmm0 # sched: [10:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: slto4f32_mem:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vcvtqq2psy (%rdi), %xmm0 # sched: [11:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %a1 = load <4 x i64>, <4 x i64>* %a, align 8
-  %b = sitofp <4 x i64> %a1 to <4 x float>
-  ret <4 x float>%b
-}
-
-define <4 x i64> @f64to4sl(<4 x double> %a) {
-; GENERIC-LABEL: f64to4sl:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vcvttpd2qq %ymm0, %ymm0 # sched: [4:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: f64to4sl:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vcvttpd2qq %ymm0, %ymm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %b = fptosi <4 x double> %a to <4 x i64>
-  ret <4 x i64> %b
-}
-
-define <4 x i64> @f32to4sl(<4 x float> %a) {
-; GENERIC-LABEL: f32to4sl:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vcvttps2qq %xmm0, %ymm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: f32to4sl:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vcvttps2qq %xmm0, %ymm0 # sched: [7:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %b = fptosi <4 x float> %a to <4 x i64>
-  ret <4 x i64> %b
-}
-
-define <4 x float> @slto4f32(<4 x i64> %a) {
-; GENERIC-LABEL: slto4f32:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vcvtqq2ps %ymm0, %xmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    vzeroupper # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: slto4f32:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vcvtqq2ps %ymm0, %xmm0 # sched: [7:1.00]
-; SKX-NEXT:    vzeroupper # sched: [0:0.67]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %b = sitofp <4 x i64> %a to <4 x float>
-  ret <4 x float> %b
-}
-
-define <4 x float> @ulto4f32(<4 x i64> %a) {
-; GENERIC-LABEL: ulto4f32:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vcvtuqq2ps %ymm0, %xmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    vzeroupper # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: ulto4f32:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vcvtuqq2ps %ymm0, %xmm0 # sched: [7:1.00]
-; SKX-NEXT:    vzeroupper # sched: [0:0.67]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %b = uitofp <4 x i64> %a to <4 x float>
-  ret <4 x float> %b
-}
-
-define <8 x double> @ulto8f64(<8 x i64> %a) {
-; GENERIC-LABEL: ulto8f64:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vcvtuqq2pd %zmm0, %zmm0 # sched: [4:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: ulto8f64:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vcvtuqq2pd %zmm0, %zmm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %b = uitofp <8 x i64> %a to <8 x double>
-  ret <8 x double> %b
-}
-
-define <16 x double> @ulto16f64(<16 x i64> %a) {
-; GENERIC-LABEL: ulto16f64:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vcvtuqq2pd %zmm0, %zmm0 # sched: [4:1.00]
-; GENERIC-NEXT:    vcvtuqq2pd %zmm1, %zmm1 # sched: [4:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: ulto16f64:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vcvtuqq2pd %zmm0, %zmm0 # sched: [4:0.50]
-; SKX-NEXT:    vcvtuqq2pd %zmm1, %zmm1 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %b = uitofp <16 x i64> %a to <16 x double>
-  ret <16 x double> %b
-}
-
-define <16 x i32> @f64to16si(<16 x float> %a) nounwind {
-; GENERIC-LABEL: f64to16si:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vcvttps2dq %zmm0, %zmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: f64to16si:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vcvttps2dq %zmm0, %zmm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %b = fptosi <16 x float> %a to <16 x i32>
-  ret <16 x i32> %b
-}
-
-define <16 x i32> @f32to16ui(<16 x float> %a) nounwind {
-; GENERIC-LABEL: f32to16ui:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vcvttps2udq %zmm0, %zmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: f32to16ui:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vcvttps2udq %zmm0, %zmm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %b = fptoui <16 x float> %a to <16 x i32>
-  ret <16 x i32> %b
-}
-
-define <16 x i8> @f32to16uc(<16 x float> %f) {
-; GENERIC-LABEL: f32to16uc:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vcvttps2dq %zmm0, %zmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    vpmovdb %zmm0, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vzeroupper # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: f32to16uc:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vcvttps2dq %zmm0, %zmm0 # sched: [4:0.50]
-; SKX-NEXT:    vpmovdb %zmm0, %xmm0 # sched: [4:2.00]
-; SKX-NEXT:    vzeroupper # sched: [0:0.67]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %res = fptoui <16 x float> %f to <16 x i8>
-  ret <16 x i8> %res
-}
-
-define <16 x i16> @f32to16us(<16 x float> %f) {
-; GENERIC-LABEL: f32to16us:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vcvttps2dq %zmm0, %zmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    vpmovdw %zmm0, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: f32to16us:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vcvttps2dq %zmm0, %zmm0 # sched: [4:0.50]
-; SKX-NEXT:    vpmovdw %zmm0, %ymm0 # sched: [4:2.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %res = fptoui <16 x float> %f to <16 x i16>
-  ret <16 x i16> %res
-}
-
-define <8 x i32> @f32to8ui(<8 x float> %a) nounwind {
-; GENERIC-LABEL: f32to8ui:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vcvttps2udq %ymm0, %ymm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: f32to8ui:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vcvttps2udq %ymm0, %ymm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %b = fptoui <8 x float> %a to <8 x i32>
-  ret <8 x i32> %b
-}
-
-define <4 x i32> @f32to4ui(<4 x float> %a) nounwind {
-; GENERIC-LABEL: f32to4ui:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vcvttps2udq %xmm0, %xmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: f32to4ui:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vcvttps2udq %xmm0, %xmm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %b = fptoui <4 x float> %a to <4 x i32>
-  ret <4 x i32> %b
-}
-
-define <8 x i32> @f64to8ui(<8 x double> %a) nounwind {
-; GENERIC-LABEL: f64to8ui:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vcvttpd2udq %zmm0, %ymm0 # sched: [4:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: f64to8ui:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vcvttpd2udq %zmm0, %ymm0 # sched: [7:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %b = fptoui <8 x double> %a to <8 x i32>
-  ret <8 x i32> %b
-}
-
-define <8 x i16> @f64to8us(<8 x double> %f) {
-; GENERIC-LABEL: f64to8us:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vcvttpd2dq %zmm0, %ymm0 # sched: [4:1.00]
-; GENERIC-NEXT:    vpmovdw %ymm0, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vzeroupper # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: f64to8us:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vcvttpd2dq %zmm0, %ymm0 # sched: [7:1.00]
-; SKX-NEXT:    vpmovdw %ymm0, %xmm0 # sched: [4:2.00]
-; SKX-NEXT:    vzeroupper # sched: [0:0.67]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %res = fptoui <8 x double> %f to <8 x i16>
-  ret <8 x i16> %res
-}
-
-define <8 x i8> @f64to8uc(<8 x double> %f) {
-; GENERIC-LABEL: f64to8uc:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vcvttpd2dq %zmm0, %ymm0 # sched: [4:1.00]
-; GENERIC-NEXT:    vpmovdw %ymm0, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vzeroupper # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: f64to8uc:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vcvttpd2dq %zmm0, %ymm0 # sched: [7:1.00]
-; SKX-NEXT:    vpmovdw %ymm0, %xmm0 # sched: [4:2.00]
-; SKX-NEXT:    vzeroupper # sched: [0:0.67]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %res = fptoui <8 x double> %f to <8 x i8>
-  ret <8 x i8> %res
-}
-
-define <4 x i32> @f64to4ui(<4 x double> %a) nounwind {
-; GENERIC-LABEL: f64to4ui:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vcvttpd2udq %ymm0, %xmm0 # sched: [4:1.00]
-; GENERIC-NEXT:    vzeroupper # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: f64to4ui:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vcvttpd2udq %ymm0, %xmm0 # sched: [7:1.00]
-; SKX-NEXT:    vzeroupper # sched: [0:0.67]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %b = fptoui <4 x double> %a to <4 x i32>
-  ret <4 x i32> %b
-}
-
-define <8 x double> @sito8f64(<8 x i32> %a) {
-; GENERIC-LABEL: sito8f64:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vcvtdq2pd %ymm0, %zmm0 # sched: [4:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: sito8f64:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vcvtdq2pd %ymm0, %zmm0 # sched: [7:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %b = sitofp <8 x i32> %a to <8 x double>
-  ret <8 x double> %b
-}
-define <8 x double> @i32to8f64_mask(<8 x double> %a, <8 x i32> %b, i8 %c) nounwind {
-; GENERIC-LABEL: i32to8f64_mask:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    kmovd %edi, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vcvtdq2pd %ymm1, %zmm0 {%k1} # sched: [4:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: i32to8f64_mask:
-; SKX:       # %bb.0:
-; SKX-NEXT:    kmovd %edi, %k1 # sched: [1:1.00]
-; SKX-NEXT:    vcvtdq2pd %ymm1, %zmm0 {%k1} # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-; VLNOBW-LABEL: i32to8f64_mask:
-; VLNOBW:       # %bb.0:
-; VLNOBW-NEXT:    kmovw %edi, %k1
-; VLNOBW-NEXT:    vcvtdq2pd %ymm1, %zmm0 {%k1}
-; VLNOBW-NEXT:    ret{{[l|q]}}
-  %1 = bitcast i8 %c to <8 x i1>
-  %2 = sitofp <8 x i32> %b to <8 x double>
-  %3 = select <8 x i1> %1, <8 x double> %2, <8 x double> %a
-  ret <8 x double> %3
-}
-define <8 x double> @sito8f64_maskz(<8 x i32> %a, i8 %b) nounwind {
-; GENERIC-LABEL: sito8f64_maskz:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    kmovd %edi, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vcvtdq2pd %ymm0, %zmm0 {%k1} {z} # sched: [4:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: sito8f64_maskz:
-; SKX:       # %bb.0:
-; SKX-NEXT:    kmovd %edi, %k1 # sched: [1:1.00]
-; SKX-NEXT:    vcvtdq2pd %ymm0, %zmm0 {%k1} {z} # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-; VLNOBW-LABEL: sito8f64_maskz:
-; VLNOBW:       # %bb.0:
-; VLNOBW-NEXT:    kmovw %edi, %k1
-; VLNOBW-NEXT:    vcvtdq2pd %ymm0, %zmm0 {%k1} {z}
-; VLNOBW-NEXT:    ret{{[l|q]}}
-  %1 = bitcast i8 %b to <8 x i1>
-  %2 = sitofp <8 x i32> %a to <8 x double>
-  %3 = select <8 x i1> %1, <8 x double> %2, <8 x double> zeroinitializer
-  ret <8 x double> %3
-}
-
-define <8 x i32> @f64to8si(<8 x double> %a) {
-; GENERIC-LABEL: f64to8si:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vcvttpd2dq %zmm0, %ymm0 # sched: [4:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: f64to8si:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vcvttpd2dq %zmm0, %ymm0 # sched: [7:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %b = fptosi <8 x double> %a to <8 x i32>
-  ret <8 x i32> %b
-}
-
-define <4 x i32> @f64to4si(<4 x double> %a) {
-; GENERIC-LABEL: f64to4si:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vcvttpd2dq %ymm0, %xmm0 # sched: [4:1.00]
-; GENERIC-NEXT:    vzeroupper # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: f64to4si:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vcvttpd2dq %ymm0, %xmm0 # sched: [7:1.00]
-; SKX-NEXT:    vzeroupper # sched: [0:0.67]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %b = fptosi <4 x double> %a to <4 x i32>
-  ret <4 x i32> %b
-}
-
-define <16 x float> @f64to16f32(<16 x double> %b) nounwind {
-; GENERIC-LABEL: f64to16f32:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vcvtpd2ps %zmm0, %ymm0 # sched: [4:1.00]
-; GENERIC-NEXT:    vcvtpd2ps %zmm1, %ymm1 # sched: [4:1.00]
-; GENERIC-NEXT:    vinsertf64x4 $1, %ymm1, %zmm0, %zmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: f64to16f32:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vcvtpd2ps %zmm0, %ymm0 # sched: [7:1.00]
-; SKX-NEXT:    vcvtpd2ps %zmm1, %ymm1 # sched: [7:1.00]
-; SKX-NEXT:    vinsertf64x4 $1, %ymm1, %zmm0, %zmm0 # sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %a = fptrunc <16 x double> %b to <16 x float>
-  ret <16 x float> %a
-}
-
-define <4 x float> @f64to4f32(<4 x double> %b) {
-; GENERIC-LABEL: f64to4f32:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vcvtpd2ps %ymm0, %xmm0 # sched: [4:1.00]
-; GENERIC-NEXT:    vzeroupper # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: f64to4f32:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vcvtpd2ps %ymm0, %xmm0 # sched: [7:1.00]
-; SKX-NEXT:    vzeroupper # sched: [0:0.67]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %a = fptrunc <4 x double> %b to <4 x float>
-  ret <4 x float> %a
-}
-
-define <4 x float> @f64to4f32_mask(<4 x double> %b, <4 x i1> %mask) {
-; GENERIC-LABEL: f64to4f32_mask:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpslld $31, %xmm1, %xmm1 # sched: [1:1.00]
-; GENERIC-NEXT:    vpmovd2m %xmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vcvtpd2ps %ymm0, %xmm0 {%k1} {z} # sched: [4:1.00]
-; GENERIC-NEXT:    vzeroupper # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: f64to4f32_mask:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpslld $31, %xmm1, %xmm1 # sched: [1:0.50]
-; SKX-NEXT:    vpmovd2m %xmm1, %k1 # sched: [1:1.00]
-; SKX-NEXT:    vcvtpd2ps %ymm0, %xmm0 {%k1} {z} # sched: [7:1.00]
-; SKX-NEXT:    vzeroupper # sched: [0:0.67]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %a = fptrunc <4 x double> %b to <4 x float>
-  %c = select <4 x i1>%mask, <4 x float>%a, <4 x float> zeroinitializer
-  ret <4 x float> %c
-}
-
-define <4 x float> @f64tof32_inreg(<2 x double> %a0, <4 x float> %a1) nounwind {
-; GENERIC-LABEL: f64tof32_inreg:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vcvtsd2ss %xmm0, %xmm1, %xmm0 # sched: [4:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: f64tof32_inreg:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vcvtsd2ss %xmm0, %xmm1, %xmm0 # sched: [5:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %ext = extractelement <2 x double> %a0, i32 0
-  %cvt = fptrunc double %ext to float
-  %res = insertelement <4 x float> %a1, float %cvt, i32 0
-  ret <4 x float> %res
-}
-
-define <8 x double> @f32to8f64(<8 x float> %b) nounwind {
-; GENERIC-LABEL: f32to8f64:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vcvtps2pd %ymm0, %zmm0 # sched: [2:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: f32to8f64:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vcvtps2pd %ymm0, %zmm0 # sched: [7:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %a = fpext <8 x float> %b to <8 x double>
-  ret <8 x double> %a
-}
-
-define <4 x double> @f32to4f64_mask(<4 x float> %b, <4 x double> %b1, <4 x double> %a1) {
-; GENERIC-LABEL: f32to4f64_mask:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vcmpltpd %ymm2, %ymm1, %k1 # sched: [3:1.00]
-; GENERIC-NEXT:    vcvtps2pd %xmm0, %ymm0 {%k1} {z} # sched: [2:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: f32to4f64_mask:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vcmpltpd %ymm2, %ymm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vcvtps2pd %xmm0, %ymm0 {%k1} {z} # sched: [7:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %a = fpext <4 x float> %b to <4 x double>
-  %mask = fcmp ogt <4 x double> %a1, %b1
-  %c = select <4 x i1> %mask, <4 x double> %a, <4 x double> zeroinitializer
-  ret <4 x double> %c
-}
-
-define <2 x double> @f32tof64_inreg(<2 x double> %a0, <4 x float> %a1) nounwind {
-; GENERIC-LABEL: f32tof64_inreg:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vcvtss2sd %xmm1, %xmm0, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: f32tof64_inreg:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vcvtss2sd %xmm1, %xmm0, %xmm0 # sched: [5:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %ext = extractelement <4 x float> %a1, i32 0
-  %cvt = fpext float %ext to double
-  %res = insertelement <2 x double> %a0, double %cvt, i32 0
-  ret <2 x double> %res
-}
-
-define double @sltof64_load(i64* nocapture %e) {
-; GENERIC-LABEL: sltof64_load:
-; GENERIC:       # %bb.0: # %entry
-; GENERIC-NEXT:    vcvtsi2sdq (%rdi), %xmm0, %xmm0 # sched: [9:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: sltof64_load:
-; SKX:       # %bb.0: # %entry
-; SKX-NEXT:    vcvtsi2sdq (%rdi), %xmm0, %xmm0 # sched: [9:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-entry:
-  %tmp1 = load i64, i64* %e, align 8
-  %conv = sitofp i64 %tmp1 to double
-  ret double %conv
-}
-
-define double @sitof64_load(i32* %e) {
-; GENERIC-LABEL: sitof64_load:
-; GENERIC:       # %bb.0: # %entry
-; GENERIC-NEXT:    vcvtsi2sdl (%rdi), %xmm0, %xmm0 # sched: [9:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: sitof64_load:
-; SKX:       # %bb.0: # %entry
-; SKX-NEXT:    vcvtsi2sdl (%rdi), %xmm0, %xmm0 # sched: [9:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-entry:
-  %tmp1 = load i32, i32* %e, align 4
-  %conv = sitofp i32 %tmp1 to double
-  ret double %conv
-}
-
-define float @sitof32_load(i32* %e) {
-; GENERIC-LABEL: sitof32_load:
-; GENERIC:       # %bb.0: # %entry
-; GENERIC-NEXT:    vcvtsi2ssl (%rdi), %xmm0, %xmm0 # sched: [10:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: sitof32_load:
-; SKX:       # %bb.0: # %entry
-; SKX-NEXT:    vcvtsi2ssl (%rdi), %xmm0, %xmm0 # sched: [9:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-entry:
-  %tmp1 = load i32, i32* %e, align 4
-  %conv = sitofp i32 %tmp1 to float
-  ret float %conv
-}
-
-define float @sltof32_load(i64* %e) {
-; GENERIC-LABEL: sltof32_load:
-; GENERIC:       # %bb.0: # %entry
-; GENERIC-NEXT:    vcvtsi2ssq (%rdi), %xmm0, %xmm0 # sched: [10:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: sltof32_load:
-; SKX:       # %bb.0: # %entry
-; SKX-NEXT:    vcvtsi2ssq (%rdi), %xmm0, %xmm0 # sched: [9:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-entry:
-  %tmp1 = load i64, i64* %e, align 8
-  %conv = sitofp i64 %tmp1 to float
-  ret float %conv
-}
-
-define void @f32tof64_loadstore() {
-; GENERIC-LABEL: f32tof64_loadstore:
-; GENERIC:       # %bb.0: # %entry
-; GENERIC-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero sched: [6:0.50]
-; GENERIC-NEXT:    vcvtss2sd %xmm0, %xmm0, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vmovsd %xmm0, -{{[0-9]+}}(%rsp) # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: f32tof64_loadstore:
-; SKX:       # %bb.0: # %entry
-; SKX-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero sched: [5:0.50]
-; SKX-NEXT:    vcvtss2sd %xmm0, %xmm0, %xmm0 # sched: [5:1.00]
-; SKX-NEXT:    vmovsd %xmm0, -{{[0-9]+}}(%rsp) # sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-entry:
-  %f = alloca float, align 4
-  %d = alloca double, align 8
-  %tmp = load float, float* %f, align 4
-  %conv = fpext float %tmp to double
-  store double %conv, double* %d, align 8
-  ret void
-}
-
-define void @f64tof32_loadstore() nounwind uwtable {
-; GENERIC-LABEL: f64tof32_loadstore:
-; GENERIC:       # %bb.0: # %entry
-; GENERIC-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero sched: [6:0.50]
-; GENERIC-NEXT:    vcvtsd2ss %xmm0, %xmm0, %xmm0 # sched: [4:1.00]
-; GENERIC-NEXT:    vmovss %xmm0, -{{[0-9]+}}(%rsp) # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: f64tof32_loadstore:
-; SKX:       # %bb.0: # %entry
-; SKX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero sched: [5:0.50]
-; SKX-NEXT:    vcvtsd2ss %xmm0, %xmm0, %xmm0 # sched: [5:1.00]
-; SKX-NEXT:    vmovss %xmm0, -{{[0-9]+}}(%rsp) # sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-entry:
-  %f = alloca float, align 4
-  %d = alloca double, align 8
-  %tmp = load double, double* %d, align 8
-  %conv = fptrunc double %tmp to float
-  store float %conv, float* %f, align 4
-  ret void
-}
-
-define double @long_to_double(i64 %x) {
-; GENERIC-LABEL: long_to_double:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovq %rdi, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: long_to_double:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovq %rdi, %xmm0 # sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-   %res = bitcast i64 %x to double
-   ret double %res
-}
-
-define i64 @double_to_long(double %x) {
-; GENERIC-LABEL: double_to_long:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovq %xmm0, %rax # sched: [2:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: double_to_long:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovq %xmm0, %rax # sched: [2:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-   %res = bitcast double %x to i64
-   ret i64 %res
-}
-
-define float @int_to_float(i32 %x) {
-; GENERIC-LABEL: int_to_float:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovd %edi, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: int_to_float:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovd %edi, %xmm0 # sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-   %res = bitcast i32 %x to float
-   ret float %res
-}
-
-define i32 @float_to_int(float %x) {
-; GENERIC-LABEL: float_to_int:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovd %xmm0, %eax # sched: [2:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: float_to_int:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovd %xmm0, %eax # sched: [2:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-   %res = bitcast float %x to i32
-   ret i32 %res
-}
-
-define <16 x double> @uito16f64(<16 x i32> %a) nounwind {
-; GENERIC-LABEL: uito16f64:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vcvtudq2pd %ymm0, %zmm2 # sched: [4:1.00]
-; GENERIC-NEXT:    vextractf64x4 $1, %zmm0, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vcvtudq2pd %ymm0, %zmm1 # sched: [4:1.00]
-; GENERIC-NEXT:    vmovaps %zmm2, %zmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: uito16f64:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vcvtudq2pd %ymm0, %zmm2 # sched: [7:1.00]
-; SKX-NEXT:    vextractf64x4 $1, %zmm0, %ymm0 # sched: [3:1.00]
-; SKX-NEXT:    vcvtudq2pd %ymm0, %zmm1 # sched: [7:1.00]
-; SKX-NEXT:    vmovaps %zmm2, %zmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %b = uitofp <16 x i32> %a to <16 x double>
-  ret <16 x double> %b
-}
-
-define <8 x float> @slto8f32(<8 x i64> %a) {
-; GENERIC-LABEL: slto8f32:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vcvtqq2ps %zmm0, %ymm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: slto8f32:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vcvtqq2ps %zmm0, %ymm0 # sched: [7:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %b = sitofp <8 x i64> %a to <8 x float>
-  ret <8 x float> %b
-}
-
-define <16 x float> @slto16f32(<16 x i64> %a) {
-; GENERIC-LABEL: slto16f32:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vcvtqq2ps %zmm0, %ymm0 # sched: [3:1.00]
-; GENERIC-NEXT:    vcvtqq2ps %zmm1, %ymm1 # sched: [3:1.00]
-; GENERIC-NEXT:    vinsertf64x4 $1, %ymm1, %zmm0, %zmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: slto16f32:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vcvtqq2ps %zmm0, %ymm0 # sched: [7:1.00]
-; SKX-NEXT:    vcvtqq2ps %zmm1, %ymm1 # sched: [7:1.00]
-; SKX-NEXT:    vinsertf64x4 $1, %ymm1, %zmm0, %zmm0 # sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %b = sitofp <16 x i64> %a to <16 x float>
-  ret <16 x float> %b
-}
-
-define <8 x double> @slto8f64(<8 x i64> %a) {
-; GENERIC-LABEL: slto8f64:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vcvtqq2pd %zmm0, %zmm0 # sched: [4:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: slto8f64:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vcvtqq2pd %zmm0, %zmm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %b = sitofp <8 x i64> %a to <8 x double>
-  ret <8 x double> %b
-}
-
-define <16 x double> @slto16f64(<16 x i64> %a) {
-; GENERIC-LABEL: slto16f64:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vcvtqq2pd %zmm0, %zmm0 # sched: [4:1.00]
-; GENERIC-NEXT:    vcvtqq2pd %zmm1, %zmm1 # sched: [4:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: slto16f64:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vcvtqq2pd %zmm0, %zmm0 # sched: [4:0.50]
-; SKX-NEXT:    vcvtqq2pd %zmm1, %zmm1 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %b = sitofp <16 x i64> %a to <16 x double>
-  ret <16 x double> %b
-}
-
-define <8 x float> @ulto8f32(<8 x i64> %a) {
-; GENERIC-LABEL: ulto8f32:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vcvtuqq2ps %zmm0, %ymm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: ulto8f32:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vcvtuqq2ps %zmm0, %ymm0 # sched: [7:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %b = uitofp <8 x i64> %a to <8 x float>
-  ret <8 x float> %b
-}
-
-define <16 x float> @ulto16f32(<16 x i64> %a) {
-; GENERIC-LABEL: ulto16f32:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vcvtuqq2ps %zmm0, %ymm0 # sched: [3:1.00]
-; GENERIC-NEXT:    vcvtuqq2ps %zmm1, %ymm1 # sched: [3:1.00]
-; GENERIC-NEXT:    vinsertf64x4 $1, %ymm1, %zmm0, %zmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: ulto16f32:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vcvtuqq2ps %zmm0, %ymm0 # sched: [7:1.00]
-; SKX-NEXT:    vcvtuqq2ps %zmm1, %ymm1 # sched: [7:1.00]
-; SKX-NEXT:    vinsertf64x4 $1, %ymm1, %zmm0, %zmm0 # sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %b = uitofp <16 x i64> %a to <16 x float>
-  ret <16 x float> %b
-}
-
-define <8 x double> @uito8f64_mask(<8 x double> %a, <8 x i32> %b, i8 %c) nounwind {
-; GENERIC-LABEL: uito8f64_mask:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    kmovd %edi, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vcvtudq2pd %ymm1, %zmm0 {%k1} # sched: [4:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: uito8f64_mask:
-; SKX:       # %bb.0:
-; SKX-NEXT:    kmovd %edi, %k1 # sched: [1:1.00]
-; SKX-NEXT:    vcvtudq2pd %ymm1, %zmm0 {%k1} # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-; VLNOBW-LABEL: uito8f64_mask:
-; VLNOBW:       # %bb.0:
-; VLNOBW-NEXT:    kmovw %edi, %k1
-; VLNOBW-NEXT:    vcvtudq2pd %ymm1, %zmm0 {%k1}
-; VLNOBW-NEXT:    ret{{[l|q]}}
-  %1 = bitcast i8 %c to <8 x i1>
-  %2 = uitofp <8 x i32> %b to <8 x double>
-  %3 = select <8 x i1> %1, <8 x double> %2, <8 x double> %a
-  ret <8 x double> %3
-}
-define <8 x double> @uito8f64_maskz(<8 x i32> %a, i8 %b) nounwind {
-; GENERIC-LABEL: uito8f64_maskz:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    kmovd %edi, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vcvtudq2pd %ymm0, %zmm0 {%k1} {z} # sched: [4:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: uito8f64_maskz:
-; SKX:       # %bb.0:
-; SKX-NEXT:    kmovd %edi, %k1 # sched: [1:1.00]
-; SKX-NEXT:    vcvtudq2pd %ymm0, %zmm0 {%k1} {z} # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %1 = bitcast i8 %b to <8 x i1>
-  %2 = uitofp <8 x i32> %a to <8 x double>
-  %3 = select <8 x i1> %1, <8 x double> %2, <8 x double> zeroinitializer
-  ret <8 x double> %3
-}
-
-define <4 x double> @uito4f64(<4 x i32> %a) nounwind {
-; GENERIC-LABEL: uito4f64:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vcvtudq2pd %xmm0, %ymm0 # sched: [4:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: uito4f64:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vcvtudq2pd %xmm0, %ymm0 # sched: [7:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %b = uitofp <4 x i32> %a to <4 x double>
-  ret <4 x double> %b
-}
-
-define <16 x float> @uito16f32(<16 x i32> %a) nounwind {
-; GENERIC-LABEL: uito16f32:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vcvtudq2ps %zmm0, %zmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: uito16f32:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vcvtudq2ps %zmm0, %zmm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %b = uitofp <16 x i32> %a to <16 x float>
-  ret <16 x float> %b
-}
-
-define <8 x double> @uito8f64(<8 x i32> %a) {
-; GENERIC-LABEL: uito8f64:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vcvtudq2pd %ymm0, %zmm0 # sched: [4:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: uito8f64:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vcvtudq2pd %ymm0, %zmm0 # sched: [7:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %b = uitofp <8 x i32> %a to <8 x double>
-  ret <8 x double> %b
-}
-
-define <8 x float> @uito8f32(<8 x i32> %a) nounwind {
-; GENERIC-LABEL: uito8f32:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vcvtudq2ps %ymm0, %ymm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: uito8f32:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vcvtudq2ps %ymm0, %ymm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %b = uitofp <8 x i32> %a to <8 x float>
-  ret <8 x float> %b
-}
-
-define <4 x float> @uito4f32(<4 x i32> %a) nounwind {
-; GENERIC-LABEL: uito4f32:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vcvtudq2ps %xmm0, %xmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: uito4f32:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vcvtudq2ps %xmm0, %xmm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %b = uitofp <4 x i32> %a to <4 x float>
-  ret <4 x float> %b
-}
-
-define i32 @fptosi(float %a) nounwind {
-; GENERIC-LABEL: fptosi:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vcvttss2si %xmm0, %eax # sched: [5:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: fptosi:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vcvttss2si %xmm0, %eax # sched: [6:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %b = fptosi float %a to i32
-  ret i32 %b
-}
-
-define i32 @fptoui(float %a) nounwind {
-; GENERIC-LABEL: fptoui:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vcvttss2usi %xmm0, %eax # sched: [5:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: fptoui:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vcvttss2usi %xmm0, %eax # sched: [6:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %b = fptoui float %a to i32
-  ret i32 %b
-}
-
-define float @uitof32(i32 %a) nounwind {
-; GENERIC-LABEL: uitof32:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vcvtusi2ssl %edi, %xmm0, %xmm0 # sched: [5:2.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: uitof32:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vcvtusi2ssl %edi, %xmm0, %xmm0 # sched: [5:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %b = uitofp i32 %a to float
-  ret float %b
-}
-
-define double @uitof64(i32 %a) nounwind {
-; GENERIC-LABEL: uitof64:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vcvtusi2sdl %edi, %xmm0, %xmm0 # sched: [4:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: uitof64:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vcvtusi2sdl %edi, %xmm0, %xmm0 # sched: [5:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %b = uitofp i32 %a to double
-  ret double %b
-}
-
-define <16 x float> @sbto16f32(<16 x i32> %a) {
-; GENERIC-LABEL: sbto16f32:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpmovd2m %zmm0, %k0 # sched: [1:0.33]
-; GENERIC-NEXT:    vpmovm2d %k0, %zmm0 # sched: [1:0.33]
-; GENERIC-NEXT:    vcvtdq2ps %zmm0, %zmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: sbto16f32:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpmovd2m %zmm0, %k0 # sched: [1:1.00]
-; SKX-NEXT:    vpmovm2d %k0, %zmm0 # sched: [1:0.25]
-; SKX-NEXT:    vcvtdq2ps %zmm0, %zmm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %mask = icmp slt <16 x i32> %a, zeroinitializer
-  %1 = sitofp <16 x i1> %mask to <16 x float>
-  ret <16 x float> %1
-}
-
-define <16 x float> @scto16f32(<16 x i8> %a) {
-; GENERIC-LABEL: scto16f32:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpmovsxbd %xmm0, %zmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vcvtdq2ps %zmm0, %zmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: scto16f32:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpmovsxbd %xmm0, %zmm0 # sched: [3:1.00]
-; SKX-NEXT:    vcvtdq2ps %zmm0, %zmm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %1 = sitofp <16 x i8> %a to <16 x float>
-  ret <16 x float> %1
-}
-
-define <16 x float> @ssto16f32(<16 x i16> %a) {
-; GENERIC-LABEL: ssto16f32:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpmovsxwd %ymm0, %zmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vcvtdq2ps %zmm0, %zmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: ssto16f32:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpmovsxwd %ymm0, %zmm0 # sched: [3:1.00]
-; SKX-NEXT:    vcvtdq2ps %zmm0, %zmm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %1 = sitofp <16 x i16> %a to <16 x float>
-  ret <16 x float> %1
-}
-
-define <8 x double> @ssto16f64(<8 x i16> %a) {
-; GENERIC-LABEL: ssto16f64:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpmovsxwd %xmm0, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vcvtdq2pd %ymm0, %zmm0 # sched: [4:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: ssto16f64:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpmovsxwd %xmm0, %ymm0 # sched: [3:1.00]
-; SKX-NEXT:    vcvtdq2pd %ymm0, %zmm0 # sched: [7:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %1 = sitofp <8 x i16> %a to <8 x double>
-  ret <8 x double> %1
-}
-
-define <8 x double> @scto8f64(<8 x i8> %a) {
-; GENERIC-LABEL: scto8f64:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero sched: [1:1.00]
-; GENERIC-NEXT:    vpslld $24, %ymm0, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vpsrad $24, %ymm0, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vcvtdq2pd %ymm0, %zmm0 # sched: [4:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: scto8f64:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero sched: [3:1.00]
-; SKX-NEXT:    vpslld $24, %ymm0, %ymm0 # sched: [1:0.50]
-; SKX-NEXT:    vpsrad $24, %ymm0, %ymm0 # sched: [1:0.50]
-; SKX-NEXT:    vcvtdq2pd %ymm0, %zmm0 # sched: [7:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %1 = sitofp <8 x i8> %a to <8 x double>
-  ret <8 x double> %1
-}
-
-define <16 x double> @scto16f64(<16 x i8> %a) {
-; GENERIC-LABEL: scto16f64:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpmovsxbd %xmm0, %zmm1 # sched: [1:1.00]
-; GENERIC-NEXT:    vcvtdq2pd %ymm1, %zmm0 # sched: [4:1.00]
-; GENERIC-NEXT:    vextracti64x4 $1, %zmm1, %ymm1 # sched: [1:1.00]
-; GENERIC-NEXT:    vcvtdq2pd %ymm1, %zmm1 # sched: [4:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: scto16f64:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpmovsxbd %xmm0, %zmm1 # sched: [3:1.00]
-; SKX-NEXT:    vcvtdq2pd %ymm1, %zmm0 # sched: [7:1.00]
-; SKX-NEXT:    vextracti64x4 $1, %zmm1, %ymm1 # sched: [3:1.00]
-; SKX-NEXT:    vcvtdq2pd %ymm1, %zmm1 # sched: [7:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %b = sitofp <16 x i8> %a to <16 x double>
-  ret <16 x double> %b
-}
-
-define <16 x double> @sbto16f64(<16 x double> %a) {
-; GENERIC-LABEL: sbto16f64:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vxorpd %xmm2, %xmm2, %xmm2 # sched: [0:0.25]
-; GENERIC-NEXT:    vcmpltpd %zmm0, %zmm2, %k0 # sched: [3:1.00]
-; GENERIC-NEXT:    vcmpltpd %zmm1, %zmm2, %k1 # sched: [3:1.00]
-; GENERIC-NEXT:    kunpckbw %k0, %k1, %k0 # sched: [1:1.00]
-; GENERIC-NEXT:    vpmovm2d %k0, %zmm1 # sched: [1:0.33]
-; GENERIC-NEXT:    vcvtdq2pd %ymm1, %zmm0 # sched: [4:1.00]
-; GENERIC-NEXT:    vextracti64x4 $1, %zmm1, %ymm1 # sched: [1:1.00]
-; GENERIC-NEXT:    vcvtdq2pd %ymm1, %zmm1 # sched: [4:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: sbto16f64:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vxorpd %xmm2, %xmm2, %xmm2 # sched: [1:0.33]
-; SKX-NEXT:    vcmpltpd %zmm0, %zmm2, %k0 # sched: [3:1.00]
-; SKX-NEXT:    vcmpltpd %zmm1, %zmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    kunpckbw %k0, %k1, %k0 # sched: [3:1.00]
-; SKX-NEXT:    vpmovm2d %k0, %zmm1 # sched: [1:0.25]
-; SKX-NEXT:    vcvtdq2pd %ymm1, %zmm0 # sched: [7:1.00]
-; SKX-NEXT:    vextracti64x4 $1, %zmm1, %ymm1 # sched: [3:1.00]
-; SKX-NEXT:    vcvtdq2pd %ymm1, %zmm1 # sched: [7:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %cmpres = fcmp ogt <16 x double> %a, zeroinitializer
-  %1 = sitofp <16 x i1> %cmpres to <16 x double>
-  ret <16 x double> %1
-}
-
-define <8 x double> @sbto8f64(<8 x double> %a) {
-; GENERIC-LABEL: sbto8f64:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vxorpd %xmm1, %xmm1, %xmm1 # sched: [0:0.25]
-; GENERIC-NEXT:    vcmpltpd %zmm0, %zmm1, %k0 # sched: [3:1.00]
-; GENERIC-NEXT:    vpmovm2d %k0, %ymm0 # sched: [1:0.33]
-; GENERIC-NEXT:    vcvtdq2pd %ymm0, %zmm0 # sched: [4:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: sbto8f64:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vxorpd %xmm1, %xmm1, %xmm1 # sched: [1:0.33]
-; SKX-NEXT:    vcmpltpd %zmm0, %zmm1, %k0 # sched: [3:1.00]
-; SKX-NEXT:    vpmovm2d %k0, %ymm0 # sched: [1:0.25]
-; SKX-NEXT:    vcvtdq2pd %ymm0, %zmm0 # sched: [7:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %cmpres = fcmp ogt <8 x double> %a, zeroinitializer
-  %1 = sitofp <8 x i1> %cmpres to <8 x double>
-  ret <8 x double> %1
-}
-
-define <8 x float> @sbto8f32(<8 x float> %a) {
-; GENERIC-LABEL: sbto8f32:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vxorps %xmm1, %xmm1, %xmm1 # sched: [0:0.25]
-; GENERIC-NEXT:    vcmpltps %ymm0, %ymm1, %ymm0 # sched: [3:1.00]
-; GENERIC-NEXT:    vcvtdq2ps %ymm0, %ymm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: sbto8f32:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vxorps %xmm1, %xmm1, %xmm1 # sched: [1:0.33]
-; SKX-NEXT:    vcmpltps %ymm0, %ymm1, %ymm0 # sched: [4:0.50]
-; SKX-NEXT:    vcvtdq2ps %ymm0, %ymm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %cmpres = fcmp ogt <8 x float> %a, zeroinitializer
-  %1 = sitofp <8 x i1> %cmpres to <8 x float>
-  ret <8 x float> %1
-}
-
-define <4 x float> @sbto4f32(<4 x float> %a) {
-; GENERIC-LABEL: sbto4f32:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vxorps %xmm1, %xmm1, %xmm1 # sched: [0:0.25]
-; GENERIC-NEXT:    vcmpltps %xmm0, %xmm1, %xmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    vcvtdq2ps %xmm0, %xmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: sbto4f32:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vxorps %xmm1, %xmm1, %xmm1 # sched: [1:0.33]
-; SKX-NEXT:    vcmpltps %xmm0, %xmm1, %xmm0 # sched: [4:0.50]
-; SKX-NEXT:    vcvtdq2ps %xmm0, %xmm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %cmpres = fcmp ogt <4 x float> %a, zeroinitializer
-  %1 = sitofp <4 x i1> %cmpres to <4 x float>
-  ret <4 x float> %1
-}
-
-define <4 x double> @sbto4f64(<4 x double> %a) {
-; GENERIC-LABEL: sbto4f64:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vxorpd %xmm1, %xmm1, %xmm1 # sched: [0:0.25]
-; GENERIC-NEXT:    vcmpltpd %ymm0, %ymm1, %k0 # sched: [3:1.00]
-; GENERIC-NEXT:    vpmovm2d %k0, %xmm0 # sched: [1:0.33]
-; GENERIC-NEXT:    vcvtdq2pd %xmm0, %ymm0 # sched: [4:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: sbto4f64:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vxorpd %xmm1, %xmm1, %xmm1 # sched: [1:0.33]
-; SKX-NEXT:    vcmpltpd %ymm0, %ymm1, %k0 # sched: [3:1.00]
-; SKX-NEXT:    vpmovm2d %k0, %xmm0 # sched: [1:0.25]
-; SKX-NEXT:    vcvtdq2pd %xmm0, %ymm0 # sched: [7:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %cmpres = fcmp ogt <4 x double> %a, zeroinitializer
-  %1 = sitofp <4 x i1> %cmpres to <4 x double>
-  ret <4 x double> %1
-}
-
-define <2 x float> @sbto2f32(<2 x float> %a) {
-; GENERIC-LABEL: sbto2f32:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vxorps %xmm1, %xmm1, %xmm1 # sched: [0:0.25]
-; GENERIC-NEXT:    vcmpltps %xmm0, %xmm1, %xmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    vcvtdq2ps %xmm0, %xmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: sbto2f32:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vxorps %xmm1, %xmm1, %xmm1 # sched: [1:0.33]
-; SKX-NEXT:    vcmpltps %xmm0, %xmm1, %xmm0 # sched: [4:0.50]
-; SKX-NEXT:    vcvtdq2ps %xmm0, %xmm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %cmpres = fcmp ogt <2 x float> %a, zeroinitializer
-  %1 = sitofp <2 x i1> %cmpres to <2 x float>
-  ret <2 x float> %1
-}
-
-define <2 x double> @sbto2f64(<2 x double> %a) {
-; GENERIC-LABEL: sbto2f64:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vxorpd %xmm1, %xmm1, %xmm1 # sched: [0:0.25]
-; GENERIC-NEXT:    vcmpltpd %xmm0, %xmm1, %xmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,2,2,3] sched: [1:1.00]
-; GENERIC-NEXT:    vcvtdq2pd %xmm0, %xmm0 # sched: [4:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: sbto2f64:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vxorpd %xmm1, %xmm1, %xmm1 # sched: [1:0.33]
-; SKX-NEXT:    vcmpltpd %xmm0, %xmm1, %xmm0 # sched: [4:0.50]
-; SKX-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,2,2,3] sched: [1:1.00]
-; SKX-NEXT:    vcvtdq2pd %xmm0, %xmm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %cmpres = fcmp ogt <2 x double> %a, zeroinitializer
-  %1 = sitofp <2 x i1> %cmpres to <2 x double>
-  ret <2 x double> %1
-}
-
-define <16 x float> @ucto16f32(<16 x i8> %a) {
-; GENERIC-LABEL: ucto16f32:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero sched: [1:1.00]
-; GENERIC-NEXT:    vcvtdq2ps %zmm0, %zmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: ucto16f32:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero sched: [3:1.00]
-; SKX-NEXT:    vcvtdq2ps %zmm0, %zmm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %b = uitofp <16 x i8> %a to <16 x float>
-  ret <16 x float>%b
-}
-
-define <8 x double> @ucto8f64(<8 x i8> %a) {
-; GENERIC-LABEL: ucto8f64:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0 # sched: [7:0.50]
-; GENERIC-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero sched: [1:1.00]
-; GENERIC-NEXT:    vcvtdq2pd %ymm0, %zmm0 # sched: [4:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: ucto8f64:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0 # sched: [7:0.50]
-; SKX-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero sched: [3:1.00]
-; SKX-NEXT:    vcvtdq2pd %ymm0, %zmm0 # sched: [7:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %b = uitofp <8 x i8> %a to <8 x double>
-  ret <8 x double> %b
-}
-
-define <16 x float> @swto16f32(<16 x i16> %a) {
-; GENERIC-LABEL: swto16f32:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpmovsxwd %ymm0, %zmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vcvtdq2ps %zmm0, %zmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: swto16f32:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpmovsxwd %ymm0, %zmm0 # sched: [3:1.00]
-; SKX-NEXT:    vcvtdq2ps %zmm0, %zmm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %b = sitofp <16 x i16> %a to <16 x float>
-  ret <16 x float> %b
-}
-
-define <8 x double> @swto8f64(<8 x i16> %a) {
-; GENERIC-LABEL: swto8f64:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpmovsxwd %xmm0, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vcvtdq2pd %ymm0, %zmm0 # sched: [4:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: swto8f64:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpmovsxwd %xmm0, %ymm0 # sched: [3:1.00]
-; SKX-NEXT:    vcvtdq2pd %ymm0, %zmm0 # sched: [7:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %b = sitofp <8 x i16> %a to <8 x double>
-  ret <8 x double> %b
-}
-
-define <16 x double> @swto16f64(<16 x i16> %a) {
-; GENERIC-LABEL: swto16f64:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpmovsxwd %ymm0, %zmm1 # sched: [1:1.00]
-; GENERIC-NEXT:    vcvtdq2pd %ymm1, %zmm0 # sched: [4:1.00]
-; GENERIC-NEXT:    vextracti64x4 $1, %zmm1, %ymm1 # sched: [1:1.00]
-; GENERIC-NEXT:    vcvtdq2pd %ymm1, %zmm1 # sched: [4:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: swto16f64:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpmovsxwd %ymm0, %zmm1 # sched: [3:1.00]
-; SKX-NEXT:    vcvtdq2pd %ymm1, %zmm0 # sched: [7:1.00]
-; SKX-NEXT:    vextracti64x4 $1, %zmm1, %ymm1 # sched: [3:1.00]
-; SKX-NEXT:    vcvtdq2pd %ymm1, %zmm1 # sched: [7:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %b = sitofp <16 x i16> %a to <16 x double>
-  ret <16 x double> %b
-}
-
-define <16 x double> @ucto16f64(<16 x i8> %a) {
-; GENERIC-LABEL: ucto16f64:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpmovzxbd {{.*#+}} zmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero sched: [1:1.00]
-; GENERIC-NEXT:    vcvtdq2pd %ymm1, %zmm0 # sched: [4:1.00]
-; GENERIC-NEXT:    vextracti64x4 $1, %zmm1, %ymm1 # sched: [1:1.00]
-; GENERIC-NEXT:    vcvtdq2pd %ymm1, %zmm1 # sched: [4:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: ucto16f64:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpmovzxbd {{.*#+}} zmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero sched: [3:1.00]
-; SKX-NEXT:    vcvtdq2pd %ymm1, %zmm0 # sched: [7:1.00]
-; SKX-NEXT:    vextracti64x4 $1, %zmm1, %ymm1 # sched: [3:1.00]
-; SKX-NEXT:    vcvtdq2pd %ymm1, %zmm1 # sched: [7:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %b = uitofp <16 x i8> %a to <16 x double>
-  ret <16 x double> %b
-}
-
-define <16 x float> @uwto16f32(<16 x i16> %a) {
-; GENERIC-LABEL: uwto16f32:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero sched: [1:1.00]
-; GENERIC-NEXT:    vcvtdq2ps %zmm0, %zmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: uwto16f32:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero sched: [3:1.00]
-; SKX-NEXT:    vcvtdq2ps %zmm0, %zmm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %b = uitofp <16 x i16> %a to <16 x float>
-  ret <16 x float> %b
-}
-
-define <8 x double> @uwto8f64(<8 x i16> %a) {
-; GENERIC-LABEL: uwto8f64:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero sched: [1:1.00]
-; GENERIC-NEXT:    vcvtdq2pd %ymm0, %zmm0 # sched: [4:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: uwto8f64:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero sched: [3:1.00]
-; SKX-NEXT:    vcvtdq2pd %ymm0, %zmm0 # sched: [7:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %b = uitofp <8 x i16> %a to <8 x double>
-  ret <8 x double> %b
-}
-
-define <16 x double> @uwto16f64(<16 x i16> %a) {
-; GENERIC-LABEL: uwto16f64:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpmovzxwd {{.*#+}} zmm1 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero sched: [1:1.00]
-; GENERIC-NEXT:    vcvtdq2pd %ymm1, %zmm0 # sched: [4:1.00]
-; GENERIC-NEXT:    vextracti64x4 $1, %zmm1, %ymm1 # sched: [1:1.00]
-; GENERIC-NEXT:    vcvtdq2pd %ymm1, %zmm1 # sched: [4:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: uwto16f64:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpmovzxwd {{.*#+}} zmm1 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero sched: [3:1.00]
-; SKX-NEXT:    vcvtdq2pd %ymm1, %zmm0 # sched: [7:1.00]
-; SKX-NEXT:    vextracti64x4 $1, %zmm1, %ymm1 # sched: [3:1.00]
-; SKX-NEXT:    vcvtdq2pd %ymm1, %zmm1 # sched: [7:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %b = uitofp <16 x i16> %a to <16 x double>
-  ret <16 x double> %b
-}
-
-define <16 x float> @sito16f32(<16 x i32> %a) {
-; GENERIC-LABEL: sito16f32:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vcvtdq2ps %zmm0, %zmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: sito16f32:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vcvtdq2ps %zmm0, %zmm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %b = sitofp <16 x i32> %a to <16 x float>
-  ret <16 x float> %b
-}
-
-define <16 x double> @sito16f64(<16 x i32> %a) {
-; GENERIC-LABEL: sito16f64:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vcvtdq2pd %ymm0, %zmm2 # sched: [4:1.00]
-; GENERIC-NEXT:    vextractf64x4 $1, %zmm0, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vcvtdq2pd %ymm0, %zmm1 # sched: [4:1.00]
-; GENERIC-NEXT:    vmovaps %zmm2, %zmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: sito16f64:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vcvtdq2pd %ymm0, %zmm2 # sched: [7:1.00]
-; SKX-NEXT:    vextractf64x4 $1, %zmm0, %ymm0 # sched: [3:1.00]
-; SKX-NEXT:    vcvtdq2pd %ymm0, %zmm1 # sched: [7:1.00]
-; SKX-NEXT:    vmovaps %zmm2, %zmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %b = sitofp <16 x i32> %a to <16 x double>
-  ret <16 x double> %b
-}
-
-define <16 x float> @usto16f32(<16 x i16> %a) {
-; GENERIC-LABEL: usto16f32:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero sched: [1:1.00]
-; GENERIC-NEXT:    vcvtdq2ps %zmm0, %zmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: usto16f32:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero sched: [3:1.00]
-; SKX-NEXT:    vcvtdq2ps %zmm0, %zmm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %b = uitofp <16 x i16> %a to <16 x float>
-  ret <16 x float> %b
-}
-
-define <16 x float> @ubto16f32(<16 x i32> %a) {
-; GENERIC-LABEL: ubto16f32:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpmovd2m %zmm0, %k0 # sched: [1:0.33]
-; GENERIC-NEXT:    vpmovm2d %k0, %zmm0 # sched: [1:0.33]
-; GENERIC-NEXT:    vpsrld $31, %zmm0, %zmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vcvtdq2ps %zmm0, %zmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: ubto16f32:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpmovd2m %zmm0, %k0 # sched: [1:1.00]
-; SKX-NEXT:    vpmovm2d %k0, %zmm0 # sched: [1:0.25]
-; SKX-NEXT:    vpsrld $31, %zmm0, %zmm0 # sched: [1:1.00]
-; SKX-NEXT:    vcvtdq2ps %zmm0, %zmm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %mask = icmp slt <16 x i32> %a, zeroinitializer
-  %1 = uitofp <16 x i1> %mask to <16 x float>
-  ret <16 x float> %1
-}
-
-define <16 x double> @ubto16f64(<16 x i32> %a) {
-; GENERIC-LABEL: ubto16f64:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpmovd2m %zmm0, %k0 # sched: [1:0.33]
-; GENERIC-NEXT:    vpmovm2d %k0, %zmm0 # sched: [1:0.33]
-; GENERIC-NEXT:    vpsrld $31, %zmm0, %zmm1 # sched: [1:1.00]
-; GENERIC-NEXT:    vcvtdq2pd %ymm1, %zmm0 # sched: [4:1.00]
-; GENERIC-NEXT:    vextracti64x4 $1, %zmm1, %ymm1 # sched: [1:1.00]
-; GENERIC-NEXT:    vcvtdq2pd %ymm1, %zmm1 # sched: [4:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: ubto16f64:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpmovd2m %zmm0, %k0 # sched: [1:1.00]
-; SKX-NEXT:    vpmovm2d %k0, %zmm0 # sched: [1:0.25]
-; SKX-NEXT:    vpsrld $31, %zmm0, %zmm1 # sched: [1:1.00]
-; SKX-NEXT:    vcvtdq2pd %ymm1, %zmm0 # sched: [7:1.00]
-; SKX-NEXT:    vextracti64x4 $1, %zmm1, %ymm1 # sched: [3:1.00]
-; SKX-NEXT:    vcvtdq2pd %ymm1, %zmm1 # sched: [7:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %mask = icmp slt <16 x i32> %a, zeroinitializer
-  %1 = uitofp <16 x i1> %mask to <16 x double>
-  ret <16 x double> %1
-}
-
-define <8 x float> @ubto8f32(<8 x i32> %a) {
-; GENERIC-LABEL: ubto8f32:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpxor %xmm1, %xmm1, %xmm1 # sched: [0:0.25]
-; GENERIC-NEXT:    vpcmpgtd %ymm0, %ymm1, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    vpandd {{.*}}(%rip){1to8}, %ymm0, %ymm0 # sched: [8:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: ubto8f32:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpxor %xmm1, %xmm1, %xmm1 # sched: [1:0.33]
-; SKX-NEXT:    vpcmpgtd %ymm0, %ymm1, %ymm0 # sched: [1:0.50]
-; SKX-NEXT:    vpandd {{.*}}(%rip){1to8}, %ymm0, %ymm0 # sched: [8:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %mask = icmp slt <8 x i32> %a, zeroinitializer
-  %1 = uitofp <8 x i1> %mask to <8 x float>
-  ret <8 x float> %1
-}
-
-define <8 x double> @ubto8f64(<8 x i32> %a) {
-; GENERIC-LABEL: ubto8f64:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpxor %xmm1, %xmm1, %xmm1 # sched: [0:0.25]
-; GENERIC-NEXT:    vpcmpgtd %ymm0, %ymm1, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    vpsrld $31, %ymm0, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vcvtdq2pd %ymm0, %zmm0 # sched: [4:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: ubto8f64:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpxor %xmm1, %xmm1, %xmm1 # sched: [1:0.33]
-; SKX-NEXT:    vpcmpgtd %ymm0, %ymm1, %ymm0 # sched: [1:0.50]
-; SKX-NEXT:    vpsrld $31, %ymm0, %ymm0 # sched: [1:0.50]
-; SKX-NEXT:    vcvtdq2pd %ymm0, %zmm0 # sched: [7:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %mask = icmp slt <8 x i32> %a, zeroinitializer
-  %1 = uitofp <8 x i1> %mask to <8 x double>
-  ret <8 x double> %1
-}
-
-define <4 x float> @ubto4f32(<4 x i32> %a) {
-; GENERIC-LABEL: ubto4f32:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpxor %xmm1, %xmm1, %xmm1 # sched: [0:0.25]
-; GENERIC-NEXT:    vpcmpgtd %xmm0, %xmm1, %xmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    vpandd {{.*}}(%rip){1to4}, %xmm0, %xmm0 # sched: [7:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: ubto4f32:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpxor %xmm1, %xmm1, %xmm1 # sched: [1:0.33]
-; SKX-NEXT:    vpcmpgtd %xmm0, %xmm1, %xmm0 # sched: [1:0.50]
-; SKX-NEXT:    vpandd {{.*}}(%rip){1to4}, %xmm0, %xmm0 # sched: [7:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %mask = icmp slt <4 x i32> %a, zeroinitializer
-  %1 = uitofp <4 x i1> %mask to <4 x float>
-  ret <4 x float> %1
-}
-
-define <4 x double> @ubto4f64(<4 x i32> %a) {
-; GENERIC-LABEL: ubto4f64:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpxor %xmm1, %xmm1, %xmm1 # sched: [0:0.25]
-; GENERIC-NEXT:    vpcmpgtd %xmm0, %xmm1, %xmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    vpsrld $31, %xmm0, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vcvtdq2pd %xmm0, %ymm0 # sched: [4:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: ubto4f64:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpxor %xmm1, %xmm1, %xmm1 # sched: [1:0.33]
-; SKX-NEXT:    vpcmpgtd %xmm0, %xmm1, %xmm0 # sched: [1:0.50]
-; SKX-NEXT:    vpsrld $31, %xmm0, %xmm0 # sched: [1:0.50]
-; SKX-NEXT:    vcvtdq2pd %xmm0, %ymm0 # sched: [7:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %mask = icmp slt <4 x i32> %a, zeroinitializer
-  %1 = uitofp <4 x i1> %mask to <4 x double>
-  ret <4 x double> %1
-}
-
-define <2 x float> @ubto2f32(<2 x i32> %a) {
-; GENERIC-LABEL: ubto2f32:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpxor %xmm1, %xmm1, %xmm1 # sched: [0:0.25]
-; GENERIC-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3] sched: [1:0.50]
-; GENERIC-NEXT:    vpcmpeqq %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    vpandn {{.*}}(%rip), %xmm0, %xmm0 # sched: [7:0.50]
-; GENERIC-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: ubto2f32:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpxor %xmm1, %xmm1, %xmm1 # sched: [1:0.33]
-; SKX-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3] sched: [1:0.33]
-; SKX-NEXT:    vpcmpeqq %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SKX-NEXT:    vpandn {{.*}}(%rip), %xmm0, %xmm0 # sched: [7:0.50]
-; SKX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %mask = icmp ne <2 x i32> %a, zeroinitializer
-  %1 = uitofp <2 x i1> %mask to <2 x float>
-  ret <2 x float> %1
-}
-
-define <2 x double> @ubto2f64(<2 x i32> %a) {
-; GENERIC-LABEL: ubto2f64:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpxor %xmm1, %xmm1, %xmm1 # sched: [0:0.25]
-; GENERIC-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3] sched: [1:0.50]
-; GENERIC-NEXT:    vpcmpeqq %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    vpandn {{.*}}(%rip), %xmm0, %xmm0 # sched: [7:0.50]
-; GENERIC-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] sched: [1:0.50]
-; GENERIC-NEXT:    vcvtdq2pd %xmm0, %xmm0 # sched: [4:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: ubto2f64:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpxor %xmm1, %xmm1, %xmm1 # sched: [1:0.33]
-; SKX-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3] sched: [1:0.33]
-; SKX-NEXT:    vpcmpeqq %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SKX-NEXT:    vpandn {{.*}}(%rip), %xmm0, %xmm0 # sched: [7:0.50]
-; SKX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] sched: [1:1.00]
-; SKX-NEXT:    vcvtdq2pd %xmm0, %xmm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %mask = icmp ne <2 x i32> %a, zeroinitializer
-  %1 = uitofp <2 x i1> %mask to <2 x double>
-  ret <2 x double> %1
-}
-
-define <8 x i16> @zext_8x8mem_to_8x16(<8 x i8> *%i , <8 x i1> %mask) nounwind readnone {
-; GENERIC-LABEL: zext_8x8mem_to_8x16:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpsllw $15, %xmm0, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vpmovw2m %xmm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpmovzxbw {{.*#+}} xmm0 {%k1} {z} = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: zext_8x8mem_to_8x16:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpsllw $15, %xmm0, %xmm0 # sched: [1:0.50]
-; SKX-NEXT:    vpmovw2m %xmm0, %k1 # sched: [1:1.00]
-; SKX-NEXT:    vpmovzxbw {{.*#+}} xmm0 {%k1} {z} = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero sched: [9:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %a   = load <8 x i8>,<8 x i8> *%i,align 1
-  %x   = zext <8 x i8> %a to <8 x i16>
-  %ret = select <8 x i1> %mask, <8 x i16> %x, <8 x i16> zeroinitializer
-  ret <8 x i16> %ret
-}
-
-define <8 x i16> @sext_8x8mem_to_8x16(<8 x i8> *%i , <8 x i1> %mask) nounwind readnone {
-; GENERIC-LABEL: sext_8x8mem_to_8x16:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpsllw $15, %xmm0, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vpmovw2m %xmm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpmovsxbw (%rdi), %xmm0 {%k1} {z} # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: sext_8x8mem_to_8x16:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpsllw $15, %xmm0, %xmm0 # sched: [1:0.50]
-; SKX-NEXT:    vpmovw2m %xmm0, %k1 # sched: [1:1.00]
-; SKX-NEXT:    vpmovsxbw (%rdi), %xmm0 {%k1} {z} # sched: [9:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %a   = load <8 x i8>,<8 x i8> *%i,align 1
-  %x   = sext <8 x i8> %a to <8 x i16>
-  %ret = select <8 x i1> %mask, <8 x i16> %x, <8 x i16> zeroinitializer
-  ret <8 x i16> %ret
-}
-
-
-define <16 x i16> @zext_16x8mem_to_16x16(<16 x i8> *%i , <16 x i1> %mask) nounwind readnone {
-; GENERIC-LABEL: zext_16x8mem_to_16x16:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpsllw $7, %xmm0, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vpmovb2m %xmm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpmovzxbw {{.*#+}} ymm0 {%k1} {z} = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero,mem[8],zero,mem[9],zero,mem[10],zero,mem[11],zero,mem[12],zero,mem[13],zero,mem[14],zero,mem[15],zero sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: zext_16x8mem_to_16x16:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpsllw $7, %xmm0, %xmm0 # sched: [1:0.50]
-; SKX-NEXT:    vpmovb2m %xmm0, %k1 # sched: [1:1.00]
-; SKX-NEXT:    vpmovzxbw {{.*#+}} ymm0 {%k1} {z} = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero,mem[8],zero,mem[9],zero,mem[10],zero,mem[11],zero,mem[12],zero,mem[13],zero,mem[14],zero,mem[15],zero sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %a   = load <16 x i8>,<16 x i8> *%i,align 1
-  %x   = zext <16 x i8> %a to <16 x i16>
-  %ret = select <16 x i1> %mask, <16 x i16> %x, <16 x i16> zeroinitializer
-  ret <16 x i16> %ret
-}
-
-define <16 x i16> @sext_16x8mem_to_16x16(<16 x i8> *%i , <16 x i1> %mask) nounwind readnone {
-; GENERIC-LABEL: sext_16x8mem_to_16x16:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpsllw $7, %xmm0, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vpmovb2m %xmm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpmovsxbw (%rdi), %ymm0 {%k1} {z} # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: sext_16x8mem_to_16x16:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpsllw $7, %xmm0, %xmm0 # sched: [1:0.50]
-; SKX-NEXT:    vpmovb2m %xmm0, %k1 # sched: [1:1.00]
-; SKX-NEXT:    vpmovsxbw (%rdi), %ymm0 {%k1} {z} # sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %a   = load <16 x i8>,<16 x i8> *%i,align 1
-  %x   = sext <16 x i8> %a to <16 x i16>
-  %ret = select <16 x i1> %mask, <16 x i16> %x, <16 x i16> zeroinitializer
-  ret <16 x i16> %ret
-}
-
-define <16 x i16> @zext_16x8_to_16x16(<16 x i8> %a ) nounwind readnone {
-; GENERIC-LABEL: zext_16x8_to_16x16:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: zext_16x8_to_16x16:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %x   = zext <16 x i8> %a to <16 x i16>
-  ret <16 x i16> %x
-}
-
-define <16 x i16> @zext_16x8_to_16x16_mask(<16 x i8> %a ,<16 x i1> %mask) nounwind readnone {
-; GENERIC-LABEL: zext_16x8_to_16x16_mask:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpsllw $7, %xmm1, %xmm1 # sched: [1:1.00]
-; GENERIC-NEXT:    vpmovb2m %xmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpmovzxbw {{.*#+}} ymm0 {%k1} {z} = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: zext_16x8_to_16x16_mask:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpsllw $7, %xmm1, %xmm1 # sched: [1:0.50]
-; SKX-NEXT:    vpmovb2m %xmm1, %k1 # sched: [1:1.00]
-; SKX-NEXT:    vpmovzxbw {{.*#+}} ymm0 {%k1} {z} = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %x   = zext <16 x i8> %a to <16 x i16>
-  %ret = select <16 x i1> %mask, <16 x i16> %x, <16 x i16> zeroinitializer
-  ret <16 x i16> %ret
-}
-
-define <16 x i16> @sext_16x8_to_16x16(<16 x i8> %a ) nounwind readnone {
-; GENERIC-LABEL: sext_16x8_to_16x16:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpmovsxbw %xmm0, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: sext_16x8_to_16x16:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpmovsxbw %xmm0, %ymm0 # sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %x   = sext <16 x i8> %a to <16 x i16>
-  ret <16 x i16> %x
-}
-
-define <16 x i16> @sext_16x8_to_16x16_mask(<16 x i8> %a ,<16 x i1> %mask) nounwind readnone {
-; GENERIC-LABEL: sext_16x8_to_16x16_mask:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpsllw $7, %xmm1, %xmm1 # sched: [1:1.00]
-; GENERIC-NEXT:    vpmovb2m %xmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpmovsxbw %xmm0, %ymm0 {%k1} {z} # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: sext_16x8_to_16x16_mask:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpsllw $7, %xmm1, %xmm1 # sched: [1:0.50]
-; SKX-NEXT:    vpmovb2m %xmm1, %k1 # sched: [1:1.00]
-; SKX-NEXT:    vpmovsxbw %xmm0, %ymm0 {%k1} {z} # sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %x   = sext <16 x i8> %a to <16 x i16>
-  %ret = select <16 x i1> %mask, <16 x i16> %x, <16 x i16> zeroinitializer
-  ret <16 x i16> %ret
-}
-
-define <32 x i16> @zext_32x8mem_to_32x16(<32 x i8> *%i , <32 x i1> %mask) nounwind readnone {
-; GENERIC-LABEL: zext_32x8mem_to_32x16:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpsllw $7, %ymm0, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vpmovb2m %ymm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpmovzxbw {{.*#+}} zmm0 {%k1} {z} = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero,mem[8],zero,mem[9],zero,mem[10],zero,mem[11],zero,mem[12],zero,mem[13],zero,mem[14],zero,mem[15],zero,mem[16],zero,mem[17],zero,mem[18],zero,mem[19],zero,mem[20],zero,mem[21],zero,mem[22],zero,mem[23],zero,mem[24],zero,mem[25],zero,mem[26],zero,mem[27],zero,mem[28],zero,mem[29],zero,mem[30],zero,mem[31],zero sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: zext_32x8mem_to_32x16:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpsllw $7, %ymm0, %ymm0 # sched: [1:0.50]
-; SKX-NEXT:    vpmovb2m %ymm0, %k1 # sched: [1:1.00]
-; SKX-NEXT:    vpmovzxbw {{.*#+}} zmm0 {%k1} {z} = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero,mem[8],zero,mem[9],zero,mem[10],zero,mem[11],zero,mem[12],zero,mem[13],zero,mem[14],zero,mem[15],zero,mem[16],zero,mem[17],zero,mem[18],zero,mem[19],zero,mem[20],zero,mem[21],zero,mem[22],zero,mem[23],zero,mem[24],zero,mem[25],zero,mem[26],zero,mem[27],zero,mem[28],zero,mem[29],zero,mem[30],zero,mem[31],zero sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %a   = load <32 x i8>,<32 x i8> *%i,align 1
-  %x   = zext <32 x i8> %a to <32 x i16>
-  %ret = select <32 x i1> %mask, <32 x i16> %x, <32 x i16> zeroinitializer
-  ret <32 x i16> %ret
-}
-
-define <32 x i16> @sext_32x8mem_to_32x16(<32 x i8> *%i , <32 x i1> %mask) nounwind readnone {
-; GENERIC-LABEL: sext_32x8mem_to_32x16:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpsllw $7, %ymm0, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vpmovb2m %ymm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpmovsxbw (%rdi), %zmm0 {%k1} {z} # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: sext_32x8mem_to_32x16:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpsllw $7, %ymm0, %ymm0 # sched: [1:0.50]
-; SKX-NEXT:    vpmovb2m %ymm0, %k1 # sched: [1:1.00]
-; SKX-NEXT:    vpmovsxbw (%rdi), %zmm0 {%k1} {z} # sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %a   = load <32 x i8>,<32 x i8> *%i,align 1
-  %x   = sext <32 x i8> %a to <32 x i16>
-  %ret = select <32 x i1> %mask, <32 x i16> %x, <32 x i16> zeroinitializer
-  ret <32 x i16> %ret
-}
-
-define <32 x i16> @zext_32x8_to_32x16(<32 x i8> %a ) nounwind readnone {
-; GENERIC-LABEL: zext_32x8_to_32x16:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpmovzxbw {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero,ymm0[16],zero,ymm0[17],zero,ymm0[18],zero,ymm0[19],zero,ymm0[20],zero,ymm0[21],zero,ymm0[22],zero,ymm0[23],zero,ymm0[24],zero,ymm0[25],zero,ymm0[26],zero,ymm0[27],zero,ymm0[28],zero,ymm0[29],zero,ymm0[30],zero,ymm0[31],zero sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: zext_32x8_to_32x16:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpmovzxbw {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero,ymm0[16],zero,ymm0[17],zero,ymm0[18],zero,ymm0[19],zero,ymm0[20],zero,ymm0[21],zero,ymm0[22],zero,ymm0[23],zero,ymm0[24],zero,ymm0[25],zero,ymm0[26],zero,ymm0[27],zero,ymm0[28],zero,ymm0[29],zero,ymm0[30],zero,ymm0[31],zero sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %x   = zext <32 x i8> %a to <32 x i16>
-  ret <32 x i16> %x
-}
-
-define <32 x i16> @zext_32x8_to_32x16_mask(<32 x i8> %a ,<32 x i1> %mask) nounwind readnone {
-; GENERIC-LABEL: zext_32x8_to_32x16_mask:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpsllw $7, %ymm1, %ymm1 # sched: [1:1.00]
-; GENERIC-NEXT:    vpmovb2m %ymm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpmovzxbw {{.*#+}} zmm0 {%k1} {z} = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero,ymm0[16],zero,ymm0[17],zero,ymm0[18],zero,ymm0[19],zero,ymm0[20],zero,ymm0[21],zero,ymm0[22],zero,ymm0[23],zero,ymm0[24],zero,ymm0[25],zero,ymm0[26],zero,ymm0[27],zero,ymm0[28],zero,ymm0[29],zero,ymm0[30],zero,ymm0[31],zero sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: zext_32x8_to_32x16_mask:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpsllw $7, %ymm1, %ymm1 # sched: [1:0.50]
-; SKX-NEXT:    vpmovb2m %ymm1, %k1 # sched: [1:1.00]
-; SKX-NEXT:    vpmovzxbw {{.*#+}} zmm0 {%k1} {z} = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero,ymm0[16],zero,ymm0[17],zero,ymm0[18],zero,ymm0[19],zero,ymm0[20],zero,ymm0[21],zero,ymm0[22],zero,ymm0[23],zero,ymm0[24],zero,ymm0[25],zero,ymm0[26],zero,ymm0[27],zero,ymm0[28],zero,ymm0[29],zero,ymm0[30],zero,ymm0[31],zero sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %x   = zext <32 x i8> %a to <32 x i16>
-  %ret = select <32 x i1> %mask, <32 x i16> %x, <32 x i16> zeroinitializer
-  ret <32 x i16> %ret
-}
-
-define <32 x i16> @sext_32x8_to_32x16(<32 x i8> %a ) nounwind readnone {
-; GENERIC-LABEL: sext_32x8_to_32x16:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpmovsxbw %ymm0, %zmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: sext_32x8_to_32x16:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpmovsxbw %ymm0, %zmm0 # sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %x   = sext <32 x i8> %a to <32 x i16>
-  ret <32 x i16> %x
-}
-
-define <32 x i16> @sext_32x8_to_32x16_mask(<32 x i8> %a ,<32 x i1> %mask) nounwind readnone {
-; GENERIC-LABEL: sext_32x8_to_32x16_mask:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpsllw $7, %ymm1, %ymm1 # sched: [1:1.00]
-; GENERIC-NEXT:    vpmovb2m %ymm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpmovsxbw %ymm0, %zmm0 {%k1} {z} # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: sext_32x8_to_32x16_mask:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpsllw $7, %ymm1, %ymm1 # sched: [1:0.50]
-; SKX-NEXT:    vpmovb2m %ymm1, %k1 # sched: [1:1.00]
-; SKX-NEXT:    vpmovsxbw %ymm0, %zmm0 {%k1} {z} # sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %x   = sext <32 x i8> %a to <32 x i16>
-  %ret = select <32 x i1> %mask, <32 x i16> %x, <32 x i16> zeroinitializer
-  ret <32 x i16> %ret
-}
-
-define <4 x i32> @zext_4x8mem_to_4x32(<4 x i8> *%i , <4 x i1> %mask) nounwind readnone {
-; GENERIC-LABEL: zext_4x8mem_to_4x32:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpslld $31, %xmm0, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vpmovd2m %xmm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpmovzxbd {{.*#+}} xmm0 {%k1} {z} = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: zext_4x8mem_to_4x32:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpslld $31, %xmm0, %xmm0 # sched: [1:0.50]
-; SKX-NEXT:    vpmovd2m %xmm0, %k1 # sched: [1:1.00]
-; SKX-NEXT:    vpmovzxbd {{.*#+}} xmm0 {%k1} {z} = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero sched: [9:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %a   = load <4 x i8>,<4 x i8> *%i,align 1
-  %x   = zext <4 x i8> %a to <4 x i32>
-  %ret = select <4 x i1> %mask, <4 x i32> %x, <4 x i32> zeroinitializer
-  ret <4 x i32> %ret
-}
-
-define <4 x i32> @sext_4x8mem_to_4x32(<4 x i8> *%i , <4 x i1> %mask) nounwind readnone {
-; GENERIC-LABEL: sext_4x8mem_to_4x32:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpslld $31, %xmm0, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vpmovd2m %xmm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpmovsxbd (%rdi), %xmm0 {%k1} {z} # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: sext_4x8mem_to_4x32:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpslld $31, %xmm0, %xmm0 # sched: [1:0.50]
-; SKX-NEXT:    vpmovd2m %xmm0, %k1 # sched: [1:1.00]
-; SKX-NEXT:    vpmovsxbd (%rdi), %xmm0 {%k1} {z} # sched: [9:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %a   = load <4 x i8>,<4 x i8> *%i,align 1
-  %x   = sext <4 x i8> %a to <4 x i32>
-  %ret = select <4 x i1> %mask, <4 x i32> %x, <4 x i32> zeroinitializer
-  ret <4 x i32> %ret
-}
-
-define <8 x i32> @zext_8x8mem_to_8x32(<8 x i8> *%i , <8 x i1> %mask) nounwind readnone {
-; GENERIC-LABEL: zext_8x8mem_to_8x32:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpsllw $15, %xmm0, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vpmovw2m %xmm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpmovzxbd {{.*#+}} ymm0 {%k1} {z} = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: zext_8x8mem_to_8x32:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpsllw $15, %xmm0, %xmm0 # sched: [1:0.50]
-; SKX-NEXT:    vpmovw2m %xmm0, %k1 # sched: [1:1.00]
-; SKX-NEXT:    vpmovzxbd {{.*#+}} ymm0 {%k1} {z} = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %a   = load <8 x i8>,<8 x i8> *%i,align 1
-  %x   = zext <8 x i8> %a to <8 x i32>
-  %ret = select <8 x i1> %mask, <8 x i32> %x, <8 x i32> zeroinitializer
-  ret <8 x i32> %ret
-}
-
-define <8 x i32> @sext_8x8mem_to_8x32(<8 x i8> *%i , <8 x i1> %mask) nounwind readnone {
-; GENERIC-LABEL: sext_8x8mem_to_8x32:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpsllw $15, %xmm0, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vpmovw2m %xmm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpmovsxbd (%rdi), %ymm0 {%k1} {z} # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: sext_8x8mem_to_8x32:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpsllw $15, %xmm0, %xmm0 # sched: [1:0.50]
-; SKX-NEXT:    vpmovw2m %xmm0, %k1 # sched: [1:1.00]
-; SKX-NEXT:    vpmovsxbd (%rdi), %ymm0 {%k1} {z} # sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %a   = load <8 x i8>,<8 x i8> *%i,align 1
-  %x   = sext <8 x i8> %a to <8 x i32>
-  %ret = select <8 x i1> %mask, <8 x i32> %x, <8 x i32> zeroinitializer
-  ret <8 x i32> %ret
-}
-
-define <16 x i32> @zext_16x8mem_to_16x32(<16 x i8> *%i , <16 x i1> %mask) nounwind readnone {
-; GENERIC-LABEL: zext_16x8mem_to_16x32:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpsllw $7, %xmm0, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vpmovb2m %xmm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpmovzxbd {{.*#+}} zmm0 {%k1} {z} = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero,mem[8],zero,zero,zero,mem[9],zero,zero,zero,mem[10],zero,zero,zero,mem[11],zero,zero,zero,mem[12],zero,zero,zero,mem[13],zero,zero,zero,mem[14],zero,zero,zero,mem[15],zero,zero,zero sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: zext_16x8mem_to_16x32:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpsllw $7, %xmm0, %xmm0 # sched: [1:0.50]
-; SKX-NEXT:    vpmovb2m %xmm0, %k1 # sched: [1:1.00]
-; SKX-NEXT:    vpmovzxbd {{.*#+}} zmm0 {%k1} {z} = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero,mem[8],zero,zero,zero,mem[9],zero,zero,zero,mem[10],zero,zero,zero,mem[11],zero,zero,zero,mem[12],zero,zero,zero,mem[13],zero,zero,zero,mem[14],zero,zero,zero,mem[15],zero,zero,zero sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %a   = load <16 x i8>,<16 x i8> *%i,align 1
-  %x   = zext <16 x i8> %a to <16 x i32>
-  %ret = select <16 x i1> %mask, <16 x i32> %x, <16 x i32> zeroinitializer
-  ret <16 x i32> %ret
-}
-
-define <16 x i32> @sext_16x8mem_to_16x32(<16 x i8> *%i , <16 x i1> %mask) nounwind readnone {
-; GENERIC-LABEL: sext_16x8mem_to_16x32:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpsllw $7, %xmm0, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vpmovb2m %xmm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpmovsxbd (%rdi), %zmm0 {%k1} {z} # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: sext_16x8mem_to_16x32:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpsllw $7, %xmm0, %xmm0 # sched: [1:0.50]
-; SKX-NEXT:    vpmovb2m %xmm0, %k1 # sched: [1:1.00]
-; SKX-NEXT:    vpmovsxbd (%rdi), %zmm0 {%k1} {z} # sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %a   = load <16 x i8>,<16 x i8> *%i,align 1
-  %x   = sext <16 x i8> %a to <16 x i32>
-  %ret = select <16 x i1> %mask, <16 x i32> %x, <16 x i32> zeroinitializer
-  ret <16 x i32> %ret
-}
-
-define <16 x i32> @zext_16x8_to_16x32_mask(<16 x i8> %a , <16 x i1> %mask) nounwind readnone {
-; GENERIC-LABEL: zext_16x8_to_16x32_mask:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpsllw $7, %xmm1, %xmm1 # sched: [1:1.00]
-; GENERIC-NEXT:    vpmovb2m %xmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpmovzxbd {{.*#+}} zmm0 {%k1} {z} = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: zext_16x8_to_16x32_mask:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpsllw $7, %xmm1, %xmm1 # sched: [1:0.50]
-; SKX-NEXT:    vpmovb2m %xmm1, %k1 # sched: [1:1.00]
-; SKX-NEXT:    vpmovzxbd {{.*#+}} zmm0 {%k1} {z} = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %x   = zext <16 x i8> %a to <16 x i32>
-  %ret = select <16 x i1> %mask, <16 x i32> %x, <16 x i32> zeroinitializer
-  ret <16 x i32> %ret
-}
-
-define <16 x i32> @sext_16x8_to_16x32_mask(<16 x i8> %a , <16 x i1> %mask) nounwind readnone {
-; GENERIC-LABEL: sext_16x8_to_16x32_mask:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpsllw $7, %xmm1, %xmm1 # sched: [1:1.00]
-; GENERIC-NEXT:    vpmovb2m %xmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpmovsxbd %xmm0, %zmm0 {%k1} {z} # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: sext_16x8_to_16x32_mask:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpsllw $7, %xmm1, %xmm1 # sched: [1:0.50]
-; SKX-NEXT:    vpmovb2m %xmm1, %k1 # sched: [1:1.00]
-; SKX-NEXT:    vpmovsxbd %xmm0, %zmm0 {%k1} {z} # sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %x   = sext <16 x i8> %a to <16 x i32>
-  %ret = select <16 x i1> %mask, <16 x i32> %x, <16 x i32> zeroinitializer
-  ret <16 x i32> %ret
-}
-
-define <16 x i32> @zext_16x8_to_16x32(<16 x i8> %i) nounwind readnone {
-; GENERIC-LABEL: zext_16x8_to_16x32:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: zext_16x8_to_16x32:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %x = zext <16 x i8> %i to <16 x i32>
-  ret <16 x i32> %x
-}
-
-define <16 x i32> @sext_16x8_to_16x32(<16 x i8> %i) nounwind readnone {
-; GENERIC-LABEL: sext_16x8_to_16x32:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpmovsxbd %xmm0, %zmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: sext_16x8_to_16x32:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpmovsxbd %xmm0, %zmm0 # sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %x = sext <16 x i8> %i to <16 x i32>
-  ret <16 x i32> %x
-}
-
-define <2 x i64> @zext_2x8mem_to_2x64(<2 x i8> *%i , <2 x i1> %mask) nounwind readnone {
-; GENERIC-LABEL: zext_2x8mem_to_2x64:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpsllq $63, %xmm0, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vpmovq2m %xmm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpmovzxbq {{.*#+}} xmm0 {%k1} {z} = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: zext_2x8mem_to_2x64:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpsllq $63, %xmm0, %xmm0 # sched: [1:0.50]
-; SKX-NEXT:    vpmovq2m %xmm0, %k1 # sched: [1:1.00]
-; SKX-NEXT:    vpmovzxbq {{.*#+}} xmm0 {%k1} {z} = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero sched: [9:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %a   = load <2 x i8>,<2 x i8> *%i,align 1
-  %x   = zext <2 x i8> %a to <2 x i64>
-  %ret = select <2 x  i1> %mask, <2 x i64> %x, <2 x i64> zeroinitializer
-  ret <2 x i64> %ret
-}
-define <2 x i64> @sext_2x8mem_to_2x64mask(<2 x i8> *%i , <2 x i1> %mask) nounwind readnone {
-; GENERIC-LABEL: sext_2x8mem_to_2x64mask:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpsllq $63, %xmm0, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vpmovq2m %xmm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpmovsxbq (%rdi), %xmm0 {%k1} {z} # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: sext_2x8mem_to_2x64mask:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpsllq $63, %xmm0, %xmm0 # sched: [1:0.50]
-; SKX-NEXT:    vpmovq2m %xmm0, %k1 # sched: [1:1.00]
-; SKX-NEXT:    vpmovsxbq (%rdi), %xmm0 {%k1} {z} # sched: [9:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %a   = load <2 x i8>,<2 x i8> *%i,align 1
-  %x   = sext <2 x i8> %a to <2 x i64>
-  %ret = select <2 x i1> %mask, <2 x i64> %x, <2 x i64> zeroinitializer
-  ret <2 x i64> %ret
-}
-define <2 x i64> @sext_2x8mem_to_2x64(<2 x i8> *%i) nounwind readnone {
-; GENERIC-LABEL: sext_2x8mem_to_2x64:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpmovsxbq (%rdi), %xmm0 # sched: [7:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: sext_2x8mem_to_2x64:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpmovsxbq (%rdi), %xmm0 # sched: [6:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %a   = load <2 x i8>,<2 x i8> *%i,align 1
-  %x   = sext <2 x i8> %a to <2 x i64>
-  ret <2 x i64> %x
-}
-
-define <4 x i64> @zext_4x8mem_to_4x64(<4 x i8> *%i , <4 x i1> %mask) nounwind readnone {
-; GENERIC-LABEL: zext_4x8mem_to_4x64:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpslld $31, %xmm0, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vpmovd2m %xmm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpmovzxbq {{.*#+}} ymm0 {%k1} {z} = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero,mem[2],zero,zero,zero,zero,zero,zero,zero,mem[3],zero,zero,zero,zero,zero,zero,zero sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: zext_4x8mem_to_4x64:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpslld $31, %xmm0, %xmm0 # sched: [1:0.50]
-; SKX-NEXT:    vpmovd2m %xmm0, %k1 # sched: [1:1.00]
-; SKX-NEXT:    vpmovzxbq {{.*#+}} ymm0 {%k1} {z} = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero,mem[2],zero,zero,zero,zero,zero,zero,zero,mem[3],zero,zero,zero,zero,zero,zero,zero sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %a   = load <4 x i8>,<4 x i8> *%i,align 1
-  %x   = zext <4 x i8> %a to <4 x i64>
-  %ret = select <4 x  i1> %mask, <4 x i64> %x, <4 x i64> zeroinitializer
-  ret <4 x i64> %ret
-}
-
-define <4 x i64> @sext_4x8mem_to_4x64mask(<4 x i8> *%i , <4 x i1> %mask) nounwind readnone {
-; GENERIC-LABEL: sext_4x8mem_to_4x64mask:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpslld $31, %xmm0, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vpmovd2m %xmm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpmovsxbq (%rdi), %ymm0 {%k1} {z} # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: sext_4x8mem_to_4x64mask:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpslld $31, %xmm0, %xmm0 # sched: [1:0.50]
-; SKX-NEXT:    vpmovd2m %xmm0, %k1 # sched: [1:1.00]
-; SKX-NEXT:    vpmovsxbq (%rdi), %ymm0 {%k1} {z} # sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %a   = load <4 x i8>,<4 x i8> *%i,align 1
-  %x   = sext <4 x i8> %a to <4 x i64>
-  %ret = select <4 x i1> %mask, <4 x i64> %x, <4 x i64> zeroinitializer
-  ret <4 x i64> %ret
-}
-
-define <4 x i64> @sext_4x8mem_to_4x64(<4 x i8> *%i) nounwind readnone {
-; GENERIC-LABEL: sext_4x8mem_to_4x64:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpmovsxbq (%rdi), %ymm0 # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: sext_4x8mem_to_4x64:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpmovsxbq (%rdi), %ymm0 # sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %a   = load <4 x i8>,<4 x i8> *%i,align 1
-  %x   = sext <4 x i8> %a to <4 x i64>
-  ret <4 x i64> %x
-}
-
-define <8 x i64> @zext_8x8mem_to_8x64(<8 x i8> *%i , <8 x i1> %mask) nounwind readnone {
-; GENERIC-LABEL: zext_8x8mem_to_8x64:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpsllw $15, %xmm0, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vpmovw2m %xmm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpmovzxbq {{.*#+}} zmm0 {%k1} {z} = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero,mem[2],zero,zero,zero,zero,zero,zero,zero,mem[3],zero,zero,zero,zero,zero,zero,zero,mem[4],zero,zero,zero,zero,zero,zero,zero,mem[5],zero,zero,zero,zero,zero,zero,zero,mem[6],zero,zero,zero,zero,zero,zero,zero,mem[7],zero,zero,zero,zero,zero,zero,zero sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: zext_8x8mem_to_8x64:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpsllw $15, %xmm0, %xmm0 # sched: [1:0.50]
-; SKX-NEXT:    vpmovw2m %xmm0, %k1 # sched: [1:1.00]
-; SKX-NEXT:    vpmovzxbq {{.*#+}} zmm0 {%k1} {z} = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero,mem[2],zero,zero,zero,zero,zero,zero,zero,mem[3],zero,zero,zero,zero,zero,zero,zero,mem[4],zero,zero,zero,zero,zero,zero,zero,mem[5],zero,zero,zero,zero,zero,zero,zero,mem[6],zero,zero,zero,zero,zero,zero,zero,mem[7],zero,zero,zero,zero,zero,zero,zero sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %a   = load <8 x i8>,<8 x i8> *%i,align 1
-  %x   = zext <8 x i8> %a to <8 x i64>
-  %ret = select <8 x  i1> %mask, <8 x i64> %x, <8 x i64> zeroinitializer
-  ret <8 x i64> %ret
-}
-
-define <8 x i64> @sext_8x8mem_to_8x64mask(<8 x i8> *%i , <8 x i1> %mask) nounwind readnone {
-; GENERIC-LABEL: sext_8x8mem_to_8x64mask:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpsllw $15, %xmm0, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vpmovw2m %xmm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpmovsxbq (%rdi), %zmm0 {%k1} {z} # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: sext_8x8mem_to_8x64mask:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpsllw $15, %xmm0, %xmm0 # sched: [1:0.50]
-; SKX-NEXT:    vpmovw2m %xmm0, %k1 # sched: [1:1.00]
-; SKX-NEXT:    vpmovsxbq (%rdi), %zmm0 {%k1} {z} # sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %a   = load <8 x i8>,<8 x i8> *%i,align 1
-  %x   = sext <8 x i8> %a to <8 x i64>
-  %ret = select <8 x i1> %mask, <8 x i64> %x, <8 x i64> zeroinitializer
-  ret <8 x i64> %ret
-}
-
-define <8 x i64> @sext_8x8mem_to_8x64(<8 x i8> *%i) nounwind readnone {
-; GENERIC-LABEL: sext_8x8mem_to_8x64:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpmovsxbq (%rdi), %zmm0 # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: sext_8x8mem_to_8x64:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpmovsxbq (%rdi), %zmm0 # sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %a   = load <8 x i8>,<8 x i8> *%i,align 1
-  %x   = sext <8 x i8> %a to <8 x i64>
-  ret <8 x i64> %x
-}
-
-define <4 x i32> @zext_4x16mem_to_4x32(<4 x i16> *%i , <4 x i1> %mask) nounwind readnone {
-; GENERIC-LABEL: zext_4x16mem_to_4x32:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpslld $31, %xmm0, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vpmovd2m %xmm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpmovzxwd {{.*#+}} xmm0 {%k1} {z} = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: zext_4x16mem_to_4x32:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpslld $31, %xmm0, %xmm0 # sched: [1:0.50]
-; SKX-NEXT:    vpmovd2m %xmm0, %k1 # sched: [1:1.00]
-; SKX-NEXT:    vpmovzxwd {{.*#+}} xmm0 {%k1} {z} = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero sched: [9:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %a   = load <4 x i16>,<4 x i16> *%i,align 1
-  %x   = zext <4 x i16> %a to <4 x i32>
-  %ret = select <4 x i1> %mask, <4 x i32> %x, <4 x i32> zeroinitializer
-  ret <4 x i32> %ret
-}
-
-define <4 x i32> @sext_4x16mem_to_4x32mask(<4 x i16> *%i , <4 x i1> %mask) nounwind readnone {
-; GENERIC-LABEL: sext_4x16mem_to_4x32mask:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpslld $31, %xmm0, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vpmovd2m %xmm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpmovsxwd (%rdi), %xmm0 {%k1} {z} # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: sext_4x16mem_to_4x32mask:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpslld $31, %xmm0, %xmm0 # sched: [1:0.50]
-; SKX-NEXT:    vpmovd2m %xmm0, %k1 # sched: [1:1.00]
-; SKX-NEXT:    vpmovsxwd (%rdi), %xmm0 {%k1} {z} # sched: [9:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %a   = load <4 x i16>,<4 x i16> *%i,align 1
-  %x   = sext <4 x i16> %a to <4 x i32>
-  %ret = select <4 x i1> %mask, <4 x i32> %x, <4 x i32> zeroinitializer
-  ret <4 x i32> %ret
-}
-
-define <4 x i32> @sext_4x16mem_to_4x32(<4 x i16> *%i) nounwind readnone {
-; GENERIC-LABEL: sext_4x16mem_to_4x32:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpmovsxwd (%rdi), %xmm0 # sched: [7:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: sext_4x16mem_to_4x32:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpmovsxwd (%rdi), %xmm0 # sched: [6:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %a   = load <4 x i16>,<4 x i16> *%i,align 1
-  %x   = sext <4 x i16> %a to <4 x i32>
-  ret <4 x i32> %x
-}
-
-
-define <8 x i32> @zext_8x16mem_to_8x32(<8 x i16> *%i , <8 x i1> %mask) nounwind readnone {
-; GENERIC-LABEL: zext_8x16mem_to_8x32:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpsllw $15, %xmm0, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vpmovw2m %xmm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpmovzxwd {{.*#+}} ymm0 {%k1} {z} = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: zext_8x16mem_to_8x32:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpsllw $15, %xmm0, %xmm0 # sched: [1:0.50]
-; SKX-NEXT:    vpmovw2m %xmm0, %k1 # sched: [1:1.00]
-; SKX-NEXT:    vpmovzxwd {{.*#+}} ymm0 {%k1} {z} = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %a   = load <8 x i16>,<8 x i16> *%i,align 1
-  %x   = zext <8 x i16> %a to <8 x i32>
-  %ret = select <8 x i1> %mask, <8 x i32> %x, <8 x i32> zeroinitializer
-  ret <8 x i32> %ret
-}
-
-define <8 x i32> @sext_8x16mem_to_8x32mask(<8 x i16> *%i , <8 x i1> %mask) nounwind readnone {
-; GENERIC-LABEL: sext_8x16mem_to_8x32mask:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpsllw $15, %xmm0, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vpmovw2m %xmm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpmovsxwd (%rdi), %ymm0 {%k1} {z} # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: sext_8x16mem_to_8x32mask:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpsllw $15, %xmm0, %xmm0 # sched: [1:0.50]
-; SKX-NEXT:    vpmovw2m %xmm0, %k1 # sched: [1:1.00]
-; SKX-NEXT:    vpmovsxwd (%rdi), %ymm0 {%k1} {z} # sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %a   = load <8 x i16>,<8 x i16> *%i,align 1
-  %x   = sext <8 x i16> %a to <8 x i32>
-  %ret = select <8 x i1> %mask, <8 x i32> %x, <8 x i32> zeroinitializer
-  ret <8 x i32> %ret
-}
-
-define <8 x i32> @sext_8x16mem_to_8x32(<8 x i16> *%i) nounwind readnone {
-; GENERIC-LABEL: sext_8x16mem_to_8x32:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpmovsxwd (%rdi), %ymm0 # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: sext_8x16mem_to_8x32:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpmovsxwd (%rdi), %ymm0 # sched: [9:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %a   = load <8 x i16>,<8 x i16> *%i,align 1
-  %x   = sext <8 x i16> %a to <8 x i32>
-  ret <8 x i32> %x
-}
-
-define <8 x i32> @zext_8x16_to_8x32mask(<8 x i16> %a , <8 x i1> %mask) nounwind readnone {
-; GENERIC-LABEL: zext_8x16_to_8x32mask:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpsllw $15, %xmm1, %xmm1 # sched: [1:1.00]
-; GENERIC-NEXT:    vpmovw2m %xmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpmovzxwd {{.*#+}} ymm0 {%k1} {z} = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: zext_8x16_to_8x32mask:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpsllw $15, %xmm1, %xmm1 # sched: [1:0.50]
-; SKX-NEXT:    vpmovw2m %xmm1, %k1 # sched: [1:1.00]
-; SKX-NEXT:    vpmovzxwd {{.*#+}} ymm0 {%k1} {z} = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %x   = zext <8 x i16> %a to <8 x i32>
-  %ret = select <8 x i1> %mask, <8 x i32> %x, <8 x i32> zeroinitializer
-  ret <8 x i32> %ret
-}
-
-define <8 x i32> @zext_8x16_to_8x32(<8 x i16> %a ) nounwind readnone {
-; GENERIC-LABEL: zext_8x16_to_8x32:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: zext_8x16_to_8x32:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %x   = zext <8 x i16> %a to <8 x i32>
-  ret <8 x i32> %x
-}
-
-define <16 x i32> @zext_16x16mem_to_16x32(<16 x i16> *%i , <16 x i1> %mask) nounwind readnone {
-; GENERIC-LABEL: zext_16x16mem_to_16x32:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpsllw $7, %xmm0, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vpmovb2m %xmm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpmovzxwd {{.*#+}} zmm0 {%k1} {z} = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero,mem[8],zero,mem[9],zero,mem[10],zero,mem[11],zero,mem[12],zero,mem[13],zero,mem[14],zero,mem[15],zero sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: zext_16x16mem_to_16x32:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpsllw $7, %xmm0, %xmm0 # sched: [1:0.50]
-; SKX-NEXT:    vpmovb2m %xmm0, %k1 # sched: [1:1.00]
-; SKX-NEXT:    vpmovzxwd {{.*#+}} zmm0 {%k1} {z} = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero,mem[8],zero,mem[9],zero,mem[10],zero,mem[11],zero,mem[12],zero,mem[13],zero,mem[14],zero,mem[15],zero sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %a   = load <16 x i16>,<16 x i16> *%i,align 1
-  %x   = zext <16 x i16> %a to <16 x i32>
-  %ret = select <16 x i1> %mask, <16 x i32> %x, <16 x i32> zeroinitializer
-  ret <16 x i32> %ret
-}
-
-define <16 x i32> @sext_16x16mem_to_16x32mask(<16 x i16> *%i , <16 x i1> %mask) nounwind readnone {
-; GENERIC-LABEL: sext_16x16mem_to_16x32mask:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpsllw $7, %xmm0, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vpmovb2m %xmm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpmovsxwd (%rdi), %zmm0 {%k1} {z} # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: sext_16x16mem_to_16x32mask:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpsllw $7, %xmm0, %xmm0 # sched: [1:0.50]
-; SKX-NEXT:    vpmovb2m %xmm0, %k1 # sched: [1:1.00]
-; SKX-NEXT:    vpmovsxwd (%rdi), %zmm0 {%k1} {z} # sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %a   = load <16 x i16>,<16 x i16> *%i,align 1
-  %x   = sext <16 x i16> %a to <16 x i32>
-  %ret = select <16 x i1> %mask, <16 x i32> %x, <16 x i32> zeroinitializer
-  ret <16 x i32> %ret
-}
-
-define <16 x i32> @sext_16x16mem_to_16x32(<16 x i16> *%i) nounwind readnone {
-; GENERIC-LABEL: sext_16x16mem_to_16x32:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpmovsxwd (%rdi), %zmm0 # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: sext_16x16mem_to_16x32:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpmovsxwd (%rdi), %zmm0 # sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %a   = load <16 x i16>,<16 x i16> *%i,align 1
-  %x   = sext <16 x i16> %a to <16 x i32>
-  ret <16 x i32> %x
-}
-define <16 x i32> @zext_16x16_to_16x32mask(<16 x i16> %a , <16 x i1> %mask) nounwind readnone {
-; GENERIC-LABEL: zext_16x16_to_16x32mask:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpsllw $7, %xmm1, %xmm1 # sched: [1:1.00]
-; GENERIC-NEXT:    vpmovb2m %xmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpmovzxwd {{.*#+}} zmm0 {%k1} {z} = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: zext_16x16_to_16x32mask:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpsllw $7, %xmm1, %xmm1 # sched: [1:0.50]
-; SKX-NEXT:    vpmovb2m %xmm1, %k1 # sched: [1:1.00]
-; SKX-NEXT:    vpmovzxwd {{.*#+}} zmm0 {%k1} {z} = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %x   = zext <16 x i16> %a to <16 x i32>
-  %ret = select <16 x i1> %mask, <16 x i32> %x, <16 x i32> zeroinitializer
-  ret <16 x i32> %ret
-}
-
-define <16 x i32> @zext_16x16_to_16x32(<16 x i16> %a ) nounwind readnone {
-; GENERIC-LABEL: zext_16x16_to_16x32:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: zext_16x16_to_16x32:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %x   = zext <16 x i16> %a to <16 x i32>
-  ret <16 x i32> %x
-}
-
-define <2 x i64> @zext_2x16mem_to_2x64(<2 x i16> *%i , <2 x i1> %mask) nounwind readnone {
-; GENERIC-LABEL: zext_2x16mem_to_2x64:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpsllq $63, %xmm0, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vpmovq2m %xmm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpmovzxwq {{.*#+}} xmm0 {%k1} {z} = mem[0],zero,zero,zero,mem[1],zero,zero,zero sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: zext_2x16mem_to_2x64:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpsllq $63, %xmm0, %xmm0 # sched: [1:0.50]
-; SKX-NEXT:    vpmovq2m %xmm0, %k1 # sched: [1:1.00]
-; SKX-NEXT:    vpmovzxwq {{.*#+}} xmm0 {%k1} {z} = mem[0],zero,zero,zero,mem[1],zero,zero,zero sched: [9:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %a   = load <2 x i16>,<2 x i16> *%i,align 1
-  %x   = zext <2 x i16> %a to <2 x i64>
-  %ret = select <2 x  i1> %mask, <2 x i64> %x, <2 x i64> zeroinitializer
-  ret <2 x i64> %ret
-}
-
-define <2 x i64> @sext_2x16mem_to_2x64mask(<2 x i16> *%i , <2 x i1> %mask) nounwind readnone {
-; GENERIC-LABEL: sext_2x16mem_to_2x64mask:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpsllq $63, %xmm0, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vpmovq2m %xmm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpmovsxwq (%rdi), %xmm0 {%k1} {z} # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: sext_2x16mem_to_2x64mask:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpsllq $63, %xmm0, %xmm0 # sched: [1:0.50]
-; SKX-NEXT:    vpmovq2m %xmm0, %k1 # sched: [1:1.00]
-; SKX-NEXT:    vpmovsxwq (%rdi), %xmm0 {%k1} {z} # sched: [9:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %a   = load <2 x i16>,<2 x i16> *%i,align 1
-  %x   = sext <2 x i16> %a to <2 x i64>
-  %ret = select <2 x i1> %mask, <2 x i64> %x, <2 x i64> zeroinitializer
-  ret <2 x i64> %ret
-}
-
-define <2 x i64> @sext_2x16mem_to_2x64(<2 x i16> *%i) nounwind readnone {
-; GENERIC-LABEL: sext_2x16mem_to_2x64:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpmovsxwq (%rdi), %xmm0 # sched: [7:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: sext_2x16mem_to_2x64:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpmovsxwq (%rdi), %xmm0 # sched: [6:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %a   = load <2 x i16>,<2 x i16> *%i,align 1
-  %x   = sext <2 x i16> %a to <2 x i64>
-  ret <2 x i64> %x
-}
-
-define <4 x i64> @zext_4x16mem_to_4x64(<4 x i16> *%i , <4 x i1> %mask) nounwind readnone {
-; GENERIC-LABEL: zext_4x16mem_to_4x64:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpslld $31, %xmm0, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vpmovd2m %xmm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpmovzxwq {{.*#+}} ymm0 {%k1} {z} = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: zext_4x16mem_to_4x64:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpslld $31, %xmm0, %xmm0 # sched: [1:0.50]
-; SKX-NEXT:    vpmovd2m %xmm0, %k1 # sched: [1:1.00]
-; SKX-NEXT:    vpmovzxwq {{.*#+}} ymm0 {%k1} {z} = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %a   = load <4 x i16>,<4 x i16> *%i,align 1
-  %x   = zext <4 x i16> %a to <4 x i64>
-  %ret = select <4 x  i1> %mask, <4 x i64> %x, <4 x i64> zeroinitializer
-  ret <4 x i64> %ret
-}
-
-define <4 x i64> @sext_4x16mem_to_4x64mask(<4 x i16> *%i , <4 x i1> %mask) nounwind readnone {
-; GENERIC-LABEL: sext_4x16mem_to_4x64mask:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpslld $31, %xmm0, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vpmovd2m %xmm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpmovsxwq (%rdi), %ymm0 {%k1} {z} # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: sext_4x16mem_to_4x64mask:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpslld $31, %xmm0, %xmm0 # sched: [1:0.50]
-; SKX-NEXT:    vpmovd2m %xmm0, %k1 # sched: [1:1.00]
-; SKX-NEXT:    vpmovsxwq (%rdi), %ymm0 {%k1} {z} # sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %a   = load <4 x i16>,<4 x i16> *%i,align 1
-  %x   = sext <4 x i16> %a to <4 x i64>
-  %ret = select <4 x i1> %mask, <4 x i64> %x, <4 x i64> zeroinitializer
-  ret <4 x i64> %ret
-}
-
-define <4 x i64> @sext_4x16mem_to_4x64(<4 x i16> *%i) nounwind readnone {
-; GENERIC-LABEL: sext_4x16mem_to_4x64:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpmovsxwq (%rdi), %ymm0 # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: sext_4x16mem_to_4x64:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpmovsxwq (%rdi), %ymm0 # sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %a   = load <4 x i16>,<4 x i16> *%i,align 1
-  %x   = sext <4 x i16> %a to <4 x i64>
-  ret <4 x i64> %x
-}
-
-define <8 x i64> @zext_8x16mem_to_8x64(<8 x i16> *%i , <8 x i1> %mask) nounwind readnone {
-; GENERIC-LABEL: zext_8x16mem_to_8x64:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpsllw $15, %xmm0, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vpmovw2m %xmm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpmovzxwq {{.*#+}} zmm0 {%k1} {z} = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: zext_8x16mem_to_8x64:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpsllw $15, %xmm0, %xmm0 # sched: [1:0.50]
-; SKX-NEXT:    vpmovw2m %xmm0, %k1 # sched: [1:1.00]
-; SKX-NEXT:    vpmovzxwq {{.*#+}} zmm0 {%k1} {z} = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %a   = load <8 x i16>,<8 x i16> *%i,align 1
-  %x   = zext <8 x i16> %a to <8 x i64>
-  %ret = select <8 x  i1> %mask, <8 x i64> %x, <8 x i64> zeroinitializer
-  ret <8 x i64> %ret
-}
-
-define <8 x i64> @sext_8x16mem_to_8x64mask(<8 x i16> *%i , <8 x i1> %mask) nounwind readnone {
-; GENERIC-LABEL: sext_8x16mem_to_8x64mask:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpsllw $15, %xmm0, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vpmovw2m %xmm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpmovsxwq (%rdi), %zmm0 {%k1} {z} # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: sext_8x16mem_to_8x64mask:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpsllw $15, %xmm0, %xmm0 # sched: [1:0.50]
-; SKX-NEXT:    vpmovw2m %xmm0, %k1 # sched: [1:1.00]
-; SKX-NEXT:    vpmovsxwq (%rdi), %zmm0 {%k1} {z} # sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %a   = load <8 x i16>,<8 x i16> *%i,align 1
-  %x   = sext <8 x i16> %a to <8 x i64>
-  %ret = select <8 x i1> %mask, <8 x i64> %x, <8 x i64> zeroinitializer
-  ret <8 x i64> %ret
-}
-
-define <8 x i64> @sext_8x16mem_to_8x64(<8 x i16> *%i) nounwind readnone {
-; GENERIC-LABEL: sext_8x16mem_to_8x64:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpmovsxwq (%rdi), %zmm0 # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: sext_8x16mem_to_8x64:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpmovsxwq (%rdi), %zmm0 # sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %a   = load <8 x i16>,<8 x i16> *%i,align 1
-  %x   = sext <8 x i16> %a to <8 x i64>
-  ret <8 x i64> %x
-}
-
-define <8 x i64> @zext_8x16_to_8x64mask(<8 x i16> %a , <8 x i1> %mask) nounwind readnone {
-; GENERIC-LABEL: zext_8x16_to_8x64mask:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpsllw $15, %xmm1, %xmm1 # sched: [1:1.00]
-; GENERIC-NEXT:    vpmovw2m %xmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpmovzxwq {{.*#+}} zmm0 {%k1} {z} = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: zext_8x16_to_8x64mask:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpsllw $15, %xmm1, %xmm1 # sched: [1:0.50]
-; SKX-NEXT:    vpmovw2m %xmm1, %k1 # sched: [1:1.00]
-; SKX-NEXT:    vpmovzxwq {{.*#+}} zmm0 {%k1} {z} = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %x   = zext <8 x i16> %a to <8 x i64>
-  %ret = select <8 x  i1> %mask, <8 x i64> %x, <8 x i64> zeroinitializer
-  ret <8 x i64> %ret
-}
-
-define <8 x i64> @zext_8x16_to_8x64(<8 x i16> %a) nounwind readnone {
-; GENERIC-LABEL: zext_8x16_to_8x64:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpmovzxwq {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: zext_8x16_to_8x64:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpmovzxwq {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %ret   = zext <8 x i16> %a to <8 x i64>
-  ret <8 x i64> %ret
-}
-
-define <2 x i64> @zext_2x32mem_to_2x64(<2 x i32> *%i , <2 x i1> %mask) nounwind readnone {
-; GENERIC-LABEL: zext_2x32mem_to_2x64:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpsllq $63, %xmm0, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vpmovq2m %xmm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpmovzxdq {{.*#+}} xmm0 {%k1} {z} = mem[0],zero,mem[1],zero sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: zext_2x32mem_to_2x64:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpsllq $63, %xmm0, %xmm0 # sched: [1:0.50]
-; SKX-NEXT:    vpmovq2m %xmm0, %k1 # sched: [1:1.00]
-; SKX-NEXT:    vpmovzxdq {{.*#+}} xmm0 {%k1} {z} = mem[0],zero,mem[1],zero sched: [9:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %a   = load <2 x i32>,<2 x i32> *%i,align 1
-  %x   = zext <2 x i32> %a to <2 x i64>
-  %ret = select <2 x  i1> %mask, <2 x i64> %x, <2 x i64> zeroinitializer
-  ret <2 x i64> %ret
-}
-
-define <2 x i64> @sext_2x32mem_to_2x64mask(<2 x i32> *%i , <2 x i1> %mask) nounwind readnone {
-; GENERIC-LABEL: sext_2x32mem_to_2x64mask:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpsllq $63, %xmm0, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vpmovq2m %xmm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpmovsxdq (%rdi), %xmm0 {%k1} {z} # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: sext_2x32mem_to_2x64mask:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpsllq $63, %xmm0, %xmm0 # sched: [1:0.50]
-; SKX-NEXT:    vpmovq2m %xmm0, %k1 # sched: [1:1.00]
-; SKX-NEXT:    vpmovsxdq (%rdi), %xmm0 {%k1} {z} # sched: [9:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %a   = load <2 x i32>,<2 x i32> *%i,align 1
-  %x   = sext <2 x i32> %a to <2 x i64>
-  %ret = select <2 x i1> %mask, <2 x i64> %x, <2 x i64> zeroinitializer
-  ret <2 x i64> %ret
-}
-
-define <2 x i64> @sext_2x32mem_to_2x64(<2 x i32> *%i) nounwind readnone {
-; GENERIC-LABEL: sext_2x32mem_to_2x64:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpmovsxdq (%rdi), %xmm0 # sched: [7:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: sext_2x32mem_to_2x64:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpmovsxdq (%rdi), %xmm0 # sched: [6:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %a   = load <2 x i32>,<2 x i32> *%i,align 1
-  %x   = sext <2 x i32> %a to <2 x i64>
-  ret <2 x i64> %x
-}
-
-define <4 x i64> @zext_4x32mem_to_4x64(<4 x i32> *%i , <4 x i1> %mask) nounwind readnone {
-; GENERIC-LABEL: zext_4x32mem_to_4x64:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpslld $31, %xmm0, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vpmovd2m %xmm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpmovzxdq {{.*#+}} ymm0 {%k1} {z} = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: zext_4x32mem_to_4x64:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpslld $31, %xmm0, %xmm0 # sched: [1:0.50]
-; SKX-NEXT:    vpmovd2m %xmm0, %k1 # sched: [1:1.00]
-; SKX-NEXT:    vpmovzxdq {{.*#+}} ymm0 {%k1} {z} = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %a   = load <4 x i32>,<4 x i32> *%i,align 1
-  %x   = zext <4 x i32> %a to <4 x i64>
-  %ret = select <4 x  i1> %mask, <4 x i64> %x, <4 x i64> zeroinitializer
-  ret <4 x i64> %ret
-}
-
-define <4 x i64> @sext_4x32mem_to_4x64mask(<4 x i32> *%i , <4 x i1> %mask) nounwind readnone {
-; GENERIC-LABEL: sext_4x32mem_to_4x64mask:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpslld $31, %xmm0, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vpmovd2m %xmm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpmovsxdq (%rdi), %ymm0 {%k1} {z} # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: sext_4x32mem_to_4x64mask:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpslld $31, %xmm0, %xmm0 # sched: [1:0.50]
-; SKX-NEXT:    vpmovd2m %xmm0, %k1 # sched: [1:1.00]
-; SKX-NEXT:    vpmovsxdq (%rdi), %ymm0 {%k1} {z} # sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %a   = load <4 x i32>,<4 x i32> *%i,align 1
-  %x   = sext <4 x i32> %a to <4 x i64>
-  %ret = select <4 x i1> %mask, <4 x i64> %x, <4 x i64> zeroinitializer
-  ret <4 x i64> %ret
-}
-
-define <4 x i64> @sext_4x32mem_to_4x64(<4 x i32> *%i) nounwind readnone {
-; GENERIC-LABEL: sext_4x32mem_to_4x64:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpmovsxdq (%rdi), %ymm0 # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: sext_4x32mem_to_4x64:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpmovsxdq (%rdi), %ymm0 # sched: [9:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %a   = load <4 x i32>,<4 x i32> *%i,align 1
-  %x   = sext <4 x i32> %a to <4 x i64>
-  ret <4 x i64> %x
-}
-
-define <4 x i64> @sext_4x32_to_4x64(<4 x i32> %a) nounwind readnone {
-; GENERIC-LABEL: sext_4x32_to_4x64:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpmovsxdq %xmm0, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: sext_4x32_to_4x64:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpmovsxdq %xmm0, %ymm0 # sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %x   = sext <4 x i32> %a to <4 x i64>
-  ret <4 x i64> %x
-}
-
-define <4 x i64> @zext_4x32_to_4x64mask(<4 x i32> %a , <4 x i1> %mask) nounwind readnone {
-; GENERIC-LABEL: zext_4x32_to_4x64mask:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpslld $31, %xmm1, %xmm1 # sched: [1:1.00]
-; GENERIC-NEXT:    vpmovd2m %xmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpmovzxdq {{.*#+}} ymm0 {%k1} {z} = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: zext_4x32_to_4x64mask:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpslld $31, %xmm1, %xmm1 # sched: [1:0.50]
-; SKX-NEXT:    vpmovd2m %xmm1, %k1 # sched: [1:1.00]
-; SKX-NEXT:    vpmovzxdq {{.*#+}} ymm0 {%k1} {z} = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %x   = zext <4 x i32> %a to <4 x i64>
-  %ret = select <4 x  i1> %mask, <4 x i64> %x, <4 x i64> zeroinitializer
-  ret <4 x i64> %ret
-}
-
-define <8 x i64> @zext_8x32mem_to_8x64(<8 x i32> *%i , <8 x i1> %mask) nounwind readnone {
-; GENERIC-LABEL: zext_8x32mem_to_8x64:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpsllw $15, %xmm0, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vpmovw2m %xmm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpmovzxdq {{.*#+}} zmm0 {%k1} {z} = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: zext_8x32mem_to_8x64:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpsllw $15, %xmm0, %xmm0 # sched: [1:0.50]
-; SKX-NEXT:    vpmovw2m %xmm0, %k1 # sched: [1:1.00]
-; SKX-NEXT:    vpmovzxdq {{.*#+}} zmm0 {%k1} {z} = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %a   = load <8 x i32>,<8 x i32> *%i,align 1
-  %x   = zext <8 x i32> %a to <8 x i64>
-  %ret = select <8 x  i1> %mask, <8 x i64> %x, <8 x i64> zeroinitializer
-  ret <8 x i64> %ret
-}
-
-define <8 x i64> @sext_8x32mem_to_8x64mask(<8 x i32> *%i , <8 x i1> %mask) nounwind readnone {
-; GENERIC-LABEL: sext_8x32mem_to_8x64mask:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpsllw $15, %xmm0, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vpmovw2m %xmm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpmovsxdq (%rdi), %zmm0 {%k1} {z} # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: sext_8x32mem_to_8x64mask:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpsllw $15, %xmm0, %xmm0 # sched: [1:0.50]
-; SKX-NEXT:    vpmovw2m %xmm0, %k1 # sched: [1:1.00]
-; SKX-NEXT:    vpmovsxdq (%rdi), %zmm0 {%k1} {z} # sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %a   = load <8 x i32>,<8 x i32> *%i,align 1
-  %x   = sext <8 x i32> %a to <8 x i64>
-  %ret = select <8 x i1> %mask, <8 x i64> %x, <8 x i64> zeroinitializer
-  ret <8 x i64> %ret
-}
-
-define <8 x i64> @sext_8x32mem_to_8x64(<8 x i32> *%i) nounwind readnone {
-; GENERIC-LABEL: sext_8x32mem_to_8x64:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpmovsxdq (%rdi), %zmm0 # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: sext_8x32mem_to_8x64:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpmovsxdq (%rdi), %zmm0 # sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %a   = load <8 x i32>,<8 x i32> *%i,align 1
-  %x   = sext <8 x i32> %a to <8 x i64>
-  ret <8 x i64> %x
-}
-
-define <8 x i64> @sext_8x32_to_8x64(<8 x i32> %a) nounwind readnone {
-; GENERIC-LABEL: sext_8x32_to_8x64:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpmovsxdq %ymm0, %zmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: sext_8x32_to_8x64:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpmovsxdq %ymm0, %zmm0 # sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %x   = sext <8 x i32> %a to <8 x i64>
-  ret <8 x i64> %x
-}
-
-define <8 x i64> @zext_8x32_to_8x64mask(<8 x i32> %a , <8 x i1> %mask) nounwind readnone {
-; GENERIC-LABEL: zext_8x32_to_8x64mask:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpsllw $15, %xmm1, %xmm1 # sched: [1:1.00]
-; GENERIC-NEXT:    vpmovw2m %xmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpmovzxdq {{.*#+}} zmm0 {%k1} {z} = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: zext_8x32_to_8x64mask:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpsllw $15, %xmm1, %xmm1 # sched: [1:0.50]
-; SKX-NEXT:    vpmovw2m %xmm1, %k1 # sched: [1:1.00]
-; SKX-NEXT:    vpmovzxdq {{.*#+}} zmm0 {%k1} {z} = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %x   = zext <8 x i32> %a to <8 x i64>
-  %ret = select <8 x  i1> %mask, <8 x i64> %x, <8 x i64> zeroinitializer
-  ret <8 x i64> %ret
-}
-define <8 x float> @fptrunc_test(<8 x double> %a) nounwind readnone {
-; GENERIC-LABEL: fptrunc_test:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vcvtpd2ps %zmm0, %ymm0 # sched: [4:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: fptrunc_test:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vcvtpd2ps %zmm0, %ymm0 # sched: [7:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %b = fptrunc <8 x double> %a to <8 x float>
-  ret <8 x float> %b
-}
-
-define <8 x double> @fpext_test(<8 x float> %a) nounwind readnone {
-; GENERIC-LABEL: fpext_test:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vcvtps2pd %ymm0, %zmm0 # sched: [2:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: fpext_test:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vcvtps2pd %ymm0, %zmm0 # sched: [7:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %b = fpext <8 x float> %a to <8 x double>
-  ret <8 x double> %b
-}
-
-define   <16 x i32> @zext_16i1_to_16xi32(i16 %b) {
-; GENERIC-LABEL: zext_16i1_to_16xi32:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    kmovd %edi, %k0 # sched: [1:0.33]
-; GENERIC-NEXT:    vpmovm2d %k0, %zmm0 # sched: [1:0.33]
-; GENERIC-NEXT:    vpsrld $31, %zmm0, %zmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: zext_16i1_to_16xi32:
-; SKX:       # %bb.0:
-; SKX-NEXT:    kmovd %edi, %k0 # sched: [1:1.00]
-; SKX-NEXT:    vpmovm2d %k0, %zmm0 # sched: [1:0.25]
-; SKX-NEXT:    vpsrld $31, %zmm0, %zmm0 # sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %a = bitcast i16 %b to <16 x i1>
-  %c = zext <16 x i1> %a to <16 x i32>
-  ret <16 x i32> %c
-}
-
-define   <8 x i64> @zext_8i1_to_8xi64(i8 %b) {
-; GENERIC-LABEL: zext_8i1_to_8xi64:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    kmovd %edi, %k0 # sched: [1:0.33]
-; GENERIC-NEXT:    vpmovm2q %k0, %zmm0 # sched: [1:0.33]
-; GENERIC-NEXT:    vpsrlq $63, %zmm0, %zmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: zext_8i1_to_8xi64:
-; SKX:       # %bb.0:
-; SKX-NEXT:    kmovd %edi, %k0 # sched: [1:1.00]
-; SKX-NEXT:    vpmovm2q %k0, %zmm0 # sched: [1:0.25]
-; SKX-NEXT:    vpsrlq $63, %zmm0, %zmm0 # sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %a = bitcast i8 %b to <8 x i1>
-  %c = zext <8 x i1> %a to <8 x i64>
-  ret <8 x i64> %c
-}
-
-define i16 @trunc_16i8_to_16i1(<16 x i8> %a) {
-; GENERIC-LABEL: trunc_16i8_to_16i1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpsllw $7, %xmm0, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vpmovb2m %xmm0, %k0 # sched: [1:0.33]
-; GENERIC-NEXT:    kmovd %k0, %eax # sched: [1:0.33]
-; GENERIC-NEXT:    # kill: def $ax killed $ax killed $eax
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: trunc_16i8_to_16i1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpsllw $7, %xmm0, %xmm0 # sched: [1:0.50]
-; SKX-NEXT:    vpmovb2m %xmm0, %k0 # sched: [1:1.00]
-; SKX-NEXT:    kmovd %k0, %eax # sched: [3:1.00]
-; SKX-NEXT:    # kill: def $ax killed $ax killed $eax
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %mask_b = trunc <16 x i8>%a to <16 x i1>
-  %mask = bitcast <16 x i1> %mask_b to i16
-  ret i16 %mask
-}
-
-define i16 @trunc_16i32_to_16i1(<16 x i32> %a) {
-; GENERIC-LABEL: trunc_16i32_to_16i1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpslld $31, %zmm0, %zmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vpmovd2m %zmm0, %k0 # sched: [1:0.33]
-; GENERIC-NEXT:    kmovd %k0, %eax # sched: [1:0.33]
-; GENERIC-NEXT:    # kill: def $ax killed $ax killed $eax
-; GENERIC-NEXT:    vzeroupper # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: trunc_16i32_to_16i1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpslld $31, %zmm0, %zmm0 # sched: [1:1.00]
-; SKX-NEXT:    vpmovd2m %zmm0, %k0 # sched: [1:1.00]
-; SKX-NEXT:    kmovd %k0, %eax # sched: [3:1.00]
-; SKX-NEXT:    # kill: def $ax killed $ax killed $eax
-; SKX-NEXT:    vzeroupper # sched: [0:0.67]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %mask_b = trunc <16 x i32>%a to <16 x i1>
-  %mask = bitcast <16 x i1> %mask_b to i16
-  ret i16 %mask
-}
-
-define <4 x i32> @trunc_4i32_to_4i1(<4 x i32> %a, <4 x i32> %b) {
-; GENERIC-LABEL: trunc_4i32_to_4i1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpand %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; GENERIC-NEXT:    vpslld $31, %xmm0, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vpsrad $31, %xmm0, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: trunc_4i32_to_4i1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpand %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    vpslld $31, %xmm0, %xmm0 # sched: [1:0.50]
-; SKX-NEXT:    vpsrad $31, %xmm0, %xmm0 # sched: [1:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %mask_a = trunc <4 x i32>%a to <4 x i1>
-  %mask_b = trunc <4 x i32>%b to <4 x i1>
-  %a_and_b = and <4 x i1>%mask_a, %mask_b
-  %res = sext <4 x i1>%a_and_b to <4 x i32>
-  ret <4 x i32>%res
-}
-
-
-define i8 @trunc_8i16_to_8i1(<8 x i16> %a) {
-; GENERIC-LABEL: trunc_8i16_to_8i1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpsllw $15, %xmm0, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vpmovw2m %xmm0, %k0 # sched: [1:0.33]
-; GENERIC-NEXT:    kmovd %k0, %eax # sched: [1:0.33]
-; GENERIC-NEXT:    # kill: def $al killed $al killed $eax
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: trunc_8i16_to_8i1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpsllw $15, %xmm0, %xmm0 # sched: [1:0.50]
-; SKX-NEXT:    vpmovw2m %xmm0, %k0 # sched: [1:1.00]
-; SKX-NEXT:    kmovd %k0, %eax # sched: [3:1.00]
-; SKX-NEXT:    # kill: def $al killed $al killed $eax
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %mask_b = trunc <8 x i16>%a to <8 x i1>
-  %mask = bitcast <8 x i1> %mask_b to i8
-  ret i8 %mask
-}
-
-define <8 x i32> @sext_8i1_8i32(<8 x i32> %a1, <8 x i32> %a2) nounwind {
-; GENERIC-LABEL: sext_8i1_8i32:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpcmpgtd %ymm0, %ymm1, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    vpternlogq $15, %ymm0, %ymm0, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: sext_8i1_8i32:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpcmpgtd %ymm0, %ymm1, %ymm0 # sched: [1:0.50]
-; SKX-NEXT:    vpternlogq $15, %ymm0, %ymm0, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %x = icmp slt <8 x i32> %a1, %a2
-  %x1 = xor <8 x i1>%x, <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>
-  %y = sext <8 x i1> %x1 to <8 x i32>
-  ret <8 x i32> %y
-}
-
-
-define i16 @trunc_i32_to_i1(i32 %a) {
-; GENERIC-LABEL: trunc_i32_to_i1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    movw $-4, %ax # sched: [1:0.33]
-; GENERIC-NEXT:    kmovd %eax, %k0 # sched: [1:0.33]
-; GENERIC-NEXT:    kshiftrw $1, %k0, %k0 # sched: [1:1.00]
-; GENERIC-NEXT:    kshiftlw $1, %k0, %k0 # sched: [1:1.00]
-; GENERIC-NEXT:    andl $1, %edi # sched: [1:0.33]
-; GENERIC-NEXT:    kmovw %edi, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    korw %k1, %k0, %k0 # sched: [1:0.33]
-; GENERIC-NEXT:    kmovd %k0, %eax # sched: [1:0.33]
-; GENERIC-NEXT:    # kill: def $ax killed $ax killed $eax
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: trunc_i32_to_i1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    movw $-4, %ax # sched: [1:0.25]
-; SKX-NEXT:    kmovd %eax, %k0 # sched: [1:1.00]
-; SKX-NEXT:    kshiftrw $1, %k0, %k0 # sched: [3:1.00]
-; SKX-NEXT:    kshiftlw $1, %k0, %k0 # sched: [3:1.00]
-; SKX-NEXT:    andl $1, %edi # sched: [1:0.25]
-; SKX-NEXT:    kmovw %edi, %k1 # sched: [1:1.00]
-; SKX-NEXT:    korw %k1, %k0, %k0 # sched: [1:1.00]
-; SKX-NEXT:    kmovd %k0, %eax # sched: [3:1.00]
-; SKX-NEXT:    # kill: def $ax killed $ax killed $eax
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %a_i = trunc i32 %a to i1
-  %maskv = insertelement <16 x i1> <i1 true, i1 false, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, i1 %a_i, i32 0
-  %res = bitcast <16 x i1> %maskv to i16
-  ret i16 %res
-}
-
-define <8 x i16> @sext_8i1_8i16(<8 x i32> %a1, <8 x i32> %a2) nounwind {
-; GENERIC-LABEL: sext_8i1_8i16:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpcmpgtd %ymm0, %ymm1, %k0 # sched: [1:0.50]
-; GENERIC-NEXT:    vpmovm2w %k0, %xmm0 # sched: [1:0.33]
-; GENERIC-NEXT:    vzeroupper # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: sext_8i1_8i16:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpcmpgtd %ymm0, %ymm1, %k0 # sched: [3:1.00]
-; SKX-NEXT:    vpmovm2w %k0, %xmm0 # sched: [1:0.25]
-; SKX-NEXT:    vzeroupper # sched: [0:0.67]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %x = icmp slt <8 x i32> %a1, %a2
-  %y = sext <8 x i1> %x to <8 x i16>
-  ret <8 x i16> %y
-}
-
-define <16 x i32> @sext_16i1_16i32(<16 x i32> %a1, <16 x i32> %a2) nounwind {
-; GENERIC-LABEL: sext_16i1_16i32:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpcmpgtd %zmm0, %zmm1, %k0 # sched: [1:0.50]
-; GENERIC-NEXT:    vpmovm2d %k0, %zmm0 # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: sext_16i1_16i32:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpcmpgtd %zmm0, %zmm1, %k0 # sched: [3:1.00]
-; SKX-NEXT:    vpmovm2d %k0, %zmm0 # sched: [1:0.25]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %x = icmp slt <16 x i32> %a1, %a2
-  %y = sext <16 x i1> %x to <16 x i32>
-  ret <16 x i32> %y
-}
-
-define <8 x i64> @sext_8i1_8i64(<8 x i32> %a1, <8 x i32> %a2) nounwind {
-; GENERIC-LABEL: sext_8i1_8i64:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpcmpgtd %ymm0, %ymm1, %k0 # sched: [1:0.50]
-; GENERIC-NEXT:    vpmovm2q %k0, %zmm0 # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: sext_8i1_8i64:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpcmpgtd %ymm0, %ymm1, %k0 # sched: [3:1.00]
-; SKX-NEXT:    vpmovm2q %k0, %zmm0 # sched: [1:0.25]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %x = icmp slt <8 x i32> %a1, %a2
-  %y = sext <8 x i1> %x to <8 x i64>
-  ret <8 x i64> %y
-}
-
-define void @extload_v8i64(<8 x i8>* %a, <8 x i64>* %res) {
-; GENERIC-LABEL: extload_v8i64:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpmovsxbq (%rdi), %zmm0 # sched: [8:1.00]
-; GENERIC-NEXT:    vmovdqa64 %zmm0, (%rsi) # sched: [1:1.00]
-; GENERIC-NEXT:    vzeroupper # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: extload_v8i64:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpmovsxbq (%rdi), %zmm0 # sched: [10:1.00]
-; SKX-NEXT:    vmovdqa64 %zmm0, (%rsi) # sched: [1:1.00]
-; SKX-NEXT:    vzeroupper # sched: [0:0.67]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %sign_load = load <8 x i8>, <8 x i8>* %a
-  %c = sext <8 x i8> %sign_load to <8 x i64>
-  store <8 x i64> %c, <8 x i64>* %res
-  ret void
-}
-
-define <64 x i16> @test21(<64 x i16> %x , <64 x i1> %mask) nounwind readnone {
-; GENERIC-LABEL: test21:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpsllw $7, %zmm2, %zmm2 # sched: [1:1.00]
-; GENERIC-NEXT:    vpmovb2m %zmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vmovdqu16 %zmm0, %zmm0 {%k1} {z} # sched: [1:0.50]
-; GENERIC-NEXT:    kshiftrq $32, %k1, %k1 # sched: [1:1.00]
-; GENERIC-NEXT:    vmovdqu16 %zmm1, %zmm1 {%k1} {z} # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test21:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpsllw $7, %zmm2, %zmm2 # sched: [1:1.00]
-; SKX-NEXT:    vpmovb2m %zmm2, %k1 # sched: [1:1.00]
-; SKX-NEXT:    vmovdqu16 %zmm0, %zmm0 {%k1} {z} # sched: [1:0.33]
-; SKX-NEXT:    kshiftrq $32, %k1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vmovdqu16 %zmm1, %zmm1 {%k1} {z} # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %ret = select <64 x i1> %mask, <64 x i16> %x, <64 x i16> zeroinitializer
-  ret <64 x i16> %ret
-}
-
-define <16 x i16> @shuffle_zext_16x8_to_16x16(<16 x i8> %a) nounwind readnone {
-; GENERIC-LABEL: shuffle_zext_16x8_to_16x16:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: shuffle_zext_16x8_to_16x16:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %1 = shufflevector <16 x i8> %a, <16 x i8> zeroinitializer, <32 x i32> <i32 0, i32 16, i32 1, i32 16, i32 2, i32 16, i32 3, i32 16, i32 4, i32 16, i32 5, i32 16, i32 6, i32 16, i32 7, i32 16, i32 8, i32 16, i32 9, i32 16, i32 10, i32 16, i32 11, i32 16, i32 12, i32 16, i32 13, i32 16, i32 14, i32 16, i32 15, i32 16>
-  %2 = bitcast <32 x i8> %1 to <16 x i16>
-  ret <16 x i16> %2
-}
-
-define <16 x i16> @shuffle_zext_16x8_to_16x16_mask(<16 x i8> %a, <16 x i1> %mask) nounwind readnone {
-; GENERIC-LABEL: shuffle_zext_16x8_to_16x16_mask:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpsllw $7, %xmm1, %xmm1 # sched: [1:1.00]
-; GENERIC-NEXT:    vpmovb2m %xmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpmovzxbw {{.*#+}} ymm0 {%k1} {z} = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: shuffle_zext_16x8_to_16x16_mask:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpsllw $7, %xmm1, %xmm1 # sched: [1:0.50]
-; SKX-NEXT:    vpmovb2m %xmm1, %k1 # sched: [1:1.00]
-; SKX-NEXT:    vpmovzxbw {{.*#+}} ymm0 {%k1} {z} = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %x   = shufflevector <16 x i8> %a, <16 x i8> zeroinitializer, <32 x i32> <i32 0, i32 16, i32 1, i32 16, i32 2, i32 16, i32 3, i32 16, i32 4, i32 16, i32 5, i32 16, i32 6, i32 16, i32 7, i32 16, i32 8, i32 16, i32 9, i32 16, i32 10, i32 16, i32 11, i32 16, i32 12, i32 16, i32 13, i32 16, i32 14, i32 16, i32 15, i32 16>
-  %bc  = bitcast <32 x i8> %x to <16 x i16>
-  %ret = select <16 x i1> %mask, <16 x i16> %bc, <16 x i16> zeroinitializer
-  ret <16 x i16> %ret
-}
-
-define <16 x i16> @zext_32x8_to_16x16(<32 x i8> %a) {
-; GENERIC-LABEL: zext_32x8_to_16x16:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: zext_32x8_to_16x16:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %1 = shufflevector <32 x i8> %a, <32 x i8> zeroinitializer, <32 x i32> <i32 0, i32 32, i32 1, i32 32, i32 2, i32 32, i32 3, i32 32, i32 4, i32 32, i32 5, i32 32, i32 6, i32 32, i32 7, i32 32, i32 8, i32 32, i32 9, i32 32, i32 10, i32 32, i32 11, i32 32, i32 12, i32 32, i32 13, i32 32, i32 14, i32 32, i32 15, i32 32>
-  %2 = bitcast <32 x i8> %1 to <16 x i16>
-  ret <16 x i16> %2
-}
-
-define <8 x i32> @zext_32x8_to_8x32(<32 x i8> %a) {
-; GENERIC-LABEL: zext_32x8_to_8x32:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpmovzxbd {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: zext_32x8_to_8x32:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpmovzxbd {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %1 = shufflevector <32 x i8> %a, <32 x i8> zeroinitializer, <32 x i32> <i32 0, i32 32, i32 32, i32 32, i32 1, i32 32, i32 32, i32 32, i32 2, i32 32, i32 32, i32 32, i32 3, i32 32, i32 32, i32 32, i32 4, i32 32, i32 32, i32 32, i32 5, i32 32, i32 32, i32 32, i32 6, i32 32, i32 32, i32 32, i32 7, i32 32, i32 32, i32 32>
-  %2 = bitcast <32 x i8> %1 to <8 x i32>
-  ret <8 x i32> %2
-}
-
-define <4 x i64> @zext_32x8_to_4x64(<32 x i8> %a) {
-; GENERIC-LABEL: zext_32x8_to_4x64:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpmovzxbq {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero,xmm0[2],zero,zero,zero,zero,zero,zero,zero,xmm0[3],zero,zero,zero,zero,zero,zero,zero sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: zext_32x8_to_4x64:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpmovzxbq {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero,xmm0[2],zero,zero,zero,zero,zero,zero,zero,xmm0[3],zero,zero,zero,zero,zero,zero,zero sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %1 = shufflevector <32 x i8> %a, <32 x i8> zeroinitializer, <32 x i32> <i32 0, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 1, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 2, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 3, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32>
-  %2 = bitcast <32 x i8> %1 to <4 x i64>
-  ret <4 x i64> %2
-}
-
-define <8 x i32> @zext_16x16_to_8x32(<16 x i16> %a) {
-; GENERIC-LABEL: zext_16x16_to_8x32:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: zext_16x16_to_8x32:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %1 = shufflevector <16 x i16> %a, <16 x i16> zeroinitializer, <16 x i32> <i32 0, i32 16, i32 1, i32 16, i32 2, i32 16, i32 3, i32 16, i32 4, i32 16, i32 5, i32 16, i32 6, i32 16, i32 7, i32 16>
-  %2 = bitcast <16 x i16> %1 to <8 x i32>
-  ret <8 x i32> %2
-}
-
-define <4 x i64> @zext_16x16_to_4x64(<16 x i16> %a) {
-; GENERIC-LABEL: zext_16x16_to_4x64:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpmovzxwq {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: zext_16x16_to_4x64:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpmovzxwq {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %1 = shufflevector <16 x i16> %a, <16 x i16> zeroinitializer, <16 x i32> <i32 0, i32 16, i32 16, i32 16, i32 1, i32 16, i32 16, i32 16, i32 2, i32 16, i32 16, i32 16, i32 3, i32 16, i32 16, i32 16>
-  %2 = bitcast <16 x i16> %1 to <4 x i64>
-  ret <4 x i64> %2
-}
-
-define <4 x i64> @zext_8x32_to_4x64(<8 x i32> %a) {
-; GENERIC-LABEL: zext_8x32_to_4x64:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpmovzxdq {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: zext_8x32_to_4x64:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpmovzxdq {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %1 = shufflevector <8 x i32> %a, <8 x i32> zeroinitializer, <8 x i32> <i32 0, i32 8, i32 1, i32 8, i32 2, i32 8, i32 3, i32 8>
-  %2 = bitcast <8 x i32> %1 to <4 x i64>
-  ret <4 x i64> %2
-}
-
-define <64 x i8> @zext_64xi1_to_64xi8(<64 x i8> %x, <64 x i8> %y) #0 {
-; GENERIC-LABEL: zext_64xi1_to_64xi8:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpcmpeqb %zmm1, %zmm0, %k1 # sched: [1:0.50]
-; GENERIC-NEXT:    vmovdqu8 {{.*}}(%rip), %zmm0 {%k1} {z} # sched: [7:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: zext_64xi1_to_64xi8:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpcmpeqb %zmm1, %zmm0, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vmovdqu8 {{.*}}(%rip), %zmm0 {%k1} {z} # sched: [8:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %mask = icmp eq <64 x i8> %x, %y
-  %1 = zext <64 x i1> %mask to <64 x i8>
-  ret <64 x i8> %1
-}
-
-define <32 x i16> @zext_32xi1_to_32xi16(<32 x i16> %x, <32 x i16> %y) #0 {
-; GENERIC-LABEL: zext_32xi1_to_32xi16:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpcmpeqw %zmm1, %zmm0, %k0 # sched: [1:0.50]
-; GENERIC-NEXT:    vpmovm2w %k0, %zmm0 # sched: [1:0.33]
-; GENERIC-NEXT:    vpsrlw $15, %zmm0, %zmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: zext_32xi1_to_32xi16:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpcmpeqw %zmm1, %zmm0, %k0 # sched: [3:1.00]
-; SKX-NEXT:    vpmovm2w %k0, %zmm0 # sched: [1:0.25]
-; SKX-NEXT:    vpsrlw $15, %zmm0, %zmm0 # sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %mask = icmp eq <32 x i16> %x, %y
-  %1 = zext <32 x i1> %mask to <32 x i16>
-  ret <32 x i16> %1
-}
-
-define <16 x i16> @zext_16xi1_to_16xi16(<16 x i16> %x, <16 x i16> %y) #0 {
-; GENERIC-LABEL: zext_16xi1_to_16xi16:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpcmpeqw %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    vpsrlw $15, %ymm0, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: zext_16xi1_to_16xi16:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpcmpeqw %ymm1, %ymm0, %ymm0 # sched: [1:0.50]
-; SKX-NEXT:    vpsrlw $15, %ymm0, %ymm0 # sched: [1:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %mask = icmp eq <16 x i16> %x, %y
-  %1 = zext <16 x i1> %mask to <16 x i16>
-  ret <16 x i16> %1
-}
-
-
-define <32 x i8> @zext_32xi1_to_32xi8(<32 x i16> %x, <32 x i16> %y) #0 {
-; GENERIC-LABEL: zext_32xi1_to_32xi8:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpcmpeqw %zmm1, %zmm0, %k1 # sched: [1:0.50]
-; GENERIC-NEXT:    vmovdqu8 {{.*}}(%rip), %ymm0 {%k1} {z} # sched: [7:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: zext_32xi1_to_32xi8:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpcmpeqw %zmm1, %zmm0, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vmovdqu8 {{.*}}(%rip), %ymm0 {%k1} {z} # sched: [8:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %mask = icmp eq <32 x i16> %x, %y
-  %1 = zext <32 x i1> %mask to <32 x i8>
-  ret <32 x i8> %1
-}
-
-define <4 x i32> @zext_4xi1_to_4x32(<4 x i8> %x, <4 x i8> %y) #0 {
-; GENERIC-LABEL: zext_4xi1_to_4x32:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [255,255,255,255] sched: [7:0.50]
-; GENERIC-NEXT:    vpand %xmm2, %xmm1, %xmm1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpand %xmm2, %xmm0, %xmm0 # sched: [1:0.33]
-; GENERIC-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    vpsrld $31, %xmm0, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: zext_4xi1_to_4x32:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [255,255,255,255] sched: [6:0.50]
-; SKX-NEXT:    vpand %xmm2, %xmm1, %xmm1 # sched: [1:0.33]
-; SKX-NEXT:    vpand %xmm2, %xmm0, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SKX-NEXT:    vpsrld $31, %xmm0, %xmm0 # sched: [1:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %mask = icmp eq <4 x i8> %x, %y
-  %1 = zext <4 x i1> %mask to <4 x i32>
-  ret <4 x i32> %1
-}
-
-define <2 x i64> @zext_2xi1_to_2xi64(<2 x i8> %x, <2 x i8> %y) #0 {
-; GENERIC-LABEL: zext_2xi1_to_2xi64:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpbroadcastq {{.*#+}} xmm2 = [255,255] sched: [7:0.50]
-; GENERIC-NEXT:    vpand %xmm2, %xmm1, %xmm1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpand %xmm2, %xmm0, %xmm0 # sched: [1:0.33]
-; GENERIC-NEXT:    vpcmpeqq %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    vpsrlq $63, %xmm0, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: zext_2xi1_to_2xi64:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpbroadcastq {{.*#+}} xmm2 = [255,255] sched: [6:0.50]
-; SKX-NEXT:    vpand %xmm2, %xmm1, %xmm1 # sched: [1:0.33]
-; SKX-NEXT:    vpand %xmm2, %xmm0, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    vpcmpeqq %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SKX-NEXT:    vpsrlq $63, %xmm0, %xmm0 # sched: [1:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %mask = icmp eq <2 x i8> %x, %y
-  %1 = zext <2 x i1> %mask to <2 x i64>
-  ret <2 x i64> %1
-}
-
-define <16 x float> @test_x86_fmadd_ps_z(<16 x float> %a0, <16 x float> %a1, <16 x float> %a2) {
-; GENERIC-LABEL: test_x86_fmadd_ps_z:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmulps %zmm1, %zmm0, %zmm0 # sched: [5:1.00]
-; GENERIC-NEXT:    vaddps %zmm2, %zmm0, %zmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_x86_fmadd_ps_z:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmulps %zmm1, %zmm0, %zmm0 # sched: [4:0.50]
-; SKX-NEXT:    vaddps %zmm2, %zmm0, %zmm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %x = fmul <16 x float> %a0, %a1
-  %res = fadd <16 x float> %x, %a2
-  ret <16 x float> %res
-}
-
-define <16 x float> @test_x86_fmsub_ps_z(<16 x float> %a0, <16 x float> %a1, <16 x float> %a2) {
-; GENERIC-LABEL: test_x86_fmsub_ps_z:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmulps %zmm1, %zmm0, %zmm0 # sched: [5:1.00]
-; GENERIC-NEXT:    vsubps %zmm2, %zmm0, %zmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_x86_fmsub_ps_z:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmulps %zmm1, %zmm0, %zmm0 # sched: [4:0.50]
-; SKX-NEXT:    vsubps %zmm2, %zmm0, %zmm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %x = fmul <16 x float> %a0, %a1
-  %res = fsub <16 x float> %x, %a2
-  ret <16 x float> %res
-}
-
-define <16 x float> @test_x86_fnmadd_ps_z(<16 x float> %a0, <16 x float> %a1, <16 x float> %a2) {
-; GENERIC-LABEL: test_x86_fnmadd_ps_z:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmulps %zmm1, %zmm0, %zmm0 # sched: [5:1.00]
-; GENERIC-NEXT:    vsubps %zmm0, %zmm2, %zmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_x86_fnmadd_ps_z:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmulps %zmm1, %zmm0, %zmm0 # sched: [4:0.50]
-; SKX-NEXT:    vsubps %zmm0, %zmm2, %zmm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %x = fmul <16 x float> %a0, %a1
-  %res = fsub <16 x float> %a2, %x
-  ret <16 x float> %res
-}
-
-define <16 x float> @test_x86_fnmsub_ps_z(<16 x float> %a0, <16 x float> %a1, <16 x float> %a2) {
-; GENERIC-LABEL: test_x86_fnmsub_ps_z:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmulps %zmm1, %zmm0, %zmm0 # sched: [5:1.00]
-; GENERIC-NEXT:    vxorps {{.*}}(%rip){1to16}, %zmm0, %zmm0 # sched: [8:1.00]
-; GENERIC-NEXT:    vsubps %zmm2, %zmm0, %zmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_x86_fnmsub_ps_z:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmulps %zmm1, %zmm0, %zmm0 # sched: [4:0.50]
-; SKX-NEXT:    vxorps {{.*}}(%rip){1to16}, %zmm0, %zmm0 # sched: [8:0.50]
-; SKX-NEXT:    vsubps %zmm2, %zmm0, %zmm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %x = fmul <16 x float> %a0, %a1
-  %y = fsub <16 x float> <float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00,
-                          float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00,
-                          float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00,
-                          float -0.000000e+00>, %x
-  %res = fsub <16 x float> %y, %a2
-  ret <16 x float> %res
-}
-
-define <8 x double> @test_x86_fmadd_pd_z(<8 x double> %a0, <8 x double> %a1, <8 x double> %a2) {
-; GENERIC-LABEL: test_x86_fmadd_pd_z:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmulpd %zmm1, %zmm0, %zmm0 # sched: [5:1.00]
-; GENERIC-NEXT:    vaddpd %zmm2, %zmm0, %zmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_x86_fmadd_pd_z:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmulpd %zmm1, %zmm0, %zmm0 # sched: [4:0.50]
-; SKX-NEXT:    vaddpd %zmm2, %zmm0, %zmm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %x = fmul <8 x double> %a0, %a1
-  %res = fadd <8 x double> %x, %a2
-  ret <8 x double> %res
-}
-
-define <8 x double> @test_x86_fmsub_pd_z(<8 x double> %a0, <8 x double> %a1, <8 x double> %a2) {
-; GENERIC-LABEL: test_x86_fmsub_pd_z:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmulpd %zmm1, %zmm0, %zmm0 # sched: [5:1.00]
-; GENERIC-NEXT:    vsubpd %zmm2, %zmm0, %zmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_x86_fmsub_pd_z:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmulpd %zmm1, %zmm0, %zmm0 # sched: [4:0.50]
-; SKX-NEXT:    vsubpd %zmm2, %zmm0, %zmm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %x = fmul <8 x double> %a0, %a1
-  %res = fsub <8 x double> %x, %a2
-  ret <8 x double> %res
-}
-
-define double @test_x86_fmsub_213(double %a0, double %a1, double %a2) {
-; GENERIC-LABEL: test_x86_fmsub_213:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmulsd %xmm1, %xmm0, %xmm0 # sched: [5:1.00]
-; GENERIC-NEXT:    vsubsd %xmm2, %xmm0, %xmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_x86_fmsub_213:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmulsd %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKX-NEXT:    vsubsd %xmm2, %xmm0, %xmm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %x = fmul double %a0, %a1
-  %res = fsub double %x, %a2
-  ret double %res
-}
-
-define double @test_x86_fmsub_213_m(double %a0, double %a1, double * %a2_ptr) {
-; GENERIC-LABEL: test_x86_fmsub_213_m:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmulsd %xmm1, %xmm0, %xmm0 # sched: [5:1.00]
-; GENERIC-NEXT:    vsubsd (%rdi), %xmm0, %xmm0 # sched: [9:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_x86_fmsub_213_m:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmulsd %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKX-NEXT:    vsubsd (%rdi), %xmm0, %xmm0 # sched: [9:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %a2 = load double , double *%a2_ptr
-  %x = fmul double %a0, %a1
-  %res = fsub double %x, %a2
-  ret double %res
-}
-
-define double @test_x86_fmsub_231_m(double %a0, double %a1, double * %a2_ptr) {
-; GENERIC-LABEL: test_x86_fmsub_231_m:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmulsd (%rdi), %xmm0, %xmm0 # sched: [11:1.00]
-; GENERIC-NEXT:    vsubsd %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_x86_fmsub_231_m:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmulsd (%rdi), %xmm0, %xmm0 # sched: [9:0.50]
-; SKX-NEXT:    vsubsd %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %a2 = load double , double *%a2_ptr
-  %x = fmul double %a0, %a2
-  %res = fsub double %x, %a1
-  ret double %res
-}
-
-define <16 x float> @test231_br(<16 x float> %a1, <16 x float> %a2) nounwind {
-; GENERIC-LABEL: test231_br:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmulps {{.*}}(%rip){1to16}, %zmm0, %zmm0 # sched: [12:1.00]
-; GENERIC-NEXT:    vaddps %zmm1, %zmm0, %zmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test231_br:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmulps {{.*}}(%rip){1to16}, %zmm0, %zmm0 # sched: [11:0.50]
-; SKX-NEXT:    vaddps %zmm1, %zmm0, %zmm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %b1 = fmul <16 x float> %a1, <float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000>
-  %b2 = fadd <16 x float> %b1, %a2
-  ret <16 x float> %b2
-}
-
-define <16 x float> @test213_br(<16 x float> %a1, <16 x float> %a2) nounwind {
-; GENERIC-LABEL: test213_br:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmulps %zmm1, %zmm0, %zmm0 # sched: [5:1.00]
-; GENERIC-NEXT:    vaddps {{.*}}(%rip){1to16}, %zmm0, %zmm0 # sched: [10:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test213_br:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmulps %zmm1, %zmm0, %zmm0 # sched: [4:0.50]
-; SKX-NEXT:    vaddps {{.*}}(%rip){1to16}, %zmm0, %zmm0 # sched: [11:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %b1 = fmul <16 x float> %a1, %a2
-  %b2 = fadd <16 x float> %b1, <float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000, float 0x3FB99999A0000000>
-  ret <16 x float> %b2
-}
-
-;mask (a*c+b , a)
-define <16 x float> @test_x86_fmadd132_ps(<16 x float> %a0, <16 x float> %a1, <16 x float> *%a2_ptrt, <16 x i1> %mask) {
-; GENERIC-LABEL: test_x86_fmadd132_ps:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpsllw $7, %xmm2, %xmm2 # sched: [1:1.00]
-; GENERIC-NEXT:    vpmovb2m %xmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vmulps (%rdi), %zmm0, %zmm2 # sched: [12:1.00]
-; GENERIC-NEXT:    vaddps %zmm1, %zmm2, %zmm0 {%k1} # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_x86_fmadd132_ps:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpsllw $7, %xmm2, %xmm2 # sched: [1:0.50]
-; SKX-NEXT:    vpmovb2m %xmm2, %k1 # sched: [1:1.00]
-; SKX-NEXT:    vmulps (%rdi), %zmm0, %zmm2 # sched: [11:0.50]
-; SKX-NEXT:    vaddps %zmm1, %zmm2, %zmm0 {%k1} # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %a2   = load <16 x float>,<16 x float> *%a2_ptrt,align 1
-  %x = fmul <16 x float> %a0, %a2
-  %y = fadd <16 x float> %x, %a1
-  %res = select <16 x i1> %mask, <16 x float> %y, <16 x float> %a0
-  ret <16 x float> %res
-}
-
-;mask (a*c+b , b)
-define <16 x float> @test_x86_fmadd231_ps(<16 x float> %a0, <16 x float> %a1, <16 x float> *%a2_ptrt, <16 x i1> %mask) {
-; GENERIC-LABEL: test_x86_fmadd231_ps:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpsllw $7, %xmm2, %xmm2 # sched: [1:1.00]
-; GENERIC-NEXT:    vpmovb2m %xmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vmulps (%rdi), %zmm0, %zmm0 # sched: [12:1.00]
-; GENERIC-NEXT:    vaddps %zmm1, %zmm0, %zmm1 {%k1} # sched: [3:1.00]
-; GENERIC-NEXT:    vmovaps %zmm1, %zmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_x86_fmadd231_ps:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpsllw $7, %xmm2, %xmm2 # sched: [1:0.50]
-; SKX-NEXT:    vpmovb2m %xmm2, %k1 # sched: [1:1.00]
-; SKX-NEXT:    vmulps (%rdi), %zmm0, %zmm0 # sched: [11:0.50]
-; SKX-NEXT:    vaddps %zmm1, %zmm0, %zmm1 {%k1} # sched: [4:0.50]
-; SKX-NEXT:    vmovaps %zmm1, %zmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %a2   = load <16 x float>,<16 x float> *%a2_ptrt,align 1
-  %x = fmul <16 x float> %a0, %a2
-  %y = fadd <16 x float> %x, %a1
-  %res = select <16 x i1> %mask, <16 x float> %y, <16 x float> %a1
-  ret <16 x float> %res
-}
-
-;mask (b*a+c , b)
-define <16 x float> @test_x86_fmadd213_ps(<16 x float> %a0, <16 x float> %a1, <16 x float> *%a2_ptrt, <16 x i1> %mask) {
-; GENERIC-LABEL: test_x86_fmadd213_ps:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpsllw $7, %xmm2, %xmm2 # sched: [1:1.00]
-; GENERIC-NEXT:    vpmovb2m %xmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vmulps %zmm0, %zmm1, %zmm0 # sched: [5:1.00]
-; GENERIC-NEXT:    vaddps (%rdi), %zmm0, %zmm1 {%k1} # sched: [10:1.00]
-; GENERIC-NEXT:    vmovaps %zmm1, %zmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_x86_fmadd213_ps:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpsllw $7, %xmm2, %xmm2 # sched: [1:0.50]
-; SKX-NEXT:    vpmovb2m %xmm2, %k1 # sched: [1:1.00]
-; SKX-NEXT:    vmulps %zmm0, %zmm1, %zmm0 # sched: [4:0.50]
-; SKX-NEXT:    vaddps (%rdi), %zmm0, %zmm1 {%k1} # sched: [11:0.50]
-; SKX-NEXT:    vmovaps %zmm1, %zmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %a2   = load <16 x float>,<16 x float> *%a2_ptrt,align 1
-  %x = fmul <16 x float> %a1, %a0
-  %y = fadd <16 x float> %x, %a2
-  %res = select <16 x i1> %mask, <16 x float> %y, <16 x float> %a1
-  ret <16 x float> %res
-}
-
-define <16 x i32> @vpandd(<16 x i32> %a, <16 x i32> %b) nounwind uwtable readnone ssp {
-; GENERIC-LABEL: vpandd:
-; GENERIC:       # %bb.0: # %entry
-; GENERIC-NEXT:    vpaddd {{.*}}(%rip){1to16}, %zmm0, %zmm0 # sched: [8:0.50]
-; GENERIC-NEXT:    vpandd %zmm1, %zmm0, %zmm0 # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: vpandd:
-; SKX:       # %bb.0: # %entry
-; SKX-NEXT:    vpaddd {{.*}}(%rip){1to16}, %zmm0, %zmm0 # sched: [8:0.50]
-; SKX-NEXT:    vpandd %zmm1, %zmm0, %zmm0 # sched: [1:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-entry:
-  ; Force the execution domain with an add.
-  %a2 = add <16 x i32> %a, <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2,
-                            i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
-  %x = and <16 x i32> %a2, %b
-  ret <16 x i32> %x
-}
-
-define <16 x i32> @vpandnd(<16 x i32> %a, <16 x i32> %b) nounwind uwtable readnone ssp {
-; GENERIC-LABEL: vpandnd:
-; GENERIC:       # %bb.0: # %entry
-; GENERIC-NEXT:    vpaddd {{.*}}(%rip){1to16}, %zmm0, %zmm0 # sched: [8:0.50]
-; GENERIC-NEXT:    vpandnd %zmm0, %zmm1, %zmm0 # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: vpandnd:
-; SKX:       # %bb.0: # %entry
-; SKX-NEXT:    vpaddd {{.*}}(%rip){1to16}, %zmm0, %zmm0 # sched: [8:0.50]
-; SKX-NEXT:    vpandnd %zmm0, %zmm1, %zmm0 # sched: [1:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-entry:
-  ; Force the execution domain with an add.
-  %a2 = add <16 x i32> %a, <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3,
-                            i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
-  %b2 = xor <16 x i32> %b, <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1,
-                            i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1>
-  %x = and <16 x i32> %a2, %b2
-  ret <16 x i32> %x
-}
-
-define <16 x i32> @vpord(<16 x i32> %a, <16 x i32> %b) nounwind uwtable readnone ssp {
-; GENERIC-LABEL: vpord:
-; GENERIC:       # %bb.0: # %entry
-; GENERIC-NEXT:    vpaddd {{.*}}(%rip){1to16}, %zmm0, %zmm0 # sched: [8:0.50]
-; GENERIC-NEXT:    vpord %zmm1, %zmm0, %zmm0 # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: vpord:
-; SKX:       # %bb.0: # %entry
-; SKX-NEXT:    vpaddd {{.*}}(%rip){1to16}, %zmm0, %zmm0 # sched: [8:0.50]
-; SKX-NEXT:    vpord %zmm1, %zmm0, %zmm0 # sched: [1:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-entry:
-  ; Force the execution domain with an add.
-  %a2 = add <16 x i32> %a, <i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4,
-                            i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4>
-  %x = or <16 x i32> %a2, %b
-  ret <16 x i32> %x
-}
-
-define <16 x i32> @vpxord(<16 x i32> %a, <16 x i32> %b) nounwind uwtable readnone ssp {
-; GENERIC-LABEL: vpxord:
-; GENERIC:       # %bb.0: # %entry
-; GENERIC-NEXT:    vpaddd {{.*}}(%rip){1to16}, %zmm0, %zmm0 # sched: [8:0.50]
-; GENERIC-NEXT:    vpxord %zmm1, %zmm0, %zmm0 # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: vpxord:
-; SKX:       # %bb.0: # %entry
-; SKX-NEXT:    vpaddd {{.*}}(%rip){1to16}, %zmm0, %zmm0 # sched: [8:0.50]
-; SKX-NEXT:    vpxord %zmm1, %zmm0, %zmm0 # sched: [1:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-entry:
-  ; Force the execution domain with an add.
-  %a2 = add <16 x i32> %a, <i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5,
-                            i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5>
-  %x = xor <16 x i32> %a2, %b
-  ret <16 x i32> %x
-}
-
-define <8 x i64> @vpandq(<8 x i64> %a, <8 x i64> %b) nounwind uwtable readnone ssp {
-; GENERIC-LABEL: vpandq:
-; GENERIC:       # %bb.0: # %entry
-; GENERIC-NEXT:    vpaddq {{.*}}(%rip){1to8}, %zmm0, %zmm0 # sched: [8:0.50]
-; GENERIC-NEXT:    vpandq %zmm1, %zmm0, %zmm0 # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: vpandq:
-; SKX:       # %bb.0: # %entry
-; SKX-NEXT:    vpaddq {{.*}}(%rip){1to8}, %zmm0, %zmm0 # sched: [8:0.50]
-; SKX-NEXT:    vpandq %zmm1, %zmm0, %zmm0 # sched: [1:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-entry:
-  ; Force the execution domain with an add.
-  %a2 = add <8 x i64> %a, <i64 6, i64 6, i64 6, i64 6, i64 6, i64 6, i64 6, i64 6>
-  %x = and <8 x i64> %a2, %b
-  ret <8 x i64> %x
-}
-
-define <8 x i64> @vpandnq(<8 x i64> %a, <8 x i64> %b) nounwind uwtable readnone ssp {
-; GENERIC-LABEL: vpandnq:
-; GENERIC:       # %bb.0: # %entry
-; GENERIC-NEXT:    vpaddq {{.*}}(%rip){1to8}, %zmm0, %zmm0 # sched: [8:0.50]
-; GENERIC-NEXT:    vpandnq %zmm0, %zmm1, %zmm0 # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: vpandnq:
-; SKX:       # %bb.0: # %entry
-; SKX-NEXT:    vpaddq {{.*}}(%rip){1to8}, %zmm0, %zmm0 # sched: [8:0.50]
-; SKX-NEXT:    vpandnq %zmm0, %zmm1, %zmm0 # sched: [1:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-entry:
-  ; Force the execution domain with an add.
-  %a2 = add <8 x i64> %a, <i64 7, i64 7, i64 7, i64 7, i64 7, i64 7, i64 7, i64 7>
-  %b2 = xor <8 x i64> %b, <i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1>
-  %x = and <8 x i64> %a2, %b2
-  ret <8 x i64> %x
-}
-
-define <8 x i64> @vporq(<8 x i64> %a, <8 x i64> %b) nounwind uwtable readnone ssp {
-; GENERIC-LABEL: vporq:
-; GENERIC:       # %bb.0: # %entry
-; GENERIC-NEXT:    vpaddq {{.*}}(%rip){1to8}, %zmm0, %zmm0 # sched: [8:0.50]
-; GENERIC-NEXT:    vporq %zmm1, %zmm0, %zmm0 # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: vporq:
-; SKX:       # %bb.0: # %entry
-; SKX-NEXT:    vpaddq {{.*}}(%rip){1to8}, %zmm0, %zmm0 # sched: [8:0.50]
-; SKX-NEXT:    vporq %zmm1, %zmm0, %zmm0 # sched: [1:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-entry:
-  ; Force the execution domain with an add.
-  %a2 = add <8 x i64> %a, <i64 8, i64 8, i64 8, i64 8, i64 8, i64 8, i64 8, i64 8>
-  %x = or <8 x i64> %a2, %b
-  ret <8 x i64> %x
-}
-
-define <8 x i64> @vpxorq(<8 x i64> %a, <8 x i64> %b) nounwind uwtable readnone ssp {
-; GENERIC-LABEL: vpxorq:
-; GENERIC:       # %bb.0: # %entry
-; GENERIC-NEXT:    vpaddq {{.*}}(%rip){1to8}, %zmm0, %zmm0 # sched: [8:0.50]
-; GENERIC-NEXT:    vpxorq %zmm1, %zmm0, %zmm0 # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: vpxorq:
-; SKX:       # %bb.0: # %entry
-; SKX-NEXT:    vpaddq {{.*}}(%rip){1to8}, %zmm0, %zmm0 # sched: [8:0.50]
-; SKX-NEXT:    vpxorq %zmm1, %zmm0, %zmm0 # sched: [1:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-entry:
-  ; Force the execution domain with an add.
-  %a2 = add <8 x i64> %a, <i64 9, i64 9, i64 9, i64 9, i64 9, i64 9, i64 9, i64 9>
-  %x = xor <8 x i64> %a2, %b
-  ret <8 x i64> %x
-}
-
-define <64 x i8> @and_v64i8(<64 x i8> %a, <64 x i8> %b) {
-; GENERIC-LABEL: and_v64i8:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vandps %zmm1, %zmm0, %zmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: and_v64i8:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vandps %zmm1, %zmm0, %zmm0 # sched: [1:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %res = and <64 x i8> %a, %b
-  ret <64 x i8> %res
-}
-
-define <64 x i8> @andn_v64i8(<64 x i8> %a, <64 x i8> %b) {
-; GENERIC-LABEL: andn_v64i8:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vandnps %zmm0, %zmm1, %zmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: andn_v64i8:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vandnps %zmm0, %zmm1, %zmm0 # sched: [1:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %b2 = xor <64 x i8> %b, <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1,
-                           i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1,
-                           i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1,
-                           i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>
-  %res = and <64 x i8> %a, %b2
-  ret <64 x i8> %res
-}
-
-define <64 x i8> @or_v64i8(<64 x i8> %a, <64 x i8> %b) {
-; GENERIC-LABEL: or_v64i8:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vorps %zmm1, %zmm0, %zmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: or_v64i8:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vorps %zmm1, %zmm0, %zmm0 # sched: [1:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %res = or <64 x i8> %a, %b
-  ret <64 x i8> %res
-}
-
-define <64 x i8> @xor_v64i8(<64 x i8> %a, <64 x i8> %b) {
-; GENERIC-LABEL: xor_v64i8:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vxorps %zmm1, %zmm0, %zmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: xor_v64i8:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vxorps %zmm1, %zmm0, %zmm0 # sched: [1:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %res = xor <64 x i8> %a, %b
-  ret <64 x i8> %res
-}
-
-define <32 x i16> @and_v32i16(<32 x i16> %a, <32 x i16> %b) {
-; GENERIC-LABEL: and_v32i16:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vandps %zmm1, %zmm0, %zmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: and_v32i16:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vandps %zmm1, %zmm0, %zmm0 # sched: [1:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %res = and <32 x i16> %a, %b
-  ret <32 x i16> %res
-}
-
-define <32 x i16> @andn_v32i16(<32 x i16> %a, <32 x i16> %b) {
-; GENERIC-LABEL: andn_v32i16:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vandnps %zmm0, %zmm1, %zmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: andn_v32i16:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vandnps %zmm0, %zmm1, %zmm0 # sched: [1:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %b2 = xor <32 x i16> %b, <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1,
-                            i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>
-  %res = and <32 x i16> %a, %b2
-  ret <32 x i16> %res
-}
-
-define <32 x i16> @or_v32i16(<32 x i16> %a, <32 x i16> %b) {
-; GENERIC-LABEL: or_v32i16:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vorps %zmm1, %zmm0, %zmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: or_v32i16:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vorps %zmm1, %zmm0, %zmm0 # sched: [1:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %res = or <32 x i16> %a, %b
-  ret <32 x i16> %res
-}
-
-define <32 x i16> @xor_v32i16(<32 x i16> %a, <32 x i16> %b) {
-; GENERIC-LABEL: xor_v32i16:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vxorps %zmm1, %zmm0, %zmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: xor_v32i16:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vxorps %zmm1, %zmm0, %zmm0 # sched: [1:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %res = xor <32 x i16> %a, %b
-  ret <32 x i16> %res
-}
-
-define <16 x float> @masked_and_v16f32(<16 x float> %a, <16 x float> %b, <16 x float> %passThru, i16 %mask, <16 x float> %c) {
-; GENERIC-LABEL: masked_and_v16f32:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    kmovd %edi, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vandps %zmm1, %zmm0, %zmm2 {%k1} # sched: [1:1.00]
-; GENERIC-NEXT:    vaddps %zmm2, %zmm3, %zmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: masked_and_v16f32:
-; SKX:       # %bb.0:
-; SKX-NEXT:    kmovd %edi, %k1 # sched: [1:1.00]
-; SKX-NEXT:    vandps %zmm1, %zmm0, %zmm2 {%k1} # sched: [1:0.50]
-; SKX-NEXT:    vaddps %zmm2, %zmm3, %zmm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %a1 = bitcast <16 x float> %a to <16 x i32>
-  %b1 = bitcast <16 x float> %b to <16 x i32>
-  %passThru1 = bitcast <16 x float> %passThru to <16 x i32>
-  %mask1 = bitcast i16 %mask to <16 x i1>
-  %op = and <16 x i32> %a1, %b1
-  %select = select <16 x i1> %mask1, <16 x i32> %op, <16 x i32> %passThru1
-  %cast = bitcast <16 x i32> %select to <16 x float>
-  %add = fadd <16 x float> %c, %cast
-  ret <16 x float> %add
-}
-
-define <16 x float> @masked_or_v16f32(<16 x float> %a, <16 x float> %b, <16 x float> %passThru, i16 %mask, <16 x float> %c) {
-; GENERIC-LABEL: masked_or_v16f32:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    kmovd %edi, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vandps %zmm1, %zmm0, %zmm2 {%k1} # sched: [1:1.00]
-; GENERIC-NEXT:    vaddps %zmm2, %zmm3, %zmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: masked_or_v16f32:
-; SKX:       # %bb.0:
-; SKX-NEXT:    kmovd %edi, %k1 # sched: [1:1.00]
-; SKX-NEXT:    vandps %zmm1, %zmm0, %zmm2 {%k1} # sched: [1:0.50]
-; SKX-NEXT:    vaddps %zmm2, %zmm3, %zmm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %a1 = bitcast <16 x float> %a to <16 x i32>
-  %b1 = bitcast <16 x float> %b to <16 x i32>
-  %passThru1 = bitcast <16 x float> %passThru to <16 x i32>
-  %mask1 = bitcast i16 %mask to <16 x i1>
-  %op = and <16 x i32> %a1, %b1
-  %select = select <16 x i1> %mask1, <16 x i32> %op, <16 x i32> %passThru1
-  %cast = bitcast <16 x i32> %select to <16 x float>
-  %add = fadd <16 x float> %c, %cast
-  ret <16 x float> %add
-}
-
-define <16 x float> @masked_xor_v16f32(<16 x float> %a, <16 x float> %b, <16 x float> %passThru, i16 %mask, <16 x float> %c) {
-; GENERIC-LABEL: masked_xor_v16f32:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    kmovd %edi, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vandps %zmm1, %zmm0, %zmm2 {%k1} # sched: [1:1.00]
-; GENERIC-NEXT:    vaddps %zmm2, %zmm3, %zmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: masked_xor_v16f32:
-; SKX:       # %bb.0:
-; SKX-NEXT:    kmovd %edi, %k1 # sched: [1:1.00]
-; SKX-NEXT:    vandps %zmm1, %zmm0, %zmm2 {%k1} # sched: [1:0.50]
-; SKX-NEXT:    vaddps %zmm2, %zmm3, %zmm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %a1 = bitcast <16 x float> %a to <16 x i32>
-  %b1 = bitcast <16 x float> %b to <16 x i32>
-  %passThru1 = bitcast <16 x float> %passThru to <16 x i32>
-  %mask1 = bitcast i16 %mask to <16 x i1>
-  %op = and <16 x i32> %a1, %b1
-  %select = select <16 x i1> %mask1, <16 x i32> %op, <16 x i32> %passThru1
-  %cast = bitcast <16 x i32> %select to <16 x float>
-  %add = fadd <16 x float> %c, %cast
-  ret <16 x float> %add
-}
-
-define <8 x double> @masked_and_v8f64(<8 x double> %a, <8 x double> %b, <8 x double> %passThru, i8 %mask, <8 x double> %c) {
-; GENERIC-LABEL: masked_and_v8f64:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    kmovd %edi, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vandpd %zmm1, %zmm0, %zmm2 {%k1} # sched: [1:1.00]
-; GENERIC-NEXT:    vaddpd %zmm2, %zmm3, %zmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: masked_and_v8f64:
-; SKX:       # %bb.0:
-; SKX-NEXT:    kmovd %edi, %k1 # sched: [1:1.00]
-; SKX-NEXT:    vandpd %zmm1, %zmm0, %zmm2 {%k1} # sched: [1:0.50]
-; SKX-NEXT:    vaddpd %zmm2, %zmm3, %zmm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %a1 = bitcast <8 x double> %a to <8 x i64>
-  %b1 = bitcast <8 x double> %b to <8 x i64>
-  %passThru1 = bitcast <8 x double> %passThru to <8 x i64>
-  %mask1 = bitcast i8 %mask to <8 x i1>
-  %op = and <8 x i64> %a1, %b1
-  %select = select <8 x i1> %mask1, <8 x i64> %op, <8 x i64> %passThru1
-  %cast = bitcast <8 x i64> %select to <8 x double>
-  %add = fadd <8 x double> %c, %cast
-  ret <8 x double> %add
-}
-
-define <8 x double> @masked_or_v8f64(<8 x double> %a, <8 x double> %b, <8 x double> %passThru, i8 %mask, <8 x double> %c) {
-; GENERIC-LABEL: masked_or_v8f64:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    kmovd %edi, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vandpd %zmm1, %zmm0, %zmm2 {%k1} # sched: [1:1.00]
-; GENERIC-NEXT:    vaddpd %zmm2, %zmm3, %zmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: masked_or_v8f64:
-; SKX:       # %bb.0:
-; SKX-NEXT:    kmovd %edi, %k1 # sched: [1:1.00]
-; SKX-NEXT:    vandpd %zmm1, %zmm0, %zmm2 {%k1} # sched: [1:0.50]
-; SKX-NEXT:    vaddpd %zmm2, %zmm3, %zmm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %a1 = bitcast <8 x double> %a to <8 x i64>
-  %b1 = bitcast <8 x double> %b to <8 x i64>
-  %passThru1 = bitcast <8 x double> %passThru to <8 x i64>
-  %mask1 = bitcast i8 %mask to <8 x i1>
-  %op = and <8 x i64> %a1, %b1
-  %select = select <8 x i1> %mask1, <8 x i64> %op, <8 x i64> %passThru1
-  %cast = bitcast <8 x i64> %select to <8 x double>
-  %add = fadd <8 x double> %c, %cast
-  ret <8 x double> %add
-}
-
-define <8 x double> @masked_xor_v8f64(<8 x double> %a, <8 x double> %b, <8 x double> %passThru, i8 %mask, <8 x double> %c) {
-; GENERIC-LABEL: masked_xor_v8f64:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    kmovd %edi, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vandpd %zmm1, %zmm0, %zmm2 {%k1} # sched: [1:1.00]
-; GENERIC-NEXT:    vaddpd %zmm2, %zmm3, %zmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: masked_xor_v8f64:
-; SKX:       # %bb.0:
-; SKX-NEXT:    kmovd %edi, %k1 # sched: [1:1.00]
-; SKX-NEXT:    vandpd %zmm1, %zmm0, %zmm2 {%k1} # sched: [1:0.50]
-; SKX-NEXT:    vaddpd %zmm2, %zmm3, %zmm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %a1 = bitcast <8 x double> %a to <8 x i64>
-  %b1 = bitcast <8 x double> %b to <8 x i64>
-  %passThru1 = bitcast <8 x double> %passThru to <8 x i64>
-  %mask1 = bitcast i8 %mask to <8 x i1>
-  %op = and <8 x i64> %a1, %b1
-  %select = select <8 x i1> %mask1, <8 x i64> %op, <8 x i64> %passThru1
-  %cast = bitcast <8 x i64> %select to <8 x double>
-  %add = fadd <8 x double> %c, %cast
-  ret <8 x double> %add
-}
-
-define <8 x i64> @test_mm512_mask_and_epi32(<8 x i64> %__src, i16 zeroext %__k, <8 x i64> %__a, <8 x i64> %__b) {
-; GENERIC-LABEL: test_mm512_mask_and_epi32:
-; GENERIC:       # %bb.0: # %entry
-; GENERIC-NEXT:    kmovd %edi, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vandps %zmm2, %zmm1, %zmm0 {%k1} # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_mm512_mask_and_epi32:
-; SKX:       # %bb.0: # %entry
-; SKX-NEXT:    kmovd %edi, %k1 # sched: [1:1.00]
-; SKX-NEXT:    vandps %zmm2, %zmm1, %zmm0 {%k1} # sched: [1:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-entry:
-  %and1.i.i = and <8 x i64> %__a, %__b
-  %0 = bitcast <8 x i64> %and1.i.i to <16 x i32>
-  %1 = bitcast <8 x i64> %__src to <16 x i32>
-  %2 = bitcast i16 %__k to <16 x i1>
-  %3 = select <16 x i1> %2, <16 x i32> %0, <16 x i32> %1
-  %4 = bitcast <16 x i32> %3 to <8 x i64>
-  ret <8 x i64> %4
-}
-
-define <8 x i64> @test_mm512_mask_or_epi32(<8 x i64> %__src, i16 zeroext %__k, <8 x i64> %__a, <8 x i64> %__b) {
-; GENERIC-LABEL: test_mm512_mask_or_epi32:
-; GENERIC:       # %bb.0: # %entry
-; GENERIC-NEXT:    kmovd %edi, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vorps %zmm2, %zmm1, %zmm0 {%k1} # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_mm512_mask_or_epi32:
-; SKX:       # %bb.0: # %entry
-; SKX-NEXT:    kmovd %edi, %k1 # sched: [1:1.00]
-; SKX-NEXT:    vorps %zmm2, %zmm1, %zmm0 {%k1} # sched: [1:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-entry:
-  %or1.i.i = or <8 x i64> %__a, %__b
-  %0 = bitcast <8 x i64> %or1.i.i to <16 x i32>
-  %1 = bitcast <8 x i64> %__src to <16 x i32>
-  %2 = bitcast i16 %__k to <16 x i1>
-  %3 = select <16 x i1> %2, <16 x i32> %0, <16 x i32> %1
-  %4 = bitcast <16 x i32> %3 to <8 x i64>
-  ret <8 x i64> %4
-}
-
-define <8 x i64> @test_mm512_mask_xor_epi32(<8 x i64> %__src, i16 zeroext %__k, <8 x i64> %__a, <8 x i64> %__b) {
-; GENERIC-LABEL: test_mm512_mask_xor_epi32:
-; GENERIC:       # %bb.0: # %entry
-; GENERIC-NEXT:    kmovd %edi, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vxorps %zmm2, %zmm1, %zmm0 {%k1} # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_mm512_mask_xor_epi32:
-; SKX:       # %bb.0: # %entry
-; SKX-NEXT:    kmovd %edi, %k1 # sched: [1:1.00]
-; SKX-NEXT:    vxorps %zmm2, %zmm1, %zmm0 {%k1} # sched: [1:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-entry:
-  %xor1.i.i = xor <8 x i64> %__a, %__b
-  %0 = bitcast <8 x i64> %xor1.i.i to <16 x i32>
-  %1 = bitcast <8 x i64> %__src to <16 x i32>
-  %2 = bitcast i16 %__k to <16 x i1>
-  %3 = select <16 x i1> %2, <16 x i32> %0, <16 x i32> %1
-  %4 = bitcast <16 x i32> %3 to <8 x i64>
-  ret <8 x i64> %4
-}
-
-define <8 x double> @test_mm512_mask_xor_pd(<8 x double> %__W, i8 zeroext %__U, <8 x double> %__A, <8 x double> %__B) {
-; GENERIC-LABEL: test_mm512_mask_xor_pd:
-; GENERIC:       # %bb.0: # %entry
-; GENERIC-NEXT:    kmovd %edi, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vxorpd %zmm2, %zmm1, %zmm0 {%k1} # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_mm512_mask_xor_pd:
-; SKX:       # %bb.0: # %entry
-; SKX-NEXT:    kmovd %edi, %k1 # sched: [1:1.00]
-; SKX-NEXT:    vxorpd %zmm2, %zmm1, %zmm0 {%k1} # sched: [1:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-entry:
-  %0 = bitcast <8 x double> %__A to <8 x i64>
-  %1 = bitcast <8 x double> %__B to <8 x i64>
-  %xor.i.i = xor <8 x i64> %0, %1
-  %2 = bitcast <8 x i64> %xor.i.i to <8 x double>
-  %3 = bitcast i8 %__U to <8 x i1>
-  %4 = select <8 x i1> %3, <8 x double> %2, <8 x double> %__W
-  ret <8 x double> %4
-}
-
-define <8 x double> @test_mm512_maskz_xor_pd(i8 zeroext %__U, <8 x double> %__A, <8 x double> %__B) {
-; GENERIC-LABEL: test_mm512_maskz_xor_pd:
-; GENERIC:       # %bb.0: # %entry
-; GENERIC-NEXT:    kmovd %edi, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vxorpd %zmm1, %zmm0, %zmm0 {%k1} {z} # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_mm512_maskz_xor_pd:
-; SKX:       # %bb.0: # %entry
-; SKX-NEXT:    kmovd %edi, %k1 # sched: [1:1.00]
-; SKX-NEXT:    vxorpd %zmm1, %zmm0, %zmm0 {%k1} {z} # sched: [1:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-entry:
-  %0 = bitcast <8 x double> %__A to <8 x i64>
-  %1 = bitcast <8 x double> %__B to <8 x i64>
-  %xor.i.i = xor <8 x i64> %0, %1
-  %2 = bitcast <8 x i64> %xor.i.i to <8 x double>
-  %3 = bitcast i8 %__U to <8 x i1>
-  %4 = select <8 x i1> %3, <8 x double> %2, <8 x double> zeroinitializer
-  ret <8 x double> %4
-}
-
-define <16 x float> @test_mm512_mask_xor_ps(<16 x float> %__W, i16 zeroext %__U, <16 x float> %__A, <16 x float> %__B) {
-; GENERIC-LABEL: test_mm512_mask_xor_ps:
-; GENERIC:       # %bb.0: # %entry
-; GENERIC-NEXT:    kmovd %edi, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vxorps %zmm2, %zmm1, %zmm0 {%k1} # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_mm512_mask_xor_ps:
-; SKX:       # %bb.0: # %entry
-; SKX-NEXT:    kmovd %edi, %k1 # sched: [1:1.00]
-; SKX-NEXT:    vxorps %zmm2, %zmm1, %zmm0 {%k1} # sched: [1:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-entry:
-  %0 = bitcast <16 x float> %__A to <16 x i32>
-  %1 = bitcast <16 x float> %__B to <16 x i32>
-  %xor.i.i = xor <16 x i32> %0, %1
-  %2 = bitcast <16 x i32> %xor.i.i to <16 x float>
-  %3 = bitcast i16 %__U to <16 x i1>
-  %4 = select <16 x i1> %3, <16 x float> %2, <16 x float> %__W
-  ret <16 x float> %4
-}
-
-define <16 x float> @test_mm512_maskz_xor_ps(i16 zeroext %__U, <16 x float> %__A, <16 x float> %__B) {
-; GENERIC-LABEL: test_mm512_maskz_xor_ps:
-; GENERIC:       # %bb.0: # %entry
-; GENERIC-NEXT:    kmovd %edi, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vxorps %zmm1, %zmm0, %zmm0 {%k1} {z} # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_mm512_maskz_xor_ps:
-; SKX:       # %bb.0: # %entry
-; SKX-NEXT:    kmovd %edi, %k1 # sched: [1:1.00]
-; SKX-NEXT:    vxorps %zmm1, %zmm0, %zmm0 {%k1} {z} # sched: [1:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-entry:
-  %0 = bitcast <16 x float> %__A to <16 x i32>
-  %1 = bitcast <16 x float> %__B to <16 x i32>
-  %xor.i.i = xor <16 x i32> %0, %1
-  %2 = bitcast <16 x i32> %xor.i.i to <16 x float>
-  %3 = bitcast i16 %__U to <16 x i1>
-  %4 = select <16 x i1> %3, <16 x float> %2, <16 x float> zeroinitializer
-  ret <16 x float> %4
-}
-
-define <8 x double> @test_mm512_mask_or_pd(<8 x double> %__W, i8 zeroext %__U, <8 x double> %__A, <8 x double> %__B) {
-; GENERIC-LABEL: test_mm512_mask_or_pd:
-; GENERIC:       # %bb.0: # %entry
-; GENERIC-NEXT:    kmovd %edi, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vorpd %zmm1, %zmm2, %zmm0 {%k1} # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_mm512_mask_or_pd:
-; SKX:       # %bb.0: # %entry
-; SKX-NEXT:    kmovd %edi, %k1 # sched: [1:1.00]
-; SKX-NEXT:    vorpd %zmm1, %zmm2, %zmm0 {%k1} # sched: [1:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-entry:
-  %0 = bitcast <8 x double> %__A to <8 x i64>
-  %1 = bitcast <8 x double> %__B to <8 x i64>
-  %or.i.i = or <8 x i64> %1, %0
-  %2 = bitcast <8 x i64> %or.i.i to <8 x double>
-  %3 = bitcast i8 %__U to <8 x i1>
-  %4 = select <8 x i1> %3, <8 x double> %2, <8 x double> %__W
-  ret <8 x double> %4
-}
-
-define <8 x double> @test_mm512_maskz_or_pd(i8 zeroext %__U, <8 x double> %__A, <8 x double> %__B) {
-; GENERIC-LABEL: test_mm512_maskz_or_pd:
-; GENERIC:       # %bb.0: # %entry
-; GENERIC-NEXT:    kmovd %edi, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vorpd %zmm0, %zmm1, %zmm0 {%k1} {z} # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_mm512_maskz_or_pd:
-; SKX:       # %bb.0: # %entry
-; SKX-NEXT:    kmovd %edi, %k1 # sched: [1:1.00]
-; SKX-NEXT:    vorpd %zmm0, %zmm1, %zmm0 {%k1} {z} # sched: [1:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-entry:
-  %0 = bitcast <8 x double> %__A to <8 x i64>
-  %1 = bitcast <8 x double> %__B to <8 x i64>
-  %or.i.i = or <8 x i64> %1, %0
-  %2 = bitcast <8 x i64> %or.i.i to <8 x double>
-  %3 = bitcast i8 %__U to <8 x i1>
-  %4 = select <8 x i1> %3, <8 x double> %2, <8 x double> zeroinitializer
-  ret <8 x double> %4
-}
-
-define <16 x float> @test_mm512_mask_or_ps(<16 x float> %__W, i16 zeroext %__U, <16 x float> %__A, <16 x float> %__B) {
-; GENERIC-LABEL: test_mm512_mask_or_ps:
-; GENERIC:       # %bb.0: # %entry
-; GENERIC-NEXT:    kmovd %edi, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vorps %zmm1, %zmm2, %zmm0 {%k1} # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_mm512_mask_or_ps:
-; SKX:       # %bb.0: # %entry
-; SKX-NEXT:    kmovd %edi, %k1 # sched: [1:1.00]
-; SKX-NEXT:    vorps %zmm1, %zmm2, %zmm0 {%k1} # sched: [1:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-entry:
-  %0 = bitcast <16 x float> %__A to <16 x i32>
-  %1 = bitcast <16 x float> %__B to <16 x i32>
-  %or.i.i = or <16 x i32> %1, %0
-  %2 = bitcast <16 x i32> %or.i.i to <16 x float>
-  %3 = bitcast i16 %__U to <16 x i1>
-  %4 = select <16 x i1> %3, <16 x float> %2, <16 x float> %__W
-  ret <16 x float> %4
-}
-
-define <16 x float> @test_mm512_maskz_or_ps(i16 zeroext %__U, <16 x float> %__A, <16 x float> %__B) {
-; GENERIC-LABEL: test_mm512_maskz_or_ps:
-; GENERIC:       # %bb.0: # %entry
-; GENERIC-NEXT:    kmovd %edi, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vorps %zmm0, %zmm1, %zmm0 {%k1} {z} # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_mm512_maskz_or_ps:
-; SKX:       # %bb.0: # %entry
-; SKX-NEXT:    kmovd %edi, %k1 # sched: [1:1.00]
-; SKX-NEXT:    vorps %zmm0, %zmm1, %zmm0 {%k1} {z} # sched: [1:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-entry:
-  %0 = bitcast <16 x float> %__A to <16 x i32>
-  %1 = bitcast <16 x float> %__B to <16 x i32>
-  %or.i.i = or <16 x i32> %1, %0
-  %2 = bitcast <16 x i32> %or.i.i to <16 x float>
-  %3 = bitcast i16 %__U to <16 x i1>
-  %4 = select <16 x i1> %3, <16 x float> %2, <16 x float> zeroinitializer
-  ret <16 x float> %4
-}
-
-define <8 x double> @test_mm512_mask_and_pd(<8 x double> %__W, i8 zeroext %__U, <8 x double> %__A, <8 x double> %__B) {
-; GENERIC-LABEL: test_mm512_mask_and_pd:
-; GENERIC:       # %bb.0: # %entry
-; GENERIC-NEXT:    kmovd %edi, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vandpd %zmm1, %zmm2, %zmm0 {%k1} # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_mm512_mask_and_pd:
-; SKX:       # %bb.0: # %entry
-; SKX-NEXT:    kmovd %edi, %k1 # sched: [1:1.00]
-; SKX-NEXT:    vandpd %zmm1, %zmm2, %zmm0 {%k1} # sched: [1:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-entry:
-  %0 = bitcast <8 x double> %__A to <8 x i64>
-  %1 = bitcast <8 x double> %__B to <8 x i64>
-  %and.i.i = and <8 x i64> %1, %0
-  %2 = bitcast <8 x i64> %and.i.i to <8 x double>
-  %3 = bitcast i8 %__U to <8 x i1>
-  %4 = select <8 x i1> %3, <8 x double> %2, <8 x double> %__W
-  ret <8 x double> %4
-}
-
-define <8 x double> @test_mm512_maskz_and_pd(i8 zeroext %__U, <8 x double> %__A, <8 x double> %__B) {
-; GENERIC-LABEL: test_mm512_maskz_and_pd:
-; GENERIC:       # %bb.0: # %entry
-; GENERIC-NEXT:    kmovd %edi, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vandpd %zmm0, %zmm1, %zmm0 {%k1} {z} # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_mm512_maskz_and_pd:
-; SKX:       # %bb.0: # %entry
-; SKX-NEXT:    kmovd %edi, %k1 # sched: [1:1.00]
-; SKX-NEXT:    vandpd %zmm0, %zmm1, %zmm0 {%k1} {z} # sched: [1:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-entry:
-  %0 = bitcast <8 x double> %__A to <8 x i64>
-  %1 = bitcast <8 x double> %__B to <8 x i64>
-  %and.i.i = and <8 x i64> %1, %0
-  %2 = bitcast <8 x i64> %and.i.i to <8 x double>
-  %3 = bitcast i8 %__U to <8 x i1>
-  %4 = select <8 x i1> %3, <8 x double> %2, <8 x double> zeroinitializer
-  ret <8 x double> %4
-}
-
-define <16 x float> @test_mm512_mask_and_ps(<16 x float> %__W, i16 zeroext %__U, <16 x float> %__A, <16 x float> %__B) {
-; GENERIC-LABEL: test_mm512_mask_and_ps:
-; GENERIC:       # %bb.0: # %entry
-; GENERIC-NEXT:    kmovd %edi, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vandps %zmm1, %zmm2, %zmm0 {%k1} # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_mm512_mask_and_ps:
-; SKX:       # %bb.0: # %entry
-; SKX-NEXT:    kmovd %edi, %k1 # sched: [1:1.00]
-; SKX-NEXT:    vandps %zmm1, %zmm2, %zmm0 {%k1} # sched: [1:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-entry:
-  %0 = bitcast <16 x float> %__A to <16 x i32>
-  %1 = bitcast <16 x float> %__B to <16 x i32>
-  %and.i.i = and <16 x i32> %1, %0
-  %2 = bitcast <16 x i32> %and.i.i to <16 x float>
-  %3 = bitcast i16 %__U to <16 x i1>
-  %4 = select <16 x i1> %3, <16 x float> %2, <16 x float> %__W
-  ret <16 x float> %4
-}
-
-define <16 x float> @test_mm512_maskz_and_ps(i16 zeroext %__U, <16 x float> %__A, <16 x float> %__B) {
-; GENERIC-LABEL: test_mm512_maskz_and_ps:
-; GENERIC:       # %bb.0: # %entry
-; GENERIC-NEXT:    kmovd %edi, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vandps %zmm0, %zmm1, %zmm0 {%k1} {z} # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_mm512_maskz_and_ps:
-; SKX:       # %bb.0: # %entry
-; SKX-NEXT:    kmovd %edi, %k1 # sched: [1:1.00]
-; SKX-NEXT:    vandps %zmm0, %zmm1, %zmm0 {%k1} {z} # sched: [1:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-entry:
-  %0 = bitcast <16 x float> %__A to <16 x i32>
-  %1 = bitcast <16 x float> %__B to <16 x i32>
-  %and.i.i = and <16 x i32> %1, %0
-  %2 = bitcast <16 x i32> %and.i.i to <16 x float>
-  %3 = bitcast i16 %__U to <16 x i1>
-  %4 = select <16 x i1> %3, <16 x float> %2, <16 x float> zeroinitializer
-  ret <16 x float> %4
-}
-
-define <8 x double> @test_mm512_mask_andnot_pd(<8 x double> %__W, i8 zeroext %__U, <8 x double> %__A, <8 x double> %__B) {
-; GENERIC-LABEL: test_mm512_mask_andnot_pd:
-; GENERIC:       # %bb.0: # %entry
-; GENERIC-NEXT:    kmovd %edi, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vandnpd %zmm2, %zmm1, %zmm0 {%k1} # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_mm512_mask_andnot_pd:
-; SKX:       # %bb.0: # %entry
-; SKX-NEXT:    kmovd %edi, %k1 # sched: [1:1.00]
-; SKX-NEXT:    vandnpd %zmm2, %zmm1, %zmm0 {%k1} # sched: [1:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-entry:
-  %0 = bitcast <8 x double> %__A to <8 x i64>
-  %neg.i.i = xor <8 x i64> %0, <i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1>
-  %1 = bitcast <8 x double> %__B to <8 x i64>
-  %and.i.i = and <8 x i64> %1, %neg.i.i
-  %2 = bitcast <8 x i64> %and.i.i to <8 x double>
-  %3 = bitcast i8 %__U to <8 x i1>
-  %4 = select <8 x i1> %3, <8 x double> %2, <8 x double> %__W
-  ret <8 x double> %4
-}
-
-define <8 x double> @test_mm512_maskz_andnot_pd(i8 zeroext %__U, <8 x double> %__A, <8 x double> %__B) {
-; GENERIC-LABEL: test_mm512_maskz_andnot_pd:
-; GENERIC:       # %bb.0: # %entry
-; GENERIC-NEXT:    kmovd %edi, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vandnpd %zmm1, %zmm0, %zmm0 {%k1} {z} # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_mm512_maskz_andnot_pd:
-; SKX:       # %bb.0: # %entry
-; SKX-NEXT:    kmovd %edi, %k1 # sched: [1:1.00]
-; SKX-NEXT:    vandnpd %zmm1, %zmm0, %zmm0 {%k1} {z} # sched: [1:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-entry:
-  %0 = bitcast <8 x double> %__A to <8 x i64>
-  %neg.i.i = xor <8 x i64> %0, <i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1>
-  %1 = bitcast <8 x double> %__B to <8 x i64>
-  %and.i.i = and <8 x i64> %1, %neg.i.i
-  %2 = bitcast <8 x i64> %and.i.i to <8 x double>
-  %3 = bitcast i8 %__U to <8 x i1>
-  %4 = select <8 x i1> %3, <8 x double> %2, <8 x double> zeroinitializer
-  ret <8 x double> %4
-}
-
-define <16 x float> @test_mm512_mask_andnot_ps(<16 x float> %__W, i16 zeroext %__U, <16 x float> %__A, <16 x float> %__B) {
-; GENERIC-LABEL: test_mm512_mask_andnot_ps:
-; GENERIC:       # %bb.0: # %entry
-; GENERIC-NEXT:    kmovd %edi, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vandnps %zmm2, %zmm1, %zmm0 {%k1} # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_mm512_mask_andnot_ps:
-; SKX:       # %bb.0: # %entry
-; SKX-NEXT:    kmovd %edi, %k1 # sched: [1:1.00]
-; SKX-NEXT:    vandnps %zmm2, %zmm1, %zmm0 {%k1} # sched: [1:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-entry:
-  %0 = bitcast <16 x float> %__A to <16 x i32>
-  %neg.i.i = xor <16 x i32> %0, <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1>
-  %1 = bitcast <16 x float> %__B to <16 x i32>
-  %and.i.i = and <16 x i32> %1, %neg.i.i
-  %2 = bitcast <16 x i32> %and.i.i to <16 x float>
-  %3 = bitcast i16 %__U to <16 x i1>
-  %4 = select <16 x i1> %3, <16 x float> %2, <16 x float> %__W
-  ret <16 x float> %4
-}
-
-define <16 x float> @test_mm512_maskz_andnot_ps(i16 zeroext %__U, <16 x float> %__A, <16 x float> %__B) {
-; GENERIC-LABEL: test_mm512_maskz_andnot_ps:
-; GENERIC:       # %bb.0: # %entry
-; GENERIC-NEXT:    kmovd %edi, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vandnps %zmm1, %zmm0, %zmm0 {%k1} {z} # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_mm512_maskz_andnot_ps:
-; SKX:       # %bb.0: # %entry
-; SKX-NEXT:    kmovd %edi, %k1 # sched: [1:1.00]
-; SKX-NEXT:    vandnps %zmm1, %zmm0, %zmm0 {%k1} {z} # sched: [1:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-entry:
-  %0 = bitcast <16 x float> %__A to <16 x i32>
-  %neg.i.i = xor <16 x i32> %0, <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1>
-  %1 = bitcast <16 x float> %__B to <16 x i32>
-  %and.i.i = and <16 x i32> %1, %neg.i.i
-  %2 = bitcast <16 x i32> %and.i.i to <16 x float>
-  %3 = bitcast i16 %__U to <16 x i1>
-  %4 = select <16 x i1> %3, <16 x float> %2, <16 x float> zeroinitializer
-  ret <16 x float> %4
-}
-
-define i32 @mov_test1(float %x) {
-; GENERIC-LABEL: mov_test1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovd %xmm0, %eax # sched: [2:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: mov_test1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovd %xmm0, %eax # sched: [2:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-   %res = bitcast float %x to i32
-   ret i32 %res
-}
-
-define <4 x i32> @mov_test2(i32 %x) {
-; GENERIC-LABEL: mov_test2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovd %edi, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: mov_test2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovd %edi, %xmm0 # sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-   %res = insertelement <4 x i32>undef, i32 %x, i32 0
-   ret <4 x i32>%res
-}
-
-define <2 x i64> @mov_test3(i64 %x) {
-; GENERIC-LABEL: mov_test3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovq %rdi, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: mov_test3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovq %rdi, %xmm0 # sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-   %res = insertelement <2 x i64>undef, i64 %x, i32 0
-   ret <2 x i64>%res
-}
-
-define <4 x i32> @mov_test4(i32* %x) {
-; GENERIC-LABEL: mov_test4:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero sched: [6:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: mov_test4:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero sched: [5:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-   %y = load i32, i32* %x
-   %res = insertelement <4 x i32>undef, i32 %y, i32 0
-   ret <4 x i32>%res
-}
-
-define void @mov_test5(float %x, float* %y) {
-; GENERIC-LABEL: mov_test5:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovss %xmm0, (%rdi) # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: mov_test5:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovss %xmm0, (%rdi) # sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-   store float %x, float* %y, align 4
-   ret void
-}
-
-define void @mov_test6(double %x, double* %y) {
-; GENERIC-LABEL: mov_test6:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovsd %xmm0, (%rdi) # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: mov_test6:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovsd %xmm0, (%rdi) # sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-   store double %x, double* %y, align 8
-   ret void
-}
-
-define float @mov_test7(i32* %x) {
-; GENERIC-LABEL: mov_test7:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero sched: [6:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: mov_test7:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero sched: [5:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-   %y = load i32, i32* %x
-   %res = bitcast i32 %y to float
-   ret float %res
-}
-
-define i32 @mov_test8(<4 x i32> %x) {
-; GENERIC-LABEL: mov_test8:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovd %xmm0, %eax # sched: [2:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: mov_test8:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovd %xmm0, %eax # sched: [2:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-   %res = extractelement <4 x i32> %x, i32 0
-   ret i32 %res
-}
-
-define i64 @mov_test9(<2 x i64> %x) {
-; GENERIC-LABEL: mov_test9:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovq %xmm0, %rax # sched: [2:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: mov_test9:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovq %xmm0, %rax # sched: [2:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-   %res = extractelement <2 x i64> %x, i32 0
-   ret i64 %res
-}
-
-define <4 x i32> @mov_test10(i32* %x) {
-; GENERIC-LABEL: mov_test10:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero sched: [6:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: mov_test10:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero sched: [5:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-   %y = load i32, i32* %x, align 4
-   %res = insertelement <4 x i32>zeroinitializer, i32 %y, i32 0
-   ret <4 x i32>%res
-}
-
-define <4 x float> @mov_test11(float* %x) {
-; GENERIC-LABEL: mov_test11:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero sched: [6:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: mov_test11:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero sched: [5:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-   %y = load float, float* %x, align 4
-   %res = insertelement <4 x float>zeroinitializer, float %y, i32 0
-   ret <4 x float>%res
-}
-
-define <2 x double> @mov_test12(double* %x) {
-; GENERIC-LABEL: mov_test12:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero sched: [6:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: mov_test12:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero sched: [5:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-   %y = load double, double* %x, align 8
-   %res = insertelement <2 x double>zeroinitializer, double %y, i32 0
-   ret <2 x double>%res
-}
-
-define <2 x i64> @mov_test13(i64 %x) {
-; GENERIC-LABEL: mov_test13:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovq %rdi, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: mov_test13:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovq %rdi, %xmm0 # sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-   %res = insertelement <2 x i64>zeroinitializer, i64 %x, i32 0
-   ret <2 x i64>%res
-}
-
-define <4 x i32> @mov_test14(i32 %x) {
-; GENERIC-LABEL: mov_test14:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovd %edi, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: mov_test14:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovd %edi, %xmm0 # sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-   %res = insertelement <4 x i32>zeroinitializer, i32 %x, i32 0
-   ret <4 x i32>%res
-}
-
-define <4 x i32> @mov_test15(i32* %x) {
-; GENERIC-LABEL: mov_test15:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero sched: [6:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: mov_test15:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero sched: [5:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-   %y = load i32, i32* %x, align 4
-   %res = insertelement <4 x i32>zeroinitializer, i32 %y, i32 0
-   ret <4 x i32>%res
-}
-
-define <16 x i32> @mov_test16(i8 * %addr) {
-; GENERIC-LABEL: mov_test16:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovups (%rdi), %zmm0 # sched: [7:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: mov_test16:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovups (%rdi), %zmm0 # sched: [8:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vaddr = bitcast i8* %addr to <16 x i32>*
-  %res = load <16 x i32>, <16 x i32>* %vaddr, align 1
-  ret <16 x i32>%res
-}
-
-define <16 x i32> @mov_test17(i8 * %addr) {
-; GENERIC-LABEL: mov_test17:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovaps (%rdi), %zmm0 # sched: [7:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: mov_test17:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovaps (%rdi), %zmm0 # sched: [8:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vaddr = bitcast i8* %addr to <16 x i32>*
-  %res = load <16 x i32>, <16 x i32>* %vaddr, align 64
-  ret <16 x i32>%res
-}
-
-define void @mov_test18(i8 * %addr, <8 x i64> %data) {
-; GENERIC-LABEL: mov_test18:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovaps %zmm0, (%rdi) # sched: [1:1.00]
-; GENERIC-NEXT:    vzeroupper # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: mov_test18:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovaps %zmm0, (%rdi) # sched: [1:1.00]
-; SKX-NEXT:    vzeroupper # sched: [0:0.67]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vaddr = bitcast i8* %addr to <8 x i64>*
-  store <8 x i64>%data, <8 x i64>* %vaddr, align 64
-  ret void
-}
-
-define void @mov_test19(i8 * %addr, <16 x i32> %data) {
-; GENERIC-LABEL: mov_test19:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovups %zmm0, (%rdi) # sched: [1:1.00]
-; GENERIC-NEXT:    vzeroupper # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: mov_test19:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovups %zmm0, (%rdi) # sched: [1:1.00]
-; SKX-NEXT:    vzeroupper # sched: [0:0.67]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vaddr = bitcast i8* %addr to <16 x i32>*
-  store <16 x i32>%data, <16 x i32>* %vaddr, align 1
-  ret void
-}
-
-define void @mov_test20(i8 * %addr, <16 x i32> %data) {
-; GENERIC-LABEL: mov_test20:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovaps %zmm0, (%rdi) # sched: [1:1.00]
-; GENERIC-NEXT:    vzeroupper # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: mov_test20:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovaps %zmm0, (%rdi) # sched: [1:1.00]
-; SKX-NEXT:    vzeroupper # sched: [0:0.67]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vaddr = bitcast i8* %addr to <16 x i32>*
-  store <16 x i32>%data, <16 x i32>* %vaddr, align 64
-  ret void
-}
-
-define  <8 x i64> @mov_test21(i8 * %addr) {
-; GENERIC-LABEL: mov_test21:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovaps (%rdi), %zmm0 # sched: [7:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: mov_test21:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovaps (%rdi), %zmm0 # sched: [8:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vaddr = bitcast i8* %addr to <8 x i64>*
-  %res = load <8 x i64>, <8 x i64>* %vaddr, align 64
-  ret <8 x i64>%res
-}
-
-define void @mov_test22(i8 * %addr, <8 x i64> %data) {
-; GENERIC-LABEL: mov_test22:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovups %zmm0, (%rdi) # sched: [1:1.00]
-; GENERIC-NEXT:    vzeroupper # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: mov_test22:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovups %zmm0, (%rdi) # sched: [1:1.00]
-; SKX-NEXT:    vzeroupper # sched: [0:0.67]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vaddr = bitcast i8* %addr to <8 x i64>*
-  store <8 x i64>%data, <8 x i64>* %vaddr, align 1
-  ret void
-}
-
-define <8 x i64> @mov_test23(i8 * %addr) {
-; GENERIC-LABEL: mov_test23:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovups (%rdi), %zmm0 # sched: [7:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: mov_test23:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovups (%rdi), %zmm0 # sched: [8:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vaddr = bitcast i8* %addr to <8 x i64>*
-  %res = load <8 x i64>, <8 x i64>* %vaddr, align 1
-  ret <8 x i64>%res
-}
-
-define void @mov_test24(i8 * %addr, <8 x double> %data) {
-; GENERIC-LABEL: mov_test24:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovaps %zmm0, (%rdi) # sched: [1:1.00]
-; GENERIC-NEXT:    vzeroupper # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: mov_test24:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovaps %zmm0, (%rdi) # sched: [1:1.00]
-; SKX-NEXT:    vzeroupper # sched: [0:0.67]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vaddr = bitcast i8* %addr to <8 x double>*
-  store <8 x double>%data, <8 x double>* %vaddr, align 64
-  ret void
-}
-
-define <8 x double> @mov_test25(i8 * %addr) {
-; GENERIC-LABEL: mov_test25:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovaps (%rdi), %zmm0 # sched: [7:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: mov_test25:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovaps (%rdi), %zmm0 # sched: [8:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vaddr = bitcast i8* %addr to <8 x double>*
-  %res = load <8 x double>, <8 x double>* %vaddr, align 64
-  ret <8 x double>%res
-}
-
-define void @mov_test26(i8 * %addr, <16 x float> %data) {
-; GENERIC-LABEL: mov_test26:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovaps %zmm0, (%rdi) # sched: [1:1.00]
-; GENERIC-NEXT:    vzeroupper # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: mov_test26:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovaps %zmm0, (%rdi) # sched: [1:1.00]
-; SKX-NEXT:    vzeroupper # sched: [0:0.67]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vaddr = bitcast i8* %addr to <16 x float>*
-  store <16 x float>%data, <16 x float>* %vaddr, align 64
-  ret void
-}
-
-define <16 x float> @mov_test27(i8 * %addr) {
-; GENERIC-LABEL: mov_test27:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovaps (%rdi), %zmm0 # sched: [7:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: mov_test27:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovaps (%rdi), %zmm0 # sched: [8:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vaddr = bitcast i8* %addr to <16 x float>*
-  %res = load <16 x float>, <16 x float>* %vaddr, align 64
-  ret <16 x float>%res
-}
-
-define void @mov_test28(i8 * %addr, <8 x double> %data) {
-; GENERIC-LABEL: mov_test28:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovups %zmm0, (%rdi) # sched: [1:1.00]
-; GENERIC-NEXT:    vzeroupper # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: mov_test28:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovups %zmm0, (%rdi) # sched: [1:1.00]
-; SKX-NEXT:    vzeroupper # sched: [0:0.67]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vaddr = bitcast i8* %addr to <8 x double>*
-  store <8 x double>%data, <8 x double>* %vaddr, align 1
-  ret void
-}
-
-define <8 x double> @mov_test29(i8 * %addr) {
-; GENERIC-LABEL: mov_test29:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovups (%rdi), %zmm0 # sched: [7:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: mov_test29:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovups (%rdi), %zmm0 # sched: [8:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vaddr = bitcast i8* %addr to <8 x double>*
-  %res = load <8 x double>, <8 x double>* %vaddr, align 1
-  ret <8 x double>%res
-}
-
-define void @mov_test30(i8 * %addr, <16 x float> %data) {
-; GENERIC-LABEL: mov_test30:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovups %zmm0, (%rdi) # sched: [1:1.00]
-; GENERIC-NEXT:    vzeroupper # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: mov_test30:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovups %zmm0, (%rdi) # sched: [1:1.00]
-; SKX-NEXT:    vzeroupper # sched: [0:0.67]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vaddr = bitcast i8* %addr to <16 x float>*
-  store <16 x float>%data, <16 x float>* %vaddr, align 1
-  ret void
-}
-
-define <16 x float> @mov_test31(i8 * %addr) {
-; GENERIC-LABEL: mov_test31:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovups (%rdi), %zmm0 # sched: [7:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: mov_test31:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovups (%rdi), %zmm0 # sched: [8:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vaddr = bitcast i8* %addr to <16 x float>*
-  %res = load <16 x float>, <16 x float>* %vaddr, align 1
-  ret <16 x float>%res
-}
-
-define <16 x i32> @mov_test32(i8 * %addr, <16 x i32> %old, <16 x i32> %mask1) {
-; GENERIC-LABEL: mov_test32:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestmd %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vmovdqa32 (%rdi), %zmm0 {%k1} # sched: [7:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: mov_test32:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestmd %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vmovdqa32 (%rdi), %zmm0 {%k1} # sched: [8:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %mask = icmp ne <16 x i32> %mask1, zeroinitializer
-  %vaddr = bitcast i8* %addr to <16 x i32>*
-  %r = load <16 x i32>, <16 x i32>* %vaddr, align 64
-  %res = select <16 x i1> %mask, <16 x i32> %r, <16 x i32> %old
-  ret <16 x i32>%res
-}
-
-define <16 x i32> @mov_test33(i8 * %addr, <16 x i32> %old, <16 x i32> %mask1) {
-; GENERIC-LABEL: mov_test33:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestmd %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vmovdqu32 (%rdi), %zmm0 {%k1} # sched: [7:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: mov_test33:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestmd %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vmovdqu32 (%rdi), %zmm0 {%k1} # sched: [8:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %mask = icmp ne <16 x i32> %mask1, zeroinitializer
-  %vaddr = bitcast i8* %addr to <16 x i32>*
-  %r = load <16 x i32>, <16 x i32>* %vaddr, align 1
-  %res = select <16 x i1> %mask, <16 x i32> %r, <16 x i32> %old
-  ret <16 x i32>%res
-}
-
-define <16 x i32> @mov_test34(i8 * %addr, <16 x i32> %mask1) {
-; GENERIC-LABEL: mov_test34:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestmd %zmm0, %zmm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vmovdqa32 (%rdi), %zmm0 {%k1} {z} # sched: [7:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: mov_test34:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestmd %zmm0, %zmm0, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vmovdqa32 (%rdi), %zmm0 {%k1} {z} # sched: [8:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %mask = icmp ne <16 x i32> %mask1, zeroinitializer
-  %vaddr = bitcast i8* %addr to <16 x i32>*
-  %r = load <16 x i32>, <16 x i32>* %vaddr, align 64
-  %res = select <16 x i1> %mask, <16 x i32> %r, <16 x i32> zeroinitializer
-  ret <16 x i32>%res
-}
-
-define <16 x i32> @mov_test35(i8 * %addr, <16 x i32> %mask1) {
-; GENERIC-LABEL: mov_test35:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestmd %zmm0, %zmm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vmovdqu32 (%rdi), %zmm0 {%k1} {z} # sched: [7:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: mov_test35:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestmd %zmm0, %zmm0, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vmovdqu32 (%rdi), %zmm0 {%k1} {z} # sched: [8:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %mask = icmp ne <16 x i32> %mask1, zeroinitializer
-  %vaddr = bitcast i8* %addr to <16 x i32>*
-  %r = load <16 x i32>, <16 x i32>* %vaddr, align 1
-  %res = select <16 x i1> %mask, <16 x i32> %r, <16 x i32> zeroinitializer
-  ret <16 x i32>%res
-}
-
-define <8 x i64> @mov_test36(i8 * %addr, <8 x i64> %old, <8 x i64> %mask1) {
-; GENERIC-LABEL: mov_test36:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestmq %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vmovdqa64 (%rdi), %zmm0 {%k1} # sched: [7:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: mov_test36:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestmq %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vmovdqa64 (%rdi), %zmm0 {%k1} # sched: [8:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %mask = icmp ne <8 x i64> %mask1, zeroinitializer
-  %vaddr = bitcast i8* %addr to <8 x i64>*
-  %r = load <8 x i64>, <8 x i64>* %vaddr, align 64
-  %res = select <8 x i1> %mask, <8 x i64> %r, <8 x i64> %old
-  ret <8 x i64>%res
-}
-
-define <8 x i64> @mov_test37(i8 * %addr, <8 x i64> %old, <8 x i64> %mask1) {
-; GENERIC-LABEL: mov_test37:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestmq %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vmovdqu64 (%rdi), %zmm0 {%k1} # sched: [7:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: mov_test37:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestmq %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vmovdqu64 (%rdi), %zmm0 {%k1} # sched: [8:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %mask = icmp ne <8 x i64> %mask1, zeroinitializer
-  %vaddr = bitcast i8* %addr to <8 x i64>*
-  %r = load <8 x i64>, <8 x i64>* %vaddr, align 1
-  %res = select <8 x i1> %mask, <8 x i64> %r, <8 x i64> %old
-  ret <8 x i64>%res
-}
-
-define <8 x i64> @mov_test38(i8 * %addr, <8 x i64> %mask1) {
-; GENERIC-LABEL: mov_test38:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestmq %zmm0, %zmm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vmovdqa64 (%rdi), %zmm0 {%k1} {z} # sched: [7:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: mov_test38:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestmq %zmm0, %zmm0, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vmovdqa64 (%rdi), %zmm0 {%k1} {z} # sched: [8:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %mask = icmp ne <8 x i64> %mask1, zeroinitializer
-  %vaddr = bitcast i8* %addr to <8 x i64>*
-  %r = load <8 x i64>, <8 x i64>* %vaddr, align 64
-  %res = select <8 x i1> %mask, <8 x i64> %r, <8 x i64> zeroinitializer
-  ret <8 x i64>%res
-}
-
-define <8 x i64> @mov_test39(i8 * %addr, <8 x i64> %mask1) {
-; GENERIC-LABEL: mov_test39:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestmq %zmm0, %zmm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vmovdqu64 (%rdi), %zmm0 {%k1} {z} # sched: [7:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: mov_test39:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestmq %zmm0, %zmm0, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vmovdqu64 (%rdi), %zmm0 {%k1} {z} # sched: [8:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %mask = icmp ne <8 x i64> %mask1, zeroinitializer
-  %vaddr = bitcast i8* %addr to <8 x i64>*
-  %r = load <8 x i64>, <8 x i64>* %vaddr, align 1
-  %res = select <8 x i1> %mask, <8 x i64> %r, <8 x i64> zeroinitializer
-  ret <8 x i64>%res
-}
-
-define <16 x float> @mov_test40(i8 * %addr, <16 x float> %old, <16 x float> %mask1) {
-; GENERIC-LABEL: mov_test40:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vxorps %xmm2, %xmm2, %xmm2 # sched: [0:0.25]
-; GENERIC-NEXT:    vcmpneq_oqps %zmm2, %zmm1, %k1 # sched: [3:1.00]
-; GENERIC-NEXT:    vmovaps (%rdi), %zmm0 {%k1} # sched: [7:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: mov_test40:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vxorps %xmm2, %xmm2, %xmm2 # sched: [1:0.33]
-; SKX-NEXT:    vcmpneq_oqps %zmm2, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vmovaps (%rdi), %zmm0 {%k1} # sched: [8:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %mask = fcmp one <16 x float> %mask1, zeroinitializer
-  %vaddr = bitcast i8* %addr to <16 x float>*
-  %r = load <16 x float>, <16 x float>* %vaddr, align 64
-  %res = select <16 x i1> %mask, <16 x float> %r, <16 x float> %old
-  ret <16 x float>%res
-}
-
-define <16 x float> @mov_test41(i8 * %addr, <16 x float> %old, <16 x float> %mask1) {
-; GENERIC-LABEL: mov_test41:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vxorps %xmm2, %xmm2, %xmm2 # sched: [0:0.25]
-; GENERIC-NEXT:    vcmpneq_oqps %zmm2, %zmm1, %k1 # sched: [3:1.00]
-; GENERIC-NEXT:    vmovups (%rdi), %zmm0 {%k1} # sched: [7:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: mov_test41:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vxorps %xmm2, %xmm2, %xmm2 # sched: [1:0.33]
-; SKX-NEXT:    vcmpneq_oqps %zmm2, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vmovups (%rdi), %zmm0 {%k1} # sched: [8:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %mask = fcmp one <16 x float> %mask1, zeroinitializer
-  %vaddr = bitcast i8* %addr to <16 x float>*
-  %r = load <16 x float>, <16 x float>* %vaddr, align 1
-  %res = select <16 x i1> %mask, <16 x float> %r, <16 x float> %old
-  ret <16 x float>%res
-}
-
-define <16 x float> @mov_test42(i8 * %addr, <16 x float> %mask1) {
-; GENERIC-LABEL: mov_test42:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vxorps %xmm1, %xmm1, %xmm1 # sched: [0:0.25]
-; GENERIC-NEXT:    vcmpneq_oqps %zmm1, %zmm0, %k1 # sched: [3:1.00]
-; GENERIC-NEXT:    vmovaps (%rdi), %zmm0 {%k1} {z} # sched: [7:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: mov_test42:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vxorps %xmm1, %xmm1, %xmm1 # sched: [1:0.33]
-; SKX-NEXT:    vcmpneq_oqps %zmm1, %zmm0, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vmovaps (%rdi), %zmm0 {%k1} {z} # sched: [8:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %mask = fcmp one <16 x float> %mask1, zeroinitializer
-  %vaddr = bitcast i8* %addr to <16 x float>*
-  %r = load <16 x float>, <16 x float>* %vaddr, align 64
-  %res = select <16 x i1> %mask, <16 x float> %r, <16 x float> zeroinitializer
-  ret <16 x float>%res
-}
-
-define <16 x float> @mov_test43(i8 * %addr, <16 x float> %mask1) {
-; GENERIC-LABEL: mov_test43:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vxorps %xmm1, %xmm1, %xmm1 # sched: [0:0.25]
-; GENERIC-NEXT:    vcmpneq_oqps %zmm1, %zmm0, %k1 # sched: [3:1.00]
-; GENERIC-NEXT:    vmovups (%rdi), %zmm0 {%k1} {z} # sched: [7:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: mov_test43:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vxorps %xmm1, %xmm1, %xmm1 # sched: [1:0.33]
-; SKX-NEXT:    vcmpneq_oqps %zmm1, %zmm0, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vmovups (%rdi), %zmm0 {%k1} {z} # sched: [8:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %mask = fcmp one <16 x float> %mask1, zeroinitializer
-  %vaddr = bitcast i8* %addr to <16 x float>*
-  %r = load <16 x float>, <16 x float>* %vaddr, align 1
-  %res = select <16 x i1> %mask, <16 x float> %r, <16 x float> zeroinitializer
-  ret <16 x float>%res
-}
-
-define <8 x double> @mov_test44(i8 * %addr, <8 x double> %old, <8 x double> %mask1) {
-; GENERIC-LABEL: mov_test44:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vxorpd %xmm2, %xmm2, %xmm2 # sched: [0:0.25]
-; GENERIC-NEXT:    vcmpneq_oqpd %zmm2, %zmm1, %k1 # sched: [3:1.00]
-; GENERIC-NEXT:    vmovapd (%rdi), %zmm0 {%k1} # sched: [7:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: mov_test44:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vxorpd %xmm2, %xmm2, %xmm2 # sched: [1:0.33]
-; SKX-NEXT:    vcmpneq_oqpd %zmm2, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vmovapd (%rdi), %zmm0 {%k1} # sched: [8:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %mask = fcmp one <8 x double> %mask1, zeroinitializer
-  %vaddr = bitcast i8* %addr to <8 x double>*
-  %r = load <8 x double>, <8 x double>* %vaddr, align 64
-  %res = select <8 x i1> %mask, <8 x double> %r, <8 x double> %old
-  ret <8 x double>%res
-}
-
-define <8 x double> @mov_test45(i8 * %addr, <8 x double> %old, <8 x double> %mask1) {
-; GENERIC-LABEL: mov_test45:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vxorpd %xmm2, %xmm2, %xmm2 # sched: [0:0.25]
-; GENERIC-NEXT:    vcmpneq_oqpd %zmm2, %zmm1, %k1 # sched: [3:1.00]
-; GENERIC-NEXT:    vmovupd (%rdi), %zmm0 {%k1} # sched: [7:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: mov_test45:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vxorpd %xmm2, %xmm2, %xmm2 # sched: [1:0.33]
-; SKX-NEXT:    vcmpneq_oqpd %zmm2, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vmovupd (%rdi), %zmm0 {%k1} # sched: [8:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %mask = fcmp one <8 x double> %mask1, zeroinitializer
-  %vaddr = bitcast i8* %addr to <8 x double>*
-  %r = load <8 x double>, <8 x double>* %vaddr, align 1
-  %res = select <8 x i1> %mask, <8 x double> %r, <8 x double> %old
-  ret <8 x double>%res
-}
-
-define <8 x double> @mov_test46(i8 * %addr, <8 x double> %mask1) {
-; GENERIC-LABEL: mov_test46:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vxorpd %xmm1, %xmm1, %xmm1 # sched: [0:0.25]
-; GENERIC-NEXT:    vcmpneq_oqpd %zmm1, %zmm0, %k1 # sched: [3:1.00]
-; GENERIC-NEXT:    vmovapd (%rdi), %zmm0 {%k1} {z} # sched: [7:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: mov_test46:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vxorpd %xmm1, %xmm1, %xmm1 # sched: [1:0.33]
-; SKX-NEXT:    vcmpneq_oqpd %zmm1, %zmm0, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vmovapd (%rdi), %zmm0 {%k1} {z} # sched: [8:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %mask = fcmp one <8 x double> %mask1, zeroinitializer
-  %vaddr = bitcast i8* %addr to <8 x double>*
-  %r = load <8 x double>, <8 x double>* %vaddr, align 64
-  %res = select <8 x i1> %mask, <8 x double> %r, <8 x double> zeroinitializer
-  ret <8 x double>%res
-}
-
-define <8 x double> @mov_test47(i8 * %addr, <8 x double> %mask1) {
-; GENERIC-LABEL: mov_test47:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vxorpd %xmm1, %xmm1, %xmm1 # sched: [0:0.25]
-; GENERIC-NEXT:    vcmpneq_oqpd %zmm1, %zmm0, %k1 # sched: [3:1.00]
-; GENERIC-NEXT:    vmovupd (%rdi), %zmm0 {%k1} {z} # sched: [7:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: mov_test47:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vxorpd %xmm1, %xmm1, %xmm1 # sched: [1:0.33]
-; SKX-NEXT:    vcmpneq_oqpd %zmm1, %zmm0, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vmovupd (%rdi), %zmm0 {%k1} {z} # sched: [8:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %mask = fcmp one <8 x double> %mask1, zeroinitializer
-  %vaddr = bitcast i8* %addr to <8 x double>*
-  %r = load <8 x double>, <8 x double>* %vaddr, align 1
-  %res = select <8 x i1> %mask, <8 x double> %r, <8 x double> zeroinitializer
-  ret <8 x double>%res
-}
-
-define i16 @mask16(i16 %x) {
-; GENERIC-LABEL: mask16:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    movl %edi, %eax # sched: [1:0.33]
-; GENERIC-NEXT:    notl %eax # sched: [1:0.33]
-; GENERIC-NEXT:    # kill: def $ax killed $ax killed $eax
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: mask16:
-; SKX:       # %bb.0:
-; SKX-NEXT:    movl %edi, %eax # sched: [1:0.25]
-; SKX-NEXT:    notl %eax # sched: [1:0.25]
-; SKX-NEXT:    # kill: def $ax killed $ax killed $eax
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %m0 = bitcast i16 %x to <16 x i1>
-  %m1 = xor <16 x i1> %m0, <i1 -1, i1 -1, i1 -1, i1 -1, i1 -1, i1 -1, i1 -1, i1 -1, i1 -1, i1 -1, i1 -1, i1 -1, i1 -1, i1 -1, i1 -1, i1 -1>
-  %ret = bitcast <16 x i1> %m1 to i16
-  ret i16 %ret
-}
-
-define i32 @mask16_zext(i16 %x) {
-; GENERIC-LABEL: mask16_zext:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    notl %edi # sched: [1:0.33]
-; GENERIC-NEXT:    movzwl %di, %eax # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: mask16_zext:
-; SKX:       # %bb.0:
-; SKX-NEXT:    notl %edi # sched: [1:0.25]
-; SKX-NEXT:    movzwl %di, %eax # sched: [1:0.25]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %m0 = bitcast i16 %x to <16 x i1>
-  %m1 = xor <16 x i1> %m0, <i1 -1, i1 -1, i1 -1, i1 -1, i1 -1, i1 -1, i1 -1, i1 -1, i1 -1, i1 -1, i1 -1, i1 -1, i1 -1, i1 -1, i1 -1, i1 -1>
-  %m2 = bitcast <16 x i1> %m1 to i16
-  %ret = zext i16 %m2 to i32
-  ret i32 %ret
-}
-
-define i8 @mask8(i8 %x) {
-; GENERIC-LABEL: mask8:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    movl %edi, %eax # sched: [1:0.33]
-; GENERIC-NEXT:    notb %al # sched: [1:0.33]
-; GENERIC-NEXT:    # kill: def $al killed $al killed $eax
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: mask8:
-; SKX:       # %bb.0:
-; SKX-NEXT:    movl %edi, %eax # sched: [1:0.25]
-; SKX-NEXT:    notb %al # sched: [1:0.25]
-; SKX-NEXT:    # kill: def $al killed $al killed $eax
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %m0 = bitcast i8 %x to <8 x i1>
-  %m1 = xor <8 x i1> %m0, <i1 -1, i1 -1, i1 -1, i1 -1, i1 -1, i1 -1, i1 -1, i1 -1>
-  %ret = bitcast <8 x i1> %m1 to i8
-  ret i8 %ret
-}
-
-define i32 @mask8_zext(i8 %x) {
-; GENERIC-LABEL: mask8_zext:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    notb %dil # sched: [1:0.33]
-; GENERIC-NEXT:    movzbl %dil, %eax # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: mask8_zext:
-; SKX:       # %bb.0:
-; SKX-NEXT:    notb %dil # sched: [1:0.25]
-; SKX-NEXT:    movzbl %dil, %eax # sched: [1:0.25]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %m0 = bitcast i8 %x to <8 x i1>
-  %m1 = xor <8 x i1> %m0, <i1 -1, i1 -1, i1 -1, i1 -1, i1 -1, i1 -1, i1 -1, i1 -1>
-  %m2 = bitcast <8 x i1> %m1 to i8
-  %ret = zext i8 %m2 to i32
-  ret i32 %ret
-}
-
-define void @mask16_mem(i16* %ptr) {
-; GENERIC-LABEL: mask16_mem:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    kmovw (%rdi), %k0 # sched: [5:0.50]
-; GENERIC-NEXT:    knotw %k0, %k0 # sched: [1:0.33]
-; GENERIC-NEXT:    kmovw %k0, (%rdi) # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: mask16_mem:
-; SKX:       # %bb.0:
-; SKX-NEXT:    kmovw (%rdi), %k0 # sched: [7:1.00]
-; SKX-NEXT:    knotw %k0, %k0 # sched: [1:1.00]
-; SKX-NEXT:    kmovw %k0, (%rdi) # sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %x = load i16, i16* %ptr, align 4
-  %m0 = bitcast i16 %x to <16 x i1>
-  %m1 = xor <16 x i1> %m0, <i1 -1, i1 -1, i1 -1, i1 -1, i1 -1, i1 -1, i1 -1, i1 -1, i1 -1, i1 -1, i1 -1, i1 -1, i1 -1, i1 -1, i1 -1, i1 -1>
-  %ret = bitcast <16 x i1> %m1 to i16
-  store i16 %ret, i16* %ptr, align 4
-  ret void
-}
-
-define void @mask8_mem(i8* %ptr) {
-; GENERIC-LABEL: mask8_mem:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    kmovb (%rdi), %k0 # sched: [5:0.50]
-; GENERIC-NEXT:    knotb %k0, %k0 # sched: [1:0.33]
-; GENERIC-NEXT:    kmovb %k0, (%rdi) # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: mask8_mem:
-; SKX:       # %bb.0:
-; SKX-NEXT:    kmovb (%rdi), %k0 # sched: [7:1.00]
-; SKX-NEXT:    knotb %k0, %k0 # sched: [1:1.00]
-; SKX-NEXT:    kmovb %k0, (%rdi) # sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %x = load i8, i8* %ptr, align 4
-  %m0 = bitcast i8 %x to <8 x i1>
-  %m1 = xor <8 x i1> %m0, <i1 -1, i1 -1, i1 -1, i1 -1, i1 -1, i1 -1, i1 -1, i1 -1>
-  %ret = bitcast <8 x i1> %m1 to i8
-  store i8 %ret, i8* %ptr, align 4
-  ret void
-}
-
-define i16 @mand16(i16 %x, i16 %y) {
-; GENERIC-LABEL: mand16:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    movl %edi, %eax # sched: [1:0.33]
-; GENERIC-NEXT:    movl %edi, %ecx # sched: [1:0.33]
-; GENERIC-NEXT:    andl %esi, %ecx # sched: [1:0.33]
-; GENERIC-NEXT:    xorl %esi, %eax # sched: [1:0.33]
-; GENERIC-NEXT:    orl %ecx, %eax # sched: [1:0.33]
-; GENERIC-NEXT:    # kill: def $ax killed $ax killed $eax
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: mand16:
-; SKX:       # %bb.0:
-; SKX-NEXT:    movl %edi, %eax # sched: [1:0.25]
-; SKX-NEXT:    movl %edi, %ecx # sched: [1:0.25]
-; SKX-NEXT:    andl %esi, %ecx # sched: [1:0.25]
-; SKX-NEXT:    xorl %esi, %eax # sched: [1:0.25]
-; SKX-NEXT:    orl %ecx, %eax # sched: [1:0.25]
-; SKX-NEXT:    # kill: def $ax killed $ax killed $eax
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %ma = bitcast i16 %x to <16 x i1>
-  %mb = bitcast i16 %y to <16 x i1>
-  %mc = and <16 x i1> %ma, %mb
-  %md = xor <16 x i1> %ma, %mb
-  %me = or <16 x i1> %mc, %md
-  %ret = bitcast <16 x i1> %me to i16
-  ret i16 %ret
-}
-
-define i16 @mand16_mem(<16 x i1>* %x, <16 x i1>* %y) {
-; GENERIC-LABEL: mand16_mem:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    kmovw (%rdi), %k0 # sched: [5:0.50]
-; GENERIC-NEXT:    kmovw (%rsi), %k1 # sched: [5:0.50]
-; GENERIC-NEXT:    kandw %k1, %k0, %k2 # sched: [1:0.33]
-; GENERIC-NEXT:    kxorw %k1, %k0, %k0 # sched: [1:0.33]
-; GENERIC-NEXT:    korw %k0, %k2, %k0 # sched: [1:0.33]
-; GENERIC-NEXT:    kmovd %k0, %eax # sched: [1:0.33]
-; GENERIC-NEXT:    # kill: def $ax killed $ax killed $eax
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: mand16_mem:
-; SKX:       # %bb.0:
-; SKX-NEXT:    kmovw (%rdi), %k0 # sched: [7:1.00]
-; SKX-NEXT:    kmovw (%rsi), %k1 # sched: [7:1.00]
-; SKX-NEXT:    kandw %k1, %k0, %k2 # sched: [1:1.00]
-; SKX-NEXT:    kxorw %k1, %k0, %k0 # sched: [1:1.00]
-; SKX-NEXT:    korw %k0, %k2, %k0 # sched: [1:1.00]
-; SKX-NEXT:    kmovd %k0, %eax # sched: [3:1.00]
-; SKX-NEXT:    # kill: def $ax killed $ax killed $eax
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %ma = load <16 x i1>, <16 x i1>* %x
-  %mb = load <16 x i1>, <16 x i1>* %y
-  %mc = and <16 x i1> %ma, %mb
-  %md = xor <16 x i1> %ma, %mb
-  %me = or <16 x i1> %mc, %md
-  %ret = bitcast <16 x i1> %me to i16
-  ret i16 %ret
-}
-
-define i8 @shuf_test1(i16 %v) nounwind {
-; GENERIC-LABEL: shuf_test1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    kmovd %edi, %k0 # sched: [1:0.33]
-; GENERIC-NEXT:    kshiftrw $8, %k0, %k0 # sched: [1:1.00]
-; GENERIC-NEXT:    kmovd %k0, %eax # sched: [1:0.33]
-; GENERIC-NEXT:    # kill: def $al killed $al killed $eax
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: shuf_test1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    kmovd %edi, %k0 # sched: [1:1.00]
-; SKX-NEXT:    kshiftrw $8, %k0, %k0 # sched: [3:1.00]
-; SKX-NEXT:    kmovd %k0, %eax # sched: [3:1.00]
-; SKX-NEXT:    # kill: def $al killed $al killed $eax
-; SKX-NEXT:    retq # sched: [7:1.00]
-   %v1 = bitcast i16 %v to <16 x i1>
-   %mask = shufflevector <16 x i1> %v1, <16 x i1> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
-   %mask1 = bitcast <8 x i1> %mask to i8
-   ret i8 %mask1
-}
-
-define i32 @zext_test1(<16 x i32> %a, <16 x i32> %b) {
-; GENERIC-LABEL: zext_test1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpcmpnleud %zmm1, %zmm0, %k0 # sched: [1:0.50]
-; GENERIC-NEXT:    kshiftrw $5, %k0, %k0 # sched: [1:1.00]
-; GENERIC-NEXT:    kmovd %k0, %eax # sched: [1:0.33]
-; GENERIC-NEXT:    andl $1, %eax # sched: [1:0.33]
-; GENERIC-NEXT:    vzeroupper # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: zext_test1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpcmpnleud %zmm1, %zmm0, %k0 # sched: [3:1.00]
-; SKX-NEXT:    kshiftrw $5, %k0, %k0 # sched: [3:1.00]
-; SKX-NEXT:    kmovd %k0, %eax # sched: [3:1.00]
-; SKX-NEXT:    andl $1, %eax # sched: [1:0.25]
-; SKX-NEXT:    vzeroupper # sched: [0:0.67]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %cmp_res = icmp ugt <16 x i32> %a, %b
-  %cmp_res.i1 = extractelement <16 x i1> %cmp_res, i32 5
-  %res = zext i1 %cmp_res.i1 to i32
-  ret i32 %res
-}
-
-define i16 @zext_test2(<16 x i32> %a, <16 x i32> %b) {
-; GENERIC-LABEL: zext_test2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpcmpnleud %zmm1, %zmm0, %k0 # sched: [1:0.50]
-; GENERIC-NEXT:    kshiftrw $5, %k0, %k0 # sched: [1:1.00]
-; GENERIC-NEXT:    kmovd %k0, %eax # sched: [1:0.33]
-; GENERIC-NEXT:    andl $1, %eax # sched: [1:0.33]
-; GENERIC-NEXT:    # kill: def $ax killed $ax killed $eax
-; GENERIC-NEXT:    vzeroupper # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: zext_test2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpcmpnleud %zmm1, %zmm0, %k0 # sched: [3:1.00]
-; SKX-NEXT:    kshiftrw $5, %k0, %k0 # sched: [3:1.00]
-; SKX-NEXT:    kmovd %k0, %eax # sched: [3:1.00]
-; SKX-NEXT:    andl $1, %eax # sched: [1:0.25]
-; SKX-NEXT:    # kill: def $ax killed $ax killed $eax
-; SKX-NEXT:    vzeroupper # sched: [0:0.67]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %cmp_res = icmp ugt <16 x i32> %a, %b
-  %cmp_res.i1 = extractelement <16 x i1> %cmp_res, i32 5
-  %res = zext i1 %cmp_res.i1 to i16
-  ret i16 %res
-}
-
-define i8 @zext_test3(<16 x i32> %a, <16 x i32> %b) {
-; GENERIC-LABEL: zext_test3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpcmpnleud %zmm1, %zmm0, %k0 # sched: [1:0.50]
-; GENERIC-NEXT:    kshiftrw $5, %k0, %k0 # sched: [1:1.00]
-; GENERIC-NEXT:    kmovd %k0, %eax # sched: [1:0.33]
-; GENERIC-NEXT:    andb $1, %al # sched: [1:0.33]
-; GENERIC-NEXT:    # kill: def $al killed $al killed $eax
-; GENERIC-NEXT:    vzeroupper # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: zext_test3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpcmpnleud %zmm1, %zmm0, %k0 # sched: [3:1.00]
-; SKX-NEXT:    kshiftrw $5, %k0, %k0 # sched: [3:1.00]
-; SKX-NEXT:    kmovd %k0, %eax # sched: [3:1.00]
-; SKX-NEXT:    andb $1, %al # sched: [1:0.25]
-; SKX-NEXT:    # kill: def $al killed $al killed $eax
-; SKX-NEXT:    vzeroupper # sched: [0:0.67]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %cmp_res = icmp ugt <16 x i32> %a, %b
-  %cmp_res.i1 = extractelement <16 x i1> %cmp_res, i32 5
-  %res = zext i1 %cmp_res.i1 to i8
-  ret i8 %res
-}
-
-define i8 @conv1(<8 x i1>* %R) {
-; GENERIC-LABEL: conv1:
-; GENERIC:       # %bb.0: # %entry
-; GENERIC-NEXT:    movb $-1, (%rdi) # sched: [1:1.00]
-; GENERIC-NEXT:    movb $-2, -{{[0-9]+}}(%rsp) # sched: [1:1.00]
-; GENERIC-NEXT:    movb $-2, %al # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: conv1:
-; SKX:       # %bb.0: # %entry
-; SKX-NEXT:    movb $-1, (%rdi) # sched: [1:1.00]
-; SKX-NEXT:    movb $-2, -{{[0-9]+}}(%rsp) # sched: [1:1.00]
-; SKX-NEXT:    movb $-2, %al # sched: [1:0.25]
-; SKX-NEXT:    retq # sched: [7:1.00]
-entry:
-  store <8 x i1> <i1 1, i1 1, i1 1, i1 1, i1 1, i1 1, i1 1, i1 1>, <8 x i1>* %R
-
-  %maskPtr = alloca <8 x i1>
-  store <8 x i1> <i1 0, i1 1, i1 1, i1 1, i1 1, i1 1, i1 1, i1 1>, <8 x i1>* %maskPtr
-  %mask = load <8 x i1>, <8 x i1>* %maskPtr
-  %mask_convert = bitcast <8 x i1> %mask to i8
-  ret i8 %mask_convert
-}
-
-define <4 x i32> @test4(<4 x i64> %x, <4 x i64> %y, <4 x i64> %x1, <4 x i64> %y1) {
-; GENERIC-LABEL: test4:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpcmpleq %ymm1, %ymm0, %k1 # sched: [1:0.50]
-; GENERIC-NEXT:    vpcmpgtq %ymm3, %ymm2, %k0 {%k1} # sched: [1:0.50]
-; GENERIC-NEXT:    vpmovm2d %k0, %xmm0 # sched: [1:0.33]
-; GENERIC-NEXT:    vzeroupper # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test4:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpcmpleq %ymm1, %ymm0, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpcmpgtq %ymm3, %ymm2, %k0 {%k1} # sched: [3:1.00]
-; SKX-NEXT:    vpmovm2d %k0, %xmm0 # sched: [1:0.25]
-; SKX-NEXT:    vzeroupper # sched: [0:0.67]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %x_gt_y = icmp sgt <4 x i64> %x, %y
-  %x1_gt_y1 = icmp sgt <4 x i64> %x1, %y1
-  %res = icmp sgt <4 x i1>%x_gt_y, %x1_gt_y1
-  %resse = sext <4 x i1>%res to <4 x i32>
-  ret <4 x i32> %resse
-}
-
-define <2 x i64> @vcmp_test5(<2 x i64> %x, <2 x i64> %y, <2 x i64> %x1, <2 x i64> %y1) {
-; GENERIC-LABEL: vcmp_test5:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpcmpleq %xmm3, %xmm2, %k1 # sched: [1:0.50]
-; GENERIC-NEXT:    vpcmpgtq %xmm0, %xmm1, %k0 {%k1} # sched: [1:0.50]
-; GENERIC-NEXT:    vpmovm2q %k0, %xmm0 # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: vcmp_test5:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpcmpleq %xmm3, %xmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpcmpgtq %xmm0, %xmm1, %k0 {%k1} # sched: [3:1.00]
-; SKX-NEXT:    vpmovm2q %k0, %xmm0 # sched: [1:0.25]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %x_gt_y = icmp slt <2 x i64> %x, %y
-  %x1_gt_y1 = icmp sgt <2 x i64> %x1, %y1
-  %res = icmp slt <2 x i1>%x_gt_y, %x1_gt_y1
-  %resse = sext <2 x i1>%res to <2 x i64>
-  ret <2 x i64> %resse
-}define void @vcmp_test6(<16 x i1> %mask)  {
-allocas:
-  %a= and <16 x i1> %mask, <i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false>
-  %b = bitcast <16 x i1> %a to i16
-  %c = icmp eq i16 %b, 0
-  br i1 %c, label %true, label %false
-
-true:
-  ret void
-
-false:
-  ret void
-}
-define void @vcmp_test7(<8 x i1> %mask)  {
-; GENERIC-LABEL: vcmp_test7:
-; GENERIC:       # %bb.0: # %allocas
-; GENERIC-NEXT:    vpsllw $15, %xmm0, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vpmovw2m %xmm0, %k0 # sched: [1:0.33]
-; GENERIC-NEXT:    kmovd %k0, %eax # sched: [1:0.33]
-; GENERIC-NEXT:    orb $85, %al # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: vcmp_test7:
-; SKX:       # %bb.0: # %allocas
-; SKX-NEXT:    vpsllw $15, %xmm0, %xmm0 # sched: [1:0.50]
-; SKX-NEXT:    vpmovw2m %xmm0, %k0 # sched: [1:1.00]
-; SKX-NEXT:    kmovd %k0, %eax # sched: [3:1.00]
-; SKX-NEXT:    orb $85, %al # sched: [1:0.25]
-; SKX-NEXT:    retq # sched: [7:1.00]
-allocas:
-  %a= or <8 x i1> %mask, <i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false>
-  %b = bitcast <8 x i1> %a to i8
-  %c = icmp eq i8 %b, 0
-  br i1 %c, label %true, label %false
-
-true:
-  ret void
-
-false:
-  ret void
-}
-define <16 x i8> @vcmp_test8(<16 x i32>%a, <16 x i32>%b, i32 %a1, i32 %b1) {
-; GENERIC-LABEL: vcmp_test8:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    cmpl %esi, %edi # sched: [1:0.33]
-; GENERIC-NEXT:    jg .LBB386_1 # sched: [1:1.00]
-; GENERIC-NEXT:  # %bb.2:
-; GENERIC-NEXT:    kxorw %k0, %k0, %k0 # sched: [1:0.33]
-; GENERIC-NEXT:    vpmovm2b %k0, %xmm0 # sched: [1:0.33]
-; GENERIC-NEXT:    vzeroupper # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-; GENERIC-NEXT:  .LBB386_1:
-; GENERIC-NEXT:    vpxor %xmm1, %xmm1, %xmm1 # sched: [0:0.25]
-; GENERIC-NEXT:    vpcmpgtd %zmm1, %zmm0, %k0 # sched: [1:0.50]
-; GENERIC-NEXT:    vpmovm2b %k0, %xmm0 # sched: [1:0.33]
-; GENERIC-NEXT:    vzeroupper # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: vcmp_test8:
-; SKX:       # %bb.0:
-; SKX-NEXT:    cmpl %esi, %edi # sched: [1:0.25]
-; SKX-NEXT:    jg .LBB386_1 # sched: [1:0.50]
-; SKX-NEXT:  # %bb.2:
-; SKX-NEXT:    kxorw %k0, %k0, %k0 # sched: [1:1.00]
-; SKX-NEXT:    vpmovm2b %k0, %xmm0 # sched: [1:0.25]
-; SKX-NEXT:    vzeroupper # sched: [0:0.67]
-; SKX-NEXT:    retq # sched: [7:1.00]
-; SKX-NEXT:  .LBB386_1:
-; SKX-NEXT:    vpxor %xmm1, %xmm1, %xmm1 # sched: [1:0.33]
-; SKX-NEXT:    vpcmpgtd %zmm1, %zmm0, %k0 # sched: [3:1.00]
-; SKX-NEXT:    vpmovm2b %k0, %xmm0 # sched: [1:0.25]
-; SKX-NEXT:    vzeroupper # sched: [0:0.67]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %cond = icmp sgt i32 %a1, %b1
-  %cmp1 = icmp sgt <16 x i32> %a, zeroinitializer
-  %cmp2 = icmp ult <16 x i32> %b, zeroinitializer
-  %mix = select i1 %cond, <16 x i1> %cmp1, <16 x i1> %cmp2
-  %res = sext <16 x i1> %mix to <16 x i8>
-  ret <16 x i8> %res
-}
-define <16 x i1> @vpmov_test9(<16 x i1>%a, <16 x i1>%b, i32 %a1, i32 %b1) {
-; GENERIC-LABEL: vpmov_test9:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    cmpl %esi, %edi # sched: [1:0.33]
-; GENERIC-NEXT:    jg .LBB387_1 # sched: [1:1.00]
-; GENERIC-NEXT:  # %bb.2:
-; GENERIC-NEXT:    vpsllw $7, %xmm1, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    jmp .LBB387_3 # sched: [1:1.00]
-; GENERIC-NEXT:  .LBB387_1:
-; GENERIC-NEXT:    vpsllw $7, %xmm0, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:  .LBB387_3:
-; GENERIC-NEXT:    vpmovb2m %xmm0, %k0 # sched: [1:0.33]
-; GENERIC-NEXT:    vpmovm2b %k0, %xmm0 # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: vpmov_test9:
-; SKX:       # %bb.0:
-; SKX-NEXT:    cmpl %esi, %edi # sched: [1:0.25]
-; SKX-NEXT:    jg .LBB387_1 # sched: [1:0.50]
-; SKX-NEXT:  # %bb.2:
-; SKX-NEXT:    vpsllw $7, %xmm1, %xmm0 # sched: [1:0.50]
-; SKX-NEXT:    jmp .LBB387_3 # sched: [1:0.50]
-; SKX-NEXT:  .LBB387_1:
-; SKX-NEXT:    vpsllw $7, %xmm0, %xmm0 # sched: [1:0.50]
-; SKX-NEXT:  .LBB387_3:
-; SKX-NEXT:    vpmovb2m %xmm0, %k0 # sched: [1:1.00]
-; SKX-NEXT:    vpmovm2b %k0, %xmm0 # sched: [1:0.25]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %mask = icmp sgt i32 %a1, %b1
-  %c = select i1 %mask, <16 x i1>%a, <16 x i1>%b
-  ret <16 x i1>%c
-}define <8 x i1> @vpmov_test10(<8 x i1>%a, <8 x i1>%b, i32 %a1, i32 %b1) {
-  %mask = icmp sgt i32 %a1, %b1
-  %c = select i1 %mask, <8 x i1>%a, <8 x i1>%b
-  ret <8 x i1>%c
-}
-
-define <4 x i1> @vmov_test11(<4 x i1>%a, <4 x i1>%b, i32 %a1, i32 %b1) {
-; GENERIC-LABEL: vmov_test11:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    cmpl %esi, %edi # sched: [1:0.33]
-; GENERIC-NEXT:    jg .LBB389_1 # sched: [1:1.00]
-; GENERIC-NEXT:  # %bb.2:
-; GENERIC-NEXT:    vpslld $31, %xmm1, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    jmp .LBB389_3 # sched: [1:1.00]
-; GENERIC-NEXT:  .LBB389_1:
-; GENERIC-NEXT:    vpslld $31, %xmm0, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:  .LBB389_3:
-; GENERIC-NEXT:    vpmovd2m %xmm0, %k0 # sched: [1:0.33]
-; GENERIC-NEXT:    vpmovm2d %k0, %xmm0 # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: vmov_test11:
-; SKX:       # %bb.0:
-; SKX-NEXT:    cmpl %esi, %edi # sched: [1:0.25]
-; SKX-NEXT:    jg .LBB389_1 # sched: [1:0.50]
-; SKX-NEXT:  # %bb.2:
-; SKX-NEXT:    vpslld $31, %xmm1, %xmm0 # sched: [1:0.50]
-; SKX-NEXT:    jmp .LBB389_3 # sched: [1:0.50]
-; SKX-NEXT:  .LBB389_1:
-; SKX-NEXT:    vpslld $31, %xmm0, %xmm0 # sched: [1:0.50]
-; SKX-NEXT:  .LBB389_3:
-; SKX-NEXT:    vpmovd2m %xmm0, %k0 # sched: [1:1.00]
-; SKX-NEXT:    vpmovm2d %k0, %xmm0 # sched: [1:0.25]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %mask = icmp sgt i32 %a1, %b1
-  %c = select i1 %mask, <4 x i1>%a, <4 x i1>%b
-  ret <4 x i1>%c
-}
-
-define i32 @vmov_test12(i32 %x, i32 %y)  {
-; GENERIC-LABEL: vmov_test12:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    movl %edi, %eax # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: vmov_test12:
-; SKX:       # %bb.0:
-; SKX-NEXT:    movl %edi, %eax # sched: [1:0.25]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %a = bitcast i16 21845 to <16 x i1>
-  %b = extractelement <16 x i1> %a, i32 0
-  %c = select i1 %b, i32 %x, i32 %y
-  ret i32 %c
-}
-
-define i32 @vmov_test13(i32 %x, i32 %y)  {
-; GENERIC-LABEL: vmov_test13:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    movl %esi, %eax # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: vmov_test13:
-; SKX:       # %bb.0:
-; SKX-NEXT:    movl %esi, %eax # sched: [1:0.25]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %a = bitcast i16 21845 to <16 x i1>
-  %b = extractelement <16 x i1> %a, i32 3
-  %c = select i1 %b, i32 %x, i32 %y
-  ret i32 %c
-}define <4 x i1> @vmov_test14()  {
-  %a = bitcast i16 21845 to <16 x i1>
-  %b = extractelement <16 x i1> %a, i32 2
-  %c = insertelement <4 x i1> <i1 true, i1 false, i1 false, i1 true>, i1 %b, i32 1
-  ret <4 x i1> %c
-}
-
-define <16 x i1> @vmov_test15(i32 %x, i32 %y)  {
-; GENERIC-LABEL: vmov_test15:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    cmpl %esi, %edi # sched: [1:0.33]
-; GENERIC-NEXT:    movl $21845, %eax # imm = 0x5555
-; GENERIC-NEXT:    # sched: [1:0.33]
-; GENERIC-NEXT:    movl $1, %ecx # sched: [1:0.33]
-; GENERIC-NEXT:    cmovgl %eax, %ecx # sched: [2:0.67]
-; GENERIC-NEXT:    kmovd %ecx, %k0 # sched: [1:0.33]
-; GENERIC-NEXT:    vpmovm2b %k0, %xmm0 # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: vmov_test15:
-; SKX:       # %bb.0:
-; SKX-NEXT:    cmpl %esi, %edi # sched: [1:0.25]
-; SKX-NEXT:    movl $21845, %eax # imm = 0x5555
-; SKX-NEXT:    # sched: [1:0.25]
-; SKX-NEXT:    movl $1, %ecx # sched: [1:0.25]
-; SKX-NEXT:    cmovgl %eax, %ecx # sched: [1:0.50]
-; SKX-NEXT:    kmovd %ecx, %k0 # sched: [1:1.00]
-; SKX-NEXT:    vpmovm2b %k0, %xmm0 # sched: [1:0.25]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %a = bitcast i16 21845 to <16 x i1>
-  %b = bitcast i16 1 to <16 x i1>
-  %mask = icmp sgt i32 %x, %y
-  %c = select i1 %mask, <16 x i1> %a, <16 x i1> %b
-  ret <16 x i1> %c
-}
-
-define <64 x i8> @vmov_test16(i64 %x) {
-;
-; GENERIC-LABEL: vmov_test16:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    kmovq %rdi, %k0 # sched: [1:0.33]
-; GENERIC-NEXT:    movb $1, %al # sched: [1:0.33]
-; GENERIC-NEXT:    kmovd %eax, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    kshiftrq $5, %k0, %k2 # sched: [1:1.00]
-; GENERIC-NEXT:    kxorq %k1, %k2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    kshiftlq $63, %k1, %k1 # sched: [1:1.00]
-; GENERIC-NEXT:    kshiftrq $58, %k1, %k1 # sched: [1:1.00]
-; GENERIC-NEXT:    kxorq %k1, %k0, %k0 # sched: [1:0.33]
-; GENERIC-NEXT:    vpmovm2b %k0, %zmm0 # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: vmov_test16:
-; SKX:       # %bb.0:
-; SKX-NEXT:    kmovq %rdi, %k0 # sched: [1:1.00]
-; SKX-NEXT:    movb $1, %al # sched: [1:0.25]
-; SKX-NEXT:    kmovd %eax, %k1 # sched: [1:1.00]
-; SKX-NEXT:    kshiftrq $5, %k0, %k2 # sched: [3:1.00]
-; SKX-NEXT:    kxorq %k1, %k2, %k1 # sched: [1:1.00]
-; SKX-NEXT:    kshiftlq $63, %k1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    kshiftrq $58, %k1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    kxorq %k1, %k0, %k0 # sched: [1:1.00]
-; SKX-NEXT:    vpmovm2b %k0, %zmm0 # sched: [1:0.25]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %a = bitcast i64 %x to <64 x i1>
-  %b = insertelement <64 x i1>%a, i1 true, i32 5
-  %c = sext <64 x i1>%b to <64 x i8>
-  ret <64 x i8>%c
-}
-
-define <64 x i8> @vmov_test17(i64 %x, i32 %y, i32 %z) {
-;
-; GENERIC-LABEL: vmov_test17:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    kmovq %rdi, %k0 # sched: [1:0.33]
-; GENERIC-NEXT:    cmpl %edx, %esi # sched: [1:0.33]
-; GENERIC-NEXT:    setg %al # sched: [1:0.50]
-; GENERIC-NEXT:    kmovd %eax, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    kshiftrq $5, %k0, %k2 # sched: [1:1.00]
-; GENERIC-NEXT:    kxorq %k1, %k2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    kshiftlq $63, %k1, %k1 # sched: [1:1.00]
-; GENERIC-NEXT:    kshiftrq $58, %k1, %k1 # sched: [1:1.00]
-; GENERIC-NEXT:    kxorq %k1, %k0, %k0 # sched: [1:0.33]
-; GENERIC-NEXT:    vpmovm2b %k0, %zmm0 # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: vmov_test17:
-; SKX:       # %bb.0:
-; SKX-NEXT:    kmovq %rdi, %k0 # sched: [1:1.00]
-; SKX-NEXT:    cmpl %edx, %esi # sched: [1:0.25]
-; SKX-NEXT:    setg %al # sched: [1:0.50]
-; SKX-NEXT:    kmovd %eax, %k1 # sched: [1:1.00]
-; SKX-NEXT:    kshiftrq $5, %k0, %k2 # sched: [3:1.00]
-; SKX-NEXT:    kxorq %k1, %k2, %k1 # sched: [1:1.00]
-; SKX-NEXT:    kshiftlq $63, %k1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    kshiftrq $58, %k1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    kxorq %k1, %k0, %k0 # sched: [1:1.00]
-; SKX-NEXT:    vpmovm2b %k0, %zmm0 # sched: [1:0.25]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %a = bitcast i64 %x to <64 x i1>
-  %b = icmp sgt i32 %y, %z
-  %c = insertelement <64 x i1>%a, i1 %b, i32 5
-  %d = sext <64 x i1>%c to <64 x i8>
-  ret <64 x i8>%d
-}
-
-define <8 x i1> @vmov_test18(i8 %a, i16 %y) {
-; GENERIC-LABEL: vmov_test18:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    kmovd %edi, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    kmovd %esi, %k2 # sched: [1:0.33]
-; GENERIC-NEXT:    kshiftrw $8, %k2, %k0 # sched: [1:1.00]
-; GENERIC-NEXT:    kshiftrw $9, %k2, %k2 # sched: [1:1.00]
-; GENERIC-NEXT:    kshiftrb $6, %k1, %k3 # sched: [1:1.00]
-; GENERIC-NEXT:    kxorb %k2, %k3, %k2 # sched: [1:0.33]
-; GENERIC-NEXT:    kshiftlb $7, %k2, %k2 # sched: [1:1.00]
-; GENERIC-NEXT:    kshiftrb $1, %k2, %k2 # sched: [1:1.00]
-; GENERIC-NEXT:    kxorb %k2, %k1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    kshiftlb $1, %k1, %k1 # sched: [1:1.00]
-; GENERIC-NEXT:    kshiftrb $1, %k1, %k1 # sched: [1:1.00]
-; GENERIC-NEXT:    kshiftlb $7, %k0, %k0 # sched: [1:1.00]
-; GENERIC-NEXT:    korb %k0, %k1, %k0 # sched: [1:0.33]
-; GENERIC-NEXT:    vpmovm2w %k0, %xmm0 # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: vmov_test18:
-; SKX:       # %bb.0:
-; SKX-NEXT:    kmovd %edi, %k1 # sched: [1:1.00]
-; SKX-NEXT:    kmovd %esi, %k2 # sched: [1:1.00]
-; SKX-NEXT:    kshiftrw $8, %k2, %k0 # sched: [3:1.00]
-; SKX-NEXT:    kshiftrw $9, %k2, %k2 # sched: [3:1.00]
-; SKX-NEXT:    kshiftrb $6, %k1, %k3 # sched: [3:1.00]
-; SKX-NEXT:    kxorb %k2, %k3, %k2 # sched: [1:1.00]
-; SKX-NEXT:    kshiftlb $7, %k2, %k2 # sched: [3:1.00]
-; SKX-NEXT:    kshiftrb $1, %k2, %k2 # sched: [3:1.00]
-; SKX-NEXT:    kxorb %k2, %k1, %k1 # sched: [1:1.00]
-; SKX-NEXT:    kshiftlb $1, %k1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    kshiftrb $1, %k1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    kshiftlb $7, %k0, %k0 # sched: [3:1.00]
-; SKX-NEXT:    korb %k0, %k1, %k0 # sched: [1:1.00]
-; SKX-NEXT:    vpmovm2w %k0, %xmm0 # sched: [1:0.25]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %b = bitcast i8 %a to <8 x i1>
-  %b1 = bitcast i16 %y to <16 x i1>
-  %el1 = extractelement <16 x i1>%b1, i32 8
-  %el2 = extractelement <16 x i1>%b1, i32 9
-  %c = insertelement <8 x i1>%b, i1 %el1, i32 7
-  %d = insertelement <8 x i1>%c, i1 %el2, i32 6
-  ret <8 x i1>%d
-}
-define <32 x i16> @vmov_test21(<32 x i16> %x , <32 x i1> %mask) nounwind readnone {
-; GENERIC-LABEL: vmov_test21:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpsllw $7, %ymm1, %ymm1 # sched: [1:1.00]
-; GENERIC-NEXT:    vpmovb2m %ymm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vmovdqu16 %zmm0, %zmm0 {%k1} {z} # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: vmov_test21:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpsllw $7, %ymm1, %ymm1 # sched: [1:0.50]
-; SKX-NEXT:    vpmovb2m %ymm1, %k1 # sched: [1:1.00]
-; SKX-NEXT:    vmovdqu16 %zmm0, %zmm0 {%k1} {z} # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %ret = select <32 x i1> %mask, <32 x i16> %x, <32 x i16> zeroinitializer
-  ret <32 x i16> %ret
-}
-
-define void @vmov_test22(<4 x i1> %a, <4 x i1>* %addr) {
-; GENERIC-LABEL: vmov_test22:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpslld $31, %xmm0, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vpmovd2m %xmm0, %k0 # sched: [1:0.33]
-; GENERIC-NEXT:    kmovb %k0, (%rdi) # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: vmov_test22:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpslld $31, %xmm0, %xmm0 # sched: [1:0.50]
-; SKX-NEXT:    vpmovd2m %xmm0, %k0 # sched: [1:1.00]
-; SKX-NEXT:    kmovb %k0, (%rdi) # sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  store <4 x i1> %a, <4 x i1>* %addr
-  ret void
-}
-
-define void @vmov_test23(<2 x i1> %a, <2 x i1>* %addr) {
-; GENERIC-LABEL: vmov_test23:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpsllq $63, %xmm0, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vpmovq2m %xmm0, %k0 # sched: [1:0.33]
-; GENERIC-NEXT:    kmovb %k0, (%rdi) # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: vmov_test23:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpsllq $63, %xmm0, %xmm0 # sched: [1:0.50]
-; SKX-NEXT:    vpmovq2m %xmm0, %k0 # sched: [1:1.00]
-; SKX-NEXT:    kmovb %k0, (%rdi) # sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  store <2 x i1> %a, <2 x i1>* %addr
-  ret void
-}
-
-define void @store_v1i1(<1 x i1> %c , <1 x i1>* %ptr) {
-; GENERIC-LABEL: store_v1i1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    kmovd %edi, %k0 # sched: [1:0.33]
-; GENERIC-NEXT:    kxnorw %k0, %k0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    kxorw %k1, %k0, %k0 # sched: [1:0.33]
-; GENERIC-NEXT:    kmovb %k0, (%rsi) # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: store_v1i1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    kmovd %edi, %k0 # sched: [1:1.00]
-; SKX-NEXT:    kxnorw %k0, %k0, %k1 # sched: [1:1.00]
-; SKX-NEXT:    kxorw %k1, %k0, %k0 # sched: [1:1.00]
-; SKX-NEXT:    kmovb %k0, (%rsi) # sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %x = xor <1 x i1> %c, <i1 1>
-  store <1 x i1> %x, <1 x i1>*  %ptr, align 4
-  ret void
-}
-
-define void @store_v2i1(<2 x i1> %c , <2 x i1>* %ptr) {
-; GENERIC-LABEL: store_v2i1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpsllq $63, %xmm0, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vpmovq2m %xmm0, %k0 # sched: [1:0.33]
-; GENERIC-NEXT:    knotw %k0, %k0 # sched: [1:0.33]
-; GENERIC-NEXT:    kmovb %k0, (%rdi) # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: store_v2i1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpsllq $63, %xmm0, %xmm0 # sched: [1:0.50]
-; SKX-NEXT:    vpmovq2m %xmm0, %k0 # sched: [1:1.00]
-; SKX-NEXT:    knotw %k0, %k0 # sched: [1:1.00]
-; SKX-NEXT:    kmovb %k0, (%rdi) # sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %x = xor <2 x i1> %c, <i1 1, i1 1>
-  store <2 x i1> %x, <2 x i1>*  %ptr, align 4
-  ret void
-}
-
-define void @store_v4i1(<4 x i1> %c , <4 x i1>* %ptr) {
-; GENERIC-LABEL: store_v4i1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpslld $31, %xmm0, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vpmovd2m %xmm0, %k0 # sched: [1:0.33]
-; GENERIC-NEXT:    knotw %k0, %k0 # sched: [1:0.33]
-; GENERIC-NEXT:    kmovb %k0, (%rdi) # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: store_v4i1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpslld $31, %xmm0, %xmm0 # sched: [1:0.50]
-; SKX-NEXT:    vpmovd2m %xmm0, %k0 # sched: [1:1.00]
-; SKX-NEXT:    knotw %k0, %k0 # sched: [1:1.00]
-; SKX-NEXT:    kmovb %k0, (%rdi) # sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %x = xor <4 x i1> %c, <i1 1, i1 1, i1 1, i1 1>
-  store <4 x i1> %x, <4 x i1>*  %ptr, align 4
-  ret void
-}
-
-define void @store_v8i1(<8 x i1> %c , <8 x i1>* %ptr) {
-; GENERIC-LABEL: store_v8i1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpsllw $15, %xmm0, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vpmovw2m %xmm0, %k0 # sched: [1:0.33]
-; GENERIC-NEXT:    knotb %k0, %k0 # sched: [1:0.33]
-; GENERIC-NEXT:    kmovb %k0, (%rdi) # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: store_v8i1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpsllw $15, %xmm0, %xmm0 # sched: [1:0.50]
-; SKX-NEXT:    vpmovw2m %xmm0, %k0 # sched: [1:1.00]
-; SKX-NEXT:    knotb %k0, %k0 # sched: [1:1.00]
-; SKX-NEXT:    kmovb %k0, (%rdi) # sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %x = xor <8 x i1> %c, <i1 1, i1 1, i1 1, i1 1, i1 1, i1 1, i1 1, i1 1>
-  store <8 x i1> %x, <8 x i1>*  %ptr, align 4
-  ret void
-}
-
-define void @store_v16i1(<16 x i1> %c , <16 x i1>* %ptr) {
-; GENERIC-LABEL: store_v16i1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpsllw $7, %xmm0, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vpmovb2m %xmm0, %k0 # sched: [1:0.33]
-; GENERIC-NEXT:    knotw %k0, %k0 # sched: [1:0.33]
-; GENERIC-NEXT:    kmovw %k0, (%rdi) # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: store_v16i1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpsllw $7, %xmm0, %xmm0 # sched: [1:0.50]
-; SKX-NEXT:    vpmovb2m %xmm0, %k0 # sched: [1:1.00]
-; SKX-NEXT:    knotw %k0, %k0 # sched: [1:1.00]
-; SKX-NEXT:    kmovw %k0, (%rdi) # sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %x = xor <16 x i1> %c, <i1 1, i1 1, i1 1, i1 1, i1 1, i1 1, i1 1, i1 1, i1 1, i1 1, i1 1, i1 1, i1 1, i1 1, i1 1, i1 1>
-  store <16 x i1> %x, <16 x i1>*  %ptr, align 4
-  ret void
-}
-
-;void f2(int);
-;void f1(int c)
-;{
-;  static int v = 0;
-;  if (v == 0)
-;    v = 1;
-;  else
-;    v = 0;
-;  f2(v);
-;}
-
-@f1.v = internal unnamed_addr global i1 false, align 4
-
-define void @f1(i32 %c) {
-; GENERIC-LABEL: f1:
-; GENERIC:       # %bb.0: # %entry
-; GENERIC-NEXT:    movzbl {{.*}}(%rip), %edi # sched: [5:0.50]
-; GENERIC-NEXT:    xorl $1, %edi # sched: [1:0.33]
-; GENERIC-NEXT:    movb %dil, {{.*}}(%rip) # sched: [1:1.00]
-; GENERIC-NEXT:    jmp f2 # TAILCALL
-;
-; SKX-LABEL: f1:
-; SKX:       # %bb.0: # %entry
-; SKX-NEXT:    movzbl {{.*}}(%rip), %edi # sched: [5:0.50]
-; SKX-NEXT:    xorl $1, %edi # sched: [1:0.25]
-; SKX-NEXT:    movb %dil, {{.*}}(%rip) # sched: [1:1.00]
-; SKX-NEXT:    jmp f2 # TAILCALL
-entry:
-  %.b1 = load i1, i1* @f1.v, align 4
-  %not..b1 = xor i1 %.b1, true
-  store i1 %not..b1, i1* @f1.v, align 4
-  %0 = zext i1 %not..b1 to i32
-  tail call void @f2(i32 %0) #2
-  ret void
-}
-
-declare void @f2(i32) #1
-
-define void @store_i16_i1(i16 %x, i1 *%y) {
-; GENERIC-LABEL: store_i16_i1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    andl $1, %edi # sched: [1:0.33]
-; GENERIC-NEXT:    movb %dil, (%rsi) # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: store_i16_i1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    andl $1, %edi # sched: [1:0.25]
-; SKX-NEXT:    movb %dil, (%rsi) # sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %c = trunc i16 %x to i1
-  store i1 %c, i1* %y
-  ret void
-}
-
-define void @store_i8_i1(i8 %x, i1 *%y) {
-; GENERIC-LABEL: store_i8_i1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    andl $1, %edi # sched: [1:0.33]
-; GENERIC-NEXT:    movb %dil, (%rsi) # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: store_i8_i1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    andl $1, %edi # sched: [1:0.25]
-; SKX-NEXT:    movb %dil, (%rsi) # sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %c = trunc i8 %x to i1
-  store i1 %c, i1* %y
-  ret void
-}
-
-define <32 x i16> @test_build_vec_v32i1(<32 x i16> %x) {
-; GENERIC-LABEL: test_build_vec_v32i1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    movl $1497715861, %eax # imm = 0x59455495
-; GENERIC-NEXT:    # sched: [1:0.33]
-; GENERIC-NEXT:    kmovd %eax, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vmovdqu16 %zmm0, %zmm0 {%k1} {z} # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_build_vec_v32i1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    movl $1497715861, %eax # imm = 0x59455495
-; SKX-NEXT:    # sched: [1:0.25]
-; SKX-NEXT:    kmovd %eax, %k1 # sched: [1:1.00]
-; SKX-NEXT:    vmovdqu16 %zmm0, %zmm0 {%k1} {z} # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %ret = select <32 x i1> <i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 true, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 true, i1 true, i1 false, i1 true, i1 false>, <32 x i16> %x, <32 x i16> zeroinitializer
-  ret <32 x i16> %ret
-}
-
-define <64 x i8> @test_build_vec_v64i1(<64 x i8> %x) {
-; GENERIC-LABEL: test_build_vec_v64i1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpshufb {{.*#+}} zmm0 = zero,zero,zmm0[2],zero,zero,zero,zmm0[6],zero,zmm0[8],zero,zmm0[10],zero,zmm0[12],zero,zero,zmm0[15],zero,zero,zmm0[18],zero,zmm0[20],zero,zmm0[22],zero,zmm0[24],zero,zero,zmm0[27],zero,zero,zmm0[30],zero,zmm0[32],zero,zmm0[34],zero,zero,zero,zmm0[38],zero,zmm0[40],zero,zero,zmm0[43,44],zero,zmm0[46],zero,zmm0[48],zero,zmm0[50],zero,zero,zero,zmm0[54],zero,zmm0[56],zero,zero,zmm0[59,60],zero,zmm0[62],zero sched: [8:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_build_vec_v64i1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpshufb {{.*#+}} zmm0 = zero,zero,zmm0[2],zero,zero,zero,zmm0[6],zero,zmm0[8],zero,zmm0[10],zero,zmm0[12],zero,zero,zmm0[15],zero,zero,zmm0[18],zero,zmm0[20],zero,zmm0[22],zero,zmm0[24],zero,zero,zmm0[27],zero,zero,zmm0[30],zero,zmm0[32],zero,zmm0[34],zero,zero,zero,zmm0[38],zero,zmm0[40],zero,zero,zmm0[43,44],zero,zmm0[46],zero,zmm0[48],zero,zmm0[50],zero,zero,zero,zmm0[54],zero,zmm0[56],zero,zero,zmm0[59,60],zero,zmm0[62],zero sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %ret = select <64 x i1> <i1 false, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 true, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 true, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 true, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 false, i1 true, i1 false, i1 true, i1 false, i1 false, i1 true, i1 true, i1 false, i1 true, i1 false>, <64 x i8> %x, <64 x i8> zeroinitializer
-  ret <64 x i8> %ret
-}
-
-define void @ktest_1(<8 x double> %in, double * %base) {
-; GENERIC-LABEL: ktest_1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovupd (%rdi), %zmm1 # sched: [7:0.50]
-; GENERIC-NEXT:    vcmpltpd %zmm0, %zmm1, %k1 # sched: [3:1.00]
-; GENERIC-NEXT:    vmovupd 8(%rdi), %zmm1 {%k1} {z} # sched: [7:0.50]
-; GENERIC-NEXT:    vcmpltpd %zmm1, %zmm0, %k0 {%k1} # sched: [3:1.00]
-; GENERIC-NEXT:    kortestb %k0, %k0 # sched: [1:0.33]
-; GENERIC-NEXT:    je .LBB410_2 # sched: [1:1.00]
-; GENERIC-NEXT:  # %bb.1: # %L1
-; GENERIC-NEXT:    vmovapd %zmm0, (%rdi) # sched: [1:1.00]
-; GENERIC-NEXT:    vzeroupper # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-; GENERIC-NEXT:  .LBB410_2: # %L2
-; GENERIC-NEXT:    vmovapd %zmm0, 8(%rdi) # sched: [1:1.00]
-; GENERIC-NEXT:    vzeroupper # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: ktest_1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovupd (%rdi), %zmm1 # sched: [8:0.50]
-; SKX-NEXT:    vcmpltpd %zmm0, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vmovupd 8(%rdi), %zmm1 {%k1} {z} # sched: [8:0.50]
-; SKX-NEXT:    vcmpltpd %zmm1, %zmm0, %k0 {%k1} # sched: [3:1.00]
-; SKX-NEXT:    kortestb %k0, %k0 # sched: [3:1.00]
-; SKX-NEXT:    je .LBB410_2 # sched: [1:0.50]
-; SKX-NEXT:  # %bb.1: # %L1
-; SKX-NEXT:    vmovapd %zmm0, (%rdi) # sched: [1:1.00]
-; SKX-NEXT:    vzeroupper # sched: [0:0.67]
-; SKX-NEXT:    retq # sched: [7:1.00]
-; SKX-NEXT:  .LBB410_2: # %L2
-; SKX-NEXT:    vmovapd %zmm0, 8(%rdi) # sched: [1:1.00]
-; SKX-NEXT:    vzeroupper # sched: [0:0.67]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %addr1 = getelementptr double, double * %base, i64 0
-  %addr2 = getelementptr double, double * %base, i64 1
-
-  %vaddr1 = bitcast double* %addr1 to <8 x double>*
-  %vaddr2 = bitcast double* %addr2 to <8 x double>*
-
-  %val1 = load <8 x double>, <8 x double> *%vaddr1, align 1
-  %val2 = load <8 x double>, <8 x double> *%vaddr2, align 1
-
-  %sel1 = fcmp ogt <8 x double>%in, %val1
-  %val3 = select <8 x i1> %sel1, <8 x double> %val2, <8 x double> zeroinitializer
-  %sel2 = fcmp olt <8 x double> %in, %val3
-  %sel3 = and <8 x i1> %sel1, %sel2
-
-  %int_sel3 = bitcast <8 x i1> %sel3 to i8
-  %res = icmp eq i8 %int_sel3, zeroinitializer
-  br i1 %res, label %L2, label %L1
-L1:
-  store <8 x double> %in, <8 x double>* %vaddr1
-  br label %End
-L2:
-  store <8 x double> %in, <8 x double>* %vaddr2
-  br label %End
-End:
-  ret void
-}
-
-define void @ktest_2(<32 x float> %in, float * %base) {
-;
-; GENERIC-LABEL: ktest_2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovups (%rdi), %zmm2 # sched: [7:0.50]
-; GENERIC-NEXT:    vmovups 64(%rdi), %zmm3 # sched: [7:0.50]
-; GENERIC-NEXT:    vcmpltps %zmm0, %zmm2, %k1 # sched: [3:1.00]
-; GENERIC-NEXT:    vcmpltps %zmm1, %zmm3, %k2 # sched: [3:1.00]
-; GENERIC-NEXT:    kunpckwd %k1, %k2, %k0 # sched: [1:1.00]
-; GENERIC-NEXT:    vmovups 68(%rdi), %zmm2 {%k2} {z} # sched: [7:0.50]
-; GENERIC-NEXT:    vmovups 4(%rdi), %zmm3 {%k1} {z} # sched: [7:0.50]
-; GENERIC-NEXT:    vcmpltps %zmm3, %zmm0, %k1 # sched: [3:1.00]
-; GENERIC-NEXT:    vcmpltps %zmm2, %zmm1, %k2 # sched: [3:1.00]
-; GENERIC-NEXT:    kunpckwd %k1, %k2, %k1 # sched: [1:1.00]
-; GENERIC-NEXT:    kortestd %k1, %k0 # sched: [1:0.33]
-; GENERIC-NEXT:    je .LBB411_2 # sched: [1:1.00]
-; GENERIC-NEXT:  # %bb.1: # %L1
-; GENERIC-NEXT:    vmovaps %zmm0, (%rdi) # sched: [1:1.00]
-; GENERIC-NEXT:    vmovaps %zmm1, 64(%rdi) # sched: [1:1.00]
-; GENERIC-NEXT:    vzeroupper # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-; GENERIC-NEXT:  .LBB411_2: # %L2
-; GENERIC-NEXT:    vmovaps %zmm0, 4(%rdi) # sched: [1:1.00]
-; GENERIC-NEXT:    vmovaps %zmm1, 68(%rdi) # sched: [1:1.00]
-; GENERIC-NEXT:    vzeroupper # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: ktest_2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovups (%rdi), %zmm2 # sched: [8:0.50]
-; SKX-NEXT:    vmovups 64(%rdi), %zmm3 # sched: [8:0.50]
-; SKX-NEXT:    vcmpltps %zmm0, %zmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vcmpltps %zmm1, %zmm3, %k2 # sched: [3:1.00]
-; SKX-NEXT:    kunpckwd %k1, %k2, %k0 # sched: [3:1.00]
-; SKX-NEXT:    vmovups 68(%rdi), %zmm2 {%k2} {z} # sched: [8:0.50]
-; SKX-NEXT:    vmovups 4(%rdi), %zmm3 {%k1} {z} # sched: [8:0.50]
-; SKX-NEXT:    vcmpltps %zmm3, %zmm0, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vcmpltps %zmm2, %zmm1, %k2 # sched: [3:1.00]
-; SKX-NEXT:    kunpckwd %k1, %k2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    kortestd %k1, %k0 # sched: [3:1.00]
-; SKX-NEXT:    je .LBB411_2 # sched: [1:0.50]
-; SKX-NEXT:  # %bb.1: # %L1
-; SKX-NEXT:    vmovaps %zmm0, (%rdi) # sched: [1:1.00]
-; SKX-NEXT:    vmovaps %zmm1, 64(%rdi) # sched: [1:1.00]
-; SKX-NEXT:    vzeroupper # sched: [0:0.67]
-; SKX-NEXT:    retq # sched: [7:1.00]
-; SKX-NEXT:  .LBB411_2: # %L2
-; SKX-NEXT:    vmovaps %zmm0, 4(%rdi) # sched: [1:1.00]
-; SKX-NEXT:    vmovaps %zmm1, 68(%rdi) # sched: [1:1.00]
-; SKX-NEXT:    vzeroupper # sched: [0:0.67]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %addr1 = getelementptr float, float * %base, i64 0
-  %addr2 = getelementptr float, float * %base, i64 1
-
-  %vaddr1 = bitcast float* %addr1 to <32 x float>*
-  %vaddr2 = bitcast float* %addr2 to <32 x float>*
-
-  %val1 = load <32 x float>, <32 x float> *%vaddr1, align 1
-  %val2 = load <32 x float>, <32 x float> *%vaddr2, align 1
-
-  %sel1 = fcmp ogt <32 x float>%in, %val1
-  %val3 = select <32 x i1> %sel1, <32 x float> %val2, <32 x float> zeroinitializer
-  %sel2 = fcmp olt <32 x float> %in, %val3
-  %sel3 = or <32 x i1> %sel1, %sel2
-
-  %int_sel3 = bitcast <32 x i1> %sel3 to i32
-  %res = icmp eq i32 %int_sel3, zeroinitializer
-  br i1 %res, label %L2, label %L1
-L1:
-  store <32 x float> %in, <32 x float>* %vaddr1
-  br label %End
-L2:
-  store <32 x float> %in, <32 x float>* %vaddr2
-  br label %End
-End:
-  ret void
-}
-
-define <8 x i64> @load_8i1(<8 x i1>* %a) {
-; GENERIC-LABEL: load_8i1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    kmovb (%rdi), %k0 # sched: [5:0.50]
-; GENERIC-NEXT:    vpmovm2q %k0, %zmm0 # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: load_8i1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    kmovb (%rdi), %k0 # sched: [7:1.00]
-; SKX-NEXT:    vpmovm2q %k0, %zmm0 # sched: [1:0.25]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %b = load <8 x i1>, <8 x i1>* %a
-  %c = sext <8 x i1> %b to <8 x i64>
-  ret <8 x i64> %c
-}
-
-define <16 x i32> @load_16i1(<16 x i1>* %a) {
-; GENERIC-LABEL: load_16i1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    kmovw (%rdi), %k0 # sched: [5:0.50]
-; GENERIC-NEXT:    vpmovm2d %k0, %zmm0 # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: load_16i1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    kmovw (%rdi), %k0 # sched: [7:1.00]
-; SKX-NEXT:    vpmovm2d %k0, %zmm0 # sched: [1:0.25]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %b = load <16 x i1>, <16 x i1>* %a
-  %c = sext <16 x i1> %b to <16 x i32>
-  ret <16 x i32> %c
-}
-
-define <2 x i16> @load_2i1(<2 x i1>* %a) {
-; GENERIC-LABEL: load_2i1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    kmovb (%rdi), %k0 # sched: [5:0.50]
-; GENERIC-NEXT:    vpmovm2q %k0, %xmm0 # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: load_2i1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    kmovb (%rdi), %k0 # sched: [7:1.00]
-; SKX-NEXT:    vpmovm2q %k0, %xmm0 # sched: [1:0.25]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %b = load <2 x i1>, <2 x i1>* %a
-  %c = sext <2 x i1> %b to <2 x i16>
-  ret <2 x i16> %c
-}
-
-define <4 x i16> @load_4i1(<4 x i1>* %a) {
-; GENERIC-LABEL: load_4i1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    kmovb (%rdi), %k0 # sched: [5:0.50]
-; GENERIC-NEXT:    vpmovm2d %k0, %xmm0 # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: load_4i1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    kmovb (%rdi), %k0 # sched: [7:1.00]
-; SKX-NEXT:    vpmovm2d %k0, %xmm0 # sched: [1:0.25]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %b = load <4 x i1>, <4 x i1>* %a
-  %c = sext <4 x i1> %b to <4 x i16>
-  ret <4 x i16> %c
-}
-
-define <32 x i16> @load_32i1(<32 x i1>* %a) {
-; GENERIC-LABEL: load_32i1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    kmovd (%rdi), %k0 # sched: [5:0.50]
-; GENERIC-NEXT:    vpmovm2w %k0, %zmm0 # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: load_32i1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    kmovd (%rdi), %k0 # sched: [7:1.00]
-; SKX-NEXT:    vpmovm2w %k0, %zmm0 # sched: [1:0.25]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %b = load <32 x i1>, <32 x i1>* %a
-  %c = sext <32 x i1> %b to <32 x i16>
-  ret <32 x i16> %c
-}
-
-define <64 x i8> @load_64i1(<64 x i1>* %a) {
-; GENERIC-LABEL: load_64i1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    kmovq (%rdi), %k0 # sched: [5:0.50]
-; GENERIC-NEXT:    vpmovm2b %k0, %zmm0 # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: load_64i1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    kmovq (%rdi), %k0 # sched: [7:1.00]
-; SKX-NEXT:    vpmovm2b %k0, %zmm0 # sched: [1:0.25]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %b = load <64 x i1>, <64 x i1>* %a
-  %c = sext <64 x i1> %b to <64 x i8>
-  ret <64 x i8> %c
-}
-
-define void @store_8i1(<8 x i1>* %a, <8 x i1> %v) {
-; GENERIC-LABEL: store_8i1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpsllw $15, %xmm0, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vpmovw2m %xmm0, %k0 # sched: [1:0.33]
-; GENERIC-NEXT:    kmovb %k0, (%rdi) # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: store_8i1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpsllw $15, %xmm0, %xmm0 # sched: [1:0.50]
-; SKX-NEXT:    vpmovw2m %xmm0, %k0 # sched: [1:1.00]
-; SKX-NEXT:    kmovb %k0, (%rdi) # sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  store <8 x i1> %v, <8 x i1>* %a
-  ret void
-}
-
-define void @store_8i1_1(<8 x i1>* %a, <8 x i16> %v) {
-; GENERIC-LABEL: store_8i1_1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpsllw $15, %xmm0, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vpmovw2m %xmm0, %k0 # sched: [1:0.33]
-; GENERIC-NEXT:    kmovb %k0, (%rdi) # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: store_8i1_1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpsllw $15, %xmm0, %xmm0 # sched: [1:0.50]
-; SKX-NEXT:    vpmovw2m %xmm0, %k0 # sched: [1:1.00]
-; SKX-NEXT:    kmovb %k0, (%rdi) # sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %v1 = trunc <8 x i16> %v to <8 x i1>
-  store <8 x i1> %v1, <8 x i1>* %a
-  ret void
-}
-
-define void @store_16i1(<16 x i1>* %a, <16 x i1> %v) {
-; GENERIC-LABEL: store_16i1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpsllw $7, %xmm0, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vpmovb2m %xmm0, %k0 # sched: [1:0.33]
-; GENERIC-NEXT:    kmovw %k0, (%rdi) # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: store_16i1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpsllw $7, %xmm0, %xmm0 # sched: [1:0.50]
-; SKX-NEXT:    vpmovb2m %xmm0, %k0 # sched: [1:1.00]
-; SKX-NEXT:    kmovw %k0, (%rdi) # sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  store <16 x i1> %v, <16 x i1>* %a
-  ret void
-}
-
-define void @store_32i1(<32 x i1>* %a, <32 x i1> %v) {
-; GENERIC-LABEL: store_32i1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpsllw $7, %ymm0, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vpmovb2m %ymm0, %k0 # sched: [1:0.33]
-; GENERIC-NEXT:    kmovd %k0, (%rdi) # sched: [1:1.00]
-; GENERIC-NEXT:    vzeroupper # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: store_32i1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpsllw $7, %ymm0, %ymm0 # sched: [1:0.50]
-; SKX-NEXT:    vpmovb2m %ymm0, %k0 # sched: [1:1.00]
-; SKX-NEXT:    kmovd %k0, (%rdi) # sched: [1:1.00]
-; SKX-NEXT:    vzeroupper # sched: [0:0.67]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  store <32 x i1> %v, <32 x i1>* %a
-  ret void
-}
-
-define void @store_32i1_1(<32 x i1>* %a, <32 x i16> %v) {
-; GENERIC-LABEL: store_32i1_1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpsllw $15, %zmm0, %zmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vpmovw2m %zmm0, %k0 # sched: [1:0.33]
-; GENERIC-NEXT:    kmovd %k0, (%rdi) # sched: [1:1.00]
-; GENERIC-NEXT:    vzeroupper # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: store_32i1_1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpsllw $15, %zmm0, %zmm0 # sched: [1:1.00]
-; SKX-NEXT:    vpmovw2m %zmm0, %k0 # sched: [1:1.00]
-; SKX-NEXT:    kmovd %k0, (%rdi) # sched: [1:1.00]
-; SKX-NEXT:    vzeroupper # sched: [0:0.67]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %v1 = trunc <32 x i16> %v to <32 x i1>
-  store <32 x i1> %v1, <32 x i1>* %a
-  ret void
-}
-
-
-define void @store_64i1(<64 x i1>* %a, <64 x i1> %v) {
-;
-; GENERIC-LABEL: store_64i1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpsllw $7, %zmm0, %zmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vpmovb2m %zmm0, %k0 # sched: [1:0.33]
-; GENERIC-NEXT:    kmovq %k0, (%rdi) # sched: [1:1.00]
-; GENERIC-NEXT:    vzeroupper # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: store_64i1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpsllw $7, %zmm0, %zmm0 # sched: [1:1.00]
-; SKX-NEXT:    vpmovb2m %zmm0, %k0 # sched: [1:1.00]
-; SKX-NEXT:    kmovq %k0, (%rdi) # sched: [1:1.00]
-; SKX-NEXT:    vzeroupper # sched: [0:0.67]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  store <64 x i1> %v, <64 x i1>* %a
-  ret void
-}
-
-define i32 @test_bitcast_v8i1_zext(<16 x i32> %a) {
-; GENERIC-LABEL: test_bitcast_v8i1_zext:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %zmm0, %zmm0, %k0 # sched: [1:0.33]
-; GENERIC-NEXT:    kmovb %k0, %eax # sched: [1:0.33]
-; GENERIC-NEXT:    addl %eax, %eax # sched: [1:0.33]
-; GENERIC-NEXT:    vzeroupper # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_bitcast_v8i1_zext:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %zmm0, %zmm0, %k0 # sched: [3:1.00]
-; SKX-NEXT:    kmovb %k0, %eax # sched: [3:1.00]
-; SKX-NEXT:    addl %eax, %eax # sched: [1:0.25]
-; SKX-NEXT:    vzeroupper # sched: [0:0.67]
-; SKX-NEXT:    retq # sched: [7:1.00]
-   %v1 = icmp eq <16 x i32> %a, zeroinitializer
-   %mask = shufflevector <16 x i1> %v1, <16 x i1> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-   %mask1 = bitcast <8 x i1> %mask to i8
-   %val = zext i8 %mask1 to i32
-   %val1 = add i32 %val, %val
-   ret i32 %val1
-}
-
-define i32 @test_bitcast_v16i1_zext(<16 x i32> %a) {
-; GENERIC-LABEL: test_bitcast_v16i1_zext:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %zmm0, %zmm0, %k0 # sched: [1:0.33]
-; GENERIC-NEXT:    kmovw %k0, %eax # sched: [1:0.33]
-; GENERIC-NEXT:    addl %eax, %eax # sched: [1:0.33]
-; GENERIC-NEXT:    vzeroupper # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_bitcast_v16i1_zext:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %zmm0, %zmm0, %k0 # sched: [3:1.00]
-; SKX-NEXT:    kmovw %k0, %eax # sched: [3:1.00]
-; SKX-NEXT:    addl %eax, %eax # sched: [1:0.25]
-; SKX-NEXT:    vzeroupper # sched: [0:0.67]
-; SKX-NEXT:    retq # sched: [7:1.00]
-   %v1 = icmp eq <16 x i32> %a, zeroinitializer
-   %mask1 = bitcast <16 x i1> %v1 to i16
-   %val = zext i16 %mask1 to i32
-   %val1 = add i32 %val, %val
-   ret i32 %val1
-}
-
-define i16 @test_v16i1_add(i16 %x, i16 %y) {
-; GENERIC-LABEL: test_v16i1_add:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    kmovd %edi, %k0 # sched: [1:0.33]
-; GENERIC-NEXT:    kmovd %esi, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    kxorw %k1, %k0, %k0 # sched: [1:0.33]
-; GENERIC-NEXT:    kmovd %k0, %eax # sched: [1:0.33]
-; GENERIC-NEXT:    # kill: def $ax killed $ax killed $eax
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_v16i1_add:
-; SKX:       # %bb.0:
-; SKX-NEXT:    kmovd %edi, %k0 # sched: [1:1.00]
-; SKX-NEXT:    kmovd %esi, %k1 # sched: [1:1.00]
-; SKX-NEXT:    kxorw %k1, %k0, %k0 # sched: [1:1.00]
-; SKX-NEXT:    kmovd %k0, %eax # sched: [3:1.00]
-; SKX-NEXT:    # kill: def $ax killed $ax killed $eax
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %m0 = bitcast i16 %x to <16 x i1>
-  %m1 = bitcast i16 %y to <16 x i1>
-  %m2 = add <16 x i1> %m0,  %m1
-  %ret = bitcast <16 x i1> %m2 to i16
-  ret i16 %ret
-}
-
-define i16 @test_v16i1_sub(i16 %x, i16 %y) {
-; GENERIC-LABEL: test_v16i1_sub:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    kmovd %edi, %k0 # sched: [1:0.33]
-; GENERIC-NEXT:    kmovd %esi, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    kxorw %k1, %k0, %k0 # sched: [1:0.33]
-; GENERIC-NEXT:    kmovd %k0, %eax # sched: [1:0.33]
-; GENERIC-NEXT:    # kill: def $ax killed $ax killed $eax
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_v16i1_sub:
-; SKX:       # %bb.0:
-; SKX-NEXT:    kmovd %edi, %k0 # sched: [1:1.00]
-; SKX-NEXT:    kmovd %esi, %k1 # sched: [1:1.00]
-; SKX-NEXT:    kxorw %k1, %k0, %k0 # sched: [1:1.00]
-; SKX-NEXT:    kmovd %k0, %eax # sched: [3:1.00]
-; SKX-NEXT:    # kill: def $ax killed $ax killed $eax
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %m0 = bitcast i16 %x to <16 x i1>
-  %m1 = bitcast i16 %y to <16 x i1>
-  %m2 = sub <16 x i1> %m0,  %m1
-  %ret = bitcast <16 x i1> %m2 to i16
-  ret i16 %ret
-}
-
-define i16 @test_v16i1_mul(i16 %x, i16 %y) {
-; GENERIC-LABEL: test_v16i1_mul:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    kmovd %edi, %k0 # sched: [1:0.33]
-; GENERIC-NEXT:    kmovd %esi, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    kandw %k1, %k0, %k0 # sched: [1:0.33]
-; GENERIC-NEXT:    kmovd %k0, %eax # sched: [1:0.33]
-; GENERIC-NEXT:    # kill: def $ax killed $ax killed $eax
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_v16i1_mul:
-; SKX:       # %bb.0:
-; SKX-NEXT:    kmovd %edi, %k0 # sched: [1:1.00]
-; SKX-NEXT:    kmovd %esi, %k1 # sched: [1:1.00]
-; SKX-NEXT:    kandw %k1, %k0, %k0 # sched: [1:1.00]
-; SKX-NEXT:    kmovd %k0, %eax # sched: [3:1.00]
-; SKX-NEXT:    # kill: def $ax killed $ax killed $eax
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %m0 = bitcast i16 %x to <16 x i1>
-  %m1 = bitcast i16 %y to <16 x i1>
-  %m2 = mul <16 x i1> %m0,  %m1
-  %ret = bitcast <16 x i1> %m2 to i16
-  ret i16 %ret
-}
-
-define i8 @test_v8i1_add(i8 %x, i8 %y) {
-; GENERIC-LABEL: test_v8i1_add:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    kmovd %edi, %k0 # sched: [1:0.33]
-; GENERIC-NEXT:    kmovd %esi, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    kxorb %k1, %k0, %k0 # sched: [1:0.33]
-; GENERIC-NEXT:    kmovd %k0, %eax # sched: [1:0.33]
-; GENERIC-NEXT:    # kill: def $al killed $al killed $eax
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_v8i1_add:
-; SKX:       # %bb.0:
-; SKX-NEXT:    kmovd %edi, %k0 # sched: [1:1.00]
-; SKX-NEXT:    kmovd %esi, %k1 # sched: [1:1.00]
-; SKX-NEXT:    kxorb %k1, %k0, %k0 # sched: [1:1.00]
-; SKX-NEXT:    kmovd %k0, %eax # sched: [3:1.00]
-; SKX-NEXT:    # kill: def $al killed $al killed $eax
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %m0 = bitcast i8 %x to <8 x i1>
-  %m1 = bitcast i8 %y to <8 x i1>
-  %m2 = add <8 x i1> %m0,  %m1
-  %ret = bitcast <8 x i1> %m2 to i8
-  ret i8 %ret
-}
-
-define i8 @test_v8i1_sub(i8 %x, i8 %y) {
-; GENERIC-LABEL: test_v8i1_sub:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    kmovd %edi, %k0 # sched: [1:0.33]
-; GENERIC-NEXT:    kmovd %esi, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    kxorb %k1, %k0, %k0 # sched: [1:0.33]
-; GENERIC-NEXT:    kmovd %k0, %eax # sched: [1:0.33]
-; GENERIC-NEXT:    # kill: def $al killed $al killed $eax
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_v8i1_sub:
-; SKX:       # %bb.0:
-; SKX-NEXT:    kmovd %edi, %k0 # sched: [1:1.00]
-; SKX-NEXT:    kmovd %esi, %k1 # sched: [1:1.00]
-; SKX-NEXT:    kxorb %k1, %k0, %k0 # sched: [1:1.00]
-; SKX-NEXT:    kmovd %k0, %eax # sched: [3:1.00]
-; SKX-NEXT:    # kill: def $al killed $al killed $eax
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %m0 = bitcast i8 %x to <8 x i1>
-  %m1 = bitcast i8 %y to <8 x i1>
-  %m2 = sub <8 x i1> %m0,  %m1
-  %ret = bitcast <8 x i1> %m2 to i8
-  ret i8 %ret
-}
-
-define i8 @test_v8i1_mul(i8 %x, i8 %y) {
-; GENERIC-LABEL: test_v8i1_mul:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    kmovd %edi, %k0 # sched: [1:0.33]
-; GENERIC-NEXT:    kmovd %esi, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    kandb %k1, %k0, %k0 # sched: [1:0.33]
-; GENERIC-NEXT:    kmovd %k0, %eax # sched: [1:0.33]
-; GENERIC-NEXT:    # kill: def $al killed $al killed $eax
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_v8i1_mul:
-; SKX:       # %bb.0:
-; SKX-NEXT:    kmovd %edi, %k0 # sched: [1:1.00]
-; SKX-NEXT:    kmovd %esi, %k1 # sched: [1:1.00]
-; SKX-NEXT:    kandb %k1, %k0, %k0 # sched: [1:1.00]
-; SKX-NEXT:    kmovd %k0, %eax # sched: [3:1.00]
-; SKX-NEXT:    # kill: def $al killed $al killed $eax
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %m0 = bitcast i8 %x to <8 x i1>
-  %m1 = bitcast i8 %y to <8 x i1>
-  %m2 = mul <8 x i1> %m0,  %m1
-  %ret = bitcast <8 x i1> %m2 to i8
-  ret i8 %ret
-}
-
-define   <16 x i32> @_inreg16xi32(i32 %a) {
-; GENERIC-LABEL: _inreg16xi32:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpbroadcastd %edi, %zmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: _inreg16xi32:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpbroadcastd %edi, %zmm0 # sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %b = insertelement <16 x i32> undef, i32 %a, i32 0
-  %c = shufflevector <16 x i32> %b, <16 x i32> undef, <16 x i32> zeroinitializer
-  ret <16 x i32> %c
-}
-
-define   <8 x i64> @_inreg8xi64(i64 %a) {
-; GENERIC-LABEL: _inreg8xi64:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpbroadcastq %rdi, %zmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: _inreg8xi64:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpbroadcastq %rdi, %zmm0 # sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %b = insertelement <8 x i64> undef, i64 %a, i32 0
-  %c = shufflevector <8 x i64> %b, <8 x i64> undef, <8 x i32> zeroinitializer
-  ret <8 x i64> %c
-}
-
-define   <16 x float> @_ss16xfloat_v4(<4 x float> %a) {
-; GENERIC-LABEL: _ss16xfloat_v4:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vbroadcastss %xmm0, %zmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: _ss16xfloat_v4:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vbroadcastss %xmm0, %zmm0 # sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %b = shufflevector <4 x float> %a, <4 x float> undef, <16 x i32> zeroinitializer
-  ret <16 x float> %b
-}
-
-define   <16 x float> @_inreg16xfloat(float %a) {
-; GENERIC-LABEL: _inreg16xfloat:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vbroadcastss %xmm0, %zmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: _inreg16xfloat:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vbroadcastss %xmm0, %zmm0 # sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %b = insertelement <16 x float> undef, float %a, i32 0
-  %c = shufflevector <16 x float> %b, <16 x float> undef, <16 x i32> zeroinitializer
-  ret <16 x float> %c
-}
-
-define   <16 x float> @_ss16xfloat_mask(float %a, <16 x float> %i, <16 x i32> %mask1) {
-; GENERIC-LABEL: _ss16xfloat_mask:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestmd %zmm2, %zmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vbroadcastss %xmm0, %zmm1 {%k1} # sched: [1:1.00]
-; GENERIC-NEXT:    vmovaps %zmm1, %zmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: _ss16xfloat_mask:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestmd %zmm2, %zmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vbroadcastss %xmm0, %zmm1 {%k1} # sched: [3:1.00]
-; SKX-NEXT:    vmovaps %zmm1, %zmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %mask = icmp ne <16 x i32> %mask1, zeroinitializer
-  %b = insertelement <16 x float> undef, float %a, i32 0
-  %c = shufflevector <16 x float> %b, <16 x float> undef, <16 x i32> zeroinitializer
-  %r = select <16 x i1> %mask, <16 x float> %c, <16 x float> %i
-  ret <16 x float> %r
-}
-
-define   <16 x float> @_ss16xfloat_maskz(float %a, <16 x i32> %mask1) {
-; GENERIC-LABEL: _ss16xfloat_maskz:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestmd %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vbroadcastss %xmm0, %zmm0 {%k1} {z} # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: _ss16xfloat_maskz:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestmd %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vbroadcastss %xmm0, %zmm0 {%k1} {z} # sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %mask = icmp ne <16 x i32> %mask1, zeroinitializer
-  %b = insertelement <16 x float> undef, float %a, i32 0
-  %c = shufflevector <16 x float> %b, <16 x float> undef, <16 x i32> zeroinitializer
-  %r = select <16 x i1> %mask, <16 x float> %c, <16 x float> zeroinitializer
-  ret <16 x float> %r
-}
-
-define   <16 x float> @_ss16xfloat_load(float* %a.ptr) {
-; GENERIC-LABEL: _ss16xfloat_load:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vbroadcastss (%rdi), %zmm0 # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: _ss16xfloat_load:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vbroadcastss (%rdi), %zmm0 # sched: [8:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %a = load float, float* %a.ptr
-  %b = insertelement <16 x float> undef, float %a, i32 0
-  %c = shufflevector <16 x float> %b, <16 x float> undef, <16 x i32> zeroinitializer
-  ret <16 x float> %c
-}
-
-define   <16 x float> @_ss16xfloat_mask_load(float* %a.ptr, <16 x float> %i, <16 x i32> %mask1) {
-; GENERIC-LABEL: _ss16xfloat_mask_load:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestmd %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vbroadcastss (%rdi), %zmm0 {%k1} # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: _ss16xfloat_mask_load:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestmd %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vbroadcastss (%rdi), %zmm0 {%k1} # sched: [8:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %a = load float, float* %a.ptr
-  %mask = icmp ne <16 x i32> %mask1, zeroinitializer
-  %b = insertelement <16 x float> undef, float %a, i32 0
-  %c = shufflevector <16 x float> %b, <16 x float> undef, <16 x i32> zeroinitializer
-  %r = select <16 x i1> %mask, <16 x float> %c, <16 x float> %i
-  ret <16 x float> %r
-}
-
-define   <16 x float> @_ss16xfloat_maskz_load(float* %a.ptr, <16 x i32> %mask1) {
-; GENERIC-LABEL: _ss16xfloat_maskz_load:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestmd %zmm0, %zmm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vbroadcastss (%rdi), %zmm0 {%k1} {z} # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: _ss16xfloat_maskz_load:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestmd %zmm0, %zmm0, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vbroadcastss (%rdi), %zmm0 {%k1} {z} # sched: [8:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %a = load float, float* %a.ptr
-  %mask = icmp ne <16 x i32> %mask1, zeroinitializer
-  %b = insertelement <16 x float> undef, float %a, i32 0
-  %c = shufflevector <16 x float> %b, <16 x float> undef, <16 x i32> zeroinitializer
-  %r = select <16 x i1> %mask, <16 x float> %c, <16 x float> zeroinitializer
-  ret <16 x float> %r
-}
-
-define   <8 x double> @_inreg8xdouble(double %a) {
-; GENERIC-LABEL: _inreg8xdouble:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vbroadcastsd %xmm0, %zmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: _inreg8xdouble:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vbroadcastsd %xmm0, %zmm0 # sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %b = insertelement <8 x double> undef, double %a, i32 0
-  %c = shufflevector <8 x double> %b, <8 x double> undef, <8 x i32> zeroinitializer
-  ret <8 x double> %c
-}
-
-define   <8 x double> @_sd8xdouble_mask(double %a, <8 x double> %i, <8 x i32> %mask1) {
-; GENERIC-LABEL: _sd8xdouble_mask:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestmd %ymm2, %ymm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vbroadcastsd %xmm0, %zmm1 {%k1} # sched: [1:1.00]
-; GENERIC-NEXT:    vmovapd %zmm1, %zmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: _sd8xdouble_mask:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestmd %ymm2, %ymm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vbroadcastsd %xmm0, %zmm1 {%k1} # sched: [3:1.00]
-; SKX-NEXT:    vmovapd %zmm1, %zmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %mask = icmp ne <8 x i32> %mask1, zeroinitializer
-  %b = insertelement <8 x double> undef, double %a, i32 0
-  %c = shufflevector <8 x double> %b, <8 x double> undef, <8 x i32> zeroinitializer
-  %r = select <8 x i1> %mask, <8 x double> %c, <8 x double> %i
-  ret <8 x double> %r
-}
-
-define   <8 x double> @_sd8xdouble_maskz(double %a, <8 x i32> %mask1) {
-; GENERIC-LABEL: _sd8xdouble_maskz:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestmd %ymm1, %ymm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vbroadcastsd %xmm0, %zmm0 {%k1} {z} # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: _sd8xdouble_maskz:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestmd %ymm1, %ymm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vbroadcastsd %xmm0, %zmm0 {%k1} {z} # sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %mask = icmp ne <8 x i32> %mask1, zeroinitializer
-  %b = insertelement <8 x double> undef, double %a, i32 0
-  %c = shufflevector <8 x double> %b, <8 x double> undef, <8 x i32> zeroinitializer
-  %r = select <8 x i1> %mask, <8 x double> %c, <8 x double> zeroinitializer
-  ret <8 x double> %r
-}
-
-define   <8 x double> @_sd8xdouble_load(double* %a.ptr) {
-; GENERIC-LABEL: _sd8xdouble_load:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vbroadcastsd (%rdi), %zmm0 # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: _sd8xdouble_load:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vbroadcastsd (%rdi), %zmm0 # sched: [8:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %a = load double, double* %a.ptr
-  %b = insertelement <8 x double> undef, double %a, i32 0
-  %c = shufflevector <8 x double> %b, <8 x double> undef, <8 x i32> zeroinitializer
-  ret <8 x double> %c
-}
-
-define   <8 x double> @_sd8xdouble_mask_load(double* %a.ptr, <8 x double> %i, <8 x i32> %mask1) {
-; GENERIC-LABEL: _sd8xdouble_mask_load:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestmd %ymm1, %ymm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vbroadcastsd (%rdi), %zmm0 {%k1} # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: _sd8xdouble_mask_load:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestmd %ymm1, %ymm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vbroadcastsd (%rdi), %zmm0 {%k1} # sched: [8:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %a = load double, double* %a.ptr
-  %mask = icmp ne <8 x i32> %mask1, zeroinitializer
-  %b = insertelement <8 x double> undef, double %a, i32 0
-  %c = shufflevector <8 x double> %b, <8 x double> undef, <8 x i32> zeroinitializer
-  %r = select <8 x i1> %mask, <8 x double> %c, <8 x double> %i
-  ret <8 x double> %r
-}
-
-define   <8 x double> @_sd8xdouble_maskz_load(double* %a.ptr, <8 x i32> %mask1) {
-; GENERIC-LABEL: _sd8xdouble_maskz_load:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestmd %ymm0, %ymm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vbroadcastsd (%rdi), %zmm0 {%k1} {z} # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: _sd8xdouble_maskz_load:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestmd %ymm0, %ymm0, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vbroadcastsd (%rdi), %zmm0 {%k1} {z} # sched: [8:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %a = load double, double* %a.ptr
-  %mask = icmp ne <8 x i32> %mask1, zeroinitializer
-  %b = insertelement <8 x double> undef, double %a, i32 0
-  %c = shufflevector <8 x double> %b, <8 x double> undef, <8 x i32> zeroinitializer
-  %r = select <8 x i1> %mask, <8 x double> %c, <8 x double> zeroinitializer
-  ret <8 x double> %r
-}
-
-define   <16 x i32> @_xmm16xi32(<16 x i32> %a) {
-; GENERIC-LABEL: _xmm16xi32:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vbroadcastss %xmm0, %zmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: _xmm16xi32:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vbroadcastss %xmm0, %zmm0 # sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %b = shufflevector <16 x i32> %a, <16 x i32> undef, <16 x i32> zeroinitializer
-  ret <16 x i32> %b
-}
-
-define   <16 x float> @_xmm16xfloat(<16 x float> %a) {
-; GENERIC-LABEL: _xmm16xfloat:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vbroadcastss %xmm0, %zmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: _xmm16xfloat:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vbroadcastss %xmm0, %zmm0 # sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %b = shufflevector <16 x float> %a, <16 x float> undef, <16 x i32> zeroinitializer
-  ret <16 x float> %b
-}
-
-define <16 x i32> @test_vbroadcast() {
-; GENERIC-LABEL: test_vbroadcast:
-; GENERIC:       # %bb.0: # %entry
-; GENERIC-NEXT:    vxorps %xmm0, %xmm0, %xmm0 # sched: [0:0.25]
-; GENERIC-NEXT:    vcmpunordps %zmm0, %zmm0, %k0 # sched: [3:1.00]
-; GENERIC-NEXT:    vpmovm2d %k0, %zmm0 # sched: [1:0.33]
-; GENERIC-NEXT:    knotw %k0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vmovdqa32 %zmm0, %zmm0 {%k1} {z} # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_vbroadcast:
-; SKX:       # %bb.0: # %entry
-; SKX-NEXT:    vxorps %xmm0, %xmm0, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    vcmpunordps %zmm0, %zmm0, %k0 # sched: [3:1.00]
-; SKX-NEXT:    vpmovm2d %k0, %zmm0 # sched: [1:0.25]
-; SKX-NEXT:    knotw %k0, %k1 # sched: [1:1.00]
-; SKX-NEXT:    vmovdqa32 %zmm0, %zmm0 {%k1} {z} # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-entry:
-  %0 = sext <16 x i1> zeroinitializer to <16 x i32>
-  %1 = fcmp uno <16 x float> undef, zeroinitializer
-  %2 = sext <16 x i1> %1 to <16 x i32>
-  %3 = select <16 x i1> %1, <16 x i32> %0, <16 x i32> %2
-  ret <16 x i32> %3
-}
-
-; We implement the set1 intrinsics with vector initializers.  Verify that the
-; IR generated will produce broadcasts at the end.
-define <8 x double> @test_set1_pd(double %d) #2 {
-; GENERIC-LABEL: test_set1_pd:
-; GENERIC:       # %bb.0: # %entry
-; GENERIC-NEXT:    vbroadcastsd %xmm0, %zmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_set1_pd:
-; SKX:       # %bb.0: # %entry
-; SKX-NEXT:    vbroadcastsd %xmm0, %zmm0 # sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-entry:
-  %vecinit.i = insertelement <8 x double> undef, double %d, i32 0
-  %vecinit1.i = insertelement <8 x double> %vecinit.i, double %d, i32 1
-  %vecinit2.i = insertelement <8 x double> %vecinit1.i, double %d, i32 2
-  %vecinit3.i = insertelement <8 x double> %vecinit2.i, double %d, i32 3
-  %vecinit4.i = insertelement <8 x double> %vecinit3.i, double %d, i32 4
-  %vecinit5.i = insertelement <8 x double> %vecinit4.i, double %d, i32 5
-  %vecinit6.i = insertelement <8 x double> %vecinit5.i, double %d, i32 6
-  %vecinit7.i = insertelement <8 x double> %vecinit6.i, double %d, i32 7
-  ret <8 x double> %vecinit7.i
-}
-
-define <8 x i64> @test_set1_epi64(i64 %d) #2 {
-; GENERIC-LABEL: test_set1_epi64:
-; GENERIC:       # %bb.0: # %entry
-; GENERIC-NEXT:    vpbroadcastq %rdi, %zmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_set1_epi64:
-; SKX:       # %bb.0: # %entry
-; SKX-NEXT:    vpbroadcastq %rdi, %zmm0 # sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-entry:
-  %vecinit.i = insertelement <8 x i64> undef, i64 %d, i32 0
-  %vecinit1.i = insertelement <8 x i64> %vecinit.i, i64 %d, i32 1
-  %vecinit2.i = insertelement <8 x i64> %vecinit1.i, i64 %d, i32 2
-  %vecinit3.i = insertelement <8 x i64> %vecinit2.i, i64 %d, i32 3
-  %vecinit4.i = insertelement <8 x i64> %vecinit3.i, i64 %d, i32 4
-  %vecinit5.i = insertelement <8 x i64> %vecinit4.i, i64 %d, i32 5
-  %vecinit6.i = insertelement <8 x i64> %vecinit5.i, i64 %d, i32 6
-  %vecinit7.i = insertelement <8 x i64> %vecinit6.i, i64 %d, i32 7
-  ret <8 x i64> %vecinit7.i
-}
-
-define <16 x float> @test_set1_ps(float %f) #2 {
-; GENERIC-LABEL: test_set1_ps:
-; GENERIC:       # %bb.0: # %entry
-; GENERIC-NEXT:    vbroadcastss %xmm0, %zmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_set1_ps:
-; SKX:       # %bb.0: # %entry
-; SKX-NEXT:    vbroadcastss %xmm0, %zmm0 # sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-entry:
-  %vecinit.i = insertelement <16 x float> undef, float %f, i32 0
-  %vecinit1.i = insertelement <16 x float> %vecinit.i, float %f, i32 1
-  %vecinit2.i = insertelement <16 x float> %vecinit1.i, float %f, i32 2
-  %vecinit3.i = insertelement <16 x float> %vecinit2.i, float %f, i32 3
-  %vecinit4.i = insertelement <16 x float> %vecinit3.i, float %f, i32 4
-  %vecinit5.i = insertelement <16 x float> %vecinit4.i, float %f, i32 5
-  %vecinit6.i = insertelement <16 x float> %vecinit5.i, float %f, i32 6
-  %vecinit7.i = insertelement <16 x float> %vecinit6.i, float %f, i32 7
-  %vecinit8.i = insertelement <16 x float> %vecinit7.i, float %f, i32 8
-  %vecinit9.i = insertelement <16 x float> %vecinit8.i, float %f, i32 9
-  %vecinit10.i = insertelement <16 x float> %vecinit9.i, float %f, i32 10
-  %vecinit11.i = insertelement <16 x float> %vecinit10.i, float %f, i32 11
-  %vecinit12.i = insertelement <16 x float> %vecinit11.i, float %f, i32 12
-  %vecinit13.i = insertelement <16 x float> %vecinit12.i, float %f, i32 13
-  %vecinit14.i = insertelement <16 x float> %vecinit13.i, float %f, i32 14
-  %vecinit15.i = insertelement <16 x float> %vecinit14.i, float %f, i32 15
-  ret <16 x float> %vecinit15.i
-}
-
-define <16 x i32> @test_set1_epi32(i32 %f) #2 {
-; GENERIC-LABEL: test_set1_epi32:
-; GENERIC:       # %bb.0: # %entry
-; GENERIC-NEXT:    vpbroadcastd %edi, %zmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_set1_epi32:
-; SKX:       # %bb.0: # %entry
-; SKX-NEXT:    vpbroadcastd %edi, %zmm0 # sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-entry:
-  %vecinit.i = insertelement <16 x i32> undef, i32 %f, i32 0
-  %vecinit1.i = insertelement <16 x i32> %vecinit.i, i32 %f, i32 1
-  %vecinit2.i = insertelement <16 x i32> %vecinit1.i, i32 %f, i32 2
-  %vecinit3.i = insertelement <16 x i32> %vecinit2.i, i32 %f, i32 3
-  %vecinit4.i = insertelement <16 x i32> %vecinit3.i, i32 %f, i32 4
-  %vecinit5.i = insertelement <16 x i32> %vecinit4.i, i32 %f, i32 5
-  %vecinit6.i = insertelement <16 x i32> %vecinit5.i, i32 %f, i32 6
-  %vecinit7.i = insertelement <16 x i32> %vecinit6.i, i32 %f, i32 7
-  %vecinit8.i = insertelement <16 x i32> %vecinit7.i, i32 %f, i32 8
-  %vecinit9.i = insertelement <16 x i32> %vecinit8.i, i32 %f, i32 9
-  %vecinit10.i = insertelement <16 x i32> %vecinit9.i, i32 %f, i32 10
-  %vecinit11.i = insertelement <16 x i32> %vecinit10.i, i32 %f, i32 11
-  %vecinit12.i = insertelement <16 x i32> %vecinit11.i, i32 %f, i32 12
-  %vecinit13.i = insertelement <16 x i32> %vecinit12.i, i32 %f, i32 13
-  %vecinit14.i = insertelement <16 x i32> %vecinit13.i, i32 %f, i32 14
-  %vecinit15.i = insertelement <16 x i32> %vecinit14.i, i32 %f, i32 15
-  ret <16 x i32> %vecinit15.i
-}
-
-; We implement the scalar broadcast intrinsics with vector initializers.
-; Verify that the IR generated will produce the broadcast at the end.
-define <8 x double> @test_mm512_broadcastsd_pd(<2 x double> %a) {
-; GENERIC-LABEL: test_mm512_broadcastsd_pd:
-; GENERIC:       # %bb.0: # %entry
-; GENERIC-NEXT:    vbroadcastsd %xmm0, %zmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_mm512_broadcastsd_pd:
-; SKX:       # %bb.0: # %entry
-; SKX-NEXT:    vbroadcastsd %xmm0, %zmm0 # sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-entry:
-  %0 = extractelement <2 x double> %a, i32 0
-  %vecinit.i = insertelement <8 x double> undef, double %0, i32 0
-  %vecinit1.i = insertelement <8 x double> %vecinit.i, double %0, i32 1
-  %vecinit2.i = insertelement <8 x double> %vecinit1.i, double %0, i32 2
-  %vecinit3.i = insertelement <8 x double> %vecinit2.i, double %0, i32 3
-  %vecinit4.i = insertelement <8 x double> %vecinit3.i, double %0, i32 4
-  %vecinit5.i = insertelement <8 x double> %vecinit4.i, double %0, i32 5
-  %vecinit6.i = insertelement <8 x double> %vecinit5.i, double %0, i32 6
-  %vecinit7.i = insertelement <8 x double> %vecinit6.i, double %0, i32 7
-  ret <8 x double> %vecinit7.i
-}
-
-define <16 x float> @suff_test1(<8 x float>%a)  {
-; GENERIC-LABEL: suff_test1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vbroadcastss %xmm0, %zmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: suff_test1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vbroadcastss %xmm0, %zmm0 # sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %res = shufflevector <8 x float> %a, <8 x float> undef, <16 x i32> zeroinitializer
-  ret <16 x float>%res
-}
-
-define <8 x double> @suff_test2(<4 x double>%a)  {
-; GENERIC-LABEL: suff_test2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vbroadcastsd %xmm0, %zmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: suff_test2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vbroadcastsd %xmm0, %zmm0 # sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %res = shufflevector <4 x double> %a, <4 x double> undef, <8 x i32> zeroinitializer
-  ret <8 x double>%res
-}
-
-define <64 x i8> @_invec32xi8(<32 x i8>%a)  {
-; GENERIC-LABEL: _invec32xi8:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpbroadcastb %xmm0, %zmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: _invec32xi8:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpbroadcastb %xmm0, %zmm0 # sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %res = shufflevector <32 x i8> %a, <32 x i8> undef, <64 x i32> zeroinitializer
-  ret <64 x i8>%res
-}
-
-define <32 x i16> @_invec16xi16(<16 x i16>%a)  {
-; GENERIC-LABEL: _invec16xi16:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpbroadcastw %xmm0, %zmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: _invec16xi16:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpbroadcastw %xmm0, %zmm0 # sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %res = shufflevector <16 x i16> %a, <16 x i16> undef, <32 x i32> zeroinitializer
-  ret <32 x i16>%res
-}
-
-define <16 x i32> @_invec8xi32(<8 x i32>%a)  {
-; GENERIC-LABEL: _invec8xi32:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vbroadcastss %xmm0, %zmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: _invec8xi32:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vbroadcastss %xmm0, %zmm0 # sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %res = shufflevector <8 x i32> %a, <8 x i32> undef, <16 x i32> zeroinitializer
-  ret <16 x i32>%res
-}
-
-define <8 x i64> @_invec4xi64(<4 x i64>%a)  {
-; GENERIC-LABEL: _invec4xi64:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vbroadcastsd %xmm0, %zmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: _invec4xi64:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vbroadcastsd %xmm0, %zmm0 # sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %res = shufflevector <4 x i64> %a, <4 x i64> undef, <8 x i32> zeroinitializer
-  ret <8 x i64>%res
-}
-
-declare void @func_f32(float)
-define <16 x float> @broadcast_ss_spill(float %x) {
-; GENERIC-LABEL: broadcast_ss_spill:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    subq $24, %rsp # sched: [1:0.33]
-; GENERIC-NEXT:    .cfi_def_cfa_offset 32
-; GENERIC-NEXT:    vaddss %xmm0, %xmm0, %xmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    vmovaps %xmm0, (%rsp) # 16-byte Spill sched: [1:1.00]
-; GENERIC-NEXT:    callq func_f32
-; GENERIC-NEXT:    vbroadcastss (%rsp), %zmm0 # 16-byte Folded Reload sched: [8:1.00]
-; GENERIC-NEXT:    addq $24, %rsp # sched: [1:0.33]
-; GENERIC-NEXT:    .cfi_def_cfa_offset 8
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: broadcast_ss_spill:
-; SKX:       # %bb.0:
-; SKX-NEXT:    subq $24, %rsp # sched: [1:0.25]
-; SKX-NEXT:    .cfi_def_cfa_offset 32
-; SKX-NEXT:    vaddss %xmm0, %xmm0, %xmm0 # sched: [4:0.50]
-; SKX-NEXT:    vmovaps %xmm0, (%rsp) # 16-byte Spill sched: [1:1.00]
-; SKX-NEXT:    callq func_f32
-; SKX-NEXT:    vbroadcastss (%rsp), %zmm0 # 16-byte Folded Reload sched: [8:0.50]
-; SKX-NEXT:    addq $24, %rsp # sched: [1:0.25]
-; SKX-NEXT:    .cfi_def_cfa_offset 8
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %a  = fadd float %x, %x
-  call void @func_f32(float %a)
-  %b = insertelement <16 x float> undef, float %a, i32 0
-  %c = shufflevector <16 x float> %b, <16 x float> undef, <16 x i32> zeroinitializer
-  ret <16 x float> %c
-}
-
-declare void @func_f64(double)
-define <8 x double> @broadcast_sd_spill(double %x) {
-; GENERIC-LABEL: broadcast_sd_spill:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    subq $24, %rsp # sched: [1:0.33]
-; GENERIC-NEXT:    .cfi_def_cfa_offset 32
-; GENERIC-NEXT:    vaddsd %xmm0, %xmm0, %xmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    vmovapd %xmm0, (%rsp) # 16-byte Spill sched: [1:1.00]
-; GENERIC-NEXT:    callq func_f64
-; GENERIC-NEXT:    vbroadcastsd (%rsp), %zmm0 # 16-byte Folded Reload sched: [8:1.00]
-; GENERIC-NEXT:    addq $24, %rsp # sched: [1:0.33]
-; GENERIC-NEXT:    .cfi_def_cfa_offset 8
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: broadcast_sd_spill:
-; SKX:       # %bb.0:
-; SKX-NEXT:    subq $24, %rsp # sched: [1:0.25]
-; SKX-NEXT:    .cfi_def_cfa_offset 32
-; SKX-NEXT:    vaddsd %xmm0, %xmm0, %xmm0 # sched: [4:0.50]
-; SKX-NEXT:    vmovapd %xmm0, (%rsp) # 16-byte Spill sched: [1:1.00]
-; SKX-NEXT:    callq func_f64
-; SKX-NEXT:    vbroadcastsd (%rsp), %zmm0 # 16-byte Folded Reload sched: [8:0.50]
-; SKX-NEXT:    addq $24, %rsp # sched: [1:0.25]
-; SKX-NEXT:    .cfi_def_cfa_offset 8
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %a  = fadd double %x, %x
-  call void @func_f64(double %a)
-  %b = insertelement <8 x double> undef, double %a, i32 0
-  %c = shufflevector <8 x double> %b, <8 x double> undef, <8 x i32> zeroinitializer
-  ret <8 x double> %c
-}
diff --git a/test/CodeGen/X86/avx512-select.ll b/test/CodeGen/X86/avx512-select.ll
index 1a5f78db17febcc445ca1f15603bc2fe79effee1..a34c64a375010b24887ee1112df95e4066e252ba 100644
--- a/test/CodeGen/X86/avx512-select.ll
+++ b/test/CodeGen/X86/avx512-select.ll
@@ -151,10 +151,8 @@ define i8 @select05_mem(<8 x i1>* %a.0, <8 x i1>* %m) {
 ; X86:       # %bb.0:
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT:    movzbl (%ecx), %ecx
-; X86-NEXT:    kmovw %ecx, %k0
-; X86-NEXT:    movzbl (%eax), %eax
-; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    kmovw (%ecx), %k0
+; X86-NEXT:    kmovw (%eax), %k1
 ; X86-NEXT:    korw %k1, %k0, %k0
 ; X86-NEXT:    kmovw %k0, %eax
 ; X86-NEXT:    # kill: def $al killed $al killed $eax
@@ -162,10 +160,8 @@ define i8 @select05_mem(<8 x i1>* %a.0, <8 x i1>* %m) {
 ;
 ; X64-LABEL: select05_mem:
 ; X64:       # %bb.0:
-; X64-NEXT:    movzbl (%rsi), %eax
-; X64-NEXT:    kmovw %eax, %k0
-; X64-NEXT:    movzbl (%rdi), %eax
-; X64-NEXT:    kmovw %eax, %k1
+; X64-NEXT:    kmovw (%rsi), %k0
+; X64-NEXT:    kmovw (%rdi), %k1
 ; X64-NEXT:    korw %k1, %k0, %k0
 ; X64-NEXT:    kmovw %k0, %eax
 ; X64-NEXT:    # kill: def $al killed $al killed $eax
@@ -202,10 +198,8 @@ define i8 @select06_mem(<8 x i1>* %a.0, <8 x i1>* %m) {
 ; X86:       # %bb.0:
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT:    movzbl (%ecx), %ecx
-; X86-NEXT:    kmovw %ecx, %k0
-; X86-NEXT:    movzbl (%eax), %eax
-; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    kmovw (%ecx), %k0
+; X86-NEXT:    kmovw (%eax), %k1
 ; X86-NEXT:    kandw %k1, %k0, %k0
 ; X86-NEXT:    kmovw %k0, %eax
 ; X86-NEXT:    # kill: def $al killed $al killed $eax
@@ -213,10 +207,8 @@ define i8 @select06_mem(<8 x i1>* %a.0, <8 x i1>* %m) {
 ;
 ; X64-LABEL: select06_mem:
 ; X64:       # %bb.0:
-; X64-NEXT:    movzbl (%rsi), %eax
-; X64-NEXT:    kmovw %eax, %k0
-; X64-NEXT:    movzbl (%rdi), %eax
-; X64-NEXT:    kmovw %eax, %k1
+; X64-NEXT:    kmovw (%rsi), %k0
+; X64-NEXT:    kmovw (%rdi), %k1
 ; X64-NEXT:    kandw %k1, %k0, %k0
 ; X64-NEXT:    kmovw %k0, %eax
 ; X64-NEXT:    # kill: def $al killed $al killed $eax
diff --git a/test/CodeGen/X86/avx512-shift.ll b/test/CodeGen/X86/avx512-shift.ll
index eda27c21adfe2d49fb4b67f552baeee8177d33c6..8cf08b852892a0ccb1da6a612ee02de4393d172d 100644
--- a/test/CodeGen/X86/avx512-shift.ll
+++ b/test/CodeGen/X86/avx512-shift.ll
@@ -1,52 +1,93 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-;RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=knl | FileCheck %s --check-prefix=CHECK --check-prefix=KNL
-;RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=skx | FileCheck %s --check-prefix=CHECK --check-prefix=SKX
+;RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=knl | FileCheck %s --check-prefixes=CHECK,KNL
+;RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=skx | FileCheck %s --check-prefixes=CHECK,SKX
 
-define <16 x i32> @shift_16_i32(<16 x i32> %a) {
-; CHECK-LABEL: shift_16_i32:
+define <16 x i32> @ashr_16_i32(<16 x i32> %a) {
+; CHECK-LABEL: ashr_16_i32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vpsrld $1, %zmm0, %zmm0
-; CHECK-NEXT:    vpslld $12, %zmm0, %zmm0
 ; CHECK-NEXT:    vpsrad $12, %zmm0, %zmm0
+; CHECK-NEXT:    retq
+   %b = ashr <16 x i32> %a, <i32 12, i32 12, i32 12, i32 12, i32 12, i32 12, i32 12, i32 12, i32 12, i32 12, i32 12, i32 12, i32 12, i32 12, i32 12, i32 12>
+   ret <16 x i32> %b
+}
+
+define <16 x i32> @lshr_16_i32(<16 x i32> %a) {
+; CHECK-LABEL: lshr_16_i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpsrld $1, %zmm0, %zmm0
 ; CHECK-NEXT:    retq
    %b = lshr <16 x i32> %a, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
-   %c = shl <16 x i32> %b, <i32 12, i32 12, i32 12, i32 12, i32 12, i32 12, i32 12, i32 12, i32 12, i32 12, i32 12, i32 12, i32 12, i32 12, i32 12, i32 12>
-   %d = ashr <16 x i32> %c, <i32 12, i32 12, i32 12, i32 12, i32 12, i32 12, i32 12, i32 12, i32 12, i32 12, i32 12, i32 12, i32 12, i32 12, i32 12, i32 12>
-   ret <16 x i32> %d;
+   ret <16 x i32> %b
 }
 
-define <8 x i64> @shift_8_i64(<8 x i64> %a) {
-; CHECK-LABEL: shift_8_i64:
+define <16 x i32> @shl_16_i32(<16 x i32> %a) {
+; CHECK-LABEL: shl_16_i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpslld $12, %zmm0, %zmm0
+; CHECK-NEXT:    retq
+   %b = shl <16 x i32> %a, <i32 12, i32 12, i32 12, i32 12, i32 12, i32 12, i32 12, i32 12, i32 12, i32 12, i32 12, i32 12, i32 12, i32 12, i32 12, i32 12>
+   ret <16 x i32> %b
+}
+
+define <8 x i64> @ashr_8_i64(<8 x i64> %a) {
+; CHECK-LABEL: ashr_8_i64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vpsrlq $1, %zmm0, %zmm0
-; CHECK-NEXT:    vpsllq $12, %zmm0, %zmm0
 ; CHECK-NEXT:    vpsraq $12, %zmm0, %zmm0
+; CHECK-NEXT:    retq
+   %b = ashr <8 x i64> %a, <i64 12, i64 12, i64 12, i64 12, i64 12, i64 12, i64 12, i64 12>
+   ret <8 x i64> %b
+}
+
+define <8 x i64> @lshr_8_i64(<8 x i64> %a) {
+; CHECK-LABEL: lshr_8_i64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpsrlq $1, %zmm0, %zmm0
 ; CHECK-NEXT:    retq
    %b = lshr <8 x i64> %a, <i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1>
-   %c = shl <8 x i64> %b,  <i64 12, i64 12, i64 12, i64 12, i64 12, i64 12, i64 12, i64 12>
-   %d = ashr <8 x i64> %c, <i64 12, i64 12, i64 12, i64 12, i64 12, i64 12, i64 12, i64 12>
-   ret <8 x i64> %d;
+   ret <8 x i64> %b
+}
+
+define <8 x i64> @shl_8_i64(<8 x i64> %a) {
+; CHECK-LABEL: shl_8_i64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpsllq $12, %zmm0, %zmm0
+; CHECK-NEXT:    retq
+   %b = shl <8 x i64> %a, <i64 12, i64 12, i64 12, i64 12, i64 12, i64 12, i64 12, i64 12>
+   ret <8 x i64> %b
 }
 
-define <4 x i64> @shift_4_i64(<4 x i64> %a) {
-; KNL-LABEL: shift_4_i64:
+define <4 x i64> @ashr_4_i64(<4 x i64> %a) {
+; KNL-LABEL: ashr_4_i64:
 ; KNL:       # %bb.0:
-; KNL-NEXT:    vpsrlq $1, %ymm0, %ymm0
-; KNL-NEXT:    vpsllq $12, %ymm0, %ymm0
+; KNL-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
 ; KNL-NEXT:    vpsraq $12, %zmm0, %zmm0
 ; KNL-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
 ; KNL-NEXT:    retq
 ;
-; SKX-LABEL: shift_4_i64:
+; SKX-LABEL: ashr_4_i64:
 ; SKX:       # %bb.0:
-; SKX-NEXT:    vpsrlq $1, %ymm0, %ymm0
-; SKX-NEXT:    vpsllq $12, %ymm0, %ymm0
 ; SKX-NEXT:    vpsraq $12, %ymm0, %ymm0
 ; SKX-NEXT:    retq
+   %b = ashr <4 x i64> %a, <i64 12, i64 12, i64 12, i64 12>
+   ret <4 x i64> %b
+}
+
+define <4 x i64> @lshr_4_i64(<4 x i64> %a) {
+; CHECK-LABEL: lshr_4_i64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpsrlq $1, %ymm0, %ymm0
+; CHECK-NEXT:    retq
    %b = lshr <4 x i64> %a, <i64 1, i64 1, i64 1, i64 1>
-   %c = shl <4 x i64> %b,  <i64 12, i64 12, i64 12, i64 12>
-   %d = ashr <4 x i64> %c, <i64 12, i64 12, i64 12, i64 12>
-   ret <4 x i64> %d;
+   ret <4 x i64> %b
+}
+
+define <4 x i64> @shl_4_i64(<4 x i64> %a) {
+; CHECK-LABEL: shl_4_i64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpsllq $12, %ymm0, %ymm0
+; CHECK-NEXT:    retq
+   %b = shl <4 x i64> %a,  <i64 12, i64 12, i64 12, i64 12>
+   ret <4 x i64> %b
 }
 
 define <8 x i64> @variable_shl4(<8 x i64> %x, <8 x i64> %y) {
diff --git a/test/CodeGen/X86/avx512-shuffle-schedule.ll b/test/CodeGen/X86/avx512-shuffle-schedule.ll
deleted file mode 100755
index 26f4bd578c24dbb0257f891af579bc1d5a5bfb5f..0000000000000000000000000000000000000000
--- a/test/CodeGen/X86/avx512-shuffle-schedule.ll
+++ /dev/null
@@ -1,15629 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=x86-64 -mattr=+avx512f,+avx512dq,+avx512bw,+avx512vl | FileCheck %s --check-prefix=GENERIC
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=skx | FileCheck %s --check-prefix=SKX
-
-; This test is an assembly of avx512 shuffling instructions to check their scheduling
-
-define <16 x i16> @test_16xi16_perm_mask0(<16 x i16> %vec) {
-; GENERIC-LABEL: test_16xi16_perm_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovdqa {{.*#+}} ymm1 = [8,6,12,4,7,9,14,8,4,12,9,4,14,15,12,14] sched: [7:0.50]
-; GENERIC-NEXT:    vpermw %ymm0, %ymm1, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_16xi16_perm_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovdqa {{.*#+}} ymm1 = [8,6,12,4,7,9,14,8,4,12,9,4,14,15,12,14] sched: [7:0.50]
-; SKX-NEXT:    vpermw %ymm0, %ymm1, %ymm0 # sched: [6:2.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %res = shufflevector <16 x i16> %vec, <16 x i16> undef, <16 x i32> <i32 8, i32 6, i32 12, i32 4, i32 7, i32 9, i32 14, i32 8, i32 4, i32 12, i32 9, i32 4, i32 14, i32 15, i32 12, i32 14>
-  ret <16 x i16> %res
-}
-define <16 x i16> @test_masked_16xi16_perm_mask0(<16 x i16> %vec, <16 x i16> %vec2, <16 x i16> %mask) {
-; GENERIC-LABEL: test_masked_16xi16_perm_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovdqa {{.*#+}} ymm3 = [8,6,12,4,7,9,14,8,4,12,9,4,14,15,12,14] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmw %ymm2, %ymm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermw %ymm0, %ymm3, %ymm1 {%k1} # sched: [1:1.00]
-; GENERIC-NEXT:    vmovdqa %ymm1, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_16xi16_perm_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovdqa {{.*#+}} ymm3 = [8,6,12,4,7,9,14,8,4,12,9,4,14,15,12,14] sched: [7:0.50]
-; SKX-NEXT:    vptestnmw %ymm2, %ymm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermw %ymm0, %ymm3, %ymm1 {%k1} # sched: [6:2.00]
-; SKX-NEXT:    vmovdqa %ymm1, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <16 x i16> %vec, <16 x i16> undef, <16 x i32> <i32 8, i32 6, i32 12, i32 4, i32 7, i32 9, i32 14, i32 8, i32 4, i32 12, i32 9, i32 4, i32 14, i32 15, i32 12, i32 14>
-  %cmp = icmp eq <16 x i16> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x i16> %shuf, <16 x i16> %vec2
-  ret <16 x i16> %res
-}
-
-define <16 x i16> @test_masked_z_16xi16_perm_mask0(<16 x i16> %vec, <16 x i16> %mask) {
-; GENERIC-LABEL: test_masked_z_16xi16_perm_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovdqa {{.*#+}} ymm2 = [8,6,12,4,7,9,14,8,4,12,9,4,14,15,12,14] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmw %ymm1, %ymm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermw %ymm0, %ymm2, %ymm0 {%k1} {z} # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_16xi16_perm_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovdqa {{.*#+}} ymm2 = [8,6,12,4,7,9,14,8,4,12,9,4,14,15,12,14] sched: [7:0.50]
-; SKX-NEXT:    vptestnmw %ymm1, %ymm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermw %ymm0, %ymm2, %ymm0 {%k1} {z} # sched: [6:2.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <16 x i16> %vec, <16 x i16> undef, <16 x i32> <i32 8, i32 6, i32 12, i32 4, i32 7, i32 9, i32 14, i32 8, i32 4, i32 12, i32 9, i32 4, i32 14, i32 15, i32 12, i32 14>
-  %cmp = icmp eq <16 x i16> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x i16> %shuf, <16 x i16> zeroinitializer
-  ret <16 x i16> %res
-}
-define <16 x i16> @test_masked_16xi16_perm_mask1(<16 x i16> %vec, <16 x i16> %vec2, <16 x i16> %mask) {
-; GENERIC-LABEL: test_masked_16xi16_perm_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovdqa {{.*#+}} ymm3 = [4,11,14,10,7,1,6,9,14,15,7,13,4,12,8,0] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmw %ymm2, %ymm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermw %ymm0, %ymm3, %ymm1 {%k1} # sched: [1:1.00]
-; GENERIC-NEXT:    vmovdqa %ymm1, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_16xi16_perm_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovdqa {{.*#+}} ymm3 = [4,11,14,10,7,1,6,9,14,15,7,13,4,12,8,0] sched: [7:0.50]
-; SKX-NEXT:    vptestnmw %ymm2, %ymm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermw %ymm0, %ymm3, %ymm1 {%k1} # sched: [6:2.00]
-; SKX-NEXT:    vmovdqa %ymm1, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <16 x i16> %vec, <16 x i16> undef, <16 x i32> <i32 4, i32 11, i32 14, i32 10, i32 7, i32 1, i32 6, i32 9, i32 14, i32 15, i32 7, i32 13, i32 4, i32 12, i32 8, i32 0>
-  %cmp = icmp eq <16 x i16> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x i16> %shuf, <16 x i16> %vec2
-  ret <16 x i16> %res
-}
-
-define <16 x i16> @test_masked_z_16xi16_perm_mask1(<16 x i16> %vec, <16 x i16> %mask) {
-; GENERIC-LABEL: test_masked_z_16xi16_perm_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovdqa {{.*#+}} ymm2 = [4,11,14,10,7,1,6,9,14,15,7,13,4,12,8,0] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmw %ymm1, %ymm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermw %ymm0, %ymm2, %ymm0 {%k1} {z} # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_16xi16_perm_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovdqa {{.*#+}} ymm2 = [4,11,14,10,7,1,6,9,14,15,7,13,4,12,8,0] sched: [7:0.50]
-; SKX-NEXT:    vptestnmw %ymm1, %ymm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermw %ymm0, %ymm2, %ymm0 {%k1} {z} # sched: [6:2.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <16 x i16> %vec, <16 x i16> undef, <16 x i32> <i32 4, i32 11, i32 14, i32 10, i32 7, i32 1, i32 6, i32 9, i32 14, i32 15, i32 7, i32 13, i32 4, i32 12, i32 8, i32 0>
-  %cmp = icmp eq <16 x i16> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x i16> %shuf, <16 x i16> zeroinitializer
-  ret <16 x i16> %res
-}
-define <16 x i16> @test_masked_16xi16_perm_mask2(<16 x i16> %vec, <16 x i16> %vec2, <16 x i16> %mask) {
-; GENERIC-LABEL: test_masked_16xi16_perm_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovdqa {{.*#+}} ymm3 = [11,6,13,10,0,7,13,3,5,13,3,9,3,15,12,7] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmw %ymm2, %ymm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermw %ymm0, %ymm3, %ymm1 {%k1} # sched: [1:1.00]
-; GENERIC-NEXT:    vmovdqa %ymm1, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_16xi16_perm_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovdqa {{.*#+}} ymm3 = [11,6,13,10,0,7,13,3,5,13,3,9,3,15,12,7] sched: [7:0.50]
-; SKX-NEXT:    vptestnmw %ymm2, %ymm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermw %ymm0, %ymm3, %ymm1 {%k1} # sched: [6:2.00]
-; SKX-NEXT:    vmovdqa %ymm1, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <16 x i16> %vec, <16 x i16> undef, <16 x i32> <i32 11, i32 6, i32 13, i32 10, i32 0, i32 7, i32 13, i32 3, i32 5, i32 13, i32 3, i32 9, i32 3, i32 15, i32 12, i32 7>
-  %cmp = icmp eq <16 x i16> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x i16> %shuf, <16 x i16> %vec2
-  ret <16 x i16> %res
-}
-
-define <16 x i16> @test_masked_z_16xi16_perm_mask2(<16 x i16> %vec, <16 x i16> %mask) {
-; GENERIC-LABEL: test_masked_z_16xi16_perm_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovdqa {{.*#+}} ymm2 = [11,6,13,10,0,7,13,3,5,13,3,9,3,15,12,7] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmw %ymm1, %ymm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermw %ymm0, %ymm2, %ymm0 {%k1} {z} # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_16xi16_perm_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovdqa {{.*#+}} ymm2 = [11,6,13,10,0,7,13,3,5,13,3,9,3,15,12,7] sched: [7:0.50]
-; SKX-NEXT:    vptestnmw %ymm1, %ymm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermw %ymm0, %ymm2, %ymm0 {%k1} {z} # sched: [6:2.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <16 x i16> %vec, <16 x i16> undef, <16 x i32> <i32 11, i32 6, i32 13, i32 10, i32 0, i32 7, i32 13, i32 3, i32 5, i32 13, i32 3, i32 9, i32 3, i32 15, i32 12, i32 7>
-  %cmp = icmp eq <16 x i16> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x i16> %shuf, <16 x i16> zeroinitializer
-  ret <16 x i16> %res
-}
-define <16 x i16> @test_16xi16_perm_mask3(<16 x i16> %vec) {
-; GENERIC-LABEL: test_16xi16_perm_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovdqa {{.*#+}} ymm1 = [1,5,8,14,1,8,11,8,13,8,15,9,9,7,9,6] sched: [7:0.50]
-; GENERIC-NEXT:    vpermw %ymm0, %ymm1, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_16xi16_perm_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovdqa {{.*#+}} ymm1 = [1,5,8,14,1,8,11,8,13,8,15,9,9,7,9,6] sched: [7:0.50]
-; SKX-NEXT:    vpermw %ymm0, %ymm1, %ymm0 # sched: [6:2.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %res = shufflevector <16 x i16> %vec, <16 x i16> undef, <16 x i32> <i32 1, i32 5, i32 8, i32 14, i32 1, i32 8, i32 11, i32 8, i32 13, i32 8, i32 15, i32 9, i32 9, i32 7, i32 9, i32 6>
-  ret <16 x i16> %res
-}
-define <16 x i16> @test_masked_16xi16_perm_mask3(<16 x i16> %vec, <16 x i16> %vec2, <16 x i16> %mask) {
-; GENERIC-LABEL: test_masked_16xi16_perm_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovdqa {{.*#+}} ymm3 = [1,5,8,14,1,8,11,8,13,8,15,9,9,7,9,6] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmw %ymm2, %ymm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermw %ymm0, %ymm3, %ymm1 {%k1} # sched: [1:1.00]
-; GENERIC-NEXT:    vmovdqa %ymm1, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_16xi16_perm_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovdqa {{.*#+}} ymm3 = [1,5,8,14,1,8,11,8,13,8,15,9,9,7,9,6] sched: [7:0.50]
-; SKX-NEXT:    vptestnmw %ymm2, %ymm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermw %ymm0, %ymm3, %ymm1 {%k1} # sched: [6:2.00]
-; SKX-NEXT:    vmovdqa %ymm1, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <16 x i16> %vec, <16 x i16> undef, <16 x i32> <i32 1, i32 5, i32 8, i32 14, i32 1, i32 8, i32 11, i32 8, i32 13, i32 8, i32 15, i32 9, i32 9, i32 7, i32 9, i32 6>
-  %cmp = icmp eq <16 x i16> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x i16> %shuf, <16 x i16> %vec2
-  ret <16 x i16> %res
-}
-
-define <16 x i16> @test_masked_z_16xi16_perm_mask3(<16 x i16> %vec, <16 x i16> %mask) {
-; GENERIC-LABEL: test_masked_z_16xi16_perm_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovdqa {{.*#+}} ymm2 = [1,5,8,14,1,8,11,8,13,8,15,9,9,7,9,6] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmw %ymm1, %ymm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermw %ymm0, %ymm2, %ymm0 {%k1} {z} # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_16xi16_perm_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovdqa {{.*#+}} ymm2 = [1,5,8,14,1,8,11,8,13,8,15,9,9,7,9,6] sched: [7:0.50]
-; SKX-NEXT:    vptestnmw %ymm1, %ymm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermw %ymm0, %ymm2, %ymm0 {%k1} {z} # sched: [6:2.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <16 x i16> %vec, <16 x i16> undef, <16 x i32> <i32 1, i32 5, i32 8, i32 14, i32 1, i32 8, i32 11, i32 8, i32 13, i32 8, i32 15, i32 9, i32 9, i32 7, i32 9, i32 6>
-  %cmp = icmp eq <16 x i16> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x i16> %shuf, <16 x i16> zeroinitializer
-  ret <16 x i16> %res
-}
-define <16 x i16> @test_16xi16_perm_mem_mask0(<16 x i16>* %vp) {
-; GENERIC-LABEL: test_16xi16_perm_mem_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovdqa {{.*#+}} ymm0 = [9,10,7,1,12,14,14,13,14,14,8,6,11,4,12,13] sched: [7:0.50]
-; GENERIC-NEXT:    vpermw (%rdi), %ymm0, %ymm0 # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_16xi16_perm_mem_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovdqa {{.*#+}} ymm0 = [9,10,7,1,12,14,14,13,14,14,8,6,11,4,12,13] sched: [7:0.50]
-; SKX-NEXT:    vpermw (%rdi), %ymm0, %ymm0 # sched: [13:2.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <16 x i16>, <16 x i16>* %vp
-  %res = shufflevector <16 x i16> %vec, <16 x i16> undef, <16 x i32> <i32 9, i32 10, i32 7, i32 1, i32 12, i32 14, i32 14, i32 13, i32 14, i32 14, i32 8, i32 6, i32 11, i32 4, i32 12, i32 13>
-  ret <16 x i16> %res
-}
-define <16 x i16> @test_masked_16xi16_perm_mem_mask0(<16 x i16>* %vp, <16 x i16> %vec2, <16 x i16> %mask) {
-; GENERIC-LABEL: test_masked_16xi16_perm_mem_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovdqa {{.*#+}} ymm2 = [9,10,7,1,12,14,14,13,14,14,8,6,11,4,12,13] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmw %ymm1, %ymm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermw (%rdi), %ymm2, %ymm0 {%k1} # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_16xi16_perm_mem_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovdqa {{.*#+}} ymm2 = [9,10,7,1,12,14,14,13,14,14,8,6,11,4,12,13] sched: [7:0.50]
-; SKX-NEXT:    vptestnmw %ymm1, %ymm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermw (%rdi), %ymm2, %ymm0 {%k1} # sched: [13:2.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <16 x i16>, <16 x i16>* %vp
-  %shuf = shufflevector <16 x i16> %vec, <16 x i16> undef, <16 x i32> <i32 9, i32 10, i32 7, i32 1, i32 12, i32 14, i32 14, i32 13, i32 14, i32 14, i32 8, i32 6, i32 11, i32 4, i32 12, i32 13>
-  %cmp = icmp eq <16 x i16> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x i16> %shuf, <16 x i16> %vec2
-  ret <16 x i16> %res
-}
-
-define <16 x i16> @test_masked_z_16xi16_perm_mem_mask0(<16 x i16>* %vp, <16 x i16> %mask) {
-; GENERIC-LABEL: test_masked_z_16xi16_perm_mem_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovdqa {{.*#+}} ymm1 = [9,10,7,1,12,14,14,13,14,14,8,6,11,4,12,13] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmw %ymm0, %ymm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermw (%rdi), %ymm1, %ymm0 {%k1} {z} # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_16xi16_perm_mem_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovdqa {{.*#+}} ymm1 = [9,10,7,1,12,14,14,13,14,14,8,6,11,4,12,13] sched: [7:0.50]
-; SKX-NEXT:    vptestnmw %ymm0, %ymm0, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermw (%rdi), %ymm1, %ymm0 {%k1} {z} # sched: [13:2.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <16 x i16>, <16 x i16>* %vp
-  %shuf = shufflevector <16 x i16> %vec, <16 x i16> undef, <16 x i32> <i32 9, i32 10, i32 7, i32 1, i32 12, i32 14, i32 14, i32 13, i32 14, i32 14, i32 8, i32 6, i32 11, i32 4, i32 12, i32 13>
-  %cmp = icmp eq <16 x i16> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x i16> %shuf, <16 x i16> zeroinitializer
-  ret <16 x i16> %res
-}
-
-define <16 x i16> @test_masked_16xi16_perm_mem_mask1(<16 x i16>* %vp, <16 x i16> %vec2, <16 x i16> %mask) {
-; GENERIC-LABEL: test_masked_16xi16_perm_mem_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovdqa {{.*#+}} ymm2 = [14,9,15,9,7,10,15,14,12,1,9,7,10,13,3,11] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmw %ymm1, %ymm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermw (%rdi), %ymm2, %ymm0 {%k1} # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_16xi16_perm_mem_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovdqa {{.*#+}} ymm2 = [14,9,15,9,7,10,15,14,12,1,9,7,10,13,3,11] sched: [7:0.50]
-; SKX-NEXT:    vptestnmw %ymm1, %ymm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermw (%rdi), %ymm2, %ymm0 {%k1} # sched: [13:2.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <16 x i16>, <16 x i16>* %vp
-  %shuf = shufflevector <16 x i16> %vec, <16 x i16> undef, <16 x i32> <i32 14, i32 9, i32 15, i32 9, i32 7, i32 10, i32 15, i32 14, i32 12, i32 1, i32 9, i32 7, i32 10, i32 13, i32 3, i32 11>
-  %cmp = icmp eq <16 x i16> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x i16> %shuf, <16 x i16> %vec2
-  ret <16 x i16> %res
-}
-
-define <16 x i16> @test_masked_z_16xi16_perm_mem_mask1(<16 x i16>* %vp, <16 x i16> %mask) {
-; GENERIC-LABEL: test_masked_z_16xi16_perm_mem_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovdqa {{.*#+}} ymm1 = [14,9,15,9,7,10,15,14,12,1,9,7,10,13,3,11] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmw %ymm0, %ymm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermw (%rdi), %ymm1, %ymm0 {%k1} {z} # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_16xi16_perm_mem_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovdqa {{.*#+}} ymm1 = [14,9,15,9,7,10,15,14,12,1,9,7,10,13,3,11] sched: [7:0.50]
-; SKX-NEXT:    vptestnmw %ymm0, %ymm0, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermw (%rdi), %ymm1, %ymm0 {%k1} {z} # sched: [13:2.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <16 x i16>, <16 x i16>* %vp
-  %shuf = shufflevector <16 x i16> %vec, <16 x i16> undef, <16 x i32> <i32 14, i32 9, i32 15, i32 9, i32 7, i32 10, i32 15, i32 14, i32 12, i32 1, i32 9, i32 7, i32 10, i32 13, i32 3, i32 11>
-  %cmp = icmp eq <16 x i16> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x i16> %shuf, <16 x i16> zeroinitializer
-  ret <16 x i16> %res
-}
-
-define <16 x i16> @test_masked_16xi16_perm_mem_mask2(<16 x i16>* %vp, <16 x i16> %vec2, <16 x i16> %mask) {
-; GENERIC-LABEL: test_masked_16xi16_perm_mem_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovdqa {{.*#+}} ymm2 = [1,3,12,5,13,1,2,11,0,9,14,8,10,0,10,9] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmw %ymm1, %ymm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermw (%rdi), %ymm2, %ymm0 {%k1} # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_16xi16_perm_mem_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovdqa {{.*#+}} ymm2 = [1,3,12,5,13,1,2,11,0,9,14,8,10,0,10,9] sched: [7:0.50]
-; SKX-NEXT:    vptestnmw %ymm1, %ymm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermw (%rdi), %ymm2, %ymm0 {%k1} # sched: [13:2.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <16 x i16>, <16 x i16>* %vp
-  %shuf = shufflevector <16 x i16> %vec, <16 x i16> undef, <16 x i32> <i32 1, i32 3, i32 12, i32 5, i32 13, i32 1, i32 2, i32 11, i32 0, i32 9, i32 14, i32 8, i32 10, i32 0, i32 10, i32 9>
-  %cmp = icmp eq <16 x i16> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x i16> %shuf, <16 x i16> %vec2
-  ret <16 x i16> %res
-}
-
-define <16 x i16> @test_masked_z_16xi16_perm_mem_mask2(<16 x i16>* %vp, <16 x i16> %mask) {
-; GENERIC-LABEL: test_masked_z_16xi16_perm_mem_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovdqa {{.*#+}} ymm1 = [1,3,12,5,13,1,2,11,0,9,14,8,10,0,10,9] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmw %ymm0, %ymm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermw (%rdi), %ymm1, %ymm0 {%k1} {z} # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_16xi16_perm_mem_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovdqa {{.*#+}} ymm1 = [1,3,12,5,13,1,2,11,0,9,14,8,10,0,10,9] sched: [7:0.50]
-; SKX-NEXT:    vptestnmw %ymm0, %ymm0, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermw (%rdi), %ymm1, %ymm0 {%k1} {z} # sched: [13:2.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <16 x i16>, <16 x i16>* %vp
-  %shuf = shufflevector <16 x i16> %vec, <16 x i16> undef, <16 x i32> <i32 1, i32 3, i32 12, i32 5, i32 13, i32 1, i32 2, i32 11, i32 0, i32 9, i32 14, i32 8, i32 10, i32 0, i32 10, i32 9>
-  %cmp = icmp eq <16 x i16> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x i16> %shuf, <16 x i16> zeroinitializer
-  ret <16 x i16> %res
-}
-
-define <16 x i16> @test_16xi16_perm_mem_mask3(<16 x i16>* %vp) {
-; GENERIC-LABEL: test_16xi16_perm_mem_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovdqa {{.*#+}} ymm0 = [9,6,5,15,0,0,15,2,1,3,12,14,0,6,1,4] sched: [7:0.50]
-; GENERIC-NEXT:    vpermw (%rdi), %ymm0, %ymm0 # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_16xi16_perm_mem_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovdqa {{.*#+}} ymm0 = [9,6,5,15,0,0,15,2,1,3,12,14,0,6,1,4] sched: [7:0.50]
-; SKX-NEXT:    vpermw (%rdi), %ymm0, %ymm0 # sched: [13:2.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <16 x i16>, <16 x i16>* %vp
-  %res = shufflevector <16 x i16> %vec, <16 x i16> undef, <16 x i32> <i32 9, i32 6, i32 5, i32 15, i32 0, i32 0, i32 15, i32 2, i32 1, i32 3, i32 12, i32 14, i32 0, i32 6, i32 1, i32 4>
-  ret <16 x i16> %res
-}
-define <16 x i16> @test_masked_16xi16_perm_mem_mask3(<16 x i16>* %vp, <16 x i16> %vec2, <16 x i16> %mask) {
-; GENERIC-LABEL: test_masked_16xi16_perm_mem_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovdqa {{.*#+}} ymm2 = [9,6,5,15,0,0,15,2,1,3,12,14,0,6,1,4] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmw %ymm1, %ymm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermw (%rdi), %ymm2, %ymm0 {%k1} # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_16xi16_perm_mem_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovdqa {{.*#+}} ymm2 = [9,6,5,15,0,0,15,2,1,3,12,14,0,6,1,4] sched: [7:0.50]
-; SKX-NEXT:    vptestnmw %ymm1, %ymm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermw (%rdi), %ymm2, %ymm0 {%k1} # sched: [13:2.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <16 x i16>, <16 x i16>* %vp
-  %shuf = shufflevector <16 x i16> %vec, <16 x i16> undef, <16 x i32> <i32 9, i32 6, i32 5, i32 15, i32 0, i32 0, i32 15, i32 2, i32 1, i32 3, i32 12, i32 14, i32 0, i32 6, i32 1, i32 4>
-  %cmp = icmp eq <16 x i16> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x i16> %shuf, <16 x i16> %vec2
-  ret <16 x i16> %res
-}
-
-define <16 x i16> @test_masked_z_16xi16_perm_mem_mask3(<16 x i16>* %vp, <16 x i16> %mask) {
-; GENERIC-LABEL: test_masked_z_16xi16_perm_mem_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovdqa {{.*#+}} ymm1 = [9,6,5,15,0,0,15,2,1,3,12,14,0,6,1,4] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmw %ymm0, %ymm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermw (%rdi), %ymm1, %ymm0 {%k1} {z} # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_16xi16_perm_mem_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovdqa {{.*#+}} ymm1 = [9,6,5,15,0,0,15,2,1,3,12,14,0,6,1,4] sched: [7:0.50]
-; SKX-NEXT:    vptestnmw %ymm0, %ymm0, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermw (%rdi), %ymm1, %ymm0 {%k1} {z} # sched: [13:2.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <16 x i16>, <16 x i16>* %vp
-  %shuf = shufflevector <16 x i16> %vec, <16 x i16> undef, <16 x i32> <i32 9, i32 6, i32 5, i32 15, i32 0, i32 0, i32 15, i32 2, i32 1, i32 3, i32 12, i32 14, i32 0, i32 6, i32 1, i32 4>
-  %cmp = icmp eq <16 x i16> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x i16> %shuf, <16 x i16> zeroinitializer
-  ret <16 x i16> %res
-}
-
-define <32 x i16> @test_32xi16_perm_mask0(<32 x i16> %vec) {
-; GENERIC-LABEL: test_32xi16_perm_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [16,1,3,31,6,11,23,26,29,5,21,30,1,21,27,10,8,19,14,5,15,13,18,16,9,11,26,8,17,0,23,10] sched: [7:0.50]
-; GENERIC-NEXT:    vpermw %zmm0, %zmm1, %zmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_32xi16_perm_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [16,1,3,31,6,11,23,26,29,5,21,30,1,21,27,10,8,19,14,5,15,13,18,16,9,11,26,8,17,0,23,10] sched: [8:0.50]
-; SKX-NEXT:    vpermw %zmm0, %zmm1, %zmm0 # sched: [6:2.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %res = shufflevector <32 x i16> %vec, <32 x i16> undef, <32 x i32> <i32 16, i32 1, i32 3, i32 31, i32 6, i32 11, i32 23, i32 26, i32 29, i32 5, i32 21, i32 30, i32 1, i32 21, i32 27, i32 10, i32 8, i32 19, i32 14, i32 5, i32 15, i32 13, i32 18, i32 16, i32 9, i32 11, i32 26, i32 8, i32 17, i32 0, i32 23, i32 10>
-  ret <32 x i16> %res
-}
-define <32 x i16> @test_masked_32xi16_perm_mask0(<32 x i16> %vec, <32 x i16> %vec2, <32 x i16> %mask) {
-; GENERIC-LABEL: test_masked_32xi16_perm_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [16,1,3,31,6,11,23,26,29,5,21,30,1,21,27,10,8,19,14,5,15,13,18,16,9,11,26,8,17,0,23,10] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmw %zmm2, %zmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermw %zmm0, %zmm3, %zmm1 {%k1} # sched: [1:1.00]
-; GENERIC-NEXT:    vmovdqa64 %zmm1, %zmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_32xi16_perm_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [16,1,3,31,6,11,23,26,29,5,21,30,1,21,27,10,8,19,14,5,15,13,18,16,9,11,26,8,17,0,23,10] sched: [8:0.50]
-; SKX-NEXT:    vptestnmw %zmm2, %zmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermw %zmm0, %zmm3, %zmm1 {%k1} # sched: [6:2.00]
-; SKX-NEXT:    vmovdqa64 %zmm1, %zmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <32 x i16> %vec, <32 x i16> undef, <32 x i32> <i32 16, i32 1, i32 3, i32 31, i32 6, i32 11, i32 23, i32 26, i32 29, i32 5, i32 21, i32 30, i32 1, i32 21, i32 27, i32 10, i32 8, i32 19, i32 14, i32 5, i32 15, i32 13, i32 18, i32 16, i32 9, i32 11, i32 26, i32 8, i32 17, i32 0, i32 23, i32 10>
-  %cmp = icmp eq <32 x i16> %mask, zeroinitializer
-  %res = select <32 x i1> %cmp, <32 x i16> %shuf, <32 x i16> %vec2
-  ret <32 x i16> %res
-}
-
-define <32 x i16> @test_masked_z_32xi16_perm_mask0(<32 x i16> %vec, <32 x i16> %mask) {
-; GENERIC-LABEL: test_masked_z_32xi16_perm_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [16,1,3,31,6,11,23,26,29,5,21,30,1,21,27,10,8,19,14,5,15,13,18,16,9,11,26,8,17,0,23,10] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmw %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermw %zmm0, %zmm2, %zmm0 {%k1} {z} # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_32xi16_perm_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [16,1,3,31,6,11,23,26,29,5,21,30,1,21,27,10,8,19,14,5,15,13,18,16,9,11,26,8,17,0,23,10] sched: [8:0.50]
-; SKX-NEXT:    vptestnmw %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermw %zmm0, %zmm2, %zmm0 {%k1} {z} # sched: [6:2.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <32 x i16> %vec, <32 x i16> undef, <32 x i32> <i32 16, i32 1, i32 3, i32 31, i32 6, i32 11, i32 23, i32 26, i32 29, i32 5, i32 21, i32 30, i32 1, i32 21, i32 27, i32 10, i32 8, i32 19, i32 14, i32 5, i32 15, i32 13, i32 18, i32 16, i32 9, i32 11, i32 26, i32 8, i32 17, i32 0, i32 23, i32 10>
-  %cmp = icmp eq <32 x i16> %mask, zeroinitializer
-  %res = select <32 x i1> %cmp, <32 x i16> %shuf, <32 x i16> zeroinitializer
-  ret <32 x i16> %res
-}
-define <32 x i16> @test_masked_32xi16_perm_mask1(<32 x i16> %vec, <32 x i16> %vec2, <32 x i16> %mask) {
-; GENERIC-LABEL: test_masked_32xi16_perm_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [1,8,7,30,11,9,11,30,20,19,22,12,13,20,0,6,10,7,20,12,28,18,13,12,22,13,21,1,14,8,5,16] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmw %zmm2, %zmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermw %zmm0, %zmm3, %zmm1 {%k1} # sched: [1:1.00]
-; GENERIC-NEXT:    vmovdqa64 %zmm1, %zmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_32xi16_perm_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [1,8,7,30,11,9,11,30,20,19,22,12,13,20,0,6,10,7,20,12,28,18,13,12,22,13,21,1,14,8,5,16] sched: [8:0.50]
-; SKX-NEXT:    vptestnmw %zmm2, %zmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermw %zmm0, %zmm3, %zmm1 {%k1} # sched: [6:2.00]
-; SKX-NEXT:    vmovdqa64 %zmm1, %zmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <32 x i16> %vec, <32 x i16> undef, <32 x i32> <i32 1, i32 8, i32 7, i32 30, i32 11, i32 9, i32 11, i32 30, i32 20, i32 19, i32 22, i32 12, i32 13, i32 20, i32 0, i32 6, i32 10, i32 7, i32 20, i32 12, i32 28, i32 18, i32 13, i32 12, i32 22, i32 13, i32 21, i32 1, i32 14, i32 8, i32 5, i32 16>
-  %cmp = icmp eq <32 x i16> %mask, zeroinitializer
-  %res = select <32 x i1> %cmp, <32 x i16> %shuf, <32 x i16> %vec2
-  ret <32 x i16> %res
-}
-
-define <32 x i16> @test_masked_z_32xi16_perm_mask1(<32 x i16> %vec, <32 x i16> %mask) {
-; GENERIC-LABEL: test_masked_z_32xi16_perm_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [1,8,7,30,11,9,11,30,20,19,22,12,13,20,0,6,10,7,20,12,28,18,13,12,22,13,21,1,14,8,5,16] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmw %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermw %zmm0, %zmm2, %zmm0 {%k1} {z} # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_32xi16_perm_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [1,8,7,30,11,9,11,30,20,19,22,12,13,20,0,6,10,7,20,12,28,18,13,12,22,13,21,1,14,8,5,16] sched: [8:0.50]
-; SKX-NEXT:    vptestnmw %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermw %zmm0, %zmm2, %zmm0 {%k1} {z} # sched: [6:2.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <32 x i16> %vec, <32 x i16> undef, <32 x i32> <i32 1, i32 8, i32 7, i32 30, i32 11, i32 9, i32 11, i32 30, i32 20, i32 19, i32 22, i32 12, i32 13, i32 20, i32 0, i32 6, i32 10, i32 7, i32 20, i32 12, i32 28, i32 18, i32 13, i32 12, i32 22, i32 13, i32 21, i32 1, i32 14, i32 8, i32 5, i32 16>
-  %cmp = icmp eq <32 x i16> %mask, zeroinitializer
-  %res = select <32 x i1> %cmp, <32 x i16> %shuf, <32 x i16> zeroinitializer
-  ret <32 x i16> %res
-}
-define <32 x i16> @test_masked_32xi16_perm_mask2(<32 x i16> %vec, <32 x i16> %vec2, <32 x i16> %mask) {
-; GENERIC-LABEL: test_masked_32xi16_perm_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [15,17,24,28,15,9,14,25,28,25,6,31,20,2,23,31,12,21,10,6,22,0,26,16,3,3,20,27,8,31,3,27] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmw %zmm2, %zmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermw %zmm0, %zmm3, %zmm1 {%k1} # sched: [1:1.00]
-; GENERIC-NEXT:    vmovdqa64 %zmm1, %zmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_32xi16_perm_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [15,17,24,28,15,9,14,25,28,25,6,31,20,2,23,31,12,21,10,6,22,0,26,16,3,3,20,27,8,31,3,27] sched: [8:0.50]
-; SKX-NEXT:    vptestnmw %zmm2, %zmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermw %zmm0, %zmm3, %zmm1 {%k1} # sched: [6:2.00]
-; SKX-NEXT:    vmovdqa64 %zmm1, %zmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <32 x i16> %vec, <32 x i16> undef, <32 x i32> <i32 15, i32 17, i32 24, i32 28, i32 15, i32 9, i32 14, i32 25, i32 28, i32 25, i32 6, i32 31, i32 20, i32 2, i32 23, i32 31, i32 12, i32 21, i32 10, i32 6, i32 22, i32 0, i32 26, i32 16, i32 3, i32 3, i32 20, i32 27, i32 8, i32 31, i32 3, i32 27>
-  %cmp = icmp eq <32 x i16> %mask, zeroinitializer
-  %res = select <32 x i1> %cmp, <32 x i16> %shuf, <32 x i16> %vec2
-  ret <32 x i16> %res
-}
-
-define <32 x i16> @test_masked_z_32xi16_perm_mask2(<32 x i16> %vec, <32 x i16> %mask) {
-; GENERIC-LABEL: test_masked_z_32xi16_perm_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [15,17,24,28,15,9,14,25,28,25,6,31,20,2,23,31,12,21,10,6,22,0,26,16,3,3,20,27,8,31,3,27] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmw %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermw %zmm0, %zmm2, %zmm0 {%k1} {z} # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_32xi16_perm_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [15,17,24,28,15,9,14,25,28,25,6,31,20,2,23,31,12,21,10,6,22,0,26,16,3,3,20,27,8,31,3,27] sched: [8:0.50]
-; SKX-NEXT:    vptestnmw %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermw %zmm0, %zmm2, %zmm0 {%k1} {z} # sched: [6:2.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <32 x i16> %vec, <32 x i16> undef, <32 x i32> <i32 15, i32 17, i32 24, i32 28, i32 15, i32 9, i32 14, i32 25, i32 28, i32 25, i32 6, i32 31, i32 20, i32 2, i32 23, i32 31, i32 12, i32 21, i32 10, i32 6, i32 22, i32 0, i32 26, i32 16, i32 3, i32 3, i32 20, i32 27, i32 8, i32 31, i32 3, i32 27>
-  %cmp = icmp eq <32 x i16> %mask, zeroinitializer
-  %res = select <32 x i1> %cmp, <32 x i16> %shuf, <32 x i16> zeroinitializer
-  ret <32 x i16> %res
-}
-define <32 x i16> @test_32xi16_perm_mask3(<32 x i16> %vec) {
-; GENERIC-LABEL: test_32xi16_perm_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [12,2,8,14,25,27,4,16,20,11,27,8,0,1,21,17,30,30,29,1,23,22,20,22,28,20,11,17,6,18,0,4] sched: [7:0.50]
-; GENERIC-NEXT:    vpermw %zmm0, %zmm1, %zmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_32xi16_perm_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [12,2,8,14,25,27,4,16,20,11,27,8,0,1,21,17,30,30,29,1,23,22,20,22,28,20,11,17,6,18,0,4] sched: [8:0.50]
-; SKX-NEXT:    vpermw %zmm0, %zmm1, %zmm0 # sched: [6:2.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %res = shufflevector <32 x i16> %vec, <32 x i16> undef, <32 x i32> <i32 12, i32 2, i32 8, i32 14, i32 25, i32 27, i32 4, i32 16, i32 20, i32 11, i32 27, i32 8, i32 0, i32 1, i32 21, i32 17, i32 30, i32 30, i32 29, i32 1, i32 23, i32 22, i32 20, i32 22, i32 28, i32 20, i32 11, i32 17, i32 6, i32 18, i32 0, i32 4>
-  ret <32 x i16> %res
-}
-define <32 x i16> @test_masked_32xi16_perm_mask3(<32 x i16> %vec, <32 x i16> %vec2, <32 x i16> %mask) {
-; GENERIC-LABEL: test_masked_32xi16_perm_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [12,2,8,14,25,27,4,16,20,11,27,8,0,1,21,17,30,30,29,1,23,22,20,22,28,20,11,17,6,18,0,4] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmw %zmm2, %zmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermw %zmm0, %zmm3, %zmm1 {%k1} # sched: [1:1.00]
-; GENERIC-NEXT:    vmovdqa64 %zmm1, %zmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_32xi16_perm_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [12,2,8,14,25,27,4,16,20,11,27,8,0,1,21,17,30,30,29,1,23,22,20,22,28,20,11,17,6,18,0,4] sched: [8:0.50]
-; SKX-NEXT:    vptestnmw %zmm2, %zmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermw %zmm0, %zmm3, %zmm1 {%k1} # sched: [6:2.00]
-; SKX-NEXT:    vmovdqa64 %zmm1, %zmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <32 x i16> %vec, <32 x i16> undef, <32 x i32> <i32 12, i32 2, i32 8, i32 14, i32 25, i32 27, i32 4, i32 16, i32 20, i32 11, i32 27, i32 8, i32 0, i32 1, i32 21, i32 17, i32 30, i32 30, i32 29, i32 1, i32 23, i32 22, i32 20, i32 22, i32 28, i32 20, i32 11, i32 17, i32 6, i32 18, i32 0, i32 4>
-  %cmp = icmp eq <32 x i16> %mask, zeroinitializer
-  %res = select <32 x i1> %cmp, <32 x i16> %shuf, <32 x i16> %vec2
-  ret <32 x i16> %res
-}
-
-define <32 x i16> @test_masked_z_32xi16_perm_mask3(<32 x i16> %vec, <32 x i16> %mask) {
-; GENERIC-LABEL: test_masked_z_32xi16_perm_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [12,2,8,14,25,27,4,16,20,11,27,8,0,1,21,17,30,30,29,1,23,22,20,22,28,20,11,17,6,18,0,4] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmw %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermw %zmm0, %zmm2, %zmm0 {%k1} {z} # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_32xi16_perm_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [12,2,8,14,25,27,4,16,20,11,27,8,0,1,21,17,30,30,29,1,23,22,20,22,28,20,11,17,6,18,0,4] sched: [8:0.50]
-; SKX-NEXT:    vptestnmw %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermw %zmm0, %zmm2, %zmm0 {%k1} {z} # sched: [6:2.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <32 x i16> %vec, <32 x i16> undef, <32 x i32> <i32 12, i32 2, i32 8, i32 14, i32 25, i32 27, i32 4, i32 16, i32 20, i32 11, i32 27, i32 8, i32 0, i32 1, i32 21, i32 17, i32 30, i32 30, i32 29, i32 1, i32 23, i32 22, i32 20, i32 22, i32 28, i32 20, i32 11, i32 17, i32 6, i32 18, i32 0, i32 4>
-  %cmp = icmp eq <32 x i16> %mask, zeroinitializer
-  %res = select <32 x i1> %cmp, <32 x i16> %shuf, <32 x i16> zeroinitializer
-  ret <32 x i16> %res
-}
-define <32 x i16> @test_32xi16_perm_mem_mask0(<32 x i16>* %vp) {
-; GENERIC-LABEL: test_32xi16_perm_mem_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [19,1,5,31,9,12,17,9,15,7,1,5,16,2,12,10,13,3,29,15,26,31,10,15,22,13,9,23,28,29,20,12] sched: [7:0.50]
-; GENERIC-NEXT:    vpermw (%rdi), %zmm0, %zmm0 # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_32xi16_perm_mem_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [19,1,5,31,9,12,17,9,15,7,1,5,16,2,12,10,13,3,29,15,26,31,10,15,22,13,9,23,28,29,20,12] sched: [8:0.50]
-; SKX-NEXT:    vpermw (%rdi), %zmm0, %zmm0 # sched: [13:2.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <32 x i16>, <32 x i16>* %vp
-  %res = shufflevector <32 x i16> %vec, <32 x i16> undef, <32 x i32> <i32 19, i32 1, i32 5, i32 31, i32 9, i32 12, i32 17, i32 9, i32 15, i32 7, i32 1, i32 5, i32 16, i32 2, i32 12, i32 10, i32 13, i32 3, i32 29, i32 15, i32 26, i32 31, i32 10, i32 15, i32 22, i32 13, i32 9, i32 23, i32 28, i32 29, i32 20, i32 12>
-  ret <32 x i16> %res
-}
-define <32 x i16> @test_masked_32xi16_perm_mem_mask0(<32 x i16>* %vp, <32 x i16> %vec2, <32 x i16> %mask) {
-; GENERIC-LABEL: test_masked_32xi16_perm_mem_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [19,1,5,31,9,12,17,9,15,7,1,5,16,2,12,10,13,3,29,15,26,31,10,15,22,13,9,23,28,29,20,12] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmw %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermw (%rdi), %zmm2, %zmm0 {%k1} # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_32xi16_perm_mem_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [19,1,5,31,9,12,17,9,15,7,1,5,16,2,12,10,13,3,29,15,26,31,10,15,22,13,9,23,28,29,20,12] sched: [8:0.50]
-; SKX-NEXT:    vptestnmw %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermw (%rdi), %zmm2, %zmm0 {%k1} # sched: [13:2.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <32 x i16>, <32 x i16>* %vp
-  %shuf = shufflevector <32 x i16> %vec, <32 x i16> undef, <32 x i32> <i32 19, i32 1, i32 5, i32 31, i32 9, i32 12, i32 17, i32 9, i32 15, i32 7, i32 1, i32 5, i32 16, i32 2, i32 12, i32 10, i32 13, i32 3, i32 29, i32 15, i32 26, i32 31, i32 10, i32 15, i32 22, i32 13, i32 9, i32 23, i32 28, i32 29, i32 20, i32 12>
-  %cmp = icmp eq <32 x i16> %mask, zeroinitializer
-  %res = select <32 x i1> %cmp, <32 x i16> %shuf, <32 x i16> %vec2
-  ret <32 x i16> %res
-}
-
-define <32 x i16> @test_masked_z_32xi16_perm_mem_mask0(<32 x i16>* %vp, <32 x i16> %mask) {
-; GENERIC-LABEL: test_masked_z_32xi16_perm_mem_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [19,1,5,31,9,12,17,9,15,7,1,5,16,2,12,10,13,3,29,15,26,31,10,15,22,13,9,23,28,29,20,12] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmw %zmm0, %zmm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermw (%rdi), %zmm1, %zmm0 {%k1} {z} # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_32xi16_perm_mem_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [19,1,5,31,9,12,17,9,15,7,1,5,16,2,12,10,13,3,29,15,26,31,10,15,22,13,9,23,28,29,20,12] sched: [8:0.50]
-; SKX-NEXT:    vptestnmw %zmm0, %zmm0, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermw (%rdi), %zmm1, %zmm0 {%k1} {z} # sched: [13:2.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <32 x i16>, <32 x i16>* %vp
-  %shuf = shufflevector <32 x i16> %vec, <32 x i16> undef, <32 x i32> <i32 19, i32 1, i32 5, i32 31, i32 9, i32 12, i32 17, i32 9, i32 15, i32 7, i32 1, i32 5, i32 16, i32 2, i32 12, i32 10, i32 13, i32 3, i32 29, i32 15, i32 26, i32 31, i32 10, i32 15, i32 22, i32 13, i32 9, i32 23, i32 28, i32 29, i32 20, i32 12>
-  %cmp = icmp eq <32 x i16> %mask, zeroinitializer
-  %res = select <32 x i1> %cmp, <32 x i16> %shuf, <32 x i16> zeroinitializer
-  ret <32 x i16> %res
-}
-
-define <32 x i16> @test_masked_32xi16_perm_mem_mask1(<32 x i16>* %vp, <32 x i16> %vec2, <32 x i16> %mask) {
-; GENERIC-LABEL: test_masked_32xi16_perm_mem_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [31,20,2,2,23,1,0,12,16,14,15,18,21,13,11,31,8,24,13,11,2,27,22,28,14,21,3,12,6,1,30,6] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmw %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermw (%rdi), %zmm2, %zmm0 {%k1} # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_32xi16_perm_mem_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [31,20,2,2,23,1,0,12,16,14,15,18,21,13,11,31,8,24,13,11,2,27,22,28,14,21,3,12,6,1,30,6] sched: [8:0.50]
-; SKX-NEXT:    vptestnmw %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermw (%rdi), %zmm2, %zmm0 {%k1} # sched: [13:2.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <32 x i16>, <32 x i16>* %vp
-  %shuf = shufflevector <32 x i16> %vec, <32 x i16> undef, <32 x i32> <i32 31, i32 20, i32 2, i32 2, i32 23, i32 1, i32 0, i32 12, i32 16, i32 14, i32 15, i32 18, i32 21, i32 13, i32 11, i32 31, i32 8, i32 24, i32 13, i32 11, i32 2, i32 27, i32 22, i32 28, i32 14, i32 21, i32 3, i32 12, i32 6, i32 1, i32 30, i32 6>
-  %cmp = icmp eq <32 x i16> %mask, zeroinitializer
-  %res = select <32 x i1> %cmp, <32 x i16> %shuf, <32 x i16> %vec2
-  ret <32 x i16> %res
-}
-
-define <32 x i16> @test_masked_z_32xi16_perm_mem_mask1(<32 x i16>* %vp, <32 x i16> %mask) {
-; GENERIC-LABEL: test_masked_z_32xi16_perm_mem_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [31,20,2,2,23,1,0,12,16,14,15,18,21,13,11,31,8,24,13,11,2,27,22,28,14,21,3,12,6,1,30,6] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmw %zmm0, %zmm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermw (%rdi), %zmm1, %zmm0 {%k1} {z} # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_32xi16_perm_mem_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [31,20,2,2,23,1,0,12,16,14,15,18,21,13,11,31,8,24,13,11,2,27,22,28,14,21,3,12,6,1,30,6] sched: [8:0.50]
-; SKX-NEXT:    vptestnmw %zmm0, %zmm0, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermw (%rdi), %zmm1, %zmm0 {%k1} {z} # sched: [13:2.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <32 x i16>, <32 x i16>* %vp
-  %shuf = shufflevector <32 x i16> %vec, <32 x i16> undef, <32 x i32> <i32 31, i32 20, i32 2, i32 2, i32 23, i32 1, i32 0, i32 12, i32 16, i32 14, i32 15, i32 18, i32 21, i32 13, i32 11, i32 31, i32 8, i32 24, i32 13, i32 11, i32 2, i32 27, i32 22, i32 28, i32 14, i32 21, i32 3, i32 12, i32 6, i32 1, i32 30, i32 6>
-  %cmp = icmp eq <32 x i16> %mask, zeroinitializer
-  %res = select <32 x i1> %cmp, <32 x i16> %shuf, <32 x i16> zeroinitializer
-  ret <32 x i16> %res
-}
-
-define <32 x i16> @test_masked_32xi16_perm_mem_mask2(<32 x i16>* %vp, <32 x i16> %vec2, <32 x i16> %mask) {
-; GENERIC-LABEL: test_masked_32xi16_perm_mem_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [4,6,12,17,4,31,31,4,12,21,28,15,29,10,15,15,21,6,19,7,10,30,28,26,1,4,8,25,26,18,22,25] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmw %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermw (%rdi), %zmm2, %zmm0 {%k1} # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_32xi16_perm_mem_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [4,6,12,17,4,31,31,4,12,21,28,15,29,10,15,15,21,6,19,7,10,30,28,26,1,4,8,25,26,18,22,25] sched: [8:0.50]
-; SKX-NEXT:    vptestnmw %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermw (%rdi), %zmm2, %zmm0 {%k1} # sched: [13:2.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <32 x i16>, <32 x i16>* %vp
-  %shuf = shufflevector <32 x i16> %vec, <32 x i16> undef, <32 x i32> <i32 4, i32 6, i32 12, i32 17, i32 4, i32 31, i32 31, i32 4, i32 12, i32 21, i32 28, i32 15, i32 29, i32 10, i32 15, i32 15, i32 21, i32 6, i32 19, i32 7, i32 10, i32 30, i32 28, i32 26, i32 1, i32 4, i32 8, i32 25, i32 26, i32 18, i32 22, i32 25>
-  %cmp = icmp eq <32 x i16> %mask, zeroinitializer
-  %res = select <32 x i1> %cmp, <32 x i16> %shuf, <32 x i16> %vec2
-  ret <32 x i16> %res
-}
-
-define <32 x i16> @test_masked_z_32xi16_perm_mem_mask2(<32 x i16>* %vp, <32 x i16> %mask) {
-; GENERIC-LABEL: test_masked_z_32xi16_perm_mem_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [4,6,12,17,4,31,31,4,12,21,28,15,29,10,15,15,21,6,19,7,10,30,28,26,1,4,8,25,26,18,22,25] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmw %zmm0, %zmm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermw (%rdi), %zmm1, %zmm0 {%k1} {z} # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_32xi16_perm_mem_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [4,6,12,17,4,31,31,4,12,21,28,15,29,10,15,15,21,6,19,7,10,30,28,26,1,4,8,25,26,18,22,25] sched: [8:0.50]
-; SKX-NEXT:    vptestnmw %zmm0, %zmm0, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermw (%rdi), %zmm1, %zmm0 {%k1} {z} # sched: [13:2.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <32 x i16>, <32 x i16>* %vp
-  %shuf = shufflevector <32 x i16> %vec, <32 x i16> undef, <32 x i32> <i32 4, i32 6, i32 12, i32 17, i32 4, i32 31, i32 31, i32 4, i32 12, i32 21, i32 28, i32 15, i32 29, i32 10, i32 15, i32 15, i32 21, i32 6, i32 19, i32 7, i32 10, i32 30, i32 28, i32 26, i32 1, i32 4, i32 8, i32 25, i32 26, i32 18, i32 22, i32 25>
-  %cmp = icmp eq <32 x i16> %mask, zeroinitializer
-  %res = select <32 x i1> %cmp, <32 x i16> %shuf, <32 x i16> zeroinitializer
-  ret <32 x i16> %res
-}
-
-define <32 x i16> @test_32xi16_perm_mem_mask3(<32 x i16>* %vp) {
-; GENERIC-LABEL: test_32xi16_perm_mem_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [2,2,27,1,7,1,0,27,10,5,4,20,30,16,28,16,18,21,25,24,31,23,28,6,17,19,26,15,25,12,18,27] sched: [7:0.50]
-; GENERIC-NEXT:    vpermw (%rdi), %zmm0, %zmm0 # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_32xi16_perm_mem_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [2,2,27,1,7,1,0,27,10,5,4,20,30,16,28,16,18,21,25,24,31,23,28,6,17,19,26,15,25,12,18,27] sched: [8:0.50]
-; SKX-NEXT:    vpermw (%rdi), %zmm0, %zmm0 # sched: [13:2.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <32 x i16>, <32 x i16>* %vp
-  %res = shufflevector <32 x i16> %vec, <32 x i16> undef, <32 x i32> <i32 2, i32 2, i32 27, i32 1, i32 7, i32 1, i32 0, i32 27, i32 10, i32 5, i32 4, i32 20, i32 30, i32 16, i32 28, i32 16, i32 18, i32 21, i32 25, i32 24, i32 31, i32 23, i32 28, i32 6, i32 17, i32 19, i32 26, i32 15, i32 25, i32 12, i32 18, i32 27>
-  ret <32 x i16> %res
-}
-define <32 x i16> @test_masked_32xi16_perm_mem_mask3(<32 x i16>* %vp, <32 x i16> %vec2, <32 x i16> %mask) {
-; GENERIC-LABEL: test_masked_32xi16_perm_mem_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [2,2,27,1,7,1,0,27,10,5,4,20,30,16,28,16,18,21,25,24,31,23,28,6,17,19,26,15,25,12,18,27] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmw %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermw (%rdi), %zmm2, %zmm0 {%k1} # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_32xi16_perm_mem_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [2,2,27,1,7,1,0,27,10,5,4,20,30,16,28,16,18,21,25,24,31,23,28,6,17,19,26,15,25,12,18,27] sched: [8:0.50]
-; SKX-NEXT:    vptestnmw %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermw (%rdi), %zmm2, %zmm0 {%k1} # sched: [13:2.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <32 x i16>, <32 x i16>* %vp
-  %shuf = shufflevector <32 x i16> %vec, <32 x i16> undef, <32 x i32> <i32 2, i32 2, i32 27, i32 1, i32 7, i32 1, i32 0, i32 27, i32 10, i32 5, i32 4, i32 20, i32 30, i32 16, i32 28, i32 16, i32 18, i32 21, i32 25, i32 24, i32 31, i32 23, i32 28, i32 6, i32 17, i32 19, i32 26, i32 15, i32 25, i32 12, i32 18, i32 27>
-  %cmp = icmp eq <32 x i16> %mask, zeroinitializer
-  %res = select <32 x i1> %cmp, <32 x i16> %shuf, <32 x i16> %vec2
-  ret <32 x i16> %res
-}
-
-define <32 x i16> @test_masked_z_32xi16_perm_mem_mask3(<32 x i16>* %vp, <32 x i16> %mask) {
-; GENERIC-LABEL: test_masked_z_32xi16_perm_mem_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [2,2,27,1,7,1,0,27,10,5,4,20,30,16,28,16,18,21,25,24,31,23,28,6,17,19,26,15,25,12,18,27] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmw %zmm0, %zmm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermw (%rdi), %zmm1, %zmm0 {%k1} {z} # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_32xi16_perm_mem_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [2,2,27,1,7,1,0,27,10,5,4,20,30,16,28,16,18,21,25,24,31,23,28,6,17,19,26,15,25,12,18,27] sched: [8:0.50]
-; SKX-NEXT:    vptestnmw %zmm0, %zmm0, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermw (%rdi), %zmm1, %zmm0 {%k1} {z} # sched: [13:2.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <32 x i16>, <32 x i16>* %vp
-  %shuf = shufflevector <32 x i16> %vec, <32 x i16> undef, <32 x i32> <i32 2, i32 2, i32 27, i32 1, i32 7, i32 1, i32 0, i32 27, i32 10, i32 5, i32 4, i32 20, i32 30, i32 16, i32 28, i32 16, i32 18, i32 21, i32 25, i32 24, i32 31, i32 23, i32 28, i32 6, i32 17, i32 19, i32 26, i32 15, i32 25, i32 12, i32 18, i32 27>
-  %cmp = icmp eq <32 x i16> %mask, zeroinitializer
-  %res = select <32 x i1> %cmp, <32 x i16> %shuf, <32 x i16> zeroinitializer
-  ret <32 x i16> %res
-}
-
-define <8 x i32> @test_8xi32_perm_mask0(<8 x i32> %vec) {
-; GENERIC-LABEL: test_8xi32_perm_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovaps {{.*#+}} ymm1 = [4,2,0,6,7,2,3,6] sched: [7:0.50]
-; GENERIC-NEXT:    vpermps %ymm0, %ymm1, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xi32_perm_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovaps {{.*#+}} ymm1 = [4,2,0,6,7,2,3,6] sched: [7:0.50]
-; SKX-NEXT:    vpermps %ymm0, %ymm1, %ymm0 # sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %res = shufflevector <8 x i32> %vec, <8 x i32> undef, <8 x i32> <i32 4, i32 2, i32 0, i32 6, i32 7, i32 2, i32 3, i32 6>
-  ret <8 x i32> %res
-}
-define <8 x i32> @test_masked_8xi32_perm_mask0(<8 x i32> %vec, <8 x i32> %vec2, <8 x i32> %mask) {
-; GENERIC-LABEL: test_masked_8xi32_perm_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovdqa {{.*#+}} ymm3 = [4,2,0,6,7,2,3,6] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmd %ymm2, %ymm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermd %ymm0, %ymm3, %ymm1 {%k1} # sched: [1:1.00]
-; GENERIC-NEXT:    vmovdqa %ymm1, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_8xi32_perm_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovdqa {{.*#+}} ymm3 = [4,2,0,6,7,2,3,6] sched: [7:0.50]
-; SKX-NEXT:    vptestnmd %ymm2, %ymm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermd %ymm0, %ymm3, %ymm1 {%k1} # sched: [3:1.00]
-; SKX-NEXT:    vmovdqa %ymm1, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x i32> %vec, <8 x i32> undef, <8 x i32> <i32 4, i32 2, i32 0, i32 6, i32 7, i32 2, i32 3, i32 6>
-  %cmp = icmp eq <8 x i32> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i32> %shuf, <8 x i32> %vec2
-  ret <8 x i32> %res
-}
-
-define <8 x i32> @test_masked_z_8xi32_perm_mask0(<8 x i32> %vec, <8 x i32> %mask) {
-; GENERIC-LABEL: test_masked_z_8xi32_perm_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovdqa {{.*#+}} ymm2 = [4,2,0,6,7,2,3,6] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmd %ymm1, %ymm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermd %ymm0, %ymm2, %ymm0 {%k1} {z} # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_8xi32_perm_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovdqa {{.*#+}} ymm2 = [4,2,0,6,7,2,3,6] sched: [7:0.50]
-; SKX-NEXT:    vptestnmd %ymm1, %ymm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermd %ymm0, %ymm2, %ymm0 {%k1} {z} # sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x i32> %vec, <8 x i32> undef, <8 x i32> <i32 4, i32 2, i32 0, i32 6, i32 7, i32 2, i32 3, i32 6>
-  %cmp = icmp eq <8 x i32> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i32> %shuf, <8 x i32> zeroinitializer
-  ret <8 x i32> %res
-}
-define <8 x i32> @test_masked_8xi32_perm_mask1(<8 x i32> %vec, <8 x i32> %vec2, <8 x i32> %mask) {
-; GENERIC-LABEL: test_masked_8xi32_perm_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovdqa {{.*#+}} ymm3 = [0,5,1,2,6,0,0,3] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmd %ymm2, %ymm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermd %ymm0, %ymm3, %ymm1 {%k1} # sched: [1:1.00]
-; GENERIC-NEXT:    vmovdqa %ymm1, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_8xi32_perm_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovdqa {{.*#+}} ymm3 = [0,5,1,2,6,0,0,3] sched: [7:0.50]
-; SKX-NEXT:    vptestnmd %ymm2, %ymm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermd %ymm0, %ymm3, %ymm1 {%k1} # sched: [3:1.00]
-; SKX-NEXT:    vmovdqa %ymm1, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x i32> %vec, <8 x i32> undef, <8 x i32> <i32 0, i32 5, i32 1, i32 2, i32 6, i32 0, i32 0, i32 3>
-  %cmp = icmp eq <8 x i32> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i32> %shuf, <8 x i32> %vec2
-  ret <8 x i32> %res
-}
-
-define <8 x i32> @test_masked_z_8xi32_perm_mask1(<8 x i32> %vec, <8 x i32> %mask) {
-; GENERIC-LABEL: test_masked_z_8xi32_perm_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovdqa {{.*#+}} ymm2 = [0,5,1,2,6,0,0,3] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmd %ymm1, %ymm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermd %ymm0, %ymm2, %ymm0 {%k1} {z} # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_8xi32_perm_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovdqa {{.*#+}} ymm2 = [0,5,1,2,6,0,0,3] sched: [7:0.50]
-; SKX-NEXT:    vptestnmd %ymm1, %ymm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermd %ymm0, %ymm2, %ymm0 {%k1} {z} # sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x i32> %vec, <8 x i32> undef, <8 x i32> <i32 0, i32 5, i32 1, i32 2, i32 6, i32 0, i32 0, i32 3>
-  %cmp = icmp eq <8 x i32> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i32> %shuf, <8 x i32> zeroinitializer
-  ret <8 x i32> %res
-}
-define <8 x i32> @test_masked_8xi32_perm_mask2(<8 x i32> %vec, <8 x i32> %vec2, <8 x i32> %mask) {
-; GENERIC-LABEL: test_masked_8xi32_perm_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovdqa {{.*#+}} ymm3 = [3,6,5,5,1,7,3,4] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmd %ymm2, %ymm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermd %ymm0, %ymm3, %ymm1 {%k1} # sched: [1:1.00]
-; GENERIC-NEXT:    vmovdqa %ymm1, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_8xi32_perm_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovdqa {{.*#+}} ymm3 = [3,6,5,5,1,7,3,4] sched: [7:0.50]
-; SKX-NEXT:    vptestnmd %ymm2, %ymm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermd %ymm0, %ymm3, %ymm1 {%k1} # sched: [3:1.00]
-; SKX-NEXT:    vmovdqa %ymm1, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x i32> %vec, <8 x i32> undef, <8 x i32> <i32 3, i32 6, i32 5, i32 5, i32 1, i32 7, i32 3, i32 4>
-  %cmp = icmp eq <8 x i32> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i32> %shuf, <8 x i32> %vec2
-  ret <8 x i32> %res
-}
-
-define <8 x i32> @test_masked_z_8xi32_perm_mask2(<8 x i32> %vec, <8 x i32> %mask) {
-; GENERIC-LABEL: test_masked_z_8xi32_perm_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovdqa {{.*#+}} ymm2 = [3,6,5,5,1,7,3,4] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmd %ymm1, %ymm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermd %ymm0, %ymm2, %ymm0 {%k1} {z} # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_8xi32_perm_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovdqa {{.*#+}} ymm2 = [3,6,5,5,1,7,3,4] sched: [7:0.50]
-; SKX-NEXT:    vptestnmd %ymm1, %ymm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermd %ymm0, %ymm2, %ymm0 {%k1} {z} # sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x i32> %vec, <8 x i32> undef, <8 x i32> <i32 3, i32 6, i32 5, i32 5, i32 1, i32 7, i32 3, i32 4>
-  %cmp = icmp eq <8 x i32> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i32> %shuf, <8 x i32> zeroinitializer
-  ret <8 x i32> %res
-}
-define <8 x i32> @test_8xi32_perm_mask3(<8 x i32> %vec) {
-; GENERIC-LABEL: test_8xi32_perm_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovaps {{.*#+}} ymm1 = [3,0,3,1,0,4,5,0] sched: [7:0.50]
-; GENERIC-NEXT:    vpermps %ymm0, %ymm1, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xi32_perm_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovaps {{.*#+}} ymm1 = [3,0,3,1,0,4,5,0] sched: [7:0.50]
-; SKX-NEXT:    vpermps %ymm0, %ymm1, %ymm0 # sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %res = shufflevector <8 x i32> %vec, <8 x i32> undef, <8 x i32> <i32 3, i32 0, i32 3, i32 1, i32 0, i32 4, i32 5, i32 0>
-  ret <8 x i32> %res
-}
-define <8 x i32> @test_masked_8xi32_perm_mask3(<8 x i32> %vec, <8 x i32> %vec2, <8 x i32> %mask) {
-; GENERIC-LABEL: test_masked_8xi32_perm_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovdqa {{.*#+}} ymm3 = [3,0,3,1,0,4,5,0] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmd %ymm2, %ymm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermd %ymm0, %ymm3, %ymm1 {%k1} # sched: [1:1.00]
-; GENERIC-NEXT:    vmovdqa %ymm1, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_8xi32_perm_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovdqa {{.*#+}} ymm3 = [3,0,3,1,0,4,5,0] sched: [7:0.50]
-; SKX-NEXT:    vptestnmd %ymm2, %ymm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermd %ymm0, %ymm3, %ymm1 {%k1} # sched: [3:1.00]
-; SKX-NEXT:    vmovdqa %ymm1, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x i32> %vec, <8 x i32> undef, <8 x i32> <i32 3, i32 0, i32 3, i32 1, i32 0, i32 4, i32 5, i32 0>
-  %cmp = icmp eq <8 x i32> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i32> %shuf, <8 x i32> %vec2
-  ret <8 x i32> %res
-}
-
-define <8 x i32> @test_masked_z_8xi32_perm_mask3(<8 x i32> %vec, <8 x i32> %mask) {
-; GENERIC-LABEL: test_masked_z_8xi32_perm_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovdqa {{.*#+}} ymm2 = [3,0,3,1,0,4,5,0] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmd %ymm1, %ymm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermd %ymm0, %ymm2, %ymm0 {%k1} {z} # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_8xi32_perm_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovdqa {{.*#+}} ymm2 = [3,0,3,1,0,4,5,0] sched: [7:0.50]
-; SKX-NEXT:    vptestnmd %ymm1, %ymm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermd %ymm0, %ymm2, %ymm0 {%k1} {z} # sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x i32> %vec, <8 x i32> undef, <8 x i32> <i32 3, i32 0, i32 3, i32 1, i32 0, i32 4, i32 5, i32 0>
-  %cmp = icmp eq <8 x i32> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i32> %shuf, <8 x i32> zeroinitializer
-  ret <8 x i32> %res
-}
-define <8 x i32> @test_8xi32_perm_mem_mask0(<8 x i32>* %vp) {
-; GENERIC-LABEL: test_8xi32_perm_mem_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovaps {{.*#+}} ymm0 = [3,7,4,3,5,2,0,5] sched: [7:0.50]
-; GENERIC-NEXT:    vpermps (%rdi), %ymm0, %ymm0 # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xi32_perm_mem_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovaps {{.*#+}} ymm0 = [3,7,4,3,5,2,0,5] sched: [7:0.50]
-; SKX-NEXT:    vpermps (%rdi), %ymm0, %ymm0 # sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <8 x i32>, <8 x i32>* %vp
-  %res = shufflevector <8 x i32> %vec, <8 x i32> undef, <8 x i32> <i32 3, i32 7, i32 4, i32 3, i32 5, i32 2, i32 0, i32 5>
-  ret <8 x i32> %res
-}
-define <8 x i32> @test_masked_8xi32_perm_mem_mask0(<8 x i32>* %vp, <8 x i32> %vec2, <8 x i32> %mask) {
-; GENERIC-LABEL: test_masked_8xi32_perm_mem_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovdqa {{.*#+}} ymm2 = [3,7,4,3,5,2,0,5] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmd %ymm1, %ymm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermd (%rdi), %ymm2, %ymm0 {%k1} # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_8xi32_perm_mem_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovdqa {{.*#+}} ymm2 = [3,7,4,3,5,2,0,5] sched: [7:0.50]
-; SKX-NEXT:    vptestnmd %ymm1, %ymm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermd (%rdi), %ymm2, %ymm0 {%k1} # sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <8 x i32>, <8 x i32>* %vp
-  %shuf = shufflevector <8 x i32> %vec, <8 x i32> undef, <8 x i32> <i32 3, i32 7, i32 4, i32 3, i32 5, i32 2, i32 0, i32 5>
-  %cmp = icmp eq <8 x i32> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i32> %shuf, <8 x i32> %vec2
-  ret <8 x i32> %res
-}
-
-define <8 x i32> @test_masked_z_8xi32_perm_mem_mask0(<8 x i32>* %vp, <8 x i32> %mask) {
-; GENERIC-LABEL: test_masked_z_8xi32_perm_mem_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovdqa {{.*#+}} ymm1 = [3,7,4,3,5,2,0,5] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmd %ymm0, %ymm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermd (%rdi), %ymm1, %ymm0 {%k1} {z} # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_8xi32_perm_mem_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovdqa {{.*#+}} ymm1 = [3,7,4,3,5,2,0,5] sched: [7:0.50]
-; SKX-NEXT:    vptestnmd %ymm0, %ymm0, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermd (%rdi), %ymm1, %ymm0 {%k1} {z} # sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <8 x i32>, <8 x i32>* %vp
-  %shuf = shufflevector <8 x i32> %vec, <8 x i32> undef, <8 x i32> <i32 3, i32 7, i32 4, i32 3, i32 5, i32 2, i32 0, i32 5>
-  %cmp = icmp eq <8 x i32> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i32> %shuf, <8 x i32> zeroinitializer
-  ret <8 x i32> %res
-}
-
-define <8 x i32> @test_masked_8xi32_perm_mem_mask1(<8 x i32>* %vp, <8 x i32> %vec2, <8 x i32> %mask) {
-; GENERIC-LABEL: test_masked_8xi32_perm_mem_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovdqa {{.*#+}} ymm2 = [4,6,1,7,6,7,6,5] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmd %ymm1, %ymm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermd (%rdi), %ymm2, %ymm0 {%k1} # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_8xi32_perm_mem_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovdqa {{.*#+}} ymm2 = [4,6,1,7,6,7,6,5] sched: [7:0.50]
-; SKX-NEXT:    vptestnmd %ymm1, %ymm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermd (%rdi), %ymm2, %ymm0 {%k1} # sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <8 x i32>, <8 x i32>* %vp
-  %shuf = shufflevector <8 x i32> %vec, <8 x i32> undef, <8 x i32> <i32 4, i32 6, i32 1, i32 7, i32 6, i32 7, i32 6, i32 5>
-  %cmp = icmp eq <8 x i32> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i32> %shuf, <8 x i32> %vec2
-  ret <8 x i32> %res
-}
-
-define <8 x i32> @test_masked_z_8xi32_perm_mem_mask1(<8 x i32>* %vp, <8 x i32> %mask) {
-; GENERIC-LABEL: test_masked_z_8xi32_perm_mem_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovdqa {{.*#+}} ymm1 = [4,6,1,7,6,7,6,5] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmd %ymm0, %ymm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermd (%rdi), %ymm1, %ymm0 {%k1} {z} # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_8xi32_perm_mem_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovdqa {{.*#+}} ymm1 = [4,6,1,7,6,7,6,5] sched: [7:0.50]
-; SKX-NEXT:    vptestnmd %ymm0, %ymm0, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermd (%rdi), %ymm1, %ymm0 {%k1} {z} # sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <8 x i32>, <8 x i32>* %vp
-  %shuf = shufflevector <8 x i32> %vec, <8 x i32> undef, <8 x i32> <i32 4, i32 6, i32 1, i32 7, i32 6, i32 7, i32 6, i32 5>
-  %cmp = icmp eq <8 x i32> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i32> %shuf, <8 x i32> zeroinitializer
-  ret <8 x i32> %res
-}
-
-define <8 x i32> @test_masked_8xi32_perm_mem_mask2(<8 x i32>* %vp, <8 x i32> %vec2, <8 x i32> %mask) {
-; GENERIC-LABEL: test_masked_8xi32_perm_mem_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovdqa {{.*#+}} ymm2 = [6,4,6,1,6,3,6,3] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmd %ymm1, %ymm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermd (%rdi), %ymm2, %ymm0 {%k1} # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_8xi32_perm_mem_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovdqa {{.*#+}} ymm2 = [6,4,6,1,6,3,6,3] sched: [7:0.50]
-; SKX-NEXT:    vptestnmd %ymm1, %ymm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermd (%rdi), %ymm2, %ymm0 {%k1} # sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <8 x i32>, <8 x i32>* %vp
-  %shuf = shufflevector <8 x i32> %vec, <8 x i32> undef, <8 x i32> <i32 6, i32 4, i32 6, i32 1, i32 6, i32 3, i32 6, i32 3>
-  %cmp = icmp eq <8 x i32> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i32> %shuf, <8 x i32> %vec2
-  ret <8 x i32> %res
-}
-
-define <8 x i32> @test_masked_z_8xi32_perm_mem_mask2(<8 x i32>* %vp, <8 x i32> %mask) {
-; GENERIC-LABEL: test_masked_z_8xi32_perm_mem_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovdqa {{.*#+}} ymm1 = [6,4,6,1,6,3,6,3] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmd %ymm0, %ymm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermd (%rdi), %ymm1, %ymm0 {%k1} {z} # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_8xi32_perm_mem_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovdqa {{.*#+}} ymm1 = [6,4,6,1,6,3,6,3] sched: [7:0.50]
-; SKX-NEXT:    vptestnmd %ymm0, %ymm0, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermd (%rdi), %ymm1, %ymm0 {%k1} {z} # sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <8 x i32>, <8 x i32>* %vp
-  %shuf = shufflevector <8 x i32> %vec, <8 x i32> undef, <8 x i32> <i32 6, i32 4, i32 6, i32 1, i32 6, i32 3, i32 6, i32 3>
-  %cmp = icmp eq <8 x i32> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i32> %shuf, <8 x i32> zeroinitializer
-  ret <8 x i32> %res
-}
-
-define <8 x i32> @test_8xi32_perm_mem_mask3(<8 x i32>* %vp) {
-; GENERIC-LABEL: test_8xi32_perm_mem_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovaps {{.*#+}} ymm0 = [6,0,0,7,3,7,7,5] sched: [7:0.50]
-; GENERIC-NEXT:    vpermps (%rdi), %ymm0, %ymm0 # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xi32_perm_mem_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovaps {{.*#+}} ymm0 = [6,0,0,7,3,7,7,5] sched: [7:0.50]
-; SKX-NEXT:    vpermps (%rdi), %ymm0, %ymm0 # sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <8 x i32>, <8 x i32>* %vp
-  %res = shufflevector <8 x i32> %vec, <8 x i32> undef, <8 x i32> <i32 6, i32 0, i32 0, i32 7, i32 3, i32 7, i32 7, i32 5>
-  ret <8 x i32> %res
-}
-define <8 x i32> @test_masked_8xi32_perm_mem_mask3(<8 x i32>* %vp, <8 x i32> %vec2, <8 x i32> %mask) {
-; GENERIC-LABEL: test_masked_8xi32_perm_mem_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovdqa {{.*#+}} ymm2 = [6,0,0,7,3,7,7,5] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmd %ymm1, %ymm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermd (%rdi), %ymm2, %ymm0 {%k1} # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_8xi32_perm_mem_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovdqa {{.*#+}} ymm2 = [6,0,0,7,3,7,7,5] sched: [7:0.50]
-; SKX-NEXT:    vptestnmd %ymm1, %ymm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermd (%rdi), %ymm2, %ymm0 {%k1} # sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <8 x i32>, <8 x i32>* %vp
-  %shuf = shufflevector <8 x i32> %vec, <8 x i32> undef, <8 x i32> <i32 6, i32 0, i32 0, i32 7, i32 3, i32 7, i32 7, i32 5>
-  %cmp = icmp eq <8 x i32> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i32> %shuf, <8 x i32> %vec2
-  ret <8 x i32> %res
-}
-
-define <8 x i32> @test_masked_z_8xi32_perm_mem_mask3(<8 x i32>* %vp, <8 x i32> %mask) {
-; GENERIC-LABEL: test_masked_z_8xi32_perm_mem_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovdqa {{.*#+}} ymm1 = [6,0,0,7,3,7,7,5] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmd %ymm0, %ymm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermd (%rdi), %ymm1, %ymm0 {%k1} {z} # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_8xi32_perm_mem_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovdqa {{.*#+}} ymm1 = [6,0,0,7,3,7,7,5] sched: [7:0.50]
-; SKX-NEXT:    vptestnmd %ymm0, %ymm0, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermd (%rdi), %ymm1, %ymm0 {%k1} {z} # sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <8 x i32>, <8 x i32>* %vp
-  %shuf = shufflevector <8 x i32> %vec, <8 x i32> undef, <8 x i32> <i32 6, i32 0, i32 0, i32 7, i32 3, i32 7, i32 7, i32 5>
-  %cmp = icmp eq <8 x i32> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i32> %shuf, <8 x i32> zeroinitializer
-  ret <8 x i32> %res
-}
-
-define <16 x i32> @test_16xi32_perm_mask0(<16 x i32> %vec, <16 x i32> %mask) {
-; GENERIC-LABEL: test_16xi32_perm_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovaps {{.*#+}} zmm1 = [14,12,11,6,4,1,6,9,14,14,6,1,12,11,0,7] sched: [7:0.50]
-; GENERIC-NEXT:    vpermps %zmm0, %zmm1, %zmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_16xi32_perm_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovaps {{.*#+}} zmm1 = [14,12,11,6,4,1,6,9,14,14,6,1,12,11,0,7] sched: [8:0.50]
-; SKX-NEXT:    vpermps %zmm0, %zmm1, %zmm0 # sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %res = shufflevector <16 x i32> %vec, <16 x i32> undef, <16 x i32> <i32 14, i32 12, i32 11, i32 6, i32 4, i32 1, i32 6, i32 9, i32 14, i32 14, i32 6, i32 1, i32 12, i32 11, i32 0, i32 7>
-  ret <16 x i32> %res
-}
-define <16 x i32> @test_masked_16xi32_perm_mask0(<16 x i32> %vec, <16 x i32> %vec2, <16 x i32> %mask) {
-; GENERIC-LABEL: test_masked_16xi32_perm_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [14,12,11,6,4,1,6,9,14,14,6,1,12,11,0,7] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmd %zmm2, %zmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermd %zmm0, %zmm3, %zmm1 {%k1} # sched: [1:1.00]
-; GENERIC-NEXT:    vmovdqa64 %zmm1, %zmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_16xi32_perm_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [14,12,11,6,4,1,6,9,14,14,6,1,12,11,0,7] sched: [8:0.50]
-; SKX-NEXT:    vptestnmd %zmm2, %zmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermd %zmm0, %zmm3, %zmm1 {%k1} # sched: [3:1.00]
-; SKX-NEXT:    vmovdqa64 %zmm1, %zmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <16 x i32> %vec, <16 x i32> undef, <16 x i32> <i32 14, i32 12, i32 11, i32 6, i32 4, i32 1, i32 6, i32 9, i32 14, i32 14, i32 6, i32 1, i32 12, i32 11, i32 0, i32 7>
-  %cmp = icmp eq <16 x i32> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x i32> %shuf, <16 x i32> %vec2
-  ret <16 x i32> %res
-}
-
-define <16 x i32> @test_masked_z_16xi32_perm_mask0(<16 x i32> %vec, <16 x i32> %mask) {
-; GENERIC-LABEL: test_masked_z_16xi32_perm_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [14,12,11,6,4,1,6,9,14,14,6,1,12,11,0,7] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmd %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermd %zmm0, %zmm2, %zmm0 {%k1} {z} # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_16xi32_perm_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [14,12,11,6,4,1,6,9,14,14,6,1,12,11,0,7] sched: [8:0.50]
-; SKX-NEXT:    vptestnmd %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermd %zmm0, %zmm2, %zmm0 {%k1} {z} # sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <16 x i32> %vec, <16 x i32> undef, <16 x i32> <i32 14, i32 12, i32 11, i32 6, i32 4, i32 1, i32 6, i32 9, i32 14, i32 14, i32 6, i32 1, i32 12, i32 11, i32 0, i32 7>
-  %cmp = icmp eq <16 x i32> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x i32> %shuf, <16 x i32> zeroinitializer
-  ret <16 x i32> %res
-}
-define <16 x i32> @test_masked_16xi32_perm_mask1(<16 x i32> %vec, <16 x i32> %vec2, <16 x i32> %mask) {
-; GENERIC-LABEL: test_masked_16xi32_perm_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [10,0,14,15,11,1,1,5,0,5,0,15,13,1,14,3] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmd %zmm2, %zmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermd %zmm0, %zmm3, %zmm1 {%k1} # sched: [1:1.00]
-; GENERIC-NEXT:    vmovdqa64 %zmm1, %zmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_16xi32_perm_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [10,0,14,15,11,1,1,5,0,5,0,15,13,1,14,3] sched: [8:0.50]
-; SKX-NEXT:    vptestnmd %zmm2, %zmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermd %zmm0, %zmm3, %zmm1 {%k1} # sched: [3:1.00]
-; SKX-NEXT:    vmovdqa64 %zmm1, %zmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <16 x i32> %vec, <16 x i32> undef, <16 x i32> <i32 10, i32 0, i32 14, i32 15, i32 11, i32 1, i32 1, i32 5, i32 0, i32 5, i32 0, i32 15, i32 13, i32 1, i32 14, i32 3>
-  %cmp = icmp eq <16 x i32> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x i32> %shuf, <16 x i32> %vec2
-  ret <16 x i32> %res
-}
-
-define <16 x i32> @test_masked_z_16xi32_perm_mask1(<16 x i32> %vec, <16 x i32> %mask) {
-; GENERIC-LABEL: test_masked_z_16xi32_perm_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [10,0,14,15,11,1,1,5,0,5,0,15,13,1,14,3] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmd %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermd %zmm0, %zmm2, %zmm0 {%k1} {z} # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_16xi32_perm_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [10,0,14,15,11,1,1,5,0,5,0,15,13,1,14,3] sched: [8:0.50]
-; SKX-NEXT:    vptestnmd %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermd %zmm0, %zmm2, %zmm0 {%k1} {z} # sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <16 x i32> %vec, <16 x i32> undef, <16 x i32> <i32 10, i32 0, i32 14, i32 15, i32 11, i32 1, i32 1, i32 5, i32 0, i32 5, i32 0, i32 15, i32 13, i32 1, i32 14, i32 3>
-  %cmp = icmp eq <16 x i32> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x i32> %shuf, <16 x i32> zeroinitializer
-  ret <16 x i32> %res
-}
-define <16 x i32> @test_masked_16xi32_perm_mask2(<16 x i32> %vec, <16 x i32> %vec2, <16 x i32> %mask) {
-; GENERIC-LABEL: test_masked_16xi32_perm_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [3,10,15,1,0,5,0,9,13,2,1,5,15,2,15,5] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmd %zmm2, %zmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermd %zmm0, %zmm3, %zmm1 {%k1} # sched: [1:1.00]
-; GENERIC-NEXT:    vmovdqa64 %zmm1, %zmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_16xi32_perm_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [3,10,15,1,0,5,0,9,13,2,1,5,15,2,15,5] sched: [8:0.50]
-; SKX-NEXT:    vptestnmd %zmm2, %zmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermd %zmm0, %zmm3, %zmm1 {%k1} # sched: [3:1.00]
-; SKX-NEXT:    vmovdqa64 %zmm1, %zmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <16 x i32> %vec, <16 x i32> undef, <16 x i32> <i32 3, i32 10, i32 15, i32 1, i32 0, i32 5, i32 0, i32 9, i32 13, i32 2, i32 1, i32 5, i32 15, i32 2, i32 15, i32 5>
-  %cmp = icmp eq <16 x i32> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x i32> %shuf, <16 x i32> %vec2
-  ret <16 x i32> %res
-}
-
-define <16 x i32> @test_masked_z_16xi32_perm_mask2(<16 x i32> %vec, <16 x i32> %mask) {
-; GENERIC-LABEL: test_masked_z_16xi32_perm_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [3,10,15,1,0,5,0,9,13,2,1,5,15,2,15,5] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmd %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermd %zmm0, %zmm2, %zmm0 {%k1} {z} # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_16xi32_perm_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [3,10,15,1,0,5,0,9,13,2,1,5,15,2,15,5] sched: [8:0.50]
-; SKX-NEXT:    vptestnmd %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermd %zmm0, %zmm2, %zmm0 {%k1} {z} # sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <16 x i32> %vec, <16 x i32> undef, <16 x i32> <i32 3, i32 10, i32 15, i32 1, i32 0, i32 5, i32 0, i32 9, i32 13, i32 2, i32 1, i32 5, i32 15, i32 2, i32 15, i32 5>
-  %cmp = icmp eq <16 x i32> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x i32> %shuf, <16 x i32> zeroinitializer
-  ret <16 x i32> %res
-}
-define <16 x i32> @test_16xi32_perm_mask3(<16 x i32> %vec) {
-; GENERIC-LABEL: test_16xi32_perm_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovaps {{.*#+}} zmm1 = [7,4,14,15,10,2,15,1,9,2,14,15,12,5,3,12] sched: [7:0.50]
-; GENERIC-NEXT:    vpermps %zmm0, %zmm1, %zmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_16xi32_perm_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovaps {{.*#+}} zmm1 = [7,4,14,15,10,2,15,1,9,2,14,15,12,5,3,12] sched: [8:0.50]
-; SKX-NEXT:    vpermps %zmm0, %zmm1, %zmm0 # sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %res = shufflevector <16 x i32> %vec, <16 x i32> undef, <16 x i32> <i32 7, i32 4, i32 14, i32 15, i32 10, i32 2, i32 15, i32 1, i32 9, i32 2, i32 14, i32 15, i32 12, i32 5, i32 3, i32 12>
-  ret <16 x i32> %res
-}
-define <16 x i32> @test_masked_16xi32_perm_mask3(<16 x i32> %vec, <16 x i32> %vec2, <16 x i32> %mask) {
-; GENERIC-LABEL: test_masked_16xi32_perm_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [7,4,14,15,10,2,15,1,9,2,14,15,12,5,3,12] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmd %zmm2, %zmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermd %zmm0, %zmm3, %zmm1 {%k1} # sched: [1:1.00]
-; GENERIC-NEXT:    vmovdqa64 %zmm1, %zmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_16xi32_perm_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [7,4,14,15,10,2,15,1,9,2,14,15,12,5,3,12] sched: [8:0.50]
-; SKX-NEXT:    vptestnmd %zmm2, %zmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermd %zmm0, %zmm3, %zmm1 {%k1} # sched: [3:1.00]
-; SKX-NEXT:    vmovdqa64 %zmm1, %zmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <16 x i32> %vec, <16 x i32> undef, <16 x i32> <i32 7, i32 4, i32 14, i32 15, i32 10, i32 2, i32 15, i32 1, i32 9, i32 2, i32 14, i32 15, i32 12, i32 5, i32 3, i32 12>
-  %cmp = icmp eq <16 x i32> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x i32> %shuf, <16 x i32> %vec2
-  ret <16 x i32> %res
-}
-
-define <16 x i32> @test_masked_z_16xi32_perm_mask3(<16 x i32> %vec, <16 x i32> %mask) {
-; GENERIC-LABEL: test_masked_z_16xi32_perm_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [7,4,14,15,10,2,15,1,9,2,14,15,12,5,3,12] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmd %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermd %zmm0, %zmm2, %zmm0 {%k1} {z} # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_16xi32_perm_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [7,4,14,15,10,2,15,1,9,2,14,15,12,5,3,12] sched: [8:0.50]
-; SKX-NEXT:    vptestnmd %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermd %zmm0, %zmm2, %zmm0 {%k1} {z} # sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <16 x i32> %vec, <16 x i32> undef, <16 x i32> <i32 7, i32 4, i32 14, i32 15, i32 10, i32 2, i32 15, i32 1, i32 9, i32 2, i32 14, i32 15, i32 12, i32 5, i32 3, i32 12>
-  %cmp = icmp eq <16 x i32> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x i32> %shuf, <16 x i32> zeroinitializer
-  ret <16 x i32> %res
-}
-define <16 x i32> @test_16xi32_perm_mem_mask0(<16 x i32>* %vp) {
-; GENERIC-LABEL: test_16xi32_perm_mem_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovaps {{.*#+}} zmm0 = [0,1,1,6,8,11,2,6,10,1,7,5,15,0,6,6] sched: [7:0.50]
-; GENERIC-NEXT:    vpermps (%rdi), %zmm0, %zmm0 # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_16xi32_perm_mem_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovaps {{.*#+}} zmm0 = [0,1,1,6,8,11,2,6,10,1,7,5,15,0,6,6] sched: [8:0.50]
-; SKX-NEXT:    vpermps (%rdi), %zmm0, %zmm0 # sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <16 x i32>, <16 x i32>* %vp
-  %res = shufflevector <16 x i32> %vec, <16 x i32> undef, <16 x i32> <i32 0, i32 1, i32 1, i32 6, i32 8, i32 11, i32 2, i32 6, i32 10, i32 1, i32 7, i32 5, i32 15, i32 0, i32 6, i32 6>
-  ret <16 x i32> %res
-}
-define <16 x i32> @test_masked_16xi32_perm_mem_mask0(<16 x i32>* %vp, <16 x i32> %vec2, <16 x i32> %mask) {
-; GENERIC-LABEL: test_masked_16xi32_perm_mem_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [0,1,1,6,8,11,2,6,10,1,7,5,15,0,6,6] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmd %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermd (%rdi), %zmm2, %zmm0 {%k1} # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_16xi32_perm_mem_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [0,1,1,6,8,11,2,6,10,1,7,5,15,0,6,6] sched: [8:0.50]
-; SKX-NEXT:    vptestnmd %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermd (%rdi), %zmm2, %zmm0 {%k1} # sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <16 x i32>, <16 x i32>* %vp
-  %shuf = shufflevector <16 x i32> %vec, <16 x i32> undef, <16 x i32> <i32 0, i32 1, i32 1, i32 6, i32 8, i32 11, i32 2, i32 6, i32 10, i32 1, i32 7, i32 5, i32 15, i32 0, i32 6, i32 6>
-  %cmp = icmp eq <16 x i32> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x i32> %shuf, <16 x i32> %vec2
-  ret <16 x i32> %res
-}
-
-define <16 x i32> @test_masked_z_16xi32_perm_mem_mask0(<16 x i32>* %vp, <16 x i32> %mask) {
-; GENERIC-LABEL: test_masked_z_16xi32_perm_mem_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [0,1,1,6,8,11,2,6,10,1,7,5,15,0,6,6] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmd %zmm0, %zmm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermd (%rdi), %zmm1, %zmm0 {%k1} {z} # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_16xi32_perm_mem_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [0,1,1,6,8,11,2,6,10,1,7,5,15,0,6,6] sched: [8:0.50]
-; SKX-NEXT:    vptestnmd %zmm0, %zmm0, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermd (%rdi), %zmm1, %zmm0 {%k1} {z} # sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <16 x i32>, <16 x i32>* %vp
-  %shuf = shufflevector <16 x i32> %vec, <16 x i32> undef, <16 x i32> <i32 0, i32 1, i32 1, i32 6, i32 8, i32 11, i32 2, i32 6, i32 10, i32 1, i32 7, i32 5, i32 15, i32 0, i32 6, i32 6>
-  %cmp = icmp eq <16 x i32> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x i32> %shuf, <16 x i32> zeroinitializer
-  ret <16 x i32> %res
-}
-
-define <16 x i32> @test_masked_16xi32_perm_mem_mask1(<16 x i32>* %vp, <16 x i32> %vec2, <16 x i32> %mask) {
-; GENERIC-LABEL: test_masked_16xi32_perm_mem_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [11,5,3,4,7,15,12,4,8,11,12,7,6,12,6,3] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmd %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermd (%rdi), %zmm2, %zmm0 {%k1} # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_16xi32_perm_mem_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [11,5,3,4,7,15,12,4,8,11,12,7,6,12,6,3] sched: [8:0.50]
-; SKX-NEXT:    vptestnmd %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermd (%rdi), %zmm2, %zmm0 {%k1} # sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <16 x i32>, <16 x i32>* %vp
-  %shuf = shufflevector <16 x i32> %vec, <16 x i32> undef, <16 x i32> <i32 11, i32 5, i32 3, i32 4, i32 7, i32 15, i32 12, i32 4, i32 8, i32 11, i32 12, i32 7, i32 6, i32 12, i32 6, i32 3>
-  %cmp = icmp eq <16 x i32> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x i32> %shuf, <16 x i32> %vec2
-  ret <16 x i32> %res
-}
-
-define <16 x i32> @test_masked_z_16xi32_perm_mem_mask1(<16 x i32>* %vp, <16 x i32> %mask) {
-; GENERIC-LABEL: test_masked_z_16xi32_perm_mem_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [11,5,3,4,7,15,12,4,8,11,12,7,6,12,6,3] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmd %zmm0, %zmm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermd (%rdi), %zmm1, %zmm0 {%k1} {z} # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_16xi32_perm_mem_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [11,5,3,4,7,15,12,4,8,11,12,7,6,12,6,3] sched: [8:0.50]
-; SKX-NEXT:    vptestnmd %zmm0, %zmm0, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermd (%rdi), %zmm1, %zmm0 {%k1} {z} # sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <16 x i32>, <16 x i32>* %vp
-  %shuf = shufflevector <16 x i32> %vec, <16 x i32> undef, <16 x i32> <i32 11, i32 5, i32 3, i32 4, i32 7, i32 15, i32 12, i32 4, i32 8, i32 11, i32 12, i32 7, i32 6, i32 12, i32 6, i32 3>
-  %cmp = icmp eq <16 x i32> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x i32> %shuf, <16 x i32> zeroinitializer
-  ret <16 x i32> %res
-}
-
-define <16 x i32> @test_masked_16xi32_perm_mem_mask2(<16 x i32>* %vp, <16 x i32> %vec2, <16 x i32> %mask) {
-; GENERIC-LABEL: test_masked_16xi32_perm_mem_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [7,14,2,7,10,7,3,0,11,9,0,4,12,10,8,2] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmd %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermd (%rdi), %zmm2, %zmm0 {%k1} # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_16xi32_perm_mem_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [7,14,2,7,10,7,3,0,11,9,0,4,12,10,8,2] sched: [8:0.50]
-; SKX-NEXT:    vptestnmd %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermd (%rdi), %zmm2, %zmm0 {%k1} # sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <16 x i32>, <16 x i32>* %vp
-  %shuf = shufflevector <16 x i32> %vec, <16 x i32> undef, <16 x i32> <i32 7, i32 14, i32 2, i32 7, i32 10, i32 7, i32 3, i32 0, i32 11, i32 9, i32 0, i32 4, i32 12, i32 10, i32 8, i32 2>
-  %cmp = icmp eq <16 x i32> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x i32> %shuf, <16 x i32> %vec2
-  ret <16 x i32> %res
-}
-
-define <16 x i32> @test_masked_z_16xi32_perm_mem_mask2(<16 x i32>* %vp, <16 x i32> %mask) {
-; GENERIC-LABEL: test_masked_z_16xi32_perm_mem_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [7,14,2,7,10,7,3,0,11,9,0,4,12,10,8,2] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmd %zmm0, %zmm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermd (%rdi), %zmm1, %zmm0 {%k1} {z} # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_16xi32_perm_mem_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [7,14,2,7,10,7,3,0,11,9,0,4,12,10,8,2] sched: [8:0.50]
-; SKX-NEXT:    vptestnmd %zmm0, %zmm0, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermd (%rdi), %zmm1, %zmm0 {%k1} {z} # sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <16 x i32>, <16 x i32>* %vp
-  %shuf = shufflevector <16 x i32> %vec, <16 x i32> undef, <16 x i32> <i32 7, i32 14, i32 2, i32 7, i32 10, i32 7, i32 3, i32 0, i32 11, i32 9, i32 0, i32 4, i32 12, i32 10, i32 8, i32 2>
-  %cmp = icmp eq <16 x i32> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x i32> %shuf, <16 x i32> zeroinitializer
-  ret <16 x i32> %res
-}
-
-define <16 x i32> @test_16xi32_perm_mem_mask3(<16 x i32>* %vp) {
-; GENERIC-LABEL: test_16xi32_perm_mem_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovaps {{.*#+}} zmm0 = [11,7,10,12,3,12,4,15,1,14,0,4,8,9,6,1] sched: [7:0.50]
-; GENERIC-NEXT:    vpermps (%rdi), %zmm0, %zmm0 # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_16xi32_perm_mem_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovaps {{.*#+}} zmm0 = [11,7,10,12,3,12,4,15,1,14,0,4,8,9,6,1] sched: [8:0.50]
-; SKX-NEXT:    vpermps (%rdi), %zmm0, %zmm0 # sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <16 x i32>, <16 x i32>* %vp
-  %res = shufflevector <16 x i32> %vec, <16 x i32> undef, <16 x i32> <i32 11, i32 7, i32 10, i32 12, i32 3, i32 12, i32 4, i32 15, i32 1, i32 14, i32 0, i32 4, i32 8, i32 9, i32 6, i32 1>
-  ret <16 x i32> %res
-}
-define <16 x i32> @test_masked_16xi32_perm_mem_mask3(<16 x i32>* %vp, <16 x i32> %vec2, <16 x i32> %mask) {
-; GENERIC-LABEL: test_masked_16xi32_perm_mem_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [11,7,10,12,3,12,4,15,1,14,0,4,8,9,6,1] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmd %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermd (%rdi), %zmm2, %zmm0 {%k1} # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_16xi32_perm_mem_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [11,7,10,12,3,12,4,15,1,14,0,4,8,9,6,1] sched: [8:0.50]
-; SKX-NEXT:    vptestnmd %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermd (%rdi), %zmm2, %zmm0 {%k1} # sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <16 x i32>, <16 x i32>* %vp
-  %shuf = shufflevector <16 x i32> %vec, <16 x i32> undef, <16 x i32> <i32 11, i32 7, i32 10, i32 12, i32 3, i32 12, i32 4, i32 15, i32 1, i32 14, i32 0, i32 4, i32 8, i32 9, i32 6, i32 1>
-  %cmp = icmp eq <16 x i32> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x i32> %shuf, <16 x i32> %vec2
-  ret <16 x i32> %res
-}
-
-define <16 x i32> @test_masked_z_16xi32_perm_mem_mask3(<16 x i32>* %vp, <16 x i32> %mask) {
-; GENERIC-LABEL: test_masked_z_16xi32_perm_mem_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [11,7,10,12,3,12,4,15,1,14,0,4,8,9,6,1] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmd %zmm0, %zmm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermd (%rdi), %zmm1, %zmm0 {%k1} {z} # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_16xi32_perm_mem_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [11,7,10,12,3,12,4,15,1,14,0,4,8,9,6,1] sched: [8:0.50]
-; SKX-NEXT:    vptestnmd %zmm0, %zmm0, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermd (%rdi), %zmm1, %zmm0 {%k1} {z} # sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <16 x i32>, <16 x i32>* %vp
-  %shuf = shufflevector <16 x i32> %vec, <16 x i32> undef, <16 x i32> <i32 11, i32 7, i32 10, i32 12, i32 3, i32 12, i32 4, i32 15, i32 1, i32 14, i32 0, i32 4, i32 8, i32 9, i32 6, i32 1>
-  %cmp = icmp eq <16 x i32> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x i32> %shuf, <16 x i32> zeroinitializer
-  ret <16 x i32> %res
-}
-
-define <4 x i64> @test_4xi64_perm_mask0(<4 x i64> %vec) {
-; GENERIC-LABEL: test_4xi64_perm_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[2,0,3,1] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xi64_perm_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[2,0,3,1] sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %res = shufflevector <4 x i64> %vec, <4 x i64> undef, <4 x i32> <i32 2, i32 0, i32 3, i32 1>
-  ret <4 x i64> %res
-}
-define <4 x i64> @test_masked_4xi64_perm_mask0(<4 x i64> %vec, <4 x i64> %vec2, <4 x i64> %mask) {
-; GENERIC-LABEL: test_masked_4xi64_perm_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %ymm2, %ymm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermq {{.*#+}} ymm1 {%k1} = ymm0[2,0,3,1] sched: [1:1.00]
-; GENERIC-NEXT:    vmovdqa %ymm1, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_4xi64_perm_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %ymm2, %ymm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermq {{.*#+}} ymm1 {%k1} = ymm0[2,0,3,1] sched: [3:1.00]
-; SKX-NEXT:    vmovdqa %ymm1, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <4 x i64> %vec, <4 x i64> undef, <4 x i32> <i32 2, i32 0, i32 3, i32 1>
-  %cmp = icmp eq <4 x i64> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x i64> %shuf, <4 x i64> %vec2
-  ret <4 x i64> %res
-}
-
-define <4 x i64> @test_masked_z_4xi64_perm_mask0(<4 x i64> %vec, <4 x i64> %mask) {
-; GENERIC-LABEL: test_masked_z_4xi64_perm_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %ymm1, %ymm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermq {{.*#+}} ymm0 {%k1} {z} = ymm0[2,0,3,1] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_4xi64_perm_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %ymm1, %ymm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermq {{.*#+}} ymm0 {%k1} {z} = ymm0[2,0,3,1] sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <4 x i64> %vec, <4 x i64> undef, <4 x i32> <i32 2, i32 0, i32 3, i32 1>
-  %cmp = icmp eq <4 x i64> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x i64> %shuf, <4 x i64> zeroinitializer
-  ret <4 x i64> %res
-}
-define <4 x i64> @test_masked_4xi64_perm_mask1(<4 x i64> %vec, <4 x i64> %vec2, <4 x i64> %mask) {
-; GENERIC-LABEL: test_masked_4xi64_perm_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %ymm2, %ymm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermq {{.*#+}} ymm1 {%k1} = ymm0[1,2,0,3] sched: [1:1.00]
-; GENERIC-NEXT:    vmovdqa %ymm1, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_4xi64_perm_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %ymm2, %ymm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermq {{.*#+}} ymm1 {%k1} = ymm0[1,2,0,3] sched: [3:1.00]
-; SKX-NEXT:    vmovdqa %ymm1, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <4 x i64> %vec, <4 x i64> undef, <4 x i32> <i32 1, i32 2, i32 0, i32 3>
-  %cmp = icmp eq <4 x i64> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x i64> %shuf, <4 x i64> %vec2
-  ret <4 x i64> %res
-}
-
-define <4 x i64> @test_masked_z_4xi64_perm_mask1(<4 x i64> %vec, <4 x i64> %mask) {
-; GENERIC-LABEL: test_masked_z_4xi64_perm_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %ymm1, %ymm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermq {{.*#+}} ymm0 {%k1} {z} = ymm0[1,2,0,3] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_4xi64_perm_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %ymm1, %ymm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermq {{.*#+}} ymm0 {%k1} {z} = ymm0[1,2,0,3] sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <4 x i64> %vec, <4 x i64> undef, <4 x i32> <i32 1, i32 2, i32 0, i32 3>
-  %cmp = icmp eq <4 x i64> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x i64> %shuf, <4 x i64> zeroinitializer
-  ret <4 x i64> %res
-}
-define <4 x i64> @test_masked_4xi64_perm_mask2(<4 x i64> %vec, <4 x i64> %vec2, <4 x i64> %mask) {
-; GENERIC-LABEL: test_masked_4xi64_perm_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %ymm2, %ymm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermq {{.*#+}} ymm1 {%k1} = ymm0[2,2,2,1] sched: [1:1.00]
-; GENERIC-NEXT:    vmovdqa %ymm1, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_4xi64_perm_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %ymm2, %ymm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermq {{.*#+}} ymm1 {%k1} = ymm0[2,2,2,1] sched: [3:1.00]
-; SKX-NEXT:    vmovdqa %ymm1, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <4 x i64> %vec, <4 x i64> undef, <4 x i32> <i32 2, i32 2, i32 2, i32 1>
-  %cmp = icmp eq <4 x i64> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x i64> %shuf, <4 x i64> %vec2
-  ret <4 x i64> %res
-}
-
-define <4 x i64> @test_masked_z_4xi64_perm_mask2(<4 x i64> %vec, <4 x i64> %mask) {
-; GENERIC-LABEL: test_masked_z_4xi64_perm_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %ymm1, %ymm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermq {{.*#+}} ymm0 {%k1} {z} = ymm0[2,2,2,1] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_4xi64_perm_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %ymm1, %ymm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermq {{.*#+}} ymm0 {%k1} {z} = ymm0[2,2,2,1] sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <4 x i64> %vec, <4 x i64> undef, <4 x i32> <i32 2, i32 2, i32 2, i32 1>
-  %cmp = icmp eq <4 x i64> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x i64> %shuf, <4 x i64> zeroinitializer
-  ret <4 x i64> %res
-}
-define <4 x i64> @test_4xi64_perm_mask3(<4 x i64> %vec) {
-; GENERIC-LABEL: test_4xi64_perm_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[2,1,3,3] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xi64_perm_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[2,1,3,3] sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %res = shufflevector <4 x i64> %vec, <4 x i64> undef, <4 x i32> <i32 2, i32 1, i32 3, i32 3>
-  ret <4 x i64> %res
-}
-define <4 x i64> @test_masked_4xi64_perm_mask3(<4 x i64> %vec, <4 x i64> %vec2, <4 x i64> %mask) {
-; GENERIC-LABEL: test_masked_4xi64_perm_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %ymm2, %ymm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermq {{.*#+}} ymm1 {%k1} = ymm0[2,1,3,3] sched: [1:1.00]
-; GENERIC-NEXT:    vmovdqa %ymm1, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_4xi64_perm_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %ymm2, %ymm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermq {{.*#+}} ymm1 {%k1} = ymm0[2,1,3,3] sched: [3:1.00]
-; SKX-NEXT:    vmovdqa %ymm1, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <4 x i64> %vec, <4 x i64> undef, <4 x i32> <i32 2, i32 1, i32 3, i32 3>
-  %cmp = icmp eq <4 x i64> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x i64> %shuf, <4 x i64> %vec2
-  ret <4 x i64> %res
-}
-
-define <4 x i64> @test_masked_z_4xi64_perm_mask3(<4 x i64> %vec, <4 x i64> %mask) {
-; GENERIC-LABEL: test_masked_z_4xi64_perm_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %ymm1, %ymm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermq {{.*#+}} ymm0 {%k1} {z} = ymm0[2,1,3,3] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_4xi64_perm_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %ymm1, %ymm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermq {{.*#+}} ymm0 {%k1} {z} = ymm0[2,1,3,3] sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <4 x i64> %vec, <4 x i64> undef, <4 x i32> <i32 2, i32 1, i32 3, i32 3>
-  %cmp = icmp eq <4 x i64> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x i64> %shuf, <4 x i64> zeroinitializer
-  ret <4 x i64> %res
-}
-define <4 x i64> @test_4xi64_perm_mem_mask0(<4 x i64>* %vp) {
-; GENERIC-LABEL: test_4xi64_perm_mem_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpermpd {{.*#+}} ymm0 = mem[2,1,2,0] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xi64_perm_mem_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpermpd {{.*#+}} ymm0 = mem[2,1,2,0] sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <4 x i64>, <4 x i64>* %vp
-  %res = shufflevector <4 x i64> %vec, <4 x i64> undef, <4 x i32> <i32 2, i32 1, i32 2, i32 0>
-  ret <4 x i64> %res
-}
-define <4 x i64> @test_masked_4xi64_perm_mem_mask0(<4 x i64>* %vp, <4 x i64> %vec2, <4 x i64> %mask) {
-; GENERIC-LABEL: test_masked_4xi64_perm_mem_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %ymm1, %ymm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermq {{.*#+}} ymm0 {%k1} = mem[2,1,2,0] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_4xi64_perm_mem_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %ymm1, %ymm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermq {{.*#+}} ymm0 {%k1} = mem[2,1,2,0] sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <4 x i64>, <4 x i64>* %vp
-  %shuf = shufflevector <4 x i64> %vec, <4 x i64> undef, <4 x i32> <i32 2, i32 1, i32 2, i32 0>
-  %cmp = icmp eq <4 x i64> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x i64> %shuf, <4 x i64> %vec2
-  ret <4 x i64> %res
-}
-
-define <4 x i64> @test_masked_z_4xi64_perm_mem_mask0(<4 x i64>* %vp, <4 x i64> %mask) {
-; GENERIC-LABEL: test_masked_z_4xi64_perm_mem_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %ymm0, %ymm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermq {{.*#+}} ymm0 {%k1} {z} = mem[2,1,2,0] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_4xi64_perm_mem_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %ymm0, %ymm0, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermq {{.*#+}} ymm0 {%k1} {z} = mem[2,1,2,0] sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <4 x i64>, <4 x i64>* %vp
-  %shuf = shufflevector <4 x i64> %vec, <4 x i64> undef, <4 x i32> <i32 2, i32 1, i32 2, i32 0>
-  %cmp = icmp eq <4 x i64> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x i64> %shuf, <4 x i64> zeroinitializer
-  ret <4 x i64> %res
-}
-
-define <4 x i64> @test_masked_4xi64_perm_mem_mask1(<4 x i64>* %vp, <4 x i64> %vec2, <4 x i64> %mask) {
-; GENERIC-LABEL: test_masked_4xi64_perm_mem_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %ymm1, %ymm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermq {{.*#+}} ymm0 {%k1} = mem[2,1,1,1] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_4xi64_perm_mem_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %ymm1, %ymm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermq {{.*#+}} ymm0 {%k1} = mem[2,1,1,1] sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <4 x i64>, <4 x i64>* %vp
-  %shuf = shufflevector <4 x i64> %vec, <4 x i64> undef, <4 x i32> <i32 2, i32 1, i32 1, i32 1>
-  %cmp = icmp eq <4 x i64> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x i64> %shuf, <4 x i64> %vec2
-  ret <4 x i64> %res
-}
-
-define <4 x i64> @test_masked_z_4xi64_perm_mem_mask1(<4 x i64>* %vp, <4 x i64> %mask) {
-; GENERIC-LABEL: test_masked_z_4xi64_perm_mem_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %ymm0, %ymm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermq {{.*#+}} ymm0 {%k1} {z} = mem[2,1,1,1] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_4xi64_perm_mem_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %ymm0, %ymm0, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermq {{.*#+}} ymm0 {%k1} {z} = mem[2,1,1,1] sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <4 x i64>, <4 x i64>* %vp
-  %shuf = shufflevector <4 x i64> %vec, <4 x i64> undef, <4 x i32> <i32 2, i32 1, i32 1, i32 1>
-  %cmp = icmp eq <4 x i64> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x i64> %shuf, <4 x i64> zeroinitializer
-  ret <4 x i64> %res
-}
-
-define <4 x i64> @test_masked_4xi64_perm_mem_mask2(<4 x i64>* %vp, <4 x i64> %vec2, <4 x i64> %mask) {
-; GENERIC-LABEL: test_masked_4xi64_perm_mem_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %ymm1, %ymm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermq {{.*#+}} ymm0 {%k1} = mem[0,1,2,0] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_4xi64_perm_mem_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %ymm1, %ymm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermq {{.*#+}} ymm0 {%k1} = mem[0,1,2,0] sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <4 x i64>, <4 x i64>* %vp
-  %shuf = shufflevector <4 x i64> %vec, <4 x i64> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 0>
-  %cmp = icmp eq <4 x i64> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x i64> %shuf, <4 x i64> %vec2
-  ret <4 x i64> %res
-}
-
-define <4 x i64> @test_masked_z_4xi64_perm_mem_mask2(<4 x i64>* %vp, <4 x i64> %mask) {
-; GENERIC-LABEL: test_masked_z_4xi64_perm_mem_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %ymm0, %ymm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermq {{.*#+}} ymm0 {%k1} {z} = mem[0,1,2,0] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_4xi64_perm_mem_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %ymm0, %ymm0, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermq {{.*#+}} ymm0 {%k1} {z} = mem[0,1,2,0] sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <4 x i64>, <4 x i64>* %vp
-  %shuf = shufflevector <4 x i64> %vec, <4 x i64> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 0>
-  %cmp = icmp eq <4 x i64> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x i64> %shuf, <4 x i64> zeroinitializer
-  ret <4 x i64> %res
-}
-
-define <4 x i64> @test_4xi64_perm_mem_mask3(<4 x i64>* %vp) {
-; GENERIC-LABEL: test_4xi64_perm_mem_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpermpd {{.*#+}} ymm0 = mem[2,0,1,3] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xi64_perm_mem_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpermpd {{.*#+}} ymm0 = mem[2,0,1,3] sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <4 x i64>, <4 x i64>* %vp
-  %res = shufflevector <4 x i64> %vec, <4 x i64> undef, <4 x i32> <i32 2, i32 0, i32 1, i32 3>
-  ret <4 x i64> %res
-}
-define <4 x i64> @test_masked_4xi64_perm_mem_mask3(<4 x i64>* %vp, <4 x i64> %vec2, <4 x i64> %mask) {
-; GENERIC-LABEL: test_masked_4xi64_perm_mem_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %ymm1, %ymm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermq {{.*#+}} ymm0 {%k1} = mem[2,0,1,3] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_4xi64_perm_mem_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %ymm1, %ymm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermq {{.*#+}} ymm0 {%k1} = mem[2,0,1,3] sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <4 x i64>, <4 x i64>* %vp
-  %shuf = shufflevector <4 x i64> %vec, <4 x i64> undef, <4 x i32> <i32 2, i32 0, i32 1, i32 3>
-  %cmp = icmp eq <4 x i64> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x i64> %shuf, <4 x i64> %vec2
-  ret <4 x i64> %res
-}
-
-define <4 x i64> @test_masked_z_4xi64_perm_mem_mask3(<4 x i64>* %vp, <4 x i64> %mask) {
-; GENERIC-LABEL: test_masked_z_4xi64_perm_mem_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %ymm0, %ymm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermq {{.*#+}} ymm0 {%k1} {z} = mem[2,0,1,3] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_4xi64_perm_mem_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %ymm0, %ymm0, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermq {{.*#+}} ymm0 {%k1} {z} = mem[2,0,1,3] sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <4 x i64>, <4 x i64>* %vp
-  %shuf = shufflevector <4 x i64> %vec, <4 x i64> undef, <4 x i32> <i32 2, i32 0, i32 1, i32 3>
-  %cmp = icmp eq <4 x i64> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x i64> %shuf, <4 x i64> zeroinitializer
-  ret <4 x i64> %res
-}
-
-define <8 x i64> @test_8xi64_perm_mask0(<8 x i64> %vec) {
-; GENERIC-LABEL: test_8xi64_perm_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovaps {{.*#+}} zmm1 = [0,4,7,6,5,5,1,6] sched: [7:0.50]
-; GENERIC-NEXT:    vpermpd %zmm0, %zmm1, %zmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xi64_perm_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovaps {{.*#+}} zmm1 = [0,4,7,6,5,5,1,6] sched: [8:0.50]
-; SKX-NEXT:    vpermpd %zmm0, %zmm1, %zmm0 # sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %res = shufflevector <8 x i64> %vec, <8 x i64> undef, <8 x i32> <i32 0, i32 4, i32 7, i32 6, i32 5, i32 5, i32 1, i32 6>
-  ret <8 x i64> %res
-}
-define <8 x i64> @test_masked_8xi64_perm_mask0(<8 x i64> %vec, <8 x i64> %vec2, <8 x i64> %mask) {
-; GENERIC-LABEL: test_masked_8xi64_perm_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [0,4,7,6,5,5,1,6] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmq %zmm2, %zmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermq %zmm0, %zmm3, %zmm1 {%k1} # sched: [1:1.00]
-; GENERIC-NEXT:    vmovdqa64 %zmm1, %zmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_8xi64_perm_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [0,4,7,6,5,5,1,6] sched: [8:0.50]
-; SKX-NEXT:    vptestnmq %zmm2, %zmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermq %zmm0, %zmm3, %zmm1 {%k1} # sched: [3:1.00]
-; SKX-NEXT:    vmovdqa64 %zmm1, %zmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x i64> %vec, <8 x i64> undef, <8 x i32> <i32 0, i32 4, i32 7, i32 6, i32 5, i32 5, i32 1, i32 6>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i64> %shuf, <8 x i64> %vec2
-  ret <8 x i64> %res
-}
-
-define <8 x i64> @test_masked_z_8xi64_perm_mask0(<8 x i64> %vec, <8 x i64> %mask) {
-; GENERIC-LABEL: test_masked_z_8xi64_perm_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [0,4,7,6,5,5,1,6] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmq %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermq %zmm0, %zmm2, %zmm0 {%k1} {z} # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_8xi64_perm_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [0,4,7,6,5,5,1,6] sched: [8:0.50]
-; SKX-NEXT:    vptestnmq %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermq %zmm0, %zmm2, %zmm0 {%k1} {z} # sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x i64> %vec, <8 x i64> undef, <8 x i32> <i32 0, i32 4, i32 7, i32 6, i32 5, i32 5, i32 1, i32 6>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i64> %shuf, <8 x i64> zeroinitializer
-  ret <8 x i64> %res
-}
-define <8 x i64> @test_masked_8xi64_perm_imm_mask1(<8 x i64> %vec, <8 x i64> %vec2, <8 x i64> %mask) {
-; GENERIC-LABEL: test_masked_8xi64_perm_imm_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %zmm2, %zmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermq {{.*#+}} zmm1 {%k1} = zmm0[1,0,1,1,5,4,5,5] sched: [1:1.00]
-; GENERIC-NEXT:    vmovdqa64 %zmm1, %zmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_8xi64_perm_imm_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %zmm2, %zmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermq {{.*#+}} zmm1 {%k1} = zmm0[1,0,1,1,5,4,5,5] sched: [3:1.00]
-; SKX-NEXT:    vmovdqa64 %zmm1, %zmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x i64> %vec, <8 x i64> undef, <8 x i32> <i32 1, i32 0, i32 1, i32 1, i32 5, i32 4, i32 5, i32 5>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i64> %shuf, <8 x i64> %vec2
-  ret <8 x i64> %res
-}
-
-define <8 x i64> @test_masked_z_8xi64_perm_imm_mask1(<8 x i64> %vec, <8 x i64> %mask) {
-; GENERIC-LABEL: test_masked_z_8xi64_perm_imm_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermq {{.*#+}} zmm0 {%k1} {z} = zmm0[1,0,1,1,5,4,5,5] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_8xi64_perm_imm_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermq {{.*#+}} zmm0 {%k1} {z} = zmm0[1,0,1,1,5,4,5,5] sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x i64> %vec, <8 x i64> undef, <8 x i32> <i32 1, i32 0, i32 1, i32 1, i32 5, i32 4, i32 5, i32 5>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i64> %shuf, <8 x i64> zeroinitializer
-  ret <8 x i64> %res
-}
-define <8 x i64> @test_masked_8xi64_perm_mask2(<8 x i64> %vec, <8 x i64> %vec2, <8 x i64> %mask) {
-; GENERIC-LABEL: test_masked_8xi64_perm_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [1,3,7,3,3,5,4,1] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmq %zmm2, %zmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermq %zmm0, %zmm3, %zmm1 {%k1} # sched: [1:1.00]
-; GENERIC-NEXT:    vmovdqa64 %zmm1, %zmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_8xi64_perm_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [1,3,7,3,3,5,4,1] sched: [8:0.50]
-; SKX-NEXT:    vptestnmq %zmm2, %zmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermq %zmm0, %zmm3, %zmm1 {%k1} # sched: [3:1.00]
-; SKX-NEXT:    vmovdqa64 %zmm1, %zmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x i64> %vec, <8 x i64> undef, <8 x i32> <i32 1, i32 3, i32 7, i32 3, i32 3, i32 5, i32 4, i32 1>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i64> %shuf, <8 x i64> %vec2
-  ret <8 x i64> %res
-}
-
-define <8 x i64> @test_masked_z_8xi64_perm_mask2(<8 x i64> %vec, <8 x i64> %mask) {
-; GENERIC-LABEL: test_masked_z_8xi64_perm_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [1,3,7,3,3,5,4,1] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmq %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermq %zmm0, %zmm2, %zmm0 {%k1} {z} # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_8xi64_perm_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [1,3,7,3,3,5,4,1] sched: [8:0.50]
-; SKX-NEXT:    vptestnmq %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermq %zmm0, %zmm2, %zmm0 {%k1} {z} # sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x i64> %vec, <8 x i64> undef, <8 x i32> <i32 1, i32 3, i32 7, i32 3, i32 3, i32 5, i32 4, i32 1>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i64> %shuf, <8 x i64> zeroinitializer
-  ret <8 x i64> %res
-}
-define <8 x i64> @test_8xi64_perm_imm_mask3(<8 x i64> %vec) {
-; GENERIC-LABEL: test_8xi64_perm_imm_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpermpd {{.*#+}} zmm0 = zmm0[3,1,3,1,7,5,7,5] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xi64_perm_imm_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpermpd {{.*#+}} zmm0 = zmm0[3,1,3,1,7,5,7,5] sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %res = shufflevector <8 x i64> %vec, <8 x i64> undef, <8 x i32> <i32 3, i32 1, i32 3, i32 1, i32 7, i32 5, i32 7, i32 5>
-  ret <8 x i64> %res
-}
-define <8 x i64> @test_masked_8xi64_perm_imm_mask3(<8 x i64> %vec, <8 x i64> %vec2, <8 x i64> %mask) {
-; GENERIC-LABEL: test_masked_8xi64_perm_imm_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %zmm2, %zmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermq {{.*#+}} zmm1 {%k1} = zmm0[3,1,3,1,7,5,7,5] sched: [1:1.00]
-; GENERIC-NEXT:    vmovdqa64 %zmm1, %zmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_8xi64_perm_imm_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %zmm2, %zmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermq {{.*#+}} zmm1 {%k1} = zmm0[3,1,3,1,7,5,7,5] sched: [3:1.00]
-; SKX-NEXT:    vmovdqa64 %zmm1, %zmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x i64> %vec, <8 x i64> undef, <8 x i32> <i32 3, i32 1, i32 3, i32 1, i32 7, i32 5, i32 7, i32 5>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i64> %shuf, <8 x i64> %vec2
-  ret <8 x i64> %res
-}
-
-define <8 x i64> @test_masked_z_8xi64_perm_imm_mask3(<8 x i64> %vec, <8 x i64> %mask) {
-; GENERIC-LABEL: test_masked_z_8xi64_perm_imm_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermq {{.*#+}} zmm0 {%k1} {z} = zmm0[3,1,3,1,7,5,7,5] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_8xi64_perm_imm_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermq {{.*#+}} zmm0 {%k1} {z} = zmm0[3,1,3,1,7,5,7,5] sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x i64> %vec, <8 x i64> undef, <8 x i32> <i32 3, i32 1, i32 3, i32 1, i32 7, i32 5, i32 7, i32 5>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i64> %shuf, <8 x i64> zeroinitializer
-  ret <8 x i64> %res
-}
-define <8 x i64> @test_masked_8xi64_perm_mask4(<8 x i64> %vec, <8 x i64> %vec2, <8 x i64> %mask) {
-; GENERIC-LABEL: test_masked_8xi64_perm_mask4:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [6,3,1,1,7,4,0,3] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmq %zmm2, %zmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermq %zmm0, %zmm3, %zmm1 {%k1} # sched: [1:1.00]
-; GENERIC-NEXT:    vmovdqa64 %zmm1, %zmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_8xi64_perm_mask4:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [6,3,1,1,7,4,0,3] sched: [8:0.50]
-; SKX-NEXT:    vptestnmq %zmm2, %zmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermq %zmm0, %zmm3, %zmm1 {%k1} # sched: [3:1.00]
-; SKX-NEXT:    vmovdqa64 %zmm1, %zmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x i64> %vec, <8 x i64> undef, <8 x i32> <i32 6, i32 3, i32 1, i32 1, i32 7, i32 4, i32 0, i32 3>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i64> %shuf, <8 x i64> %vec2
-  ret <8 x i64> %res
-}
-
-define <8 x i64> @test_masked_z_8xi64_perm_mask4(<8 x i64> %vec, <8 x i64> %mask) {
-; GENERIC-LABEL: test_masked_z_8xi64_perm_mask4:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [6,3,1,1,7,4,0,3] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmq %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermq %zmm0, %zmm2, %zmm0 {%k1} {z} # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_8xi64_perm_mask4:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [6,3,1,1,7,4,0,3] sched: [8:0.50]
-; SKX-NEXT:    vptestnmq %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermq %zmm0, %zmm2, %zmm0 {%k1} {z} # sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x i64> %vec, <8 x i64> undef, <8 x i32> <i32 6, i32 3, i32 1, i32 1, i32 7, i32 4, i32 0, i32 3>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i64> %shuf, <8 x i64> zeroinitializer
-  ret <8 x i64> %res
-}
-define <8 x i64> @test_masked_8xi64_perm_imm_mask5(<8 x i64> %vec, <8 x i64> %vec2, <8 x i64> %mask) {
-; GENERIC-LABEL: test_masked_8xi64_perm_imm_mask5:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %zmm2, %zmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermq {{.*#+}} zmm1 {%k1} = zmm0[0,0,0,0,4,4,4,4] sched: [1:1.00]
-; GENERIC-NEXT:    vmovdqa64 %zmm1, %zmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_8xi64_perm_imm_mask5:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %zmm2, %zmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermq {{.*#+}} zmm1 {%k1} = zmm0[0,0,0,0,4,4,4,4] sched: [3:1.00]
-; SKX-NEXT:    vmovdqa64 %zmm1, %zmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x i64> %vec, <8 x i64> undef, <8 x i32> <i32 0, i32 0, i32 0, i32 0, i32 4, i32 4, i32 4, i32 4>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i64> %shuf, <8 x i64> %vec2
-  ret <8 x i64> %res
-}
-
-define <8 x i64> @test_masked_z_8xi64_perm_imm_mask5(<8 x i64> %vec, <8 x i64> %mask) {
-; GENERIC-LABEL: test_masked_z_8xi64_perm_imm_mask5:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermq {{.*#+}} zmm0 {%k1} {z} = zmm0[0,0,0,0,4,4,4,4] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_8xi64_perm_imm_mask5:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermq {{.*#+}} zmm0 {%k1} {z} = zmm0[0,0,0,0,4,4,4,4] sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x i64> %vec, <8 x i64> undef, <8 x i32> <i32 0, i32 0, i32 0, i32 0, i32 4, i32 4, i32 4, i32 4>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i64> %shuf, <8 x i64> zeroinitializer
-  ret <8 x i64> %res
-}
-define <8 x i64> @test_8xi64_perm_mask6(<8 x i64> %vec) {
-; GENERIC-LABEL: test_8xi64_perm_mask6:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovaps {{.*#+}} zmm1 = [5,1,4,4,5,4,2,7] sched: [7:0.50]
-; GENERIC-NEXT:    vpermpd %zmm0, %zmm1, %zmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xi64_perm_mask6:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovaps {{.*#+}} zmm1 = [5,1,4,4,5,4,2,7] sched: [8:0.50]
-; SKX-NEXT:    vpermpd %zmm0, %zmm1, %zmm0 # sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %res = shufflevector <8 x i64> %vec, <8 x i64> undef, <8 x i32> <i32 5, i32 1, i32 4, i32 4, i32 5, i32 4, i32 2, i32 7>
-  ret <8 x i64> %res
-}
-define <8 x i64> @test_masked_8xi64_perm_mask6(<8 x i64> %vec, <8 x i64> %vec2, <8 x i64> %mask) {
-; GENERIC-LABEL: test_masked_8xi64_perm_mask6:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [5,1,4,4,5,4,2,7] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmq %zmm2, %zmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermq %zmm0, %zmm3, %zmm1 {%k1} # sched: [1:1.00]
-; GENERIC-NEXT:    vmovdqa64 %zmm1, %zmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_8xi64_perm_mask6:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [5,1,4,4,5,4,2,7] sched: [8:0.50]
-; SKX-NEXT:    vptestnmq %zmm2, %zmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermq %zmm0, %zmm3, %zmm1 {%k1} # sched: [3:1.00]
-; SKX-NEXT:    vmovdqa64 %zmm1, %zmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x i64> %vec, <8 x i64> undef, <8 x i32> <i32 5, i32 1, i32 4, i32 4, i32 5, i32 4, i32 2, i32 7>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i64> %shuf, <8 x i64> %vec2
-  ret <8 x i64> %res
-}
-
-define <8 x i64> @test_masked_z_8xi64_perm_mask6(<8 x i64> %vec, <8 x i64> %mask) {
-; GENERIC-LABEL: test_masked_z_8xi64_perm_mask6:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [5,1,4,4,5,4,2,7] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmq %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermq %zmm0, %zmm2, %zmm0 {%k1} {z} # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_8xi64_perm_mask6:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [5,1,4,4,5,4,2,7] sched: [8:0.50]
-; SKX-NEXT:    vptestnmq %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermq %zmm0, %zmm2, %zmm0 {%k1} {z} # sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x i64> %vec, <8 x i64> undef, <8 x i32> <i32 5, i32 1, i32 4, i32 4, i32 5, i32 4, i32 2, i32 7>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i64> %shuf, <8 x i64> zeroinitializer
-  ret <8 x i64> %res
-}
-define <8 x i64> @test_masked_8xi64_perm_imm_mask7(<8 x i64> %vec, <8 x i64> %vec2, <8 x i64> %mask) {
-; GENERIC-LABEL: test_masked_8xi64_perm_imm_mask7:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %zmm2, %zmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermq {{.*#+}} zmm1 {%k1} = zmm0[3,3,3,3,7,7,7,7] sched: [1:1.00]
-; GENERIC-NEXT:    vmovdqa64 %zmm1, %zmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_8xi64_perm_imm_mask7:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %zmm2, %zmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermq {{.*#+}} zmm1 {%k1} = zmm0[3,3,3,3,7,7,7,7] sched: [3:1.00]
-; SKX-NEXT:    vmovdqa64 %zmm1, %zmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x i64> %vec, <8 x i64> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 7, i32 7, i32 7, i32 7>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i64> %shuf, <8 x i64> %vec2
-  ret <8 x i64> %res
-}
-
-define <8 x i64> @test_masked_z_8xi64_perm_imm_mask7(<8 x i64> %vec, <8 x i64> %mask) {
-; GENERIC-LABEL: test_masked_z_8xi64_perm_imm_mask7:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermq {{.*#+}} zmm0 {%k1} {z} = zmm0[3,3,3,3,7,7,7,7] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_8xi64_perm_imm_mask7:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermq {{.*#+}} zmm0 {%k1} {z} = zmm0[3,3,3,3,7,7,7,7] sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x i64> %vec, <8 x i64> undef, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 7, i32 7, i32 7, i32 7>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i64> %shuf, <8 x i64> zeroinitializer
-  ret <8 x i64> %res
-}
-define <8 x i64> @test_8xi64_perm_mem_mask0(<8 x i64>* %vp) {
-; GENERIC-LABEL: test_8xi64_perm_mem_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovaps {{.*#+}} zmm0 = [5,1,6,5,7,3,7,3] sched: [7:0.50]
-; GENERIC-NEXT:    vpermpd (%rdi), %zmm0, %zmm0 # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xi64_perm_mem_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovaps {{.*#+}} zmm0 = [5,1,6,5,7,3,7,3] sched: [8:0.50]
-; SKX-NEXT:    vpermpd (%rdi), %zmm0, %zmm0 # sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <8 x i64>, <8 x i64>* %vp
-  %res = shufflevector <8 x i64> %vec, <8 x i64> undef, <8 x i32> <i32 5, i32 1, i32 6, i32 5, i32 7, i32 3, i32 7, i32 3>
-  ret <8 x i64> %res
-}
-define <8 x i64> @test_masked_8xi64_perm_mem_mask0(<8 x i64>* %vp, <8 x i64> %vec2, <8 x i64> %mask) {
-; GENERIC-LABEL: test_masked_8xi64_perm_mem_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [5,1,6,5,7,3,7,3] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmq %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermq (%rdi), %zmm2, %zmm0 {%k1} # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_8xi64_perm_mem_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [5,1,6,5,7,3,7,3] sched: [8:0.50]
-; SKX-NEXT:    vptestnmq %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermq (%rdi), %zmm2, %zmm0 {%k1} # sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <8 x i64>, <8 x i64>* %vp
-  %shuf = shufflevector <8 x i64> %vec, <8 x i64> undef, <8 x i32> <i32 5, i32 1, i32 6, i32 5, i32 7, i32 3, i32 7, i32 3>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i64> %shuf, <8 x i64> %vec2
-  ret <8 x i64> %res
-}
-
-define <8 x i64> @test_masked_z_8xi64_perm_mem_mask0(<8 x i64>* %vp, <8 x i64> %mask) {
-; GENERIC-LABEL: test_masked_z_8xi64_perm_mem_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [5,1,6,5,7,3,7,3] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmq %zmm0, %zmm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermq (%rdi), %zmm1, %zmm0 {%k1} {z} # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_8xi64_perm_mem_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [5,1,6,5,7,3,7,3] sched: [8:0.50]
-; SKX-NEXT:    vptestnmq %zmm0, %zmm0, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermq (%rdi), %zmm1, %zmm0 {%k1} {z} # sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <8 x i64>, <8 x i64>* %vp
-  %shuf = shufflevector <8 x i64> %vec, <8 x i64> undef, <8 x i32> <i32 5, i32 1, i32 6, i32 5, i32 7, i32 3, i32 7, i32 3>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i64> %shuf, <8 x i64> zeroinitializer
-  ret <8 x i64> %res
-}
-
-define <8 x i64> @test_masked_8xi64_perm_imm_mem_mask1(<8 x i64>* %vp, <8 x i64> %vec2, <8 x i64> %mask) {
-; GENERIC-LABEL: test_masked_8xi64_perm_imm_mem_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermq {{.*#+}} zmm0 {%k1} = mem[1,1,1,0,5,5,5,4] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_8xi64_perm_imm_mem_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermq {{.*#+}} zmm0 {%k1} = mem[1,1,1,0,5,5,5,4] sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <8 x i64>, <8 x i64>* %vp
-  %shuf = shufflevector <8 x i64> %vec, <8 x i64> undef, <8 x i32> <i32 1, i32 1, i32 1, i32 0, i32 5, i32 5, i32 5, i32 4>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i64> %shuf, <8 x i64> %vec2
-  ret <8 x i64> %res
-}
-
-define <8 x i64> @test_masked_z_8xi64_perm_imm_mem_mask1(<8 x i64>* %vp, <8 x i64> %mask) {
-; GENERIC-LABEL: test_masked_z_8xi64_perm_imm_mem_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %zmm0, %zmm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermq {{.*#+}} zmm0 {%k1} {z} = mem[1,1,1,0,5,5,5,4] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_8xi64_perm_imm_mem_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %zmm0, %zmm0, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermq {{.*#+}} zmm0 {%k1} {z} = mem[1,1,1,0,5,5,5,4] sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <8 x i64>, <8 x i64>* %vp
-  %shuf = shufflevector <8 x i64> %vec, <8 x i64> undef, <8 x i32> <i32 1, i32 1, i32 1, i32 0, i32 5, i32 5, i32 5, i32 4>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i64> %shuf, <8 x i64> zeroinitializer
-  ret <8 x i64> %res
-}
-
-define <8 x i64> @test_masked_8xi64_perm_mem_mask2(<8 x i64>* %vp, <8 x i64> %vec2, <8 x i64> %mask) {
-; GENERIC-LABEL: test_masked_8xi64_perm_mem_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [0,2,1,4,1,1,5,5] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmq %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermq (%rdi), %zmm2, %zmm0 {%k1} # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_8xi64_perm_mem_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [0,2,1,4,1,1,5,5] sched: [8:0.50]
-; SKX-NEXT:    vptestnmq %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermq (%rdi), %zmm2, %zmm0 {%k1} # sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <8 x i64>, <8 x i64>* %vp
-  %shuf = shufflevector <8 x i64> %vec, <8 x i64> undef, <8 x i32> <i32 0, i32 2, i32 1, i32 4, i32 1, i32 1, i32 5, i32 5>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i64> %shuf, <8 x i64> %vec2
-  ret <8 x i64> %res
-}
-
-define <8 x i64> @test_masked_z_8xi64_perm_mem_mask2(<8 x i64>* %vp, <8 x i64> %mask) {
-; GENERIC-LABEL: test_masked_z_8xi64_perm_mem_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [0,2,1,4,1,1,5,5] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmq %zmm0, %zmm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermq (%rdi), %zmm1, %zmm0 {%k1} {z} # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_8xi64_perm_mem_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [0,2,1,4,1,1,5,5] sched: [8:0.50]
-; SKX-NEXT:    vptestnmq %zmm0, %zmm0, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermq (%rdi), %zmm1, %zmm0 {%k1} {z} # sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <8 x i64>, <8 x i64>* %vp
-  %shuf = shufflevector <8 x i64> %vec, <8 x i64> undef, <8 x i32> <i32 0, i32 2, i32 1, i32 4, i32 1, i32 1, i32 5, i32 5>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i64> %shuf, <8 x i64> zeroinitializer
-  ret <8 x i64> %res
-}
-
-define <8 x i64> @test_8xi64_perm_imm_mem_mask3(<8 x i64>* %vp) {
-; GENERIC-LABEL: test_8xi64_perm_imm_mem_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpermpd {{.*#+}} zmm0 = mem[1,3,1,1,5,7,5,5] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xi64_perm_imm_mem_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpermpd {{.*#+}} zmm0 = mem[1,3,1,1,5,7,5,5] sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <8 x i64>, <8 x i64>* %vp
-  %res = shufflevector <8 x i64> %vec, <8 x i64> undef, <8 x i32> <i32 1, i32 3, i32 1, i32 1, i32 5, i32 7, i32 5, i32 5>
-  ret <8 x i64> %res
-}
-define <8 x i64> @test_masked_8xi64_perm_imm_mem_mask3(<8 x i64>* %vp, <8 x i64> %vec2, <8 x i64> %mask) {
-; GENERIC-LABEL: test_masked_8xi64_perm_imm_mem_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermq {{.*#+}} zmm0 {%k1} = mem[1,3,1,1,5,7,5,5] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_8xi64_perm_imm_mem_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermq {{.*#+}} zmm0 {%k1} = mem[1,3,1,1,5,7,5,5] sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <8 x i64>, <8 x i64>* %vp
-  %shuf = shufflevector <8 x i64> %vec, <8 x i64> undef, <8 x i32> <i32 1, i32 3, i32 1, i32 1, i32 5, i32 7, i32 5, i32 5>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i64> %shuf, <8 x i64> %vec2
-  ret <8 x i64> %res
-}
-
-define <8 x i64> @test_masked_z_8xi64_perm_imm_mem_mask3(<8 x i64>* %vp, <8 x i64> %mask) {
-; GENERIC-LABEL: test_masked_z_8xi64_perm_imm_mem_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %zmm0, %zmm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermq {{.*#+}} zmm0 {%k1} {z} = mem[1,3,1,1,5,7,5,5] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_8xi64_perm_imm_mem_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %zmm0, %zmm0, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermq {{.*#+}} zmm0 {%k1} {z} = mem[1,3,1,1,5,7,5,5] sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <8 x i64>, <8 x i64>* %vp
-  %shuf = shufflevector <8 x i64> %vec, <8 x i64> undef, <8 x i32> <i32 1, i32 3, i32 1, i32 1, i32 5, i32 7, i32 5, i32 5>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i64> %shuf, <8 x i64> zeroinitializer
-  ret <8 x i64> %res
-}
-
-define <8 x i64> @test_masked_8xi64_perm_mem_mask4(<8 x i64>* %vp, <8 x i64> %vec2, <8 x i64> %mask) {
-; GENERIC-LABEL: test_masked_8xi64_perm_mem_mask4:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [5,0,7,0,3,5,0,6] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmq %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermq (%rdi), %zmm2, %zmm0 {%k1} # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_8xi64_perm_mem_mask4:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [5,0,7,0,3,5,0,6] sched: [8:0.50]
-; SKX-NEXT:    vptestnmq %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermq (%rdi), %zmm2, %zmm0 {%k1} # sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <8 x i64>, <8 x i64>* %vp
-  %shuf = shufflevector <8 x i64> %vec, <8 x i64> undef, <8 x i32> <i32 5, i32 0, i32 7, i32 0, i32 3, i32 5, i32 0, i32 6>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i64> %shuf, <8 x i64> %vec2
-  ret <8 x i64> %res
-}
-
-define <8 x i64> @test_masked_z_8xi64_perm_mem_mask4(<8 x i64>* %vp, <8 x i64> %mask) {
-; GENERIC-LABEL: test_masked_z_8xi64_perm_mem_mask4:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [5,0,7,0,3,5,0,6] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmq %zmm0, %zmm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermq (%rdi), %zmm1, %zmm0 {%k1} {z} # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_8xi64_perm_mem_mask4:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [5,0,7,0,3,5,0,6] sched: [8:0.50]
-; SKX-NEXT:    vptestnmq %zmm0, %zmm0, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermq (%rdi), %zmm1, %zmm0 {%k1} {z} # sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <8 x i64>, <8 x i64>* %vp
-  %shuf = shufflevector <8 x i64> %vec, <8 x i64> undef, <8 x i32> <i32 5, i32 0, i32 7, i32 0, i32 3, i32 5, i32 0, i32 6>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i64> %shuf, <8 x i64> zeroinitializer
-  ret <8 x i64> %res
-}
-
-define <8 x i64> @test_masked_8xi64_perm_imm_mem_mask5(<8 x i64>* %vp, <8 x i64> %vec2, <8 x i64> %mask) {
-; GENERIC-LABEL: test_masked_8xi64_perm_imm_mem_mask5:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermq {{.*#+}} zmm0 {%k1} = mem[3,1,0,0,7,5,4,4] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_8xi64_perm_imm_mem_mask5:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermq {{.*#+}} zmm0 {%k1} = mem[3,1,0,0,7,5,4,4] sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <8 x i64>, <8 x i64>* %vp
-  %shuf = shufflevector <8 x i64> %vec, <8 x i64> undef, <8 x i32> <i32 3, i32 1, i32 0, i32 0, i32 7, i32 5, i32 4, i32 4>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i64> %shuf, <8 x i64> %vec2
-  ret <8 x i64> %res
-}
-
-define <8 x i64> @test_masked_z_8xi64_perm_imm_mem_mask5(<8 x i64>* %vp, <8 x i64> %mask) {
-; GENERIC-LABEL: test_masked_z_8xi64_perm_imm_mem_mask5:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %zmm0, %zmm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermq {{.*#+}} zmm0 {%k1} {z} = mem[3,1,0,0,7,5,4,4] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_8xi64_perm_imm_mem_mask5:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %zmm0, %zmm0, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermq {{.*#+}} zmm0 {%k1} {z} = mem[3,1,0,0,7,5,4,4] sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <8 x i64>, <8 x i64>* %vp
-  %shuf = shufflevector <8 x i64> %vec, <8 x i64> undef, <8 x i32> <i32 3, i32 1, i32 0, i32 0, i32 7, i32 5, i32 4, i32 4>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i64> %shuf, <8 x i64> zeroinitializer
-  ret <8 x i64> %res
-}
-
-define <8 x i64> @test_8xi64_perm_mem_mask6(<8 x i64>* %vp) {
-; GENERIC-LABEL: test_8xi64_perm_mem_mask6:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovaps {{.*#+}} zmm0 = [0,6,3,7,3,0,3,6] sched: [7:0.50]
-; GENERIC-NEXT:    vpermpd (%rdi), %zmm0, %zmm0 # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xi64_perm_mem_mask6:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovaps {{.*#+}} zmm0 = [0,6,3,7,3,0,3,6] sched: [8:0.50]
-; SKX-NEXT:    vpermpd (%rdi), %zmm0, %zmm0 # sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <8 x i64>, <8 x i64>* %vp
-  %res = shufflevector <8 x i64> %vec, <8 x i64> undef, <8 x i32> <i32 0, i32 6, i32 3, i32 7, i32 3, i32 0, i32 3, i32 6>
-  ret <8 x i64> %res
-}
-define <8 x i64> @test_masked_8xi64_perm_mem_mask6(<8 x i64>* %vp, <8 x i64> %vec2, <8 x i64> %mask) {
-; GENERIC-LABEL: test_masked_8xi64_perm_mem_mask6:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [0,6,3,7,3,0,3,6] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmq %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermq (%rdi), %zmm2, %zmm0 {%k1} # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_8xi64_perm_mem_mask6:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [0,6,3,7,3,0,3,6] sched: [8:0.50]
-; SKX-NEXT:    vptestnmq %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermq (%rdi), %zmm2, %zmm0 {%k1} # sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <8 x i64>, <8 x i64>* %vp
-  %shuf = shufflevector <8 x i64> %vec, <8 x i64> undef, <8 x i32> <i32 0, i32 6, i32 3, i32 7, i32 3, i32 0, i32 3, i32 6>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i64> %shuf, <8 x i64> %vec2
-  ret <8 x i64> %res
-}
-
-define <8 x i64> @test_masked_z_8xi64_perm_mem_mask6(<8 x i64>* %vp, <8 x i64> %mask) {
-; GENERIC-LABEL: test_masked_z_8xi64_perm_mem_mask6:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [0,6,3,7,3,0,3,6] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmq %zmm0, %zmm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermq (%rdi), %zmm1, %zmm0 {%k1} {z} # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_8xi64_perm_mem_mask6:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [0,6,3,7,3,0,3,6] sched: [8:0.50]
-; SKX-NEXT:    vptestnmq %zmm0, %zmm0, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermq (%rdi), %zmm1, %zmm0 {%k1} {z} # sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <8 x i64>, <8 x i64>* %vp
-  %shuf = shufflevector <8 x i64> %vec, <8 x i64> undef, <8 x i32> <i32 0, i32 6, i32 3, i32 7, i32 3, i32 0, i32 3, i32 6>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i64> %shuf, <8 x i64> zeroinitializer
-  ret <8 x i64> %res
-}
-
-define <8 x i64> @test_masked_8xi64_perm_imm_mem_mask7(<8 x i64>* %vp, <8 x i64> %vec2, <8 x i64> %mask) {
-; GENERIC-LABEL: test_masked_8xi64_perm_imm_mem_mask7:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermq {{.*#+}} zmm0 {%k1} = mem[3,0,0,1,7,4,4,5] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_8xi64_perm_imm_mem_mask7:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermq {{.*#+}} zmm0 {%k1} = mem[3,0,0,1,7,4,4,5] sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <8 x i64>, <8 x i64>* %vp
-  %shuf = shufflevector <8 x i64> %vec, <8 x i64> undef, <8 x i32> <i32 3, i32 0, i32 0, i32 1, i32 7, i32 4, i32 4, i32 5>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i64> %shuf, <8 x i64> %vec2
-  ret <8 x i64> %res
-}
-
-define <8 x i64> @test_masked_z_8xi64_perm_imm_mem_mask7(<8 x i64>* %vp, <8 x i64> %mask) {
-; GENERIC-LABEL: test_masked_z_8xi64_perm_imm_mem_mask7:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %zmm0, %zmm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermq {{.*#+}} zmm0 {%k1} {z} = mem[3,0,0,1,7,4,4,5] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_8xi64_perm_imm_mem_mask7:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %zmm0, %zmm0, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermq {{.*#+}} zmm0 {%k1} {z} = mem[3,0,0,1,7,4,4,5] sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <8 x i64>, <8 x i64>* %vp
-  %shuf = shufflevector <8 x i64> %vec, <8 x i64> undef, <8 x i32> <i32 3, i32 0, i32 0, i32 1, i32 7, i32 4, i32 4, i32 5>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i64> %shuf, <8 x i64> zeroinitializer
-  ret <8 x i64> %res
-}
-
-define <8 x float> @test_8xfloat_perm_mask0(<8 x float> %vec) {
-; GENERIC-LABEL: test_8xfloat_perm_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovaps {{.*#+}} ymm1 = [3,4,2,4,1,2,3,4] sched: [7:0.50]
-; GENERIC-NEXT:    vpermps %ymm0, %ymm1, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xfloat_perm_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovaps {{.*#+}} ymm1 = [3,4,2,4,1,2,3,4] sched: [7:0.50]
-; SKX-NEXT:    vpermps %ymm0, %ymm1, %ymm0 # sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %res = shufflevector <8 x float> %vec, <8 x float> undef, <8 x i32> <i32 3, i32 4, i32 2, i32 4, i32 1, i32 2, i32 3, i32 4>
-  ret <8 x float> %res
-}
-define <8 x float> @test_masked_8xfloat_perm_mask0(<8 x float> %vec, <8 x float> %vec2, <8 x i32> %mask) {
-; GENERIC-LABEL: test_masked_8xfloat_perm_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovaps {{.*#+}} ymm3 = [3,4,2,4,1,2,3,4] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmd %ymm2, %ymm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermps %ymm0, %ymm3, %ymm1 {%k1} # sched: [1:1.00]
-; GENERIC-NEXT:    vmovaps %ymm1, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_8xfloat_perm_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovaps {{.*#+}} ymm3 = [3,4,2,4,1,2,3,4] sched: [7:0.50]
-; SKX-NEXT:    vptestnmd %ymm2, %ymm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermps %ymm0, %ymm3, %ymm1 {%k1} # sched: [3:1.00]
-; SKX-NEXT:    vmovaps %ymm1, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x float> %vec, <8 x float> undef, <8 x i32> <i32 3, i32 4, i32 2, i32 4, i32 1, i32 2, i32 3, i32 4>
-  %cmp = icmp eq <8 x i32> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x float> %shuf, <8 x float> %vec2
-  ret <8 x float> %res
-}
-
-define <8 x float> @test_masked_z_8xfloat_perm_mask0(<8 x float> %vec, <8 x i32> %mask) {
-; GENERIC-LABEL: test_masked_z_8xfloat_perm_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovaps {{.*#+}} ymm2 = [3,4,2,4,1,2,3,4] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmd %ymm1, %ymm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermps %ymm0, %ymm2, %ymm0 {%k1} {z} # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_8xfloat_perm_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovaps {{.*#+}} ymm2 = [3,4,2,4,1,2,3,4] sched: [7:0.50]
-; SKX-NEXT:    vptestnmd %ymm1, %ymm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermps %ymm0, %ymm2, %ymm0 {%k1} {z} # sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x float> %vec, <8 x float> undef, <8 x i32> <i32 3, i32 4, i32 2, i32 4, i32 1, i32 2, i32 3, i32 4>
-  %cmp = icmp eq <8 x i32> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x float> %shuf, <8 x float> zeroinitializer
-  ret <8 x float> %res
-}
-define <8 x float> @test_masked_8xfloat_perm_mask1(<8 x float> %vec, <8 x float> %vec2, <8 x i32> %mask) {
-; GENERIC-LABEL: test_masked_8xfloat_perm_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovaps {{.*#+}} ymm3 = [4,2,1,0,6,0,5,1] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmd %ymm2, %ymm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermps %ymm0, %ymm3, %ymm1 {%k1} # sched: [1:1.00]
-; GENERIC-NEXT:    vmovaps %ymm1, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_8xfloat_perm_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovaps {{.*#+}} ymm3 = [4,2,1,0,6,0,5,1] sched: [7:0.50]
-; SKX-NEXT:    vptestnmd %ymm2, %ymm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermps %ymm0, %ymm3, %ymm1 {%k1} # sched: [3:1.00]
-; SKX-NEXT:    vmovaps %ymm1, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x float> %vec, <8 x float> undef, <8 x i32> <i32 4, i32 2, i32 1, i32 0, i32 6, i32 0, i32 5, i32 1>
-  %cmp = icmp eq <8 x i32> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x float> %shuf, <8 x float> %vec2
-  ret <8 x float> %res
-}
-
-define <8 x float> @test_masked_z_8xfloat_perm_mask1(<8 x float> %vec, <8 x i64> %mask) {
-; GENERIC-LABEL: test_masked_z_8xfloat_perm_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovaps {{.*#+}} ymm2 = [4,2,1,0,6,0,5,1] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmq %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermps %ymm0, %ymm2, %ymm0 {%k1} {z} # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_8xfloat_perm_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovaps {{.*#+}} ymm2 = [4,2,1,0,6,0,5,1] sched: [7:0.50]
-; SKX-NEXT:    vptestnmq %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermps %ymm0, %ymm2, %ymm0 {%k1} {z} # sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x float> %vec, <8 x float> undef, <8 x i32> <i32 4, i32 2, i32 1, i32 0, i32 6, i32 0, i32 5, i32 1>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x float> %shuf, <8 x float> zeroinitializer
-  ret <8 x float> %res
-}
-define <8 x float> @test_masked_8xfloat_perm_mask2(<8 x float> %vec, <8 x float> %vec2, <8 x i32> %mask) {
-; GENERIC-LABEL: test_masked_8xfloat_perm_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovaps {{.*#+}} ymm3 = [2,5,5,5,4,6,0,5] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmd %ymm2, %ymm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermps %ymm0, %ymm3, %ymm1 {%k1} # sched: [1:1.00]
-; GENERIC-NEXT:    vmovaps %ymm1, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_8xfloat_perm_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovaps {{.*#+}} ymm3 = [2,5,5,5,4,6,0,5] sched: [7:0.50]
-; SKX-NEXT:    vptestnmd %ymm2, %ymm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermps %ymm0, %ymm3, %ymm1 {%k1} # sched: [3:1.00]
-; SKX-NEXT:    vmovaps %ymm1, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x float> %vec, <8 x float> undef, <8 x i32> <i32 2, i32 5, i32 5, i32 5, i32 4, i32 6, i32 0, i32 5>
-  %cmp = icmp eq <8 x i32> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x float> %shuf, <8 x float> %vec2
-  ret <8 x float> %res
-}
-
-define <8 x float> @test_masked_z_8xfloat_perm_mask2(<8 x float> %vec, <8 x i32> %mask) {
-; GENERIC-LABEL: test_masked_z_8xfloat_perm_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovaps {{.*#+}} ymm2 = [2,5,5,5,4,6,0,5] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmd %ymm1, %ymm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermps %ymm0, %ymm2, %ymm0 {%k1} {z} # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_8xfloat_perm_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovaps {{.*#+}} ymm2 = [2,5,5,5,4,6,0,5] sched: [7:0.50]
-; SKX-NEXT:    vptestnmd %ymm1, %ymm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermps %ymm0, %ymm2, %ymm0 {%k1} {z} # sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x float> %vec, <8 x float> undef, <8 x i32> <i32 2, i32 5, i32 5, i32 5, i32 4, i32 6, i32 0, i32 5>
-  %cmp = icmp eq <8 x i32> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x float> %shuf, <8 x float> zeroinitializer
-  ret <8 x float> %res
-}
-define <8 x float> @test_8xfloat_perm_mask3(<8 x float> %vec) {
-; GENERIC-LABEL: test_8xfloat_perm_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovaps {{.*#+}} ymm1 = [0,5,2,5,5,5,1,6] sched: [7:0.50]
-; GENERIC-NEXT:    vpermps %ymm0, %ymm1, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xfloat_perm_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovaps {{.*#+}} ymm1 = [0,5,2,5,5,5,1,6] sched: [7:0.50]
-; SKX-NEXT:    vpermps %ymm0, %ymm1, %ymm0 # sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %res = shufflevector <8 x float> %vec, <8 x float> undef, <8 x i32> <i32 0, i32 5, i32 2, i32 5, i32 5, i32 5, i32 1, i32 6>
-  ret <8 x float> %res
-}
-define <8 x float> @test_masked_8xfloat_perm_mask3(<8 x float> %vec, <8 x float> %vec2, <8 x i32> %mask) {
-; GENERIC-LABEL: test_masked_8xfloat_perm_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovaps {{.*#+}} ymm3 = [0,5,2,5,5,5,1,6] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmd %ymm2, %ymm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermps %ymm0, %ymm3, %ymm1 {%k1} # sched: [1:1.00]
-; GENERIC-NEXT:    vmovaps %ymm1, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_8xfloat_perm_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovaps {{.*#+}} ymm3 = [0,5,2,5,5,5,1,6] sched: [7:0.50]
-; SKX-NEXT:    vptestnmd %ymm2, %ymm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermps %ymm0, %ymm3, %ymm1 {%k1} # sched: [3:1.00]
-; SKX-NEXT:    vmovaps %ymm1, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x float> %vec, <8 x float> undef, <8 x i32> <i32 0, i32 5, i32 2, i32 5, i32 5, i32 5, i32 1, i32 6>
-  %cmp = icmp eq <8 x i32> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x float> %shuf, <8 x float> %vec2
-  ret <8 x float> %res
-}
-
-define <8 x float> @test_masked_z_8xfloat_perm_mask3(<8 x float> %vec, <8 x i32> %mask) {
-; GENERIC-LABEL: test_masked_z_8xfloat_perm_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovaps {{.*#+}} ymm2 = [0,5,2,5,5,5,1,6] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmd %ymm1, %ymm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermps %ymm0, %ymm2, %ymm0 {%k1} {z} # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_8xfloat_perm_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovaps {{.*#+}} ymm2 = [0,5,2,5,5,5,1,6] sched: [7:0.50]
-; SKX-NEXT:    vptestnmd %ymm1, %ymm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermps %ymm0, %ymm2, %ymm0 {%k1} {z} # sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x float> %vec, <8 x float> undef, <8 x i32> <i32 0, i32 5, i32 2, i32 5, i32 5, i32 5, i32 1, i32 6>
-  %cmp = icmp eq <8 x i32> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x float> %shuf, <8 x float> zeroinitializer
-  ret <8 x float> %res
-}
-define <8 x float> @test_8xfloat_perm_mem_mask0(<8 x float>* %vp) {
-; GENERIC-LABEL: test_8xfloat_perm_mem_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovaps {{.*#+}} ymm0 = [5,2,1,6,4,2,4,0] sched: [7:0.50]
-; GENERIC-NEXT:    vpermps (%rdi), %ymm0, %ymm0 # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xfloat_perm_mem_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovaps {{.*#+}} ymm0 = [5,2,1,6,4,2,4,0] sched: [7:0.50]
-; SKX-NEXT:    vpermps (%rdi), %ymm0, %ymm0 # sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <8 x float>, <8 x float>* %vp
-  %res = shufflevector <8 x float> %vec, <8 x float> undef, <8 x i32> <i32 5, i32 2, i32 1, i32 6, i32 4, i32 2, i32 4, i32 0>
-  ret <8 x float> %res
-}
-define <8 x float> @test_masked_8xfloat_perm_mem_mask0(<8 x float>* %vp, <8 x float> %vec2, <8 x i32> %mask) {
-; GENERIC-LABEL: test_masked_8xfloat_perm_mem_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovaps {{.*#+}} ymm2 = [5,2,1,6,4,2,4,0] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmd %ymm1, %ymm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermps (%rdi), %ymm2, %ymm0 {%k1} # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_8xfloat_perm_mem_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovaps {{.*#+}} ymm2 = [5,2,1,6,4,2,4,0] sched: [7:0.50]
-; SKX-NEXT:    vptestnmd %ymm1, %ymm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermps (%rdi), %ymm2, %ymm0 {%k1} # sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <8 x float>, <8 x float>* %vp
-  %shuf = shufflevector <8 x float> %vec, <8 x float> undef, <8 x i32> <i32 5, i32 2, i32 1, i32 6, i32 4, i32 2, i32 4, i32 0>
-  %cmp = icmp eq <8 x i32> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x float> %shuf, <8 x float> %vec2
-  ret <8 x float> %res
-}
-
-define <8 x float> @test_masked_z_8xfloat_perm_mem_mask0(<8 x float>* %vp, <8 x i32> %mask) {
-; GENERIC-LABEL: test_masked_z_8xfloat_perm_mem_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovaps {{.*#+}} ymm1 = [5,2,1,6,4,2,4,0] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmd %ymm0, %ymm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermps (%rdi), %ymm1, %ymm0 {%k1} {z} # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_8xfloat_perm_mem_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovaps {{.*#+}} ymm1 = [5,2,1,6,4,2,4,0] sched: [7:0.50]
-; SKX-NEXT:    vptestnmd %ymm0, %ymm0, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermps (%rdi), %ymm1, %ymm0 {%k1} {z} # sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <8 x float>, <8 x float>* %vp
-  %shuf = shufflevector <8 x float> %vec, <8 x float> undef, <8 x i32> <i32 5, i32 2, i32 1, i32 6, i32 4, i32 2, i32 4, i32 0>
-  %cmp = icmp eq <8 x i32> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x float> %shuf, <8 x float> zeroinitializer
-  ret <8 x float> %res
-}
-
-define <8 x float> @test_masked_8xfloat_perm_mem_mask1(<8 x float>* %vp, <8 x float> %vec2, <8 x i32> %mask) {
-; GENERIC-LABEL: test_masked_8xfloat_perm_mem_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovaps {{.*#+}} ymm2 = [1,3,7,4,0,6,6,6] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmd %ymm1, %ymm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermps (%rdi), %ymm2, %ymm0 {%k1} # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_8xfloat_perm_mem_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovaps {{.*#+}} ymm2 = [1,3,7,4,0,6,6,6] sched: [7:0.50]
-; SKX-NEXT:    vptestnmd %ymm1, %ymm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermps (%rdi), %ymm2, %ymm0 {%k1} # sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <8 x float>, <8 x float>* %vp
-  %shuf = shufflevector <8 x float> %vec, <8 x float> undef, <8 x i32> <i32 1, i32 3, i32 7, i32 4, i32 0, i32 6, i32 6, i32 6>
-  %cmp = icmp eq <8 x i32> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x float> %shuf, <8 x float> %vec2
-  ret <8 x float> %res
-}
-
-define <8 x float> @test_masked_z_8xfloat_perm_mem_mask1(<8 x float>* %vp, <8 x i32> %mask) {
-; GENERIC-LABEL: test_masked_z_8xfloat_perm_mem_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovaps {{.*#+}} ymm1 = [1,3,7,4,0,6,6,6] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmd %ymm0, %ymm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermps (%rdi), %ymm1, %ymm0 {%k1} {z} # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_8xfloat_perm_mem_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovaps {{.*#+}} ymm1 = [1,3,7,4,0,6,6,6] sched: [7:0.50]
-; SKX-NEXT:    vptestnmd %ymm0, %ymm0, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermps (%rdi), %ymm1, %ymm0 {%k1} {z} # sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <8 x float>, <8 x float>* %vp
-  %shuf = shufflevector <8 x float> %vec, <8 x float> undef, <8 x i32> <i32 1, i32 3, i32 7, i32 4, i32 0, i32 6, i32 6, i32 6>
-  %cmp = icmp eq <8 x i32> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x float> %shuf, <8 x float> zeroinitializer
-  ret <8 x float> %res
-}
-
-define <8 x float> @test_masked_8xfloat_perm_mem_mask2(<8 x float>* %vp, <8 x float> %vec2, <8 x i32> %mask) {
-; GENERIC-LABEL: test_masked_8xfloat_perm_mem_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovaps {{.*#+}} ymm2 = [4,5,1,5,6,6,2,4] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmd %ymm1, %ymm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermps (%rdi), %ymm2, %ymm0 {%k1} # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_8xfloat_perm_mem_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovaps {{.*#+}} ymm2 = [4,5,1,5,6,6,2,4] sched: [7:0.50]
-; SKX-NEXT:    vptestnmd %ymm1, %ymm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermps (%rdi), %ymm2, %ymm0 {%k1} # sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <8 x float>, <8 x float>* %vp
-  %shuf = shufflevector <8 x float> %vec, <8 x float> undef, <8 x i32> <i32 4, i32 5, i32 1, i32 5, i32 6, i32 6, i32 2, i32 4>
-  %cmp = icmp eq <8 x i32> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x float> %shuf, <8 x float> %vec2
-  ret <8 x float> %res
-}
-
-define <8 x float> @test_masked_z_8xfloat_perm_mem_mask2(<8 x float>* %vp, <8 x i32> %mask) {
-; GENERIC-LABEL: test_masked_z_8xfloat_perm_mem_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovaps {{.*#+}} ymm1 = [4,5,1,5,6,6,2,4] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmd %ymm0, %ymm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermps (%rdi), %ymm1, %ymm0 {%k1} {z} # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_8xfloat_perm_mem_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovaps {{.*#+}} ymm1 = [4,5,1,5,6,6,2,4] sched: [7:0.50]
-; SKX-NEXT:    vptestnmd %ymm0, %ymm0, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermps (%rdi), %ymm1, %ymm0 {%k1} {z} # sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <8 x float>, <8 x float>* %vp
-  %shuf = shufflevector <8 x float> %vec, <8 x float> undef, <8 x i32> <i32 4, i32 5, i32 1, i32 5, i32 6, i32 6, i32 2, i32 4>
-  %cmp = icmp eq <8 x i32> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x float> %shuf, <8 x float> zeroinitializer
-  ret <8 x float> %res
-}
-
-define <8 x float> @test_8xfloat_perm_mem_mask3(<8 x float>* %vp, <8 x i32> %mask) {
-; GENERIC-LABEL: test_8xfloat_perm_mem_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovaps {{.*#+}} ymm0 = [5,7,0,6,4,2,3,0] sched: [7:0.50]
-; GENERIC-NEXT:    vpermps (%rdi), %ymm0, %ymm0 # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xfloat_perm_mem_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovaps {{.*#+}} ymm0 = [5,7,0,6,4,2,3,0] sched: [7:0.50]
-; SKX-NEXT:    vpermps (%rdi), %ymm0, %ymm0 # sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <8 x float>, <8 x float>* %vp
-  %res = shufflevector <8 x float> %vec, <8 x float> undef, <8 x i32> <i32 5, i32 7, i32 0, i32 6, i32 4, i32 2, i32 3, i32 0>
-  ret <8 x float> %res
-}
-define <8 x float> @test_masked_8xfloat_perm_mem_mask3(<8 x float>* %vp, <8 x float> %vec2, <8 x i32> %mask) {
-; GENERIC-LABEL: test_masked_8xfloat_perm_mem_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovaps {{.*#+}} ymm2 = [5,7,0,6,4,2,3,0] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmd %ymm1, %ymm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermps (%rdi), %ymm2, %ymm0 {%k1} # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_8xfloat_perm_mem_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovaps {{.*#+}} ymm2 = [5,7,0,6,4,2,3,0] sched: [7:0.50]
-; SKX-NEXT:    vptestnmd %ymm1, %ymm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermps (%rdi), %ymm2, %ymm0 {%k1} # sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <8 x float>, <8 x float>* %vp
-  %shuf = shufflevector <8 x float> %vec, <8 x float> undef, <8 x i32> <i32 5, i32 7, i32 0, i32 6, i32 4, i32 2, i32 3, i32 0>
-  %cmp = icmp eq <8 x i32> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x float> %shuf, <8 x float> %vec2
-  ret <8 x float> %res
-}
-
-define <8 x float> @test_masked_z_8xfloat_perm_mem_mask3(<8 x float>* %vp, <8 x i32> %mask) {
-; GENERIC-LABEL: test_masked_z_8xfloat_perm_mem_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovaps {{.*#+}} ymm1 = [5,7,0,6,4,2,3,0] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmd %ymm0, %ymm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermps (%rdi), %ymm1, %ymm0 {%k1} {z} # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_8xfloat_perm_mem_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovaps {{.*#+}} ymm1 = [5,7,0,6,4,2,3,0] sched: [7:0.50]
-; SKX-NEXT:    vptestnmd %ymm0, %ymm0, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermps (%rdi), %ymm1, %ymm0 {%k1} {z} # sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <8 x float>, <8 x float>* %vp
-  %shuf = shufflevector <8 x float> %vec, <8 x float> undef, <8 x i32> <i32 5, i32 7, i32 0, i32 6, i32 4, i32 2, i32 3, i32 0>
-  %cmp = icmp eq <8 x i32> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x float> %shuf, <8 x float> zeroinitializer
-  ret <8 x float> %res
-}
-
-define <16 x float> @test_16xfloat_perm_mask0(<16 x float> %vec) {
-; GENERIC-LABEL: test_16xfloat_perm_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovaps {{.*#+}} zmm1 = [15,7,5,13,4,9,11,13,12,6,0,0,11,15,5,7] sched: [7:0.50]
-; GENERIC-NEXT:    vpermps %zmm0, %zmm1, %zmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_16xfloat_perm_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovaps {{.*#+}} zmm1 = [15,7,5,13,4,9,11,13,12,6,0,0,11,15,5,7] sched: [8:0.50]
-; SKX-NEXT:    vpermps %zmm0, %zmm1, %zmm0 # sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %res = shufflevector <16 x float> %vec, <16 x float> undef, <16 x i32> <i32 15, i32 7, i32 5, i32 13, i32 4, i32 9, i32 11, i32 13, i32 12, i32 6, i32 0, i32 0, i32 11, i32 15, i32 5, i32 7>
-  ret <16 x float> %res
-}
-define <16 x float> @test_masked_16xfloat_perm_mask0(<16 x float> %vec, <16 x float> %vec2, <16 x i32> %mask) {
-; GENERIC-LABEL: test_masked_16xfloat_perm_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovaps {{.*#+}} zmm3 = [15,7,5,13,4,9,11,13,12,6,0,0,11,15,5,7] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmd %zmm2, %zmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermps %zmm0, %zmm3, %zmm1 {%k1} # sched: [1:1.00]
-; GENERIC-NEXT:    vmovaps %zmm1, %zmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_16xfloat_perm_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovaps {{.*#+}} zmm3 = [15,7,5,13,4,9,11,13,12,6,0,0,11,15,5,7] sched: [8:0.50]
-; SKX-NEXT:    vptestnmd %zmm2, %zmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermps %zmm0, %zmm3, %zmm1 {%k1} # sched: [3:1.00]
-; SKX-NEXT:    vmovaps %zmm1, %zmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <16 x float> %vec, <16 x float> undef, <16 x i32> <i32 15, i32 7, i32 5, i32 13, i32 4, i32 9, i32 11, i32 13, i32 12, i32 6, i32 0, i32 0, i32 11, i32 15, i32 5, i32 7>
-  %cmp = icmp eq <16 x i32> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x float> %shuf, <16 x float> %vec2
-  ret <16 x float> %res
-}
-
-define <16 x float> @test_masked_z_16xfloat_perm_mask0(<16 x float> %vec, <16 x i32> %mask) {
-; GENERIC-LABEL: test_masked_z_16xfloat_perm_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovaps {{.*#+}} zmm2 = [15,7,5,13,4,9,11,13,12,6,0,0,11,15,5,7] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmd %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermps %zmm0, %zmm2, %zmm0 {%k1} {z} # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_16xfloat_perm_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovaps {{.*#+}} zmm2 = [15,7,5,13,4,9,11,13,12,6,0,0,11,15,5,7] sched: [8:0.50]
-; SKX-NEXT:    vptestnmd %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermps %zmm0, %zmm2, %zmm0 {%k1} {z} # sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <16 x float> %vec, <16 x float> undef, <16 x i32> <i32 15, i32 7, i32 5, i32 13, i32 4, i32 9, i32 11, i32 13, i32 12, i32 6, i32 0, i32 0, i32 11, i32 15, i32 5, i32 7>
-  %cmp = icmp eq <16 x i32> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x float> %shuf, <16 x float> zeroinitializer
-  ret <16 x float> %res
-}
-define <16 x float> @test_masked_16xfloat_perm_mask1(<16 x float> %vec, <16 x float> %vec2, <16 x i32> %mask) {
-; GENERIC-LABEL: test_masked_16xfloat_perm_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovaps {{.*#+}} zmm3 = [11,10,4,10,4,5,8,11,2,0,10,0,0,3,10,1] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmd %zmm2, %zmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermps %zmm0, %zmm3, %zmm1 {%k1} # sched: [1:1.00]
-; GENERIC-NEXT:    vmovaps %zmm1, %zmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_16xfloat_perm_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovaps {{.*#+}} zmm3 = [11,10,4,10,4,5,8,11,2,0,10,0,0,3,10,1] sched: [8:0.50]
-; SKX-NEXT:    vptestnmd %zmm2, %zmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermps %zmm0, %zmm3, %zmm1 {%k1} # sched: [3:1.00]
-; SKX-NEXT:    vmovaps %zmm1, %zmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <16 x float> %vec, <16 x float> undef, <16 x i32> <i32 11, i32 10, i32 4, i32 10, i32 4, i32 5, i32 8, i32 11, i32 2, i32 0, i32 10, i32 0, i32 0, i32 3, i32 10, i32 1>
-  %cmp = icmp eq <16 x i32> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x float> %shuf, <16 x float> %vec2
-  ret <16 x float> %res
-}
-
-define <16 x float> @test_masked_z_16xfloat_perm_mask1(<16 x float> %vec, <16 x i32> %mask) {
-; GENERIC-LABEL: test_masked_z_16xfloat_perm_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovaps {{.*#+}} zmm2 = [11,10,4,10,4,5,8,11,2,0,10,0,0,3,10,1] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmd %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermps %zmm0, %zmm2, %zmm0 {%k1} {z} # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_16xfloat_perm_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovaps {{.*#+}} zmm2 = [11,10,4,10,4,5,8,11,2,0,10,0,0,3,10,1] sched: [8:0.50]
-; SKX-NEXT:    vptestnmd %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermps %zmm0, %zmm2, %zmm0 {%k1} {z} # sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <16 x float> %vec, <16 x float> undef, <16 x i32> <i32 11, i32 10, i32 4, i32 10, i32 4, i32 5, i32 8, i32 11, i32 2, i32 0, i32 10, i32 0, i32 0, i32 3, i32 10, i32 1>
-  %cmp = icmp eq <16 x i32> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x float> %shuf, <16 x float> zeroinitializer
-  ret <16 x float> %res
-}
-define <16 x float> @test_masked_16xfloat_perm_mask2(<16 x float> %vec, <16 x float> %vec2, <16 x i32> %mask) {
-; GENERIC-LABEL: test_masked_16xfloat_perm_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovaps {{.*#+}} zmm3 = [0,15,6,14,3,6,5,2,5,15,11,6,6,4,8,11] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmd %zmm2, %zmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermps %zmm0, %zmm3, %zmm1 {%k1} # sched: [1:1.00]
-; GENERIC-NEXT:    vmovaps %zmm1, %zmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_16xfloat_perm_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovaps {{.*#+}} zmm3 = [0,15,6,14,3,6,5,2,5,15,11,6,6,4,8,11] sched: [8:0.50]
-; SKX-NEXT:    vptestnmd %zmm2, %zmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermps %zmm0, %zmm3, %zmm1 {%k1} # sched: [3:1.00]
-; SKX-NEXT:    vmovaps %zmm1, %zmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <16 x float> %vec, <16 x float> undef, <16 x i32> <i32 0, i32 15, i32 6, i32 14, i32 3, i32 6, i32 5, i32 2, i32 5, i32 15, i32 11, i32 6, i32 6, i32 4, i32 8, i32 11>
-  %cmp = icmp eq <16 x i32> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x float> %shuf, <16 x float> %vec2
-  ret <16 x float> %res
-}
-
-define <16 x float> @test_masked_z_16xfloat_perm_mask2(<16 x float> %vec, <16 x i32> %mask) {
-; GENERIC-LABEL: test_masked_z_16xfloat_perm_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovaps {{.*#+}} zmm2 = [0,15,6,14,3,6,5,2,5,15,11,6,6,4,8,11] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmd %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermps %zmm0, %zmm2, %zmm0 {%k1} {z} # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_16xfloat_perm_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovaps {{.*#+}} zmm2 = [0,15,6,14,3,6,5,2,5,15,11,6,6,4,8,11] sched: [8:0.50]
-; SKX-NEXT:    vptestnmd %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermps %zmm0, %zmm2, %zmm0 {%k1} {z} # sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <16 x float> %vec, <16 x float> undef, <16 x i32> <i32 0, i32 15, i32 6, i32 14, i32 3, i32 6, i32 5, i32 2, i32 5, i32 15, i32 11, i32 6, i32 6, i32 4, i32 8, i32 11>
-  %cmp = icmp eq <16 x i32> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x float> %shuf, <16 x float> zeroinitializer
-  ret <16 x float> %res
-}
-define <16 x float> @test_16xfloat_perm_mask3(<16 x float> %vec) {
-; GENERIC-LABEL: test_16xfloat_perm_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovaps {{.*#+}} zmm1 = [10,7,0,14,6,6,0,2,13,8,11,2,5,13,13,3] sched: [7:0.50]
-; GENERIC-NEXT:    vpermps %zmm0, %zmm1, %zmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_16xfloat_perm_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovaps {{.*#+}} zmm1 = [10,7,0,14,6,6,0,2,13,8,11,2,5,13,13,3] sched: [8:0.50]
-; SKX-NEXT:    vpermps %zmm0, %zmm1, %zmm0 # sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %res = shufflevector <16 x float> %vec, <16 x float> undef, <16 x i32> <i32 10, i32 7, i32 0, i32 14, i32 6, i32 6, i32 0, i32 2, i32 13, i32 8, i32 11, i32 2, i32 5, i32 13, i32 13, i32 3>
-  ret <16 x float> %res
-}
-define <16 x float> @test_masked_16xfloat_perm_mask3(<16 x float> %vec, <16 x float> %vec2, <16 x i32> %mask) {
-; GENERIC-LABEL: test_masked_16xfloat_perm_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovaps {{.*#+}} zmm3 = [10,7,0,14,6,6,0,2,13,8,11,2,5,13,13,3] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmd %zmm2, %zmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermps %zmm0, %zmm3, %zmm1 {%k1} # sched: [1:1.00]
-; GENERIC-NEXT:    vmovaps %zmm1, %zmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_16xfloat_perm_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovaps {{.*#+}} zmm3 = [10,7,0,14,6,6,0,2,13,8,11,2,5,13,13,3] sched: [8:0.50]
-; SKX-NEXT:    vptestnmd %zmm2, %zmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermps %zmm0, %zmm3, %zmm1 {%k1} # sched: [3:1.00]
-; SKX-NEXT:    vmovaps %zmm1, %zmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <16 x float> %vec, <16 x float> undef, <16 x i32> <i32 10, i32 7, i32 0, i32 14, i32 6, i32 6, i32 0, i32 2, i32 13, i32 8, i32 11, i32 2, i32 5, i32 13, i32 13, i32 3>
-  %cmp = icmp eq <16 x i32> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x float> %shuf, <16 x float> %vec2
-  ret <16 x float> %res
-}
-
-define <16 x float> @test_masked_z_16xfloat_perm_mask3(<16 x float> %vec, <16 x i32> %mask) {
-; GENERIC-LABEL: test_masked_z_16xfloat_perm_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovaps {{.*#+}} zmm2 = [10,7,0,14,6,6,0,2,13,8,11,2,5,13,13,3] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmd %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermps %zmm0, %zmm2, %zmm0 {%k1} {z} # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_16xfloat_perm_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovaps {{.*#+}} zmm2 = [10,7,0,14,6,6,0,2,13,8,11,2,5,13,13,3] sched: [8:0.50]
-; SKX-NEXT:    vptestnmd %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermps %zmm0, %zmm2, %zmm0 {%k1} {z} # sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <16 x float> %vec, <16 x float> undef, <16 x i32> <i32 10, i32 7, i32 0, i32 14, i32 6, i32 6, i32 0, i32 2, i32 13, i32 8, i32 11, i32 2, i32 5, i32 13, i32 13, i32 3>
-  %cmp = icmp eq <16 x i32> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x float> %shuf, <16 x float> zeroinitializer
-  ret <16 x float> %res
-}
-define <16 x float> @test_16xfloat_perm_mem_mask0(<16 x float>* %vp) {
-; GENERIC-LABEL: test_16xfloat_perm_mem_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovaps {{.*#+}} zmm0 = [10,2,1,14,9,9,7,2,9,4,12,11,0,14,0,1] sched: [7:0.50]
-; GENERIC-NEXT:    vpermps (%rdi), %zmm0, %zmm0 # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_16xfloat_perm_mem_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovaps {{.*#+}} zmm0 = [10,2,1,14,9,9,7,2,9,4,12,11,0,14,0,1] sched: [8:0.50]
-; SKX-NEXT:    vpermps (%rdi), %zmm0, %zmm0 # sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <16 x float>, <16 x float>* %vp
-  %res = shufflevector <16 x float> %vec, <16 x float> undef, <16 x i32> <i32 10, i32 2, i32 1, i32 14, i32 9, i32 9, i32 7, i32 2, i32 9, i32 4, i32 12, i32 11, i32 0, i32 14, i32 0, i32 1>
-  ret <16 x float> %res
-}
-define <16 x float> @test_masked_16xfloat_perm_mem_mask0(<16 x float>* %vp, <16 x float> %vec2, <16 x i32> %mask) {
-; GENERIC-LABEL: test_masked_16xfloat_perm_mem_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovaps {{.*#+}} zmm2 = [10,2,1,14,9,9,7,2,9,4,12,11,0,14,0,1] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmd %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermps (%rdi), %zmm2, %zmm0 {%k1} # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_16xfloat_perm_mem_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovaps {{.*#+}} zmm2 = [10,2,1,14,9,9,7,2,9,4,12,11,0,14,0,1] sched: [8:0.50]
-; SKX-NEXT:    vptestnmd %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermps (%rdi), %zmm2, %zmm0 {%k1} # sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <16 x float>, <16 x float>* %vp
-  %shuf = shufflevector <16 x float> %vec, <16 x float> undef, <16 x i32> <i32 10, i32 2, i32 1, i32 14, i32 9, i32 9, i32 7, i32 2, i32 9, i32 4, i32 12, i32 11, i32 0, i32 14, i32 0, i32 1>
-  %cmp = icmp eq <16 x i32> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x float> %shuf, <16 x float> %vec2
-  ret <16 x float> %res
-}
-
-define <16 x float> @test_masked_z_16xfloat_perm_mem_mask0(<16 x float>* %vp, <16 x i32> %mask) {
-; GENERIC-LABEL: test_masked_z_16xfloat_perm_mem_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovaps {{.*#+}} zmm1 = [10,2,1,14,9,9,7,2,9,4,12,11,0,14,0,1] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmd %zmm0, %zmm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermps (%rdi), %zmm1, %zmm0 {%k1} {z} # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_16xfloat_perm_mem_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovaps {{.*#+}} zmm1 = [10,2,1,14,9,9,7,2,9,4,12,11,0,14,0,1] sched: [8:0.50]
-; SKX-NEXT:    vptestnmd %zmm0, %zmm0, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermps (%rdi), %zmm1, %zmm0 {%k1} {z} # sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <16 x float>, <16 x float>* %vp
-  %shuf = shufflevector <16 x float> %vec, <16 x float> undef, <16 x i32> <i32 10, i32 2, i32 1, i32 14, i32 9, i32 9, i32 7, i32 2, i32 9, i32 4, i32 12, i32 11, i32 0, i32 14, i32 0, i32 1>
-  %cmp = icmp eq <16 x i32> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x float> %shuf, <16 x float> zeroinitializer
-  ret <16 x float> %res
-}
-
-define <16 x float> @test_masked_16xfloat_perm_mem_mask1(<16 x float>* %vp, <16 x float> %vec2, <16 x i32> %mask) {
-; GENERIC-LABEL: test_masked_16xfloat_perm_mem_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovaps {{.*#+}} zmm2 = [4,2,3,5,11,6,4,7,6,4,14,8,15,12,9,4] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmd %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermps (%rdi), %zmm2, %zmm0 {%k1} # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_16xfloat_perm_mem_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovaps {{.*#+}} zmm2 = [4,2,3,5,11,6,4,7,6,4,14,8,15,12,9,4] sched: [8:0.50]
-; SKX-NEXT:    vptestnmd %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermps (%rdi), %zmm2, %zmm0 {%k1} # sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <16 x float>, <16 x float>* %vp
-  %shuf = shufflevector <16 x float> %vec, <16 x float> undef, <16 x i32> <i32 4, i32 2, i32 3, i32 5, i32 11, i32 6, i32 4, i32 7, i32 6, i32 4, i32 14, i32 8, i32 15, i32 12, i32 9, i32 4>
-  %cmp = icmp eq <16 x i32> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x float> %shuf, <16 x float> %vec2
-  ret <16 x float> %res
-}
-
-define <16 x float> @test_masked_z_16xfloat_perm_mem_mask1(<16 x float>* %vp, <16 x i32> %mask) {
-; GENERIC-LABEL: test_masked_z_16xfloat_perm_mem_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovaps {{.*#+}} zmm1 = [4,2,3,5,11,6,4,7,6,4,14,8,15,12,9,4] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmd %zmm0, %zmm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermps (%rdi), %zmm1, %zmm0 {%k1} {z} # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_16xfloat_perm_mem_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovaps {{.*#+}} zmm1 = [4,2,3,5,11,6,4,7,6,4,14,8,15,12,9,4] sched: [8:0.50]
-; SKX-NEXT:    vptestnmd %zmm0, %zmm0, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermps (%rdi), %zmm1, %zmm0 {%k1} {z} # sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <16 x float>, <16 x float>* %vp
-  %shuf = shufflevector <16 x float> %vec, <16 x float> undef, <16 x i32> <i32 4, i32 2, i32 3, i32 5, i32 11, i32 6, i32 4, i32 7, i32 6, i32 4, i32 14, i32 8, i32 15, i32 12, i32 9, i32 4>
-  %cmp = icmp eq <16 x i32> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x float> %shuf, <16 x float> zeroinitializer
-  ret <16 x float> %res
-}
-
-define <16 x float> @test_masked_16xfloat_perm_mem_mask2(<16 x float>* %vp, <16 x float> %vec2, <16 x i32> %mask) {
-; GENERIC-LABEL: test_masked_16xfloat_perm_mem_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovaps {{.*#+}} zmm2 = [10,7,11,6,7,0,11,0,10,9,12,4,10,3,8,5] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmd %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermps (%rdi), %zmm2, %zmm0 {%k1} # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_16xfloat_perm_mem_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovaps {{.*#+}} zmm2 = [10,7,11,6,7,0,11,0,10,9,12,4,10,3,8,5] sched: [8:0.50]
-; SKX-NEXT:    vptestnmd %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermps (%rdi), %zmm2, %zmm0 {%k1} # sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <16 x float>, <16 x float>* %vp
-  %shuf = shufflevector <16 x float> %vec, <16 x float> undef, <16 x i32> <i32 10, i32 7, i32 11, i32 6, i32 7, i32 0, i32 11, i32 0, i32 10, i32 9, i32 12, i32 4, i32 10, i32 3, i32 8, i32 5>
-  %cmp = icmp eq <16 x i32> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x float> %shuf, <16 x float> %vec2
-  ret <16 x float> %res
-}
-
-define <16 x float> @test_masked_z_16xfloat_perm_mem_mask2(<16 x float>* %vp, <16 x i32> %mask) {
-; GENERIC-LABEL: test_masked_z_16xfloat_perm_mem_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovaps {{.*#+}} zmm1 = [10,7,11,6,7,0,11,0,10,9,12,4,10,3,8,5] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmd %zmm0, %zmm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermps (%rdi), %zmm1, %zmm0 {%k1} {z} # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_16xfloat_perm_mem_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovaps {{.*#+}} zmm1 = [10,7,11,6,7,0,11,0,10,9,12,4,10,3,8,5] sched: [8:0.50]
-; SKX-NEXT:    vptestnmd %zmm0, %zmm0, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermps (%rdi), %zmm1, %zmm0 {%k1} {z} # sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <16 x float>, <16 x float>* %vp
-  %shuf = shufflevector <16 x float> %vec, <16 x float> undef, <16 x i32> <i32 10, i32 7, i32 11, i32 6, i32 7, i32 0, i32 11, i32 0, i32 10, i32 9, i32 12, i32 4, i32 10, i32 3, i32 8, i32 5>
-  %cmp = icmp eq <16 x i32> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x float> %shuf, <16 x float> zeroinitializer
-  ret <16 x float> %res
-}
-
-define <16 x float> @test_16xfloat_perm_mem_mask3(<16 x float>* %vp) {
-; GENERIC-LABEL: test_16xfloat_perm_mem_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovaps {{.*#+}} zmm0 = [15,15,3,9,5,15,14,9,11,10,5,14,14,5,11,0] sched: [7:0.50]
-; GENERIC-NEXT:    vpermps (%rdi), %zmm0, %zmm0 # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_16xfloat_perm_mem_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovaps {{.*#+}} zmm0 = [15,15,3,9,5,15,14,9,11,10,5,14,14,5,11,0] sched: [8:0.50]
-; SKX-NEXT:    vpermps (%rdi), %zmm0, %zmm0 # sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <16 x float>, <16 x float>* %vp
-  %res = shufflevector <16 x float> %vec, <16 x float> undef, <16 x i32> <i32 15, i32 15, i32 3, i32 9, i32 5, i32 15, i32 14, i32 9, i32 11, i32 10, i32 5, i32 14, i32 14, i32 5, i32 11, i32 0>
-  ret <16 x float> %res
-}
-define <16 x float> @test_masked_16xfloat_perm_mem_mask3(<16 x float>* %vp, <16 x float> %vec2, <16 x i32> %mask) {
-; GENERIC-LABEL: test_masked_16xfloat_perm_mem_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovaps {{.*#+}} zmm2 = [15,15,3,9,5,15,14,9,11,10,5,14,14,5,11,0] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmd %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermps (%rdi), %zmm2, %zmm0 {%k1} # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_16xfloat_perm_mem_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovaps {{.*#+}} zmm2 = [15,15,3,9,5,15,14,9,11,10,5,14,14,5,11,0] sched: [8:0.50]
-; SKX-NEXT:    vptestnmd %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermps (%rdi), %zmm2, %zmm0 {%k1} # sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <16 x float>, <16 x float>* %vp
-  %shuf = shufflevector <16 x float> %vec, <16 x float> undef, <16 x i32> <i32 15, i32 15, i32 3, i32 9, i32 5, i32 15, i32 14, i32 9, i32 11, i32 10, i32 5, i32 14, i32 14, i32 5, i32 11, i32 0>
-  %cmp = icmp eq <16 x i32> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x float> %shuf, <16 x float> %vec2
-  ret <16 x float> %res
-}
-
-define <16 x float> @test_masked_z_16xfloat_perm_mem_mask3(<16 x float>* %vp, <16 x i32> %mask) {
-; GENERIC-LABEL: test_masked_z_16xfloat_perm_mem_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovaps {{.*#+}} zmm1 = [15,15,3,9,5,15,14,9,11,10,5,14,14,5,11,0] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmd %zmm0, %zmm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermps (%rdi), %zmm1, %zmm0 {%k1} {z} # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_16xfloat_perm_mem_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovaps {{.*#+}} zmm1 = [15,15,3,9,5,15,14,9,11,10,5,14,14,5,11,0] sched: [8:0.50]
-; SKX-NEXT:    vptestnmd %zmm0, %zmm0, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermps (%rdi), %zmm1, %zmm0 {%k1} {z} # sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <16 x float>, <16 x float>* %vp
-  %shuf = shufflevector <16 x float> %vec, <16 x float> undef, <16 x i32> <i32 15, i32 15, i32 3, i32 9, i32 5, i32 15, i32 14, i32 9, i32 11, i32 10, i32 5, i32 14, i32 14, i32 5, i32 11, i32 0>
-  %cmp = icmp eq <16 x i32> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x float> %shuf, <16 x float> zeroinitializer
-  ret <16 x float> %res
-}
-
-define <4 x double> @test_4xdouble_perm_mask0(<4 x double> %vec) {
-; GENERIC-LABEL: test_4xdouble_perm_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[2,1,3,2] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xdouble_perm_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[2,1,3,2] sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %res = shufflevector <4 x double> %vec, <4 x double> undef, <4 x i32> <i32 2, i32 1, i32 3, i32 2>
-  ret <4 x double> %res
-}
-define <4 x double> @test_masked_4xdouble_perm_mask0(<4 x double> %vec, <4 x double> %vec2, <4 x i64> %mask) {
-; GENERIC-LABEL: test_masked_4xdouble_perm_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %ymm2, %ymm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermpd {{.*#+}} ymm1 {%k1} = ymm0[2,1,3,2] sched: [1:1.00]
-; GENERIC-NEXT:    vmovapd %ymm1, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_4xdouble_perm_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %ymm2, %ymm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermpd {{.*#+}} ymm1 {%k1} = ymm0[2,1,3,2] sched: [3:1.00]
-; SKX-NEXT:    vmovapd %ymm1, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <4 x double> %vec, <4 x double> undef, <4 x i32> <i32 2, i32 1, i32 3, i32 2>
-  %cmp = icmp eq <4 x i64> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x double> %shuf, <4 x double> %vec2
-  ret <4 x double> %res
-}
-
-define <4 x double> @test_masked_z_4xdouble_perm_mask0(<4 x double> %vec, <4 x i64> %mask) {
-; GENERIC-LABEL: test_masked_z_4xdouble_perm_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %ymm1, %ymm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermpd {{.*#+}} ymm0 {%k1} {z} = ymm0[2,1,3,2] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_4xdouble_perm_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %ymm1, %ymm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermpd {{.*#+}} ymm0 {%k1} {z} = ymm0[2,1,3,2] sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <4 x double> %vec, <4 x double> undef, <4 x i32> <i32 2, i32 1, i32 3, i32 2>
-  %cmp = icmp eq <4 x i64> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x double> %shuf, <4 x double> zeroinitializer
-  ret <4 x double> %res
-}
-define <4 x double> @test_masked_4xdouble_perm_mask1(<4 x double> %vec, <4 x double> %vec2, <4 x i64> %mask) {
-; GENERIC-LABEL: test_masked_4xdouble_perm_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %ymm2, %ymm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermpd {{.*#+}} ymm1 {%k1} = ymm0[3,0,0,0] sched: [1:1.00]
-; GENERIC-NEXT:    vmovapd %ymm1, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_4xdouble_perm_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %ymm2, %ymm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermpd {{.*#+}} ymm1 {%k1} = ymm0[3,0,0,0] sched: [3:1.00]
-; SKX-NEXT:    vmovapd %ymm1, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <4 x double> %vec, <4 x double> undef, <4 x i32> <i32 3, i32 0, i32 0, i32 0>
-  %cmp = icmp eq <4 x i64> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x double> %shuf, <4 x double> %vec2
-  ret <4 x double> %res
-}
-
-define <4 x double> @test_masked_z_4xdouble_perm_mask1(<4 x double> %vec, <4 x i64> %mask) {
-; GENERIC-LABEL: test_masked_z_4xdouble_perm_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %ymm1, %ymm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermpd {{.*#+}} ymm0 {%k1} {z} = ymm0[3,0,0,0] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_4xdouble_perm_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %ymm1, %ymm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermpd {{.*#+}} ymm0 {%k1} {z} = ymm0[3,0,0,0] sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <4 x double> %vec, <4 x double> undef, <4 x i32> <i32 3, i32 0, i32 0, i32 0>
-  %cmp = icmp eq <4 x i64> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x double> %shuf, <4 x double> zeroinitializer
-  ret <4 x double> %res
-}
-define <4 x double> @test_masked_4xdouble_perm_mask2(<4 x double> %vec, <4 x double> %vec2, <4 x i64> %mask) {
-; GENERIC-LABEL: test_masked_4xdouble_perm_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %ymm2, %ymm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermpd {{.*#+}} ymm1 {%k1} = ymm0[0,3,3,1] sched: [1:1.00]
-; GENERIC-NEXT:    vmovapd %ymm1, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_4xdouble_perm_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %ymm2, %ymm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermpd {{.*#+}} ymm1 {%k1} = ymm0[0,3,3,1] sched: [3:1.00]
-; SKX-NEXT:    vmovapd %ymm1, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <4 x double> %vec, <4 x double> undef, <4 x i32> <i32 0, i32 3, i32 3, i32 1>
-  %cmp = icmp eq <4 x i64> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x double> %shuf, <4 x double> %vec2
-  ret <4 x double> %res
-}
-
-define <4 x double> @test_masked_z_4xdouble_perm_mask2(<4 x double> %vec, <4 x i64> %mask) {
-; GENERIC-LABEL: test_masked_z_4xdouble_perm_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %ymm1, %ymm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermpd {{.*#+}} ymm0 {%k1} {z} = ymm0[0,3,3,1] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_4xdouble_perm_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %ymm1, %ymm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermpd {{.*#+}} ymm0 {%k1} {z} = ymm0[0,3,3,1] sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <4 x double> %vec, <4 x double> undef, <4 x i32> <i32 0, i32 3, i32 3, i32 1>
-  %cmp = icmp eq <4 x i64> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x double> %shuf, <4 x double> zeroinitializer
-  ret <4 x double> %res
-}
-define <4 x double> @test_4xdouble_perm_mask3(<4 x double> %vec) {
-; GENERIC-LABEL: test_4xdouble_perm_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[3,3,3,2] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xdouble_perm_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[3,3,3,2] sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %res = shufflevector <4 x double> %vec, <4 x double> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 2>
-  ret <4 x double> %res
-}
-define <4 x double> @test_masked_4xdouble_perm_mask3(<4 x double> %vec, <4 x double> %vec2, <4 x i64> %mask) {
-; GENERIC-LABEL: test_masked_4xdouble_perm_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %ymm2, %ymm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermpd {{.*#+}} ymm1 {%k1} = ymm0[3,3,3,2] sched: [1:1.00]
-; GENERIC-NEXT:    vmovapd %ymm1, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_4xdouble_perm_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %ymm2, %ymm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermpd {{.*#+}} ymm1 {%k1} = ymm0[3,3,3,2] sched: [3:1.00]
-; SKX-NEXT:    vmovapd %ymm1, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <4 x double> %vec, <4 x double> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 2>
-  %cmp = icmp eq <4 x i64> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x double> %shuf, <4 x double> %vec2
-  ret <4 x double> %res
-}
-
-define <4 x double> @test_masked_z_4xdouble_perm_mask3(<4 x double> %vec, <4 x i64> %mask) {
-; GENERIC-LABEL: test_masked_z_4xdouble_perm_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %ymm1, %ymm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermpd {{.*#+}} ymm0 {%k1} {z} = ymm0[3,3,3,2] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_4xdouble_perm_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %ymm1, %ymm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermpd {{.*#+}} ymm0 {%k1} {z} = ymm0[3,3,3,2] sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <4 x double> %vec, <4 x double> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 2>
-  %cmp = icmp eq <4 x i64> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x double> %shuf, <4 x double> zeroinitializer
-  ret <4 x double> %res
-}
-define <4 x double> @test_4xdouble_perm_mem_mask0(<4 x double>* %vp) {
-; GENERIC-LABEL: test_4xdouble_perm_mem_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpermpd {{.*#+}} ymm0 = mem[0,0,2,0] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xdouble_perm_mem_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpermpd {{.*#+}} ymm0 = mem[0,0,2,0] sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <4 x double>, <4 x double>* %vp
-  %res = shufflevector <4 x double> %vec, <4 x double> undef, <4 x i32> <i32 0, i32 0, i32 2, i32 0>
-  ret <4 x double> %res
-}
-define <4 x double> @test_masked_4xdouble_perm_mem_mask0(<4 x double>* %vp, <4 x double> %vec2, <4 x i64> %mask) {
-; GENERIC-LABEL: test_masked_4xdouble_perm_mem_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %ymm1, %ymm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermpd {{.*#+}} ymm0 {%k1} = mem[0,0,2,0] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_4xdouble_perm_mem_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %ymm1, %ymm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermpd {{.*#+}} ymm0 {%k1} = mem[0,0,2,0] sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <4 x double>, <4 x double>* %vp
-  %shuf = shufflevector <4 x double> %vec, <4 x double> undef, <4 x i32> <i32 0, i32 0, i32 2, i32 0>
-  %cmp = icmp eq <4 x i64> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x double> %shuf, <4 x double> %vec2
-  ret <4 x double> %res
-}
-
-define <4 x double> @test_masked_z_4xdouble_perm_mem_mask0(<4 x double>* %vp, <4 x i64> %mask) {
-; GENERIC-LABEL: test_masked_z_4xdouble_perm_mem_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %ymm0, %ymm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermpd {{.*#+}} ymm0 {%k1} {z} = mem[0,0,2,0] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_4xdouble_perm_mem_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %ymm0, %ymm0, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermpd {{.*#+}} ymm0 {%k1} {z} = mem[0,0,2,0] sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <4 x double>, <4 x double>* %vp
-  %shuf = shufflevector <4 x double> %vec, <4 x double> undef, <4 x i32> <i32 0, i32 0, i32 2, i32 0>
-  %cmp = icmp eq <4 x i64> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x double> %shuf, <4 x double> zeroinitializer
-  ret <4 x double> %res
-}
-
-define <4 x double> @test_masked_4xdouble_perm_mem_mask1(<4 x double>* %vp, <4 x double> %vec2, <4 x i64> %mask) {
-; GENERIC-LABEL: test_masked_4xdouble_perm_mem_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %ymm1, %ymm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermpd {{.*#+}} ymm0 {%k1} = mem[0,2,3,2] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_4xdouble_perm_mem_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %ymm1, %ymm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermpd {{.*#+}} ymm0 {%k1} = mem[0,2,3,2] sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <4 x double>, <4 x double>* %vp
-  %shuf = shufflevector <4 x double> %vec, <4 x double> undef, <4 x i32> <i32 0, i32 2, i32 3, i32 2>
-  %cmp = icmp eq <4 x i64> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x double> %shuf, <4 x double> %vec2
-  ret <4 x double> %res
-}
-
-define <4 x double> @test_masked_z_4xdouble_perm_mem_mask1(<4 x double>* %vp, <4 x i64> %mask) {
-; GENERIC-LABEL: test_masked_z_4xdouble_perm_mem_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %ymm0, %ymm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermpd {{.*#+}} ymm0 {%k1} {z} = mem[0,2,3,2] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_4xdouble_perm_mem_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %ymm0, %ymm0, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermpd {{.*#+}} ymm0 {%k1} {z} = mem[0,2,3,2] sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <4 x double>, <4 x double>* %vp
-  %shuf = shufflevector <4 x double> %vec, <4 x double> undef, <4 x i32> <i32 0, i32 2, i32 3, i32 2>
-  %cmp = icmp eq <4 x i64> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x double> %shuf, <4 x double> zeroinitializer
-  ret <4 x double> %res
-}
-
-define <4 x double> @test_masked_4xdouble_perm_mem_mask2(<4 x double>* %vp, <4 x double> %vec2, <4 x i64> %mask) {
-; GENERIC-LABEL: test_masked_4xdouble_perm_mem_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %ymm1, %ymm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermpd {{.*#+}} ymm0 {%k1} = mem[3,1,1,1] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_4xdouble_perm_mem_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %ymm1, %ymm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermpd {{.*#+}} ymm0 {%k1} = mem[3,1,1,1] sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <4 x double>, <4 x double>* %vp
-  %shuf = shufflevector <4 x double> %vec, <4 x double> undef, <4 x i32> <i32 3, i32 1, i32 1, i32 1>
-  %cmp = icmp eq <4 x i64> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x double> %shuf, <4 x double> %vec2
-  ret <4 x double> %res
-}
-
-define <4 x double> @test_masked_z_4xdouble_perm_mem_mask2(<4 x double>* %vp, <4 x i64> %mask) {
-; GENERIC-LABEL: test_masked_z_4xdouble_perm_mem_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %ymm0, %ymm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermpd {{.*#+}} ymm0 {%k1} {z} = mem[3,1,1,1] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_4xdouble_perm_mem_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %ymm0, %ymm0, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermpd {{.*#+}} ymm0 {%k1} {z} = mem[3,1,1,1] sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <4 x double>, <4 x double>* %vp
-  %shuf = shufflevector <4 x double> %vec, <4 x double> undef, <4 x i32> <i32 3, i32 1, i32 1, i32 1>
-  %cmp = icmp eq <4 x i64> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x double> %shuf, <4 x double> zeroinitializer
-  ret <4 x double> %res
-}
-
-define <4 x double> @test_4xdouble_perm_mem_mask3(<4 x double>* %vp) {
-; GENERIC-LABEL: test_4xdouble_perm_mem_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpermpd {{.*#+}} ymm0 = mem[3,2,3,2] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xdouble_perm_mem_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpermpd {{.*#+}} ymm0 = mem[3,2,3,2] sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <4 x double>, <4 x double>* %vp
-  %res = shufflevector <4 x double> %vec, <4 x double> undef, <4 x i32> <i32 3, i32 2, i32 3, i32 2>
-  ret <4 x double> %res
-}
-define <4 x double> @test_masked_4xdouble_perm_mem_mask3(<4 x double>* %vp, <4 x double> %vec2, <4 x i64> %mask) {
-; GENERIC-LABEL: test_masked_4xdouble_perm_mem_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %ymm1, %ymm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermpd {{.*#+}} ymm0 {%k1} = mem[3,2,3,2] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_4xdouble_perm_mem_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %ymm1, %ymm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermpd {{.*#+}} ymm0 {%k1} = mem[3,2,3,2] sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <4 x double>, <4 x double>* %vp
-  %shuf = shufflevector <4 x double> %vec, <4 x double> undef, <4 x i32> <i32 3, i32 2, i32 3, i32 2>
-  %cmp = icmp eq <4 x i64> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x double> %shuf, <4 x double> %vec2
-  ret <4 x double> %res
-}
-
-define <4 x double> @test_masked_z_4xdouble_perm_mem_mask3(<4 x double>* %vp, <4 x i64> %mask) {
-; GENERIC-LABEL: test_masked_z_4xdouble_perm_mem_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %ymm0, %ymm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermpd {{.*#+}} ymm0 {%k1} {z} = mem[3,2,3,2] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_4xdouble_perm_mem_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %ymm0, %ymm0, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermpd {{.*#+}} ymm0 {%k1} {z} = mem[3,2,3,2] sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <4 x double>, <4 x double>* %vp
-  %shuf = shufflevector <4 x double> %vec, <4 x double> undef, <4 x i32> <i32 3, i32 2, i32 3, i32 2>
-  %cmp = icmp eq <4 x i64> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x double> %shuf, <4 x double> zeroinitializer
-  ret <4 x double> %res
-}
-
-define <8 x double> @test_8xdouble_perm_mask0(<8 x double> %vec) {
-; GENERIC-LABEL: test_8xdouble_perm_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovaps {{.*#+}} zmm1 = [5,7,4,2,7,4,3,4] sched: [7:0.50]
-; GENERIC-NEXT:    vpermpd %zmm0, %zmm1, %zmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xdouble_perm_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovaps {{.*#+}} zmm1 = [5,7,4,2,7,4,3,4] sched: [8:0.50]
-; SKX-NEXT:    vpermpd %zmm0, %zmm1, %zmm0 # sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %res = shufflevector <8 x double> %vec, <8 x double> undef, <8 x i32> <i32 5, i32 7, i32 4, i32 2, i32 7, i32 4, i32 3, i32 4>
-  ret <8 x double> %res
-}
-define <8 x double> @test_masked_8xdouble_perm_mask0(<8 x double> %vec, <8 x double> %vec2, <8 x i64> %mask) {
-; GENERIC-LABEL: test_masked_8xdouble_perm_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovapd {{.*#+}} zmm3 = [5,7,4,2,7,4,3,4] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmq %zmm2, %zmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermpd %zmm0, %zmm3, %zmm1 {%k1} # sched: [1:1.00]
-; GENERIC-NEXT:    vmovapd %zmm1, %zmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_8xdouble_perm_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovapd {{.*#+}} zmm3 = [5,7,4,2,7,4,3,4] sched: [8:0.50]
-; SKX-NEXT:    vptestnmq %zmm2, %zmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermpd %zmm0, %zmm3, %zmm1 {%k1} # sched: [3:1.00]
-; SKX-NEXT:    vmovapd %zmm1, %zmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x double> %vec, <8 x double> undef, <8 x i32> <i32 5, i32 7, i32 4, i32 2, i32 7, i32 4, i32 3, i32 4>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x double> %shuf, <8 x double> %vec2
-  ret <8 x double> %res
-}
-
-define <8 x double> @test_masked_z_8xdouble_perm_mask0(<8 x double> %vec, <8 x i64> %mask) {
-; GENERIC-LABEL: test_masked_z_8xdouble_perm_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovapd {{.*#+}} zmm2 = [5,7,4,2,7,4,3,4] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmq %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermpd %zmm0, %zmm2, %zmm0 {%k1} {z} # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_8xdouble_perm_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovapd {{.*#+}} zmm2 = [5,7,4,2,7,4,3,4] sched: [8:0.50]
-; SKX-NEXT:    vptestnmq %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermpd %zmm0, %zmm2, %zmm0 {%k1} {z} # sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x double> %vec, <8 x double> undef, <8 x i32> <i32 5, i32 7, i32 4, i32 2, i32 7, i32 4, i32 3, i32 4>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x double> %shuf, <8 x double> zeroinitializer
-  ret <8 x double> %res
-}
-define <8 x double> @test_masked_8xdouble_perm_imm_mask1(<8 x double> %vec, <8 x double> %vec2, <8 x i64> %mask) {
-; GENERIC-LABEL: test_masked_8xdouble_perm_imm_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %zmm2, %zmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermpd {{.*#+}} zmm1 {%k1} = zmm0[3,0,0,2,7,4,4,6] sched: [1:1.00]
-; GENERIC-NEXT:    vmovapd %zmm1, %zmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_8xdouble_perm_imm_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %zmm2, %zmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermpd {{.*#+}} zmm1 {%k1} = zmm0[3,0,0,2,7,4,4,6] sched: [3:1.00]
-; SKX-NEXT:    vmovapd %zmm1, %zmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x double> %vec, <8 x double> undef, <8 x i32> <i32 3, i32 0, i32 0, i32 2, i32 7, i32 4, i32 4, i32 6>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x double> %shuf, <8 x double> %vec2
-  ret <8 x double> %res
-}
-
-define <8 x double> @test_masked_z_8xdouble_perm_imm_mask1(<8 x double> %vec, <8 x i64> %mask) {
-; GENERIC-LABEL: test_masked_z_8xdouble_perm_imm_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermpd {{.*#+}} zmm0 {%k1} {z} = zmm0[3,0,0,2,7,4,4,6] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_8xdouble_perm_imm_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermpd {{.*#+}} zmm0 {%k1} {z} = zmm0[3,0,0,2,7,4,4,6] sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x double> %vec, <8 x double> undef, <8 x i32> <i32 3, i32 0, i32 0, i32 2, i32 7, i32 4, i32 4, i32 6>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x double> %shuf, <8 x double> zeroinitializer
-  ret <8 x double> %res
-}
-define <8 x double> @test_masked_8xdouble_perm_mask2(<8 x double> %vec, <8 x double> %vec2, <8 x i64> %mask) {
-; GENERIC-LABEL: test_masked_8xdouble_perm_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovapd {{.*#+}} zmm3 = [7,5,5,5,3,5,1,7] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmq %zmm2, %zmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermpd %zmm0, %zmm3, %zmm1 {%k1} # sched: [1:1.00]
-; GENERIC-NEXT:    vmovapd %zmm1, %zmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_8xdouble_perm_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovapd {{.*#+}} zmm3 = [7,5,5,5,3,5,1,7] sched: [8:0.50]
-; SKX-NEXT:    vptestnmq %zmm2, %zmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermpd %zmm0, %zmm3, %zmm1 {%k1} # sched: [3:1.00]
-; SKX-NEXT:    vmovapd %zmm1, %zmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x double> %vec, <8 x double> undef, <8 x i32> <i32 7, i32 5, i32 5, i32 5, i32 3, i32 5, i32 1, i32 7>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x double> %shuf, <8 x double> %vec2
-  ret <8 x double> %res
-}
-
-define <8 x double> @test_masked_z_8xdouble_perm_mask2(<8 x double> %vec, <8 x i64> %mask) {
-; GENERIC-LABEL: test_masked_z_8xdouble_perm_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovapd {{.*#+}} zmm2 = [7,5,5,5,3,5,1,7] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmq %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermpd %zmm0, %zmm2, %zmm0 {%k1} {z} # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_8xdouble_perm_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovapd {{.*#+}} zmm2 = [7,5,5,5,3,5,1,7] sched: [8:0.50]
-; SKX-NEXT:    vptestnmq %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermpd %zmm0, %zmm2, %zmm0 {%k1} {z} # sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x double> %vec, <8 x double> undef, <8 x i32> <i32 7, i32 5, i32 5, i32 5, i32 3, i32 5, i32 1, i32 7>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x double> %shuf, <8 x double> zeroinitializer
-  ret <8 x double> %res
-}
-define <8 x double> @test_8xdouble_perm_imm_mask3(<8 x double> %vec) {
-; GENERIC-LABEL: test_8xdouble_perm_imm_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpermpd {{.*#+}} zmm0 = zmm0[1,3,3,0,5,7,7,4] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xdouble_perm_imm_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpermpd {{.*#+}} zmm0 = zmm0[1,3,3,0,5,7,7,4] sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %res = shufflevector <8 x double> %vec, <8 x double> undef, <8 x i32> <i32 1, i32 3, i32 3, i32 0, i32 5, i32 7, i32 7, i32 4>
-  ret <8 x double> %res
-}
-define <8 x double> @test_masked_8xdouble_perm_imm_mask3(<8 x double> %vec, <8 x double> %vec2, <8 x i64> %mask) {
-; GENERIC-LABEL: test_masked_8xdouble_perm_imm_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %zmm2, %zmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermpd {{.*#+}} zmm1 {%k1} = zmm0[1,3,3,0,5,7,7,4] sched: [1:1.00]
-; GENERIC-NEXT:    vmovapd %zmm1, %zmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_8xdouble_perm_imm_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %zmm2, %zmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermpd {{.*#+}} zmm1 {%k1} = zmm0[1,3,3,0,5,7,7,4] sched: [3:1.00]
-; SKX-NEXT:    vmovapd %zmm1, %zmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x double> %vec, <8 x double> undef, <8 x i32> <i32 1, i32 3, i32 3, i32 0, i32 5, i32 7, i32 7, i32 4>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x double> %shuf, <8 x double> %vec2
-  ret <8 x double> %res
-}
-
-define <8 x double> @test_masked_z_8xdouble_perm_imm_mask3(<8 x double> %vec, <8 x i64> %mask) {
-; GENERIC-LABEL: test_masked_z_8xdouble_perm_imm_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermpd {{.*#+}} zmm0 {%k1} {z} = zmm0[1,3,3,0,5,7,7,4] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_8xdouble_perm_imm_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermpd {{.*#+}} zmm0 {%k1} {z} = zmm0[1,3,3,0,5,7,7,4] sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x double> %vec, <8 x double> undef, <8 x i32> <i32 1, i32 3, i32 3, i32 0, i32 5, i32 7, i32 7, i32 4>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x double> %shuf, <8 x double> zeroinitializer
-  ret <8 x double> %res
-}
-define <8 x double> @test_masked_8xdouble_perm_mask4(<8 x double> %vec, <8 x double> %vec2, <8 x i64> %mask) {
-; GENERIC-LABEL: test_masked_8xdouble_perm_mask4:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovapd {{.*#+}} zmm3 = [3,5,3,4,6,5,7,1] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmq %zmm2, %zmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermpd %zmm0, %zmm3, %zmm1 {%k1} # sched: [1:1.00]
-; GENERIC-NEXT:    vmovapd %zmm1, %zmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_8xdouble_perm_mask4:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovapd {{.*#+}} zmm3 = [3,5,3,4,6,5,7,1] sched: [8:0.50]
-; SKX-NEXT:    vptestnmq %zmm2, %zmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermpd %zmm0, %zmm3, %zmm1 {%k1} # sched: [3:1.00]
-; SKX-NEXT:    vmovapd %zmm1, %zmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x double> %vec, <8 x double> undef, <8 x i32> <i32 3, i32 5, i32 3, i32 4, i32 6, i32 5, i32 7, i32 1>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x double> %shuf, <8 x double> %vec2
-  ret <8 x double> %res
-}
-
-define <8 x double> @test_masked_z_8xdouble_perm_mask4(<8 x double> %vec, <8 x i64> %mask) {
-; GENERIC-LABEL: test_masked_z_8xdouble_perm_mask4:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovapd {{.*#+}} zmm2 = [3,5,3,4,6,5,7,1] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmq %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermpd %zmm0, %zmm2, %zmm0 {%k1} {z} # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_8xdouble_perm_mask4:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovapd {{.*#+}} zmm2 = [3,5,3,4,6,5,7,1] sched: [8:0.50]
-; SKX-NEXT:    vptestnmq %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermpd %zmm0, %zmm2, %zmm0 {%k1} {z} # sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x double> %vec, <8 x double> undef, <8 x i32> <i32 3, i32 5, i32 3, i32 4, i32 6, i32 5, i32 7, i32 1>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x double> %shuf, <8 x double> zeroinitializer
-  ret <8 x double> %res
-}
-define <8 x double> @test_masked_8xdouble_perm_imm_mask5(<8 x double> %vec, <8 x double> %vec2, <8 x i64> %mask) {
-; GENERIC-LABEL: test_masked_8xdouble_perm_imm_mask5:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %zmm2, %zmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermpd {{.*#+}} zmm1 {%k1} = zmm0[3,3,2,3,7,7,6,7] sched: [1:1.00]
-; GENERIC-NEXT:    vmovapd %zmm1, %zmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_8xdouble_perm_imm_mask5:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %zmm2, %zmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermpd {{.*#+}} zmm1 {%k1} = zmm0[3,3,2,3,7,7,6,7] sched: [3:1.00]
-; SKX-NEXT:    vmovapd %zmm1, %zmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x double> %vec, <8 x double> undef, <8 x i32> <i32 3, i32 3, i32 2, i32 3, i32 7, i32 7, i32 6, i32 7>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x double> %shuf, <8 x double> %vec2
-  ret <8 x double> %res
-}
-
-define <8 x double> @test_masked_z_8xdouble_perm_imm_mask5(<8 x double> %vec, <8 x i64> %mask) {
-; GENERIC-LABEL: test_masked_z_8xdouble_perm_imm_mask5:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermpd {{.*#+}} zmm0 {%k1} {z} = zmm0[3,3,2,3,7,7,6,7] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_8xdouble_perm_imm_mask5:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermpd {{.*#+}} zmm0 {%k1} {z} = zmm0[3,3,2,3,7,7,6,7] sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x double> %vec, <8 x double> undef, <8 x i32> <i32 3, i32 3, i32 2, i32 3, i32 7, i32 7, i32 6, i32 7>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x double> %shuf, <8 x double> zeroinitializer
-  ret <8 x double> %res
-}
-define <8 x double> @test_8xdouble_perm_mask6(<8 x double> %vec) {
-; GENERIC-LABEL: test_8xdouble_perm_mask6:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovaps {{.*#+}} zmm1 = [2,7,6,4,0,0,0,2] sched: [7:0.50]
-; GENERIC-NEXT:    vpermpd %zmm0, %zmm1, %zmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xdouble_perm_mask6:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovaps {{.*#+}} zmm1 = [2,7,6,4,0,0,0,2] sched: [8:0.50]
-; SKX-NEXT:    vpermpd %zmm0, %zmm1, %zmm0 # sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %res = shufflevector <8 x double> %vec, <8 x double> undef, <8 x i32> <i32 2, i32 7, i32 6, i32 4, i32 0, i32 0, i32 0, i32 2>
-  ret <8 x double> %res
-}
-define <8 x double> @test_masked_8xdouble_perm_mask6(<8 x double> %vec, <8 x double> %vec2, <8 x i64> %mask) {
-; GENERIC-LABEL: test_masked_8xdouble_perm_mask6:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovapd {{.*#+}} zmm3 = [2,7,6,4,0,0,0,2] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmq %zmm2, %zmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermpd %zmm0, %zmm3, %zmm1 {%k1} # sched: [1:1.00]
-; GENERIC-NEXT:    vmovapd %zmm1, %zmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_8xdouble_perm_mask6:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovapd {{.*#+}} zmm3 = [2,7,6,4,0,0,0,2] sched: [8:0.50]
-; SKX-NEXT:    vptestnmq %zmm2, %zmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermpd %zmm0, %zmm3, %zmm1 {%k1} # sched: [3:1.00]
-; SKX-NEXT:    vmovapd %zmm1, %zmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x double> %vec, <8 x double> undef, <8 x i32> <i32 2, i32 7, i32 6, i32 4, i32 0, i32 0, i32 0, i32 2>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x double> %shuf, <8 x double> %vec2
-  ret <8 x double> %res
-}
-
-define <8 x double> @test_masked_z_8xdouble_perm_mask6(<8 x double> %vec, <8 x i64> %mask) {
-; GENERIC-LABEL: test_masked_z_8xdouble_perm_mask6:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovapd {{.*#+}} zmm2 = [2,7,6,4,0,0,0,2] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmq %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermpd %zmm0, %zmm2, %zmm0 {%k1} {z} # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_8xdouble_perm_mask6:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovapd {{.*#+}} zmm2 = [2,7,6,4,0,0,0,2] sched: [8:0.50]
-; SKX-NEXT:    vptestnmq %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermpd %zmm0, %zmm2, %zmm0 {%k1} {z} # sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x double> %vec, <8 x double> undef, <8 x i32> <i32 2, i32 7, i32 6, i32 4, i32 0, i32 0, i32 0, i32 2>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x double> %shuf, <8 x double> zeroinitializer
-  ret <8 x double> %res
-}
-define <8 x double> @test_masked_8xdouble_perm_imm_mask7(<8 x double> %vec, <8 x double> %vec2, <8 x i64> %mask) {
-; GENERIC-LABEL: test_masked_8xdouble_perm_imm_mask7:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %zmm2, %zmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermpd {{.*#+}} zmm1 {%k1} = zmm0[3,1,3,2,7,5,7,6] sched: [1:1.00]
-; GENERIC-NEXT:    vmovapd %zmm1, %zmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_8xdouble_perm_imm_mask7:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %zmm2, %zmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermpd {{.*#+}} zmm1 {%k1} = zmm0[3,1,3,2,7,5,7,6] sched: [3:1.00]
-; SKX-NEXT:    vmovapd %zmm1, %zmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x double> %vec, <8 x double> undef, <8 x i32> <i32 3, i32 1, i32 3, i32 2, i32 7, i32 5, i32 7, i32 6>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x double> %shuf, <8 x double> %vec2
-  ret <8 x double> %res
-}
-
-define <8 x double> @test_masked_z_8xdouble_perm_imm_mask7(<8 x double> %vec, <8 x i64> %mask) {
-; GENERIC-LABEL: test_masked_z_8xdouble_perm_imm_mask7:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermpd {{.*#+}} zmm0 {%k1} {z} = zmm0[3,1,3,2,7,5,7,6] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_8xdouble_perm_imm_mask7:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermpd {{.*#+}} zmm0 {%k1} {z} = zmm0[3,1,3,2,7,5,7,6] sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x double> %vec, <8 x double> undef, <8 x i32> <i32 3, i32 1, i32 3, i32 2, i32 7, i32 5, i32 7, i32 6>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x double> %shuf, <8 x double> zeroinitializer
-  ret <8 x double> %res
-}
-define <8 x double> @test_8xdouble_perm_mem_mask0(<8 x double>* %vp) {
-; GENERIC-LABEL: test_8xdouble_perm_mem_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovaps {{.*#+}} zmm0 = [0,3,4,0,4,2,0,1] sched: [7:0.50]
-; GENERIC-NEXT:    vpermpd (%rdi), %zmm0, %zmm0 # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xdouble_perm_mem_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovaps {{.*#+}} zmm0 = [0,3,4,0,4,2,0,1] sched: [8:0.50]
-; SKX-NEXT:    vpermpd (%rdi), %zmm0, %zmm0 # sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <8 x double>, <8 x double>* %vp
-  %res = shufflevector <8 x double> %vec, <8 x double> undef, <8 x i32> <i32 0, i32 3, i32 4, i32 0, i32 4, i32 2, i32 0, i32 1>
-  ret <8 x double> %res
-}
-define <8 x double> @test_masked_8xdouble_perm_mem_mask0(<8 x double>* %vp, <8 x double> %vec2, <8 x i64> %mask) {
-; GENERIC-LABEL: test_masked_8xdouble_perm_mem_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovapd {{.*#+}} zmm2 = [0,3,4,0,4,2,0,1] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmq %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermpd (%rdi), %zmm2, %zmm0 {%k1} # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_8xdouble_perm_mem_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovapd {{.*#+}} zmm2 = [0,3,4,0,4,2,0,1] sched: [8:0.50]
-; SKX-NEXT:    vptestnmq %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermpd (%rdi), %zmm2, %zmm0 {%k1} # sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <8 x double>, <8 x double>* %vp
-  %shuf = shufflevector <8 x double> %vec, <8 x double> undef, <8 x i32> <i32 0, i32 3, i32 4, i32 0, i32 4, i32 2, i32 0, i32 1>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x double> %shuf, <8 x double> %vec2
-  ret <8 x double> %res
-}
-
-define <8 x double> @test_masked_z_8xdouble_perm_mem_mask0(<8 x double>* %vp, <8 x i64> %mask) {
-; GENERIC-LABEL: test_masked_z_8xdouble_perm_mem_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovapd {{.*#+}} zmm1 = [0,3,4,0,4,2,0,1] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmq %zmm0, %zmm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermpd (%rdi), %zmm1, %zmm0 {%k1} {z} # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_8xdouble_perm_mem_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovapd {{.*#+}} zmm1 = [0,3,4,0,4,2,0,1] sched: [8:0.50]
-; SKX-NEXT:    vptestnmq %zmm0, %zmm0, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermpd (%rdi), %zmm1, %zmm0 {%k1} {z} # sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <8 x double>, <8 x double>* %vp
-  %shuf = shufflevector <8 x double> %vec, <8 x double> undef, <8 x i32> <i32 0, i32 3, i32 4, i32 0, i32 4, i32 2, i32 0, i32 1>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x double> %shuf, <8 x double> zeroinitializer
-  ret <8 x double> %res
-}
-
-define <8 x double> @test_masked_8xdouble_perm_imm_mem_mask1(<8 x double>* %vp, <8 x double> %vec2, <8 x i64> %mask) {
-; GENERIC-LABEL: test_masked_8xdouble_perm_imm_mem_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermpd {{.*#+}} zmm0 {%k1} = mem[0,2,0,3,4,6,4,7] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_8xdouble_perm_imm_mem_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermpd {{.*#+}} zmm0 {%k1} = mem[0,2,0,3,4,6,4,7] sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <8 x double>, <8 x double>* %vp
-  %shuf = shufflevector <8 x double> %vec, <8 x double> undef, <8 x i32> <i32 0, i32 2, i32 0, i32 3, i32 4, i32 6, i32 4, i32 7>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x double> %shuf, <8 x double> %vec2
-  ret <8 x double> %res
-}
-
-define <8 x double> @test_masked_z_8xdouble_perm_imm_mem_mask1(<8 x double>* %vp, <8 x i64> %mask) {
-; GENERIC-LABEL: test_masked_z_8xdouble_perm_imm_mem_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %zmm0, %zmm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermpd {{.*#+}} zmm0 {%k1} {z} = mem[0,2,0,3,4,6,4,7] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_8xdouble_perm_imm_mem_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %zmm0, %zmm0, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermpd {{.*#+}} zmm0 {%k1} {z} = mem[0,2,0,3,4,6,4,7] sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <8 x double>, <8 x double>* %vp
-  %shuf = shufflevector <8 x double> %vec, <8 x double> undef, <8 x i32> <i32 0, i32 2, i32 0, i32 3, i32 4, i32 6, i32 4, i32 7>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x double> %shuf, <8 x double> zeroinitializer
-  ret <8 x double> %res
-}
-
-define <8 x double> @test_masked_8xdouble_perm_mem_mask2(<8 x double>* %vp, <8 x double> %vec2, <8 x i64> %mask) {
-; GENERIC-LABEL: test_masked_8xdouble_perm_mem_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovapd {{.*#+}} zmm2 = [6,7,2,7,7,6,2,5] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmq %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermpd (%rdi), %zmm2, %zmm0 {%k1} # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_8xdouble_perm_mem_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovapd {{.*#+}} zmm2 = [6,7,2,7,7,6,2,5] sched: [8:0.50]
-; SKX-NEXT:    vptestnmq %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermpd (%rdi), %zmm2, %zmm0 {%k1} # sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <8 x double>, <8 x double>* %vp
-  %shuf = shufflevector <8 x double> %vec, <8 x double> undef, <8 x i32> <i32 6, i32 7, i32 2, i32 7, i32 7, i32 6, i32 2, i32 5>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x double> %shuf, <8 x double> %vec2
-  ret <8 x double> %res
-}
-
-define <8 x double> @test_masked_z_8xdouble_perm_mem_mask2(<8 x double>* %vp, <8 x i64> %mask) {
-; GENERIC-LABEL: test_masked_z_8xdouble_perm_mem_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovapd {{.*#+}} zmm1 = [6,7,2,7,7,6,2,5] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmq %zmm0, %zmm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermpd (%rdi), %zmm1, %zmm0 {%k1} {z} # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_8xdouble_perm_mem_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovapd {{.*#+}} zmm1 = [6,7,2,7,7,6,2,5] sched: [8:0.50]
-; SKX-NEXT:    vptestnmq %zmm0, %zmm0, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermpd (%rdi), %zmm1, %zmm0 {%k1} {z} # sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <8 x double>, <8 x double>* %vp
-  %shuf = shufflevector <8 x double> %vec, <8 x double> undef, <8 x i32> <i32 6, i32 7, i32 2, i32 7, i32 7, i32 6, i32 2, i32 5>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x double> %shuf, <8 x double> zeroinitializer
-  ret <8 x double> %res
-}
-
-define <8 x double> @test_8xdouble_perm_imm_mem_mask3(<8 x double>* %vp) {
-; GENERIC-LABEL: test_8xdouble_perm_imm_mem_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpermpd {{.*#+}} zmm0 = mem[2,1,1,0,6,5,5,4] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xdouble_perm_imm_mem_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpermpd {{.*#+}} zmm0 = mem[2,1,1,0,6,5,5,4] sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <8 x double>, <8 x double>* %vp
-  %res = shufflevector <8 x double> %vec, <8 x double> undef, <8 x i32> <i32 2, i32 1, i32 1, i32 0, i32 6, i32 5, i32 5, i32 4>
-  ret <8 x double> %res
-}
-define <8 x double> @test_masked_8xdouble_perm_imm_mem_mask3(<8 x double>* %vp, <8 x double> %vec2, <8 x i64> %mask) {
-; GENERIC-LABEL: test_masked_8xdouble_perm_imm_mem_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermpd {{.*#+}} zmm0 {%k1} = mem[2,1,1,0,6,5,5,4] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_8xdouble_perm_imm_mem_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermpd {{.*#+}} zmm0 {%k1} = mem[2,1,1,0,6,5,5,4] sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <8 x double>, <8 x double>* %vp
-  %shuf = shufflevector <8 x double> %vec, <8 x double> undef, <8 x i32> <i32 2, i32 1, i32 1, i32 0, i32 6, i32 5, i32 5, i32 4>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x double> %shuf, <8 x double> %vec2
-  ret <8 x double> %res
-}
-
-define <8 x double> @test_masked_z_8xdouble_perm_imm_mem_mask3(<8 x double>* %vp, <8 x i64> %mask) {
-; GENERIC-LABEL: test_masked_z_8xdouble_perm_imm_mem_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %zmm0, %zmm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermpd {{.*#+}} zmm0 {%k1} {z} = mem[2,1,1,0,6,5,5,4] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_8xdouble_perm_imm_mem_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %zmm0, %zmm0, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermpd {{.*#+}} zmm0 {%k1} {z} = mem[2,1,1,0,6,5,5,4] sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <8 x double>, <8 x double>* %vp
-  %shuf = shufflevector <8 x double> %vec, <8 x double> undef, <8 x i32> <i32 2, i32 1, i32 1, i32 0, i32 6, i32 5, i32 5, i32 4>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x double> %shuf, <8 x double> zeroinitializer
-  ret <8 x double> %res
-}
-
-define <8 x double> @test_masked_8xdouble_perm_mem_mask4(<8 x double>* %vp, <8 x double> %vec2, <8 x i64> %mask) {
-; GENERIC-LABEL: test_masked_8xdouble_perm_mem_mask4:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovapd {{.*#+}} zmm2 = [1,1,3,5,6,0,6,0] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmq %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermpd (%rdi), %zmm2, %zmm0 {%k1} # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_8xdouble_perm_mem_mask4:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovapd {{.*#+}} zmm2 = [1,1,3,5,6,0,6,0] sched: [8:0.50]
-; SKX-NEXT:    vptestnmq %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermpd (%rdi), %zmm2, %zmm0 {%k1} # sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <8 x double>, <8 x double>* %vp
-  %shuf = shufflevector <8 x double> %vec, <8 x double> undef, <8 x i32> <i32 1, i32 1, i32 3, i32 5, i32 6, i32 0, i32 6, i32 0>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x double> %shuf, <8 x double> %vec2
-  ret <8 x double> %res
-}
-
-define <8 x double> @test_masked_z_8xdouble_perm_mem_mask4(<8 x double>* %vp, <8 x i64> %mask) {
-; GENERIC-LABEL: test_masked_z_8xdouble_perm_mem_mask4:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovapd {{.*#+}} zmm1 = [1,1,3,5,6,0,6,0] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmq %zmm0, %zmm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermpd (%rdi), %zmm1, %zmm0 {%k1} {z} # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_8xdouble_perm_mem_mask4:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovapd {{.*#+}} zmm1 = [1,1,3,5,6,0,6,0] sched: [8:0.50]
-; SKX-NEXT:    vptestnmq %zmm0, %zmm0, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermpd (%rdi), %zmm1, %zmm0 {%k1} {z} # sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <8 x double>, <8 x double>* %vp
-  %shuf = shufflevector <8 x double> %vec, <8 x double> undef, <8 x i32> <i32 1, i32 1, i32 3, i32 5, i32 6, i32 0, i32 6, i32 0>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x double> %shuf, <8 x double> zeroinitializer
-  ret <8 x double> %res
-}
-
-define <8 x double> @test_masked_8xdouble_perm_imm_mem_mask5(<8 x double>* %vp, <8 x double> %vec2, <8 x i64> %mask) {
-; GENERIC-LABEL: test_masked_8xdouble_perm_imm_mem_mask5:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermpd {{.*#+}} zmm0 {%k1} = mem[2,2,2,3,6,6,6,7] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_8xdouble_perm_imm_mem_mask5:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermpd {{.*#+}} zmm0 {%k1} = mem[2,2,2,3,6,6,6,7] sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <8 x double>, <8 x double>* %vp
-  %shuf = shufflevector <8 x double> %vec, <8 x double> undef, <8 x i32> <i32 2, i32 2, i32 2, i32 3, i32 6, i32 6, i32 6, i32 7>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x double> %shuf, <8 x double> %vec2
-  ret <8 x double> %res
-}
-
-define <8 x double> @test_masked_z_8xdouble_perm_imm_mem_mask5(<8 x double>* %vp, <8 x i64> %mask) {
-; GENERIC-LABEL: test_masked_z_8xdouble_perm_imm_mem_mask5:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %zmm0, %zmm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermpd {{.*#+}} zmm0 {%k1} {z} = mem[2,2,2,3,6,6,6,7] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_8xdouble_perm_imm_mem_mask5:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %zmm0, %zmm0, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermpd {{.*#+}} zmm0 {%k1} {z} = mem[2,2,2,3,6,6,6,7] sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <8 x double>, <8 x double>* %vp
-  %shuf = shufflevector <8 x double> %vec, <8 x double> undef, <8 x i32> <i32 2, i32 2, i32 2, i32 3, i32 6, i32 6, i32 6, i32 7>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x double> %shuf, <8 x double> zeroinitializer
-  ret <8 x double> %res
-}
-
-define <8 x double> @test_8xdouble_perm_mem_mask6(<8 x double>* %vp) {
-; GENERIC-LABEL: test_8xdouble_perm_mem_mask6:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovaps {{.*#+}} zmm0 = [2,4,0,4,6,1,2,5] sched: [7:0.50]
-; GENERIC-NEXT:    vpermpd (%rdi), %zmm0, %zmm0 # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xdouble_perm_mem_mask6:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovaps {{.*#+}} zmm0 = [2,4,0,4,6,1,2,5] sched: [8:0.50]
-; SKX-NEXT:    vpermpd (%rdi), %zmm0, %zmm0 # sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <8 x double>, <8 x double>* %vp
-  %res = shufflevector <8 x double> %vec, <8 x double> undef, <8 x i32> <i32 2, i32 4, i32 0, i32 4, i32 6, i32 1, i32 2, i32 5>
-  ret <8 x double> %res
-}
-define <8 x double> @test_masked_8xdouble_perm_mem_mask6(<8 x double>* %vp, <8 x double> %vec2, <8 x i64> %mask) {
-; GENERIC-LABEL: test_masked_8xdouble_perm_mem_mask6:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovapd {{.*#+}} zmm2 = [2,4,0,4,6,1,2,5] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmq %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermpd (%rdi), %zmm2, %zmm0 {%k1} # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_8xdouble_perm_mem_mask6:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovapd {{.*#+}} zmm2 = [2,4,0,4,6,1,2,5] sched: [8:0.50]
-; SKX-NEXT:    vptestnmq %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermpd (%rdi), %zmm2, %zmm0 {%k1} # sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <8 x double>, <8 x double>* %vp
-  %shuf = shufflevector <8 x double> %vec, <8 x double> undef, <8 x i32> <i32 2, i32 4, i32 0, i32 4, i32 6, i32 1, i32 2, i32 5>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x double> %shuf, <8 x double> %vec2
-  ret <8 x double> %res
-}
-
-define <8 x double> @test_masked_z_8xdouble_perm_mem_mask6(<8 x double>* %vp, <8 x i64> %mask) {
-; GENERIC-LABEL: test_masked_z_8xdouble_perm_mem_mask6:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovapd {{.*#+}} zmm1 = [2,4,0,4,6,1,2,5] sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmq %zmm0, %zmm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermpd (%rdi), %zmm1, %zmm0 {%k1} {z} # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_8xdouble_perm_mem_mask6:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovapd {{.*#+}} zmm1 = [2,4,0,4,6,1,2,5] sched: [8:0.50]
-; SKX-NEXT:    vptestnmq %zmm0, %zmm0, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermpd (%rdi), %zmm1, %zmm0 {%k1} {z} # sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <8 x double>, <8 x double>* %vp
-  %shuf = shufflevector <8 x double> %vec, <8 x double> undef, <8 x i32> <i32 2, i32 4, i32 0, i32 4, i32 6, i32 1, i32 2, i32 5>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x double> %shuf, <8 x double> zeroinitializer
-  ret <8 x double> %res
-}
-
-define <8 x double> @test_masked_8xdouble_perm_imm_mem_mask7(<8 x double>* %vp, <8 x double> %vec2, <8 x i64> %mask) {
-; GENERIC-LABEL: test_masked_8xdouble_perm_imm_mem_mask7:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermpd {{.*#+}} zmm0 {%k1} = mem[0,3,2,0,4,7,6,4] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_8xdouble_perm_imm_mem_mask7:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermpd {{.*#+}} zmm0 {%k1} = mem[0,3,2,0,4,7,6,4] sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <8 x double>, <8 x double>* %vp
-  %shuf = shufflevector <8 x double> %vec, <8 x double> undef, <8 x i32> <i32 0, i32 3, i32 2, i32 0, i32 4, i32 7, i32 6, i32 4>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x double> %shuf, <8 x double> %vec2
-  ret <8 x double> %res
-}
-
-define <8 x double> @test_masked_z_8xdouble_perm_imm_mem_mask7(<8 x double>* %vp, <8 x i64> %mask) {
-; GENERIC-LABEL: test_masked_z_8xdouble_perm_imm_mem_mask7:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %zmm0, %zmm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpermpd {{.*#+}} zmm0 {%k1} {z} = mem[0,3,2,0,4,7,6,4] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_8xdouble_perm_imm_mem_mask7:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %zmm0, %zmm0, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpermpd {{.*#+}} zmm0 {%k1} {z} = mem[0,3,2,0,4,7,6,4] sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <8 x double>, <8 x double>* %vp
-  %shuf = shufflevector <8 x double> %vec, <8 x double> undef, <8 x i32> <i32 0, i32 3, i32 2, i32 0, i32 4, i32 7, i32 6, i32 4>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x double> %shuf, <8 x double> zeroinitializer
-  ret <8 x double> %res
-}
-
-define <16 x i8> @test_16xi8_perm_mask0(<16 x i8> %vec) {
-; GENERIC-LABEL: test_16xi8_perm_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[8,6,12,4,7,9,14,8,4,12,9,4,14,15,12,14] sched: [7:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_16xi8_perm_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[8,6,12,4,7,9,14,8,4,12,9,4,14,15,12,14] sched: [7:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %res = shufflevector <16 x i8> %vec, <16 x i8> undef, <16 x i32> <i32 8, i32 6, i32 12, i32 4, i32 7, i32 9, i32 14, i32 8, i32 4, i32 12, i32 9, i32 4, i32 14, i32 15, i32 12, i32 14>
-  ret <16 x i8> %res
-}
-define <16 x i8> @test_masked_16xi8_perm_mask0(<16 x i8> %vec, <16 x i8> %vec2, <16 x i8> %mask) {
-; GENERIC-LABEL: test_masked_16xi8_perm_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmb %xmm2, %xmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufb {{.*#+}} xmm1 {%k1} = xmm0[8,6,12,4,7,9,14,8,4,12,9,4,14,15,12,14] sched: [7:0.50]
-; GENERIC-NEXT:    vmovdqa %xmm1, %xmm0 # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_16xi8_perm_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmb %xmm2, %xmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufb {{.*#+}} xmm1 {%k1} = xmm0[8,6,12,4,7,9,14,8,4,12,9,4,14,15,12,14] sched: [7:1.00]
-; SKX-NEXT:    vmovdqa %xmm1, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <16 x i8> %vec, <16 x i8> undef, <16 x i32> <i32 8, i32 6, i32 12, i32 4, i32 7, i32 9, i32 14, i32 8, i32 4, i32 12, i32 9, i32 4, i32 14, i32 15, i32 12, i32 14>
-  %cmp = icmp eq <16 x i8> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x i8> %shuf, <16 x i8> %vec2
-  ret <16 x i8> %res
-}
-
-define <16 x i8> @test_masked_z_16xi8_perm_mask0(<16 x i8> %vec, <16 x i8> %mask) {
-; GENERIC-LABEL: test_masked_z_16xi8_perm_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmb %xmm1, %xmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufb {{.*#+}} xmm0 {%k1} {z} = xmm0[8,6,12,4,7,9,14,8,4,12,9,4,14,15,12,14] sched: [7:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_16xi8_perm_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmb %xmm1, %xmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufb {{.*#+}} xmm0 {%k1} {z} = xmm0[8,6,12,4,7,9,14,8,4,12,9,4,14,15,12,14] sched: [7:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <16 x i8> %vec, <16 x i8> undef, <16 x i32> <i32 8, i32 6, i32 12, i32 4, i32 7, i32 9, i32 14, i32 8, i32 4, i32 12, i32 9, i32 4, i32 14, i32 15, i32 12, i32 14>
-  %cmp = icmp eq <16 x i8> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x i8> %shuf, <16 x i8> zeroinitializer
-  ret <16 x i8> %res
-}
-define <16 x i8> @test_masked_16xi8_perm_mask1(<16 x i8> %vec, <16 x i8> %vec2, <16 x i8> %mask) {
-; GENERIC-LABEL: test_masked_16xi8_perm_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmb %xmm2, %xmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufb {{.*#+}} xmm1 {%k1} = xmm0[4,11,14,10,7,1,6,9,14,15,7,13,4,12,8,0] sched: [7:0.50]
-; GENERIC-NEXT:    vmovdqa %xmm1, %xmm0 # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_16xi8_perm_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmb %xmm2, %xmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufb {{.*#+}} xmm1 {%k1} = xmm0[4,11,14,10,7,1,6,9,14,15,7,13,4,12,8,0] sched: [7:1.00]
-; SKX-NEXT:    vmovdqa %xmm1, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <16 x i8> %vec, <16 x i8> undef, <16 x i32> <i32 4, i32 11, i32 14, i32 10, i32 7, i32 1, i32 6, i32 9, i32 14, i32 15, i32 7, i32 13, i32 4, i32 12, i32 8, i32 0>
-  %cmp = icmp eq <16 x i8> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x i8> %shuf, <16 x i8> %vec2
-  ret <16 x i8> %res
-}
-
-define <16 x i8> @test_masked_z_16xi8_perm_mask1(<16 x i8> %vec, <16 x i8> %mask) {
-; GENERIC-LABEL: test_masked_z_16xi8_perm_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmb %xmm1, %xmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufb {{.*#+}} xmm0 {%k1} {z} = xmm0[4,11,14,10,7,1,6,9,14,15,7,13,4,12,8,0] sched: [7:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_16xi8_perm_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmb %xmm1, %xmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufb {{.*#+}} xmm0 {%k1} {z} = xmm0[4,11,14,10,7,1,6,9,14,15,7,13,4,12,8,0] sched: [7:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <16 x i8> %vec, <16 x i8> undef, <16 x i32> <i32 4, i32 11, i32 14, i32 10, i32 7, i32 1, i32 6, i32 9, i32 14, i32 15, i32 7, i32 13, i32 4, i32 12, i32 8, i32 0>
-  %cmp = icmp eq <16 x i8> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x i8> %shuf, <16 x i8> zeroinitializer
-  ret <16 x i8> %res
-}
-define <16 x i8> @test_masked_16xi8_perm_mask2(<16 x i8> %vec, <16 x i8> %vec2, <16 x i8> %mask) {
-; GENERIC-LABEL: test_masked_16xi8_perm_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmb %xmm2, %xmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufb {{.*#+}} xmm1 {%k1} = xmm0[11,6,13,10,0,7,13,3,5,13,3,9,3,15,12,7] sched: [7:0.50]
-; GENERIC-NEXT:    vmovdqa %xmm1, %xmm0 # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_16xi8_perm_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmb %xmm2, %xmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufb {{.*#+}} xmm1 {%k1} = xmm0[11,6,13,10,0,7,13,3,5,13,3,9,3,15,12,7] sched: [7:1.00]
-; SKX-NEXT:    vmovdqa %xmm1, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <16 x i8> %vec, <16 x i8> undef, <16 x i32> <i32 11, i32 6, i32 13, i32 10, i32 0, i32 7, i32 13, i32 3, i32 5, i32 13, i32 3, i32 9, i32 3, i32 15, i32 12, i32 7>
-  %cmp = icmp eq <16 x i8> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x i8> %shuf, <16 x i8> %vec2
-  ret <16 x i8> %res
-}
-
-define <16 x i8> @test_masked_z_16xi8_perm_mask2(<16 x i8> %vec, <16 x i8> %mask) {
-; GENERIC-LABEL: test_masked_z_16xi8_perm_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmb %xmm1, %xmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufb {{.*#+}} xmm0 {%k1} {z} = xmm0[11,6,13,10,0,7,13,3,5,13,3,9,3,15,12,7] sched: [7:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_16xi8_perm_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmb %xmm1, %xmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufb {{.*#+}} xmm0 {%k1} {z} = xmm0[11,6,13,10,0,7,13,3,5,13,3,9,3,15,12,7] sched: [7:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <16 x i8> %vec, <16 x i8> undef, <16 x i32> <i32 11, i32 6, i32 13, i32 10, i32 0, i32 7, i32 13, i32 3, i32 5, i32 13, i32 3, i32 9, i32 3, i32 15, i32 12, i32 7>
-  %cmp = icmp eq <16 x i8> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x i8> %shuf, <16 x i8> zeroinitializer
-  ret <16 x i8> %res
-}
-define <16 x i8> @test_16xi8_perm_mask3(<16 x i8> %vec) {
-; GENERIC-LABEL: test_16xi8_perm_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[1,5,8,14,1,8,11,8,13,8,15,9,9,7,9,6] sched: [7:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_16xi8_perm_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[1,5,8,14,1,8,11,8,13,8,15,9,9,7,9,6] sched: [7:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %res = shufflevector <16 x i8> %vec, <16 x i8> undef, <16 x i32> <i32 1, i32 5, i32 8, i32 14, i32 1, i32 8, i32 11, i32 8, i32 13, i32 8, i32 15, i32 9, i32 9, i32 7, i32 9, i32 6>
-  ret <16 x i8> %res
-}
-define <16 x i8> @test_masked_16xi8_perm_mask3(<16 x i8> %vec, <16 x i8> %vec2, <16 x i8> %mask) {
-; GENERIC-LABEL: test_masked_16xi8_perm_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmb %xmm2, %xmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufb {{.*#+}} xmm1 {%k1} = xmm0[1,5,8,14,1,8,11,8,13,8,15,9,9,7,9,6] sched: [7:0.50]
-; GENERIC-NEXT:    vmovdqa %xmm1, %xmm0 # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_16xi8_perm_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmb %xmm2, %xmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufb {{.*#+}} xmm1 {%k1} = xmm0[1,5,8,14,1,8,11,8,13,8,15,9,9,7,9,6] sched: [7:1.00]
-; SKX-NEXT:    vmovdqa %xmm1, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <16 x i8> %vec, <16 x i8> undef, <16 x i32> <i32 1, i32 5, i32 8, i32 14, i32 1, i32 8, i32 11, i32 8, i32 13, i32 8, i32 15, i32 9, i32 9, i32 7, i32 9, i32 6>
-  %cmp = icmp eq <16 x i8> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x i8> %shuf, <16 x i8> %vec2
-  ret <16 x i8> %res
-}
-
-define <16 x i8> @test_masked_z_16xi8_perm_mask3(<16 x i8> %vec, <16 x i8> %mask) {
-; GENERIC-LABEL: test_masked_z_16xi8_perm_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmb %xmm1, %xmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufb {{.*#+}} xmm0 {%k1} {z} = xmm0[1,5,8,14,1,8,11,8,13,8,15,9,9,7,9,6] sched: [7:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_16xi8_perm_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmb %xmm1, %xmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufb {{.*#+}} xmm0 {%k1} {z} = xmm0[1,5,8,14,1,8,11,8,13,8,15,9,9,7,9,6] sched: [7:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <16 x i8> %vec, <16 x i8> undef, <16 x i32> <i32 1, i32 5, i32 8, i32 14, i32 1, i32 8, i32 11, i32 8, i32 13, i32 8, i32 15, i32 9, i32 9, i32 7, i32 9, i32 6>
-  %cmp = icmp eq <16 x i8> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x i8> %shuf, <16 x i8> zeroinitializer
-  ret <16 x i8> %res
-}
-define <16 x i8> @test_16xi8_perm_mem_mask0(<16 x i8>* %vp) {
-; GENERIC-LABEL: test_16xi8_perm_mem_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovdqa (%rdi), %xmm0 # sched: [6:0.50]
-; GENERIC-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[9,10,7,1,12,14,14,13,14,14,8,6,11,4,12,13] sched: [7:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_16xi8_perm_mem_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovdqa (%rdi), %xmm0 # sched: [6:0.50]
-; SKX-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[9,10,7,1,12,14,14,13,14,14,8,6,11,4,12,13] sched: [7:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <16 x i8>, <16 x i8>* %vp
-  %res = shufflevector <16 x i8> %vec, <16 x i8> undef, <16 x i32> <i32 9, i32 10, i32 7, i32 1, i32 12, i32 14, i32 14, i32 13, i32 14, i32 14, i32 8, i32 6, i32 11, i32 4, i32 12, i32 13>
-  ret <16 x i8> %res
-}
-define <16 x i8> @test_masked_16xi8_perm_mem_mask0(<16 x i8>* %vp, <16 x i8> %vec2, <16 x i8> %mask) {
-; GENERIC-LABEL: test_masked_16xi8_perm_mem_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovdqa (%rdi), %xmm2 # sched: [6:0.50]
-; GENERIC-NEXT:    vptestnmb %xmm1, %xmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufb {{.*#+}} xmm0 {%k1} = xmm2[9,10,7,1,12,14,14,13,14,14,8,6,11,4,12,13] sched: [7:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_16xi8_perm_mem_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovdqa (%rdi), %xmm2 # sched: [6:0.50]
-; SKX-NEXT:    vptestnmb %xmm1, %xmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufb {{.*#+}} xmm0 {%k1} = xmm2[9,10,7,1,12,14,14,13,14,14,8,6,11,4,12,13] sched: [7:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <16 x i8>, <16 x i8>* %vp
-  %shuf = shufflevector <16 x i8> %vec, <16 x i8> undef, <16 x i32> <i32 9, i32 10, i32 7, i32 1, i32 12, i32 14, i32 14, i32 13, i32 14, i32 14, i32 8, i32 6, i32 11, i32 4, i32 12, i32 13>
-  %cmp = icmp eq <16 x i8> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x i8> %shuf, <16 x i8> %vec2
-  ret <16 x i8> %res
-}
-
-define <16 x i8> @test_masked_z_16xi8_perm_mem_mask0(<16 x i8>* %vp, <16 x i8> %mask) {
-; GENERIC-LABEL: test_masked_z_16xi8_perm_mem_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovdqa (%rdi), %xmm1 # sched: [6:0.50]
-; GENERIC-NEXT:    vptestnmb %xmm0, %xmm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufb {{.*#+}} xmm0 {%k1} {z} = xmm1[9,10,7,1,12,14,14,13,14,14,8,6,11,4,12,13] sched: [7:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_16xi8_perm_mem_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovdqa (%rdi), %xmm1 # sched: [6:0.50]
-; SKX-NEXT:    vptestnmb %xmm0, %xmm0, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufb {{.*#+}} xmm0 {%k1} {z} = xmm1[9,10,7,1,12,14,14,13,14,14,8,6,11,4,12,13] sched: [7:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <16 x i8>, <16 x i8>* %vp
-  %shuf = shufflevector <16 x i8> %vec, <16 x i8> undef, <16 x i32> <i32 9, i32 10, i32 7, i32 1, i32 12, i32 14, i32 14, i32 13, i32 14, i32 14, i32 8, i32 6, i32 11, i32 4, i32 12, i32 13>
-  %cmp = icmp eq <16 x i8> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x i8> %shuf, <16 x i8> zeroinitializer
-  ret <16 x i8> %res
-}
-
-define <16 x i8> @test_masked_16xi8_perm_mem_mask1(<16 x i8>* %vp, <16 x i8> %vec2, <16 x i8> %mask) {
-; GENERIC-LABEL: test_masked_16xi8_perm_mem_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovdqa (%rdi), %xmm2 # sched: [6:0.50]
-; GENERIC-NEXT:    vptestnmb %xmm1, %xmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufb {{.*#+}} xmm0 {%k1} = xmm2[14,9,15,9,7,10,15,14,12,1,9,7,10,13,3,11] sched: [7:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_16xi8_perm_mem_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovdqa (%rdi), %xmm2 # sched: [6:0.50]
-; SKX-NEXT:    vptestnmb %xmm1, %xmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufb {{.*#+}} xmm0 {%k1} = xmm2[14,9,15,9,7,10,15,14,12,1,9,7,10,13,3,11] sched: [7:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <16 x i8>, <16 x i8>* %vp
-  %shuf = shufflevector <16 x i8> %vec, <16 x i8> undef, <16 x i32> <i32 14, i32 9, i32 15, i32 9, i32 7, i32 10, i32 15, i32 14, i32 12, i32 1, i32 9, i32 7, i32 10, i32 13, i32 3, i32 11>
-  %cmp = icmp eq <16 x i8> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x i8> %shuf, <16 x i8> %vec2
-  ret <16 x i8> %res
-}
-
-define <16 x i8> @test_masked_z_16xi8_perm_mem_mask1(<16 x i8>* %vp, <16 x i8> %mask) {
-; GENERIC-LABEL: test_masked_z_16xi8_perm_mem_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovdqa (%rdi), %xmm1 # sched: [6:0.50]
-; GENERIC-NEXT:    vptestnmb %xmm0, %xmm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufb {{.*#+}} xmm0 {%k1} {z} = xmm1[14,9,15,9,7,10,15,14,12,1,9,7,10,13,3,11] sched: [7:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_16xi8_perm_mem_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovdqa (%rdi), %xmm1 # sched: [6:0.50]
-; SKX-NEXT:    vptestnmb %xmm0, %xmm0, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufb {{.*#+}} xmm0 {%k1} {z} = xmm1[14,9,15,9,7,10,15,14,12,1,9,7,10,13,3,11] sched: [7:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <16 x i8>, <16 x i8>* %vp
-  %shuf = shufflevector <16 x i8> %vec, <16 x i8> undef, <16 x i32> <i32 14, i32 9, i32 15, i32 9, i32 7, i32 10, i32 15, i32 14, i32 12, i32 1, i32 9, i32 7, i32 10, i32 13, i32 3, i32 11>
-  %cmp = icmp eq <16 x i8> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x i8> %shuf, <16 x i8> zeroinitializer
-  ret <16 x i8> %res
-}
-
-define <16 x i8> @test_masked_16xi8_perm_mem_mask2(<16 x i8>* %vp, <16 x i8> %vec2, <16 x i8> %mask) {
-; GENERIC-LABEL: test_masked_16xi8_perm_mem_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovdqa (%rdi), %xmm2 # sched: [6:0.50]
-; GENERIC-NEXT:    vptestnmb %xmm1, %xmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufb {{.*#+}} xmm0 {%k1} = xmm2[1,3,12,5,13,1,2,11,0,9,14,8,10,0,10,9] sched: [7:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_16xi8_perm_mem_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovdqa (%rdi), %xmm2 # sched: [6:0.50]
-; SKX-NEXT:    vptestnmb %xmm1, %xmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufb {{.*#+}} xmm0 {%k1} = xmm2[1,3,12,5,13,1,2,11,0,9,14,8,10,0,10,9] sched: [7:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <16 x i8>, <16 x i8>* %vp
-  %shuf = shufflevector <16 x i8> %vec, <16 x i8> undef, <16 x i32> <i32 1, i32 3, i32 12, i32 5, i32 13, i32 1, i32 2, i32 11, i32 0, i32 9, i32 14, i32 8, i32 10, i32 0, i32 10, i32 9>
-  %cmp = icmp eq <16 x i8> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x i8> %shuf, <16 x i8> %vec2
-  ret <16 x i8> %res
-}
-
-define <16 x i8> @test_masked_z_16xi8_perm_mem_mask2(<16 x i8>* %vp, <16 x i8> %mask) {
-; GENERIC-LABEL: test_masked_z_16xi8_perm_mem_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovdqa (%rdi), %xmm1 # sched: [6:0.50]
-; GENERIC-NEXT:    vptestnmb %xmm0, %xmm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufb {{.*#+}} xmm0 {%k1} {z} = xmm1[1,3,12,5,13,1,2,11,0,9,14,8,10,0,10,9] sched: [7:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_16xi8_perm_mem_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovdqa (%rdi), %xmm1 # sched: [6:0.50]
-; SKX-NEXT:    vptestnmb %xmm0, %xmm0, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufb {{.*#+}} xmm0 {%k1} {z} = xmm1[1,3,12,5,13,1,2,11,0,9,14,8,10,0,10,9] sched: [7:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <16 x i8>, <16 x i8>* %vp
-  %shuf = shufflevector <16 x i8> %vec, <16 x i8> undef, <16 x i32> <i32 1, i32 3, i32 12, i32 5, i32 13, i32 1, i32 2, i32 11, i32 0, i32 9, i32 14, i32 8, i32 10, i32 0, i32 10, i32 9>
-  %cmp = icmp eq <16 x i8> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x i8> %shuf, <16 x i8> zeroinitializer
-  ret <16 x i8> %res
-}
-
-define <16 x i8> @test_16xi8_perm_mem_mask3(<16 x i8>* %vp) {
-; GENERIC-LABEL: test_16xi8_perm_mem_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovdqa (%rdi), %xmm0 # sched: [6:0.50]
-; GENERIC-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[9,6,5,15,0,0,15,2,1,3,12,14,0,6,1,4] sched: [7:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_16xi8_perm_mem_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovdqa (%rdi), %xmm0 # sched: [6:0.50]
-; SKX-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[9,6,5,15,0,0,15,2,1,3,12,14,0,6,1,4] sched: [7:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <16 x i8>, <16 x i8>* %vp
-  %res = shufflevector <16 x i8> %vec, <16 x i8> undef, <16 x i32> <i32 9, i32 6, i32 5, i32 15, i32 0, i32 0, i32 15, i32 2, i32 1, i32 3, i32 12, i32 14, i32 0, i32 6, i32 1, i32 4>
-  ret <16 x i8> %res
-}
-define <16 x i8> @test_masked_16xi8_perm_mem_mask3(<16 x i8>* %vp, <16 x i8> %vec2, <16 x i8> %mask) {
-; GENERIC-LABEL: test_masked_16xi8_perm_mem_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovdqa (%rdi), %xmm2 # sched: [6:0.50]
-; GENERIC-NEXT:    vptestnmb %xmm1, %xmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufb {{.*#+}} xmm0 {%k1} = xmm2[9,6,5,15,0,0,15,2,1,3,12,14,0,6,1,4] sched: [7:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_16xi8_perm_mem_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovdqa (%rdi), %xmm2 # sched: [6:0.50]
-; SKX-NEXT:    vptestnmb %xmm1, %xmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufb {{.*#+}} xmm0 {%k1} = xmm2[9,6,5,15,0,0,15,2,1,3,12,14,0,6,1,4] sched: [7:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <16 x i8>, <16 x i8>* %vp
-  %shuf = shufflevector <16 x i8> %vec, <16 x i8> undef, <16 x i32> <i32 9, i32 6, i32 5, i32 15, i32 0, i32 0, i32 15, i32 2, i32 1, i32 3, i32 12, i32 14, i32 0, i32 6, i32 1, i32 4>
-  %cmp = icmp eq <16 x i8> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x i8> %shuf, <16 x i8> %vec2
-  ret <16 x i8> %res
-}
-
-define <16 x i8> @test_masked_z_16xi8_perm_mem_mask3(<16 x i8>* %vp, <16 x i8> %mask) {
-; GENERIC-LABEL: test_masked_z_16xi8_perm_mem_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovdqa (%rdi), %xmm1 # sched: [6:0.50]
-; GENERIC-NEXT:    vptestnmb %xmm0, %xmm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufb {{.*#+}} xmm0 {%k1} {z} = xmm1[9,6,5,15,0,0,15,2,1,3,12,14,0,6,1,4] sched: [7:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_16xi8_perm_mem_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovdqa (%rdi), %xmm1 # sched: [6:0.50]
-; SKX-NEXT:    vptestnmb %xmm0, %xmm0, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufb {{.*#+}} xmm0 {%k1} {z} = xmm1[9,6,5,15,0,0,15,2,1,3,12,14,0,6,1,4] sched: [7:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <16 x i8>, <16 x i8>* %vp
-  %shuf = shufflevector <16 x i8> %vec, <16 x i8> undef, <16 x i32> <i32 9, i32 6, i32 5, i32 15, i32 0, i32 0, i32 15, i32 2, i32 1, i32 3, i32 12, i32 14, i32 0, i32 6, i32 1, i32 4>
-  %cmp = icmp eq <16 x i8> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x i8> %shuf, <16 x i8> zeroinitializer
-  ret <16 x i8> %res
-}
-
-define <32 x i8> @test_32xi8_perm_mask0(<32 x i8> %vec) {
-; GENERIC-LABEL: test_32xi8_perm_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[8,0,1,15,3,5,11,13,14,2,10,15,0,10,13,5,20,25,23,18,23,22,25,24,20,21,29,20,24,16,27,21] sched: [8:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_32xi8_perm_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[8,0,1,15,3,5,11,13,14,2,10,15,0,10,13,5,20,25,23,18,23,22,25,24,20,21,29,20,24,16,27,21] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %res = shufflevector <32 x i8> %vec, <32 x i8> undef, <32 x i32> <i32 8, i32 0, i32 1, i32 15, i32 3, i32 5, i32 11, i32 13, i32 14, i32 2, i32 10, i32 15, i32 0, i32 10, i32 13, i32 5, i32 20, i32 25, i32 23, i32 18, i32 23, i32 22, i32 25, i32 24, i32 20, i32 21, i32 29, i32 20, i32 24, i32 16, i32 27, i32 21>
-  ret <32 x i8> %res
-}
-define <32 x i8> @test_masked_32xi8_perm_mask0(<32 x i8> %vec, <32 x i8> %vec2, <32 x i8> %mask) {
-; GENERIC-LABEL: test_masked_32xi8_perm_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmb %ymm2, %ymm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufb {{.*#+}} ymm1 {%k1} = ymm0[8,0,1,15,3,5,11,13,14,2,10,15,0,10,13,5,20,25,23,18,23,22,25,24,20,21,29,20,24,16,27,21] sched: [8:0.50]
-; GENERIC-NEXT:    vmovdqa %ymm1, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_32xi8_perm_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmb %ymm2, %ymm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufb {{.*#+}} ymm1 {%k1} = ymm0[8,0,1,15,3,5,11,13,14,2,10,15,0,10,13,5,20,25,23,18,23,22,25,24,20,21,29,20,24,16,27,21] sched: [8:1.00]
-; SKX-NEXT:    vmovdqa %ymm1, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <32 x i8> %vec, <32 x i8> undef, <32 x i32> <i32 8, i32 0, i32 1, i32 15, i32 3, i32 5, i32 11, i32 13, i32 14, i32 2, i32 10, i32 15, i32 0, i32 10, i32 13, i32 5, i32 20, i32 25, i32 23, i32 18, i32 23, i32 22, i32 25, i32 24, i32 20, i32 21, i32 29, i32 20, i32 24, i32 16, i32 27, i32 21>
-  %cmp = icmp eq <32 x i8> %mask, zeroinitializer
-  %res = select <32 x i1> %cmp, <32 x i8> %shuf, <32 x i8> %vec2
-  ret <32 x i8> %res
-}
-
-define <32 x i8> @test_masked_z_32xi8_perm_mask0(<32 x i8> %vec, <32 x i8> %mask) {
-; GENERIC-LABEL: test_masked_z_32xi8_perm_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmb %ymm1, %ymm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufb {{.*#+}} ymm0 {%k1} {z} = ymm0[8,0,1,15,3,5,11,13,14,2,10,15,0,10,13,5,20,25,23,18,23,22,25,24,20,21,29,20,24,16,27,21] sched: [8:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_32xi8_perm_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmb %ymm1, %ymm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufb {{.*#+}} ymm0 {%k1} {z} = ymm0[8,0,1,15,3,5,11,13,14,2,10,15,0,10,13,5,20,25,23,18,23,22,25,24,20,21,29,20,24,16,27,21] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <32 x i8> %vec, <32 x i8> undef, <32 x i32> <i32 8, i32 0, i32 1, i32 15, i32 3, i32 5, i32 11, i32 13, i32 14, i32 2, i32 10, i32 15, i32 0, i32 10, i32 13, i32 5, i32 20, i32 25, i32 23, i32 18, i32 23, i32 22, i32 25, i32 24, i32 20, i32 21, i32 29, i32 20, i32 24, i32 16, i32 27, i32 21>
-  %cmp = icmp eq <32 x i8> %mask, zeroinitializer
-  %res = select <32 x i1> %cmp, <32 x i8> %shuf, <32 x i8> zeroinitializer
-  ret <32 x i8> %res
-}
-define <32 x i8> @test_masked_32xi8_perm_mask1(<32 x i8> %vec, <32 x i8> %vec2, <32 x i8> %mask) {
-; GENERIC-LABEL: test_masked_32xi8_perm_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmb %ymm2, %ymm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufb {{.*#+}} ymm1 {%k1} = ymm0[0,4,3,15,5,4,5,15,10,9,11,6,6,10,0,3,21,19,26,22,30,25,22,22,27,22,26,16,23,20,18,24] sched: [8:0.50]
-; GENERIC-NEXT:    vmovdqa %ymm1, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_32xi8_perm_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmb %ymm2, %ymm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufb {{.*#+}} ymm1 {%k1} = ymm0[0,4,3,15,5,4,5,15,10,9,11,6,6,10,0,3,21,19,26,22,30,25,22,22,27,22,26,16,23,20,18,24] sched: [8:1.00]
-; SKX-NEXT:    vmovdqa %ymm1, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <32 x i8> %vec, <32 x i8> undef, <32 x i32> <i32 0, i32 4, i32 3, i32 15, i32 5, i32 4, i32 5, i32 15, i32 10, i32 9, i32 11, i32 6, i32 6, i32 10, i32 0, i32 3, i32 21, i32 19, i32 26, i32 22, i32 30, i32 25, i32 22, i32 22, i32 27, i32 22, i32 26, i32 16, i32 23, i32 20, i32 18, i32 24>
-  %cmp = icmp eq <32 x i8> %mask, zeroinitializer
-  %res = select <32 x i1> %cmp, <32 x i8> %shuf, <32 x i8> %vec2
-  ret <32 x i8> %res
-}
-
-define <32 x i8> @test_masked_z_32xi8_perm_mask1(<32 x i8> %vec, <32 x i8> %mask) {
-; GENERIC-LABEL: test_masked_z_32xi8_perm_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmb %ymm1, %ymm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufb {{.*#+}} ymm0 {%k1} {z} = ymm0[0,4,3,15,5,4,5,15,10,9,11,6,6,10,0,3,21,19,26,22,30,25,22,22,27,22,26,16,23,20,18,24] sched: [8:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_32xi8_perm_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmb %ymm1, %ymm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufb {{.*#+}} ymm0 {%k1} {z} = ymm0[0,4,3,15,5,4,5,15,10,9,11,6,6,10,0,3,21,19,26,22,30,25,22,22,27,22,26,16,23,20,18,24] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <32 x i8> %vec, <32 x i8> undef, <32 x i32> <i32 0, i32 4, i32 3, i32 15, i32 5, i32 4, i32 5, i32 15, i32 10, i32 9, i32 11, i32 6, i32 6, i32 10, i32 0, i32 3, i32 21, i32 19, i32 26, i32 22, i32 30, i32 25, i32 22, i32 22, i32 27, i32 22, i32 26, i32 16, i32 23, i32 20, i32 18, i32 24>
-  %cmp = icmp eq <32 x i8> %mask, zeroinitializer
-  %res = select <32 x i1> %cmp, <32 x i8> %shuf, <32 x i8> zeroinitializer
-  ret <32 x i8> %res
-}
-define <32 x i8> @test_masked_32xi8_perm_mask2(<32 x i8> %vec, <32 x i8> %vec2, <32 x i8> %mask) {
-; GENERIC-LABEL: test_masked_32xi8_perm_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmb %ymm2, %ymm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufb {{.*#+}} ymm1 {%k1} = ymm0[7,8,12,14,7,4,7,12,14,12,3,15,10,1,11,15,22,26,21,19,27,16,29,24,17,17,26,29,20,31,17,29] sched: [8:0.50]
-; GENERIC-NEXT:    vmovdqa %ymm1, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_32xi8_perm_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmb %ymm2, %ymm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufb {{.*#+}} ymm1 {%k1} = ymm0[7,8,12,14,7,4,7,12,14,12,3,15,10,1,11,15,22,26,21,19,27,16,29,24,17,17,26,29,20,31,17,29] sched: [8:1.00]
-; SKX-NEXT:    vmovdqa %ymm1, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <32 x i8> %vec, <32 x i8> undef, <32 x i32> <i32 7, i32 8, i32 12, i32 14, i32 7, i32 4, i32 7, i32 12, i32 14, i32 12, i32 3, i32 15, i32 10, i32 1, i32 11, i32 15, i32 22, i32 26, i32 21, i32 19, i32 27, i32 16, i32 29, i32 24, i32 17, i32 17, i32 26, i32 29, i32 20, i32 31, i32 17, i32 29>
-  %cmp = icmp eq <32 x i8> %mask, zeroinitializer
-  %res = select <32 x i1> %cmp, <32 x i8> %shuf, <32 x i8> %vec2
-  ret <32 x i8> %res
-}
-
-define <32 x i8> @test_masked_z_32xi8_perm_mask2(<32 x i8> %vec, <32 x i8> %mask) {
-; GENERIC-LABEL: test_masked_z_32xi8_perm_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmb %ymm1, %ymm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufb {{.*#+}} ymm0 {%k1} {z} = ymm0[7,8,12,14,7,4,7,12,14,12,3,15,10,1,11,15,22,26,21,19,27,16,29,24,17,17,26,29,20,31,17,29] sched: [8:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_32xi8_perm_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmb %ymm1, %ymm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufb {{.*#+}} ymm0 {%k1} {z} = ymm0[7,8,12,14,7,4,7,12,14,12,3,15,10,1,11,15,22,26,21,19,27,16,29,24,17,17,26,29,20,31,17,29] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <32 x i8> %vec, <32 x i8> undef, <32 x i32> <i32 7, i32 8, i32 12, i32 14, i32 7, i32 4, i32 7, i32 12, i32 14, i32 12, i32 3, i32 15, i32 10, i32 1, i32 11, i32 15, i32 22, i32 26, i32 21, i32 19, i32 27, i32 16, i32 29, i32 24, i32 17, i32 17, i32 26, i32 29, i32 20, i32 31, i32 17, i32 29>
-  %cmp = icmp eq <32 x i8> %mask, zeroinitializer
-  %res = select <32 x i1> %cmp, <32 x i8> %shuf, <32 x i8> zeroinitializer
-  ret <32 x i8> %res
-}
-define <32 x i8> @test_32xi8_perm_mask3(<32 x i8> %vec) {
-; GENERIC-LABEL: test_32xi8_perm_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[6,1,4,7,12,13,2,8,10,5,13,4,0,0,10,8,31,31,30,16,27,27,26,27,30,26,21,24,19,25,16,18] sched: [8:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_32xi8_perm_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[6,1,4,7,12,13,2,8,10,5,13,4,0,0,10,8,31,31,30,16,27,27,26,27,30,26,21,24,19,25,16,18] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %res = shufflevector <32 x i8> %vec, <32 x i8> undef, <32 x i32> <i32 6, i32 1, i32 4, i32 7, i32 12, i32 13, i32 2, i32 8, i32 10, i32 5, i32 13, i32 4, i32 0, i32 0, i32 10, i32 8, i32 31, i32 31, i32 30, i32 16, i32 27, i32 27, i32 26, i32 27, i32 30, i32 26, i32 21, i32 24, i32 19, i32 25, i32 16, i32 18>
-  ret <32 x i8> %res
-}
-define <32 x i8> @test_masked_32xi8_perm_mask3(<32 x i8> %vec, <32 x i8> %vec2, <32 x i8> %mask) {
-; GENERIC-LABEL: test_masked_32xi8_perm_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmb %ymm2, %ymm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufb {{.*#+}} ymm1 {%k1} = ymm0[6,1,4,7,12,13,2,8,10,5,13,4,0,0,10,8,31,31,30,16,27,27,26,27,30,26,21,24,19,25,16,18] sched: [8:0.50]
-; GENERIC-NEXT:    vmovdqa %ymm1, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_32xi8_perm_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmb %ymm2, %ymm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufb {{.*#+}} ymm1 {%k1} = ymm0[6,1,4,7,12,13,2,8,10,5,13,4,0,0,10,8,31,31,30,16,27,27,26,27,30,26,21,24,19,25,16,18] sched: [8:1.00]
-; SKX-NEXT:    vmovdqa %ymm1, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <32 x i8> %vec, <32 x i8> undef, <32 x i32> <i32 6, i32 1, i32 4, i32 7, i32 12, i32 13, i32 2, i32 8, i32 10, i32 5, i32 13, i32 4, i32 0, i32 0, i32 10, i32 8, i32 31, i32 31, i32 30, i32 16, i32 27, i32 27, i32 26, i32 27, i32 30, i32 26, i32 21, i32 24, i32 19, i32 25, i32 16, i32 18>
-  %cmp = icmp eq <32 x i8> %mask, zeroinitializer
-  %res = select <32 x i1> %cmp, <32 x i8> %shuf, <32 x i8> %vec2
-  ret <32 x i8> %res
-}
-
-define <32 x i8> @test_masked_z_32xi8_perm_mask3(<32 x i8> %vec, <32 x i8> %mask) {
-; GENERIC-LABEL: test_masked_z_32xi8_perm_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmb %ymm1, %ymm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufb {{.*#+}} ymm0 {%k1} {z} = ymm0[6,1,4,7,12,13,2,8,10,5,13,4,0,0,10,8,31,31,30,16,27,27,26,27,30,26,21,24,19,25,16,18] sched: [8:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_32xi8_perm_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmb %ymm1, %ymm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufb {{.*#+}} ymm0 {%k1} {z} = ymm0[6,1,4,7,12,13,2,8,10,5,13,4,0,0,10,8,31,31,30,16,27,27,26,27,30,26,21,24,19,25,16,18] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <32 x i8> %vec, <32 x i8> undef, <32 x i32> <i32 6, i32 1, i32 4, i32 7, i32 12, i32 13, i32 2, i32 8, i32 10, i32 5, i32 13, i32 4, i32 0, i32 0, i32 10, i32 8, i32 31, i32 31, i32 30, i32 16, i32 27, i32 27, i32 26, i32 27, i32 30, i32 26, i32 21, i32 24, i32 19, i32 25, i32 16, i32 18>
-  %cmp = icmp eq <32 x i8> %mask, zeroinitializer
-  %res = select <32 x i1> %cmp, <32 x i8> %shuf, <32 x i8> zeroinitializer
-  ret <32 x i8> %res
-}
-define <32 x i8> @test_32xi8_perm_mem_mask0(<32 x i8>* %vp) {
-; GENERIC-LABEL: test_32xi8_perm_mem_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovdqa (%rdi), %ymm0 # sched: [7:0.50]
-; GENERIC-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[9,0,2,15,4,6,8,4,7,3,0,2,8,1,6,5,22,17,30,23,29,31,21,23,27,22,20,27,30,30,26,22] sched: [8:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_32xi8_perm_mem_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovdqa (%rdi), %ymm0 # sched: [7:0.50]
-; SKX-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[9,0,2,15,4,6,8,4,7,3,0,2,8,1,6,5,22,17,30,23,29,31,21,23,27,22,20,27,30,30,26,22] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <32 x i8>, <32 x i8>* %vp
-  %res = shufflevector <32 x i8> %vec, <32 x i8> undef, <32 x i32> <i32 9, i32 0, i32 2, i32 15, i32 4, i32 6, i32 8, i32 4, i32 7, i32 3, i32 0, i32 2, i32 8, i32 1, i32 6, i32 5, i32 22, i32 17, i32 30, i32 23, i32 29, i32 31, i32 21, i32 23, i32 27, i32 22, i32 20, i32 27, i32 30, i32 30, i32 26, i32 22>
-  ret <32 x i8> %res
-}
-define <32 x i8> @test_masked_32xi8_perm_mem_mask0(<32 x i8>* %vp, <32 x i8> %vec2, <32 x i8> %mask) {
-; GENERIC-LABEL: test_masked_32xi8_perm_mem_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovdqa (%rdi), %ymm2 # sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmb %ymm1, %ymm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufb {{.*#+}} ymm0 {%k1} = ymm2[9,0,2,15,4,6,8,4,7,3,0,2,8,1,6,5,22,17,30,23,29,31,21,23,27,22,20,27,30,30,26,22] sched: [8:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_32xi8_perm_mem_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovdqa (%rdi), %ymm2 # sched: [7:0.50]
-; SKX-NEXT:    vptestnmb %ymm1, %ymm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufb {{.*#+}} ymm0 {%k1} = ymm2[9,0,2,15,4,6,8,4,7,3,0,2,8,1,6,5,22,17,30,23,29,31,21,23,27,22,20,27,30,30,26,22] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <32 x i8>, <32 x i8>* %vp
-  %shuf = shufflevector <32 x i8> %vec, <32 x i8> undef, <32 x i32> <i32 9, i32 0, i32 2, i32 15, i32 4, i32 6, i32 8, i32 4, i32 7, i32 3, i32 0, i32 2, i32 8, i32 1, i32 6, i32 5, i32 22, i32 17, i32 30, i32 23, i32 29, i32 31, i32 21, i32 23, i32 27, i32 22, i32 20, i32 27, i32 30, i32 30, i32 26, i32 22>
-  %cmp = icmp eq <32 x i8> %mask, zeroinitializer
-  %res = select <32 x i1> %cmp, <32 x i8> %shuf, <32 x i8> %vec2
-  ret <32 x i8> %res
-}
-
-define <32 x i8> @test_masked_z_32xi8_perm_mem_mask0(<32 x i8>* %vp, <32 x i8> %mask) {
-; GENERIC-LABEL: test_masked_z_32xi8_perm_mem_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovdqa (%rdi), %ymm1 # sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmb %ymm0, %ymm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufb {{.*#+}} ymm0 {%k1} {z} = ymm1[9,0,2,15,4,6,8,4,7,3,0,2,8,1,6,5,22,17,30,23,29,31,21,23,27,22,20,27,30,30,26,22] sched: [8:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_32xi8_perm_mem_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovdqa (%rdi), %ymm1 # sched: [7:0.50]
-; SKX-NEXT:    vptestnmb %ymm0, %ymm0, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufb {{.*#+}} ymm0 {%k1} {z} = ymm1[9,0,2,15,4,6,8,4,7,3,0,2,8,1,6,5,22,17,30,23,29,31,21,23,27,22,20,27,30,30,26,22] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <32 x i8>, <32 x i8>* %vp
-  %shuf = shufflevector <32 x i8> %vec, <32 x i8> undef, <32 x i32> <i32 9, i32 0, i32 2, i32 15, i32 4, i32 6, i32 8, i32 4, i32 7, i32 3, i32 0, i32 2, i32 8, i32 1, i32 6, i32 5, i32 22, i32 17, i32 30, i32 23, i32 29, i32 31, i32 21, i32 23, i32 27, i32 22, i32 20, i32 27, i32 30, i32 30, i32 26, i32 22>
-  %cmp = icmp eq <32 x i8> %mask, zeroinitializer
-  %res = select <32 x i1> %cmp, <32 x i8> %shuf, <32 x i8> zeroinitializer
-  ret <32 x i8> %res
-}
-
-define <32 x i8> @test_masked_32xi8_perm_mem_mask1(<32 x i8>* %vp, <32 x i8> %vec2, <32 x i8> %mask) {
-; GENERIC-LABEL: test_masked_32xi8_perm_mem_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovdqa (%rdi), %ymm2 # sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmb %ymm1, %ymm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufb {{.*#+}} ymm0 {%k1} = ymm2[15,10,1,1,11,0,0,6,8,7,7,9,10,6,5,15,20,28,22,21,17,29,27,30,23,26,17,22,19,16,31,19] sched: [8:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_32xi8_perm_mem_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovdqa (%rdi), %ymm2 # sched: [7:0.50]
-; SKX-NEXT:    vptestnmb %ymm1, %ymm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufb {{.*#+}} ymm0 {%k1} = ymm2[15,10,1,1,11,0,0,6,8,7,7,9,10,6,5,15,20,28,22,21,17,29,27,30,23,26,17,22,19,16,31,19] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <32 x i8>, <32 x i8>* %vp
-  %shuf = shufflevector <32 x i8> %vec, <32 x i8> undef, <32 x i32> <i32 15, i32 10, i32 1, i32 1, i32 11, i32 0, i32 0, i32 6, i32 8, i32 7, i32 7, i32 9, i32 10, i32 6, i32 5, i32 15, i32 20, i32 28, i32 22, i32 21, i32 17, i32 29, i32 27, i32 30, i32 23, i32 26, i32 17, i32 22, i32 19, i32 16, i32 31, i32 19>
-  %cmp = icmp eq <32 x i8> %mask, zeroinitializer
-  %res = select <32 x i1> %cmp, <32 x i8> %shuf, <32 x i8> %vec2
-  ret <32 x i8> %res
-}
-
-define <32 x i8> @test_masked_z_32xi8_perm_mem_mask1(<32 x i8>* %vp, <32 x i8> %mask) {
-; GENERIC-LABEL: test_masked_z_32xi8_perm_mem_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovdqa (%rdi), %ymm1 # sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmb %ymm0, %ymm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufb {{.*#+}} ymm0 {%k1} {z} = ymm1[15,10,1,1,11,0,0,6,8,7,7,9,10,6,5,15,20,28,22,21,17,29,27,30,23,26,17,22,19,16,31,19] sched: [8:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_32xi8_perm_mem_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovdqa (%rdi), %ymm1 # sched: [7:0.50]
-; SKX-NEXT:    vptestnmb %ymm0, %ymm0, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufb {{.*#+}} ymm0 {%k1} {z} = ymm1[15,10,1,1,11,0,0,6,8,7,7,9,10,6,5,15,20,28,22,21,17,29,27,30,23,26,17,22,19,16,31,19] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <32 x i8>, <32 x i8>* %vp
-  %shuf = shufflevector <32 x i8> %vec, <32 x i8> undef, <32 x i32> <i32 15, i32 10, i32 1, i32 1, i32 11, i32 0, i32 0, i32 6, i32 8, i32 7, i32 7, i32 9, i32 10, i32 6, i32 5, i32 15, i32 20, i32 28, i32 22, i32 21, i32 17, i32 29, i32 27, i32 30, i32 23, i32 26, i32 17, i32 22, i32 19, i32 16, i32 31, i32 19>
-  %cmp = icmp eq <32 x i8> %mask, zeroinitializer
-  %res = select <32 x i1> %cmp, <32 x i8> %shuf, <32 x i8> zeroinitializer
-  ret <32 x i8> %res
-}
-
-define <32 x i8> @test_masked_32xi8_perm_mem_mask2(<32 x i8>* %vp, <32 x i8> %vec2, <32 x i8> %mask) {
-; GENERIC-LABEL: test_masked_32xi8_perm_mem_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovdqa (%rdi), %ymm2 # sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmb %ymm1, %ymm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufb {{.*#+}} ymm0 {%k1} = ymm2[2,3,6,8,2,15,15,2,6,10,14,7,14,5,7,7,26,19,25,19,21,31,30,29,16,18,20,28,29,25,27,28] sched: [8:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_32xi8_perm_mem_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovdqa (%rdi), %ymm2 # sched: [7:0.50]
-; SKX-NEXT:    vptestnmb %ymm1, %ymm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufb {{.*#+}} ymm0 {%k1} = ymm2[2,3,6,8,2,15,15,2,6,10,14,7,14,5,7,7,26,19,25,19,21,31,30,29,16,18,20,28,29,25,27,28] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <32 x i8>, <32 x i8>* %vp
-  %shuf = shufflevector <32 x i8> %vec, <32 x i8> undef, <32 x i32> <i32 2, i32 3, i32 6, i32 8, i32 2, i32 15, i32 15, i32 2, i32 6, i32 10, i32 14, i32 7, i32 14, i32 5, i32 7, i32 7, i32 26, i32 19, i32 25, i32 19, i32 21, i32 31, i32 30, i32 29, i32 16, i32 18, i32 20, i32 28, i32 29, i32 25, i32 27, i32 28>
-  %cmp = icmp eq <32 x i8> %mask, zeroinitializer
-  %res = select <32 x i1> %cmp, <32 x i8> %shuf, <32 x i8> %vec2
-  ret <32 x i8> %res
-}
-
-define <32 x i8> @test_masked_z_32xi8_perm_mem_mask2(<32 x i8>* %vp, <32 x i8> %mask) {
-; GENERIC-LABEL: test_masked_z_32xi8_perm_mem_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovdqa (%rdi), %ymm1 # sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmb %ymm0, %ymm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufb {{.*#+}} ymm0 {%k1} {z} = ymm1[2,3,6,8,2,15,15,2,6,10,14,7,14,5,7,7,26,19,25,19,21,31,30,29,16,18,20,28,29,25,27,28] sched: [8:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_32xi8_perm_mem_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovdqa (%rdi), %ymm1 # sched: [7:0.50]
-; SKX-NEXT:    vptestnmb %ymm0, %ymm0, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufb {{.*#+}} ymm0 {%k1} {z} = ymm1[2,3,6,8,2,15,15,2,6,10,14,7,14,5,7,7,26,19,25,19,21,31,30,29,16,18,20,28,29,25,27,28] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <32 x i8>, <32 x i8>* %vp
-  %shuf = shufflevector <32 x i8> %vec, <32 x i8> undef, <32 x i32> <i32 2, i32 3, i32 6, i32 8, i32 2, i32 15, i32 15, i32 2, i32 6, i32 10, i32 14, i32 7, i32 14, i32 5, i32 7, i32 7, i32 26, i32 19, i32 25, i32 19, i32 21, i32 31, i32 30, i32 29, i32 16, i32 18, i32 20, i32 28, i32 29, i32 25, i32 27, i32 28>
-  %cmp = icmp eq <32 x i8> %mask, zeroinitializer
-  %res = select <32 x i1> %cmp, <32 x i8> %shuf, <32 x i8> zeroinitializer
-  ret <32 x i8> %res
-}
-
-define <32 x i8> @test_32xi8_perm_mem_mask3(<32 x i8>* %vp) {
-; GENERIC-LABEL: test_32xi8_perm_mem_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovdqa (%rdi), %ymm0 # sched: [7:0.50]
-; GENERIC-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[1,1,13,0,3,0,0,13,5,2,2,10,15,8,14,8,25,26,28,28,31,27,30,19,24,25,29,23,28,22,25,29] sched: [8:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_32xi8_perm_mem_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovdqa (%rdi), %ymm0 # sched: [7:0.50]
-; SKX-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[1,1,13,0,3,0,0,13,5,2,2,10,15,8,14,8,25,26,28,28,31,27,30,19,24,25,29,23,28,22,25,29] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <32 x i8>, <32 x i8>* %vp
-  %res = shufflevector <32 x i8> %vec, <32 x i8> undef, <32 x i32> <i32 1, i32 1, i32 13, i32 0, i32 3, i32 0, i32 0, i32 13, i32 5, i32 2, i32 2, i32 10, i32 15, i32 8, i32 14, i32 8, i32 25, i32 26, i32 28, i32 28, i32 31, i32 27, i32 30, i32 19, i32 24, i32 25, i32 29, i32 23, i32 28, i32 22, i32 25, i32 29>
-  ret <32 x i8> %res
-}
-define <32 x i8> @test_masked_32xi8_perm_mem_mask3(<32 x i8>* %vp, <32 x i8> %vec2, <32 x i8> %mask) {
-; GENERIC-LABEL: test_masked_32xi8_perm_mem_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovdqa (%rdi), %ymm2 # sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmb %ymm1, %ymm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufb {{.*#+}} ymm0 {%k1} = ymm2[1,1,13,0,3,0,0,13,5,2,2,10,15,8,14,8,25,26,28,28,31,27,30,19,24,25,29,23,28,22,25,29] sched: [8:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_32xi8_perm_mem_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovdqa (%rdi), %ymm2 # sched: [7:0.50]
-; SKX-NEXT:    vptestnmb %ymm1, %ymm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufb {{.*#+}} ymm0 {%k1} = ymm2[1,1,13,0,3,0,0,13,5,2,2,10,15,8,14,8,25,26,28,28,31,27,30,19,24,25,29,23,28,22,25,29] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <32 x i8>, <32 x i8>* %vp
-  %shuf = shufflevector <32 x i8> %vec, <32 x i8> undef, <32 x i32> <i32 1, i32 1, i32 13, i32 0, i32 3, i32 0, i32 0, i32 13, i32 5, i32 2, i32 2, i32 10, i32 15, i32 8, i32 14, i32 8, i32 25, i32 26, i32 28, i32 28, i32 31, i32 27, i32 30, i32 19, i32 24, i32 25, i32 29, i32 23, i32 28, i32 22, i32 25, i32 29>
-  %cmp = icmp eq <32 x i8> %mask, zeroinitializer
-  %res = select <32 x i1> %cmp, <32 x i8> %shuf, <32 x i8> %vec2
-  ret <32 x i8> %res
-}
-
-define <32 x i8> @test_masked_z_32xi8_perm_mem_mask3(<32 x i8>* %vp, <32 x i8> %mask) {
-; GENERIC-LABEL: test_masked_z_32xi8_perm_mem_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovdqa (%rdi), %ymm1 # sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmb %ymm0, %ymm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufb {{.*#+}} ymm0 {%k1} {z} = ymm1[1,1,13,0,3,0,0,13,5,2,2,10,15,8,14,8,25,26,28,28,31,27,30,19,24,25,29,23,28,22,25,29] sched: [8:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_32xi8_perm_mem_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovdqa (%rdi), %ymm1 # sched: [7:0.50]
-; SKX-NEXT:    vptestnmb %ymm0, %ymm0, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufb {{.*#+}} ymm0 {%k1} {z} = ymm1[1,1,13,0,3,0,0,13,5,2,2,10,15,8,14,8,25,26,28,28,31,27,30,19,24,25,29,23,28,22,25,29] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <32 x i8>, <32 x i8>* %vp
-  %shuf = shufflevector <32 x i8> %vec, <32 x i8> undef, <32 x i32> <i32 1, i32 1, i32 13, i32 0, i32 3, i32 0, i32 0, i32 13, i32 5, i32 2, i32 2, i32 10, i32 15, i32 8, i32 14, i32 8, i32 25, i32 26, i32 28, i32 28, i32 31, i32 27, i32 30, i32 19, i32 24, i32 25, i32 29, i32 23, i32 28, i32 22, i32 25, i32 29>
-  %cmp = icmp eq <32 x i8> %mask, zeroinitializer
-  %res = select <32 x i1> %cmp, <32 x i8> %shuf, <32 x i8> zeroinitializer
-  ret <32 x i8> %res
-}
-
-define <64 x i8> @test_64xi8_perm_mask0(<64 x i8> %vec) {
-; GENERIC-LABEL: test_64xi8_perm_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpshufb {{.*#+}} zmm0 = zmm0[8,4,1,13,15,4,6,12,0,10,2,4,13,0,0,6,23,29,27,26,18,31,22,25,22,16,23,18,16,25,26,17,40,37,38,44,39,46,41,39,42,37,33,42,41,44,34,46,60,62,61,58,60,56,60,51,60,55,60,55,60,49,48,62] sched: [8:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_64xi8_perm_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpshufb {{.*#+}} zmm0 = zmm0[8,4,1,13,15,4,6,12,0,10,2,4,13,0,0,6,23,29,27,26,18,31,22,25,22,16,23,18,16,25,26,17,40,37,38,44,39,46,41,39,42,37,33,42,41,44,34,46,60,62,61,58,60,56,60,51,60,55,60,55,60,49,48,62] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %res = shufflevector <64 x i8> %vec, <64 x i8> undef, <64 x i32> <i32 8, i32 4, i32 1, i32 13, i32 15, i32 4, i32 6, i32 12, i32 0, i32 10, i32 2, i32 4, i32 13, i32 0, i32 0, i32 6, i32 23, i32 29, i32 27, i32 26, i32 18, i32 31, i32 22, i32 25, i32 22, i32 16, i32 23, i32 18, i32 16, i32 25, i32 26, i32 17, i32 40, i32 37, i32 38, i32 44, i32 39, i32 46, i32 41, i32 39, i32 42, i32 37, i32 33, i32 42, i32 41, i32 44, i32 34, i32 46, i32 60, i32 62, i32 61, i32 58, i32 60, i32 56, i32 60, i32 51, i32 60, i32 55, i32 60, i32 55, i32 60, i32 49, i32 48, i32 62>
-  ret <64 x i8> %res
-}
-define <64 x i8> @test_masked_64xi8_perm_mask0(<64 x i8> %vec, <64 x i8> %vec2, <64 x i8> %mask) {
-; GENERIC-LABEL: test_masked_64xi8_perm_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmb %zmm2, %zmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufb {{.*#+}} zmm1 {%k1} = zmm0[8,4,1,13,15,4,6,12,0,10,2,4,13,0,0,6,23,29,27,26,18,31,22,25,22,16,23,18,16,25,26,17,40,37,38,44,39,46,41,39,42,37,33,42,41,44,34,46,60,62,61,58,60,56,60,51,60,55,60,55,60,49,48,62] sched: [8:0.50]
-; GENERIC-NEXT:    vmovdqa64 %zmm1, %zmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_64xi8_perm_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmb %zmm2, %zmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufb {{.*#+}} zmm1 {%k1} = zmm0[8,4,1,13,15,4,6,12,0,10,2,4,13,0,0,6,23,29,27,26,18,31,22,25,22,16,23,18,16,25,26,17,40,37,38,44,39,46,41,39,42,37,33,42,41,44,34,46,60,62,61,58,60,56,60,51,60,55,60,55,60,49,48,62] sched: [8:1.00]
-; SKX-NEXT:    vmovdqa64 %zmm1, %zmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <64 x i8> %vec, <64 x i8> undef, <64 x i32> <i32 8, i32 4, i32 1, i32 13, i32 15, i32 4, i32 6, i32 12, i32 0, i32 10, i32 2, i32 4, i32 13, i32 0, i32 0, i32 6, i32 23, i32 29, i32 27, i32 26, i32 18, i32 31, i32 22, i32 25, i32 22, i32 16, i32 23, i32 18, i32 16, i32 25, i32 26, i32 17, i32 40, i32 37, i32 38, i32 44, i32 39, i32 46, i32 41, i32 39, i32 42, i32 37, i32 33, i32 42, i32 41, i32 44, i32 34, i32 46, i32 60, i32 62, i32 61, i32 58, i32 60, i32 56, i32 60, i32 51, i32 60, i32 55, i32 60, i32 55, i32 60, i32 49, i32 48, i32 62>
-  %cmp = icmp eq <64 x i8> %mask, zeroinitializer
-  %res = select <64 x i1> %cmp, <64 x i8> %shuf, <64 x i8> %vec2
-  ret <64 x i8> %res
-}
-
-define <64 x i8> @test_masked_z_64xi8_perm_mask0(<64 x i8> %vec, <64 x i8> %mask) {
-; GENERIC-LABEL: test_masked_z_64xi8_perm_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmb %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufb {{.*#+}} zmm0 {%k1} {z} = zmm0[8,4,1,13,15,4,6,12,0,10,2,4,13,0,0,6,23,29,27,26,18,31,22,25,22,16,23,18,16,25,26,17,40,37,38,44,39,46,41,39,42,37,33,42,41,44,34,46,60,62,61,58,60,56,60,51,60,55,60,55,60,49,48,62] sched: [8:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_64xi8_perm_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmb %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufb {{.*#+}} zmm0 {%k1} {z} = zmm0[8,4,1,13,15,4,6,12,0,10,2,4,13,0,0,6,23,29,27,26,18,31,22,25,22,16,23,18,16,25,26,17,40,37,38,44,39,46,41,39,42,37,33,42,41,44,34,46,60,62,61,58,60,56,60,51,60,55,60,55,60,49,48,62] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <64 x i8> %vec, <64 x i8> undef, <64 x i32> <i32 8, i32 4, i32 1, i32 13, i32 15, i32 4, i32 6, i32 12, i32 0, i32 10, i32 2, i32 4, i32 13, i32 0, i32 0, i32 6, i32 23, i32 29, i32 27, i32 26, i32 18, i32 31, i32 22, i32 25, i32 22, i32 16, i32 23, i32 18, i32 16, i32 25, i32 26, i32 17, i32 40, i32 37, i32 38, i32 44, i32 39, i32 46, i32 41, i32 39, i32 42, i32 37, i32 33, i32 42, i32 41, i32 44, i32 34, i32 46, i32 60, i32 62, i32 61, i32 58, i32 60, i32 56, i32 60, i32 51, i32 60, i32 55, i32 60, i32 55, i32 60, i32 49, i32 48, i32 62>
-  %cmp = icmp eq <64 x i8> %mask, zeroinitializer
-  %res = select <64 x i1> %cmp, <64 x i8> %shuf, <64 x i8> zeroinitializer
-  ret <64 x i8> %res
-}
-define <64 x i8> @test_masked_64xi8_perm_mask1(<64 x i8> %vec, <64 x i8> %vec2, <64 x i8> %mask) {
-; GENERIC-LABEL: test_masked_64xi8_perm_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmb %zmm2, %zmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufb {{.*#+}} zmm1 {%k1} = zmm0[7,14,15,10,9,3,1,13,14,12,11,6,4,1,6,9,30,30,22,17,28,27,16,23,26,16,30,31,27,17,17,21,32,37,32,47,45,33,46,35,35,42,47,33,32,37,32,41,61,50,49,53,63,50,63,53,55,52,62,63,58,50,63,49] sched: [8:0.50]
-; GENERIC-NEXT:    vmovdqa64 %zmm1, %zmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_64xi8_perm_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmb %zmm2, %zmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufb {{.*#+}} zmm1 {%k1} = zmm0[7,14,15,10,9,3,1,13,14,12,11,6,4,1,6,9,30,30,22,17,28,27,16,23,26,16,30,31,27,17,17,21,32,37,32,47,45,33,46,35,35,42,47,33,32,37,32,41,61,50,49,53,63,50,63,53,55,52,62,63,58,50,63,49] sched: [8:1.00]
-; SKX-NEXT:    vmovdqa64 %zmm1, %zmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <64 x i8> %vec, <64 x i8> undef, <64 x i32> <i32 7, i32 14, i32 15, i32 10, i32 9, i32 3, i32 1, i32 13, i32 14, i32 12, i32 11, i32 6, i32 4, i32 1, i32 6, i32 9, i32 30, i32 30, i32 22, i32 17, i32 28, i32 27, i32 16, i32 23, i32 26, i32 16, i32 30, i32 31, i32 27, i32 17, i32 17, i32 21, i32 32, i32 37, i32 32, i32 47, i32 45, i32 33, i32 46, i32 35, i32 35, i32 42, i32 47, i32 33, i32 32, i32 37, i32 32, i32 41, i32 61, i32 50, i32 49, i32 53, i32 63, i32 50, i32 63, i32 53, i32 55, i32 52, i32 62, i32 63, i32 58, i32 50, i32 63, i32 49>
-  %cmp = icmp eq <64 x i8> %mask, zeroinitializer
-  %res = select <64 x i1> %cmp, <64 x i8> %shuf, <64 x i8> %vec2
-  ret <64 x i8> %res
-}
-
-define <64 x i8> @test_masked_z_64xi8_perm_mask1(<64 x i8> %vec, <64 x i8> %mask) {
-; GENERIC-LABEL: test_masked_z_64xi8_perm_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmb %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufb {{.*#+}} zmm0 {%k1} {z} = zmm0[7,14,15,10,9,3,1,13,14,12,11,6,4,1,6,9,30,30,22,17,28,27,16,23,26,16,30,31,27,17,17,21,32,37,32,47,45,33,46,35,35,42,47,33,32,37,32,41,61,50,49,53,63,50,63,53,55,52,62,63,58,50,63,49] sched: [8:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_64xi8_perm_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmb %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufb {{.*#+}} zmm0 {%k1} {z} = zmm0[7,14,15,10,9,3,1,13,14,12,11,6,4,1,6,9,30,30,22,17,28,27,16,23,26,16,30,31,27,17,17,21,32,37,32,47,45,33,46,35,35,42,47,33,32,37,32,41,61,50,49,53,63,50,63,53,55,52,62,63,58,50,63,49] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <64 x i8> %vec, <64 x i8> undef, <64 x i32> <i32 7, i32 14, i32 15, i32 10, i32 9, i32 3, i32 1, i32 13, i32 14, i32 12, i32 11, i32 6, i32 4, i32 1, i32 6, i32 9, i32 30, i32 30, i32 22, i32 17, i32 28, i32 27, i32 16, i32 23, i32 26, i32 16, i32 30, i32 31, i32 27, i32 17, i32 17, i32 21, i32 32, i32 37, i32 32, i32 47, i32 45, i32 33, i32 46, i32 35, i32 35, i32 42, i32 47, i32 33, i32 32, i32 37, i32 32, i32 41, i32 61, i32 50, i32 49, i32 53, i32 63, i32 50, i32 63, i32 53, i32 55, i32 52, i32 62, i32 63, i32 58, i32 50, i32 63, i32 49>
-  %cmp = icmp eq <64 x i8> %mask, zeroinitializer
-  %res = select <64 x i1> %cmp, <64 x i8> %shuf, <64 x i8> zeroinitializer
-  ret <64 x i8> %res
-}
-define <64 x i8> @test_masked_64xi8_perm_mask2(<64 x i8> %vec, <64 x i8> %vec2, <64 x i8> %mask) {
-; GENERIC-LABEL: test_masked_64xi8_perm_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmb %zmm2, %zmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufb {{.*#+}} zmm1 {%k1} = zmm0[9,2,14,15,12,5,3,12,4,6,0,2,0,1,1,6,24,27,18,22,26,17,23,21,31,16,22,22,27,21,19,20,39,47,44,36,40,43,44,39,38,44,38,35,39,46,34,39,58,55,51,48,59,57,48,52,60,58,56,50,59,55,58,60] sched: [8:0.50]
-; GENERIC-NEXT:    vmovdqa64 %zmm1, %zmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_64xi8_perm_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmb %zmm2, %zmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufb {{.*#+}} zmm1 {%k1} = zmm0[9,2,14,15,12,5,3,12,4,6,0,2,0,1,1,6,24,27,18,22,26,17,23,21,31,16,22,22,27,21,19,20,39,47,44,36,40,43,44,39,38,44,38,35,39,46,34,39,58,55,51,48,59,57,48,52,60,58,56,50,59,55,58,60] sched: [8:1.00]
-; SKX-NEXT:    vmovdqa64 %zmm1, %zmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <64 x i8> %vec, <64 x i8> undef, <64 x i32> <i32 9, i32 2, i32 14, i32 15, i32 12, i32 5, i32 3, i32 12, i32 4, i32 6, i32 0, i32 2, i32 0, i32 1, i32 1, i32 6, i32 24, i32 27, i32 18, i32 22, i32 26, i32 17, i32 23, i32 21, i32 31, i32 16, i32 22, i32 22, i32 27, i32 21, i32 19, i32 20, i32 39, i32 47, i32 44, i32 36, i32 40, i32 43, i32 44, i32 39, i32 38, i32 44, i32 38, i32 35, i32 39, i32 46, i32 34, i32 39, i32 58, i32 55, i32 51, i32 48, i32 59, i32 57, i32 48, i32 52, i32 60, i32 58, i32 56, i32 50, i32 59, i32 55, i32 58, i32 60>
-  %cmp = icmp eq <64 x i8> %mask, zeroinitializer
-  %res = select <64 x i1> %cmp, <64 x i8> %shuf, <64 x i8> %vec2
-  ret <64 x i8> %res
-}
-
-define <64 x i8> @test_masked_z_64xi8_perm_mask2(<64 x i8> %vec, <64 x i8> %mask) {
-; GENERIC-LABEL: test_masked_z_64xi8_perm_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmb %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufb {{.*#+}} zmm0 {%k1} {z} = zmm0[9,2,14,15,12,5,3,12,4,6,0,2,0,1,1,6,24,27,18,22,26,17,23,21,31,16,22,22,27,21,19,20,39,47,44,36,40,43,44,39,38,44,38,35,39,46,34,39,58,55,51,48,59,57,48,52,60,58,56,50,59,55,58,60] sched: [8:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_64xi8_perm_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmb %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufb {{.*#+}} zmm0 {%k1} {z} = zmm0[9,2,14,15,12,5,3,12,4,6,0,2,0,1,1,6,24,27,18,22,26,17,23,21,31,16,22,22,27,21,19,20,39,47,44,36,40,43,44,39,38,44,38,35,39,46,34,39,58,55,51,48,59,57,48,52,60,58,56,50,59,55,58,60] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <64 x i8> %vec, <64 x i8> undef, <64 x i32> <i32 9, i32 2, i32 14, i32 15, i32 12, i32 5, i32 3, i32 12, i32 4, i32 6, i32 0, i32 2, i32 0, i32 1, i32 1, i32 6, i32 24, i32 27, i32 18, i32 22, i32 26, i32 17, i32 23, i32 21, i32 31, i32 16, i32 22, i32 22, i32 27, i32 21, i32 19, i32 20, i32 39, i32 47, i32 44, i32 36, i32 40, i32 43, i32 44, i32 39, i32 38, i32 44, i32 38, i32 35, i32 39, i32 46, i32 34, i32 39, i32 58, i32 55, i32 51, i32 48, i32 59, i32 57, i32 48, i32 52, i32 60, i32 58, i32 56, i32 50, i32 59, i32 55, i32 58, i32 60>
-  %cmp = icmp eq <64 x i8> %mask, zeroinitializer
-  %res = select <64 x i1> %cmp, <64 x i8> %shuf, <64 x i8> zeroinitializer
-  ret <64 x i8> %res
-}
-define <64 x i8> @test_64xi8_perm_mask3(<64 x i8> %vec) {
-; GENERIC-LABEL: test_64xi8_perm_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpshufb {{.*#+}} zmm0 = zmm0[3,12,4,15,1,14,0,4,8,9,6,1,4,4,12,14,25,16,28,20,21,24,19,30,18,22,20,24,25,26,24,22,42,38,44,44,36,37,42,34,43,38,41,34,42,37,39,38,55,59,53,58,48,52,59,48,57,48,55,62,48,56,49,61] sched: [8:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_64xi8_perm_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpshufb {{.*#+}} zmm0 = zmm0[3,12,4,15,1,14,0,4,8,9,6,1,4,4,12,14,25,16,28,20,21,24,19,30,18,22,20,24,25,26,24,22,42,38,44,44,36,37,42,34,43,38,41,34,42,37,39,38,55,59,53,58,48,52,59,48,57,48,55,62,48,56,49,61] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %res = shufflevector <64 x i8> %vec, <64 x i8> undef, <64 x i32> <i32 3, i32 12, i32 4, i32 15, i32 1, i32 14, i32 0, i32 4, i32 8, i32 9, i32 6, i32 1, i32 4, i32 4, i32 12, i32 14, i32 25, i32 16, i32 28, i32 20, i32 21, i32 24, i32 19, i32 30, i32 18, i32 22, i32 20, i32 24, i32 25, i32 26, i32 24, i32 22, i32 42, i32 38, i32 44, i32 44, i32 36, i32 37, i32 42, i32 34, i32 43, i32 38, i32 41, i32 34, i32 42, i32 37, i32 39, i32 38, i32 55, i32 59, i32 53, i32 58, i32 48, i32 52, i32 59, i32 48, i32 57, i32 48, i32 55, i32 62, i32 48, i32 56, i32 49, i32 61>
-  ret <64 x i8> %res
-}
-define <64 x i8> @test_masked_64xi8_perm_mask3(<64 x i8> %vec, <64 x i8> %vec2, <64 x i8> %mask) {
-; GENERIC-LABEL: test_masked_64xi8_perm_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmb %zmm2, %zmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufb {{.*#+}} zmm1 {%k1} = zmm0[3,12,4,15,1,14,0,4,8,9,6,1,4,4,12,14,25,16,28,20,21,24,19,30,18,22,20,24,25,26,24,22,42,38,44,44,36,37,42,34,43,38,41,34,42,37,39,38,55,59,53,58,48,52,59,48,57,48,55,62,48,56,49,61] sched: [8:0.50]
-; GENERIC-NEXT:    vmovdqa64 %zmm1, %zmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_64xi8_perm_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmb %zmm2, %zmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufb {{.*#+}} zmm1 {%k1} = zmm0[3,12,4,15,1,14,0,4,8,9,6,1,4,4,12,14,25,16,28,20,21,24,19,30,18,22,20,24,25,26,24,22,42,38,44,44,36,37,42,34,43,38,41,34,42,37,39,38,55,59,53,58,48,52,59,48,57,48,55,62,48,56,49,61] sched: [8:1.00]
-; SKX-NEXT:    vmovdqa64 %zmm1, %zmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <64 x i8> %vec, <64 x i8> undef, <64 x i32> <i32 3, i32 12, i32 4, i32 15, i32 1, i32 14, i32 0, i32 4, i32 8, i32 9, i32 6, i32 1, i32 4, i32 4, i32 12, i32 14, i32 25, i32 16, i32 28, i32 20, i32 21, i32 24, i32 19, i32 30, i32 18, i32 22, i32 20, i32 24, i32 25, i32 26, i32 24, i32 22, i32 42, i32 38, i32 44, i32 44, i32 36, i32 37, i32 42, i32 34, i32 43, i32 38, i32 41, i32 34, i32 42, i32 37, i32 39, i32 38, i32 55, i32 59, i32 53, i32 58, i32 48, i32 52, i32 59, i32 48, i32 57, i32 48, i32 55, i32 62, i32 48, i32 56, i32 49, i32 61>
-  %cmp = icmp eq <64 x i8> %mask, zeroinitializer
-  %res = select <64 x i1> %cmp, <64 x i8> %shuf, <64 x i8> %vec2
-  ret <64 x i8> %res
-}
-
-define <64 x i8> @test_masked_z_64xi8_perm_mask3(<64 x i8> %vec, <64 x i8> %mask) {
-; GENERIC-LABEL: test_masked_z_64xi8_perm_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmb %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufb {{.*#+}} zmm0 {%k1} {z} = zmm0[3,12,4,15,1,14,0,4,8,9,6,1,4,4,12,14,25,16,28,20,21,24,19,30,18,22,20,24,25,26,24,22,42,38,44,44,36,37,42,34,43,38,41,34,42,37,39,38,55,59,53,58,48,52,59,48,57,48,55,62,48,56,49,61] sched: [8:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_64xi8_perm_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmb %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufb {{.*#+}} zmm0 {%k1} {z} = zmm0[3,12,4,15,1,14,0,4,8,9,6,1,4,4,12,14,25,16,28,20,21,24,19,30,18,22,20,24,25,26,24,22,42,38,44,44,36,37,42,34,43,38,41,34,42,37,39,38,55,59,53,58,48,52,59,48,57,48,55,62,48,56,49,61] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <64 x i8> %vec, <64 x i8> undef, <64 x i32> <i32 3, i32 12, i32 4, i32 15, i32 1, i32 14, i32 0, i32 4, i32 8, i32 9, i32 6, i32 1, i32 4, i32 4, i32 12, i32 14, i32 25, i32 16, i32 28, i32 20, i32 21, i32 24, i32 19, i32 30, i32 18, i32 22, i32 20, i32 24, i32 25, i32 26, i32 24, i32 22, i32 42, i32 38, i32 44, i32 44, i32 36, i32 37, i32 42, i32 34, i32 43, i32 38, i32 41, i32 34, i32 42, i32 37, i32 39, i32 38, i32 55, i32 59, i32 53, i32 58, i32 48, i32 52, i32 59, i32 48, i32 57, i32 48, i32 55, i32 62, i32 48, i32 56, i32 49, i32 61>
-  %cmp = icmp eq <64 x i8> %mask, zeroinitializer
-  %res = select <64 x i1> %cmp, <64 x i8> %shuf, <64 x i8> zeroinitializer
-  ret <64 x i8> %res
-}
-define <64 x i8> @test_64xi8_perm_mem_mask0(<64 x i8>* %vp) {
-; GENERIC-LABEL: test_64xi8_perm_mem_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovdqa64 (%rdi), %zmm0 # sched: [7:0.50]
-; GENERIC-NEXT:    vpshufb {{.*#+}} zmm0 = zmm0[0,9,15,13,11,11,3,12,4,1,7,5,2,6,14,6,23,27,24,18,30,23,28,22,28,22,19,19,31,25,16,22,35,33,34,32,42,34,41,41,43,40,36,46,37,39,42,40,63,63,62,62,57,55,59,51,52,48,50,48,58,50,60,58] sched: [8:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_64xi8_perm_mem_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovdqa64 (%rdi), %zmm0 # sched: [8:0.50]
-; SKX-NEXT:    vpshufb {{.*#+}} zmm0 = zmm0[0,9,15,13,11,11,3,12,4,1,7,5,2,6,14,6,23,27,24,18,30,23,28,22,28,22,19,19,31,25,16,22,35,33,34,32,42,34,41,41,43,40,36,46,37,39,42,40,63,63,62,62,57,55,59,51,52,48,50,48,58,50,60,58] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <64 x i8>, <64 x i8>* %vp
-  %res = shufflevector <64 x i8> %vec, <64 x i8> undef, <64 x i32> <i32 0, i32 9, i32 15, i32 13, i32 11, i32 11, i32 3, i32 12, i32 4, i32 1, i32 7, i32 5, i32 2, i32 6, i32 14, i32 6, i32 23, i32 27, i32 24, i32 18, i32 30, i32 23, i32 28, i32 22, i32 28, i32 22, i32 19, i32 19, i32 31, i32 25, i32 16, i32 22, i32 35, i32 33, i32 34, i32 32, i32 42, i32 34, i32 41, i32 41, i32 43, i32 40, i32 36, i32 46, i32 37, i32 39, i32 42, i32 40, i32 63, i32 63, i32 62, i32 62, i32 57, i32 55, i32 59, i32 51, i32 52, i32 48, i32 50, i32 48, i32 58, i32 50, i32 60, i32 58>
-  ret <64 x i8> %res
-}
-define <64 x i8> @test_masked_64xi8_perm_mem_mask0(<64 x i8>* %vp, <64 x i8> %vec2, <64 x i8> %mask) {
-; GENERIC-LABEL: test_masked_64xi8_perm_mem_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovdqa64 (%rdi), %zmm2 # sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmb %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufb {{.*#+}} zmm0 {%k1} = zmm2[0,9,15,13,11,11,3,12,4,1,7,5,2,6,14,6,23,27,24,18,30,23,28,22,28,22,19,19,31,25,16,22,35,33,34,32,42,34,41,41,43,40,36,46,37,39,42,40,63,63,62,62,57,55,59,51,52,48,50,48,58,50,60,58] sched: [8:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_64xi8_perm_mem_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovdqa64 (%rdi), %zmm2 # sched: [8:0.50]
-; SKX-NEXT:    vptestnmb %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufb {{.*#+}} zmm0 {%k1} = zmm2[0,9,15,13,11,11,3,12,4,1,7,5,2,6,14,6,23,27,24,18,30,23,28,22,28,22,19,19,31,25,16,22,35,33,34,32,42,34,41,41,43,40,36,46,37,39,42,40,63,63,62,62,57,55,59,51,52,48,50,48,58,50,60,58] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <64 x i8>, <64 x i8>* %vp
-  %shuf = shufflevector <64 x i8> %vec, <64 x i8> undef, <64 x i32> <i32 0, i32 9, i32 15, i32 13, i32 11, i32 11, i32 3, i32 12, i32 4, i32 1, i32 7, i32 5, i32 2, i32 6, i32 14, i32 6, i32 23, i32 27, i32 24, i32 18, i32 30, i32 23, i32 28, i32 22, i32 28, i32 22, i32 19, i32 19, i32 31, i32 25, i32 16, i32 22, i32 35, i32 33, i32 34, i32 32, i32 42, i32 34, i32 41, i32 41, i32 43, i32 40, i32 36, i32 46, i32 37, i32 39, i32 42, i32 40, i32 63, i32 63, i32 62, i32 62, i32 57, i32 55, i32 59, i32 51, i32 52, i32 48, i32 50, i32 48, i32 58, i32 50, i32 60, i32 58>
-  %cmp = icmp eq <64 x i8> %mask, zeroinitializer
-  %res = select <64 x i1> %cmp, <64 x i8> %shuf, <64 x i8> %vec2
-  ret <64 x i8> %res
-}
-
-define <64 x i8> @test_masked_z_64xi8_perm_mem_mask0(<64 x i8>* %vp, <64 x i8> %mask) {
-; GENERIC-LABEL: test_masked_z_64xi8_perm_mem_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovdqa64 (%rdi), %zmm1 # sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmb %zmm0, %zmm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufb {{.*#+}} zmm0 {%k1} {z} = zmm1[0,9,15,13,11,11,3,12,4,1,7,5,2,6,14,6,23,27,24,18,30,23,28,22,28,22,19,19,31,25,16,22,35,33,34,32,42,34,41,41,43,40,36,46,37,39,42,40,63,63,62,62,57,55,59,51,52,48,50,48,58,50,60,58] sched: [8:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_64xi8_perm_mem_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovdqa64 (%rdi), %zmm1 # sched: [8:0.50]
-; SKX-NEXT:    vptestnmb %zmm0, %zmm0, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufb {{.*#+}} zmm0 {%k1} {z} = zmm1[0,9,15,13,11,11,3,12,4,1,7,5,2,6,14,6,23,27,24,18,30,23,28,22,28,22,19,19,31,25,16,22,35,33,34,32,42,34,41,41,43,40,36,46,37,39,42,40,63,63,62,62,57,55,59,51,52,48,50,48,58,50,60,58] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <64 x i8>, <64 x i8>* %vp
-  %shuf = shufflevector <64 x i8> %vec, <64 x i8> undef, <64 x i32> <i32 0, i32 9, i32 15, i32 13, i32 11, i32 11, i32 3, i32 12, i32 4, i32 1, i32 7, i32 5, i32 2, i32 6, i32 14, i32 6, i32 23, i32 27, i32 24, i32 18, i32 30, i32 23, i32 28, i32 22, i32 28, i32 22, i32 19, i32 19, i32 31, i32 25, i32 16, i32 22, i32 35, i32 33, i32 34, i32 32, i32 42, i32 34, i32 41, i32 41, i32 43, i32 40, i32 36, i32 46, i32 37, i32 39, i32 42, i32 40, i32 63, i32 63, i32 62, i32 62, i32 57, i32 55, i32 59, i32 51, i32 52, i32 48, i32 50, i32 48, i32 58, i32 50, i32 60, i32 58>
-  %cmp = icmp eq <64 x i8> %mask, zeroinitializer
-  %res = select <64 x i1> %cmp, <64 x i8> %shuf, <64 x i8> zeroinitializer
-  ret <64 x i8> %res
-}
-
-define <64 x i8> @test_masked_64xi8_perm_mem_mask1(<64 x i8>* %vp, <64 x i8> %vec2, <64 x i8> %mask) {
-; GENERIC-LABEL: test_masked_64xi8_perm_mem_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovdqa64 (%rdi), %zmm2 # sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmb %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufb {{.*#+}} zmm0 {%k1} = zmm2[15,6,14,7,5,1,14,12,5,7,5,0,0,5,3,8,19,19,26,27,20,29,20,21,27,16,30,17,23,27,16,28,47,39,33,33,33,44,38,46,39,33,38,44,45,32,34,39,50,61,62,53,54,56,52,56,51,52,55,57,56,52,51,49] sched: [8:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_64xi8_perm_mem_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovdqa64 (%rdi), %zmm2 # sched: [8:0.50]
-; SKX-NEXT:    vptestnmb %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufb {{.*#+}} zmm0 {%k1} = zmm2[15,6,14,7,5,1,14,12,5,7,5,0,0,5,3,8,19,19,26,27,20,29,20,21,27,16,30,17,23,27,16,28,47,39,33,33,33,44,38,46,39,33,38,44,45,32,34,39,50,61,62,53,54,56,52,56,51,52,55,57,56,52,51,49] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <64 x i8>, <64 x i8>* %vp
-  %shuf = shufflevector <64 x i8> %vec, <64 x i8> undef, <64 x i32> <i32 15, i32 6, i32 14, i32 7, i32 5, i32 1, i32 14, i32 12, i32 5, i32 7, i32 5, i32 0, i32 0, i32 5, i32 3, i32 8, i32 19, i32 19, i32 26, i32 27, i32 20, i32 29, i32 20, i32 21, i32 27, i32 16, i32 30, i32 17, i32 23, i32 27, i32 16, i32 28, i32 47, i32 39, i32 33, i32 33, i32 33, i32 44, i32 38, i32 46, i32 39, i32 33, i32 38, i32 44, i32 45, i32 32, i32 34, i32 39, i32 50, i32 61, i32 62, i32 53, i32 54, i32 56, i32 52, i32 56, i32 51, i32 52, i32 55, i32 57, i32 56, i32 52, i32 51, i32 49>
-  %cmp = icmp eq <64 x i8> %mask, zeroinitializer
-  %res = select <64 x i1> %cmp, <64 x i8> %shuf, <64 x i8> %vec2
-  ret <64 x i8> %res
-}
-
-define <64 x i8> @test_masked_z_64xi8_perm_mem_mask1(<64 x i8>* %vp, <64 x i8> %mask) {
-; GENERIC-LABEL: test_masked_z_64xi8_perm_mem_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovdqa64 (%rdi), %zmm1 # sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmb %zmm0, %zmm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufb {{.*#+}} zmm0 {%k1} {z} = zmm1[15,6,14,7,5,1,14,12,5,7,5,0,0,5,3,8,19,19,26,27,20,29,20,21,27,16,30,17,23,27,16,28,47,39,33,33,33,44,38,46,39,33,38,44,45,32,34,39,50,61,62,53,54,56,52,56,51,52,55,57,56,52,51,49] sched: [8:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_64xi8_perm_mem_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovdqa64 (%rdi), %zmm1 # sched: [8:0.50]
-; SKX-NEXT:    vptestnmb %zmm0, %zmm0, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufb {{.*#+}} zmm0 {%k1} {z} = zmm1[15,6,14,7,5,1,14,12,5,7,5,0,0,5,3,8,19,19,26,27,20,29,20,21,27,16,30,17,23,27,16,28,47,39,33,33,33,44,38,46,39,33,38,44,45,32,34,39,50,61,62,53,54,56,52,56,51,52,55,57,56,52,51,49] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <64 x i8>, <64 x i8>* %vp
-  %shuf = shufflevector <64 x i8> %vec, <64 x i8> undef, <64 x i32> <i32 15, i32 6, i32 14, i32 7, i32 5, i32 1, i32 14, i32 12, i32 5, i32 7, i32 5, i32 0, i32 0, i32 5, i32 3, i32 8, i32 19, i32 19, i32 26, i32 27, i32 20, i32 29, i32 20, i32 21, i32 27, i32 16, i32 30, i32 17, i32 23, i32 27, i32 16, i32 28, i32 47, i32 39, i32 33, i32 33, i32 33, i32 44, i32 38, i32 46, i32 39, i32 33, i32 38, i32 44, i32 45, i32 32, i32 34, i32 39, i32 50, i32 61, i32 62, i32 53, i32 54, i32 56, i32 52, i32 56, i32 51, i32 52, i32 55, i32 57, i32 56, i32 52, i32 51, i32 49>
-  %cmp = icmp eq <64 x i8> %mask, zeroinitializer
-  %res = select <64 x i1> %cmp, <64 x i8> %shuf, <64 x i8> zeroinitializer
-  ret <64 x i8> %res
-}
-
-define <64 x i8> @test_masked_64xi8_perm_mem_mask2(<64 x i8>* %vp, <64 x i8> %vec2, <64 x i8> %mask) {
-; GENERIC-LABEL: test_masked_64xi8_perm_mem_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovdqa64 (%rdi), %zmm2 # sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmb %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufb {{.*#+}} zmm0 {%k1} = zmm2[12,1,11,3,4,11,10,11,8,13,1,10,1,11,5,10,27,26,19,29,19,24,26,19,26,20,18,28,24,21,25,16,34,38,47,40,33,44,44,44,41,43,35,43,45,44,37,41,58,62,49,61,56,53,55,48,51,58,58,55,63,55,53,61] sched: [8:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_64xi8_perm_mem_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovdqa64 (%rdi), %zmm2 # sched: [8:0.50]
-; SKX-NEXT:    vptestnmb %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufb {{.*#+}} zmm0 {%k1} = zmm2[12,1,11,3,4,11,10,11,8,13,1,10,1,11,5,10,27,26,19,29,19,24,26,19,26,20,18,28,24,21,25,16,34,38,47,40,33,44,44,44,41,43,35,43,45,44,37,41,58,62,49,61,56,53,55,48,51,58,58,55,63,55,53,61] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <64 x i8>, <64 x i8>* %vp
-  %shuf = shufflevector <64 x i8> %vec, <64 x i8> undef, <64 x i32> <i32 12, i32 1, i32 11, i32 3, i32 4, i32 11, i32 10, i32 11, i32 8, i32 13, i32 1, i32 10, i32 1, i32 11, i32 5, i32 10, i32 27, i32 26, i32 19, i32 29, i32 19, i32 24, i32 26, i32 19, i32 26, i32 20, i32 18, i32 28, i32 24, i32 21, i32 25, i32 16, i32 34, i32 38, i32 47, i32 40, i32 33, i32 44, i32 44, i32 44, i32 41, i32 43, i32 35, i32 43, i32 45, i32 44, i32 37, i32 41, i32 58, i32 62, i32 49, i32 61, i32 56, i32 53, i32 55, i32 48, i32 51, i32 58, i32 58, i32 55, i32 63, i32 55, i32 53, i32 61>
-  %cmp = icmp eq <64 x i8> %mask, zeroinitializer
-  %res = select <64 x i1> %cmp, <64 x i8> %shuf, <64 x i8> %vec2
-  ret <64 x i8> %res
-}
-
-define <64 x i8> @test_masked_z_64xi8_perm_mem_mask2(<64 x i8>* %vp, <64 x i8> %mask) {
-; GENERIC-LABEL: test_masked_z_64xi8_perm_mem_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovdqa64 (%rdi), %zmm1 # sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmb %zmm0, %zmm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufb {{.*#+}} zmm0 {%k1} {z} = zmm1[12,1,11,3,4,11,10,11,8,13,1,10,1,11,5,10,27,26,19,29,19,24,26,19,26,20,18,28,24,21,25,16,34,38,47,40,33,44,44,44,41,43,35,43,45,44,37,41,58,62,49,61,56,53,55,48,51,58,58,55,63,55,53,61] sched: [8:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_64xi8_perm_mem_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovdqa64 (%rdi), %zmm1 # sched: [8:0.50]
-; SKX-NEXT:    vptestnmb %zmm0, %zmm0, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufb {{.*#+}} zmm0 {%k1} {z} = zmm1[12,1,11,3,4,11,10,11,8,13,1,10,1,11,5,10,27,26,19,29,19,24,26,19,26,20,18,28,24,21,25,16,34,38,47,40,33,44,44,44,41,43,35,43,45,44,37,41,58,62,49,61,56,53,55,48,51,58,58,55,63,55,53,61] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <64 x i8>, <64 x i8>* %vp
-  %shuf = shufflevector <64 x i8> %vec, <64 x i8> undef, <64 x i32> <i32 12, i32 1, i32 11, i32 3, i32 4, i32 11, i32 10, i32 11, i32 8, i32 13, i32 1, i32 10, i32 1, i32 11, i32 5, i32 10, i32 27, i32 26, i32 19, i32 29, i32 19, i32 24, i32 26, i32 19, i32 26, i32 20, i32 18, i32 28, i32 24, i32 21, i32 25, i32 16, i32 34, i32 38, i32 47, i32 40, i32 33, i32 44, i32 44, i32 44, i32 41, i32 43, i32 35, i32 43, i32 45, i32 44, i32 37, i32 41, i32 58, i32 62, i32 49, i32 61, i32 56, i32 53, i32 55, i32 48, i32 51, i32 58, i32 58, i32 55, i32 63, i32 55, i32 53, i32 61>
-  %cmp = icmp eq <64 x i8> %mask, zeroinitializer
-  %res = select <64 x i1> %cmp, <64 x i8> %shuf, <64 x i8> zeroinitializer
-  ret <64 x i8> %res
-}
-
-define <64 x i8> @test_64xi8_perm_mem_mask3(<64 x i8>* %vp) {
-; GENERIC-LABEL: test_64xi8_perm_mem_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovdqa64 (%rdi), %zmm0 # sched: [7:0.50]
-; GENERIC-NEXT:    vpshufb {{.*#+}} zmm0 = zmm0[4,9,11,13,12,6,0,0,11,15,5,7,11,10,4,10,20,21,24,27,18,16,26,16,16,19,26,17,16,31,22,30,35,38,37,34,37,47,43,38,38,36,40,43,42,39,32,46,54,54,48,50,61,56,59,50,53,61,61,51,48,60,50,60] sched: [8:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_64xi8_perm_mem_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovdqa64 (%rdi), %zmm0 # sched: [8:0.50]
-; SKX-NEXT:    vpshufb {{.*#+}} zmm0 = zmm0[4,9,11,13,12,6,0,0,11,15,5,7,11,10,4,10,20,21,24,27,18,16,26,16,16,19,26,17,16,31,22,30,35,38,37,34,37,47,43,38,38,36,40,43,42,39,32,46,54,54,48,50,61,56,59,50,53,61,61,51,48,60,50,60] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <64 x i8>, <64 x i8>* %vp
-  %res = shufflevector <64 x i8> %vec, <64 x i8> undef, <64 x i32> <i32 4, i32 9, i32 11, i32 13, i32 12, i32 6, i32 0, i32 0, i32 11, i32 15, i32 5, i32 7, i32 11, i32 10, i32 4, i32 10, i32 20, i32 21, i32 24, i32 27, i32 18, i32 16, i32 26, i32 16, i32 16, i32 19, i32 26, i32 17, i32 16, i32 31, i32 22, i32 30, i32 35, i32 38, i32 37, i32 34, i32 37, i32 47, i32 43, i32 38, i32 38, i32 36, i32 40, i32 43, i32 42, i32 39, i32 32, i32 46, i32 54, i32 54, i32 48, i32 50, i32 61, i32 56, i32 59, i32 50, i32 53, i32 61, i32 61, i32 51, i32 48, i32 60, i32 50, i32 60>
-  ret <64 x i8> %res
-}
-define <64 x i8> @test_masked_64xi8_perm_mem_mask3(<64 x i8>* %vp, <64 x i8> %vec2, <64 x i8> %mask) {
-; GENERIC-LABEL: test_masked_64xi8_perm_mem_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovdqa64 (%rdi), %zmm2 # sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmb %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufb {{.*#+}} zmm0 {%k1} = zmm2[4,9,11,13,12,6,0,0,11,15,5,7,11,10,4,10,20,21,24,27,18,16,26,16,16,19,26,17,16,31,22,30,35,38,37,34,37,47,43,38,38,36,40,43,42,39,32,46,54,54,48,50,61,56,59,50,53,61,61,51,48,60,50,60] sched: [8:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_64xi8_perm_mem_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovdqa64 (%rdi), %zmm2 # sched: [8:0.50]
-; SKX-NEXT:    vptestnmb %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufb {{.*#+}} zmm0 {%k1} = zmm2[4,9,11,13,12,6,0,0,11,15,5,7,11,10,4,10,20,21,24,27,18,16,26,16,16,19,26,17,16,31,22,30,35,38,37,34,37,47,43,38,38,36,40,43,42,39,32,46,54,54,48,50,61,56,59,50,53,61,61,51,48,60,50,60] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <64 x i8>, <64 x i8>* %vp
-  %shuf = shufflevector <64 x i8> %vec, <64 x i8> undef, <64 x i32> <i32 4, i32 9, i32 11, i32 13, i32 12, i32 6, i32 0, i32 0, i32 11, i32 15, i32 5, i32 7, i32 11, i32 10, i32 4, i32 10, i32 20, i32 21, i32 24, i32 27, i32 18, i32 16, i32 26, i32 16, i32 16, i32 19, i32 26, i32 17, i32 16, i32 31, i32 22, i32 30, i32 35, i32 38, i32 37, i32 34, i32 37, i32 47, i32 43, i32 38, i32 38, i32 36, i32 40, i32 43, i32 42, i32 39, i32 32, i32 46, i32 54, i32 54, i32 48, i32 50, i32 61, i32 56, i32 59, i32 50, i32 53, i32 61, i32 61, i32 51, i32 48, i32 60, i32 50, i32 60>
-  %cmp = icmp eq <64 x i8> %mask, zeroinitializer
-  %res = select <64 x i1> %cmp, <64 x i8> %shuf, <64 x i8> %vec2
-  ret <64 x i8> %res
-}
-
-define <64 x i8> @test_masked_z_64xi8_perm_mem_mask3(<64 x i8>* %vp, <64 x i8> %mask) {
-; GENERIC-LABEL: test_masked_z_64xi8_perm_mem_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovdqa64 (%rdi), %zmm1 # sched: [7:0.50]
-; GENERIC-NEXT:    vptestnmb %zmm0, %zmm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufb {{.*#+}} zmm0 {%k1} {z} = zmm1[4,9,11,13,12,6,0,0,11,15,5,7,11,10,4,10,20,21,24,27,18,16,26,16,16,19,26,17,16,31,22,30,35,38,37,34,37,47,43,38,38,36,40,43,42,39,32,46,54,54,48,50,61,56,59,50,53,61,61,51,48,60,50,60] sched: [8:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_64xi8_perm_mem_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovdqa64 (%rdi), %zmm1 # sched: [8:0.50]
-; SKX-NEXT:    vptestnmb %zmm0, %zmm0, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufb {{.*#+}} zmm0 {%k1} {z} = zmm1[4,9,11,13,12,6,0,0,11,15,5,7,11,10,4,10,20,21,24,27,18,16,26,16,16,19,26,17,16,31,22,30,35,38,37,34,37,47,43,38,38,36,40,43,42,39,32,46,54,54,48,50,61,56,59,50,53,61,61,51,48,60,50,60] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <64 x i8>, <64 x i8>* %vp
-  %shuf = shufflevector <64 x i8> %vec, <64 x i8> undef, <64 x i32> <i32 4, i32 9, i32 11, i32 13, i32 12, i32 6, i32 0, i32 0, i32 11, i32 15, i32 5, i32 7, i32 11, i32 10, i32 4, i32 10, i32 20, i32 21, i32 24, i32 27, i32 18, i32 16, i32 26, i32 16, i32 16, i32 19, i32 26, i32 17, i32 16, i32 31, i32 22, i32 30, i32 35, i32 38, i32 37, i32 34, i32 37, i32 47, i32 43, i32 38, i32 38, i32 36, i32 40, i32 43, i32 42, i32 39, i32 32, i32 46, i32 54, i32 54, i32 48, i32 50, i32 61, i32 56, i32 59, i32 50, i32 53, i32 61, i32 61, i32 51, i32 48, i32 60, i32 50, i32 60>
-  %cmp = icmp eq <64 x i8> %mask, zeroinitializer
-  %res = select <64 x i1> %cmp, <64 x i8> %shuf, <64 x i8> zeroinitializer
-  ret <64 x i8> %res
-}
-
-define <8 x i16> @test_8xi16_perm_high_mask0(<8 x i16> %vec) {
-; GENERIC-LABEL: test_8xi16_perm_high_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,6,5,7,6] sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xi16_perm_high_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,6,5,7,6] sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %res = shufflevector <8 x i16> %vec, <8 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 6, i32 5, i32 7, i32 6>
-  ret <8 x i16> %res
-}
-define <8 x i16> @test_masked_8xi16_perm_high_mask0(<8 x i16> %vec, <8 x i16> %vec2, <8 x i16> %mask) {
-; GENERIC-LABEL: test_masked_8xi16_perm_high_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmw %xmm2, %xmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufhw {{.*#+}} xmm1 {%k1} = xmm0[0,1,2,3,6,5,7,6] sched: [1:0.50]
-; GENERIC-NEXT:    vmovdqa %xmm1, %xmm0 # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_8xi16_perm_high_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmw %xmm2, %xmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufhw {{.*#+}} xmm1 {%k1} = xmm0[0,1,2,3,6,5,7,6] sched: [1:1.00]
-; SKX-NEXT:    vmovdqa %xmm1, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x i16> %vec, <8 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 6, i32 5, i32 7, i32 6>
-  %cmp = icmp eq <8 x i16> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i16> %shuf, <8 x i16> %vec2
-  ret <8 x i16> %res
-}
-
-define <8 x i16> @test_masked_z_8xi16_perm_high_mask0(<8 x i16> %vec, <8 x i16> %mask) {
-; GENERIC-LABEL: test_masked_z_8xi16_perm_high_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmw %xmm1, %xmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufhw {{.*#+}} xmm0 {%k1} {z} = xmm0[0,1,2,3,6,5,7,6] sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_8xi16_perm_high_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmw %xmm1, %xmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufhw {{.*#+}} xmm0 {%k1} {z} = xmm0[0,1,2,3,6,5,7,6] sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x i16> %vec, <8 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 6, i32 5, i32 7, i32 6>
-  %cmp = icmp eq <8 x i16> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i16> %shuf, <8 x i16> zeroinitializer
-  ret <8 x i16> %res
-}
-define <8 x i16> @test_masked_8xi16_perm_low_mask1(<8 x i16> %vec, <8 x i16> %vec2, <8 x i16> %mask) {
-; GENERIC-LABEL: test_masked_8xi16_perm_low_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmw %xmm2, %xmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshuflw {{.*#+}} xmm1 {%k1} = xmm0[0,3,0,0,4,5,6,7] sched: [1:0.50]
-; GENERIC-NEXT:    vmovdqa %xmm1, %xmm0 # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_8xi16_perm_low_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmw %xmm2, %xmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshuflw {{.*#+}} xmm1 {%k1} = xmm0[0,3,0,0,4,5,6,7] sched: [1:1.00]
-; SKX-NEXT:    vmovdqa %xmm1, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x i16> %vec, <8 x i16> undef, <8 x i32> <i32 0, i32 3, i32 0, i32 0, i32 4, i32 5, i32 6, i32 7>
-  %cmp = icmp eq <8 x i16> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i16> %shuf, <8 x i16> %vec2
-  ret <8 x i16> %res
-}
-
-define <8 x i16> @test_masked_z_8xi16_perm_low_mask1(<8 x i16> %vec, <8 x i16> %mask) {
-; GENERIC-LABEL: test_masked_z_8xi16_perm_low_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmw %xmm1, %xmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshuflw {{.*#+}} xmm0 {%k1} {z} = xmm0[0,3,0,0,4,5,6,7] sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_8xi16_perm_low_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmw %xmm1, %xmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshuflw {{.*#+}} xmm0 {%k1} {z} = xmm0[0,3,0,0,4,5,6,7] sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x i16> %vec, <8 x i16> undef, <8 x i32> <i32 0, i32 3, i32 0, i32 0, i32 4, i32 5, i32 6, i32 7>
-  %cmp = icmp eq <8 x i16> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i16> %shuf, <8 x i16> zeroinitializer
-  ret <8 x i16> %res
-}
-define <8 x i16> @test_masked_8xi16_perm_high_mask2(<8 x i16> %vec, <8 x i16> %vec2, <8 x i16> %mask) {
-; GENERIC-LABEL: test_masked_8xi16_perm_high_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmw %xmm2, %xmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufhw {{.*#+}} xmm1 {%k1} = xmm0[0,1,2,3,5,4,4,5] sched: [1:0.50]
-; GENERIC-NEXT:    vmovdqa %xmm1, %xmm0 # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_8xi16_perm_high_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmw %xmm2, %xmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufhw {{.*#+}} xmm1 {%k1} = xmm0[0,1,2,3,5,4,4,5] sched: [1:1.00]
-; SKX-NEXT:    vmovdqa %xmm1, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x i16> %vec, <8 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 5, i32 4, i32 4, i32 5>
-  %cmp = icmp eq <8 x i16> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i16> %shuf, <8 x i16> %vec2
-  ret <8 x i16> %res
-}
-
-define <8 x i16> @test_masked_z_8xi16_perm_high_mask2(<8 x i16> %vec, <8 x i16> %mask) {
-; GENERIC-LABEL: test_masked_z_8xi16_perm_high_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmw %xmm1, %xmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufhw {{.*#+}} xmm0 {%k1} {z} = xmm0[0,1,2,3,5,4,4,5] sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_8xi16_perm_high_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmw %xmm1, %xmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufhw {{.*#+}} xmm0 {%k1} {z} = xmm0[0,1,2,3,5,4,4,5] sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x i16> %vec, <8 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 5, i32 4, i32 4, i32 5>
-  %cmp = icmp eq <8 x i16> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i16> %shuf, <8 x i16> zeroinitializer
-  ret <8 x i16> %res
-}
-define <8 x i16> @test_8xi16_perm_low_mask3(<8 x i16> %vec) {
-; GENERIC-LABEL: test_8xi16_perm_low_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[2,1,1,1,4,5,6,7] sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xi16_perm_low_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[2,1,1,1,4,5,6,7] sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %res = shufflevector <8 x i16> %vec, <8 x i16> undef, <8 x i32> <i32 2, i32 1, i32 1, i32 1, i32 4, i32 5, i32 6, i32 7>
-  ret <8 x i16> %res
-}
-define <8 x i16> @test_masked_8xi16_perm_low_mask3(<8 x i16> %vec, <8 x i16> %vec2, <8 x i16> %mask) {
-; GENERIC-LABEL: test_masked_8xi16_perm_low_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmw %xmm2, %xmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshuflw {{.*#+}} xmm1 {%k1} = xmm0[2,1,1,1,4,5,6,7] sched: [1:0.50]
-; GENERIC-NEXT:    vmovdqa %xmm1, %xmm0 # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_8xi16_perm_low_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmw %xmm2, %xmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshuflw {{.*#+}} xmm1 {%k1} = xmm0[2,1,1,1,4,5,6,7] sched: [1:1.00]
-; SKX-NEXT:    vmovdqa %xmm1, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x i16> %vec, <8 x i16> undef, <8 x i32> <i32 2, i32 1, i32 1, i32 1, i32 4, i32 5, i32 6, i32 7>
-  %cmp = icmp eq <8 x i16> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i16> %shuf, <8 x i16> %vec2
-  ret <8 x i16> %res
-}
-
-define <8 x i16> @test_masked_z_8xi16_perm_low_mask3(<8 x i16> %vec, <8 x i16> %mask) {
-; GENERIC-LABEL: test_masked_z_8xi16_perm_low_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmw %xmm1, %xmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshuflw {{.*#+}} xmm0 {%k1} {z} = xmm0[2,1,1,1,4,5,6,7] sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_8xi16_perm_low_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmw %xmm1, %xmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshuflw {{.*#+}} xmm0 {%k1} {z} = xmm0[2,1,1,1,4,5,6,7] sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x i16> %vec, <8 x i16> undef, <8 x i32> <i32 2, i32 1, i32 1, i32 1, i32 4, i32 5, i32 6, i32 7>
-  %cmp = icmp eq <8 x i16> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i16> %shuf, <8 x i16> zeroinitializer
-  ret <8 x i16> %res
-}
-define <8 x i16> @test_masked_8xi16_perm_high_mask4(<8 x i16> %vec, <8 x i16> %vec2, <8 x i16> %mask) {
-; GENERIC-LABEL: test_masked_8xi16_perm_high_mask4:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmw %xmm2, %xmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufhw {{.*#+}} xmm1 {%k1} = xmm0[0,1,2,3,5,5,7,6] sched: [1:0.50]
-; GENERIC-NEXT:    vmovdqa %xmm1, %xmm0 # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_8xi16_perm_high_mask4:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmw %xmm2, %xmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufhw {{.*#+}} xmm1 {%k1} = xmm0[0,1,2,3,5,5,7,6] sched: [1:1.00]
-; SKX-NEXT:    vmovdqa %xmm1, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x i16> %vec, <8 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 5, i32 5, i32 7, i32 6>
-  %cmp = icmp eq <8 x i16> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i16> %shuf, <8 x i16> %vec2
-  ret <8 x i16> %res
-}
-
-define <8 x i16> @test_masked_z_8xi16_perm_high_mask4(<8 x i16> %vec, <8 x i16> %mask) {
-; GENERIC-LABEL: test_masked_z_8xi16_perm_high_mask4:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmw %xmm1, %xmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufhw {{.*#+}} xmm0 {%k1} {z} = xmm0[0,1,2,3,5,5,7,6] sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_8xi16_perm_high_mask4:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmw %xmm1, %xmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufhw {{.*#+}} xmm0 {%k1} {z} = xmm0[0,1,2,3,5,5,7,6] sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x i16> %vec, <8 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 5, i32 5, i32 7, i32 6>
-  %cmp = icmp eq <8 x i16> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i16> %shuf, <8 x i16> zeroinitializer
-  ret <8 x i16> %res
-}
-define <8 x i16> @test_masked_8xi16_perm_low_mask5(<8 x i16> %vec, <8 x i16> %vec2, <8 x i16> %mask) {
-; GENERIC-LABEL: test_masked_8xi16_perm_low_mask5:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmw %xmm2, %xmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshuflw {{.*#+}} xmm1 {%k1} = xmm0[3,3,2,1,4,5,6,7] sched: [1:0.50]
-; GENERIC-NEXT:    vmovdqa %xmm1, %xmm0 # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_8xi16_perm_low_mask5:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmw %xmm2, %xmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshuflw {{.*#+}} xmm1 {%k1} = xmm0[3,3,2,1,4,5,6,7] sched: [1:1.00]
-; SKX-NEXT:    vmovdqa %xmm1, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x i16> %vec, <8 x i16> undef, <8 x i32> <i32 3, i32 3, i32 2, i32 1, i32 4, i32 5, i32 6, i32 7>
-  %cmp = icmp eq <8 x i16> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i16> %shuf, <8 x i16> %vec2
-  ret <8 x i16> %res
-}
-
-define <8 x i16> @test_masked_z_8xi16_perm_low_mask5(<8 x i16> %vec, <8 x i16> %mask) {
-; GENERIC-LABEL: test_masked_z_8xi16_perm_low_mask5:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmw %xmm1, %xmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshuflw {{.*#+}} xmm0 {%k1} {z} = xmm0[3,3,2,1,4,5,6,7] sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_8xi16_perm_low_mask5:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmw %xmm1, %xmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshuflw {{.*#+}} xmm0 {%k1} {z} = xmm0[3,3,2,1,4,5,6,7] sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x i16> %vec, <8 x i16> undef, <8 x i32> <i32 3, i32 3, i32 2, i32 1, i32 4, i32 5, i32 6, i32 7>
-  %cmp = icmp eq <8 x i16> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i16> %shuf, <8 x i16> zeroinitializer
-  ret <8 x i16> %res
-}
-define <8 x i16> @test_8xi16_perm_high_mask6(<8 x i16> %vec) {
-; GENERIC-LABEL: test_8xi16_perm_high_mask6:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,6,5,6,5] sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xi16_perm_high_mask6:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,6,5,6,5] sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %res = shufflevector <8 x i16> %vec, <8 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 6, i32 5, i32 6, i32 5>
-  ret <8 x i16> %res
-}
-define <8 x i16> @test_masked_8xi16_perm_high_mask6(<8 x i16> %vec, <8 x i16> %vec2, <8 x i16> %mask) {
-; GENERIC-LABEL: test_masked_8xi16_perm_high_mask6:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmw %xmm2, %xmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufhw {{.*#+}} xmm1 {%k1} = xmm0[0,1,2,3,6,5,6,5] sched: [1:0.50]
-; GENERIC-NEXT:    vmovdqa %xmm1, %xmm0 # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_8xi16_perm_high_mask6:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmw %xmm2, %xmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufhw {{.*#+}} xmm1 {%k1} = xmm0[0,1,2,3,6,5,6,5] sched: [1:1.00]
-; SKX-NEXT:    vmovdqa %xmm1, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x i16> %vec, <8 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 6, i32 5, i32 6, i32 5>
-  %cmp = icmp eq <8 x i16> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i16> %shuf, <8 x i16> %vec2
-  ret <8 x i16> %res
-}
-
-define <8 x i16> @test_masked_z_8xi16_perm_high_mask6(<8 x i16> %vec, <8 x i16> %mask) {
-; GENERIC-LABEL: test_masked_z_8xi16_perm_high_mask6:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmw %xmm1, %xmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufhw {{.*#+}} xmm0 {%k1} {z} = xmm0[0,1,2,3,6,5,6,5] sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_8xi16_perm_high_mask6:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmw %xmm1, %xmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufhw {{.*#+}} xmm0 {%k1} {z} = xmm0[0,1,2,3,6,5,6,5] sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x i16> %vec, <8 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 6, i32 5, i32 6, i32 5>
-  %cmp = icmp eq <8 x i16> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i16> %shuf, <8 x i16> zeroinitializer
-  ret <8 x i16> %res
-}
-define <8 x i16> @test_masked_8xi16_perm_low_mask7(<8 x i16> %vec, <8 x i16> %vec2, <8 x i16> %mask) {
-; GENERIC-LABEL: test_masked_8xi16_perm_low_mask7:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmw %xmm2, %xmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshuflw {{.*#+}} xmm1 {%k1} = xmm0[1,0,2,0,4,5,6,7] sched: [1:0.50]
-; GENERIC-NEXT:    vmovdqa %xmm1, %xmm0 # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_8xi16_perm_low_mask7:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmw %xmm2, %xmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshuflw {{.*#+}} xmm1 {%k1} = xmm0[1,0,2,0,4,5,6,7] sched: [1:1.00]
-; SKX-NEXT:    vmovdqa %xmm1, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x i16> %vec, <8 x i16> undef, <8 x i32> <i32 1, i32 0, i32 2, i32 0, i32 4, i32 5, i32 6, i32 7>
-  %cmp = icmp eq <8 x i16> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i16> %shuf, <8 x i16> %vec2
-  ret <8 x i16> %res
-}
-
-define <8 x i16> @test_masked_z_8xi16_perm_low_mask7(<8 x i16> %vec, <8 x i16> %mask) {
-; GENERIC-LABEL: test_masked_z_8xi16_perm_low_mask7:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmw %xmm1, %xmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshuflw {{.*#+}} xmm0 {%k1} {z} = xmm0[1,0,2,0,4,5,6,7] sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_8xi16_perm_low_mask7:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmw %xmm1, %xmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshuflw {{.*#+}} xmm0 {%k1} {z} = xmm0[1,0,2,0,4,5,6,7] sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x i16> %vec, <8 x i16> undef, <8 x i32> <i32 1, i32 0, i32 2, i32 0, i32 4, i32 5, i32 6, i32 7>
-  %cmp = icmp eq <8 x i16> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i16> %shuf, <8 x i16> zeroinitializer
-  ret <8 x i16> %res
-}
-define <8 x i16> @test_8xi16_perm_high_mem_mask0(<8 x i16>* %vp) {
-; GENERIC-LABEL: test_8xi16_perm_high_mem_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpshufhw {{.*#+}} xmm0 = mem[0,1,2,3,7,7,4,6] sched: [7:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xi16_perm_high_mem_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpshufhw {{.*#+}} xmm0 = mem[0,1,2,3,7,7,4,6] sched: [7:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <8 x i16>, <8 x i16>* %vp
-  %res = shufflevector <8 x i16> %vec, <8 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 7, i32 7, i32 4, i32 6>
-  ret <8 x i16> %res
-}
-define <8 x i16> @test_masked_8xi16_perm_high_mem_mask0(<8 x i16>* %vp, <8 x i16> %vec2, <8 x i16> %mask) {
-; GENERIC-LABEL: test_masked_8xi16_perm_high_mem_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmw %xmm1, %xmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufhw {{.*#+}} xmm0 {%k1} = mem[0,1,2,3,7,7,4,6] sched: [7:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_8xi16_perm_high_mem_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmw %xmm1, %xmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufhw {{.*#+}} xmm0 {%k1} = mem[0,1,2,3,7,7,4,6] sched: [7:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <8 x i16>, <8 x i16>* %vp
-  %shuf = shufflevector <8 x i16> %vec, <8 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 7, i32 7, i32 4, i32 6>
-  %cmp = icmp eq <8 x i16> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i16> %shuf, <8 x i16> %vec2
-  ret <8 x i16> %res
-}
-
-define <8 x i16> @test_masked_z_8xi16_perm_high_mem_mask0(<8 x i16>* %vp, <8 x i16> %mask) {
-; GENERIC-LABEL: test_masked_z_8xi16_perm_high_mem_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmw %xmm0, %xmm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufhw {{.*#+}} xmm0 {%k1} {z} = mem[0,1,2,3,7,7,4,6] sched: [7:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_8xi16_perm_high_mem_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmw %xmm0, %xmm0, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufhw {{.*#+}} xmm0 {%k1} {z} = mem[0,1,2,3,7,7,4,6] sched: [7:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <8 x i16>, <8 x i16>* %vp
-  %shuf = shufflevector <8 x i16> %vec, <8 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 7, i32 7, i32 4, i32 6>
-  %cmp = icmp eq <8 x i16> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i16> %shuf, <8 x i16> zeroinitializer
-  ret <8 x i16> %res
-}
-
-define <8 x i16> @test_masked_8xi16_perm_low_mem_mask1(<8 x i16>* %vp, <8 x i16> %vec2, <8 x i16> %mask) {
-; GENERIC-LABEL: test_masked_8xi16_perm_low_mem_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmw %xmm1, %xmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshuflw {{.*#+}} xmm0 {%k1} = mem[1,3,3,2,4,5,6,7] sched: [7:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_8xi16_perm_low_mem_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmw %xmm1, %xmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshuflw {{.*#+}} xmm0 {%k1} = mem[1,3,3,2,4,5,6,7] sched: [7:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <8 x i16>, <8 x i16>* %vp
-  %shuf = shufflevector <8 x i16> %vec, <8 x i16> undef, <8 x i32> <i32 1, i32 3, i32 3, i32 2, i32 4, i32 5, i32 6, i32 7>
-  %cmp = icmp eq <8 x i16> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i16> %shuf, <8 x i16> %vec2
-  ret <8 x i16> %res
-}
-
-define <8 x i16> @test_masked_z_8xi16_perm_low_mem_mask1(<8 x i16>* %vp, <8 x i16> %mask) {
-; GENERIC-LABEL: test_masked_z_8xi16_perm_low_mem_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmw %xmm0, %xmm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshuflw {{.*#+}} xmm0 {%k1} {z} = mem[1,3,3,2,4,5,6,7] sched: [7:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_8xi16_perm_low_mem_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmw %xmm0, %xmm0, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshuflw {{.*#+}} xmm0 {%k1} {z} = mem[1,3,3,2,4,5,6,7] sched: [7:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <8 x i16>, <8 x i16>* %vp
-  %shuf = shufflevector <8 x i16> %vec, <8 x i16> undef, <8 x i32> <i32 1, i32 3, i32 3, i32 2, i32 4, i32 5, i32 6, i32 7>
-  %cmp = icmp eq <8 x i16> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i16> %shuf, <8 x i16> zeroinitializer
-  ret <8 x i16> %res
-}
-
-define <8 x i16> @test_masked_8xi16_perm_high_mem_mask2(<8 x i16>* %vp, <8 x i16> %vec2, <8 x i16> %mask) {
-; GENERIC-LABEL: test_masked_8xi16_perm_high_mem_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmw %xmm1, %xmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufhw {{.*#+}} xmm0 {%k1} = mem[0,1,2,3,6,6,5,7] sched: [7:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_8xi16_perm_high_mem_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmw %xmm1, %xmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufhw {{.*#+}} xmm0 {%k1} = mem[0,1,2,3,6,6,5,7] sched: [7:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <8 x i16>, <8 x i16>* %vp
-  %shuf = shufflevector <8 x i16> %vec, <8 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 6, i32 6, i32 5, i32 7>
-  %cmp = icmp eq <8 x i16> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i16> %shuf, <8 x i16> %vec2
-  ret <8 x i16> %res
-}
-
-define <8 x i16> @test_masked_z_8xi16_perm_high_mem_mask2(<8 x i16>* %vp, <8 x i16> %mask) {
-; GENERIC-LABEL: test_masked_z_8xi16_perm_high_mem_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmw %xmm0, %xmm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufhw {{.*#+}} xmm0 {%k1} {z} = mem[0,1,2,3,6,6,5,7] sched: [7:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_8xi16_perm_high_mem_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmw %xmm0, %xmm0, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufhw {{.*#+}} xmm0 {%k1} {z} = mem[0,1,2,3,6,6,5,7] sched: [7:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <8 x i16>, <8 x i16>* %vp
-  %shuf = shufflevector <8 x i16> %vec, <8 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 6, i32 6, i32 5, i32 7>
-  %cmp = icmp eq <8 x i16> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i16> %shuf, <8 x i16> zeroinitializer
-  ret <8 x i16> %res
-}
-
-define <8 x i16> @test_8xi16_perm_low_mem_mask3(<8 x i16>* %vp) {
-; GENERIC-LABEL: test_8xi16_perm_low_mem_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpshuflw {{.*#+}} xmm0 = mem[3,1,2,0,4,5,6,7] sched: [7:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xi16_perm_low_mem_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpshuflw {{.*#+}} xmm0 = mem[3,1,2,0,4,5,6,7] sched: [7:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <8 x i16>, <8 x i16>* %vp
-  %res = shufflevector <8 x i16> %vec, <8 x i16> undef, <8 x i32> <i32 3, i32 1, i32 2, i32 0, i32 4, i32 5, i32 6, i32 7>
-  ret <8 x i16> %res
-}
-define <8 x i16> @test_masked_8xi16_perm_low_mem_mask3(<8 x i16>* %vp, <8 x i16> %vec2, <8 x i16> %mask) {
-; GENERIC-LABEL: test_masked_8xi16_perm_low_mem_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmw %xmm1, %xmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshuflw {{.*#+}} xmm0 {%k1} = mem[3,1,2,0,4,5,6,7] sched: [7:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_8xi16_perm_low_mem_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmw %xmm1, %xmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshuflw {{.*#+}} xmm0 {%k1} = mem[3,1,2,0,4,5,6,7] sched: [7:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <8 x i16>, <8 x i16>* %vp
-  %shuf = shufflevector <8 x i16> %vec, <8 x i16> undef, <8 x i32> <i32 3, i32 1, i32 2, i32 0, i32 4, i32 5, i32 6, i32 7>
-  %cmp = icmp eq <8 x i16> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i16> %shuf, <8 x i16> %vec2
-  ret <8 x i16> %res
-}
-
-define <8 x i16> @test_masked_z_8xi16_perm_low_mem_mask3(<8 x i16>* %vp, <8 x i16> %mask) {
-; GENERIC-LABEL: test_masked_z_8xi16_perm_low_mem_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmw %xmm0, %xmm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshuflw {{.*#+}} xmm0 {%k1} {z} = mem[3,1,2,0,4,5,6,7] sched: [7:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_8xi16_perm_low_mem_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmw %xmm0, %xmm0, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshuflw {{.*#+}} xmm0 {%k1} {z} = mem[3,1,2,0,4,5,6,7] sched: [7:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <8 x i16>, <8 x i16>* %vp
-  %shuf = shufflevector <8 x i16> %vec, <8 x i16> undef, <8 x i32> <i32 3, i32 1, i32 2, i32 0, i32 4, i32 5, i32 6, i32 7>
-  %cmp = icmp eq <8 x i16> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i16> %shuf, <8 x i16> zeroinitializer
-  ret <8 x i16> %res
-}
-
-define <8 x i16> @test_masked_8xi16_perm_high_mem_mask4(<8 x i16>* %vp, <8 x i16> %vec2, <8 x i16> %mask) {
-; GENERIC-LABEL: test_masked_8xi16_perm_high_mem_mask4:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmw %xmm1, %xmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufhw {{.*#+}} xmm0 {%k1} = mem[0,1,2,3,7,6,7,5] sched: [7:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_8xi16_perm_high_mem_mask4:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmw %xmm1, %xmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufhw {{.*#+}} xmm0 {%k1} = mem[0,1,2,3,7,6,7,5] sched: [7:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <8 x i16>, <8 x i16>* %vp
-  %shuf = shufflevector <8 x i16> %vec, <8 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 7, i32 6, i32 7, i32 5>
-  %cmp = icmp eq <8 x i16> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i16> %shuf, <8 x i16> %vec2
-  ret <8 x i16> %res
-}
-
-define <8 x i16> @test_masked_z_8xi16_perm_high_mem_mask4(<8 x i16>* %vp, <8 x i16> %mask) {
-; GENERIC-LABEL: test_masked_z_8xi16_perm_high_mem_mask4:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmw %xmm0, %xmm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufhw {{.*#+}} xmm0 {%k1} {z} = mem[0,1,2,3,7,6,7,5] sched: [7:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_8xi16_perm_high_mem_mask4:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmw %xmm0, %xmm0, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufhw {{.*#+}} xmm0 {%k1} {z} = mem[0,1,2,3,7,6,7,5] sched: [7:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <8 x i16>, <8 x i16>* %vp
-  %shuf = shufflevector <8 x i16> %vec, <8 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 7, i32 6, i32 7, i32 5>
-  %cmp = icmp eq <8 x i16> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i16> %shuf, <8 x i16> zeroinitializer
-  ret <8 x i16> %res
-}
-
-define <8 x i16> @test_masked_8xi16_perm_low_mem_mask5(<8 x i16>* %vp, <8 x i16> %vec2, <8 x i16> %mask) {
-; GENERIC-LABEL: test_masked_8xi16_perm_low_mem_mask5:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmw %xmm1, %xmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshuflw {{.*#+}} xmm0 {%k1} = mem[2,1,3,2,4,5,6,7] sched: [7:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_8xi16_perm_low_mem_mask5:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmw %xmm1, %xmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshuflw {{.*#+}} xmm0 {%k1} = mem[2,1,3,2,4,5,6,7] sched: [7:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <8 x i16>, <8 x i16>* %vp
-  %shuf = shufflevector <8 x i16> %vec, <8 x i16> undef, <8 x i32> <i32 2, i32 1, i32 3, i32 2, i32 4, i32 5, i32 6, i32 7>
-  %cmp = icmp eq <8 x i16> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i16> %shuf, <8 x i16> %vec2
-  ret <8 x i16> %res
-}
-
-define <8 x i16> @test_masked_z_8xi16_perm_low_mem_mask5(<8 x i16>* %vp, <8 x i16> %mask) {
-; GENERIC-LABEL: test_masked_z_8xi16_perm_low_mem_mask5:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmw %xmm0, %xmm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshuflw {{.*#+}} xmm0 {%k1} {z} = mem[2,1,3,2,4,5,6,7] sched: [7:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_8xi16_perm_low_mem_mask5:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmw %xmm0, %xmm0, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshuflw {{.*#+}} xmm0 {%k1} {z} = mem[2,1,3,2,4,5,6,7] sched: [7:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <8 x i16>, <8 x i16>* %vp
-  %shuf = shufflevector <8 x i16> %vec, <8 x i16> undef, <8 x i32> <i32 2, i32 1, i32 3, i32 2, i32 4, i32 5, i32 6, i32 7>
-  %cmp = icmp eq <8 x i16> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i16> %shuf, <8 x i16> zeroinitializer
-  ret <8 x i16> %res
-}
-
-define <8 x i16> @test_8xi16_perm_high_mem_mask6(<8 x i16>* %vp) {
-; GENERIC-LABEL: test_8xi16_perm_high_mem_mask6:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpshufhw {{.*#+}} xmm0 = mem[0,1,2,3,7,4,4,4] sched: [7:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xi16_perm_high_mem_mask6:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpshufhw {{.*#+}} xmm0 = mem[0,1,2,3,7,4,4,4] sched: [7:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <8 x i16>, <8 x i16>* %vp
-  %res = shufflevector <8 x i16> %vec, <8 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 7, i32 4, i32 4, i32 4>
-  ret <8 x i16> %res
-}
-define <8 x i16> @test_masked_8xi16_perm_high_mem_mask6(<8 x i16>* %vp, <8 x i16> %vec2, <8 x i16> %mask) {
-; GENERIC-LABEL: test_masked_8xi16_perm_high_mem_mask6:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmw %xmm1, %xmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufhw {{.*#+}} xmm0 {%k1} = mem[0,1,2,3,7,4,4,4] sched: [7:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_8xi16_perm_high_mem_mask6:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmw %xmm1, %xmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufhw {{.*#+}} xmm0 {%k1} = mem[0,1,2,3,7,4,4,4] sched: [7:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <8 x i16>, <8 x i16>* %vp
-  %shuf = shufflevector <8 x i16> %vec, <8 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 7, i32 4, i32 4, i32 4>
-  %cmp = icmp eq <8 x i16> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i16> %shuf, <8 x i16> %vec2
-  ret <8 x i16> %res
-}
-
-define <8 x i16> @test_masked_z_8xi16_perm_high_mem_mask6(<8 x i16>* %vp, <8 x i16> %mask) {
-; GENERIC-LABEL: test_masked_z_8xi16_perm_high_mem_mask6:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmw %xmm0, %xmm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufhw {{.*#+}} xmm0 {%k1} {z} = mem[0,1,2,3,7,4,4,4] sched: [7:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_8xi16_perm_high_mem_mask6:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmw %xmm0, %xmm0, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufhw {{.*#+}} xmm0 {%k1} {z} = mem[0,1,2,3,7,4,4,4] sched: [7:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <8 x i16>, <8 x i16>* %vp
-  %shuf = shufflevector <8 x i16> %vec, <8 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 7, i32 4, i32 4, i32 4>
-  %cmp = icmp eq <8 x i16> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i16> %shuf, <8 x i16> zeroinitializer
-  ret <8 x i16> %res
-}
-
-define <8 x i16> @test_masked_8xi16_perm_low_mem_mask7(<8 x i16>* %vp, <8 x i16> %vec2, <8 x i16> %mask) {
-; GENERIC-LABEL: test_masked_8xi16_perm_low_mem_mask7:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmw %xmm1, %xmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshuflw {{.*#+}} xmm0 {%k1} = mem[0,3,3,1,4,5,6,7] sched: [7:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_8xi16_perm_low_mem_mask7:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmw %xmm1, %xmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshuflw {{.*#+}} xmm0 {%k1} = mem[0,3,3,1,4,5,6,7] sched: [7:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <8 x i16>, <8 x i16>* %vp
-  %shuf = shufflevector <8 x i16> %vec, <8 x i16> undef, <8 x i32> <i32 0, i32 3, i32 3, i32 1, i32 4, i32 5, i32 6, i32 7>
-  %cmp = icmp eq <8 x i16> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i16> %shuf, <8 x i16> %vec2
-  ret <8 x i16> %res
-}
-
-define <8 x i16> @test_masked_z_8xi16_perm_low_mem_mask7(<8 x i16>* %vp, <8 x i16> %mask) {
-; GENERIC-LABEL: test_masked_z_8xi16_perm_low_mem_mask7:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmw %xmm0, %xmm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshuflw {{.*#+}} xmm0 {%k1} {z} = mem[0,3,3,1,4,5,6,7] sched: [7:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_8xi16_perm_low_mem_mask7:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmw %xmm0, %xmm0, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshuflw {{.*#+}} xmm0 {%k1} {z} = mem[0,3,3,1,4,5,6,7] sched: [7:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <8 x i16>, <8 x i16>* %vp
-  %shuf = shufflevector <8 x i16> %vec, <8 x i16> undef, <8 x i32> <i32 0, i32 3, i32 3, i32 1, i32 4, i32 5, i32 6, i32 7>
-  %cmp = icmp eq <8 x i16> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i16> %shuf, <8 x i16> zeroinitializer
-  ret <8 x i16> %res
-}
-
-define <16 x i16> @test_16xi16_perm_high_mask0(<16 x i16> %vec) {
-; GENERIC-LABEL: test_16xi16_perm_high_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpshufhw {{.*#+}} ymm0 = ymm0[0,1,2,3,4,4,6,4,8,9,10,11,12,12,14,12] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_16xi16_perm_high_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpshufhw {{.*#+}} ymm0 = ymm0[0,1,2,3,4,4,6,4,8,9,10,11,12,12,14,12] sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %res = shufflevector <16 x i16> %vec, <16 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 4, i32 6, i32 4, i32 8, i32 9, i32 10, i32 11, i32 12, i32 12, i32 14, i32 12>
-  ret <16 x i16> %res
-}
-define <16 x i16> @test_masked_16xi16_perm_high_mask0(<16 x i16> %vec, <16 x i16> %vec2, <16 x i16> %mask) {
-; GENERIC-LABEL: test_masked_16xi16_perm_high_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmw %ymm2, %ymm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufhw {{.*#+}} ymm1 {%k1} = ymm0[0,1,2,3,4,4,6,4,8,9,10,11,12,12,14,12] sched: [1:1.00]
-; GENERIC-NEXT:    vmovdqa %ymm1, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_16xi16_perm_high_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmw %ymm2, %ymm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufhw {{.*#+}} ymm1 {%k1} = ymm0[0,1,2,3,4,4,6,4,8,9,10,11,12,12,14,12] sched: [1:1.00]
-; SKX-NEXT:    vmovdqa %ymm1, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <16 x i16> %vec, <16 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 4, i32 6, i32 4, i32 8, i32 9, i32 10, i32 11, i32 12, i32 12, i32 14, i32 12>
-  %cmp = icmp eq <16 x i16> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x i16> %shuf, <16 x i16> %vec2
-  ret <16 x i16> %res
-}
-
-define <16 x i16> @test_masked_z_16xi16_perm_high_mask0(<16 x i16> %vec, <16 x i16> %mask) {
-; GENERIC-LABEL: test_masked_z_16xi16_perm_high_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmw %ymm1, %ymm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufhw {{.*#+}} ymm0 {%k1} {z} = ymm0[0,1,2,3,4,4,6,4,8,9,10,11,12,12,14,12] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_16xi16_perm_high_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmw %ymm1, %ymm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufhw {{.*#+}} ymm0 {%k1} {z} = ymm0[0,1,2,3,4,4,6,4,8,9,10,11,12,12,14,12] sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <16 x i16> %vec, <16 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 4, i32 6, i32 4, i32 8, i32 9, i32 10, i32 11, i32 12, i32 12, i32 14, i32 12>
-  %cmp = icmp eq <16 x i16> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x i16> %shuf, <16 x i16> zeroinitializer
-  ret <16 x i16> %res
-}
-define <16 x i16> @test_masked_16xi16_perm_low_mask1(<16 x i16> %vec, <16 x i16> %vec2, <16 x i16> %mask) {
-; GENERIC-LABEL: test_masked_16xi16_perm_low_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmw %ymm2, %ymm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshuflw {{.*#+}} ymm1 {%k1} = ymm0[0,2,3,2,4,5,6,7,8,10,11,10,12,13,14,15] sched: [1:1.00]
-; GENERIC-NEXT:    vmovdqa %ymm1, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_16xi16_perm_low_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmw %ymm2, %ymm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshuflw {{.*#+}} ymm1 {%k1} = ymm0[0,2,3,2,4,5,6,7,8,10,11,10,12,13,14,15] sched: [1:1.00]
-; SKX-NEXT:    vmovdqa %ymm1, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <16 x i16> %vec, <16 x i16> undef, <16 x i32> <i32 0, i32 2, i32 3, i32 2, i32 4, i32 5, i32 6, i32 7, i32 8, i32 10, i32 11, i32 10, i32 12, i32 13, i32 14, i32 15>
-  %cmp = icmp eq <16 x i16> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x i16> %shuf, <16 x i16> %vec2
-  ret <16 x i16> %res
-}
-
-define <16 x i16> @test_masked_z_16xi16_perm_low_mask1(<16 x i16> %vec, <16 x i16> %mask) {
-; GENERIC-LABEL: test_masked_z_16xi16_perm_low_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmw %ymm1, %ymm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshuflw {{.*#+}} ymm0 {%k1} {z} = ymm0[0,2,3,2,4,5,6,7,8,10,11,10,12,13,14,15] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_16xi16_perm_low_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmw %ymm1, %ymm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshuflw {{.*#+}} ymm0 {%k1} {z} = ymm0[0,2,3,2,4,5,6,7,8,10,11,10,12,13,14,15] sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <16 x i16> %vec, <16 x i16> undef, <16 x i32> <i32 0, i32 2, i32 3, i32 2, i32 4, i32 5, i32 6, i32 7, i32 8, i32 10, i32 11, i32 10, i32 12, i32 13, i32 14, i32 15>
-  %cmp = icmp eq <16 x i16> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x i16> %shuf, <16 x i16> zeroinitializer
-  ret <16 x i16> %res
-}
-define <16 x i16> @test_masked_16xi16_perm_high_mask2(<16 x i16> %vec, <16 x i16> %vec2, <16 x i16> %mask) {
-; GENERIC-LABEL: test_masked_16xi16_perm_high_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmw %ymm2, %ymm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufhw {{.*#+}} ymm1 {%k1} = ymm0[0,1,2,3,7,5,5,5,8,9,10,11,15,13,13,13] sched: [1:1.00]
-; GENERIC-NEXT:    vmovdqa %ymm1, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_16xi16_perm_high_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmw %ymm2, %ymm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufhw {{.*#+}} ymm1 {%k1} = ymm0[0,1,2,3,7,5,5,5,8,9,10,11,15,13,13,13] sched: [1:1.00]
-; SKX-NEXT:    vmovdqa %ymm1, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <16 x i16> %vec, <16 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 7, i32 5, i32 5, i32 5, i32 8, i32 9, i32 10, i32 11, i32 15, i32 13, i32 13, i32 13>
-  %cmp = icmp eq <16 x i16> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x i16> %shuf, <16 x i16> %vec2
-  ret <16 x i16> %res
-}
-
-define <16 x i16> @test_masked_z_16xi16_perm_high_mask2(<16 x i16> %vec, <16 x i16> %mask) {
-; GENERIC-LABEL: test_masked_z_16xi16_perm_high_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmw %ymm1, %ymm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufhw {{.*#+}} ymm0 {%k1} {z} = ymm0[0,1,2,3,7,5,5,5,8,9,10,11,15,13,13,13] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_16xi16_perm_high_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmw %ymm1, %ymm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufhw {{.*#+}} ymm0 {%k1} {z} = ymm0[0,1,2,3,7,5,5,5,8,9,10,11,15,13,13,13] sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <16 x i16> %vec, <16 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 7, i32 5, i32 5, i32 5, i32 8, i32 9, i32 10, i32 11, i32 15, i32 13, i32 13, i32 13>
-  %cmp = icmp eq <16 x i16> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x i16> %shuf, <16 x i16> zeroinitializer
-  ret <16 x i16> %res
-}
-define <16 x i16> @test_16xi16_perm_low_mask3(<16 x i16> %vec) {
-; GENERIC-LABEL: test_16xi16_perm_low_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpshuflw {{.*#+}} ymm0 = ymm0[3,2,3,2,4,5,6,7,11,10,11,10,12,13,14,15] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_16xi16_perm_low_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpshuflw {{.*#+}} ymm0 = ymm0[3,2,3,2,4,5,6,7,11,10,11,10,12,13,14,15] sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %res = shufflevector <16 x i16> %vec, <16 x i16> undef, <16 x i32> <i32 3, i32 2, i32 3, i32 2, i32 4, i32 5, i32 6, i32 7, i32 11, i32 10, i32 11, i32 10, i32 12, i32 13, i32 14, i32 15>
-  ret <16 x i16> %res
-}
-define <16 x i16> @test_masked_16xi16_perm_low_mask3(<16 x i16> %vec, <16 x i16> %vec2, <16 x i16> %mask) {
-; GENERIC-LABEL: test_masked_16xi16_perm_low_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmw %ymm2, %ymm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshuflw {{.*#+}} ymm1 {%k1} = ymm0[3,2,3,2,4,5,6,7,11,10,11,10,12,13,14,15] sched: [1:1.00]
-; GENERIC-NEXT:    vmovdqa %ymm1, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_16xi16_perm_low_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmw %ymm2, %ymm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshuflw {{.*#+}} ymm1 {%k1} = ymm0[3,2,3,2,4,5,6,7,11,10,11,10,12,13,14,15] sched: [1:1.00]
-; SKX-NEXT:    vmovdqa %ymm1, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <16 x i16> %vec, <16 x i16> undef, <16 x i32> <i32 3, i32 2, i32 3, i32 2, i32 4, i32 5, i32 6, i32 7, i32 11, i32 10, i32 11, i32 10, i32 12, i32 13, i32 14, i32 15>
-  %cmp = icmp eq <16 x i16> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x i16> %shuf, <16 x i16> %vec2
-  ret <16 x i16> %res
-}
-
-define <16 x i16> @test_masked_z_16xi16_perm_low_mask3(<16 x i16> %vec, <16 x i16> %mask) {
-; GENERIC-LABEL: test_masked_z_16xi16_perm_low_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmw %ymm1, %ymm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshuflw {{.*#+}} ymm0 {%k1} {z} = ymm0[3,2,3,2,4,5,6,7,11,10,11,10,12,13,14,15] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_16xi16_perm_low_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmw %ymm1, %ymm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshuflw {{.*#+}} ymm0 {%k1} {z} = ymm0[3,2,3,2,4,5,6,7,11,10,11,10,12,13,14,15] sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <16 x i16> %vec, <16 x i16> undef, <16 x i32> <i32 3, i32 2, i32 3, i32 2, i32 4, i32 5, i32 6, i32 7, i32 11, i32 10, i32 11, i32 10, i32 12, i32 13, i32 14, i32 15>
-  %cmp = icmp eq <16 x i16> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x i16> %shuf, <16 x i16> zeroinitializer
-  ret <16 x i16> %res
-}
-define <16 x i16> @test_masked_16xi16_perm_high_mask4(<16 x i16> %vec, <16 x i16> %vec2, <16 x i16> %mask) {
-; GENERIC-LABEL: test_masked_16xi16_perm_high_mask4:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmw %ymm2, %ymm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufhw {{.*#+}} ymm1 {%k1} = ymm0[0,1,2,3,6,7,4,7,8,9,10,11,14,15,12,15] sched: [1:1.00]
-; GENERIC-NEXT:    vmovdqa %ymm1, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_16xi16_perm_high_mask4:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmw %ymm2, %ymm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufhw {{.*#+}} ymm1 {%k1} = ymm0[0,1,2,3,6,7,4,7,8,9,10,11,14,15,12,15] sched: [1:1.00]
-; SKX-NEXT:    vmovdqa %ymm1, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <16 x i16> %vec, <16 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 6, i32 7, i32 4, i32 7, i32 8, i32 9, i32 10, i32 11, i32 14, i32 15, i32 12, i32 15>
-  %cmp = icmp eq <16 x i16> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x i16> %shuf, <16 x i16> %vec2
-  ret <16 x i16> %res
-}
-
-define <16 x i16> @test_masked_z_16xi16_perm_high_mask4(<16 x i16> %vec, <16 x i16> %mask) {
-; GENERIC-LABEL: test_masked_z_16xi16_perm_high_mask4:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmw %ymm1, %ymm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufhw {{.*#+}} ymm0 {%k1} {z} = ymm0[0,1,2,3,6,7,4,7,8,9,10,11,14,15,12,15] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_16xi16_perm_high_mask4:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmw %ymm1, %ymm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufhw {{.*#+}} ymm0 {%k1} {z} = ymm0[0,1,2,3,6,7,4,7,8,9,10,11,14,15,12,15] sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <16 x i16> %vec, <16 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 6, i32 7, i32 4, i32 7, i32 8, i32 9, i32 10, i32 11, i32 14, i32 15, i32 12, i32 15>
-  %cmp = icmp eq <16 x i16> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x i16> %shuf, <16 x i16> zeroinitializer
-  ret <16 x i16> %res
-}
-define <16 x i16> @test_masked_16xi16_perm_low_mask5(<16 x i16> %vec, <16 x i16> %vec2, <16 x i16> %mask) {
-; GENERIC-LABEL: test_masked_16xi16_perm_low_mask5:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmw %ymm2, %ymm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshuflw {{.*#+}} ymm1 {%k1} = ymm0[3,3,3,0,4,5,6,7,11,11,11,8,12,13,14,15] sched: [1:1.00]
-; GENERIC-NEXT:    vmovdqa %ymm1, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_16xi16_perm_low_mask5:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmw %ymm2, %ymm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshuflw {{.*#+}} ymm1 {%k1} = ymm0[3,3,3,0,4,5,6,7,11,11,11,8,12,13,14,15] sched: [1:1.00]
-; SKX-NEXT:    vmovdqa %ymm1, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <16 x i16> %vec, <16 x i16> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 0, i32 4, i32 5, i32 6, i32 7, i32 11, i32 11, i32 11, i32 8, i32 12, i32 13, i32 14, i32 15>
-  %cmp = icmp eq <16 x i16> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x i16> %shuf, <16 x i16> %vec2
-  ret <16 x i16> %res
-}
-
-define <16 x i16> @test_masked_z_16xi16_perm_low_mask5(<16 x i16> %vec, <16 x i16> %mask) {
-; GENERIC-LABEL: test_masked_z_16xi16_perm_low_mask5:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmw %ymm1, %ymm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshuflw {{.*#+}} ymm0 {%k1} {z} = ymm0[3,3,3,0,4,5,6,7,11,11,11,8,12,13,14,15] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_16xi16_perm_low_mask5:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmw %ymm1, %ymm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshuflw {{.*#+}} ymm0 {%k1} {z} = ymm0[3,3,3,0,4,5,6,7,11,11,11,8,12,13,14,15] sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <16 x i16> %vec, <16 x i16> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 0, i32 4, i32 5, i32 6, i32 7, i32 11, i32 11, i32 11, i32 8, i32 12, i32 13, i32 14, i32 15>
-  %cmp = icmp eq <16 x i16> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x i16> %shuf, <16 x i16> zeroinitializer
-  ret <16 x i16> %res
-}
-define <16 x i16> @test_16xi16_perm_high_mask6(<16 x i16> %vec) {
-; GENERIC-LABEL: test_16xi16_perm_high_mask6:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpshufhw {{.*#+}} ymm0 = ymm0[0,1,2,3,6,7,6,5,8,9,10,11,14,15,14,13] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_16xi16_perm_high_mask6:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpshufhw {{.*#+}} ymm0 = ymm0[0,1,2,3,6,7,6,5,8,9,10,11,14,15,14,13] sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %res = shufflevector <16 x i16> %vec, <16 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 6, i32 7, i32 6, i32 5, i32 8, i32 9, i32 10, i32 11, i32 14, i32 15, i32 14, i32 13>
-  ret <16 x i16> %res
-}
-define <16 x i16> @test_masked_16xi16_perm_high_mask6(<16 x i16> %vec, <16 x i16> %vec2, <16 x i16> %mask) {
-; GENERIC-LABEL: test_masked_16xi16_perm_high_mask6:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmw %ymm2, %ymm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufhw {{.*#+}} ymm1 {%k1} = ymm0[0,1,2,3,6,7,6,5,8,9,10,11,14,15,14,13] sched: [1:1.00]
-; GENERIC-NEXT:    vmovdqa %ymm1, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_16xi16_perm_high_mask6:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmw %ymm2, %ymm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufhw {{.*#+}} ymm1 {%k1} = ymm0[0,1,2,3,6,7,6,5,8,9,10,11,14,15,14,13] sched: [1:1.00]
-; SKX-NEXT:    vmovdqa %ymm1, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <16 x i16> %vec, <16 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 6, i32 7, i32 6, i32 5, i32 8, i32 9, i32 10, i32 11, i32 14, i32 15, i32 14, i32 13>
-  %cmp = icmp eq <16 x i16> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x i16> %shuf, <16 x i16> %vec2
-  ret <16 x i16> %res
-}
-
-define <16 x i16> @test_masked_z_16xi16_perm_high_mask6(<16 x i16> %vec, <16 x i16> %mask) {
-; GENERIC-LABEL: test_masked_z_16xi16_perm_high_mask6:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmw %ymm1, %ymm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufhw {{.*#+}} ymm0 {%k1} {z} = ymm0[0,1,2,3,6,7,6,5,8,9,10,11,14,15,14,13] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_16xi16_perm_high_mask6:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmw %ymm1, %ymm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufhw {{.*#+}} ymm0 {%k1} {z} = ymm0[0,1,2,3,6,7,6,5,8,9,10,11,14,15,14,13] sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <16 x i16> %vec, <16 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 6, i32 7, i32 6, i32 5, i32 8, i32 9, i32 10, i32 11, i32 14, i32 15, i32 14, i32 13>
-  %cmp = icmp eq <16 x i16> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x i16> %shuf, <16 x i16> zeroinitializer
-  ret <16 x i16> %res
-}
-define <16 x i16> @test_masked_16xi16_perm_low_mask7(<16 x i16> %vec, <16 x i16> %vec2, <16 x i16> %mask) {
-; GENERIC-LABEL: test_masked_16xi16_perm_low_mask7:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmw %ymm2, %ymm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshuflw {{.*#+}} ymm1 {%k1} = ymm0[3,2,1,2,4,5,6,7,11,10,9,10,12,13,14,15] sched: [1:1.00]
-; GENERIC-NEXT:    vmovdqa %ymm1, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_16xi16_perm_low_mask7:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmw %ymm2, %ymm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshuflw {{.*#+}} ymm1 {%k1} = ymm0[3,2,1,2,4,5,6,7,11,10,9,10,12,13,14,15] sched: [1:1.00]
-; SKX-NEXT:    vmovdqa %ymm1, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <16 x i16> %vec, <16 x i16> undef, <16 x i32> <i32 3, i32 2, i32 1, i32 2, i32 4, i32 5, i32 6, i32 7, i32 11, i32 10, i32 9, i32 10, i32 12, i32 13, i32 14, i32 15>
-  %cmp = icmp eq <16 x i16> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x i16> %shuf, <16 x i16> %vec2
-  ret <16 x i16> %res
-}
-
-define <16 x i16> @test_masked_z_16xi16_perm_low_mask7(<16 x i16> %vec, <16 x i16> %mask) {
-; GENERIC-LABEL: test_masked_z_16xi16_perm_low_mask7:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmw %ymm1, %ymm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshuflw {{.*#+}} ymm0 {%k1} {z} = ymm0[3,2,1,2,4,5,6,7,11,10,9,10,12,13,14,15] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_16xi16_perm_low_mask7:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmw %ymm1, %ymm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshuflw {{.*#+}} ymm0 {%k1} {z} = ymm0[3,2,1,2,4,5,6,7,11,10,9,10,12,13,14,15] sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <16 x i16> %vec, <16 x i16> undef, <16 x i32> <i32 3, i32 2, i32 1, i32 2, i32 4, i32 5, i32 6, i32 7, i32 11, i32 10, i32 9, i32 10, i32 12, i32 13, i32 14, i32 15>
-  %cmp = icmp eq <16 x i16> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x i16> %shuf, <16 x i16> zeroinitializer
-  ret <16 x i16> %res
-}
-define <16 x i16> @test_16xi16_perm_high_mem_mask0(<16 x i16>* %vp) {
-; GENERIC-LABEL: test_16xi16_perm_high_mem_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpshufhw {{.*#+}} ymm0 = mem[0,1,2,3,5,6,4,7,8,9,10,11,13,14,12,15] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_16xi16_perm_high_mem_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpshufhw {{.*#+}} ymm0 = mem[0,1,2,3,5,6,4,7,8,9,10,11,13,14,12,15] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <16 x i16>, <16 x i16>* %vp
-  %res = shufflevector <16 x i16> %vec, <16 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 5, i32 6, i32 4, i32 7, i32 8, i32 9, i32 10, i32 11, i32 13, i32 14, i32 12, i32 15>
-  ret <16 x i16> %res
-}
-define <16 x i16> @test_masked_16xi16_perm_high_mem_mask0(<16 x i16>* %vp, <16 x i16> %vec2, <16 x i16> %mask) {
-; GENERIC-LABEL: test_masked_16xi16_perm_high_mem_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmw %ymm1, %ymm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufhw {{.*#+}} ymm0 {%k1} = mem[0,1,2,3,5,6,4,7,8,9,10,11,13,14,12,15] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_16xi16_perm_high_mem_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmw %ymm1, %ymm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufhw {{.*#+}} ymm0 {%k1} = mem[0,1,2,3,5,6,4,7,8,9,10,11,13,14,12,15] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <16 x i16>, <16 x i16>* %vp
-  %shuf = shufflevector <16 x i16> %vec, <16 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 5, i32 6, i32 4, i32 7, i32 8, i32 9, i32 10, i32 11, i32 13, i32 14, i32 12, i32 15>
-  %cmp = icmp eq <16 x i16> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x i16> %shuf, <16 x i16> %vec2
-  ret <16 x i16> %res
-}
-
-define <16 x i16> @test_masked_z_16xi16_perm_high_mem_mask0(<16 x i16>* %vp, <16 x i16> %mask) {
-; GENERIC-LABEL: test_masked_z_16xi16_perm_high_mem_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmw %ymm0, %ymm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufhw {{.*#+}} ymm0 {%k1} {z} = mem[0,1,2,3,5,6,4,7,8,9,10,11,13,14,12,15] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_16xi16_perm_high_mem_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmw %ymm0, %ymm0, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufhw {{.*#+}} ymm0 {%k1} {z} = mem[0,1,2,3,5,6,4,7,8,9,10,11,13,14,12,15] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <16 x i16>, <16 x i16>* %vp
-  %shuf = shufflevector <16 x i16> %vec, <16 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 5, i32 6, i32 4, i32 7, i32 8, i32 9, i32 10, i32 11, i32 13, i32 14, i32 12, i32 15>
-  %cmp = icmp eq <16 x i16> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x i16> %shuf, <16 x i16> zeroinitializer
-  ret <16 x i16> %res
-}
-
-define <16 x i16> @test_masked_16xi16_perm_low_mem_mask1(<16 x i16>* %vp, <16 x i16> %vec2, <16 x i16> %mask) {
-; GENERIC-LABEL: test_masked_16xi16_perm_low_mem_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmw %ymm1, %ymm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshuflw {{.*#+}} ymm0 {%k1} = mem[1,3,3,0,4,5,6,7,9,11,11,8,12,13,14,15] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_16xi16_perm_low_mem_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmw %ymm1, %ymm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshuflw {{.*#+}} ymm0 {%k1} = mem[1,3,3,0,4,5,6,7,9,11,11,8,12,13,14,15] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <16 x i16>, <16 x i16>* %vp
-  %shuf = shufflevector <16 x i16> %vec, <16 x i16> undef, <16 x i32> <i32 1, i32 3, i32 3, i32 0, i32 4, i32 5, i32 6, i32 7, i32 9, i32 11, i32 11, i32 8, i32 12, i32 13, i32 14, i32 15>
-  %cmp = icmp eq <16 x i16> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x i16> %shuf, <16 x i16> %vec2
-  ret <16 x i16> %res
-}
-
-define <16 x i16> @test_masked_z_16xi16_perm_low_mem_mask1(<16 x i16>* %vp, <16 x i16> %mask) {
-; GENERIC-LABEL: test_masked_z_16xi16_perm_low_mem_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmw %ymm0, %ymm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshuflw {{.*#+}} ymm0 {%k1} {z} = mem[1,3,3,0,4,5,6,7,9,11,11,8,12,13,14,15] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_16xi16_perm_low_mem_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmw %ymm0, %ymm0, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshuflw {{.*#+}} ymm0 {%k1} {z} = mem[1,3,3,0,4,5,6,7,9,11,11,8,12,13,14,15] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <16 x i16>, <16 x i16>* %vp
-  %shuf = shufflevector <16 x i16> %vec, <16 x i16> undef, <16 x i32> <i32 1, i32 3, i32 3, i32 0, i32 4, i32 5, i32 6, i32 7, i32 9, i32 11, i32 11, i32 8, i32 12, i32 13, i32 14, i32 15>
-  %cmp = icmp eq <16 x i16> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x i16> %shuf, <16 x i16> zeroinitializer
-  ret <16 x i16> %res
-}
-
-define <16 x i16> @test_masked_16xi16_perm_high_mem_mask2(<16 x i16>* %vp, <16 x i16> %vec2, <16 x i16> %mask) {
-; GENERIC-LABEL: test_masked_16xi16_perm_high_mem_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmw %ymm1, %ymm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufhw {{.*#+}} ymm0 {%k1} = mem[0,1,2,3,5,6,5,6,8,9,10,11,13,14,13,14] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_16xi16_perm_high_mem_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmw %ymm1, %ymm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufhw {{.*#+}} ymm0 {%k1} = mem[0,1,2,3,5,6,5,6,8,9,10,11,13,14,13,14] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <16 x i16>, <16 x i16>* %vp
-  %shuf = shufflevector <16 x i16> %vec, <16 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 5, i32 6, i32 5, i32 6, i32 8, i32 9, i32 10, i32 11, i32 13, i32 14, i32 13, i32 14>
-  %cmp = icmp eq <16 x i16> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x i16> %shuf, <16 x i16> %vec2
-  ret <16 x i16> %res
-}
-
-define <16 x i16> @test_masked_z_16xi16_perm_high_mem_mask2(<16 x i16>* %vp, <16 x i16> %mask) {
-; GENERIC-LABEL: test_masked_z_16xi16_perm_high_mem_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmw %ymm0, %ymm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufhw {{.*#+}} ymm0 {%k1} {z} = mem[0,1,2,3,5,6,5,6,8,9,10,11,13,14,13,14] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_16xi16_perm_high_mem_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmw %ymm0, %ymm0, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufhw {{.*#+}} ymm0 {%k1} {z} = mem[0,1,2,3,5,6,5,6,8,9,10,11,13,14,13,14] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <16 x i16>, <16 x i16>* %vp
-  %shuf = shufflevector <16 x i16> %vec, <16 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 5, i32 6, i32 5, i32 6, i32 8, i32 9, i32 10, i32 11, i32 13, i32 14, i32 13, i32 14>
-  %cmp = icmp eq <16 x i16> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x i16> %shuf, <16 x i16> zeroinitializer
-  ret <16 x i16> %res
-}
-
-define <16 x i16> @test_16xi16_perm_low_mem_mask3(<16 x i16>* %vp) {
-; GENERIC-LABEL: test_16xi16_perm_low_mem_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpshuflw {{.*#+}} ymm0 = mem[3,2,3,0,4,5,6,7,11,10,11,8,12,13,14,15] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_16xi16_perm_low_mem_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpshuflw {{.*#+}} ymm0 = mem[3,2,3,0,4,5,6,7,11,10,11,8,12,13,14,15] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <16 x i16>, <16 x i16>* %vp
-  %res = shufflevector <16 x i16> %vec, <16 x i16> undef, <16 x i32> <i32 3, i32 2, i32 3, i32 0, i32 4, i32 5, i32 6, i32 7, i32 11, i32 10, i32 11, i32 8, i32 12, i32 13, i32 14, i32 15>
-  ret <16 x i16> %res
-}
-define <16 x i16> @test_masked_16xi16_perm_low_mem_mask3(<16 x i16>* %vp, <16 x i16> %vec2, <16 x i16> %mask) {
-; GENERIC-LABEL: test_masked_16xi16_perm_low_mem_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmw %ymm1, %ymm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshuflw {{.*#+}} ymm0 {%k1} = mem[3,2,3,0,4,5,6,7,11,10,11,8,12,13,14,15] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_16xi16_perm_low_mem_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmw %ymm1, %ymm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshuflw {{.*#+}} ymm0 {%k1} = mem[3,2,3,0,4,5,6,7,11,10,11,8,12,13,14,15] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <16 x i16>, <16 x i16>* %vp
-  %shuf = shufflevector <16 x i16> %vec, <16 x i16> undef, <16 x i32> <i32 3, i32 2, i32 3, i32 0, i32 4, i32 5, i32 6, i32 7, i32 11, i32 10, i32 11, i32 8, i32 12, i32 13, i32 14, i32 15>
-  %cmp = icmp eq <16 x i16> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x i16> %shuf, <16 x i16> %vec2
-  ret <16 x i16> %res
-}
-
-define <16 x i16> @test_masked_z_16xi16_perm_low_mem_mask3(<16 x i16>* %vp, <16 x i16> %mask) {
-; GENERIC-LABEL: test_masked_z_16xi16_perm_low_mem_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmw %ymm0, %ymm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshuflw {{.*#+}} ymm0 {%k1} {z} = mem[3,2,3,0,4,5,6,7,11,10,11,8,12,13,14,15] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_16xi16_perm_low_mem_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmw %ymm0, %ymm0, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshuflw {{.*#+}} ymm0 {%k1} {z} = mem[3,2,3,0,4,5,6,7,11,10,11,8,12,13,14,15] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <16 x i16>, <16 x i16>* %vp
-  %shuf = shufflevector <16 x i16> %vec, <16 x i16> undef, <16 x i32> <i32 3, i32 2, i32 3, i32 0, i32 4, i32 5, i32 6, i32 7, i32 11, i32 10, i32 11, i32 8, i32 12, i32 13, i32 14, i32 15>
-  %cmp = icmp eq <16 x i16> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x i16> %shuf, <16 x i16> zeroinitializer
-  ret <16 x i16> %res
-}
-
-define <16 x i16> @test_masked_16xi16_perm_high_mem_mask4(<16 x i16>* %vp, <16 x i16> %vec2, <16 x i16> %mask) {
-; GENERIC-LABEL: test_masked_16xi16_perm_high_mem_mask4:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmw %ymm1, %ymm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufhw {{.*#+}} ymm0 {%k1} = mem[0,1,2,3,7,7,6,7,8,9,10,11,15,15,14,15] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_16xi16_perm_high_mem_mask4:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmw %ymm1, %ymm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufhw {{.*#+}} ymm0 {%k1} = mem[0,1,2,3,7,7,6,7,8,9,10,11,15,15,14,15] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <16 x i16>, <16 x i16>* %vp
-  %shuf = shufflevector <16 x i16> %vec, <16 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 7, i32 7, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 15, i32 15, i32 14, i32 15>
-  %cmp = icmp eq <16 x i16> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x i16> %shuf, <16 x i16> %vec2
-  ret <16 x i16> %res
-}
-
-define <16 x i16> @test_masked_z_16xi16_perm_high_mem_mask4(<16 x i16>* %vp, <16 x i16> %mask) {
-; GENERIC-LABEL: test_masked_z_16xi16_perm_high_mem_mask4:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmw %ymm0, %ymm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufhw {{.*#+}} ymm0 {%k1} {z} = mem[0,1,2,3,7,7,6,7,8,9,10,11,15,15,14,15] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_16xi16_perm_high_mem_mask4:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmw %ymm0, %ymm0, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufhw {{.*#+}} ymm0 {%k1} {z} = mem[0,1,2,3,7,7,6,7,8,9,10,11,15,15,14,15] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <16 x i16>, <16 x i16>* %vp
-  %shuf = shufflevector <16 x i16> %vec, <16 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 7, i32 7, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 15, i32 15, i32 14, i32 15>
-  %cmp = icmp eq <16 x i16> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x i16> %shuf, <16 x i16> zeroinitializer
-  ret <16 x i16> %res
-}
-
-define <16 x i16> @test_masked_16xi16_perm_low_mem_mask5(<16 x i16>* %vp, <16 x i16> %vec2, <16 x i16> %mask) {
-; GENERIC-LABEL: test_masked_16xi16_perm_low_mem_mask5:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmw %ymm1, %ymm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshuflw {{.*#+}} ymm0 {%k1} = mem[1,3,3,2,4,5,6,7,9,11,11,10,12,13,14,15] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_16xi16_perm_low_mem_mask5:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmw %ymm1, %ymm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshuflw {{.*#+}} ymm0 {%k1} = mem[1,3,3,2,4,5,6,7,9,11,11,10,12,13,14,15] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <16 x i16>, <16 x i16>* %vp
-  %shuf = shufflevector <16 x i16> %vec, <16 x i16> undef, <16 x i32> <i32 1, i32 3, i32 3, i32 2, i32 4, i32 5, i32 6, i32 7, i32 9, i32 11, i32 11, i32 10, i32 12, i32 13, i32 14, i32 15>
-  %cmp = icmp eq <16 x i16> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x i16> %shuf, <16 x i16> %vec2
-  ret <16 x i16> %res
-}
-
-define <16 x i16> @test_masked_z_16xi16_perm_low_mem_mask5(<16 x i16>* %vp, <16 x i16> %mask) {
-; GENERIC-LABEL: test_masked_z_16xi16_perm_low_mem_mask5:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmw %ymm0, %ymm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshuflw {{.*#+}} ymm0 {%k1} {z} = mem[1,3,3,2,4,5,6,7,9,11,11,10,12,13,14,15] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_16xi16_perm_low_mem_mask5:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmw %ymm0, %ymm0, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshuflw {{.*#+}} ymm0 {%k1} {z} = mem[1,3,3,2,4,5,6,7,9,11,11,10,12,13,14,15] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <16 x i16>, <16 x i16>* %vp
-  %shuf = shufflevector <16 x i16> %vec, <16 x i16> undef, <16 x i32> <i32 1, i32 3, i32 3, i32 2, i32 4, i32 5, i32 6, i32 7, i32 9, i32 11, i32 11, i32 10, i32 12, i32 13, i32 14, i32 15>
-  %cmp = icmp eq <16 x i16> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x i16> %shuf, <16 x i16> zeroinitializer
-  ret <16 x i16> %res
-}
-
-define <16 x i16> @test_16xi16_perm_high_mem_mask6(<16 x i16>* %vp) {
-; GENERIC-LABEL: test_16xi16_perm_high_mem_mask6:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpshufhw {{.*#+}} ymm0 = mem[0,1,2,3,4,4,4,5,8,9,10,11,12,12,12,13] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_16xi16_perm_high_mem_mask6:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpshufhw {{.*#+}} ymm0 = mem[0,1,2,3,4,4,4,5,8,9,10,11,12,12,12,13] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <16 x i16>, <16 x i16>* %vp
-  %res = shufflevector <16 x i16> %vec, <16 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 4, i32 4, i32 5, i32 8, i32 9, i32 10, i32 11, i32 12, i32 12, i32 12, i32 13>
-  ret <16 x i16> %res
-}
-define <16 x i16> @test_masked_16xi16_perm_high_mem_mask6(<16 x i16>* %vp, <16 x i16> %vec2, <16 x i16> %mask) {
-; GENERIC-LABEL: test_masked_16xi16_perm_high_mem_mask6:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmw %ymm1, %ymm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufhw {{.*#+}} ymm0 {%k1} = mem[0,1,2,3,4,4,4,5,8,9,10,11,12,12,12,13] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_16xi16_perm_high_mem_mask6:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmw %ymm1, %ymm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufhw {{.*#+}} ymm0 {%k1} = mem[0,1,2,3,4,4,4,5,8,9,10,11,12,12,12,13] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <16 x i16>, <16 x i16>* %vp
-  %shuf = shufflevector <16 x i16> %vec, <16 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 4, i32 4, i32 5, i32 8, i32 9, i32 10, i32 11, i32 12, i32 12, i32 12, i32 13>
-  %cmp = icmp eq <16 x i16> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x i16> %shuf, <16 x i16> %vec2
-  ret <16 x i16> %res
-}
-
-define <16 x i16> @test_masked_z_16xi16_perm_high_mem_mask6(<16 x i16>* %vp, <16 x i16> %mask) {
-; GENERIC-LABEL: test_masked_z_16xi16_perm_high_mem_mask6:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmw %ymm0, %ymm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufhw {{.*#+}} ymm0 {%k1} {z} = mem[0,1,2,3,4,4,4,5,8,9,10,11,12,12,12,13] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_16xi16_perm_high_mem_mask6:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmw %ymm0, %ymm0, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufhw {{.*#+}} ymm0 {%k1} {z} = mem[0,1,2,3,4,4,4,5,8,9,10,11,12,12,12,13] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <16 x i16>, <16 x i16>* %vp
-  %shuf = shufflevector <16 x i16> %vec, <16 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 4, i32 4, i32 5, i32 8, i32 9, i32 10, i32 11, i32 12, i32 12, i32 12, i32 13>
-  %cmp = icmp eq <16 x i16> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x i16> %shuf, <16 x i16> zeroinitializer
-  ret <16 x i16> %res
-}
-
-define <16 x i16> @test_masked_16xi16_perm_low_mem_mask7(<16 x i16>* %vp, <16 x i16> %vec2, <16 x i16> %mask) {
-; GENERIC-LABEL: test_masked_16xi16_perm_low_mem_mask7:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmw %ymm1, %ymm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshuflw {{.*#+}} ymm0 {%k1} = mem[3,1,3,2,4,5,6,7,11,9,11,10,12,13,14,15] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_16xi16_perm_low_mem_mask7:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmw %ymm1, %ymm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshuflw {{.*#+}} ymm0 {%k1} = mem[3,1,3,2,4,5,6,7,11,9,11,10,12,13,14,15] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <16 x i16>, <16 x i16>* %vp
-  %shuf = shufflevector <16 x i16> %vec, <16 x i16> undef, <16 x i32> <i32 3, i32 1, i32 3, i32 2, i32 4, i32 5, i32 6, i32 7, i32 11, i32 9, i32 11, i32 10, i32 12, i32 13, i32 14, i32 15>
-  %cmp = icmp eq <16 x i16> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x i16> %shuf, <16 x i16> %vec2
-  ret <16 x i16> %res
-}
-
-define <16 x i16> @test_masked_z_16xi16_perm_low_mem_mask7(<16 x i16>* %vp, <16 x i16> %mask) {
-; GENERIC-LABEL: test_masked_z_16xi16_perm_low_mem_mask7:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmw %ymm0, %ymm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshuflw {{.*#+}} ymm0 {%k1} {z} = mem[3,1,3,2,4,5,6,7,11,9,11,10,12,13,14,15] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_16xi16_perm_low_mem_mask7:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmw %ymm0, %ymm0, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshuflw {{.*#+}} ymm0 {%k1} {z} = mem[3,1,3,2,4,5,6,7,11,9,11,10,12,13,14,15] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <16 x i16>, <16 x i16>* %vp
-  %shuf = shufflevector <16 x i16> %vec, <16 x i16> undef, <16 x i32> <i32 3, i32 1, i32 3, i32 2, i32 4, i32 5, i32 6, i32 7, i32 11, i32 9, i32 11, i32 10, i32 12, i32 13, i32 14, i32 15>
-  %cmp = icmp eq <16 x i16> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x i16> %shuf, <16 x i16> zeroinitializer
-  ret <16 x i16> %res
-}
-
-define <32 x i16> @test_32xi16_perm_high_mask0(<32 x i16> %vec) {
-; GENERIC-LABEL: test_32xi16_perm_high_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpshufhw {{.*#+}} zmm0 = zmm0[0,1,2,3,4,5,6,4,8,9,10,11,12,13,14,12,16,17,18,19,20,21,22,20,24,25,26,27,28,29,30,28] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_32xi16_perm_high_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpshufhw {{.*#+}} zmm0 = zmm0[0,1,2,3,4,5,6,4,8,9,10,11,12,13,14,12,16,17,18,19,20,21,22,20,24,25,26,27,28,29,30,28] sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %res = shufflevector <32 x i16> %vec, <32 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 4, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 12, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 20, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 28>
-  ret <32 x i16> %res
-}
-define <32 x i16> @test_masked_32xi16_perm_high_mask0(<32 x i16> %vec, <32 x i16> %vec2, <32 x i16> %mask) {
-; GENERIC-LABEL: test_masked_32xi16_perm_high_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmw %zmm2, %zmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufhw {{.*#+}} zmm1 {%k1} = zmm0[0,1,2,3,4,5,6,4,8,9,10,11,12,13,14,12,16,17,18,19,20,21,22,20,24,25,26,27,28,29,30,28] sched: [1:1.00]
-; GENERIC-NEXT:    vmovdqa64 %zmm1, %zmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_32xi16_perm_high_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmw %zmm2, %zmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufhw {{.*#+}} zmm1 {%k1} = zmm0[0,1,2,3,4,5,6,4,8,9,10,11,12,13,14,12,16,17,18,19,20,21,22,20,24,25,26,27,28,29,30,28] sched: [1:1.00]
-; SKX-NEXT:    vmovdqa64 %zmm1, %zmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <32 x i16> %vec, <32 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 4, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 12, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 20, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 28>
-  %cmp = icmp eq <32 x i16> %mask, zeroinitializer
-  %res = select <32 x i1> %cmp, <32 x i16> %shuf, <32 x i16> %vec2
-  ret <32 x i16> %res
-}
-
-define <32 x i16> @test_masked_z_32xi16_perm_high_mask0(<32 x i16> %vec, <32 x i16> %mask) {
-; GENERIC-LABEL: test_masked_z_32xi16_perm_high_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmw %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufhw {{.*#+}} zmm0 {%k1} {z} = zmm0[0,1,2,3,4,5,6,4,8,9,10,11,12,13,14,12,16,17,18,19,20,21,22,20,24,25,26,27,28,29,30,28] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_32xi16_perm_high_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmw %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufhw {{.*#+}} zmm0 {%k1} {z} = zmm0[0,1,2,3,4,5,6,4,8,9,10,11,12,13,14,12,16,17,18,19,20,21,22,20,24,25,26,27,28,29,30,28] sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <32 x i16> %vec, <32 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 4, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 12, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 20, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 28>
-  %cmp = icmp eq <32 x i16> %mask, zeroinitializer
-  %res = select <32 x i1> %cmp, <32 x i16> %shuf, <32 x i16> zeroinitializer
-  ret <32 x i16> %res
-}
-define <32 x i16> @test_masked_32xi16_perm_low_mask1(<32 x i16> %vec, <32 x i16> %vec2, <32 x i16> %mask) {
-; GENERIC-LABEL: test_masked_32xi16_perm_low_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmw %zmm2, %zmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshuflw {{.*#+}} zmm1 {%k1} = zmm0[2,1,0,0,4,5,6,7,10,9,8,8,12,13,14,15,18,17,16,16,20,21,22,23,26,25,24,24,28,29,30,31] sched: [1:1.00]
-; GENERIC-NEXT:    vmovdqa64 %zmm1, %zmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_32xi16_perm_low_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmw %zmm2, %zmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshuflw {{.*#+}} zmm1 {%k1} = zmm0[2,1,0,0,4,5,6,7,10,9,8,8,12,13,14,15,18,17,16,16,20,21,22,23,26,25,24,24,28,29,30,31] sched: [1:1.00]
-; SKX-NEXT:    vmovdqa64 %zmm1, %zmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <32 x i16> %vec, <32 x i16> undef, <32 x i32> <i32 2, i32 1, i32 0, i32 0, i32 4, i32 5, i32 6, i32 7, i32 10, i32 9, i32 8, i32 8, i32 12, i32 13, i32 14, i32 15, i32 18, i32 17, i32 16, i32 16, i32 20, i32 21, i32 22, i32 23, i32 26, i32 25, i32 24, i32 24, i32 28, i32 29, i32 30, i32 31>
-  %cmp = icmp eq <32 x i16> %mask, zeroinitializer
-  %res = select <32 x i1> %cmp, <32 x i16> %shuf, <32 x i16> %vec2
-  ret <32 x i16> %res
-}
-
-define <32 x i16> @test_masked_z_32xi16_perm_low_mask1(<32 x i16> %vec, <32 x i16> %mask) {
-; GENERIC-LABEL: test_masked_z_32xi16_perm_low_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmw %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshuflw {{.*#+}} zmm0 {%k1} {z} = zmm0[2,1,0,0,4,5,6,7,10,9,8,8,12,13,14,15,18,17,16,16,20,21,22,23,26,25,24,24,28,29,30,31] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_32xi16_perm_low_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmw %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshuflw {{.*#+}} zmm0 {%k1} {z} = zmm0[2,1,0,0,4,5,6,7,10,9,8,8,12,13,14,15,18,17,16,16,20,21,22,23,26,25,24,24,28,29,30,31] sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <32 x i16> %vec, <32 x i16> undef, <32 x i32> <i32 2, i32 1, i32 0, i32 0, i32 4, i32 5, i32 6, i32 7, i32 10, i32 9, i32 8, i32 8, i32 12, i32 13, i32 14, i32 15, i32 18, i32 17, i32 16, i32 16, i32 20, i32 21, i32 22, i32 23, i32 26, i32 25, i32 24, i32 24, i32 28, i32 29, i32 30, i32 31>
-  %cmp = icmp eq <32 x i16> %mask, zeroinitializer
-  %res = select <32 x i1> %cmp, <32 x i16> %shuf, <32 x i16> zeroinitializer
-  ret <32 x i16> %res
-}
-define <32 x i16> @test_masked_32xi16_perm_high_mask2(<32 x i16> %vec, <32 x i16> %vec2, <32 x i16> %mask) {
-; GENERIC-LABEL: test_masked_32xi16_perm_high_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmw %zmm2, %zmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufhw {{.*#+}} zmm1 {%k1} = zmm0[0,1,2,3,4,6,4,7,8,9,10,11,12,14,12,15,16,17,18,19,20,22,20,23,24,25,26,27,28,30,28,31] sched: [1:1.00]
-; GENERIC-NEXT:    vmovdqa64 %zmm1, %zmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_32xi16_perm_high_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmw %zmm2, %zmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufhw {{.*#+}} zmm1 {%k1} = zmm0[0,1,2,3,4,6,4,7,8,9,10,11,12,14,12,15,16,17,18,19,20,22,20,23,24,25,26,27,28,30,28,31] sched: [1:1.00]
-; SKX-NEXT:    vmovdqa64 %zmm1, %zmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <32 x i16> %vec, <32 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 6, i32 4, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 14, i32 12, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 22, i32 20, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 30, i32 28, i32 31>
-  %cmp = icmp eq <32 x i16> %mask, zeroinitializer
-  %res = select <32 x i1> %cmp, <32 x i16> %shuf, <32 x i16> %vec2
-  ret <32 x i16> %res
-}
-
-define <32 x i16> @test_masked_z_32xi16_perm_high_mask2(<32 x i16> %vec, <32 x i16> %mask) {
-; GENERIC-LABEL: test_masked_z_32xi16_perm_high_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmw %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufhw {{.*#+}} zmm0 {%k1} {z} = zmm0[0,1,2,3,4,6,4,7,8,9,10,11,12,14,12,15,16,17,18,19,20,22,20,23,24,25,26,27,28,30,28,31] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_32xi16_perm_high_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmw %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufhw {{.*#+}} zmm0 {%k1} {z} = zmm0[0,1,2,3,4,6,4,7,8,9,10,11,12,14,12,15,16,17,18,19,20,22,20,23,24,25,26,27,28,30,28,31] sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <32 x i16> %vec, <32 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 6, i32 4, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 14, i32 12, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 22, i32 20, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 30, i32 28, i32 31>
-  %cmp = icmp eq <32 x i16> %mask, zeroinitializer
-  %res = select <32 x i1> %cmp, <32 x i16> %shuf, <32 x i16> zeroinitializer
-  ret <32 x i16> %res
-}
-define <32 x i16> @test_32xi16_perm_low_mask3(<32 x i16> %vec) {
-; GENERIC-LABEL: test_32xi16_perm_low_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpshuflw {{.*#+}} zmm0 = zmm0[3,3,1,3,4,5,6,7,11,11,9,11,12,13,14,15,19,19,17,19,20,21,22,23,27,27,25,27,28,29,30,31] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_32xi16_perm_low_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpshuflw {{.*#+}} zmm0 = zmm0[3,3,1,3,4,5,6,7,11,11,9,11,12,13,14,15,19,19,17,19,20,21,22,23,27,27,25,27,28,29,30,31] sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %res = shufflevector <32 x i16> %vec, <32 x i16> undef, <32 x i32> <i32 3, i32 3, i32 1, i32 3, i32 4, i32 5, i32 6, i32 7, i32 11, i32 11, i32 9, i32 11, i32 12, i32 13, i32 14, i32 15, i32 19, i32 19, i32 17, i32 19, i32 20, i32 21, i32 22, i32 23, i32 27, i32 27, i32 25, i32 27, i32 28, i32 29, i32 30, i32 31>
-  ret <32 x i16> %res
-}
-define <32 x i16> @test_masked_32xi16_perm_low_mask3(<32 x i16> %vec, <32 x i16> %vec2, <32 x i16> %mask) {
-; GENERIC-LABEL: test_masked_32xi16_perm_low_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmw %zmm2, %zmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshuflw {{.*#+}} zmm1 {%k1} = zmm0[3,3,1,3,4,5,6,7,11,11,9,11,12,13,14,15,19,19,17,19,20,21,22,23,27,27,25,27,28,29,30,31] sched: [1:1.00]
-; GENERIC-NEXT:    vmovdqa64 %zmm1, %zmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_32xi16_perm_low_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmw %zmm2, %zmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshuflw {{.*#+}} zmm1 {%k1} = zmm0[3,3,1,3,4,5,6,7,11,11,9,11,12,13,14,15,19,19,17,19,20,21,22,23,27,27,25,27,28,29,30,31] sched: [1:1.00]
-; SKX-NEXT:    vmovdqa64 %zmm1, %zmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <32 x i16> %vec, <32 x i16> undef, <32 x i32> <i32 3, i32 3, i32 1, i32 3, i32 4, i32 5, i32 6, i32 7, i32 11, i32 11, i32 9, i32 11, i32 12, i32 13, i32 14, i32 15, i32 19, i32 19, i32 17, i32 19, i32 20, i32 21, i32 22, i32 23, i32 27, i32 27, i32 25, i32 27, i32 28, i32 29, i32 30, i32 31>
-  %cmp = icmp eq <32 x i16> %mask, zeroinitializer
-  %res = select <32 x i1> %cmp, <32 x i16> %shuf, <32 x i16> %vec2
-  ret <32 x i16> %res
-}
-
-define <32 x i16> @test_masked_z_32xi16_perm_low_mask3(<32 x i16> %vec, <32 x i16> %mask) {
-; GENERIC-LABEL: test_masked_z_32xi16_perm_low_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmw %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshuflw {{.*#+}} zmm0 {%k1} {z} = zmm0[3,3,1,3,4,5,6,7,11,11,9,11,12,13,14,15,19,19,17,19,20,21,22,23,27,27,25,27,28,29,30,31] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_32xi16_perm_low_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmw %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshuflw {{.*#+}} zmm0 {%k1} {z} = zmm0[3,3,1,3,4,5,6,7,11,11,9,11,12,13,14,15,19,19,17,19,20,21,22,23,27,27,25,27,28,29,30,31] sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <32 x i16> %vec, <32 x i16> undef, <32 x i32> <i32 3, i32 3, i32 1, i32 3, i32 4, i32 5, i32 6, i32 7, i32 11, i32 11, i32 9, i32 11, i32 12, i32 13, i32 14, i32 15, i32 19, i32 19, i32 17, i32 19, i32 20, i32 21, i32 22, i32 23, i32 27, i32 27, i32 25, i32 27, i32 28, i32 29, i32 30, i32 31>
-  %cmp = icmp eq <32 x i16> %mask, zeroinitializer
-  %res = select <32 x i1> %cmp, <32 x i16> %shuf, <32 x i16> zeroinitializer
-  ret <32 x i16> %res
-}
-define <32 x i16> @test_masked_32xi16_perm_high_mask4(<32 x i16> %vec, <32 x i16> %vec2, <32 x i16> %mask) {
-; GENERIC-LABEL: test_masked_32xi16_perm_high_mask4:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmw %zmm2, %zmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufhw {{.*#+}} zmm1 {%k1} = zmm0[0,1,2,3,7,7,5,6,8,9,10,11,15,15,13,14,16,17,18,19,23,23,21,22,24,25,26,27,31,31,29,30] sched: [1:1.00]
-; GENERIC-NEXT:    vmovdqa64 %zmm1, %zmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_32xi16_perm_high_mask4:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmw %zmm2, %zmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufhw {{.*#+}} zmm1 {%k1} = zmm0[0,1,2,3,7,7,5,6,8,9,10,11,15,15,13,14,16,17,18,19,23,23,21,22,24,25,26,27,31,31,29,30] sched: [1:1.00]
-; SKX-NEXT:    vmovdqa64 %zmm1, %zmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <32 x i16> %vec, <32 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 7, i32 7, i32 5, i32 6, i32 8, i32 9, i32 10, i32 11, i32 15, i32 15, i32 13, i32 14, i32 16, i32 17, i32 18, i32 19, i32 23, i32 23, i32 21, i32 22, i32 24, i32 25, i32 26, i32 27, i32 31, i32 31, i32 29, i32 30>
-  %cmp = icmp eq <32 x i16> %mask, zeroinitializer
-  %res = select <32 x i1> %cmp, <32 x i16> %shuf, <32 x i16> %vec2
-  ret <32 x i16> %res
-}
-
-define <32 x i16> @test_masked_z_32xi16_perm_high_mask4(<32 x i16> %vec, <32 x i16> %mask) {
-; GENERIC-LABEL: test_masked_z_32xi16_perm_high_mask4:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmw %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufhw {{.*#+}} zmm0 {%k1} {z} = zmm0[0,1,2,3,7,7,5,6,8,9,10,11,15,15,13,14,16,17,18,19,23,23,21,22,24,25,26,27,31,31,29,30] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_32xi16_perm_high_mask4:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmw %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufhw {{.*#+}} zmm0 {%k1} {z} = zmm0[0,1,2,3,7,7,5,6,8,9,10,11,15,15,13,14,16,17,18,19,23,23,21,22,24,25,26,27,31,31,29,30] sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <32 x i16> %vec, <32 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 7, i32 7, i32 5, i32 6, i32 8, i32 9, i32 10, i32 11, i32 15, i32 15, i32 13, i32 14, i32 16, i32 17, i32 18, i32 19, i32 23, i32 23, i32 21, i32 22, i32 24, i32 25, i32 26, i32 27, i32 31, i32 31, i32 29, i32 30>
-  %cmp = icmp eq <32 x i16> %mask, zeroinitializer
-  %res = select <32 x i1> %cmp, <32 x i16> %shuf, <32 x i16> zeroinitializer
-  ret <32 x i16> %res
-}
-define <32 x i16> @test_masked_32xi16_perm_low_mask5(<32 x i16> %vec, <32 x i16> %vec2, <32 x i16> %mask) {
-; GENERIC-LABEL: test_masked_32xi16_perm_low_mask5:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmw %zmm2, %zmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshuflw {{.*#+}} zmm1 {%k1} = zmm0[2,1,1,0,4,5,6,7,10,9,9,8,12,13,14,15,18,17,17,16,20,21,22,23,26,25,25,24,28,29,30,31] sched: [1:1.00]
-; GENERIC-NEXT:    vmovdqa64 %zmm1, %zmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_32xi16_perm_low_mask5:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmw %zmm2, %zmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshuflw {{.*#+}} zmm1 {%k1} = zmm0[2,1,1,0,4,5,6,7,10,9,9,8,12,13,14,15,18,17,17,16,20,21,22,23,26,25,25,24,28,29,30,31] sched: [1:1.00]
-; SKX-NEXT:    vmovdqa64 %zmm1, %zmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <32 x i16> %vec, <32 x i16> undef, <32 x i32> <i32 2, i32 1, i32 1, i32 0, i32 4, i32 5, i32 6, i32 7, i32 10, i32 9, i32 9, i32 8, i32 12, i32 13, i32 14, i32 15, i32 18, i32 17, i32 17, i32 16, i32 20, i32 21, i32 22, i32 23, i32 26, i32 25, i32 25, i32 24, i32 28, i32 29, i32 30, i32 31>
-  %cmp = icmp eq <32 x i16> %mask, zeroinitializer
-  %res = select <32 x i1> %cmp, <32 x i16> %shuf, <32 x i16> %vec2
-  ret <32 x i16> %res
-}
-
-define <32 x i16> @test_masked_z_32xi16_perm_low_mask5(<32 x i16> %vec, <32 x i16> %mask) {
-; GENERIC-LABEL: test_masked_z_32xi16_perm_low_mask5:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmw %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshuflw {{.*#+}} zmm0 {%k1} {z} = zmm0[2,1,1,0,4,5,6,7,10,9,9,8,12,13,14,15,18,17,17,16,20,21,22,23,26,25,25,24,28,29,30,31] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_32xi16_perm_low_mask5:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmw %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshuflw {{.*#+}} zmm0 {%k1} {z} = zmm0[2,1,1,0,4,5,6,7,10,9,9,8,12,13,14,15,18,17,17,16,20,21,22,23,26,25,25,24,28,29,30,31] sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <32 x i16> %vec, <32 x i16> undef, <32 x i32> <i32 2, i32 1, i32 1, i32 0, i32 4, i32 5, i32 6, i32 7, i32 10, i32 9, i32 9, i32 8, i32 12, i32 13, i32 14, i32 15, i32 18, i32 17, i32 17, i32 16, i32 20, i32 21, i32 22, i32 23, i32 26, i32 25, i32 25, i32 24, i32 28, i32 29, i32 30, i32 31>
-  %cmp = icmp eq <32 x i16> %mask, zeroinitializer
-  %res = select <32 x i1> %cmp, <32 x i16> %shuf, <32 x i16> zeroinitializer
-  ret <32 x i16> %res
-}
-define <32 x i16> @test_32xi16_perm_high_mask6(<32 x i16> %vec) {
-; GENERIC-LABEL: test_32xi16_perm_high_mask6:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpshufhw {{.*#+}} zmm0 = zmm0[0,1,2,3,4,4,5,6,8,9,10,11,12,12,13,14,16,17,18,19,20,20,21,22,24,25,26,27,28,28,29,30] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_32xi16_perm_high_mask6:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpshufhw {{.*#+}} zmm0 = zmm0[0,1,2,3,4,4,5,6,8,9,10,11,12,12,13,14,16,17,18,19,20,20,21,22,24,25,26,27,28,28,29,30] sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %res = shufflevector <32 x i16> %vec, <32 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 4, i32 5, i32 6, i32 8, i32 9, i32 10, i32 11, i32 12, i32 12, i32 13, i32 14, i32 16, i32 17, i32 18, i32 19, i32 20, i32 20, i32 21, i32 22, i32 24, i32 25, i32 26, i32 27, i32 28, i32 28, i32 29, i32 30>
-  ret <32 x i16> %res
-}
-define <32 x i16> @test_masked_32xi16_perm_high_mask6(<32 x i16> %vec, <32 x i16> %vec2, <32 x i16> %mask) {
-; GENERIC-LABEL: test_masked_32xi16_perm_high_mask6:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmw %zmm2, %zmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufhw {{.*#+}} zmm1 {%k1} = zmm0[0,1,2,3,4,4,5,6,8,9,10,11,12,12,13,14,16,17,18,19,20,20,21,22,24,25,26,27,28,28,29,30] sched: [1:1.00]
-; GENERIC-NEXT:    vmovdqa64 %zmm1, %zmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_32xi16_perm_high_mask6:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmw %zmm2, %zmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufhw {{.*#+}} zmm1 {%k1} = zmm0[0,1,2,3,4,4,5,6,8,9,10,11,12,12,13,14,16,17,18,19,20,20,21,22,24,25,26,27,28,28,29,30] sched: [1:1.00]
-; SKX-NEXT:    vmovdqa64 %zmm1, %zmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <32 x i16> %vec, <32 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 4, i32 5, i32 6, i32 8, i32 9, i32 10, i32 11, i32 12, i32 12, i32 13, i32 14, i32 16, i32 17, i32 18, i32 19, i32 20, i32 20, i32 21, i32 22, i32 24, i32 25, i32 26, i32 27, i32 28, i32 28, i32 29, i32 30>
-  %cmp = icmp eq <32 x i16> %mask, zeroinitializer
-  %res = select <32 x i1> %cmp, <32 x i16> %shuf, <32 x i16> %vec2
-  ret <32 x i16> %res
-}
-
-define <32 x i16> @test_masked_z_32xi16_perm_high_mask6(<32 x i16> %vec, <32 x i16> %mask) {
-; GENERIC-LABEL: test_masked_z_32xi16_perm_high_mask6:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmw %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufhw {{.*#+}} zmm0 {%k1} {z} = zmm0[0,1,2,3,4,4,5,6,8,9,10,11,12,12,13,14,16,17,18,19,20,20,21,22,24,25,26,27,28,28,29,30] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_32xi16_perm_high_mask6:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmw %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufhw {{.*#+}} zmm0 {%k1} {z} = zmm0[0,1,2,3,4,4,5,6,8,9,10,11,12,12,13,14,16,17,18,19,20,20,21,22,24,25,26,27,28,28,29,30] sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <32 x i16> %vec, <32 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 4, i32 5, i32 6, i32 8, i32 9, i32 10, i32 11, i32 12, i32 12, i32 13, i32 14, i32 16, i32 17, i32 18, i32 19, i32 20, i32 20, i32 21, i32 22, i32 24, i32 25, i32 26, i32 27, i32 28, i32 28, i32 29, i32 30>
-  %cmp = icmp eq <32 x i16> %mask, zeroinitializer
-  %res = select <32 x i1> %cmp, <32 x i16> %shuf, <32 x i16> zeroinitializer
-  ret <32 x i16> %res
-}
-define <32 x i16> @test_masked_32xi16_perm_low_mask7(<32 x i16> %vec, <32 x i16> %vec2, <32 x i16> %mask) {
-; GENERIC-LABEL: test_masked_32xi16_perm_low_mask7:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmw %zmm2, %zmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshuflw {{.*#+}} zmm1 {%k1} = zmm0[3,0,3,0,4,5,6,7,11,8,11,8,12,13,14,15,19,16,19,16,20,21,22,23,27,24,27,24,28,29,30,31] sched: [1:1.00]
-; GENERIC-NEXT:    vmovdqa64 %zmm1, %zmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_32xi16_perm_low_mask7:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmw %zmm2, %zmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshuflw {{.*#+}} zmm1 {%k1} = zmm0[3,0,3,0,4,5,6,7,11,8,11,8,12,13,14,15,19,16,19,16,20,21,22,23,27,24,27,24,28,29,30,31] sched: [1:1.00]
-; SKX-NEXT:    vmovdqa64 %zmm1, %zmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <32 x i16> %vec, <32 x i16> undef, <32 x i32> <i32 3, i32 0, i32 3, i32 0, i32 4, i32 5, i32 6, i32 7, i32 11, i32 8, i32 11, i32 8, i32 12, i32 13, i32 14, i32 15, i32 19, i32 16, i32 19, i32 16, i32 20, i32 21, i32 22, i32 23, i32 27, i32 24, i32 27, i32 24, i32 28, i32 29, i32 30, i32 31>
-  %cmp = icmp eq <32 x i16> %mask, zeroinitializer
-  %res = select <32 x i1> %cmp, <32 x i16> %shuf, <32 x i16> %vec2
-  ret <32 x i16> %res
-}
-
-define <32 x i16> @test_masked_z_32xi16_perm_low_mask7(<32 x i16> %vec, <32 x i16> %mask) {
-; GENERIC-LABEL: test_masked_z_32xi16_perm_low_mask7:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmw %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshuflw {{.*#+}} zmm0 {%k1} {z} = zmm0[3,0,3,0,4,5,6,7,11,8,11,8,12,13,14,15,19,16,19,16,20,21,22,23,27,24,27,24,28,29,30,31] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_32xi16_perm_low_mask7:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmw %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshuflw {{.*#+}} zmm0 {%k1} {z} = zmm0[3,0,3,0,4,5,6,7,11,8,11,8,12,13,14,15,19,16,19,16,20,21,22,23,27,24,27,24,28,29,30,31] sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <32 x i16> %vec, <32 x i16> undef, <32 x i32> <i32 3, i32 0, i32 3, i32 0, i32 4, i32 5, i32 6, i32 7, i32 11, i32 8, i32 11, i32 8, i32 12, i32 13, i32 14, i32 15, i32 19, i32 16, i32 19, i32 16, i32 20, i32 21, i32 22, i32 23, i32 27, i32 24, i32 27, i32 24, i32 28, i32 29, i32 30, i32 31>
-  %cmp = icmp eq <32 x i16> %mask, zeroinitializer
-  %res = select <32 x i1> %cmp, <32 x i16> %shuf, <32 x i16> zeroinitializer
-  ret <32 x i16> %res
-}
-define <32 x i16> @test_32xi16_perm_high_mem_mask0(<32 x i16>* %vp) {
-; GENERIC-LABEL: test_32xi16_perm_high_mem_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpshufhw {{.*#+}} zmm0 = mem[0,1,2,3,7,4,5,6,8,9,10,11,15,12,13,14,16,17,18,19,23,20,21,22,24,25,26,27,31,28,29,30] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_32xi16_perm_high_mem_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpshufhw {{.*#+}} zmm0 = mem[0,1,2,3,7,4,5,6,8,9,10,11,15,12,13,14,16,17,18,19,23,20,21,22,24,25,26,27,31,28,29,30] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <32 x i16>, <32 x i16>* %vp
-  %res = shufflevector <32 x i16> %vec, <32 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 7, i32 4, i32 5, i32 6, i32 8, i32 9, i32 10, i32 11, i32 15, i32 12, i32 13, i32 14, i32 16, i32 17, i32 18, i32 19, i32 23, i32 20, i32 21, i32 22, i32 24, i32 25, i32 26, i32 27, i32 31, i32 28, i32 29, i32 30>
-  ret <32 x i16> %res
-}
-define <32 x i16> @test_masked_32xi16_perm_high_mem_mask0(<32 x i16>* %vp, <32 x i16> %vec2, <32 x i16> %mask) {
-; GENERIC-LABEL: test_masked_32xi16_perm_high_mem_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmw %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufhw {{.*#+}} zmm0 {%k1} = mem[0,1,2,3,7,4,5,6,8,9,10,11,15,12,13,14,16,17,18,19,23,20,21,22,24,25,26,27,31,28,29,30] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_32xi16_perm_high_mem_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmw %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufhw {{.*#+}} zmm0 {%k1} = mem[0,1,2,3,7,4,5,6,8,9,10,11,15,12,13,14,16,17,18,19,23,20,21,22,24,25,26,27,31,28,29,30] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <32 x i16>, <32 x i16>* %vp
-  %shuf = shufflevector <32 x i16> %vec, <32 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 7, i32 4, i32 5, i32 6, i32 8, i32 9, i32 10, i32 11, i32 15, i32 12, i32 13, i32 14, i32 16, i32 17, i32 18, i32 19, i32 23, i32 20, i32 21, i32 22, i32 24, i32 25, i32 26, i32 27, i32 31, i32 28, i32 29, i32 30>
-  %cmp = icmp eq <32 x i16> %mask, zeroinitializer
-  %res = select <32 x i1> %cmp, <32 x i16> %shuf, <32 x i16> %vec2
-  ret <32 x i16> %res
-}
-
-define <32 x i16> @test_masked_z_32xi16_perm_high_mem_mask0(<32 x i16>* %vp, <32 x i16> %mask) {
-; GENERIC-LABEL: test_masked_z_32xi16_perm_high_mem_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmw %zmm0, %zmm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufhw {{.*#+}} zmm0 {%k1} {z} = mem[0,1,2,3,7,4,5,6,8,9,10,11,15,12,13,14,16,17,18,19,23,20,21,22,24,25,26,27,31,28,29,30] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_32xi16_perm_high_mem_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmw %zmm0, %zmm0, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufhw {{.*#+}} zmm0 {%k1} {z} = mem[0,1,2,3,7,4,5,6,8,9,10,11,15,12,13,14,16,17,18,19,23,20,21,22,24,25,26,27,31,28,29,30] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <32 x i16>, <32 x i16>* %vp
-  %shuf = shufflevector <32 x i16> %vec, <32 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 7, i32 4, i32 5, i32 6, i32 8, i32 9, i32 10, i32 11, i32 15, i32 12, i32 13, i32 14, i32 16, i32 17, i32 18, i32 19, i32 23, i32 20, i32 21, i32 22, i32 24, i32 25, i32 26, i32 27, i32 31, i32 28, i32 29, i32 30>
-  %cmp = icmp eq <32 x i16> %mask, zeroinitializer
-  %res = select <32 x i1> %cmp, <32 x i16> %shuf, <32 x i16> zeroinitializer
-  ret <32 x i16> %res
-}
-
-define <32 x i16> @test_masked_32xi16_perm_low_mem_mask1(<32 x i16>* %vp, <32 x i16> %vec2, <32 x i16> %mask) {
-; GENERIC-LABEL: test_masked_32xi16_perm_low_mem_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmw %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshuflw {{.*#+}} zmm0 {%k1} = mem[1,1,3,3,4,5,6,7,9,9,11,11,12,13,14,15,17,17,19,19,20,21,22,23,25,25,27,27,28,29,30,31] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_32xi16_perm_low_mem_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmw %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshuflw {{.*#+}} zmm0 {%k1} = mem[1,1,3,3,4,5,6,7,9,9,11,11,12,13,14,15,17,17,19,19,20,21,22,23,25,25,27,27,28,29,30,31] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <32 x i16>, <32 x i16>* %vp
-  %shuf = shufflevector <32 x i16> %vec, <32 x i16> undef, <32 x i32> <i32 1, i32 1, i32 3, i32 3, i32 4, i32 5, i32 6, i32 7, i32 9, i32 9, i32 11, i32 11, i32 12, i32 13, i32 14, i32 15, i32 17, i32 17, i32 19, i32 19, i32 20, i32 21, i32 22, i32 23, i32 25, i32 25, i32 27, i32 27, i32 28, i32 29, i32 30, i32 31>
-  %cmp = icmp eq <32 x i16> %mask, zeroinitializer
-  %res = select <32 x i1> %cmp, <32 x i16> %shuf, <32 x i16> %vec2
-  ret <32 x i16> %res
-}
-
-define <32 x i16> @test_masked_z_32xi16_perm_low_mem_mask1(<32 x i16>* %vp, <32 x i16> %mask) {
-; GENERIC-LABEL: test_masked_z_32xi16_perm_low_mem_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmw %zmm0, %zmm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshuflw {{.*#+}} zmm0 {%k1} {z} = mem[1,1,3,3,4,5,6,7,9,9,11,11,12,13,14,15,17,17,19,19,20,21,22,23,25,25,27,27,28,29,30,31] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_32xi16_perm_low_mem_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmw %zmm0, %zmm0, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshuflw {{.*#+}} zmm0 {%k1} {z} = mem[1,1,3,3,4,5,6,7,9,9,11,11,12,13,14,15,17,17,19,19,20,21,22,23,25,25,27,27,28,29,30,31] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <32 x i16>, <32 x i16>* %vp
-  %shuf = shufflevector <32 x i16> %vec, <32 x i16> undef, <32 x i32> <i32 1, i32 1, i32 3, i32 3, i32 4, i32 5, i32 6, i32 7, i32 9, i32 9, i32 11, i32 11, i32 12, i32 13, i32 14, i32 15, i32 17, i32 17, i32 19, i32 19, i32 20, i32 21, i32 22, i32 23, i32 25, i32 25, i32 27, i32 27, i32 28, i32 29, i32 30, i32 31>
-  %cmp = icmp eq <32 x i16> %mask, zeroinitializer
-  %res = select <32 x i1> %cmp, <32 x i16> %shuf, <32 x i16> zeroinitializer
-  ret <32 x i16> %res
-}
-
-define <32 x i16> @test_masked_32xi16_perm_high_mem_mask2(<32 x i16>* %vp, <32 x i16> %vec2, <32 x i16> %mask) {
-; GENERIC-LABEL: test_masked_32xi16_perm_high_mem_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmw %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufhw {{.*#+}} zmm0 {%k1} = mem[0,1,2,3,4,7,6,4,8,9,10,11,12,15,14,12,16,17,18,19,20,23,22,20,24,25,26,27,28,31,30,28] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_32xi16_perm_high_mem_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmw %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufhw {{.*#+}} zmm0 {%k1} = mem[0,1,2,3,4,7,6,4,8,9,10,11,12,15,14,12,16,17,18,19,20,23,22,20,24,25,26,27,28,31,30,28] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <32 x i16>, <32 x i16>* %vp
-  %shuf = shufflevector <32 x i16> %vec, <32 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 7, i32 6, i32 4, i32 8, i32 9, i32 10, i32 11, i32 12, i32 15, i32 14, i32 12, i32 16, i32 17, i32 18, i32 19, i32 20, i32 23, i32 22, i32 20, i32 24, i32 25, i32 26, i32 27, i32 28, i32 31, i32 30, i32 28>
-  %cmp = icmp eq <32 x i16> %mask, zeroinitializer
-  %res = select <32 x i1> %cmp, <32 x i16> %shuf, <32 x i16> %vec2
-  ret <32 x i16> %res
-}
-
-define <32 x i16> @test_masked_z_32xi16_perm_high_mem_mask2(<32 x i16>* %vp, <32 x i16> %mask) {
-; GENERIC-LABEL: test_masked_z_32xi16_perm_high_mem_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmw %zmm0, %zmm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufhw {{.*#+}} zmm0 {%k1} {z} = mem[0,1,2,3,4,7,6,4,8,9,10,11,12,15,14,12,16,17,18,19,20,23,22,20,24,25,26,27,28,31,30,28] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_32xi16_perm_high_mem_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmw %zmm0, %zmm0, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufhw {{.*#+}} zmm0 {%k1} {z} = mem[0,1,2,3,4,7,6,4,8,9,10,11,12,15,14,12,16,17,18,19,20,23,22,20,24,25,26,27,28,31,30,28] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <32 x i16>, <32 x i16>* %vp
-  %shuf = shufflevector <32 x i16> %vec, <32 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 7, i32 6, i32 4, i32 8, i32 9, i32 10, i32 11, i32 12, i32 15, i32 14, i32 12, i32 16, i32 17, i32 18, i32 19, i32 20, i32 23, i32 22, i32 20, i32 24, i32 25, i32 26, i32 27, i32 28, i32 31, i32 30, i32 28>
-  %cmp = icmp eq <32 x i16> %mask, zeroinitializer
-  %res = select <32 x i1> %cmp, <32 x i16> %shuf, <32 x i16> zeroinitializer
-  ret <32 x i16> %res
-}
-
-define <32 x i16> @test_32xi16_perm_low_mem_mask3(<32 x i16>* %vp) {
-; GENERIC-LABEL: test_32xi16_perm_low_mem_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpshuflw {{.*#+}} zmm0 = mem[2,2,0,3,4,5,6,7,10,10,8,11,12,13,14,15,18,18,16,19,20,21,22,23,26,26,24,27,28,29,30,31] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_32xi16_perm_low_mem_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpshuflw {{.*#+}} zmm0 = mem[2,2,0,3,4,5,6,7,10,10,8,11,12,13,14,15,18,18,16,19,20,21,22,23,26,26,24,27,28,29,30,31] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <32 x i16>, <32 x i16>* %vp
-  %res = shufflevector <32 x i16> %vec, <32 x i16> undef, <32 x i32> <i32 2, i32 2, i32 0, i32 3, i32 4, i32 5, i32 6, i32 7, i32 10, i32 10, i32 8, i32 11, i32 12, i32 13, i32 14, i32 15, i32 18, i32 18, i32 16, i32 19, i32 20, i32 21, i32 22, i32 23, i32 26, i32 26, i32 24, i32 27, i32 28, i32 29, i32 30, i32 31>
-  ret <32 x i16> %res
-}
-define <32 x i16> @test_masked_32xi16_perm_low_mem_mask3(<32 x i16>* %vp, <32 x i16> %vec2, <32 x i16> %mask) {
-; GENERIC-LABEL: test_masked_32xi16_perm_low_mem_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmw %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshuflw {{.*#+}} zmm0 {%k1} = mem[2,2,0,3,4,5,6,7,10,10,8,11,12,13,14,15,18,18,16,19,20,21,22,23,26,26,24,27,28,29,30,31] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_32xi16_perm_low_mem_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmw %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshuflw {{.*#+}} zmm0 {%k1} = mem[2,2,0,3,4,5,6,7,10,10,8,11,12,13,14,15,18,18,16,19,20,21,22,23,26,26,24,27,28,29,30,31] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <32 x i16>, <32 x i16>* %vp
-  %shuf = shufflevector <32 x i16> %vec, <32 x i16> undef, <32 x i32> <i32 2, i32 2, i32 0, i32 3, i32 4, i32 5, i32 6, i32 7, i32 10, i32 10, i32 8, i32 11, i32 12, i32 13, i32 14, i32 15, i32 18, i32 18, i32 16, i32 19, i32 20, i32 21, i32 22, i32 23, i32 26, i32 26, i32 24, i32 27, i32 28, i32 29, i32 30, i32 31>
-  %cmp = icmp eq <32 x i16> %mask, zeroinitializer
-  %res = select <32 x i1> %cmp, <32 x i16> %shuf, <32 x i16> %vec2
-  ret <32 x i16> %res
-}
-
-define <32 x i16> @test_masked_z_32xi16_perm_low_mem_mask3(<32 x i16>* %vp, <32 x i16> %mask) {
-; GENERIC-LABEL: test_masked_z_32xi16_perm_low_mem_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmw %zmm0, %zmm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshuflw {{.*#+}} zmm0 {%k1} {z} = mem[2,2,0,3,4,5,6,7,10,10,8,11,12,13,14,15,18,18,16,19,20,21,22,23,26,26,24,27,28,29,30,31] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_32xi16_perm_low_mem_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmw %zmm0, %zmm0, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshuflw {{.*#+}} zmm0 {%k1} {z} = mem[2,2,0,3,4,5,6,7,10,10,8,11,12,13,14,15,18,18,16,19,20,21,22,23,26,26,24,27,28,29,30,31] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <32 x i16>, <32 x i16>* %vp
-  %shuf = shufflevector <32 x i16> %vec, <32 x i16> undef, <32 x i32> <i32 2, i32 2, i32 0, i32 3, i32 4, i32 5, i32 6, i32 7, i32 10, i32 10, i32 8, i32 11, i32 12, i32 13, i32 14, i32 15, i32 18, i32 18, i32 16, i32 19, i32 20, i32 21, i32 22, i32 23, i32 26, i32 26, i32 24, i32 27, i32 28, i32 29, i32 30, i32 31>
-  %cmp = icmp eq <32 x i16> %mask, zeroinitializer
-  %res = select <32 x i1> %cmp, <32 x i16> %shuf, <32 x i16> zeroinitializer
-  ret <32 x i16> %res
-}
-
-define <32 x i16> @test_masked_32xi16_perm_high_mem_mask4(<32 x i16>* %vp, <32 x i16> %vec2, <32 x i16> %mask) {
-; GENERIC-LABEL: test_masked_32xi16_perm_high_mem_mask4:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmw %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufhw {{.*#+}} zmm0 {%k1} = mem[0,1,2,3,7,4,6,5,8,9,10,11,15,12,14,13,16,17,18,19,23,20,22,21,24,25,26,27,31,28,30,29] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_32xi16_perm_high_mem_mask4:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmw %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufhw {{.*#+}} zmm0 {%k1} = mem[0,1,2,3,7,4,6,5,8,9,10,11,15,12,14,13,16,17,18,19,23,20,22,21,24,25,26,27,31,28,30,29] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <32 x i16>, <32 x i16>* %vp
-  %shuf = shufflevector <32 x i16> %vec, <32 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 7, i32 4, i32 6, i32 5, i32 8, i32 9, i32 10, i32 11, i32 15, i32 12, i32 14, i32 13, i32 16, i32 17, i32 18, i32 19, i32 23, i32 20, i32 22, i32 21, i32 24, i32 25, i32 26, i32 27, i32 31, i32 28, i32 30, i32 29>
-  %cmp = icmp eq <32 x i16> %mask, zeroinitializer
-  %res = select <32 x i1> %cmp, <32 x i16> %shuf, <32 x i16> %vec2
-  ret <32 x i16> %res
-}
-
-define <32 x i16> @test_masked_z_32xi16_perm_high_mem_mask4(<32 x i16>* %vp, <32 x i16> %mask) {
-; GENERIC-LABEL: test_masked_z_32xi16_perm_high_mem_mask4:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmw %zmm0, %zmm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufhw {{.*#+}} zmm0 {%k1} {z} = mem[0,1,2,3,7,4,6,5,8,9,10,11,15,12,14,13,16,17,18,19,23,20,22,21,24,25,26,27,31,28,30,29] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_32xi16_perm_high_mem_mask4:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmw %zmm0, %zmm0, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufhw {{.*#+}} zmm0 {%k1} {z} = mem[0,1,2,3,7,4,6,5,8,9,10,11,15,12,14,13,16,17,18,19,23,20,22,21,24,25,26,27,31,28,30,29] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <32 x i16>, <32 x i16>* %vp
-  %shuf = shufflevector <32 x i16> %vec, <32 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 7, i32 4, i32 6, i32 5, i32 8, i32 9, i32 10, i32 11, i32 15, i32 12, i32 14, i32 13, i32 16, i32 17, i32 18, i32 19, i32 23, i32 20, i32 22, i32 21, i32 24, i32 25, i32 26, i32 27, i32 31, i32 28, i32 30, i32 29>
-  %cmp = icmp eq <32 x i16> %mask, zeroinitializer
-  %res = select <32 x i1> %cmp, <32 x i16> %shuf, <32 x i16> zeroinitializer
-  ret <32 x i16> %res
-}
-
-define <32 x i16> @test_masked_32xi16_perm_low_mem_mask5(<32 x i16>* %vp, <32 x i16> %vec2, <32 x i16> %mask) {
-; GENERIC-LABEL: test_masked_32xi16_perm_low_mem_mask5:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpshufd {{.*#+}} zmm2 = mem[0,0,2,3,4,4,6,7,8,8,10,11,12,12,14,15] sched: [8:1.00]
-; GENERIC-NEXT:    vptestnmw %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vmovdqu16 %zmm2, %zmm0 {%k1} # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_32xi16_perm_low_mem_mask5:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpshufd {{.*#+}} zmm2 = mem[0,0,2,3,4,4,6,7,8,8,10,11,12,12,14,15] sched: [8:1.00]
-; SKX-NEXT:    vptestnmw %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vmovdqu16 %zmm2, %zmm0 {%k1} # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <32 x i16>, <32 x i16>* %vp
-  %shuf = shufflevector <32 x i16> %vec, <32 x i16> undef, <32 x i32> <i32 0, i32 1, i32 0, i32 1, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 8, i32 9, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 16, i32 17, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 24, i32 25, i32 28, i32 29, i32 30, i32 31>
-  %cmp = icmp eq <32 x i16> %mask, zeroinitializer
-  %res = select <32 x i1> %cmp, <32 x i16> %shuf, <32 x i16> %vec2
-  ret <32 x i16> %res
-}
-
-define <32 x i16> @test_masked_z_32xi16_perm_low_mem_mask5(<32 x i16>* %vp, <32 x i16> %mask) {
-; GENERIC-LABEL: test_masked_z_32xi16_perm_low_mem_mask5:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpshufd {{.*#+}} zmm1 = mem[0,0,2,3,4,4,6,7,8,8,10,11,12,12,14,15] sched: [8:1.00]
-; GENERIC-NEXT:    vptestnmw %zmm0, %zmm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vmovdqu16 %zmm1, %zmm0 {%k1} {z} # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_32xi16_perm_low_mem_mask5:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpshufd {{.*#+}} zmm1 = mem[0,0,2,3,4,4,6,7,8,8,10,11,12,12,14,15] sched: [8:1.00]
-; SKX-NEXT:    vptestnmw %zmm0, %zmm0, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vmovdqu16 %zmm1, %zmm0 {%k1} {z} # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <32 x i16>, <32 x i16>* %vp
-  %shuf = shufflevector <32 x i16> %vec, <32 x i16> undef, <32 x i32> <i32 0, i32 1, i32 0, i32 1, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 8, i32 9, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 16, i32 17, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 24, i32 25, i32 28, i32 29, i32 30, i32 31>
-  %cmp = icmp eq <32 x i16> %mask, zeroinitializer
-  %res = select <32 x i1> %cmp, <32 x i16> %shuf, <32 x i16> zeroinitializer
-  ret <32 x i16> %res
-}
-
-define <32 x i16> @test_32xi16_perm_high_mem_mask6(<32 x i16>* %vp) {
-; GENERIC-LABEL: test_32xi16_perm_high_mem_mask6:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpshufhw {{.*#+}} zmm0 = mem[0,1,2,3,6,5,6,6,8,9,10,11,14,13,14,14,16,17,18,19,22,21,22,22,24,25,26,27,30,29,30,30] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_32xi16_perm_high_mem_mask6:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpshufhw {{.*#+}} zmm0 = mem[0,1,2,3,6,5,6,6,8,9,10,11,14,13,14,14,16,17,18,19,22,21,22,22,24,25,26,27,30,29,30,30] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <32 x i16>, <32 x i16>* %vp
-  %res = shufflevector <32 x i16> %vec, <32 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 6, i32 5, i32 6, i32 6, i32 8, i32 9, i32 10, i32 11, i32 14, i32 13, i32 14, i32 14, i32 16, i32 17, i32 18, i32 19, i32 22, i32 21, i32 22, i32 22, i32 24, i32 25, i32 26, i32 27, i32 30, i32 29, i32 30, i32 30>
-  ret <32 x i16> %res
-}
-define <32 x i16> @test_masked_32xi16_perm_high_mem_mask6(<32 x i16>* %vp, <32 x i16> %vec2, <32 x i16> %mask) {
-; GENERIC-LABEL: test_masked_32xi16_perm_high_mem_mask6:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmw %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufhw {{.*#+}} zmm0 {%k1} = mem[0,1,2,3,6,5,6,6,8,9,10,11,14,13,14,14,16,17,18,19,22,21,22,22,24,25,26,27,30,29,30,30] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_32xi16_perm_high_mem_mask6:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmw %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufhw {{.*#+}} zmm0 {%k1} = mem[0,1,2,3,6,5,6,6,8,9,10,11,14,13,14,14,16,17,18,19,22,21,22,22,24,25,26,27,30,29,30,30] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <32 x i16>, <32 x i16>* %vp
-  %shuf = shufflevector <32 x i16> %vec, <32 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 6, i32 5, i32 6, i32 6, i32 8, i32 9, i32 10, i32 11, i32 14, i32 13, i32 14, i32 14, i32 16, i32 17, i32 18, i32 19, i32 22, i32 21, i32 22, i32 22, i32 24, i32 25, i32 26, i32 27, i32 30, i32 29, i32 30, i32 30>
-  %cmp = icmp eq <32 x i16> %mask, zeroinitializer
-  %res = select <32 x i1> %cmp, <32 x i16> %shuf, <32 x i16> %vec2
-  ret <32 x i16> %res
-}
-
-define <32 x i16> @test_masked_z_32xi16_perm_high_mem_mask6(<32 x i16>* %vp, <32 x i16> %mask) {
-; GENERIC-LABEL: test_masked_z_32xi16_perm_high_mem_mask6:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmw %zmm0, %zmm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufhw {{.*#+}} zmm0 {%k1} {z} = mem[0,1,2,3,6,5,6,6,8,9,10,11,14,13,14,14,16,17,18,19,22,21,22,22,24,25,26,27,30,29,30,30] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_32xi16_perm_high_mem_mask6:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmw %zmm0, %zmm0, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufhw {{.*#+}} zmm0 {%k1} {z} = mem[0,1,2,3,6,5,6,6,8,9,10,11,14,13,14,14,16,17,18,19,22,21,22,22,24,25,26,27,30,29,30,30] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <32 x i16>, <32 x i16>* %vp
-  %shuf = shufflevector <32 x i16> %vec, <32 x i16> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 6, i32 5, i32 6, i32 6, i32 8, i32 9, i32 10, i32 11, i32 14, i32 13, i32 14, i32 14, i32 16, i32 17, i32 18, i32 19, i32 22, i32 21, i32 22, i32 22, i32 24, i32 25, i32 26, i32 27, i32 30, i32 29, i32 30, i32 30>
-  %cmp = icmp eq <32 x i16> %mask, zeroinitializer
-  %res = select <32 x i1> %cmp, <32 x i16> %shuf, <32 x i16> zeroinitializer
-  ret <32 x i16> %res
-}
-
-define <32 x i16> @test_masked_32xi16_perm_low_mem_mask7(<32 x i16>* %vp, <32 x i16> %vec2, <32 x i16> %mask) {
-; GENERIC-LABEL: test_masked_32xi16_perm_low_mem_mask7:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmw %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshuflw {{.*#+}} zmm0 {%k1} = mem[3,1,3,0,4,5,6,7,11,9,11,8,12,13,14,15,19,17,19,16,20,21,22,23,27,25,27,24,28,29,30,31] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_32xi16_perm_low_mem_mask7:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmw %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshuflw {{.*#+}} zmm0 {%k1} = mem[3,1,3,0,4,5,6,7,11,9,11,8,12,13,14,15,19,17,19,16,20,21,22,23,27,25,27,24,28,29,30,31] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <32 x i16>, <32 x i16>* %vp
-  %shuf = shufflevector <32 x i16> %vec, <32 x i16> undef, <32 x i32> <i32 3, i32 1, i32 3, i32 0, i32 4, i32 5, i32 6, i32 7, i32 11, i32 9, i32 11, i32 8, i32 12, i32 13, i32 14, i32 15, i32 19, i32 17, i32 19, i32 16, i32 20, i32 21, i32 22, i32 23, i32 27, i32 25, i32 27, i32 24, i32 28, i32 29, i32 30, i32 31>
-  %cmp = icmp eq <32 x i16> %mask, zeroinitializer
-  %res = select <32 x i1> %cmp, <32 x i16> %shuf, <32 x i16> %vec2
-  ret <32 x i16> %res
-}
-
-define <32 x i16> @test_masked_z_32xi16_perm_low_mem_mask7(<32 x i16>* %vp, <32 x i16> %mask) {
-; GENERIC-LABEL: test_masked_z_32xi16_perm_low_mem_mask7:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmw %zmm0, %zmm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshuflw {{.*#+}} zmm0 {%k1} {z} = mem[3,1,3,0,4,5,6,7,11,9,11,8,12,13,14,15,19,17,19,16,20,21,22,23,27,25,27,24,28,29,30,31] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_32xi16_perm_low_mem_mask7:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmw %zmm0, %zmm0, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshuflw {{.*#+}} zmm0 {%k1} {z} = mem[3,1,3,0,4,5,6,7,11,9,11,8,12,13,14,15,19,17,19,16,20,21,22,23,27,25,27,24,28,29,30,31] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <32 x i16>, <32 x i16>* %vp
-  %shuf = shufflevector <32 x i16> %vec, <32 x i16> undef, <32 x i32> <i32 3, i32 1, i32 3, i32 0, i32 4, i32 5, i32 6, i32 7, i32 11, i32 9, i32 11, i32 8, i32 12, i32 13, i32 14, i32 15, i32 19, i32 17, i32 19, i32 16, i32 20, i32 21, i32 22, i32 23, i32 27, i32 25, i32 27, i32 24, i32 28, i32 29, i32 30, i32 31>
-  %cmp = icmp eq <32 x i16> %mask, zeroinitializer
-  %res = select <32 x i1> %cmp, <32 x i16> %shuf, <32 x i16> zeroinitializer
-  ret <32 x i16> %res
-}
-
-define <4 x i32> @test_4xi32_perm_mask0(<4 x i32> %vec) {
-; GENERIC-LABEL: test_4xi32_perm_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[2,3,3,0] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xi32_perm_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[2,3,3,0] sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %res = shufflevector <4 x i32> %vec, <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 3, i32 0>
-  ret <4 x i32> %res
-}
-define <4 x i32> @test_masked_4xi32_perm_mask0(<4 x i32> %vec, <4 x i32> %vec2, <4 x i32> %mask) {
-; GENERIC-LABEL: test_masked_4xi32_perm_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %xmm2, %xmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufd {{.*#+}} xmm1 {%k1} = xmm0[2,3,3,0] sched: [1:0.50]
-; GENERIC-NEXT:    vmovdqa %xmm1, %xmm0 # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_4xi32_perm_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %xmm2, %xmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufd {{.*#+}} xmm1 {%k1} = xmm0[2,3,3,0] sched: [1:1.00]
-; SKX-NEXT:    vmovdqa %xmm1, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <4 x i32> %vec, <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 3, i32 0>
-  %cmp = icmp eq <4 x i32> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x i32> %shuf, <4 x i32> %vec2
-  ret <4 x i32> %res
-}
-
-define <4 x i32> @test_masked_z_4xi32_perm_mask0(<4 x i32> %vec, <4 x i32> %mask) {
-; GENERIC-LABEL: test_masked_z_4xi32_perm_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %xmm1, %xmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufd {{.*#+}} xmm0 {%k1} {z} = xmm0[2,3,3,0] sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_4xi32_perm_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %xmm1, %xmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufd {{.*#+}} xmm0 {%k1} {z} = xmm0[2,3,3,0] sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <4 x i32> %vec, <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 3, i32 0>
-  %cmp = icmp eq <4 x i32> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x i32> %shuf, <4 x i32> zeroinitializer
-  ret <4 x i32> %res
-}
-define <4 x i32> @test_masked_4xi32_perm_mask1(<4 x i32> %vec, <4 x i32> %vec2, <4 x i32> %mask) {
-; GENERIC-LABEL: test_masked_4xi32_perm_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %xmm2, %xmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufd {{.*#+}} xmm1 {%k1} = xmm0[1,0,2,0] sched: [1:0.50]
-; GENERIC-NEXT:    vmovdqa %xmm1, %xmm0 # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_4xi32_perm_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %xmm2, %xmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufd {{.*#+}} xmm1 {%k1} = xmm0[1,0,2,0] sched: [1:1.00]
-; SKX-NEXT:    vmovdqa %xmm1, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <4 x i32> %vec, <4 x i32> undef, <4 x i32> <i32 1, i32 0, i32 2, i32 0>
-  %cmp = icmp eq <4 x i32> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x i32> %shuf, <4 x i32> %vec2
-  ret <4 x i32> %res
-}
-
-define <4 x i32> @test_masked_z_4xi32_perm_mask1(<4 x i32> %vec, <4 x i32> %mask) {
-; GENERIC-LABEL: test_masked_z_4xi32_perm_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %xmm1, %xmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufd {{.*#+}} xmm0 {%k1} {z} = xmm0[1,0,2,0] sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_4xi32_perm_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %xmm1, %xmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufd {{.*#+}} xmm0 {%k1} {z} = xmm0[1,0,2,0] sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <4 x i32> %vec, <4 x i32> undef, <4 x i32> <i32 1, i32 0, i32 2, i32 0>
-  %cmp = icmp eq <4 x i32> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x i32> %shuf, <4 x i32> zeroinitializer
-  ret <4 x i32> %res
-}
-define <4 x i32> @test_masked_4xi32_perm_mask2(<4 x i32> %vec, <4 x i32> %vec2, <4 x i32> %mask) {
-; GENERIC-LABEL: test_masked_4xi32_perm_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %xmm2, %xmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufd {{.*#+}} xmm1 {%k1} = xmm0[3,0,1,0] sched: [1:0.50]
-; GENERIC-NEXT:    vmovdqa %xmm1, %xmm0 # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_4xi32_perm_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %xmm2, %xmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufd {{.*#+}} xmm1 {%k1} = xmm0[3,0,1,0] sched: [1:1.00]
-; SKX-NEXT:    vmovdqa %xmm1, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <4 x i32> %vec, <4 x i32> undef, <4 x i32> <i32 3, i32 0, i32 1, i32 0>
-  %cmp = icmp eq <4 x i32> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x i32> %shuf, <4 x i32> %vec2
-  ret <4 x i32> %res
-}
-
-define <4 x i32> @test_masked_z_4xi32_perm_mask2(<4 x i32> %vec, <4 x i32> %mask) {
-; GENERIC-LABEL: test_masked_z_4xi32_perm_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %xmm1, %xmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufd {{.*#+}} xmm0 {%k1} {z} = xmm0[3,0,1,0] sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_4xi32_perm_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %xmm1, %xmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufd {{.*#+}} xmm0 {%k1} {z} = xmm0[3,0,1,0] sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <4 x i32> %vec, <4 x i32> undef, <4 x i32> <i32 3, i32 0, i32 1, i32 0>
-  %cmp = icmp eq <4 x i32> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x i32> %shuf, <4 x i32> zeroinitializer
-  ret <4 x i32> %res
-}
-define <4 x i32> @test_4xi32_perm_mask3(<4 x i32> %vec) {
-; GENERIC-LABEL: test_4xi32_perm_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[1,1,0,3] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xi32_perm_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[1,1,0,3] sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %res = shufflevector <4 x i32> %vec, <4 x i32> undef, <4 x i32> <i32 1, i32 1, i32 0, i32 3>
-  ret <4 x i32> %res
-}
-define <4 x i32> @test_masked_4xi32_perm_mask3(<4 x i32> %vec, <4 x i32> %vec2, <4 x i32> %mask) {
-; GENERIC-LABEL: test_masked_4xi32_perm_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %xmm2, %xmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufd {{.*#+}} xmm1 {%k1} = xmm0[1,1,0,3] sched: [1:0.50]
-; GENERIC-NEXT:    vmovdqa %xmm1, %xmm0 # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_4xi32_perm_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %xmm2, %xmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufd {{.*#+}} xmm1 {%k1} = xmm0[1,1,0,3] sched: [1:1.00]
-; SKX-NEXT:    vmovdqa %xmm1, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <4 x i32> %vec, <4 x i32> undef, <4 x i32> <i32 1, i32 1, i32 0, i32 3>
-  %cmp = icmp eq <4 x i32> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x i32> %shuf, <4 x i32> %vec2
-  ret <4 x i32> %res
-}
-
-define <4 x i32> @test_masked_z_4xi32_perm_mask3(<4 x i32> %vec, <4 x i32> %mask) {
-; GENERIC-LABEL: test_masked_z_4xi32_perm_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %xmm1, %xmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufd {{.*#+}} xmm0 {%k1} {z} = xmm0[1,1,0,3] sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_4xi32_perm_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %xmm1, %xmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufd {{.*#+}} xmm0 {%k1} {z} = xmm0[1,1,0,3] sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <4 x i32> %vec, <4 x i32> undef, <4 x i32> <i32 1, i32 1, i32 0, i32 3>
-  %cmp = icmp eq <4 x i32> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x i32> %shuf, <4 x i32> zeroinitializer
-  ret <4 x i32> %res
-}
-define <4 x i32> @test_4xi32_perm_mem_mask0(<4 x i32>* %vp) {
-; GENERIC-LABEL: test_4xi32_perm_mem_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpermilps {{.*#+}} xmm0 = mem[0,1,3,3] sched: [7:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xi32_perm_mem_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpermilps {{.*#+}} xmm0 = mem[0,1,3,3] sched: [7:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <4 x i32>, <4 x i32>* %vp
-  %res = shufflevector <4 x i32> %vec, <4 x i32> undef, <4 x i32> <i32 0, i32 1, i32 3, i32 3>
-  ret <4 x i32> %res
-}
-define <4 x i32> @test_masked_4xi32_perm_mem_mask0(<4 x i32>* %vp, <4 x i32> %vec2, <4 x i32> %mask) {
-; GENERIC-LABEL: test_masked_4xi32_perm_mem_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %xmm1, %xmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufd {{.*#+}} xmm0 {%k1} = mem[0,1,3,3] sched: [7:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_4xi32_perm_mem_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %xmm1, %xmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufd {{.*#+}} xmm0 {%k1} = mem[0,1,3,3] sched: [7:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <4 x i32>, <4 x i32>* %vp
-  %shuf = shufflevector <4 x i32> %vec, <4 x i32> undef, <4 x i32> <i32 0, i32 1, i32 3, i32 3>
-  %cmp = icmp eq <4 x i32> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x i32> %shuf, <4 x i32> %vec2
-  ret <4 x i32> %res
-}
-
-define <4 x i32> @test_masked_z_4xi32_perm_mem_mask0(<4 x i32>* %vp, <4 x i32> %mask) {
-; GENERIC-LABEL: test_masked_z_4xi32_perm_mem_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %xmm0, %xmm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufd {{.*#+}} xmm0 {%k1} {z} = mem[0,1,3,3] sched: [7:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_4xi32_perm_mem_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %xmm0, %xmm0, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufd {{.*#+}} xmm0 {%k1} {z} = mem[0,1,3,3] sched: [7:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <4 x i32>, <4 x i32>* %vp
-  %shuf = shufflevector <4 x i32> %vec, <4 x i32> undef, <4 x i32> <i32 0, i32 1, i32 3, i32 3>
-  %cmp = icmp eq <4 x i32> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x i32> %shuf, <4 x i32> zeroinitializer
-  ret <4 x i32> %res
-}
-
-define <4 x i32> @test_masked_4xi32_perm_mem_mask1(<4 x i32>* %vp, <4 x i32> %vec2, <4 x i32> %mask) {
-; GENERIC-LABEL: test_masked_4xi32_perm_mem_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %xmm1, %xmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufd {{.*#+}} xmm0 {%k1} = mem[2,2,3,1] sched: [7:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_4xi32_perm_mem_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %xmm1, %xmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufd {{.*#+}} xmm0 {%k1} = mem[2,2,3,1] sched: [7:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <4 x i32>, <4 x i32>* %vp
-  %shuf = shufflevector <4 x i32> %vec, <4 x i32> undef, <4 x i32> <i32 2, i32 2, i32 3, i32 1>
-  %cmp = icmp eq <4 x i32> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x i32> %shuf, <4 x i32> %vec2
-  ret <4 x i32> %res
-}
-
-define <4 x i32> @test_masked_z_4xi32_perm_mem_mask1(<4 x i32>* %vp, <4 x i32> %mask) {
-; GENERIC-LABEL: test_masked_z_4xi32_perm_mem_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %xmm0, %xmm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufd {{.*#+}} xmm0 {%k1} {z} = mem[2,2,3,1] sched: [7:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_4xi32_perm_mem_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %xmm0, %xmm0, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufd {{.*#+}} xmm0 {%k1} {z} = mem[2,2,3,1] sched: [7:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <4 x i32>, <4 x i32>* %vp
-  %shuf = shufflevector <4 x i32> %vec, <4 x i32> undef, <4 x i32> <i32 2, i32 2, i32 3, i32 1>
-  %cmp = icmp eq <4 x i32> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x i32> %shuf, <4 x i32> zeroinitializer
-  ret <4 x i32> %res
-}
-
-define <4 x i32> @test_masked_4xi32_perm_mem_mask2(<4 x i32>* %vp, <4 x i32> %vec2, <4 x i32> %mask) {
-; GENERIC-LABEL: test_masked_4xi32_perm_mem_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %xmm1, %xmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufd {{.*#+}} xmm0 {%k1} = mem[0,3,0,1] sched: [7:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_4xi32_perm_mem_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %xmm1, %xmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufd {{.*#+}} xmm0 {%k1} = mem[0,3,0,1] sched: [7:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <4 x i32>, <4 x i32>* %vp
-  %shuf = shufflevector <4 x i32> %vec, <4 x i32> undef, <4 x i32> <i32 0, i32 3, i32 0, i32 1>
-  %cmp = icmp eq <4 x i32> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x i32> %shuf, <4 x i32> %vec2
-  ret <4 x i32> %res
-}
-
-define <4 x i32> @test_masked_z_4xi32_perm_mem_mask2(<4 x i32>* %vp, <4 x i32> %mask) {
-; GENERIC-LABEL: test_masked_z_4xi32_perm_mem_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %xmm0, %xmm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufd {{.*#+}} xmm0 {%k1} {z} = mem[0,3,0,1] sched: [7:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_4xi32_perm_mem_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %xmm0, %xmm0, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufd {{.*#+}} xmm0 {%k1} {z} = mem[0,3,0,1] sched: [7:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <4 x i32>, <4 x i32>* %vp
-  %shuf = shufflevector <4 x i32> %vec, <4 x i32> undef, <4 x i32> <i32 0, i32 3, i32 0, i32 1>
-  %cmp = icmp eq <4 x i32> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x i32> %shuf, <4 x i32> zeroinitializer
-  ret <4 x i32> %res
-}
-
-define <4 x i32> @test_4xi32_perm_mem_mask3(<4 x i32>* %vp) {
-; GENERIC-LABEL: test_4xi32_perm_mem_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpermilps {{.*#+}} xmm0 = mem[1,0,1,0] sched: [7:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xi32_perm_mem_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpermilps {{.*#+}} xmm0 = mem[1,0,1,0] sched: [7:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <4 x i32>, <4 x i32>* %vp
-  %res = shufflevector <4 x i32> %vec, <4 x i32> undef, <4 x i32> <i32 1, i32 0, i32 1, i32 0>
-  ret <4 x i32> %res
-}
-define <4 x i32> @test_masked_4xi32_perm_mem_mask3(<4 x i32>* %vp, <4 x i32> %vec2, <4 x i32> %mask) {
-; GENERIC-LABEL: test_masked_4xi32_perm_mem_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %xmm1, %xmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufd {{.*#+}} xmm0 {%k1} = mem[1,0,1,0] sched: [7:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_4xi32_perm_mem_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %xmm1, %xmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufd {{.*#+}} xmm0 {%k1} = mem[1,0,1,0] sched: [7:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <4 x i32>, <4 x i32>* %vp
-  %shuf = shufflevector <4 x i32> %vec, <4 x i32> undef, <4 x i32> <i32 1, i32 0, i32 1, i32 0>
-  %cmp = icmp eq <4 x i32> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x i32> %shuf, <4 x i32> %vec2
-  ret <4 x i32> %res
-}
-
-define <4 x i32> @test_masked_z_4xi32_perm_mem_mask3(<4 x i32>* %vp, <4 x i32> %mask) {
-; GENERIC-LABEL: test_masked_z_4xi32_perm_mem_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %xmm0, %xmm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufd {{.*#+}} xmm0 {%k1} {z} = mem[1,0,1,0] sched: [7:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_masked_z_4xi32_perm_mem_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %xmm0, %xmm0, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufd {{.*#+}} xmm0 {%k1} {z} = mem[1,0,1,0] sched: [7:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <4 x i32>, <4 x i32>* %vp
-  %shuf = shufflevector <4 x i32> %vec, <4 x i32> undef, <4 x i32> <i32 1, i32 0, i32 1, i32 0>
-  %cmp = icmp eq <4 x i32> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x i32> %shuf, <4 x i32> zeroinitializer
-  ret <4 x i32> %res
-}
-
-define <8 x i32> @test2_8xi32_perm_mask0(<8 x i32> %vec) {
-; GENERIC-LABEL: test2_8xi32_perm_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpermilps {{.*#+}} ymm0 = ymm0[2,3,1,0,6,7,5,4] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test2_8xi32_perm_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpermilps {{.*#+}} ymm0 = ymm0[2,3,1,0,6,7,5,4] sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %res = shufflevector <8 x i32> %vec, <8 x i32> undef, <8 x i32> <i32 2, i32 3, i32 1, i32 0, i32 6, i32 7, i32 5, i32 4>
-  ret <8 x i32> %res
-}
-define <8 x i32> @test2_masked_8xi32_perm_mask0(<8 x i32> %vec, <8 x i32> %vec2, <8 x i32> %mask) {
-; GENERIC-LABEL: test2_masked_8xi32_perm_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %ymm2, %ymm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufd {{.*#+}} ymm1 {%k1} = ymm0[2,3,1,0,6,7,5,4] sched: [1:1.00]
-; GENERIC-NEXT:    vmovdqa %ymm1, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test2_masked_8xi32_perm_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %ymm2, %ymm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufd {{.*#+}} ymm1 {%k1} = ymm0[2,3,1,0,6,7,5,4] sched: [1:1.00]
-; SKX-NEXT:    vmovdqa %ymm1, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x i32> %vec, <8 x i32> undef, <8 x i32> <i32 2, i32 3, i32 1, i32 0, i32 6, i32 7, i32 5, i32 4>
-  %cmp = icmp eq <8 x i32> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i32> %shuf, <8 x i32> %vec2
-  ret <8 x i32> %res
-}
-
-define <8 x i32> @test2_masked_z_8xi32_perm_mask0(<8 x i32> %vec, <8 x i32> %mask) {
-; GENERIC-LABEL: test2_masked_z_8xi32_perm_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %ymm1, %ymm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufd {{.*#+}} ymm0 {%k1} {z} = ymm0[2,3,1,0,6,7,5,4] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test2_masked_z_8xi32_perm_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %ymm1, %ymm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufd {{.*#+}} ymm0 {%k1} {z} = ymm0[2,3,1,0,6,7,5,4] sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x i32> %vec, <8 x i32> undef, <8 x i32> <i32 2, i32 3, i32 1, i32 0, i32 6, i32 7, i32 5, i32 4>
-  %cmp = icmp eq <8 x i32> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i32> %shuf, <8 x i32> zeroinitializer
-  ret <8 x i32> %res
-}
-define <8 x i32> @test2_masked_8xi32_perm_mask1(<8 x i32> %vec, <8 x i32> %vec2, <8 x i32> %mask) {
-; GENERIC-LABEL: test2_masked_8xi32_perm_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %ymm2, %ymm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufd {{.*#+}} ymm1 {%k1} = ymm0[0,3,3,3,4,7,7,7] sched: [1:1.00]
-; GENERIC-NEXT:    vmovdqa %ymm1, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test2_masked_8xi32_perm_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %ymm2, %ymm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufd {{.*#+}} ymm1 {%k1} = ymm0[0,3,3,3,4,7,7,7] sched: [1:1.00]
-; SKX-NEXT:    vmovdqa %ymm1, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x i32> %vec, <8 x i32> undef, <8 x i32> <i32 0, i32 3, i32 3, i32 3, i32 4, i32 7, i32 7, i32 7>
-  %cmp = icmp eq <8 x i32> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i32> %shuf, <8 x i32> %vec2
-  ret <8 x i32> %res
-}
-
-define <8 x i32> @test2_masked_z_8xi32_perm_mask1(<8 x i32> %vec, <8 x i32> %mask) {
-; GENERIC-LABEL: test2_masked_z_8xi32_perm_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %ymm1, %ymm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufd {{.*#+}} ymm0 {%k1} {z} = ymm0[0,3,3,3,4,7,7,7] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test2_masked_z_8xi32_perm_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %ymm1, %ymm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufd {{.*#+}} ymm0 {%k1} {z} = ymm0[0,3,3,3,4,7,7,7] sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x i32> %vec, <8 x i32> undef, <8 x i32> <i32 0, i32 3, i32 3, i32 3, i32 4, i32 7, i32 7, i32 7>
-  %cmp = icmp eq <8 x i32> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i32> %shuf, <8 x i32> zeroinitializer
-  ret <8 x i32> %res
-}
-define <8 x i32> @test2_masked_8xi32_perm_mask2(<8 x i32> %vec, <8 x i32> %vec2, <8 x i32> %mask) {
-; GENERIC-LABEL: test2_masked_8xi32_perm_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %ymm2, %ymm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufd {{.*#+}} ymm1 {%k1} = ymm0[1,2,0,3,5,6,4,7] sched: [1:1.00]
-; GENERIC-NEXT:    vmovdqa %ymm1, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test2_masked_8xi32_perm_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %ymm2, %ymm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufd {{.*#+}} ymm1 {%k1} = ymm0[1,2,0,3,5,6,4,7] sched: [1:1.00]
-; SKX-NEXT:    vmovdqa %ymm1, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x i32> %vec, <8 x i32> undef, <8 x i32> <i32 1, i32 2, i32 0, i32 3, i32 5, i32 6, i32 4, i32 7>
-  %cmp = icmp eq <8 x i32> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i32> %shuf, <8 x i32> %vec2
-  ret <8 x i32> %res
-}
-
-define <8 x i32> @test2_masked_z_8xi32_perm_mask2(<8 x i32> %vec, <8 x i32> %mask) {
-; GENERIC-LABEL: test2_masked_z_8xi32_perm_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %ymm1, %ymm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufd {{.*#+}} ymm0 {%k1} {z} = ymm0[1,2,0,3,5,6,4,7] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test2_masked_z_8xi32_perm_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %ymm1, %ymm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufd {{.*#+}} ymm0 {%k1} {z} = ymm0[1,2,0,3,5,6,4,7] sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x i32> %vec, <8 x i32> undef, <8 x i32> <i32 1, i32 2, i32 0, i32 3, i32 5, i32 6, i32 4, i32 7>
-  %cmp = icmp eq <8 x i32> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i32> %shuf, <8 x i32> zeroinitializer
-  ret <8 x i32> %res
-}
-define <8 x i32> @test2_8xi32_perm_mask3(<8 x i32> %vec) {
-; GENERIC-LABEL: test2_8xi32_perm_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpermilps {{.*#+}} ymm0 = ymm0[1,3,1,0,5,7,5,4] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test2_8xi32_perm_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpermilps {{.*#+}} ymm0 = ymm0[1,3,1,0,5,7,5,4] sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %res = shufflevector <8 x i32> %vec, <8 x i32> undef, <8 x i32> <i32 1, i32 3, i32 1, i32 0, i32 5, i32 7, i32 5, i32 4>
-  ret <8 x i32> %res
-}
-define <8 x i32> @test2_masked_8xi32_perm_mask3(<8 x i32> %vec, <8 x i32> %vec2, <8 x i32> %mask) {
-; GENERIC-LABEL: test2_masked_8xi32_perm_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %ymm2, %ymm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufd {{.*#+}} ymm1 {%k1} = ymm0[1,3,1,0,5,7,5,4] sched: [1:1.00]
-; GENERIC-NEXT:    vmovdqa %ymm1, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test2_masked_8xi32_perm_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %ymm2, %ymm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufd {{.*#+}} ymm1 {%k1} = ymm0[1,3,1,0,5,7,5,4] sched: [1:1.00]
-; SKX-NEXT:    vmovdqa %ymm1, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x i32> %vec, <8 x i32> undef, <8 x i32> <i32 1, i32 3, i32 1, i32 0, i32 5, i32 7, i32 5, i32 4>
-  %cmp = icmp eq <8 x i32> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i32> %shuf, <8 x i32> %vec2
-  ret <8 x i32> %res
-}
-
-define <8 x i32> @test2_masked_z_8xi32_perm_mask3(<8 x i32> %vec, <8 x i32> %mask) {
-; GENERIC-LABEL: test2_masked_z_8xi32_perm_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %ymm1, %ymm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufd {{.*#+}} ymm0 {%k1} {z} = ymm0[1,3,1,0,5,7,5,4] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test2_masked_z_8xi32_perm_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %ymm1, %ymm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufd {{.*#+}} ymm0 {%k1} {z} = ymm0[1,3,1,0,5,7,5,4] sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x i32> %vec, <8 x i32> undef, <8 x i32> <i32 1, i32 3, i32 1, i32 0, i32 5, i32 7, i32 5, i32 4>
-  %cmp = icmp eq <8 x i32> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i32> %shuf, <8 x i32> zeroinitializer
-  ret <8 x i32> %res
-}
-define <8 x i32> @test2_8xi32_perm_mem_mask0(<8 x i32>* %vp) {
-; GENERIC-LABEL: test2_8xi32_perm_mem_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpermilps {{.*#+}} ymm0 = mem[1,0,2,0,5,4,6,4] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test2_8xi32_perm_mem_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpermilps {{.*#+}} ymm0 = mem[1,0,2,0,5,4,6,4] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <8 x i32>, <8 x i32>* %vp
-  %res = shufflevector <8 x i32> %vec, <8 x i32> undef, <8 x i32> <i32 1, i32 0, i32 2, i32 0, i32 5, i32 4, i32 6, i32 4>
-  ret <8 x i32> %res
-}
-define <8 x i32> @test2_masked_8xi32_perm_mem_mask0(<8 x i32>* %vp, <8 x i32> %vec2, <8 x i32> %mask) {
-; GENERIC-LABEL: test2_masked_8xi32_perm_mem_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %ymm1, %ymm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufd {{.*#+}} ymm0 {%k1} = mem[1,0,2,0,5,4,6,4] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test2_masked_8xi32_perm_mem_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %ymm1, %ymm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufd {{.*#+}} ymm0 {%k1} = mem[1,0,2,0,5,4,6,4] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <8 x i32>, <8 x i32>* %vp
-  %shuf = shufflevector <8 x i32> %vec, <8 x i32> undef, <8 x i32> <i32 1, i32 0, i32 2, i32 0, i32 5, i32 4, i32 6, i32 4>
-  %cmp = icmp eq <8 x i32> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i32> %shuf, <8 x i32> %vec2
-  ret <8 x i32> %res
-}
-
-define <8 x i32> @test2_masked_z_8xi32_perm_mem_mask0(<8 x i32>* %vp, <8 x i32> %mask) {
-; GENERIC-LABEL: test2_masked_z_8xi32_perm_mem_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %ymm0, %ymm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufd {{.*#+}} ymm0 {%k1} {z} = mem[1,0,2,0,5,4,6,4] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test2_masked_z_8xi32_perm_mem_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %ymm0, %ymm0, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufd {{.*#+}} ymm0 {%k1} {z} = mem[1,0,2,0,5,4,6,4] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <8 x i32>, <8 x i32>* %vp
-  %shuf = shufflevector <8 x i32> %vec, <8 x i32> undef, <8 x i32> <i32 1, i32 0, i32 2, i32 0, i32 5, i32 4, i32 6, i32 4>
-  %cmp = icmp eq <8 x i32> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i32> %shuf, <8 x i32> zeroinitializer
-  ret <8 x i32> %res
-}
-
-define <8 x i32> @test2_masked_8xi32_perm_mem_mask1(<8 x i32>* %vp, <8 x i32> %vec2, <8 x i32> %mask) {
-; GENERIC-LABEL: test2_masked_8xi32_perm_mem_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %ymm1, %ymm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufd {{.*#+}} ymm0 {%k1} = mem[0,3,2,0,4,7,6,4] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test2_masked_8xi32_perm_mem_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %ymm1, %ymm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufd {{.*#+}} ymm0 {%k1} = mem[0,3,2,0,4,7,6,4] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <8 x i32>, <8 x i32>* %vp
-  %shuf = shufflevector <8 x i32> %vec, <8 x i32> undef, <8 x i32> <i32 0, i32 3, i32 2, i32 0, i32 4, i32 7, i32 6, i32 4>
-  %cmp = icmp eq <8 x i32> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i32> %shuf, <8 x i32> %vec2
-  ret <8 x i32> %res
-}
-
-define <8 x i32> @test2_masked_z_8xi32_perm_mem_mask1(<8 x i32>* %vp, <8 x i32> %mask) {
-; GENERIC-LABEL: test2_masked_z_8xi32_perm_mem_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %ymm0, %ymm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufd {{.*#+}} ymm0 {%k1} {z} = mem[0,3,2,0,4,7,6,4] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test2_masked_z_8xi32_perm_mem_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %ymm0, %ymm0, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufd {{.*#+}} ymm0 {%k1} {z} = mem[0,3,2,0,4,7,6,4] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <8 x i32>, <8 x i32>* %vp
-  %shuf = shufflevector <8 x i32> %vec, <8 x i32> undef, <8 x i32> <i32 0, i32 3, i32 2, i32 0, i32 4, i32 7, i32 6, i32 4>
-  %cmp = icmp eq <8 x i32> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i32> %shuf, <8 x i32> zeroinitializer
-  ret <8 x i32> %res
-}
-
-define <8 x i32> @test2_masked_8xi32_perm_mem_mask2(<8 x i32>* %vp, <8 x i32> %vec2, <8 x i32> %mask) {
-; GENERIC-LABEL: test2_masked_8xi32_perm_mem_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %ymm1, %ymm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufd {{.*#+}} ymm0 {%k1} = mem[3,2,3,1,7,6,7,5] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test2_masked_8xi32_perm_mem_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %ymm1, %ymm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufd {{.*#+}} ymm0 {%k1} = mem[3,2,3,1,7,6,7,5] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <8 x i32>, <8 x i32>* %vp
-  %shuf = shufflevector <8 x i32> %vec, <8 x i32> undef, <8 x i32> <i32 3, i32 2, i32 3, i32 1, i32 7, i32 6, i32 7, i32 5>
-  %cmp = icmp eq <8 x i32> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i32> %shuf, <8 x i32> %vec2
-  ret <8 x i32> %res
-}
-
-define <8 x i32> @test2_masked_z_8xi32_perm_mem_mask2(<8 x i32>* %vp, <8 x i32> %mask) {
-; GENERIC-LABEL: test2_masked_z_8xi32_perm_mem_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %ymm0, %ymm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufd {{.*#+}} ymm0 {%k1} {z} = mem[3,2,3,1,7,6,7,5] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test2_masked_z_8xi32_perm_mem_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %ymm0, %ymm0, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufd {{.*#+}} ymm0 {%k1} {z} = mem[3,2,3,1,7,6,7,5] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <8 x i32>, <8 x i32>* %vp
-  %shuf = shufflevector <8 x i32> %vec, <8 x i32> undef, <8 x i32> <i32 3, i32 2, i32 3, i32 1, i32 7, i32 6, i32 7, i32 5>
-  %cmp = icmp eq <8 x i32> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i32> %shuf, <8 x i32> zeroinitializer
-  ret <8 x i32> %res
-}
-
-define <8 x i32> @test2_8xi32_perm_mem_mask3(<8 x i32>* %vp) {
-; GENERIC-LABEL: test2_8xi32_perm_mem_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpermilps {{.*#+}} ymm0 = mem[3,2,0,0,7,6,4,4] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test2_8xi32_perm_mem_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpermilps {{.*#+}} ymm0 = mem[3,2,0,0,7,6,4,4] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <8 x i32>, <8 x i32>* %vp
-  %res = shufflevector <8 x i32> %vec, <8 x i32> undef, <8 x i32> <i32 3, i32 2, i32 0, i32 0, i32 7, i32 6, i32 4, i32 4>
-  ret <8 x i32> %res
-}
-define <8 x i32> @test2_masked_8xi32_perm_mem_mask3(<8 x i32>* %vp, <8 x i32> %vec2, <8 x i32> %mask) {
-; GENERIC-LABEL: test2_masked_8xi32_perm_mem_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %ymm1, %ymm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufd {{.*#+}} ymm0 {%k1} = mem[3,2,0,0,7,6,4,4] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test2_masked_8xi32_perm_mem_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %ymm1, %ymm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufd {{.*#+}} ymm0 {%k1} = mem[3,2,0,0,7,6,4,4] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <8 x i32>, <8 x i32>* %vp
-  %shuf = shufflevector <8 x i32> %vec, <8 x i32> undef, <8 x i32> <i32 3, i32 2, i32 0, i32 0, i32 7, i32 6, i32 4, i32 4>
-  %cmp = icmp eq <8 x i32> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i32> %shuf, <8 x i32> %vec2
-  ret <8 x i32> %res
-}
-
-define <8 x i32> @test2_masked_z_8xi32_perm_mem_mask3(<8 x i32>* %vp, <8 x i32> %mask) {
-; GENERIC-LABEL: test2_masked_z_8xi32_perm_mem_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %ymm0, %ymm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufd {{.*#+}} ymm0 {%k1} {z} = mem[3,2,0,0,7,6,4,4] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test2_masked_z_8xi32_perm_mem_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %ymm0, %ymm0, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufd {{.*#+}} ymm0 {%k1} {z} = mem[3,2,0,0,7,6,4,4] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <8 x i32>, <8 x i32>* %vp
-  %shuf = shufflevector <8 x i32> %vec, <8 x i32> undef, <8 x i32> <i32 3, i32 2, i32 0, i32 0, i32 7, i32 6, i32 4, i32 4>
-  %cmp = icmp eq <8 x i32> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i32> %shuf, <8 x i32> zeroinitializer
-  ret <8 x i32> %res
-}
-
-define <16 x i32> @test2_16xi32_perm_mask0(<16 x i32> %vec) {
-; GENERIC-LABEL: test2_16xi32_perm_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpermilps {{.*#+}} zmm0 = zmm0[3,1,3,0,7,5,7,4,11,9,11,8,15,13,15,12] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test2_16xi32_perm_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpermilps {{.*#+}} zmm0 = zmm0[3,1,3,0,7,5,7,4,11,9,11,8,15,13,15,12] sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %res = shufflevector <16 x i32> %vec, <16 x i32> undef, <16 x i32> <i32 3, i32 1, i32 3, i32 0, i32 7, i32 5, i32 7, i32 4, i32 11, i32 9, i32 11, i32 8, i32 15, i32 13, i32 15, i32 12>
-  ret <16 x i32> %res
-}
-define <16 x i32> @test2_masked_16xi32_perm_mask0(<16 x i32> %vec, <16 x i32> %vec2, <16 x i32> %mask) {
-; GENERIC-LABEL: test2_masked_16xi32_perm_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %zmm2, %zmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufd {{.*#+}} zmm1 {%k1} = zmm0[3,1,3,0,7,5,7,4,11,9,11,8,15,13,15,12] sched: [1:1.00]
-; GENERIC-NEXT:    vmovdqa64 %zmm1, %zmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test2_masked_16xi32_perm_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %zmm2, %zmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufd {{.*#+}} zmm1 {%k1} = zmm0[3,1,3,0,7,5,7,4,11,9,11,8,15,13,15,12] sched: [1:1.00]
-; SKX-NEXT:    vmovdqa64 %zmm1, %zmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <16 x i32> %vec, <16 x i32> undef, <16 x i32> <i32 3, i32 1, i32 3, i32 0, i32 7, i32 5, i32 7, i32 4, i32 11, i32 9, i32 11, i32 8, i32 15, i32 13, i32 15, i32 12>
-  %cmp = icmp eq <16 x i32> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x i32> %shuf, <16 x i32> %vec2
-  ret <16 x i32> %res
-}
-
-define <16 x i32> @test2_masked_z_16xi32_perm_mask0(<16 x i32> %vec, <16 x i32> %mask) {
-; GENERIC-LABEL: test2_masked_z_16xi32_perm_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufd {{.*#+}} zmm0 {%k1} {z} = zmm0[3,1,3,0,7,5,7,4,11,9,11,8,15,13,15,12] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test2_masked_z_16xi32_perm_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufd {{.*#+}} zmm0 {%k1} {z} = zmm0[3,1,3,0,7,5,7,4,11,9,11,8,15,13,15,12] sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <16 x i32> %vec, <16 x i32> undef, <16 x i32> <i32 3, i32 1, i32 3, i32 0, i32 7, i32 5, i32 7, i32 4, i32 11, i32 9, i32 11, i32 8, i32 15, i32 13, i32 15, i32 12>
-  %cmp = icmp eq <16 x i32> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x i32> %shuf, <16 x i32> zeroinitializer
-  ret <16 x i32> %res
-}
-define <16 x i32> @test2_masked_16xi32_perm_mask1(<16 x i32> %vec, <16 x i32> %vec2, <16 x i32> %mask) {
-; GENERIC-LABEL: test2_masked_16xi32_perm_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %zmm2, %zmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufd {{.*#+}} zmm1 {%k1} = zmm0[2,0,3,0,6,4,7,4,10,8,11,8,14,12,15,12] sched: [1:1.00]
-; GENERIC-NEXT:    vmovdqa64 %zmm1, %zmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test2_masked_16xi32_perm_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %zmm2, %zmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufd {{.*#+}} zmm1 {%k1} = zmm0[2,0,3,0,6,4,7,4,10,8,11,8,14,12,15,12] sched: [1:1.00]
-; SKX-NEXT:    vmovdqa64 %zmm1, %zmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <16 x i32> %vec, <16 x i32> undef, <16 x i32> <i32 2, i32 0, i32 3, i32 0, i32 6, i32 4, i32 7, i32 4, i32 10, i32 8, i32 11, i32 8, i32 14, i32 12, i32 15, i32 12>
-  %cmp = icmp eq <16 x i32> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x i32> %shuf, <16 x i32> %vec2
-  ret <16 x i32> %res
-}
-
-define <16 x i32> @test2_masked_z_16xi32_perm_mask1(<16 x i32> %vec, <16 x i32> %mask) {
-; GENERIC-LABEL: test2_masked_z_16xi32_perm_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufd {{.*#+}} zmm0 {%k1} {z} = zmm0[2,0,3,0,6,4,7,4,10,8,11,8,14,12,15,12] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test2_masked_z_16xi32_perm_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufd {{.*#+}} zmm0 {%k1} {z} = zmm0[2,0,3,0,6,4,7,4,10,8,11,8,14,12,15,12] sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <16 x i32> %vec, <16 x i32> undef, <16 x i32> <i32 2, i32 0, i32 3, i32 0, i32 6, i32 4, i32 7, i32 4, i32 10, i32 8, i32 11, i32 8, i32 14, i32 12, i32 15, i32 12>
-  %cmp = icmp eq <16 x i32> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x i32> %shuf, <16 x i32> zeroinitializer
-  ret <16 x i32> %res
-}
-define <16 x i32> @test2_masked_16xi32_perm_mask2(<16 x i32> %vec, <16 x i32> %vec2, <16 x i32> %mask) {
-; GENERIC-LABEL: test2_masked_16xi32_perm_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %zmm2, %zmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufd {{.*#+}} zmm1 {%k1} = zmm0[1,3,3,0,5,7,7,4,9,11,11,8,13,15,15,12] sched: [1:1.00]
-; GENERIC-NEXT:    vmovdqa64 %zmm1, %zmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test2_masked_16xi32_perm_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %zmm2, %zmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufd {{.*#+}} zmm1 {%k1} = zmm0[1,3,3,0,5,7,7,4,9,11,11,8,13,15,15,12] sched: [1:1.00]
-; SKX-NEXT:    vmovdqa64 %zmm1, %zmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <16 x i32> %vec, <16 x i32> undef, <16 x i32> <i32 1, i32 3, i32 3, i32 0, i32 5, i32 7, i32 7, i32 4, i32 9, i32 11, i32 11, i32 8, i32 13, i32 15, i32 15, i32 12>
-  %cmp = icmp eq <16 x i32> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x i32> %shuf, <16 x i32> %vec2
-  ret <16 x i32> %res
-}
-
-define <16 x i32> @test2_masked_z_16xi32_perm_mask2(<16 x i32> %vec, <16 x i32> %mask) {
-; GENERIC-LABEL: test2_masked_z_16xi32_perm_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufd {{.*#+}} zmm0 {%k1} {z} = zmm0[1,3,3,0,5,7,7,4,9,11,11,8,13,15,15,12] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test2_masked_z_16xi32_perm_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufd {{.*#+}} zmm0 {%k1} {z} = zmm0[1,3,3,0,5,7,7,4,9,11,11,8,13,15,15,12] sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <16 x i32> %vec, <16 x i32> undef, <16 x i32> <i32 1, i32 3, i32 3, i32 0, i32 5, i32 7, i32 7, i32 4, i32 9, i32 11, i32 11, i32 8, i32 13, i32 15, i32 15, i32 12>
-  %cmp = icmp eq <16 x i32> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x i32> %shuf, <16 x i32> zeroinitializer
-  ret <16 x i32> %res
-}
-define <16 x i32> @test2_16xi32_perm_mask3(<16 x i32> %vec) {
-; GENERIC-LABEL: test2_16xi32_perm_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpermilps {{.*#+}} zmm0 = zmm0[3,2,0,3,7,6,4,7,11,10,8,11,15,14,12,15] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test2_16xi32_perm_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpermilps {{.*#+}} zmm0 = zmm0[3,2,0,3,7,6,4,7,11,10,8,11,15,14,12,15] sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %res = shufflevector <16 x i32> %vec, <16 x i32> undef, <16 x i32> <i32 3, i32 2, i32 0, i32 3, i32 7, i32 6, i32 4, i32 7, i32 11, i32 10, i32 8, i32 11, i32 15, i32 14, i32 12, i32 15>
-  ret <16 x i32> %res
-}
-define <16 x i32> @test2_masked_16xi32_perm_mask3(<16 x i32> %vec, <16 x i32> %vec2, <16 x i32> %mask) {
-; GENERIC-LABEL: test2_masked_16xi32_perm_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %zmm2, %zmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufd {{.*#+}} zmm1 {%k1} = zmm0[3,2,0,3,7,6,4,7,11,10,8,11,15,14,12,15] sched: [1:1.00]
-; GENERIC-NEXT:    vmovdqa64 %zmm1, %zmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test2_masked_16xi32_perm_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %zmm2, %zmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufd {{.*#+}} zmm1 {%k1} = zmm0[3,2,0,3,7,6,4,7,11,10,8,11,15,14,12,15] sched: [1:1.00]
-; SKX-NEXT:    vmovdqa64 %zmm1, %zmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <16 x i32> %vec, <16 x i32> undef, <16 x i32> <i32 3, i32 2, i32 0, i32 3, i32 7, i32 6, i32 4, i32 7, i32 11, i32 10, i32 8, i32 11, i32 15, i32 14, i32 12, i32 15>
-  %cmp = icmp eq <16 x i32> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x i32> %shuf, <16 x i32> %vec2
-  ret <16 x i32> %res
-}
-
-define <16 x i32> @test2_masked_z_16xi32_perm_mask3(<16 x i32> %vec, <16 x i32> %mask) {
-; GENERIC-LABEL: test2_masked_z_16xi32_perm_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufd {{.*#+}} zmm0 {%k1} {z} = zmm0[3,2,0,3,7,6,4,7,11,10,8,11,15,14,12,15] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test2_masked_z_16xi32_perm_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufd {{.*#+}} zmm0 {%k1} {z} = zmm0[3,2,0,3,7,6,4,7,11,10,8,11,15,14,12,15] sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <16 x i32> %vec, <16 x i32> undef, <16 x i32> <i32 3, i32 2, i32 0, i32 3, i32 7, i32 6, i32 4, i32 7, i32 11, i32 10, i32 8, i32 11, i32 15, i32 14, i32 12, i32 15>
-  %cmp = icmp eq <16 x i32> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x i32> %shuf, <16 x i32> zeroinitializer
-  ret <16 x i32> %res
-}
-define <16 x i32> @test2_16xi32_perm_mem_mask0(<16 x i32>* %vp) {
-; GENERIC-LABEL: test2_16xi32_perm_mem_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpermilps {{.*#+}} zmm0 = mem[1,0,1,3,5,4,5,7,9,8,9,11,13,12,13,15] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test2_16xi32_perm_mem_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpermilps {{.*#+}} zmm0 = mem[1,0,1,3,5,4,5,7,9,8,9,11,13,12,13,15] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <16 x i32>, <16 x i32>* %vp
-  %res = shufflevector <16 x i32> %vec, <16 x i32> undef, <16 x i32> <i32 1, i32 0, i32 1, i32 3, i32 5, i32 4, i32 5, i32 7, i32 9, i32 8, i32 9, i32 11, i32 13, i32 12, i32 13, i32 15>
-  ret <16 x i32> %res
-}
-define <16 x i32> @test2_masked_16xi32_perm_mem_mask0(<16 x i32>* %vp, <16 x i32> %vec2, <16 x i32> %mask) {
-; GENERIC-LABEL: test2_masked_16xi32_perm_mem_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufd {{.*#+}} zmm0 {%k1} = mem[1,0,1,3,5,4,5,7,9,8,9,11,13,12,13,15] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test2_masked_16xi32_perm_mem_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufd {{.*#+}} zmm0 {%k1} = mem[1,0,1,3,5,4,5,7,9,8,9,11,13,12,13,15] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <16 x i32>, <16 x i32>* %vp
-  %shuf = shufflevector <16 x i32> %vec, <16 x i32> undef, <16 x i32> <i32 1, i32 0, i32 1, i32 3, i32 5, i32 4, i32 5, i32 7, i32 9, i32 8, i32 9, i32 11, i32 13, i32 12, i32 13, i32 15>
-  %cmp = icmp eq <16 x i32> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x i32> %shuf, <16 x i32> %vec2
-  ret <16 x i32> %res
-}
-
-define <16 x i32> @test2_masked_z_16xi32_perm_mem_mask0(<16 x i32>* %vp, <16 x i32> %mask) {
-; GENERIC-LABEL: test2_masked_z_16xi32_perm_mem_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %zmm0, %zmm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufd {{.*#+}} zmm0 {%k1} {z} = mem[1,0,1,3,5,4,5,7,9,8,9,11,13,12,13,15] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test2_masked_z_16xi32_perm_mem_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %zmm0, %zmm0, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufd {{.*#+}} zmm0 {%k1} {z} = mem[1,0,1,3,5,4,5,7,9,8,9,11,13,12,13,15] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <16 x i32>, <16 x i32>* %vp
-  %shuf = shufflevector <16 x i32> %vec, <16 x i32> undef, <16 x i32> <i32 1, i32 0, i32 1, i32 3, i32 5, i32 4, i32 5, i32 7, i32 9, i32 8, i32 9, i32 11, i32 13, i32 12, i32 13, i32 15>
-  %cmp = icmp eq <16 x i32> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x i32> %shuf, <16 x i32> zeroinitializer
-  ret <16 x i32> %res
-}
-
-define <16 x i32> @test2_masked_16xi32_perm_mem_mask1(<16 x i32>* %vp, <16 x i32> %vec2, <16 x i32> %mask) {
-; GENERIC-LABEL: test2_masked_16xi32_perm_mem_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufd {{.*#+}} zmm0 {%k1} = mem[1,0,0,2,5,4,4,6,9,8,8,10,13,12,12,14] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test2_masked_16xi32_perm_mem_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufd {{.*#+}} zmm0 {%k1} = mem[1,0,0,2,5,4,4,6,9,8,8,10,13,12,12,14] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <16 x i32>, <16 x i32>* %vp
-  %shuf = shufflevector <16 x i32> %vec, <16 x i32> undef, <16 x i32> <i32 1, i32 0, i32 0, i32 2, i32 5, i32 4, i32 4, i32 6, i32 9, i32 8, i32 8, i32 10, i32 13, i32 12, i32 12, i32 14>
-  %cmp = icmp eq <16 x i32> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x i32> %shuf, <16 x i32> %vec2
-  ret <16 x i32> %res
-}
-
-define <16 x i32> @test2_masked_z_16xi32_perm_mem_mask1(<16 x i32>* %vp, <16 x i32> %mask) {
-; GENERIC-LABEL: test2_masked_z_16xi32_perm_mem_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %zmm0, %zmm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufd {{.*#+}} zmm0 {%k1} {z} = mem[1,0,0,2,5,4,4,6,9,8,8,10,13,12,12,14] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test2_masked_z_16xi32_perm_mem_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %zmm0, %zmm0, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufd {{.*#+}} zmm0 {%k1} {z} = mem[1,0,0,2,5,4,4,6,9,8,8,10,13,12,12,14] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <16 x i32>, <16 x i32>* %vp
-  %shuf = shufflevector <16 x i32> %vec, <16 x i32> undef, <16 x i32> <i32 1, i32 0, i32 0, i32 2, i32 5, i32 4, i32 4, i32 6, i32 9, i32 8, i32 8, i32 10, i32 13, i32 12, i32 12, i32 14>
-  %cmp = icmp eq <16 x i32> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x i32> %shuf, <16 x i32> zeroinitializer
-  ret <16 x i32> %res
-}
-
-define <16 x i32> @test2_masked_16xi32_perm_mem_mask2(<16 x i32>* %vp, <16 x i32> %vec2, <16 x i32> %mask) {
-; GENERIC-LABEL: test2_masked_16xi32_perm_mem_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufd {{.*#+}} zmm0 {%k1} = mem[2,0,1,2,6,4,5,6,10,8,9,10,14,12,13,14] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test2_masked_16xi32_perm_mem_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufd {{.*#+}} zmm0 {%k1} = mem[2,0,1,2,6,4,5,6,10,8,9,10,14,12,13,14] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <16 x i32>, <16 x i32>* %vp
-  %shuf = shufflevector <16 x i32> %vec, <16 x i32> undef, <16 x i32> <i32 2, i32 0, i32 1, i32 2, i32 6, i32 4, i32 5, i32 6, i32 10, i32 8, i32 9, i32 10, i32 14, i32 12, i32 13, i32 14>
-  %cmp = icmp eq <16 x i32> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x i32> %shuf, <16 x i32> %vec2
-  ret <16 x i32> %res
-}
-
-define <16 x i32> @test2_masked_z_16xi32_perm_mem_mask2(<16 x i32>* %vp, <16 x i32> %mask) {
-; GENERIC-LABEL: test2_masked_z_16xi32_perm_mem_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %zmm0, %zmm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufd {{.*#+}} zmm0 {%k1} {z} = mem[2,0,1,2,6,4,5,6,10,8,9,10,14,12,13,14] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test2_masked_z_16xi32_perm_mem_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %zmm0, %zmm0, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufd {{.*#+}} zmm0 {%k1} {z} = mem[2,0,1,2,6,4,5,6,10,8,9,10,14,12,13,14] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <16 x i32>, <16 x i32>* %vp
-  %shuf = shufflevector <16 x i32> %vec, <16 x i32> undef, <16 x i32> <i32 2, i32 0, i32 1, i32 2, i32 6, i32 4, i32 5, i32 6, i32 10, i32 8, i32 9, i32 10, i32 14, i32 12, i32 13, i32 14>
-  %cmp = icmp eq <16 x i32> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x i32> %shuf, <16 x i32> zeroinitializer
-  ret <16 x i32> %res
-}
-
-define <16 x i32> @test2_16xi32_perm_mem_mask3(<16 x i32>* %vp) {
-; GENERIC-LABEL: test2_16xi32_perm_mem_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vpermilps {{.*#+}} zmm0 = mem[3,1,1,1,7,5,5,5,11,9,9,9,15,13,13,13] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test2_16xi32_perm_mem_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpermilps {{.*#+}} zmm0 = mem[3,1,1,1,7,5,5,5,11,9,9,9,15,13,13,13] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <16 x i32>, <16 x i32>* %vp
-  %res = shufflevector <16 x i32> %vec, <16 x i32> undef, <16 x i32> <i32 3, i32 1, i32 1, i32 1, i32 7, i32 5, i32 5, i32 5, i32 11, i32 9, i32 9, i32 9, i32 15, i32 13, i32 13, i32 13>
-  ret <16 x i32> %res
-}
-define <16 x i32> @test2_masked_16xi32_perm_mem_mask3(<16 x i32>* %vp, <16 x i32> %vec2, <16 x i32> %mask) {
-; GENERIC-LABEL: test2_masked_16xi32_perm_mem_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufd {{.*#+}} zmm0 {%k1} = mem[3,1,1,1,7,5,5,5,11,9,9,9,15,13,13,13] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test2_masked_16xi32_perm_mem_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufd {{.*#+}} zmm0 {%k1} = mem[3,1,1,1,7,5,5,5,11,9,9,9,15,13,13,13] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <16 x i32>, <16 x i32>* %vp
-  %shuf = shufflevector <16 x i32> %vec, <16 x i32> undef, <16 x i32> <i32 3, i32 1, i32 1, i32 1, i32 7, i32 5, i32 5, i32 5, i32 11, i32 9, i32 9, i32 9, i32 15, i32 13, i32 13, i32 13>
-  %cmp = icmp eq <16 x i32> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x i32> %shuf, <16 x i32> %vec2
-  ret <16 x i32> %res
-}
-
-define <16 x i32> @test2_masked_z_16xi32_perm_mem_mask3(<16 x i32>* %vp, <16 x i32> %mask) {
-; GENERIC-LABEL: test2_masked_z_16xi32_perm_mem_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %zmm0, %zmm0, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vpshufd {{.*#+}} zmm0 {%k1} {z} = mem[3,1,1,1,7,5,5,5,11,9,9,9,15,13,13,13] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test2_masked_z_16xi32_perm_mem_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %zmm0, %zmm0, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vpshufd {{.*#+}} zmm0 {%k1} {z} = mem[3,1,1,1,7,5,5,5,11,9,9,9,15,13,13,13] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec = load <16 x i32>, <16 x i32>* %vp
-  %shuf = shufflevector <16 x i32> %vec, <16 x i32> undef, <16 x i32> <i32 3, i32 1, i32 1, i32 1, i32 7, i32 5, i32 5, i32 5, i32 11, i32 9, i32 9, i32 9, i32 15, i32 13, i32 13, i32 13>
-  %cmp = icmp eq <16 x i32> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x i32> %shuf, <16 x i32> zeroinitializer
-  ret <16 x i32> %res
-}
-
-define <8 x float> @test2_8xfloat_shuff_mask0(<8 x float> %vec1, <8 x float> %vec2) {
-; GENERIC-LABEL: test2_8xfloat_shuff_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm0[2,3],ymm1[0,1] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test2_8xfloat_shuff_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm0[2,3],ymm1[0,1] sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %res = shufflevector <8 x float> %vec1, <8 x float> %vec2, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
-  ret <8 x float> %res
-}
-define <8 x float> @test2_8xfloat_masked_shuff_mask0(<8 x float> %vec1, <8 x float> %vec2, <8 x float> %vec3, <8 x i32> %mask) {
-; GENERIC-LABEL: test2_8xfloat_masked_shuff_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %ymm3, %ymm3, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshuff32x4 {{.*#+}} ymm2 {%k1} = ymm0[4,5,6,7],ymm1[0,1,2,3] sched: [1:1.00]
-; GENERIC-NEXT:    vmovaps %ymm2, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test2_8xfloat_masked_shuff_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %ymm3, %ymm3, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshuff32x4 {{.*#+}} ymm2 {%k1} = ymm0[4,5,6,7],ymm1[0,1,2,3] sched: [3:1.00]
-; SKX-NEXT:    vmovaps %ymm2, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x float> %vec1, <8 x float> %vec2, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
-  %cmp = icmp eq <8 x i32> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x float> %shuf, <8 x float> %vec3
-  ret <8 x float> %res
-}
-
-define <8 x float> @test2_8xfloat_zero_masked_shuff_mask0(<8 x float> %vec1, <8 x float> %vec2, <8 x i32> %mask) {
-; GENERIC-LABEL: test2_8xfloat_zero_masked_shuff_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %ymm2, %ymm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshuff32x4 {{.*#+}} ymm0 {%k1} {z} = ymm0[4,5,6,7],ymm1[0,1,2,3] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test2_8xfloat_zero_masked_shuff_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %ymm2, %ymm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshuff32x4 {{.*#+}} ymm0 {%k1} {z} = ymm0[4,5,6,7],ymm1[0,1,2,3] sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x float> %vec1, <8 x float> %vec2, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
-  %cmp = icmp eq <8 x i32> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x float> %shuf, <8 x float> zeroinitializer
-  ret <8 x float> %res
-}
-define <8 x float> @test2_8xfloat_masked_shuff_mask1(<8 x float> %vec1, <8 x float> %vec2, <8 x float> %vec3, <8 x i32> %mask) {
-; GENERIC-LABEL: test2_8xfloat_masked_shuff_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %ymm3, %ymm3, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshuff32x4 {{.*#+}} ymm2 {%k1} = ymm0[4,5,6,7],ymm1[0,1,2,3] sched: [1:1.00]
-; GENERIC-NEXT:    vmovaps %ymm2, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test2_8xfloat_masked_shuff_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %ymm3, %ymm3, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshuff32x4 {{.*#+}} ymm2 {%k1} = ymm0[4,5,6,7],ymm1[0,1,2,3] sched: [3:1.00]
-; SKX-NEXT:    vmovaps %ymm2, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x float> %vec1, <8 x float> %vec2, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
-  %cmp = icmp eq <8 x i32> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x float> %shuf, <8 x float> %vec3
-  ret <8 x float> %res
-}
-
-define <8 x float> @test2_8xfloat_zero_masked_shuff_mask1(<8 x float> %vec1, <8 x float> %vec2, <8 x i32> %mask) {
-; GENERIC-LABEL: test2_8xfloat_zero_masked_shuff_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %ymm2, %ymm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshuff32x4 {{.*#+}} ymm0 {%k1} {z} = ymm0[4,5,6,7],ymm1[0,1,2,3] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test2_8xfloat_zero_masked_shuff_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %ymm2, %ymm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshuff32x4 {{.*#+}} ymm0 {%k1} {z} = ymm0[4,5,6,7],ymm1[0,1,2,3] sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x float> %vec1, <8 x float> %vec2, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
-  %cmp = icmp eq <8 x i32> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x float> %shuf, <8 x float> zeroinitializer
-  ret <8 x float> %res
-}
-define <8 x float> @test2_8xfloat_masked_shuff_mask2(<8 x float> %vec1, <8 x float> %vec2, <8 x float> %vec3, <8 x i32> %mask) {
-; GENERIC-LABEL: test2_8xfloat_masked_shuff_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %ymm3, %ymm3, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshuff32x4 {{.*#+}} ymm2 {%k1} = ymm0[4,5,6,7],ymm1[4,5,6,7] sched: [1:1.00]
-; GENERIC-NEXT:    vmovaps %ymm2, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test2_8xfloat_masked_shuff_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %ymm3, %ymm3, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshuff32x4 {{.*#+}} ymm2 {%k1} = ymm0[4,5,6,7],ymm1[4,5,6,7] sched: [3:1.00]
-; SKX-NEXT:    vmovaps %ymm2, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x float> %vec1, <8 x float> %vec2, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 12, i32 13, i32 14, i32 15>
-  %cmp = icmp eq <8 x i32> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x float> %shuf, <8 x float> %vec3
-  ret <8 x float> %res
-}
-
-define <8 x float> @test2_8xfloat_zero_masked_shuff_mask2(<8 x float> %vec1, <8 x float> %vec2, <8 x i32> %mask) {
-; GENERIC-LABEL: test2_8xfloat_zero_masked_shuff_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %ymm2, %ymm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshuff32x4 {{.*#+}} ymm0 {%k1} {z} = ymm0[4,5,6,7],ymm1[4,5,6,7] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test2_8xfloat_zero_masked_shuff_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %ymm2, %ymm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshuff32x4 {{.*#+}} ymm0 {%k1} {z} = ymm0[4,5,6,7],ymm1[4,5,6,7] sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x float> %vec1, <8 x float> %vec2, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 12, i32 13, i32 14, i32 15>
-  %cmp = icmp eq <8 x i32> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x float> %shuf, <8 x float> zeroinitializer
-  ret <8 x float> %res
-}
-define <8 x float> @test2_8xfloat_shuff_mask3(<8 x float> %vec1, <8 x float> %vec2) {
-; GENERIC-LABEL: test2_8xfloat_shuff_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm0[2,3],ymm1[0,1] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test2_8xfloat_shuff_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm0[2,3],ymm1[0,1] sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %res = shufflevector <8 x float> %vec1, <8 x float> %vec2, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
-  ret <8 x float> %res
-}
-define <8 x float> @test2_8xfloat_masked_shuff_mask3(<8 x float> %vec1, <8 x float> %vec2, <8 x float> %vec3, <8 x i32> %mask) {
-; GENERIC-LABEL: test2_8xfloat_masked_shuff_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %ymm3, %ymm3, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshuff32x4 {{.*#+}} ymm2 {%k1} = ymm0[4,5,6,7],ymm1[0,1,2,3] sched: [1:1.00]
-; GENERIC-NEXT:    vmovaps %ymm2, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test2_8xfloat_masked_shuff_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %ymm3, %ymm3, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshuff32x4 {{.*#+}} ymm2 {%k1} = ymm0[4,5,6,7],ymm1[0,1,2,3] sched: [3:1.00]
-; SKX-NEXT:    vmovaps %ymm2, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x float> %vec1, <8 x float> %vec2, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
-  %cmp = icmp eq <8 x i32> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x float> %shuf, <8 x float> %vec3
-  ret <8 x float> %res
-}
-
-define <8 x float> @test_8xfloat_zero_masked_shuff_mask3(<8 x float> %vec1, <8 x float> %vec2, <8 x i32> %mask) {
-; GENERIC-LABEL: test_8xfloat_zero_masked_shuff_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %ymm2, %ymm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshuff32x4 {{.*#+}} ymm0 {%k1} {z} = ymm0[4,5,6,7],ymm1[0,1,2,3] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xfloat_zero_masked_shuff_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %ymm2, %ymm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshuff32x4 {{.*#+}} ymm0 {%k1} {z} = ymm0[4,5,6,7],ymm1[0,1,2,3] sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x float> %vec1, <8 x float> %vec2, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
-  %cmp = icmp eq <8 x i32> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x float> %shuf, <8 x float> zeroinitializer
-  ret <8 x float> %res
-}
-define <8 x float> @test_8xfloat_shuff_mem_mask0(<8 x float> %vec1, <8 x float>* %vec2p) {
-; GENERIC-LABEL: test_8xfloat_shuff_mem_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm0[2,3],mem[2,3] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xfloat_shuff_mem_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm0[2,3],mem[2,3] sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <8 x float>, <8 x float>* %vec2p
-  %res = shufflevector <8 x float> %vec1, <8 x float> %vec2, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 12, i32 13, i32 14, i32 15>
-  ret <8 x float> %res
-}
-define <8 x float> @test_8xfloat_masked_shuff_mem_mask0(<8 x float> %vec1, <8 x float>* %vec2p, <8 x float> %vec3, <8 x i32> %mask) {
-; GENERIC-LABEL: test_8xfloat_masked_shuff_mem_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %ymm2, %ymm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshuff32x4 {{.*#+}} ymm1 {%k1} = ymm0[4,5,6,7],mem[4,5,6,7] sched: [8:1.00]
-; GENERIC-NEXT:    vmovaps %ymm1, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xfloat_masked_shuff_mem_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %ymm2, %ymm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshuff32x4 {{.*#+}} ymm1 {%k1} = ymm0[4,5,6,7],mem[4,5,6,7] sched: [10:1.00]
-; SKX-NEXT:    vmovaps %ymm1, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <8 x float>, <8 x float>* %vec2p
-  %shuf = shufflevector <8 x float> %vec1, <8 x float> %vec2, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 12, i32 13, i32 14, i32 15>
-  %cmp = icmp eq <8 x i32> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x float> %shuf, <8 x float> %vec3
-  ret <8 x float> %res
-}
-
-define <8 x float> @test_8xfloat_zero_masked_shuff_mem_mask0(<8 x float> %vec1, <8 x float>* %vec2p, <8 x i32> %mask) {
-; GENERIC-LABEL: test_8xfloat_zero_masked_shuff_mem_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %ymm1, %ymm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshuff32x4 {{.*#+}} ymm0 {%k1} {z} = ymm0[4,5,6,7],mem[4,5,6,7] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xfloat_zero_masked_shuff_mem_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %ymm1, %ymm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshuff32x4 {{.*#+}} ymm0 {%k1} {z} = ymm0[4,5,6,7],mem[4,5,6,7] sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <8 x float>, <8 x float>* %vec2p
-  %shuf = shufflevector <8 x float> %vec1, <8 x float> %vec2, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 12, i32 13, i32 14, i32 15>
-  %cmp = icmp eq <8 x i32> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x float> %shuf, <8 x float> zeroinitializer
-  ret <8 x float> %res
-}
-
-define <8 x float> @test_8xfloat_masked_shuff_mem_mask1(<8 x float> %vec1, <8 x float>* %vec2p, <8 x float> %vec3, <8 x i32> %mask) {
-; GENERIC-LABEL: test_8xfloat_masked_shuff_mem_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %ymm2, %ymm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshuff32x4 {{.*#+}} ymm1 {%k1} = ymm0[4,5,6,7],mem[4,5,6,7] sched: [8:1.00]
-; GENERIC-NEXT:    vmovaps %ymm1, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xfloat_masked_shuff_mem_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %ymm2, %ymm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshuff32x4 {{.*#+}} ymm1 {%k1} = ymm0[4,5,6,7],mem[4,5,6,7] sched: [10:1.00]
-; SKX-NEXT:    vmovaps %ymm1, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <8 x float>, <8 x float>* %vec2p
-  %shuf = shufflevector <8 x float> %vec1, <8 x float> %vec2, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 12, i32 13, i32 14, i32 15>
-  %cmp = icmp eq <8 x i32> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x float> %shuf, <8 x float> %vec3
-  ret <8 x float> %res
-}
-
-define <8 x float> @test_8xfloat_zero_masked_shuff_mem_mask1(<8 x float> %vec1, <8 x float>* %vec2p, <8 x i32> %mask) {
-; GENERIC-LABEL: test_8xfloat_zero_masked_shuff_mem_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %ymm1, %ymm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshuff32x4 {{.*#+}} ymm0 {%k1} {z} = ymm0[4,5,6,7],mem[4,5,6,7] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xfloat_zero_masked_shuff_mem_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %ymm1, %ymm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshuff32x4 {{.*#+}} ymm0 {%k1} {z} = ymm0[4,5,6,7],mem[4,5,6,7] sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <8 x float>, <8 x float>* %vec2p
-  %shuf = shufflevector <8 x float> %vec1, <8 x float> %vec2, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 12, i32 13, i32 14, i32 15>
-  %cmp = icmp eq <8 x i32> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x float> %shuf, <8 x float> zeroinitializer
-  ret <8 x float> %res
-}
-
-define <8 x float> @test_8xfloat_masked_shuff_mem_mask2(<8 x float> %vec1, <8 x float>* %vec2p, <8 x float> %vec3, <8 x i32> %mask) {
-; GENERIC-LABEL: test_8xfloat_masked_shuff_mem_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %ymm2, %ymm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshuff32x4 {{.*#+}} ymm1 {%k1} = ymm0[4,5,6,7],mem[0,1,2,3] sched: [8:1.00]
-; GENERIC-NEXT:    vmovaps %ymm1, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xfloat_masked_shuff_mem_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %ymm2, %ymm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshuff32x4 {{.*#+}} ymm1 {%k1} = ymm0[4,5,6,7],mem[0,1,2,3] sched: [10:1.00]
-; SKX-NEXT:    vmovaps %ymm1, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <8 x float>, <8 x float>* %vec2p
-  %shuf = shufflevector <8 x float> %vec1, <8 x float> %vec2, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
-  %cmp = icmp eq <8 x i32> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x float> %shuf, <8 x float> %vec3
-  ret <8 x float> %res
-}
-
-define <8 x float> @test_8xfloat_zero_masked_shuff_mem_mask2(<8 x float> %vec1, <8 x float>* %vec2p, <8 x i32> %mask) {
-; GENERIC-LABEL: test_8xfloat_zero_masked_shuff_mem_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %ymm1, %ymm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshuff32x4 {{.*#+}} ymm0 {%k1} {z} = ymm0[4,5,6,7],mem[0,1,2,3] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xfloat_zero_masked_shuff_mem_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %ymm1, %ymm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshuff32x4 {{.*#+}} ymm0 {%k1} {z} = ymm0[4,5,6,7],mem[0,1,2,3] sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <8 x float>, <8 x float>* %vec2p
-  %shuf = shufflevector <8 x float> %vec1, <8 x float> %vec2, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
-  %cmp = icmp eq <8 x i32> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x float> %shuf, <8 x float> zeroinitializer
-  ret <8 x float> %res
-}
-
-define <8 x float> @test_8xfloat_shuff_mem_mask3(<8 x float> %vec1, <8 x float>* %vec2p) {
-; GENERIC-LABEL: test_8xfloat_shuff_mem_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm0[2,3],mem[0,1] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xfloat_shuff_mem_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm0[2,3],mem[0,1] sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <8 x float>, <8 x float>* %vec2p
-  %res = shufflevector <8 x float> %vec1, <8 x float> %vec2, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
-  ret <8 x float> %res
-}
-define <8 x float> @test_8xfloat_masked_shuff_mem_mask3(<8 x float> %vec1, <8 x float>* %vec2p, <8 x float> %vec3, <8 x i32> %mask) {
-; GENERIC-LABEL: test_8xfloat_masked_shuff_mem_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %ymm2, %ymm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshuff32x4 {{.*#+}} ymm1 {%k1} = ymm0[4,5,6,7],mem[0,1,2,3] sched: [8:1.00]
-; GENERIC-NEXT:    vmovaps %ymm1, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xfloat_masked_shuff_mem_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %ymm2, %ymm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshuff32x4 {{.*#+}} ymm1 {%k1} = ymm0[4,5,6,7],mem[0,1,2,3] sched: [10:1.00]
-; SKX-NEXT:    vmovaps %ymm1, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <8 x float>, <8 x float>* %vec2p
-  %shuf = shufflevector <8 x float> %vec1, <8 x float> %vec2, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
-  %cmp = icmp eq <8 x i32> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x float> %shuf, <8 x float> %vec3
-  ret <8 x float> %res
-}
-
-define <8 x float> @test_8xfloat_zero_masked_shuff_mem_mask3(<8 x float> %vec1, <8 x float>* %vec2p, <8 x i32> %mask) {
-; GENERIC-LABEL: test_8xfloat_zero_masked_shuff_mem_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %ymm1, %ymm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshuff32x4 {{.*#+}} ymm0 {%k1} {z} = ymm0[4,5,6,7],mem[0,1,2,3] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xfloat_zero_masked_shuff_mem_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %ymm1, %ymm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshuff32x4 {{.*#+}} ymm0 {%k1} {z} = ymm0[4,5,6,7],mem[0,1,2,3] sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <8 x float>, <8 x float>* %vec2p
-  %shuf = shufflevector <8 x float> %vec1, <8 x float> %vec2, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
-  %cmp = icmp eq <8 x i32> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x float> %shuf, <8 x float> zeroinitializer
-  ret <8 x float> %res
-}
-
-define <16 x float> @test_16xfloat_shuff_mask0(<16 x float> %vec1, <16 x float> %vec2, <16 x i32> %mask) {
-; GENERIC-LABEL: test_16xfloat_shuff_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vshuff64x2 {{.*#+}} zmm0 = zmm0[6,7,0,1],zmm1[2,3,6,7] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_16xfloat_shuff_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vshuff64x2 {{.*#+}} zmm0 = zmm0[6,7,0,1],zmm1[2,3,6,7] sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %res = shufflevector <16 x float> %vec1, <16 x float> %vec2, <16 x i32> <i32 12, i32 13, i32 14, i32 15, i32 0, i32 1, i32 2, i32 3, i32 20, i32 21, i32 22, i32 23, i32 28, i32 29, i32 30, i32 31>
-  ret <16 x float> %res
-}
-define <16 x float> @test_16xfloat_masked_shuff_mask0(<16 x float> %vec1, <16 x float> %vec2, <16 x float> %vec3, <16 x i32> %mask) {
-; GENERIC-LABEL: test_16xfloat_masked_shuff_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %zmm3, %zmm3, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshuff32x4 {{.*#+}} zmm2 {%k1} = zmm0[12,13,14,15,0,1,2,3],zmm1[4,5,6,7,12,13,14,15] sched: [1:1.00]
-; GENERIC-NEXT:    vmovaps %zmm2, %zmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_16xfloat_masked_shuff_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %zmm3, %zmm3, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshuff32x4 {{.*#+}} zmm2 {%k1} = zmm0[12,13,14,15,0,1,2,3],zmm1[4,5,6,7,12,13,14,15] sched: [3:1.00]
-; SKX-NEXT:    vmovaps %zmm2, %zmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <16 x float> %vec1, <16 x float> %vec2, <16 x i32> <i32 12, i32 13, i32 14, i32 15, i32 0, i32 1, i32 2, i32 3, i32 20, i32 21, i32 22, i32 23, i32 28, i32 29, i32 30, i32 31>
-  %cmp = icmp eq <16 x i32> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x float> %shuf, <16 x float> %vec3
-  ret <16 x float> %res
-}
-
-define <16 x float> @test_16xfloat_zero_masked_shuff_mask0(<16 x float> %vec1, <16 x float> %vec2, <16 x i32> %mask) {
-; GENERIC-LABEL: test_16xfloat_zero_masked_shuff_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %zmm2, %zmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshuff32x4 {{.*#+}} zmm0 {%k1} {z} = zmm0[12,13,14,15,0,1,2,3],zmm1[4,5,6,7,12,13,14,15] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_16xfloat_zero_masked_shuff_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %zmm2, %zmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshuff32x4 {{.*#+}} zmm0 {%k1} {z} = zmm0[12,13,14,15,0,1,2,3],zmm1[4,5,6,7,12,13,14,15] sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <16 x float> %vec1, <16 x float> %vec2, <16 x i32> <i32 12, i32 13, i32 14, i32 15, i32 0, i32 1, i32 2, i32 3, i32 20, i32 21, i32 22, i32 23, i32 28, i32 29, i32 30, i32 31>
-  %cmp = icmp eq <16 x i32> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x float> %shuf, <16 x float> zeroinitializer
-  ret <16 x float> %res
-}
-define <16 x float> @test_16xfloat_masked_shuff_mask1(<16 x float> %vec1, <16 x float> %vec2, <16 x float> %vec3, <16 x i32> %mask) {
-; GENERIC-LABEL: test_16xfloat_masked_shuff_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %zmm3, %zmm3, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshuff32x4 {{.*#+}} zmm2 {%k1} = zmm0[0,1,2,3,8,9,10,11],zmm1[0,1,2,3,12,13,14,15] sched: [1:1.00]
-; GENERIC-NEXT:    vmovaps %zmm2, %zmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_16xfloat_masked_shuff_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %zmm3, %zmm3, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshuff32x4 {{.*#+}} zmm2 {%k1} = zmm0[0,1,2,3,8,9,10,11],zmm1[0,1,2,3,12,13,14,15] sched: [3:1.00]
-; SKX-NEXT:    vmovaps %zmm2, %zmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <16 x float> %vec1, <16 x float> %vec2, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19, i32 28, i32 29, i32 30, i32 31>
-  %cmp = icmp eq <16 x i32> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x float> %shuf, <16 x float> %vec3
-  ret <16 x float> %res
-}
-
-define <16 x float> @test_16xfloat_zero_masked_shuff_mask1(<16 x float> %vec1, <16 x float> %vec2, <16 x i32> %mask) {
-; GENERIC-LABEL: test_16xfloat_zero_masked_shuff_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %zmm2, %zmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshuff32x4 {{.*#+}} zmm0 {%k1} {z} = zmm0[0,1,2,3,8,9,10,11],zmm1[0,1,2,3,12,13,14,15] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_16xfloat_zero_masked_shuff_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %zmm2, %zmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshuff32x4 {{.*#+}} zmm0 {%k1} {z} = zmm0[0,1,2,3,8,9,10,11],zmm1[0,1,2,3,12,13,14,15] sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <16 x float> %vec1, <16 x float> %vec2, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19, i32 28, i32 29, i32 30, i32 31>
-  %cmp = icmp eq <16 x i32> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x float> %shuf, <16 x float> zeroinitializer
-  ret <16 x float> %res
-}
-define <16 x float> @test_16xfloat_masked_shuff_mask2(<16 x float> %vec1, <16 x float> %vec2, <16 x float> %vec3, <16 x i32> %mask) {
-; GENERIC-LABEL: test_16xfloat_masked_shuff_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %zmm3, %zmm3, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshuff32x4 {{.*#+}} zmm2 {%k1} = zmm0[12,13,14,15,4,5,6,7],zmm1[0,1,2,3,4,5,6,7] sched: [1:1.00]
-; GENERIC-NEXT:    vmovaps %zmm2, %zmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_16xfloat_masked_shuff_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %zmm3, %zmm3, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshuff32x4 {{.*#+}} zmm2 {%k1} = zmm0[12,13,14,15,4,5,6,7],zmm1[0,1,2,3,4,5,6,7] sched: [3:1.00]
-; SKX-NEXT:    vmovaps %zmm2, %zmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <16 x float> %vec1, <16 x float> %vec2, <16 x i32> <i32 12, i32 13, i32 14, i32 15, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
-  %cmp = icmp eq <16 x i32> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x float> %shuf, <16 x float> %vec3
-  ret <16 x float> %res
-}
-
-define <16 x float> @test_16xfloat_zero_masked_shuff_mask2(<16 x float> %vec1, <16 x float> %vec2, <16 x i32> %mask) {
-; GENERIC-LABEL: test_16xfloat_zero_masked_shuff_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %zmm2, %zmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshuff32x4 {{.*#+}} zmm0 {%k1} {z} = zmm0[12,13,14,15,4,5,6,7],zmm1[0,1,2,3,4,5,6,7] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_16xfloat_zero_masked_shuff_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %zmm2, %zmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshuff32x4 {{.*#+}} zmm0 {%k1} {z} = zmm0[12,13,14,15,4,5,6,7],zmm1[0,1,2,3,4,5,6,7] sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <16 x float> %vec1, <16 x float> %vec2, <16 x i32> <i32 12, i32 13, i32 14, i32 15, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
-  %cmp = icmp eq <16 x i32> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x float> %shuf, <16 x float> zeroinitializer
-  ret <16 x float> %res
-}
-define <16 x float> @test_16xfloat_shuff_mask3(<16 x float> %vec1, <16 x float> %vec2) {
-; GENERIC-LABEL: test_16xfloat_shuff_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vshuff64x2 {{.*#+}} zmm0 = zmm0[4,5,6,7],zmm1[0,1,4,5] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_16xfloat_shuff_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vshuff64x2 {{.*#+}} zmm0 = zmm0[4,5,6,7],zmm1[0,1,4,5] sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %res = shufflevector <16 x float> %vec1, <16 x float> %vec2, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 24, i32 25, i32 26, i32 27>
-  ret <16 x float> %res
-}
-define <16 x float> @test_16xfloat_masked_shuff_mask3(<16 x float> %vec1, <16 x float> %vec2, <16 x float> %vec3, <16 x i32> %mask) {
-; GENERIC-LABEL: test_16xfloat_masked_shuff_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %zmm3, %zmm3, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshuff32x4 {{.*#+}} zmm2 {%k1} = zmm0[8,9,10,11,12,13,14,15],zmm1[0,1,2,3,8,9,10,11] sched: [1:1.00]
-; GENERIC-NEXT:    vmovaps %zmm2, %zmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_16xfloat_masked_shuff_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %zmm3, %zmm3, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshuff32x4 {{.*#+}} zmm2 {%k1} = zmm0[8,9,10,11,12,13,14,15],zmm1[0,1,2,3,8,9,10,11] sched: [3:1.00]
-; SKX-NEXT:    vmovaps %zmm2, %zmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <16 x float> %vec1, <16 x float> %vec2, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 24, i32 25, i32 26, i32 27>
-  %cmp = icmp eq <16 x i32> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x float> %shuf, <16 x float> %vec3
-  ret <16 x float> %res
-}
-
-define <16 x float> @test_16xfloat_zero_masked_shuff_mask3(<16 x float> %vec1, <16 x float> %vec2, <16 x i32> %mask) {
-; GENERIC-LABEL: test_16xfloat_zero_masked_shuff_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %zmm2, %zmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshuff32x4 {{.*#+}} zmm0 {%k1} {z} = zmm0[8,9,10,11,12,13,14,15],zmm1[0,1,2,3,8,9,10,11] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_16xfloat_zero_masked_shuff_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %zmm2, %zmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshuff32x4 {{.*#+}} zmm0 {%k1} {z} = zmm0[8,9,10,11,12,13,14,15],zmm1[0,1,2,3,8,9,10,11] sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <16 x float> %vec1, <16 x float> %vec2, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 24, i32 25, i32 26, i32 27>
-  %cmp = icmp eq <16 x i32> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x float> %shuf, <16 x float> zeroinitializer
-  ret <16 x float> %res
-}
-define <16 x float> @test_16xfloat_shuff_mem_mask0(<16 x float> %vec1, <16 x float>* %vec2p) {
-; GENERIC-LABEL: test_16xfloat_shuff_mem_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vshuff64x2 {{.*#+}} zmm0 = zmm0[6,7,4,5],mem[4,5,2,3] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_16xfloat_shuff_mem_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vshuff64x2 {{.*#+}} zmm0 = zmm0[6,7,4,5],mem[4,5,2,3] sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <16 x float>, <16 x float>* %vec2p
-  %res = shufflevector <16 x float> %vec1, <16 x float> %vec2, <16 x i32> <i32 12, i32 13, i32 14, i32 15, i32 8, i32 9, i32 10, i32 11, i32 24, i32 25, i32 26, i32 27, i32 20, i32 21, i32 22, i32 23>
-  ret <16 x float> %res
-}
-define <16 x float> @test_16xfloat_masked_shuff_mem_mask0(<16 x float> %vec1, <16 x float>* %vec2p, <16 x float> %vec3, <16 x i32> %mask) {
-; GENERIC-LABEL: test_16xfloat_masked_shuff_mem_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %zmm2, %zmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshuff32x4 {{.*#+}} zmm1 {%k1} = zmm0[12,13,14,15,8,9,10,11],mem[8,9,10,11,4,5,6,7] sched: [8:1.00]
-; GENERIC-NEXT:    vmovaps %zmm1, %zmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_16xfloat_masked_shuff_mem_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %zmm2, %zmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshuff32x4 {{.*#+}} zmm1 {%k1} = zmm0[12,13,14,15,8,9,10,11],mem[8,9,10,11,4,5,6,7] sched: [10:1.00]
-; SKX-NEXT:    vmovaps %zmm1, %zmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <16 x float>, <16 x float>* %vec2p
-  %shuf = shufflevector <16 x float> %vec1, <16 x float> %vec2, <16 x i32> <i32 12, i32 13, i32 14, i32 15, i32 8, i32 9, i32 10, i32 11, i32 24, i32 25, i32 26, i32 27, i32 20, i32 21, i32 22, i32 23>
-  %cmp = icmp eq <16 x i32> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x float> %shuf, <16 x float> %vec3
-  ret <16 x float> %res
-}
-
-define <16 x float> @test_16xfloat_zero_masked_shuff_mem_mask0(<16 x float> %vec1, <16 x float>* %vec2p, <16 x i32> %mask) {
-; GENERIC-LABEL: test_16xfloat_zero_masked_shuff_mem_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshuff32x4 {{.*#+}} zmm0 {%k1} {z} = zmm0[12,13,14,15,8,9,10,11],mem[8,9,10,11,4,5,6,7] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_16xfloat_zero_masked_shuff_mem_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshuff32x4 {{.*#+}} zmm0 {%k1} {z} = zmm0[12,13,14,15,8,9,10,11],mem[8,9,10,11,4,5,6,7] sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <16 x float>, <16 x float>* %vec2p
-  %shuf = shufflevector <16 x float> %vec1, <16 x float> %vec2, <16 x i32> <i32 12, i32 13, i32 14, i32 15, i32 8, i32 9, i32 10, i32 11, i32 24, i32 25, i32 26, i32 27, i32 20, i32 21, i32 22, i32 23>
-  %cmp = icmp eq <16 x i32> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x float> %shuf, <16 x float> zeroinitializer
-  ret <16 x float> %res
-}
-
-define <16 x float> @test_16xfloat_masked_shuff_mem_mask1(<16 x float> %vec1, <16 x float>* %vec2p, <16 x float> %vec3, <16 x i32> %mask) {
-; GENERIC-LABEL: test_16xfloat_masked_shuff_mem_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %zmm2, %zmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshuff32x4 {{.*#+}} zmm1 {%k1} = zmm0[8,9,10,11,4,5,6,7],mem[8,9,10,11,4,5,6,7] sched: [8:1.00]
-; GENERIC-NEXT:    vmovaps %zmm1, %zmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_16xfloat_masked_shuff_mem_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %zmm2, %zmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshuff32x4 {{.*#+}} zmm1 {%k1} = zmm0[8,9,10,11,4,5,6,7],mem[8,9,10,11,4,5,6,7] sched: [10:1.00]
-; SKX-NEXT:    vmovaps %zmm1, %zmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <16 x float>, <16 x float>* %vec2p
-  %shuf = shufflevector <16 x float> %vec1, <16 x float> %vec2, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7, i32 24, i32 25, i32 26, i32 27, i32 20, i32 21, i32 22, i32 23>
-  %cmp = icmp eq <16 x i32> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x float> %shuf, <16 x float> %vec3
-  ret <16 x float> %res
-}
-
-define <16 x float> @test_16xfloat_zero_masked_shuff_mem_mask1(<16 x float> %vec1, <16 x float>* %vec2p, <16 x i32> %mask) {
-; GENERIC-LABEL: test_16xfloat_zero_masked_shuff_mem_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshuff32x4 {{.*#+}} zmm0 {%k1} {z} = zmm0[8,9,10,11,4,5,6,7],mem[8,9,10,11,4,5,6,7] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_16xfloat_zero_masked_shuff_mem_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshuff32x4 {{.*#+}} zmm0 {%k1} {z} = zmm0[8,9,10,11,4,5,6,7],mem[8,9,10,11,4,5,6,7] sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <16 x float>, <16 x float>* %vec2p
-  %shuf = shufflevector <16 x float> %vec1, <16 x float> %vec2, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7, i32 24, i32 25, i32 26, i32 27, i32 20, i32 21, i32 22, i32 23>
-  %cmp = icmp eq <16 x i32> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x float> %shuf, <16 x float> zeroinitializer
-  ret <16 x float> %res
-}
-
-define <16 x float> @test_16xfloat_masked_shuff_mem_mask2(<16 x float> %vec1, <16 x float>* %vec2p, <16 x float> %vec3, <16 x i32> %mask) {
-; GENERIC-LABEL: test_16xfloat_masked_shuff_mem_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %zmm2, %zmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshuff32x4 {{.*#+}} zmm1 {%k1} = zmm0[0,1,2,3,0,1,2,3],mem[8,9,10,11,8,9,10,11] sched: [8:1.00]
-; GENERIC-NEXT:    vmovaps %zmm1, %zmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_16xfloat_masked_shuff_mem_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %zmm2, %zmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshuff32x4 {{.*#+}} zmm1 {%k1} = zmm0[0,1,2,3,0,1,2,3],mem[8,9,10,11,8,9,10,11] sched: [10:1.00]
-; SKX-NEXT:    vmovaps %zmm1, %zmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <16 x float>, <16 x float>* %vec2p
-  %shuf = shufflevector <16 x float> %vec1, <16 x float> %vec2, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 24, i32 25, i32 26, i32 27, i32 24, i32 25, i32 26, i32 27>
-  %cmp = icmp eq <16 x i32> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x float> %shuf, <16 x float> %vec3
-  ret <16 x float> %res
-}
-
-define <16 x float> @test_16xfloat_zero_masked_shuff_mem_mask2(<16 x float> %vec1, <16 x float>* %vec2p, <16 x i32> %mask) {
-; GENERIC-LABEL: test_16xfloat_zero_masked_shuff_mem_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshuff32x4 {{.*#+}} zmm0 {%k1} {z} = zmm0[0,1,2,3,0,1,2,3],mem[8,9,10,11,8,9,10,11] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_16xfloat_zero_masked_shuff_mem_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshuff32x4 {{.*#+}} zmm0 {%k1} {z} = zmm0[0,1,2,3,0,1,2,3],mem[8,9,10,11,8,9,10,11] sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <16 x float>, <16 x float>* %vec2p
-  %shuf = shufflevector <16 x float> %vec1, <16 x float> %vec2, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 24, i32 25, i32 26, i32 27, i32 24, i32 25, i32 26, i32 27>
-  %cmp = icmp eq <16 x i32> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x float> %shuf, <16 x float> zeroinitializer
-  ret <16 x float> %res
-}
-
-define <16 x float> @test_16xfloat_shuff_mem_mask3(<16 x float> %vec1, <16 x float>* %vec2p) {
-; GENERIC-LABEL: test_16xfloat_shuff_mem_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vshuff64x2 {{.*#+}} zmm0 = zmm0[2,3,0,1],mem[6,7,6,7] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_16xfloat_shuff_mem_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vshuff64x2 {{.*#+}} zmm0 = zmm0[2,3,0,1],mem[6,7,6,7] sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <16 x float>, <16 x float>* %vec2p
-  %res = shufflevector <16 x float> %vec1, <16 x float> %vec2, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 28, i32 29, i32 30, i32 31, i32 28, i32 29, i32 30, i32 31>
-  ret <16 x float> %res
-}
-define <16 x float> @test_16xfloat_masked_shuff_mem_mask3(<16 x float> %vec1, <16 x float>* %vec2p, <16 x float> %vec3, <16 x i32> %mask) {
-; GENERIC-LABEL: test_16xfloat_masked_shuff_mem_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %zmm2, %zmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshuff32x4 {{.*#+}} zmm1 {%k1} = zmm0[4,5,6,7,0,1,2,3],mem[12,13,14,15,12,13,14,15] sched: [8:1.00]
-; GENERIC-NEXT:    vmovaps %zmm1, %zmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_16xfloat_masked_shuff_mem_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %zmm2, %zmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshuff32x4 {{.*#+}} zmm1 {%k1} = zmm0[4,5,6,7,0,1,2,3],mem[12,13,14,15,12,13,14,15] sched: [10:1.00]
-; SKX-NEXT:    vmovaps %zmm1, %zmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <16 x float>, <16 x float>* %vec2p
-  %shuf = shufflevector <16 x float> %vec1, <16 x float> %vec2, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 28, i32 29, i32 30, i32 31, i32 28, i32 29, i32 30, i32 31>
-  %cmp = icmp eq <16 x i32> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x float> %shuf, <16 x float> %vec3
-  ret <16 x float> %res
-}
-
-define <16 x float> @test_16xfloat_zero_masked_shuff_mem_mask3(<16 x float> %vec1, <16 x float>* %vec2p, <16 x i32> %mask) {
-; GENERIC-LABEL: test_16xfloat_zero_masked_shuff_mem_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshuff32x4 {{.*#+}} zmm0 {%k1} {z} = zmm0[4,5,6,7,0,1,2,3],mem[12,13,14,15,12,13,14,15] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_16xfloat_zero_masked_shuff_mem_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshuff32x4 {{.*#+}} zmm0 {%k1} {z} = zmm0[4,5,6,7,0,1,2,3],mem[12,13,14,15,12,13,14,15] sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <16 x float>, <16 x float>* %vec2p
-  %shuf = shufflevector <16 x float> %vec1, <16 x float> %vec2, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 28, i32 29, i32 30, i32 31, i32 28, i32 29, i32 30, i32 31>
-  %cmp = icmp eq <16 x i32> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x float> %shuf, <16 x float> zeroinitializer
-  ret <16 x float> %res
-}
-
-define <4 x double> @test_4xdouble_shuff_mask0(<4 x double> %vec1, <4 x double> %vec2) {
-; GENERIC-LABEL: test_4xdouble_shuff_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm0[2,3],ymm1[0,1] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xdouble_shuff_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm0[2,3],ymm1[0,1] sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %res = shufflevector <4 x double> %vec1, <4 x double> %vec2, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
-  ret <4 x double> %res
-}
-define <4 x double> @test_4xdouble_masked_shuff_mask0(<4 x double> %vec1, <4 x double> %vec2, <4 x double> %vec3, <4 x i64> %mask) {
-; GENERIC-LABEL: test_4xdouble_masked_shuff_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %ymm3, %ymm3, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshuff64x2 {{.*#+}} ymm2 {%k1} = ymm0[2,3],ymm1[0,1] sched: [1:1.00]
-; GENERIC-NEXT:    vmovapd %ymm2, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xdouble_masked_shuff_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %ymm3, %ymm3, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshuff64x2 {{.*#+}} ymm2 {%k1} = ymm0[2,3],ymm1[0,1] sched: [3:1.00]
-; SKX-NEXT:    vmovapd %ymm2, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <4 x double> %vec1, <4 x double> %vec2, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
-  %cmp = icmp eq <4 x i64> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x double> %shuf, <4 x double> %vec3
-  ret <4 x double> %res
-}
-
-define <4 x double> @test_4xdouble_zero_masked_shuff_mask0(<4 x double> %vec1, <4 x double> %vec2, <4 x i64> %mask) {
-; GENERIC-LABEL: test_4xdouble_zero_masked_shuff_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %ymm2, %ymm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshuff64x2 {{.*#+}} ymm0 {%k1} {z} = ymm0[2,3],ymm1[0,1] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xdouble_zero_masked_shuff_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %ymm2, %ymm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshuff64x2 {{.*#+}} ymm0 {%k1} {z} = ymm0[2,3],ymm1[0,1] sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <4 x double> %vec1, <4 x double> %vec2, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
-  %cmp = icmp eq <4 x i64> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x double> %shuf, <4 x double> zeroinitializer
-  ret <4 x double> %res
-}
-define <4 x double> @test_4xdouble_masked_shuff_mask1(<4 x double> %vec1, <4 x double> %vec2, <4 x double> %vec3, <4 x i64> %mask) {
-; GENERIC-LABEL: test_4xdouble_masked_shuff_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %ymm3, %ymm3, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshuff64x2 {{.*#+}} ymm2 {%k1} = ymm0[2,3],ymm1[0,1] sched: [1:1.00]
-; GENERIC-NEXT:    vmovapd %ymm2, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xdouble_masked_shuff_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %ymm3, %ymm3, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshuff64x2 {{.*#+}} ymm2 {%k1} = ymm0[2,3],ymm1[0,1] sched: [3:1.00]
-; SKX-NEXT:    vmovapd %ymm2, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <4 x double> %vec1, <4 x double> %vec2, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
-  %cmp = icmp eq <4 x i64> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x double> %shuf, <4 x double> %vec3
-  ret <4 x double> %res
-}
-
-define <4 x double> @test_4xdouble_zero_masked_shuff_mask1(<4 x double> %vec1, <4 x double> %vec2, <4 x i64> %mask) {
-; GENERIC-LABEL: test_4xdouble_zero_masked_shuff_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %ymm2, %ymm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshuff64x2 {{.*#+}} ymm0 {%k1} {z} = ymm0[2,3],ymm1[0,1] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xdouble_zero_masked_shuff_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %ymm2, %ymm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshuff64x2 {{.*#+}} ymm0 {%k1} {z} = ymm0[2,3],ymm1[0,1] sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <4 x double> %vec1, <4 x double> %vec2, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
-  %cmp = icmp eq <4 x i64> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x double> %shuf, <4 x double> zeroinitializer
-  ret <4 x double> %res
-}
-define <4 x double> @test_4xdouble_masked_shuff_mask2(<4 x double> %vec1, <4 x double> %vec2, <4 x double> %vec3, <4 x i64> %mask) {
-; GENERIC-LABEL: test_4xdouble_masked_shuff_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %ymm3, %ymm3, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshuff64x2 {{.*#+}} ymm2 {%k1} = ymm0[2,3],ymm1[2,3] sched: [1:1.00]
-; GENERIC-NEXT:    vmovapd %ymm2, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xdouble_masked_shuff_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %ymm3, %ymm3, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshuff64x2 {{.*#+}} ymm2 {%k1} = ymm0[2,3],ymm1[2,3] sched: [3:1.00]
-; SKX-NEXT:    vmovapd %ymm2, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <4 x double> %vec1, <4 x double> %vec2, <4 x i32> <i32 2, i32 3, i32 6, i32 7>
-  %cmp = icmp eq <4 x i64> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x double> %shuf, <4 x double> %vec3
-  ret <4 x double> %res
-}
-
-define <4 x double> @test_4xdouble_zero_masked_shuff_mask2(<4 x double> %vec1, <4 x double> %vec2, <4 x i64> %mask) {
-; GENERIC-LABEL: test_4xdouble_zero_masked_shuff_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %ymm2, %ymm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshuff64x2 {{.*#+}} ymm0 {%k1} {z} = ymm0[2,3],ymm1[2,3] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xdouble_zero_masked_shuff_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %ymm2, %ymm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshuff64x2 {{.*#+}} ymm0 {%k1} {z} = ymm0[2,3],ymm1[2,3] sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <4 x double> %vec1, <4 x double> %vec2, <4 x i32> <i32 2, i32 3, i32 6, i32 7>
-  %cmp = icmp eq <4 x i64> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x double> %shuf, <4 x double> zeroinitializer
-  ret <4 x double> %res
-}
-define <4 x double> @test_4xdouble_shuff_mask3(<4 x double> %vec1, <4 x double> %vec2) {
-; GENERIC-LABEL: test_4xdouble_shuff_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm0[2,3],ymm1[2,3] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xdouble_shuff_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm0[2,3],ymm1[2,3] sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %res = shufflevector <4 x double> %vec1, <4 x double> %vec2, <4 x i32> <i32 2, i32 3, i32 6, i32 7>
-  ret <4 x double> %res
-}
-define <4 x double> @test_4xdouble_masked_shuff_mask3(<4 x double> %vec1, <4 x double> %vec2, <4 x double> %vec3, <4 x i64> %mask) {
-; GENERIC-LABEL: test_4xdouble_masked_shuff_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %ymm3, %ymm3, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshuff64x2 {{.*#+}} ymm2 {%k1} = ymm0[2,3],ymm1[2,3] sched: [1:1.00]
-; GENERIC-NEXT:    vmovapd %ymm2, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xdouble_masked_shuff_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %ymm3, %ymm3, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshuff64x2 {{.*#+}} ymm2 {%k1} = ymm0[2,3],ymm1[2,3] sched: [3:1.00]
-; SKX-NEXT:    vmovapd %ymm2, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <4 x double> %vec1, <4 x double> %vec2, <4 x i32> <i32 2, i32 3, i32 6, i32 7>
-  %cmp = icmp eq <4 x i64> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x double> %shuf, <4 x double> %vec3
-  ret <4 x double> %res
-}
-
-define <4 x double> @test_4xdouble_zero_masked_shuff_mask3(<4 x double> %vec1, <4 x double> %vec2, <4 x i64> %mask) {
-; GENERIC-LABEL: test_4xdouble_zero_masked_shuff_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %ymm2, %ymm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshuff64x2 {{.*#+}} ymm0 {%k1} {z} = ymm0[2,3],ymm1[2,3] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xdouble_zero_masked_shuff_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %ymm2, %ymm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshuff64x2 {{.*#+}} ymm0 {%k1} {z} = ymm0[2,3],ymm1[2,3] sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <4 x double> %vec1, <4 x double> %vec2, <4 x i32> <i32 2, i32 3, i32 6, i32 7>
-  %cmp = icmp eq <4 x i64> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x double> %shuf, <4 x double> zeroinitializer
-  ret <4 x double> %res
-}
-define <4 x double> @test_4xdouble_shuff_mem_mask0(<4 x double> %vec1, <4 x double>* %vec2p) {
-; GENERIC-LABEL: test_4xdouble_shuff_mem_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm0[2,3],mem[2,3] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xdouble_shuff_mem_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm0[2,3],mem[2,3] sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <4 x double>, <4 x double>* %vec2p
-  %res = shufflevector <4 x double> %vec1, <4 x double> %vec2, <4 x i32> <i32 2, i32 3, i32 6, i32 7>
-  ret <4 x double> %res
-}
-define <4 x double> @test_4xdouble_masked_shuff_mem_mask0(<4 x double> %vec1, <4 x double>* %vec2p, <4 x double> %vec3, <4 x i64> %mask) {
-; GENERIC-LABEL: test_4xdouble_masked_shuff_mem_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %ymm2, %ymm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshuff64x2 {{.*#+}} ymm1 {%k1} = ymm0[2,3],mem[2,3] sched: [8:1.00]
-; GENERIC-NEXT:    vmovapd %ymm1, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xdouble_masked_shuff_mem_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %ymm2, %ymm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshuff64x2 {{.*#+}} ymm1 {%k1} = ymm0[2,3],mem[2,3] sched: [10:1.00]
-; SKX-NEXT:    vmovapd %ymm1, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <4 x double>, <4 x double>* %vec2p
-  %shuf = shufflevector <4 x double> %vec1, <4 x double> %vec2, <4 x i32> <i32 2, i32 3, i32 6, i32 7>
-  %cmp = icmp eq <4 x i64> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x double> %shuf, <4 x double> %vec3
-  ret <4 x double> %res
-}
-
-define <4 x double> @test_4xdouble_zero_masked_shuff_mem_mask0(<4 x double> %vec1, <4 x double>* %vec2p, <4 x i64> %mask) {
-; GENERIC-LABEL: test_4xdouble_zero_masked_shuff_mem_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %ymm1, %ymm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshuff64x2 {{.*#+}} ymm0 {%k1} {z} = ymm0[2,3],mem[2,3] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xdouble_zero_masked_shuff_mem_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %ymm1, %ymm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshuff64x2 {{.*#+}} ymm0 {%k1} {z} = ymm0[2,3],mem[2,3] sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <4 x double>, <4 x double>* %vec2p
-  %shuf = shufflevector <4 x double> %vec1, <4 x double> %vec2, <4 x i32> <i32 2, i32 3, i32 6, i32 7>
-  %cmp = icmp eq <4 x i64> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x double> %shuf, <4 x double> zeroinitializer
-  ret <4 x double> %res
-}
-
-define <4 x double> @test_4xdouble_masked_shuff_mem_mask1(<4 x double> %vec1, <4 x double>* %vec2p, <4 x double> %vec3, <4 x i64> %mask) {
-; GENERIC-LABEL: test_4xdouble_masked_shuff_mem_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %ymm2, %ymm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshuff64x2 {{.*#+}} ymm1 {%k1} = ymm0[2,3],mem[0,1] sched: [8:1.00]
-; GENERIC-NEXT:    vmovapd %ymm1, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xdouble_masked_shuff_mem_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %ymm2, %ymm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshuff64x2 {{.*#+}} ymm1 {%k1} = ymm0[2,3],mem[0,1] sched: [10:1.00]
-; SKX-NEXT:    vmovapd %ymm1, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <4 x double>, <4 x double>* %vec2p
-  %shuf = shufflevector <4 x double> %vec1, <4 x double> %vec2, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
-  %cmp = icmp eq <4 x i64> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x double> %shuf, <4 x double> %vec3
-  ret <4 x double> %res
-}
-
-define <4 x double> @test_4xdouble_zero_masked_shuff_mem_mask1(<4 x double> %vec1, <4 x double>* %vec2p, <4 x i64> %mask) {
-; GENERIC-LABEL: test_4xdouble_zero_masked_shuff_mem_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %ymm1, %ymm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshuff64x2 {{.*#+}} ymm0 {%k1} {z} = ymm0[2,3],mem[0,1] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xdouble_zero_masked_shuff_mem_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %ymm1, %ymm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshuff64x2 {{.*#+}} ymm0 {%k1} {z} = ymm0[2,3],mem[0,1] sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <4 x double>, <4 x double>* %vec2p
-  %shuf = shufflevector <4 x double> %vec1, <4 x double> %vec2, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
-  %cmp = icmp eq <4 x i64> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x double> %shuf, <4 x double> zeroinitializer
-  ret <4 x double> %res
-}
-
-define <4 x double> @test_4xdouble_masked_shuff_mem_mask2(<4 x double> %vec1, <4 x double>* %vec2p, <4 x double> %vec3, <4 x i64> %mask) {
-; GENERIC-LABEL: test_4xdouble_masked_shuff_mem_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %ymm2, %ymm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshuff64x2 {{.*#+}} ymm1 {%k1} = ymm0[2,3],mem[0,1] sched: [8:1.00]
-; GENERIC-NEXT:    vmovapd %ymm1, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xdouble_masked_shuff_mem_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %ymm2, %ymm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshuff64x2 {{.*#+}} ymm1 {%k1} = ymm0[2,3],mem[0,1] sched: [10:1.00]
-; SKX-NEXT:    vmovapd %ymm1, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <4 x double>, <4 x double>* %vec2p
-  %shuf = shufflevector <4 x double> %vec1, <4 x double> %vec2, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
-  %cmp = icmp eq <4 x i64> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x double> %shuf, <4 x double> %vec3
-  ret <4 x double> %res
-}
-
-define <4 x double> @test_4xdouble_zero_masked_shuff_mem_mask2(<4 x double> %vec1, <4 x double>* %vec2p, <4 x i64> %mask) {
-; GENERIC-LABEL: test_4xdouble_zero_masked_shuff_mem_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %ymm1, %ymm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshuff64x2 {{.*#+}} ymm0 {%k1} {z} = ymm0[2,3],mem[0,1] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xdouble_zero_masked_shuff_mem_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %ymm1, %ymm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshuff64x2 {{.*#+}} ymm0 {%k1} {z} = ymm0[2,3],mem[0,1] sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <4 x double>, <4 x double>* %vec2p
-  %shuf = shufflevector <4 x double> %vec1, <4 x double> %vec2, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
-  %cmp = icmp eq <4 x i64> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x double> %shuf, <4 x double> zeroinitializer
-  ret <4 x double> %res
-}
-
-define <4 x double> @test_4xdouble_shuff_mem_mask3(<4 x double> %vec1, <4 x double>* %vec2p) {
-; GENERIC-LABEL: test_4xdouble_shuff_mem_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm0[2,3],mem[2,3] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xdouble_shuff_mem_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm0[2,3],mem[2,3] sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <4 x double>, <4 x double>* %vec2p
-  %res = shufflevector <4 x double> %vec1, <4 x double> %vec2, <4 x i32> <i32 2, i32 3, i32 6, i32 7>
-  ret <4 x double> %res
-}
-define <4 x double> @test_4xdouble_masked_shuff_mem_mask3(<4 x double> %vec1, <4 x double>* %vec2p, <4 x double> %vec3, <4 x i64> %mask) {
-; GENERIC-LABEL: test_4xdouble_masked_shuff_mem_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %ymm2, %ymm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshuff64x2 {{.*#+}} ymm1 {%k1} = ymm0[2,3],mem[2,3] sched: [8:1.00]
-; GENERIC-NEXT:    vmovapd %ymm1, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xdouble_masked_shuff_mem_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %ymm2, %ymm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshuff64x2 {{.*#+}} ymm1 {%k1} = ymm0[2,3],mem[2,3] sched: [10:1.00]
-; SKX-NEXT:    vmovapd %ymm1, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <4 x double>, <4 x double>* %vec2p
-  %shuf = shufflevector <4 x double> %vec1, <4 x double> %vec2, <4 x i32> <i32 2, i32 3, i32 6, i32 7>
-  %cmp = icmp eq <4 x i64> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x double> %shuf, <4 x double> %vec3
-  ret <4 x double> %res
-}
-
-define <4 x double> @test_4xdouble_zero_masked_shuff_mem_mask3(<4 x double> %vec1, <4 x double>* %vec2p, <4 x i64> %mask) {
-; GENERIC-LABEL: test_4xdouble_zero_masked_shuff_mem_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %ymm1, %ymm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshuff64x2 {{.*#+}} ymm0 {%k1} {z} = ymm0[2,3],mem[2,3] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xdouble_zero_masked_shuff_mem_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %ymm1, %ymm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshuff64x2 {{.*#+}} ymm0 {%k1} {z} = ymm0[2,3],mem[2,3] sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <4 x double>, <4 x double>* %vec2p
-  %shuf = shufflevector <4 x double> %vec1, <4 x double> %vec2, <4 x i32> <i32 2, i32 3, i32 6, i32 7>
-  %cmp = icmp eq <4 x i64> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x double> %shuf, <4 x double> zeroinitializer
-  ret <4 x double> %res
-}
-
-define <8 x double> @test_8xdouble_shuff_mask0(<8 x double> %vec1, <8 x double> %vec2) {
-; GENERIC-LABEL: test_8xdouble_shuff_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vshuff64x2 {{.*#+}} zmm0 = zmm0[6,7,2,3],zmm1[6,7,0,1] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xdouble_shuff_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vshuff64x2 {{.*#+}} zmm0 = zmm0[6,7,2,3],zmm1[6,7,0,1] sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %res = shufflevector <8 x double> %vec1, <8 x double> %vec2, <8 x i32> <i32 6, i32 7, i32 2, i32 3, i32 14, i32 15, i32 8, i32 9>
-  ret <8 x double> %res
-}
-define <8 x double> @test_8xdouble_masked_shuff_mask0(<8 x double> %vec1, <8 x double> %vec2, <8 x double> %vec3, <8 x i64> %mask) {
-; GENERIC-LABEL: test_8xdouble_masked_shuff_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %zmm3, %zmm3, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshuff64x2 {{.*#+}} zmm2 {%k1} = zmm0[6,7,2,3],zmm1[6,7,0,1] sched: [1:1.00]
-; GENERIC-NEXT:    vmovapd %zmm2, %zmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xdouble_masked_shuff_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %zmm3, %zmm3, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshuff64x2 {{.*#+}} zmm2 {%k1} = zmm0[6,7,2,3],zmm1[6,7,0,1] sched: [3:1.00]
-; SKX-NEXT:    vmovapd %zmm2, %zmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x double> %vec1, <8 x double> %vec2, <8 x i32> <i32 6, i32 7, i32 2, i32 3, i32 14, i32 15, i32 8, i32 9>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x double> %shuf, <8 x double> %vec3
-  ret <8 x double> %res
-}
-
-define <8 x double> @test_8xdouble_zero_masked_shuff_mask0(<8 x double> %vec1, <8 x double> %vec2, <8 x i64> %mask) {
-; GENERIC-LABEL: test_8xdouble_zero_masked_shuff_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %zmm2, %zmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshuff64x2 {{.*#+}} zmm0 {%k1} {z} = zmm0[6,7,2,3],zmm1[6,7,0,1] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xdouble_zero_masked_shuff_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %zmm2, %zmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshuff64x2 {{.*#+}} zmm0 {%k1} {z} = zmm0[6,7,2,3],zmm1[6,7,0,1] sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x double> %vec1, <8 x double> %vec2, <8 x i32> <i32 6, i32 7, i32 2, i32 3, i32 14, i32 15, i32 8, i32 9>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x double> %shuf, <8 x double> zeroinitializer
-  ret <8 x double> %res
-}
-define <8 x double> @test_8xdouble_masked_shuff_mask1(<8 x double> %vec1, <8 x double> %vec2, <8 x double> %vec3, <8 x i64> %mask) {
-; GENERIC-LABEL: test_8xdouble_masked_shuff_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %zmm3, %zmm3, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshuff64x2 {{.*#+}} zmm2 {%k1} = zmm0[0,1,4,5],zmm1[0,1,4,5] sched: [1:1.00]
-; GENERIC-NEXT:    vmovapd %zmm2, %zmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xdouble_masked_shuff_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %zmm3, %zmm3, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshuff64x2 {{.*#+}} zmm2 {%k1} = zmm0[0,1,4,5],zmm1[0,1,4,5] sched: [3:1.00]
-; SKX-NEXT:    vmovapd %zmm2, %zmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x double> %vec1, <8 x double> %vec2, <8 x i32> <i32 0, i32 1, i32 4, i32 5, i32 8, i32 9, i32 12, i32 13>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x double> %shuf, <8 x double> %vec3
-  ret <8 x double> %res
-}
-
-define <8 x double> @test_8xdouble_zero_masked_shuff_mask1(<8 x double> %vec1, <8 x double> %vec2, <8 x i64> %mask) {
-; GENERIC-LABEL: test_8xdouble_zero_masked_shuff_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %zmm2, %zmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshuff64x2 {{.*#+}} zmm0 {%k1} {z} = zmm0[0,1,4,5],zmm1[0,1,4,5] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xdouble_zero_masked_shuff_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %zmm2, %zmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshuff64x2 {{.*#+}} zmm0 {%k1} {z} = zmm0[0,1,4,5],zmm1[0,1,4,5] sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x double> %vec1, <8 x double> %vec2, <8 x i32> <i32 0, i32 1, i32 4, i32 5, i32 8, i32 9, i32 12, i32 13>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x double> %shuf, <8 x double> zeroinitializer
-  ret <8 x double> %res
-}
-define <8 x double> @test_8xdouble_masked_shuff_mask2(<8 x double> %vec1, <8 x double> %vec2, <8 x double> %vec3, <8 x i64> %mask) {
-; GENERIC-LABEL: test_8xdouble_masked_shuff_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %zmm3, %zmm3, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshuff64x2 {{.*#+}} zmm2 {%k1} = zmm0[6,7,4,5],zmm1[4,5,0,1] sched: [1:1.00]
-; GENERIC-NEXT:    vmovapd %zmm2, %zmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xdouble_masked_shuff_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %zmm3, %zmm3, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshuff64x2 {{.*#+}} zmm2 {%k1} = zmm0[6,7,4,5],zmm1[4,5,0,1] sched: [3:1.00]
-; SKX-NEXT:    vmovapd %zmm2, %zmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x double> %vec1, <8 x double> %vec2, <8 x i32> <i32 6, i32 7, i32 4, i32 5, i32 12, i32 13, i32 8, i32 9>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x double> %shuf, <8 x double> %vec3
-  ret <8 x double> %res
-}
-
-define <8 x double> @test_8xdouble_zero_masked_shuff_mask2(<8 x double> %vec1, <8 x double> %vec2, <8 x i64> %mask) {
-; GENERIC-LABEL: test_8xdouble_zero_masked_shuff_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %zmm2, %zmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshuff64x2 {{.*#+}} zmm0 {%k1} {z} = zmm0[6,7,4,5],zmm1[4,5,0,1] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xdouble_zero_masked_shuff_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %zmm2, %zmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshuff64x2 {{.*#+}} zmm0 {%k1} {z} = zmm0[6,7,4,5],zmm1[4,5,0,1] sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x double> %vec1, <8 x double> %vec2, <8 x i32> <i32 6, i32 7, i32 4, i32 5, i32 12, i32 13, i32 8, i32 9>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x double> %shuf, <8 x double> zeroinitializer
-  ret <8 x double> %res
-}
-define <8 x double> @test_8xdouble_shuff_mask3(<8 x double> %vec1, <8 x double> %vec2) {
-; GENERIC-LABEL: test_8xdouble_shuff_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vshuff64x2 {{.*#+}} zmm0 = zmm0[4,5,4,5],zmm1[4,5,2,3] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xdouble_shuff_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vshuff64x2 {{.*#+}} zmm0 = zmm0[4,5,4,5],zmm1[4,5,2,3] sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %res = shufflevector <8 x double> %vec1, <8 x double> %vec2, <8 x i32> <i32 4, i32 5, i32 4, i32 5, i32 12, i32 13, i32 10, i32 11>
-  ret <8 x double> %res
-}
-define <8 x double> @test_8xdouble_masked_shuff_mask3(<8 x double> %vec1, <8 x double> %vec2, <8 x double> %vec3, <8 x i64> %mask) {
-; GENERIC-LABEL: test_8xdouble_masked_shuff_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %zmm3, %zmm3, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshuff64x2 {{.*#+}} zmm2 {%k1} = zmm0[4,5,4,5],zmm1[4,5,2,3] sched: [1:1.00]
-; GENERIC-NEXT:    vmovapd %zmm2, %zmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xdouble_masked_shuff_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %zmm3, %zmm3, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshuff64x2 {{.*#+}} zmm2 {%k1} = zmm0[4,5,4,5],zmm1[4,5,2,3] sched: [3:1.00]
-; SKX-NEXT:    vmovapd %zmm2, %zmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x double> %vec1, <8 x double> %vec2, <8 x i32> <i32 4, i32 5, i32 4, i32 5, i32 12, i32 13, i32 10, i32 11>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x double> %shuf, <8 x double> %vec3
-  ret <8 x double> %res
-}
-
-define <8 x double> @test_8xdouble_zero_masked_shuff_mask3(<8 x double> %vec1, <8 x double> %vec2, <8 x i64> %mask) {
-; GENERIC-LABEL: test_8xdouble_zero_masked_shuff_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %zmm2, %zmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshuff64x2 {{.*#+}} zmm0 {%k1} {z} = zmm0[4,5,4,5],zmm1[4,5,2,3] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xdouble_zero_masked_shuff_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %zmm2, %zmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshuff64x2 {{.*#+}} zmm0 {%k1} {z} = zmm0[4,5,4,5],zmm1[4,5,2,3] sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x double> %vec1, <8 x double> %vec2, <8 x i32> <i32 4, i32 5, i32 4, i32 5, i32 12, i32 13, i32 10, i32 11>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x double> %shuf, <8 x double> zeroinitializer
-  ret <8 x double> %res
-}
-define <8 x double> @test_8xdouble_shuff_mem_mask0(<8 x double> %vec1, <8 x double>* %vec2p) {
-; GENERIC-LABEL: test_8xdouble_shuff_mem_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vshuff64x2 {{.*#+}} zmm0 = zmm0[6,7,0,1],mem[0,1,0,1] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xdouble_shuff_mem_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vshuff64x2 {{.*#+}} zmm0 = zmm0[6,7,0,1],mem[0,1,0,1] sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <8 x double>, <8 x double>* %vec2p
-  %res = shufflevector <8 x double> %vec1, <8 x double> %vec2, <8 x i32> <i32 6, i32 7, i32 0, i32 1, i32 8, i32 9, i32 8, i32 9>
-  ret <8 x double> %res
-}
-define <8 x double> @test_8xdouble_masked_shuff_mem_mask0(<8 x double> %vec1, <8 x double>* %vec2p, <8 x double> %vec3, <8 x i64> %mask) {
-; GENERIC-LABEL: test_8xdouble_masked_shuff_mem_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %zmm2, %zmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshuff64x2 {{.*#+}} zmm1 {%k1} = zmm0[6,7,0,1],mem[0,1,0,1] sched: [8:1.00]
-; GENERIC-NEXT:    vmovapd %zmm1, %zmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xdouble_masked_shuff_mem_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %zmm2, %zmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshuff64x2 {{.*#+}} zmm1 {%k1} = zmm0[6,7,0,1],mem[0,1,0,1] sched: [10:1.00]
-; SKX-NEXT:    vmovapd %zmm1, %zmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <8 x double>, <8 x double>* %vec2p
-  %shuf = shufflevector <8 x double> %vec1, <8 x double> %vec2, <8 x i32> <i32 6, i32 7, i32 0, i32 1, i32 8, i32 9, i32 8, i32 9>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x double> %shuf, <8 x double> %vec3
-  ret <8 x double> %res
-}
-
-define <8 x double> @test_8xdouble_zero_masked_shuff_mem_mask0(<8 x double> %vec1, <8 x double>* %vec2p, <8 x i64> %mask) {
-; GENERIC-LABEL: test_8xdouble_zero_masked_shuff_mem_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshuff64x2 {{.*#+}} zmm0 {%k1} {z} = zmm0[6,7,0,1],mem[0,1,0,1] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xdouble_zero_masked_shuff_mem_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshuff64x2 {{.*#+}} zmm0 {%k1} {z} = zmm0[6,7,0,1],mem[0,1,0,1] sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <8 x double>, <8 x double>* %vec2p
-  %shuf = shufflevector <8 x double> %vec1, <8 x double> %vec2, <8 x i32> <i32 6, i32 7, i32 0, i32 1, i32 8, i32 9, i32 8, i32 9>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x double> %shuf, <8 x double> zeroinitializer
-  ret <8 x double> %res
-}
-
-define <8 x double> @test_8xdouble_masked_shuff_mem_mask1(<8 x double> %vec1, <8 x double>* %vec2p, <8 x double> %vec3, <8 x i64> %mask) {
-; GENERIC-LABEL: test_8xdouble_masked_shuff_mem_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %zmm2, %zmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshuff64x2 {{.*#+}} zmm1 {%k1} = zmm0[6,7,6,7],mem[0,1,2,3] sched: [8:1.00]
-; GENERIC-NEXT:    vmovapd %zmm1, %zmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xdouble_masked_shuff_mem_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %zmm2, %zmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshuff64x2 {{.*#+}} zmm1 {%k1} = zmm0[6,7,6,7],mem[0,1,2,3] sched: [10:1.00]
-; SKX-NEXT:    vmovapd %zmm1, %zmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <8 x double>, <8 x double>* %vec2p
-  %shuf = shufflevector <8 x double> %vec1, <8 x double> %vec2, <8 x i32> <i32 6, i32 7, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x double> %shuf, <8 x double> %vec3
-  ret <8 x double> %res
-}
-
-define <8 x double> @test_8xdouble_zero_masked_shuff_mem_mask1(<8 x double> %vec1, <8 x double>* %vec2p, <8 x i64> %mask) {
-; GENERIC-LABEL: test_8xdouble_zero_masked_shuff_mem_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshuff64x2 {{.*#+}} zmm0 {%k1} {z} = zmm0[6,7,6,7],mem[0,1,2,3] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xdouble_zero_masked_shuff_mem_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshuff64x2 {{.*#+}} zmm0 {%k1} {z} = zmm0[6,7,6,7],mem[0,1,2,3] sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <8 x double>, <8 x double>* %vec2p
-  %shuf = shufflevector <8 x double> %vec1, <8 x double> %vec2, <8 x i32> <i32 6, i32 7, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x double> %shuf, <8 x double> zeroinitializer
-  ret <8 x double> %res
-}
-
-define <8 x double> @test_8xdouble_masked_shuff_mem_mask2(<8 x double> %vec1, <8 x double>* %vec2p, <8 x double> %vec3, <8 x i64> %mask) {
-; GENERIC-LABEL: test_8xdouble_masked_shuff_mem_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %zmm2, %zmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshuff64x2 {{.*#+}} zmm1 {%k1} = zmm0[0,1,2,3],mem[0,1,4,5] sched: [8:1.00]
-; GENERIC-NEXT:    vmovapd %zmm1, %zmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xdouble_masked_shuff_mem_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %zmm2, %zmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshuff64x2 {{.*#+}} zmm1 {%k1} = zmm0[0,1,2,3],mem[0,1,4,5] sched: [10:1.00]
-; SKX-NEXT:    vmovapd %zmm1, %zmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <8 x double>, <8 x double>* %vec2p
-  %shuf = shufflevector <8 x double> %vec1, <8 x double> %vec2, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 12, i32 13>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x double> %shuf, <8 x double> %vec3
-  ret <8 x double> %res
-}
-
-define <8 x double> @test_8xdouble_zero_masked_shuff_mem_mask2(<8 x double> %vec1, <8 x double>* %vec2p, <8 x i64> %mask) {
-; GENERIC-LABEL: test_8xdouble_zero_masked_shuff_mem_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshuff64x2 {{.*#+}} zmm0 {%k1} {z} = zmm0[0,1,2,3],mem[0,1,4,5] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xdouble_zero_masked_shuff_mem_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshuff64x2 {{.*#+}} zmm0 {%k1} {z} = zmm0[0,1,2,3],mem[0,1,4,5] sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <8 x double>, <8 x double>* %vec2p
-  %shuf = shufflevector <8 x double> %vec1, <8 x double> %vec2, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 12, i32 13>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x double> %shuf, <8 x double> zeroinitializer
-  ret <8 x double> %res
-}
-
-define <8 x double> @test_8xdouble_shuff_mem_mask3(<8 x double> %vec1, <8 x double>* %vec2p) {
-; GENERIC-LABEL: test_8xdouble_shuff_mem_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vshuff64x2 {{.*#+}} zmm0 = zmm0[2,3,0,1],mem[4,5,0,1] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xdouble_shuff_mem_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vshuff64x2 {{.*#+}} zmm0 = zmm0[2,3,0,1],mem[4,5,0,1] sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <8 x double>, <8 x double>* %vec2p
-  %res = shufflevector <8 x double> %vec1, <8 x double> %vec2, <8 x i32> <i32 2, i32 3, i32 0, i32 1, i32 12, i32 13, i32 8, i32 9>
-  ret <8 x double> %res
-}
-define <8 x double> @test_8xdouble_masked_shuff_mem_mask3(<8 x double> %vec1, <8 x double>* %vec2p, <8 x double> %vec3, <8 x i64> %mask) {
-; GENERIC-LABEL: test_8xdouble_masked_shuff_mem_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %zmm2, %zmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshuff64x2 {{.*#+}} zmm1 {%k1} = zmm0[2,3,0,1],mem[4,5,0,1] sched: [8:1.00]
-; GENERIC-NEXT:    vmovapd %zmm1, %zmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xdouble_masked_shuff_mem_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %zmm2, %zmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshuff64x2 {{.*#+}} zmm1 {%k1} = zmm0[2,3,0,1],mem[4,5,0,1] sched: [10:1.00]
-; SKX-NEXT:    vmovapd %zmm1, %zmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <8 x double>, <8 x double>* %vec2p
-  %shuf = shufflevector <8 x double> %vec1, <8 x double> %vec2, <8 x i32> <i32 2, i32 3, i32 0, i32 1, i32 12, i32 13, i32 8, i32 9>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x double> %shuf, <8 x double> %vec3
-  ret <8 x double> %res
-}
-
-define <8 x double> @test_8xdouble_zero_masked_shuff_mem_mask3(<8 x double> %vec1, <8 x double>* %vec2p, <8 x i64> %mask) {
-; GENERIC-LABEL: test_8xdouble_zero_masked_shuff_mem_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshuff64x2 {{.*#+}} zmm0 {%k1} {z} = zmm0[2,3,0,1],mem[4,5,0,1] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xdouble_zero_masked_shuff_mem_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshuff64x2 {{.*#+}} zmm0 {%k1} {z} = zmm0[2,3,0,1],mem[4,5,0,1] sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <8 x double>, <8 x double>* %vec2p
-  %shuf = shufflevector <8 x double> %vec1, <8 x double> %vec2, <8 x i32> <i32 2, i32 3, i32 0, i32 1, i32 12, i32 13, i32 8, i32 9>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x double> %shuf, <8 x double> zeroinitializer
-  ret <8 x double> %res
-}
-
-define <8 x i32> @test_8xi32_shuff_mask0(<8 x i32> %vec1, <8 x i32> %vec2) {
-; GENERIC-LABEL: test_8xi32_shuff_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vperm2i128 {{.*#+}} ymm0 = ymm0[2,3],ymm1[2,3] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xi32_shuff_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vperm2i128 {{.*#+}} ymm0 = ymm0[2,3],ymm1[2,3] sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %res = shufflevector <8 x i32> %vec1, <8 x i32> %vec2, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 12, i32 13, i32 14, i32 15>
-  ret <8 x i32> %res
-}
-define <8 x i32> @test_8xi32_masked_shuff_mask0(<8 x i32> %vec1, <8 x i32> %vec2, <8 x i32> %vec3, <8 x i32> %mask) {
-; GENERIC-LABEL: test_8xi32_masked_shuff_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %ymm3, %ymm3, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshufi32x4 {{.*#+}} ymm2 {%k1} = ymm0[4,5,6,7],ymm1[4,5,6,7] sched: [1:1.00]
-; GENERIC-NEXT:    vmovdqa %ymm2, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xi32_masked_shuff_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %ymm3, %ymm3, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshufi32x4 {{.*#+}} ymm2 {%k1} = ymm0[4,5,6,7],ymm1[4,5,6,7] sched: [3:1.00]
-; SKX-NEXT:    vmovdqa %ymm2, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x i32> %vec1, <8 x i32> %vec2, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 12, i32 13, i32 14, i32 15>
-  %cmp = icmp eq <8 x i32> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i32> %shuf, <8 x i32> %vec3
-  ret <8 x i32> %res
-}
-
-define <8 x i32> @test_8xi32_zero_masked_shuff_mask0(<8 x i32> %vec1, <8 x i32> %vec2, <8 x i32> %mask) {
-; GENERIC-LABEL: test_8xi32_zero_masked_shuff_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %ymm2, %ymm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshufi32x4 {{.*#+}} ymm0 {%k1} {z} = ymm0[4,5,6,7],ymm1[4,5,6,7] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xi32_zero_masked_shuff_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %ymm2, %ymm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshufi32x4 {{.*#+}} ymm0 {%k1} {z} = ymm0[4,5,6,7],ymm1[4,5,6,7] sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x i32> %vec1, <8 x i32> %vec2, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 12, i32 13, i32 14, i32 15>
-  %cmp = icmp eq <8 x i32> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i32> %shuf, <8 x i32> zeroinitializer
-  ret <8 x i32> %res
-}
-define <8 x i32> @test_8xi32_masked_shuff_mask1(<8 x i32> %vec1, <8 x i32> %vec2, <8 x i32> %vec3, <8 x i32> %mask) {
-; GENERIC-LABEL: test_8xi32_masked_shuff_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %ymm3, %ymm3, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshufi32x4 {{.*#+}} ymm2 {%k1} = ymm0[4,5,6,7],ymm1[0,1,2,3] sched: [1:1.00]
-; GENERIC-NEXT:    vmovdqa %ymm2, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xi32_masked_shuff_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %ymm3, %ymm3, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshufi32x4 {{.*#+}} ymm2 {%k1} = ymm0[4,5,6,7],ymm1[0,1,2,3] sched: [3:1.00]
-; SKX-NEXT:    vmovdqa %ymm2, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x i32> %vec1, <8 x i32> %vec2, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
-  %cmp = icmp eq <8 x i32> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i32> %shuf, <8 x i32> %vec3
-  ret <8 x i32> %res
-}
-
-define <8 x i32> @test_8xi32_zero_masked_shuff_mask1(<8 x i32> %vec1, <8 x i32> %vec2, <8 x i32> %mask) {
-; GENERIC-LABEL: test_8xi32_zero_masked_shuff_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %ymm2, %ymm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshufi32x4 {{.*#+}} ymm0 {%k1} {z} = ymm0[4,5,6,7],ymm1[0,1,2,3] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xi32_zero_masked_shuff_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %ymm2, %ymm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshufi32x4 {{.*#+}} ymm0 {%k1} {z} = ymm0[4,5,6,7],ymm1[0,1,2,3] sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x i32> %vec1, <8 x i32> %vec2, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
-  %cmp = icmp eq <8 x i32> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i32> %shuf, <8 x i32> zeroinitializer
-  ret <8 x i32> %res
-}
-define <8 x i32> @test_8xi32_masked_shuff_mask2(<8 x i32> %vec1, <8 x i32> %vec2, <8 x i32> %vec3, <8 x i32> %mask) {
-; GENERIC-LABEL: test_8xi32_masked_shuff_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %ymm3, %ymm3, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshufi32x4 {{.*#+}} ymm2 {%k1} = ymm0[4,5,6,7],ymm1[4,5,6,7] sched: [1:1.00]
-; GENERIC-NEXT:    vmovdqa %ymm2, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xi32_masked_shuff_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %ymm3, %ymm3, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshufi32x4 {{.*#+}} ymm2 {%k1} = ymm0[4,5,6,7],ymm1[4,5,6,7] sched: [3:1.00]
-; SKX-NEXT:    vmovdqa %ymm2, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x i32> %vec1, <8 x i32> %vec2, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 12, i32 13, i32 14, i32 15>
-  %cmp = icmp eq <8 x i32> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i32> %shuf, <8 x i32> %vec3
-  ret <8 x i32> %res
-}
-
-define <8 x i32> @test_8xi32_zero_masked_shuff_mask2(<8 x i32> %vec1, <8 x i32> %vec2, <8 x i32> %mask) {
-; GENERIC-LABEL: test_8xi32_zero_masked_shuff_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %ymm2, %ymm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshufi32x4 {{.*#+}} ymm0 {%k1} {z} = ymm0[4,5,6,7],ymm1[4,5,6,7] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xi32_zero_masked_shuff_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %ymm2, %ymm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshufi32x4 {{.*#+}} ymm0 {%k1} {z} = ymm0[4,5,6,7],ymm1[4,5,6,7] sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x i32> %vec1, <8 x i32> %vec2, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 12, i32 13, i32 14, i32 15>
-  %cmp = icmp eq <8 x i32> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i32> %shuf, <8 x i32> zeroinitializer
-  ret <8 x i32> %res
-}
-define <8 x i32> @test_8xi32_shuff_mask3(<8 x i32> %vec1, <8 x i32> %vec2) {
-; GENERIC-LABEL: test_8xi32_shuff_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vperm2i128 {{.*#+}} ymm0 = ymm0[2,3],ymm1[0,1] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xi32_shuff_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vperm2i128 {{.*#+}} ymm0 = ymm0[2,3],ymm1[0,1] sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %res = shufflevector <8 x i32> %vec1, <8 x i32> %vec2, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
-  ret <8 x i32> %res
-}
-define <8 x i32> @test_8xi32_masked_shuff_mask3(<8 x i32> %vec1, <8 x i32> %vec2, <8 x i32> %vec3, <8 x i32> %mask) {
-; GENERIC-LABEL: test_8xi32_masked_shuff_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %ymm3, %ymm3, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshufi32x4 {{.*#+}} ymm2 {%k1} = ymm0[4,5,6,7],ymm1[0,1,2,3] sched: [1:1.00]
-; GENERIC-NEXT:    vmovdqa %ymm2, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xi32_masked_shuff_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %ymm3, %ymm3, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshufi32x4 {{.*#+}} ymm2 {%k1} = ymm0[4,5,6,7],ymm1[0,1,2,3] sched: [3:1.00]
-; SKX-NEXT:    vmovdqa %ymm2, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x i32> %vec1, <8 x i32> %vec2, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
-  %cmp = icmp eq <8 x i32> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i32> %shuf, <8 x i32> %vec3
-  ret <8 x i32> %res
-}
-
-define <8 x i32> @test_8xi32_zero_masked_shuff_mask3(<8 x i32> %vec1, <8 x i32> %vec2, <8 x i32> %mask) {
-; GENERIC-LABEL: test_8xi32_zero_masked_shuff_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %ymm2, %ymm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshufi32x4 {{.*#+}} ymm0 {%k1} {z} = ymm0[4,5,6,7],ymm1[0,1,2,3] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xi32_zero_masked_shuff_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %ymm2, %ymm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshufi32x4 {{.*#+}} ymm0 {%k1} {z} = ymm0[4,5,6,7],ymm1[0,1,2,3] sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x i32> %vec1, <8 x i32> %vec2, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
-  %cmp = icmp eq <8 x i32> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i32> %shuf, <8 x i32> zeroinitializer
-  ret <8 x i32> %res
-}
-define <8 x i32> @test_8xi32_shuff_mem_mask0(<8 x i32> %vec1, <8 x i32>* %vec2p) {
-; GENERIC-LABEL: test_8xi32_shuff_mem_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vperm2i128 {{.*#+}} ymm0 = ymm0[2,3],mem[2,3] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xi32_shuff_mem_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vperm2i128 {{.*#+}} ymm0 = ymm0[2,3],mem[2,3] sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <8 x i32>, <8 x i32>* %vec2p
-  %res = shufflevector <8 x i32> %vec1, <8 x i32> %vec2, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 12, i32 13, i32 14, i32 15>
-  ret <8 x i32> %res
-}
-define <8 x i32> @test_8xi32_masked_shuff_mem_mask0(<8 x i32> %vec1, <8 x i32>* %vec2p, <8 x i32> %vec3, <8 x i32> %mask) {
-; GENERIC-LABEL: test_8xi32_masked_shuff_mem_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %ymm2, %ymm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshufi32x4 {{.*#+}} ymm1 {%k1} = ymm0[4,5,6,7],mem[4,5,6,7] sched: [8:1.00]
-; GENERIC-NEXT:    vmovdqa %ymm1, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xi32_masked_shuff_mem_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %ymm2, %ymm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshufi32x4 {{.*#+}} ymm1 {%k1} = ymm0[4,5,6,7],mem[4,5,6,7] sched: [10:1.00]
-; SKX-NEXT:    vmovdqa %ymm1, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <8 x i32>, <8 x i32>* %vec2p
-  %shuf = shufflevector <8 x i32> %vec1, <8 x i32> %vec2, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 12, i32 13, i32 14, i32 15>
-  %cmp = icmp eq <8 x i32> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i32> %shuf, <8 x i32> %vec3
-  ret <8 x i32> %res
-}
-
-define <8 x i32> @test_8xi32_zero_masked_shuff_mem_mask0(<8 x i32> %vec1, <8 x i32>* %vec2p, <8 x i32> %mask) {
-; GENERIC-LABEL: test_8xi32_zero_masked_shuff_mem_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %ymm1, %ymm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshufi32x4 {{.*#+}} ymm0 {%k1} {z} = ymm0[4,5,6,7],mem[4,5,6,7] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xi32_zero_masked_shuff_mem_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %ymm1, %ymm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshufi32x4 {{.*#+}} ymm0 {%k1} {z} = ymm0[4,5,6,7],mem[4,5,6,7] sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <8 x i32>, <8 x i32>* %vec2p
-  %shuf = shufflevector <8 x i32> %vec1, <8 x i32> %vec2, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 12, i32 13, i32 14, i32 15>
-  %cmp = icmp eq <8 x i32> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i32> %shuf, <8 x i32> zeroinitializer
-  ret <8 x i32> %res
-}
-
-define <8 x i32> @test_8xi32_masked_shuff_mem_mask1(<8 x i32> %vec1, <8 x i32>* %vec2p, <8 x i32> %vec3, <8 x i32> %mask) {
-; GENERIC-LABEL: test_8xi32_masked_shuff_mem_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %ymm2, %ymm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshufi32x4 {{.*#+}} ymm1 {%k1} = ymm0[4,5,6,7],mem[0,1,2,3] sched: [8:1.00]
-; GENERIC-NEXT:    vmovdqa %ymm1, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xi32_masked_shuff_mem_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %ymm2, %ymm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshufi32x4 {{.*#+}} ymm1 {%k1} = ymm0[4,5,6,7],mem[0,1,2,3] sched: [10:1.00]
-; SKX-NEXT:    vmovdqa %ymm1, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <8 x i32>, <8 x i32>* %vec2p
-  %shuf = shufflevector <8 x i32> %vec1, <8 x i32> %vec2, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
-  %cmp = icmp eq <8 x i32> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i32> %shuf, <8 x i32> %vec3
-  ret <8 x i32> %res
-}
-
-define <8 x i32> @test_8xi32_zero_masked_shuff_mem_mask1(<8 x i32> %vec1, <8 x i32>* %vec2p, <8 x i32> %mask) {
-; GENERIC-LABEL: test_8xi32_zero_masked_shuff_mem_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %ymm1, %ymm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshufi32x4 {{.*#+}} ymm0 {%k1} {z} = ymm0[4,5,6,7],mem[0,1,2,3] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xi32_zero_masked_shuff_mem_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %ymm1, %ymm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshufi32x4 {{.*#+}} ymm0 {%k1} {z} = ymm0[4,5,6,7],mem[0,1,2,3] sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <8 x i32>, <8 x i32>* %vec2p
-  %shuf = shufflevector <8 x i32> %vec1, <8 x i32> %vec2, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
-  %cmp = icmp eq <8 x i32> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i32> %shuf, <8 x i32> zeroinitializer
-  ret <8 x i32> %res
-}
-
-define <8 x i32> @test_8xi32_masked_shuff_mem_mask2(<8 x i32> %vec1, <8 x i32>* %vec2p, <8 x i32> %vec3, <8 x i32> %mask) {
-; GENERIC-LABEL: test_8xi32_masked_shuff_mem_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %ymm2, %ymm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshufi32x4 {{.*#+}} ymm1 {%k1} = ymm0[4,5,6,7],mem[0,1,2,3] sched: [8:1.00]
-; GENERIC-NEXT:    vmovdqa %ymm1, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xi32_masked_shuff_mem_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %ymm2, %ymm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshufi32x4 {{.*#+}} ymm1 {%k1} = ymm0[4,5,6,7],mem[0,1,2,3] sched: [10:1.00]
-; SKX-NEXT:    vmovdqa %ymm1, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <8 x i32>, <8 x i32>* %vec2p
-  %shuf = shufflevector <8 x i32> %vec1, <8 x i32> %vec2, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
-  %cmp = icmp eq <8 x i32> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i32> %shuf, <8 x i32> %vec3
-  ret <8 x i32> %res
-}
-
-define <8 x i32> @test_8xi32_zero_masked_shuff_mem_mask2(<8 x i32> %vec1, <8 x i32>* %vec2p, <8 x i32> %mask) {
-; GENERIC-LABEL: test_8xi32_zero_masked_shuff_mem_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %ymm1, %ymm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshufi32x4 {{.*#+}} ymm0 {%k1} {z} = ymm0[4,5,6,7],mem[0,1,2,3] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xi32_zero_masked_shuff_mem_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %ymm1, %ymm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshufi32x4 {{.*#+}} ymm0 {%k1} {z} = ymm0[4,5,6,7],mem[0,1,2,3] sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <8 x i32>, <8 x i32>* %vec2p
-  %shuf = shufflevector <8 x i32> %vec1, <8 x i32> %vec2, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
-  %cmp = icmp eq <8 x i32> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i32> %shuf, <8 x i32> zeroinitializer
-  ret <8 x i32> %res
-}
-
-define <8 x i32> @test_8xi32_shuff_mem_mask3(<8 x i32> %vec1, <8 x i32>* %vec2p) {
-; GENERIC-LABEL: test_8xi32_shuff_mem_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vperm2i128 {{.*#+}} ymm0 = ymm0[2,3],mem[0,1] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xi32_shuff_mem_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vperm2i128 {{.*#+}} ymm0 = ymm0[2,3],mem[0,1] sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <8 x i32>, <8 x i32>* %vec2p
-  %res = shufflevector <8 x i32> %vec1, <8 x i32> %vec2, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
-  ret <8 x i32> %res
-}
-define <8 x i32> @test_8xi32_masked_shuff_mem_mask3(<8 x i32> %vec1, <8 x i32>* %vec2p, <8 x i32> %vec3, <8 x i32> %mask) {
-; GENERIC-LABEL: test_8xi32_masked_shuff_mem_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %ymm2, %ymm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshufi32x4 {{.*#+}} ymm1 {%k1} = ymm0[4,5,6,7],mem[0,1,2,3] sched: [8:1.00]
-; GENERIC-NEXT:    vmovdqa %ymm1, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xi32_masked_shuff_mem_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %ymm2, %ymm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshufi32x4 {{.*#+}} ymm1 {%k1} = ymm0[4,5,6,7],mem[0,1,2,3] sched: [10:1.00]
-; SKX-NEXT:    vmovdqa %ymm1, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <8 x i32>, <8 x i32>* %vec2p
-  %shuf = shufflevector <8 x i32> %vec1, <8 x i32> %vec2, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
-  %cmp = icmp eq <8 x i32> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i32> %shuf, <8 x i32> %vec3
-  ret <8 x i32> %res
-}
-
-define <8 x i32> @test_8xi32_zero_masked_shuff_mem_mask3(<8 x i32> %vec1, <8 x i32>* %vec2p, <8 x i32> %mask) {
-; GENERIC-LABEL: test_8xi32_zero_masked_shuff_mem_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %ymm1, %ymm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshufi32x4 {{.*#+}} ymm0 {%k1} {z} = ymm0[4,5,6,7],mem[0,1,2,3] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xi32_zero_masked_shuff_mem_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %ymm1, %ymm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshufi32x4 {{.*#+}} ymm0 {%k1} {z} = ymm0[4,5,6,7],mem[0,1,2,3] sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <8 x i32>, <8 x i32>* %vec2p
-  %shuf = shufflevector <8 x i32> %vec1, <8 x i32> %vec2, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>
-  %cmp = icmp eq <8 x i32> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i32> %shuf, <8 x i32> zeroinitializer
-  ret <8 x i32> %res
-}
-
-define <16 x i32> @test_16xi32_shuff_mask0(<16 x i32> %vec1, <16 x i32> %vec2) {
-; GENERIC-LABEL: test_16xi32_shuff_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm0[2,3,2,3],zmm1[2,3,6,7] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_16xi32_shuff_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm0[2,3,2,3],zmm1[2,3,6,7] sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %res = shufflevector <16 x i32> %vec1, <16 x i32> %vec2, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 4, i32 5, i32 6, i32 7, i32 20, i32 21, i32 22, i32 23, i32 28, i32 29, i32 30, i32 31>
-  ret <16 x i32> %res
-}
-define <16 x i32> @test_16xi32_masked_shuff_mask0(<16 x i32> %vec1, <16 x i32> %vec2, <16 x i32> %vec3, <16 x i32> %mask) {
-; GENERIC-LABEL: test_16xi32_masked_shuff_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %zmm3, %zmm3, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshufi32x4 {{.*#+}} zmm2 {%k1} = zmm0[4,5,6,7,4,5,6,7],zmm1[4,5,6,7,12,13,14,15] sched: [1:1.00]
-; GENERIC-NEXT:    vmovdqa64 %zmm2, %zmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_16xi32_masked_shuff_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %zmm3, %zmm3, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshufi32x4 {{.*#+}} zmm2 {%k1} = zmm0[4,5,6,7,4,5,6,7],zmm1[4,5,6,7,12,13,14,15] sched: [3:1.00]
-; SKX-NEXT:    vmovdqa64 %zmm2, %zmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <16 x i32> %vec1, <16 x i32> %vec2, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 4, i32 5, i32 6, i32 7, i32 20, i32 21, i32 22, i32 23, i32 28, i32 29, i32 30, i32 31>
-  %cmp = icmp eq <16 x i32> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x i32> %shuf, <16 x i32> %vec3
-  ret <16 x i32> %res
-}
-
-define <16 x i32> @test_16xi32_zero_masked_shuff_mask0(<16 x i32> %vec1, <16 x i32> %vec2, <16 x i32> %mask) {
-; GENERIC-LABEL: test_16xi32_zero_masked_shuff_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %zmm2, %zmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshufi32x4 {{.*#+}} zmm0 {%k1} {z} = zmm0[4,5,6,7,4,5,6,7],zmm1[4,5,6,7,12,13,14,15] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_16xi32_zero_masked_shuff_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %zmm2, %zmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshufi32x4 {{.*#+}} zmm0 {%k1} {z} = zmm0[4,5,6,7,4,5,6,7],zmm1[4,5,6,7,12,13,14,15] sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <16 x i32> %vec1, <16 x i32> %vec2, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 4, i32 5, i32 6, i32 7, i32 20, i32 21, i32 22, i32 23, i32 28, i32 29, i32 30, i32 31>
-  %cmp = icmp eq <16 x i32> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x i32> %shuf, <16 x i32> zeroinitializer
-  ret <16 x i32> %res
-}
-define <16 x i32> @test_16xi32_masked_shuff_mask1(<16 x i32> %vec1, <16 x i32> %vec2, <16 x i32> %vec3, <16 x i32> %mask) {
-; GENERIC-LABEL: test_16xi32_masked_shuff_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %zmm3, %zmm3, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshufi32x4 {{.*#+}} zmm2 {%k1} = zmm0[8,9,10,11,8,9,10,11],zmm1[8,9,10,11,4,5,6,7] sched: [1:1.00]
-; GENERIC-NEXT:    vmovdqa64 %zmm2, %zmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_16xi32_masked_shuff_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %zmm3, %zmm3, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshufi32x4 {{.*#+}} zmm2 {%k1} = zmm0[8,9,10,11,8,9,10,11],zmm1[8,9,10,11,4,5,6,7] sched: [3:1.00]
-; SKX-NEXT:    vmovdqa64 %zmm2, %zmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <16 x i32> %vec1, <16 x i32> %vec2, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 8, i32 9, i32 10, i32 11, i32 24, i32 25, i32 26, i32 27, i32 20, i32 21, i32 22, i32 23>
-  %cmp = icmp eq <16 x i32> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x i32> %shuf, <16 x i32> %vec3
-  ret <16 x i32> %res
-}
-
-define <16 x i32> @test_16xi32_zero_masked_shuff_mask1(<16 x i32> %vec1, <16 x i32> %vec2, <16 x i32> %mask) {
-; GENERIC-LABEL: test_16xi32_zero_masked_shuff_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %zmm2, %zmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshufi32x4 {{.*#+}} zmm0 {%k1} {z} = zmm0[8,9,10,11,8,9,10,11],zmm1[8,9,10,11,4,5,6,7] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_16xi32_zero_masked_shuff_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %zmm2, %zmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshufi32x4 {{.*#+}} zmm0 {%k1} {z} = zmm0[8,9,10,11,8,9,10,11],zmm1[8,9,10,11,4,5,6,7] sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <16 x i32> %vec1, <16 x i32> %vec2, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 8, i32 9, i32 10, i32 11, i32 24, i32 25, i32 26, i32 27, i32 20, i32 21, i32 22, i32 23>
-  %cmp = icmp eq <16 x i32> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x i32> %shuf, <16 x i32> zeroinitializer
-  ret <16 x i32> %res
-}
-define <16 x i32> @test_16xi32_masked_shuff_mask2(<16 x i32> %vec1, <16 x i32> %vec2, <16 x i32> %vec3, <16 x i32> %mask) {
-; GENERIC-LABEL: test_16xi32_masked_shuff_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %zmm3, %zmm3, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshufi32x4 {{.*#+}} zmm2 {%k1} = zmm0[4,5,6,7,8,9,10,11],zmm1[0,1,2,3,0,1,2,3] sched: [1:1.00]
-; GENERIC-NEXT:    vmovdqa64 %zmm2, %zmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_16xi32_masked_shuff_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %zmm3, %zmm3, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshufi32x4 {{.*#+}} zmm2 {%k1} = zmm0[4,5,6,7,8,9,10,11],zmm1[0,1,2,3,0,1,2,3] sched: [3:1.00]
-; SKX-NEXT:    vmovdqa64 %zmm2, %zmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <16 x i32> %vec1, <16 x i32> %vec2, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19, i32 16, i32 17, i32 18, i32 19>
-  %cmp = icmp eq <16 x i32> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x i32> %shuf, <16 x i32> %vec3
-  ret <16 x i32> %res
-}
-
-define <16 x i32> @test_16xi32_zero_masked_shuff_mask2(<16 x i32> %vec1, <16 x i32> %vec2, <16 x i32> %mask) {
-; GENERIC-LABEL: test_16xi32_zero_masked_shuff_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %zmm2, %zmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshufi32x4 {{.*#+}} zmm0 {%k1} {z} = zmm0[4,5,6,7,8,9,10,11],zmm1[0,1,2,3,0,1,2,3] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_16xi32_zero_masked_shuff_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %zmm2, %zmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshufi32x4 {{.*#+}} zmm0 {%k1} {z} = zmm0[4,5,6,7,8,9,10,11],zmm1[0,1,2,3,0,1,2,3] sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <16 x i32> %vec1, <16 x i32> %vec2, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 16, i32 17, i32 18, i32 19, i32 16, i32 17, i32 18, i32 19>
-  %cmp = icmp eq <16 x i32> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x i32> %shuf, <16 x i32> zeroinitializer
-  ret <16 x i32> %res
-}
-define <16 x i32> @test_16xi32_shuff_mask3(<16 x i32> %vec1, <16 x i32> %vec2) {
-; GENERIC-LABEL: test_16xi32_shuff_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm0[2,3,0,1],zmm1[4,5,2,3] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_16xi32_shuff_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm0[2,3,0,1],zmm1[4,5,2,3] sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %res = shufflevector <16 x i32> %vec1, <16 x i32> %vec2, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 24, i32 25, i32 26, i32 27, i32 20, i32 21, i32 22, i32 23>
-  ret <16 x i32> %res
-}
-define <16 x i32> @test_16xi32_masked_shuff_mask3(<16 x i32> %vec1, <16 x i32> %vec2, <16 x i32> %vec3, <16 x i32> %mask) {
-; GENERIC-LABEL: test_16xi32_masked_shuff_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %zmm3, %zmm3, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshufi32x4 {{.*#+}} zmm2 {%k1} = zmm0[4,5,6,7,0,1,2,3],zmm1[8,9,10,11,4,5,6,7] sched: [1:1.00]
-; GENERIC-NEXT:    vmovdqa64 %zmm2, %zmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_16xi32_masked_shuff_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %zmm3, %zmm3, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshufi32x4 {{.*#+}} zmm2 {%k1} = zmm0[4,5,6,7,0,1,2,3],zmm1[8,9,10,11,4,5,6,7] sched: [3:1.00]
-; SKX-NEXT:    vmovdqa64 %zmm2, %zmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <16 x i32> %vec1, <16 x i32> %vec2, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 24, i32 25, i32 26, i32 27, i32 20, i32 21, i32 22, i32 23>
-  %cmp = icmp eq <16 x i32> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x i32> %shuf, <16 x i32> %vec3
-  ret <16 x i32> %res
-}
-
-define <16 x i32> @test_16xi32_zero_masked_shuff_mask3(<16 x i32> %vec1, <16 x i32> %vec2, <16 x i32> %mask) {
-; GENERIC-LABEL: test_16xi32_zero_masked_shuff_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %zmm2, %zmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshufi32x4 {{.*#+}} zmm0 {%k1} {z} = zmm0[4,5,6,7,0,1,2,3],zmm1[8,9,10,11,4,5,6,7] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_16xi32_zero_masked_shuff_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %zmm2, %zmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshufi32x4 {{.*#+}} zmm0 {%k1} {z} = zmm0[4,5,6,7,0,1,2,3],zmm1[8,9,10,11,4,5,6,7] sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <16 x i32> %vec1, <16 x i32> %vec2, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 24, i32 25, i32 26, i32 27, i32 20, i32 21, i32 22, i32 23>
-  %cmp = icmp eq <16 x i32> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x i32> %shuf, <16 x i32> zeroinitializer
-  ret <16 x i32> %res
-}
-define <16 x i32> @test_16xi32_shuff_mem_mask0(<16 x i32> %vec1, <16 x i32>* %vec2p) {
-; GENERIC-LABEL: test_16xi32_shuff_mem_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm0[4,5,2,3],mem[4,5,0,1] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_16xi32_shuff_mem_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm0[4,5,2,3],mem[4,5,0,1] sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <16 x i32>, <16 x i32>* %vec2p
-  %res = shufflevector <16 x i32> %vec1, <16 x i32> %vec2, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7, i32 24, i32 25, i32 26, i32 27, i32 16, i32 17, i32 18, i32 19>
-  ret <16 x i32> %res
-}
-define <16 x i32> @test_16xi32_masked_shuff_mem_mask0(<16 x i32> %vec1, <16 x i32>* %vec2p, <16 x i32> %vec3, <16 x i32> %mask) {
-; GENERIC-LABEL: test_16xi32_masked_shuff_mem_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %zmm2, %zmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshufi32x4 {{.*#+}} zmm1 {%k1} = zmm0[8,9,10,11,4,5,6,7],mem[8,9,10,11,0,1,2,3] sched: [8:1.00]
-; GENERIC-NEXT:    vmovdqa64 %zmm1, %zmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_16xi32_masked_shuff_mem_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %zmm2, %zmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshufi32x4 {{.*#+}} zmm1 {%k1} = zmm0[8,9,10,11,4,5,6,7],mem[8,9,10,11,0,1,2,3] sched: [10:1.00]
-; SKX-NEXT:    vmovdqa64 %zmm1, %zmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <16 x i32>, <16 x i32>* %vec2p
-  %shuf = shufflevector <16 x i32> %vec1, <16 x i32> %vec2, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7, i32 24, i32 25, i32 26, i32 27, i32 16, i32 17, i32 18, i32 19>
-  %cmp = icmp eq <16 x i32> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x i32> %shuf, <16 x i32> %vec3
-  ret <16 x i32> %res
-}
-
-define <16 x i32> @test_16xi32_zero_masked_shuff_mem_mask0(<16 x i32> %vec1, <16 x i32>* %vec2p, <16 x i32> %mask) {
-; GENERIC-LABEL: test_16xi32_zero_masked_shuff_mem_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshufi32x4 {{.*#+}} zmm0 {%k1} {z} = zmm0[8,9,10,11,4,5,6,7],mem[8,9,10,11,0,1,2,3] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_16xi32_zero_masked_shuff_mem_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshufi32x4 {{.*#+}} zmm0 {%k1} {z} = zmm0[8,9,10,11,4,5,6,7],mem[8,9,10,11,0,1,2,3] sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <16 x i32>, <16 x i32>* %vec2p
-  %shuf = shufflevector <16 x i32> %vec1, <16 x i32> %vec2, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7, i32 24, i32 25, i32 26, i32 27, i32 16, i32 17, i32 18, i32 19>
-  %cmp = icmp eq <16 x i32> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x i32> %shuf, <16 x i32> zeroinitializer
-  ret <16 x i32> %res
-}
-
-define <16 x i32> @test_16xi32_masked_shuff_mem_mask1(<16 x i32> %vec1, <16 x i32>* %vec2p, <16 x i32> %vec3, <16 x i32> %mask) {
-; GENERIC-LABEL: test_16xi32_masked_shuff_mem_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %zmm2, %zmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshufi32x4 {{.*#+}} zmm1 {%k1} = zmm0[4,5,6,7,4,5,6,7],mem[0,1,2,3,8,9,10,11] sched: [8:1.00]
-; GENERIC-NEXT:    vmovdqa64 %zmm1, %zmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_16xi32_masked_shuff_mem_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %zmm2, %zmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshufi32x4 {{.*#+}} zmm1 {%k1} = zmm0[4,5,6,7,4,5,6,7],mem[0,1,2,3,8,9,10,11] sched: [10:1.00]
-; SKX-NEXT:    vmovdqa64 %zmm1, %zmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <16 x i32>, <16 x i32>* %vec2p
-  %shuf = shufflevector <16 x i32> %vec1, <16 x i32> %vec2, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 24, i32 25, i32 26, i32 27>
-  %cmp = icmp eq <16 x i32> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x i32> %shuf, <16 x i32> %vec3
-  ret <16 x i32> %res
-}
-
-define <16 x i32> @test_16xi32_zero_masked_shuff_mem_mask1(<16 x i32> %vec1, <16 x i32>* %vec2p, <16 x i32> %mask) {
-; GENERIC-LABEL: test_16xi32_zero_masked_shuff_mem_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshufi32x4 {{.*#+}} zmm0 {%k1} {z} = zmm0[4,5,6,7,4,5,6,7],mem[0,1,2,3,8,9,10,11] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_16xi32_zero_masked_shuff_mem_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshufi32x4 {{.*#+}} zmm0 {%k1} {z} = zmm0[4,5,6,7,4,5,6,7],mem[0,1,2,3,8,9,10,11] sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <16 x i32>, <16 x i32>* %vec2p
-  %shuf = shufflevector <16 x i32> %vec1, <16 x i32> %vec2, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 24, i32 25, i32 26, i32 27>
-  %cmp = icmp eq <16 x i32> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x i32> %shuf, <16 x i32> zeroinitializer
-  ret <16 x i32> %res
-}
-
-define <16 x i32> @test_16xi32_masked_shuff_mem_mask2(<16 x i32> %vec1, <16 x i32>* %vec2p, <16 x i32> %vec3, <16 x i32> %mask) {
-; GENERIC-LABEL: test_16xi32_masked_shuff_mem_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %zmm2, %zmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshufi32x4 {{.*#+}} zmm1 {%k1} = zmm0[4,5,6,7,8,9,10,11],mem[12,13,14,15,12,13,14,15] sched: [8:1.00]
-; GENERIC-NEXT:    vmovdqa64 %zmm1, %zmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_16xi32_masked_shuff_mem_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %zmm2, %zmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshufi32x4 {{.*#+}} zmm1 {%k1} = zmm0[4,5,6,7,8,9,10,11],mem[12,13,14,15,12,13,14,15] sched: [10:1.00]
-; SKX-NEXT:    vmovdqa64 %zmm1, %zmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <16 x i32>, <16 x i32>* %vec2p
-  %shuf = shufflevector <16 x i32> %vec1, <16 x i32> %vec2, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 28, i32 29, i32 30, i32 31, i32 28, i32 29, i32 30, i32 31>
-  %cmp = icmp eq <16 x i32> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x i32> %shuf, <16 x i32> %vec3
-  ret <16 x i32> %res
-}
-
-define <16 x i32> @test_16xi32_zero_masked_shuff_mem_mask2(<16 x i32> %vec1, <16 x i32>* %vec2p, <16 x i32> %mask) {
-; GENERIC-LABEL: test_16xi32_zero_masked_shuff_mem_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshufi32x4 {{.*#+}} zmm0 {%k1} {z} = zmm0[4,5,6,7,8,9,10,11],mem[12,13,14,15,12,13,14,15] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_16xi32_zero_masked_shuff_mem_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshufi32x4 {{.*#+}} zmm0 {%k1} {z} = zmm0[4,5,6,7,8,9,10,11],mem[12,13,14,15,12,13,14,15] sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <16 x i32>, <16 x i32>* %vec2p
-  %shuf = shufflevector <16 x i32> %vec1, <16 x i32> %vec2, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 28, i32 29, i32 30, i32 31, i32 28, i32 29, i32 30, i32 31>
-  %cmp = icmp eq <16 x i32> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x i32> %shuf, <16 x i32> zeroinitializer
-  ret <16 x i32> %res
-}
-
-define <16 x i32> @test_16xi32_shuff_mem_mask3(<16 x i32> %vec1, <16 x i32>* %vec2p) {
-; GENERIC-LABEL: test_16xi32_shuff_mem_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm0[2,3,2,3],mem[2,3,6,7] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_16xi32_shuff_mem_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm0[2,3,2,3],mem[2,3,6,7] sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <16 x i32>, <16 x i32>* %vec2p
-  %res = shufflevector <16 x i32> %vec1, <16 x i32> %vec2, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 4, i32 5, i32 6, i32 7, i32 20, i32 21, i32 22, i32 23, i32 28, i32 29, i32 30, i32 31>
-  ret <16 x i32> %res
-}
-define <16 x i32> @test_16xi32_masked_shuff_mem_mask3(<16 x i32> %vec1, <16 x i32>* %vec2p, <16 x i32> %vec3, <16 x i32> %mask) {
-; GENERIC-LABEL: test_16xi32_masked_shuff_mem_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %zmm2, %zmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshufi32x4 {{.*#+}} zmm1 {%k1} = zmm0[4,5,6,7,4,5,6,7],mem[4,5,6,7,12,13,14,15] sched: [8:1.00]
-; GENERIC-NEXT:    vmovdqa64 %zmm1, %zmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_16xi32_masked_shuff_mem_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %zmm2, %zmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshufi32x4 {{.*#+}} zmm1 {%k1} = zmm0[4,5,6,7,4,5,6,7],mem[4,5,6,7,12,13,14,15] sched: [10:1.00]
-; SKX-NEXT:    vmovdqa64 %zmm1, %zmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <16 x i32>, <16 x i32>* %vec2p
-  %shuf = shufflevector <16 x i32> %vec1, <16 x i32> %vec2, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 4, i32 5, i32 6, i32 7, i32 20, i32 21, i32 22, i32 23, i32 28, i32 29, i32 30, i32 31>
-  %cmp = icmp eq <16 x i32> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x i32> %shuf, <16 x i32> %vec3
-  ret <16 x i32> %res
-}
-
-define <16 x i32> @test_16xi32_zero_masked_shuff_mem_mask3(<16 x i32> %vec1, <16 x i32>* %vec2p, <16 x i32> %mask) {
-; GENERIC-LABEL: test_16xi32_zero_masked_shuff_mem_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshufi32x4 {{.*#+}} zmm0 {%k1} {z} = zmm0[4,5,6,7,4,5,6,7],mem[4,5,6,7,12,13,14,15] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_16xi32_zero_masked_shuff_mem_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshufi32x4 {{.*#+}} zmm0 {%k1} {z} = zmm0[4,5,6,7,4,5,6,7],mem[4,5,6,7,12,13,14,15] sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <16 x i32>, <16 x i32>* %vec2p
-  %shuf = shufflevector <16 x i32> %vec1, <16 x i32> %vec2, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 4, i32 5, i32 6, i32 7, i32 20, i32 21, i32 22, i32 23, i32 28, i32 29, i32 30, i32 31>
-  %cmp = icmp eq <16 x i32> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x i32> %shuf, <16 x i32> zeroinitializer
-  ret <16 x i32> %res
-}
-
-define <4 x i64> @test_4xi64_shuff_mask0(<4 x i64> %vec1, <4 x i64> %vec2) {
-; GENERIC-LABEL: test_4xi64_shuff_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vperm2i128 {{.*#+}} ymm0 = ymm0[2,3],ymm1[0,1] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xi64_shuff_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vperm2i128 {{.*#+}} ymm0 = ymm0[2,3],ymm1[0,1] sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %res = shufflevector <4 x i64> %vec1, <4 x i64> %vec2, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
-  ret <4 x i64> %res
-}
-define <4 x i64> @test_4xi64_masked_shuff_mask0(<4 x i64> %vec1, <4 x i64> %vec2, <4 x i64> %vec3, <4 x i64> %mask) {
-; GENERIC-LABEL: test_4xi64_masked_shuff_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %ymm3, %ymm3, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshufi64x2 {{.*#+}} ymm2 {%k1} = ymm0[2,3],ymm1[0,1] sched: [1:1.00]
-; GENERIC-NEXT:    vmovdqa %ymm2, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xi64_masked_shuff_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %ymm3, %ymm3, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshufi64x2 {{.*#+}} ymm2 {%k1} = ymm0[2,3],ymm1[0,1] sched: [3:1.00]
-; SKX-NEXT:    vmovdqa %ymm2, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <4 x i64> %vec1, <4 x i64> %vec2, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
-  %cmp = icmp eq <4 x i64> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x i64> %shuf, <4 x i64> %vec3
-  ret <4 x i64> %res
-}
-
-define <4 x i64> @test_4xi64_zero_masked_shuff_mask0(<4 x i64> %vec1, <4 x i64> %vec2, <4 x i64> %mask) {
-; GENERIC-LABEL: test_4xi64_zero_masked_shuff_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %ymm2, %ymm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshufi64x2 {{.*#+}} ymm0 {%k1} {z} = ymm0[2,3],ymm1[0,1] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xi64_zero_masked_shuff_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %ymm2, %ymm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshufi64x2 {{.*#+}} ymm0 {%k1} {z} = ymm0[2,3],ymm1[0,1] sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <4 x i64> %vec1, <4 x i64> %vec2, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
-  %cmp = icmp eq <4 x i64> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x i64> %shuf, <4 x i64> zeroinitializer
-  ret <4 x i64> %res
-}
-define <4 x i64> @test_4xi64_masked_shuff_mask1(<4 x i64> %vec1, <4 x i64> %vec2, <4 x i64> %vec3, <4 x i64> %mask) {
-; GENERIC-LABEL: test_4xi64_masked_shuff_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %ymm3, %ymm3, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshufi64x2 {{.*#+}} ymm2 {%k1} = ymm0[2,3],ymm1[2,3] sched: [1:1.00]
-; GENERIC-NEXT:    vmovdqa %ymm2, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xi64_masked_shuff_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %ymm3, %ymm3, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshufi64x2 {{.*#+}} ymm2 {%k1} = ymm0[2,3],ymm1[2,3] sched: [3:1.00]
-; SKX-NEXT:    vmovdqa %ymm2, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <4 x i64> %vec1, <4 x i64> %vec2, <4 x i32> <i32 2, i32 3, i32 6, i32 7>
-  %cmp = icmp eq <4 x i64> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x i64> %shuf, <4 x i64> %vec3
-  ret <4 x i64> %res
-}
-
-define <4 x i64> @test_4xi64_zero_masked_shuff_mask1(<4 x i64> %vec1, <4 x i64> %vec2, <4 x i64> %mask) {
-; GENERIC-LABEL: test_4xi64_zero_masked_shuff_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %ymm2, %ymm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshufi64x2 {{.*#+}} ymm0 {%k1} {z} = ymm0[2,3],ymm1[2,3] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xi64_zero_masked_shuff_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %ymm2, %ymm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshufi64x2 {{.*#+}} ymm0 {%k1} {z} = ymm0[2,3],ymm1[2,3] sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <4 x i64> %vec1, <4 x i64> %vec2, <4 x i32> <i32 2, i32 3, i32 6, i32 7>
-  %cmp = icmp eq <4 x i64> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x i64> %shuf, <4 x i64> zeroinitializer
-  ret <4 x i64> %res
-}
-define <4 x i64> @test_4xi64_masked_shuff_mask2(<4 x i64> %vec1, <4 x i64> %vec2, <4 x i64> %vec3, <4 x i64> %mask) {
-; GENERIC-LABEL: test_4xi64_masked_shuff_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %ymm3, %ymm3, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshufi64x2 {{.*#+}} ymm2 {%k1} = ymm0[2,3],ymm1[0,1] sched: [1:1.00]
-; GENERIC-NEXT:    vmovdqa %ymm2, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xi64_masked_shuff_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %ymm3, %ymm3, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshufi64x2 {{.*#+}} ymm2 {%k1} = ymm0[2,3],ymm1[0,1] sched: [3:1.00]
-; SKX-NEXT:    vmovdqa %ymm2, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <4 x i64> %vec1, <4 x i64> %vec2, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
-  %cmp = icmp eq <4 x i64> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x i64> %shuf, <4 x i64> %vec3
-  ret <4 x i64> %res
-}
-
-define <4 x i64> @test_4xi64_zero_masked_shuff_mask2(<4 x i64> %vec1, <4 x i64> %vec2, <4 x i64> %mask) {
-; GENERIC-LABEL: test_4xi64_zero_masked_shuff_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %ymm2, %ymm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshufi64x2 {{.*#+}} ymm0 {%k1} {z} = ymm0[2,3],ymm1[0,1] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xi64_zero_masked_shuff_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %ymm2, %ymm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshufi64x2 {{.*#+}} ymm0 {%k1} {z} = ymm0[2,3],ymm1[0,1] sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <4 x i64> %vec1, <4 x i64> %vec2, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
-  %cmp = icmp eq <4 x i64> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x i64> %shuf, <4 x i64> zeroinitializer
-  ret <4 x i64> %res
-}
-define <4 x i64> @test_4xi64_shuff_mask3(<4 x i64> %vec1, <4 x i64> %vec2) {
-; GENERIC-LABEL: test_4xi64_shuff_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vperm2i128 {{.*#+}} ymm0 = ymm0[2,3],ymm1[2,3] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xi64_shuff_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vperm2i128 {{.*#+}} ymm0 = ymm0[2,3],ymm1[2,3] sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %res = shufflevector <4 x i64> %vec1, <4 x i64> %vec2, <4 x i32> <i32 2, i32 3, i32 6, i32 7>
-  ret <4 x i64> %res
-}
-define <4 x i64> @test_4xi64_masked_shuff_mask3(<4 x i64> %vec1, <4 x i64> %vec2, <4 x i64> %vec3, <4 x i64> %mask) {
-; GENERIC-LABEL: test_4xi64_masked_shuff_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %ymm3, %ymm3, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshufi64x2 {{.*#+}} ymm2 {%k1} = ymm0[2,3],ymm1[2,3] sched: [1:1.00]
-; GENERIC-NEXT:    vmovdqa %ymm2, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xi64_masked_shuff_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %ymm3, %ymm3, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshufi64x2 {{.*#+}} ymm2 {%k1} = ymm0[2,3],ymm1[2,3] sched: [3:1.00]
-; SKX-NEXT:    vmovdqa %ymm2, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <4 x i64> %vec1, <4 x i64> %vec2, <4 x i32> <i32 2, i32 3, i32 6, i32 7>
-  %cmp = icmp eq <4 x i64> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x i64> %shuf, <4 x i64> %vec3
-  ret <4 x i64> %res
-}
-
-define <4 x i64> @test_4xi64_zero_masked_shuff_mask3(<4 x i64> %vec1, <4 x i64> %vec2, <4 x i64> %mask) {
-; GENERIC-LABEL: test_4xi64_zero_masked_shuff_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %ymm2, %ymm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshufi64x2 {{.*#+}} ymm0 {%k1} {z} = ymm0[2,3],ymm1[2,3] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xi64_zero_masked_shuff_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %ymm2, %ymm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshufi64x2 {{.*#+}} ymm0 {%k1} {z} = ymm0[2,3],ymm1[2,3] sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <4 x i64> %vec1, <4 x i64> %vec2, <4 x i32> <i32 2, i32 3, i32 6, i32 7>
-  %cmp = icmp eq <4 x i64> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x i64> %shuf, <4 x i64> zeroinitializer
-  ret <4 x i64> %res
-}
-define <4 x i64> @test_4xi64_shuff_mem_mask0(<4 x i64> %vec1, <4 x i64>* %vec2p) {
-; GENERIC-LABEL: test_4xi64_shuff_mem_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vperm2i128 {{.*#+}} ymm0 = ymm0[2,3],mem[2,3] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xi64_shuff_mem_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vperm2i128 {{.*#+}} ymm0 = ymm0[2,3],mem[2,3] sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <4 x i64>, <4 x i64>* %vec2p
-  %res = shufflevector <4 x i64> %vec1, <4 x i64> %vec2, <4 x i32> <i32 2, i32 3, i32 6, i32 7>
-  ret <4 x i64> %res
-}
-define <4 x i64> @test_4xi64_masked_shuff_mem_mask0(<4 x i64> %vec1, <4 x i64>* %vec2p, <4 x i64> %vec3, <4 x i64> %mask) {
-; GENERIC-LABEL: test_4xi64_masked_shuff_mem_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %ymm2, %ymm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshufi64x2 {{.*#+}} ymm1 {%k1} = ymm0[2,3],mem[2,3] sched: [8:1.00]
-; GENERIC-NEXT:    vmovdqa %ymm1, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xi64_masked_shuff_mem_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %ymm2, %ymm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshufi64x2 {{.*#+}} ymm1 {%k1} = ymm0[2,3],mem[2,3] sched: [10:1.00]
-; SKX-NEXT:    vmovdqa %ymm1, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <4 x i64>, <4 x i64>* %vec2p
-  %shuf = shufflevector <4 x i64> %vec1, <4 x i64> %vec2, <4 x i32> <i32 2, i32 3, i32 6, i32 7>
-  %cmp = icmp eq <4 x i64> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x i64> %shuf, <4 x i64> %vec3
-  ret <4 x i64> %res
-}
-
-define <4 x i64> @test_4xi64_zero_masked_shuff_mem_mask0(<4 x i64> %vec1, <4 x i64>* %vec2p, <4 x i64> %mask) {
-; GENERIC-LABEL: test_4xi64_zero_masked_shuff_mem_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %ymm1, %ymm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshufi64x2 {{.*#+}} ymm0 {%k1} {z} = ymm0[2,3],mem[2,3] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xi64_zero_masked_shuff_mem_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %ymm1, %ymm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshufi64x2 {{.*#+}} ymm0 {%k1} {z} = ymm0[2,3],mem[2,3] sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <4 x i64>, <4 x i64>* %vec2p
-  %shuf = shufflevector <4 x i64> %vec1, <4 x i64> %vec2, <4 x i32> <i32 2, i32 3, i32 6, i32 7>
-  %cmp = icmp eq <4 x i64> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x i64> %shuf, <4 x i64> zeroinitializer
-  ret <4 x i64> %res
-}
-
-define <4 x i64> @test_4xi64_masked_shuff_mem_mask1(<4 x i64> %vec1, <4 x i64>* %vec2p, <4 x i64> %vec3, <4 x i64> %mask) {
-; GENERIC-LABEL: test_4xi64_masked_shuff_mem_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %ymm2, %ymm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshufi64x2 {{.*#+}} ymm1 {%k1} = ymm0[2,3],mem[0,1] sched: [8:1.00]
-; GENERIC-NEXT:    vmovdqa %ymm1, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xi64_masked_shuff_mem_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %ymm2, %ymm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshufi64x2 {{.*#+}} ymm1 {%k1} = ymm0[2,3],mem[0,1] sched: [10:1.00]
-; SKX-NEXT:    vmovdqa %ymm1, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <4 x i64>, <4 x i64>* %vec2p
-  %shuf = shufflevector <4 x i64> %vec1, <4 x i64> %vec2, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
-  %cmp = icmp eq <4 x i64> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x i64> %shuf, <4 x i64> %vec3
-  ret <4 x i64> %res
-}
-
-define <4 x i64> @test_4xi64_zero_masked_shuff_mem_mask1(<4 x i64> %vec1, <4 x i64>* %vec2p, <4 x i64> %mask) {
-; GENERIC-LABEL: test_4xi64_zero_masked_shuff_mem_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %ymm1, %ymm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshufi64x2 {{.*#+}} ymm0 {%k1} {z} = ymm0[2,3],mem[0,1] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xi64_zero_masked_shuff_mem_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %ymm1, %ymm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshufi64x2 {{.*#+}} ymm0 {%k1} {z} = ymm0[2,3],mem[0,1] sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <4 x i64>, <4 x i64>* %vec2p
-  %shuf = shufflevector <4 x i64> %vec1, <4 x i64> %vec2, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
-  %cmp = icmp eq <4 x i64> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x i64> %shuf, <4 x i64> zeroinitializer
-  ret <4 x i64> %res
-}
-
-define <4 x i64> @test_4xi64_masked_shuff_mem_mask2(<4 x i64> %vec1, <4 x i64>* %vec2p, <4 x i64> %vec3, <4 x i64> %mask) {
-; GENERIC-LABEL: test_4xi64_masked_shuff_mem_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %ymm2, %ymm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshufi64x2 {{.*#+}} ymm1 {%k1} = ymm0[2,3],mem[0,1] sched: [8:1.00]
-; GENERIC-NEXT:    vmovdqa %ymm1, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xi64_masked_shuff_mem_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %ymm2, %ymm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshufi64x2 {{.*#+}} ymm1 {%k1} = ymm0[2,3],mem[0,1] sched: [10:1.00]
-; SKX-NEXT:    vmovdqa %ymm1, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <4 x i64>, <4 x i64>* %vec2p
-  %shuf = shufflevector <4 x i64> %vec1, <4 x i64> %vec2, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
-  %cmp = icmp eq <4 x i64> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x i64> %shuf, <4 x i64> %vec3
-  ret <4 x i64> %res
-}
-
-define <4 x i64> @test_4xi64_zero_masked_shuff_mem_mask2(<4 x i64> %vec1, <4 x i64>* %vec2p, <4 x i64> %mask) {
-; GENERIC-LABEL: test_4xi64_zero_masked_shuff_mem_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %ymm1, %ymm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshufi64x2 {{.*#+}} ymm0 {%k1} {z} = ymm0[2,3],mem[0,1] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xi64_zero_masked_shuff_mem_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %ymm1, %ymm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshufi64x2 {{.*#+}} ymm0 {%k1} {z} = ymm0[2,3],mem[0,1] sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <4 x i64>, <4 x i64>* %vec2p
-  %shuf = shufflevector <4 x i64> %vec1, <4 x i64> %vec2, <4 x i32> <i32 2, i32 3, i32 4, i32 5>
-  %cmp = icmp eq <4 x i64> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x i64> %shuf, <4 x i64> zeroinitializer
-  ret <4 x i64> %res
-}
-
-define <4 x i64> @test_4xi64_shuff_mem_mask3(<4 x i64> %vec1, <4 x i64>* %vec2p) {
-; GENERIC-LABEL: test_4xi64_shuff_mem_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vperm2i128 {{.*#+}} ymm0 = ymm0[2,3],mem[2,3] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xi64_shuff_mem_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vperm2i128 {{.*#+}} ymm0 = ymm0[2,3],mem[2,3] sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <4 x i64>, <4 x i64>* %vec2p
-  %res = shufflevector <4 x i64> %vec1, <4 x i64> %vec2, <4 x i32> <i32 2, i32 3, i32 6, i32 7>
-  ret <4 x i64> %res
-}
-define <4 x i64> @test_4xi64_masked_shuff_mem_mask3(<4 x i64> %vec1, <4 x i64>* %vec2p, <4 x i64> %vec3, <4 x i64> %mask) {
-; GENERIC-LABEL: test_4xi64_masked_shuff_mem_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %ymm2, %ymm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshufi64x2 {{.*#+}} ymm1 {%k1} = ymm0[2,3],mem[2,3] sched: [8:1.00]
-; GENERIC-NEXT:    vmovdqa %ymm1, %ymm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xi64_masked_shuff_mem_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %ymm2, %ymm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshufi64x2 {{.*#+}} ymm1 {%k1} = ymm0[2,3],mem[2,3] sched: [10:1.00]
-; SKX-NEXT:    vmovdqa %ymm1, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <4 x i64>, <4 x i64>* %vec2p
-  %shuf = shufflevector <4 x i64> %vec1, <4 x i64> %vec2, <4 x i32> <i32 2, i32 3, i32 6, i32 7>
-  %cmp = icmp eq <4 x i64> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x i64> %shuf, <4 x i64> %vec3
-  ret <4 x i64> %res
-}
-
-define <4 x i64> @test_4xi64_zero_masked_shuff_mem_mask3(<4 x i64> %vec1, <4 x i64>* %vec2p, <4 x i64> %mask) {
-; GENERIC-LABEL: test_4xi64_zero_masked_shuff_mem_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %ymm1, %ymm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshufi64x2 {{.*#+}} ymm0 {%k1} {z} = ymm0[2,3],mem[2,3] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xi64_zero_masked_shuff_mem_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %ymm1, %ymm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshufi64x2 {{.*#+}} ymm0 {%k1} {z} = ymm0[2,3],mem[2,3] sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <4 x i64>, <4 x i64>* %vec2p
-  %shuf = shufflevector <4 x i64> %vec1, <4 x i64> %vec2, <4 x i32> <i32 2, i32 3, i32 6, i32 7>
-  %cmp = icmp eq <4 x i64> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x i64> %shuf, <4 x i64> zeroinitializer
-  ret <4 x i64> %res
-}
-
-define <8 x i64> @test_8xi64_shuff_mask0(<8 x i64> %vec1, <8 x i64> %vec2) {
-; GENERIC-LABEL: test_8xi64_shuff_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm0[4,5,4,5],zmm1[4,5,4,5] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xi64_shuff_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm0[4,5,4,5],zmm1[4,5,4,5] sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %res = shufflevector <8 x i64> %vec1, <8 x i64> %vec2, <8 x i32> <i32 4, i32 5, i32 4, i32 5, i32 12, i32 13, i32 12, i32 13>
-  ret <8 x i64> %res
-}
-define <8 x i64> @test_8xi64_masked_shuff_mask0(<8 x i64> %vec1, <8 x i64> %vec2, <8 x i64> %vec3, <8 x i64> %mask) {
-; GENERIC-LABEL: test_8xi64_masked_shuff_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %zmm3, %zmm3, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshufi64x2 {{.*#+}} zmm2 {%k1} = zmm0[4,5,4,5],zmm1[4,5,4,5] sched: [1:1.00]
-; GENERIC-NEXT:    vmovdqa64 %zmm2, %zmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xi64_masked_shuff_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %zmm3, %zmm3, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshufi64x2 {{.*#+}} zmm2 {%k1} = zmm0[4,5,4,5],zmm1[4,5,4,5] sched: [3:1.00]
-; SKX-NEXT:    vmovdqa64 %zmm2, %zmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x i64> %vec1, <8 x i64> %vec2, <8 x i32> <i32 4, i32 5, i32 4, i32 5, i32 12, i32 13, i32 12, i32 13>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i64> %shuf, <8 x i64> %vec3
-  ret <8 x i64> %res
-}
-
-define <8 x i64> @test_8xi64_zero_masked_shuff_mask0(<8 x i64> %vec1, <8 x i64> %vec2, <8 x i64> %mask) {
-; GENERIC-LABEL: test_8xi64_zero_masked_shuff_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %zmm2, %zmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshufi64x2 {{.*#+}} zmm0 {%k1} {z} = zmm0[4,5,4,5],zmm1[4,5,4,5] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xi64_zero_masked_shuff_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %zmm2, %zmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshufi64x2 {{.*#+}} zmm0 {%k1} {z} = zmm0[4,5,4,5],zmm1[4,5,4,5] sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x i64> %vec1, <8 x i64> %vec2, <8 x i32> <i32 4, i32 5, i32 4, i32 5, i32 12, i32 13, i32 12, i32 13>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i64> %shuf, <8 x i64> zeroinitializer
-  ret <8 x i64> %res
-}
-define <8 x i64> @test_8xi64_masked_shuff_mask1(<8 x i64> %vec1, <8 x i64> %vec2, <8 x i64> %vec3, <8 x i64> %mask) {
-; GENERIC-LABEL: test_8xi64_masked_shuff_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %zmm3, %zmm3, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshufi64x2 {{.*#+}} zmm2 {%k1} = zmm0[6,7,4,5],zmm1[2,3,4,5] sched: [1:1.00]
-; GENERIC-NEXT:    vmovdqa64 %zmm2, %zmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xi64_masked_shuff_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %zmm3, %zmm3, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshufi64x2 {{.*#+}} zmm2 {%k1} = zmm0[6,7,4,5],zmm1[2,3,4,5] sched: [3:1.00]
-; SKX-NEXT:    vmovdqa64 %zmm2, %zmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x i64> %vec1, <8 x i64> %vec2, <8 x i32> <i32 6, i32 7, i32 4, i32 5, i32 10, i32 11, i32 12, i32 13>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i64> %shuf, <8 x i64> %vec3
-  ret <8 x i64> %res
-}
-
-define <8 x i64> @test_8xi64_zero_masked_shuff_mask1(<8 x i64> %vec1, <8 x i64> %vec2, <8 x i64> %mask) {
-; GENERIC-LABEL: test_8xi64_zero_masked_shuff_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %zmm2, %zmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshufi64x2 {{.*#+}} zmm0 {%k1} {z} = zmm0[6,7,4,5],zmm1[2,3,4,5] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xi64_zero_masked_shuff_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %zmm2, %zmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshufi64x2 {{.*#+}} zmm0 {%k1} {z} = zmm0[6,7,4,5],zmm1[2,3,4,5] sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x i64> %vec1, <8 x i64> %vec2, <8 x i32> <i32 6, i32 7, i32 4, i32 5, i32 10, i32 11, i32 12, i32 13>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i64> %shuf, <8 x i64> zeroinitializer
-  ret <8 x i64> %res
-}
-define <8 x i64> @test_8xi64_masked_shuff_mask2(<8 x i64> %vec1, <8 x i64> %vec2, <8 x i64> %vec3, <8 x i64> %mask) {
-; GENERIC-LABEL: test_8xi64_masked_shuff_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %zmm3, %zmm3, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshufi64x2 {{.*#+}} zmm2 {%k1} = zmm0[0,1,4,5],zmm1[0,1,0,1] sched: [1:1.00]
-; GENERIC-NEXT:    vmovdqa64 %zmm2, %zmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xi64_masked_shuff_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %zmm3, %zmm3, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshufi64x2 {{.*#+}} zmm2 {%k1} = zmm0[0,1,4,5],zmm1[0,1,0,1] sched: [3:1.00]
-; SKX-NEXT:    vmovdqa64 %zmm2, %zmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x i64> %vec1, <8 x i64> %vec2, <8 x i32> <i32 0, i32 1, i32 4, i32 5, i32 8, i32 9, i32 8, i32 9>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i64> %shuf, <8 x i64> %vec3
-  ret <8 x i64> %res
-}
-
-define <8 x i64> @test_8xi64_zero_masked_shuff_mask2(<8 x i64> %vec1, <8 x i64> %vec2, <8 x i64> %mask) {
-; GENERIC-LABEL: test_8xi64_zero_masked_shuff_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %zmm2, %zmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshufi64x2 {{.*#+}} zmm0 {%k1} {z} = zmm0[0,1,4,5],zmm1[0,1,0,1] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xi64_zero_masked_shuff_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %zmm2, %zmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshufi64x2 {{.*#+}} zmm0 {%k1} {z} = zmm0[0,1,4,5],zmm1[0,1,0,1] sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x i64> %vec1, <8 x i64> %vec2, <8 x i32> <i32 0, i32 1, i32 4, i32 5, i32 8, i32 9, i32 8, i32 9>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i64> %shuf, <8 x i64> zeroinitializer
-  ret <8 x i64> %res
-}
-define <8 x i64> @test_8xi64_shuff_mask3(<8 x i64> %vec1, <8 x i64> %vec2) {
-; GENERIC-LABEL: test_8xi64_shuff_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm0[2,3,6,7],zmm1[4,5,2,3] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xi64_shuff_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm0[2,3,6,7],zmm1[4,5,2,3] sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %res = shufflevector <8 x i64> %vec1, <8 x i64> %vec2, <8 x i32> <i32 2, i32 3, i32 6, i32 7, i32 12, i32 13, i32 10, i32 11>
-  ret <8 x i64> %res
-}
-define <8 x i64> @test_8xi64_masked_shuff_mask3(<8 x i64> %vec1, <8 x i64> %vec2, <8 x i64> %vec3, <8 x i64> %mask) {
-; GENERIC-LABEL: test_8xi64_masked_shuff_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %zmm3, %zmm3, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshufi64x2 {{.*#+}} zmm2 {%k1} = zmm0[2,3,6,7],zmm1[4,5,2,3] sched: [1:1.00]
-; GENERIC-NEXT:    vmovdqa64 %zmm2, %zmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xi64_masked_shuff_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %zmm3, %zmm3, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshufi64x2 {{.*#+}} zmm2 {%k1} = zmm0[2,3,6,7],zmm1[4,5,2,3] sched: [3:1.00]
-; SKX-NEXT:    vmovdqa64 %zmm2, %zmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x i64> %vec1, <8 x i64> %vec2, <8 x i32> <i32 2, i32 3, i32 6, i32 7, i32 12, i32 13, i32 10, i32 11>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i64> %shuf, <8 x i64> %vec3
-  ret <8 x i64> %res
-}
-
-define <8 x i64> @test_8xi64_zero_masked_shuff_mask3(<8 x i64> %vec1, <8 x i64> %vec2, <8 x i64> %mask) {
-; GENERIC-LABEL: test_8xi64_zero_masked_shuff_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %zmm2, %zmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshufi64x2 {{.*#+}} zmm0 {%k1} {z} = zmm0[2,3,6,7],zmm1[4,5,2,3] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xi64_zero_masked_shuff_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %zmm2, %zmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshufi64x2 {{.*#+}} zmm0 {%k1} {z} = zmm0[2,3,6,7],zmm1[4,5,2,3] sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x i64> %vec1, <8 x i64> %vec2, <8 x i32> <i32 2, i32 3, i32 6, i32 7, i32 12, i32 13, i32 10, i32 11>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i64> %shuf, <8 x i64> zeroinitializer
-  ret <8 x i64> %res
-}
-define <8 x i64> @test_8xi64_shuff_mem_mask0(<8 x i64> %vec1, <8 x i64>* %vec2p) {
-; GENERIC-LABEL: test_8xi64_shuff_mem_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm0[2,3,2,3],mem[4,5,2,3] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xi64_shuff_mem_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm0[2,3,2,3],mem[4,5,2,3] sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <8 x i64>, <8 x i64>* %vec2p
-  %res = shufflevector <8 x i64> %vec1, <8 x i64> %vec2, <8 x i32> <i32 2, i32 3, i32 2, i32 3, i32 12, i32 13, i32 10, i32 11>
-  ret <8 x i64> %res
-}
-define <8 x i64> @test_8xi64_masked_shuff_mem_mask0(<8 x i64> %vec1, <8 x i64>* %vec2p, <8 x i64> %vec3, <8 x i64> %mask) {
-; GENERIC-LABEL: test_8xi64_masked_shuff_mem_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %zmm2, %zmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshufi64x2 {{.*#+}} zmm1 {%k1} = zmm0[2,3,2,3],mem[4,5,2,3] sched: [8:1.00]
-; GENERIC-NEXT:    vmovdqa64 %zmm1, %zmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xi64_masked_shuff_mem_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %zmm2, %zmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshufi64x2 {{.*#+}} zmm1 {%k1} = zmm0[2,3,2,3],mem[4,5,2,3] sched: [10:1.00]
-; SKX-NEXT:    vmovdqa64 %zmm1, %zmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <8 x i64>, <8 x i64>* %vec2p
-  %shuf = shufflevector <8 x i64> %vec1, <8 x i64> %vec2, <8 x i32> <i32 2, i32 3, i32 2, i32 3, i32 12, i32 13, i32 10, i32 11>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i64> %shuf, <8 x i64> %vec3
-  ret <8 x i64> %res
-}
-
-define <8 x i64> @test_8xi64_zero_masked_shuff_mem_mask0(<8 x i64> %vec1, <8 x i64>* %vec2p, <8 x i64> %mask) {
-; GENERIC-LABEL: test_8xi64_zero_masked_shuff_mem_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshufi64x2 {{.*#+}} zmm0 {%k1} {z} = zmm0[2,3,2,3],mem[4,5,2,3] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xi64_zero_masked_shuff_mem_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshufi64x2 {{.*#+}} zmm0 {%k1} {z} = zmm0[2,3,2,3],mem[4,5,2,3] sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <8 x i64>, <8 x i64>* %vec2p
-  %shuf = shufflevector <8 x i64> %vec1, <8 x i64> %vec2, <8 x i32> <i32 2, i32 3, i32 2, i32 3, i32 12, i32 13, i32 10, i32 11>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i64> %shuf, <8 x i64> zeroinitializer
-  ret <8 x i64> %res
-}
-
-define <8 x i64> @test_8xi64_masked_shuff_mem_mask1(<8 x i64> %vec1, <8 x i64>* %vec2p, <8 x i64> %vec3, <8 x i64> %mask) {
-; GENERIC-LABEL: test_8xi64_masked_shuff_mem_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %zmm2, %zmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshufi64x2 {{.*#+}} zmm1 {%k1} = zmm0[2,3,0,1],mem[0,1,0,1] sched: [8:1.00]
-; GENERIC-NEXT:    vmovdqa64 %zmm1, %zmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xi64_masked_shuff_mem_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %zmm2, %zmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshufi64x2 {{.*#+}} zmm1 {%k1} = zmm0[2,3,0,1],mem[0,1,0,1] sched: [10:1.00]
-; SKX-NEXT:    vmovdqa64 %zmm1, %zmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <8 x i64>, <8 x i64>* %vec2p
-  %shuf = shufflevector <8 x i64> %vec1, <8 x i64> %vec2, <8 x i32> <i32 2, i32 3, i32 0, i32 1, i32 8, i32 9, i32 8, i32 9>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i64> %shuf, <8 x i64> %vec3
-  ret <8 x i64> %res
-}
-
-define <8 x i64> @test_8xi64_zero_masked_shuff_mem_mask1(<8 x i64> %vec1, <8 x i64>* %vec2p, <8 x i64> %mask) {
-; GENERIC-LABEL: test_8xi64_zero_masked_shuff_mem_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshufi64x2 {{.*#+}} zmm0 {%k1} {z} = zmm0[2,3,0,1],mem[0,1,0,1] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xi64_zero_masked_shuff_mem_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshufi64x2 {{.*#+}} zmm0 {%k1} {z} = zmm0[2,3,0,1],mem[0,1,0,1] sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <8 x i64>, <8 x i64>* %vec2p
-  %shuf = shufflevector <8 x i64> %vec1, <8 x i64> %vec2, <8 x i32> <i32 2, i32 3, i32 0, i32 1, i32 8, i32 9, i32 8, i32 9>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i64> %shuf, <8 x i64> zeroinitializer
-  ret <8 x i64> %res
-}
-
-define <8 x i64> @test_8xi64_masked_shuff_mem_mask2(<8 x i64> %vec1, <8 x i64>* %vec2p, <8 x i64> %vec3, <8 x i64> %mask) {
-; GENERIC-LABEL: test_8xi64_masked_shuff_mem_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %zmm2, %zmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshufi64x2 {{.*#+}} zmm1 {%k1} = zmm0[4,5,0,1],mem[2,3,2,3] sched: [8:1.00]
-; GENERIC-NEXT:    vmovdqa64 %zmm1, %zmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xi64_masked_shuff_mem_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %zmm2, %zmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshufi64x2 {{.*#+}} zmm1 {%k1} = zmm0[4,5,0,1],mem[2,3,2,3] sched: [10:1.00]
-; SKX-NEXT:    vmovdqa64 %zmm1, %zmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <8 x i64>, <8 x i64>* %vec2p
-  %shuf = shufflevector <8 x i64> %vec1, <8 x i64> %vec2, <8 x i32> <i32 4, i32 5, i32 0, i32 1, i32 10, i32 11, i32 10, i32 11>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i64> %shuf, <8 x i64> %vec3
-  ret <8 x i64> %res
-}
-
-define <8 x i64> @test_8xi64_zero_masked_shuff_mem_mask2(<8 x i64> %vec1, <8 x i64>* %vec2p, <8 x i64> %mask) {
-; GENERIC-LABEL: test_8xi64_zero_masked_shuff_mem_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshufi64x2 {{.*#+}} zmm0 {%k1} {z} = zmm0[4,5,0,1],mem[2,3,2,3] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xi64_zero_masked_shuff_mem_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshufi64x2 {{.*#+}} zmm0 {%k1} {z} = zmm0[4,5,0,1],mem[2,3,2,3] sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <8 x i64>, <8 x i64>* %vec2p
-  %shuf = shufflevector <8 x i64> %vec1, <8 x i64> %vec2, <8 x i32> <i32 4, i32 5, i32 0, i32 1, i32 10, i32 11, i32 10, i32 11>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i64> %shuf, <8 x i64> zeroinitializer
-  ret <8 x i64> %res
-}
-
-define <8 x i64> @test_8xi64_shuff_mem_mask3(<8 x i64> %vec1, <8 x i64>* %vec2p) {
-; GENERIC-LABEL: test_8xi64_shuff_mem_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm0[2,3,0,1],mem[6,7,2,3] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xi64_shuff_mem_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm0[2,3,0,1],mem[6,7,2,3] sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <8 x i64>, <8 x i64>* %vec2p
-  %res = shufflevector <8 x i64> %vec1, <8 x i64> %vec2, <8 x i32> <i32 2, i32 3, i32 0, i32 1, i32 14, i32 15, i32 10, i32 11>
-  ret <8 x i64> %res
-}
-define <8 x i64> @test_8xi64_masked_shuff_mem_mask3(<8 x i64> %vec1, <8 x i64>* %vec2p, <8 x i64> %vec3, <8 x i64> %mask) {
-; GENERIC-LABEL: test_8xi64_masked_shuff_mem_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %zmm2, %zmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshufi64x2 {{.*#+}} zmm1 {%k1} = zmm0[2,3,0,1],mem[6,7,2,3] sched: [8:1.00]
-; GENERIC-NEXT:    vmovdqa64 %zmm1, %zmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xi64_masked_shuff_mem_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %zmm2, %zmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshufi64x2 {{.*#+}} zmm1 {%k1} = zmm0[2,3,0,1],mem[6,7,2,3] sched: [10:1.00]
-; SKX-NEXT:    vmovdqa64 %zmm1, %zmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <8 x i64>, <8 x i64>* %vec2p
-  %shuf = shufflevector <8 x i64> %vec1, <8 x i64> %vec2, <8 x i32> <i32 2, i32 3, i32 0, i32 1, i32 14, i32 15, i32 10, i32 11>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i64> %shuf, <8 x i64> %vec3
-  ret <8 x i64> %res
-}
-
-define <8 x i64> @test_8xi64_zero_masked_shuff_mem_mask3(<8 x i64> %vec1, <8 x i64>* %vec2p, <8 x i64> %mask) {
-; GENERIC-LABEL: test_8xi64_zero_masked_shuff_mem_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vshufi64x2 {{.*#+}} zmm0 {%k1} {z} = zmm0[2,3,0,1],mem[6,7,2,3] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xi64_zero_masked_shuff_mem_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vshufi64x2 {{.*#+}} zmm0 {%k1} {z} = zmm0[2,3,0,1],mem[6,7,2,3] sched: [10:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <8 x i64>, <8 x i64>* %vec2p
-  %shuf = shufflevector <8 x i64> %vec1, <8 x i64> %vec2, <8 x i32> <i32 2, i32 3, i32 0, i32 1, i32 14, i32 15, i32 10, i32 11>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x i64> %shuf, <8 x i64> zeroinitializer
-  ret <8 x i64> %res
-}
-
-define <4 x float> @test_4xfloat_unpack_low_mask0(<4 x float> %vec1, <4 x float> %vec2) {
-; GENERIC-LABEL: test_4xfloat_unpack_low_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vunpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xfloat_unpack_low_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vunpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %res = shufflevector <4 x float> %vec1, <4 x float> %vec2, <4 x i32> <i32 0, i32 4, i32 1, i32 5>
-  ret <4 x float> %res
-}
-define <4 x float> @test_4xfloat_masked_unpack_low_mask0(<4 x float> %vec1, <4 x float> %vec2, <4 x float> %vec3, <4 x i32> %mask) {
-; GENERIC-LABEL: test_4xfloat_masked_unpack_low_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %xmm3, %xmm3, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpcklps {{.*#+}} xmm2 {%k1} = xmm0[0],xmm1[0],xmm0[1],xmm1[1] sched: [1:1.00]
-; GENERIC-NEXT:    vmovaps %xmm2, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xfloat_masked_unpack_low_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %xmm3, %xmm3, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpcklps {{.*#+}} xmm2 {%k1} = xmm0[0],xmm1[0],xmm0[1],xmm1[1] sched: [1:1.00]
-; SKX-NEXT:    vmovaps %xmm2, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <4 x float> %vec1, <4 x float> %vec2, <4 x i32> <i32 0, i32 4, i32 1, i32 5>
-  %cmp = icmp eq <4 x i32> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x float> %shuf, <4 x float> %vec3
-  ret <4 x float> %res
-}
-
-define <4 x float> @test_4xfloat_zero_masked_unpack_low_mask0(<4 x float> %vec1, <4 x float> %vec2, <4 x i32> %mask) {
-; GENERIC-LABEL: test_4xfloat_zero_masked_unpack_low_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %xmm2, %xmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpcklps {{.*#+}} xmm0 {%k1} {z} = xmm0[0],xmm1[0],xmm0[1],xmm1[1] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xfloat_zero_masked_unpack_low_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %xmm2, %xmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpcklps {{.*#+}} xmm0 {%k1} {z} = xmm0[0],xmm1[0],xmm0[1],xmm1[1] sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <4 x float> %vec1, <4 x float> %vec2, <4 x i32> <i32 0, i32 4, i32 1, i32 5>
-  %cmp = icmp eq <4 x i32> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x float> %shuf, <4 x float> zeroinitializer
-  ret <4 x float> %res
-}
-define <4 x float> @test_4xfloat_masked_unpack_low_mask1(<4 x float> %vec1, <4 x float> %vec2, <4 x float> %vec3, <4 x i32> %mask) {
-; GENERIC-LABEL: test_4xfloat_masked_unpack_low_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %xmm3, %xmm3, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpcklps {{.*#+}} xmm2 {%k1} = xmm0[0],xmm1[0],xmm0[1],xmm1[1] sched: [1:1.00]
-; GENERIC-NEXT:    vmovaps %xmm2, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xfloat_masked_unpack_low_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %xmm3, %xmm3, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpcklps {{.*#+}} xmm2 {%k1} = xmm0[0],xmm1[0],xmm0[1],xmm1[1] sched: [1:1.00]
-; SKX-NEXT:    vmovaps %xmm2, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <4 x float> %vec1, <4 x float> %vec2, <4 x i32> <i32 0, i32 4, i32 1, i32 5>
-  %cmp = icmp eq <4 x i32> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x float> %shuf, <4 x float> %vec3
-  ret <4 x float> %res
-}
-
-define <4 x float> @test_4xfloat_zero_masked_unpack_low_mask1(<4 x float> %vec1, <4 x float> %vec2, <4 x i32> %mask) {
-; GENERIC-LABEL: test_4xfloat_zero_masked_unpack_low_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %xmm2, %xmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpcklps {{.*#+}} xmm0 {%k1} {z} = xmm0[0],xmm1[0],xmm0[1],xmm1[1] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xfloat_zero_masked_unpack_low_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %xmm2, %xmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpcklps {{.*#+}} xmm0 {%k1} {z} = xmm0[0],xmm1[0],xmm0[1],xmm1[1] sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <4 x float> %vec1, <4 x float> %vec2, <4 x i32> <i32 0, i32 4, i32 1, i32 5>
-  %cmp = icmp eq <4 x i32> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x float> %shuf, <4 x float> zeroinitializer
-  ret <4 x float> %res
-}
-define <4 x float> @test_4xfloat_masked_unpack_low_mask2(<4 x float> %vec1, <4 x float> %vec2, <4 x float> %vec3, <4 x i32> %mask) {
-; GENERIC-LABEL: test_4xfloat_masked_unpack_low_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %xmm3, %xmm3, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpcklps {{.*#+}} xmm2 {%k1} = xmm0[0],xmm1[0],xmm0[1],xmm1[1] sched: [1:1.00]
-; GENERIC-NEXT:    vmovaps %xmm2, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xfloat_masked_unpack_low_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %xmm3, %xmm3, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpcklps {{.*#+}} xmm2 {%k1} = xmm0[0],xmm1[0],xmm0[1],xmm1[1] sched: [1:1.00]
-; SKX-NEXT:    vmovaps %xmm2, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <4 x float> %vec1, <4 x float> %vec2, <4 x i32> <i32 0, i32 4, i32 1, i32 5>
-  %cmp = icmp eq <4 x i32> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x float> %shuf, <4 x float> %vec3
-  ret <4 x float> %res
-}
-
-define <4 x float> @test_4xfloat_zero_masked_unpack_low_mask2(<4 x float> %vec1, <4 x float> %vec2, <4 x i32> %mask) {
-; GENERIC-LABEL: test_4xfloat_zero_masked_unpack_low_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %xmm2, %xmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpcklps {{.*#+}} xmm0 {%k1} {z} = xmm0[0],xmm1[0],xmm0[1],xmm1[1] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xfloat_zero_masked_unpack_low_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %xmm2, %xmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpcklps {{.*#+}} xmm0 {%k1} {z} = xmm0[0],xmm1[0],xmm0[1],xmm1[1] sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <4 x float> %vec1, <4 x float> %vec2, <4 x i32> <i32 0, i32 4, i32 1, i32 5>
-  %cmp = icmp eq <4 x i32> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x float> %shuf, <4 x float> zeroinitializer
-  ret <4 x float> %res
-}
-define <4 x float> @test_4xfloat_unpack_low_mask3(<4 x float> %vec1, <4 x float> %vec2) {
-; GENERIC-LABEL: test_4xfloat_unpack_low_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vunpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xfloat_unpack_low_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vunpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %res = shufflevector <4 x float> %vec1, <4 x float> %vec2, <4 x i32> <i32 0, i32 4, i32 1, i32 5>
-  ret <4 x float> %res
-}
-define <4 x float> @test_4xfloat_masked_unpack_low_mask3(<4 x float> %vec1, <4 x float> %vec2, <4 x float> %vec3, <4 x i32> %mask) {
-; GENERIC-LABEL: test_4xfloat_masked_unpack_low_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %xmm3, %xmm3, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpcklps {{.*#+}} xmm2 {%k1} = xmm0[0],xmm1[0],xmm0[1],xmm1[1] sched: [1:1.00]
-; GENERIC-NEXT:    vmovaps %xmm2, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xfloat_masked_unpack_low_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %xmm3, %xmm3, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpcklps {{.*#+}} xmm2 {%k1} = xmm0[0],xmm1[0],xmm0[1],xmm1[1] sched: [1:1.00]
-; SKX-NEXT:    vmovaps %xmm2, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <4 x float> %vec1, <4 x float> %vec2, <4 x i32> <i32 0, i32 4, i32 1, i32 5>
-  %cmp = icmp eq <4 x i32> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x float> %shuf, <4 x float> %vec3
-  ret <4 x float> %res
-}
-
-define <4 x float> @test_4xfloat_zero_masked_unpack_low_mask3(<4 x float> %vec1, <4 x float> %vec2, <4 x i32> %mask) {
-; GENERIC-LABEL: test_4xfloat_zero_masked_unpack_low_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %xmm2, %xmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpcklps {{.*#+}} xmm0 {%k1} {z} = xmm0[0],xmm1[0],xmm0[1],xmm1[1] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xfloat_zero_masked_unpack_low_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %xmm2, %xmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpcklps {{.*#+}} xmm0 {%k1} {z} = xmm0[0],xmm1[0],xmm0[1],xmm1[1] sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <4 x float> %vec1, <4 x float> %vec2, <4 x i32> <i32 0, i32 4, i32 1, i32 5>
-  %cmp = icmp eq <4 x i32> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x float> %shuf, <4 x float> zeroinitializer
-  ret <4 x float> %res
-}
-define <4 x float> @test_4xfloat_unpack_low_mem_mask0(<4 x float> %vec1, <4 x float>* %vec2p) {
-; GENERIC-LABEL: test_4xfloat_unpack_low_mem_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vunpcklps {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1] sched: [7:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xfloat_unpack_low_mem_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vunpcklps {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1] sched: [7:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <4 x float>, <4 x float>* %vec2p
-  %res = shufflevector <4 x float> %vec1, <4 x float> %vec2, <4 x i32> <i32 0, i32 4, i32 1, i32 5>
-  ret <4 x float> %res
-}
-define <4 x float> @test_4xfloat_masked_unpack_low_mem_mask0(<4 x float> %vec1, <4 x float>* %vec2p, <4 x float> %vec3, <4 x i32> %mask) {
-; GENERIC-LABEL: test_4xfloat_masked_unpack_low_mem_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %xmm2, %xmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpcklps {{.*#+}} xmm1 {%k1} = xmm0[0],mem[0],xmm0[1],mem[1] sched: [7:1.00]
-; GENERIC-NEXT:    vmovaps %xmm1, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xfloat_masked_unpack_low_mem_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %xmm2, %xmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpcklps {{.*#+}} xmm1 {%k1} = xmm0[0],mem[0],xmm0[1],mem[1] sched: [7:1.00]
-; SKX-NEXT:    vmovaps %xmm1, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <4 x float>, <4 x float>* %vec2p
-  %shuf = shufflevector <4 x float> %vec1, <4 x float> %vec2, <4 x i32> <i32 0, i32 4, i32 1, i32 5>
-  %cmp = icmp eq <4 x i32> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x float> %shuf, <4 x float> %vec3
-  ret <4 x float> %res
-}
-
-define <4 x float> @test_4xfloat_zero_masked_unpack_low_mem_mask0(<4 x float> %vec1, <4 x float>* %vec2p, <4 x i32> %mask) {
-; GENERIC-LABEL: test_4xfloat_zero_masked_unpack_low_mem_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %xmm1, %xmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpcklps {{.*#+}} xmm0 {%k1} {z} = xmm0[0],mem[0],xmm0[1],mem[1] sched: [7:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xfloat_zero_masked_unpack_low_mem_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %xmm1, %xmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpcklps {{.*#+}} xmm0 {%k1} {z} = xmm0[0],mem[0],xmm0[1],mem[1] sched: [7:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <4 x float>, <4 x float>* %vec2p
-  %shuf = shufflevector <4 x float> %vec1, <4 x float> %vec2, <4 x i32> <i32 0, i32 4, i32 1, i32 5>
-  %cmp = icmp eq <4 x i32> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x float> %shuf, <4 x float> zeroinitializer
-  ret <4 x float> %res
-}
-
-define <4 x float> @test_4xfloat_masked_unpack_low_mem_mask1(<4 x float> %vec1, <4 x float>* %vec2p, <4 x float> %vec3, <4 x i32> %mask) {
-; GENERIC-LABEL: test_4xfloat_masked_unpack_low_mem_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %xmm2, %xmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpcklps {{.*#+}} xmm1 {%k1} = xmm0[0],mem[0],xmm0[1],mem[1] sched: [7:1.00]
-; GENERIC-NEXT:    vmovaps %xmm1, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xfloat_masked_unpack_low_mem_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %xmm2, %xmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpcklps {{.*#+}} xmm1 {%k1} = xmm0[0],mem[0],xmm0[1],mem[1] sched: [7:1.00]
-; SKX-NEXT:    vmovaps %xmm1, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <4 x float>, <4 x float>* %vec2p
-  %shuf = shufflevector <4 x float> %vec1, <4 x float> %vec2, <4 x i32> <i32 0, i32 4, i32 1, i32 5>
-  %cmp = icmp eq <4 x i32> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x float> %shuf, <4 x float> %vec3
-  ret <4 x float> %res
-}
-
-define <4 x float> @test_4xfloat_zero_masked_unpack_low_mem_mask1(<4 x float> %vec1, <4 x float>* %vec2p, <4 x i32> %mask) {
-; GENERIC-LABEL: test_4xfloat_zero_masked_unpack_low_mem_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %xmm1, %xmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpcklps {{.*#+}} xmm0 {%k1} {z} = xmm0[0],mem[0],xmm0[1],mem[1] sched: [7:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xfloat_zero_masked_unpack_low_mem_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %xmm1, %xmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpcklps {{.*#+}} xmm0 {%k1} {z} = xmm0[0],mem[0],xmm0[1],mem[1] sched: [7:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <4 x float>, <4 x float>* %vec2p
-  %shuf = shufflevector <4 x float> %vec1, <4 x float> %vec2, <4 x i32> <i32 0, i32 4, i32 1, i32 5>
-  %cmp = icmp eq <4 x i32> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x float> %shuf, <4 x float> zeroinitializer
-  ret <4 x float> %res
-}
-
-define <4 x float> @test_4xfloat_masked_unpack_low_mem_mask2(<4 x float> %vec1, <4 x float>* %vec2p, <4 x float> %vec3, <4 x i32> %mask) {
-; GENERIC-LABEL: test_4xfloat_masked_unpack_low_mem_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %xmm2, %xmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpcklps {{.*#+}} xmm1 {%k1} = xmm0[0],mem[0],xmm0[1],mem[1] sched: [7:1.00]
-; GENERIC-NEXT:    vmovaps %xmm1, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xfloat_masked_unpack_low_mem_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %xmm2, %xmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpcklps {{.*#+}} xmm1 {%k1} = xmm0[0],mem[0],xmm0[1],mem[1] sched: [7:1.00]
-; SKX-NEXT:    vmovaps %xmm1, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <4 x float>, <4 x float>* %vec2p
-  %shuf = shufflevector <4 x float> %vec1, <4 x float> %vec2, <4 x i32> <i32 0, i32 4, i32 1, i32 5>
-  %cmp = icmp eq <4 x i32> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x float> %shuf, <4 x float> %vec3
-  ret <4 x float> %res
-}
-
-define <4 x float> @test_4xfloat_zero_masked_unpack_low_mem_mask2(<4 x float> %vec1, <4 x float>* %vec2p, <4 x i32> %mask) {
-; GENERIC-LABEL: test_4xfloat_zero_masked_unpack_low_mem_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %xmm1, %xmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpcklps {{.*#+}} xmm0 {%k1} {z} = xmm0[0],mem[0],xmm0[1],mem[1] sched: [7:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xfloat_zero_masked_unpack_low_mem_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %xmm1, %xmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpcklps {{.*#+}} xmm0 {%k1} {z} = xmm0[0],mem[0],xmm0[1],mem[1] sched: [7:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <4 x float>, <4 x float>* %vec2p
-  %shuf = shufflevector <4 x float> %vec1, <4 x float> %vec2, <4 x i32> <i32 0, i32 4, i32 1, i32 5>
-  %cmp = icmp eq <4 x i32> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x float> %shuf, <4 x float> zeroinitializer
-  ret <4 x float> %res
-}
-
-define <4 x float> @test_4xfloat_unpack_low_mem_mask3(<4 x float> %vec1, <4 x float>* %vec2p) {
-; GENERIC-LABEL: test_4xfloat_unpack_low_mem_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vunpcklps {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1] sched: [7:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xfloat_unpack_low_mem_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vunpcklps {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1] sched: [7:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <4 x float>, <4 x float>* %vec2p
-  %res = shufflevector <4 x float> %vec1, <4 x float> %vec2, <4 x i32> <i32 0, i32 4, i32 1, i32 5>
-  ret <4 x float> %res
-}
-define <4 x float> @test_4xfloat_masked_unpack_low_mem_mask3(<4 x float> %vec1, <4 x float>* %vec2p, <4 x float> %vec3, <4 x i32> %mask) {
-; GENERIC-LABEL: test_4xfloat_masked_unpack_low_mem_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %xmm2, %xmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpcklps {{.*#+}} xmm1 {%k1} = xmm0[0],mem[0],xmm0[1],mem[1] sched: [7:1.00]
-; GENERIC-NEXT:    vmovaps %xmm1, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xfloat_masked_unpack_low_mem_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %xmm2, %xmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpcklps {{.*#+}} xmm1 {%k1} = xmm0[0],mem[0],xmm0[1],mem[1] sched: [7:1.00]
-; SKX-NEXT:    vmovaps %xmm1, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <4 x float>, <4 x float>* %vec2p
-  %shuf = shufflevector <4 x float> %vec1, <4 x float> %vec2, <4 x i32> <i32 0, i32 4, i32 1, i32 5>
-  %cmp = icmp eq <4 x i32> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x float> %shuf, <4 x float> %vec3
-  ret <4 x float> %res
-}
-
-define <4 x float> @test_4xfloat_zero_masked_unpack_low_mem_mask3(<4 x float> %vec1, <4 x float>* %vec2p, <4 x i32> %mask) {
-; GENERIC-LABEL: test_4xfloat_zero_masked_unpack_low_mem_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %xmm1, %xmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpcklps {{.*#+}} xmm0 {%k1} {z} = xmm0[0],mem[0],xmm0[1],mem[1] sched: [7:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xfloat_zero_masked_unpack_low_mem_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %xmm1, %xmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpcklps {{.*#+}} xmm0 {%k1} {z} = xmm0[0],mem[0],xmm0[1],mem[1] sched: [7:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <4 x float>, <4 x float>* %vec2p
-  %shuf = shufflevector <4 x float> %vec1, <4 x float> %vec2, <4 x i32> <i32 0, i32 4, i32 1, i32 5>
-  %cmp = icmp eq <4 x i32> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x float> %shuf, <4 x float> zeroinitializer
-  ret <4 x float> %res
-}
-
-define <8 x float> @test_8xfloat_unpack_low_mask0(<8 x float> %vec1, <8 x float> %vec2) {
-; GENERIC-LABEL: test_8xfloat_unpack_low_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vunpcklps {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xfloat_unpack_low_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vunpcklps {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5] sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %res = shufflevector <8 x float> %vec1, <8 x float> %vec2, <8 x i32> <i32 0, i32 8, i32 1, i32 9, i32 4, i32 12, i32 5, i32 13>
-  ret <8 x float> %res
-}
-define <8 x float> @test_8xfloat_masked_unpack_low_mask0(<8 x float> %vec1, <8 x float> %vec2, <8 x float> %vec3, <8 x i32> %mask) {
-; GENERIC-LABEL: test_8xfloat_masked_unpack_low_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %ymm3, %ymm3, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpcklps {{.*#+}} ymm2 {%k1} = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5] sched: [1:1.00]
-; GENERIC-NEXT:    vmovaps %ymm2, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xfloat_masked_unpack_low_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %ymm3, %ymm3, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpcklps {{.*#+}} ymm2 {%k1} = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5] sched: [1:1.00]
-; SKX-NEXT:    vmovaps %ymm2, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x float> %vec1, <8 x float> %vec2, <8 x i32> <i32 0, i32 8, i32 1, i32 9, i32 4, i32 12, i32 5, i32 13>
-  %cmp = icmp eq <8 x i32> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x float> %shuf, <8 x float> %vec3
-  ret <8 x float> %res
-}
-
-define <8 x float> @test_8xfloat_zero_masked_unpack_low_mask0(<8 x float> %vec1, <8 x float> %vec2, <8 x i32> %mask) {
-; GENERIC-LABEL: test_8xfloat_zero_masked_unpack_low_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %ymm2, %ymm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpcklps {{.*#+}} ymm0 {%k1} {z} = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xfloat_zero_masked_unpack_low_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %ymm2, %ymm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpcklps {{.*#+}} ymm0 {%k1} {z} = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5] sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x float> %vec1, <8 x float> %vec2, <8 x i32> <i32 0, i32 8, i32 1, i32 9, i32 4, i32 12, i32 5, i32 13>
-  %cmp = icmp eq <8 x i32> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x float> %shuf, <8 x float> zeroinitializer
-  ret <8 x float> %res
-}
-define <8 x float> @test_8xfloat_masked_unpack_low_mask1(<8 x float> %vec1, <8 x float> %vec2, <8 x float> %vec3, <8 x i32> %mask) {
-; GENERIC-LABEL: test_8xfloat_masked_unpack_low_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %ymm3, %ymm3, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpcklps {{.*#+}} ymm2 {%k1} = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5] sched: [1:1.00]
-; GENERIC-NEXT:    vmovaps %ymm2, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xfloat_masked_unpack_low_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %ymm3, %ymm3, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpcklps {{.*#+}} ymm2 {%k1} = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5] sched: [1:1.00]
-; SKX-NEXT:    vmovaps %ymm2, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x float> %vec1, <8 x float> %vec2, <8 x i32> <i32 0, i32 8, i32 1, i32 9, i32 4, i32 12, i32 5, i32 13>
-  %cmp = icmp eq <8 x i32> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x float> %shuf, <8 x float> %vec3
-  ret <8 x float> %res
-}
-
-define <8 x float> @test_8xfloat_zero_masked_unpack_low_mask1(<8 x float> %vec1, <8 x float> %vec2, <8 x i32> %mask) {
-; GENERIC-LABEL: test_8xfloat_zero_masked_unpack_low_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %ymm2, %ymm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpcklps {{.*#+}} ymm0 {%k1} {z} = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xfloat_zero_masked_unpack_low_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %ymm2, %ymm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpcklps {{.*#+}} ymm0 {%k1} {z} = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5] sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x float> %vec1, <8 x float> %vec2, <8 x i32> <i32 0, i32 8, i32 1, i32 9, i32 4, i32 12, i32 5, i32 13>
-  %cmp = icmp eq <8 x i32> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x float> %shuf, <8 x float> zeroinitializer
-  ret <8 x float> %res
-}
-define <8 x float> @test_8xfloat_masked_unpack_low_mask2(<8 x float> %vec1, <8 x float> %vec2, <8 x float> %vec3, <8 x i32> %mask) {
-; GENERIC-LABEL: test_8xfloat_masked_unpack_low_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %ymm3, %ymm3, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpcklps {{.*#+}} ymm2 {%k1} = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5] sched: [1:1.00]
-; GENERIC-NEXT:    vmovaps %ymm2, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xfloat_masked_unpack_low_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %ymm3, %ymm3, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpcklps {{.*#+}} ymm2 {%k1} = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5] sched: [1:1.00]
-; SKX-NEXT:    vmovaps %ymm2, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x float> %vec1, <8 x float> %vec2, <8 x i32> <i32 0, i32 8, i32 1, i32 9, i32 4, i32 12, i32 5, i32 13>
-  %cmp = icmp eq <8 x i32> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x float> %shuf, <8 x float> %vec3
-  ret <8 x float> %res
-}
-
-define <8 x float> @test_8xfloat_zero_masked_unpack_low_mask2(<8 x float> %vec1, <8 x float> %vec2, <8 x i32> %mask) {
-; GENERIC-LABEL: test_8xfloat_zero_masked_unpack_low_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %ymm2, %ymm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpcklps {{.*#+}} ymm0 {%k1} {z} = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xfloat_zero_masked_unpack_low_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %ymm2, %ymm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpcklps {{.*#+}} ymm0 {%k1} {z} = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5] sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x float> %vec1, <8 x float> %vec2, <8 x i32> <i32 0, i32 8, i32 1, i32 9, i32 4, i32 12, i32 5, i32 13>
-  %cmp = icmp eq <8 x i32> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x float> %shuf, <8 x float> zeroinitializer
-  ret <8 x float> %res
-}
-define <8 x float> @test_8xfloat_unpack_low_mask3(<8 x float> %vec1, <8 x float> %vec2) {
-; GENERIC-LABEL: test_8xfloat_unpack_low_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vunpcklps {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xfloat_unpack_low_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vunpcklps {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5] sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %res = shufflevector <8 x float> %vec1, <8 x float> %vec2, <8 x i32> <i32 0, i32 8, i32 1, i32 9, i32 4, i32 12, i32 5, i32 13>
-  ret <8 x float> %res
-}
-define <8 x float> @test_8xfloat_masked_unpack_low_mask3(<8 x float> %vec1, <8 x float> %vec2, <8 x float> %vec3, <8 x i32> %mask) {
-; GENERIC-LABEL: test_8xfloat_masked_unpack_low_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %ymm3, %ymm3, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpcklps {{.*#+}} ymm2 {%k1} = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5] sched: [1:1.00]
-; GENERIC-NEXT:    vmovaps %ymm2, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xfloat_masked_unpack_low_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %ymm3, %ymm3, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpcklps {{.*#+}} ymm2 {%k1} = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5] sched: [1:1.00]
-; SKX-NEXT:    vmovaps %ymm2, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x float> %vec1, <8 x float> %vec2, <8 x i32> <i32 0, i32 8, i32 1, i32 9, i32 4, i32 12, i32 5, i32 13>
-  %cmp = icmp eq <8 x i32> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x float> %shuf, <8 x float> %vec3
-  ret <8 x float> %res
-}
-
-define <8 x float> @test_8xfloat_zero_masked_unpack_low_mask3(<8 x float> %vec1, <8 x float> %vec2, <8 x i32> %mask) {
-; GENERIC-LABEL: test_8xfloat_zero_masked_unpack_low_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %ymm2, %ymm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpcklps {{.*#+}} ymm0 {%k1} {z} = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xfloat_zero_masked_unpack_low_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %ymm2, %ymm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpcklps {{.*#+}} ymm0 {%k1} {z} = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5] sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x float> %vec1, <8 x float> %vec2, <8 x i32> <i32 0, i32 8, i32 1, i32 9, i32 4, i32 12, i32 5, i32 13>
-  %cmp = icmp eq <8 x i32> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x float> %shuf, <8 x float> zeroinitializer
-  ret <8 x float> %res
-}
-define <8 x float> @test_8xfloat_unpack_low_mem_mask0(<8 x float> %vec1, <8 x float>* %vec2p) {
-; GENERIC-LABEL: test_8xfloat_unpack_low_mem_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vunpcklps {{.*#+}} ymm0 = ymm0[0],mem[0],ymm0[1],mem[1],ymm0[4],mem[4],ymm0[5],mem[5] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xfloat_unpack_low_mem_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vunpcklps {{.*#+}} ymm0 = ymm0[0],mem[0],ymm0[1],mem[1],ymm0[4],mem[4],ymm0[5],mem[5] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <8 x float>, <8 x float>* %vec2p
-  %res = shufflevector <8 x float> %vec1, <8 x float> %vec2, <8 x i32> <i32 0, i32 8, i32 1, i32 9, i32 4, i32 12, i32 5, i32 13>
-  ret <8 x float> %res
-}
-define <8 x float> @test_8xfloat_masked_unpack_low_mem_mask0(<8 x float> %vec1, <8 x float>* %vec2p, <8 x float> %vec3, <8 x i32> %mask) {
-; GENERIC-LABEL: test_8xfloat_masked_unpack_low_mem_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %ymm2, %ymm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpcklps {{.*#+}} ymm1 {%k1} = ymm0[0],mem[0],ymm0[1],mem[1],ymm0[4],mem[4],ymm0[5],mem[5] sched: [8:1.00]
-; GENERIC-NEXT:    vmovaps %ymm1, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xfloat_masked_unpack_low_mem_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %ymm2, %ymm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpcklps {{.*#+}} ymm1 {%k1} = ymm0[0],mem[0],ymm0[1],mem[1],ymm0[4],mem[4],ymm0[5],mem[5] sched: [8:1.00]
-; SKX-NEXT:    vmovaps %ymm1, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <8 x float>, <8 x float>* %vec2p
-  %shuf = shufflevector <8 x float> %vec1, <8 x float> %vec2, <8 x i32> <i32 0, i32 8, i32 1, i32 9, i32 4, i32 12, i32 5, i32 13>
-  %cmp = icmp eq <8 x i32> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x float> %shuf, <8 x float> %vec3
-  ret <8 x float> %res
-}
-
-define <8 x float> @test_8xfloat_zero_masked_unpack_low_mem_mask0(<8 x float> %vec1, <8 x float>* %vec2p, <8 x i32> %mask) {
-; GENERIC-LABEL: test_8xfloat_zero_masked_unpack_low_mem_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %ymm1, %ymm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpcklps {{.*#+}} ymm0 {%k1} {z} = ymm0[0],mem[0],ymm0[1],mem[1],ymm0[4],mem[4],ymm0[5],mem[5] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xfloat_zero_masked_unpack_low_mem_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %ymm1, %ymm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpcklps {{.*#+}} ymm0 {%k1} {z} = ymm0[0],mem[0],ymm0[1],mem[1],ymm0[4],mem[4],ymm0[5],mem[5] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <8 x float>, <8 x float>* %vec2p
-  %shuf = shufflevector <8 x float> %vec1, <8 x float> %vec2, <8 x i32> <i32 0, i32 8, i32 1, i32 9, i32 4, i32 12, i32 5, i32 13>
-  %cmp = icmp eq <8 x i32> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x float> %shuf, <8 x float> zeroinitializer
-  ret <8 x float> %res
-}
-
-define <8 x float> @test_8xfloat_masked_unpack_low_mem_mask1(<8 x float> %vec1, <8 x float>* %vec2p, <8 x float> %vec3, <8 x i32> %mask) {
-; GENERIC-LABEL: test_8xfloat_masked_unpack_low_mem_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %ymm2, %ymm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpcklps {{.*#+}} ymm1 {%k1} = ymm0[0],mem[0],ymm0[1],mem[1],ymm0[4],mem[4],ymm0[5],mem[5] sched: [8:1.00]
-; GENERIC-NEXT:    vmovaps %ymm1, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xfloat_masked_unpack_low_mem_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %ymm2, %ymm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpcklps {{.*#+}} ymm1 {%k1} = ymm0[0],mem[0],ymm0[1],mem[1],ymm0[4],mem[4],ymm0[5],mem[5] sched: [8:1.00]
-; SKX-NEXT:    vmovaps %ymm1, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <8 x float>, <8 x float>* %vec2p
-  %shuf = shufflevector <8 x float> %vec1, <8 x float> %vec2, <8 x i32> <i32 0, i32 8, i32 1, i32 9, i32 4, i32 12, i32 5, i32 13>
-  %cmp = icmp eq <8 x i32> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x float> %shuf, <8 x float> %vec3
-  ret <8 x float> %res
-}
-
-define <8 x float> @test_8xfloat_zero_masked_unpack_low_mem_mask1(<8 x float> %vec1, <8 x float>* %vec2p, <8 x i32> %mask) {
-; GENERIC-LABEL: test_8xfloat_zero_masked_unpack_low_mem_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %ymm1, %ymm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpcklps {{.*#+}} ymm0 {%k1} {z} = ymm0[0],mem[0],ymm0[1],mem[1],ymm0[4],mem[4],ymm0[5],mem[5] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xfloat_zero_masked_unpack_low_mem_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %ymm1, %ymm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpcklps {{.*#+}} ymm0 {%k1} {z} = ymm0[0],mem[0],ymm0[1],mem[1],ymm0[4],mem[4],ymm0[5],mem[5] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <8 x float>, <8 x float>* %vec2p
-  %shuf = shufflevector <8 x float> %vec1, <8 x float> %vec2, <8 x i32> <i32 0, i32 8, i32 1, i32 9, i32 4, i32 12, i32 5, i32 13>
-  %cmp = icmp eq <8 x i32> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x float> %shuf, <8 x float> zeroinitializer
-  ret <8 x float> %res
-}
-
-define <8 x float> @test_8xfloat_masked_unpack_low_mem_mask2(<8 x float> %vec1, <8 x float>* %vec2p, <8 x float> %vec3, <8 x i32> %mask) {
-; GENERIC-LABEL: test_8xfloat_masked_unpack_low_mem_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %ymm2, %ymm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpcklps {{.*#+}} ymm1 {%k1} = ymm0[0],mem[0],ymm0[1],mem[1],ymm0[4],mem[4],ymm0[5],mem[5] sched: [8:1.00]
-; GENERIC-NEXT:    vmovaps %ymm1, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xfloat_masked_unpack_low_mem_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %ymm2, %ymm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpcklps {{.*#+}} ymm1 {%k1} = ymm0[0],mem[0],ymm0[1],mem[1],ymm0[4],mem[4],ymm0[5],mem[5] sched: [8:1.00]
-; SKX-NEXT:    vmovaps %ymm1, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <8 x float>, <8 x float>* %vec2p
-  %shuf = shufflevector <8 x float> %vec1, <8 x float> %vec2, <8 x i32> <i32 0, i32 8, i32 1, i32 9, i32 4, i32 12, i32 5, i32 13>
-  %cmp = icmp eq <8 x i32> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x float> %shuf, <8 x float> %vec3
-  ret <8 x float> %res
-}
-
-define <8 x float> @test_8xfloat_zero_masked_unpack_low_mem_mask2(<8 x float> %vec1, <8 x float>* %vec2p, <8 x i32> %mask) {
-; GENERIC-LABEL: test_8xfloat_zero_masked_unpack_low_mem_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %ymm1, %ymm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpcklps {{.*#+}} ymm0 {%k1} {z} = ymm0[0],mem[0],ymm0[1],mem[1],ymm0[4],mem[4],ymm0[5],mem[5] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xfloat_zero_masked_unpack_low_mem_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %ymm1, %ymm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpcklps {{.*#+}} ymm0 {%k1} {z} = ymm0[0],mem[0],ymm0[1],mem[1],ymm0[4],mem[4],ymm0[5],mem[5] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <8 x float>, <8 x float>* %vec2p
-  %shuf = shufflevector <8 x float> %vec1, <8 x float> %vec2, <8 x i32> <i32 0, i32 8, i32 1, i32 9, i32 4, i32 12, i32 5, i32 13>
-  %cmp = icmp eq <8 x i32> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x float> %shuf, <8 x float> zeroinitializer
-  ret <8 x float> %res
-}
-
-define <8 x float> @test_8xfloat_unpack_low_mem_mask3(<8 x float> %vec1, <8 x float>* %vec2p) {
-; GENERIC-LABEL: test_8xfloat_unpack_low_mem_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vunpcklps {{.*#+}} ymm0 = ymm0[0],mem[0],ymm0[1],mem[1],ymm0[4],mem[4],ymm0[5],mem[5] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xfloat_unpack_low_mem_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vunpcklps {{.*#+}} ymm0 = ymm0[0],mem[0],ymm0[1],mem[1],ymm0[4],mem[4],ymm0[5],mem[5] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <8 x float>, <8 x float>* %vec2p
-  %res = shufflevector <8 x float> %vec1, <8 x float> %vec2, <8 x i32> <i32 0, i32 8, i32 1, i32 9, i32 4, i32 12, i32 5, i32 13>
-  ret <8 x float> %res
-}
-define <8 x float> @test_8xfloat_masked_unpack_low_mem_mask3(<8 x float> %vec1, <8 x float>* %vec2p, <8 x float> %vec3, <8 x i32> %mask) {
-; GENERIC-LABEL: test_8xfloat_masked_unpack_low_mem_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %ymm2, %ymm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpcklps {{.*#+}} ymm1 {%k1} = ymm0[0],mem[0],ymm0[1],mem[1],ymm0[4],mem[4],ymm0[5],mem[5] sched: [8:1.00]
-; GENERIC-NEXT:    vmovaps %ymm1, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xfloat_masked_unpack_low_mem_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %ymm2, %ymm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpcklps {{.*#+}} ymm1 {%k1} = ymm0[0],mem[0],ymm0[1],mem[1],ymm0[4],mem[4],ymm0[5],mem[5] sched: [8:1.00]
-; SKX-NEXT:    vmovaps %ymm1, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <8 x float>, <8 x float>* %vec2p
-  %shuf = shufflevector <8 x float> %vec1, <8 x float> %vec2, <8 x i32> <i32 0, i32 8, i32 1, i32 9, i32 4, i32 12, i32 5, i32 13>
-  %cmp = icmp eq <8 x i32> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x float> %shuf, <8 x float> %vec3
-  ret <8 x float> %res
-}
-
-define <8 x float> @test_8xfloat_zero_masked_unpack_low_mem_mask3(<8 x float> %vec1, <8 x float>* %vec2p, <8 x i32> %mask) {
-; GENERIC-LABEL: test_8xfloat_zero_masked_unpack_low_mem_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %ymm1, %ymm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpcklps {{.*#+}} ymm0 {%k1} {z} = ymm0[0],mem[0],ymm0[1],mem[1],ymm0[4],mem[4],ymm0[5],mem[5] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xfloat_zero_masked_unpack_low_mem_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %ymm1, %ymm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpcklps {{.*#+}} ymm0 {%k1} {z} = ymm0[0],mem[0],ymm0[1],mem[1],ymm0[4],mem[4],ymm0[5],mem[5] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <8 x float>, <8 x float>* %vec2p
-  %shuf = shufflevector <8 x float> %vec1, <8 x float> %vec2, <8 x i32> <i32 0, i32 8, i32 1, i32 9, i32 4, i32 12, i32 5, i32 13>
-  %cmp = icmp eq <8 x i32> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x float> %shuf, <8 x float> zeroinitializer
-  ret <8 x float> %res
-}
-
-define <16 x float> @test_16xfloat_unpack_low_mask0(<16 x float> %vec1, <16 x float> %vec2) {
-; GENERIC-LABEL: test_16xfloat_unpack_low_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vunpcklps {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[1],zmm1[1],zmm0[4],zmm1[4],zmm0[5],zmm1[5],zmm0[8],zmm1[8],zmm0[9],zmm1[9],zmm0[12],zmm1[12],zmm0[13],zmm1[13] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_16xfloat_unpack_low_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vunpcklps {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[1],zmm1[1],zmm0[4],zmm1[4],zmm0[5],zmm1[5],zmm0[8],zmm1[8],zmm0[9],zmm1[9],zmm0[12],zmm1[12],zmm0[13],zmm1[13] sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %res = shufflevector <16 x float> %vec1, <16 x float> %vec2, <16 x i32> <i32 0, i32 16, i32 1, i32 17, i32 4, i32 20, i32 5, i32 21, i32 8, i32 24, i32 9, i32 25, i32 12, i32 28, i32 13, i32 29>
-  ret <16 x float> %res
-}
-define <16 x float> @test_16xfloat_masked_unpack_low_mask0(<16 x float> %vec1, <16 x float> %vec2, <16 x float> %vec3, <16 x i32> %mask) {
-; GENERIC-LABEL: test_16xfloat_masked_unpack_low_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %zmm3, %zmm3, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpcklps {{.*#+}} zmm2 {%k1} = zmm0[0],zmm1[0],zmm0[1],zmm1[1],zmm0[4],zmm1[4],zmm0[5],zmm1[5],zmm0[8],zmm1[8],zmm0[9],zmm1[9],zmm0[12],zmm1[12],zmm0[13],zmm1[13] sched: [1:1.00]
-; GENERIC-NEXT:    vmovaps %zmm2, %zmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_16xfloat_masked_unpack_low_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %zmm3, %zmm3, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpcklps {{.*#+}} zmm2 {%k1} = zmm0[0],zmm1[0],zmm0[1],zmm1[1],zmm0[4],zmm1[4],zmm0[5],zmm1[5],zmm0[8],zmm1[8],zmm0[9],zmm1[9],zmm0[12],zmm1[12],zmm0[13],zmm1[13] sched: [1:1.00]
-; SKX-NEXT:    vmovaps %zmm2, %zmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <16 x float> %vec1, <16 x float> %vec2, <16 x i32> <i32 0, i32 16, i32 1, i32 17, i32 4, i32 20, i32 5, i32 21, i32 8, i32 24, i32 9, i32 25, i32 12, i32 28, i32 13, i32 29>
-  %cmp = icmp eq <16 x i32> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x float> %shuf, <16 x float> %vec3
-  ret <16 x float> %res
-}
-
-define <16 x float> @test_16xfloat_zero_masked_unpack_low_mask0(<16 x float> %vec1, <16 x float> %vec2, <16 x i32> %mask) {
-; GENERIC-LABEL: test_16xfloat_zero_masked_unpack_low_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %zmm2, %zmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpcklps {{.*#+}} zmm0 {%k1} {z} = zmm0[0],zmm1[0],zmm0[1],zmm1[1],zmm0[4],zmm1[4],zmm0[5],zmm1[5],zmm0[8],zmm1[8],zmm0[9],zmm1[9],zmm0[12],zmm1[12],zmm0[13],zmm1[13] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_16xfloat_zero_masked_unpack_low_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %zmm2, %zmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpcklps {{.*#+}} zmm0 {%k1} {z} = zmm0[0],zmm1[0],zmm0[1],zmm1[1],zmm0[4],zmm1[4],zmm0[5],zmm1[5],zmm0[8],zmm1[8],zmm0[9],zmm1[9],zmm0[12],zmm1[12],zmm0[13],zmm1[13] sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <16 x float> %vec1, <16 x float> %vec2, <16 x i32> <i32 0, i32 16, i32 1, i32 17, i32 4, i32 20, i32 5, i32 21, i32 8, i32 24, i32 9, i32 25, i32 12, i32 28, i32 13, i32 29>
-  %cmp = icmp eq <16 x i32> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x float> %shuf, <16 x float> zeroinitializer
-  ret <16 x float> %res
-}
-define <16 x float> @test_16xfloat_masked_unpack_low_mask1(<16 x float> %vec1, <16 x float> %vec2, <16 x float> %vec3, <16 x i32> %mask) {
-; GENERIC-LABEL: test_16xfloat_masked_unpack_low_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %zmm3, %zmm3, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpcklps {{.*#+}} zmm2 {%k1} = zmm0[0],zmm1[0],zmm0[1],zmm1[1],zmm0[4],zmm1[4],zmm0[5],zmm1[5],zmm0[8],zmm1[8],zmm0[9],zmm1[9],zmm0[12],zmm1[12],zmm0[13],zmm1[13] sched: [1:1.00]
-; GENERIC-NEXT:    vmovaps %zmm2, %zmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_16xfloat_masked_unpack_low_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %zmm3, %zmm3, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpcklps {{.*#+}} zmm2 {%k1} = zmm0[0],zmm1[0],zmm0[1],zmm1[1],zmm0[4],zmm1[4],zmm0[5],zmm1[5],zmm0[8],zmm1[8],zmm0[9],zmm1[9],zmm0[12],zmm1[12],zmm0[13],zmm1[13] sched: [1:1.00]
-; SKX-NEXT:    vmovaps %zmm2, %zmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <16 x float> %vec1, <16 x float> %vec2, <16 x i32> <i32 0, i32 16, i32 1, i32 17, i32 4, i32 20, i32 5, i32 21, i32 8, i32 24, i32 9, i32 25, i32 12, i32 28, i32 13, i32 29>
-  %cmp = icmp eq <16 x i32> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x float> %shuf, <16 x float> %vec3
-  ret <16 x float> %res
-}
-
-define <16 x float> @test_16xfloat_zero_masked_unpack_low_mask1(<16 x float> %vec1, <16 x float> %vec2, <16 x i32> %mask) {
-; GENERIC-LABEL: test_16xfloat_zero_masked_unpack_low_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %zmm2, %zmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpcklps {{.*#+}} zmm0 {%k1} {z} = zmm0[0],zmm1[0],zmm0[1],zmm1[1],zmm0[4],zmm1[4],zmm0[5],zmm1[5],zmm0[8],zmm1[8],zmm0[9],zmm1[9],zmm0[12],zmm1[12],zmm0[13],zmm1[13] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_16xfloat_zero_masked_unpack_low_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %zmm2, %zmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpcklps {{.*#+}} zmm0 {%k1} {z} = zmm0[0],zmm1[0],zmm0[1],zmm1[1],zmm0[4],zmm1[4],zmm0[5],zmm1[5],zmm0[8],zmm1[8],zmm0[9],zmm1[9],zmm0[12],zmm1[12],zmm0[13],zmm1[13] sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <16 x float> %vec1, <16 x float> %vec2, <16 x i32> <i32 0, i32 16, i32 1, i32 17, i32 4, i32 20, i32 5, i32 21, i32 8, i32 24, i32 9, i32 25, i32 12, i32 28, i32 13, i32 29>
-  %cmp = icmp eq <16 x i32> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x float> %shuf, <16 x float> zeroinitializer
-  ret <16 x float> %res
-}
-define <16 x float> @test_16xfloat_masked_unpack_low_mask2(<16 x float> %vec1, <16 x float> %vec2, <16 x float> %vec3, <16 x i32> %mask) {
-; GENERIC-LABEL: test_16xfloat_masked_unpack_low_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %zmm3, %zmm3, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpcklps {{.*#+}} zmm2 {%k1} = zmm0[0],zmm1[0],zmm0[1],zmm1[1],zmm0[4],zmm1[4],zmm0[5],zmm1[5],zmm0[8],zmm1[8],zmm0[9],zmm1[9],zmm0[12],zmm1[12],zmm0[13],zmm1[13] sched: [1:1.00]
-; GENERIC-NEXT:    vmovaps %zmm2, %zmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_16xfloat_masked_unpack_low_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %zmm3, %zmm3, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpcklps {{.*#+}} zmm2 {%k1} = zmm0[0],zmm1[0],zmm0[1],zmm1[1],zmm0[4],zmm1[4],zmm0[5],zmm1[5],zmm0[8],zmm1[8],zmm0[9],zmm1[9],zmm0[12],zmm1[12],zmm0[13],zmm1[13] sched: [1:1.00]
-; SKX-NEXT:    vmovaps %zmm2, %zmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <16 x float> %vec1, <16 x float> %vec2, <16 x i32> <i32 0, i32 16, i32 1, i32 17, i32 4, i32 20, i32 5, i32 21, i32 8, i32 24, i32 9, i32 25, i32 12, i32 28, i32 13, i32 29>
-  %cmp = icmp eq <16 x i32> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x float> %shuf, <16 x float> %vec3
-  ret <16 x float> %res
-}
-
-define <16 x float> @test_16xfloat_zero_masked_unpack_low_mask2(<16 x float> %vec1, <16 x float> %vec2, <16 x i32> %mask) {
-; GENERIC-LABEL: test_16xfloat_zero_masked_unpack_low_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %zmm2, %zmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpcklps {{.*#+}} zmm0 {%k1} {z} = zmm0[0],zmm1[0],zmm0[1],zmm1[1],zmm0[4],zmm1[4],zmm0[5],zmm1[5],zmm0[8],zmm1[8],zmm0[9],zmm1[9],zmm0[12],zmm1[12],zmm0[13],zmm1[13] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_16xfloat_zero_masked_unpack_low_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %zmm2, %zmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpcklps {{.*#+}} zmm0 {%k1} {z} = zmm0[0],zmm1[0],zmm0[1],zmm1[1],zmm0[4],zmm1[4],zmm0[5],zmm1[5],zmm0[8],zmm1[8],zmm0[9],zmm1[9],zmm0[12],zmm1[12],zmm0[13],zmm1[13] sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <16 x float> %vec1, <16 x float> %vec2, <16 x i32> <i32 0, i32 16, i32 1, i32 17, i32 4, i32 20, i32 5, i32 21, i32 8, i32 24, i32 9, i32 25, i32 12, i32 28, i32 13, i32 29>
-  %cmp = icmp eq <16 x i32> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x float> %shuf, <16 x float> zeroinitializer
-  ret <16 x float> %res
-}
-define <16 x float> @test_16xfloat_unpack_low_mask3(<16 x float> %vec1, <16 x float> %vec2) {
-; GENERIC-LABEL: test_16xfloat_unpack_low_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vunpcklps {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[1],zmm1[1],zmm0[4],zmm1[4],zmm0[5],zmm1[5],zmm0[8],zmm1[8],zmm0[9],zmm1[9],zmm0[12],zmm1[12],zmm0[13],zmm1[13] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_16xfloat_unpack_low_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vunpcklps {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[1],zmm1[1],zmm0[4],zmm1[4],zmm0[5],zmm1[5],zmm0[8],zmm1[8],zmm0[9],zmm1[9],zmm0[12],zmm1[12],zmm0[13],zmm1[13] sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %res = shufflevector <16 x float> %vec1, <16 x float> %vec2, <16 x i32> <i32 0, i32 16, i32 1, i32 17, i32 4, i32 20, i32 5, i32 21, i32 8, i32 24, i32 9, i32 25, i32 12, i32 28, i32 13, i32 29>
-  ret <16 x float> %res
-}
-define <16 x float> @test_16xfloat_masked_unpack_low_mask3(<16 x float> %vec1, <16 x float> %vec2, <16 x float> %vec3, <16 x i32> %mask) {
-; GENERIC-LABEL: test_16xfloat_masked_unpack_low_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %zmm3, %zmm3, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpcklps {{.*#+}} zmm2 {%k1} = zmm0[0],zmm1[0],zmm0[1],zmm1[1],zmm0[4],zmm1[4],zmm0[5],zmm1[5],zmm0[8],zmm1[8],zmm0[9],zmm1[9],zmm0[12],zmm1[12],zmm0[13],zmm1[13] sched: [1:1.00]
-; GENERIC-NEXT:    vmovaps %zmm2, %zmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_16xfloat_masked_unpack_low_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %zmm3, %zmm3, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpcklps {{.*#+}} zmm2 {%k1} = zmm0[0],zmm1[0],zmm0[1],zmm1[1],zmm0[4],zmm1[4],zmm0[5],zmm1[5],zmm0[8],zmm1[8],zmm0[9],zmm1[9],zmm0[12],zmm1[12],zmm0[13],zmm1[13] sched: [1:1.00]
-; SKX-NEXT:    vmovaps %zmm2, %zmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <16 x float> %vec1, <16 x float> %vec2, <16 x i32> <i32 0, i32 16, i32 1, i32 17, i32 4, i32 20, i32 5, i32 21, i32 8, i32 24, i32 9, i32 25, i32 12, i32 28, i32 13, i32 29>
-  %cmp = icmp eq <16 x i32> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x float> %shuf, <16 x float> %vec3
-  ret <16 x float> %res
-}
-
-define <16 x float> @test_16xfloat_zero_masked_unpack_low_mask3(<16 x float> %vec1, <16 x float> %vec2, <16 x i32> %mask) {
-; GENERIC-LABEL: test_16xfloat_zero_masked_unpack_low_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %zmm2, %zmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpcklps {{.*#+}} zmm0 {%k1} {z} = zmm0[0],zmm1[0],zmm0[1],zmm1[1],zmm0[4],zmm1[4],zmm0[5],zmm1[5],zmm0[8],zmm1[8],zmm0[9],zmm1[9],zmm0[12],zmm1[12],zmm0[13],zmm1[13] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_16xfloat_zero_masked_unpack_low_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %zmm2, %zmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpcklps {{.*#+}} zmm0 {%k1} {z} = zmm0[0],zmm1[0],zmm0[1],zmm1[1],zmm0[4],zmm1[4],zmm0[5],zmm1[5],zmm0[8],zmm1[8],zmm0[9],zmm1[9],zmm0[12],zmm1[12],zmm0[13],zmm1[13] sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <16 x float> %vec1, <16 x float> %vec2, <16 x i32> <i32 0, i32 16, i32 1, i32 17, i32 4, i32 20, i32 5, i32 21, i32 8, i32 24, i32 9, i32 25, i32 12, i32 28, i32 13, i32 29>
-  %cmp = icmp eq <16 x i32> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x float> %shuf, <16 x float> zeroinitializer
-  ret <16 x float> %res
-}
-define <16 x float> @test_16xfloat_unpack_low_mem_mask0(<16 x float> %vec1, <16 x float>* %vec2p) {
-; GENERIC-LABEL: test_16xfloat_unpack_low_mem_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vunpcklps {{.*#+}} zmm0 = zmm0[0],mem[0],zmm0[1],mem[1],zmm0[4],mem[4],zmm0[5],mem[5],zmm0[8],mem[8],zmm0[9],mem[9],zmm0[12],mem[12],zmm0[13],mem[13] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_16xfloat_unpack_low_mem_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vunpcklps {{.*#+}} zmm0 = zmm0[0],mem[0],zmm0[1],mem[1],zmm0[4],mem[4],zmm0[5],mem[5],zmm0[8],mem[8],zmm0[9],mem[9],zmm0[12],mem[12],zmm0[13],mem[13] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <16 x float>, <16 x float>* %vec2p
-  %res = shufflevector <16 x float> %vec1, <16 x float> %vec2, <16 x i32> <i32 0, i32 16, i32 1, i32 17, i32 4, i32 20, i32 5, i32 21, i32 8, i32 24, i32 9, i32 25, i32 12, i32 28, i32 13, i32 29>
-  ret <16 x float> %res
-}
-define <16 x float> @test_16xfloat_masked_unpack_low_mem_mask0(<16 x float> %vec1, <16 x float>* %vec2p, <16 x float> %vec3, <16 x i32> %mask) {
-; GENERIC-LABEL: test_16xfloat_masked_unpack_low_mem_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %zmm2, %zmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpcklps {{.*#+}} zmm1 {%k1} = zmm0[0],mem[0],zmm0[1],mem[1],zmm0[4],mem[4],zmm0[5],mem[5],zmm0[8],mem[8],zmm0[9],mem[9],zmm0[12],mem[12],zmm0[13],mem[13] sched: [8:1.00]
-; GENERIC-NEXT:    vmovaps %zmm1, %zmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_16xfloat_masked_unpack_low_mem_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %zmm2, %zmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpcklps {{.*#+}} zmm1 {%k1} = zmm0[0],mem[0],zmm0[1],mem[1],zmm0[4],mem[4],zmm0[5],mem[5],zmm0[8],mem[8],zmm0[9],mem[9],zmm0[12],mem[12],zmm0[13],mem[13] sched: [8:1.00]
-; SKX-NEXT:    vmovaps %zmm1, %zmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <16 x float>, <16 x float>* %vec2p
-  %shuf = shufflevector <16 x float> %vec1, <16 x float> %vec2, <16 x i32> <i32 0, i32 16, i32 1, i32 17, i32 4, i32 20, i32 5, i32 21, i32 8, i32 24, i32 9, i32 25, i32 12, i32 28, i32 13, i32 29>
-  %cmp = icmp eq <16 x i32> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x float> %shuf, <16 x float> %vec3
-  ret <16 x float> %res
-}
-
-define <16 x float> @test_16xfloat_zero_masked_unpack_low_mem_mask0(<16 x float> %vec1, <16 x float>* %vec2p, <16 x i32> %mask) {
-; GENERIC-LABEL: test_16xfloat_zero_masked_unpack_low_mem_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpcklps {{.*#+}} zmm0 {%k1} {z} = zmm0[0],mem[0],zmm0[1],mem[1],zmm0[4],mem[4],zmm0[5],mem[5],zmm0[8],mem[8],zmm0[9],mem[9],zmm0[12],mem[12],zmm0[13],mem[13] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_16xfloat_zero_masked_unpack_low_mem_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpcklps {{.*#+}} zmm0 {%k1} {z} = zmm0[0],mem[0],zmm0[1],mem[1],zmm0[4],mem[4],zmm0[5],mem[5],zmm0[8],mem[8],zmm0[9],mem[9],zmm0[12],mem[12],zmm0[13],mem[13] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <16 x float>, <16 x float>* %vec2p
-  %shuf = shufflevector <16 x float> %vec1, <16 x float> %vec2, <16 x i32> <i32 0, i32 16, i32 1, i32 17, i32 4, i32 20, i32 5, i32 21, i32 8, i32 24, i32 9, i32 25, i32 12, i32 28, i32 13, i32 29>
-  %cmp = icmp eq <16 x i32> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x float> %shuf, <16 x float> zeroinitializer
-  ret <16 x float> %res
-}
-
-define <16 x float> @test_16xfloat_masked_unpack_low_mem_mask1(<16 x float> %vec1, <16 x float>* %vec2p, <16 x float> %vec3, <16 x i32> %mask) {
-; GENERIC-LABEL: test_16xfloat_masked_unpack_low_mem_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %zmm2, %zmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpcklps {{.*#+}} zmm1 {%k1} = zmm0[0],mem[0],zmm0[1],mem[1],zmm0[4],mem[4],zmm0[5],mem[5],zmm0[8],mem[8],zmm0[9],mem[9],zmm0[12],mem[12],zmm0[13],mem[13] sched: [8:1.00]
-; GENERIC-NEXT:    vmovaps %zmm1, %zmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_16xfloat_masked_unpack_low_mem_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %zmm2, %zmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpcklps {{.*#+}} zmm1 {%k1} = zmm0[0],mem[0],zmm0[1],mem[1],zmm0[4],mem[4],zmm0[5],mem[5],zmm0[8],mem[8],zmm0[9],mem[9],zmm0[12],mem[12],zmm0[13],mem[13] sched: [8:1.00]
-; SKX-NEXT:    vmovaps %zmm1, %zmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <16 x float>, <16 x float>* %vec2p
-  %shuf = shufflevector <16 x float> %vec1, <16 x float> %vec2, <16 x i32> <i32 0, i32 16, i32 1, i32 17, i32 4, i32 20, i32 5, i32 21, i32 8, i32 24, i32 9, i32 25, i32 12, i32 28, i32 13, i32 29>
-  %cmp = icmp eq <16 x i32> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x float> %shuf, <16 x float> %vec3
-  ret <16 x float> %res
-}
-
-define <16 x float> @test_16xfloat_zero_masked_unpack_low_mem_mask1(<16 x float> %vec1, <16 x float>* %vec2p, <16 x i32> %mask) {
-; GENERIC-LABEL: test_16xfloat_zero_masked_unpack_low_mem_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpcklps {{.*#+}} zmm0 {%k1} {z} = zmm0[0],mem[0],zmm0[1],mem[1],zmm0[4],mem[4],zmm0[5],mem[5],zmm0[8],mem[8],zmm0[9],mem[9],zmm0[12],mem[12],zmm0[13],mem[13] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_16xfloat_zero_masked_unpack_low_mem_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpcklps {{.*#+}} zmm0 {%k1} {z} = zmm0[0],mem[0],zmm0[1],mem[1],zmm0[4],mem[4],zmm0[5],mem[5],zmm0[8],mem[8],zmm0[9],mem[9],zmm0[12],mem[12],zmm0[13],mem[13] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <16 x float>, <16 x float>* %vec2p
-  %shuf = shufflevector <16 x float> %vec1, <16 x float> %vec2, <16 x i32> <i32 0, i32 16, i32 1, i32 17, i32 4, i32 20, i32 5, i32 21, i32 8, i32 24, i32 9, i32 25, i32 12, i32 28, i32 13, i32 29>
-  %cmp = icmp eq <16 x i32> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x float> %shuf, <16 x float> zeroinitializer
-  ret <16 x float> %res
-}
-
-define <16 x float> @test_16xfloat_masked_unpack_low_mem_mask2(<16 x float> %vec1, <16 x float>* %vec2p, <16 x float> %vec3, <16 x i32> %mask) {
-; GENERIC-LABEL: test_16xfloat_masked_unpack_low_mem_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %zmm2, %zmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpcklps {{.*#+}} zmm1 {%k1} = zmm0[0],mem[0],zmm0[1],mem[1],zmm0[4],mem[4],zmm0[5],mem[5],zmm0[8],mem[8],zmm0[9],mem[9],zmm0[12],mem[12],zmm0[13],mem[13] sched: [8:1.00]
-; GENERIC-NEXT:    vmovaps %zmm1, %zmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_16xfloat_masked_unpack_low_mem_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %zmm2, %zmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpcklps {{.*#+}} zmm1 {%k1} = zmm0[0],mem[0],zmm0[1],mem[1],zmm0[4],mem[4],zmm0[5],mem[5],zmm0[8],mem[8],zmm0[9],mem[9],zmm0[12],mem[12],zmm0[13],mem[13] sched: [8:1.00]
-; SKX-NEXT:    vmovaps %zmm1, %zmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <16 x float>, <16 x float>* %vec2p
-  %shuf = shufflevector <16 x float> %vec1, <16 x float> %vec2, <16 x i32> <i32 0, i32 16, i32 1, i32 17, i32 4, i32 20, i32 5, i32 21, i32 8, i32 24, i32 9, i32 25, i32 12, i32 28, i32 13, i32 29>
-  %cmp = icmp eq <16 x i32> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x float> %shuf, <16 x float> %vec3
-  ret <16 x float> %res
-}
-
-define <16 x float> @test_16xfloat_zero_masked_unpack_low_mem_mask2(<16 x float> %vec1, <16 x float>* %vec2p, <16 x i32> %mask) {
-; GENERIC-LABEL: test_16xfloat_zero_masked_unpack_low_mem_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpcklps {{.*#+}} zmm0 {%k1} {z} = zmm0[0],mem[0],zmm0[1],mem[1],zmm0[4],mem[4],zmm0[5],mem[5],zmm0[8],mem[8],zmm0[9],mem[9],zmm0[12],mem[12],zmm0[13],mem[13] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_16xfloat_zero_masked_unpack_low_mem_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpcklps {{.*#+}} zmm0 {%k1} {z} = zmm0[0],mem[0],zmm0[1],mem[1],zmm0[4],mem[4],zmm0[5],mem[5],zmm0[8],mem[8],zmm0[9],mem[9],zmm0[12],mem[12],zmm0[13],mem[13] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <16 x float>, <16 x float>* %vec2p
-  %shuf = shufflevector <16 x float> %vec1, <16 x float> %vec2, <16 x i32> <i32 0, i32 16, i32 1, i32 17, i32 4, i32 20, i32 5, i32 21, i32 8, i32 24, i32 9, i32 25, i32 12, i32 28, i32 13, i32 29>
-  %cmp = icmp eq <16 x i32> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x float> %shuf, <16 x float> zeroinitializer
-  ret <16 x float> %res
-}
-
-define <16 x float> @test_16xfloat_unpack_low_mem_mask3(<16 x float> %vec1, <16 x float>* %vec2p) {
-; GENERIC-LABEL: test_16xfloat_unpack_low_mem_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vunpcklps {{.*#+}} zmm0 = zmm0[0],mem[0],zmm0[1],mem[1],zmm0[4],mem[4],zmm0[5],mem[5],zmm0[8],mem[8],zmm0[9],mem[9],zmm0[12],mem[12],zmm0[13],mem[13] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_16xfloat_unpack_low_mem_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vunpcklps {{.*#+}} zmm0 = zmm0[0],mem[0],zmm0[1],mem[1],zmm0[4],mem[4],zmm0[5],mem[5],zmm0[8],mem[8],zmm0[9],mem[9],zmm0[12],mem[12],zmm0[13],mem[13] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <16 x float>, <16 x float>* %vec2p
-  %res = shufflevector <16 x float> %vec1, <16 x float> %vec2, <16 x i32> <i32 0, i32 16, i32 1, i32 17, i32 4, i32 20, i32 5, i32 21, i32 8, i32 24, i32 9, i32 25, i32 12, i32 28, i32 13, i32 29>
-  ret <16 x float> %res
-}
-define <16 x float> @test_16xfloat_masked_unpack_low_mem_mask3(<16 x float> %vec1, <16 x float>* %vec2p, <16 x float> %vec3, <16 x i32> %mask) {
-; GENERIC-LABEL: test_16xfloat_masked_unpack_low_mem_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %zmm2, %zmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpcklps {{.*#+}} zmm1 {%k1} = zmm0[0],mem[0],zmm0[1],mem[1],zmm0[4],mem[4],zmm0[5],mem[5],zmm0[8],mem[8],zmm0[9],mem[9],zmm0[12],mem[12],zmm0[13],mem[13] sched: [8:1.00]
-; GENERIC-NEXT:    vmovaps %zmm1, %zmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_16xfloat_masked_unpack_low_mem_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %zmm2, %zmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpcklps {{.*#+}} zmm1 {%k1} = zmm0[0],mem[0],zmm0[1],mem[1],zmm0[4],mem[4],zmm0[5],mem[5],zmm0[8],mem[8],zmm0[9],mem[9],zmm0[12],mem[12],zmm0[13],mem[13] sched: [8:1.00]
-; SKX-NEXT:    vmovaps %zmm1, %zmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <16 x float>, <16 x float>* %vec2p
-  %shuf = shufflevector <16 x float> %vec1, <16 x float> %vec2, <16 x i32> <i32 0, i32 16, i32 1, i32 17, i32 4, i32 20, i32 5, i32 21, i32 8, i32 24, i32 9, i32 25, i32 12, i32 28, i32 13, i32 29>
-  %cmp = icmp eq <16 x i32> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x float> %shuf, <16 x float> %vec3
-  ret <16 x float> %res
-}
-
-define <16 x float> @test_16xfloat_zero_masked_unpack_low_mem_mask3(<16 x float> %vec1, <16 x float>* %vec2p, <16 x i32> %mask) {
-; GENERIC-LABEL: test_16xfloat_zero_masked_unpack_low_mem_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpcklps {{.*#+}} zmm0 {%k1} {z} = zmm0[0],mem[0],zmm0[1],mem[1],zmm0[4],mem[4],zmm0[5],mem[5],zmm0[8],mem[8],zmm0[9],mem[9],zmm0[12],mem[12],zmm0[13],mem[13] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_16xfloat_zero_masked_unpack_low_mem_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpcklps {{.*#+}} zmm0 {%k1} {z} = zmm0[0],mem[0],zmm0[1],mem[1],zmm0[4],mem[4],zmm0[5],mem[5],zmm0[8],mem[8],zmm0[9],mem[9],zmm0[12],mem[12],zmm0[13],mem[13] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <16 x float>, <16 x float>* %vec2p
-  %shuf = shufflevector <16 x float> %vec1, <16 x float> %vec2, <16 x i32> <i32 0, i32 16, i32 1, i32 17, i32 4, i32 20, i32 5, i32 21, i32 8, i32 24, i32 9, i32 25, i32 12, i32 28, i32 13, i32 29>
-  %cmp = icmp eq <16 x i32> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x float> %shuf, <16 x float> zeroinitializer
-  ret <16 x float> %res
-}
-
-define <2 x double> @test_2xdouble_unpack_low_mask0(<2 x double> %vec1, <2 x double> %vec2) {
-; GENERIC-LABEL: test_2xdouble_unpack_low_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_2xdouble_unpack_low_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0] sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %res = shufflevector <2 x double> %vec1, <2 x double> %vec2, <2 x i32> <i32 0, i32 2>
-  ret <2 x double> %res
-}
-define <2 x double> @test_2xdouble_masked_unpack_low_mask0(<2 x double> %vec1, <2 x double> %vec2, <2 x double> %vec3, <2 x i64> %mask) {
-; GENERIC-LABEL: test_2xdouble_masked_unpack_low_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %xmm3, %xmm3, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpcklpd {{.*#+}} xmm2 {%k1} = xmm0[0],xmm1[0] sched: [1:1.00]
-; GENERIC-NEXT:    vmovapd %xmm2, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_2xdouble_masked_unpack_low_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %xmm3, %xmm3, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpcklpd {{.*#+}} xmm2 {%k1} = xmm0[0],xmm1[0] sched: [1:1.00]
-; SKX-NEXT:    vmovapd %xmm2, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <2 x double> %vec1, <2 x double> %vec2, <2 x i32> <i32 0, i32 2>
-  %cmp = icmp eq <2 x i64> %mask, zeroinitializer
-  %res = select <2 x i1> %cmp, <2 x double> %shuf, <2 x double> %vec3
-  ret <2 x double> %res
-}
-
-define <2 x double> @test_2xdouble_zero_masked_unpack_low_mask0(<2 x double> %vec1, <2 x double> %vec2, <2 x i64> %mask) {
-; GENERIC-LABEL: test_2xdouble_zero_masked_unpack_low_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %xmm2, %xmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpcklpd {{.*#+}} xmm0 {%k1} {z} = xmm0[0],xmm1[0] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_2xdouble_zero_masked_unpack_low_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %xmm2, %xmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpcklpd {{.*#+}} xmm0 {%k1} {z} = xmm0[0],xmm1[0] sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <2 x double> %vec1, <2 x double> %vec2, <2 x i32> <i32 0, i32 2>
-  %cmp = icmp eq <2 x i64> %mask, zeroinitializer
-  %res = select <2 x i1> %cmp, <2 x double> %shuf, <2 x double> zeroinitializer
-  ret <2 x double> %res
-}
-define <2 x double> @test_2xdouble_masked_unpack_low_mask1(<2 x double> %vec1, <2 x double> %vec2, <2 x double> %vec3, <2 x i64> %mask) {
-; GENERIC-LABEL: test_2xdouble_masked_unpack_low_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %xmm3, %xmm3, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpcklpd {{.*#+}} xmm2 {%k1} = xmm0[0],xmm1[0] sched: [1:1.00]
-; GENERIC-NEXT:    vmovapd %xmm2, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_2xdouble_masked_unpack_low_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %xmm3, %xmm3, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpcklpd {{.*#+}} xmm2 {%k1} = xmm0[0],xmm1[0] sched: [1:1.00]
-; SKX-NEXT:    vmovapd %xmm2, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <2 x double> %vec1, <2 x double> %vec2, <2 x i32> <i32 0, i32 2>
-  %cmp = icmp eq <2 x i64> %mask, zeroinitializer
-  %res = select <2 x i1> %cmp, <2 x double> %shuf, <2 x double> %vec3
-  ret <2 x double> %res
-}
-
-define <2 x double> @test_2xdouble_zero_masked_unpack_low_mask1(<2 x double> %vec1, <2 x double> %vec2, <2 x i64> %mask) {
-; GENERIC-LABEL: test_2xdouble_zero_masked_unpack_low_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %xmm2, %xmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpcklpd {{.*#+}} xmm0 {%k1} {z} = xmm0[0],xmm1[0] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_2xdouble_zero_masked_unpack_low_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %xmm2, %xmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpcklpd {{.*#+}} xmm0 {%k1} {z} = xmm0[0],xmm1[0] sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <2 x double> %vec1, <2 x double> %vec2, <2 x i32> <i32 0, i32 2>
-  %cmp = icmp eq <2 x i64> %mask, zeroinitializer
-  %res = select <2 x i1> %cmp, <2 x double> %shuf, <2 x double> zeroinitializer
-  ret <2 x double> %res
-}
-define <2 x double> @test_2xdouble_unpack_low_mem_mask0(<2 x double> %vec1, <2 x double>* %vec2p) {
-; GENERIC-LABEL: test_2xdouble_unpack_low_mem_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vunpcklpd {{.*#+}} xmm0 = xmm0[0],mem[0] sched: [7:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_2xdouble_unpack_low_mem_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vunpcklpd {{.*#+}} xmm0 = xmm0[0],mem[0] sched: [7:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <2 x double>, <2 x double>* %vec2p
-  %res = shufflevector <2 x double> %vec1, <2 x double> %vec2, <2 x i32> <i32 0, i32 2>
-  ret <2 x double> %res
-}
-define <2 x double> @test_2xdouble_masked_unpack_low_mem_mask0(<2 x double> %vec1, <2 x double>* %vec2p, <2 x double> %vec3, <2 x i64> %mask) {
-; GENERIC-LABEL: test_2xdouble_masked_unpack_low_mem_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %xmm2, %xmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpcklpd {{.*#+}} xmm1 {%k1} = xmm0[0],mem[0] sched: [7:1.00]
-; GENERIC-NEXT:    vmovapd %xmm1, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_2xdouble_masked_unpack_low_mem_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %xmm2, %xmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpcklpd {{.*#+}} xmm1 {%k1} = xmm0[0],mem[0] sched: [7:1.00]
-; SKX-NEXT:    vmovapd %xmm1, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <2 x double>, <2 x double>* %vec2p
-  %shuf = shufflevector <2 x double> %vec1, <2 x double> %vec2, <2 x i32> <i32 0, i32 2>
-  %cmp = icmp eq <2 x i64> %mask, zeroinitializer
-  %res = select <2 x i1> %cmp, <2 x double> %shuf, <2 x double> %vec3
-  ret <2 x double> %res
-}
-
-define <2 x double> @test_2xdouble_zero_masked_unpack_low_mem_mask0(<2 x double> %vec1, <2 x double>* %vec2p, <2 x i64> %mask) {
-; GENERIC-LABEL: test_2xdouble_zero_masked_unpack_low_mem_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %xmm1, %xmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpcklpd {{.*#+}} xmm0 {%k1} {z} = xmm0[0],mem[0] sched: [7:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_2xdouble_zero_masked_unpack_low_mem_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %xmm1, %xmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpcklpd {{.*#+}} xmm0 {%k1} {z} = xmm0[0],mem[0] sched: [7:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <2 x double>, <2 x double>* %vec2p
-  %shuf = shufflevector <2 x double> %vec1, <2 x double> %vec2, <2 x i32> <i32 0, i32 2>
-  %cmp = icmp eq <2 x i64> %mask, zeroinitializer
-  %res = select <2 x i1> %cmp, <2 x double> %shuf, <2 x double> zeroinitializer
-  ret <2 x double> %res
-}
-
-define <2 x double> @test_2xdouble_masked_unpack_low_mem_mask1(<2 x double> %vec1, <2 x double>* %vec2p, <2 x double> %vec3, <2 x i64> %mask) {
-; GENERIC-LABEL: test_2xdouble_masked_unpack_low_mem_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %xmm2, %xmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpcklpd {{.*#+}} xmm1 {%k1} = xmm0[0],mem[0] sched: [7:1.00]
-; GENERIC-NEXT:    vmovapd %xmm1, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_2xdouble_masked_unpack_low_mem_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %xmm2, %xmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpcklpd {{.*#+}} xmm1 {%k1} = xmm0[0],mem[0] sched: [7:1.00]
-; SKX-NEXT:    vmovapd %xmm1, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <2 x double>, <2 x double>* %vec2p
-  %shuf = shufflevector <2 x double> %vec1, <2 x double> %vec2, <2 x i32> <i32 0, i32 2>
-  %cmp = icmp eq <2 x i64> %mask, zeroinitializer
-  %res = select <2 x i1> %cmp, <2 x double> %shuf, <2 x double> %vec3
-  ret <2 x double> %res
-}
-
-define <2 x double> @test_2xdouble_zero_masked_unpack_low_mem_mask1(<2 x double> %vec1, <2 x double>* %vec2p, <2 x i64> %mask) {
-; GENERIC-LABEL: test_2xdouble_zero_masked_unpack_low_mem_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %xmm1, %xmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpcklpd {{.*#+}} xmm0 {%k1} {z} = xmm0[0],mem[0] sched: [7:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_2xdouble_zero_masked_unpack_low_mem_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %xmm1, %xmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpcklpd {{.*#+}} xmm0 {%k1} {z} = xmm0[0],mem[0] sched: [7:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <2 x double>, <2 x double>* %vec2p
-  %shuf = shufflevector <2 x double> %vec1, <2 x double> %vec2, <2 x i32> <i32 0, i32 2>
-  %cmp = icmp eq <2 x i64> %mask, zeroinitializer
-  %res = select <2 x i1> %cmp, <2 x double> %shuf, <2 x double> zeroinitializer
-  ret <2 x double> %res
-}
-
-define <4 x double> @test_4xdouble_unpack_low_mask0(<4 x double> %vec1, <4 x double> %vec2) {
-; GENERIC-LABEL: test_4xdouble_unpack_low_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vunpcklpd {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[2],ymm1[2] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xdouble_unpack_low_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vunpcklpd {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[2],ymm1[2] sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %res = shufflevector <4 x double> %vec1, <4 x double> %vec2, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
-  ret <4 x double> %res
-}
-define <4 x double> @test_4xdouble_masked_unpack_low_mask0(<4 x double> %vec1, <4 x double> %vec2, <4 x double> %vec3, <4 x i64> %mask) {
-; GENERIC-LABEL: test_4xdouble_masked_unpack_low_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %ymm3, %ymm3, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpcklpd {{.*#+}} ymm2 {%k1} = ymm0[0],ymm1[0],ymm0[2],ymm1[2] sched: [1:1.00]
-; GENERIC-NEXT:    vmovapd %ymm2, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xdouble_masked_unpack_low_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %ymm3, %ymm3, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpcklpd {{.*#+}} ymm2 {%k1} = ymm0[0],ymm1[0],ymm0[2],ymm1[2] sched: [1:1.00]
-; SKX-NEXT:    vmovapd %ymm2, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <4 x double> %vec1, <4 x double> %vec2, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
-  %cmp = icmp eq <4 x i64> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x double> %shuf, <4 x double> %vec3
-  ret <4 x double> %res
-}
-
-define <4 x double> @test_4xdouble_zero_masked_unpack_low_mask0(<4 x double> %vec1, <4 x double> %vec2, <4 x i64> %mask) {
-; GENERIC-LABEL: test_4xdouble_zero_masked_unpack_low_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %ymm2, %ymm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpcklpd {{.*#+}} ymm0 {%k1} {z} = ymm0[0],ymm1[0],ymm0[2],ymm1[2] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xdouble_zero_masked_unpack_low_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %ymm2, %ymm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpcklpd {{.*#+}} ymm0 {%k1} {z} = ymm0[0],ymm1[0],ymm0[2],ymm1[2] sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <4 x double> %vec1, <4 x double> %vec2, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
-  %cmp = icmp eq <4 x i64> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x double> %shuf, <4 x double> zeroinitializer
-  ret <4 x double> %res
-}
-define <4 x double> @test_4xdouble_masked_unpack_low_mask1(<4 x double> %vec1, <4 x double> %vec2, <4 x double> %vec3, <4 x i64> %mask) {
-; GENERIC-LABEL: test_4xdouble_masked_unpack_low_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %ymm3, %ymm3, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpcklpd {{.*#+}} ymm2 {%k1} = ymm0[0],ymm1[0],ymm0[2],ymm1[2] sched: [1:1.00]
-; GENERIC-NEXT:    vmovapd %ymm2, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xdouble_masked_unpack_low_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %ymm3, %ymm3, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpcklpd {{.*#+}} ymm2 {%k1} = ymm0[0],ymm1[0],ymm0[2],ymm1[2] sched: [1:1.00]
-; SKX-NEXT:    vmovapd %ymm2, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <4 x double> %vec1, <4 x double> %vec2, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
-  %cmp = icmp eq <4 x i64> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x double> %shuf, <4 x double> %vec3
-  ret <4 x double> %res
-}
-
-define <4 x double> @test_4xdouble_zero_masked_unpack_low_mask1(<4 x double> %vec1, <4 x double> %vec2, <4 x i64> %mask) {
-; GENERIC-LABEL: test_4xdouble_zero_masked_unpack_low_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %ymm2, %ymm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpcklpd {{.*#+}} ymm0 {%k1} {z} = ymm0[0],ymm1[0],ymm0[2],ymm1[2] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xdouble_zero_masked_unpack_low_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %ymm2, %ymm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpcklpd {{.*#+}} ymm0 {%k1} {z} = ymm0[0],ymm1[0],ymm0[2],ymm1[2] sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <4 x double> %vec1, <4 x double> %vec2, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
-  %cmp = icmp eq <4 x i64> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x double> %shuf, <4 x double> zeroinitializer
-  ret <4 x double> %res
-}
-define <4 x double> @test_4xdouble_masked_unpack_low_mask2(<4 x double> %vec1, <4 x double> %vec2, <4 x double> %vec3, <4 x i64> %mask) {
-; GENERIC-LABEL: test_4xdouble_masked_unpack_low_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %ymm3, %ymm3, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpcklpd {{.*#+}} ymm2 {%k1} = ymm0[0],ymm1[0],ymm0[2],ymm1[2] sched: [1:1.00]
-; GENERIC-NEXT:    vmovapd %ymm2, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xdouble_masked_unpack_low_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %ymm3, %ymm3, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpcklpd {{.*#+}} ymm2 {%k1} = ymm0[0],ymm1[0],ymm0[2],ymm1[2] sched: [1:1.00]
-; SKX-NEXT:    vmovapd %ymm2, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <4 x double> %vec1, <4 x double> %vec2, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
-  %cmp = icmp eq <4 x i64> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x double> %shuf, <4 x double> %vec3
-  ret <4 x double> %res
-}
-
-define <4 x double> @test_4xdouble_zero_masked_unpack_low_mask2(<4 x double> %vec1, <4 x double> %vec2, <4 x i64> %mask) {
-; GENERIC-LABEL: test_4xdouble_zero_masked_unpack_low_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %ymm2, %ymm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpcklpd {{.*#+}} ymm0 {%k1} {z} = ymm0[0],ymm1[0],ymm0[2],ymm1[2] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xdouble_zero_masked_unpack_low_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %ymm2, %ymm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpcklpd {{.*#+}} ymm0 {%k1} {z} = ymm0[0],ymm1[0],ymm0[2],ymm1[2] sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <4 x double> %vec1, <4 x double> %vec2, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
-  %cmp = icmp eq <4 x i64> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x double> %shuf, <4 x double> zeroinitializer
-  ret <4 x double> %res
-}
-define <4 x double> @test_4xdouble_unpack_low_mask3(<4 x double> %vec1, <4 x double> %vec2) {
-; GENERIC-LABEL: test_4xdouble_unpack_low_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vunpcklpd {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[2],ymm1[2] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xdouble_unpack_low_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vunpcklpd {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[2],ymm1[2] sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %res = shufflevector <4 x double> %vec1, <4 x double> %vec2, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
-  ret <4 x double> %res
-}
-define <4 x double> @test_4xdouble_masked_unpack_low_mask3(<4 x double> %vec1, <4 x double> %vec2, <4 x double> %vec3, <4 x i64> %mask) {
-; GENERIC-LABEL: test_4xdouble_masked_unpack_low_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %ymm3, %ymm3, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpcklpd {{.*#+}} ymm2 {%k1} = ymm0[0],ymm1[0],ymm0[2],ymm1[2] sched: [1:1.00]
-; GENERIC-NEXT:    vmovapd %ymm2, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xdouble_masked_unpack_low_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %ymm3, %ymm3, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpcklpd {{.*#+}} ymm2 {%k1} = ymm0[0],ymm1[0],ymm0[2],ymm1[2] sched: [1:1.00]
-; SKX-NEXT:    vmovapd %ymm2, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <4 x double> %vec1, <4 x double> %vec2, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
-  %cmp = icmp eq <4 x i64> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x double> %shuf, <4 x double> %vec3
-  ret <4 x double> %res
-}
-
-define <4 x double> @test_4xdouble_zero_masked_unpack_low_mask3(<4 x double> %vec1, <4 x double> %vec2, <4 x i64> %mask) {
-; GENERIC-LABEL: test_4xdouble_zero_masked_unpack_low_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %ymm2, %ymm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpcklpd {{.*#+}} ymm0 {%k1} {z} = ymm0[0],ymm1[0],ymm0[2],ymm1[2] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xdouble_zero_masked_unpack_low_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %ymm2, %ymm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpcklpd {{.*#+}} ymm0 {%k1} {z} = ymm0[0],ymm1[0],ymm0[2],ymm1[2] sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <4 x double> %vec1, <4 x double> %vec2, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
-  %cmp = icmp eq <4 x i64> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x double> %shuf, <4 x double> zeroinitializer
-  ret <4 x double> %res
-}
-define <4 x double> @test_4xdouble_unpack_low_mem_mask0(<4 x double> %vec1, <4 x double>* %vec2p) {
-; GENERIC-LABEL: test_4xdouble_unpack_low_mem_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vunpcklpd {{.*#+}} ymm0 = ymm0[0],mem[0],ymm0[2],mem[2] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xdouble_unpack_low_mem_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vunpcklpd {{.*#+}} ymm0 = ymm0[0],mem[0],ymm0[2],mem[2] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <4 x double>, <4 x double>* %vec2p
-  %res = shufflevector <4 x double> %vec1, <4 x double> %vec2, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
-  ret <4 x double> %res
-}
-define <4 x double> @test_4xdouble_masked_unpack_low_mem_mask0(<4 x double> %vec1, <4 x double>* %vec2p, <4 x double> %vec3, <4 x i64> %mask) {
-; GENERIC-LABEL: test_4xdouble_masked_unpack_low_mem_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %ymm2, %ymm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpcklpd {{.*#+}} ymm1 {%k1} = ymm0[0],mem[0],ymm0[2],mem[2] sched: [8:1.00]
-; GENERIC-NEXT:    vmovapd %ymm1, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xdouble_masked_unpack_low_mem_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %ymm2, %ymm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpcklpd {{.*#+}} ymm1 {%k1} = ymm0[0],mem[0],ymm0[2],mem[2] sched: [8:1.00]
-; SKX-NEXT:    vmovapd %ymm1, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <4 x double>, <4 x double>* %vec2p
-  %shuf = shufflevector <4 x double> %vec1, <4 x double> %vec2, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
-  %cmp = icmp eq <4 x i64> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x double> %shuf, <4 x double> %vec3
-  ret <4 x double> %res
-}
-
-define <4 x double> @test_4xdouble_zero_masked_unpack_low_mem_mask0(<4 x double> %vec1, <4 x double>* %vec2p, <4 x i64> %mask) {
-; GENERIC-LABEL: test_4xdouble_zero_masked_unpack_low_mem_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %ymm1, %ymm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpcklpd {{.*#+}} ymm0 {%k1} {z} = ymm0[0],mem[0],ymm0[2],mem[2] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xdouble_zero_masked_unpack_low_mem_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %ymm1, %ymm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpcklpd {{.*#+}} ymm0 {%k1} {z} = ymm0[0],mem[0],ymm0[2],mem[2] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <4 x double>, <4 x double>* %vec2p
-  %shuf = shufflevector <4 x double> %vec1, <4 x double> %vec2, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
-  %cmp = icmp eq <4 x i64> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x double> %shuf, <4 x double> zeroinitializer
-  ret <4 x double> %res
-}
-
-define <4 x double> @test_4xdouble_masked_unpack_low_mem_mask1(<4 x double> %vec1, <4 x double>* %vec2p, <4 x double> %vec3, <4 x i64> %mask) {
-; GENERIC-LABEL: test_4xdouble_masked_unpack_low_mem_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %ymm2, %ymm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpcklpd {{.*#+}} ymm1 {%k1} = ymm0[0],mem[0],ymm0[2],mem[2] sched: [8:1.00]
-; GENERIC-NEXT:    vmovapd %ymm1, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xdouble_masked_unpack_low_mem_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %ymm2, %ymm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpcklpd {{.*#+}} ymm1 {%k1} = ymm0[0],mem[0],ymm0[2],mem[2] sched: [8:1.00]
-; SKX-NEXT:    vmovapd %ymm1, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <4 x double>, <4 x double>* %vec2p
-  %shuf = shufflevector <4 x double> %vec1, <4 x double> %vec2, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
-  %cmp = icmp eq <4 x i64> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x double> %shuf, <4 x double> %vec3
-  ret <4 x double> %res
-}
-
-define <4 x double> @test_4xdouble_zero_masked_unpack_low_mem_mask1(<4 x double> %vec1, <4 x double>* %vec2p, <4 x i64> %mask) {
-; GENERIC-LABEL: test_4xdouble_zero_masked_unpack_low_mem_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %ymm1, %ymm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpcklpd {{.*#+}} ymm0 {%k1} {z} = ymm0[0],mem[0],ymm0[2],mem[2] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xdouble_zero_masked_unpack_low_mem_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %ymm1, %ymm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpcklpd {{.*#+}} ymm0 {%k1} {z} = ymm0[0],mem[0],ymm0[2],mem[2] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <4 x double>, <4 x double>* %vec2p
-  %shuf = shufflevector <4 x double> %vec1, <4 x double> %vec2, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
-  %cmp = icmp eq <4 x i64> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x double> %shuf, <4 x double> zeroinitializer
-  ret <4 x double> %res
-}
-
-define <4 x double> @test_4xdouble_masked_unpack_low_mem_mask2(<4 x double> %vec1, <4 x double>* %vec2p, <4 x double> %vec3, <4 x i64> %mask) {
-; GENERIC-LABEL: test_4xdouble_masked_unpack_low_mem_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %ymm2, %ymm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpcklpd {{.*#+}} ymm1 {%k1} = ymm0[0],mem[0],ymm0[2],mem[2] sched: [8:1.00]
-; GENERIC-NEXT:    vmovapd %ymm1, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xdouble_masked_unpack_low_mem_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %ymm2, %ymm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpcklpd {{.*#+}} ymm1 {%k1} = ymm0[0],mem[0],ymm0[2],mem[2] sched: [8:1.00]
-; SKX-NEXT:    vmovapd %ymm1, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <4 x double>, <4 x double>* %vec2p
-  %shuf = shufflevector <4 x double> %vec1, <4 x double> %vec2, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
-  %cmp = icmp eq <4 x i64> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x double> %shuf, <4 x double> %vec3
-  ret <4 x double> %res
-}
-
-define <4 x double> @test_4xdouble_zero_masked_unpack_low_mem_mask2(<4 x double> %vec1, <4 x double>* %vec2p, <4 x i64> %mask) {
-; GENERIC-LABEL: test_4xdouble_zero_masked_unpack_low_mem_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %ymm1, %ymm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpcklpd {{.*#+}} ymm0 {%k1} {z} = ymm0[0],mem[0],ymm0[2],mem[2] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xdouble_zero_masked_unpack_low_mem_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %ymm1, %ymm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpcklpd {{.*#+}} ymm0 {%k1} {z} = ymm0[0],mem[0],ymm0[2],mem[2] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <4 x double>, <4 x double>* %vec2p
-  %shuf = shufflevector <4 x double> %vec1, <4 x double> %vec2, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
-  %cmp = icmp eq <4 x i64> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x double> %shuf, <4 x double> zeroinitializer
-  ret <4 x double> %res
-}
-
-define <4 x double> @test_4xdouble_unpack_low_mem_mask3(<4 x double> %vec1, <4 x double>* %vec2p) {
-; GENERIC-LABEL: test_4xdouble_unpack_low_mem_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vunpcklpd {{.*#+}} ymm0 = ymm0[0],mem[0],ymm0[2],mem[2] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xdouble_unpack_low_mem_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vunpcklpd {{.*#+}} ymm0 = ymm0[0],mem[0],ymm0[2],mem[2] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <4 x double>, <4 x double>* %vec2p
-  %res = shufflevector <4 x double> %vec1, <4 x double> %vec2, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
-  ret <4 x double> %res
-}
-define <4 x double> @test_4xdouble_masked_unpack_low_mem_mask3(<4 x double> %vec1, <4 x double>* %vec2p, <4 x double> %vec3, <4 x i64> %mask) {
-; GENERIC-LABEL: test_4xdouble_masked_unpack_low_mem_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %ymm2, %ymm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpcklpd {{.*#+}} ymm1 {%k1} = ymm0[0],mem[0],ymm0[2],mem[2] sched: [8:1.00]
-; GENERIC-NEXT:    vmovapd %ymm1, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xdouble_masked_unpack_low_mem_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %ymm2, %ymm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpcklpd {{.*#+}} ymm1 {%k1} = ymm0[0],mem[0],ymm0[2],mem[2] sched: [8:1.00]
-; SKX-NEXT:    vmovapd %ymm1, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <4 x double>, <4 x double>* %vec2p
-  %shuf = shufflevector <4 x double> %vec1, <4 x double> %vec2, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
-  %cmp = icmp eq <4 x i64> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x double> %shuf, <4 x double> %vec3
-  ret <4 x double> %res
-}
-
-define <4 x double> @test_4xdouble_zero_masked_unpack_low_mem_mask3(<4 x double> %vec1, <4 x double>* %vec2p, <4 x i64> %mask) {
-; GENERIC-LABEL: test_4xdouble_zero_masked_unpack_low_mem_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %ymm1, %ymm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpcklpd {{.*#+}} ymm0 {%k1} {z} = ymm0[0],mem[0],ymm0[2],mem[2] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xdouble_zero_masked_unpack_low_mem_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %ymm1, %ymm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpcklpd {{.*#+}} ymm0 {%k1} {z} = ymm0[0],mem[0],ymm0[2],mem[2] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <4 x double>, <4 x double>* %vec2p
-  %shuf = shufflevector <4 x double> %vec1, <4 x double> %vec2, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
-  %cmp = icmp eq <4 x i64> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x double> %shuf, <4 x double> zeroinitializer
-  ret <4 x double> %res
-}
-
-define <8 x double> @test_8xdouble_unpack_low_mask0(<8 x double> %vec1, <8 x double> %vec2) {
-; GENERIC-LABEL: test_8xdouble_unpack_low_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vunpcklpd {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[2],zmm1[2],zmm0[4],zmm1[4],zmm0[6],zmm1[6] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xdouble_unpack_low_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vunpcklpd {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[2],zmm1[2],zmm0[4],zmm1[4],zmm0[6],zmm1[6] sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %res = shufflevector <8 x double> %vec1, <8 x double> %vec2, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
-  ret <8 x double> %res
-}
-define <8 x double> @test_8xdouble_masked_unpack_low_mask0(<8 x double> %vec1, <8 x double> %vec2, <8 x double> %vec3, <8 x i64> %mask) {
-; GENERIC-LABEL: test_8xdouble_masked_unpack_low_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %zmm3, %zmm3, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpcklpd {{.*#+}} zmm2 {%k1} = zmm0[0],zmm1[0],zmm0[2],zmm1[2],zmm0[4],zmm1[4],zmm0[6],zmm1[6] sched: [1:1.00]
-; GENERIC-NEXT:    vmovapd %zmm2, %zmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xdouble_masked_unpack_low_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %zmm3, %zmm3, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpcklpd {{.*#+}} zmm2 {%k1} = zmm0[0],zmm1[0],zmm0[2],zmm1[2],zmm0[4],zmm1[4],zmm0[6],zmm1[6] sched: [1:1.00]
-; SKX-NEXT:    vmovapd %zmm2, %zmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x double> %vec1, <8 x double> %vec2, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x double> %shuf, <8 x double> %vec3
-  ret <8 x double> %res
-}
-
-define <8 x double> @test_8xdouble_zero_masked_unpack_low_mask0(<8 x double> %vec1, <8 x double> %vec2, <8 x i64> %mask) {
-; GENERIC-LABEL: test_8xdouble_zero_masked_unpack_low_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %zmm2, %zmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpcklpd {{.*#+}} zmm0 {%k1} {z} = zmm0[0],zmm1[0],zmm0[2],zmm1[2],zmm0[4],zmm1[4],zmm0[6],zmm1[6] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xdouble_zero_masked_unpack_low_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %zmm2, %zmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpcklpd {{.*#+}} zmm0 {%k1} {z} = zmm0[0],zmm1[0],zmm0[2],zmm1[2],zmm0[4],zmm1[4],zmm0[6],zmm1[6] sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x double> %vec1, <8 x double> %vec2, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x double> %shuf, <8 x double> zeroinitializer
-  ret <8 x double> %res
-}
-define <8 x double> @test_8xdouble_masked_unpack_low_mask1(<8 x double> %vec1, <8 x double> %vec2, <8 x double> %vec3, <8 x i64> %mask) {
-; GENERIC-LABEL: test_8xdouble_masked_unpack_low_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %zmm3, %zmm3, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpcklpd {{.*#+}} zmm2 {%k1} = zmm0[0],zmm1[0],zmm0[2],zmm1[2],zmm0[4],zmm1[4],zmm0[6],zmm1[6] sched: [1:1.00]
-; GENERIC-NEXT:    vmovapd %zmm2, %zmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xdouble_masked_unpack_low_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %zmm3, %zmm3, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpcklpd {{.*#+}} zmm2 {%k1} = zmm0[0],zmm1[0],zmm0[2],zmm1[2],zmm0[4],zmm1[4],zmm0[6],zmm1[6] sched: [1:1.00]
-; SKX-NEXT:    vmovapd %zmm2, %zmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x double> %vec1, <8 x double> %vec2, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x double> %shuf, <8 x double> %vec3
-  ret <8 x double> %res
-}
-
-define <8 x double> @test_8xdouble_zero_masked_unpack_low_mask1(<8 x double> %vec1, <8 x double> %vec2, <8 x i64> %mask) {
-; GENERIC-LABEL: test_8xdouble_zero_masked_unpack_low_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %zmm2, %zmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpcklpd {{.*#+}} zmm0 {%k1} {z} = zmm0[0],zmm1[0],zmm0[2],zmm1[2],zmm0[4],zmm1[4],zmm0[6],zmm1[6] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xdouble_zero_masked_unpack_low_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %zmm2, %zmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpcklpd {{.*#+}} zmm0 {%k1} {z} = zmm0[0],zmm1[0],zmm0[2],zmm1[2],zmm0[4],zmm1[4],zmm0[6],zmm1[6] sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x double> %vec1, <8 x double> %vec2, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x double> %shuf, <8 x double> zeroinitializer
-  ret <8 x double> %res
-}
-define <8 x double> @test_8xdouble_masked_unpack_low_mask2(<8 x double> %vec1, <8 x double> %vec2, <8 x double> %vec3, <8 x i64> %mask) {
-; GENERIC-LABEL: test_8xdouble_masked_unpack_low_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %zmm3, %zmm3, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpcklpd {{.*#+}} zmm2 {%k1} = zmm0[0],zmm1[0],zmm0[2],zmm1[2],zmm0[4],zmm1[4],zmm0[6],zmm1[6] sched: [1:1.00]
-; GENERIC-NEXT:    vmovapd %zmm2, %zmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xdouble_masked_unpack_low_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %zmm3, %zmm3, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpcklpd {{.*#+}} zmm2 {%k1} = zmm0[0],zmm1[0],zmm0[2],zmm1[2],zmm0[4],zmm1[4],zmm0[6],zmm1[6] sched: [1:1.00]
-; SKX-NEXT:    vmovapd %zmm2, %zmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x double> %vec1, <8 x double> %vec2, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x double> %shuf, <8 x double> %vec3
-  ret <8 x double> %res
-}
-
-define <8 x double> @test_8xdouble_zero_masked_unpack_low_mask2(<8 x double> %vec1, <8 x double> %vec2, <8 x i64> %mask) {
-; GENERIC-LABEL: test_8xdouble_zero_masked_unpack_low_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %zmm2, %zmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpcklpd {{.*#+}} zmm0 {%k1} {z} = zmm0[0],zmm1[0],zmm0[2],zmm1[2],zmm0[4],zmm1[4],zmm0[6],zmm1[6] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xdouble_zero_masked_unpack_low_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %zmm2, %zmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpcklpd {{.*#+}} zmm0 {%k1} {z} = zmm0[0],zmm1[0],zmm0[2],zmm1[2],zmm0[4],zmm1[4],zmm0[6],zmm1[6] sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x double> %vec1, <8 x double> %vec2, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x double> %shuf, <8 x double> zeroinitializer
-  ret <8 x double> %res
-}
-define <8 x double> @test_8xdouble_unpack_low_mask3(<8 x double> %vec1, <8 x double> %vec2) {
-; GENERIC-LABEL: test_8xdouble_unpack_low_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vunpcklpd {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[2],zmm1[2],zmm0[4],zmm1[4],zmm0[6],zmm1[6] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xdouble_unpack_low_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vunpcklpd {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[2],zmm1[2],zmm0[4],zmm1[4],zmm0[6],zmm1[6] sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %res = shufflevector <8 x double> %vec1, <8 x double> %vec2, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
-  ret <8 x double> %res
-}
-define <8 x double> @test_8xdouble_masked_unpack_low_mask3(<8 x double> %vec1, <8 x double> %vec2, <8 x double> %vec3, <8 x i64> %mask) {
-; GENERIC-LABEL: test_8xdouble_masked_unpack_low_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %zmm3, %zmm3, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpcklpd {{.*#+}} zmm2 {%k1} = zmm0[0],zmm1[0],zmm0[2],zmm1[2],zmm0[4],zmm1[4],zmm0[6],zmm1[6] sched: [1:1.00]
-; GENERIC-NEXT:    vmovapd %zmm2, %zmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xdouble_masked_unpack_low_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %zmm3, %zmm3, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpcklpd {{.*#+}} zmm2 {%k1} = zmm0[0],zmm1[0],zmm0[2],zmm1[2],zmm0[4],zmm1[4],zmm0[6],zmm1[6] sched: [1:1.00]
-; SKX-NEXT:    vmovapd %zmm2, %zmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x double> %vec1, <8 x double> %vec2, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x double> %shuf, <8 x double> %vec3
-  ret <8 x double> %res
-}
-
-define <8 x double> @test_8xdouble_zero_masked_unpack_low_mask3(<8 x double> %vec1, <8 x double> %vec2, <8 x i64> %mask) {
-; GENERIC-LABEL: test_8xdouble_zero_masked_unpack_low_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %zmm2, %zmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpcklpd {{.*#+}} zmm0 {%k1} {z} = zmm0[0],zmm1[0],zmm0[2],zmm1[2],zmm0[4],zmm1[4],zmm0[6],zmm1[6] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xdouble_zero_masked_unpack_low_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %zmm2, %zmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpcklpd {{.*#+}} zmm0 {%k1} {z} = zmm0[0],zmm1[0],zmm0[2],zmm1[2],zmm0[4],zmm1[4],zmm0[6],zmm1[6] sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x double> %vec1, <8 x double> %vec2, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x double> %shuf, <8 x double> zeroinitializer
-  ret <8 x double> %res
-}
-define <8 x double> @test_8xdouble_unpack_low_mem_mask0(<8 x double> %vec1, <8 x double>* %vec2p) {
-; GENERIC-LABEL: test_8xdouble_unpack_low_mem_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vunpcklpd {{.*#+}} zmm0 = zmm0[0],mem[0],zmm0[2],mem[2],zmm0[4],mem[4],zmm0[6],mem[6] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xdouble_unpack_low_mem_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vunpcklpd {{.*#+}} zmm0 = zmm0[0],mem[0],zmm0[2],mem[2],zmm0[4],mem[4],zmm0[6],mem[6] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <8 x double>, <8 x double>* %vec2p
-  %res = shufflevector <8 x double> %vec1, <8 x double> %vec2, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
-  ret <8 x double> %res
-}
-define <8 x double> @test_8xdouble_masked_unpack_low_mem_mask0(<8 x double> %vec1, <8 x double>* %vec2p, <8 x double> %vec3, <8 x i64> %mask) {
-; GENERIC-LABEL: test_8xdouble_masked_unpack_low_mem_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %zmm2, %zmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpcklpd {{.*#+}} zmm1 {%k1} = zmm0[0],mem[0],zmm0[2],mem[2],zmm0[4],mem[4],zmm0[6],mem[6] sched: [8:1.00]
-; GENERIC-NEXT:    vmovapd %zmm1, %zmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xdouble_masked_unpack_low_mem_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %zmm2, %zmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpcklpd {{.*#+}} zmm1 {%k1} = zmm0[0],mem[0],zmm0[2],mem[2],zmm0[4],mem[4],zmm0[6],mem[6] sched: [8:1.00]
-; SKX-NEXT:    vmovapd %zmm1, %zmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <8 x double>, <8 x double>* %vec2p
-  %shuf = shufflevector <8 x double> %vec1, <8 x double> %vec2, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x double> %shuf, <8 x double> %vec3
-  ret <8 x double> %res
-}
-
-define <8 x double> @test_8xdouble_zero_masked_unpack_low_mem_mask0(<8 x double> %vec1, <8 x double>* %vec2p, <8 x i64> %mask) {
-; GENERIC-LABEL: test_8xdouble_zero_masked_unpack_low_mem_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpcklpd {{.*#+}} zmm0 {%k1} {z} = zmm0[0],mem[0],zmm0[2],mem[2],zmm0[4],mem[4],zmm0[6],mem[6] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xdouble_zero_masked_unpack_low_mem_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpcklpd {{.*#+}} zmm0 {%k1} {z} = zmm0[0],mem[0],zmm0[2],mem[2],zmm0[4],mem[4],zmm0[6],mem[6] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <8 x double>, <8 x double>* %vec2p
-  %shuf = shufflevector <8 x double> %vec1, <8 x double> %vec2, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x double> %shuf, <8 x double> zeroinitializer
-  ret <8 x double> %res
-}
-
-define <8 x double> @test_8xdouble_masked_unpack_low_mem_mask1(<8 x double> %vec1, <8 x double>* %vec2p, <8 x double> %vec3, <8 x i64> %mask) {
-; GENERIC-LABEL: test_8xdouble_masked_unpack_low_mem_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %zmm2, %zmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpcklpd {{.*#+}} zmm1 {%k1} = zmm0[0],mem[0],zmm0[2],mem[2],zmm0[4],mem[4],zmm0[6],mem[6] sched: [8:1.00]
-; GENERIC-NEXT:    vmovapd %zmm1, %zmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xdouble_masked_unpack_low_mem_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %zmm2, %zmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpcklpd {{.*#+}} zmm1 {%k1} = zmm0[0],mem[0],zmm0[2],mem[2],zmm0[4],mem[4],zmm0[6],mem[6] sched: [8:1.00]
-; SKX-NEXT:    vmovapd %zmm1, %zmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <8 x double>, <8 x double>* %vec2p
-  %shuf = shufflevector <8 x double> %vec1, <8 x double> %vec2, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x double> %shuf, <8 x double> %vec3
-  ret <8 x double> %res
-}
-
-define <8 x double> @test_8xdouble_zero_masked_unpack_low_mem_mask1(<8 x double> %vec1, <8 x double>* %vec2p, <8 x i64> %mask) {
-; GENERIC-LABEL: test_8xdouble_zero_masked_unpack_low_mem_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpcklpd {{.*#+}} zmm0 {%k1} {z} = zmm0[0],mem[0],zmm0[2],mem[2],zmm0[4],mem[4],zmm0[6],mem[6] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xdouble_zero_masked_unpack_low_mem_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpcklpd {{.*#+}} zmm0 {%k1} {z} = zmm0[0],mem[0],zmm0[2],mem[2],zmm0[4],mem[4],zmm0[6],mem[6] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <8 x double>, <8 x double>* %vec2p
-  %shuf = shufflevector <8 x double> %vec1, <8 x double> %vec2, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x double> %shuf, <8 x double> zeroinitializer
-  ret <8 x double> %res
-}
-
-define <8 x double> @test_8xdouble_masked_unpack_low_mem_mask2(<8 x double> %vec1, <8 x double>* %vec2p, <8 x double> %vec3, <8 x i64> %mask) {
-; GENERIC-LABEL: test_8xdouble_masked_unpack_low_mem_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %zmm2, %zmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpcklpd {{.*#+}} zmm1 {%k1} = zmm0[0],mem[0],zmm0[2],mem[2],zmm0[4],mem[4],zmm0[6],mem[6] sched: [8:1.00]
-; GENERIC-NEXT:    vmovapd %zmm1, %zmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xdouble_masked_unpack_low_mem_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %zmm2, %zmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpcklpd {{.*#+}} zmm1 {%k1} = zmm0[0],mem[0],zmm0[2],mem[2],zmm0[4],mem[4],zmm0[6],mem[6] sched: [8:1.00]
-; SKX-NEXT:    vmovapd %zmm1, %zmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <8 x double>, <8 x double>* %vec2p
-  %shuf = shufflevector <8 x double> %vec1, <8 x double> %vec2, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x double> %shuf, <8 x double> %vec3
-  ret <8 x double> %res
-}
-
-define <8 x double> @test_8xdouble_zero_masked_unpack_low_mem_mask2(<8 x double> %vec1, <8 x double>* %vec2p, <8 x i64> %mask) {
-; GENERIC-LABEL: test_8xdouble_zero_masked_unpack_low_mem_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpcklpd {{.*#+}} zmm0 {%k1} {z} = zmm0[0],mem[0],zmm0[2],mem[2],zmm0[4],mem[4],zmm0[6],mem[6] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xdouble_zero_masked_unpack_low_mem_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpcklpd {{.*#+}} zmm0 {%k1} {z} = zmm0[0],mem[0],zmm0[2],mem[2],zmm0[4],mem[4],zmm0[6],mem[6] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <8 x double>, <8 x double>* %vec2p
-  %shuf = shufflevector <8 x double> %vec1, <8 x double> %vec2, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x double> %shuf, <8 x double> zeroinitializer
-  ret <8 x double> %res
-}
-
-define <8 x double> @test_8xdouble_unpack_low_mem_mask3(<8 x double> %vec1, <8 x double>* %vec2p) {
-; GENERIC-LABEL: test_8xdouble_unpack_low_mem_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vunpcklpd {{.*#+}} zmm0 = zmm0[0],mem[0],zmm0[2],mem[2],zmm0[4],mem[4],zmm0[6],mem[6] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xdouble_unpack_low_mem_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vunpcklpd {{.*#+}} zmm0 = zmm0[0],mem[0],zmm0[2],mem[2],zmm0[4],mem[4],zmm0[6],mem[6] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <8 x double>, <8 x double>* %vec2p
-  %res = shufflevector <8 x double> %vec1, <8 x double> %vec2, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
-  ret <8 x double> %res
-}
-define <8 x double> @test_8xdouble_masked_unpack_low_mem_mask3(<8 x double> %vec1, <8 x double>* %vec2p, <8 x double> %vec3, <8 x i64> %mask) {
-; GENERIC-LABEL: test_8xdouble_masked_unpack_low_mem_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %zmm2, %zmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpcklpd {{.*#+}} zmm1 {%k1} = zmm0[0],mem[0],zmm0[2],mem[2],zmm0[4],mem[4],zmm0[6],mem[6] sched: [8:1.00]
-; GENERIC-NEXT:    vmovapd %zmm1, %zmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xdouble_masked_unpack_low_mem_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %zmm2, %zmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpcklpd {{.*#+}} zmm1 {%k1} = zmm0[0],mem[0],zmm0[2],mem[2],zmm0[4],mem[4],zmm0[6],mem[6] sched: [8:1.00]
-; SKX-NEXT:    vmovapd %zmm1, %zmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <8 x double>, <8 x double>* %vec2p
-  %shuf = shufflevector <8 x double> %vec1, <8 x double> %vec2, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x double> %shuf, <8 x double> %vec3
-  ret <8 x double> %res
-}
-
-define <8 x double> @test_8xdouble_zero_masked_unpack_low_mem_mask3(<8 x double> %vec1, <8 x double>* %vec2p, <8 x i64> %mask) {
-; GENERIC-LABEL: test_8xdouble_zero_masked_unpack_low_mem_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpcklpd {{.*#+}} zmm0 {%k1} {z} = zmm0[0],mem[0],zmm0[2],mem[2],zmm0[4],mem[4],zmm0[6],mem[6] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xdouble_zero_masked_unpack_low_mem_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpcklpd {{.*#+}} zmm0 {%k1} {z} = zmm0[0],mem[0],zmm0[2],mem[2],zmm0[4],mem[4],zmm0[6],mem[6] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <8 x double>, <8 x double>* %vec2p
-  %shuf = shufflevector <8 x double> %vec1, <8 x double> %vec2, <8 x i32> <i32 0, i32 8, i32 2, i32 10, i32 4, i32 12, i32 6, i32 14>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x double> %shuf, <8 x double> zeroinitializer
-  ret <8 x double> %res
-}
-
-define <4 x float> @test_4xfloat_unpack_high_mask0(<4 x float> %vec1, <4 x float> %vec2) {
-; GENERIC-LABEL: test_4xfloat_unpack_high_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vunpckhps {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xfloat_unpack_high_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vunpckhps {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3] sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %res = shufflevector <4 x float> %vec1, <4 x float> %vec2, <4 x i32> <i32 2, i32 6, i32 3, i32 7>
-  ret <4 x float> %res
-}
-define <4 x float> @test_4xfloat_masked_unpack_high_mask0(<4 x float> %vec1, <4 x float> %vec2, <4 x float> %vec3, <4 x i32> %mask) {
-; GENERIC-LABEL: test_4xfloat_masked_unpack_high_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %xmm3, %xmm3, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpckhps {{.*#+}} xmm2 {%k1} = xmm0[2],xmm1[2],xmm0[3],xmm1[3] sched: [1:1.00]
-; GENERIC-NEXT:    vmovaps %xmm2, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xfloat_masked_unpack_high_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %xmm3, %xmm3, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpckhps {{.*#+}} xmm2 {%k1} = xmm0[2],xmm1[2],xmm0[3],xmm1[3] sched: [1:1.00]
-; SKX-NEXT:    vmovaps %xmm2, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <4 x float> %vec1, <4 x float> %vec2, <4 x i32> <i32 2, i32 6, i32 3, i32 7>
-  %cmp = icmp eq <4 x i32> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x float> %shuf, <4 x float> %vec3
-  ret <4 x float> %res
-}
-
-define <4 x float> @test_4xfloat_zero_masked_unpack_high_mask0(<4 x float> %vec1, <4 x float> %vec2, <4 x i32> %mask) {
-; GENERIC-LABEL: test_4xfloat_zero_masked_unpack_high_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %xmm2, %xmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpckhps {{.*#+}} xmm0 {%k1} {z} = xmm0[2],xmm1[2],xmm0[3],xmm1[3] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xfloat_zero_masked_unpack_high_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %xmm2, %xmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpckhps {{.*#+}} xmm0 {%k1} {z} = xmm0[2],xmm1[2],xmm0[3],xmm1[3] sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <4 x float> %vec1, <4 x float> %vec2, <4 x i32> <i32 2, i32 6, i32 3, i32 7>
-  %cmp = icmp eq <4 x i32> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x float> %shuf, <4 x float> zeroinitializer
-  ret <4 x float> %res
-}
-define <4 x float> @test_4xfloat_masked_unpack_high_mask1(<4 x float> %vec1, <4 x float> %vec2, <4 x float> %vec3, <4 x i32> %mask) {
-; GENERIC-LABEL: test_4xfloat_masked_unpack_high_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %xmm3, %xmm3, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpckhps {{.*#+}} xmm2 {%k1} = xmm0[2],xmm1[2],xmm0[3],xmm1[3] sched: [1:1.00]
-; GENERIC-NEXT:    vmovaps %xmm2, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xfloat_masked_unpack_high_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %xmm3, %xmm3, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpckhps {{.*#+}} xmm2 {%k1} = xmm0[2],xmm1[2],xmm0[3],xmm1[3] sched: [1:1.00]
-; SKX-NEXT:    vmovaps %xmm2, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <4 x float> %vec1, <4 x float> %vec2, <4 x i32> <i32 2, i32 6, i32 3, i32 7>
-  %cmp = icmp eq <4 x i32> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x float> %shuf, <4 x float> %vec3
-  ret <4 x float> %res
-}
-
-define <4 x float> @test_4xfloat_zero_masked_unpack_high_mask1(<4 x float> %vec1, <4 x float> %vec2, <4 x i32> %mask) {
-; GENERIC-LABEL: test_4xfloat_zero_masked_unpack_high_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %xmm2, %xmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpckhps {{.*#+}} xmm0 {%k1} {z} = xmm0[2],xmm1[2],xmm0[3],xmm1[3] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xfloat_zero_masked_unpack_high_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %xmm2, %xmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpckhps {{.*#+}} xmm0 {%k1} {z} = xmm0[2],xmm1[2],xmm0[3],xmm1[3] sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <4 x float> %vec1, <4 x float> %vec2, <4 x i32> <i32 2, i32 6, i32 3, i32 7>
-  %cmp = icmp eq <4 x i32> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x float> %shuf, <4 x float> zeroinitializer
-  ret <4 x float> %res
-}
-define <4 x float> @test_4xfloat_masked_unpack_high_mask2(<4 x float> %vec1, <4 x float> %vec2, <4 x float> %vec3, <4 x i32> %mask) {
-; GENERIC-LABEL: test_4xfloat_masked_unpack_high_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %xmm3, %xmm3, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpckhps {{.*#+}} xmm2 {%k1} = xmm0[2],xmm1[2],xmm0[3],xmm1[3] sched: [1:1.00]
-; GENERIC-NEXT:    vmovaps %xmm2, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xfloat_masked_unpack_high_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %xmm3, %xmm3, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpckhps {{.*#+}} xmm2 {%k1} = xmm0[2],xmm1[2],xmm0[3],xmm1[3] sched: [1:1.00]
-; SKX-NEXT:    vmovaps %xmm2, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <4 x float> %vec1, <4 x float> %vec2, <4 x i32> <i32 2, i32 6, i32 3, i32 7>
-  %cmp = icmp eq <4 x i32> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x float> %shuf, <4 x float> %vec3
-  ret <4 x float> %res
-}
-
-define <4 x float> @test_4xfloat_zero_masked_unpack_high_mask2(<4 x float> %vec1, <4 x float> %vec2, <4 x i32> %mask) {
-; GENERIC-LABEL: test_4xfloat_zero_masked_unpack_high_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %xmm2, %xmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpckhps {{.*#+}} xmm0 {%k1} {z} = xmm0[2],xmm1[2],xmm0[3],xmm1[3] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xfloat_zero_masked_unpack_high_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %xmm2, %xmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpckhps {{.*#+}} xmm0 {%k1} {z} = xmm0[2],xmm1[2],xmm0[3],xmm1[3] sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <4 x float> %vec1, <4 x float> %vec2, <4 x i32> <i32 2, i32 6, i32 3, i32 7>
-  %cmp = icmp eq <4 x i32> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x float> %shuf, <4 x float> zeroinitializer
-  ret <4 x float> %res
-}
-define <4 x float> @test_4xfloat_unpack_high_mask3(<4 x float> %vec1, <4 x float> %vec2) {
-; GENERIC-LABEL: test_4xfloat_unpack_high_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vunpckhps {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xfloat_unpack_high_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vunpckhps {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3] sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %res = shufflevector <4 x float> %vec1, <4 x float> %vec2, <4 x i32> <i32 2, i32 6, i32 3, i32 7>
-  ret <4 x float> %res
-}
-define <4 x float> @test_4xfloat_masked_unpack_high_mask3(<4 x float> %vec1, <4 x float> %vec2, <4 x float> %vec3, <4 x i32> %mask) {
-; GENERIC-LABEL: test_4xfloat_masked_unpack_high_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %xmm3, %xmm3, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpckhps {{.*#+}} xmm2 {%k1} = xmm0[2],xmm1[2],xmm0[3],xmm1[3] sched: [1:1.00]
-; GENERIC-NEXT:    vmovaps %xmm2, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xfloat_masked_unpack_high_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %xmm3, %xmm3, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpckhps {{.*#+}} xmm2 {%k1} = xmm0[2],xmm1[2],xmm0[3],xmm1[3] sched: [1:1.00]
-; SKX-NEXT:    vmovaps %xmm2, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <4 x float> %vec1, <4 x float> %vec2, <4 x i32> <i32 2, i32 6, i32 3, i32 7>
-  %cmp = icmp eq <4 x i32> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x float> %shuf, <4 x float> %vec3
-  ret <4 x float> %res
-}
-
-define <4 x float> @test_4xfloat_zero_masked_unpack_high_mask3(<4 x float> %vec1, <4 x float> %vec2, <4 x i32> %mask) {
-; GENERIC-LABEL: test_4xfloat_zero_masked_unpack_high_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %xmm2, %xmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpckhps {{.*#+}} xmm0 {%k1} {z} = xmm0[2],xmm1[2],xmm0[3],xmm1[3] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xfloat_zero_masked_unpack_high_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %xmm2, %xmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpckhps {{.*#+}} xmm0 {%k1} {z} = xmm0[2],xmm1[2],xmm0[3],xmm1[3] sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <4 x float> %vec1, <4 x float> %vec2, <4 x i32> <i32 2, i32 6, i32 3, i32 7>
-  %cmp = icmp eq <4 x i32> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x float> %shuf, <4 x float> zeroinitializer
-  ret <4 x float> %res
-}
-define <4 x float> @test_4xfloat_unpack_high_mem_mask0(<4 x float> %vec1, <4 x float>* %vec2p) {
-; GENERIC-LABEL: test_4xfloat_unpack_high_mem_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vunpckhps {{.*#+}} xmm0 = xmm0[2],mem[2],xmm0[3],mem[3] sched: [7:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xfloat_unpack_high_mem_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vunpckhps {{.*#+}} xmm0 = xmm0[2],mem[2],xmm0[3],mem[3] sched: [7:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <4 x float>, <4 x float>* %vec2p
-  %res = shufflevector <4 x float> %vec1, <4 x float> %vec2, <4 x i32> <i32 2, i32 6, i32 3, i32 7>
-  ret <4 x float> %res
-}
-define <4 x float> @test_4xfloat_masked_unpack_high_mem_mask0(<4 x float> %vec1, <4 x float>* %vec2p, <4 x float> %vec3, <4 x i32> %mask) {
-; GENERIC-LABEL: test_4xfloat_masked_unpack_high_mem_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %xmm2, %xmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpckhps {{.*#+}} xmm1 {%k1} = xmm0[2],mem[2],xmm0[3],mem[3] sched: [7:1.00]
-; GENERIC-NEXT:    vmovaps %xmm1, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xfloat_masked_unpack_high_mem_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %xmm2, %xmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpckhps {{.*#+}} xmm1 {%k1} = xmm0[2],mem[2],xmm0[3],mem[3] sched: [7:1.00]
-; SKX-NEXT:    vmovaps %xmm1, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <4 x float>, <4 x float>* %vec2p
-  %shuf = shufflevector <4 x float> %vec1, <4 x float> %vec2, <4 x i32> <i32 2, i32 6, i32 3, i32 7>
-  %cmp = icmp eq <4 x i32> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x float> %shuf, <4 x float> %vec3
-  ret <4 x float> %res
-}
-
-define <4 x float> @test_4xfloat_zero_masked_unpack_high_mem_mask0(<4 x float> %vec1, <4 x float>* %vec2p, <4 x i32> %mask) {
-; GENERIC-LABEL: test_4xfloat_zero_masked_unpack_high_mem_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %xmm1, %xmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpckhps {{.*#+}} xmm0 {%k1} {z} = xmm0[2],mem[2],xmm0[3],mem[3] sched: [7:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xfloat_zero_masked_unpack_high_mem_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %xmm1, %xmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpckhps {{.*#+}} xmm0 {%k1} {z} = xmm0[2],mem[2],xmm0[3],mem[3] sched: [7:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <4 x float>, <4 x float>* %vec2p
-  %shuf = shufflevector <4 x float> %vec1, <4 x float> %vec2, <4 x i32> <i32 2, i32 6, i32 3, i32 7>
-  %cmp = icmp eq <4 x i32> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x float> %shuf, <4 x float> zeroinitializer
-  ret <4 x float> %res
-}
-
-define <4 x float> @test_4xfloat_masked_unpack_high_mem_mask1(<4 x float> %vec1, <4 x float>* %vec2p, <4 x float> %vec3, <4 x i32> %mask) {
-; GENERIC-LABEL: test_4xfloat_masked_unpack_high_mem_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %xmm2, %xmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpckhps {{.*#+}} xmm1 {%k1} = xmm0[2],mem[2],xmm0[3],mem[3] sched: [7:1.00]
-; GENERIC-NEXT:    vmovaps %xmm1, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xfloat_masked_unpack_high_mem_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %xmm2, %xmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpckhps {{.*#+}} xmm1 {%k1} = xmm0[2],mem[2],xmm0[3],mem[3] sched: [7:1.00]
-; SKX-NEXT:    vmovaps %xmm1, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <4 x float>, <4 x float>* %vec2p
-  %shuf = shufflevector <4 x float> %vec1, <4 x float> %vec2, <4 x i32> <i32 2, i32 6, i32 3, i32 7>
-  %cmp = icmp eq <4 x i32> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x float> %shuf, <4 x float> %vec3
-  ret <4 x float> %res
-}
-
-define <4 x float> @test_4xfloat_zero_masked_unpack_high_mem_mask1(<4 x float> %vec1, <4 x float>* %vec2p, <4 x i32> %mask) {
-; GENERIC-LABEL: test_4xfloat_zero_masked_unpack_high_mem_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %xmm1, %xmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpckhps {{.*#+}} xmm0 {%k1} {z} = xmm0[2],mem[2],xmm0[3],mem[3] sched: [7:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xfloat_zero_masked_unpack_high_mem_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %xmm1, %xmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpckhps {{.*#+}} xmm0 {%k1} {z} = xmm0[2],mem[2],xmm0[3],mem[3] sched: [7:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <4 x float>, <4 x float>* %vec2p
-  %shuf = shufflevector <4 x float> %vec1, <4 x float> %vec2, <4 x i32> <i32 2, i32 6, i32 3, i32 7>
-  %cmp = icmp eq <4 x i32> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x float> %shuf, <4 x float> zeroinitializer
-  ret <4 x float> %res
-}
-
-define <4 x float> @test_4xfloat_masked_unpack_high_mem_mask2(<4 x float> %vec1, <4 x float>* %vec2p, <4 x float> %vec3, <4 x i32> %mask) {
-; GENERIC-LABEL: test_4xfloat_masked_unpack_high_mem_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %xmm2, %xmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpckhps {{.*#+}} xmm1 {%k1} = xmm0[2],mem[2],xmm0[3],mem[3] sched: [7:1.00]
-; GENERIC-NEXT:    vmovaps %xmm1, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xfloat_masked_unpack_high_mem_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %xmm2, %xmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpckhps {{.*#+}} xmm1 {%k1} = xmm0[2],mem[2],xmm0[3],mem[3] sched: [7:1.00]
-; SKX-NEXT:    vmovaps %xmm1, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <4 x float>, <4 x float>* %vec2p
-  %shuf = shufflevector <4 x float> %vec1, <4 x float> %vec2, <4 x i32> <i32 2, i32 6, i32 3, i32 7>
-  %cmp = icmp eq <4 x i32> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x float> %shuf, <4 x float> %vec3
-  ret <4 x float> %res
-}
-
-define <4 x float> @test_4xfloat_zero_masked_unpack_high_mem_mask2(<4 x float> %vec1, <4 x float>* %vec2p, <4 x i32> %mask) {
-; GENERIC-LABEL: test_4xfloat_zero_masked_unpack_high_mem_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %xmm1, %xmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpckhps {{.*#+}} xmm0 {%k1} {z} = xmm0[2],mem[2],xmm0[3],mem[3] sched: [7:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xfloat_zero_masked_unpack_high_mem_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %xmm1, %xmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpckhps {{.*#+}} xmm0 {%k1} {z} = xmm0[2],mem[2],xmm0[3],mem[3] sched: [7:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <4 x float>, <4 x float>* %vec2p
-  %shuf = shufflevector <4 x float> %vec1, <4 x float> %vec2, <4 x i32> <i32 2, i32 6, i32 3, i32 7>
-  %cmp = icmp eq <4 x i32> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x float> %shuf, <4 x float> zeroinitializer
-  ret <4 x float> %res
-}
-
-define <4 x float> @test_4xfloat_unpack_high_mem_mask3(<4 x float> %vec1, <4 x float>* %vec2p) {
-; GENERIC-LABEL: test_4xfloat_unpack_high_mem_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vunpckhps {{.*#+}} xmm0 = xmm0[2],mem[2],xmm0[3],mem[3] sched: [7:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xfloat_unpack_high_mem_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vunpckhps {{.*#+}} xmm0 = xmm0[2],mem[2],xmm0[3],mem[3] sched: [7:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <4 x float>, <4 x float>* %vec2p
-  %res = shufflevector <4 x float> %vec1, <4 x float> %vec2, <4 x i32> <i32 2, i32 6, i32 3, i32 7>
-  ret <4 x float> %res
-}
-define <4 x float> @test_4xfloat_masked_unpack_high_mem_mask3(<4 x float> %vec1, <4 x float>* %vec2p, <4 x float> %vec3, <4 x i32> %mask) {
-; GENERIC-LABEL: test_4xfloat_masked_unpack_high_mem_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %xmm2, %xmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpckhps {{.*#+}} xmm1 {%k1} = xmm0[2],mem[2],xmm0[3],mem[3] sched: [7:1.00]
-; GENERIC-NEXT:    vmovaps %xmm1, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xfloat_masked_unpack_high_mem_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %xmm2, %xmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpckhps {{.*#+}} xmm1 {%k1} = xmm0[2],mem[2],xmm0[3],mem[3] sched: [7:1.00]
-; SKX-NEXT:    vmovaps %xmm1, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <4 x float>, <4 x float>* %vec2p
-  %shuf = shufflevector <4 x float> %vec1, <4 x float> %vec2, <4 x i32> <i32 2, i32 6, i32 3, i32 7>
-  %cmp = icmp eq <4 x i32> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x float> %shuf, <4 x float> %vec3
-  ret <4 x float> %res
-}
-
-define <4 x float> @test_4xfloat_zero_masked_unpack_high_mem_mask3(<4 x float> %vec1, <4 x float>* %vec2p, <4 x i32> %mask) {
-; GENERIC-LABEL: test_4xfloat_zero_masked_unpack_high_mem_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %xmm1, %xmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpckhps {{.*#+}} xmm0 {%k1} {z} = xmm0[2],mem[2],xmm0[3],mem[3] sched: [7:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xfloat_zero_masked_unpack_high_mem_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %xmm1, %xmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpckhps {{.*#+}} xmm0 {%k1} {z} = xmm0[2],mem[2],xmm0[3],mem[3] sched: [7:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <4 x float>, <4 x float>* %vec2p
-  %shuf = shufflevector <4 x float> %vec1, <4 x float> %vec2, <4 x i32> <i32 2, i32 6, i32 3, i32 7>
-  %cmp = icmp eq <4 x i32> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x float> %shuf, <4 x float> zeroinitializer
-  ret <4 x float> %res
-}
-
-define <8 x float> @test_8xfloat_unpack_high_mask0(<8 x float> %vec1, <8 x float> %vec2) {
-; GENERIC-LABEL: test_8xfloat_unpack_high_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vunpckhps {{.*#+}} ymm0 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xfloat_unpack_high_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vunpckhps {{.*#+}} ymm0 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7] sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %res = shufflevector <8 x float> %vec1, <8 x float> %vec2, <8 x i32> <i32 2, i32 10, i32 3, i32 11, i32 6, i32 14, i32 7, i32 15>
-  ret <8 x float> %res
-}
-define <8 x float> @test_8xfloat_masked_unpack_high_mask0(<8 x float> %vec1, <8 x float> %vec2, <8 x float> %vec3, <8 x i32> %mask) {
-; GENERIC-LABEL: test_8xfloat_masked_unpack_high_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %ymm3, %ymm3, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpckhps {{.*#+}} ymm2 {%k1} = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7] sched: [1:1.00]
-; GENERIC-NEXT:    vmovaps %ymm2, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xfloat_masked_unpack_high_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %ymm3, %ymm3, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpckhps {{.*#+}} ymm2 {%k1} = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7] sched: [1:1.00]
-; SKX-NEXT:    vmovaps %ymm2, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x float> %vec1, <8 x float> %vec2, <8 x i32> <i32 2, i32 10, i32 3, i32 11, i32 6, i32 14, i32 7, i32 15>
-  %cmp = icmp eq <8 x i32> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x float> %shuf, <8 x float> %vec3
-  ret <8 x float> %res
-}
-
-define <8 x float> @test_8xfloat_zero_masked_unpack_high_mask0(<8 x float> %vec1, <8 x float> %vec2, <8 x i32> %mask) {
-; GENERIC-LABEL: test_8xfloat_zero_masked_unpack_high_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %ymm2, %ymm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpckhps {{.*#+}} ymm0 {%k1} {z} = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xfloat_zero_masked_unpack_high_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %ymm2, %ymm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpckhps {{.*#+}} ymm0 {%k1} {z} = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7] sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x float> %vec1, <8 x float> %vec2, <8 x i32> <i32 2, i32 10, i32 3, i32 11, i32 6, i32 14, i32 7, i32 15>
-  %cmp = icmp eq <8 x i32> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x float> %shuf, <8 x float> zeroinitializer
-  ret <8 x float> %res
-}
-define <8 x float> @test_8xfloat_masked_unpack_high_mask1(<8 x float> %vec1, <8 x float> %vec2, <8 x float> %vec3, <8 x i32> %mask) {
-; GENERIC-LABEL: test_8xfloat_masked_unpack_high_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %ymm3, %ymm3, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpckhps {{.*#+}} ymm2 {%k1} = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7] sched: [1:1.00]
-; GENERIC-NEXT:    vmovaps %ymm2, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xfloat_masked_unpack_high_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %ymm3, %ymm3, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpckhps {{.*#+}} ymm2 {%k1} = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7] sched: [1:1.00]
-; SKX-NEXT:    vmovaps %ymm2, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x float> %vec1, <8 x float> %vec2, <8 x i32> <i32 2, i32 10, i32 3, i32 11, i32 6, i32 14, i32 7, i32 15>
-  %cmp = icmp eq <8 x i32> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x float> %shuf, <8 x float> %vec3
-  ret <8 x float> %res
-}
-
-define <8 x float> @test_8xfloat_zero_masked_unpack_high_mask1(<8 x float> %vec1, <8 x float> %vec2, <8 x i32> %mask) {
-; GENERIC-LABEL: test_8xfloat_zero_masked_unpack_high_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %ymm2, %ymm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpckhps {{.*#+}} ymm0 {%k1} {z} = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xfloat_zero_masked_unpack_high_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %ymm2, %ymm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpckhps {{.*#+}} ymm0 {%k1} {z} = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7] sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x float> %vec1, <8 x float> %vec2, <8 x i32> <i32 2, i32 10, i32 3, i32 11, i32 6, i32 14, i32 7, i32 15>
-  %cmp = icmp eq <8 x i32> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x float> %shuf, <8 x float> zeroinitializer
-  ret <8 x float> %res
-}
-define <8 x float> @test_8xfloat_masked_unpack_high_mask2(<8 x float> %vec1, <8 x float> %vec2, <8 x float> %vec3, <8 x i32> %mask) {
-; GENERIC-LABEL: test_8xfloat_masked_unpack_high_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %ymm3, %ymm3, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpckhps {{.*#+}} ymm2 {%k1} = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7] sched: [1:1.00]
-; GENERIC-NEXT:    vmovaps %ymm2, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xfloat_masked_unpack_high_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %ymm3, %ymm3, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpckhps {{.*#+}} ymm2 {%k1} = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7] sched: [1:1.00]
-; SKX-NEXT:    vmovaps %ymm2, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x float> %vec1, <8 x float> %vec2, <8 x i32> <i32 2, i32 10, i32 3, i32 11, i32 6, i32 14, i32 7, i32 15>
-  %cmp = icmp eq <8 x i32> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x float> %shuf, <8 x float> %vec3
-  ret <8 x float> %res
-}
-
-define <8 x float> @test_8xfloat_zero_masked_unpack_high_mask2(<8 x float> %vec1, <8 x float> %vec2, <8 x i32> %mask) {
-; GENERIC-LABEL: test_8xfloat_zero_masked_unpack_high_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %ymm2, %ymm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpckhps {{.*#+}} ymm0 {%k1} {z} = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xfloat_zero_masked_unpack_high_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %ymm2, %ymm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpckhps {{.*#+}} ymm0 {%k1} {z} = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7] sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x float> %vec1, <8 x float> %vec2, <8 x i32> <i32 2, i32 10, i32 3, i32 11, i32 6, i32 14, i32 7, i32 15>
-  %cmp = icmp eq <8 x i32> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x float> %shuf, <8 x float> zeroinitializer
-  ret <8 x float> %res
-}
-define <8 x float> @test_8xfloat_unpack_high_mask3(<8 x float> %vec1, <8 x float> %vec2) {
-; GENERIC-LABEL: test_8xfloat_unpack_high_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vunpckhps {{.*#+}} ymm0 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xfloat_unpack_high_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vunpckhps {{.*#+}} ymm0 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7] sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %res = shufflevector <8 x float> %vec1, <8 x float> %vec2, <8 x i32> <i32 2, i32 10, i32 3, i32 11, i32 6, i32 14, i32 7, i32 15>
-  ret <8 x float> %res
-}
-define <8 x float> @test_8xfloat_masked_unpack_high_mask3(<8 x float> %vec1, <8 x float> %vec2, <8 x float> %vec3, <8 x i32> %mask) {
-; GENERIC-LABEL: test_8xfloat_masked_unpack_high_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %ymm3, %ymm3, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpckhps {{.*#+}} ymm2 {%k1} = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7] sched: [1:1.00]
-; GENERIC-NEXT:    vmovaps %ymm2, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xfloat_masked_unpack_high_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %ymm3, %ymm3, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpckhps {{.*#+}} ymm2 {%k1} = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7] sched: [1:1.00]
-; SKX-NEXT:    vmovaps %ymm2, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x float> %vec1, <8 x float> %vec2, <8 x i32> <i32 2, i32 10, i32 3, i32 11, i32 6, i32 14, i32 7, i32 15>
-  %cmp = icmp eq <8 x i32> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x float> %shuf, <8 x float> %vec3
-  ret <8 x float> %res
-}
-
-define <8 x float> @test_8xfloat_zero_masked_unpack_high_mask3(<8 x float> %vec1, <8 x float> %vec2, <8 x i32> %mask) {
-; GENERIC-LABEL: test_8xfloat_zero_masked_unpack_high_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %ymm2, %ymm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpckhps {{.*#+}} ymm0 {%k1} {z} = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xfloat_zero_masked_unpack_high_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %ymm2, %ymm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpckhps {{.*#+}} ymm0 {%k1} {z} = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7] sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x float> %vec1, <8 x float> %vec2, <8 x i32> <i32 2, i32 10, i32 3, i32 11, i32 6, i32 14, i32 7, i32 15>
-  %cmp = icmp eq <8 x i32> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x float> %shuf, <8 x float> zeroinitializer
-  ret <8 x float> %res
-}
-define <8 x float> @test_8xfloat_unpack_high_mem_mask0(<8 x float> %vec1, <8 x float>* %vec2p) {
-; GENERIC-LABEL: test_8xfloat_unpack_high_mem_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vunpckhps {{.*#+}} ymm0 = ymm0[2],mem[2],ymm0[3],mem[3],ymm0[6],mem[6],ymm0[7],mem[7] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xfloat_unpack_high_mem_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vunpckhps {{.*#+}} ymm0 = ymm0[2],mem[2],ymm0[3],mem[3],ymm0[6],mem[6],ymm0[7],mem[7] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <8 x float>, <8 x float>* %vec2p
-  %res = shufflevector <8 x float> %vec1, <8 x float> %vec2, <8 x i32> <i32 2, i32 10, i32 3, i32 11, i32 6, i32 14, i32 7, i32 15>
-  ret <8 x float> %res
-}
-define <8 x float> @test_8xfloat_masked_unpack_high_mem_mask0(<8 x float> %vec1, <8 x float>* %vec2p, <8 x float> %vec3, <8 x i32> %mask) {
-; GENERIC-LABEL: test_8xfloat_masked_unpack_high_mem_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %ymm2, %ymm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpckhps {{.*#+}} ymm1 {%k1} = ymm0[2],mem[2],ymm0[3],mem[3],ymm0[6],mem[6],ymm0[7],mem[7] sched: [8:1.00]
-; GENERIC-NEXT:    vmovaps %ymm1, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xfloat_masked_unpack_high_mem_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %ymm2, %ymm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpckhps {{.*#+}} ymm1 {%k1} = ymm0[2],mem[2],ymm0[3],mem[3],ymm0[6],mem[6],ymm0[7],mem[7] sched: [8:1.00]
-; SKX-NEXT:    vmovaps %ymm1, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <8 x float>, <8 x float>* %vec2p
-  %shuf = shufflevector <8 x float> %vec1, <8 x float> %vec2, <8 x i32> <i32 2, i32 10, i32 3, i32 11, i32 6, i32 14, i32 7, i32 15>
-  %cmp = icmp eq <8 x i32> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x float> %shuf, <8 x float> %vec3
-  ret <8 x float> %res
-}
-
-define <8 x float> @test_8xfloat_zero_masked_unpack_high_mem_mask0(<8 x float> %vec1, <8 x float>* %vec2p, <8 x i32> %mask) {
-; GENERIC-LABEL: test_8xfloat_zero_masked_unpack_high_mem_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %ymm1, %ymm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpckhps {{.*#+}} ymm0 {%k1} {z} = ymm0[2],mem[2],ymm0[3],mem[3],ymm0[6],mem[6],ymm0[7],mem[7] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xfloat_zero_masked_unpack_high_mem_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %ymm1, %ymm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpckhps {{.*#+}} ymm0 {%k1} {z} = ymm0[2],mem[2],ymm0[3],mem[3],ymm0[6],mem[6],ymm0[7],mem[7] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <8 x float>, <8 x float>* %vec2p
-  %shuf = shufflevector <8 x float> %vec1, <8 x float> %vec2, <8 x i32> <i32 2, i32 10, i32 3, i32 11, i32 6, i32 14, i32 7, i32 15>
-  %cmp = icmp eq <8 x i32> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x float> %shuf, <8 x float> zeroinitializer
-  ret <8 x float> %res
-}
-
-define <8 x float> @test_8xfloat_masked_unpack_high_mem_mask1(<8 x float> %vec1, <8 x float>* %vec2p, <8 x float> %vec3, <8 x i32> %mask) {
-; GENERIC-LABEL: test_8xfloat_masked_unpack_high_mem_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %ymm2, %ymm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpckhps {{.*#+}} ymm1 {%k1} = ymm0[2],mem[2],ymm0[3],mem[3],ymm0[6],mem[6],ymm0[7],mem[7] sched: [8:1.00]
-; GENERIC-NEXT:    vmovaps %ymm1, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xfloat_masked_unpack_high_mem_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %ymm2, %ymm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpckhps {{.*#+}} ymm1 {%k1} = ymm0[2],mem[2],ymm0[3],mem[3],ymm0[6],mem[6],ymm0[7],mem[7] sched: [8:1.00]
-; SKX-NEXT:    vmovaps %ymm1, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <8 x float>, <8 x float>* %vec2p
-  %shuf = shufflevector <8 x float> %vec1, <8 x float> %vec2, <8 x i32> <i32 2, i32 10, i32 3, i32 11, i32 6, i32 14, i32 7, i32 15>
-  %cmp = icmp eq <8 x i32> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x float> %shuf, <8 x float> %vec3
-  ret <8 x float> %res
-}
-
-define <8 x float> @test_8xfloat_zero_masked_unpack_high_mem_mask1(<8 x float> %vec1, <8 x float>* %vec2p, <8 x i32> %mask) {
-; GENERIC-LABEL: test_8xfloat_zero_masked_unpack_high_mem_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %ymm1, %ymm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpckhps {{.*#+}} ymm0 {%k1} {z} = ymm0[2],mem[2],ymm0[3],mem[3],ymm0[6],mem[6],ymm0[7],mem[7] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xfloat_zero_masked_unpack_high_mem_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %ymm1, %ymm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpckhps {{.*#+}} ymm0 {%k1} {z} = ymm0[2],mem[2],ymm0[3],mem[3],ymm0[6],mem[6],ymm0[7],mem[7] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <8 x float>, <8 x float>* %vec2p
-  %shuf = shufflevector <8 x float> %vec1, <8 x float> %vec2, <8 x i32> <i32 2, i32 10, i32 3, i32 11, i32 6, i32 14, i32 7, i32 15>
-  %cmp = icmp eq <8 x i32> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x float> %shuf, <8 x float> zeroinitializer
-  ret <8 x float> %res
-}
-
-define <8 x float> @test_8xfloat_masked_unpack_high_mem_mask2(<8 x float> %vec1, <8 x float>* %vec2p, <8 x float> %vec3, <8 x i32> %mask) {
-; GENERIC-LABEL: test_8xfloat_masked_unpack_high_mem_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %ymm2, %ymm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpckhps {{.*#+}} ymm1 {%k1} = ymm0[2],mem[2],ymm0[3],mem[3],ymm0[6],mem[6],ymm0[7],mem[7] sched: [8:1.00]
-; GENERIC-NEXT:    vmovaps %ymm1, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xfloat_masked_unpack_high_mem_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %ymm2, %ymm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpckhps {{.*#+}} ymm1 {%k1} = ymm0[2],mem[2],ymm0[3],mem[3],ymm0[6],mem[6],ymm0[7],mem[7] sched: [8:1.00]
-; SKX-NEXT:    vmovaps %ymm1, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <8 x float>, <8 x float>* %vec2p
-  %shuf = shufflevector <8 x float> %vec1, <8 x float> %vec2, <8 x i32> <i32 2, i32 10, i32 3, i32 11, i32 6, i32 14, i32 7, i32 15>
-  %cmp = icmp eq <8 x i32> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x float> %shuf, <8 x float> %vec3
-  ret <8 x float> %res
-}
-
-define <8 x float> @test_8xfloat_zero_masked_unpack_high_mem_mask2(<8 x float> %vec1, <8 x float>* %vec2p, <8 x i32> %mask) {
-; GENERIC-LABEL: test_8xfloat_zero_masked_unpack_high_mem_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %ymm1, %ymm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpckhps {{.*#+}} ymm0 {%k1} {z} = ymm0[2],mem[2],ymm0[3],mem[3],ymm0[6],mem[6],ymm0[7],mem[7] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xfloat_zero_masked_unpack_high_mem_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %ymm1, %ymm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpckhps {{.*#+}} ymm0 {%k1} {z} = ymm0[2],mem[2],ymm0[3],mem[3],ymm0[6],mem[6],ymm0[7],mem[7] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <8 x float>, <8 x float>* %vec2p
-  %shuf = shufflevector <8 x float> %vec1, <8 x float> %vec2, <8 x i32> <i32 2, i32 10, i32 3, i32 11, i32 6, i32 14, i32 7, i32 15>
-  %cmp = icmp eq <8 x i32> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x float> %shuf, <8 x float> zeroinitializer
-  ret <8 x float> %res
-}
-
-define <8 x float> @test_8xfloat_unpack_high_mem_mask3(<8 x float> %vec1, <8 x float>* %vec2p) {
-; GENERIC-LABEL: test_8xfloat_unpack_high_mem_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vunpckhps {{.*#+}} ymm0 = ymm0[2],mem[2],ymm0[3],mem[3],ymm0[6],mem[6],ymm0[7],mem[7] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xfloat_unpack_high_mem_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vunpckhps {{.*#+}} ymm0 = ymm0[2],mem[2],ymm0[3],mem[3],ymm0[6],mem[6],ymm0[7],mem[7] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <8 x float>, <8 x float>* %vec2p
-  %res = shufflevector <8 x float> %vec1, <8 x float> %vec2, <8 x i32> <i32 2, i32 10, i32 3, i32 11, i32 6, i32 14, i32 7, i32 15>
-  ret <8 x float> %res
-}
-define <8 x float> @test_8xfloat_masked_unpack_high_mem_mask3(<8 x float> %vec1, <8 x float>* %vec2p, <8 x float> %vec3, <8 x i32> %mask) {
-; GENERIC-LABEL: test_8xfloat_masked_unpack_high_mem_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %ymm2, %ymm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpckhps {{.*#+}} ymm1 {%k1} = ymm0[2],mem[2],ymm0[3],mem[3],ymm0[6],mem[6],ymm0[7],mem[7] sched: [8:1.00]
-; GENERIC-NEXT:    vmovaps %ymm1, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xfloat_masked_unpack_high_mem_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %ymm2, %ymm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpckhps {{.*#+}} ymm1 {%k1} = ymm0[2],mem[2],ymm0[3],mem[3],ymm0[6],mem[6],ymm0[7],mem[7] sched: [8:1.00]
-; SKX-NEXT:    vmovaps %ymm1, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <8 x float>, <8 x float>* %vec2p
-  %shuf = shufflevector <8 x float> %vec1, <8 x float> %vec2, <8 x i32> <i32 2, i32 10, i32 3, i32 11, i32 6, i32 14, i32 7, i32 15>
-  %cmp = icmp eq <8 x i32> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x float> %shuf, <8 x float> %vec3
-  ret <8 x float> %res
-}
-
-define <8 x float> @test_8xfloat_zero_masked_unpack_high_mem_mask3(<8 x float> %vec1, <8 x float>* %vec2p, <8 x i32> %mask) {
-; GENERIC-LABEL: test_8xfloat_zero_masked_unpack_high_mem_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %ymm1, %ymm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpckhps {{.*#+}} ymm0 {%k1} {z} = ymm0[2],mem[2],ymm0[3],mem[3],ymm0[6],mem[6],ymm0[7],mem[7] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xfloat_zero_masked_unpack_high_mem_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %ymm1, %ymm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpckhps {{.*#+}} ymm0 {%k1} {z} = ymm0[2],mem[2],ymm0[3],mem[3],ymm0[6],mem[6],ymm0[7],mem[7] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <8 x float>, <8 x float>* %vec2p
-  %shuf = shufflevector <8 x float> %vec1, <8 x float> %vec2, <8 x i32> <i32 2, i32 10, i32 3, i32 11, i32 6, i32 14, i32 7, i32 15>
-  %cmp = icmp eq <8 x i32> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x float> %shuf, <8 x float> zeroinitializer
-  ret <8 x float> %res
-}
-
-define <16 x float> @test_16xfloat_unpack_high_mask0(<16 x float> %vec1, <16 x float> %vec2) {
-; GENERIC-LABEL: test_16xfloat_unpack_high_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vunpckhps {{.*#+}} zmm0 = zmm0[2],zmm1[2],zmm0[3],zmm1[3],zmm0[6],zmm1[6],zmm0[7],zmm1[7],zmm0[10],zmm1[10],zmm0[11],zmm1[11],zmm0[14],zmm1[14],zmm0[15],zmm1[15] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_16xfloat_unpack_high_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vunpckhps {{.*#+}} zmm0 = zmm0[2],zmm1[2],zmm0[3],zmm1[3],zmm0[6],zmm1[6],zmm0[7],zmm1[7],zmm0[10],zmm1[10],zmm0[11],zmm1[11],zmm0[14],zmm1[14],zmm0[15],zmm1[15] sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %res = shufflevector <16 x float> %vec1, <16 x float> %vec2, <16 x i32> <i32 2, i32 18, i32 3, i32 19, i32 6, i32 22, i32 7, i32 23, i32 10, i32 26, i32 11, i32 27, i32 14, i32 30, i32 15, i32 31>
-  ret <16 x float> %res
-}
-define <16 x float> @test_16xfloat_masked_unpack_high_mask0(<16 x float> %vec1, <16 x float> %vec2, <16 x float> %vec3, <16 x i32> %mask) {
-; GENERIC-LABEL: test_16xfloat_masked_unpack_high_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %zmm3, %zmm3, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpckhps {{.*#+}} zmm2 {%k1} = zmm0[2],zmm1[2],zmm0[3],zmm1[3],zmm0[6],zmm1[6],zmm0[7],zmm1[7],zmm0[10],zmm1[10],zmm0[11],zmm1[11],zmm0[14],zmm1[14],zmm0[15],zmm1[15] sched: [1:1.00]
-; GENERIC-NEXT:    vmovaps %zmm2, %zmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_16xfloat_masked_unpack_high_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %zmm3, %zmm3, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpckhps {{.*#+}} zmm2 {%k1} = zmm0[2],zmm1[2],zmm0[3],zmm1[3],zmm0[6],zmm1[6],zmm0[7],zmm1[7],zmm0[10],zmm1[10],zmm0[11],zmm1[11],zmm0[14],zmm1[14],zmm0[15],zmm1[15] sched: [1:1.00]
-; SKX-NEXT:    vmovaps %zmm2, %zmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <16 x float> %vec1, <16 x float> %vec2, <16 x i32> <i32 2, i32 18, i32 3, i32 19, i32 6, i32 22, i32 7, i32 23, i32 10, i32 26, i32 11, i32 27, i32 14, i32 30, i32 15, i32 31>
-  %cmp = icmp eq <16 x i32> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x float> %shuf, <16 x float> %vec3
-  ret <16 x float> %res
-}
-
-define <16 x float> @test_16xfloat_zero_masked_unpack_high_mask0(<16 x float> %vec1, <16 x float> %vec2, <16 x i32> %mask) {
-; GENERIC-LABEL: test_16xfloat_zero_masked_unpack_high_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %zmm2, %zmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpckhps {{.*#+}} zmm0 {%k1} {z} = zmm0[2],zmm1[2],zmm0[3],zmm1[3],zmm0[6],zmm1[6],zmm0[7],zmm1[7],zmm0[10],zmm1[10],zmm0[11],zmm1[11],zmm0[14],zmm1[14],zmm0[15],zmm1[15] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_16xfloat_zero_masked_unpack_high_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %zmm2, %zmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpckhps {{.*#+}} zmm0 {%k1} {z} = zmm0[2],zmm1[2],zmm0[3],zmm1[3],zmm0[6],zmm1[6],zmm0[7],zmm1[7],zmm0[10],zmm1[10],zmm0[11],zmm1[11],zmm0[14],zmm1[14],zmm0[15],zmm1[15] sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <16 x float> %vec1, <16 x float> %vec2, <16 x i32> <i32 2, i32 18, i32 3, i32 19, i32 6, i32 22, i32 7, i32 23, i32 10, i32 26, i32 11, i32 27, i32 14, i32 30, i32 15, i32 31>
-  %cmp = icmp eq <16 x i32> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x float> %shuf, <16 x float> zeroinitializer
-  ret <16 x float> %res
-}
-define <16 x float> @test_16xfloat_masked_unpack_high_mask1(<16 x float> %vec1, <16 x float> %vec2, <16 x float> %vec3, <16 x i32> %mask) {
-; GENERIC-LABEL: test_16xfloat_masked_unpack_high_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %zmm3, %zmm3, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpckhps {{.*#+}} zmm2 {%k1} = zmm0[2],zmm1[2],zmm0[3],zmm1[3],zmm0[6],zmm1[6],zmm0[7],zmm1[7],zmm0[10],zmm1[10],zmm0[11],zmm1[11],zmm0[14],zmm1[14],zmm0[15],zmm1[15] sched: [1:1.00]
-; GENERIC-NEXT:    vmovaps %zmm2, %zmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_16xfloat_masked_unpack_high_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %zmm3, %zmm3, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpckhps {{.*#+}} zmm2 {%k1} = zmm0[2],zmm1[2],zmm0[3],zmm1[3],zmm0[6],zmm1[6],zmm0[7],zmm1[7],zmm0[10],zmm1[10],zmm0[11],zmm1[11],zmm0[14],zmm1[14],zmm0[15],zmm1[15] sched: [1:1.00]
-; SKX-NEXT:    vmovaps %zmm2, %zmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <16 x float> %vec1, <16 x float> %vec2, <16 x i32> <i32 2, i32 18, i32 3, i32 19, i32 6, i32 22, i32 7, i32 23, i32 10, i32 26, i32 11, i32 27, i32 14, i32 30, i32 15, i32 31>
-  %cmp = icmp eq <16 x i32> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x float> %shuf, <16 x float> %vec3
-  ret <16 x float> %res
-}
-
-define <16 x float> @test_16xfloat_zero_masked_unpack_high_mask1(<16 x float> %vec1, <16 x float> %vec2, <16 x i32> %mask) {
-; GENERIC-LABEL: test_16xfloat_zero_masked_unpack_high_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %zmm2, %zmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpckhps {{.*#+}} zmm0 {%k1} {z} = zmm0[2],zmm1[2],zmm0[3],zmm1[3],zmm0[6],zmm1[6],zmm0[7],zmm1[7],zmm0[10],zmm1[10],zmm0[11],zmm1[11],zmm0[14],zmm1[14],zmm0[15],zmm1[15] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_16xfloat_zero_masked_unpack_high_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %zmm2, %zmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpckhps {{.*#+}} zmm0 {%k1} {z} = zmm0[2],zmm1[2],zmm0[3],zmm1[3],zmm0[6],zmm1[6],zmm0[7],zmm1[7],zmm0[10],zmm1[10],zmm0[11],zmm1[11],zmm0[14],zmm1[14],zmm0[15],zmm1[15] sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <16 x float> %vec1, <16 x float> %vec2, <16 x i32> <i32 2, i32 18, i32 3, i32 19, i32 6, i32 22, i32 7, i32 23, i32 10, i32 26, i32 11, i32 27, i32 14, i32 30, i32 15, i32 31>
-  %cmp = icmp eq <16 x i32> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x float> %shuf, <16 x float> zeroinitializer
-  ret <16 x float> %res
-}
-define <16 x float> @test_16xfloat_masked_unpack_high_mask2(<16 x float> %vec1, <16 x float> %vec2, <16 x float> %vec3, <16 x i32> %mask) {
-; GENERIC-LABEL: test_16xfloat_masked_unpack_high_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %zmm3, %zmm3, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpckhps {{.*#+}} zmm2 {%k1} = zmm0[2],zmm1[2],zmm0[3],zmm1[3],zmm0[6],zmm1[6],zmm0[7],zmm1[7],zmm0[10],zmm1[10],zmm0[11],zmm1[11],zmm0[14],zmm1[14],zmm0[15],zmm1[15] sched: [1:1.00]
-; GENERIC-NEXT:    vmovaps %zmm2, %zmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_16xfloat_masked_unpack_high_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %zmm3, %zmm3, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpckhps {{.*#+}} zmm2 {%k1} = zmm0[2],zmm1[2],zmm0[3],zmm1[3],zmm0[6],zmm1[6],zmm0[7],zmm1[7],zmm0[10],zmm1[10],zmm0[11],zmm1[11],zmm0[14],zmm1[14],zmm0[15],zmm1[15] sched: [1:1.00]
-; SKX-NEXT:    vmovaps %zmm2, %zmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <16 x float> %vec1, <16 x float> %vec2, <16 x i32> <i32 2, i32 18, i32 3, i32 19, i32 6, i32 22, i32 7, i32 23, i32 10, i32 26, i32 11, i32 27, i32 14, i32 30, i32 15, i32 31>
-  %cmp = icmp eq <16 x i32> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x float> %shuf, <16 x float> %vec3
-  ret <16 x float> %res
-}
-
-define <16 x float> @test_16xfloat_zero_masked_unpack_high_mask2(<16 x float> %vec1, <16 x float> %vec2, <16 x i32> %mask) {
-; GENERIC-LABEL: test_16xfloat_zero_masked_unpack_high_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %zmm2, %zmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpckhps {{.*#+}} zmm0 {%k1} {z} = zmm0[2],zmm1[2],zmm0[3],zmm1[3],zmm0[6],zmm1[6],zmm0[7],zmm1[7],zmm0[10],zmm1[10],zmm0[11],zmm1[11],zmm0[14],zmm1[14],zmm0[15],zmm1[15] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_16xfloat_zero_masked_unpack_high_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %zmm2, %zmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpckhps {{.*#+}} zmm0 {%k1} {z} = zmm0[2],zmm1[2],zmm0[3],zmm1[3],zmm0[6],zmm1[6],zmm0[7],zmm1[7],zmm0[10],zmm1[10],zmm0[11],zmm1[11],zmm0[14],zmm1[14],zmm0[15],zmm1[15] sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <16 x float> %vec1, <16 x float> %vec2, <16 x i32> <i32 2, i32 18, i32 3, i32 19, i32 6, i32 22, i32 7, i32 23, i32 10, i32 26, i32 11, i32 27, i32 14, i32 30, i32 15, i32 31>
-  %cmp = icmp eq <16 x i32> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x float> %shuf, <16 x float> zeroinitializer
-  ret <16 x float> %res
-}
-define <16 x float> @test_16xfloat_unpack_high_mask3(<16 x float> %vec1, <16 x float> %vec2) {
-; GENERIC-LABEL: test_16xfloat_unpack_high_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vunpckhps {{.*#+}} zmm0 = zmm0[2],zmm1[2],zmm0[3],zmm1[3],zmm0[6],zmm1[6],zmm0[7],zmm1[7],zmm0[10],zmm1[10],zmm0[11],zmm1[11],zmm0[14],zmm1[14],zmm0[15],zmm1[15] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_16xfloat_unpack_high_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vunpckhps {{.*#+}} zmm0 = zmm0[2],zmm1[2],zmm0[3],zmm1[3],zmm0[6],zmm1[6],zmm0[7],zmm1[7],zmm0[10],zmm1[10],zmm0[11],zmm1[11],zmm0[14],zmm1[14],zmm0[15],zmm1[15] sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %res = shufflevector <16 x float> %vec1, <16 x float> %vec2, <16 x i32> <i32 2, i32 18, i32 3, i32 19, i32 6, i32 22, i32 7, i32 23, i32 10, i32 26, i32 11, i32 27, i32 14, i32 30, i32 15, i32 31>
-  ret <16 x float> %res
-}
-define <16 x float> @test_16xfloat_masked_unpack_high_mask3(<16 x float> %vec1, <16 x float> %vec2, <16 x float> %vec3, <16 x i32> %mask) {
-; GENERIC-LABEL: test_16xfloat_masked_unpack_high_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %zmm3, %zmm3, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpckhps {{.*#+}} zmm2 {%k1} = zmm0[2],zmm1[2],zmm0[3],zmm1[3],zmm0[6],zmm1[6],zmm0[7],zmm1[7],zmm0[10],zmm1[10],zmm0[11],zmm1[11],zmm0[14],zmm1[14],zmm0[15],zmm1[15] sched: [1:1.00]
-; GENERIC-NEXT:    vmovaps %zmm2, %zmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_16xfloat_masked_unpack_high_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %zmm3, %zmm3, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpckhps {{.*#+}} zmm2 {%k1} = zmm0[2],zmm1[2],zmm0[3],zmm1[3],zmm0[6],zmm1[6],zmm0[7],zmm1[7],zmm0[10],zmm1[10],zmm0[11],zmm1[11],zmm0[14],zmm1[14],zmm0[15],zmm1[15] sched: [1:1.00]
-; SKX-NEXT:    vmovaps %zmm2, %zmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <16 x float> %vec1, <16 x float> %vec2, <16 x i32> <i32 2, i32 18, i32 3, i32 19, i32 6, i32 22, i32 7, i32 23, i32 10, i32 26, i32 11, i32 27, i32 14, i32 30, i32 15, i32 31>
-  %cmp = icmp eq <16 x i32> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x float> %shuf, <16 x float> %vec3
-  ret <16 x float> %res
-}
-
-define <16 x float> @test_16xfloat_zero_masked_unpack_high_mask3(<16 x float> %vec1, <16 x float> %vec2, <16 x i32> %mask) {
-; GENERIC-LABEL: test_16xfloat_zero_masked_unpack_high_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %zmm2, %zmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpckhps {{.*#+}} zmm0 {%k1} {z} = zmm0[2],zmm1[2],zmm0[3],zmm1[3],zmm0[6],zmm1[6],zmm0[7],zmm1[7],zmm0[10],zmm1[10],zmm0[11],zmm1[11],zmm0[14],zmm1[14],zmm0[15],zmm1[15] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_16xfloat_zero_masked_unpack_high_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %zmm2, %zmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpckhps {{.*#+}} zmm0 {%k1} {z} = zmm0[2],zmm1[2],zmm0[3],zmm1[3],zmm0[6],zmm1[6],zmm0[7],zmm1[7],zmm0[10],zmm1[10],zmm0[11],zmm1[11],zmm0[14],zmm1[14],zmm0[15],zmm1[15] sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <16 x float> %vec1, <16 x float> %vec2, <16 x i32> <i32 2, i32 18, i32 3, i32 19, i32 6, i32 22, i32 7, i32 23, i32 10, i32 26, i32 11, i32 27, i32 14, i32 30, i32 15, i32 31>
-  %cmp = icmp eq <16 x i32> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x float> %shuf, <16 x float> zeroinitializer
-  ret <16 x float> %res
-}
-define <16 x float> @test_16xfloat_unpack_high_mem_mask0(<16 x float> %vec1, <16 x float>* %vec2p) {
-; GENERIC-LABEL: test_16xfloat_unpack_high_mem_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vunpckhps {{.*#+}} zmm0 = zmm0[2],mem[2],zmm0[3],mem[3],zmm0[6],mem[6],zmm0[7],mem[7],zmm0[10],mem[10],zmm0[11],mem[11],zmm0[14],mem[14],zmm0[15],mem[15] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_16xfloat_unpack_high_mem_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vunpckhps {{.*#+}} zmm0 = zmm0[2],mem[2],zmm0[3],mem[3],zmm0[6],mem[6],zmm0[7],mem[7],zmm0[10],mem[10],zmm0[11],mem[11],zmm0[14],mem[14],zmm0[15],mem[15] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <16 x float>, <16 x float>* %vec2p
-  %res = shufflevector <16 x float> %vec1, <16 x float> %vec2, <16 x i32> <i32 2, i32 18, i32 3, i32 19, i32 6, i32 22, i32 7, i32 23, i32 10, i32 26, i32 11, i32 27, i32 14, i32 30, i32 15, i32 31>
-  ret <16 x float> %res
-}
-define <16 x float> @test_16xfloat_masked_unpack_high_mem_mask0(<16 x float> %vec1, <16 x float>* %vec2p, <16 x float> %vec3, <16 x i32> %mask) {
-; GENERIC-LABEL: test_16xfloat_masked_unpack_high_mem_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %zmm2, %zmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpckhps {{.*#+}} zmm1 {%k1} = zmm0[2],mem[2],zmm0[3],mem[3],zmm0[6],mem[6],zmm0[7],mem[7],zmm0[10],mem[10],zmm0[11],mem[11],zmm0[14],mem[14],zmm0[15],mem[15] sched: [8:1.00]
-; GENERIC-NEXT:    vmovaps %zmm1, %zmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_16xfloat_masked_unpack_high_mem_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %zmm2, %zmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpckhps {{.*#+}} zmm1 {%k1} = zmm0[2],mem[2],zmm0[3],mem[3],zmm0[6],mem[6],zmm0[7],mem[7],zmm0[10],mem[10],zmm0[11],mem[11],zmm0[14],mem[14],zmm0[15],mem[15] sched: [8:1.00]
-; SKX-NEXT:    vmovaps %zmm1, %zmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <16 x float>, <16 x float>* %vec2p
-  %shuf = shufflevector <16 x float> %vec1, <16 x float> %vec2, <16 x i32> <i32 2, i32 18, i32 3, i32 19, i32 6, i32 22, i32 7, i32 23, i32 10, i32 26, i32 11, i32 27, i32 14, i32 30, i32 15, i32 31>
-  %cmp = icmp eq <16 x i32> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x float> %shuf, <16 x float> %vec3
-  ret <16 x float> %res
-}
-
-define <16 x float> @test_16xfloat_zero_masked_unpack_high_mem_mask0(<16 x float> %vec1, <16 x float>* %vec2p, <16 x i32> %mask) {
-; GENERIC-LABEL: test_16xfloat_zero_masked_unpack_high_mem_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpckhps {{.*#+}} zmm0 {%k1} {z} = zmm0[2],mem[2],zmm0[3],mem[3],zmm0[6],mem[6],zmm0[7],mem[7],zmm0[10],mem[10],zmm0[11],mem[11],zmm0[14],mem[14],zmm0[15],mem[15] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_16xfloat_zero_masked_unpack_high_mem_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpckhps {{.*#+}} zmm0 {%k1} {z} = zmm0[2],mem[2],zmm0[3],mem[3],zmm0[6],mem[6],zmm0[7],mem[7],zmm0[10],mem[10],zmm0[11],mem[11],zmm0[14],mem[14],zmm0[15],mem[15] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <16 x float>, <16 x float>* %vec2p
-  %shuf = shufflevector <16 x float> %vec1, <16 x float> %vec2, <16 x i32> <i32 2, i32 18, i32 3, i32 19, i32 6, i32 22, i32 7, i32 23, i32 10, i32 26, i32 11, i32 27, i32 14, i32 30, i32 15, i32 31>
-  %cmp = icmp eq <16 x i32> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x float> %shuf, <16 x float> zeroinitializer
-  ret <16 x float> %res
-}
-
-define <16 x float> @test_16xfloat_masked_unpack_high_mem_mask1(<16 x float> %vec1, <16 x float>* %vec2p, <16 x float> %vec3, <16 x i32> %mask) {
-; GENERIC-LABEL: test_16xfloat_masked_unpack_high_mem_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %zmm2, %zmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpckhps {{.*#+}} zmm1 {%k1} = zmm0[2],mem[2],zmm0[3],mem[3],zmm0[6],mem[6],zmm0[7],mem[7],zmm0[10],mem[10],zmm0[11],mem[11],zmm0[14],mem[14],zmm0[15],mem[15] sched: [8:1.00]
-; GENERIC-NEXT:    vmovaps %zmm1, %zmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_16xfloat_masked_unpack_high_mem_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %zmm2, %zmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpckhps {{.*#+}} zmm1 {%k1} = zmm0[2],mem[2],zmm0[3],mem[3],zmm0[6],mem[6],zmm0[7],mem[7],zmm0[10],mem[10],zmm0[11],mem[11],zmm0[14],mem[14],zmm0[15],mem[15] sched: [8:1.00]
-; SKX-NEXT:    vmovaps %zmm1, %zmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <16 x float>, <16 x float>* %vec2p
-  %shuf = shufflevector <16 x float> %vec1, <16 x float> %vec2, <16 x i32> <i32 2, i32 18, i32 3, i32 19, i32 6, i32 22, i32 7, i32 23, i32 10, i32 26, i32 11, i32 27, i32 14, i32 30, i32 15, i32 31>
-  %cmp = icmp eq <16 x i32> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x float> %shuf, <16 x float> %vec3
-  ret <16 x float> %res
-}
-
-define <16 x float> @test_16xfloat_zero_masked_unpack_high_mem_mask1(<16 x float> %vec1, <16 x float>* %vec2p, <16 x i32> %mask) {
-; GENERIC-LABEL: test_16xfloat_zero_masked_unpack_high_mem_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpckhps {{.*#+}} zmm0 {%k1} {z} = zmm0[2],mem[2],zmm0[3],mem[3],zmm0[6],mem[6],zmm0[7],mem[7],zmm0[10],mem[10],zmm0[11],mem[11],zmm0[14],mem[14],zmm0[15],mem[15] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_16xfloat_zero_masked_unpack_high_mem_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpckhps {{.*#+}} zmm0 {%k1} {z} = zmm0[2],mem[2],zmm0[3],mem[3],zmm0[6],mem[6],zmm0[7],mem[7],zmm0[10],mem[10],zmm0[11],mem[11],zmm0[14],mem[14],zmm0[15],mem[15] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <16 x float>, <16 x float>* %vec2p
-  %shuf = shufflevector <16 x float> %vec1, <16 x float> %vec2, <16 x i32> <i32 2, i32 18, i32 3, i32 19, i32 6, i32 22, i32 7, i32 23, i32 10, i32 26, i32 11, i32 27, i32 14, i32 30, i32 15, i32 31>
-  %cmp = icmp eq <16 x i32> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x float> %shuf, <16 x float> zeroinitializer
-  ret <16 x float> %res
-}
-
-define <16 x float> @test_16xfloat_masked_unpack_high_mem_mask2(<16 x float> %vec1, <16 x float>* %vec2p, <16 x float> %vec3, <16 x i32> %mask) {
-; GENERIC-LABEL: test_16xfloat_masked_unpack_high_mem_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %zmm2, %zmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpckhps {{.*#+}} zmm1 {%k1} = zmm0[2],mem[2],zmm0[3],mem[3],zmm0[6],mem[6],zmm0[7],mem[7],zmm0[10],mem[10],zmm0[11],mem[11],zmm0[14],mem[14],zmm0[15],mem[15] sched: [8:1.00]
-; GENERIC-NEXT:    vmovaps %zmm1, %zmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_16xfloat_masked_unpack_high_mem_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %zmm2, %zmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpckhps {{.*#+}} zmm1 {%k1} = zmm0[2],mem[2],zmm0[3],mem[3],zmm0[6],mem[6],zmm0[7],mem[7],zmm0[10],mem[10],zmm0[11],mem[11],zmm0[14],mem[14],zmm0[15],mem[15] sched: [8:1.00]
-; SKX-NEXT:    vmovaps %zmm1, %zmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <16 x float>, <16 x float>* %vec2p
-  %shuf = shufflevector <16 x float> %vec1, <16 x float> %vec2, <16 x i32> <i32 2, i32 18, i32 3, i32 19, i32 6, i32 22, i32 7, i32 23, i32 10, i32 26, i32 11, i32 27, i32 14, i32 30, i32 15, i32 31>
-  %cmp = icmp eq <16 x i32> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x float> %shuf, <16 x float> %vec3
-  ret <16 x float> %res
-}
-
-define <16 x float> @test_16xfloat_zero_masked_unpack_high_mem_mask2(<16 x float> %vec1, <16 x float>* %vec2p, <16 x i32> %mask) {
-; GENERIC-LABEL: test_16xfloat_zero_masked_unpack_high_mem_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpckhps {{.*#+}} zmm0 {%k1} {z} = zmm0[2],mem[2],zmm0[3],mem[3],zmm0[6],mem[6],zmm0[7],mem[7],zmm0[10],mem[10],zmm0[11],mem[11],zmm0[14],mem[14],zmm0[15],mem[15] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_16xfloat_zero_masked_unpack_high_mem_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpckhps {{.*#+}} zmm0 {%k1} {z} = zmm0[2],mem[2],zmm0[3],mem[3],zmm0[6],mem[6],zmm0[7],mem[7],zmm0[10],mem[10],zmm0[11],mem[11],zmm0[14],mem[14],zmm0[15],mem[15] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <16 x float>, <16 x float>* %vec2p
-  %shuf = shufflevector <16 x float> %vec1, <16 x float> %vec2, <16 x i32> <i32 2, i32 18, i32 3, i32 19, i32 6, i32 22, i32 7, i32 23, i32 10, i32 26, i32 11, i32 27, i32 14, i32 30, i32 15, i32 31>
-  %cmp = icmp eq <16 x i32> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x float> %shuf, <16 x float> zeroinitializer
-  ret <16 x float> %res
-}
-
-define <16 x float> @test_16xfloat_unpack_high_mem_mask3(<16 x float> %vec1, <16 x float>* %vec2p) {
-; GENERIC-LABEL: test_16xfloat_unpack_high_mem_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vunpckhps {{.*#+}} zmm0 = zmm0[2],mem[2],zmm0[3],mem[3],zmm0[6],mem[6],zmm0[7],mem[7],zmm0[10],mem[10],zmm0[11],mem[11],zmm0[14],mem[14],zmm0[15],mem[15] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_16xfloat_unpack_high_mem_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vunpckhps {{.*#+}} zmm0 = zmm0[2],mem[2],zmm0[3],mem[3],zmm0[6],mem[6],zmm0[7],mem[7],zmm0[10],mem[10],zmm0[11],mem[11],zmm0[14],mem[14],zmm0[15],mem[15] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <16 x float>, <16 x float>* %vec2p
-  %res = shufflevector <16 x float> %vec1, <16 x float> %vec2, <16 x i32> <i32 2, i32 18, i32 3, i32 19, i32 6, i32 22, i32 7, i32 23, i32 10, i32 26, i32 11, i32 27, i32 14, i32 30, i32 15, i32 31>
-  ret <16 x float> %res
-}
-define <16 x float> @test_16xfloat_masked_unpack_high_mem_mask3(<16 x float> %vec1, <16 x float>* %vec2p, <16 x float> %vec3, <16 x i32> %mask) {
-; GENERIC-LABEL: test_16xfloat_masked_unpack_high_mem_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %zmm2, %zmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpckhps {{.*#+}} zmm1 {%k1} = zmm0[2],mem[2],zmm0[3],mem[3],zmm0[6],mem[6],zmm0[7],mem[7],zmm0[10],mem[10],zmm0[11],mem[11],zmm0[14],mem[14],zmm0[15],mem[15] sched: [8:1.00]
-; GENERIC-NEXT:    vmovaps %zmm1, %zmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_16xfloat_masked_unpack_high_mem_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %zmm2, %zmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpckhps {{.*#+}} zmm1 {%k1} = zmm0[2],mem[2],zmm0[3],mem[3],zmm0[6],mem[6],zmm0[7],mem[7],zmm0[10],mem[10],zmm0[11],mem[11],zmm0[14],mem[14],zmm0[15],mem[15] sched: [8:1.00]
-; SKX-NEXT:    vmovaps %zmm1, %zmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <16 x float>, <16 x float>* %vec2p
-  %shuf = shufflevector <16 x float> %vec1, <16 x float> %vec2, <16 x i32> <i32 2, i32 18, i32 3, i32 19, i32 6, i32 22, i32 7, i32 23, i32 10, i32 26, i32 11, i32 27, i32 14, i32 30, i32 15, i32 31>
-  %cmp = icmp eq <16 x i32> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x float> %shuf, <16 x float> %vec3
-  ret <16 x float> %res
-}
-
-define <16 x float> @test_16xfloat_zero_masked_unpack_high_mem_mask3(<16 x float> %vec1, <16 x float>* %vec2p, <16 x i32> %mask) {
-; GENERIC-LABEL: test_16xfloat_zero_masked_unpack_high_mem_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmd %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpckhps {{.*#+}} zmm0 {%k1} {z} = zmm0[2],mem[2],zmm0[3],mem[3],zmm0[6],mem[6],zmm0[7],mem[7],zmm0[10],mem[10],zmm0[11],mem[11],zmm0[14],mem[14],zmm0[15],mem[15] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_16xfloat_zero_masked_unpack_high_mem_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmd %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpckhps {{.*#+}} zmm0 {%k1} {z} = zmm0[2],mem[2],zmm0[3],mem[3],zmm0[6],mem[6],zmm0[7],mem[7],zmm0[10],mem[10],zmm0[11],mem[11],zmm0[14],mem[14],zmm0[15],mem[15] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <16 x float>, <16 x float>* %vec2p
-  %shuf = shufflevector <16 x float> %vec1, <16 x float> %vec2, <16 x i32> <i32 2, i32 18, i32 3, i32 19, i32 6, i32 22, i32 7, i32 23, i32 10, i32 26, i32 11, i32 27, i32 14, i32 30, i32 15, i32 31>
-  %cmp = icmp eq <16 x i32> %mask, zeroinitializer
-  %res = select <16 x i1> %cmp, <16 x float> %shuf, <16 x float> zeroinitializer
-  ret <16 x float> %res
-}
-
-define <2 x double> @test_2xdouble_unpack_high_mask0(<2 x double> %vec1, <2 x double> %vec2) {
-; GENERIC-LABEL: test_2xdouble_unpack_high_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vunpckhpd {{.*#+}} xmm0 = xmm0[1],xmm1[1] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_2xdouble_unpack_high_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vunpckhpd {{.*#+}} xmm0 = xmm0[1],xmm1[1] sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %res = shufflevector <2 x double> %vec1, <2 x double> %vec2, <2 x i32> <i32 1, i32 3>
-  ret <2 x double> %res
-}
-define <2 x double> @test_2xdouble_masked_unpack_high_mask0(<2 x double> %vec1, <2 x double> %vec2, <2 x double> %vec3, <2 x i64> %mask) {
-; GENERIC-LABEL: test_2xdouble_masked_unpack_high_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %xmm3, %xmm3, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpckhpd {{.*#+}} xmm2 {%k1} = xmm0[1],xmm1[1] sched: [1:1.00]
-; GENERIC-NEXT:    vmovapd %xmm2, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_2xdouble_masked_unpack_high_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %xmm3, %xmm3, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpckhpd {{.*#+}} xmm2 {%k1} = xmm0[1],xmm1[1] sched: [1:1.00]
-; SKX-NEXT:    vmovapd %xmm2, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <2 x double> %vec1, <2 x double> %vec2, <2 x i32> <i32 1, i32 3>
-  %cmp = icmp eq <2 x i64> %mask, zeroinitializer
-  %res = select <2 x i1> %cmp, <2 x double> %shuf, <2 x double> %vec3
-  ret <2 x double> %res
-}
-
-define <2 x double> @test_2xdouble_zero_masked_unpack_high_mask0(<2 x double> %vec1, <2 x double> %vec2, <2 x i64> %mask) {
-; GENERIC-LABEL: test_2xdouble_zero_masked_unpack_high_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %xmm2, %xmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpckhpd {{.*#+}} xmm0 {%k1} {z} = xmm0[1],xmm1[1] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_2xdouble_zero_masked_unpack_high_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %xmm2, %xmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpckhpd {{.*#+}} xmm0 {%k1} {z} = xmm0[1],xmm1[1] sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <2 x double> %vec1, <2 x double> %vec2, <2 x i32> <i32 1, i32 3>
-  %cmp = icmp eq <2 x i64> %mask, zeroinitializer
-  %res = select <2 x i1> %cmp, <2 x double> %shuf, <2 x double> zeroinitializer
-  ret <2 x double> %res
-}
-define <2 x double> @test_2xdouble_masked_unpack_high_mask1(<2 x double> %vec1, <2 x double> %vec2, <2 x double> %vec3, <2 x i64> %mask) {
-; GENERIC-LABEL: test_2xdouble_masked_unpack_high_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %xmm3, %xmm3, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpckhpd {{.*#+}} xmm2 {%k1} = xmm0[1],xmm1[1] sched: [1:1.00]
-; GENERIC-NEXT:    vmovapd %xmm2, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_2xdouble_masked_unpack_high_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %xmm3, %xmm3, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpckhpd {{.*#+}} xmm2 {%k1} = xmm0[1],xmm1[1] sched: [1:1.00]
-; SKX-NEXT:    vmovapd %xmm2, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <2 x double> %vec1, <2 x double> %vec2, <2 x i32> <i32 1, i32 3>
-  %cmp = icmp eq <2 x i64> %mask, zeroinitializer
-  %res = select <2 x i1> %cmp, <2 x double> %shuf, <2 x double> %vec3
-  ret <2 x double> %res
-}
-
-define <2 x double> @test_2xdouble_zero_masked_unpack_high_mask1(<2 x double> %vec1, <2 x double> %vec2, <2 x i64> %mask) {
-; GENERIC-LABEL: test_2xdouble_zero_masked_unpack_high_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %xmm2, %xmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpckhpd {{.*#+}} xmm0 {%k1} {z} = xmm0[1],xmm1[1] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_2xdouble_zero_masked_unpack_high_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %xmm2, %xmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpckhpd {{.*#+}} xmm0 {%k1} {z} = xmm0[1],xmm1[1] sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <2 x double> %vec1, <2 x double> %vec2, <2 x i32> <i32 1, i32 3>
-  %cmp = icmp eq <2 x i64> %mask, zeroinitializer
-  %res = select <2 x i1> %cmp, <2 x double> %shuf, <2 x double> zeroinitializer
-  ret <2 x double> %res
-}
-define <2 x double> @test_2xdouble_unpack_high_mem_mask0(<2 x double> %vec1, <2 x double>* %vec2p) {
-; GENERIC-LABEL: test_2xdouble_unpack_high_mem_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vunpckhpd {{.*#+}} xmm0 = xmm0[1],mem[1] sched: [7:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_2xdouble_unpack_high_mem_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vunpckhpd {{.*#+}} xmm0 = xmm0[1],mem[1] sched: [7:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <2 x double>, <2 x double>* %vec2p
-  %res = shufflevector <2 x double> %vec1, <2 x double> %vec2, <2 x i32> <i32 1, i32 3>
-  ret <2 x double> %res
-}
-define <2 x double> @test_2xdouble_masked_unpack_high_mem_mask0(<2 x double> %vec1, <2 x double>* %vec2p, <2 x double> %vec3, <2 x i64> %mask) {
-; GENERIC-LABEL: test_2xdouble_masked_unpack_high_mem_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %xmm2, %xmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpckhpd {{.*#+}} xmm1 {%k1} = xmm0[1],mem[1] sched: [7:1.00]
-; GENERIC-NEXT:    vmovapd %xmm1, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_2xdouble_masked_unpack_high_mem_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %xmm2, %xmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpckhpd {{.*#+}} xmm1 {%k1} = xmm0[1],mem[1] sched: [7:1.00]
-; SKX-NEXT:    vmovapd %xmm1, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <2 x double>, <2 x double>* %vec2p
-  %shuf = shufflevector <2 x double> %vec1, <2 x double> %vec2, <2 x i32> <i32 1, i32 3>
-  %cmp = icmp eq <2 x i64> %mask, zeroinitializer
-  %res = select <2 x i1> %cmp, <2 x double> %shuf, <2 x double> %vec3
-  ret <2 x double> %res
-}
-
-define <2 x double> @test_2xdouble_zero_masked_unpack_high_mem_mask0(<2 x double> %vec1, <2 x double>* %vec2p, <2 x i64> %mask) {
-; GENERIC-LABEL: test_2xdouble_zero_masked_unpack_high_mem_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %xmm1, %xmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpckhpd {{.*#+}} xmm0 {%k1} {z} = xmm0[1],mem[1] sched: [7:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_2xdouble_zero_masked_unpack_high_mem_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %xmm1, %xmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpckhpd {{.*#+}} xmm0 {%k1} {z} = xmm0[1],mem[1] sched: [7:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <2 x double>, <2 x double>* %vec2p
-  %shuf = shufflevector <2 x double> %vec1, <2 x double> %vec2, <2 x i32> <i32 1, i32 3>
-  %cmp = icmp eq <2 x i64> %mask, zeroinitializer
-  %res = select <2 x i1> %cmp, <2 x double> %shuf, <2 x double> zeroinitializer
-  ret <2 x double> %res
-}
-
-define <2 x double> @test_2xdouble_masked_unpack_high_mem_mask1(<2 x double> %vec1, <2 x double>* %vec2p, <2 x double> %vec3, <2 x i64> %mask) {
-; GENERIC-LABEL: test_2xdouble_masked_unpack_high_mem_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %xmm2, %xmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpckhpd {{.*#+}} xmm1 {%k1} = xmm0[1],mem[1] sched: [7:1.00]
-; GENERIC-NEXT:    vmovapd %xmm1, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_2xdouble_masked_unpack_high_mem_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %xmm2, %xmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpckhpd {{.*#+}} xmm1 {%k1} = xmm0[1],mem[1] sched: [7:1.00]
-; SKX-NEXT:    vmovapd %xmm1, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <2 x double>, <2 x double>* %vec2p
-  %shuf = shufflevector <2 x double> %vec1, <2 x double> %vec2, <2 x i32> <i32 1, i32 3>
-  %cmp = icmp eq <2 x i64> %mask, zeroinitializer
-  %res = select <2 x i1> %cmp, <2 x double> %shuf, <2 x double> %vec3
-  ret <2 x double> %res
-}
-
-define <2 x double> @test_2xdouble_zero_masked_unpack_high_mem_mask1(<2 x double> %vec1, <2 x double>* %vec2p, <2 x i64> %mask) {
-; GENERIC-LABEL: test_2xdouble_zero_masked_unpack_high_mem_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %xmm1, %xmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpckhpd {{.*#+}} xmm0 {%k1} {z} = xmm0[1],mem[1] sched: [7:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_2xdouble_zero_masked_unpack_high_mem_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %xmm1, %xmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpckhpd {{.*#+}} xmm0 {%k1} {z} = xmm0[1],mem[1] sched: [7:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <2 x double>, <2 x double>* %vec2p
-  %shuf = shufflevector <2 x double> %vec1, <2 x double> %vec2, <2 x i32> <i32 1, i32 3>
-  %cmp = icmp eq <2 x i64> %mask, zeroinitializer
-  %res = select <2 x i1> %cmp, <2 x double> %shuf, <2 x double> zeroinitializer
-  ret <2 x double> %res
-}
-
-define <4 x double> @test_4xdouble_unpack_high_mask0(<4 x double> %vec1, <4 x double> %vec2) {
-; GENERIC-LABEL: test_4xdouble_unpack_high_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vunpckhpd {{.*#+}} ymm0 = ymm0[1],ymm1[1],ymm0[3],ymm1[3] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xdouble_unpack_high_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vunpckhpd {{.*#+}} ymm0 = ymm0[1],ymm1[1],ymm0[3],ymm1[3] sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %res = shufflevector <4 x double> %vec1, <4 x double> %vec2, <4 x i32> <i32 1, i32 5, i32 3, i32 7>
-  ret <4 x double> %res
-}
-define <4 x double> @test_4xdouble_masked_unpack_high_mask0(<4 x double> %vec1, <4 x double> %vec2, <4 x double> %vec3, <4 x i64> %mask) {
-; GENERIC-LABEL: test_4xdouble_masked_unpack_high_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %ymm3, %ymm3, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpckhpd {{.*#+}} ymm2 {%k1} = ymm0[1],ymm1[1],ymm0[3],ymm1[3] sched: [1:1.00]
-; GENERIC-NEXT:    vmovapd %ymm2, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xdouble_masked_unpack_high_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %ymm3, %ymm3, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpckhpd {{.*#+}} ymm2 {%k1} = ymm0[1],ymm1[1],ymm0[3],ymm1[3] sched: [1:1.00]
-; SKX-NEXT:    vmovapd %ymm2, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <4 x double> %vec1, <4 x double> %vec2, <4 x i32> <i32 1, i32 5, i32 3, i32 7>
-  %cmp = icmp eq <4 x i64> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x double> %shuf, <4 x double> %vec3
-  ret <4 x double> %res
-}
-
-define <4 x double> @test_4xdouble_zero_masked_unpack_high_mask0(<4 x double> %vec1, <4 x double> %vec2, <4 x i64> %mask) {
-; GENERIC-LABEL: test_4xdouble_zero_masked_unpack_high_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %ymm2, %ymm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpckhpd {{.*#+}} ymm0 {%k1} {z} = ymm0[1],ymm1[1],ymm0[3],ymm1[3] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xdouble_zero_masked_unpack_high_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %ymm2, %ymm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpckhpd {{.*#+}} ymm0 {%k1} {z} = ymm0[1],ymm1[1],ymm0[3],ymm1[3] sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <4 x double> %vec1, <4 x double> %vec2, <4 x i32> <i32 1, i32 5, i32 3, i32 7>
-  %cmp = icmp eq <4 x i64> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x double> %shuf, <4 x double> zeroinitializer
-  ret <4 x double> %res
-}
-define <4 x double> @test_4xdouble_masked_unpack_high_mask1(<4 x double> %vec1, <4 x double> %vec2, <4 x double> %vec3, <4 x i64> %mask) {
-; GENERIC-LABEL: test_4xdouble_masked_unpack_high_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %ymm3, %ymm3, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpckhpd {{.*#+}} ymm2 {%k1} = ymm0[1],ymm1[1],ymm0[3],ymm1[3] sched: [1:1.00]
-; GENERIC-NEXT:    vmovapd %ymm2, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xdouble_masked_unpack_high_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %ymm3, %ymm3, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpckhpd {{.*#+}} ymm2 {%k1} = ymm0[1],ymm1[1],ymm0[3],ymm1[3] sched: [1:1.00]
-; SKX-NEXT:    vmovapd %ymm2, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <4 x double> %vec1, <4 x double> %vec2, <4 x i32> <i32 1, i32 5, i32 3, i32 7>
-  %cmp = icmp eq <4 x i64> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x double> %shuf, <4 x double> %vec3
-  ret <4 x double> %res
-}
-
-define <4 x double> @test_4xdouble_zero_masked_unpack_high_mask1(<4 x double> %vec1, <4 x double> %vec2, <4 x i64> %mask) {
-; GENERIC-LABEL: test_4xdouble_zero_masked_unpack_high_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %ymm2, %ymm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpckhpd {{.*#+}} ymm0 {%k1} {z} = ymm0[1],ymm1[1],ymm0[3],ymm1[3] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xdouble_zero_masked_unpack_high_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %ymm2, %ymm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpckhpd {{.*#+}} ymm0 {%k1} {z} = ymm0[1],ymm1[1],ymm0[3],ymm1[3] sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <4 x double> %vec1, <4 x double> %vec2, <4 x i32> <i32 1, i32 5, i32 3, i32 7>
-  %cmp = icmp eq <4 x i64> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x double> %shuf, <4 x double> zeroinitializer
-  ret <4 x double> %res
-}
-define <4 x double> @test_4xdouble_masked_unpack_high_mask2(<4 x double> %vec1, <4 x double> %vec2, <4 x double> %vec3, <4 x i64> %mask) {
-; GENERIC-LABEL: test_4xdouble_masked_unpack_high_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %ymm3, %ymm3, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpckhpd {{.*#+}} ymm2 {%k1} = ymm0[1],ymm1[1],ymm0[3],ymm1[3] sched: [1:1.00]
-; GENERIC-NEXT:    vmovapd %ymm2, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xdouble_masked_unpack_high_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %ymm3, %ymm3, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpckhpd {{.*#+}} ymm2 {%k1} = ymm0[1],ymm1[1],ymm0[3],ymm1[3] sched: [1:1.00]
-; SKX-NEXT:    vmovapd %ymm2, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <4 x double> %vec1, <4 x double> %vec2, <4 x i32> <i32 1, i32 5, i32 3, i32 7>
-  %cmp = icmp eq <4 x i64> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x double> %shuf, <4 x double> %vec3
-  ret <4 x double> %res
-}
-
-define <4 x double> @test_4xdouble_zero_masked_unpack_high_mask2(<4 x double> %vec1, <4 x double> %vec2, <4 x i64> %mask) {
-; GENERIC-LABEL: test_4xdouble_zero_masked_unpack_high_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %ymm2, %ymm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpckhpd {{.*#+}} ymm0 {%k1} {z} = ymm0[1],ymm1[1],ymm0[3],ymm1[3] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xdouble_zero_masked_unpack_high_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %ymm2, %ymm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpckhpd {{.*#+}} ymm0 {%k1} {z} = ymm0[1],ymm1[1],ymm0[3],ymm1[3] sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <4 x double> %vec1, <4 x double> %vec2, <4 x i32> <i32 1, i32 5, i32 3, i32 7>
-  %cmp = icmp eq <4 x i64> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x double> %shuf, <4 x double> zeroinitializer
-  ret <4 x double> %res
-}
-define <4 x double> @test_4xdouble_unpack_high_mask3(<4 x double> %vec1, <4 x double> %vec2) {
-; GENERIC-LABEL: test_4xdouble_unpack_high_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vunpckhpd {{.*#+}} ymm0 = ymm0[1],ymm1[1],ymm0[3],ymm1[3] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xdouble_unpack_high_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vunpckhpd {{.*#+}} ymm0 = ymm0[1],ymm1[1],ymm0[3],ymm1[3] sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %res = shufflevector <4 x double> %vec1, <4 x double> %vec2, <4 x i32> <i32 1, i32 5, i32 3, i32 7>
-  ret <4 x double> %res
-}
-define <4 x double> @test_4xdouble_masked_unpack_high_mask3(<4 x double> %vec1, <4 x double> %vec2, <4 x double> %vec3, <4 x i64> %mask) {
-; GENERIC-LABEL: test_4xdouble_masked_unpack_high_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %ymm3, %ymm3, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpckhpd {{.*#+}} ymm2 {%k1} = ymm0[1],ymm1[1],ymm0[3],ymm1[3] sched: [1:1.00]
-; GENERIC-NEXT:    vmovapd %ymm2, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xdouble_masked_unpack_high_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %ymm3, %ymm3, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpckhpd {{.*#+}} ymm2 {%k1} = ymm0[1],ymm1[1],ymm0[3],ymm1[3] sched: [1:1.00]
-; SKX-NEXT:    vmovapd %ymm2, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <4 x double> %vec1, <4 x double> %vec2, <4 x i32> <i32 1, i32 5, i32 3, i32 7>
-  %cmp = icmp eq <4 x i64> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x double> %shuf, <4 x double> %vec3
-  ret <4 x double> %res
-}
-
-define <4 x double> @test_4xdouble_zero_masked_unpack_high_mask3(<4 x double> %vec1, <4 x double> %vec2, <4 x i64> %mask) {
-; GENERIC-LABEL: test_4xdouble_zero_masked_unpack_high_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %ymm2, %ymm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpckhpd {{.*#+}} ymm0 {%k1} {z} = ymm0[1],ymm1[1],ymm0[3],ymm1[3] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xdouble_zero_masked_unpack_high_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %ymm2, %ymm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpckhpd {{.*#+}} ymm0 {%k1} {z} = ymm0[1],ymm1[1],ymm0[3],ymm1[3] sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <4 x double> %vec1, <4 x double> %vec2, <4 x i32> <i32 1, i32 5, i32 3, i32 7>
-  %cmp = icmp eq <4 x i64> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x double> %shuf, <4 x double> zeroinitializer
-  ret <4 x double> %res
-}
-define <4 x double> @test_4xdouble_unpack_high_mem_mask0(<4 x double> %vec1, <4 x double>* %vec2p) {
-; GENERIC-LABEL: test_4xdouble_unpack_high_mem_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vunpckhpd {{.*#+}} ymm0 = ymm0[1],mem[1],ymm0[3],mem[3] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xdouble_unpack_high_mem_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vunpckhpd {{.*#+}} ymm0 = ymm0[1],mem[1],ymm0[3],mem[3] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <4 x double>, <4 x double>* %vec2p
-  %res = shufflevector <4 x double> %vec1, <4 x double> %vec2, <4 x i32> <i32 1, i32 5, i32 3, i32 7>
-  ret <4 x double> %res
-}
-define <4 x double> @test_4xdouble_masked_unpack_high_mem_mask0(<4 x double> %vec1, <4 x double>* %vec2p, <4 x double> %vec3, <4 x i64> %mask) {
-; GENERIC-LABEL: test_4xdouble_masked_unpack_high_mem_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %ymm2, %ymm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpckhpd {{.*#+}} ymm1 {%k1} = ymm0[1],mem[1],ymm0[3],mem[3] sched: [8:1.00]
-; GENERIC-NEXT:    vmovapd %ymm1, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xdouble_masked_unpack_high_mem_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %ymm2, %ymm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpckhpd {{.*#+}} ymm1 {%k1} = ymm0[1],mem[1],ymm0[3],mem[3] sched: [8:1.00]
-; SKX-NEXT:    vmovapd %ymm1, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <4 x double>, <4 x double>* %vec2p
-  %shuf = shufflevector <4 x double> %vec1, <4 x double> %vec2, <4 x i32> <i32 1, i32 5, i32 3, i32 7>
-  %cmp = icmp eq <4 x i64> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x double> %shuf, <4 x double> %vec3
-  ret <4 x double> %res
-}
-
-define <4 x double> @test_4xdouble_zero_masked_unpack_high_mem_mask0(<4 x double> %vec1, <4 x double>* %vec2p, <4 x i64> %mask) {
-; GENERIC-LABEL: test_4xdouble_zero_masked_unpack_high_mem_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %ymm1, %ymm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpckhpd {{.*#+}} ymm0 {%k1} {z} = ymm0[1],mem[1],ymm0[3],mem[3] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xdouble_zero_masked_unpack_high_mem_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %ymm1, %ymm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpckhpd {{.*#+}} ymm0 {%k1} {z} = ymm0[1],mem[1],ymm0[3],mem[3] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <4 x double>, <4 x double>* %vec2p
-  %shuf = shufflevector <4 x double> %vec1, <4 x double> %vec2, <4 x i32> <i32 1, i32 5, i32 3, i32 7>
-  %cmp = icmp eq <4 x i64> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x double> %shuf, <4 x double> zeroinitializer
-  ret <4 x double> %res
-}
-
-define <4 x double> @test_4xdouble_masked_unpack_high_mem_mask1(<4 x double> %vec1, <4 x double>* %vec2p, <4 x double> %vec3, <4 x i64> %mask) {
-; GENERIC-LABEL: test_4xdouble_masked_unpack_high_mem_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %ymm2, %ymm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpckhpd {{.*#+}} ymm1 {%k1} = ymm0[1],mem[1],ymm0[3],mem[3] sched: [8:1.00]
-; GENERIC-NEXT:    vmovapd %ymm1, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xdouble_masked_unpack_high_mem_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %ymm2, %ymm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpckhpd {{.*#+}} ymm1 {%k1} = ymm0[1],mem[1],ymm0[3],mem[3] sched: [8:1.00]
-; SKX-NEXT:    vmovapd %ymm1, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <4 x double>, <4 x double>* %vec2p
-  %shuf = shufflevector <4 x double> %vec1, <4 x double> %vec2, <4 x i32> <i32 1, i32 5, i32 3, i32 7>
-  %cmp = icmp eq <4 x i64> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x double> %shuf, <4 x double> %vec3
-  ret <4 x double> %res
-}
-
-define <4 x double> @test_4xdouble_zero_masked_unpack_high_mem_mask1(<4 x double> %vec1, <4 x double>* %vec2p, <4 x i64> %mask) {
-; GENERIC-LABEL: test_4xdouble_zero_masked_unpack_high_mem_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %ymm1, %ymm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpckhpd {{.*#+}} ymm0 {%k1} {z} = ymm0[1],mem[1],ymm0[3],mem[3] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xdouble_zero_masked_unpack_high_mem_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %ymm1, %ymm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpckhpd {{.*#+}} ymm0 {%k1} {z} = ymm0[1],mem[1],ymm0[3],mem[3] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <4 x double>, <4 x double>* %vec2p
-  %shuf = shufflevector <4 x double> %vec1, <4 x double> %vec2, <4 x i32> <i32 1, i32 5, i32 3, i32 7>
-  %cmp = icmp eq <4 x i64> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x double> %shuf, <4 x double> zeroinitializer
-  ret <4 x double> %res
-}
-
-define <4 x double> @test_4xdouble_masked_unpack_high_mem_mask2(<4 x double> %vec1, <4 x double>* %vec2p, <4 x double> %vec3, <4 x i64> %mask) {
-; GENERIC-LABEL: test_4xdouble_masked_unpack_high_mem_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %ymm2, %ymm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpckhpd {{.*#+}} ymm1 {%k1} = ymm0[1],mem[1],ymm0[3],mem[3] sched: [8:1.00]
-; GENERIC-NEXT:    vmovapd %ymm1, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xdouble_masked_unpack_high_mem_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %ymm2, %ymm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpckhpd {{.*#+}} ymm1 {%k1} = ymm0[1],mem[1],ymm0[3],mem[3] sched: [8:1.00]
-; SKX-NEXT:    vmovapd %ymm1, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <4 x double>, <4 x double>* %vec2p
-  %shuf = shufflevector <4 x double> %vec1, <4 x double> %vec2, <4 x i32> <i32 1, i32 5, i32 3, i32 7>
-  %cmp = icmp eq <4 x i64> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x double> %shuf, <4 x double> %vec3
-  ret <4 x double> %res
-}
-
-define <4 x double> @test_4xdouble_zero_masked_unpack_high_mem_mask2(<4 x double> %vec1, <4 x double>* %vec2p, <4 x i64> %mask) {
-; GENERIC-LABEL: test_4xdouble_zero_masked_unpack_high_mem_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %ymm1, %ymm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpckhpd {{.*#+}} ymm0 {%k1} {z} = ymm0[1],mem[1],ymm0[3],mem[3] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xdouble_zero_masked_unpack_high_mem_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %ymm1, %ymm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpckhpd {{.*#+}} ymm0 {%k1} {z} = ymm0[1],mem[1],ymm0[3],mem[3] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <4 x double>, <4 x double>* %vec2p
-  %shuf = shufflevector <4 x double> %vec1, <4 x double> %vec2, <4 x i32> <i32 1, i32 5, i32 3, i32 7>
-  %cmp = icmp eq <4 x i64> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x double> %shuf, <4 x double> zeroinitializer
-  ret <4 x double> %res
-}
-
-define <4 x double> @test_4xdouble_unpack_high_mem_mask3(<4 x double> %vec1, <4 x double>* %vec2p) {
-; GENERIC-LABEL: test_4xdouble_unpack_high_mem_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vunpckhpd {{.*#+}} ymm0 = ymm0[1],mem[1],ymm0[3],mem[3] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xdouble_unpack_high_mem_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vunpckhpd {{.*#+}} ymm0 = ymm0[1],mem[1],ymm0[3],mem[3] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <4 x double>, <4 x double>* %vec2p
-  %res = shufflevector <4 x double> %vec1, <4 x double> %vec2, <4 x i32> <i32 1, i32 5, i32 3, i32 7>
-  ret <4 x double> %res
-}
-define <4 x double> @test_4xdouble_masked_unpack_high_mem_mask3(<4 x double> %vec1, <4 x double>* %vec2p, <4 x double> %vec3, <4 x i64> %mask) {
-; GENERIC-LABEL: test_4xdouble_masked_unpack_high_mem_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %ymm2, %ymm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpckhpd {{.*#+}} ymm1 {%k1} = ymm0[1],mem[1],ymm0[3],mem[3] sched: [8:1.00]
-; GENERIC-NEXT:    vmovapd %ymm1, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xdouble_masked_unpack_high_mem_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %ymm2, %ymm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpckhpd {{.*#+}} ymm1 {%k1} = ymm0[1],mem[1],ymm0[3],mem[3] sched: [8:1.00]
-; SKX-NEXT:    vmovapd %ymm1, %ymm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <4 x double>, <4 x double>* %vec2p
-  %shuf = shufflevector <4 x double> %vec1, <4 x double> %vec2, <4 x i32> <i32 1, i32 5, i32 3, i32 7>
-  %cmp = icmp eq <4 x i64> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x double> %shuf, <4 x double> %vec3
-  ret <4 x double> %res
-}
-
-define <4 x double> @test_4xdouble_zero_masked_unpack_high_mem_mask3(<4 x double> %vec1, <4 x double>* %vec2p, <4 x i64> %mask) {
-; GENERIC-LABEL: test_4xdouble_zero_masked_unpack_high_mem_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %ymm1, %ymm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpckhpd {{.*#+}} ymm0 {%k1} {z} = ymm0[1],mem[1],ymm0[3],mem[3] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_4xdouble_zero_masked_unpack_high_mem_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %ymm1, %ymm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpckhpd {{.*#+}} ymm0 {%k1} {z} = ymm0[1],mem[1],ymm0[3],mem[3] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <4 x double>, <4 x double>* %vec2p
-  %shuf = shufflevector <4 x double> %vec1, <4 x double> %vec2, <4 x i32> <i32 1, i32 5, i32 3, i32 7>
-  %cmp = icmp eq <4 x i64> %mask, zeroinitializer
-  %res = select <4 x i1> %cmp, <4 x double> %shuf, <4 x double> zeroinitializer
-  ret <4 x double> %res
-}
-
-define <8 x double> @test_8xdouble_unpack_high_mask0(<8 x double> %vec1, <8 x double> %vec2) {
-; GENERIC-LABEL: test_8xdouble_unpack_high_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vunpckhpd {{.*#+}} zmm0 = zmm0[1],zmm1[1],zmm0[3],zmm1[3],zmm0[5],zmm1[5],zmm0[7],zmm1[7] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xdouble_unpack_high_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vunpckhpd {{.*#+}} zmm0 = zmm0[1],zmm1[1],zmm0[3],zmm1[3],zmm0[5],zmm1[5],zmm0[7],zmm1[7] sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %res = shufflevector <8 x double> %vec1, <8 x double> %vec2, <8 x i32> <i32 1, i32 9, i32 3, i32 11, i32 5, i32 13, i32 7, i32 15>
-  ret <8 x double> %res
-}
-define <8 x double> @test_8xdouble_masked_unpack_high_mask0(<8 x double> %vec1, <8 x double> %vec2, <8 x double> %vec3, <8 x i64> %mask) {
-; GENERIC-LABEL: test_8xdouble_masked_unpack_high_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %zmm3, %zmm3, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpckhpd {{.*#+}} zmm2 {%k1} = zmm0[1],zmm1[1],zmm0[3],zmm1[3],zmm0[5],zmm1[5],zmm0[7],zmm1[7] sched: [1:1.00]
-; GENERIC-NEXT:    vmovapd %zmm2, %zmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xdouble_masked_unpack_high_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %zmm3, %zmm3, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpckhpd {{.*#+}} zmm2 {%k1} = zmm0[1],zmm1[1],zmm0[3],zmm1[3],zmm0[5],zmm1[5],zmm0[7],zmm1[7] sched: [1:1.00]
-; SKX-NEXT:    vmovapd %zmm2, %zmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x double> %vec1, <8 x double> %vec2, <8 x i32> <i32 1, i32 9, i32 3, i32 11, i32 5, i32 13, i32 7, i32 15>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x double> %shuf, <8 x double> %vec3
-  ret <8 x double> %res
-}
-
-define <8 x double> @test_8xdouble_zero_masked_unpack_high_mask0(<8 x double> %vec1, <8 x double> %vec2, <8 x i64> %mask) {
-; GENERIC-LABEL: test_8xdouble_zero_masked_unpack_high_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %zmm2, %zmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpckhpd {{.*#+}} zmm0 {%k1} {z} = zmm0[1],zmm1[1],zmm0[3],zmm1[3],zmm0[5],zmm1[5],zmm0[7],zmm1[7] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xdouble_zero_masked_unpack_high_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %zmm2, %zmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpckhpd {{.*#+}} zmm0 {%k1} {z} = zmm0[1],zmm1[1],zmm0[3],zmm1[3],zmm0[5],zmm1[5],zmm0[7],zmm1[7] sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x double> %vec1, <8 x double> %vec2, <8 x i32> <i32 1, i32 9, i32 3, i32 11, i32 5, i32 13, i32 7, i32 15>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x double> %shuf, <8 x double> zeroinitializer
-  ret <8 x double> %res
-}
-define <8 x double> @test_8xdouble_masked_unpack_high_mask1(<8 x double> %vec1, <8 x double> %vec2, <8 x double> %vec3, <8 x i64> %mask) {
-; GENERIC-LABEL: test_8xdouble_masked_unpack_high_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %zmm3, %zmm3, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpckhpd {{.*#+}} zmm2 {%k1} = zmm0[1],zmm1[1],zmm0[3],zmm1[3],zmm0[5],zmm1[5],zmm0[7],zmm1[7] sched: [1:1.00]
-; GENERIC-NEXT:    vmovapd %zmm2, %zmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xdouble_masked_unpack_high_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %zmm3, %zmm3, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpckhpd {{.*#+}} zmm2 {%k1} = zmm0[1],zmm1[1],zmm0[3],zmm1[3],zmm0[5],zmm1[5],zmm0[7],zmm1[7] sched: [1:1.00]
-; SKX-NEXT:    vmovapd %zmm2, %zmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x double> %vec1, <8 x double> %vec2, <8 x i32> <i32 1, i32 9, i32 3, i32 11, i32 5, i32 13, i32 7, i32 15>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x double> %shuf, <8 x double> %vec3
-  ret <8 x double> %res
-}
-
-define <8 x double> @test_8xdouble_zero_masked_unpack_high_mask1(<8 x double> %vec1, <8 x double> %vec2, <8 x i64> %mask) {
-; GENERIC-LABEL: test_8xdouble_zero_masked_unpack_high_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %zmm2, %zmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpckhpd {{.*#+}} zmm0 {%k1} {z} = zmm0[1],zmm1[1],zmm0[3],zmm1[3],zmm0[5],zmm1[5],zmm0[7],zmm1[7] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xdouble_zero_masked_unpack_high_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %zmm2, %zmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpckhpd {{.*#+}} zmm0 {%k1} {z} = zmm0[1],zmm1[1],zmm0[3],zmm1[3],zmm0[5],zmm1[5],zmm0[7],zmm1[7] sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x double> %vec1, <8 x double> %vec2, <8 x i32> <i32 1, i32 9, i32 3, i32 11, i32 5, i32 13, i32 7, i32 15>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x double> %shuf, <8 x double> zeroinitializer
-  ret <8 x double> %res
-}
-define <8 x double> @test_8xdouble_masked_unpack_high_mask2(<8 x double> %vec1, <8 x double> %vec2, <8 x double> %vec3, <8 x i64> %mask) {
-; GENERIC-LABEL: test_8xdouble_masked_unpack_high_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %zmm3, %zmm3, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpckhpd {{.*#+}} zmm2 {%k1} = zmm0[1],zmm1[1],zmm0[3],zmm1[3],zmm0[5],zmm1[5],zmm0[7],zmm1[7] sched: [1:1.00]
-; GENERIC-NEXT:    vmovapd %zmm2, %zmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xdouble_masked_unpack_high_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %zmm3, %zmm3, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpckhpd {{.*#+}} zmm2 {%k1} = zmm0[1],zmm1[1],zmm0[3],zmm1[3],zmm0[5],zmm1[5],zmm0[7],zmm1[7] sched: [1:1.00]
-; SKX-NEXT:    vmovapd %zmm2, %zmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x double> %vec1, <8 x double> %vec2, <8 x i32> <i32 1, i32 9, i32 3, i32 11, i32 5, i32 13, i32 7, i32 15>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x double> %shuf, <8 x double> %vec3
-  ret <8 x double> %res
-}
-
-define <8 x double> @test_8xdouble_zero_masked_unpack_high_mask2(<8 x double> %vec1, <8 x double> %vec2, <8 x i64> %mask) {
-; GENERIC-LABEL: test_8xdouble_zero_masked_unpack_high_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %zmm2, %zmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpckhpd {{.*#+}} zmm0 {%k1} {z} = zmm0[1],zmm1[1],zmm0[3],zmm1[3],zmm0[5],zmm1[5],zmm0[7],zmm1[7] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xdouble_zero_masked_unpack_high_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %zmm2, %zmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpckhpd {{.*#+}} zmm0 {%k1} {z} = zmm0[1],zmm1[1],zmm0[3],zmm1[3],zmm0[5],zmm1[5],zmm0[7],zmm1[7] sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x double> %vec1, <8 x double> %vec2, <8 x i32> <i32 1, i32 9, i32 3, i32 11, i32 5, i32 13, i32 7, i32 15>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x double> %shuf, <8 x double> zeroinitializer
-  ret <8 x double> %res
-}
-define <8 x double> @test_8xdouble_unpack_high_mask3(<8 x double> %vec1, <8 x double> %vec2) {
-; GENERIC-LABEL: test_8xdouble_unpack_high_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vunpckhpd {{.*#+}} zmm0 = zmm0[1],zmm1[1],zmm0[3],zmm1[3],zmm0[5],zmm1[5],zmm0[7],zmm1[7] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xdouble_unpack_high_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vunpckhpd {{.*#+}} zmm0 = zmm0[1],zmm1[1],zmm0[3],zmm1[3],zmm0[5],zmm1[5],zmm0[7],zmm1[7] sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %res = shufflevector <8 x double> %vec1, <8 x double> %vec2, <8 x i32> <i32 1, i32 9, i32 3, i32 11, i32 5, i32 13, i32 7, i32 15>
-  ret <8 x double> %res
-}
-define <8 x double> @test_8xdouble_masked_unpack_high_mask3(<8 x double> %vec1, <8 x double> %vec2, <8 x double> %vec3, <8 x i64> %mask) {
-; GENERIC-LABEL: test_8xdouble_masked_unpack_high_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %zmm3, %zmm3, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpckhpd {{.*#+}} zmm2 {%k1} = zmm0[1],zmm1[1],zmm0[3],zmm1[3],zmm0[5],zmm1[5],zmm0[7],zmm1[7] sched: [1:1.00]
-; GENERIC-NEXT:    vmovapd %zmm2, %zmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xdouble_masked_unpack_high_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %zmm3, %zmm3, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpckhpd {{.*#+}} zmm2 {%k1} = zmm0[1],zmm1[1],zmm0[3],zmm1[3],zmm0[5],zmm1[5],zmm0[7],zmm1[7] sched: [1:1.00]
-; SKX-NEXT:    vmovapd %zmm2, %zmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x double> %vec1, <8 x double> %vec2, <8 x i32> <i32 1, i32 9, i32 3, i32 11, i32 5, i32 13, i32 7, i32 15>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x double> %shuf, <8 x double> %vec3
-  ret <8 x double> %res
-}
-
-define <8 x double> @test_8xdouble_zero_masked_unpack_high_mask3(<8 x double> %vec1, <8 x double> %vec2, <8 x i64> %mask) {
-; GENERIC-LABEL: test_8xdouble_zero_masked_unpack_high_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %zmm2, %zmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpckhpd {{.*#+}} zmm0 {%k1} {z} = zmm0[1],zmm1[1],zmm0[3],zmm1[3],zmm0[5],zmm1[5],zmm0[7],zmm1[7] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xdouble_zero_masked_unpack_high_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %zmm2, %zmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpckhpd {{.*#+}} zmm0 {%k1} {z} = zmm0[1],zmm1[1],zmm0[3],zmm1[3],zmm0[5],zmm1[5],zmm0[7],zmm1[7] sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %shuf = shufflevector <8 x double> %vec1, <8 x double> %vec2, <8 x i32> <i32 1, i32 9, i32 3, i32 11, i32 5, i32 13, i32 7, i32 15>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x double> %shuf, <8 x double> zeroinitializer
-  ret <8 x double> %res
-}
-define <8 x double> @test_8xdouble_unpack_high_mem_mask0(<8 x double> %vec1, <8 x double>* %vec2p) {
-; GENERIC-LABEL: test_8xdouble_unpack_high_mem_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vunpckhpd {{.*#+}} zmm0 = zmm0[1],mem[1],zmm0[3],mem[3],zmm0[5],mem[5],zmm0[7],mem[7] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xdouble_unpack_high_mem_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vunpckhpd {{.*#+}} zmm0 = zmm0[1],mem[1],zmm0[3],mem[3],zmm0[5],mem[5],zmm0[7],mem[7] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <8 x double>, <8 x double>* %vec2p
-  %res = shufflevector <8 x double> %vec1, <8 x double> %vec2, <8 x i32> <i32 1, i32 9, i32 3, i32 11, i32 5, i32 13, i32 7, i32 15>
-  ret <8 x double> %res
-}
-define <8 x double> @test_8xdouble_masked_unpack_high_mem_mask0(<8 x double> %vec1, <8 x double>* %vec2p, <8 x double> %vec3, <8 x i64> %mask) {
-; GENERIC-LABEL: test_8xdouble_masked_unpack_high_mem_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %zmm2, %zmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpckhpd {{.*#+}} zmm1 {%k1} = zmm0[1],mem[1],zmm0[3],mem[3],zmm0[5],mem[5],zmm0[7],mem[7] sched: [8:1.00]
-; GENERIC-NEXT:    vmovapd %zmm1, %zmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xdouble_masked_unpack_high_mem_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %zmm2, %zmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpckhpd {{.*#+}} zmm1 {%k1} = zmm0[1],mem[1],zmm0[3],mem[3],zmm0[5],mem[5],zmm0[7],mem[7] sched: [8:1.00]
-; SKX-NEXT:    vmovapd %zmm1, %zmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <8 x double>, <8 x double>* %vec2p
-  %shuf = shufflevector <8 x double> %vec1, <8 x double> %vec2, <8 x i32> <i32 1, i32 9, i32 3, i32 11, i32 5, i32 13, i32 7, i32 15>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x double> %shuf, <8 x double> %vec3
-  ret <8 x double> %res
-}
-
-define <8 x double> @test_8xdouble_zero_masked_unpack_high_mem_mask0(<8 x double> %vec1, <8 x double>* %vec2p, <8 x i64> %mask) {
-; GENERIC-LABEL: test_8xdouble_zero_masked_unpack_high_mem_mask0:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpckhpd {{.*#+}} zmm0 {%k1} {z} = zmm0[1],mem[1],zmm0[3],mem[3],zmm0[5],mem[5],zmm0[7],mem[7] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xdouble_zero_masked_unpack_high_mem_mask0:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpckhpd {{.*#+}} zmm0 {%k1} {z} = zmm0[1],mem[1],zmm0[3],mem[3],zmm0[5],mem[5],zmm0[7],mem[7] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <8 x double>, <8 x double>* %vec2p
-  %shuf = shufflevector <8 x double> %vec1, <8 x double> %vec2, <8 x i32> <i32 1, i32 9, i32 3, i32 11, i32 5, i32 13, i32 7, i32 15>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x double> %shuf, <8 x double> zeroinitializer
-  ret <8 x double> %res
-}
-
-define <8 x double> @test_8xdouble_masked_unpack_high_mem_mask1(<8 x double> %vec1, <8 x double>* %vec2p, <8 x double> %vec3, <8 x i64> %mask) {
-; GENERIC-LABEL: test_8xdouble_masked_unpack_high_mem_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %zmm2, %zmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpckhpd {{.*#+}} zmm1 {%k1} = zmm0[1],mem[1],zmm0[3],mem[3],zmm0[5],mem[5],zmm0[7],mem[7] sched: [8:1.00]
-; GENERIC-NEXT:    vmovapd %zmm1, %zmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xdouble_masked_unpack_high_mem_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %zmm2, %zmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpckhpd {{.*#+}} zmm1 {%k1} = zmm0[1],mem[1],zmm0[3],mem[3],zmm0[5],mem[5],zmm0[7],mem[7] sched: [8:1.00]
-; SKX-NEXT:    vmovapd %zmm1, %zmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <8 x double>, <8 x double>* %vec2p
-  %shuf = shufflevector <8 x double> %vec1, <8 x double> %vec2, <8 x i32> <i32 1, i32 9, i32 3, i32 11, i32 5, i32 13, i32 7, i32 15>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x double> %shuf, <8 x double> %vec3
-  ret <8 x double> %res
-}
-
-define <8 x double> @test_8xdouble_zero_masked_unpack_high_mem_mask1(<8 x double> %vec1, <8 x double>* %vec2p, <8 x i64> %mask) {
-; GENERIC-LABEL: test_8xdouble_zero_masked_unpack_high_mem_mask1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpckhpd {{.*#+}} zmm0 {%k1} {z} = zmm0[1],mem[1],zmm0[3],mem[3],zmm0[5],mem[5],zmm0[7],mem[7] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xdouble_zero_masked_unpack_high_mem_mask1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpckhpd {{.*#+}} zmm0 {%k1} {z} = zmm0[1],mem[1],zmm0[3],mem[3],zmm0[5],mem[5],zmm0[7],mem[7] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <8 x double>, <8 x double>* %vec2p
-  %shuf = shufflevector <8 x double> %vec1, <8 x double> %vec2, <8 x i32> <i32 1, i32 9, i32 3, i32 11, i32 5, i32 13, i32 7, i32 15>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x double> %shuf, <8 x double> zeroinitializer
-  ret <8 x double> %res
-}
-
-define <8 x double> @test_8xdouble_masked_unpack_high_mem_mask2(<8 x double> %vec1, <8 x double>* %vec2p, <8 x double> %vec3, <8 x i64> %mask) {
-; GENERIC-LABEL: test_8xdouble_masked_unpack_high_mem_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %zmm2, %zmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpckhpd {{.*#+}} zmm1 {%k1} = zmm0[1],mem[1],zmm0[3],mem[3],zmm0[5],mem[5],zmm0[7],mem[7] sched: [8:1.00]
-; GENERIC-NEXT:    vmovapd %zmm1, %zmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xdouble_masked_unpack_high_mem_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %zmm2, %zmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpckhpd {{.*#+}} zmm1 {%k1} = zmm0[1],mem[1],zmm0[3],mem[3],zmm0[5],mem[5],zmm0[7],mem[7] sched: [8:1.00]
-; SKX-NEXT:    vmovapd %zmm1, %zmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <8 x double>, <8 x double>* %vec2p
-  %shuf = shufflevector <8 x double> %vec1, <8 x double> %vec2, <8 x i32> <i32 1, i32 9, i32 3, i32 11, i32 5, i32 13, i32 7, i32 15>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x double> %shuf, <8 x double> %vec3
-  ret <8 x double> %res
-}
-
-define <8 x double> @test_8xdouble_zero_masked_unpack_high_mem_mask2(<8 x double> %vec1, <8 x double>* %vec2p, <8 x i64> %mask) {
-; GENERIC-LABEL: test_8xdouble_zero_masked_unpack_high_mem_mask2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpckhpd {{.*#+}} zmm0 {%k1} {z} = zmm0[1],mem[1],zmm0[3],mem[3],zmm0[5],mem[5],zmm0[7],mem[7] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xdouble_zero_masked_unpack_high_mem_mask2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpckhpd {{.*#+}} zmm0 {%k1} {z} = zmm0[1],mem[1],zmm0[3],mem[3],zmm0[5],mem[5],zmm0[7],mem[7] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <8 x double>, <8 x double>* %vec2p
-  %shuf = shufflevector <8 x double> %vec1, <8 x double> %vec2, <8 x i32> <i32 1, i32 9, i32 3, i32 11, i32 5, i32 13, i32 7, i32 15>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x double> %shuf, <8 x double> zeroinitializer
-  ret <8 x double> %res
-}
-
-define <8 x double> @test_8xdouble_unpack_high_mem_mask3(<8 x double> %vec1, <8 x double>* %vec2p) {
-; GENERIC-LABEL: test_8xdouble_unpack_high_mem_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vunpckhpd {{.*#+}} zmm0 = zmm0[1],mem[1],zmm0[3],mem[3],zmm0[5],mem[5],zmm0[7],mem[7] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xdouble_unpack_high_mem_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vunpckhpd {{.*#+}} zmm0 = zmm0[1],mem[1],zmm0[3],mem[3],zmm0[5],mem[5],zmm0[7],mem[7] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <8 x double>, <8 x double>* %vec2p
-  %res = shufflevector <8 x double> %vec1, <8 x double> %vec2, <8 x i32> <i32 1, i32 9, i32 3, i32 11, i32 5, i32 13, i32 7, i32 15>
-  ret <8 x double> %res
-}
-define <8 x double> @test_8xdouble_masked_unpack_high_mem_mask3(<8 x double> %vec1, <8 x double>* %vec2p, <8 x double> %vec3, <8 x i64> %mask) {
-; GENERIC-LABEL: test_8xdouble_masked_unpack_high_mem_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %zmm2, %zmm2, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpckhpd {{.*#+}} zmm1 {%k1} = zmm0[1],mem[1],zmm0[3],mem[3],zmm0[5],mem[5],zmm0[7],mem[7] sched: [8:1.00]
-; GENERIC-NEXT:    vmovapd %zmm1, %zmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xdouble_masked_unpack_high_mem_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %zmm2, %zmm2, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpckhpd {{.*#+}} zmm1 {%k1} = zmm0[1],mem[1],zmm0[3],mem[3],zmm0[5],mem[5],zmm0[7],mem[7] sched: [8:1.00]
-; SKX-NEXT:    vmovapd %zmm1, %zmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <8 x double>, <8 x double>* %vec2p
-  %shuf = shufflevector <8 x double> %vec1, <8 x double> %vec2, <8 x i32> <i32 1, i32 9, i32 3, i32 11, i32 5, i32 13, i32 7, i32 15>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x double> %shuf, <8 x double> %vec3
-  ret <8 x double> %res
-}
-
-define <8 x double> @test_8xdouble_zero_masked_unpack_high_mem_mask3(<8 x double> %vec1, <8 x double>* %vec2p, <8 x i64> %mask) {
-; GENERIC-LABEL: test_8xdouble_zero_masked_unpack_high_mem_mask3:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vptestnmq %zmm1, %zmm1, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    vunpckhpd {{.*#+}} zmm0 {%k1} {z} = zmm0[1],mem[1],zmm0[3],mem[3],zmm0[5],mem[5],zmm0[7],mem[7] sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: test_8xdouble_zero_masked_unpack_high_mem_mask3:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptestnmq %zmm1, %zmm1, %k1 # sched: [3:1.00]
-; SKX-NEXT:    vunpckhpd {{.*#+}} zmm0 {%k1} {z} = zmm0[1],mem[1],zmm0[3],mem[3],zmm0[5],mem[5],zmm0[7],mem[7] sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  %vec2 = load <8 x double>, <8 x double>* %vec2p
-  %shuf = shufflevector <8 x double> %vec1, <8 x double> %vec2, <8 x i32> <i32 1, i32 9, i32 3, i32 11, i32 5, i32 13, i32 7, i32 15>
-  %cmp = icmp eq <8 x i64> %mask, zeroinitializer
-  %res = select <8 x i1> %cmp, <8 x double> %shuf, <8 x double> zeroinitializer
-  ret <8 x double> %res
-}
-
diff --git a/test/CodeGen/X86/avx512-shuffles/broadcast-scalar-int.ll b/test/CodeGen/X86/avx512-shuffles/broadcast-scalar-int.ll
index 74005debfedd865d991026494fa7ccb46a73f0f7..e1ab24a615684031b64ec7ed9443398cfdc9af18 100644
--- a/test/CodeGen/X86/avx512-shuffles/broadcast-scalar-int.ll
+++ b/test/CodeGen/X86/avx512-shuffles/broadcast-scalar-int.ll
@@ -2324,7 +2324,7 @@ define <16 x i32> @test_masked_z_i32_to_16_mem_mask3(i32* %p, <16 x i32> %mask)
 define <2 x i64> @test_i64_to_2_mem(i64* %p) {
 ; CHECK-LABEL: test_i64_to_2_mem:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vpbroadcastq (%rdi), %xmm0
+; CHECK-NEXT:    vmovddup {{.*#+}} xmm0 = mem[0,0]
 ; CHECK-NEXT:    retq
   %s = load i64, i64* %p
   %vec = insertelement <2 x i64> undef, i64 %s, i32 0
diff --git a/test/CodeGen/X86/avx512-shuffles/broadcast-vector-int.ll b/test/CodeGen/X86/avx512-shuffles/broadcast-vector-int.ll
index 218aa3ffe07ba4ff1daef38d3e57a5107de5e967..9c9df48c03c09e96ac2744ed9344ddd2f69d64bd 100644
--- a/test/CodeGen/X86/avx512-shuffles/broadcast-vector-int.ll
+++ b/test/CodeGen/X86/avx512-shuffles/broadcast-vector-int.ll
@@ -6,7 +6,7 @@
 define <4 x i32> @test_2xi32_to_4xi32(<4 x i32> %vec) {
 ; CHECK-LABEL: test_2xi32_to_4xi32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vpbroadcastq %xmm0, %xmm0
+; CHECK-NEXT:    vmovddup {{.*#+}} xmm0 = xmm0[0,0]
 ; CHECK-NEXT:    retq
   %res = shufflevector <4 x i32> %vec, <4 x i32> undef, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
   ret <4 x i32> %res
@@ -318,7 +318,7 @@ define <16 x i32> @test_masked_z_2xi32_to_16xi32_mask3(<16 x i32> %vec, <16 x i3
 define <4 x i32> @test_2xi32_to_4xi32_mem(<2 x i32>* %vp) {
 ; CHECK-LABEL: test_2xi32_to_4xi32_mem:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vpbroadcastq (%rdi), %xmm0
+; CHECK-NEXT:    vmovddup {{.*#+}} xmm0 = mem[0,0]
 ; CHECK-NEXT:    retq
   %vec = load <2 x i32>, <2 x i32>* %vp
   %res = shufflevector <2 x i32> %vec, <2 x i32> undef, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
@@ -536,9 +536,7 @@ define <8 x i32> @test_masked_z_2xi32_to_8xi32_mem_mask3(<2 x i32>* %vp, <8 x i3
 define <16 x i32> @test_2xi32_to_16xi32_mem(<2 x i32>* %vp) {
 ; CHECK-LABEL: test_2xi32_to_16xi32_mem:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vpmovzxdq {{.*#+}} xmm0 = mem[0],zero,mem[1],zero
-; CHECK-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [0,2,0,2,0,2,0,2,0,2,0,2,0,2,0,2]
-; CHECK-NEXT:    vpermd %zmm0, %zmm1, %zmm0
+; CHECK-NEXT:    vbroadcastsd (%rdi), %zmm0
 ; CHECK-NEXT:    retq
   %vec = load <2 x i32>, <2 x i32>* %vp
   %res = shufflevector <2 x i32> %vec, <2 x i32> undef, <16 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1>
@@ -547,10 +545,8 @@ define <16 x i32> @test_2xi32_to_16xi32_mem(<2 x i32>* %vp) {
 define <16 x i32> @test_masked_2xi32_to_16xi32_mem_mask0(<2 x i32>* %vp, <16 x i32> %default, <16 x i32> %mask) {
 ; CHECK-LABEL: test_masked_2xi32_to_16xi32_mem_mask0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vpmovzxdq {{.*#+}} xmm2 = mem[0],zero,mem[1],zero
-; CHECK-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [0,2,0,2,0,2,0,2,0,2,0,2,0,2,0,2]
 ; CHECK-NEXT:    vptestnmd %zmm1, %zmm1, %k1
-; CHECK-NEXT:    vpermd %zmm2, %zmm3, %zmm0 {%k1}
+; CHECK-NEXT:    vbroadcasti32x2 {{.*#+}} zmm0 {%k1} = mem[0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1]
 ; CHECK-NEXT:    retq
   %vec = load <2 x i32>, <2 x i32>* %vp
   %shuf = shufflevector <2 x i32> %vec, <2 x i32> undef, <16 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1>
@@ -562,10 +558,8 @@ define <16 x i32> @test_masked_2xi32_to_16xi32_mem_mask0(<2 x i32>* %vp, <16 x i
 define <16 x i32> @test_masked_z_2xi32_to_16xi32_mem_mask0(<2 x i32>* %vp, <16 x i32> %mask) {
 ; CHECK-LABEL: test_masked_z_2xi32_to_16xi32_mem_mask0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vpmovzxdq {{.*#+}} xmm1 = mem[0],zero,mem[1],zero
-; CHECK-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [0,2,0,2,0,2,0,2,0,2,0,2,0,2,0,2]
 ; CHECK-NEXT:    vptestnmd %zmm0, %zmm0, %k1
-; CHECK-NEXT:    vpermd %zmm1, %zmm2, %zmm0 {%k1} {z}
+; CHECK-NEXT:    vbroadcasti32x2 {{.*#+}} zmm0 {%k1} {z} = mem[0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1]
 ; CHECK-NEXT:    retq
   %vec = load <2 x i32>, <2 x i32>* %vp
   %shuf = shufflevector <2 x i32> %vec, <2 x i32> undef, <16 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1>
@@ -576,10 +570,8 @@ define <16 x i32> @test_masked_z_2xi32_to_16xi32_mem_mask0(<2 x i32>* %vp, <16 x
 define <16 x i32> @test_masked_2xi32_to_16xi32_mem_mask1(<2 x i32>* %vp, <16 x i32> %default, <16 x i32> %mask) {
 ; CHECK-LABEL: test_masked_2xi32_to_16xi32_mem_mask1:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vpmovzxdq {{.*#+}} xmm2 = mem[0],zero,mem[1],zero
-; CHECK-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [0,2,0,2,0,2,0,2,0,2,0,2,0,2,0,2]
 ; CHECK-NEXT:    vptestnmd %zmm1, %zmm1, %k1
-; CHECK-NEXT:    vpermd %zmm2, %zmm3, %zmm0 {%k1}
+; CHECK-NEXT:    vbroadcasti32x2 {{.*#+}} zmm0 {%k1} = mem[0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1]
 ; CHECK-NEXT:    retq
   %vec = load <2 x i32>, <2 x i32>* %vp
   %shuf = shufflevector <2 x i32> %vec, <2 x i32> undef, <16 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1>
@@ -591,10 +583,8 @@ define <16 x i32> @test_masked_2xi32_to_16xi32_mem_mask1(<2 x i32>* %vp, <16 x i
 define <16 x i32> @test_masked_z_2xi32_to_16xi32_mem_mask1(<2 x i32>* %vp, <16 x i32> %mask) {
 ; CHECK-LABEL: test_masked_z_2xi32_to_16xi32_mem_mask1:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vpmovzxdq {{.*#+}} xmm1 = mem[0],zero,mem[1],zero
-; CHECK-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [0,2,0,2,0,2,0,2,0,2,0,2,0,2,0,2]
 ; CHECK-NEXT:    vptestnmd %zmm0, %zmm0, %k1
-; CHECK-NEXT:    vpermd %zmm1, %zmm2, %zmm0 {%k1} {z}
+; CHECK-NEXT:    vbroadcasti32x2 {{.*#+}} zmm0 {%k1} {z} = mem[0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1]
 ; CHECK-NEXT:    retq
   %vec = load <2 x i32>, <2 x i32>* %vp
   %shuf = shufflevector <2 x i32> %vec, <2 x i32> undef, <16 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1>
@@ -605,10 +595,8 @@ define <16 x i32> @test_masked_z_2xi32_to_16xi32_mem_mask1(<2 x i32>* %vp, <16 x
 define <16 x i32> @test_masked_2xi32_to_16xi32_mem_mask2(<2 x i32>* %vp, <16 x i32> %default, <16 x i32> %mask) {
 ; CHECK-LABEL: test_masked_2xi32_to_16xi32_mem_mask2:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vpmovzxdq {{.*#+}} xmm2 = mem[0],zero,mem[1],zero
-; CHECK-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [0,2,0,2,0,2,0,2,0,2,0,2,0,2,0,2]
 ; CHECK-NEXT:    vptestnmd %zmm1, %zmm1, %k1
-; CHECK-NEXT:    vpermd %zmm2, %zmm3, %zmm0 {%k1}
+; CHECK-NEXT:    vbroadcasti32x2 {{.*#+}} zmm0 {%k1} = mem[0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1]
 ; CHECK-NEXT:    retq
   %vec = load <2 x i32>, <2 x i32>* %vp
   %shuf = shufflevector <2 x i32> %vec, <2 x i32> undef, <16 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1>
@@ -620,10 +608,8 @@ define <16 x i32> @test_masked_2xi32_to_16xi32_mem_mask2(<2 x i32>* %vp, <16 x i
 define <16 x i32> @test_masked_z_2xi32_to_16xi32_mem_mask2(<2 x i32>* %vp, <16 x i32> %mask) {
 ; CHECK-LABEL: test_masked_z_2xi32_to_16xi32_mem_mask2:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vpmovzxdq {{.*#+}} xmm1 = mem[0],zero,mem[1],zero
-; CHECK-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [0,2,0,2,0,2,0,2,0,2,0,2,0,2,0,2]
 ; CHECK-NEXT:    vptestnmd %zmm0, %zmm0, %k1
-; CHECK-NEXT:    vpermd %zmm1, %zmm2, %zmm0 {%k1} {z}
+; CHECK-NEXT:    vbroadcasti32x2 {{.*#+}} zmm0 {%k1} {z} = mem[0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1]
 ; CHECK-NEXT:    retq
   %vec = load <2 x i32>, <2 x i32>* %vp
   %shuf = shufflevector <2 x i32> %vec, <2 x i32> undef, <16 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1>
@@ -634,10 +620,8 @@ define <16 x i32> @test_masked_z_2xi32_to_16xi32_mem_mask2(<2 x i32>* %vp, <16 x
 define <16 x i32> @test_masked_2xi32_to_16xi32_mem_mask3(<2 x i32>* %vp, <16 x i32> %default, <16 x i32> %mask) {
 ; CHECK-LABEL: test_masked_2xi32_to_16xi32_mem_mask3:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vpmovzxdq {{.*#+}} xmm2 = mem[0],zero,mem[1],zero
-; CHECK-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [0,2,0,2,0,2,0,2,0,2,0,2,0,2,0,2]
 ; CHECK-NEXT:    vptestnmd %zmm1, %zmm1, %k1
-; CHECK-NEXT:    vpermd %zmm2, %zmm3, %zmm0 {%k1}
+; CHECK-NEXT:    vbroadcasti32x2 {{.*#+}} zmm0 {%k1} = mem[0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1]
 ; CHECK-NEXT:    retq
   %vec = load <2 x i32>, <2 x i32>* %vp
   %shuf = shufflevector <2 x i32> %vec, <2 x i32> undef, <16 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1>
@@ -649,10 +633,8 @@ define <16 x i32> @test_masked_2xi32_to_16xi32_mem_mask3(<2 x i32>* %vp, <16 x i
 define <16 x i32> @test_masked_z_2xi32_to_16xi32_mem_mask3(<2 x i32>* %vp, <16 x i32> %mask) {
 ; CHECK-LABEL: test_masked_z_2xi32_to_16xi32_mem_mask3:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vpmovzxdq {{.*#+}} xmm1 = mem[0],zero,mem[1],zero
-; CHECK-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [0,2,0,2,0,2,0,2,0,2,0,2,0,2,0,2]
 ; CHECK-NEXT:    vptestnmd %zmm0, %zmm0, %k1
-; CHECK-NEXT:    vpermd %zmm1, %zmm2, %zmm0 {%k1} {z}
+; CHECK-NEXT:    vbroadcasti32x2 {{.*#+}} zmm0 {%k1} {z} = mem[0,1,0,1,0,1,0,1,0,1,0,1,0,1,0,1]
 ; CHECK-NEXT:    retq
   %vec = load <2 x i32>, <2 x i32>* %vp
   %shuf = shufflevector <2 x i32> %vec, <2 x i32> undef, <16 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1>
diff --git a/test/CodeGen/X86/avx512-shuffles/partial_permute.ll b/test/CodeGen/X86/avx512-shuffles/partial_permute.ll
index 50dec563d575e205970539a1fbc48bf8cca48936..8ac883d1e46ab9c2462c17d1fbaccbb5a03a997b 100644
--- a/test/CodeGen/X86/avx512-shuffles/partial_permute.ll
+++ b/test/CodeGen/X86/avx512-shuffles/partial_permute.ll
@@ -922,9 +922,9 @@ define <8 x i16> @test_masked_z_32xi16_to_8xi16_perm_mem_mask3(<32 x i16>* %vp,
 define <4 x i32> @test_8xi32_to_4xi32_perm_mask0(<8 x i32> %vec) {
 ; CHECK-LABEL: test_8xi32_to_4xi32_perm_mask0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; CHECK-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,0,3,2]
-; CHECK-NEXT:    vblendps {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
+; CHECK-NEXT:    vmovaps {{.*#+}} ymm1 = <4,0,3,2,u,u,u,u>
+; CHECK-NEXT:    vpermps %ymm0, %ymm1, %ymm0
+; CHECK-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retq
   %res = shufflevector <8 x i32> %vec, <8 x i32> undef, <4 x i32> <i32 4, i32 0, i32 3, i32 2>
@@ -933,9 +933,8 @@ define <4 x i32> @test_8xi32_to_4xi32_perm_mask0(<8 x i32> %vec) {
 define <4 x i32> @test_masked_8xi32_to_4xi32_perm_mask0(<8 x i32> %vec, <4 x i32> %vec2, <4 x i32> %mask) {
 ; CHECK-LABEL: test_masked_8xi32_to_4xi32_perm_mask0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vextracti128 $1, %ymm0, %xmm3
-; CHECK-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,3,2]
-; CHECK-NEXT:    vpblendd {{.*#+}} xmm0 = xmm3[0],xmm0[1,2,3]
+; CHECK-NEXT:    vmovdqa {{.*#+}} ymm3 = <4,0,3,2,u,u,u,u>
+; CHECK-NEXT:    vpermd %ymm0, %ymm3, %ymm0
 ; CHECK-NEXT:    vptestnmd %xmm2, %xmm2, %k1
 ; CHECK-NEXT:    vpblendmd %xmm0, %xmm1, %xmm0 {%k1}
 ; CHECK-NEXT:    vzeroupper
@@ -949,9 +948,8 @@ define <4 x i32> @test_masked_8xi32_to_4xi32_perm_mask0(<8 x i32> %vec, <4 x i32
 define <4 x i32> @test_masked_z_8xi32_to_4xi32_perm_mask0(<8 x i32> %vec, <4 x i32> %mask) {
 ; CHECK-LABEL: test_masked_z_8xi32_to_4xi32_perm_mask0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vextracti128 $1, %ymm0, %xmm2
-; CHECK-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,3,2]
-; CHECK-NEXT:    vpblendd {{.*#+}} xmm0 = xmm2[0],xmm0[1,2,3]
+; CHECK-NEXT:    vmovdqa {{.*#+}} ymm2 = <4,0,3,2,u,u,u,u>
+; CHECK-NEXT:    vpermd %ymm0, %ymm2, %ymm0
 ; CHECK-NEXT:    vptestnmd %xmm1, %xmm1, %k1
 ; CHECK-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
 ; CHECK-NEXT:    vzeroupper
@@ -964,10 +962,8 @@ define <4 x i32> @test_masked_z_8xi32_to_4xi32_perm_mask0(<8 x i32> %vec, <4 x i
 define <4 x i32> @test_masked_8xi32_to_4xi32_perm_mask1(<8 x i32> %vec, <4 x i32> %vec2, <4 x i32> %mask) {
 ; CHECK-LABEL: test_masked_8xi32_to_4xi32_perm_mask1:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vextracti128 $1, %ymm0, %xmm3
-; CHECK-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[2,2,3,3]
-; CHECK-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[3,0,2,3]
-; CHECK-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0,1],xmm3[2],xmm0[3]
+; CHECK-NEXT:    vmovdqa {{.*#+}} ymm3 = <3,0,7,3,u,u,u,u>
+; CHECK-NEXT:    vpermd %ymm0, %ymm3, %ymm0
 ; CHECK-NEXT:    vptestnmd %xmm2, %xmm2, %k1
 ; CHECK-NEXT:    vpblendmd %xmm0, %xmm1, %xmm0 {%k1}
 ; CHECK-NEXT:    vzeroupper
@@ -981,10 +977,8 @@ define <4 x i32> @test_masked_8xi32_to_4xi32_perm_mask1(<8 x i32> %vec, <4 x i32
 define <4 x i32> @test_masked_z_8xi32_to_4xi32_perm_mask1(<8 x i32> %vec, <4 x i32> %mask) {
 ; CHECK-LABEL: test_masked_z_8xi32_to_4xi32_perm_mask1:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vextracti128 $1, %ymm0, %xmm2
-; CHECK-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,2,3,3]
-; CHECK-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[3,0,2,3]
-; CHECK-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0,1],xmm2[2],xmm0[3]
+; CHECK-NEXT:    vmovdqa {{.*#+}} ymm2 = <3,0,7,3,u,u,u,u>
+; CHECK-NEXT:    vpermd %ymm0, %ymm2, %ymm0
 ; CHECK-NEXT:    vptestnmd %xmm1, %xmm1, %k1
 ; CHECK-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
 ; CHECK-NEXT:    vzeroupper
@@ -1026,9 +1020,9 @@ define <4 x i32> @test_masked_z_8xi32_to_4xi32_perm_mask2(<8 x i32> %vec, <4 x i
 define <4 x i32> @test_8xi32_to_4xi32_perm_mask3(<8 x i32> %vec) {
 ; CHECK-LABEL: test_8xi32_to_4xi32_perm_mask3:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; CHECK-NEXT:    vblendps {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
-; CHECK-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[1,3,2,1]
+; CHECK-NEXT:    vmovaps {{.*#+}} ymm1 = <5,3,2,5,u,u,u,u>
+; CHECK-NEXT:    vpermps %ymm0, %ymm1, %ymm0
+; CHECK-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retq
   %res = shufflevector <8 x i32> %vec, <8 x i32> undef, <4 x i32> <i32 5, i32 3, i32 2, i32 5>
@@ -1037,11 +1031,10 @@ define <4 x i32> @test_8xi32_to_4xi32_perm_mask3(<8 x i32> %vec) {
 define <4 x i32> @test_masked_8xi32_to_4xi32_perm_mask3(<8 x i32> %vec, <4 x i32> %vec2, <4 x i32> %mask) {
 ; CHECK-LABEL: test_masked_8xi32_to_4xi32_perm_mask3:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vextracti128 $1, %ymm0, %xmm3
-; CHECK-NEXT:    vpblendd {{.*#+}} xmm0 = xmm3[0,1],xmm0[2,3]
+; CHECK-NEXT:    vmovdqa {{.*#+}} ymm3 = <5,3,2,5,u,u,u,u>
+; CHECK-NEXT:    vpermd %ymm0, %ymm3, %ymm0
 ; CHECK-NEXT:    vptestnmd %xmm2, %xmm2, %k1
-; CHECK-NEXT:    vpshufd {{.*#+}} xmm1 {%k1} = xmm0[1,3,2,1]
-; CHECK-NEXT:    vmovdqa %xmm1, %xmm0
+; CHECK-NEXT:    vpblendmd %xmm0, %xmm1, %xmm0 {%k1}
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retq
   %shuf = shufflevector <8 x i32> %vec, <8 x i32> undef, <4 x i32> <i32 5, i32 3, i32 2, i32 5>
@@ -1053,10 +1046,10 @@ define <4 x i32> @test_masked_8xi32_to_4xi32_perm_mask3(<8 x i32> %vec, <4 x i32
 define <4 x i32> @test_masked_z_8xi32_to_4xi32_perm_mask3(<8 x i32> %vec, <4 x i32> %mask) {
 ; CHECK-LABEL: test_masked_z_8xi32_to_4xi32_perm_mask3:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vextracti128 $1, %ymm0, %xmm2
-; CHECK-NEXT:    vpblendd {{.*#+}} xmm0 = xmm2[0,1],xmm0[2,3]
+; CHECK-NEXT:    vmovdqa {{.*#+}} ymm2 = <5,3,2,5,u,u,u,u>
+; CHECK-NEXT:    vpermd %ymm0, %ymm2, %ymm0
 ; CHECK-NEXT:    vptestnmd %xmm1, %xmm1, %k1
-; CHECK-NEXT:    vpshufd {{.*#+}} xmm0 {%k1} {z} = xmm0[1,3,2,1]
+; CHECK-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retq
   %shuf = shufflevector <8 x i32> %vec, <8 x i32> undef, <4 x i32> <i32 5, i32 3, i32 2, i32 5>
@@ -1167,9 +1160,9 @@ define <4 x i32> @test_masked_z_8xi32_to_4xi32_perm_mem_mask2(<8 x i32>* %vp, <4
 define <4 x i32> @test_8xi32_to_4xi32_perm_mem_mask3(<8 x i32>* %vp) {
 ; CHECK-LABEL: test_8xi32_to_4xi32_perm_mem_mask3:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vpshufd {{.*#+}} xmm0 = mem[1,1,2,3]
-; CHECK-NEXT:    vpbroadcastq 8(%rdi), %xmm1
-; CHECK-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1,2],xmm0[3]
+; CHECK-NEXT:    vpermilps {{.*#+}} xmm0 = mem[1,1,2,3]
+; CHECK-NEXT:    vmovddup {{.*#+}} xmm1 = mem[0,0]
+; CHECK-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2],xmm0[3]
 ; CHECK-NEXT:    retq
   %vec = load <8 x i32>, <8 x i32>* %vp
   %res = shufflevector <8 x i32> %vec, <8 x i32> undef, <4 x i32> <i32 5, i32 3, i32 2, i32 7>
@@ -1817,8 +1810,8 @@ define <4 x i32> @test_masked_z_16xi32_to_4xi32_perm_mem_mask3(<16 x i32>* %vp,
 define <2 x i64> @test_4xi64_to_2xi64_perm_mask0(<4 x i64> %vec) {
 ; CHECK-LABEL: test_4xi64_to_2xi64_perm_mask0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; CHECK-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; CHECK-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[2,0,2,3]
+; CHECK-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retq
   %res = shufflevector <4 x i64> %vec, <4 x i64> undef, <2 x i32> <i32 2, i32 0>
@@ -1827,10 +1820,9 @@ define <2 x i64> @test_4xi64_to_2xi64_perm_mask0(<4 x i64> %vec) {
 define <2 x i64> @test_masked_4xi64_to_2xi64_perm_mask0(<4 x i64> %vec, <2 x i64> %vec2, <2 x i64> %mask) {
 ; CHECK-LABEL: test_masked_4xi64_to_2xi64_perm_mask0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vextracti128 $1, %ymm0, %xmm3
+; CHECK-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,0,2,3]
 ; CHECK-NEXT:    vptestnmq %xmm2, %xmm2, %k1
-; CHECK-NEXT:    vpunpcklqdq {{.*#+}} xmm1 {%k1} = xmm3[0],xmm0[0]
-; CHECK-NEXT:    vmovdqa %xmm1, %xmm0
+; CHECK-NEXT:    vpblendmq %xmm0, %xmm1, %xmm0 {%k1}
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retq
   %shuf = shufflevector <4 x i64> %vec, <4 x i64> undef, <2 x i32> <i32 2, i32 0>
@@ -1842,9 +1834,9 @@ define <2 x i64> @test_masked_4xi64_to_2xi64_perm_mask0(<4 x i64> %vec, <2 x i64
 define <2 x i64> @test_masked_z_4xi64_to_2xi64_perm_mask0(<4 x i64> %vec, <2 x i64> %mask) {
 ; CHECK-LABEL: test_masked_z_4xi64_to_2xi64_perm_mask0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vextracti128 $1, %ymm0, %xmm2
+; CHECK-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,0,2,3]
 ; CHECK-NEXT:    vptestnmq %xmm1, %xmm1, %k1
-; CHECK-NEXT:    vpunpcklqdq {{.*#+}} xmm0 {%k1} {z} = xmm2[0],xmm0[0]
+; CHECK-NEXT:    vmovdqa64 %xmm0, %xmm0 {%k1} {z}
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retq
   %shuf = shufflevector <4 x i64> %vec, <4 x i64> undef, <2 x i32> <i32 2, i32 0>
@@ -1855,8 +1847,7 @@ define <2 x i64> @test_masked_z_4xi64_to_2xi64_perm_mask0(<4 x i64> %vec, <2 x i
 define <2 x i64> @test_masked_4xi64_to_2xi64_perm_mask1(<4 x i64> %vec, <2 x i64> %vec2, <2 x i64> %mask) {
 ; CHECK-LABEL: test_masked_4xi64_to_2xi64_perm_mask1:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vextracti128 $1, %ymm0, %xmm3
-; CHECK-NEXT:    vpblendd {{.*#+}} xmm0 = xmm3[0,1],xmm0[2,3]
+; CHECK-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,1,2,3]
 ; CHECK-NEXT:    vptestnmq %xmm2, %xmm2, %k1
 ; CHECK-NEXT:    vpblendmq %xmm0, %xmm1, %xmm0 {%k1}
 ; CHECK-NEXT:    vzeroupper
@@ -1870,8 +1861,7 @@ define <2 x i64> @test_masked_4xi64_to_2xi64_perm_mask1(<4 x i64> %vec, <2 x i64
 define <2 x i64> @test_masked_z_4xi64_to_2xi64_perm_mask1(<4 x i64> %vec, <2 x i64> %mask) {
 ; CHECK-LABEL: test_masked_z_4xi64_to_2xi64_perm_mask1:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vextracti128 $1, %ymm0, %xmm2
-; CHECK-NEXT:    vpblendd {{.*#+}} xmm0 = xmm2[0,1],xmm0[2,3]
+; CHECK-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,1,2,3]
 ; CHECK-NEXT:    vptestnmq %xmm1, %xmm1, %k1
 ; CHECK-NEXT:    vmovdqa64 %xmm0, %xmm0 {%k1} {z}
 ; CHECK-NEXT:    vzeroupper
@@ -2183,7 +2173,7 @@ define <4 x i64> @test_masked_z_8xi64_to_4xi64_perm_mask6(<8 x i64> %vec, <4 x i
 define <4 x i64> @test_masked_8xi64_to_4xi64_perm_mask7(<8 x i64> %vec, <4 x i64> %vec2, <4 x i64> %mask) {
 ; CHECK-LABEL: test_masked_8xi64_to_4xi64_perm_mask7:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vextracti64x4 $1, %zmm0, %ymm3
+; CHECK-NEXT:    vextracti32x4 $2, %zmm0, %xmm3
 ; CHECK-NEXT:    vmovdqa {{.*#+}} ymm4 = [2,0,3,4]
 ; CHECK-NEXT:    vpermi2q %ymm3, %ymm0, %ymm4
 ; CHECK-NEXT:    vptestnmq %ymm2, %ymm2, %k1
@@ -2198,7 +2188,7 @@ define <4 x i64> @test_masked_8xi64_to_4xi64_perm_mask7(<8 x i64> %vec, <4 x i64
 define <4 x i64> @test_masked_z_8xi64_to_4xi64_perm_mask7(<8 x i64> %vec, <4 x i64> %mask) {
 ; CHECK-LABEL: test_masked_z_8xi64_to_4xi64_perm_mask7:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vextracti64x4 $1, %zmm0, %ymm3
+; CHECK-NEXT:    vextracti32x4 $2, %zmm0, %xmm3
 ; CHECK-NEXT:    vmovdqa {{.*#+}} ymm2 = [2,0,3,4]
 ; CHECK-NEXT:    vptestnmq %ymm1, %ymm1, %k1
 ; CHECK-NEXT:    vpermi2q %ymm3, %ymm0, %ymm2 {%k1} {z}
@@ -2678,12 +2668,11 @@ define <4 x float> @test_masked_z_8xfloat_to_4xfloat_perm_mask0(<8 x float> %vec
 define <4 x float> @test_masked_8xfloat_to_4xfloat_perm_mask1(<8 x float> %vec, <4 x float> %vec2, <4 x float> %mask) {
 ; CHECK-LABEL: test_masked_8xfloat_to_4xfloat_perm_mask1:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vextractf128 $1, %ymm0, %xmm3
-; CHECK-NEXT:    vshufps {{.*#+}} xmm3 = xmm3[1,0],xmm0[0,0]
-; CHECK-NEXT:    vxorps %xmm4, %xmm4, %xmm4
-; CHECK-NEXT:    vcmpeqps %xmm4, %xmm2, %k1
-; CHECK-NEXT:    vshufps {{.*#+}} xmm1 {%k1} = xmm0[1,3],xmm3[0,2]
-; CHECK-NEXT:    vmovaps %xmm1, %xmm0
+; CHECK-NEXT:    vmovaps {{.*#+}} ymm3 = <1,3,5,0,u,u,u,u>
+; CHECK-NEXT:    vpermps %ymm0, %ymm3, %ymm0
+; CHECK-NEXT:    vxorps %xmm3, %xmm3, %xmm3
+; CHECK-NEXT:    vcmpeqps %xmm3, %xmm2, %k1
+; CHECK-NEXT:    vblendmps %xmm0, %xmm1, %xmm0 {%k1}
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retq
   %shuf = shufflevector <8 x float> %vec, <8 x float> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 0>
@@ -2695,11 +2684,11 @@ define <4 x float> @test_masked_8xfloat_to_4xfloat_perm_mask1(<8 x float> %vec,
 define <4 x float> @test_masked_z_8xfloat_to_4xfloat_perm_mask1(<8 x float> %vec, <4 x float> %mask) {
 ; CHECK-LABEL: test_masked_z_8xfloat_to_4xfloat_perm_mask1:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vextractf128 $1, %ymm0, %xmm2
-; CHECK-NEXT:    vshufps {{.*#+}} xmm2 = xmm2[1,0],xmm0[0,0]
-; CHECK-NEXT:    vxorps %xmm3, %xmm3, %xmm3
-; CHECK-NEXT:    vcmpeqps %xmm3, %xmm1, %k1
-; CHECK-NEXT:    vshufps {{.*#+}} xmm0 {%k1} {z} = xmm0[1,3],xmm2[0,2]
+; CHECK-NEXT:    vmovaps {{.*#+}} ymm2 = <1,3,5,0,u,u,u,u>
+; CHECK-NEXT:    vpermps %ymm0, %ymm2, %ymm0
+; CHECK-NEXT:    vxorps %xmm2, %xmm2, %xmm2
+; CHECK-NEXT:    vcmpeqps %xmm2, %xmm1, %k1
+; CHECK-NEXT:    vmovaps %xmm0, %xmm0 {%k1} {z}
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retq
   %shuf = shufflevector <8 x float> %vec, <8 x float> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 0>
@@ -2710,12 +2699,11 @@ define <4 x float> @test_masked_z_8xfloat_to_4xfloat_perm_mask1(<8 x float> %vec
 define <4 x float> @test_masked_8xfloat_to_4xfloat_perm_mask2(<8 x float> %vec, <4 x float> %vec2, <4 x float> %mask) {
 ; CHECK-LABEL: test_masked_8xfloat_to_4xfloat_perm_mask2:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vextractf128 $1, %ymm0, %xmm3
-; CHECK-NEXT:    vshufps {{.*#+}} xmm3 = xmm3[3,0],xmm0[0,0]
-; CHECK-NEXT:    vxorps %xmm4, %xmm4, %xmm4
-; CHECK-NEXT:    vcmpeqps %xmm4, %xmm2, %k1
-; CHECK-NEXT:    vshufps {{.*#+}} xmm1 {%k1} = xmm0[3,2],xmm3[0,2]
-; CHECK-NEXT:    vmovaps %xmm1, %xmm0
+; CHECK-NEXT:    vmovaps {{.*#+}} ymm3 = <3,2,7,0,u,u,u,u>
+; CHECK-NEXT:    vpermps %ymm0, %ymm3, %ymm0
+; CHECK-NEXT:    vxorps %xmm3, %xmm3, %xmm3
+; CHECK-NEXT:    vcmpeqps %xmm3, %xmm2, %k1
+; CHECK-NEXT:    vblendmps %xmm0, %xmm1, %xmm0 {%k1}
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retq
   %shuf = shufflevector <8 x float> %vec, <8 x float> undef, <4 x i32> <i32 3, i32 2, i32 7, i32 0>
@@ -2727,11 +2715,11 @@ define <4 x float> @test_masked_8xfloat_to_4xfloat_perm_mask2(<8 x float> %vec,
 define <4 x float> @test_masked_z_8xfloat_to_4xfloat_perm_mask2(<8 x float> %vec, <4 x float> %mask) {
 ; CHECK-LABEL: test_masked_z_8xfloat_to_4xfloat_perm_mask2:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vextractf128 $1, %ymm0, %xmm2
-; CHECK-NEXT:    vshufps {{.*#+}} xmm2 = xmm2[3,0],xmm0[0,0]
-; CHECK-NEXT:    vxorps %xmm3, %xmm3, %xmm3
-; CHECK-NEXT:    vcmpeqps %xmm3, %xmm1, %k1
-; CHECK-NEXT:    vshufps {{.*#+}} xmm0 {%k1} {z} = xmm0[3,2],xmm2[0,2]
+; CHECK-NEXT:    vmovaps {{.*#+}} ymm2 = <3,2,7,0,u,u,u,u>
+; CHECK-NEXT:    vpermps %ymm0, %ymm2, %ymm0
+; CHECK-NEXT:    vxorps %xmm2, %xmm2, %xmm2
+; CHECK-NEXT:    vcmpeqps %xmm2, %xmm1, %k1
+; CHECK-NEXT:    vmovaps %xmm0, %xmm0 {%k1} {z}
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retq
   %shuf = shufflevector <8 x float> %vec, <8 x float> undef, <4 x i32> <i32 3, i32 2, i32 7, i32 0>
@@ -2742,9 +2730,9 @@ define <4 x float> @test_masked_z_8xfloat_to_4xfloat_perm_mask2(<8 x float> %vec
 define <4 x float> @test_8xfloat_to_4xfloat_perm_mask3(<8 x float> %vec) {
 ; CHECK-LABEL: test_8xfloat_to_4xfloat_perm_mask3:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; CHECK-NEXT:    vblendps {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
-; CHECK-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,3,1,2]
+; CHECK-NEXT:    vmovaps {{.*#+}} ymm1 = <3,3,5,2,u,u,u,u>
+; CHECK-NEXT:    vpermps %ymm0, %ymm1, %ymm0
+; CHECK-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retq
   %res = shufflevector <8 x float> %vec, <8 x float> undef, <4 x i32> <i32 3, i32 3, i32 5, i32 2>
@@ -2753,12 +2741,11 @@ define <4 x float> @test_8xfloat_to_4xfloat_perm_mask3(<8 x float> %vec) {
 define <4 x float> @test_masked_8xfloat_to_4xfloat_perm_mask3(<8 x float> %vec, <4 x float> %vec2, <4 x float> %mask) {
 ; CHECK-LABEL: test_masked_8xfloat_to_4xfloat_perm_mask3:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vextractf128 $1, %ymm0, %xmm3
-; CHECK-NEXT:    vblendps {{.*#+}} xmm0 = xmm3[0,1],xmm0[2,3]
+; CHECK-NEXT:    vmovaps {{.*#+}} ymm3 = <3,3,5,2,u,u,u,u>
+; CHECK-NEXT:    vpermps %ymm0, %ymm3, %ymm0
 ; CHECK-NEXT:    vxorps %xmm3, %xmm3, %xmm3
 ; CHECK-NEXT:    vcmpeqps %xmm3, %xmm2, %k1
-; CHECK-NEXT:    vpermilps {{.*#+}} xmm1 {%k1} = xmm0[3,3,1,2]
-; CHECK-NEXT:    vmovaps %xmm1, %xmm0
+; CHECK-NEXT:    vblendmps %xmm0, %xmm1, %xmm0 {%k1}
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retq
   %shuf = shufflevector <8 x float> %vec, <8 x float> undef, <4 x i32> <i32 3, i32 3, i32 5, i32 2>
@@ -2770,11 +2757,11 @@ define <4 x float> @test_masked_8xfloat_to_4xfloat_perm_mask3(<8 x float> %vec,
 define <4 x float> @test_masked_z_8xfloat_to_4xfloat_perm_mask3(<8 x float> %vec, <4 x float> %mask) {
 ; CHECK-LABEL: test_masked_z_8xfloat_to_4xfloat_perm_mask3:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vextractf128 $1, %ymm0, %xmm2
-; CHECK-NEXT:    vblendps {{.*#+}} xmm0 = xmm2[0,1],xmm0[2,3]
+; CHECK-NEXT:    vmovaps {{.*#+}} ymm2 = <3,3,5,2,u,u,u,u>
+; CHECK-NEXT:    vpermps %ymm0, %ymm2, %ymm0
 ; CHECK-NEXT:    vxorps %xmm2, %xmm2, %xmm2
 ; CHECK-NEXT:    vcmpeqps %xmm2, %xmm1, %k1
-; CHECK-NEXT:    vpermilps {{.*#+}} xmm0 {%k1} {z} = xmm0[3,3,1,2]
+; CHECK-NEXT:    vmovaps %xmm0, %xmm0 {%k1} {z}
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retq
   %shuf = shufflevector <8 x float> %vec, <8 x float> undef, <4 x i32> <i32 3, i32 3, i32 5, i32 2>
@@ -3134,7 +3121,7 @@ define <4 x float> @test_masked_z_16xfloat_to_4xfloat_perm_mask0(<16 x float> %v
 define <4 x float> @test_masked_16xfloat_to_4xfloat_perm_mask1(<16 x float> %vec, <4 x float> %vec2, <4 x float> %mask) {
 ; CHECK-LABEL: test_masked_16xfloat_to_4xfloat_perm_mask1:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vextractf64x4 $1, %zmm0, %ymm3
+; CHECK-NEXT:    vextractf32x4 $2, %zmm0, %xmm3
 ; CHECK-NEXT:    vextractf128 $1, %ymm0, %xmm0
 ; CHECK-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,2,2,2]
 ; CHECK-NEXT:    vblendps {{.*#+}} xmm0 = xmm3[0],xmm0[1],xmm3[2],xmm0[3]
@@ -3152,7 +3139,7 @@ define <4 x float> @test_masked_16xfloat_to_4xfloat_perm_mask1(<16 x float> %vec
 define <4 x float> @test_masked_z_16xfloat_to_4xfloat_perm_mask1(<16 x float> %vec, <4 x float> %mask) {
 ; CHECK-LABEL: test_masked_z_16xfloat_to_4xfloat_perm_mask1:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vextractf64x4 $1, %zmm0, %ymm2
+; CHECK-NEXT:    vextractf32x4 $2, %zmm0, %xmm2
 ; CHECK-NEXT:    vextractf128 $1, %ymm0, %xmm0
 ; CHECK-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,2,2,2]
 ; CHECK-NEXT:    vblendps {{.*#+}} xmm0 = xmm2[0],xmm0[1],xmm2[2],xmm0[3]
@@ -3203,7 +3190,7 @@ define <4 x float> @test_16xfloat_to_4xfloat_perm_mask3(<16 x float> %vec) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vmovaps {{.*#+}} ymm1 = [0,2,4,6,4,6,6,7]
 ; CHECK-NEXT:    vpermps %ymm0, %ymm1, %ymm1
-; CHECK-NEXT:    vextractf64x4 $1, %zmm0, %ymm0
+; CHECK-NEXT:    vextractf32x4 $2, %zmm0, %xmm0
 ; CHECK-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[2,1,3,3]
 ; CHECK-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3]
 ; CHECK-NEXT:    vzeroupper
@@ -3216,7 +3203,7 @@ define <4 x float> @test_masked_16xfloat_to_4xfloat_perm_mask3(<16 x float> %vec
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vmovaps {{.*#+}} ymm3 = [0,2,4,6,4,6,6,7]
 ; CHECK-NEXT:    vpermps %ymm0, %ymm3, %ymm3
-; CHECK-NEXT:    vextractf64x4 $1, %zmm0, %ymm0
+; CHECK-NEXT:    vextractf32x4 $2, %zmm0, %xmm0
 ; CHECK-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[2,1,3,3]
 ; CHECK-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm3[1],xmm0[2],xmm3[3]
 ; CHECK-NEXT:    vxorps %xmm3, %xmm3, %xmm3
@@ -3235,7 +3222,7 @@ define <4 x float> @test_masked_z_16xfloat_to_4xfloat_perm_mask3(<16 x float> %v
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vmovaps {{.*#+}} ymm2 = [0,2,4,6,4,6,6,7]
 ; CHECK-NEXT:    vpermps %ymm0, %ymm2, %ymm2
-; CHECK-NEXT:    vextractf64x4 $1, %zmm0, %ymm0
+; CHECK-NEXT:    vextractf32x4 $2, %zmm0, %xmm0
 ; CHECK-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[2,1,3,3]
 ; CHECK-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2],xmm2[3]
 ; CHECK-NEXT:    vxorps %xmm2, %xmm2, %xmm2
@@ -3578,8 +3565,8 @@ define <4 x float> @test_masked_z_16xfloat_to_4xfloat_perm_mem_mask3(<16 x float
 define <2 x double> @test_4xdouble_to_2xdouble_perm_mask0(<4 x double> %vec) {
 ; CHECK-LABEL: test_4xdouble_to_2xdouble_perm_mask0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; CHECK-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; CHECK-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[2,0,2,3]
+; CHECK-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retq
   %res = shufflevector <4 x double> %vec, <4 x double> undef, <2 x i32> <i32 2, i32 0>
@@ -3588,11 +3575,10 @@ define <2 x double> @test_4xdouble_to_2xdouble_perm_mask0(<4 x double> %vec) {
 define <2 x double> @test_masked_4xdouble_to_2xdouble_perm_mask0(<4 x double> %vec, <2 x double> %vec2, <2 x double> %mask) {
 ; CHECK-LABEL: test_masked_4xdouble_to_2xdouble_perm_mask0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vextractf128 $1, %ymm0, %xmm3
-; CHECK-NEXT:    vxorpd %xmm4, %xmm4, %xmm4
-; CHECK-NEXT:    vcmpeqpd %xmm4, %xmm2, %k1
-; CHECK-NEXT:    vunpcklpd {{.*#+}} xmm1 {%k1} = xmm3[0],xmm0[0]
-; CHECK-NEXT:    vmovapd %xmm1, %xmm0
+; CHECK-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[2,0,2,3]
+; CHECK-NEXT:    vxorpd %xmm3, %xmm3, %xmm3
+; CHECK-NEXT:    vcmpeqpd %xmm3, %xmm2, %k1
+; CHECK-NEXT:    vblendmpd %xmm0, %xmm1, %xmm0 {%k1}
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retq
   %shuf = shufflevector <4 x double> %vec, <4 x double> undef, <2 x i32> <i32 2, i32 0>
@@ -3604,10 +3590,10 @@ define <2 x double> @test_masked_4xdouble_to_2xdouble_perm_mask0(<4 x double> %v
 define <2 x double> @test_masked_z_4xdouble_to_2xdouble_perm_mask0(<4 x double> %vec, <2 x double> %mask) {
 ; CHECK-LABEL: test_masked_z_4xdouble_to_2xdouble_perm_mask0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vextractf128 $1, %ymm0, %xmm2
-; CHECK-NEXT:    vxorpd %xmm3, %xmm3, %xmm3
-; CHECK-NEXT:    vcmpeqpd %xmm3, %xmm1, %k1
-; CHECK-NEXT:    vunpcklpd {{.*#+}} xmm0 {%k1} {z} = xmm2[0],xmm0[0]
+; CHECK-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[2,0,2,3]
+; CHECK-NEXT:    vxorpd %xmm2, %xmm2, %xmm2
+; CHECK-NEXT:    vcmpeqpd %xmm2, %xmm1, %k1
+; CHECK-NEXT:    vmovapd %xmm0, %xmm0 {%k1} {z}
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retq
   %shuf = shufflevector <4 x double> %vec, <4 x double> undef, <2 x i32> <i32 2, i32 0>
@@ -3618,11 +3604,10 @@ define <2 x double> @test_masked_z_4xdouble_to_2xdouble_perm_mask0(<4 x double>
 define <2 x double> @test_masked_4xdouble_to_2xdouble_perm_mask1(<4 x double> %vec, <2 x double> %vec2, <2 x double> %mask) {
 ; CHECK-LABEL: test_masked_4xdouble_to_2xdouble_perm_mask1:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vextractf128 $1, %ymm0, %xmm3
-; CHECK-NEXT:    vxorpd %xmm4, %xmm4, %xmm4
-; CHECK-NEXT:    vcmpeqpd %xmm4, %xmm2, %k1
-; CHECK-NEXT:    vunpckhpd {{.*#+}} xmm1 {%k1} = xmm0[1],xmm3[1]
-; CHECK-NEXT:    vmovapd %xmm1, %xmm0
+; CHECK-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[1,3,2,3]
+; CHECK-NEXT:    vxorpd %xmm3, %xmm3, %xmm3
+; CHECK-NEXT:    vcmpeqpd %xmm3, %xmm2, %k1
+; CHECK-NEXT:    vblendmpd %xmm0, %xmm1, %xmm0 {%k1}
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retq
   %shuf = shufflevector <4 x double> %vec, <4 x double> undef, <2 x i32> <i32 1, i32 3>
@@ -3634,10 +3619,10 @@ define <2 x double> @test_masked_4xdouble_to_2xdouble_perm_mask1(<4 x double> %v
 define <2 x double> @test_masked_z_4xdouble_to_2xdouble_perm_mask1(<4 x double> %vec, <2 x double> %mask) {
 ; CHECK-LABEL: test_masked_z_4xdouble_to_2xdouble_perm_mask1:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vextractf128 $1, %ymm0, %xmm2
-; CHECK-NEXT:    vxorpd %xmm3, %xmm3, %xmm3
-; CHECK-NEXT:    vcmpeqpd %xmm3, %xmm1, %k1
-; CHECK-NEXT:    vunpckhpd {{.*#+}} xmm0 {%k1} {z} = xmm0[1],xmm2[1]
+; CHECK-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[1,3,2,3]
+; CHECK-NEXT:    vxorpd %xmm2, %xmm2, %xmm2
+; CHECK-NEXT:    vcmpeqpd %xmm2, %xmm1, %k1
+; CHECK-NEXT:    vmovapd %xmm0, %xmm0 {%k1} {z}
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retq
   %shuf = shufflevector <4 x double> %vec, <4 x double> undef, <2 x i32> <i32 1, i32 3>
@@ -3819,7 +3804,7 @@ define <4 x double> @test_masked_z_8xdouble_to_4xdouble_perm_mask2(<8 x double>
 define <4 x double> @test_8xdouble_to_4xdouble_perm_mask3(<8 x double> %vec) {
 ; CHECK-LABEL: test_8xdouble_to_4xdouble_perm_mask3:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vextractf64x4 $1, %zmm0, %ymm2
+; CHECK-NEXT:    vextractf32x4 $2, %zmm0, %xmm2
 ; CHECK-NEXT:    vmovapd {{.*#+}} ymm1 = [0,2,1,4]
 ; CHECK-NEXT:    vpermi2pd %ymm2, %ymm0, %ymm1
 ; CHECK-NEXT:    vmovapd %ymm1, %ymm0
@@ -3830,7 +3815,7 @@ define <4 x double> @test_8xdouble_to_4xdouble_perm_mask3(<8 x double> %vec) {
 define <4 x double> @test_masked_8xdouble_to_4xdouble_perm_mask3(<8 x double> %vec, <4 x double> %vec2, <4 x double> %mask) {
 ; CHECK-LABEL: test_masked_8xdouble_to_4xdouble_perm_mask3:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vextractf64x4 $1, %zmm0, %ymm3
+; CHECK-NEXT:    vextractf32x4 $2, %zmm0, %xmm3
 ; CHECK-NEXT:    vmovapd {{.*#+}} ymm4 = [0,2,1,4]
 ; CHECK-NEXT:    vpermi2pd %ymm3, %ymm0, %ymm4
 ; CHECK-NEXT:    vxorpd %xmm0, %xmm0, %xmm0
@@ -3846,7 +3831,7 @@ define <4 x double> @test_masked_8xdouble_to_4xdouble_perm_mask3(<8 x double> %v
 define <4 x double> @test_masked_z_8xdouble_to_4xdouble_perm_mask3(<8 x double> %vec, <4 x double> %mask) {
 ; CHECK-LABEL: test_masked_z_8xdouble_to_4xdouble_perm_mask3:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vextractf64x4 $1, %zmm0, %ymm3
+; CHECK-NEXT:    vextractf32x4 $2, %zmm0, %xmm3
 ; CHECK-NEXT:    vmovapd {{.*#+}} ymm2 = [0,2,1,4]
 ; CHECK-NEXT:    vxorpd %xmm4, %xmm4, %xmm4
 ; CHECK-NEXT:    vcmpeqpd %ymm4, %ymm1, %k1
@@ -3861,7 +3846,7 @@ define <4 x double> @test_masked_z_8xdouble_to_4xdouble_perm_mask3(<8 x double>
 define <4 x double> @test_masked_8xdouble_to_4xdouble_perm_mask4(<8 x double> %vec, <4 x double> %vec2, <4 x double> %mask) {
 ; CHECK-LABEL: test_masked_8xdouble_to_4xdouble_perm_mask4:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vextractf64x4 $1, %zmm0, %ymm3
+; CHECK-NEXT:    vextractf32x4 $2, %zmm0, %xmm3
 ; CHECK-NEXT:    vmovapd {{.*#+}} ymm4 = [1,1,5,5]
 ; CHECK-NEXT:    vpermi2pd %ymm3, %ymm0, %ymm4
 ; CHECK-NEXT:    vxorpd %xmm0, %xmm0, %xmm0
@@ -3877,7 +3862,7 @@ define <4 x double> @test_masked_8xdouble_to_4xdouble_perm_mask4(<8 x double> %v
 define <4 x double> @test_masked_z_8xdouble_to_4xdouble_perm_mask4(<8 x double> %vec, <4 x double> %mask) {
 ; CHECK-LABEL: test_masked_z_8xdouble_to_4xdouble_perm_mask4:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vextractf64x4 $1, %zmm0, %ymm3
+; CHECK-NEXT:    vextractf32x4 $2, %zmm0, %xmm3
 ; CHECK-NEXT:    vmovapd {{.*#+}} ymm2 = [1,1,5,5]
 ; CHECK-NEXT:    vxorpd %xmm4, %xmm4, %xmm4
 ; CHECK-NEXT:    vcmpeqpd %ymm4, %ymm1, %k1
@@ -3996,10 +3981,9 @@ define <4 x double> @test_masked_z_8xdouble_to_4xdouble_perm_mask7(<8 x double>
 define <2 x double> @test_8xdouble_to_2xdouble_perm_mask0(<8 x double> %vec) {
 ; CHECK-LABEL: test_8xdouble_to_2xdouble_perm_mask0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vextractf64x4 $1, %zmm0, %ymm2
-; CHECK-NEXT:    vmovapd {{.*#+}} ymm1 = [0,6,2,6]
-; CHECK-NEXT:    vpermi2pd %ymm2, %ymm0, %ymm1
-; CHECK-NEXT:    vmovapd %xmm1, %xmm0
+; CHECK-NEXT:    vextractf64x4 $1, %zmm0, %ymm1
+; CHECK-NEXT:    vextractf128 $1, %ymm1, %xmm1
+; CHECK-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0]
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retq
   %res = shufflevector <8 x double> %vec, <8 x double> undef, <2 x i32> <i32 0, i32 6>
@@ -4009,11 +3993,11 @@ define <2 x double> @test_masked_8xdouble_to_2xdouble_perm_mask0(<8 x double> %v
 ; CHECK-LABEL: test_masked_8xdouble_to_2xdouble_perm_mask0:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vextractf64x4 $1, %zmm0, %ymm3
-; CHECK-NEXT:    vmovapd {{.*#+}} ymm4 = [0,6,2,6]
-; CHECK-NEXT:    vpermi2pd %ymm3, %ymm0, %ymm4
-; CHECK-NEXT:    vxorpd %xmm0, %xmm0, %xmm0
-; CHECK-NEXT:    vcmpeqpd %xmm0, %xmm2, %k1
-; CHECK-NEXT:    vblendmpd %xmm4, %xmm1, %xmm0 {%k1}
+; CHECK-NEXT:    vextractf128 $1, %ymm3, %xmm3
+; CHECK-NEXT:    vxorpd %xmm4, %xmm4, %xmm4
+; CHECK-NEXT:    vcmpeqpd %xmm4, %xmm2, %k1
+; CHECK-NEXT:    vunpcklpd {{.*#+}} xmm1 {%k1} = xmm0[0],xmm3[0]
+; CHECK-NEXT:    vmovapd %xmm1, %xmm0
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retq
   %shuf = shufflevector <8 x double> %vec, <8 x double> undef, <2 x i32> <i32 0, i32 6>
@@ -4026,11 +4010,10 @@ define <2 x double> @test_masked_z_8xdouble_to_2xdouble_perm_mask0(<8 x double>
 ; CHECK-LABEL: test_masked_z_8xdouble_to_2xdouble_perm_mask0:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vextractf64x4 $1, %zmm0, %ymm2
-; CHECK-NEXT:    vmovapd {{.*#+}} ymm3 = [0,6,2,6]
-; CHECK-NEXT:    vpermi2pd %ymm2, %ymm0, %ymm3
-; CHECK-NEXT:    vxorpd %xmm0, %xmm0, %xmm0
-; CHECK-NEXT:    vcmpeqpd %xmm0, %xmm1, %k1
-; CHECK-NEXT:    vmovapd %xmm3, %xmm0 {%k1} {z}
+; CHECK-NEXT:    vextractf128 $1, %ymm2, %xmm2
+; CHECK-NEXT:    vxorpd %xmm3, %xmm3, %xmm3
+; CHECK-NEXT:    vcmpeqpd %xmm3, %xmm1, %k1
+; CHECK-NEXT:    vunpcklpd {{.*#+}} xmm0 {%k1} {z} = xmm0[0],xmm2[0]
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retq
   %shuf = shufflevector <8 x double> %vec, <8 x double> undef, <2 x i32> <i32 0, i32 6>
@@ -4382,11 +4365,8 @@ define <4 x double> @test_masked_z_8xdouble_to_4xdouble_perm_mem_mask7(<8 x doub
 define <2 x double> @test_8xdouble_to_2xdouble_perm_mem_mask0(<8 x double>* %vp) {
 ; CHECK-LABEL: test_8xdouble_to_2xdouble_perm_mem_mask0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmovapd (%rdi), %ymm1
-; CHECK-NEXT:    vmovapd {{.*#+}} ymm0 = [1,6,3,6]
-; CHECK-NEXT:    vpermi2pd 32(%rdi), %ymm1, %ymm0
-; CHECK-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
-; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    vmovapd (%rdi), %xmm0
+; CHECK-NEXT:    vshufpd {{.*#+}} xmm0 = xmm0[1],mem[0]
 ; CHECK-NEXT:    retq
   %vec = load <8 x double>, <8 x double>* %vp
   %res = shufflevector <8 x double> %vec, <8 x double> undef, <2 x i32> <i32 1, i32 6>
@@ -4395,13 +4375,10 @@ define <2 x double> @test_8xdouble_to_2xdouble_perm_mem_mask0(<8 x double>* %vp)
 define <2 x double> @test_masked_8xdouble_to_2xdouble_perm_mem_mask0(<8 x double>* %vp, <2 x double> %vec2, <2 x double> %mask) {
 ; CHECK-LABEL: test_masked_8xdouble_to_2xdouble_perm_mem_mask0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmovapd (%rdi), %ymm2
-; CHECK-NEXT:    vmovapd {{.*#+}} ymm3 = [1,6,3,6]
-; CHECK-NEXT:    vpermi2pd 32(%rdi), %ymm2, %ymm3
-; CHECK-NEXT:    vxorpd %xmm2, %xmm2, %xmm2
-; CHECK-NEXT:    vcmpeqpd %xmm2, %xmm1, %k1
-; CHECK-NEXT:    vmovapd %xmm3, %xmm0 {%k1}
-; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    vmovapd (%rdi), %xmm2
+; CHECK-NEXT:    vxorpd %xmm3, %xmm3, %xmm3
+; CHECK-NEXT:    vcmpeqpd %xmm3, %xmm1, %k1
+; CHECK-NEXT:    vshufpd {{.*#+}} xmm0 {%k1} = xmm2[1],mem[0]
 ; CHECK-NEXT:    retq
   %vec = load <8 x double>, <8 x double>* %vp
   %shuf = shufflevector <8 x double> %vec, <8 x double> undef, <2 x i32> <i32 1, i32 6>
@@ -4413,13 +4390,10 @@ define <2 x double> @test_masked_8xdouble_to_2xdouble_perm_mem_mask0(<8 x double
 define <2 x double> @test_masked_z_8xdouble_to_2xdouble_perm_mem_mask0(<8 x double>* %vp, <2 x double> %mask) {
 ; CHECK-LABEL: test_masked_z_8xdouble_to_2xdouble_perm_mem_mask0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmovapd (%rdi), %ymm1
-; CHECK-NEXT:    vmovapd {{.*#+}} ymm2 = [1,6,3,6]
-; CHECK-NEXT:    vpermi2pd 32(%rdi), %ymm1, %ymm2
-; CHECK-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
-; CHECK-NEXT:    vcmpeqpd %xmm1, %xmm0, %k1
-; CHECK-NEXT:    vmovapd %xmm2, %xmm0 {%k1} {z}
-; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    vmovapd (%rdi), %xmm1
+; CHECK-NEXT:    vxorpd %xmm2, %xmm2, %xmm2
+; CHECK-NEXT:    vcmpeqpd %xmm2, %xmm0, %k1
+; CHECK-NEXT:    vshufpd {{.*#+}} xmm0 {%k1} {z} = xmm1[1],mem[0]
 ; CHECK-NEXT:    retq
   %vec = load <8 x double>, <8 x double>* %vp
   %shuf = shufflevector <8 x double> %vec, <8 x double> undef, <2 x i32> <i32 1, i32 6>
diff --git a/test/CodeGen/X86/avx512bw-intrinsics-canonical.ll b/test/CodeGen/X86/avx512bw-intrinsics-canonical.ll
index ba67a6200959f934e2915301d5615f6b549d3155..7ae02e99165d388597ac87ac3399c1311263618f 100644
--- a/test/CodeGen/X86/avx512bw-intrinsics-canonical.ll
+++ b/test/CodeGen/X86/avx512bw-intrinsics-canonical.ll
@@ -2,8 +2,292 @@
 ; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=knl -mattr=+avx512bw | FileCheck %s --check-prefix=ALL --check-prefix=AVX512BW
 ; RUN: llc < %s -mtriple=i386-unknown-linux-gnu -mcpu=knl -mattr=+avx512bw | FileCheck %s --check-prefix=ALL --check-prefix=AVX512F-32
 
-; NOTE: This should use IR equivalent to what is generated by clang/test/CodeGen/sse2-builtins.c
+; NOTE: This should use IR equivalent to what is generated by clang/test/CodeGen/avx512bw-builtins.c
 
+;
+; Signed Saturation
+;
+
+define <32 x i16> @test_mask_adds_epi16_rr_512(<32 x i16> %a, <32 x i16> %b) {
+; AVX512BW-LABEL: test_mask_adds_epi16_rr_512:
+; AVX512BW:       ## %bb.0:
+; AVX512BW-NEXT:    vpaddsw %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    retq
+;
+; AVX512F-32-LABEL: test_mask_adds_epi16_rr_512:
+; AVX512F-32:       # %bb.0:
+; AVX512F-32-NEXT:    vpaddsw %zmm1, %zmm0, %zmm0
+; AVX512F-32-NEXT:    retl
+  %res = call <32 x i16> @llvm.sadd.sat.v32i16(<32 x i16> %a, <32 x i16> %b)
+  ret <32 x i16> %res
+}
+declare <32 x i16> @llvm.sadd.sat.v32i16(<32 x i16>, <32 x i16>)
+
+define <32 x i16> @test_mask_adds_epi16_rrk_512(<32 x i16> %a, <32 x i16> %b, <32 x i16> %passThru, i32 %mask) {
+; AVX512BW-LABEL: test_mask_adds_epi16_rrk_512:
+; AVX512BW:       ## %bb.0:
+; AVX512BW-NEXT:    kmovd %edi, %k1
+; AVX512BW-NEXT:    vpaddsw %zmm1, %zmm0, %zmm2 {%k1}
+; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm0
+; AVX512BW-NEXT:    retq
+;
+; AVX512F-32-LABEL: test_mask_adds_epi16_rrk_512:
+; AVX512F-32:       # %bb.0:
+; AVX512F-32-NEXT:    kmovd {{[0-9]+}}(%esp), %k1
+; AVX512F-32-NEXT:    vpaddsw %zmm1, %zmm0, %zmm2 {%k1}
+; AVX512F-32-NEXT:    vmovdqa64 %zmm2, %zmm0
+; AVX512F-32-NEXT:    retl
+  %1 = call <32 x i16> @llvm.sadd.sat.v32i16(<32 x i16> %a, <32 x i16> %b)
+  %2 = bitcast i32 %mask to <32 x i1>
+  %3 = select <32 x i1> %2, <32 x i16> %1, <32 x i16> %passThru
+  ret <32 x i16> %3
+}
+
+define <32 x i16> @test_mask_adds_epi16_rrkz_512(<32 x i16> %a, <32 x i16> %b, i32 %mask) {
+; AVX512BW-LABEL: test_mask_adds_epi16_rrkz_512:
+; AVX512BW:       ## %bb.0:
+; AVX512BW-NEXT:    kmovd %edi, %k1
+; AVX512BW-NEXT:    vpaddsw %zmm1, %zmm0, %zmm0 {%k1} {z}
+; AVX512BW-NEXT:    retq
+;
+; AVX512F-32-LABEL: test_mask_adds_epi16_rrkz_512:
+; AVX512F-32:       # %bb.0:
+; AVX512F-32-NEXT:    kmovd {{[0-9]+}}(%esp), %k1
+; AVX512F-32-NEXT:    vpaddsw %zmm1, %zmm0, %zmm0 {%k1} {z}
+; AVX512F-32-NEXT:    retl
+  %1 = call <32 x i16> @llvm.sadd.sat.v32i16(<32 x i16> %a, <32 x i16> %b)
+  %2 = bitcast i32 %mask to <32 x i1>
+  %3 = select <32 x i1> %2, <32 x i16> %1, <32 x i16> zeroinitializer
+  ret <32 x i16> %3
+}
+
+define <32 x i16> @test_mask_adds_epi16_rm_512(<32 x i16> %a, <32 x i16>* %ptr_b) {
+; AVX512BW-LABEL: test_mask_adds_epi16_rm_512:
+; AVX512BW:       ## %bb.0:
+; AVX512BW-NEXT:    vpaddsw (%rdi), %zmm0, %zmm0
+; AVX512BW-NEXT:    retq
+;
+; AVX512F-32-LABEL: test_mask_adds_epi16_rm_512:
+; AVX512F-32:       # %bb.0:
+; AVX512F-32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; AVX512F-32-NEXT:    vpaddsw (%eax), %zmm0, %zmm0
+; AVX512F-32-NEXT:    retl
+  %b = load <32 x i16>, <32 x i16>* %ptr_b
+  %1 = call <32 x i16> @llvm.sadd.sat.v32i16(<32 x i16> %a, <32 x i16> %b)
+  ret <32 x i16> %1
+}
+
+define <32 x i16> @test_mask_adds_epi16_rmk_512(<32 x i16> %a, <32 x i16>* %ptr_b, <32 x i16> %passThru, i32 %mask) {
+; AVX512BW-LABEL: test_mask_adds_epi16_rmk_512:
+; AVX512BW:       ## %bb.0:
+; AVX512BW-NEXT:    kmovd %esi, %k1
+; AVX512BW-NEXT:    vpaddsw (%rdi), %zmm0, %zmm1 {%k1}
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm0
+; AVX512BW-NEXT:    retq
+;
+; AVX512F-32-LABEL: test_mask_adds_epi16_rmk_512:
+; AVX512F-32:       # %bb.0:
+; AVX512F-32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; AVX512F-32-NEXT:    kmovd {{[0-9]+}}(%esp), %k1
+; AVX512F-32-NEXT:    vpaddsw (%eax), %zmm0, %zmm1 {%k1}
+; AVX512F-32-NEXT:    vmovdqa64 %zmm1, %zmm0
+; AVX512F-32-NEXT:    retl
+  %b = load <32 x i16>, <32 x i16>* %ptr_b
+  %1 = call <32 x i16> @llvm.sadd.sat.v32i16(<32 x i16> %a, <32 x i16> %b)
+  %2 = bitcast i32 %mask to <32 x i1>
+  %3 = select <32 x i1> %2, <32 x i16> %1, <32 x i16> %passThru
+  ret <32 x i16> %3
+}
+
+define <32 x i16> @test_mask_adds_epi16_rmkz_512(<32 x i16> %a, <32 x i16>* %ptr_b, i32 %mask) {
+; AVX512BW-LABEL: test_mask_adds_epi16_rmkz_512:
+; AVX512BW:       ## %bb.0:
+; AVX512BW-NEXT:    kmovd %esi, %k1
+; AVX512BW-NEXT:    vpaddsw (%rdi), %zmm0, %zmm0 {%k1} {z}
+; AVX512BW-NEXT:    retq
+;
+; AVX512F-32-LABEL: test_mask_adds_epi16_rmkz_512:
+; AVX512F-32:       # %bb.0:
+; AVX512F-32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; AVX512F-32-NEXT:    kmovd {{[0-9]+}}(%esp), %k1
+; AVX512F-32-NEXT:    vpaddsw (%eax), %zmm0, %zmm0 {%k1} {z}
+; AVX512F-32-NEXT:    retl
+  %b = load <32 x i16>, <32 x i16>* %ptr_b
+  %1 = call <32 x i16> @llvm.sadd.sat.v32i16(<32 x i16> %a, <32 x i16> %b)
+  %2 = bitcast i32 %mask to <32 x i1>
+  %3 = select <32 x i1> %2, <32 x i16> %1, <32 x i16> zeroinitializer
+  ret <32 x i16> %3
+}
+
+define <32 x i16> @test_mask_subs_epi16_rr_512(<32 x i16> %a, <32 x i16> %b) {
+; AVX512BW-LABEL: test_mask_subs_epi16_rr_512:
+; AVX512BW:       ## %bb.0:
+; AVX512BW-NEXT:    vpsubsw %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    retq
+;
+; AVX512F-32-LABEL: test_mask_subs_epi16_rr_512:
+; AVX512F-32:       # %bb.0:
+; AVX512F-32-NEXT:    vpsubsw %zmm1, %zmm0, %zmm0
+; AVX512F-32-NEXT:    retl
+  %sub = call <32 x i16> @llvm.ssub.sat.v32i16(<32 x i16> %a, <32 x i16> %b)
+  ret <32 x i16> %sub
+}
+declare <32 x i16> @llvm.ssub.sat.v32i16(<32 x i16>, <32 x i16>)
+
+define <32 x i16> @test_mask_subs_epi16_rrk_512(<32 x i16> %a, <32 x i16> %b, <32 x i16> %passThru, i32 %mask) {
+; AVX512BW-LABEL: test_mask_subs_epi16_rrk_512:
+; AVX512BW:       ## %bb.0:
+; AVX512BW-NEXT:    kmovd %edi, %k1
+; AVX512BW-NEXT:    vpsubsw %zmm1, %zmm0, %zmm2 {%k1}
+; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm0
+; AVX512BW-NEXT:    retq
+;
+; AVX512F-32-LABEL: test_mask_subs_epi16_rrk_512:
+; AVX512F-32:       # %bb.0:
+; AVX512F-32-NEXT:    kmovd {{[0-9]+}}(%esp), %k1
+; AVX512F-32-NEXT:    vpsubsw %zmm1, %zmm0, %zmm2 {%k1}
+; AVX512F-32-NEXT:    vmovdqa64 %zmm2, %zmm0
+; AVX512F-32-NEXT:    retl
+  %sub = call <32 x i16> @llvm.ssub.sat.v32i16(<32 x i16> %a, <32 x i16> %b)
+  %bc = bitcast i32 %mask to <32 x i1>
+  %res = select <32 x i1> %bc, <32 x i16> %sub, <32 x i16> %passThru
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_mask_subs_epi16_rrkz_512(<32 x i16> %a, <32 x i16> %b, i32 %mask) {
+; AVX512BW-LABEL: test_mask_subs_epi16_rrkz_512:
+; AVX512BW:       ## %bb.0:
+; AVX512BW-NEXT:    kmovd %edi, %k1
+; AVX512BW-NEXT:    vpsubsw %zmm1, %zmm0, %zmm0 {%k1} {z}
+; AVX512BW-NEXT:    retq
+;
+; AVX512F-32-LABEL: test_mask_subs_epi16_rrkz_512:
+; AVX512F-32:       # %bb.0:
+; AVX512F-32-NEXT:    kmovd {{[0-9]+}}(%esp), %k1
+; AVX512F-32-NEXT:    vpsubsw %zmm1, %zmm0, %zmm0 {%k1} {z}
+; AVX512F-32-NEXT:    retl
+  %sub = call <32 x i16> @llvm.ssub.sat.v32i16(<32 x i16> %a, <32 x i16> %b)
+  %bc = bitcast i32 %mask to <32 x i1>
+  %res = select <32 x i1> %bc, <32 x i16> %sub, <32 x i16> zeroinitializer
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_mask_subs_epi16_rm_512(<32 x i16> %a, <32 x i16>* %ptr_b) {
+; AVX512BW-LABEL: test_mask_subs_epi16_rm_512:
+; AVX512BW:       ## %bb.0:
+; AVX512BW-NEXT:    vpsubsw (%rdi), %zmm0, %zmm0
+; AVX512BW-NEXT:    retq
+;
+; AVX512F-32-LABEL: test_mask_subs_epi16_rm_512:
+; AVX512F-32:       # %bb.0:
+; AVX512F-32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; AVX512F-32-NEXT:    vpsubsw (%eax), %zmm0, %zmm0
+; AVX512F-32-NEXT:    retl
+  %b = load <32 x i16>, <32 x i16>* %ptr_b
+  %sub = call <32 x i16> @llvm.ssub.sat.v32i16(<32 x i16> %a, <32 x i16> %b)
+  ret <32 x i16> %sub
+}
+
+define <32 x i16> @test_mask_subs_epi16_rmk_512(<32 x i16> %a, <32 x i16>* %ptr_b, <32 x i16> %passThru, i32 %mask) {
+; AVX512BW-LABEL: test_mask_subs_epi16_rmk_512:
+; AVX512BW:       ## %bb.0:
+; AVX512BW-NEXT:    kmovd %esi, %k1
+; AVX512BW-NEXT:    vpsubsw (%rdi), %zmm0, %zmm1 {%k1}
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm0
+; AVX512BW-NEXT:    retq
+;
+; AVX512F-32-LABEL: test_mask_subs_epi16_rmk_512:
+; AVX512F-32:       # %bb.0:
+; AVX512F-32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; AVX512F-32-NEXT:    kmovd {{[0-9]+}}(%esp), %k1
+; AVX512F-32-NEXT:    vpsubsw (%eax), %zmm0, %zmm1 {%k1}
+; AVX512F-32-NEXT:    vmovdqa64 %zmm1, %zmm0
+; AVX512F-32-NEXT:    retl
+  %b = load <32 x i16>, <32 x i16>* %ptr_b
+  %sub = call <32 x i16> @llvm.ssub.sat.v32i16(<32 x i16> %a, <32 x i16> %b)
+  %bc = bitcast i32 %mask to <32 x i1>
+  %res = select <32 x i1> %bc, <32 x i16> %sub, <32 x i16> %passThru
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_mask_subs_epi16_rmkz_512(<32 x i16> %a, <32 x i16>* %ptr_b, i32 %mask) {
+; AVX512BW-LABEL: test_mask_subs_epi16_rmkz_512:
+; AVX512BW:       ## %bb.0:
+; AVX512BW-NEXT:    kmovd %esi, %k1
+; AVX512BW-NEXT:    vpsubsw (%rdi), %zmm0, %zmm0 {%k1} {z}
+; AVX512BW-NEXT:    retq
+;
+; AVX512F-32-LABEL: test_mask_subs_epi16_rmkz_512:
+; AVX512F-32:       # %bb.0:
+; AVX512F-32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; AVX512F-32-NEXT:    kmovd {{[0-9]+}}(%esp), %k1
+; AVX512F-32-NEXT:    vpsubsw (%eax), %zmm0, %zmm0 {%k1} {z}
+; AVX512F-32-NEXT:    retl
+  %b = load <32 x i16>, <32 x i16>* %ptr_b
+  %sub = call <32 x i16> @llvm.ssub.sat.v32i16(<32 x i16> %a, <32 x i16> %b)
+  %bc = bitcast i32 %mask to <32 x i1>
+  %res = select <32 x i1> %bc, <32 x i16> %sub, <32 x i16> zeroinitializer
+  ret <32 x i16> %res
+}
+
+
+define <64 x i16> @test_mask_adds_epi16_rr_1024(<64 x i16> %a, <64 x i16> %b) {
+; AVX512BW-LABEL: test_mask_adds_epi16_rr_1024:
+; AVX512BW:       ## %bb.0:
+; AVX512BW-NEXT:    vpaddsw %zmm2, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpaddsw %zmm3, %zmm1, %zmm1
+; AVX512BW-NEXT:    retq
+;
+; AVX512F-32-LABEL: test_mask_adds_epi16_rr_1024:
+; AVX512F-32:       # %bb.0:
+; AVX512F-32-NEXT:    pushl %ebp
+; AVX512F-32-NEXT:    .cfi_def_cfa_offset 8
+; AVX512F-32-NEXT:    .cfi_offset %ebp, -8
+; AVX512F-32-NEXT:    movl %esp, %ebp
+; AVX512F-32-NEXT:    .cfi_def_cfa_register %ebp
+; AVX512F-32-NEXT:    andl $-64, %esp
+; AVX512F-32-NEXT:    subl $64, %esp
+; AVX512F-32-NEXT:    vpaddsw %zmm2, %zmm0, %zmm0
+; AVX512F-32-NEXT:    vpaddsw 8(%ebp), %zmm1, %zmm1
+; AVX512F-32-NEXT:    movl %ebp, %esp
+; AVX512F-32-NEXT:    popl %ebp
+; AVX512F-32-NEXT:    .cfi_def_cfa %esp, 4
+; AVX512F-32-NEXT:    retl
+  %1 = call <64 x i16> @llvm.sadd.sat.v64i16(<64 x i16> %a, <64 x i16> %b)
+  ret <64 x i16> %1
+}
+declare <64 x i16> @llvm.sadd.sat.v64i16(<64 x i16>, <64 x i16>)
+
+define <64 x i16> @test_mask_subs_epi16_rr_1024(<64 x i16> %a, <64 x i16> %b) {
+; AVX512BW-LABEL: test_mask_subs_epi16_rr_1024:
+; AVX512BW:       ## %bb.0:
+; AVX512BW-NEXT:    vpsubsw %zmm2, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpsubsw %zmm3, %zmm1, %zmm1
+; AVX512BW-NEXT:    retq
+;
+; AVX512F-32-LABEL: test_mask_subs_epi16_rr_1024:
+; AVX512F-32:       # %bb.0:
+; AVX512F-32-NEXT:    pushl %ebp
+; AVX512F-32-NEXT:    .cfi_def_cfa_offset 8
+; AVX512F-32-NEXT:    .cfi_offset %ebp, -8
+; AVX512F-32-NEXT:    movl %esp, %ebp
+; AVX512F-32-NEXT:    .cfi_def_cfa_register %ebp
+; AVX512F-32-NEXT:    andl $-64, %esp
+; AVX512F-32-NEXT:    subl $64, %esp
+; AVX512F-32-NEXT:    vpsubsw %zmm2, %zmm0, %zmm0
+; AVX512F-32-NEXT:    vpsubsw 8(%ebp), %zmm1, %zmm1
+; AVX512F-32-NEXT:    movl %ebp, %esp
+; AVX512F-32-NEXT:    popl %ebp
+; AVX512F-32-NEXT:    .cfi_def_cfa %esp, 4
+; AVX512F-32-NEXT:    retl
+  %sub = call <64 x i16> @llvm.ssub.sat.v64i16(<64 x i16> %a, <64 x i16> %b)
+  ret <64 x i16> %sub
+}
+declare <64 x i16> @llvm.ssub.sat.v64i16(<64 x i16>, <64 x i16>);
+
+;
+; Unsigned Saturation
+;
 
 define <32 x i16> @test_mask_adds_epu16_rr_512(<32 x i16> %a, <32 x i16> %b) {
 ; AVX512BW-LABEL: test_mask_adds_epu16_rr_512:
@@ -15,11 +299,10 @@ define <32 x i16> @test_mask_adds_epu16_rr_512(<32 x i16> %a, <32 x i16> %b) {
 ; AVX512F-32:       # %bb.0:
 ; AVX512F-32-NEXT:    vpaddusw %zmm1, %zmm0, %zmm0
 ; AVX512F-32-NEXT:    retl
-  %1 = add <32 x i16> %a, %b
-  %2 = icmp ugt <32 x i16> %a, %1
-  %3 = select <32 x i1> %2, <32 x i16> <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>, <32 x i16> %1
-  ret <32 x i16> %3
+  %res = call <32 x i16> @llvm.uadd.sat.v32i16(<32 x i16> %a, <32 x i16> %b)
+  ret <32 x i16> %res
 }
+declare <32 x i16> @llvm.uadd.sat.v32i16(<32 x i16>, <32 x i16>)
 
 define <32 x i16> @test_mask_adds_epu16_rrk_512(<32 x i16> %a, <32 x i16> %b, <32 x i16> %passThru, i32 %mask) {
 ; AVX512BW-LABEL: test_mask_adds_epu16_rrk_512:
@@ -35,12 +318,10 @@ define <32 x i16> @test_mask_adds_epu16_rrk_512(<32 x i16> %a, <32 x i16> %b, <3
 ; AVX512F-32-NEXT:    vpaddusw %zmm1, %zmm0, %zmm2 {%k1}
 ; AVX512F-32-NEXT:    vmovdqa64 %zmm2, %zmm0
 ; AVX512F-32-NEXT:    retl
-  %1 = add <32 x i16> %a, %b
-  %2 = icmp ugt <32 x i16> %a, %1
-  %3 = select <32 x i1> %2, <32 x i16> <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>, <32 x i16> %1
-  %4 = bitcast i32 %mask to <32 x i1>
-  %5 = select <32 x i1> %4, <32 x i16> %3, <32 x i16> %passThru
-  ret <32 x i16> %5
+  %1 = call <32 x i16> @llvm.uadd.sat.v32i16(<32 x i16> %a, <32 x i16> %b)
+  %2 = bitcast i32 %mask to <32 x i1>
+  %3 = select <32 x i1> %2, <32 x i16> %1, <32 x i16> %passThru
+  ret <32 x i16> %3
 }
 
 define <32 x i16> @test_mask_adds_epu16_rrkz_512(<32 x i16> %a, <32 x i16> %b, i32 %mask) {
@@ -55,12 +336,10 @@ define <32 x i16> @test_mask_adds_epu16_rrkz_512(<32 x i16> %a, <32 x i16> %b, i
 ; AVX512F-32-NEXT:    kmovd {{[0-9]+}}(%esp), %k1
 ; AVX512F-32-NEXT:    vpaddusw %zmm1, %zmm0, %zmm0 {%k1} {z}
 ; AVX512F-32-NEXT:    retl
-  %1 = add <32 x i16> %a, %b
-  %2 = icmp ugt <32 x i16> %a, %1
-  %3 = select <32 x i1> %2, <32 x i16> <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>, <32 x i16> %1
-  %4 = bitcast i32 %mask to <32 x i1>
-  %5 = select <32 x i1> %4, <32 x i16> %3, <32 x i16> zeroinitializer
-  ret <32 x i16> %5
+  %1 = call <32 x i16> @llvm.uadd.sat.v32i16(<32 x i16> %a, <32 x i16> %b)
+  %2 = bitcast i32 %mask to <32 x i1>
+  %3 = select <32 x i1> %2, <32 x i16> %1, <32 x i16> zeroinitializer
+  ret <32 x i16> %3
 }
 
 define <32 x i16> @test_mask_adds_epu16_rm_512(<32 x i16> %a, <32 x i16>* %ptr_b) {
@@ -75,10 +354,8 @@ define <32 x i16> @test_mask_adds_epu16_rm_512(<32 x i16> %a, <32 x i16>* %ptr_b
 ; AVX512F-32-NEXT:    vpaddusw (%eax), %zmm0, %zmm0
 ; AVX512F-32-NEXT:    retl
   %b = load <32 x i16>, <32 x i16>* %ptr_b
-  %1 = add <32 x i16> %a, %b
-  %2 = icmp ugt <32 x i16> %a, %1
-  %3 = select <32 x i1> %2, <32 x i16> <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>, <32 x i16> %1
-  ret <32 x i16> %3
+  %1 = call <32 x i16> @llvm.uadd.sat.v32i16(<32 x i16> %a, <32 x i16> %b)
+  ret <32 x i16> %1
 }
 
 define <32 x i16> @test_mask_adds_epu16_rmk_512(<32 x i16> %a, <32 x i16>* %ptr_b, <32 x i16> %passThru, i32 %mask) {
@@ -97,12 +374,10 @@ define <32 x i16> @test_mask_adds_epu16_rmk_512(<32 x i16> %a, <32 x i16>* %ptr_
 ; AVX512F-32-NEXT:    vmovdqa64 %zmm1, %zmm0
 ; AVX512F-32-NEXT:    retl
   %b = load <32 x i16>, <32 x i16>* %ptr_b
-  %1 = add <32 x i16> %a, %b
-  %2 = icmp ugt <32 x i16> %a, %1
-  %3 = select <32 x i1> %2, <32 x i16> <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>, <32 x i16> %1
-  %4 = bitcast i32 %mask to <32 x i1>
-  %5 = select <32 x i1> %4, <32 x i16> %3, <32 x i16> %passThru
-  ret <32 x i16> %5
+  %1 = call <32 x i16> @llvm.uadd.sat.v32i16(<32 x i16> %a, <32 x i16> %b)
+  %2 = bitcast i32 %mask to <32 x i1>
+  %3 = select <32 x i1> %2, <32 x i16> %1, <32 x i16> %passThru
+  ret <32 x i16> %3
 }
 
 define <32 x i16> @test_mask_adds_epu16_rmkz_512(<32 x i16> %a, <32 x i16>* %ptr_b, i32 %mask) {
@@ -119,12 +394,10 @@ define <32 x i16> @test_mask_adds_epu16_rmkz_512(<32 x i16> %a, <32 x i16>* %ptr
 ; AVX512F-32-NEXT:    vpaddusw (%eax), %zmm0, %zmm0 {%k1} {z}
 ; AVX512F-32-NEXT:    retl
   %b = load <32 x i16>, <32 x i16>* %ptr_b
-  %1 = add <32 x i16> %a, %b
-  %2 = icmp ugt <32 x i16> %a, %1
-  %3 = select <32 x i1> %2, <32 x i16> <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>, <32 x i16> %1
-  %4 = bitcast i32 %mask to <32 x i1>
-  %5 = select <32 x i1> %4, <32 x i16> %3, <32 x i16> zeroinitializer
-  ret <32 x i16> %5
+  %1 = call <32 x i16> @llvm.uadd.sat.v32i16(<32 x i16> %a, <32 x i16> %b)
+  %2 = bitcast i32 %mask to <32 x i1>
+  %3 = select <32 x i1> %2, <32 x i16> %1, <32 x i16> zeroinitializer
+  ret <32 x i16> %3
 }
 
 define <32 x i16> @test_mask_subs_epu16_rr_512(<32 x i16> %a, <32 x i16> %b) {
@@ -137,11 +410,10 @@ define <32 x i16> @test_mask_subs_epu16_rr_512(<32 x i16> %a, <32 x i16> %b) {
 ; AVX512F-32:       # %bb.0:
 ; AVX512F-32-NEXT:    vpsubusw %zmm1, %zmm0, %zmm0
 ; AVX512F-32-NEXT:    retl
-  %cmp = icmp ugt <32 x i16> %a, %b
-  %sel = select <32 x i1> %cmp, <32 x i16> %a, <32 x i16> %b
-  %sub = sub <32 x i16> %sel, %b
+  %sub = call <32 x i16> @llvm.usub.sat.v32i16(<32 x i16> %a, <32 x i16> %b)
   ret <32 x i16> %sub
 }
+declare <32 x i16> @llvm.usub.sat.v32i16(<32 x i16>, <32 x i16>)
 
 define <32 x i16> @test_mask_subs_epu16_rrk_512(<32 x i16> %a, <32 x i16> %b, <32 x i16> %passThru, i32 %mask) {
 ; AVX512BW-LABEL: test_mask_subs_epu16_rrk_512:
@@ -157,9 +429,7 @@ define <32 x i16> @test_mask_subs_epu16_rrk_512(<32 x i16> %a, <32 x i16> %b, <3
 ; AVX512F-32-NEXT:    vpsubusw %zmm1, %zmm0, %zmm2 {%k1}
 ; AVX512F-32-NEXT:    vmovdqa64 %zmm2, %zmm0
 ; AVX512F-32-NEXT:    retl
-  %cmp = icmp ugt <32 x i16> %a, %b
-  %sel = select <32 x i1> %cmp, <32 x i16> %a, <32 x i16> %b
-  %sub = sub <32 x i16> %sel, %b
+  %sub = call <32 x i16> @llvm.usub.sat.v32i16(<32 x i16> %a, <32 x i16> %b)
   %bc = bitcast i32 %mask to <32 x i1>
   %res = select <32 x i1> %bc, <32 x i16> %sub, <32 x i16> %passThru
   ret <32 x i16> %res
@@ -177,9 +447,7 @@ define <32 x i16> @test_mask_subs_epu16_rrkz_512(<32 x i16> %a, <32 x i16> %b, i
 ; AVX512F-32-NEXT:    kmovd {{[0-9]+}}(%esp), %k1
 ; AVX512F-32-NEXT:    vpsubusw %zmm1, %zmm0, %zmm0 {%k1} {z}
 ; AVX512F-32-NEXT:    retl
-  %cmp = icmp ugt <32 x i16> %a, %b
-  %sel = select <32 x i1> %cmp, <32 x i16> %a, <32 x i16> %b
-  %sub = sub <32 x i16> %sel, %b
+  %sub = call <32 x i16> @llvm.usub.sat.v32i16(<32 x i16> %a, <32 x i16> %b)
   %bc = bitcast i32 %mask to <32 x i1>
   %res = select <32 x i1> %bc, <32 x i16> %sub, <32 x i16> zeroinitializer
   ret <32 x i16> %res
@@ -197,9 +465,7 @@ define <32 x i16> @test_mask_subs_epu16_rm_512(<32 x i16> %a, <32 x i16>* %ptr_b
 ; AVX512F-32-NEXT:    vpsubusw (%eax), %zmm0, %zmm0
 ; AVX512F-32-NEXT:    retl
   %b = load <32 x i16>, <32 x i16>* %ptr_b
-  %cmp = icmp ugt <32 x i16> %a, %b
-  %sel = select <32 x i1> %cmp, <32 x i16> %a, <32 x i16> %b
-  %sub = sub <32 x i16> %sel, %b
+  %sub = call <32 x i16> @llvm.usub.sat.v32i16(<32 x i16> %a, <32 x i16> %b)
   ret <32 x i16> %sub
 }
 
@@ -219,9 +485,7 @@ define <32 x i16> @test_mask_subs_epu16_rmk_512(<32 x i16> %a, <32 x i16>* %ptr_
 ; AVX512F-32-NEXT:    vmovdqa64 %zmm1, %zmm0
 ; AVX512F-32-NEXT:    retl
   %b = load <32 x i16>, <32 x i16>* %ptr_b
-  %cmp = icmp ugt <32 x i16> %a, %b
-  %sel = select <32 x i1> %cmp, <32 x i16> %a, <32 x i16> %b
-  %sub = sub <32 x i16> %sel, %b
+  %sub = call <32 x i16> @llvm.usub.sat.v32i16(<32 x i16> %a, <32 x i16> %b)
   %bc = bitcast i32 %mask to <32 x i1>
   %res = select <32 x i1> %bc, <32 x i16> %sub, <32 x i16> %passThru
   ret <32 x i16> %res
@@ -241,9 +505,7 @@ define <32 x i16> @test_mask_subs_epu16_rmkz_512(<32 x i16> %a, <32 x i16>* %ptr
 ; AVX512F-32-NEXT:    vpsubusw (%eax), %zmm0, %zmm0 {%k1} {z}
 ; AVX512F-32-NEXT:    retl
   %b = load <32 x i16>, <32 x i16>* %ptr_b
-  %cmp = icmp ugt <32 x i16> %a, %b
-  %sel = select <32 x i1> %cmp, <32 x i16> %a, <32 x i16> %b
-  %sub = sub <32 x i16> %sel, %b
+  %sub = call <32 x i16> @llvm.usub.sat.v32i16(<32 x i16> %a, <32 x i16> %b)
   %bc = bitcast i32 %mask to <32 x i1>
   %res = select <32 x i1> %bc, <32 x i16> %sub, <32 x i16> zeroinitializer
   ret <32 x i16> %res
@@ -272,11 +534,10 @@ define <64 x i16> @test_mask_adds_epu16_rr_1024(<64 x i16> %a, <64 x i16> %b) {
 ; AVX512F-32-NEXT:    popl %ebp
 ; AVX512F-32-NEXT:    .cfi_def_cfa %esp, 4
 ; AVX512F-32-NEXT:    retl
-  %1 = add <64 x i16> %a, %b
-  %2 = icmp ugt <64 x i16> %a, %1
-  %3 = select <64 x i1> %2, <64 x i16> <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>, <64 x i16> %1
-  ret <64 x i16> %3
+  %1 = call <64 x i16> @llvm.uadd.sat.v64i16(<64 x i16> %a, <64 x i16> %b)
+  ret <64 x i16> %1
 }
+declare <64 x i16> @llvm.uadd.sat.v64i16(<64 x i16>, <64 x i16>)
 
 define <64 x i16> @test_mask_subs_epu16_rr_1024(<64 x i16> %a, <64 x i16> %b) {
 ; AVX512BW-LABEL: test_mask_subs_epu16_rr_1024:
@@ -300,9 +561,7 @@ define <64 x i16> @test_mask_subs_epu16_rr_1024(<64 x i16> %a, <64 x i16> %b) {
 ; AVX512F-32-NEXT:    popl %ebp
 ; AVX512F-32-NEXT:    .cfi_def_cfa %esp, 4
 ; AVX512F-32-NEXT:    retl
-  %cmp = icmp ugt <64 x i16> %a, %b
-  %sel = select <64 x i1> %cmp, <64 x i16> %a, <64 x i16> %b
-  %sub = sub <64 x i16> %sel, %b
+  %sub = call <64 x i16> @llvm.usub.sat.v64i16(<64 x i16> %a, <64 x i16> %b)
   ret <64 x i16> %sub
 }
-
+declare <64 x i16> @llvm.usub.sat.v64i16(<64 x i16>, <64 x i16>)
diff --git a/test/CodeGen/X86/avx512bw-intrinsics-upgrade.ll b/test/CodeGen/X86/avx512bw-intrinsics-upgrade.ll
index c623d8bbf2dee18cd4210a6bc73a430da6efc92d..beea392fc17300f4269f3bb34f1550eb8f7e1a4d 100644
--- a/test/CodeGen/X86/avx512bw-intrinsics-upgrade.ll
+++ b/test/CodeGen/X86/avx512bw-intrinsics-upgrade.ll
@@ -3285,6 +3285,113 @@ define <64 x i8> @test_mask_subs_epu8_rmkz_512(<64 x i8> %a, <64 x i8>* %ptr_b,
 
 declare <64 x i8> @llvm.x86.avx512.mask.psubus.b.512(<64 x i8>, <64 x i8>, <64 x i8>, i64)
 
+define <32 x i16> @test_adds_epi16_rr_512(<32 x i16> %a, <32 x i16> %b) {
+; CHECK-LABEL: test_adds_epi16_rr_512:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpaddsw %zmm1, %zmm0, %zmm0 # encoding: [0x62,0xf1,0x7d,0x48,0xed,0xc1]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %1 = call <32 x i16> @llvm.x86.avx512.padds.w.512(<32 x i16> %a, <32 x i16> %b)
+  ret <32 x i16> %1
+}
+
+define <32 x i16> @test_adds_epi16_rrk_512(<32 x i16> %a, <32 x i16> %b, <32 x i16> %passThru, i32 %mask) {
+; X86-LABEL: test_adds_epi16_rrk_512:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf9,0x90,0x4c,0x24,0x04]
+; X86-NEXT:    vpaddsw %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf1,0x7d,0x49,0xed,0xd1]
+; X86-NEXT:    vmovdqa64 %zmm2, %zmm0 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc2]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_adds_epi16_rrk_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; X64-NEXT:    vpaddsw %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf1,0x7d,0x49,0xed,0xd1]
+; X64-NEXT:    vmovdqa64 %zmm2, %zmm0 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc2]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %1 = call <32 x i16> @llvm.x86.avx512.padds.w.512(<32 x i16> %a, <32 x i16> %b)
+  %2 = bitcast i32 %mask to <32 x i1>
+  %3 = select <32 x i1> %2, <32 x i16> %1, <32 x i16> %passThru
+  ret <32 x i16> %3
+}
+
+define <32 x i16> @test_adds_epi16_rrkz_512(<32 x i16> %a, <32 x i16> %b, i32 %mask) {
+; X86-LABEL: test_adds_epi16_rrkz_512:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf9,0x90,0x4c,0x24,0x04]
+; X86-NEXT:    vpaddsw %zmm1, %zmm0, %zmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xc9,0xed,0xc1]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_adds_epi16_rrkz_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; X64-NEXT:    vpaddsw %zmm1, %zmm0, %zmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xc9,0xed,0xc1]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %1 = call <32 x i16> @llvm.x86.avx512.padds.w.512(<32 x i16> %a, <32 x i16> %b)
+  %2 = bitcast i32 %mask to <32 x i1>
+  %3 = select <32 x i1> %2, <32 x i16> %1, <32 x i16> zeroinitializer
+  ret <32 x i16> %3
+}
+
+define <32 x i16> @test_adds_epi16_rm_512(<32 x i16> %a, <32 x i16>* %ptr_b) {
+; X86-LABEL: test_adds_epi16_rm_512:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-NEXT:    vpaddsw (%eax), %zmm0, %zmm0 # encoding: [0x62,0xf1,0x7d,0x48,0xed,0x00]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_adds_epi16_rm_512:
+; X64:       # %bb.0:
+; X64-NEXT:    vpaddsw (%rdi), %zmm0, %zmm0 # encoding: [0x62,0xf1,0x7d,0x48,0xed,0x07]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %b = load <32 x i16>, <32 x i16>* %ptr_b
+  %1 = call <32 x i16> @llvm.x86.avx512.padds.w.512(<32 x i16> %a, <32 x i16> %b)
+  ret <32 x i16> %1
+}
+
+define <32 x i16> @test_adds_epi16_rmk_512(<32 x i16> %a, <32 x i16>* %ptr_b, <32 x i16> %passThru, i32 %mask) {
+; X86-LABEL: test_adds_epi16_rmk_512:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf9,0x90,0x4c,0x24,0x08]
+; X86-NEXT:    vpaddsw (%eax), %zmm0, %zmm1 {%k1} # encoding: [0x62,0xf1,0x7d,0x49,0xed,0x08]
+; X86-NEXT:    vmovdqa64 %zmm1, %zmm0 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc1]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_adds_epi16_rmk_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovd %esi, %k1 # encoding: [0xc5,0xfb,0x92,0xce]
+; X64-NEXT:    vpaddsw (%rdi), %zmm0, %zmm1 {%k1} # encoding: [0x62,0xf1,0x7d,0x49,0xed,0x0f]
+; X64-NEXT:    vmovdqa64 %zmm1, %zmm0 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc1]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %b = load <32 x i16>, <32 x i16>* %ptr_b
+  %1 = call <32 x i16> @llvm.x86.avx512.padds.w.512(<32 x i16> %a, <32 x i16> %b)
+  %2 = bitcast i32 %mask to <32 x i1>
+  %3 = select <32 x i1> %2, <32 x i16> %1, <32 x i16> %passThru
+  ret <32 x i16> %3
+}
+
+define <32 x i16> @test_adds_epi16_rmkz_512(<32 x i16> %a, <32 x i16>* %ptr_b, i32 %mask) {
+; X86-LABEL: test_adds_epi16_rmkz_512:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf9,0x90,0x4c,0x24,0x08]
+; X86-NEXT:    vpaddsw (%eax), %zmm0, %zmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xc9,0xed,0x00]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_adds_epi16_rmkz_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovd %esi, %k1 # encoding: [0xc5,0xfb,0x92,0xce]
+; X64-NEXT:    vpaddsw (%rdi), %zmm0, %zmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xc9,0xed,0x07]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %b = load <32 x i16>, <32 x i16>* %ptr_b
+  %1 = call <32 x i16> @llvm.x86.avx512.padds.w.512(<32 x i16> %a, <32 x i16> %b)
+  %2 = bitcast i32 %mask to <32 x i1>
+  %3 = select <32 x i1> %2, <32 x i16> %1, <32 x i16> zeroinitializer
+  ret <32 x i16> %3
+}
+
+declare <32 x i16> @llvm.x86.avx512.padds.w.512(<32 x i16>, <32 x i16>)
+
 define <32 x i16> @test_mask_adds_epi16_rr_512(<32 x i16> %a, <32 x i16> %b) {
 ; CHECK-LABEL: test_mask_adds_epi16_rr_512:
 ; CHECK:       # %bb.0:
@@ -3384,6 +3491,113 @@ define <32 x i16> @test_mask_adds_epi16_rmkz_512(<32 x i16> %a, <32 x i16>* %ptr
 
 declare <32 x i16> @llvm.x86.avx512.mask.padds.w.512(<32 x i16>, <32 x i16>, <32 x i16>, i32)
 
+define <32 x i16> @test_subs_epi16_rr_512(<32 x i16> %a, <32 x i16> %b) {
+; CHECK-LABEL: test_subs_epi16_rr_512:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpsubsw %zmm1, %zmm0, %zmm0 # encoding: [0x62,0xf1,0x7d,0x48,0xe9,0xc1]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %1 = call <32 x i16> @llvm.x86.avx512.psubs.w.512(<32 x i16> %a, <32 x i16> %b)
+  ret <32 x i16> %1
+}
+
+define <32 x i16> @test_subs_epi16_rrk_512(<32 x i16> %a, <32 x i16> %b, <32 x i16> %passThru, i32 %mask) {
+; X86-LABEL: test_subs_epi16_rrk_512:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf9,0x90,0x4c,0x24,0x04]
+; X86-NEXT:    vpsubsw %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf1,0x7d,0x49,0xe9,0xd1]
+; X86-NEXT:    vmovdqa64 %zmm2, %zmm0 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc2]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_subs_epi16_rrk_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; X64-NEXT:    vpsubsw %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf1,0x7d,0x49,0xe9,0xd1]
+; X64-NEXT:    vmovdqa64 %zmm2, %zmm0 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc2]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %1 = call <32 x i16> @llvm.x86.avx512.psubs.w.512(<32 x i16> %a, <32 x i16> %b)
+  %2 = bitcast i32 %mask to <32 x i1>
+  %3 = select <32 x i1> %2, <32 x i16> %1, <32 x i16> %passThru
+  ret <32 x i16> %3
+}
+
+define <32 x i16> @test_subs_epi16_rrkz_512(<32 x i16> %a, <32 x i16> %b, i32 %mask) {
+; X86-LABEL: test_subs_epi16_rrkz_512:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf9,0x90,0x4c,0x24,0x04]
+; X86-NEXT:    vpsubsw %zmm1, %zmm0, %zmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xc9,0xe9,0xc1]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_subs_epi16_rrkz_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; X64-NEXT:    vpsubsw %zmm1, %zmm0, %zmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xc9,0xe9,0xc1]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %1 = call <32 x i16> @llvm.x86.avx512.psubs.w.512(<32 x i16> %a, <32 x i16> %b)
+  %2 = bitcast i32 %mask to <32 x i1>
+  %3 = select <32 x i1> %2, <32 x i16> %1, <32 x i16> zeroinitializer
+  ret <32 x i16> %3
+}
+
+define <32 x i16> @test_subs_epi16_rm_512(<32 x i16> %a, <32 x i16>* %ptr_b) {
+; X86-LABEL: test_subs_epi16_rm_512:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-NEXT:    vpsubsw (%eax), %zmm0, %zmm0 # encoding: [0x62,0xf1,0x7d,0x48,0xe9,0x00]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_subs_epi16_rm_512:
+; X64:       # %bb.0:
+; X64-NEXT:    vpsubsw (%rdi), %zmm0, %zmm0 # encoding: [0x62,0xf1,0x7d,0x48,0xe9,0x07]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %b = load <32 x i16>, <32 x i16>* %ptr_b
+  %1 = call <32 x i16> @llvm.x86.avx512.psubs.w.512(<32 x i16> %a, <32 x i16> %b)
+  ret <32 x i16> %1
+}
+
+define <32 x i16> @test_subs_epi16_rmk_512(<32 x i16> %a, <32 x i16>* %ptr_b, <32 x i16> %passThru, i32 %mask) {
+; X86-LABEL: test_subs_epi16_rmk_512:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf9,0x90,0x4c,0x24,0x08]
+; X86-NEXT:    vpsubsw (%eax), %zmm0, %zmm1 {%k1} # encoding: [0x62,0xf1,0x7d,0x49,0xe9,0x08]
+; X86-NEXT:    vmovdqa64 %zmm1, %zmm0 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc1]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_subs_epi16_rmk_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovd %esi, %k1 # encoding: [0xc5,0xfb,0x92,0xce]
+; X64-NEXT:    vpsubsw (%rdi), %zmm0, %zmm1 {%k1} # encoding: [0x62,0xf1,0x7d,0x49,0xe9,0x0f]
+; X64-NEXT:    vmovdqa64 %zmm1, %zmm0 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc1]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %b = load <32 x i16>, <32 x i16>* %ptr_b
+  %1 = call <32 x i16> @llvm.x86.avx512.psubs.w.512(<32 x i16> %a, <32 x i16> %b)
+  %2 = bitcast i32 %mask to <32 x i1>
+  %3 = select <32 x i1> %2, <32 x i16> %1, <32 x i16> %passThru
+  ret <32 x i16> %3
+}
+
+define <32 x i16> @test_subs_epi16_rmkz_512(<32 x i16> %a, <32 x i16>* %ptr_b, i32 %mask) {
+; X86-LABEL: test_subs_epi16_rmkz_512:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf9,0x90,0x4c,0x24,0x08]
+; X86-NEXT:    vpsubsw (%eax), %zmm0, %zmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xc9,0xe9,0x00]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_subs_epi16_rmkz_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovd %esi, %k1 # encoding: [0xc5,0xfb,0x92,0xce]
+; X64-NEXT:    vpsubsw (%rdi), %zmm0, %zmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xc9,0xe9,0x07]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %b = load <32 x i16>, <32 x i16>* %ptr_b
+  %1 = call <32 x i16> @llvm.x86.avx512.psubs.w.512(<32 x i16> %a, <32 x i16> %b)
+  %2 = bitcast i32 %mask to <32 x i1>
+  %3 = select <32 x i1> %2, <32 x i16> %1, <32 x i16> zeroinitializer
+  ret <32 x i16> %3
+}
+
+declare <32 x i16> @llvm.x86.avx512.psubs.w.512(<32 x i16>, <32 x i16>)
+
 define <32 x i16> @test_mask_subs_epi16_rr_512(<32 x i16> %a, <32 x i16> %b) {
 ; CHECK-LABEL: test_mask_subs_epi16_rr_512:
 ; CHECK:       # %bb.0:
@@ -3680,3 +3894,123 @@ define <64 x i8> @test_mask_subs_epi8_rmkz_512(<64 x i8> %a, <64 x i8>* %ptr_b,
 }
 
 declare <64 x i8> @llvm.x86.avx512.mask.psubs.b.512(<64 x i8>, <64 x i8>, <64 x i8>, i64)
+
+declare <32 x i16> @llvm.x86.avx512.mask.psrlv32hi(<32 x i16>, <32 x i16>, <32 x i16>, i32)
+
+define <32 x i16>@test_int_x86_avx512_mask_psrlv32hi(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3) {
+; X86-LABEL: test_int_x86_avx512_mask_psrlv32hi:
+; X86:       # %bb.0:
+; X86-NEXT:    vpsrlvw %zmm1, %zmm0, %zmm3 # encoding: [0x62,0xf2,0xfd,0x48,0x10,0xd9]
+; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf9,0x90,0x4c,0x24,0x04]
+; X86-NEXT:    vpsrlvw %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x49,0x10,0xd1]
+; X86-NEXT:    vpsrlvw %zmm1, %zmm0, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xc9,0x10,0xc1]
+; X86-NEXT:    vpaddw %zmm3, %zmm0, %zmm0 # encoding: [0x62,0xf1,0x7d,0x48,0xfd,0xc3]
+; X86-NEXT:    vpaddw %zmm0, %zmm2, %zmm0 # encoding: [0x62,0xf1,0x6d,0x48,0xfd,0xc0]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_mask_psrlv32hi:
+; X64:       # %bb.0:
+; X64-NEXT:    vpsrlvw %zmm1, %zmm0, %zmm3 # encoding: [0x62,0xf2,0xfd,0x48,0x10,0xd9]
+; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; X64-NEXT:    vpsrlvw %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x49,0x10,0xd1]
+; X64-NEXT:    vpsrlvw %zmm1, %zmm0, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xc9,0x10,0xc1]
+; X64-NEXT:    vpaddw %zmm3, %zmm0, %zmm0 # encoding: [0x62,0xf1,0x7d,0x48,0xfd,0xc3]
+; X64-NEXT:    vpaddw %zmm0, %zmm2, %zmm0 # encoding: [0x62,0xf1,0x6d,0x48,0xfd,0xc0]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <32 x i16> @llvm.x86.avx512.mask.psrlv32hi(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3)
+  %res1 = call <32 x i16> @llvm.x86.avx512.mask.psrlv32hi(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> zeroinitializer, i32 %x3)
+  %res2 = call <32 x i16> @llvm.x86.avx512.mask.psrlv32hi(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 -1)
+  %res3 = add <32 x i16> %res, %res1
+  %res4 = add <32 x i16> %res3, %res2
+  ret <32 x i16> %res4
+}
+
+declare <32 x i16> @llvm.x86.avx512.mask.psrav32.hi(<32 x i16>, <32 x i16>, <32 x i16>, i32)
+
+define <32 x i16>@test_int_x86_avx512_mask_psrav32_hi(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3) {
+; X86-LABEL: test_int_x86_avx512_mask_psrav32_hi:
+; X86:       # %bb.0:
+; X86-NEXT:    vpsravw %zmm1, %zmm0, %zmm3 # encoding: [0x62,0xf2,0xfd,0x48,0x11,0xd9]
+; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf9,0x90,0x4c,0x24,0x04]
+; X86-NEXT:    vpsravw %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x49,0x11,0xd1]
+; X86-NEXT:    vpsravw %zmm1, %zmm0, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xc9,0x11,0xc1]
+; X86-NEXT:    vpaddw %zmm3, %zmm0, %zmm0 # encoding: [0x62,0xf1,0x7d,0x48,0xfd,0xc3]
+; X86-NEXT:    vpaddw %zmm0, %zmm2, %zmm0 # encoding: [0x62,0xf1,0x6d,0x48,0xfd,0xc0]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_mask_psrav32_hi:
+; X64:       # %bb.0:
+; X64-NEXT:    vpsravw %zmm1, %zmm0, %zmm3 # encoding: [0x62,0xf2,0xfd,0x48,0x11,0xd9]
+; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; X64-NEXT:    vpsravw %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x49,0x11,0xd1]
+; X64-NEXT:    vpsravw %zmm1, %zmm0, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xc9,0x11,0xc1]
+; X64-NEXT:    vpaddw %zmm3, %zmm0, %zmm0 # encoding: [0x62,0xf1,0x7d,0x48,0xfd,0xc3]
+; X64-NEXT:    vpaddw %zmm0, %zmm2, %zmm0 # encoding: [0x62,0xf1,0x6d,0x48,0xfd,0xc0]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <32 x i16> @llvm.x86.avx512.mask.psrav32.hi(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3)
+  %res1 = call <32 x i16> @llvm.x86.avx512.mask.psrav32.hi(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> zeroinitializer, i32 %x3)
+  %res2 = call <32 x i16> @llvm.x86.avx512.mask.psrav32.hi(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 -1)
+  %res3 = add <32 x i16> %res, %res1
+  %res4 = add <32 x i16> %res3, %res2
+  ret <32 x i16> %res4
+}
+
+declare <32 x i16> @llvm.x86.avx512.mask.psllv32hi(<32 x i16>, <32 x i16>, <32 x i16>, i32)
+
+define <32 x i16>@test_int_x86_avx512_mask_psllv32hi(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3) {
+; X86-LABEL: test_int_x86_avx512_mask_psllv32hi:
+; X86:       # %bb.0:
+; X86-NEXT:    vpsllvw %zmm1, %zmm0, %zmm3 # encoding: [0x62,0xf2,0xfd,0x48,0x12,0xd9]
+; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf9,0x90,0x4c,0x24,0x04]
+; X86-NEXT:    vpsllvw %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x49,0x12,0xd1]
+; X86-NEXT:    vpsllvw %zmm1, %zmm0, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xc9,0x12,0xc1]
+; X86-NEXT:    vpaddw %zmm3, %zmm0, %zmm0 # encoding: [0x62,0xf1,0x7d,0x48,0xfd,0xc3]
+; X86-NEXT:    vpaddw %zmm0, %zmm2, %zmm0 # encoding: [0x62,0xf1,0x6d,0x48,0xfd,0xc0]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_mask_psllv32hi:
+; X64:       # %bb.0:
+; X64-NEXT:    vpsllvw %zmm1, %zmm0, %zmm3 # encoding: [0x62,0xf2,0xfd,0x48,0x12,0xd9]
+; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; X64-NEXT:    vpsllvw %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x49,0x12,0xd1]
+; X64-NEXT:    vpsllvw %zmm1, %zmm0, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xc9,0x12,0xc1]
+; X64-NEXT:    vpaddw %zmm3, %zmm0, %zmm0 # encoding: [0x62,0xf1,0x7d,0x48,0xfd,0xc3]
+; X64-NEXT:    vpaddw %zmm0, %zmm2, %zmm0 # encoding: [0x62,0xf1,0x6d,0x48,0xfd,0xc0]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <32 x i16> @llvm.x86.avx512.mask.psllv32hi(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3)
+  %res1 = call <32 x i16> @llvm.x86.avx512.mask.psllv32hi(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> zeroinitializer, i32 %x3)
+  %res2 = call <32 x i16> @llvm.x86.avx512.mask.psllv32hi(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 -1)
+  %res3 = add <32 x i16> %res, %res1
+  %res4 = add <32 x i16> %res3, %res2
+  ret <32 x i16> %res4
+}
+
+declare <32 x i8> @llvm.x86.avx512.mask.pmov.wb.512(<32 x i16>, <32 x i8>, i32)
+
+define <32 x i8>@test_int_x86_avx512_mask_pmov_wb_512(<32 x i16> %x0, <32 x i8> %x1, i32 %x2) {
+; X86-LABEL: test_int_x86_avx512_mask_pmov_wb_512:
+; X86:       # %bb.0:
+; X86-NEXT:    vpmovwb %zmm0, %ymm2 # encoding: [0x62,0xf2,0x7e,0x48,0x30,0xc2]
+; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf9,0x90,0x4c,0x24,0x04]
+; X86-NEXT:    vpmovwb %zmm0, %ymm1 {%k1} # encoding: [0x62,0xf2,0x7e,0x49,0x30,0xc1]
+; X86-NEXT:    vpmovwb %zmm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0xc9,0x30,0xc0]
+; X86-NEXT:    vpaddb %ymm0, %ymm1, %ymm0 # encoding: [0xc5,0xf5,0xfc,0xc0]
+; X86-NEXT:    vpaddb %ymm0, %ymm2, %ymm0 # encoding: [0xc5,0xed,0xfc,0xc0]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_mask_pmov_wb_512:
+; X64:       # %bb.0:
+; X64-NEXT:    vpmovwb %zmm0, %ymm2 # encoding: [0x62,0xf2,0x7e,0x48,0x30,0xc2]
+; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; X64-NEXT:    vpmovwb %zmm0, %ymm1 {%k1} # encoding: [0x62,0xf2,0x7e,0x49,0x30,0xc1]
+; X64-NEXT:    vpmovwb %zmm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0xc9,0x30,0xc0]
+; X64-NEXT:    vpaddb %ymm0, %ymm1, %ymm0 # encoding: [0xc5,0xf5,0xfc,0xc0]
+; X64-NEXT:    vpaddb %ymm0, %ymm2, %ymm0 # encoding: [0xc5,0xed,0xfc,0xc0]
+; X64-NEXT:    retq # encoding: [0xc3]
+    %res0 = call <32 x i8> @llvm.x86.avx512.mask.pmov.wb.512(<32 x i16> %x0, <32 x i8> %x1, i32 -1)
+    %res1 = call <32 x i8> @llvm.x86.avx512.mask.pmov.wb.512(<32 x i16> %x0, <32 x i8> %x1, i32 %x2)
+    %res2 = call <32 x i8> @llvm.x86.avx512.mask.pmov.wb.512(<32 x i16> %x0, <32 x i8> zeroinitializer, i32 %x2)
+    %res3 = add <32 x i8> %res0, %res1
+    %res4 = add <32 x i8> %res3, %res2
+    ret <32 x i8> %res4
+}
diff --git a/test/CodeGen/X86/avx512bw-intrinsics.ll b/test/CodeGen/X86/avx512bw-intrinsics.ll
index cf52746c3a533ff69cf607147a9764c05df01abe..71cfd891bb188c8059e9affa479dcab015df6e8e 100644
--- a/test/CodeGen/X86/avx512bw-intrinsics.ll
+++ b/test/CodeGen/X86/avx512bw-intrinsics.ll
@@ -683,419 +683,6 @@ define <64 x i8> @test_mask_packus_epi16_rmkz_512(<32 x i16> %a, <32 x i16>* %pt
 
 declare <64 x i8> @llvm.x86.avx512.packuswb.512(<32 x i16>, <32 x i16>)
 
-define <32 x i16> @test_mask_adds_epi16_rr_512(<32 x i16> %a, <32 x i16> %b) {
-; CHECK-LABEL: test_mask_adds_epi16_rr_512:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vpaddsw %zmm1, %zmm0, %zmm0 # encoding: [0x62,0xf1,0x7d,0x48,0xed,0xc1]
-; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
-  %1 = call <32 x i16> @llvm.x86.avx512.padds.w.512(<32 x i16> %a, <32 x i16> %b)
-  ret <32 x i16> %1
-}
-
-define <32 x i16> @test_mask_adds_epi16_rrk_512(<32 x i16> %a, <32 x i16> %b, <32 x i16> %passThru, i32 %mask) {
-; X86-LABEL: test_mask_adds_epi16_rrk_512:
-; X86:       # %bb.0:
-; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf9,0x90,0x4c,0x24,0x04]
-; X86-NEXT:    vpaddsw %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf1,0x7d,0x49,0xed,0xd1]
-; X86-NEXT:    vmovdqa64 %zmm2, %zmm0 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc2]
-; X86-NEXT:    retl # encoding: [0xc3]
-;
-; X64-LABEL: test_mask_adds_epi16_rrk_512:
-; X64:       # %bb.0:
-; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
-; X64-NEXT:    vpaddsw %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf1,0x7d,0x49,0xed,0xd1]
-; X64-NEXT:    vmovdqa64 %zmm2, %zmm0 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc2]
-; X64-NEXT:    retq # encoding: [0xc3]
-  %1 = call <32 x i16> @llvm.x86.avx512.padds.w.512(<32 x i16> %a, <32 x i16> %b)
-  %2 = bitcast i32 %mask to <32 x i1>
-  %3 = select <32 x i1> %2, <32 x i16> %1, <32 x i16> %passThru
-  ret <32 x i16> %3
-}
-
-define <32 x i16> @test_mask_adds_epi16_rrkz_512(<32 x i16> %a, <32 x i16> %b, i32 %mask) {
-; X86-LABEL: test_mask_adds_epi16_rrkz_512:
-; X86:       # %bb.0:
-; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf9,0x90,0x4c,0x24,0x04]
-; X86-NEXT:    vpaddsw %zmm1, %zmm0, %zmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xc9,0xed,0xc1]
-; X86-NEXT:    retl # encoding: [0xc3]
-;
-; X64-LABEL: test_mask_adds_epi16_rrkz_512:
-; X64:       # %bb.0:
-; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
-; X64-NEXT:    vpaddsw %zmm1, %zmm0, %zmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xc9,0xed,0xc1]
-; X64-NEXT:    retq # encoding: [0xc3]
-  %1 = call <32 x i16> @llvm.x86.avx512.padds.w.512(<32 x i16> %a, <32 x i16> %b)
-  %2 = bitcast i32 %mask to <32 x i1>
-  %3 = select <32 x i1> %2, <32 x i16> %1, <32 x i16> zeroinitializer
-  ret <32 x i16> %3
-}
-
-define <32 x i16> @test_mask_adds_epi16_rm_512(<32 x i16> %a, <32 x i16>* %ptr_b) {
-; X86-LABEL: test_mask_adds_epi16_rm_512:
-; X86:       # %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
-; X86-NEXT:    vpaddsw (%eax), %zmm0, %zmm0 # encoding: [0x62,0xf1,0x7d,0x48,0xed,0x00]
-; X86-NEXT:    retl # encoding: [0xc3]
-;
-; X64-LABEL: test_mask_adds_epi16_rm_512:
-; X64:       # %bb.0:
-; X64-NEXT:    vpaddsw (%rdi), %zmm0, %zmm0 # encoding: [0x62,0xf1,0x7d,0x48,0xed,0x07]
-; X64-NEXT:    retq # encoding: [0xc3]
-  %b = load <32 x i16>, <32 x i16>* %ptr_b
-  %1 = call <32 x i16> @llvm.x86.avx512.padds.w.512(<32 x i16> %a, <32 x i16> %b)
-  ret <32 x i16> %1
-}
-
-define <32 x i16> @test_mask_adds_epi16_rmk_512(<32 x i16> %a, <32 x i16>* %ptr_b, <32 x i16> %passThru, i32 %mask) {
-; X86-LABEL: test_mask_adds_epi16_rmk_512:
-; X86:       # %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
-; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf9,0x90,0x4c,0x24,0x08]
-; X86-NEXT:    vpaddsw (%eax), %zmm0, %zmm1 {%k1} # encoding: [0x62,0xf1,0x7d,0x49,0xed,0x08]
-; X86-NEXT:    vmovdqa64 %zmm1, %zmm0 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc1]
-; X86-NEXT:    retl # encoding: [0xc3]
-;
-; X64-LABEL: test_mask_adds_epi16_rmk_512:
-; X64:       # %bb.0:
-; X64-NEXT:    kmovd %esi, %k1 # encoding: [0xc5,0xfb,0x92,0xce]
-; X64-NEXT:    vpaddsw (%rdi), %zmm0, %zmm1 {%k1} # encoding: [0x62,0xf1,0x7d,0x49,0xed,0x0f]
-; X64-NEXT:    vmovdqa64 %zmm1, %zmm0 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc1]
-; X64-NEXT:    retq # encoding: [0xc3]
-  %b = load <32 x i16>, <32 x i16>* %ptr_b
-  %1 = call <32 x i16> @llvm.x86.avx512.padds.w.512(<32 x i16> %a, <32 x i16> %b)
-  %2 = bitcast i32 %mask to <32 x i1>
-  %3 = select <32 x i1> %2, <32 x i16> %1, <32 x i16> %passThru
-  ret <32 x i16> %3
-}
-
-define <32 x i16> @test_mask_adds_epi16_rmkz_512(<32 x i16> %a, <32 x i16>* %ptr_b, i32 %mask) {
-; X86-LABEL: test_mask_adds_epi16_rmkz_512:
-; X86:       # %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
-; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf9,0x90,0x4c,0x24,0x08]
-; X86-NEXT:    vpaddsw (%eax), %zmm0, %zmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xc9,0xed,0x00]
-; X86-NEXT:    retl # encoding: [0xc3]
-;
-; X64-LABEL: test_mask_adds_epi16_rmkz_512:
-; X64:       # %bb.0:
-; X64-NEXT:    kmovd %esi, %k1 # encoding: [0xc5,0xfb,0x92,0xce]
-; X64-NEXT:    vpaddsw (%rdi), %zmm0, %zmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xc9,0xed,0x07]
-; X64-NEXT:    retq # encoding: [0xc3]
-  %b = load <32 x i16>, <32 x i16>* %ptr_b
-  %1 = call <32 x i16> @llvm.x86.avx512.padds.w.512(<32 x i16> %a, <32 x i16> %b)
-  %2 = bitcast i32 %mask to <32 x i1>
-  %3 = select <32 x i1> %2, <32 x i16> %1, <32 x i16> zeroinitializer
-  ret <32 x i16> %3
-}
-
-declare <32 x i16> @llvm.x86.avx512.padds.w.512(<32 x i16>, <32 x i16>)
-
-define <32 x i16> @test_mask_subs_epi16_rr_512(<32 x i16> %a, <32 x i16> %b) {
-; CHECK-LABEL: test_mask_subs_epi16_rr_512:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vpsubsw %zmm1, %zmm0, %zmm0 # encoding: [0x62,0xf1,0x7d,0x48,0xe9,0xc1]
-; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
-  %1 = call <32 x i16> @llvm.x86.avx512.psubs.w.512(<32 x i16> %a, <32 x i16> %b)
-  ret <32 x i16> %1
-}
-
-define <32 x i16> @test_mask_subs_epi16_rrk_512(<32 x i16> %a, <32 x i16> %b, <32 x i16> %passThru, i32 %mask) {
-; X86-LABEL: test_mask_subs_epi16_rrk_512:
-; X86:       # %bb.0:
-; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf9,0x90,0x4c,0x24,0x04]
-; X86-NEXT:    vpsubsw %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf1,0x7d,0x49,0xe9,0xd1]
-; X86-NEXT:    vmovdqa64 %zmm2, %zmm0 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc2]
-; X86-NEXT:    retl # encoding: [0xc3]
-;
-; X64-LABEL: test_mask_subs_epi16_rrk_512:
-; X64:       # %bb.0:
-; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
-; X64-NEXT:    vpsubsw %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf1,0x7d,0x49,0xe9,0xd1]
-; X64-NEXT:    vmovdqa64 %zmm2, %zmm0 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc2]
-; X64-NEXT:    retq # encoding: [0xc3]
-  %1 = call <32 x i16> @llvm.x86.avx512.psubs.w.512(<32 x i16> %a, <32 x i16> %b)
-  %2 = bitcast i32 %mask to <32 x i1>
-  %3 = select <32 x i1> %2, <32 x i16> %1, <32 x i16> %passThru
-  ret <32 x i16> %3
-}
-
-define <32 x i16> @test_mask_subs_epi16_rrkz_512(<32 x i16> %a, <32 x i16> %b, i32 %mask) {
-; X86-LABEL: test_mask_subs_epi16_rrkz_512:
-; X86:       # %bb.0:
-; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf9,0x90,0x4c,0x24,0x04]
-; X86-NEXT:    vpsubsw %zmm1, %zmm0, %zmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xc9,0xe9,0xc1]
-; X86-NEXT:    retl # encoding: [0xc3]
-;
-; X64-LABEL: test_mask_subs_epi16_rrkz_512:
-; X64:       # %bb.0:
-; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
-; X64-NEXT:    vpsubsw %zmm1, %zmm0, %zmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xc9,0xe9,0xc1]
-; X64-NEXT:    retq # encoding: [0xc3]
-  %1 = call <32 x i16> @llvm.x86.avx512.psubs.w.512(<32 x i16> %a, <32 x i16> %b)
-  %2 = bitcast i32 %mask to <32 x i1>
-  %3 = select <32 x i1> %2, <32 x i16> %1, <32 x i16> zeroinitializer
-  ret <32 x i16> %3
-}
-
-define <32 x i16> @test_mask_subs_epi16_rm_512(<32 x i16> %a, <32 x i16>* %ptr_b) {
-; X86-LABEL: test_mask_subs_epi16_rm_512:
-; X86:       # %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
-; X86-NEXT:    vpsubsw (%eax), %zmm0, %zmm0 # encoding: [0x62,0xf1,0x7d,0x48,0xe9,0x00]
-; X86-NEXT:    retl # encoding: [0xc3]
-;
-; X64-LABEL: test_mask_subs_epi16_rm_512:
-; X64:       # %bb.0:
-; X64-NEXT:    vpsubsw (%rdi), %zmm0, %zmm0 # encoding: [0x62,0xf1,0x7d,0x48,0xe9,0x07]
-; X64-NEXT:    retq # encoding: [0xc3]
-  %b = load <32 x i16>, <32 x i16>* %ptr_b
-  %1 = call <32 x i16> @llvm.x86.avx512.psubs.w.512(<32 x i16> %a, <32 x i16> %b)
-  ret <32 x i16> %1
-}
-
-define <32 x i16> @test_mask_subs_epi16_rmk_512(<32 x i16> %a, <32 x i16>* %ptr_b, <32 x i16> %passThru, i32 %mask) {
-; X86-LABEL: test_mask_subs_epi16_rmk_512:
-; X86:       # %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
-; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf9,0x90,0x4c,0x24,0x08]
-; X86-NEXT:    vpsubsw (%eax), %zmm0, %zmm1 {%k1} # encoding: [0x62,0xf1,0x7d,0x49,0xe9,0x08]
-; X86-NEXT:    vmovdqa64 %zmm1, %zmm0 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc1]
-; X86-NEXT:    retl # encoding: [0xc3]
-;
-; X64-LABEL: test_mask_subs_epi16_rmk_512:
-; X64:       # %bb.0:
-; X64-NEXT:    kmovd %esi, %k1 # encoding: [0xc5,0xfb,0x92,0xce]
-; X64-NEXT:    vpsubsw (%rdi), %zmm0, %zmm1 {%k1} # encoding: [0x62,0xf1,0x7d,0x49,0xe9,0x0f]
-; X64-NEXT:    vmovdqa64 %zmm1, %zmm0 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc1]
-; X64-NEXT:    retq # encoding: [0xc3]
-  %b = load <32 x i16>, <32 x i16>* %ptr_b
-  %1 = call <32 x i16> @llvm.x86.avx512.psubs.w.512(<32 x i16> %a, <32 x i16> %b)
-  %2 = bitcast i32 %mask to <32 x i1>
-  %3 = select <32 x i1> %2, <32 x i16> %1, <32 x i16> %passThru
-  ret <32 x i16> %3
-}
-
-define <32 x i16> @test_mask_subs_epi16_rmkz_512(<32 x i16> %a, <32 x i16>* %ptr_b, i32 %mask) {
-; X86-LABEL: test_mask_subs_epi16_rmkz_512:
-; X86:       # %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
-; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf9,0x90,0x4c,0x24,0x08]
-; X86-NEXT:    vpsubsw (%eax), %zmm0, %zmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xc9,0xe9,0x00]
-; X86-NEXT:    retl # encoding: [0xc3]
-;
-; X64-LABEL: test_mask_subs_epi16_rmkz_512:
-; X64:       # %bb.0:
-; X64-NEXT:    kmovd %esi, %k1 # encoding: [0xc5,0xfb,0x92,0xce]
-; X64-NEXT:    vpsubsw (%rdi), %zmm0, %zmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xc9,0xe9,0x07]
-; X64-NEXT:    retq # encoding: [0xc3]
-  %b = load <32 x i16>, <32 x i16>* %ptr_b
-  %1 = call <32 x i16> @llvm.x86.avx512.psubs.w.512(<32 x i16> %a, <32 x i16> %b)
-  %2 = bitcast i32 %mask to <32 x i1>
-  %3 = select <32 x i1> %2, <32 x i16> %1, <32 x i16> zeroinitializer
-  ret <32 x i16> %3
-}
-
-declare <32 x i16> @llvm.x86.avx512.psubs.w.512(<32 x i16>, <32 x i16>)
-
-define <64 x i8> @test_mask_adds_epi8_rr_512(<64 x i8> %a, <64 x i8> %b) {
-; CHECK-LABEL: test_mask_adds_epi8_rr_512:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vpaddsb %zmm1, %zmm0, %zmm0 # encoding: [0x62,0xf1,0x7d,0x48,0xec,0xc1]
-; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
-  %res = call <64 x i8> @llvm.x86.avx512.mask.padds.b.512(<64 x i8> %a, <64 x i8> %b, <64 x i8> zeroinitializer, i64 -1)
-  ret <64 x i8> %res
-}
-
-define <64 x i8> @test_mask_adds_epi8_rrk_512(<64 x i8> %a, <64 x i8> %b, <64 x i8> %passThru, i64 %mask) {
-; X86-LABEL: test_mask_adds_epi8_rrk_512:
-; X86:       # %bb.0:
-; X86-NEXT:    kmovq {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf8,0x90,0x4c,0x24,0x04]
-; X86-NEXT:    vpaddsb %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf1,0x7d,0x49,0xec,0xd1]
-; X86-NEXT:    vmovdqa64 %zmm2, %zmm0 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc2]
-; X86-NEXT:    retl # encoding: [0xc3]
-;
-; X64-LABEL: test_mask_adds_epi8_rrk_512:
-; X64:       # %bb.0:
-; X64-NEXT:    kmovq %rdi, %k1 # encoding: [0xc4,0xe1,0xfb,0x92,0xcf]
-; X64-NEXT:    vpaddsb %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf1,0x7d,0x49,0xec,0xd1]
-; X64-NEXT:    vmovdqa64 %zmm2, %zmm0 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc2]
-; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <64 x i8> @llvm.x86.avx512.mask.padds.b.512(<64 x i8> %a, <64 x i8> %b, <64 x i8> %passThru, i64 %mask)
-  ret <64 x i8> %res
-}
-
-define <64 x i8> @test_mask_adds_epi8_rrkz_512(<64 x i8> %a, <64 x i8> %b, i64 %mask) {
-; X86-LABEL: test_mask_adds_epi8_rrkz_512:
-; X86:       # %bb.0:
-; X86-NEXT:    kmovq {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf8,0x90,0x4c,0x24,0x04]
-; X86-NEXT:    vpaddsb %zmm1, %zmm0, %zmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xc9,0xec,0xc1]
-; X86-NEXT:    retl # encoding: [0xc3]
-;
-; X64-LABEL: test_mask_adds_epi8_rrkz_512:
-; X64:       # %bb.0:
-; X64-NEXT:    kmovq %rdi, %k1 # encoding: [0xc4,0xe1,0xfb,0x92,0xcf]
-; X64-NEXT:    vpaddsb %zmm1, %zmm0, %zmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xc9,0xec,0xc1]
-; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <64 x i8> @llvm.x86.avx512.mask.padds.b.512(<64 x i8> %a, <64 x i8> %b, <64 x i8> zeroinitializer, i64 %mask)
-  ret <64 x i8> %res
-}
-
-define <64 x i8> @test_mask_adds_epi8_rm_512(<64 x i8> %a, <64 x i8>* %ptr_b) {
-; X86-LABEL: test_mask_adds_epi8_rm_512:
-; X86:       # %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
-; X86-NEXT:    vpaddsb (%eax), %zmm0, %zmm0 # encoding: [0x62,0xf1,0x7d,0x48,0xec,0x00]
-; X86-NEXT:    retl # encoding: [0xc3]
-;
-; X64-LABEL: test_mask_adds_epi8_rm_512:
-; X64:       # %bb.0:
-; X64-NEXT:    vpaddsb (%rdi), %zmm0, %zmm0 # encoding: [0x62,0xf1,0x7d,0x48,0xec,0x07]
-; X64-NEXT:    retq # encoding: [0xc3]
-  %b = load <64 x i8>, <64 x i8>* %ptr_b
-  %res = call <64 x i8> @llvm.x86.avx512.mask.padds.b.512(<64 x i8> %a, <64 x i8> %b, <64 x i8> zeroinitializer, i64 -1)
-  ret <64 x i8> %res
-}
-
-define <64 x i8> @test_mask_adds_epi8_rmk_512(<64 x i8> %a, <64 x i8>* %ptr_b, <64 x i8> %passThru, i64 %mask) {
-; X86-LABEL: test_mask_adds_epi8_rmk_512:
-; X86:       # %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
-; X86-NEXT:    kmovq {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf8,0x90,0x4c,0x24,0x08]
-; X86-NEXT:    vpaddsb (%eax), %zmm0, %zmm1 {%k1} # encoding: [0x62,0xf1,0x7d,0x49,0xec,0x08]
-; X86-NEXT:    vmovdqa64 %zmm1, %zmm0 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc1]
-; X86-NEXT:    retl # encoding: [0xc3]
-;
-; X64-LABEL: test_mask_adds_epi8_rmk_512:
-; X64:       # %bb.0:
-; X64-NEXT:    kmovq %rsi, %k1 # encoding: [0xc4,0xe1,0xfb,0x92,0xce]
-; X64-NEXT:    vpaddsb (%rdi), %zmm0, %zmm1 {%k1} # encoding: [0x62,0xf1,0x7d,0x49,0xec,0x0f]
-; X64-NEXT:    vmovdqa64 %zmm1, %zmm0 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc1]
-; X64-NEXT:    retq # encoding: [0xc3]
-  %b = load <64 x i8>, <64 x i8>* %ptr_b
-  %res = call <64 x i8> @llvm.x86.avx512.mask.padds.b.512(<64 x i8> %a, <64 x i8> %b, <64 x i8> %passThru, i64 %mask)
-  ret <64 x i8> %res
-}
-
-define <64 x i8> @test_mask_adds_epi8_rmkz_512(<64 x i8> %a, <64 x i8>* %ptr_b, i64 %mask) {
-; X86-LABEL: test_mask_adds_epi8_rmkz_512:
-; X86:       # %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
-; X86-NEXT:    kmovq {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf8,0x90,0x4c,0x24,0x08]
-; X86-NEXT:    vpaddsb (%eax), %zmm0, %zmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xc9,0xec,0x00]
-; X86-NEXT:    retl # encoding: [0xc3]
-;
-; X64-LABEL: test_mask_adds_epi8_rmkz_512:
-; X64:       # %bb.0:
-; X64-NEXT:    kmovq %rsi, %k1 # encoding: [0xc4,0xe1,0xfb,0x92,0xce]
-; X64-NEXT:    vpaddsb (%rdi), %zmm0, %zmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xc9,0xec,0x07]
-; X64-NEXT:    retq # encoding: [0xc3]
-  %b = load <64 x i8>, <64 x i8>* %ptr_b
-  %res = call <64 x i8> @llvm.x86.avx512.mask.padds.b.512(<64 x i8> %a, <64 x i8> %b, <64 x i8> zeroinitializer, i64 %mask)
-  ret <64 x i8> %res
-}
-
-declare <64 x i8> @llvm.x86.avx512.mask.padds.b.512(<64 x i8>, <64 x i8>, <64 x i8>, i64)
-
-define <64 x i8> @test_mask_subs_epi8_rr_512(<64 x i8> %a, <64 x i8> %b) {
-; CHECK-LABEL: test_mask_subs_epi8_rr_512:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vpsubsb %zmm1, %zmm0, %zmm0 # encoding: [0x62,0xf1,0x7d,0x48,0xe8,0xc1]
-; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
-  %res = call <64 x i8> @llvm.x86.avx512.mask.psubs.b.512(<64 x i8> %a, <64 x i8> %b, <64 x i8> zeroinitializer, i64 -1)
-  ret <64 x i8> %res
-}
-
-define <64 x i8> @test_mask_subs_epi8_rrk_512(<64 x i8> %a, <64 x i8> %b, <64 x i8> %passThru, i64 %mask) {
-; X86-LABEL: test_mask_subs_epi8_rrk_512:
-; X86:       # %bb.0:
-; X86-NEXT:    kmovq {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf8,0x90,0x4c,0x24,0x04]
-; X86-NEXT:    vpsubsb %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf1,0x7d,0x49,0xe8,0xd1]
-; X86-NEXT:    vmovdqa64 %zmm2, %zmm0 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc2]
-; X86-NEXT:    retl # encoding: [0xc3]
-;
-; X64-LABEL: test_mask_subs_epi8_rrk_512:
-; X64:       # %bb.0:
-; X64-NEXT:    kmovq %rdi, %k1 # encoding: [0xc4,0xe1,0xfb,0x92,0xcf]
-; X64-NEXT:    vpsubsb %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf1,0x7d,0x49,0xe8,0xd1]
-; X64-NEXT:    vmovdqa64 %zmm2, %zmm0 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc2]
-; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <64 x i8> @llvm.x86.avx512.mask.psubs.b.512(<64 x i8> %a, <64 x i8> %b, <64 x i8> %passThru, i64 %mask)
-  ret <64 x i8> %res
-}
-
-define <64 x i8> @test_mask_subs_epi8_rrkz_512(<64 x i8> %a, <64 x i8> %b, i64 %mask) {
-; X86-LABEL: test_mask_subs_epi8_rrkz_512:
-; X86:       # %bb.0:
-; X86-NEXT:    kmovq {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf8,0x90,0x4c,0x24,0x04]
-; X86-NEXT:    vpsubsb %zmm1, %zmm0, %zmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xc9,0xe8,0xc1]
-; X86-NEXT:    retl # encoding: [0xc3]
-;
-; X64-LABEL: test_mask_subs_epi8_rrkz_512:
-; X64:       # %bb.0:
-; X64-NEXT:    kmovq %rdi, %k1 # encoding: [0xc4,0xe1,0xfb,0x92,0xcf]
-; X64-NEXT:    vpsubsb %zmm1, %zmm0, %zmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xc9,0xe8,0xc1]
-; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <64 x i8> @llvm.x86.avx512.mask.psubs.b.512(<64 x i8> %a, <64 x i8> %b, <64 x i8> zeroinitializer, i64 %mask)
-  ret <64 x i8> %res
-}
-
-define <64 x i8> @test_mask_subs_epi8_rm_512(<64 x i8> %a, <64 x i8>* %ptr_b) {
-; X86-LABEL: test_mask_subs_epi8_rm_512:
-; X86:       # %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
-; X86-NEXT:    vpsubsb (%eax), %zmm0, %zmm0 # encoding: [0x62,0xf1,0x7d,0x48,0xe8,0x00]
-; X86-NEXT:    retl # encoding: [0xc3]
-;
-; X64-LABEL: test_mask_subs_epi8_rm_512:
-; X64:       # %bb.0:
-; X64-NEXT:    vpsubsb (%rdi), %zmm0, %zmm0 # encoding: [0x62,0xf1,0x7d,0x48,0xe8,0x07]
-; X64-NEXT:    retq # encoding: [0xc3]
-  %b = load <64 x i8>, <64 x i8>* %ptr_b
-  %res = call <64 x i8> @llvm.x86.avx512.mask.psubs.b.512(<64 x i8> %a, <64 x i8> %b, <64 x i8> zeroinitializer, i64 -1)
-  ret <64 x i8> %res
-}
-
-define <64 x i8> @test_mask_subs_epi8_rmk_512(<64 x i8> %a, <64 x i8>* %ptr_b, <64 x i8> %passThru, i64 %mask) {
-; X86-LABEL: test_mask_subs_epi8_rmk_512:
-; X86:       # %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
-; X86-NEXT:    kmovq {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf8,0x90,0x4c,0x24,0x08]
-; X86-NEXT:    vpsubsb (%eax), %zmm0, %zmm1 {%k1} # encoding: [0x62,0xf1,0x7d,0x49,0xe8,0x08]
-; X86-NEXT:    vmovdqa64 %zmm1, %zmm0 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc1]
-; X86-NEXT:    retl # encoding: [0xc3]
-;
-; X64-LABEL: test_mask_subs_epi8_rmk_512:
-; X64:       # %bb.0:
-; X64-NEXT:    kmovq %rsi, %k1 # encoding: [0xc4,0xe1,0xfb,0x92,0xce]
-; X64-NEXT:    vpsubsb (%rdi), %zmm0, %zmm1 {%k1} # encoding: [0x62,0xf1,0x7d,0x49,0xe8,0x0f]
-; X64-NEXT:    vmovdqa64 %zmm1, %zmm0 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc1]
-; X64-NEXT:    retq # encoding: [0xc3]
-  %b = load <64 x i8>, <64 x i8>* %ptr_b
-  %res = call <64 x i8> @llvm.x86.avx512.mask.psubs.b.512(<64 x i8> %a, <64 x i8> %b, <64 x i8> %passThru, i64 %mask)
-  ret <64 x i8> %res
-}
-
-define <64 x i8> @test_mask_subs_epi8_rmkz_512(<64 x i8> %a, <64 x i8>* %ptr_b, i64 %mask) {
-; X86-LABEL: test_mask_subs_epi8_rmkz_512:
-; X86:       # %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
-; X86-NEXT:    kmovq {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf8,0x90,0x4c,0x24,0x08]
-; X86-NEXT:    vpsubsb (%eax), %zmm0, %zmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xc9,0xe8,0x00]
-; X86-NEXT:    retl # encoding: [0xc3]
-;
-; X64-LABEL: test_mask_subs_epi8_rmkz_512:
-; X64:       # %bb.0:
-; X64-NEXT:    kmovq %rsi, %k1 # encoding: [0xc4,0xe1,0xfb,0x92,0xce]
-; X64-NEXT:    vpsubsb (%rdi), %zmm0, %zmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xc9,0xe8,0x07]
-; X64-NEXT:    retq # encoding: [0xc3]
-  %b = load <64 x i8>, <64 x i8>* %ptr_b
-  %res = call <64 x i8> @llvm.x86.avx512.mask.psubs.b.512(<64 x i8> %a, <64 x i8> %b, <64 x i8> zeroinitializer, i64 %mask)
-  ret <64 x i8> %res
-}
-
-declare <64 x i8> @llvm.x86.avx512.mask.psubs.b.512(<64 x i8>, <64 x i8>, <64 x i8>, i64)
-
-
 define <32 x i16>@test_int_x86_avx512_mask_vpermt2var_hi_512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpermt2var_hi_512:
 ; X86:       # %bb.0:
@@ -1303,34 +890,36 @@ define <32 x i16> @test_int_x86_avx512_mask_pmulhr_sw_512(<32 x i16> %x0, <32 x
   ret <32 x i16> %res2
 }
 
-declare <32 x i8> @llvm.x86.avx512.mask.pmov.wb.512(<32 x i16>, <32 x i8>, i32)
-
 define <32 x i8>@test_int_x86_avx512_mask_pmov_wb_512(<32 x i16> %x0, <32 x i8> %x1, i32 %x2) {
 ; X86-LABEL: test_int_x86_avx512_mask_pmov_wb_512:
 ; X86:       # %bb.0:
+; X86-NEXT:    vpmovwb %zmm0, %ymm2 # encoding: [0x62,0xf2,0x7e,0x48,0x30,0xc2]
 ; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf9,0x90,0x4c,0x24,0x04]
 ; X86-NEXT:    vpmovwb %zmm0, %ymm1 {%k1} # encoding: [0x62,0xf2,0x7e,0x49,0x30,0xc1]
-; X86-NEXT:    vpmovwb %zmm0, %ymm2 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0xc9,0x30,0xc2]
-; X86-NEXT:    vpaddb %ymm2, %ymm1, %ymm1 # encoding: [0xc5,0xf5,0xfc,0xca]
-; X86-NEXT:    vpmovwb %zmm0, %ymm0 # encoding: [0x62,0xf2,0x7e,0x48,0x30,0xc0]
-; X86-NEXT:    vpaddb %ymm1, %ymm0, %ymm0 # encoding: [0xc5,0xfd,0xfc,0xc1]
+; X86-NEXT:    vpmovwb %zmm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0xc9,0x30,0xc0]
+; X86-NEXT:    vpaddb %ymm0, %ymm1, %ymm0 # encoding: [0xc5,0xf5,0xfc,0xc0]
+; X86-NEXT:    vpaddb %ymm0, %ymm2, %ymm0 # encoding: [0xc5,0xed,0xfc,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_pmov_wb_512:
 ; X64:       # %bb.0:
+; X64-NEXT:    vpmovwb %zmm0, %ymm2 # encoding: [0x62,0xf2,0x7e,0x48,0x30,0xc2]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpmovwb %zmm0, %ymm1 {%k1} # encoding: [0x62,0xf2,0x7e,0x49,0x30,0xc1]
-; X64-NEXT:    vpmovwb %zmm0, %ymm2 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0xc9,0x30,0xc2]
-; X64-NEXT:    vpaddb %ymm2, %ymm1, %ymm1 # encoding: [0xc5,0xf5,0xfc,0xca]
-; X64-NEXT:    vpmovwb %zmm0, %ymm0 # encoding: [0x62,0xf2,0x7e,0x48,0x30,0xc0]
-; X64-NEXT:    vpaddb %ymm1, %ymm0, %ymm0 # encoding: [0xc5,0xfd,0xfc,0xc1]
-; X64-NEXT:    retq # encoding: [0xc3]
-    %res0 = call <32 x i8> @llvm.x86.avx512.mask.pmov.wb.512(<32 x i16> %x0, <32 x i8> %x1, i32 -1)
-    %res1 = call <32 x i8> @llvm.x86.avx512.mask.pmov.wb.512(<32 x i16> %x0, <32 x i8> %x1, i32 %x2)
-    %res2 = call <32 x i8> @llvm.x86.avx512.mask.pmov.wb.512(<32 x i16> %x0, <32 x i8> zeroinitializer, i32 %x2)
-    %res3 = add <32 x i8> %res0, %res1
-    %res4 = add <32 x i8> %res3, %res2
-    ret <32 x i8> %res4
+; X64-NEXT:    vpmovwb %zmm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0xc9,0x30,0xc0]
+; X64-NEXT:    vpaddb %ymm0, %ymm1, %ymm0 # encoding: [0xc5,0xf5,0xfc,0xc0]
+; X64-NEXT:    vpaddb %ymm0, %ymm2, %ymm0 # encoding: [0xc5,0xed,0xfc,0xc0]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %1 = trunc <32 x i16> %x0 to <32 x i8>
+  %2 = trunc <32 x i16> %x0 to <32 x i8>
+  %3 = bitcast i32 %x2 to <32 x i1>
+  %4 = select <32 x i1> %3, <32 x i8> %2, <32 x i8> %x1
+  %5 = trunc <32 x i16> %x0 to <32 x i8>
+  %6 = bitcast i32 %x2 to <32 x i1>
+  %7 = select <32 x i1> %6, <32 x i8> %5, <32 x i8> zeroinitializer
+  %res3 = add <32 x i8> %1, %4
+  %res4 = add <32 x i8> %res3, %7
+  ret <32 x i8> %res4
 }
 
 declare void @llvm.x86.avx512.mask.pmov.wb.mem.512(i8* %ptr, <32 x i16>, i32)
@@ -1566,7 +1155,29 @@ define  <8 x i64>@test_int_x86_avx512_mask_psadb_w_512(<64 x i8> %x0, <64 x i8>
   ret  <8 x i64> %res2
 }
 
-declare <32 x i16> @llvm.x86.avx512.mask.psrlv32hi(<32 x i16>, <32 x i16>, <32 x i16>, i32)
+declare <32 x i16> @llvm.x86.avx512.psrlv.w.512(<32 x i16>, <32 x i16>) nounwind readnone
+
+define <32 x i16> @test_x86_avx512_psrlv_w_512_const() optsize {
+; X86-LABEL: test_x86_avx512_psrlv_w_512_const:
+; X86:       # %bb.0:
+; X86-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,65535]
+; X86-NEXT:    # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0x05,A,A,A,A]
+; X86-NEXT:    # fixup A - offset: 6, value: {{\.LCPI.*}}, kind: FK_Data_4
+; X86-NEXT:    vpsrlvw {{\.LCPI.*}}, %zmm0, %zmm0 # encoding: [0x62,0xf2,0xfd,0x48,0x10,0x05,A,A,A,A]
+; X86-NEXT:    # fixup A - offset: 6, value: {{\.LCPI.*}}, kind: FK_Data_4
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_x86_avx512_psrlv_w_512_const:
+; X64:       # %bb.0:
+; X64-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,65535]
+; X64-NEXT:    # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0x05,A,A,A,A]
+; X64-NEXT:    # fixup A - offset: 6, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-NEXT:    vpsrlvw {{.*}}(%rip), %zmm0, %zmm0 # encoding: [0x62,0xf2,0xfd,0x48,0x10,0x05,A,A,A,A]
+; X64-NEXT:    # fixup A - offset: 6, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res1 = call <32 x i16> @llvm.x86.avx512.psrlv.w.512(<32 x i16> <i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 -1>, <32 x i16> <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1,  i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 -1>)
+  ret <32 x i16> %res1
+}
 
 define <32 x i16>@test_int_x86_avx512_mask_psrlv32hi(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_psrlv32hi:
@@ -1588,15 +1199,19 @@ define <32 x i16>@test_int_x86_avx512_mask_psrlv32hi(<32 x i16> %x0, <32 x i16>
 ; X64-NEXT:    vpaddw %zmm3, %zmm0, %zmm0 # encoding: [0x62,0xf1,0x7d,0x48,0xfd,0xc3]
 ; X64-NEXT:    vpaddw %zmm0, %zmm2, %zmm0 # encoding: [0x62,0xf1,0x6d,0x48,0xfd,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <32 x i16> @llvm.x86.avx512.mask.psrlv32hi(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3)
-  %res1 = call <32 x i16> @llvm.x86.avx512.mask.psrlv32hi(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> zeroinitializer, i32 %x3)
-  %res2 = call <32 x i16> @llvm.x86.avx512.mask.psrlv32hi(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 -1)
-  %res3 = add <32 x i16> %res, %res1
-  %res4 = add <32 x i16> %res3, %res2
+  %1 = call <32 x i16> @llvm.x86.avx512.psrlv.w.512(<32 x i16> %x0, <32 x i16> %x1)
+  %2 = bitcast i32 %x3 to <32 x i1>
+  %3 = select <32 x i1> %2, <32 x i16> %1, <32 x i16> %x2
+  %4 = call <32 x i16> @llvm.x86.avx512.psrlv.w.512(<32 x i16> %x0, <32 x i16> %x1)
+  %5 = bitcast i32 %x3 to <32 x i1>
+  %6 = select <32 x i1> %5, <32 x i16> %4, <32 x i16> zeroinitializer
+  %7 = call <32 x i16> @llvm.x86.avx512.psrlv.w.512(<32 x i16> %x0, <32 x i16> %x1)
+  %res3 = add <32 x i16> %3, %6
+  %res4 = add <32 x i16> %res3, %7
   ret <32 x i16> %res4
 }
 
-declare <32 x i16> @llvm.x86.avx512.mask.psrav32.hi(<32 x i16>, <32 x i16>, <32 x i16>, i32)
+declare <32 x i16> @llvm.x86.avx512.psrav.w.512(<32 x i16>, <32 x i16>)
 
 define <32 x i16>@test_int_x86_avx512_mask_psrav32_hi(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_psrav32_hi:
@@ -1618,11 +1233,15 @@ define <32 x i16>@test_int_x86_avx512_mask_psrav32_hi(<32 x i16> %x0, <32 x i16>
 ; X64-NEXT:    vpaddw %zmm3, %zmm0, %zmm0 # encoding: [0x62,0xf1,0x7d,0x48,0xfd,0xc3]
 ; X64-NEXT:    vpaddw %zmm0, %zmm2, %zmm0 # encoding: [0x62,0xf1,0x6d,0x48,0xfd,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <32 x i16> @llvm.x86.avx512.mask.psrav32.hi(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3)
-  %res1 = call <32 x i16> @llvm.x86.avx512.mask.psrav32.hi(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> zeroinitializer, i32 %x3)
-  %res2 = call <32 x i16> @llvm.x86.avx512.mask.psrav32.hi(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 -1)
-  %res3 = add <32 x i16> %res, %res1
-  %res4 = add <32 x i16> %res3, %res2
+  %1 = call <32 x i16> @llvm.x86.avx512.psrav.w.512(<32 x i16> %x0, <32 x i16> %x1)
+  %2 = bitcast i32 %x3 to <32 x i1>
+  %3 = select <32 x i1> %2, <32 x i16> %1, <32 x i16> %x2
+  %4 = call <32 x i16> @llvm.x86.avx512.psrav.w.512(<32 x i16> %x0, <32 x i16> %x1)
+  %5 = bitcast i32 %x3 to <32 x i1>
+  %6 = select <32 x i1> %5, <32 x i16> %4, <32 x i16> zeroinitializer
+  %7 = call <32 x i16> @llvm.x86.avx512.psrav.w.512(<32 x i16> %x0, <32 x i16> %x1)
+  %res3 = add <32 x i16> %3, %6
+  %res4 = add <32 x i16> %res3, %7
   ret <32 x i16> %res4
 }
 
@@ -1644,14 +1263,10 @@ define <32 x i16>@test_int_x86_avx512_mask_psrav32_hi_const(<32 x i16> %x0, <32
 ; X64-NEXT:    vpsravw {{.*}}(%rip), %zmm0, %zmm0 # encoding: [0x62,0xf2,0xfd,0x48,0x11,0x05,A,A,A,A]
 ; X64-NEXT:    # fixup A - offset: 6, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <32 x i16> @llvm.x86.avx512.mask.psrav32.hi(<32 x i16> <i16 2, i16 9,  i16 -12, i16 23, i16 -26, i16 37, i16 -40, i16 51, i16 2, i16 9,  i16 -12, i16 23, i16 -26, i16 37, i16 -40, i16 51, i16 2, i16 9,  i16 -12, i16 23, i16 -26, i16 37, i16 -40, i16 51, i16 2, i16 9,  i16 -12, i16 23, i16 -26, i16 37, i16 -40, i16 51>,
-                                                          <32 x i16> <i16 1, i16 10, i16 35,  i16 52, i16 69,  i16 9,  i16 16,  i16 49, i16 1, i16 10, i16 35,  i16 52, i16 69,  i16 9,  i16 16,  i16 49, i16 1, i16 10, i16 35,  i16 52, i16 69,  i16 9,  i16 16,  i16 49, i16 1, i16 10, i16 35,  i16 52, i16 69,  i16 9,  i16 16,  i16 49>,
-                                                          <32 x i16> zeroinitializer, i32 -1)
-  ret <32 x i16> %res
+  %1 = call <32 x i16> @llvm.x86.avx512.psrav.w.512(<32 x i16> <i16 2, i16 9, i16 -12, i16 23, i16 -26, i16 37, i16 -40, i16 51, i16 2, i16 9, i16 -12, i16 23, i16 -26, i16 37, i16 -40, i16 51, i16 2, i16 9, i16 -12, i16 23, i16 -26, i16 37, i16 -40, i16 51, i16 2, i16 9, i16 -12, i16 23, i16 -26, i16 37, i16 -40, i16 51>, <32 x i16> <i16 1, i16 10, i16 35, i16 52, i16 69, i16 9, i16 16, i16 49, i16 1, i16 10, i16 35, i16 52, i16 69, i16 9, i16 16, i16 49, i16 1, i16 10, i16 35, i16 52, i16 69, i16 9, i16 16, i16 49, i16 1, i16 10, i16 35, i16 52, i16 69, i16 9, i16 16, i16 49>)
+  ret <32 x i16> %1
 }
 
-declare <32 x i16> @llvm.x86.avx512.mask.psllv32hi(<32 x i16>, <32 x i16>, <32 x i16>, i32)
-
 define <32 x i16>@test_int_x86_avx512_mask_psllv32hi(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_psllv32hi:
 ; X86:       # %bb.0:
@@ -1672,11 +1287,15 @@ define <32 x i16>@test_int_x86_avx512_mask_psllv32hi(<32 x i16> %x0, <32 x i16>
 ; X64-NEXT:    vpaddw %zmm3, %zmm0, %zmm0 # encoding: [0x62,0xf1,0x7d,0x48,0xfd,0xc3]
 ; X64-NEXT:    vpaddw %zmm0, %zmm2, %zmm0 # encoding: [0x62,0xf1,0x6d,0x48,0xfd,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <32 x i16> @llvm.x86.avx512.mask.psllv32hi(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3)
-  %res1 = call <32 x i16> @llvm.x86.avx512.mask.psllv32hi(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> zeroinitializer, i32 %x3)
-  %res2 = call <32 x i16> @llvm.x86.avx512.mask.psllv32hi(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 -1)
-  %res3 = add <32 x i16> %res, %res1
-  %res4 = add <32 x i16> %res3, %res2
+  %1 = call <32 x i16> @llvm.x86.avx512.psllv.w.512(<32 x i16> %x0, <32 x i16> %x1)
+  %2 = bitcast i32 %x3 to <32 x i1>
+  %3 = select <32 x i1> %2, <32 x i16> %1, <32 x i16> %x2
+  %4 = call <32 x i16> @llvm.x86.avx512.psllv.w.512(<32 x i16> %x0, <32 x i16> %x1)
+  %5 = bitcast i32 %x3 to <32 x i1>
+  %6 = select <32 x i1> %5, <32 x i16> %4, <32 x i16> zeroinitializer
+  %7 = call <32 x i16> @llvm.x86.avx512.psllv.w.512(<32 x i16> %x0, <32 x i16> %x1)
+  %res3 = add <32 x i16> %3, %6
+  %res4 = add <32 x i16> %res3, %7
   ret <32 x i16> %res4
 }
 
@@ -1761,6 +1380,29 @@ define <32 x i16> @test_x86_avx512_maskz_psll_w_512(<32 x i16> %a0, <8 x i16> %a
 declare <32 x i16> @llvm.x86.avx512.psll.w.512(<32 x i16>, <8 x i16>) nounwind readnone
 
 
+define <32 x i16> @test_x86_avx512_psllv_w_512_const() optsize {
+; X86-LABEL: test_x86_avx512_psllv_w_512_const:
+; X86:       # %bb.0:
+; X86-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,65535]
+; X86-NEXT:    # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0x05,A,A,A,A]
+; X86-NEXT:    # fixup A - offset: 6, value: {{\.LCPI.*}}, kind: FK_Data_4
+; X86-NEXT:    vpsllvw {{\.LCPI.*}}, %zmm0, %zmm0 # encoding: [0x62,0xf2,0xfd,0x48,0x12,0x05,A,A,A,A]
+; X86-NEXT:    # fixup A - offset: 6, value: {{\.LCPI.*}}, kind: FK_Data_4
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_x86_avx512_psllv_w_512_const:
+; X64:       # %bb.0:
+; X64-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,65535]
+; X64-NEXT:    # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0x05,A,A,A,A]
+; X64-NEXT:    # fixup A - offset: 6, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-NEXT:    vpsllvw {{.*}}(%rip), %zmm0, %zmm0 # encoding: [0x62,0xf2,0xfd,0x48,0x12,0x05,A,A,A,A]
+; X64-NEXT:    # fixup A - offset: 6, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res1 = call <32 x i16> @llvm.x86.avx512.psllv.w.512(<32 x i16> <i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4,  i16 4, i16 4, i16 4, i16 4, i16 4, i16 -1>, <32 x i16> <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1,  i16 1, i16 1, i16 -1>)
+  ret <32 x i16> %res1
+}
+declare <32 x i16> @llvm.x86.avx512.psllv.w.512(<32 x i16>, <32 x i16>) nounwind readnone
+
 define <32 x i16> @test_x86_avx512_pslli_w_512(<32 x i16> %a0) {
 ; CHECK-LABEL: test_x86_avx512_pslli_w_512:
 ; CHECK:       # %bb.0:
diff --git a/test/CodeGen/X86/avx512bwvl-intrinsics-canonical.ll b/test/CodeGen/X86/avx512bwvl-intrinsics-canonical.ll
index 748e29d390a1b824c66bd71e2ae329d3877d59ec..fc994069eec01047443568e21ce43c7125b8512e 100644
--- a/test/CodeGen/X86/avx512bwvl-intrinsics-canonical.ll
+++ b/test/CodeGen/X86/avx512bwvl-intrinsics-canonical.ll
@@ -1,19 +1,602 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=knl -mattr=+avx512bw -mattr=+avx512vl --show-mc-encoding| FileCheck %s
 
-; NOTE: This should use IR equivalent to what is generated by clang/test/CodeGen/sse2-builtins.c
+; NOTE: This should use IR equivalent to what is generated by clang/test/CodeGen/avx512vlbw-builtins.c
 
 
+;
+; Signed Saturation
+;
+
+define <8 x i16> @test_mask_adds_epi16_rr_128(<8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: test_mask_adds_epi16_rr_128:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    vpaddsw %xmm1, %xmm0, %xmm0 ## EVEX TO VEX Compression encoding: [0xc5,0xf9,0xed,0xc1]
+; CHECK-NEXT:    retq ## encoding: [0xc3]
+  %1 = call <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16> %a, <8 x i16> %b)
+  ret <8 x i16> %1
+}
+declare <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16>, <8 x i16>)
+
+define <8 x i16> @test_mask_adds_epi16_rrk_128(<8 x i16> %a, <8 x i16> %b, <8 x i16> %passThru, i8 %mask) {
+; CHECK-LABEL: test_mask_adds_epi16_rrk_128:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    kmovd %edi, %k1 ## encoding: [0xc5,0xfb,0x92,0xcf]
+; CHECK-NEXT:    vpaddsw %xmm1, %xmm0, %xmm2 {%k1} ## encoding: [0x62,0xf1,0x7d,0x09,0xed,0xd1]
+; CHECK-NEXT:    vmovdqa %xmm2, %xmm0 ## EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc2]
+; CHECK-NEXT:    retq ## encoding: [0xc3]
+  %1 = call <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16> %a, <8 x i16> %b)
+  %2 = bitcast i8 %mask to <8 x i1>
+  %3 = select <8 x i1> %2, <8 x i16> %1, <8 x i16> %passThru
+  ret <8 x i16> %3
+}
+
+define <8 x i16> @test_mask_adds_epi16_rrkz_128(<8 x i16> %a, <8 x i16> %b, i8 %mask) {
+; CHECK-LABEL: test_mask_adds_epi16_rrkz_128:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    kmovd %edi, %k1 ## encoding: [0xc5,0xfb,0x92,0xcf]
+; CHECK-NEXT:    vpaddsw %xmm1, %xmm0, %xmm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0x89,0xed,0xc1]
+; CHECK-NEXT:    retq ## encoding: [0xc3]
+  %1 = call <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16> %a, <8 x i16> %b)
+  %2 = bitcast i8 %mask to <8 x i1>
+  %3 = select <8 x i1> %2, <8 x i16> %1, <8 x i16> zeroinitializer
+  ret <8 x i16> %3
+}
+
+define <8 x i16> @test_mask_adds_epi16_rm_128(<8 x i16> %a, <8 x i16>* %ptr_b) {
+; CHECK-LABEL: test_mask_adds_epi16_rm_128:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    vpaddsw (%rdi), %xmm0, %xmm0 ## EVEX TO VEX Compression encoding: [0xc5,0xf9,0xed,0x07]
+; CHECK-NEXT:    retq ## encoding: [0xc3]
+  %b = load <8 x i16>, <8 x i16>* %ptr_b
+  %1 = call <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16> %a, <8 x i16> %b)
+  ret <8 x i16> %1
+}
+
+define <8 x i16> @test_mask_adds_epi16_rmk_128(<8 x i16> %a, <8 x i16>* %ptr_b, <8 x i16> %passThru, i8 %mask) {
+; CHECK-LABEL: test_mask_adds_epi16_rmk_128:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    kmovd %esi, %k1 ## encoding: [0xc5,0xfb,0x92,0xce]
+; CHECK-NEXT:    vpaddsw (%rdi), %xmm0, %xmm1 {%k1} ## encoding: [0x62,0xf1,0x7d,0x09,0xed,0x0f]
+; CHECK-NEXT:    vmovdqa %xmm1, %xmm0 ## EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc1]
+; CHECK-NEXT:    retq ## encoding: [0xc3]
+  %b = load <8 x i16>, <8 x i16>* %ptr_b
+  %1 = call <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16> %a, <8 x i16> %b)
+  %2 = bitcast i8 %mask to <8 x i1>
+  %3 = select <8 x i1> %2, <8 x i16> %1, <8 x i16> %passThru
+  ret <8 x i16> %3
+}
+
+define <8 x i16> @test_mask_adds_epi16_rmkz_128(<8 x i16> %a, <8 x i16>* %ptr_b, i8 %mask) {
+; CHECK-LABEL: test_mask_adds_epi16_rmkz_128:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    kmovd %esi, %k1 ## encoding: [0xc5,0xfb,0x92,0xce]
+; CHECK-NEXT:    vpaddsw (%rdi), %xmm0, %xmm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0x89,0xed,0x07]
+; CHECK-NEXT:    retq ## encoding: [0xc3]
+  %b = load <8 x i16>, <8 x i16>* %ptr_b
+  %1 = call <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16> %a, <8 x i16> %b)
+  %2 = bitcast i8 %mask to <8 x i1>
+  %3 = select <8 x i1> %2, <8 x i16> %1, <8 x i16> zeroinitializer
+  ret <8 x i16> %3
+}
+
+define <16 x i16> @test_mask_adds_epi16_rr_256(<16 x i16> %a, <16 x i16> %b) {
+; CHECK-LABEL: test_mask_adds_epi16_rr_256:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    vpaddsw %ymm1, %ymm0, %ymm0 ## EVEX TO VEX Compression encoding: [0xc5,0xfd,0xed,0xc1]
+; CHECK-NEXT:    retq ## encoding: [0xc3]
+  %1 = call <16 x i16> @llvm.sadd.sat.v16i16(<16 x i16> %a, <16 x i16> %b)
+  ret <16 x i16> %1
+}
+declare <16 x i16> @llvm.sadd.sat.v16i16(<16 x i16>, <16 x i16>)
+
+define <16 x i16> @test_mask_adds_epi16_rrk_256(<16 x i16> %a, <16 x i16> %b, <16 x i16> %passThru, i16 %mask) {
+; CHECK-LABEL: test_mask_adds_epi16_rrk_256:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    kmovd %edi, %k1 ## encoding: [0xc5,0xfb,0x92,0xcf]
+; CHECK-NEXT:    vpaddsw %ymm1, %ymm0, %ymm2 {%k1} ## encoding: [0x62,0xf1,0x7d,0x29,0xed,0xd1]
+; CHECK-NEXT:    vmovdqa %ymm2, %ymm0 ## EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc2]
+; CHECK-NEXT:    retq ## encoding: [0xc3]
+  %1 = call <16 x i16> @llvm.sadd.sat.v16i16(<16 x i16> %a, <16 x i16> %b)
+  %2 = bitcast i16 %mask to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x i16> %1, <16 x i16> %passThru
+  ret <16 x i16> %3
+}
+
+define <16 x i16> @test_mask_adds_epi16_rrkz_256(<16 x i16> %a, <16 x i16> %b, i16 %mask) {
+; CHECK-LABEL: test_mask_adds_epi16_rrkz_256:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    kmovd %edi, %k1 ## encoding: [0xc5,0xfb,0x92,0xcf]
+; CHECK-NEXT:    vpaddsw %ymm1, %ymm0, %ymm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0xa9,0xed,0xc1]
+; CHECK-NEXT:    retq ## encoding: [0xc3]
+  %1 = call <16 x i16> @llvm.sadd.sat.v16i16(<16 x i16> %a, <16 x i16> %b)
+  %2 = bitcast i16 %mask to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x i16> %1, <16 x i16> zeroinitializer
+  ret <16 x i16> %3
+}
+
+define <16 x i16> @test_mask_adds_epi16_rm_256(<16 x i16> %a, <16 x i16>* %ptr_b) {
+; CHECK-LABEL: test_mask_adds_epi16_rm_256:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    vpaddsw (%rdi), %ymm0, %ymm0 ## EVEX TO VEX Compression encoding: [0xc5,0xfd,0xed,0x07]
+; CHECK-NEXT:    retq ## encoding: [0xc3]
+  %b = load <16 x i16>, <16 x i16>* %ptr_b
+  %1 = call <16 x i16> @llvm.sadd.sat.v16i16(<16 x i16> %a, <16 x i16> %b)
+  ret <16 x i16> %1
+}
+
+define <16 x i16> @test_mask_adds_epi16_rmk_256(<16 x i16> %a, <16 x i16>* %ptr_b, <16 x i16> %passThru, i16 %mask) {
+; CHECK-LABEL: test_mask_adds_epi16_rmk_256:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    kmovd %esi, %k1 ## encoding: [0xc5,0xfb,0x92,0xce]
+; CHECK-NEXT:    vpaddsw (%rdi), %ymm0, %ymm1 {%k1} ## encoding: [0x62,0xf1,0x7d,0x29,0xed,0x0f]
+; CHECK-NEXT:    vmovdqa %ymm1, %ymm0 ## EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc1]
+; CHECK-NEXT:    retq ## encoding: [0xc3]
+  %b = load <16 x i16>, <16 x i16>* %ptr_b
+  %1 = call <16 x i16> @llvm.sadd.sat.v16i16(<16 x i16> %a, <16 x i16> %b)
+  %2 = bitcast i16 %mask to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x i16> %1, <16 x i16> %passThru
+  ret <16 x i16> %3
+}
+
+define <16 x i16> @test_mask_adds_epi16_rmkz_256(<16 x i16> %a, <16 x i16>* %ptr_b, i16 %mask) {
+; CHECK-LABEL: test_mask_adds_epi16_rmkz_256:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    kmovd %esi, %k1 ## encoding: [0xc5,0xfb,0x92,0xce]
+; CHECK-NEXT:    vpaddsw (%rdi), %ymm0, %ymm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0xa9,0xed,0x07]
+; CHECK-NEXT:    retq ## encoding: [0xc3]
+  %b = load <16 x i16>, <16 x i16>* %ptr_b
+  %1 = call <16 x i16> @llvm.sadd.sat.v16i16(<16 x i16> %a, <16 x i16> %b)
+  %2 = bitcast i16 %mask to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x i16> %1, <16 x i16> zeroinitializer
+  ret <16 x i16> %3
+}
+
+define <8 x i16> @test_mask_subs_epi16_rr_128(<8 x i16> %a, <8 x i16> %b) {
+; CHECK-LABEL: test_mask_subs_epi16_rr_128:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    vpsubsw %xmm1, %xmm0, %xmm0 ## EVEX TO VEX Compression encoding: [0xc5,0xf9,0xe9,0xc1]
+; CHECK-NEXT:    retq ## encoding: [0xc3]
+  %sub = call <8 x i16> @llvm.ssub.sat.v8i16(<8 x i16> %a, <8 x i16> %b)
+  ret <8 x i16> %sub
+}
+declare <8 x i16> @llvm.ssub.sat.v8i16(<8 x i16>, <8 x i16>)
+
+define <8 x i16> @test_mask_subs_epi16_rrk_128(<8 x i16> %a, <8 x i16> %b, <8 x i16> %passThru, i8 %mask) {
+; CHECK-LABEL: test_mask_subs_epi16_rrk_128:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    kmovd %edi, %k1 ## encoding: [0xc5,0xfb,0x92,0xcf]
+; CHECK-NEXT:    vpsubsw %xmm1, %xmm0, %xmm2 {%k1} ## encoding: [0x62,0xf1,0x7d,0x09,0xe9,0xd1]
+; CHECK-NEXT:    vmovdqa %xmm2, %xmm0 ## EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc2]
+; CHECK-NEXT:    retq ## encoding: [0xc3]
+  %sub = call <8 x i16> @llvm.ssub.sat.v8i16(<8 x i16> %a, <8 x i16> %b)
+  %bc = bitcast i8 %mask to <8 x i1>
+  %res = select <8 x i1> %bc, <8 x i16> %sub, <8 x i16> %passThru
+  ret <8 x i16> %res
+}
+
+define <8 x i16> @test_mask_subs_epi16_rrkz_128(<8 x i16> %a, <8 x i16> %b, i8 %mask) {
+; CHECK-LABEL: test_mask_subs_epi16_rrkz_128:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    kmovd %edi, %k1 ## encoding: [0xc5,0xfb,0x92,0xcf]
+; CHECK-NEXT:    vpsubsw %xmm1, %xmm0, %xmm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0x89,0xe9,0xc1]
+; CHECK-NEXT:    retq ## encoding: [0xc3]
+  %sub = call <8 x i16> @llvm.ssub.sat.v8i16(<8 x i16> %a, <8 x i16> %b)
+  %bc = bitcast i8 %mask to <8 x i1>
+  %res = select <8 x i1> %bc, <8 x i16> %sub, <8 x i16> zeroinitializer
+  ret <8 x i16> %res
+}
+
+define <8 x i16> @test_mask_subs_epi16_rm_128(<8 x i16> %a, <8 x i16>* %ptr_b) {
+; CHECK-LABEL: test_mask_subs_epi16_rm_128:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    vpsubsw (%rdi), %xmm0, %xmm0 ## EVEX TO VEX Compression encoding: [0xc5,0xf9,0xe9,0x07]
+; CHECK-NEXT:    retq ## encoding: [0xc3]
+  %b = load <8 x i16>, <8 x i16>* %ptr_b
+  %sub = call <8 x i16> @llvm.ssub.sat.v8i16(<8 x i16> %a, <8 x i16> %b)
+  ret <8 x i16> %sub
+}
+
+define <8 x i16> @test_mask_subs_epi16_rmk_128(<8 x i16> %a, <8 x i16>* %ptr_b, <8 x i16> %passThru, i8 %mask) {
+; CHECK-LABEL: test_mask_subs_epi16_rmk_128:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    kmovd %esi, %k1 ## encoding: [0xc5,0xfb,0x92,0xce]
+; CHECK-NEXT:    vpsubsw (%rdi), %xmm0, %xmm1 {%k1} ## encoding: [0x62,0xf1,0x7d,0x09,0xe9,0x0f]
+; CHECK-NEXT:    vmovdqa %xmm1, %xmm0 ## EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc1]
+; CHECK-NEXT:    retq ## encoding: [0xc3]
+  %b = load <8 x i16>, <8 x i16>* %ptr_b
+  %sub = call <8 x i16> @llvm.ssub.sat.v8i16(<8 x i16> %a, <8 x i16> %b)
+  %bc = bitcast i8 %mask to <8 x i1>
+  %res = select <8 x i1> %bc, <8 x i16> %sub, <8 x i16> %passThru
+  ret <8 x i16> %res
+}
+
+define <8 x i16> @test_mask_subs_epi16_rmkz_128(<8 x i16> %a, <8 x i16>* %ptr_b, i8 %mask) {
+; CHECK-LABEL: test_mask_subs_epi16_rmkz_128:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    kmovd %esi, %k1 ## encoding: [0xc5,0xfb,0x92,0xce]
+; CHECK-NEXT:    vpsubsw (%rdi), %xmm0, %xmm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0x89,0xe9,0x07]
+; CHECK-NEXT:    retq ## encoding: [0xc3]
+  %b = load <8 x i16>, <8 x i16>* %ptr_b
+  %sub = call <8 x i16> @llvm.ssub.sat.v8i16(<8 x i16> %a, <8 x i16> %b)
+  %bc = bitcast i8 %mask to <8 x i1>
+  %res = select <8 x i1> %bc, <8 x i16> %sub, <8 x i16> zeroinitializer
+  ret <8 x i16> %res
+}
+
+define <16 x i16> @test_mask_subs_epi16_rr_256(<16 x i16> %a, <16 x i16> %b) {
+; CHECK-LABEL: test_mask_subs_epi16_rr_256:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    vpsubsw %ymm1, %ymm0, %ymm0 ## EVEX TO VEX Compression encoding: [0xc5,0xfd,0xe9,0xc1]
+; CHECK-NEXT:    retq ## encoding: [0xc3]
+  %sub = call <16 x i16> @llvm.ssub.sat.v16i16(<16 x i16> %a, <16 x i16> %b)
+  ret <16 x i16> %sub
+}
+declare <16 x i16> @llvm.ssub.sat.v16i16(<16 x i16>, <16 x i16>)
+
+define <16 x i16> @test_mask_subs_epi16_rrk_256(<16 x i16> %a, <16 x i16> %b, <16 x i16> %passThru, i16 %mask) {
+; CHECK-LABEL: test_mask_subs_epi16_rrk_256:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    kmovd %edi, %k1 ## encoding: [0xc5,0xfb,0x92,0xcf]
+; CHECK-NEXT:    vpsubsw %ymm1, %ymm0, %ymm2 {%k1} ## encoding: [0x62,0xf1,0x7d,0x29,0xe9,0xd1]
+; CHECK-NEXT:    vmovdqa %ymm2, %ymm0 ## EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc2]
+; CHECK-NEXT:    retq ## encoding: [0xc3]
+  %sub = call <16 x i16> @llvm.ssub.sat.v16i16(<16 x i16> %a, <16 x i16> %b)
+  %bc = bitcast i16 %mask to <16 x i1>
+  %res = select <16 x i1> %bc, <16 x i16> %sub, <16 x i16> %passThru
+  ret <16 x i16> %res
+}
+
+define <16 x i16> @test_mask_subs_epi16_rrkz_256(<16 x i16> %a, <16 x i16> %b, i16 %mask) {
+; CHECK-LABEL: test_mask_subs_epi16_rrkz_256:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    kmovd %edi, %k1 ## encoding: [0xc5,0xfb,0x92,0xcf]
+; CHECK-NEXT:    vpsubsw %ymm1, %ymm0, %ymm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0xa9,0xe9,0xc1]
+; CHECK-NEXT:    retq ## encoding: [0xc3]
+  %sub = call <16 x i16> @llvm.ssub.sat.v16i16(<16 x i16> %a, <16 x i16> %b)
+  %bc = bitcast i16 %mask to <16 x i1>
+  %res = select <16 x i1> %bc, <16 x i16> %sub, <16 x i16> zeroinitializer
+  ret <16 x i16> %res
+}
+
+define <16 x i16> @test_mask_subs_epi16_rm_256(<16 x i16> %a, <16 x i16>* %ptr_b) {
+; CHECK-LABEL: test_mask_subs_epi16_rm_256:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    vpsubsw (%rdi), %ymm0, %ymm0 ## EVEX TO VEX Compression encoding: [0xc5,0xfd,0xe9,0x07]
+; CHECK-NEXT:    retq ## encoding: [0xc3]
+  %b = load <16 x i16>, <16 x i16>* %ptr_b
+  %sub = call <16 x i16> @llvm.ssub.sat.v16i16(<16 x i16> %a, <16 x i16> %b)
+  ret <16 x i16> %sub
+}
+
+define <16 x i16> @test_mask_subs_epi16_rmk_256(<16 x i16> %a, <16 x i16>* %ptr_b, <16 x i16> %passThru, i16 %mask) {
+; CHECK-LABEL: test_mask_subs_epi16_rmk_256:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    kmovd %esi, %k1 ## encoding: [0xc5,0xfb,0x92,0xce]
+; CHECK-NEXT:    vpsubsw (%rdi), %ymm0, %ymm1 {%k1} ## encoding: [0x62,0xf1,0x7d,0x29,0xe9,0x0f]
+; CHECK-NEXT:    vmovdqa %ymm1, %ymm0 ## EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc1]
+; CHECK-NEXT:    retq ## encoding: [0xc3]
+  %b = load <16 x i16>, <16 x i16>* %ptr_b
+  %sub = call <16 x i16> @llvm.ssub.sat.v16i16(<16 x i16> %a, <16 x i16> %b)
+  %bc = bitcast i16 %mask to <16 x i1>
+  %res = select <16 x i1> %bc, <16 x i16> %sub, <16 x i16> %passThru
+  ret <16 x i16> %res
+}
+
+define <16 x i16> @test_mask_subs_epi16_rmkz_256(<16 x i16> %a, <16 x i16>* %ptr_b, i16 %mask) {
+; CHECK-LABEL: test_mask_subs_epi16_rmkz_256:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    kmovd %esi, %k1 ## encoding: [0xc5,0xfb,0x92,0xce]
+; CHECK-NEXT:    vpsubsw (%rdi), %ymm0, %ymm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0xa9,0xe9,0x07]
+; CHECK-NEXT:    retq ## encoding: [0xc3]
+  %b = load <16 x i16>, <16 x i16>* %ptr_b
+  %sub = call <16 x i16> @llvm.ssub.sat.v16i16(<16 x i16> %a, <16 x i16> %b)
+  %bc = bitcast i16 %mask to <16 x i1>
+  %res = select <16 x i1> %bc, <16 x i16> %sub, <16 x i16> zeroinitializer
+  ret <16 x i16> %res
+}
+
+define <16 x i8> @test_mask_adds_epi8_rr_128(<16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: test_mask_adds_epi8_rr_128:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    vpaddsb %xmm1, %xmm0, %xmm0 ## EVEX TO VEX Compression encoding: [0xc5,0xf9,0xec,0xc1]
+; CHECK-NEXT:    retq ## encoding: [0xc3]
+  %1 = call <16 x i8> @llvm.sadd.sat.v16i8(<16 x i8> %a, <16 x i8> %b)
+  ret <16 x i8> %1
+}
+declare <16 x i8> @llvm.sadd.sat.v16i8(<16 x i8>, <16 x i8>)
+
+define <16 x i8> @test_mask_adds_epi8_rrk_128(<16 x i8> %a, <16 x i8> %b, <16 x i8> %passThru, i16 %mask) {
+; CHECK-LABEL: test_mask_adds_epi8_rrk_128:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    kmovd %edi, %k1 ## encoding: [0xc5,0xfb,0x92,0xcf]
+; CHECK-NEXT:    vpaddsb %xmm1, %xmm0, %xmm2 {%k1} ## encoding: [0x62,0xf1,0x7d,0x09,0xec,0xd1]
+; CHECK-NEXT:    vmovdqa %xmm2, %xmm0 ## EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc2]
+; CHECK-NEXT:    retq ## encoding: [0xc3]
+  %1 = call <16 x i8> @llvm.sadd.sat.v16i8(<16 x i8> %a, <16 x i8> %b)
+  %2 = bitcast i16 %mask to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x i8> %1, <16 x i8> %passThru
+  ret <16 x i8> %3
+}
+
+define <16 x i8> @test_mask_adds_epi8_rrkz_128(<16 x i8> %a, <16 x i8> %b, i16 %mask) {
+; CHECK-LABEL: test_mask_adds_epi8_rrkz_128:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    kmovd %edi, %k1 ## encoding: [0xc5,0xfb,0x92,0xcf]
+; CHECK-NEXT:    vpaddsb %xmm1, %xmm0, %xmm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0x89,0xec,0xc1]
+; CHECK-NEXT:    retq ## encoding: [0xc3]
+  %1 = call <16 x i8> @llvm.sadd.sat.v16i8(<16 x i8> %a, <16 x i8> %b)
+  %2 = bitcast i16 %mask to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x i8> %1, <16 x i8> zeroinitializer
+  ret <16 x i8> %3
+}
+
+define <16 x i8> @test_mask_adds_epi8_rm_128(<16 x i8> %a, <16 x i8>* %ptr_b) {
+; CHECK-LABEL: test_mask_adds_epi8_rm_128:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    vpaddsb (%rdi), %xmm0, %xmm0 ## EVEX TO VEX Compression encoding: [0xc5,0xf9,0xec,0x07]
+; CHECK-NEXT:    retq ## encoding: [0xc3]
+  %b = load <16 x i8>, <16 x i8>* %ptr_b
+  %1 = call <16 x i8> @llvm.sadd.sat.v16i8(<16 x i8> %a, <16 x i8> %b)
+  ret <16 x i8> %1
+}
+
+define <16 x i8> @test_mask_adds_epi8_rmk_128(<16 x i8> %a, <16 x i8>* %ptr_b, <16 x i8> %passThru, i16 %mask) {
+; CHECK-LABEL: test_mask_adds_epi8_rmk_128:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    kmovd %esi, %k1 ## encoding: [0xc5,0xfb,0x92,0xce]
+; CHECK-NEXT:    vpaddsb (%rdi), %xmm0, %xmm1 {%k1} ## encoding: [0x62,0xf1,0x7d,0x09,0xec,0x0f]
+; CHECK-NEXT:    vmovdqa %xmm1, %xmm0 ## EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc1]
+; CHECK-NEXT:    retq ## encoding: [0xc3]
+  %b = load <16 x i8>, <16 x i8>* %ptr_b
+  %1 = call <16 x i8> @llvm.sadd.sat.v16i8(<16 x i8> %a, <16 x i8> %b)
+  %2 = bitcast i16 %mask to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x i8> %1, <16 x i8> %passThru
+  ret <16 x i8> %3
+}
+
+define <16 x i8> @test_mask_adds_epi8_rmkz_128(<16 x i8> %a, <16 x i8>* %ptr_b, i16 %mask) {
+; CHECK-LABEL: test_mask_adds_epi8_rmkz_128:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    kmovd %esi, %k1 ## encoding: [0xc5,0xfb,0x92,0xce]
+; CHECK-NEXT:    vpaddsb (%rdi), %xmm0, %xmm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0x89,0xec,0x07]
+; CHECK-NEXT:    retq ## encoding: [0xc3]
+  %b = load <16 x i8>, <16 x i8>* %ptr_b
+  %1 = call <16 x i8> @llvm.sadd.sat.v16i8(<16 x i8> %a, <16 x i8> %b)
+  %2 = bitcast i16 %mask to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x i8> %1, <16 x i8> zeroinitializer
+  ret <16 x i8> %3
+}
+
+define <32 x i8> @test_mask_adds_epi8_rr_256(<32 x i8> %a, <32 x i8> %b) {
+; CHECK-LABEL: test_mask_adds_epi8_rr_256:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    vpaddsb %ymm1, %ymm0, %ymm0 ## EVEX TO VEX Compression encoding: [0xc5,0xfd,0xec,0xc1]
+; CHECK-NEXT:    retq ## encoding: [0xc3]
+  %1 = call <32 x i8> @llvm.sadd.sat.v32i8(<32 x i8> %a, <32 x i8> %b)
+  ret <32 x i8> %1
+}
+declare <32 x i8> @llvm.sadd.sat.v32i8(<32 x i8>, <32 x i8>)
+
+define <32 x i8> @test_mask_adds_epi8_rrk_256(<32 x i8> %a, <32 x i8> %b, <32 x i8> %passThru, i32 %mask) {
+; CHECK-LABEL: test_mask_adds_epi8_rrk_256:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    kmovd %edi, %k1 ## encoding: [0xc5,0xfb,0x92,0xcf]
+; CHECK-NEXT:    vpaddsb %ymm1, %ymm0, %ymm2 {%k1} ## encoding: [0x62,0xf1,0x7d,0x29,0xec,0xd1]
+; CHECK-NEXT:    vmovdqa %ymm2, %ymm0 ## EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc2]
+; CHECK-NEXT:    retq ## encoding: [0xc3]
+  %1 = call <32 x i8> @llvm.sadd.sat.v32i8(<32 x i8> %a, <32 x i8> %b)
+  %2 = bitcast i32 %mask to <32 x i1>
+  %3 = select <32 x i1> %2, <32 x i8> %1, <32 x i8> %passThru
+  ret <32 x i8> %3
+}
+
+define <32 x i8> @test_mask_adds_epi8_rrkz_256(<32 x i8> %a, <32 x i8> %b, i32 %mask) {
+; CHECK-LABEL: test_mask_adds_epi8_rrkz_256:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    kmovd %edi, %k1 ## encoding: [0xc5,0xfb,0x92,0xcf]
+; CHECK-NEXT:    vpaddsb %ymm1, %ymm0, %ymm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0xa9,0xec,0xc1]
+; CHECK-NEXT:    retq ## encoding: [0xc3]
+  %1 = call <32 x i8> @llvm.sadd.sat.v32i8(<32 x i8> %a, <32 x i8> %b)
+  %2 = bitcast i32 %mask to <32 x i1>
+  %3 = select <32 x i1> %2, <32 x i8> %1, <32 x i8> zeroinitializer
+  ret <32 x i8> %3
+}
+
+define <32 x i8> @test_mask_adds_epi8_rm_256(<32 x i8> %a, <32 x i8>* %ptr_b) {
+; CHECK-LABEL: test_mask_adds_epi8_rm_256:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    vpaddsb (%rdi), %ymm0, %ymm0 ## EVEX TO VEX Compression encoding: [0xc5,0xfd,0xec,0x07]
+; CHECK-NEXT:    retq ## encoding: [0xc3]
+  %b = load <32 x i8>, <32 x i8>* %ptr_b
+  %1 = call <32 x i8> @llvm.sadd.sat.v32i8(<32 x i8> %a, <32 x i8> %b)
+  ret <32 x i8> %1
+}
+
+define <32 x i8> @test_mask_adds_epi8_rmk_256(<32 x i8> %a, <32 x i8>* %ptr_b, <32 x i8> %passThru, i32 %mask) {
+; CHECK-LABEL: test_mask_adds_epi8_rmk_256:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    kmovd %esi, %k1 ## encoding: [0xc5,0xfb,0x92,0xce]
+; CHECK-NEXT:    vpaddsb (%rdi), %ymm0, %ymm1 {%k1} ## encoding: [0x62,0xf1,0x7d,0x29,0xec,0x0f]
+; CHECK-NEXT:    vmovdqa %ymm1, %ymm0 ## EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc1]
+; CHECK-NEXT:    retq ## encoding: [0xc3]
+  %b = load <32 x i8>, <32 x i8>* %ptr_b
+  %1 = call <32 x i8> @llvm.sadd.sat.v32i8(<32 x i8> %a, <32 x i8> %b)
+  %2 = bitcast i32 %mask to <32 x i1>
+  %3 = select <32 x i1> %2, <32 x i8> %1, <32 x i8> %passThru
+  ret <32 x i8> %3
+}
+
+define <32 x i8> @test_mask_adds_epi8_rmkz_256(<32 x i8> %a, <32 x i8>* %ptr_b, i32 %mask) {
+; CHECK-LABEL: test_mask_adds_epi8_rmkz_256:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    kmovd %esi, %k1 ## encoding: [0xc5,0xfb,0x92,0xce]
+; CHECK-NEXT:    vpaddsb (%rdi), %ymm0, %ymm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0xa9,0xec,0x07]
+; CHECK-NEXT:    retq ## encoding: [0xc3]
+  %b = load <32 x i8>, <32 x i8>* %ptr_b
+  %1 = call <32 x i8> @llvm.sadd.sat.v32i8(<32 x i8> %a, <32 x i8> %b)
+  %2 = bitcast i32 %mask to <32 x i1>
+  %3 = select <32 x i1> %2, <32 x i8> %1, <32 x i8> zeroinitializer
+  ret <32 x i8> %3
+}
+
+define <16 x i8> @test_mask_subs_epi8_rr_128(<16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: test_mask_subs_epi8_rr_128:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    vpsubsb %xmm1, %xmm0, %xmm0 ## EVEX TO VEX Compression encoding: [0xc5,0xf9,0xe8,0xc1]
+; CHECK-NEXT:    retq ## encoding: [0xc3]
+  %sub = call <16 x i8> @llvm.ssub.sat.v16i8(<16 x i8> %a, <16 x i8> %b)
+  ret <16 x i8> %sub
+}
+declare <16 x i8> @llvm.ssub.sat.v16i8(<16 x i8>, <16 x i8>)
+
+define <16 x i8> @test_mask_subs_epi8_rrk_128(<16 x i8> %a, <16 x i8> %b, <16 x i8> %passThru, i16 %mask) {
+; CHECK-LABEL: test_mask_subs_epi8_rrk_128:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    kmovd %edi, %k1 ## encoding: [0xc5,0xfb,0x92,0xcf]
+; CHECK-NEXT:    vpsubsb %xmm1, %xmm0, %xmm2 {%k1} ## encoding: [0x62,0xf1,0x7d,0x09,0xe8,0xd1]
+; CHECK-NEXT:    vmovdqa %xmm2, %xmm0 ## EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc2]
+; CHECK-NEXT:    retq ## encoding: [0xc3]
+  %sub = call <16 x i8> @llvm.ssub.sat.v16i8(<16 x i8> %a, <16 x i8> %b)
+  %bc = bitcast i16 %mask to <16 x i1>
+  %res = select <16 x i1> %bc, <16 x i8> %sub, <16 x i8> %passThru
+  ret <16 x i8> %res
+}
+
+define <16 x i8> @test_mask_subs_epi8_rrkz_128(<16 x i8> %a, <16 x i8> %b, i16 %mask) {
+; CHECK-LABEL: test_mask_subs_epi8_rrkz_128:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    kmovd %edi, %k1 ## encoding: [0xc5,0xfb,0x92,0xcf]
+; CHECK-NEXT:    vpsubsb %xmm1, %xmm0, %xmm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0x89,0xe8,0xc1]
+; CHECK-NEXT:    retq ## encoding: [0xc3]
+  %sub = call <16 x i8> @llvm.ssub.sat.v16i8(<16 x i8> %a, <16 x i8> %b)
+  %bc = bitcast i16 %mask to <16 x i1>
+  %res = select <16 x i1> %bc, <16 x i8> %sub, <16 x i8> zeroinitializer
+  ret <16 x i8> %res
+}
+
+define <16 x i8> @test_mask_subs_epi8_rm_128(<16 x i8> %a, <16 x i8>* %ptr_b) {
+; CHECK-LABEL: test_mask_subs_epi8_rm_128:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    vpsubsb (%rdi), %xmm0, %xmm0 ## EVEX TO VEX Compression encoding: [0xc5,0xf9,0xe8,0x07]
+; CHECK-NEXT:    retq ## encoding: [0xc3]
+  %b = load <16 x i8>, <16 x i8>* %ptr_b
+  %sub = call <16 x i8> @llvm.ssub.sat.v16i8(<16 x i8> %a, <16 x i8> %b)
+  ret <16 x i8> %sub
+}
+
+define <16 x i8> @test_mask_subs_epi8_rmk_128(<16 x i8> %a, <16 x i8>* %ptr_b, <16 x i8> %passThru, i16 %mask) {
+; CHECK-LABEL: test_mask_subs_epi8_rmk_128:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    kmovd %esi, %k1 ## encoding: [0xc5,0xfb,0x92,0xce]
+; CHECK-NEXT:    vpsubsb (%rdi), %xmm0, %xmm1 {%k1} ## encoding: [0x62,0xf1,0x7d,0x09,0xe8,0x0f]
+; CHECK-NEXT:    vmovdqa %xmm1, %xmm0 ## EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc1]
+; CHECK-NEXT:    retq ## encoding: [0xc3]
+  %b = load <16 x i8>, <16 x i8>* %ptr_b
+  %sub = call <16 x i8> @llvm.ssub.sat.v16i8(<16 x i8> %a, <16 x i8> %b)
+  %bc = bitcast i16 %mask to <16 x i1>
+  %res = select <16 x i1> %bc, <16 x i8> %sub, <16 x i8> %passThru
+  ret <16 x i8> %res
+}
+
+define <16 x i8> @test_mask_subs_epi8_rmkz_128(<16 x i8> %a, <16 x i8>* %ptr_b, i16 %mask) {
+; CHECK-LABEL: test_mask_subs_epi8_rmkz_128:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    kmovd %esi, %k1 ## encoding: [0xc5,0xfb,0x92,0xce]
+; CHECK-NEXT:    vpsubsb (%rdi), %xmm0, %xmm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0x89,0xe8,0x07]
+; CHECK-NEXT:    retq ## encoding: [0xc3]
+  %b = load <16 x i8>, <16 x i8>* %ptr_b
+  %sub = call <16 x i8> @llvm.ssub.sat.v16i8(<16 x i8> %a, <16 x i8> %b)
+  %bc = bitcast i16 %mask to <16 x i1>
+  %res = select <16 x i1> %bc, <16 x i8> %sub, <16 x i8> zeroinitializer
+  ret <16 x i8> %res
+}
+
+define <32 x i8> @test_mask_subs_epi8_rr_256(<32 x i8> %a, <32 x i8> %b) {
+; CHECK-LABEL: test_mask_subs_epi8_rr_256:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    vpsubsb %ymm1, %ymm0, %ymm0 ## EVEX TO VEX Compression encoding: [0xc5,0xfd,0xe8,0xc1]
+; CHECK-NEXT:    retq ## encoding: [0xc3]
+  %sub = call <32 x i8> @llvm.ssub.sat.v32i8(<32 x i8> %a, <32 x i8> %b)
+  ret <32 x i8> %sub
+}
+declare <32 x i8> @llvm.ssub.sat.v32i8(<32 x i8>, <32 x i8>)
+
+define <32 x i8> @test_mask_subs_epi8_rrk_256(<32 x i8> %a, <32 x i8> %b, <32 x i8> %passThru, i32 %mask) {
+; CHECK-LABEL: test_mask_subs_epi8_rrk_256:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    kmovd %edi, %k1 ## encoding: [0xc5,0xfb,0x92,0xcf]
+; CHECK-NEXT:    vpsubsb %ymm1, %ymm0, %ymm2 {%k1} ## encoding: [0x62,0xf1,0x7d,0x29,0xe8,0xd1]
+; CHECK-NEXT:    vmovdqa %ymm2, %ymm0 ## EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc2]
+; CHECK-NEXT:    retq ## encoding: [0xc3]
+  %sub = call <32 x i8> @llvm.ssub.sat.v32i8(<32 x i8> %a, <32 x i8> %b)
+  %bc = bitcast i32 %mask to <32 x i1>
+  %res = select <32 x i1> %bc, <32 x i8> %sub, <32 x i8> %passThru
+  ret <32 x i8> %res
+}
+
+define <32 x i8> @test_mask_subs_epi8_rrkz_256(<32 x i8> %a, <32 x i8> %b, i32 %mask) {
+; CHECK-LABEL: test_mask_subs_epi8_rrkz_256:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    kmovd %edi, %k1 ## encoding: [0xc5,0xfb,0x92,0xcf]
+; CHECK-NEXT:    vpsubsb %ymm1, %ymm0, %ymm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0xa9,0xe8,0xc1]
+; CHECK-NEXT:    retq ## encoding: [0xc3]
+  %sub = call <32 x i8> @llvm.ssub.sat.v32i8(<32 x i8> %a, <32 x i8> %b)
+  %bc = bitcast i32 %mask to <32 x i1>
+  %res = select <32 x i1> %bc, <32 x i8> %sub, <32 x i8> zeroinitializer
+  ret <32 x i8> %res
+}
+
+define <32 x i8> @test_mask_subs_epi8_rm_256(<32 x i8> %a, <32 x i8>* %ptr_b) {
+; CHECK-LABEL: test_mask_subs_epi8_rm_256:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    vpsubsb (%rdi), %ymm0, %ymm0 ## EVEX TO VEX Compression encoding: [0xc5,0xfd,0xe8,0x07]
+; CHECK-NEXT:    retq ## encoding: [0xc3]
+  %b = load <32 x i8>, <32 x i8>* %ptr_b
+  %sub = call <32 x i8> @llvm.ssub.sat.v32i8(<32 x i8> %a, <32 x i8> %b)
+  ret <32 x i8> %sub
+}
+
+define <32 x i8> @test_mask_subs_epi8_rmk_256(<32 x i8> %a, <32 x i8>* %ptr_b, <32 x i8> %passThru, i32 %mask) {
+; CHECK-LABEL: test_mask_subs_epi8_rmk_256:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    kmovd %esi, %k1 ## encoding: [0xc5,0xfb,0x92,0xce]
+; CHECK-NEXT:    vpsubsb (%rdi), %ymm0, %ymm1 {%k1} ## encoding: [0x62,0xf1,0x7d,0x29,0xe8,0x0f]
+; CHECK-NEXT:    vmovdqa %ymm1, %ymm0 ## EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc1]
+; CHECK-NEXT:    retq ## encoding: [0xc3]
+  %b = load <32 x i8>, <32 x i8>* %ptr_b
+  %sub = call <32 x i8> @llvm.ssub.sat.v32i8(<32 x i8> %a, <32 x i8> %b)
+  %bc = bitcast i32 %mask to <32 x i1>
+  %res = select <32 x i1> %bc, <32 x i8> %sub, <32 x i8> %passThru
+  ret <32 x i8> %res
+}
+
+define <32 x i8> @test_mask_subs_epi8_rmkz_256(<32 x i8> %a, <32 x i8>* %ptr_b, i32 %mask) {
+; CHECK-LABEL: test_mask_subs_epi8_rmkz_256:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    kmovd %esi, %k1 ## encoding: [0xc5,0xfb,0x92,0xce]
+; CHECK-NEXT:    vpsubsb (%rdi), %ymm0, %ymm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0xa9,0xe8,0x07]
+; CHECK-NEXT:    retq ## encoding: [0xc3]
+  %b = load <32 x i8>, <32 x i8>* %ptr_b
+  %sub = call <32 x i8> @llvm.ssub.sat.v32i8(<32 x i8> %a, <32 x i8> %b)
+  %bc = bitcast i32 %mask to <32 x i1>
+  %res = select <32 x i1> %bc, <32 x i8> %sub, <32 x i8> zeroinitializer
+  ret <32 x i8> %res
+}
+
+;
+; Unsigned Saturation
+;
+
 define <8 x i16> @test_mask_adds_epu16_rr_128(<8 x i16> %a, <8 x i16> %b) {
 ; CHECK-LABEL: test_mask_adds_epu16_rr_128:
 ; CHECK:       ## %bb.0:
 ; CHECK-NEXT:    vpaddusw %xmm1, %xmm0, %xmm0 ## EVEX TO VEX Compression encoding: [0xc5,0xf9,0xdd,0xc1]
 ; CHECK-NEXT:    retq ## encoding: [0xc3]
-  %1 = add <8 x i16> %a, %b
-  %2 = icmp ugt <8 x i16> %a, %1
-  %3 = select <8 x i1> %2, <8 x i16> <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>, <8 x i16> %1
-  ret <8 x i16> %3
+  %1 = call <8 x i16> @llvm.uadd.sat.v8i16(<8 x i16> %a, <8 x i16> %b)
+  ret <8 x i16> %1
 }
+declare <8 x i16> @llvm.uadd.sat.v8i16(<8 x i16>, <8 x i16>)
 
 define <8 x i16> @test_mask_adds_epu16_rrk_128(<8 x i16> %a, <8 x i16> %b, <8 x i16> %passThru, i8 %mask) {
 ; CHECK-LABEL: test_mask_adds_epu16_rrk_128:
@@ -22,12 +605,10 @@ define <8 x i16> @test_mask_adds_epu16_rrk_128(<8 x i16> %a, <8 x i16> %b, <8 x
 ; CHECK-NEXT:    vpaddusw %xmm1, %xmm0, %xmm2 {%k1} ## encoding: [0x62,0xf1,0x7d,0x09,0xdd,0xd1]
 ; CHECK-NEXT:    vmovdqa %xmm2, %xmm0 ## EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc2]
 ; CHECK-NEXT:    retq ## encoding: [0xc3]
-  %1 = add <8 x i16> %a, %b
-  %2 = icmp ugt <8 x i16> %a, %1
-  %3 = select <8 x i1> %2, <8 x i16> <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>, <8 x i16> %1
-  %4 = bitcast i8 %mask to <8 x i1>
-  %5 = select <8 x i1> %4, <8 x i16> %3, <8 x i16> %passThru
-  ret <8 x i16> %5
+  %1 = call <8 x i16> @llvm.uadd.sat.v8i16(<8 x i16> %a, <8 x i16> %b)
+  %2 = bitcast i8 %mask to <8 x i1>
+  %3 = select <8 x i1> %2, <8 x i16> %1, <8 x i16> %passThru
+  ret <8 x i16> %3
 }
 
 define <8 x i16> @test_mask_adds_epu16_rrkz_128(<8 x i16> %a, <8 x i16> %b, i8 %mask) {
@@ -36,12 +617,10 @@ define <8 x i16> @test_mask_adds_epu16_rrkz_128(<8 x i16> %a, <8 x i16> %b, i8 %
 ; CHECK-NEXT:    kmovd %edi, %k1 ## encoding: [0xc5,0xfb,0x92,0xcf]
 ; CHECK-NEXT:    vpaddusw %xmm1, %xmm0, %xmm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0x89,0xdd,0xc1]
 ; CHECK-NEXT:    retq ## encoding: [0xc3]
-  %1 = add <8 x i16> %a, %b
-  %2 = icmp ugt <8 x i16> %a, %1
-  %3 = select <8 x i1> %2, <8 x i16> <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>, <8 x i16> %1
-  %4 = bitcast i8 %mask to <8 x i1>
-  %5 = select <8 x i1> %4, <8 x i16> %3, <8 x i16> zeroinitializer
-  ret <8 x i16> %5
+  %1 = call <8 x i16> @llvm.uadd.sat.v8i16(<8 x i16> %a, <8 x i16> %b)
+  %2 = bitcast i8 %mask to <8 x i1>
+  %3 = select <8 x i1> %2, <8 x i16> %1, <8 x i16> zeroinitializer
+  ret <8 x i16> %3
 }
 
 define <8 x i16> @test_mask_adds_epu16_rm_128(<8 x i16> %a, <8 x i16>* %ptr_b) {
@@ -50,10 +629,8 @@ define <8 x i16> @test_mask_adds_epu16_rm_128(<8 x i16> %a, <8 x i16>* %ptr_b) {
 ; CHECK-NEXT:    vpaddusw (%rdi), %xmm0, %xmm0 ## EVEX TO VEX Compression encoding: [0xc5,0xf9,0xdd,0x07]
 ; CHECK-NEXT:    retq ## encoding: [0xc3]
   %b = load <8 x i16>, <8 x i16>* %ptr_b
-  %1 = add <8 x i16> %a, %b
-  %2 = icmp ugt <8 x i16> %a, %1
-  %3 = select <8 x i1> %2, <8 x i16> <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>, <8 x i16> %1
-  ret <8 x i16> %3
+  %1 = call <8 x i16> @llvm.uadd.sat.v8i16(<8 x i16> %a, <8 x i16> %b)
+  ret <8 x i16> %1
 }
 
 define <8 x i16> @test_mask_adds_epu16_rmk_128(<8 x i16> %a, <8 x i16>* %ptr_b, <8 x i16> %passThru, i8 %mask) {
@@ -64,12 +641,10 @@ define <8 x i16> @test_mask_adds_epu16_rmk_128(<8 x i16> %a, <8 x i16>* %ptr_b,
 ; CHECK-NEXT:    vmovdqa %xmm1, %xmm0 ## EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc1]
 ; CHECK-NEXT:    retq ## encoding: [0xc3]
   %b = load <8 x i16>, <8 x i16>* %ptr_b
-  %1 = add <8 x i16> %a, %b
-  %2 = icmp ugt <8 x i16> %a, %1
-  %3 = select <8 x i1> %2, <8 x i16> <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>, <8 x i16> %1
-  %4 = bitcast i8 %mask to <8 x i1>
-  %5 = select <8 x i1> %4, <8 x i16> %3, <8 x i16> %passThru
-  ret <8 x i16> %5
+  %1 = call <8 x i16> @llvm.uadd.sat.v8i16(<8 x i16> %a, <8 x i16> %b)
+  %2 = bitcast i8 %mask to <8 x i1>
+  %3 = select <8 x i1> %2, <8 x i16> %1, <8 x i16> %passThru
+  ret <8 x i16> %3
 }
 
 define <8 x i16> @test_mask_adds_epu16_rmkz_128(<8 x i16> %a, <8 x i16>* %ptr_b, i8 %mask) {
@@ -79,12 +654,10 @@ define <8 x i16> @test_mask_adds_epu16_rmkz_128(<8 x i16> %a, <8 x i16>* %ptr_b,
 ; CHECK-NEXT:    vpaddusw (%rdi), %xmm0, %xmm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0x89,0xdd,0x07]
 ; CHECK-NEXT:    retq ## encoding: [0xc3]
   %b = load <8 x i16>, <8 x i16>* %ptr_b
-  %1 = add <8 x i16> %a, %b
-  %2 = icmp ugt <8 x i16> %a, %1
-  %3 = select <8 x i1> %2, <8 x i16> <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>, <8 x i16> %1
-  %4 = bitcast i8 %mask to <8 x i1>
-  %5 = select <8 x i1> %4, <8 x i16> %3, <8 x i16> zeroinitializer
-  ret <8 x i16> %5
+  %1 = call <8 x i16> @llvm.uadd.sat.v8i16(<8 x i16> %a, <8 x i16> %b)
+  %2 = bitcast i8 %mask to <8 x i1>
+  %3 = select <8 x i1> %2, <8 x i16> %1, <8 x i16> zeroinitializer
+  ret <8 x i16> %3
 }
 
 define <16 x i16> @test_mask_adds_epu16_rr_256(<16 x i16> %a, <16 x i16> %b) {
@@ -92,11 +665,10 @@ define <16 x i16> @test_mask_adds_epu16_rr_256(<16 x i16> %a, <16 x i16> %b) {
 ; CHECK:       ## %bb.0:
 ; CHECK-NEXT:    vpaddusw %ymm1, %ymm0, %ymm0 ## EVEX TO VEX Compression encoding: [0xc5,0xfd,0xdd,0xc1]
 ; CHECK-NEXT:    retq ## encoding: [0xc3]
-  %1 = add <16 x i16> %a, %b
-  %2 = icmp ugt <16 x i16> %a, %1
-  %3 = select <16 x i1> %2, <16 x i16> <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>, <16 x i16> %1
-  ret <16 x i16> %3
+  %1 = call <16 x i16> @llvm.uadd.sat.v16i16(<16 x i16> %a, <16 x i16> %b)
+  ret <16 x i16> %1
 }
+declare <16 x i16> @llvm.uadd.sat.v16i16(<16 x i16>, <16 x i16>)
 
 define <16 x i16> @test_mask_adds_epu16_rrk_256(<16 x i16> %a, <16 x i16> %b, <16 x i16> %passThru, i16 %mask) {
 ; CHECK-LABEL: test_mask_adds_epu16_rrk_256:
@@ -105,12 +677,10 @@ define <16 x i16> @test_mask_adds_epu16_rrk_256(<16 x i16> %a, <16 x i16> %b, <1
 ; CHECK-NEXT:    vpaddusw %ymm1, %ymm0, %ymm2 {%k1} ## encoding: [0x62,0xf1,0x7d,0x29,0xdd,0xd1]
 ; CHECK-NEXT:    vmovdqa %ymm2, %ymm0 ## EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc2]
 ; CHECK-NEXT:    retq ## encoding: [0xc3]
-  %1 = add <16 x i16> %a, %b
-  %2 = icmp ugt <16 x i16> %a, %1
-  %3 = select <16 x i1> %2, <16 x i16> <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>, <16 x i16> %1
-  %4 = bitcast i16 %mask to <16 x i1>
-  %5 = select <16 x i1> %4, <16 x i16> %3, <16 x i16> %passThru
-  ret <16 x i16> %5
+  %1 = call <16 x i16> @llvm.uadd.sat.v16i16(<16 x i16> %a, <16 x i16> %b)
+  %2 = bitcast i16 %mask to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x i16> %1, <16 x i16> %passThru
+  ret <16 x i16> %3
 }
 
 define <16 x i16> @test_mask_adds_epu16_rrkz_256(<16 x i16> %a, <16 x i16> %b, i16 %mask) {
@@ -119,12 +689,10 @@ define <16 x i16> @test_mask_adds_epu16_rrkz_256(<16 x i16> %a, <16 x i16> %b, i
 ; CHECK-NEXT:    kmovd %edi, %k1 ## encoding: [0xc5,0xfb,0x92,0xcf]
 ; CHECK-NEXT:    vpaddusw %ymm1, %ymm0, %ymm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0xa9,0xdd,0xc1]
 ; CHECK-NEXT:    retq ## encoding: [0xc3]
-  %1 = add <16 x i16> %a, %b
-  %2 = icmp ugt <16 x i16> %a, %1
-  %3 = select <16 x i1> %2, <16 x i16> <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>, <16 x i16> %1
-  %4 = bitcast i16 %mask to <16 x i1>
-  %5 = select <16 x i1> %4, <16 x i16> %3, <16 x i16> zeroinitializer
-  ret <16 x i16> %5
+  %1 = call <16 x i16> @llvm.uadd.sat.v16i16(<16 x i16> %a, <16 x i16> %b)
+  %2 = bitcast i16 %mask to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x i16> %1, <16 x i16> zeroinitializer
+  ret <16 x i16> %3
 }
 
 define <16 x i16> @test_mask_adds_epu16_rm_256(<16 x i16> %a, <16 x i16>* %ptr_b) {
@@ -133,10 +701,8 @@ define <16 x i16> @test_mask_adds_epu16_rm_256(<16 x i16> %a, <16 x i16>* %ptr_b
 ; CHECK-NEXT:    vpaddusw (%rdi), %ymm0, %ymm0 ## EVEX TO VEX Compression encoding: [0xc5,0xfd,0xdd,0x07]
 ; CHECK-NEXT:    retq ## encoding: [0xc3]
   %b = load <16 x i16>, <16 x i16>* %ptr_b
-  %1 = add <16 x i16> %a, %b
-  %2 = icmp ugt <16 x i16> %a, %1
-  %3 = select <16 x i1> %2, <16 x i16> <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>, <16 x i16> %1
-  ret <16 x i16> %3
+  %1 = call <16 x i16> @llvm.uadd.sat.v16i16(<16 x i16> %a, <16 x i16> %b)
+  ret <16 x i16> %1
 }
 
 define <16 x i16> @test_mask_adds_epu16_rmk_256(<16 x i16> %a, <16 x i16>* %ptr_b, <16 x i16> %passThru, i16 %mask) {
@@ -147,12 +713,10 @@ define <16 x i16> @test_mask_adds_epu16_rmk_256(<16 x i16> %a, <16 x i16>* %ptr_
 ; CHECK-NEXT:    vmovdqa %ymm1, %ymm0 ## EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc1]
 ; CHECK-NEXT:    retq ## encoding: [0xc3]
   %b = load <16 x i16>, <16 x i16>* %ptr_b
-  %1 = add <16 x i16> %a, %b
-  %2 = icmp ugt <16 x i16> %a, %1
-  %3 = select <16 x i1> %2, <16 x i16> <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>, <16 x i16> %1
-  %4 = bitcast i16 %mask to <16 x i1>
-  %5 = select <16 x i1> %4, <16 x i16> %3, <16 x i16> %passThru
-  ret <16 x i16> %5
+  %1 = call <16 x i16> @llvm.uadd.sat.v16i16(<16 x i16> %a, <16 x i16> %b)
+  %2 = bitcast i16 %mask to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x i16> %1, <16 x i16> %passThru
+  ret <16 x i16> %3
 }
 
 define <16 x i16> @test_mask_adds_epu16_rmkz_256(<16 x i16> %a, <16 x i16>* %ptr_b, i16 %mask) {
@@ -162,12 +726,10 @@ define <16 x i16> @test_mask_adds_epu16_rmkz_256(<16 x i16> %a, <16 x i16>* %ptr
 ; CHECK-NEXT:    vpaddusw (%rdi), %ymm0, %ymm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0xa9,0xdd,0x07]
 ; CHECK-NEXT:    retq ## encoding: [0xc3]
   %b = load <16 x i16>, <16 x i16>* %ptr_b
-  %1 = add <16 x i16> %a, %b
-  %2 = icmp ugt <16 x i16> %a, %1
-  %3 = select <16 x i1> %2, <16 x i16> <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>, <16 x i16> %1
-  %4 = bitcast i16 %mask to <16 x i1>
-  %5 = select <16 x i1> %4, <16 x i16> %3, <16 x i16> zeroinitializer
-  ret <16 x i16> %5
+  %1 = call <16 x i16> @llvm.uadd.sat.v16i16(<16 x i16> %a, <16 x i16> %b)
+  %2 = bitcast i16 %mask to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x i16> %1, <16 x i16> zeroinitializer
+  ret <16 x i16> %3
 }
 
 define <8 x i16> @test_mask_subs_epu16_rr_128(<8 x i16> %a, <8 x i16> %b) {
@@ -175,11 +737,10 @@ define <8 x i16> @test_mask_subs_epu16_rr_128(<8 x i16> %a, <8 x i16> %b) {
 ; CHECK:       ## %bb.0:
 ; CHECK-NEXT:    vpsubusw %xmm1, %xmm0, %xmm0 ## EVEX TO VEX Compression encoding: [0xc5,0xf9,0xd9,0xc1]
 ; CHECK-NEXT:    retq ## encoding: [0xc3]
-  %cmp = icmp ugt <8 x i16> %a, %b
-  %sel = select <8 x i1> %cmp, <8 x i16> %a, <8 x i16> %b
-  %sub = sub <8 x i16> %sel, %b
+  %sub = call <8 x i16> @llvm.usub.sat.v8i16(<8 x i16> %a, <8 x i16> %b)
   ret <8 x i16> %sub
 }
+declare <8 x i16> @llvm.usub.sat.v8i16(<8 x i16>, <8 x i16>)
 
 define <8 x i16> @test_mask_subs_epu16_rrk_128(<8 x i16> %a, <8 x i16> %b, <8 x i16> %passThru, i8 %mask) {
 ; CHECK-LABEL: test_mask_subs_epu16_rrk_128:
@@ -188,9 +749,7 @@ define <8 x i16> @test_mask_subs_epu16_rrk_128(<8 x i16> %a, <8 x i16> %b, <8 x
 ; CHECK-NEXT:    vpsubusw %xmm1, %xmm0, %xmm2 {%k1} ## encoding: [0x62,0xf1,0x7d,0x09,0xd9,0xd1]
 ; CHECK-NEXT:    vmovdqa %xmm2, %xmm0 ## EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc2]
 ; CHECK-NEXT:    retq ## encoding: [0xc3]
-  %cmp = icmp ugt <8 x i16> %a, %b
-  %sel = select <8 x i1> %cmp, <8 x i16> %a, <8 x i16> %b
-  %sub = sub <8 x i16> %sel, %b
+  %sub = call <8 x i16> @llvm.usub.sat.v8i16(<8 x i16> %a, <8 x i16> %b)
   %bc = bitcast i8 %mask to <8 x i1>
   %res = select <8 x i1> %bc, <8 x i16> %sub, <8 x i16> %passThru
   ret <8 x i16> %res
@@ -202,9 +761,7 @@ define <8 x i16> @test_mask_subs_epu16_rrkz_128(<8 x i16> %a, <8 x i16> %b, i8 %
 ; CHECK-NEXT:    kmovd %edi, %k1 ## encoding: [0xc5,0xfb,0x92,0xcf]
 ; CHECK-NEXT:    vpsubusw %xmm1, %xmm0, %xmm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0x89,0xd9,0xc1]
 ; CHECK-NEXT:    retq ## encoding: [0xc3]
-  %cmp = icmp ugt <8 x i16> %a, %b
-  %sel = select <8 x i1> %cmp, <8 x i16> %a, <8 x i16> %b
-  %sub = sub <8 x i16> %sel, %b
+  %sub = call <8 x i16> @llvm.usub.sat.v8i16(<8 x i16> %a, <8 x i16> %b)
   %bc = bitcast i8 %mask to <8 x i1>
   %res = select <8 x i1> %bc, <8 x i16> %sub, <8 x i16> zeroinitializer
   ret <8 x i16> %res
@@ -216,9 +773,7 @@ define <8 x i16> @test_mask_subs_epu16_rm_128(<8 x i16> %a, <8 x i16>* %ptr_b) {
 ; CHECK-NEXT:    vpsubusw (%rdi), %xmm0, %xmm0 ## EVEX TO VEX Compression encoding: [0xc5,0xf9,0xd9,0x07]
 ; CHECK-NEXT:    retq ## encoding: [0xc3]
   %b = load <8 x i16>, <8 x i16>* %ptr_b
-  %cmp = icmp ugt <8 x i16> %a, %b
-  %sel = select <8 x i1> %cmp, <8 x i16> %a, <8 x i16> %b
-  %sub = sub <8 x i16> %sel, %b
+  %sub = call <8 x i16> @llvm.usub.sat.v8i16(<8 x i16> %a, <8 x i16> %b)
   ret <8 x i16> %sub
 }
 
@@ -230,9 +785,7 @@ define <8 x i16> @test_mask_subs_epu16_rmk_128(<8 x i16> %a, <8 x i16>* %ptr_b,
 ; CHECK-NEXT:    vmovdqa %xmm1, %xmm0 ## EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc1]
 ; CHECK-NEXT:    retq ## encoding: [0xc3]
   %b = load <8 x i16>, <8 x i16>* %ptr_b
-  %cmp = icmp ugt <8 x i16> %a, %b
-  %sel = select <8 x i1> %cmp, <8 x i16> %a, <8 x i16> %b
-  %sub = sub <8 x i16> %sel, %b
+  %sub = call <8 x i16> @llvm.usub.sat.v8i16(<8 x i16> %a, <8 x i16> %b)
   %bc = bitcast i8 %mask to <8 x i1>
   %res = select <8 x i1> %bc, <8 x i16> %sub, <8 x i16> %passThru
   ret <8 x i16> %res
@@ -245,9 +798,7 @@ define <8 x i16> @test_mask_subs_epu16_rmkz_128(<8 x i16> %a, <8 x i16>* %ptr_b,
 ; CHECK-NEXT:    vpsubusw (%rdi), %xmm0, %xmm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0x89,0xd9,0x07]
 ; CHECK-NEXT:    retq ## encoding: [0xc3]
   %b = load <8 x i16>, <8 x i16>* %ptr_b
-  %cmp = icmp ugt <8 x i16> %a, %b
-  %sel = select <8 x i1> %cmp, <8 x i16> %a, <8 x i16> %b
-  %sub = sub <8 x i16> %sel, %b
+  %sub = call <8 x i16> @llvm.usub.sat.v8i16(<8 x i16> %a, <8 x i16> %b)
   %bc = bitcast i8 %mask to <8 x i1>
   %res = select <8 x i1> %bc, <8 x i16> %sub, <8 x i16> zeroinitializer
   ret <8 x i16> %res
@@ -258,11 +809,10 @@ define <16 x i16> @test_mask_subs_epu16_rr_256(<16 x i16> %a, <16 x i16> %b) {
 ; CHECK:       ## %bb.0:
 ; CHECK-NEXT:    vpsubusw %ymm1, %ymm0, %ymm0 ## EVEX TO VEX Compression encoding: [0xc5,0xfd,0xd9,0xc1]
 ; CHECK-NEXT:    retq ## encoding: [0xc3]
-  %cmp = icmp ugt <16 x i16> %a, %b
-  %sel = select <16 x i1> %cmp, <16 x i16> %a, <16 x i16> %b
-  %sub = sub <16 x i16> %sel, %b
+  %sub = call <16 x i16> @llvm.usub.sat.v16i16(<16 x i16> %a, <16 x i16> %b)
   ret <16 x i16> %sub
 }
+declare <16 x i16> @llvm.usub.sat.v16i16(<16 x i16>, <16 x i16>)
 
 define <16 x i16> @test_mask_subs_epu16_rrk_256(<16 x i16> %a, <16 x i16> %b, <16 x i16> %passThru, i16 %mask) {
 ; CHECK-LABEL: test_mask_subs_epu16_rrk_256:
@@ -271,9 +821,7 @@ define <16 x i16> @test_mask_subs_epu16_rrk_256(<16 x i16> %a, <16 x i16> %b, <1
 ; CHECK-NEXT:    vpsubusw %ymm1, %ymm0, %ymm2 {%k1} ## encoding: [0x62,0xf1,0x7d,0x29,0xd9,0xd1]
 ; CHECK-NEXT:    vmovdqa %ymm2, %ymm0 ## EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc2]
 ; CHECK-NEXT:    retq ## encoding: [0xc3]
-  %cmp = icmp ugt <16 x i16> %a, %b
-  %sel = select <16 x i1> %cmp, <16 x i16> %a, <16 x i16> %b
-  %sub = sub <16 x i16> %sel, %b
+  %sub = call <16 x i16> @llvm.usub.sat.v16i16(<16 x i16> %a, <16 x i16> %b)
   %bc = bitcast i16 %mask to <16 x i1>
   %res = select <16 x i1> %bc, <16 x i16> %sub, <16 x i16> %passThru
   ret <16 x i16> %res
@@ -285,9 +833,7 @@ define <16 x i16> @test_mask_subs_epu16_rrkz_256(<16 x i16> %a, <16 x i16> %b, i
 ; CHECK-NEXT:    kmovd %edi, %k1 ## encoding: [0xc5,0xfb,0x92,0xcf]
 ; CHECK-NEXT:    vpsubusw %ymm1, %ymm0, %ymm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0xa9,0xd9,0xc1]
 ; CHECK-NEXT:    retq ## encoding: [0xc3]
-  %cmp = icmp ugt <16 x i16> %a, %b
-  %sel = select <16 x i1> %cmp, <16 x i16> %a, <16 x i16> %b
-  %sub = sub <16 x i16> %sel, %b
+  %sub = call <16 x i16> @llvm.usub.sat.v16i16(<16 x i16> %a, <16 x i16> %b)
   %bc = bitcast i16 %mask to <16 x i1>
   %res = select <16 x i1> %bc, <16 x i16> %sub, <16 x i16> zeroinitializer
   ret <16 x i16> %res
@@ -299,9 +845,7 @@ define <16 x i16> @test_mask_subs_epu16_rm_256(<16 x i16> %a, <16 x i16>* %ptr_b
 ; CHECK-NEXT:    vpsubusw (%rdi), %ymm0, %ymm0 ## EVEX TO VEX Compression encoding: [0xc5,0xfd,0xd9,0x07]
 ; CHECK-NEXT:    retq ## encoding: [0xc3]
   %b = load <16 x i16>, <16 x i16>* %ptr_b
-  %cmp = icmp ugt <16 x i16> %a, %b
-  %sel = select <16 x i1> %cmp, <16 x i16> %a, <16 x i16> %b
-  %sub = sub <16 x i16> %sel, %b
+  %sub = call <16 x i16> @llvm.usub.sat.v16i16(<16 x i16> %a, <16 x i16> %b)
   ret <16 x i16> %sub
 }
 
@@ -313,9 +857,7 @@ define <16 x i16> @test_mask_subs_epu16_rmk_256(<16 x i16> %a, <16 x i16>* %ptr_
 ; CHECK-NEXT:    vmovdqa %ymm1, %ymm0 ## EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc1]
 ; CHECK-NEXT:    retq ## encoding: [0xc3]
   %b = load <16 x i16>, <16 x i16>* %ptr_b
-  %cmp = icmp ugt <16 x i16> %a, %b
-  %sel = select <16 x i1> %cmp, <16 x i16> %a, <16 x i16> %b
-  %sub = sub <16 x i16> %sel, %b
+  %sub = call <16 x i16> @llvm.usub.sat.v16i16(<16 x i16> %a, <16 x i16> %b)
   %bc = bitcast i16 %mask to <16 x i1>
   %res = select <16 x i1> %bc, <16 x i16> %sub, <16 x i16> %passThru
   ret <16 x i16> %res
@@ -328,9 +870,7 @@ define <16 x i16> @test_mask_subs_epu16_rmkz_256(<16 x i16> %a, <16 x i16>* %ptr
 ; CHECK-NEXT:    vpsubusw (%rdi), %ymm0, %ymm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0xa9,0xd9,0x07]
 ; CHECK-NEXT:    retq ## encoding: [0xc3]
   %b = load <16 x i16>, <16 x i16>* %ptr_b
-  %cmp = icmp ugt <16 x i16> %a, %b
-  %sel = select <16 x i1> %cmp, <16 x i16> %a, <16 x i16> %b
-  %sub = sub <16 x i16> %sel, %b
+  %sub = call <16 x i16> @llvm.usub.sat.v16i16(<16 x i16> %a, <16 x i16> %b)
   %bc = bitcast i16 %mask to <16 x i1>
   %res = select <16 x i1> %bc, <16 x i16> %sub, <16 x i16> zeroinitializer
   ret <16 x i16> %res
@@ -341,11 +881,10 @@ define <16 x i8> @test_mask_adds_epu8_rr_128(<16 x i8> %a, <16 x i8> %b) {
 ; CHECK:       ## %bb.0:
 ; CHECK-NEXT:    vpaddusb %xmm1, %xmm0, %xmm0 ## EVEX TO VEX Compression encoding: [0xc5,0xf9,0xdc,0xc1]
 ; CHECK-NEXT:    retq ## encoding: [0xc3]
-  %1 = add <16 x i8> %a, %b
-  %2 = icmp ugt <16 x i8> %a, %1
-  %3 = select <16 x i1> %2, <16 x i8> <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>, <16 x i8> %1
-  ret <16 x i8> %3
+  %1 = call <16 x i8> @llvm.uadd.sat.v16i8(<16 x i8> %a, <16 x i8> %b)
+  ret <16 x i8> %1
 }
+declare <16 x i8> @llvm.uadd.sat.v16i8(<16 x i8>, <16 x i8>)
 
 define <16 x i8> @test_mask_adds_epu8_rrk_128(<16 x i8> %a, <16 x i8> %b, <16 x i8> %passThru, i16 %mask) {
 ; CHECK-LABEL: test_mask_adds_epu8_rrk_128:
@@ -354,12 +893,10 @@ define <16 x i8> @test_mask_adds_epu8_rrk_128(<16 x i8> %a, <16 x i8> %b, <16 x
 ; CHECK-NEXT:    vpaddusb %xmm1, %xmm0, %xmm2 {%k1} ## encoding: [0x62,0xf1,0x7d,0x09,0xdc,0xd1]
 ; CHECK-NEXT:    vmovdqa %xmm2, %xmm0 ## EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc2]
 ; CHECK-NEXT:    retq ## encoding: [0xc3]
-  %1 = add <16 x i8> %a, %b
-  %2 = icmp ugt <16 x i8> %a, %1
-  %3 = select <16 x i1> %2, <16 x i8> <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>, <16 x i8> %1
-  %4 = bitcast i16 %mask to <16 x i1>
-  %5 = select <16 x i1> %4, <16 x i8> %3, <16 x i8> %passThru
-  ret <16 x i8> %5
+  %1 = call <16 x i8> @llvm.uadd.sat.v16i8(<16 x i8> %a, <16 x i8> %b)
+  %2 = bitcast i16 %mask to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x i8> %1, <16 x i8> %passThru
+  ret <16 x i8> %3
 }
 
 define <16 x i8> @test_mask_adds_epu8_rrkz_128(<16 x i8> %a, <16 x i8> %b, i16 %mask) {
@@ -368,12 +905,10 @@ define <16 x i8> @test_mask_adds_epu8_rrkz_128(<16 x i8> %a, <16 x i8> %b, i16 %
 ; CHECK-NEXT:    kmovd %edi, %k1 ## encoding: [0xc5,0xfb,0x92,0xcf]
 ; CHECK-NEXT:    vpaddusb %xmm1, %xmm0, %xmm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0x89,0xdc,0xc1]
 ; CHECK-NEXT:    retq ## encoding: [0xc3]
-  %1 = add <16 x i8> %a, %b
-  %2 = icmp ugt <16 x i8> %a, %1
-  %3 = select <16 x i1> %2, <16 x i8> <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>, <16 x i8> %1
-  %4 = bitcast i16 %mask to <16 x i1>
-  %5 = select <16 x i1> %4, <16 x i8> %3, <16 x i8> zeroinitializer
-  ret <16 x i8> %5
+  %1 = call <16 x i8> @llvm.uadd.sat.v16i8(<16 x i8> %a, <16 x i8> %b)
+  %2 = bitcast i16 %mask to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x i8> %1, <16 x i8> zeroinitializer
+  ret <16 x i8> %3
 }
 
 define <16 x i8> @test_mask_adds_epu8_rm_128(<16 x i8> %a, <16 x i8>* %ptr_b) {
@@ -382,10 +917,8 @@ define <16 x i8> @test_mask_adds_epu8_rm_128(<16 x i8> %a, <16 x i8>* %ptr_b) {
 ; CHECK-NEXT:    vpaddusb (%rdi), %xmm0, %xmm0 ## EVEX TO VEX Compression encoding: [0xc5,0xf9,0xdc,0x07]
 ; CHECK-NEXT:    retq ## encoding: [0xc3]
   %b = load <16 x i8>, <16 x i8>* %ptr_b
-  %1 = add <16 x i8> %a, %b
-  %2 = icmp ugt <16 x i8> %a, %1
-  %3 = select <16 x i1> %2, <16 x i8> <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>, <16 x i8> %1
-  ret <16 x i8> %3
+  %1 = call <16 x i8> @llvm.uadd.sat.v16i8(<16 x i8> %a, <16 x i8> %b)
+  ret <16 x i8> %1
 }
 
 define <16 x i8> @test_mask_adds_epu8_rmk_128(<16 x i8> %a, <16 x i8>* %ptr_b, <16 x i8> %passThru, i16 %mask) {
@@ -396,12 +929,10 @@ define <16 x i8> @test_mask_adds_epu8_rmk_128(<16 x i8> %a, <16 x i8>* %ptr_b, <
 ; CHECK-NEXT:    vmovdqa %xmm1, %xmm0 ## EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc1]
 ; CHECK-NEXT:    retq ## encoding: [0xc3]
   %b = load <16 x i8>, <16 x i8>* %ptr_b
-  %1 = add <16 x i8> %a, %b
-  %2 = icmp ugt <16 x i8> %a, %1
-  %3 = select <16 x i1> %2, <16 x i8> <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>, <16 x i8> %1
-  %4 = bitcast i16 %mask to <16 x i1>
-  %5 = select <16 x i1> %4, <16 x i8> %3, <16 x i8> %passThru
-  ret <16 x i8> %5
+  %1 = call <16 x i8> @llvm.uadd.sat.v16i8(<16 x i8> %a, <16 x i8> %b)
+  %2 = bitcast i16 %mask to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x i8> %1, <16 x i8> %passThru
+  ret <16 x i8> %3
 }
 
 define <16 x i8> @test_mask_adds_epu8_rmkz_128(<16 x i8> %a, <16 x i8>* %ptr_b, i16 %mask) {
@@ -411,12 +942,10 @@ define <16 x i8> @test_mask_adds_epu8_rmkz_128(<16 x i8> %a, <16 x i8>* %ptr_b,
 ; CHECK-NEXT:    vpaddusb (%rdi), %xmm0, %xmm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0x89,0xdc,0x07]
 ; CHECK-NEXT:    retq ## encoding: [0xc3]
   %b = load <16 x i8>, <16 x i8>* %ptr_b
-  %1 = add <16 x i8> %a, %b
-  %2 = icmp ugt <16 x i8> %a, %1
-  %3 = select <16 x i1> %2, <16 x i8> <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>, <16 x i8> %1
-  %4 = bitcast i16 %mask to <16 x i1>
-  %5 = select <16 x i1> %4, <16 x i8> %3, <16 x i8> zeroinitializer
-  ret <16 x i8> %5
+  %1 = call <16 x i8> @llvm.uadd.sat.v16i8(<16 x i8> %a, <16 x i8> %b)
+  %2 = bitcast i16 %mask to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x i8> %1, <16 x i8> zeroinitializer
+  ret <16 x i8> %3
 }
 
 define <32 x i8> @test_mask_adds_epu8_rr_256(<32 x i8> %a, <32 x i8> %b) {
@@ -424,11 +953,10 @@ define <32 x i8> @test_mask_adds_epu8_rr_256(<32 x i8> %a, <32 x i8> %b) {
 ; CHECK:       ## %bb.0:
 ; CHECK-NEXT:    vpaddusb %ymm1, %ymm0, %ymm0 ## EVEX TO VEX Compression encoding: [0xc5,0xfd,0xdc,0xc1]
 ; CHECK-NEXT:    retq ## encoding: [0xc3]
-  %1 = add <32 x i8> %a, %b
-  %2 = icmp ugt <32 x i8> %a, %1
-  %3 = select <32 x i1> %2, <32 x i8> <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>, <32 x i8> %1
-  ret <32 x i8> %3
+  %1 = call <32 x i8> @llvm.uadd.sat.v32i8(<32 x i8> %a, <32 x i8> %b)
+  ret <32 x i8> %1
 }
+declare <32 x i8> @llvm.uadd.sat.v32i8(<32 x i8>, <32 x i8>)
 
 define <32 x i8> @test_mask_adds_epu8_rrk_256(<32 x i8> %a, <32 x i8> %b, <32 x i8> %passThru, i32 %mask) {
 ; CHECK-LABEL: test_mask_adds_epu8_rrk_256:
@@ -437,12 +965,10 @@ define <32 x i8> @test_mask_adds_epu8_rrk_256(<32 x i8> %a, <32 x i8> %b, <32 x
 ; CHECK-NEXT:    vpaddusb %ymm1, %ymm0, %ymm2 {%k1} ## encoding: [0x62,0xf1,0x7d,0x29,0xdc,0xd1]
 ; CHECK-NEXT:    vmovdqa %ymm2, %ymm0 ## EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc2]
 ; CHECK-NEXT:    retq ## encoding: [0xc3]
-  %1 = add <32 x i8> %a, %b
-  %2 = icmp ugt <32 x i8> %a, %1
-  %3 = select <32 x i1> %2, <32 x i8> <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>, <32 x i8> %1
-  %4 = bitcast i32 %mask to <32 x i1>
-  %5 = select <32 x i1> %4, <32 x i8> %3, <32 x i8> %passThru
-  ret <32 x i8> %5
+  %1 = call <32 x i8> @llvm.uadd.sat.v32i8(<32 x i8> %a, <32 x i8> %b)
+  %2 = bitcast i32 %mask to <32 x i1>
+  %3 = select <32 x i1> %2, <32 x i8> %1, <32 x i8> %passThru
+  ret <32 x i8> %3
 }
 
 define <32 x i8> @test_mask_adds_epu8_rrkz_256(<32 x i8> %a, <32 x i8> %b, i32 %mask) {
@@ -451,12 +977,10 @@ define <32 x i8> @test_mask_adds_epu8_rrkz_256(<32 x i8> %a, <32 x i8> %b, i32 %
 ; CHECK-NEXT:    kmovd %edi, %k1 ## encoding: [0xc5,0xfb,0x92,0xcf]
 ; CHECK-NEXT:    vpaddusb %ymm1, %ymm0, %ymm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0xa9,0xdc,0xc1]
 ; CHECK-NEXT:    retq ## encoding: [0xc3]
-  %1 = add <32 x i8> %a, %b
-  %2 = icmp ugt <32 x i8> %a, %1
-  %3 = select <32 x i1> %2, <32 x i8> <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>, <32 x i8> %1
-  %4 = bitcast i32 %mask to <32 x i1>
-  %5 = select <32 x i1> %4, <32 x i8> %3, <32 x i8> zeroinitializer
-  ret <32 x i8> %5
+  %1 = call <32 x i8> @llvm.uadd.sat.v32i8(<32 x i8> %a, <32 x i8> %b)
+  %2 = bitcast i32 %mask to <32 x i1>
+  %3 = select <32 x i1> %2, <32 x i8> %1, <32 x i8> zeroinitializer
+  ret <32 x i8> %3
 }
 
 define <32 x i8> @test_mask_adds_epu8_rm_256(<32 x i8> %a, <32 x i8>* %ptr_b) {
@@ -465,10 +989,8 @@ define <32 x i8> @test_mask_adds_epu8_rm_256(<32 x i8> %a, <32 x i8>* %ptr_b) {
 ; CHECK-NEXT:    vpaddusb (%rdi), %ymm0, %ymm0 ## EVEX TO VEX Compression encoding: [0xc5,0xfd,0xdc,0x07]
 ; CHECK-NEXT:    retq ## encoding: [0xc3]
   %b = load <32 x i8>, <32 x i8>* %ptr_b
-  %1 = add <32 x i8> %a, %b
-  %2 = icmp ugt <32 x i8> %a, %1
-  %3 = select <32 x i1> %2, <32 x i8> <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>, <32 x i8> %1
-  ret <32 x i8> %3
+  %1 = call <32 x i8> @llvm.uadd.sat.v32i8(<32 x i8> %a, <32 x i8> %b)
+  ret <32 x i8> %1
 }
 
 define <32 x i8> @test_mask_adds_epu8_rmk_256(<32 x i8> %a, <32 x i8>* %ptr_b, <32 x i8> %passThru, i32 %mask) {
@@ -479,12 +1001,10 @@ define <32 x i8> @test_mask_adds_epu8_rmk_256(<32 x i8> %a, <32 x i8>* %ptr_b, <
 ; CHECK-NEXT:    vmovdqa %ymm1, %ymm0 ## EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc1]
 ; CHECK-NEXT:    retq ## encoding: [0xc3]
   %b = load <32 x i8>, <32 x i8>* %ptr_b
-  %1 = add <32 x i8> %a, %b
-  %2 = icmp ugt <32 x i8> %a, %1
-  %3 = select <32 x i1> %2, <32 x i8> <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>, <32 x i8> %1
-  %4 = bitcast i32 %mask to <32 x i1>
-  %5 = select <32 x i1> %4, <32 x i8> %3, <32 x i8> %passThru
-  ret <32 x i8> %5
+  %1 = call <32 x i8> @llvm.uadd.sat.v32i8(<32 x i8> %a, <32 x i8> %b)
+  %2 = bitcast i32 %mask to <32 x i1>
+  %3 = select <32 x i1> %2, <32 x i8> %1, <32 x i8> %passThru
+  ret <32 x i8> %3
 }
 
 define <32 x i8> @test_mask_adds_epu8_rmkz_256(<32 x i8> %a, <32 x i8>* %ptr_b, i32 %mask) {
@@ -494,12 +1014,10 @@ define <32 x i8> @test_mask_adds_epu8_rmkz_256(<32 x i8> %a, <32 x i8>* %ptr_b,
 ; CHECK-NEXT:    vpaddusb (%rdi), %ymm0, %ymm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0xa9,0xdc,0x07]
 ; CHECK-NEXT:    retq ## encoding: [0xc3]
   %b = load <32 x i8>, <32 x i8>* %ptr_b
-  %1 = add <32 x i8> %a, %b
-  %2 = icmp ugt <32 x i8> %a, %1
-  %3 = select <32 x i1> %2, <32 x i8> <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>, <32 x i8> %1
-  %4 = bitcast i32 %mask to <32 x i1>
-  %5 = select <32 x i1> %4, <32 x i8> %3, <32 x i8> zeroinitializer
-  ret <32 x i8> %5
+  %1 = call <32 x i8> @llvm.uadd.sat.v32i8(<32 x i8> %a, <32 x i8> %b)
+  %2 = bitcast i32 %mask to <32 x i1>
+  %3 = select <32 x i1> %2, <32 x i8> %1, <32 x i8> zeroinitializer
+  ret <32 x i8> %3
 }
 
 define <16 x i8> @test_mask_subs_epu8_rr_128(<16 x i8> %a, <16 x i8> %b) {
@@ -507,11 +1025,10 @@ define <16 x i8> @test_mask_subs_epu8_rr_128(<16 x i8> %a, <16 x i8> %b) {
 ; CHECK:       ## %bb.0:
 ; CHECK-NEXT:    vpsubusb %xmm1, %xmm0, %xmm0 ## EVEX TO VEX Compression encoding: [0xc5,0xf9,0xd8,0xc1]
 ; CHECK-NEXT:    retq ## encoding: [0xc3]
-  %cmp = icmp ugt <16 x i8> %a, %b
-  %sel = select <16 x i1> %cmp, <16 x i8> %a, <16 x i8> %b
-  %sub = sub <16 x i8> %sel, %b
+  %sub = call <16 x i8> @llvm.usub.sat.v16i8(<16 x i8> %a, <16 x i8> %b)
   ret <16 x i8> %sub
 }
+declare <16 x i8> @llvm.usub.sat.v16i8(<16 x i8>, <16 x i8>)
 
 define <16 x i8> @test_mask_subs_epu8_rrk_128(<16 x i8> %a, <16 x i8> %b, <16 x i8> %passThru, i16 %mask) {
 ; CHECK-LABEL: test_mask_subs_epu8_rrk_128:
@@ -520,9 +1037,7 @@ define <16 x i8> @test_mask_subs_epu8_rrk_128(<16 x i8> %a, <16 x i8> %b, <16 x
 ; CHECK-NEXT:    vpsubusb %xmm1, %xmm0, %xmm2 {%k1} ## encoding: [0x62,0xf1,0x7d,0x09,0xd8,0xd1]
 ; CHECK-NEXT:    vmovdqa %xmm2, %xmm0 ## EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc2]
 ; CHECK-NEXT:    retq ## encoding: [0xc3]
-  %cmp = icmp ugt <16 x i8> %a, %b
-  %sel = select <16 x i1> %cmp, <16 x i8> %a, <16 x i8> %b
-  %sub = sub <16 x i8> %sel, %b
+  %sub = call <16 x i8> @llvm.usub.sat.v16i8(<16 x i8> %a, <16 x i8> %b)
   %bc = bitcast i16 %mask to <16 x i1>
   %res = select <16 x i1> %bc, <16 x i8> %sub, <16 x i8> %passThru
   ret <16 x i8> %res
@@ -534,9 +1049,7 @@ define <16 x i8> @test_mask_subs_epu8_rrkz_128(<16 x i8> %a, <16 x i8> %b, i16 %
 ; CHECK-NEXT:    kmovd %edi, %k1 ## encoding: [0xc5,0xfb,0x92,0xcf]
 ; CHECK-NEXT:    vpsubusb %xmm1, %xmm0, %xmm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0x89,0xd8,0xc1]
 ; CHECK-NEXT:    retq ## encoding: [0xc3]
-  %cmp = icmp ugt <16 x i8> %a, %b
-  %sel = select <16 x i1> %cmp, <16 x i8> %a, <16 x i8> %b
-  %sub = sub <16 x i8> %sel, %b
+  %sub = call <16 x i8> @llvm.usub.sat.v16i8(<16 x i8> %a, <16 x i8> %b)
   %bc = bitcast i16 %mask to <16 x i1>
   %res = select <16 x i1> %bc, <16 x i8> %sub, <16 x i8> zeroinitializer
   ret <16 x i8> %res
@@ -548,9 +1061,7 @@ define <16 x i8> @test_mask_subs_epu8_rm_128(<16 x i8> %a, <16 x i8>* %ptr_b) {
 ; CHECK-NEXT:    vpsubusb (%rdi), %xmm0, %xmm0 ## EVEX TO VEX Compression encoding: [0xc5,0xf9,0xd8,0x07]
 ; CHECK-NEXT:    retq ## encoding: [0xc3]
   %b = load <16 x i8>, <16 x i8>* %ptr_b
-  %cmp = icmp ugt <16 x i8> %a, %b
-  %sel = select <16 x i1> %cmp, <16 x i8> %a, <16 x i8> %b
-  %sub = sub <16 x i8> %sel, %b
+  %sub = call <16 x i8> @llvm.usub.sat.v16i8(<16 x i8> %a, <16 x i8> %b)
   ret <16 x i8> %sub
 }
 
@@ -562,9 +1073,7 @@ define <16 x i8> @test_mask_subs_epu8_rmk_128(<16 x i8> %a, <16 x i8>* %ptr_b, <
 ; CHECK-NEXT:    vmovdqa %xmm1, %xmm0 ## EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc1]
 ; CHECK-NEXT:    retq ## encoding: [0xc3]
   %b = load <16 x i8>, <16 x i8>* %ptr_b
-  %cmp = icmp ugt <16 x i8> %a, %b
-  %sel = select <16 x i1> %cmp, <16 x i8> %a, <16 x i8> %b
-  %sub = sub <16 x i8> %sel, %b
+  %sub = call <16 x i8> @llvm.usub.sat.v16i8(<16 x i8> %a, <16 x i8> %b)
   %bc = bitcast i16 %mask to <16 x i1>
   %res = select <16 x i1> %bc, <16 x i8> %sub, <16 x i8> %passThru
   ret <16 x i8> %res
@@ -577,9 +1086,7 @@ define <16 x i8> @test_mask_subs_epu8_rmkz_128(<16 x i8> %a, <16 x i8>* %ptr_b,
 ; CHECK-NEXT:    vpsubusb (%rdi), %xmm0, %xmm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0x89,0xd8,0x07]
 ; CHECK-NEXT:    retq ## encoding: [0xc3]
   %b = load <16 x i8>, <16 x i8>* %ptr_b
-  %cmp = icmp ugt <16 x i8> %a, %b
-  %sel = select <16 x i1> %cmp, <16 x i8> %a, <16 x i8> %b
-  %sub = sub <16 x i8> %sel, %b
+  %sub = call <16 x i8> @llvm.usub.sat.v16i8(<16 x i8> %a, <16 x i8> %b)
   %bc = bitcast i16 %mask to <16 x i1>
   %res = select <16 x i1> %bc, <16 x i8> %sub, <16 x i8> zeroinitializer
   ret <16 x i8> %res
@@ -590,11 +1097,10 @@ define <32 x i8> @test_mask_subs_epu8_rr_256(<32 x i8> %a, <32 x i8> %b) {
 ; CHECK:       ## %bb.0:
 ; CHECK-NEXT:    vpsubusb %ymm1, %ymm0, %ymm0 ## EVEX TO VEX Compression encoding: [0xc5,0xfd,0xd8,0xc1]
 ; CHECK-NEXT:    retq ## encoding: [0xc3]
-  %cmp = icmp ugt <32 x i8> %a, %b
-  %sel = select <32 x i1> %cmp, <32 x i8> %a, <32 x i8> %b
-  %sub = sub <32 x i8> %sel, %b
+  %sub = call <32 x i8> @llvm.usub.sat.v32i8(<32 x i8> %a, <32 x i8> %b)
   ret <32 x i8> %sub
 }
+declare <32 x i8> @llvm.usub.sat.v32i8(<32 x i8>, <32 x i8>)
 
 define <32 x i8> @test_mask_subs_epu8_rrk_256(<32 x i8> %a, <32 x i8> %b, <32 x i8> %passThru, i32 %mask) {
 ; CHECK-LABEL: test_mask_subs_epu8_rrk_256:
@@ -603,9 +1109,7 @@ define <32 x i8> @test_mask_subs_epu8_rrk_256(<32 x i8> %a, <32 x i8> %b, <32 x
 ; CHECK-NEXT:    vpsubusb %ymm1, %ymm0, %ymm2 {%k1} ## encoding: [0x62,0xf1,0x7d,0x29,0xd8,0xd1]
 ; CHECK-NEXT:    vmovdqa %ymm2, %ymm0 ## EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc2]
 ; CHECK-NEXT:    retq ## encoding: [0xc3]
-  %cmp = icmp ugt <32 x i8> %a, %b
-  %sel = select <32 x i1> %cmp, <32 x i8> %a, <32 x i8> %b
-  %sub = sub <32 x i8> %sel, %b
+  %sub = call <32 x i8> @llvm.usub.sat.v32i8(<32 x i8> %a, <32 x i8> %b)
   %bc = bitcast i32 %mask to <32 x i1>
   %res = select <32 x i1> %bc, <32 x i8> %sub, <32 x i8> %passThru
   ret <32 x i8> %res
@@ -617,9 +1121,7 @@ define <32 x i8> @test_mask_subs_epu8_rrkz_256(<32 x i8> %a, <32 x i8> %b, i32 %
 ; CHECK-NEXT:    kmovd %edi, %k1 ## encoding: [0xc5,0xfb,0x92,0xcf]
 ; CHECK-NEXT:    vpsubusb %ymm1, %ymm0, %ymm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0xa9,0xd8,0xc1]
 ; CHECK-NEXT:    retq ## encoding: [0xc3]
-  %cmp = icmp ugt <32 x i8> %a, %b
-  %sel = select <32 x i1> %cmp, <32 x i8> %a, <32 x i8> %b
-  %sub = sub <32 x i8> %sel, %b
+  %sub = call <32 x i8> @llvm.usub.sat.v32i8(<32 x i8> %a, <32 x i8> %b)
   %bc = bitcast i32 %mask to <32 x i1>
   %res = select <32 x i1> %bc, <32 x i8> %sub, <32 x i8> zeroinitializer
   ret <32 x i8> %res
@@ -631,9 +1133,7 @@ define <32 x i8> @test_mask_subs_epu8_rm_256(<32 x i8> %a, <32 x i8>* %ptr_b) {
 ; CHECK-NEXT:    vpsubusb (%rdi), %ymm0, %ymm0 ## EVEX TO VEX Compression encoding: [0xc5,0xfd,0xd8,0x07]
 ; CHECK-NEXT:    retq ## encoding: [0xc3]
   %b = load <32 x i8>, <32 x i8>* %ptr_b
-  %cmp = icmp ugt <32 x i8> %a, %b
-  %sel = select <32 x i1> %cmp, <32 x i8> %a, <32 x i8> %b
-  %sub = sub <32 x i8> %sel, %b
+  %sub = call <32 x i8> @llvm.usub.sat.v32i8(<32 x i8> %a, <32 x i8> %b)
   ret <32 x i8> %sub
 }
 
@@ -645,9 +1145,7 @@ define <32 x i8> @test_mask_subs_epu8_rmk_256(<32 x i8> %a, <32 x i8>* %ptr_b, <
 ; CHECK-NEXT:    vmovdqa %ymm1, %ymm0 ## EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc1]
 ; CHECK-NEXT:    retq ## encoding: [0xc3]
   %b = load <32 x i8>, <32 x i8>* %ptr_b
-  %cmp = icmp ugt <32 x i8> %a, %b
-  %sel = select <32 x i1> %cmp, <32 x i8> %a, <32 x i8> %b
-  %sub = sub <32 x i8> %sel, %b
+  %sub = call <32 x i8> @llvm.usub.sat.v32i8(<32 x i8> %a, <32 x i8> %b)
   %bc = bitcast i32 %mask to <32 x i1>
   %res = select <32 x i1> %bc, <32 x i8> %sub, <32 x i8> %passThru
   ret <32 x i8> %res
@@ -660,9 +1158,7 @@ define <32 x i8> @test_mask_subs_epu8_rmkz_256(<32 x i8> %a, <32 x i8>* %ptr_b,
 ; CHECK-NEXT:    vpsubusb (%rdi), %ymm0, %ymm0 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0xa9,0xd8,0x07]
 ; CHECK-NEXT:    retq ## encoding: [0xc3]
   %b = load <32 x i8>, <32 x i8>* %ptr_b
-  %cmp = icmp ugt <32 x i8> %a, %b
-  %sel = select <32 x i1> %cmp, <32 x i8> %a, <32 x i8> %b
-  %sub = sub <32 x i8> %sel, %b
+  %sub = call <32 x i8> @llvm.usub.sat.v32i8(<32 x i8> %a, <32 x i8> %b)
   %bc = bitcast i32 %mask to <32 x i1>
   %res = select <32 x i1> %bc, <32 x i8> %sub, <32 x i8> zeroinitializer
   ret <32 x i8> %res
diff --git a/test/CodeGen/X86/avx512bwvl-intrinsics-upgrade.ll b/test/CodeGen/X86/avx512bwvl-intrinsics-upgrade.ll
index 21231b071092fd81a7d31cf0457c34e68cfb2edd..0c20b2679848382306fad892c47210f94e681d02 100644
--- a/test/CodeGen/X86/avx512bwvl-intrinsics-upgrade.ll
+++ b/test/CodeGen/X86/avx512bwvl-intrinsics-upgrade.ll
@@ -7348,6 +7348,338 @@ define <16 x i16> @test_mask_adds_epi16_rmkz_256(<16 x i16> %a, <16 x i16>* %ptr
 
 declare <16 x i16> @llvm.x86.avx512.mask.padds.w.256(<16 x i16>, <16 x i16>, <16 x i16>, i16)
 
+declare <8 x i16> @llvm.x86.sse2.psubs.w(<8 x i16>, <8 x i16>) #0
+
+define <8 x i16> @test_test_subs_epi16_rrk_128(<8 x i16> %a, <8 x i16> %b, <8 x i16> %passThru, i8 %mask) {
+; X86-LABEL: test_test_subs_epi16_rrk_128:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
+; X86-NEXT:    vpsubsw %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf1,0x7d,0x09,0xe9,0xd1]
+; X86-NEXT:    vmovdqa %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc2]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_test_subs_epi16_rrk_128:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; X64-NEXT:    vpsubsw %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf1,0x7d,0x09,0xe9,0xd1]
+; X64-NEXT:    vmovdqa %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc2]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %1 = call <8 x i16> @llvm.x86.sse2.psubs.w(<8 x i16> %a, <8 x i16> %b)
+  %2 = bitcast i8 %mask to <8 x i1>
+  %3 = select <8 x i1> %2, <8 x i16> %1, <8 x i16> %passThru
+  ret <8 x i16> %3
+}
+
+define <8 x i16> @test_test_subs_epi16_rrkz_128(<8 x i16> %a, <8 x i16> %b, i8 %mask) {
+; X86-LABEL: test_test_subs_epi16_rrkz_128:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
+; X86-NEXT:    vpsubsw %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0x89,0xe9,0xc1]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_test_subs_epi16_rrkz_128:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; X64-NEXT:    vpsubsw %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0x89,0xe9,0xc1]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %1 = call <8 x i16> @llvm.x86.sse2.psubs.w(<8 x i16> %a, <8 x i16> %b)
+  %2 = bitcast i8 %mask to <8 x i1>
+  %3 = select <8 x i1> %2, <8 x i16> %1, <8 x i16> zeroinitializer
+  ret <8 x i16> %3
+}
+
+define <8 x i16> @test_test_subs_epi16_rmk_128(<8 x i16> %a, <8 x i16>* %ptr_b, <8 x i16> %passThru, i8 %mask) {
+; X86-LABEL: test_test_subs_epi16_rmk_128:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx # encoding: [0x0f,0xb6,0x4c,0x24,0x08]
+; X86-NEXT:    kmovd %ecx, %k1 # encoding: [0xc5,0xfb,0x92,0xc9]
+; X86-NEXT:    vpsubsw (%eax), %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf1,0x7d,0x09,0xe9,0x08]
+; X86-NEXT:    vmovdqa %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc1]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_test_subs_epi16_rmk_128:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovd %esi, %k1 # encoding: [0xc5,0xfb,0x92,0xce]
+; X64-NEXT:    vpsubsw (%rdi), %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf1,0x7d,0x09,0xe9,0x0f]
+; X64-NEXT:    vmovdqa %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc1]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %b = load <8 x i16>, <8 x i16>* %ptr_b
+  %1 = call <8 x i16> @llvm.x86.sse2.psubs.w(<8 x i16> %a, <8 x i16> %b)
+  %2 = bitcast i8 %mask to <8 x i1>
+  %3 = select <8 x i1> %2, <8 x i16> %1, <8 x i16> %passThru
+  ret <8 x i16> %3
+}
+
+define <8 x i16> @test_test_subs_epi16_rmkz_128(<8 x i16> %a, <8 x i16>* %ptr_b, i8 %mask) {
+; X86-LABEL: test_test_subs_epi16_rmkz_128:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx # encoding: [0x0f,0xb6,0x4c,0x24,0x08]
+; X86-NEXT:    kmovd %ecx, %k1 # encoding: [0xc5,0xfb,0x92,0xc9]
+; X86-NEXT:    vpsubsw (%eax), %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0x89,0xe9,0x00]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_test_subs_epi16_rmkz_128:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovd %esi, %k1 # encoding: [0xc5,0xfb,0x92,0xce]
+; X64-NEXT:    vpsubsw (%rdi), %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0x89,0xe9,0x07]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %b = load <8 x i16>, <8 x i16>* %ptr_b
+  %1 = call <8 x i16> @llvm.x86.sse2.psubs.w(<8 x i16> %a, <8 x i16> %b)
+  %2 = bitcast i8 %mask to <8 x i1>
+  %3 = select <8 x i1> %2, <8 x i16> %1, <8 x i16> zeroinitializer
+  ret <8 x i16> %3
+}
+
+declare <16 x i16> @llvm.x86.avx2.psubs.w(<16 x i16>, <16 x i16>) #0
+
+define <16 x i16> @test_test_subs_epi16_rrk_256(<16 x i16> %a, <16 x i16> %b, <16 x i16> %passThru, i16 %mask) {
+; X86-LABEL: test_test_subs_epi16_rrk_256:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
+; X86-NEXT:    vpsubsw %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf1,0x7d,0x29,0xe9,0xd1]
+; X86-NEXT:    vmovdqa %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc2]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_test_subs_epi16_rrk_256:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; X64-NEXT:    vpsubsw %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf1,0x7d,0x29,0xe9,0xd1]
+; X64-NEXT:    vmovdqa %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc2]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %1 = call <16 x i16> @llvm.x86.avx2.psubs.w(<16 x i16> %a, <16 x i16> %b)
+  %2 = bitcast i16 %mask to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x i16> %1, <16 x i16> %passThru
+  ret <16 x i16> %3
+}
+
+define <16 x i16> @test_test_subs_epi16_rrkz_256(<16 x i16> %a, <16 x i16> %b, i16 %mask) {
+; X86-LABEL: test_test_subs_epi16_rrkz_256:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
+; X86-NEXT:    vpsubsw %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xa9,0xe9,0xc1]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_test_subs_epi16_rrkz_256:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; X64-NEXT:    vpsubsw %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xa9,0xe9,0xc1]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %1 = call <16 x i16> @llvm.x86.avx2.psubs.w(<16 x i16> %a, <16 x i16> %b)
+  %2 = bitcast i16 %mask to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x i16> %1, <16 x i16> zeroinitializer
+  ret <16 x i16> %3
+}
+
+define <16 x i16> @test_test_subs_epi16_rmk_256(<16 x i16> %a, <16 x i16>* %ptr_b, <16 x i16> %passThru, i16 %mask) {
+; X86-LABEL: test_test_subs_epi16_rmk_256:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x08]
+; X86-NEXT:    vpsubsw (%eax), %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf1,0x7d,0x29,0xe9,0x08]
+; X86-NEXT:    vmovdqa %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc1]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_test_subs_epi16_rmk_256:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovd %esi, %k1 # encoding: [0xc5,0xfb,0x92,0xce]
+; X64-NEXT:    vpsubsw (%rdi), %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf1,0x7d,0x29,0xe9,0x0f]
+; X64-NEXT:    vmovdqa %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc1]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %b = load <16 x i16>, <16 x i16>* %ptr_b
+  %1 = call <16 x i16> @llvm.x86.avx2.psubs.w(<16 x i16> %a, <16 x i16> %b)
+  %2 = bitcast i16 %mask to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x i16> %1, <16 x i16> %passThru
+  ret <16 x i16> %3
+}
+
+define <16 x i16> @test_test_subs_epi16_rmkz_256(<16 x i16> %a, <16 x i16>* %ptr_b, i16 %mask) {
+; X86-LABEL: test_test_subs_epi16_rmkz_256:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x08]
+; X86-NEXT:    vpsubsw (%eax), %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xa9,0xe9,0x00]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_test_subs_epi16_rmkz_256:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovd %esi, %k1 # encoding: [0xc5,0xfb,0x92,0xce]
+; X64-NEXT:    vpsubsw (%rdi), %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xa9,0xe9,0x07]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %b = load <16 x i16>, <16 x i16>* %ptr_b
+  %1 = call <16 x i16> @llvm.x86.avx2.psubs.w(<16 x i16> %a, <16 x i16> %b)
+  %2 = bitcast i16 %mask to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x i16> %1, <16 x i16> zeroinitializer
+  ret <16 x i16> %3
+}
+
+declare <16 x i8> @llvm.x86.sse2.psubs.b(<16 x i8>, <16 x i8>) #0
+
+define <16 x i8> @test_test_subs_epi8_rrk_128(<16 x i8> %a, <16 x i8> %b, <16 x i8> %passThru, i16 %mask) {
+; X86-LABEL: test_test_subs_epi8_rrk_128:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
+; X86-NEXT:    vpsubsb %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf1,0x7d,0x09,0xe8,0xd1]
+; X86-NEXT:    vmovdqa %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc2]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_test_subs_epi8_rrk_128:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; X64-NEXT:    vpsubsb %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf1,0x7d,0x09,0xe8,0xd1]
+; X64-NEXT:    vmovdqa %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc2]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %1 = call <16 x i8> @llvm.x86.sse2.psubs.b(<16 x i8> %a, <16 x i8> %b)
+  %2 = bitcast i16 %mask to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x i8> %1, <16 x i8> %passThru
+  ret <16 x i8> %3
+}
+
+define <16 x i8> @test_test_subs_epi8_rrkz_128(<16 x i8> %a, <16 x i8> %b, i16 %mask) {
+; X86-LABEL: test_test_subs_epi8_rrkz_128:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
+; X86-NEXT:    vpsubsb %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0x89,0xe8,0xc1]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_test_subs_epi8_rrkz_128:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; X64-NEXT:    vpsubsb %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0x89,0xe8,0xc1]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %1 = call <16 x i8> @llvm.x86.sse2.psubs.b(<16 x i8> %a, <16 x i8> %b)
+  %2 = bitcast i16 %mask to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x i8> %1, <16 x i8> zeroinitializer
+  ret <16 x i8> %3
+}
+
+define <16 x i8> @test_test_subs_epi8_rmk_128(<16 x i8> %a, <16 x i8>* %ptr_b, <16 x i8> %passThru, i16 %mask) {
+; X86-LABEL: test_test_subs_epi8_rmk_128:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x08]
+; X86-NEXT:    vpsubsb (%eax), %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf1,0x7d,0x09,0xe8,0x08]
+; X86-NEXT:    vmovdqa %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc1]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_test_subs_epi8_rmk_128:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovd %esi, %k1 # encoding: [0xc5,0xfb,0x92,0xce]
+; X64-NEXT:    vpsubsb (%rdi), %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf1,0x7d,0x09,0xe8,0x0f]
+; X64-NEXT:    vmovdqa %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc1]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %b = load <16 x i8>, <16 x i8>* %ptr_b
+  %1 = call <16 x i8> @llvm.x86.sse2.psubs.b(<16 x i8> %a, <16 x i8> %b)
+  %2 = bitcast i16 %mask to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x i8> %1, <16 x i8> %passThru
+  ret <16 x i8> %3
+}
+
+define <16 x i8> @test_test_subs_epi8_rmkz_128(<16 x i8> %a, <16 x i8>* %ptr_b, i16 %mask) {
+; X86-LABEL: test_test_subs_epi8_rmkz_128:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x08]
+; X86-NEXT:    vpsubsb (%eax), %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0x89,0xe8,0x00]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_test_subs_epi8_rmkz_128:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovd %esi, %k1 # encoding: [0xc5,0xfb,0x92,0xce]
+; X64-NEXT:    vpsubsb (%rdi), %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0x89,0xe8,0x07]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %b = load <16 x i8>, <16 x i8>* %ptr_b
+  %1 = call <16 x i8> @llvm.x86.sse2.psubs.b(<16 x i8> %a, <16 x i8> %b)
+  %2 = bitcast i16 %mask to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x i8> %1, <16 x i8> zeroinitializer
+  ret <16 x i8> %3
+}
+
+declare <32 x i8> @llvm.x86.avx2.psubs.b(<32 x i8>, <32 x i8>) #0
+
+define <32 x i8> @test_test_subs_epi8_rrk_256(<32 x i8> %a, <32 x i8> %b, <32 x i8> %passThru, i32 %mask) {
+; X86-LABEL: test_test_subs_epi8_rrk_256:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf9,0x90,0x4c,0x24,0x04]
+; X86-NEXT:    vpsubsb %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf1,0x7d,0x29,0xe8,0xd1]
+; X86-NEXT:    vmovdqa %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc2]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_test_subs_epi8_rrk_256:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; X64-NEXT:    vpsubsb %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf1,0x7d,0x29,0xe8,0xd1]
+; X64-NEXT:    vmovdqa %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc2]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %1 = call <32 x i8> @llvm.x86.avx2.psubs.b(<32 x i8> %a, <32 x i8> %b)
+  %2 = bitcast i32 %mask to <32 x i1>
+  %3 = select <32 x i1> %2, <32 x i8> %1, <32 x i8> %passThru
+  ret <32 x i8> %3
+}
+
+define <32 x i8> @test_test_subs_epi8_rrkz_256(<32 x i8> %a, <32 x i8> %b, i32 %mask) {
+; X86-LABEL: test_test_subs_epi8_rrkz_256:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf9,0x90,0x4c,0x24,0x04]
+; X86-NEXT:    vpsubsb %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xa9,0xe8,0xc1]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_test_subs_epi8_rrkz_256:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; X64-NEXT:    vpsubsb %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xa9,0xe8,0xc1]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %1 = call <32 x i8> @llvm.x86.avx2.psubs.b(<32 x i8> %a, <32 x i8> %b)
+  %2 = bitcast i32 %mask to <32 x i1>
+  %3 = select <32 x i1> %2, <32 x i8> %1, <32 x i8> zeroinitializer
+  ret <32 x i8> %3
+}
+
+define <32 x i8> @test_test_subs_epi8_rmk_256(<32 x i8> %a, <32 x i8>* %ptr_b, <32 x i8> %passThru, i32 %mask) {
+; X86-LABEL: test_test_subs_epi8_rmk_256:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf9,0x90,0x4c,0x24,0x08]
+; X86-NEXT:    vpsubsb (%eax), %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf1,0x7d,0x29,0xe8,0x08]
+; X86-NEXT:    vmovdqa %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc1]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_test_subs_epi8_rmk_256:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovd %esi, %k1 # encoding: [0xc5,0xfb,0x92,0xce]
+; X64-NEXT:    vpsubsb (%rdi), %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf1,0x7d,0x29,0xe8,0x0f]
+; X64-NEXT:    vmovdqa %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc1]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %b = load <32 x i8>, <32 x i8>* %ptr_b
+  %1 = call <32 x i8> @llvm.x86.avx2.psubs.b(<32 x i8> %a, <32 x i8> %b)
+  %2 = bitcast i32 %mask to <32 x i1>
+  %3 = select <32 x i1> %2, <32 x i8> %1, <32 x i8> %passThru
+  ret <32 x i8> %3
+}
+
+define <32 x i8> @test_test_subs_epi8_rmkz_256(<32 x i8> %a, <32 x i8>* %ptr_b, i32 %mask) {
+; X86-LABEL: test_test_subs_epi8_rmkz_256:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf9,0x90,0x4c,0x24,0x08]
+; X86-NEXT:    vpsubsb (%eax), %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xa9,0xe8,0x00]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_test_subs_epi8_rmkz_256:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovd %esi, %k1 # encoding: [0xc5,0xfb,0x92,0xce]
+; X64-NEXT:    vpsubsb (%rdi), %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xa9,0xe8,0x07]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %b = load <32 x i8>, <32 x i8>* %ptr_b
+  %1 = call <32 x i8> @llvm.x86.avx2.psubs.b(<32 x i8> %a, <32 x i8> %b)
+  %2 = bitcast i32 %mask to <32 x i1>
+  %3 = select <32 x i1> %2, <32 x i8> %1, <32 x i8> zeroinitializer
+  ret <32 x i8> %3
+}
+
 define <8 x i16> @test_mask_subs_epi16_rr_128(<8 x i16> %a, <8 x i16> %b) {
 ; CHECK-LABEL: test_mask_subs_epi16_rr_128:
 ; CHECK:       # %bb.0:
@@ -7366,190 +7698,538 @@ define <8 x i16> @test_mask_subs_epi16_rrk_128(<8 x i16> %a, <8 x i16> %b, <8 x
 ; X86-NEXT:    vmovdqa %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc2]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
-; X64-LABEL: test_mask_subs_epi16_rrk_128:
+; X64-LABEL: test_mask_subs_epi16_rrk_128:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; X64-NEXT:    vpsubsw %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf1,0x7d,0x09,0xe9,0xd1]
+; X64-NEXT:    vmovdqa %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc2]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <8 x i16> @llvm.x86.avx512.mask.psubs.w.128(<8 x i16> %a, <8 x i16> %b, <8 x i16> %passThru, i8 %mask)
+  ret <8 x i16> %res
+}
+
+define <8 x i16> @test_mask_subs_epi16_rrkz_128(<8 x i16> %a, <8 x i16> %b, i8 %mask) {
+; X86-LABEL: test_mask_subs_epi16_rrkz_128:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
+; X86-NEXT:    vpsubsw %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0x89,0xe9,0xc1]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_mask_subs_epi16_rrkz_128:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; X64-NEXT:    vpsubsw %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0x89,0xe9,0xc1]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <8 x i16> @llvm.x86.avx512.mask.psubs.w.128(<8 x i16> %a, <8 x i16> %b, <8 x i16> zeroinitializer, i8 %mask)
+  ret <8 x i16> %res
+}
+
+define <8 x i16> @test_mask_subs_epi16_rm_128(<8 x i16> %a, <8 x i16>* %ptr_b) {
+; X86-LABEL: test_mask_subs_epi16_rm_128:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-NEXT:    vpsubsw (%eax), %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xe9,0x00]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_mask_subs_epi16_rm_128:
+; X64:       # %bb.0:
+; X64-NEXT:    vpsubsw (%rdi), %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xe9,0x07]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %b = load <8 x i16>, <8 x i16>* %ptr_b
+  %res = call <8 x i16> @llvm.x86.avx512.mask.psubs.w.128(<8 x i16> %a, <8 x i16> %b, <8 x i16> zeroinitializer, i8 -1)
+  ret <8 x i16> %res
+}
+
+define <8 x i16> @test_mask_subs_epi16_rmk_128(<8 x i16> %a, <8 x i16>* %ptr_b, <8 x i16> %passThru, i8 %mask) {
+; X86-LABEL: test_mask_subs_epi16_rmk_128:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx # encoding: [0x0f,0xb6,0x4c,0x24,0x08]
+; X86-NEXT:    kmovd %ecx, %k1 # encoding: [0xc5,0xfb,0x92,0xc9]
+; X86-NEXT:    vpsubsw (%eax), %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf1,0x7d,0x09,0xe9,0x08]
+; X86-NEXT:    vmovdqa %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc1]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_mask_subs_epi16_rmk_128:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovd %esi, %k1 # encoding: [0xc5,0xfb,0x92,0xce]
+; X64-NEXT:    vpsubsw (%rdi), %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf1,0x7d,0x09,0xe9,0x0f]
+; X64-NEXT:    vmovdqa %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc1]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %b = load <8 x i16>, <8 x i16>* %ptr_b
+  %res = call <8 x i16> @llvm.x86.avx512.mask.psubs.w.128(<8 x i16> %a, <8 x i16> %b, <8 x i16> %passThru, i8 %mask)
+  ret <8 x i16> %res
+}
+
+define <8 x i16> @test_mask_subs_epi16_rmkz_128(<8 x i16> %a, <8 x i16>* %ptr_b, i8 %mask) {
+; X86-LABEL: test_mask_subs_epi16_rmkz_128:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx # encoding: [0x0f,0xb6,0x4c,0x24,0x08]
+; X86-NEXT:    kmovd %ecx, %k1 # encoding: [0xc5,0xfb,0x92,0xc9]
+; X86-NEXT:    vpsubsw (%eax), %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0x89,0xe9,0x00]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_mask_subs_epi16_rmkz_128:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovd %esi, %k1 # encoding: [0xc5,0xfb,0x92,0xce]
+; X64-NEXT:    vpsubsw (%rdi), %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0x89,0xe9,0x07]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %b = load <8 x i16>, <8 x i16>* %ptr_b
+  %res = call <8 x i16> @llvm.x86.avx512.mask.psubs.w.128(<8 x i16> %a, <8 x i16> %b, <8 x i16> zeroinitializer, i8 %mask)
+  ret <8 x i16> %res
+}
+
+declare <8 x i16> @llvm.x86.avx512.mask.psubs.w.128(<8 x i16>, <8 x i16>, <8 x i16>, i8)
+
+define <16 x i16> @test_mask_subs_epi16_rr_256(<16 x i16> %a, <16 x i16> %b) {
+; CHECK-LABEL: test_mask_subs_epi16_rr_256:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpsubsw %ymm1, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xe9,0xc1]
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <16 x i16> @llvm.x86.avx512.mask.psubs.w.256(<16 x i16> %a, <16 x i16> %b, <16 x i16> zeroinitializer, i16 -1)
+  ret <16 x i16> %res
+}
+
+define <16 x i16> @test_mask_subs_epi16_rrk_256(<16 x i16> %a, <16 x i16> %b, <16 x i16> %passThru, i16 %mask) {
+; X86-LABEL: test_mask_subs_epi16_rrk_256:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
+; X86-NEXT:    vpsubsw %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf1,0x7d,0x29,0xe9,0xd1]
+; X86-NEXT:    vmovdqa %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc2]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_mask_subs_epi16_rrk_256:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; X64-NEXT:    vpsubsw %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf1,0x7d,0x29,0xe9,0xd1]
+; X64-NEXT:    vmovdqa %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc2]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <16 x i16> @llvm.x86.avx512.mask.psubs.w.256(<16 x i16> %a, <16 x i16> %b, <16 x i16> %passThru, i16 %mask)
+  ret <16 x i16> %res
+}
+
+define <16 x i16> @test_mask_subs_epi16_rrkz_256(<16 x i16> %a, <16 x i16> %b, i16 %mask) {
+; X86-LABEL: test_mask_subs_epi16_rrkz_256:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
+; X86-NEXT:    vpsubsw %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xa9,0xe9,0xc1]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_mask_subs_epi16_rrkz_256:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; X64-NEXT:    vpsubsw %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xa9,0xe9,0xc1]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <16 x i16> @llvm.x86.avx512.mask.psubs.w.256(<16 x i16> %a, <16 x i16> %b, <16 x i16> zeroinitializer, i16 %mask)
+  ret <16 x i16> %res
+}
+
+define <16 x i16> @test_mask_subs_epi16_rm_256(<16 x i16> %a, <16 x i16>* %ptr_b) {
+; X86-LABEL: test_mask_subs_epi16_rm_256:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-NEXT:    vpsubsw (%eax), %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xe9,0x00]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_mask_subs_epi16_rm_256:
+; X64:       # %bb.0:
+; X64-NEXT:    vpsubsw (%rdi), %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xe9,0x07]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %b = load <16 x i16>, <16 x i16>* %ptr_b
+  %res = call <16 x i16> @llvm.x86.avx512.mask.psubs.w.256(<16 x i16> %a, <16 x i16> %b, <16 x i16> zeroinitializer, i16 -1)
+  ret <16 x i16> %res
+}
+
+define <16 x i16> @test_mask_subs_epi16_rmk_256(<16 x i16> %a, <16 x i16>* %ptr_b, <16 x i16> %passThru, i16 %mask) {
+; X86-LABEL: test_mask_subs_epi16_rmk_256:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x08]
+; X86-NEXT:    vpsubsw (%eax), %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf1,0x7d,0x29,0xe9,0x08]
+; X86-NEXT:    vmovdqa %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc1]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_mask_subs_epi16_rmk_256:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovd %esi, %k1 # encoding: [0xc5,0xfb,0x92,0xce]
+; X64-NEXT:    vpsubsw (%rdi), %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf1,0x7d,0x29,0xe9,0x0f]
+; X64-NEXT:    vmovdqa %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc1]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %b = load <16 x i16>, <16 x i16>* %ptr_b
+  %res = call <16 x i16> @llvm.x86.avx512.mask.psubs.w.256(<16 x i16> %a, <16 x i16> %b, <16 x i16> %passThru, i16 %mask)
+  ret <16 x i16> %res
+}
+
+define <16 x i16> @test_mask_subs_epi16_rmkz_256(<16 x i16> %a, <16 x i16>* %ptr_b, i16 %mask) {
+; X86-LABEL: test_mask_subs_epi16_rmkz_256:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x08]
+; X86-NEXT:    vpsubsw (%eax), %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xa9,0xe9,0x00]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_mask_subs_epi16_rmkz_256:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovd %esi, %k1 # encoding: [0xc5,0xfb,0x92,0xce]
+; X64-NEXT:    vpsubsw (%rdi), %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xa9,0xe9,0x07]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %b = load <16 x i16>, <16 x i16>* %ptr_b
+  %res = call <16 x i16> @llvm.x86.avx512.mask.psubs.w.256(<16 x i16> %a, <16 x i16> %b, <16 x i16> zeroinitializer, i16 %mask)
+  ret <16 x i16> %res
+}
+
+declare <16 x i16> @llvm.x86.avx512.mask.psubs.w.256(<16 x i16>, <16 x i16>, <16 x i16>, i16)
+
+declare <8 x i16> @llvm.x86.sse2.padds.w(<8 x i16>, <8 x i16>) #0
+
+define <8 x i16> @test_adds_epi16_rrk_128(<8 x i16> %a, <8 x i16> %b, <8 x i16> %passThru, i8 %mask) {
+; X86-LABEL: test_adds_epi16_rrk_128:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
+; X86-NEXT:    vpaddsw %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf1,0x7d,0x09,0xed,0xd1]
+; X86-NEXT:    vmovdqa %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc2]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_adds_epi16_rrk_128:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; X64-NEXT:    vpaddsw %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf1,0x7d,0x09,0xed,0xd1]
+; X64-NEXT:    vmovdqa %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc2]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %1 = call <8 x i16> @llvm.x86.sse2.padds.w(<8 x i16> %a, <8 x i16> %b)
+  %2 = bitcast i8 %mask to <8 x i1>
+  %3 = select <8 x i1> %2, <8 x i16> %1, <8 x i16> %passThru
+  ret <8 x i16> %3
+}
+
+define <8 x i16> @test_adds_epi16_rrkz_128(<8 x i16> %a, <8 x i16> %b, i8 %mask) {
+; X86-LABEL: test_adds_epi16_rrkz_128:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
+; X86-NEXT:    vpaddsw %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0x89,0xed,0xc1]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_adds_epi16_rrkz_128:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; X64-NEXT:    vpaddsw %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0x89,0xed,0xc1]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %1 = call <8 x i16> @llvm.x86.sse2.padds.w(<8 x i16> %a, <8 x i16> %b)
+  %2 = bitcast i8 %mask to <8 x i1>
+  %3 = select <8 x i1> %2, <8 x i16> %1, <8 x i16> zeroinitializer
+  ret <8 x i16> %3
+}
+
+define <8 x i16> @test_adds_epi16_rmk_128(<8 x i16> %a, <8 x i16>* %ptr_b, <8 x i16> %passThru, i8 %mask) {
+; X86-LABEL: test_adds_epi16_rmk_128:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx # encoding: [0x0f,0xb6,0x4c,0x24,0x08]
+; X86-NEXT:    kmovd %ecx, %k1 # encoding: [0xc5,0xfb,0x92,0xc9]
+; X86-NEXT:    vpaddsw (%eax), %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf1,0x7d,0x09,0xed,0x08]
+; X86-NEXT:    vmovdqa %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc1]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_adds_epi16_rmk_128:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovd %esi, %k1 # encoding: [0xc5,0xfb,0x92,0xce]
+; X64-NEXT:    vpaddsw (%rdi), %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf1,0x7d,0x09,0xed,0x0f]
+; X64-NEXT:    vmovdqa %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc1]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %b = load <8 x i16>, <8 x i16>* %ptr_b
+  %1 = call <8 x i16> @llvm.x86.sse2.padds.w(<8 x i16> %a, <8 x i16> %b)
+  %2 = bitcast i8 %mask to <8 x i1>
+  %3 = select <8 x i1> %2, <8 x i16> %1, <8 x i16> %passThru
+  ret <8 x i16> %3
+}
+
+define <8 x i16> @test_adds_epi16_rmkz_128(<8 x i16> %a, <8 x i16>* %ptr_b, i8 %mask) {
+; X86-LABEL: test_adds_epi16_rmkz_128:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx # encoding: [0x0f,0xb6,0x4c,0x24,0x08]
+; X86-NEXT:    kmovd %ecx, %k1 # encoding: [0xc5,0xfb,0x92,0xc9]
+; X86-NEXT:    vpaddsw (%eax), %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0x89,0xed,0x00]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_adds_epi16_rmkz_128:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovd %esi, %k1 # encoding: [0xc5,0xfb,0x92,0xce]
+; X64-NEXT:    vpaddsw (%rdi), %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0x89,0xed,0x07]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %b = load <8 x i16>, <8 x i16>* %ptr_b
+  %1 = call <8 x i16> @llvm.x86.sse2.padds.w(<8 x i16> %a, <8 x i16> %b)
+  %2 = bitcast i8 %mask to <8 x i1>
+  %3 = select <8 x i1> %2, <8 x i16> %1, <8 x i16> zeroinitializer
+  ret <8 x i16> %3
+}
+
+declare <16 x i16> @llvm.x86.avx2.padds.w(<16 x i16>, <16 x i16>) #0
+
+define <16 x i16> @test_adds_epi16_rrk_256(<16 x i16> %a, <16 x i16> %b, <16 x i16> %passThru, i16 %mask) {
+; X86-LABEL: test_adds_epi16_rrk_256:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
+; X86-NEXT:    vpaddsw %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf1,0x7d,0x29,0xed,0xd1]
+; X86-NEXT:    vmovdqa %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc2]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_adds_epi16_rrk_256:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; X64-NEXT:    vpaddsw %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf1,0x7d,0x29,0xed,0xd1]
+; X64-NEXT:    vmovdqa %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc2]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %1 = call <16 x i16> @llvm.x86.avx2.padds.w(<16 x i16> %a, <16 x i16> %b)
+  %2 = bitcast i16 %mask to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x i16> %1, <16 x i16> %passThru
+  ret <16 x i16> %3
+}
+
+define <16 x i16> @test_adds_epi16_rrkz_256(<16 x i16> %a, <16 x i16> %b, i16 %mask) {
+; X86-LABEL: test_adds_epi16_rrkz_256:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
+; X86-NEXT:    vpaddsw %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xa9,0xed,0xc1]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_adds_epi16_rrkz_256:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; X64-NEXT:    vpaddsw %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xa9,0xed,0xc1]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %1 = call <16 x i16> @llvm.x86.avx2.padds.w(<16 x i16> %a, <16 x i16> %b)
+  %2 = bitcast i16 %mask to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x i16> %1, <16 x i16> zeroinitializer
+  ret <16 x i16> %3
+}
+
+define <16 x i16> @test_adds_epi16_rmk_256(<16 x i16> %a, <16 x i16>* %ptr_b, <16 x i16> %passThru, i16 %mask) {
+; X86-LABEL: test_adds_epi16_rmk_256:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x08]
+; X86-NEXT:    vpaddsw (%eax), %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf1,0x7d,0x29,0xed,0x08]
+; X86-NEXT:    vmovdqa %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc1]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_adds_epi16_rmk_256:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovd %esi, %k1 # encoding: [0xc5,0xfb,0x92,0xce]
+; X64-NEXT:    vpaddsw (%rdi), %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf1,0x7d,0x29,0xed,0x0f]
+; X64-NEXT:    vmovdqa %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc1]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %b = load <16 x i16>, <16 x i16>* %ptr_b
+  %1 = call <16 x i16> @llvm.x86.avx2.padds.w(<16 x i16> %a, <16 x i16> %b)
+  %2 = bitcast i16 %mask to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x i16> %1, <16 x i16> %passThru
+  ret <16 x i16> %3
+}
+
+define <16 x i16> @test_adds_epi16_rmkz_256(<16 x i16> %a, <16 x i16>* %ptr_b, i16 %mask) {
+; X86-LABEL: test_adds_epi16_rmkz_256:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x08]
+; X86-NEXT:    vpaddsw (%eax), %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xa9,0xed,0x00]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_adds_epi16_rmkz_256:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovd %esi, %k1 # encoding: [0xc5,0xfb,0x92,0xce]
+; X64-NEXT:    vpaddsw (%rdi), %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xa9,0xed,0x07]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %b = load <16 x i16>, <16 x i16>* %ptr_b
+  %1 = call <16 x i16> @llvm.x86.avx2.padds.w(<16 x i16> %a, <16 x i16> %b)
+  %2 = bitcast i16 %mask to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x i16> %1, <16 x i16> zeroinitializer
+  ret <16 x i16> %3
+}
+
+declare <16 x i8> @llvm.x86.sse2.padds.b(<16 x i8>, <16 x i8>) #0
+
+define <16 x i8> @test_adds_epi8_rrk_128(<16 x i8> %a, <16 x i8> %b, <16 x i8> %passThru, i16 %mask) {
+; X86-LABEL: test_adds_epi8_rrk_128:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
+; X86-NEXT:    vpaddsb %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf1,0x7d,0x09,0xec,0xd1]
+; X86-NEXT:    vmovdqa %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc2]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_adds_epi8_rrk_128:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
-; X64-NEXT:    vpsubsw %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf1,0x7d,0x09,0xe9,0xd1]
+; X64-NEXT:    vpaddsb %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf1,0x7d,0x09,0xec,0xd1]
 ; X64-NEXT:    vmovdqa %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc2]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <8 x i16> @llvm.x86.avx512.mask.psubs.w.128(<8 x i16> %a, <8 x i16> %b, <8 x i16> %passThru, i8 %mask)
-  ret <8 x i16> %res
+  %1 = call <16 x i8> @llvm.x86.sse2.padds.b(<16 x i8> %a, <16 x i8> %b)
+  %2 = bitcast i16 %mask to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x i8> %1, <16 x i8> %passThru
+  ret <16 x i8> %3
 }
 
-define <8 x i16> @test_mask_subs_epi16_rrkz_128(<8 x i16> %a, <8 x i16> %b, i8 %mask) {
-; X86-LABEL: test_mask_subs_epi16_rrkz_128:
+define <16 x i8> @test_adds_epi8_rrkz_128(<16 x i8> %a, <16 x i8> %b, i16 %mask) {
+; X86-LABEL: test_adds_epi8_rrkz_128:
 ; X86:       # %bb.0:
-; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
-; X86-NEXT:    kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
-; X86-NEXT:    vpsubsw %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0x89,0xe9,0xc1]
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
+; X86-NEXT:    vpaddsb %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0x89,0xec,0xc1]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
-; X64-LABEL: test_mask_subs_epi16_rrkz_128:
+; X64-LABEL: test_adds_epi8_rrkz_128:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
-; X64-NEXT:    vpsubsw %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0x89,0xe9,0xc1]
+; X64-NEXT:    vpaddsb %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0x89,0xec,0xc1]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <8 x i16> @llvm.x86.avx512.mask.psubs.w.128(<8 x i16> %a, <8 x i16> %b, <8 x i16> zeroinitializer, i8 %mask)
-  ret <8 x i16> %res
+  %1 = call <16 x i8> @llvm.x86.sse2.padds.b(<16 x i8> %a, <16 x i8> %b)
+  %2 = bitcast i16 %mask to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x i8> %1, <16 x i8> zeroinitializer
+  ret <16 x i8> %3
 }
 
-define <8 x i16> @test_mask_subs_epi16_rm_128(<8 x i16> %a, <8 x i16>* %ptr_b) {
-; X86-LABEL: test_mask_subs_epi16_rm_128:
+define <16 x i8> @test_adds_epi8_rm_128(<16 x i8> %a, <16 x i8>* %ptr_b) {
+; X86-LABEL: test_adds_epi8_rm_128:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
-; X86-NEXT:    vpsubsw (%eax), %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xe9,0x00]
+; X86-NEXT:    vpaddsb (%eax), %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xec,0x00]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
-; X64-LABEL: test_mask_subs_epi16_rm_128:
+; X64-LABEL: test_adds_epi8_rm_128:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpsubsw (%rdi), %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xe9,0x07]
+; X64-NEXT:    vpaddsb (%rdi), %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xec,0x07]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %b = load <8 x i16>, <8 x i16>* %ptr_b
-  %res = call <8 x i16> @llvm.x86.avx512.mask.psubs.w.128(<8 x i16> %a, <8 x i16> %b, <8 x i16> zeroinitializer, i8 -1)
-  ret <8 x i16> %res
+  %b = load <16 x i8>, <16 x i8>* %ptr_b
+  %1 = call <16 x i8> @llvm.x86.sse2.padds.b(<16 x i8> %a, <16 x i8> %b)
+  ret <16 x i8> %1
 }
 
-define <8 x i16> @test_mask_subs_epi16_rmk_128(<8 x i16> %a, <8 x i16>* %ptr_b, <8 x i16> %passThru, i8 %mask) {
-; X86-LABEL: test_mask_subs_epi16_rmk_128:
+define <16 x i8> @test_adds_epi8_rmk_128(<16 x i8> %a, <16 x i8>* %ptr_b, <16 x i8> %passThru, i16 %mask) {
+; X86-LABEL: test_adds_epi8_rmk_128:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
-; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx # encoding: [0x0f,0xb6,0x4c,0x24,0x08]
-; X86-NEXT:    kmovd %ecx, %k1 # encoding: [0xc5,0xfb,0x92,0xc9]
-; X86-NEXT:    vpsubsw (%eax), %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf1,0x7d,0x09,0xe9,0x08]
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x08]
+; X86-NEXT:    vpaddsb (%eax), %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf1,0x7d,0x09,0xec,0x08]
 ; X86-NEXT:    vmovdqa %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc1]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
-; X64-LABEL: test_mask_subs_epi16_rmk_128:
+; X64-LABEL: test_adds_epi8_rmk_128:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovd %esi, %k1 # encoding: [0xc5,0xfb,0x92,0xce]
-; X64-NEXT:    vpsubsw (%rdi), %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf1,0x7d,0x09,0xe9,0x0f]
+; X64-NEXT:    vpaddsb (%rdi), %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf1,0x7d,0x09,0xec,0x0f]
 ; X64-NEXT:    vmovdqa %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc1]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %b = load <8 x i16>, <8 x i16>* %ptr_b
-  %res = call <8 x i16> @llvm.x86.avx512.mask.psubs.w.128(<8 x i16> %a, <8 x i16> %b, <8 x i16> %passThru, i8 %mask)
-  ret <8 x i16> %res
+  %b = load <16 x i8>, <16 x i8>* %ptr_b
+  %1 = call <16 x i8> @llvm.x86.sse2.padds.b(<16 x i8> %a, <16 x i8> %b)
+  %2 = bitcast i16 %mask to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x i8> %1, <16 x i8> %passThru
+  ret <16 x i8> %3
 }
 
-define <8 x i16> @test_mask_subs_epi16_rmkz_128(<8 x i16> %a, <8 x i16>* %ptr_b, i8 %mask) {
-; X86-LABEL: test_mask_subs_epi16_rmkz_128:
+define <16 x i8> @test_adds_epi8_rmkz_128(<16 x i8> %a, <16 x i8>* %ptr_b, i16 %mask) {
+; X86-LABEL: test_adds_epi8_rmkz_128:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
-; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx # encoding: [0x0f,0xb6,0x4c,0x24,0x08]
-; X86-NEXT:    kmovd %ecx, %k1 # encoding: [0xc5,0xfb,0x92,0xc9]
-; X86-NEXT:    vpsubsw (%eax), %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0x89,0xe9,0x00]
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x08]
+; X86-NEXT:    vpaddsb (%eax), %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0x89,0xec,0x00]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
-; X64-LABEL: test_mask_subs_epi16_rmkz_128:
+; X64-LABEL: test_adds_epi8_rmkz_128:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovd %esi, %k1 # encoding: [0xc5,0xfb,0x92,0xce]
-; X64-NEXT:    vpsubsw (%rdi), %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0x89,0xe9,0x07]
+; X64-NEXT:    vpaddsb (%rdi), %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0x89,0xec,0x07]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %b = load <8 x i16>, <8 x i16>* %ptr_b
-  %res = call <8 x i16> @llvm.x86.avx512.mask.psubs.w.128(<8 x i16> %a, <8 x i16> %b, <8 x i16> zeroinitializer, i8 %mask)
-  ret <8 x i16> %res
+  %b = load <16 x i8>, <16 x i8>* %ptr_b
+  %1 = call <16 x i8> @llvm.x86.sse2.padds.b(<16 x i8> %a, <16 x i8> %b)
+  %2 = bitcast i16 %mask to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x i8> %1, <16 x i8> zeroinitializer
+  ret <16 x i8> %3
 }
 
-declare <8 x i16> @llvm.x86.avx512.mask.psubs.w.128(<8 x i16>, <8 x i16>, <8 x i16>, i8)
-
-define <16 x i16> @test_mask_subs_epi16_rr_256(<16 x i16> %a, <16 x i16> %b) {
-; CHECK-LABEL: test_mask_subs_epi16_rr_256:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vpsubsw %ymm1, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xe9,0xc1]
-; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
-  %res = call <16 x i16> @llvm.x86.avx512.mask.psubs.w.256(<16 x i16> %a, <16 x i16> %b, <16 x i16> zeroinitializer, i16 -1)
-  ret <16 x i16> %res
-}
+declare <32 x i8> @llvm.x86.avx2.padds.b(<32 x i8>, <32 x i8>) #0
 
-define <16 x i16> @test_mask_subs_epi16_rrk_256(<16 x i16> %a, <16 x i16> %b, <16 x i16> %passThru, i16 %mask) {
-; X86-LABEL: test_mask_subs_epi16_rrk_256:
+define <32 x i8> @test_adds_epi8_rrk_256(<32 x i8> %a, <32 x i8> %b, <32 x i8> %passThru, i32 %mask) {
+; X86-LABEL: test_adds_epi8_rrk_256:
 ; X86:       # %bb.0:
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
-; X86-NEXT:    vpsubsw %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf1,0x7d,0x29,0xe9,0xd1]
+; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf9,0x90,0x4c,0x24,0x04]
+; X86-NEXT:    vpaddsb %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf1,0x7d,0x29,0xec,0xd1]
 ; X86-NEXT:    vmovdqa %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc2]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
-; X64-LABEL: test_mask_subs_epi16_rrk_256:
+; X64-LABEL: test_adds_epi8_rrk_256:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
-; X64-NEXT:    vpsubsw %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf1,0x7d,0x29,0xe9,0xd1]
+; X64-NEXT:    vpaddsb %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf1,0x7d,0x29,0xec,0xd1]
 ; X64-NEXT:    vmovdqa %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc2]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <16 x i16> @llvm.x86.avx512.mask.psubs.w.256(<16 x i16> %a, <16 x i16> %b, <16 x i16> %passThru, i16 %mask)
-  ret <16 x i16> %res
+  %1 = call <32 x i8> @llvm.x86.avx2.padds.b(<32 x i8> %a, <32 x i8> %b)
+  %2 = bitcast i32 %mask to <32 x i1>
+  %3 = select <32 x i1> %2, <32 x i8> %1, <32 x i8> %passThru
+  ret <32 x i8> %3
 }
 
-define <16 x i16> @test_mask_subs_epi16_rrkz_256(<16 x i16> %a, <16 x i16> %b, i16 %mask) {
-; X86-LABEL: test_mask_subs_epi16_rrkz_256:
+define <32 x i8> @test_adds_epi8_rrkz_256(<32 x i8> %a, <32 x i8> %b, i32 %mask) {
+; X86-LABEL: test_adds_epi8_rrkz_256:
 ; X86:       # %bb.0:
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
-; X86-NEXT:    vpsubsw %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xa9,0xe9,0xc1]
+; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf9,0x90,0x4c,0x24,0x04]
+; X86-NEXT:    vpaddsb %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xa9,0xec,0xc1]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
-; X64-LABEL: test_mask_subs_epi16_rrkz_256:
+; X64-LABEL: test_adds_epi8_rrkz_256:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
-; X64-NEXT:    vpsubsw %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xa9,0xe9,0xc1]
-; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <16 x i16> @llvm.x86.avx512.mask.psubs.w.256(<16 x i16> %a, <16 x i16> %b, <16 x i16> zeroinitializer, i16 %mask)
-  ret <16 x i16> %res
-}
-
-define <16 x i16> @test_mask_subs_epi16_rm_256(<16 x i16> %a, <16 x i16>* %ptr_b) {
-; X86-LABEL: test_mask_subs_epi16_rm_256:
-; X86:       # %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
-; X86-NEXT:    vpsubsw (%eax), %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xe9,0x00]
-; X86-NEXT:    retl # encoding: [0xc3]
-;
-; X64-LABEL: test_mask_subs_epi16_rm_256:
-; X64:       # %bb.0:
-; X64-NEXT:    vpsubsw (%rdi), %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xe9,0x07]
+; X64-NEXT:    vpaddsb %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xa9,0xec,0xc1]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %b = load <16 x i16>, <16 x i16>* %ptr_b
-  %res = call <16 x i16> @llvm.x86.avx512.mask.psubs.w.256(<16 x i16> %a, <16 x i16> %b, <16 x i16> zeroinitializer, i16 -1)
-  ret <16 x i16> %res
+  %1 = call <32 x i8> @llvm.x86.avx2.padds.b(<32 x i8> %a, <32 x i8> %b)
+  %2 = bitcast i32 %mask to <32 x i1>
+  %3 = select <32 x i1> %2, <32 x i8> %1, <32 x i8> zeroinitializer
+  ret <32 x i8> %3
 }
 
-define <16 x i16> @test_mask_subs_epi16_rmk_256(<16 x i16> %a, <16 x i16>* %ptr_b, <16 x i16> %passThru, i16 %mask) {
-; X86-LABEL: test_mask_subs_epi16_rmk_256:
+define <32 x i8> @test_adds_epi8_rmk_256(<32 x i8> %a, <32 x i8>* %ptr_b, <32 x i8> %passThru, i32 %mask) {
+; X86-LABEL: test_adds_epi8_rmk_256:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x08]
-; X86-NEXT:    vpsubsw (%eax), %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf1,0x7d,0x29,0xe9,0x08]
+; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf9,0x90,0x4c,0x24,0x08]
+; X86-NEXT:    vpaddsb (%eax), %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf1,0x7d,0x29,0xec,0x08]
 ; X86-NEXT:    vmovdqa %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc1]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
-; X64-LABEL: test_mask_subs_epi16_rmk_256:
+; X64-LABEL: test_adds_epi8_rmk_256:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovd %esi, %k1 # encoding: [0xc5,0xfb,0x92,0xce]
-; X64-NEXT:    vpsubsw (%rdi), %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf1,0x7d,0x29,0xe9,0x0f]
+; X64-NEXT:    vpaddsb (%rdi), %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf1,0x7d,0x29,0xec,0x0f]
 ; X64-NEXT:    vmovdqa %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc1]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %b = load <16 x i16>, <16 x i16>* %ptr_b
-  %res = call <16 x i16> @llvm.x86.avx512.mask.psubs.w.256(<16 x i16> %a, <16 x i16> %b, <16 x i16> %passThru, i16 %mask)
-  ret <16 x i16> %res
+  %b = load <32 x i8>, <32 x i8>* %ptr_b
+  %1 = call <32 x i8> @llvm.x86.avx2.padds.b(<32 x i8> %a, <32 x i8> %b)
+  %2 = bitcast i32 %mask to <32 x i1>
+  %3 = select <32 x i1> %2, <32 x i8> %1, <32 x i8> %passThru
+  ret <32 x i8> %3
 }
 
-define <16 x i16> @test_mask_subs_epi16_rmkz_256(<16 x i16> %a, <16 x i16>* %ptr_b, i16 %mask) {
-; X86-LABEL: test_mask_subs_epi16_rmkz_256:
+define <32 x i8> @test_adds_epi8_rmkz_256(<32 x i8> %a, <32 x i8>* %ptr_b, i32 %mask) {
+; X86-LABEL: test_adds_epi8_rmkz_256:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x08]
-; X86-NEXT:    vpsubsw (%eax), %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xa9,0xe9,0x00]
+; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf9,0x90,0x4c,0x24,0x08]
+; X86-NEXT:    vpaddsb (%eax), %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xa9,0xec,0x00]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
-; X64-LABEL: test_mask_subs_epi16_rmkz_256:
+; X64-LABEL: test_adds_epi8_rmkz_256:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovd %esi, %k1 # encoding: [0xc5,0xfb,0x92,0xce]
-; X64-NEXT:    vpsubsw (%rdi), %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xa9,0xe9,0x07]
+; X64-NEXT:    vpaddsb (%rdi), %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xa9,0xec,0x07]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %b = load <16 x i16>, <16 x i16>* %ptr_b
-  %res = call <16 x i16> @llvm.x86.avx512.mask.psubs.w.256(<16 x i16> %a, <16 x i16> %b, <16 x i16> zeroinitializer, i16 %mask)
-  ret <16 x i16> %res
+  %b = load <32 x i8>, <32 x i8>* %ptr_b
+  %1 = call <32 x i8> @llvm.x86.avx2.padds.b(<32 x i8> %a, <32 x i8> %b)
+  %2 = bitcast i32 %mask to <32 x i1>
+  %3 = select <32 x i1> %2, <32 x i8> %1, <32 x i8> zeroinitializer
+  ret <32 x i8> %3
 }
 
-declare <16 x i16> @llvm.x86.avx512.mask.psubs.w.256(<16 x i16>, <16 x i16>, <16 x i16>, i16)
-
 define <16 x i8> @test_mask_adds_epi8_rr_128(<16 x i8> %a, <16 x i8> %b) {
 ; CHECK-LABEL: test_mask_adds_epi8_rr_128:
 ; CHECK:       # %bb.0:
@@ -7946,3 +8626,217 @@ define <32 x i8> @test_mask_subs_epi8_rmkz_256(<32 x i8> %a, <32 x i8>* %ptr_b,
 
 declare <32 x i8> @llvm.x86.avx512.mask.psubs.b.256(<32 x i8>, <32 x i8>, <32 x i8>, i32)
 
+declare <16 x i16> @llvm.x86.avx512.mask.psrav16.hi(<16 x i16>, <16 x i16>, <16 x i16>, i16)
+
+define <16 x i16>@test_int_x86_avx512_mask_psrav16_hi(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x2, i16 %x3) {
+; X86-LABEL: test_int_x86_avx512_mask_psrav16_hi:
+; X86:       # %bb.0:
+; X86-NEXT:    vpsravw %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf2,0xfd,0x28,0x11,0xd9]
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
+; X86-NEXT:    vpsravw %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x29,0x11,0xd1]
+; X86-NEXT:    vpsravw %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xa9,0x11,0xc1]
+; X86-NEXT:    vpaddw %ymm3, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xfd,0xc3]
+; X86-NEXT:    vpaddw %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfd,0xc0]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_mask_psrav16_hi:
+; X64:       # %bb.0:
+; X64-NEXT:    vpsravw %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf2,0xfd,0x28,0x11,0xd9]
+; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; X64-NEXT:    vpsravw %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x29,0x11,0xd1]
+; X64-NEXT:    vpsravw %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xa9,0x11,0xc1]
+; X64-NEXT:    vpaddw %ymm3, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xfd,0xc3]
+; X64-NEXT:    vpaddw %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfd,0xc0]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <16 x i16> @llvm.x86.avx512.mask.psrav16.hi(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x2, i16 %x3)
+  %res1 = call <16 x i16> @llvm.x86.avx512.mask.psrav16.hi(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> zeroinitializer, i16 %x3)
+  %res2 = call <16 x i16> @llvm.x86.avx512.mask.psrav16.hi(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x2, i16 -1)
+  %res3 = add <16 x i16> %res, %res1
+  %res4 = add <16 x i16> %res3, %res2
+  ret <16 x i16> %res4
+}
+
+declare <8 x i16> @llvm.x86.avx512.mask.psrav8.hi(<8 x i16>, <8 x i16>, <8 x i16>, i8)
+
+define <8 x i16>@test_int_x86_avx512_mask_psrav8_hi(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_mask_psrav8_hi:
+; X86:       # %bb.0:
+; X86-NEXT:    vpsravw %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf2,0xfd,0x08,0x11,0xd9]
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
+; X86-NEXT:    vpsravw %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x09,0x11,0xd1]
+; X86-NEXT:    vpsravw %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0x89,0x11,0xc1]
+; X86-NEXT:    vpaddw %xmm3, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfd,0xc3]
+; X86-NEXT:    vpaddw %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfd,0xc0]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_mask_psrav8_hi:
+; X64:       # %bb.0:
+; X64-NEXT:    vpsravw %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf2,0xfd,0x08,0x11,0xd9]
+; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; X64-NEXT:    vpsravw %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x09,0x11,0xd1]
+; X64-NEXT:    vpsravw %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0x89,0x11,0xc1]
+; X64-NEXT:    vpaddw %xmm3, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfd,0xc3]
+; X64-NEXT:    vpaddw %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfd,0xc0]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <8 x i16> @llvm.x86.avx512.mask.psrav8.hi(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2, i8 %x3)
+  %res1 = call <8 x i16> @llvm.x86.avx512.mask.psrav8.hi(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> zeroinitializer, i8 %x3)
+  %res2 = call <8 x i16> @llvm.x86.avx512.mask.psrav8.hi(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2, i8 -1)
+  %res3 = add <8 x i16> %res, %res1
+  %res4 = add <8 x i16> %res3, %res2
+  ret <8 x i16> %res4
+}
+
+declare <16 x i16> @llvm.x86.avx512.mask.psllv16.hi(<16 x i16>, <16 x i16>, <16 x i16>, i16)
+
+define <16 x i16>@test_int_x86_avx512_mask_psllv16_hi(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x2, i16 %x3) {
+; X86-LABEL: test_int_x86_avx512_mask_psllv16_hi:
+; X86:       # %bb.0:
+; X86-NEXT:    vpsllvw %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf2,0xfd,0x28,0x12,0xd9]
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
+; X86-NEXT:    vpsllvw %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x29,0x12,0xd1]
+; X86-NEXT:    vpsllvw %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xa9,0x12,0xc1]
+; X86-NEXT:    vpaddw %ymm3, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xfd,0xc3]
+; X86-NEXT:    vpaddw %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfd,0xc0]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_mask_psllv16_hi:
+; X64:       # %bb.0:
+; X64-NEXT:    vpsllvw %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf2,0xfd,0x28,0x12,0xd9]
+; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; X64-NEXT:    vpsllvw %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x29,0x12,0xd1]
+; X64-NEXT:    vpsllvw %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xa9,0x12,0xc1]
+; X64-NEXT:    vpaddw %ymm3, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xfd,0xc3]
+; X64-NEXT:    vpaddw %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfd,0xc0]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <16 x i16> @llvm.x86.avx512.mask.psllv16.hi(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x2, i16 %x3)
+  %res1 = call <16 x i16> @llvm.x86.avx512.mask.psllv16.hi(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> zeroinitializer, i16 %x3)
+  %res2 = call <16 x i16> @llvm.x86.avx512.mask.psllv16.hi(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x2, i16 -1)
+  %res3 = add <16 x i16> %res, %res1
+  %res4 = add <16 x i16> %res3, %res2
+  ret <16 x i16> %res4
+}
+
+declare <8 x i16> @llvm.x86.avx512.mask.psllv8.hi(<8 x i16>, <8 x i16>, <8 x i16>, i8)
+
+define <8 x i16>@test_int_x86_avx512_mask_psllv8_hi(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_mask_psllv8_hi:
+; X86:       # %bb.0:
+; X86-NEXT:    vpsllvw %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf2,0xfd,0x08,0x12,0xd9]
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
+; X86-NEXT:    vpsllvw %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x09,0x12,0xd1]
+; X86-NEXT:    vpsllvw %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0x89,0x12,0xc1]
+; X86-NEXT:    vpaddw %xmm3, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfd,0xc3]
+; X86-NEXT:    vpaddw %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfd,0xc0]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_mask_psllv8_hi:
+; X64:       # %bb.0:
+; X64-NEXT:    vpsllvw %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf2,0xfd,0x08,0x12,0xd9]
+; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; X64-NEXT:    vpsllvw %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x09,0x12,0xd1]
+; X64-NEXT:    vpsllvw %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0x89,0x12,0xc1]
+; X64-NEXT:    vpaddw %xmm3, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfd,0xc3]
+; X64-NEXT:    vpaddw %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfd,0xc0]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <8 x i16> @llvm.x86.avx512.mask.psllv8.hi(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2, i8 %x3)
+  %res1 = call <8 x i16> @llvm.x86.avx512.mask.psllv8.hi(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> zeroinitializer, i8 %x3)
+  %res2 = call <8 x i16> @llvm.x86.avx512.mask.psllv8.hi(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2, i8 -1)
+  %res3 = add <8 x i16> %res, %res1
+  %res4 = add <8 x i16> %res3, %res2
+  ret <8 x i16> %res4
+}
+
+declare <16 x i16> @llvm.x86.avx512.mask.psrlv16.hi(<16 x i16>, <16 x i16>, <16 x i16>, i16)
+
+define <16 x i16>@test_int_x86_avx512_mask_psrlv16_hi(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x2, i16 %x3) {
+; X86-LABEL: test_int_x86_avx512_mask_psrlv16_hi:
+; X86:       # %bb.0:
+; X86-NEXT:    vpsrlvw %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf2,0xfd,0x28,0x10,0xd9]
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
+; X86-NEXT:    vpsrlvw %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x29,0x10,0xd1]
+; X86-NEXT:    vpsrlvw %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xa9,0x10,0xc1]
+; X86-NEXT:    vpaddw %ymm3, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xfd,0xc3]
+; X86-NEXT:    vpaddw %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfd,0xc0]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_mask_psrlv16_hi:
+; X64:       # %bb.0:
+; X64-NEXT:    vpsrlvw %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf2,0xfd,0x28,0x10,0xd9]
+; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; X64-NEXT:    vpsrlvw %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x29,0x10,0xd1]
+; X64-NEXT:    vpsrlvw %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xa9,0x10,0xc1]
+; X64-NEXT:    vpaddw %ymm3, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xfd,0xc3]
+; X64-NEXT:    vpaddw %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfd,0xc0]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <16 x i16> @llvm.x86.avx512.mask.psrlv16.hi(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x2, i16 %x3)
+  %res1 = call <16 x i16> @llvm.x86.avx512.mask.psrlv16.hi(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> zeroinitializer, i16 %x3)
+  %res2 = call <16 x i16> @llvm.x86.avx512.mask.psrlv16.hi(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x2, i16 -1)
+  %res3 = add <16 x i16> %res, %res1
+  %res4 = add <16 x i16> %res3, %res2
+  ret <16 x i16> %res4
+}
+
+declare <8 x i16> @llvm.x86.avx512.mask.psrlv8.hi(<8 x i16>, <8 x i16>, <8 x i16>, i8)
+
+define <8 x i16>@test_int_x86_avx512_mask_psrlv8_hi(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_mask_psrlv8_hi:
+; X86:       # %bb.0:
+; X86-NEXT:    vpsrlvw %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf2,0xfd,0x08,0x10,0xd9]
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
+; X86-NEXT:    vpsrlvw %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x09,0x10,0xd1]
+; X86-NEXT:    vpsrlvw %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0x89,0x10,0xc1]
+; X86-NEXT:    vpaddw %xmm3, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfd,0xc3]
+; X86-NEXT:    vpaddw %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfd,0xc0]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_mask_psrlv8_hi:
+; X64:       # %bb.0:
+; X64-NEXT:    vpsrlvw %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf2,0xfd,0x08,0x10,0xd9]
+; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; X64-NEXT:    vpsrlvw %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x09,0x10,0xd1]
+; X64-NEXT:    vpsrlvw %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0x89,0x10,0xc1]
+; X64-NEXT:    vpaddw %xmm3, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfd,0xc3]
+; X64-NEXT:    vpaddw %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfd,0xc0]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <8 x i16> @llvm.x86.avx512.mask.psrlv8.hi(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2, i8 %x3)
+  %res1 = call <8 x i16> @llvm.x86.avx512.mask.psrlv8.hi(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> zeroinitializer, i8 %x3)
+  %res2 = call <8 x i16> @llvm.x86.avx512.mask.psrlv8.hi(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2, i8 -1)
+  %res3 = add <8 x i16> %res, %res1
+  %res4 = add <8 x i16> %res3, %res2
+  ret <8 x i16> %res4
+}
+
+declare <16 x i8> @llvm.x86.avx512.mask.pmov.wb.256(<16 x i16>, <16 x i8>, i16)
+
+define <16 x i8>@test_int_x86_avx512_mask_pmov_wb_256(<16 x i16> %x0, <16 x i8> %x1, i16 %x2) {
+; X86-LABEL: test_int_x86_avx512_mask_pmov_wb_256:
+; X86:       # %bb.0:
+; X86-NEXT:    vpmovwb %ymm0, %xmm2 # encoding: [0x62,0xf2,0x7e,0x28,0x30,0xc2]
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
+; X86-NEXT:    vpmovwb %ymm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0x7e,0x29,0x30,0xc1]
+; X86-NEXT:    vpmovwb %ymm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0xa9,0x30,0xc0]
+; X86-NEXT:    vpaddb %xmm0, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0xfc,0xc0]
+; X86-NEXT:    vpaddb %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfc,0xc0]
+; X86-NEXT:    vzeroupper # encoding: [0xc5,0xf8,0x77]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_mask_pmov_wb_256:
+; X64:       # %bb.0:
+; X64-NEXT:    vpmovwb %ymm0, %xmm2 # encoding: [0x62,0xf2,0x7e,0x28,0x30,0xc2]
+; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; X64-NEXT:    vpmovwb %ymm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0x7e,0x29,0x30,0xc1]
+; X64-NEXT:    vpmovwb %ymm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0xa9,0x30,0xc0]
+; X64-NEXT:    vpaddb %xmm0, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0xfc,0xc0]
+; X64-NEXT:    vpaddb %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfc,0xc0]
+; X64-NEXT:    vzeroupper # encoding: [0xc5,0xf8,0x77]
+; X64-NEXT:    retq # encoding: [0xc3]
+    %res0 = call <16 x i8> @llvm.x86.avx512.mask.pmov.wb.256(<16 x i16> %x0, <16 x i8> %x1, i16 -1)
+    %res1 = call <16 x i8> @llvm.x86.avx512.mask.pmov.wb.256(<16 x i16> %x0, <16 x i8> %x1, i16 %x2)
+    %res2 = call <16 x i8> @llvm.x86.avx512.mask.pmov.wb.256(<16 x i16> %x0, <16 x i8> zeroinitializer, i16 %x2)
+    %res3 = add <16 x i8> %res0, %res1
+    %res4 = add <16 x i8> %res3, %res2
+    ret <16 x i8> %res4
+}
diff --git a/test/CodeGen/X86/avx512bwvl-intrinsics.ll b/test/CodeGen/X86/avx512bwvl-intrinsics.ll
index 568cdeccdec5261335dbdfdc67551a2cc8c3274a..b9937603f1679022f5723ba0f41759f31fea3f71 100644
--- a/test/CodeGen/X86/avx512bwvl-intrinsics.ll
+++ b/test/CodeGen/X86/avx512bwvl-intrinsics.ll
@@ -1127,686 +1127,6 @@ define <32 x i8> @test_mask_packus_epi16_rmkz_256(<16 x i16> %a, <16 x i16>* %pt
 
 declare <32 x i8> @llvm.x86.avx2.packuswb(<16 x i16>, <16 x i16>)
 
-declare <8 x i16> @llvm.x86.sse2.padds.w(<8 x i16>, <8 x i16>) #0
-
-define <8 x i16> @test_mask_adds_epi16_rrk_128(<8 x i16> %a, <8 x i16> %b, <8 x i16> %passThru, i8 %mask) {
-; X86-LABEL: test_mask_adds_epi16_rrk_128:
-; X86:       # %bb.0:
-; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
-; X86-NEXT:    kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
-; X86-NEXT:    vpaddsw %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf1,0x7d,0x09,0xed,0xd1]
-; X86-NEXT:    vmovdqa %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc2]
-; X86-NEXT:    retl # encoding: [0xc3]
-;
-; X64-LABEL: test_mask_adds_epi16_rrk_128:
-; X64:       # %bb.0:
-; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
-; X64-NEXT:    vpaddsw %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf1,0x7d,0x09,0xed,0xd1]
-; X64-NEXT:    vmovdqa %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc2]
-; X64-NEXT:    retq # encoding: [0xc3]
-  %1 = call <8 x i16> @llvm.x86.sse2.padds.w(<8 x i16> %a, <8 x i16> %b)
-  %2 = bitcast i8 %mask to <8 x i1>
-  %3 = select <8 x i1> %2, <8 x i16> %1, <8 x i16> %passThru
-  ret <8 x i16> %3
-}
-
-define <8 x i16> @test_mask_adds_epi16_rrkz_128(<8 x i16> %a, <8 x i16> %b, i8 %mask) {
-; X86-LABEL: test_mask_adds_epi16_rrkz_128:
-; X86:       # %bb.0:
-; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
-; X86-NEXT:    kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
-; X86-NEXT:    vpaddsw %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0x89,0xed,0xc1]
-; X86-NEXT:    retl # encoding: [0xc3]
-;
-; X64-LABEL: test_mask_adds_epi16_rrkz_128:
-; X64:       # %bb.0:
-; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
-; X64-NEXT:    vpaddsw %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0x89,0xed,0xc1]
-; X64-NEXT:    retq # encoding: [0xc3]
-  %1 = call <8 x i16> @llvm.x86.sse2.padds.w(<8 x i16> %a, <8 x i16> %b)
-  %2 = bitcast i8 %mask to <8 x i1>
-  %3 = select <8 x i1> %2, <8 x i16> %1, <8 x i16> zeroinitializer
-  ret <8 x i16> %3
-}
-
-define <8 x i16> @test_mask_adds_epi16_rmk_128(<8 x i16> %a, <8 x i16>* %ptr_b, <8 x i16> %passThru, i8 %mask) {
-; X86-LABEL: test_mask_adds_epi16_rmk_128:
-; X86:       # %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
-; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx # encoding: [0x0f,0xb6,0x4c,0x24,0x08]
-; X86-NEXT:    kmovd %ecx, %k1 # encoding: [0xc5,0xfb,0x92,0xc9]
-; X86-NEXT:    vpaddsw (%eax), %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf1,0x7d,0x09,0xed,0x08]
-; X86-NEXT:    vmovdqa %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc1]
-; X86-NEXT:    retl # encoding: [0xc3]
-;
-; X64-LABEL: test_mask_adds_epi16_rmk_128:
-; X64:       # %bb.0:
-; X64-NEXT:    kmovd %esi, %k1 # encoding: [0xc5,0xfb,0x92,0xce]
-; X64-NEXT:    vpaddsw (%rdi), %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf1,0x7d,0x09,0xed,0x0f]
-; X64-NEXT:    vmovdqa %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc1]
-; X64-NEXT:    retq # encoding: [0xc3]
-  %b = load <8 x i16>, <8 x i16>* %ptr_b
-  %1 = call <8 x i16> @llvm.x86.sse2.padds.w(<8 x i16> %a, <8 x i16> %b)
-  %2 = bitcast i8 %mask to <8 x i1>
-  %3 = select <8 x i1> %2, <8 x i16> %1, <8 x i16> %passThru
-  ret <8 x i16> %3
-}
-
-define <8 x i16> @test_mask_adds_epi16_rmkz_128(<8 x i16> %a, <8 x i16>* %ptr_b, i8 %mask) {
-; X86-LABEL: test_mask_adds_epi16_rmkz_128:
-; X86:       # %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
-; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx # encoding: [0x0f,0xb6,0x4c,0x24,0x08]
-; X86-NEXT:    kmovd %ecx, %k1 # encoding: [0xc5,0xfb,0x92,0xc9]
-; X86-NEXT:    vpaddsw (%eax), %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0x89,0xed,0x00]
-; X86-NEXT:    retl # encoding: [0xc3]
-;
-; X64-LABEL: test_mask_adds_epi16_rmkz_128:
-; X64:       # %bb.0:
-; X64-NEXT:    kmovd %esi, %k1 # encoding: [0xc5,0xfb,0x92,0xce]
-; X64-NEXT:    vpaddsw (%rdi), %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0x89,0xed,0x07]
-; X64-NEXT:    retq # encoding: [0xc3]
-  %b = load <8 x i16>, <8 x i16>* %ptr_b
-  %1 = call <8 x i16> @llvm.x86.sse2.padds.w(<8 x i16> %a, <8 x i16> %b)
-  %2 = bitcast i8 %mask to <8 x i1>
-  %3 = select <8 x i1> %2, <8 x i16> %1, <8 x i16> zeroinitializer
-  ret <8 x i16> %3
-}
-
-declare <16 x i16> @llvm.x86.avx2.padds.w(<16 x i16>, <16 x i16>) #0
-
-define <16 x i16> @test_mask_adds_epi16_rrk_256(<16 x i16> %a, <16 x i16> %b, <16 x i16> %passThru, i16 %mask) {
-; X86-LABEL: test_mask_adds_epi16_rrk_256:
-; X86:       # %bb.0:
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
-; X86-NEXT:    vpaddsw %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf1,0x7d,0x29,0xed,0xd1]
-; X86-NEXT:    vmovdqa %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc2]
-; X86-NEXT:    retl # encoding: [0xc3]
-;
-; X64-LABEL: test_mask_adds_epi16_rrk_256:
-; X64:       # %bb.0:
-; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
-; X64-NEXT:    vpaddsw %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf1,0x7d,0x29,0xed,0xd1]
-; X64-NEXT:    vmovdqa %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc2]
-; X64-NEXT:    retq # encoding: [0xc3]
-  %1 = call <16 x i16> @llvm.x86.avx2.padds.w(<16 x i16> %a, <16 x i16> %b)
-  %2 = bitcast i16 %mask to <16 x i1>
-  %3 = select <16 x i1> %2, <16 x i16> %1, <16 x i16> %passThru
-  ret <16 x i16> %3
-}
-
-define <16 x i16> @test_mask_adds_epi16_rrkz_256(<16 x i16> %a, <16 x i16> %b, i16 %mask) {
-; X86-LABEL: test_mask_adds_epi16_rrkz_256:
-; X86:       # %bb.0:
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
-; X86-NEXT:    vpaddsw %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xa9,0xed,0xc1]
-; X86-NEXT:    retl # encoding: [0xc3]
-;
-; X64-LABEL: test_mask_adds_epi16_rrkz_256:
-; X64:       # %bb.0:
-; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
-; X64-NEXT:    vpaddsw %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xa9,0xed,0xc1]
-; X64-NEXT:    retq # encoding: [0xc3]
-  %1 = call <16 x i16> @llvm.x86.avx2.padds.w(<16 x i16> %a, <16 x i16> %b)
-  %2 = bitcast i16 %mask to <16 x i1>
-  %3 = select <16 x i1> %2, <16 x i16> %1, <16 x i16> zeroinitializer
-  ret <16 x i16> %3
-}
-
-define <16 x i16> @test_mask_adds_epi16_rmk_256(<16 x i16> %a, <16 x i16>* %ptr_b, <16 x i16> %passThru, i16 %mask) {
-; X86-LABEL: test_mask_adds_epi16_rmk_256:
-; X86:       # %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x08]
-; X86-NEXT:    vpaddsw (%eax), %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf1,0x7d,0x29,0xed,0x08]
-; X86-NEXT:    vmovdqa %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc1]
-; X86-NEXT:    retl # encoding: [0xc3]
-;
-; X64-LABEL: test_mask_adds_epi16_rmk_256:
-; X64:       # %bb.0:
-; X64-NEXT:    kmovd %esi, %k1 # encoding: [0xc5,0xfb,0x92,0xce]
-; X64-NEXT:    vpaddsw (%rdi), %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf1,0x7d,0x29,0xed,0x0f]
-; X64-NEXT:    vmovdqa %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc1]
-; X64-NEXT:    retq # encoding: [0xc3]
-  %b = load <16 x i16>, <16 x i16>* %ptr_b
-  %1 = call <16 x i16> @llvm.x86.avx2.padds.w(<16 x i16> %a, <16 x i16> %b)
-  %2 = bitcast i16 %mask to <16 x i1>
-  %3 = select <16 x i1> %2, <16 x i16> %1, <16 x i16> %passThru
-  ret <16 x i16> %3
-}
-
-define <16 x i16> @test_mask_adds_epi16_rmkz_256(<16 x i16> %a, <16 x i16>* %ptr_b, i16 %mask) {
-; X86-LABEL: test_mask_adds_epi16_rmkz_256:
-; X86:       # %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x08]
-; X86-NEXT:    vpaddsw (%eax), %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xa9,0xed,0x00]
-; X86-NEXT:    retl # encoding: [0xc3]
-;
-; X64-LABEL: test_mask_adds_epi16_rmkz_256:
-; X64:       # %bb.0:
-; X64-NEXT:    kmovd %esi, %k1 # encoding: [0xc5,0xfb,0x92,0xce]
-; X64-NEXT:    vpaddsw (%rdi), %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xa9,0xed,0x07]
-; X64-NEXT:    retq # encoding: [0xc3]
-  %b = load <16 x i16>, <16 x i16>* %ptr_b
-  %1 = call <16 x i16> @llvm.x86.avx2.padds.w(<16 x i16> %a, <16 x i16> %b)
-  %2 = bitcast i16 %mask to <16 x i1>
-  %3 = select <16 x i1> %2, <16 x i16> %1, <16 x i16> zeroinitializer
-  ret <16 x i16> %3
-}
-
-declare <8 x i16> @llvm.x86.sse2.psubs.w(<8 x i16>, <8 x i16>) #0
-
-define <8 x i16> @test_mask_subs_epi16_rrk_128(<8 x i16> %a, <8 x i16> %b, <8 x i16> %passThru, i8 %mask) {
-; X86-LABEL: test_mask_subs_epi16_rrk_128:
-; X86:       # %bb.0:
-; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
-; X86-NEXT:    kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
-; X86-NEXT:    vpsubsw %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf1,0x7d,0x09,0xe9,0xd1]
-; X86-NEXT:    vmovdqa %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc2]
-; X86-NEXT:    retl # encoding: [0xc3]
-;
-; X64-LABEL: test_mask_subs_epi16_rrk_128:
-; X64:       # %bb.0:
-; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
-; X64-NEXT:    vpsubsw %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf1,0x7d,0x09,0xe9,0xd1]
-; X64-NEXT:    vmovdqa %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc2]
-; X64-NEXT:    retq # encoding: [0xc3]
-  %1 = call <8 x i16> @llvm.x86.sse2.psubs.w(<8 x i16> %a, <8 x i16> %b)
-  %2 = bitcast i8 %mask to <8 x i1>
-  %3 = select <8 x i1> %2, <8 x i16> %1, <8 x i16> %passThru
-  ret <8 x i16> %3
-}
-
-define <8 x i16> @test_mask_subs_epi16_rrkz_128(<8 x i16> %a, <8 x i16> %b, i8 %mask) {
-; X86-LABEL: test_mask_subs_epi16_rrkz_128:
-; X86:       # %bb.0:
-; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
-; X86-NEXT:    kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
-; X86-NEXT:    vpsubsw %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0x89,0xe9,0xc1]
-; X86-NEXT:    retl # encoding: [0xc3]
-;
-; X64-LABEL: test_mask_subs_epi16_rrkz_128:
-; X64:       # %bb.0:
-; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
-; X64-NEXT:    vpsubsw %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0x89,0xe9,0xc1]
-; X64-NEXT:    retq # encoding: [0xc3]
-  %1 = call <8 x i16> @llvm.x86.sse2.psubs.w(<8 x i16> %a, <8 x i16> %b)
-  %2 = bitcast i8 %mask to <8 x i1>
-  %3 = select <8 x i1> %2, <8 x i16> %1, <8 x i16> zeroinitializer
-  ret <8 x i16> %3
-}
-
-define <8 x i16> @test_mask_subs_epi16_rmk_128(<8 x i16> %a, <8 x i16>* %ptr_b, <8 x i16> %passThru, i8 %mask) {
-; X86-LABEL: test_mask_subs_epi16_rmk_128:
-; X86:       # %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
-; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx # encoding: [0x0f,0xb6,0x4c,0x24,0x08]
-; X86-NEXT:    kmovd %ecx, %k1 # encoding: [0xc5,0xfb,0x92,0xc9]
-; X86-NEXT:    vpsubsw (%eax), %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf1,0x7d,0x09,0xe9,0x08]
-; X86-NEXT:    vmovdqa %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc1]
-; X86-NEXT:    retl # encoding: [0xc3]
-;
-; X64-LABEL: test_mask_subs_epi16_rmk_128:
-; X64:       # %bb.0:
-; X64-NEXT:    kmovd %esi, %k1 # encoding: [0xc5,0xfb,0x92,0xce]
-; X64-NEXT:    vpsubsw (%rdi), %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf1,0x7d,0x09,0xe9,0x0f]
-; X64-NEXT:    vmovdqa %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc1]
-; X64-NEXT:    retq # encoding: [0xc3]
-  %b = load <8 x i16>, <8 x i16>* %ptr_b
-  %1 = call <8 x i16> @llvm.x86.sse2.psubs.w(<8 x i16> %a, <8 x i16> %b)
-  %2 = bitcast i8 %mask to <8 x i1>
-  %3 = select <8 x i1> %2, <8 x i16> %1, <8 x i16> %passThru
-  ret <8 x i16> %3
-}
-
-define <8 x i16> @test_mask_subs_epi16_rmkz_128(<8 x i16> %a, <8 x i16>* %ptr_b, i8 %mask) {
-; X86-LABEL: test_mask_subs_epi16_rmkz_128:
-; X86:       # %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
-; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx # encoding: [0x0f,0xb6,0x4c,0x24,0x08]
-; X86-NEXT:    kmovd %ecx, %k1 # encoding: [0xc5,0xfb,0x92,0xc9]
-; X86-NEXT:    vpsubsw (%eax), %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0x89,0xe9,0x00]
-; X86-NEXT:    retl # encoding: [0xc3]
-;
-; X64-LABEL: test_mask_subs_epi16_rmkz_128:
-; X64:       # %bb.0:
-; X64-NEXT:    kmovd %esi, %k1 # encoding: [0xc5,0xfb,0x92,0xce]
-; X64-NEXT:    vpsubsw (%rdi), %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0x89,0xe9,0x07]
-; X64-NEXT:    retq # encoding: [0xc3]
-  %b = load <8 x i16>, <8 x i16>* %ptr_b
-  %1 = call <8 x i16> @llvm.x86.sse2.psubs.w(<8 x i16> %a, <8 x i16> %b)
-  %2 = bitcast i8 %mask to <8 x i1>
-  %3 = select <8 x i1> %2, <8 x i16> %1, <8 x i16> zeroinitializer
-  ret <8 x i16> %3
-}
-
-declare <16 x i16> @llvm.x86.avx2.psubs.w(<16 x i16>, <16 x i16>) #0
-
-define <16 x i16> @test_mask_subs_epi16_rrk_256(<16 x i16> %a, <16 x i16> %b, <16 x i16> %passThru, i16 %mask) {
-; X86-LABEL: test_mask_subs_epi16_rrk_256:
-; X86:       # %bb.0:
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
-; X86-NEXT:    vpsubsw %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf1,0x7d,0x29,0xe9,0xd1]
-; X86-NEXT:    vmovdqa %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc2]
-; X86-NEXT:    retl # encoding: [0xc3]
-;
-; X64-LABEL: test_mask_subs_epi16_rrk_256:
-; X64:       # %bb.0:
-; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
-; X64-NEXT:    vpsubsw %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf1,0x7d,0x29,0xe9,0xd1]
-; X64-NEXT:    vmovdqa %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc2]
-; X64-NEXT:    retq # encoding: [0xc3]
-  %1 = call <16 x i16> @llvm.x86.avx2.psubs.w(<16 x i16> %a, <16 x i16> %b)
-  %2 = bitcast i16 %mask to <16 x i1>
-  %3 = select <16 x i1> %2, <16 x i16> %1, <16 x i16> %passThru
-  ret <16 x i16> %3
-}
-
-define <16 x i16> @test_mask_subs_epi16_rrkz_256(<16 x i16> %a, <16 x i16> %b, i16 %mask) {
-; X86-LABEL: test_mask_subs_epi16_rrkz_256:
-; X86:       # %bb.0:
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
-; X86-NEXT:    vpsubsw %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xa9,0xe9,0xc1]
-; X86-NEXT:    retl # encoding: [0xc3]
-;
-; X64-LABEL: test_mask_subs_epi16_rrkz_256:
-; X64:       # %bb.0:
-; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
-; X64-NEXT:    vpsubsw %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xa9,0xe9,0xc1]
-; X64-NEXT:    retq # encoding: [0xc3]
-  %1 = call <16 x i16> @llvm.x86.avx2.psubs.w(<16 x i16> %a, <16 x i16> %b)
-  %2 = bitcast i16 %mask to <16 x i1>
-  %3 = select <16 x i1> %2, <16 x i16> %1, <16 x i16> zeroinitializer
-  ret <16 x i16> %3
-}
-
-define <16 x i16> @test_mask_subs_epi16_rmk_256(<16 x i16> %a, <16 x i16>* %ptr_b, <16 x i16> %passThru, i16 %mask) {
-; X86-LABEL: test_mask_subs_epi16_rmk_256:
-; X86:       # %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x08]
-; X86-NEXT:    vpsubsw (%eax), %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf1,0x7d,0x29,0xe9,0x08]
-; X86-NEXT:    vmovdqa %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc1]
-; X86-NEXT:    retl # encoding: [0xc3]
-;
-; X64-LABEL: test_mask_subs_epi16_rmk_256:
-; X64:       # %bb.0:
-; X64-NEXT:    kmovd %esi, %k1 # encoding: [0xc5,0xfb,0x92,0xce]
-; X64-NEXT:    vpsubsw (%rdi), %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf1,0x7d,0x29,0xe9,0x0f]
-; X64-NEXT:    vmovdqa %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc1]
-; X64-NEXT:    retq # encoding: [0xc3]
-  %b = load <16 x i16>, <16 x i16>* %ptr_b
-  %1 = call <16 x i16> @llvm.x86.avx2.psubs.w(<16 x i16> %a, <16 x i16> %b)
-  %2 = bitcast i16 %mask to <16 x i1>
-  %3 = select <16 x i1> %2, <16 x i16> %1, <16 x i16> %passThru
-  ret <16 x i16> %3
-}
-
-define <16 x i16> @test_mask_subs_epi16_rmkz_256(<16 x i16> %a, <16 x i16>* %ptr_b, i16 %mask) {
-; X86-LABEL: test_mask_subs_epi16_rmkz_256:
-; X86:       # %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x08]
-; X86-NEXT:    vpsubsw (%eax), %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xa9,0xe9,0x00]
-; X86-NEXT:    retl # encoding: [0xc3]
-;
-; X64-LABEL: test_mask_subs_epi16_rmkz_256:
-; X64:       # %bb.0:
-; X64-NEXT:    kmovd %esi, %k1 # encoding: [0xc5,0xfb,0x92,0xce]
-; X64-NEXT:    vpsubsw (%rdi), %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xa9,0xe9,0x07]
-; X64-NEXT:    retq # encoding: [0xc3]
-  %b = load <16 x i16>, <16 x i16>* %ptr_b
-  %1 = call <16 x i16> @llvm.x86.avx2.psubs.w(<16 x i16> %a, <16 x i16> %b)
-  %2 = bitcast i16 %mask to <16 x i1>
-  %3 = select <16 x i1> %2, <16 x i16> %1, <16 x i16> zeroinitializer
-  ret <16 x i16> %3
-}
-
-declare <16 x i8> @llvm.x86.sse2.padds.b(<16 x i8>, <16 x i8>) #0
-
-define <16 x i8> @test_mask_adds_epi8_rrk_128(<16 x i8> %a, <16 x i8> %b, <16 x i8> %passThru, i16 %mask) {
-; X86-LABEL: test_mask_adds_epi8_rrk_128:
-; X86:       # %bb.0:
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
-; X86-NEXT:    vpaddsb %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf1,0x7d,0x09,0xec,0xd1]
-; X86-NEXT:    vmovdqa %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc2]
-; X86-NEXT:    retl # encoding: [0xc3]
-;
-; X64-LABEL: test_mask_adds_epi8_rrk_128:
-; X64:       # %bb.0:
-; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
-; X64-NEXT:    vpaddsb %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf1,0x7d,0x09,0xec,0xd1]
-; X64-NEXT:    vmovdqa %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc2]
-; X64-NEXT:    retq # encoding: [0xc3]
-  %1 = call <16 x i8> @llvm.x86.sse2.padds.b(<16 x i8> %a, <16 x i8> %b)
-  %2 = bitcast i16 %mask to <16 x i1>
-  %3 = select <16 x i1> %2, <16 x i8> %1, <16 x i8> %passThru
-  ret <16 x i8> %3
-}
-
-define <16 x i8> @test_mask_adds_epi8_rrkz_128(<16 x i8> %a, <16 x i8> %b, i16 %mask) {
-; X86-LABEL: test_mask_adds_epi8_rrkz_128:
-; X86:       # %bb.0:
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
-; X86-NEXT:    vpaddsb %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0x89,0xec,0xc1]
-; X86-NEXT:    retl # encoding: [0xc3]
-;
-; X64-LABEL: test_mask_adds_epi8_rrkz_128:
-; X64:       # %bb.0:
-; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
-; X64-NEXT:    vpaddsb %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0x89,0xec,0xc1]
-; X64-NEXT:    retq # encoding: [0xc3]
-  %1 = call <16 x i8> @llvm.x86.sse2.padds.b(<16 x i8> %a, <16 x i8> %b)
-  %2 = bitcast i16 %mask to <16 x i1>
-  %3 = select <16 x i1> %2, <16 x i8> %1, <16 x i8> zeroinitializer
-  ret <16 x i8> %3
-}
-
-define <16 x i8> @test_mask_adds_epi8_rm_128(<16 x i8> %a, <16 x i8>* %ptr_b) {
-; X86-LABEL: test_mask_adds_epi8_rm_128:
-; X86:       # %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
-; X86-NEXT:    vpaddsb (%eax), %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xec,0x00]
-; X86-NEXT:    retl # encoding: [0xc3]
-;
-; X64-LABEL: test_mask_adds_epi8_rm_128:
-; X64:       # %bb.0:
-; X64-NEXT:    vpaddsb (%rdi), %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xec,0x07]
-; X64-NEXT:    retq # encoding: [0xc3]
-  %b = load <16 x i8>, <16 x i8>* %ptr_b
-  %1 = call <16 x i8> @llvm.x86.sse2.padds.b(<16 x i8> %a, <16 x i8> %b)
-  ret <16 x i8> %1
-}
-
-define <16 x i8> @test_mask_adds_epi8_rmk_128(<16 x i8> %a, <16 x i8>* %ptr_b, <16 x i8> %passThru, i16 %mask) {
-; X86-LABEL: test_mask_adds_epi8_rmk_128:
-; X86:       # %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x08]
-; X86-NEXT:    vpaddsb (%eax), %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf1,0x7d,0x09,0xec,0x08]
-; X86-NEXT:    vmovdqa %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc1]
-; X86-NEXT:    retl # encoding: [0xc3]
-;
-; X64-LABEL: test_mask_adds_epi8_rmk_128:
-; X64:       # %bb.0:
-; X64-NEXT:    kmovd %esi, %k1 # encoding: [0xc5,0xfb,0x92,0xce]
-; X64-NEXT:    vpaddsb (%rdi), %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf1,0x7d,0x09,0xec,0x0f]
-; X64-NEXT:    vmovdqa %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc1]
-; X64-NEXT:    retq # encoding: [0xc3]
-  %b = load <16 x i8>, <16 x i8>* %ptr_b
-  %1 = call <16 x i8> @llvm.x86.sse2.padds.b(<16 x i8> %a, <16 x i8> %b)
-  %2 = bitcast i16 %mask to <16 x i1>
-  %3 = select <16 x i1> %2, <16 x i8> %1, <16 x i8> %passThru
-  ret <16 x i8> %3
-}
-
-define <16 x i8> @test_mask_adds_epi8_rmkz_128(<16 x i8> %a, <16 x i8>* %ptr_b, i16 %mask) {
-; X86-LABEL: test_mask_adds_epi8_rmkz_128:
-; X86:       # %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x08]
-; X86-NEXT:    vpaddsb (%eax), %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0x89,0xec,0x00]
-; X86-NEXT:    retl # encoding: [0xc3]
-;
-; X64-LABEL: test_mask_adds_epi8_rmkz_128:
-; X64:       # %bb.0:
-; X64-NEXT:    kmovd %esi, %k1 # encoding: [0xc5,0xfb,0x92,0xce]
-; X64-NEXT:    vpaddsb (%rdi), %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0x89,0xec,0x07]
-; X64-NEXT:    retq # encoding: [0xc3]
-  %b = load <16 x i8>, <16 x i8>* %ptr_b
-  %1 = call <16 x i8> @llvm.x86.sse2.padds.b(<16 x i8> %a, <16 x i8> %b)
-  %2 = bitcast i16 %mask to <16 x i1>
-  %3 = select <16 x i1> %2, <16 x i8> %1, <16 x i8> zeroinitializer
-  ret <16 x i8> %3
-}
-
-declare <32 x i8> @llvm.x86.avx2.padds.b(<32 x i8>, <32 x i8>) #0
-
-define <32 x i8> @test_mask_adds_epi8_rrk_256(<32 x i8> %a, <32 x i8> %b, <32 x i8> %passThru, i32 %mask) {
-; X86-LABEL: test_mask_adds_epi8_rrk_256:
-; X86:       # %bb.0:
-; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf9,0x90,0x4c,0x24,0x04]
-; X86-NEXT:    vpaddsb %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf1,0x7d,0x29,0xec,0xd1]
-; X86-NEXT:    vmovdqa %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc2]
-; X86-NEXT:    retl # encoding: [0xc3]
-;
-; X64-LABEL: test_mask_adds_epi8_rrk_256:
-; X64:       # %bb.0:
-; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
-; X64-NEXT:    vpaddsb %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf1,0x7d,0x29,0xec,0xd1]
-; X64-NEXT:    vmovdqa %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc2]
-; X64-NEXT:    retq # encoding: [0xc3]
-  %1 = call <32 x i8> @llvm.x86.avx2.padds.b(<32 x i8> %a, <32 x i8> %b)
-  %2 = bitcast i32 %mask to <32 x i1>
-  %3 = select <32 x i1> %2, <32 x i8> %1, <32 x i8> %passThru
-  ret <32 x i8> %3
-}
-
-define <32 x i8> @test_mask_adds_epi8_rrkz_256(<32 x i8> %a, <32 x i8> %b, i32 %mask) {
-; X86-LABEL: test_mask_adds_epi8_rrkz_256:
-; X86:       # %bb.0:
-; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf9,0x90,0x4c,0x24,0x04]
-; X86-NEXT:    vpaddsb %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xa9,0xec,0xc1]
-; X86-NEXT:    retl # encoding: [0xc3]
-;
-; X64-LABEL: test_mask_adds_epi8_rrkz_256:
-; X64:       # %bb.0:
-; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
-; X64-NEXT:    vpaddsb %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xa9,0xec,0xc1]
-; X64-NEXT:    retq # encoding: [0xc3]
-  %1 = call <32 x i8> @llvm.x86.avx2.padds.b(<32 x i8> %a, <32 x i8> %b)
-  %2 = bitcast i32 %mask to <32 x i1>
-  %3 = select <32 x i1> %2, <32 x i8> %1, <32 x i8> zeroinitializer
-  ret <32 x i8> %3
-}
-
-define <32 x i8> @test_mask_adds_epi8_rmk_256(<32 x i8> %a, <32 x i8>* %ptr_b, <32 x i8> %passThru, i32 %mask) {
-; X86-LABEL: test_mask_adds_epi8_rmk_256:
-; X86:       # %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
-; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf9,0x90,0x4c,0x24,0x08]
-; X86-NEXT:    vpaddsb (%eax), %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf1,0x7d,0x29,0xec,0x08]
-; X86-NEXT:    vmovdqa %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc1]
-; X86-NEXT:    retl # encoding: [0xc3]
-;
-; X64-LABEL: test_mask_adds_epi8_rmk_256:
-; X64:       # %bb.0:
-; X64-NEXT:    kmovd %esi, %k1 # encoding: [0xc5,0xfb,0x92,0xce]
-; X64-NEXT:    vpaddsb (%rdi), %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf1,0x7d,0x29,0xec,0x0f]
-; X64-NEXT:    vmovdqa %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc1]
-; X64-NEXT:    retq # encoding: [0xc3]
-  %b = load <32 x i8>, <32 x i8>* %ptr_b
-  %1 = call <32 x i8> @llvm.x86.avx2.padds.b(<32 x i8> %a, <32 x i8> %b)
-  %2 = bitcast i32 %mask to <32 x i1>
-  %3 = select <32 x i1> %2, <32 x i8> %1, <32 x i8> %passThru
-  ret <32 x i8> %3
-}
-
-define <32 x i8> @test_mask_adds_epi8_rmkz_256(<32 x i8> %a, <32 x i8>* %ptr_b, i32 %mask) {
-; X86-LABEL: test_mask_adds_epi8_rmkz_256:
-; X86:       # %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
-; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf9,0x90,0x4c,0x24,0x08]
-; X86-NEXT:    vpaddsb (%eax), %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xa9,0xec,0x00]
-; X86-NEXT:    retl # encoding: [0xc3]
-;
-; X64-LABEL: test_mask_adds_epi8_rmkz_256:
-; X64:       # %bb.0:
-; X64-NEXT:    kmovd %esi, %k1 # encoding: [0xc5,0xfb,0x92,0xce]
-; X64-NEXT:    vpaddsb (%rdi), %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xa9,0xec,0x07]
-; X64-NEXT:    retq # encoding: [0xc3]
-  %b = load <32 x i8>, <32 x i8>* %ptr_b
-  %1 = call <32 x i8> @llvm.x86.avx2.padds.b(<32 x i8> %a, <32 x i8> %b)
-  %2 = bitcast i32 %mask to <32 x i1>
-  %3 = select <32 x i1> %2, <32 x i8> %1, <32 x i8> zeroinitializer
-  ret <32 x i8> %3
-}
-
-declare <16 x i8> @llvm.x86.sse2.psubs.b(<16 x i8>, <16 x i8>) #0
-
-define <16 x i8> @test_mask_subs_epi8_rrk_128(<16 x i8> %a, <16 x i8> %b, <16 x i8> %passThru, i16 %mask) {
-; X86-LABEL: test_mask_subs_epi8_rrk_128:
-; X86:       # %bb.0:
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
-; X86-NEXT:    vpsubsb %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf1,0x7d,0x09,0xe8,0xd1]
-; X86-NEXT:    vmovdqa %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc2]
-; X86-NEXT:    retl # encoding: [0xc3]
-;
-; X64-LABEL: test_mask_subs_epi8_rrk_128:
-; X64:       # %bb.0:
-; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
-; X64-NEXT:    vpsubsb %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf1,0x7d,0x09,0xe8,0xd1]
-; X64-NEXT:    vmovdqa %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc2]
-; X64-NEXT:    retq # encoding: [0xc3]
-  %1 = call <16 x i8> @llvm.x86.sse2.psubs.b(<16 x i8> %a, <16 x i8> %b)
-  %2 = bitcast i16 %mask to <16 x i1>
-  %3 = select <16 x i1> %2, <16 x i8> %1, <16 x i8> %passThru
-  ret <16 x i8> %3
-}
-
-define <16 x i8> @test_mask_subs_epi8_rrkz_128(<16 x i8> %a, <16 x i8> %b, i16 %mask) {
-; X86-LABEL: test_mask_subs_epi8_rrkz_128:
-; X86:       # %bb.0:
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
-; X86-NEXT:    vpsubsb %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0x89,0xe8,0xc1]
-; X86-NEXT:    retl # encoding: [0xc3]
-;
-; X64-LABEL: test_mask_subs_epi8_rrkz_128:
-; X64:       # %bb.0:
-; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
-; X64-NEXT:    vpsubsb %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0x89,0xe8,0xc1]
-; X64-NEXT:    retq # encoding: [0xc3]
-  %1 = call <16 x i8> @llvm.x86.sse2.psubs.b(<16 x i8> %a, <16 x i8> %b)
-  %2 = bitcast i16 %mask to <16 x i1>
-  %3 = select <16 x i1> %2, <16 x i8> %1, <16 x i8> zeroinitializer
-  ret <16 x i8> %3
-}
-
-define <16 x i8> @test_mask_subs_epi8_rmk_128(<16 x i8> %a, <16 x i8>* %ptr_b, <16 x i8> %passThru, i16 %mask) {
-; X86-LABEL: test_mask_subs_epi8_rmk_128:
-; X86:       # %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x08]
-; X86-NEXT:    vpsubsb (%eax), %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf1,0x7d,0x09,0xe8,0x08]
-; X86-NEXT:    vmovdqa %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc1]
-; X86-NEXT:    retl # encoding: [0xc3]
-;
-; X64-LABEL: test_mask_subs_epi8_rmk_128:
-; X64:       # %bb.0:
-; X64-NEXT:    kmovd %esi, %k1 # encoding: [0xc5,0xfb,0x92,0xce]
-; X64-NEXT:    vpsubsb (%rdi), %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf1,0x7d,0x09,0xe8,0x0f]
-; X64-NEXT:    vmovdqa %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc1]
-; X64-NEXT:    retq # encoding: [0xc3]
-  %b = load <16 x i8>, <16 x i8>* %ptr_b
-  %1 = call <16 x i8> @llvm.x86.sse2.psubs.b(<16 x i8> %a, <16 x i8> %b)
-  %2 = bitcast i16 %mask to <16 x i1>
-  %3 = select <16 x i1> %2, <16 x i8> %1, <16 x i8> %passThru
-  ret <16 x i8> %3
-}
-
-define <16 x i8> @test_mask_subs_epi8_rmkz_128(<16 x i8> %a, <16 x i8>* %ptr_b, i16 %mask) {
-; X86-LABEL: test_mask_subs_epi8_rmkz_128:
-; X86:       # %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x08]
-; X86-NEXT:    vpsubsb (%eax), %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0x89,0xe8,0x00]
-; X86-NEXT:    retl # encoding: [0xc3]
-;
-; X64-LABEL: test_mask_subs_epi8_rmkz_128:
-; X64:       # %bb.0:
-; X64-NEXT:    kmovd %esi, %k1 # encoding: [0xc5,0xfb,0x92,0xce]
-; X64-NEXT:    vpsubsb (%rdi), %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0x89,0xe8,0x07]
-; X64-NEXT:    retq # encoding: [0xc3]
-  %b = load <16 x i8>, <16 x i8>* %ptr_b
-  %1 = call <16 x i8> @llvm.x86.sse2.psubs.b(<16 x i8> %a, <16 x i8> %b)
-  %2 = bitcast i16 %mask to <16 x i1>
-  %3 = select <16 x i1> %2, <16 x i8> %1, <16 x i8> zeroinitializer
-  ret <16 x i8> %3
-}
-
-declare <32 x i8> @llvm.x86.avx2.psubs.b(<32 x i8>, <32 x i8>) #0
-
-define <32 x i8> @test_mask_subs_epi8_rrk_256(<32 x i8> %a, <32 x i8> %b, <32 x i8> %passThru, i32 %mask) {
-; X86-LABEL: test_mask_subs_epi8_rrk_256:
-; X86:       # %bb.0:
-; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf9,0x90,0x4c,0x24,0x04]
-; X86-NEXT:    vpsubsb %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf1,0x7d,0x29,0xe8,0xd1]
-; X86-NEXT:    vmovdqa %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc2]
-; X86-NEXT:    retl # encoding: [0xc3]
-;
-; X64-LABEL: test_mask_subs_epi8_rrk_256:
-; X64:       # %bb.0:
-; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
-; X64-NEXT:    vpsubsb %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf1,0x7d,0x29,0xe8,0xd1]
-; X64-NEXT:    vmovdqa %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc2]
-; X64-NEXT:    retq # encoding: [0xc3]
-  %1 = call <32 x i8> @llvm.x86.avx2.psubs.b(<32 x i8> %a, <32 x i8> %b)
-  %2 = bitcast i32 %mask to <32 x i1>
-  %3 = select <32 x i1> %2, <32 x i8> %1, <32 x i8> %passThru
-  ret <32 x i8> %3
-}
-
-define <32 x i8> @test_mask_subs_epi8_rrkz_256(<32 x i8> %a, <32 x i8> %b, i32 %mask) {
-; X86-LABEL: test_mask_subs_epi8_rrkz_256:
-; X86:       # %bb.0:
-; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf9,0x90,0x4c,0x24,0x04]
-; X86-NEXT:    vpsubsb %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xa9,0xe8,0xc1]
-; X86-NEXT:    retl # encoding: [0xc3]
-;
-; X64-LABEL: test_mask_subs_epi8_rrkz_256:
-; X64:       # %bb.0:
-; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
-; X64-NEXT:    vpsubsb %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xa9,0xe8,0xc1]
-; X64-NEXT:    retq # encoding: [0xc3]
-  %1 = call <32 x i8> @llvm.x86.avx2.psubs.b(<32 x i8> %a, <32 x i8> %b)
-  %2 = bitcast i32 %mask to <32 x i1>
-  %3 = select <32 x i1> %2, <32 x i8> %1, <32 x i8> zeroinitializer
-  ret <32 x i8> %3
-}
-
-define <32 x i8> @test_mask_subs_epi8_rmk_256(<32 x i8> %a, <32 x i8>* %ptr_b, <32 x i8> %passThru, i32 %mask) {
-; X86-LABEL: test_mask_subs_epi8_rmk_256:
-; X86:       # %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
-; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf9,0x90,0x4c,0x24,0x08]
-; X86-NEXT:    vpsubsb (%eax), %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf1,0x7d,0x29,0xe8,0x08]
-; X86-NEXT:    vmovdqa %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc1]
-; X86-NEXT:    retl # encoding: [0xc3]
-;
-; X64-LABEL: test_mask_subs_epi8_rmk_256:
-; X64:       # %bb.0:
-; X64-NEXT:    kmovd %esi, %k1 # encoding: [0xc5,0xfb,0x92,0xce]
-; X64-NEXT:    vpsubsb (%rdi), %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf1,0x7d,0x29,0xe8,0x0f]
-; X64-NEXT:    vmovdqa %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc1]
-; X64-NEXT:    retq # encoding: [0xc3]
-  %b = load <32 x i8>, <32 x i8>* %ptr_b
-  %1 = call <32 x i8> @llvm.x86.avx2.psubs.b(<32 x i8> %a, <32 x i8> %b)
-  %2 = bitcast i32 %mask to <32 x i1>
-  %3 = select <32 x i1> %2, <32 x i8> %1, <32 x i8> %passThru
-  ret <32 x i8> %3
-}
-
-define <32 x i8> @test_mask_subs_epi8_rmkz_256(<32 x i8> %a, <32 x i8>* %ptr_b, i32 %mask) {
-; X86-LABEL: test_mask_subs_epi8_rmkz_256:
-; X86:       # %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
-; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf9,0x90,0x4c,0x24,0x08]
-; X86-NEXT:    vpsubsb (%eax), %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xa9,0xe8,0x00]
-; X86-NEXT:    retl # encoding: [0xc3]
-;
-; X64-LABEL: test_mask_subs_epi8_rmkz_256:
-; X64:       # %bb.0:
-; X64-NEXT:    kmovd %esi, %k1 # encoding: [0xc5,0xfb,0x92,0xce]
-; X64-NEXT:    vpsubsb (%rdi), %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xa9,0xe8,0x07]
-; X64-NEXT:    retq # encoding: [0xc3]
-  %b = load <32 x i8>, <32 x i8>* %ptr_b
-  %1 = call <32 x i8> @llvm.x86.avx2.psubs.b(<32 x i8> %a, <32 x i8> %b)
-  %2 = bitcast i32 %mask to <32 x i1>
-  %3 = select <32 x i1> %2, <32 x i8> %1, <32 x i8> zeroinitializer
-  ret <32 x i8> %3
-}
-
 define <8 x i16>@test_int_x86_avx512_mask_vpermt2var_hi_128(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpermt2var_hi_128:
 ; X86:       # %bb.0:
@@ -2134,22 +1454,23 @@ declare <16 x i8> @llvm.x86.avx512.mask.pmov.wb.128(<8 x i16>, <16 x i8>, i8)
 define <16 x i8>@test_int_x86_avx512_mask_pmov_wb_128(<8 x i16> %x0, <16 x i8> %x1, i8 %x2) {
 ; X86-LABEL: test_int_x86_avx512_mask_pmov_wb_128:
 ; X86:       # %bb.0:
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
-; X86-NEXT:    vpmovwb %xmm0, %xmm2 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0x89,0x30,0xc2]
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
+; X86-NEXT:    vpmovwb %xmm0, %xmm2 # encoding: [0x62,0xf2,0x7e,0x08,0x30,0xc2]
 ; X86-NEXT:    vpmovwb %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0x7e,0x09,0x30,0xc1]
-; X86-NEXT:    vpaddb %xmm2, %xmm1, %xmm1 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0xfc,0xca]
-; X86-NEXT:    vpmovwb %xmm0, %xmm0 # encoding: [0x62,0xf2,0x7e,0x08,0x30,0xc0]
-; X86-NEXT:    vpaddb %xmm1, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfc,0xc1]
+; X86-NEXT:    vpmovwb %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0x89,0x30,0xc0]
+; X86-NEXT:    vpaddb %xmm0, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0xfc,0xc0]
+; X86-NEXT:    vpaddb %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfc,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_pmov_wb_128:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
-; X64-NEXT:    vpmovwb %xmm0, %xmm2 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0x89,0x30,0xc2]
+; X64-NEXT:    vpmovwb %xmm0, %xmm2 # encoding: [0x62,0xf2,0x7e,0x08,0x30,0xc2]
 ; X64-NEXT:    vpmovwb %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0x7e,0x09,0x30,0xc1]
-; X64-NEXT:    vpaddb %xmm2, %xmm1, %xmm1 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0xfc,0xca]
-; X64-NEXT:    vpmovwb %xmm0, %xmm0 # encoding: [0x62,0xf2,0x7e,0x08,0x30,0xc0]
-; X64-NEXT:    vpaddb %xmm1, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfc,0xc1]
+; X64-NEXT:    vpmovwb %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0x89,0x30,0xc0]
+; X64-NEXT:    vpaddb %xmm0, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0xfc,0xc0]
+; X64-NEXT:    vpaddb %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfc,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
     %res0 = call <16 x i8> @llvm.x86.avx512.mask.pmov.wb.128(<8 x i16> %x0, <16 x i8> %x1, i8 -1)
     %res1 = call <16 x i8> @llvm.x86.avx512.mask.pmov.wb.128(<8 x i16> %x0, <16 x i8> %x1, i8 %x2)
@@ -2187,22 +1508,23 @@ declare <16 x i8> @llvm.x86.avx512.mask.pmovs.wb.128(<8 x i16>, <16 x i8>, i8)
 define <16 x i8>@test_int_x86_avx512_mask_pmovs_wb_128(<8 x i16> %x0, <16 x i8> %x1, i8 %x2) {
 ; X86-LABEL: test_int_x86_avx512_mask_pmovs_wb_128:
 ; X86:       # %bb.0:
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
-; X86-NEXT:    vpmovswb %xmm0, %xmm2 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0x89,0x20,0xc2]
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
+; X86-NEXT:    vpmovswb %xmm0, %xmm2 # encoding: [0x62,0xf2,0x7e,0x08,0x20,0xc2]
 ; X86-NEXT:    vpmovswb %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0x7e,0x09,0x20,0xc1]
-; X86-NEXT:    vpaddb %xmm2, %xmm1, %xmm1 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0xfc,0xca]
-; X86-NEXT:    vpmovswb %xmm0, %xmm0 # encoding: [0x62,0xf2,0x7e,0x08,0x20,0xc0]
-; X86-NEXT:    vpaddb %xmm1, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfc,0xc1]
+; X86-NEXT:    vpmovswb %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0x89,0x20,0xc0]
+; X86-NEXT:    vpaddb %xmm0, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0xfc,0xc0]
+; X86-NEXT:    vpaddb %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfc,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_pmovs_wb_128:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
-; X64-NEXT:    vpmovswb %xmm0, %xmm2 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0x89,0x20,0xc2]
+; X64-NEXT:    vpmovswb %xmm0, %xmm2 # encoding: [0x62,0xf2,0x7e,0x08,0x20,0xc2]
 ; X64-NEXT:    vpmovswb %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0x7e,0x09,0x20,0xc1]
-; X64-NEXT:    vpaddb %xmm2, %xmm1, %xmm1 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0xfc,0xca]
-; X64-NEXT:    vpmovswb %xmm0, %xmm0 # encoding: [0x62,0xf2,0x7e,0x08,0x20,0xc0]
-; X64-NEXT:    vpaddb %xmm1, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfc,0xc1]
+; X64-NEXT:    vpmovswb %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0x89,0x20,0xc0]
+; X64-NEXT:    vpaddb %xmm0, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0xfc,0xc0]
+; X64-NEXT:    vpaddb %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfc,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
     %res0 = call <16 x i8> @llvm.x86.avx512.mask.pmovs.wb.128(<8 x i16> %x0, <16 x i8> %x1, i8 -1)
     %res1 = call <16 x i8> @llvm.x86.avx512.mask.pmovs.wb.128(<8 x i16> %x0, <16 x i8> %x1, i8 %x2)
@@ -2240,22 +1562,23 @@ declare <16 x i8> @llvm.x86.avx512.mask.pmovus.wb.128(<8 x i16>, <16 x i8>, i8)
 define <16 x i8>@test_int_x86_avx512_mask_pmovus_wb_128(<8 x i16> %x0, <16 x i8> %x1, i8 %x2) {
 ; X86-LABEL: test_int_x86_avx512_mask_pmovus_wb_128:
 ; X86:       # %bb.0:
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
-; X86-NEXT:    vpmovuswb %xmm0, %xmm2 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0x89,0x10,0xc2]
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
+; X86-NEXT:    vpmovuswb %xmm0, %xmm2 # encoding: [0x62,0xf2,0x7e,0x08,0x10,0xc2]
 ; X86-NEXT:    vpmovuswb %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0x7e,0x09,0x10,0xc1]
-; X86-NEXT:    vpaddb %xmm2, %xmm1, %xmm1 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0xfc,0xca]
-; X86-NEXT:    vpmovuswb %xmm0, %xmm0 # encoding: [0x62,0xf2,0x7e,0x08,0x10,0xc0]
-; X86-NEXT:    vpaddb %xmm1, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfc,0xc1]
+; X86-NEXT:    vpmovuswb %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0x89,0x10,0xc0]
+; X86-NEXT:    vpaddb %xmm0, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0xfc,0xc0]
+; X86-NEXT:    vpaddb %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfc,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_pmovus_wb_128:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
-; X64-NEXT:    vpmovuswb %xmm0, %xmm2 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0x89,0x10,0xc2]
+; X64-NEXT:    vpmovuswb %xmm0, %xmm2 # encoding: [0x62,0xf2,0x7e,0x08,0x10,0xc2]
 ; X64-NEXT:    vpmovuswb %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0x7e,0x09,0x10,0xc1]
-; X64-NEXT:    vpaddb %xmm2, %xmm1, %xmm1 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0xfc,0xca]
-; X64-NEXT:    vpmovuswb %xmm0, %xmm0 # encoding: [0x62,0xf2,0x7e,0x08,0x10,0xc0]
-; X64-NEXT:    vpaddb %xmm1, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfc,0xc1]
+; X64-NEXT:    vpmovuswb %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0x89,0x10,0xc0]
+; X64-NEXT:    vpaddb %xmm0, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0xfc,0xc0]
+; X64-NEXT:    vpaddb %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfc,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
     %res0 = call <16 x i8> @llvm.x86.avx512.mask.pmovus.wb.128(<8 x i16> %x0, <16 x i8> %x1, i8 -1)
     %res1 = call <16 x i8> @llvm.x86.avx512.mask.pmovus.wb.128(<8 x i16> %x0, <16 x i8> %x1, i8 %x2)
@@ -2288,36 +1611,38 @@ define void @test_int_x86_avx512_mask_pmovus_wb_mem_128(i8* %ptr, <8 x i16> %x1,
     ret void
 }
 
-declare <16 x i8> @llvm.x86.avx512.mask.pmov.wb.256(<16 x i16>, <16 x i8>, i16)
-
 define <16 x i8>@test_int_x86_avx512_mask_pmov_wb_256(<16 x i16> %x0, <16 x i8> %x1, i16 %x2) {
 ; X86-LABEL: test_int_x86_avx512_mask_pmov_wb_256:
 ; X86:       # %bb.0:
+; X86-NEXT:    vpmovwb %ymm0, %xmm2 # encoding: [0x62,0xf2,0x7e,0x28,0x30,0xc2]
 ; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
 ; X86-NEXT:    vpmovwb %ymm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0x7e,0x29,0x30,0xc1]
-; X86-NEXT:    vpmovwb %ymm0, %xmm2 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0xa9,0x30,0xc2]
-; X86-NEXT:    vpaddb %xmm2, %xmm1, %xmm1 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0xfc,0xca]
-; X86-NEXT:    vpmovwb %ymm0, %xmm0 # encoding: [0x62,0xf2,0x7e,0x28,0x30,0xc0]
-; X86-NEXT:    vpaddb %xmm1, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfc,0xc1]
+; X86-NEXT:    vpmovwb %ymm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0xa9,0x30,0xc0]
+; X86-NEXT:    vpaddb %xmm0, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0xfc,0xc0]
+; X86-NEXT:    vpaddb %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfc,0xc0]
 ; X86-NEXT:    vzeroupper # encoding: [0xc5,0xf8,0x77]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_pmov_wb_256:
 ; X64:       # %bb.0:
+; X64-NEXT:    vpmovwb %ymm0, %xmm2 # encoding: [0x62,0xf2,0x7e,0x28,0x30,0xc2]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
-; X64-NEXT:    vpmovwb %ymm0, %xmm2 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0xa9,0x30,0xc2]
 ; X64-NEXT:    vpmovwb %ymm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0x7e,0x29,0x30,0xc1]
-; X64-NEXT:    vpaddb %xmm2, %xmm1, %xmm1 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0xfc,0xca]
-; X64-NEXT:    vpmovwb %ymm0, %xmm0 # encoding: [0x62,0xf2,0x7e,0x28,0x30,0xc0]
-; X64-NEXT:    vpaddb %xmm1, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfc,0xc1]
+; X64-NEXT:    vpmovwb %ymm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0xa9,0x30,0xc0]
+; X64-NEXT:    vpaddb %xmm0, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0xfc,0xc0]
+; X64-NEXT:    vpaddb %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfc,0xc0]
 ; X64-NEXT:    vzeroupper # encoding: [0xc5,0xf8,0x77]
 ; X64-NEXT:    retq # encoding: [0xc3]
-    %res0 = call <16 x i8> @llvm.x86.avx512.mask.pmov.wb.256(<16 x i16> %x0, <16 x i8> %x1, i16 -1)
-    %res1 = call <16 x i8> @llvm.x86.avx512.mask.pmov.wb.256(<16 x i16> %x0, <16 x i8> %x1, i16 %x2)
-    %res2 = call <16 x i8> @llvm.x86.avx512.mask.pmov.wb.256(<16 x i16> %x0, <16 x i8> zeroinitializer, i16 %x2)
-    %res3 = add <16 x i8> %res0, %res1
-    %res4 = add <16 x i8> %res3, %res2
-    ret <16 x i8> %res4
+  %1 = trunc <16 x i16> %x0 to <16 x i8>
+  %2 = trunc <16 x i16> %x0 to <16 x i8>
+  %3 = bitcast i16 %x2 to <16 x i1>
+  %4 = select <16 x i1> %3, <16 x i8> %2, <16 x i8> %x1
+  %5 = trunc <16 x i16> %x0 to <16 x i8>
+  %6 = bitcast i16 %x2 to <16 x i1>
+  %7 = select <16 x i1> %6, <16 x i8> %5, <16 x i8> zeroinitializer
+  %res3 = add <16 x i8> %1, %4
+  %res4 = add <16 x i8> %res3, %7
+  ret <16 x i8> %res4
 }
 
 declare void @llvm.x86.avx512.mask.pmov.wb.mem.256(i8* %ptr, <16 x i16>, i16)
@@ -2694,7 +2019,56 @@ define <8 x i16>@test_int_x86_avx512_mask_psrlv8_hi(<8 x i16> %x0, <8 x i16> %x1
   ret <8 x i16> %res4
 }
 
-declare <16 x i16> @llvm.x86.avx512.mask.psrav16.hi(<16 x i16>, <16 x i16>, <16 x i16>, i16)
+
+define <8 x i16> @test_int_x86_avx512_psrlv_w_128_const() optsize {
+; X86-LABEL: test_int_x86_avx512_psrlv_w_128_const:
+; X86:       # %bb.0:
+; X86-NEXT:    vmovdqa {{\.LCPI.*}}, %xmm0 # EVEX TO VEX Compression xmm0 = [4,4,4,4,4,4,4,65535]
+; X86-NEXT:    # encoding: [0xc5,0xf9,0x6f,0x05,A,A,A,A]
+; X86-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}, kind: FK_Data_4
+; X86-NEXT:    vpsrlvw {{\.LCPI.*}}, %xmm0, %xmm0 # encoding: [0x62,0xf2,0xfd,0x08,0x10,0x05,A,A,A,A]
+; X86-NEXT:    # fixup A - offset: 6, value: {{\.LCPI.*}}, kind: FK_Data_4
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_psrlv_w_128_const:
+; X64:       # %bb.0:
+; X64-NEXT:    vmovdqa {{.*}}(%rip), %xmm0 # EVEX TO VEX Compression xmm0 = [4,4,4,4,4,4,4,65535]
+; X64-NEXT:    # encoding: [0xc5,0xf9,0x6f,0x05,A,A,A,A]
+; X64-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-NEXT:    vpsrlvw {{.*}}(%rip), %xmm0, %xmm0 # encoding: [0x62,0xf2,0xfd,0x08,0x10,0x05,A,A,A,A]
+; X64-NEXT:    # fixup A - offset: 6, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <8 x i16> @llvm.x86.avx512.psrlv.w.128(<8 x i16> <i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 -1>, <8 x i16> <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 -1>)
+  ret <8 x i16> %res
+}
+
+declare <8 x i16> @llvm.x86.avx512.psrlv.w.128(<8 x i16>, <8 x i16>)
+
+define <16 x i16> @test_int_x86_avx512_psrlv_w_256_const() optsize {
+; X86-LABEL: test_int_x86_avx512_psrlv_w_256_const:
+; X86:       # %bb.0:
+; X86-NEXT:    vmovdqa {{\.LCPI.*}}, %ymm0 # EVEX TO VEX Compression ymm0 = [4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,65535]
+; X86-NEXT:    # encoding: [0xc5,0xfd,0x6f,0x05,A,A,A,A]
+; X86-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}, kind: FK_Data_4
+; X86-NEXT:    vpsrlvw {{\.LCPI.*}}, %ymm0, %ymm0 # encoding: [0x62,0xf2,0xfd,0x28,0x10,0x05,A,A,A,A]
+; X86-NEXT:    # fixup A - offset: 6, value: {{\.LCPI.*}}, kind: FK_Data_4
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_psrlv_w_256_const:
+; X64:       # %bb.0:
+; X64-NEXT:    vmovdqa {{.*}}(%rip), %ymm0 # EVEX TO VEX Compression ymm0 = [4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,65535]
+; X64-NEXT:    # encoding: [0xc5,0xfd,0x6f,0x05,A,A,A,A]
+; X64-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-NEXT:    vpsrlvw {{.*}}(%rip), %ymm0, %ymm0 # encoding: [0x62,0xf2,0xfd,0x28,0x10,0x05,A,A,A,A]
+; X64-NEXT:    # fixup A - offset: 6, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <16 x i16> @llvm.x86.avx512.psrlv.w.256(<16 x i16> <i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 -1>, <16 x i16> <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 -1>)
+  ret <16 x i16> %res
+}
+
+declare <16 x i16> @llvm.x86.avx512.psrlv.w.256(<16 x i16>, <16 x i16>)
+
+declare <16 x i16> @llvm.x86.avx512.psrav.w.256(<16 x i16>, <16 x i16>)
 
 define <16 x i16>@test_int_x86_avx512_mask_psrav16_hi(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x2, i16 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_psrav16_hi:
@@ -2716,15 +2090,19 @@ define <16 x i16>@test_int_x86_avx512_mask_psrav16_hi(<16 x i16> %x0, <16 x i16>
 ; X64-NEXT:    vpaddw %ymm3, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xfd,0xc3]
 ; X64-NEXT:    vpaddw %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfd,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <16 x i16> @llvm.x86.avx512.mask.psrav16.hi(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x2, i16 %x3)
-  %res1 = call <16 x i16> @llvm.x86.avx512.mask.psrav16.hi(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> zeroinitializer, i16 %x3)
-  %res2 = call <16 x i16> @llvm.x86.avx512.mask.psrav16.hi(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x2, i16 -1)
-  %res3 = add <16 x i16> %res, %res1
-  %res4 = add <16 x i16> %res3, %res2
+  %1 = call <16 x i16> @llvm.x86.avx512.psrav.w.256(<16 x i16> %x0, <16 x i16> %x1)
+  %2 = bitcast i16 %x3 to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x i16> %1, <16 x i16> %x2
+  %4 = call <16 x i16> @llvm.x86.avx512.psrav.w.256(<16 x i16> %x0, <16 x i16> %x1)
+  %5 = bitcast i16 %x3 to <16 x i1>
+  %6 = select <16 x i1> %5, <16 x i16> %4, <16 x i16> zeroinitializer
+  %7 = call <16 x i16> @llvm.x86.avx512.psrav.w.256(<16 x i16> %x0, <16 x i16> %x1)
+  %res3 = add <16 x i16> %3, %6
+  %res4 = add <16 x i16> %res3, %7
   ret <16 x i16> %res4
 }
 
-declare <8 x i16> @llvm.x86.avx512.mask.psrav8.hi(<8 x i16>, <8 x i16>, <8 x i16>, i8)
+declare <8 x i16> @llvm.x86.avx512.psrav.w.128(<8 x i16>, <8 x i16>)
 
 define <8 x i16>@test_int_x86_avx512_mask_psrav8_hi(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_psrav8_hi:
@@ -2747,16 +2125,18 @@ define <8 x i16>@test_int_x86_avx512_mask_psrav8_hi(<8 x i16> %x0, <8 x i16> %x1
 ; X64-NEXT:    vpaddw %xmm3, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfd,0xc3]
 ; X64-NEXT:    vpaddw %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfd,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <8 x i16> @llvm.x86.avx512.mask.psrav8.hi(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2, i8 %x3)
-  %res1 = call <8 x i16> @llvm.x86.avx512.mask.psrav8.hi(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> zeroinitializer, i8 %x3)
-  %res2 = call <8 x i16> @llvm.x86.avx512.mask.psrav8.hi(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2, i8 -1)
-  %res3 = add <8 x i16> %res, %res1
-  %res4 = add <8 x i16> %res3, %res2
+  %1 = call <8 x i16> @llvm.x86.avx512.psrav.w.128(<8 x i16> %x0, <8 x i16> %x1)
+  %2 = bitcast i8 %x3 to <8 x i1>
+  %3 = select <8 x i1> %2, <8 x i16> %1, <8 x i16> %x2
+  %4 = call <8 x i16> @llvm.x86.avx512.psrav.w.128(<8 x i16> %x0, <8 x i16> %x1)
+  %5 = bitcast i8 %x3 to <8 x i1>
+  %6 = select <8 x i1> %5, <8 x i16> %4, <8 x i16> zeroinitializer
+  %7 = call <8 x i16> @llvm.x86.avx512.psrav.w.128(<8 x i16> %x0, <8 x i16> %x1)
+  %res3 = add <8 x i16> %3, %6
+  %res4 = add <8 x i16> %res3, %7
   ret <8 x i16> %res4
 }
 
-declare <16 x i16> @llvm.x86.avx512.mask.psllv16.hi(<16 x i16>, <16 x i16>, <16 x i16>, i16)
-
 define <16 x i16>@test_int_x86_avx512_mask_psllv16_hi(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x2, i16 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_psllv16_hi:
 ; X86:       # %bb.0:
@@ -2777,16 +2157,18 @@ define <16 x i16>@test_int_x86_avx512_mask_psllv16_hi(<16 x i16> %x0, <16 x i16>
 ; X64-NEXT:    vpaddw %ymm3, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xfd,0xc3]
 ; X64-NEXT:    vpaddw %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfd,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <16 x i16> @llvm.x86.avx512.mask.psllv16.hi(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x2, i16 %x3)
-  %res1 = call <16 x i16> @llvm.x86.avx512.mask.psllv16.hi(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> zeroinitializer, i16 %x3)
-  %res2 = call <16 x i16> @llvm.x86.avx512.mask.psllv16.hi(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x2, i16 -1)
-  %res3 = add <16 x i16> %res, %res1
-  %res4 = add <16 x i16> %res3, %res2
+  %1 = call <16 x i16> @llvm.x86.avx512.psllv.w.256(<16 x i16> %x0, <16 x i16> %x1)
+  %2 = bitcast i16 %x3 to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x i16> %1, <16 x i16> %x2
+  %4 = call <16 x i16> @llvm.x86.avx512.psllv.w.256(<16 x i16> %x0, <16 x i16> %x1)
+  %5 = bitcast i16 %x3 to <16 x i1>
+  %6 = select <16 x i1> %5, <16 x i16> %4, <16 x i16> zeroinitializer
+  %7 = call <16 x i16> @llvm.x86.avx512.psllv.w.256(<16 x i16> %x0, <16 x i16> %x1)
+  %res3 = add <16 x i16> %3, %6
+  %res4 = add <16 x i16> %res3, %7
   ret <16 x i16> %res4
 }
 
-declare <8 x i16> @llvm.x86.avx512.mask.psllv8.hi(<8 x i16>, <8 x i16>, <8 x i16>, i8)
-
 define <8 x i16>@test_int_x86_avx512_mask_psllv8_hi(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_psllv8_hi:
 ; X86:       # %bb.0:
@@ -2808,14 +2190,69 @@ define <8 x i16>@test_int_x86_avx512_mask_psllv8_hi(<8 x i16> %x0, <8 x i16> %x1
 ; X64-NEXT:    vpaddw %xmm3, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfd,0xc3]
 ; X64-NEXT:    vpaddw %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfd,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <8 x i16> @llvm.x86.avx512.mask.psllv8.hi(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2, i8 %x3)
-  %res1 = call <8 x i16> @llvm.x86.avx512.mask.psllv8.hi(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> zeroinitializer, i8 %x3)
-  %res2 = call <8 x i16> @llvm.x86.avx512.mask.psllv8.hi(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2, i8 -1)
-  %res3 = add <8 x i16> %res, %res1
-  %res4 = add <8 x i16> %res3, %res2
+  %1 = call <8 x i16> @llvm.x86.avx512.psllv.w.128(<8 x i16> %x0, <8 x i16> %x1)
+  %2 = bitcast i8 %x3 to <8 x i1>
+  %3 = select <8 x i1> %2, <8 x i16> %1, <8 x i16> %x2
+  %4 = call <8 x i16> @llvm.x86.avx512.psllv.w.128(<8 x i16> %x0, <8 x i16> %x1)
+  %5 = bitcast i8 %x3 to <8 x i1>
+  %6 = select <8 x i1> %5, <8 x i16> %4, <8 x i16> zeroinitializer
+  %7 = call <8 x i16> @llvm.x86.avx512.psllv.w.128(<8 x i16> %x0, <8 x i16> %x1)
+  %res3 = add <8 x i16> %3, %6
+  %res4 = add <8 x i16> %res3, %7
   ret <8 x i16> %res4
 }
 
+define <8 x i16> @test_int_x86_avx512_psllv_w_128_const() optsize {
+; X86-LABEL: test_int_x86_avx512_psllv_w_128_const:
+; X86:       # %bb.0:
+; X86-NEXT:    vmovdqa {{\.LCPI.*}}, %xmm0 # EVEX TO VEX Compression xmm0 = [4,4,4,4,4,4,4,65535]
+; X86-NEXT:    # encoding: [0xc5,0xf9,0x6f,0x05,A,A,A,A]
+; X86-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}, kind: FK_Data_4
+; X86-NEXT:    vpsllvw {{\.LCPI.*}}, %xmm0, %xmm0 # encoding: [0x62,0xf2,0xfd,0x08,0x12,0x05,A,A,A,A]
+; X86-NEXT:    # fixup A - offset: 6, value: {{\.LCPI.*}}, kind: FK_Data_4
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_psllv_w_128_const:
+; X64:       # %bb.0:
+; X64-NEXT:    vmovdqa {{.*}}(%rip), %xmm0 # EVEX TO VEX Compression xmm0 = [4,4,4,4,4,4,4,65535]
+; X64-NEXT:    # encoding: [0xc5,0xf9,0x6f,0x05,A,A,A,A]
+; X64-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-NEXT:    vpsllvw {{.*}}(%rip), %xmm0, %xmm0 # encoding: [0x62,0xf2,0xfd,0x08,0x12,0x05,A,A,A,A]
+; X64-NEXT:    # fixup A - offset: 6, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <8 x i16> @llvm.x86.avx512.psllv.w.128(<8 x i16> <i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 -1>, <8 x i16> <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 -1>)
+  ret <8 x i16> %res
+}
+
+declare <8 x i16> @llvm.x86.avx512.psllv.w.128(<8 x i16>, <8 x i16>)
+
+
+define <16 x i16> @test_int_x86_avx512_psllv_w_256_const() optsize {
+; X86-LABEL: test_int_x86_avx512_psllv_w_256_const:
+; X86:       # %bb.0:
+; X86-NEXT:    vmovdqa {{\.LCPI.*}}, %ymm0 # EVEX TO VEX Compression ymm0 = [4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,65535]
+; X86-NEXT:    # encoding: [0xc5,0xfd,0x6f,0x05,A,A,A,A]
+; X86-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}, kind: FK_Data_4
+; X86-NEXT:    vpsllvw {{\.LCPI.*}}, %ymm0, %ymm0 # encoding: [0x62,0xf2,0xfd,0x28,0x12,0x05,A,A,A,A]
+; X86-NEXT:    # fixup A - offset: 6, value: {{\.LCPI.*}}, kind: FK_Data_4
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_psllv_w_256_const:
+; X64:       # %bb.0:
+; X64-NEXT:    vmovdqa {{.*}}(%rip), %ymm0 # EVEX TO VEX Compression ymm0 = [4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,65535]
+; X64-NEXT:    # encoding: [0xc5,0xfd,0x6f,0x05,A,A,A,A]
+; X64-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-NEXT:    vpsllvw {{.*}}(%rip), %ymm0, %ymm0 # encoding: [0x62,0xf2,0xfd,0x28,0x12,0x05,A,A,A,A]
+; X64-NEXT:    # fixup A - offset: 6, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <16 x i16> @llvm.x86.avx512.psllv.w.256(<16 x i16> <i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 4, i16 -1>, <16 x i16> <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 -1>)
+  ret <16 x i16> %res
+}
+
+declare <16 x i16> @llvm.x86.avx512.psllv.w.256(<16 x i16>, <16 x i16>)
+
+
+
 declare <8 x i16> @llvm.x86.avx512.permvar.hi.128(<8 x i16>, <8 x i16>)
 
 define <8 x i16>@test_int_x86_avx512_mask_permvar_hi_128(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2, i8 %x3) {
diff --git a/test/CodeGen/X86/avx512cd-intrinsics-upgrade.ll b/test/CodeGen/X86/avx512cd-intrinsics-upgrade.ll
index e5164eea3c831af129438a334054b80f3d00eef8..5e076c1f38c48d1c751fcd291cca0224ac1e7bfb 100644
--- a/test/CodeGen/X86/avx512cd-intrinsics-upgrade.ll
+++ b/test/CodeGen/X86/avx512cd-intrinsics-upgrade.ll
@@ -2,6 +2,98 @@
 ; RUN: llc < %s -mtriple=i686-unknown -mattr=+avx512cd | FileCheck %s --check-prefixes=CHECK,X86
 ; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx512cd | FileCheck %s --check-prefixes=CHECK,X64
 
+declare <16 x i32> @llvm.x86.avx512.mask.conflict.d.512(<16 x i32>, <16 x i32>, i16) nounwind readonly
+
+define <16 x i32> @test_conflict_d(<16 x i32> %a) {
+; CHECK-LABEL: test_conflict_d:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpconflictd %zmm0, %zmm0
+; CHECK-NEXT:    ret{{[l|q]}}
+  %res = call <16 x i32> @llvm.x86.avx512.mask.conflict.d.512(<16 x i32> %a, <16 x i32> undef, i16 -1)
+  ret <16 x i32> %res
+}
+
+define <16 x i32> @test_mask_conflict_d(<16 x i32> %a, <16 x i32> %b, i16 %mask) {
+; X86-LABEL: test_mask_conflict_d:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vpconflictd %zmm0, %zmm1 {%k1}
+; X86-NEXT:    vmovdqa64 %zmm1, %zmm0
+; X86-NEXT:    retl
+;
+; X64-LABEL: test_mask_conflict_d:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vpconflictd %zmm0, %zmm1 {%k1}
+; X64-NEXT:    vmovdqa64 %zmm1, %zmm0
+; X64-NEXT:    retq
+  %res = call <16 x i32> @llvm.x86.avx512.mask.conflict.d.512(<16 x i32> %a, <16 x i32> %b, i16 %mask)
+  ret <16 x i32> %res
+}
+
+define <16 x i32> @test_maskz_conflict_d(<16 x i32> %a, i16 %mask) {
+; X86-LABEL: test_maskz_conflict_d:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vpconflictd %zmm0, %zmm0 {%k1} {z}
+; X86-NEXT:    retl
+;
+; X64-LABEL: test_maskz_conflict_d:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vpconflictd %zmm0, %zmm0 {%k1} {z}
+; X64-NEXT:    retq
+  %res = call <16 x i32> @llvm.x86.avx512.mask.conflict.d.512(<16 x i32> %a, <16 x i32> zeroinitializer, i16 %mask)
+  ret <16 x i32> %res
+}
+
+declare <8 x i64> @llvm.x86.avx512.mask.conflict.q.512(<8 x i64>, <8 x i64>, i8) nounwind readonly
+
+define <8 x i64> @test_conflict_q(<8 x i64> %a) {
+; CHECK-LABEL: test_conflict_q:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpconflictq %zmm0, %zmm0
+; CHECK-NEXT:    ret{{[l|q]}}
+  %res = call <8 x i64> @llvm.x86.avx512.mask.conflict.q.512(<8 x i64> %a, <8 x i64> undef, i8 -1)
+  ret <8 x i64> %res
+}
+
+define <8 x i64> @test_mask_conflict_q(<8 x i64> %a, <8 x i64> %b, i8 %mask) {
+; X86-LABEL: test_mask_conflict_q:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vpconflictq %zmm0, %zmm1 {%k1}
+; X86-NEXT:    vmovdqa64 %zmm1, %zmm0
+; X86-NEXT:    retl
+;
+; X64-LABEL: test_mask_conflict_q:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vpconflictq %zmm0, %zmm1 {%k1}
+; X64-NEXT:    vmovdqa64 %zmm1, %zmm0
+; X64-NEXT:    retq
+  %res = call <8 x i64> @llvm.x86.avx512.mask.conflict.q.512(<8 x i64> %a, <8 x i64> %b, i8 %mask)
+  ret <8 x i64> %res
+}
+
+define <8 x i64> @test_maskz_conflict_q(<8 x i64> %a, i8 %mask) {
+; X86-LABEL: test_maskz_conflict_q:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vpconflictq %zmm0, %zmm0 {%k1} {z}
+; X86-NEXT:    retl
+;
+; X64-LABEL: test_maskz_conflict_q:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vpconflictq %zmm0, %zmm0 {%k1} {z}
+; X64-NEXT:    retq
+  %res = call <8 x i64> @llvm.x86.avx512.mask.conflict.q.512(<8 x i64> %a, <8 x i64> zeroinitializer, i8 %mask)
+  ret <8 x i64> %res
+}
+
 define <16 x i32> @test_lzcnt_d(<16 x i32> %a) {
 ; CHECK-LABEL: test_lzcnt_d:
 ; CHECK:       # %bb.0:
diff --git a/test/CodeGen/X86/avx512cd-intrinsics.ll b/test/CodeGen/X86/avx512cd-intrinsics.ll
index fd18951747ab142d725a161bb90142e551b7bf45..ea248f47f8d1b487de0b7e5f9f6f8cd8969ccdd3 100644
--- a/test/CodeGen/X86/avx512cd-intrinsics.ll
+++ b/test/CodeGen/X86/avx512cd-intrinsics.ll
@@ -2,18 +2,34 @@
 ; RUN: llc < %s -mtriple=i686-unknown -mattr=+avx512cd | FileCheck %s --check-prefixes=CHECK,X86
 ; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx512cd | FileCheck %s --check-prefixes=CHECK,X64
 
-declare <16 x i32> @llvm.x86.avx512.mask.conflict.d.512(<16 x i32>, <16 x i32>, i16) nounwind readonly
-
-define <8 x i64> @test_conflict_q(<8 x i64> %a) {
-; CHECK-LABEL: test_conflict_q:
+define <16 x i32> @test_conflict_d(<16 x i32> %a) {
+; CHECK-LABEL: test_conflict_d:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vpconflictq %zmm0, %zmm0
+; CHECK-NEXT:    vpconflictd %zmm0, %zmm0
 ; CHECK-NEXT:    ret{{[l|q]}}
-  %res = call <8 x i64> @llvm.x86.avx512.mask.conflict.q.512(<8 x i64> %a, <8 x i64> zeroinitializer, i8 -1)
-  ret <8 x i64> %res
+  %1 = call <16 x i32> @llvm.x86.avx512.conflict.d.512(<16 x i32> %a)
+  ret <16 x i32> %1
 }
 
-declare <8 x i64> @llvm.x86.avx512.mask.conflict.q.512(<8 x i64>, <8 x i64>, i8) nounwind readonly
+define <16 x i32> @test_mask_conflict_d(<16 x i32> %a, <16 x i32> %b, i16 %mask) {
+; X86-LABEL: test_mask_conflict_d:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vpconflictd %zmm0, %zmm1 {%k1}
+; X86-NEXT:    vmovdqa64 %zmm1, %zmm0
+; X86-NEXT:    retl
+;
+; X64-LABEL: test_mask_conflict_d:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vpconflictd %zmm0, %zmm1 {%k1}
+; X64-NEXT:    vmovdqa64 %zmm1, %zmm0
+; X64-NEXT:    retq
+  %1 = call <16 x i32> @llvm.x86.avx512.conflict.d.512(<16 x i32> %a)
+  %2 = bitcast i16 %mask to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x i32> %1, <16 x i32> %b
+  ret <16 x i32> %3
+}
 
 define <16 x i32> @test_maskz_conflict_d(<16 x i32> %a, i16 %mask) {
 ; X86-LABEL: test_maskz_conflict_d:
@@ -27,8 +43,19 @@ define <16 x i32> @test_maskz_conflict_d(<16 x i32> %a, i16 %mask) {
 ; X64-NEXT:    kmovw %edi, %k1
 ; X64-NEXT:    vpconflictd %zmm0, %zmm0 {%k1} {z}
 ; X64-NEXT:    retq
-  %res = call <16 x i32> @llvm.x86.avx512.mask.conflict.d.512(<16 x i32> %a, <16 x i32> zeroinitializer, i16 %mask)
-  ret <16 x i32> %res
+  %1 = call <16 x i32> @llvm.x86.avx512.conflict.d.512(<16 x i32> %a)
+  %2 = bitcast i16 %mask to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x i32> %1, <16 x i32> zeroinitializer
+  ret <16 x i32> %3
+}
+
+define <8 x i64> @test_conflict_q(<8 x i64> %a) {
+; CHECK-LABEL: test_conflict_q:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpconflictq %zmm0, %zmm0
+; CHECK-NEXT:    ret{{[l|q]}}
+  %1 = call <8 x i64> @llvm.x86.avx512.conflict.q.512(<8 x i64> %a)
+  ret <8 x i64> %1
 }
 
 define <8 x i64> @test_mask_conflict_q(<8 x i64> %a, <8 x i64> %b, i8 %mask) {
@@ -46,8 +73,29 @@ define <8 x i64> @test_mask_conflict_q(<8 x i64> %a, <8 x i64> %b, i8 %mask) {
 ; X64-NEXT:    vpconflictq %zmm0, %zmm1 {%k1}
 ; X64-NEXT:    vmovdqa64 %zmm1, %zmm0
 ; X64-NEXT:    retq
-  %res = call <8 x i64> @llvm.x86.avx512.mask.conflict.q.512(<8 x i64> %a, <8 x i64> %b, i8 %mask)
-  ret <8 x i64> %res
+  %1 = call <8 x i64> @llvm.x86.avx512.conflict.q.512(<8 x i64> %a)
+  %2 = bitcast i8 %mask to <8 x i1>
+  %3 = select <8 x i1> %2, <8 x i64> %1, <8 x i64> %b
+  ret <8 x i64> %3
+}
+
+define <8 x i64> @test_maskz_conflict_q(<8 x i64> %a, i8 %mask) {
+; X86-LABEL: test_maskz_conflict_q:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vpconflictq %zmm0, %zmm0 {%k1} {z}
+; X86-NEXT:    retl
+;
+; X64-LABEL: test_maskz_conflict_q:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vpconflictq %zmm0, %zmm0 {%k1} {z}
+; X64-NEXT:    retq
+  %1 = call <8 x i64> @llvm.x86.avx512.conflict.q.512(<8 x i64> %a)
+  %2 = bitcast i8 %mask to <8 x i1>
+  %3 = select <8 x i1> %2, <8 x i64> %1, <8 x i64> zeroinitializer
+  ret <8 x i64> %3
 }
 
 define <16 x i32> @test_lzcnt_d(<16 x i32> %a) {
@@ -110,3 +158,6 @@ define <8 x i64> @test_mask_lzcnt_q(<8 x i64> %a, <8 x i64> %b, i8 %mask) {
   %3 = select <8 x i1> %2, <8 x i64> %1, <8 x i64> %b
   ret <8 x i64> %3
 }
+
+declare <16 x i32> @llvm.x86.avx512.conflict.d.512(<16 x i32>)
+declare <8 x i64> @llvm.x86.avx512.conflict.q.512(<8 x i64>)
diff --git a/test/CodeGen/X86/avx512cdvl-intrinsics-upgrade.ll b/test/CodeGen/X86/avx512cdvl-intrinsics-upgrade.ll
index e8c8fc700a93860e8493ef621121435f4c28063c..6f4fc11495f20b120063a6fa2de2c2469af01af1 100644
--- a/test/CodeGen/X86/avx512cdvl-intrinsics-upgrade.ll
+++ b/test/CodeGen/X86/avx512cdvl-intrinsics-upgrade.ll
@@ -180,3 +180,127 @@ define <2 x i64> @test_x86_broadcastmb_128(i8 %a0) {
 }
 declare <2 x i64> @llvm.x86.avx512.broadcastmb.128(i8)
 
+declare <4 x i32> @llvm.x86.avx512.mask.conflict.d.128(<4 x i32>, <4 x i32>, i8)
+
+define <4 x i32> @test_int_x86_avx512_mask_vpconflict_d_128(<4 x i32> %x0, <4 x i32> %x1, i8 %x2) {
+; X86-LABEL: test_int_x86_avx512_mask_vpconflict_d_128:
+; X86:       # %bb.0:
+; X86-NEXT:    vpconflictd %xmm0, %xmm2
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vpconflictd %xmm0, %xmm1 {%k1}
+; X86-NEXT:    vpconflictd %xmm0, %xmm0 {%k1} {z}
+; X86-NEXT:    vpaddd %xmm0, %xmm2, %xmm0
+; X86-NEXT:    vpaddd %xmm0, %xmm1, %xmm0
+; X86-NEXT:    retl
+;
+; X64-LABEL: test_int_x86_avx512_mask_vpconflict_d_128:
+; X64:       # %bb.0:
+; X64-NEXT:    vpconflictd %xmm0, %xmm2
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vpconflictd %xmm0, %xmm1 {%k1}
+; X64-NEXT:    vpconflictd %xmm0, %xmm0 {%k1} {z}
+; X64-NEXT:    vpaddd %xmm0, %xmm2, %xmm0
+; X64-NEXT:    vpaddd %xmm0, %xmm1, %xmm0
+; X64-NEXT:    retq
+  %res = call <4 x i32> @llvm.x86.avx512.mask.conflict.d.128(<4 x i32> %x0, <4 x i32> %x1, i8 %x2)
+  %res1 = call <4 x i32> @llvm.x86.avx512.mask.conflict.d.128(<4 x i32> %x0, <4 x i32> %x1, i8 -1)
+  %res3 = call <4 x i32> @llvm.x86.avx512.mask.conflict.d.128(<4 x i32> %x0, <4 x i32> zeroinitializer, i8 %x2)
+  %res2 = add <4 x i32> %res, %res1
+  %res4 = add <4 x i32> %res2, %res3
+  ret <4 x i32> %res4
+}
+
+declare <8 x i32> @llvm.x86.avx512.mask.conflict.d.256(<8 x i32>, <8 x i32>, i8)
+
+define <8 x i32> @test_int_x86_avx512_mask_vpconflict_d_256(<8 x i32> %x0, <8 x i32> %x1, i8 %x2) {
+; X86-LABEL: test_int_x86_avx512_mask_vpconflict_d_256:
+; X86:       # %bb.0:
+; X86-NEXT:    vpconflictd %ymm0, %ymm2
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vpconflictd %ymm0, %ymm1 {%k1}
+; X86-NEXT:    vpconflictd %ymm0, %ymm0 {%k1} {z}
+; X86-NEXT:    vpaddd %ymm0, %ymm2, %ymm0
+; X86-NEXT:    vpaddd %ymm0, %ymm1, %ymm0
+; X86-NEXT:    retl
+;
+; X64-LABEL: test_int_x86_avx512_mask_vpconflict_d_256:
+; X64:       # %bb.0:
+; X64-NEXT:    vpconflictd %ymm0, %ymm2
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vpconflictd %ymm0, %ymm1 {%k1}
+; X64-NEXT:    vpconflictd %ymm0, %ymm0 {%k1} {z}
+; X64-NEXT:    vpaddd %ymm0, %ymm2, %ymm0
+; X64-NEXT:    vpaddd %ymm0, %ymm1, %ymm0
+; X64-NEXT:    retq
+  %res = call <8 x i32> @llvm.x86.avx512.mask.conflict.d.256(<8 x i32> %x0, <8 x i32> %x1, i8 %x2)
+  %res1 = call <8 x i32> @llvm.x86.avx512.mask.conflict.d.256(<8 x i32> %x0, <8 x i32> %x1, i8 -1)
+  %res2 = call <8 x i32> @llvm.x86.avx512.mask.conflict.d.256(<8 x i32> %x0, <8 x i32> zeroinitializer, i8 %x2)
+  %res3 = add <8 x i32> %res, %res1
+  %res4 = add <8 x i32> %res2, %res3
+  ret <8 x i32> %res4
+}
+
+declare <2 x i64> @llvm.x86.avx512.mask.conflict.q.128(<2 x i64>, <2 x i64>, i8)
+
+define <2 x i64> @test_int_x86_avx512_mask_vpconflict_q_128(<2 x i64> %x0, <2 x i64> %x1, i8 %x2) {
+; X86-LABEL: test_int_x86_avx512_mask_vpconflict_q_128:
+; X86:       # %bb.0:
+; X86-NEXT:    vpconflictq %xmm0, %xmm2
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vpconflictq %xmm0, %xmm1 {%k1}
+; X86-NEXT:    vpconflictq %xmm0, %xmm0 {%k1} {z}
+; X86-NEXT:    vpaddq %xmm0, %xmm2, %xmm0
+; X86-NEXT:    vpaddq %xmm0, %xmm1, %xmm0
+; X86-NEXT:    retl
+;
+; X64-LABEL: test_int_x86_avx512_mask_vpconflict_q_128:
+; X64:       # %bb.0:
+; X64-NEXT:    vpconflictq %xmm0, %xmm2
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vpconflictq %xmm0, %xmm1 {%k1}
+; X64-NEXT:    vpconflictq %xmm0, %xmm0 {%k1} {z}
+; X64-NEXT:    vpaddq %xmm0, %xmm2, %xmm0
+; X64-NEXT:    vpaddq %xmm0, %xmm1, %xmm0
+; X64-NEXT:    retq
+  %res = call <2 x i64> @llvm.x86.avx512.mask.conflict.q.128(<2 x i64> %x0, <2 x i64> %x1, i8 %x2)
+  %res1 = call <2 x i64> @llvm.x86.avx512.mask.conflict.q.128(<2 x i64> %x0, <2 x i64> %x1, i8 -1)
+  %res2 = call <2 x i64> @llvm.x86.avx512.mask.conflict.q.128(<2 x i64> %x0, <2 x i64> zeroinitializer, i8 %x2)
+  %res3 = add <2 x i64> %res, %res1
+  %res4 = add <2 x i64> %res2, %res3
+  ret <2 x i64> %res4
+}
+
+declare <4 x i64> @llvm.x86.avx512.mask.conflict.q.256(<4 x i64>, <4 x i64>, i8)
+
+define <4 x i64> @test_int_x86_avx512_mask_vpconflict_q_256(<4 x i64> %x0, <4 x i64> %x1, i8 %x2) {
+; X86-LABEL: test_int_x86_avx512_mask_vpconflict_q_256:
+; X86:       # %bb.0:
+; X86-NEXT:    vpconflictq %ymm0, %ymm2
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw %eax, %k1
+; X86-NEXT:    vpconflictq %ymm0, %ymm1 {%k1}
+; X86-NEXT:    vpconflictq %ymm0, %ymm0 {%k1} {z}
+; X86-NEXT:    vpaddq %ymm0, %ymm2, %ymm0
+; X86-NEXT:    vpaddq %ymm0, %ymm1, %ymm0
+; X86-NEXT:    retl
+;
+; X64-LABEL: test_int_x86_avx512_mask_vpconflict_q_256:
+; X64:       # %bb.0:
+; X64-NEXT:    vpconflictq %ymm0, %ymm2
+; X64-NEXT:    kmovw %edi, %k1
+; X64-NEXT:    vpconflictq %ymm0, %ymm1 {%k1}
+; X64-NEXT:    vpconflictq %ymm0, %ymm0 {%k1} {z}
+; X64-NEXT:    vpaddq %ymm0, %ymm2, %ymm0
+; X64-NEXT:    vpaddq %ymm0, %ymm1, %ymm0
+; X64-NEXT:    retq
+  %res = call <4 x i64> @llvm.x86.avx512.mask.conflict.q.256(<4 x i64> %x0, <4 x i64> %x1, i8 %x2)
+  %res1 = call <4 x i64> @llvm.x86.avx512.mask.conflict.q.256(<4 x i64> %x0, <4 x i64> %x1, i8 -1)
+  %res2 = call <4 x i64> @llvm.x86.avx512.mask.conflict.q.256(<4 x i64> %x0, <4 x i64> zeroinitializer, i8 %x2)
+  %res3 = add <4 x i64> %res, %res1
+  %res4 = add <4 x i64> %res2, %res3
+  ret <4 x i64> %res4
+}
+
diff --git a/test/CodeGen/X86/avx512cdvl-intrinsics.ll b/test/CodeGen/X86/avx512cdvl-intrinsics.ll
index cd811e584eebae9d2dbb2c8609fa4be4f8048bc3..4d7e19c38a225853dd9867f213a18114564facff 100644
--- a/test/CodeGen/X86/avx512cdvl-intrinsics.ll
+++ b/test/CodeGen/X86/avx512cdvl-intrinsics.ll
@@ -118,109 +118,145 @@ define <4 x i64> @test_int_x86_avx512_mask_vplzcnt_q_256(<4 x i64> %x0, <4 x i64
 }
 declare <4 x i64> @llvm.ctlz.v4i64(<4 x i64>, i1) #0
 
-declare <4 x i32> @llvm.x86.avx512.mask.conflict.d.128(<4 x i32>, <4 x i32>, i8)
-
-define <4 x i32>@test_int_x86_avx512_mask_vpconflict_d_128(<4 x i32> %x0, <4 x i32> %x1, i8 %x2) {
+define <4 x i32> @test_int_x86_avx512_mask_vpconflict_d_128(<4 x i32> %x0, <4 x i32> %x1, i8 %x2) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpconflict_d_128:
 ; X86:       # %bb.0:
+; X86-NEXT:    vpconflictd %xmm0, %xmm2
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vpconflictd %xmm0, %xmm1 {%k1}
-; X86-NEXT:    vpconflictd %xmm0, %xmm2 {%k1} {z}
-; X86-NEXT:    vpconflictd %xmm0, %xmm0
-; X86-NEXT:    vpaddd %xmm2, %xmm0, %xmm0
+; X86-NEXT:    vpconflictd %xmm0, %xmm0 {%k1} {z}
+; X86-NEXT:    vpaddd %xmm0, %xmm2, %xmm0
 ; X86-NEXT:    vpaddd %xmm0, %xmm1, %xmm0
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_vpconflict_d_128:
 ; X64:       # %bb.0:
+; X64-NEXT:    vpconflictd %xmm0, %xmm2
 ; X64-NEXT:    kmovw %edi, %k1
-; X64-NEXT:    vpconflictd %xmm0, %xmm2 {%k1} {z}
 ; X64-NEXT:    vpconflictd %xmm0, %xmm1 {%k1}
-; X64-NEXT:    vpconflictd %xmm0, %xmm0
-; X64-NEXT:    vpaddd %xmm2, %xmm0, %xmm0
+; X64-NEXT:    vpconflictd %xmm0, %xmm0 {%k1} {z}
+; X64-NEXT:    vpaddd %xmm0, %xmm2, %xmm0
 ; X64-NEXT:    vpaddd %xmm0, %xmm1, %xmm0
 ; X64-NEXT:    retq
-  %res = call <4 x i32> @llvm.x86.avx512.mask.conflict.d.128(<4 x i32> %x0, <4 x i32> %x1, i8 %x2)
-  %res1 = call <4 x i32> @llvm.x86.avx512.mask.conflict.d.128(<4 x i32> %x0, <4 x i32> %x1, i8 -1)
-  %res3 = call <4 x i32> @llvm.x86.avx512.mask.conflict.d.128(<4 x i32> %x0, <4 x i32> zeroinitializer, i8 %x2)
-  %res2 = add <4 x i32> %res, %res1
-  %res4 = add <4 x i32> %res2, %res3
+  %1 = call <4 x i32> @llvm.x86.avx512.conflict.d.128(<4 x i32> %x0)
+  %2 = bitcast i8 %x2 to <8 x i1>
+  %extract1 = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %3 = select <4 x i1> %extract1, <4 x i32> %1, <4 x i32> %x1
+  %4 = call <4 x i32> @llvm.x86.avx512.conflict.d.128(<4 x i32> %x0)
+  %5 = call <4 x i32> @llvm.x86.avx512.conflict.d.128(<4 x i32> %x0)
+  %6 = bitcast i8 %x2 to <8 x i1>
+  %extract = shufflevector <8 x i1> %6, <8 x i1> %6, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %7 = select <4 x i1> %extract, <4 x i32> %5, <4 x i32> zeroinitializer
+  %res2 = add <4 x i32> %3, %4
+  %res4 = add <4 x i32> %res2, %7
   ret <4 x i32> %res4
 }
 
-declare <8 x i32> @llvm.x86.avx512.mask.conflict.d.256(<8 x i32>, <8 x i32>, i8)
-
-define <8 x i32>@test_int_x86_avx512_mask_vpconflict_d_256(<8 x i32> %x0, <8 x i32> %x1, i8 %x2) {
+define <8 x i32> @test_int_x86_avx512_mask_vpconflict_d_256(<8 x i32> %x0, <8 x i32> %x1, i8 %x2) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpconflict_d_256:
 ; X86:       # %bb.0:
+; X86-NEXT:    vpconflictd %ymm0, %ymm2
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vpconflictd %ymm0, %ymm1 {%k1}
-; X86-NEXT:    vpconflictd %ymm0, %ymm0
+; X86-NEXT:    vpconflictd %ymm0, %ymm0 {%k1} {z}
+; X86-NEXT:    vpaddd %ymm0, %ymm2, %ymm0
 ; X86-NEXT:    vpaddd %ymm0, %ymm1, %ymm0
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_vpconflict_d_256:
 ; X64:       # %bb.0:
+; X64-NEXT:    vpconflictd %ymm0, %ymm2
 ; X64-NEXT:    kmovw %edi, %k1
 ; X64-NEXT:    vpconflictd %ymm0, %ymm1 {%k1}
-; X64-NEXT:    vpconflictd %ymm0, %ymm0
+; X64-NEXT:    vpconflictd %ymm0, %ymm0 {%k1} {z}
+; X64-NEXT:    vpaddd %ymm0, %ymm2, %ymm0
 ; X64-NEXT:    vpaddd %ymm0, %ymm1, %ymm0
 ; X64-NEXT:    retq
-  %res = call <8 x i32> @llvm.x86.avx512.mask.conflict.d.256(<8 x i32> %x0, <8 x i32> %x1, i8 %x2)
-  %res1 = call <8 x i32> @llvm.x86.avx512.mask.conflict.d.256(<8 x i32> %x0, <8 x i32> %x1, i8 -1)
-  %res2 = add <8 x i32> %res, %res1
-  ret <8 x i32> %res2
+  %1 = call <8 x i32> @llvm.x86.avx512.conflict.d.256(<8 x i32> %x0)
+  %2 = bitcast i8 %x2 to <8 x i1>
+  %3 = select <8 x i1> %2, <8 x i32> %1, <8 x i32> %x1
+  %4 = call <8 x i32> @llvm.x86.avx512.conflict.d.256(<8 x i32> %x0)
+  %5 = call <8 x i32> @llvm.x86.avx512.conflict.d.256(<8 x i32> %x0)
+  %6 = bitcast i8 %x2 to <8 x i1>
+  %7 = select <8 x i1> %6, <8 x i32> %5, <8 x i32> zeroinitializer
+  %res3 = add <8 x i32> %3, %4
+  %res4 = add <8 x i32> %7, %res3
+  ret <8 x i32> %res4
 }
 
-declare <2 x i64> @llvm.x86.avx512.mask.conflict.q.128(<2 x i64>, <2 x i64>, i8)
-
-define <2 x i64>@test_int_x86_avx512_mask_vpconflict_q_128(<2 x i64> %x0, <2 x i64> %x1, i8 %x2) {
+define <2 x i64> @test_int_x86_avx512_mask_vpconflict_q_128(<2 x i64> %x0, <2 x i64> %x1, i8 %x2) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpconflict_q_128:
 ; X86:       # %bb.0:
+; X86-NEXT:    vpconflictq %xmm0, %xmm2
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vpconflictq %xmm0, %xmm1 {%k1}
-; X86-NEXT:    vpconflictq %xmm0, %xmm0
+; X86-NEXT:    vpconflictq %xmm0, %xmm0 {%k1} {z}
+; X86-NEXT:    vpaddq %xmm0, %xmm2, %xmm0
 ; X86-NEXT:    vpaddq %xmm0, %xmm1, %xmm0
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_vpconflict_q_128:
 ; X64:       # %bb.0:
+; X64-NEXT:    vpconflictq %xmm0, %xmm2
 ; X64-NEXT:    kmovw %edi, %k1
 ; X64-NEXT:    vpconflictq %xmm0, %xmm1 {%k1}
-; X64-NEXT:    vpconflictq %xmm0, %xmm0
+; X64-NEXT:    vpconflictq %xmm0, %xmm0 {%k1} {z}
+; X64-NEXT:    vpaddq %xmm0, %xmm2, %xmm0
 ; X64-NEXT:    vpaddq %xmm0, %xmm1, %xmm0
 ; X64-NEXT:    retq
-  %res = call <2 x i64> @llvm.x86.avx512.mask.conflict.q.128(<2 x i64> %x0, <2 x i64> %x1, i8 %x2)
-  %res1 = call <2 x i64> @llvm.x86.avx512.mask.conflict.q.128(<2 x i64> %x0, <2 x i64> %x1, i8 -1)
-  %res2 = add <2 x i64> %res, %res1
-  ret <2 x i64> %res2
+  %1 = call <2 x i64> @llvm.x86.avx512.conflict.q.128(<2 x i64> %x0)
+  %2 = bitcast i8 %x2 to <8 x i1>
+  %extract1 = shufflevector <8 x i1> %2, <8 x i1> %2, <2 x i32> <i32 0, i32 1>
+  %3 = select <2 x i1> %extract1, <2 x i64> %1, <2 x i64> %x1
+  %4 = call <2 x i64> @llvm.x86.avx512.conflict.q.128(<2 x i64> %x0)
+  %5 = call <2 x i64> @llvm.x86.avx512.conflict.q.128(<2 x i64> %x0)
+  %6 = bitcast i8 %x2 to <8 x i1>
+  %extract = shufflevector <8 x i1> %6, <8 x i1> %6, <2 x i32> <i32 0, i32 1>
+  %7 = select <2 x i1> %extract, <2 x i64> %5, <2 x i64> zeroinitializer
+  %res3 = add <2 x i64> %3, %4
+  %res4 = add <2 x i64> %7, %res3
+  ret <2 x i64> %res4
 }
 
-declare <4 x i64> @llvm.x86.avx512.mask.conflict.q.256(<4 x i64>, <4 x i64>, i8)
-
-define <4 x i64>@test_int_x86_avx512_mask_vpconflict_q_256(<4 x i64> %x0, <4 x i64> %x1, i8 %x2) {
+define <4 x i64> @test_int_x86_avx512_mask_vpconflict_q_256(<4 x i64> %x0, <4 x i64> %x1, i8 %x2) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpconflict_q_256:
 ; X86:       # %bb.0:
+; X86-NEXT:    vpconflictq %ymm0, %ymm2
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    kmovw %eax, %k1
 ; X86-NEXT:    vpconflictq %ymm0, %ymm1 {%k1}
-; X86-NEXT:    vpconflictq %ymm0, %ymm0
+; X86-NEXT:    vpconflictq %ymm0, %ymm0 {%k1} {z}
+; X86-NEXT:    vpaddq %ymm0, %ymm2, %ymm0
 ; X86-NEXT:    vpaddq %ymm0, %ymm1, %ymm0
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_vpconflict_q_256:
 ; X64:       # %bb.0:
+; X64-NEXT:    vpconflictq %ymm0, %ymm2
 ; X64-NEXT:    kmovw %edi, %k1
 ; X64-NEXT:    vpconflictq %ymm0, %ymm1 {%k1}
-; X64-NEXT:    vpconflictq %ymm0, %ymm0
+; X64-NEXT:    vpconflictq %ymm0, %ymm0 {%k1} {z}
+; X64-NEXT:    vpaddq %ymm0, %ymm2, %ymm0
 ; X64-NEXT:    vpaddq %ymm0, %ymm1, %ymm0
 ; X64-NEXT:    retq
-  %res = call <4 x i64> @llvm.x86.avx512.mask.conflict.q.256(<4 x i64> %x0, <4 x i64> %x1, i8 %x2)
-  %res1 = call <4 x i64> @llvm.x86.avx512.mask.conflict.q.256(<4 x i64> %x0, <4 x i64> %x1, i8 -1)
-  %res2 = add <4 x i64> %res, %res1
-  ret <4 x i64> %res2
+  %1 = call <4 x i64> @llvm.x86.avx512.conflict.q.256(<4 x i64> %x0)
+  %2 = bitcast i8 %x2 to <8 x i1>
+  %extract1 = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %3 = select <4 x i1> %extract1, <4 x i64> %1, <4 x i64> %x1
+  %4 = call <4 x i64> @llvm.x86.avx512.conflict.q.256(<4 x i64> %x0)
+  %5 = call <4 x i64> @llvm.x86.avx512.conflict.q.256(<4 x i64> %x0)
+  %6 = bitcast i8 %x2 to <8 x i1>
+  %extract = shufflevector <8 x i1> %6, <8 x i1> %6, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %7 = select <4 x i1> %extract, <4 x i64> %5, <4 x i64> zeroinitializer
+  %res3 = add <4 x i64> %3, %4
+  %res4 = add <4 x i64> %7, %res3
+  ret <4 x i64> %res4
 }
 
+declare <4 x i32> @llvm.x86.avx512.conflict.d.128(<4 x i32>)
+declare <8 x i32> @llvm.x86.avx512.conflict.d.256(<8 x i32>)
+declare <2 x i64> @llvm.x86.avx512.conflict.q.128(<2 x i64>)
+declare <4 x i64> @llvm.x86.avx512.conflict.q.256(<4 x i64>)
diff --git a/test/CodeGen/X86/avx512dq-intrinsics-upgrade.ll b/test/CodeGen/X86/avx512dq-intrinsics-upgrade.ll
index d1eb47a9cc74414dc31ca417b0b7bed7c1e013f7..4ddbfde7624dcd285d46be8ae5d2ea0635a70876 100644
--- a/test/CodeGen/X86/avx512dq-intrinsics-upgrade.ll
+++ b/test/CodeGen/X86/avx512dq-intrinsics-upgrade.ll
@@ -567,3 +567,99 @@ define i16@test_int_x86_avx512_mask_fpclass_ps_512(<16 x float> %x0) {
     %res1 = call i16 @llvm.x86.avx512.mask.fpclass.ps.512(<16 x float> %x0, i32 2, i16 %res)
     ret i16 %res1
 }
+
+declare <8 x double> @llvm.x86.avx512.mask.cvtqq2pd.512(<8 x i64>, <8 x double>, i8, i32)
+
+define <8 x double>@test_int_x86_avx512_mask_cvt_qq2pd_512(<8 x i64> %x0, <8 x double> %x1, i8 %x2) {
+; X86-LABEL: test_int_x86_avx512_mask_cvt_qq2pd_512:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovb {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf9,0x90,0x4c,0x24,0x04]
+; X86-NEXT:    vcvtqq2pd %zmm0, %zmm1 {%k1} # encoding: [0x62,0xf1,0xfe,0x49,0xe6,0xc8]
+; X86-NEXT:    vcvtqq2pd {rn-sae}, %zmm0, %zmm0 # encoding: [0x62,0xf1,0xfe,0x18,0xe6,0xc0]
+; X86-NEXT:    vaddpd %zmm0, %zmm1, %zmm0 # encoding: [0x62,0xf1,0xf5,0x48,0x58,0xc0]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_mask_cvt_qq2pd_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vcvtqq2pd %zmm0, %zmm1 {%k1} # encoding: [0x62,0xf1,0xfe,0x49,0xe6,0xc8]
+; X64-NEXT:    vcvtqq2pd {rn-sae}, %zmm0, %zmm0 # encoding: [0x62,0xf1,0xfe,0x18,0xe6,0xc0]
+; X64-NEXT:    vaddpd %zmm0, %zmm1, %zmm0 # encoding: [0x62,0xf1,0xf5,0x48,0x58,0xc0]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <8 x double> @llvm.x86.avx512.mask.cvtqq2pd.512(<8 x i64> %x0, <8 x double> %x1, i8 %x2, i32 4)
+  %res1 = call <8 x double> @llvm.x86.avx512.mask.cvtqq2pd.512(<8 x i64> %x0, <8 x double> %x1, i8 -1, i32 0)
+  %res2 = fadd <8 x double> %res, %res1
+  ret <8 x double> %res2
+}
+
+declare <8 x float> @llvm.x86.avx512.mask.cvtqq2ps.512(<8 x i64>, <8 x float>, i8, i32)
+
+define <8 x float> @test_int_x86_avx512_mask_cvt_qq2ps_512(<8 x i64> %x0, <8 x float> %x1, i8 %x2) {
+; X86-LABEL: test_int_x86_avx512_mask_cvt_qq2ps_512:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovb {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf9,0x90,0x4c,0x24,0x04]
+; X86-NEXT:    vcvtqq2ps %zmm0, %ymm1 {%k1} # encoding: [0x62,0xf1,0xfc,0x49,0x5b,0xc8]
+; X86-NEXT:    vcvtqq2ps {rn-sae}, %zmm0, %ymm0 # encoding: [0x62,0xf1,0xfc,0x18,0x5b,0xc0]
+; X86-NEXT:    vaddps %ymm0, %ymm1, %ymm0 # encoding: [0xc5,0xf4,0x58,0xc0]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_mask_cvt_qq2ps_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vcvtqq2ps %zmm0, %ymm1 {%k1} # encoding: [0x62,0xf1,0xfc,0x49,0x5b,0xc8]
+; X64-NEXT:    vcvtqq2ps {rn-sae}, %zmm0, %ymm0 # encoding: [0x62,0xf1,0xfc,0x18,0x5b,0xc0]
+; X64-NEXT:    vaddps %ymm0, %ymm1, %ymm0 # encoding: [0xc5,0xf4,0x58,0xc0]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <8 x float> @llvm.x86.avx512.mask.cvtqq2ps.512(<8 x i64> %x0, <8 x float> %x1, i8 %x2, i32 4)
+  %res1 = call <8 x float> @llvm.x86.avx512.mask.cvtqq2ps.512(<8 x i64> %x0, <8 x float> %x1, i8 -1, i32 0)
+  %res2 = fadd <8 x float> %res, %res1
+  ret <8 x float> %res2
+}
+
+declare <8 x double> @llvm.x86.avx512.mask.cvtuqq2pd.512(<8 x i64>, <8 x double>, i8, i32)
+
+define <8 x double> @test_int_x86_avx512_mask_cvt_uqq2pd_512(<8 x i64> %x0, <8 x double> %x1, i8 %x2) {
+; X86-LABEL: test_int_x86_avx512_mask_cvt_uqq2pd_512:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovb {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf9,0x90,0x4c,0x24,0x04]
+; X86-NEXT:    vcvtuqq2pd %zmm0, %zmm1 {%k1} # encoding: [0x62,0xf1,0xfe,0x49,0x7a,0xc8]
+; X86-NEXT:    vcvtuqq2pd {rn-sae}, %zmm0, %zmm0 # encoding: [0x62,0xf1,0xfe,0x18,0x7a,0xc0]
+; X86-NEXT:    vaddpd %zmm0, %zmm1, %zmm0 # encoding: [0x62,0xf1,0xf5,0x48,0x58,0xc0]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_mask_cvt_uqq2pd_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vcvtuqq2pd %zmm0, %zmm1 {%k1} # encoding: [0x62,0xf1,0xfe,0x49,0x7a,0xc8]
+; X64-NEXT:    vcvtuqq2pd {rn-sae}, %zmm0, %zmm0 # encoding: [0x62,0xf1,0xfe,0x18,0x7a,0xc0]
+; X64-NEXT:    vaddpd %zmm0, %zmm1, %zmm0 # encoding: [0x62,0xf1,0xf5,0x48,0x58,0xc0]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <8 x double> @llvm.x86.avx512.mask.cvtuqq2pd.512(<8 x i64> %x0, <8 x double> %x1, i8 %x2, i32 4)
+  %res1 = call <8 x double> @llvm.x86.avx512.mask.cvtuqq2pd.512(<8 x i64> %x0, <8 x double> %x1, i8 -1, i32 0)
+  %res2 = fadd <8 x double> %res, %res1
+  ret <8 x double> %res2
+}
+
+declare <8 x float> @llvm.x86.avx512.mask.cvtuqq2ps.512(<8 x i64>, <8 x float>, i8, i32)
+
+define <8 x float> @test_int_x86_avx512_mask_cvt_uqq2ps_512(<8 x i64> %x0, <8 x float> %x1, i8 %x2) {
+; X86-LABEL: test_int_x86_avx512_mask_cvt_uqq2ps_512:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovb {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf9,0x90,0x4c,0x24,0x04]
+; X86-NEXT:    vcvtuqq2ps %zmm0, %ymm1 {%k1} # encoding: [0x62,0xf1,0xff,0x49,0x7a,0xc8]
+; X86-NEXT:    vcvtuqq2ps {rn-sae}, %zmm0, %ymm0 # encoding: [0x62,0xf1,0xff,0x18,0x7a,0xc0]
+; X86-NEXT:    vaddps %ymm0, %ymm1, %ymm0 # encoding: [0xc5,0xf4,0x58,0xc0]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_mask_cvt_uqq2ps_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vcvtuqq2ps %zmm0, %ymm1 {%k1} # encoding: [0x62,0xf1,0xff,0x49,0x7a,0xc8]
+; X64-NEXT:    vcvtuqq2ps {rn-sae}, %zmm0, %ymm0 # encoding: [0x62,0xf1,0xff,0x18,0x7a,0xc0]
+; X64-NEXT:    vaddps %ymm0, %ymm1, %ymm0 # encoding: [0xc5,0xf4,0x58,0xc0]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <8 x float> @llvm.x86.avx512.mask.cvtuqq2ps.512(<8 x i64> %x0, <8 x float> %x1, i8 %x2, i32 4)
+  %res1 = call <8 x float> @llvm.x86.avx512.mask.cvtuqq2ps.512(<8 x i64> %x0, <8 x float> %x1, i8 -1, i32 0)
+  %res2 = fadd <8 x float> %res, %res1
+  ret <8 x float> %res2
+}
diff --git a/test/CodeGen/X86/avx512dq-intrinsics.ll b/test/CodeGen/X86/avx512dq-intrinsics.ll
index 0e18fa4ef26dcb16f9b5554915fe371e3215a74c..3d488795141ca77da84df35ee00135935cb333dc 100644
--- a/test/CodeGen/X86/avx512dq-intrinsics.ll
+++ b/test/CodeGen/X86/avx512dq-intrinsics.ll
@@ -212,7 +212,7 @@ define <8 x i64>@test_int_x86_avx512_mask_cvt_ps2uqq_512(<8 x float> %x0, <8 x i
   ret <8 x i64> %res2
 }
 
-declare <8 x double> @llvm.x86.avx512.mask.cvtqq2pd.512(<8 x i64>, <8 x double>, i8, i32)
+declare <8 x double> @llvm.x86.avx512.sitofp.round.v8f64.v8i64(<8 x i64>, i32)
 
 define <8 x double>@test_int_x86_avx512_mask_cvt_qq2pd_512(<8 x i64> %x0, <8 x double> %x1, i8 %x2) {
 ; X86-LABEL: test_int_x86_avx512_mask_cvt_qq2pd_512:
@@ -230,13 +230,15 @@ define <8 x double>@test_int_x86_avx512_mask_cvt_qq2pd_512(<8 x i64> %x0, <8 x d
 ; X64-NEXT:    vcvtqq2pd {rn-sae}, %zmm0, %zmm0 # encoding: [0x62,0xf1,0xfe,0x18,0xe6,0xc0]
 ; X64-NEXT:    vaddpd %zmm0, %zmm1, %zmm0 # encoding: [0x62,0xf1,0xf5,0x48,0x58,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <8 x double> @llvm.x86.avx512.mask.cvtqq2pd.512(<8 x i64> %x0, <8 x double> %x1, i8 %x2, i32 4)
-  %res1 = call <8 x double> @llvm.x86.avx512.mask.cvtqq2pd.512(<8 x i64> %x0, <8 x double> %x1, i8 -1, i32 0)
-  %res2 = fadd <8 x double> %res, %res1
+  %cvt = sitofp <8 x i64> %x0 to <8 x double>
+  %1 = bitcast i8 %x2 to <8 x i1>
+  %2 = select <8 x i1> %1, <8 x double> %cvt, <8 x double> %x1
+  %3 = call <8 x double> @llvm.x86.avx512.sitofp.round.v8f64.v8i64(<8 x i64> %x0, i32 0)
+  %res2 = fadd <8 x double> %2, %3
   ret <8 x double> %res2
 }
 
-declare <8 x float> @llvm.x86.avx512.mask.cvtqq2ps.512(<8 x i64>, <8 x float>, i8, i32)
+declare <8 x float> @llvm.x86.avx512.sitofp.round.v8f32.v8i64(<8 x i64>, i32)
 
 define <8 x float>@test_int_x86_avx512_mask_cvt_qq2ps_512(<8 x i64> %x0, <8 x float> %x1, i8 %x2) {
 ; X86-AVX512DQ-LABEL: test_int_x86_avx512_mask_cvt_qq2ps_512:
@@ -270,9 +272,11 @@ define <8 x float>@test_int_x86_avx512_mask_cvt_qq2ps_512(<8 x i64> %x0, <8 x fl
 ; X64-AVX512DQVL-NEXT:    vcvtqq2ps {rn-sae}, %zmm0, %ymm0 # encoding: [0x62,0xf1,0xfc,0x18,0x5b,0xc0]
 ; X64-AVX512DQVL-NEXT:    vaddps %ymm0, %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xf4,0x58,0xc0]
 ; X64-AVX512DQVL-NEXT:    retq # encoding: [0xc3]
-  %res = call <8 x float> @llvm.x86.avx512.mask.cvtqq2ps.512(<8 x i64> %x0, <8 x float> %x1, i8 %x2, i32 4)
-  %res1 = call <8 x float> @llvm.x86.avx512.mask.cvtqq2ps.512(<8 x i64> %x0, <8 x float> %x1, i8 -1, i32 0)
-  %res2 = fadd <8 x float> %res, %res1
+  %cvt = sitofp <8 x i64> %x0 to <8 x float>
+  %1 = bitcast i8 %x2 to <8 x i1>
+  %2 = select <8 x i1> %1, <8 x float> %cvt, <8 x float> %x1
+  %3 = call <8 x float> @llvm.x86.avx512.sitofp.round.v8f32.v8i64(<8 x i64> %x0, i32 0)
+  %res2 = fadd <8 x float> %2, %3
   ret <8 x float> %res2
 }
 
@@ -372,7 +376,7 @@ define <8 x i64>@test_int_x86_avx512_mask_cvtt_ps2uqq_512(<8 x float> %x0, <8 x
   ret <8 x i64> %res2
 }
 
-declare <8 x double> @llvm.x86.avx512.mask.cvtuqq2pd.512(<8 x i64>, <8 x double>, i8, i32)
+declare <8 x double> @llvm.x86.avx512.uitofp.round.v8f64.v8i64(<8 x i64>, i32)
 
 define <8 x double>@test_int_x86_avx512_mask_cvt_uqq2pd_512(<8 x i64> %x0, <8 x double> %x1, i8 %x2) {
 ; X86-LABEL: test_int_x86_avx512_mask_cvt_uqq2pd_512:
@@ -390,13 +394,15 @@ define <8 x double>@test_int_x86_avx512_mask_cvt_uqq2pd_512(<8 x i64> %x0, <8 x
 ; X64-NEXT:    vcvtuqq2pd {rn-sae}, %zmm0, %zmm0 # encoding: [0x62,0xf1,0xfe,0x18,0x7a,0xc0]
 ; X64-NEXT:    vaddpd %zmm0, %zmm1, %zmm0 # encoding: [0x62,0xf1,0xf5,0x48,0x58,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <8 x double> @llvm.x86.avx512.mask.cvtuqq2pd.512(<8 x i64> %x0, <8 x double> %x1, i8 %x2, i32 4)
-  %res1 = call <8 x double> @llvm.x86.avx512.mask.cvtuqq2pd.512(<8 x i64> %x0, <8 x double> %x1, i8 -1, i32 0)
-  %res2 = fadd <8 x double> %res, %res1
+  %cvt = uitofp <8 x i64> %x0 to <8 x double>
+  %1 = bitcast i8 %x2 to <8 x i1>
+  %2 = select <8 x i1> %1, <8 x double> %cvt, <8 x double> %x1
+  %3 = call <8 x double> @llvm.x86.avx512.uitofp.round.v8f64.v8i64(<8 x i64> %x0, i32 0)
+  %res2 = fadd <8 x double> %2, %3
   ret <8 x double> %res2
 }
 
-declare <8 x float> @llvm.x86.avx512.mask.cvtuqq2ps.512(<8 x i64>, <8 x float>, i8, i32)
+declare <8 x float> @llvm.x86.avx512.uitofp.round.v8f32.v8i64(<8 x i64>, i32)
 
 define <8 x float>@test_int_x86_avx512_mask_cvt_uqq2ps_512(<8 x i64> %x0, <8 x float> %x1, i8 %x2) {
 ; X86-AVX512DQ-LABEL: test_int_x86_avx512_mask_cvt_uqq2ps_512:
@@ -430,9 +436,11 @@ define <8 x float>@test_int_x86_avx512_mask_cvt_uqq2ps_512(<8 x i64> %x0, <8 x f
 ; X64-AVX512DQVL-NEXT:    vcvtuqq2ps {rn-sae}, %zmm0, %ymm0 # encoding: [0x62,0xf1,0xff,0x18,0x7a,0xc0]
 ; X64-AVX512DQVL-NEXT:    vaddps %ymm0, %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xf4,0x58,0xc0]
 ; X64-AVX512DQVL-NEXT:    retq # encoding: [0xc3]
-  %res = call <8 x float> @llvm.x86.avx512.mask.cvtuqq2ps.512(<8 x i64> %x0, <8 x float> %x1, i8 %x2, i32 4)
-  %res1 = call <8 x float> @llvm.x86.avx512.mask.cvtuqq2ps.512(<8 x i64> %x0, <8 x float> %x1, i8 -1, i32 0)
-  %res2 = fadd <8 x float> %res, %res1
+  %cvt = uitofp <8 x i64> %x0 to <8 x float>
+  %1 = bitcast i8 %x2 to <8 x i1>
+  %2 = select <8 x i1> %1, <8 x float> %cvt, <8 x float> %x1
+  %3 = call <8 x float> @llvm.x86.avx512.uitofp.round.v8f32.v8i64(<8 x i64> %x0, i32 0)
+  %res2 = fadd <8 x float> %2, %3
   ret <8 x float> %res2
 }
 
diff --git a/test/CodeGen/X86/avx512dqvl-intrinsics-upgrade.ll b/test/CodeGen/X86/avx512dqvl-intrinsics-upgrade.ll
index a7a44253c8067594c7fbe3ef903384889f9f88e1..3bb86b1b67a1ed44aab8c23cecfae4281ebfed8c 100644
--- a/test/CodeGen/X86/avx512dqvl-intrinsics-upgrade.ll
+++ b/test/CodeGen/X86/avx512dqvl-intrinsics-upgrade.ll
@@ -2990,3 +2990,55 @@ define i8 @test_int_x86_avx512_mask_fpclass_pd_256(<4 x double> %x0) {
   %res1 = call i8 @llvm.x86.avx512.mask.fpclass.pd.256(<4 x double> %x0, i32 4, i8 %res)
   ret i8 %res1
 }
+
+declare <4 x float> @llvm.x86.avx512.mask.cvtqq2ps.256(<4 x i64>, <4 x float>, i8)
+
+define <4 x float> @test_int_x86_avx512_mask_cvt_qq2ps_256(<4 x i64> %x0, <4 x float> %x1, i8 %x2) {
+; X86-LABEL: test_int_x86_avx512_mask_cvt_qq2ps_256:
+; X86:       # %bb.0:
+; X86-NEXT:    vcvtqq2ps %ymm0, %xmm2 # encoding: [0x62,0xf1,0xfc,0x28,0x5b,0xd0]
+; X86-NEXT:    kmovb {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf9,0x90,0x4c,0x24,0x04]
+; X86-NEXT:    vcvtqq2ps %ymm0, %xmm1 {%k1} # encoding: [0x62,0xf1,0xfc,0x29,0x5b,0xc8]
+; X86-NEXT:    vaddps %xmm2, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf0,0x58,0xc2]
+; X86-NEXT:    vzeroupper # encoding: [0xc5,0xf8,0x77]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_mask_cvt_qq2ps_256:
+; X64:       # %bb.0:
+; X64-NEXT:    vcvtqq2ps %ymm0, %xmm2 # encoding: [0x62,0xf1,0xfc,0x28,0x5b,0xd0]
+; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vcvtqq2ps %ymm0, %xmm1 {%k1} # encoding: [0x62,0xf1,0xfc,0x29,0x5b,0xc8]
+; X64-NEXT:    vaddps %xmm2, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf0,0x58,0xc2]
+; X64-NEXT:    vzeroupper # encoding: [0xc5,0xf8,0x77]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <4 x float> @llvm.x86.avx512.mask.cvtqq2ps.256(<4 x i64> %x0, <4 x float> %x1, i8 %x2)
+  %res1 = call <4 x float> @llvm.x86.avx512.mask.cvtqq2ps.256(<4 x i64> %x0, <4 x float> %x1, i8 -1)
+  %res2 = fadd <4 x float> %res, %res1
+  ret <4 x float> %res2
+}
+
+declare <4 x float> @llvm.x86.avx512.mask.cvtuqq2ps.256(<4 x i64>, <4 x float>, i8)
+
+define <4 x float> @test_int_x86_avx512_mask_cvt_uqq2ps_256(<4 x i64> %x0, <4 x float> %x1, i8 %x2) {
+; X86-LABEL: test_int_x86_avx512_mask_cvt_uqq2ps_256:
+; X86:       # %bb.0:
+; X86-NEXT:    vcvtuqq2ps %ymm0, %xmm2 # encoding: [0x62,0xf1,0xff,0x28,0x7a,0xd0]
+; X86-NEXT:    kmovb {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf9,0x90,0x4c,0x24,0x04]
+; X86-NEXT:    vcvtuqq2ps %ymm0, %xmm1 {%k1} # encoding: [0x62,0xf1,0xff,0x29,0x7a,0xc8]
+; X86-NEXT:    vaddps %xmm2, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf0,0x58,0xc2]
+; X86-NEXT:    vzeroupper # encoding: [0xc5,0xf8,0x77]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_mask_cvt_uqq2ps_256:
+; X64:       # %bb.0:
+; X64-NEXT:    vcvtuqq2ps %ymm0, %xmm2 # encoding: [0x62,0xf1,0xff,0x28,0x7a,0xd0]
+; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vcvtuqq2ps %ymm0, %xmm1 {%k1} # encoding: [0x62,0xf1,0xff,0x29,0x7a,0xc8]
+; X64-NEXT:    vaddps %xmm2, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf0,0x58,0xc2]
+; X64-NEXT:    vzeroupper # encoding: [0xc5,0xf8,0x77]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <4 x float> @llvm.x86.avx512.mask.cvtuqq2ps.256(<4 x i64> %x0, <4 x float> %x1, i8 %x2)
+  %res1 = call <4 x float> @llvm.x86.avx512.mask.cvtuqq2ps.256(<4 x i64> %x0, <4 x float> %x1, i8 -1)
+  %res2 = fadd <4 x float> %res, %res1
+  ret <4 x float> %res2
+}
diff --git a/test/CodeGen/X86/avx512dqvl-intrinsics.ll b/test/CodeGen/X86/avx512dqvl-intrinsics.ll
index 6ba69e7be951c34d4cd4ffa523a147ef800eb130..5925fb13aed3a0ca1549e78f0c87f9537b1749e1 100644
--- a/test/CodeGen/X86/avx512dqvl-intrinsics.ll
+++ b/test/CodeGen/X86/avx512dqvl-intrinsics.ll
@@ -223,8 +223,6 @@ define <4 x float>@test_int_x86_avx512_mask_cvt_qq2ps_128_zext(<2 x i64> %x0, <4
 ; X86:       # %bb.0:
 ; X86-NEXT:    kmovb {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf9,0x90,0x4c,0x24,0x04]
 ; X86-NEXT:    vcvtqq2ps %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf1,0xfc,0x09,0x5b,0xc8]
-; X86-NEXT:    vmovq %xmm1, %xmm1 # EVEX TO VEX Compression encoding: [0xc5,0xfa,0x7e,0xc9]
-; X86-NEXT:    # xmm1 = xmm1[0],zero
 ; X86-NEXT:    vcvtqq2ps %xmm0, %xmm0 # encoding: [0x62,0xf1,0xfc,0x08,0x5b,0xc0]
 ; X86-NEXT:    vaddps %xmm0, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf0,0x58,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
@@ -233,8 +231,6 @@ define <4 x float>@test_int_x86_avx512_mask_cvt_qq2ps_128_zext(<2 x i64> %x0, <4
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vcvtqq2ps %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf1,0xfc,0x09,0x5b,0xc8]
-; X64-NEXT:    vmovq %xmm1, %xmm1 # EVEX TO VEX Compression encoding: [0xc5,0xfa,0x7e,0xc9]
-; X64-NEXT:    # xmm1 = xmm1[0],zero
 ; X64-NEXT:    vcvtqq2ps %xmm0, %xmm0 # encoding: [0x62,0xf1,0xfc,0x08,0x5b,0xc0]
 ; X64-NEXT:    vaddps %xmm0, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf0,0x58,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
@@ -246,29 +242,30 @@ define <4 x float>@test_int_x86_avx512_mask_cvt_qq2ps_128_zext(<2 x i64> %x0, <4
   ret <4 x float> %res4
 }
 
-declare <4 x float> @llvm.x86.avx512.mask.cvtqq2ps.256(<4 x i64>, <4 x float>, i8)
-
 define <4 x float>@test_int_x86_avx512_mask_cvt_qq2ps_256(<4 x i64> %x0, <4 x float> %x1, i8 %x2) {
 ; X86-LABEL: test_int_x86_avx512_mask_cvt_qq2ps_256:
 ; X86:       # %bb.0:
+; X86-NEXT:    vcvtqq2ps %ymm0, %xmm2 # encoding: [0x62,0xf1,0xfc,0x28,0x5b,0xd0]
 ; X86-NEXT:    kmovb {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf9,0x90,0x4c,0x24,0x04]
 ; X86-NEXT:    vcvtqq2ps %ymm0, %xmm1 {%k1} # encoding: [0x62,0xf1,0xfc,0x29,0x5b,0xc8]
-; X86-NEXT:    vcvtqq2ps %ymm0, %xmm0 # encoding: [0x62,0xf1,0xfc,0x28,0x5b,0xc0]
-; X86-NEXT:    vaddps %xmm0, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf0,0x58,0xc0]
+; X86-NEXT:    vaddps %xmm2, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf0,0x58,0xc2]
 ; X86-NEXT:    vzeroupper # encoding: [0xc5,0xf8,0x77]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_cvt_qq2ps_256:
 ; X64:       # %bb.0:
+; X64-NEXT:    vcvtqq2ps %ymm0, %xmm2 # encoding: [0x62,0xf1,0xfc,0x28,0x5b,0xd0]
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vcvtqq2ps %ymm0, %xmm1 {%k1} # encoding: [0x62,0xf1,0xfc,0x29,0x5b,0xc8]
-; X64-NEXT:    vcvtqq2ps %ymm0, %xmm0 # encoding: [0x62,0xf1,0xfc,0x28,0x5b,0xc0]
-; X64-NEXT:    vaddps %xmm0, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf0,0x58,0xc0]
+; X64-NEXT:    vaddps %xmm2, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf0,0x58,0xc2]
 ; X64-NEXT:    vzeroupper # encoding: [0xc5,0xf8,0x77]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <4 x float> @llvm.x86.avx512.mask.cvtqq2ps.256(<4 x i64> %x0, <4 x float> %x1, i8 %x2)
-  %res1 = call <4 x float> @llvm.x86.avx512.mask.cvtqq2ps.256(<4 x i64> %x0, <4 x float> %x1, i8 -1)
-  %res2 = fadd <4 x float> %res, %res1
+  %cvt1 = sitofp <4 x i64> %x0 to <4 x float>
+  %1 = bitcast i8 %x2 to <8 x i1>
+  %extract = shufflevector <8 x i1> %1, <8 x i1> %1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %2 = select <4 x i1> %extract, <4 x float> %cvt1, <4 x float> %x1
+  %cvt = sitofp <4 x i64> %x0 to <4 x float>
+  %res2 = fadd <4 x float> %2, %cvt
   ret <4 x float> %res2
 }
 
@@ -445,8 +442,6 @@ define <4 x float>@test_int_x86_avx512_mask_cvt_uqq2ps_128_zext(<2 x i64> %x0, <
 ; X86:       # %bb.0:
 ; X86-NEXT:    kmovb {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf9,0x90,0x4c,0x24,0x04]
 ; X86-NEXT:    vcvtuqq2ps %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf1,0xff,0x09,0x7a,0xc8]
-; X86-NEXT:    vmovq %xmm1, %xmm1 # EVEX TO VEX Compression encoding: [0xc5,0xfa,0x7e,0xc9]
-; X86-NEXT:    # xmm1 = xmm1[0],zero
 ; X86-NEXT:    vcvtuqq2ps %xmm0, %xmm0 # encoding: [0x62,0xf1,0xff,0x08,0x7a,0xc0]
 ; X86-NEXT:    vaddps %xmm0, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf0,0x58,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
@@ -455,8 +450,6 @@ define <4 x float>@test_int_x86_avx512_mask_cvt_uqq2ps_128_zext(<2 x i64> %x0, <
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vcvtuqq2ps %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf1,0xff,0x09,0x7a,0xc8]
-; X64-NEXT:    vmovq %xmm1, %xmm1 # EVEX TO VEX Compression encoding: [0xc5,0xfa,0x7e,0xc9]
-; X64-NEXT:    # xmm1 = xmm1[0],zero
 ; X64-NEXT:    vcvtuqq2ps %xmm0, %xmm0 # encoding: [0x62,0xf1,0xff,0x08,0x7a,0xc0]
 ; X64-NEXT:    vaddps %xmm0, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf0,0x58,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
@@ -473,24 +466,27 @@ declare <4 x float> @llvm.x86.avx512.mask.cvtuqq2ps.256(<4 x i64>, <4 x float>,
 define <4 x float>@test_int_x86_avx512_mask_cvt_uqq2ps_256(<4 x i64> %x0, <4 x float> %x1, i8 %x2) {
 ; X86-LABEL: test_int_x86_avx512_mask_cvt_uqq2ps_256:
 ; X86:       # %bb.0:
+; X86-NEXT:    vcvtuqq2ps %ymm0, %xmm2 # encoding: [0x62,0xf1,0xff,0x28,0x7a,0xd0]
 ; X86-NEXT:    kmovb {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf9,0x90,0x4c,0x24,0x04]
 ; X86-NEXT:    vcvtuqq2ps %ymm0, %xmm1 {%k1} # encoding: [0x62,0xf1,0xff,0x29,0x7a,0xc8]
-; X86-NEXT:    vcvtuqq2ps %ymm0, %xmm0 # encoding: [0x62,0xf1,0xff,0x28,0x7a,0xc0]
-; X86-NEXT:    vaddps %xmm0, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf0,0x58,0xc0]
+; X86-NEXT:    vaddps %xmm2, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf0,0x58,0xc2]
 ; X86-NEXT:    vzeroupper # encoding: [0xc5,0xf8,0x77]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_cvt_uqq2ps_256:
 ; X64:       # %bb.0:
+; X64-NEXT:    vcvtuqq2ps %ymm0, %xmm2 # encoding: [0x62,0xf1,0xff,0x28,0x7a,0xd0]
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vcvtuqq2ps %ymm0, %xmm1 {%k1} # encoding: [0x62,0xf1,0xff,0x29,0x7a,0xc8]
-; X64-NEXT:    vcvtuqq2ps %ymm0, %xmm0 # encoding: [0x62,0xf1,0xff,0x28,0x7a,0xc0]
-; X64-NEXT:    vaddps %xmm0, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf0,0x58,0xc0]
+; X64-NEXT:    vaddps %xmm2, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf0,0x58,0xc2]
 ; X64-NEXT:    vzeroupper # encoding: [0xc5,0xf8,0x77]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <4 x float> @llvm.x86.avx512.mask.cvtuqq2ps.256(<4 x i64> %x0, <4 x float> %x1, i8 %x2)
-  %res1 = call <4 x float> @llvm.x86.avx512.mask.cvtuqq2ps.256(<4 x i64> %x0, <4 x float> %x1, i8 -1)
-  %res2 = fadd <4 x float> %res, %res1
+  %cvt1 = uitofp <4 x i64> %x0 to <4 x float>
+  %1 = bitcast i8 %x2 to <8 x i1>
+  %extract = shufflevector <8 x i1> %1, <8 x i1> %1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %2 = select <4 x i1> %extract, <4 x float> %cvt1, <4 x float> %x1
+  %cvt = uitofp <4 x i64> %x0 to <4 x float>
+  %res2 = fadd <4 x float> %2, %cvt
   ret <4 x float> %res2
 }
 
diff --git a/test/CodeGen/X86/avx512vbmi-intrinsics-fast-isel.ll b/test/CodeGen/X86/avx512vbmi-intrinsics-fast-isel.ll
index 495f612ff42561e2715985e0e5eebb8b730110c1..8eb9a7c3866b5ac88fa9a655aee91246e4a1ad6b 100644
--- a/test/CodeGen/X86/avx512vbmi-intrinsics-fast-isel.ll
+++ b/test/CodeGen/X86/avx512vbmi-intrinsics-fast-isel.ll
@@ -96,3 +96,67 @@ entry:
 }
 
 declare <64 x i8> @llvm.x86.avx512.vpermi2var.qi.512(<64 x i8>, <64 x i8>, <64 x i8>)
+
+define <8 x i64> @test_mm512_mask_multishift_epi64_epi8(<8 x i64> %__W, i64 %__M, <8 x i64> %__X, <8 x i64> %__Y) {
+; X86-LABEL: test_mm512_mask_multishift_epi64_epi8:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k0
+; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    kunpckdq %k1, %k0, %k1
+; X86-NEXT:    vpmultishiftqb %zmm2, %zmm1, %zmm0 {%k1}
+; X86-NEXT:    retl
+;
+; X64-LABEL: test_mm512_mask_multishift_epi64_epi8:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    kmovq %rdi, %k1
+; X64-NEXT:    vpmultishiftqb %zmm2, %zmm1, %zmm0 {%k1}
+; X64-NEXT:    retq
+entry:
+  %0 = bitcast <8 x i64> %__X to <64 x i8>
+  %1 = bitcast <8 x i64> %__Y to <64 x i8>
+  %2 = tail call <64 x i8> @llvm.x86.avx512.pmultishift.qb.512(<64 x i8> %0, <64 x i8> %1)
+  %3 = bitcast <8 x i64> %__W to <64 x i8>
+  %4 = bitcast i64 %__M to <64 x i1>
+  %5 = select <64 x i1> %4, <64 x i8> %2, <64 x i8> %3
+  %6 = bitcast <64 x i8> %5 to <8 x i64>
+  ret <8 x i64> %6
+}
+
+define <8 x i64> @test_mm512_maskz_multishift_epi64_epi8(i64 %__M, <8 x i64> %__X, <8 x i64> %__Y) {
+; X86-LABEL: test_mm512_maskz_multishift_epi64_epi8:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k0
+; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    kunpckdq %k1, %k0, %k1
+; X86-NEXT:    vpmultishiftqb %zmm1, %zmm0, %zmm0 {%k1} {z}
+; X86-NEXT:    retl
+;
+; X64-LABEL: test_mm512_maskz_multishift_epi64_epi8:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    kmovq %rdi, %k1
+; X64-NEXT:    vpmultishiftqb %zmm1, %zmm0, %zmm0 {%k1} {z}
+; X64-NEXT:    retq
+entry:
+  %0 = bitcast <8 x i64> %__X to <64 x i8>
+  %1 = bitcast <8 x i64> %__Y to <64 x i8>
+  %2 = tail call <64 x i8> @llvm.x86.avx512.pmultishift.qb.512(<64 x i8> %0, <64 x i8> %1)
+  %3 = bitcast i64 %__M to <64 x i1>
+  %4 = select <64 x i1> %3, <64 x i8> %2, <64 x i8> zeroinitializer
+  %5 = bitcast <64 x i8> %4 to <8 x i64>
+  ret <8 x i64> %5
+}
+
+define <8 x i64> @test_mm512_multishift_epi64_epi8(<8 x i64> %__X, <8 x i64> %__Y) {
+; CHECK-LABEL: test_mm512_multishift_epi64_epi8:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vpmultishiftqb %zmm1, %zmm0, %zmm0
+; CHECK-NEXT:    ret{{[l|q]}}
+entry:
+  %0 = bitcast <8 x i64> %__X to <64 x i8>
+  %1 = bitcast <8 x i64> %__Y to <64 x i8>
+  %2 = tail call <64 x i8> @llvm.x86.avx512.pmultishift.qb.512(<64 x i8> %0, <64 x i8> %1)
+  %3 = bitcast <64 x i8> %2 to <8 x i64>
+  ret <8 x i64> %3
+}
+
+declare <64 x i8> @llvm.x86.avx512.pmultishift.qb.512(<64 x i8>, <64 x i8>)
diff --git a/test/CodeGen/X86/avx512vbmi-intrinsics-upgrade.ll b/test/CodeGen/X86/avx512vbmi-intrinsics-upgrade.ll
index a09cf095aef70b1edcb727f00784541472e329ce..8c6b982fe7475a4ce960dcff8f39a5ba4c5ce213 100644
--- a/test/CodeGen/X86/avx512vbmi-intrinsics-upgrade.ll
+++ b/test/CodeGen/X86/avx512vbmi-intrinsics-upgrade.ll
@@ -32,6 +32,36 @@ define <64 x i8>@test_int_x86_avx512_mask_permvar_qi_512(<64 x i8> %x0, <64 x i8
  ret <64 x i8> %res4
 }
 
+declare <64 x i8> @llvm.x86.avx512.mask.pmultishift.qb.512(<64 x i8>, <64 x i8>, <64 x i8>, i64)
+
+define <64 x i8>@test_int_x86_avx512_mask_pmultishift_qb_512(<64 x i8> %x0, <64 x i8> %x1, <64 x i8> %x2, i64 %x3) {
+; X86-LABEL: test_int_x86_avx512_mask_pmultishift_qb_512:
+; X86:       # %bb.0:
+; X86-NEXT:    vpmultishiftqb %zmm1, %zmm0, %zmm3 # encoding: [0x62,0xf2,0xfd,0x48,0x83,0xd9]
+; X86-NEXT:    kmovq {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf8,0x90,0x4c,0x24,0x04]
+; X86-NEXT:    vpmultishiftqb %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x49,0x83,0xd1]
+; X86-NEXT:    vpmultishiftqb %zmm1, %zmm0, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xc9,0x83,0xc1]
+; X86-NEXT:    vpaddb %zmm3, %zmm0, %zmm0 # encoding: [0x62,0xf1,0x7d,0x48,0xfc,0xc3]
+; X86-NEXT:    vpaddb %zmm0, %zmm2, %zmm0 # encoding: [0x62,0xf1,0x6d,0x48,0xfc,0xc0]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_mask_pmultishift_qb_512:
+; X64:       # %bb.0:
+; X64-NEXT:    vpmultishiftqb %zmm1, %zmm0, %zmm3 # encoding: [0x62,0xf2,0xfd,0x48,0x83,0xd9]
+; X64-NEXT:    kmovq %rdi, %k1 # encoding: [0xc4,0xe1,0xfb,0x92,0xcf]
+; X64-NEXT:    vpmultishiftqb %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x49,0x83,0xd1]
+; X64-NEXT:    vpmultishiftqb %zmm1, %zmm0, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xc9,0x83,0xc1]
+; X64-NEXT:    vpaddb %zmm3, %zmm0, %zmm0 # encoding: [0x62,0xf1,0x7d,0x48,0xfc,0xc3]
+; X64-NEXT:    vpaddb %zmm0, %zmm2, %zmm0 # encoding: [0x62,0xf1,0x6d,0x48,0xfc,0xc0]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <64 x i8> @llvm.x86.avx512.mask.pmultishift.qb.512(<64 x i8> %x0, <64 x i8> %x1, <64 x i8> %x2, i64 %x3)
+  %res1 = call <64 x i8> @llvm.x86.avx512.mask.pmultishift.qb.512(<64 x i8> %x0, <64 x i8> %x1, <64 x i8> zeroinitializer, i64 %x3)
+  %res2 = call <64 x i8> @llvm.x86.avx512.mask.pmultishift.qb.512(<64 x i8> %x0, <64 x i8> %x1, <64 x i8> %x2, i64 -1)
+  %res3 = add <64 x i8> %res, %res1
+  %res4 = add <64 x i8> %res3, %res2
+  ret <64 x i8> %res4
+}
+
 declare <64 x i8> @llvm.x86.avx512.mask.vpermi2var.qi.512(<64 x i8>, <64 x i8>, <64 x i8>, i64)
 
 define <64 x i8>@test_int_x86_avx512_mask_vpermi2var_qi_512(<64 x i8> %x0, <64 x i8> %x1, <64 x i8> %x2, i64 %x3) {
diff --git a/test/CodeGen/X86/avx512vbmi-intrinsics.ll b/test/CodeGen/X86/avx512vbmi-intrinsics.ll
index ffce664b8d6d8822b795de6db595232318c2d071..23a7e2ac3e96597486f6a8e7abccff726d62de34 100644
--- a/test/CodeGen/X86/avx512vbmi-intrinsics.ll
+++ b/test/CodeGen/X86/avx512vbmi-intrinsics.ll
@@ -36,15 +36,13 @@ define <64 x i8>@test_int_x86_avx512_mask_permvar_qi_512(<64 x i8> %x0, <64 x i8
   ret <64 x i8> %res4
 }
 
-declare <64 x i8> @llvm.x86.avx512.mask.pmultishift.qb.512(<64 x i8>, <64 x i8>, <64 x i8>, i64)
+declare <64 x i8> @llvm.x86.avx512.pmultishift.qb.512(<64 x i8>, <64 x i8>)
 
 define <64 x i8>@test_int_x86_avx512_mask_pmultishift_qb_512(<64 x i8> %x0, <64 x i8> %x1, <64 x i8> %x2, i64 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_pmultishift_qb_512:
 ; X86:       # %bb.0:
 ; X86-NEXT:    vpmultishiftqb %zmm1, %zmm0, %zmm3 # encoding: [0x62,0xf2,0xfd,0x48,0x83,0xd9]
-; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k0 # encoding: [0xc4,0xe1,0xf9,0x90,0x44,0x24,0x04]
-; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf9,0x90,0x4c,0x24,0x08]
-; X86-NEXT:    kunpckdq %k0, %k1, %k1 # encoding: [0xc4,0xe1,0xf4,0x4b,0xc8]
+; X86-NEXT:    kmovq {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf8,0x90,0x4c,0x24,0x04]
 ; X86-NEXT:    vpmultishiftqb %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x49,0x83,0xd1]
 ; X86-NEXT:    vpmultishiftqb %zmm1, %zmm0, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xc9,0x83,0xc1]
 ; X86-NEXT:    vpaddb %zmm3, %zmm0, %zmm0 # encoding: [0x62,0xf1,0x7d,0x48,0xfc,0xc3]
@@ -53,18 +51,22 @@ define <64 x i8>@test_int_x86_avx512_mask_pmultishift_qb_512(<64 x i8> %x0, <64
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_pmultishift_qb_512:
 ; X64:       # %bb.0:
+; X64-NEXT:    vpmultishiftqb %zmm1, %zmm0, %zmm3 # encoding: [0x62,0xf2,0xfd,0x48,0x83,0xd9]
 ; X64-NEXT:    kmovq %rdi, %k1 # encoding: [0xc4,0xe1,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpmultishiftqb %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x49,0x83,0xd1]
-; X64-NEXT:    vpmultishiftqb %zmm1, %zmm0, %zmm3 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xc9,0x83,0xd9]
-; X64-NEXT:    vpmultishiftqb %zmm1, %zmm0, %zmm0 # encoding: [0x62,0xf2,0xfd,0x48,0x83,0xc1]
-; X64-NEXT:    vpaddb %zmm0, %zmm3, %zmm0 # encoding: [0x62,0xf1,0x65,0x48,0xfc,0xc0]
+; X64-NEXT:    vpmultishiftqb %zmm1, %zmm0, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xc9,0x83,0xc1]
+; X64-NEXT:    vpaddb %zmm3, %zmm0, %zmm0 # encoding: [0x62,0xf1,0x7d,0x48,0xfc,0xc3]
 ; X64-NEXT:    vpaddb %zmm0, %zmm2, %zmm0 # encoding: [0x62,0xf1,0x6d,0x48,0xfc,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <64 x i8> @llvm.x86.avx512.mask.pmultishift.qb.512(<64 x i8> %x0, <64 x i8> %x1, <64 x i8> %x2, i64 %x3)
-  %res1 = call <64 x i8> @llvm.x86.avx512.mask.pmultishift.qb.512(<64 x i8> %x0, <64 x i8> %x1, <64 x i8> zeroinitializer, i64 %x3)
-  %res2 = call <64 x i8> @llvm.x86.avx512.mask.pmultishift.qb.512(<64 x i8> %x0, <64 x i8> %x1, <64 x i8> %x2, i64 -1)
-  %res3 = add <64 x i8> %res, %res1
-  %res4 = add <64 x i8> %res3, %res2
+  %1 = call <64 x i8> @llvm.x86.avx512.pmultishift.qb.512(<64 x i8> %x0, <64 x i8> %x1)
+  %2 = bitcast i64 %x3 to <64 x i1>
+  %3 = select <64 x i1> %2, <64 x i8> %1, <64 x i8> %x2
+  %4 = call <64 x i8> @llvm.x86.avx512.pmultishift.qb.512(<64 x i8> %x0, <64 x i8> %x1)
+  %5 = bitcast i64 %x3 to <64 x i1>
+  %6 = select <64 x i1> %5, <64 x i8> %4, <64 x i8> zeroinitializer
+  %7 = call <64 x i8> @llvm.x86.avx512.pmultishift.qb.512(<64 x i8> %x0, <64 x i8> %x1)
+  %res3 = add <64 x i8> %3, %6
+  %res4 = add <64 x i8> %res3, %7
   ret <64 x i8> %res4
 }
 
diff --git a/test/CodeGen/X86/avx512vbmi2-intrinsics-fast-isel.ll b/test/CodeGen/X86/avx512vbmi2-intrinsics-fast-isel.ll
index 0a0837d267151fc23f2fbd56b3dad91ebe303c9d..0ba766cbfe8a48a7fe0f8c2034bbfb069f928f7d 100644
--- a/test/CodeGen/X86/avx512vbmi2-intrinsics-fast-isel.ll
+++ b/test/CodeGen/X86/avx512vbmi2-intrinsics-fast-isel.ll
@@ -308,22 +308,22 @@ define <8 x i64> @test_mm512_mask_shldi_epi64(<8 x i64> %__S, i8 zeroext %__U, <
 ; X86:       # %bb.0: # %entry
 ; X86-NEXT:    movb {{[0-9]+}}(%esp), %al
 ; X86-NEXT:    kmovd %eax, %k1
-; X86-NEXT:    vpshldq $127, %zmm2, %zmm1, %zmm0 {%k1}
+; X86-NEXT:    vpshldq $47, %zmm2, %zmm1, %zmm0 {%k1}
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: test_mm512_mask_shldi_epi64:
 ; X64:       # %bb.0: # %entry
 ; X64-NEXT:    kmovd %edi, %k1
-; X64-NEXT:    vpshldq $127, %zmm2, %zmm1, %zmm0 {%k1}
+; X64-NEXT:    vpshldq $47, %zmm2, %zmm1, %zmm0 {%k1}
 ; X64-NEXT:    retq
 entry:
-  %0 = tail call <8 x i64> @llvm.x86.avx512.vpshld.q.512(<8 x i64> %__A, <8 x i64> %__B, i32 127)
+  %0 = tail call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %__A, <8 x i64> %__B, <8 x i64> <i64 47, i64 47, i64 47, i64 47, i64 47, i64 47, i64 47, i64 47>)
   %1 = bitcast i8 %__U to <8 x i1>
   %2 = select <8 x i1> %1, <8 x i64> %0, <8 x i64> %__S
   ret <8 x i64> %2
 }
 
-declare <8 x i64> @llvm.x86.avx512.vpshld.q.512(<8 x i64>, <8 x i64>, i32)
+declare <8 x i64> @llvm.fshl.v8i64(<8 x i64>, <8 x i64>, <8 x i64>)
 
 define <8 x i64> @test_mm512_maskz_shldi_epi64(i8 zeroext %__U, <8 x i64> %__A, <8 x i64> %__B) {
 ; X86-LABEL: test_mm512_maskz_shldi_epi64:
@@ -339,7 +339,7 @@ define <8 x i64> @test_mm512_maskz_shldi_epi64(i8 zeroext %__U, <8 x i64> %__A,
 ; X64-NEXT:    vpshldq $63, %zmm1, %zmm0, %zmm0 {%k1} {z}
 ; X64-NEXT:    retq
 entry:
-  %0 = tail call <8 x i64> @llvm.x86.avx512.vpshld.q.512(<8 x i64> %__A, <8 x i64> %__B, i32 63)
+  %0 = tail call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %__A, <8 x i64> %__B, <8 x i64> <i64 63, i64 63, i64 63, i64 63, i64 63, i64 63, i64 63, i64 63>)
   %1 = bitcast i8 %__U to <8 x i1>
   %2 = select <8 x i1> %1, <8 x i64> %0, <8 x i64> zeroinitializer
   ret <8 x i64> %2
@@ -351,7 +351,7 @@ define <8 x i64> @test_mm512_shldi_epi64(<8 x i64> %__A, <8 x i64> %__B) {
 ; CHECK-NEXT:    vpshldq $31, %zmm1, %zmm0, %zmm0
 ; CHECK-NEXT:    ret{{[l|q]}}
 entry:
-  %0 = tail call <8 x i64> @llvm.x86.avx512.vpshld.q.512(<8 x i64> %__A, <8 x i64> %__B, i32 31)
+  %0 = tail call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %__A, <8 x i64> %__B, <8 x i64> <i64 31, i64 31, i64 31, i64 31, i64 31, i64 31, i64 31, i64 31>)
   ret <8 x i64> %0
 }
 
@@ -359,18 +359,18 @@ define <8 x i64> @test_mm512_mask_shldi_epi32(<8 x i64> %__S, i16 zeroext %__U,
 ; X86-LABEL: test_mm512_mask_shldi_epi32:
 ; X86:       # %bb.0: # %entry
 ; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
-; X86-NEXT:    vpshldd $127, %zmm2, %zmm1, %zmm0 {%k1}
+; X86-NEXT:    vpshldd $7, %zmm2, %zmm1, %zmm0 {%k1}
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: test_mm512_mask_shldi_epi32:
 ; X64:       # %bb.0: # %entry
 ; X64-NEXT:    kmovd %edi, %k1
-; X64-NEXT:    vpshldd $127, %zmm2, %zmm1, %zmm0 {%k1}
+; X64-NEXT:    vpshldd $7, %zmm2, %zmm1, %zmm0 {%k1}
 ; X64-NEXT:    retq
 entry:
   %0 = bitcast <8 x i64> %__A to <16 x i32>
   %1 = bitcast <8 x i64> %__B to <16 x i32>
-  %2 = tail call <16 x i32> @llvm.x86.avx512.vpshld.d.512(<16 x i32> %0, <16 x i32> %1, i32 127)
+  %2 = tail call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %0, <16 x i32> %1, <16 x i32> <i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7>)
   %3 = bitcast <8 x i64> %__S to <16 x i32>
   %4 = bitcast i16 %__U to <16 x i1>
   %5 = select <16 x i1> %4, <16 x i32> %2, <16 x i32> %3
@@ -378,24 +378,24 @@ entry:
   ret <8 x i64> %6
 }
 
-declare <16 x i32> @llvm.x86.avx512.vpshld.d.512(<16 x i32>, <16 x i32>, i32)
+declare <16 x i32> @llvm.fshl.v16i32(<16 x i32>, <16 x i32>, <16 x i32>)
 
 define <8 x i64> @test_mm512_maskz_shldi_epi32(i16 zeroext %__U, <8 x i64> %__A, <8 x i64> %__B) {
 ; X86-LABEL: test_mm512_maskz_shldi_epi32:
 ; X86:       # %bb.0: # %entry
 ; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
-; X86-NEXT:    vpshldd $63, %zmm1, %zmm0, %zmm0 {%k1} {z}
+; X86-NEXT:    vpshldd $15, %zmm1, %zmm0, %zmm0 {%k1} {z}
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: test_mm512_maskz_shldi_epi32:
 ; X64:       # %bb.0: # %entry
 ; X64-NEXT:    kmovd %edi, %k1
-; X64-NEXT:    vpshldd $63, %zmm1, %zmm0, %zmm0 {%k1} {z}
+; X64-NEXT:    vpshldd $15, %zmm1, %zmm0, %zmm0 {%k1} {z}
 ; X64-NEXT:    retq
 entry:
   %0 = bitcast <8 x i64> %__A to <16 x i32>
   %1 = bitcast <8 x i64> %__B to <16 x i32>
-  %2 = tail call <16 x i32> @llvm.x86.avx512.vpshld.d.512(<16 x i32> %0, <16 x i32> %1, i32 63)
+  %2 = tail call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %0, <16 x i32> %1, <16 x i32> <i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15>)
   %3 = bitcast i16 %__U to <16 x i1>
   %4 = select <16 x i1> %3, <16 x i32> %2, <16 x i32> zeroinitializer
   %5 = bitcast <16 x i32> %4 to <8 x i64>
@@ -410,7 +410,7 @@ define <8 x i64> @test_mm512_shldi_epi32(<8 x i64> %__A, <8 x i64> %__B) {
 entry:
   %0 = bitcast <8 x i64> %__A to <16 x i32>
   %1 = bitcast <8 x i64> %__B to <16 x i32>
-  %2 = tail call <16 x i32> @llvm.x86.avx512.vpshld.d.512(<16 x i32> %0, <16 x i32> %1, i32 31)
+  %2 = tail call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %0, <16 x i32> %1, <16 x i32> <i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31>)
   %3 = bitcast <16 x i32> %2 to <8 x i64>
   ret <8 x i64> %3
 }
@@ -419,18 +419,18 @@ define <8 x i64> @test_mm512_mask_shldi_epi16(<8 x i64> %__S, i32 %__U, <8 x i64
 ; X86-LABEL: test_mm512_mask_shldi_epi16:
 ; X86:       # %bb.0: # %entry
 ; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1
-; X86-NEXT:    vpshldw $127, %zmm2, %zmm1, %zmm0 {%k1}
+; X86-NEXT:    vpshldw $3, %zmm2, %zmm1, %zmm0 {%k1}
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: test_mm512_mask_shldi_epi16:
 ; X64:       # %bb.0: # %entry
 ; X64-NEXT:    kmovd %edi, %k1
-; X64-NEXT:    vpshldw $127, %zmm2, %zmm1, %zmm0 {%k1}
+; X64-NEXT:    vpshldw $3, %zmm2, %zmm1, %zmm0 {%k1}
 ; X64-NEXT:    retq
 entry:
   %0 = bitcast <8 x i64> %__A to <32 x i16>
   %1 = bitcast <8 x i64> %__B to <32 x i16>
-  %2 = tail call <32 x i16> @llvm.x86.avx512.vpshld.w.512(<32 x i16> %0, <32 x i16> %1, i32 127)
+  %2 = tail call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %0, <32 x i16> %1, <32 x i16> <i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3>)
   %3 = bitcast <8 x i64> %__S to <32 x i16>
   %4 = bitcast i32 %__U to <32 x i1>
   %5 = select <32 x i1> %4, <32 x i16> %2, <32 x i16> %3
@@ -438,24 +438,24 @@ entry:
   ret <8 x i64> %6
 }
 
-declare <32 x i16> @llvm.x86.avx512.vpshld.w.512(<32 x i16>, <32 x i16>, i32)
+declare <32 x i16> @llvm.fshl.v32i16(<32 x i16>, <32 x i16>, <32 x i16>)
 
 define <8 x i64> @test_mm512_maskz_shldi_epi16(i32 %__U, <8 x i64> %__A, <8 x i64> %__B) {
 ; X86-LABEL: test_mm512_maskz_shldi_epi16:
 ; X86:       # %bb.0: # %entry
 ; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1
-; X86-NEXT:    vpshldw $63, %zmm1, %zmm0, %zmm0 {%k1} {z}
+; X86-NEXT:    vpshldw $7, %zmm1, %zmm0, %zmm0 {%k1} {z}
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: test_mm512_maskz_shldi_epi16:
 ; X64:       # %bb.0: # %entry
 ; X64-NEXT:    kmovd %edi, %k1
-; X64-NEXT:    vpshldw $63, %zmm1, %zmm0, %zmm0 {%k1} {z}
+; X64-NEXT:    vpshldw $7, %zmm1, %zmm0, %zmm0 {%k1} {z}
 ; X64-NEXT:    retq
 entry:
   %0 = bitcast <8 x i64> %__A to <32 x i16>
   %1 = bitcast <8 x i64> %__B to <32 x i16>
-  %2 = tail call <32 x i16> @llvm.x86.avx512.vpshld.w.512(<32 x i16> %0, <32 x i16> %1, i32 63)
+  %2 = tail call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %0, <32 x i16> %1, <32 x i16> <i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7>)
   %3 = bitcast i32 %__U to <32 x i1>
   %4 = select <32 x i1> %3, <32 x i16> %2, <32 x i16> zeroinitializer
   %5 = bitcast <32 x i16> %4 to <8 x i64>
@@ -465,12 +465,12 @@ entry:
 define <8 x i64> @test_mm512_shldi_epi16(<8 x i64> %__A, <8 x i64> %__B) {
 ; CHECK-LABEL: test_mm512_shldi_epi16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vpshldw $31, %zmm1, %zmm0, %zmm0
+; CHECK-NEXT:    vpshldw $15, %zmm1, %zmm0, %zmm0
 ; CHECK-NEXT:    ret{{[l|q]}}
 entry:
   %0 = bitcast <8 x i64> %__A to <32 x i16>
   %1 = bitcast <8 x i64> %__B to <32 x i16>
-  %2 = tail call <32 x i16> @llvm.x86.avx512.vpshld.w.512(<32 x i16> %0, <32 x i16> %1, i32 31)
+  %2 = tail call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %0, <32 x i16> %1, <32 x i16> <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>)
   %3 = bitcast <32 x i16> %2 to <8 x i64>
   ret <8 x i64> %3
 }
@@ -480,22 +480,22 @@ define <8 x i64> @test_mm512_mask_shrdi_epi64(<8 x i64> %__S, i8 zeroext %__U, <
 ; X86:       # %bb.0: # %entry
 ; X86-NEXT:    movb {{[0-9]+}}(%esp), %al
 ; X86-NEXT:    kmovd %eax, %k1
-; X86-NEXT:    vpshrdq $127, %zmm2, %zmm1, %zmm0 {%k1}
+; X86-NEXT:    vpshrdq $47, %zmm2, %zmm1, %zmm0 {%k1}
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: test_mm512_mask_shrdi_epi64:
 ; X64:       # %bb.0: # %entry
 ; X64-NEXT:    kmovd %edi, %k1
-; X64-NEXT:    vpshrdq $127, %zmm2, %zmm1, %zmm0 {%k1}
+; X64-NEXT:    vpshrdq $47, %zmm2, %zmm1, %zmm0 {%k1}
 ; X64-NEXT:    retq
 entry:
-  %0 = tail call <8 x i64> @llvm.x86.avx512.vpshrd.q.512(<8 x i64> %__A, <8 x i64> %__B, i32 127)
+  %0 = tail call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %__B, <8 x i64> %__A, <8 x i64> <i64 47, i64 47, i64 47, i64 47, i64 47, i64 47, i64 47, i64 47>)
   %1 = bitcast i8 %__U to <8 x i1>
   %2 = select <8 x i1> %1, <8 x i64> %0, <8 x i64> %__S
   ret <8 x i64> %2
 }
 
-declare <8 x i64> @llvm.x86.avx512.vpshrd.q.512(<8 x i64>, <8 x i64>, i32)
+declare <8 x i64> @llvm.fshr.v8i64(<8 x i64>, <8 x i64>, <8 x i64>)
 
 define <8 x i64> @test_mm512_maskz_shrdi_epi64(i8 zeroext %__U, <8 x i64> %__A, <8 x i64> %__B) {
 ; X86-LABEL: test_mm512_maskz_shrdi_epi64:
@@ -511,7 +511,7 @@ define <8 x i64> @test_mm512_maskz_shrdi_epi64(i8 zeroext %__U, <8 x i64> %__A,
 ; X64-NEXT:    vpshrdq $63, %zmm1, %zmm0, %zmm0 {%k1} {z}
 ; X64-NEXT:    retq
 entry:
-  %0 = tail call <8 x i64> @llvm.x86.avx512.vpshrd.q.512(<8 x i64> %__A, <8 x i64> %__B, i32 63)
+  %0 = tail call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %__B, <8 x i64> %__A, <8 x i64> <i64 63, i64 63, i64 63, i64 63, i64 63, i64 63, i64 63, i64 63>)
   %1 = bitcast i8 %__U to <8 x i1>
   %2 = select <8 x i1> %1, <8 x i64> %0, <8 x i64> zeroinitializer
   ret <8 x i64> %2
@@ -523,7 +523,7 @@ define <8 x i64> @test_mm512_shrdi_epi64(<8 x i64> %__A, <8 x i64> %__B) {
 ; CHECK-NEXT:    vpshrdq $31, %zmm1, %zmm0, %zmm0
 ; CHECK-NEXT:    ret{{[l|q]}}
 entry:
-  %0 = tail call <8 x i64> @llvm.x86.avx512.vpshrd.q.512(<8 x i64> %__A, <8 x i64> %__B, i32 31)
+  %0 = tail call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %__B, <8 x i64> %__A, <8 x i64> <i64 31, i64 31, i64 31, i64 31, i64 31, i64 31, i64 31, i64 31>)
   ret <8 x i64> %0
 }
 
@@ -531,18 +531,18 @@ define <8 x i64> @test_mm512_mask_shrdi_epi32(<8 x i64> %__S, i16 zeroext %__U,
 ; X86-LABEL: test_mm512_mask_shrdi_epi32:
 ; X86:       # %bb.0: # %entry
 ; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
-; X86-NEXT:    vpshrdd $127, %zmm2, %zmm1, %zmm0 {%k1}
+; X86-NEXT:    vpshrdd $7, %zmm2, %zmm1, %zmm0 {%k1}
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: test_mm512_mask_shrdi_epi32:
 ; X64:       # %bb.0: # %entry
 ; X64-NEXT:    kmovd %edi, %k1
-; X64-NEXT:    vpshrdd $127, %zmm2, %zmm1, %zmm0 {%k1}
+; X64-NEXT:    vpshrdd $7, %zmm2, %zmm1, %zmm0 {%k1}
 ; X64-NEXT:    retq
 entry:
   %0 = bitcast <8 x i64> %__A to <16 x i32>
   %1 = bitcast <8 x i64> %__B to <16 x i32>
-  %2 = tail call <16 x i32> @llvm.x86.avx512.vpshrd.d.512(<16 x i32> %0, <16 x i32> %1, i32 127)
+  %2 = tail call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %1, <16 x i32> %0, <16 x i32> <i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7>)
   %3 = bitcast <8 x i64> %__S to <16 x i32>
   %4 = bitcast i16 %__U to <16 x i1>
   %5 = select <16 x i1> %4, <16 x i32> %2, <16 x i32> %3
@@ -550,24 +550,24 @@ entry:
   ret <8 x i64> %6
 }
 
-declare <16 x i32> @llvm.x86.avx512.vpshrd.d.512(<16 x i32>, <16 x i32>, i32)
+declare <16 x i32> @llvm.fshr.v16i32(<16 x i32>, <16 x i32>, <16 x i32>)
 
 define <8 x i64> @test_mm512_maskz_shrdi_epi32(i16 zeroext %__U, <8 x i64> %__A, <8 x i64> %__B) {
 ; X86-LABEL: test_mm512_maskz_shrdi_epi32:
 ; X86:       # %bb.0: # %entry
 ; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
-; X86-NEXT:    vpshrdd $63, %zmm1, %zmm0, %zmm0 {%k1} {z}
+; X86-NEXT:    vpshrdd $15, %zmm1, %zmm0, %zmm0 {%k1} {z}
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: test_mm512_maskz_shrdi_epi32:
 ; X64:       # %bb.0: # %entry
 ; X64-NEXT:    kmovd %edi, %k1
-; X64-NEXT:    vpshrdd $63, %zmm1, %zmm0, %zmm0 {%k1} {z}
+; X64-NEXT:    vpshrdd $15, %zmm1, %zmm0, %zmm0 {%k1} {z}
 ; X64-NEXT:    retq
 entry:
   %0 = bitcast <8 x i64> %__A to <16 x i32>
   %1 = bitcast <8 x i64> %__B to <16 x i32>
-  %2 = tail call <16 x i32> @llvm.x86.avx512.vpshrd.d.512(<16 x i32> %0, <16 x i32> %1, i32 63)
+  %2 = tail call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %1, <16 x i32> %0, <16 x i32> <i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15>)
   %3 = bitcast i16 %__U to <16 x i1>
   %4 = select <16 x i1> %3, <16 x i32> %2, <16 x i32> zeroinitializer
   %5 = bitcast <16 x i32> %4 to <8 x i64>
@@ -582,7 +582,7 @@ define <8 x i64> @test_mm512_shrdi_epi32(<8 x i64> %__A, <8 x i64> %__B) {
 entry:
   %0 = bitcast <8 x i64> %__A to <16 x i32>
   %1 = bitcast <8 x i64> %__B to <16 x i32>
-  %2 = tail call <16 x i32> @llvm.x86.avx512.vpshrd.d.512(<16 x i32> %0, <16 x i32> %1, i32 31)
+  %2 = tail call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %1, <16 x i32> %0, <16 x i32> <i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31>)
   %3 = bitcast <16 x i32> %2 to <8 x i64>
   ret <8 x i64> %3
 }
@@ -591,18 +591,18 @@ define <8 x i64> @test_mm512_mask_shrdi_epi16(<8 x i64> %__S, i32 %__U, <8 x i64
 ; X86-LABEL: test_mm512_mask_shrdi_epi16:
 ; X86:       # %bb.0: # %entry
 ; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1
-; X86-NEXT:    vpshrdw $127, %zmm2, %zmm1, %zmm0 {%k1}
+; X86-NEXT:    vpshrdw $3, %zmm2, %zmm1, %zmm0 {%k1}
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: test_mm512_mask_shrdi_epi16:
 ; X64:       # %bb.0: # %entry
 ; X64-NEXT:    kmovd %edi, %k1
-; X64-NEXT:    vpshrdw $127, %zmm2, %zmm1, %zmm0 {%k1}
+; X64-NEXT:    vpshrdw $3, %zmm2, %zmm1, %zmm0 {%k1}
 ; X64-NEXT:    retq
 entry:
   %0 = bitcast <8 x i64> %__A to <32 x i16>
   %1 = bitcast <8 x i64> %__B to <32 x i16>
-  %2 = tail call <32 x i16> @llvm.x86.avx512.vpshrd.w.512(<32 x i16> %0, <32 x i16> %1, i32 127)
+  %2 = tail call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %1, <32 x i16> %0, <32 x i16> <i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3>)
   %3 = bitcast <8 x i64> %__S to <32 x i16>
   %4 = bitcast i32 %__U to <32 x i1>
   %5 = select <32 x i1> %4, <32 x i16> %2, <32 x i16> %3
@@ -610,24 +610,24 @@ entry:
   ret <8 x i64> %6
 }
 
-declare <32 x i16> @llvm.x86.avx512.vpshrd.w.512(<32 x i16>, <32 x i16>, i32)
+declare <32 x i16> @llvm.fshr.v32i16(<32 x i16>, <32 x i16>, <32 x i16>)
 
 define <8 x i64> @test_mm512_maskz_shrdi_epi16(i32 %__U, <8 x i64> %__A, <8 x i64> %__B) {
 ; X86-LABEL: test_mm512_maskz_shrdi_epi16:
 ; X86:       # %bb.0: # %entry
 ; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1
-; X86-NEXT:    vpshrdw $63, %zmm1, %zmm0, %zmm0 {%k1} {z}
+; X86-NEXT:    vpshrdw $15, %zmm1, %zmm0, %zmm0 {%k1} {z}
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: test_mm512_maskz_shrdi_epi16:
 ; X64:       # %bb.0: # %entry
 ; X64-NEXT:    kmovd %edi, %k1
-; X64-NEXT:    vpshrdw $63, %zmm1, %zmm0, %zmm0 {%k1} {z}
+; X64-NEXT:    vpshrdw $15, %zmm1, %zmm0, %zmm0 {%k1} {z}
 ; X64-NEXT:    retq
 entry:
   %0 = bitcast <8 x i64> %__A to <32 x i16>
   %1 = bitcast <8 x i64> %__B to <32 x i16>
-  %2 = tail call <32 x i16> @llvm.x86.avx512.vpshrd.w.512(<32 x i16> %0, <32 x i16> %1, i32 63)
+  %2 = tail call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %1, <32 x i16> %0, <32 x i16> <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>)
   %3 = bitcast i32 %__U to <32 x i1>
   %4 = select <32 x i1> %3, <32 x i16> %2, <32 x i16> zeroinitializer
   %5 = bitcast <32 x i16> %4 to <8 x i64>
@@ -637,12 +637,12 @@ entry:
 define <8 x i64> @test_mm512_shrdi_epi16(<8 x i64> %__A, <8 x i64> %__B) {
 ; CHECK-LABEL: test_mm512_shrdi_epi16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vpshrdw $31, %zmm1, %zmm0, %zmm0
+; CHECK-NEXT:    vpshrdw $15, %zmm1, %zmm0, %zmm0
 ; CHECK-NEXT:    ret{{[l|q]}}
 entry:
   %0 = bitcast <8 x i64> %__A to <32 x i16>
   %1 = bitcast <8 x i64> %__B to <32 x i16>
-  %2 = tail call <32 x i16> @llvm.x86.avx512.vpshrd.w.512(<32 x i16> %0, <32 x i16> %1, i32 31)
+  %2 = tail call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %1, <32 x i16> %0, <32 x i16> <i16 31, i16 31, i16 31, i16 31, i16 31, i16 31, i16 31, i16 31, i16 31, i16 31, i16 31, i16 31, i16 31, i16 31, i16 31, i16 31, i16 31, i16 31, i16 31, i16 31, i16 31, i16 31, i16 31, i16 31, i16 31, i16 31, i16 31, i16 31, i16 31, i16 31, i16 31, i16 31>)
   %3 = bitcast <32 x i16> %2 to <8 x i64>
   ret <8 x i64> %3
 }
@@ -661,8 +661,10 @@ define <8 x i64> @test_mm512_mask_shldv_epi64(<8 x i64> %__S, i8 zeroext %__U, <
 ; X64-NEXT:    vpshldvq %zmm2, %zmm1, %zmm0 {%k1}
 ; X64-NEXT:    retq
 entry:
-  %0 = tail call <8 x i64> @llvm.x86.avx512.mask.vpshldv.q.512(<8 x i64> %__S, <8 x i64> %__A, <8 x i64> %__B, i8 %__U)
-  ret <8 x i64> %0
+  %0 = tail call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %__S, <8 x i64> %__A, <8 x i64> %__B)
+  %1 = bitcast i8 %__U to <8 x i1>
+  %2 = select <8 x i1> %1, <8 x i64> %0, <8 x i64> %__S
+  ret <8 x i64> %2
 }
 
 define <8 x i64> @test_mm512_maskz_shldv_epi64(i8 zeroext %__U, <8 x i64> %__S, <8 x i64> %__A, <8 x i64> %__B) {
@@ -679,8 +681,10 @@ define <8 x i64> @test_mm512_maskz_shldv_epi64(i8 zeroext %__U, <8 x i64> %__S,
 ; X64-NEXT:    vpshldvq %zmm2, %zmm1, %zmm0 {%k1} {z}
 ; X64-NEXT:    retq
 entry:
-  %0 = tail call <8 x i64> @llvm.x86.avx512.maskz.vpshldv.q.512(<8 x i64> %__S, <8 x i64> %__A, <8 x i64> %__B, i8 %__U)
-  ret <8 x i64> %0
+  %0 = tail call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %__S, <8 x i64> %__A, <8 x i64> %__B)
+  %1 = bitcast i8 %__U to <8 x i1>
+  %2 = select <8 x i1> %1, <8 x i64> %0, <8 x i64> zeroinitializer
+  ret <8 x i64> %2
 }
 
 define <8 x i64> @test_mm512_shldv_epi64(<8 x i64> %__S, <8 x i64> %__A, <8 x i64> %__B) {
@@ -689,7 +693,7 @@ define <8 x i64> @test_mm512_shldv_epi64(<8 x i64> %__S, <8 x i64> %__A, <8 x i6
 ; CHECK-NEXT:    vpshldvq %zmm2, %zmm1, %zmm0
 ; CHECK-NEXT:    ret{{[l|q]}}
 entry:
-  %0 = tail call <8 x i64> @llvm.x86.avx512.mask.vpshldv.q.512(<8 x i64> %__S, <8 x i64> %__A, <8 x i64> %__B, i8 -1)
+  %0 = tail call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %__S, <8 x i64> %__A, <8 x i64> %__B)
   ret <8 x i64> %0
 }
 
@@ -709,9 +713,11 @@ entry:
   %0 = bitcast <8 x i64> %__S to <16 x i32>
   %1 = bitcast <8 x i64> %__A to <16 x i32>
   %2 = bitcast <8 x i64> %__B to <16 x i32>
-  %3 = tail call <16 x i32> @llvm.x86.avx512.mask.vpshldv.d.512(<16 x i32> %0, <16 x i32> %1, <16 x i32> %2, i16 %__U)
-  %4 = bitcast <16 x i32> %3 to <8 x i64>
-  ret <8 x i64> %4
+  %3 = tail call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %0, <16 x i32> %1, <16 x i32> %2)
+  %4 = bitcast i16 %__U to <16 x i1>
+  %5 = select <16 x i1> %4, <16 x i32> %3, <16 x i32> %0
+  %6 = bitcast <16 x i32> %5 to <8 x i64>
+  ret <8 x i64> %6
 }
 
 define <8 x i64> @test_mm512_maskz_shldv_epi32(i16 zeroext %__U, <8 x i64> %__S, <8 x i64> %__A, <8 x i64> %__B) {
@@ -730,9 +736,11 @@ entry:
   %0 = bitcast <8 x i64> %__S to <16 x i32>
   %1 = bitcast <8 x i64> %__A to <16 x i32>
   %2 = bitcast <8 x i64> %__B to <16 x i32>
-  %3 = tail call <16 x i32> @llvm.x86.avx512.maskz.vpshldv.d.512(<16 x i32> %0, <16 x i32> %1, <16 x i32> %2, i16 %__U)
-  %4 = bitcast <16 x i32> %3 to <8 x i64>
-  ret <8 x i64> %4
+  %3 = tail call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %0, <16 x i32> %1, <16 x i32> %2)
+  %4 = bitcast i16 %__U to <16 x i1>
+  %5 = select <16 x i1> %4, <16 x i32> %3, <16 x i32> zeroinitializer
+  %6 = bitcast <16 x i32> %5 to <8 x i64>
+  ret <8 x i64> %6
 }
 
 define <8 x i64> @test_mm512_shldv_epi32(<8 x i64> %__S, <8 x i64> %__A, <8 x i64> %__B) {
@@ -744,7 +752,7 @@ entry:
   %0 = bitcast <8 x i64> %__S to <16 x i32>
   %1 = bitcast <8 x i64> %__A to <16 x i32>
   %2 = bitcast <8 x i64> %__B to <16 x i32>
-  %3 = tail call <16 x i32> @llvm.x86.avx512.mask.vpshldv.d.512(<16 x i32> %0, <16 x i32> %1, <16 x i32> %2, i16 -1)
+  %3 = tail call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %0, <16 x i32> %1, <16 x i32> %2)
   %4 = bitcast <16 x i32> %3 to <8 x i64>
   ret <8 x i64> %4
 }
@@ -765,9 +773,11 @@ entry:
   %0 = bitcast <8 x i64> %__S to <32 x i16>
   %1 = bitcast <8 x i64> %__A to <32 x i16>
   %2 = bitcast <8 x i64> %__B to <32 x i16>
-  %3 = tail call <32 x i16> @llvm.x86.avx512.mask.vpshldv.w.512(<32 x i16> %0, <32 x i16> %1, <32 x i16> %2, i32 %__U)
-  %4 = bitcast <32 x i16> %3 to <8 x i64>
-  ret <8 x i64> %4
+  %3 = tail call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %0, <32 x i16> %1, <32 x i16> %2)
+  %4 = bitcast i32 %__U to <32 x i1>
+  %5 = select <32 x i1> %4, <32 x i16> %3, <32 x i16> %0
+  %6 = bitcast <32 x i16> %5 to <8 x i64>
+  ret <8 x i64> %6
 }
 
 define <8 x i64> @test_mm512_maskz_shldv_epi16(i32 %__U, <8 x i64> %__S, <8 x i64> %__A, <8 x i64> %__B) {
@@ -786,9 +796,11 @@ entry:
   %0 = bitcast <8 x i64> %__S to <32 x i16>
   %1 = bitcast <8 x i64> %__A to <32 x i16>
   %2 = bitcast <8 x i64> %__B to <32 x i16>
-  %3 = tail call <32 x i16> @llvm.x86.avx512.maskz.vpshldv.w.512(<32 x i16> %0, <32 x i16> %1, <32 x i16> %2, i32 %__U)
-  %4 = bitcast <32 x i16> %3 to <8 x i64>
-  ret <8 x i64> %4
+  %3 = tail call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %0, <32 x i16> %1, <32 x i16> %2)
+  %4 = bitcast i32 %__U to <32 x i1>
+  %5 = select <32 x i1> %4, <32 x i16> %3, <32 x i16> zeroinitializer
+  %6 = bitcast <32 x i16> %5 to <8 x i64>
+  ret <8 x i64> %6
 }
 
 define <8 x i64> @test_mm512_shldv_epi16(<8 x i64> %__S, <8 x i64> %__A, <8 x i64> %__B) {
@@ -800,7 +812,7 @@ entry:
   %0 = bitcast <8 x i64> %__S to <32 x i16>
   %1 = bitcast <8 x i64> %__A to <32 x i16>
   %2 = bitcast <8 x i64> %__B to <32 x i16>
-  %3 = tail call <32 x i16> @llvm.x86.avx512.mask.vpshldv.w.512(<32 x i16> %0, <32 x i16> %1, <32 x i16> %2, i32 -1)
+  %3 = tail call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %0, <32 x i16> %1, <32 x i16> %2)
   %4 = bitcast <32 x i16> %3 to <8 x i64>
   ret <8 x i64> %4
 }
@@ -819,8 +831,10 @@ define <8 x i64> @test_mm512_mask_shrdv_epi64(<8 x i64> %__S, i8 zeroext %__U, <
 ; X64-NEXT:    vpshrdvq %zmm2, %zmm1, %zmm0 {%k1}
 ; X64-NEXT:    retq
 entry:
-  %0 = tail call <8 x i64> @llvm.x86.avx512.mask.vpshrdv.q.512(<8 x i64> %__S, <8 x i64> %__A, <8 x i64> %__B, i8 %__U)
-  ret <8 x i64> %0
+  %0 = tail call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %__A, <8 x i64> %__S, <8 x i64> %__B)
+  %1 = bitcast i8 %__U to <8 x i1>
+  %2 = select <8 x i1> %1, <8 x i64> %0, <8 x i64> %__S
+  ret <8 x i64> %2
 }
 
 define <8 x i64> @test_mm512_maskz_shrdv_epi64(i8 zeroext %__U, <8 x i64> %__S, <8 x i64> %__A, <8 x i64> %__B) {
@@ -837,8 +851,10 @@ define <8 x i64> @test_mm512_maskz_shrdv_epi64(i8 zeroext %__U, <8 x i64> %__S,
 ; X64-NEXT:    vpshrdvq %zmm2, %zmm1, %zmm0 {%k1} {z}
 ; X64-NEXT:    retq
 entry:
-  %0 = tail call <8 x i64> @llvm.x86.avx512.maskz.vpshrdv.q.512(<8 x i64> %__S, <8 x i64> %__A, <8 x i64> %__B, i8 %__U)
-  ret <8 x i64> %0
+  %0 = tail call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %__A, <8 x i64> %__S, <8 x i64> %__B)
+  %1 = bitcast i8 %__U to <8 x i1>
+  %2 = select <8 x i1> %1, <8 x i64> %0, <8 x i64> zeroinitializer
+  ret <8 x i64> %2
 }
 
 define <8 x i64> @test_mm512_shrdv_epi64(<8 x i64> %__S, <8 x i64> %__A, <8 x i64> %__B) {
@@ -847,7 +863,7 @@ define <8 x i64> @test_mm512_shrdv_epi64(<8 x i64> %__S, <8 x i64> %__A, <8 x i6
 ; CHECK-NEXT:    vpshrdvq %zmm2, %zmm1, %zmm0
 ; CHECK-NEXT:    ret{{[l|q]}}
 entry:
-  %0 = tail call <8 x i64> @llvm.x86.avx512.mask.vpshrdv.q.512(<8 x i64> %__S, <8 x i64> %__A, <8 x i64> %__B, i8 -1)
+  %0 = tail call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %__A, <8 x i64> %__S, <8 x i64> %__B)
   ret <8 x i64> %0
 }
 
@@ -867,9 +883,11 @@ entry:
   %0 = bitcast <8 x i64> %__S to <16 x i32>
   %1 = bitcast <8 x i64> %__A to <16 x i32>
   %2 = bitcast <8 x i64> %__B to <16 x i32>
-  %3 = tail call <16 x i32> @llvm.x86.avx512.mask.vpshrdv.d.512(<16 x i32> %0, <16 x i32> %1, <16 x i32> %2, i16 %__U)
-  %4 = bitcast <16 x i32> %3 to <8 x i64>
-  ret <8 x i64> %4
+  %3 = tail call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %1, <16 x i32> %0, <16 x i32> %2)
+  %4 = bitcast i16 %__U to <16 x i1>
+  %5 = select <16 x i1> %4, <16 x i32> %3, <16 x i32> %0
+  %6 = bitcast <16 x i32> %5 to <8 x i64>
+  ret <8 x i64> %6
 }
 
 define <8 x i64> @test_mm512_maskz_shrdv_epi32(i16 zeroext %__U, <8 x i64> %__S, <8 x i64> %__A, <8 x i64> %__B) {
@@ -888,9 +906,11 @@ entry:
   %0 = bitcast <8 x i64> %__S to <16 x i32>
   %1 = bitcast <8 x i64> %__A to <16 x i32>
   %2 = bitcast <8 x i64> %__B to <16 x i32>
-  %3 = tail call <16 x i32> @llvm.x86.avx512.maskz.vpshrdv.d.512(<16 x i32> %0, <16 x i32> %1, <16 x i32> %2, i16 %__U)
-  %4 = bitcast <16 x i32> %3 to <8 x i64>
-  ret <8 x i64> %4
+  %3 = tail call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %1, <16 x i32> %0, <16 x i32> %2)
+  %4 = bitcast i16 %__U to <16 x i1>
+  %5 = select <16 x i1> %4, <16 x i32> %3, <16 x i32> zeroinitializer
+  %6 = bitcast <16 x i32> %5 to <8 x i64>
+  ret <8 x i64> %6
 }
 
 define <8 x i64> @test_mm512_shrdv_epi32(<8 x i64> %__S, <8 x i64> %__A, <8 x i64> %__B) {
@@ -902,7 +922,7 @@ entry:
   %0 = bitcast <8 x i64> %__S to <16 x i32>
   %1 = bitcast <8 x i64> %__A to <16 x i32>
   %2 = bitcast <8 x i64> %__B to <16 x i32>
-  %3 = tail call <16 x i32> @llvm.x86.avx512.mask.vpshrdv.d.512(<16 x i32> %0, <16 x i32> %1, <16 x i32> %2, i16 -1)
+  %3 = tail call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %1, <16 x i32> %0, <16 x i32> %2)
   %4 = bitcast <16 x i32> %3 to <8 x i64>
   ret <8 x i64> %4
 }
@@ -923,9 +943,11 @@ entry:
   %0 = bitcast <8 x i64> %__S to <32 x i16>
   %1 = bitcast <8 x i64> %__A to <32 x i16>
   %2 = bitcast <8 x i64> %__B to <32 x i16>
-  %3 = tail call <32 x i16> @llvm.x86.avx512.mask.vpshrdv.w.512(<32 x i16> %0, <32 x i16> %1, <32 x i16> %2, i32 %__U)
-  %4 = bitcast <32 x i16> %3 to <8 x i64>
-  ret <8 x i64> %4
+  %3 = tail call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %1, <32 x i16> %0, <32 x i16> %2)
+  %4 = bitcast i32 %__U to <32 x i1>
+  %5 = select <32 x i1> %4, <32 x i16> %3, <32 x i16> %0
+  %6 = bitcast <32 x i16> %5 to <8 x i64>
+  ret <8 x i64> %6
 }
 
 define <8 x i64> @test_mm512_maskz_shrdv_epi16(i32 %__U, <8 x i64> %__S, <8 x i64> %__A, <8 x i64> %__B) {
@@ -944,9 +966,11 @@ entry:
   %0 = bitcast <8 x i64> %__S to <32 x i16>
   %1 = bitcast <8 x i64> %__A to <32 x i16>
   %2 = bitcast <8 x i64> %__B to <32 x i16>
-  %3 = tail call <32 x i16> @llvm.x86.avx512.maskz.vpshrdv.w.512(<32 x i16> %0, <32 x i16> %1, <32 x i16> %2, i32 %__U)
-  %4 = bitcast <32 x i16> %3 to <8 x i64>
-  ret <8 x i64> %4
+  %3 = tail call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %1, <32 x i16> %0, <32 x i16> %2)
+  %4 = bitcast i32 %__U to <32 x i1>
+  %5 = select <32 x i1> %4, <32 x i16> %3, <32 x i16> zeroinitializer
+  %6 = bitcast <32 x i16> %5 to <8 x i64>
+  ret <8 x i64> %6
 }
 
 define <8 x i64> @test_mm512_shrdv_epi16(<8 x i64> %__S, <8 x i64> %__A, <8 x i64> %__B) {
@@ -958,7 +982,7 @@ entry:
   %0 = bitcast <8 x i64> %__S to <32 x i16>
   %1 = bitcast <8 x i64> %__A to <32 x i16>
   %2 = bitcast <8 x i64> %__B to <32 x i16>
-  %3 = tail call <32 x i16> @llvm.x86.avx512.mask.vpshrdv.w.512(<32 x i16> %0, <32 x i16> %1, <32 x i16> %2, i32 -1)
+  %3 = tail call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %1, <32 x i16> %0, <32 x i16> %2)
   %4 = bitcast <32 x i16> %3 to <8 x i64>
   ret <8 x i64> %4
 }
@@ -971,15 +995,3 @@ declare <32 x i16> @llvm.x86.avx512.mask.expand.w.512(<32 x i16>, <32 x i16>, i3
 declare <64 x i8> @llvm.x86.avx512.mask.expand.b.512(<64 x i8>, <64 x i8>, i64)
 declare <32 x i16> @llvm.masked.expandload.v32i16(i16*, <32 x i1>, <32 x i16>)
 declare <64 x i8> @llvm.masked.expandload.v64i8(i8*, <64 x i1>, <64 x i8>)
-declare <8 x i64> @llvm.x86.avx512.mask.vpshldv.q.512(<8 x i64>, <8 x i64>, <8 x i64>, i8)
-declare <8 x i64> @llvm.x86.avx512.maskz.vpshldv.q.512(<8 x i64>, <8 x i64>, <8 x i64>, i8)
-declare <16 x i32> @llvm.x86.avx512.mask.vpshldv.d.512(<16 x i32>, <16 x i32>, <16 x i32>, i16)
-declare <16 x i32> @llvm.x86.avx512.maskz.vpshldv.d.512(<16 x i32>, <16 x i32>, <16 x i32>, i16)
-declare <32 x i16> @llvm.x86.avx512.mask.vpshldv.w.512(<32 x i16>, <32 x i16>, <32 x i16>, i32)
-declare <32 x i16> @llvm.x86.avx512.maskz.vpshldv.w.512(<32 x i16>, <32 x i16>, <32 x i16>, i32)
-declare <8 x i64> @llvm.x86.avx512.mask.vpshrdv.q.512(<8 x i64>, <8 x i64>, <8 x i64>, i8)
-declare <8 x i64> @llvm.x86.avx512.maskz.vpshrdv.q.512(<8 x i64>, <8 x i64>, <8 x i64>, i8)
-declare <16 x i32> @llvm.x86.avx512.mask.vpshrdv.d.512(<16 x i32>, <16 x i32>, <16 x i32>, i16)
-declare <16 x i32> @llvm.x86.avx512.maskz.vpshrdv.d.512(<16 x i32>, <16 x i32>, <16 x i32>, i16)
-declare <32 x i16> @llvm.x86.avx512.mask.vpshrdv.w.512(<32 x i16>, <32 x i16>, <32 x i16>, i32)
-declare <32 x i16> @llvm.x86.avx512.maskz.vpshrdv.w.512(<32 x i16>, <32 x i16>, <32 x i16>, i32)
diff --git a/test/CodeGen/X86/avx512vbmi2-intrinsics-upgrade.ll b/test/CodeGen/X86/avx512vbmi2-intrinsics-upgrade.ll
index 319008f4f29a27c61c7a637225daaed04a857e5d..96eba89f6a40e66cc674548a7e19987bd0ceca0d 100644
--- a/test/CodeGen/X86/avx512vbmi2-intrinsics-upgrade.ll
+++ b/test/CodeGen/X86/avx512vbmi2-intrinsics-upgrade.ll
@@ -55,6 +55,50 @@ define <32 x i16> @test_expand_load_w_512(i8* %addr, <32 x i16> %data) {
   ret <32 x i16> %res
 }
 
+define <32 x i16> @test_expand_w_512(<32 x i16> %data) {
+; CHECK-LABEL: test_expand_w_512:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <32 x i16> @llvm.x86.avx512.mask.expand.w.512(<32 x i16> %data, <32 x i16> undef, i32 -1)
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_mask_expand_w_512(<32 x i16> %data, <32 x i16> %passthru, i32 %mask) {
+; X86-LABEL: test_mask_expand_w_512:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf9,0x90,0x4c,0x24,0x04]
+; X86-NEXT:    vpexpandw %zmm0, %zmm1 {%k1} # encoding: [0x62,0xf2,0xfd,0x49,0x62,0xc8]
+; X86-NEXT:    vmovdqa64 %zmm1, %zmm0 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc1]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_mask_expand_w_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; X64-NEXT:    vpexpandw %zmm0, %zmm1 {%k1} # encoding: [0x62,0xf2,0xfd,0x49,0x62,0xc8]
+; X64-NEXT:    vmovdqa64 %zmm1, %zmm0 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc1]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <32 x i16> @llvm.x86.avx512.mask.expand.w.512(<32 x i16> %data, <32 x i16> %passthru, i32 %mask)
+  ret <32 x i16> %res
+}
+
+define <32 x i16> @test_maskz_expand_w_512(<32 x i16> %data, i32 %mask) {
+; X86-LABEL: test_maskz_expand_w_512:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf9,0x90,0x4c,0x24,0x04]
+; X86-NEXT:    vpexpandw %zmm0, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xc9,0x62,0xc0]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_maskz_expand_w_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; X64-NEXT:    vpexpandw %zmm0, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xc9,0x62,0xc0]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <32 x i16> @llvm.x86.avx512.mask.expand.w.512(<32 x i16> %data, <32 x i16> zeroinitializer, i32 %mask)
+  ret <32 x i16> %res
+}
+
+declare <32 x i16> @llvm.x86.avx512.mask.expand.w.512(<32 x i16> %data, <32 x i16> %src0, i32 %mask)
+
 define <64 x i8> @test_mask_expand_load_b_512(i8* %addr, <64 x i8> %data, i64 %mask) {
 ; X86-LABEL: test_mask_expand_load_b_512:
 ; X86:       # %bb.0:
@@ -108,6 +152,50 @@ define <64 x i8> @test_expand_load_b_512(i8* %addr, <64 x i8> %data) {
   ret <64 x i8> %res
 }
 
+define <64 x i8> @test_expand_b_512(<64 x i8> %data) {
+; CHECK-LABEL: test_expand_b_512:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <64 x i8> @llvm.x86.avx512.mask.expand.b.512(<64 x i8> %data, <64 x i8> undef, i64 -1)
+  ret <64 x i8> %res
+}
+
+define <64 x i8> @test_mask_expand_b_512(<64 x i8> %data, <64 x i8> %passthru, i64 %mask) {
+; X86-LABEL: test_mask_expand_b_512:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovq {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf8,0x90,0x4c,0x24,0x04]
+; X86-NEXT:    vpexpandb %zmm0, %zmm1 {%k1} # encoding: [0x62,0xf2,0x7d,0x49,0x62,0xc8]
+; X86-NEXT:    vmovdqa64 %zmm1, %zmm0 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc1]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_mask_expand_b_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovq %rdi, %k1 # encoding: [0xc4,0xe1,0xfb,0x92,0xcf]
+; X64-NEXT:    vpexpandb %zmm0, %zmm1 {%k1} # encoding: [0x62,0xf2,0x7d,0x49,0x62,0xc8]
+; X64-NEXT:    vmovdqa64 %zmm1, %zmm0 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc1]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <64 x i8> @llvm.x86.avx512.mask.expand.b.512(<64 x i8> %data, <64 x i8> %passthru, i64 %mask)
+  ret <64 x i8> %res
+}
+
+define <64 x i8> @test_maskz_expand_b_512(<64 x i8> %data, i64 %mask) {
+; X86-LABEL: test_maskz_expand_b_512:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovq {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf8,0x90,0x4c,0x24,0x04]
+; X86-NEXT:    vpexpandb %zmm0, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0xc9,0x62,0xc0]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_maskz_expand_b_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovq %rdi, %k1 # encoding: [0xc4,0xe1,0xfb,0x92,0xcf]
+; X64-NEXT:    vpexpandb %zmm0, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0xc9,0x62,0xc0]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <64 x i8> @llvm.x86.avx512.mask.expand.b.512(<64 x i8> %data, <64 x i8> zeroinitializer, i64 %mask)
+  ret <64 x i8> %res
+}
+
+declare <64 x i8> @llvm.x86.avx512.mask.expand.b.512(<64 x i8> %data, <64 x i8> %src0, i64 %mask)
+
 define void @test_mask_compress_store_w_512(i8* %addr, <32 x i16> %data, i32 %mask) {
 ; X86-LABEL: test_mask_compress_store_w_512:
 ; X86:       # %bb.0:
@@ -129,6 +217,25 @@ define void @test_mask_compress_store_w_512(i8* %addr, <32 x i16> %data, i32 %ma
 
 declare void @llvm.x86.avx512.mask.compress.store.w.512(i8* %addr, <32 x i16> %data, i32 %mask)
 
+define void @test_compress_store_w_512(i8* %addr, <32 x i16> %data) {
+; X86-LABEL: test_compress_store_w_512:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-NEXT:    kxnord %k0, %k0, %k1 # encoding: [0xc4,0xe1,0xfd,0x46,0xc8]
+; X86-NEXT:    vpcompressw %zmm0, (%eax) {%k1} # encoding: [0x62,0xf2,0xfd,0x49,0x63,0x00]
+; X86-NEXT:    vzeroupper # encoding: [0xc5,0xf8,0x77]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_compress_store_w_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kxnord %k0, %k0, %k1 # encoding: [0xc4,0xe1,0xfd,0x46,0xc8]
+; X64-NEXT:    vpcompressw %zmm0, (%rdi) {%k1} # encoding: [0x62,0xf2,0xfd,0x49,0x63,0x07]
+; X64-NEXT:    vzeroupper # encoding: [0xc5,0xf8,0x77]
+; X64-NEXT:    retq # encoding: [0xc3]
+  call void @llvm.x86.avx512.mask.compress.store.w.512(i8* %addr, <32 x i16> %data, i32 -1)
+  ret void
+}
+
 define <32 x i16> @test_mask_compress_w_512(<32 x i16> %data, <32 x i16> %passthru, i32 %mask) {
 ; X86-LABEL: test_mask_compress_w_512:
 ; X86:       # %bb.0:
@@ -173,52 +280,50 @@ define <32 x i16> @test_compress_w_512(<32 x i16> %data) {
 
 declare <32 x i16> @llvm.x86.avx512.mask.compress.w.512(<32 x i16> %data, <32 x i16> %src0, i32 %mask)
 
-define void @test_compress_store_w_512(i8* %addr, <32 x i16> %data) {
-; X86-LABEL: test_compress_store_w_512:
+define void @test_mask_compress_store_b_512(i8* %addr, <64 x i8> %data, i64 %mask) {
+; X86-LABEL: test_mask_compress_store_b_512:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
-; X86-NEXT:    kxnord %k0, %k0, %k1 # encoding: [0xc4,0xe1,0xfd,0x46,0xc8]
-; X86-NEXT:    vpcompressw %zmm0, (%eax) {%k1} # encoding: [0x62,0xf2,0xfd,0x49,0x63,0x00]
+; X86-NEXT:    kmovq {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf8,0x90,0x4c,0x24,0x08]
+; X86-NEXT:    vpcompressb %zmm0, (%eax) {%k1} # encoding: [0x62,0xf2,0x7d,0x49,0x63,0x00]
 ; X86-NEXT:    vzeroupper # encoding: [0xc5,0xf8,0x77]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
-; X64-LABEL: test_compress_store_w_512:
+; X64-LABEL: test_mask_compress_store_b_512:
 ; X64:       # %bb.0:
-; X64-NEXT:    kxnord %k0, %k0, %k1 # encoding: [0xc4,0xe1,0xfd,0x46,0xc8]
-; X64-NEXT:    vpcompressw %zmm0, (%rdi) {%k1} # encoding: [0x62,0xf2,0xfd,0x49,0x63,0x07]
+; X64-NEXT:    kmovq %rsi, %k1 # encoding: [0xc4,0xe1,0xfb,0x92,0xce]
+; X64-NEXT:    vpcompressb %zmm0, (%rdi) {%k1} # encoding: [0x62,0xf2,0x7d,0x49,0x63,0x07]
 ; X64-NEXT:    vzeroupper # encoding: [0xc5,0xf8,0x77]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  call void @llvm.x86.avx512.mask.compress.store.w.512(i8* %addr, <32 x i16> %data, i32 -1)
+  call void @llvm.x86.avx512.mask.compress.store.b.512(i8* %addr, <64 x i8> %data, i64 %mask)
   ret void
 }
 
-define void @test_mask_compress_store_b_512(i8* %addr, <64 x i8> %data, i64 %mask) {
-; X86-LABEL: test_mask_compress_store_b_512:
+declare void @llvm.x86.avx512.mask.compress.store.b.512(i8* %addr, <64 x i8> %data, i64 %mask)
+
+define void @test_compress_store_b_512(i8* %addr, <64 x i8> %data) {
+; X86-LABEL: test_compress_store_b_512:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
-; X86-NEXT:    kmovq {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf8,0x90,0x4c,0x24,0x08]
+; X86-NEXT:    kxnorq %k0, %k0, %k1 # encoding: [0xc4,0xe1,0xfc,0x46,0xc8]
 ; X86-NEXT:    vpcompressb %zmm0, (%eax) {%k1} # encoding: [0x62,0xf2,0x7d,0x49,0x63,0x00]
 ; X86-NEXT:    vzeroupper # encoding: [0xc5,0xf8,0x77]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
-; X64-LABEL: test_mask_compress_store_b_512:
+; X64-LABEL: test_compress_store_b_512:
 ; X64:       # %bb.0:
-; X64-NEXT:    kmovq %rsi, %k1 # encoding: [0xc4,0xe1,0xfb,0x92,0xce]
+; X64-NEXT:    kxnorq %k0, %k0, %k1 # encoding: [0xc4,0xe1,0xfc,0x46,0xc8]
 ; X64-NEXT:    vpcompressb %zmm0, (%rdi) {%k1} # encoding: [0x62,0xf2,0x7d,0x49,0x63,0x07]
 ; X64-NEXT:    vzeroupper # encoding: [0xc5,0xf8,0x77]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  call void @llvm.x86.avx512.mask.compress.store.b.512(i8* %addr, <64 x i8> %data, i64 %mask)
+  call void @llvm.x86.avx512.mask.compress.store.b.512(i8* %addr, <64 x i8> %data, i64 -1)
   ret void
 }
 
-declare void @llvm.x86.avx512.mask.compress.store.b.512(i8* %addr, <64 x i8> %data, i64 %mask)
-
 define <64 x i8> @test_mask_compress_b_512(<64 x i8> %data, <64 x i8> %passthru, i64 %mask) {
 ; X86-LABEL: test_mask_compress_b_512:
 ; X86:       # %bb.0:
-; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k0 # encoding: [0xc4,0xe1,0xf9,0x90,0x44,0x24,0x04]
-; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf9,0x90,0x4c,0x24,0x08]
-; X86-NEXT:    kunpckdq %k0, %k1, %k1 # encoding: [0xc4,0xe1,0xf4,0x4b,0xc8]
+; X86-NEXT:    kmovq {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf8,0x90,0x4c,0x24,0x04]
 ; X86-NEXT:    vpcompressb %zmm0, %zmm1 {%k1} # encoding: [0x62,0xf2,0x7d,0x49,0x63,0xc1]
 ; X86-NEXT:    vmovdqa64 %zmm1, %zmm0 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc1]
 ; X86-NEXT:    retl # encoding: [0xc3]
@@ -236,9 +341,7 @@ define <64 x i8> @test_mask_compress_b_512(<64 x i8> %data, <64 x i8> %passthru,
 define <64 x i8> @test_maskz_compress_b_512(<64 x i8> %data, i64 %mask) {
 ; X86-LABEL: test_maskz_compress_b_512:
 ; X86:       # %bb.0:
-; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k0 # encoding: [0xc4,0xe1,0xf9,0x90,0x44,0x24,0x04]
-; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf9,0x90,0x4c,0x24,0x08]
-; X86-NEXT:    kunpckdq %k0, %k1, %k1 # encoding: [0xc4,0xe1,0xf4,0x4b,0xc8]
+; X86-NEXT:    kmovq {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf8,0x90,0x4c,0x24,0x04]
 ; X86-NEXT:    vpcompressb %zmm0, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0xc9,0x63,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
@@ -261,25 +364,6 @@ define <64 x i8> @test_compress_b_512(<64 x i8> %data) {
 
 declare <64 x i8> @llvm.x86.avx512.mask.compress.b.512(<64 x i8> %data, <64 x i8> %src0, i64 %mask)
 
-define void @test_compress_store_b_512(i8* %addr, <64 x i8> %data) {
-; X86-LABEL: test_compress_store_b_512:
-; X86:       # %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
-; X86-NEXT:    kxnorq %k0, %k0, %k1 # encoding: [0xc4,0xe1,0xfc,0x46,0xc8]
-; X86-NEXT:    vpcompressb %zmm0, (%eax) {%k1} # encoding: [0x62,0xf2,0x7d,0x49,0x63,0x00]
-; X86-NEXT:    vzeroupper # encoding: [0xc5,0xf8,0x77]
-; X86-NEXT:    retl # encoding: [0xc3]
-;
-; X64-LABEL: test_compress_store_b_512:
-; X64:       # %bb.0:
-; X64-NEXT:    kxnorq %k0, %k0, %k1 # encoding: [0xc4,0xe1,0xfc,0x46,0xc8]
-; X64-NEXT:    vpcompressb %zmm0, (%rdi) {%k1} # encoding: [0x62,0xf2,0x7d,0x49,0x63,0x07]
-; X64-NEXT:    vzeroupper # encoding: [0xc5,0xf8,0x77]
-; X64-NEXT:    retq # encoding: [0xc3]
-  call void @llvm.x86.avx512.mask.compress.store.b.512(i8* %addr, <64 x i8> %data, i64 -1)
-  ret void
-}
-
 define <16 x i32>@test_int_x86_avx512_mask_vpshld_d_512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x3, i16 %x4) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpshld_d_512:
 ; X86:       # %bb.0:
@@ -330,21 +414,21 @@ declare <8 x i64> @llvm.x86.avx512.mask.vpshld.q.512(<8 x i64>, <8 x i64>, i32,
 define <32 x i16>@test_int_x86_avx512_mask_vpshld_w_512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x3, i32 %x4) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpshld_w_512:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpshldw $22, %zmm1, %zmm0, %zmm3 # encoding: [0x62,0xf3,0xfd,0x48,0x70,0xd9,0x16]
+; X86-NEXT:    vpshldw $6, %zmm1, %zmm0, %zmm3 # encoding: [0x62,0xf3,0xfd,0x48,0x70,0xd9,0x06]
 ; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf9,0x90,0x4c,0x24,0x04]
-; X86-NEXT:    vpshldw $22, %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x49,0x70,0xd1,0x16]
+; X86-NEXT:    vpshldw $6, %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x49,0x70,0xd1,0x06]
 ; X86-NEXT:    vpaddw %zmm3, %zmm2, %zmm0 # encoding: [0x62,0xf1,0x6d,0x48,0xfd,0xc3]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_vpshld_w_512:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpshldw $22, %zmm1, %zmm0, %zmm3 # encoding: [0x62,0xf3,0xfd,0x48,0x70,0xd9,0x16]
+; X64-NEXT:    vpshldw $6, %zmm1, %zmm0, %zmm3 # encoding: [0x62,0xf3,0xfd,0x48,0x70,0xd9,0x06]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
-; X64-NEXT:    vpshldw $22, %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x49,0x70,0xd1,0x16]
+; X64-NEXT:    vpshldw $6, %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x49,0x70,0xd1,0x06]
 ; X64-NEXT:    vpaddw %zmm3, %zmm2, %zmm0 # encoding: [0x62,0xf1,0x6d,0x48,0xfd,0xc3]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <32 x i16> @llvm.x86.avx512.mask.vpshld.w.512(<32 x i16> %x0, <32 x i16> %x1, i32 22, <32 x i16> %x3, i32 %x4)
-  %res1 = call <32 x i16> @llvm.x86.avx512.mask.vpshld.w.512(<32 x i16> %x0, <32 x i16> %x1, i32 22, <32 x i16> %x3, i32 -1)
+  %res = call <32 x i16> @llvm.x86.avx512.mask.vpshld.w.512(<32 x i16> %x0, <32 x i16> %x1, i32 6, <32 x i16> %x3, i32 %x4)
+  %res1 = call <32 x i16> @llvm.x86.avx512.mask.vpshld.w.512(<32 x i16> %x0, <32 x i16> %x1, i32 6, <32 x i16> %x3, i32 -1)
   %res2 = add <32 x i16> %res, %res1
   ret <32 x i16> %res2
 }
@@ -400,22 +484,398 @@ declare <8 x i64> @llvm.x86.avx512.mask.vpshrd.q.512(<8 x i64>, <8 x i64>, i32,
 define <32 x i16>@test_int_x86_avx512_mask_vpshrd_w_512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x3, i32 %x4) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpshrd_w_512:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpshrdw $22, %zmm1, %zmm0, %zmm3 # encoding: [0x62,0xf3,0xfd,0x48,0x72,0xd9,0x16]
+; X86-NEXT:    vpshrdw $6, %zmm1, %zmm0, %zmm3 # encoding: [0x62,0xf3,0xfd,0x48,0x72,0xd9,0x06]
 ; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf9,0x90,0x4c,0x24,0x04]
-; X86-NEXT:    vpshrdw $22, %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x49,0x72,0xd1,0x16]
+; X86-NEXT:    vpshrdw $6, %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x49,0x72,0xd1,0x06]
 ; X86-NEXT:    vpaddw %zmm3, %zmm2, %zmm0 # encoding: [0x62,0xf1,0x6d,0x48,0xfd,0xc3]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_vpshrd_w_512:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpshrdw $22, %zmm1, %zmm0, %zmm3 # encoding: [0x62,0xf3,0xfd,0x48,0x72,0xd9,0x16]
+; X64-NEXT:    vpshrdw $6, %zmm1, %zmm0, %zmm3 # encoding: [0x62,0xf3,0xfd,0x48,0x72,0xd9,0x06]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
-; X64-NEXT:    vpshrdw $22, %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x49,0x72,0xd1,0x16]
+; X64-NEXT:    vpshrdw $6, %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x49,0x72,0xd1,0x06]
 ; X64-NEXT:    vpaddw %zmm3, %zmm2, %zmm0 # encoding: [0x62,0xf1,0x6d,0x48,0xfd,0xc3]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <32 x i16> @llvm.x86.avx512.mask.vpshrd.w.512(<32 x i16> %x0, <32 x i16> %x1, i32 22, <32 x i16> %x3, i32 %x4)
-  %res1 = call <32 x i16> @llvm.x86.avx512.mask.vpshrd.w.512(<32 x i16> %x0, <32 x i16> %x1, i32 22, <32 x i16> %x3, i32 -1)
+  %res = call <32 x i16> @llvm.x86.avx512.mask.vpshrd.w.512(<32 x i16> %x0, <32 x i16> %x1, i32 6, <32 x i16> %x3, i32 %x4)
+  %res1 = call <32 x i16> @llvm.x86.avx512.mask.vpshrd.w.512(<32 x i16> %x0, <32 x i16> %x1, i32 6, <32 x i16> %x3, i32 -1)
   %res2 = add <32 x i16> %res, %res1
   ret <32 x i16> %res2
 }
 declare <32 x i16> @llvm.x86.avx512.mask.vpshrd.w.512(<32 x i16>, <32 x i16>, i32, <32 x i16>, i32)
+
+define <16 x i32>@test_int_x86_avx512_mask_vpshld_d_512_2(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x3, i16 %x4) {
+; X86-LABEL: test_int_x86_avx512_mask_vpshld_d_512_2:
+; X86:       # %bb.0:
+; X86-NEXT:    vpshldd $22, %zmm1, %zmm0, %zmm3 # encoding: [0x62,0xf3,0x7d,0x48,0x71,0xd9,0x16]
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
+; X86-NEXT:    vpshldd $22, %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf3,0x7d,0x49,0x71,0xd1,0x16]
+; X86-NEXT:    vpaddd %zmm3, %zmm2, %zmm0 # encoding: [0x62,0xf1,0x6d,0x48,0xfe,0xc3]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_mask_vpshld_d_512_2:
+; X64:       # %bb.0:
+; X64-NEXT:    vpshldd $22, %zmm1, %zmm0, %zmm3 # encoding: [0x62,0xf3,0x7d,0x48,0x71,0xd9,0x16]
+; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; X64-NEXT:    vpshldd $22, %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf3,0x7d,0x49,0x71,0xd1,0x16]
+; X64-NEXT:    vpaddd %zmm3, %zmm2, %zmm0 # encoding: [0x62,0xf1,0x6d,0x48,0xfe,0xc3]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %1 = call <16 x i32> @llvm.x86.avx512.vpshld.d.512(<16 x i32> %x0, <16 x i32> %x1, i32 22)
+  %2 = bitcast i16 %x4 to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x i32> %1, <16 x i32> %x3
+  %4 = call <16 x i32> @llvm.x86.avx512.vpshld.d.512(<16 x i32> %x0, <16 x i32> %x1, i32 22)
+  %res2 = add <16 x i32> %3, %4
+  ret <16 x i32> %res2
+}
+declare <16 x i32> @llvm.x86.avx512.vpshld.d.512(<16 x i32>, <16 x i32>, i32)
+
+define <8 x i64>@test_int_x86_avx512_mask_vpshld_q_512_2(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x3, i8 %x4) {
+; X86-LABEL: test_int_x86_avx512_mask_vpshld_q_512_2:
+; X86:       # %bb.0:
+; X86-NEXT:    vpshldq $22, %zmm1, %zmm0, %zmm3 # encoding: [0x62,0xf3,0xfd,0x48,0x71,0xd9,0x16]
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
+; X86-NEXT:    vpshldq $22, %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x49,0x71,0xd1,0x16]
+; X86-NEXT:    vpaddq %zmm3, %zmm2, %zmm0 # encoding: [0x62,0xf1,0xed,0x48,0xd4,0xc3]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_mask_vpshld_q_512_2:
+; X64:       # %bb.0:
+; X64-NEXT:    vpshldq $22, %zmm1, %zmm0, %zmm3 # encoding: [0x62,0xf3,0xfd,0x48,0x71,0xd9,0x16]
+; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; X64-NEXT:    vpshldq $22, %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x49,0x71,0xd1,0x16]
+; X64-NEXT:    vpaddq %zmm3, %zmm2, %zmm0 # encoding: [0x62,0xf1,0xed,0x48,0xd4,0xc3]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %1 = call <8 x i64> @llvm.x86.avx512.vpshld.q.512(<8 x i64> %x0, <8 x i64> %x1, i32 22)
+  %2 = bitcast i8 %x4 to <8 x i1>
+  %3 = select <8 x i1> %2, <8 x i64> %1, <8 x i64> %x3
+  %4 = call <8 x i64> @llvm.x86.avx512.vpshld.q.512(<8 x i64> %x0, <8 x i64> %x1, i32 22)
+  %res2 = add <8 x i64> %3, %4
+  ret <8 x i64> %res2
+}
+declare <8 x i64> @llvm.x86.avx512.vpshld.q.512(<8 x i64>, <8 x i64>, i32)
+
+define <32 x i16>@test_int_x86_avx512_mask_vpshld_w_512_2(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x3, i32 %x4) {
+; X86-LABEL: test_int_x86_avx512_mask_vpshld_w_512_2:
+; X86:       # %bb.0:
+; X86-NEXT:    vpshldw $6, %zmm1, %zmm0, %zmm3 # encoding: [0x62,0xf3,0xfd,0x48,0x70,0xd9,0x06]
+; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf9,0x90,0x4c,0x24,0x04]
+; X86-NEXT:    vpshldw $6, %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x49,0x70,0xd1,0x06]
+; X86-NEXT:    vpaddw %zmm3, %zmm2, %zmm0 # encoding: [0x62,0xf1,0x6d,0x48,0xfd,0xc3]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_mask_vpshld_w_512_2:
+; X64:       # %bb.0:
+; X64-NEXT:    vpshldw $6, %zmm1, %zmm0, %zmm3 # encoding: [0x62,0xf3,0xfd,0x48,0x70,0xd9,0x06]
+; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; X64-NEXT:    vpshldw $6, %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x49,0x70,0xd1,0x06]
+; X64-NEXT:    vpaddw %zmm3, %zmm2, %zmm0 # encoding: [0x62,0xf1,0x6d,0x48,0xfd,0xc3]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %1 = call <32 x i16> @llvm.x86.avx512.vpshld.w.512(<32 x i16> %x0, <32 x i16> %x1, i32 6)
+  %2 = bitcast i32 %x4 to <32 x i1>
+  %3 = select <32 x i1> %2, <32 x i16> %1, <32 x i16> %x3
+  %4 = call <32 x i16> @llvm.x86.avx512.vpshld.w.512(<32 x i16> %x0, <32 x i16> %x1, i32 6)
+  %res2 = add <32 x i16> %3, %4
+  ret <32 x i16> %res2
+}
+declare <32 x i16> @llvm.x86.avx512.vpshld.w.512(<32 x i16>, <32 x i16>, i32)
+
+define <16 x i32>@test_int_x86_avx512_mask_vpshrd_d_512_2(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x3, i16 %x4) {
+; X86-LABEL: test_int_x86_avx512_mask_vpshrd_d_512_2:
+; X86:       # %bb.0:
+; X86-NEXT:    vpshrdd $22, %zmm1, %zmm0, %zmm3 # encoding: [0x62,0xf3,0x7d,0x48,0x73,0xd9,0x16]
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
+; X86-NEXT:    vpshrdd $22, %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf3,0x7d,0x49,0x73,0xd1,0x16]
+; X86-NEXT:    vpaddd %zmm3, %zmm2, %zmm0 # encoding: [0x62,0xf1,0x6d,0x48,0xfe,0xc3]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_mask_vpshrd_d_512_2:
+; X64:       # %bb.0:
+; X64-NEXT:    vpshrdd $22, %zmm1, %zmm0, %zmm3 # encoding: [0x62,0xf3,0x7d,0x48,0x73,0xd9,0x16]
+; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; X64-NEXT:    vpshrdd $22, %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf3,0x7d,0x49,0x73,0xd1,0x16]
+; X64-NEXT:    vpaddd %zmm3, %zmm2, %zmm0 # encoding: [0x62,0xf1,0x6d,0x48,0xfe,0xc3]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %1 = call <16 x i32> @llvm.x86.avx512.vpshrd.d.512(<16 x i32> %x0, <16 x i32> %x1, i32 22)
+  %2 = bitcast i16 %x4 to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x i32> %1, <16 x i32> %x3
+  %4 = call <16 x i32> @llvm.x86.avx512.vpshrd.d.512(<16 x i32> %x0, <16 x i32> %x1, i32 22)
+  %res2 = add <16 x i32> %3, %4
+  ret <16 x i32> %res2
+}
+declare <16 x i32> @llvm.x86.avx512.vpshrd.d.512(<16 x i32>, <16 x i32>, i32)
+
+define <8 x i64>@test_int_x86_avx512_mask_vpshrd_q_512_2(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x3, i8 %x4) {
+; X86-LABEL: test_int_x86_avx512_mask_vpshrd_q_512_2:
+; X86:       # %bb.0:
+; X86-NEXT:    vpshrdq $22, %zmm1, %zmm0, %zmm3 # encoding: [0x62,0xf3,0xfd,0x48,0x73,0xd9,0x16]
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
+; X86-NEXT:    vpshrdq $22, %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x49,0x73,0xd1,0x16]
+; X86-NEXT:    vpaddq %zmm3, %zmm2, %zmm0 # encoding: [0x62,0xf1,0xed,0x48,0xd4,0xc3]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_mask_vpshrd_q_512_2:
+; X64:       # %bb.0:
+; X64-NEXT:    vpshrdq $22, %zmm1, %zmm0, %zmm3 # encoding: [0x62,0xf3,0xfd,0x48,0x73,0xd9,0x16]
+; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; X64-NEXT:    vpshrdq $22, %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x49,0x73,0xd1,0x16]
+; X64-NEXT:    vpaddq %zmm3, %zmm2, %zmm0 # encoding: [0x62,0xf1,0xed,0x48,0xd4,0xc3]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %1 = call <8 x i64> @llvm.x86.avx512.vpshrd.q.512(<8 x i64> %x0, <8 x i64> %x1, i32 22)
+  %2 = bitcast i8 %x4 to <8 x i1>
+  %3 = select <8 x i1> %2, <8 x i64> %1, <8 x i64> %x3
+  %4 = call <8 x i64> @llvm.x86.avx512.vpshrd.q.512(<8 x i64> %x0, <8 x i64> %x1, i32 22)
+  %res2 = add <8 x i64> %3, %4
+  ret <8 x i64> %res2
+}
+declare <8 x i64> @llvm.x86.avx512.vpshrd.q.512(<8 x i64>, <8 x i64>, i32)
+
+define <32 x i16>@test_int_x86_avx512_mask_vpshrd_w_512_2(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x3, i32 %x4) {
+; X86-LABEL: test_int_x86_avx512_mask_vpshrd_w_512_2:
+; X86:       # %bb.0:
+; X86-NEXT:    vpshrdw $6, %zmm1, %zmm0, %zmm3 # encoding: [0x62,0xf3,0xfd,0x48,0x72,0xd9,0x06]
+; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf9,0x90,0x4c,0x24,0x04]
+; X86-NEXT:    vpshrdw $6, %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x49,0x72,0xd1,0x06]
+; X86-NEXT:    vpaddw %zmm3, %zmm2, %zmm0 # encoding: [0x62,0xf1,0x6d,0x48,0xfd,0xc3]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_mask_vpshrd_w_512_2:
+; X64:       # %bb.0:
+; X64-NEXT:    vpshrdw $6, %zmm1, %zmm0, %zmm3 # encoding: [0x62,0xf3,0xfd,0x48,0x72,0xd9,0x06]
+; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; X64-NEXT:    vpshrdw $6, %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x49,0x72,0xd1,0x06]
+; X64-NEXT:    vpaddw %zmm3, %zmm2, %zmm0 # encoding: [0x62,0xf1,0x6d,0x48,0xfd,0xc3]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %1 = call <32 x i16> @llvm.x86.avx512.vpshrd.w.512(<32 x i16> %x0, <32 x i16> %x1, i32 6)
+  %2 = bitcast i32 %x4 to <32 x i1>
+  %3 = select <32 x i1> %2, <32 x i16> %1, <32 x i16> %x3
+  %4 = call <32 x i16> @llvm.x86.avx512.vpshrd.w.512(<32 x i16> %x0, <32 x i16> %x1, i32 6)
+  %res2 = add <32 x i16> %3, %4
+  ret <32 x i16> %res2
+}
+declare <32 x i16> @llvm.x86.avx512.vpshrd.w.512(<32 x i16>, <32 x i16>, i32)
+
+declare <16 x i32> @llvm.x86.avx512.mask.vpshrdv.d.512(<16 x i32>, <16 x i32>, <16 x i32>, i16)
+declare <16 x i32> @llvm.x86.avx512.maskz.vpshrdv.d.512(<16 x i32>, <16 x i32>, <16 x i32>, i16)
+
+define <16 x i32>@test_int_x86_avx512_mask_vpshrdv_d_512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32>* %x2p, <16 x i32> %x4, i16 %x3) {
+; X86-LABEL: test_int_x86_avx512_mask_vpshrdv_d_512:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x08]
+; X86-NEXT:    vmovdqa64 %zmm0, %zmm3 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xd8]
+; X86-NEXT:    vpshrdvd (%eax), %zmm1, %zmm3 {%k1} # encoding: [0x62,0xf2,0x75,0x49,0x73,0x18]
+; X86-NEXT:    vmovdqa64 %zmm0, %zmm4 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xe0]
+; X86-NEXT:    vpshrdvd %zmm2, %zmm1, %zmm4 # encoding: [0x62,0xf2,0x75,0x48,0x73,0xe2]
+; X86-NEXT:    vpshrdvd %zmm2, %zmm1, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0xc9,0x73,0xc2]
+; X86-NEXT:    vpaddd %zmm0, %zmm4, %zmm0 # encoding: [0x62,0xf1,0x5d,0x48,0xfe,0xc0]
+; X86-NEXT:    vpaddd %zmm0, %zmm3, %zmm0 # encoding: [0x62,0xf1,0x65,0x48,0xfe,0xc0]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_mask_vpshrdv_d_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovd %esi, %k1 # encoding: [0xc5,0xfb,0x92,0xce]
+; X64-NEXT:    vmovdqa64 %zmm0, %zmm3 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xd8]
+; X64-NEXT:    vpshrdvd (%rdi), %zmm1, %zmm3 {%k1} # encoding: [0x62,0xf2,0x75,0x49,0x73,0x1f]
+; X64-NEXT:    vmovdqa64 %zmm0, %zmm4 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xe0]
+; X64-NEXT:    vpshrdvd %zmm2, %zmm1, %zmm4 # encoding: [0x62,0xf2,0x75,0x48,0x73,0xe2]
+; X64-NEXT:    vpshrdvd %zmm2, %zmm1, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0xc9,0x73,0xc2]
+; X64-NEXT:    vpaddd %zmm0, %zmm4, %zmm0 # encoding: [0x62,0xf1,0x5d,0x48,0xfe,0xc0]
+; X64-NEXT:    vpaddd %zmm0, %zmm3, %zmm0 # encoding: [0x62,0xf1,0x65,0x48,0xfe,0xc0]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %x2 = load <16 x i32>, <16 x i32>* %x2p
+  %res = call <16 x i32> @llvm.x86.avx512.mask.vpshrdv.d.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 %x3)
+  %res1 = call <16 x i32> @llvm.x86.avx512.mask.vpshrdv.d.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x4, i16 -1)
+  %res2 = call <16 x i32> @llvm.x86.avx512.maskz.vpshrdv.d.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x4, i16  %x3)
+  %res3 = add <16 x i32> %res, %res1
+  %res4 = add <16 x i32> %res2, %res3
+  ret <16 x i32> %res4
+}
+
+declare <8 x i64> @llvm.x86.avx512.mask.vpshrdv.q.512(<8 x i64>, <8 x i64>, <8 x i64>, i8)
+declare <8 x i64> @llvm.x86.avx512.maskz.vpshrdv.q.512(<8 x i64>, <8 x i64>, <8 x i64>, i8)
+
+define <8 x i64>@test_int_x86_avx512_mask_vpshrdv_q_512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64>* %x2p, <8 x i64> %x4, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_mask_vpshrdv_q_512:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx # encoding: [0x0f,0xb6,0x4c,0x24,0x08]
+; X86-NEXT:    kmovd %ecx, %k1 # encoding: [0xc5,0xfb,0x92,0xc9]
+; X86-NEXT:    vmovdqa64 %zmm0, %zmm3 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xd8]
+; X86-NEXT:    vpshrdvq (%eax), %zmm1, %zmm3 {%k1} # encoding: [0x62,0xf2,0xf5,0x49,0x73,0x18]
+; X86-NEXT:    vmovdqa64 %zmm0, %zmm4 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xe0]
+; X86-NEXT:    vpshrdvq %zmm2, %zmm1, %zmm4 # encoding: [0x62,0xf2,0xf5,0x48,0x73,0xe2]
+; X86-NEXT:    vpshrdvq %zmm2, %zmm1, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xc9,0x73,0xc2]
+; X86-NEXT:    vpaddq %zmm0, %zmm4, %zmm0 # encoding: [0x62,0xf1,0xdd,0x48,0xd4,0xc0]
+; X86-NEXT:    vpaddq %zmm0, %zmm3, %zmm0 # encoding: [0x62,0xf1,0xe5,0x48,0xd4,0xc0]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_mask_vpshrdv_q_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovd %esi, %k1 # encoding: [0xc5,0xfb,0x92,0xce]
+; X64-NEXT:    vmovdqa64 %zmm0, %zmm3 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xd8]
+; X64-NEXT:    vpshrdvq (%rdi), %zmm1, %zmm3 {%k1} # encoding: [0x62,0xf2,0xf5,0x49,0x73,0x1f]
+; X64-NEXT:    vmovdqa64 %zmm0, %zmm4 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xe0]
+; X64-NEXT:    vpshrdvq %zmm2, %zmm1, %zmm4 # encoding: [0x62,0xf2,0xf5,0x48,0x73,0xe2]
+; X64-NEXT:    vpshrdvq %zmm2, %zmm1, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xc9,0x73,0xc2]
+; X64-NEXT:    vpaddq %zmm0, %zmm4, %zmm0 # encoding: [0x62,0xf1,0xdd,0x48,0xd4,0xc0]
+; X64-NEXT:    vpaddq %zmm0, %zmm3, %zmm0 # encoding: [0x62,0xf1,0xe5,0x48,0xd4,0xc0]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %x2 = load <8 x i64>, <8 x i64>* %x2p
+  %res = call <8 x i64> @llvm.x86.avx512.mask.vpshrdv.q.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 %x3)
+  %res1 = call <8 x i64> @llvm.x86.avx512.mask.vpshrdv.q.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x4, i8 -1)
+  %res2 = call <8 x i64> @llvm.x86.avx512.maskz.vpshrdv.q.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x4, i8  %x3)
+  %res3 = add <8 x i64> %res, %res1
+  %res4 = add <8 x i64> %res2, %res3
+  ret <8 x i64> %res4
+}
+
+declare <32 x i16> @llvm.x86.avx512.mask.vpshrdv.w.512(<32 x i16>, <32 x i16>, <32 x i16>, i32)
+declare <32 x i16> @llvm.x86.avx512.maskz.vpshrdv.w.512(<32 x i16>, <32 x i16>, <32 x i16>, i32)
+
+define <32 x i16>@test_int_x86_avx512_mask_vpshrdv_w_512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16>* %x2p, <32 x i16> %x4, i32 %x3) {
+; X86-LABEL: test_int_x86_avx512_mask_vpshrdv_w_512:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf9,0x90,0x4c,0x24,0x08]
+; X86-NEXT:    vmovdqa64 %zmm0, %zmm3 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xd8]
+; X86-NEXT:    vpshrdvw (%eax), %zmm1, %zmm3 {%k1} # encoding: [0x62,0xf2,0xf5,0x49,0x72,0x18]
+; X86-NEXT:    vmovdqa64 %zmm0, %zmm4 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xe0]
+; X86-NEXT:    vpshrdvw %zmm2, %zmm1, %zmm4 # encoding: [0x62,0xf2,0xf5,0x48,0x72,0xe2]
+; X86-NEXT:    vpshrdvw %zmm2, %zmm1, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xc9,0x72,0xc2]
+; X86-NEXT:    vpaddw %zmm0, %zmm4, %zmm0 # encoding: [0x62,0xf1,0x5d,0x48,0xfd,0xc0]
+; X86-NEXT:    vpaddw %zmm0, %zmm3, %zmm0 # encoding: [0x62,0xf1,0x65,0x48,0xfd,0xc0]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_mask_vpshrdv_w_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovd %esi, %k1 # encoding: [0xc5,0xfb,0x92,0xce]
+; X64-NEXT:    vmovdqa64 %zmm0, %zmm3 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xd8]
+; X64-NEXT:    vpshrdvw (%rdi), %zmm1, %zmm3 {%k1} # encoding: [0x62,0xf2,0xf5,0x49,0x72,0x1f]
+; X64-NEXT:    vmovdqa64 %zmm0, %zmm4 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xe0]
+; X64-NEXT:    vpshrdvw %zmm2, %zmm1, %zmm4 # encoding: [0x62,0xf2,0xf5,0x48,0x72,0xe2]
+; X64-NEXT:    vpshrdvw %zmm2, %zmm1, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xc9,0x72,0xc2]
+; X64-NEXT:    vpaddw %zmm0, %zmm4, %zmm0 # encoding: [0x62,0xf1,0x5d,0x48,0xfd,0xc0]
+; X64-NEXT:    vpaddw %zmm0, %zmm3, %zmm0 # encoding: [0x62,0xf1,0x65,0x48,0xfd,0xc0]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %x2 = load <32 x i16>, <32 x i16>* %x2p
+  %res = call <32 x i16> @llvm.x86.avx512.mask.vpshrdv.w.512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3)
+  %res1 = call <32 x i16> @llvm.x86.avx512.mask.vpshrdv.w.512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x4, i32 -1)
+  %res2 = call <32 x i16> @llvm.x86.avx512.maskz.vpshrdv.w.512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x4, i32  %x3)
+  %res3 = add <32 x i16> %res, %res1
+  %res4 = add <32 x i16> %res2, %res3
+  ret <32 x i16> %res4
+}
+
+declare <16 x i32> @llvm.x86.avx512.mask.vpshldv.d.512(<16 x i32>, <16 x i32>, <16 x i32>, i16)
+declare <16 x i32> @llvm.x86.avx512.maskz.vpshldv.d.512(<16 x i32>, <16 x i32>, <16 x i32>, i16)
+
+define <16 x i32>@test_int_x86_avx512_mask_vpshldv_d_512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32>* %x2p, <16 x i32> %x4, i16 %x3) {
+; X86-LABEL: test_int_x86_avx512_mask_vpshldv_d_512:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x08]
+; X86-NEXT:    vmovdqa64 %zmm0, %zmm3 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xd8]
+; X86-NEXT:    vpshldvd (%eax), %zmm1, %zmm3 {%k1} # encoding: [0x62,0xf2,0x75,0x49,0x71,0x18]
+; X86-NEXT:    vmovdqa64 %zmm0, %zmm4 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xe0]
+; X86-NEXT:    vpshldvd %zmm2, %zmm1, %zmm4 # encoding: [0x62,0xf2,0x75,0x48,0x71,0xe2]
+; X86-NEXT:    vpshldvd %zmm2, %zmm1, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0xc9,0x71,0xc2]
+; X86-NEXT:    vpaddd %zmm0, %zmm4, %zmm0 # encoding: [0x62,0xf1,0x5d,0x48,0xfe,0xc0]
+; X86-NEXT:    vpaddd %zmm0, %zmm3, %zmm0 # encoding: [0x62,0xf1,0x65,0x48,0xfe,0xc0]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_mask_vpshldv_d_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovd %esi, %k1 # encoding: [0xc5,0xfb,0x92,0xce]
+; X64-NEXT:    vmovdqa64 %zmm0, %zmm3 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xd8]
+; X64-NEXT:    vpshldvd (%rdi), %zmm1, %zmm3 {%k1} # encoding: [0x62,0xf2,0x75,0x49,0x71,0x1f]
+; X64-NEXT:    vmovdqa64 %zmm0, %zmm4 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xe0]
+; X64-NEXT:    vpshldvd %zmm2, %zmm1, %zmm4 # encoding: [0x62,0xf2,0x75,0x48,0x71,0xe2]
+; X64-NEXT:    vpshldvd %zmm2, %zmm1, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0xc9,0x71,0xc2]
+; X64-NEXT:    vpaddd %zmm0, %zmm4, %zmm0 # encoding: [0x62,0xf1,0x5d,0x48,0xfe,0xc0]
+; X64-NEXT:    vpaddd %zmm0, %zmm3, %zmm0 # encoding: [0x62,0xf1,0x65,0x48,0xfe,0xc0]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %x2 = load <16 x i32>, <16 x i32>* %x2p
+  %res = call <16 x i32> @llvm.x86.avx512.mask.vpshldv.d.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 %x3)
+  %res1 = call <16 x i32> @llvm.x86.avx512.mask.vpshldv.d.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x4, i16 -1)
+  %res2 = call <16 x i32> @llvm.x86.avx512.maskz.vpshldv.d.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x4, i16  %x3)
+  %res3 = add <16 x i32> %res, %res1
+  %res4 = add <16 x i32> %res2, %res3
+  ret <16 x i32> %res4
+}
+
+declare <8 x i64> @llvm.x86.avx512.mask.vpshldv.q.512(<8 x i64>, <8 x i64>, <8 x i64>, i8)
+declare <8 x i64> @llvm.x86.avx512.maskz.vpshldv.q.512(<8 x i64>, <8 x i64>, <8 x i64>, i8)
+
+define <8 x i64>@test_int_x86_avx512_mask_vpshldv_q_512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64>* %x2p, <8 x i64> %x4, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_mask_vpshldv_q_512:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx # encoding: [0x0f,0xb6,0x4c,0x24,0x08]
+; X86-NEXT:    kmovd %ecx, %k1 # encoding: [0xc5,0xfb,0x92,0xc9]
+; X86-NEXT:    vmovdqa64 %zmm0, %zmm3 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xd8]
+; X86-NEXT:    vpshldvq (%eax), %zmm1, %zmm3 {%k1} # encoding: [0x62,0xf2,0xf5,0x49,0x71,0x18]
+; X86-NEXT:    vmovdqa64 %zmm0, %zmm4 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xe0]
+; X86-NEXT:    vpshldvq %zmm2, %zmm1, %zmm4 # encoding: [0x62,0xf2,0xf5,0x48,0x71,0xe2]
+; X86-NEXT:    vpshldvq %zmm2, %zmm1, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xc9,0x71,0xc2]
+; X86-NEXT:    vpaddq %zmm0, %zmm4, %zmm0 # encoding: [0x62,0xf1,0xdd,0x48,0xd4,0xc0]
+; X86-NEXT:    vpaddq %zmm0, %zmm3, %zmm0 # encoding: [0x62,0xf1,0xe5,0x48,0xd4,0xc0]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_mask_vpshldv_q_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovd %esi, %k1 # encoding: [0xc5,0xfb,0x92,0xce]
+; X64-NEXT:    vmovdqa64 %zmm0, %zmm3 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xd8]
+; X64-NEXT:    vpshldvq (%rdi), %zmm1, %zmm3 {%k1} # encoding: [0x62,0xf2,0xf5,0x49,0x71,0x1f]
+; X64-NEXT:    vmovdqa64 %zmm0, %zmm4 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xe0]
+; X64-NEXT:    vpshldvq %zmm2, %zmm1, %zmm4 # encoding: [0x62,0xf2,0xf5,0x48,0x71,0xe2]
+; X64-NEXT:    vpshldvq %zmm2, %zmm1, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xc9,0x71,0xc2]
+; X64-NEXT:    vpaddq %zmm0, %zmm4, %zmm0 # encoding: [0x62,0xf1,0xdd,0x48,0xd4,0xc0]
+; X64-NEXT:    vpaddq %zmm0, %zmm3, %zmm0 # encoding: [0x62,0xf1,0xe5,0x48,0xd4,0xc0]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %x2 = load <8 x i64>, <8 x i64>* %x2p
+  %res = call <8 x i64> @llvm.x86.avx512.mask.vpshldv.q.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 %x3)
+  %res1 = call <8 x i64> @llvm.x86.avx512.mask.vpshldv.q.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x4, i8 -1)
+  %res2 = call <8 x i64> @llvm.x86.avx512.maskz.vpshldv.q.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x4, i8  %x3)
+  %res3 = add <8 x i64> %res, %res1
+  %res4 = add <8 x i64> %res2, %res3
+  ret <8 x i64> %res4
+}
+
+declare <32 x i16> @llvm.x86.avx512.mask.vpshldv.w.512(<32 x i16>, <32 x i16>, <32 x i16>, i32)
+declare <32 x i16> @llvm.x86.avx512.maskz.vpshldv.w.512(<32 x i16>, <32 x i16>, <32 x i16>, i32)
+
+define <32 x i16>@test_int_x86_avx512_mask_vpshldv_w_512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16>* %x2p, <32 x i16> %x4, i32 %x3) {
+; X86-LABEL: test_int_x86_avx512_mask_vpshldv_w_512:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf9,0x90,0x4c,0x24,0x08]
+; X86-NEXT:    vmovdqa64 %zmm0, %zmm3 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xd8]
+; X86-NEXT:    vpshldvw (%eax), %zmm1, %zmm3 {%k1} # encoding: [0x62,0xf2,0xf5,0x49,0x70,0x18]
+; X86-NEXT:    vmovdqa64 %zmm0, %zmm4 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xe0]
+; X86-NEXT:    vpshldvw %zmm2, %zmm1, %zmm4 # encoding: [0x62,0xf2,0xf5,0x48,0x70,0xe2]
+; X86-NEXT:    vpshldvw %zmm2, %zmm1, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xc9,0x70,0xc2]
+; X86-NEXT:    vpaddw %zmm0, %zmm4, %zmm0 # encoding: [0x62,0xf1,0x5d,0x48,0xfd,0xc0]
+; X86-NEXT:    vpaddw %zmm0, %zmm3, %zmm0 # encoding: [0x62,0xf1,0x65,0x48,0xfd,0xc0]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_mask_vpshldv_w_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovd %esi, %k1 # encoding: [0xc5,0xfb,0x92,0xce]
+; X64-NEXT:    vmovdqa64 %zmm0, %zmm3 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xd8]
+; X64-NEXT:    vpshldvw (%rdi), %zmm1, %zmm3 {%k1} # encoding: [0x62,0xf2,0xf5,0x49,0x70,0x1f]
+; X64-NEXT:    vmovdqa64 %zmm0, %zmm4 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xe0]
+; X64-NEXT:    vpshldvw %zmm2, %zmm1, %zmm4 # encoding: [0x62,0xf2,0xf5,0x48,0x70,0xe2]
+; X64-NEXT:    vpshldvw %zmm2, %zmm1, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xc9,0x70,0xc2]
+; X64-NEXT:    vpaddw %zmm0, %zmm4, %zmm0 # encoding: [0x62,0xf1,0x5d,0x48,0xfd,0xc0]
+; X64-NEXT:    vpaddw %zmm0, %zmm3, %zmm0 # encoding: [0x62,0xf1,0x65,0x48,0xfd,0xc0]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %x2 = load <32 x i16>, <32 x i16>* %x2p
+  %res = call <32 x i16> @llvm.x86.avx512.mask.vpshldv.w.512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3)
+  %res1 = call <32 x i16> @llvm.x86.avx512.mask.vpshldv.w.512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x4, i32 -1)
+  %res2 = call <32 x i16> @llvm.x86.avx512.maskz.vpshldv.w.512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x4, i32  %x3)
+  %res3 = add <32 x i16> %res, %res1
+  %res4 = add <32 x i16> %res2, %res3
+  ret <32 x i16> %res4
+}
diff --git a/test/CodeGen/X86/avx512vbmi2-intrinsics.ll b/test/CodeGen/X86/avx512vbmi2-intrinsics.ll
index fd98c16dab6fcb8af8601c2273ae91c0fada20cd..a16b91b6218e379b471a681b6f14ce1ab827c751 100644
--- a/test/CodeGen/X86/avx512vbmi2-intrinsics.ll
+++ b/test/CodeGen/X86/avx512vbmi2-intrinsics.ll
@@ -2,12 +2,68 @@
 ; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+avx512vbmi2 --show-mc-encoding | FileCheck %s --check-prefixes=CHECK,X86
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512vbmi2 --show-mc-encoding | FileCheck %s --check-prefixes=CHECK,X64
 
+define <32 x i16> @test_mask_expand_load_w_512(i8* %addr, <32 x i16> %data, i32 %mask) {
+; X86-LABEL: test_mask_expand_load_w_512:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf9,0x90,0x4c,0x24,0x08]
+; X86-NEXT:    vpexpandw (%eax), %zmm0 {%k1} # encoding: [0x62,0xf2,0xfd,0x49,0x62,0x00]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_mask_expand_load_w_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovd %esi, %k1 # encoding: [0xc5,0xfb,0x92,0xce]
+; X64-NEXT:    vpexpandw (%rdi), %zmm0 {%k1} # encoding: [0x62,0xf2,0xfd,0x49,0x62,0x07]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %1 = bitcast i8* %addr to i16*
+  %2 = bitcast i32 %mask to <32 x i1>
+  %3 = call <32 x i16> @llvm.masked.expandload.v32i16(i16* %1, <32 x i1> %2, <32 x i16> %data)
+  ret <32 x i16> %3
+}
+
+define <32 x i16> @test_maskz_expand_load_w_512(i8* %addr, i32 %mask) {
+; X86-LABEL: test_maskz_expand_load_w_512:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf9,0x90,0x4c,0x24,0x08]
+; X86-NEXT:    vpexpandw (%eax), %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xc9,0x62,0x00]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_maskz_expand_load_w_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovd %esi, %k1 # encoding: [0xc5,0xfb,0x92,0xce]
+; X64-NEXT:    vpexpandw (%rdi), %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xc9,0x62,0x07]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %1 = bitcast i8* %addr to i16*
+  %2 = bitcast i32 %mask to <32 x i1>
+  %3 = call <32 x i16> @llvm.masked.expandload.v32i16(i16* %1, <32 x i1> %2, <32 x i16> zeroinitializer)
+  ret <32 x i16> %3
+}
+
+define <32 x i16> @test_expand_load_w_512(i8* %addr, <32 x i16> %data) {
+; X86-LABEL: test_expand_load_w_512:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-NEXT:    kxnord %k0, %k0, %k1 # encoding: [0xc4,0xe1,0xfd,0x46,0xc8]
+; X86-NEXT:    vpexpandw (%eax), %zmm0 {%k1} # encoding: [0x62,0xf2,0xfd,0x49,0x62,0x00]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_expand_load_w_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kxnord %k0, %k0, %k1 # encoding: [0xc4,0xe1,0xfd,0x46,0xc8]
+; X64-NEXT:    vpexpandw (%rdi), %zmm0 {%k1} # encoding: [0x62,0xf2,0xfd,0x49,0x62,0x07]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %1 = bitcast i8* %addr to i16*
+  %2 = call <32 x i16> @llvm.masked.expandload.v32i16(i16* %1, <32 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <32 x i16> %data)
+  ret <32 x i16> %2
+}
+
 define <32 x i16> @test_expand_w_512(<32 x i16> %data) {
 ; CHECK-LABEL: test_expand_w_512:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
-  %res = call <32 x i16> @llvm.x86.avx512.mask.expand.w.512(<32 x i16> %data, <32 x i16> undef, i32 -1)
-  ret <32 x i16> %res
+  %1 = call <32 x i16> @llvm.x86.avx512.mask.expand.v32i16(<32 x i16> %data, <32 x i16> undef, <32 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>)
+  ret <32 x i16> %1
 }
 
 define <32 x i16> @test_mask_expand_w_512(<32 x i16> %data, <32 x i16> %passthru, i32 %mask) {
@@ -24,8 +80,9 @@ define <32 x i16> @test_mask_expand_w_512(<32 x i16> %data, <32 x i16> %passthru
 ; X64-NEXT:    vpexpandw %zmm0, %zmm1 {%k1} # encoding: [0x62,0xf2,0xfd,0x49,0x62,0xc8]
 ; X64-NEXT:    vmovdqa64 %zmm1, %zmm0 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc1]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <32 x i16> @llvm.x86.avx512.mask.expand.w.512(<32 x i16> %data, <32 x i16> %passthru, i32 %mask)
-  ret <32 x i16> %res
+  %1 = bitcast i32 %mask to <32 x i1>
+  %2 = call <32 x i16> @llvm.x86.avx512.mask.expand.v32i16(<32 x i16> %data, <32 x i16> %passthru, <32 x i1> %1)
+  ret <32 x i16> %2
 }
 
 define <32 x i16> @test_maskz_expand_w_512(<32 x i16> %data, i32 %mask) {
@@ -40,26 +97,76 @@ define <32 x i16> @test_maskz_expand_w_512(<32 x i16> %data, i32 %mask) {
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpexpandw %zmm0, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xc9,0x62,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <32 x i16> @llvm.x86.avx512.mask.expand.w.512(<32 x i16> %data, <32 x i16> zeroinitializer, i32 %mask)
-  ret <32 x i16> %res
+  %1 = bitcast i32 %mask to <32 x i1>
+  %2 = call <32 x i16> @llvm.x86.avx512.mask.expand.v32i16(<32 x i16> %data, <32 x i16> zeroinitializer, <32 x i1> %1)
+  ret <32 x i16> %2
+}
+
+define <64 x i8> @test_mask_expand_load_b_512(i8* %addr, <64 x i8> %data, i64 %mask) {
+; X86-LABEL: test_mask_expand_load_b_512:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-NEXT:    kmovq {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf8,0x90,0x4c,0x24,0x08]
+; X86-NEXT:    vpexpandb (%eax), %zmm0 {%k1} # encoding: [0x62,0xf2,0x7d,0x49,0x62,0x00]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_mask_expand_load_b_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovq %rsi, %k1 # encoding: [0xc4,0xe1,0xfb,0x92,0xce]
+; X64-NEXT:    vpexpandb (%rdi), %zmm0 {%k1} # encoding: [0x62,0xf2,0x7d,0x49,0x62,0x07]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %1 = bitcast i64 %mask to <64 x i1>
+  %2 = call <64 x i8> @llvm.masked.expandload.v64i8(i8* %addr, <64 x i1> %1, <64 x i8> %data)
+  ret <64 x i8> %2
+}
+
+define <64 x i8> @test_maskz_expand_load_b_512(i8* %addr, i64 %mask) {
+; X86-LABEL: test_maskz_expand_load_b_512:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-NEXT:    kmovq {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf8,0x90,0x4c,0x24,0x08]
+; X86-NEXT:    vpexpandb (%eax), %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0xc9,0x62,0x00]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_maskz_expand_load_b_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovq %rsi, %k1 # encoding: [0xc4,0xe1,0xfb,0x92,0xce]
+; X64-NEXT:    vpexpandb (%rdi), %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0xc9,0x62,0x07]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %1 = bitcast i64 %mask to <64 x i1>
+  %2 = call <64 x i8> @llvm.masked.expandload.v64i8(i8* %addr, <64 x i1> %1, <64 x i8> zeroinitializer)
+  ret <64 x i8> %2
 }
 
-declare <32 x i16> @llvm.x86.avx512.mask.expand.w.512(<32 x i16> %data, <32 x i16> %src0, i32 %mask)
+define <64 x i8> @test_expand_load_b_512(i8* %addr, <64 x i8> %data) {
+; X86-LABEL: test_expand_load_b_512:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-NEXT:    kxnorq %k0, %k0, %k1 # encoding: [0xc4,0xe1,0xfc,0x46,0xc8]
+; X86-NEXT:    vpexpandb (%eax), %zmm0 {%k1} # encoding: [0x62,0xf2,0x7d,0x49,0x62,0x00]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_expand_load_b_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kxnorq %k0, %k0, %k1 # encoding: [0xc4,0xe1,0xfc,0x46,0xc8]
+; X64-NEXT:    vpexpandb (%rdi), %zmm0 {%k1} # encoding: [0x62,0xf2,0x7d,0x49,0x62,0x07]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %1 = call <64 x i8> @llvm.masked.expandload.v64i8(i8* %addr, <64 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <64 x i8> %data)
+  ret <64 x i8> %1
+}
 
 define <64 x i8> @test_expand_b_512(<64 x i8> %data) {
 ; CHECK-LABEL: test_expand_b_512:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
-  %res = call <64 x i8> @llvm.x86.avx512.mask.expand.b.512(<64 x i8> %data, <64 x i8> undef, i64 -1)
-  ret <64 x i8> %res
+  %1 = call <64 x i8> @llvm.x86.avx512.mask.expand.v64i8(<64 x i8> %data, <64 x i8> undef, <64 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>)
+  ret <64 x i8> %1
 }
 
 define <64 x i8> @test_mask_expand_b_512(<64 x i8> %data, <64 x i8> %passthru, i64 %mask) {
 ; X86-LABEL: test_mask_expand_b_512:
 ; X86:       # %bb.0:
-; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k0 # encoding: [0xc4,0xe1,0xf9,0x90,0x44,0x24,0x04]
-; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf9,0x90,0x4c,0x24,0x08]
-; X86-NEXT:    kunpckdq %k0, %k1, %k1 # encoding: [0xc4,0xe1,0xf4,0x4b,0xc8]
+; X86-NEXT:    kmovq {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf8,0x90,0x4c,0x24,0x04]
 ; X86-NEXT:    vpexpandb %zmm0, %zmm1 {%k1} # encoding: [0x62,0xf2,0x7d,0x49,0x62,0xc8]
 ; X86-NEXT:    vmovdqa64 %zmm1, %zmm0 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc1]
 ; X86-NEXT:    retl # encoding: [0xc3]
@@ -70,16 +177,15 @@ define <64 x i8> @test_mask_expand_b_512(<64 x i8> %data, <64 x i8> %passthru, i
 ; X64-NEXT:    vpexpandb %zmm0, %zmm1 {%k1} # encoding: [0x62,0xf2,0x7d,0x49,0x62,0xc8]
 ; X64-NEXT:    vmovdqa64 %zmm1, %zmm0 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc1]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <64 x i8> @llvm.x86.avx512.mask.expand.b.512(<64 x i8> %data, <64 x i8> %passthru, i64 %mask)
-  ret <64 x i8> %res
+  %1 = bitcast i64 %mask to <64 x i1>
+  %2 = call <64 x i8> @llvm.x86.avx512.mask.expand.v64i8(<64 x i8> %data, <64 x i8> %passthru, <64 x i1> %1)
+  ret <64 x i8> %2
 }
 
 define <64 x i8> @test_maskz_expand_b_512(<64 x i8> %data, i64 %mask) {
 ; X86-LABEL: test_maskz_expand_b_512:
 ; X86:       # %bb.0:
-; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k0 # encoding: [0xc4,0xe1,0xf9,0x90,0x44,0x24,0x04]
-; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf9,0x90,0x4c,0x24,0x08]
-; X86-NEXT:    kunpckdq %k0, %k1, %k1 # encoding: [0xc4,0xe1,0xf4,0x4b,0xc8]
+; X86-NEXT:    kmovq {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf8,0x90,0x4c,0x24,0x04]
 ; X86-NEXT:    vpexpandb %zmm0, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0xc9,0x62,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
@@ -88,13 +194,180 @@ define <64 x i8> @test_maskz_expand_b_512(<64 x i8> %data, i64 %mask) {
 ; X64-NEXT:    kmovq %rdi, %k1 # encoding: [0xc4,0xe1,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpexpandb %zmm0, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0xc9,0x62,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <64 x i8> @llvm.x86.avx512.mask.expand.b.512(<64 x i8> %data, <64 x i8> zeroinitializer, i64 %mask)
-  ret <64 x i8> %res
+  %1 = bitcast i64 %mask to <64 x i1>
+  %2 = call <64 x i8> @llvm.x86.avx512.mask.expand.v64i8(<64 x i8> %data, <64 x i8> zeroinitializer, <64 x i1> %1)
+  ret <64 x i8> %2
+}
+
+define void @test_mask_compress_store_w_512(i8* %addr, <32 x i16> %data, i32 %mask) {
+; X86-LABEL: test_mask_compress_store_w_512:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf9,0x90,0x4c,0x24,0x08]
+; X86-NEXT:    vpcompressw %zmm0, (%eax) {%k1} # encoding: [0x62,0xf2,0xfd,0x49,0x63,0x00]
+; X86-NEXT:    vzeroupper # encoding: [0xc5,0xf8,0x77]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_mask_compress_store_w_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovd %esi, %k1 # encoding: [0xc5,0xfb,0x92,0xce]
+; X64-NEXT:    vpcompressw %zmm0, (%rdi) {%k1} # encoding: [0x62,0xf2,0xfd,0x49,0x63,0x07]
+; X64-NEXT:    vzeroupper # encoding: [0xc5,0xf8,0x77]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %1 = bitcast i8* %addr to i16*
+  %2 = bitcast i32 %mask to <32 x i1>
+  call void @llvm.masked.compressstore.v32i16(<32 x i16> %data, i16* %1, <32 x i1> %2)
+  ret void
+}
+
+define void @test_compress_store_w_512(i8* %addr, <32 x i16> %data) {
+; X86-LABEL: test_compress_store_w_512:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-NEXT:    kxnord %k0, %k0, %k1 # encoding: [0xc4,0xe1,0xfd,0x46,0xc8]
+; X86-NEXT:    vpcompressw %zmm0, (%eax) {%k1} # encoding: [0x62,0xf2,0xfd,0x49,0x63,0x00]
+; X86-NEXT:    vzeroupper # encoding: [0xc5,0xf8,0x77]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_compress_store_w_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kxnord %k0, %k0, %k1 # encoding: [0xc4,0xe1,0xfd,0x46,0xc8]
+; X64-NEXT:    vpcompressw %zmm0, (%rdi) {%k1} # encoding: [0x62,0xf2,0xfd,0x49,0x63,0x07]
+; X64-NEXT:    vzeroupper # encoding: [0xc5,0xf8,0x77]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %1 = bitcast i8* %addr to i16*
+  call void @llvm.masked.compressstore.v32i16(<32 x i16> %data, i16* %1, <32 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>)
+  ret void
+}
+
+define <32 x i16> @test_mask_compress_w_512(<32 x i16> %data, <32 x i16> %passthru, i32 %mask) {
+; X86-LABEL: test_mask_compress_w_512:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf9,0x90,0x4c,0x24,0x04]
+; X86-NEXT:    vpcompressw %zmm0, %zmm1 {%k1} # encoding: [0x62,0xf2,0xfd,0x49,0x63,0xc1]
+; X86-NEXT:    vmovdqa64 %zmm1, %zmm0 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc1]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_mask_compress_w_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; X64-NEXT:    vpcompressw %zmm0, %zmm1 {%k1} # encoding: [0x62,0xf2,0xfd,0x49,0x63,0xc1]
+; X64-NEXT:    vmovdqa64 %zmm1, %zmm0 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc1]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %1 = bitcast i32 %mask to <32 x i1>
+  %2 = call <32 x i16> @llvm.x86.avx512.mask.compress.v32i16(<32 x i16> %data, <32 x i16> %passthru, <32 x i1> %1)
+  ret <32 x i16> %2
+}
+
+define <32 x i16> @test_maskz_compress_w_512(<32 x i16> %data, i32 %mask) {
+; X86-LABEL: test_maskz_compress_w_512:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf9,0x90,0x4c,0x24,0x04]
+; X86-NEXT:    vpcompressw %zmm0, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xc9,0x63,0xc0]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_maskz_compress_w_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; X64-NEXT:    vpcompressw %zmm0, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xc9,0x63,0xc0]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %1 = bitcast i32 %mask to <32 x i1>
+  %2 = call <32 x i16> @llvm.x86.avx512.mask.compress.v32i16(<32 x i16> %data, <32 x i16> zeroinitializer, <32 x i1> %1)
+  ret <32 x i16> %2
+}
+
+define <32 x i16> @test_compress_w_512(<32 x i16> %data) {
+; CHECK-LABEL: test_compress_w_512:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %1 = call <32 x i16> @llvm.x86.avx512.mask.compress.v32i16(<32 x i16> %data, <32 x i16> undef, <32 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>)
+  ret <32 x i16> %1
+}
+
+define void @test_mask_compress_store_b_512(i8* %addr, <64 x i8> %data, i64 %mask) {
+; X86-LABEL: test_mask_compress_store_b_512:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-NEXT:    kmovq {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf8,0x90,0x4c,0x24,0x08]
+; X86-NEXT:    vpcompressb %zmm0, (%eax) {%k1} # encoding: [0x62,0xf2,0x7d,0x49,0x63,0x00]
+; X86-NEXT:    vzeroupper # encoding: [0xc5,0xf8,0x77]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_mask_compress_store_b_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovq %rsi, %k1 # encoding: [0xc4,0xe1,0xfb,0x92,0xce]
+; X64-NEXT:    vpcompressb %zmm0, (%rdi) {%k1} # encoding: [0x62,0xf2,0x7d,0x49,0x63,0x07]
+; X64-NEXT:    vzeroupper # encoding: [0xc5,0xf8,0x77]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %1 = bitcast i64 %mask to <64 x i1>
+  call void @llvm.masked.compressstore.v64i8(<64 x i8> %data, i8* %addr, <64 x i1> %1)
+  ret void
+}
+
+define void @test_compress_store_b_512(i8* %addr, <64 x i8> %data) {
+; X86-LABEL: test_compress_store_b_512:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-NEXT:    kxnorq %k0, %k0, %k1 # encoding: [0xc4,0xe1,0xfc,0x46,0xc8]
+; X86-NEXT:    vpcompressb %zmm0, (%eax) {%k1} # encoding: [0x62,0xf2,0x7d,0x49,0x63,0x00]
+; X86-NEXT:    vzeroupper # encoding: [0xc5,0xf8,0x77]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_compress_store_b_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kxnorq %k0, %k0, %k1 # encoding: [0xc4,0xe1,0xfc,0x46,0xc8]
+; X64-NEXT:    vpcompressb %zmm0, (%rdi) {%k1} # encoding: [0x62,0xf2,0x7d,0x49,0x63,0x07]
+; X64-NEXT:    vzeroupper # encoding: [0xc5,0xf8,0x77]
+; X64-NEXT:    retq # encoding: [0xc3]
+  call void @llvm.masked.compressstore.v64i8(<64 x i8> %data, i8* %addr, <64 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>)
+  ret void
 }
 
-declare <64 x i8> @llvm.x86.avx512.mask.expand.b.512(<64 x i8> %data, <64 x i8> %src0, i64 %mask)
+define <64 x i8> @test_mask_compress_b_512(<64 x i8> %data, <64 x i8> %passthru, i64 %mask) {
+; X86-LABEL: test_mask_compress_b_512:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovq {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf8,0x90,0x4c,0x24,0x04]
+; X86-NEXT:    vpcompressb %zmm0, %zmm1 {%k1} # encoding: [0x62,0xf2,0x7d,0x49,0x63,0xc1]
+; X86-NEXT:    vmovdqa64 %zmm1, %zmm0 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc1]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_mask_compress_b_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovq %rdi, %k1 # encoding: [0xc4,0xe1,0xfb,0x92,0xcf]
+; X64-NEXT:    vpcompressb %zmm0, %zmm1 {%k1} # encoding: [0x62,0xf2,0x7d,0x49,0x63,0xc1]
+; X64-NEXT:    vmovdqa64 %zmm1, %zmm0 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc1]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %1 = bitcast i64 %mask to <64 x i1>
+  %2 = call <64 x i8> @llvm.x86.avx512.mask.compress.v64i8(<64 x i8> %data, <64 x i8> %passthru, <64 x i1> %1)
+  ret <64 x i8> %2
+}
 
-define <16 x i32>@test_int_x86_avx512_mask_vpshld_d_512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x3, i16 %x4) {
+define <64 x i8> @test_maskz_compress_b_512(<64 x i8> %data, i64 %mask) {
+; X86-LABEL: test_maskz_compress_b_512:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovq {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf8,0x90,0x4c,0x24,0x04]
+; X86-NEXT:    vpcompressb %zmm0, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0xc9,0x63,0xc0]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_maskz_compress_b_512:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovq %rdi, %k1 # encoding: [0xc4,0xe1,0xfb,0x92,0xcf]
+; X64-NEXT:    vpcompressb %zmm0, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0xc9,0x63,0xc0]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %1 = bitcast i64 %mask to <64 x i1>
+  %2 = call <64 x i8> @llvm.x86.avx512.mask.compress.v64i8(<64 x i8> %data, <64 x i8> zeroinitializer, <64 x i1> %1)
+  ret <64 x i8> %2
+}
+
+define <64 x i8> @test_compress_b_512(<64 x i8> %data) {
+; CHECK-LABEL: test_compress_b_512:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %1 = call <64 x i8> @llvm.x86.avx512.mask.compress.v64i8(<64 x i8> %data, <64 x i8> undef, <64 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>)
+  ret <64 x i8> %1
+}
+
+define <16 x i32> @test_int_x86_avx512_mask_vpshld_d_512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x3, i16 %x4) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpshld_d_512:
 ; X86:       # %bb.0:
 ; X86-NEXT:    vpshldd $22, %zmm1, %zmm0, %zmm3 # encoding: [0x62,0xf3,0x7d,0x48,0x71,0xd9,0x16]
@@ -110,16 +383,15 @@ define <16 x i32>@test_int_x86_avx512_mask_vpshld_d_512(<16 x i32> %x0, <16 x i3
 ; X64-NEXT:    vpshldd $22, %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf3,0x7d,0x49,0x71,0xd1,0x16]
 ; X64-NEXT:    vpaddd %zmm3, %zmm2, %zmm0 # encoding: [0x62,0xf1,0x6d,0x48,0xfe,0xc3]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %1 = call <16 x i32> @llvm.x86.avx512.vpshld.d.512(<16 x i32> %x0, <16 x i32> %x1, i32 22)
+  %1 = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> <i32 22, i32 22, i32 22, i32 22, i32 22, i32 22, i32 22, i32 22, i32 22, i32 22, i32 22, i32 22, i32 22, i32 22, i32 22, i32 22>)
   %2 = bitcast i16 %x4 to <16 x i1>
   %3 = select <16 x i1> %2, <16 x i32> %1, <16 x i32> %x3
-  %4 = call <16 x i32> @llvm.x86.avx512.vpshld.d.512(<16 x i32> %x0, <16 x i32> %x1, i32 22)
+  %4 = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> <i32 22, i32 22, i32 22, i32 22, i32 22, i32 22, i32 22, i32 22, i32 22, i32 22, i32 22, i32 22, i32 22, i32 22, i32 22, i32 22>)
   %res2 = add <16 x i32> %3, %4
   ret <16 x i32> %res2
 }
-declare <16 x i32> @llvm.x86.avx512.vpshld.d.512(<16 x i32>, <16 x i32>, i32)
 
-define <8 x i64>@test_int_x86_avx512_mask_vpshld_q_512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x3, i8 %x4) {
+define <8 x i64> @test_int_x86_avx512_mask_vpshld_q_512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x3, i8 %x4) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpshld_q_512:
 ; X86:       # %bb.0:
 ; X86-NEXT:    vpshldq $22, %zmm1, %zmm0, %zmm3 # encoding: [0x62,0xf3,0xfd,0x48,0x71,0xd9,0x16]
@@ -136,41 +408,39 @@ define <8 x i64>@test_int_x86_avx512_mask_vpshld_q_512(<8 x i64> %x0, <8 x i64>
 ; X64-NEXT:    vpshldq $22, %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x49,0x71,0xd1,0x16]
 ; X64-NEXT:    vpaddq %zmm3, %zmm2, %zmm0 # encoding: [0x62,0xf1,0xed,0x48,0xd4,0xc3]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %1 = call <8 x i64> @llvm.x86.avx512.vpshld.q.512(<8 x i64> %x0, <8 x i64> %x1, i32 22)
+  %1 = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> <i64 22, i64 22, i64 22, i64 22, i64 22, i64 22, i64 22, i64 22>)
   %2 = bitcast i8 %x4 to <8 x i1>
   %3 = select <8 x i1> %2, <8 x i64> %1, <8 x i64> %x3
-  %4 = call <8 x i64> @llvm.x86.avx512.vpshld.q.512(<8 x i64> %x0, <8 x i64> %x1, i32 22)
+  %4 = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> <i64 22, i64 22, i64 22, i64 22, i64 22, i64 22, i64 22, i64 22>)
   %res2 = add <8 x i64> %3, %4
   ret <8 x i64> %res2
 }
-declare <8 x i64> @llvm.x86.avx512.vpshld.q.512(<8 x i64>, <8 x i64>, i32)
 
-define <32 x i16>@test_int_x86_avx512_mask_vpshld_w_512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x3, i32 %x4) {
+define <32 x i16> @test_int_x86_avx512_mask_vpshld_w_512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x3, i32 %x4) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpshld_w_512:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpshldw $22, %zmm1, %zmm0, %zmm3 # encoding: [0x62,0xf3,0xfd,0x48,0x70,0xd9,0x16]
+; X86-NEXT:    vpshldw $6, %zmm1, %zmm0, %zmm3 # encoding: [0x62,0xf3,0xfd,0x48,0x70,0xd9,0x06]
 ; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf9,0x90,0x4c,0x24,0x04]
-; X86-NEXT:    vpshldw $22, %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x49,0x70,0xd1,0x16]
+; X86-NEXT:    vpshldw $6, %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x49,0x70,0xd1,0x06]
 ; X86-NEXT:    vpaddw %zmm3, %zmm2, %zmm0 # encoding: [0x62,0xf1,0x6d,0x48,0xfd,0xc3]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_vpshld_w_512:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpshldw $22, %zmm1, %zmm0, %zmm3 # encoding: [0x62,0xf3,0xfd,0x48,0x70,0xd9,0x16]
+; X64-NEXT:    vpshldw $6, %zmm1, %zmm0, %zmm3 # encoding: [0x62,0xf3,0xfd,0x48,0x70,0xd9,0x06]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
-; X64-NEXT:    vpshldw $22, %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x49,0x70,0xd1,0x16]
+; X64-NEXT:    vpshldw $6, %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x49,0x70,0xd1,0x06]
 ; X64-NEXT:    vpaddw %zmm3, %zmm2, %zmm0 # encoding: [0x62,0xf1,0x6d,0x48,0xfd,0xc3]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %1 = call <32 x i16> @llvm.x86.avx512.vpshld.w.512(<32 x i16> %x0, <32 x i16> %x1, i32 22)
+  %1 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> <i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6>)
   %2 = bitcast i32 %x4 to <32 x i1>
   %3 = select <32 x i1> %2, <32 x i16> %1, <32 x i16> %x3
-  %4 = call <32 x i16> @llvm.x86.avx512.vpshld.w.512(<32 x i16> %x0, <32 x i16> %x1, i32 22)
+  %4 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> <i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6>)
   %res2 = add <32 x i16> %3, %4
   ret <32 x i16> %res2
 }
-declare <32 x i16> @llvm.x86.avx512.vpshld.w.512(<32 x i16>, <32 x i16>, i32)
 
-define <16 x i32>@test_int_x86_avx512_mask_vpshrd_d_512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x3, i16 %x4) {
+define <16 x i32> @test_int_x86_avx512_mask_vpshrd_d_512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x3, i16 %x4) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpshrd_d_512:
 ; X86:       # %bb.0:
 ; X86-NEXT:    vpshrdd $22, %zmm1, %zmm0, %zmm3 # encoding: [0x62,0xf3,0x7d,0x48,0x73,0xd9,0x16]
@@ -186,16 +456,15 @@ define <16 x i32>@test_int_x86_avx512_mask_vpshrd_d_512(<16 x i32> %x0, <16 x i3
 ; X64-NEXT:    vpshrdd $22, %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf3,0x7d,0x49,0x73,0xd1,0x16]
 ; X64-NEXT:    vpaddd %zmm3, %zmm2, %zmm0 # encoding: [0x62,0xf1,0x6d,0x48,0xfe,0xc3]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %1 = call <16 x i32> @llvm.x86.avx512.vpshrd.d.512(<16 x i32> %x0, <16 x i32> %x1, i32 22)
+  %1 = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %x1, <16 x i32> %x0, <16 x i32> <i32 22, i32 22, i32 22, i32 22, i32 22, i32 22, i32 22, i32 22, i32 22, i32 22, i32 22, i32 22, i32 22, i32 22, i32 22, i32 22>)
   %2 = bitcast i16 %x4 to <16 x i1>
   %3 = select <16 x i1> %2, <16 x i32> %1, <16 x i32> %x3
-  %4 = call <16 x i32> @llvm.x86.avx512.vpshrd.d.512(<16 x i32> %x0, <16 x i32> %x1, i32 22)
+  %4 = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %x1, <16 x i32> %x0, <16 x i32> <i32 22, i32 22, i32 22, i32 22, i32 22, i32 22, i32 22, i32 22, i32 22, i32 22, i32 22, i32 22, i32 22, i32 22, i32 22, i32 22>)
   %res2 = add <16 x i32> %3, %4
   ret <16 x i32> %res2
 }
-declare <16 x i32> @llvm.x86.avx512.vpshrd.d.512(<16 x i32>, <16 x i32>, i32)
 
-define <8 x i64>@test_int_x86_avx512_mask_vpshrd_q_512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x3, i8 %x4) {
+define <8 x i64> @test_int_x86_avx512_mask_vpshrd_q_512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x3, i8 %x4) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpshrd_q_512:
 ; X86:       # %bb.0:
 ; X86-NEXT:    vpshrdq $22, %zmm1, %zmm0, %zmm3 # encoding: [0x62,0xf3,0xfd,0x48,0x73,0xd9,0x16]
@@ -212,44 +481,39 @@ define <8 x i64>@test_int_x86_avx512_mask_vpshrd_q_512(<8 x i64> %x0, <8 x i64>
 ; X64-NEXT:    vpshrdq $22, %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x49,0x73,0xd1,0x16]
 ; X64-NEXT:    vpaddq %zmm3, %zmm2, %zmm0 # encoding: [0x62,0xf1,0xed,0x48,0xd4,0xc3]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %1 = call <8 x i64> @llvm.x86.avx512.vpshrd.q.512(<8 x i64> %x0, <8 x i64> %x1, i32 22)
+  %1 = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %x1, <8 x i64> %x0, <8 x i64> <i64 22, i64 22, i64 22, i64 22, i64 22, i64 22, i64 22, i64 22>)
   %2 = bitcast i8 %x4 to <8 x i1>
   %3 = select <8 x i1> %2, <8 x i64> %1, <8 x i64> %x3
-  %4 = call <8 x i64> @llvm.x86.avx512.vpshrd.q.512(<8 x i64> %x0, <8 x i64> %x1, i32 22)
+  %4 = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %x1, <8 x i64> %x0, <8 x i64> <i64 22, i64 22, i64 22, i64 22, i64 22, i64 22, i64 22, i64 22>)
   %res2 = add <8 x i64> %3, %4
   ret <8 x i64> %res2
 }
-declare <8 x i64> @llvm.x86.avx512.vpshrd.q.512(<8 x i64>, <8 x i64>, i32)
 
-define <32 x i16>@test_int_x86_avx512_mask_vpshrd_w_512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x3, i32 %x4) {
+define <32 x i16> @test_int_x86_avx512_mask_vpshrd_w_512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x3, i32 %x4) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpshrd_w_512:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpshrdw $22, %zmm1, %zmm0, %zmm3 # encoding: [0x62,0xf3,0xfd,0x48,0x72,0xd9,0x16]
+; X86-NEXT:    vpshrdw $6, %zmm1, %zmm0, %zmm3 # encoding: [0x62,0xf3,0xfd,0x48,0x72,0xd9,0x06]
 ; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf9,0x90,0x4c,0x24,0x04]
-; X86-NEXT:    vpshrdw $22, %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x49,0x72,0xd1,0x16]
+; X86-NEXT:    vpshrdw $6, %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x49,0x72,0xd1,0x06]
 ; X86-NEXT:    vpaddw %zmm3, %zmm2, %zmm0 # encoding: [0x62,0xf1,0x6d,0x48,0xfd,0xc3]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_vpshrd_w_512:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpshrdw $22, %zmm1, %zmm0, %zmm3 # encoding: [0x62,0xf3,0xfd,0x48,0x72,0xd9,0x16]
+; X64-NEXT:    vpshrdw $6, %zmm1, %zmm0, %zmm3 # encoding: [0x62,0xf3,0xfd,0x48,0x72,0xd9,0x06]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
-; X64-NEXT:    vpshrdw $22, %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x49,0x72,0xd1,0x16]
+; X64-NEXT:    vpshrdw $6, %zmm1, %zmm0, %zmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x49,0x72,0xd1,0x06]
 ; X64-NEXT:    vpaddw %zmm3, %zmm2, %zmm0 # encoding: [0x62,0xf1,0x6d,0x48,0xfd,0xc3]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %1 = call <32 x i16> @llvm.x86.avx512.vpshrd.w.512(<32 x i16> %x0, <32 x i16> %x1, i32 22)
+  %1 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %x1, <32 x i16> %x0, <32 x i16> <i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6>)
   %2 = bitcast i32 %x4 to <32 x i1>
   %3 = select <32 x i1> %2, <32 x i16> %1, <32 x i16> %x3
-  %4 = call <32 x i16> @llvm.x86.avx512.vpshrd.w.512(<32 x i16> %x0, <32 x i16> %x1, i32 22)
+  %4 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %x1, <32 x i16> %x0, <32 x i16> <i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6>)
   %res2 = add <32 x i16> %3, %4
   ret <32 x i16> %res2
 }
-declare <32 x i16> @llvm.x86.avx512.vpshrd.w.512(<32 x i16>, <32 x i16>, i32)
 
-declare <16 x i32> @llvm.x86.avx512.mask.vpshrdv.d.512(<16 x i32>, <16 x i32>, <16 x i32>, i16)
-declare <16 x i32> @llvm.x86.avx512.maskz.vpshrdv.d.512(<16 x i32>, <16 x i32>, <16 x i32>, i16)
-
-define <16 x i32>@test_int_x86_avx512_mask_vpshrdv_d_512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32>* %x2p, <16 x i32> %x4, i16 %x3) {
+define <16 x i32> @test_int_x86_avx512_mask_vpshrdv_d_512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32>* %x2p, <16 x i32> %x4, i16 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpshrdv_d_512:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
@@ -257,9 +521,9 @@ define <16 x i32>@test_int_x86_avx512_mask_vpshrdv_d_512(<16 x i32> %x0, <16 x i
 ; X86-NEXT:    vmovdqa64 %zmm0, %zmm3 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xd8]
 ; X86-NEXT:    vpshrdvd (%eax), %zmm1, %zmm3 {%k1} # encoding: [0x62,0xf2,0x75,0x49,0x73,0x18]
 ; X86-NEXT:    vmovdqa64 %zmm0, %zmm4 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xe0]
-; X86-NEXT:    vpshrdvd %zmm2, %zmm1, %zmm4 {%k1} {z} # encoding: [0x62,0xf2,0x75,0xc9,0x73,0xe2]
-; X86-NEXT:    vpshrdvd %zmm2, %zmm1, %zmm0 # encoding: [0x62,0xf2,0x75,0x48,0x73,0xc2]
-; X86-NEXT:    vpaddd %zmm4, %zmm0, %zmm0 # encoding: [0x62,0xf1,0x7d,0x48,0xfe,0xc4]
+; X86-NEXT:    vpshrdvd %zmm2, %zmm1, %zmm4 # encoding: [0x62,0xf2,0x75,0x48,0x73,0xe2]
+; X86-NEXT:    vpshrdvd %zmm2, %zmm1, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0xc9,0x73,0xc2]
+; X86-NEXT:    vpaddd %zmm0, %zmm4, %zmm0 # encoding: [0x62,0xf1,0x5d,0x48,0xfe,0xc0]
 ; X86-NEXT:    vpaddd %zmm0, %zmm3, %zmm0 # encoding: [0x62,0xf1,0x65,0x48,0xfe,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
@@ -275,29 +539,30 @@ define <16 x i32>@test_int_x86_avx512_mask_vpshrdv_d_512(<16 x i32> %x0, <16 x i
 ; X64-NEXT:    vpaddd %zmm0, %zmm3, %zmm0 # encoding: [0x62,0xf1,0x65,0x48,0xfe,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %x2 = load <16 x i32>, <16 x i32>* %x2p
-  %res = call <16 x i32> @llvm.x86.avx512.mask.vpshrdv.d.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 %x3)
-  %res1 = call <16 x i32> @llvm.x86.avx512.mask.vpshrdv.d.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x4, i16 -1)
-  %res2 = call <16 x i32> @llvm.x86.avx512.maskz.vpshrdv.d.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x4, i16  %x3)
-  %res3 = add <16 x i32> %res, %res1
-  %res4 = add <16 x i32> %res2, %res3
+  %1 = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %x1, <16 x i32> %x0, <16 x i32> %x2)
+  %2 = bitcast i16 %x3 to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x i32> %1, <16 x i32> %x0
+  %4 = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %x1, <16 x i32> %x0, <16 x i32> %x4)
+  %5 = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %x1, <16 x i32> %x0, <16 x i32> %x4)
+  %6 = bitcast i16 %x3 to <16 x i1>
+  %7 = select <16 x i1> %6, <16 x i32> %5, <16 x i32> zeroinitializer
+  %res3 = add <16 x i32> %3, %4
+  %res4 = add <16 x i32> %7, %res3
   ret <16 x i32> %res4
 }
 
-declare <8 x i64> @llvm.x86.avx512.mask.vpshrdv.q.512(<8 x i64>, <8 x i64>, <8 x i64>, i8)
-declare <8 x i64> @llvm.x86.avx512.maskz.vpshrdv.q.512(<8 x i64>, <8 x i64>, <8 x i64>, i8)
-
-define <8 x i64>@test_int_x86_avx512_mask_vpshrdv_q_512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64>* %x2p, <8 x i64> %x4, i8 %x3) {
+define <8 x i64> @test_int_x86_avx512_mask_vpshrdv_q_512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64>* %x2p, <8 x i64> %x4, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpshrdv_q_512:
 ; X86:       # %bb.0:
-; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x08]
-; X86-NEXT:    kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx # encoding: [0x0f,0xb6,0x4c,0x24,0x08]
+; X86-NEXT:    kmovd %ecx, %k1 # encoding: [0xc5,0xfb,0x92,0xc9]
 ; X86-NEXT:    vmovdqa64 %zmm0, %zmm3 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xd8]
 ; X86-NEXT:    vpshrdvq (%eax), %zmm1, %zmm3 {%k1} # encoding: [0x62,0xf2,0xf5,0x49,0x73,0x18]
 ; X86-NEXT:    vmovdqa64 %zmm0, %zmm4 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xe0]
-; X86-NEXT:    vpshrdvq %zmm2, %zmm1, %zmm4 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xc9,0x73,0xe2]
-; X86-NEXT:    vpshrdvq %zmm2, %zmm1, %zmm0 # encoding: [0x62,0xf2,0xf5,0x48,0x73,0xc2]
-; X86-NEXT:    vpaddq %zmm4, %zmm0, %zmm0 # encoding: [0x62,0xf1,0xfd,0x48,0xd4,0xc4]
+; X86-NEXT:    vpshrdvq %zmm2, %zmm1, %zmm4 # encoding: [0x62,0xf2,0xf5,0x48,0x73,0xe2]
+; X86-NEXT:    vpshrdvq %zmm2, %zmm1, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xc9,0x73,0xc2]
+; X86-NEXT:    vpaddq %zmm0, %zmm4, %zmm0 # encoding: [0x62,0xf1,0xdd,0x48,0xd4,0xc0]
 ; X86-NEXT:    vpaddq %zmm0, %zmm3, %zmm0 # encoding: [0x62,0xf1,0xe5,0x48,0xd4,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
@@ -313,18 +578,19 @@ define <8 x i64>@test_int_x86_avx512_mask_vpshrdv_q_512(<8 x i64> %x0, <8 x i64>
 ; X64-NEXT:    vpaddq %zmm0, %zmm3, %zmm0 # encoding: [0x62,0xf1,0xe5,0x48,0xd4,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %x2 = load <8 x i64>, <8 x i64>* %x2p
-  %res = call <8 x i64> @llvm.x86.avx512.mask.vpshrdv.q.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 %x3)
-  %res1 = call <8 x i64> @llvm.x86.avx512.mask.vpshrdv.q.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x4, i8 -1)
-  %res2 = call <8 x i64> @llvm.x86.avx512.maskz.vpshrdv.q.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x4, i8  %x3)
-  %res3 = add <8 x i64> %res, %res1
-  %res4 = add <8 x i64> %res2, %res3
+  %1 = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %x1, <8 x i64> %x0, <8 x i64> %x2)
+  %2 = bitcast i8 %x3 to <8 x i1>
+  %3 = select <8 x i1> %2, <8 x i64> %1, <8 x i64> %x0
+  %4 = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %x1, <8 x i64> %x0, <8 x i64> %x4)
+  %5 = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %x1, <8 x i64> %x0, <8 x i64> %x4)
+  %6 = bitcast i8 %x3 to <8 x i1>
+  %7 = select <8 x i1> %6, <8 x i64> %5, <8 x i64> zeroinitializer
+  %res3 = add <8 x i64> %3, %4
+  %res4 = add <8 x i64> %7, %res3
   ret <8 x i64> %res4
 }
 
-declare <32 x i16> @llvm.x86.avx512.mask.vpshrdv.w.512(<32 x i16>, <32 x i16>, <32 x i16>, i32)
-declare <32 x i16> @llvm.x86.avx512.maskz.vpshrdv.w.512(<32 x i16>, <32 x i16>, <32 x i16>, i32)
-
-define <32 x i16>@test_int_x86_avx512_mask_vpshrdv_w_512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16>* %x2p, <32 x i16> %x4, i32 %x3) {
+define <32 x i16> @test_int_x86_avx512_mask_vpshrdv_w_512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16>* %x2p, <32 x i16> %x4, i32 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpshrdv_w_512:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
@@ -332,9 +598,9 @@ define <32 x i16>@test_int_x86_avx512_mask_vpshrdv_w_512(<32 x i16> %x0, <32 x i
 ; X86-NEXT:    vmovdqa64 %zmm0, %zmm3 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xd8]
 ; X86-NEXT:    vpshrdvw (%eax), %zmm1, %zmm3 {%k1} # encoding: [0x62,0xf2,0xf5,0x49,0x72,0x18]
 ; X86-NEXT:    vmovdqa64 %zmm0, %zmm4 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xe0]
-; X86-NEXT:    vpshrdvw %zmm2, %zmm1, %zmm4 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xc9,0x72,0xe2]
-; X86-NEXT:    vpshrdvw %zmm2, %zmm1, %zmm0 # encoding: [0x62,0xf2,0xf5,0x48,0x72,0xc2]
-; X86-NEXT:    vpaddw %zmm4, %zmm0, %zmm0 # encoding: [0x62,0xf1,0x7d,0x48,0xfd,0xc4]
+; X86-NEXT:    vpshrdvw %zmm2, %zmm1, %zmm4 # encoding: [0x62,0xf2,0xf5,0x48,0x72,0xe2]
+; X86-NEXT:    vpshrdvw %zmm2, %zmm1, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xc9,0x72,0xc2]
+; X86-NEXT:    vpaddw %zmm0, %zmm4, %zmm0 # encoding: [0x62,0xf1,0x5d,0x48,0xfd,0xc0]
 ; X86-NEXT:    vpaddw %zmm0, %zmm3, %zmm0 # encoding: [0x62,0xf1,0x65,0x48,0xfd,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
@@ -344,24 +610,25 @@ define <32 x i16>@test_int_x86_avx512_mask_vpshrdv_w_512(<32 x i16> %x0, <32 x i
 ; X64-NEXT:    vmovdqa64 %zmm0, %zmm3 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xd8]
 ; X64-NEXT:    vpshrdvw (%rdi), %zmm1, %zmm3 {%k1} # encoding: [0x62,0xf2,0xf5,0x49,0x72,0x1f]
 ; X64-NEXT:    vmovdqa64 %zmm0, %zmm4 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xe0]
-; X64-NEXT:    vpshrdvw %zmm2, %zmm1, %zmm4 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xc9,0x72,0xe2]
-; X64-NEXT:    vpshrdvw %zmm2, %zmm1, %zmm0 # encoding: [0x62,0xf2,0xf5,0x48,0x72,0xc2]
-; X64-NEXT:    vpaddw %zmm4, %zmm0, %zmm0 # encoding: [0x62,0xf1,0x7d,0x48,0xfd,0xc4]
+; X64-NEXT:    vpshrdvw %zmm2, %zmm1, %zmm4 # encoding: [0x62,0xf2,0xf5,0x48,0x72,0xe2]
+; X64-NEXT:    vpshrdvw %zmm2, %zmm1, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xc9,0x72,0xc2]
+; X64-NEXT:    vpaddw %zmm0, %zmm4, %zmm0 # encoding: [0x62,0xf1,0x5d,0x48,0xfd,0xc0]
 ; X64-NEXT:    vpaddw %zmm0, %zmm3, %zmm0 # encoding: [0x62,0xf1,0x65,0x48,0xfd,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %x2 = load <32 x i16>, <32 x i16>* %x2p
-  %res = call <32 x i16> @llvm.x86.avx512.mask.vpshrdv.w.512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3)
-  %res1 = call <32 x i16> @llvm.x86.avx512.mask.vpshrdv.w.512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x4, i32 -1)
-  %res2 = call <32 x i16> @llvm.x86.avx512.maskz.vpshrdv.w.512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x4, i32  %x3)
-  %res3 = add <32 x i16> %res, %res1
-  %res4 = add <32 x i16> %res2, %res3
+  %1 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %x1, <32 x i16> %x0, <32 x i16> %x2)
+  %2 = bitcast i32 %x3 to <32 x i1>
+  %3 = select <32 x i1> %2, <32 x i16> %1, <32 x i16> %x0
+  %4 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %x1, <32 x i16> %x0, <32 x i16> %x4)
+  %5 = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %x1, <32 x i16> %x0, <32 x i16> %x4)
+  %6 = bitcast i32 %x3 to <32 x i1>
+  %7 = select <32 x i1> %6, <32 x i16> %5, <32 x i16> zeroinitializer
+  %res3 = add <32 x i16> %3, %4
+  %res4 = add <32 x i16> %7, %res3
   ret <32 x i16> %res4
 }
 
-declare <16 x i32> @llvm.x86.avx512.mask.vpshldv.d.512(<16 x i32>, <16 x i32>, <16 x i32>, i16)
-declare <16 x i32> @llvm.x86.avx512.maskz.vpshldv.d.512(<16 x i32>, <16 x i32>, <16 x i32>, i16)
-
-define <16 x i32>@test_int_x86_avx512_mask_vpshldv_d_512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32>* %x2p, <16 x i32> %x4, i16 %x3) {
+define <16 x i32> @test_int_x86_avx512_mask_vpshldv_d_512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32>* %x2p, <16 x i32> %x4, i16 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpshldv_d_512:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
@@ -369,9 +636,9 @@ define <16 x i32>@test_int_x86_avx512_mask_vpshldv_d_512(<16 x i32> %x0, <16 x i
 ; X86-NEXT:    vmovdqa64 %zmm0, %zmm3 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xd8]
 ; X86-NEXT:    vpshldvd (%eax), %zmm1, %zmm3 {%k1} # encoding: [0x62,0xf2,0x75,0x49,0x71,0x18]
 ; X86-NEXT:    vmovdqa64 %zmm0, %zmm4 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xe0]
-; X86-NEXT:    vpshldvd %zmm2, %zmm1, %zmm4 {%k1} {z} # encoding: [0x62,0xf2,0x75,0xc9,0x71,0xe2]
-; X86-NEXT:    vpshldvd %zmm2, %zmm1, %zmm0 # encoding: [0x62,0xf2,0x75,0x48,0x71,0xc2]
-; X86-NEXT:    vpaddd %zmm4, %zmm0, %zmm0 # encoding: [0x62,0xf1,0x7d,0x48,0xfe,0xc4]
+; X86-NEXT:    vpshldvd %zmm2, %zmm1, %zmm4 # encoding: [0x62,0xf2,0x75,0x48,0x71,0xe2]
+; X86-NEXT:    vpshldvd %zmm2, %zmm1, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0xc9,0x71,0xc2]
+; X86-NEXT:    vpaddd %zmm0, %zmm4, %zmm0 # encoding: [0x62,0xf1,0x5d,0x48,0xfe,0xc0]
 ; X86-NEXT:    vpaddd %zmm0, %zmm3, %zmm0 # encoding: [0x62,0xf1,0x65,0x48,0xfe,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
@@ -387,29 +654,30 @@ define <16 x i32>@test_int_x86_avx512_mask_vpshldv_d_512(<16 x i32> %x0, <16 x i
 ; X64-NEXT:    vpaddd %zmm0, %zmm3, %zmm0 # encoding: [0x62,0xf1,0x65,0x48,0xfe,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %x2 = load <16 x i32>, <16 x i32>* %x2p
-  %res = call <16 x i32> @llvm.x86.avx512.mask.vpshldv.d.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2, i16 %x3)
-  %res1 = call <16 x i32> @llvm.x86.avx512.mask.vpshldv.d.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x4, i16 -1)
-  %res2 = call <16 x i32> @llvm.x86.avx512.maskz.vpshldv.d.512(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x4, i16  %x3)
-  %res3 = add <16 x i32> %res, %res1
-  %res4 = add <16 x i32> %res2, %res3
+  %1 = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x2)
+  %2 = bitcast i16 %x3 to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x i32> %1, <16 x i32> %x0
+  %4 = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x4)
+  %5 = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %x0, <16 x i32> %x1, <16 x i32> %x4)
+  %6 = bitcast i16 %x3 to <16 x i1>
+  %7 = select <16 x i1> %6, <16 x i32> %5, <16 x i32> zeroinitializer
+  %res3 = add <16 x i32> %3, %4
+  %res4 = add <16 x i32> %7, %res3
   ret <16 x i32> %res4
 }
 
-declare <8 x i64> @llvm.x86.avx512.mask.vpshldv.q.512(<8 x i64>, <8 x i64>, <8 x i64>, i8)
-declare <8 x i64> @llvm.x86.avx512.maskz.vpshldv.q.512(<8 x i64>, <8 x i64>, <8 x i64>, i8)
-
-define <8 x i64>@test_int_x86_avx512_mask_vpshldv_q_512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64>* %x2p, <8 x i64> %x4, i8 %x3) {
+define <8 x i64> @test_int_x86_avx512_mask_vpshldv_q_512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64>* %x2p, <8 x i64> %x4, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpshldv_q_512:
 ; X86:       # %bb.0:
-; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x08]
-; X86-NEXT:    kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx # encoding: [0x0f,0xb6,0x4c,0x24,0x08]
+; X86-NEXT:    kmovd %ecx, %k1 # encoding: [0xc5,0xfb,0x92,0xc9]
 ; X86-NEXT:    vmovdqa64 %zmm0, %zmm3 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xd8]
 ; X86-NEXT:    vpshldvq (%eax), %zmm1, %zmm3 {%k1} # encoding: [0x62,0xf2,0xf5,0x49,0x71,0x18]
 ; X86-NEXT:    vmovdqa64 %zmm0, %zmm4 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xe0]
-; X86-NEXT:    vpshldvq %zmm2, %zmm1, %zmm4 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xc9,0x71,0xe2]
-; X86-NEXT:    vpshldvq %zmm2, %zmm1, %zmm0 # encoding: [0x62,0xf2,0xf5,0x48,0x71,0xc2]
-; X86-NEXT:    vpaddq %zmm4, %zmm0, %zmm0 # encoding: [0x62,0xf1,0xfd,0x48,0xd4,0xc4]
+; X86-NEXT:    vpshldvq %zmm2, %zmm1, %zmm4 # encoding: [0x62,0xf2,0xf5,0x48,0x71,0xe2]
+; X86-NEXT:    vpshldvq %zmm2, %zmm1, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xc9,0x71,0xc2]
+; X86-NEXT:    vpaddq %zmm0, %zmm4, %zmm0 # encoding: [0x62,0xf1,0xdd,0x48,0xd4,0xc0]
 ; X86-NEXT:    vpaddq %zmm0, %zmm3, %zmm0 # encoding: [0x62,0xf1,0xe5,0x48,0xd4,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
@@ -425,18 +693,19 @@ define <8 x i64>@test_int_x86_avx512_mask_vpshldv_q_512(<8 x i64> %x0, <8 x i64>
 ; X64-NEXT:    vpaddq %zmm0, %zmm3, %zmm0 # encoding: [0x62,0xf1,0xe5,0x48,0xd4,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %x2 = load <8 x i64>, <8 x i64>* %x2p
-  %res = call <8 x i64> @llvm.x86.avx512.mask.vpshldv.q.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2, i8 %x3)
-  %res1 = call <8 x i64> @llvm.x86.avx512.mask.vpshldv.q.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x4, i8 -1)
-  %res2 = call <8 x i64> @llvm.x86.avx512.maskz.vpshldv.q.512(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x4, i8  %x3)
-  %res3 = add <8 x i64> %res, %res1
-  %res4 = add <8 x i64> %res2, %res3
+  %1 = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x2)
+  %2 = bitcast i8 %x3 to <8 x i1>
+  %3 = select <8 x i1> %2, <8 x i64> %1, <8 x i64> %x0
+  %4 = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x4)
+  %5 = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %x0, <8 x i64> %x1, <8 x i64> %x4)
+  %6 = bitcast i8 %x3 to <8 x i1>
+  %7 = select <8 x i1> %6, <8 x i64> %5, <8 x i64> zeroinitializer
+  %res3 = add <8 x i64> %3, %4
+  %res4 = add <8 x i64> %7, %res3
   ret <8 x i64> %res4
 }
 
-declare <32 x i16> @llvm.x86.avx512.mask.vpshldv.w.512(<32 x i16>, <32 x i16>, <32 x i16>, i32)
-declare <32 x i16> @llvm.x86.avx512.maskz.vpshldv.w.512(<32 x i16>, <32 x i16>, <32 x i16>, i32)
-
-define <32 x i16>@test_int_x86_avx512_mask_vpshldv_w_512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16>* %x2p, <32 x i16> %x4, i32 %x3) {
+define <32 x i16> @test_int_x86_avx512_mask_vpshldv_w_512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16>* %x2p, <32 x i16> %x4, i32 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpshldv_w_512:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
@@ -444,9 +713,9 @@ define <32 x i16>@test_int_x86_avx512_mask_vpshldv_w_512(<32 x i16> %x0, <32 x i
 ; X86-NEXT:    vmovdqa64 %zmm0, %zmm3 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xd8]
 ; X86-NEXT:    vpshldvw (%eax), %zmm1, %zmm3 {%k1} # encoding: [0x62,0xf2,0xf5,0x49,0x70,0x18]
 ; X86-NEXT:    vmovdqa64 %zmm0, %zmm4 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xe0]
-; X86-NEXT:    vpshldvw %zmm2, %zmm1, %zmm4 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xc9,0x70,0xe2]
-; X86-NEXT:    vpshldvw %zmm2, %zmm1, %zmm0 # encoding: [0x62,0xf2,0xf5,0x48,0x70,0xc2]
-; X86-NEXT:    vpaddw %zmm4, %zmm0, %zmm0 # encoding: [0x62,0xf1,0x7d,0x48,0xfd,0xc4]
+; X86-NEXT:    vpshldvw %zmm2, %zmm1, %zmm4 # encoding: [0x62,0xf2,0xf5,0x48,0x70,0xe2]
+; X86-NEXT:    vpshldvw %zmm2, %zmm1, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xc9,0x70,0xc2]
+; X86-NEXT:    vpaddw %zmm0, %zmm4, %zmm0 # encoding: [0x62,0xf1,0x5d,0x48,0xfd,0xc0]
 ; X86-NEXT:    vpaddw %zmm0, %zmm3, %zmm0 # encoding: [0x62,0xf1,0x65,0x48,0xfd,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
@@ -456,17 +725,35 @@ define <32 x i16>@test_int_x86_avx512_mask_vpshldv_w_512(<32 x i16> %x0, <32 x i
 ; X64-NEXT:    vmovdqa64 %zmm0, %zmm3 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xd8]
 ; X64-NEXT:    vpshldvw (%rdi), %zmm1, %zmm3 {%k1} # encoding: [0x62,0xf2,0xf5,0x49,0x70,0x1f]
 ; X64-NEXT:    vmovdqa64 %zmm0, %zmm4 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xe0]
-; X64-NEXT:    vpshldvw %zmm2, %zmm1, %zmm4 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xc9,0x70,0xe2]
-; X64-NEXT:    vpshldvw %zmm2, %zmm1, %zmm0 # encoding: [0x62,0xf2,0xf5,0x48,0x70,0xc2]
-; X64-NEXT:    vpaddw %zmm4, %zmm0, %zmm0 # encoding: [0x62,0xf1,0x7d,0x48,0xfd,0xc4]
+; X64-NEXT:    vpshldvw %zmm2, %zmm1, %zmm4 # encoding: [0x62,0xf2,0xf5,0x48,0x70,0xe2]
+; X64-NEXT:    vpshldvw %zmm2, %zmm1, %zmm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xc9,0x70,0xc2]
+; X64-NEXT:    vpaddw %zmm0, %zmm4, %zmm0 # encoding: [0x62,0xf1,0x5d,0x48,0xfd,0xc0]
 ; X64-NEXT:    vpaddw %zmm0, %zmm3, %zmm0 # encoding: [0x62,0xf1,0x65,0x48,0xfd,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %x2 = load <32 x i16>, <32 x i16>* %x2p
-  %res = call <32 x i16> @llvm.x86.avx512.mask.vpshldv.w.512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2, i32 %x3)
-  %res1 = call <32 x i16> @llvm.x86.avx512.mask.vpshldv.w.512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x4, i32 -1)
-  %res2 = call <32 x i16> @llvm.x86.avx512.maskz.vpshldv.w.512(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x4, i32  %x3)
-  %res3 = add <32 x i16> %res, %res1
-  %res4 = add <32 x i16> %res2, %res3
+  %1 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x2)
+  %2 = bitcast i32 %x3 to <32 x i1>
+  %3 = select <32 x i1> %2, <32 x i16> %1, <32 x i16> %x0
+  %4 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x4)
+  %5 = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %x0, <32 x i16> %x1, <32 x i16> %x4)
+  %6 = bitcast i32 %x3 to <32 x i1>
+  %7 = select <32 x i1> %6, <32 x i16> %5, <32 x i16> zeroinitializer
+  %res3 = add <32 x i16> %3, %4
+  %res4 = add <32 x i16> %7, %res3
   ret <32 x i16> %res4
 }
 
+declare <16 x i32> @llvm.fshl.v16i32(<16 x i32>, <16 x i32>, <16 x i32>)
+declare <8 x i64> @llvm.fshl.v8i64(<8 x i64>, <8 x i64>, <8 x i64>)
+declare <32 x i16> @llvm.fshl.v32i16(<32 x i16>, <32 x i16>, <32 x i16>)
+declare <16 x i32> @llvm.fshr.v16i32(<16 x i32>, <16 x i32>, <16 x i32>)
+declare <8 x i64> @llvm.fshr.v8i64(<8 x i64>, <8 x i64>, <8 x i64>)
+declare <32 x i16> @llvm.fshr.v32i16(<32 x i16>, <32 x i16>, <32 x i16>)
+declare <32 x i16> @llvm.masked.expandload.v32i16(i16*, <32 x i1>, <32 x i16>)
+declare <64 x i8> @llvm.masked.expandload.v64i8(i8*, <64 x i1>, <64 x i8>)
+declare void @llvm.masked.compressstore.v32i16(<32 x i16>, i16*, <32 x i1>)
+declare void @llvm.masked.compressstore.v64i8(<64 x i8>, i8*, <64 x i1>)
+declare <32 x i16> @llvm.x86.avx512.mask.expand.v32i16(<32 x i16>, <32 x i16>, <32 x i1>)
+declare <64 x i8> @llvm.x86.avx512.mask.expand.v64i8(<64 x i8>, <64 x i8>, <64 x i1>)
+declare <32 x i16> @llvm.x86.avx512.mask.compress.v32i16(<32 x i16>, <32 x i16>, <32 x i1>)
+declare <64 x i8> @llvm.x86.avx512.mask.compress.v64i8(<64 x i8>, <64 x i8>, <64 x i1>)
diff --git a/test/CodeGen/X86/avx512vbmi2vl-intrinsics-fast-isel.ll b/test/CodeGen/X86/avx512vbmi2vl-intrinsics-fast-isel.ll
index 392ea0b21f55932d549f938d0654b189e006be2e..03594dfa36af11704525ffc0aa60aeeb705fdd2e 100644
--- a/test/CodeGen/X86/avx512vbmi2vl-intrinsics-fast-isel.ll
+++ b/test/CodeGen/X86/avx512vbmi2vl-intrinsics-fast-isel.ll
@@ -582,23 +582,23 @@ define <4 x i64> @test_mm256_mask_shldi_epi64(<4 x i64> %__S, i8 zeroext %__U, <
 ; X86:       # %bb.0: # %entry
 ; X86-NEXT:    movb {{[0-9]+}}(%esp), %al
 ; X86-NEXT:    kmovd %eax, %k1
-; X86-NEXT:    vpshldq $127, %ymm2, %ymm1, %ymm0 {%k1}
+; X86-NEXT:    vpshldq $47, %ymm2, %ymm1, %ymm0 {%k1}
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: test_mm256_mask_shldi_epi64:
 ; X64:       # %bb.0: # %entry
 ; X64-NEXT:    kmovd %edi, %k1
-; X64-NEXT:    vpshldq $127, %ymm2, %ymm1, %ymm0 {%k1}
+; X64-NEXT:    vpshldq $47, %ymm2, %ymm1, %ymm0 {%k1}
 ; X64-NEXT:    retq
 entry:
-  %0 = tail call <4 x i64> @llvm.x86.avx512.vpshld.q.256(<4 x i64> %__A, <4 x i64> %__B, i32 127)
+  %0 = tail call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %__A, <4 x i64> %__B, <4 x i64> <i64 47, i64 47, i64 47, i64 47>)
   %1 = bitcast i8 %__U to <8 x i1>
   %extract = shufflevector <8 x i1> %1, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %2 = select <4 x i1> %extract, <4 x i64> %0, <4 x i64> %__S
   ret <4 x i64> %2
 }
 
-declare <4 x i64> @llvm.x86.avx512.vpshld.q.256(<4 x i64>, <4 x i64>, i32)
+declare <4 x i64> @llvm.fshl.v4i64(<4 x i64>, <4 x i64>, <4 x i64>)
 
 define <4 x i64> @test_mm256_maskz_shldi_epi64(i8 zeroext %__U, <4 x i64> %__A, <4 x i64> %__B) {
 ; X86-LABEL: test_mm256_maskz_shldi_epi64:
@@ -614,7 +614,7 @@ define <4 x i64> @test_mm256_maskz_shldi_epi64(i8 zeroext %__U, <4 x i64> %__A,
 ; X64-NEXT:    vpshldq $63, %ymm1, %ymm0, %ymm0 {%k1} {z}
 ; X64-NEXT:    retq
 entry:
-  %0 = tail call <4 x i64> @llvm.x86.avx512.vpshld.q.256(<4 x i64> %__A, <4 x i64> %__B, i32 63)
+  %0 = tail call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %__A, <4 x i64> %__B, <4 x i64> <i64 63, i64 63, i64 63, i64 63>)
   %1 = bitcast i8 %__U to <8 x i1>
   %extract = shufflevector <8 x i1> %1, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %2 = select <4 x i1> %extract, <4 x i64> %0, <4 x i64> zeroinitializer
@@ -627,7 +627,7 @@ define <4 x i64> @test_mm256_shldi_epi64(<4 x i64> %__A, <4 x i64> %__B) {
 ; CHECK-NEXT:    vpshldq $31, %ymm1, %ymm0, %ymm0
 ; CHECK-NEXT:    ret{{[l|q]}}
 entry:
-  %0 = tail call <4 x i64> @llvm.x86.avx512.vpshld.q.256(<4 x i64> %__A, <4 x i64> %__B, i32 31)
+  %0 = tail call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %__A, <4 x i64> %__B, <4 x i64> <i64 31, i64 31, i64 31, i64 31>)
   ret <4 x i64> %0
 }
 
@@ -636,23 +636,23 @@ define <2 x i64> @test_mm_mask_shldi_epi64(<2 x i64> %__S, i8 zeroext %__U, <2 x
 ; X86:       # %bb.0: # %entry
 ; X86-NEXT:    movb {{[0-9]+}}(%esp), %al
 ; X86-NEXT:    kmovd %eax, %k1
-; X86-NEXT:    vpshldq $127, %xmm2, %xmm1, %xmm0 {%k1}
+; X86-NEXT:    vpshldq $47, %xmm2, %xmm1, %xmm0 {%k1}
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: test_mm_mask_shldi_epi64:
 ; X64:       # %bb.0: # %entry
 ; X64-NEXT:    kmovd %edi, %k1
-; X64-NEXT:    vpshldq $127, %xmm2, %xmm1, %xmm0 {%k1}
+; X64-NEXT:    vpshldq $47, %xmm2, %xmm1, %xmm0 {%k1}
 ; X64-NEXT:    retq
 entry:
-  %0 = tail call <2 x i64> @llvm.x86.avx512.vpshld.q.128(<2 x i64> %__A, <2 x i64> %__B, i32 127)
+  %0 = tail call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %__A, <2 x i64> %__B, <2 x i64> <i64 47, i64 47>)
   %1 = bitcast i8 %__U to <8 x i1>
   %extract = shufflevector <8 x i1> %1, <8 x i1> undef, <2 x i32> <i32 0, i32 1>
   %2 = select <2 x i1> %extract, <2 x i64> %0, <2 x i64> %__S
   ret <2 x i64> %2
 }
 
-declare <2 x i64> @llvm.x86.avx512.vpshld.q.128(<2 x i64>, <2 x i64>, i32) #3
+declare <2 x i64> @llvm.fshl.v2i64(<2 x i64>, <2 x i64>, <2 x i64>)
 
 define <2 x i64> @test_mm_maskz_shldi_epi64(i8 zeroext %__U, <2 x i64> %__A, <2 x i64> %__B) {
 ; X86-LABEL: test_mm_maskz_shldi_epi64:
@@ -668,7 +668,7 @@ define <2 x i64> @test_mm_maskz_shldi_epi64(i8 zeroext %__U, <2 x i64> %__A, <2
 ; X64-NEXT:    vpshldq $63, %xmm1, %xmm0, %xmm0 {%k1} {z}
 ; X64-NEXT:    retq
 entry:
-  %0 = tail call <2 x i64> @llvm.x86.avx512.vpshld.q.128(<2 x i64> %__A, <2 x i64> %__B, i32 63)
+  %0 = tail call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %__A, <2 x i64> %__B, <2 x i64> <i64 63, i64 63>)
   %1 = bitcast i8 %__U to <8 x i1>
   %extract = shufflevector <8 x i1> %1, <8 x i1> undef, <2 x i32> <i32 0, i32 1>
   %2 = select <2 x i1> %extract, <2 x i64> %0, <2 x i64> zeroinitializer
@@ -681,7 +681,7 @@ define <2 x i64> @test_mm_shldi_epi64(<2 x i64> %__A, <2 x i64> %__B) {
 ; CHECK-NEXT:    vpshldq $31, %xmm1, %xmm0, %xmm0
 ; CHECK-NEXT:    ret{{[l|q]}}
 entry:
-  %0 = tail call <2 x i64> @llvm.x86.avx512.vpshld.q.128(<2 x i64> %__A, <2 x i64> %__B, i32 31)
+  %0 = tail call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %__A, <2 x i64> %__B, <2 x i64> <i64 31, i64 31>)
   ret <2 x i64> %0
 }
 
@@ -690,18 +690,18 @@ define <4 x i64> @test_mm256_mask_shldi_epi32(<4 x i64> %__S, i8 zeroext %__U, <
 ; X86:       # %bb.0: # %entry
 ; X86-NEXT:    movb {{[0-9]+}}(%esp), %al
 ; X86-NEXT:    kmovd %eax, %k1
-; X86-NEXT:    vpshldd $127, %ymm2, %ymm1, %ymm0 {%k1}
+; X86-NEXT:    vpshldd $7, %ymm2, %ymm1, %ymm0 {%k1}
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: test_mm256_mask_shldi_epi32:
 ; X64:       # %bb.0: # %entry
 ; X64-NEXT:    kmovd %edi, %k1
-; X64-NEXT:    vpshldd $127, %ymm2, %ymm1, %ymm0 {%k1}
+; X64-NEXT:    vpshldd $7, %ymm2, %ymm1, %ymm0 {%k1}
 ; X64-NEXT:    retq
 entry:
   %0 = bitcast <4 x i64> %__A to <8 x i32>
   %1 = bitcast <4 x i64> %__B to <8 x i32>
-  %2 = tail call <8 x i32> @llvm.x86.avx512.vpshld.d.256(<8 x i32> %0, <8 x i32> %1, i32 127)
+  %2 = tail call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %0, <8 x i32> %1, <8 x i32> <i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7>)
   %3 = bitcast <4 x i64> %__S to <8 x i32>
   %4 = bitcast i8 %__U to <8 x i1>
   %5 = select <8 x i1> %4, <8 x i32> %2, <8 x i32> %3
@@ -709,25 +709,25 @@ entry:
   ret <4 x i64> %6
 }
 
-declare <8 x i32> @llvm.x86.avx512.vpshld.d.256(<8 x i32>, <8 x i32>, i32)
+declare <8 x i32> @llvm.fshl.v8i32(<8 x i32>, <8 x i32>, <8 x i32>)
 
 define <4 x i64> @test_mm256_maskz_shldi_epi32(i8 zeroext %__U, <4 x i64> %__A, <4 x i64> %__B) {
 ; X86-LABEL: test_mm256_maskz_shldi_epi32:
 ; X86:       # %bb.0: # %entry
 ; X86-NEXT:    movb {{[0-9]+}}(%esp), %al
 ; X86-NEXT:    kmovd %eax, %k1
-; X86-NEXT:    vpshldd $63, %ymm1, %ymm0, %ymm0 {%k1} {z}
+; X86-NEXT:    vpshldd $15, %ymm1, %ymm0, %ymm0 {%k1} {z}
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: test_mm256_maskz_shldi_epi32:
 ; X64:       # %bb.0: # %entry
 ; X64-NEXT:    kmovd %edi, %k1
-; X64-NEXT:    vpshldd $63, %ymm1, %ymm0, %ymm0 {%k1} {z}
+; X64-NEXT:    vpshldd $15, %ymm1, %ymm0, %ymm0 {%k1} {z}
 ; X64-NEXT:    retq
 entry:
   %0 = bitcast <4 x i64> %__A to <8 x i32>
   %1 = bitcast <4 x i64> %__B to <8 x i32>
-  %2 = tail call <8 x i32> @llvm.x86.avx512.vpshld.d.256(<8 x i32> %0, <8 x i32> %1, i32 63)
+  %2 = tail call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %0, <8 x i32> %1, <8 x i32> <i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15>)
   %3 = bitcast i8 %__U to <8 x i1>
   %4 = select <8 x i1> %3, <8 x i32> %2, <8 x i32> zeroinitializer
   %5 = bitcast <8 x i32> %4 to <4 x i64>
@@ -742,7 +742,7 @@ define <4 x i64> @test_mm256_shldi_epi32(<4 x i64> %__A, <4 x i64> %__B) {
 entry:
   %0 = bitcast <4 x i64> %__A to <8 x i32>
   %1 = bitcast <4 x i64> %__B to <8 x i32>
-  %2 = tail call <8 x i32> @llvm.x86.avx512.vpshld.d.256(<8 x i32> %0, <8 x i32> %1, i32 31)
+  %2 = tail call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %0, <8 x i32> %1, <8 x i32> <i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31>)
   %3 = bitcast <8 x i32> %2 to <4 x i64>
   ret <4 x i64> %3
 }
@@ -752,18 +752,18 @@ define <2 x i64> @test_mm_mask_shldi_epi32(<2 x i64> %__S, i8 zeroext %__U, <2 x
 ; X86:       # %bb.0: # %entry
 ; X86-NEXT:    movb {{[0-9]+}}(%esp), %al
 ; X86-NEXT:    kmovd %eax, %k1
-; X86-NEXT:    vpshldd $127, %xmm2, %xmm1, %xmm0 {%k1}
+; X86-NEXT:    vpshldd $7, %xmm2, %xmm1, %xmm0 {%k1}
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: test_mm_mask_shldi_epi32:
 ; X64:       # %bb.0: # %entry
 ; X64-NEXT:    kmovd %edi, %k1
-; X64-NEXT:    vpshldd $127, %xmm2, %xmm1, %xmm0 {%k1}
+; X64-NEXT:    vpshldd $7, %xmm2, %xmm1, %xmm0 {%k1}
 ; X64-NEXT:    retq
 entry:
   %0 = bitcast <2 x i64> %__A to <4 x i32>
   %1 = bitcast <2 x i64> %__B to <4 x i32>
-  %2 = tail call <4 x i32> @llvm.x86.avx512.vpshld.d.128(<4 x i32> %0, <4 x i32> %1, i32 127)
+  %2 = tail call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %0, <4 x i32> %1, <4 x i32> <i32 7, i32 7, i32 7, i32 7>)
   %3 = bitcast <2 x i64> %__S to <4 x i32>
   %4 = bitcast i8 %__U to <8 x i1>
   %extract = shufflevector <8 x i1> %4, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
@@ -772,25 +772,25 @@ entry:
   ret <2 x i64> %6
 }
 
-declare <4 x i32> @llvm.x86.avx512.vpshld.d.128(<4 x i32>, <4 x i32>, i32)
+declare <4 x i32> @llvm.fshl.v4i32(<4 x i32>, <4 x i32>, <4 x i32>)
 
 define <2 x i64> @test_mm_maskz_shldi_epi32(i8 zeroext %__U, <2 x i64> %__A, <2 x i64> %__B) {
 ; X86-LABEL: test_mm_maskz_shldi_epi32:
 ; X86:       # %bb.0: # %entry
 ; X86-NEXT:    movb {{[0-9]+}}(%esp), %al
 ; X86-NEXT:    kmovd %eax, %k1
-; X86-NEXT:    vpshldd $63, %xmm1, %xmm0, %xmm0 {%k1} {z}
+; X86-NEXT:    vpshldd $15, %xmm1, %xmm0, %xmm0 {%k1} {z}
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: test_mm_maskz_shldi_epi32:
 ; X64:       # %bb.0: # %entry
 ; X64-NEXT:    kmovd %edi, %k1
-; X64-NEXT:    vpshldd $63, %xmm1, %xmm0, %xmm0 {%k1} {z}
+; X64-NEXT:    vpshldd $15, %xmm1, %xmm0, %xmm0 {%k1} {z}
 ; X64-NEXT:    retq
 entry:
   %0 = bitcast <2 x i64> %__A to <4 x i32>
   %1 = bitcast <2 x i64> %__B to <4 x i32>
-  %2 = tail call <4 x i32> @llvm.x86.avx512.vpshld.d.128(<4 x i32> %0, <4 x i32> %1, i32 63)
+  %2 = tail call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %0, <4 x i32> %1, <4 x i32> <i32 15, i32 15, i32 15, i32 15>)
   %3 = bitcast i8 %__U to <8 x i1>
   %extract = shufflevector <8 x i1> %3, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %4 = select <4 x i1> %extract, <4 x i32> %2, <4 x i32> zeroinitializer
@@ -806,7 +806,7 @@ define <2 x i64> @test_mm_shldi_epi32(<2 x i64> %__A, <2 x i64> %__B) {
 entry:
   %0 = bitcast <2 x i64> %__A to <4 x i32>
   %1 = bitcast <2 x i64> %__B to <4 x i32>
-  %2 = tail call <4 x i32> @llvm.x86.avx512.vpshld.d.128(<4 x i32> %0, <4 x i32> %1, i32 31)
+  %2 = tail call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %0, <4 x i32> %1, <4 x i32> <i32 31, i32 31, i32 31, i32 31>)
   %3 = bitcast <4 x i32> %2 to <2 x i64>
   ret <2 x i64> %3
 }
@@ -815,18 +815,18 @@ define <4 x i64> @test_mm256_mask_shldi_epi16(<4 x i64> %__S, i16 zeroext %__U,
 ; X86-LABEL: test_mm256_mask_shldi_epi16:
 ; X86:       # %bb.0: # %entry
 ; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
-; X86-NEXT:    vpshldw $127, %ymm2, %ymm1, %ymm0 {%k1}
+; X86-NEXT:    vpshldw $3, %ymm2, %ymm1, %ymm0 {%k1}
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: test_mm256_mask_shldi_epi16:
 ; X64:       # %bb.0: # %entry
 ; X64-NEXT:    kmovd %edi, %k1
-; X64-NEXT:    vpshldw $127, %ymm2, %ymm1, %ymm0 {%k1}
+; X64-NEXT:    vpshldw $3, %ymm2, %ymm1, %ymm0 {%k1}
 ; X64-NEXT:    retq
 entry:
   %0 = bitcast <4 x i64> %__A to <16 x i16>
   %1 = bitcast <4 x i64> %__B to <16 x i16>
-  %2 = tail call <16 x i16> @llvm.x86.avx512.vpshld.w.256(<16 x i16> %0, <16 x i16> %1, i32 127)
+  %2 = tail call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %0, <16 x i16> %1, <16 x i16> <i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3>)
   %3 = bitcast <4 x i64> %__S to <16 x i16>
   %4 = bitcast i16 %__U to <16 x i1>
   %5 = select <16 x i1> %4, <16 x i16> %2, <16 x i16> %3
@@ -834,24 +834,24 @@ entry:
   ret <4 x i64> %6
 }
 
-declare <16 x i16> @llvm.x86.avx512.vpshld.w.256(<16 x i16>, <16 x i16>, i32)
+declare <16 x i16> @llvm.fshl.v16i16(<16 x i16>, <16 x i16>, <16 x i16>)
 
 define <4 x i64> @test_mm256_maskz_shldi_epi16(i16 zeroext %__U, <4 x i64> %__A, <4 x i64> %__B) {
 ; X86-LABEL: test_mm256_maskz_shldi_epi16:
 ; X86:       # %bb.0: # %entry
 ; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
-; X86-NEXT:    vpshldw $63, %ymm1, %ymm0, %ymm0 {%k1} {z}
+; X86-NEXT:    vpshldw $7, %ymm1, %ymm0, %ymm0 {%k1} {z}
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: test_mm256_maskz_shldi_epi16:
 ; X64:       # %bb.0: # %entry
 ; X64-NEXT:    kmovd %edi, %k1
-; X64-NEXT:    vpshldw $63, %ymm1, %ymm0, %ymm0 {%k1} {z}
+; X64-NEXT:    vpshldw $7, %ymm1, %ymm0, %ymm0 {%k1} {z}
 ; X64-NEXT:    retq
 entry:
   %0 = bitcast <4 x i64> %__A to <16 x i16>
   %1 = bitcast <4 x i64> %__B to <16 x i16>
-  %2 = tail call <16 x i16> @llvm.x86.avx512.vpshld.w.256(<16 x i16> %0, <16 x i16> %1, i32 63)
+  %2 = tail call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %0, <16 x i16> %1, <16 x i16> <i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7>)
   %3 = bitcast i16 %__U to <16 x i1>
   %4 = select <16 x i1> %3, <16 x i16> %2, <16 x i16> zeroinitializer
   %5 = bitcast <16 x i16> %4 to <4 x i64>
@@ -861,12 +861,12 @@ entry:
 define <4 x i64> @test_mm256_shldi_epi16(<4 x i64> %__A, <4 x i64> %__B) {
 ; CHECK-LABEL: test_mm256_shldi_epi16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vpshldw $31, %ymm1, %ymm0, %ymm0
+; CHECK-NEXT:    vpshldw $15, %ymm1, %ymm0, %ymm0
 ; CHECK-NEXT:    ret{{[l|q]}}
 entry:
   %0 = bitcast <4 x i64> %__A to <16 x i16>
   %1 = bitcast <4 x i64> %__B to <16 x i16>
-  %2 = tail call <16 x i16> @llvm.x86.avx512.vpshld.w.256(<16 x i16> %0, <16 x i16> %1, i32 31)
+  %2 = tail call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %0, <16 x i16> %1, <16 x i16> <i16 31, i16 31, i16 31, i16 31, i16 31, i16 31, i16 31, i16 31, i16 31, i16 31, i16 31, i16 31, i16 31, i16 31, i16 31, i16 31>)
   %3 = bitcast <16 x i16> %2 to <4 x i64>
   ret <4 x i64> %3
 }
@@ -876,18 +876,18 @@ define <2 x i64> @test_mm_mask_shldi_epi16(<2 x i64> %__S, i8 zeroext %__U, <2 x
 ; X86:       # %bb.0: # %entry
 ; X86-NEXT:    movb {{[0-9]+}}(%esp), %al
 ; X86-NEXT:    kmovd %eax, %k1
-; X86-NEXT:    vpshldw $127, %xmm2, %xmm1, %xmm0 {%k1}
+; X86-NEXT:    vpshldw $3, %xmm2, %xmm1, %xmm0 {%k1}
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: test_mm_mask_shldi_epi16:
 ; X64:       # %bb.0: # %entry
 ; X64-NEXT:    kmovd %edi, %k1
-; X64-NEXT:    vpshldw $127, %xmm2, %xmm1, %xmm0 {%k1}
+; X64-NEXT:    vpshldw $3, %xmm2, %xmm1, %xmm0 {%k1}
 ; X64-NEXT:    retq
 entry:
   %0 = bitcast <2 x i64> %__A to <8 x i16>
   %1 = bitcast <2 x i64> %__B to <8 x i16>
-  %2 = tail call <8 x i16> @llvm.x86.avx512.vpshld.w.128(<8 x i16> %0, <8 x i16> %1, i32 127)
+  %2 = tail call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %0, <8 x i16> %1, <8 x i16> <i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3>)
   %3 = bitcast <2 x i64> %__S to <8 x i16>
   %4 = bitcast i8 %__U to <8 x i1>
   %5 = select <8 x i1> %4, <8 x i16> %2, <8 x i16> %3
@@ -895,25 +895,25 @@ entry:
   ret <2 x i64> %6
 }
 
-declare <8 x i16> @llvm.x86.avx512.vpshld.w.128(<8 x i16>, <8 x i16>, i32)
+declare <8 x i16> @llvm.fshl.v8i16(<8 x i16>, <8 x i16>, <8 x i16>)
 
 define <2 x i64> @test_mm_maskz_shldi_epi16(i8 zeroext %__U, <2 x i64> %__A, <2 x i64> %__B) {
 ; X86-LABEL: test_mm_maskz_shldi_epi16:
 ; X86:       # %bb.0: # %entry
 ; X86-NEXT:    movb {{[0-9]+}}(%esp), %al
 ; X86-NEXT:    kmovd %eax, %k1
-; X86-NEXT:    vpshldw $63, %xmm1, %xmm0, %xmm0 {%k1} {z}
+; X86-NEXT:    vpshldw $7, %xmm1, %xmm0, %xmm0 {%k1} {z}
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: test_mm_maskz_shldi_epi16:
 ; X64:       # %bb.0: # %entry
 ; X64-NEXT:    kmovd %edi, %k1
-; X64-NEXT:    vpshldw $63, %xmm1, %xmm0, %xmm0 {%k1} {z}
+; X64-NEXT:    vpshldw $7, %xmm1, %xmm0, %xmm0 {%k1} {z}
 ; X64-NEXT:    retq
 entry:
   %0 = bitcast <2 x i64> %__A to <8 x i16>
   %1 = bitcast <2 x i64> %__B to <8 x i16>
-  %2 = tail call <8 x i16> @llvm.x86.avx512.vpshld.w.128(<8 x i16> %0, <8 x i16> %1, i32 63)
+  %2 = tail call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %0, <8 x i16> %1, <8 x i16> <i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7>)
   %3 = bitcast i8 %__U to <8 x i1>
   %4 = select <8 x i1> %3, <8 x i16> %2, <8 x i16> zeroinitializer
   %5 = bitcast <8 x i16> %4 to <2 x i64>
@@ -923,12 +923,12 @@ entry:
 define <2 x i64> @test_mm_shldi_epi16(<2 x i64> %__A, <2 x i64> %__B) {
 ; CHECK-LABEL: test_mm_shldi_epi16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vpshldw $31, %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    vpshldw $15, %xmm1, %xmm0, %xmm0
 ; CHECK-NEXT:    ret{{[l|q]}}
 entry:
   %0 = bitcast <2 x i64> %__A to <8 x i16>
   %1 = bitcast <2 x i64> %__B to <8 x i16>
-  %2 = tail call <8 x i16> @llvm.x86.avx512.vpshld.w.128(<8 x i16> %0, <8 x i16> %1, i32 31)
+  %2 = tail call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %0, <8 x i16> %1, <8 x i16> <i16 31, i16 31, i16 31, i16 31, i16 31, i16 31, i16 31, i16 31>)
   %3 = bitcast <8 x i16> %2 to <2 x i64>
   ret <2 x i64> %3
 }
@@ -938,23 +938,23 @@ define <4 x i64> @test_mm256_mask_shrdi_epi64(<4 x i64> %__S, i8 zeroext %__U, <
 ; X86:       # %bb.0: # %entry
 ; X86-NEXT:    movb {{[0-9]+}}(%esp), %al
 ; X86-NEXT:    kmovd %eax, %k1
-; X86-NEXT:    vpshrdq $127, %ymm2, %ymm1, %ymm0 {%k1}
+; X86-NEXT:    vpshrdq $47, %ymm2, %ymm1, %ymm0 {%k1}
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: test_mm256_mask_shrdi_epi64:
 ; X64:       # %bb.0: # %entry
 ; X64-NEXT:    kmovd %edi, %k1
-; X64-NEXT:    vpshrdq $127, %ymm2, %ymm1, %ymm0 {%k1}
+; X64-NEXT:    vpshrdq $47, %ymm2, %ymm1, %ymm0 {%k1}
 ; X64-NEXT:    retq
 entry:
-  %0 = tail call <4 x i64> @llvm.x86.avx512.vpshrd.q.256(<4 x i64> %__A, <4 x i64> %__B, i32 127)
+  %0 = tail call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %__B, <4 x i64> %__A, <4 x i64> <i64 47, i64 47, i64 47, i64 47>)
   %1 = bitcast i8 %__U to <8 x i1>
   %extract = shufflevector <8 x i1> %1, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %2 = select <4 x i1> %extract, <4 x i64> %0, <4 x i64> %__S
   ret <4 x i64> %2
 }
 
-declare <4 x i64> @llvm.x86.avx512.vpshrd.q.256(<4 x i64>, <4 x i64>, i32)
+declare <4 x i64> @llvm.fshr.v4i64(<4 x i64>, <4 x i64>, <4 x i64>)
 
 define <4 x i64> @test_mm256_maskz_shrdi_epi64(i8 zeroext %__U, <4 x i64> %__A, <4 x i64> %__B) {
 ; X86-LABEL: test_mm256_maskz_shrdi_epi64:
@@ -970,7 +970,7 @@ define <4 x i64> @test_mm256_maskz_shrdi_epi64(i8 zeroext %__U, <4 x i64> %__A,
 ; X64-NEXT:    vpshrdq $63, %ymm1, %ymm0, %ymm0 {%k1} {z}
 ; X64-NEXT:    retq
 entry:
-  %0 = tail call <4 x i64> @llvm.x86.avx512.vpshrd.q.256(<4 x i64> %__A, <4 x i64> %__B, i32 63)
+  %0 = tail call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %__B, <4 x i64> %__A, <4 x i64> <i64 63, i64 63, i64 63, i64 63>)
   %1 = bitcast i8 %__U to <8 x i1>
   %extract = shufflevector <8 x i1> %1, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %2 = select <4 x i1> %extract, <4 x i64> %0, <4 x i64> zeroinitializer
@@ -983,7 +983,7 @@ define <4 x i64> @test_mm256_shrdi_epi64(<4 x i64> %__A, <4 x i64> %__B) {
 ; CHECK-NEXT:    vpshrdq $31, %ymm1, %ymm0, %ymm0
 ; CHECK-NEXT:    ret{{[l|q]}}
 entry:
-  %0 = tail call <4 x i64> @llvm.x86.avx512.vpshrd.q.256(<4 x i64> %__A, <4 x i64> %__B, i32 31)
+  %0 = tail call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %__B, <4 x i64> %__A, <4 x i64> <i64 31, i64 31, i64 31, i64 31>)
   ret <4 x i64> %0
 }
 
@@ -992,23 +992,23 @@ define <2 x i64> @test_mm_mask_shrdi_epi64(<2 x i64> %__S, i8 zeroext %__U, <2 x
 ; X86:       # %bb.0: # %entry
 ; X86-NEXT:    movb {{[0-9]+}}(%esp), %al
 ; X86-NEXT:    kmovd %eax, %k1
-; X86-NEXT:    vpshrdq $127, %xmm2, %xmm1, %xmm0 {%k1}
+; X86-NEXT:    vpshrdq $47, %xmm2, %xmm1, %xmm0 {%k1}
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: test_mm_mask_shrdi_epi64:
 ; X64:       # %bb.0: # %entry
 ; X64-NEXT:    kmovd %edi, %k1
-; X64-NEXT:    vpshrdq $127, %xmm2, %xmm1, %xmm0 {%k1}
+; X64-NEXT:    vpshrdq $47, %xmm2, %xmm1, %xmm0 {%k1}
 ; X64-NEXT:    retq
 entry:
-  %0 = tail call <2 x i64> @llvm.x86.avx512.vpshrd.q.128(<2 x i64> %__A, <2 x i64> %__B, i32 127)
+  %0 = tail call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %__B, <2 x i64> %__A, <2 x i64> <i64 47, i64 47>)
   %1 = bitcast i8 %__U to <8 x i1>
   %extract = shufflevector <8 x i1> %1, <8 x i1> undef, <2 x i32> <i32 0, i32 1>
   %2 = select <2 x i1> %extract, <2 x i64> %0, <2 x i64> %__S
   ret <2 x i64> %2
 }
 
-declare <2 x i64> @llvm.x86.avx512.vpshrd.q.128(<2 x i64>, <2 x i64>, i32)
+declare <2 x i64> @llvm.fshr.v2i64(<2 x i64>, <2 x i64>, <2 x i64>)
 
 define <2 x i64> @test_mm_maskz_shrdi_epi64(i8 zeroext %__U, <2 x i64> %__A, <2 x i64> %__B) {
 ; X86-LABEL: test_mm_maskz_shrdi_epi64:
@@ -1024,7 +1024,7 @@ define <2 x i64> @test_mm_maskz_shrdi_epi64(i8 zeroext %__U, <2 x i64> %__A, <2
 ; X64-NEXT:    vpshrdq $63, %xmm1, %xmm0, %xmm0 {%k1} {z}
 ; X64-NEXT:    retq
 entry:
-  %0 = tail call <2 x i64> @llvm.x86.avx512.vpshrd.q.128(<2 x i64> %__A, <2 x i64> %__B, i32 63)
+  %0 = tail call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %__B, <2 x i64> %__A, <2 x i64> <i64 63, i64 63>)
   %1 = bitcast i8 %__U to <8 x i1>
   %extract = shufflevector <8 x i1> %1, <8 x i1> undef, <2 x i32> <i32 0, i32 1>
   %2 = select <2 x i1> %extract, <2 x i64> %0, <2 x i64> zeroinitializer
@@ -1037,7 +1037,7 @@ define <2 x i64> @test_mm_shrdi_epi64(<2 x i64> %__A, <2 x i64> %__B) {
 ; CHECK-NEXT:    vpshrdq $31, %xmm1, %xmm0, %xmm0
 ; CHECK-NEXT:    ret{{[l|q]}}
 entry:
-  %0 = tail call <2 x i64> @llvm.x86.avx512.vpshrd.q.128(<2 x i64> %__A, <2 x i64> %__B, i32 31)
+  %0 = tail call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %__B, <2 x i64> %__A, <2 x i64> <i64 31, i64 31>)
   ret <2 x i64> %0
 }
 
@@ -1046,18 +1046,18 @@ define <4 x i64> @test_mm256_mask_shrdi_epi32(<4 x i64> %__S, i8 zeroext %__U, <
 ; X86:       # %bb.0: # %entry
 ; X86-NEXT:    movb {{[0-9]+}}(%esp), %al
 ; X86-NEXT:    kmovd %eax, %k1
-; X86-NEXT:    vpshrdd $127, %ymm2, %ymm1, %ymm0 {%k1}
+; X86-NEXT:    vpshrdd $7, %ymm2, %ymm1, %ymm0 {%k1}
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: test_mm256_mask_shrdi_epi32:
 ; X64:       # %bb.0: # %entry
 ; X64-NEXT:    kmovd %edi, %k1
-; X64-NEXT:    vpshrdd $127, %ymm2, %ymm1, %ymm0 {%k1}
+; X64-NEXT:    vpshrdd $7, %ymm2, %ymm1, %ymm0 {%k1}
 ; X64-NEXT:    retq
 entry:
   %0 = bitcast <4 x i64> %__A to <8 x i32>
   %1 = bitcast <4 x i64> %__B to <8 x i32>
-  %2 = tail call <8 x i32> @llvm.x86.avx512.vpshrd.d.256(<8 x i32> %0, <8 x i32> %1, i32 127)
+  %2 = tail call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %1, <8 x i32> %0, <8 x i32> <i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7>)
   %3 = bitcast <4 x i64> %__S to <8 x i32>
   %4 = bitcast i8 %__U to <8 x i1>
   %5 = select <8 x i1> %4, <8 x i32> %2, <8 x i32> %3
@@ -1065,25 +1065,25 @@ entry:
   ret <4 x i64> %6
 }
 
-declare <8 x i32> @llvm.x86.avx512.vpshrd.d.256(<8 x i32>, <8 x i32>, i32)
+declare <8 x i32> @llvm.fshr.v8i32(<8 x i32>, <8 x i32>, <8 x i32>)
 
 define <4 x i64> @test_mm256_maskz_shrdi_epi32(i8 zeroext %__U, <4 x i64> %__A, <4 x i64> %__B) {
 ; X86-LABEL: test_mm256_maskz_shrdi_epi32:
 ; X86:       # %bb.0: # %entry
 ; X86-NEXT:    movb {{[0-9]+}}(%esp), %al
 ; X86-NEXT:    kmovd %eax, %k1
-; X86-NEXT:    vpshrdd $63, %ymm1, %ymm0, %ymm0 {%k1} {z}
+; X86-NEXT:    vpshrdd $15, %ymm1, %ymm0, %ymm0 {%k1} {z}
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: test_mm256_maskz_shrdi_epi32:
 ; X64:       # %bb.0: # %entry
 ; X64-NEXT:    kmovd %edi, %k1
-; X64-NEXT:    vpshrdd $63, %ymm1, %ymm0, %ymm0 {%k1} {z}
+; X64-NEXT:    vpshrdd $15, %ymm1, %ymm0, %ymm0 {%k1} {z}
 ; X64-NEXT:    retq
 entry:
   %0 = bitcast <4 x i64> %__A to <8 x i32>
   %1 = bitcast <4 x i64> %__B to <8 x i32>
-  %2 = tail call <8 x i32> @llvm.x86.avx512.vpshrd.d.256(<8 x i32> %0, <8 x i32> %1, i32 63)
+  %2 = tail call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %1, <8 x i32> %0, <8 x i32> <i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15>)
   %3 = bitcast i8 %__U to <8 x i1>
   %4 = select <8 x i1> %3, <8 x i32> %2, <8 x i32> zeroinitializer
   %5 = bitcast <8 x i32> %4 to <4 x i64>
@@ -1098,7 +1098,7 @@ define <4 x i64> @test_mm256_shrdi_epi32(<4 x i64> %__A, <4 x i64> %__B) {
 entry:
   %0 = bitcast <4 x i64> %__A to <8 x i32>
   %1 = bitcast <4 x i64> %__B to <8 x i32>
-  %2 = tail call <8 x i32> @llvm.x86.avx512.vpshrd.d.256(<8 x i32> %0, <8 x i32> %1, i32 31)
+  %2 = tail call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %1, <8 x i32> %0, <8 x i32> <i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31>)
   %3 = bitcast <8 x i32> %2 to <4 x i64>
   ret <4 x i64> %3
 }
@@ -1108,18 +1108,18 @@ define <2 x i64> @test_mm_mask_shrdi_epi32(<2 x i64> %__S, i8 zeroext %__U, <2 x
 ; X86:       # %bb.0: # %entry
 ; X86-NEXT:    movb {{[0-9]+}}(%esp), %al
 ; X86-NEXT:    kmovd %eax, %k1
-; X86-NEXT:    vpshrdd $127, %xmm2, %xmm1, %xmm0 {%k1}
+; X86-NEXT:    vpshrdd $7, %xmm2, %xmm1, %xmm0 {%k1}
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: test_mm_mask_shrdi_epi32:
 ; X64:       # %bb.0: # %entry
 ; X64-NEXT:    kmovd %edi, %k1
-; X64-NEXT:    vpshrdd $127, %xmm2, %xmm1, %xmm0 {%k1}
+; X64-NEXT:    vpshrdd $7, %xmm2, %xmm1, %xmm0 {%k1}
 ; X64-NEXT:    retq
 entry:
   %0 = bitcast <2 x i64> %__A to <4 x i32>
   %1 = bitcast <2 x i64> %__B to <4 x i32>
-  %2 = tail call <4 x i32> @llvm.x86.avx512.vpshrd.d.128(<4 x i32> %0, <4 x i32> %1, i32 127)
+  %2 = tail call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %1, <4 x i32> %0, <4 x i32> <i32 7, i32 7, i32 7, i32 7>)
   %3 = bitcast <2 x i64> %__S to <4 x i32>
   %4 = bitcast i8 %__U to <8 x i1>
   %extract = shufflevector <8 x i1> %4, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
@@ -1128,25 +1128,25 @@ entry:
   ret <2 x i64> %6
 }
 
-declare <4 x i32> @llvm.x86.avx512.vpshrd.d.128(<4 x i32>, <4 x i32>, i32)
+declare <4 x i32> @llvm.fshr.v4i32(<4 x i32>, <4 x i32>, <4 x i32>)
 
 define <2 x i64> @test_mm_maskz_shrdi_epi32(i8 zeroext %__U, <2 x i64> %__A, <2 x i64> %__B) {
 ; X86-LABEL: test_mm_maskz_shrdi_epi32:
 ; X86:       # %bb.0: # %entry
 ; X86-NEXT:    movb {{[0-9]+}}(%esp), %al
 ; X86-NEXT:    kmovd %eax, %k1
-; X86-NEXT:    vpshrdd $63, %xmm1, %xmm0, %xmm0 {%k1} {z}
+; X86-NEXT:    vpshrdd $15, %xmm1, %xmm0, %xmm0 {%k1} {z}
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: test_mm_maskz_shrdi_epi32:
 ; X64:       # %bb.0: # %entry
 ; X64-NEXT:    kmovd %edi, %k1
-; X64-NEXT:    vpshrdd $63, %xmm1, %xmm0, %xmm0 {%k1} {z}
+; X64-NEXT:    vpshrdd $15, %xmm1, %xmm0, %xmm0 {%k1} {z}
 ; X64-NEXT:    retq
 entry:
   %0 = bitcast <2 x i64> %__A to <4 x i32>
   %1 = bitcast <2 x i64> %__B to <4 x i32>
-  %2 = tail call <4 x i32> @llvm.x86.avx512.vpshrd.d.128(<4 x i32> %0, <4 x i32> %1, i32 63)
+  %2 = tail call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %1, <4 x i32> %0, <4 x i32> <i32 15, i32 15, i32 15, i32 15>)
   %3 = bitcast i8 %__U to <8 x i1>
   %extract = shufflevector <8 x i1> %3, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %4 = select <4 x i1> %extract, <4 x i32> %2, <4 x i32> zeroinitializer
@@ -1162,7 +1162,7 @@ define <2 x i64> @test_mm_shrdi_epi32(<2 x i64> %__A, <2 x i64> %__B) {
 entry:
   %0 = bitcast <2 x i64> %__A to <4 x i32>
   %1 = bitcast <2 x i64> %__B to <4 x i32>
-  %2 = tail call <4 x i32> @llvm.x86.avx512.vpshrd.d.128(<4 x i32> %0, <4 x i32> %1, i32 31)
+  %2 = tail call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %1, <4 x i32> %0, <4 x i32> <i32 31, i32 31, i32 31, i32 31>)
   %3 = bitcast <4 x i32> %2 to <2 x i64>
   ret <2 x i64> %3
 }
@@ -1171,18 +1171,18 @@ define <4 x i64> @test_mm256_mask_shrdi_epi16(<4 x i64> %__S, i16 zeroext %__U,
 ; X86-LABEL: test_mm256_mask_shrdi_epi16:
 ; X86:       # %bb.0: # %entry
 ; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
-; X86-NEXT:    vpshrdw $127, %ymm2, %ymm1, %ymm0 {%k1}
+; X86-NEXT:    vpshrdw $3, %ymm2, %ymm1, %ymm0 {%k1}
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: test_mm256_mask_shrdi_epi16:
 ; X64:       # %bb.0: # %entry
 ; X64-NEXT:    kmovd %edi, %k1
-; X64-NEXT:    vpshrdw $127, %ymm2, %ymm1, %ymm0 {%k1}
+; X64-NEXT:    vpshrdw $3, %ymm2, %ymm1, %ymm0 {%k1}
 ; X64-NEXT:    retq
 entry:
   %0 = bitcast <4 x i64> %__A to <16 x i16>
   %1 = bitcast <4 x i64> %__B to <16 x i16>
-  %2 = tail call <16 x i16> @llvm.x86.avx512.vpshrd.w.256(<16 x i16> %0, <16 x i16> %1, i32 127)
+  %2 = tail call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %1, <16 x i16> %0, <16 x i16> <i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3>)
   %3 = bitcast <4 x i64> %__S to <16 x i16>
   %4 = bitcast i16 %__U to <16 x i1>
   %5 = select <16 x i1> %4, <16 x i16> %2, <16 x i16> %3
@@ -1190,24 +1190,24 @@ entry:
   ret <4 x i64> %6
 }
 
-declare <16 x i16> @llvm.x86.avx512.vpshrd.w.256(<16 x i16>, <16 x i16>, i32)
+declare <16 x i16> @llvm.fshr.v16i16(<16 x i16>, <16 x i16>, <16 x i16>)
 
 define <4 x i64> @test_mm256_maskz_shrdi_epi16(i16 zeroext %__U, <4 x i64> %__A, <4 x i64> %__B) {
 ; X86-LABEL: test_mm256_maskz_shrdi_epi16:
 ; X86:       # %bb.0: # %entry
 ; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
-; X86-NEXT:    vpshrdw $63, %ymm1, %ymm0, %ymm0 {%k1} {z}
+; X86-NEXT:    vpshrdw $7, %ymm1, %ymm0, %ymm0 {%k1} {z}
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: test_mm256_maskz_shrdi_epi16:
 ; X64:       # %bb.0: # %entry
 ; X64-NEXT:    kmovd %edi, %k1
-; X64-NEXT:    vpshrdw $63, %ymm1, %ymm0, %ymm0 {%k1} {z}
+; X64-NEXT:    vpshrdw $7, %ymm1, %ymm0, %ymm0 {%k1} {z}
 ; X64-NEXT:    retq
 entry:
   %0 = bitcast <4 x i64> %__A to <16 x i16>
   %1 = bitcast <4 x i64> %__B to <16 x i16>
-  %2 = tail call <16 x i16> @llvm.x86.avx512.vpshrd.w.256(<16 x i16> %0, <16 x i16> %1, i32 63)
+  %2 = tail call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %1, <16 x i16> %0, <16 x i16> <i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7>)
   %3 = bitcast i16 %__U to <16 x i1>
   %4 = select <16 x i1> %3, <16 x i16> %2, <16 x i16> zeroinitializer
   %5 = bitcast <16 x i16> %4 to <4 x i64>
@@ -1217,12 +1217,12 @@ entry:
 define <4 x i64> @test_mm256_shrdi_epi16(<4 x i64> %__A, <4 x i64> %__B) {
 ; CHECK-LABEL: test_mm256_shrdi_epi16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vpshrdw $31, %ymm1, %ymm0, %ymm0
+; CHECK-NEXT:    vpshrdw $15, %ymm1, %ymm0, %ymm0
 ; CHECK-NEXT:    ret{{[l|q]}}
 entry:
   %0 = bitcast <4 x i64> %__A to <16 x i16>
   %1 = bitcast <4 x i64> %__B to <16 x i16>
-  %2 = tail call <16 x i16> @llvm.x86.avx512.vpshrd.w.256(<16 x i16> %0, <16 x i16> %1, i32 31)
+  %2 = tail call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %1, <16 x i16> %0, <16 x i16> <i16 31, i16 31, i16 31, i16 31, i16 31, i16 31, i16 31, i16 31, i16 31, i16 31, i16 31, i16 31, i16 31, i16 31, i16 31, i16 31>)
   %3 = bitcast <16 x i16> %2 to <4 x i64>
   ret <4 x i64> %3
 }
@@ -1232,18 +1232,18 @@ define <2 x i64> @test_mm_mask_shrdi_epi16(<2 x i64> %__S, i8 zeroext %__U, <2 x
 ; X86:       # %bb.0: # %entry
 ; X86-NEXT:    movb {{[0-9]+}}(%esp), %al
 ; X86-NEXT:    kmovd %eax, %k1
-; X86-NEXT:    vpshrdw $127, %xmm2, %xmm1, %xmm0 {%k1}
+; X86-NEXT:    vpshrdw $3, %xmm2, %xmm1, %xmm0 {%k1}
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: test_mm_mask_shrdi_epi16:
 ; X64:       # %bb.0: # %entry
 ; X64-NEXT:    kmovd %edi, %k1
-; X64-NEXT:    vpshrdw $127, %xmm2, %xmm1, %xmm0 {%k1}
+; X64-NEXT:    vpshrdw $3, %xmm2, %xmm1, %xmm0 {%k1}
 ; X64-NEXT:    retq
 entry:
   %0 = bitcast <2 x i64> %__A to <8 x i16>
   %1 = bitcast <2 x i64> %__B to <8 x i16>
-  %2 = tail call <8 x i16> @llvm.x86.avx512.vpshrd.w.128(<8 x i16> %0, <8 x i16> %1, i32 127)
+  %2 = tail call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %1, <8 x i16> %0, <8 x i16> <i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3, i16 3>)
   %3 = bitcast <2 x i64> %__S to <8 x i16>
   %4 = bitcast i8 %__U to <8 x i1>
   %5 = select <8 x i1> %4, <8 x i16> %2, <8 x i16> %3
@@ -1251,25 +1251,25 @@ entry:
   ret <2 x i64> %6
 }
 
-declare <8 x i16> @llvm.x86.avx512.vpshrd.w.128(<8 x i16>, <8 x i16>, i32)
+declare <8 x i16> @llvm.fshr.v8i16(<8 x i16>, <8 x i16>, <8 x i16>)
 
 define <2 x i64> @test_mm_maskz_shrdi_epi16(i8 zeroext %__U, <2 x i64> %__A, <2 x i64> %__B) {
 ; X86-LABEL: test_mm_maskz_shrdi_epi16:
 ; X86:       # %bb.0: # %entry
 ; X86-NEXT:    movb {{[0-9]+}}(%esp), %al
 ; X86-NEXT:    kmovd %eax, %k1
-; X86-NEXT:    vpshrdw $63, %xmm1, %xmm0, %xmm0 {%k1} {z}
+; X86-NEXT:    vpshrdw $7, %xmm1, %xmm0, %xmm0 {%k1} {z}
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: test_mm_maskz_shrdi_epi16:
 ; X64:       # %bb.0: # %entry
 ; X64-NEXT:    kmovd %edi, %k1
-; X64-NEXT:    vpshrdw $63, %xmm1, %xmm0, %xmm0 {%k1} {z}
+; X64-NEXT:    vpshrdw $7, %xmm1, %xmm0, %xmm0 {%k1} {z}
 ; X64-NEXT:    retq
 entry:
   %0 = bitcast <2 x i64> %__A to <8 x i16>
   %1 = bitcast <2 x i64> %__B to <8 x i16>
-  %2 = tail call <8 x i16> @llvm.x86.avx512.vpshrd.w.128(<8 x i16> %0, <8 x i16> %1, i32 63)
+  %2 = tail call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %1, <8 x i16> %0, <8 x i16> <i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7>)
   %3 = bitcast i8 %__U to <8 x i1>
   %4 = select <8 x i1> %3, <8 x i16> %2, <8 x i16> zeroinitializer
   %5 = bitcast <8 x i16> %4 to <2 x i64>
@@ -1279,12 +1279,12 @@ entry:
 define <2 x i64> @test_mm_shrdi_epi16(<2 x i64> %__A, <2 x i64> %__B) {
 ; CHECK-LABEL: test_mm_shrdi_epi16:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vpshrdw $31, %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    vpshrdw $15, %xmm1, %xmm0, %xmm0
 ; CHECK-NEXT:    ret{{[l|q]}}
 entry:
   %0 = bitcast <2 x i64> %__A to <8 x i16>
   %1 = bitcast <2 x i64> %__B to <8 x i16>
-  %2 = tail call <8 x i16> @llvm.x86.avx512.vpshrd.w.128(<8 x i16> %0, <8 x i16> %1, i32 31)
+  %2 = tail call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %1, <8 x i16> %0, <8 x i16> <i16 31, i16 31, i16 31, i16 31, i16 31, i16 31, i16 31, i16 31>)
   %3 = bitcast <8 x i16> %2 to <2 x i64>
   ret <2 x i64> %3
 }
@@ -1303,8 +1303,11 @@ define <4 x i64> @test_mm256_mask_shldv_epi64(<4 x i64> %__S, i8 zeroext %__U, <
 ; X64-NEXT:    vpshldvq %ymm2, %ymm1, %ymm0 {%k1}
 ; X64-NEXT:    retq
 entry:
-  %0 = tail call <4 x i64> @llvm.x86.avx512.mask.vpshldv.q.256(<4 x i64> %__S, <4 x i64> %__A, <4 x i64> %__B, i8 %__U)
-  ret <4 x i64> %0
+  %0 = tail call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %__S, <4 x i64> %__A, <4 x i64> %__B)
+  %1 = bitcast i8 %__U to <8 x i1>
+  %extract.i = shufflevector <8 x i1> %1, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %2 = select <4 x i1> %extract.i, <4 x i64> %0, <4 x i64> %__S
+  ret <4 x i64> %2
 }
 
 define <4 x i64> @test_mm256_maskz_shldv_epi64(i8 zeroext %__U, <4 x i64> %__S, <4 x i64> %__A, <4 x i64> %__B) {
@@ -1321,8 +1324,11 @@ define <4 x i64> @test_mm256_maskz_shldv_epi64(i8 zeroext %__U, <4 x i64> %__S,
 ; X64-NEXT:    vpshldvq %ymm2, %ymm1, %ymm0 {%k1} {z}
 ; X64-NEXT:    retq
 entry:
-  %0 = tail call <4 x i64> @llvm.x86.avx512.maskz.vpshldv.q.256(<4 x i64> %__S, <4 x i64> %__A, <4 x i64> %__B, i8 %__U)
-  ret <4 x i64> %0
+  %0 = tail call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %__S, <4 x i64> %__A, <4 x i64> %__B)
+  %1 = bitcast i8 %__U to <8 x i1>
+  %extract.i = shufflevector <8 x i1> %1, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %2 = select <4 x i1> %extract.i, <4 x i64> %0, <4 x i64> zeroinitializer
+  ret <4 x i64> %2
 }
 
 define <4 x i64> @test_mm256_shldv_epi64(<4 x i64> %__S, <4 x i64> %__A, <4 x i64> %__B) {
@@ -1331,7 +1337,7 @@ define <4 x i64> @test_mm256_shldv_epi64(<4 x i64> %__S, <4 x i64> %__A, <4 x i6
 ; CHECK-NEXT:    vpshldvq %ymm2, %ymm1, %ymm0
 ; CHECK-NEXT:    ret{{[l|q]}}
 entry:
-  %0 = tail call <4 x i64> @llvm.x86.avx512.mask.vpshldv.q.256(<4 x i64> %__S, <4 x i64> %__A, <4 x i64> %__B, i8 -1)
+  %0 = tail call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %__S, <4 x i64> %__A, <4 x i64> %__B)
   ret <4 x i64> %0
 }
 
@@ -1349,8 +1355,11 @@ define <2 x i64> @test_mm_mask_shldv_epi64(<2 x i64> %__S, i8 zeroext %__U, <2 x
 ; X64-NEXT:    vpshldvq %xmm2, %xmm1, %xmm0 {%k1}
 ; X64-NEXT:    retq
 entry:
-  %0 = tail call <2 x i64> @llvm.x86.avx512.mask.vpshldv.q.128(<2 x i64> %__S, <2 x i64> %__A, <2 x i64> %__B, i8 %__U)
-  ret <2 x i64> %0
+  %0 = tail call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %__S, <2 x i64> %__A, <2 x i64> %__B)
+  %1 = bitcast i8 %__U to <8 x i1>
+  %extract.i = shufflevector <8 x i1> %1, <8 x i1> undef, <2 x i32> <i32 0, i32 1>
+  %2 = select <2 x i1> %extract.i, <2 x i64> %0, <2 x i64> %__S
+  ret <2 x i64> %2
 }
 
 define <2 x i64> @test_mm_maskz_shldv_epi64(i8 zeroext %__U, <2 x i64> %__S, <2 x i64> %__A, <2 x i64> %__B) {
@@ -1367,8 +1376,11 @@ define <2 x i64> @test_mm_maskz_shldv_epi64(i8 zeroext %__U, <2 x i64> %__S, <2
 ; X64-NEXT:    vpshldvq %xmm2, %xmm1, %xmm0 {%k1} {z}
 ; X64-NEXT:    retq
 entry:
-  %0 = tail call <2 x i64> @llvm.x86.avx512.maskz.vpshldv.q.128(<2 x i64> %__S, <2 x i64> %__A, <2 x i64> %__B, i8 %__U)
-  ret <2 x i64> %0
+  %0 = tail call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %__S, <2 x i64> %__A, <2 x i64> %__B)
+  %1 = bitcast i8 %__U to <8 x i1>
+  %extract.i = shufflevector <8 x i1> %1, <8 x i1> undef, <2 x i32> <i32 0, i32 1>
+  %2 = select <2 x i1> %extract.i, <2 x i64> %0, <2 x i64> zeroinitializer
+  ret <2 x i64> %2
 }
 
 define <2 x i64> @test_mm_shldv_epi64(<2 x i64> %__S, <2 x i64> %__A, <2 x i64> %__B) {
@@ -1377,7 +1389,7 @@ define <2 x i64> @test_mm_shldv_epi64(<2 x i64> %__S, <2 x i64> %__A, <2 x i64>
 ; CHECK-NEXT:    vpshldvq %xmm2, %xmm1, %xmm0
 ; CHECK-NEXT:    ret{{[l|q]}}
 entry:
-  %0 = tail call <2 x i64> @llvm.x86.avx512.mask.vpshldv.q.128(<2 x i64> %__S, <2 x i64> %__A, <2 x i64> %__B, i8 -1)
+  %0 = tail call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %__S, <2 x i64> %__A, <2 x i64> %__B)
   ret <2 x i64> %0
 }
 
@@ -1398,9 +1410,11 @@ entry:
   %0 = bitcast <4 x i64> %__S to <8 x i32>
   %1 = bitcast <4 x i64> %__A to <8 x i32>
   %2 = bitcast <4 x i64> %__B to <8 x i32>
-  %3 = tail call <8 x i32> @llvm.x86.avx512.mask.vpshldv.d.256(<8 x i32> %0, <8 x i32> %1, <8 x i32> %2, i8 %__U)
-  %4 = bitcast <8 x i32> %3 to <4 x i64>
-  ret <4 x i64> %4
+  %3 = tail call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %0, <8 x i32> %1, <8 x i32> %2)
+  %4 = bitcast i8 %__U to <8 x i1>
+  %5 = select <8 x i1> %4, <8 x i32> %3, <8 x i32> %0
+  %6 = bitcast <8 x i32> %5 to <4 x i64>
+  ret <4 x i64> %6
 }
 
 define <4 x i64> @test_mm256_maskz_shldv_epi32(i8 zeroext %__U, <4 x i64> %__S, <4 x i64> %__A, <4 x i64> %__B) {
@@ -1420,9 +1434,11 @@ entry:
   %0 = bitcast <4 x i64> %__S to <8 x i32>
   %1 = bitcast <4 x i64> %__A to <8 x i32>
   %2 = bitcast <4 x i64> %__B to <8 x i32>
-  %3 = tail call <8 x i32> @llvm.x86.avx512.maskz.vpshldv.d.256(<8 x i32> %0, <8 x i32> %1, <8 x i32> %2, i8 %__U)
-  %4 = bitcast <8 x i32> %3 to <4 x i64>
-  ret <4 x i64> %4
+  %3 = tail call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %0, <8 x i32> %1, <8 x i32> %2)
+  %4 = bitcast i8 %__U to <8 x i1>
+  %5 = select <8 x i1> %4, <8 x i32> %3, <8 x i32> zeroinitializer
+  %6 = bitcast <8 x i32> %5 to <4 x i64>
+  ret <4 x i64> %6
 }
 
 define <4 x i64> @test_mm256_shldv_epi32(<4 x i64> %__S, <4 x i64> %__A, <4 x i64> %__B) {
@@ -1434,7 +1450,7 @@ entry:
   %0 = bitcast <4 x i64> %__S to <8 x i32>
   %1 = bitcast <4 x i64> %__A to <8 x i32>
   %2 = bitcast <4 x i64> %__B to <8 x i32>
-  %3 = tail call <8 x i32> @llvm.x86.avx512.mask.vpshldv.d.256(<8 x i32> %0, <8 x i32> %1, <8 x i32> %2, i8 -1)
+  %3 = tail call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %0, <8 x i32> %1, <8 x i32> %2)
   %4 = bitcast <8 x i32> %3 to <4 x i64>
   ret <4 x i64> %4
 }
@@ -1456,9 +1472,12 @@ entry:
   %0 = bitcast <2 x i64> %__S to <4 x i32>
   %1 = bitcast <2 x i64> %__A to <4 x i32>
   %2 = bitcast <2 x i64> %__B to <4 x i32>
-  %3 = tail call <4 x i32> @llvm.x86.avx512.mask.vpshldv.d.128(<4 x i32> %0, <4 x i32> %1, <4 x i32> %2, i8 %__U)
-  %4 = bitcast <4 x i32> %3 to <2 x i64>
-  ret <2 x i64> %4
+  %3 = tail call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %0, <4 x i32> %1, <4 x i32> %2)
+  %4 = bitcast i8 %__U to <8 x i1>
+  %extract.i = shufflevector <8 x i1> %4, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %5 = select <4 x i1> %extract.i, <4 x i32> %3, <4 x i32> %0
+  %6 = bitcast <4 x i32> %5 to <2 x i64>
+  ret <2 x i64> %6
 }
 
 define <2 x i64> @test_mm_maskz_shldv_epi32(i8 zeroext %__U, <2 x i64> %__S, <2 x i64> %__A, <2 x i64> %__B) {
@@ -1478,9 +1497,12 @@ entry:
   %0 = bitcast <2 x i64> %__S to <4 x i32>
   %1 = bitcast <2 x i64> %__A to <4 x i32>
   %2 = bitcast <2 x i64> %__B to <4 x i32>
-  %3 = tail call <4 x i32> @llvm.x86.avx512.maskz.vpshldv.d.128(<4 x i32> %0, <4 x i32> %1, <4 x i32> %2, i8 %__U)
-  %4 = bitcast <4 x i32> %3 to <2 x i64>
-  ret <2 x i64> %4
+  %3 = tail call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %0, <4 x i32> %1, <4 x i32> %2)
+  %4 = bitcast i8 %__U to <8 x i1>
+  %extract.i = shufflevector <8 x i1> %4, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %5 = select <4 x i1> %extract.i, <4 x i32> %3, <4 x i32> zeroinitializer
+  %6 = bitcast <4 x i32> %5 to <2 x i64>
+  ret <2 x i64> %6
 }
 
 define <2 x i64> @test_mm_shldv_epi32(<2 x i64> %__S, <2 x i64> %__A, <2 x i64> %__B) {
@@ -1492,7 +1514,7 @@ entry:
   %0 = bitcast <2 x i64> %__S to <4 x i32>
   %1 = bitcast <2 x i64> %__A to <4 x i32>
   %2 = bitcast <2 x i64> %__B to <4 x i32>
-  %3 = tail call <4 x i32> @llvm.x86.avx512.mask.vpshldv.d.128(<4 x i32> %0, <4 x i32> %1, <4 x i32> %2, i8 -1)
+  %3 = tail call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %0, <4 x i32> %1, <4 x i32> %2)
   %4 = bitcast <4 x i32> %3 to <2 x i64>
   ret <2 x i64> %4
 }
@@ -1513,9 +1535,11 @@ entry:
   %0 = bitcast <4 x i64> %__S to <16 x i16>
   %1 = bitcast <4 x i64> %__A to <16 x i16>
   %2 = bitcast <4 x i64> %__B to <16 x i16>
-  %3 = tail call <16 x i16> @llvm.x86.avx512.mask.vpshldv.w.256(<16 x i16> %0, <16 x i16> %1, <16 x i16> %2, i16 %__U)
-  %4 = bitcast <16 x i16> %3 to <4 x i64>
-  ret <4 x i64> %4
+  %3 = tail call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %0, <16 x i16> %1, <16 x i16> %2)
+  %4 = bitcast i16 %__U to <16 x i1>
+  %5 = select <16 x i1> %4, <16 x i16> %3, <16 x i16> %0
+  %6 = bitcast <16 x i16> %5 to <4 x i64>
+  ret <4 x i64> %6
 }
 
 define <4 x i64> @test_mm256_maskz_shldv_epi16(i16 zeroext %__U, <4 x i64> %__S, <4 x i64> %__A, <4 x i64> %__B) {
@@ -1534,9 +1558,11 @@ entry:
   %0 = bitcast <4 x i64> %__S to <16 x i16>
   %1 = bitcast <4 x i64> %__A to <16 x i16>
   %2 = bitcast <4 x i64> %__B to <16 x i16>
-  %3 = tail call <16 x i16> @llvm.x86.avx512.maskz.vpshldv.w.256(<16 x i16> %0, <16 x i16> %1, <16 x i16> %2, i16 %__U)
-  %4 = bitcast <16 x i16> %3 to <4 x i64>
-  ret <4 x i64> %4
+  %3 = tail call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %0, <16 x i16> %1, <16 x i16> %2)
+  %4 = bitcast i16 %__U to <16 x i1>
+  %5 = select <16 x i1> %4, <16 x i16> %3, <16 x i16> zeroinitializer
+  %6 = bitcast <16 x i16> %5 to <4 x i64>
+  ret <4 x i64> %6
 }
 
 define <4 x i64> @test_mm256_shldv_epi16(<4 x i64> %__S, <4 x i64> %__A, <4 x i64> %__B) {
@@ -1548,7 +1574,7 @@ entry:
   %0 = bitcast <4 x i64> %__S to <16 x i16>
   %1 = bitcast <4 x i64> %__A to <16 x i16>
   %2 = bitcast <4 x i64> %__B to <16 x i16>
-  %3 = tail call <16 x i16> @llvm.x86.avx512.mask.vpshldv.w.256(<16 x i16> %0, <16 x i16> %1, <16 x i16> %2, i16 -1)
+  %3 = tail call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %0, <16 x i16> %1, <16 x i16> %2)
   %4 = bitcast <16 x i16> %3 to <4 x i64>
   ret <4 x i64> %4
 }
@@ -1570,9 +1596,11 @@ entry:
   %0 = bitcast <2 x i64> %__S to <8 x i16>
   %1 = bitcast <2 x i64> %__A to <8 x i16>
   %2 = bitcast <2 x i64> %__B to <8 x i16>
-  %3 = tail call <8 x i16> @llvm.x86.avx512.mask.vpshldv.w.128(<8 x i16> %0, <8 x i16> %1, <8 x i16> %2, i8 %__U)
-  %4 = bitcast <8 x i16> %3 to <2 x i64>
-  ret <2 x i64> %4
+  %3 = tail call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %0, <8 x i16> %1, <8 x i16> %2)
+  %4 = bitcast i8 %__U to <8 x i1>
+  %5 = select <8 x i1> %4, <8 x i16> %3, <8 x i16> %0
+  %6 = bitcast <8 x i16> %5 to <2 x i64>
+  ret <2 x i64> %6
 }
 
 define <2 x i64> @test_mm_maskz_shldv_epi16(i8 zeroext %__U, <2 x i64> %__S, <2 x i64> %__A, <2 x i64> %__B) {
@@ -1592,9 +1620,11 @@ entry:
   %0 = bitcast <2 x i64> %__S to <8 x i16>
   %1 = bitcast <2 x i64> %__A to <8 x i16>
   %2 = bitcast <2 x i64> %__B to <8 x i16>
-  %3 = tail call <8 x i16> @llvm.x86.avx512.maskz.vpshldv.w.128(<8 x i16> %0, <8 x i16> %1, <8 x i16> %2, i8 %__U)
-  %4 = bitcast <8 x i16> %3 to <2 x i64>
-  ret <2 x i64> %4
+  %3 = tail call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %0, <8 x i16> %1, <8 x i16> %2)
+  %4 = bitcast i8 %__U to <8 x i1>
+  %5 = select <8 x i1> %4, <8 x i16> %3, <8 x i16> zeroinitializer
+  %6 = bitcast <8 x i16> %5 to <2 x i64>
+  ret <2 x i64> %6
 }
 
 define <2 x i64> @test_mm_shldv_epi16(<2 x i64> %__S, <2 x i64> %__A, <2 x i64> %__B) {
@@ -1606,7 +1636,7 @@ entry:
   %0 = bitcast <2 x i64> %__S to <8 x i16>
   %1 = bitcast <2 x i64> %__A to <8 x i16>
   %2 = bitcast <2 x i64> %__B to <8 x i16>
-  %3 = tail call <8 x i16> @llvm.x86.avx512.mask.vpshldv.w.128(<8 x i16> %0, <8 x i16> %1, <8 x i16> %2, i8 -1)
+  %3 = tail call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %0, <8 x i16> %1, <8 x i16> %2)
   %4 = bitcast <8 x i16> %3 to <2 x i64>
   ret <2 x i64> %4
 }
@@ -1625,8 +1655,11 @@ define <4 x i64> @test_mm256_mask_shrdv_epi64(<4 x i64> %__S, i8 zeroext %__U, <
 ; X64-NEXT:    vpshrdvq %ymm2, %ymm1, %ymm0 {%k1}
 ; X64-NEXT:    retq
 entry:
-  %0 = tail call <4 x i64> @llvm.x86.avx512.mask.vpshrdv.q.256(<4 x i64> %__S, <4 x i64> %__A, <4 x i64> %__B, i8 %__U)
-  ret <4 x i64> %0
+  %0 = tail call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %__A, <4 x i64> %__S, <4 x i64> %__B)
+  %1 = bitcast i8 %__U to <8 x i1>
+  %extract.i = shufflevector <8 x i1> %1, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %2 = select <4 x i1> %extract.i, <4 x i64> %0, <4 x i64> %__S
+  ret <4 x i64> %2
 }
 
 define <4 x i64> @test_mm256_maskz_shrdv_epi64(i8 zeroext %__U, <4 x i64> %__S, <4 x i64> %__A, <4 x i64> %__B) {
@@ -1643,8 +1676,11 @@ define <4 x i64> @test_mm256_maskz_shrdv_epi64(i8 zeroext %__U, <4 x i64> %__S,
 ; X64-NEXT:    vpshrdvq %ymm2, %ymm1, %ymm0 {%k1} {z}
 ; X64-NEXT:    retq
 entry:
-  %0 = tail call <4 x i64> @llvm.x86.avx512.maskz.vpshrdv.q.256(<4 x i64> %__S, <4 x i64> %__A, <4 x i64> %__B, i8 %__U)
-  ret <4 x i64> %0
+  %0 = tail call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %__A, <4 x i64> %__S, <4 x i64> %__B)
+  %1 = bitcast i8 %__U to <8 x i1>
+  %extract.i = shufflevector <8 x i1> %1, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %2 = select <4 x i1> %extract.i, <4 x i64> %0, <4 x i64> zeroinitializer
+  ret <4 x i64> %2
 }
 
 define <4 x i64> @test_mm256_shrdv_epi64(<4 x i64> %__S, <4 x i64> %__A, <4 x i64> %__B) {
@@ -1653,7 +1689,7 @@ define <4 x i64> @test_mm256_shrdv_epi64(<4 x i64> %__S, <4 x i64> %__A, <4 x i6
 ; CHECK-NEXT:    vpshrdvq %ymm2, %ymm1, %ymm0
 ; CHECK-NEXT:    ret{{[l|q]}}
 entry:
-  %0 = tail call <4 x i64> @llvm.x86.avx512.mask.vpshrdv.q.256(<4 x i64> %__S, <4 x i64> %__A, <4 x i64> %__B, i8 -1)
+  %0 = tail call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %__A, <4 x i64> %__S, <4 x i64> %__B)
   ret <4 x i64> %0
 }
 
@@ -1671,8 +1707,11 @@ define <2 x i64> @test_mm_mask_shrdv_epi64(<2 x i64> %__S, i8 zeroext %__U, <2 x
 ; X64-NEXT:    vpshrdvq %xmm2, %xmm1, %xmm0 {%k1}
 ; X64-NEXT:    retq
 entry:
-  %0 = tail call <2 x i64> @llvm.x86.avx512.mask.vpshrdv.q.128(<2 x i64> %__S, <2 x i64> %__A, <2 x i64> %__B, i8 %__U)
-  ret <2 x i64> %0
+  %0 = tail call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %__A, <2 x i64> %__S, <2 x i64> %__B)
+  %1 = bitcast i8 %__U to <8 x i1>
+  %extract.i = shufflevector <8 x i1> %1, <8 x i1> undef, <2 x i32> <i32 0, i32 1>
+  %2 = select <2 x i1> %extract.i, <2 x i64> %0, <2 x i64> %__S
+  ret <2 x i64> %2
 }
 
 define <2 x i64> @test_mm_maskz_shrdv_epi64(i8 zeroext %__U, <2 x i64> %__S, <2 x i64> %__A, <2 x i64> %__B) {
@@ -1689,8 +1728,11 @@ define <2 x i64> @test_mm_maskz_shrdv_epi64(i8 zeroext %__U, <2 x i64> %__S, <2
 ; X64-NEXT:    vpshrdvq %xmm2, %xmm1, %xmm0 {%k1} {z}
 ; X64-NEXT:    retq
 entry:
-  %0 = tail call <2 x i64> @llvm.x86.avx512.maskz.vpshrdv.q.128(<2 x i64> %__S, <2 x i64> %__A, <2 x i64> %__B, i8 %__U)
-  ret <2 x i64> %0
+  %0 = tail call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %__A, <2 x i64> %__S, <2 x i64> %__B)
+  %1 = bitcast i8 %__U to <8 x i1>
+  %extract.i = shufflevector <8 x i1> %1, <8 x i1> undef, <2 x i32> <i32 0, i32 1>
+  %2 = select <2 x i1> %extract.i, <2 x i64> %0, <2 x i64> zeroinitializer
+  ret <2 x i64> %2
 }
 
 define <2 x i64> @test_mm_shrdv_epi64(<2 x i64> %__S, <2 x i64> %__A, <2 x i64> %__B) {
@@ -1699,7 +1741,7 @@ define <2 x i64> @test_mm_shrdv_epi64(<2 x i64> %__S, <2 x i64> %__A, <2 x i64>
 ; CHECK-NEXT:    vpshrdvq %xmm2, %xmm1, %xmm0
 ; CHECK-NEXT:    ret{{[l|q]}}
 entry:
-  %0 = tail call <2 x i64> @llvm.x86.avx512.mask.vpshrdv.q.128(<2 x i64> %__S, <2 x i64> %__A, <2 x i64> %__B, i8 -1)
+  %0 = tail call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %__A, <2 x i64> %__S, <2 x i64> %__B)
   ret <2 x i64> %0
 }
 
@@ -1720,9 +1762,11 @@ entry:
   %0 = bitcast <4 x i64> %__S to <8 x i32>
   %1 = bitcast <4 x i64> %__A to <8 x i32>
   %2 = bitcast <4 x i64> %__B to <8 x i32>
-  %3 = tail call <8 x i32> @llvm.x86.avx512.mask.vpshrdv.d.256(<8 x i32> %0, <8 x i32> %1, <8 x i32> %2, i8 %__U)
-  %4 = bitcast <8 x i32> %3 to <4 x i64>
-  ret <4 x i64> %4
+  %3 = tail call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %1, <8 x i32> %0, <8 x i32> %2)
+  %4 = bitcast i8 %__U to <8 x i1>
+  %5 = select <8 x i1> %4, <8 x i32> %3, <8 x i32> %0
+  %6 = bitcast <8 x i32> %5 to <4 x i64>
+  ret <4 x i64> %6
 }
 
 define <4 x i64> @test_mm256_maskz_shrdv_epi32(i8 zeroext %__U, <4 x i64> %__S, <4 x i64> %__A, <4 x i64> %__B) {
@@ -1742,9 +1786,11 @@ entry:
   %0 = bitcast <4 x i64> %__S to <8 x i32>
   %1 = bitcast <4 x i64> %__A to <8 x i32>
   %2 = bitcast <4 x i64> %__B to <8 x i32>
-  %3 = tail call <8 x i32> @llvm.x86.avx512.maskz.vpshrdv.d.256(<8 x i32> %0, <8 x i32> %1, <8 x i32> %2, i8 %__U)
-  %4 = bitcast <8 x i32> %3 to <4 x i64>
-  ret <4 x i64> %4
+  %3 = tail call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %1, <8 x i32> %0, <8 x i32> %2)
+  %4 = bitcast i8 %__U to <8 x i1>
+  %5 = select <8 x i1> %4, <8 x i32> %3, <8 x i32> zeroinitializer
+  %6 = bitcast <8 x i32> %5 to <4 x i64>
+  ret <4 x i64> %6
 }
 
 define <4 x i64> @test_mm256_shrdv_epi32(<4 x i64> %__S, <4 x i64> %__A, <4 x i64> %__B) {
@@ -1756,7 +1802,7 @@ entry:
   %0 = bitcast <4 x i64> %__S to <8 x i32>
   %1 = bitcast <4 x i64> %__A to <8 x i32>
   %2 = bitcast <4 x i64> %__B to <8 x i32>
-  %3 = tail call <8 x i32> @llvm.x86.avx512.mask.vpshrdv.d.256(<8 x i32> %0, <8 x i32> %1, <8 x i32> %2, i8 -1)
+  %3 = tail call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %1, <8 x i32> %0, <8 x i32> %2)
   %4 = bitcast <8 x i32> %3 to <4 x i64>
   ret <4 x i64> %4
 }
@@ -1778,9 +1824,12 @@ entry:
   %0 = bitcast <2 x i64> %__S to <4 x i32>
   %1 = bitcast <2 x i64> %__A to <4 x i32>
   %2 = bitcast <2 x i64> %__B to <4 x i32>
-  %3 = tail call <4 x i32> @llvm.x86.avx512.mask.vpshrdv.d.128(<4 x i32> %0, <4 x i32> %1, <4 x i32> %2, i8 %__U)
-  %4 = bitcast <4 x i32> %3 to <2 x i64>
-  ret <2 x i64> %4
+  %3 = tail call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %1, <4 x i32> %0, <4 x i32> %2)
+  %4 = bitcast i8 %__U to <8 x i1>
+  %extract.i = shufflevector <8 x i1> %4, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %5 = select <4 x i1> %extract.i, <4 x i32> %3, <4 x i32> %0
+  %6 = bitcast <4 x i32> %5 to <2 x i64>
+  ret <2 x i64> %6
 }
 
 define <2 x i64> @test_mm_maskz_shrdv_epi32(i8 zeroext %__U, <2 x i64> %__S, <2 x i64> %__A, <2 x i64> %__B) {
@@ -1800,9 +1849,12 @@ entry:
   %0 = bitcast <2 x i64> %__S to <4 x i32>
   %1 = bitcast <2 x i64> %__A to <4 x i32>
   %2 = bitcast <2 x i64> %__B to <4 x i32>
-  %3 = tail call <4 x i32> @llvm.x86.avx512.maskz.vpshrdv.d.128(<4 x i32> %0, <4 x i32> %1, <4 x i32> %2, i8 %__U)
-  %4 = bitcast <4 x i32> %3 to <2 x i64>
-  ret <2 x i64> %4
+  %3 = tail call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %1, <4 x i32> %0, <4 x i32> %2)
+  %4 = bitcast i8 %__U to <8 x i1>
+  %extract.i = shufflevector <8 x i1> %4, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %5 = select <4 x i1> %extract.i, <4 x i32> %3, <4 x i32> zeroinitializer
+  %6 = bitcast <4 x i32> %5 to <2 x i64>
+  ret <2 x i64> %6
 }
 
 define <2 x i64> @test_mm_shrdv_epi32(<2 x i64> %__S, <2 x i64> %__A, <2 x i64> %__B) {
@@ -1814,7 +1866,7 @@ entry:
   %0 = bitcast <2 x i64> %__S to <4 x i32>
   %1 = bitcast <2 x i64> %__A to <4 x i32>
   %2 = bitcast <2 x i64> %__B to <4 x i32>
-  %3 = tail call <4 x i32> @llvm.x86.avx512.mask.vpshrdv.d.128(<4 x i32> %0, <4 x i32> %1, <4 x i32> %2, i8 -1)
+  %3 = tail call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %1, <4 x i32> %0, <4 x i32> %2)
   %4 = bitcast <4 x i32> %3 to <2 x i64>
   ret <2 x i64> %4
 }
@@ -1835,9 +1887,11 @@ entry:
   %0 = bitcast <4 x i64> %__S to <16 x i16>
   %1 = bitcast <4 x i64> %__A to <16 x i16>
   %2 = bitcast <4 x i64> %__B to <16 x i16>
-  %3 = tail call <16 x i16> @llvm.x86.avx512.mask.vpshrdv.w.256(<16 x i16> %0, <16 x i16> %1, <16 x i16> %2, i16 %__U)
-  %4 = bitcast <16 x i16> %3 to <4 x i64>
-  ret <4 x i64> %4
+  %3 = tail call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %1, <16 x i16> %0, <16 x i16> %2)
+  %4 = bitcast i16 %__U to <16 x i1>
+  %5 = select <16 x i1> %4, <16 x i16> %3, <16 x i16> %0
+  %6 = bitcast <16 x i16> %5 to <4 x i64>
+  ret <4 x i64> %6
 }
 
 define <4 x i64> @test_mm256_maskz_shrdv_epi16(i16 zeroext %__U, <4 x i64> %__S, <4 x i64> %__A, <4 x i64> %__B) {
@@ -1856,9 +1910,11 @@ entry:
   %0 = bitcast <4 x i64> %__S to <16 x i16>
   %1 = bitcast <4 x i64> %__A to <16 x i16>
   %2 = bitcast <4 x i64> %__B to <16 x i16>
-  %3 = tail call <16 x i16> @llvm.x86.avx512.maskz.vpshrdv.w.256(<16 x i16> %0, <16 x i16> %1, <16 x i16> %2, i16 %__U)
-  %4 = bitcast <16 x i16> %3 to <4 x i64>
-  ret <4 x i64> %4
+  %3 = tail call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %1, <16 x i16> %0, <16 x i16> %2)
+  %4 = bitcast i16 %__U to <16 x i1>
+  %5 = select <16 x i1> %4, <16 x i16> %3, <16 x i16> zeroinitializer
+  %6 = bitcast <16 x i16> %5 to <4 x i64>
+  ret <4 x i64> %6
 }
 
 define <4 x i64> @test_mm256_shrdv_epi16(<4 x i64> %__S, <4 x i64> %__A, <4 x i64> %__B) {
@@ -1870,7 +1926,7 @@ entry:
   %0 = bitcast <4 x i64> %__S to <16 x i16>
   %1 = bitcast <4 x i64> %__A to <16 x i16>
   %2 = bitcast <4 x i64> %__B to <16 x i16>
-  %3 = tail call <16 x i16> @llvm.x86.avx512.mask.vpshrdv.w.256(<16 x i16> %0, <16 x i16> %1, <16 x i16> %2, i16 -1)
+  %3 = tail call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %1, <16 x i16> %0, <16 x i16> %2)
   %4 = bitcast <16 x i16> %3 to <4 x i64>
   ret <4 x i64> %4
 }
@@ -1892,9 +1948,11 @@ entry:
   %0 = bitcast <2 x i64> %__S to <8 x i16>
   %1 = bitcast <2 x i64> %__A to <8 x i16>
   %2 = bitcast <2 x i64> %__B to <8 x i16>
-  %3 = tail call <8 x i16> @llvm.x86.avx512.mask.vpshrdv.w.128(<8 x i16> %0, <8 x i16> %1, <8 x i16> %2, i8 %__U)
-  %4 = bitcast <8 x i16> %3 to <2 x i64>
-  ret <2 x i64> %4
+  %3 = tail call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %1, <8 x i16> %0, <8 x i16> %2)
+  %4 = bitcast i8 %__U to <8 x i1>
+  %5 = select <8 x i1> %4, <8 x i16> %3, <8 x i16> %0
+  %6 = bitcast <8 x i16> %5 to <2 x i64>
+  ret <2 x i64> %6
 }
 
 define <2 x i64> @test_mm_maskz_shrdv_epi16(i8 zeroext %__U, <2 x i64> %__S, <2 x i64> %__A, <2 x i64> %__B) {
@@ -1914,9 +1972,11 @@ entry:
   %0 = bitcast <2 x i64> %__S to <8 x i16>
   %1 = bitcast <2 x i64> %__A to <8 x i16>
   %2 = bitcast <2 x i64> %__B to <8 x i16>
-  %3 = tail call <8 x i16> @llvm.x86.avx512.maskz.vpshrdv.w.128(<8 x i16> %0, <8 x i16> %1, <8 x i16> %2, i8 %__U)
-  %4 = bitcast <8 x i16> %3 to <2 x i64>
-  ret <2 x i64> %4
+  %3 = tail call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %1, <8 x i16> %0, <8 x i16> %2)
+  %4 = bitcast i8 %__U to <8 x i1>
+  %5 = select <8 x i1> %4, <8 x i16> %3, <8 x i16> zeroinitializer
+  %6 = bitcast <8 x i16> %5 to <2 x i64>
+  ret <2 x i64> %6
 }
 
 define <2 x i64> @test_mm_shrdv_epi16(<2 x i64> %__S, <2 x i64> %__A, <2 x i64> %__B) {
@@ -1928,7 +1988,7 @@ entry:
   %0 = bitcast <2 x i64> %__S to <8 x i16>
   %1 = bitcast <2 x i64> %__A to <8 x i16>
   %2 = bitcast <2 x i64> %__B to <8 x i16>
-  %3 = tail call <8 x i16> @llvm.x86.avx512.mask.vpshrdv.w.128(<8 x i16> %0, <8 x i16> %1, <8 x i16> %2, i8 -1)
+  %3 = tail call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %1, <8 x i16> %0, <8 x i16> %2)
   %4 = bitcast <8 x i16> %3 to <2 x i64>
   ret <2 x i64> %4
 }
@@ -1949,27 +2009,3 @@ declare <16 x i16> @llvm.x86.avx512.mask.expand.w.256(<16 x i16>, <16 x i16>, i1
 declare <32 x i8> @llvm.x86.avx512.mask.expand.b.256(<32 x i8>, <32 x i8>, i32)
 declare <16 x i16> @llvm.masked.expandload.v16i16(i16*, <16 x i1>, <16 x i16>)
 declare <32 x i8> @llvm.masked.expandload.v32i8(i8*, <32 x i1>, <32 x i8>)
-declare <4 x i64> @llvm.x86.avx512.mask.vpshldv.q.256(<4 x i64>, <4 x i64>, <4 x i64>, i8)
-declare <4 x i64> @llvm.x86.avx512.maskz.vpshldv.q.256(<4 x i64>, <4 x i64>, <4 x i64>, i8)
-declare <2 x i64> @llvm.x86.avx512.mask.vpshldv.q.128(<2 x i64>, <2 x i64>, <2 x i64>, i8)
-declare <2 x i64> @llvm.x86.avx512.maskz.vpshldv.q.128(<2 x i64>, <2 x i64>, <2 x i64>, i8)
-declare <8 x i32> @llvm.x86.avx512.mask.vpshldv.d.256(<8 x i32>, <8 x i32>, <8 x i32>, i8)
-declare <8 x i32> @llvm.x86.avx512.maskz.vpshldv.d.256(<8 x i32>, <8 x i32>, <8 x i32>, i8)
-declare <4 x i32> @llvm.x86.avx512.mask.vpshldv.d.128(<4 x i32>, <4 x i32>, <4 x i32>, i8)
-declare <4 x i32> @llvm.x86.avx512.maskz.vpshldv.d.128(<4 x i32>, <4 x i32>, <4 x i32>, i8)
-declare <16 x i16> @llvm.x86.avx512.mask.vpshldv.w.256(<16 x i16>, <16 x i16>, <16 x i16>, i16)
-declare <16 x i16> @llvm.x86.avx512.maskz.vpshldv.w.256(<16 x i16>, <16 x i16>, <16 x i16>, i16)
-declare <8 x i16> @llvm.x86.avx512.mask.vpshldv.w.128(<8 x i16>, <8 x i16>, <8 x i16>, i8)
-declare <8 x i16> @llvm.x86.avx512.maskz.vpshldv.w.128(<8 x i16>, <8 x i16>, <8 x i16>, i8)
-declare <4 x i64> @llvm.x86.avx512.mask.vpshrdv.q.256(<4 x i64>, <4 x i64>, <4 x i64>, i8)
-declare <4 x i64> @llvm.x86.avx512.maskz.vpshrdv.q.256(<4 x i64>, <4 x i64>, <4 x i64>, i8)
-declare <2 x i64> @llvm.x86.avx512.mask.vpshrdv.q.128(<2 x i64>, <2 x i64>, <2 x i64>, i8)
-declare <2 x i64> @llvm.x86.avx512.maskz.vpshrdv.q.128(<2 x i64>, <2 x i64>, <2 x i64>, i8)
-declare <8 x i32> @llvm.x86.avx512.mask.vpshrdv.d.256(<8 x i32>, <8 x i32>, <8 x i32>, i8)
-declare <8 x i32> @llvm.x86.avx512.maskz.vpshrdv.d.256(<8 x i32>, <8 x i32>, <8 x i32>, i8)
-declare <4 x i32> @llvm.x86.avx512.mask.vpshrdv.d.128(<4 x i32>, <4 x i32>, <4 x i32>, i8)
-declare <4 x i32> @llvm.x86.avx512.maskz.vpshrdv.d.128(<4 x i32>, <4 x i32>, <4 x i32>, i8)
-declare <16 x i16> @llvm.x86.avx512.mask.vpshrdv.w.256(<16 x i16>, <16 x i16>, <16 x i16>, i16)
-declare <16 x i16> @llvm.x86.avx512.maskz.vpshrdv.w.256(<16 x i16>, <16 x i16>, <16 x i16>, i16)
-declare <8 x i16> @llvm.x86.avx512.mask.vpshrdv.w.128(<8 x i16>, <8 x i16>, <8 x i16>, i8)
-declare <8 x i16> @llvm.x86.avx512.maskz.vpshrdv.w.128(<8 x i16>, <8 x i16>, <8 x i16>, i8)
diff --git a/test/CodeGen/X86/avx512vbmi2vl-intrinsics-upgrade.ll b/test/CodeGen/X86/avx512vbmi2vl-intrinsics-upgrade.ll
index 072f96276f654cfbdefe9e5d96f16bf4e6a96a42..9c4ff050316184cc64eb160240aaeda4c7cb3427 100644
--- a/test/CodeGen/X86/avx512vbmi2vl-intrinsics-upgrade.ll
+++ b/test/CodeGen/X86/avx512vbmi2vl-intrinsics-upgrade.ll
@@ -57,6 +57,52 @@ define <8 x i16> @test_expand_load_w_128(i8* %addr, <8 x i16> %data) {
   ret <8 x i16> %res
 }
 
+define <8 x i16> @test_expand_w_128(<8 x i16> %data) {
+; CHECK-LABEL: test_expand_w_128:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <8 x i16> @llvm.x86.avx512.mask.expand.w.128(<8 x i16> %data, <8 x i16> undef, i8 -1)
+  ret <8 x i16> %res
+}
+
+define <8 x i16> @test_mask_expand_w_128(<8 x i16> %data, <8 x i16> %passthru, i8 %mask) {
+; X86-LABEL: test_mask_expand_w_128:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
+; X86-NEXT:    vpexpandw %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0xfd,0x09,0x62,0xc8]
+; X86-NEXT:    vmovdqa %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc1]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_mask_expand_w_128:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; X64-NEXT:    vpexpandw %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0xfd,0x09,0x62,0xc8]
+; X64-NEXT:    vmovdqa %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc1]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <8 x i16> @llvm.x86.avx512.mask.expand.w.128(<8 x i16> %data, <8 x i16> %passthru, i8 %mask)
+  ret <8 x i16> %res
+}
+
+define <8 x i16> @test_maskz_expand_w_128(<8 x i16> %data, i8 %mask) {
+; X86-LABEL: test_maskz_expand_w_128:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
+; X86-NEXT:    vpexpandw %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0x89,0x62,0xc0]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_maskz_expand_w_128:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; X64-NEXT:    vpexpandw %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0x89,0x62,0xc0]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <8 x i16> @llvm.x86.avx512.mask.expand.w.128(<8 x i16> %data, <8 x i16> zeroinitializer, i8 %mask)
+  ret <8 x i16> %res
+}
+
+declare <8 x i16> @llvm.x86.avx512.mask.expand.w.128(<8 x i16> %data, <8 x i16> %src0, i8 %mask)
+
 define <16 x i8> @test_mask_expand_load_b_128(i8* %addr, <16 x i8> %data, i16 %mask) {
 ; X86-LABEL: test_mask_expand_load_b_128:
 ; X86:       # %bb.0:
@@ -110,6 +156,50 @@ define <16 x i8> @test_expand_load_b_128(i8* %addr, <16 x i8> %data) {
   ret <16 x i8> %res
 }
 
+define <16 x i8> @test_expand_b_128(<16 x i8> %data) {
+; CHECK-LABEL: test_expand_b_128:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <16 x i8> @llvm.x86.avx512.mask.expand.b.128(<16 x i8> %data, <16 x i8> undef, i16 -1)
+  ret <16 x i8> %res
+}
+
+define <16 x i8> @test_mask_expand_b_128(<16 x i8> %data, <16 x i8> %passthru, i16 %mask) {
+; X86-LABEL: test_mask_expand_b_128:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
+; X86-NEXT:    vpexpandb %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0x7d,0x09,0x62,0xc8]
+; X86-NEXT:    vmovdqa %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc1]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_mask_expand_b_128:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; X64-NEXT:    vpexpandb %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0x7d,0x09,0x62,0xc8]
+; X64-NEXT:    vmovdqa %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc1]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <16 x i8> @llvm.x86.avx512.mask.expand.b.128(<16 x i8> %data, <16 x i8> %passthru, i16 %mask)
+  ret <16 x i8> %res
+}
+
+define <16 x i8> @test_maskz_expand_b_128(<16 x i8> %data, i16 %mask) {
+; X86-LABEL: test_maskz_expand_b_128:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
+; X86-NEXT:    vpexpandb %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0x89,0x62,0xc0]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_maskz_expand_b_128:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; X64-NEXT:    vpexpandb %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0x89,0x62,0xc0]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <16 x i8> @llvm.x86.avx512.mask.expand.b.128(<16 x i8> %data, <16 x i8> zeroinitializer, i16 %mask)
+  ret <16 x i8> %res
+}
+
+declare <16 x i8> @llvm.x86.avx512.mask.expand.b.128(<16 x i8> %data, <16 x i8> %src0, i16 %mask)
+
 define void @test_mask_compress_store_w_128(i8* %addr, <8 x i16> %data, i8 %mask) {
 ; X86-LABEL: test_mask_compress_store_w_128:
 ; X86:       # %bb.0:
@@ -147,6 +237,52 @@ define void @test_compress_store_w_128(i8* %addr, <8 x i16> %data) {
   ret void
 }
 
+define <8 x i16> @test_mask_compress_w_128(<8 x i16> %data, <8 x i16> %passthru, i8 %mask) {
+; X86-LABEL: test_mask_compress_w_128:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
+; X86-NEXT:    vpcompressw %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0xfd,0x09,0x63,0xc1]
+; X86-NEXT:    vmovdqa %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc1]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_mask_compress_w_128:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; X64-NEXT:    vpcompressw %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0xfd,0x09,0x63,0xc1]
+; X64-NEXT:    vmovdqa %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc1]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <8 x i16> @llvm.x86.avx512.mask.compress.w.128(<8 x i16> %data, <8 x i16> %passthru, i8 %mask)
+  ret <8 x i16> %res
+}
+
+define <8 x i16> @test_maskz_compress_w_128(<8 x i16> %data, i8 %mask) {
+; X86-LABEL: test_maskz_compress_w_128:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
+; X86-NEXT:    vpcompressw %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0x89,0x63,0xc0]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_maskz_compress_w_128:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; X64-NEXT:    vpcompressw %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0x89,0x63,0xc0]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <8 x i16> @llvm.x86.avx512.mask.compress.w.128(<8 x i16> %data, <8 x i16> zeroinitializer, i8 %mask)
+  ret <8 x i16> %res
+}
+
+define <8 x i16> @test_compress_w_128(<8 x i16> %data) {
+; CHECK-LABEL: test_compress_w_128:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <8 x i16> @llvm.x86.avx512.mask.compress.w.128(<8 x i16> %data, <8 x i16> undef, i8 -1)
+  ret <8 x i16> %res
+}
+
+declare <8 x i16> @llvm.x86.avx512.mask.compress.w.128(<8 x i16> %data, <8 x i16> %src0, i8 %mask)
+
 define void @test_mask_compress_store_b_128(i8* %addr, <16 x i8> %data, i16 %mask) {
 ; X86-LABEL: test_mask_compress_store_b_128:
 ; X86:       # %bb.0:
@@ -183,6 +319,50 @@ define void @test_compress_store_b_128(i8* %addr, <16 x i8> %data) {
   ret void
 }
 
+define <16 x i8> @test_mask_compress_b_128(<16 x i8> %data, <16 x i8> %passthru, i16 %mask) {
+; X86-LABEL: test_mask_compress_b_128:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
+; X86-NEXT:    vpcompressb %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0x7d,0x09,0x63,0xc1]
+; X86-NEXT:    vmovdqa %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc1]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_mask_compress_b_128:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; X64-NEXT:    vpcompressb %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0x7d,0x09,0x63,0xc1]
+; X64-NEXT:    vmovdqa %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc1]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <16 x i8> @llvm.x86.avx512.mask.compress.b.128(<16 x i8> %data, <16 x i8> %passthru, i16 %mask)
+  ret <16 x i8> %res
+}
+
+define <16 x i8> @test_maskz_compress_b_128(<16 x i8> %data, i16 %mask) {
+; X86-LABEL: test_maskz_compress_b_128:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
+; X86-NEXT:    vpcompressb %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0x89,0x63,0xc0]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_maskz_compress_b_128:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; X64-NEXT:    vpcompressb %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0x89,0x63,0xc0]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <16 x i8> @llvm.x86.avx512.mask.compress.b.128(<16 x i8> %data, <16 x i8> zeroinitializer, i16 %mask)
+  ret <16 x i8> %res
+}
+
+define <16 x i8> @test_compress_b_128(<16 x i8> %data) {
+; CHECK-LABEL: test_compress_b_128:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <16 x i8> @llvm.x86.avx512.mask.compress.b.128(<16 x i8> %data, <16 x i8> undef, i16 -1)
+  ret <16 x i8> %res
+}
+
+declare <16 x i8> @llvm.x86.avx512.mask.compress.b.128(<16 x i8> %data, <16 x i8> %src0, i16 %mask)
+
 define <16 x i16> @test_mask_expand_load_w_256(i8* %addr, <16 x i16> %data, i16 %mask) {
 ; X86-LABEL: test_mask_expand_load_w_256:
 ; X86:       # %bb.0:
@@ -236,6 +416,50 @@ define <16 x i16> @test_expand_load_w_256(i8* %addr, <16 x i16> %data) {
   ret <16 x i16> %res
 }
 
+define <16 x i16> @test_expand_w_256(<16 x i16> %data) {
+; CHECK-LABEL: test_expand_w_256:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <16 x i16> @llvm.x86.avx512.mask.expand.w.256(<16 x i16> %data, <16 x i16> undef, i16 -1)
+  ret <16 x i16> %res
+}
+
+define <16 x i16> @test_mask_expand_w_256(<16 x i16> %data, <16 x i16> %passthru, i16 %mask) {
+; X86-LABEL: test_mask_expand_w_256:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
+; X86-NEXT:    vpexpandw %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf2,0xfd,0x29,0x62,0xc8]
+; X86-NEXT:    vmovdqa %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc1]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_mask_expand_w_256:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; X64-NEXT:    vpexpandw %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf2,0xfd,0x29,0x62,0xc8]
+; X64-NEXT:    vmovdqa %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc1]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <16 x i16> @llvm.x86.avx512.mask.expand.w.256(<16 x i16> %data, <16 x i16> %passthru, i16 %mask)
+  ret <16 x i16> %res
+}
+
+define <16 x i16> @test_maskz_expand_w_256(<16 x i16> %data, i16 %mask) {
+; X86-LABEL: test_maskz_expand_w_256:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
+; X86-NEXT:    vpexpandw %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xa9,0x62,0xc0]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_maskz_expand_w_256:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; X64-NEXT:    vpexpandw %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xa9,0x62,0xc0]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <16 x i16> @llvm.x86.avx512.mask.expand.w.256(<16 x i16> %data, <16 x i16> zeroinitializer, i16 %mask)
+  ret <16 x i16> %res
+}
+
+declare <16 x i16> @llvm.x86.avx512.mask.expand.w.256(<16 x i16> %data, <16 x i16> %src0, i16 %mask)
+
 define <32 x i8> @test_mask_expand_load_b_256(i8* %addr, <32 x i8> %data, i32 %mask) {
 ; X86-LABEL: test_mask_expand_load_b_256:
 ; X86:       # %bb.0:
@@ -289,6 +513,50 @@ define <32 x i8> @test_expand_load_b_256(i8* %addr, <32 x i8> %data) {
   ret <32 x i8> %res
 }
 
+define <32 x i8> @test_expand_b_256(<32 x i8> %data) {
+; CHECK-LABEL: test_expand_b_256:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <32 x i8> @llvm.x86.avx512.mask.expand.b.256(<32 x i8> %data, <32 x i8> undef, i32 -1)
+  ret <32 x i8> %res
+}
+
+define <32 x i8> @test_mask_expand_b_256(<32 x i8> %data, <32 x i8> %passthru, i32 %mask) {
+; X86-LABEL: test_mask_expand_b_256:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf9,0x90,0x4c,0x24,0x04]
+; X86-NEXT:    vpexpandb %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf2,0x7d,0x29,0x62,0xc8]
+; X86-NEXT:    vmovdqa %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc1]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_mask_expand_b_256:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; X64-NEXT:    vpexpandb %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf2,0x7d,0x29,0x62,0xc8]
+; X64-NEXT:    vmovdqa %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc1]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <32 x i8> @llvm.x86.avx512.mask.expand.b.256(<32 x i8> %data, <32 x i8> %passthru, i32 %mask)
+  ret <32 x i8> %res
+}
+
+define <32 x i8> @test_maskz_expand_b_256(<32 x i8> %data, i32 %mask) {
+; X86-LABEL: test_maskz_expand_b_256:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf9,0x90,0x4c,0x24,0x04]
+; X86-NEXT:    vpexpandb %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0xa9,0x62,0xc0]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_maskz_expand_b_256:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; X64-NEXT:    vpexpandb %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0xa9,0x62,0xc0]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <32 x i8> @llvm.x86.avx512.mask.expand.b.256(<32 x i8> %data, <32 x i8> zeroinitializer, i32 %mask)
+  ret <32 x i8> %res
+}
+
+declare <32 x i8> @llvm.x86.avx512.mask.expand.b.256(<32 x i8> %data, <32 x i8> %src0, i32 %mask)
+
 define void @test_mask_compress_store_w_256(i8* %addr, <16 x i16> %data, i16 %mask) {
 ; X86-LABEL: test_mask_compress_store_w_256:
 ; X86:       # %bb.0:
@@ -329,6 +597,50 @@ define void @test_compress_store_w_256(i8* %addr, <16 x i16> %data) {
   ret void
 }
 
+define <16 x i16> @test_mask_compress_w_256(<16 x i16> %data, <16 x i16> %passthru, i16 %mask) {
+; X86-LABEL: test_mask_compress_w_256:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
+; X86-NEXT:    vpcompressw %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf2,0xfd,0x29,0x63,0xc1]
+; X86-NEXT:    vmovdqa %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc1]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_mask_compress_w_256:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; X64-NEXT:    vpcompressw %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf2,0xfd,0x29,0x63,0xc1]
+; X64-NEXT:    vmovdqa %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc1]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <16 x i16> @llvm.x86.avx512.mask.compress.w.256(<16 x i16> %data, <16 x i16> %passthru, i16 %mask)
+  ret <16 x i16> %res
+}
+
+define <16 x i16> @test_maskz_compress_w_256(<16 x i16> %data, i16 %mask) {
+; X86-LABEL: test_maskz_compress_w_256:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
+; X86-NEXT:    vpcompressw %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xa9,0x63,0xc0]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_maskz_compress_w_256:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; X64-NEXT:    vpcompressw %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xa9,0x63,0xc0]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <16 x i16> @llvm.x86.avx512.mask.compress.w.256(<16 x i16> %data, <16 x i16> zeroinitializer, i16 %mask)
+  ret <16 x i16> %res
+}
+
+define <16 x i16> @test_compress_w_256(<16 x i16> %data) {
+; CHECK-LABEL: test_compress_w_256:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <16 x i16> @llvm.x86.avx512.mask.compress.w.256(<16 x i16> %data, <16 x i16> undef, i16 -1)
+  ret <16 x i16> %res
+}
+
+declare <16 x i16> @llvm.x86.avx512.mask.compress.w.256(<16 x i16> %data, <16 x i16> %src0, i16 %mask)
+
 define void @test_mask_compress_store_b_256(i8* %addr, <32 x i8> %data, i32 %mask) {
 ; X86-LABEL: test_mask_compress_store_b_256:
 ; X86:       # %bb.0:
@@ -369,6 +681,50 @@ define void @test_compress_store_b_256(i8* %addr, <32 x i8> %data) {
   ret void
 }
 
+define <32 x i8> @test_mask_compress_b_256(<32 x i8> %data, <32 x i8> %passthru, i32 %mask) {
+; X86-LABEL: test_mask_compress_b_256:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf9,0x90,0x4c,0x24,0x04]
+; X86-NEXT:    vpcompressb %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf2,0x7d,0x29,0x63,0xc1]
+; X86-NEXT:    vmovdqa %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc1]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_mask_compress_b_256:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; X64-NEXT:    vpcompressb %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf2,0x7d,0x29,0x63,0xc1]
+; X64-NEXT:    vmovdqa %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc1]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <32 x i8> @llvm.x86.avx512.mask.compress.b.256(<32 x i8> %data, <32 x i8> %passthru, i32 %mask)
+  ret <32 x i8> %res
+}
+
+define <32 x i8> @test_maskz_compress_b_256(<32 x i8> %data, i32 %mask) {
+; X86-LABEL: test_maskz_compress_b_256:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf9,0x90,0x4c,0x24,0x04]
+; X86-NEXT:    vpcompressb %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0xa9,0x63,0xc0]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_maskz_compress_b_256:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; X64-NEXT:    vpcompressb %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0xa9,0x63,0xc0]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <32 x i8> @llvm.x86.avx512.mask.compress.b.256(<32 x i8> %data, <32 x i8> zeroinitializer, i32 %mask)
+  ret <32 x i8> %res
+}
+
+define <32 x i8> @test_compress_b_256(<32 x i8> %data) {
+; CHECK-LABEL: test_compress_b_256:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <32 x i8> @llvm.x86.avx512.mask.compress.b.256(<32 x i8> %data, <32 x i8> undef, i32 -1)
+  ret <32 x i8> %res
+}
+
+declare <32 x i8> @llvm.x86.avx512.mask.compress.b.256(<32 x i8> %data, <32 x i8> %src0, i32 %mask)
+
 define <4 x i32>@test_int_x86_avx512_mask_vpshld_d_128(<4 x i32> %x0, <4 x i32> %x1,<4 x i32> %x3, i8 %x4) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpshld_d_128:
 ; X86:       # %bb.0:
@@ -474,22 +830,22 @@ declare <4 x i64> @llvm.x86.avx512.mask.vpshld.q.256(<4 x i64>, <4 x i64>, i32,
 define <8 x i16>@test_int_x86_avx512_mask_vpshld_w_128(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x3, i8 %x4) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpshld_w_128:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpshldw $22, %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf3,0xfd,0x08,0x70,0xd9,0x16]
+; X86-NEXT:    vpshldw $6, %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf3,0xfd,0x08,0x70,0xd9,0x06]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
-; X86-NEXT:    vpshldw $22, %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x09,0x70,0xd1,0x16]
+; X86-NEXT:    vpshldw $6, %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x09,0x70,0xd1,0x06]
 ; X86-NEXT:    vpaddw %xmm3, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfd,0xc3]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_vpshld_w_128:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpshldw $22, %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf3,0xfd,0x08,0x70,0xd9,0x16]
+; X64-NEXT:    vpshldw $6, %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf3,0xfd,0x08,0x70,0xd9,0x06]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
-; X64-NEXT:    vpshldw $22, %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x09,0x70,0xd1,0x16]
+; X64-NEXT:    vpshldw $6, %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x09,0x70,0xd1,0x06]
 ; X64-NEXT:    vpaddw %xmm3, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfd,0xc3]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <8 x i16> @llvm.x86.avx512.mask.vpshld.w.128(<8 x i16> %x0, <8 x i16> %x1, i32 22, <8 x i16> %x3, i8 %x4)
-  %res1 = call <8 x i16> @llvm.x86.avx512.mask.vpshld.w.128(<8 x i16> %x0, <8 x i16> %x1, i32 22, <8 x i16> %x3, i8 -1)
+  %res = call <8 x i16> @llvm.x86.avx512.mask.vpshld.w.128(<8 x i16> %x0, <8 x i16> %x1, i32 6, <8 x i16> %x3, i8 %x4)
+  %res1 = call <8 x i16> @llvm.x86.avx512.mask.vpshld.w.128(<8 x i16> %x0, <8 x i16> %x1, i32 6, <8 x i16> %x3, i8 -1)
   %res2 = add <8 x i16> %res, %res1
   ret <8 x i16> %res2
 }
@@ -498,21 +854,21 @@ declare <8 x i16> @llvm.x86.avx512.mask.vpshld.w.128(<8 x i16>, <8 x i16>, i32,
 define <16 x i16>@test_int_x86_avx512_mask_vpshld_w_256(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x3, i16 %x4) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpshld_w_256:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpshldw $22, %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf3,0xfd,0x28,0x70,0xd9,0x16]
+; X86-NEXT:    vpshldw $6, %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf3,0xfd,0x28,0x70,0xd9,0x06]
 ; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
-; X86-NEXT:    vpshldw $22, %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x29,0x70,0xd1,0x16]
+; X86-NEXT:    vpshldw $6, %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x29,0x70,0xd1,0x06]
 ; X86-NEXT:    vpaddw %ymm3, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfd,0xc3]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_vpshld_w_256:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpshldw $22, %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf3,0xfd,0x28,0x70,0xd9,0x16]
+; X64-NEXT:    vpshldw $6, %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf3,0xfd,0x28,0x70,0xd9,0x06]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
-; X64-NEXT:    vpshldw $22, %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x29,0x70,0xd1,0x16]
+; X64-NEXT:    vpshldw $6, %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x29,0x70,0xd1,0x06]
 ; X64-NEXT:    vpaddw %ymm3, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfd,0xc3]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <16 x i16> @llvm.x86.avx512.mask.vpshld.w.256(<16 x i16> %x0, <16 x i16> %x1, i32 22, <16 x i16> %x3, i16 %x4)
-  %res1 = call <16 x i16> @llvm.x86.avx512.mask.vpshld.w.256(<16 x i16> %x0, <16 x i16> %x1, i32 22, <16 x i16> %x3, i16 -1)
+  %res = call <16 x i16> @llvm.x86.avx512.mask.vpshld.w.256(<16 x i16> %x0, <16 x i16> %x1, i32 6, <16 x i16> %x3, i16 %x4)
+  %res1 = call <16 x i16> @llvm.x86.avx512.mask.vpshld.w.256(<16 x i16> %x0, <16 x i16> %x1, i32 6, <16 x i16> %x3, i16 -1)
   %res2 = add <16 x i16> %res, %res1
   ret <16 x i16> %res2
 }
@@ -623,22 +979,22 @@ declare <4 x i64> @llvm.x86.avx512.mask.vpshrd.q.256(<4 x i64>, <4 x i64>, i32,
 define <8 x i16>@test_int_x86_avx512_mask_vpshrd_w_128(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x3, i8 %x4) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpshrd_w_128:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpshrdw $22, %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf3,0xfd,0x08,0x72,0xd9,0x16]
+; X86-NEXT:    vpshrdw $6, %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf3,0xfd,0x08,0x72,0xd9,0x06]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
-; X86-NEXT:    vpshrdw $22, %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x09,0x72,0xd1,0x16]
+; X86-NEXT:    vpshrdw $6, %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x09,0x72,0xd1,0x06]
 ; X86-NEXT:    vpaddw %xmm3, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfd,0xc3]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_vpshrd_w_128:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpshrdw $22, %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf3,0xfd,0x08,0x72,0xd9,0x16]
+; X64-NEXT:    vpshrdw $6, %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf3,0xfd,0x08,0x72,0xd9,0x06]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
-; X64-NEXT:    vpshrdw $22, %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x09,0x72,0xd1,0x16]
+; X64-NEXT:    vpshrdw $6, %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x09,0x72,0xd1,0x06]
 ; X64-NEXT:    vpaddw %xmm3, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfd,0xc3]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <8 x i16> @llvm.x86.avx512.mask.vpshrd.w.128(<8 x i16> %x0, <8 x i16> %x1, i32 22, <8 x i16> %x3, i8 %x4)
-  %res1 = call <8 x i16> @llvm.x86.avx512.mask.vpshrd.w.128(<8 x i16> %x0, <8 x i16> %x1, i32 22, <8 x i16> %x3, i8 -1)
+  %res = call <8 x i16> @llvm.x86.avx512.mask.vpshrd.w.128(<8 x i16> %x0, <8 x i16> %x1, i32 6, <8 x i16> %x3, i8 %x4)
+  %res1 = call <8 x i16> @llvm.x86.avx512.mask.vpshrd.w.128(<8 x i16> %x0, <8 x i16> %x1, i32 6, <8 x i16> %x3, i8 -1)
   %res2 = add <8 x i16> %res, %res1
   ret <8 x i16> %res2
 }
@@ -647,22 +1003,810 @@ declare <8 x i16> @llvm.x86.avx512.mask.vpshrd.w.128(<8 x i16>, <8 x i16>, i32,
 define <16 x i16>@test_int_x86_avx512_mask_vpshrd_w_256(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x3, i16 %x4) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpshrd_w_256:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpshrdw $22, %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf3,0xfd,0x28,0x72,0xd9,0x16]
+; X86-NEXT:    vpshrdw $6, %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf3,0xfd,0x28,0x72,0xd9,0x06]
 ; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
-; X86-NEXT:    vpshrdw $22, %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x29,0x72,0xd1,0x16]
+; X86-NEXT:    vpshrdw $6, %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x29,0x72,0xd1,0x06]
 ; X86-NEXT:    vpaddw %ymm3, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfd,0xc3]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_vpshrd_w_256:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpshrdw $22, %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf3,0xfd,0x28,0x72,0xd9,0x16]
+; X64-NEXT:    vpshrdw $6, %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf3,0xfd,0x28,0x72,0xd9,0x06]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
-; X64-NEXT:    vpshrdw $22, %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x29,0x72,0xd1,0x16]
+; X64-NEXT:    vpshrdw $6, %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x29,0x72,0xd1,0x06]
 ; X64-NEXT:    vpaddw %ymm3, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfd,0xc3]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <16 x i16> @llvm.x86.avx512.mask.vpshrd.w.256(<16 x i16> %x0, <16 x i16> %x1, i32 22, <16 x i16> %x3, i16 %x4)
-  %res1 = call <16 x i16> @llvm.x86.avx512.mask.vpshrd.w.256(<16 x i16> %x0, <16 x i16> %x1, i32 22, <16 x i16> %x3, i16 -1)
+  %res = call <16 x i16> @llvm.x86.avx512.mask.vpshrd.w.256(<16 x i16> %x0, <16 x i16> %x1, i32 6, <16 x i16> %x3, i16 %x4)
+  %res1 = call <16 x i16> @llvm.x86.avx512.mask.vpshrd.w.256(<16 x i16> %x0, <16 x i16> %x1, i32 6, <16 x i16> %x3, i16 -1)
   %res2 = add <16 x i16> %res, %res1
   ret <16 x i16> %res2
 }
 declare <16 x i16> @llvm.x86.avx512.mask.vpshrd.w.256(<16 x i16>, <16 x i16>, i32, <16 x i16>, i16)
+
+define <4 x i32>@test_int_x86_avx512_mask_vpshld_d_128_2(<4 x i32> %x0, <4 x i32> %x1,<4 x i32> %x3, i8 %x4) {
+; X86-LABEL: test_int_x86_avx512_mask_vpshld_d_128_2:
+; X86:       # %bb.0:
+; X86-NEXT:    vpshldd $22, %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf3,0x7d,0x08,0x71,0xd9,0x16]
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
+; X86-NEXT:    vpshldd $22, %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf3,0x7d,0x09,0x71,0xd1,0x16]
+; X86-NEXT:    vpshldd $22, %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf3,0x7d,0x89,0x71,0xc1,0x16]
+; X86-NEXT:    vpaddd %xmm0, %xmm3, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe1,0xfe,0xc0]
+; X86-NEXT:    vpaddd %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfe,0xc0]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_mask_vpshld_d_128_2:
+; X64:       # %bb.0:
+; X64-NEXT:    vpshldd $22, %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf3,0x7d,0x08,0x71,0xd9,0x16]
+; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; X64-NEXT:    vpshldd $22, %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf3,0x7d,0x09,0x71,0xd1,0x16]
+; X64-NEXT:    vpshldd $22, %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf3,0x7d,0x89,0x71,0xc1,0x16]
+; X64-NEXT:    vpaddd %xmm0, %xmm3, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe1,0xfe,0xc0]
+; X64-NEXT:    vpaddd %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfe,0xc0]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %1 = call <4 x i32> @llvm.x86.avx512.vpshld.d.128(<4 x i32> %x0, <4 x i32> %x1, i32 22)
+  %2 = bitcast i8 %x4 to <8 x i1>
+  %extract1 = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %3 = select <4 x i1> %extract1, <4 x i32> %1, <4 x i32> %x3
+  %4 = call <4 x i32> @llvm.x86.avx512.vpshld.d.128(<4 x i32> %x0, <4 x i32> %x1, i32 22)
+  %5 = call <4 x i32> @llvm.x86.avx512.vpshld.d.128(<4 x i32> %x0, <4 x i32> %x1, i32 22)
+  %6 = bitcast i8 %x4 to <8 x i1>
+  %extract = shufflevector <8 x i1> %6, <8 x i1> %6, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %7 = select <4 x i1> %extract, <4 x i32> %5, <4 x i32> zeroinitializer
+  %res3 = add <4 x i32> %3, %4
+  %res4 = add <4 x i32> %res3, %7
+  ret <4 x i32> %res4
+}
+declare <4 x i32> @llvm.x86.avx512.vpshld.d.128(<4 x i32>, <4 x i32>, i32)
+
+define <8 x i32>@test_int_x86_avx512_mask_vpshld_d_256_2(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x3, i8 %x4) {
+; X86-LABEL: test_int_x86_avx512_mask_vpshld_d_256_2:
+; X86:       # %bb.0:
+; X86-NEXT:    vpshldd $22, %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf3,0x7d,0x28,0x71,0xd9,0x16]
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
+; X86-NEXT:    vpshldd $22, %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf3,0x7d,0x29,0x71,0xd1,0x16]
+; X86-NEXT:    vpaddd %ymm3, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfe,0xc3]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_mask_vpshld_d_256_2:
+; X64:       # %bb.0:
+; X64-NEXT:    vpshldd $22, %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf3,0x7d,0x28,0x71,0xd9,0x16]
+; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; X64-NEXT:    vpshldd $22, %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf3,0x7d,0x29,0x71,0xd1,0x16]
+; X64-NEXT:    vpaddd %ymm3, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfe,0xc3]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %1 = call <8 x i32> @llvm.x86.avx512.vpshld.d.256(<8 x i32> %x0, <8 x i32> %x1, i32 22)
+  %2 = bitcast i8 %x4 to <8 x i1>
+  %3 = select <8 x i1> %2, <8 x i32> %1, <8 x i32> %x3
+  %4 = call <8 x i32> @llvm.x86.avx512.vpshld.d.256(<8 x i32> %x0, <8 x i32> %x1, i32 22)
+  %res2 = add <8 x i32> %3, %4
+  ret <8 x i32> %res2
+}
+declare <8 x i32> @llvm.x86.avx512.vpshld.d.256(<8 x i32>, <8 x i32>, i32)
+
+define <2 x i64>@test_int_x86_avx512_mask_vpshld_q_128_2(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x3, i8 %x4) {
+; X86-LABEL: test_int_x86_avx512_mask_vpshld_q_128_2:
+; X86:       # %bb.0:
+; X86-NEXT:    vpshldq $22, %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf3,0xfd,0x08,0x71,0xd9,0x16]
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
+; X86-NEXT:    vpshldq $22, %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x09,0x71,0xd1,0x16]
+; X86-NEXT:    vpaddq %xmm3, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xd4,0xc3]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_mask_vpshld_q_128_2:
+; X64:       # %bb.0:
+; X64-NEXT:    vpshldq $22, %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf3,0xfd,0x08,0x71,0xd9,0x16]
+; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; X64-NEXT:    vpshldq $22, %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x09,0x71,0xd1,0x16]
+; X64-NEXT:    vpaddq %xmm3, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xd4,0xc3]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %1 = call <2 x i64> @llvm.x86.avx512.vpshld.q.128(<2 x i64> %x0, <2 x i64> %x1, i32 22)
+  %2 = bitcast i8 %x4 to <8 x i1>
+  %extract = shufflevector <8 x i1> %2, <8 x i1> %2, <2 x i32> <i32 0, i32 1>
+  %3 = select <2 x i1> %extract, <2 x i64> %1, <2 x i64> %x3
+  %4 = call <2 x i64> @llvm.x86.avx512.vpshld.q.128(<2 x i64> %x0, <2 x i64> %x1, i32 22)
+  %res2 = add <2 x i64> %3, %4
+  ret <2 x i64> %res2
+}
+declare <2 x i64> @llvm.x86.avx512.vpshld.q.128(<2 x i64>, <2 x i64>, i32)
+
+define <4 x i64>@test_int_x86_avx512_mask_vpshld_q_256_2(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x3, i8 %x4) {
+; X86-LABEL: test_int_x86_avx512_mask_vpshld_q_256_2:
+; X86:       # %bb.0:
+; X86-NEXT:    vpshldq $22, %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf3,0xfd,0x28,0x71,0xd9,0x16]
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
+; X86-NEXT:    vpshldq $22, %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x29,0x71,0xd1,0x16]
+; X86-NEXT:    vpaddq %ymm3, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xd4,0xc3]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_mask_vpshld_q_256_2:
+; X64:       # %bb.0:
+; X64-NEXT:    vpshldq $22, %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf3,0xfd,0x28,0x71,0xd9,0x16]
+; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; X64-NEXT:    vpshldq $22, %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x29,0x71,0xd1,0x16]
+; X64-NEXT:    vpaddq %ymm3, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xd4,0xc3]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %1 = call <4 x i64> @llvm.x86.avx512.vpshld.q.256(<4 x i64> %x0, <4 x i64> %x1, i32 22)
+  %2 = bitcast i8 %x4 to <8 x i1>
+  %extract = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %3 = select <4 x i1> %extract, <4 x i64> %1, <4 x i64> %x3
+  %4 = call <4 x i64> @llvm.x86.avx512.vpshld.q.256(<4 x i64> %x0, <4 x i64> %x1, i32 22)
+  %res2 = add <4 x i64> %3, %4
+  ret <4 x i64> %res2
+}
+declare <4 x i64> @llvm.x86.avx512.vpshld.q.256(<4 x i64>, <4 x i64>, i32)
+
+define <8 x i16>@test_int_x86_avx512_mask_vpshld_w_128_2(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x3, i8 %x4) {
+; X86-LABEL: test_int_x86_avx512_mask_vpshld_w_128_2:
+; X86:       # %bb.0:
+; X86-NEXT:    vpshldw $6, %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf3,0xfd,0x08,0x70,0xd9,0x06]
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
+; X86-NEXT:    vpshldw $6, %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x09,0x70,0xd1,0x06]
+; X86-NEXT:    vpaddw %xmm3, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfd,0xc3]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_mask_vpshld_w_128_2:
+; X64:       # %bb.0:
+; X64-NEXT:    vpshldw $6, %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf3,0xfd,0x08,0x70,0xd9,0x06]
+; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; X64-NEXT:    vpshldw $6, %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x09,0x70,0xd1,0x06]
+; X64-NEXT:    vpaddw %xmm3, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfd,0xc3]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %1 = call <8 x i16> @llvm.x86.avx512.vpshld.w.128(<8 x i16> %x0, <8 x i16> %x1, i32 6)
+  %2 = bitcast i8 %x4 to <8 x i1>
+  %3 = select <8 x i1> %2, <8 x i16> %1, <8 x i16> %x3
+  %4 = call <8 x i16> @llvm.x86.avx512.vpshld.w.128(<8 x i16> %x0, <8 x i16> %x1, i32 6)
+  %res2 = add <8 x i16> %3, %4
+  ret <8 x i16> %res2
+}
+declare <8 x i16> @llvm.x86.avx512.vpshld.w.128(<8 x i16>, <8 x i16>, i32)
+
+define <16 x i16>@test_int_x86_avx512_mask_vpshld_w_256_2(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x3, i16 %x4) {
+; X86-LABEL: test_int_x86_avx512_mask_vpshld_w_256_2:
+; X86:       # %bb.0:
+; X86-NEXT:    vpshldw $6, %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf3,0xfd,0x28,0x70,0xd9,0x06]
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
+; X86-NEXT:    vpshldw $6, %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x29,0x70,0xd1,0x06]
+; X86-NEXT:    vpaddw %ymm3, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfd,0xc3]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_mask_vpshld_w_256_2:
+; X64:       # %bb.0:
+; X64-NEXT:    vpshldw $6, %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf3,0xfd,0x28,0x70,0xd9,0x06]
+; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; X64-NEXT:    vpshldw $6, %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x29,0x70,0xd1,0x06]
+; X64-NEXT:    vpaddw %ymm3, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfd,0xc3]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %1 = call <16 x i16> @llvm.x86.avx512.vpshld.w.256(<16 x i16> %x0, <16 x i16> %x1, i32 6)
+  %2 = bitcast i16 %x4 to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x i16> %1, <16 x i16> %x3
+  %4 = call <16 x i16> @llvm.x86.avx512.vpshld.w.256(<16 x i16> %x0, <16 x i16> %x1, i32 6)
+  %res2 = add <16 x i16> %3, %4
+  ret <16 x i16> %res2
+}
+declare <16 x i16> @llvm.x86.avx512.vpshld.w.256(<16 x i16>, <16 x i16>, i32)
+
+define <4 x i32>@test_int_x86_avx512_mask_vpshrd_d_128_2(<4 x i32> %x0, <4 x i32> %x1,<4 x i32> %x3, i8 %x4) {
+; X86-LABEL: test_int_x86_avx512_mask_vpshrd_d_128_2:
+; X86:       # %bb.0:
+; X86-NEXT:    vpshrdd $22, %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf3,0x7d,0x08,0x73,0xd9,0x16]
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
+; X86-NEXT:    vpshrdd $22, %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf3,0x7d,0x09,0x73,0xd1,0x16]
+; X86-NEXT:    vpshrdd $22, %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf3,0x7d,0x89,0x73,0xc1,0x16]
+; X86-NEXT:    vpaddd %xmm0, %xmm3, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe1,0xfe,0xc0]
+; X86-NEXT:    vpaddd %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfe,0xc0]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_mask_vpshrd_d_128_2:
+; X64:       # %bb.0:
+; X64-NEXT:    vpshrdd $22, %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf3,0x7d,0x08,0x73,0xd9,0x16]
+; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; X64-NEXT:    vpshrdd $22, %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf3,0x7d,0x09,0x73,0xd1,0x16]
+; X64-NEXT:    vpshrdd $22, %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf3,0x7d,0x89,0x73,0xc1,0x16]
+; X64-NEXT:    vpaddd %xmm0, %xmm3, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe1,0xfe,0xc0]
+; X64-NEXT:    vpaddd %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfe,0xc0]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %1 = call <4 x i32> @llvm.x86.avx512.vpshrd.d.128(<4 x i32> %x0, <4 x i32> %x1, i32 22)
+  %2 = bitcast i8 %x4 to <8 x i1>
+  %extract1 = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %3 = select <4 x i1> %extract1, <4 x i32> %1, <4 x i32> %x3
+  %4 = call <4 x i32> @llvm.x86.avx512.vpshrd.d.128(<4 x i32> %x0, <4 x i32> %x1, i32 22)
+  %5 = call <4 x i32> @llvm.x86.avx512.vpshrd.d.128(<4 x i32> %x0, <4 x i32> %x1, i32 22)
+  %6 = bitcast i8 %x4 to <8 x i1>
+  %extract = shufflevector <8 x i1> %6, <8 x i1> %6, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %7 = select <4 x i1> %extract, <4 x i32> %5, <4 x i32> zeroinitializer
+  %res3 = add <4 x i32> %3, %4
+  %res4 = add <4 x i32> %res3, %7
+  ret <4 x i32> %res4
+}
+declare <4 x i32> @llvm.x86.avx512.vpshrd.d.128(<4 x i32>, <4 x i32>, i32)
+
+define <8 x i32>@test_int_x86_avx512_mask_vpshrd_d_256_2(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x3, i8 %x4) {
+; X86-LABEL: test_int_x86_avx512_mask_vpshrd_d_256_2:
+; X86:       # %bb.0:
+; X86-NEXT:    vpshrdd $22, %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf3,0x7d,0x28,0x73,0xd9,0x16]
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
+; X86-NEXT:    vpshrdd $22, %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf3,0x7d,0x29,0x73,0xd1,0x16]
+; X86-NEXT:    vpaddd %ymm3, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfe,0xc3]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_mask_vpshrd_d_256_2:
+; X64:       # %bb.0:
+; X64-NEXT:    vpshrdd $22, %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf3,0x7d,0x28,0x73,0xd9,0x16]
+; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; X64-NEXT:    vpshrdd $22, %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf3,0x7d,0x29,0x73,0xd1,0x16]
+; X64-NEXT:    vpaddd %ymm3, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfe,0xc3]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %1 = call <8 x i32> @llvm.x86.avx512.vpshrd.d.256(<8 x i32> %x0, <8 x i32> %x1, i32 22)
+  %2 = bitcast i8 %x4 to <8 x i1>
+  %3 = select <8 x i1> %2, <8 x i32> %1, <8 x i32> %x3
+  %4 = call <8 x i32> @llvm.x86.avx512.vpshrd.d.256(<8 x i32> %x0, <8 x i32> %x1, i32 22)
+  %res2 = add <8 x i32> %3, %4
+  ret <8 x i32> %res2
+}
+declare <8 x i32> @llvm.x86.avx512.vpshrd.d.256(<8 x i32>, <8 x i32>, i32)
+
+define <2 x i64>@test_int_x86_avx512_mask_vpshrd_q_128_2(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x3, i8 %x4) {
+; X86-LABEL: test_int_x86_avx512_mask_vpshrd_q_128_2:
+; X86:       # %bb.0:
+; X86-NEXT:    vpshrdq $22, %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf3,0xfd,0x08,0x73,0xd9,0x16]
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
+; X86-NEXT:    vpshrdq $22, %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x09,0x73,0xd1,0x16]
+; X86-NEXT:    vpaddq %xmm3, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xd4,0xc3]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_mask_vpshrd_q_128_2:
+; X64:       # %bb.0:
+; X64-NEXT:    vpshrdq $22, %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf3,0xfd,0x08,0x73,0xd9,0x16]
+; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; X64-NEXT:    vpshrdq $22, %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x09,0x73,0xd1,0x16]
+; X64-NEXT:    vpaddq %xmm3, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xd4,0xc3]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %1 = call <2 x i64> @llvm.x86.avx512.vpshrd.q.128(<2 x i64> %x0, <2 x i64> %x1, i32 22)
+  %2 = bitcast i8 %x4 to <8 x i1>
+  %extract = shufflevector <8 x i1> %2, <8 x i1> %2, <2 x i32> <i32 0, i32 1>
+  %3 = select <2 x i1> %extract, <2 x i64> %1, <2 x i64> %x3
+  %4 = call <2 x i64> @llvm.x86.avx512.vpshrd.q.128(<2 x i64> %x0, <2 x i64> %x1, i32 22)
+  %res2 = add <2 x i64> %3, %4
+  ret <2 x i64> %res2
+}
+declare <2 x i64> @llvm.x86.avx512.vpshrd.q.128(<2 x i64>, <2 x i64>, i32)
+
+define <4 x i64>@test_int_x86_avx512_mask_vpshrd_q_256_2(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x3, i8 %x4) {
+; X86-LABEL: test_int_x86_avx512_mask_vpshrd_q_256_2:
+; X86:       # %bb.0:
+; X86-NEXT:    vpshrdq $22, %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf3,0xfd,0x28,0x73,0xd9,0x16]
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
+; X86-NEXT:    vpshrdq $22, %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x29,0x73,0xd1,0x16]
+; X86-NEXT:    vpaddq %ymm3, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xd4,0xc3]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_mask_vpshrd_q_256_2:
+; X64:       # %bb.0:
+; X64-NEXT:    vpshrdq $22, %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf3,0xfd,0x28,0x73,0xd9,0x16]
+; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; X64-NEXT:    vpshrdq $22, %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x29,0x73,0xd1,0x16]
+; X64-NEXT:    vpaddq %ymm3, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xd4,0xc3]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %1 = call <4 x i64> @llvm.x86.avx512.vpshrd.q.256(<4 x i64> %x0, <4 x i64> %x1, i32 22)
+  %2 = bitcast i8 %x4 to <8 x i1>
+  %extract = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %3 = select <4 x i1> %extract, <4 x i64> %1, <4 x i64> %x3
+  %4 = call <4 x i64> @llvm.x86.avx512.vpshrd.q.256(<4 x i64> %x0, <4 x i64> %x1, i32 22)
+  %res2 = add <4 x i64> %3, %4
+  ret <4 x i64> %res2
+}
+declare <4 x i64> @llvm.x86.avx512.vpshrd.q.256(<4 x i64>, <4 x i64>, i32)
+
+define <8 x i16>@test_int_x86_avx512_mask_vpshrd_w_128_2(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x3, i8 %x4) {
+; X86-LABEL: test_int_x86_avx512_mask_vpshrd_w_128_2:
+; X86:       # %bb.0:
+; X86-NEXT:    vpshrdw $6, %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf3,0xfd,0x08,0x72,0xd9,0x06]
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
+; X86-NEXT:    vpshrdw $6, %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x09,0x72,0xd1,0x06]
+; X86-NEXT:    vpaddw %xmm3, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfd,0xc3]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_mask_vpshrd_w_128_2:
+; X64:       # %bb.0:
+; X64-NEXT:    vpshrdw $6, %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf3,0xfd,0x08,0x72,0xd9,0x06]
+; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; X64-NEXT:    vpshrdw $6, %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x09,0x72,0xd1,0x06]
+; X64-NEXT:    vpaddw %xmm3, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfd,0xc3]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %1 = call <8 x i16> @llvm.x86.avx512.vpshrd.w.128(<8 x i16> %x0, <8 x i16> %x1, i32 6)
+  %2 = bitcast i8 %x4 to <8 x i1>
+  %3 = select <8 x i1> %2, <8 x i16> %1, <8 x i16> %x3
+  %4 = call <8 x i16> @llvm.x86.avx512.vpshrd.w.128(<8 x i16> %x0, <8 x i16> %x1, i32 6)
+  %res2 = add <8 x i16> %3, %4
+  ret <8 x i16> %res2
+}
+declare <8 x i16> @llvm.x86.avx512.vpshrd.w.128(<8 x i16>, <8 x i16>, i32)
+
+define <16 x i16>@test_int_x86_avx512_mask_vpshrd_w_256_2(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x3, i16 %x4) {
+; X86-LABEL: test_int_x86_avx512_mask_vpshrd_w_256_2:
+; X86:       # %bb.0:
+; X86-NEXT:    vpshrdw $6, %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf3,0xfd,0x28,0x72,0xd9,0x06]
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
+; X86-NEXT:    vpshrdw $6, %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x29,0x72,0xd1,0x06]
+; X86-NEXT:    vpaddw %ymm3, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfd,0xc3]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_mask_vpshrd_w_256_2:
+; X64:       # %bb.0:
+; X64-NEXT:    vpshrdw $6, %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf3,0xfd,0x28,0x72,0xd9,0x06]
+; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; X64-NEXT:    vpshrdw $6, %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x29,0x72,0xd1,0x06]
+; X64-NEXT:    vpaddw %ymm3, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfd,0xc3]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %1 = call <16 x i16> @llvm.x86.avx512.vpshrd.w.256(<16 x i16> %x0, <16 x i16> %x1, i32 6)
+  %2 = bitcast i16 %x4 to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x i16> %1, <16 x i16> %x3
+  %4 = call <16 x i16> @llvm.x86.avx512.vpshrd.w.256(<16 x i16> %x0, <16 x i16> %x1, i32 6)
+  %res2 = add <16 x i16> %3, %4
+  ret <16 x i16> %res2
+}
+declare <16 x i16> @llvm.x86.avx512.vpshrd.w.256(<16 x i16>, <16 x i16>, i32)
+
+declare <8 x i32> @llvm.x86.avx512.mask.vpshrdv.d.256(<8 x i32>, <8 x i32>, <8 x i32>, i8)
+declare <8 x i32> @llvm.x86.avx512.maskz.vpshrdv.d.256(<8 x i32>, <8 x i32>, <8 x i32>, i8)
+
+define <8 x i32>@test_int_x86_avx512_mask_vpshrdv_d_256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32>* %x2p, <8 x i32> %x4, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_mask_vpshrdv_d_256:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx # encoding: [0x0f,0xb6,0x4c,0x24,0x08]
+; X86-NEXT:    kmovd %ecx, %k1 # encoding: [0xc5,0xfb,0x92,0xc9]
+; X86-NEXT:    vmovdqa %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xd8]
+; X86-NEXT:    vpshrdvd (%eax), %ymm1, %ymm3 {%k1} # encoding: [0x62,0xf2,0x75,0x29,0x73,0x18]
+; X86-NEXT:    vmovdqa %ymm0, %ymm4 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xe0]
+; X86-NEXT:    vpshrdvd %ymm2, %ymm1, %ymm4 # encoding: [0x62,0xf2,0x75,0x28,0x73,0xe2]
+; X86-NEXT:    vpshrdvd %ymm2, %ymm1, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0xa9,0x73,0xc2]
+; X86-NEXT:    vpaddd %ymm0, %ymm4, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xdd,0xfe,0xc0]
+; X86-NEXT:    vpaddd %ymm0, %ymm3, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xe5,0xfe,0xc0]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_mask_vpshrdv_d_256:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovd %esi, %k1 # encoding: [0xc5,0xfb,0x92,0xce]
+; X64-NEXT:    vmovdqa %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xd8]
+; X64-NEXT:    vpshrdvd (%rdi), %ymm1, %ymm3 {%k1} # encoding: [0x62,0xf2,0x75,0x29,0x73,0x1f]
+; X64-NEXT:    vmovdqa %ymm0, %ymm4 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xe0]
+; X64-NEXT:    vpshrdvd %ymm2, %ymm1, %ymm4 # encoding: [0x62,0xf2,0x75,0x28,0x73,0xe2]
+; X64-NEXT:    vpshrdvd %ymm2, %ymm1, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0xa9,0x73,0xc2]
+; X64-NEXT:    vpaddd %ymm0, %ymm4, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xdd,0xfe,0xc0]
+; X64-NEXT:    vpaddd %ymm0, %ymm3, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xe5,0xfe,0xc0]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %x2 = load <8 x i32>, <8 x i32>* %x2p
+  %res = call <8 x i32> @llvm.x86.avx512.mask.vpshrdv.d.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i8 %x3)
+  %res1 = call <8 x i32> @llvm.x86.avx512.mask.vpshrdv.d.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x4, i8 -1)
+  %res2 = call <8 x i32> @llvm.x86.avx512.maskz.vpshrdv.d.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x4, i8  %x3)
+  %res3 = add <8 x i32> %res, %res1
+  %res4 = add <8 x i32> %res2, %res3
+  ret <8 x i32> %res4
+}
+
+declare <4 x i32> @llvm.x86.avx512.mask.vpshrdv.d.128(<4 x i32>, <4 x i32>, <4 x i32>, i8)
+declare <4 x i32> @llvm.x86.avx512.maskz.vpshrdv.d.128(<4 x i32>, <4 x i32>, <4 x i32>, i8)
+
+define <4 x i32>@test_int_x86_avx512_mask_vpshrdv_d_128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32>* %x2p, <4 x i32> %x4, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_mask_vpshrdv_d_128:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx # encoding: [0x0f,0xb6,0x4c,0x24,0x08]
+; X86-NEXT:    kmovd %ecx, %k1 # encoding: [0xc5,0xfb,0x92,0xc9]
+; X86-NEXT:    vmovdqa %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xd8]
+; X86-NEXT:    vpshrdvd (%eax), %xmm1, %xmm3 {%k1} # encoding: [0x62,0xf2,0x75,0x09,0x73,0x18]
+; X86-NEXT:    vmovdqa %xmm0, %xmm4 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xe0]
+; X86-NEXT:    vpshrdvd %xmm2, %xmm1, %xmm4 # encoding: [0x62,0xf2,0x75,0x08,0x73,0xe2]
+; X86-NEXT:    vpshrdvd %xmm2, %xmm1, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0x89,0x73,0xc2]
+; X86-NEXT:    vpaddd %xmm0, %xmm4, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xd9,0xfe,0xc0]
+; X86-NEXT:    vpaddd %xmm0, %xmm3, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe1,0xfe,0xc0]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_mask_vpshrdv_d_128:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovd %esi, %k1 # encoding: [0xc5,0xfb,0x92,0xce]
+; X64-NEXT:    vmovdqa %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xd8]
+; X64-NEXT:    vpshrdvd (%rdi), %xmm1, %xmm3 {%k1} # encoding: [0x62,0xf2,0x75,0x09,0x73,0x1f]
+; X64-NEXT:    vmovdqa %xmm0, %xmm4 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xe0]
+; X64-NEXT:    vpshrdvd %xmm2, %xmm1, %xmm4 # encoding: [0x62,0xf2,0x75,0x08,0x73,0xe2]
+; X64-NEXT:    vpshrdvd %xmm2, %xmm1, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0x89,0x73,0xc2]
+; X64-NEXT:    vpaddd %xmm0, %xmm4, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xd9,0xfe,0xc0]
+; X64-NEXT:    vpaddd %xmm0, %xmm3, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe1,0xfe,0xc0]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %x2 = load <4 x i32>, <4 x i32>* %x2p
+  %res = call <4 x i32> @llvm.x86.avx512.mask.vpshrdv.d.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i8 %x3)
+  %res1 = call <4 x i32> @llvm.x86.avx512.mask.vpshrdv.d.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x4, i8 -1)
+  %res2 = call <4 x i32> @llvm.x86.avx512.maskz.vpshrdv.d.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x4, i8  %x3)
+  %res3 = add <4 x i32> %res, %res1
+  %res4 = add <4 x i32> %res2, %res3
+  ret <4 x i32> %res4
+}
+
+declare <4 x i64> @llvm.x86.avx512.mask.vpshrdv.q.256(<4 x i64>, <4 x i64>, <4 x i64>, i8)
+declare <4 x i64> @llvm.x86.avx512.maskz.vpshrdv.q.256(<4 x i64>, <4 x i64>, <4 x i64>, i8)
+
+define <4 x i64>@test_int_x86_avx512_mask_vpshrdv_q_256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64>* %x2p, <4 x i64> %x4, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_mask_vpshrdv_q_256:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx # encoding: [0x0f,0xb6,0x4c,0x24,0x08]
+; X86-NEXT:    kmovd %ecx, %k1 # encoding: [0xc5,0xfb,0x92,0xc9]
+; X86-NEXT:    vmovdqa %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xd8]
+; X86-NEXT:    vpshrdvq (%eax), %ymm1, %ymm3 {%k1} # encoding: [0x62,0xf2,0xf5,0x29,0x73,0x18]
+; X86-NEXT:    vmovdqa %ymm0, %ymm4 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xe0]
+; X86-NEXT:    vpshrdvq %ymm2, %ymm1, %ymm4 # encoding: [0x62,0xf2,0xf5,0x28,0x73,0xe2]
+; X86-NEXT:    vpshrdvq %ymm2, %ymm1, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xa9,0x73,0xc2]
+; X86-NEXT:    vpaddq %ymm0, %ymm4, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xdd,0xd4,0xc0]
+; X86-NEXT:    vpaddq %ymm0, %ymm3, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xe5,0xd4,0xc0]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_mask_vpshrdv_q_256:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovd %esi, %k1 # encoding: [0xc5,0xfb,0x92,0xce]
+; X64-NEXT:    vmovdqa %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xd8]
+; X64-NEXT:    vpshrdvq (%rdi), %ymm1, %ymm3 {%k1} # encoding: [0x62,0xf2,0xf5,0x29,0x73,0x1f]
+; X64-NEXT:    vmovdqa %ymm0, %ymm4 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xe0]
+; X64-NEXT:    vpshrdvq %ymm2, %ymm1, %ymm4 # encoding: [0x62,0xf2,0xf5,0x28,0x73,0xe2]
+; X64-NEXT:    vpshrdvq %ymm2, %ymm1, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xa9,0x73,0xc2]
+; X64-NEXT:    vpaddq %ymm0, %ymm4, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xdd,0xd4,0xc0]
+; X64-NEXT:    vpaddq %ymm0, %ymm3, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xe5,0xd4,0xc0]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %x2 = load <4 x i64>, <4 x i64>* %x2p
+  %res = call <4 x i64> @llvm.x86.avx512.mask.vpshrdv.q.256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2, i8 %x3)
+  %res1 = call <4 x i64> @llvm.x86.avx512.mask.vpshrdv.q.256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x4, i8 -1)
+  %res2 = call <4 x i64> @llvm.x86.avx512.maskz.vpshrdv.q.256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x4, i8  %x3)
+  %res3 = add <4 x i64> %res, %res1
+  %res4 = add <4 x i64> %res2, %res3
+  ret <4 x i64> %res4
+}
+
+declare <2 x i64> @llvm.x86.avx512.mask.vpshrdv.q.128(<2 x i64>, <2 x i64>, <2 x i64>, i8)
+declare <2 x i64> @llvm.x86.avx512.maskz.vpshrdv.q.128(<2 x i64>, <2 x i64>, <2 x i64>, i8)
+
+define <2 x i64>@test_int_x86_avx512_mask_vpshrdv_q_128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64>* %x2p, <2 x i64> %x4, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_mask_vpshrdv_q_128:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx # encoding: [0x0f,0xb6,0x4c,0x24,0x08]
+; X86-NEXT:    kmovd %ecx, %k1 # encoding: [0xc5,0xfb,0x92,0xc9]
+; X86-NEXT:    vmovdqa %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xd8]
+; X86-NEXT:    vpshrdvq (%eax), %xmm1, %xmm3 {%k1} # encoding: [0x62,0xf2,0xf5,0x09,0x73,0x18]
+; X86-NEXT:    vmovdqa %xmm0, %xmm4 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xe0]
+; X86-NEXT:    vpshrdvq %xmm2, %xmm1, %xmm4 # encoding: [0x62,0xf2,0xf5,0x08,0x73,0xe2]
+; X86-NEXT:    vpshrdvq %xmm2, %xmm1, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0x89,0x73,0xc2]
+; X86-NEXT:    vpaddq %xmm0, %xmm4, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xd9,0xd4,0xc0]
+; X86-NEXT:    vpaddq %xmm0, %xmm3, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe1,0xd4,0xc0]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_mask_vpshrdv_q_128:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovd %esi, %k1 # encoding: [0xc5,0xfb,0x92,0xce]
+; X64-NEXT:    vmovdqa %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xd8]
+; X64-NEXT:    vpshrdvq (%rdi), %xmm1, %xmm3 {%k1} # encoding: [0x62,0xf2,0xf5,0x09,0x73,0x1f]
+; X64-NEXT:    vmovdqa %xmm0, %xmm4 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xe0]
+; X64-NEXT:    vpshrdvq %xmm2, %xmm1, %xmm4 # encoding: [0x62,0xf2,0xf5,0x08,0x73,0xe2]
+; X64-NEXT:    vpshrdvq %xmm2, %xmm1, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0x89,0x73,0xc2]
+; X64-NEXT:    vpaddq %xmm0, %xmm4, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xd9,0xd4,0xc0]
+; X64-NEXT:    vpaddq %xmm0, %xmm3, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe1,0xd4,0xc0]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %x2 = load <2 x i64>, <2 x i64>* %x2p
+  %res = call <2 x i64> @llvm.x86.avx512.mask.vpshrdv.q.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i8 %x3)
+  %res1 = call <2 x i64> @llvm.x86.avx512.mask.vpshrdv.q.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x4, i8 -1)
+  %res2 = call <2 x i64> @llvm.x86.avx512.maskz.vpshrdv.q.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x4, i8  %x3)
+  %res3 = add <2 x i64> %res, %res1
+  %res4 = add <2 x i64> %res2, %res3
+  ret <2 x i64> %res4
+}
+
+declare <16 x i16> @llvm.x86.avx512.mask.vpshrdv.w.256(<16 x i16>, <16 x i16>, <16 x i16>, i16)
+declare <16 x i16> @llvm.x86.avx512.maskz.vpshrdv.w.256(<16 x i16>, <16 x i16>, <16 x i16>, i16)
+
+define <16 x i16>@test_int_x86_avx512_mask_vpshrdv_w_256(<16 x i16> %x0, <16 x i16> %x1, <16 x i16>* %x2p, <16 x i16> %x4, i16 %x3) {
+; X86-LABEL: test_int_x86_avx512_mask_vpshrdv_w_256:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x08]
+; X86-NEXT:    vmovdqa %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xd8]
+; X86-NEXT:    vpshrdvw (%eax), %ymm1, %ymm3 {%k1} # encoding: [0x62,0xf2,0xf5,0x29,0x72,0x18]
+; X86-NEXT:    vmovdqa %ymm0, %ymm4 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xe0]
+; X86-NEXT:    vpshrdvw %ymm2, %ymm1, %ymm4 # encoding: [0x62,0xf2,0xf5,0x28,0x72,0xe2]
+; X86-NEXT:    vpshrdvw %ymm2, %ymm1, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xa9,0x72,0xc2]
+; X86-NEXT:    vpaddw %ymm0, %ymm4, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xdd,0xfd,0xc0]
+; X86-NEXT:    vpaddw %ymm0, %ymm3, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xe5,0xfd,0xc0]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_mask_vpshrdv_w_256:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovd %esi, %k1 # encoding: [0xc5,0xfb,0x92,0xce]
+; X64-NEXT:    vmovdqa %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xd8]
+; X64-NEXT:    vpshrdvw (%rdi), %ymm1, %ymm3 {%k1} # encoding: [0x62,0xf2,0xf5,0x29,0x72,0x1f]
+; X64-NEXT:    vmovdqa %ymm0, %ymm4 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xe0]
+; X64-NEXT:    vpshrdvw %ymm2, %ymm1, %ymm4 # encoding: [0x62,0xf2,0xf5,0x28,0x72,0xe2]
+; X64-NEXT:    vpshrdvw %ymm2, %ymm1, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xa9,0x72,0xc2]
+; X64-NEXT:    vpaddw %ymm0, %ymm4, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xdd,0xfd,0xc0]
+; X64-NEXT:    vpaddw %ymm0, %ymm3, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xe5,0xfd,0xc0]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %x2 = load <16 x i16>, <16 x i16>* %x2p
+  %res = call <16 x i16> @llvm.x86.avx512.mask.vpshrdv.w.256(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x2, i16 %x3)
+  %res1 = call <16 x i16> @llvm.x86.avx512.mask.vpshrdv.w.256(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x4, i16 -1)
+  %res2 = call <16 x i16> @llvm.x86.avx512.maskz.vpshrdv.w.256(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x4, i16  %x3)
+  %res3 = add <16 x i16> %res, %res1
+  %res4 = add <16 x i16> %res2, %res3
+  ret <16 x i16> %res4
+}
+
+declare <8 x i16> @llvm.x86.avx512.mask.vpshrdv.w.128(<8 x i16>, <8 x i16>, <8 x i16>, i8)
+declare <8 x i16> @llvm.x86.avx512.maskz.vpshrdv.w.128(<8 x i16>, <8 x i16>, <8 x i16>, i8)
+
+define <8 x i16>@test_int_x86_avx512_mask_vpshrdv_w_128(<8 x i16> %x0, <8 x i16> %x1, <8 x i16>* %x2p, <8 x i16> %x4, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_mask_vpshrdv_w_128:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx # encoding: [0x0f,0xb6,0x4c,0x24,0x08]
+; X86-NEXT:    kmovd %ecx, %k1 # encoding: [0xc5,0xfb,0x92,0xc9]
+; X86-NEXT:    vmovdqa %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xd8]
+; X86-NEXT:    vpshrdvw (%eax), %xmm1, %xmm3 {%k1} # encoding: [0x62,0xf2,0xf5,0x09,0x72,0x18]
+; X86-NEXT:    vmovdqa %xmm0, %xmm4 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xe0]
+; X86-NEXT:    vpshrdvw %xmm2, %xmm1, %xmm4 # encoding: [0x62,0xf2,0xf5,0x08,0x72,0xe2]
+; X86-NEXT:    vpshrdvw %xmm2, %xmm1, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0x89,0x72,0xc2]
+; X86-NEXT:    vpaddw %xmm0, %xmm4, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xd9,0xfd,0xc0]
+; X86-NEXT:    vpaddw %xmm0, %xmm3, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe1,0xfd,0xc0]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_mask_vpshrdv_w_128:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovd %esi, %k1 # encoding: [0xc5,0xfb,0x92,0xce]
+; X64-NEXT:    vmovdqa %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xd8]
+; X64-NEXT:    vpshrdvw (%rdi), %xmm1, %xmm3 {%k1} # encoding: [0x62,0xf2,0xf5,0x09,0x72,0x1f]
+; X64-NEXT:    vmovdqa %xmm0, %xmm4 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xe0]
+; X64-NEXT:    vpshrdvw %xmm2, %xmm1, %xmm4 # encoding: [0x62,0xf2,0xf5,0x08,0x72,0xe2]
+; X64-NEXT:    vpshrdvw %xmm2, %xmm1, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0x89,0x72,0xc2]
+; X64-NEXT:    vpaddw %xmm0, %xmm4, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xd9,0xfd,0xc0]
+; X64-NEXT:    vpaddw %xmm0, %xmm3, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe1,0xfd,0xc0]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %x2 = load <8 x i16>, <8 x i16>* %x2p
+  %res = call <8 x i16> @llvm.x86.avx512.mask.vpshrdv.w.128(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2, i8 %x3)
+  %res1 = call <8 x i16> @llvm.x86.avx512.mask.vpshrdv.w.128(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x4, i8 -1)
+  %res2 = call <8 x i16> @llvm.x86.avx512.maskz.vpshrdv.w.128(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x4, i8  %x3)
+  %res3 = add <8 x i16> %res, %res1
+  %res4 = add <8 x i16> %res2, %res3
+  ret <8 x i16> %res4
+}
+
+declare <8 x i32> @llvm.x86.avx512.mask.vpshldv.d.256(<8 x i32>, <8 x i32>, <8 x i32>, i8)
+declare <8 x i32> @llvm.x86.avx512.maskz.vpshldv.d.256(<8 x i32>, <8 x i32>, <8 x i32>, i8)
+
+define <8 x i32>@test_int_x86_avx512_mask_vpshldv_d_256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32>* %x2p, <8 x i32> %x4, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_mask_vpshldv_d_256:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx # encoding: [0x0f,0xb6,0x4c,0x24,0x08]
+; X86-NEXT:    kmovd %ecx, %k1 # encoding: [0xc5,0xfb,0x92,0xc9]
+; X86-NEXT:    vmovdqa %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xd8]
+; X86-NEXT:    vpshldvd (%eax), %ymm1, %ymm3 {%k1} # encoding: [0x62,0xf2,0x75,0x29,0x71,0x18]
+; X86-NEXT:    vmovdqa %ymm0, %ymm4 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xe0]
+; X86-NEXT:    vpshldvd %ymm2, %ymm1, %ymm4 # encoding: [0x62,0xf2,0x75,0x28,0x71,0xe2]
+; X86-NEXT:    vpshldvd %ymm2, %ymm1, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0xa9,0x71,0xc2]
+; X86-NEXT:    vpaddd %ymm0, %ymm4, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xdd,0xfe,0xc0]
+; X86-NEXT:    vpaddd %ymm0, %ymm3, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xe5,0xfe,0xc0]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_mask_vpshldv_d_256:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovd %esi, %k1 # encoding: [0xc5,0xfb,0x92,0xce]
+; X64-NEXT:    vmovdqa %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xd8]
+; X64-NEXT:    vpshldvd (%rdi), %ymm1, %ymm3 {%k1} # encoding: [0x62,0xf2,0x75,0x29,0x71,0x1f]
+; X64-NEXT:    vmovdqa %ymm0, %ymm4 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xe0]
+; X64-NEXT:    vpshldvd %ymm2, %ymm1, %ymm4 # encoding: [0x62,0xf2,0x75,0x28,0x71,0xe2]
+; X64-NEXT:    vpshldvd %ymm2, %ymm1, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0xa9,0x71,0xc2]
+; X64-NEXT:    vpaddd %ymm0, %ymm4, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xdd,0xfe,0xc0]
+; X64-NEXT:    vpaddd %ymm0, %ymm3, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xe5,0xfe,0xc0]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %x2 = load <8 x i32>, <8 x i32>* %x2p
+  %res = call <8 x i32> @llvm.x86.avx512.mask.vpshldv.d.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i8 %x3)
+  %res1 = call <8 x i32> @llvm.x86.avx512.mask.vpshldv.d.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x4, i8 -1)
+  %res2 = call <8 x i32> @llvm.x86.avx512.maskz.vpshldv.d.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x4, i8  %x3)
+  %res3 = add <8 x i32> %res, %res1
+  %res4 = add <8 x i32> %res2, %res3
+  ret <8 x i32> %res4
+}
+
+declare <4 x i32> @llvm.x86.avx512.mask.vpshldv.d.128(<4 x i32>, <4 x i32>, <4 x i32>, i8)
+declare <4 x i32> @llvm.x86.avx512.maskz.vpshldv.d.128(<4 x i32>, <4 x i32>, <4 x i32>, i8)
+
+define <4 x i32>@test_int_x86_avx512_mask_vpshldv_d_128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32>* %x2p, <4 x i32> %x4, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_mask_vpshldv_d_128:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx # encoding: [0x0f,0xb6,0x4c,0x24,0x08]
+; X86-NEXT:    kmovd %ecx, %k1 # encoding: [0xc5,0xfb,0x92,0xc9]
+; X86-NEXT:    vmovdqa %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xd8]
+; X86-NEXT:    vpshldvd (%eax), %xmm1, %xmm3 {%k1} # encoding: [0x62,0xf2,0x75,0x09,0x71,0x18]
+; X86-NEXT:    vmovdqa %xmm0, %xmm4 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xe0]
+; X86-NEXT:    vpshldvd %xmm2, %xmm1, %xmm4 # encoding: [0x62,0xf2,0x75,0x08,0x71,0xe2]
+; X86-NEXT:    vpshldvd %xmm2, %xmm1, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0x89,0x71,0xc2]
+; X86-NEXT:    vpaddd %xmm0, %xmm4, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xd9,0xfe,0xc0]
+; X86-NEXT:    vpaddd %xmm0, %xmm3, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe1,0xfe,0xc0]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_mask_vpshldv_d_128:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovd %esi, %k1 # encoding: [0xc5,0xfb,0x92,0xce]
+; X64-NEXT:    vmovdqa %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xd8]
+; X64-NEXT:    vpshldvd (%rdi), %xmm1, %xmm3 {%k1} # encoding: [0x62,0xf2,0x75,0x09,0x71,0x1f]
+; X64-NEXT:    vmovdqa %xmm0, %xmm4 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xe0]
+; X64-NEXT:    vpshldvd %xmm2, %xmm1, %xmm4 # encoding: [0x62,0xf2,0x75,0x08,0x71,0xe2]
+; X64-NEXT:    vpshldvd %xmm2, %xmm1, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0x89,0x71,0xc2]
+; X64-NEXT:    vpaddd %xmm0, %xmm4, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xd9,0xfe,0xc0]
+; X64-NEXT:    vpaddd %xmm0, %xmm3, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe1,0xfe,0xc0]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %x2 = load <4 x i32>, <4 x i32>* %x2p
+  %res = call <4 x i32> @llvm.x86.avx512.mask.vpshldv.d.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i8 %x3)
+  %res1 = call <4 x i32> @llvm.x86.avx512.mask.vpshldv.d.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x4, i8 -1)
+  %res2 = call <4 x i32> @llvm.x86.avx512.maskz.vpshldv.d.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x4, i8  %x3)
+  %res3 = add <4 x i32> %res, %res1
+  %res4 = add <4 x i32> %res2, %res3
+  ret <4 x i32> %res4
+}
+
+declare <4 x i64> @llvm.x86.avx512.mask.vpshldv.q.256(<4 x i64>, <4 x i64>, <4 x i64>, i8)
+declare <4 x i64> @llvm.x86.avx512.maskz.vpshldv.q.256(<4 x i64>, <4 x i64>, <4 x i64>, i8)
+
+define <4 x i64>@test_int_x86_avx512_mask_vpshldv_q_256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64>* %x2p, <4 x i64> %x4, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_mask_vpshldv_q_256:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx # encoding: [0x0f,0xb6,0x4c,0x24,0x08]
+; X86-NEXT:    kmovd %ecx, %k1 # encoding: [0xc5,0xfb,0x92,0xc9]
+; X86-NEXT:    vmovdqa %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xd8]
+; X86-NEXT:    vpshldvq (%eax), %ymm1, %ymm3 {%k1} # encoding: [0x62,0xf2,0xf5,0x29,0x71,0x18]
+; X86-NEXT:    vmovdqa %ymm0, %ymm4 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xe0]
+; X86-NEXT:    vpshldvq %ymm2, %ymm1, %ymm4 # encoding: [0x62,0xf2,0xf5,0x28,0x71,0xe2]
+; X86-NEXT:    vpshldvq %ymm2, %ymm1, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xa9,0x71,0xc2]
+; X86-NEXT:    vpaddq %ymm0, %ymm4, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xdd,0xd4,0xc0]
+; X86-NEXT:    vpaddq %ymm0, %ymm3, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xe5,0xd4,0xc0]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_mask_vpshldv_q_256:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovd %esi, %k1 # encoding: [0xc5,0xfb,0x92,0xce]
+; X64-NEXT:    vmovdqa %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xd8]
+; X64-NEXT:    vpshldvq (%rdi), %ymm1, %ymm3 {%k1} # encoding: [0x62,0xf2,0xf5,0x29,0x71,0x1f]
+; X64-NEXT:    vmovdqa %ymm0, %ymm4 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xe0]
+; X64-NEXT:    vpshldvq %ymm2, %ymm1, %ymm4 # encoding: [0x62,0xf2,0xf5,0x28,0x71,0xe2]
+; X64-NEXT:    vpshldvq %ymm2, %ymm1, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xa9,0x71,0xc2]
+; X64-NEXT:    vpaddq %ymm0, %ymm4, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xdd,0xd4,0xc0]
+; X64-NEXT:    vpaddq %ymm0, %ymm3, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xe5,0xd4,0xc0]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %x2 = load <4 x i64>, <4 x i64>* %x2p
+  %res = call <4 x i64> @llvm.x86.avx512.mask.vpshldv.q.256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2, i8 %x3)
+  %res1 = call <4 x i64> @llvm.x86.avx512.mask.vpshldv.q.256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x4, i8 -1)
+  %res2 = call <4 x i64> @llvm.x86.avx512.maskz.vpshldv.q.256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x4, i8  %x3)
+  %res3 = add <4 x i64> %res, %res1
+  %res4 = add <4 x i64> %res2, %res3
+  ret <4 x i64> %res4
+}
+
+declare <2 x i64> @llvm.x86.avx512.mask.vpshldv.q.128(<2 x i64>, <2 x i64>, <2 x i64>, i8)
+declare <2 x i64> @llvm.x86.avx512.maskz.vpshldv.q.128(<2 x i64>, <2 x i64>, <2 x i64>, i8)
+
+define <2 x i64>@test_int_x86_avx512_mask_vpshldv_q_128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64>* %x2p, <2 x i64> %x4, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_mask_vpshldv_q_128:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx # encoding: [0x0f,0xb6,0x4c,0x24,0x08]
+; X86-NEXT:    kmovd %ecx, %k1 # encoding: [0xc5,0xfb,0x92,0xc9]
+; X86-NEXT:    vmovdqa %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xd8]
+; X86-NEXT:    vpshldvq (%eax), %xmm1, %xmm3 {%k1} # encoding: [0x62,0xf2,0xf5,0x09,0x71,0x18]
+; X86-NEXT:    vmovdqa %xmm0, %xmm4 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xe0]
+; X86-NEXT:    vpshldvq %xmm2, %xmm1, %xmm4 # encoding: [0x62,0xf2,0xf5,0x08,0x71,0xe2]
+; X86-NEXT:    vpshldvq %xmm2, %xmm1, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0x89,0x71,0xc2]
+; X86-NEXT:    vpaddq %xmm0, %xmm4, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xd9,0xd4,0xc0]
+; X86-NEXT:    vpaddq %xmm0, %xmm3, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe1,0xd4,0xc0]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_mask_vpshldv_q_128:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovd %esi, %k1 # encoding: [0xc5,0xfb,0x92,0xce]
+; X64-NEXT:    vmovdqa %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xd8]
+; X64-NEXT:    vpshldvq (%rdi), %xmm1, %xmm3 {%k1} # encoding: [0x62,0xf2,0xf5,0x09,0x71,0x1f]
+; X64-NEXT:    vmovdqa %xmm0, %xmm4 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xe0]
+; X64-NEXT:    vpshldvq %xmm2, %xmm1, %xmm4 # encoding: [0x62,0xf2,0xf5,0x08,0x71,0xe2]
+; X64-NEXT:    vpshldvq %xmm2, %xmm1, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0x89,0x71,0xc2]
+; X64-NEXT:    vpaddq %xmm0, %xmm4, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xd9,0xd4,0xc0]
+; X64-NEXT:    vpaddq %xmm0, %xmm3, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe1,0xd4,0xc0]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %x2 = load <2 x i64>, <2 x i64>* %x2p
+  %res = call <2 x i64> @llvm.x86.avx512.mask.vpshldv.q.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i8 %x3)
+  %res1 = call <2 x i64> @llvm.x86.avx512.mask.vpshldv.q.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x4, i8 -1)
+  %res2 = call <2 x i64> @llvm.x86.avx512.maskz.vpshldv.q.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x4, i8  %x3)
+  %res3 = add <2 x i64> %res, %res1
+  %res4 = add <2 x i64> %res2, %res3
+  ret <2 x i64> %res4
+}
+
+declare <16 x i16> @llvm.x86.avx512.mask.vpshldv.w.256(<16 x i16>, <16 x i16>, <16 x i16>, i16)
+declare <16 x i16> @llvm.x86.avx512.maskz.vpshldv.w.256(<16 x i16>, <16 x i16>, <16 x i16>, i16)
+
+define <16 x i16>@test_int_x86_avx512_mask_vpshldv_w_256(<16 x i16> %x0, <16 x i16> %x1, <16 x i16>* %x2p, <16 x i16> %x4, i16 %x3) {
+; X86-LABEL: test_int_x86_avx512_mask_vpshldv_w_256:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x08]
+; X86-NEXT:    vmovdqa %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xd8]
+; X86-NEXT:    vpshldvw (%eax), %ymm1, %ymm3 {%k1} # encoding: [0x62,0xf2,0xf5,0x29,0x70,0x18]
+; X86-NEXT:    vmovdqa %ymm0, %ymm4 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xe0]
+; X86-NEXT:    vpshldvw %ymm2, %ymm1, %ymm4 # encoding: [0x62,0xf2,0xf5,0x28,0x70,0xe2]
+; X86-NEXT:    vpshldvw %ymm2, %ymm1, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xa9,0x70,0xc2]
+; X86-NEXT:    vpaddw %ymm0, %ymm4, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xdd,0xfd,0xc0]
+; X86-NEXT:    vpaddw %ymm0, %ymm3, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xe5,0xfd,0xc0]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_mask_vpshldv_w_256:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovd %esi, %k1 # encoding: [0xc5,0xfb,0x92,0xce]
+; X64-NEXT:    vmovdqa %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xd8]
+; X64-NEXT:    vpshldvw (%rdi), %ymm1, %ymm3 {%k1} # encoding: [0x62,0xf2,0xf5,0x29,0x70,0x1f]
+; X64-NEXT:    vmovdqa %ymm0, %ymm4 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xe0]
+; X64-NEXT:    vpshldvw %ymm2, %ymm1, %ymm4 # encoding: [0x62,0xf2,0xf5,0x28,0x70,0xe2]
+; X64-NEXT:    vpshldvw %ymm2, %ymm1, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xa9,0x70,0xc2]
+; X64-NEXT:    vpaddw %ymm0, %ymm4, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xdd,0xfd,0xc0]
+; X64-NEXT:    vpaddw %ymm0, %ymm3, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xe5,0xfd,0xc0]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %x2 = load <16 x i16>, <16 x i16>* %x2p
+  %res = call <16 x i16> @llvm.x86.avx512.mask.vpshldv.w.256(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x2, i16 %x3)
+  %res1 = call <16 x i16> @llvm.x86.avx512.mask.vpshldv.w.256(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x4, i16 -1)
+  %res2 = call <16 x i16> @llvm.x86.avx512.maskz.vpshldv.w.256(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x4, i16  %x3)
+  %res3 = add <16 x i16> %res, %res1
+  %res4 = add <16 x i16> %res2, %res3
+  ret <16 x i16> %res4
+}
+
+declare <8 x i16> @llvm.x86.avx512.mask.vpshldv.w.128(<8 x i16>, <8 x i16>, <8 x i16>, i8)
+declare <8 x i16> @llvm.x86.avx512.maskz.vpshldv.w.128(<8 x i16>, <8 x i16>, <8 x i16>, i8)
+
+define <8 x i16>@test_int_x86_avx512_mask_vpshldv_w_128(<8 x i16> %x0, <8 x i16> %x1, <8 x i16>* %x2p, <8 x i16> %x4, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_mask_vpshldv_w_128:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx # encoding: [0x0f,0xb6,0x4c,0x24,0x08]
+; X86-NEXT:    kmovd %ecx, %k1 # encoding: [0xc5,0xfb,0x92,0xc9]
+; X86-NEXT:    vmovdqa %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xd8]
+; X86-NEXT:    vpshldvw (%eax), %xmm1, %xmm3 {%k1} # encoding: [0x62,0xf2,0xf5,0x09,0x70,0x18]
+; X86-NEXT:    vmovdqa %xmm0, %xmm4 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xe0]
+; X86-NEXT:    vpshldvw %xmm2, %xmm1, %xmm4 # encoding: [0x62,0xf2,0xf5,0x08,0x70,0xe2]
+; X86-NEXT:    vpshldvw %xmm2, %xmm1, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0x89,0x70,0xc2]
+; X86-NEXT:    vpaddw %xmm0, %xmm4, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xd9,0xfd,0xc0]
+; X86-NEXT:    vpaddw %xmm0, %xmm3, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe1,0xfd,0xc0]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_mask_vpshldv_w_128:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovd %esi, %k1 # encoding: [0xc5,0xfb,0x92,0xce]
+; X64-NEXT:    vmovdqa %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xd8]
+; X64-NEXT:    vpshldvw (%rdi), %xmm1, %xmm3 {%k1} # encoding: [0x62,0xf2,0xf5,0x09,0x70,0x1f]
+; X64-NEXT:    vmovdqa %xmm0, %xmm4 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xe0]
+; X64-NEXT:    vpshldvw %xmm2, %xmm1, %xmm4 # encoding: [0x62,0xf2,0xf5,0x08,0x70,0xe2]
+; X64-NEXT:    vpshldvw %xmm2, %xmm1, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0x89,0x70,0xc2]
+; X64-NEXT:    vpaddw %xmm0, %xmm4, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xd9,0xfd,0xc0]
+; X64-NEXT:    vpaddw %xmm0, %xmm3, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe1,0xfd,0xc0]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %x2 = load <8 x i16>, <8 x i16>* %x2p
+  %res = call <8 x i16> @llvm.x86.avx512.mask.vpshldv.w.128(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2, i8 %x3)
+  %res1 = call <8 x i16> @llvm.x86.avx512.mask.vpshldv.w.128(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x4, i8 -1)
+  %res2 = call <8 x i16> @llvm.x86.avx512.maskz.vpshldv.w.128(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x4, i8  %x3)
+  %res3 = add <8 x i16> %res, %res1
+  %res4 = add <8 x i16> %res2, %res3
+  ret <8 x i16> %res4
+}
diff --git a/test/CodeGen/X86/avx512vbmi2vl-intrinsics.ll b/test/CodeGen/X86/avx512vbmi2vl-intrinsics.ll
index 341adb92e1d60a186d510c1a2d2aefb66679982e..371499c476e24c0bca74731c17b46523e8700ffc 100644
--- a/test/CodeGen/X86/avx512vbmi2vl-intrinsics.ll
+++ b/test/CodeGen/X86/avx512vbmi2vl-intrinsics.ll
@@ -2,12 +2,70 @@
 ; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+avx512vbmi2,+avx512vl --show-mc-encoding | FileCheck %s --check-prefixes=CHECK,X86
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512vbmi2,+avx512vl --show-mc-encoding | FileCheck %s --check-prefixes=CHECK,X64
 
+define <8 x i16> @test_mask_expand_load_w_128(i8* %addr, <8 x i16> %data, i8 %mask) {
+; X86-LABEL: test_mask_expand_load_w_128:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx # encoding: [0x0f,0xb6,0x4c,0x24,0x08]
+; X86-NEXT:    kmovd %ecx, %k1 # encoding: [0xc5,0xfb,0x92,0xc9]
+; X86-NEXT:    vpexpandw (%eax), %xmm0 {%k1} # encoding: [0x62,0xf2,0xfd,0x09,0x62,0x00]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_mask_expand_load_w_128:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovd %esi, %k1 # encoding: [0xc5,0xfb,0x92,0xce]
+; X64-NEXT:    vpexpandw (%rdi), %xmm0 {%k1} # encoding: [0x62,0xf2,0xfd,0x09,0x62,0x07]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %1 = bitcast i8* %addr to i16*
+  %2 = bitcast i8 %mask to <8 x i1>
+  %3 = call <8 x i16> @llvm.masked.expandload.v8i16(i16* %1, <8 x i1> %2, <8 x i16> %data)
+  ret <8 x i16> %3
+}
+
+define <8 x i16> @test_maskz_expand_load_w_128(i8* %addr, i8 %mask) {
+; X86-LABEL: test_maskz_expand_load_w_128:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx # encoding: [0x0f,0xb6,0x4c,0x24,0x08]
+; X86-NEXT:    kmovd %ecx, %k1 # encoding: [0xc5,0xfb,0x92,0xc9]
+; X86-NEXT:    vpexpandw (%eax), %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0x89,0x62,0x00]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_maskz_expand_load_w_128:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovd %esi, %k1 # encoding: [0xc5,0xfb,0x92,0xce]
+; X64-NEXT:    vpexpandw (%rdi), %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0x89,0x62,0x07]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %1 = bitcast i8* %addr to i16*
+  %2 = bitcast i8 %mask to <8 x i1>
+  %3 = call <8 x i16> @llvm.masked.expandload.v8i16(i16* %1, <8 x i1> %2, <8 x i16> zeroinitializer)
+  ret <8 x i16> %3
+}
+
+define <8 x i16> @test_expand_load_w_128(i8* %addr, <8 x i16> %data) {
+; X86-LABEL: test_expand_load_w_128:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-NEXT:    kxnorw %k0, %k0, %k1 # encoding: [0xc5,0xfc,0x46,0xc8]
+; X86-NEXT:    vpexpandw (%eax), %xmm0 {%k1} # encoding: [0x62,0xf2,0xfd,0x09,0x62,0x00]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_expand_load_w_128:
+; X64:       # %bb.0:
+; X64-NEXT:    kxnorw %k0, %k0, %k1 # encoding: [0xc5,0xfc,0x46,0xc8]
+; X64-NEXT:    vpexpandw (%rdi), %xmm0 {%k1} # encoding: [0x62,0xf2,0xfd,0x09,0x62,0x07]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %1 = bitcast i8* %addr to i16*
+  %2 = call <8 x i16> @llvm.masked.expandload.v8i16(i16* %1, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <8 x i16> %data)
+  ret <8 x i16> %2
+}
+
 define <8 x i16> @test_expand_w_128(<8 x i16> %data) {
 ; CHECK-LABEL: test_expand_w_128:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
-  %res = call <8 x i16> @llvm.x86.avx512.mask.expand.w.128(<8 x i16> %data, <8 x i16> undef, i8 -1)
-  ret <8 x i16> %res
+  %1 = call <8 x i16> @llvm.x86.avx512.mask.expand.v8i16(<8 x i16> %data, <8 x i16> undef, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>)
+  ret <8 x i16> %1
 }
 
 define <8 x i16> @test_mask_expand_w_128(<8 x i16> %data, <8 x i16> %passthru, i8 %mask) {
@@ -25,8 +83,9 @@ define <8 x i16> @test_mask_expand_w_128(<8 x i16> %data, <8 x i16> %passthru, i
 ; X64-NEXT:    vpexpandw %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0xfd,0x09,0x62,0xc8]
 ; X64-NEXT:    vmovdqa %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc1]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <8 x i16> @llvm.x86.avx512.mask.expand.w.128(<8 x i16> %data, <8 x i16> %passthru, i8 %mask)
-  ret <8 x i16> %res
+  %1 = bitcast i8 %mask to <8 x i1>
+  %2 = call <8 x i16> @llvm.x86.avx512.mask.expand.v8i16(<8 x i16> %data, <8 x i16> %passthru, <8 x i1> %1)
+  ret <8 x i16> %2
 }
 
 define <8 x i16> @test_maskz_expand_w_128(<8 x i16> %data, i8 %mask) {
@@ -42,18 +101,70 @@ define <8 x i16> @test_maskz_expand_w_128(<8 x i16> %data, i8 %mask) {
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpexpandw %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0x89,0x62,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <8 x i16> @llvm.x86.avx512.mask.expand.w.128(<8 x i16> %data, <8 x i16> zeroinitializer, i8 %mask)
-  ret <8 x i16> %res
+  %1 = bitcast i8 %mask to <8 x i1>
+  %2 = call <8 x i16> @llvm.x86.avx512.mask.expand.v8i16(<8 x i16> %data, <8 x i16> zeroinitializer, <8 x i1> %1)
+  ret <8 x i16> %2
 }
 
-declare <8 x i16> @llvm.x86.avx512.mask.expand.w.128(<8 x i16> %data, <8 x i16> %src0, i8 %mask)
+define <16 x i8> @test_mask_expand_load_b_128(i8* %addr, <16 x i8> %data, i16 %mask) {
+; X86-LABEL: test_mask_expand_load_b_128:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x08]
+; X86-NEXT:    vpexpandb (%eax), %xmm0 {%k1} # encoding: [0x62,0xf2,0x7d,0x09,0x62,0x00]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_mask_expand_load_b_128:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovd %esi, %k1 # encoding: [0xc5,0xfb,0x92,0xce]
+; X64-NEXT:    vpexpandb (%rdi), %xmm0 {%k1} # encoding: [0x62,0xf2,0x7d,0x09,0x62,0x07]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %1 = bitcast i16 %mask to <16 x i1>
+  %2 = call <16 x i8> @llvm.masked.expandload.v16i8(i8* %addr, <16 x i1> %1, <16 x i8> %data)
+  ret <16 x i8> %2
+}
+
+define <16 x i8> @test_maskz_expand_load_b_128(i8* %addr, i16 %mask) {
+; X86-LABEL: test_maskz_expand_load_b_128:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x08]
+; X86-NEXT:    vpexpandb (%eax), %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0x89,0x62,0x00]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_maskz_expand_load_b_128:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovd %esi, %k1 # encoding: [0xc5,0xfb,0x92,0xce]
+; X64-NEXT:    vpexpandb (%rdi), %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0x89,0x62,0x07]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %1 = bitcast i16 %mask to <16 x i1>
+  %2 = call <16 x i8> @llvm.masked.expandload.v16i8(i8* %addr, <16 x i1> %1, <16 x i8> zeroinitializer)
+  ret <16 x i8> %2
+}
+
+define <16 x i8> @test_expand_load_b_128(i8* %addr, <16 x i8> %data) {
+; X86-LABEL: test_expand_load_b_128:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-NEXT:    kxnorw %k0, %k0, %k1 # encoding: [0xc5,0xfc,0x46,0xc8]
+; X86-NEXT:    vpexpandb (%eax), %xmm0 {%k1} # encoding: [0x62,0xf2,0x7d,0x09,0x62,0x00]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_expand_load_b_128:
+; X64:       # %bb.0:
+; X64-NEXT:    kxnorw %k0, %k0, %k1 # encoding: [0xc5,0xfc,0x46,0xc8]
+; X64-NEXT:    vpexpandb (%rdi), %xmm0 {%k1} # encoding: [0x62,0xf2,0x7d,0x09,0x62,0x07]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %1 = call <16 x i8> @llvm.masked.expandload.v16i8(i8* %addr, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x i8> %data)
+  ret <16 x i8> %1
+}
 
 define <16 x i8> @test_expand_b_128(<16 x i8> %data) {
 ; CHECK-LABEL: test_expand_b_128:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
-  %res = call <16 x i8> @llvm.x86.avx512.mask.expand.b.128(<16 x i8> %data, <16 x i8> undef, i16 -1)
-  ret <16 x i8> %res
+  %1 = call <16 x i8> @llvm.x86.avx512.mask.expand.v16i8(<16 x i8> %data, <16 x i8> undef, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>)
+  ret <16 x i8> %1
 }
 
 define <16 x i8> @test_mask_expand_b_128(<16 x i8> %data, <16 x i8> %passthru, i16 %mask) {
@@ -70,8 +181,9 @@ define <16 x i8> @test_mask_expand_b_128(<16 x i8> %data, <16 x i8> %passthru, i
 ; X64-NEXT:    vpexpandb %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0x7d,0x09,0x62,0xc8]
 ; X64-NEXT:    vmovdqa %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc1]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <16 x i8> @llvm.x86.avx512.mask.expand.b.128(<16 x i8> %data, <16 x i8> %passthru, i16 %mask)
-  ret <16 x i8> %res
+  %1 = bitcast i16 %mask to <16 x i1>
+  %2 = call <16 x i8> @llvm.x86.avx512.mask.expand.v16i8(<16 x i8> %data, <16 x i8> %passthru, <16 x i1> %1)
+  ret <16 x i8> %2
 }
 
 define <16 x i8> @test_maskz_expand_b_128(<16 x i8> %data, i16 %mask) {
@@ -86,11 +198,48 @@ define <16 x i8> @test_maskz_expand_b_128(<16 x i8> %data, i16 %mask) {
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpexpandb %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0x89,0x62,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <16 x i8> @llvm.x86.avx512.mask.expand.b.128(<16 x i8> %data, <16 x i8> zeroinitializer, i16 %mask)
-  ret <16 x i8> %res
+  %1 = bitcast i16 %mask to <16 x i1>
+  %2 = call <16 x i8> @llvm.x86.avx512.mask.expand.v16i8(<16 x i8> %data, <16 x i8> zeroinitializer, <16 x i1> %1)
+  ret <16 x i8> %2
 }
 
-declare <16 x i8> @llvm.x86.avx512.mask.expand.b.128(<16 x i8> %data, <16 x i8> %src0, i16 %mask)
+define void @test_mask_compress_store_w_128(i8* %addr, <8 x i16> %data, i8 %mask) {
+; X86-LABEL: test_mask_compress_store_w_128:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx # encoding: [0x0f,0xb6,0x4c,0x24,0x08]
+; X86-NEXT:    kmovd %ecx, %k1 # encoding: [0xc5,0xfb,0x92,0xc9]
+; X86-NEXT:    vpcompressw %xmm0, (%eax) {%k1} # encoding: [0x62,0xf2,0xfd,0x09,0x63,0x00]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_mask_compress_store_w_128:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovd %esi, %k1 # encoding: [0xc5,0xfb,0x92,0xce]
+; X64-NEXT:    vpcompressw %xmm0, (%rdi) {%k1} # encoding: [0x62,0xf2,0xfd,0x09,0x63,0x07]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %1 = bitcast i8* %addr to i16*
+  %2 = bitcast i8 %mask to <8 x i1>
+  call void @llvm.masked.compressstore.v8i16(<8 x i16> %data, i16* %1, <8 x i1> %2)
+  ret void
+}
+
+define void @test_compress_store_w_128(i8* %addr, <8 x i16> %data) {
+; X86-LABEL: test_compress_store_w_128:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-NEXT:    kxnorw %k0, %k0, %k1 # encoding: [0xc5,0xfc,0x46,0xc8]
+; X86-NEXT:    vpcompressw %xmm0, (%eax) {%k1} # encoding: [0x62,0xf2,0xfd,0x09,0x63,0x00]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_compress_store_w_128:
+; X64:       # %bb.0:
+; X64-NEXT:    kxnorw %k0, %k0, %k1 # encoding: [0xc5,0xfc,0x46,0xc8]
+; X64-NEXT:    vpcompressw %xmm0, (%rdi) {%k1} # encoding: [0x62,0xf2,0xfd,0x09,0x63,0x07]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %1 = bitcast i8* %addr to i16*
+  call void @llvm.masked.compressstore.v8i16(<8 x i16> %data, i16* %1, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>)
+  ret void
+}
 
 define <8 x i16> @test_mask_compress_w_128(<8 x i16> %data, <8 x i16> %passthru, i8 %mask) {
 ; X86-LABEL: test_mask_compress_w_128:
@@ -107,8 +256,9 @@ define <8 x i16> @test_mask_compress_w_128(<8 x i16> %data, <8 x i16> %passthru,
 ; X64-NEXT:    vpcompressw %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0xfd,0x09,0x63,0xc1]
 ; X64-NEXT:    vmovdqa %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc1]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <8 x i16> @llvm.x86.avx512.mask.compress.w.128(<8 x i16> %data, <8 x i16> %passthru, i8 %mask)
-  ret <8 x i16> %res
+  %1 = bitcast i8 %mask to <8 x i1>
+  %2 = call <8 x i16> @llvm.x86.avx512.mask.compress.v8i16(<8 x i16> %data, <8 x i16> %passthru, <8 x i1> %1)
+  ret <8 x i16> %2
 }
 
 define <8 x i16> @test_maskz_compress_w_128(<8 x i16> %data, i8 %mask) {
@@ -124,19 +274,53 @@ define <8 x i16> @test_maskz_compress_w_128(<8 x i16> %data, i8 %mask) {
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpcompressw %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0x89,0x63,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <8 x i16> @llvm.x86.avx512.mask.compress.w.128(<8 x i16> %data, <8 x i16> zeroinitializer, i8 %mask)
-  ret <8 x i16> %res
+  %1 = bitcast i8 %mask to <8 x i1>
+  %2 = call <8 x i16> @llvm.x86.avx512.mask.compress.v8i16(<8 x i16> %data, <8 x i16> zeroinitializer, <8 x i1> %1)
+  ret <8 x i16> %2
 }
 
 define <8 x i16> @test_compress_w_128(<8 x i16> %data) {
 ; CHECK-LABEL: test_compress_w_128:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
-  %res = call <8 x i16> @llvm.x86.avx512.mask.compress.w.128(<8 x i16> %data, <8 x i16> undef, i8 -1)
-  ret <8 x i16> %res
+  %1 = call <8 x i16> @llvm.x86.avx512.mask.compress.v8i16(<8 x i16> %data, <8 x i16> undef, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>)
+  ret <8 x i16> %1
+}
+
+define void @test_mask_compress_store_b_128(i8* %addr, <16 x i8> %data, i16 %mask) {
+; X86-LABEL: test_mask_compress_store_b_128:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x08]
+; X86-NEXT:    vpcompressb %xmm0, (%eax) {%k1} # encoding: [0x62,0xf2,0x7d,0x09,0x63,0x00]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_mask_compress_store_b_128:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovd %esi, %k1 # encoding: [0xc5,0xfb,0x92,0xce]
+; X64-NEXT:    vpcompressb %xmm0, (%rdi) {%k1} # encoding: [0x62,0xf2,0x7d,0x09,0x63,0x07]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %1 = bitcast i16 %mask to <16 x i1>
+  call void @llvm.masked.compressstore.v16i8(<16 x i8> %data, i8* %addr, <16 x i1> %1)
+  ret void
 }
 
-declare <8 x i16> @llvm.x86.avx512.mask.compress.w.128(<8 x i16> %data, <8 x i16> %src0, i8 %mask)
+define void @test_compress_store_b_128(i8* %addr, <16 x i8> %data) {
+; X86-LABEL: test_compress_store_b_128:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-NEXT:    kxnorw %k0, %k0, %k1 # encoding: [0xc5,0xfc,0x46,0xc8]
+; X86-NEXT:    vpcompressb %xmm0, (%eax) {%k1} # encoding: [0x62,0xf2,0x7d,0x09,0x63,0x00]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_compress_store_b_128:
+; X64:       # %bb.0:
+; X64-NEXT:    kxnorw %k0, %k0, %k1 # encoding: [0xc5,0xfc,0x46,0xc8]
+; X64-NEXT:    vpcompressb %xmm0, (%rdi) {%k1} # encoding: [0x62,0xf2,0x7d,0x09,0x63,0x07]
+; X64-NEXT:    retq # encoding: [0xc3]
+  call void @llvm.masked.compressstore.v16i8(<16 x i8> %data, i8* %addr, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>)
+  ret void
+}
 
 define <16 x i8> @test_mask_compress_b_128(<16 x i8> %data, <16 x i8> %passthru, i16 %mask) {
 ; X86-LABEL: test_mask_compress_b_128:
@@ -152,8 +336,9 @@ define <16 x i8> @test_mask_compress_b_128(<16 x i8> %data, <16 x i8> %passthru,
 ; X64-NEXT:    vpcompressb %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0x7d,0x09,0x63,0xc1]
 ; X64-NEXT:    vmovdqa %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc1]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <16 x i8> @llvm.x86.avx512.mask.compress.b.128(<16 x i8> %data, <16 x i8> %passthru, i16 %mask)
-  ret <16 x i8> %res
+  %1 = bitcast i16 %mask to <16 x i1>
+  %2 = call <16 x i8> @llvm.x86.avx512.mask.compress.v16i8(<16 x i8> %data, <16 x i8> %passthru, <16 x i1> %1)
+  ret <16 x i8> %2
 }
 
 define <16 x i8> @test_maskz_compress_b_128(<16 x i8> %data, i16 %mask) {
@@ -168,26 +353,81 @@ define <16 x i8> @test_maskz_compress_b_128(<16 x i8> %data, i16 %mask) {
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpcompressb %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0x89,0x63,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <16 x i8> @llvm.x86.avx512.mask.compress.b.128(<16 x i8> %data, <16 x i8> zeroinitializer, i16 %mask)
-  ret <16 x i8> %res
+  %1 = bitcast i16 %mask to <16 x i1>
+  %2 = call <16 x i8> @llvm.x86.avx512.mask.compress.v16i8(<16 x i8> %data, <16 x i8> zeroinitializer, <16 x i1> %1)
+  ret <16 x i8> %2
 }
 
 define <16 x i8> @test_compress_b_128(<16 x i8> %data) {
 ; CHECK-LABEL: test_compress_b_128:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
-  %res = call <16 x i8> @llvm.x86.avx512.mask.compress.b.128(<16 x i8> %data, <16 x i8> undef, i16 -1)
-  ret <16 x i8> %res
+  %1 = call <16 x i8> @llvm.x86.avx512.mask.compress.v16i8(<16 x i8> %data, <16 x i8> undef, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>)
+  ret <16 x i8> %1
+}
+
+define <16 x i16> @test_mask_expand_load_w_256(i8* %addr, <16 x i16> %data, i16 %mask) {
+; X86-LABEL: test_mask_expand_load_w_256:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x08]
+; X86-NEXT:    vpexpandw (%eax), %ymm0 {%k1} # encoding: [0x62,0xf2,0xfd,0x29,0x62,0x00]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_mask_expand_load_w_256:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovd %esi, %k1 # encoding: [0xc5,0xfb,0x92,0xce]
+; X64-NEXT:    vpexpandw (%rdi), %ymm0 {%k1} # encoding: [0x62,0xf2,0xfd,0x29,0x62,0x07]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %1 = bitcast i8* %addr to i16*
+  %2 = bitcast i16 %mask to <16 x i1>
+  %3 = call <16 x i16> @llvm.masked.expandload.v16i16(i16* %1, <16 x i1> %2, <16 x i16> %data)
+  ret <16 x i16> %3
+}
+
+define <16 x i16> @test_maskz_expand_load_w_256(i8* %addr, i16 %mask) {
+; X86-LABEL: test_maskz_expand_load_w_256:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x08]
+; X86-NEXT:    vpexpandw (%eax), %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xa9,0x62,0x00]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_maskz_expand_load_w_256:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovd %esi, %k1 # encoding: [0xc5,0xfb,0x92,0xce]
+; X64-NEXT:    vpexpandw (%rdi), %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xa9,0x62,0x07]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %1 = bitcast i8* %addr to i16*
+  %2 = bitcast i16 %mask to <16 x i1>
+  %3 = call <16 x i16> @llvm.masked.expandload.v16i16(i16* %1, <16 x i1> %2, <16 x i16> zeroinitializer)
+  ret <16 x i16> %3
 }
 
-declare <16 x i8> @llvm.x86.avx512.mask.compress.b.128(<16 x i8> %data, <16 x i8> %src0, i16 %mask)
+define <16 x i16> @test_expand_load_w_256(i8* %addr, <16 x i16> %data) {
+; X86-LABEL: test_expand_load_w_256:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-NEXT:    kxnorw %k0, %k0, %k1 # encoding: [0xc5,0xfc,0x46,0xc8]
+; X86-NEXT:    vpexpandw (%eax), %ymm0 {%k1} # encoding: [0x62,0xf2,0xfd,0x29,0x62,0x00]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_expand_load_w_256:
+; X64:       # %bb.0:
+; X64-NEXT:    kxnorw %k0, %k0, %k1 # encoding: [0xc5,0xfc,0x46,0xc8]
+; X64-NEXT:    vpexpandw (%rdi), %ymm0 {%k1} # encoding: [0x62,0xf2,0xfd,0x29,0x62,0x07]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %1 = bitcast i8* %addr to i16*
+  %2 = call <16 x i16> @llvm.masked.expandload.v16i16(i16* %1, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <16 x i16> %data)
+  ret <16 x i16> %2
+}
 
 define <16 x i16> @test_expand_w_256(<16 x i16> %data) {
 ; CHECK-LABEL: test_expand_w_256:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
-  %res = call <16 x i16> @llvm.x86.avx512.mask.expand.w.256(<16 x i16> %data, <16 x i16> undef, i16 -1)
-  ret <16 x i16> %res
+  %1 = call <16 x i16> @llvm.x86.avx512.mask.expand.v16i16(<16 x i16> %data, <16 x i16> undef, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>)
+  ret <16 x i16> %1
 }
 
 define <16 x i16> @test_mask_expand_w_256(<16 x i16> %data, <16 x i16> %passthru, i16 %mask) {
@@ -204,8 +444,9 @@ define <16 x i16> @test_mask_expand_w_256(<16 x i16> %data, <16 x i16> %passthru
 ; X64-NEXT:    vpexpandw %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf2,0xfd,0x29,0x62,0xc8]
 ; X64-NEXT:    vmovdqa %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc1]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <16 x i16> @llvm.x86.avx512.mask.expand.w.256(<16 x i16> %data, <16 x i16> %passthru, i16 %mask)
-  ret <16 x i16> %res
+  %1 = bitcast i16 %mask to <16 x i1>
+  %2 = call <16 x i16> @llvm.x86.avx512.mask.expand.v16i16(<16 x i16> %data, <16 x i16> %passthru, <16 x i1> %1)
+  ret <16 x i16> %2
 }
 
 define <16 x i16> @test_maskz_expand_w_256(<16 x i16> %data, i16 %mask) {
@@ -220,18 +461,70 @@ define <16 x i16> @test_maskz_expand_w_256(<16 x i16> %data, i16 %mask) {
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpexpandw %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xa9,0x62,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <16 x i16> @llvm.x86.avx512.mask.expand.w.256(<16 x i16> %data, <16 x i16> zeroinitializer, i16 %mask)
-  ret <16 x i16> %res
+  %1 = bitcast i16 %mask to <16 x i1>
+  %2 = call <16 x i16> @llvm.x86.avx512.mask.expand.v16i16(<16 x i16> %data, <16 x i16> zeroinitializer, <16 x i1> %1)
+  ret <16 x i16> %2
 }
 
-declare <16 x i16> @llvm.x86.avx512.mask.expand.w.256(<16 x i16> %data, <16 x i16> %src0, i16 %mask)
+define <32 x i8> @test_mask_expand_load_b_256(i8* %addr, <32 x i8> %data, i32 %mask) {
+; X86-LABEL: test_mask_expand_load_b_256:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf9,0x90,0x4c,0x24,0x08]
+; X86-NEXT:    vpexpandb (%eax), %ymm0 {%k1} # encoding: [0x62,0xf2,0x7d,0x29,0x62,0x00]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_mask_expand_load_b_256:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovd %esi, %k1 # encoding: [0xc5,0xfb,0x92,0xce]
+; X64-NEXT:    vpexpandb (%rdi), %ymm0 {%k1} # encoding: [0x62,0xf2,0x7d,0x29,0x62,0x07]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %1 = bitcast i32 %mask to <32 x i1>
+  %2 = call <32 x i8> @llvm.masked.expandload.v32i8(i8* %addr, <32 x i1> %1, <32 x i8> %data)
+  ret <32 x i8> %2
+}
+
+define <32 x i8> @test_maskz_expand_load_b_256(i8* %addr, i32 %mask) {
+; X86-LABEL: test_maskz_expand_load_b_256:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf9,0x90,0x4c,0x24,0x08]
+; X86-NEXT:    vpexpandb (%eax), %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0xa9,0x62,0x00]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_maskz_expand_load_b_256:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovd %esi, %k1 # encoding: [0xc5,0xfb,0x92,0xce]
+; X64-NEXT:    vpexpandb (%rdi), %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0xa9,0x62,0x07]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %1 = bitcast i32 %mask to <32 x i1>
+  %2 = call <32 x i8> @llvm.masked.expandload.v32i8(i8* %addr, <32 x i1> %1, <32 x i8> zeroinitializer)
+  ret <32 x i8> %2
+}
+
+define <32 x i8> @test_expand_load_b_256(i8* %addr, <32 x i8> %data) {
+; X86-LABEL: test_expand_load_b_256:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-NEXT:    kxnord %k0, %k0, %k1 # encoding: [0xc4,0xe1,0xfd,0x46,0xc8]
+; X86-NEXT:    vpexpandb (%eax), %ymm0 {%k1} # encoding: [0x62,0xf2,0x7d,0x29,0x62,0x00]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_expand_load_b_256:
+; X64:       # %bb.0:
+; X64-NEXT:    kxnord %k0, %k0, %k1 # encoding: [0xc4,0xe1,0xfd,0x46,0xc8]
+; X64-NEXT:    vpexpandb (%rdi), %ymm0 {%k1} # encoding: [0x62,0xf2,0x7d,0x29,0x62,0x07]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %1 = call <32 x i8> @llvm.masked.expandload.v32i8(i8* %addr, <32 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, <32 x i8> %data)
+  ret <32 x i8> %1
+}
 
 define <32 x i8> @test_expand_b_256(<32 x i8> %data) {
 ; CHECK-LABEL: test_expand_b_256:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
-  %res = call <32 x i8> @llvm.x86.avx512.mask.expand.b.256(<32 x i8> %data, <32 x i8> undef, i32 -1)
-  ret <32 x i8> %res
+  %1 = call <32 x i8> @llvm.x86.avx512.mask.expand.v32i8(<32 x i8> %data, <32 x i8> undef, <32 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>)
+  ret <32 x i8> %1
 }
 
 define <32 x i8> @test_mask_expand_b_256(<32 x i8> %data, <32 x i8> %passthru, i32 %mask) {
@@ -248,8 +541,9 @@ define <32 x i8> @test_mask_expand_b_256(<32 x i8> %data, <32 x i8> %passthru, i
 ; X64-NEXT:    vpexpandb %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf2,0x7d,0x29,0x62,0xc8]
 ; X64-NEXT:    vmovdqa %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc1]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <32 x i8> @llvm.x86.avx512.mask.expand.b.256(<32 x i8> %data, <32 x i8> %passthru, i32 %mask)
-  ret <32 x i8> %res
+  %1 = bitcast i32 %mask to <32 x i1>
+  %2 = call <32 x i8> @llvm.x86.avx512.mask.expand.v32i8(<32 x i8> %data, <32 x i8> %passthru, <32 x i1> %1)
+  ret <32 x i8> %2
 }
 
 define <32 x i8> @test_maskz_expand_b_256(<32 x i8> %data, i32 %mask) {
@@ -264,11 +558,51 @@ define <32 x i8> @test_maskz_expand_b_256(<32 x i8> %data, i32 %mask) {
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpexpandb %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0xa9,0x62,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <32 x i8> @llvm.x86.avx512.mask.expand.b.256(<32 x i8> %data, <32 x i8> zeroinitializer, i32 %mask)
-  ret <32 x i8> %res
+  %1 = bitcast i32 %mask to <32 x i1>
+  %2 = call <32 x i8> @llvm.x86.avx512.mask.expand.v32i8(<32 x i8> %data, <32 x i8> zeroinitializer, <32 x i1> %1)
+  ret <32 x i8> %2
 }
 
-declare <32 x i8> @llvm.x86.avx512.mask.expand.b.256(<32 x i8> %data, <32 x i8> %src0, i32 %mask)
+define void @test_mask_compress_store_w_256(i8* %addr, <16 x i16> %data, i16 %mask) {
+; X86-LABEL: test_mask_compress_store_w_256:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x08]
+; X86-NEXT:    vpcompressw %ymm0, (%eax) {%k1} # encoding: [0x62,0xf2,0xfd,0x29,0x63,0x00]
+; X86-NEXT:    vzeroupper # encoding: [0xc5,0xf8,0x77]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_mask_compress_store_w_256:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovd %esi, %k1 # encoding: [0xc5,0xfb,0x92,0xce]
+; X64-NEXT:    vpcompressw %ymm0, (%rdi) {%k1} # encoding: [0x62,0xf2,0xfd,0x29,0x63,0x07]
+; X64-NEXT:    vzeroupper # encoding: [0xc5,0xf8,0x77]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %1 = bitcast i8* %addr to i16*
+  %2 = bitcast i16 %mask to <16 x i1>
+  call void @llvm.masked.compressstore.v16i16(<16 x i16> %data, i16* %1, <16 x i1> %2)
+  ret void
+}
+
+define void @test_compress_store_w_256(i8* %addr, <16 x i16> %data) {
+; X86-LABEL: test_compress_store_w_256:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-NEXT:    kxnorw %k0, %k0, %k1 # encoding: [0xc5,0xfc,0x46,0xc8]
+; X86-NEXT:    vpcompressw %ymm0, (%eax) {%k1} # encoding: [0x62,0xf2,0xfd,0x29,0x63,0x00]
+; X86-NEXT:    vzeroupper # encoding: [0xc5,0xf8,0x77]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_compress_store_w_256:
+; X64:       # %bb.0:
+; X64-NEXT:    kxnorw %k0, %k0, %k1 # encoding: [0xc5,0xfc,0x46,0xc8]
+; X64-NEXT:    vpcompressw %ymm0, (%rdi) {%k1} # encoding: [0x62,0xf2,0xfd,0x29,0x63,0x07]
+; X64-NEXT:    vzeroupper # encoding: [0xc5,0xf8,0x77]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %1 = bitcast i8* %addr to i16*
+  call void @llvm.masked.compressstore.v16i16(<16 x i16> %data, i16* %1, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>)
+  ret void
+}
 
 define <16 x i16> @test_mask_compress_w_256(<16 x i16> %data, <16 x i16> %passthru, i16 %mask) {
 ; X86-LABEL: test_mask_compress_w_256:
@@ -284,8 +618,9 @@ define <16 x i16> @test_mask_compress_w_256(<16 x i16> %data, <16 x i16> %passth
 ; X64-NEXT:    vpcompressw %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf2,0xfd,0x29,0x63,0xc1]
 ; X64-NEXT:    vmovdqa %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc1]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <16 x i16> @llvm.x86.avx512.mask.compress.w.256(<16 x i16> %data, <16 x i16> %passthru, i16 %mask)
-  ret <16 x i16> %res
+  %1 = bitcast i16 %mask to <16 x i1>
+  %2 = call <16 x i16> @llvm.x86.avx512.mask.compress.v16i16(<16 x i16> %data, <16 x i16> %passthru, <16 x i1> %1)
+  ret <16 x i16> %2
 }
 
 define <16 x i16> @test_maskz_compress_w_256(<16 x i16> %data, i16 %mask) {
@@ -300,19 +635,57 @@ define <16 x i16> @test_maskz_compress_w_256(<16 x i16> %data, i16 %mask) {
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpcompressw %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xa9,0x63,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <16 x i16> @llvm.x86.avx512.mask.compress.w.256(<16 x i16> %data, <16 x i16> zeroinitializer, i16 %mask)
-  ret <16 x i16> %res
+  %1 = bitcast i16 %mask to <16 x i1>
+  %2 = call <16 x i16> @llvm.x86.avx512.mask.compress.v16i16(<16 x i16> %data, <16 x i16> zeroinitializer, <16 x i1> %1)
+  ret <16 x i16> %2
 }
 
 define <16 x i16> @test_compress_w_256(<16 x i16> %data) {
 ; CHECK-LABEL: test_compress_w_256:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
-  %res = call <16 x i16> @llvm.x86.avx512.mask.compress.w.256(<16 x i16> %data, <16 x i16> undef, i16 -1)
-  ret <16 x i16> %res
+  %1 = call <16 x i16> @llvm.x86.avx512.mask.compress.v16i16(<16 x i16> %data, <16 x i16> undef, <16 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>)
+  ret <16 x i16> %1
 }
 
-declare <16 x i16> @llvm.x86.avx512.mask.compress.w.256(<16 x i16> %data, <16 x i16> %src0, i16 %mask)
+define void @test_mask_compress_store_b_256(i8* %addr, <32 x i8> %data, i32 %mask) {
+; X86-LABEL: test_mask_compress_store_b_256:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf9,0x90,0x4c,0x24,0x08]
+; X86-NEXT:    vpcompressb %ymm0, (%eax) {%k1} # encoding: [0x62,0xf2,0x7d,0x29,0x63,0x00]
+; X86-NEXT:    vzeroupper # encoding: [0xc5,0xf8,0x77]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_mask_compress_store_b_256:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovd %esi, %k1 # encoding: [0xc5,0xfb,0x92,0xce]
+; X64-NEXT:    vpcompressb %ymm0, (%rdi) {%k1} # encoding: [0x62,0xf2,0x7d,0x29,0x63,0x07]
+; X64-NEXT:    vzeroupper # encoding: [0xc5,0xf8,0x77]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %1 = bitcast i32 %mask to <32 x i1>
+  call void @llvm.masked.compressstore.v32i8(<32 x i8> %data, i8* %addr, <32 x i1> %1)
+  ret void
+}
+
+define void @test_compress_store_b_256(i8* %addr, <32 x i8> %data) {
+; X86-LABEL: test_compress_store_b_256:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-NEXT:    kxnord %k0, %k0, %k1 # encoding: [0xc4,0xe1,0xfd,0x46,0xc8]
+; X86-NEXT:    vpcompressb %ymm0, (%eax) {%k1} # encoding: [0x62,0xf2,0x7d,0x29,0x63,0x00]
+; X86-NEXT:    vzeroupper # encoding: [0xc5,0xf8,0x77]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_compress_store_b_256:
+; X64:       # %bb.0:
+; X64-NEXT:    kxnord %k0, %k0, %k1 # encoding: [0xc4,0xe1,0xfd,0x46,0xc8]
+; X64-NEXT:    vpcompressb %ymm0, (%rdi) {%k1} # encoding: [0x62,0xf2,0x7d,0x29,0x63,0x07]
+; X64-NEXT:    vzeroupper # encoding: [0xc5,0xf8,0x77]
+; X64-NEXT:    retq # encoding: [0xc3]
+  call void @llvm.masked.compressstore.v32i8(<32 x i8> %data, i8* %addr, <32 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>)
+  ret void
+}
 
 define <32 x i8> @test_mask_compress_b_256(<32 x i8> %data, <32 x i8> %passthru, i32 %mask) {
 ; X86-LABEL: test_mask_compress_b_256:
@@ -328,8 +701,9 @@ define <32 x i8> @test_mask_compress_b_256(<32 x i8> %data, <32 x i8> %passthru,
 ; X64-NEXT:    vpcompressb %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf2,0x7d,0x29,0x63,0xc1]
 ; X64-NEXT:    vmovdqa %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc1]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <32 x i8> @llvm.x86.avx512.mask.compress.b.256(<32 x i8> %data, <32 x i8> %passthru, i32 %mask)
-  ret <32 x i8> %res
+  %1 = bitcast i32 %mask to <32 x i1>
+  %2 = call <32 x i8> @llvm.x86.avx512.mask.compress.v32i8(<32 x i8> %data, <32 x i8> %passthru, <32 x i1> %1)
+  ret <32 x i8> %2
 }
 
 define <32 x i8> @test_maskz_compress_b_256(<32 x i8> %data, i32 %mask) {
@@ -344,21 +718,20 @@ define <32 x i8> @test_maskz_compress_b_256(<32 x i8> %data, i32 %mask) {
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpcompressb %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0xa9,0x63,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <32 x i8> @llvm.x86.avx512.mask.compress.b.256(<32 x i8> %data, <32 x i8> zeroinitializer, i32 %mask)
-  ret <32 x i8> %res
+  %1 = bitcast i32 %mask to <32 x i1>
+  %2 = call <32 x i8> @llvm.x86.avx512.mask.compress.v32i8(<32 x i8> %data, <32 x i8> zeroinitializer, <32 x i1> %1)
+  ret <32 x i8> %2
 }
 
 define <32 x i8> @test_compress_b_256(<32 x i8> %data) {
 ; CHECK-LABEL: test_compress_b_256:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
-  %res = call <32 x i8> @llvm.x86.avx512.mask.compress.b.256(<32 x i8> %data, <32 x i8> undef, i32 -1)
-  ret <32 x i8> %res
+  %1 = call <32 x i8> @llvm.x86.avx512.mask.compress.v32i8(<32 x i8> %data, <32 x i8> undef, <32 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>)
+  ret <32 x i8> %1
 }
 
-declare <32 x i8> @llvm.x86.avx512.mask.compress.b.256(<32 x i8> %data, <32 x i8> %src0, i32 %mask)
-
-define <4 x i32>@test_int_x86_avx512_mask_vpshld_d_128(<4 x i32> %x0, <4 x i32> %x1,<4 x i32> %x3, i8 %x4) {
+define <4 x i32> @test_int_x86_avx512_mask_vpshld_d_128(<4 x i32> %x0, <4 x i32> %x1,<4 x i32> %x3, i8 %x4) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpshld_d_128:
 ; X86:       # %bb.0:
 ; X86-NEXT:    vpshldd $22, %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf3,0x7d,0x08,0x71,0xd9,0x16]
@@ -379,12 +752,12 @@ define <4 x i32>@test_int_x86_avx512_mask_vpshld_d_128(<4 x i32> %x0, <4 x i32>
 ; X64-NEXT:    vpaddd %xmm0, %xmm3, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe1,0xfe,0xc0]
 ; X64-NEXT:    vpaddd %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfe,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %1 = call <4 x i32> @llvm.x86.avx512.vpshld.d.128(<4 x i32> %x0, <4 x i32> %x1, i32 22)
+  %1 = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> <i32 22, i32 22, i32 22, i32 22>)
   %2 = bitcast i8 %x4 to <8 x i1>
   %extract1 = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %3 = select <4 x i1> %extract1, <4 x i32> %1, <4 x i32> %x3
-  %4 = call <4 x i32> @llvm.x86.avx512.vpshld.d.128(<4 x i32> %x0, <4 x i32> %x1, i32 22)
-  %5 = call <4 x i32> @llvm.x86.avx512.vpshld.d.128(<4 x i32> %x0, <4 x i32> %x1, i32 22)
+  %4 = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> <i32 22, i32 22, i32 22, i32 22>)
+  %5 = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> <i32 22, i32 22, i32 22, i32 22>)
   %6 = bitcast i8 %x4 to <8 x i1>
   %extract = shufflevector <8 x i1> %6, <8 x i1> %6, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %7 = select <4 x i1> %extract, <4 x i32> %5, <4 x i32> zeroinitializer
@@ -392,9 +765,8 @@ define <4 x i32>@test_int_x86_avx512_mask_vpshld_d_128(<4 x i32> %x0, <4 x i32>
   %res4 = add <4 x i32> %res3, %7
   ret <4 x i32> %res4
 }
-declare <4 x i32> @llvm.x86.avx512.vpshld.d.128(<4 x i32>, <4 x i32>, i32)
 
-define <8 x i32>@test_int_x86_avx512_mask_vpshld_d_256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x3, i8 %x4) {
+define <8 x i32> @test_int_x86_avx512_mask_vpshld_d_256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x3, i8 %x4) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpshld_d_256:
 ; X86:       # %bb.0:
 ; X86-NEXT:    vpshldd $22, %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf3,0x7d,0x28,0x71,0xd9,0x16]
@@ -411,16 +783,15 @@ define <8 x i32>@test_int_x86_avx512_mask_vpshld_d_256(<8 x i32> %x0, <8 x i32>
 ; X64-NEXT:    vpshldd $22, %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf3,0x7d,0x29,0x71,0xd1,0x16]
 ; X64-NEXT:    vpaddd %ymm3, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfe,0xc3]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %1 = call <8 x i32> @llvm.x86.avx512.vpshld.d.256(<8 x i32> %x0, <8 x i32> %x1, i32 22)
+  %1 = call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> <i32 22, i32 22, i32 22, i32 22, i32 22, i32 22, i32 22, i32 22>)
   %2 = bitcast i8 %x4 to <8 x i1>
   %3 = select <8 x i1> %2, <8 x i32> %1, <8 x i32> %x3
-  %4 = call <8 x i32> @llvm.x86.avx512.vpshld.d.256(<8 x i32> %x0, <8 x i32> %x1, i32 22)
+  %4 = call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> <i32 22, i32 22, i32 22, i32 22, i32 22, i32 22, i32 22, i32 22>)
   %res2 = add <8 x i32> %3, %4
   ret <8 x i32> %res2
 }
-declare <8 x i32> @llvm.x86.avx512.vpshld.d.256(<8 x i32>, <8 x i32>, i32)
 
-define <2 x i64>@test_int_x86_avx512_mask_vpshld_q_128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x3, i8 %x4) {
+define <2 x i64> @test_int_x86_avx512_mask_vpshld_q_128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x3, i8 %x4) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpshld_q_128:
 ; X86:       # %bb.0:
 ; X86-NEXT:    vpshldq $22, %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf3,0xfd,0x08,0x71,0xd9,0x16]
@@ -437,17 +808,16 @@ define <2 x i64>@test_int_x86_avx512_mask_vpshld_q_128(<2 x i64> %x0, <2 x i64>
 ; X64-NEXT:    vpshldq $22, %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x09,0x71,0xd1,0x16]
 ; X64-NEXT:    vpaddq %xmm3, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xd4,0xc3]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %1 = call <2 x i64> @llvm.x86.avx512.vpshld.q.128(<2 x i64> %x0, <2 x i64> %x1, i32 22)
+  %1 = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> <i64 22, i64 22>)
   %2 = bitcast i8 %x4 to <8 x i1>
   %extract = shufflevector <8 x i1> %2, <8 x i1> %2, <2 x i32> <i32 0, i32 1>
   %3 = select <2 x i1> %extract, <2 x i64> %1, <2 x i64> %x3
-  %4 = call <2 x i64> @llvm.x86.avx512.vpshld.q.128(<2 x i64> %x0, <2 x i64> %x1, i32 22)
+  %4 = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> <i64 22, i64 22>)
   %res2 = add <2 x i64> %3, %4
   ret <2 x i64> %res2
 }
-declare <2 x i64> @llvm.x86.avx512.vpshld.q.128(<2 x i64>, <2 x i64>, i32)
 
-define <4 x i64>@test_int_x86_avx512_mask_vpshld_q_256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x3, i8 %x4) {
+define <4 x i64> @test_int_x86_avx512_mask_vpshld_q_256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x3, i8 %x4) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpshld_q_256:
 ; X86:       # %bb.0:
 ; X86-NEXT:    vpshldq $22, %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf3,0xfd,0x28,0x71,0xd9,0x16]
@@ -464,68 +834,65 @@ define <4 x i64>@test_int_x86_avx512_mask_vpshld_q_256(<4 x i64> %x0, <4 x i64>
 ; X64-NEXT:    vpshldq $22, %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x29,0x71,0xd1,0x16]
 ; X64-NEXT:    vpaddq %ymm3, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xd4,0xc3]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %1 = call <4 x i64> @llvm.x86.avx512.vpshld.q.256(<4 x i64> %x0, <4 x i64> %x1, i32 22)
+  %1 = call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> <i64 22, i64 22, i64 22, i64 22>)
   %2 = bitcast i8 %x4 to <8 x i1>
   %extract = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %3 = select <4 x i1> %extract, <4 x i64> %1, <4 x i64> %x3
-  %4 = call <4 x i64> @llvm.x86.avx512.vpshld.q.256(<4 x i64> %x0, <4 x i64> %x1, i32 22)
+  %4 = call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> <i64 22, i64 22, i64 22, i64 22>)
   %res2 = add <4 x i64> %3, %4
   ret <4 x i64> %res2
 }
-declare <4 x i64> @llvm.x86.avx512.vpshld.q.256(<4 x i64>, <4 x i64>, i32)
 
-define <8 x i16>@test_int_x86_avx512_mask_vpshld_w_128(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x3, i8 %x4) {
+define <8 x i16> @test_int_x86_avx512_mask_vpshld_w_128(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x3, i8 %x4) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpshld_w_128:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpshldw $22, %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf3,0xfd,0x08,0x70,0xd9,0x16]
+; X86-NEXT:    vpshldw $6, %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf3,0xfd,0x08,0x70,0xd9,0x06]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
-; X86-NEXT:    vpshldw $22, %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x09,0x70,0xd1,0x16]
+; X86-NEXT:    vpshldw $6, %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x09,0x70,0xd1,0x06]
 ; X86-NEXT:    vpaddw %xmm3, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfd,0xc3]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_vpshld_w_128:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpshldw $22, %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf3,0xfd,0x08,0x70,0xd9,0x16]
+; X64-NEXT:    vpshldw $6, %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf3,0xfd,0x08,0x70,0xd9,0x06]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
-; X64-NEXT:    vpshldw $22, %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x09,0x70,0xd1,0x16]
+; X64-NEXT:    vpshldw $6, %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x09,0x70,0xd1,0x06]
 ; X64-NEXT:    vpaddw %xmm3, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfd,0xc3]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %1 = call <8 x i16> @llvm.x86.avx512.vpshld.w.128(<8 x i16> %x0, <8 x i16> %x1, i32 22)
+  %1 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> <i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6>)
   %2 = bitcast i8 %x4 to <8 x i1>
   %3 = select <8 x i1> %2, <8 x i16> %1, <8 x i16> %x3
-  %4 = call <8 x i16> @llvm.x86.avx512.vpshld.w.128(<8 x i16> %x0, <8 x i16> %x1, i32 22)
+  %4 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> <i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6>)
   %res2 = add <8 x i16> %3, %4
   ret <8 x i16> %res2
 }
-declare <8 x i16> @llvm.x86.avx512.vpshld.w.128(<8 x i16>, <8 x i16>, i32)
 
-define <16 x i16>@test_int_x86_avx512_mask_vpshld_w_256(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x3, i16 %x4) {
+define <16 x i16> @test_int_x86_avx512_mask_vpshld_w_256(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x3, i16 %x4) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpshld_w_256:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpshldw $22, %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf3,0xfd,0x28,0x70,0xd9,0x16]
+; X86-NEXT:    vpshldw $6, %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf3,0xfd,0x28,0x70,0xd9,0x06]
 ; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
-; X86-NEXT:    vpshldw $22, %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x29,0x70,0xd1,0x16]
+; X86-NEXT:    vpshldw $6, %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x29,0x70,0xd1,0x06]
 ; X86-NEXT:    vpaddw %ymm3, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfd,0xc3]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_vpshld_w_256:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpshldw $22, %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf3,0xfd,0x28,0x70,0xd9,0x16]
+; X64-NEXT:    vpshldw $6, %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf3,0xfd,0x28,0x70,0xd9,0x06]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
-; X64-NEXT:    vpshldw $22, %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x29,0x70,0xd1,0x16]
+; X64-NEXT:    vpshldw $6, %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x29,0x70,0xd1,0x06]
 ; X64-NEXT:    vpaddw %ymm3, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfd,0xc3]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %1 = call <16 x i16> @llvm.x86.avx512.vpshld.w.256(<16 x i16> %x0, <16 x i16> %x1, i32 22)
+  %1 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> <i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6>)
   %2 = bitcast i16 %x4 to <16 x i1>
   %3 = select <16 x i1> %2, <16 x i16> %1, <16 x i16> %x3
-  %4 = call <16 x i16> @llvm.x86.avx512.vpshld.w.256(<16 x i16> %x0, <16 x i16> %x1, i32 22)
+  %4 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> <i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6>)
   %res2 = add <16 x i16> %3, %4
   ret <16 x i16> %res2
 }
-declare <16 x i16> @llvm.x86.avx512.vpshld.w.256(<16 x i16>, <16 x i16>, i32)
 
-define <4 x i32>@test_int_x86_avx512_mask_vpshrd_d_128(<4 x i32> %x0, <4 x i32> %x1,<4 x i32> %x3, i8 %x4) {
+define <4 x i32> @test_int_x86_avx512_mask_vpshrd_d_128(<4 x i32> %x0, <4 x i32> %x1,<4 x i32> %x3, i8 %x4) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpshrd_d_128:
 ; X86:       # %bb.0:
 ; X86-NEXT:    vpshrdd $22, %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf3,0x7d,0x08,0x73,0xd9,0x16]
@@ -546,12 +913,12 @@ define <4 x i32>@test_int_x86_avx512_mask_vpshrd_d_128(<4 x i32> %x0, <4 x i32>
 ; X64-NEXT:    vpaddd %xmm0, %xmm3, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe1,0xfe,0xc0]
 ; X64-NEXT:    vpaddd %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfe,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %1 = call <4 x i32> @llvm.x86.avx512.vpshrd.d.128(<4 x i32> %x0, <4 x i32> %x1, i32 22)
+  %1 = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %x1, <4 x i32> %x0, <4 x i32> <i32 22, i32 22, i32 22, i32 22>)
   %2 = bitcast i8 %x4 to <8 x i1>
   %extract1 = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %3 = select <4 x i1> %extract1, <4 x i32> %1, <4 x i32> %x3
-  %4 = call <4 x i32> @llvm.x86.avx512.vpshrd.d.128(<4 x i32> %x0, <4 x i32> %x1, i32 22)
-  %5 = call <4 x i32> @llvm.x86.avx512.vpshrd.d.128(<4 x i32> %x0, <4 x i32> %x1, i32 22)
+  %4 = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %x1, <4 x i32> %x0, <4 x i32> <i32 22, i32 22, i32 22, i32 22>)
+  %5 = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %x1, <4 x i32> %x0, <4 x i32> <i32 22, i32 22, i32 22, i32 22>)
   %6 = bitcast i8 %x4 to <8 x i1>
   %extract = shufflevector <8 x i1> %6, <8 x i1> %6, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %7 = select <4 x i1> %extract, <4 x i32> %5, <4 x i32> zeroinitializer
@@ -559,9 +926,8 @@ define <4 x i32>@test_int_x86_avx512_mask_vpshrd_d_128(<4 x i32> %x0, <4 x i32>
   %res4 = add <4 x i32> %res3, %7
   ret <4 x i32> %res4
 }
-declare <4 x i32> @llvm.x86.avx512.vpshrd.d.128(<4 x i32>, <4 x i32>, i32)
 
-define <8 x i32>@test_int_x86_avx512_mask_vpshrd_d_256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x3, i8 %x4) {
+define <8 x i32> @test_int_x86_avx512_mask_vpshrd_d_256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x3, i8 %x4) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpshrd_d_256:
 ; X86:       # %bb.0:
 ; X86-NEXT:    vpshrdd $22, %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf3,0x7d,0x28,0x73,0xd9,0x16]
@@ -578,16 +944,15 @@ define <8 x i32>@test_int_x86_avx512_mask_vpshrd_d_256(<8 x i32> %x0, <8 x i32>
 ; X64-NEXT:    vpshrdd $22, %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf3,0x7d,0x29,0x73,0xd1,0x16]
 ; X64-NEXT:    vpaddd %ymm3, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfe,0xc3]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %1 = call <8 x i32> @llvm.x86.avx512.vpshrd.d.256(<8 x i32> %x0, <8 x i32> %x1, i32 22)
+  %1 = call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %x1, <8 x i32> %x0, <8 x i32> <i32 22, i32 22, i32 22, i32 22, i32 22, i32 22, i32 22, i32 22>)
   %2 = bitcast i8 %x4 to <8 x i1>
   %3 = select <8 x i1> %2, <8 x i32> %1, <8 x i32> %x3
-  %4 = call <8 x i32> @llvm.x86.avx512.vpshrd.d.256(<8 x i32> %x0, <8 x i32> %x1, i32 22)
+  %4 = call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %x1, <8 x i32> %x0, <8 x i32> <i32 22, i32 22, i32 22, i32 22, i32 22, i32 22, i32 22, i32 22>)
   %res2 = add <8 x i32> %3, %4
   ret <8 x i32> %res2
 }
-declare <8 x i32> @llvm.x86.avx512.vpshrd.d.256(<8 x i32>, <8 x i32>, i32)
 
-define <2 x i64>@test_int_x86_avx512_mask_vpshrd_q_128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x3, i8 %x4) {
+define <2 x i64> @test_int_x86_avx512_mask_vpshrd_q_128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x3, i8 %x4) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpshrd_q_128:
 ; X86:       # %bb.0:
 ; X86-NEXT:    vpshrdq $22, %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf3,0xfd,0x08,0x73,0xd9,0x16]
@@ -604,17 +969,16 @@ define <2 x i64>@test_int_x86_avx512_mask_vpshrd_q_128(<2 x i64> %x0, <2 x i64>
 ; X64-NEXT:    vpshrdq $22, %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x09,0x73,0xd1,0x16]
 ; X64-NEXT:    vpaddq %xmm3, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xd4,0xc3]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %1 = call <2 x i64> @llvm.x86.avx512.vpshrd.q.128(<2 x i64> %x0, <2 x i64> %x1, i32 22)
+  %1 = call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %x1, <2 x i64> %x0, <2 x i64> <i64 22, i64 22>)
   %2 = bitcast i8 %x4 to <8 x i1>
   %extract = shufflevector <8 x i1> %2, <8 x i1> %2, <2 x i32> <i32 0, i32 1>
   %3 = select <2 x i1> %extract, <2 x i64> %1, <2 x i64> %x3
-  %4 = call <2 x i64> @llvm.x86.avx512.vpshrd.q.128(<2 x i64> %x0, <2 x i64> %x1, i32 22)
+  %4 = call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %x1, <2 x i64> %x0, <2 x i64> <i64 22, i64 22>)
   %res2 = add <2 x i64> %3, %4
   ret <2 x i64> %res2
 }
-declare <2 x i64> @llvm.x86.avx512.vpshrd.q.128(<2 x i64>, <2 x i64>, i32)
 
-define <4 x i64>@test_int_x86_avx512_mask_vpshrd_q_256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x3, i8 %x4) {
+define <4 x i64> @test_int_x86_avx512_mask_vpshrd_q_256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x3, i8 %x4) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpshrd_q_256:
 ; X86:       # %bb.0:
 ; X86-NEXT:    vpshrdq $22, %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf3,0xfd,0x28,0x73,0xd9,0x16]
@@ -631,82 +995,76 @@ define <4 x i64>@test_int_x86_avx512_mask_vpshrd_q_256(<4 x i64> %x0, <4 x i64>
 ; X64-NEXT:    vpshrdq $22, %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x29,0x73,0xd1,0x16]
 ; X64-NEXT:    vpaddq %ymm3, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xd4,0xc3]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %1 = call <4 x i64> @llvm.x86.avx512.vpshrd.q.256(<4 x i64> %x0, <4 x i64> %x1, i32 22)
+  %1 = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %x1, <4 x i64> %x0, <4 x i64> <i64 22, i64 22, i64 22, i64 22>)
   %2 = bitcast i8 %x4 to <8 x i1>
   %extract = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %3 = select <4 x i1> %extract, <4 x i64> %1, <4 x i64> %x3
-  %4 = call <4 x i64> @llvm.x86.avx512.vpshrd.q.256(<4 x i64> %x0, <4 x i64> %x1, i32 22)
+  %4 = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %x1, <4 x i64> %x0, <4 x i64> <i64 22, i64 22, i64 22, i64 22>)
   %res2 = add <4 x i64> %3, %4
   ret <4 x i64> %res2
 }
-declare <4 x i64> @llvm.x86.avx512.vpshrd.q.256(<4 x i64>, <4 x i64>, i32)
 
-define <8 x i16>@test_int_x86_avx512_mask_vpshrd_w_128(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x3, i8 %x4) {
+define <8 x i16> @test_int_x86_avx512_mask_vpshrd_w_128(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x3, i8 %x4) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpshrd_w_128:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpshrdw $22, %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf3,0xfd,0x08,0x72,0xd9,0x16]
+; X86-NEXT:    vpshrdw $6, %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf3,0xfd,0x08,0x72,0xd9,0x06]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
-; X86-NEXT:    vpshrdw $22, %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x09,0x72,0xd1,0x16]
+; X86-NEXT:    vpshrdw $6, %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x09,0x72,0xd1,0x06]
 ; X86-NEXT:    vpaddw %xmm3, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfd,0xc3]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_vpshrd_w_128:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpshrdw $22, %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf3,0xfd,0x08,0x72,0xd9,0x16]
+; X64-NEXT:    vpshrdw $6, %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf3,0xfd,0x08,0x72,0xd9,0x06]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
-; X64-NEXT:    vpshrdw $22, %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x09,0x72,0xd1,0x16]
+; X64-NEXT:    vpshrdw $6, %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x09,0x72,0xd1,0x06]
 ; X64-NEXT:    vpaddw %xmm3, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfd,0xc3]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %1 = call <8 x i16> @llvm.x86.avx512.vpshrd.w.128(<8 x i16> %x0, <8 x i16> %x1, i32 22)
+  %1 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %x1, <8 x i16> %x0, <8 x i16> <i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6>)
   %2 = bitcast i8 %x4 to <8 x i1>
   %3 = select <8 x i1> %2, <8 x i16> %1, <8 x i16> %x3
-  %4 = call <8 x i16> @llvm.x86.avx512.vpshrd.w.128(<8 x i16> %x0, <8 x i16> %x1, i32 22)
+  %4 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %x1, <8 x i16> %x0, <8 x i16> <i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6>)
   %res2 = add <8 x i16> %3, %4
   ret <8 x i16> %res2
 }
-declare <8 x i16> @llvm.x86.avx512.vpshrd.w.128(<8 x i16>, <8 x i16>, i32)
 
-define <16 x i16>@test_int_x86_avx512_mask_vpshrd_w_256(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x3, i16 %x4) {
+define <16 x i16> @test_int_x86_avx512_mask_vpshrd_w_256(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x3, i16 %x4) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpshrd_w_256:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpshrdw $22, %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf3,0xfd,0x28,0x72,0xd9,0x16]
+; X86-NEXT:    vpshrdw $6, %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf3,0xfd,0x28,0x72,0xd9,0x06]
 ; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
-; X86-NEXT:    vpshrdw $22, %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x29,0x72,0xd1,0x16]
+; X86-NEXT:    vpshrdw $6, %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x29,0x72,0xd1,0x06]
 ; X86-NEXT:    vpaddw %ymm3, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfd,0xc3]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_vpshrd_w_256:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpshrdw $22, %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf3,0xfd,0x28,0x72,0xd9,0x16]
+; X64-NEXT:    vpshrdw $6, %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf3,0xfd,0x28,0x72,0xd9,0x06]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
-; X64-NEXT:    vpshrdw $22, %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x29,0x72,0xd1,0x16]
+; X64-NEXT:    vpshrdw $6, %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf3,0xfd,0x29,0x72,0xd1,0x06]
 ; X64-NEXT:    vpaddw %ymm3, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfd,0xc3]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %1 = call <16 x i16> @llvm.x86.avx512.vpshrd.w.256(<16 x i16> %x0, <16 x i16> %x1, i32 22)
+  %1 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %x1, <16 x i16> %x0, <16 x i16> <i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6>)
   %2 = bitcast i16 %x4 to <16 x i1>
   %3 = select <16 x i1> %2, <16 x i16> %1, <16 x i16> %x3
-  %4 = call <16 x i16> @llvm.x86.avx512.vpshrd.w.256(<16 x i16> %x0, <16 x i16> %x1, i32 22)
+  %4 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %x1, <16 x i16> %x0, <16 x i16> <i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6, i16 6>)
   %res2 = add <16 x i16> %3, %4
   ret <16 x i16> %res2
 }
-declare <16 x i16> @llvm.x86.avx512.vpshrd.w.256(<16 x i16>, <16 x i16>, i32)
-
-declare <8 x i32> @llvm.x86.avx512.mask.vpshrdv.d.256(<8 x i32>, <8 x i32>, <8 x i32>, i8)
-declare <8 x i32> @llvm.x86.avx512.maskz.vpshrdv.d.256(<8 x i32>, <8 x i32>, <8 x i32>, i8)
 
-define <8 x i32>@test_int_x86_avx512_mask_vpshrdv_d_256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32>* %x2p, <8 x i32> %x4, i8 %x3) {
+define <8 x i32> @test_int_x86_avx512_mask_vpshrdv_d_256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32>* %x2p, <8 x i32> %x4, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpshrdv_d_256:
 ; X86:       # %bb.0:
-; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x08]
-; X86-NEXT:    kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx # encoding: [0x0f,0xb6,0x4c,0x24,0x08]
+; X86-NEXT:    kmovd %ecx, %k1 # encoding: [0xc5,0xfb,0x92,0xc9]
 ; X86-NEXT:    vmovdqa %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xd8]
 ; X86-NEXT:    vpshrdvd (%eax), %ymm1, %ymm3 {%k1} # encoding: [0x62,0xf2,0x75,0x29,0x73,0x18]
 ; X86-NEXT:    vmovdqa %ymm0, %ymm4 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xe0]
-; X86-NEXT:    vpshrdvd %ymm2, %ymm1, %ymm4 {%k1} {z} # encoding: [0x62,0xf2,0x75,0xa9,0x73,0xe2]
-; X86-NEXT:    vpshrdvd %ymm2, %ymm1, %ymm0 # encoding: [0x62,0xf2,0x75,0x28,0x73,0xc2]
-; X86-NEXT:    vpaddd %ymm4, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xfe,0xc4]
+; X86-NEXT:    vpshrdvd %ymm2, %ymm1, %ymm4 # encoding: [0x62,0xf2,0x75,0x28,0x73,0xe2]
+; X86-NEXT:    vpshrdvd %ymm2, %ymm1, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0xa9,0x73,0xc2]
+; X86-NEXT:    vpaddd %ymm0, %ymm4, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xdd,0xfe,0xc0]
 ; X86-NEXT:    vpaddd %ymm0, %ymm3, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xe5,0xfe,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
@@ -722,29 +1080,30 @@ define <8 x i32>@test_int_x86_avx512_mask_vpshrdv_d_256(<8 x i32> %x0, <8 x i32>
 ; X64-NEXT:    vpaddd %ymm0, %ymm3, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xe5,0xfe,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %x2 = load <8 x i32>, <8 x i32>* %x2p
-  %res = call <8 x i32> @llvm.x86.avx512.mask.vpshrdv.d.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i8 %x3)
-  %res1 = call <8 x i32> @llvm.x86.avx512.mask.vpshrdv.d.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x4, i8 -1)
-  %res2 = call <8 x i32> @llvm.x86.avx512.maskz.vpshrdv.d.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x4, i8  %x3)
-  %res3 = add <8 x i32> %res, %res1
-  %res4 = add <8 x i32> %res2, %res3
+  %1 = call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %x1, <8 x i32> %x0, <8 x i32> %x2)
+  %2 = bitcast i8 %x3 to <8 x i1>
+  %3 = select <8 x i1> %2, <8 x i32> %1, <8 x i32> %x0
+  %4 = call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %x1, <8 x i32> %x0, <8 x i32> %x4)
+  %5 = call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %x1, <8 x i32> %x0, <8 x i32> %x4)
+  %6 = bitcast i8 %x3 to <8 x i1>
+  %7 = select <8 x i1> %6, <8 x i32> %5, <8 x i32> zeroinitializer
+  %res3 = add <8 x i32> %3, %4
+  %res4 = add <8 x i32> %7, %res3
   ret <8 x i32> %res4
 }
 
-declare <4 x i32> @llvm.x86.avx512.mask.vpshrdv.d.128(<4 x i32>, <4 x i32>, <4 x i32>, i8)
-declare <4 x i32> @llvm.x86.avx512.maskz.vpshrdv.d.128(<4 x i32>, <4 x i32>, <4 x i32>, i8)
-
-define <4 x i32>@test_int_x86_avx512_mask_vpshrdv_d_128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32>* %x2p, <4 x i32> %x4, i8 %x3) {
+define <4 x i32> @test_int_x86_avx512_mask_vpshrdv_d_128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32>* %x2p, <4 x i32> %x4, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpshrdv_d_128:
 ; X86:       # %bb.0:
-; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x08]
-; X86-NEXT:    kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx # encoding: [0x0f,0xb6,0x4c,0x24,0x08]
+; X86-NEXT:    kmovd %ecx, %k1 # encoding: [0xc5,0xfb,0x92,0xc9]
 ; X86-NEXT:    vmovdqa %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xd8]
 ; X86-NEXT:    vpshrdvd (%eax), %xmm1, %xmm3 {%k1} # encoding: [0x62,0xf2,0x75,0x09,0x73,0x18]
 ; X86-NEXT:    vmovdqa %xmm0, %xmm4 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xe0]
-; X86-NEXT:    vpshrdvd %xmm2, %xmm1, %xmm4 {%k1} {z} # encoding: [0x62,0xf2,0x75,0x89,0x73,0xe2]
-; X86-NEXT:    vpshrdvd %xmm2, %xmm1, %xmm0 # encoding: [0x62,0xf2,0x75,0x08,0x73,0xc2]
-; X86-NEXT:    vpaddd %xmm4, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfe,0xc4]
+; X86-NEXT:    vpshrdvd %xmm2, %xmm1, %xmm4 # encoding: [0x62,0xf2,0x75,0x08,0x73,0xe2]
+; X86-NEXT:    vpshrdvd %xmm2, %xmm1, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0x89,0x73,0xc2]
+; X86-NEXT:    vpaddd %xmm0, %xmm4, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xd9,0xfe,0xc0]
 ; X86-NEXT:    vpaddd %xmm0, %xmm3, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe1,0xfe,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
@@ -760,29 +1119,32 @@ define <4 x i32>@test_int_x86_avx512_mask_vpshrdv_d_128(<4 x i32> %x0, <4 x i32>
 ; X64-NEXT:    vpaddd %xmm0, %xmm3, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe1,0xfe,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %x2 = load <4 x i32>, <4 x i32>* %x2p
-  %res = call <4 x i32> @llvm.x86.avx512.mask.vpshrdv.d.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i8 %x3)
-  %res1 = call <4 x i32> @llvm.x86.avx512.mask.vpshrdv.d.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x4, i8 -1)
-  %res2 = call <4 x i32> @llvm.x86.avx512.maskz.vpshrdv.d.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x4, i8  %x3)
-  %res3 = add <4 x i32> %res, %res1
-  %res4 = add <4 x i32> %res2, %res3
+  %1 = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %x1, <4 x i32> %x0, <4 x i32> %x2)
+  %2 = bitcast i8 %x3 to <8 x i1>
+  %extract = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %3 = select <4 x i1> %extract, <4 x i32> %1, <4 x i32> %x0
+  %4 = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %x1, <4 x i32> %x0, <4 x i32> %x4)
+  %5 = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %x1, <4 x i32> %x0, <4 x i32> %x4)
+  %6 = bitcast i8 %x3 to <8 x i1>
+  %extract1 = shufflevector <8 x i1> %6, <8 x i1> %6, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %7 = select <4 x i1> %extract1, <4 x i32> %5, <4 x i32> zeroinitializer
+  %res3 = add <4 x i32> %3, %4
+  %res4 = add <4 x i32> %7, %res3
   ret <4 x i32> %res4
 }
 
-declare <4 x i64> @llvm.x86.avx512.mask.vpshrdv.q.256(<4 x i64>, <4 x i64>, <4 x i64>, i8)
-declare <4 x i64> @llvm.x86.avx512.maskz.vpshrdv.q.256(<4 x i64>, <4 x i64>, <4 x i64>, i8)
-
-define <4 x i64>@test_int_x86_avx512_mask_vpshrdv_q_256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64>* %x2p, <4 x i64> %x4, i8 %x3) {
+define <4 x i64> @test_int_x86_avx512_mask_vpshrdv_q_256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64>* %x2p, <4 x i64> %x4, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpshrdv_q_256:
 ; X86:       # %bb.0:
-; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x08]
-; X86-NEXT:    kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx # encoding: [0x0f,0xb6,0x4c,0x24,0x08]
+; X86-NEXT:    kmovd %ecx, %k1 # encoding: [0xc5,0xfb,0x92,0xc9]
 ; X86-NEXT:    vmovdqa %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xd8]
 ; X86-NEXT:    vpshrdvq (%eax), %ymm1, %ymm3 {%k1} # encoding: [0x62,0xf2,0xf5,0x29,0x73,0x18]
 ; X86-NEXT:    vmovdqa %ymm0, %ymm4 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xe0]
-; X86-NEXT:    vpshrdvq %ymm2, %ymm1, %ymm4 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xa9,0x73,0xe2]
-; X86-NEXT:    vpshrdvq %ymm2, %ymm1, %ymm0 # encoding: [0x62,0xf2,0xf5,0x28,0x73,0xc2]
-; X86-NEXT:    vpaddq %ymm4, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xd4,0xc4]
+; X86-NEXT:    vpshrdvq %ymm2, %ymm1, %ymm4 # encoding: [0x62,0xf2,0xf5,0x28,0x73,0xe2]
+; X86-NEXT:    vpshrdvq %ymm2, %ymm1, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xa9,0x73,0xc2]
+; X86-NEXT:    vpaddq %ymm0, %ymm4, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xdd,0xd4,0xc0]
 ; X86-NEXT:    vpaddq %ymm0, %ymm3, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xe5,0xd4,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
@@ -798,29 +1160,32 @@ define <4 x i64>@test_int_x86_avx512_mask_vpshrdv_q_256(<4 x i64> %x0, <4 x i64>
 ; X64-NEXT:    vpaddq %ymm0, %ymm3, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xe5,0xd4,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %x2 = load <4 x i64>, <4 x i64>* %x2p
-  %res = call <4 x i64> @llvm.x86.avx512.mask.vpshrdv.q.256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2, i8 %x3)
-  %res1 = call <4 x i64> @llvm.x86.avx512.mask.vpshrdv.q.256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x4, i8 -1)
-  %res2 = call <4 x i64> @llvm.x86.avx512.maskz.vpshrdv.q.256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x4, i8  %x3)
-  %res3 = add <4 x i64> %res, %res1
-  %res4 = add <4 x i64> %res2, %res3
+  %1 = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %x1, <4 x i64> %x0, <4 x i64> %x2)
+  %2 = bitcast i8 %x3 to <8 x i1>
+  %extract = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %3 = select <4 x i1> %extract, <4 x i64> %1, <4 x i64> %x0
+  %4 = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %x1, <4 x i64> %x0, <4 x i64> %x4)
+  %5 = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %x1, <4 x i64> %x0, <4 x i64> %x4)
+  %6 = bitcast i8 %x3 to <8 x i1>
+  %extract1 = shufflevector <8 x i1> %6, <8 x i1> %6, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %7 = select <4 x i1> %extract1, <4 x i64> %5, <4 x i64> zeroinitializer
+  %res3 = add <4 x i64> %3, %4
+  %res4 = add <4 x i64> %7, %res3
   ret <4 x i64> %res4
 }
 
-declare <2 x i64> @llvm.x86.avx512.mask.vpshrdv.q.128(<2 x i64>, <2 x i64>, <2 x i64>, i8)
-declare <2 x i64> @llvm.x86.avx512.maskz.vpshrdv.q.128(<2 x i64>, <2 x i64>, <2 x i64>, i8)
-
-define <2 x i64>@test_int_x86_avx512_mask_vpshrdv_q_128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64>* %x2p, <2 x i64> %x4, i8 %x3) {
+define <2 x i64> @test_int_x86_avx512_mask_vpshrdv_q_128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64>* %x2p, <2 x i64> %x4, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpshrdv_q_128:
 ; X86:       # %bb.0:
-; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x08]
-; X86-NEXT:    kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx # encoding: [0x0f,0xb6,0x4c,0x24,0x08]
+; X86-NEXT:    kmovd %ecx, %k1 # encoding: [0xc5,0xfb,0x92,0xc9]
 ; X86-NEXT:    vmovdqa %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xd8]
 ; X86-NEXT:    vpshrdvq (%eax), %xmm1, %xmm3 {%k1} # encoding: [0x62,0xf2,0xf5,0x09,0x73,0x18]
 ; X86-NEXT:    vmovdqa %xmm0, %xmm4 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xe0]
-; X86-NEXT:    vpshrdvq %xmm2, %xmm1, %xmm4 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0x89,0x73,0xe2]
-; X86-NEXT:    vpshrdvq %xmm2, %xmm1, %xmm0 # encoding: [0x62,0xf2,0xf5,0x08,0x73,0xc2]
-; X86-NEXT:    vpaddq %xmm4, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xd4,0xc4]
+; X86-NEXT:    vpshrdvq %xmm2, %xmm1, %xmm4 # encoding: [0x62,0xf2,0xf5,0x08,0x73,0xe2]
+; X86-NEXT:    vpshrdvq %xmm2, %xmm1, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0x89,0x73,0xc2]
+; X86-NEXT:    vpaddq %xmm0, %xmm4, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xd9,0xd4,0xc0]
 ; X86-NEXT:    vpaddq %xmm0, %xmm3, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe1,0xd4,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
@@ -836,18 +1201,21 @@ define <2 x i64>@test_int_x86_avx512_mask_vpshrdv_q_128(<2 x i64> %x0, <2 x i64>
 ; X64-NEXT:    vpaddq %xmm0, %xmm3, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe1,0xd4,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %x2 = load <2 x i64>, <2 x i64>* %x2p
-  %res = call <2 x i64> @llvm.x86.avx512.mask.vpshrdv.q.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i8 %x3)
-  %res1 = call <2 x i64> @llvm.x86.avx512.mask.vpshrdv.q.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x4, i8 -1)
-  %res2 = call <2 x i64> @llvm.x86.avx512.maskz.vpshrdv.q.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x4, i8  %x3)
-  %res3 = add <2 x i64> %res, %res1
-  %res4 = add <2 x i64> %res2, %res3
+  %1 = call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %x1, <2 x i64> %x0, <2 x i64> %x2)
+  %2 = bitcast i8 %x3 to <8 x i1>
+  %extract = shufflevector <8 x i1> %2, <8 x i1> %2, <2 x i32> <i32 0, i32 1>
+  %3 = select <2 x i1> %extract, <2 x i64> %1, <2 x i64> %x0
+  %4 = call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %x1, <2 x i64> %x0, <2 x i64> %x4)
+  %5 = call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %x1, <2 x i64> %x0, <2 x i64> %x4)
+  %6 = bitcast i8 %x3 to <8 x i1>
+  %extract1 = shufflevector <8 x i1> %6, <8 x i1> %6, <2 x i32> <i32 0, i32 1>
+  %7 = select <2 x i1> %extract1, <2 x i64> %5, <2 x i64> zeroinitializer
+  %res3 = add <2 x i64> %3, %4
+  %res4 = add <2 x i64> %7, %res3
   ret <2 x i64> %res4
 }
 
-declare <16 x i16> @llvm.x86.avx512.mask.vpshrdv.w.256(<16 x i16>, <16 x i16>, <16 x i16>, i16)
-declare <16 x i16> @llvm.x86.avx512.maskz.vpshrdv.w.256(<16 x i16>, <16 x i16>, <16 x i16>, i16)
-
-define <16 x i16>@test_int_x86_avx512_mask_vpshrdv_w_256(<16 x i16> %x0, <16 x i16> %x1, <16 x i16>* %x2p, <16 x i16> %x4, i16 %x3) {
+define <16 x i16> @test_int_x86_avx512_mask_vpshrdv_w_256(<16 x i16> %x0, <16 x i16> %x1, <16 x i16>* %x2p, <16 x i16> %x4, i16 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpshrdv_w_256:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
@@ -855,9 +1223,9 @@ define <16 x i16>@test_int_x86_avx512_mask_vpshrdv_w_256(<16 x i16> %x0, <16 x i
 ; X86-NEXT:    vmovdqa %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xd8]
 ; X86-NEXT:    vpshrdvw (%eax), %ymm1, %ymm3 {%k1} # encoding: [0x62,0xf2,0xf5,0x29,0x72,0x18]
 ; X86-NEXT:    vmovdqa %ymm0, %ymm4 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xe0]
-; X86-NEXT:    vpshrdvw %ymm2, %ymm1, %ymm4 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xa9,0x72,0xe2]
-; X86-NEXT:    vpshrdvw %ymm2, %ymm1, %ymm0 # encoding: [0x62,0xf2,0xf5,0x28,0x72,0xc2]
-; X86-NEXT:    vpaddw %ymm4, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xfd,0xc4]
+; X86-NEXT:    vpshrdvw %ymm2, %ymm1, %ymm4 # encoding: [0x62,0xf2,0xf5,0x28,0x72,0xe2]
+; X86-NEXT:    vpshrdvw %ymm2, %ymm1, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xa9,0x72,0xc2]
+; X86-NEXT:    vpaddw %ymm0, %ymm4, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xdd,0xfd,0xc0]
 ; X86-NEXT:    vpaddw %ymm0, %ymm3, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xe5,0xfd,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
@@ -873,29 +1241,30 @@ define <16 x i16>@test_int_x86_avx512_mask_vpshrdv_w_256(<16 x i16> %x0, <16 x i
 ; X64-NEXT:    vpaddw %ymm0, %ymm3, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xe5,0xfd,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %x2 = load <16 x i16>, <16 x i16>* %x2p
-  %res = call <16 x i16> @llvm.x86.avx512.mask.vpshrdv.w.256(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x2, i16 %x3)
-  %res1 = call <16 x i16> @llvm.x86.avx512.mask.vpshrdv.w.256(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x4, i16 -1)
-  %res2 = call <16 x i16> @llvm.x86.avx512.maskz.vpshrdv.w.256(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x4, i16  %x3)
-  %res3 = add <16 x i16> %res, %res1
-  %res4 = add <16 x i16> %res2, %res3
+  %1 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %x1, <16 x i16> %x0, <16 x i16> %x2)
+  %2 = bitcast i16 %x3 to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x i16> %1, <16 x i16> %x0
+  %4 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %x1, <16 x i16> %x0, <16 x i16> %x4)
+  %5 = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %x1, <16 x i16> %x0, <16 x i16> %x4)
+  %6 = bitcast i16 %x3 to <16 x i1>
+  %7 = select <16 x i1> %6, <16 x i16> %5, <16 x i16> zeroinitializer
+  %res3 = add <16 x i16> %3, %4
+  %res4 = add <16 x i16> %7, %res3
   ret <16 x i16> %res4
 }
 
-declare <8 x i16> @llvm.x86.avx512.mask.vpshrdv.w.128(<8 x i16>, <8 x i16>, <8 x i16>, i8)
-declare <8 x i16> @llvm.x86.avx512.maskz.vpshrdv.w.128(<8 x i16>, <8 x i16>, <8 x i16>, i8)
-
-define <8 x i16>@test_int_x86_avx512_mask_vpshrdv_w_128(<8 x i16> %x0, <8 x i16> %x1, <8 x i16>* %x2p, <8 x i16> %x4, i8 %x3) {
+define <8 x i16> @test_int_x86_avx512_mask_vpshrdv_w_128(<8 x i16> %x0, <8 x i16> %x1, <8 x i16>* %x2p, <8 x i16> %x4, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpshrdv_w_128:
 ; X86:       # %bb.0:
-; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x08]
-; X86-NEXT:    kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx # encoding: [0x0f,0xb6,0x4c,0x24,0x08]
+; X86-NEXT:    kmovd %ecx, %k1 # encoding: [0xc5,0xfb,0x92,0xc9]
 ; X86-NEXT:    vmovdqa %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xd8]
 ; X86-NEXT:    vpshrdvw (%eax), %xmm1, %xmm3 {%k1} # encoding: [0x62,0xf2,0xf5,0x09,0x72,0x18]
 ; X86-NEXT:    vmovdqa %xmm0, %xmm4 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xe0]
-; X86-NEXT:    vpshrdvw %xmm2, %xmm1, %xmm4 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0x89,0x72,0xe2]
-; X86-NEXT:    vpshrdvw %xmm2, %xmm1, %xmm0 # encoding: [0x62,0xf2,0xf5,0x08,0x72,0xc2]
-; X86-NEXT:    vpaddw %xmm4, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfd,0xc4]
+; X86-NEXT:    vpshrdvw %xmm2, %xmm1, %xmm4 # encoding: [0x62,0xf2,0xf5,0x08,0x72,0xe2]
+; X86-NEXT:    vpshrdvw %xmm2, %xmm1, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0x89,0x72,0xc2]
+; X86-NEXT:    vpaddw %xmm0, %xmm4, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xd9,0xfd,0xc0]
 ; X86-NEXT:    vpaddw %xmm0, %xmm3, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe1,0xfd,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
@@ -911,29 +1280,30 @@ define <8 x i16>@test_int_x86_avx512_mask_vpshrdv_w_128(<8 x i16> %x0, <8 x i16>
 ; X64-NEXT:    vpaddw %xmm0, %xmm3, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe1,0xfd,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %x2 = load <8 x i16>, <8 x i16>* %x2p
-  %res = call <8 x i16> @llvm.x86.avx512.mask.vpshrdv.w.128(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2, i8 %x3)
-  %res1 = call <8 x i16> @llvm.x86.avx512.mask.vpshrdv.w.128(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x4, i8 -1)
-  %res2 = call <8 x i16> @llvm.x86.avx512.maskz.vpshrdv.w.128(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x4, i8  %x3)
-  %res3 = add <8 x i16> %res, %res1
-  %res4 = add <8 x i16> %res2, %res3
+  %1 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %x1, <8 x i16> %x0, <8 x i16> %x2)
+  %2 = bitcast i8 %x3 to <8 x i1>
+  %3 = select <8 x i1> %2, <8 x i16> %1, <8 x i16> %x0
+  %4 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %x1, <8 x i16> %x0, <8 x i16> %x4)
+  %5 = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %x1, <8 x i16> %x0, <8 x i16> %x4)
+  %6 = bitcast i8 %x3 to <8 x i1>
+  %7 = select <8 x i1> %6, <8 x i16> %5, <8 x i16> zeroinitializer
+  %res3 = add <8 x i16> %3, %4
+  %res4 = add <8 x i16> %7, %res3
   ret <8 x i16> %res4
 }
 
-declare <8 x i32> @llvm.x86.avx512.mask.vpshldv.d.256(<8 x i32>, <8 x i32>, <8 x i32>, i8)
-declare <8 x i32> @llvm.x86.avx512.maskz.vpshldv.d.256(<8 x i32>, <8 x i32>, <8 x i32>, i8)
-
-define <8 x i32>@test_int_x86_avx512_mask_vpshldv_d_256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32>* %x2p, <8 x i32> %x4, i8 %x3) {
+define <8 x i32> @test_int_x86_avx512_mask_vpshldv_d_256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32>* %x2p, <8 x i32> %x4, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpshldv_d_256:
 ; X86:       # %bb.0:
-; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x08]
-; X86-NEXT:    kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx # encoding: [0x0f,0xb6,0x4c,0x24,0x08]
+; X86-NEXT:    kmovd %ecx, %k1 # encoding: [0xc5,0xfb,0x92,0xc9]
 ; X86-NEXT:    vmovdqa %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xd8]
 ; X86-NEXT:    vpshldvd (%eax), %ymm1, %ymm3 {%k1} # encoding: [0x62,0xf2,0x75,0x29,0x71,0x18]
 ; X86-NEXT:    vmovdqa %ymm0, %ymm4 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xe0]
-; X86-NEXT:    vpshldvd %ymm2, %ymm1, %ymm4 {%k1} {z} # encoding: [0x62,0xf2,0x75,0xa9,0x71,0xe2]
-; X86-NEXT:    vpshldvd %ymm2, %ymm1, %ymm0 # encoding: [0x62,0xf2,0x75,0x28,0x71,0xc2]
-; X86-NEXT:    vpaddd %ymm4, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xfe,0xc4]
+; X86-NEXT:    vpshldvd %ymm2, %ymm1, %ymm4 # encoding: [0x62,0xf2,0x75,0x28,0x71,0xe2]
+; X86-NEXT:    vpshldvd %ymm2, %ymm1, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0xa9,0x71,0xc2]
+; X86-NEXT:    vpaddd %ymm0, %ymm4, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xdd,0xfe,0xc0]
 ; X86-NEXT:    vpaddd %ymm0, %ymm3, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xe5,0xfe,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
@@ -949,29 +1319,30 @@ define <8 x i32>@test_int_x86_avx512_mask_vpshldv_d_256(<8 x i32> %x0, <8 x i32>
 ; X64-NEXT:    vpaddd %ymm0, %ymm3, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xe5,0xfe,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %x2 = load <8 x i32>, <8 x i32>* %x2p
-  %res = call <8 x i32> @llvm.x86.avx512.mask.vpshldv.d.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i8 %x3)
-  %res1 = call <8 x i32> @llvm.x86.avx512.mask.vpshldv.d.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x4, i8 -1)
-  %res2 = call <8 x i32> @llvm.x86.avx512.maskz.vpshldv.d.256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x4, i8  %x3)
-  %res3 = add <8 x i32> %res, %res1
-  %res4 = add <8 x i32> %res2, %res3
+  %1 = call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2)
+  %2 = bitcast i8 %x3 to <8 x i1>
+  %3 = select <8 x i1> %2, <8 x i32> %1, <8 x i32> %x0
+  %4 = call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x4)
+  %5 = call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x4)
+  %6 = bitcast i8 %x3 to <8 x i1>
+  %7 = select <8 x i1> %6, <8 x i32> %5, <8 x i32> zeroinitializer
+  %res3 = add <8 x i32> %3, %4
+  %res4 = add <8 x i32> %7, %res3
   ret <8 x i32> %res4
 }
 
-declare <4 x i32> @llvm.x86.avx512.mask.vpshldv.d.128(<4 x i32>, <4 x i32>, <4 x i32>, i8)
-declare <4 x i32> @llvm.x86.avx512.maskz.vpshldv.d.128(<4 x i32>, <4 x i32>, <4 x i32>, i8)
-
-define <4 x i32>@test_int_x86_avx512_mask_vpshldv_d_128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32>* %x2p, <4 x i32> %x4, i8 %x3) {
+define <4 x i32> @test_int_x86_avx512_mask_vpshldv_d_128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32>* %x2p, <4 x i32> %x4, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpshldv_d_128:
 ; X86:       # %bb.0:
-; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x08]
-; X86-NEXT:    kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx # encoding: [0x0f,0xb6,0x4c,0x24,0x08]
+; X86-NEXT:    kmovd %ecx, %k1 # encoding: [0xc5,0xfb,0x92,0xc9]
 ; X86-NEXT:    vmovdqa %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xd8]
 ; X86-NEXT:    vpshldvd (%eax), %xmm1, %xmm3 {%k1} # encoding: [0x62,0xf2,0x75,0x09,0x71,0x18]
 ; X86-NEXT:    vmovdqa %xmm0, %xmm4 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xe0]
-; X86-NEXT:    vpshldvd %xmm2, %xmm1, %xmm4 {%k1} {z} # encoding: [0x62,0xf2,0x75,0x89,0x71,0xe2]
-; X86-NEXT:    vpshldvd %xmm2, %xmm1, %xmm0 # encoding: [0x62,0xf2,0x75,0x08,0x71,0xc2]
-; X86-NEXT:    vpaddd %xmm4, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfe,0xc4]
+; X86-NEXT:    vpshldvd %xmm2, %xmm1, %xmm4 # encoding: [0x62,0xf2,0x75,0x08,0x71,0xe2]
+; X86-NEXT:    vpshldvd %xmm2, %xmm1, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0x89,0x71,0xc2]
+; X86-NEXT:    vpaddd %xmm0, %xmm4, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xd9,0xfe,0xc0]
 ; X86-NEXT:    vpaddd %xmm0, %xmm3, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe1,0xfe,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
@@ -987,29 +1358,32 @@ define <4 x i32>@test_int_x86_avx512_mask_vpshldv_d_128(<4 x i32> %x0, <4 x i32>
 ; X64-NEXT:    vpaddd %xmm0, %xmm3, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe1,0xfe,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %x2 = load <4 x i32>, <4 x i32>* %x2p
-  %res = call <4 x i32> @llvm.x86.avx512.mask.vpshldv.d.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i8 %x3)
-  %res1 = call <4 x i32> @llvm.x86.avx512.mask.vpshldv.d.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x4, i8 -1)
-  %res2 = call <4 x i32> @llvm.x86.avx512.maskz.vpshldv.d.128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x4, i8  %x3)
-  %res3 = add <4 x i32> %res, %res1
-  %res4 = add <4 x i32> %res2, %res3
+  %1 = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2)
+  %2 = bitcast i8 %x3 to <8 x i1>
+  %extract = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %3 = select <4 x i1> %extract, <4 x i32> %1, <4 x i32> %x0
+  %4 = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x4)
+  %5 = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x4)
+  %6 = bitcast i8 %x3 to <8 x i1>
+  %extract1 = shufflevector <8 x i1> %6, <8 x i1> %6, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %7 = select <4 x i1> %extract1, <4 x i32> %5, <4 x i32> zeroinitializer
+  %res3 = add <4 x i32> %3, %4
+  %res4 = add <4 x i32> %7, %res3
   ret <4 x i32> %res4
 }
 
-declare <4 x i64> @llvm.x86.avx512.mask.vpshldv.q.256(<4 x i64>, <4 x i64>, <4 x i64>, i8)
-declare <4 x i64> @llvm.x86.avx512.maskz.vpshldv.q.256(<4 x i64>, <4 x i64>, <4 x i64>, i8)
-
-define <4 x i64>@test_int_x86_avx512_mask_vpshldv_q_256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64>* %x2p, <4 x i64> %x4, i8 %x3) {
+define <4 x i64> @test_int_x86_avx512_mask_vpshldv_q_256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64>* %x2p, <4 x i64> %x4, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpshldv_q_256:
 ; X86:       # %bb.0:
-; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x08]
-; X86-NEXT:    kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx # encoding: [0x0f,0xb6,0x4c,0x24,0x08]
+; X86-NEXT:    kmovd %ecx, %k1 # encoding: [0xc5,0xfb,0x92,0xc9]
 ; X86-NEXT:    vmovdqa %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xd8]
 ; X86-NEXT:    vpshldvq (%eax), %ymm1, %ymm3 {%k1} # encoding: [0x62,0xf2,0xf5,0x29,0x71,0x18]
 ; X86-NEXT:    vmovdqa %ymm0, %ymm4 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xe0]
-; X86-NEXT:    vpshldvq %ymm2, %ymm1, %ymm4 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xa9,0x71,0xe2]
-; X86-NEXT:    vpshldvq %ymm2, %ymm1, %ymm0 # encoding: [0x62,0xf2,0xf5,0x28,0x71,0xc2]
-; X86-NEXT:    vpaddq %ymm4, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xd4,0xc4]
+; X86-NEXT:    vpshldvq %ymm2, %ymm1, %ymm4 # encoding: [0x62,0xf2,0xf5,0x28,0x71,0xe2]
+; X86-NEXT:    vpshldvq %ymm2, %ymm1, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xa9,0x71,0xc2]
+; X86-NEXT:    vpaddq %ymm0, %ymm4, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xdd,0xd4,0xc0]
 ; X86-NEXT:    vpaddq %ymm0, %ymm3, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xe5,0xd4,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
@@ -1025,29 +1399,32 @@ define <4 x i64>@test_int_x86_avx512_mask_vpshldv_q_256(<4 x i64> %x0, <4 x i64>
 ; X64-NEXT:    vpaddq %ymm0, %ymm3, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xe5,0xd4,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %x2 = load <4 x i64>, <4 x i64>* %x2p
-  %res = call <4 x i64> @llvm.x86.avx512.mask.vpshldv.q.256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2, i8 %x3)
-  %res1 = call <4 x i64> @llvm.x86.avx512.mask.vpshldv.q.256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x4, i8 -1)
-  %res2 = call <4 x i64> @llvm.x86.avx512.maskz.vpshldv.q.256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x4, i8  %x3)
-  %res3 = add <4 x i64> %res, %res1
-  %res4 = add <4 x i64> %res2, %res3
+  %1 = call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2)
+  %2 = bitcast i8 %x3 to <8 x i1>
+  %extract = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %3 = select <4 x i1> %extract, <4 x i64> %1, <4 x i64> %x0
+  %4 = call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x4)
+  %5 = call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x4)
+  %6 = bitcast i8 %x3 to <8 x i1>
+  %extract1 = shufflevector <8 x i1> %6, <8 x i1> %6, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %7 = select <4 x i1> %extract1, <4 x i64> %5, <4 x i64> zeroinitializer
+  %res3 = add <4 x i64> %3, %4
+  %res4 = add <4 x i64> %7, %res3
   ret <4 x i64> %res4
 }
 
-declare <2 x i64> @llvm.x86.avx512.mask.vpshldv.q.128(<2 x i64>, <2 x i64>, <2 x i64>, i8)
-declare <2 x i64> @llvm.x86.avx512.maskz.vpshldv.q.128(<2 x i64>, <2 x i64>, <2 x i64>, i8)
-
-define <2 x i64>@test_int_x86_avx512_mask_vpshldv_q_128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64>* %x2p, <2 x i64> %x4, i8 %x3) {
+define <2 x i64> @test_int_x86_avx512_mask_vpshldv_q_128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64>* %x2p, <2 x i64> %x4, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpshldv_q_128:
 ; X86:       # %bb.0:
-; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x08]
-; X86-NEXT:    kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx # encoding: [0x0f,0xb6,0x4c,0x24,0x08]
+; X86-NEXT:    kmovd %ecx, %k1 # encoding: [0xc5,0xfb,0x92,0xc9]
 ; X86-NEXT:    vmovdqa %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xd8]
 ; X86-NEXT:    vpshldvq (%eax), %xmm1, %xmm3 {%k1} # encoding: [0x62,0xf2,0xf5,0x09,0x71,0x18]
 ; X86-NEXT:    vmovdqa %xmm0, %xmm4 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xe0]
-; X86-NEXT:    vpshldvq %xmm2, %xmm1, %xmm4 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0x89,0x71,0xe2]
-; X86-NEXT:    vpshldvq %xmm2, %xmm1, %xmm0 # encoding: [0x62,0xf2,0xf5,0x08,0x71,0xc2]
-; X86-NEXT:    vpaddq %xmm4, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xd4,0xc4]
+; X86-NEXT:    vpshldvq %xmm2, %xmm1, %xmm4 # encoding: [0x62,0xf2,0xf5,0x08,0x71,0xe2]
+; X86-NEXT:    vpshldvq %xmm2, %xmm1, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0x89,0x71,0xc2]
+; X86-NEXT:    vpaddq %xmm0, %xmm4, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xd9,0xd4,0xc0]
 ; X86-NEXT:    vpaddq %xmm0, %xmm3, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe1,0xd4,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
@@ -1063,18 +1440,21 @@ define <2 x i64>@test_int_x86_avx512_mask_vpshldv_q_128(<2 x i64> %x0, <2 x i64>
 ; X64-NEXT:    vpaddq %xmm0, %xmm3, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe1,0xd4,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %x2 = load <2 x i64>, <2 x i64>* %x2p
-  %res = call <2 x i64> @llvm.x86.avx512.mask.vpshldv.q.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i8 %x3)
-  %res1 = call <2 x i64> @llvm.x86.avx512.mask.vpshldv.q.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x4, i8 -1)
-  %res2 = call <2 x i64> @llvm.x86.avx512.maskz.vpshldv.q.128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x4, i8  %x3)
-  %res3 = add <2 x i64> %res, %res1
-  %res4 = add <2 x i64> %res2, %res3
+  %1 = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2)
+  %2 = bitcast i8 %x3 to <8 x i1>
+  %extract = shufflevector <8 x i1> %2, <8 x i1> %2, <2 x i32> <i32 0, i32 1>
+  %3 = select <2 x i1> %extract, <2 x i64> %1, <2 x i64> %x0
+  %4 = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x4)
+  %5 = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x4)
+  %6 = bitcast i8 %x3 to <8 x i1>
+  %extract1 = shufflevector <8 x i1> %6, <8 x i1> %6, <2 x i32> <i32 0, i32 1>
+  %7 = select <2 x i1> %extract1, <2 x i64> %5, <2 x i64> zeroinitializer
+  %res3 = add <2 x i64> %3, %4
+  %res4 = add <2 x i64> %7, %res3
   ret <2 x i64> %res4
 }
 
-declare <16 x i16> @llvm.x86.avx512.mask.vpshldv.w.256(<16 x i16>, <16 x i16>, <16 x i16>, i16)
-declare <16 x i16> @llvm.x86.avx512.maskz.vpshldv.w.256(<16 x i16>, <16 x i16>, <16 x i16>, i16)
-
-define <16 x i16>@test_int_x86_avx512_mask_vpshldv_w_256(<16 x i16> %x0, <16 x i16> %x1, <16 x i16>* %x2p, <16 x i16> %x4, i16 %x3) {
+define <16 x i16> @test_int_x86_avx512_mask_vpshldv_w_256(<16 x i16> %x0, <16 x i16> %x1, <16 x i16>* %x2p, <16 x i16> %x4, i16 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpshldv_w_256:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
@@ -1082,9 +1462,9 @@ define <16 x i16>@test_int_x86_avx512_mask_vpshldv_w_256(<16 x i16> %x0, <16 x i
 ; X86-NEXT:    vmovdqa %ymm0, %ymm3 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xd8]
 ; X86-NEXT:    vpshldvw (%eax), %ymm1, %ymm3 {%k1} # encoding: [0x62,0xf2,0xf5,0x29,0x70,0x18]
 ; X86-NEXT:    vmovdqa %ymm0, %ymm4 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xe0]
-; X86-NEXT:    vpshldvw %ymm2, %ymm1, %ymm4 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xa9,0x70,0xe2]
-; X86-NEXT:    vpshldvw %ymm2, %ymm1, %ymm0 # encoding: [0x62,0xf2,0xf5,0x28,0x70,0xc2]
-; X86-NEXT:    vpaddw %ymm4, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xfd,0xc4]
+; X86-NEXT:    vpshldvw %ymm2, %ymm1, %ymm4 # encoding: [0x62,0xf2,0xf5,0x28,0x70,0xe2]
+; X86-NEXT:    vpshldvw %ymm2, %ymm1, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xa9,0x70,0xc2]
+; X86-NEXT:    vpaddw %ymm0, %ymm4, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xdd,0xfd,0xc0]
 ; X86-NEXT:    vpaddw %ymm0, %ymm3, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xe5,0xfd,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
@@ -1100,29 +1480,30 @@ define <16 x i16>@test_int_x86_avx512_mask_vpshldv_w_256(<16 x i16> %x0, <16 x i
 ; X64-NEXT:    vpaddw %ymm0, %ymm3, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xe5,0xfd,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %x2 = load <16 x i16>, <16 x i16>* %x2p
-  %res = call <16 x i16> @llvm.x86.avx512.mask.vpshldv.w.256(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x2, i16 %x3)
-  %res1 = call <16 x i16> @llvm.x86.avx512.mask.vpshldv.w.256(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x4, i16 -1)
-  %res2 = call <16 x i16> @llvm.x86.avx512.maskz.vpshldv.w.256(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x4, i16  %x3)
-  %res3 = add <16 x i16> %res, %res1
-  %res4 = add <16 x i16> %res2, %res3
+  %1 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x2)
+  %2 = bitcast i16 %x3 to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x i16> %1, <16 x i16> %x0
+  %4 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x4)
+  %5 = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %x0, <16 x i16> %x1, <16 x i16> %x4)
+  %6 = bitcast i16 %x3 to <16 x i1>
+  %7 = select <16 x i1> %6, <16 x i16> %5, <16 x i16> zeroinitializer
+  %res3 = add <16 x i16> %3, %4
+  %res4 = add <16 x i16> %7, %res3
   ret <16 x i16> %res4
 }
 
-declare <8 x i16> @llvm.x86.avx512.mask.vpshldv.w.128(<8 x i16>, <8 x i16>, <8 x i16>, i8)
-declare <8 x i16> @llvm.x86.avx512.maskz.vpshldv.w.128(<8 x i16>, <8 x i16>, <8 x i16>, i8)
-
-define <8 x i16>@test_int_x86_avx512_mask_vpshldv_w_128(<8 x i16> %x0, <8 x i16> %x1, <8 x i16>* %x2p, <8 x i16> %x4, i8 %x3) {
+define <8 x i16> @test_int_x86_avx512_mask_vpshldv_w_128(<8 x i16> %x0, <8 x i16> %x1, <8 x i16>* %x2p, <8 x i16> %x4, i8 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_vpshldv_w_128:
 ; X86:       # %bb.0:
-; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x08]
-; X86-NEXT:    kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx # encoding: [0x0f,0xb6,0x4c,0x24,0x08]
+; X86-NEXT:    kmovd %ecx, %k1 # encoding: [0xc5,0xfb,0x92,0xc9]
 ; X86-NEXT:    vmovdqa %xmm0, %xmm3 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xd8]
 ; X86-NEXT:    vpshldvw (%eax), %xmm1, %xmm3 {%k1} # encoding: [0x62,0xf2,0xf5,0x09,0x70,0x18]
 ; X86-NEXT:    vmovdqa %xmm0, %xmm4 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xe0]
-; X86-NEXT:    vpshldvw %xmm2, %xmm1, %xmm4 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0x89,0x70,0xe2]
-; X86-NEXT:    vpshldvw %xmm2, %xmm1, %xmm0 # encoding: [0x62,0xf2,0xf5,0x08,0x70,0xc2]
-; X86-NEXT:    vpaddw %xmm4, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfd,0xc4]
+; X86-NEXT:    vpshldvw %xmm2, %xmm1, %xmm4 # encoding: [0x62,0xf2,0xf5,0x08,0x70,0xe2]
+; X86-NEXT:    vpshldvw %xmm2, %xmm1, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0x89,0x70,0xc2]
+; X86-NEXT:    vpaddw %xmm0, %xmm4, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xd9,0xfd,0xc0]
 ; X86-NEXT:    vpaddw %xmm0, %xmm3, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe1,0xfd,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
@@ -1138,11 +1519,43 @@ define <8 x i16>@test_int_x86_avx512_mask_vpshldv_w_128(<8 x i16> %x0, <8 x i16>
 ; X64-NEXT:    vpaddw %xmm0, %xmm3, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe1,0xfd,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %x2 = load <8 x i16>, <8 x i16>* %x2p
-  %res = call <8 x i16> @llvm.x86.avx512.mask.vpshldv.w.128(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2, i8 %x3)
-  %res1 = call <8 x i16> @llvm.x86.avx512.mask.vpshldv.w.128(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x4, i8 -1)
-  %res2 = call <8 x i16> @llvm.x86.avx512.maskz.vpshldv.w.128(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x4, i8  %x3)
-  %res3 = add <8 x i16> %res, %res1
-  %res4 = add <8 x i16> %res2, %res3
+  %1 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x2)
+  %2 = bitcast i8 %x3 to <8 x i1>
+  %3 = select <8 x i1> %2, <8 x i16> %1, <8 x i16> %x0
+  %4 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x4)
+  %5 = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %x0, <8 x i16> %x1, <8 x i16> %x4)
+  %6 = bitcast i8 %x3 to <8 x i1>
+  %7 = select <8 x i1> %6, <8 x i16> %5, <8 x i16> zeroinitializer
+  %res3 = add <8 x i16> %3, %4
+  %res4 = add <8 x i16> %7, %res3
   ret <8 x i16> %res4
 }
 
+declare <4 x i32> @llvm.fshl.v4i32(<4 x i32>, <4 x i32>, <4 x i32>)
+declare <8 x i32> @llvm.fshl.v8i32(<8 x i32>, <8 x i32>, <8 x i32>)
+declare <2 x i64> @llvm.fshl.v2i64(<2 x i64>, <2 x i64>, <2 x i64>)
+declare <4 x i64> @llvm.fshl.v4i64(<4 x i64>, <4 x i64>, <4 x i64>)
+declare <8 x i16> @llvm.fshl.v8i16(<8 x i16>, <8 x i16>, <8 x i16>)
+declare <16 x i16> @llvm.fshl.v16i16(<16 x i16>, <16 x i16>, <16 x i16>)
+declare <4 x i32> @llvm.fshr.v4i32(<4 x i32>, <4 x i32>, <4 x i32>)
+declare <8 x i32> @llvm.fshr.v8i32(<8 x i32>, <8 x i32>, <8 x i32>)
+declare <2 x i64> @llvm.fshr.v2i64(<2 x i64>, <2 x i64>, <2 x i64>)
+declare <4 x i64> @llvm.fshr.v4i64(<4 x i64>, <4 x i64>, <4 x i64>)
+declare <8 x i16> @llvm.fshr.v8i16(<8 x i16>, <8 x i16>, <8 x i16>)
+declare <16 x i16> @llvm.fshr.v16i16(<16 x i16>, <16 x i16>, <16 x i16>)
+declare <8 x i16> @llvm.masked.expandload.v8i16(i16*, <8 x i1>, <8 x i16>)
+declare <16 x i8> @llvm.masked.expandload.v16i8(i8*, <16 x i1>, <16 x i8>)
+declare void @llvm.masked.compressstore.v8i16(<8 x i16>, i16*, <8 x i1>)
+declare void @llvm.masked.compressstore.v16i8(<16 x i8>, i8*, <16 x i1>)
+declare <16 x i16> @llvm.masked.expandload.v16i16(i16*, <16 x i1>, <16 x i16>)
+declare <32 x i8> @llvm.masked.expandload.v32i8(i8*, <32 x i1>, <32 x i8>)
+declare void @llvm.masked.compressstore.v16i16(<16 x i16>, i16*, <16 x i1>)
+declare void @llvm.masked.compressstore.v32i8(<32 x i8>, i8*, <32 x i1>)
+declare <8 x i16> @llvm.x86.avx512.mask.expand.v8i16(<8 x i16>, <8 x i16>, <8 x i1>)
+declare <16 x i8> @llvm.x86.avx512.mask.expand.v16i8(<16 x i8>, <16 x i8>, <16 x i1>)
+declare <8 x i16> @llvm.x86.avx512.mask.compress.v8i16(<8 x i16>, <8 x i16>, <8 x i1>)
+declare <16 x i8> @llvm.x86.avx512.mask.compress.v16i8(<16 x i8>, <16 x i8>, <16 x i1>)
+declare <16 x i16> @llvm.x86.avx512.mask.expand.v16i16(<16 x i16>, <16 x i16>, <16 x i1>)
+declare <32 x i8> @llvm.x86.avx512.mask.expand.v32i8(<32 x i8>, <32 x i8>, <32 x i1>)
+declare <16 x i16> @llvm.x86.avx512.mask.compress.v16i16(<16 x i16>, <16 x i16>, <16 x i1>)
+declare <32 x i8> @llvm.x86.avx512.mask.compress.v32i8(<32 x i8>, <32 x i8>, <32 x i1>)
diff --git a/test/CodeGen/X86/avx512vbmivl-intrinsics-fast-isel.ll b/test/CodeGen/X86/avx512vbmivl-intrinsics-fast-isel.ll
index 70848be4a6d7bc94bbcd50dced17611d6fe72ef1..896a30af5f45b8b58bb8d479cd63444ab58544a8 100644
--- a/test/CodeGen/X86/avx512vbmivl-intrinsics-fast-isel.ll
+++ b/test/CodeGen/X86/avx512vbmivl-intrinsics-fast-isel.ll
@@ -174,5 +174,123 @@ entry:
   ret <4 x i64> %6
 }
 
+define <2 x i64> @test_mm_mask_multishift_epi64_epi8(<2 x i64> %__W, i16 zeroext %__M, <2 x i64> %__X, <2 x i64> %__Y) {
+; X86-LABEL: test_mm_mask_multishift_epi64_epi8:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vpmultishiftqb %xmm2, %xmm1, %xmm0 {%k1}
+; X86-NEXT:    retl
+;
+; X64-LABEL: test_mm_mask_multishift_epi64_epi8:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    kmovd %edi, %k1
+; X64-NEXT:    vpmultishiftqb %xmm2, %xmm1, %xmm0 {%k1}
+; X64-NEXT:    retq
+entry:
+  %0 = bitcast <2 x i64> %__X to <16 x i8>
+  %1 = bitcast <2 x i64> %__Y to <16 x i8>
+  %2 = tail call <16 x i8> @llvm.x86.avx512.pmultishift.qb.128(<16 x i8> %0, <16 x i8> %1)
+  %3 = bitcast <2 x i64> %__W to <16 x i8>
+  %4 = bitcast i16 %__M to <16 x i1>
+  %5 = select <16 x i1> %4, <16 x i8> %2, <16 x i8> %3
+  %6 = bitcast <16 x i8> %5 to <2 x i64>
+  ret <2 x i64> %6
+}
+
+define <2 x i64> @test_mm_maskz_multishift_epi64_epi8(i16 zeroext %__M, <2 x i64> %__X, <2 x i64> %__Y) {
+; X86-LABEL: test_mm_maskz_multishift_epi64_epi8:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vpmultishiftqb %xmm1, %xmm0, %xmm0 {%k1} {z}
+; X86-NEXT:    retl
+;
+; X64-LABEL: test_mm_maskz_multishift_epi64_epi8:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    kmovd %edi, %k1
+; X64-NEXT:    vpmultishiftqb %xmm1, %xmm0, %xmm0 {%k1} {z}
+; X64-NEXT:    retq
+entry:
+  %0 = bitcast <2 x i64> %__X to <16 x i8>
+  %1 = bitcast <2 x i64> %__Y to <16 x i8>
+  %2 = tail call <16 x i8> @llvm.x86.avx512.pmultishift.qb.128(<16 x i8> %0, <16 x i8> %1)
+  %3 = bitcast i16 %__M to <16 x i1>
+  %4 = select <16 x i1> %3, <16 x i8> %2, <16 x i8> zeroinitializer
+  %5 = bitcast <16 x i8> %4 to <2 x i64>
+  ret <2 x i64> %5
+}
+
+define <2 x i64> @test_mm_multishift_epi64_epi8(<2 x i64> %__X, <2 x i64> %__Y) {
+; CHECK-LABEL: test_mm_multishift_epi64_epi8:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vpmultishiftqb %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    ret{{[l|q]}}
+entry:
+  %0 = bitcast <2 x i64> %__X to <16 x i8>
+  %1 = bitcast <2 x i64> %__Y to <16 x i8>
+  %2 = tail call <16 x i8> @llvm.x86.avx512.pmultishift.qb.128(<16 x i8> %0, <16 x i8> %1)
+  %3 = bitcast <16 x i8> %2 to <2 x i64>
+  ret <2 x i64> %3
+}
+
+define <4 x i64> @test_mm256_mask_multishift_epi64_epi8(<4 x i64> %__W, i32 %__M, <4 x i64> %__X, <4 x i64> %__Y) {
+; X86-LABEL: test_mm256_mask_multishift_epi64_epi8:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vpmultishiftqb %ymm2, %ymm1, %ymm0 {%k1}
+; X86-NEXT:    retl
+;
+; X64-LABEL: test_mm256_mask_multishift_epi64_epi8:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    kmovd %edi, %k1
+; X64-NEXT:    vpmultishiftqb %ymm2, %ymm1, %ymm0 {%k1}
+; X64-NEXT:    retq
+entry:
+  %0 = bitcast <4 x i64> %__X to <32 x i8>
+  %1 = bitcast <4 x i64> %__Y to <32 x i8>
+  %2 = tail call <32 x i8> @llvm.x86.avx512.pmultishift.qb.256(<32 x i8> %0, <32 x i8> %1)
+  %3 = bitcast <4 x i64> %__W to <32 x i8>
+  %4 = bitcast i32 %__M to <32 x i1>
+  %5 = select <32 x i1> %4, <32 x i8> %2, <32 x i8> %3
+  %6 = bitcast <32 x i8> %5 to <4 x i64>
+  ret <4 x i64> %6
+}
+
+define <4 x i64> @test_mm256_maskz_multishift_epi64_epi8(i32 %__M, <4 x i64> %__X, <4 x i64> %__Y) {
+; X86-LABEL: test_mm256_maskz_multishift_epi64_epi8:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vpmultishiftqb %ymm1, %ymm0, %ymm0 {%k1} {z}
+; X86-NEXT:    retl
+;
+; X64-LABEL: test_mm256_maskz_multishift_epi64_epi8:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    kmovd %edi, %k1
+; X64-NEXT:    vpmultishiftqb %ymm1, %ymm0, %ymm0 {%k1} {z}
+; X64-NEXT:    retq
+entry:
+  %0 = bitcast <4 x i64> %__X to <32 x i8>
+  %1 = bitcast <4 x i64> %__Y to <32 x i8>
+  %2 = tail call <32 x i8> @llvm.x86.avx512.pmultishift.qb.256(<32 x i8> %0, <32 x i8> %1) #3
+  %3 = bitcast i32 %__M to <32 x i1>
+  %4 = select <32 x i1> %3, <32 x i8> %2, <32 x i8> zeroinitializer
+  %5 = bitcast <32 x i8> %4 to <4 x i64>
+  ret <4 x i64> %5
+}
+
+define <4 x i64> @test_mm256_multishift_epi64_epi8(<4 x i64> %__X, <4 x i64> %__Y) {
+; CHECK-LABEL: test_mm256_multishift_epi64_epi8:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    vpmultishiftqb %ymm1, %ymm0, %ymm0
+; CHECK-NEXT:    ret{{[l|q]}}
+entry:
+  %0 = bitcast <4 x i64> %__X to <32 x i8>
+  %1 = bitcast <4 x i64> %__Y to <32 x i8>
+  %2 = tail call <32 x i8> @llvm.x86.avx512.pmultishift.qb.256(<32 x i8> %0, <32 x i8> %1)
+  %3 = bitcast <32 x i8> %2 to <4 x i64>
+  ret <4 x i64> %3
+}
+
 declare <16 x i8> @llvm.x86.avx512.vpermi2var.qi.128(<16 x i8>, <16 x i8>, <16 x i8>)
 declare <32 x i8> @llvm.x86.avx512.vpermi2var.qi.256(<32 x i8>, <32 x i8>, <32 x i8>)
+declare <16 x i8> @llvm.x86.avx512.pmultishift.qb.128(<16 x i8>, <16 x i8>)
+declare <32 x i8> @llvm.x86.avx512.pmultishift.qb.256(<32 x i8>, <32 x i8>)
diff --git a/test/CodeGen/X86/avx512vbmivl-intrinsics-upgrade.ll b/test/CodeGen/X86/avx512vbmivl-intrinsics-upgrade.ll
index 7572e40f0b2ad0165185b78bf63e711b8db54c1a..4b1d51da46fbe62cd6a0ba9cda7c95cb07d66cf2 100644
--- a/test/CodeGen/X86/avx512vbmivl-intrinsics-upgrade.ll
+++ b/test/CodeGen/X86/avx512vbmivl-intrinsics-upgrade.ll
@@ -62,6 +62,66 @@ define <32 x i8>@test_int_x86_avx512_mask_permvar_qi_256(<32 x i8> %x0, <32 x i8
   ret <32 x i8> %res4
 }
 
+declare <16 x i8> @llvm.x86.avx512.mask.pmultishift.qb.128(<16 x i8>, <16 x i8>, <16 x i8>, i16)
+
+define <16 x i8>@test_int_x86_avx512_mask_pmultishift_qb_128(<16 x i8> %x0, <16 x i8> %x1, <16 x i8> %x2, i16 %x3) {
+; X86-LABEL: test_int_x86_avx512_mask_pmultishift_qb_128:
+; X86:       # %bb.0:
+; X86-NEXT:    vpmultishiftqb %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf2,0xfd,0x08,0x83,0xd9]
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
+; X86-NEXT:    vpmultishiftqb %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x09,0x83,0xd1]
+; X86-NEXT:    vpmultishiftqb %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0x89,0x83,0xc1]
+; X86-NEXT:    vpaddb %xmm3, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfc,0xc3]
+; X86-NEXT:    vpaddb %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfc,0xc0]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_mask_pmultishift_qb_128:
+; X64:       # %bb.0:
+; X64-NEXT:    vpmultishiftqb %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf2,0xfd,0x08,0x83,0xd9]
+; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; X64-NEXT:    vpmultishiftqb %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x09,0x83,0xd1]
+; X64-NEXT:    vpmultishiftqb %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0x89,0x83,0xc1]
+; X64-NEXT:    vpaddb %xmm3, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfc,0xc3]
+; X64-NEXT:    vpaddb %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfc,0xc0]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <16 x i8> @llvm.x86.avx512.mask.pmultishift.qb.128(<16 x i8> %x0, <16 x i8> %x1, <16 x i8> %x2, i16 %x3)
+  %res1 = call <16 x i8> @llvm.x86.avx512.mask.pmultishift.qb.128(<16 x i8> %x0, <16 x i8> %x1, <16 x i8> zeroinitializer, i16 %x3)
+  %res2 = call <16 x i8> @llvm.x86.avx512.mask.pmultishift.qb.128(<16 x i8> %x0, <16 x i8> %x1, <16 x i8> %x2, i16 -1)
+  %res3 = add <16 x i8> %res, %res1
+  %res4 = add <16 x i8> %res3, %res2
+  ret <16 x i8> %res4
+}
+
+declare <32 x i8> @llvm.x86.avx512.mask.pmultishift.qb.256(<32 x i8>, <32 x i8>, <32 x i8>, i32)
+
+define <32 x i8>@test_int_x86_avx512_mask_pmultishift_qb_256(<32 x i8> %x0, <32 x i8> %x1, <32 x i8> %x2, i32 %x3) {
+; X86-LABEL: test_int_x86_avx512_mask_pmultishift_qb_256:
+; X86:       # %bb.0:
+; X86-NEXT:    vpmultishiftqb %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf2,0xfd,0x28,0x83,0xd9]
+; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf9,0x90,0x4c,0x24,0x04]
+; X86-NEXT:    vpmultishiftqb %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x29,0x83,0xd1]
+; X86-NEXT:    vpmultishiftqb %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xa9,0x83,0xc1]
+; X86-NEXT:    vpaddb %ymm3, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xfc,0xc3]
+; X86-NEXT:    vpaddb %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfc,0xc0]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_mask_pmultishift_qb_256:
+; X64:       # %bb.0:
+; X64-NEXT:    vpmultishiftqb %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf2,0xfd,0x28,0x83,0xd9]
+; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
+; X64-NEXT:    vpmultishiftqb %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x29,0x83,0xd1]
+; X64-NEXT:    vpmultishiftqb %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xa9,0x83,0xc1]
+; X64-NEXT:    vpaddb %ymm3, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xfc,0xc3]
+; X64-NEXT:    vpaddb %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfc,0xc0]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <32 x i8> @llvm.x86.avx512.mask.pmultishift.qb.256(<32 x i8> %x0, <32 x i8> %x1, <32 x i8> %x2, i32 %x3)
+  %res1 = call <32 x i8> @llvm.x86.avx512.mask.pmultishift.qb.256(<32 x i8> %x0, <32 x i8> %x1, <32 x i8> zeroinitializer, i32 %x3)
+  %res2 = call <32 x i8> @llvm.x86.avx512.mask.pmultishift.qb.256(<32 x i8> %x0, <32 x i8> %x1, <32 x i8> %x2, i32 -1)
+  %res3 = add <32 x i8> %res, %res1
+  %res4 = add <32 x i8> %res3, %res2
+  ret <32 x i8> %res4
+}
+
 declare <16 x i8> @llvm.x86.avx512.mask.vpermi2var.qi.128(<16 x i8>, <16 x i8>, <16 x i8>, i16)
 
 define <16 x i8>@test_int_x86_avx512_mask_vpermi2var_qi_128(<16 x i8> %x0, <16 x i8> %x1, <16 x i8> %x2, i16 %x3) {
diff --git a/test/CodeGen/X86/avx512vbmivl-intrinsics.ll b/test/CodeGen/X86/avx512vbmivl-intrinsics.ll
index 79f32103dddeca8617576287025a11633fe64a67..7c03d78c825f32de221c6f90a773a953db94de63 100644
--- a/test/CodeGen/X86/avx512vbmivl-intrinsics.ll
+++ b/test/CodeGen/X86/avx512vbmivl-intrinsics.ll
@@ -70,63 +70,71 @@ define <32 x i8>@test_int_x86_avx512_mask_permvar_qi_256(<32 x i8> %x0, <32 x i8
   ret <32 x i8> %res4
 }
 
-declare <16 x i8> @llvm.x86.avx512.mask.pmultishift.qb.128(<16 x i8>, <16 x i8>, <16 x i8>, i16)
+declare <16 x i8> @llvm.x86.avx512.pmultishift.qb.128(<16 x i8>, <16 x i8>)
 
 define <16 x i8>@test_int_x86_avx512_mask_pmultishift_qb_128(<16 x i8> %x0, <16 x i8> %x1, <16 x i8> %x2, i16 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_pmultishift_qb_128:
 ; X86:       # %bb.0:
+; X86-NEXT:    vpmultishiftqb %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf2,0xfd,0x08,0x83,0xd9]
 ; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
 ; X86-NEXT:    vpmultishiftqb %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x09,0x83,0xd1]
-; X86-NEXT:    vpmultishiftqb %xmm1, %xmm0, %xmm3 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0x89,0x83,0xd9]
-; X86-NEXT:    vpmultishiftqb %xmm1, %xmm0, %xmm0 # encoding: [0x62,0xf2,0xfd,0x08,0x83,0xc1]
-; X86-NEXT:    vpaddb %xmm0, %xmm3, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe1,0xfc,0xc0]
+; X86-NEXT:    vpmultishiftqb %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0x89,0x83,0xc1]
+; X86-NEXT:    vpaddb %xmm3, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfc,0xc3]
 ; X86-NEXT:    vpaddb %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfc,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_pmultishift_qb_128:
 ; X64:       # %bb.0:
-; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpmultishiftqb %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf2,0xfd,0x08,0x83,0xd9]
+; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpmultishiftqb %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x09,0x83,0xd1]
 ; X64-NEXT:    vpmultishiftqb %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0x89,0x83,0xc1]
 ; X64-NEXT:    vpaddb %xmm3, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfc,0xc3]
 ; X64-NEXT:    vpaddb %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfc,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <16 x i8> @llvm.x86.avx512.mask.pmultishift.qb.128(<16 x i8> %x0, <16 x i8> %x1, <16 x i8> %x2, i16 %x3)
-  %res1 = call <16 x i8> @llvm.x86.avx512.mask.pmultishift.qb.128(<16 x i8> %x0, <16 x i8> %x1, <16 x i8> zeroinitializer, i16 %x3)
-  %res2 = call <16 x i8> @llvm.x86.avx512.mask.pmultishift.qb.128(<16 x i8> %x0, <16 x i8> %x1, <16 x i8> %x2, i16 -1)
-  %res3 = add <16 x i8> %res, %res1
-  %res4 = add <16 x i8> %res3, %res2
+  %1 = call <16 x i8> @llvm.x86.avx512.pmultishift.qb.128(<16 x i8> %x0, <16 x i8> %x1)
+  %2 = bitcast i16 %x3 to <16 x i1>
+  %3 = select <16 x i1> %2, <16 x i8> %1, <16 x i8> %x2
+  %4 = call <16 x i8> @llvm.x86.avx512.pmultishift.qb.128(<16 x i8> %x0, <16 x i8> %x1)
+  %5 = bitcast i16 %x3 to <16 x i1>
+  %6 = select <16 x i1> %5, <16 x i8> %4, <16 x i8> zeroinitializer
+  %7 = call <16 x i8> @llvm.x86.avx512.pmultishift.qb.128(<16 x i8> %x0, <16 x i8> %x1)
+  %res3 = add <16 x i8> %3, %6
+  %res4 = add <16 x i8> %res3, %7
   ret <16 x i8> %res4
 }
 
-declare <32 x i8> @llvm.x86.avx512.mask.pmultishift.qb.256(<32 x i8>, <32 x i8>, <32 x i8>, i32)
+declare <32 x i8> @llvm.x86.avx512.pmultishift.qb.256(<32 x i8>, <32 x i8>)
 
 define <32 x i8>@test_int_x86_avx512_mask_pmultishift_qb_256(<32 x i8> %x0, <32 x i8> %x1, <32 x i8> %x2, i32 %x3) {
 ; X86-LABEL: test_int_x86_avx512_mask_pmultishift_qb_256:
 ; X86:       # %bb.0:
+; X86-NEXT:    vpmultishiftqb %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf2,0xfd,0x28,0x83,0xd9]
 ; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf9,0x90,0x4c,0x24,0x04]
 ; X86-NEXT:    vpmultishiftqb %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x29,0x83,0xd1]
-; X86-NEXT:    vpmultishiftqb %ymm1, %ymm0, %ymm3 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xa9,0x83,0xd9]
-; X86-NEXT:    vpmultishiftqb %ymm1, %ymm0, %ymm0 # encoding: [0x62,0xf2,0xfd,0x28,0x83,0xc1]
-; X86-NEXT:    vpaddb %ymm0, %ymm3, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xe5,0xfc,0xc0]
+; X86-NEXT:    vpmultishiftqb %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xa9,0x83,0xc1]
+; X86-NEXT:    vpaddb %ymm3, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xfc,0xc3]
 ; X86-NEXT:    vpaddb %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfc,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_pmultishift_qb_256:
 ; X64:       # %bb.0:
+; X64-NEXT:    vpmultishiftqb %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf2,0xfd,0x28,0x83,0xd9]
 ; X64-NEXT:    kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
 ; X64-NEXT:    vpmultishiftqb %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x29,0x83,0xd1]
-; X64-NEXT:    vpmultishiftqb %ymm1, %ymm0, %ymm3 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xa9,0x83,0xd9]
-; X64-NEXT:    vpmultishiftqb %ymm1, %ymm0, %ymm0 # encoding: [0x62,0xf2,0xfd,0x28,0x83,0xc1]
-; X64-NEXT:    vpaddb %ymm0, %ymm3, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xe5,0xfc,0xc0]
+; X64-NEXT:    vpmultishiftqb %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xa9,0x83,0xc1]
+; X64-NEXT:    vpaddb %ymm3, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xfc,0xc3]
 ; X64-NEXT:    vpaddb %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfc,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <32 x i8> @llvm.x86.avx512.mask.pmultishift.qb.256(<32 x i8> %x0, <32 x i8> %x1, <32 x i8> %x2, i32 %x3)
-  %res1 = call <32 x i8> @llvm.x86.avx512.mask.pmultishift.qb.256(<32 x i8> %x0, <32 x i8> %x1, <32 x i8> zeroinitializer, i32 %x3)
-  %res2 = call <32 x i8> @llvm.x86.avx512.mask.pmultishift.qb.256(<32 x i8> %x0, <32 x i8> %x1, <32 x i8> %x2, i32 -1)
-  %res3 = add <32 x i8> %res, %res1
-  %res4 = add <32 x i8> %res3, %res2
+  %1 = call <32 x i8> @llvm.x86.avx512.pmultishift.qb.256(<32 x i8> %x0, <32 x i8> %x1)
+  %2 = bitcast i32 %x3 to <32 x i1>
+  %3 = select <32 x i1> %2, <32 x i8> %1, <32 x i8> %x2
+  %4 = call <32 x i8> @llvm.x86.avx512.pmultishift.qb.256(<32 x i8> %x0, <32 x i8> %x1)
+  %5 = bitcast i32 %x3 to <32 x i1>
+  %6 = select <32 x i1> %5, <32 x i8> %4, <32 x i8> zeroinitializer
+  %7 = call <32 x i8> @llvm.x86.avx512.pmultishift.qb.256(<32 x i8> %x0, <32 x i8> %x1)
+  %res3 = add <32 x i8> %3, %6
+  %res4 = add <32 x i8> %res3, %7
   ret <32 x i8> %res4
 }
 
diff --git a/test/CodeGen/X86/avx512vl-intrinsics-fast-isel.ll b/test/CodeGen/X86/avx512vl-intrinsics-fast-isel.ll
index 6c1cfa8fc0937d36f064fc2a9a9cd63d0acdcdc7..4f1affb94e61f4c42fd4d7304d366dff505d6a4a 100644
--- a/test/CodeGen/X86/avx512vl-intrinsics-fast-isel.ll
+++ b/test/CodeGen/X86/avx512vl-intrinsics-fast-isel.ll
@@ -2104,7 +2104,7 @@ define <4 x i64> @test_mm256_maskz_broadcastd_epi32(i8 %a0, <2 x i64> %a1) {
 define <2 x i64> @test_mm_broadcastq_epi64(<2 x i64> %a0) {
 ; CHECK-LABEL: test_mm_broadcastq_epi64:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vpbroadcastq %xmm0, %xmm0
+; CHECK-NEXT:    vmovddup {{.*#+}} xmm0 = xmm0[0,0]
 ; CHECK-NEXT:    ret{{[l|q]}}
   %res = shufflevector <2 x i64> %a0, <2 x i64> undef, <2 x i32> zeroinitializer
   ret <2 x i64> %res
@@ -6626,13 +6626,11 @@ define <2 x i64> @test_mm_rol_epi32(<2 x i64> %__A) {
 ; CHECK-NEXT:    ret{{[l|q]}}
 entry:
   %0 = bitcast <2 x i64> %__A to <4 x i32>
-  %1 = tail call <4 x i32> @llvm.x86.avx512.prol.d.128(<4 x i32> %0, i32 5)
+  %1 = tail call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %0, <4 x i32> %0, <4 x i32> <i32 5, i32 5, i32 5, i32 5>)
   %2 = bitcast <4 x i32> %1 to <2 x i64>
   ret <2 x i64> %2
 }
 
-declare <4 x i32> @llvm.x86.avx512.prol.d.128(<4 x i32>, i32)
-
 define <2 x i64> @test_mm_mask_rol_epi32(<2 x i64> %__W, i8 zeroext %__U, <2 x i64> %__A) {
 ; X86-LABEL: test_mm_mask_rol_epi32:
 ; X86:       # %bb.0: # %entry
@@ -6648,7 +6646,7 @@ define <2 x i64> @test_mm_mask_rol_epi32(<2 x i64> %__W, i8 zeroext %__U, <2 x i
 ; X64-NEXT:    retq
 entry:
   %0 = bitcast <2 x i64> %__A to <4 x i32>
-  %1 = tail call <4 x i32> @llvm.x86.avx512.prol.d.128(<4 x i32> %0, i32 5)
+  %1 = tail call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %0, <4 x i32> %0, <4 x i32> <i32 5, i32 5, i32 5, i32 5>)
   %2 = bitcast <2 x i64> %__W to <4 x i32>
   %3 = bitcast i8 %__U to <8 x i1>
   %extract = shufflevector <8 x i1> %3, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
@@ -6672,7 +6670,7 @@ define <2 x i64> @test_mm_maskz_rol_epi32(i8 zeroext %__U, <2 x i64> %__A) {
 ; X64-NEXT:    retq
 entry:
   %0 = bitcast <2 x i64> %__A to <4 x i32>
-  %1 = tail call <4 x i32> @llvm.x86.avx512.prol.d.128(<4 x i32> %0, i32 5)
+  %1 = tail call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %0, <4 x i32> %0, <4 x i32> <i32 5, i32 5, i32 5, i32 5>)
   %2 = bitcast i8 %__U to <8 x i1>
   %extract = shufflevector <8 x i1> %2, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %3 = select <4 x i1> %extract, <4 x i32> %1, <4 x i32> zeroinitializer
@@ -6687,13 +6685,11 @@ define <4 x i64> @test_mm256_rol_epi32(<4 x i64> %__A) {
 ; CHECK-NEXT:    ret{{[l|q]}}
 entry:
   %0 = bitcast <4 x i64> %__A to <8 x i32>
-  %1 = tail call <8 x i32> @llvm.x86.avx512.prol.d.256(<8 x i32> %0, i32 5)
+  %1 = tail call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %0, <8 x i32> %0, <8 x i32> <i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5>)
   %2 = bitcast <8 x i32> %1 to <4 x i64>
   ret <4 x i64> %2
 }
 
-declare <8 x i32> @llvm.x86.avx512.prol.d.256(<8 x i32>, i32)
-
 define <4 x i64> @test_mm256_mask_rol_epi32(<4 x i64> %__W, i8 zeroext %__U, <4 x i64> %__A) {
 ; X86-LABEL: test_mm256_mask_rol_epi32:
 ; X86:       # %bb.0: # %entry
@@ -6709,7 +6705,7 @@ define <4 x i64> @test_mm256_mask_rol_epi32(<4 x i64> %__W, i8 zeroext %__U, <4
 ; X64-NEXT:    retq
 entry:
   %0 = bitcast <4 x i64> %__A to <8 x i32>
-  %1 = tail call <8 x i32> @llvm.x86.avx512.prol.d.256(<8 x i32> %0, i32 5)
+  %1 = tail call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %0, <8 x i32> %0, <8 x i32> <i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5>)
   %2 = bitcast <4 x i64> %__W to <8 x i32>
   %3 = bitcast i8 %__U to <8 x i1>
   %4 = select <8 x i1> %3, <8 x i32> %1, <8 x i32> %2
@@ -6732,7 +6728,7 @@ define <4 x i64> @test_mm256_maskz_rol_epi32(i8 zeroext %__U, <4 x i64> %__A) {
 ; X64-NEXT:    retq
 entry:
   %0 = bitcast <4 x i64> %__A to <8 x i32>
-  %1 = tail call <8 x i32> @llvm.x86.avx512.prol.d.256(<8 x i32> %0, i32 5)
+  %1 = tail call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %0, <8 x i32> %0, <8 x i32> <i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5>)
   %2 = bitcast i8 %__U to <8 x i1>
   %3 = select <8 x i1> %2, <8 x i32> %1, <8 x i32> zeroinitializer
   %4 = bitcast <8 x i32> %3 to <4 x i64>
@@ -6745,12 +6741,10 @@ define <2 x i64> @test_mm_rol_epi64(<2 x i64> %__A) {
 ; CHECK-NEXT:    vprolq $5, %xmm0, %xmm0
 ; CHECK-NEXT:    ret{{[l|q]}}
 entry:
-  %0 = tail call <2 x i64> @llvm.x86.avx512.prol.q.128(<2 x i64> %__A, i32 5)
+  %0 = tail call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %__A, <2 x i64> %__A, <2 x i64> <i64 5, i64 5>)
   ret <2 x i64> %0
 }
 
-declare <2 x i64> @llvm.x86.avx512.prol.q.128(<2 x i64>, i32)
-
 define <2 x i64> @test_mm_mask_rol_epi64(<2 x i64> %__W, i8 zeroext %__U, <2 x i64> %__A) {
 ; X86-LABEL: test_mm_mask_rol_epi64:
 ; X86:       # %bb.0: # %entry
@@ -6765,7 +6759,7 @@ define <2 x i64> @test_mm_mask_rol_epi64(<2 x i64> %__W, i8 zeroext %__U, <2 x i
 ; X64-NEXT:    vprolq $5, %xmm1, %xmm0 {%k1}
 ; X64-NEXT:    retq
 entry:
-  %0 = tail call <2 x i64> @llvm.x86.avx512.prol.q.128(<2 x i64> %__A, i32 5)
+  %0 = tail call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %__A, <2 x i64> %__A, <2 x i64> <i64 5, i64 5>)
   %1 = bitcast i8 %__U to <8 x i1>
   %extract = shufflevector <8 x i1> %1, <8 x i1> undef, <2 x i32> <i32 0, i32 1>
   %2 = select <2 x i1> %extract, <2 x i64> %0, <2 x i64> %__W
@@ -6786,7 +6780,7 @@ define <2 x i64> @test_mm_maskz_rol_epi64(i8 zeroext %__U, <2 x i64> %__A) {
 ; X64-NEXT:    vprolq $5, %xmm0, %xmm0 {%k1} {z}
 ; X64-NEXT:    retq
 entry:
-  %0 = tail call <2 x i64> @llvm.x86.avx512.prol.q.128(<2 x i64> %__A, i32 5)
+  %0 = tail call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %__A, <2 x i64> %__A, <2 x i64> <i64 5, i64 5>)
   %1 = bitcast i8 %__U to <8 x i1>
   %extract = shufflevector <8 x i1> %1, <8 x i1> undef, <2 x i32> <i32 0, i32 1>
   %2 = select <2 x i1> %extract, <2 x i64> %0, <2 x i64> zeroinitializer
@@ -6799,12 +6793,10 @@ define <4 x i64> @test_mm256_rol_epi64(<4 x i64> %__A) {
 ; CHECK-NEXT:    vprolq $5, %ymm0, %ymm0
 ; CHECK-NEXT:    ret{{[l|q]}}
 entry:
-  %0 = tail call <4 x i64> @llvm.x86.avx512.prol.q.256(<4 x i64> %__A, i32 5)
+  %0 = tail call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %__A, <4 x i64> %__A, <4 x i64> <i64 5, i64 5,i64 5, i64 5>)
   ret <4 x i64> %0
 }
 
-declare <4 x i64> @llvm.x86.avx512.prol.q.256(<4 x i64>, i32)
-
 define <4 x i64> @test_mm256_mask_rol_epi64(<4 x i64> %__W, i8 zeroext %__U, <4 x i64> %__A) {
 ; X86-LABEL: test_mm256_mask_rol_epi64:
 ; X86:       # %bb.0: # %entry
@@ -6819,7 +6811,7 @@ define <4 x i64> @test_mm256_mask_rol_epi64(<4 x i64> %__W, i8 zeroext %__U, <4
 ; X64-NEXT:    vprolq $5, %ymm1, %ymm0 {%k1}
 ; X64-NEXT:    retq
 entry:
-  %0 = tail call <4 x i64> @llvm.x86.avx512.prol.q.256(<4 x i64> %__A, i32 5)
+  %0 = tail call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %__A, <4 x i64> %__A, <4 x i64> <i64 5, i64 5,i64 5, i64 5>)
   %1 = bitcast i8 %__U to <8 x i1>
   %extract = shufflevector <8 x i1> %1, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %2 = select <4 x i1> %extract, <4 x i64> %0, <4 x i64> %__W
@@ -6840,7 +6832,7 @@ define <4 x i64> @test_mm256_maskz_rol_epi64(i8 zeroext %__U, <4 x i64> %__A) {
 ; X64-NEXT:    vprolq $5, %ymm0, %ymm0 {%k1} {z}
 ; X64-NEXT:    retq
 entry:
-  %0 = tail call <4 x i64> @llvm.x86.avx512.prol.q.256(<4 x i64> %__A, i32 5)
+  %0 = tail call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %__A, <4 x i64> %__A, <4 x i64> <i64 5, i64 5,i64 5, i64 5>)
   %1 = bitcast i8 %__U to <8 x i1>
   %extract = shufflevector <8 x i1> %1, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %2 = select <4 x i1> %extract, <4 x i64> %0, <4 x i64> zeroinitializer
@@ -6855,7 +6847,7 @@ define <2 x i64> @test_mm_rolv_epi32(<2 x i64> %__A, <2 x i64> %__B) {
 entry:
   %0 = bitcast <2 x i64> %__A to <4 x i32>
   %1 = bitcast <2 x i64> %__B to <4 x i32>
-  %2 = tail call <4 x i32> @llvm.x86.avx512.prolv.d.128(<4 x i32> %0, <4 x i32> %1)
+  %2 = tail call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %0, <4 x i32> %0, <4 x i32> %1)
   %3 = bitcast <4 x i32> %2 to <2 x i64>
   ret <2 x i64> %3
 }
@@ -6876,7 +6868,7 @@ define <2 x i64> @test_mm_mask_rolv_epi32(<2 x i64> %__W, i8 zeroext %__U, <2 x
 entry:
   %0 = bitcast <2 x i64> %__A to <4 x i32>
   %1 = bitcast <2 x i64> %__B to <4 x i32>
-  %2 = tail call <4 x i32> @llvm.x86.avx512.prolv.d.128(<4 x i32> %0, <4 x i32> %1)
+  %2 = tail call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %0, <4 x i32> %0, <4 x i32> %1)
   %3 = bitcast <2 x i64> %__W to <4 x i32>
   %4 = bitcast i8 %__U to <8 x i1>
   %extract.i = shufflevector <8 x i1> %4, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
@@ -6901,7 +6893,7 @@ define <2 x i64> @test_mm_maskz_rolv_epi32(i8 zeroext %__U, <2 x i64> %__A, <2 x
 entry:
   %0 = bitcast <2 x i64> %__A to <4 x i32>
   %1 = bitcast <2 x i64> %__B to <4 x i32>
-  %2 = tail call <4 x i32> @llvm.x86.avx512.prolv.d.128(<4 x i32> %0, <4 x i32> %1)
+  %2 = tail call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %0, <4 x i32> %0, <4 x i32> %1)
   %3 = bitcast i8 %__U to <8 x i1>
   %extract.i = shufflevector <8 x i1> %3, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %4 = select <4 x i1> %extract.i, <4 x i32> %2, <4 x i32> zeroinitializer
@@ -6917,7 +6909,7 @@ define <4 x i64> @test_mm256_rolv_epi32(<4 x i64> %__A, <4 x i64> %__B) {
 entry:
   %0 = bitcast <4 x i64> %__A to <8 x i32>
   %1 = bitcast <4 x i64> %__B to <8 x i32>
-  %2 = tail call <8 x i32> @llvm.x86.avx512.prolv.d.256(<8 x i32> %0, <8 x i32> %1)
+  %2 = tail call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %0, <8 x i32> %0, <8 x i32> %1)
   %3 = bitcast <8 x i32> %2 to <4 x i64>
   ret <4 x i64> %3
 }
@@ -6938,7 +6930,7 @@ define <4 x i64> @test_mm256_mask_rolv_epi32(<4 x i64> %__W, i8 zeroext %__U, <4
 entry:
   %0 = bitcast <4 x i64> %__A to <8 x i32>
   %1 = bitcast <4 x i64> %__B to <8 x i32>
-  %2 = tail call <8 x i32> @llvm.x86.avx512.prolv.d.256(<8 x i32> %0, <8 x i32> %1)
+  %2 = tail call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %0, <8 x i32> %0, <8 x i32> %1)
   %3 = bitcast <4 x i64> %__W to <8 x i32>
   %4 = bitcast i8 %__U to <8 x i1>
   %5 = select <8 x i1> %4, <8 x i32> %2, <8 x i32> %3
@@ -6962,7 +6954,7 @@ define <4 x i64> @test_mm256_maskz_rolv_epi32(i8 zeroext %__U, <4 x i64> %__A, <
 entry:
   %0 = bitcast <4 x i64> %__A to <8 x i32>
   %1 = bitcast <4 x i64> %__B to <8 x i32>
-  %2 = tail call <8 x i32> @llvm.x86.avx512.prolv.d.256(<8 x i32> %0, <8 x i32> %1)
+  %2 = tail call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %0, <8 x i32> %0, <8 x i32> %1)
   %3 = bitcast i8 %__U to <8 x i1>
   %4 = select <8 x i1> %3, <8 x i32> %2, <8 x i32> zeroinitializer
   %5 = bitcast <8 x i32> %4 to <4 x i64>
@@ -6975,7 +6967,7 @@ define <2 x i64> @test_mm_rolv_epi64(<2 x i64> %__A, <2 x i64> %__B) {
 ; CHECK-NEXT:    vprolvq %xmm1, %xmm0, %xmm0
 ; CHECK-NEXT:    ret{{[l|q]}}
 entry:
-  %0 = tail call <2 x i64> @llvm.x86.avx512.prolv.q.128(<2 x i64> %__A, <2 x i64> %__B)
+  %0 = tail call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %__A, <2 x i64> %__A, <2 x i64> %__B)
   ret <2 x i64> %0
 }
 
@@ -6993,7 +6985,7 @@ define <2 x i64> @test_mm_mask_rolv_epi64(<2 x i64> %__W, i8 zeroext %__U, <2 x
 ; X64-NEXT:    vprolvq %xmm2, %xmm1, %xmm0 {%k1}
 ; X64-NEXT:    retq
 entry:
-  %0 = tail call <2 x i64> @llvm.x86.avx512.prolv.q.128(<2 x i64> %__A, <2 x i64> %__B)
+  %0 = tail call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %__A, <2 x i64> %__A, <2 x i64> %__B)
   %1 = bitcast i8 %__U to <8 x i1>
   %extract.i = shufflevector <8 x i1> %1, <8 x i1> undef, <2 x i32> <i32 0, i32 1>
   %2 = select <2 x i1> %extract.i, <2 x i64> %0, <2 x i64> %__W
@@ -7014,7 +7006,7 @@ define <2 x i64> @test_mm_maskz_rolv_epi64(i8 zeroext %__U, <2 x i64> %__A, <2 x
 ; X64-NEXT:    vprolvq %xmm1, %xmm0, %xmm0 {%k1} {z}
 ; X64-NEXT:    retq
 entry:
-  %0 = tail call <2 x i64> @llvm.x86.avx512.prolv.q.128(<2 x i64> %__A, <2 x i64> %__B)
+  %0 = tail call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %__A, <2 x i64> %__A, <2 x i64> %__B)
   %1 = bitcast i8 %__U to <8 x i1>
   %extract.i = shufflevector <8 x i1> %1, <8 x i1> undef, <2 x i32> <i32 0, i32 1>
   %2 = select <2 x i1> %extract.i, <2 x i64> %0, <2 x i64> zeroinitializer
@@ -7027,7 +7019,7 @@ define <4 x i64> @test_mm256_rolv_epi64(<4 x i64> %__A, <4 x i64> %__B) {
 ; CHECK-NEXT:    vprolvq %ymm1, %ymm0, %ymm0
 ; CHECK-NEXT:    ret{{[l|q]}}
 entry:
-  %0 = tail call <4 x i64> @llvm.x86.avx512.prolv.q.256(<4 x i64> %__A, <4 x i64> %__B)
+  %0 = tail call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %__A, <4 x i64> %__A, <4 x i64> %__B)
   ret <4 x i64> %0
 }
 
@@ -7045,7 +7037,7 @@ define <4 x i64> @test_mm256_mask_rolv_epi64(<4 x i64> %__W, i8 zeroext %__U, <4
 ; X64-NEXT:    vprolvq %ymm2, %ymm1, %ymm0 {%k1}
 ; X64-NEXT:    retq
 entry:
-  %0 = tail call <4 x i64> @llvm.x86.avx512.prolv.q.256(<4 x i64> %__A, <4 x i64> %__B)
+  %0 = tail call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %__A, <4 x i64> %__A, <4 x i64> %__B)
   %1 = bitcast i8 %__U to <8 x i1>
   %extract.i = shufflevector <8 x i1> %1, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %2 = select <4 x i1> %extract.i, <4 x i64> %0, <4 x i64> %__W
@@ -7066,7 +7058,7 @@ define <4 x i64> @test_mm256_maskz_rolv_epi64(i8 zeroext %__U, <4 x i64> %__A, <
 ; X64-NEXT:    vprolvq %ymm1, %ymm0, %ymm0 {%k1} {z}
 ; X64-NEXT:    retq
 entry:
-  %0 = tail call <4 x i64> @llvm.x86.avx512.prolv.q.256(<4 x i64> %__A, <4 x i64> %__B)
+  %0 = tail call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %__A, <4 x i64> %__A, <4 x i64> %__B)
   %1 = bitcast i8 %__U to <8 x i1>
   %extract.i = shufflevector <8 x i1> %1, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %2 = select <4 x i1> %extract.i, <4 x i64> %0, <4 x i64> zeroinitializer
@@ -7080,13 +7072,11 @@ define <2 x i64> @test_mm_ror_epi32(<2 x i64> %__A) {
 ; CHECK-NEXT:    ret{{[l|q]}}
 entry:
   %0 = bitcast <2 x i64> %__A to <4 x i32>
-  %1 = tail call <4 x i32> @llvm.x86.avx512.pror.d.128(<4 x i32> %0, i32 5)
+  %1 = tail call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %0, <4 x i32> %0, <4 x i32> <i32 5, i32 5, i32 5, i32 5>)
   %2 = bitcast <4 x i32> %1 to <2 x i64>
   ret <2 x i64> %2
 }
 
-declare <4 x i32> @llvm.x86.avx512.pror.d.128(<4 x i32>, i32)
-
 define <2 x i64> @test_mm_mask_ror_epi32(<2 x i64> %__W, i8 zeroext %__U, <2 x i64> %__A) {
 ; X86-LABEL: test_mm_mask_ror_epi32:
 ; X86:       # %bb.0: # %entry
@@ -7102,7 +7092,7 @@ define <2 x i64> @test_mm_mask_ror_epi32(<2 x i64> %__W, i8 zeroext %__U, <2 x i
 ; X64-NEXT:    retq
 entry:
   %0 = bitcast <2 x i64> %__A to <4 x i32>
-  %1 = tail call <4 x i32> @llvm.x86.avx512.pror.d.128(<4 x i32> %0, i32 5)
+  %1 = tail call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %0, <4 x i32> %0, <4 x i32> <i32 5, i32 5, i32 5, i32 5>)
   %2 = bitcast <2 x i64> %__W to <4 x i32>
   %3 = bitcast i8 %__U to <8 x i1>
   %extract = shufflevector <8 x i1> %3, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
@@ -7126,7 +7116,7 @@ define <2 x i64> @test_mm_maskz_ror_epi32(i8 zeroext %__U, <2 x i64> %__A) {
 ; X64-NEXT:    retq
 entry:
   %0 = bitcast <2 x i64> %__A to <4 x i32>
-  %1 = tail call <4 x i32> @llvm.x86.avx512.pror.d.128(<4 x i32> %0, i32 5)
+  %1 = tail call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %0, <4 x i32> %0, <4 x i32> <i32 5, i32 5, i32 5, i32 5>)
   %2 = bitcast i8 %__U to <8 x i1>
   %extract = shufflevector <8 x i1> %2, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %3 = select <4 x i1> %extract, <4 x i32> %1, <4 x i32> zeroinitializer
@@ -7141,13 +7131,11 @@ define <4 x i64> @test_mm256_ror_epi32(<4 x i64> %__A) {
 ; CHECK-NEXT:    ret{{[l|q]}}
 entry:
   %0 = bitcast <4 x i64> %__A to <8 x i32>
-  %1 = tail call <8 x i32> @llvm.x86.avx512.pror.d.256(<8 x i32> %0, i32 5)
+  %1 = tail call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %0, <8 x i32> %0, <8 x i32> <i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5>)
   %2 = bitcast <8 x i32> %1 to <4 x i64>
   ret <4 x i64> %2
 }
 
-declare <8 x i32> @llvm.x86.avx512.pror.d.256(<8 x i32>, i32)
-
 define <4 x i64> @test_mm256_mask_ror_epi32(<4 x i64> %__W, i8 zeroext %__U, <4 x i64> %__A) {
 ; X86-LABEL: test_mm256_mask_ror_epi32:
 ; X86:       # %bb.0: # %entry
@@ -7163,7 +7151,7 @@ define <4 x i64> @test_mm256_mask_ror_epi32(<4 x i64> %__W, i8 zeroext %__U, <4
 ; X64-NEXT:    retq
 entry:
   %0 = bitcast <4 x i64> %__A to <8 x i32>
-  %1 = tail call <8 x i32> @llvm.x86.avx512.pror.d.256(<8 x i32> %0, i32 5)
+  %1 = tail call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %0, <8 x i32> %0, <8 x i32> <i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5>)
   %2 = bitcast <4 x i64> %__W to <8 x i32>
   %3 = bitcast i8 %__U to <8 x i1>
   %4 = select <8 x i1> %3, <8 x i32> %1, <8 x i32> %2
@@ -7186,7 +7174,7 @@ define <4 x i64> @test_mm256_maskz_ror_epi32(i8 zeroext %__U, <4 x i64> %__A) {
 ; X64-NEXT:    retq
 entry:
   %0 = bitcast <4 x i64> %__A to <8 x i32>
-  %1 = tail call <8 x i32> @llvm.x86.avx512.pror.d.256(<8 x i32> %0, i32 5)
+  %1 = tail call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %0, <8 x i32> %0, <8 x i32> <i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5, i32 5>)
   %2 = bitcast i8 %__U to <8 x i1>
   %3 = select <8 x i1> %2, <8 x i32> %1, <8 x i32> zeroinitializer
   %4 = bitcast <8 x i32> %3 to <4 x i64>
@@ -7199,12 +7187,10 @@ define <2 x i64> @test_mm_ror_epi64(<2 x i64> %__A) {
 ; CHECK-NEXT:    vprorq $5, %xmm0, %xmm0
 ; CHECK-NEXT:    ret{{[l|q]}}
 entry:
-  %0 = tail call <2 x i64> @llvm.x86.avx512.pror.q.128(<2 x i64> %__A, i32 5)
+  %0 = tail call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %__A, <2 x i64> %__A, <2 x i64> <i64 5, i64 5>)
   ret <2 x i64> %0
 }
 
-declare <2 x i64> @llvm.x86.avx512.pror.q.128(<2 x i64>, i32)
-
 define <2 x i64> @test_mm_mask_ror_epi64(<2 x i64> %__W, i8 zeroext %__U, <2 x i64> %__A) {
 ; X86-LABEL: test_mm_mask_ror_epi64:
 ; X86:       # %bb.0: # %entry
@@ -7219,7 +7205,7 @@ define <2 x i64> @test_mm_mask_ror_epi64(<2 x i64> %__W, i8 zeroext %__U, <2 x i
 ; X64-NEXT:    vprorq $5, %xmm1, %xmm0 {%k1}
 ; X64-NEXT:    retq
 entry:
-  %0 = tail call <2 x i64> @llvm.x86.avx512.pror.q.128(<2 x i64> %__A, i32 5)
+  %0 = tail call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %__A, <2 x i64> %__A, <2 x i64> <i64 5, i64 5>)
   %1 = bitcast i8 %__U to <8 x i1>
   %extract = shufflevector <8 x i1> %1, <8 x i1> undef, <2 x i32> <i32 0, i32 1>
   %2 = select <2 x i1> %extract, <2 x i64> %0, <2 x i64> %__W
@@ -7240,7 +7226,7 @@ define <2 x i64> @test_mm_maskz_ror_epi64(i8 zeroext %__U, <2 x i64> %__A) {
 ; X64-NEXT:    vprorq $5, %xmm0, %xmm0 {%k1} {z}
 ; X64-NEXT:    retq
 entry:
-  %0 = tail call <2 x i64> @llvm.x86.avx512.pror.q.128(<2 x i64> %__A, i32 5)
+  %0 = tail call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %__A, <2 x i64> %__A, <2 x i64> <i64 5, i64 5>)
   %1 = bitcast i8 %__U to <8 x i1>
   %extract = shufflevector <8 x i1> %1, <8 x i1> undef, <2 x i32> <i32 0, i32 1>
   %2 = select <2 x i1> %extract, <2 x i64> %0, <2 x i64> zeroinitializer
@@ -7253,12 +7239,10 @@ define <4 x i64> @test_mm256_ror_epi64(<4 x i64> %__A) {
 ; CHECK-NEXT:    vprorq $5, %ymm0, %ymm0
 ; CHECK-NEXT:    ret{{[l|q]}}
 entry:
-  %0 = tail call <4 x i64> @llvm.x86.avx512.pror.q.256(<4 x i64> %__A, i32 5)
+  %0 = tail call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %__A, <4 x i64> %__A, <4 x i64> <i64 5, i64 5, i64 5, i64 5>)
   ret <4 x i64> %0
 }
 
-declare <4 x i64> @llvm.x86.avx512.pror.q.256(<4 x i64>, i32)
-
 define <4 x i64> @test_mm256_mask_ror_epi64(<4 x i64> %__W, i8 zeroext %__U, <4 x i64> %__A) {
 ; X86-LABEL: test_mm256_mask_ror_epi64:
 ; X86:       # %bb.0: # %entry
@@ -7273,7 +7257,7 @@ define <4 x i64> @test_mm256_mask_ror_epi64(<4 x i64> %__W, i8 zeroext %__U, <4
 ; X64-NEXT:    vprorq $5, %ymm1, %ymm0 {%k1}
 ; X64-NEXT:    retq
 entry:
-  %0 = tail call <4 x i64> @llvm.x86.avx512.pror.q.256(<4 x i64> %__A, i32 5)
+  %0 = tail call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %__A, <4 x i64> %__A, <4 x i64> <i64 5, i64 5, i64 5, i64 5>)
   %1 = bitcast i8 %__U to <8 x i1>
   %extract = shufflevector <8 x i1> %1, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %2 = select <4 x i1> %extract, <4 x i64> %0, <4 x i64> %__W
@@ -7294,7 +7278,7 @@ define <4 x i64> @test_mm256_maskz_ror_epi64(i8 zeroext %__U, <4 x i64> %__A) {
 ; X64-NEXT:    vprorq $5, %ymm0, %ymm0 {%k1} {z}
 ; X64-NEXT:    retq
 entry:
-  %0 = tail call <4 x i64> @llvm.x86.avx512.pror.q.256(<4 x i64> %__A, i32 5)
+  %0 = tail call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %__A, <4 x i64> %__A, <4 x i64> <i64 5, i64 5, i64 5, i64 5>)
   %1 = bitcast i8 %__U to <8 x i1>
   %extract = shufflevector <8 x i1> %1, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %2 = select <4 x i1> %extract, <4 x i64> %0, <4 x i64> zeroinitializer
@@ -7309,7 +7293,7 @@ define <2 x i64> @test_mm_rorv_epi32(<2 x i64> %__A, <2 x i64> %__B) {
 entry:
   %0 = bitcast <2 x i64> %__A to <4 x i32>
   %1 = bitcast <2 x i64> %__B to <4 x i32>
-  %2 = tail call <4 x i32> @llvm.x86.avx512.prorv.d.128(<4 x i32> %0, <4 x i32> %1)
+  %2 = tail call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %0, <4 x i32> %0, <4 x i32> %1)
   %3 = bitcast <4 x i32> %2 to <2 x i64>
   ret <2 x i64> %3
 }
@@ -7330,7 +7314,7 @@ define <2 x i64> @test_mm_mask_rorv_epi32(<2 x i64> %__W, i8 zeroext %__U, <2 x
 entry:
   %0 = bitcast <2 x i64> %__A to <4 x i32>
   %1 = bitcast <2 x i64> %__B to <4 x i32>
-  %2 = tail call <4 x i32> @llvm.x86.avx512.prorv.d.128(<4 x i32> %0, <4 x i32> %1)
+  %2 = tail call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %0, <4 x i32> %0, <4 x i32> %1)
   %3 = bitcast <2 x i64> %__W to <4 x i32>
   %4 = bitcast i8 %__U to <8 x i1>
   %extract.i = shufflevector <8 x i1> %4, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
@@ -7355,7 +7339,7 @@ define <2 x i64> @test_mm_maskz_rorv_epi32(i8 zeroext %__U, <2 x i64> %__A, <2 x
 entry:
   %0 = bitcast <2 x i64> %__A to <4 x i32>
   %1 = bitcast <2 x i64> %__B to <4 x i32>
-  %2 = tail call <4 x i32> @llvm.x86.avx512.prorv.d.128(<4 x i32> %0, <4 x i32> %1)
+  %2 = tail call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %0, <4 x i32> %0, <4 x i32> %1)
   %3 = bitcast i8 %__U to <8 x i1>
   %extract.i = shufflevector <8 x i1> %3, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %4 = select <4 x i1> %extract.i, <4 x i32> %2, <4 x i32> zeroinitializer
@@ -7371,7 +7355,7 @@ define <4 x i64> @test_mm256_rorv_epi32(<4 x i64> %__A, <4 x i64> %__B) {
 entry:
   %0 = bitcast <4 x i64> %__A to <8 x i32>
   %1 = bitcast <4 x i64> %__B to <8 x i32>
-  %2 = tail call <8 x i32> @llvm.x86.avx512.prorv.d.256(<8 x i32> %0, <8 x i32> %1)
+  %2 = tail call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %0, <8 x i32> %0, <8 x i32> %1)
   %3 = bitcast <8 x i32> %2 to <4 x i64>
   ret <4 x i64> %3
 }
@@ -7392,7 +7376,7 @@ define <4 x i64> @test_mm256_mask_rorv_epi32(<4 x i64> %__W, i8 zeroext %__U, <4
 entry:
   %0 = bitcast <4 x i64> %__A to <8 x i32>
   %1 = bitcast <4 x i64> %__B to <8 x i32>
-  %2 = tail call <8 x i32> @llvm.x86.avx512.prorv.d.256(<8 x i32> %0, <8 x i32> %1)
+  %2 = tail call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %0, <8 x i32> %0, <8 x i32> %1)
   %3 = bitcast <4 x i64> %__W to <8 x i32>
   %4 = bitcast i8 %__U to <8 x i1>
   %5 = select <8 x i1> %4, <8 x i32> %2, <8 x i32> %3
@@ -7416,7 +7400,7 @@ define <4 x i64> @test_mm256_maskz_rorv_epi32(i8 zeroext %__U, <4 x i64> %__A, <
 entry:
   %0 = bitcast <4 x i64> %__A to <8 x i32>
   %1 = bitcast <4 x i64> %__B to <8 x i32>
-  %2 = tail call <8 x i32> @llvm.x86.avx512.prorv.d.256(<8 x i32> %0, <8 x i32> %1)
+  %2 = tail call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %0, <8 x i32> %0, <8 x i32> %1)
   %3 = bitcast i8 %__U to <8 x i1>
   %4 = select <8 x i1> %3, <8 x i32> %2, <8 x i32> zeroinitializer
   %5 = bitcast <8 x i32> %4 to <4 x i64>
@@ -7429,7 +7413,7 @@ define <2 x i64> @test_mm_rorv_epi64(<2 x i64> %__A, <2 x i64> %__B) {
 ; CHECK-NEXT:    vprorvq %xmm1, %xmm0, %xmm0
 ; CHECK-NEXT:    ret{{[l|q]}}
 entry:
-  %0 = tail call <2 x i64> @llvm.x86.avx512.prorv.q.128(<2 x i64> %__A, <2 x i64> %__B)
+  %0 = tail call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %__A, <2 x i64> %__A, <2 x i64> %__B)
   ret <2 x i64> %0
 }
 
@@ -7447,7 +7431,7 @@ define <2 x i64> @test_mm_mask_rorv_epi64(<2 x i64> %__W, i8 zeroext %__U, <2 x
 ; X64-NEXT:    vprorvq %xmm2, %xmm1, %xmm0 {%k1}
 ; X64-NEXT:    retq
 entry:
-  %0 = tail call <2 x i64> @llvm.x86.avx512.prorv.q.128(<2 x i64> %__A, <2 x i64> %__B)
+  %0 = tail call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %__A, <2 x i64> %__A, <2 x i64> %__B)
   %1 = bitcast i8 %__U to <8 x i1>
   %extract.i = shufflevector <8 x i1> %1, <8 x i1> undef, <2 x i32> <i32 0, i32 1>
   %2 = select <2 x i1> %extract.i, <2 x i64> %0, <2 x i64> %__W
@@ -7468,7 +7452,7 @@ define <2 x i64> @test_mm_maskz_rorv_epi64(i8 zeroext %__U, <2 x i64> %__A, <2 x
 ; X64-NEXT:    vprorvq %xmm1, %xmm0, %xmm0 {%k1} {z}
 ; X64-NEXT:    retq
 entry:
-  %0 = tail call <2 x i64> @llvm.x86.avx512.prorv.q.128(<2 x i64> %__A, <2 x i64> %__B)
+  %0 = tail call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %__A, <2 x i64> %__A, <2 x i64> %__B)
   %1 = bitcast i8 %__U to <8 x i1>
   %extract.i = shufflevector <8 x i1> %1, <8 x i1> undef, <2 x i32> <i32 0, i32 1>
   %2 = select <2 x i1> %extract.i, <2 x i64> %0, <2 x i64> zeroinitializer
@@ -7481,7 +7465,7 @@ define <4 x i64> @test_mm256_rorv_epi64(<4 x i64> %__A, <4 x i64> %__B) {
 ; CHECK-NEXT:    vprorvq %ymm1, %ymm0, %ymm0
 ; CHECK-NEXT:    ret{{[l|q]}}
 entry:
-  %0 = tail call <4 x i64> @llvm.x86.avx512.prorv.q.256(<4 x i64> %__A, <4 x i64> %__B)
+  %0 = tail call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %__A, <4 x i64> %__A, <4 x i64> %__B)
   ret <4 x i64> %0
 }
 
@@ -7499,7 +7483,7 @@ define <4 x i64> @test_mm256_mask_rorv_epi64(<4 x i64> %__W, i8 zeroext %__U, <4
 ; X64-NEXT:    vprorvq %ymm2, %ymm1, %ymm0 {%k1}
 ; X64-NEXT:    retq
 entry:
-  %0 = tail call <4 x i64> @llvm.x86.avx512.prorv.q.256(<4 x i64> %__A, <4 x i64> %__B)
+  %0 = tail call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %__A, <4 x i64> %__A, <4 x i64> %__B)
   %1 = bitcast i8 %__U to <8 x i1>
   %extract.i = shufflevector <8 x i1> %1, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %2 = select <4 x i1> %extract.i, <4 x i64> %0, <4 x i64> %__W
@@ -7520,7 +7504,7 @@ define <4 x i64> @test_mm256_maskz_rorv_epi64(i8 zeroext %__U, <4 x i64> %__A, <
 ; X64-NEXT:    vprorvq %ymm1, %ymm0, %ymm0 {%k1} {z}
 ; X64-NEXT:    retq
 entry:
-  %0 = tail call <4 x i64> @llvm.x86.avx512.prorv.q.256(<4 x i64> %__A, <4 x i64> %__B)
+  %0 = tail call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %__A, <4 x i64> %__A, <4 x i64> %__B)
   %1 = bitcast i8 %__U to <8 x i1>
   %extract.i = shufflevector <8 x i1> %1, <8 x i1> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %2 = select <4 x i1> %extract.i, <4 x i64> %0, <4 x i64> zeroinitializer
@@ -7572,13 +7556,13 @@ declare void @llvm.masked.compressstore.v4f32(<4 x float>, float*, <4 x i1>)
 declare void @llvm.masked.compressstore.v8f32(<8 x float>, float*, <8 x i1>)
 declare void @llvm.masked.compressstore.v4i32(<4 x i32>, i32*, <4 x i1>)
 declare void @llvm.masked.compressstore.v8i32(<8 x i32>, i32*, <8 x i1>)
-declare <4 x i32> @llvm.x86.avx512.prolv.d.128(<4 x i32>, <4 x i32>)
-declare <8 x i32> @llvm.x86.avx512.prolv.d.256(<8 x i32>, <8 x i32>)
-declare <2 x i64> @llvm.x86.avx512.prolv.q.128(<2 x i64>, <2 x i64>)
-declare <4 x i64> @llvm.x86.avx512.prolv.q.256(<4 x i64>, <4 x i64>)
-declare <4 x i32> @llvm.x86.avx512.prorv.d.128(<4 x i32>, <4 x i32>)
-declare <8 x i32> @llvm.x86.avx512.prorv.d.256(<8 x i32>, <8 x i32>)
-declare <2 x i64> @llvm.x86.avx512.prorv.q.128(<2 x i64>, <2 x i64>)
-declare <4 x i64> @llvm.x86.avx512.prorv.q.256(<4 x i64>, <4 x i64>)
+declare <4 x i32> @llvm.fshl.v4i32(<4 x i32>, <4 x i32>, <4 x i32>)
+declare <8 x i32> @llvm.fshl.v8i32(<8 x i32>, <8 x i32>, <8 x i32>)
+declare <2 x i64> @llvm.fshl.v2i64(<2 x i64>, <2 x i64>, <2 x i64>)
+declare <4 x i64> @llvm.fshl.v4i64(<4 x i64>, <4 x i64>, <4 x i64>)
+declare <4 x i32> @llvm.fshr.v4i32(<4 x i32>, <4 x i32>, <4 x i32>)
+declare <8 x i32> @llvm.fshr.v8i32(<8 x i32>, <8 x i32>, <8 x i32>)
+declare <2 x i64> @llvm.fshr.v2i64(<2 x i64>, <2 x i64>, <2 x i64>)
+declare <4 x i64> @llvm.fshr.v4i64(<4 x i64>, <4 x i64>, <4 x i64>)
 
 !0 = !{i32 1}
diff --git a/test/CodeGen/X86/avx512vl-intrinsics-upgrade.ll b/test/CodeGen/X86/avx512vl-intrinsics-upgrade.ll
index 6e78ab9826a9a83bd1aa03cde13479f568b65d83..6fd1bbbcb1f244a9ef8a9947258fa86913e850d3 100644
--- a/test/CodeGen/X86/avx512vl-intrinsics-upgrade.ll
+++ b/test/CodeGen/X86/avx512vl-intrinsics-upgrade.ll
@@ -3910,8 +3910,9 @@ define <2 x i64> @test_mask_andnot_epi64_rmb_128(<2 x i64> %a, i64* %ptr_b) {
 ; X86-LABEL: test_mask_andnot_epi64_rmb_128:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
-; X86-NEXT:    vpbroadcastq (%eax), %xmm1 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x79,0x59,0x08]
-; X86-NEXT:    vpandn %xmm1, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xdf,0xc1]
+; X86-NEXT:    vmovddup (%eax), %xmm1 # EVEX TO VEX Compression encoding: [0xc5,0xfb,0x12,0x08]
+; X86-NEXT:    # xmm1 = mem[0,0]
+; X86-NEXT:    vandnps %xmm1, %xmm0, %xmm0 # encoding: [0xc5,0xf8,0x55,0xc1]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_mask_andnot_epi64_rmb_128:
@@ -12592,1379 +12593,2732 @@ define <4 x i64>@test_int_x86_avx512_mask_pror_q_256(<4 x i64> %x0, i32 %x1, <4
   ret <4 x i64> %res4
 }
 
-declare <8 x float> @llvm.x86.avx512.mask.vfmadd.ps.256(<8 x float>, <8 x float>, <8 x float>, i8) nounwind readnone
-
-define <8 x float> @test_vfmadd256_ps(<8 x float> %a0, <8 x float> %a1, <8 x float> %a2) {
-; CHECK-LABEL: test_vfmadd256_ps:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vfmadd213ps %ymm2, %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x75,0xa8,0xc2]
-; CHECK-NEXT:    # ymm0 = (ymm1 * ymm0) + ymm2
-; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
-  %res = call <8 x float> @llvm.x86.avx512.mask.vfmadd.ps.256(<8 x float> %a0, <8 x float> %a1, <8 x float> %a2, i8 -1) nounwind
-  ret <8 x float> %res
-}
+declare <4 x i32> @llvm.x86.avx512.prorv.d.128(<4 x i32>, <4 x i32>)
 
-define <8 x float> @test_mask_vfmadd256_ps(<8 x float> %a0, <8 x float> %a1, <8 x float> %a2, i8 %mask) {
-; X86-LABEL: test_mask_vfmadd256_ps:
+define <4 x i32>@test_int_x86_avx512_prorv_d_128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_prorv_d_128:
 ; X86:       # %bb.0:
+; X86-NEXT:    vprorvd %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf2,0x7d,0x08,0x14,0xd9]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
-; X86-NEXT:    vfmadd132ps %ymm1, %ymm2, %ymm0 {%k1} # encoding: [0x62,0xf2,0x6d,0x29,0x98,0xc1]
-; X86-NEXT:    # ymm0 = (ymm0 * ymm1) + ymm2
+; X86-NEXT:    vprorvd %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf2,0x7d,0x09,0x14,0xd1]
+; X86-NEXT:    vprorvd %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0x89,0x14,0xc1]
+; X86-NEXT:    vpaddd %xmm3, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfe,0xc3]
+; X86-NEXT:    vpaddd %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfe,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
-; X64-LABEL: test_mask_vfmadd256_ps:
+; X64-LABEL: test_int_x86_avx512_prorv_d_128:
 ; X64:       # %bb.0:
+; X64-NEXT:    vprorvd %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf2,0x7d,0x08,0x14,0xd9]
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
-; X64-NEXT:    vfmadd132ps %ymm1, %ymm2, %ymm0 {%k1} # encoding: [0x62,0xf2,0x6d,0x29,0x98,0xc1]
-; X64-NEXT:    # ymm0 = (ymm0 * ymm1) + ymm2
+; X64-NEXT:    vprorvd %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf2,0x7d,0x09,0x14,0xd1]
+; X64-NEXT:    vprorvd %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0x89,0x14,0xc1]
+; X64-NEXT:    vpaddd %xmm3, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfe,0xc3]
+; X64-NEXT:    vpaddd %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfe,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <8 x float> @llvm.x86.avx512.mask.vfmadd.ps.256(<8 x float> %a0, <8 x float> %a1, <8 x float> %a2, i8 %mask) nounwind
-  ret <8 x float> %res
+  %1 = call <4 x i32> @llvm.x86.avx512.prorv.d.128(<4 x i32> %x0, <4 x i32> %x1)
+  %2 = bitcast i8 %x3 to <8 x i1>
+  %extract1 = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %3 = select <4 x i1> %extract1, <4 x i32> %1, <4 x i32> %x2
+  %4 = call <4 x i32> @llvm.x86.avx512.prorv.d.128(<4 x i32> %x0, <4 x i32> %x1)
+  %5 = bitcast i8 %x3 to <8 x i1>
+  %extract = shufflevector <8 x i1> %5, <8 x i1> %5, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %6 = select <4 x i1> %extract, <4 x i32> %4, <4 x i32> zeroinitializer
+  %7 = call <4 x i32> @llvm.x86.avx512.prorv.d.128(<4 x i32> %x0, <4 x i32> %x1)
+  %res3 = add <4 x i32> %3, %6
+  %res4 = add <4 x i32> %res3, %7
+  ret <4 x i32> %res4
 }
 
-declare <4 x float> @llvm.x86.avx512.mask.vfmadd.ps.128(<4 x float>, <4 x float>, <4 x float>, i8) nounwind readnone
-
-define <4 x float> @test_vfmadd128_ps(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2) {
-; CHECK-LABEL: test_vfmadd128_ps:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vfmadd213ps %xmm2, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x71,0xa8,0xc2]
-; CHECK-NEXT:    # xmm0 = (xmm1 * xmm0) + xmm2
-; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
-  %res = call <4 x float> @llvm.x86.avx512.mask.vfmadd.ps.128(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, i8 -1) nounwind
-  ret <4 x float> %res
-}
+declare <8 x i32> @llvm.x86.avx512.prorv.d.256(<8 x i32>, <8 x i32>)
 
-define <4 x float> @test_mask_vfmadd128_ps(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, i8 %mask) {
-; X86-LABEL: test_mask_vfmadd128_ps:
+define <8 x i32>@test_int_x86_avx512_prorv_d_256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_prorv_d_256:
 ; X86:       # %bb.0:
+; X86-NEXT:    vprorvd %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf2,0x7d,0x28,0x14,0xd9]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
-; X86-NEXT:    vfmadd132ps %xmm1, %xmm2, %xmm0 {%k1} # encoding: [0x62,0xf2,0x6d,0x09,0x98,0xc1]
-; X86-NEXT:    # xmm0 = (xmm0 * xmm1) + xmm2
+; X86-NEXT:    vprorvd %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf2,0x7d,0x29,0x14,0xd1]
+; X86-NEXT:    vprorvd %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0xa9,0x14,0xc1]
+; X86-NEXT:    vpaddd %ymm3, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xfe,0xc3]
+; X86-NEXT:    vpaddd %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfe,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
-; X64-LABEL: test_mask_vfmadd128_ps:
+; X64-LABEL: test_int_x86_avx512_prorv_d_256:
 ; X64:       # %bb.0:
+; X64-NEXT:    vprorvd %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf2,0x7d,0x28,0x14,0xd9]
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
-; X64-NEXT:    vfmadd132ps %xmm1, %xmm2, %xmm0 {%k1} # encoding: [0x62,0xf2,0x6d,0x09,0x98,0xc1]
-; X64-NEXT:    # xmm0 = (xmm0 * xmm1) + xmm2
+; X64-NEXT:    vprorvd %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf2,0x7d,0x29,0x14,0xd1]
+; X64-NEXT:    vprorvd %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0xa9,0x14,0xc1]
+; X64-NEXT:    vpaddd %ymm3, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xfe,0xc3]
+; X64-NEXT:    vpaddd %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfe,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <4 x float> @llvm.x86.avx512.mask.vfmadd.ps.128(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, i8 %mask) nounwind
-  ret <4 x float> %res
+  %1 = call <8 x i32> @llvm.x86.avx512.prorv.d.256(<8 x i32> %x0, <8 x i32> %x1)
+  %2 = bitcast i8 %x3 to <8 x i1>
+  %3 = select <8 x i1> %2, <8 x i32> %1, <8 x i32> %x2
+  %4 = call <8 x i32> @llvm.x86.avx512.prorv.d.256(<8 x i32> %x0, <8 x i32> %x1)
+  %5 = bitcast i8 %x3 to <8 x i1>
+  %6 = select <8 x i1> %5, <8 x i32> %4, <8 x i32> zeroinitializer
+  %7 = call <8 x i32> @llvm.x86.avx512.prorv.d.256(<8 x i32> %x0, <8 x i32> %x1)
+  %res3 = add <8 x i32> %3, %6
+  %res4 = add <8 x i32> %res3, %7
+  ret <8 x i32> %res4
 }
 
-declare <4 x double> @llvm.x86.avx512.mask.vfmadd.pd.256(<4 x double>, <4 x double>, <4 x double>, i8)
-
-define <4 x double> @test_fmadd256_pd(<4 x double> %a, <4 x double> %b, <4 x double> %c) {
-; CHECK-LABEL: test_fmadd256_pd:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vfmadd213pd %ymm2, %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0xf5,0xa8,0xc2]
-; CHECK-NEXT:    # ymm0 = (ymm1 * ymm0) + ymm2
-; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
-  %res = call <4 x double> @llvm.x86.avx512.mask.vfmadd.pd.256(<4 x double> %a, <4 x double> %b, <4 x double> %c, i8 -1)
-  ret <4 x double> %res
-}
+declare <2 x i64> @llvm.x86.avx512.prorv.q.128(<2 x i64>, <2 x i64>)
 
-define <4 x double> @test_mask_fmadd256_pd(<4 x double> %a, <4 x double> %b, <4 x double> %c, i8 %mask) {
-; X86-LABEL: test_mask_fmadd256_pd:
+define <2 x i64>@test_int_x86_avx512_prorv_q_128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_prorv_q_128:
 ; X86:       # %bb.0:
+; X86-NEXT:    vprorvq %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf2,0xfd,0x08,0x14,0xd9]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
-; X86-NEXT:    vfmadd132pd %ymm1, %ymm2, %ymm0 {%k1} # encoding: [0x62,0xf2,0xed,0x29,0x98,0xc1]
-; X86-NEXT:    # ymm0 = (ymm0 * ymm1) + ymm2
+; X86-NEXT:    vprorvq %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x09,0x14,0xd1]
+; X86-NEXT:    vprorvq %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0x89,0x14,0xc1]
+; X86-NEXT:    vpaddq %xmm3, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xd4,0xc3]
+; X86-NEXT:    vpaddq %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xd4,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
-; X64-LABEL: test_mask_fmadd256_pd:
+; X64-LABEL: test_int_x86_avx512_prorv_q_128:
 ; X64:       # %bb.0:
+; X64-NEXT:    vprorvq %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf2,0xfd,0x08,0x14,0xd9]
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
-; X64-NEXT:    vfmadd132pd %ymm1, %ymm2, %ymm0 {%k1} # encoding: [0x62,0xf2,0xed,0x29,0x98,0xc1]
-; X64-NEXT:    # ymm0 = (ymm0 * ymm1) + ymm2
+; X64-NEXT:    vprorvq %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x09,0x14,0xd1]
+; X64-NEXT:    vprorvq %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0x89,0x14,0xc1]
+; X64-NEXT:    vpaddq %xmm3, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xd4,0xc3]
+; X64-NEXT:    vpaddq %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xd4,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <4 x double> @llvm.x86.avx512.mask.vfmadd.pd.256(<4 x double> %a, <4 x double> %b, <4 x double> %c, i8 %mask)
-  ret <4 x double> %res
+  %1 = call <2 x i64> @llvm.x86.avx512.prorv.q.128(<2 x i64> %x0, <2 x i64> %x1)
+  %2 = bitcast i8 %x3 to <8 x i1>
+  %extract1 = shufflevector <8 x i1> %2, <8 x i1> %2, <2 x i32> <i32 0, i32 1>
+  %3 = select <2 x i1> %extract1, <2 x i64> %1, <2 x i64> %x2
+  %4 = call <2 x i64> @llvm.x86.avx512.prorv.q.128(<2 x i64> %x0, <2 x i64> %x1)
+  %5 = bitcast i8 %x3 to <8 x i1>
+  %extract = shufflevector <8 x i1> %5, <8 x i1> %5, <2 x i32> <i32 0, i32 1>
+  %6 = select <2 x i1> %extract, <2 x i64> %4, <2 x i64> zeroinitializer
+  %7 = call <2 x i64> @llvm.x86.avx512.prorv.q.128(<2 x i64> %x0, <2 x i64> %x1)
+  %res3 = add <2 x i64> %3, %6
+  %res4 = add <2 x i64> %res3, %7
+  ret <2 x i64> %res4
 }
 
-declare <2 x double> @llvm.x86.avx512.mask.vfmadd.pd.128(<2 x double>, <2 x double>, <2 x double>, i8)
-
-define <2 x double> @test_fmadd128_pd(<2 x double> %a, <2 x double> %b, <2 x double> %c) {
-; CHECK-LABEL: test_fmadd128_pd:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vfmadd213pd %xmm2, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0xf1,0xa8,0xc2]
-; CHECK-NEXT:    # xmm0 = (xmm1 * xmm0) + xmm2
-; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
-  %res = call <2 x double> @llvm.x86.avx512.mask.vfmadd.pd.128(<2 x double> %a, <2 x double> %b, <2 x double> %c, i8 -1)
-  ret <2 x double> %res
-}
+declare <4 x i64> @llvm.x86.avx512.prorv.q.256(<4 x i64>, <4 x i64>)
 
-define <2 x double> @test_mask_fmadd128_pd(<2 x double> %a, <2 x double> %b, <2 x double> %c, i8 %mask) {
-; X86-LABEL: test_mask_fmadd128_pd:
+define <4 x i64>@test_int_x86_avx512_prorv_q_256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_prorv_q_256:
 ; X86:       # %bb.0:
+; X86-NEXT:    vprorvq %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf2,0xfd,0x28,0x14,0xd9]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
-; X86-NEXT:    vfmadd132pd %xmm1, %xmm2, %xmm0 {%k1} # encoding: [0x62,0xf2,0xed,0x09,0x98,0xc1]
-; X86-NEXT:    # xmm0 = (xmm0 * xmm1) + xmm2
+; X86-NEXT:    vprorvq %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x29,0x14,0xd1]
+; X86-NEXT:    vprorvq %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xa9,0x14,0xc1]
+; X86-NEXT:    vpaddq %ymm3, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xd4,0xc3]
+; X86-NEXT:    vpaddq %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xd4,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
-; X64-LABEL: test_mask_fmadd128_pd:
+; X64-LABEL: test_int_x86_avx512_prorv_q_256:
 ; X64:       # %bb.0:
+; X64-NEXT:    vprorvq %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf2,0xfd,0x28,0x14,0xd9]
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
-; X64-NEXT:    vfmadd132pd %xmm1, %xmm2, %xmm0 {%k1} # encoding: [0x62,0xf2,0xed,0x09,0x98,0xc1]
-; X64-NEXT:    # xmm0 = (xmm0 * xmm1) + xmm2
+; X64-NEXT:    vprorvq %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x29,0x14,0xd1]
+; X64-NEXT:    vprorvq %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xa9,0x14,0xc1]
+; X64-NEXT:    vpaddq %ymm3, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xd4,0xc3]
+; X64-NEXT:    vpaddq %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xd4,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <2 x double> @llvm.x86.avx512.mask.vfmadd.pd.128(<2 x double> %a, <2 x double> %b, <2 x double> %c, i8 %mask)
-  ret <2 x double> %res
+  %1 = call <4 x i64> @llvm.x86.avx512.prorv.q.256(<4 x i64> %x0, <4 x i64> %x1)
+  %2 = bitcast i8 %x3 to <8 x i1>
+  %extract1 = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %3 = select <4 x i1> %extract1, <4 x i64> %1, <4 x i64> %x2
+  %4 = call <4 x i64> @llvm.x86.avx512.prorv.q.256(<4 x i64> %x0, <4 x i64> %x1)
+  %5 = bitcast i8 %x3 to <8 x i1>
+  %extract = shufflevector <8 x i1> %5, <8 x i1> %5, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %6 = select <4 x i1> %extract, <4 x i64> %4, <4 x i64> zeroinitializer
+  %7 = call <4 x i64> @llvm.x86.avx512.prorv.q.256(<4 x i64> %x0, <4 x i64> %x1)
+  %res3 = add <4 x i64> %3, %6
+  %res4 = add <4 x i64> %res3, %7
+  ret <4 x i64> %res4
 }
 
-declare <2 x double> @llvm.x86.avx512.mask3.vfmadd.pd.128(<2 x double>, <2 x double>, <2 x double>, i8)
+declare <4 x i32> @llvm.x86.avx512.prol.d.128(<4 x i32>, i32)
 
-define <2 x double>@test_int_x86_avx512_mask3_vfmadd_pd_128(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3) {
-; X86-LABEL: test_int_x86_avx512_mask3_vfmadd_pd_128:
+define <4 x i32>@test_int_x86_avx512_prol_d_128(<4 x i32> %x0, i32 %x1, <4 x i32> %x2, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_prol_d_128:
 ; X86:       # %bb.0:
-; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    vprold $3, %xmm0, %xmm2 # encoding: [0x62,0xf1,0x6d,0x08,0x72,0xc8,0x03]
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x08]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
-; X86-NEXT:    vfmadd231pd %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x09,0xb8,0xd1]
-; X86-NEXT:    # xmm2 = (xmm0 * xmm1) + xmm2
-; X86-NEXT:    vmovapd %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x28,0xc2]
+; X86-NEXT:    vprold $3, %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf1,0x75,0x09,0x72,0xc8,0x03]
+; X86-NEXT:    vprold $3, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0x89,0x72,0xc8,0x03]
+; X86-NEXT:    vpaddd %xmm2, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfe,0xc2]
+; X86-NEXT:    vpaddd %xmm0, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0xfe,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
-; X64-LABEL: test_int_x86_avx512_mask3_vfmadd_pd_128:
+; X64-LABEL: test_int_x86_avx512_prol_d_128:
 ; X64:       # %bb.0:
-; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
-; X64-NEXT:    vfmadd231pd %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x09,0xb8,0xd1]
-; X64-NEXT:    # xmm2 = (xmm0 * xmm1) + xmm2
-; X64-NEXT:    vmovapd %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x28,0xc2]
+; X64-NEXT:    vprold $3, %xmm0, %xmm2 # encoding: [0x62,0xf1,0x6d,0x08,0x72,0xc8,0x03]
+; X64-NEXT:    kmovw %esi, %k1 # encoding: [0xc5,0xf8,0x92,0xce]
+; X64-NEXT:    vprold $3, %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf1,0x75,0x09,0x72,0xc8,0x03]
+; X64-NEXT:    vprold $3, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0x89,0x72,0xc8,0x03]
+; X64-NEXT:    vpaddd %xmm2, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfe,0xc2]
+; X64-NEXT:    vpaddd %xmm0, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0xfe,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <2 x double> @llvm.x86.avx512.mask3.vfmadd.pd.128(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3)
-  ret <2 x double> %res
+  %1 = call <4 x i32> @llvm.x86.avx512.prol.d.128(<4 x i32> %x0, i32 3)
+  %2 = bitcast i8 %x3 to <8 x i1>
+  %extract1 = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %3 = select <4 x i1> %extract1, <4 x i32> %1, <4 x i32> %x2
+  %4 = call <4 x i32> @llvm.x86.avx512.prol.d.128(<4 x i32> %x0, i32 3)
+  %5 = bitcast i8 %x3 to <8 x i1>
+  %extract = shufflevector <8 x i1> %5, <8 x i1> %5, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %6 = select <4 x i1> %extract, <4 x i32> %4, <4 x i32> zeroinitializer
+  %7 = call <4 x i32> @llvm.x86.avx512.prol.d.128(<4 x i32> %x0, i32 3)
+  %res3 = add <4 x i32> %3, %6
+  %res4 = add <4 x i32> %res3, %7
+  ret <4 x i32> %res4
 }
 
-declare <2 x double> @llvm.x86.avx512.maskz.vfmadd.pd.128(<2 x double>, <2 x double>, <2 x double>, i8)
+declare <8 x i32> @llvm.x86.avx512.prol.d.256(<8 x i32>, i32)
 
-define <2 x double>@test_int_x86_avx512_maskz_vfmadd_pd_128(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3) {
-; X86-LABEL: test_int_x86_avx512_maskz_vfmadd_pd_128:
+define <8 x i32>@test_int_x86_avx512_prol_d_256(<8 x i32> %x0, i32 %x1, <8 x i32> %x2, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_prol_d_256:
 ; X86:       # %bb.0:
-; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    vprold $3, %ymm0, %ymm2 # encoding: [0x62,0xf1,0x6d,0x28,0x72,0xc8,0x03]
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x08]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
-; X86-NEXT:    vfmadd213pd %xmm2, %xmm1, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0x89,0xa8,0xc2]
-; X86-NEXT:    # xmm0 = (xmm1 * xmm0) + xmm2
+; X86-NEXT:    vprold $3, %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf1,0x75,0x29,0x72,0xc8,0x03]
+; X86-NEXT:    vprold $3, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xa9,0x72,0xc8,0x03]
+; X86-NEXT:    vpaddd %ymm2, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xfe,0xc2]
+; X86-NEXT:    vpaddd %ymm0, %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xf5,0xfe,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
-; X64-LABEL: test_int_x86_avx512_maskz_vfmadd_pd_128:
+; X64-LABEL: test_int_x86_avx512_prol_d_256:
 ; X64:       # %bb.0:
-; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
-; X64-NEXT:    vfmadd213pd %xmm2, %xmm1, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0x89,0xa8,0xc2]
-; X64-NEXT:    # xmm0 = (xmm1 * xmm0) + xmm2
+; X64-NEXT:    vprold $3, %ymm0, %ymm2 # encoding: [0x62,0xf1,0x6d,0x28,0x72,0xc8,0x03]
+; X64-NEXT:    kmovw %esi, %k1 # encoding: [0xc5,0xf8,0x92,0xce]
+; X64-NEXT:    vprold $3, %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf1,0x75,0x29,0x72,0xc8,0x03]
+; X64-NEXT:    vprold $3, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xa9,0x72,0xc8,0x03]
+; X64-NEXT:    vpaddd %ymm2, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xfe,0xc2]
+; X64-NEXT:    vpaddd %ymm0, %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xf5,0xfe,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <2 x double> @llvm.x86.avx512.maskz.vfmadd.pd.128(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3)
-  ret <2 x double> %res
+  %1 = call <8 x i32> @llvm.x86.avx512.prol.d.256(<8 x i32> %x0, i32 3)
+  %2 = bitcast i8 %x3 to <8 x i1>
+  %3 = select <8 x i1> %2, <8 x i32> %1, <8 x i32> %x2
+  %4 = call <8 x i32> @llvm.x86.avx512.prol.d.256(<8 x i32> %x0, i32 3)
+  %5 = bitcast i8 %x3 to <8 x i1>
+  %6 = select <8 x i1> %5, <8 x i32> %4, <8 x i32> zeroinitializer
+  %7 = call <8 x i32> @llvm.x86.avx512.prol.d.256(<8 x i32> %x0, i32 3)
+  %res3 = add <8 x i32> %3, %6
+  %res4 = add <8 x i32> %res3, %7
+  ret <8 x i32> %res4
 }
 
-declare <4 x double> @llvm.x86.avx512.mask3.vfmadd.pd.256(<4 x double>, <4 x double>, <4 x double>, i8)
+declare <2 x i64> @llvm.x86.avx512.prol.q.128(<2 x i64>, i32)
 
-define <4 x double>@test_int_x86_avx512_mask3_vfmadd_pd_256(<4 x double> %x0, <4 x double> %x1, <4 x double> %x2, i8 %x3) {
-; X86-LABEL: test_int_x86_avx512_mask3_vfmadd_pd_256:
+define <2 x i64>@test_int_x86_avx512_prol_q_128(<2 x i64> %x0, i32 %x1, <2 x i64> %x2, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_prol_q_128:
 ; X86:       # %bb.0:
-; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    vprolq $3, %xmm0, %xmm2 # encoding: [0x62,0xf1,0xed,0x08,0x72,0xc8,0x03]
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x08]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
-; X86-NEXT:    vfmadd231pd %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x29,0xb8,0xd1]
-; X86-NEXT:    # ymm2 = (ymm0 * ymm1) + ymm2
-; X86-NEXT:    vmovapd %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x28,0xc2]
+; X86-NEXT:    vprolq $3, %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf1,0xf5,0x09,0x72,0xc8,0x03]
+; X86-NEXT:    vprolq $3, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0xfd,0x89,0x72,0xc8,0x03]
+; X86-NEXT:    vpaddq %xmm2, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xd4,0xc2]
+; X86-NEXT:    vpaddq %xmm0, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0xd4,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
-; X64-LABEL: test_int_x86_avx512_mask3_vfmadd_pd_256:
+; X64-LABEL: test_int_x86_avx512_prol_q_128:
 ; X64:       # %bb.0:
-; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
-; X64-NEXT:    vfmadd231pd %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x29,0xb8,0xd1]
-; X64-NEXT:    # ymm2 = (ymm0 * ymm1) + ymm2
-; X64-NEXT:    vmovapd %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x28,0xc2]
+; X64-NEXT:    vprolq $3, %xmm0, %xmm2 # encoding: [0x62,0xf1,0xed,0x08,0x72,0xc8,0x03]
+; X64-NEXT:    kmovw %esi, %k1 # encoding: [0xc5,0xf8,0x92,0xce]
+; X64-NEXT:    vprolq $3, %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf1,0xf5,0x09,0x72,0xc8,0x03]
+; X64-NEXT:    vprolq $3, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0xfd,0x89,0x72,0xc8,0x03]
+; X64-NEXT:    vpaddq %xmm2, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xd4,0xc2]
+; X64-NEXT:    vpaddq %xmm0, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0xd4,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <4 x double> @llvm.x86.avx512.mask3.vfmadd.pd.256(<4 x double> %x0, <4 x double> %x1, <4 x double> %x2, i8 %x3)
-  ret <4 x double> %res
+  %1 = call <2 x i64> @llvm.x86.avx512.prol.q.128(<2 x i64> %x0, i32 3)
+  %2 = bitcast i8 %x3 to <8 x i1>
+  %extract1 = shufflevector <8 x i1> %2, <8 x i1> %2, <2 x i32> <i32 0, i32 1>
+  %3 = select <2 x i1> %extract1, <2 x i64> %1, <2 x i64> %x2
+  %4 = call <2 x i64> @llvm.x86.avx512.prol.q.128(<2 x i64> %x0, i32 3)
+  %5 = bitcast i8 %x3 to <8 x i1>
+  %extract = shufflevector <8 x i1> %5, <8 x i1> %5, <2 x i32> <i32 0, i32 1>
+  %6 = select <2 x i1> %extract, <2 x i64> %4, <2 x i64> zeroinitializer
+  %7 = call <2 x i64> @llvm.x86.avx512.prol.q.128(<2 x i64> %x0, i32 3)
+  %res3 = add <2 x i64> %3, %6
+  %res4 = add <2 x i64> %res3, %7
+  ret <2 x i64> %res4
 }
 
-declare <4 x double> @llvm.x86.avx512.maskz.vfmadd.pd.256(<4 x double>, <4 x double>, <4 x double>, i8)
+declare <4 x i64> @llvm.x86.avx512.prol.q.256(<4 x i64>, i32)
 
-define <4 x double>@test_int_x86_avx512_maskz_vfmadd_pd_256(<4 x double> %x0, <4 x double> %x1, <4 x double> %x2, i8 %x3) {
-; X86-LABEL: test_int_x86_avx512_maskz_vfmadd_pd_256:
+define <4 x i64>@test_int_x86_avx512_prol_q_256(<4 x i64> %x0, i32 %x1, <4 x i64> %x2, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_prol_q_256:
 ; X86:       # %bb.0:
-; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    vprolq $3, %ymm0, %ymm2 # encoding: [0x62,0xf1,0xed,0x28,0x72,0xc8,0x03]
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x08]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
-; X86-NEXT:    vfmadd213pd %ymm2, %ymm1, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xa9,0xa8,0xc2]
-; X86-NEXT:    # ymm0 = (ymm1 * ymm0) + ymm2
+; X86-NEXT:    vprolq $3, %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf1,0xf5,0x29,0x72,0xc8,0x03]
+; X86-NEXT:    vprolq $3, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0xfd,0xa9,0x72,0xc8,0x03]
+; X86-NEXT:    vpaddq %ymm2, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xd4,0xc2]
+; X86-NEXT:    vpaddq %ymm0, %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xf5,0xd4,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
-; X64-LABEL: test_int_x86_avx512_maskz_vfmadd_pd_256:
+; X64-LABEL: test_int_x86_avx512_prol_q_256:
 ; X64:       # %bb.0:
-; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
-; X64-NEXT:    vfmadd213pd %ymm2, %ymm1, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xa9,0xa8,0xc2]
-; X64-NEXT:    # ymm0 = (ymm1 * ymm0) + ymm2
+; X64-NEXT:    vprolq $3, %ymm0, %ymm2 # encoding: [0x62,0xf1,0xed,0x28,0x72,0xc8,0x03]
+; X64-NEXT:    kmovw %esi, %k1 # encoding: [0xc5,0xf8,0x92,0xce]
+; X64-NEXT:    vprolq $3, %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf1,0xf5,0x29,0x72,0xc8,0x03]
+; X64-NEXT:    vprolq $3, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0xfd,0xa9,0x72,0xc8,0x03]
+; X64-NEXT:    vpaddq %ymm2, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xd4,0xc2]
+; X64-NEXT:    vpaddq %ymm0, %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xf5,0xd4,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <4 x double> @llvm.x86.avx512.maskz.vfmadd.pd.256(<4 x double> %x0, <4 x double> %x1, <4 x double> %x2, i8 %x3)
-  ret <4 x double> %res
+  %1 = call <4 x i64> @llvm.x86.avx512.prol.q.256(<4 x i64> %x0, i32 3)
+  %2 = bitcast i8 %x3 to <8 x i1>
+  %extract1 = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %3 = select <4 x i1> %extract1, <4 x i64> %1, <4 x i64> %x2
+  %4 = call <4 x i64> @llvm.x86.avx512.prol.q.256(<4 x i64> %x0, i32 3)
+  %5 = bitcast i8 %x3 to <8 x i1>
+  %extract = shufflevector <8 x i1> %5, <8 x i1> %5, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %6 = select <4 x i1> %extract, <4 x i64> %4, <4 x i64> zeroinitializer
+  %7 = call <4 x i64> @llvm.x86.avx512.prol.q.256(<4 x i64> %x0, i32 3)
+  %res3 = add <4 x i64> %3, %6
+  %res4 = add <4 x i64> %res3, %7
+  ret <4 x i64> %res4
 }
 
-declare <4 x float> @llvm.x86.avx512.mask3.vfmadd.ps.128(<4 x float>, <4 x float>, <4 x float>, i8)
+declare <4 x i32> @llvm.x86.avx512.prolv.d.128(<4 x i32>, <4 x i32>)
 
-define <4 x float>@test_int_x86_avx512_mask3_vfmadd_ps_128(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3) {
-; X86-LABEL: test_int_x86_avx512_mask3_vfmadd_ps_128:
+define <4 x i32>@test_int_x86_avx512_prolv_d_128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_prolv_d_128:
 ; X86:       # %bb.0:
+; X86-NEXT:    vprolvd %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf2,0x7d,0x08,0x15,0xd9]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
-; X86-NEXT:    vfmadd231ps %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf2,0x7d,0x09,0xb8,0xd1]
-; X86-NEXT:    # xmm2 = (xmm0 * xmm1) + xmm2
-; X86-NEXT:    vmovaps %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf8,0x28,0xc2]
+; X86-NEXT:    vprolvd %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf2,0x7d,0x09,0x15,0xd1]
+; X86-NEXT:    vprolvd %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0x89,0x15,0xc1]
+; X86-NEXT:    vpaddd %xmm3, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfe,0xc3]
+; X86-NEXT:    vpaddd %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfe,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
-; X64-LABEL: test_int_x86_avx512_mask3_vfmadd_ps_128:
+; X64-LABEL: test_int_x86_avx512_prolv_d_128:
 ; X64:       # %bb.0:
+; X64-NEXT:    vprolvd %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf2,0x7d,0x08,0x15,0xd9]
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
-; X64-NEXT:    vfmadd231ps %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf2,0x7d,0x09,0xb8,0xd1]
-; X64-NEXT:    # xmm2 = (xmm0 * xmm1) + xmm2
-; X64-NEXT:    vmovaps %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf8,0x28,0xc2]
+; X64-NEXT:    vprolvd %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf2,0x7d,0x09,0x15,0xd1]
+; X64-NEXT:    vprolvd %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0x89,0x15,0xc1]
+; X64-NEXT:    vpaddd %xmm3, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfe,0xc3]
+; X64-NEXT:    vpaddd %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfe,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <4 x float> @llvm.x86.avx512.mask3.vfmadd.ps.128(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3)
-  ret <4 x float> %res
+  %1 = call <4 x i32> @llvm.x86.avx512.prolv.d.128(<4 x i32> %x0, <4 x i32> %x1)
+  %2 = bitcast i8 %x3 to <8 x i1>
+  %extract1 = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %3 = select <4 x i1> %extract1, <4 x i32> %1, <4 x i32> %x2
+  %4 = call <4 x i32> @llvm.x86.avx512.prolv.d.128(<4 x i32> %x0, <4 x i32> %x1)
+  %5 = bitcast i8 %x3 to <8 x i1>
+  %extract = shufflevector <8 x i1> %5, <8 x i1> %5, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %6 = select <4 x i1> %extract, <4 x i32> %4, <4 x i32> zeroinitializer
+  %7 = call <4 x i32> @llvm.x86.avx512.prolv.d.128(<4 x i32> %x0, <4 x i32> %x1)
+  %res3 = add <4 x i32> %3, %6
+  %res4 = add <4 x i32> %res3, %7
+  ret <4 x i32> %res4
 }
 
-declare <4 x float> @llvm.x86.avx512.maskz.vfmadd.ps.128(<4 x float>, <4 x float>, <4 x float>, i8)
+declare <8 x i32> @llvm.x86.avx512.prolv.d.256(<8 x i32>, <8 x i32>)
 
-define <4 x float>@test_int_x86_avx512_maskz_vfmadd_ps_128(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3) {
-; X86-LABEL: test_int_x86_avx512_maskz_vfmadd_ps_128:
+define <8 x i32>@test_int_x86_avx512_prolv_d_256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_prolv_d_256:
 ; X86:       # %bb.0:
+; X86-NEXT:    vprolvd %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf2,0x7d,0x28,0x15,0xd9]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
-; X86-NEXT:    vfmadd213ps %xmm2, %xmm1, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0x89,0xa8,0xc2]
-; X86-NEXT:    # xmm0 = (xmm1 * xmm0) + xmm2
+; X86-NEXT:    vprolvd %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf2,0x7d,0x29,0x15,0xd1]
+; X86-NEXT:    vprolvd %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0xa9,0x15,0xc1]
+; X86-NEXT:    vpaddd %ymm3, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xfe,0xc3]
+; X86-NEXT:    vpaddd %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfe,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
-; X64-LABEL: test_int_x86_avx512_maskz_vfmadd_ps_128:
+; X64-LABEL: test_int_x86_avx512_prolv_d_256:
 ; X64:       # %bb.0:
+; X64-NEXT:    vprolvd %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf2,0x7d,0x28,0x15,0xd9]
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
-; X64-NEXT:    vfmadd213ps %xmm2, %xmm1, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0x89,0xa8,0xc2]
-; X64-NEXT:    # xmm0 = (xmm1 * xmm0) + xmm2
+; X64-NEXT:    vprolvd %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf2,0x7d,0x29,0x15,0xd1]
+; X64-NEXT:    vprolvd %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0xa9,0x15,0xc1]
+; X64-NEXT:    vpaddd %ymm3, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xfe,0xc3]
+; X64-NEXT:    vpaddd %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfe,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <4 x float> @llvm.x86.avx512.maskz.vfmadd.ps.128(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3)
-  ret <4 x float> %res
+  %1 = call <8 x i32> @llvm.x86.avx512.prolv.d.256(<8 x i32> %x0, <8 x i32> %x1)
+  %2 = bitcast i8 %x3 to <8 x i1>
+  %3 = select <8 x i1> %2, <8 x i32> %1, <8 x i32> %x2
+  %4 = call <8 x i32> @llvm.x86.avx512.prolv.d.256(<8 x i32> %x0, <8 x i32> %x1)
+  %5 = bitcast i8 %x3 to <8 x i1>
+  %6 = select <8 x i1> %5, <8 x i32> %4, <8 x i32> zeroinitializer
+  %7 = call <8 x i32> @llvm.x86.avx512.prolv.d.256(<8 x i32> %x0, <8 x i32> %x1)
+  %res3 = add <8 x i32> %3, %6
+  %res4 = add <8 x i32> %res3, %7
+  ret <8 x i32> %res4
 }
 
-declare <8 x float> @llvm.x86.avx512.mask3.vfmadd.ps.256(<8 x float>, <8 x float>, <8 x float>, i8)
+declare <2 x i64> @llvm.x86.avx512.prolv.q.128(<2 x i64>, <2 x i64>)
 
-define <8 x float>@test_int_x86_avx512_mask3_vfmadd_ps_256(<8 x float> %x0, <8 x float> %x1, <8 x float> %x2, i8 %x3) {
-; X86-LABEL: test_int_x86_avx512_mask3_vfmadd_ps_256:
+define <2 x i64>@test_int_x86_avx512_prolv_q_128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_prolv_q_128:
 ; X86:       # %bb.0:
+; X86-NEXT:    vprolvq %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf2,0xfd,0x08,0x15,0xd9]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
-; X86-NEXT:    vfmadd231ps %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf2,0x7d,0x29,0xb8,0xd1]
-; X86-NEXT:    # ymm2 = (ymm0 * ymm1) + ymm2
-; X86-NEXT:    vmovaps %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfc,0x28,0xc2]
+; X86-NEXT:    vprolvq %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x09,0x15,0xd1]
+; X86-NEXT:    vprolvq %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0x89,0x15,0xc1]
+; X86-NEXT:    vpaddq %xmm3, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xd4,0xc3]
+; X86-NEXT:    vpaddq %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xd4,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
-; X64-LABEL: test_int_x86_avx512_mask3_vfmadd_ps_256:
+; X64-LABEL: test_int_x86_avx512_prolv_q_128:
 ; X64:       # %bb.0:
+; X64-NEXT:    vprolvq %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf2,0xfd,0x08,0x15,0xd9]
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
-; X64-NEXT:    vfmadd231ps %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf2,0x7d,0x29,0xb8,0xd1]
-; X64-NEXT:    # ymm2 = (ymm0 * ymm1) + ymm2
-; X64-NEXT:    vmovaps %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfc,0x28,0xc2]
+; X64-NEXT:    vprolvq %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x09,0x15,0xd1]
+; X64-NEXT:    vprolvq %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0x89,0x15,0xc1]
+; X64-NEXT:    vpaddq %xmm3, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xd4,0xc3]
+; X64-NEXT:    vpaddq %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xd4,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <8 x float> @llvm.x86.avx512.mask3.vfmadd.ps.256(<8 x float> %x0, <8 x float> %x1, <8 x float> %x2, i8 %x3)
-  ret <8 x float> %res
+  %1 = call <2 x i64> @llvm.x86.avx512.prolv.q.128(<2 x i64> %x0, <2 x i64> %x1)
+  %2 = bitcast i8 %x3 to <8 x i1>
+  %extract1 = shufflevector <8 x i1> %2, <8 x i1> %2, <2 x i32> <i32 0, i32 1>
+  %3 = select <2 x i1> %extract1, <2 x i64> %1, <2 x i64> %x2
+  %4 = call <2 x i64> @llvm.x86.avx512.prolv.q.128(<2 x i64> %x0, <2 x i64> %x1)
+  %5 = bitcast i8 %x3 to <8 x i1>
+  %extract = shufflevector <8 x i1> %5, <8 x i1> %5, <2 x i32> <i32 0, i32 1>
+  %6 = select <2 x i1> %extract, <2 x i64> %4, <2 x i64> zeroinitializer
+  %7 = call <2 x i64> @llvm.x86.avx512.prolv.q.128(<2 x i64> %x0, <2 x i64> %x1)
+  %res3 = add <2 x i64> %3, %6
+  %res4 = add <2 x i64> %res3, %7
+  ret <2 x i64> %res4
 }
 
-declare <8 x float> @llvm.x86.avx512.maskz.vfmadd.ps.256(<8 x float>, <8 x float>, <8 x float>, i8)
+declare <4 x i64> @llvm.x86.avx512.prolv.q.256(<4 x i64>, <4 x i64>)
 
-define <8 x float>@test_int_x86_avx512_maskz_vfmadd_ps_256(<8 x float> %x0, <8 x float> %x1, <8 x float> %x2, i8 %x3) {
-; X86-LABEL: test_int_x86_avx512_maskz_vfmadd_ps_256:
+define <4 x i64>@test_int_x86_avx512_prolv_q_256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_prolv_q_256:
 ; X86:       # %bb.0:
+; X86-NEXT:    vprolvq %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf2,0xfd,0x28,0x15,0xd9]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
-; X86-NEXT:    vfmadd213ps %ymm2, %ymm1, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0xa9,0xa8,0xc2]
-; X86-NEXT:    # ymm0 = (ymm1 * ymm0) + ymm2
+; X86-NEXT:    vprolvq %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x29,0x15,0xd1]
+; X86-NEXT:    vprolvq %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xa9,0x15,0xc1]
+; X86-NEXT:    vpaddq %ymm3, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xd4,0xc3]
+; X86-NEXT:    vpaddq %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xd4,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
-; X64-LABEL: test_int_x86_avx512_maskz_vfmadd_ps_256:
+; X64-LABEL: test_int_x86_avx512_prolv_q_256:
 ; X64:       # %bb.0:
+; X64-NEXT:    vprolvq %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf2,0xfd,0x28,0x15,0xd9]
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
-; X64-NEXT:    vfmadd213ps %ymm2, %ymm1, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0xa9,0xa8,0xc2]
-; X64-NEXT:    # ymm0 = (ymm1 * ymm0) + ymm2
+; X64-NEXT:    vprolvq %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x29,0x15,0xd1]
+; X64-NEXT:    vprolvq %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xa9,0x15,0xc1]
+; X64-NEXT:    vpaddq %ymm3, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xd4,0xc3]
+; X64-NEXT:    vpaddq %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xd4,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <8 x float> @llvm.x86.avx512.maskz.vfmadd.ps.256(<8 x float> %x0, <8 x float> %x1, <8 x float> %x2, i8 %x3)
-  ret <8 x float> %res
+  %1 = call <4 x i64> @llvm.x86.avx512.prolv.q.256(<4 x i64> %x0, <4 x i64> %x1)
+  %2 = bitcast i8 %x3 to <8 x i1>
+  %extract1 = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %3 = select <4 x i1> %extract1, <4 x i64> %1, <4 x i64> %x2
+  %4 = call <4 x i64> @llvm.x86.avx512.prolv.q.256(<4 x i64> %x0, <4 x i64> %x1)
+  %5 = bitcast i8 %x3 to <8 x i1>
+  %extract = shufflevector <8 x i1> %5, <8 x i1> %5, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %6 = select <4 x i1> %extract, <4 x i64> %4, <4 x i64> zeroinitializer
+  %7 = call <4 x i64> @llvm.x86.avx512.prolv.q.256(<4 x i64> %x0, <4 x i64> %x1)
+  %res3 = add <4 x i64> %3, %6
+  %res4 = add <4 x i64> %res3, %7
+  ret <4 x i64> %res4
 }
 
+declare <4 x i32> @llvm.x86.avx512.pror.d.128(<4 x i32>, i32)
 
-declare <2 x double> @llvm.x86.avx512.mask3.vfmsub.pd.128(<2 x double>, <2 x double>, <2 x double>, i8)
-
-define <2 x double>@test_int_x86_avx512_mask3_vfmsub_pd_128(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3) {
-; X86-LABEL: test_int_x86_avx512_mask3_vfmsub_pd_128:
+define <4 x i32>@test_int_x86_avx512_pror_d_128(<4 x i32> %x0, i32 %x1, <4 x i32> %x2, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_pror_d_128:
 ; X86:       # %bb.0:
-; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    vprord $3, %xmm0, %xmm2 # encoding: [0x62,0xf1,0x6d,0x08,0x72,0xc0,0x03]
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x08]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
-; X86-NEXT:    vfmsub231pd %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x09,0xba,0xd1]
-; X86-NEXT:    # xmm2 = (xmm0 * xmm1) - xmm2
-; X86-NEXT:    vmovapd %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x28,0xc2]
+; X86-NEXT:    vprord $3, %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf1,0x75,0x09,0x72,0xc0,0x03]
+; X86-NEXT:    vprord $3, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0x89,0x72,0xc0,0x03]
+; X86-NEXT:    vpaddd %xmm2, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfe,0xc2]
+; X86-NEXT:    vpaddd %xmm0, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0xfe,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
-; X64-LABEL: test_int_x86_avx512_mask3_vfmsub_pd_128:
+; X64-LABEL: test_int_x86_avx512_pror_d_128:
 ; X64:       # %bb.0:
-; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
-; X64-NEXT:    vfmsub231pd %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x09,0xba,0xd1]
-; X64-NEXT:    # xmm2 = (xmm0 * xmm1) - xmm2
-; X64-NEXT:    vmovapd %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x28,0xc2]
+; X64-NEXT:    vprord $3, %xmm0, %xmm2 # encoding: [0x62,0xf1,0x6d,0x08,0x72,0xc0,0x03]
+; X64-NEXT:    kmovw %esi, %k1 # encoding: [0xc5,0xf8,0x92,0xce]
+; X64-NEXT:    vprord $3, %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf1,0x75,0x09,0x72,0xc0,0x03]
+; X64-NEXT:    vprord $3, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0x89,0x72,0xc0,0x03]
+; X64-NEXT:    vpaddd %xmm2, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfe,0xc2]
+; X64-NEXT:    vpaddd %xmm0, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0xfe,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <2 x double> @llvm.x86.avx512.mask3.vfmsub.pd.128(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3)
-  ret <2 x double> %res
+  %1 = call <4 x i32> @llvm.x86.avx512.pror.d.128(<4 x i32> %x0, i32 3)
+  %2 = bitcast i8 %x3 to <8 x i1>
+  %extract1 = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %3 = select <4 x i1> %extract1, <4 x i32> %1, <4 x i32> %x2
+  %4 = call <4 x i32> @llvm.x86.avx512.pror.d.128(<4 x i32> %x0, i32 3)
+  %5 = bitcast i8 %x3 to <8 x i1>
+  %extract = shufflevector <8 x i1> %5, <8 x i1> %5, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %6 = select <4 x i1> %extract, <4 x i32> %4, <4 x i32> zeroinitializer
+  %7 = call <4 x i32> @llvm.x86.avx512.pror.d.128(<4 x i32> %x0, i32 3)
+  %res3 = add <4 x i32> %3, %6
+  %res4 = add <4 x i32> %res3, %7
+  ret <4 x i32> %res4
 }
 
+declare <8 x i32> @llvm.x86.avx512.pror.d.256(<8 x i32>, i32)
 
-declare <4 x double> @llvm.x86.avx512.mask3.vfmsub.pd.256(<4 x double>, <4 x double>, <4 x double>, i8)
-
-define <4 x double>@test_int_x86_avx512_mask3_vfmsub_pd_256(<4 x double> %x0, <4 x double> %x1, <4 x double> %x2, i8 %x3) {
-; X86-LABEL: test_int_x86_avx512_mask3_vfmsub_pd_256:
+define <8 x i32>@test_int_x86_avx512_pror_d_256(<8 x i32> %x0, i32 %x1, <8 x i32> %x2, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_pror_d_256:
 ; X86:       # %bb.0:
-; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    vprord $3, %ymm0, %ymm2 # encoding: [0x62,0xf1,0x6d,0x28,0x72,0xc0,0x03]
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x08]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
-; X86-NEXT:    vfmsub231pd %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x29,0xba,0xd1]
-; X86-NEXT:    # ymm2 = (ymm0 * ymm1) - ymm2
-; X86-NEXT:    vmovapd %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x28,0xc2]
+; X86-NEXT:    vprord $3, %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf1,0x75,0x29,0x72,0xc0,0x03]
+; X86-NEXT:    vprord $3, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xa9,0x72,0xc0,0x03]
+; X86-NEXT:    vpaddd %ymm2, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xfe,0xc2]
+; X86-NEXT:    vpaddd %ymm0, %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xf5,0xfe,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
-; X64-LABEL: test_int_x86_avx512_mask3_vfmsub_pd_256:
+; X64-LABEL: test_int_x86_avx512_pror_d_256:
 ; X64:       # %bb.0:
-; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
-; X64-NEXT:    vfmsub231pd %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x29,0xba,0xd1]
-; X64-NEXT:    # ymm2 = (ymm0 * ymm1) - ymm2
-; X64-NEXT:    vmovapd %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x28,0xc2]
+; X64-NEXT:    vprord $3, %ymm0, %ymm2 # encoding: [0x62,0xf1,0x6d,0x28,0x72,0xc0,0x03]
+; X64-NEXT:    kmovw %esi, %k1 # encoding: [0xc5,0xf8,0x92,0xce]
+; X64-NEXT:    vprord $3, %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf1,0x75,0x29,0x72,0xc0,0x03]
+; X64-NEXT:    vprord $3, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xa9,0x72,0xc0,0x03]
+; X64-NEXT:    vpaddd %ymm2, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xfe,0xc2]
+; X64-NEXT:    vpaddd %ymm0, %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xf5,0xfe,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <4 x double> @llvm.x86.avx512.mask3.vfmsub.pd.256(<4 x double> %x0, <4 x double> %x1, <4 x double> %x2, i8 %x3)
-  ret <4 x double> %res
+  %1 = call <8 x i32> @llvm.x86.avx512.pror.d.256(<8 x i32> %x0, i32 3)
+  %2 = bitcast i8 %x3 to <8 x i1>
+  %3 = select <8 x i1> %2, <8 x i32> %1, <8 x i32> %x2
+  %4 = call <8 x i32> @llvm.x86.avx512.pror.d.256(<8 x i32> %x0, i32 3)
+  %5 = bitcast i8 %x3 to <8 x i1>
+  %6 = select <8 x i1> %5, <8 x i32> %4, <8 x i32> zeroinitializer
+  %7 = call <8 x i32> @llvm.x86.avx512.pror.d.256(<8 x i32> %x0, i32 3)
+  %res3 = add <8 x i32> %3, %6
+  %res4 = add <8 x i32> %res3, %7
+  ret <8 x i32> %res4
 }
 
-declare <4 x float> @llvm.x86.avx512.mask3.vfmsub.ps.128(<4 x float>, <4 x float>, <4 x float>, i8)
+declare <2 x i64> @llvm.x86.avx512.pror.q.128(<2 x i64>, i32)
 
-define <4 x float>@test_int_x86_avx512_mask3_vfmsub_ps_128(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3) {
-; X86-LABEL: test_int_x86_avx512_mask3_vfmsub_ps_128:
+define <2 x i64>@test_int_x86_avx512_pror_q_128(<2 x i64> %x0, i32 %x1, <2 x i64> %x2, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_pror_q_128:
 ; X86:       # %bb.0:
-; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    vprorq $3, %xmm0, %xmm2 # encoding: [0x62,0xf1,0xed,0x08,0x72,0xc0,0x03]
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x08]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
-; X86-NEXT:    vfmsub231ps %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf2,0x7d,0x09,0xba,0xd1]
-; X86-NEXT:    # xmm2 = (xmm0 * xmm1) - xmm2
-; X86-NEXT:    vmovaps %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf8,0x28,0xc2]
+; X86-NEXT:    vprorq $3, %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf1,0xf5,0x09,0x72,0xc0,0x03]
+; X86-NEXT:    vprorq $3, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0xfd,0x89,0x72,0xc0,0x03]
+; X86-NEXT:    vpaddq %xmm2, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xd4,0xc2]
+; X86-NEXT:    vpaddq %xmm0, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0xd4,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
-; X64-LABEL: test_int_x86_avx512_mask3_vfmsub_ps_128:
+; X64-LABEL: test_int_x86_avx512_pror_q_128:
 ; X64:       # %bb.0:
-; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
-; X64-NEXT:    vfmsub231ps %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf2,0x7d,0x09,0xba,0xd1]
-; X64-NEXT:    # xmm2 = (xmm0 * xmm1) - xmm2
-; X64-NEXT:    vmovaps %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf8,0x28,0xc2]
+; X64-NEXT:    vprorq $3, %xmm0, %xmm2 # encoding: [0x62,0xf1,0xed,0x08,0x72,0xc0,0x03]
+; X64-NEXT:    kmovw %esi, %k1 # encoding: [0xc5,0xf8,0x92,0xce]
+; X64-NEXT:    vprorq $3, %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf1,0xf5,0x09,0x72,0xc0,0x03]
+; X64-NEXT:    vprorq $3, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0xfd,0x89,0x72,0xc0,0x03]
+; X64-NEXT:    vpaddq %xmm2, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xd4,0xc2]
+; X64-NEXT:    vpaddq %xmm0, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0xd4,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <4 x float> @llvm.x86.avx512.mask3.vfmsub.ps.128(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3)
-  ret <4 x float> %res
+  %1 = call <2 x i64> @llvm.x86.avx512.pror.q.128(<2 x i64> %x0, i32 3)
+  %2 = bitcast i8 %x3 to <8 x i1>
+  %extract1 = shufflevector <8 x i1> %2, <8 x i1> %2, <2 x i32> <i32 0, i32 1>
+  %3 = select <2 x i1> %extract1, <2 x i64> %1, <2 x i64> %x2
+  %4 = call <2 x i64> @llvm.x86.avx512.pror.q.128(<2 x i64> %x0, i32 3)
+  %5 = bitcast i8 %x3 to <8 x i1>
+  %extract = shufflevector <8 x i1> %5, <8 x i1> %5, <2 x i32> <i32 0, i32 1>
+  %6 = select <2 x i1> %extract, <2 x i64> %4, <2 x i64> zeroinitializer
+  %7 = call <2 x i64> @llvm.x86.avx512.pror.q.128(<2 x i64> %x0, i32 3)
+  %res3 = add <2 x i64> %3, %6
+  %res4 = add <2 x i64> %res3, %7
+  ret <2 x i64> %res4
 }
 
-declare <8 x float> @llvm.x86.avx512.mask3.vfmsub.ps.256(<8 x float>, <8 x float>, <8 x float>, i8)
+declare <4 x i64> @llvm.x86.avx512.pror.q.256(<4 x i64>, i32)
 
-define <8 x float>@test_int_x86_avx512_mask3_vfmsub_ps_256(<8 x float> %x0, <8 x float> %x1, <8 x float> %x2, i8 %x3) {
-; X86-LABEL: test_int_x86_avx512_mask3_vfmsub_ps_256:
+define <4 x i64>@test_int_x86_avx512_pror_q_256(<4 x i64> %x0, i32 %x1, <4 x i64> %x2, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_pror_q_256:
 ; X86:       # %bb.0:
-; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    vprorq $3, %ymm0, %ymm2 # encoding: [0x62,0xf1,0xed,0x28,0x72,0xc0,0x03]
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x08]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
-; X86-NEXT:    vfmsub231ps %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf2,0x7d,0x29,0xba,0xd1]
-; X86-NEXT:    # ymm2 = (ymm0 * ymm1) - ymm2
-; X86-NEXT:    vmovaps %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfc,0x28,0xc2]
+; X86-NEXT:    vprorq $3, %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf1,0xf5,0x29,0x72,0xc0,0x03]
+; X86-NEXT:    vprorq $3, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0xfd,0xa9,0x72,0xc0,0x03]
+; X86-NEXT:    vpaddq %ymm2, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xd4,0xc2]
+; X86-NEXT:    vpaddq %ymm0, %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xf5,0xd4,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
-; X64-LABEL: test_int_x86_avx512_mask3_vfmsub_ps_256:
+; X64-LABEL: test_int_x86_avx512_pror_q_256:
 ; X64:       # %bb.0:
-; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
-; X64-NEXT:    vfmsub231ps %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf2,0x7d,0x29,0xba,0xd1]
-; X64-NEXT:    # ymm2 = (ymm0 * ymm1) - ymm2
-; X64-NEXT:    vmovaps %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfc,0x28,0xc2]
+; X64-NEXT:    vprorq $3, %ymm0, %ymm2 # encoding: [0x62,0xf1,0xed,0x28,0x72,0xc0,0x03]
+; X64-NEXT:    kmovw %esi, %k1 # encoding: [0xc5,0xf8,0x92,0xce]
+; X64-NEXT:    vprorq $3, %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf1,0xf5,0x29,0x72,0xc0,0x03]
+; X64-NEXT:    vprorq $3, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0xfd,0xa9,0x72,0xc0,0x03]
+; X64-NEXT:    vpaddq %ymm2, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xd4,0xc2]
+; X64-NEXT:    vpaddq %ymm0, %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xf5,0xd4,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <8 x float> @llvm.x86.avx512.mask3.vfmsub.ps.256(<8 x float> %x0, <8 x float> %x1, <8 x float> %x2, i8 %x3)
-  ret <8 x float> %res
+  %1 = call <4 x i64> @llvm.x86.avx512.pror.q.256(<4 x i64> %x0, i32 3)
+  %2 = bitcast i8 %x3 to <8 x i1>
+  %extract1 = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %3 = select <4 x i1> %extract1, <4 x i64> %1, <4 x i64> %x2
+  %4 = call <4 x i64> @llvm.x86.avx512.pror.q.256(<4 x i64> %x0, i32 3)
+  %5 = bitcast i8 %x3 to <8 x i1>
+  %extract = shufflevector <8 x i1> %5, <8 x i1> %5, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %6 = select <4 x i1> %extract, <4 x i64> %4, <4 x i64> zeroinitializer
+  %7 = call <4 x i64> @llvm.x86.avx512.pror.q.256(<4 x i64> %x0, i32 3)
+  %res3 = add <4 x i64> %3, %6
+  %res4 = add <4 x i64> %res3, %7
+  ret <4 x i64> %res4
 }
 
-declare <8 x float> @llvm.x86.avx512.mask.vfnmadd.ps.256(<8 x float>, <8 x float>, <8 x float>, i8) nounwind readnone
+declare <8 x float> @llvm.x86.avx512.mask.vfmadd.ps.256(<8 x float>, <8 x float>, <8 x float>, i8) nounwind readnone
 
-define <8 x float> @test_vfnmadd256_ps(<8 x float> %a0, <8 x float> %a1, <8 x float> %a2) {
-; CHECK-LABEL: test_vfnmadd256_ps:
+define <8 x float> @test_vfmadd256_ps(<8 x float> %a0, <8 x float> %a1, <8 x float> %a2) {
+; CHECK-LABEL: test_vfmadd256_ps:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vfnmadd213ps %ymm2, %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x75,0xac,0xc2]
-; CHECK-NEXT:    # ymm0 = -(ymm1 * ymm0) + ymm2
+; CHECK-NEXT:    vfmadd213ps %ymm2, %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x75,0xa8,0xc2]
+; CHECK-NEXT:    # ymm0 = (ymm1 * ymm0) + ymm2
 ; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
-  %res = call <8 x float> @llvm.x86.avx512.mask.vfnmadd.ps.256(<8 x float> %a0, <8 x float> %a1, <8 x float> %a2, i8 -1) nounwind
+  %res = call <8 x float> @llvm.x86.avx512.mask.vfmadd.ps.256(<8 x float> %a0, <8 x float> %a1, <8 x float> %a2, i8 -1) nounwind
   ret <8 x float> %res
 }
 
-define <8 x float> @test_mask_vfnmadd256_ps(<8 x float> %a0, <8 x float> %a1, <8 x float> %a2, i8 %mask) {
-; X86-LABEL: test_mask_vfnmadd256_ps:
+define <8 x float> @test_mask_vfmadd256_ps(<8 x float> %a0, <8 x float> %a1, <8 x float> %a2, i8 %mask) {
+; X86-LABEL: test_mask_vfmadd256_ps:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
-; X86-NEXT:    vfnmadd132ps %ymm1, %ymm2, %ymm0 {%k1} # encoding: [0x62,0xf2,0x6d,0x29,0x9c,0xc1]
-; X86-NEXT:    # ymm0 = -(ymm0 * ymm1) + ymm2
+; X86-NEXT:    vfmadd132ps %ymm1, %ymm2, %ymm0 {%k1} # encoding: [0x62,0xf2,0x6d,0x29,0x98,0xc1]
+; X86-NEXT:    # ymm0 = (ymm0 * ymm1) + ymm2
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
-; X64-LABEL: test_mask_vfnmadd256_ps:
+; X64-LABEL: test_mask_vfmadd256_ps:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
-; X64-NEXT:    vfnmadd132ps %ymm1, %ymm2, %ymm0 {%k1} # encoding: [0x62,0xf2,0x6d,0x29,0x9c,0xc1]
-; X64-NEXT:    # ymm0 = -(ymm0 * ymm1) + ymm2
+; X64-NEXT:    vfmadd132ps %ymm1, %ymm2, %ymm0 {%k1} # encoding: [0x62,0xf2,0x6d,0x29,0x98,0xc1]
+; X64-NEXT:    # ymm0 = (ymm0 * ymm1) + ymm2
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <8 x float> @llvm.x86.avx512.mask.vfnmadd.ps.256(<8 x float> %a0, <8 x float> %a1, <8 x float> %a2, i8 %mask) nounwind
+  %res = call <8 x float> @llvm.x86.avx512.mask.vfmadd.ps.256(<8 x float> %a0, <8 x float> %a1, <8 x float> %a2, i8 %mask) nounwind
   ret <8 x float> %res
 }
 
-declare <4 x float> @llvm.x86.avx512.mask.vfnmadd.ps.128(<4 x float>, <4 x float>, <4 x float>, i8) nounwind readnone
+declare <4 x float> @llvm.x86.avx512.mask.vfmadd.ps.128(<4 x float>, <4 x float>, <4 x float>, i8) nounwind readnone
 
-define <4 x float> @test_vfnmadd128_ps(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2) {
-; CHECK-LABEL: test_vfnmadd128_ps:
+define <4 x float> @test_vfmadd128_ps(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2) {
+; CHECK-LABEL: test_vfmadd128_ps:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vfnmadd213ps %xmm2, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x71,0xac,0xc2]
-; CHECK-NEXT:    # xmm0 = -(xmm1 * xmm0) + xmm2
+; CHECK-NEXT:    vfmadd213ps %xmm2, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x71,0xa8,0xc2]
+; CHECK-NEXT:    # xmm0 = (xmm1 * xmm0) + xmm2
 ; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
-  %res = call <4 x float> @llvm.x86.avx512.mask.vfnmadd.ps.128(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, i8 -1) nounwind
+  %res = call <4 x float> @llvm.x86.avx512.mask.vfmadd.ps.128(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, i8 -1) nounwind
   ret <4 x float> %res
 }
 
-define <4 x float> @test_mask_vfnmadd128_ps(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, i8 %mask) {
-; X86-LABEL: test_mask_vfnmadd128_ps:
+define <4 x float> @test_mask_vfmadd128_ps(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, i8 %mask) {
+; X86-LABEL: test_mask_vfmadd128_ps:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
-; X86-NEXT:    vfnmadd132ps %xmm1, %xmm2, %xmm0 {%k1} # encoding: [0x62,0xf2,0x6d,0x09,0x9c,0xc1]
-; X86-NEXT:    # xmm0 = -(xmm0 * xmm1) + xmm2
+; X86-NEXT:    vfmadd132ps %xmm1, %xmm2, %xmm0 {%k1} # encoding: [0x62,0xf2,0x6d,0x09,0x98,0xc1]
+; X86-NEXT:    # xmm0 = (xmm0 * xmm1) + xmm2
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
-; X64-LABEL: test_mask_vfnmadd128_ps:
+; X64-LABEL: test_mask_vfmadd128_ps:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
-; X64-NEXT:    vfnmadd132ps %xmm1, %xmm2, %xmm0 {%k1} # encoding: [0x62,0xf2,0x6d,0x09,0x9c,0xc1]
-; X64-NEXT:    # xmm0 = -(xmm0 * xmm1) + xmm2
+; X64-NEXT:    vfmadd132ps %xmm1, %xmm2, %xmm0 {%k1} # encoding: [0x62,0xf2,0x6d,0x09,0x98,0xc1]
+; X64-NEXT:    # xmm0 = (xmm0 * xmm1) + xmm2
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <4 x float> @llvm.x86.avx512.mask.vfnmadd.ps.128(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, i8 %mask) nounwind
+  %res = call <4 x float> @llvm.x86.avx512.mask.vfmadd.ps.128(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, i8 %mask) nounwind
   ret <4 x float> %res
 }
 
-declare <4 x double> @llvm.x86.avx512.mask.vfnmadd.pd.256(<4 x double>, <4 x double>, <4 x double>, i8) nounwind readnone
+declare <4 x double> @llvm.x86.avx512.mask.vfmadd.pd.256(<4 x double>, <4 x double>, <4 x double>, i8)
 
-define <4 x double> @test_vfnmadd256_pd(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2) {
-; CHECK-LABEL: test_vfnmadd256_pd:
+define <4 x double> @test_fmadd256_pd(<4 x double> %a, <4 x double> %b, <4 x double> %c) {
+; CHECK-LABEL: test_fmadd256_pd:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vfnmadd213pd %ymm2, %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0xf5,0xac,0xc2]
-; CHECK-NEXT:    # ymm0 = -(ymm1 * ymm0) + ymm2
+; CHECK-NEXT:    vfmadd213pd %ymm2, %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0xf5,0xa8,0xc2]
+; CHECK-NEXT:    # ymm0 = (ymm1 * ymm0) + ymm2
 ; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
-  %res = call <4 x double> @llvm.x86.avx512.mask.vfnmadd.pd.256(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2, i8 -1) nounwind
+  %res = call <4 x double> @llvm.x86.avx512.mask.vfmadd.pd.256(<4 x double> %a, <4 x double> %b, <4 x double> %c, i8 -1)
   ret <4 x double> %res
 }
 
-define <4 x double> @test_mask_vfnmadd256_pd(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2, i8 %mask) {
-; X86-LABEL: test_mask_vfnmadd256_pd:
+define <4 x double> @test_mask_fmadd256_pd(<4 x double> %a, <4 x double> %b, <4 x double> %c, i8 %mask) {
+; X86-LABEL: test_mask_fmadd256_pd:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
-; X86-NEXT:    vfnmadd132pd %ymm1, %ymm2, %ymm0 {%k1} # encoding: [0x62,0xf2,0xed,0x29,0x9c,0xc1]
-; X86-NEXT:    # ymm0 = -(ymm0 * ymm1) + ymm2
+; X86-NEXT:    vfmadd132pd %ymm1, %ymm2, %ymm0 {%k1} # encoding: [0x62,0xf2,0xed,0x29,0x98,0xc1]
+; X86-NEXT:    # ymm0 = (ymm0 * ymm1) + ymm2
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
-; X64-LABEL: test_mask_vfnmadd256_pd:
+; X64-LABEL: test_mask_fmadd256_pd:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
-; X64-NEXT:    vfnmadd132pd %ymm1, %ymm2, %ymm0 {%k1} # encoding: [0x62,0xf2,0xed,0x29,0x9c,0xc1]
-; X64-NEXT:    # ymm0 = -(ymm0 * ymm1) + ymm2
+; X64-NEXT:    vfmadd132pd %ymm1, %ymm2, %ymm0 {%k1} # encoding: [0x62,0xf2,0xed,0x29,0x98,0xc1]
+; X64-NEXT:    # ymm0 = (ymm0 * ymm1) + ymm2
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <4 x double> @llvm.x86.avx512.mask.vfnmadd.pd.256(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2, i8 %mask) nounwind
+  %res = call <4 x double> @llvm.x86.avx512.mask.vfmadd.pd.256(<4 x double> %a, <4 x double> %b, <4 x double> %c, i8 %mask)
   ret <4 x double> %res
 }
 
-declare <2 x double> @llvm.x86.avx512.mask.vfnmadd.pd.128(<2 x double>, <2 x double>, <2 x double>, i8) nounwind readnone
+declare <2 x double> @llvm.x86.avx512.mask.vfmadd.pd.128(<2 x double>, <2 x double>, <2 x double>, i8)
 
-define <2 x double> @test_vfnmadd128_pd(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2) {
-; CHECK-LABEL: test_vfnmadd128_pd:
+define <2 x double> @test_fmadd128_pd(<2 x double> %a, <2 x double> %b, <2 x double> %c) {
+; CHECK-LABEL: test_fmadd128_pd:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vfnmadd213pd %xmm2, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0xf1,0xac,0xc2]
-; CHECK-NEXT:    # xmm0 = -(xmm1 * xmm0) + xmm2
+; CHECK-NEXT:    vfmadd213pd %xmm2, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0xf1,0xa8,0xc2]
+; CHECK-NEXT:    # xmm0 = (xmm1 * xmm0) + xmm2
 ; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
-  %res = call <2 x double> @llvm.x86.avx512.mask.vfnmadd.pd.128(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, i8 -1) nounwind
+  %res = call <2 x double> @llvm.x86.avx512.mask.vfmadd.pd.128(<2 x double> %a, <2 x double> %b, <2 x double> %c, i8 -1)
   ret <2 x double> %res
 }
 
-define <2 x double> @test_mask_vfnmadd128_pd(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, i8 %mask) {
-; X86-LABEL: test_mask_vfnmadd128_pd:
+define <2 x double> @test_mask_fmadd128_pd(<2 x double> %a, <2 x double> %b, <2 x double> %c, i8 %mask) {
+; X86-LABEL: test_mask_fmadd128_pd:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
-; X86-NEXT:    vfnmadd132pd %xmm1, %xmm2, %xmm0 {%k1} # encoding: [0x62,0xf2,0xed,0x09,0x9c,0xc1]
-; X86-NEXT:    # xmm0 = -(xmm0 * xmm1) + xmm2
+; X86-NEXT:    vfmadd132pd %xmm1, %xmm2, %xmm0 {%k1} # encoding: [0x62,0xf2,0xed,0x09,0x98,0xc1]
+; X86-NEXT:    # xmm0 = (xmm0 * xmm1) + xmm2
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
-; X64-LABEL: test_mask_vfnmadd128_pd:
+; X64-LABEL: test_mask_fmadd128_pd:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
-; X64-NEXT:    vfnmadd132pd %xmm1, %xmm2, %xmm0 {%k1} # encoding: [0x62,0xf2,0xed,0x09,0x9c,0xc1]
-; X64-NEXT:    # xmm0 = -(xmm0 * xmm1) + xmm2
+; X64-NEXT:    vfmadd132pd %xmm1, %xmm2, %xmm0 {%k1} # encoding: [0x62,0xf2,0xed,0x09,0x98,0xc1]
+; X64-NEXT:    # xmm0 = (xmm0 * xmm1) + xmm2
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <2 x double> @llvm.x86.avx512.mask.vfnmadd.pd.128(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, i8 %mask) nounwind
+  %res = call <2 x double> @llvm.x86.avx512.mask.vfmadd.pd.128(<2 x double> %a, <2 x double> %b, <2 x double> %c, i8 %mask)
   ret <2 x double> %res
 }
 
-declare <8 x float> @llvm.x86.avx512.mask.vfnmsub.ps.256(<8 x float>, <8 x float>, <8 x float>, i8) nounwind readnone
-
-define <8 x float> @test_vfnmsub256_ps(<8 x float> %a0, <8 x float> %a1, <8 x float> %a2) {
-; CHECK-LABEL: test_vfnmsub256_ps:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vfnmsub213ps %ymm2, %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x75,0xae,0xc2]
-; CHECK-NEXT:    # ymm0 = -(ymm1 * ymm0) - ymm2
-; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
-  %res = call <8 x float> @llvm.x86.avx512.mask.vfnmsub.ps.256(<8 x float> %a0, <8 x float> %a1, <8 x float> %a2, i8 -1) nounwind
-  ret <8 x float> %res
-}
+declare <2 x double> @llvm.x86.avx512.mask3.vfmadd.pd.128(<2 x double>, <2 x double>, <2 x double>, i8)
 
-define <8 x float> @test_mask_vfnmsub256_ps(<8 x float> %a0, <8 x float> %a1, <8 x float> %a2, i8 %mask) {
-; X86-LABEL: test_mask_vfnmsub256_ps:
+define <2 x double>@test_int_x86_avx512_mask3_vfmadd_pd_128(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_mask3_vfmadd_pd_128:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
-; X86-NEXT:    vfnmsub132ps %ymm1, %ymm2, %ymm0 {%k1} # encoding: [0x62,0xf2,0x6d,0x29,0x9e,0xc1]
-; X86-NEXT:    # ymm0 = -(ymm0 * ymm1) - ymm2
+; X86-NEXT:    vfmadd231pd %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x09,0xb8,0xd1]
+; X86-NEXT:    # xmm2 = (xmm0 * xmm1) + xmm2
+; X86-NEXT:    vmovapd %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x28,0xc2]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
-; X64-LABEL: test_mask_vfnmsub256_ps:
+; X64-LABEL: test_int_x86_avx512_mask3_vfmadd_pd_128:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
-; X64-NEXT:    vfnmsub132ps %ymm1, %ymm2, %ymm0 {%k1} # encoding: [0x62,0xf2,0x6d,0x29,0x9e,0xc1]
-; X64-NEXT:    # ymm0 = -(ymm0 * ymm1) - ymm2
+; X64-NEXT:    vfmadd231pd %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x09,0xb8,0xd1]
+; X64-NEXT:    # xmm2 = (xmm0 * xmm1) + xmm2
+; X64-NEXT:    vmovapd %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x28,0xc2]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <8 x float> @llvm.x86.avx512.mask.vfnmsub.ps.256(<8 x float> %a0, <8 x float> %a1, <8 x float> %a2, i8 %mask) nounwind
-  ret <8 x float> %res
+  %res = call <2 x double> @llvm.x86.avx512.mask3.vfmadd.pd.128(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3)
+  ret <2 x double> %res
 }
 
-declare <4 x float> @llvm.x86.avx512.mask.vfnmsub.ps.128(<4 x float>, <4 x float>, <4 x float>, i8) nounwind readnone
-
-define <4 x float> @test_vfnmsub128_ps(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2) {
-; CHECK-LABEL: test_vfnmsub128_ps:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vfnmsub213ps %xmm2, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x71,0xae,0xc2]
-; CHECK-NEXT:    # xmm0 = -(xmm1 * xmm0) - xmm2
-; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
-  %res = call <4 x float> @llvm.x86.avx512.mask.vfnmsub.ps.128(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, i8 -1) nounwind
-  ret <4 x float> %res
-}
+declare <2 x double> @llvm.x86.avx512.maskz.vfmadd.pd.128(<2 x double>, <2 x double>, <2 x double>, i8)
 
-define <4 x float> @test_mask_vfnmsub128_ps(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, i8 %mask) {
-; X86-LABEL: test_mask_vfnmsub128_ps:
+define <2 x double>@test_int_x86_avx512_maskz_vfmadd_pd_128(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_maskz_vfmadd_pd_128:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
-; X86-NEXT:    vfnmsub132ps %xmm1, %xmm2, %xmm0 {%k1} # encoding: [0x62,0xf2,0x6d,0x09,0x9e,0xc1]
-; X86-NEXT:    # xmm0 = -(xmm0 * xmm1) - xmm2
+; X86-NEXT:    vfmadd213pd %xmm2, %xmm1, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0x89,0xa8,0xc2]
+; X86-NEXT:    # xmm0 = (xmm1 * xmm0) + xmm2
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
-; X64-LABEL: test_mask_vfnmsub128_ps:
+; X64-LABEL: test_int_x86_avx512_maskz_vfmadd_pd_128:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
-; X64-NEXT:    vfnmsub132ps %xmm1, %xmm2, %xmm0 {%k1} # encoding: [0x62,0xf2,0x6d,0x09,0x9e,0xc1]
-; X64-NEXT:    # xmm0 = -(xmm0 * xmm1) - xmm2
+; X64-NEXT:    vfmadd213pd %xmm2, %xmm1, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0x89,0xa8,0xc2]
+; X64-NEXT:    # xmm0 = (xmm1 * xmm0) + xmm2
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <4 x float> @llvm.x86.avx512.mask.vfnmsub.ps.128(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, i8 %mask) nounwind
-  ret <4 x float> %res
+  %res = call <2 x double> @llvm.x86.avx512.maskz.vfmadd.pd.128(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3)
+  ret <2 x double> %res
 }
 
-declare <4 x double> @llvm.x86.avx512.mask.vfnmsub.pd.256(<4 x double>, <4 x double>, <4 x double>, i8) nounwind readnone
-
-define <4 x double> @test_vfnmsub256_pd(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2) {
-; CHECK-LABEL: test_vfnmsub256_pd:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vfnmsub213pd %ymm2, %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0xf5,0xae,0xc2]
-; CHECK-NEXT:    # ymm0 = -(ymm1 * ymm0) - ymm2
-; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
-  %res = call <4 x double> @llvm.x86.avx512.mask.vfnmsub.pd.256(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2, i8 -1) nounwind
-  ret <4 x double> %res
-}
+declare <4 x double> @llvm.x86.avx512.mask3.vfmadd.pd.256(<4 x double>, <4 x double>, <4 x double>, i8)
 
-define <4 x double> @test_mask_vfnmsub256_pd(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2, i8 %mask) {
-; X86-LABEL: test_mask_vfnmsub256_pd:
+define <4 x double>@test_int_x86_avx512_mask3_vfmadd_pd_256(<4 x double> %x0, <4 x double> %x1, <4 x double> %x2, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_mask3_vfmadd_pd_256:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
-; X86-NEXT:    vfnmsub132pd %ymm1, %ymm2, %ymm0 {%k1} # encoding: [0x62,0xf2,0xed,0x29,0x9e,0xc1]
-; X86-NEXT:    # ymm0 = -(ymm0 * ymm1) - ymm2
+; X86-NEXT:    vfmadd231pd %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x29,0xb8,0xd1]
+; X86-NEXT:    # ymm2 = (ymm0 * ymm1) + ymm2
+; X86-NEXT:    vmovapd %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x28,0xc2]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
-; X64-LABEL: test_mask_vfnmsub256_pd:
+; X64-LABEL: test_int_x86_avx512_mask3_vfmadd_pd_256:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
-; X64-NEXT:    vfnmsub132pd %ymm1, %ymm2, %ymm0 {%k1} # encoding: [0x62,0xf2,0xed,0x29,0x9e,0xc1]
-; X64-NEXT:    # ymm0 = -(ymm0 * ymm1) - ymm2
+; X64-NEXT:    vfmadd231pd %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x29,0xb8,0xd1]
+; X64-NEXT:    # ymm2 = (ymm0 * ymm1) + ymm2
+; X64-NEXT:    vmovapd %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x28,0xc2]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <4 x double> @llvm.x86.avx512.mask.vfnmsub.pd.256(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2, i8 %mask) nounwind
+  %res = call <4 x double> @llvm.x86.avx512.mask3.vfmadd.pd.256(<4 x double> %x0, <4 x double> %x1, <4 x double> %x2, i8 %x3)
   ret <4 x double> %res
 }
 
-declare <2 x double> @llvm.x86.avx512.mask.vfnmsub.pd.128(<2 x double>, <2 x double>, <2 x double>, i8) nounwind readnone
-
-define <2 x double> @test_vfnmsub128_pd(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2) {
-; CHECK-LABEL: test_vfnmsub128_pd:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vfnmsub213pd %xmm2, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0xf1,0xae,0xc2]
-; CHECK-NEXT:    # xmm0 = -(xmm1 * xmm0) - xmm2
-; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
-  %res = call <2 x double> @llvm.x86.avx512.mask.vfnmsub.pd.128(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, i8 -1) nounwind
-  ret <2 x double> %res
-}
+declare <4 x double> @llvm.x86.avx512.maskz.vfmadd.pd.256(<4 x double>, <4 x double>, <4 x double>, i8)
 
-define <2 x double> @test_mask_vfnmsub128_pd(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, i8 %mask) {
-; X86-LABEL: test_mask_vfnmsub128_pd:
+define <4 x double>@test_int_x86_avx512_maskz_vfmadd_pd_256(<4 x double> %x0, <4 x double> %x1, <4 x double> %x2, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_maskz_vfmadd_pd_256:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
-; X86-NEXT:    vfnmsub132pd %xmm1, %xmm2, %xmm0 {%k1} # encoding: [0x62,0xf2,0xed,0x09,0x9e,0xc1]
-; X86-NEXT:    # xmm0 = -(xmm0 * xmm1) - xmm2
+; X86-NEXT:    vfmadd213pd %ymm2, %ymm1, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xa9,0xa8,0xc2]
+; X86-NEXT:    # ymm0 = (ymm1 * ymm0) + ymm2
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
-; X64-LABEL: test_mask_vfnmsub128_pd:
+; X64-LABEL: test_int_x86_avx512_maskz_vfmadd_pd_256:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
-; X64-NEXT:    vfnmsub132pd %xmm1, %xmm2, %xmm0 {%k1} # encoding: [0x62,0xf2,0xed,0x09,0x9e,0xc1]
-; X64-NEXT:    # xmm0 = -(xmm0 * xmm1) - xmm2
+; X64-NEXT:    vfmadd213pd %ymm2, %ymm1, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xa9,0xa8,0xc2]
+; X64-NEXT:    # ymm0 = (ymm1 * ymm0) + ymm2
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <2 x double> @llvm.x86.avx512.mask.vfnmsub.pd.128(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, i8 %mask) nounwind
-  ret <2 x double> %res
+  %res = call <4 x double> @llvm.x86.avx512.maskz.vfmadd.pd.256(<4 x double> %x0, <4 x double> %x1, <4 x double> %x2, i8 %x3)
+  ret <4 x double> %res
 }
 
-declare <2 x double> @llvm.x86.avx512.mask3.vfnmsub.pd.128(<2 x double>, <2 x double>, <2 x double>, i8)
+declare <4 x float> @llvm.x86.avx512.mask3.vfmadd.ps.128(<4 x float>, <4 x float>, <4 x float>, i8)
 
-define <2 x double>@test_int_x86_avx512_mask3_vfnmsub_pd_128(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3) {
-; X86-LABEL: test_int_x86_avx512_mask3_vfnmsub_pd_128:
+define <4 x float>@test_int_x86_avx512_mask3_vfmadd_ps_128(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_mask3_vfmadd_ps_128:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
-; X86-NEXT:    vfnmsub231pd %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x09,0xbe,0xd1]
-; X86-NEXT:    # xmm2 = -(xmm0 * xmm1) - xmm2
-; X86-NEXT:    vmovapd %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x28,0xc2]
+; X86-NEXT:    vfmadd231ps %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf2,0x7d,0x09,0xb8,0xd1]
+; X86-NEXT:    # xmm2 = (xmm0 * xmm1) + xmm2
+; X86-NEXT:    vmovaps %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf8,0x28,0xc2]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
-; X64-LABEL: test_int_x86_avx512_mask3_vfnmsub_pd_128:
+; X64-LABEL: test_int_x86_avx512_mask3_vfmadd_ps_128:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
-; X64-NEXT:    vfnmsub231pd %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x09,0xbe,0xd1]
-; X64-NEXT:    # xmm2 = -(xmm0 * xmm1) - xmm2
-; X64-NEXT:    vmovapd %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x28,0xc2]
+; X64-NEXT:    vfmadd231ps %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf2,0x7d,0x09,0xb8,0xd1]
+; X64-NEXT:    # xmm2 = (xmm0 * xmm1) + xmm2
+; X64-NEXT:    vmovaps %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf8,0x28,0xc2]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <2 x double> @llvm.x86.avx512.mask3.vfnmsub.pd.128(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3)
-  ret <2 x double> %res
+  %res = call <4 x float> @llvm.x86.avx512.mask3.vfmadd.ps.128(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3)
+  ret <4 x float> %res
 }
 
-declare <4 x double> @llvm.x86.avx512.mask3.vfnmsub.pd.256(<4 x double>, <4 x double>, <4 x double>, i8)
+declare <4 x float> @llvm.x86.avx512.maskz.vfmadd.ps.128(<4 x float>, <4 x float>, <4 x float>, i8)
 
-define <4 x double>@test_int_x86_avx512_mask3_vfnmsub_pd_256(<4 x double> %x0, <4 x double> %x1, <4 x double> %x2, i8 %x3) {
-; X86-LABEL: test_int_x86_avx512_mask3_vfnmsub_pd_256:
+define <4 x float>@test_int_x86_avx512_maskz_vfmadd_ps_128(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_maskz_vfmadd_ps_128:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
-; X86-NEXT:    vfnmsub231pd %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x29,0xbe,0xd1]
-; X86-NEXT:    # ymm2 = -(ymm0 * ymm1) - ymm2
-; X86-NEXT:    vmovapd %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x28,0xc2]
+; X86-NEXT:    vfmadd213ps %xmm2, %xmm1, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0x89,0xa8,0xc2]
+; X86-NEXT:    # xmm0 = (xmm1 * xmm0) + xmm2
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
-; X64-LABEL: test_int_x86_avx512_mask3_vfnmsub_pd_256:
+; X64-LABEL: test_int_x86_avx512_maskz_vfmadd_ps_128:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
-; X64-NEXT:    vfnmsub231pd %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x29,0xbe,0xd1]
-; X64-NEXT:    # ymm2 = -(ymm0 * ymm1) - ymm2
-; X64-NEXT:    vmovapd %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x28,0xc2]
+; X64-NEXT:    vfmadd213ps %xmm2, %xmm1, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0x89,0xa8,0xc2]
+; X64-NEXT:    # xmm0 = (xmm1 * xmm0) + xmm2
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <4 x double> @llvm.x86.avx512.mask3.vfnmsub.pd.256(<4 x double> %x0, <4 x double> %x1, <4 x double> %x2, i8 %x3)
-  ret <4 x double> %res
+  %res = call <4 x float> @llvm.x86.avx512.maskz.vfmadd.ps.128(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3)
+  ret <4 x float> %res
 }
 
-declare <4 x float> @llvm.x86.avx512.mask3.vfnmsub.ps.128(<4 x float>, <4 x float>, <4 x float>, i8)
+declare <8 x float> @llvm.x86.avx512.mask3.vfmadd.ps.256(<8 x float>, <8 x float>, <8 x float>, i8)
 
-define <4 x float>@test_int_x86_avx512_mask3_vfnmsub_ps_128(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3) {
-; X86-LABEL: test_int_x86_avx512_mask3_vfnmsub_ps_128:
+define <8 x float>@test_int_x86_avx512_mask3_vfmadd_ps_256(<8 x float> %x0, <8 x float> %x1, <8 x float> %x2, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_mask3_vfmadd_ps_256:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
-; X86-NEXT:    vfnmsub231ps %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf2,0x7d,0x09,0xbe,0xd1]
-; X86-NEXT:    # xmm2 = -(xmm0 * xmm1) - xmm2
-; X86-NEXT:    vmovaps %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf8,0x28,0xc2]
+; X86-NEXT:    vfmadd231ps %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf2,0x7d,0x29,0xb8,0xd1]
+; X86-NEXT:    # ymm2 = (ymm0 * ymm1) + ymm2
+; X86-NEXT:    vmovaps %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfc,0x28,0xc2]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
-; X64-LABEL: test_int_x86_avx512_mask3_vfnmsub_ps_128:
+; X64-LABEL: test_int_x86_avx512_mask3_vfmadd_ps_256:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
-; X64-NEXT:    vfnmsub231ps %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf2,0x7d,0x09,0xbe,0xd1]
-; X64-NEXT:    # xmm2 = -(xmm0 * xmm1) - xmm2
-; X64-NEXT:    vmovaps %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf8,0x28,0xc2]
+; X64-NEXT:    vfmadd231ps %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf2,0x7d,0x29,0xb8,0xd1]
+; X64-NEXT:    # ymm2 = (ymm0 * ymm1) + ymm2
+; X64-NEXT:    vmovaps %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfc,0x28,0xc2]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <4 x float> @llvm.x86.avx512.mask3.vfnmsub.ps.128(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3)
-  ret <4 x float> %res
+  %res = call <8 x float> @llvm.x86.avx512.mask3.vfmadd.ps.256(<8 x float> %x0, <8 x float> %x1, <8 x float> %x2, i8 %x3)
+  ret <8 x float> %res
 }
 
-declare <8 x float> @llvm.x86.avx512.mask3.vfnmsub.ps.256(<8 x float>, <8 x float>, <8 x float>, i8)
+declare <8 x float> @llvm.x86.avx512.maskz.vfmadd.ps.256(<8 x float>, <8 x float>, <8 x float>, i8)
 
-define <8 x float>@test_int_x86_avx512_mask3_vfnmsub_ps_256(<8 x float> %x0, <8 x float> %x1, <8 x float> %x2, i8 %x3) {
-; X86-LABEL: test_int_x86_avx512_mask3_vfnmsub_ps_256:
+define <8 x float>@test_int_x86_avx512_maskz_vfmadd_ps_256(<8 x float> %x0, <8 x float> %x1, <8 x float> %x2, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_maskz_vfmadd_ps_256:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
-; X86-NEXT:    vfnmsub231ps %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf2,0x7d,0x29,0xbe,0xd1]
-; X86-NEXT:    # ymm2 = -(ymm0 * ymm1) - ymm2
-; X86-NEXT:    vmovaps %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfc,0x28,0xc2]
+; X86-NEXT:    vfmadd213ps %ymm2, %ymm1, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0xa9,0xa8,0xc2]
+; X86-NEXT:    # ymm0 = (ymm1 * ymm0) + ymm2
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
-; X64-LABEL: test_int_x86_avx512_mask3_vfnmsub_ps_256:
+; X64-LABEL: test_int_x86_avx512_maskz_vfmadd_ps_256:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
-; X64-NEXT:    vfnmsub231ps %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf2,0x7d,0x29,0xbe,0xd1]
-; X64-NEXT:    # ymm2 = -(ymm0 * ymm1) - ymm2
-; X64-NEXT:    vmovaps %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfc,0x28,0xc2]
+; X64-NEXT:    vfmadd213ps %ymm2, %ymm1, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0xa9,0xa8,0xc2]
+; X64-NEXT:    # ymm0 = (ymm1 * ymm0) + ymm2
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <8 x float> @llvm.x86.avx512.mask3.vfnmsub.ps.256(<8 x float> %x0, <8 x float> %x1, <8 x float> %x2, i8 %x3)
+  %res = call <8 x float> @llvm.x86.avx512.maskz.vfmadd.ps.256(<8 x float> %x0, <8 x float> %x1, <8 x float> %x2, i8 %x3)
   ret <8 x float> %res
 }
 
-declare <8 x float> @llvm.x86.avx512.mask.vfmaddsub.ps.256(<8 x float>, <8 x float>, <8 x float>, i8) nounwind readnone
 
-define <8 x float> @test_fmaddsub256_ps(<8 x float> %a, <8 x float> %b, <8 x float> %c) {
-; CHECK-LABEL: test_fmaddsub256_ps:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vfmaddsub213ps %ymm2, %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x75,0xa6,0xc2]
-; CHECK-NEXT:    # ymm0 = (ymm1 * ymm0) +/- ymm2
-; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
-  %res = call <8 x float> @llvm.x86.avx512.mask.vfmaddsub.ps.256(<8 x float> %a, <8 x float> %b, <8 x float> %c, i8 -1)
-  ret <8 x float> %res
-}
+declare <2 x double> @llvm.x86.avx512.mask3.vfmsub.pd.128(<2 x double>, <2 x double>, <2 x double>, i8)
 
-define <8 x float> @test_mask_fmaddsub256_ps(<8 x float> %a, <8 x float> %b, <8 x float> %c, i8 %mask) {
-; X86-LABEL: test_mask_fmaddsub256_ps:
+define <2 x double>@test_int_x86_avx512_mask3_vfmsub_pd_128(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_mask3_vfmsub_pd_128:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
-; X86-NEXT:    vfmaddsub132ps %ymm1, %ymm2, %ymm0 {%k1} # encoding: [0x62,0xf2,0x6d,0x29,0x96,0xc1]
-; X86-NEXT:    # ymm0 = (ymm0 * ymm1) +/- ymm2
+; X86-NEXT:    vfmsub231pd %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x09,0xba,0xd1]
+; X86-NEXT:    # xmm2 = (xmm0 * xmm1) - xmm2
+; X86-NEXT:    vmovapd %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x28,0xc2]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
-; X64-LABEL: test_mask_fmaddsub256_ps:
+; X64-LABEL: test_int_x86_avx512_mask3_vfmsub_pd_128:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
-; X64-NEXT:    vfmaddsub132ps %ymm1, %ymm2, %ymm0 {%k1} # encoding: [0x62,0xf2,0x6d,0x29,0x96,0xc1]
-; X64-NEXT:    # ymm0 = (ymm0 * ymm1) +/- ymm2
+; X64-NEXT:    vfmsub231pd %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x09,0xba,0xd1]
+; X64-NEXT:    # xmm2 = (xmm0 * xmm1) - xmm2
+; X64-NEXT:    vmovapd %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x28,0xc2]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <8 x float> @llvm.x86.avx512.mask.vfmaddsub.ps.256(<8 x float> %a, <8 x float> %b, <8 x float> %c, i8 %mask)
-  ret <8 x float> %res
+  %res = call <2 x double> @llvm.x86.avx512.mask3.vfmsub.pd.128(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3)
+  ret <2 x double> %res
 }
 
-declare <4 x float> @llvm.x86.avx512.mask.vfmaddsub.ps.128(<4 x float>, <4 x float>, <4 x float>, i8) nounwind readnone
 
-define <4 x float> @test_fmaddsub128_ps(<4 x float> %a, <4 x float> %b, <4 x float> %c) {
-; CHECK-LABEL: test_fmaddsub128_ps:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vfmaddsub213ps %xmm2, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x71,0xa6,0xc2]
-; CHECK-NEXT:    # xmm0 = (xmm1 * xmm0) +/- xmm2
-; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
-  %res = call <4 x float> @llvm.x86.avx512.mask.vfmaddsub.ps.128(<4 x float> %a, <4 x float> %b, <4 x float> %c, i8 -1)
-  ret <4 x float> %res
-}
+declare <4 x double> @llvm.x86.avx512.mask3.vfmsub.pd.256(<4 x double>, <4 x double>, <4 x double>, i8)
 
-define <4 x float> @test_mask_fmaddsub128_ps(<4 x float> %a, <4 x float> %b, <4 x float> %c, i8 %mask) {
-; X86-LABEL: test_mask_fmaddsub128_ps:
+define <4 x double>@test_int_x86_avx512_mask3_vfmsub_pd_256(<4 x double> %x0, <4 x double> %x1, <4 x double> %x2, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_mask3_vfmsub_pd_256:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
-; X86-NEXT:    vfmaddsub132ps %xmm1, %xmm2, %xmm0 {%k1} # encoding: [0x62,0xf2,0x6d,0x09,0x96,0xc1]
-; X86-NEXT:    # xmm0 = (xmm0 * xmm1) +/- xmm2
+; X86-NEXT:    vfmsub231pd %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x29,0xba,0xd1]
+; X86-NEXT:    # ymm2 = (ymm0 * ymm1) - ymm2
+; X86-NEXT:    vmovapd %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x28,0xc2]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
-; X64-LABEL: test_mask_fmaddsub128_ps:
+; X64-LABEL: test_int_x86_avx512_mask3_vfmsub_pd_256:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
-; X64-NEXT:    vfmaddsub132ps %xmm1, %xmm2, %xmm0 {%k1} # encoding: [0x62,0xf2,0x6d,0x09,0x96,0xc1]
-; X64-NEXT:    # xmm0 = (xmm0 * xmm1) +/- xmm2
+; X64-NEXT:    vfmsub231pd %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x29,0xba,0xd1]
+; X64-NEXT:    # ymm2 = (ymm0 * ymm1) - ymm2
+; X64-NEXT:    vmovapd %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x28,0xc2]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <4 x float> @llvm.x86.avx512.mask.vfmaddsub.ps.128(<4 x float> %a, <4 x float> %b, <4 x float> %c, i8 %mask)
-  ret <4 x float> %res
-}
-
-declare <4 x double> @llvm.x86.avx512.mask.vfmaddsub.pd.256(<4 x double>, <4 x double>, <4 x double>, i8) nounwind readnone
-
-define <4 x double> @test_vfmaddsub256_pd(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2) {
-; CHECK-LABEL: test_vfmaddsub256_pd:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vfmaddsub213pd %ymm2, %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0xf5,0xa6,0xc2]
-; CHECK-NEXT:    # ymm0 = (ymm1 * ymm0) +/- ymm2
-; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
-  %res = call <4 x double> @llvm.x86.avx512.mask.vfmaddsub.pd.256(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2, i8 -1) nounwind
+  %res = call <4 x double> @llvm.x86.avx512.mask3.vfmsub.pd.256(<4 x double> %x0, <4 x double> %x1, <4 x double> %x2, i8 %x3)
   ret <4 x double> %res
 }
 
-define <4 x double> @test_mask_vfmaddsub256_pd(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2, i8 %mask) {
-; X86-LABEL: test_mask_vfmaddsub256_pd:
+declare <4 x float> @llvm.x86.avx512.mask3.vfmsub.ps.128(<4 x float>, <4 x float>, <4 x float>, i8)
+
+define <4 x float>@test_int_x86_avx512_mask3_vfmsub_ps_128(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_mask3_vfmsub_ps_128:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
-; X86-NEXT:    vfmaddsub132pd %ymm1, %ymm2, %ymm0 {%k1} # encoding: [0x62,0xf2,0xed,0x29,0x96,0xc1]
-; X86-NEXT:    # ymm0 = (ymm0 * ymm1) +/- ymm2
+; X86-NEXT:    vfmsub231ps %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf2,0x7d,0x09,0xba,0xd1]
+; X86-NEXT:    # xmm2 = (xmm0 * xmm1) - xmm2
+; X86-NEXT:    vmovaps %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf8,0x28,0xc2]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
-; X64-LABEL: test_mask_vfmaddsub256_pd:
+; X64-LABEL: test_int_x86_avx512_mask3_vfmsub_ps_128:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
-; X64-NEXT:    vfmaddsub132pd %ymm1, %ymm2, %ymm0 {%k1} # encoding: [0x62,0xf2,0xed,0x29,0x96,0xc1]
-; X64-NEXT:    # ymm0 = (ymm0 * ymm1) +/- ymm2
+; X64-NEXT:    vfmsub231ps %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf2,0x7d,0x09,0xba,0xd1]
+; X64-NEXT:    # xmm2 = (xmm0 * xmm1) - xmm2
+; X64-NEXT:    vmovaps %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf8,0x28,0xc2]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <4 x double> @llvm.x86.avx512.mask.vfmaddsub.pd.256(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2, i8 %mask) nounwind
-  ret <4 x double> %res
+  %res = call <4 x float> @llvm.x86.avx512.mask3.vfmsub.ps.128(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3)
+  ret <4 x float> %res
 }
 
-declare <2 x double> @llvm.x86.avx512.mask.vfmaddsub.pd.128(<2 x double>, <2 x double>, <2 x double>, i8) nounwind readnone
-
-define <2 x double> @test_vfmaddsub128_pd(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2) {
-; CHECK-LABEL: test_vfmaddsub128_pd:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vfmaddsub213pd %xmm2, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0xf1,0xa6,0xc2]
-; CHECK-NEXT:    # xmm0 = (xmm1 * xmm0) +/- xmm2
-; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
-  %res = call <2 x double> @llvm.x86.avx512.mask.vfmaddsub.pd.128(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, i8 -1) nounwind
-  ret <2 x double> %res
-}
+declare <8 x float> @llvm.x86.avx512.mask3.vfmsub.ps.256(<8 x float>, <8 x float>, <8 x float>, i8)
 
-define <2 x double> @test_mask_vfmaddsub128_pd(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, i8 %mask) {
-; X86-LABEL: test_mask_vfmaddsub128_pd:
+define <8 x float>@test_int_x86_avx512_mask3_vfmsub_ps_256(<8 x float> %x0, <8 x float> %x1, <8 x float> %x2, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_mask3_vfmsub_ps_256:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
-; X86-NEXT:    vfmaddsub132pd %xmm1, %xmm2, %xmm0 {%k1} # encoding: [0x62,0xf2,0xed,0x09,0x96,0xc1]
-; X86-NEXT:    # xmm0 = (xmm0 * xmm1) +/- xmm2
+; X86-NEXT:    vfmsub231ps %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf2,0x7d,0x29,0xba,0xd1]
+; X86-NEXT:    # ymm2 = (ymm0 * ymm1) - ymm2
+; X86-NEXT:    vmovaps %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfc,0x28,0xc2]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
-; X64-LABEL: test_mask_vfmaddsub128_pd:
+; X64-LABEL: test_int_x86_avx512_mask3_vfmsub_ps_256:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
-; X64-NEXT:    vfmaddsub132pd %xmm1, %xmm2, %xmm0 {%k1} # encoding: [0x62,0xf2,0xed,0x09,0x96,0xc1]
-; X64-NEXT:    # xmm0 = (xmm0 * xmm1) +/- xmm2
+; X64-NEXT:    vfmsub231ps %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf2,0x7d,0x29,0xba,0xd1]
+; X64-NEXT:    # ymm2 = (ymm0 * ymm1) - ymm2
+; X64-NEXT:    vmovaps %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfc,0x28,0xc2]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <2 x double> @llvm.x86.avx512.mask.vfmaddsub.pd.128(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, i8 %mask) nounwind
-  ret <2 x double> %res
+  %res = call <8 x float> @llvm.x86.avx512.mask3.vfmsub.ps.256(<8 x float> %x0, <8 x float> %x1, <8 x float> %x2, i8 %x3)
+  ret <8 x float> %res
 }
 
-declare <2 x double> @llvm.x86.avx512.mask3.vfmaddsub.pd.128(<2 x double>, <2 x double>, <2 x double>, i8)
+declare <8 x float> @llvm.x86.avx512.mask.vfnmadd.ps.256(<8 x float>, <8 x float>, <8 x float>, i8) nounwind readnone
 
-define <2 x double>@test_int_x86_avx512_mask3_vfmaddsub_pd_128(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3) {
-; X86-LABEL: test_int_x86_avx512_mask3_vfmaddsub_pd_128:
+define <8 x float> @test_vfnmadd256_ps(<8 x float> %a0, <8 x float> %a1, <8 x float> %a2) {
+; CHECK-LABEL: test_vfnmadd256_ps:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vfnmadd213ps %ymm2, %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x75,0xac,0xc2]
+; CHECK-NEXT:    # ymm0 = -(ymm1 * ymm0) + ymm2
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <8 x float> @llvm.x86.avx512.mask.vfnmadd.ps.256(<8 x float> %a0, <8 x float> %a1, <8 x float> %a2, i8 -1) nounwind
+  ret <8 x float> %res
+}
+
+define <8 x float> @test_mask_vfnmadd256_ps(<8 x float> %a0, <8 x float> %a1, <8 x float> %a2, i8 %mask) {
+; X86-LABEL: test_mask_vfnmadd256_ps:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
-; X86-NEXT:    vfmaddsub231pd %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x09,0xb6,0xd1]
-; X86-NEXT:    # xmm2 = (xmm0 * xmm1) +/- xmm2
-; X86-NEXT:    vmovapd %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x28,0xc2]
+; X86-NEXT:    vfnmadd132ps %ymm1, %ymm2, %ymm0 {%k1} # encoding: [0x62,0xf2,0x6d,0x29,0x9c,0xc1]
+; X86-NEXT:    # ymm0 = -(ymm0 * ymm1) + ymm2
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
-; X64-LABEL: test_int_x86_avx512_mask3_vfmaddsub_pd_128:
+; X64-LABEL: test_mask_vfnmadd256_ps:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
-; X64-NEXT:    vfmaddsub231pd %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x09,0xb6,0xd1]
-; X64-NEXT:    # xmm2 = (xmm0 * xmm1) +/- xmm2
-; X64-NEXT:    vmovapd %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x28,0xc2]
+; X64-NEXT:    vfnmadd132ps %ymm1, %ymm2, %ymm0 {%k1} # encoding: [0x62,0xf2,0x6d,0x29,0x9c,0xc1]
+; X64-NEXT:    # ymm0 = -(ymm0 * ymm1) + ymm2
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <2 x double> @llvm.x86.avx512.mask3.vfmaddsub.pd.128(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3)
-  ret <2 x double> %res
+  %res = call <8 x float> @llvm.x86.avx512.mask.vfnmadd.ps.256(<8 x float> %a0, <8 x float> %a1, <8 x float> %a2, i8 %mask) nounwind
+  ret <8 x float> %res
 }
 
-declare <2 x double> @llvm.x86.avx512.maskz.vfmaddsub.pd.128(<2 x double>, <2 x double>, <2 x double>, i8)
+declare <4 x float> @llvm.x86.avx512.mask.vfnmadd.ps.128(<4 x float>, <4 x float>, <4 x float>, i8) nounwind readnone
 
-define <2 x double>@test_int_x86_avx512_maskz_vfmaddsub_pd_128(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3) {
-; X86-LABEL: test_int_x86_avx512_maskz_vfmaddsub_pd_128:
+define <4 x float> @test_vfnmadd128_ps(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2) {
+; CHECK-LABEL: test_vfnmadd128_ps:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vfnmadd213ps %xmm2, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x71,0xac,0xc2]
+; CHECK-NEXT:    # xmm0 = -(xmm1 * xmm0) + xmm2
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <4 x float> @llvm.x86.avx512.mask.vfnmadd.ps.128(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, i8 -1) nounwind
+  ret <4 x float> %res
+}
+
+define <4 x float> @test_mask_vfnmadd128_ps(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, i8 %mask) {
+; X86-LABEL: test_mask_vfnmadd128_ps:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
-; X86-NEXT:    vfmaddsub213pd %xmm2, %xmm1, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0x89,0xa6,0xc2]
-; X86-NEXT:    # xmm0 = (xmm1 * xmm0) +/- xmm2
+; X86-NEXT:    vfnmadd132ps %xmm1, %xmm2, %xmm0 {%k1} # encoding: [0x62,0xf2,0x6d,0x09,0x9c,0xc1]
+; X86-NEXT:    # xmm0 = -(xmm0 * xmm1) + xmm2
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
-; X64-LABEL: test_int_x86_avx512_maskz_vfmaddsub_pd_128:
+; X64-LABEL: test_mask_vfnmadd128_ps:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
-; X64-NEXT:    vfmaddsub213pd %xmm2, %xmm1, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0x89,0xa6,0xc2]
-; X64-NEXT:    # xmm0 = (xmm1 * xmm0) +/- xmm2
+; X64-NEXT:    vfnmadd132ps %xmm1, %xmm2, %xmm0 {%k1} # encoding: [0x62,0xf2,0x6d,0x09,0x9c,0xc1]
+; X64-NEXT:    # xmm0 = -(xmm0 * xmm1) + xmm2
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <2 x double> @llvm.x86.avx512.maskz.vfmaddsub.pd.128(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3)
-  ret <2 x double> %res
+  %res = call <4 x float> @llvm.x86.avx512.mask.vfnmadd.ps.128(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, i8 %mask) nounwind
+  ret <4 x float> %res
 }
 
-declare <4 x double> @llvm.x86.avx512.mask3.vfmaddsub.pd.256(<4 x double>, <4 x double>, <4 x double>, i8)
+declare <4 x double> @llvm.x86.avx512.mask.vfnmadd.pd.256(<4 x double>, <4 x double>, <4 x double>, i8) nounwind readnone
 
-define <4 x double>@test_int_x86_avx512_mask3_vfmaddsub_pd_256(<4 x double> %x0, <4 x double> %x1, <4 x double> %x2, i8 %x3) {
-; X86-LABEL: test_int_x86_avx512_mask3_vfmaddsub_pd_256:
+define <4 x double> @test_vfnmadd256_pd(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2) {
+; CHECK-LABEL: test_vfnmadd256_pd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vfnmadd213pd %ymm2, %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0xf5,0xac,0xc2]
+; CHECK-NEXT:    # ymm0 = -(ymm1 * ymm0) + ymm2
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <4 x double> @llvm.x86.avx512.mask.vfnmadd.pd.256(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2, i8 -1) nounwind
+  ret <4 x double> %res
+}
+
+define <4 x double> @test_mask_vfnmadd256_pd(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2, i8 %mask) {
+; X86-LABEL: test_mask_vfnmadd256_pd:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
-; X86-NEXT:    vfmaddsub231pd %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x29,0xb6,0xd1]
-; X86-NEXT:    # ymm2 = (ymm0 * ymm1) +/- ymm2
-; X86-NEXT:    vmovapd %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x28,0xc2]
+; X86-NEXT:    vfnmadd132pd %ymm1, %ymm2, %ymm0 {%k1} # encoding: [0x62,0xf2,0xed,0x29,0x9c,0xc1]
+; X86-NEXT:    # ymm0 = -(ymm0 * ymm1) + ymm2
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
-; X64-LABEL: test_int_x86_avx512_mask3_vfmaddsub_pd_256:
+; X64-LABEL: test_mask_vfnmadd256_pd:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
-; X64-NEXT:    vfmaddsub231pd %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x29,0xb6,0xd1]
-; X64-NEXT:    # ymm2 = (ymm0 * ymm1) +/- ymm2
-; X64-NEXT:    vmovapd %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x28,0xc2]
+; X64-NEXT:    vfnmadd132pd %ymm1, %ymm2, %ymm0 {%k1} # encoding: [0x62,0xf2,0xed,0x29,0x9c,0xc1]
+; X64-NEXT:    # ymm0 = -(ymm0 * ymm1) + ymm2
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <4 x double> @llvm.x86.avx512.mask3.vfmaddsub.pd.256(<4 x double> %x0, <4 x double> %x1, <4 x double> %x2, i8 %x3)
+  %res = call <4 x double> @llvm.x86.avx512.mask.vfnmadd.pd.256(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2, i8 %mask) nounwind
   ret <4 x double> %res
 }
 
-declare <4 x double> @llvm.x86.avx512.maskz.vfmaddsub.pd.256(<4 x double>, <4 x double>, <4 x double>, i8)
+declare <2 x double> @llvm.x86.avx512.mask.vfnmadd.pd.128(<2 x double>, <2 x double>, <2 x double>, i8) nounwind readnone
 
-define <4 x double>@test_int_x86_avx512_maskz_vfmaddsub_pd_256(<4 x double> %x0, <4 x double> %x1, <4 x double> %x2, i8 %x3) {
-; X86-LABEL: test_int_x86_avx512_maskz_vfmaddsub_pd_256:
+define <2 x double> @test_vfnmadd128_pd(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2) {
+; CHECK-LABEL: test_vfnmadd128_pd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vfnmadd213pd %xmm2, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0xf1,0xac,0xc2]
+; CHECK-NEXT:    # xmm0 = -(xmm1 * xmm0) + xmm2
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <2 x double> @llvm.x86.avx512.mask.vfnmadd.pd.128(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, i8 -1) nounwind
+  ret <2 x double> %res
+}
+
+define <2 x double> @test_mask_vfnmadd128_pd(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, i8 %mask) {
+; X86-LABEL: test_mask_vfnmadd128_pd:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
-; X86-NEXT:    vfmaddsub213pd %ymm2, %ymm1, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xa9,0xa6,0xc2]
-; X86-NEXT:    # ymm0 = (ymm1 * ymm0) +/- ymm2
+; X86-NEXT:    vfnmadd132pd %xmm1, %xmm2, %xmm0 {%k1} # encoding: [0x62,0xf2,0xed,0x09,0x9c,0xc1]
+; X86-NEXT:    # xmm0 = -(xmm0 * xmm1) + xmm2
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
-; X64-LABEL: test_int_x86_avx512_maskz_vfmaddsub_pd_256:
+; X64-LABEL: test_mask_vfnmadd128_pd:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
-; X64-NEXT:    vfmaddsub213pd %ymm2, %ymm1, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xa9,0xa6,0xc2]
-; X64-NEXT:    # ymm0 = (ymm1 * ymm0) +/- ymm2
+; X64-NEXT:    vfnmadd132pd %xmm1, %xmm2, %xmm0 {%k1} # encoding: [0x62,0xf2,0xed,0x09,0x9c,0xc1]
+; X64-NEXT:    # xmm0 = -(xmm0 * xmm1) + xmm2
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <4 x double> @llvm.x86.avx512.maskz.vfmaddsub.pd.256(<4 x double> %x0, <4 x double> %x1, <4 x double> %x2, i8 %x3)
-  ret <4 x double> %res
+  %res = call <2 x double> @llvm.x86.avx512.mask.vfnmadd.pd.128(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, i8 %mask) nounwind
+  ret <2 x double> %res
 }
 
-declare <4 x float> @llvm.x86.avx512.mask3.vfmaddsub.ps.128(<4 x float>, <4 x float>, <4 x float>, i8)
+declare <8 x float> @llvm.x86.avx512.mask.vfnmsub.ps.256(<8 x float>, <8 x float>, <8 x float>, i8) nounwind readnone
 
-define <4 x float>@test_int_x86_avx512_mask3_vfmaddsub_ps_128(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3) {
-; X86-LABEL: test_int_x86_avx512_mask3_vfmaddsub_ps_128:
+define <8 x float> @test_vfnmsub256_ps(<8 x float> %a0, <8 x float> %a1, <8 x float> %a2) {
+; CHECK-LABEL: test_vfnmsub256_ps:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vfnmsub213ps %ymm2, %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x75,0xae,0xc2]
+; CHECK-NEXT:    # ymm0 = -(ymm1 * ymm0) - ymm2
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <8 x float> @llvm.x86.avx512.mask.vfnmsub.ps.256(<8 x float> %a0, <8 x float> %a1, <8 x float> %a2, i8 -1) nounwind
+  ret <8 x float> %res
+}
+
+define <8 x float> @test_mask_vfnmsub256_ps(<8 x float> %a0, <8 x float> %a1, <8 x float> %a2, i8 %mask) {
+; X86-LABEL: test_mask_vfnmsub256_ps:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
-; X86-NEXT:    vfmaddsub231ps %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf2,0x7d,0x09,0xb6,0xd1]
-; X86-NEXT:    # xmm2 = (xmm0 * xmm1) +/- xmm2
-; X86-NEXT:    vmovaps %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf8,0x28,0xc2]
+; X86-NEXT:    vfnmsub132ps %ymm1, %ymm2, %ymm0 {%k1} # encoding: [0x62,0xf2,0x6d,0x29,0x9e,0xc1]
+; X86-NEXT:    # ymm0 = -(ymm0 * ymm1) - ymm2
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
-; X64-LABEL: test_int_x86_avx512_mask3_vfmaddsub_ps_128:
+; X64-LABEL: test_mask_vfnmsub256_ps:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
-; X64-NEXT:    vfmaddsub231ps %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf2,0x7d,0x09,0xb6,0xd1]
-; X64-NEXT:    # xmm2 = (xmm0 * xmm1) +/- xmm2
-; X64-NEXT:    vmovaps %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf8,0x28,0xc2]
+; X64-NEXT:    vfnmsub132ps %ymm1, %ymm2, %ymm0 {%k1} # encoding: [0x62,0xf2,0x6d,0x29,0x9e,0xc1]
+; X64-NEXT:    # ymm0 = -(ymm0 * ymm1) - ymm2
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <4 x float> @llvm.x86.avx512.mask3.vfmaddsub.ps.128(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3)
-  ret <4 x float> %res
+  %res = call <8 x float> @llvm.x86.avx512.mask.vfnmsub.ps.256(<8 x float> %a0, <8 x float> %a1, <8 x float> %a2, i8 %mask) nounwind
+  ret <8 x float> %res
 }
 
-declare <4 x float> @llvm.x86.avx512.maskz.vfmaddsub.ps.128(<4 x float>, <4 x float>, <4 x float>, i8)
+declare <4 x float> @llvm.x86.avx512.mask.vfnmsub.ps.128(<4 x float>, <4 x float>, <4 x float>, i8) nounwind readnone
 
-define <4 x float>@test_int_x86_avx512_maskz_vfmaddsub_ps_128(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3) {
-; X86-LABEL: test_int_x86_avx512_maskz_vfmaddsub_ps_128:
+define <4 x float> @test_vfnmsub128_ps(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2) {
+; CHECK-LABEL: test_vfnmsub128_ps:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vfnmsub213ps %xmm2, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x71,0xae,0xc2]
+; CHECK-NEXT:    # xmm0 = -(xmm1 * xmm0) - xmm2
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <4 x float> @llvm.x86.avx512.mask.vfnmsub.ps.128(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, i8 -1) nounwind
+  ret <4 x float> %res
+}
+
+define <4 x float> @test_mask_vfnmsub128_ps(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, i8 %mask) {
+; X86-LABEL: test_mask_vfnmsub128_ps:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
-; X86-NEXT:    vfmaddsub213ps %xmm2, %xmm1, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0x89,0xa6,0xc2]
-; X86-NEXT:    # xmm0 = (xmm1 * xmm0) +/- xmm2
+; X86-NEXT:    vfnmsub132ps %xmm1, %xmm2, %xmm0 {%k1} # encoding: [0x62,0xf2,0x6d,0x09,0x9e,0xc1]
+; X86-NEXT:    # xmm0 = -(xmm0 * xmm1) - xmm2
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
-; X64-LABEL: test_int_x86_avx512_maskz_vfmaddsub_ps_128:
+; X64-LABEL: test_mask_vfnmsub128_ps:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
-; X64-NEXT:    vfmaddsub213ps %xmm2, %xmm1, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0x89,0xa6,0xc2]
-; X64-NEXT:    # xmm0 = (xmm1 * xmm0) +/- xmm2
+; X64-NEXT:    vfnmsub132ps %xmm1, %xmm2, %xmm0 {%k1} # encoding: [0x62,0xf2,0x6d,0x09,0x9e,0xc1]
+; X64-NEXT:    # xmm0 = -(xmm0 * xmm1) - xmm2
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <4 x float> @llvm.x86.avx512.maskz.vfmaddsub.ps.128(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3)
+  %res = call <4 x float> @llvm.x86.avx512.mask.vfnmsub.ps.128(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, i8 %mask) nounwind
   ret <4 x float> %res
 }
 
-declare <8 x float> @llvm.x86.avx512.mask3.vfmaddsub.ps.256(<8 x float>, <8 x float>, <8 x float>, i8)
+declare <4 x double> @llvm.x86.avx512.mask.vfnmsub.pd.256(<4 x double>, <4 x double>, <4 x double>, i8) nounwind readnone
 
-define <8 x float>@test_int_x86_avx512_mask3_vfmaddsub_ps_256(<8 x float> %x0, <8 x float> %x1, <8 x float> %x2, i8 %x3) {
-; X86-LABEL: test_int_x86_avx512_mask3_vfmaddsub_ps_256:
+define <4 x double> @test_vfnmsub256_pd(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2) {
+; CHECK-LABEL: test_vfnmsub256_pd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vfnmsub213pd %ymm2, %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0xf5,0xae,0xc2]
+; CHECK-NEXT:    # ymm0 = -(ymm1 * ymm0) - ymm2
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <4 x double> @llvm.x86.avx512.mask.vfnmsub.pd.256(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2, i8 -1) nounwind
+  ret <4 x double> %res
+}
+
+define <4 x double> @test_mask_vfnmsub256_pd(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2, i8 %mask) {
+; X86-LABEL: test_mask_vfnmsub256_pd:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
-; X86-NEXT:    vfmaddsub231ps %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf2,0x7d,0x29,0xb6,0xd1]
-; X86-NEXT:    # ymm2 = (ymm0 * ymm1) +/- ymm2
-; X86-NEXT:    vmovaps %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfc,0x28,0xc2]
+; X86-NEXT:    vfnmsub132pd %ymm1, %ymm2, %ymm0 {%k1} # encoding: [0x62,0xf2,0xed,0x29,0x9e,0xc1]
+; X86-NEXT:    # ymm0 = -(ymm0 * ymm1) - ymm2
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
-; X64-LABEL: test_int_x86_avx512_mask3_vfmaddsub_ps_256:
+; X64-LABEL: test_mask_vfnmsub256_pd:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
-; X64-NEXT:    vfmaddsub231ps %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf2,0x7d,0x29,0xb6,0xd1]
-; X64-NEXT:    # ymm2 = (ymm0 * ymm1) +/- ymm2
-; X64-NEXT:    vmovaps %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfc,0x28,0xc2]
+; X64-NEXT:    vfnmsub132pd %ymm1, %ymm2, %ymm0 {%k1} # encoding: [0x62,0xf2,0xed,0x29,0x9e,0xc1]
+; X64-NEXT:    # ymm0 = -(ymm0 * ymm1) - ymm2
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <8 x float> @llvm.x86.avx512.mask3.vfmaddsub.ps.256(<8 x float> %x0, <8 x float> %x1, <8 x float> %x2, i8 %x3)
-  ret <8 x float> %res
+  %res = call <4 x double> @llvm.x86.avx512.mask.vfnmsub.pd.256(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2, i8 %mask) nounwind
+  ret <4 x double> %res
 }
 
-declare <8 x float> @llvm.x86.avx512.maskz.vfmaddsub.ps.256(<8 x float>, <8 x float>, <8 x float>, i8)
+declare <2 x double> @llvm.x86.avx512.mask.vfnmsub.pd.128(<2 x double>, <2 x double>, <2 x double>, i8) nounwind readnone
 
-define <8 x float>@test_int_x86_avx512_maskz_vfmaddsub_ps_256(<8 x float> %x0, <8 x float> %x1, <8 x float> %x2, i8 %x3) {
-; X86-LABEL: test_int_x86_avx512_maskz_vfmaddsub_ps_256:
+define <2 x double> @test_vfnmsub128_pd(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2) {
+; CHECK-LABEL: test_vfnmsub128_pd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vfnmsub213pd %xmm2, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0xf1,0xae,0xc2]
+; CHECK-NEXT:    # xmm0 = -(xmm1 * xmm0) - xmm2
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <2 x double> @llvm.x86.avx512.mask.vfnmsub.pd.128(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, i8 -1) nounwind
+  ret <2 x double> %res
+}
+
+define <2 x double> @test_mask_vfnmsub128_pd(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, i8 %mask) {
+; X86-LABEL: test_mask_vfnmsub128_pd:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
-; X86-NEXT:    vfmaddsub213ps %ymm2, %ymm1, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0xa9,0xa6,0xc2]
-; X86-NEXT:    # ymm0 = (ymm1 * ymm0) +/- ymm2
+; X86-NEXT:    vfnmsub132pd %xmm1, %xmm2, %xmm0 {%k1} # encoding: [0x62,0xf2,0xed,0x09,0x9e,0xc1]
+; X86-NEXT:    # xmm0 = -(xmm0 * xmm1) - xmm2
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
-; X64-LABEL: test_int_x86_avx512_maskz_vfmaddsub_ps_256:
+; X64-LABEL: test_mask_vfnmsub128_pd:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
-; X64-NEXT:    vfmaddsub213ps %ymm2, %ymm1, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0xa9,0xa6,0xc2]
-; X64-NEXT:    # ymm0 = (ymm1 * ymm0) +/- ymm2
+; X64-NEXT:    vfnmsub132pd %xmm1, %xmm2, %xmm0 {%k1} # encoding: [0x62,0xf2,0xed,0x09,0x9e,0xc1]
+; X64-NEXT:    # xmm0 = -(xmm0 * xmm1) - xmm2
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <8 x float> @llvm.x86.avx512.maskz.vfmaddsub.ps.256(<8 x float> %x0, <8 x float> %x1, <8 x float> %x2, i8 %x3)
-  ret <8 x float> %res
+  %res = call <2 x double> @llvm.x86.avx512.mask.vfnmsub.pd.128(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, i8 %mask) nounwind
+  ret <2 x double> %res
 }
 
-declare <2 x double> @llvm.x86.avx512.mask3.vfmsubadd.pd.128(<2 x double>, <2 x double>, <2 x double>, i8)
+declare <2 x double> @llvm.x86.avx512.mask3.vfnmsub.pd.128(<2 x double>, <2 x double>, <2 x double>, i8)
 
-define <2 x double>@test_int_x86_avx512_mask3_vfmsubadd_pd_128(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3) {
-; X86-LABEL: test_int_x86_avx512_mask3_vfmsubadd_pd_128:
+define <2 x double>@test_int_x86_avx512_mask3_vfnmsub_pd_128(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_mask3_vfnmsub_pd_128:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
-; X86-NEXT:    vfmsubadd231pd %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x09,0xb7,0xd1]
-; X86-NEXT:    # xmm2 = (xmm0 * xmm1) -/+ xmm2
+; X86-NEXT:    vfnmsub231pd %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x09,0xbe,0xd1]
+; X86-NEXT:    # xmm2 = -(xmm0 * xmm1) - xmm2
 ; X86-NEXT:    vmovapd %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x28,0xc2]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
-; X64-LABEL: test_int_x86_avx512_mask3_vfmsubadd_pd_128:
+; X64-LABEL: test_int_x86_avx512_mask3_vfnmsub_pd_128:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
-; X64-NEXT:    vfmsubadd231pd %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x09,0xb7,0xd1]
-; X64-NEXT:    # xmm2 = (xmm0 * xmm1) -/+ xmm2
+; X64-NEXT:    vfnmsub231pd %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x09,0xbe,0xd1]
+; X64-NEXT:    # xmm2 = -(xmm0 * xmm1) - xmm2
 ; X64-NEXT:    vmovapd %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x28,0xc2]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <2 x double> @llvm.x86.avx512.mask3.vfmsubadd.pd.128(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3)
+  %res = call <2 x double> @llvm.x86.avx512.mask3.vfnmsub.pd.128(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3)
   ret <2 x double> %res
 }
 
-declare <4 x double> @llvm.x86.avx512.mask3.vfmsubadd.pd.256(<4 x double>, <4 x double>, <4 x double>, i8)
+declare <4 x double> @llvm.x86.avx512.mask3.vfnmsub.pd.256(<4 x double>, <4 x double>, <4 x double>, i8)
 
-define <4 x double>@test_int_x86_avx512_mask3_vfmsubadd_pd_256(<4 x double> %x0, <4 x double> %x1, <4 x double> %x2, i8 %x3) {
-; X86-LABEL: test_int_x86_avx512_mask3_vfmsubadd_pd_256:
+define <4 x double>@test_int_x86_avx512_mask3_vfnmsub_pd_256(<4 x double> %x0, <4 x double> %x1, <4 x double> %x2, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_mask3_vfnmsub_pd_256:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
-; X86-NEXT:    vfmsubadd231pd %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x29,0xb7,0xd1]
-; X86-NEXT:    # ymm2 = (ymm0 * ymm1) -/+ ymm2
+; X86-NEXT:    vfnmsub231pd %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x29,0xbe,0xd1]
+; X86-NEXT:    # ymm2 = -(ymm0 * ymm1) - ymm2
 ; X86-NEXT:    vmovapd %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x28,0xc2]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
-; X64-LABEL: test_int_x86_avx512_mask3_vfmsubadd_pd_256:
+; X64-LABEL: test_int_x86_avx512_mask3_vfnmsub_pd_256:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
-; X64-NEXT:    vfmsubadd231pd %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x29,0xb7,0xd1]
-; X64-NEXT:    # ymm2 = (ymm0 * ymm1) -/+ ymm2
+; X64-NEXT:    vfnmsub231pd %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x29,0xbe,0xd1]
+; X64-NEXT:    # ymm2 = -(ymm0 * ymm1) - ymm2
 ; X64-NEXT:    vmovapd %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x28,0xc2]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <4 x double> @llvm.x86.avx512.mask3.vfmsubadd.pd.256(<4 x double> %x0, <4 x double> %x1, <4 x double> %x2, i8 %x3)
+  %res = call <4 x double> @llvm.x86.avx512.mask3.vfnmsub.pd.256(<4 x double> %x0, <4 x double> %x1, <4 x double> %x2, i8 %x3)
   ret <4 x double> %res
 }
 
-declare <4 x float> @llvm.x86.avx512.mask3.vfmsubadd.ps.128(<4 x float>, <4 x float>, <4 x float>, i8)
+declare <4 x float> @llvm.x86.avx512.mask3.vfnmsub.ps.128(<4 x float>, <4 x float>, <4 x float>, i8)
 
-define <4 x float>@test_int_x86_avx512_mask3_vfmsubadd_ps_128(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3) {
-; X86-LABEL: test_int_x86_avx512_mask3_vfmsubadd_ps_128:
+define <4 x float>@test_int_x86_avx512_mask3_vfnmsub_ps_128(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_mask3_vfnmsub_ps_128:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
-; X86-NEXT:    vfmsubadd231ps %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf2,0x7d,0x09,0xb7,0xd1]
-; X86-NEXT:    # xmm2 = (xmm0 * xmm1) -/+ xmm2
+; X86-NEXT:    vfnmsub231ps %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf2,0x7d,0x09,0xbe,0xd1]
+; X86-NEXT:    # xmm2 = -(xmm0 * xmm1) - xmm2
 ; X86-NEXT:    vmovaps %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf8,0x28,0xc2]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
-; X64-LABEL: test_int_x86_avx512_mask3_vfmsubadd_ps_128:
+; X64-LABEL: test_int_x86_avx512_mask3_vfnmsub_ps_128:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
-; X64-NEXT:    vfmsubadd231ps %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf2,0x7d,0x09,0xb7,0xd1]
-; X64-NEXT:    # xmm2 = (xmm0 * xmm1) -/+ xmm2
+; X64-NEXT:    vfnmsub231ps %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf2,0x7d,0x09,0xbe,0xd1]
+; X64-NEXT:    # xmm2 = -(xmm0 * xmm1) - xmm2
 ; X64-NEXT:    vmovaps %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf8,0x28,0xc2]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <4 x float> @llvm.x86.avx512.mask3.vfmsubadd.ps.128(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3)
+  %res = call <4 x float> @llvm.x86.avx512.mask3.vfnmsub.ps.128(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3)
   ret <4 x float> %res
 }
 
-declare <8 x float> @llvm.x86.avx512.mask3.vfmsubadd.ps.256(<8 x float>, <8 x float>, <8 x float>, i8)
+declare <8 x float> @llvm.x86.avx512.mask3.vfnmsub.ps.256(<8 x float>, <8 x float>, <8 x float>, i8)
 
-define <8 x float>@test_int_x86_avx512_mask3_vfmsubadd_ps_256(<8 x float> %x0, <8 x float> %x1, <8 x float> %x2, i8 %x3) {
-; X86-LABEL: test_int_x86_avx512_mask3_vfmsubadd_ps_256:
+define <8 x float>@test_int_x86_avx512_mask3_vfnmsub_ps_256(<8 x float> %x0, <8 x float> %x1, <8 x float> %x2, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_mask3_vfnmsub_ps_256:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
-; X86-NEXT:    vfmsubadd231ps %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf2,0x7d,0x29,0xb7,0xd1]
-; X86-NEXT:    # ymm2 = (ymm0 * ymm1) -/+ ymm2
+; X86-NEXT:    vfnmsub231ps %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf2,0x7d,0x29,0xbe,0xd1]
+; X86-NEXT:    # ymm2 = -(ymm0 * ymm1) - ymm2
 ; X86-NEXT:    vmovaps %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfc,0x28,0xc2]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
-; X64-LABEL: test_int_x86_avx512_mask3_vfmsubadd_ps_256:
+; X64-LABEL: test_int_x86_avx512_mask3_vfnmsub_ps_256:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
-; X64-NEXT:    vfmsubadd231ps %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf2,0x7d,0x29,0xb7,0xd1]
-; X64-NEXT:    # ymm2 = (ymm0 * ymm1) -/+ ymm2
+; X64-NEXT:    vfnmsub231ps %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf2,0x7d,0x29,0xbe,0xd1]
+; X64-NEXT:    # ymm2 = -(ymm0 * ymm1) - ymm2
 ; X64-NEXT:    vmovaps %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfc,0x28,0xc2]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <8 x float> @llvm.x86.avx512.mask3.vfmsubadd.ps.256(<8 x float> %x0, <8 x float> %x1, <8 x float> %x2, i8 %x3)
+  %res = call <8 x float> @llvm.x86.avx512.mask3.vfnmsub.ps.256(<8 x float> %x0, <8 x float> %x1, <8 x float> %x2, i8 %x3)
   ret <8 x float> %res
 }
 
+declare <8 x float> @llvm.x86.avx512.mask.vfmaddsub.ps.256(<8 x float>, <8 x float>, <8 x float>, i8) nounwind readnone
 
-define <4 x float> @test_mask_vfmadd128_ps_rmk(<4 x float> %a0, <4 x float> %a1, <4 x float>* %ptr_a2, i8 %mask) {
-; X86-LABEL: test_mask_vfmadd128_ps_rmk:
-; X86:       # %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
-; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx # encoding: [0x0f,0xb6,0x4c,0x24,0x08]
-; X86-NEXT:    kmovw %ecx, %k1 # encoding: [0xc5,0xf8,0x92,0xc9]
-; X86-NEXT:    vfmadd213ps (%eax), %xmm1, %xmm0 {%k1} # encoding: [0x62,0xf2,0x75,0x09,0xa8,0x00]
-; X86-NEXT:    # xmm0 = (xmm1 * xmm0) + mem
-; X86-NEXT:    retl # encoding: [0xc3]
-;
-; X64-LABEL: test_mask_vfmadd128_ps_rmk:
-; X64:       # %bb.0:
-; X64-NEXT:    kmovw %esi, %k1 # encoding: [0xc5,0xf8,0x92,0xce]
-; X64-NEXT:    vfmadd213ps (%rdi), %xmm1, %xmm0 {%k1} # encoding: [0x62,0xf2,0x75,0x09,0xa8,0x07]
-; X64-NEXT:    # xmm0 = (xmm1 * xmm0) + mem
-; X64-NEXT:    retq # encoding: [0xc3]
-  %a2 = load <4 x float>, <4 x float>* %ptr_a2
-  %res = call <4 x float> @llvm.x86.avx512.mask.vfmadd.ps.128(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, i8 %mask) nounwind
-  ret <4 x float> %res
+define <8 x float> @test_fmaddsub256_ps(<8 x float> %a, <8 x float> %b, <8 x float> %c) {
+; CHECK-LABEL: test_fmaddsub256_ps:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vfmaddsub213ps %ymm2, %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x75,0xa6,0xc2]
+; CHECK-NEXT:    # ymm0 = (ymm1 * ymm0) +/- ymm2
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <8 x float> @llvm.x86.avx512.mask.vfmaddsub.ps.256(<8 x float> %a, <8 x float> %b, <8 x float> %c, i8 -1)
+  ret <8 x float> %res
 }
 
-define <4 x float> @test_mask_vfmadd128_ps_rmka(<4 x float> %a0, <4 x float> %a1, <4 x float>* %ptr_a2, i8 %mask) {
-; X86-LABEL: test_mask_vfmadd128_ps_rmka:
+define <8 x float> @test_mask_fmaddsub256_ps(<8 x float> %a, <8 x float> %b, <8 x float> %c, i8 %mask) {
+; X86-LABEL: test_mask_fmaddsub256_ps:
 ; X86:       # %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
-; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx # encoding: [0x0f,0xb6,0x4c,0x24,0x08]
-; X86-NEXT:    kmovw %ecx, %k1 # encoding: [0xc5,0xf8,0x92,0xc9]
-; X86-NEXT:    vfmadd213ps (%eax), %xmm1, %xmm0 {%k1} # encoding: [0x62,0xf2,0x75,0x09,0xa8,0x00]
-; X86-NEXT:    # xmm0 = (xmm1 * xmm0) + mem
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vfmaddsub132ps %ymm1, %ymm2, %ymm0 {%k1} # encoding: [0x62,0xf2,0x6d,0x29,0x96,0xc1]
+; X86-NEXT:    # ymm0 = (ymm0 * ymm1) +/- ymm2
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
-; X64-LABEL: test_mask_vfmadd128_ps_rmka:
+; X64-LABEL: test_mask_fmaddsub256_ps:
 ; X64:       # %bb.0:
-; X64-NEXT:    kmovw %esi, %k1 # encoding: [0xc5,0xf8,0x92,0xce]
-; X64-NEXT:    vfmadd213ps (%rdi), %xmm1, %xmm0 {%k1} # encoding: [0x62,0xf2,0x75,0x09,0xa8,0x07]
-; X64-NEXT:    # xmm0 = (xmm1 * xmm0) + mem
+; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vfmaddsub132ps %ymm1, %ymm2, %ymm0 {%k1} # encoding: [0x62,0xf2,0x6d,0x29,0x96,0xc1]
+; X64-NEXT:    # ymm0 = (ymm0 * ymm1) +/- ymm2
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %a2 = load <4 x float>, <4 x float>* %ptr_a2, align 8
-  %res = call <4 x float> @llvm.x86.avx512.mask.vfmadd.ps.128(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, i8 %mask) nounwind
+  %res = call <8 x float> @llvm.x86.avx512.mask.vfmaddsub.ps.256(<8 x float> %a, <8 x float> %b, <8 x float> %c, i8 %mask)
+  ret <8 x float> %res
+}
+
+declare <4 x float> @llvm.x86.avx512.mask.vfmaddsub.ps.128(<4 x float>, <4 x float>, <4 x float>, i8) nounwind readnone
+
+define <4 x float> @test_fmaddsub128_ps(<4 x float> %a, <4 x float> %b, <4 x float> %c) {
+; CHECK-LABEL: test_fmaddsub128_ps:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vfmaddsub213ps %xmm2, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x71,0xa6,0xc2]
+; CHECK-NEXT:    # xmm0 = (xmm1 * xmm0) +/- xmm2
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <4 x float> @llvm.x86.avx512.mask.vfmaddsub.ps.128(<4 x float> %a, <4 x float> %b, <4 x float> %c, i8 -1)
   ret <4 x float> %res
 }
 
-define <4 x float> @test_mask_vfmadd128_ps_rmkz(<4 x float> %a0, <4 x float> %a1, <4 x float>* %ptr_a2) {
-; X86-LABEL: test_mask_vfmadd128_ps_rmkz:
+define <4 x float> @test_mask_fmaddsub128_ps(<4 x float> %a, <4 x float> %b, <4 x float> %c, i8 %mask) {
+; X86-LABEL: test_mask_fmaddsub128_ps:
 ; X86:       # %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
-; X86-NEXT:    vfmadd213ps (%eax), %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x71,0xa8,0x00]
-; X86-NEXT:    # xmm0 = (xmm1 * xmm0) + mem
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vfmaddsub132ps %xmm1, %xmm2, %xmm0 {%k1} # encoding: [0x62,0xf2,0x6d,0x09,0x96,0xc1]
+; X86-NEXT:    # xmm0 = (xmm0 * xmm1) +/- xmm2
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
-; X64-LABEL: test_mask_vfmadd128_ps_rmkz:
+; X64-LABEL: test_mask_fmaddsub128_ps:
 ; X64:       # %bb.0:
-; X64-NEXT:    vfmadd213ps (%rdi), %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x71,0xa8,0x07]
-; X64-NEXT:    # xmm0 = (xmm1 * xmm0) + mem
+; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vfmaddsub132ps %xmm1, %xmm2, %xmm0 {%k1} # encoding: [0x62,0xf2,0x6d,0x09,0x96,0xc1]
+; X64-NEXT:    # xmm0 = (xmm0 * xmm1) +/- xmm2
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %a2 = load <4 x float>, <4 x float>* %ptr_a2
-  %res = call <4 x float> @llvm.x86.avx512.mask.vfmadd.ps.128(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, i8 -1) nounwind
+  %res = call <4 x float> @llvm.x86.avx512.mask.vfmaddsub.ps.128(<4 x float> %a, <4 x float> %b, <4 x float> %c, i8 %mask)
   ret <4 x float> %res
 }
 
-define <4 x float> @test_mask_vfmadd128_ps_rmkza(<4 x float> %a0, <4 x float> %a1, <4 x float>* %ptr_a2) {
-; X86-LABEL: test_mask_vfmadd128_ps_rmkza:
+declare <4 x double> @llvm.x86.avx512.mask.vfmaddsub.pd.256(<4 x double>, <4 x double>, <4 x double>, i8) nounwind readnone
+
+define <4 x double> @test_vfmaddsub256_pd(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2) {
+; CHECK-LABEL: test_vfmaddsub256_pd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vfmaddsub213pd %ymm2, %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0xf5,0xa6,0xc2]
+; CHECK-NEXT:    # ymm0 = (ymm1 * ymm0) +/- ymm2
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <4 x double> @llvm.x86.avx512.mask.vfmaddsub.pd.256(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2, i8 -1) nounwind
+  ret <4 x double> %res
+}
+
+define <4 x double> @test_mask_vfmaddsub256_pd(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2, i8 %mask) {
+; X86-LABEL: test_mask_vfmaddsub256_pd:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vfmaddsub132pd %ymm1, %ymm2, %ymm0 {%k1} # encoding: [0x62,0xf2,0xed,0x29,0x96,0xc1]
+; X86-NEXT:    # ymm0 = (ymm0 * ymm1) +/- ymm2
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_mask_vfmaddsub256_pd:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vfmaddsub132pd %ymm1, %ymm2, %ymm0 {%k1} # encoding: [0x62,0xf2,0xed,0x29,0x96,0xc1]
+; X64-NEXT:    # ymm0 = (ymm0 * ymm1) +/- ymm2
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <4 x double> @llvm.x86.avx512.mask.vfmaddsub.pd.256(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2, i8 %mask) nounwind
+  ret <4 x double> %res
+}
+
+declare <2 x double> @llvm.x86.avx512.mask.vfmaddsub.pd.128(<2 x double>, <2 x double>, <2 x double>, i8) nounwind readnone
+
+define <2 x double> @test_vfmaddsub128_pd(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2) {
+; CHECK-LABEL: test_vfmaddsub128_pd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vfmaddsub213pd %xmm2, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0xf1,0xa6,0xc2]
+; CHECK-NEXT:    # xmm0 = (xmm1 * xmm0) +/- xmm2
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <2 x double> @llvm.x86.avx512.mask.vfmaddsub.pd.128(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, i8 -1) nounwind
+  ret <2 x double> %res
+}
+
+define <2 x double> @test_mask_vfmaddsub128_pd(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, i8 %mask) {
+; X86-LABEL: test_mask_vfmaddsub128_pd:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vfmaddsub132pd %xmm1, %xmm2, %xmm0 {%k1} # encoding: [0x62,0xf2,0xed,0x09,0x96,0xc1]
+; X86-NEXT:    # xmm0 = (xmm0 * xmm1) +/- xmm2
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_mask_vfmaddsub128_pd:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vfmaddsub132pd %xmm1, %xmm2, %xmm0 {%k1} # encoding: [0x62,0xf2,0xed,0x09,0x96,0xc1]
+; X64-NEXT:    # xmm0 = (xmm0 * xmm1) +/- xmm2
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <2 x double> @llvm.x86.avx512.mask.vfmaddsub.pd.128(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, i8 %mask) nounwind
+  ret <2 x double> %res
+}
+
+declare <2 x double> @llvm.x86.avx512.mask3.vfmaddsub.pd.128(<2 x double>, <2 x double>, <2 x double>, i8)
+
+define <2 x double>@test_int_x86_avx512_mask3_vfmaddsub_pd_128(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_mask3_vfmaddsub_pd_128:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vfmaddsub231pd %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x09,0xb6,0xd1]
+; X86-NEXT:    # xmm2 = (xmm0 * xmm1) +/- xmm2
+; X86-NEXT:    vmovapd %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x28,0xc2]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_mask3_vfmaddsub_pd_128:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vfmaddsub231pd %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x09,0xb6,0xd1]
+; X64-NEXT:    # xmm2 = (xmm0 * xmm1) +/- xmm2
+; X64-NEXT:    vmovapd %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x28,0xc2]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <2 x double> @llvm.x86.avx512.mask3.vfmaddsub.pd.128(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3)
+  ret <2 x double> %res
+}
+
+declare <2 x double> @llvm.x86.avx512.maskz.vfmaddsub.pd.128(<2 x double>, <2 x double>, <2 x double>, i8)
+
+define <2 x double>@test_int_x86_avx512_maskz_vfmaddsub_pd_128(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_maskz_vfmaddsub_pd_128:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vfmaddsub213pd %xmm2, %xmm1, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0x89,0xa6,0xc2]
+; X86-NEXT:    # xmm0 = (xmm1 * xmm0) +/- xmm2
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_maskz_vfmaddsub_pd_128:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vfmaddsub213pd %xmm2, %xmm1, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0x89,0xa6,0xc2]
+; X64-NEXT:    # xmm0 = (xmm1 * xmm0) +/- xmm2
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <2 x double> @llvm.x86.avx512.maskz.vfmaddsub.pd.128(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3)
+  ret <2 x double> %res
+}
+
+declare <4 x double> @llvm.x86.avx512.mask3.vfmaddsub.pd.256(<4 x double>, <4 x double>, <4 x double>, i8)
+
+define <4 x double>@test_int_x86_avx512_mask3_vfmaddsub_pd_256(<4 x double> %x0, <4 x double> %x1, <4 x double> %x2, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_mask3_vfmaddsub_pd_256:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vfmaddsub231pd %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x29,0xb6,0xd1]
+; X86-NEXT:    # ymm2 = (ymm0 * ymm1) +/- ymm2
+; X86-NEXT:    vmovapd %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x28,0xc2]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_mask3_vfmaddsub_pd_256:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vfmaddsub231pd %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x29,0xb6,0xd1]
+; X64-NEXT:    # ymm2 = (ymm0 * ymm1) +/- ymm2
+; X64-NEXT:    vmovapd %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x28,0xc2]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <4 x double> @llvm.x86.avx512.mask3.vfmaddsub.pd.256(<4 x double> %x0, <4 x double> %x1, <4 x double> %x2, i8 %x3)
+  ret <4 x double> %res
+}
+
+declare <4 x double> @llvm.x86.avx512.maskz.vfmaddsub.pd.256(<4 x double>, <4 x double>, <4 x double>, i8)
+
+define <4 x double>@test_int_x86_avx512_maskz_vfmaddsub_pd_256(<4 x double> %x0, <4 x double> %x1, <4 x double> %x2, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_maskz_vfmaddsub_pd_256:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vfmaddsub213pd %ymm2, %ymm1, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xa9,0xa6,0xc2]
+; X86-NEXT:    # ymm0 = (ymm1 * ymm0) +/- ymm2
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_maskz_vfmaddsub_pd_256:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vfmaddsub213pd %ymm2, %ymm1, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xf5,0xa9,0xa6,0xc2]
+; X64-NEXT:    # ymm0 = (ymm1 * ymm0) +/- ymm2
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <4 x double> @llvm.x86.avx512.maskz.vfmaddsub.pd.256(<4 x double> %x0, <4 x double> %x1, <4 x double> %x2, i8 %x3)
+  ret <4 x double> %res
+}
+
+declare <4 x float> @llvm.x86.avx512.mask3.vfmaddsub.ps.128(<4 x float>, <4 x float>, <4 x float>, i8)
+
+define <4 x float>@test_int_x86_avx512_mask3_vfmaddsub_ps_128(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_mask3_vfmaddsub_ps_128:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vfmaddsub231ps %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf2,0x7d,0x09,0xb6,0xd1]
+; X86-NEXT:    # xmm2 = (xmm0 * xmm1) +/- xmm2
+; X86-NEXT:    vmovaps %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf8,0x28,0xc2]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_mask3_vfmaddsub_ps_128:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vfmaddsub231ps %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf2,0x7d,0x09,0xb6,0xd1]
+; X64-NEXT:    # xmm2 = (xmm0 * xmm1) +/- xmm2
+; X64-NEXT:    vmovaps %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf8,0x28,0xc2]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <4 x float> @llvm.x86.avx512.mask3.vfmaddsub.ps.128(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3)
+  ret <4 x float> %res
+}
+
+declare <4 x float> @llvm.x86.avx512.maskz.vfmaddsub.ps.128(<4 x float>, <4 x float>, <4 x float>, i8)
+
+define <4 x float>@test_int_x86_avx512_maskz_vfmaddsub_ps_128(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_maskz_vfmaddsub_ps_128:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vfmaddsub213ps %xmm2, %xmm1, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0x89,0xa6,0xc2]
+; X86-NEXT:    # xmm0 = (xmm1 * xmm0) +/- xmm2
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_maskz_vfmaddsub_ps_128:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vfmaddsub213ps %xmm2, %xmm1, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0x89,0xa6,0xc2]
+; X64-NEXT:    # xmm0 = (xmm1 * xmm0) +/- xmm2
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <4 x float> @llvm.x86.avx512.maskz.vfmaddsub.ps.128(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3)
+  ret <4 x float> %res
+}
+
+declare <8 x float> @llvm.x86.avx512.mask3.vfmaddsub.ps.256(<8 x float>, <8 x float>, <8 x float>, i8)
+
+define <8 x float>@test_int_x86_avx512_mask3_vfmaddsub_ps_256(<8 x float> %x0, <8 x float> %x1, <8 x float> %x2, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_mask3_vfmaddsub_ps_256:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vfmaddsub231ps %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf2,0x7d,0x29,0xb6,0xd1]
+; X86-NEXT:    # ymm2 = (ymm0 * ymm1) +/- ymm2
+; X86-NEXT:    vmovaps %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfc,0x28,0xc2]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_mask3_vfmaddsub_ps_256:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vfmaddsub231ps %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf2,0x7d,0x29,0xb6,0xd1]
+; X64-NEXT:    # ymm2 = (ymm0 * ymm1) +/- ymm2
+; X64-NEXT:    vmovaps %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfc,0x28,0xc2]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <8 x float> @llvm.x86.avx512.mask3.vfmaddsub.ps.256(<8 x float> %x0, <8 x float> %x1, <8 x float> %x2, i8 %x3)
+  ret <8 x float> %res
+}
+
+declare <8 x float> @llvm.x86.avx512.maskz.vfmaddsub.ps.256(<8 x float>, <8 x float>, <8 x float>, i8)
+
+define <8 x float>@test_int_x86_avx512_maskz_vfmaddsub_ps_256(<8 x float> %x0, <8 x float> %x1, <8 x float> %x2, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_maskz_vfmaddsub_ps_256:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vfmaddsub213ps %ymm2, %ymm1, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0xa9,0xa6,0xc2]
+; X86-NEXT:    # ymm0 = (ymm1 * ymm0) +/- ymm2
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_maskz_vfmaddsub_ps_256:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vfmaddsub213ps %ymm2, %ymm1, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0x75,0xa9,0xa6,0xc2]
+; X64-NEXT:    # ymm0 = (ymm1 * ymm0) +/- ymm2
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <8 x float> @llvm.x86.avx512.maskz.vfmaddsub.ps.256(<8 x float> %x0, <8 x float> %x1, <8 x float> %x2, i8 %x3)
+  ret <8 x float> %res
+}
+
+declare <2 x double> @llvm.x86.avx512.mask3.vfmsubadd.pd.128(<2 x double>, <2 x double>, <2 x double>, i8)
+
+define <2 x double>@test_int_x86_avx512_mask3_vfmsubadd_pd_128(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_mask3_vfmsubadd_pd_128:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vfmsubadd231pd %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x09,0xb7,0xd1]
+; X86-NEXT:    # xmm2 = (xmm0 * xmm1) -/+ xmm2
+; X86-NEXT:    vmovapd %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x28,0xc2]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_mask3_vfmsubadd_pd_128:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vfmsubadd231pd %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x09,0xb7,0xd1]
+; X64-NEXT:    # xmm2 = (xmm0 * xmm1) -/+ xmm2
+; X64-NEXT:    vmovapd %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x28,0xc2]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <2 x double> @llvm.x86.avx512.mask3.vfmsubadd.pd.128(<2 x double> %x0, <2 x double> %x1, <2 x double> %x2, i8 %x3)
+  ret <2 x double> %res
+}
+
+declare <4 x double> @llvm.x86.avx512.mask3.vfmsubadd.pd.256(<4 x double>, <4 x double>, <4 x double>, i8)
+
+define <4 x double>@test_int_x86_avx512_mask3_vfmsubadd_pd_256(<4 x double> %x0, <4 x double> %x1, <4 x double> %x2, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_mask3_vfmsubadd_pd_256:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vfmsubadd231pd %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x29,0xb7,0xd1]
+; X86-NEXT:    # ymm2 = (ymm0 * ymm1) -/+ ymm2
+; X86-NEXT:    vmovapd %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x28,0xc2]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_mask3_vfmsubadd_pd_256:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vfmsubadd231pd %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x29,0xb7,0xd1]
+; X64-NEXT:    # ymm2 = (ymm0 * ymm1) -/+ ymm2
+; X64-NEXT:    vmovapd %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x28,0xc2]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <4 x double> @llvm.x86.avx512.mask3.vfmsubadd.pd.256(<4 x double> %x0, <4 x double> %x1, <4 x double> %x2, i8 %x3)
+  ret <4 x double> %res
+}
+
+declare <4 x float> @llvm.x86.avx512.mask3.vfmsubadd.ps.128(<4 x float>, <4 x float>, <4 x float>, i8)
+
+define <4 x float>@test_int_x86_avx512_mask3_vfmsubadd_ps_128(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_mask3_vfmsubadd_ps_128:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vfmsubadd231ps %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf2,0x7d,0x09,0xb7,0xd1]
+; X86-NEXT:    # xmm2 = (xmm0 * xmm1) -/+ xmm2
+; X86-NEXT:    vmovaps %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf8,0x28,0xc2]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_mask3_vfmsubadd_ps_128:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vfmsubadd231ps %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf2,0x7d,0x09,0xb7,0xd1]
+; X64-NEXT:    # xmm2 = (xmm0 * xmm1) -/+ xmm2
+; X64-NEXT:    vmovaps %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf8,0x28,0xc2]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <4 x float> @llvm.x86.avx512.mask3.vfmsubadd.ps.128(<4 x float> %x0, <4 x float> %x1, <4 x float> %x2, i8 %x3)
+  ret <4 x float> %res
+}
+
+declare <8 x float> @llvm.x86.avx512.mask3.vfmsubadd.ps.256(<8 x float>, <8 x float>, <8 x float>, i8)
+
+define <8 x float>@test_int_x86_avx512_mask3_vfmsubadd_ps_256(<8 x float> %x0, <8 x float> %x1, <8 x float> %x2, i8 %x3) {
+; X86-LABEL: test_int_x86_avx512_mask3_vfmsubadd_ps_256:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vfmsubadd231ps %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf2,0x7d,0x29,0xb7,0xd1]
+; X86-NEXT:    # ymm2 = (ymm0 * ymm1) -/+ ymm2
+; X86-NEXT:    vmovaps %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfc,0x28,0xc2]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_mask3_vfmsubadd_ps_256:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vfmsubadd231ps %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf2,0x7d,0x29,0xb7,0xd1]
+; X64-NEXT:    # ymm2 = (ymm0 * ymm1) -/+ ymm2
+; X64-NEXT:    vmovaps %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfc,0x28,0xc2]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <8 x float> @llvm.x86.avx512.mask3.vfmsubadd.ps.256(<8 x float> %x0, <8 x float> %x1, <8 x float> %x2, i8 %x3)
+  ret <8 x float> %res
+}
+
+
+define <4 x float> @test_mask_vfmadd128_ps_rmk(<4 x float> %a0, <4 x float> %a1, <4 x float>* %ptr_a2, i8 %mask) {
+; X86-LABEL: test_mask_vfmadd128_ps_rmk:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx # encoding: [0x0f,0xb6,0x4c,0x24,0x08]
+; X86-NEXT:    kmovw %ecx, %k1 # encoding: [0xc5,0xf8,0x92,0xc9]
+; X86-NEXT:    vfmadd213ps (%eax), %xmm1, %xmm0 {%k1} # encoding: [0x62,0xf2,0x75,0x09,0xa8,0x00]
+; X86-NEXT:    # xmm0 = (xmm1 * xmm0) + mem
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_mask_vfmadd128_ps_rmk:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %esi, %k1 # encoding: [0xc5,0xf8,0x92,0xce]
+; X64-NEXT:    vfmadd213ps (%rdi), %xmm1, %xmm0 {%k1} # encoding: [0x62,0xf2,0x75,0x09,0xa8,0x07]
+; X64-NEXT:    # xmm0 = (xmm1 * xmm0) + mem
+; X64-NEXT:    retq # encoding: [0xc3]
+  %a2 = load <4 x float>, <4 x float>* %ptr_a2
+  %res = call <4 x float> @llvm.x86.avx512.mask.vfmadd.ps.128(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, i8 %mask) nounwind
+  ret <4 x float> %res
+}
+
+define <4 x float> @test_mask_vfmadd128_ps_rmka(<4 x float> %a0, <4 x float> %a1, <4 x float>* %ptr_a2, i8 %mask) {
+; X86-LABEL: test_mask_vfmadd128_ps_rmka:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx # encoding: [0x0f,0xb6,0x4c,0x24,0x08]
+; X86-NEXT:    kmovw %ecx, %k1 # encoding: [0xc5,0xf8,0x92,0xc9]
+; X86-NEXT:    vfmadd213ps (%eax), %xmm1, %xmm0 {%k1} # encoding: [0x62,0xf2,0x75,0x09,0xa8,0x00]
+; X86-NEXT:    # xmm0 = (xmm1 * xmm0) + mem
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_mask_vfmadd128_ps_rmka:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %esi, %k1 # encoding: [0xc5,0xf8,0x92,0xce]
+; X64-NEXT:    vfmadd213ps (%rdi), %xmm1, %xmm0 {%k1} # encoding: [0x62,0xf2,0x75,0x09,0xa8,0x07]
+; X64-NEXT:    # xmm0 = (xmm1 * xmm0) + mem
+; X64-NEXT:    retq # encoding: [0xc3]
+  %a2 = load <4 x float>, <4 x float>* %ptr_a2, align 8
+  %res = call <4 x float> @llvm.x86.avx512.mask.vfmadd.ps.128(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, i8 %mask) nounwind
+  ret <4 x float> %res
+}
+
+define <4 x float> @test_mask_vfmadd128_ps_rmkz(<4 x float> %a0, <4 x float> %a1, <4 x float>* %ptr_a2) {
+; X86-LABEL: test_mask_vfmadd128_ps_rmkz:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-NEXT:    vfmadd213ps (%eax), %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x71,0xa8,0x00]
+; X86-NEXT:    # xmm0 = (xmm1 * xmm0) + mem
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_mask_vfmadd128_ps_rmkz:
+; X64:       # %bb.0:
+; X64-NEXT:    vfmadd213ps (%rdi), %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x71,0xa8,0x07]
+; X64-NEXT:    # xmm0 = (xmm1 * xmm0) + mem
+; X64-NEXT:    retq # encoding: [0xc3]
+  %a2 = load <4 x float>, <4 x float>* %ptr_a2
+  %res = call <4 x float> @llvm.x86.avx512.mask.vfmadd.ps.128(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, i8 -1) nounwind
+  ret <4 x float> %res
+}
+
+define <4 x float> @test_mask_vfmadd128_ps_rmkza(<4 x float> %a0, <4 x float> %a1, <4 x float>* %ptr_a2) {
+; X86-LABEL: test_mask_vfmadd128_ps_rmkza:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-NEXT:    vfmadd213ps (%eax), %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x71,0xa8,0x00]
+; X86-NEXT:    # xmm0 = (xmm1 * xmm0) + mem
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_mask_vfmadd128_ps_rmkza:
+; X64:       # %bb.0:
+; X64-NEXT:    vfmadd213ps (%rdi), %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x71,0xa8,0x07]
+; X64-NEXT:    # xmm0 = (xmm1 * xmm0) + mem
+; X64-NEXT:    retq # encoding: [0xc3]
+  %a2 = load <4 x float>, <4 x float>* %ptr_a2, align 4
+  %res = call <4 x float> @llvm.x86.avx512.mask.vfmadd.ps.128(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, i8 -1) nounwind
+  ret <4 x float> %res
+}
+
+define <4 x float> @test_mask_vfmadd128_ps_rmb(<4 x float> %a0, <4 x float> %a1, float* %ptr_a2, i8 %mask) {
+; X86-LABEL: test_mask_vfmadd128_ps_rmb:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx # encoding: [0x0f,0xb6,0x4c,0x24,0x08]
+; X86-NEXT:    kmovw %ecx, %k1 # encoding: [0xc5,0xf8,0x92,0xc9]
+; X86-NEXT:    vfmadd213ps (%eax){1to4}, %xmm1, %xmm0 {%k1} # encoding: [0x62,0xf2,0x75,0x19,0xa8,0x00]
+; X86-NEXT:    # xmm0 = (xmm1 * xmm0) + mem
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_mask_vfmadd128_ps_rmb:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %esi, %k1 # encoding: [0xc5,0xf8,0x92,0xce]
+; X64-NEXT:    vfmadd213ps (%rdi){1to4}, %xmm1, %xmm0 {%k1} # encoding: [0x62,0xf2,0x75,0x19,0xa8,0x07]
+; X64-NEXT:    # xmm0 = (xmm1 * xmm0) + mem
+; X64-NEXT:    retq # encoding: [0xc3]
+  %q = load float, float* %ptr_a2
+  %vecinit.i = insertelement <4 x float> undef, float %q, i32 0
+  %vecinit2.i = insertelement <4 x float> %vecinit.i, float %q, i32 1
+  %vecinit4.i = insertelement <4 x float> %vecinit2.i, float %q, i32 2
+  %vecinit6.i = insertelement <4 x float> %vecinit4.i, float %q, i32 3
+  %res = call <4 x float> @llvm.x86.avx512.mask.vfmadd.ps.128(<4 x float> %a0, <4 x float> %a1, <4 x float> %vecinit6.i, i8 %mask) nounwind
+  ret <4 x float> %res
+}
+
+define <4 x float> @test_mask_vfmadd128_ps_rmba(<4 x float> %a0, <4 x float> %a1, float* %ptr_a2, i8 %mask) {
+; X86-LABEL: test_mask_vfmadd128_ps_rmba:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx # encoding: [0x0f,0xb6,0x4c,0x24,0x08]
+; X86-NEXT:    kmovw %ecx, %k1 # encoding: [0xc5,0xf8,0x92,0xc9]
+; X86-NEXT:    vfmadd213ps (%eax){1to4}, %xmm1, %xmm0 {%k1} # encoding: [0x62,0xf2,0x75,0x19,0xa8,0x00]
+; X86-NEXT:    # xmm0 = (xmm1 * xmm0) + mem
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_mask_vfmadd128_ps_rmba:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %esi, %k1 # encoding: [0xc5,0xf8,0x92,0xce]
+; X64-NEXT:    vfmadd213ps (%rdi){1to4}, %xmm1, %xmm0 {%k1} # encoding: [0x62,0xf2,0x75,0x19,0xa8,0x07]
+; X64-NEXT:    # xmm0 = (xmm1 * xmm0) + mem
+; X64-NEXT:    retq # encoding: [0xc3]
+  %q = load float, float* %ptr_a2, align 4
+  %vecinit.i = insertelement <4 x float> undef, float %q, i32 0
+  %vecinit2.i = insertelement <4 x float> %vecinit.i, float %q, i32 1
+  %vecinit4.i = insertelement <4 x float> %vecinit2.i, float %q, i32 2
+  %vecinit6.i = insertelement <4 x float> %vecinit4.i, float %q, i32 3
+  %res = call <4 x float> @llvm.x86.avx512.mask.vfmadd.ps.128(<4 x float> %a0, <4 x float> %a1, <4 x float> %vecinit6.i, i8 %mask) nounwind
+  ret <4 x float> %res
+}
+
+define <4 x float> @test_mask_vfmadd128_ps_rmbz(<4 x float> %a0, <4 x float> %a1, float* %ptr_a2) {
+; X86-LABEL: test_mask_vfmadd128_ps_rmbz:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-NEXT:    vfmadd213ps (%eax){1to4}, %xmm1, %xmm0 # encoding: [0x62,0xf2,0x75,0x18,0xa8,0x00]
+; X86-NEXT:    # xmm0 = (xmm1 * xmm0) + mem
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_mask_vfmadd128_ps_rmbz:
+; X64:       # %bb.0:
+; X64-NEXT:    vfmadd213ps (%rdi){1to4}, %xmm1, %xmm0 # encoding: [0x62,0xf2,0x75,0x18,0xa8,0x07]
+; X64-NEXT:    # xmm0 = (xmm1 * xmm0) + mem
+; X64-NEXT:    retq # encoding: [0xc3]
+  %q = load float, float* %ptr_a2
+  %vecinit.i = insertelement <4 x float> undef, float %q, i32 0
+  %vecinit2.i = insertelement <4 x float> %vecinit.i, float %q, i32 1
+  %vecinit4.i = insertelement <4 x float> %vecinit2.i, float %q, i32 2
+  %vecinit6.i = insertelement <4 x float> %vecinit4.i, float %q, i32 3
+  %res = call <4 x float> @llvm.x86.avx512.mask.vfmadd.ps.128(<4 x float> %a0, <4 x float> %a1, <4 x float> %vecinit6.i, i8 -1) nounwind
+  ret <4 x float> %res
+}
+
+define <4 x float> @test_mask_vfmadd128_ps_rmbza(<4 x float> %a0, <4 x float> %a1, float* %ptr_a2) {
+; X86-LABEL: test_mask_vfmadd128_ps_rmbza:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-NEXT:    vfmadd213ps (%eax){1to4}, %xmm1, %xmm0 # encoding: [0x62,0xf2,0x75,0x18,0xa8,0x00]
+; X86-NEXT:    # xmm0 = (xmm1 * xmm0) + mem
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_mask_vfmadd128_ps_rmbza:
+; X64:       # %bb.0:
+; X64-NEXT:    vfmadd213ps (%rdi){1to4}, %xmm1, %xmm0 # encoding: [0x62,0xf2,0x75,0x18,0xa8,0x07]
+; X64-NEXT:    # xmm0 = (xmm1 * xmm0) + mem
+; X64-NEXT:    retq # encoding: [0xc3]
+  %q = load float, float* %ptr_a2, align 4
+  %vecinit.i = insertelement <4 x float> undef, float %q, i32 0
+  %vecinit2.i = insertelement <4 x float> %vecinit.i, float %q, i32 1
+  %vecinit4.i = insertelement <4 x float> %vecinit2.i, float %q, i32 2
+  %vecinit6.i = insertelement <4 x float> %vecinit4.i, float %q, i32 3
+  %res = call <4 x float> @llvm.x86.avx512.mask.vfmadd.ps.128(<4 x float> %a0, <4 x float> %a1, <4 x float> %vecinit6.i, i8 -1) nounwind
+  ret <4 x float> %res
+}
+
+define <2 x double> @test_mask_vfmadd128_pd_rmk(<2 x double> %a0, <2 x double> %a1, <2 x double>* %ptr_a2, i8 %mask) {
+; X86-LABEL: test_mask_vfmadd128_pd_rmk:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx # encoding: [0x0f,0xb6,0x4c,0x24,0x08]
+; X86-NEXT:    kmovw %ecx, %k1 # encoding: [0xc5,0xf8,0x92,0xc9]
+; X86-NEXT:    vfmadd213pd (%eax), %xmm1, %xmm0 {%k1} # encoding: [0x62,0xf2,0xf5,0x09,0xa8,0x00]
+; X86-NEXT:    # xmm0 = (xmm1 * xmm0) + mem
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_mask_vfmadd128_pd_rmk:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %esi, %k1 # encoding: [0xc5,0xf8,0x92,0xce]
+; X64-NEXT:    vfmadd213pd (%rdi), %xmm1, %xmm0 {%k1} # encoding: [0x62,0xf2,0xf5,0x09,0xa8,0x07]
+; X64-NEXT:    # xmm0 = (xmm1 * xmm0) + mem
+; X64-NEXT:    retq # encoding: [0xc3]
+  %a2 = load <2 x double>, <2 x double>* %ptr_a2
+  %res = call <2 x double> @llvm.x86.avx512.mask.vfmadd.pd.128(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, i8 %mask) nounwind
+  ret <2 x double> %res
+}
+
+define <2 x double> @test_mask_vfmadd128_pd_rmkz(<2 x double> %a0, <2 x double> %a1, <2 x double>* %ptr_a2) {
+; X86-LABEL: test_mask_vfmadd128_pd_rmkz:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-NEXT:    vfmadd213pd (%eax), %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0xf1,0xa8,0x00]
+; X86-NEXT:    # xmm0 = (xmm1 * xmm0) + mem
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_mask_vfmadd128_pd_rmkz:
+; X64:       # %bb.0:
+; X64-NEXT:    vfmadd213pd (%rdi), %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0xf1,0xa8,0x07]
+; X64-NEXT:    # xmm0 = (xmm1 * xmm0) + mem
+; X64-NEXT:    retq # encoding: [0xc3]
+  %a2 = load <2 x double>, <2 x double>* %ptr_a2
+  %res = call <2 x double> @llvm.x86.avx512.mask.vfmadd.pd.128(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, i8 -1) nounwind
+  ret <2 x double> %res
+}
+
+define <4 x double> @test_mask_vfmadd256_pd_rmk(<4 x double> %a0, <4 x double> %a1, <4 x double>* %ptr_a2, i8 %mask) {
+; X86-LABEL: test_mask_vfmadd256_pd_rmk:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx # encoding: [0x0f,0xb6,0x4c,0x24,0x08]
+; X86-NEXT:    kmovw %ecx, %k1 # encoding: [0xc5,0xf8,0x92,0xc9]
+; X86-NEXT:    vfmadd213pd (%eax), %ymm1, %ymm0 {%k1} # encoding: [0x62,0xf2,0xf5,0x29,0xa8,0x00]
+; X86-NEXT:    # ymm0 = (ymm1 * ymm0) + mem
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_mask_vfmadd256_pd_rmk:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %esi, %k1 # encoding: [0xc5,0xf8,0x92,0xce]
+; X64-NEXT:    vfmadd213pd (%rdi), %ymm1, %ymm0 {%k1} # encoding: [0x62,0xf2,0xf5,0x29,0xa8,0x07]
+; X64-NEXT:    # ymm0 = (ymm1 * ymm0) + mem
+; X64-NEXT:    retq # encoding: [0xc3]
+  %a2 = load <4 x double>, <4 x double>* %ptr_a2
+  %res = call <4 x double> @llvm.x86.avx512.mask.vfmadd.pd.256(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2, i8 %mask) nounwind
+  ret <4 x double> %res
+}
+
+define <4 x double> @test_mask_vfmadd256_pd_rmkz(<4 x double> %a0, <4 x double> %a1, <4 x double>* %ptr_a2) {
+; X86-LABEL: test_mask_vfmadd256_pd_rmkz:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-NEXT:    vfmadd213pd (%eax), %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0xf5,0xa8,0x00]
+; X86-NEXT:    # ymm0 = (ymm1 * ymm0) + mem
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_mask_vfmadd256_pd_rmkz:
+; X64:       # %bb.0:
+; X64-NEXT:    vfmadd213pd (%rdi), %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0xf5,0xa8,0x07]
+; X64-NEXT:    # ymm0 = (ymm1 * ymm0) + mem
+; X64-NEXT:    retq # encoding: [0xc3]
+  %a2 = load <4 x double>, <4 x double>* %ptr_a2
+  %res = call <4 x double> @llvm.x86.avx512.mask.vfmadd.pd.256(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2, i8 -1) nounwind
+  ret <4 x double> %res
+}
+
+declare <4 x i32> @llvm.x86.avx512.mask.pmov.qd.256(<4 x i64>, <4 x i32>, i8)
+
+define <4 x i32>@test_int_x86_avx512_mask_pmov_qd_256(<4 x i64> %x0, <4 x i32> %x1, i8 %x2) {
+; X86-LABEL: test_int_x86_avx512_mask_pmov_qd_256:
+; X86:       # %bb.0:
+; X86-NEXT:    vpmovqd %ymm0, %xmm2 # encoding: [0x62,0xf2,0x7e,0x28,0x35,0xc2]
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vpmovqd %ymm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0x7e,0x29,0x35,0xc1]
+; X86-NEXT:    vpmovqd %ymm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0xa9,0x35,0xc0]
+; X86-NEXT:    vpaddd %xmm0, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0xfe,0xc0]
+; X86-NEXT:    vpaddd %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfe,0xc0]
+; X86-NEXT:    vzeroupper # encoding: [0xc5,0xf8,0x77]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_int_x86_avx512_mask_pmov_qd_256:
+; X64:       # %bb.0:
+; X64-NEXT:    vpmovqd %ymm0, %xmm2 # encoding: [0x62,0xf2,0x7e,0x28,0x35,0xc2]
+; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vpmovqd %ymm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0x7e,0x29,0x35,0xc1]
+; X64-NEXT:    vpmovqd %ymm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0xa9,0x35,0xc0]
+; X64-NEXT:    vpaddd %xmm0, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0xfe,0xc0]
+; X64-NEXT:    vpaddd %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfe,0xc0]
+; X64-NEXT:    vzeroupper # encoding: [0xc5,0xf8,0x77]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res0 = call <4 x i32> @llvm.x86.avx512.mask.pmov.qd.256(<4 x i64> %x0, <4 x i32> %x1, i8 -1)
+  %res1 = call <4 x i32> @llvm.x86.avx512.mask.pmov.qd.256(<4 x i64> %x0, <4 x i32> %x1, i8 %x2)
+  %res2 = call <4 x i32> @llvm.x86.avx512.mask.pmov.qd.256(<4 x i64> %x0, <4 x i32> zeroinitializer, i8 %x2)
+  %res3 = add <4 x i32> %res0, %res1
+  %res4 = add <4 x i32> %res3, %res2
+  ret <4 x i32> %res4
+}
+
+define <2 x double> @test_mask_compress_pd_128(<2 x double> %data, <2 x double> %passthru, i8 %mask) {
+; X86-LABEL: test_mask_compress_pd_128:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vcompresspd %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0xfd,0x09,0x8a,0xc1]
+; X86-NEXT:    vmovdqa %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc1]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_mask_compress_pd_128:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vcompresspd %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0xfd,0x09,0x8a,0xc1]
+; X64-NEXT:    vmovdqa %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc1]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <2 x double> @llvm.x86.avx512.mask.compress.pd.128(<2 x double> %data, <2 x double> %passthru, i8 %mask)
+  ret <2 x double> %res
+}
+
+define <2 x double> @test_maskz_compress_pd_128(<2 x double> %data, i8 %mask) {
+; X86-LABEL: test_maskz_compress_pd_128:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vcompresspd %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0x89,0x8a,0xc0]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_maskz_compress_pd_128:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vcompresspd %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0x89,0x8a,0xc0]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <2 x double> @llvm.x86.avx512.mask.compress.pd.128(<2 x double> %data, <2 x double> zeroinitializer, i8 %mask)
+  ret <2 x double> %res
+}
+
+define <2 x double> @test_compress_pd_128(<2 x double> %data) {
+; CHECK-LABEL: test_compress_pd_128:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <2 x double> @llvm.x86.avx512.mask.compress.pd.128(<2 x double> %data, <2 x double> undef, i8 -1)
+  ret <2 x double> %res
+}
+
+declare <2 x double> @llvm.x86.avx512.mask.compress.pd.128(<2 x double> %data, <2 x double> %src0, i8 %mask)
+
+define <4 x float> @test_mask_compress_ps_128(<4 x float> %data, <4 x float> %passthru, i8 %mask) {
+; X86-LABEL: test_mask_compress_ps_128:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vcompressps %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0x7d,0x09,0x8a,0xc1]
+; X86-NEXT:    vmovdqa %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc1]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_mask_compress_ps_128:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vcompressps %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0x7d,0x09,0x8a,0xc1]
+; X64-NEXT:    vmovdqa %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc1]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <4 x float> @llvm.x86.avx512.mask.compress.ps.128(<4 x float> %data, <4 x float> %passthru, i8 %mask)
+  ret <4 x float> %res
+}
+
+define <4 x float> @test_maskz_compress_ps_128(<4 x float> %data, i8 %mask) {
+; X86-LABEL: test_maskz_compress_ps_128:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vcompressps %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0x89,0x8a,0xc0]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_maskz_compress_ps_128:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vcompressps %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0x89,0x8a,0xc0]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <4 x float> @llvm.x86.avx512.mask.compress.ps.128(<4 x float> %data, <4 x float> zeroinitializer, i8 %mask)
+  ret <4 x float> %res
+}
+
+define <4 x float> @test_compress_ps_128(<4 x float> %data) {
+; CHECK-LABEL: test_compress_ps_128:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <4 x float> @llvm.x86.avx512.mask.compress.ps.128(<4 x float> %data, <4 x float> undef, i8 -1)
+  ret <4 x float> %res
+}
+
+declare <4 x float> @llvm.x86.avx512.mask.compress.ps.128(<4 x float> %data, <4 x float> %src0, i8 %mask)
+
+define <2 x i64> @test_mask_compress_q_128(<2 x i64> %data, <2 x i64> %passthru, i8 %mask) {
+; X86-LABEL: test_mask_compress_q_128:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vpcompressq %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0xfd,0x09,0x8b,0xc1]
+; X86-NEXT:    vmovdqa %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc1]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_mask_compress_q_128:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vpcompressq %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0xfd,0x09,0x8b,0xc1]
+; X64-NEXT:    vmovdqa %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc1]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <2 x i64> @llvm.x86.avx512.mask.compress.q.128(<2 x i64> %data, <2 x i64> %passthru, i8 %mask)
+  ret <2 x i64> %res
+}
+
+define <2 x i64> @test_maskz_compress_q_128(<2 x i64> %data, i8 %mask) {
+; X86-LABEL: test_maskz_compress_q_128:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vpcompressq %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0x89,0x8b,0xc0]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_maskz_compress_q_128:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vpcompressq %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0x89,0x8b,0xc0]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <2 x i64> @llvm.x86.avx512.mask.compress.q.128(<2 x i64> %data, <2 x i64> zeroinitializer, i8 %mask)
+  ret <2 x i64> %res
+}
+
+define <2 x i64> @test_compress_q_128(<2 x i64> %data) {
+; CHECK-LABEL: test_compress_q_128:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <2 x i64> @llvm.x86.avx512.mask.compress.q.128(<2 x i64> %data, <2 x i64> undef, i8 -1)
+  ret <2 x i64> %res
+}
+
+declare <2 x i64> @llvm.x86.avx512.mask.compress.q.128(<2 x i64> %data, <2 x i64> %src0, i8 %mask)
+
+define <4 x i32> @test_mask_compress_d_128(<4 x i32> %data, <4 x i32> %passthru, i8 %mask) {
+; X86-LABEL: test_mask_compress_d_128:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vpcompressd %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0x7d,0x09,0x8b,0xc1]
+; X86-NEXT:    vmovdqa %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc1]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_mask_compress_d_128:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vpcompressd %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0x7d,0x09,0x8b,0xc1]
+; X64-NEXT:    vmovdqa %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc1]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <4 x i32> @llvm.x86.avx512.mask.compress.d.128(<4 x i32> %data, <4 x i32> %passthru, i8 %mask)
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @test_maskz_compress_d_128(<4 x i32> %data, i8 %mask) {
+; X86-LABEL: test_maskz_compress_d_128:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vpcompressd %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0x89,0x8b,0xc0]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_maskz_compress_d_128:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vpcompressd %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0x89,0x8b,0xc0]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <4 x i32> @llvm.x86.avx512.mask.compress.d.128(<4 x i32> %data, <4 x i32> zeroinitializer, i8 %mask)
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @test_compress_d_128(<4 x i32> %data) {
+; CHECK-LABEL: test_compress_d_128:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <4 x i32> @llvm.x86.avx512.mask.compress.d.128(<4 x i32> %data, <4 x i32> undef, i8 -1)
+  ret <4 x i32> %res
+}
+
+declare <4 x i32> @llvm.x86.avx512.mask.compress.d.128(<4 x i32> %data, <4 x i32> %src0, i8 %mask)
+
+define <2 x double> @test_expand_pd_128(<2 x double> %data) {
+; CHECK-LABEL: test_expand_pd_128:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <2 x double> @llvm.x86.avx512.mask.expand.pd.128(<2 x double> %data, <2 x double> undef, i8 -1)
+  ret <2 x double> %res
+}
+
+define <2 x double> @test_mask_expand_pd_128(<2 x double> %data, <2 x double> %passthru, i8 %mask) {
+; X86-LABEL: test_mask_expand_pd_128:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vexpandpd %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0xfd,0x09,0x88,0xc8]
+; X86-NEXT:    vmovdqa %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc1]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_mask_expand_pd_128:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vexpandpd %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0xfd,0x09,0x88,0xc8]
+; X64-NEXT:    vmovdqa %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc1]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <2 x double> @llvm.x86.avx512.mask.expand.pd.128(<2 x double> %data, <2 x double> %passthru, i8 %mask)
+  ret <2 x double> %res
+}
+
+define <2 x double> @test_maskz_expand_pd_128(<2 x double> %data, i8 %mask) {
+; X86-LABEL: test_maskz_expand_pd_128:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vexpandpd %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0x89,0x88,0xc0]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_maskz_expand_pd_128:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vexpandpd %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0x89,0x88,0xc0]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <2 x double> @llvm.x86.avx512.mask.expand.pd.128(<2 x double> %data, <2 x double> zeroinitializer, i8 %mask)
+  ret <2 x double> %res
+}
+
+declare <2 x double> @llvm.x86.avx512.mask.expand.pd.128(<2 x double> %data, <2 x double> %src0, i8 %mask)
+
+define <4 x float> @test_expand_ps_128(<4 x float> %data) {
+; CHECK-LABEL: test_expand_ps_128:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <4 x float> @llvm.x86.avx512.mask.expand.ps.128(<4 x float> %data, <4 x float> undef, i8 -1)
+  ret <4 x float> %res
+}
+
+define <4 x float> @test_mask_expand_ps_128(<4 x float> %data, <4 x float> %passthru, i8 %mask) {
+; X86-LABEL: test_mask_expand_ps_128:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vexpandps %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0x7d,0x09,0x88,0xc8]
+; X86-NEXT:    vmovdqa %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc1]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_mask_expand_ps_128:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vexpandps %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0x7d,0x09,0x88,0xc8]
+; X64-NEXT:    vmovdqa %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc1]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <4 x float> @llvm.x86.avx512.mask.expand.ps.128(<4 x float> %data, <4 x float> %passthru, i8 %mask)
+  ret <4 x float> %res
+}
+
+define <4 x float> @test_maskz_expand_ps_128(<4 x float> %data, i8 %mask) {
+; X86-LABEL: test_maskz_expand_ps_128:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vexpandps %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0x89,0x88,0xc0]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_maskz_expand_ps_128:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vexpandps %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0x89,0x88,0xc0]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <4 x float> @llvm.x86.avx512.mask.expand.ps.128(<4 x float> %data, <4 x float> zeroinitializer, i8 %mask)
+  ret <4 x float> %res
+}
+
+declare <4 x float> @llvm.x86.avx512.mask.expand.ps.128(<4 x float> %data, <4 x float> %src0, i8 %mask)
+
+define <2 x i64> @test_expand_q_128(<2 x i64> %data) {
+; CHECK-LABEL: test_expand_q_128:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <2 x i64> @llvm.x86.avx512.mask.expand.q.128(<2 x i64> %data, <2 x i64> undef, i8 -1)
+  ret <2 x i64> %res
+}
+
+define <2 x i64> @test_mask_expand_q_128(<2 x i64> %data, <2 x i64> %passthru, i8 %mask) {
+; X86-LABEL: test_mask_expand_q_128:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vpexpandq %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0xfd,0x09,0x89,0xc8]
+; X86-NEXT:    vmovdqa %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc1]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_mask_expand_q_128:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vpexpandq %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0xfd,0x09,0x89,0xc8]
+; X64-NEXT:    vmovdqa %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc1]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <2 x i64> @llvm.x86.avx512.mask.expand.q.128(<2 x i64> %data, <2 x i64> %passthru, i8 %mask)
+  ret <2 x i64> %res
+}
+
+define <2 x i64> @test_maskz_expand_q_128(<2 x i64> %data, i8 %mask) {
+; X86-LABEL: test_maskz_expand_q_128:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vpexpandq %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0x89,0x89,0xc0]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_maskz_expand_q_128:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vpexpandq %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0x89,0x89,0xc0]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <2 x i64> @llvm.x86.avx512.mask.expand.q.128(<2 x i64> %data, <2 x i64> zeroinitializer, i8 %mask)
+  ret <2 x i64> %res
+}
+
+declare <2 x i64> @llvm.x86.avx512.mask.expand.q.128(<2 x i64> %data, <2 x i64> %src0, i8 %mask)
+
+define <4 x i32> @test_expand_d_128(<4 x i32> %data) {
+; CHECK-LABEL: test_expand_d_128:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <4 x i32> @llvm.x86.avx512.mask.expand.d.128(<4 x i32> %data, <4 x i32> undef, i8 -1)
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @test_mask_expand_d_128(<4 x i32> %data, <4 x i32> %passthru, i8 %mask) {
+; X86-LABEL: test_mask_expand_d_128:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vpexpandd %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0x7d,0x09,0x89,0xc8]
+; X86-NEXT:    vmovdqa %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc1]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_mask_expand_d_128:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vpexpandd %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0x7d,0x09,0x89,0xc8]
+; X64-NEXT:    vmovdqa %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc1]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <4 x i32> @llvm.x86.avx512.mask.expand.d.128(<4 x i32> %data, <4 x i32> %passthru, i8 %mask)
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @test_maskz_expand_d_128(<4 x i32> %data, i8 %mask) {
+; X86-LABEL: test_maskz_expand_d_128:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vpexpandd %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0x89,0x89,0xc0]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_maskz_expand_d_128:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vpexpandd %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0x89,0x89,0xc0]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <4 x i32> @llvm.x86.avx512.mask.expand.d.128(<4 x i32> %data, <4 x i32> zeroinitializer, i8 %mask)
+  ret <4 x i32> %res
+}
+
+declare <4 x i32> @llvm.x86.avx512.mask.expand.d.128(<4 x i32> %data, <4 x i32> %src0, i8 %mask)
+
+define <4 x double> @test_mask_compress_pd_256(<4 x double> %data, <4 x double> %passthru, i8 %mask) {
+; X86-LABEL: test_mask_compress_pd_256:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vcompresspd %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf2,0xfd,0x29,0x8a,0xc1]
+; X86-NEXT:    vmovdqa %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc1]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_mask_compress_pd_256:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vcompresspd %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf2,0xfd,0x29,0x8a,0xc1]
+; X64-NEXT:    vmovdqa %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc1]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <4 x double> @llvm.x86.avx512.mask.compress.pd.256(<4 x double> %data, <4 x double> %passthru, i8 %mask)
+  ret <4 x double> %res
+}
+
+define <4 x double> @test_maskz_compress_pd_256(<4 x double> %data, i8 %mask) {
+; X86-LABEL: test_maskz_compress_pd_256:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vcompresspd %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xa9,0x8a,0xc0]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_maskz_compress_pd_256:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vcompresspd %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xa9,0x8a,0xc0]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <4 x double> @llvm.x86.avx512.mask.compress.pd.256(<4 x double> %data, <4 x double> zeroinitializer, i8 %mask)
+  ret <4 x double> %res
+}
+
+define <4 x double> @test_compress_pd_256(<4 x double> %data) {
+; CHECK-LABEL: test_compress_pd_256:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <4 x double> @llvm.x86.avx512.mask.compress.pd.256(<4 x double> %data, <4 x double> undef, i8 -1)
+  ret <4 x double> %res
+}
+
+declare <4 x double> @llvm.x86.avx512.mask.compress.pd.256(<4 x double> %data, <4 x double> %src0, i8 %mask)
+
+define <8 x float> @test_mask_compress_ps_256(<8 x float> %data, <8 x float> %passthru, i8 %mask) {
+; X86-LABEL: test_mask_compress_ps_256:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vcompressps %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf2,0x7d,0x29,0x8a,0xc1]
+; X86-NEXT:    vmovdqa %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc1]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_mask_compress_ps_256:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vcompressps %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf2,0x7d,0x29,0x8a,0xc1]
+; X64-NEXT:    vmovdqa %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc1]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <8 x float> @llvm.x86.avx512.mask.compress.ps.256(<8 x float> %data, <8 x float> %passthru, i8 %mask)
+  ret <8 x float> %res
+}
+
+define <8 x float> @test_maskz_compress_ps_256(<8 x float> %data, i8 %mask) {
+; X86-LABEL: test_maskz_compress_ps_256:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vcompressps %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0xa9,0x8a,0xc0]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_maskz_compress_ps_256:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vcompressps %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0xa9,0x8a,0xc0]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <8 x float> @llvm.x86.avx512.mask.compress.ps.256(<8 x float> %data, <8 x float> zeroinitializer, i8 %mask)
+  ret <8 x float> %res
+}
+
+define <8 x float> @test_compress_ps_256(<8 x float> %data) {
+; CHECK-LABEL: test_compress_ps_256:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <8 x float> @llvm.x86.avx512.mask.compress.ps.256(<8 x float> %data, <8 x float> undef, i8 -1)
+  ret <8 x float> %res
+}
+
+declare <8 x float> @llvm.x86.avx512.mask.compress.ps.256(<8 x float> %data, <8 x float> %src0, i8 %mask)
+
+define <4 x i64> @test_mask_compress_q_256(<4 x i64> %data, <4 x i64> %passthru, i8 %mask) {
+; X86-LABEL: test_mask_compress_q_256:
 ; X86:       # %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
-; X86-NEXT:    vfmadd213ps (%eax), %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x71,0xa8,0x00]
-; X86-NEXT:    # xmm0 = (xmm1 * xmm0) + mem
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vpcompressq %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf2,0xfd,0x29,0x8b,0xc1]
+; X86-NEXT:    vmovdqa %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc1]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
-; X64-LABEL: test_mask_vfmadd128_ps_rmkza:
+; X64-LABEL: test_mask_compress_q_256:
 ; X64:       # %bb.0:
-; X64-NEXT:    vfmadd213ps (%rdi), %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x71,0xa8,0x07]
-; X64-NEXT:    # xmm0 = (xmm1 * xmm0) + mem
+; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vpcompressq %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf2,0xfd,0x29,0x8b,0xc1]
+; X64-NEXT:    vmovdqa %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc1]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %a2 = load <4 x float>, <4 x float>* %ptr_a2, align 4
-  %res = call <4 x float> @llvm.x86.avx512.mask.vfmadd.ps.128(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, i8 -1) nounwind
-  ret <4 x float> %res
+  %res = call <4 x i64> @llvm.x86.avx512.mask.compress.q.256(<4 x i64> %data, <4 x i64> %passthru, i8 %mask)
+  ret <4 x i64> %res
 }
 
-define <4 x float> @test_mask_vfmadd128_ps_rmb(<4 x float> %a0, <4 x float> %a1, float* %ptr_a2, i8 %mask) {
-; X86-LABEL: test_mask_vfmadd128_ps_rmb:
+define <4 x i64> @test_maskz_compress_q_256(<4 x i64> %data, i8 %mask) {
+; X86-LABEL: test_maskz_compress_q_256:
 ; X86:       # %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
-; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx # encoding: [0x0f,0xb6,0x4c,0x24,0x08]
-; X86-NEXT:    kmovw %ecx, %k1 # encoding: [0xc5,0xf8,0x92,0xc9]
-; X86-NEXT:    vfmadd213ps (%eax){1to4}, %xmm1, %xmm0 {%k1} # encoding: [0x62,0xf2,0x75,0x19,0xa8,0x00]
-; X86-NEXT:    # xmm0 = (xmm1 * xmm0) + mem
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vpcompressq %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xa9,0x8b,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
-; X64-LABEL: test_mask_vfmadd128_ps_rmb:
+; X64-LABEL: test_maskz_compress_q_256:
 ; X64:       # %bb.0:
-; X64-NEXT:    kmovw %esi, %k1 # encoding: [0xc5,0xf8,0x92,0xce]
-; X64-NEXT:    vfmadd213ps (%rdi){1to4}, %xmm1, %xmm0 {%k1} # encoding: [0x62,0xf2,0x75,0x19,0xa8,0x07]
-; X64-NEXT:    # xmm0 = (xmm1 * xmm0) + mem
+; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vpcompressq %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xa9,0x8b,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %q = load float, float* %ptr_a2
-  %vecinit.i = insertelement <4 x float> undef, float %q, i32 0
-  %vecinit2.i = insertelement <4 x float> %vecinit.i, float %q, i32 1
-  %vecinit4.i = insertelement <4 x float> %vecinit2.i, float %q, i32 2
-  %vecinit6.i = insertelement <4 x float> %vecinit4.i, float %q, i32 3
-  %res = call <4 x float> @llvm.x86.avx512.mask.vfmadd.ps.128(<4 x float> %a0, <4 x float> %a1, <4 x float> %vecinit6.i, i8 %mask) nounwind
-  ret <4 x float> %res
+  %res = call <4 x i64> @llvm.x86.avx512.mask.compress.q.256(<4 x i64> %data, <4 x i64> zeroinitializer, i8 %mask)
+  ret <4 x i64> %res
 }
 
-define <4 x float> @test_mask_vfmadd128_ps_rmba(<4 x float> %a0, <4 x float> %a1, float* %ptr_a2, i8 %mask) {
-; X86-LABEL: test_mask_vfmadd128_ps_rmba:
+define <4 x i64> @test_compress_q_256(<4 x i64> %data) {
+; CHECK-LABEL: test_compress_q_256:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <4 x i64> @llvm.x86.avx512.mask.compress.q.256(<4 x i64> %data, <4 x i64> undef, i8 -1)
+  ret <4 x i64> %res
+}
+
+declare <4 x i64> @llvm.x86.avx512.mask.compress.q.256(<4 x i64> %data, <4 x i64> %src0, i8 %mask)
+
+define <8 x i32> @test_mask_compress_d_256(<8 x i32> %data, <8 x i32> %passthru, i8 %mask) {
+; X86-LABEL: test_mask_compress_d_256:
 ; X86:       # %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
-; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx # encoding: [0x0f,0xb6,0x4c,0x24,0x08]
-; X86-NEXT:    kmovw %ecx, %k1 # encoding: [0xc5,0xf8,0x92,0xc9]
-; X86-NEXT:    vfmadd213ps (%eax){1to4}, %xmm1, %xmm0 {%k1} # encoding: [0x62,0xf2,0x75,0x19,0xa8,0x00]
-; X86-NEXT:    # xmm0 = (xmm1 * xmm0) + mem
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vpcompressd %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf2,0x7d,0x29,0x8b,0xc1]
+; X86-NEXT:    vmovdqa %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc1]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
-; X64-LABEL: test_mask_vfmadd128_ps_rmba:
+; X64-LABEL: test_mask_compress_d_256:
 ; X64:       # %bb.0:
-; X64-NEXT:    kmovw %esi, %k1 # encoding: [0xc5,0xf8,0x92,0xce]
-; X64-NEXT:    vfmadd213ps (%rdi){1to4}, %xmm1, %xmm0 {%k1} # encoding: [0x62,0xf2,0x75,0x19,0xa8,0x07]
-; X64-NEXT:    # xmm0 = (xmm1 * xmm0) + mem
+; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vpcompressd %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf2,0x7d,0x29,0x8b,0xc1]
+; X64-NEXT:    vmovdqa %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc1]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %q = load float, float* %ptr_a2, align 4
-  %vecinit.i = insertelement <4 x float> undef, float %q, i32 0
-  %vecinit2.i = insertelement <4 x float> %vecinit.i, float %q, i32 1
-  %vecinit4.i = insertelement <4 x float> %vecinit2.i, float %q, i32 2
-  %vecinit6.i = insertelement <4 x float> %vecinit4.i, float %q, i32 3
-  %res = call <4 x float> @llvm.x86.avx512.mask.vfmadd.ps.128(<4 x float> %a0, <4 x float> %a1, <4 x float> %vecinit6.i, i8 %mask) nounwind
-  ret <4 x float> %res
+  %res = call <8 x i32> @llvm.x86.avx512.mask.compress.d.256(<8 x i32> %data, <8 x i32> %passthru, i8 %mask)
+  ret <8 x i32> %res
 }
 
-define <4 x float> @test_mask_vfmadd128_ps_rmbz(<4 x float> %a0, <4 x float> %a1, float* %ptr_a2) {
-; X86-LABEL: test_mask_vfmadd128_ps_rmbz:
+define <8 x i32> @test_maskz_compress_d_256(<8 x i32> %data, i8 %mask) {
+; X86-LABEL: test_maskz_compress_d_256:
 ; X86:       # %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
-; X86-NEXT:    vfmadd213ps (%eax){1to4}, %xmm1, %xmm0 # encoding: [0x62,0xf2,0x75,0x18,0xa8,0x00]
-; X86-NEXT:    # xmm0 = (xmm1 * xmm0) + mem
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vpcompressd %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0xa9,0x8b,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
-; X64-LABEL: test_mask_vfmadd128_ps_rmbz:
+; X64-LABEL: test_maskz_compress_d_256:
 ; X64:       # %bb.0:
-; X64-NEXT:    vfmadd213ps (%rdi){1to4}, %xmm1, %xmm0 # encoding: [0x62,0xf2,0x75,0x18,0xa8,0x07]
-; X64-NEXT:    # xmm0 = (xmm1 * xmm0) + mem
+; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vpcompressd %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0xa9,0x8b,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %q = load float, float* %ptr_a2
-  %vecinit.i = insertelement <4 x float> undef, float %q, i32 0
-  %vecinit2.i = insertelement <4 x float> %vecinit.i, float %q, i32 1
-  %vecinit4.i = insertelement <4 x float> %vecinit2.i, float %q, i32 2
-  %vecinit6.i = insertelement <4 x float> %vecinit4.i, float %q, i32 3
-  %res = call <4 x float> @llvm.x86.avx512.mask.vfmadd.ps.128(<4 x float> %a0, <4 x float> %a1, <4 x float> %vecinit6.i, i8 -1) nounwind
-  ret <4 x float> %res
+  %res = call <8 x i32> @llvm.x86.avx512.mask.compress.d.256(<8 x i32> %data, <8 x i32> zeroinitializer, i8 %mask)
+  ret <8 x i32> %res
 }
 
-define <4 x float> @test_mask_vfmadd128_ps_rmbza(<4 x float> %a0, <4 x float> %a1, float* %ptr_a2) {
-; X86-LABEL: test_mask_vfmadd128_ps_rmbza:
+define <8 x i32> @test_compress_d_256(<8 x i32> %data) {
+; CHECK-LABEL: test_compress_d_256:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <8 x i32> @llvm.x86.avx512.mask.compress.d.256(<8 x i32> %data, <8 x i32> undef, i8 -1)
+  ret <8 x i32> %res
+}
+
+declare <8 x i32> @llvm.x86.avx512.mask.compress.d.256(<8 x i32> %data, <8 x i32> %src0, i8 %mask)
+
+define <4 x double> @test_expand_pd_256(<4 x double> %data) {
+; CHECK-LABEL: test_expand_pd_256:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <4 x double> @llvm.x86.avx512.mask.expand.pd.256(<4 x double> %data, <4 x double> undef, i8 -1)
+  ret <4 x double> %res
+}
+
+define <4 x double> @test_mask_expand_pd_256(<4 x double> %data, <4 x double> %passthru, i8 %mask) {
+; X86-LABEL: test_mask_expand_pd_256:
 ; X86:       # %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
-; X86-NEXT:    vfmadd213ps (%eax){1to4}, %xmm1, %xmm0 # encoding: [0x62,0xf2,0x75,0x18,0xa8,0x00]
-; X86-NEXT:    # xmm0 = (xmm1 * xmm0) + mem
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vexpandpd %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf2,0xfd,0x29,0x88,0xc8]
+; X86-NEXT:    vmovdqa %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc1]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
-; X64-LABEL: test_mask_vfmadd128_ps_rmbza:
+; X64-LABEL: test_mask_expand_pd_256:
 ; X64:       # %bb.0:
-; X64-NEXT:    vfmadd213ps (%rdi){1to4}, %xmm1, %xmm0 # encoding: [0x62,0xf2,0x75,0x18,0xa8,0x07]
-; X64-NEXT:    # xmm0 = (xmm1 * xmm0) + mem
+; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vexpandpd %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf2,0xfd,0x29,0x88,0xc8]
+; X64-NEXT:    vmovdqa %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc1]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %q = load float, float* %ptr_a2, align 4
-  %vecinit.i = insertelement <4 x float> undef, float %q, i32 0
-  %vecinit2.i = insertelement <4 x float> %vecinit.i, float %q, i32 1
-  %vecinit4.i = insertelement <4 x float> %vecinit2.i, float %q, i32 2
-  %vecinit6.i = insertelement <4 x float> %vecinit4.i, float %q, i32 3
-  %res = call <4 x float> @llvm.x86.avx512.mask.vfmadd.ps.128(<4 x float> %a0, <4 x float> %a1, <4 x float> %vecinit6.i, i8 -1) nounwind
-  ret <4 x float> %res
+  %res = call <4 x double> @llvm.x86.avx512.mask.expand.pd.256(<4 x double> %data, <4 x double> %passthru, i8 %mask)
+  ret <4 x double> %res
 }
 
-define <2 x double> @test_mask_vfmadd128_pd_rmk(<2 x double> %a0, <2 x double> %a1, <2 x double>* %ptr_a2, i8 %mask) {
-; X86-LABEL: test_mask_vfmadd128_pd_rmk:
+define <4 x double> @test_maskz_expand_pd_256(<4 x double> %data, i8 %mask) {
+; X86-LABEL: test_maskz_expand_pd_256:
 ; X86:       # %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
-; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx # encoding: [0x0f,0xb6,0x4c,0x24,0x08]
-; X86-NEXT:    kmovw %ecx, %k1 # encoding: [0xc5,0xf8,0x92,0xc9]
-; X86-NEXT:    vfmadd213pd (%eax), %xmm1, %xmm0 {%k1} # encoding: [0x62,0xf2,0xf5,0x09,0xa8,0x00]
-; X86-NEXT:    # xmm0 = (xmm1 * xmm0) + mem
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vexpandpd %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xa9,0x88,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
-; X64-LABEL: test_mask_vfmadd128_pd_rmk:
+; X64-LABEL: test_maskz_expand_pd_256:
 ; X64:       # %bb.0:
-; X64-NEXT:    kmovw %esi, %k1 # encoding: [0xc5,0xf8,0x92,0xce]
-; X64-NEXT:    vfmadd213pd (%rdi), %xmm1, %xmm0 {%k1} # encoding: [0x62,0xf2,0xf5,0x09,0xa8,0x07]
-; X64-NEXT:    # xmm0 = (xmm1 * xmm0) + mem
+; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vexpandpd %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xa9,0x88,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %a2 = load <2 x double>, <2 x double>* %ptr_a2
-  %res = call <2 x double> @llvm.x86.avx512.mask.vfmadd.pd.128(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, i8 %mask) nounwind
-  ret <2 x double> %res
+  %res = call <4 x double> @llvm.x86.avx512.mask.expand.pd.256(<4 x double> %data, <4 x double> zeroinitializer, i8 %mask)
+  ret <4 x double> %res
 }
 
-define <2 x double> @test_mask_vfmadd128_pd_rmkz(<2 x double> %a0, <2 x double> %a1, <2 x double>* %ptr_a2) {
-; X86-LABEL: test_mask_vfmadd128_pd_rmkz:
+declare <4 x double> @llvm.x86.avx512.mask.expand.pd.256(<4 x double> %data, <4 x double> %src0, i8 %mask)
+
+define <8 x float> @test_expand_ps_256(<8 x float> %data) {
+; CHECK-LABEL: test_expand_ps_256:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <8 x float> @llvm.x86.avx512.mask.expand.ps.256(<8 x float> %data, <8 x float> undef, i8 -1)
+  ret <8 x float> %res
+}
+
+define <8 x float> @test_mask_expand_ps_256(<8 x float> %data, <8 x float> %passthru, i8 %mask) {
+; X86-LABEL: test_mask_expand_ps_256:
 ; X86:       # %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
-; X86-NEXT:    vfmadd213pd (%eax), %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0xf1,0xa8,0x00]
-; X86-NEXT:    # xmm0 = (xmm1 * xmm0) + mem
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vexpandps %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf2,0x7d,0x29,0x88,0xc8]
+; X86-NEXT:    vmovdqa %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc1]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
-; X64-LABEL: test_mask_vfmadd128_pd_rmkz:
+; X64-LABEL: test_mask_expand_ps_256:
 ; X64:       # %bb.0:
-; X64-NEXT:    vfmadd213pd (%rdi), %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0xf1,0xa8,0x07]
-; X64-NEXT:    # xmm0 = (xmm1 * xmm0) + mem
+; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vexpandps %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf2,0x7d,0x29,0x88,0xc8]
+; X64-NEXT:    vmovdqa %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc1]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %a2 = load <2 x double>, <2 x double>* %ptr_a2
-  %res = call <2 x double> @llvm.x86.avx512.mask.vfmadd.pd.128(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, i8 -1) nounwind
-  ret <2 x double> %res
+  %res = call <8 x float> @llvm.x86.avx512.mask.expand.ps.256(<8 x float> %data, <8 x float> %passthru, i8 %mask)
+  ret <8 x float> %res
 }
 
-define <4 x double> @test_mask_vfmadd256_pd_rmk(<4 x double> %a0, <4 x double> %a1, <4 x double>* %ptr_a2, i8 %mask) {
-; X86-LABEL: test_mask_vfmadd256_pd_rmk:
+define <8 x float> @test_maskz_expand_ps_256(<8 x float> %data, i8 %mask) {
+; X86-LABEL: test_maskz_expand_ps_256:
 ; X86:       # %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
-; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx # encoding: [0x0f,0xb6,0x4c,0x24,0x08]
-; X86-NEXT:    kmovw %ecx, %k1 # encoding: [0xc5,0xf8,0x92,0xc9]
-; X86-NEXT:    vfmadd213pd (%eax), %ymm1, %ymm0 {%k1} # encoding: [0x62,0xf2,0xf5,0x29,0xa8,0x00]
-; X86-NEXT:    # ymm0 = (ymm1 * ymm0) + mem
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vexpandps %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0xa9,0x88,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
-; X64-LABEL: test_mask_vfmadd256_pd_rmk:
+; X64-LABEL: test_maskz_expand_ps_256:
 ; X64:       # %bb.0:
-; X64-NEXT:    kmovw %esi, %k1 # encoding: [0xc5,0xf8,0x92,0xce]
-; X64-NEXT:    vfmadd213pd (%rdi), %ymm1, %ymm0 {%k1} # encoding: [0x62,0xf2,0xf5,0x29,0xa8,0x07]
-; X64-NEXT:    # ymm0 = (ymm1 * ymm0) + mem
+; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vexpandps %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0xa9,0x88,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %a2 = load <4 x double>, <4 x double>* %ptr_a2
-  %res = call <4 x double> @llvm.x86.avx512.mask.vfmadd.pd.256(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2, i8 %mask) nounwind
-  ret <4 x double> %res
+  %res = call <8 x float> @llvm.x86.avx512.mask.expand.ps.256(<8 x float> %data, <8 x float> zeroinitializer, i8 %mask)
+  ret <8 x float> %res
 }
 
-define <4 x double> @test_mask_vfmadd256_pd_rmkz(<4 x double> %a0, <4 x double> %a1, <4 x double>* %ptr_a2) {
-; X86-LABEL: test_mask_vfmadd256_pd_rmkz:
+declare <8 x float> @llvm.x86.avx512.mask.expand.ps.256(<8 x float> %data, <8 x float> %src0, i8 %mask)
+
+define <4 x i64> @test_expand_q_256(<4 x i64> %data) {
+; CHECK-LABEL: test_expand_q_256:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <4 x i64> @llvm.x86.avx512.mask.expand.q.256(<4 x i64> %data, <4 x i64> undef, i8 -1)
+  ret <4 x i64> %res
+}
+
+define <4 x i64> @test_mask_expand_q_256(<4 x i64> %data, <4 x i64> %passthru, i8 %mask) {
+; X86-LABEL: test_mask_expand_q_256:
 ; X86:       # %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
-; X86-NEXT:    vfmadd213pd (%eax), %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0xf5,0xa8,0x00]
-; X86-NEXT:    # ymm0 = (ymm1 * ymm0) + mem
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vpexpandq %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf2,0xfd,0x29,0x89,0xc8]
+; X86-NEXT:    vmovdqa %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc1]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
-; X64-LABEL: test_mask_vfmadd256_pd_rmkz:
+; X64-LABEL: test_mask_expand_q_256:
 ; X64:       # %bb.0:
-; X64-NEXT:    vfmadd213pd (%rdi), %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0xf5,0xa8,0x07]
-; X64-NEXT:    # ymm0 = (ymm1 * ymm0) + mem
+; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vpexpandq %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf2,0xfd,0x29,0x89,0xc8]
+; X64-NEXT:    vmovdqa %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc1]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %a2 = load <4 x double>, <4 x double>* %ptr_a2
-  %res = call <4 x double> @llvm.x86.avx512.mask.vfmadd.pd.256(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2, i8 -1) nounwind
-  ret <4 x double> %res
+  %res = call <4 x i64> @llvm.x86.avx512.mask.expand.q.256(<4 x i64> %data, <4 x i64> %passthru, i8 %mask)
+  ret <4 x i64> %res
+}
+
+define <4 x i64> @test_maskz_expand_q_256(<4 x i64> %data, i8 %mask) {
+; X86-LABEL: test_maskz_expand_q_256:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vpexpandq %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xa9,0x89,0xc0]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_maskz_expand_q_256:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vpexpandq %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xa9,0x89,0xc0]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <4 x i64> @llvm.x86.avx512.mask.expand.q.256(<4 x i64> %data, <4 x i64> zeroinitializer, i8 %mask)
+  ret <4 x i64> %res
+}
+
+declare <4 x i64> @llvm.x86.avx512.mask.expand.q.256(<4 x i64> %data, <4 x i64> %src0, i8 %mask)
+
+define <8 x i32> @test_expand_d_256(<8 x i32> %data) {
+; CHECK-LABEL: test_expand_d_256:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
+  %res = call <8 x i32> @llvm.x86.avx512.mask.expand.d.256(<8 x i32> %data, <8 x i32> undef, i8 -1)
+  ret <8 x i32> %res
+}
+
+define <8 x i32> @test_mask_expand_d_256(<8 x i32> %data, <8 x i32> %passthru, i8 %mask) {
+; X86-LABEL: test_mask_expand_d_256:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vpexpandd %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf2,0x7d,0x29,0x89,0xc8]
+; X86-NEXT:    vmovdqa %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc1]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_mask_expand_d_256:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vpexpandd %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf2,0x7d,0x29,0x89,0xc8]
+; X64-NEXT:    vmovdqa %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc1]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <8 x i32> @llvm.x86.avx512.mask.expand.d.256(<8 x i32> %data, <8 x i32> %passthru, i8 %mask)
+  ret <8 x i32> %res
+}
+
+define <8 x i32> @test_maskz_expand_d_256(<8 x i32> %data, i8 %mask) {
+; X86-LABEL: test_maskz_expand_d_256:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vpexpandd %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0xa9,0x89,0xc0]
+; X86-NEXT:    retl # encoding: [0xc3]
+;
+; X64-LABEL: test_maskz_expand_d_256:
+; X64:       # %bb.0:
+; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vpexpandd %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0xa9,0x89,0xc0]
+; X64-NEXT:    retq # encoding: [0xc3]
+  %res = call <8 x i32> @llvm.x86.avx512.mask.expand.d.256(<8 x i32> %data, <8 x i32> zeroinitializer, i8 %mask)
+  ret <8 x i32> %res
 }
+
+declare <8 x i32> @llvm.x86.avx512.mask.expand.d.256(<8 x i32> %data, <8 x i32> %src0, i8 %mask)
diff --git a/test/CodeGen/X86/avx512vl-intrinsics.ll b/test/CodeGen/X86/avx512vl-intrinsics.ll
index f4ec74b66dc8bfa009952ddd9d75738a0c43f75e..b5dfa2f11ade884ea5d6a8aac5e61b4a8fdc0f04 100644
--- a/test/CodeGen/X86/avx512vl-intrinsics.ll
+++ b/test/CodeGen/X86/avx512vl-intrinsics.ll
@@ -17,8 +17,10 @@ define <2 x double> @test_mask_compress_pd_128(<2 x double> %data, <2 x double>
 ; X64-NEXT:    vcompresspd %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0xfd,0x09,0x8a,0xc1]
 ; X64-NEXT:    vmovdqa %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc1]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <2 x double> @llvm.x86.avx512.mask.compress.pd.128(<2 x double> %data, <2 x double> %passthru, i8 %mask)
-  ret <2 x double> %res
+  %1 = bitcast i8 %mask to <8 x i1>
+  %extract = shufflevector <8 x i1> %1, <8 x i1> %1, <2 x i32> <i32 0, i32 1>
+  %2 = call <2 x double> @llvm.x86.avx512.mask.compress.v2f64(<2 x double> %data, <2 x double> %passthru, <2 x i1> %extract)
+  ret <2 x double> %2
 }
 
 define <2 x double> @test_maskz_compress_pd_128(<2 x double> %data, i8 %mask) {
@@ -34,20 +36,20 @@ define <2 x double> @test_maskz_compress_pd_128(<2 x double> %data, i8 %mask) {
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vcompresspd %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0x89,0x8a,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <2 x double> @llvm.x86.avx512.mask.compress.pd.128(<2 x double> %data, <2 x double> zeroinitializer, i8 %mask)
-  ret <2 x double> %res
+  %1 = bitcast i8 %mask to <8 x i1>
+  %extract = shufflevector <8 x i1> %1, <8 x i1> %1, <2 x i32> <i32 0, i32 1>
+  %2 = call <2 x double> @llvm.x86.avx512.mask.compress.v2f64(<2 x double> %data, <2 x double> zeroinitializer, <2 x i1> %extract)
+  ret <2 x double> %2
 }
 
 define <2 x double> @test_compress_pd_128(<2 x double> %data) {
 ; CHECK-LABEL: test_compress_pd_128:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
-  %res = call <2 x double> @llvm.x86.avx512.mask.compress.pd.128(<2 x double> %data, <2 x double> undef, i8 -1)
-  ret <2 x double> %res
+  %1 = call <2 x double> @llvm.x86.avx512.mask.compress.v2f64(<2 x double> %data, <2 x double> undef, <2 x i1> <i1 true, i1 true>)
+  ret <2 x double> %1
 }
 
-declare <2 x double> @llvm.x86.avx512.mask.compress.pd.128(<2 x double> %data, <2 x double> %src0, i8 %mask)
-
 define <4 x float> @test_mask_compress_ps_128(<4 x float> %data, <4 x float> %passthru, i8 %mask) {
 ; X86-LABEL: test_mask_compress_ps_128:
 ; X86:       # %bb.0:
@@ -63,8 +65,10 @@ define <4 x float> @test_mask_compress_ps_128(<4 x float> %data, <4 x float> %pa
 ; X64-NEXT:    vcompressps %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0x7d,0x09,0x8a,0xc1]
 ; X64-NEXT:    vmovdqa %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc1]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <4 x float> @llvm.x86.avx512.mask.compress.ps.128(<4 x float> %data, <4 x float> %passthru, i8 %mask)
-  ret <4 x float> %res
+  %1 = bitcast i8 %mask to <8 x i1>
+  %extract = shufflevector <8 x i1> %1, <8 x i1> %1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %2 = call <4 x float> @llvm.x86.avx512.mask.compress.v4f32(<4 x float> %data, <4 x float> %passthru, <4 x i1> %extract)
+  ret <4 x float> %2
 }
 
 define <4 x float> @test_maskz_compress_ps_128(<4 x float> %data, i8 %mask) {
@@ -80,20 +84,20 @@ define <4 x float> @test_maskz_compress_ps_128(<4 x float> %data, i8 %mask) {
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vcompressps %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0x89,0x8a,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <4 x float> @llvm.x86.avx512.mask.compress.ps.128(<4 x float> %data, <4 x float> zeroinitializer, i8 %mask)
-  ret <4 x float> %res
+  %1 = bitcast i8 %mask to <8 x i1>
+  %extract = shufflevector <8 x i1> %1, <8 x i1> %1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %2 = call <4 x float> @llvm.x86.avx512.mask.compress.v4f32(<4 x float> %data, <4 x float> zeroinitializer, <4 x i1> %extract)
+  ret <4 x float> %2
 }
 
 define <4 x float> @test_compress_ps_128(<4 x float> %data) {
 ; CHECK-LABEL: test_compress_ps_128:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
-  %res = call <4 x float> @llvm.x86.avx512.mask.compress.ps.128(<4 x float> %data, <4 x float> undef, i8 -1)
-  ret <4 x float> %res
+  %1 = call <4 x float> @llvm.x86.avx512.mask.compress.v4f32(<4 x float> %data, <4 x float> undef, <4 x i1> <i1 true, i1 true, i1 true, i1 true>)
+  ret <4 x float> %1
 }
 
-declare <4 x float> @llvm.x86.avx512.mask.compress.ps.128(<4 x float> %data, <4 x float> %src0, i8 %mask)
-
 define <2 x i64> @test_mask_compress_q_128(<2 x i64> %data, <2 x i64> %passthru, i8 %mask) {
 ; X86-LABEL: test_mask_compress_q_128:
 ; X86:       # %bb.0:
@@ -109,8 +113,10 @@ define <2 x i64> @test_mask_compress_q_128(<2 x i64> %data, <2 x i64> %passthru,
 ; X64-NEXT:    vpcompressq %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0xfd,0x09,0x8b,0xc1]
 ; X64-NEXT:    vmovdqa %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc1]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <2 x i64> @llvm.x86.avx512.mask.compress.q.128(<2 x i64> %data, <2 x i64> %passthru, i8 %mask)
-  ret <2 x i64> %res
+  %1 = bitcast i8 %mask to <8 x i1>
+  %extract = shufflevector <8 x i1> %1, <8 x i1> %1, <2 x i32> <i32 0, i32 1>
+  %2 = call <2 x i64> @llvm.x86.avx512.mask.compress.v2i64(<2 x i64> %data, <2 x i64> %passthru, <2 x i1> %extract)
+  ret <2 x i64> %2
 }
 
 define <2 x i64> @test_maskz_compress_q_128(<2 x i64> %data, i8 %mask) {
@@ -126,20 +132,20 @@ define <2 x i64> @test_maskz_compress_q_128(<2 x i64> %data, i8 %mask) {
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vpcompressq %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0x89,0x8b,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <2 x i64> @llvm.x86.avx512.mask.compress.q.128(<2 x i64> %data, <2 x i64> zeroinitializer, i8 %mask)
-  ret <2 x i64> %res
+  %1 = bitcast i8 %mask to <8 x i1>
+  %extract = shufflevector <8 x i1> %1, <8 x i1> %1, <2 x i32> <i32 0, i32 1>
+  %2 = call <2 x i64> @llvm.x86.avx512.mask.compress.v2i64(<2 x i64> %data, <2 x i64> zeroinitializer, <2 x i1> %extract)
+  ret <2 x i64> %2
 }
 
 define <2 x i64> @test_compress_q_128(<2 x i64> %data) {
 ; CHECK-LABEL: test_compress_q_128:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
-  %res = call <2 x i64> @llvm.x86.avx512.mask.compress.q.128(<2 x i64> %data, <2 x i64> undef, i8 -1)
-  ret <2 x i64> %res
+  %1 = call <2 x i64> @llvm.x86.avx512.mask.compress.v2i64(<2 x i64> %data, <2 x i64> undef, <2 x i1> <i1 true, i1 true>)
+  ret <2 x i64> %1
 }
 
-declare <2 x i64> @llvm.x86.avx512.mask.compress.q.128(<2 x i64> %data, <2 x i64> %src0, i8 %mask)
-
 define <4 x i32> @test_mask_compress_d_128(<4 x i32> %data, <4 x i32> %passthru, i8 %mask) {
 ; X86-LABEL: test_mask_compress_d_128:
 ; X86:       # %bb.0:
@@ -155,8 +161,10 @@ define <4 x i32> @test_mask_compress_d_128(<4 x i32> %data, <4 x i32> %passthru,
 ; X64-NEXT:    vpcompressd %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0x7d,0x09,0x8b,0xc1]
 ; X64-NEXT:    vmovdqa %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc1]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <4 x i32> @llvm.x86.avx512.mask.compress.d.128(<4 x i32> %data, <4 x i32> %passthru, i8 %mask)
-  ret <4 x i32> %res
+  %1 = bitcast i8 %mask to <8 x i1>
+  %extract = shufflevector <8 x i1> %1, <8 x i1> %1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %2 = call <4 x i32> @llvm.x86.avx512.mask.compress.v4i32(<4 x i32> %data, <4 x i32> %passthru, <4 x i1> %extract)
+  ret <4 x i32> %2
 }
 
 define <4 x i32> @test_maskz_compress_d_128(<4 x i32> %data, i8 %mask) {
@@ -172,26 +180,26 @@ define <4 x i32> @test_maskz_compress_d_128(<4 x i32> %data, i8 %mask) {
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vpcompressd %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0x89,0x8b,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <4 x i32> @llvm.x86.avx512.mask.compress.d.128(<4 x i32> %data, <4 x i32> zeroinitializer, i8 %mask)
-  ret <4 x i32> %res
+  %1 = bitcast i8 %mask to <8 x i1>
+  %extract = shufflevector <8 x i1> %1, <8 x i1> %1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %2 = call <4 x i32> @llvm.x86.avx512.mask.compress.v4i32(<4 x i32> %data, <4 x i32> zeroinitializer, <4 x i1> %extract)
+  ret <4 x i32> %2
 }
 
 define <4 x i32> @test_compress_d_128(<4 x i32> %data) {
 ; CHECK-LABEL: test_compress_d_128:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
-  %res = call <4 x i32> @llvm.x86.avx512.mask.compress.d.128(<4 x i32> %data, <4 x i32> undef, i8 -1)
-  ret <4 x i32> %res
+  %1 = call <4 x i32> @llvm.x86.avx512.mask.compress.v4i32(<4 x i32> %data, <4 x i32> undef, <4 x i1> <i1 true, i1 true, i1 true, i1 true>)
+  ret <4 x i32> %1
 }
 
-declare <4 x i32> @llvm.x86.avx512.mask.compress.d.128(<4 x i32> %data, <4 x i32> %src0, i8 %mask)
-
 define <2 x double> @test_expand_pd_128(<2 x double> %data) {
 ; CHECK-LABEL: test_expand_pd_128:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
-  %res = call <2 x double> @llvm.x86.avx512.mask.expand.pd.128(<2 x double> %data, <2 x double> undef, i8 -1)
-  ret <2 x double> %res
+  %1 = call <2 x double> @llvm.x86.avx512.mask.expand.v2f64(<2 x double> %data, <2 x double> undef, <2 x i1> <i1 true, i1 true>)
+  ret <2 x double> %1
 }
 
 define <2 x double> @test_mask_expand_pd_128(<2 x double> %data, <2 x double> %passthru, i8 %mask) {
@@ -209,8 +217,10 @@ define <2 x double> @test_mask_expand_pd_128(<2 x double> %data, <2 x double> %p
 ; X64-NEXT:    vexpandpd %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0xfd,0x09,0x88,0xc8]
 ; X64-NEXT:    vmovdqa %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc1]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <2 x double> @llvm.x86.avx512.mask.expand.pd.128(<2 x double> %data, <2 x double> %passthru, i8 %mask)
-  ret <2 x double> %res
+  %1 = bitcast i8 %mask to <8 x i1>
+  %extract = shufflevector <8 x i1> %1, <8 x i1> %1, <2 x i32> <i32 0, i32 1>
+  %2 = call <2 x double> @llvm.x86.avx512.mask.expand.v2f64(<2 x double> %data, <2 x double> %passthru, <2 x i1> %extract)
+  ret <2 x double> %2
 }
 
 define <2 x double> @test_maskz_expand_pd_128(<2 x double> %data, i8 %mask) {
@@ -226,18 +236,18 @@ define <2 x double> @test_maskz_expand_pd_128(<2 x double> %data, i8 %mask) {
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vexpandpd %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0x89,0x88,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <2 x double> @llvm.x86.avx512.mask.expand.pd.128(<2 x double> %data, <2 x double> zeroinitializer, i8 %mask)
-  ret <2 x double> %res
+  %1 = bitcast i8 %mask to <8 x i1>
+  %extract = shufflevector <8 x i1> %1, <8 x i1> %1, <2 x i32> <i32 0, i32 1>
+  %2 = call <2 x double> @llvm.x86.avx512.mask.expand.v2f64(<2 x double> %data, <2 x double> zeroinitializer, <2 x i1> %extract)
+  ret <2 x double> %2
 }
 
-declare <2 x double> @llvm.x86.avx512.mask.expand.pd.128(<2 x double> %data, <2 x double> %src0, i8 %mask)
-
 define <4 x float> @test_expand_ps_128(<4 x float> %data) {
 ; CHECK-LABEL: test_expand_ps_128:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
-  %res = call <4 x float> @llvm.x86.avx512.mask.expand.ps.128(<4 x float> %data, <4 x float> undef, i8 -1)
-  ret <4 x float> %res
+  %1 = call <4 x float> @llvm.x86.avx512.mask.expand.v4f32(<4 x float> %data, <4 x float> undef, <4 x i1> <i1 true, i1 true, i1 true, i1 true>)
+  ret <4 x float> %1
 }
 
 define <4 x float> @test_mask_expand_ps_128(<4 x float> %data, <4 x float> %passthru, i8 %mask) {
@@ -255,8 +265,10 @@ define <4 x float> @test_mask_expand_ps_128(<4 x float> %data, <4 x float> %pass
 ; X64-NEXT:    vexpandps %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0x7d,0x09,0x88,0xc8]
 ; X64-NEXT:    vmovdqa %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc1]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <4 x float> @llvm.x86.avx512.mask.expand.ps.128(<4 x float> %data, <4 x float> %passthru, i8 %mask)
-  ret <4 x float> %res
+  %1 = bitcast i8 %mask to <8 x i1>
+  %extract = shufflevector <8 x i1> %1, <8 x i1> %1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %2 = call <4 x float> @llvm.x86.avx512.mask.expand.v4f32(<4 x float> %data, <4 x float> %passthru, <4 x i1> %extract)
+  ret <4 x float> %2
 }
 
 define <4 x float> @test_maskz_expand_ps_128(<4 x float> %data, i8 %mask) {
@@ -272,18 +284,18 @@ define <4 x float> @test_maskz_expand_ps_128(<4 x float> %data, i8 %mask) {
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vexpandps %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0x89,0x88,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <4 x float> @llvm.x86.avx512.mask.expand.ps.128(<4 x float> %data, <4 x float> zeroinitializer, i8 %mask)
-  ret <4 x float> %res
+  %1 = bitcast i8 %mask to <8 x i1>
+  %extract = shufflevector <8 x i1> %1, <8 x i1> %1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %2 = call <4 x float> @llvm.x86.avx512.mask.expand.v4f32(<4 x float> %data, <4 x float> zeroinitializer, <4 x i1> %extract)
+  ret <4 x float> %2
 }
 
-declare <4 x float> @llvm.x86.avx512.mask.expand.ps.128(<4 x float> %data, <4 x float> %src0, i8 %mask)
-
 define <2 x i64> @test_expand_q_128(<2 x i64> %data) {
 ; CHECK-LABEL: test_expand_q_128:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
-  %res = call <2 x i64> @llvm.x86.avx512.mask.expand.q.128(<2 x i64> %data, <2 x i64> undef, i8 -1)
-  ret <2 x i64> %res
+  %1 = call <2 x i64> @llvm.x86.avx512.mask.expand.v2i64(<2 x i64> %data, <2 x i64> undef, <2 x i1> <i1 true, i1 true>)
+  ret <2 x i64> %1
 }
 
 define <2 x i64> @test_mask_expand_q_128(<2 x i64> %data, <2 x i64> %passthru, i8 %mask) {
@@ -301,8 +313,10 @@ define <2 x i64> @test_mask_expand_q_128(<2 x i64> %data, <2 x i64> %passthru, i
 ; X64-NEXT:    vpexpandq %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0xfd,0x09,0x89,0xc8]
 ; X64-NEXT:    vmovdqa %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc1]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <2 x i64> @llvm.x86.avx512.mask.expand.q.128(<2 x i64> %data, <2 x i64> %passthru, i8 %mask)
-  ret <2 x i64> %res
+  %1 = bitcast i8 %mask to <8 x i1>
+  %extract = shufflevector <8 x i1> %1, <8 x i1> %1, <2 x i32> <i32 0, i32 1>
+  %2 = call <2 x i64> @llvm.x86.avx512.mask.expand.v2i64(<2 x i64> %data, <2 x i64> %passthru, <2 x i1> %extract)
+  ret <2 x i64> %2
 }
 
 define <2 x i64> @test_maskz_expand_q_128(<2 x i64> %data, i8 %mask) {
@@ -318,18 +332,18 @@ define <2 x i64> @test_maskz_expand_q_128(<2 x i64> %data, i8 %mask) {
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vpexpandq %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0x89,0x89,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <2 x i64> @llvm.x86.avx512.mask.expand.q.128(<2 x i64> %data, <2 x i64> zeroinitializer, i8 %mask)
-  ret <2 x i64> %res
+  %1 = bitcast i8 %mask to <8 x i1>
+  %extract = shufflevector <8 x i1> %1, <8 x i1> %1, <2 x i32> <i32 0, i32 1>
+  %2 = call <2 x i64> @llvm.x86.avx512.mask.expand.v2i64(<2 x i64> %data, <2 x i64> zeroinitializer, <2 x i1> %extract)
+  ret <2 x i64> %2
 }
 
-declare <2 x i64> @llvm.x86.avx512.mask.expand.q.128(<2 x i64> %data, <2 x i64> %src0, i8 %mask)
-
 define <4 x i32> @test_expand_d_128(<4 x i32> %data) {
 ; CHECK-LABEL: test_expand_d_128:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
-  %res = call <4 x i32> @llvm.x86.avx512.mask.expand.d.128(<4 x i32> %data, <4 x i32> undef, i8 -1)
-  ret <4 x i32> %res
+  %1 = call <4 x i32> @llvm.x86.avx512.mask.expand.v4i32(<4 x i32> %data, <4 x i32> undef, <4 x i1> <i1 true, i1 true, i1 true, i1 true>)
+  ret <4 x i32> %1
 }
 
 define <4 x i32> @test_mask_expand_d_128(<4 x i32> %data, <4 x i32> %passthru, i8 %mask) {
@@ -347,8 +361,10 @@ define <4 x i32> @test_mask_expand_d_128(<4 x i32> %data, <4 x i32> %passthru, i
 ; X64-NEXT:    vpexpandd %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0x7d,0x09,0x89,0xc8]
 ; X64-NEXT:    vmovdqa %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc1]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <4 x i32> @llvm.x86.avx512.mask.expand.d.128(<4 x i32> %data, <4 x i32> %passthru, i8 %mask)
-  ret <4 x i32> %res
+  %1 = bitcast i8 %mask to <8 x i1>
+  %extract = shufflevector <8 x i1> %1, <8 x i1> %1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %2 = call <4 x i32> @llvm.x86.avx512.mask.expand.v4i32(<4 x i32> %data, <4 x i32> %passthru, <4 x i1> %extract)
+  ret <4 x i32> %2
 }
 
 define <4 x i32> @test_maskz_expand_d_128(<4 x i32> %data, i8 %mask) {
@@ -364,12 +380,12 @@ define <4 x i32> @test_maskz_expand_d_128(<4 x i32> %data, i8 %mask) {
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vpexpandd %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0x89,0x89,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <4 x i32> @llvm.x86.avx512.mask.expand.d.128(<4 x i32> %data, <4 x i32> zeroinitializer, i8 %mask)
-  ret <4 x i32> %res
+  %1 = bitcast i8 %mask to <8 x i1>
+  %extract = shufflevector <8 x i1> %1, <8 x i1> %1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %2 = call <4 x i32> @llvm.x86.avx512.mask.expand.v4i32(<4 x i32> %data, <4 x i32> zeroinitializer, <4 x i1> %extract)
+  ret <4 x i32> %2
 }
 
-declare <4 x i32> @llvm.x86.avx512.mask.expand.d.128(<4 x i32> %data, <4 x i32> %src0, i8 %mask)
-
 define <4 x double> @test_mask_compress_pd_256(<4 x double> %data, <4 x double> %passthru, i8 %mask) {
 ; X86-LABEL: test_mask_compress_pd_256:
 ; X86:       # %bb.0:
@@ -385,8 +401,10 @@ define <4 x double> @test_mask_compress_pd_256(<4 x double> %data, <4 x double>
 ; X64-NEXT:    vcompresspd %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf2,0xfd,0x29,0x8a,0xc1]
 ; X64-NEXT:    vmovdqa %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc1]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <4 x double> @llvm.x86.avx512.mask.compress.pd.256(<4 x double> %data, <4 x double> %passthru, i8 %mask)
-  ret <4 x double> %res
+  %1 = bitcast i8 %mask to <8 x i1>
+  %extract = shufflevector <8 x i1> %1, <8 x i1> %1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %2 = call <4 x double> @llvm.x86.avx512.mask.compress.v4f64(<4 x double> %data, <4 x double> %passthru, <4 x i1> %extract)
+  ret <4 x double> %2
 }
 
 define <4 x double> @test_maskz_compress_pd_256(<4 x double> %data, i8 %mask) {
@@ -402,20 +420,20 @@ define <4 x double> @test_maskz_compress_pd_256(<4 x double> %data, i8 %mask) {
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vcompresspd %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xa9,0x8a,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <4 x double> @llvm.x86.avx512.mask.compress.pd.256(<4 x double> %data, <4 x double> zeroinitializer, i8 %mask)
-  ret <4 x double> %res
+  %1 = bitcast i8 %mask to <8 x i1>
+  %extract = shufflevector <8 x i1> %1, <8 x i1> %1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %2 = call <4 x double> @llvm.x86.avx512.mask.compress.v4f64(<4 x double> %data, <4 x double> zeroinitializer, <4 x i1> %extract)
+  ret <4 x double> %2
 }
 
 define <4 x double> @test_compress_pd_256(<4 x double> %data) {
 ; CHECK-LABEL: test_compress_pd_256:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
-  %res = call <4 x double> @llvm.x86.avx512.mask.compress.pd.256(<4 x double> %data, <4 x double> undef, i8 -1)
-  ret <4 x double> %res
+  %1 = call <4 x double> @llvm.x86.avx512.mask.compress.v4f64(<4 x double> %data, <4 x double> undef, <4 x i1> <i1 true, i1 true, i1 true, i1 true>)
+  ret <4 x double> %1
 }
 
-declare <4 x double> @llvm.x86.avx512.mask.compress.pd.256(<4 x double> %data, <4 x double> %src0, i8 %mask)
-
 define <8 x float> @test_mask_compress_ps_256(<8 x float> %data, <8 x float> %passthru, i8 %mask) {
 ; X86-LABEL: test_mask_compress_ps_256:
 ; X86:       # %bb.0:
@@ -431,8 +449,9 @@ define <8 x float> @test_mask_compress_ps_256(<8 x float> %data, <8 x float> %pa
 ; X64-NEXT:    vcompressps %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf2,0x7d,0x29,0x8a,0xc1]
 ; X64-NEXT:    vmovdqa %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc1]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <8 x float> @llvm.x86.avx512.mask.compress.ps.256(<8 x float> %data, <8 x float> %passthru, i8 %mask)
-  ret <8 x float> %res
+  %1 = bitcast i8 %mask to <8 x i1>
+  %2 = call <8 x float> @llvm.x86.avx512.mask.compress.v8f32(<8 x float> %data, <8 x float> %passthru, <8 x i1> %1)
+  ret <8 x float> %2
 }
 
 define <8 x float> @test_maskz_compress_ps_256(<8 x float> %data, i8 %mask) {
@@ -448,20 +467,19 @@ define <8 x float> @test_maskz_compress_ps_256(<8 x float> %data, i8 %mask) {
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vcompressps %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0xa9,0x8a,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <8 x float> @llvm.x86.avx512.mask.compress.ps.256(<8 x float> %data, <8 x float> zeroinitializer, i8 %mask)
-  ret <8 x float> %res
+  %1 = bitcast i8 %mask to <8 x i1>
+  %2 = call <8 x float> @llvm.x86.avx512.mask.compress.v8f32(<8 x float> %data, <8 x float> zeroinitializer, <8 x i1> %1)
+  ret <8 x float> %2
 }
 
 define <8 x float> @test_compress_ps_256(<8 x float> %data) {
 ; CHECK-LABEL: test_compress_ps_256:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
-  %res = call <8 x float> @llvm.x86.avx512.mask.compress.ps.256(<8 x float> %data, <8 x float> undef, i8 -1)
-  ret <8 x float> %res
+  %1 = call <8 x float> @llvm.x86.avx512.mask.compress.v8f32(<8 x float> %data, <8 x float> undef, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>)
+  ret <8 x float> %1
 }
 
-declare <8 x float> @llvm.x86.avx512.mask.compress.ps.256(<8 x float> %data, <8 x float> %src0, i8 %mask)
-
 define <4 x i64> @test_mask_compress_q_256(<4 x i64> %data, <4 x i64> %passthru, i8 %mask) {
 ; X86-LABEL: test_mask_compress_q_256:
 ; X86:       # %bb.0:
@@ -477,8 +495,10 @@ define <4 x i64> @test_mask_compress_q_256(<4 x i64> %data, <4 x i64> %passthru,
 ; X64-NEXT:    vpcompressq %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf2,0xfd,0x29,0x8b,0xc1]
 ; X64-NEXT:    vmovdqa %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc1]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <4 x i64> @llvm.x86.avx512.mask.compress.q.256(<4 x i64> %data, <4 x i64> %passthru, i8 %mask)
-  ret <4 x i64> %res
+  %1 = bitcast i8 %mask to <8 x i1>
+  %extract = shufflevector <8 x i1> %1, <8 x i1> %1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %2 = call <4 x i64> @llvm.x86.avx512.mask.compress.v4i64(<4 x i64> %data, <4 x i64> %passthru, <4 x i1> %extract)
+  ret <4 x i64> %2
 }
 
 define <4 x i64> @test_maskz_compress_q_256(<4 x i64> %data, i8 %mask) {
@@ -494,20 +514,20 @@ define <4 x i64> @test_maskz_compress_q_256(<4 x i64> %data, i8 %mask) {
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vpcompressq %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xa9,0x8b,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <4 x i64> @llvm.x86.avx512.mask.compress.q.256(<4 x i64> %data, <4 x i64> zeroinitializer, i8 %mask)
-  ret <4 x i64> %res
+  %1 = bitcast i8 %mask to <8 x i1>
+  %extract = shufflevector <8 x i1> %1, <8 x i1> %1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %2 = call <4 x i64> @llvm.x86.avx512.mask.compress.v4i64(<4 x i64> %data, <4 x i64> zeroinitializer, <4 x i1> %extract)
+  ret <4 x i64> %2
 }
 
 define <4 x i64> @test_compress_q_256(<4 x i64> %data) {
 ; CHECK-LABEL: test_compress_q_256:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
-  %res = call <4 x i64> @llvm.x86.avx512.mask.compress.q.256(<4 x i64> %data, <4 x i64> undef, i8 -1)
-  ret <4 x i64> %res
+  %1 = call <4 x i64> @llvm.x86.avx512.mask.compress.v4i64(<4 x i64> %data, <4 x i64> undef, <4 x i1> <i1 true, i1 true, i1 true, i1 true>)
+  ret <4 x i64> %1
 }
 
-declare <4 x i64> @llvm.x86.avx512.mask.compress.q.256(<4 x i64> %data, <4 x i64> %src0, i8 %mask)
-
 define <8 x i32> @test_mask_compress_d_256(<8 x i32> %data, <8 x i32> %passthru, i8 %mask) {
 ; X86-LABEL: test_mask_compress_d_256:
 ; X86:       # %bb.0:
@@ -523,8 +543,9 @@ define <8 x i32> @test_mask_compress_d_256(<8 x i32> %data, <8 x i32> %passthru,
 ; X64-NEXT:    vpcompressd %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf2,0x7d,0x29,0x8b,0xc1]
 ; X64-NEXT:    vmovdqa %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc1]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <8 x i32> @llvm.x86.avx512.mask.compress.d.256(<8 x i32> %data, <8 x i32> %passthru, i8 %mask)
-  ret <8 x i32> %res
+  %1 = bitcast i8 %mask to <8 x i1>
+  %2 = call <8 x i32> @llvm.x86.avx512.mask.compress.v8i32(<8 x i32> %data, <8 x i32> %passthru, <8 x i1> %1)
+  ret <8 x i32> %2
 }
 
 define <8 x i32> @test_maskz_compress_d_256(<8 x i32> %data, i8 %mask) {
@@ -540,26 +561,25 @@ define <8 x i32> @test_maskz_compress_d_256(<8 x i32> %data, i8 %mask) {
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vpcompressd %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0xa9,0x8b,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <8 x i32> @llvm.x86.avx512.mask.compress.d.256(<8 x i32> %data, <8 x i32> zeroinitializer, i8 %mask)
-  ret <8 x i32> %res
+  %1 = bitcast i8 %mask to <8 x i1>
+  %2 = call <8 x i32> @llvm.x86.avx512.mask.compress.v8i32(<8 x i32> %data, <8 x i32> zeroinitializer, <8 x i1> %1)
+  ret <8 x i32> %2
 }
 
 define <8 x i32> @test_compress_d_256(<8 x i32> %data) {
 ; CHECK-LABEL: test_compress_d_256:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
-  %res = call <8 x i32> @llvm.x86.avx512.mask.compress.d.256(<8 x i32> %data, <8 x i32> undef, i8 -1)
-  ret <8 x i32> %res
+  %1 = call <8 x i32> @llvm.x86.avx512.mask.compress.v8i32(<8 x i32> %data, <8 x i32> undef, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>)
+  ret <8 x i32> %1
 }
 
-declare <8 x i32> @llvm.x86.avx512.mask.compress.d.256(<8 x i32> %data, <8 x i32> %src0, i8 %mask)
-
 define <4 x double> @test_expand_pd_256(<4 x double> %data) {
 ; CHECK-LABEL: test_expand_pd_256:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
-  %res = call <4 x double> @llvm.x86.avx512.mask.expand.pd.256(<4 x double> %data, <4 x double> undef, i8 -1)
-  ret <4 x double> %res
+  %1 = call <4 x double> @llvm.x86.avx512.mask.expand.v4f64(<4 x double> %data, <4 x double> undef, <4 x i1> <i1 true, i1 true, i1 true, i1 true>)
+  ret <4 x double> %1
 }
 
 define <4 x double> @test_mask_expand_pd_256(<4 x double> %data, <4 x double> %passthru, i8 %mask) {
@@ -577,8 +597,10 @@ define <4 x double> @test_mask_expand_pd_256(<4 x double> %data, <4 x double> %p
 ; X64-NEXT:    vexpandpd %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf2,0xfd,0x29,0x88,0xc8]
 ; X64-NEXT:    vmovdqa %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc1]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <4 x double> @llvm.x86.avx512.mask.expand.pd.256(<4 x double> %data, <4 x double> %passthru, i8 %mask)
-  ret <4 x double> %res
+  %1 = bitcast i8 %mask to <8 x i1>
+  %extract = shufflevector <8 x i1> %1, <8 x i1> %1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %2 = call <4 x double> @llvm.x86.avx512.mask.expand.v4f64(<4 x double> %data, <4 x double> %passthru, <4 x i1> %extract)
+  ret <4 x double> %2
 }
 
 define <4 x double> @test_maskz_expand_pd_256(<4 x double> %data, i8 %mask) {
@@ -594,18 +616,18 @@ define <4 x double> @test_maskz_expand_pd_256(<4 x double> %data, i8 %mask) {
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vexpandpd %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xa9,0x88,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <4 x double> @llvm.x86.avx512.mask.expand.pd.256(<4 x double> %data, <4 x double> zeroinitializer, i8 %mask)
-  ret <4 x double> %res
+  %1 = bitcast i8 %mask to <8 x i1>
+  %extract = shufflevector <8 x i1> %1, <8 x i1> %1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %2 = call <4 x double> @llvm.x86.avx512.mask.expand.v4f64(<4 x double> %data, <4 x double> zeroinitializer, <4 x i1> %extract)
+  ret <4 x double> %2
 }
 
-declare <4 x double> @llvm.x86.avx512.mask.expand.pd.256(<4 x double> %data, <4 x double> %src0, i8 %mask)
-
 define <8 x float> @test_expand_ps_256(<8 x float> %data) {
 ; CHECK-LABEL: test_expand_ps_256:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
-  %res = call <8 x float> @llvm.x86.avx512.mask.expand.ps.256(<8 x float> %data, <8 x float> undef, i8 -1)
-  ret <8 x float> %res
+  %1 = call <8 x float> @llvm.x86.avx512.mask.expand.v8f32(<8 x float> %data, <8 x float> undef, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>)
+  ret <8 x float> %1
 }
 
 define <8 x float> @test_mask_expand_ps_256(<8 x float> %data, <8 x float> %passthru, i8 %mask) {
@@ -623,8 +645,9 @@ define <8 x float> @test_mask_expand_ps_256(<8 x float> %data, <8 x float> %pass
 ; X64-NEXT:    vexpandps %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf2,0x7d,0x29,0x88,0xc8]
 ; X64-NEXT:    vmovdqa %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc1]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <8 x float> @llvm.x86.avx512.mask.expand.ps.256(<8 x float> %data, <8 x float> %passthru, i8 %mask)
-  ret <8 x float> %res
+  %1 = bitcast i8 %mask to <8 x i1>
+  %2 = call <8 x float> @llvm.x86.avx512.mask.expand.v8f32(<8 x float> %data, <8 x float> %passthru, <8 x i1> %1)
+  ret <8 x float> %2
 }
 
 define <8 x float> @test_maskz_expand_ps_256(<8 x float> %data, i8 %mask) {
@@ -640,18 +663,17 @@ define <8 x float> @test_maskz_expand_ps_256(<8 x float> %data, i8 %mask) {
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vexpandps %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0xa9,0x88,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <8 x float> @llvm.x86.avx512.mask.expand.ps.256(<8 x float> %data, <8 x float> zeroinitializer, i8 %mask)
-  ret <8 x float> %res
+  %1 = bitcast i8 %mask to <8 x i1>
+  %2 = call <8 x float> @llvm.x86.avx512.mask.expand.v8f32(<8 x float> %data, <8 x float> zeroinitializer, <8 x i1> %1)
+  ret <8 x float> %2
 }
 
-declare <8 x float> @llvm.x86.avx512.mask.expand.ps.256(<8 x float> %data, <8 x float> %src0, i8 %mask)
-
 define <4 x i64> @test_expand_q_256(<4 x i64> %data) {
 ; CHECK-LABEL: test_expand_q_256:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
-  %res = call <4 x i64> @llvm.x86.avx512.mask.expand.q.256(<4 x i64> %data, <4 x i64> undef, i8 -1)
-  ret <4 x i64> %res
+  %1 = call <4 x i64> @llvm.x86.avx512.mask.expand.v4i64(<4 x i64> %data, <4 x i64> undef, <4 x i1> <i1 true, i1 true, i1 true, i1 true>)
+  ret <4 x i64> %1
 }
 
 define <4 x i64> @test_mask_expand_q_256(<4 x i64> %data, <4 x i64> %passthru, i8 %mask) {
@@ -669,8 +691,10 @@ define <4 x i64> @test_mask_expand_q_256(<4 x i64> %data, <4 x i64> %passthru, i
 ; X64-NEXT:    vpexpandq %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf2,0xfd,0x29,0x89,0xc8]
 ; X64-NEXT:    vmovdqa %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc1]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <4 x i64> @llvm.x86.avx512.mask.expand.q.256(<4 x i64> %data, <4 x i64> %passthru, i8 %mask)
-  ret <4 x i64> %res
+  %1 = bitcast i8 %mask to <8 x i1>
+  %extract = shufflevector <8 x i1> %1, <8 x i1> %1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %2 = call <4 x i64> @llvm.x86.avx512.mask.expand.v4i64(<4 x i64> %data, <4 x i64> %passthru, <4 x i1> %extract)
+  ret <4 x i64> %2
 }
 
 define <4 x i64> @test_maskz_expand_q_256(<4 x i64> %data, i8 %mask) {
@@ -686,18 +710,18 @@ define <4 x i64> @test_maskz_expand_q_256(<4 x i64> %data, i8 %mask) {
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vpexpandq %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xa9,0x89,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <4 x i64> @llvm.x86.avx512.mask.expand.q.256(<4 x i64> %data, <4 x i64> zeroinitializer, i8 %mask)
-  ret <4 x i64> %res
+  %1 = bitcast i8 %mask to <8 x i1>
+  %extract = shufflevector <8 x i1> %1, <8 x i1> %1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %2 = call <4 x i64> @llvm.x86.avx512.mask.expand.v4i64(<4 x i64> %data, <4 x i64> zeroinitializer, <4 x i1> %extract)
+  ret <4 x i64> %2
 }
 
-declare <4 x i64> @llvm.x86.avx512.mask.expand.q.256(<4 x i64> %data, <4 x i64> %src0, i8 %mask)
-
 define <8 x i32> @test_expand_d_256(<8 x i32> %data) {
 ; CHECK-LABEL: test_expand_d_256:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
-  %res = call <8 x i32> @llvm.x86.avx512.mask.expand.d.256(<8 x i32> %data, <8 x i32> undef, i8 -1)
-  ret <8 x i32> %res
+  %1 = call <8 x i32> @llvm.x86.avx512.mask.expand.v8i32(<8 x i32> %data, <8 x i32> undef, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>)
+  ret <8 x i32> %1
 }
 
 define <8 x i32> @test_mask_expand_d_256(<8 x i32> %data, <8 x i32> %passthru, i8 %mask) {
@@ -715,8 +739,9 @@ define <8 x i32> @test_mask_expand_d_256(<8 x i32> %data, <8 x i32> %passthru, i
 ; X64-NEXT:    vpexpandd %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf2,0x7d,0x29,0x89,0xc8]
 ; X64-NEXT:    vmovdqa %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc1]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <8 x i32> @llvm.x86.avx512.mask.expand.d.256(<8 x i32> %data, <8 x i32> %passthru, i8 %mask)
-  ret <8 x i32> %res
+  %1 = bitcast i8 %mask to <8 x i1>
+  %2 = call <8 x i32> @llvm.x86.avx512.mask.expand.v8i32(<8 x i32> %data, <8 x i32> %passthru, <8 x i1> %1)
+  ret <8 x i32> %2
 }
 
 define <8 x i32> @test_maskz_expand_d_256(<8 x i32> %data, i8 %mask) {
@@ -732,12 +757,11 @@ define <8 x i32> @test_maskz_expand_d_256(<8 x i32> %data, i8 %mask) {
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vpexpandd %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0xa9,0x89,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
-  %res = call <8 x i32> @llvm.x86.avx512.mask.expand.d.256(<8 x i32> %data, <8 x i32> zeroinitializer, i8 %mask)
-  ret <8 x i32> %res
+  %1 = bitcast i8 %mask to <8 x i1>
+  %2 = call <8 x i32> @llvm.x86.avx512.mask.expand.v8i32(<8 x i32> %data, <8 x i32> zeroinitializer, <8 x i1> %1)
+  ret <8 x i32> %2
 }
 
-declare <8 x i32> @llvm.x86.avx512.mask.expand.d.256(<8 x i32> %data, <8 x i32> %src0, i8 %mask)
-
 define i8 @test_cmpps_256(<8 x float> %a, <8 x float> %b) {
 ; CHECK-LABEL: test_cmpps_256:
 ; CHECK:       # %bb.0:
@@ -1617,22 +1641,23 @@ declare <16 x i8> @llvm.x86.avx512.mask.pmov.qb.128(<2 x i64>, <16 x i8>, i8)
 define <16 x i8>@test_int_x86_avx512_mask_pmov_qb_128(<2 x i64> %x0, <16 x i8> %x1, i8 %x2) {
 ; X86-LABEL: test_int_x86_avx512_mask_pmov_qb_128:
 ; X86:       # %bb.0:
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
-; X86-NEXT:    vpmovqb %xmm0, %xmm2 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0x89,0x32,0xc2]
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vpmovqb %xmm0, %xmm2 # encoding: [0x62,0xf2,0x7e,0x08,0x32,0xc2]
 ; X86-NEXT:    vpmovqb %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0x7e,0x09,0x32,0xc1]
-; X86-NEXT:    vpaddb %xmm2, %xmm1, %xmm1 # encoding: [0xc5,0xf1,0xfc,0xca]
-; X86-NEXT:    vpmovqb %xmm0, %xmm0 # encoding: [0x62,0xf2,0x7e,0x08,0x32,0xc0]
-; X86-NEXT:    vpaddb %xmm1, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xfc,0xc1]
+; X86-NEXT:    vpmovqb %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0x89,0x32,0xc0]
+; X86-NEXT:    vpaddb %xmm0, %xmm1, %xmm0 # encoding: [0xc5,0xf1,0xfc,0xc0]
+; X86-NEXT:    vpaddb %xmm0, %xmm2, %xmm0 # encoding: [0xc5,0xe9,0xfc,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_pmov_qb_128:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
-; X64-NEXT:    vpmovqb %xmm0, %xmm2 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0x89,0x32,0xc2]
+; X64-NEXT:    vpmovqb %xmm0, %xmm2 # encoding: [0x62,0xf2,0x7e,0x08,0x32,0xc2]
 ; X64-NEXT:    vpmovqb %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0x7e,0x09,0x32,0xc1]
-; X64-NEXT:    vpaddb %xmm2, %xmm1, %xmm1 # encoding: [0xc5,0xf1,0xfc,0xca]
-; X64-NEXT:    vpmovqb %xmm0, %xmm0 # encoding: [0x62,0xf2,0x7e,0x08,0x32,0xc0]
-; X64-NEXT:    vpaddb %xmm1, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xfc,0xc1]
+; X64-NEXT:    vpmovqb %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0x89,0x32,0xc0]
+; X64-NEXT:    vpaddb %xmm0, %xmm1, %xmm0 # encoding: [0xc5,0xf1,0xfc,0xc0]
+; X64-NEXT:    vpaddb %xmm0, %xmm2, %xmm0 # encoding: [0xc5,0xe9,0xfc,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
     %res0 = call <16 x i8> @llvm.x86.avx512.mask.pmov.qb.128(<2 x i64> %x0, <16 x i8> %x1, i8 -1)
     %res1 = call <16 x i8> @llvm.x86.avx512.mask.pmov.qb.128(<2 x i64> %x0, <16 x i8> %x1, i8 %x2)
@@ -1670,22 +1695,23 @@ declare <16 x i8> @llvm.x86.avx512.mask.pmovs.qb.128(<2 x i64>, <16 x i8>, i8)
 define <16 x i8>@test_int_x86_avx512_mask_pmovs_qb_128(<2 x i64> %x0, <16 x i8> %x1, i8 %x2) {
 ; X86-LABEL: test_int_x86_avx512_mask_pmovs_qb_128:
 ; X86:       # %bb.0:
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
-; X86-NEXT:    vpmovsqb %xmm0, %xmm2 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0x89,0x22,0xc2]
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vpmovsqb %xmm0, %xmm2 # encoding: [0x62,0xf2,0x7e,0x08,0x22,0xc2]
 ; X86-NEXT:    vpmovsqb %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0x7e,0x09,0x22,0xc1]
-; X86-NEXT:    vpaddb %xmm2, %xmm1, %xmm1 # encoding: [0xc5,0xf1,0xfc,0xca]
-; X86-NEXT:    vpmovsqb %xmm0, %xmm0 # encoding: [0x62,0xf2,0x7e,0x08,0x22,0xc0]
-; X86-NEXT:    vpaddb %xmm1, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xfc,0xc1]
+; X86-NEXT:    vpmovsqb %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0x89,0x22,0xc0]
+; X86-NEXT:    vpaddb %xmm0, %xmm1, %xmm0 # encoding: [0xc5,0xf1,0xfc,0xc0]
+; X86-NEXT:    vpaddb %xmm0, %xmm2, %xmm0 # encoding: [0xc5,0xe9,0xfc,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_pmovs_qb_128:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
-; X64-NEXT:    vpmovsqb %xmm0, %xmm2 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0x89,0x22,0xc2]
+; X64-NEXT:    vpmovsqb %xmm0, %xmm2 # encoding: [0x62,0xf2,0x7e,0x08,0x22,0xc2]
 ; X64-NEXT:    vpmovsqb %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0x7e,0x09,0x22,0xc1]
-; X64-NEXT:    vpaddb %xmm2, %xmm1, %xmm1 # encoding: [0xc5,0xf1,0xfc,0xca]
-; X64-NEXT:    vpmovsqb %xmm0, %xmm0 # encoding: [0x62,0xf2,0x7e,0x08,0x22,0xc0]
-; X64-NEXT:    vpaddb %xmm1, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xfc,0xc1]
+; X64-NEXT:    vpmovsqb %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0x89,0x22,0xc0]
+; X64-NEXT:    vpaddb %xmm0, %xmm1, %xmm0 # encoding: [0xc5,0xf1,0xfc,0xc0]
+; X64-NEXT:    vpaddb %xmm0, %xmm2, %xmm0 # encoding: [0xc5,0xe9,0xfc,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
     %res0 = call <16 x i8> @llvm.x86.avx512.mask.pmovs.qb.128(<2 x i64> %x0, <16 x i8> %x1, i8 -1)
     %res1 = call <16 x i8> @llvm.x86.avx512.mask.pmovs.qb.128(<2 x i64> %x0, <16 x i8> %x1, i8 %x2)
@@ -1723,22 +1749,23 @@ declare <16 x i8> @llvm.x86.avx512.mask.pmovus.qb.128(<2 x i64>, <16 x i8>, i8)
 define <16 x i8>@test_int_x86_avx512_mask_pmovus_qb_128(<2 x i64> %x0, <16 x i8> %x1, i8 %x2) {
 ; X86-LABEL: test_int_x86_avx512_mask_pmovus_qb_128:
 ; X86:       # %bb.0:
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
-; X86-NEXT:    vpmovusqb %xmm0, %xmm2 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0x89,0x12,0xc2]
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vpmovusqb %xmm0, %xmm2 # encoding: [0x62,0xf2,0x7e,0x08,0x12,0xc2]
 ; X86-NEXT:    vpmovusqb %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0x7e,0x09,0x12,0xc1]
-; X86-NEXT:    vpaddb %xmm2, %xmm1, %xmm1 # encoding: [0xc5,0xf1,0xfc,0xca]
-; X86-NEXT:    vpmovusqb %xmm0, %xmm0 # encoding: [0x62,0xf2,0x7e,0x08,0x12,0xc0]
-; X86-NEXT:    vpaddb %xmm1, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xfc,0xc1]
+; X86-NEXT:    vpmovusqb %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0x89,0x12,0xc0]
+; X86-NEXT:    vpaddb %xmm0, %xmm1, %xmm0 # encoding: [0xc5,0xf1,0xfc,0xc0]
+; X86-NEXT:    vpaddb %xmm0, %xmm2, %xmm0 # encoding: [0xc5,0xe9,0xfc,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_pmovus_qb_128:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
-; X64-NEXT:    vpmovusqb %xmm0, %xmm2 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0x89,0x12,0xc2]
+; X64-NEXT:    vpmovusqb %xmm0, %xmm2 # encoding: [0x62,0xf2,0x7e,0x08,0x12,0xc2]
 ; X64-NEXT:    vpmovusqb %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0x7e,0x09,0x12,0xc1]
-; X64-NEXT:    vpaddb %xmm2, %xmm1, %xmm1 # encoding: [0xc5,0xf1,0xfc,0xca]
-; X64-NEXT:    vpmovusqb %xmm0, %xmm0 # encoding: [0x62,0xf2,0x7e,0x08,0x12,0xc0]
-; X64-NEXT:    vpaddb %xmm1, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xfc,0xc1]
+; X64-NEXT:    vpmovusqb %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0x89,0x12,0xc0]
+; X64-NEXT:    vpaddb %xmm0, %xmm1, %xmm0 # encoding: [0xc5,0xf1,0xfc,0xc0]
+; X64-NEXT:    vpaddb %xmm0, %xmm2, %xmm0 # encoding: [0xc5,0xe9,0xfc,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
     %res0 = call <16 x i8> @llvm.x86.avx512.mask.pmovus.qb.128(<2 x i64> %x0, <16 x i8> %x1, i8 -1)
     %res1 = call <16 x i8> @llvm.x86.avx512.mask.pmovus.qb.128(<2 x i64> %x0, <16 x i8> %x1, i8 %x2)
@@ -1776,23 +1803,24 @@ declare <16 x i8> @llvm.x86.avx512.mask.pmov.qb.256(<4 x i64>, <16 x i8>, i8)
 define <16 x i8>@test_int_x86_avx512_mask_pmov_qb_256(<4 x i64> %x0, <16 x i8> %x1, i8 %x2) {
 ; X86-LABEL: test_int_x86_avx512_mask_pmov_qb_256:
 ; X86:       # %bb.0:
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
-; X86-NEXT:    vpmovqb %ymm0, %xmm2 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0xa9,0x32,0xc2]
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vpmovqb %ymm0, %xmm2 # encoding: [0x62,0xf2,0x7e,0x28,0x32,0xc2]
 ; X86-NEXT:    vpmovqb %ymm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0x7e,0x29,0x32,0xc1]
-; X86-NEXT:    vpaddb %xmm2, %xmm1, %xmm1 # encoding: [0xc5,0xf1,0xfc,0xca]
-; X86-NEXT:    vpmovqb %ymm0, %xmm0 # encoding: [0x62,0xf2,0x7e,0x28,0x32,0xc0]
-; X86-NEXT:    vpaddb %xmm1, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xfc,0xc1]
+; X86-NEXT:    vpmovqb %ymm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0xa9,0x32,0xc0]
+; X86-NEXT:    vpaddb %xmm0, %xmm1, %xmm0 # encoding: [0xc5,0xf1,0xfc,0xc0]
+; X86-NEXT:    vpaddb %xmm0, %xmm2, %xmm0 # encoding: [0xc5,0xe9,0xfc,0xc0]
 ; X86-NEXT:    vzeroupper # encoding: [0xc5,0xf8,0x77]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_pmov_qb_256:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
-; X64-NEXT:    vpmovqb %ymm0, %xmm2 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0xa9,0x32,0xc2]
+; X64-NEXT:    vpmovqb %ymm0, %xmm2 # encoding: [0x62,0xf2,0x7e,0x28,0x32,0xc2]
 ; X64-NEXT:    vpmovqb %ymm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0x7e,0x29,0x32,0xc1]
-; X64-NEXT:    vpaddb %xmm2, %xmm1, %xmm1 # encoding: [0xc5,0xf1,0xfc,0xca]
-; X64-NEXT:    vpmovqb %ymm0, %xmm0 # encoding: [0x62,0xf2,0x7e,0x28,0x32,0xc0]
-; X64-NEXT:    vpaddb %xmm1, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xfc,0xc1]
+; X64-NEXT:    vpmovqb %ymm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0xa9,0x32,0xc0]
+; X64-NEXT:    vpaddb %xmm0, %xmm1, %xmm0 # encoding: [0xc5,0xf1,0xfc,0xc0]
+; X64-NEXT:    vpaddb %xmm0, %xmm2, %xmm0 # encoding: [0xc5,0xe9,0xfc,0xc0]
 ; X64-NEXT:    vzeroupper # encoding: [0xc5,0xf8,0x77]
 ; X64-NEXT:    retq # encoding: [0xc3]
     %res0 = call <16 x i8> @llvm.x86.avx512.mask.pmov.qb.256(<4 x i64> %x0, <16 x i8> %x1, i8 -1)
@@ -1833,23 +1861,24 @@ declare <16 x i8> @llvm.x86.avx512.mask.pmovs.qb.256(<4 x i64>, <16 x i8>, i8)
 define <16 x i8>@test_int_x86_avx512_mask_pmovs_qb_256(<4 x i64> %x0, <16 x i8> %x1, i8 %x2) {
 ; X86-LABEL: test_int_x86_avx512_mask_pmovs_qb_256:
 ; X86:       # %bb.0:
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
-; X86-NEXT:    vpmovsqb %ymm0, %xmm2 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0xa9,0x22,0xc2]
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vpmovsqb %ymm0, %xmm2 # encoding: [0x62,0xf2,0x7e,0x28,0x22,0xc2]
 ; X86-NEXT:    vpmovsqb %ymm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0x7e,0x29,0x22,0xc1]
-; X86-NEXT:    vpaddb %xmm2, %xmm1, %xmm1 # encoding: [0xc5,0xf1,0xfc,0xca]
-; X86-NEXT:    vpmovsqb %ymm0, %xmm0 # encoding: [0x62,0xf2,0x7e,0x28,0x22,0xc0]
-; X86-NEXT:    vpaddb %xmm1, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xfc,0xc1]
+; X86-NEXT:    vpmovsqb %ymm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0xa9,0x22,0xc0]
+; X86-NEXT:    vpaddb %xmm0, %xmm1, %xmm0 # encoding: [0xc5,0xf1,0xfc,0xc0]
+; X86-NEXT:    vpaddb %xmm0, %xmm2, %xmm0 # encoding: [0xc5,0xe9,0xfc,0xc0]
 ; X86-NEXT:    vzeroupper # encoding: [0xc5,0xf8,0x77]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_pmovs_qb_256:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
-; X64-NEXT:    vpmovsqb %ymm0, %xmm2 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0xa9,0x22,0xc2]
+; X64-NEXT:    vpmovsqb %ymm0, %xmm2 # encoding: [0x62,0xf2,0x7e,0x28,0x22,0xc2]
 ; X64-NEXT:    vpmovsqb %ymm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0x7e,0x29,0x22,0xc1]
-; X64-NEXT:    vpaddb %xmm2, %xmm1, %xmm1 # encoding: [0xc5,0xf1,0xfc,0xca]
-; X64-NEXT:    vpmovsqb %ymm0, %xmm0 # encoding: [0x62,0xf2,0x7e,0x28,0x22,0xc0]
-; X64-NEXT:    vpaddb %xmm1, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xfc,0xc1]
+; X64-NEXT:    vpmovsqb %ymm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0xa9,0x22,0xc0]
+; X64-NEXT:    vpaddb %xmm0, %xmm1, %xmm0 # encoding: [0xc5,0xf1,0xfc,0xc0]
+; X64-NEXT:    vpaddb %xmm0, %xmm2, %xmm0 # encoding: [0xc5,0xe9,0xfc,0xc0]
 ; X64-NEXT:    vzeroupper # encoding: [0xc5,0xf8,0x77]
 ; X64-NEXT:    retq # encoding: [0xc3]
     %res0 = call <16 x i8> @llvm.x86.avx512.mask.pmovs.qb.256(<4 x i64> %x0, <16 x i8> %x1, i8 -1)
@@ -1890,23 +1919,24 @@ declare <16 x i8> @llvm.x86.avx512.mask.pmovus.qb.256(<4 x i64>, <16 x i8>, i8)
 define <16 x i8>@test_int_x86_avx512_mask_pmovus_qb_256(<4 x i64> %x0, <16 x i8> %x1, i8 %x2) {
 ; X86-LABEL: test_int_x86_avx512_mask_pmovus_qb_256:
 ; X86:       # %bb.0:
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
-; X86-NEXT:    vpmovusqb %ymm0, %xmm2 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0xa9,0x12,0xc2]
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vpmovusqb %ymm0, %xmm2 # encoding: [0x62,0xf2,0x7e,0x28,0x12,0xc2]
 ; X86-NEXT:    vpmovusqb %ymm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0x7e,0x29,0x12,0xc1]
-; X86-NEXT:    vpaddb %xmm2, %xmm1, %xmm1 # encoding: [0xc5,0xf1,0xfc,0xca]
-; X86-NEXT:    vpmovusqb %ymm0, %xmm0 # encoding: [0x62,0xf2,0x7e,0x28,0x12,0xc0]
-; X86-NEXT:    vpaddb %xmm1, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xfc,0xc1]
+; X86-NEXT:    vpmovusqb %ymm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0xa9,0x12,0xc0]
+; X86-NEXT:    vpaddb %xmm0, %xmm1, %xmm0 # encoding: [0xc5,0xf1,0xfc,0xc0]
+; X86-NEXT:    vpaddb %xmm0, %xmm2, %xmm0 # encoding: [0xc5,0xe9,0xfc,0xc0]
 ; X86-NEXT:    vzeroupper # encoding: [0xc5,0xf8,0x77]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_pmovus_qb_256:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
-; X64-NEXT:    vpmovusqb %ymm0, %xmm2 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0xa9,0x12,0xc2]
+; X64-NEXT:    vpmovusqb %ymm0, %xmm2 # encoding: [0x62,0xf2,0x7e,0x28,0x12,0xc2]
 ; X64-NEXT:    vpmovusqb %ymm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0x7e,0x29,0x12,0xc1]
-; X64-NEXT:    vpaddb %xmm2, %xmm1, %xmm1 # encoding: [0xc5,0xf1,0xfc,0xca]
-; X64-NEXT:    vpmovusqb %ymm0, %xmm0 # encoding: [0x62,0xf2,0x7e,0x28,0x12,0xc0]
-; X64-NEXT:    vpaddb %xmm1, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xfc,0xc1]
+; X64-NEXT:    vpmovusqb %ymm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0xa9,0x12,0xc0]
+; X64-NEXT:    vpaddb %xmm0, %xmm1, %xmm0 # encoding: [0xc5,0xf1,0xfc,0xc0]
+; X64-NEXT:    vpaddb %xmm0, %xmm2, %xmm0 # encoding: [0xc5,0xe9,0xfc,0xc0]
 ; X64-NEXT:    vzeroupper # encoding: [0xc5,0xf8,0x77]
 ; X64-NEXT:    retq # encoding: [0xc3]
     %res0 = call <16 x i8> @llvm.x86.avx512.mask.pmovus.qb.256(<4 x i64> %x0, <16 x i8> %x1, i8 -1)
@@ -1949,21 +1979,21 @@ define <8 x i16>@test_int_x86_avx512_mask_pmov_qw_128(<2 x i64> %x0, <8 x i16> %
 ; X86:       # %bb.0:
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vpmovqw %xmm0, %xmm2 # encoding: [0x62,0xf2,0x7e,0x08,0x34,0xc2]
 ; X86-NEXT:    vpmovqw %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0x7e,0x09,0x34,0xc1]
-; X86-NEXT:    vpmovqw %xmm0, %xmm2 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0x89,0x34,0xc2]
-; X86-NEXT:    vpaddw %xmm2, %xmm1, %xmm1 # encoding: [0xc5,0xf1,0xfd,0xca]
-; X86-NEXT:    vpmovqw %xmm0, %xmm0 # encoding: [0x62,0xf2,0x7e,0x08,0x34,0xc0]
-; X86-NEXT:    vpaddw %xmm1, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xfd,0xc1]
+; X86-NEXT:    vpmovqw %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0x89,0x34,0xc0]
+; X86-NEXT:    vpaddw %xmm0, %xmm1, %xmm0 # encoding: [0xc5,0xf1,0xfd,0xc0]
+; X86-NEXT:    vpaddw %xmm0, %xmm2, %xmm0 # encoding: [0xc5,0xe9,0xfd,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_pmov_qw_128:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
-; X64-NEXT:    vpmovqw %xmm0, %xmm2 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0x89,0x34,0xc2]
+; X64-NEXT:    vpmovqw %xmm0, %xmm2 # encoding: [0x62,0xf2,0x7e,0x08,0x34,0xc2]
 ; X64-NEXT:    vpmovqw %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0x7e,0x09,0x34,0xc1]
-; X64-NEXT:    vpaddw %xmm2, %xmm1, %xmm1 # encoding: [0xc5,0xf1,0xfd,0xca]
-; X64-NEXT:    vpmovqw %xmm0, %xmm0 # encoding: [0x62,0xf2,0x7e,0x08,0x34,0xc0]
-; X64-NEXT:    vpaddw %xmm1, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xfd,0xc1]
+; X64-NEXT:    vpmovqw %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0x89,0x34,0xc0]
+; X64-NEXT:    vpaddw %xmm0, %xmm1, %xmm0 # encoding: [0xc5,0xf1,0xfd,0xc0]
+; X64-NEXT:    vpaddw %xmm0, %xmm2, %xmm0 # encoding: [0xc5,0xe9,0xfd,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
     %res0 = call <8 x i16> @llvm.x86.avx512.mask.pmov.qw.128(<2 x i64> %x0, <8 x i16> %x1, i8 -1)
     %res1 = call <8 x i16> @llvm.x86.avx512.mask.pmov.qw.128(<2 x i64> %x0, <8 x i16> %x1, i8 %x2)
@@ -2003,21 +2033,21 @@ define <8 x i16>@test_int_x86_avx512_mask_pmovs_qw_128(<2 x i64> %x0, <8 x i16>
 ; X86:       # %bb.0:
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vpmovsqw %xmm0, %xmm2 # encoding: [0x62,0xf2,0x7e,0x08,0x24,0xc2]
 ; X86-NEXT:    vpmovsqw %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0x7e,0x09,0x24,0xc1]
-; X86-NEXT:    vpmovsqw %xmm0, %xmm2 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0x89,0x24,0xc2]
-; X86-NEXT:    vpaddw %xmm2, %xmm1, %xmm1 # encoding: [0xc5,0xf1,0xfd,0xca]
-; X86-NEXT:    vpmovsqw %xmm0, %xmm0 # encoding: [0x62,0xf2,0x7e,0x08,0x24,0xc0]
-; X86-NEXT:    vpaddw %xmm1, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xfd,0xc1]
+; X86-NEXT:    vpmovsqw %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0x89,0x24,0xc0]
+; X86-NEXT:    vpaddw %xmm0, %xmm1, %xmm0 # encoding: [0xc5,0xf1,0xfd,0xc0]
+; X86-NEXT:    vpaddw %xmm0, %xmm2, %xmm0 # encoding: [0xc5,0xe9,0xfd,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_pmovs_qw_128:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
-; X64-NEXT:    vpmovsqw %xmm0, %xmm2 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0x89,0x24,0xc2]
+; X64-NEXT:    vpmovsqw %xmm0, %xmm2 # encoding: [0x62,0xf2,0x7e,0x08,0x24,0xc2]
 ; X64-NEXT:    vpmovsqw %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0x7e,0x09,0x24,0xc1]
-; X64-NEXT:    vpaddw %xmm2, %xmm1, %xmm1 # encoding: [0xc5,0xf1,0xfd,0xca]
-; X64-NEXT:    vpmovsqw %xmm0, %xmm0 # encoding: [0x62,0xf2,0x7e,0x08,0x24,0xc0]
-; X64-NEXT:    vpaddw %xmm1, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xfd,0xc1]
+; X64-NEXT:    vpmovsqw %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0x89,0x24,0xc0]
+; X64-NEXT:    vpaddw %xmm0, %xmm1, %xmm0 # encoding: [0xc5,0xf1,0xfd,0xc0]
+; X64-NEXT:    vpaddw %xmm0, %xmm2, %xmm0 # encoding: [0xc5,0xe9,0xfd,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
     %res0 = call <8 x i16> @llvm.x86.avx512.mask.pmovs.qw.128(<2 x i64> %x0, <8 x i16> %x1, i8 -1)
     %res1 = call <8 x i16> @llvm.x86.avx512.mask.pmovs.qw.128(<2 x i64> %x0, <8 x i16> %x1, i8 %x2)
@@ -2057,21 +2087,21 @@ define <8 x i16>@test_int_x86_avx512_mask_pmovus_qw_128(<2 x i64> %x0, <8 x i16>
 ; X86:       # %bb.0:
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vpmovusqw %xmm0, %xmm2 # encoding: [0x62,0xf2,0x7e,0x08,0x14,0xc2]
 ; X86-NEXT:    vpmovusqw %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0x7e,0x09,0x14,0xc1]
-; X86-NEXT:    vpmovusqw %xmm0, %xmm2 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0x89,0x14,0xc2]
-; X86-NEXT:    vpaddw %xmm2, %xmm1, %xmm1 # encoding: [0xc5,0xf1,0xfd,0xca]
-; X86-NEXT:    vpmovusqw %xmm0, %xmm0 # encoding: [0x62,0xf2,0x7e,0x08,0x14,0xc0]
-; X86-NEXT:    vpaddw %xmm1, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xfd,0xc1]
+; X86-NEXT:    vpmovusqw %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0x89,0x14,0xc0]
+; X86-NEXT:    vpaddw %xmm0, %xmm1, %xmm0 # encoding: [0xc5,0xf1,0xfd,0xc0]
+; X86-NEXT:    vpaddw %xmm0, %xmm2, %xmm0 # encoding: [0xc5,0xe9,0xfd,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_pmovus_qw_128:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
-; X64-NEXT:    vpmovusqw %xmm0, %xmm2 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0x89,0x14,0xc2]
+; X64-NEXT:    vpmovusqw %xmm0, %xmm2 # encoding: [0x62,0xf2,0x7e,0x08,0x14,0xc2]
 ; X64-NEXT:    vpmovusqw %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0x7e,0x09,0x14,0xc1]
-; X64-NEXT:    vpaddw %xmm2, %xmm1, %xmm1 # encoding: [0xc5,0xf1,0xfd,0xca]
-; X64-NEXT:    vpmovusqw %xmm0, %xmm0 # encoding: [0x62,0xf2,0x7e,0x08,0x14,0xc0]
-; X64-NEXT:    vpaddw %xmm1, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xfd,0xc1]
+; X64-NEXT:    vpmovusqw %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0x89,0x14,0xc0]
+; X64-NEXT:    vpaddw %xmm0, %xmm1, %xmm0 # encoding: [0xc5,0xf1,0xfd,0xc0]
+; X64-NEXT:    vpaddw %xmm0, %xmm2, %xmm0 # encoding: [0xc5,0xe9,0xfd,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
     %res0 = call <8 x i16> @llvm.x86.avx512.mask.pmovus.qw.128(<2 x i64> %x0, <8 x i16> %x1, i8 -1)
     %res1 = call <8 x i16> @llvm.x86.avx512.mask.pmovus.qw.128(<2 x i64> %x0, <8 x i16> %x1, i8 %x2)
@@ -2111,22 +2141,22 @@ define <8 x i16>@test_int_x86_avx512_mask_pmov_qw_256(<4 x i64> %x0, <8 x i16> %
 ; X86:       # %bb.0:
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vpmovqw %ymm0, %xmm2 # encoding: [0x62,0xf2,0x7e,0x28,0x34,0xc2]
 ; X86-NEXT:    vpmovqw %ymm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0x7e,0x29,0x34,0xc1]
-; X86-NEXT:    vpmovqw %ymm0, %xmm2 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0xa9,0x34,0xc2]
-; X86-NEXT:    vpaddw %xmm2, %xmm1, %xmm1 # encoding: [0xc5,0xf1,0xfd,0xca]
-; X86-NEXT:    vpmovqw %ymm0, %xmm0 # encoding: [0x62,0xf2,0x7e,0x28,0x34,0xc0]
-; X86-NEXT:    vpaddw %xmm1, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xfd,0xc1]
+; X86-NEXT:    vpmovqw %ymm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0xa9,0x34,0xc0]
+; X86-NEXT:    vpaddw %xmm0, %xmm1, %xmm0 # encoding: [0xc5,0xf1,0xfd,0xc0]
+; X86-NEXT:    vpaddw %xmm0, %xmm2, %xmm0 # encoding: [0xc5,0xe9,0xfd,0xc0]
 ; X86-NEXT:    vzeroupper # encoding: [0xc5,0xf8,0x77]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_pmov_qw_256:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
-; X64-NEXT:    vpmovqw %ymm0, %xmm2 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0xa9,0x34,0xc2]
+; X64-NEXT:    vpmovqw %ymm0, %xmm2 # encoding: [0x62,0xf2,0x7e,0x28,0x34,0xc2]
 ; X64-NEXT:    vpmovqw %ymm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0x7e,0x29,0x34,0xc1]
-; X64-NEXT:    vpaddw %xmm2, %xmm1, %xmm1 # encoding: [0xc5,0xf1,0xfd,0xca]
-; X64-NEXT:    vpmovqw %ymm0, %xmm0 # encoding: [0x62,0xf2,0x7e,0x28,0x34,0xc0]
-; X64-NEXT:    vpaddw %xmm1, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xfd,0xc1]
+; X64-NEXT:    vpmovqw %ymm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0xa9,0x34,0xc0]
+; X64-NEXT:    vpaddw %xmm0, %xmm1, %xmm0 # encoding: [0xc5,0xf1,0xfd,0xc0]
+; X64-NEXT:    vpaddw %xmm0, %xmm2, %xmm0 # encoding: [0xc5,0xe9,0xfd,0xc0]
 ; X64-NEXT:    vzeroupper # encoding: [0xc5,0xf8,0x77]
 ; X64-NEXT:    retq # encoding: [0xc3]
     %res0 = call <8 x i16> @llvm.x86.avx512.mask.pmov.qw.256(<4 x i64> %x0, <8 x i16> %x1, i8 -1)
@@ -2169,22 +2199,22 @@ define <8 x i16>@test_int_x86_avx512_mask_pmovs_qw_256(<4 x i64> %x0, <8 x i16>
 ; X86:       # %bb.0:
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vpmovsqw %ymm0, %xmm2 # encoding: [0x62,0xf2,0x7e,0x28,0x24,0xc2]
 ; X86-NEXT:    vpmovsqw %ymm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0x7e,0x29,0x24,0xc1]
-; X86-NEXT:    vpmovsqw %ymm0, %xmm2 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0xa9,0x24,0xc2]
-; X86-NEXT:    vpaddw %xmm2, %xmm1, %xmm1 # encoding: [0xc5,0xf1,0xfd,0xca]
-; X86-NEXT:    vpmovsqw %ymm0, %xmm0 # encoding: [0x62,0xf2,0x7e,0x28,0x24,0xc0]
-; X86-NEXT:    vpaddw %xmm1, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xfd,0xc1]
+; X86-NEXT:    vpmovsqw %ymm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0xa9,0x24,0xc0]
+; X86-NEXT:    vpaddw %xmm0, %xmm1, %xmm0 # encoding: [0xc5,0xf1,0xfd,0xc0]
+; X86-NEXT:    vpaddw %xmm0, %xmm2, %xmm0 # encoding: [0xc5,0xe9,0xfd,0xc0]
 ; X86-NEXT:    vzeroupper # encoding: [0xc5,0xf8,0x77]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_pmovs_qw_256:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
-; X64-NEXT:    vpmovsqw %ymm0, %xmm2 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0xa9,0x24,0xc2]
+; X64-NEXT:    vpmovsqw %ymm0, %xmm2 # encoding: [0x62,0xf2,0x7e,0x28,0x24,0xc2]
 ; X64-NEXT:    vpmovsqw %ymm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0x7e,0x29,0x24,0xc1]
-; X64-NEXT:    vpaddw %xmm2, %xmm1, %xmm1 # encoding: [0xc5,0xf1,0xfd,0xca]
-; X64-NEXT:    vpmovsqw %ymm0, %xmm0 # encoding: [0x62,0xf2,0x7e,0x28,0x24,0xc0]
-; X64-NEXT:    vpaddw %xmm1, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xfd,0xc1]
+; X64-NEXT:    vpmovsqw %ymm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0xa9,0x24,0xc0]
+; X64-NEXT:    vpaddw %xmm0, %xmm1, %xmm0 # encoding: [0xc5,0xf1,0xfd,0xc0]
+; X64-NEXT:    vpaddw %xmm0, %xmm2, %xmm0 # encoding: [0xc5,0xe9,0xfd,0xc0]
 ; X64-NEXT:    vzeroupper # encoding: [0xc5,0xf8,0x77]
 ; X64-NEXT:    retq # encoding: [0xc3]
     %res0 = call <8 x i16> @llvm.x86.avx512.mask.pmovs.qw.256(<4 x i64> %x0, <8 x i16> %x1, i8 -1)
@@ -2227,22 +2257,22 @@ define <8 x i16>@test_int_x86_avx512_mask_pmovus_qw_256(<4 x i64> %x0, <8 x i16>
 ; X86:       # %bb.0:
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vpmovusqw %ymm0, %xmm2 # encoding: [0x62,0xf2,0x7e,0x28,0x14,0xc2]
 ; X86-NEXT:    vpmovusqw %ymm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0x7e,0x29,0x14,0xc1]
-; X86-NEXT:    vpmovusqw %ymm0, %xmm2 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0xa9,0x14,0xc2]
-; X86-NEXT:    vpaddw %xmm2, %xmm1, %xmm1 # encoding: [0xc5,0xf1,0xfd,0xca]
-; X86-NEXT:    vpmovusqw %ymm0, %xmm0 # encoding: [0x62,0xf2,0x7e,0x28,0x14,0xc0]
-; X86-NEXT:    vpaddw %xmm1, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xfd,0xc1]
+; X86-NEXT:    vpmovusqw %ymm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0xa9,0x14,0xc0]
+; X86-NEXT:    vpaddw %xmm0, %xmm1, %xmm0 # encoding: [0xc5,0xf1,0xfd,0xc0]
+; X86-NEXT:    vpaddw %xmm0, %xmm2, %xmm0 # encoding: [0xc5,0xe9,0xfd,0xc0]
 ; X86-NEXT:    vzeroupper # encoding: [0xc5,0xf8,0x77]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_pmovus_qw_256:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
-; X64-NEXT:    vpmovusqw %ymm0, %xmm2 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0xa9,0x14,0xc2]
+; X64-NEXT:    vpmovusqw %ymm0, %xmm2 # encoding: [0x62,0xf2,0x7e,0x28,0x14,0xc2]
 ; X64-NEXT:    vpmovusqw %ymm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0x7e,0x29,0x14,0xc1]
-; X64-NEXT:    vpaddw %xmm2, %xmm1, %xmm1 # encoding: [0xc5,0xf1,0xfd,0xca]
-; X64-NEXT:    vpmovusqw %ymm0, %xmm0 # encoding: [0x62,0xf2,0x7e,0x28,0x14,0xc0]
-; X64-NEXT:    vpaddw %xmm1, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xfd,0xc1]
+; X64-NEXT:    vpmovusqw %ymm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0xa9,0x14,0xc0]
+; X64-NEXT:    vpaddw %xmm0, %xmm1, %xmm0 # encoding: [0xc5,0xf1,0xfd,0xc0]
+; X64-NEXT:    vpaddw %xmm0, %xmm2, %xmm0 # encoding: [0xc5,0xe9,0xfd,0xc0]
 ; X64-NEXT:    vzeroupper # encoding: [0xc5,0xf8,0x77]
 ; X64-NEXT:    retq # encoding: [0xc3]
     %res0 = call <8 x i16> @llvm.x86.avx512.mask.pmovus.qw.256(<4 x i64> %x0, <8 x i16> %x1, i8 -1)
@@ -2285,21 +2315,21 @@ define <4 x i32>@test_int_x86_avx512_mask_pmov_qd_128(<2 x i64> %x0, <4 x i32> %
 ; X86:       # %bb.0:
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vpmovqd %xmm0, %xmm2 # encoding: [0x62,0xf2,0x7e,0x08,0x35,0xc2]
 ; X86-NEXT:    vpmovqd %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0x7e,0x09,0x35,0xc1]
-; X86-NEXT:    vpmovqd %xmm0, %xmm2 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0x89,0x35,0xc2]
-; X86-NEXT:    vpaddd %xmm2, %xmm1, %xmm1 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0xfe,0xca]
-; X86-NEXT:    vpmovqd %xmm0, %xmm0 # encoding: [0x62,0xf2,0x7e,0x08,0x35,0xc0]
-; X86-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfe,0xc1]
+; X86-NEXT:    vpmovqd %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0x89,0x35,0xc0]
+; X86-NEXT:    vpaddd %xmm0, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0xfe,0xc0]
+; X86-NEXT:    vpaddd %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfe,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_pmov_qd_128:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
-; X64-NEXT:    vpmovqd %xmm0, %xmm2 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0x89,0x35,0xc2]
+; X64-NEXT:    vpmovqd %xmm0, %xmm2 # encoding: [0x62,0xf2,0x7e,0x08,0x35,0xc2]
 ; X64-NEXT:    vpmovqd %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0x7e,0x09,0x35,0xc1]
-; X64-NEXT:    vpaddd %xmm2, %xmm1, %xmm1 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0xfe,0xca]
-; X64-NEXT:    vpmovqd %xmm0, %xmm0 # encoding: [0x62,0xf2,0x7e,0x08,0x35,0xc0]
-; X64-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfe,0xc1]
+; X64-NEXT:    vpmovqd %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0x89,0x35,0xc0]
+; X64-NEXT:    vpaddd %xmm0, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0xfe,0xc0]
+; X64-NEXT:    vpaddd %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfe,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
     %res0 = call <4 x i32> @llvm.x86.avx512.mask.pmov.qd.128(<2 x i64> %x0, <4 x i32> %x1, i8 -1)
     %res1 = call <4 x i32> @llvm.x86.avx512.mask.pmov.qd.128(<2 x i64> %x0, <4 x i32> %x1, i8 %x2)
@@ -2339,20 +2369,20 @@ define <4 x i32>@test_int_x86_avx512_mask_pmovs_qd_128(<2 x i64> %x0, <4 x i32>
 ; X86:       # %bb.0:
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
-; X86-NEXT:    vpmovsqd %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0x7e,0x09,0x25,0xc1]
-; X86-NEXT:    vpmovsqd %xmm0, %xmm2 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0x89,0x25,0xc2]
-; X86-NEXT:    vpaddd %xmm2, %xmm1, %xmm1 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0xfe,0xca]
 ; X86-NEXT:    vpmovsqd %xmm0, %xmm0 # encoding: [0x62,0xf2,0x7e,0x08,0x25,0xc0]
+; X86-NEXT:    vmovdqa32 %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf1,0x7d,0x09,0x6f,0xc8]
+; X86-NEXT:    vmovdqa32 %xmm0, %xmm2 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0x89,0x6f,0xd0]
+; X86-NEXT:    vpaddd %xmm2, %xmm1, %xmm1 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0xfe,0xca]
 ; X86-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfe,0xc1]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_pmovs_qd_128:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
-; X64-NEXT:    vpmovsqd %xmm0, %xmm2 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0x89,0x25,0xc2]
-; X64-NEXT:    vpmovsqd %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0x7e,0x09,0x25,0xc1]
-; X64-NEXT:    vpaddd %xmm2, %xmm1, %xmm1 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0xfe,0xca]
 ; X64-NEXT:    vpmovsqd %xmm0, %xmm0 # encoding: [0x62,0xf2,0x7e,0x08,0x25,0xc0]
+; X64-NEXT:    vmovdqa32 %xmm0, %xmm2 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0x89,0x6f,0xd0]
+; X64-NEXT:    vmovdqa32 %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf1,0x7d,0x09,0x6f,0xc8]
+; X64-NEXT:    vpaddd %xmm2, %xmm1, %xmm1 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0xfe,0xca]
 ; X64-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfe,0xc1]
 ; X64-NEXT:    retq # encoding: [0xc3]
     %res0 = call <4 x i32> @llvm.x86.avx512.mask.pmovs.qd.128(<2 x i64> %x0, <4 x i32> %x1, i8 -1)
@@ -2393,21 +2423,21 @@ define <4 x i32>@test_int_x86_avx512_mask_pmovus_qd_128(<2 x i64> %x0, <4 x i32>
 ; X86:       # %bb.0:
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vpmovusqd %xmm0, %xmm2 # encoding: [0x62,0xf2,0x7e,0x08,0x15,0xc2]
 ; X86-NEXT:    vpmovusqd %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0x7e,0x09,0x15,0xc1]
-; X86-NEXT:    vpmovusqd %xmm0, %xmm2 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0x89,0x15,0xc2]
-; X86-NEXT:    vpaddd %xmm2, %xmm1, %xmm1 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0xfe,0xca]
-; X86-NEXT:    vpmovusqd %xmm0, %xmm0 # encoding: [0x62,0xf2,0x7e,0x08,0x15,0xc0]
-; X86-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfe,0xc1]
+; X86-NEXT:    vpmovusqd %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0x89,0x15,0xc0]
+; X86-NEXT:    vpaddd %xmm0, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0xfe,0xc0]
+; X86-NEXT:    vpaddd %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfe,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_pmovus_qd_128:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
-; X64-NEXT:    vpmovusqd %xmm0, %xmm2 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0x89,0x15,0xc2]
+; X64-NEXT:    vpmovusqd %xmm0, %xmm2 # encoding: [0x62,0xf2,0x7e,0x08,0x15,0xc2]
 ; X64-NEXT:    vpmovusqd %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0x7e,0x09,0x15,0xc1]
-; X64-NEXT:    vpaddd %xmm2, %xmm1, %xmm1 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0xfe,0xca]
-; X64-NEXT:    vpmovusqd %xmm0, %xmm0 # encoding: [0x62,0xf2,0x7e,0x08,0x15,0xc0]
-; X64-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfe,0xc1]
+; X64-NEXT:    vpmovusqd %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0x89,0x15,0xc0]
+; X64-NEXT:    vpaddd %xmm0, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0xfe,0xc0]
+; X64-NEXT:    vpaddd %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfe,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
     %res0 = call <4 x i32> @llvm.x86.avx512.mask.pmovus.qd.128(<2 x i64> %x0, <4 x i32> %x1, i8 -1)
     %res1 = call <4 x i32> @llvm.x86.avx512.mask.pmovus.qd.128(<2 x i64> %x0, <4 x i32> %x1, i8 %x2)
@@ -2440,37 +2470,41 @@ define void @test_int_x86_avx512_mask_pmovus_qd_mem_128(i8* %ptr, <2 x i64> %x1,
     ret void
 }
 
-declare <4 x i32> @llvm.x86.avx512.mask.pmov.qd.256(<4 x i64>, <4 x i32>, i8)
-
 define <4 x i32>@test_int_x86_avx512_mask_pmov_qd_256(<4 x i64> %x0, <4 x i32> %x1, i8 %x2) {
 ; X86-LABEL: test_int_x86_avx512_mask_pmov_qd_256:
 ; X86:       # %bb.0:
+; X86-NEXT:    vpmovqd %ymm0, %xmm2 # encoding: [0x62,0xf2,0x7e,0x28,0x35,0xc2]
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vpmovqd %ymm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0x7e,0x29,0x35,0xc1]
-; X86-NEXT:    vpmovqd %ymm0, %xmm2 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0xa9,0x35,0xc2]
-; X86-NEXT:    vpaddd %xmm2, %xmm1, %xmm1 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0xfe,0xca]
-; X86-NEXT:    vpmovqd %ymm0, %xmm0 # encoding: [0x62,0xf2,0x7e,0x28,0x35,0xc0]
-; X86-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfe,0xc1]
+; X86-NEXT:    vpmovqd %ymm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0xa9,0x35,0xc0]
+; X86-NEXT:    vpaddd %xmm0, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0xfe,0xc0]
+; X86-NEXT:    vpaddd %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfe,0xc0]
 ; X86-NEXT:    vzeroupper # encoding: [0xc5,0xf8,0x77]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_pmov_qd_256:
 ; X64:       # %bb.0:
+; X64-NEXT:    vpmovqd %ymm0, %xmm2 # encoding: [0x62,0xf2,0x7e,0x28,0x35,0xc2]
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
-; X64-NEXT:    vpmovqd %ymm0, %xmm2 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0xa9,0x35,0xc2]
 ; X64-NEXT:    vpmovqd %ymm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0x7e,0x29,0x35,0xc1]
-; X64-NEXT:    vpaddd %xmm2, %xmm1, %xmm1 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0xfe,0xca]
-; X64-NEXT:    vpmovqd %ymm0, %xmm0 # encoding: [0x62,0xf2,0x7e,0x28,0x35,0xc0]
-; X64-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfe,0xc1]
+; X64-NEXT:    vpmovqd %ymm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0xa9,0x35,0xc0]
+; X64-NEXT:    vpaddd %xmm0, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0xfe,0xc0]
+; X64-NEXT:    vpaddd %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfe,0xc0]
 ; X64-NEXT:    vzeroupper # encoding: [0xc5,0xf8,0x77]
 ; X64-NEXT:    retq # encoding: [0xc3]
-    %res0 = call <4 x i32> @llvm.x86.avx512.mask.pmov.qd.256(<4 x i64> %x0, <4 x i32> %x1, i8 -1)
-    %res1 = call <4 x i32> @llvm.x86.avx512.mask.pmov.qd.256(<4 x i64> %x0, <4 x i32> %x1, i8 %x2)
-    %res2 = call <4 x i32> @llvm.x86.avx512.mask.pmov.qd.256(<4 x i64> %x0, <4 x i32> zeroinitializer, i8 %x2)
-    %res3 = add <4 x i32> %res0, %res1
-    %res4 = add <4 x i32> %res3, %res2
-    ret <4 x i32> %res4
+  %1 = trunc <4 x i64> %x0 to <4 x i32>
+  %2 = trunc <4 x i64> %x0 to <4 x i32>
+  %3 = bitcast i8 %x2 to <8 x i1>
+  %extract1 = shufflevector <8 x i1> %3, <8 x i1> %3, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %4 = select <4 x i1> %extract1, <4 x i32> %2, <4 x i32> %x1
+  %5 = trunc <4 x i64> %x0 to <4 x i32>
+  %6 = bitcast i8 %x2 to <8 x i1>
+  %extract = shufflevector <8 x i1> %6, <8 x i1> %6, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %7 = select <4 x i1> %extract, <4 x i32> %5, <4 x i32> zeroinitializer
+  %res3 = add <4 x i32> %1, %4
+  %res4 = add <4 x i32> %res3, %7
+  ret <4 x i32> %res4
 }
 
 declare void @llvm.x86.avx512.mask.pmov.qd.mem.256(i8* %ptr, <4 x i64>, i8)
@@ -2619,22 +2653,23 @@ declare <16 x i8> @llvm.x86.avx512.mask.pmov.db.128(<4 x i32>, <16 x i8>, i8)
 define <16 x i8>@test_int_x86_avx512_mask_pmov_db_128(<4 x i32> %x0, <16 x i8> %x1, i8 %x2) {
 ; X86-LABEL: test_int_x86_avx512_mask_pmov_db_128:
 ; X86:       # %bb.0:
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
-; X86-NEXT:    vpmovdb %xmm0, %xmm2 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0x89,0x31,0xc2]
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vpmovdb %xmm0, %xmm2 # encoding: [0x62,0xf2,0x7e,0x08,0x31,0xc2]
 ; X86-NEXT:    vpmovdb %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0x7e,0x09,0x31,0xc1]
-; X86-NEXT:    vpaddb %xmm2, %xmm1, %xmm1 # encoding: [0xc5,0xf1,0xfc,0xca]
-; X86-NEXT:    vpmovdb %xmm0, %xmm0 # encoding: [0x62,0xf2,0x7e,0x08,0x31,0xc0]
-; X86-NEXT:    vpaddb %xmm1, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xfc,0xc1]
+; X86-NEXT:    vpmovdb %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0x89,0x31,0xc0]
+; X86-NEXT:    vpaddb %xmm0, %xmm1, %xmm0 # encoding: [0xc5,0xf1,0xfc,0xc0]
+; X86-NEXT:    vpaddb %xmm0, %xmm2, %xmm0 # encoding: [0xc5,0xe9,0xfc,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_pmov_db_128:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
-; X64-NEXT:    vpmovdb %xmm0, %xmm2 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0x89,0x31,0xc2]
+; X64-NEXT:    vpmovdb %xmm0, %xmm2 # encoding: [0x62,0xf2,0x7e,0x08,0x31,0xc2]
 ; X64-NEXT:    vpmovdb %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0x7e,0x09,0x31,0xc1]
-; X64-NEXT:    vpaddb %xmm2, %xmm1, %xmm1 # encoding: [0xc5,0xf1,0xfc,0xca]
-; X64-NEXT:    vpmovdb %xmm0, %xmm0 # encoding: [0x62,0xf2,0x7e,0x08,0x31,0xc0]
-; X64-NEXT:    vpaddb %xmm1, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xfc,0xc1]
+; X64-NEXT:    vpmovdb %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0x89,0x31,0xc0]
+; X64-NEXT:    vpaddb %xmm0, %xmm1, %xmm0 # encoding: [0xc5,0xf1,0xfc,0xc0]
+; X64-NEXT:    vpaddb %xmm0, %xmm2, %xmm0 # encoding: [0xc5,0xe9,0xfc,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
     %res0 = call <16 x i8> @llvm.x86.avx512.mask.pmov.db.128(<4 x i32> %x0, <16 x i8> %x1, i8 -1)
     %res1 = call <16 x i8> @llvm.x86.avx512.mask.pmov.db.128(<4 x i32> %x0, <16 x i8> %x1, i8 %x2)
@@ -2672,22 +2707,23 @@ declare <16 x i8> @llvm.x86.avx512.mask.pmovs.db.128(<4 x i32>, <16 x i8>, i8)
 define <16 x i8>@test_int_x86_avx512_mask_pmovs_db_128(<4 x i32> %x0, <16 x i8> %x1, i8 %x2) {
 ; X86-LABEL: test_int_x86_avx512_mask_pmovs_db_128:
 ; X86:       # %bb.0:
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
-; X86-NEXT:    vpmovsdb %xmm0, %xmm2 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0x89,0x21,0xc2]
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vpmovsdb %xmm0, %xmm2 # encoding: [0x62,0xf2,0x7e,0x08,0x21,0xc2]
 ; X86-NEXT:    vpmovsdb %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0x7e,0x09,0x21,0xc1]
-; X86-NEXT:    vpaddb %xmm2, %xmm1, %xmm1 # encoding: [0xc5,0xf1,0xfc,0xca]
-; X86-NEXT:    vpmovsdb %xmm0, %xmm0 # encoding: [0x62,0xf2,0x7e,0x08,0x21,0xc0]
-; X86-NEXT:    vpaddb %xmm1, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xfc,0xc1]
+; X86-NEXT:    vpmovsdb %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0x89,0x21,0xc0]
+; X86-NEXT:    vpaddb %xmm0, %xmm1, %xmm0 # encoding: [0xc5,0xf1,0xfc,0xc0]
+; X86-NEXT:    vpaddb %xmm0, %xmm2, %xmm0 # encoding: [0xc5,0xe9,0xfc,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_pmovs_db_128:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
-; X64-NEXT:    vpmovsdb %xmm0, %xmm2 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0x89,0x21,0xc2]
+; X64-NEXT:    vpmovsdb %xmm0, %xmm2 # encoding: [0x62,0xf2,0x7e,0x08,0x21,0xc2]
 ; X64-NEXT:    vpmovsdb %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0x7e,0x09,0x21,0xc1]
-; X64-NEXT:    vpaddb %xmm2, %xmm1, %xmm1 # encoding: [0xc5,0xf1,0xfc,0xca]
-; X64-NEXT:    vpmovsdb %xmm0, %xmm0 # encoding: [0x62,0xf2,0x7e,0x08,0x21,0xc0]
-; X64-NEXT:    vpaddb %xmm1, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xfc,0xc1]
+; X64-NEXT:    vpmovsdb %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0x89,0x21,0xc0]
+; X64-NEXT:    vpaddb %xmm0, %xmm1, %xmm0 # encoding: [0xc5,0xf1,0xfc,0xc0]
+; X64-NEXT:    vpaddb %xmm0, %xmm2, %xmm0 # encoding: [0xc5,0xe9,0xfc,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
     %res0 = call <16 x i8> @llvm.x86.avx512.mask.pmovs.db.128(<4 x i32> %x0, <16 x i8> %x1, i8 -1)
     %res1 = call <16 x i8> @llvm.x86.avx512.mask.pmovs.db.128(<4 x i32> %x0, <16 x i8> %x1, i8 %x2)
@@ -2725,22 +2761,23 @@ declare <16 x i8> @llvm.x86.avx512.mask.pmovus.db.128(<4 x i32>, <16 x i8>, i8)
 define <16 x i8>@test_int_x86_avx512_mask_pmovus_db_128(<4 x i32> %x0, <16 x i8> %x1, i8 %x2) {
 ; X86-LABEL: test_int_x86_avx512_mask_pmovus_db_128:
 ; X86:       # %bb.0:
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
-; X86-NEXT:    vpmovusdb %xmm0, %xmm2 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0x89,0x11,0xc2]
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vpmovusdb %xmm0, %xmm2 # encoding: [0x62,0xf2,0x7e,0x08,0x11,0xc2]
 ; X86-NEXT:    vpmovusdb %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0x7e,0x09,0x11,0xc1]
-; X86-NEXT:    vpaddb %xmm2, %xmm1, %xmm1 # encoding: [0xc5,0xf1,0xfc,0xca]
-; X86-NEXT:    vpmovusdb %xmm0, %xmm0 # encoding: [0x62,0xf2,0x7e,0x08,0x11,0xc0]
-; X86-NEXT:    vpaddb %xmm1, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xfc,0xc1]
+; X86-NEXT:    vpmovusdb %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0x89,0x11,0xc0]
+; X86-NEXT:    vpaddb %xmm0, %xmm1, %xmm0 # encoding: [0xc5,0xf1,0xfc,0xc0]
+; X86-NEXT:    vpaddb %xmm0, %xmm2, %xmm0 # encoding: [0xc5,0xe9,0xfc,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_pmovus_db_128:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
-; X64-NEXT:    vpmovusdb %xmm0, %xmm2 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0x89,0x11,0xc2]
+; X64-NEXT:    vpmovusdb %xmm0, %xmm2 # encoding: [0x62,0xf2,0x7e,0x08,0x11,0xc2]
 ; X64-NEXT:    vpmovusdb %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0x7e,0x09,0x11,0xc1]
-; X64-NEXT:    vpaddb %xmm2, %xmm1, %xmm1 # encoding: [0xc5,0xf1,0xfc,0xca]
-; X64-NEXT:    vpmovusdb %xmm0, %xmm0 # encoding: [0x62,0xf2,0x7e,0x08,0x11,0xc0]
-; X64-NEXT:    vpaddb %xmm1, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xfc,0xc1]
+; X64-NEXT:    vpmovusdb %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0x89,0x11,0xc0]
+; X64-NEXT:    vpaddb %xmm0, %xmm1, %xmm0 # encoding: [0xc5,0xf1,0xfc,0xc0]
+; X64-NEXT:    vpaddb %xmm0, %xmm2, %xmm0 # encoding: [0xc5,0xe9,0xfc,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
     %res0 = call <16 x i8> @llvm.x86.avx512.mask.pmovus.db.128(<4 x i32> %x0, <16 x i8> %x1, i8 -1)
     %res1 = call <16 x i8> @llvm.x86.avx512.mask.pmovus.db.128(<4 x i32> %x0, <16 x i8> %x1, i8 %x2)
@@ -2778,23 +2815,24 @@ declare <16 x i8> @llvm.x86.avx512.mask.pmov.db.256(<8 x i32>, <16 x i8>, i8)
 define <16 x i8>@test_int_x86_avx512_mask_pmov_db_256(<8 x i32> %x0, <16 x i8> %x1, i8 %x2) {
 ; X86-LABEL: test_int_x86_avx512_mask_pmov_db_256:
 ; X86:       # %bb.0:
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
-; X86-NEXT:    vpmovdb %ymm0, %xmm2 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0xa9,0x31,0xc2]
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vpmovdb %ymm0, %xmm2 # encoding: [0x62,0xf2,0x7e,0x28,0x31,0xc2]
 ; X86-NEXT:    vpmovdb %ymm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0x7e,0x29,0x31,0xc1]
-; X86-NEXT:    vpaddb %xmm2, %xmm1, %xmm1 # encoding: [0xc5,0xf1,0xfc,0xca]
-; X86-NEXT:    vpmovdb %ymm0, %xmm0 # encoding: [0x62,0xf2,0x7e,0x28,0x31,0xc0]
-; X86-NEXT:    vpaddb %xmm1, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xfc,0xc1]
+; X86-NEXT:    vpmovdb %ymm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0xa9,0x31,0xc0]
+; X86-NEXT:    vpaddb %xmm0, %xmm1, %xmm0 # encoding: [0xc5,0xf1,0xfc,0xc0]
+; X86-NEXT:    vpaddb %xmm0, %xmm2, %xmm0 # encoding: [0xc5,0xe9,0xfc,0xc0]
 ; X86-NEXT:    vzeroupper # encoding: [0xc5,0xf8,0x77]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_pmov_db_256:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
-; X64-NEXT:    vpmovdb %ymm0, %xmm2 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0xa9,0x31,0xc2]
+; X64-NEXT:    vpmovdb %ymm0, %xmm2 # encoding: [0x62,0xf2,0x7e,0x28,0x31,0xc2]
 ; X64-NEXT:    vpmovdb %ymm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0x7e,0x29,0x31,0xc1]
-; X64-NEXT:    vpaddb %xmm2, %xmm1, %xmm1 # encoding: [0xc5,0xf1,0xfc,0xca]
-; X64-NEXT:    vpmovdb %ymm0, %xmm0 # encoding: [0x62,0xf2,0x7e,0x28,0x31,0xc0]
-; X64-NEXT:    vpaddb %xmm1, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xfc,0xc1]
+; X64-NEXT:    vpmovdb %ymm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0xa9,0x31,0xc0]
+; X64-NEXT:    vpaddb %xmm0, %xmm1, %xmm0 # encoding: [0xc5,0xf1,0xfc,0xc0]
+; X64-NEXT:    vpaddb %xmm0, %xmm2, %xmm0 # encoding: [0xc5,0xe9,0xfc,0xc0]
 ; X64-NEXT:    vzeroupper # encoding: [0xc5,0xf8,0x77]
 ; X64-NEXT:    retq # encoding: [0xc3]
     %res0 = call <16 x i8> @llvm.x86.avx512.mask.pmov.db.256(<8 x i32> %x0, <16 x i8> %x1, i8 -1)
@@ -2835,23 +2873,24 @@ declare <16 x i8> @llvm.x86.avx512.mask.pmovs.db.256(<8 x i32>, <16 x i8>, i8)
 define <16 x i8>@test_int_x86_avx512_mask_pmovs_db_256(<8 x i32> %x0, <16 x i8> %x1, i8 %x2) {
 ; X86-LABEL: test_int_x86_avx512_mask_pmovs_db_256:
 ; X86:       # %bb.0:
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
-; X86-NEXT:    vpmovsdb %ymm0, %xmm2 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0xa9,0x21,0xc2]
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vpmovsdb %ymm0, %xmm2 # encoding: [0x62,0xf2,0x7e,0x28,0x21,0xc2]
 ; X86-NEXT:    vpmovsdb %ymm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0x7e,0x29,0x21,0xc1]
-; X86-NEXT:    vpaddb %xmm2, %xmm1, %xmm1 # encoding: [0xc5,0xf1,0xfc,0xca]
-; X86-NEXT:    vpmovsdb %ymm0, %xmm0 # encoding: [0x62,0xf2,0x7e,0x28,0x21,0xc0]
-; X86-NEXT:    vpaddb %xmm1, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xfc,0xc1]
+; X86-NEXT:    vpmovsdb %ymm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0xa9,0x21,0xc0]
+; X86-NEXT:    vpaddb %xmm0, %xmm1, %xmm0 # encoding: [0xc5,0xf1,0xfc,0xc0]
+; X86-NEXT:    vpaddb %xmm0, %xmm2, %xmm0 # encoding: [0xc5,0xe9,0xfc,0xc0]
 ; X86-NEXT:    vzeroupper # encoding: [0xc5,0xf8,0x77]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_pmovs_db_256:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
-; X64-NEXT:    vpmovsdb %ymm0, %xmm2 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0xa9,0x21,0xc2]
+; X64-NEXT:    vpmovsdb %ymm0, %xmm2 # encoding: [0x62,0xf2,0x7e,0x28,0x21,0xc2]
 ; X64-NEXT:    vpmovsdb %ymm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0x7e,0x29,0x21,0xc1]
-; X64-NEXT:    vpaddb %xmm2, %xmm1, %xmm1 # encoding: [0xc5,0xf1,0xfc,0xca]
-; X64-NEXT:    vpmovsdb %ymm0, %xmm0 # encoding: [0x62,0xf2,0x7e,0x28,0x21,0xc0]
-; X64-NEXT:    vpaddb %xmm1, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xfc,0xc1]
+; X64-NEXT:    vpmovsdb %ymm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0xa9,0x21,0xc0]
+; X64-NEXT:    vpaddb %xmm0, %xmm1, %xmm0 # encoding: [0xc5,0xf1,0xfc,0xc0]
+; X64-NEXT:    vpaddb %xmm0, %xmm2, %xmm0 # encoding: [0xc5,0xe9,0xfc,0xc0]
 ; X64-NEXT:    vzeroupper # encoding: [0xc5,0xf8,0x77]
 ; X64-NEXT:    retq # encoding: [0xc3]
     %res0 = call <16 x i8> @llvm.x86.avx512.mask.pmovs.db.256(<8 x i32> %x0, <16 x i8> %x1, i8 -1)
@@ -2892,23 +2931,24 @@ declare <16 x i8> @llvm.x86.avx512.mask.pmovus.db.256(<8 x i32>, <16 x i8>, i8)
 define <16 x i8>@test_int_x86_avx512_mask_pmovus_db_256(<8 x i32> %x0, <16 x i8> %x1, i8 %x2) {
 ; X86-LABEL: test_int_x86_avx512_mask_pmovus_db_256:
 ; X86:       # %bb.0:
-; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
-; X86-NEXT:    vpmovusdb %ymm0, %xmm2 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0xa9,0x11,0xc2]
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
+; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vpmovusdb %ymm0, %xmm2 # encoding: [0x62,0xf2,0x7e,0x28,0x11,0xc2]
 ; X86-NEXT:    vpmovusdb %ymm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0x7e,0x29,0x11,0xc1]
-; X86-NEXT:    vpaddb %xmm2, %xmm1, %xmm1 # encoding: [0xc5,0xf1,0xfc,0xca]
-; X86-NEXT:    vpmovusdb %ymm0, %xmm0 # encoding: [0x62,0xf2,0x7e,0x28,0x11,0xc0]
-; X86-NEXT:    vpaddb %xmm1, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xfc,0xc1]
+; X86-NEXT:    vpmovusdb %ymm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0xa9,0x11,0xc0]
+; X86-NEXT:    vpaddb %xmm0, %xmm1, %xmm0 # encoding: [0xc5,0xf1,0xfc,0xc0]
+; X86-NEXT:    vpaddb %xmm0, %xmm2, %xmm0 # encoding: [0xc5,0xe9,0xfc,0xc0]
 ; X86-NEXT:    vzeroupper # encoding: [0xc5,0xf8,0x77]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_pmovus_db_256:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
-; X64-NEXT:    vpmovusdb %ymm0, %xmm2 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0xa9,0x11,0xc2]
+; X64-NEXT:    vpmovusdb %ymm0, %xmm2 # encoding: [0x62,0xf2,0x7e,0x28,0x11,0xc2]
 ; X64-NEXT:    vpmovusdb %ymm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0x7e,0x29,0x11,0xc1]
-; X64-NEXT:    vpaddb %xmm2, %xmm1, %xmm1 # encoding: [0xc5,0xf1,0xfc,0xca]
-; X64-NEXT:    vpmovusdb %ymm0, %xmm0 # encoding: [0x62,0xf2,0x7e,0x28,0x11,0xc0]
-; X64-NEXT:    vpaddb %xmm1, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xfc,0xc1]
+; X64-NEXT:    vpmovusdb %ymm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0xa9,0x11,0xc0]
+; X64-NEXT:    vpaddb %xmm0, %xmm1, %xmm0 # encoding: [0xc5,0xf1,0xfc,0xc0]
+; X64-NEXT:    vpaddb %xmm0, %xmm2, %xmm0 # encoding: [0xc5,0xe9,0xfc,0xc0]
 ; X64-NEXT:    vzeroupper # encoding: [0xc5,0xf8,0x77]
 ; X64-NEXT:    retq # encoding: [0xc3]
     %res0 = call <16 x i8> @llvm.x86.avx512.mask.pmovus.db.256(<8 x i32> %x0, <16 x i8> %x1, i8 -1)
@@ -2951,21 +2991,21 @@ define <8 x i16>@test_int_x86_avx512_mask_pmov_dw_128(<4 x i32> %x0, <8 x i16> %
 ; X86:       # %bb.0:
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vpmovdw %xmm0, %xmm2 # encoding: [0x62,0xf2,0x7e,0x08,0x33,0xc2]
 ; X86-NEXT:    vpmovdw %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0x7e,0x09,0x33,0xc1]
-; X86-NEXT:    vpmovdw %xmm0, %xmm2 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0x89,0x33,0xc2]
-; X86-NEXT:    vpaddw %xmm2, %xmm1, %xmm1 # encoding: [0xc5,0xf1,0xfd,0xca]
-; X86-NEXT:    vpmovdw %xmm0, %xmm0 # encoding: [0x62,0xf2,0x7e,0x08,0x33,0xc0]
-; X86-NEXT:    vpaddw %xmm1, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xfd,0xc1]
+; X86-NEXT:    vpmovdw %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0x89,0x33,0xc0]
+; X86-NEXT:    vpaddw %xmm0, %xmm1, %xmm0 # encoding: [0xc5,0xf1,0xfd,0xc0]
+; X86-NEXT:    vpaddw %xmm0, %xmm2, %xmm0 # encoding: [0xc5,0xe9,0xfd,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_pmov_dw_128:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
-; X64-NEXT:    vpmovdw %xmm0, %xmm2 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0x89,0x33,0xc2]
+; X64-NEXT:    vpmovdw %xmm0, %xmm2 # encoding: [0x62,0xf2,0x7e,0x08,0x33,0xc2]
 ; X64-NEXT:    vpmovdw %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0x7e,0x09,0x33,0xc1]
-; X64-NEXT:    vpaddw %xmm2, %xmm1, %xmm1 # encoding: [0xc5,0xf1,0xfd,0xca]
-; X64-NEXT:    vpmovdw %xmm0, %xmm0 # encoding: [0x62,0xf2,0x7e,0x08,0x33,0xc0]
-; X64-NEXT:    vpaddw %xmm1, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xfd,0xc1]
+; X64-NEXT:    vpmovdw %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0x89,0x33,0xc0]
+; X64-NEXT:    vpaddw %xmm0, %xmm1, %xmm0 # encoding: [0xc5,0xf1,0xfd,0xc0]
+; X64-NEXT:    vpaddw %xmm0, %xmm2, %xmm0 # encoding: [0xc5,0xe9,0xfd,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
     %res0 = call <8 x i16> @llvm.x86.avx512.mask.pmov.dw.128(<4 x i32> %x0, <8 x i16> %x1, i8 -1)
     %res1 = call <8 x i16> @llvm.x86.avx512.mask.pmov.dw.128(<4 x i32> %x0, <8 x i16> %x1, i8 %x2)
@@ -3005,21 +3045,21 @@ define <8 x i16>@test_int_x86_avx512_mask_pmovs_dw_128(<4 x i32> %x0, <8 x i16>
 ; X86:       # %bb.0:
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vpmovsdw %xmm0, %xmm2 # encoding: [0x62,0xf2,0x7e,0x08,0x23,0xc2]
 ; X86-NEXT:    vpmovsdw %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0x7e,0x09,0x23,0xc1]
-; X86-NEXT:    vpmovsdw %xmm0, %xmm2 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0x89,0x23,0xc2]
-; X86-NEXT:    vpaddw %xmm2, %xmm1, %xmm1 # encoding: [0xc5,0xf1,0xfd,0xca]
-; X86-NEXT:    vpmovsdw %xmm0, %xmm0 # encoding: [0x62,0xf2,0x7e,0x08,0x23,0xc0]
-; X86-NEXT:    vpaddw %xmm1, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xfd,0xc1]
+; X86-NEXT:    vpmovsdw %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0x89,0x23,0xc0]
+; X86-NEXT:    vpaddw %xmm0, %xmm1, %xmm0 # encoding: [0xc5,0xf1,0xfd,0xc0]
+; X86-NEXT:    vpaddw %xmm0, %xmm2, %xmm0 # encoding: [0xc5,0xe9,0xfd,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_pmovs_dw_128:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
-; X64-NEXT:    vpmovsdw %xmm0, %xmm2 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0x89,0x23,0xc2]
+; X64-NEXT:    vpmovsdw %xmm0, %xmm2 # encoding: [0x62,0xf2,0x7e,0x08,0x23,0xc2]
 ; X64-NEXT:    vpmovsdw %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0x7e,0x09,0x23,0xc1]
-; X64-NEXT:    vpaddw %xmm2, %xmm1, %xmm1 # encoding: [0xc5,0xf1,0xfd,0xca]
-; X64-NEXT:    vpmovsdw %xmm0, %xmm0 # encoding: [0x62,0xf2,0x7e,0x08,0x23,0xc0]
-; X64-NEXT:    vpaddw %xmm1, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xfd,0xc1]
+; X64-NEXT:    vpmovsdw %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0x89,0x23,0xc0]
+; X64-NEXT:    vpaddw %xmm0, %xmm1, %xmm0 # encoding: [0xc5,0xf1,0xfd,0xc0]
+; X64-NEXT:    vpaddw %xmm0, %xmm2, %xmm0 # encoding: [0xc5,0xe9,0xfd,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
     %res0 = call <8 x i16> @llvm.x86.avx512.mask.pmovs.dw.128(<4 x i32> %x0, <8 x i16> %x1, i8 -1)
     %res1 = call <8 x i16> @llvm.x86.avx512.mask.pmovs.dw.128(<4 x i32> %x0, <8 x i16> %x1, i8 %x2)
@@ -3059,21 +3099,21 @@ define <8 x i16>@test_int_x86_avx512_mask_pmovus_dw_128(<4 x i32> %x0, <8 x i16>
 ; X86:       # %bb.0:
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vpmovusdw %xmm0, %xmm2 # encoding: [0x62,0xf2,0x7e,0x08,0x13,0xc2]
 ; X86-NEXT:    vpmovusdw %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0x7e,0x09,0x13,0xc1]
-; X86-NEXT:    vpmovusdw %xmm0, %xmm2 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0x89,0x13,0xc2]
-; X86-NEXT:    vpaddw %xmm2, %xmm1, %xmm1 # encoding: [0xc5,0xf1,0xfd,0xca]
-; X86-NEXT:    vpmovusdw %xmm0, %xmm0 # encoding: [0x62,0xf2,0x7e,0x08,0x13,0xc0]
-; X86-NEXT:    vpaddw %xmm1, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xfd,0xc1]
+; X86-NEXT:    vpmovusdw %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0x89,0x13,0xc0]
+; X86-NEXT:    vpaddw %xmm0, %xmm1, %xmm0 # encoding: [0xc5,0xf1,0xfd,0xc0]
+; X86-NEXT:    vpaddw %xmm0, %xmm2, %xmm0 # encoding: [0xc5,0xe9,0xfd,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_pmovus_dw_128:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
-; X64-NEXT:    vpmovusdw %xmm0, %xmm2 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0x89,0x13,0xc2]
+; X64-NEXT:    vpmovusdw %xmm0, %xmm2 # encoding: [0x62,0xf2,0x7e,0x08,0x13,0xc2]
 ; X64-NEXT:    vpmovusdw %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0x7e,0x09,0x13,0xc1]
-; X64-NEXT:    vpaddw %xmm2, %xmm1, %xmm1 # encoding: [0xc5,0xf1,0xfd,0xca]
-; X64-NEXT:    vpmovusdw %xmm0, %xmm0 # encoding: [0x62,0xf2,0x7e,0x08,0x13,0xc0]
-; X64-NEXT:    vpaddw %xmm1, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xfd,0xc1]
+; X64-NEXT:    vpmovusdw %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0x89,0x13,0xc0]
+; X64-NEXT:    vpaddw %xmm0, %xmm1, %xmm0 # encoding: [0xc5,0xf1,0xfd,0xc0]
+; X64-NEXT:    vpaddw %xmm0, %xmm2, %xmm0 # encoding: [0xc5,0xe9,0xfd,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
     %res0 = call <8 x i16> @llvm.x86.avx512.mask.pmovus.dw.128(<4 x i32> %x0, <8 x i16> %x1, i8 -1)
     %res1 = call <8 x i16> @llvm.x86.avx512.mask.pmovus.dw.128(<4 x i32> %x0, <8 x i16> %x1, i8 %x2)
@@ -3113,22 +3153,22 @@ define <8 x i16>@test_int_x86_avx512_mask_pmov_dw_256(<8 x i32> %x0, <8 x i16> %
 ; X86:       # %bb.0:
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vpmovdw %ymm0, %xmm2 # encoding: [0x62,0xf2,0x7e,0x28,0x33,0xc2]
 ; X86-NEXT:    vpmovdw %ymm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0x7e,0x29,0x33,0xc1]
-; X86-NEXT:    vpmovdw %ymm0, %xmm2 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0xa9,0x33,0xc2]
-; X86-NEXT:    vpaddw %xmm2, %xmm1, %xmm1 # encoding: [0xc5,0xf1,0xfd,0xca]
-; X86-NEXT:    vpmovdw %ymm0, %xmm0 # encoding: [0x62,0xf2,0x7e,0x28,0x33,0xc0]
-; X86-NEXT:    vpaddw %xmm1, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xfd,0xc1]
+; X86-NEXT:    vpmovdw %ymm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0xa9,0x33,0xc0]
+; X86-NEXT:    vpaddw %xmm0, %xmm1, %xmm0 # encoding: [0xc5,0xf1,0xfd,0xc0]
+; X86-NEXT:    vpaddw %xmm0, %xmm2, %xmm0 # encoding: [0xc5,0xe9,0xfd,0xc0]
 ; X86-NEXT:    vzeroupper # encoding: [0xc5,0xf8,0x77]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_pmov_dw_256:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
-; X64-NEXT:    vpmovdw %ymm0, %xmm2 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0xa9,0x33,0xc2]
+; X64-NEXT:    vpmovdw %ymm0, %xmm2 # encoding: [0x62,0xf2,0x7e,0x28,0x33,0xc2]
 ; X64-NEXT:    vpmovdw %ymm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0x7e,0x29,0x33,0xc1]
-; X64-NEXT:    vpaddw %xmm2, %xmm1, %xmm1 # encoding: [0xc5,0xf1,0xfd,0xca]
-; X64-NEXT:    vpmovdw %ymm0, %xmm0 # encoding: [0x62,0xf2,0x7e,0x28,0x33,0xc0]
-; X64-NEXT:    vpaddw %xmm1, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xfd,0xc1]
+; X64-NEXT:    vpmovdw %ymm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0xa9,0x33,0xc0]
+; X64-NEXT:    vpaddw %xmm0, %xmm1, %xmm0 # encoding: [0xc5,0xf1,0xfd,0xc0]
+; X64-NEXT:    vpaddw %xmm0, %xmm2, %xmm0 # encoding: [0xc5,0xe9,0xfd,0xc0]
 ; X64-NEXT:    vzeroupper # encoding: [0xc5,0xf8,0x77]
 ; X64-NEXT:    retq # encoding: [0xc3]
     %res0 = call <8 x i16> @llvm.x86.avx512.mask.pmov.dw.256(<8 x i32> %x0, <8 x i16> %x1, i8 -1)
@@ -3171,22 +3211,22 @@ define <8 x i16>@test_int_x86_avx512_mask_pmovs_dw_256(<8 x i32> %x0, <8 x i16>
 ; X86:       # %bb.0:
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vpmovsdw %ymm0, %xmm2 # encoding: [0x62,0xf2,0x7e,0x28,0x23,0xc2]
 ; X86-NEXT:    vpmovsdw %ymm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0x7e,0x29,0x23,0xc1]
-; X86-NEXT:    vpmovsdw %ymm0, %xmm2 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0xa9,0x23,0xc2]
-; X86-NEXT:    vpaddw %xmm2, %xmm1, %xmm1 # encoding: [0xc5,0xf1,0xfd,0xca]
-; X86-NEXT:    vpmovsdw %ymm0, %xmm0 # encoding: [0x62,0xf2,0x7e,0x28,0x23,0xc0]
-; X86-NEXT:    vpaddw %xmm1, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xfd,0xc1]
+; X86-NEXT:    vpmovsdw %ymm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0xa9,0x23,0xc0]
+; X86-NEXT:    vpaddw %xmm0, %xmm1, %xmm0 # encoding: [0xc5,0xf1,0xfd,0xc0]
+; X86-NEXT:    vpaddw %xmm0, %xmm2, %xmm0 # encoding: [0xc5,0xe9,0xfd,0xc0]
 ; X86-NEXT:    vzeroupper # encoding: [0xc5,0xf8,0x77]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_pmovs_dw_256:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
-; X64-NEXT:    vpmovsdw %ymm0, %xmm2 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0xa9,0x23,0xc2]
+; X64-NEXT:    vpmovsdw %ymm0, %xmm2 # encoding: [0x62,0xf2,0x7e,0x28,0x23,0xc2]
 ; X64-NEXT:    vpmovsdw %ymm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0x7e,0x29,0x23,0xc1]
-; X64-NEXT:    vpaddw %xmm2, %xmm1, %xmm1 # encoding: [0xc5,0xf1,0xfd,0xca]
-; X64-NEXT:    vpmovsdw %ymm0, %xmm0 # encoding: [0x62,0xf2,0x7e,0x28,0x23,0xc0]
-; X64-NEXT:    vpaddw %xmm1, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xfd,0xc1]
+; X64-NEXT:    vpmovsdw %ymm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0xa9,0x23,0xc0]
+; X64-NEXT:    vpaddw %xmm0, %xmm1, %xmm0 # encoding: [0xc5,0xf1,0xfd,0xc0]
+; X64-NEXT:    vpaddw %xmm0, %xmm2, %xmm0 # encoding: [0xc5,0xe9,0xfd,0xc0]
 ; X64-NEXT:    vzeroupper # encoding: [0xc5,0xf8,0x77]
 ; X64-NEXT:    retq # encoding: [0xc3]
     %res0 = call <8 x i16> @llvm.x86.avx512.mask.pmovs.dw.256(<8 x i32> %x0, <8 x i16> %x1, i8 -1)
@@ -3229,22 +3269,22 @@ define <8 x i16>@test_int_x86_avx512_mask_pmovus_dw_256(<8 x i32> %x0, <8 x i16>
 ; X86:       # %bb.0:
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vpmovusdw %ymm0, %xmm2 # encoding: [0x62,0xf2,0x7e,0x28,0x13,0xc2]
 ; X86-NEXT:    vpmovusdw %ymm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0x7e,0x29,0x13,0xc1]
-; X86-NEXT:    vpmovusdw %ymm0, %xmm2 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0xa9,0x13,0xc2]
-; X86-NEXT:    vpaddw %xmm2, %xmm1, %xmm1 # encoding: [0xc5,0xf1,0xfd,0xca]
-; X86-NEXT:    vpmovusdw %ymm0, %xmm0 # encoding: [0x62,0xf2,0x7e,0x28,0x13,0xc0]
-; X86-NEXT:    vpaddw %xmm1, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xfd,0xc1]
+; X86-NEXT:    vpmovusdw %ymm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0xa9,0x13,0xc0]
+; X86-NEXT:    vpaddw %xmm0, %xmm1, %xmm0 # encoding: [0xc5,0xf1,0xfd,0xc0]
+; X86-NEXT:    vpaddw %xmm0, %xmm2, %xmm0 # encoding: [0xc5,0xe9,0xfd,0xc0]
 ; X86-NEXT:    vzeroupper # encoding: [0xc5,0xf8,0x77]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_int_x86_avx512_mask_pmovus_dw_256:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
-; X64-NEXT:    vpmovusdw %ymm0, %xmm2 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0xa9,0x13,0xc2]
+; X64-NEXT:    vpmovusdw %ymm0, %xmm2 # encoding: [0x62,0xf2,0x7e,0x28,0x13,0xc2]
 ; X64-NEXT:    vpmovusdw %ymm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0x7e,0x29,0x13,0xc1]
-; X64-NEXT:    vpaddw %xmm2, %xmm1, %xmm1 # encoding: [0xc5,0xf1,0xfd,0xca]
-; X64-NEXT:    vpmovusdw %ymm0, %xmm0 # encoding: [0x62,0xf2,0x7e,0x28,0x13,0xc0]
-; X64-NEXT:    vpaddw %xmm1, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xfd,0xc1]
+; X64-NEXT:    vpmovusdw %ymm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7e,0xa9,0x13,0xc0]
+; X64-NEXT:    vpaddw %xmm0, %xmm1, %xmm0 # encoding: [0xc5,0xf1,0xfd,0xc0]
+; X64-NEXT:    vpaddw %xmm0, %xmm2, %xmm0 # encoding: [0xc5,0xe9,0xfd,0xc0]
 ; X64-NEXT:    vzeroupper # encoding: [0xc5,0xf8,0x77]
 ; X64-NEXT:    retq # encoding: [0xc3]
     %res0 = call <8 x i16> @llvm.x86.avx512.mask.pmovus.dw.256(<8 x i32> %x0, <8 x i16> %x1, i8 -1)
@@ -3311,8 +3351,6 @@ define <4 x i32>@test_int_x86_avx512_mask_cvt_pd2dq_128_zext(<2 x double> %x0, <
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vcvtpd2dq %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf1,0xff,0x09,0xe6,0xc8]
-; X86-NEXT:    vmovq %xmm1, %xmm1 # EVEX TO VEX Compression encoding: [0xc5,0xfa,0x7e,0xc9]
-; X86-NEXT:    # xmm1 = xmm1[0],zero
 ; X86-NEXT:    vcvtpd2dq %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xfb,0xe6,0xc0]
 ; X86-NEXT:    vpaddd %xmm0, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0xfe,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
@@ -3321,8 +3359,6 @@ define <4 x i32>@test_int_x86_avx512_mask_cvt_pd2dq_128_zext(<2 x double> %x0, <
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vcvtpd2dq %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf1,0xff,0x09,0xe6,0xc8]
-; X64-NEXT:    vmovq %xmm1, %xmm1 # EVEX TO VEX Compression encoding: [0xc5,0xfa,0x7e,0xc9]
-; X64-NEXT:    # xmm1 = xmm1[0],zero
 ; X64-NEXT:    vcvtpd2dq %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xfb,0xe6,0xc0]
 ; X64-NEXT:    vpaddd %xmm0, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0xfe,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
@@ -3365,8 +3401,6 @@ define <4 x float>@test_int_x86_avx512_mask_cvt_pd2ps_zext(<2 x double> %x0, <4
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vcvtpd2ps %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf1,0xfd,0x09,0x5a,0xc8]
-; X86-NEXT:    vmovq %xmm1, %xmm1 # EVEX TO VEX Compression encoding: [0xc5,0xfa,0x7e,0xc9]
-; X86-NEXT:    # xmm1 = xmm1[0],zero
 ; X86-NEXT:    vcvtpd2ps %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x5a,0xc0]
 ; X86-NEXT:    vaddps %xmm0, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf0,0x58,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
@@ -3375,8 +3409,6 @@ define <4 x float>@test_int_x86_avx512_mask_cvt_pd2ps_zext(<2 x double> %x0, <4
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vcvtpd2ps %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf1,0xfd,0x09,0x5a,0xc8]
-; X64-NEXT:    vmovq %xmm1, %xmm1 # EVEX TO VEX Compression encoding: [0xc5,0xfa,0x7e,0xc9]
-; X64-NEXT:    # xmm1 = xmm1[0],zero
 ; X64-NEXT:    vcvtpd2ps %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x5a,0xc0]
 ; X64-NEXT:    vaddps %xmm0, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf0,0x58,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
@@ -3419,8 +3451,6 @@ define <4 x i32>@test_int_x86_avx512_mask_cvt_pd2udq_128_zext(<2 x double> %x0,
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vcvtpd2udq %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf1,0xfc,0x09,0x79,0xc8]
-; X86-NEXT:    vmovq %xmm1, %xmm1 # EVEX TO VEX Compression encoding: [0xc5,0xfa,0x7e,0xc9]
-; X86-NEXT:    # xmm1 = xmm1[0],zero
 ; X86-NEXT:    vcvtpd2udq %xmm0, %xmm0 # encoding: [0x62,0xf1,0xfc,0x08,0x79,0xc0]
 ; X86-NEXT:    vpaddd %xmm0, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0xfe,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
@@ -3429,8 +3459,6 @@ define <4 x i32>@test_int_x86_avx512_mask_cvt_pd2udq_128_zext(<2 x double> %x0,
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vcvtpd2udq %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf1,0xfc,0x09,0x79,0xc8]
-; X64-NEXT:    vmovq %xmm1, %xmm1 # EVEX TO VEX Compression encoding: [0xc5,0xfa,0x7e,0xc9]
-; X64-NEXT:    # xmm1 = xmm1[0],zero
 ; X64-NEXT:    vcvtpd2udq %xmm0, %xmm0 # encoding: [0x62,0xf1,0xfc,0x08,0x79,0xc0]
 ; X64-NEXT:    vpaddd %xmm0, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0xfe,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
@@ -3600,8 +3628,6 @@ define <4 x i32>@test_int_x86_avx512_mask_cvtt_pd2dq_128_zext(<2 x double> %x0,
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vcvttpd2dq %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf1,0xfd,0x09,0xe6,0xc8]
-; X86-NEXT:    vmovq %xmm1, %xmm1 # EVEX TO VEX Compression encoding: [0xc5,0xfa,0x7e,0xc9]
-; X86-NEXT:    # xmm1 = xmm1[0],zero
 ; X86-NEXT:    vcvttpd2dq %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xe6,0xc0]
 ; X86-NEXT:    vpaddd %xmm0, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0xfe,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
@@ -3610,8 +3636,6 @@ define <4 x i32>@test_int_x86_avx512_mask_cvtt_pd2dq_128_zext(<2 x double> %x0,
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vcvttpd2dq %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf1,0xfd,0x09,0xe6,0xc8]
-; X64-NEXT:    vmovq %xmm1, %xmm1 # EVEX TO VEX Compression encoding: [0xc5,0xfa,0x7e,0xc9]
-; X64-NEXT:    # xmm1 = xmm1[0],zero
 ; X64-NEXT:    vcvttpd2dq %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xe6,0xc0]
 ; X64-NEXT:    vpaddd %xmm0, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0xfe,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
@@ -3654,8 +3678,6 @@ define <4 x i32>@test_int_x86_avx512_mask_cvtt_pd2udq_128_zext(<2 x double> %x0,
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
 ; X86-NEXT:    vcvttpd2udq %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf1,0xfc,0x09,0x78,0xc8]
-; X86-NEXT:    vmovq %xmm1, %xmm1 # EVEX TO VEX Compression encoding: [0xc5,0xfa,0x7e,0xc9]
-; X86-NEXT:    # xmm1 = xmm1[0],zero
 ; X86-NEXT:    vcvttpd2udq %xmm0, %xmm0 # encoding: [0x62,0xf1,0xfc,0x08,0x78,0xc0]
 ; X86-NEXT:    vpaddd %xmm0, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0xfe,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
@@ -3664,8 +3686,6 @@ define <4 x i32>@test_int_x86_avx512_mask_cvtt_pd2udq_128_zext(<2 x double> %x0,
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
 ; X64-NEXT:    vcvttpd2udq %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf1,0xfc,0x09,0x78,0xc8]
-; X64-NEXT:    vmovq %xmm1, %xmm1 # EVEX TO VEX Compression encoding: [0xc5,0xfa,0x7e,0xc9]
-; X64-NEXT:    # xmm1 = xmm1[0],zero
 ; X64-NEXT:    vcvttpd2udq %xmm0, %xmm0 # encoding: [0x62,0xf1,0xfc,0x08,0x78,0xc0]
 ; X64-NEXT:    vpaddd %xmm0, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0xfe,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
@@ -4294,21 +4314,21 @@ define <8 x i16> @test_x86_vcvtps2ph_128(<4 x float> %a0, i8 %mask, <8 x i16> %s
 ; X86:       # %bb.0:
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vcvtps2ph $2, %xmm0, %xmm2 # EVEX TO VEX Compression encoding: [0xc4,0xe3,0x79,0x1d,0xc2,0x02]
+; X86-NEXT:    vcvtps2ph $2, %xmm0, %xmm3 {%k1} {z} # encoding: [0x62,0xf3,0x7d,0x89,0x1d,0xc3,0x02]
 ; X86-NEXT:    vcvtps2ph $2, %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf3,0x7d,0x09,0x1d,0xc1,0x02]
-; X86-NEXT:    vcvtps2ph $2, %xmm0, %xmm2 {%k1} {z} # encoding: [0x62,0xf3,0x7d,0x89,0x1d,0xc2,0x02]
-; X86-NEXT:    vpaddw %xmm1, %xmm2, %xmm1 # encoding: [0xc5,0xe9,0xfd,0xc9]
-; X86-NEXT:    vcvtps2ph $2, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe3,0x79,0x1d,0xc0,0x02]
-; X86-NEXT:    vpaddw %xmm1, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xfd,0xc1]
+; X86-NEXT:    vpaddw %xmm1, %xmm3, %xmm0 # encoding: [0xc5,0xe1,0xfd,0xc1]
+; X86-NEXT:    vpaddw %xmm0, %xmm2, %xmm0 # encoding: [0xc5,0xe9,0xfd,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_x86_vcvtps2ph_128:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vcvtps2ph $2, %xmm0, %xmm2 # EVEX TO VEX Compression encoding: [0xc4,0xe3,0x79,0x1d,0xc2,0x02]
+; X64-NEXT:    vcvtps2ph $2, %xmm0, %xmm3 {%k1} {z} # encoding: [0x62,0xf3,0x7d,0x89,0x1d,0xc3,0x02]
 ; X64-NEXT:    vcvtps2ph $2, %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf3,0x7d,0x09,0x1d,0xc1,0x02]
-; X64-NEXT:    vcvtps2ph $2, %xmm0, %xmm2 {%k1} {z} # encoding: [0x62,0xf3,0x7d,0x89,0x1d,0xc2,0x02]
-; X64-NEXT:    vpaddw %xmm1, %xmm2, %xmm1 # encoding: [0xc5,0xe9,0xfd,0xc9]
-; X64-NEXT:    vcvtps2ph $2, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe3,0x79,0x1d,0xc0,0x02]
-; X64-NEXT:    vpaddw %xmm1, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xfd,0xc1]
+; X64-NEXT:    vpaddw %xmm1, %xmm3, %xmm0 # encoding: [0xc5,0xe1,0xfd,0xc1]
+; X64-NEXT:    vpaddw %xmm0, %xmm2, %xmm0 # encoding: [0xc5,0xe9,0xfd,0xc0]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res1 = call <8 x i16> @llvm.x86.avx512.mask.vcvtps2ph.128(<4 x float> %a0, i32 2, <8 x i16> zeroinitializer, i8 -1)
   %res2 = call <8 x i16> @llvm.x86.avx512.mask.vcvtps2ph.128(<4 x float> %a0, i32 2, <8 x i16> zeroinitializer, i8 %mask)
@@ -4325,22 +4345,22 @@ define <8 x i16> @test_x86_vcvtps2ph_256(<8 x float> %a0, i8 %mask, <8 x i16> %s
 ; X86:       # %bb.0:
 ; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
 ; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vcvtps2ph $2, %ymm0, %xmm2 # EVEX TO VEX Compression encoding: [0xc4,0xe3,0x7d,0x1d,0xc2,0x02]
+; X86-NEXT:    vcvtps2ph $2, %ymm0, %xmm3 {%k1} {z} # encoding: [0x62,0xf3,0x7d,0xa9,0x1d,0xc3,0x02]
 ; X86-NEXT:    vcvtps2ph $2, %ymm0, %xmm1 {%k1} # encoding: [0x62,0xf3,0x7d,0x29,0x1d,0xc1,0x02]
-; X86-NEXT:    vcvtps2ph $2, %ymm0, %xmm2 {%k1} {z} # encoding: [0x62,0xf3,0x7d,0xa9,0x1d,0xc2,0x02]
-; X86-NEXT:    vpaddw %xmm1, %xmm2, %xmm1 # encoding: [0xc5,0xe9,0xfd,0xc9]
-; X86-NEXT:    vcvtps2ph $2, %ymm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe3,0x7d,0x1d,0xc0,0x02]
-; X86-NEXT:    vpaddw %xmm1, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xfd,0xc1]
+; X86-NEXT:    vpaddw %xmm1, %xmm3, %xmm0 # encoding: [0xc5,0xe1,0xfd,0xc1]
+; X86-NEXT:    vpaddw %xmm0, %xmm2, %xmm0 # encoding: [0xc5,0xe9,0xfd,0xc0]
 ; X86-NEXT:    vzeroupper # encoding: [0xc5,0xf8,0x77]
 ; X86-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-LABEL: test_x86_vcvtps2ph_256:
 ; X64:       # %bb.0:
 ; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
+; X64-NEXT:    vcvtps2ph $2, %ymm0, %xmm2 # EVEX TO VEX Compression encoding: [0xc4,0xe3,0x7d,0x1d,0xc2,0x02]
+; X64-NEXT:    vcvtps2ph $2, %ymm0, %xmm3 {%k1} {z} # encoding: [0x62,0xf3,0x7d,0xa9,0x1d,0xc3,0x02]
 ; X64-NEXT:    vcvtps2ph $2, %ymm0, %xmm1 {%k1} # encoding: [0x62,0xf3,0x7d,0x29,0x1d,0xc1,0x02]
-; X64-NEXT:    vcvtps2ph $2, %ymm0, %xmm2 {%k1} {z} # encoding: [0x62,0xf3,0x7d,0xa9,0x1d,0xc2,0x02]
-; X64-NEXT:    vpaddw %xmm1, %xmm2, %xmm1 # encoding: [0xc5,0xe9,0xfd,0xc9]
-; X64-NEXT:    vcvtps2ph $2, %ymm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe3,0x7d,0x1d,0xc0,0x02]
-; X64-NEXT:    vpaddw %xmm1, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xfd,0xc1]
+; X64-NEXT:    vpaddw %xmm1, %xmm3, %xmm0 # encoding: [0xc5,0xe1,0xfd,0xc1]
+; X64-NEXT:    vpaddw %xmm0, %xmm2, %xmm0 # encoding: [0xc5,0xe9,0xfd,0xc0]
 ; X64-NEXT:    vzeroupper # encoding: [0xc5,0xf8,0x77]
 ; X64-NEXT:    retq # encoding: [0xc3]
   %res1 = call <8 x i16> @llvm.x86.avx512.mask.vcvtps2ph.256(<8 x float> %a0, i32 2, <8 x i16> zeroinitializer, i8 -1)
@@ -4725,590 +4745,6 @@ define <2 x double> @test_rcp_pd_128_rrk(<2 x double> %a0, <2 x double> %a1, i8
 declare <4 x double> @llvm.x86.avx512.rcp14.pd.256(<4 x double>, <4 x double>, i8) nounwind readnone
 declare <2 x double> @llvm.x86.avx512.rcp14.pd.128(<2 x double>, <2 x double>, i8) nounwind readnone
 
-declare <4 x i32> @llvm.x86.avx512.prorv.d.128(<4 x i32>, <4 x i32>)
-
-define <4 x i32>@test_int_x86_avx512_mask_prorv_d_128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i8 %x3) {
-; X86-LABEL: test_int_x86_avx512_mask_prorv_d_128:
-; X86:       # %bb.0:
-; X86-NEXT:    vprorvd %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf2,0x7d,0x08,0x14,0xd9]
-; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
-; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
-; X86-NEXT:    vprorvd %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf2,0x7d,0x09,0x14,0xd1]
-; X86-NEXT:    vprorvd %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0x89,0x14,0xc1]
-; X86-NEXT:    vpaddd %xmm3, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfe,0xc3]
-; X86-NEXT:    vpaddd %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfe,0xc0]
-; X86-NEXT:    retl # encoding: [0xc3]
-;
-; X64-LABEL: test_int_x86_avx512_mask_prorv_d_128:
-; X64:       # %bb.0:
-; X64-NEXT:    vprorvd %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf2,0x7d,0x08,0x14,0xd9]
-; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
-; X64-NEXT:    vprorvd %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf2,0x7d,0x09,0x14,0xd1]
-; X64-NEXT:    vprorvd %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0x89,0x14,0xc1]
-; X64-NEXT:    vpaddd %xmm3, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfe,0xc3]
-; X64-NEXT:    vpaddd %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfe,0xc0]
-; X64-NEXT:    retq # encoding: [0xc3]
-  %1 = call <4 x i32> @llvm.x86.avx512.prorv.d.128(<4 x i32> %x0, <4 x i32> %x1)
-  %2 = bitcast i8 %x3 to <8 x i1>
-  %extract1 = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-  %3 = select <4 x i1> %extract1, <4 x i32> %1, <4 x i32> %x2
-  %4 = call <4 x i32> @llvm.x86.avx512.prorv.d.128(<4 x i32> %x0, <4 x i32> %x1)
-  %5 = bitcast i8 %x3 to <8 x i1>
-  %extract = shufflevector <8 x i1> %5, <8 x i1> %5, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-  %6 = select <4 x i1> %extract, <4 x i32> %4, <4 x i32> zeroinitializer
-  %7 = call <4 x i32> @llvm.x86.avx512.prorv.d.128(<4 x i32> %x0, <4 x i32> %x1)
-  %res3 = add <4 x i32> %3, %6
-  %res4 = add <4 x i32> %res3, %7
-  ret <4 x i32> %res4
-}
-
-declare <8 x i32> @llvm.x86.avx512.prorv.d.256(<8 x i32>, <8 x i32>)
-
-define <8 x i32>@test_int_x86_avx512_mask_prorv_d_256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i8 %x3) {
-; X86-LABEL: test_int_x86_avx512_mask_prorv_d_256:
-; X86:       # %bb.0:
-; X86-NEXT:    vprorvd %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf2,0x7d,0x28,0x14,0xd9]
-; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
-; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
-; X86-NEXT:    vprorvd %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf2,0x7d,0x29,0x14,0xd1]
-; X86-NEXT:    vprorvd %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0xa9,0x14,0xc1]
-; X86-NEXT:    vpaddd %ymm3, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xfe,0xc3]
-; X86-NEXT:    vpaddd %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfe,0xc0]
-; X86-NEXT:    retl # encoding: [0xc3]
-;
-; X64-LABEL: test_int_x86_avx512_mask_prorv_d_256:
-; X64:       # %bb.0:
-; X64-NEXT:    vprorvd %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf2,0x7d,0x28,0x14,0xd9]
-; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
-; X64-NEXT:    vprorvd %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf2,0x7d,0x29,0x14,0xd1]
-; X64-NEXT:    vprorvd %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0xa9,0x14,0xc1]
-; X64-NEXT:    vpaddd %ymm3, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xfe,0xc3]
-; X64-NEXT:    vpaddd %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfe,0xc0]
-; X64-NEXT:    retq # encoding: [0xc3]
-  %1 = call <8 x i32> @llvm.x86.avx512.prorv.d.256(<8 x i32> %x0, <8 x i32> %x1)
-  %2 = bitcast i8 %x3 to <8 x i1>
-  %3 = select <8 x i1> %2, <8 x i32> %1, <8 x i32> %x2
-  %4 = call <8 x i32> @llvm.x86.avx512.prorv.d.256(<8 x i32> %x0, <8 x i32> %x1)
-  %5 = bitcast i8 %x3 to <8 x i1>
-  %6 = select <8 x i1> %5, <8 x i32> %4, <8 x i32> zeroinitializer
-  %7 = call <8 x i32> @llvm.x86.avx512.prorv.d.256(<8 x i32> %x0, <8 x i32> %x1)
-  %res3 = add <8 x i32> %3, %6
-  %res4 = add <8 x i32> %res3, %7
-  ret <8 x i32> %res4
-}
-
-declare <2 x i64> @llvm.x86.avx512.prorv.q.128(<2 x i64>, <2 x i64>)
-
-define <2 x i64>@test_int_x86_avx512_mask_prorv_q_128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i8 %x3) {
-; X86-LABEL: test_int_x86_avx512_mask_prorv_q_128:
-; X86:       # %bb.0:
-; X86-NEXT:    vprorvq %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf2,0xfd,0x08,0x14,0xd9]
-; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
-; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
-; X86-NEXT:    vprorvq %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x09,0x14,0xd1]
-; X86-NEXT:    vprorvq %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0x89,0x14,0xc1]
-; X86-NEXT:    vpaddq %xmm3, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xd4,0xc3]
-; X86-NEXT:    vpaddq %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xd4,0xc0]
-; X86-NEXT:    retl # encoding: [0xc3]
-;
-; X64-LABEL: test_int_x86_avx512_mask_prorv_q_128:
-; X64:       # %bb.0:
-; X64-NEXT:    vprorvq %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf2,0xfd,0x08,0x14,0xd9]
-; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
-; X64-NEXT:    vprorvq %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x09,0x14,0xd1]
-; X64-NEXT:    vprorvq %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0x89,0x14,0xc1]
-; X64-NEXT:    vpaddq %xmm3, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xd4,0xc3]
-; X64-NEXT:    vpaddq %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xd4,0xc0]
-; X64-NEXT:    retq # encoding: [0xc3]
-  %1 = call <2 x i64> @llvm.x86.avx512.prorv.q.128(<2 x i64> %x0, <2 x i64> %x1)
-  %2 = bitcast i8 %x3 to <8 x i1>
-  %extract1 = shufflevector <8 x i1> %2, <8 x i1> %2, <2 x i32> <i32 0, i32 1>
-  %3 = select <2 x i1> %extract1, <2 x i64> %1, <2 x i64> %x2
-  %4 = call <2 x i64> @llvm.x86.avx512.prorv.q.128(<2 x i64> %x0, <2 x i64> %x1)
-  %5 = bitcast i8 %x3 to <8 x i1>
-  %extract = shufflevector <8 x i1> %5, <8 x i1> %5, <2 x i32> <i32 0, i32 1>
-  %6 = select <2 x i1> %extract, <2 x i64> %4, <2 x i64> zeroinitializer
-  %7 = call <2 x i64> @llvm.x86.avx512.prorv.q.128(<2 x i64> %x0, <2 x i64> %x1)
-  %res3 = add <2 x i64> %3, %6
-  %res4 = add <2 x i64> %res3, %7
-  ret <2 x i64> %res4
-}
-
-declare <4 x i64> @llvm.x86.avx512.prorv.q.256(<4 x i64>, <4 x i64>)
-
-define <4 x i64>@test_int_x86_avx512_mask_prorv_q_256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2, i8 %x3) {
-; X86-LABEL: test_int_x86_avx512_mask_prorv_q_256:
-; X86:       # %bb.0:
-; X86-NEXT:    vprorvq %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf2,0xfd,0x28,0x14,0xd9]
-; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
-; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
-; X86-NEXT:    vprorvq %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x29,0x14,0xd1]
-; X86-NEXT:    vprorvq %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xa9,0x14,0xc1]
-; X86-NEXT:    vpaddq %ymm3, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xd4,0xc3]
-; X86-NEXT:    vpaddq %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xd4,0xc0]
-; X86-NEXT:    retl # encoding: [0xc3]
-;
-; X64-LABEL: test_int_x86_avx512_mask_prorv_q_256:
-; X64:       # %bb.0:
-; X64-NEXT:    vprorvq %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf2,0xfd,0x28,0x14,0xd9]
-; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
-; X64-NEXT:    vprorvq %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x29,0x14,0xd1]
-; X64-NEXT:    vprorvq %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xa9,0x14,0xc1]
-; X64-NEXT:    vpaddq %ymm3, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xd4,0xc3]
-; X64-NEXT:    vpaddq %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xd4,0xc0]
-; X64-NEXT:    retq # encoding: [0xc3]
-  %1 = call <4 x i64> @llvm.x86.avx512.prorv.q.256(<4 x i64> %x0, <4 x i64> %x1)
-  %2 = bitcast i8 %x3 to <8 x i1>
-  %extract1 = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-  %3 = select <4 x i1> %extract1, <4 x i64> %1, <4 x i64> %x2
-  %4 = call <4 x i64> @llvm.x86.avx512.prorv.q.256(<4 x i64> %x0, <4 x i64> %x1)
-  %5 = bitcast i8 %x3 to <8 x i1>
-  %extract = shufflevector <8 x i1> %5, <8 x i1> %5, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-  %6 = select <4 x i1> %extract, <4 x i64> %4, <4 x i64> zeroinitializer
-  %7 = call <4 x i64> @llvm.x86.avx512.prorv.q.256(<4 x i64> %x0, <4 x i64> %x1)
-  %res3 = add <4 x i64> %3, %6
-  %res4 = add <4 x i64> %res3, %7
-  ret <4 x i64> %res4
-}
-
-declare <4 x i32> @llvm.x86.avx512.prol.d.128(<4 x i32>, i32)
-
-define <4 x i32>@test_int_x86_avx512_mask_prol_d_128(<4 x i32> %x0, i32 %x1, <4 x i32> %x2, i8 %x3) {
-; X86-LABEL: test_int_x86_avx512_mask_prol_d_128:
-; X86:       # %bb.0:
-; X86-NEXT:    vprold $3, %xmm0, %xmm2 # encoding: [0x62,0xf1,0x6d,0x08,0x72,0xc8,0x03]
-; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x08]
-; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
-; X86-NEXT:    vprold $3, %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf1,0x75,0x09,0x72,0xc8,0x03]
-; X86-NEXT:    vprold $3, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0x89,0x72,0xc8,0x03]
-; X86-NEXT:    vpaddd %xmm2, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfe,0xc2]
-; X86-NEXT:    vpaddd %xmm0, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0xfe,0xc0]
-; X86-NEXT:    retl # encoding: [0xc3]
-;
-; X64-LABEL: test_int_x86_avx512_mask_prol_d_128:
-; X64:       # %bb.0:
-; X64-NEXT:    vprold $3, %xmm0, %xmm2 # encoding: [0x62,0xf1,0x6d,0x08,0x72,0xc8,0x03]
-; X64-NEXT:    kmovw %esi, %k1 # encoding: [0xc5,0xf8,0x92,0xce]
-; X64-NEXT:    vprold $3, %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf1,0x75,0x09,0x72,0xc8,0x03]
-; X64-NEXT:    vprold $3, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0x89,0x72,0xc8,0x03]
-; X64-NEXT:    vpaddd %xmm2, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfe,0xc2]
-; X64-NEXT:    vpaddd %xmm0, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0xfe,0xc0]
-; X64-NEXT:    retq # encoding: [0xc3]
-  %1 = call <4 x i32> @llvm.x86.avx512.prol.d.128(<4 x i32> %x0, i32 3)
-  %2 = bitcast i8 %x3 to <8 x i1>
-  %extract1 = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-  %3 = select <4 x i1> %extract1, <4 x i32> %1, <4 x i32> %x2
-  %4 = call <4 x i32> @llvm.x86.avx512.prol.d.128(<4 x i32> %x0, i32 3)
-  %5 = bitcast i8 %x3 to <8 x i1>
-  %extract = shufflevector <8 x i1> %5, <8 x i1> %5, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-  %6 = select <4 x i1> %extract, <4 x i32> %4, <4 x i32> zeroinitializer
-  %7 = call <4 x i32> @llvm.x86.avx512.prol.d.128(<4 x i32> %x0, i32 3)
-  %res3 = add <4 x i32> %3, %6
-  %res4 = add <4 x i32> %res3, %7
-  ret <4 x i32> %res4
-}
-
-declare <8 x i32> @llvm.x86.avx512.prol.d.256(<8 x i32>, i32)
-
-define <8 x i32>@test_int_x86_avx512_mask_prol_d_256(<8 x i32> %x0, i32 %x1, <8 x i32> %x2, i8 %x3) {
-; X86-LABEL: test_int_x86_avx512_mask_prol_d_256:
-; X86:       # %bb.0:
-; X86-NEXT:    vprold $3, %ymm0, %ymm2 # encoding: [0x62,0xf1,0x6d,0x28,0x72,0xc8,0x03]
-; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x08]
-; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
-; X86-NEXT:    vprold $3, %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf1,0x75,0x29,0x72,0xc8,0x03]
-; X86-NEXT:    vprold $3, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xa9,0x72,0xc8,0x03]
-; X86-NEXT:    vpaddd %ymm2, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xfe,0xc2]
-; X86-NEXT:    vpaddd %ymm0, %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xf5,0xfe,0xc0]
-; X86-NEXT:    retl # encoding: [0xc3]
-;
-; X64-LABEL: test_int_x86_avx512_mask_prol_d_256:
-; X64:       # %bb.0:
-; X64-NEXT:    vprold $3, %ymm0, %ymm2 # encoding: [0x62,0xf1,0x6d,0x28,0x72,0xc8,0x03]
-; X64-NEXT:    kmovw %esi, %k1 # encoding: [0xc5,0xf8,0x92,0xce]
-; X64-NEXT:    vprold $3, %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf1,0x75,0x29,0x72,0xc8,0x03]
-; X64-NEXT:    vprold $3, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xa9,0x72,0xc8,0x03]
-; X64-NEXT:    vpaddd %ymm2, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xfe,0xc2]
-; X64-NEXT:    vpaddd %ymm0, %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xf5,0xfe,0xc0]
-; X64-NEXT:    retq # encoding: [0xc3]
-  %1 = call <8 x i32> @llvm.x86.avx512.prol.d.256(<8 x i32> %x0, i32 3)
-  %2 = bitcast i8 %x3 to <8 x i1>
-  %3 = select <8 x i1> %2, <8 x i32> %1, <8 x i32> %x2
-  %4 = call <8 x i32> @llvm.x86.avx512.prol.d.256(<8 x i32> %x0, i32 3)
-  %5 = bitcast i8 %x3 to <8 x i1>
-  %6 = select <8 x i1> %5, <8 x i32> %4, <8 x i32> zeroinitializer
-  %7 = call <8 x i32> @llvm.x86.avx512.prol.d.256(<8 x i32> %x0, i32 3)
-  %res3 = add <8 x i32> %3, %6
-  %res4 = add <8 x i32> %res3, %7
-  ret <8 x i32> %res4
-}
-
-declare <2 x i64> @llvm.x86.avx512.prol.q.128(<2 x i64>, i32)
-
-define <2 x i64>@test_int_x86_avx512_mask_prol_q_128(<2 x i64> %x0, i32 %x1, <2 x i64> %x2, i8 %x3) {
-; X86-LABEL: test_int_x86_avx512_mask_prol_q_128:
-; X86:       # %bb.0:
-; X86-NEXT:    vprolq $3, %xmm0, %xmm2 # encoding: [0x62,0xf1,0xed,0x08,0x72,0xc8,0x03]
-; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x08]
-; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
-; X86-NEXT:    vprolq $3, %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf1,0xf5,0x09,0x72,0xc8,0x03]
-; X86-NEXT:    vprolq $3, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0xfd,0x89,0x72,0xc8,0x03]
-; X86-NEXT:    vpaddq %xmm2, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xd4,0xc2]
-; X86-NEXT:    vpaddq %xmm0, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0xd4,0xc0]
-; X86-NEXT:    retl # encoding: [0xc3]
-;
-; X64-LABEL: test_int_x86_avx512_mask_prol_q_128:
-; X64:       # %bb.0:
-; X64-NEXT:    vprolq $3, %xmm0, %xmm2 # encoding: [0x62,0xf1,0xed,0x08,0x72,0xc8,0x03]
-; X64-NEXT:    kmovw %esi, %k1 # encoding: [0xc5,0xf8,0x92,0xce]
-; X64-NEXT:    vprolq $3, %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf1,0xf5,0x09,0x72,0xc8,0x03]
-; X64-NEXT:    vprolq $3, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0xfd,0x89,0x72,0xc8,0x03]
-; X64-NEXT:    vpaddq %xmm2, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xd4,0xc2]
-; X64-NEXT:    vpaddq %xmm0, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0xd4,0xc0]
-; X64-NEXT:    retq # encoding: [0xc3]
-  %1 = call <2 x i64> @llvm.x86.avx512.prol.q.128(<2 x i64> %x0, i32 3)
-  %2 = bitcast i8 %x3 to <8 x i1>
-  %extract1 = shufflevector <8 x i1> %2, <8 x i1> %2, <2 x i32> <i32 0, i32 1>
-  %3 = select <2 x i1> %extract1, <2 x i64> %1, <2 x i64> %x2
-  %4 = call <2 x i64> @llvm.x86.avx512.prol.q.128(<2 x i64> %x0, i32 3)
-  %5 = bitcast i8 %x3 to <8 x i1>
-  %extract = shufflevector <8 x i1> %5, <8 x i1> %5, <2 x i32> <i32 0, i32 1>
-  %6 = select <2 x i1> %extract, <2 x i64> %4, <2 x i64> zeroinitializer
-  %7 = call <2 x i64> @llvm.x86.avx512.prol.q.128(<2 x i64> %x0, i32 3)
-  %res3 = add <2 x i64> %3, %6
-  %res4 = add <2 x i64> %res3, %7
-  ret <2 x i64> %res4
-}
-
-declare <4 x i64> @llvm.x86.avx512.prol.q.256(<4 x i64>, i32)
-
-define <4 x i64>@test_int_x86_avx512_mask_prol_q_256(<4 x i64> %x0, i32 %x1, <4 x i64> %x2, i8 %x3) {
-; X86-LABEL: test_int_x86_avx512_mask_prol_q_256:
-; X86:       # %bb.0:
-; X86-NEXT:    vprolq $3, %ymm0, %ymm2 # encoding: [0x62,0xf1,0xed,0x28,0x72,0xc8,0x03]
-; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x08]
-; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
-; X86-NEXT:    vprolq $3, %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf1,0xf5,0x29,0x72,0xc8,0x03]
-; X86-NEXT:    vprolq $3, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0xfd,0xa9,0x72,0xc8,0x03]
-; X86-NEXT:    vpaddq %ymm2, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xd4,0xc2]
-; X86-NEXT:    vpaddq %ymm0, %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xf5,0xd4,0xc0]
-; X86-NEXT:    retl # encoding: [0xc3]
-;
-; X64-LABEL: test_int_x86_avx512_mask_prol_q_256:
-; X64:       # %bb.0:
-; X64-NEXT:    vprolq $3, %ymm0, %ymm2 # encoding: [0x62,0xf1,0xed,0x28,0x72,0xc8,0x03]
-; X64-NEXT:    kmovw %esi, %k1 # encoding: [0xc5,0xf8,0x92,0xce]
-; X64-NEXT:    vprolq $3, %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf1,0xf5,0x29,0x72,0xc8,0x03]
-; X64-NEXT:    vprolq $3, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0xfd,0xa9,0x72,0xc8,0x03]
-; X64-NEXT:    vpaddq %ymm2, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xd4,0xc2]
-; X64-NEXT:    vpaddq %ymm0, %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xf5,0xd4,0xc0]
-; X64-NEXT:    retq # encoding: [0xc3]
-  %1 = call <4 x i64> @llvm.x86.avx512.prol.q.256(<4 x i64> %x0, i32 3)
-  %2 = bitcast i8 %x3 to <8 x i1>
-  %extract1 = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-  %3 = select <4 x i1> %extract1, <4 x i64> %1, <4 x i64> %x2
-  %4 = call <4 x i64> @llvm.x86.avx512.prol.q.256(<4 x i64> %x0, i32 3)
-  %5 = bitcast i8 %x3 to <8 x i1>
-  %extract = shufflevector <8 x i1> %5, <8 x i1> %5, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-  %6 = select <4 x i1> %extract, <4 x i64> %4, <4 x i64> zeroinitializer
-  %7 = call <4 x i64> @llvm.x86.avx512.prol.q.256(<4 x i64> %x0, i32 3)
-  %res3 = add <4 x i64> %3, %6
-  %res4 = add <4 x i64> %res3, %7
-  ret <4 x i64> %res4
-}
-
-declare <4 x i32> @llvm.x86.avx512.prolv.d.128(<4 x i32>, <4 x i32>)
-
-define <4 x i32>@test_int_x86_avx512_mask_prolv_d_128(<4 x i32> %x0, <4 x i32> %x1, <4 x i32> %x2, i8 %x3) {
-; X86-LABEL: test_int_x86_avx512_mask_prolv_d_128:
-; X86:       # %bb.0:
-; X86-NEXT:    vprolvd %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf2,0x7d,0x08,0x15,0xd9]
-; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
-; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
-; X86-NEXT:    vprolvd %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf2,0x7d,0x09,0x15,0xd1]
-; X86-NEXT:    vprolvd %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0x89,0x15,0xc1]
-; X86-NEXT:    vpaddd %xmm3, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfe,0xc3]
-; X86-NEXT:    vpaddd %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfe,0xc0]
-; X86-NEXT:    retl # encoding: [0xc3]
-;
-; X64-LABEL: test_int_x86_avx512_mask_prolv_d_128:
-; X64:       # %bb.0:
-; X64-NEXT:    vprolvd %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf2,0x7d,0x08,0x15,0xd9]
-; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
-; X64-NEXT:    vprolvd %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf2,0x7d,0x09,0x15,0xd1]
-; X64-NEXT:    vprolvd %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0x89,0x15,0xc1]
-; X64-NEXT:    vpaddd %xmm3, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfe,0xc3]
-; X64-NEXT:    vpaddd %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfe,0xc0]
-; X64-NEXT:    retq # encoding: [0xc3]
-  %1 = call <4 x i32> @llvm.x86.avx512.prolv.d.128(<4 x i32> %x0, <4 x i32> %x1)
-  %2 = bitcast i8 %x3 to <8 x i1>
-  %extract1 = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-  %3 = select <4 x i1> %extract1, <4 x i32> %1, <4 x i32> %x2
-  %4 = call <4 x i32> @llvm.x86.avx512.prolv.d.128(<4 x i32> %x0, <4 x i32> %x1)
-  %5 = bitcast i8 %x3 to <8 x i1>
-  %extract = shufflevector <8 x i1> %5, <8 x i1> %5, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-  %6 = select <4 x i1> %extract, <4 x i32> %4, <4 x i32> zeroinitializer
-  %7 = call <4 x i32> @llvm.x86.avx512.prolv.d.128(<4 x i32> %x0, <4 x i32> %x1)
-  %res3 = add <4 x i32> %3, %6
-  %res4 = add <4 x i32> %res3, %7
-  ret <4 x i32> %res4
-}
-
-declare <8 x i32> @llvm.x86.avx512.prolv.d.256(<8 x i32>, <8 x i32>)
-
-define <8 x i32>@test_int_x86_avx512_mask_prolv_d_256(<8 x i32> %x0, <8 x i32> %x1, <8 x i32> %x2, i8 %x3) {
-; X86-LABEL: test_int_x86_avx512_mask_prolv_d_256:
-; X86:       # %bb.0:
-; X86-NEXT:    vprolvd %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf2,0x7d,0x28,0x15,0xd9]
-; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
-; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
-; X86-NEXT:    vprolvd %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf2,0x7d,0x29,0x15,0xd1]
-; X86-NEXT:    vprolvd %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0xa9,0x15,0xc1]
-; X86-NEXT:    vpaddd %ymm3, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xfe,0xc3]
-; X86-NEXT:    vpaddd %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfe,0xc0]
-; X86-NEXT:    retl # encoding: [0xc3]
-;
-; X64-LABEL: test_int_x86_avx512_mask_prolv_d_256:
-; X64:       # %bb.0:
-; X64-NEXT:    vprolvd %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf2,0x7d,0x28,0x15,0xd9]
-; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
-; X64-NEXT:    vprolvd %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf2,0x7d,0x29,0x15,0xd1]
-; X64-NEXT:    vprolvd %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0xa9,0x15,0xc1]
-; X64-NEXT:    vpaddd %ymm3, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xfe,0xc3]
-; X64-NEXT:    vpaddd %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfe,0xc0]
-; X64-NEXT:    retq # encoding: [0xc3]
-  %1 = call <8 x i32> @llvm.x86.avx512.prolv.d.256(<8 x i32> %x0, <8 x i32> %x1)
-  %2 = bitcast i8 %x3 to <8 x i1>
-  %3 = select <8 x i1> %2, <8 x i32> %1, <8 x i32> %x2
-  %4 = call <8 x i32> @llvm.x86.avx512.prolv.d.256(<8 x i32> %x0, <8 x i32> %x1)
-  %5 = bitcast i8 %x3 to <8 x i1>
-  %6 = select <8 x i1> %5, <8 x i32> %4, <8 x i32> zeroinitializer
-  %7 = call <8 x i32> @llvm.x86.avx512.prolv.d.256(<8 x i32> %x0, <8 x i32> %x1)
-  %res3 = add <8 x i32> %3, %6
-  %res4 = add <8 x i32> %res3, %7
-  ret <8 x i32> %res4
-}
-
-declare <2 x i64> @llvm.x86.avx512.prolv.q.128(<2 x i64>, <2 x i64>)
-
-define <2 x i64>@test_int_x86_avx512_mask_prolv_q_128(<2 x i64> %x0, <2 x i64> %x1, <2 x i64> %x2, i8 %x3) {
-; X86-LABEL: test_int_x86_avx512_mask_prolv_q_128:
-; X86:       # %bb.0:
-; X86-NEXT:    vprolvq %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf2,0xfd,0x08,0x15,0xd9]
-; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
-; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
-; X86-NEXT:    vprolvq %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x09,0x15,0xd1]
-; X86-NEXT:    vprolvq %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0x89,0x15,0xc1]
-; X86-NEXT:    vpaddq %xmm3, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xd4,0xc3]
-; X86-NEXT:    vpaddq %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xd4,0xc0]
-; X86-NEXT:    retl # encoding: [0xc3]
-;
-; X64-LABEL: test_int_x86_avx512_mask_prolv_q_128:
-; X64:       # %bb.0:
-; X64-NEXT:    vprolvq %xmm1, %xmm0, %xmm3 # encoding: [0x62,0xf2,0xfd,0x08,0x15,0xd9]
-; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
-; X64-NEXT:    vprolvq %xmm1, %xmm0, %xmm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x09,0x15,0xd1]
-; X64-NEXT:    vprolvq %xmm1, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0x89,0x15,0xc1]
-; X64-NEXT:    vpaddq %xmm3, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xd4,0xc3]
-; X64-NEXT:    vpaddq %xmm0, %xmm2, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xd4,0xc0]
-; X64-NEXT:    retq # encoding: [0xc3]
-  %1 = call <2 x i64> @llvm.x86.avx512.prolv.q.128(<2 x i64> %x0, <2 x i64> %x1)
-  %2 = bitcast i8 %x3 to <8 x i1>
-  %extract1 = shufflevector <8 x i1> %2, <8 x i1> %2, <2 x i32> <i32 0, i32 1>
-  %3 = select <2 x i1> %extract1, <2 x i64> %1, <2 x i64> %x2
-  %4 = call <2 x i64> @llvm.x86.avx512.prolv.q.128(<2 x i64> %x0, <2 x i64> %x1)
-  %5 = bitcast i8 %x3 to <8 x i1>
-  %extract = shufflevector <8 x i1> %5, <8 x i1> %5, <2 x i32> <i32 0, i32 1>
-  %6 = select <2 x i1> %extract, <2 x i64> %4, <2 x i64> zeroinitializer
-  %7 = call <2 x i64> @llvm.x86.avx512.prolv.q.128(<2 x i64> %x0, <2 x i64> %x1)
-  %res3 = add <2 x i64> %3, %6
-  %res4 = add <2 x i64> %res3, %7
-  ret <2 x i64> %res4
-}
-
-declare <4 x i64> @llvm.x86.avx512.prolv.q.256(<4 x i64>, <4 x i64>)
-
-define <4 x i64>@test_int_x86_avx512_mask_prolv_q_256(<4 x i64> %x0, <4 x i64> %x1, <4 x i64> %x2, i8 %x3) {
-; X86-LABEL: test_int_x86_avx512_mask_prolv_q_256:
-; X86:       # %bb.0:
-; X86-NEXT:    vprolvq %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf2,0xfd,0x28,0x15,0xd9]
-; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
-; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
-; X86-NEXT:    vprolvq %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x29,0x15,0xd1]
-; X86-NEXT:    vprolvq %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xa9,0x15,0xc1]
-; X86-NEXT:    vpaddq %ymm3, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xd4,0xc3]
-; X86-NEXT:    vpaddq %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xd4,0xc0]
-; X86-NEXT:    retl # encoding: [0xc3]
-;
-; X64-LABEL: test_int_x86_avx512_mask_prolv_q_256:
-; X64:       # %bb.0:
-; X64-NEXT:    vprolvq %ymm1, %ymm0, %ymm3 # encoding: [0x62,0xf2,0xfd,0x28,0x15,0xd9]
-; X64-NEXT:    kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
-; X64-NEXT:    vprolvq %ymm1, %ymm0, %ymm2 {%k1} # encoding: [0x62,0xf2,0xfd,0x29,0x15,0xd1]
-; X64-NEXT:    vprolvq %ymm1, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf2,0xfd,0xa9,0x15,0xc1]
-; X64-NEXT:    vpaddq %ymm3, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xd4,0xc3]
-; X64-NEXT:    vpaddq %ymm0, %ymm2, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xd4,0xc0]
-; X64-NEXT:    retq # encoding: [0xc3]
-  %1 = call <4 x i64> @llvm.x86.avx512.prolv.q.256(<4 x i64> %x0, <4 x i64> %x1)
-  %2 = bitcast i8 %x3 to <8 x i1>
-  %extract1 = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-  %3 = select <4 x i1> %extract1, <4 x i64> %1, <4 x i64> %x2
-  %4 = call <4 x i64> @llvm.x86.avx512.prolv.q.256(<4 x i64> %x0, <4 x i64> %x1)
-  %5 = bitcast i8 %x3 to <8 x i1>
-  %extract = shufflevector <8 x i1> %5, <8 x i1> %5, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-  %6 = select <4 x i1> %extract, <4 x i64> %4, <4 x i64> zeroinitializer
-  %7 = call <4 x i64> @llvm.x86.avx512.prolv.q.256(<4 x i64> %x0, <4 x i64> %x1)
-  %res3 = add <4 x i64> %3, %6
-  %res4 = add <4 x i64> %res3, %7
-  ret <4 x i64> %res4
-}
-
-declare <4 x i32> @llvm.x86.avx512.pror.d.128(<4 x i32>, i32)
-
-define <4 x i32>@test_int_x86_avx512_mask_pror_d_128(<4 x i32> %x0, i32 %x1, <4 x i32> %x2, i8 %x3) {
-; X86-LABEL: test_int_x86_avx512_mask_pror_d_128:
-; X86:       # %bb.0:
-; X86-NEXT:    vprord $3, %xmm0, %xmm2 # encoding: [0x62,0xf1,0x6d,0x08,0x72,0xc0,0x03]
-; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x08]
-; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
-; X86-NEXT:    vprord $3, %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf1,0x75,0x09,0x72,0xc0,0x03]
-; X86-NEXT:    vprord $3, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0x89,0x72,0xc0,0x03]
-; X86-NEXT:    vpaddd %xmm2, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfe,0xc2]
-; X86-NEXT:    vpaddd %xmm0, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0xfe,0xc0]
-; X86-NEXT:    retl # encoding: [0xc3]
-;
-; X64-LABEL: test_int_x86_avx512_mask_pror_d_128:
-; X64:       # %bb.0:
-; X64-NEXT:    vprord $3, %xmm0, %xmm2 # encoding: [0x62,0xf1,0x6d,0x08,0x72,0xc0,0x03]
-; X64-NEXT:    kmovw %esi, %k1 # encoding: [0xc5,0xf8,0x92,0xce]
-; X64-NEXT:    vprord $3, %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf1,0x75,0x09,0x72,0xc0,0x03]
-; X64-NEXT:    vprord $3, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0x89,0x72,0xc0,0x03]
-; X64-NEXT:    vpaddd %xmm2, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfe,0xc2]
-; X64-NEXT:    vpaddd %xmm0, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0xfe,0xc0]
-; X64-NEXT:    retq # encoding: [0xc3]
-  %1 = call <4 x i32> @llvm.x86.avx512.pror.d.128(<4 x i32> %x0, i32 3)
-  %2 = bitcast i8 %x3 to <8 x i1>
-  %extract1 = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-  %3 = select <4 x i1> %extract1, <4 x i32> %1, <4 x i32> %x2
-  %4 = call <4 x i32> @llvm.x86.avx512.pror.d.128(<4 x i32> %x0, i32 3)
-  %5 = bitcast i8 %x3 to <8 x i1>
-  %extract = shufflevector <8 x i1> %5, <8 x i1> %5, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-  %6 = select <4 x i1> %extract, <4 x i32> %4, <4 x i32> zeroinitializer
-  %7 = call <4 x i32> @llvm.x86.avx512.pror.d.128(<4 x i32> %x0, i32 3)
-  %res3 = add <4 x i32> %3, %6
-  %res4 = add <4 x i32> %res3, %7
-  ret <4 x i32> %res4
-}
-
-declare <8 x i32> @llvm.x86.avx512.pror.d.256(<8 x i32>, i32)
-
-define <8 x i32>@test_int_x86_avx512_mask_pror_d_256(<8 x i32> %x0, i32 %x1, <8 x i32> %x2, i8 %x3) {
-; X86-LABEL: test_int_x86_avx512_mask_pror_d_256:
-; X86:       # %bb.0:
-; X86-NEXT:    vprord $3, %ymm0, %ymm2 # encoding: [0x62,0xf1,0x6d,0x28,0x72,0xc0,0x03]
-; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x08]
-; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
-; X86-NEXT:    vprord $3, %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf1,0x75,0x29,0x72,0xc0,0x03]
-; X86-NEXT:    vprord $3, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xa9,0x72,0xc0,0x03]
-; X86-NEXT:    vpaddd %ymm2, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xfe,0xc2]
-; X86-NEXT:    vpaddd %ymm0, %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xf5,0xfe,0xc0]
-; X86-NEXT:    retl # encoding: [0xc3]
-;
-; X64-LABEL: test_int_x86_avx512_mask_pror_d_256:
-; X64:       # %bb.0:
-; X64-NEXT:    vprord $3, %ymm0, %ymm2 # encoding: [0x62,0xf1,0x6d,0x28,0x72,0xc0,0x03]
-; X64-NEXT:    kmovw %esi, %k1 # encoding: [0xc5,0xf8,0x92,0xce]
-; X64-NEXT:    vprord $3, %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf1,0x75,0x29,0x72,0xc0,0x03]
-; X64-NEXT:    vprord $3, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xa9,0x72,0xc0,0x03]
-; X64-NEXT:    vpaddd %ymm2, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xfe,0xc2]
-; X64-NEXT:    vpaddd %ymm0, %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xf5,0xfe,0xc0]
-; X64-NEXT:    retq # encoding: [0xc3]
-  %1 = call <8 x i32> @llvm.x86.avx512.pror.d.256(<8 x i32> %x0, i32 3)
-  %2 = bitcast i8 %x3 to <8 x i1>
-  %3 = select <8 x i1> %2, <8 x i32> %1, <8 x i32> %x2
-  %4 = call <8 x i32> @llvm.x86.avx512.pror.d.256(<8 x i32> %x0, i32 3)
-  %5 = bitcast i8 %x3 to <8 x i1>
-  %6 = select <8 x i1> %5, <8 x i32> %4, <8 x i32> zeroinitializer
-  %7 = call <8 x i32> @llvm.x86.avx512.pror.d.256(<8 x i32> %x0, i32 3)
-  %res3 = add <8 x i32> %3, %6
-  %res4 = add <8 x i32> %res3, %7
-  ret <8 x i32> %res4
-}
-
-declare <2 x i64> @llvm.x86.avx512.pror.q.128(<2 x i64>, i32)
-
-define <2 x i64>@test_int_x86_avx512_mask_pror_q_128(<2 x i64> %x0, i32 %x1, <2 x i64> %x2, i8 %x3) {
-; X86-LABEL: test_int_x86_avx512_mask_pror_q_128:
-; X86:       # %bb.0:
-; X86-NEXT:    vprorq $3, %xmm0, %xmm2 # encoding: [0x62,0xf1,0xed,0x08,0x72,0xc0,0x03]
-; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x08]
-; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
-; X86-NEXT:    vprorq $3, %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf1,0xf5,0x09,0x72,0xc0,0x03]
-; X86-NEXT:    vprorq $3, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0xfd,0x89,0x72,0xc0,0x03]
-; X86-NEXT:    vpaddq %xmm2, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xd4,0xc2]
-; X86-NEXT:    vpaddq %xmm0, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0xd4,0xc0]
-; X86-NEXT:    retl # encoding: [0xc3]
-;
-; X64-LABEL: test_int_x86_avx512_mask_pror_q_128:
-; X64:       # %bb.0:
-; X64-NEXT:    vprorq $3, %xmm0, %xmm2 # encoding: [0x62,0xf1,0xed,0x08,0x72,0xc0,0x03]
-; X64-NEXT:    kmovw %esi, %k1 # encoding: [0xc5,0xf8,0x92,0xce]
-; X64-NEXT:    vprorq $3, %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf1,0xf5,0x09,0x72,0xc0,0x03]
-; X64-NEXT:    vprorq $3, %xmm0, %xmm0 {%k1} {z} # encoding: [0x62,0xf1,0xfd,0x89,0x72,0xc0,0x03]
-; X64-NEXT:    vpaddq %xmm2, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xd4,0xc2]
-; X64-NEXT:    vpaddq %xmm0, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0xd4,0xc0]
-; X64-NEXT:    retq # encoding: [0xc3]
-  %1 = call <2 x i64> @llvm.x86.avx512.pror.q.128(<2 x i64> %x0, i32 3)
-  %2 = bitcast i8 %x3 to <8 x i1>
-  %extract1 = shufflevector <8 x i1> %2, <8 x i1> %2, <2 x i32> <i32 0, i32 1>
-  %3 = select <2 x i1> %extract1, <2 x i64> %1, <2 x i64> %x2
-  %4 = call <2 x i64> @llvm.x86.avx512.pror.q.128(<2 x i64> %x0, i32 3)
-  %5 = bitcast i8 %x3 to <8 x i1>
-  %extract = shufflevector <8 x i1> %5, <8 x i1> %5, <2 x i32> <i32 0, i32 1>
-  %6 = select <2 x i1> %extract, <2 x i64> %4, <2 x i64> zeroinitializer
-  %7 = call <2 x i64> @llvm.x86.avx512.pror.q.128(<2 x i64> %x0, i32 3)
-  %res3 = add <2 x i64> %3, %6
-  %res4 = add <2 x i64> %res3, %7
-  ret <2 x i64> %res4
-}
-
-declare <4 x i64> @llvm.x86.avx512.pror.q.256(<4 x i64>, i32)
-
-define <4 x i64>@test_int_x86_avx512_mask_pror_q_256(<4 x i64> %x0, i32 %x1, <4 x i64> %x2, i8 %x3) {
-; X86-LABEL: test_int_x86_avx512_mask_pror_q_256:
-; X86:       # %bb.0:
-; X86-NEXT:    vprorq $3, %ymm0, %ymm2 # encoding: [0x62,0xf1,0xed,0x28,0x72,0xc0,0x03]
-; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x08]
-; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
-; X86-NEXT:    vprorq $3, %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf1,0xf5,0x29,0x72,0xc0,0x03]
-; X86-NEXT:    vprorq $3, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0xfd,0xa9,0x72,0xc0,0x03]
-; X86-NEXT:    vpaddq %ymm2, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xd4,0xc2]
-; X86-NEXT:    vpaddq %ymm0, %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xf5,0xd4,0xc0]
-; X86-NEXT:    retl # encoding: [0xc3]
-;
-; X64-LABEL: test_int_x86_avx512_mask_pror_q_256:
-; X64:       # %bb.0:
-; X64-NEXT:    vprorq $3, %ymm0, %ymm2 # encoding: [0x62,0xf1,0xed,0x28,0x72,0xc0,0x03]
-; X64-NEXT:    kmovw %esi, %k1 # encoding: [0xc5,0xf8,0x92,0xce]
-; X64-NEXT:    vprorq $3, %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf1,0xf5,0x29,0x72,0xc0,0x03]
-; X64-NEXT:    vprorq $3, %ymm0, %ymm0 {%k1} {z} # encoding: [0x62,0xf1,0xfd,0xa9,0x72,0xc0,0x03]
-; X64-NEXT:    vpaddq %ymm2, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xd4,0xc2]
-; X64-NEXT:    vpaddq %ymm0, %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xf5,0xd4,0xc0]
-; X64-NEXT:    retq # encoding: [0xc3]
-  %1 = call <4 x i64> @llvm.x86.avx512.pror.q.256(<4 x i64> %x0, i32 3)
-  %2 = bitcast i8 %x3 to <8 x i1>
-  %extract1 = shufflevector <8 x i1> %2, <8 x i1> %2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-  %3 = select <4 x i1> %extract1, <4 x i64> %1, <4 x i64> %x2
-  %4 = call <4 x i64> @llvm.x86.avx512.pror.q.256(<4 x i64> %x0, i32 3)
-  %5 = bitcast i8 %x3 to <8 x i1>
-  %extract = shufflevector <8 x i1> %5, <8 x i1> %5, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-  %6 = select <4 x i1> %extract, <4 x i64> %4, <4 x i64> zeroinitializer
-  %7 = call <4 x i64> @llvm.x86.avx512.pror.q.256(<4 x i64> %x0, i32 3)
-  %res3 = add <4 x i64> %3, %6
-  %res4 = add <4 x i64> %res3, %7
-  ret <4 x i64> %res4
-}
-
 declare <4 x double> @llvm.x86.avx512.permvar.df.256(<4 x double>, <4 x i64>)
 
 define <4 x double>@test_int_x86_avx512_mask_permvar_df_256(<4 x double> %x0, <4 x i64> %x1, <4 x double> %x2, i8 %x3) {
@@ -7503,3 +6939,19 @@ declare <8 x float> @llvm.fma.v8f32(<8 x float>, <8 x float>, <8 x float>)
 declare <4 x float> @llvm.fma.v4f32(<4 x float>, <4 x float>, <4 x float>)
 declare <4 x double> @llvm.fma.v4f64(<4 x double>, <4 x double>, <4 x double>)
 declare <2 x double> @llvm.fma.v2f64(<2 x double>, <2 x double>, <2 x double>)
+declare <2 x double> @llvm.x86.avx512.mask.compress.v2f64(<2 x double>, <2 x double>, <2 x i1>)
+declare <4 x float> @llvm.x86.avx512.mask.compress.v4f32(<4 x float>, <4 x float>, <4 x i1>)
+declare <2 x i64> @llvm.x86.avx512.mask.compress.v2i64(<2 x i64>, <2 x i64>, <2 x i1>)
+declare <4 x i32> @llvm.x86.avx512.mask.compress.v4i32(<4 x i32>, <4 x i32>, <4 x i1>)
+declare <2 x double> @llvm.x86.avx512.mask.expand.v2f64(<2 x double>, <2 x double>, <2 x i1>)
+declare <4 x float> @llvm.x86.avx512.mask.expand.v4f32(<4 x float>, <4 x float>, <4 x i1>)
+declare <2 x i64> @llvm.x86.avx512.mask.expand.v2i64(<2 x i64>, <2 x i64>, <2 x i1>)
+declare <4 x i32> @llvm.x86.avx512.mask.expand.v4i32(<4 x i32>, <4 x i32>, <4 x i1>)
+declare <4 x double> @llvm.x86.avx512.mask.compress.v4f64(<4 x double>, <4 x double>, <4 x i1>)
+declare <8 x float> @llvm.x86.avx512.mask.compress.v8f32(<8 x float>, <8 x float>, <8 x i1>)
+declare <4 x i64> @llvm.x86.avx512.mask.compress.v4i64(<4 x i64>, <4 x i64>, <4 x i1>)
+declare <8 x i32> @llvm.x86.avx512.mask.compress.v8i32(<8 x i32>, <8 x i32>, <8 x i1>)
+declare <4 x double> @llvm.x86.avx512.mask.expand.v4f64(<4 x double>, <4 x double>, <4 x i1>)
+declare <8 x float> @llvm.x86.avx512.mask.expand.v8f32(<8 x float>, <8 x float>, <8 x i1>)
+declare <4 x i64> @llvm.x86.avx512.mask.expand.v4i64(<4 x i64>, <4 x i64>, <4 x i1>)
+declare <8 x i32> @llvm.x86.avx512.mask.expand.v8i32(<8 x i32>, <8 x i32>, <8 x i1>)
diff --git a/test/CodeGen/X86/avx512vpopcntdq-schedule.ll b/test/CodeGen/X86/avx512vpopcntdq-schedule.ll
deleted file mode 100644
index f44374e802ded6a43f9473540f51b5285031e0dd..0000000000000000000000000000000000000000
--- a/test/CodeGen/X86/avx512vpopcntdq-schedule.ll
+++ /dev/null
@@ -1,80 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=x86-64 -mattr=+avx512vpopcntdq | FileCheck %s --check-prefix=GENERIC
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=icelake-client | FileCheck %s --check-prefix=ICELAKE
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=icelake-server | FileCheck %s --check-prefix=ICELAKE
-
-define void @test_vpopcntd(<16 x i32> %a0, <16 x i32> %a1, <16 x i32> *%a2, i16 %a3) {
-; GENERIC-LABEL: test_vpopcntd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    kmovw %esi, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    vpopcntd %zmm1, %zmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    vpopcntd %zmm1, %zmm0 {%k1} # sched: [1:0.50]
-; GENERIC-NEXT:    vpopcntd %zmm1, %zmm0 {%k1} {z} # sched: [1:0.50]
-; GENERIC-NEXT:    vpopcntd (%rdi), %zmm0 # sched: [8:0.50]
-; GENERIC-NEXT:    vpopcntd (%rdi), %zmm0 {%k1} # sched: [8:0.50]
-; GENERIC-NEXT:    vpopcntd (%rdi), %zmm0 {%k1} {z} # sched: [8:0.50]
-; GENERIC-NEXT:    vpopcntd (%rdi){1to16}, %zmm0 # sched: [8:0.50]
-; GENERIC-NEXT:    vpopcntd (%rdi){1to16}, %zmm0 {%k1} # sched: [8:0.50]
-; GENERIC-NEXT:    vpopcntd (%rdi){1to16}, %zmm0 {%k1} {z} # sched: [8:0.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    vzeroupper # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ICELAKE-LABEL: test_vpopcntd:
-; ICELAKE:       # %bb.0:
-; ICELAKE-NEXT:    kmovd %esi, %k1 # sched: [1:1.00]
-; ICELAKE-NEXT:    #APP
-; ICELAKE-NEXT:    vpopcntd %zmm1, %zmm0 # sched: [1:1.00]
-; ICELAKE-NEXT:    vpopcntd %zmm1, %zmm0 {%k1} # sched: [1:1.00]
-; ICELAKE-NEXT:    vpopcntd %zmm1, %zmm0 {%k1} {z} # sched: [1:1.00]
-; ICELAKE-NEXT:    vpopcntd (%rdi), %zmm0 # sched: [8:1.00]
-; ICELAKE-NEXT:    vpopcntd (%rdi), %zmm0 {%k1} # sched: [8:1.00]
-; ICELAKE-NEXT:    vpopcntd (%rdi), %zmm0 {%k1} {z} # sched: [8:1.00]
-; ICELAKE-NEXT:    vpopcntd (%rdi){1to16}, %zmm0 # sched: [8:1.00]
-; ICELAKE-NEXT:    vpopcntd (%rdi){1to16}, %zmm0 {%k1} # sched: [8:1.00]
-; ICELAKE-NEXT:    vpopcntd (%rdi){1to16}, %zmm0 {%k1} {z} # sched: [8:1.00]
-; ICELAKE-NEXT:    #NO_APP
-; ICELAKE-NEXT:    vzeroupper # sched: [0:0.67]
-; ICELAKE-NEXT:    retq # sched: [7:1.00]
-  tail call void asm "vpopcntd $1, $0 \0A\09 vpopcntd $1, $0 {$3} \0A\09 vpopcntd $1, $0 {$3} {z} \0A\09 vpopcntd $2, $0 \0A\09 vpopcntd $2, $0 {$3} \0A\09 vpopcntd $2, $0 {$3} {z} \0A\09 vpopcntd $2{1to16}, $0 \0A\09 vpopcntd $2{1to16}, $0 {$3} \0A\09 vpopcntd $2{1to16}, $0 {$3} {z}", "v,v,*m,^Yk"(<16 x i32> %a0, <16 x i32> %a1, <16 x i32> *%a2, i16 %a3) nounwind
-  ret void
-}
-
-define void @test_vpopcntq(<8 x i64> %a0, <8 x i64> %a1, <8 x i64> *%a2, i8 %a3) {
-; GENERIC-LABEL: test_vpopcntq:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    kmovw %esi, %k1 # sched: [1:0.33]
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    vpopcntq %zmm1, %zmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    vpopcntq %zmm1, %zmm0 {%k1} # sched: [1:0.50]
-; GENERIC-NEXT:    vpopcntq %zmm1, %zmm0 {%k1} {z} # sched: [1:0.50]
-; GENERIC-NEXT:    vpopcntq (%rdi), %zmm0 # sched: [8:0.50]
-; GENERIC-NEXT:    vpopcntq (%rdi), %zmm0 {%k1} # sched: [8:0.50]
-; GENERIC-NEXT:    vpopcntq (%rdi), %zmm0 {%k1} {z} # sched: [8:0.50]
-; GENERIC-NEXT:    vpopcntq (%rdi){1to8}, %zmm0 # sched: [8:0.50]
-; GENERIC-NEXT:    vpopcntq (%rdi){1to8}, %zmm0 {%k1} # sched: [8:0.50]
-; GENERIC-NEXT:    vpopcntq (%rdi){1to8}, %zmm0 {%k1} {z} # sched: [8:0.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    vzeroupper # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ICELAKE-LABEL: test_vpopcntq:
-; ICELAKE:       # %bb.0:
-; ICELAKE-NEXT:    kmovd %esi, %k1 # sched: [1:1.00]
-; ICELAKE-NEXT:    #APP
-; ICELAKE-NEXT:    vpopcntq %zmm1, %zmm0 # sched: [1:1.00]
-; ICELAKE-NEXT:    vpopcntq %zmm1, %zmm0 {%k1} # sched: [1:1.00]
-; ICELAKE-NEXT:    vpopcntq %zmm1, %zmm0 {%k1} {z} # sched: [1:1.00]
-; ICELAKE-NEXT:    vpopcntq (%rdi), %zmm0 # sched: [8:1.00]
-; ICELAKE-NEXT:    vpopcntq (%rdi), %zmm0 {%k1} # sched: [8:1.00]
-; ICELAKE-NEXT:    vpopcntq (%rdi), %zmm0 {%k1} {z} # sched: [8:1.00]
-; ICELAKE-NEXT:    vpopcntq (%rdi){1to8}, %zmm0 # sched: [8:1.00]
-; ICELAKE-NEXT:    vpopcntq (%rdi){1to8}, %zmm0 {%k1} # sched: [8:1.00]
-; ICELAKE-NEXT:    vpopcntq (%rdi){1to8}, %zmm0 {%k1} {z} # sched: [8:1.00]
-; ICELAKE-NEXT:    #NO_APP
-; ICELAKE-NEXT:    vzeroupper # sched: [0:0.67]
-; ICELAKE-NEXT:    retq # sched: [7:1.00]
-  tail call void asm "vpopcntq $1, $0 \0A\09 vpopcntq $1, $0 {$3} \0A\09 vpopcntq $1, $0 {$3} {z} \0A\09 vpopcntq $2, $0 \0A\09 vpopcntq $2, $0 {$3} \0A\09 vpopcntq $2, $0 {$3} {z} \0A\09 vpopcntq $2{1to8}, $0 \0A\09 vpopcntq $2{1to8}, $0 {$3} \0A\09 vpopcntq $2{1to8}, $0 {$3} {z}", "v,v,*m,^Yk"(<8 x i64> %a0, <8 x i64> %a1, <8 x i64> *%a2, i8 %a3) nounwind
-  ret void
-}
diff --git a/test/CodeGen/X86/barrier.ll b/test/CodeGen/X86/barrier.ll
index 9031a0eb0129fb6ef1ca7b6e53f106f360e5dcce..f85c0ae98af1dafa2619158e330029347f093615 100644
--- a/test/CodeGen/X86/barrier.ll
+++ b/test/CodeGen/X86/barrier.ll
@@ -1,7 +1,11 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=i686-- -mattr=-sse2 | FileCheck %s
 
 define void @test() {
-; CHECK: lock
+; CHECK-LABEL: test:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    lock orl $0, (%esp)
+; CHECK-NEXT:    retl
 	fence seq_cst
 	ret void
 }
diff --git a/test/CodeGen/X86/bitcast-and-setcc-128.ll b/test/CodeGen/X86/bitcast-and-setcc-128.ll
index 0a2e154f5b93f8666b0e44fe9d7d3470b7ac444c..289ddcb194bfd0f5a99ab7da4b675af9f4f9a1c0 100644
--- a/test/CodeGen/X86/bitcast-and-setcc-128.ll
+++ b/test/CodeGen/X86/bitcast-and-setcc-128.ll
@@ -159,11 +159,8 @@ define i16 @v16i8(<16 x i8> %a, <16 x i8> %b, <16 x i8> %c, <16 x i8> %d) {
 ; AVX512F-NEXT:    vpcmpgtb %xmm1, %xmm0, %xmm0
 ; AVX512F-NEXT:    vpcmpgtb %xmm3, %xmm2, %xmm1
 ; AVX512F-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX512F-NEXT:    vpmovsxbd %xmm0, %zmm0
-; AVX512F-NEXT:    vptestmd %zmm0, %zmm0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    vpmovmskb %xmm0, %eax
 ; AVX512F-NEXT:    # kill: def $ax killed $ax killed $eax
-; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512BW-LABEL: v16i8:
diff --git a/test/CodeGen/X86/bitcast-and-setcc-256.ll b/test/CodeGen/X86/bitcast-and-setcc-256.ll
index 0f3b8c945404ec983a600e84dc2bcc022d368daf..85ae7c0c42101e5f0c265af77280df1528ff02f2 100644
--- a/test/CodeGen/X86/bitcast-and-setcc-256.ll
+++ b/test/CodeGen/X86/bitcast-and-setcc-256.ll
@@ -228,8 +228,8 @@ define i16 @v16i16(<16 x i16> %a, <16 x i16> %b, <16 x i16> %c, <16 x i16> %d) {
   ret i16 %res
 }
 
-define i8 @v8i32(<8 x i32> %a, <8 x i32> %b, <8 x i32> %c, <8 x i32> %d) {
-; SSE2-SSSE3-LABEL: v8i32:
+define i8 @v8i32_and(<8 x i32> %a, <8 x i32> %b, <8 x i32> %c, <8 x i32> %d) {
+; SSE2-SSSE3-LABEL: v8i32_and:
 ; SSE2-SSSE3:       # %bb.0:
 ; SSE2-SSSE3-NEXT:    pcmpgtd %xmm3, %xmm1
 ; SSE2-SSSE3-NEXT:    pcmpgtd %xmm2, %xmm0
@@ -243,7 +243,7 @@ define i8 @v8i32(<8 x i32> %a, <8 x i32> %b, <8 x i32> %c, <8 x i32> %d) {
 ; SSE2-SSSE3-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE2-SSSE3-NEXT:    retq
 ;
-; AVX1-LABEL: v8i32:
+; AVX1-LABEL: v8i32_and:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm0, %xmm4
 ; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm1
@@ -262,7 +262,7 @@ define i8 @v8i32(<8 x i32> %a, <8 x i32> %b, <8 x i32> %c, <8 x i32> %d) {
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
 ;
-; AVX2-LABEL: v8i32:
+; AVX2-LABEL: v8i32_and:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpcmpgtd %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vpcmpgtd %ymm3, %ymm2, %ymm1
@@ -275,7 +275,7 @@ define i8 @v8i32(<8 x i32> %a, <8 x i32> %b, <8 x i32> %c, <8 x i32> %d) {
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
-; AVX512F-LABEL: v8i32:
+; AVX512F-LABEL: v8i32_and:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vpcmpgtd %ymm1, %ymm0, %k1
 ; AVX512F-NEXT:    vpcmpgtd %ymm3, %ymm2, %k0 {%k1}
@@ -284,7 +284,7 @@ define i8 @v8i32(<8 x i32> %a, <8 x i32> %b, <8 x i32> %c, <8 x i32> %d) {
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
-; AVX512BW-LABEL: v8i32:
+; AVX512BW-LABEL: v8i32_and:
 ; AVX512BW:       # %bb.0:
 ; AVX512BW-NEXT:    vpcmpgtd %ymm1, %ymm0, %k1
 ; AVX512BW-NEXT:    vpcmpgtd %ymm3, %ymm2, %k0 {%k1}
@@ -299,8 +299,178 @@ define i8 @v8i32(<8 x i32> %a, <8 x i32> %b, <8 x i32> %c, <8 x i32> %d) {
   ret i8 %res
 }
 
-define i8 @v8f32(<8 x float> %a, <8 x float> %b, <8 x float> %c, <8 x float> %d) {
-; SSE2-SSSE3-LABEL: v8f32:
+; We should see through any bitwise logic op.
+
+define i8 @v8i32_or(<8 x i32> %a, <8 x i32> %b, <8 x i32> %c, <8 x i32> %d) {
+; SSE2-SSSE3-LABEL: v8i32_or:
+; SSE2-SSSE3:       # %bb.0:
+; SSE2-SSSE3-NEXT:    pcmpgtd %xmm3, %xmm1
+; SSE2-SSSE3-NEXT:    pcmpgtd %xmm2, %xmm0
+; SSE2-SSSE3-NEXT:    packssdw %xmm1, %xmm0
+; SSE2-SSSE3-NEXT:    pcmpgtd %xmm7, %xmm5
+; SSE2-SSSE3-NEXT:    pcmpgtd %xmm6, %xmm4
+; SSE2-SSSE3-NEXT:    packssdw %xmm5, %xmm4
+; SSE2-SSSE3-NEXT:    por %xmm0, %xmm4
+; SSE2-SSSE3-NEXT:    packsswb %xmm0, %xmm4
+; SSE2-SSSE3-NEXT:    pmovmskb %xmm4, %eax
+; SSE2-SSSE3-NEXT:    # kill: def $al killed $al killed $eax
+; SSE2-SSSE3-NEXT:    retq
+;
+; AVX1-LABEL: v8i32_or:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm0, %xmm4
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm1
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpcmpgtd %xmm3, %xmm2, %xmm1
+; AVX1-NEXT:    vpor %xmm1, %xmm4, %xmm1
+; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm3
+; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm2
+; AVX1-NEXT:    vpcmpgtd %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vpor %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpackssdw %xmm0, %xmm1, %xmm0
+; AVX1-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
+; AVX1-NEXT:    vpmovmskb %xmm0, %eax
+; AVX1-NEXT:    # kill: def $al killed $al killed $eax
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: v8i32_or:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpcmpgtd %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpcmpgtd %ymm3, %ymm2, %ymm1
+; AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT:    vpackssdw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
+; AVX2-NEXT:    vpmovmskb %xmm0, %eax
+; AVX2-NEXT:    # kill: def $al killed $al killed $eax
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
+;
+; AVX512F-LABEL: v8i32_or:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpcmpgtd %ymm1, %ymm0, %k0
+; AVX512F-NEXT:    vpcmpgtd %ymm3, %ymm2, %k1
+; AVX512F-NEXT:    korw %k1, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    # kill: def $al killed $al killed $eax
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512BW-LABEL: v8i32_or:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpcmpgtd %ymm1, %ymm0, %k0
+; AVX512BW-NEXT:    vpcmpgtd %ymm3, %ymm2, %k1
+; AVX512BW-NEXT:    korw %k1, %k0, %k0
+; AVX512BW-NEXT:    kmovd %k0, %eax
+; AVX512BW-NEXT:    # kill: def $al killed $al killed $eax
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+  %x0 = icmp sgt <8 x i32> %a, %b
+  %x1 = icmp sgt <8 x i32> %c, %d
+  %y = or <8 x i1> %x0, %x1
+  %res = bitcast <8 x i1> %y to i8
+  ret i8 %res
+}
+
+; We should see through multiple bitwise logic ops.
+
+define i8 @v8i32_or_and(<8 x i32> %a, <8 x i32> %b, <8 x i32> %c, <8 x i32> %d, <8 x i32> %e, <8 x i32> %f) {
+; SSE2-SSSE3-LABEL: v8i32_or_and:
+; SSE2-SSSE3:       # %bb.0:
+; SSE2-SSSE3-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm8
+; SSE2-SSSE3-NEXT:    movdqa {{[0-9]+}}(%rsp), %xmm9
+; SSE2-SSSE3-NEXT:    pcmpgtd %xmm3, %xmm1
+; SSE2-SSSE3-NEXT:    pcmpgtd %xmm2, %xmm0
+; SSE2-SSSE3-NEXT:    packssdw %xmm1, %xmm0
+; SSE2-SSSE3-NEXT:    pcmpgtd %xmm5, %xmm7
+; SSE2-SSSE3-NEXT:    pcmpgtd %xmm4, %xmm6
+; SSE2-SSSE3-NEXT:    packssdw %xmm7, %xmm6
+; SSE2-SSSE3-NEXT:    por %xmm0, %xmm6
+; SSE2-SSSE3-NEXT:    pcmpeqd {{[0-9]+}}(%rsp), %xmm9
+; SSE2-SSSE3-NEXT:    pcmpeqd {{[0-9]+}}(%rsp), %xmm8
+; SSE2-SSSE3-NEXT:    packssdw %xmm9, %xmm8
+; SSE2-SSSE3-NEXT:    pand %xmm6, %xmm8
+; SSE2-SSSE3-NEXT:    packsswb %xmm0, %xmm8
+; SSE2-SSSE3-NEXT:    pmovmskb %xmm8, %eax
+; SSE2-SSSE3-NEXT:    # kill: def $al killed $al killed $eax
+; SSE2-SSSE3-NEXT:    retq
+;
+; AVX1-LABEL: v8i32_or_and:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm0, %xmm6
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm1
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpcmpgtd %xmm2, %xmm3, %xmm1
+; AVX1-NEXT:    vpor %xmm1, %xmm6, %xmm1
+; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm2
+; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm3
+; AVX1-NEXT:    vpcmpgtd %xmm2, %xmm3, %xmm2
+; AVX1-NEXT:    vpor %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpackssdw %xmm0, %xmm1, %xmm0
+; AVX1-NEXT:    vextractf128 $1, %ymm5, %xmm1
+; AVX1-NEXT:    vextractf128 $1, %ymm4, %xmm2
+; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm2, %xmm1
+; AVX1-NEXT:    vpcmpeqd %xmm5, %xmm4, %xmm2
+; AVX1-NEXT:    vpackssdw %xmm1, %xmm2, %xmm1
+; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
+; AVX1-NEXT:    vpmovmskb %xmm0, %eax
+; AVX1-NEXT:    # kill: def $al killed $al killed $eax
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: v8i32_or_and:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpcmpgtd %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpcmpgtd %ymm2, %ymm3, %ymm1
+; AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpcmpeqd %ymm5, %ymm4, %ymm1
+; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm2
+; AVX2-NEXT:    vpackssdw %xmm2, %xmm1, %xmm1
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm2
+; AVX2-NEXT:    vpackssdw %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
+; AVX2-NEXT:    vpmovmskb %xmm0, %eax
+; AVX2-NEXT:    # kill: def $al killed $al killed $eax
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
+;
+; AVX512F-LABEL: v8i32_or_and:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpcmpgtd %ymm1, %ymm0, %k0
+; AVX512F-NEXT:    vpcmpgtd %ymm2, %ymm3, %k1
+; AVX512F-NEXT:    korw %k1, %k0, %k1
+; AVX512F-NEXT:    vpcmpeqd %ymm5, %ymm4, %k0 {%k1}
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    # kill: def $al killed $al killed $eax
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512BW-LABEL: v8i32_or_and:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpcmpgtd %ymm1, %ymm0, %k0
+; AVX512BW-NEXT:    vpcmpgtd %ymm2, %ymm3, %k1
+; AVX512BW-NEXT:    korw %k1, %k0, %k1
+; AVX512BW-NEXT:    vpcmpeqd %ymm5, %ymm4, %k0 {%k1}
+; AVX512BW-NEXT:    kmovd %k0, %eax
+; AVX512BW-NEXT:    # kill: def $al killed $al killed $eax
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+  %x0 = icmp sgt <8 x i32> %a, %b
+  %x1 = icmp slt <8 x i32> %c, %d
+  %x2 = icmp eq <8 x i32> %e, %f
+  %y = or <8 x i1> %x0, %x1
+  %z = and <8 x i1> %y, %x2
+  %res = bitcast <8 x i1> %z to i8
+  ret i8 %res
+}
+
+define i8 @v8f32_and(<8 x float> %a, <8 x float> %b, <8 x float> %c, <8 x float> %d) {
+; SSE2-SSSE3-LABEL: v8f32_and:
 ; SSE2-SSSE3:       # %bb.0:
 ; SSE2-SSSE3-NEXT:    cmpltps %xmm1, %xmm3
 ; SSE2-SSSE3-NEXT:    cmpltps %xmm0, %xmm2
@@ -314,7 +484,7 @@ define i8 @v8f32(<8 x float> %a, <8 x float> %b, <8 x float> %c, <8 x float> %d)
 ; SSE2-SSSE3-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE2-SSSE3-NEXT:    retq
 ;
-; AVX12-LABEL: v8f32:
+; AVX12-LABEL: v8f32_and:
 ; AVX12:       # %bb.0:
 ; AVX12-NEXT:    vcmpltps %ymm0, %ymm1, %ymm0
 ; AVX12-NEXT:    vcmpltps %ymm2, %ymm3, %ymm1
@@ -327,7 +497,7 @@ define i8 @v8f32(<8 x float> %a, <8 x float> %b, <8 x float> %c, <8 x float> %d)
 ; AVX12-NEXT:    vzeroupper
 ; AVX12-NEXT:    retq
 ;
-; AVX512F-LABEL: v8f32:
+; AVX512F-LABEL: v8f32_and:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vcmpltps %ymm0, %ymm1, %k1
 ; AVX512F-NEXT:    vcmpltps %ymm2, %ymm3, %k0 {%k1}
@@ -336,7 +506,7 @@ define i8 @v8f32(<8 x float> %a, <8 x float> %b, <8 x float> %c, <8 x float> %d)
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
-; AVX512BW-LABEL: v8f32:
+; AVX512BW-LABEL: v8f32_and:
 ; AVX512BW:       # %bb.0:
 ; AVX512BW-NEXT:    vcmpltps %ymm0, %ymm1, %k1
 ; AVX512BW-NEXT:    vcmpltps %ymm2, %ymm3, %k0 {%k1}
@@ -351,6 +521,138 @@ define i8 @v8f32(<8 x float> %a, <8 x float> %b, <8 x float> %c, <8 x float> %d)
   ret i8 %res
 }
 
+; We should see through any bitwise logic op.
+
+define i8 @v8f32_xor(<8 x float> %a, <8 x float> %b, <8 x float> %c, <8 x float> %d) {
+; SSE2-SSSE3-LABEL: v8f32_xor:
+; SSE2-SSSE3:       # %bb.0:
+; SSE2-SSSE3-NEXT:    cmpltps %xmm1, %xmm3
+; SSE2-SSSE3-NEXT:    cmpltps %xmm0, %xmm2
+; SSE2-SSSE3-NEXT:    packssdw %xmm3, %xmm2
+; SSE2-SSSE3-NEXT:    cmpltps %xmm5, %xmm7
+; SSE2-SSSE3-NEXT:    cmpltps %xmm4, %xmm6
+; SSE2-SSSE3-NEXT:    packssdw %xmm7, %xmm6
+; SSE2-SSSE3-NEXT:    pxor %xmm2, %xmm6
+; SSE2-SSSE3-NEXT:    packsswb %xmm0, %xmm6
+; SSE2-SSSE3-NEXT:    pmovmskb %xmm6, %eax
+; SSE2-SSSE3-NEXT:    # kill: def $al killed $al killed $eax
+; SSE2-SSSE3-NEXT:    retq
+;
+; AVX12-LABEL: v8f32_xor:
+; AVX12:       # %bb.0:
+; AVX12-NEXT:    vcmpltps %ymm0, %ymm1, %ymm0
+; AVX12-NEXT:    vcmpltps %ymm2, %ymm3, %ymm1
+; AVX12-NEXT:    vxorps %ymm1, %ymm0, %ymm0
+; AVX12-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX12-NEXT:    vpackssdw %xmm1, %xmm0, %xmm0
+; AVX12-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
+; AVX12-NEXT:    vpmovmskb %xmm0, %eax
+; AVX12-NEXT:    # kill: def $al killed $al killed $eax
+; AVX12-NEXT:    vzeroupper
+; AVX12-NEXT:    retq
+;
+; AVX512F-LABEL: v8f32_xor:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vcmpltps %ymm0, %ymm1, %k0
+; AVX512F-NEXT:    vcmpltps %ymm2, %ymm3, %k1
+; AVX512F-NEXT:    kxorw %k1, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    # kill: def $al killed $al killed $eax
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512BW-LABEL: v8f32_xor:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vcmpltps %ymm0, %ymm1, %k0
+; AVX512BW-NEXT:    vcmpltps %ymm2, %ymm3, %k1
+; AVX512BW-NEXT:    kxorw %k1, %k0, %k0
+; AVX512BW-NEXT:    kmovd %k0, %eax
+; AVX512BW-NEXT:    # kill: def $al killed $al killed $eax
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+  %x0 = fcmp ogt <8 x float> %a, %b
+  %x1 = fcmp ogt <8 x float> %c, %d
+  %y = xor <8 x i1> %x0, %x1
+  %res = bitcast <8 x i1> %y to i8
+  ret i8 %res
+}
+
+; We should see through multiple bitwise logic ops.
+
+define i8 @v8f32_xor_and(<8 x float> %a, <8 x float> %b, <8 x float> %c, <8 x float> %d, <8 x float> %e, <8 x float> %f) {
+; SSE2-SSSE3-LABEL: v8f32_xor_and:
+; SSE2-SSSE3:       # %bb.0:
+; SSE2-SSSE3-NEXT:    movaps {{[0-9]+}}(%rsp), %xmm8
+; SSE2-SSSE3-NEXT:    movaps {{[0-9]+}}(%rsp), %xmm9
+; SSE2-SSSE3-NEXT:    cmpnleps %xmm3, %xmm1
+; SSE2-SSSE3-NEXT:    cmpnleps %xmm2, %xmm0
+; SSE2-SSSE3-NEXT:    packssdw %xmm1, %xmm0
+; SSE2-SSSE3-NEXT:    movaps %xmm5, %xmm1
+; SSE2-SSSE3-NEXT:    cmpeqps %xmm7, %xmm1
+; SSE2-SSSE3-NEXT:    cmpunordps %xmm7, %xmm5
+; SSE2-SSSE3-NEXT:    orps %xmm1, %xmm5
+; SSE2-SSSE3-NEXT:    movaps %xmm4, %xmm1
+; SSE2-SSSE3-NEXT:    cmpeqps %xmm6, %xmm1
+; SSE2-SSSE3-NEXT:    cmpunordps %xmm6, %xmm4
+; SSE2-SSSE3-NEXT:    orps %xmm1, %xmm4
+; SSE2-SSSE3-NEXT:    packssdw %xmm5, %xmm4
+; SSE2-SSSE3-NEXT:    pxor %xmm0, %xmm4
+; SSE2-SSSE3-NEXT:    cmpltps {{[0-9]+}}(%rsp), %xmm9
+; SSE2-SSSE3-NEXT:    cmpltps {{[0-9]+}}(%rsp), %xmm8
+; SSE2-SSSE3-NEXT:    packssdw %xmm9, %xmm8
+; SSE2-SSSE3-NEXT:    pand %xmm4, %xmm8
+; SSE2-SSSE3-NEXT:    packsswb %xmm0, %xmm8
+; SSE2-SSSE3-NEXT:    pmovmskb %xmm8, %eax
+; SSE2-SSSE3-NEXT:    # kill: def $al killed $al killed $eax
+; SSE2-SSSE3-NEXT:    retq
+;
+; AVX12-LABEL: v8f32_xor_and:
+; AVX12:       # %bb.0:
+; AVX12-NEXT:    vcmpnleps %ymm1, %ymm0, %ymm0
+; AVX12-NEXT:    vcmpeq_uqps %ymm3, %ymm2, %ymm1
+; AVX12-NEXT:    vxorps %ymm1, %ymm0, %ymm0
+; AVX12-NEXT:    vcmpltps %ymm4, %ymm5, %ymm1
+; AVX12-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; AVX12-NEXT:    vpackssdw %xmm2, %xmm1, %xmm1
+; AVX12-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; AVX12-NEXT:    vpackssdw %xmm2, %xmm0, %xmm0
+; AVX12-NEXT:    vpand %xmm1, %xmm0, %xmm0
+; AVX12-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
+; AVX12-NEXT:    vpmovmskb %xmm0, %eax
+; AVX12-NEXT:    # kill: def $al killed $al killed $eax
+; AVX12-NEXT:    vzeroupper
+; AVX12-NEXT:    retq
+;
+; AVX512F-LABEL: v8f32_xor_and:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vcmpnleps %ymm1, %ymm0, %k0
+; AVX512F-NEXT:    vcmpeq_uqps %ymm3, %ymm2, %k1
+; AVX512F-NEXT:    kxorw %k1, %k0, %k1
+; AVX512F-NEXT:    vcmpltps %ymm4, %ymm5, %k0 {%k1}
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    # kill: def $al killed $al killed $eax
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512BW-LABEL: v8f32_xor_and:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vcmpnleps %ymm1, %ymm0, %k0
+; AVX512BW-NEXT:    vcmpeq_uqps %ymm3, %ymm2, %k1
+; AVX512BW-NEXT:    kxorw %k1, %k0, %k1
+; AVX512BW-NEXT:    vcmpltps %ymm4, %ymm5, %k0 {%k1}
+; AVX512BW-NEXT:    kmovd %k0, %eax
+; AVX512BW-NEXT:    # kill: def $al killed $al killed $eax
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+  %x0 = fcmp ugt <8 x float> %a, %b
+  %x1 = fcmp ueq <8 x float> %c, %d
+  %x2 = fcmp ogt <8 x float> %e, %f
+  %y = xor <8 x i1> %x0, %x1
+  %z = and <8 x i1> %y, %x2
+  %res = bitcast <8 x i1> %z to i8
+  ret i8 %res
+}
+
 define i32 @v32i8(<32 x i8> %a, <32 x i8> %b, <32 x i8> %c, <32 x i8> %d) {
 ; SSE2-SSSE3-LABEL: v32i8:
 ; SSE2-SSSE3:       # %bb.0:
@@ -399,15 +701,7 @@ define i32 @v32i8(<32 x i8> %a, <32 x i8> %b, <32 x i8> %c, <32 x i8> %d) {
 ; AVX512F-NEXT:    vpcmpgtb %ymm1, %ymm0, %ymm0
 ; AVX512F-NEXT:    vpcmpgtb %ymm3, %ymm2, %ymm1
 ; AVX512F-NEXT:    vpand %ymm1, %ymm0, %ymm0
-; AVX512F-NEXT:    vpmovsxbd %xmm0, %zmm1
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kmovw %k0, %ecx
-; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm0
-; AVX512F-NEXT:    vpmovsxbd %xmm0, %zmm0
-; AVX512F-NEXT:    vptestmd %zmm0, %zmm0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    shll $16, %eax
-; AVX512F-NEXT:    orl %ecx, %eax
+; AVX512F-NEXT:    vpmovmskb %ymm0, %eax
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
diff --git a/test/CodeGen/X86/bitcast-setcc-128.ll b/test/CodeGen/X86/bitcast-setcc-128.ll
index f803901c0e78a49cb5807664ddd5007fc3877666..e9dea85e14dfca54c8a98906ea08bfe3c54716f1 100644
--- a/test/CodeGen/X86/bitcast-setcc-128.ll
+++ b/test/CodeGen/X86/bitcast-setcc-128.ll
@@ -128,11 +128,8 @@ define i16 @v16i8(<16 x i8> %a, <16 x i8> %b) {
 ; AVX512F-LABEL: v16i8:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vpcmpgtb %xmm1, %xmm0, %xmm0
-; AVX512F-NEXT:    vpmovsxbd %xmm0, %zmm0
-; AVX512F-NEXT:    vptestmd %zmm0, %zmm0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    vpmovmskb %xmm0, %eax
 ; AVX512F-NEXT:    # kill: def $ax killed $ax killed $eax
-; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512BW-LABEL: v16i8:
@@ -686,9 +683,6 @@ define i64 @v16i8_widened_with_ones(<16 x i8> %a, <16 x i8> %b) {
 ; AVX1:       # %bb.0: # %entry
 ; AVX1-NEXT:    vpcmpeqb %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsllw $7, %xmm0, %xmm0
-; AVX1-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
-; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpgtb %xmm0, %xmm1, %xmm0
 ; AVX1-NEXT:    vpmovmskb %xmm0, %ecx
 ; AVX1-NEXT:    orl $-65536, %ecx # imm = 0xFFFF0000
 ; AVX1-NEXT:    movabsq $-4294967296, %rax # imm = 0xFFFFFFFF00000000
diff --git a/test/CodeGen/X86/bitcast-setcc-256.ll b/test/CodeGen/X86/bitcast-setcc-256.ll
index d349ae3bc40dcd598f5e20957b00c820e0d53853..7489c785f145836f03c7fe5fc3ff5da8cfe5492d 100644
--- a/test/CodeGen/X86/bitcast-setcc-256.ll
+++ b/test/CodeGen/X86/bitcast-setcc-256.ll
@@ -184,15 +184,7 @@ define i32 @v32i8(<32 x i8> %a, <32 x i8> %b) {
 ; AVX512F-LABEL: v32i8:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vpcmpgtb %ymm1, %ymm0, %ymm0
-; AVX512F-NEXT:    vpmovsxbd %xmm0, %zmm1
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kmovw %k0, %ecx
-; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm0
-; AVX512F-NEXT:    vpmovsxbd %xmm0, %zmm0
-; AVX512F-NEXT:    vptestmd %zmm0, %zmm0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    shll $16, %eax
-; AVX512F-NEXT:    orl %ecx, %eax
+; AVX512F-NEXT:    vpmovmskb %ymm0, %eax
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
@@ -322,11 +314,8 @@ define void @bitcast_32i8_store(i32* %p, <32 x i8> %a0) {
 ;
 ; AVX1-LABEL: bitcast_32i8_store:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpgtb %xmm0, %xmm1, %xmm2
-; AVX1-NEXT:    vpmovmskb %xmm2, %eax
+; AVX1-NEXT:    vpmovmskb %xmm0, %eax
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
-; AVX1-NEXT:    vpcmpgtb %xmm0, %xmm1, %xmm0
 ; AVX1-NEXT:    vpmovmskb %xmm0, %ecx
 ; AVX1-NEXT:    shll $16, %ecx
 ; AVX1-NEXT:    orl %eax, %ecx
diff --git a/test/CodeGen/X86/bitcast-setcc-512.ll b/test/CodeGen/X86/bitcast-setcc-512.ll
index 1911aed297384bb3adc1770df458b500ff9ea832..b6a38aa132b86623b016e9cbb5548f960d446593 100644
--- a/test/CodeGen/X86/bitcast-setcc-512.ll
+++ b/test/CodeGen/X86/bitcast-setcc-512.ll
@@ -256,26 +256,10 @@ define i64 @v64i8(<64 x i8> %a, <64 x i8> %b) {
 ;
 ; AVX512F-LABEL: v64i8:
 ; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpcmpgtb %ymm3, %ymm1, %ymm1
 ; AVX512F-NEXT:    vpcmpgtb %ymm2, %ymm0, %ymm0
-; AVX512F-NEXT:    vpmovsxbd %xmm0, %zmm2
-; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm0
-; AVX512F-NEXT:    vpmovsxbd %xmm0, %zmm0
-; AVX512F-NEXT:    vptestmd %zmm0, %zmm0, %k0
-; AVX512F-NEXT:    kmovw %k0, %ecx
-; AVX512F-NEXT:    shll $16, %ecx
-; AVX512F-NEXT:    orl %eax, %ecx
-; AVX512F-NEXT:    vpcmpgtb %ymm3, %ymm1, %ymm0
-; AVX512F-NEXT:    vpmovsxbd %xmm0, %zmm1
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT:    kmovw %k0, %edx
-; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm0
-; AVX512F-NEXT:    vpmovsxbd %xmm0, %zmm0
-; AVX512F-NEXT:    vptestmd %zmm0, %zmm0, %k0
-; AVX512F-NEXT:    kmovw %k0, %eax
-; AVX512F-NEXT:    shll $16, %eax
-; AVX512F-NEXT:    orl %edx, %eax
+; AVX512F-NEXT:    vpmovmskb %ymm0, %ecx
+; AVX512F-NEXT:    vpmovmskb %ymm1, %eax
 ; AVX512F-NEXT:    shlq $32, %rax
 ; AVX512F-NEXT:    orq %rcx, %rax
 ; AVX512F-NEXT:    vzeroupper
@@ -434,18 +418,13 @@ define void @bitcast_64i8_store(i64* %p, <64 x i8> %a0) {
 ;
 ; AVX1-LABEL: bitcast_64i8_store:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm3
-; AVX1-NEXT:    vpmovmskb %xmm3, %eax
+; AVX1-NEXT:    vpmovmskb %xmm0, %eax
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
-; AVX1-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm0
 ; AVX1-NEXT:    vpmovmskb %xmm0, %ecx
 ; AVX1-NEXT:    shll $16, %ecx
 ; AVX1-NEXT:    orl %eax, %ecx
-; AVX1-NEXT:    vpcmpgtb %xmm1, %xmm2, %xmm0
-; AVX1-NEXT:    vpmovmskb %xmm0, %eax
+; AVX1-NEXT:    vpmovmskb %xmm1, %eax
 ; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm0
-; AVX1-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm0
 ; AVX1-NEXT:    vpmovmskb %xmm0, %edx
 ; AVX1-NEXT:    shll $16, %edx
 ; AVX1-NEXT:    orl %eax, %edx
diff --git a/test/CodeGen/X86/bitreverse.ll b/test/CodeGen/X86/bitreverse.ll
index aeac9e88dd0b5135e17912f3222b7af55bb72b41..ed3fdefce7ac67318b95774914a8e44ae094ef3b 100644
--- a/test/CodeGen/X86/bitreverse.ll
+++ b/test/CodeGen/X86/bitreverse.ll
@@ -61,18 +61,17 @@ define <2 x i16> @test_bitreverse_v2i16(<2 x i16> %a) nounwind {
 ; X64-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
 ; X64-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
 ; X64-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[3,2,1,0,4,5,6,7]
-; X64-NEXT:    pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,7,6,5,4]
-; X64-NEXT:    packuswb %xmm2, %xmm0
-; X64-NEXT:    movdqa {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; X64-NEXT:    movdqa %xmm0, %xmm2
-; X64-NEXT:    pand %xmm1, %xmm2
+; X64-NEXT:    pshufhw {{.*#+}} xmm1 = xmm0[0,1,2,3,7,6,5,4]
+; X64-NEXT:    packuswb %xmm2, %xmm1
+; X64-NEXT:    movdqa {{.*#+}} xmm0 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; X64-NEXT:    movdqa %xmm1, %xmm2
+; X64-NEXT:    pand %xmm0, %xmm2
 ; X64-NEXT:    psllw $4, %xmm2
-; X64-NEXT:    movdqa {{.*#+}} xmm3 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
-; X64-NEXT:    pand %xmm3, %xmm2
-; X64-NEXT:    pand %xmm3, %xmm0
-; X64-NEXT:    psrlw $4, %xmm0
-; X64-NEXT:    pand %xmm1, %xmm0
-; X64-NEXT:    por %xmm2, %xmm0
+; X64-NEXT:    pand {{.*}}(%rip), %xmm1
+; X64-NEXT:    psrlw $4, %xmm1
+; X64-NEXT:    pand %xmm0, %xmm1
+; X64-NEXT:    pandn %xmm2, %xmm0
+; X64-NEXT:    por %xmm1, %xmm0
 ; X64-NEXT:    movdqa {{.*#+}} xmm1 = [51,51,51,51,51,51,51,51,51,51,51,51,51,51,51,51]
 ; X64-NEXT:    pand %xmm0, %xmm1
 ; X64-NEXT:    psllw $2, %xmm1
diff --git a/test/CodeGen/X86/bmi-schedule.ll b/test/CodeGen/X86/bmi-schedule.ll
deleted file mode 100644
index 5b5b388c100c3f8505ef229a978a480a8f1abbe0..0000000000000000000000000000000000000000
--- a/test/CodeGen/X86/bmi-schedule.ll
+++ /dev/null
@@ -1,763 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=x86-64 -mattr=+bmi | FileCheck %s --check-prefix=CHECK --check-prefix=GENERIC
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=haswell | FileCheck %s --check-prefix=CHECK --check-prefix=HASWELL
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=broadwell | FileCheck %s --check-prefix=CHECK --check-prefix=BROADWELL
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=skylake | FileCheck %s --check-prefix=CHECK --check-prefix=SKYLAKE
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=knl     | FileCheck %s --check-prefix=CHECK --check-prefix=HASWELL
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=bdver2  | FileCheck %s --check-prefix=CHECK --check-prefix=BDVER2
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=btver2  | FileCheck %s --check-prefix=CHECK --check-prefix=BTVER2
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=znver1  | FileCheck %s --check-prefix=CHECK --check-prefix=ZNVER1
-
-define i32 @test_andn_i32(i32 %a0, i32 %a1, i32 *%a2) {
-; GENERIC-LABEL: test_andn_i32:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    andnl %esi, %edi, %ecx # sched: [1:0.33]
-; GENERIC-NEXT:    andnl (%rdx), %edi, %eax # sched: [6:0.50]
-; GENERIC-NEXT:    addl %ecx, %eax # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_andn_i32:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    andnl %esi, %edi, %ecx # sched: [1:0.50]
-; HASWELL-NEXT:    andnl (%rdx), %edi, %eax # sched: [6:0.50]
-; HASWELL-NEXT:    addl %ecx, %eax # sched: [1:0.25]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_andn_i32:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    andnl %esi, %edi, %ecx # sched: [1:0.50]
-; BROADWELL-NEXT:    andnl (%rdx), %edi, %eax # sched: [6:0.50]
-; BROADWELL-NEXT:    addl %ecx, %eax # sched: [1:0.25]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_andn_i32:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    andnl %esi, %edi, %ecx # sched: [1:0.50]
-; SKYLAKE-NEXT:    andnl (%rdx), %edi, %eax # sched: [6:0.50]
-; SKYLAKE-NEXT:    addl %ecx, %eax # sched: [1:0.25]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_andn_i32:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    andnl (%rdx), %edi, %eax # sched: [5:0.50]
-; BDVER2-NEXT:    andnl %esi, %edi, %ecx # sched: [1:0.50]
-; BDVER2-NEXT:    addl %ecx, %eax # sched: [1:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_andn_i32:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    andnl (%rdx), %edi, %eax # sched: [4:1.00]
-; BTVER2-NEXT:    andnl %esi, %edi, %ecx # sched: [1:0.50]
-; BTVER2-NEXT:    addl %ecx, %eax # sched: [1:0.50]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_andn_i32:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    andnl (%rdx), %edi, %eax # sched: [5:0.50]
-; ZNVER1-NEXT:    andnl %esi, %edi, %ecx # sched: [1:0.25]
-; ZNVER1-NEXT:    addl %ecx, %eax # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = load i32, i32 *%a2
-  %2 = xor i32 %a0, -1
-  %3 = and i32 %2, %a1
-  %4 = and i32 %2, %1
-  %5 = add i32 %3, %4
-  ret i32 %5
-}
-
-define i64 @test_andn_i64(i64 %a0, i64 %a1, i64 *%a2) {
-; GENERIC-LABEL: test_andn_i64:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    andnq %rsi, %rdi, %rcx # sched: [1:0.33]
-; GENERIC-NEXT:    andnq (%rdx), %rdi, %rax # sched: [6:0.50]
-; GENERIC-NEXT:    addq %rcx, %rax # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_andn_i64:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    andnq %rsi, %rdi, %rcx # sched: [1:0.50]
-; HASWELL-NEXT:    andnq (%rdx), %rdi, %rax # sched: [6:0.50]
-; HASWELL-NEXT:    addq %rcx, %rax # sched: [1:0.25]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_andn_i64:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    andnq %rsi, %rdi, %rcx # sched: [1:0.50]
-; BROADWELL-NEXT:    andnq (%rdx), %rdi, %rax # sched: [6:0.50]
-; BROADWELL-NEXT:    addq %rcx, %rax # sched: [1:0.25]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_andn_i64:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    andnq %rsi, %rdi, %rcx # sched: [1:0.50]
-; SKYLAKE-NEXT:    andnq (%rdx), %rdi, %rax # sched: [6:0.50]
-; SKYLAKE-NEXT:    addq %rcx, %rax # sched: [1:0.25]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_andn_i64:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    andnq (%rdx), %rdi, %rax # sched: [5:0.50]
-; BDVER2-NEXT:    andnq %rsi, %rdi, %rcx # sched: [1:0.50]
-; BDVER2-NEXT:    addq %rcx, %rax # sched: [1:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_andn_i64:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    andnq (%rdx), %rdi, %rax # sched: [4:1.00]
-; BTVER2-NEXT:    andnq %rsi, %rdi, %rcx # sched: [1:0.50]
-; BTVER2-NEXT:    addq %rcx, %rax # sched: [1:0.50]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_andn_i64:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    andnq (%rdx), %rdi, %rax # sched: [5:0.50]
-; ZNVER1-NEXT:    andnq %rsi, %rdi, %rcx # sched: [1:0.25]
-; ZNVER1-NEXT:    addq %rcx, %rax # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = load i64, i64 *%a2
-  %2 = xor i64 %a0, -1
-  %3 = and i64 %2, %a1
-  %4 = and i64 %2, %1
-  %5 = add i64 %3, %4
-  ret i64 %5
-}
-
-define i32 @test_bextr_i32(i32 %a0, i32 %a1, i32 *%a2) {
-; GENERIC-LABEL: test_bextr_i32:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    bextrl %edi, (%rdx), %ecx # sched: [7:1.00]
-; GENERIC-NEXT:    bextrl %edi, %esi, %eax # sched: [2:1.00]
-; GENERIC-NEXT:    addl %ecx, %eax # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_bextr_i32:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    bextrl %edi, (%rdx), %ecx # sched: [7:0.50]
-; HASWELL-NEXT:    bextrl %edi, %esi, %eax # sched: [2:0.50]
-; HASWELL-NEXT:    addl %ecx, %eax # sched: [1:0.25]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_bextr_i32:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    bextrl %edi, (%rdx), %ecx # sched: [7:0.50]
-; BROADWELL-NEXT:    bextrl %edi, %esi, %eax # sched: [2:0.50]
-; BROADWELL-NEXT:    addl %ecx, %eax # sched: [1:0.25]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_bextr_i32:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    bextrl %edi, (%rdx), %ecx # sched: [7:0.50]
-; SKYLAKE-NEXT:    bextrl %edi, %esi, %eax # sched: [2:0.50]
-; SKYLAKE-NEXT:    addl %ecx, %eax # sched: [1:0.25]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_bextr_i32:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    bextrl %edi, (%rdx), %ecx # sched: [6:0.50]
-; BDVER2-NEXT:    bextrl %edi, %esi, %eax # sched: [2:0.50]
-; BDVER2-NEXT:    addl %ecx, %eax # sched: [1:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_bextr_i32:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    bextrl %edi, (%rdx), %ecx # sched: [4:1.00]
-; BTVER2-NEXT:    bextrl %edi, %esi, %eax # sched: [1:0.50]
-; BTVER2-NEXT:    addl %ecx, %eax # sched: [1:0.50]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_bextr_i32:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    bextrl %edi, (%rdx), %ecx # sched: [5:0.50]
-; ZNVER1-NEXT:    bextrl %edi, %esi, %eax # sched: [1:0.25]
-; ZNVER1-NEXT:    addl %ecx, %eax # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = load i32, i32 *%a2
-  %2 = tail call i32 @llvm.x86.bmi.bextr.32(i32 %1, i32 %a0)
-  %3 = tail call i32 @llvm.x86.bmi.bextr.32(i32 %a1, i32 %a0)
-  %4 = add i32 %2, %3
-  ret i32 %4
-}
-declare i32 @llvm.x86.bmi.bextr.32(i32, i32)
-
-define i64 @test_bextr_i64(i64 %a0, i64 %a1, i64 *%a2) {
-; GENERIC-LABEL: test_bextr_i64:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    bextrq %rdi, (%rdx), %rcx # sched: [7:1.00]
-; GENERIC-NEXT:    bextrq %rdi, %rsi, %rax # sched: [2:1.00]
-; GENERIC-NEXT:    addq %rcx, %rax # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_bextr_i64:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    bextrq %rdi, (%rdx), %rcx # sched: [7:0.50]
-; HASWELL-NEXT:    bextrq %rdi, %rsi, %rax # sched: [2:0.50]
-; HASWELL-NEXT:    addq %rcx, %rax # sched: [1:0.25]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_bextr_i64:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    bextrq %rdi, (%rdx), %rcx # sched: [7:0.50]
-; BROADWELL-NEXT:    bextrq %rdi, %rsi, %rax # sched: [2:0.50]
-; BROADWELL-NEXT:    addq %rcx, %rax # sched: [1:0.25]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_bextr_i64:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    bextrq %rdi, (%rdx), %rcx # sched: [7:0.50]
-; SKYLAKE-NEXT:    bextrq %rdi, %rsi, %rax # sched: [2:0.50]
-; SKYLAKE-NEXT:    addq %rcx, %rax # sched: [1:0.25]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_bextr_i64:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    bextrq %rdi, (%rdx), %rcx # sched: [6:0.50]
-; BDVER2-NEXT:    bextrq %rdi, %rsi, %rax # sched: [2:0.50]
-; BDVER2-NEXT:    addq %rcx, %rax # sched: [1:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_bextr_i64:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    bextrq %rdi, (%rdx), %rcx # sched: [4:1.00]
-; BTVER2-NEXT:    bextrq %rdi, %rsi, %rax # sched: [1:0.50]
-; BTVER2-NEXT:    addq %rcx, %rax # sched: [1:0.50]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_bextr_i64:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    bextrq %rdi, (%rdx), %rcx # sched: [5:0.50]
-; ZNVER1-NEXT:    bextrq %rdi, %rsi, %rax # sched: [1:0.25]
-; ZNVER1-NEXT:    addq %rcx, %rax # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = load i64, i64 *%a2
-  %2 = tail call i64 @llvm.x86.bmi.bextr.64(i64 %1, i64 %a0)
-  %3 = tail call i64 @llvm.x86.bmi.bextr.64(i64 %a1, i64 %a0)
-  %4 = add i64 %2, %3
-  ret i64 %4
-}
-declare i64 @llvm.x86.bmi.bextr.64(i64, i64)
-
-define i32 @test_blsi_i32(i32 %a0, i32 *%a1) {
-; GENERIC-LABEL: test_blsi_i32:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    blsil (%rsi), %ecx # sched: [6:0.50]
-; GENERIC-NEXT:    blsil %edi, %eax # sched: [1:0.33]
-; GENERIC-NEXT:    addl %ecx, %eax # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_blsi_i32:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    blsil (%rsi), %ecx # sched: [6:0.50]
-; HASWELL-NEXT:    blsil %edi, %eax # sched: [1:0.50]
-; HASWELL-NEXT:    addl %ecx, %eax # sched: [1:0.25]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_blsi_i32:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    blsil (%rsi), %ecx # sched: [6:0.50]
-; BROADWELL-NEXT:    blsil %edi, %eax # sched: [1:0.50]
-; BROADWELL-NEXT:    addl %ecx, %eax # sched: [1:0.25]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_blsi_i32:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    blsil (%rsi), %ecx # sched: [6:0.50]
-; SKYLAKE-NEXT:    blsil %edi, %eax # sched: [1:0.50]
-; SKYLAKE-NEXT:    addl %ecx, %eax # sched: [1:0.25]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_blsi_i32:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    blsil (%rsi), %ecx # sched: [6:0.50]
-; BDVER2-NEXT:    blsil %edi, %eax # sched: [2:0.50]
-; BDVER2-NEXT:    addl %ecx, %eax # sched: [1:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_blsi_i32:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    blsil (%rsi), %ecx # sched: [5:1.00]
-; BTVER2-NEXT:    blsil %edi, %eax # sched: [2:1.00]
-; BTVER2-NEXT:    addl %ecx, %eax # sched: [1:0.50]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_blsi_i32:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    blsil (%rsi), %ecx # sched: [6:0.50]
-; ZNVER1-NEXT:    blsil %edi, %eax # sched: [2:0.25]
-; ZNVER1-NEXT:    addl %ecx, %eax # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = load i32, i32 *%a1
-  %2 = sub i32 0, %1
-  %3 = sub i32 0, %a0
-  %4 = and i32 %1, %2
-  %5 = and i32 %a0, %3
-  %6 = add i32 %4, %5
-  ret i32 %6
-}
-
-define i64 @test_blsi_i64(i64 %a0, i64 *%a1) {
-; GENERIC-LABEL: test_blsi_i64:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    blsiq (%rsi), %rcx # sched: [6:0.50]
-; GENERIC-NEXT:    blsiq %rdi, %rax # sched: [1:0.33]
-; GENERIC-NEXT:    addq %rcx, %rax # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_blsi_i64:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    blsiq (%rsi), %rcx # sched: [6:0.50]
-; HASWELL-NEXT:    blsiq %rdi, %rax # sched: [1:0.50]
-; HASWELL-NEXT:    addq %rcx, %rax # sched: [1:0.25]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_blsi_i64:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    blsiq (%rsi), %rcx # sched: [6:0.50]
-; BROADWELL-NEXT:    blsiq %rdi, %rax # sched: [1:0.50]
-; BROADWELL-NEXT:    addq %rcx, %rax # sched: [1:0.25]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_blsi_i64:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    blsiq (%rsi), %rcx # sched: [6:0.50]
-; SKYLAKE-NEXT:    blsiq %rdi, %rax # sched: [1:0.50]
-; SKYLAKE-NEXT:    addq %rcx, %rax # sched: [1:0.25]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_blsi_i64:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    blsiq (%rsi), %rcx # sched: [6:0.50]
-; BDVER2-NEXT:    blsiq %rdi, %rax # sched: [2:0.50]
-; BDVER2-NEXT:    addq %rcx, %rax # sched: [1:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_blsi_i64:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    blsiq (%rsi), %rcx # sched: [5:1.00]
-; BTVER2-NEXT:    blsiq %rdi, %rax # sched: [2:1.00]
-; BTVER2-NEXT:    addq %rcx, %rax # sched: [1:0.50]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_blsi_i64:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    blsiq (%rsi), %rcx # sched: [6:0.50]
-; ZNVER1-NEXT:    blsiq %rdi, %rax # sched: [2:0.25]
-; ZNVER1-NEXT:    addq %rcx, %rax # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = load i64, i64 *%a1
-  %2 = sub i64 0, %1
-  %3 = sub i64 0, %a0
-  %4 = and i64 %1, %2
-  %5 = and i64 %a0, %3
-  %6 = add i64 %4, %5
-  ret i64 %6
-}
-
-define i32 @test_blsmsk_i32(i32 %a0, i32 *%a1) {
-; GENERIC-LABEL: test_blsmsk_i32:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    blsmskl (%rsi), %ecx # sched: [6:0.50]
-; GENERIC-NEXT:    blsmskl %edi, %eax # sched: [1:0.33]
-; GENERIC-NEXT:    addl %ecx, %eax # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_blsmsk_i32:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    blsmskl (%rsi), %ecx # sched: [6:0.50]
-; HASWELL-NEXT:    blsmskl %edi, %eax # sched: [1:0.50]
-; HASWELL-NEXT:    addl %ecx, %eax # sched: [1:0.25]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_blsmsk_i32:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    blsmskl (%rsi), %ecx # sched: [6:0.50]
-; BROADWELL-NEXT:    blsmskl %edi, %eax # sched: [1:0.50]
-; BROADWELL-NEXT:    addl %ecx, %eax # sched: [1:0.25]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_blsmsk_i32:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    blsmskl (%rsi), %ecx # sched: [6:0.50]
-; SKYLAKE-NEXT:    blsmskl %edi, %eax # sched: [1:0.50]
-; SKYLAKE-NEXT:    addl %ecx, %eax # sched: [1:0.25]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_blsmsk_i32:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    blsmskl (%rsi), %ecx # sched: [6:0.50]
-; BDVER2-NEXT:    blsmskl %edi, %eax # sched: [2:0.50]
-; BDVER2-NEXT:    addl %ecx, %eax # sched: [1:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_blsmsk_i32:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    blsmskl (%rsi), %ecx # sched: [5:1.00]
-; BTVER2-NEXT:    blsmskl %edi, %eax # sched: [2:1.00]
-; BTVER2-NEXT:    addl %ecx, %eax # sched: [1:0.50]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_blsmsk_i32:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    blsmskl (%rsi), %ecx # sched: [6:0.50]
-; ZNVER1-NEXT:    blsmskl %edi, %eax # sched: [2:0.25]
-; ZNVER1-NEXT:    addl %ecx, %eax # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = load i32, i32 *%a1
-  %2 = sub i32 %1, 1
-  %3 = sub i32 %a0, 1
-  %4 = xor i32 %1, %2
-  %5 = xor i32 %a0, %3
-  %6 = add i32 %4, %5
-  ret i32 %6
-}
-
-define i64 @test_blsmsk_i64(i64 %a0, i64 *%a1) {
-; GENERIC-LABEL: test_blsmsk_i64:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    blsmskq (%rsi), %rcx # sched: [6:0.50]
-; GENERIC-NEXT:    blsmskq %rdi, %rax # sched: [1:0.33]
-; GENERIC-NEXT:    addq %rcx, %rax # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_blsmsk_i64:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    blsmskq (%rsi), %rcx # sched: [6:0.50]
-; HASWELL-NEXT:    blsmskq %rdi, %rax # sched: [1:0.50]
-; HASWELL-NEXT:    addq %rcx, %rax # sched: [1:0.25]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_blsmsk_i64:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    blsmskq (%rsi), %rcx # sched: [6:0.50]
-; BROADWELL-NEXT:    blsmskq %rdi, %rax # sched: [1:0.50]
-; BROADWELL-NEXT:    addq %rcx, %rax # sched: [1:0.25]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_blsmsk_i64:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    blsmskq (%rsi), %rcx # sched: [6:0.50]
-; SKYLAKE-NEXT:    blsmskq %rdi, %rax # sched: [1:0.50]
-; SKYLAKE-NEXT:    addq %rcx, %rax # sched: [1:0.25]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_blsmsk_i64:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    blsmskq (%rsi), %rcx # sched: [6:0.50]
-; BDVER2-NEXT:    blsmskq %rdi, %rax # sched: [2:0.50]
-; BDVER2-NEXT:    addq %rcx, %rax # sched: [1:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_blsmsk_i64:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    blsmskq (%rsi), %rcx # sched: [5:1.00]
-; BTVER2-NEXT:    blsmskq %rdi, %rax # sched: [2:1.00]
-; BTVER2-NEXT:    addq %rcx, %rax # sched: [1:0.50]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_blsmsk_i64:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    blsmskq (%rsi), %rcx # sched: [6:0.50]
-; ZNVER1-NEXT:    blsmskq %rdi, %rax # sched: [2:0.25]
-; ZNVER1-NEXT:    addq %rcx, %rax # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = load i64, i64 *%a1
-  %2 = sub i64 %1, 1
-  %3 = sub i64 %a0, 1
-  %4 = xor i64 %1, %2
-  %5 = xor i64 %a0, %3
-  %6 = add i64 %4, %5
-  ret i64 %6
-}
-
-define i32 @test_blsr_i32(i32 %a0, i32 *%a1) {
-; GENERIC-LABEL: test_blsr_i32:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    blsrl (%rsi), %ecx # sched: [6:0.50]
-; GENERIC-NEXT:    blsrl %edi, %eax # sched: [1:0.33]
-; GENERIC-NEXT:    addl %ecx, %eax # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_blsr_i32:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    blsrl (%rsi), %ecx # sched: [6:0.50]
-; HASWELL-NEXT:    blsrl %edi, %eax # sched: [1:0.50]
-; HASWELL-NEXT:    addl %ecx, %eax # sched: [1:0.25]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_blsr_i32:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    blsrl (%rsi), %ecx # sched: [6:0.50]
-; BROADWELL-NEXT:    blsrl %edi, %eax # sched: [1:0.50]
-; BROADWELL-NEXT:    addl %ecx, %eax # sched: [1:0.25]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_blsr_i32:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    blsrl (%rsi), %ecx # sched: [6:0.50]
-; SKYLAKE-NEXT:    blsrl %edi, %eax # sched: [1:0.50]
-; SKYLAKE-NEXT:    addl %ecx, %eax # sched: [1:0.25]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_blsr_i32:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    blsrl (%rsi), %ecx # sched: [6:0.50]
-; BDVER2-NEXT:    blsrl %edi, %eax # sched: [2:0.50]
-; BDVER2-NEXT:    addl %ecx, %eax # sched: [1:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_blsr_i32:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    blsrl (%rsi), %ecx # sched: [5:1.00]
-; BTVER2-NEXT:    blsrl %edi, %eax # sched: [2:1.00]
-; BTVER2-NEXT:    addl %ecx, %eax # sched: [1:0.50]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_blsr_i32:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    blsrl (%rsi), %ecx # sched: [6:0.50]
-; ZNVER1-NEXT:    blsrl %edi, %eax # sched: [2:0.25]
-; ZNVER1-NEXT:    addl %ecx, %eax # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = load i32, i32 *%a1
-  %2 = sub i32 %1, 1
-  %3 = sub i32 %a0, 1
-  %4 = and i32 %1, %2
-  %5 = and i32 %a0, %3
-  %6 = add i32 %4, %5
-  ret i32 %6
-}
-
-define i64 @test_blsr_i64(i64 %a0, i64 *%a1) {
-; GENERIC-LABEL: test_blsr_i64:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    blsrq (%rsi), %rcx # sched: [6:0.50]
-; GENERIC-NEXT:    blsrq %rdi, %rax # sched: [1:0.33]
-; GENERIC-NEXT:    addq %rcx, %rax # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_blsr_i64:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    blsrq (%rsi), %rcx # sched: [6:0.50]
-; HASWELL-NEXT:    blsrq %rdi, %rax # sched: [1:0.50]
-; HASWELL-NEXT:    addq %rcx, %rax # sched: [1:0.25]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_blsr_i64:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    blsrq (%rsi), %rcx # sched: [6:0.50]
-; BROADWELL-NEXT:    blsrq %rdi, %rax # sched: [1:0.50]
-; BROADWELL-NEXT:    addq %rcx, %rax # sched: [1:0.25]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_blsr_i64:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    blsrq (%rsi), %rcx # sched: [6:0.50]
-; SKYLAKE-NEXT:    blsrq %rdi, %rax # sched: [1:0.50]
-; SKYLAKE-NEXT:    addq %rcx, %rax # sched: [1:0.25]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_blsr_i64:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    blsrq (%rsi), %rcx # sched: [6:0.50]
-; BDVER2-NEXT:    blsrq %rdi, %rax # sched: [2:0.50]
-; BDVER2-NEXT:    addq %rcx, %rax # sched: [1:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_blsr_i64:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    blsrq (%rsi), %rcx # sched: [5:1.00]
-; BTVER2-NEXT:    blsrq %rdi, %rax # sched: [2:1.00]
-; BTVER2-NEXT:    addq %rcx, %rax # sched: [1:0.50]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_blsr_i64:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    blsrq (%rsi), %rcx # sched: [6:0.50]
-; ZNVER1-NEXT:    blsrq %rdi, %rax # sched: [2:0.25]
-; ZNVER1-NEXT:    addq %rcx, %rax # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = load i64, i64 *%a1
-  %2 = sub i64 %1, 1
-  %3 = sub i64 %a0, 1
-  %4 = and i64 %1, %2
-  %5 = and i64 %a0, %3
-  %6 = add i64 %4, %5
-  ret i64 %6
-}
-
-define i16 @test_cttz_i16(i16 zeroext %a0, i16 *%a1) {
-; GENERIC-LABEL: test_cttz_i16:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    tzcntw (%rsi), %cx # sched: [8:1.00]
-; GENERIC-NEXT:    tzcntw %di, %ax # sched: [3:1.00]
-; GENERIC-NEXT:    orl %ecx, %eax # sched: [1:0.33]
-; GENERIC-NEXT:    # kill: def $ax killed $ax killed $eax
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_cttz_i16:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    tzcntw (%rsi), %cx # sched: [8:1.00]
-; HASWELL-NEXT:    tzcntw %di, %ax # sched: [3:1.00]
-; HASWELL-NEXT:    orl %ecx, %eax # sched: [1:0.25]
-; HASWELL-NEXT:    # kill: def $ax killed $ax killed $eax
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_cttz_i16:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    tzcntw (%rsi), %cx # sched: [8:1.00]
-; BROADWELL-NEXT:    tzcntw %di, %ax # sched: [3:1.00]
-; BROADWELL-NEXT:    orl %ecx, %eax # sched: [1:0.25]
-; BROADWELL-NEXT:    # kill: def $ax killed $ax killed $eax
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_cttz_i16:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    tzcntw (%rsi), %cx # sched: [8:1.00]
-; SKYLAKE-NEXT:    tzcntw %di, %ax # sched: [3:1.00]
-; SKYLAKE-NEXT:    orl %ecx, %eax # sched: [1:0.25]
-; SKYLAKE-NEXT:    # kill: def $ax killed $ax killed $eax
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_cttz_i16:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    tzcntw (%rsi), %cx # sched: [6:1.00]
-; BDVER2-NEXT:    tzcntw %di, %ax # sched: [2:1.00]
-; BDVER2-NEXT:    orl %ecx, %eax # sched: [1:0.50]
-; BDVER2-NEXT:    # kill: def $ax killed $ax killed $eax
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_cttz_i16:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    tzcntw (%rsi), %cx # sched: [5:1.00]
-; BTVER2-NEXT:    tzcntw %di, %ax # sched: [2:1.00]
-; BTVER2-NEXT:    orl %ecx, %eax # sched: [1:0.50]
-; BTVER2-NEXT:    # kill: def $ax killed $ax killed $eax
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_cttz_i16:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    tzcntw (%rsi), %cx # sched: [6:0.50]
-; ZNVER1-NEXT:    tzcntw %di, %ax # sched: [2:0.25]
-; ZNVER1-NEXT:    orl %ecx, %eax # sched: [1:0.25]
-; ZNVER1-NEXT:    # kill: def $ax killed $ax killed $eax
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = load i16, i16 *%a1
-  %2 = tail call i16 @llvm.cttz.i16( i16 %1, i1 false )
-  %3 = tail call i16 @llvm.cttz.i16( i16 %a0, i1 false )
-  %4 = or i16 %2, %3
-  ret i16 %4
-}
-declare i16 @llvm.cttz.i16(i16, i1)
-
-define i32 @test_cttz_i32(i32 %a0, i32 *%a1) {
-; GENERIC-LABEL: test_cttz_i32:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    tzcntl (%rsi), %ecx # sched: [8:1.00]
-; GENERIC-NEXT:    tzcntl %edi, %eax # sched: [3:1.00]
-; GENERIC-NEXT:    orl %ecx, %eax # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_cttz_i32:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    tzcntl (%rsi), %ecx # sched: [8:1.00]
-; HASWELL-NEXT:    tzcntl %edi, %eax # sched: [3:1.00]
-; HASWELL-NEXT:    orl %ecx, %eax # sched: [1:0.25]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_cttz_i32:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    tzcntl (%rsi), %ecx # sched: [8:1.00]
-; BROADWELL-NEXT:    tzcntl %edi, %eax # sched: [3:1.00]
-; BROADWELL-NEXT:    orl %ecx, %eax # sched: [1:0.25]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_cttz_i32:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    tzcntl (%rsi), %ecx # sched: [8:1.00]
-; SKYLAKE-NEXT:    tzcntl %edi, %eax # sched: [3:1.00]
-; SKYLAKE-NEXT:    orl %ecx, %eax # sched: [1:0.25]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_cttz_i32:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    tzcntl (%rsi), %ecx # sched: [6:1.00]
-; BDVER2-NEXT:    tzcntl %edi, %eax # sched: [2:1.00]
-; BDVER2-NEXT:    orl %ecx, %eax # sched: [1:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_cttz_i32:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    tzcntl (%rsi), %ecx # sched: [5:1.00]
-; BTVER2-NEXT:    tzcntl %edi, %eax # sched: [2:1.00]
-; BTVER2-NEXT:    orl %ecx, %eax # sched: [1:0.50]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_cttz_i32:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    tzcntl (%rsi), %ecx # sched: [6:0.50]
-; ZNVER1-NEXT:    tzcntl %edi, %eax # sched: [2:0.25]
-; ZNVER1-NEXT:    orl %ecx, %eax # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = load i32, i32 *%a1
-  %2 = tail call i32 @llvm.cttz.i32( i32 %1, i1 false )
-  %3 = tail call i32 @llvm.cttz.i32( i32 %a0, i1 false )
-  %4 = or i32 %2, %3
-  ret i32 %4
-}
-declare i32 @llvm.cttz.i32(i32, i1)
-
-define i64 @test_cttz_i64(i64 %a0, i64 *%a1) {
-; GENERIC-LABEL: test_cttz_i64:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    tzcntq (%rsi), %rcx # sched: [8:1.00]
-; GENERIC-NEXT:    tzcntq %rdi, %rax # sched: [3:1.00]
-; GENERIC-NEXT:    orq %rcx, %rax # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_cttz_i64:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    tzcntq (%rsi), %rcx # sched: [8:1.00]
-; HASWELL-NEXT:    tzcntq %rdi, %rax # sched: [3:1.00]
-; HASWELL-NEXT:    orq %rcx, %rax # sched: [1:0.25]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_cttz_i64:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    tzcntq (%rsi), %rcx # sched: [8:1.00]
-; BROADWELL-NEXT:    tzcntq %rdi, %rax # sched: [3:1.00]
-; BROADWELL-NEXT:    orq %rcx, %rax # sched: [1:0.25]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_cttz_i64:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    tzcntq (%rsi), %rcx # sched: [8:1.00]
-; SKYLAKE-NEXT:    tzcntq %rdi, %rax # sched: [3:1.00]
-; SKYLAKE-NEXT:    orq %rcx, %rax # sched: [1:0.25]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_cttz_i64:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    tzcntq (%rsi), %rcx # sched: [6:1.00]
-; BDVER2-NEXT:    tzcntq %rdi, %rax # sched: [2:1.00]
-; BDVER2-NEXT:    orq %rcx, %rax # sched: [1:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_cttz_i64:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    tzcntq (%rsi), %rcx # sched: [5:1.00]
-; BTVER2-NEXT:    tzcntq %rdi, %rax # sched: [2:1.00]
-; BTVER2-NEXT:    orq %rcx, %rax # sched: [1:0.50]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_cttz_i64:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    tzcntq (%rsi), %rcx # sched: [6:0.50]
-; ZNVER1-NEXT:    tzcntq %rdi, %rax # sched: [2:0.25]
-; ZNVER1-NEXT:    orq %rcx, %rax # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = load i64, i64 *%a1
-  %2 = tail call i64 @llvm.cttz.i64( i64 %1, i1 false )
-  %3 = tail call i64 @llvm.cttz.i64( i64 %a0, i1 false )
-  %4 = or i64 %2, %3
-  ret i64 %4
-}
-declare i64 @llvm.cttz.i64(i64, i1)
diff --git a/test/CodeGen/X86/bmi.ll b/test/CodeGen/X86/bmi.ll
index 5642f7c6e435b4ab1811c442380fc3b406af7d57..ab159c9506600880a5a232eed8dfc0fdde87ecab 100644
--- a/test/CodeGen/X86/bmi.ll
+++ b/test/CodeGen/X86/bmi.ll
@@ -157,15 +157,15 @@ define i1 @and_cmp_const(i32 %x) {
 ; X86-LABEL: and_cmp_const:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    notl %eax
 ; X86-NEXT:    andl $43, %eax
+; X86-NEXT:    cmpl $43, %eax
 ; X86-NEXT:    sete %al
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: and_cmp_const:
 ; X64:       # %bb.0:
-; X64-NEXT:    notl %edi
 ; X64-NEXT:    andl $43, %edi
+; X64-NEXT:    cmpl $43, %edi
 ; X64-NEXT:    sete %al
 ; X64-NEXT:    retq
   %and = and i32 %x, 43
@@ -519,10 +519,7 @@ define i32 @blsi32_z(i32 %a, i32 %b) nounwind {
 define i32 @blsi32_z2(i32 %a, i32 %b, i32 %c) nounwind {
 ; X86-LABEL: blsi32_z2:
 ; X86:       # %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    movl %eax, %ecx
-; X86-NEXT:    negl %ecx
-; X86-NEXT:    testl %eax, %ecx
+; X86-NEXT:    blsil {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    leal {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    cmovel %eax, %ecx
@@ -532,9 +529,7 @@ define i32 @blsi32_z2(i32 %a, i32 %b, i32 %c) nounwind {
 ; X64-LABEL: blsi32_z2:
 ; X64:       # %bb.0:
 ; X64-NEXT:    movl %esi, %eax
-; X64-NEXT:    movl %edi, %ecx
-; X64-NEXT:    negl %ecx
-; X64-NEXT:    testl %edi, %ecx
+; X64-NEXT:    blsil %edi, %ecx
 ; X64-NEXT:    cmovnel %edx, %eax
 ; X64-NEXT:    retq
   %t0 = sub i32 0, %a
@@ -629,9 +624,7 @@ define i64 @blsi64_z2(i64 %a, i64 %b, i64 %c) nounwind {
 ; X64-LABEL: blsi64_z2:
 ; X64:       # %bb.0:
 ; X64-NEXT:    movq %rsi, %rax
-; X64-NEXT:    movq %rdi, %rcx
-; X64-NEXT:    negq %rcx
-; X64-NEXT:    testq %rdi, %rcx
+; X64-NEXT:    blsiq %rdi, %rcx
 ; X64-NEXT:    cmovneq %rdx, %rax
 ; X64-NEXT:    retq
   %t0 = sub i64 0, %a
@@ -698,9 +691,7 @@ define i32 @blsmsk32_z(i32 %a, i32 %b) nounwind {
 define i32 @blsmsk32_z2(i32 %a, i32 %b, i32 %c) nounwind {
 ; X86-LABEL: blsmsk32_z2:
 ; X86:       # %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    leal -1(%eax), %ecx
-; X86-NEXT:    xorl %eax, %ecx
+; X86-NEXT:    blsmskl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    leal {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    cmovel %eax, %ecx
@@ -710,9 +701,7 @@ define i32 @blsmsk32_z2(i32 %a, i32 %b, i32 %c) nounwind {
 ; X64-LABEL: blsmsk32_z2:
 ; X64:       # %bb.0:
 ; X64-NEXT:    movl %esi, %eax
-; X64-NEXT:    # kill: def $edi killed $edi def $rdi
-; X64-NEXT:    leal -1(%rdi), %ecx
-; X64-NEXT:    xorl %edi, %ecx
+; X64-NEXT:    blsmskl %edi, %ecx
 ; X64-NEXT:    cmovnel %edx, %eax
 ; X64-NEXT:    retq
   %t0 = sub i32 %a, 1
@@ -807,8 +796,7 @@ define i64 @blsmsk64_z2(i64 %a, i64 %b, i64 %c) nounwind {
 ; X64-LABEL: blsmsk64_z2:
 ; X64:       # %bb.0:
 ; X64-NEXT:    movq %rsi, %rax
-; X64-NEXT:    leaq -1(%rdi), %rcx
-; X64-NEXT:    xorq %rdi, %rcx
+; X64-NEXT:    blsmskq %rdi, %rcx
 ; X64-NEXT:    cmovneq %rdx, %rax
 ; X64-NEXT:    retq
   %t0 = sub i64 %a, 1
@@ -875,9 +863,7 @@ define i32 @blsr32_z(i32 %a, i32 %b) nounwind {
 define i32 @blsr32_z2(i32 %a, i32 %b, i32 %c) nounwind {
 ; X86-LABEL: blsr32_z2:
 ; X86:       # %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    leal -1(%eax), %ecx
-; X86-NEXT:    testl %eax, %ecx
+; X86-NEXT:    blsrl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    leal {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    leal {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    cmovel %eax, %ecx
@@ -887,9 +873,7 @@ define i32 @blsr32_z2(i32 %a, i32 %b, i32 %c) nounwind {
 ; X64-LABEL: blsr32_z2:
 ; X64:       # %bb.0:
 ; X64-NEXT:    movl %esi, %eax
-; X64-NEXT:    # kill: def $edi killed $edi def $rdi
-; X64-NEXT:    leal -1(%rdi), %ecx
-; X64-NEXT:    testl %edi, %ecx
+; X64-NEXT:    blsrl %edi, %ecx
 ; X64-NEXT:    cmovnel %edx, %eax
 ; X64-NEXT:    retq
   %t0 = sub i32 %a, 1
@@ -984,8 +968,7 @@ define i64 @blsr64_z2(i64 %a, i64 %b, i64 %c) nounwind {
 ; X64-LABEL: blsr64_z2:
 ; X64:       # %bb.0:
 ; X64-NEXT:    movq %rsi, %rax
-; X64-NEXT:    leaq -1(%rdi), %rcx
-; X64-NEXT:    testq %rdi, %rcx
+; X64-NEXT:    blsrq %rdi, %rcx
 ; X64-NEXT:    cmovneq %rdx, %rax
 ; X64-NEXT:    retq
   %t0 = sub i64 %a, 1
@@ -1044,6 +1027,7 @@ define void @pr40060(i32, i32) {
 ; X86:       # %bb.0:
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    bextrl %eax, {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    testl %eax, %eax
 ; X86-NEXT:    js .LBB45_1
 ; X86-NEXT:  # %bb.2:
 ; X86-NEXT:    jmp bar # TAILCALL
@@ -1053,6 +1037,7 @@ define void @pr40060(i32, i32) {
 ; X64-LABEL: pr40060:
 ; X64:       # %bb.0:
 ; X64-NEXT:    bextrl %esi, %edi, %eax
+; X64-NEXT:    testl %eax, %eax
 ; X64-NEXT:    js .LBB45_1
 ; X64-NEXT:  # %bb.2:
 ; X64-NEXT:    jmp bar # TAILCALL
@@ -1068,4 +1053,99 @@ define void @pr40060(i32, i32) {
   ret void
 }
 
+define i32 @blsr32_branch(i32 %x) {
+; X86-LABEL: blsr32_branch:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    .cfi_def_cfa_offset 8
+; X86-NEXT:    .cfi_offset %esi, -8
+; X86-NEXT:    blsrl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    jne .LBB46_2
+; X86-NEXT:  # %bb.1:
+; X86-NEXT:    calll bar
+; X86-NEXT:  .LBB46_2:
+; X86-NEXT:    movl %esi, %eax
+; X86-NEXT:    popl %esi
+; X86-NEXT:    .cfi_def_cfa_offset 4
+; X86-NEXT:    retl
+;
+; X64-LABEL: blsr32_branch:
+; X64:       # %bb.0:
+; X64-NEXT:    pushq %rbx
+; X64-NEXT:    .cfi_def_cfa_offset 16
+; X64-NEXT:    .cfi_offset %rbx, -16
+; X64-NEXT:    blsrl %edi, %ebx
+; X64-NEXT:    jne .LBB46_2
+; X64-NEXT:  # %bb.1:
+; X64-NEXT:    callq bar
+; X64-NEXT:  .LBB46_2:
+; X64-NEXT:    movl %ebx, %eax
+; X64-NEXT:    popq %rbx
+; X64-NEXT:    .cfi_def_cfa_offset 8
+; X64-NEXT:    retq
+  %tmp = sub i32 %x, 1
+  %tmp2 = and i32 %x, %tmp
+  %cmp = icmp eq i32 %tmp2, 0
+  br i1 %cmp, label %1, label %2
+
+  tail call void @bar()
+  br label %2
+  ret i32 %tmp2
+}
+
+define i64 @blsr64_branch(i64 %x) {
+; X86-LABEL: blsr64_branch:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    .cfi_def_cfa_offset 8
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    .cfi_def_cfa_offset 12
+; X86-NEXT:    .cfi_offset %esi, -12
+; X86-NEXT:    .cfi_offset %edi, -8
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl %eax, %esi
+; X86-NEXT:    addl $-1, %esi
+; X86-NEXT:    movl %ecx, %edi
+; X86-NEXT:    adcl $-1, %edi
+; X86-NEXT:    andl %eax, %esi
+; X86-NEXT:    andl %ecx, %edi
+; X86-NEXT:    movl %esi, %eax
+; X86-NEXT:    orl %edi, %eax
+; X86-NEXT:    jne .LBB47_2
+; X86-NEXT:  # %bb.1:
+; X86-NEXT:    calll bar
+; X86-NEXT:  .LBB47_2:
+; X86-NEXT:    movl %esi, %eax
+; X86-NEXT:    movl %edi, %edx
+; X86-NEXT:    popl %esi
+; X86-NEXT:    .cfi_def_cfa_offset 8
+; X86-NEXT:    popl %edi
+; X86-NEXT:    .cfi_def_cfa_offset 4
+; X86-NEXT:    retl
+;
+; X64-LABEL: blsr64_branch:
+; X64:       # %bb.0:
+; X64-NEXT:    pushq %rbx
+; X64-NEXT:    .cfi_def_cfa_offset 16
+; X64-NEXT:    .cfi_offset %rbx, -16
+; X64-NEXT:    blsrq %rdi, %rbx
+; X64-NEXT:    jne .LBB47_2
+; X64-NEXT:  # %bb.1:
+; X64-NEXT:    callq bar
+; X64-NEXT:  .LBB47_2:
+; X64-NEXT:    movq %rbx, %rax
+; X64-NEXT:    popq %rbx
+; X64-NEXT:    .cfi_def_cfa_offset 8
+; X64-NEXT:    retq
+  %tmp = sub i64 %x, 1
+  %tmp2 = and i64 %x, %tmp
+  %cmp = icmp eq i64 %tmp2, 0
+  br i1 %cmp, label %1, label %2
+
+  tail call void @bar()
+  br label %2
+  ret i64 %tmp2
+}
+
 declare void @bar()
diff --git a/test/CodeGen/X86/bmi2-schedule.ll b/test/CodeGen/X86/bmi2-schedule.ll
deleted file mode 100644
index f235e793ab9fefef67388d0d11e8df95948dd458..0000000000000000000000000000000000000000
--- a/test/CodeGen/X86/bmi2-schedule.ll
+++ /dev/null
@@ -1,811 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=x86-64 -mattr=+bmi2 | FileCheck %s --check-prefix=CHECK --check-prefix=GENERIC
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=haswell | FileCheck %s --check-prefix=CHECK --check-prefix=HASWELL
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=broadwell | FileCheck %s --check-prefix=CHECK --check-prefix=BROADWELL
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=skylake | FileCheck %s --check-prefix=CHECK --check-prefix=SKYLAKE
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=knl     | FileCheck %s --check-prefix=CHECK --check-prefix=KNL
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=znver1  | FileCheck %s --check-prefix=CHECK --check-prefix=ZNVER1
-
-define i32 @test_bzhi_i32(i32 %a0, i32 %a1, i32 *%a2) {
-; GENERIC-LABEL: test_bzhi_i32:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    bzhil %edi, (%rdx), %ecx # sched: [6:1.00]
-; GENERIC-NEXT:    bzhil %edi, %esi, %eax # sched: [1:1.00]
-; GENERIC-NEXT:    addl %ecx, %eax # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_bzhi_i32:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    bzhil %edi, (%rdx), %ecx # sched: [6:0.50]
-; HASWELL-NEXT:    bzhil %edi, %esi, %eax # sched: [1:0.50]
-; HASWELL-NEXT:    addl %ecx, %eax # sched: [1:0.25]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_bzhi_i32:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    bzhil %edi, (%rdx), %ecx # sched: [6:0.50]
-; BROADWELL-NEXT:    bzhil %edi, %esi, %eax # sched: [1:0.50]
-; BROADWELL-NEXT:    addl %ecx, %eax # sched: [1:0.25]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_bzhi_i32:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    bzhil %edi, (%rdx), %ecx # sched: [6:0.50]
-; SKYLAKE-NEXT:    bzhil %edi, %esi, %eax # sched: [1:0.50]
-; SKYLAKE-NEXT:    addl %ecx, %eax # sched: [1:0.25]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; KNL-LABEL: test_bzhi_i32:
-; KNL:       # %bb.0:
-; KNL-NEXT:    bzhil %edi, (%rdx), %ecx # sched: [6:0.50]
-; KNL-NEXT:    bzhil %edi, %esi, %eax # sched: [1:0.50]
-; KNL-NEXT:    addl %ecx, %eax # sched: [1:0.25]
-; KNL-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_bzhi_i32:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    bzhil %edi, (%rdx), %ecx # sched: [5:0.50]
-; ZNVER1-NEXT:    bzhil %edi, %esi, %eax # sched: [1:0.25]
-; ZNVER1-NEXT:    addl %ecx, %eax # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = load i32, i32 *%a2
-  %2 = tail call i32 @llvm.x86.bmi.bzhi.32(i32 %1, i32 %a0)
-  %3 = tail call i32 @llvm.x86.bmi.bzhi.32(i32 %a1, i32 %a0)
-  %4 = add i32 %2, %3
-  ret i32 %4
-}
-declare i32 @llvm.x86.bmi.bzhi.32(i32, i32)
-
-define i64 @test_bzhi_i64(i64 %a0, i64 %a1, i64 *%a2) {
-; GENERIC-LABEL: test_bzhi_i64:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    bzhiq %rdi, (%rdx), %rcx # sched: [6:1.00]
-; GENERIC-NEXT:    bzhiq %rdi, %rsi, %rax # sched: [1:1.00]
-; GENERIC-NEXT:    addq %rcx, %rax # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_bzhi_i64:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    bzhiq %rdi, (%rdx), %rcx # sched: [6:0.50]
-; HASWELL-NEXT:    bzhiq %rdi, %rsi, %rax # sched: [1:0.50]
-; HASWELL-NEXT:    addq %rcx, %rax # sched: [1:0.25]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_bzhi_i64:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    bzhiq %rdi, (%rdx), %rcx # sched: [6:0.50]
-; BROADWELL-NEXT:    bzhiq %rdi, %rsi, %rax # sched: [1:0.50]
-; BROADWELL-NEXT:    addq %rcx, %rax # sched: [1:0.25]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_bzhi_i64:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    bzhiq %rdi, (%rdx), %rcx # sched: [6:0.50]
-; SKYLAKE-NEXT:    bzhiq %rdi, %rsi, %rax # sched: [1:0.50]
-; SKYLAKE-NEXT:    addq %rcx, %rax # sched: [1:0.25]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; KNL-LABEL: test_bzhi_i64:
-; KNL:       # %bb.0:
-; KNL-NEXT:    bzhiq %rdi, (%rdx), %rcx # sched: [6:0.50]
-; KNL-NEXT:    bzhiq %rdi, %rsi, %rax # sched: [1:0.50]
-; KNL-NEXT:    addq %rcx, %rax # sched: [1:0.25]
-; KNL-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_bzhi_i64:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    bzhiq %rdi, (%rdx), %rcx # sched: [5:0.50]
-; ZNVER1-NEXT:    bzhiq %rdi, %rsi, %rax # sched: [1:0.25]
-; ZNVER1-NEXT:    addq %rcx, %rax # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = load i64, i64 *%a2
-  %2 = tail call i64 @llvm.x86.bmi.bzhi.64(i64 %1, i64 %a0)
-  %3 = tail call i64 @llvm.x86.bmi.bzhi.64(i64 %a1, i64 %a0)
-  %4 = add i64 %2, %3
-  ret i64 %4
-}
-declare i64 @llvm.x86.bmi.bzhi.64(i64, i64)
-
-define void @test_mulx_i32(i32 %a0, i32 %a1, i32* %a2) optsize {
-; GENERIC-LABEL: test_mulx_i32:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    mulxl %esi, %esi, %edi # sched: [4:1.00]
-; GENERIC-NEXT:    mulxl (%rdx), %esi, %edi # sched: [9:1.00]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_mulx_i32:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    mulxl %esi, %esi, %edi # sched: [4:1.00]
-; HASWELL-NEXT:    mulxl (%rdx), %esi, %edi # sched: [9:1.00]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_mulx_i32:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    mulxl %esi, %esi, %edi # sched: [4:1.00]
-; BROADWELL-NEXT:    mulxl (%rdx), %esi, %edi # sched: [9:1.00]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_mulx_i32:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    mulxl %esi, %esi, %edi # sched: [4:1.00]
-; SKYLAKE-NEXT:    mulxl (%rdx), %esi, %edi # sched: [9:1.00]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; KNL-LABEL: test_mulx_i32:
-; KNL:       # %bb.0:
-; KNL-NEXT:    #APP
-; KNL-NEXT:    mulxl %esi, %esi, %edi # sched: [4:1.00]
-; KNL-NEXT:    mulxl (%rdx), %esi, %edi # sched: [9:1.00]
-; KNL-NEXT:    #NO_APP
-; KNL-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_mulx_i32:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    mulxl %esi, %esi, %edi # sched: [3:2.00]
-; ZNVER1-NEXT:    mulxl (%rdx), %esi, %edi # sched: [8:2.00]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  tail call void asm "mulx $1, $1, $0 \0A\09 mulx $2, $1, $0 ", "r,r,*m"(i32 %a0, i32 %a1, i32* %a2) nounwind
-  ret void
-}
-
-define void @test_mulx_i64(i64 %a0, i64 %a1, i64 *%a2) {
-; GENERIC-LABEL: test_mulx_i64:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    mulxq %rsi, %rsi, %rdi # sched: [4:1.00]
-; GENERIC-NEXT:    mulxq (%rdx), %rsi, %rdi # sched: [9:1.00]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_mulx_i64:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    mulxq %rsi, %rsi, %rdi # sched: [4:1.00]
-; HASWELL-NEXT:    mulxq (%rdx), %rsi, %rdi # sched: [9:1.00]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_mulx_i64:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    mulxq %rsi, %rsi, %rdi # sched: [4:1.00]
-; BROADWELL-NEXT:    mulxq (%rdx), %rsi, %rdi # sched: [9:1.00]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_mulx_i64:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    mulxq %rsi, %rsi, %rdi # sched: [4:1.00]
-; SKYLAKE-NEXT:    mulxq (%rdx), %rsi, %rdi # sched: [9:1.00]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; KNL-LABEL: test_mulx_i64:
-; KNL:       # %bb.0:
-; KNL-NEXT:    #APP
-; KNL-NEXT:    mulxq %rsi, %rsi, %rdi # sched: [4:1.00]
-; KNL-NEXT:    mulxq (%rdx), %rsi, %rdi # sched: [9:1.00]
-; KNL-NEXT:    #NO_APP
-; KNL-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_mulx_i64:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    mulxq %rsi, %rsi, %rdi # sched: [3:1.00]
-; ZNVER1-NEXT:    mulxq (%rdx), %rsi, %rdi # sched: [8:1.00]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  tail call void asm "mulx $1, $1, $0 \0A\09 mulx $2, $1, $0 ", "r,r,*m"(i64 %a0, i64 %a1, i64* %a2) nounwind
-  ret void
-}
-
-define i32 @test_pdep_i32(i32 %a0, i32 %a1, i32 *%a2) {
-; GENERIC-LABEL: test_pdep_i32:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    pdepl (%rdx), %edi, %ecx # sched: [6:0.50]
-; GENERIC-NEXT:    pdepl %esi, %edi, %eax # sched: [1:0.33]
-; GENERIC-NEXT:    addl %ecx, %eax # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_pdep_i32:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    pdepl (%rdx), %edi, %ecx # sched: [8:1.00]
-; HASWELL-NEXT:    pdepl %esi, %edi, %eax # sched: [3:1.00]
-; HASWELL-NEXT:    addl %ecx, %eax # sched: [1:0.25]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pdep_i32:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    pdepl (%rdx), %edi, %ecx # sched: [8:1.00]
-; BROADWELL-NEXT:    pdepl %esi, %edi, %eax # sched: [3:1.00]
-; BROADWELL-NEXT:    addl %ecx, %eax # sched: [1:0.25]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pdep_i32:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    pdepl (%rdx), %edi, %ecx # sched: [8:1.00]
-; SKYLAKE-NEXT:    pdepl %esi, %edi, %eax # sched: [3:1.00]
-; SKYLAKE-NEXT:    addl %ecx, %eax # sched: [1:0.25]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; KNL-LABEL: test_pdep_i32:
-; KNL:       # %bb.0:
-; KNL-NEXT:    pdepl (%rdx), %edi, %ecx # sched: [8:1.00]
-; KNL-NEXT:    pdepl %esi, %edi, %eax # sched: [3:1.00]
-; KNL-NEXT:    addl %ecx, %eax # sched: [1:0.25]
-; KNL-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_pdep_i32:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    pdepl (%rdx), %edi, %ecx # sched: [100:0.25]
-; ZNVER1-NEXT:    pdepl %esi, %edi, %eax # sched: [100:0.25]
-; ZNVER1-NEXT:    addl %ecx, %eax # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = load i32, i32 *%a2
-  %2 = tail call i32 @llvm.x86.bmi.pdep.32(i32 %a0, i32 %1)
-  %3 = tail call i32 @llvm.x86.bmi.pdep.32(i32 %a0, i32 %a1)
-  %4 = add i32 %2, %3
-  ret i32 %4
-}
-declare i32 @llvm.x86.bmi.pdep.32(i32, i32)
-
-define i64 @test_pdep_i64(i64 %a0, i64 %a1, i64 *%a2) {
-; GENERIC-LABEL: test_pdep_i64:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    pdepq (%rdx), %rdi, %rcx # sched: [6:0.50]
-; GENERIC-NEXT:    pdepq %rsi, %rdi, %rax # sched: [1:0.33]
-; GENERIC-NEXT:    addq %rcx, %rax # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_pdep_i64:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    pdepq (%rdx), %rdi, %rcx # sched: [8:1.00]
-; HASWELL-NEXT:    pdepq %rsi, %rdi, %rax # sched: [3:1.00]
-; HASWELL-NEXT:    addq %rcx, %rax # sched: [1:0.25]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pdep_i64:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    pdepq (%rdx), %rdi, %rcx # sched: [8:1.00]
-; BROADWELL-NEXT:    pdepq %rsi, %rdi, %rax # sched: [3:1.00]
-; BROADWELL-NEXT:    addq %rcx, %rax # sched: [1:0.25]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pdep_i64:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    pdepq (%rdx), %rdi, %rcx # sched: [8:1.00]
-; SKYLAKE-NEXT:    pdepq %rsi, %rdi, %rax # sched: [3:1.00]
-; SKYLAKE-NEXT:    addq %rcx, %rax # sched: [1:0.25]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; KNL-LABEL: test_pdep_i64:
-; KNL:       # %bb.0:
-; KNL-NEXT:    pdepq (%rdx), %rdi, %rcx # sched: [8:1.00]
-; KNL-NEXT:    pdepq %rsi, %rdi, %rax # sched: [3:1.00]
-; KNL-NEXT:    addq %rcx, %rax # sched: [1:0.25]
-; KNL-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_pdep_i64:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    pdepq (%rdx), %rdi, %rcx # sched: [100:0.25]
-; ZNVER1-NEXT:    pdepq %rsi, %rdi, %rax # sched: [100:0.25]
-; ZNVER1-NEXT:    addq %rcx, %rax # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = load i64, i64 *%a2
-  %2 = tail call i64 @llvm.x86.bmi.pdep.64(i64 %a0, i64 %1)
-  %3 = tail call i64 @llvm.x86.bmi.pdep.64(i64 %a0, i64 %a1)
-  %4 = add i64 %2, %3
-  ret i64 %4
-}
-declare i64 @llvm.x86.bmi.pdep.64(i64, i64)
-
-define i32 @test_pext_i32(i32 %a0, i32 %a1, i32 *%a2) {
-; GENERIC-LABEL: test_pext_i32:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    pextl (%rdx), %edi, %ecx # sched: [6:0.50]
-; GENERIC-NEXT:    pextl %esi, %edi, %eax # sched: [1:0.33]
-; GENERIC-NEXT:    addl %ecx, %eax # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_pext_i32:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    pextl (%rdx), %edi, %ecx # sched: [8:1.00]
-; HASWELL-NEXT:    pextl %esi, %edi, %eax # sched: [3:1.00]
-; HASWELL-NEXT:    addl %ecx, %eax # sched: [1:0.25]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pext_i32:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    pextl (%rdx), %edi, %ecx # sched: [8:1.00]
-; BROADWELL-NEXT:    pextl %esi, %edi, %eax # sched: [3:1.00]
-; BROADWELL-NEXT:    addl %ecx, %eax # sched: [1:0.25]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pext_i32:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    pextl (%rdx), %edi, %ecx # sched: [8:1.00]
-; SKYLAKE-NEXT:    pextl %esi, %edi, %eax # sched: [3:1.00]
-; SKYLAKE-NEXT:    addl %ecx, %eax # sched: [1:0.25]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; KNL-LABEL: test_pext_i32:
-; KNL:       # %bb.0:
-; KNL-NEXT:    pextl (%rdx), %edi, %ecx # sched: [8:1.00]
-; KNL-NEXT:    pextl %esi, %edi, %eax # sched: [3:1.00]
-; KNL-NEXT:    addl %ecx, %eax # sched: [1:0.25]
-; KNL-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_pext_i32:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    pextl (%rdx), %edi, %ecx # sched: [100:0.25]
-; ZNVER1-NEXT:    pextl %esi, %edi, %eax # sched: [100:0.25]
-; ZNVER1-NEXT:    addl %ecx, %eax # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = load i32, i32 *%a2
-  %2 = tail call i32 @llvm.x86.bmi.pext.32(i32 %a0, i32 %1)
-  %3 = tail call i32 @llvm.x86.bmi.pext.32(i32 %a0, i32 %a1)
-  %4 = add i32 %2, %3
-  ret i32 %4
-}
-declare i32 @llvm.x86.bmi.pext.32(i32, i32)
-
-define i64 @test_pext_i64(i64 %a0, i64 %a1, i64 *%a2) {
-; GENERIC-LABEL: test_pext_i64:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    pextq (%rdx), %rdi, %rcx # sched: [6:0.50]
-; GENERIC-NEXT:    pextq %rsi, %rdi, %rax # sched: [1:0.33]
-; GENERIC-NEXT:    addq %rcx, %rax # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_pext_i64:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    pextq (%rdx), %rdi, %rcx # sched: [8:1.00]
-; HASWELL-NEXT:    pextq %rsi, %rdi, %rax # sched: [3:1.00]
-; HASWELL-NEXT:    addq %rcx, %rax # sched: [1:0.25]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pext_i64:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    pextq (%rdx), %rdi, %rcx # sched: [8:1.00]
-; BROADWELL-NEXT:    pextq %rsi, %rdi, %rax # sched: [3:1.00]
-; BROADWELL-NEXT:    addq %rcx, %rax # sched: [1:0.25]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pext_i64:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    pextq (%rdx), %rdi, %rcx # sched: [8:1.00]
-; SKYLAKE-NEXT:    pextq %rsi, %rdi, %rax # sched: [3:1.00]
-; SKYLAKE-NEXT:    addq %rcx, %rax # sched: [1:0.25]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; KNL-LABEL: test_pext_i64:
-; KNL:       # %bb.0:
-; KNL-NEXT:    pextq (%rdx), %rdi, %rcx # sched: [8:1.00]
-; KNL-NEXT:    pextq %rsi, %rdi, %rax # sched: [3:1.00]
-; KNL-NEXT:    addq %rcx, %rax # sched: [1:0.25]
-; KNL-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_pext_i64:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    pextq (%rdx), %rdi, %rcx # sched: [100:0.25]
-; ZNVER1-NEXT:    pextq %rsi, %rdi, %rax # sched: [100:0.25]
-; ZNVER1-NEXT:    addq %rcx, %rax # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = load i64, i64 *%a2
-  %2 = tail call i64 @llvm.x86.bmi.pext.64(i64 %a0, i64 %1)
-  %3 = tail call i64 @llvm.x86.bmi.pext.64(i64 %a0, i64 %a1)
-  %4 = add i64 %2, %3
-  ret i64 %4
-}
-declare i64 @llvm.x86.bmi.pext.64(i64, i64)
-
-define i32 @test_rorx_i32(i32 %a0, i32 %a1, i32 *%a2) {
-; GENERIC-LABEL: test_rorx_i32:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    rorxl $5, %edi, %ecx # sched: [1:0.50]
-; GENERIC-NEXT:    rorxl $5, (%rdx), %eax # sched: [6:0.50]
-; GENERIC-NEXT:    addl %ecx, %eax # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_rorx_i32:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    rorxl $5, %edi, %ecx # sched: [1:0.50]
-; HASWELL-NEXT:    rorxl $5, (%rdx), %eax # sched: [6:0.50]
-; HASWELL-NEXT:    addl %ecx, %eax # sched: [1:0.25]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_rorx_i32:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    rorxl $5, %edi, %ecx # sched: [1:0.50]
-; BROADWELL-NEXT:    rorxl $5, (%rdx), %eax # sched: [6:0.50]
-; BROADWELL-NEXT:    addl %ecx, %eax # sched: [1:0.25]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_rorx_i32:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    rorxl $5, %edi, %ecx # sched: [1:0.50]
-; SKYLAKE-NEXT:    rorxl $5, (%rdx), %eax # sched: [6:0.50]
-; SKYLAKE-NEXT:    addl %ecx, %eax # sched: [1:0.25]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; KNL-LABEL: test_rorx_i32:
-; KNL:       # %bb.0:
-; KNL-NEXT:    rorxl $5, %edi, %ecx # sched: [1:0.50]
-; KNL-NEXT:    rorxl $5, (%rdx), %eax # sched: [6:0.50]
-; KNL-NEXT:    addl %ecx, %eax # sched: [1:0.25]
-; KNL-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_rorx_i32:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    rorxl $5, (%rdx), %eax # sched: [5:0.50]
-; ZNVER1-NEXT:    rorxl $5, %edi, %ecx # sched: [1:0.25]
-; ZNVER1-NEXT:    addl %ecx, %eax # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = load i32, i32 *%a2
-  %2 = lshr i32 %a0, 5
-  %3 = shl i32 %a0, 27
-  %4 = or i32 %2, %3
-  %5 = lshr i32 %1, 5
-  %6 = shl i32 %1, 27
-  %7 = or i32 %5, %6
-  %8 = add i32 %4, %7
-  ret i32 %8
-}
-
-define i64 @test_rorx_i64(i64 %a0, i64 %a1, i64 *%a2) {
-; GENERIC-LABEL: test_rorx_i64:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    rorxq $5, %rdi, %rcx # sched: [1:0.50]
-; GENERIC-NEXT:    rorxq $5, (%rdx), %rax # sched: [6:0.50]
-; GENERIC-NEXT:    addq %rcx, %rax # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_rorx_i64:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    rorxq $5, %rdi, %rcx # sched: [1:0.50]
-; HASWELL-NEXT:    rorxq $5, (%rdx), %rax # sched: [6:0.50]
-; HASWELL-NEXT:    addq %rcx, %rax # sched: [1:0.25]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_rorx_i64:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    rorxq $5, %rdi, %rcx # sched: [1:0.50]
-; BROADWELL-NEXT:    rorxq $5, (%rdx), %rax # sched: [6:0.50]
-; BROADWELL-NEXT:    addq %rcx, %rax # sched: [1:0.25]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_rorx_i64:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    rorxq $5, %rdi, %rcx # sched: [1:0.50]
-; SKYLAKE-NEXT:    rorxq $5, (%rdx), %rax # sched: [6:0.50]
-; SKYLAKE-NEXT:    addq %rcx, %rax # sched: [1:0.25]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; KNL-LABEL: test_rorx_i64:
-; KNL:       # %bb.0:
-; KNL-NEXT:    rorxq $5, %rdi, %rcx # sched: [1:0.50]
-; KNL-NEXT:    rorxq $5, (%rdx), %rax # sched: [6:0.50]
-; KNL-NEXT:    addq %rcx, %rax # sched: [1:0.25]
-; KNL-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_rorx_i64:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    rorxq $5, (%rdx), %rax # sched: [5:0.50]
-; ZNVER1-NEXT:    rorxq $5, %rdi, %rcx # sched: [1:0.25]
-; ZNVER1-NEXT:    addq %rcx, %rax # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = load i64, i64 *%a2
-  %2 = lshr i64 %a0, 5
-  %3 = shl i64 %a0, 59
-  %4 = or i64 %2, %3
-  %5 = lshr i64 %1, 5
-  %6 = shl i64 %1, 59
-  %7 = or i64 %5, %6
-  %8 = add i64 %4, %7
-  ret i64 %8
-}
-
-define i32 @test_sarx_i32(i32 %a0, i32 %a1, i32 *%a2) {
-; GENERIC-LABEL: test_sarx_i32:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    sarxl %esi, %edi, %ecx # sched: [1:0.50]
-; GENERIC-NEXT:    sarxl %esi, (%rdx), %eax # sched: [6:0.50]
-; GENERIC-NEXT:    addl %ecx, %eax # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_sarx_i32:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    sarxl %esi, %edi, %ecx # sched: [1:0.50]
-; HASWELL-NEXT:    sarxl %esi, (%rdx), %eax # sched: [6:0.50]
-; HASWELL-NEXT:    addl %ecx, %eax # sched: [1:0.25]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_sarx_i32:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    sarxl %esi, %edi, %ecx # sched: [1:0.50]
-; BROADWELL-NEXT:    sarxl %esi, (%rdx), %eax # sched: [6:0.50]
-; BROADWELL-NEXT:    addl %ecx, %eax # sched: [1:0.25]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_sarx_i32:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    sarxl %esi, %edi, %ecx # sched: [1:0.50]
-; SKYLAKE-NEXT:    sarxl %esi, (%rdx), %eax # sched: [6:0.50]
-; SKYLAKE-NEXT:    addl %ecx, %eax # sched: [1:0.25]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; KNL-LABEL: test_sarx_i32:
-; KNL:       # %bb.0:
-; KNL-NEXT:    sarxl %esi, %edi, %ecx # sched: [1:0.50]
-; KNL-NEXT:    sarxl %esi, (%rdx), %eax # sched: [6:0.50]
-; KNL-NEXT:    addl %ecx, %eax # sched: [1:0.25]
-; KNL-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_sarx_i32:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    sarxl %esi, (%rdx), %eax # sched: [5:0.50]
-; ZNVER1-NEXT:    sarxl %esi, %edi, %ecx # sched: [1:0.25]
-; ZNVER1-NEXT:    addl %ecx, %eax # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = load i32, i32 *%a2
-  %2 = ashr i32 %a0, %a1
-  %3 = ashr i32 %1, %a1
-  %4 = add i32 %2, %3
-  ret i32 %4
-}
-
-define i64 @test_sarx_i64(i64 %a0, i64 %a1, i64 *%a2) {
-; GENERIC-LABEL: test_sarx_i64:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    sarxq %rsi, %rdi, %rcx # sched: [1:0.50]
-; GENERIC-NEXT:    sarxq %rsi, (%rdx), %rax # sched: [6:0.50]
-; GENERIC-NEXT:    addq %rcx, %rax # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_sarx_i64:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    sarxq %rsi, %rdi, %rcx # sched: [1:0.50]
-; HASWELL-NEXT:    sarxq %rsi, (%rdx), %rax # sched: [6:0.50]
-; HASWELL-NEXT:    addq %rcx, %rax # sched: [1:0.25]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_sarx_i64:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    sarxq %rsi, %rdi, %rcx # sched: [1:0.50]
-; BROADWELL-NEXT:    sarxq %rsi, (%rdx), %rax # sched: [6:0.50]
-; BROADWELL-NEXT:    addq %rcx, %rax # sched: [1:0.25]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_sarx_i64:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    sarxq %rsi, %rdi, %rcx # sched: [1:0.50]
-; SKYLAKE-NEXT:    sarxq %rsi, (%rdx), %rax # sched: [6:0.50]
-; SKYLAKE-NEXT:    addq %rcx, %rax # sched: [1:0.25]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; KNL-LABEL: test_sarx_i64:
-; KNL:       # %bb.0:
-; KNL-NEXT:    sarxq %rsi, %rdi, %rcx # sched: [1:0.50]
-; KNL-NEXT:    sarxq %rsi, (%rdx), %rax # sched: [6:0.50]
-; KNL-NEXT:    addq %rcx, %rax # sched: [1:0.25]
-; KNL-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_sarx_i64:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    sarxq %rsi, (%rdx), %rax # sched: [5:0.50]
-; ZNVER1-NEXT:    sarxq %rsi, %rdi, %rcx # sched: [1:0.25]
-; ZNVER1-NEXT:    addq %rcx, %rax # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = load i64, i64 *%a2
-  %2 = ashr i64 %a0, %a1
-  %3 = ashr i64 %1, %a1
-  %4 = add i64 %2, %3
-  ret i64 %4
-}
-
-define i32 @test_shlx_i32(i32 %a0, i32 %a1, i32 *%a2) {
-; GENERIC-LABEL: test_shlx_i32:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    shlxl %esi, %edi, %ecx # sched: [1:0.50]
-; GENERIC-NEXT:    shlxl %esi, (%rdx), %eax # sched: [6:0.50]
-; GENERIC-NEXT:    addl %ecx, %eax # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_shlx_i32:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    shlxl %esi, %edi, %ecx # sched: [1:0.50]
-; HASWELL-NEXT:    shlxl %esi, (%rdx), %eax # sched: [6:0.50]
-; HASWELL-NEXT:    addl %ecx, %eax # sched: [1:0.25]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_shlx_i32:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    shlxl %esi, %edi, %ecx # sched: [1:0.50]
-; BROADWELL-NEXT:    shlxl %esi, (%rdx), %eax # sched: [6:0.50]
-; BROADWELL-NEXT:    addl %ecx, %eax # sched: [1:0.25]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_shlx_i32:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    shlxl %esi, %edi, %ecx # sched: [1:0.50]
-; SKYLAKE-NEXT:    shlxl %esi, (%rdx), %eax # sched: [6:0.50]
-; SKYLAKE-NEXT:    addl %ecx, %eax # sched: [1:0.25]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; KNL-LABEL: test_shlx_i32:
-; KNL:       # %bb.0:
-; KNL-NEXT:    shlxl %esi, %edi, %ecx # sched: [1:0.50]
-; KNL-NEXT:    shlxl %esi, (%rdx), %eax # sched: [6:0.50]
-; KNL-NEXT:    addl %ecx, %eax # sched: [1:0.25]
-; KNL-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_shlx_i32:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    shlxl %esi, (%rdx), %eax # sched: [5:0.50]
-; ZNVER1-NEXT:    shlxl %esi, %edi, %ecx # sched: [1:0.25]
-; ZNVER1-NEXT:    addl %ecx, %eax # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = load i32, i32 *%a2
-  %2 = shl i32 %a0, %a1
-  %3 = shl i32 %1, %a1
-  %4 = add i32 %2, %3
-  ret i32 %4
-}
-
-define i64 @test_shlx_i64(i64 %a0, i64 %a1, i64 *%a2) {
-; GENERIC-LABEL: test_shlx_i64:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    shlxq %rsi, %rdi, %rcx # sched: [1:0.50]
-; GENERIC-NEXT:    shlxq %rsi, (%rdx), %rax # sched: [6:0.50]
-; GENERIC-NEXT:    addq %rcx, %rax # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_shlx_i64:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    shlxq %rsi, %rdi, %rcx # sched: [1:0.50]
-; HASWELL-NEXT:    shlxq %rsi, (%rdx), %rax # sched: [6:0.50]
-; HASWELL-NEXT:    addq %rcx, %rax # sched: [1:0.25]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_shlx_i64:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    shlxq %rsi, %rdi, %rcx # sched: [1:0.50]
-; BROADWELL-NEXT:    shlxq %rsi, (%rdx), %rax # sched: [6:0.50]
-; BROADWELL-NEXT:    addq %rcx, %rax # sched: [1:0.25]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_shlx_i64:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    shlxq %rsi, %rdi, %rcx # sched: [1:0.50]
-; SKYLAKE-NEXT:    shlxq %rsi, (%rdx), %rax # sched: [6:0.50]
-; SKYLAKE-NEXT:    addq %rcx, %rax # sched: [1:0.25]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; KNL-LABEL: test_shlx_i64:
-; KNL:       # %bb.0:
-; KNL-NEXT:    shlxq %rsi, %rdi, %rcx # sched: [1:0.50]
-; KNL-NEXT:    shlxq %rsi, (%rdx), %rax # sched: [6:0.50]
-; KNL-NEXT:    addq %rcx, %rax # sched: [1:0.25]
-; KNL-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_shlx_i64:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    shlxq %rsi, (%rdx), %rax # sched: [5:0.50]
-; ZNVER1-NEXT:    shlxq %rsi, %rdi, %rcx # sched: [1:0.25]
-; ZNVER1-NEXT:    addq %rcx, %rax # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = load i64, i64 *%a2
-  %2 = shl i64 %a0, %a1
-  %3 = shl i64 %1, %a1
-  %4 = add i64 %2, %3
-  ret i64 %4
-}
-
-define i32 @test_shrx_i32(i32 %a0, i32 %a1, i32 *%a2) {
-; GENERIC-LABEL: test_shrx_i32:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    shrxl %esi, %edi, %ecx # sched: [1:0.50]
-; GENERIC-NEXT:    shrxl %esi, (%rdx), %eax # sched: [6:0.50]
-; GENERIC-NEXT:    addl %ecx, %eax # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_shrx_i32:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    shrxl %esi, %edi, %ecx # sched: [1:0.50]
-; HASWELL-NEXT:    shrxl %esi, (%rdx), %eax # sched: [6:0.50]
-; HASWELL-NEXT:    addl %ecx, %eax # sched: [1:0.25]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_shrx_i32:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    shrxl %esi, %edi, %ecx # sched: [1:0.50]
-; BROADWELL-NEXT:    shrxl %esi, (%rdx), %eax # sched: [6:0.50]
-; BROADWELL-NEXT:    addl %ecx, %eax # sched: [1:0.25]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_shrx_i32:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    shrxl %esi, %edi, %ecx # sched: [1:0.50]
-; SKYLAKE-NEXT:    shrxl %esi, (%rdx), %eax # sched: [6:0.50]
-; SKYLAKE-NEXT:    addl %ecx, %eax # sched: [1:0.25]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; KNL-LABEL: test_shrx_i32:
-; KNL:       # %bb.0:
-; KNL-NEXT:    shrxl %esi, %edi, %ecx # sched: [1:0.50]
-; KNL-NEXT:    shrxl %esi, (%rdx), %eax # sched: [6:0.50]
-; KNL-NEXT:    addl %ecx, %eax # sched: [1:0.25]
-; KNL-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_shrx_i32:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    shrxl %esi, (%rdx), %eax # sched: [5:0.50]
-; ZNVER1-NEXT:    shrxl %esi, %edi, %ecx # sched: [1:0.25]
-; ZNVER1-NEXT:    addl %ecx, %eax # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = load i32, i32 *%a2
-  %2 = lshr i32 %a0, %a1
-  %3 = lshr i32 %1, %a1
-  %4 = add i32 %2, %3
-  ret i32 %4
-}
-
-define i64 @test_shrx_i64(i64 %a0, i64 %a1, i64 *%a2) {
-; GENERIC-LABEL: test_shrx_i64:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    shrxq %rsi, %rdi, %rcx # sched: [1:0.50]
-; GENERIC-NEXT:    shrxq %rsi, (%rdx), %rax # sched: [6:0.50]
-; GENERIC-NEXT:    addq %rcx, %rax # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_shrx_i64:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    shrxq %rsi, %rdi, %rcx # sched: [1:0.50]
-; HASWELL-NEXT:    shrxq %rsi, (%rdx), %rax # sched: [6:0.50]
-; HASWELL-NEXT:    addq %rcx, %rax # sched: [1:0.25]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_shrx_i64:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    shrxq %rsi, %rdi, %rcx # sched: [1:0.50]
-; BROADWELL-NEXT:    shrxq %rsi, (%rdx), %rax # sched: [6:0.50]
-; BROADWELL-NEXT:    addq %rcx, %rax # sched: [1:0.25]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_shrx_i64:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    shrxq %rsi, %rdi, %rcx # sched: [1:0.50]
-; SKYLAKE-NEXT:    shrxq %rsi, (%rdx), %rax # sched: [6:0.50]
-; SKYLAKE-NEXT:    addq %rcx, %rax # sched: [1:0.25]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; KNL-LABEL: test_shrx_i64:
-; KNL:       # %bb.0:
-; KNL-NEXT:    shrxq %rsi, %rdi, %rcx # sched: [1:0.50]
-; KNL-NEXT:    shrxq %rsi, (%rdx), %rax # sched: [6:0.50]
-; KNL-NEXT:    addq %rcx, %rax # sched: [1:0.25]
-; KNL-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_shrx_i64:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    shrxq %rsi, (%rdx), %rax # sched: [5:0.50]
-; ZNVER1-NEXT:    shrxq %rsi, %rdi, %rcx # sched: [1:0.25]
-; ZNVER1-NEXT:    addq %rcx, %rax # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = load i64, i64 *%a2
-  %2 = lshr i64 %a0, %a1
-  %3 = lshr i64 %1, %a1
-  %4 = add i64 %2, %3
-  ret i64 %4
-}
diff --git a/test/CodeGen/X86/bool-ext-inc.ll b/test/CodeGen/X86/bool-ext-inc.ll
index d5711fdb3ca1c5ec9edb14ab411cbeb06f3903d3..d89893f94bdaec212dc2d9e5d58b4aa352886009 100644
--- a/test/CodeGen/X86/bool-ext-inc.ll
+++ b/test/CodeGen/X86/bool-ext-inc.ll
@@ -102,3 +102,61 @@ define <4 x i32> @bool_logic_and_math_vec(<4 x i32> %a, <4 x i32> %b, <4 x i32>
   ret <4 x i32> %add
 }
 
+define <4 x i32> @sextbool_add_vector(<4 x i32> %cmp1, <4 x i32> %cmp2, <4 x i32> %x) {
+; CHECK-LABEL: sextbool_add_vector:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    vpaddd %xmm0, %xmm2, %xmm0
+; CHECK-NEXT:    retq
+  %c = icmp eq <4 x i32> %cmp1, %cmp2
+  %b = sext <4 x i1> %c to <4 x i32>
+  %s = add <4 x i32> %x, %b
+  ret <4 x i32> %s
+}
+
+define <4 x i32> @zextbool_sub_vector(<4 x i32> %cmp1, <4 x i32> %cmp2, <4 x i32> %x) {
+; CHECK-LABEL: zextbool_sub_vector:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    vpaddd %xmm0, %xmm2, %xmm0
+; CHECK-NEXT:    retq
+  %c = icmp eq <4 x i32> %cmp1, %cmp2
+  %b = zext <4 x i1> %c to <4 x i32>
+  %s = sub <4 x i32> %x, %b
+  ret <4 x i32> %s
+}
+
+define i32 @assertsext_sub_1(i1 signext %cond, i32 %y) {
+; CHECK-LABEL: assertsext_sub_1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    # kill: def $esi killed $esi def $rsi
+; CHECK-NEXT:    # kill: def $edi killed $edi def $rdi
+; CHECK-NEXT:    leal (%rdi,%rsi), %eax
+; CHECK-NEXT:    retq
+  %e = zext i1 %cond to i32
+  %r = sub i32 %y, %e
+  ret i32 %r
+}
+
+define i32 @assertsext_add_1(i1 signext %cond, i32 %y) {
+; CHECK-LABEL: assertsext_add_1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl %esi, %eax
+; CHECK-NEXT:    subl %edi, %eax
+; CHECK-NEXT:    retq
+  %e = zext i1 %cond to i32
+  %r = add i32 %e, %y
+  ret i32 %r
+}
+
+define i32 @assertsext_add_1_commute(i1 signext %cond, i32 %y) {
+; CHECK-LABEL: assertsext_add_1_commute:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl %esi, %eax
+; CHECK-NEXT:    subl %edi, %eax
+; CHECK-NEXT:    retq
+  %e = zext i1 %cond to i32
+  %r = add i32 %y, %e
+  ret i32 %r
+}
+
diff --git a/test/CodeGen/X86/broadcast-elm-cross-splat-vec.ll b/test/CodeGen/X86/broadcast-elm-cross-splat-vec.ll
index 353faabba2de2b6279fc286878d8113549b47069..7b9f1abc9ba8d9197e7f7a8ac472fa3eff11f09c 100644
--- a/test/CodeGen/X86/broadcast-elm-cross-splat-vec.ll
+++ b/test/CodeGen/X86/broadcast-elm-cross-splat-vec.ll
@@ -95,8 +95,7 @@ define <16 x i8> @f16xi8_i64(<16 x i8> %a) {
 ;
 ; ALL32-LABEL: f16xi8_i64:
 ; ALL32:       # %bb.0:
-; ALL32-NEXT:    vmovddup {{.*#+}} xmm1 = [7.9499288951273625E-275,7.9499288951273625E-275]
-; ALL32-NEXT:    # xmm1 = mem[0,0]
+; ALL32-NEXT:    vpbroadcastq {{.*#+}} xmm1 = [7.9499288951273625E-275,7.9499288951273625E-275]
 ; ALL32-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
 ; ALL32-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; ALL32-NEXT:    retl
@@ -358,17 +357,15 @@ define <64 x i8> @f64xi8_i16(<64 x i8> %a) {
 define <64 x i8> @f64i8_i32(<64 x i8> %a) {
 ; AVX-LABEL: f64i8_i32:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vextractf128 $1, %ymm1, %xmm2
-; AVX-NEXT:    vmovd {{.*#+}} xmm3 = mem[0],zero,zero,zero
-; AVX-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[0,0,0,0]
-; AVX-NEXT:    vpaddb %xmm3, %xmm2, %xmm2
-; AVX-NEXT:    vpaddb %xmm3, %xmm1, %xmm1
-; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
-; AVX-NEXT:    vextractf128 $1, %ymm0, %xmm2
-; AVX-NEXT:    vpaddb %xmm3, %xmm2, %xmm2
-; AVX-NEXT:    vpaddb %xmm3, %xmm0, %xmm0
-; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
-; AVX-NEXT:    vinsertf128 $1, %xmm3, %ymm3, %ymm2
+; AVX-NEXT:    vbroadcastss {{.*#+}} ymm2 = [3.82047143E-37,3.82047143E-37,3.82047143E-37,3.82047143E-37,3.82047143E-37,3.82047143E-37,3.82047143E-37,3.82047143E-37]
+; AVX-NEXT:    vextractf128 $1, %ymm1, %xmm3
+; AVX-NEXT:    vpaddb %xmm2, %xmm3, %xmm3
+; AVX-NEXT:    vpaddb %xmm2, %xmm1, %xmm1
+; AVX-NEXT:    vinsertf128 $1, %xmm3, %ymm1, %ymm1
+; AVX-NEXT:    vextractf128 $1, %ymm0, %xmm3
+; AVX-NEXT:    vpaddb %xmm2, %xmm3, %xmm3
+; AVX-NEXT:    vpaddb %xmm2, %xmm0, %xmm0
+; AVX-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm0
 ; AVX-NEXT:    vandps %ymm2, %ymm0, %ymm0
 ; AVX-NEXT:    vandps %ymm2, %ymm1, %ymm1
 ; AVX-NEXT:    retl
@@ -391,17 +388,15 @@ define <64 x i8> @f64i8_i32(<64 x i8> %a) {
 ;
 ; AVX-64-LABEL: f64i8_i32:
 ; AVX-64:       # %bb.0:
-; AVX-64-NEXT:    vextractf128 $1, %ymm1, %xmm2
-; AVX-64-NEXT:    vmovd {{.*#+}} xmm3 = mem[0],zero,zero,zero
-; AVX-64-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[0,0,0,0]
-; AVX-64-NEXT:    vpaddb %xmm3, %xmm2, %xmm2
-; AVX-64-NEXT:    vpaddb %xmm3, %xmm1, %xmm1
-; AVX-64-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
-; AVX-64-NEXT:    vextractf128 $1, %ymm0, %xmm2
-; AVX-64-NEXT:    vpaddb %xmm3, %xmm2, %xmm2
-; AVX-64-NEXT:    vpaddb %xmm3, %xmm0, %xmm0
-; AVX-64-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
-; AVX-64-NEXT:    vinsertf128 $1, %xmm3, %ymm3, %ymm2
+; AVX-64-NEXT:    vbroadcastss {{.*#+}} ymm2 = [3.82047143E-37,3.82047143E-37,3.82047143E-37,3.82047143E-37,3.82047143E-37,3.82047143E-37,3.82047143E-37,3.82047143E-37]
+; AVX-64-NEXT:    vextractf128 $1, %ymm1, %xmm3
+; AVX-64-NEXT:    vpaddb %xmm2, %xmm3, %xmm3
+; AVX-64-NEXT:    vpaddb %xmm2, %xmm1, %xmm1
+; AVX-64-NEXT:    vinsertf128 $1, %xmm3, %ymm1, %ymm1
+; AVX-64-NEXT:    vextractf128 $1, %ymm0, %xmm3
+; AVX-64-NEXT:    vpaddb %xmm2, %xmm3, %xmm3
+; AVX-64-NEXT:    vpaddb %xmm2, %xmm0, %xmm0
+; AVX-64-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm0
 ; AVX-64-NEXT:    vandps %ymm2, %ymm0, %ymm0
 ; AVX-64-NEXT:    vandps %ymm2, %ymm1, %ymm1
 ; AVX-64-NEXT:    retq
@@ -427,21 +422,18 @@ define <64 x i8> @f64i8_i32(<64 x i8> %a) {
 }
 
 
-; FIXME the load should be folded with the MOVDDUP with AVX1. PR39454
 define <64 x i8> @f64xi8_i64(<64 x i8> %a) {
 ; AVX-LABEL: f64xi8_i64:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vextractf128 $1, %ymm1, %xmm2
-; AVX-NEXT:    vmovsd {{.*#+}} xmm3 = mem[0],zero
-; AVX-NEXT:    vmovddup {{.*#+}} xmm3 = xmm3[0,0]
-; AVX-NEXT:    vpaddb %xmm3, %xmm2, %xmm2
-; AVX-NEXT:    vpaddb %xmm3, %xmm1, %xmm1
-; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
-; AVX-NEXT:    vextractf128 $1, %ymm0, %xmm2
-; AVX-NEXT:    vpaddb %xmm3, %xmm2, %xmm2
-; AVX-NEXT:    vpaddb %xmm3, %xmm0, %xmm0
-; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
-; AVX-NEXT:    vinsertf128 $1, %xmm3, %ymm3, %ymm2
+; AVX-NEXT:    vbroadcastsd {{.*#+}} ymm2 = [7.9499288951273625E-275,7.9499288951273625E-275,7.9499288951273625E-275,7.9499288951273625E-275]
+; AVX-NEXT:    vextractf128 $1, %ymm1, %xmm3
+; AVX-NEXT:    vpaddb %xmm2, %xmm3, %xmm3
+; AVX-NEXT:    vpaddb %xmm2, %xmm1, %xmm1
+; AVX-NEXT:    vinsertf128 $1, %xmm3, %ymm1, %ymm1
+; AVX-NEXT:    vextractf128 $1, %ymm0, %xmm3
+; AVX-NEXT:    vpaddb %xmm2, %xmm3, %xmm3
+; AVX-NEXT:    vpaddb %xmm2, %xmm0, %xmm0
+; AVX-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm0
 ; AVX-NEXT:    vandps %ymm2, %ymm0, %ymm0
 ; AVX-NEXT:    vandps %ymm2, %ymm1, %ymm1
 ; AVX-NEXT:    retl
@@ -464,17 +456,15 @@ define <64 x i8> @f64xi8_i64(<64 x i8> %a) {
 ;
 ; AVX-64-LABEL: f64xi8_i64:
 ; AVX-64:       # %bb.0:
-; AVX-64-NEXT:    vextractf128 $1, %ymm1, %xmm2
-; AVX-64-NEXT:    vmovsd {{.*#+}} xmm3 = mem[0],zero
-; AVX-64-NEXT:    vmovddup {{.*#+}} xmm3 = xmm3[0,0]
-; AVX-64-NEXT:    vpaddb %xmm3, %xmm2, %xmm2
-; AVX-64-NEXT:    vpaddb %xmm3, %xmm1, %xmm1
-; AVX-64-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
-; AVX-64-NEXT:    vextractf128 $1, %ymm0, %xmm2
-; AVX-64-NEXT:    vpaddb %xmm3, %xmm2, %xmm2
-; AVX-64-NEXT:    vpaddb %xmm3, %xmm0, %xmm0
-; AVX-64-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
-; AVX-64-NEXT:    vinsertf128 $1, %xmm3, %ymm3, %ymm2
+; AVX-64-NEXT:    vbroadcastsd {{.*#+}} ymm2 = [7.9499288951273625E-275,7.9499288951273625E-275,7.9499288951273625E-275,7.9499288951273625E-275]
+; AVX-64-NEXT:    vextractf128 $1, %ymm1, %xmm3
+; AVX-64-NEXT:    vpaddb %xmm2, %xmm3, %xmm3
+; AVX-64-NEXT:    vpaddb %xmm2, %xmm1, %xmm1
+; AVX-64-NEXT:    vinsertf128 $1, %xmm3, %ymm1, %ymm1
+; AVX-64-NEXT:    vextractf128 $1, %ymm0, %xmm3
+; AVX-64-NEXT:    vpaddb %xmm2, %xmm3, %xmm3
+; AVX-64-NEXT:    vpaddb %xmm2, %xmm0, %xmm0
+; AVX-64-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm0
 ; AVX-64-NEXT:    vandps %ymm2, %ymm0, %ymm0
 ; AVX-64-NEXT:    vandps %ymm2, %ymm1, %ymm1
 ; AVX-64-NEXT:    retq
@@ -499,6 +489,7 @@ define <64 x i8> @f64xi8_i64(<64 x i8> %a) {
   ret <64 x i8> %res2
 }
 
+
 define <64 x i8> @f64xi8_i128(<64 x i8> %a) {
 ; AVX-LABEL: f64xi8_i128:
 ; AVX:       # %bb.0:
@@ -692,8 +683,7 @@ define <8 x i16> @f8xi16_i64(<8 x i16> %a) {
 ;
 ; ALL32-LABEL: f8xi16_i64:
 ; ALL32:       # %bb.0:
-; ALL32-NEXT:    vmovddup {{.*#+}} xmm1 = [4.1720559249406128E-309,4.1720559249406128E-309]
-; ALL32-NEXT:    # xmm1 = mem[0,0]
+; ALL32-NEXT:    vpbroadcastq {{.*#+}} xmm1 = [4.1720559249406128E-309,4.1720559249406128E-309]
 ; ALL32-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
 ; ALL32-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; ALL32-NEXT:    retl
@@ -845,17 +835,15 @@ define <16 x i16> @f16xi16_i128(<16 x i16> %a) {
 define <32 x i16> @f32xi16_i32(<32 x i16> %a) {
 ; AVX-LABEL: f32xi16_i32:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vextractf128 $1, %ymm1, %xmm2
-; AVX-NEXT:    vmovd {{.*#+}} xmm3 = mem[0],zero,zero,zero
-; AVX-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[0,0,0,0]
-; AVX-NEXT:    vpaddw %xmm3, %xmm2, %xmm2
-; AVX-NEXT:    vpaddw %xmm3, %xmm1, %xmm1
-; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
-; AVX-NEXT:    vextractf128 $1, %ymm0, %xmm2
-; AVX-NEXT:    vpaddw %xmm3, %xmm2, %xmm2
-; AVX-NEXT:    vpaddw %xmm3, %xmm0, %xmm0
-; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
-; AVX-NEXT:    vinsertf128 $1, %xmm3, %ymm3, %ymm2
+; AVX-NEXT:    vbroadcastss {{.*#+}} ymm2 = [9.18354962E-41,9.18354962E-41,9.18354962E-41,9.18354962E-41,9.18354962E-41,9.18354962E-41,9.18354962E-41,9.18354962E-41]
+; AVX-NEXT:    vextractf128 $1, %ymm1, %xmm3
+; AVX-NEXT:    vpaddw %xmm2, %xmm3, %xmm3
+; AVX-NEXT:    vpaddw %xmm2, %xmm1, %xmm1
+; AVX-NEXT:    vinsertf128 $1, %xmm3, %ymm1, %ymm1
+; AVX-NEXT:    vextractf128 $1, %ymm0, %xmm3
+; AVX-NEXT:    vpaddw %xmm2, %xmm3, %xmm3
+; AVX-NEXT:    vpaddw %xmm2, %xmm0, %xmm0
+; AVX-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm0
 ; AVX-NEXT:    vandps %ymm2, %ymm0, %ymm0
 ; AVX-NEXT:    vandps %ymm2, %ymm1, %ymm1
 ; AVX-NEXT:    retl
@@ -878,17 +866,15 @@ define <32 x i16> @f32xi16_i32(<32 x i16> %a) {
 ;
 ; AVX-64-LABEL: f32xi16_i32:
 ; AVX-64:       # %bb.0:
-; AVX-64-NEXT:    vextractf128 $1, %ymm1, %xmm2
-; AVX-64-NEXT:    vmovd {{.*#+}} xmm3 = mem[0],zero,zero,zero
-; AVX-64-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[0,0,0,0]
-; AVX-64-NEXT:    vpaddw %xmm3, %xmm2, %xmm2
-; AVX-64-NEXT:    vpaddw %xmm3, %xmm1, %xmm1
-; AVX-64-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
-; AVX-64-NEXT:    vextractf128 $1, %ymm0, %xmm2
-; AVX-64-NEXT:    vpaddw %xmm3, %xmm2, %xmm2
-; AVX-64-NEXT:    vpaddw %xmm3, %xmm0, %xmm0
-; AVX-64-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
-; AVX-64-NEXT:    vinsertf128 $1, %xmm3, %ymm3, %ymm2
+; AVX-64-NEXT:    vbroadcastss {{.*#+}} ymm2 = [9.18354962E-41,9.18354962E-41,9.18354962E-41,9.18354962E-41,9.18354962E-41,9.18354962E-41,9.18354962E-41,9.18354962E-41]
+; AVX-64-NEXT:    vextractf128 $1, %ymm1, %xmm3
+; AVX-64-NEXT:    vpaddw %xmm2, %xmm3, %xmm3
+; AVX-64-NEXT:    vpaddw %xmm2, %xmm1, %xmm1
+; AVX-64-NEXT:    vinsertf128 $1, %xmm3, %ymm1, %ymm1
+; AVX-64-NEXT:    vextractf128 $1, %ymm0, %xmm3
+; AVX-64-NEXT:    vpaddw %xmm2, %xmm3, %xmm3
+; AVX-64-NEXT:    vpaddw %xmm2, %xmm0, %xmm0
+; AVX-64-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm0
 ; AVX-64-NEXT:    vandps %ymm2, %ymm0, %ymm0
 ; AVX-64-NEXT:    vandps %ymm2, %ymm1, %ymm1
 ; AVX-64-NEXT:    retq
@@ -914,21 +900,18 @@ define <32 x i16> @f32xi16_i32(<32 x i16> %a) {
 }
 
 
-; FIXME the load should be folded with the MOVDDUP with AVX1. PR39454
 define <32 x i16> @f32xi16_i64(<32 x i16> %a) {
 ; AVX-LABEL: f32xi16_i64:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vextractf128 $1, %ymm1, %xmm2
-; AVX-NEXT:    vmovsd {{.*#+}} xmm3 = mem[0],zero
-; AVX-NEXT:    vmovddup {{.*#+}} xmm3 = xmm3[0,0]
-; AVX-NEXT:    vpaddw %xmm3, %xmm2, %xmm2
-; AVX-NEXT:    vpaddw %xmm3, %xmm1, %xmm1
-; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
-; AVX-NEXT:    vextractf128 $1, %ymm0, %xmm2
-; AVX-NEXT:    vpaddw %xmm3, %xmm2, %xmm2
-; AVX-NEXT:    vpaddw %xmm3, %xmm0, %xmm0
-; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
-; AVX-NEXT:    vinsertf128 $1, %xmm3, %ymm3, %ymm2
+; AVX-NEXT:    vbroadcastsd {{.*#+}} ymm2 = [4.1720559249406128E-309,4.1720559249406128E-309,4.1720559249406128E-309,4.1720559249406128E-309]
+; AVX-NEXT:    vextractf128 $1, %ymm1, %xmm3
+; AVX-NEXT:    vpaddw %xmm2, %xmm3, %xmm3
+; AVX-NEXT:    vpaddw %xmm2, %xmm1, %xmm1
+; AVX-NEXT:    vinsertf128 $1, %xmm3, %ymm1, %ymm1
+; AVX-NEXT:    vextractf128 $1, %ymm0, %xmm3
+; AVX-NEXT:    vpaddw %xmm2, %xmm3, %xmm3
+; AVX-NEXT:    vpaddw %xmm2, %xmm0, %xmm0
+; AVX-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm0
 ; AVX-NEXT:    vandps %ymm2, %ymm0, %ymm0
 ; AVX-NEXT:    vandps %ymm2, %ymm1, %ymm1
 ; AVX-NEXT:    retl
@@ -951,17 +934,15 @@ define <32 x i16> @f32xi16_i64(<32 x i16> %a) {
 ;
 ; AVX-64-LABEL: f32xi16_i64:
 ; AVX-64:       # %bb.0:
-; AVX-64-NEXT:    vextractf128 $1, %ymm1, %xmm2
-; AVX-64-NEXT:    vmovsd {{.*#+}} xmm3 = mem[0],zero
-; AVX-64-NEXT:    vmovddup {{.*#+}} xmm3 = xmm3[0,0]
-; AVX-64-NEXT:    vpaddw %xmm3, %xmm2, %xmm2
-; AVX-64-NEXT:    vpaddw %xmm3, %xmm1, %xmm1
-; AVX-64-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
-; AVX-64-NEXT:    vextractf128 $1, %ymm0, %xmm2
-; AVX-64-NEXT:    vpaddw %xmm3, %xmm2, %xmm2
-; AVX-64-NEXT:    vpaddw %xmm3, %xmm0, %xmm0
-; AVX-64-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
-; AVX-64-NEXT:    vinsertf128 $1, %xmm3, %ymm3, %ymm2
+; AVX-64-NEXT:    vbroadcastsd {{.*#+}} ymm2 = [4.1720559249406128E-309,4.1720559249406128E-309,4.1720559249406128E-309,4.1720559249406128E-309]
+; AVX-64-NEXT:    vextractf128 $1, %ymm1, %xmm3
+; AVX-64-NEXT:    vpaddw %xmm2, %xmm3, %xmm3
+; AVX-64-NEXT:    vpaddw %xmm2, %xmm1, %xmm1
+; AVX-64-NEXT:    vinsertf128 $1, %xmm3, %ymm1, %ymm1
+; AVX-64-NEXT:    vextractf128 $1, %ymm0, %xmm3
+; AVX-64-NEXT:    vpaddw %xmm2, %xmm3, %xmm3
+; AVX-64-NEXT:    vpaddw %xmm2, %xmm0, %xmm0
+; AVX-64-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm0
 ; AVX-64-NEXT:    vandps %ymm2, %ymm0, %ymm0
 ; AVX-64-NEXT:    vandps %ymm2, %ymm1, %ymm1
 ; AVX-64-NEXT:    retq
@@ -1135,7 +1116,6 @@ define <32 x i16> @f32xi16_i256(<32 x i16> %a) {
 }
 
 
-
 define <4 x i32> @f4xi32_i64(<4 x i32> %a) {
 ; AVX-LABEL: f4xi32_i64:
 ; AVX:       # %bb.0:
@@ -1147,8 +1127,7 @@ define <4 x i32> @f4xi32_i64(<4 x i32> %a) {
 ;
 ; ALL32-LABEL: f4xi32_i64:
 ; ALL32:       # %bb.0:
-; ALL32-NEXT:    vmovddup {{.*#+}} xmm1 = [2.1219957909652723E-314,2.1219957909652723E-314]
-; ALL32-NEXT:    # xmm1 = mem[0,0]
+; ALL32-NEXT:    vpbroadcastq {{.*#+}} xmm1 = [2.1219957909652723E-314,2.1219957909652723E-314]
 ; ALL32-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; ALL32-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; ALL32-NEXT:    retl
@@ -1257,21 +1236,18 @@ define <8 x i32> @f8xi32_i128(<8 x i32> %a) {
 }
 
 
-; FIXME the load should be folded with the MOVDDUP with AVX1. PR39454
 define <16 x i32> @f16xi32_i64(<16 x i32> %a) {
 ; AVX-LABEL: f16xi32_i64:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vextractf128 $1, %ymm1, %xmm2
-; AVX-NEXT:    vmovsd {{.*#+}} xmm3 = mem[0],zero
-; AVX-NEXT:    vmovddup {{.*#+}} xmm3 = xmm3[0,0]
-; AVX-NEXT:    vpaddd %xmm3, %xmm2, %xmm2
-; AVX-NEXT:    vpaddd %xmm3, %xmm1, %xmm1
-; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
-; AVX-NEXT:    vextractf128 $1, %ymm0, %xmm2
-; AVX-NEXT:    vpaddd %xmm3, %xmm2, %xmm2
-; AVX-NEXT:    vpaddd %xmm3, %xmm0, %xmm0
-; AVX-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
-; AVX-NEXT:    vinsertf128 $1, %xmm3, %ymm3, %ymm2
+; AVX-NEXT:    vbroadcastsd {{.*#+}} ymm2 = [2.1219957909652723E-314,2.1219957909652723E-314,2.1219957909652723E-314,2.1219957909652723E-314]
+; AVX-NEXT:    vextractf128 $1, %ymm1, %xmm3
+; AVX-NEXT:    vpaddd %xmm2, %xmm3, %xmm3
+; AVX-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
+; AVX-NEXT:    vinsertf128 $1, %xmm3, %ymm1, %ymm1
+; AVX-NEXT:    vextractf128 $1, %ymm0, %xmm3
+; AVX-NEXT:    vpaddd %xmm2, %xmm3, %xmm3
+; AVX-NEXT:    vpaddd %xmm2, %xmm0, %xmm0
+; AVX-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm0
 ; AVX-NEXT:    vandps %ymm2, %ymm0, %ymm0
 ; AVX-NEXT:    vandps %ymm2, %ymm1, %ymm1
 ; AVX-NEXT:    retl
@@ -1294,17 +1270,15 @@ define <16 x i32> @f16xi32_i64(<16 x i32> %a) {
 ;
 ; AVX-64-LABEL: f16xi32_i64:
 ; AVX-64:       # %bb.0:
-; AVX-64-NEXT:    vextractf128 $1, %ymm1, %xmm2
-; AVX-64-NEXT:    vmovsd {{.*#+}} xmm3 = mem[0],zero
-; AVX-64-NEXT:    vmovddup {{.*#+}} xmm3 = xmm3[0,0]
-; AVX-64-NEXT:    vpaddd %xmm3, %xmm2, %xmm2
-; AVX-64-NEXT:    vpaddd %xmm3, %xmm1, %xmm1
-; AVX-64-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
-; AVX-64-NEXT:    vextractf128 $1, %ymm0, %xmm2
-; AVX-64-NEXT:    vpaddd %xmm3, %xmm2, %xmm2
-; AVX-64-NEXT:    vpaddd %xmm3, %xmm0, %xmm0
-; AVX-64-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
-; AVX-64-NEXT:    vinsertf128 $1, %xmm3, %ymm3, %ymm2
+; AVX-64-NEXT:    vbroadcastsd {{.*#+}} ymm2 = [2.1219957909652723E-314,2.1219957909652723E-314,2.1219957909652723E-314,2.1219957909652723E-314]
+; AVX-64-NEXT:    vextractf128 $1, %ymm1, %xmm3
+; AVX-64-NEXT:    vpaddd %xmm2, %xmm3, %xmm3
+; AVX-64-NEXT:    vpaddd %xmm2, %xmm1, %xmm1
+; AVX-64-NEXT:    vinsertf128 $1, %xmm3, %ymm1, %ymm1
+; AVX-64-NEXT:    vextractf128 $1, %ymm0, %xmm3
+; AVX-64-NEXT:    vpaddd %xmm2, %xmm3, %xmm3
+; AVX-64-NEXT:    vpaddd %xmm2, %xmm0, %xmm0
+; AVX-64-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm0
 ; AVX-64-NEXT:    vandps %ymm2, %ymm0, %ymm0
 ; AVX-64-NEXT:    vandps %ymm2, %ymm1, %ymm1
 ; AVX-64-NEXT:    retq
@@ -1624,7 +1598,8 @@ define <4 x float> @f4xf32_f64(<4 x float> %a) {
 ;
 ; ALL64-LABEL: f4xf32_f64:
 ; ALL64:       # %bb.0:
-; ALL64-NEXT:    vpbroadcastq {{.*#+}} xmm1 = [4575657222482165760,4575657222482165760]
+; ALL64-NEXT:    vmovddup {{.*#+}} xmm1 = [4575657222482165760,4575657222482165760]
+; ALL64-NEXT:    # xmm1 = mem[0,0]
 ; ALL64-NEXT:    vaddps %xmm1, %xmm0, %xmm0
 ; ALL64-NEXT:    vdivps %xmm0, %xmm1, %xmm0
 ; ALL64-NEXT:    retq
@@ -2047,7 +2022,6 @@ define <8 x double> @f8xf64_f256(<8 x double> %a) {
 }
 
 
-
 define <8 x i16> @f8xi16_i32_NaN(<8 x i16> %a) {
 ; AVX-LABEL: f8xi16_i32_NaN:
 ; AVX:       # %bb.0:
diff --git a/test/CodeGen/X86/broadcastm-lowering.ll b/test/CodeGen/X86/broadcastm-lowering.ll
index f8d670154882fed0b86df8b6f94253a3f5722922..986d313cb8d08367dfa246a6f2d9fbd5983c3c0b 100644
--- a/test/CodeGen/X86/broadcastm-lowering.ll
+++ b/test/CodeGen/X86/broadcastm-lowering.ll
@@ -43,15 +43,9 @@ define <4 x i32> @test_mm_epi32(<16 x i8> %a, <16 x i8> %b) {
 ; AVX512CD-LABEL: test_mm_epi32:
 ; AVX512CD:       # %bb.0: # %entry
 ; AVX512CD-NEXT:    vpcmpeqb %xmm1, %xmm0, %xmm0
-; AVX512CD-NEXT:    vpmovsxbd %xmm0, %zmm0
-; AVX512CD-NEXT:    vptestmd %zmm0, %zmm0, %k0
-; AVX512CD-NEXT:    kmovw %k0, %eax
-; AVX512CD-NEXT:    vpxor %xmm0, %xmm0, %xmm0
-; AVX512CD-NEXT:    vpinsrw $0, %eax, %xmm0, %xmm0
-; AVX512CD-NEXT:    vpinsrw $2, %eax, %xmm0, %xmm0
-; AVX512CD-NEXT:    vpinsrw $4, %eax, %xmm0, %xmm0
-; AVX512CD-NEXT:    vpinsrw $6, %eax, %xmm0, %xmm0
-; AVX512CD-NEXT:    vzeroupper
+; AVX512CD-NEXT:    vpmovmskb %xmm0, %eax
+; AVX512CD-NEXT:    vmovd %eax, %xmm0
+; AVX512CD-NEXT:    vpbroadcastd %xmm0, %xmm0
 ; AVX512CD-NEXT:    retq
 ;
 ; AVX512VLCDBW-LABEL: test_mm_epi32:
@@ -75,23 +69,11 @@ entry:
 }
 
 define <16 x i32> @test_mm512_epi32(<16 x i32> %a, <16 x i32> %b) {
-; AVX512CD-LABEL: test_mm512_epi32:
-; AVX512CD:       # %bb.0: # %entry
-; AVX512CD-NEXT:    vpcmpeqd %zmm1, %zmm0, %k0
-; AVX512CD-NEXT:    vpbroadcastmw2d %k0, %zmm0
-; AVX512CD-NEXT:    retq
-;
-; AVX512VLCDBW-LABEL: test_mm512_epi32:
-; AVX512VLCDBW:       # %bb.0: # %entry
-; AVX512VLCDBW-NEXT:    vpcmpeqd %zmm1, %zmm0, %k0
-; AVX512VLCDBW-NEXT:    vpbroadcastmw2d %k0, %zmm0
-; AVX512VLCDBW-NEXT:    retq
-;
-; X86-AVX512VLCDBW-LABEL: test_mm512_epi32:
-; X86-AVX512VLCDBW:       # %bb.0: # %entry
-; X86-AVX512VLCDBW-NEXT:    vpcmpeqd %zmm1, %zmm0, %k0
-; X86-AVX512VLCDBW-NEXT:    vpbroadcastmw2d %k0, %zmm0
-; X86-AVX512VLCDBW-NEXT:    retl
+; ALL-LABEL: test_mm512_epi32:
+; ALL:       # %bb.0: # %entry
+; ALL-NEXT:    vpcmpeqd %zmm1, %zmm0, %k0
+; ALL-NEXT:    vpbroadcastmw2d %k0, %zmm0
+; ALL-NEXT:    ret{{[l|q]}}
 entry:
   %0 = icmp eq <16 x i32> %a, %b
   %1 = bitcast <16 x i1> %0 to i16
diff --git a/test/CodeGen/X86/bss_pagealigned.ll b/test/CodeGen/X86/bss_pagealigned.ll
index 3635e67f22c0daa7e3bcdca8eed40c69bbf26dc7..9cc67d74ebc6ee0e3752ba8061a49f6fe9a4ceed 100644
--- a/test/CodeGen/X86/bss_pagealigned.ll
+++ b/test/CodeGen/X86/bss_pagealigned.ll
@@ -1,4 +1,4 @@
-; RUN: llc --code-model=kernel <%s -asm-verbose=0 | FileCheck %s
+; RUN: llc --code-model=kernel < %s -asm-verbose=0 | FileCheck %s
 ; PR4933
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128"
 target triple = "x86_64-unknown-linux-gnu"
diff --git a/test/CodeGen/X86/build-vector-128.ll b/test/CodeGen/X86/build-vector-128.ll
index 6c0c2d30c312d689d38f1d67a21db75b29b6a0b9..b80f6fa82d8b8bcad3425ef1606f5ee2616b8364 100644
--- a/test/CodeGen/X86/build-vector-128.ll
+++ b/test/CodeGen/X86/build-vector-128.ll
@@ -507,3 +507,54 @@ define <4 x i32> @test_buildvector_v4i32_splat_zext_i8(i8 %in) {
   %splat = shufflevector <4 x i32> %insert, <4 x i32> undef, <4 x i32> zeroinitializer
   ret <4 x i32> %splat
 }
+
+; PR37502 - https://bugs.llvm.org/show_bug.cgi?id=37502
+; Don't use a series of insertps when movddup will do.
+
+define <4 x float> @PR37502(float %x, float %y) {
+; SSE2-32-LABEL: PR37502:
+; SSE2-32:       # %bb.0:
+; SSE2-32-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero
+; SSE2-32-NEXT:    movlhps {{.*#+}} xmm0 = xmm0[0,0]
+; SSE2-32-NEXT:    retl
+;
+; SSE2-64-LABEL: PR37502:
+; SSE2-64:       # %bb.0:
+; SSE2-64-NEXT:    unpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE2-64-NEXT:    movlhps {{.*#+}} xmm0 = xmm0[0,0]
+; SSE2-64-NEXT:    retq
+;
+; SSE41-32-LABEL: PR37502:
+; SSE41-32:       # %bb.0:
+; SSE41-32-NEXT:    movddup {{.*#+}} xmm0 = mem[0,0]
+; SSE41-32-NEXT:    retl
+;
+; SSE41-64-LABEL: PR37502:
+; SSE41-64:       # %bb.0:
+; SSE41-64-NEXT:    insertps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[2,3]
+; SSE41-64-NEXT:    movddup {{.*#+}} xmm0 = xmm0[0,0]
+; SSE41-64-NEXT:    retq
+;
+; AVX-32-LABEL: PR37502:
+; AVX-32:       # %bb.0:
+; AVX-32-NEXT:    vmovddup {{.*#+}} xmm0 = mem[0,0]
+; AVX-32-NEXT:    retl
+;
+; AVX1-64-LABEL: PR37502:
+; AVX1-64:       # %bb.0:
+; AVX1-64-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[2,3]
+; AVX1-64-NEXT:    vmovddup {{.*#+}} xmm0 = xmm0[0,0]
+; AVX1-64-NEXT:    retq
+;
+; AVX2-64-LABEL: PR37502:
+; AVX2-64:       # %bb.0:
+; AVX2-64-NEXT:    vunpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; AVX2-64-NEXT:    vmovddup {{.*#+}} xmm0 = xmm0[0,0]
+; AVX2-64-NEXT:    retq
+  %i0 = insertelement <4 x float> undef, float %x, i32 0
+  %i1 = insertelement <4 x float> %i0, float %y, i32 1
+  %i2 = insertelement <4 x float> %i1, float %x, i32 2
+  %i3 = insertelement <4 x float> %i2, float %y, i32 3
+  ret <4 x float> %i3
+}
+
diff --git a/test/CodeGen/X86/buildvec-extract.ll b/test/CodeGen/X86/buildvec-extract.ll
new file mode 100644
index 0000000000000000000000000000000000000000..d1546677a69b2adc67e5db4c554ca219c62165e8
--- /dev/null
+++ b/test/CodeGen/X86/buildvec-extract.ll
@@ -0,0 +1,693 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-- -mattr=+sse2   | FileCheck %s --check-prefixes=ANY,SSE,SSE2
+; RUN: llc < %s -mtriple=x86_64-- -mattr=+sse4.1 | FileCheck %s --check-prefixes=ANY,SSE,SSE41
+; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx    | FileCheck %s --check-prefixes=ANY,AVX
+
+define <2 x i64> @extract0_i32_zext_insert0_i64_undef(<4 x i32> %x) {
+; SSE2-LABEL: extract0_i32_zext_insert0_i64_undef:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    xorps %xmm1, %xmm1
+; SSE2-NEXT:    unpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: extract0_i32_zext_insert0_i64_undef:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    pmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: extract0_i32_zext_insert0_i64_undef:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
+; AVX-NEXT:    retq
+  %e = extractelement <4 x i32> %x, i32 0
+  %z = zext i32 %e to i64
+  %r = insertelement <2 x i64> undef, i64 %z, i32 0
+  ret <2 x i64> %r
+}
+
+define <2 x i64> @extract0_i32_zext_insert0_i64_zero(<4 x i32> %x) {
+; SSE-LABEL: extract0_i32_zext_insert0_i64_zero:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movd %xmm0, %eax
+; SSE-NEXT:    movq %rax, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: extract0_i32_zext_insert0_i64_zero:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovd %xmm0, %eax
+; AVX-NEXT:    vmovq %rax, %xmm0
+; AVX-NEXT:    retq
+  %e = extractelement <4 x i32> %x, i32 0
+  %z = zext i32 %e to i64
+  %r = insertelement <2 x i64> zeroinitializer, i64 %z, i32 0
+  ret <2 x i64> %r
+}
+
+define <2 x i64> @extract1_i32_zext_insert0_i64_undef(<4 x i32> %x) {
+; SSE-LABEL: extract1_i32_zext_insert0_i64_undef:
+; SSE:       # %bb.0:
+; SSE-NEXT:    psrlq $32, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: extract1_i32_zext_insert0_i64_undef:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpsrlq $32, %xmm0, %xmm0
+; AVX-NEXT:    retq
+  %e = extractelement <4 x i32> %x, i32 1
+  %z = zext i32 %e to i64
+  %r = insertelement <2 x i64> undef, i64 %z, i32 0
+  ret <2 x i64> %r
+}
+
+define <2 x i64> @extract1_i32_zext_insert0_i64_zero(<4 x i32> %x) {
+; SSE2-LABEL: extract1_i32_zext_insert0_i64_zero:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE2-NEXT:    movd %xmm0, %eax
+; SSE2-NEXT:    movq %rax, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: extract1_i32_zext_insert0_i64_zero:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    extractps $1, %xmm0, %eax
+; SSE41-NEXT:    movq %rax, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: extract1_i32_zext_insert0_i64_zero:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vextractps $1, %xmm0, %eax
+; AVX-NEXT:    vmovq %rax, %xmm0
+; AVX-NEXT:    retq
+  %e = extractelement <4 x i32> %x, i32 1
+  %z = zext i32 %e to i64
+  %r = insertelement <2 x i64> zeroinitializer, i64 %z, i32 0
+  ret <2 x i64> %r
+}
+
+define <2 x i64> @extract2_i32_zext_insert0_i64_undef(<4 x i32> %x) {
+; SSE-LABEL: extract2_i32_zext_insert0_i64_undef:
+; SSE:       # %bb.0:
+; SSE-NEXT:    xorps %xmm1, %xmm1
+; SSE-NEXT:    unpckhps {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: extract2_i32_zext_insert0_i64_undef:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; AVX-NEXT:    vunpckhps {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; AVX-NEXT:    retq
+  %e = extractelement <4 x i32> %x, i32 2
+  %z = zext i32 %e to i64
+  %r = insertelement <2 x i64> undef, i64 %z, i32 0
+  ret <2 x i64> %r
+}
+
+define <2 x i64> @extract2_i32_zext_insert0_i64_zero(<4 x i32> %x) {
+; SSE2-LABEL: extract2_i32_zext_insert0_i64_zero:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE2-NEXT:    movd %xmm0, %eax
+; SSE2-NEXT:    movq %rax, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: extract2_i32_zext_insert0_i64_zero:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    extractps $2, %xmm0, %eax
+; SSE41-NEXT:    movq %rax, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: extract2_i32_zext_insert0_i64_zero:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vextractps $2, %xmm0, %eax
+; AVX-NEXT:    vmovq %rax, %xmm0
+; AVX-NEXT:    retq
+  %e = extractelement <4 x i32> %x, i32 2
+  %z = zext i32 %e to i64
+  %r = insertelement <2 x i64> zeroinitializer, i64 %z, i32 0
+  ret <2 x i64> %r
+}
+
+define <2 x i64> @extract3_i32_zext_insert0_i64_undef(<4 x i32> %x) {
+; SSE-LABEL: extract3_i32_zext_insert0_i64_undef:
+; SSE:       # %bb.0:
+; SSE-NEXT:    psrldq {{.*#+}} xmm0 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: extract3_i32_zext_insert0_i64_undef:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpsrldq {{.*#+}} xmm0 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX-NEXT:    retq
+  %e = extractelement <4 x i32> %x, i32 3
+  %z = zext i32 %e to i64
+  %r = insertelement <2 x i64> undef, i64 %z, i32 0
+  ret <2 x i64> %r
+}
+
+define <2 x i64> @extract3_i32_zext_insert0_i64_zero(<4 x i32> %x) {
+; SSE2-LABEL: extract3_i32_zext_insert0_i64_zero:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE2-NEXT:    movd %xmm0, %eax
+; SSE2-NEXT:    movq %rax, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: extract3_i32_zext_insert0_i64_zero:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    extractps $3, %xmm0, %eax
+; SSE41-NEXT:    movq %rax, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: extract3_i32_zext_insert0_i64_zero:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vextractps $3, %xmm0, %eax
+; AVX-NEXT:    vmovq %rax, %xmm0
+; AVX-NEXT:    retq
+  %e = extractelement <4 x i32> %x, i32 3
+  %z = zext i32 %e to i64
+  %r = insertelement <2 x i64> zeroinitializer, i64 %z, i32 0
+  ret <2 x i64> %r
+}
+
+define <2 x i64> @extract0_i32_zext_insert1_i64_undef(<4 x i32> %x) {
+; SSE2-LABEL: extract0_i32_zext_insert1_i64_undef:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,1,1]
+; SSE2-NEXT:    pxor %xmm1, %xmm1
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: extract0_i32_zext_insert1_i64_undef:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,1,0,1]
+; SSE41-NEXT:    pxor %xmm0, %xmm0
+; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm1[0,1,2,3,4,5],xmm0[6,7]
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: extract0_i32_zext_insert1_i64_undef:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,1,0,1]
+; AVX-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[3]
+; AVX-NEXT:    retq
+  %e = extractelement <4 x i32> %x, i32 0
+  %z = zext i32 %e to i64
+  %r = insertelement <2 x i64> undef, i64 %z, i32 1
+  ret <2 x i64> %r
+}
+
+define <2 x i64> @extract0_i32_zext_insert1_i64_zero(<4 x i32> %x) {
+; SSE-LABEL: extract0_i32_zext_insert1_i64_zero:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movd %xmm0, %eax
+; SSE-NEXT:    movq %rax, %xmm0
+; SSE-NEXT:    pslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1,2,3,4,5,6,7]
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: extract0_i32_zext_insert1_i64_zero:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovd %xmm0, %eax
+; AVX-NEXT:    vmovq %rax, %xmm0
+; AVX-NEXT:    vpslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1,2,3,4,5,6,7]
+; AVX-NEXT:    retq
+  %e = extractelement <4 x i32> %x, i32 0
+  %z = zext i32 %e to i64
+  %r = insertelement <2 x i64> zeroinitializer, i64 %z, i32 1
+  ret <2 x i64> %r
+}
+
+define <2 x i64> @extract1_i32_zext_insert1_i64_undef(<4 x i32> %x) {
+; SSE2-LABEL: extract1_i32_zext_insert1_i64_undef:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    xorps %xmm1, %xmm1
+; SSE2-NEXT:    unpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: extract1_i32_zext_insert1_i64_undef:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    pmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: extract1_i32_zext_insert1_i64_undef:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
+; AVX-NEXT:    retq
+  %e = extractelement <4 x i32> %x, i32 1
+  %z = zext i32 %e to i64
+  %r = insertelement <2 x i64> undef, i64 %z, i32 1
+  ret <2 x i64> %r
+}
+
+define <2 x i64> @extract1_i32_zext_insert1_i64_zero(<4 x i32> %x) {
+; SSE2-LABEL: extract1_i32_zext_insert1_i64_zero:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE2-NEXT:    movd %xmm0, %eax
+; SSE2-NEXT:    movq %rax, %xmm0
+; SSE2-NEXT:    pslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1,2,3,4,5,6,7]
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: extract1_i32_zext_insert1_i64_zero:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    extractps $1, %xmm0, %eax
+; SSE41-NEXT:    movq %rax, %xmm0
+; SSE41-NEXT:    pslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1,2,3,4,5,6,7]
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: extract1_i32_zext_insert1_i64_zero:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vextractps $1, %xmm0, %eax
+; AVX-NEXT:    vmovq %rax, %xmm0
+; AVX-NEXT:    vpslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1,2,3,4,5,6,7]
+; AVX-NEXT:    retq
+  %e = extractelement <4 x i32> %x, i32 1
+  %z = zext i32 %e to i64
+  %r = insertelement <2 x i64> zeroinitializer, i64 %z, i32 1
+  ret <2 x i64> %r
+}
+
+define <2 x i64> @extract2_i32_zext_insert1_i64_undef(<4 x i32> %x) {
+; SSE2-LABEL: extract2_i32_zext_insert1_i64_undef:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    andps {{.*}}(%rip), %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: extract2_i32_zext_insert1_i64_undef:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    xorps %xmm1, %xmm1
+; SSE41-NEXT:    blendps {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[3]
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: extract2_i32_zext_insert1_i64_undef:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; AVX-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[3]
+; AVX-NEXT:    retq
+  %e = extractelement <4 x i32> %x, i32 2
+  %z = zext i32 %e to i64
+  %r = insertelement <2 x i64> undef, i64 %z, i32 1
+  ret <2 x i64> %r
+}
+
+define <2 x i64> @extract2_i32_zext_insert1_i64_zero(<4 x i32> %x) {
+; SSE2-LABEL: extract2_i32_zext_insert1_i64_zero:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE2-NEXT:    movd %xmm0, %eax
+; SSE2-NEXT:    movq %rax, %xmm0
+; SSE2-NEXT:    pslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1,2,3,4,5,6,7]
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: extract2_i32_zext_insert1_i64_zero:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    extractps $2, %xmm0, %eax
+; SSE41-NEXT:    movq %rax, %xmm0
+; SSE41-NEXT:    pslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1,2,3,4,5,6,7]
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: extract2_i32_zext_insert1_i64_zero:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vextractps $2, %xmm0, %eax
+; AVX-NEXT:    vmovq %rax, %xmm0
+; AVX-NEXT:    vpslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1,2,3,4,5,6,7]
+; AVX-NEXT:    retq
+  %e = extractelement <4 x i32> %x, i32 2
+  %z = zext i32 %e to i64
+  %r = insertelement <2 x i64> zeroinitializer, i64 %z, i32 1
+  ret <2 x i64> %r
+}
+
+define <2 x i64> @extract3_i32_zext_insert1_i64_undef(<4 x i32> %x) {
+; SSE-LABEL: extract3_i32_zext_insert1_i64_undef:
+; SSE:       # %bb.0:
+; SSE-NEXT:    psrlq $32, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: extract3_i32_zext_insert1_i64_undef:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpsrlq $32, %xmm0, %xmm0
+; AVX-NEXT:    retq
+  %e = extractelement <4 x i32> %x, i32 3
+  %z = zext i32 %e to i64
+  %r = insertelement <2 x i64> undef, i64 %z, i32 1
+  ret <2 x i64> %r
+}
+
+define <2 x i64> @extract3_i32_zext_insert1_i64_zero(<4 x i32> %x) {
+; SSE2-LABEL: extract3_i32_zext_insert1_i64_zero:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE2-NEXT:    movd %xmm0, %eax
+; SSE2-NEXT:    movq %rax, %xmm0
+; SSE2-NEXT:    pslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1,2,3,4,5,6,7]
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: extract3_i32_zext_insert1_i64_zero:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    extractps $3, %xmm0, %eax
+; SSE41-NEXT:    movq %rax, %xmm0
+; SSE41-NEXT:    pslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1,2,3,4,5,6,7]
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: extract3_i32_zext_insert1_i64_zero:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vextractps $3, %xmm0, %eax
+; AVX-NEXT:    vmovq %rax, %xmm0
+; AVX-NEXT:    vpslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1,2,3,4,5,6,7]
+; AVX-NEXT:    retq
+  %e = extractelement <4 x i32> %x, i32 3
+  %z = zext i32 %e to i64
+  %r = insertelement <2 x i64> zeroinitializer, i64 %z, i32 1
+  ret <2 x i64> %r
+}
+
+define <2 x i64> @extract0_i16_zext_insert0_i64_undef(<8 x i16> %x) {
+; SSE2-LABEL: extract0_i16_zext_insert0_i64_undef:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    pxor %xmm1, %xmm1
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: extract0_i16_zext_insert0_i64_undef:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    pmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: extract0_i16_zext_insert0_i64_undef:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
+; AVX-NEXT:    retq
+  %e = extractelement <8 x i16> %x, i32 0
+  %z = zext i16 %e to i64
+  %r = insertelement <2 x i64> undef, i64 %z, i32 0
+  ret <2 x i64> %r
+}
+
+define <2 x i64> @extract0_i16_zext_insert0_i64_zero(<8 x i16> %x) {
+; SSE-LABEL: extract0_i16_zext_insert0_i64_zero:
+; SSE:       # %bb.0:
+; SSE-NEXT:    pextrw $0, %xmm0, %eax
+; SSE-NEXT:    movq %rax, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: extract0_i16_zext_insert0_i64_zero:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpextrw $0, %xmm0, %eax
+; AVX-NEXT:    vmovq %rax, %xmm0
+; AVX-NEXT:    retq
+  %e = extractelement <8 x i16> %x, i32 0
+  %z = zext i16 %e to i64
+  %r = insertelement <2 x i64> zeroinitializer, i64 %z, i32 0
+  ret <2 x i64> %r
+}
+
+define <2 x i64> @extract1_i16_zext_insert0_i64_undef(<8 x i16> %x) {
+; SSE-LABEL: extract1_i16_zext_insert0_i64_undef:
+; SSE:       # %bb.0:
+; SSE-NEXT:    pslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1,2,3]
+; SSE-NEXT:    psrldq {{.*#+}} xmm0 = xmm0[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: extract1_i16_zext_insert0_i64_undef:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1,2,3]
+; AVX-NEXT:    vpsrldq {{.*#+}} xmm0 = xmm0[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX-NEXT:    retq
+  %e = extractelement <8 x i16> %x, i32 1
+  %z = zext i16 %e to i64
+  %r = insertelement <2 x i64> undef, i64 %z, i32 0
+  ret <2 x i64> %r
+}
+
+define <2 x i64> @extract1_i16_zext_insert0_i64_zero(<8 x i16> %x) {
+; SSE-LABEL: extract1_i16_zext_insert0_i64_zero:
+; SSE:       # %bb.0:
+; SSE-NEXT:    pextrw $1, %xmm0, %eax
+; SSE-NEXT:    movq %rax, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: extract1_i16_zext_insert0_i64_zero:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpextrw $1, %xmm0, %eax
+; AVX-NEXT:    vmovq %rax, %xmm0
+; AVX-NEXT:    retq
+  %e = extractelement <8 x i16> %x, i32 1
+  %z = zext i16 %e to i64
+  %r = insertelement <2 x i64> zeroinitializer, i64 %z, i32 0
+  ret <2 x i64> %r
+}
+
+define <2 x i64> @extract2_i16_zext_insert0_i64_undef(<8 x i16> %x) {
+; SSE-LABEL: extract2_i16_zext_insert0_i64_undef:
+; SSE:       # %bb.0:
+; SSE-NEXT:    pslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1,2,3,4,5]
+; SSE-NEXT:    psrldq {{.*#+}} xmm0 = xmm0[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: extract2_i16_zext_insert0_i64_undef:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1,2,3,4,5]
+; AVX-NEXT:    vpsrldq {{.*#+}} xmm0 = xmm0[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX-NEXT:    retq
+  %e = extractelement <8 x i16> %x, i32 2
+  %z = zext i16 %e to i64
+  %r = insertelement <2 x i64> undef, i64 %z, i32 0
+  ret <2 x i64> %r
+}
+
+define <2 x i64> @extract2_i16_zext_insert0_i64_zero(<8 x i16> %x) {
+; SSE-LABEL: extract2_i16_zext_insert0_i64_zero:
+; SSE:       # %bb.0:
+; SSE-NEXT:    pextrw $2, %xmm0, %eax
+; SSE-NEXT:    movq %rax, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: extract2_i16_zext_insert0_i64_zero:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpextrw $2, %xmm0, %eax
+; AVX-NEXT:    vmovq %rax, %xmm0
+; AVX-NEXT:    retq
+  %e = extractelement <8 x i16> %x, i32 2
+  %z = zext i16 %e to i64
+  %r = insertelement <2 x i64> zeroinitializer, i64 %z, i32 0
+  ret <2 x i64> %r
+}
+
+define <2 x i64> @extract3_i16_zext_insert0_i64_undef(<8 x i16> %x) {
+; SSE-LABEL: extract3_i16_zext_insert0_i64_undef:
+; SSE:       # %bb.0:
+; SSE-NEXT:    psrlq $48, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: extract3_i16_zext_insert0_i64_undef:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpsrlq $48, %xmm0, %xmm0
+; AVX-NEXT:    retq
+  %e = extractelement <8 x i16> %x, i32 3
+  %z = zext i16 %e to i64
+  %r = insertelement <2 x i64> undef, i64 %z, i32 0
+  ret <2 x i64> %r
+}
+
+define <2 x i64> @extract3_i16_zext_insert0_i64_zero(<8 x i16> %x) {
+; SSE-LABEL: extract3_i16_zext_insert0_i64_zero:
+; SSE:       # %bb.0:
+; SSE-NEXT:    pextrw $3, %xmm0, %eax
+; SSE-NEXT:    movq %rax, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: extract3_i16_zext_insert0_i64_zero:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpextrw $3, %xmm0, %eax
+; AVX-NEXT:    vmovq %rax, %xmm0
+; AVX-NEXT:    retq
+  %e = extractelement <8 x i16> %x, i32 3
+  %z = zext i16 %e to i64
+  %r = insertelement <2 x i64> zeroinitializer, i64 %z, i32 0
+  ret <2 x i64> %r
+}
+
+define <2 x i64> @extract0_i16_zext_insert1_i64_undef(<8 x i16> %x) {
+; SSE2-LABEL: extract0_i16_zext_insert1_i64_undef:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    pslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1]
+; SSE2-NEXT:    psrldq {{.*#+}} xmm0 = xmm0[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; SSE2-NEXT:    pslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1,2,3,4,5,6,7]
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: extract0_i16_zext_insert1_i64_undef:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[0,1,0,1]
+; SSE41-NEXT:    pxor %xmm0, %xmm0
+; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm1[4],xmm0[5,6,7]
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: extract0_i16_zext_insert1_i64_undef:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,1,0,1]
+; AVX-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX-NEXT:    vpblendw {{.*#+}} xmm0 = xmm1[0,1,2,3],xmm0[4],xmm1[5,6,7]
+; AVX-NEXT:    retq
+  %e = extractelement <8 x i16> %x, i32 0
+  %z = zext i16 %e to i64
+  %r = insertelement <2 x i64> undef, i64 %z, i32 1
+  ret <2 x i64> %r
+}
+
+define <2 x i64> @extract0_i16_zext_insert1_i64_zero(<8 x i16> %x) {
+; SSE-LABEL: extract0_i16_zext_insert1_i64_zero:
+; SSE:       # %bb.0:
+; SSE-NEXT:    pextrw $0, %xmm0, %eax
+; SSE-NEXT:    movq %rax, %xmm0
+; SSE-NEXT:    pslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1,2,3,4,5,6,7]
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: extract0_i16_zext_insert1_i64_zero:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpextrw $0, %xmm0, %eax
+; AVX-NEXT:    vmovq %rax, %xmm0
+; AVX-NEXT:    vpslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1,2,3,4,5,6,7]
+; AVX-NEXT:    retq
+  %e = extractelement <8 x i16> %x, i32 0
+  %z = zext i16 %e to i64
+  %r = insertelement <2 x i64> zeroinitializer, i64 %z, i32 1
+  ret <2 x i64> %r
+}
+
+define <2 x i64> @extract1_i16_zext_insert1_i64_undef(<8 x i16> %x) {
+; SSE2-LABEL: extract1_i16_zext_insert1_i64_undef:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    pxor %xmm1, %xmm1
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: extract1_i16_zext_insert1_i64_undef:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    pmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: extract1_i16_zext_insert1_i64_undef:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
+; AVX-NEXT:    retq
+  %e = extractelement <8 x i16> %x, i32 1
+  %z = zext i16 %e to i64
+  %r = insertelement <2 x i64> undef, i64 %z, i32 1
+  ret <2 x i64> %r
+}
+
+define <2 x i64> @extract1_i16_zext_insert1_i64_zero(<8 x i16> %x) {
+; SSE-LABEL: extract1_i16_zext_insert1_i64_zero:
+; SSE:       # %bb.0:
+; SSE-NEXT:    pextrw $1, %xmm0, %eax
+; SSE-NEXT:    movq %rax, %xmm0
+; SSE-NEXT:    pslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1,2,3,4,5,6,7]
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: extract1_i16_zext_insert1_i64_zero:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpextrw $1, %xmm0, %eax
+; AVX-NEXT:    vmovq %rax, %xmm0
+; AVX-NEXT:    vpslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1,2,3,4,5,6,7]
+; AVX-NEXT:    retq
+  %e = extractelement <8 x i16> %x, i32 1
+  %z = zext i16 %e to i64
+  %r = insertelement <2 x i64> zeroinitializer, i64 %z, i32 1
+  ret <2 x i64> %r
+}
+
+define <2 x i64> @extract2_i16_zext_insert1_i64_undef(<8 x i16> %x) {
+; SSE2-LABEL: extract2_i16_zext_insert1_i64_undef:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    pslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1,2,3,4,5]
+; SSE2-NEXT:    psrldq {{.*#+}} xmm0 = xmm0[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; SSE2-NEXT:    pslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1,2,3,4,5,6,7]
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: extract2_i16_zext_insert1_i64_undef:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    pmovzxdq {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero
+; SSE41-NEXT:    pxor %xmm0, %xmm0
+; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm1[4],xmm0[5,6,7]
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: extract2_i16_zext_insert1_i64_undef:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
+; AVX-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX-NEXT:    vpblendw {{.*#+}} xmm0 = xmm1[0,1,2,3],xmm0[4],xmm1[5,6,7]
+; AVX-NEXT:    retq
+  %e = extractelement <8 x i16> %x, i32 2
+  %z = zext i16 %e to i64
+  %r = insertelement <2 x i64> undef, i64 %z, i32 1
+  ret <2 x i64> %r
+}
+
+define <2 x i64> @extract2_i16_zext_insert1_i64_zero(<8 x i16> %x) {
+; SSE-LABEL: extract2_i16_zext_insert1_i64_zero:
+; SSE:       # %bb.0:
+; SSE-NEXT:    pextrw $2, %xmm0, %eax
+; SSE-NEXT:    movq %rax, %xmm0
+; SSE-NEXT:    pslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1,2,3,4,5,6,7]
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: extract2_i16_zext_insert1_i64_zero:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpextrw $2, %xmm0, %eax
+; AVX-NEXT:    vmovq %rax, %xmm0
+; AVX-NEXT:    vpslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1,2,3,4,5,6,7]
+; AVX-NEXT:    retq
+  %e = extractelement <8 x i16> %x, i32 2
+  %z = zext i16 %e to i64
+  %r = insertelement <2 x i64> zeroinitializer, i64 %z, i32 1
+  ret <2 x i64> %r
+}
+
+define <2 x i64> @extract3_i16_zext_insert1_i64_undef(<8 x i16> %x) {
+; SSE2-LABEL: extract3_i16_zext_insert1_i64_undef:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    pslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1,2,3,4,5,6,7]
+; SSE2-NEXT:    psrldq {{.*#+}} xmm0 = xmm0[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; SSE2-NEXT:    pslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1,2,3,4,5,6,7]
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: extract3_i16_zext_insert1_i64_undef:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    pslldq {{.*#+}} xmm0 = zero,zero,xmm0[0,1,2,3,4,5,6,7,8,9,10,11,12,13]
+; SSE41-NEXT:    pxor %xmm1, %xmm1
+; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm1[0,1,2,3],xmm0[4],xmm1[5,6,7]
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: extract3_i16_zext_insert1_i64_undef:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpslldq {{.*#+}} xmm0 = zero,zero,xmm0[0,1,2,3,4,5,6,7,8,9,10,11,12,13]
+; AVX-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX-NEXT:    vpblendw {{.*#+}} xmm0 = xmm1[0,1,2,3],xmm0[4],xmm1[5,6,7]
+; AVX-NEXT:    retq
+  %e = extractelement <8 x i16> %x, i32 3
+  %z = zext i16 %e to i64
+  %r = insertelement <2 x i64> undef, i64 %z, i32 1
+  ret <2 x i64> %r
+}
+
+define <2 x i64> @extract3_i16_zext_insert1_i64_zero(<8 x i16> %x) {
+; SSE-LABEL: extract3_i16_zext_insert1_i64_zero:
+; SSE:       # %bb.0:
+; SSE-NEXT:    pextrw $3, %xmm0, %eax
+; SSE-NEXT:    movq %rax, %xmm0
+; SSE-NEXT:    pslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1,2,3,4,5,6,7]
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: extract3_i16_zext_insert1_i64_zero:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpextrw $3, %xmm0, %eax
+; AVX-NEXT:    vmovq %rax, %xmm0
+; AVX-NEXT:    vpslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1,2,3,4,5,6,7]
+; AVX-NEXT:    retq
+  %e = extractelement <8 x i16> %x, i32 3
+  %z = zext i16 %e to i64
+  %r = insertelement <2 x i64> zeroinitializer, i64 %z, i32 1
+  ret <2 x i64> %r
+}
+
diff --git a/test/CodeGen/X86/bypass-slow-division-tune.ll b/test/CodeGen/X86/bypass-slow-division-tune.ll
index 2439f46895202bffdedc473dc01146516dd9585f..a387d6873f3ab348fdeb232dc91dc93f7e1d93dc 100644
--- a/test/CodeGen/X86/bypass-slow-division-tune.ll
+++ b/test/CodeGen/X86/bypass-slow-division-tune.ll
@@ -1,32 +1,111 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; Check that a division is bypassed when appropriate only.
-; RUN: llc -mtriple=x86_64-unknown-linux-gnu -mcpu=atom       < %s | FileCheck -check-prefixes=ATOM,CHECK %s
-; RUN: llc -mtriple=x86_64-unknown-linux-gnu -mcpu=silvermont < %s | FileCheck -check-prefixes=REST,CHECK %s
-; RUN: llc -mtriple=x86_64-unknown-linux-gnu -mcpu=skylake    < %s | FileCheck -check-prefixes=REST,CHECK %s
+; RUN: llc -mtriple=x86_64-unknown-linux-gnu -mcpu=atom       < %s | FileCheck -check-prefixes=CHECK,ATOM %s
+; RUN: llc -mtriple=x86_64-unknown-linux-gnu -mcpu=silvermont < %s | FileCheck -check-prefixes=CHECK,REST,SLM %s
+; RUN: llc -mtriple=x86_64-unknown-linux-gnu -mcpu=skylake    < %s | FileCheck -check-prefixes=CHECK,REST,SKL %s
 ; RUN: llc -profile-summary-huge-working-set-size-threshold=1 -mtriple=x86_64-unknown-linux-gnu -mcpu=skylake    < %s | FileCheck -check-prefixes=HUGEWS %s
 
 ; Verify that div32 is bypassed only for Atoms.
 define i32 @div32(i32 %a, i32 %b) {
-entry:
 ; ATOM-LABEL: div32:
-; ATOM: orl   %{{.*}}, [[REG:%[a-z]+]]
-; ATOM: testl $-256, [[REG]]
-; ATOM: divb
+; ATOM:       # %bb.0: # %entry
+; ATOM-NEXT:    movl %edi, %eax
+; ATOM-NEXT:    orl %esi, %eax
+; ATOM-NEXT:    testl $-256, %eax
+; ATOM-NEXT:    je .LBB0_1
+; ATOM-NEXT:  # %bb.2:
+; ATOM-NEXT:    movl %edi, %eax
+; ATOM-NEXT:    cltd
+; ATOM-NEXT:    idivl %esi
+; ATOM-NEXT:    retq
+; ATOM-NEXT:  .LBB0_1:
+; ATOM-NEXT:    movzbl %dil, %eax
+; ATOM-NEXT:    # kill: def $eax killed $eax def $ax
+; ATOM-NEXT:    divb %sil
+; ATOM-NEXT:    movzbl %al, %eax
+; ATOM-NEXT:    retq
 ;
 ; REST-LABEL: div32:
-; REST-NOT: divb
+; REST:       # %bb.0: # %entry
+; REST-NEXT:    movl %edi, %eax
+; REST-NEXT:    cltd
+; REST-NEXT:    idivl %esi
+; REST-NEXT:    retq
 ;
+; HUGEWS-LABEL: div32:
+; HUGEWS:       # %bb.0: # %entry
+; HUGEWS-NEXT:    movl %edi, %eax
+; HUGEWS-NEXT:    cltd
+; HUGEWS-NEXT:    idivl %esi
+; HUGEWS-NEXT:    retq
+entry:
   %div = sdiv i32 %a, %b
   ret i32 %div
 }
 
 ; Verify that div64 is always bypassed.
 define i64 @div64(i64 %a, i64 %b) {
-entry:
-; CHECK-LABEL: div64:
-; CHECK:     orq     %{{.*}}, [[REG:%[a-z]+]]
-; CHECK:     shrq    $32, [[REG]]
-; CHECK:     divl
+; ATOM-LABEL: div64:
+; ATOM:       # %bb.0: # %entry
+; ATOM-NEXT:    movq %rdi, %rcx
+; ATOM-NEXT:    movq %rdi, %rax
+; ATOM-NEXT:    orq %rsi, %rcx
+; ATOM-NEXT:    shrq $32, %rcx
+; ATOM-NEXT:    je .LBB1_1
+; ATOM-NEXT:  # %bb.2:
+; ATOM-NEXT:    cqto
+; ATOM-NEXT:    idivq %rsi
+; ATOM-NEXT:    retq
+; ATOM-NEXT:  .LBB1_1:
+; ATOM-NEXT:    # kill: def $eax killed $eax killed $rax
+; ATOM-NEXT:    xorl %edx, %edx
+; ATOM-NEXT:    divl %esi
+; ATOM-NEXT:    # kill: def $eax killed $eax def $rax
+; ATOM-NEXT:    retq
 ;
+; SLM-LABEL: div64:
+; SLM:       # %bb.0: # %entry
+; SLM-NEXT:    movq %rdi, %rcx
+; SLM-NEXT:    movq %rdi, %rax
+; SLM-NEXT:    orq %rsi, %rcx
+; SLM-NEXT:    shrq $32, %rcx
+; SLM-NEXT:    je .LBB1_1
+; SLM-NEXT:  # %bb.2:
+; SLM-NEXT:    cqto
+; SLM-NEXT:    idivq %rsi
+; SLM-NEXT:    retq
+; SLM-NEXT:  .LBB1_1:
+; SLM-NEXT:    xorl %edx, %edx
+; SLM-NEXT:    # kill: def $eax killed $eax killed $rax
+; SLM-NEXT:    divl %esi
+; SLM-NEXT:    # kill: def $eax killed $eax def $rax
+; SLM-NEXT:    retq
+;
+; SKL-LABEL: div64:
+; SKL:       # %bb.0: # %entry
+; SKL-NEXT:    movq %rdi, %rax
+; SKL-NEXT:    movq %rdi, %rcx
+; SKL-NEXT:    orq %rsi, %rcx
+; SKL-NEXT:    shrq $32, %rcx
+; SKL-NEXT:    je .LBB1_1
+; SKL-NEXT:  # %bb.2:
+; SKL-NEXT:    cqto
+; SKL-NEXT:    idivq %rsi
+; SKL-NEXT:    retq
+; SKL-NEXT:  .LBB1_1:
+; SKL-NEXT:    # kill: def $eax killed $eax killed $rax
+; SKL-NEXT:    xorl %edx, %edx
+; SKL-NEXT:    divl %esi
+; SKL-NEXT:    # kill: def $eax killed $eax def $rax
+; SKL-NEXT:    retq
+;
+; HUGEWS-LABEL: div64:
+; HUGEWS:       # %bb.0: # %entry
+; HUGEWS-NEXT:    movq %rdi, %rax
+; HUGEWS-NEXT:    cqto
+; HUGEWS-NEXT:    idivq %rsi
+; HUGEWS-NEXT:    retq
+entry:
   %div = sdiv i64 %a, %b
   ret i64 %div
 }
@@ -36,32 +115,119 @@ entry:
 
 define i64 @div64_optsize(i64 %a, i64 %b) optsize {
 ; CHECK-LABEL: div64_optsize:
-; CHECK-NOT: divl
-; CHECK: ret
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %rdi, %rax
+; CHECK-NEXT:    cqto
+; CHECK-NEXT:    idivq %rsi
+; CHECK-NEXT:    retq
+;
+; HUGEWS-LABEL: div64_optsize:
+; HUGEWS:       # %bb.0:
+; HUGEWS-NEXT:    movq %rdi, %rax
+; HUGEWS-NEXT:    cqto
+; HUGEWS-NEXT:    idivq %rsi
+; HUGEWS-NEXT:    retq
   %div = sdiv i64 %a, %b
   ret i64 %div
 }
 
 define i64 @div64_hugews(i64 %a, i64 %b) {
+; ATOM-LABEL: div64_hugews:
+; ATOM:       # %bb.0:
+; ATOM-NEXT:    movq %rdi, %rcx
+; ATOM-NEXT:    movq %rdi, %rax
+; ATOM-NEXT:    orq %rsi, %rcx
+; ATOM-NEXT:    shrq $32, %rcx
+; ATOM-NEXT:    je .LBB3_1
+; ATOM-NEXT:  # %bb.2:
+; ATOM-NEXT:    cqto
+; ATOM-NEXT:    idivq %rsi
+; ATOM-NEXT:    retq
+; ATOM-NEXT:  .LBB3_1:
+; ATOM-NEXT:    # kill: def $eax killed $eax killed $rax
+; ATOM-NEXT:    xorl %edx, %edx
+; ATOM-NEXT:    divl %esi
+; ATOM-NEXT:    # kill: def $eax killed $eax def $rax
+; ATOM-NEXT:    retq
+;
+; SLM-LABEL: div64_hugews:
+; SLM:       # %bb.0:
+; SLM-NEXT:    movq %rdi, %rcx
+; SLM-NEXT:    movq %rdi, %rax
+; SLM-NEXT:    orq %rsi, %rcx
+; SLM-NEXT:    shrq $32, %rcx
+; SLM-NEXT:    je .LBB3_1
+; SLM-NEXT:  # %bb.2:
+; SLM-NEXT:    cqto
+; SLM-NEXT:    idivq %rsi
+; SLM-NEXT:    retq
+; SLM-NEXT:  .LBB3_1:
+; SLM-NEXT:    xorl %edx, %edx
+; SLM-NEXT:    # kill: def $eax killed $eax killed $rax
+; SLM-NEXT:    divl %esi
+; SLM-NEXT:    # kill: def $eax killed $eax def $rax
+; SLM-NEXT:    retq
+;
+; SKL-LABEL: div64_hugews:
+; SKL:       # %bb.0:
+; SKL-NEXT:    movq %rdi, %rax
+; SKL-NEXT:    movq %rdi, %rcx
+; SKL-NEXT:    orq %rsi, %rcx
+; SKL-NEXT:    shrq $32, %rcx
+; SKL-NEXT:    je .LBB3_1
+; SKL-NEXT:  # %bb.2:
+; SKL-NEXT:    cqto
+; SKL-NEXT:    idivq %rsi
+; SKL-NEXT:    retq
+; SKL-NEXT:  .LBB3_1:
+; SKL-NEXT:    # kill: def $eax killed $eax killed $rax
+; SKL-NEXT:    xorl %edx, %edx
+; SKL-NEXT:    divl %esi
+; SKL-NEXT:    # kill: def $eax killed $eax def $rax
+; SKL-NEXT:    retq
+;
 ; HUGEWS-LABEL: div64_hugews:
-; HUGEWS-NOT: divl
-; HUGEWS: ret
+; HUGEWS:       # %bb.0:
+; HUGEWS-NEXT:    movq %rdi, %rax
+; HUGEWS-NEXT:    cqto
+; HUGEWS-NEXT:    idivq %rsi
+; HUGEWS-NEXT:    retq
   %div = sdiv i64 %a, %b
   ret i64 %div
 }
 
 define i32 @div32_optsize(i32 %a, i32 %b) optsize {
 ; CHECK-LABEL: div32_optsize:
-; CHECK-NOT: divb
-; CHECK: ret
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl %edi, %eax
+; CHECK-NEXT:    cltd
+; CHECK-NEXT:    idivl %esi
+; CHECK-NEXT:    retq
+;
+; HUGEWS-LABEL: div32_optsize:
+; HUGEWS:       # %bb.0:
+; HUGEWS-NEXT:    movl %edi, %eax
+; HUGEWS-NEXT:    cltd
+; HUGEWS-NEXT:    idivl %esi
+; HUGEWS-NEXT:    retq
   %div = sdiv i32 %a, %b
   ret i32 %div
 }
 
 define i32 @div32_minsize(i32 %a, i32 %b) minsize {
 ; CHECK-LABEL: div32_minsize:
-; CHECK-NOT: divb
-; CHECK: ret
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl %edi, %eax
+; CHECK-NEXT:    cltd
+; CHECK-NEXT:    idivl %esi
+; CHECK-NEXT:    retq
+;
+; HUGEWS-LABEL: div32_minsize:
+; HUGEWS:       # %bb.0:
+; HUGEWS-NEXT:    movl %edi, %eax
+; HUGEWS-NEXT:    cltd
+; HUGEWS-NEXT:    idivl %esi
+; HUGEWS-NEXT:    retq
   %div = sdiv i32 %a, %b
   ret i32 %div
 }
diff --git a/test/CodeGen/X86/call-push.ll b/test/CodeGen/X86/call-push.ll
index e8afa1e77afa58451eb57d96353285c994a98465..aee77937e7b1c421b9a9d3891dc6a5ff0c63455f 100644
--- a/test/CodeGen/X86/call-push.ll
+++ b/test/CodeGen/X86/call-push.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -mtriple=i386-apple-darwin -disable-fp-elim -no-x86-call-frame-opt | FileCheck %s
+; RUN: llc < %s -mtriple=i386-apple-darwin -frame-pointer=all -no-x86-call-frame-opt | FileCheck %s
 
         %struct.decode_t = type { i8, i8, i8, i8, i16, i8, i8, %struct.range_t** }
         %struct.range_t = type { float, float, i32, i32, i32, [0 x i8] }
diff --git a/test/CodeGen/X86/callbr-asm-blockplacement.ll b/test/CodeGen/X86/callbr-asm-blockplacement.ll
new file mode 100644
index 0000000000000000000000000000000000000000..d0af12fa20e6443fd4fb29623e5567760683d5f1
--- /dev/null
+++ b/test/CodeGen/X86/callbr-asm-blockplacement.ll
@@ -0,0 +1,106 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu | FileCheck %s
+
+; This test asserted in MachineBlockPlacement during asm-goto bring up.
+
+%struct.wibble = type { %struct.pluto, i32, i8* }
+%struct.pluto = type { i32, i32, i32 }
+
+@global = external global [0 x %struct.wibble]
+
+define i32 @foo(i32 %arg, i32 (i8*)* %arg3) nounwind {
+; CHECK-LABEL: foo:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    pushq %r13
+; CHECK-NEXT:    pushq %r12
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    movabsq $-2305847407260205056, %rbx # imm = 0xDFFFFC0000000000
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    testb %al, %al
+; CHECK-NEXT:    jne .LBB0_5
+; CHECK-NEXT:  # %bb.1: # %bb5
+; CHECK-NEXT:    movq %rsi, %r14
+; CHECK-NEXT:    movslq %edi, %rbp
+; CHECK-NEXT:    leaq (,%rbp,8), %rax
+; CHECK-NEXT:    leaq global(%rax,%rax,2), %r15
+; CHECK-NEXT:    leaq global+4(%rax,%rax,2), %r12
+; CHECK-NEXT:    xorl %r13d, %r13d
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB0_2: # %bb8
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    callq bar
+; CHECK-NEXT:    movq %rax, %rbx
+; CHECK-NEXT:    movq %rax, %rdi
+; CHECK-NEXT:    callq *%r14
+; CHECK-NEXT:    movq %r15, %rdi
+; CHECK-NEXT:    callq hoge
+; CHECK-NEXT:    movq %r12, %rdi
+; CHECK-NEXT:    callq hoge
+; CHECK-NEXT:    testb %r13b, %r13b
+; CHECK-NEXT:    jne .LBB0_2
+; CHECK-NEXT:  # %bb.3: # %bb15
+; CHECK-NEXT:    leaq (%rbp,%rbp,2), %rax
+; CHECK-NEXT:    movq %rbx, global+16(,%rax,8)
+; CHECK-NEXT:    movabsq $-2305847407260205056, %rbx # imm = 0xDFFFFC0000000000
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:  .LBB0_4: # %bb17
+; CHECK-NEXT:    callq widget
+; CHECK-NEXT:  .Ltmp0: # Block address taken
+; CHECK-NEXT:  .LBB0_5: # %bb18
+; CHECK-NEXT:    movw $0, 14(%rbx)
+; CHECK-NEXT:    addq $8, %rsp
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    popq %r12
+; CHECK-NEXT:    popq %r13
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    retq
+bb:
+  %tmp = add i64 0, -2305847407260205056
+  %tmp4 = sext i32 %arg to i64
+  br i1 undef, label %bb18, label %bb5
+
+bb5:                                              ; preds = %bb
+  %tmp6 = getelementptr [0 x %struct.wibble], [0 x %struct.wibble]* @global, i64 0, i64 %tmp4, i32 0, i32 0
+  %tmp7 = getelementptr [0 x %struct.wibble], [0 x %struct.wibble]* @global, i64 0, i64 %tmp4, i32 0, i32 1
+  br label %bb8
+
+bb8:                                              ; preds = %bb8, %bb5
+  %tmp9 = call i8* @bar(i64 undef)
+  %tmp10 = call i32 %arg3(i8* nonnull %tmp9)
+  %tmp11 = ptrtoint i32* %tmp6 to i64
+  call void @hoge(i64 %tmp11)
+  %tmp12 = ptrtoint i32* %tmp7 to i64
+  %tmp13 = add i64 undef, -2305847407260205056
+  call void @hoge(i64 %tmp12)
+  %tmp14 = icmp eq i32 0, 0
+  br i1 %tmp14, label %bb15, label %bb8
+
+bb15:                                             ; preds = %bb8
+  %tmp16 = getelementptr [0 x %struct.wibble], [0 x %struct.wibble]* @global, i64 0, i64 %tmp4, i32 2
+  store i8* %tmp9, i8** %tmp16
+  callbr void asm sideeffect "", "X"(i8* blockaddress(@foo, %bb18))
+          to label %bb17 [label %bb18]
+
+bb17:                                             ; preds = %bb15
+  call void @widget()
+  br label %bb18
+
+bb18:                                             ; preds = %bb17, %bb15, %bb
+  %tmp19 = add i64 %tmp, 14
+  %tmp20 = inttoptr i64 %tmp19 to i16*
+  store i16 0, i16* %tmp20
+  ret i32 undef
+}
+
+declare i8* @bar(i64)
+
+declare void @widget()
+
+declare void @hoge(i64)
diff --git a/test/CodeGen/X86/callbr-asm-branch-folding.ll b/test/CodeGen/X86/callbr-asm-branch-folding.ll
new file mode 100644
index 0000000000000000000000000000000000000000..bdbff4ac89bb247b226ed4cc7c0869b05abcb962
--- /dev/null
+++ b/test/CodeGen/X86/callbr-asm-branch-folding.ll
@@ -0,0 +1,151 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu | FileCheck %s
+
+; This test hung in the BranchFolding pass during asm-goto bring up
+
+@e = global i32 0
+@j = global i32 0
+
+define void @n(i32* %o, i32 %p, i32 %u) nounwind {
+; CHECK-LABEL: n:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    pushq %r13
+; CHECK-NEXT:    pushq %r12
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    movl %edx, %ebx
+; CHECK-NEXT:    movl %esi, %r12d
+; CHECK-NEXT:    movq %rdi, %r15
+; CHECK-NEXT:    callq c
+; CHECK-NEXT:    movl %eax, %r13d
+; CHECK-NEXT:    movq %r15, %rdi
+; CHECK-NEXT:    callq l
+; CHECK-NEXT:    testl %eax, %eax
+; CHECK-NEXT:    je .LBB0_1
+; CHECK-NEXT:  .LBB0_10: # %cleanup
+; CHECK-NEXT:    addq $8, %rsp
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    popq %r12
+; CHECK-NEXT:    popq %r13
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    retq
+; CHECK-NEXT:  .LBB0_1: # %if.end
+; CHECK-NEXT:    movl %ebx, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT:    cmpl $0, {{.*}}(%rip)
+; CHECK-NEXT:    # implicit-def: $ebx
+; CHECK-NEXT:    # implicit-def: $r14d
+; CHECK-NEXT:    je .LBB0_4
+; CHECK-NEXT:  # %bb.2: # %if.then4
+; CHECK-NEXT:    movslq %r12d, %rdi
+; CHECK-NEXT:    callq m
+; CHECK-NEXT:    # implicit-def: $ebx
+; CHECK-NEXT:    # implicit-def: $ebp
+; CHECK-NEXT:  .LBB0_3: # %r
+; CHECK-NEXT:    callq c
+; CHECK-NEXT:    movl %ebp, %r14d
+; CHECK-NEXT:  .LBB0_4: # %if.end8
+; CHECK-NEXT:    movl %ebx, %edi
+; CHECK-NEXT:    callq i
+; CHECK-NEXT:    movl %eax, %ebp
+; CHECK-NEXT:    orl %r14d, %ebp
+; CHECK-NEXT:    testl %r13d, %r13d
+; CHECK-NEXT:    je .LBB0_6
+; CHECK-NEXT:  # %bb.5:
+; CHECK-NEXT:    andl $4, %ebx
+; CHECK-NEXT:    jmp .LBB0_3
+; CHECK-NEXT:  .LBB0_6: # %if.end12
+; CHECK-NEXT:    testl %ebp, %ebp
+; CHECK-NEXT:    je .LBB0_9
+; CHECK-NEXT:  # %bb.7: # %if.then14
+; CHECK-NEXT:    movl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Reload
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    jmp .LBB0_10
+; CHECK-NEXT:  .Ltmp0: # Block address taken
+; CHECK-NEXT:  .LBB0_8: # %if.then20.critedge
+; CHECK-NEXT:    movl {{.*}}(%rip), %edi
+; CHECK-NEXT:    movslq %eax, %rcx
+; CHECK-NEXT:    movl $1, %esi
+; CHECK-NEXT:    movq %r15, %rdx
+; CHECK-NEXT:    addq $8, %rsp
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    popq %r12
+; CHECK-NEXT:    popq %r13
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    jmp k # TAILCALL
+; CHECK-NEXT:  .LBB0_9: # %if.else
+; CHECK-NEXT:    incq 0
+; CHECK-NEXT:    jmp .LBB0_10
+entry:
+  %call = tail call i32 @c()
+  %call1 = tail call i32 @l(i32* %o)
+  %tobool = icmp eq i32 %call1, 0
+  br i1 %tobool, label %if.end, label %cleanup
+
+if.end:                                           ; preds = %entry
+  %0 = load i32, i32* @e
+  %tobool3 = icmp eq i32 %0, 0
+  br i1 %tobool3, label %if.end8, label %if.then4, !prof !0
+
+if.then4:                                         ; preds = %if.end
+  %conv5 = sext i32 %p to i64
+  %call6 = tail call i32 @m(i64 %conv5)
+  br label %r
+
+r:                                                ; preds = %if.end8, %if.then4
+  %flags.0 = phi i32 [ undef, %if.then4 ], [ %and, %if.end8 ]
+  %major.0 = phi i32 [ undef, %if.then4 ], [ %or, %if.end8 ]
+  %call7 = tail call i32 @c()
+  br label %if.end8
+
+if.end8:                                          ; preds = %r, %if.end
+  %flags.1 = phi i32 [ %flags.0, %r ], [ undef, %if.end ]
+  %major.1 = phi i32 [ %major.0, %r ], [ undef, %if.end ]
+  %call9 = tail call i32 @i(i32 %flags.1)
+  %or = or i32 %call9, %major.1
+  %and = and i32 %flags.1, 4
+  %tobool10 = icmp eq i32 %call, 0
+  br i1 %tobool10, label %if.end12, label %r
+
+if.end12:                                         ; preds = %if.end8
+  %tobool13 = icmp eq i32 %or, 0
+  br i1 %tobool13, label %if.else, label %if.then14
+
+if.then14:                                        ; preds = %if.end12
+  callbr void asm sideeffect "", "X,~{dirflag},~{fpsr},~{flags}"(i8* blockaddress(@n, %if.then20.critedge))
+          to label %cleanup [label %if.then20.critedge]
+
+if.then20.critedge:                               ; preds = %if.then14
+  %1 = load i32, i32* @j
+  %conv21 = sext i32 %u to i64
+  %call22 = tail call i32 @k(i32 %1, i64 1, i32* %o, i64 %conv21)
+  br label %cleanup
+
+if.else:                                          ; preds = %if.end12
+  %2 = load i64, i64* null
+  %inc = add i64 %2, 1
+  store i64 %inc, i64* null
+  br label %cleanup
+
+cleanup:                                          ; preds = %if.else, %if.then20.critedge, %if.then14, %entry
+  ret void
+}
+
+declare i32 @c()
+
+declare i32 @l(i32*)
+
+declare i32 @m(i64)
+
+declare i32 @i(i32)
+
+declare i32 @k(i32, i64, i32*, i64)
+
+!0 = !{!"branch_weights", i32 2000, i32 1}
diff --git a/test/CodeGen/X86/callbr-asm-destinations.ll b/test/CodeGen/X86/callbr-asm-destinations.ll
new file mode 100644
index 0000000000000000000000000000000000000000..91f0f2d1d656fb8735bdda0d33c163af1566ab9f
--- /dev/null
+++ b/test/CodeGen/X86/callbr-asm-destinations.ll
@@ -0,0 +1,15 @@
+; RUN: not llc -mtriple=i686-- < %s 2> %t
+; RUN: FileCheck %s < %t
+
+; CHECK: Duplicate callbr destination
+
+; A test for asm-goto duplicate labels limitation
+
+define i32 @test(i32 %a) {
+entry:
+  %0 = add i32 %a, 4
+  callbr void asm "xorl $0, $0; jmp ${1:l}", "r,X,~{dirflag},~{fpsr},~{flags}"(i32 %0, i8* blockaddress(@test, %fail)) to label %fail [label %fail]
+
+fail:
+  ret i32 1
+}
diff --git a/test/CodeGen/X86/callbr-asm-errors.ll b/test/CodeGen/X86/callbr-asm-errors.ll
new file mode 100644
index 0000000000000000000000000000000000000000..5569e3c15ac5c87336cbe2e585582cd47e5c0be5
--- /dev/null
+++ b/test/CodeGen/X86/callbr-asm-errors.ll
@@ -0,0 +1,18 @@
+; RUN: not llc -mtriple=i686-- < %s 2> %t
+; RUN: FileCheck %s < %t
+
+; CHECK: Duplicate callbr destination
+
+; A test for asm-goto duplicate labels limitation
+
+define i32 @test(i32 %a) {
+entry:
+  %0 = add i32 %a, 4
+  callbr void asm "xorl $0, $0; jmp ${1:l}", "r,X,X,~{dirflag},~{fpsr},~{flags}"(i32 %0, i8* blockaddress(@test, %fail), i8* blockaddress(@test, %fail)) to label %normal [label %fail, label %fail]
+
+normal:
+  ret i32 %0
+
+fail:
+  ret i32 1
+}
diff --git a/test/CodeGen/X86/callbr-asm-outputs.ll b/test/CodeGen/X86/callbr-asm-outputs.ll
new file mode 100644
index 0000000000000000000000000000000000000000..7595deba3364b4a332d68d4482622ae0396fd8e4
--- /dev/null
+++ b/test/CodeGen/X86/callbr-asm-outputs.ll
@@ -0,0 +1,18 @@
+; RUN: not llc -mtriple=i686-- < %s 2> %t
+; RUN: FileCheck %s < %t
+
+; CHECK: error: asm-goto outputs not supported
+
+; A test for asm-goto output prohibition
+
+define i32 @test(i32 %a) {
+entry:
+  %0 = add i32 %a, 4
+  %1 = callbr i32 asm "xorl $1, $1; jmp ${1:l}", "=&r,r,X,~{dirflag},~{fpsr},~{flags}"(i32 %0, i8* blockaddress(@test, %fail)) to label %normal [label %fail]
+
+normal:
+  ret i32 %1
+
+fail:
+  ret i32 1
+}
diff --git a/test/CodeGen/X86/callbr-asm.ll b/test/CodeGen/X86/callbr-asm.ll
new file mode 100644
index 0000000000000000000000000000000000000000..48a80ae167ba1b4833c42a845e4a1aa1d4b2ee29
--- /dev/null
+++ b/test/CodeGen/X86/callbr-asm.ll
@@ -0,0 +1,133 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-- -O3 | FileCheck %s
+
+; Tests for using callbr as an asm-goto wrapper
+
+; Test 1 - fallthrough label gets removed, but the fallthrough code that is
+; unreachable due to asm ending on a jmp is still left in.
+define i32 @test1(i32 %a) {
+; CHECK-LABEL: test1:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    addl $4, %eax
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    jmp .Ltmp00
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:  .LBB0_1: # %normal
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    retl
+; CHECK-NEXT:  .Ltmp0: # Block address taken
+; CHECK-NEXT:  .LBB0_2: # %fail
+; CHECK-NEXT:    movl $1, %eax
+; CHECK-NEXT:    retl
+entry:
+  %0 = add i32 %a, 4
+  callbr void asm "xorl $0, $0; jmp ${1:l}", "r,X,~{dirflag},~{fpsr},~{flags}"(i32 %0, i8* blockaddress(@test1, %fail)) to label %normal [label %fail]
+
+normal:
+  ret i32 0
+
+fail:
+  ret i32 1
+}
+
+; Test 2 - callbr terminates an unreachable block, function gets simplified
+; to a trivial zero return.
+define i32 @test2(i32 %a) {
+; CHECK-LABEL: test2:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    retl
+entry:
+  br label %normal
+
+unreachableasm:
+  %0 = add i32 %a, 4
+  callbr void asm sideeffect "xorl $0, $0; jmp ${1:l}", "r,X,~{dirflag},~{fpsr},~{flags}"(i32 %0, i8* blockaddress(@test2, %fail)) to label %normal [label %fail]
+
+normal:
+  ret i32 0
+
+fail:
+  ret i32 1
+}
+
+
+; Test 3 - asm-goto implements a loop. The loop gets recognized, but many loop
+; transforms fail due to canonicalization having callbr exceptions. Trivial
+; blocks at labels 1 and 3 also don't get simplified due to callbr.
+define dso_local i32 @test3(i32 %a) {
+; CHECK-LABEL: test3:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:  .Ltmp1: # Block address taken
+; CHECK-NEXT:  .LBB2_1: # %label01
+; CHECK-NEXT:    # =>This Loop Header: Depth=1
+; CHECK-NEXT:    # Child Loop BB2_2 Depth 2
+; CHECK-NEXT:    # Child Loop BB2_3 Depth 3
+; CHECK-NEXT:    # Child Loop BB2_4 Depth 4
+; CHECK-NEXT:  .Ltmp2: # Block address taken
+; CHECK-NEXT:  .LBB2_2: # %label02
+; CHECK-NEXT:    # Parent Loop BB2_1 Depth=1
+; CHECK-NEXT:    # => This Loop Header: Depth=2
+; CHECK-NEXT:    # Child Loop BB2_3 Depth 3
+; CHECK-NEXT:    # Child Loop BB2_4 Depth 4
+; CHECK-NEXT:    addl $4, {{[0-9]+}}(%esp)
+; CHECK-NEXT:  .Ltmp3: # Block address taken
+; CHECK-NEXT:  .LBB2_3: # %label03
+; CHECK-NEXT:    # Parent Loop BB2_1 Depth=1
+; CHECK-NEXT:    # Parent Loop BB2_2 Depth=2
+; CHECK-NEXT:    # => This Loop Header: Depth=3
+; CHECK-NEXT:    # Child Loop BB2_4 Depth 4
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .Ltmp4: # Block address taken
+; CHECK-NEXT:  .LBB2_4: # %label04
+; CHECK-NEXT:    # Parent Loop BB2_1 Depth=1
+; CHECK-NEXT:    # Parent Loop BB2_2 Depth=2
+; CHECK-NEXT:    # Parent Loop BB2_3 Depth=3
+; CHECK-NEXT:    # => This Inner Loop Header: Depth=4
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    jmp .Ltmp10
+; CHECK-NEXT:    jmp .Ltmp20
+; CHECK-NEXT:    jmp .Ltmp30
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:  .LBB2_5: # %normal0
+; CHECK-NEXT:    # in Loop: Header=BB2_4 Depth=4
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    jmp .Ltmp10
+; CHECK-NEXT:    jmp .Ltmp20
+; CHECK-NEXT:    jmp .Ltmp30
+; CHECK-NEXT:    jmp .Ltmp40
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:  .LBB2_6: # %normal1
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    retl
+entry:
+  %a.addr = alloca i32, align 4
+  store i32 %a, i32* %a.addr, align 4
+  br label %label01
+
+label01:                                          ; preds = %normal0, %label04, %entry
+  br label %label02
+
+label02:                                          ; preds = %normal0, %label04, %label01
+  %0 = load i32, i32* %a.addr, align 4
+  %add = add nsw i32 %0, 4
+  store i32 %add, i32* %a.addr, align 4
+  br label %label03
+
+label03:                                          ; preds = %normal0, %label04, %label02
+  br label %label04
+
+label04:                                          ; preds = %normal0, %label03
+  callbr void asm sideeffect "jmp ${0:l}; jmp ${1:l}; jmp ${2:l}", "X,X,X,~{dirflag},~{fpsr},~{flags}"(i8* blockaddress(@test3, %label01), i8* blockaddress(@test3, %label02), i8* blockaddress(@test3, %label03))
+          to label %normal0 [label %label01, label %label02, label %label03]
+
+normal0:                                          ; preds = %label04
+  callbr void asm sideeffect "jmp ${0:l}; jmp ${1:l}; jmp ${2:l}; jmp ${3:l}", "X,X,X,X,~{dirflag},~{fpsr},~{flags}"(i8* blockaddress(@test3, %label01), i8* blockaddress(@test3, %label02), i8* blockaddress(@test3, %label03), i8* blockaddress(@test3, %label04))
+          to label %normal1 [label %label01, label %label02, label %label03, label %label04]
+
+normal1:                                          ; preds = %normal0
+  %1 = load i32, i32* %a.addr, align 4
+  ret i32 %1
+}
diff --git a/test/CodeGen/X86/cgp-usubo.ll b/test/CodeGen/X86/cgp-usubo.ll
new file mode 100644
index 0000000000000000000000000000000000000000..39361d1a68fa6488f9e6e8bb6eefa9555680a573
--- /dev/null
+++ b/test/CodeGen/X86/cgp-usubo.ll
@@ -0,0 +1,205 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-- | FileCheck %s
+
+; CodeGenPrepare is expected to form overflow intrinsics to improve DAG/isel.
+
+define i1 @usubo_ult_i64(i64 %x, i64 %y, i64* %p) nounwind {
+; CHECK-LABEL: usubo_ult_i64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    subq %rsi, %rdi
+; CHECK-NEXT:    movq %rdi, (%rdx)
+; CHECK-NEXT:    setb %al
+; CHECK-NEXT:    retq
+  %s = sub i64 %x, %y
+  store i64 %s, i64* %p
+  %ov = icmp ult i64 %x, %y
+  ret i1 %ov
+}
+
+; Verify insertion point for single-BB. Toggle predicate.
+
+define i1 @usubo_ugt_i32(i32 %x, i32 %y, i32* %p) nounwind {
+; CHECK-LABEL: usubo_ugt_i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    cmpl %edi, %esi
+; CHECK-NEXT:    seta %al
+; CHECK-NEXT:    subl %esi, %edi
+; CHECK-NEXT:    movl %edi, (%rdx)
+; CHECK-NEXT:    retq
+  %ov = icmp ugt i32 %y, %x
+  %s = sub i32 %x, %y
+  store i32 %s, i32* %p
+  ret i1 %ov
+}
+
+; Constant operand should match.
+
+define i1 @usubo_ugt_constant_op0_i8(i8 %x, i8* %p) nounwind {
+; CHECK-LABEL: usubo_ugt_constant_op0_i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movb $42, %cl
+; CHECK-NEXT:    subb %dil, %cl
+; CHECK-NEXT:    cmpb $42, %dil
+; CHECK-NEXT:    seta %al
+; CHECK-NEXT:    movb %cl, (%rsi)
+; CHECK-NEXT:    retq
+  %s = sub i8 42, %x
+  %ov = icmp ugt i8 %x, 42
+  store i8 %s, i8* %p
+  ret i1 %ov
+}
+
+; Compare with constant operand 0 is canonicalized by commuting, but verify match for non-canonical form.
+
+define i1 @usubo_ult_constant_op0_i16(i16 %x, i16* %p) nounwind {
+; CHECK-LABEL: usubo_ult_constant_op0_i16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl $43, %ecx
+; CHECK-NEXT:    subl %edi, %ecx
+; CHECK-NEXT:    cmpw $43, %di
+; CHECK-NEXT:    seta %al
+; CHECK-NEXT:    movw %cx, (%rsi)
+; CHECK-NEXT:    retq
+  %s = sub i16 43, %x
+  %ov = icmp ult i16 43, %x
+  store i16 %s, i16* %p
+  ret i1 %ov
+}
+
+; Subtract with constant operand 1 is canonicalized to add.
+
+define i1 @usubo_ult_constant_op1_i16(i16 %x, i16* %p) nounwind {
+; CHECK-LABEL: usubo_ult_constant_op1_i16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl %edi, %ecx
+; CHECK-NEXT:    addl $-44, %ecx
+; CHECK-NEXT:    cmpw $44, %di
+; CHECK-NEXT:    setb %al
+; CHECK-NEXT:    movw %cx, (%rsi)
+; CHECK-NEXT:    retq
+  %s = add i16 %x, -44
+  %ov = icmp ult i16 %x, 44
+  store i16 %s, i16* %p
+  ret i1 %ov
+}
+
+define i1 @usubo_ugt_constant_op1_i8(i8 %x, i8* %p) nounwind {
+; CHECK-LABEL: usubo_ugt_constant_op1_i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    cmpb $45, %dil
+; CHECK-NEXT:    setb %al
+; CHECK-NEXT:    addb $-45, %dil
+; CHECK-NEXT:    movb %dil, (%rsi)
+; CHECK-NEXT:    retq
+  %ov = icmp ugt i8 45, %x
+  %s = add i8 %x, -45
+  store i8 %s, i8* %p
+  ret i1 %ov
+}
+
+; Special-case: subtract 1 changes the compare predicate and constant.
+
+define i1 @usubo_eq_constant1_op1_i32(i32 %x, i32* %p) nounwind {
+; CHECK-LABEL: usubo_eq_constant1_op1_i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    # kill: def $edi killed $edi def $rdi
+; CHECK-NEXT:    leal -1(%rdi), %ecx
+; CHECK-NEXT:    testl %edi, %edi
+; CHECK-NEXT:    sete %al
+; CHECK-NEXT:    movl %ecx, (%rsi)
+; CHECK-NEXT:    retq
+  %s = add i32 %x, -1
+  %ov = icmp eq i32 %x, 0
+  store i32 %s, i32* %p
+  ret i1 %ov
+}
+
+; Verify insertion point for multi-BB.
+
+declare void @call(i1)
+
+define i1 @usubo_ult_sub_dominates_i64(i64 %x, i64 %y, i64* %p, i1 %cond) nounwind {
+; CHECK-LABEL: usubo_ult_sub_dominates_i64:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    testb $1, %cl
+; CHECK-NEXT:    je .LBB7_2
+; CHECK-NEXT:  # %bb.1: # %t
+; CHECK-NEXT:    movq %rdi, %rax
+; CHECK-NEXT:    subq %rsi, %rax
+; CHECK-NEXT:    movq %rax, (%rdx)
+; CHECK-NEXT:    testb $1, %cl
+; CHECK-NEXT:    je .LBB7_2
+; CHECK-NEXT:  # %bb.3: # %end
+; CHECK-NEXT:    cmpq %rsi, %rdi
+; CHECK-NEXT:    setb %al
+; CHECK-NEXT:    retq
+; CHECK-NEXT:  .LBB7_2: # %f
+; CHECK-NEXT:    movl %ecx, %eax
+; CHECK-NEXT:    retq
+entry:
+  br i1 %cond, label %t, label %f
+
+t:
+  %s = sub i64 %x, %y
+  store i64 %s, i64* %p
+  br i1 %cond, label %end, label %f
+
+f:
+  ret i1 %cond
+
+end:
+  %ov = icmp ult i64 %x, %y
+  ret i1 %ov
+}
+
+define i1 @usubo_ult_cmp_dominates_i64(i64 %x, i64 %y, i64* %p, i1 %cond) nounwind {
+; CHECK-LABEL: usubo_ult_cmp_dominates_i64:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    movl %ecx, %ebp
+; CHECK-NEXT:    testb $1, %bpl
+; CHECK-NEXT:    je .LBB8_2
+; CHECK-NEXT:  # %bb.1: # %t
+; CHECK-NEXT:    movq %rdx, %r14
+; CHECK-NEXT:    movq %rsi, %r15
+; CHECK-NEXT:    movq %rdi, %rbx
+; CHECK-NEXT:    xorl %edi, %edi
+; CHECK-NEXT:    cmpq %rsi, %rbx
+; CHECK-NEXT:    setb %dil
+; CHECK-NEXT:    callq call
+; CHECK-NEXT:    subq %r15, %rbx
+; CHECK-NEXT:    jae .LBB8_2
+; CHECK-NEXT:  # %bb.4: # %end
+; CHECK-NEXT:    setb %al
+; CHECK-NEXT:    movq %rbx, (%r14)
+; CHECK-NEXT:    jmp .LBB8_3
+; CHECK-NEXT:  .LBB8_2: # %f
+; CHECK-NEXT:    movl %ebp, %eax
+; CHECK-NEXT:  .LBB8_3: # %f
+; CHECK-NEXT:    addq $8, %rsp
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    retq
+entry:
+  br i1 %cond, label %t, label %f
+
+t:
+  %ov = icmp ult i64 %x, %y
+  call void @call(i1 %ov)
+  br i1 %ov, label %end, label %f
+
+f:
+  ret i1 %cond
+
+end:
+  %s = sub i64 %x, %y
+  store i64 %s, i64* %p
+  ret i1 %ov
+}
+
diff --git a/test/CodeGen/X86/clflushopt-schedule.ll b/test/CodeGen/X86/clflushopt-schedule.ll
deleted file mode 100644
index 14b4551cabc26e6b074c747a358e99fdee9ef4e1..0000000000000000000000000000000000000000
--- a/test/CodeGen/X86/clflushopt-schedule.ll
+++ /dev/null
@@ -1,36 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=x86-64 -mattr=+clflushopt | FileCheck %s --check-prefix=CHECK --check-prefix=GENERIC
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=goldmont | FileCheck %s --check-prefix=CHECK --check-prefix=GLM
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=skylake | FileCheck %s --check-prefix=CHECK --check-prefix=SKYLAKE
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=skx | FileCheck %s --check-prefix=CHECK --check-prefix=SKX
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=znver1 | FileCheck %s --check-prefix=CHECK --check-prefix=ZNVER1
-
-define void @clflushopt(i8* %p) nounwind {
-; GENERIC-LABEL: clflushopt:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    clflushopt (%rdi) # sched: [5:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; GLM-LABEL: clflushopt:
-; GLM:       # %bb.0:
-; GLM-NEXT:    clflushopt (%rdi) # sched: [3:1.00]
-; GLM-NEXT:    retq # sched: [4:1.00]
-;
-; SKYLAKE-LABEL: clflushopt:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    clflushopt (%rdi) # sched: [2:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: clflushopt:
-; SKX:       # %bb.0:
-; SKX-NEXT:    clflushopt (%rdi) # sched: [2:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: clflushopt:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    clflushopt (%rdi) # sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  tail call void @llvm.x86.clflushopt(i8* %p)
-  ret void
-}
-declare void @llvm.x86.clflushopt(i8*) nounwind
diff --git a/test/CodeGen/X86/clwb-schedule.ll b/test/CodeGen/X86/clwb-schedule.ll
deleted file mode 100644
index 4d8e3e0c8aecc12f35efe15b0cc65f24e0a1ff6b..0000000000000000000000000000000000000000
--- a/test/CodeGen/X86/clwb-schedule.ll
+++ /dev/null
@@ -1,18 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=x86-64 -mattr=+clwb | FileCheck %s --check-prefix=GENERIC
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=skx | FileCheck %s --check-prefix=SKX
-
-define void @clwb(i8* %a0) nounwind {
-; GENERIC-LABEL: clwb:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    clwb (%rdi) # sched: [5:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKX-LABEL: clwb:
-; SKX:       # %bb.0:
-; SKX-NEXT:    clwb (%rdi) # sched: [5:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-  tail call void @llvm.x86.clwb(i8* %a0)
-  ret void
-}
-declare void @llvm.x86.clwb(i8*) nounwind
diff --git a/test/CodeGen/X86/clzero-schedule.ll b/test/CodeGen/X86/clzero-schedule.ll
deleted file mode 100644
index 692c261c6e24e72fa559b62889ac9a9cd5683280..0000000000000000000000000000000000000000
--- a/test/CodeGen/X86/clzero-schedule.ll
+++ /dev/null
@@ -1,20 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=x86-64 -mattr=+clzero | FileCheck %s --check-prefix=GENERIC
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=znver1 | FileCheck %s --check-prefix=ZNVER1
-
-define void @test_clzero(i8* %p) {
-; GENERIC-LABEL: test_clzero:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    leaq (%rdi), %rax # sched: [1:0.50]
-; GENERIC-NEXT:    clzero # sched: [100:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ZNVER1-LABEL: test_clzero:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    leaq (%rdi), %rax # sched: [1:0.25]
-; ZNVER1-NEXT:    clzero # sched: [100:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  tail call void @llvm.x86.clzero(i8* %p)
-  ret void
-}
-declare void @llvm.x86.clzero(i8*)
diff --git a/test/CodeGen/X86/cmov-schedule.ll b/test/CodeGen/X86/cmov-schedule.ll
deleted file mode 100644
index de3e8637a183b90c03aea3577a395de1acb9c2fd..0000000000000000000000000000000000000000
--- a/test/CodeGen/X86/cmov-schedule.ll
+++ /dev/null
@@ -1,2203 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=x86-64 | FileCheck %s --check-prefix=CHECK --check-prefix=GENERIC
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=atom | FileCheck %s --check-prefix=CHECK --check-prefix=ATOM
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=slm | FileCheck %s --check-prefix=CHECK --check-prefix=SLM
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=sandybridge | FileCheck %s --check-prefix=CHECK --check-prefix=SANDY
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=ivybridge | FileCheck %s --check-prefix=CHECK --check-prefix=SANDY
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=haswell | FileCheck %s --check-prefix=CHECK --check-prefix=HASWELL
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=broadwell | FileCheck %s --check-prefix=CHECK --check-prefix=BROADWELL
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=skylake | FileCheck %s --check-prefix=CHECK --check-prefix=SKYLAKE
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=skx | FileCheck %s --check-prefix=CHECK --check-prefix=SKX
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=bdver2 | FileCheck %s --check-prefix=CHECK --check-prefix=BDVER2
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=btver2 | FileCheck %s --check-prefix=CHECK --check-prefix=BTVER2
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=znver1 | FileCheck %s --check-prefix=CHECK --check-prefix=ZNVER1
-
-define void @test_cmov_16(i16 %a0, i16 %a1, i16 *%a2) optsize {
-; GENERIC-LABEL: test_cmov_16:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    cmovow %si, %di # sched: [2:0.67]
-; GENERIC-NEXT:    cmovnow %si, %di # sched: [2:0.67]
-; GENERIC-NEXT:    cmovbw %si, %di # sched: [2:0.67]
-; GENERIC-NEXT:    cmovbw %si, %di # sched: [2:0.67]
-; GENERIC-NEXT:    cmovbw %si, %di # sched: [2:0.67]
-; GENERIC-NEXT:    cmovaew %si, %di # sched: [2:0.67]
-; GENERIC-NEXT:    cmovaew %si, %di # sched: [2:0.67]
-; GENERIC-NEXT:    cmovaew %si, %di # sched: [2:0.67]
-; GENERIC-NEXT:    cmovew %si, %di # sched: [2:0.67]
-; GENERIC-NEXT:    cmovew %si, %di # sched: [2:0.67]
-; GENERIC-NEXT:    cmovnew %si, %di # sched: [2:0.67]
-; GENERIC-NEXT:    cmovnew %si, %di # sched: [2:0.67]
-; GENERIC-NEXT:    cmovbew %si, %di # sched: [3:1.00]
-; GENERIC-NEXT:    cmovbew %si, %di # sched: [3:1.00]
-; GENERIC-NEXT:    cmovaw %si, %di # sched: [3:1.00]
-; GENERIC-NEXT:    cmovaw %si, %di # sched: [3:1.00]
-; GENERIC-NEXT:    cmovsw %si, %di # sched: [2:0.67]
-; GENERIC-NEXT:    cmovnsw %si, %di # sched: [2:0.67]
-; GENERIC-NEXT:    cmovpw %si, %di # sched: [2:0.67]
-; GENERIC-NEXT:    cmovpw %si, %di # sched: [2:0.67]
-; GENERIC-NEXT:    cmovnpw %si, %di # sched: [2:0.67]
-; GENERIC-NEXT:    cmovnpw %si, %di # sched: [2:0.67]
-; GENERIC-NEXT:    cmovlw %si, %di # sched: [2:0.67]
-; GENERIC-NEXT:    cmovlw %si, %di # sched: [2:0.67]
-; GENERIC-NEXT:    cmovgew %si, %di # sched: [2:0.67]
-; GENERIC-NEXT:    cmovgew %si, %di # sched: [2:0.67]
-; GENERIC-NEXT:    cmovlew %si, %di # sched: [2:0.67]
-; GENERIC-NEXT:    cmovlew %si, %di # sched: [2:0.67]
-; GENERIC-NEXT:    cmovgw %si, %di # sched: [2:0.67]
-; GENERIC-NEXT:    cmovgw %si, %di # sched: [2:0.67]
-; GENERIC-NEXT:    cmovow (%rdx), %di # sched: [7:0.67]
-; GENERIC-NEXT:    cmovnow (%rdx), %di # sched: [7:0.67]
-; GENERIC-NEXT:    cmovbw (%rdx), %di # sched: [7:0.67]
-; GENERIC-NEXT:    cmovbw (%rdx), %di # sched: [7:0.67]
-; GENERIC-NEXT:    cmovbw (%rdx), %di # sched: [7:0.67]
-; GENERIC-NEXT:    cmovaew (%rdx), %di # sched: [7:0.67]
-; GENERIC-NEXT:    cmovaew (%rdx), %di # sched: [7:0.67]
-; GENERIC-NEXT:    cmovaew (%rdx), %di # sched: [7:0.67]
-; GENERIC-NEXT:    cmovew (%rdx), %di # sched: [7:0.67]
-; GENERIC-NEXT:    cmovew (%rdx), %di # sched: [7:0.67]
-; GENERIC-NEXT:    cmovnew (%rdx), %di # sched: [7:0.67]
-; GENERIC-NEXT:    cmovnew (%rdx), %di # sched: [7:0.67]
-; GENERIC-NEXT:    cmovbew (%rdx), %di # sched: [8:1.00]
-; GENERIC-NEXT:    cmovbew (%rdx), %di # sched: [8:1.00]
-; GENERIC-NEXT:    cmovaw (%rdx), %di # sched: [8:1.00]
-; GENERIC-NEXT:    cmovaw (%rdx), %di # sched: [8:1.00]
-; GENERIC-NEXT:    cmovsw (%rdx), %di # sched: [7:0.67]
-; GENERIC-NEXT:    cmovnsw (%rdx), %di # sched: [7:0.67]
-; GENERIC-NEXT:    cmovpw (%rdx), %di # sched: [7:0.67]
-; GENERIC-NEXT:    cmovpw (%rdx), %di # sched: [7:0.67]
-; GENERIC-NEXT:    cmovnpw (%rdx), %di # sched: [7:0.67]
-; GENERIC-NEXT:    cmovnpw (%rdx), %di # sched: [7:0.67]
-; GENERIC-NEXT:    cmovlw (%rdx), %di # sched: [7:0.67]
-; GENERIC-NEXT:    cmovlw (%rdx), %di # sched: [7:0.67]
-; GENERIC-NEXT:    cmovgew (%rdx), %di # sched: [7:0.67]
-; GENERIC-NEXT:    cmovgew (%rdx), %di # sched: [7:0.67]
-; GENERIC-NEXT:    cmovlew (%rdx), %di # sched: [7:0.67]
-; GENERIC-NEXT:    cmovlew (%rdx), %di # sched: [7:0.67]
-; GENERIC-NEXT:    cmovgw (%rdx), %di # sched: [7:0.67]
-; GENERIC-NEXT:    cmovgw (%rdx), %di # sched: [7:0.67]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_cmov_16:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    cmovow %si, %di # sched: [1:0.50]
-; ATOM-NEXT:    cmovnow %si, %di # sched: [1:0.50]
-; ATOM-NEXT:    cmovbw %si, %di # sched: [1:0.50]
-; ATOM-NEXT:    cmovbw %si, %di # sched: [1:0.50]
-; ATOM-NEXT:    cmovbw %si, %di # sched: [1:0.50]
-; ATOM-NEXT:    cmovaew %si, %di # sched: [1:0.50]
-; ATOM-NEXT:    cmovaew %si, %di # sched: [1:0.50]
-; ATOM-NEXT:    cmovaew %si, %di # sched: [1:0.50]
-; ATOM-NEXT:    cmovew %si, %di # sched: [1:0.50]
-; ATOM-NEXT:    cmovew %si, %di # sched: [1:0.50]
-; ATOM-NEXT:    cmovnew %si, %di # sched: [1:0.50]
-; ATOM-NEXT:    cmovnew %si, %di # sched: [1:0.50]
-; ATOM-NEXT:    cmovbew %si, %di # sched: [1:0.50]
-; ATOM-NEXT:    cmovbew %si, %di # sched: [1:0.50]
-; ATOM-NEXT:    cmovaw %si, %di # sched: [1:0.50]
-; ATOM-NEXT:    cmovaw %si, %di # sched: [1:0.50]
-; ATOM-NEXT:    cmovsw %si, %di # sched: [1:0.50]
-; ATOM-NEXT:    cmovnsw %si, %di # sched: [1:0.50]
-; ATOM-NEXT:    cmovpw %si, %di # sched: [1:0.50]
-; ATOM-NEXT:    cmovpw %si, %di # sched: [1:0.50]
-; ATOM-NEXT:    cmovnpw %si, %di # sched: [1:0.50]
-; ATOM-NEXT:    cmovnpw %si, %di # sched: [1:0.50]
-; ATOM-NEXT:    cmovlw %si, %di # sched: [1:0.50]
-; ATOM-NEXT:    cmovlw %si, %di # sched: [1:0.50]
-; ATOM-NEXT:    cmovgew %si, %di # sched: [1:0.50]
-; ATOM-NEXT:    cmovgew %si, %di # sched: [1:0.50]
-; ATOM-NEXT:    cmovlew %si, %di # sched: [1:0.50]
-; ATOM-NEXT:    cmovlew %si, %di # sched: [1:0.50]
-; ATOM-NEXT:    cmovgw %si, %di # sched: [1:0.50]
-; ATOM-NEXT:    cmovgw %si, %di # sched: [1:0.50]
-; ATOM-NEXT:    cmovow (%rdx), %di # sched: [1:1.00]
-; ATOM-NEXT:    cmovnow (%rdx), %di # sched: [1:1.00]
-; ATOM-NEXT:    cmovbw (%rdx), %di # sched: [1:1.00]
-; ATOM-NEXT:    cmovbw (%rdx), %di # sched: [1:1.00]
-; ATOM-NEXT:    cmovbw (%rdx), %di # sched: [1:1.00]
-; ATOM-NEXT:    cmovaew (%rdx), %di # sched: [1:1.00]
-; ATOM-NEXT:    cmovaew (%rdx), %di # sched: [1:1.00]
-; ATOM-NEXT:    cmovaew (%rdx), %di # sched: [1:1.00]
-; ATOM-NEXT:    cmovew (%rdx), %di # sched: [1:1.00]
-; ATOM-NEXT:    cmovew (%rdx), %di # sched: [1:1.00]
-; ATOM-NEXT:    cmovnew (%rdx), %di # sched: [1:1.00]
-; ATOM-NEXT:    cmovnew (%rdx), %di # sched: [1:1.00]
-; ATOM-NEXT:    cmovbew (%rdx), %di # sched: [1:1.00]
-; ATOM-NEXT:    cmovbew (%rdx), %di # sched: [1:1.00]
-; ATOM-NEXT:    cmovaw (%rdx), %di # sched: [1:1.00]
-; ATOM-NEXT:    cmovaw (%rdx), %di # sched: [1:1.00]
-; ATOM-NEXT:    cmovsw (%rdx), %di # sched: [1:1.00]
-; ATOM-NEXT:    cmovnsw (%rdx), %di # sched: [1:1.00]
-; ATOM-NEXT:    cmovpw (%rdx), %di # sched: [1:1.00]
-; ATOM-NEXT:    cmovpw (%rdx), %di # sched: [1:1.00]
-; ATOM-NEXT:    cmovnpw (%rdx), %di # sched: [1:1.00]
-; ATOM-NEXT:    cmovnpw (%rdx), %di # sched: [1:1.00]
-; ATOM-NEXT:    cmovlw (%rdx), %di # sched: [1:1.00]
-; ATOM-NEXT:    cmovlw (%rdx), %di # sched: [1:1.00]
-; ATOM-NEXT:    cmovgew (%rdx), %di # sched: [1:1.00]
-; ATOM-NEXT:    cmovgew (%rdx), %di # sched: [1:1.00]
-; ATOM-NEXT:    cmovlew (%rdx), %di # sched: [1:1.00]
-; ATOM-NEXT:    cmovlew (%rdx), %di # sched: [1:1.00]
-; ATOM-NEXT:    cmovgw (%rdx), %di # sched: [1:1.00]
-; ATOM-NEXT:    cmovgw (%rdx), %di # sched: [1:1.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_cmov_16:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    cmovow %si, %di # sched: [2:1.00]
-; SLM-NEXT:    cmovnow %si, %di # sched: [2:1.00]
-; SLM-NEXT:    cmovbw %si, %di # sched: [2:1.00]
-; SLM-NEXT:    cmovbw %si, %di # sched: [2:1.00]
-; SLM-NEXT:    cmovbw %si, %di # sched: [2:1.00]
-; SLM-NEXT:    cmovaew %si, %di # sched: [2:1.00]
-; SLM-NEXT:    cmovaew %si, %di # sched: [2:1.00]
-; SLM-NEXT:    cmovaew %si, %di # sched: [2:1.00]
-; SLM-NEXT:    cmovew %si, %di # sched: [2:1.00]
-; SLM-NEXT:    cmovew %si, %di # sched: [2:1.00]
-; SLM-NEXT:    cmovnew %si, %di # sched: [2:1.00]
-; SLM-NEXT:    cmovnew %si, %di # sched: [2:1.00]
-; SLM-NEXT:    cmovbew %si, %di # sched: [2:1.00]
-; SLM-NEXT:    cmovbew %si, %di # sched: [2:1.00]
-; SLM-NEXT:    cmovaw %si, %di # sched: [2:1.00]
-; SLM-NEXT:    cmovaw %si, %di # sched: [2:1.00]
-; SLM-NEXT:    cmovsw %si, %di # sched: [2:1.00]
-; SLM-NEXT:    cmovnsw %si, %di # sched: [2:1.00]
-; SLM-NEXT:    cmovpw %si, %di # sched: [2:1.00]
-; SLM-NEXT:    cmovpw %si, %di # sched: [2:1.00]
-; SLM-NEXT:    cmovnpw %si, %di # sched: [2:1.00]
-; SLM-NEXT:    cmovnpw %si, %di # sched: [2:1.00]
-; SLM-NEXT:    cmovlw %si, %di # sched: [2:1.00]
-; SLM-NEXT:    cmovlw %si, %di # sched: [2:1.00]
-; SLM-NEXT:    cmovgew %si, %di # sched: [2:1.00]
-; SLM-NEXT:    cmovgew %si, %di # sched: [2:1.00]
-; SLM-NEXT:    cmovlew %si, %di # sched: [2:1.00]
-; SLM-NEXT:    cmovlew %si, %di # sched: [2:1.00]
-; SLM-NEXT:    cmovgw %si, %di # sched: [2:1.00]
-; SLM-NEXT:    cmovgw %si, %di # sched: [2:1.00]
-; SLM-NEXT:    cmovow (%rdx), %di # sched: [5:1.00]
-; SLM-NEXT:    cmovnow (%rdx), %di # sched: [5:1.00]
-; SLM-NEXT:    cmovbw (%rdx), %di # sched: [5:1.00]
-; SLM-NEXT:    cmovbw (%rdx), %di # sched: [5:1.00]
-; SLM-NEXT:    cmovbw (%rdx), %di # sched: [5:1.00]
-; SLM-NEXT:    cmovaew (%rdx), %di # sched: [5:1.00]
-; SLM-NEXT:    cmovaew (%rdx), %di # sched: [5:1.00]
-; SLM-NEXT:    cmovaew (%rdx), %di # sched: [5:1.00]
-; SLM-NEXT:    cmovew (%rdx), %di # sched: [5:1.00]
-; SLM-NEXT:    cmovew (%rdx), %di # sched: [5:1.00]
-; SLM-NEXT:    cmovnew (%rdx), %di # sched: [5:1.00]
-; SLM-NEXT:    cmovnew (%rdx), %di # sched: [5:1.00]
-; SLM-NEXT:    cmovbew (%rdx), %di # sched: [5:1.00]
-; SLM-NEXT:    cmovbew (%rdx), %di # sched: [5:1.00]
-; SLM-NEXT:    cmovaw (%rdx), %di # sched: [5:1.00]
-; SLM-NEXT:    cmovaw (%rdx), %di # sched: [5:1.00]
-; SLM-NEXT:    cmovsw (%rdx), %di # sched: [5:1.00]
-; SLM-NEXT:    cmovnsw (%rdx), %di # sched: [5:1.00]
-; SLM-NEXT:    cmovpw (%rdx), %di # sched: [5:1.00]
-; SLM-NEXT:    cmovpw (%rdx), %di # sched: [5:1.00]
-; SLM-NEXT:    cmovnpw (%rdx), %di # sched: [5:1.00]
-; SLM-NEXT:    cmovnpw (%rdx), %di # sched: [5:1.00]
-; SLM-NEXT:    cmovlw (%rdx), %di # sched: [5:1.00]
-; SLM-NEXT:    cmovlw (%rdx), %di # sched: [5:1.00]
-; SLM-NEXT:    cmovgew (%rdx), %di # sched: [5:1.00]
-; SLM-NEXT:    cmovgew (%rdx), %di # sched: [5:1.00]
-; SLM-NEXT:    cmovlew (%rdx), %di # sched: [5:1.00]
-; SLM-NEXT:    cmovlew (%rdx), %di # sched: [5:1.00]
-; SLM-NEXT:    cmovgw (%rdx), %di # sched: [5:1.00]
-; SLM-NEXT:    cmovgw (%rdx), %di # sched: [5:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_cmov_16:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    cmovow %si, %di # sched: [2:0.67]
-; SANDY-NEXT:    cmovnow %si, %di # sched: [2:0.67]
-; SANDY-NEXT:    cmovbw %si, %di # sched: [2:0.67]
-; SANDY-NEXT:    cmovbw %si, %di # sched: [2:0.67]
-; SANDY-NEXT:    cmovbw %si, %di # sched: [2:0.67]
-; SANDY-NEXT:    cmovaew %si, %di # sched: [2:0.67]
-; SANDY-NEXT:    cmovaew %si, %di # sched: [2:0.67]
-; SANDY-NEXT:    cmovaew %si, %di # sched: [2:0.67]
-; SANDY-NEXT:    cmovew %si, %di # sched: [2:0.67]
-; SANDY-NEXT:    cmovew %si, %di # sched: [2:0.67]
-; SANDY-NEXT:    cmovnew %si, %di # sched: [2:0.67]
-; SANDY-NEXT:    cmovnew %si, %di # sched: [2:0.67]
-; SANDY-NEXT:    cmovbew %si, %di # sched: [3:1.00]
-; SANDY-NEXT:    cmovbew %si, %di # sched: [3:1.00]
-; SANDY-NEXT:    cmovaw %si, %di # sched: [3:1.00]
-; SANDY-NEXT:    cmovaw %si, %di # sched: [3:1.00]
-; SANDY-NEXT:    cmovsw %si, %di # sched: [2:0.67]
-; SANDY-NEXT:    cmovnsw %si, %di # sched: [2:0.67]
-; SANDY-NEXT:    cmovpw %si, %di # sched: [2:0.67]
-; SANDY-NEXT:    cmovpw %si, %di # sched: [2:0.67]
-; SANDY-NEXT:    cmovnpw %si, %di # sched: [2:0.67]
-; SANDY-NEXT:    cmovnpw %si, %di # sched: [2:0.67]
-; SANDY-NEXT:    cmovlw %si, %di # sched: [2:0.67]
-; SANDY-NEXT:    cmovlw %si, %di # sched: [2:0.67]
-; SANDY-NEXT:    cmovgew %si, %di # sched: [2:0.67]
-; SANDY-NEXT:    cmovgew %si, %di # sched: [2:0.67]
-; SANDY-NEXT:    cmovlew %si, %di # sched: [2:0.67]
-; SANDY-NEXT:    cmovlew %si, %di # sched: [2:0.67]
-; SANDY-NEXT:    cmovgw %si, %di # sched: [2:0.67]
-; SANDY-NEXT:    cmovgw %si, %di # sched: [2:0.67]
-; SANDY-NEXT:    cmovow (%rdx), %di # sched: [7:0.67]
-; SANDY-NEXT:    cmovnow (%rdx), %di # sched: [7:0.67]
-; SANDY-NEXT:    cmovbw (%rdx), %di # sched: [7:0.67]
-; SANDY-NEXT:    cmovbw (%rdx), %di # sched: [7:0.67]
-; SANDY-NEXT:    cmovbw (%rdx), %di # sched: [7:0.67]
-; SANDY-NEXT:    cmovaew (%rdx), %di # sched: [7:0.67]
-; SANDY-NEXT:    cmovaew (%rdx), %di # sched: [7:0.67]
-; SANDY-NEXT:    cmovaew (%rdx), %di # sched: [7:0.67]
-; SANDY-NEXT:    cmovew (%rdx), %di # sched: [7:0.67]
-; SANDY-NEXT:    cmovew (%rdx), %di # sched: [7:0.67]
-; SANDY-NEXT:    cmovnew (%rdx), %di # sched: [7:0.67]
-; SANDY-NEXT:    cmovnew (%rdx), %di # sched: [7:0.67]
-; SANDY-NEXT:    cmovbew (%rdx), %di # sched: [8:1.00]
-; SANDY-NEXT:    cmovbew (%rdx), %di # sched: [8:1.00]
-; SANDY-NEXT:    cmovaw (%rdx), %di # sched: [8:1.00]
-; SANDY-NEXT:    cmovaw (%rdx), %di # sched: [8:1.00]
-; SANDY-NEXT:    cmovsw (%rdx), %di # sched: [7:0.67]
-; SANDY-NEXT:    cmovnsw (%rdx), %di # sched: [7:0.67]
-; SANDY-NEXT:    cmovpw (%rdx), %di # sched: [7:0.67]
-; SANDY-NEXT:    cmovpw (%rdx), %di # sched: [7:0.67]
-; SANDY-NEXT:    cmovnpw (%rdx), %di # sched: [7:0.67]
-; SANDY-NEXT:    cmovnpw (%rdx), %di # sched: [7:0.67]
-; SANDY-NEXT:    cmovlw (%rdx), %di # sched: [7:0.67]
-; SANDY-NEXT:    cmovlw (%rdx), %di # sched: [7:0.67]
-; SANDY-NEXT:    cmovgew (%rdx), %di # sched: [7:0.67]
-; SANDY-NEXT:    cmovgew (%rdx), %di # sched: [7:0.67]
-; SANDY-NEXT:    cmovlew (%rdx), %di # sched: [7:0.67]
-; SANDY-NEXT:    cmovlew (%rdx), %di # sched: [7:0.67]
-; SANDY-NEXT:    cmovgw (%rdx), %di # sched: [7:0.67]
-; SANDY-NEXT:    cmovgw (%rdx), %di # sched: [7:0.67]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_cmov_16:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    cmovow %si, %di # sched: [2:0.50]
-; HASWELL-NEXT:    cmovnow %si, %di # sched: [2:0.50]
-; HASWELL-NEXT:    cmovbw %si, %di # sched: [2:0.50]
-; HASWELL-NEXT:    cmovbw %si, %di # sched: [2:0.50]
-; HASWELL-NEXT:    cmovbw %si, %di # sched: [2:0.50]
-; HASWELL-NEXT:    cmovaew %si, %di # sched: [2:0.50]
-; HASWELL-NEXT:    cmovaew %si, %di # sched: [2:0.50]
-; HASWELL-NEXT:    cmovaew %si, %di # sched: [2:0.50]
-; HASWELL-NEXT:    cmovew %si, %di # sched: [2:0.50]
-; HASWELL-NEXT:    cmovew %si, %di # sched: [2:0.50]
-; HASWELL-NEXT:    cmovnew %si, %di # sched: [2:0.50]
-; HASWELL-NEXT:    cmovnew %si, %di # sched: [2:0.50]
-; HASWELL-NEXT:    cmovbew %si, %di # sched: [3:0.75]
-; HASWELL-NEXT:    cmovbew %si, %di # sched: [3:0.75]
-; HASWELL-NEXT:    cmovaw %si, %di # sched: [3:0.75]
-; HASWELL-NEXT:    cmovaw %si, %di # sched: [3:0.75]
-; HASWELL-NEXT:    cmovsw %si, %di # sched: [2:0.50]
-; HASWELL-NEXT:    cmovnsw %si, %di # sched: [2:0.50]
-; HASWELL-NEXT:    cmovpw %si, %di # sched: [2:0.50]
-; HASWELL-NEXT:    cmovpw %si, %di # sched: [2:0.50]
-; HASWELL-NEXT:    cmovnpw %si, %di # sched: [2:0.50]
-; HASWELL-NEXT:    cmovnpw %si, %di # sched: [2:0.50]
-; HASWELL-NEXT:    cmovlw %si, %di # sched: [2:0.50]
-; HASWELL-NEXT:    cmovlw %si, %di # sched: [2:0.50]
-; HASWELL-NEXT:    cmovgew %si, %di # sched: [2:0.50]
-; HASWELL-NEXT:    cmovgew %si, %di # sched: [2:0.50]
-; HASWELL-NEXT:    cmovlew %si, %di # sched: [2:0.50]
-; HASWELL-NEXT:    cmovlew %si, %di # sched: [2:0.50]
-; HASWELL-NEXT:    cmovgw %si, %di # sched: [2:0.50]
-; HASWELL-NEXT:    cmovgw %si, %di # sched: [2:0.50]
-; HASWELL-NEXT:    cmovow (%rdx), %di # sched: [7:0.50]
-; HASWELL-NEXT:    cmovnow (%rdx), %di # sched: [7:0.50]
-; HASWELL-NEXT:    cmovbw (%rdx), %di # sched: [7:0.50]
-; HASWELL-NEXT:    cmovbw (%rdx), %di # sched: [7:0.50]
-; HASWELL-NEXT:    cmovbw (%rdx), %di # sched: [7:0.50]
-; HASWELL-NEXT:    cmovaew (%rdx), %di # sched: [7:0.50]
-; HASWELL-NEXT:    cmovaew (%rdx), %di # sched: [7:0.50]
-; HASWELL-NEXT:    cmovaew (%rdx), %di # sched: [7:0.50]
-; HASWELL-NEXT:    cmovew (%rdx), %di # sched: [7:0.50]
-; HASWELL-NEXT:    cmovew (%rdx), %di # sched: [7:0.50]
-; HASWELL-NEXT:    cmovnew (%rdx), %di # sched: [7:0.50]
-; HASWELL-NEXT:    cmovnew (%rdx), %di # sched: [7:0.50]
-; HASWELL-NEXT:    cmovbew (%rdx), %di # sched: [8:0.75]
-; HASWELL-NEXT:    cmovbew (%rdx), %di # sched: [8:0.75]
-; HASWELL-NEXT:    cmovaw (%rdx), %di # sched: [8:0.75]
-; HASWELL-NEXT:    cmovaw (%rdx), %di # sched: [8:0.75]
-; HASWELL-NEXT:    cmovsw (%rdx), %di # sched: [7:0.50]
-; HASWELL-NEXT:    cmovnsw (%rdx), %di # sched: [7:0.50]
-; HASWELL-NEXT:    cmovpw (%rdx), %di # sched: [7:0.50]
-; HASWELL-NEXT:    cmovpw (%rdx), %di # sched: [7:0.50]
-; HASWELL-NEXT:    cmovnpw (%rdx), %di # sched: [7:0.50]
-; HASWELL-NEXT:    cmovnpw (%rdx), %di # sched: [7:0.50]
-; HASWELL-NEXT:    cmovlw (%rdx), %di # sched: [7:0.50]
-; HASWELL-NEXT:    cmovlw (%rdx), %di # sched: [7:0.50]
-; HASWELL-NEXT:    cmovgew (%rdx), %di # sched: [7:0.50]
-; HASWELL-NEXT:    cmovgew (%rdx), %di # sched: [7:0.50]
-; HASWELL-NEXT:    cmovlew (%rdx), %di # sched: [7:0.50]
-; HASWELL-NEXT:    cmovlew (%rdx), %di # sched: [7:0.50]
-; HASWELL-NEXT:    cmovgw (%rdx), %di # sched: [7:0.50]
-; HASWELL-NEXT:    cmovgw (%rdx), %di # sched: [7:0.50]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_cmov_16:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    cmovow %si, %di # sched: [1:0.50]
-; BROADWELL-NEXT:    cmovnow %si, %di # sched: [1:0.50]
-; BROADWELL-NEXT:    cmovbw %si, %di # sched: [1:0.50]
-; BROADWELL-NEXT:    cmovbw %si, %di # sched: [1:0.50]
-; BROADWELL-NEXT:    cmovbw %si, %di # sched: [1:0.50]
-; BROADWELL-NEXT:    cmovaew %si, %di # sched: [1:0.50]
-; BROADWELL-NEXT:    cmovaew %si, %di # sched: [1:0.50]
-; BROADWELL-NEXT:    cmovaew %si, %di # sched: [1:0.50]
-; BROADWELL-NEXT:    cmovew %si, %di # sched: [1:0.50]
-; BROADWELL-NEXT:    cmovew %si, %di # sched: [1:0.50]
-; BROADWELL-NEXT:    cmovnew %si, %di # sched: [1:0.50]
-; BROADWELL-NEXT:    cmovnew %si, %di # sched: [1:0.50]
-; BROADWELL-NEXT:    cmovbew %si, %di # sched: [2:0.50]
-; BROADWELL-NEXT:    cmovbew %si, %di # sched: [2:0.50]
-; BROADWELL-NEXT:    cmovaw %si, %di # sched: [2:0.50]
-; BROADWELL-NEXT:    cmovaw %si, %di # sched: [2:0.50]
-; BROADWELL-NEXT:    cmovsw %si, %di # sched: [1:0.50]
-; BROADWELL-NEXT:    cmovnsw %si, %di # sched: [1:0.50]
-; BROADWELL-NEXT:    cmovpw %si, %di # sched: [1:0.50]
-; BROADWELL-NEXT:    cmovpw %si, %di # sched: [1:0.50]
-; BROADWELL-NEXT:    cmovnpw %si, %di # sched: [1:0.50]
-; BROADWELL-NEXT:    cmovnpw %si, %di # sched: [1:0.50]
-; BROADWELL-NEXT:    cmovlw %si, %di # sched: [1:0.50]
-; BROADWELL-NEXT:    cmovlw %si, %di # sched: [1:0.50]
-; BROADWELL-NEXT:    cmovgew %si, %di # sched: [1:0.50]
-; BROADWELL-NEXT:    cmovgew %si, %di # sched: [1:0.50]
-; BROADWELL-NEXT:    cmovlew %si, %di # sched: [1:0.50]
-; BROADWELL-NEXT:    cmovlew %si, %di # sched: [1:0.50]
-; BROADWELL-NEXT:    cmovgw %si, %di # sched: [1:0.50]
-; BROADWELL-NEXT:    cmovgw %si, %di # sched: [1:0.50]
-; BROADWELL-NEXT:    cmovow (%rdx), %di # sched: [6:0.50]
-; BROADWELL-NEXT:    cmovnow (%rdx), %di # sched: [6:0.50]
-; BROADWELL-NEXT:    cmovbw (%rdx), %di # sched: [6:0.50]
-; BROADWELL-NEXT:    cmovbw (%rdx), %di # sched: [6:0.50]
-; BROADWELL-NEXT:    cmovbw (%rdx), %di # sched: [6:0.50]
-; BROADWELL-NEXT:    cmovaew (%rdx), %di # sched: [6:0.50]
-; BROADWELL-NEXT:    cmovaew (%rdx), %di # sched: [6:0.50]
-; BROADWELL-NEXT:    cmovaew (%rdx), %di # sched: [6:0.50]
-; BROADWELL-NEXT:    cmovew (%rdx), %di # sched: [6:0.50]
-; BROADWELL-NEXT:    cmovew (%rdx), %di # sched: [6:0.50]
-; BROADWELL-NEXT:    cmovnew (%rdx), %di # sched: [6:0.50]
-; BROADWELL-NEXT:    cmovnew (%rdx), %di # sched: [6:0.50]
-; BROADWELL-NEXT:    cmovbew (%rdx), %di # sched: [7:0.50]
-; BROADWELL-NEXT:    cmovbew (%rdx), %di # sched: [7:0.50]
-; BROADWELL-NEXT:    cmovaw (%rdx), %di # sched: [7:0.50]
-; BROADWELL-NEXT:    cmovaw (%rdx), %di # sched: [7:0.50]
-; BROADWELL-NEXT:    cmovsw (%rdx), %di # sched: [6:0.50]
-; BROADWELL-NEXT:    cmovnsw (%rdx), %di # sched: [6:0.50]
-; BROADWELL-NEXT:    cmovpw (%rdx), %di # sched: [6:0.50]
-; BROADWELL-NEXT:    cmovpw (%rdx), %di # sched: [6:0.50]
-; BROADWELL-NEXT:    cmovnpw (%rdx), %di # sched: [6:0.50]
-; BROADWELL-NEXT:    cmovnpw (%rdx), %di # sched: [6:0.50]
-; BROADWELL-NEXT:    cmovlw (%rdx), %di # sched: [6:0.50]
-; BROADWELL-NEXT:    cmovlw (%rdx), %di # sched: [6:0.50]
-; BROADWELL-NEXT:    cmovgew (%rdx), %di # sched: [6:0.50]
-; BROADWELL-NEXT:    cmovgew (%rdx), %di # sched: [6:0.50]
-; BROADWELL-NEXT:    cmovlew (%rdx), %di # sched: [6:0.50]
-; BROADWELL-NEXT:    cmovlew (%rdx), %di # sched: [6:0.50]
-; BROADWELL-NEXT:    cmovgw (%rdx), %di # sched: [6:0.50]
-; BROADWELL-NEXT:    cmovgw (%rdx), %di # sched: [6:0.50]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_cmov_16:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    cmovow %si, %di # sched: [1:0.50]
-; SKYLAKE-NEXT:    cmovnow %si, %di # sched: [1:0.50]
-; SKYLAKE-NEXT:    cmovbw %si, %di # sched: [1:0.50]
-; SKYLAKE-NEXT:    cmovbw %si, %di # sched: [1:0.50]
-; SKYLAKE-NEXT:    cmovbw %si, %di # sched: [1:0.50]
-; SKYLAKE-NEXT:    cmovaew %si, %di # sched: [1:0.50]
-; SKYLAKE-NEXT:    cmovaew %si, %di # sched: [1:0.50]
-; SKYLAKE-NEXT:    cmovaew %si, %di # sched: [1:0.50]
-; SKYLAKE-NEXT:    cmovew %si, %di # sched: [1:0.50]
-; SKYLAKE-NEXT:    cmovew %si, %di # sched: [1:0.50]
-; SKYLAKE-NEXT:    cmovnew %si, %di # sched: [1:0.50]
-; SKYLAKE-NEXT:    cmovnew %si, %di # sched: [1:0.50]
-; SKYLAKE-NEXT:    cmovbew %si, %di # sched: [2:1.00]
-; SKYLAKE-NEXT:    cmovbew %si, %di # sched: [2:1.00]
-; SKYLAKE-NEXT:    cmovaw %si, %di # sched: [2:1.00]
-; SKYLAKE-NEXT:    cmovaw %si, %di # sched: [2:1.00]
-; SKYLAKE-NEXT:    cmovsw %si, %di # sched: [1:0.50]
-; SKYLAKE-NEXT:    cmovnsw %si, %di # sched: [1:0.50]
-; SKYLAKE-NEXT:    cmovpw %si, %di # sched: [1:0.50]
-; SKYLAKE-NEXT:    cmovpw %si, %di # sched: [1:0.50]
-; SKYLAKE-NEXT:    cmovnpw %si, %di # sched: [1:0.50]
-; SKYLAKE-NEXT:    cmovnpw %si, %di # sched: [1:0.50]
-; SKYLAKE-NEXT:    cmovlw %si, %di # sched: [1:0.50]
-; SKYLAKE-NEXT:    cmovlw %si, %di # sched: [1:0.50]
-; SKYLAKE-NEXT:    cmovgew %si, %di # sched: [1:0.50]
-; SKYLAKE-NEXT:    cmovgew %si, %di # sched: [1:0.50]
-; SKYLAKE-NEXT:    cmovlew %si, %di # sched: [1:0.50]
-; SKYLAKE-NEXT:    cmovlew %si, %di # sched: [1:0.50]
-; SKYLAKE-NEXT:    cmovgw %si, %di # sched: [1:0.50]
-; SKYLAKE-NEXT:    cmovgw %si, %di # sched: [1:0.50]
-; SKYLAKE-NEXT:    cmovow (%rdx), %di # sched: [6:0.50]
-; SKYLAKE-NEXT:    cmovnow (%rdx), %di # sched: [6:0.50]
-; SKYLAKE-NEXT:    cmovbw (%rdx), %di # sched: [6:0.50]
-; SKYLAKE-NEXT:    cmovbw (%rdx), %di # sched: [6:0.50]
-; SKYLAKE-NEXT:    cmovbw (%rdx), %di # sched: [6:0.50]
-; SKYLAKE-NEXT:    cmovaew (%rdx), %di # sched: [6:0.50]
-; SKYLAKE-NEXT:    cmovaew (%rdx), %di # sched: [6:0.50]
-; SKYLAKE-NEXT:    cmovaew (%rdx), %di # sched: [6:0.50]
-; SKYLAKE-NEXT:    cmovew (%rdx), %di # sched: [6:0.50]
-; SKYLAKE-NEXT:    cmovew (%rdx), %di # sched: [6:0.50]
-; SKYLAKE-NEXT:    cmovnew (%rdx), %di # sched: [6:0.50]
-; SKYLAKE-NEXT:    cmovnew (%rdx), %di # sched: [6:0.50]
-; SKYLAKE-NEXT:    cmovbew (%rdx), %di # sched: [7:1.00]
-; SKYLAKE-NEXT:    cmovbew (%rdx), %di # sched: [7:1.00]
-; SKYLAKE-NEXT:    cmovaw (%rdx), %di # sched: [7:1.00]
-; SKYLAKE-NEXT:    cmovaw (%rdx), %di # sched: [7:1.00]
-; SKYLAKE-NEXT:    cmovsw (%rdx), %di # sched: [6:0.50]
-; SKYLAKE-NEXT:    cmovnsw (%rdx), %di # sched: [6:0.50]
-; SKYLAKE-NEXT:    cmovpw (%rdx), %di # sched: [6:0.50]
-; SKYLAKE-NEXT:    cmovpw (%rdx), %di # sched: [6:0.50]
-; SKYLAKE-NEXT:    cmovnpw (%rdx), %di # sched: [6:0.50]
-; SKYLAKE-NEXT:    cmovnpw (%rdx), %di # sched: [6:0.50]
-; SKYLAKE-NEXT:    cmovlw (%rdx), %di # sched: [6:0.50]
-; SKYLAKE-NEXT:    cmovlw (%rdx), %di # sched: [6:0.50]
-; SKYLAKE-NEXT:    cmovgew (%rdx), %di # sched: [6:0.50]
-; SKYLAKE-NEXT:    cmovgew (%rdx), %di # sched: [6:0.50]
-; SKYLAKE-NEXT:    cmovlew (%rdx), %di # sched: [6:0.50]
-; SKYLAKE-NEXT:    cmovlew (%rdx), %di # sched: [6:0.50]
-; SKYLAKE-NEXT:    cmovgw (%rdx), %di # sched: [6:0.50]
-; SKYLAKE-NEXT:    cmovgw (%rdx), %di # sched: [6:0.50]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_cmov_16:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    cmovow %si, %di # sched: [1:0.50]
-; SKX-NEXT:    cmovnow %si, %di # sched: [1:0.50]
-; SKX-NEXT:    cmovbw %si, %di # sched: [1:0.50]
-; SKX-NEXT:    cmovbw %si, %di # sched: [1:0.50]
-; SKX-NEXT:    cmovbw %si, %di # sched: [1:0.50]
-; SKX-NEXT:    cmovaew %si, %di # sched: [1:0.50]
-; SKX-NEXT:    cmovaew %si, %di # sched: [1:0.50]
-; SKX-NEXT:    cmovaew %si, %di # sched: [1:0.50]
-; SKX-NEXT:    cmovew %si, %di # sched: [1:0.50]
-; SKX-NEXT:    cmovew %si, %di # sched: [1:0.50]
-; SKX-NEXT:    cmovnew %si, %di # sched: [1:0.50]
-; SKX-NEXT:    cmovnew %si, %di # sched: [1:0.50]
-; SKX-NEXT:    cmovbew %si, %di # sched: [2:1.00]
-; SKX-NEXT:    cmovbew %si, %di # sched: [2:1.00]
-; SKX-NEXT:    cmovaw %si, %di # sched: [2:1.00]
-; SKX-NEXT:    cmovaw %si, %di # sched: [2:1.00]
-; SKX-NEXT:    cmovsw %si, %di # sched: [1:0.50]
-; SKX-NEXT:    cmovnsw %si, %di # sched: [1:0.50]
-; SKX-NEXT:    cmovpw %si, %di # sched: [1:0.50]
-; SKX-NEXT:    cmovpw %si, %di # sched: [1:0.50]
-; SKX-NEXT:    cmovnpw %si, %di # sched: [1:0.50]
-; SKX-NEXT:    cmovnpw %si, %di # sched: [1:0.50]
-; SKX-NEXT:    cmovlw %si, %di # sched: [1:0.50]
-; SKX-NEXT:    cmovlw %si, %di # sched: [1:0.50]
-; SKX-NEXT:    cmovgew %si, %di # sched: [1:0.50]
-; SKX-NEXT:    cmovgew %si, %di # sched: [1:0.50]
-; SKX-NEXT:    cmovlew %si, %di # sched: [1:0.50]
-; SKX-NEXT:    cmovlew %si, %di # sched: [1:0.50]
-; SKX-NEXT:    cmovgw %si, %di # sched: [1:0.50]
-; SKX-NEXT:    cmovgw %si, %di # sched: [1:0.50]
-; SKX-NEXT:    cmovow (%rdx), %di # sched: [6:0.50]
-; SKX-NEXT:    cmovnow (%rdx), %di # sched: [6:0.50]
-; SKX-NEXT:    cmovbw (%rdx), %di # sched: [6:0.50]
-; SKX-NEXT:    cmovbw (%rdx), %di # sched: [6:0.50]
-; SKX-NEXT:    cmovbw (%rdx), %di # sched: [6:0.50]
-; SKX-NEXT:    cmovaew (%rdx), %di # sched: [6:0.50]
-; SKX-NEXT:    cmovaew (%rdx), %di # sched: [6:0.50]
-; SKX-NEXT:    cmovaew (%rdx), %di # sched: [6:0.50]
-; SKX-NEXT:    cmovew (%rdx), %di # sched: [6:0.50]
-; SKX-NEXT:    cmovew (%rdx), %di # sched: [6:0.50]
-; SKX-NEXT:    cmovnew (%rdx), %di # sched: [6:0.50]
-; SKX-NEXT:    cmovnew (%rdx), %di # sched: [6:0.50]
-; SKX-NEXT:    cmovbew (%rdx), %di # sched: [7:1.00]
-; SKX-NEXT:    cmovbew (%rdx), %di # sched: [7:1.00]
-; SKX-NEXT:    cmovaw (%rdx), %di # sched: [7:1.00]
-; SKX-NEXT:    cmovaw (%rdx), %di # sched: [7:1.00]
-; SKX-NEXT:    cmovsw (%rdx), %di # sched: [6:0.50]
-; SKX-NEXT:    cmovnsw (%rdx), %di # sched: [6:0.50]
-; SKX-NEXT:    cmovpw (%rdx), %di # sched: [6:0.50]
-; SKX-NEXT:    cmovpw (%rdx), %di # sched: [6:0.50]
-; SKX-NEXT:    cmovnpw (%rdx), %di # sched: [6:0.50]
-; SKX-NEXT:    cmovnpw (%rdx), %di # sched: [6:0.50]
-; SKX-NEXT:    cmovlw (%rdx), %di # sched: [6:0.50]
-; SKX-NEXT:    cmovlw (%rdx), %di # sched: [6:0.50]
-; SKX-NEXT:    cmovgew (%rdx), %di # sched: [6:0.50]
-; SKX-NEXT:    cmovgew (%rdx), %di # sched: [6:0.50]
-; SKX-NEXT:    cmovlew (%rdx), %di # sched: [6:0.50]
-; SKX-NEXT:    cmovlew (%rdx), %di # sched: [6:0.50]
-; SKX-NEXT:    cmovgw (%rdx), %di # sched: [6:0.50]
-; SKX-NEXT:    cmovgw (%rdx), %di # sched: [6:0.50]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_cmov_16:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    cmovow %si, %di # sched: [1:0.50]
-; BDVER2-NEXT:    cmovnow %si, %di # sched: [1:0.50]
-; BDVER2-NEXT:    cmovbw %si, %di # sched: [1:0.50]
-; BDVER2-NEXT:    cmovbw %si, %di # sched: [1:0.50]
-; BDVER2-NEXT:    cmovbw %si, %di # sched: [1:0.50]
-; BDVER2-NEXT:    cmovaew %si, %di # sched: [1:0.50]
-; BDVER2-NEXT:    cmovaew %si, %di # sched: [1:0.50]
-; BDVER2-NEXT:    cmovaew %si, %di # sched: [1:0.50]
-; BDVER2-NEXT:    cmovew %si, %di # sched: [1:0.50]
-; BDVER2-NEXT:    cmovew %si, %di # sched: [1:0.50]
-; BDVER2-NEXT:    cmovnew %si, %di # sched: [1:0.50]
-; BDVER2-NEXT:    cmovnew %si, %di # sched: [1:0.50]
-; BDVER2-NEXT:    cmovbew %si, %di # sched: [1:0.50]
-; BDVER2-NEXT:    cmovbew %si, %di # sched: [1:0.50]
-; BDVER2-NEXT:    cmovaw %si, %di # sched: [1:0.50]
-; BDVER2-NEXT:    cmovaw %si, %di # sched: [1:0.50]
-; BDVER2-NEXT:    cmovsw %si, %di # sched: [1:0.50]
-; BDVER2-NEXT:    cmovnsw %si, %di # sched: [1:0.50]
-; BDVER2-NEXT:    cmovpw %si, %di # sched: [1:0.50]
-; BDVER2-NEXT:    cmovpw %si, %di # sched: [1:0.50]
-; BDVER2-NEXT:    cmovnpw %si, %di # sched: [1:0.50]
-; BDVER2-NEXT:    cmovnpw %si, %di # sched: [1:0.50]
-; BDVER2-NEXT:    cmovlw %si, %di # sched: [1:0.50]
-; BDVER2-NEXT:    cmovlw %si, %di # sched: [1:0.50]
-; BDVER2-NEXT:    cmovgew %si, %di # sched: [1:0.50]
-; BDVER2-NEXT:    cmovgew %si, %di # sched: [1:0.50]
-; BDVER2-NEXT:    cmovlew %si, %di # sched: [1:0.50]
-; BDVER2-NEXT:    cmovlew %si, %di # sched: [1:0.50]
-; BDVER2-NEXT:    cmovgw %si, %di # sched: [1:0.50]
-; BDVER2-NEXT:    cmovgw %si, %di # sched: [1:0.50]
-; BDVER2-NEXT:    cmovow (%rdx), %di # sched: [5:0.50]
-; BDVER2-NEXT:    cmovnow (%rdx), %di # sched: [5:0.50]
-; BDVER2-NEXT:    cmovbw (%rdx), %di # sched: [5:0.50]
-; BDVER2-NEXT:    cmovbw (%rdx), %di # sched: [5:0.50]
-; BDVER2-NEXT:    cmovbw (%rdx), %di # sched: [5:0.50]
-; BDVER2-NEXT:    cmovaew (%rdx), %di # sched: [5:0.50]
-; BDVER2-NEXT:    cmovaew (%rdx), %di # sched: [5:0.50]
-; BDVER2-NEXT:    cmovaew (%rdx), %di # sched: [5:0.50]
-; BDVER2-NEXT:    cmovew (%rdx), %di # sched: [5:0.50]
-; BDVER2-NEXT:    cmovew (%rdx), %di # sched: [5:0.50]
-; BDVER2-NEXT:    cmovnew (%rdx), %di # sched: [5:0.50]
-; BDVER2-NEXT:    cmovnew (%rdx), %di # sched: [5:0.50]
-; BDVER2-NEXT:    cmovbew (%rdx), %di # sched: [5:0.50]
-; BDVER2-NEXT:    cmovbew (%rdx), %di # sched: [5:0.50]
-; BDVER2-NEXT:    cmovaw (%rdx), %di # sched: [5:0.50]
-; BDVER2-NEXT:    cmovaw (%rdx), %di # sched: [5:0.50]
-; BDVER2-NEXT:    cmovsw (%rdx), %di # sched: [5:0.50]
-; BDVER2-NEXT:    cmovnsw (%rdx), %di # sched: [5:0.50]
-; BDVER2-NEXT:    cmovpw (%rdx), %di # sched: [5:0.50]
-; BDVER2-NEXT:    cmovpw (%rdx), %di # sched: [5:0.50]
-; BDVER2-NEXT:    cmovnpw (%rdx), %di # sched: [5:0.50]
-; BDVER2-NEXT:    cmovnpw (%rdx), %di # sched: [5:0.50]
-; BDVER2-NEXT:    cmovlw (%rdx), %di # sched: [5:0.50]
-; BDVER2-NEXT:    cmovlw (%rdx), %di # sched: [5:0.50]
-; BDVER2-NEXT:    cmovgew (%rdx), %di # sched: [5:0.50]
-; BDVER2-NEXT:    cmovgew (%rdx), %di # sched: [5:0.50]
-; BDVER2-NEXT:    cmovlew (%rdx), %di # sched: [5:0.50]
-; BDVER2-NEXT:    cmovlew (%rdx), %di # sched: [5:0.50]
-; BDVER2-NEXT:    cmovgw (%rdx), %di # sched: [5:0.50]
-; BDVER2-NEXT:    cmovgw (%rdx), %di # sched: [5:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_cmov_16:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    cmovow %si, %di # sched: [1:0.50]
-; BTVER2-NEXT:    cmovnow %si, %di # sched: [1:0.50]
-; BTVER2-NEXT:    cmovbw %si, %di # sched: [1:0.50]
-; BTVER2-NEXT:    cmovbw %si, %di # sched: [1:0.50]
-; BTVER2-NEXT:    cmovbw %si, %di # sched: [1:0.50]
-; BTVER2-NEXT:    cmovaew %si, %di # sched: [1:0.50]
-; BTVER2-NEXT:    cmovaew %si, %di # sched: [1:0.50]
-; BTVER2-NEXT:    cmovaew %si, %di # sched: [1:0.50]
-; BTVER2-NEXT:    cmovew %si, %di # sched: [1:0.50]
-; BTVER2-NEXT:    cmovew %si, %di # sched: [1:0.50]
-; BTVER2-NEXT:    cmovnew %si, %di # sched: [1:0.50]
-; BTVER2-NEXT:    cmovnew %si, %di # sched: [1:0.50]
-; BTVER2-NEXT:    cmovbew %si, %di # sched: [1:0.50]
-; BTVER2-NEXT:    cmovbew %si, %di # sched: [1:0.50]
-; BTVER2-NEXT:    cmovaw %si, %di # sched: [1:0.50]
-; BTVER2-NEXT:    cmovaw %si, %di # sched: [1:0.50]
-; BTVER2-NEXT:    cmovsw %si, %di # sched: [1:0.50]
-; BTVER2-NEXT:    cmovnsw %si, %di # sched: [1:0.50]
-; BTVER2-NEXT:    cmovpw %si, %di # sched: [1:0.50]
-; BTVER2-NEXT:    cmovpw %si, %di # sched: [1:0.50]
-; BTVER2-NEXT:    cmovnpw %si, %di # sched: [1:0.50]
-; BTVER2-NEXT:    cmovnpw %si, %di # sched: [1:0.50]
-; BTVER2-NEXT:    cmovlw %si, %di # sched: [1:0.50]
-; BTVER2-NEXT:    cmovlw %si, %di # sched: [1:0.50]
-; BTVER2-NEXT:    cmovgew %si, %di # sched: [1:0.50]
-; BTVER2-NEXT:    cmovgew %si, %di # sched: [1:0.50]
-; BTVER2-NEXT:    cmovlew %si, %di # sched: [1:0.50]
-; BTVER2-NEXT:    cmovlew %si, %di # sched: [1:0.50]
-; BTVER2-NEXT:    cmovgw %si, %di # sched: [1:0.50]
-; BTVER2-NEXT:    cmovgw %si, %di # sched: [1:0.50]
-; BTVER2-NEXT:    cmovow (%rdx), %di # sched: [4:1.00]
-; BTVER2-NEXT:    cmovnow (%rdx), %di # sched: [4:1.00]
-; BTVER2-NEXT:    cmovbw (%rdx), %di # sched: [4:1.00]
-; BTVER2-NEXT:    cmovbw (%rdx), %di # sched: [4:1.00]
-; BTVER2-NEXT:    cmovbw (%rdx), %di # sched: [4:1.00]
-; BTVER2-NEXT:    cmovaew (%rdx), %di # sched: [4:1.00]
-; BTVER2-NEXT:    cmovaew (%rdx), %di # sched: [4:1.00]
-; BTVER2-NEXT:    cmovaew (%rdx), %di # sched: [4:1.00]
-; BTVER2-NEXT:    cmovew (%rdx), %di # sched: [4:1.00]
-; BTVER2-NEXT:    cmovew (%rdx), %di # sched: [4:1.00]
-; BTVER2-NEXT:    cmovnew (%rdx), %di # sched: [4:1.00]
-; BTVER2-NEXT:    cmovnew (%rdx), %di # sched: [4:1.00]
-; BTVER2-NEXT:    cmovbew (%rdx), %di # sched: [4:1.00]
-; BTVER2-NEXT:    cmovbew (%rdx), %di # sched: [4:1.00]
-; BTVER2-NEXT:    cmovaw (%rdx), %di # sched: [4:1.00]
-; BTVER2-NEXT:    cmovaw (%rdx), %di # sched: [4:1.00]
-; BTVER2-NEXT:    cmovsw (%rdx), %di # sched: [4:1.00]
-; BTVER2-NEXT:    cmovnsw (%rdx), %di # sched: [4:1.00]
-; BTVER2-NEXT:    cmovpw (%rdx), %di # sched: [4:1.00]
-; BTVER2-NEXT:    cmovpw (%rdx), %di # sched: [4:1.00]
-; BTVER2-NEXT:    cmovnpw (%rdx), %di # sched: [4:1.00]
-; BTVER2-NEXT:    cmovnpw (%rdx), %di # sched: [4:1.00]
-; BTVER2-NEXT:    cmovlw (%rdx), %di # sched: [4:1.00]
-; BTVER2-NEXT:    cmovlw (%rdx), %di # sched: [4:1.00]
-; BTVER2-NEXT:    cmovgew (%rdx), %di # sched: [4:1.00]
-; BTVER2-NEXT:    cmovgew (%rdx), %di # sched: [4:1.00]
-; BTVER2-NEXT:    cmovlew (%rdx), %di # sched: [4:1.00]
-; BTVER2-NEXT:    cmovlew (%rdx), %di # sched: [4:1.00]
-; BTVER2-NEXT:    cmovgw (%rdx), %di # sched: [4:1.00]
-; BTVER2-NEXT:    cmovgw (%rdx), %di # sched: [4:1.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_cmov_16:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    cmovow %si, %di # sched: [1:0.25]
-; ZNVER1-NEXT:    cmovnow %si, %di # sched: [1:0.25]
-; ZNVER1-NEXT:    cmovbw %si, %di # sched: [1:0.25]
-; ZNVER1-NEXT:    cmovbw %si, %di # sched: [1:0.25]
-; ZNVER1-NEXT:    cmovbw %si, %di # sched: [1:0.25]
-; ZNVER1-NEXT:    cmovaew %si, %di # sched: [1:0.25]
-; ZNVER1-NEXT:    cmovaew %si, %di # sched: [1:0.25]
-; ZNVER1-NEXT:    cmovaew %si, %di # sched: [1:0.25]
-; ZNVER1-NEXT:    cmovew %si, %di # sched: [1:0.25]
-; ZNVER1-NEXT:    cmovew %si, %di # sched: [1:0.25]
-; ZNVER1-NEXT:    cmovnew %si, %di # sched: [1:0.25]
-; ZNVER1-NEXT:    cmovnew %si, %di # sched: [1:0.25]
-; ZNVER1-NEXT:    cmovbew %si, %di # sched: [1:0.25]
-; ZNVER1-NEXT:    cmovbew %si, %di # sched: [1:0.25]
-; ZNVER1-NEXT:    cmovaw %si, %di # sched: [1:0.25]
-; ZNVER1-NEXT:    cmovaw %si, %di # sched: [1:0.25]
-; ZNVER1-NEXT:    cmovsw %si, %di # sched: [1:0.25]
-; ZNVER1-NEXT:    cmovnsw %si, %di # sched: [1:0.25]
-; ZNVER1-NEXT:    cmovpw %si, %di # sched: [1:0.25]
-; ZNVER1-NEXT:    cmovpw %si, %di # sched: [1:0.25]
-; ZNVER1-NEXT:    cmovnpw %si, %di # sched: [1:0.25]
-; ZNVER1-NEXT:    cmovnpw %si, %di # sched: [1:0.25]
-; ZNVER1-NEXT:    cmovlw %si, %di # sched: [1:0.25]
-; ZNVER1-NEXT:    cmovlw %si, %di # sched: [1:0.25]
-; ZNVER1-NEXT:    cmovgew %si, %di # sched: [1:0.25]
-; ZNVER1-NEXT:    cmovgew %si, %di # sched: [1:0.25]
-; ZNVER1-NEXT:    cmovlew %si, %di # sched: [1:0.25]
-; ZNVER1-NEXT:    cmovlew %si, %di # sched: [1:0.25]
-; ZNVER1-NEXT:    cmovgw %si, %di # sched: [1:0.25]
-; ZNVER1-NEXT:    cmovgw %si, %di # sched: [1:0.25]
-; ZNVER1-NEXT:    cmovow (%rdx), %di # sched: [5:0.50]
-; ZNVER1-NEXT:    cmovnow (%rdx), %di # sched: [5:0.50]
-; ZNVER1-NEXT:    cmovbw (%rdx), %di # sched: [5:0.50]
-; ZNVER1-NEXT:    cmovbw (%rdx), %di # sched: [5:0.50]
-; ZNVER1-NEXT:    cmovbw (%rdx), %di # sched: [5:0.50]
-; ZNVER1-NEXT:    cmovaew (%rdx), %di # sched: [5:0.50]
-; ZNVER1-NEXT:    cmovaew (%rdx), %di # sched: [5:0.50]
-; ZNVER1-NEXT:    cmovaew (%rdx), %di # sched: [5:0.50]
-; ZNVER1-NEXT:    cmovew (%rdx), %di # sched: [5:0.50]
-; ZNVER1-NEXT:    cmovew (%rdx), %di # sched: [5:0.50]
-; ZNVER1-NEXT:    cmovnew (%rdx), %di # sched: [5:0.50]
-; ZNVER1-NEXT:    cmovnew (%rdx), %di # sched: [5:0.50]
-; ZNVER1-NEXT:    cmovbew (%rdx), %di # sched: [5:0.50]
-; ZNVER1-NEXT:    cmovbew (%rdx), %di # sched: [5:0.50]
-; ZNVER1-NEXT:    cmovaw (%rdx), %di # sched: [5:0.50]
-; ZNVER1-NEXT:    cmovaw (%rdx), %di # sched: [5:0.50]
-; ZNVER1-NEXT:    cmovsw (%rdx), %di # sched: [5:0.50]
-; ZNVER1-NEXT:    cmovnsw (%rdx), %di # sched: [5:0.50]
-; ZNVER1-NEXT:    cmovpw (%rdx), %di # sched: [5:0.50]
-; ZNVER1-NEXT:    cmovpw (%rdx), %di # sched: [5:0.50]
-; ZNVER1-NEXT:    cmovnpw (%rdx), %di # sched: [5:0.50]
-; ZNVER1-NEXT:    cmovnpw (%rdx), %di # sched: [5:0.50]
-; ZNVER1-NEXT:    cmovlw (%rdx), %di # sched: [5:0.50]
-; ZNVER1-NEXT:    cmovlw (%rdx), %di # sched: [5:0.50]
-; ZNVER1-NEXT:    cmovgew (%rdx), %di # sched: [5:0.50]
-; ZNVER1-NEXT:    cmovgew (%rdx), %di # sched: [5:0.50]
-; ZNVER1-NEXT:    cmovlew (%rdx), %di # sched: [5:0.50]
-; ZNVER1-NEXT:    cmovlew (%rdx), %di # sched: [5:0.50]
-; ZNVER1-NEXT:    cmovgw (%rdx), %di # sched: [5:0.50]
-; ZNVER1-NEXT:    cmovgw (%rdx), %di # sched: [5:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  call void asm sideeffect "cmovow $1, $0 \0A\09 cmovnow $1, $0 \0A\09 cmovbw $1, $0 \0A\09 cmovcw $1, $0 \0A\09 cmovnaew $1, $0 \0A\09 cmovnbw $1, $0 \0A\09 cmovncw $1, $0 \0A\09 cmovaew $1, $0 \0A\09 cmovzw $1, $0 \0A\09 cmovew $1, $0 \0A\09 cmovnzw $1, $0 \0A\09 cmovnew $1, $0 \0A\09 cmovbew $1, $0 \0A\09 cmovnaw $1, $0 \0A\09 cmovnbew $1, $0 \0A\09 cmovaw $1, $0 \0A\09 cmovsw $1, $0 \0A\09 cmovnsw $1, $0 \0A\09 cmovpw $1, $0 \0A\09 cmovpew $1, $0 \0A\09 cmovnpw $1, $0 \0A\09 cmovpow $1, $0 \0A\09 cmovlw $1, $0 \0A\09 cmovngew $1, $0 \0A\09 cmovnlw $1, $0 \0A\09 cmovgew $1, $0 \0A\09 cmovlew $1, $0 \0A\09 cmovngw $1, $0 \0A\09 cmovnlew $1, $0 \0A\09 cmovgw $1, $0 \0A\09 cmovow $2, $0 \0A\09 cmovnow $2, $0 \0A\09 cmovbw $2, $0 \0A\09 cmovcw $2, $0 \0A\09 cmovnaew $2, $0 \0A\09 cmovnbw $2, $0 \0A\09 cmovncw $2, $0 \0A\09 cmovaew $2, $0 \0A\09 cmovzw $2, $0 \0A\09 cmovew $2, $0 \0A\09 cmovnzw $2, $0 \0A\09 cmovnew $2, $0 \0A\09 cmovbew $2, $0 \0A\09 cmovnaw $2, $0 \0A\09 cmovnbew $2, $0 \0A\09 cmovaw $2, $0 \0A\09 cmovsw $2, $0 \0A\09 cmovnsw $2, $0 \0A\09 cmovpw $2, $0 \0A\09 cmovpew $2, $0 \0A\09 cmovnpw $2, $0 \0A\09 cmovpow $2, $0 \0A\09 cmovlw $2, $0 \0A\09 cmovngew $2, $0 \0A\09 cmovnlw $2, $0 \0A\09 cmovgew $2, $0 \0A\09 cmovlew $2, $0 \0A\09 cmovngw $2, $0 \0A\09 cmovnlew $2, $0 \0A\09 cmovgw $2, $0", "r,r,*m"(i16 %a0, i16 %a1, i16 *%a2)
-  ret void
-}
-
-define void @test_cmov_32(i32 %a0, i32 %a1, i32 *%a2) optsize {
-; GENERIC-LABEL: test_cmov_32:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    cmovol %esi, %edi # sched: [2:0.67]
-; GENERIC-NEXT:    cmovnol %esi, %edi # sched: [2:0.67]
-; GENERIC-NEXT:    cmovbl %esi, %edi # sched: [2:0.67]
-; GENERIC-NEXT:    cmovbl %esi, %edi # sched: [2:0.67]
-; GENERIC-NEXT:    cmovbl %esi, %edi # sched: [2:0.67]
-; GENERIC-NEXT:    cmovael %esi, %edi # sched: [2:0.67]
-; GENERIC-NEXT:    cmovael %esi, %edi # sched: [2:0.67]
-; GENERIC-NEXT:    cmovael %esi, %edi # sched: [2:0.67]
-; GENERIC-NEXT:    cmovel %esi, %edi # sched: [2:0.67]
-; GENERIC-NEXT:    cmovel %esi, %edi # sched: [2:0.67]
-; GENERIC-NEXT:    cmovnel %esi, %edi # sched: [2:0.67]
-; GENERIC-NEXT:    cmovnel %esi, %edi # sched: [2:0.67]
-; GENERIC-NEXT:    cmovbel %esi, %edi # sched: [3:1.00]
-; GENERIC-NEXT:    cmovbel %esi, %edi # sched: [3:1.00]
-; GENERIC-NEXT:    cmoval %esi, %edi # sched: [3:1.00]
-; GENERIC-NEXT:    cmoval %esi, %edi # sched: [3:1.00]
-; GENERIC-NEXT:    cmovsl %esi, %edi # sched: [2:0.67]
-; GENERIC-NEXT:    cmovnsl %esi, %edi # sched: [2:0.67]
-; GENERIC-NEXT:    cmovpl %esi, %edi # sched: [2:0.67]
-; GENERIC-NEXT:    cmovpl %esi, %edi # sched: [2:0.67]
-; GENERIC-NEXT:    cmovnpl %esi, %edi # sched: [2:0.67]
-; GENERIC-NEXT:    cmovnpl %esi, %edi # sched: [2:0.67]
-; GENERIC-NEXT:    cmovll %esi, %edi # sched: [2:0.67]
-; GENERIC-NEXT:    cmovll %esi, %edi # sched: [2:0.67]
-; GENERIC-NEXT:    cmovgel %esi, %edi # sched: [2:0.67]
-; GENERIC-NEXT:    cmovgel %esi, %edi # sched: [2:0.67]
-; GENERIC-NEXT:    cmovlel %esi, %edi # sched: [2:0.67]
-; GENERIC-NEXT:    cmovlel %esi, %edi # sched: [2:0.67]
-; GENERIC-NEXT:    cmovgl %esi, %edi # sched: [2:0.67]
-; GENERIC-NEXT:    cmovgl %esi, %edi # sched: [2:0.67]
-; GENERIC-NEXT:    cmovol (%rdx), %edi # sched: [7:0.67]
-; GENERIC-NEXT:    cmovnol (%rdx), %edi # sched: [7:0.67]
-; GENERIC-NEXT:    cmovbl (%rdx), %edi # sched: [7:0.67]
-; GENERIC-NEXT:    cmovbl (%rdx), %edi # sched: [7:0.67]
-; GENERIC-NEXT:    cmovbl (%rdx), %edi # sched: [7:0.67]
-; GENERIC-NEXT:    cmovael (%rdx), %edi # sched: [7:0.67]
-; GENERIC-NEXT:    cmovael (%rdx), %edi # sched: [7:0.67]
-; GENERIC-NEXT:    cmovael (%rdx), %edi # sched: [7:0.67]
-; GENERIC-NEXT:    cmovel (%rdx), %edi # sched: [7:0.67]
-; GENERIC-NEXT:    cmovel (%rdx), %edi # sched: [7:0.67]
-; GENERIC-NEXT:    cmovnel (%rdx), %edi # sched: [7:0.67]
-; GENERIC-NEXT:    cmovnel (%rdx), %edi # sched: [7:0.67]
-; GENERIC-NEXT:    cmovbel (%rdx), %edi # sched: [8:1.00]
-; GENERIC-NEXT:    cmovbel (%rdx), %edi # sched: [8:1.00]
-; GENERIC-NEXT:    cmoval (%rdx), %edi # sched: [8:1.00]
-; GENERIC-NEXT:    cmoval (%rdx), %edi # sched: [8:1.00]
-; GENERIC-NEXT:    cmovsl (%rdx), %edi # sched: [7:0.67]
-; GENERIC-NEXT:    cmovnsl (%rdx), %edi # sched: [7:0.67]
-; GENERIC-NEXT:    cmovpl (%rdx), %edi # sched: [7:0.67]
-; GENERIC-NEXT:    cmovpl (%rdx), %edi # sched: [7:0.67]
-; GENERIC-NEXT:    cmovnpl (%rdx), %edi # sched: [7:0.67]
-; GENERIC-NEXT:    cmovnpl (%rdx), %edi # sched: [7:0.67]
-; GENERIC-NEXT:    cmovll (%rdx), %edi # sched: [7:0.67]
-; GENERIC-NEXT:    cmovll (%rdx), %edi # sched: [7:0.67]
-; GENERIC-NEXT:    cmovgel (%rdx), %edi # sched: [7:0.67]
-; GENERIC-NEXT:    cmovgel (%rdx), %edi # sched: [7:0.67]
-; GENERIC-NEXT:    cmovlel (%rdx), %edi # sched: [7:0.67]
-; GENERIC-NEXT:    cmovlel (%rdx), %edi # sched: [7:0.67]
-; GENERIC-NEXT:    cmovgl (%rdx), %edi # sched: [7:0.67]
-; GENERIC-NEXT:    cmovgl (%rdx), %edi # sched: [7:0.67]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_cmov_32:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    cmovol %esi, %edi # sched: [1:0.50]
-; ATOM-NEXT:    cmovnol %esi, %edi # sched: [1:0.50]
-; ATOM-NEXT:    cmovbl %esi, %edi # sched: [1:0.50]
-; ATOM-NEXT:    cmovbl %esi, %edi # sched: [1:0.50]
-; ATOM-NEXT:    cmovbl %esi, %edi # sched: [1:0.50]
-; ATOM-NEXT:    cmovael %esi, %edi # sched: [1:0.50]
-; ATOM-NEXT:    cmovael %esi, %edi # sched: [1:0.50]
-; ATOM-NEXT:    cmovael %esi, %edi # sched: [1:0.50]
-; ATOM-NEXT:    cmovel %esi, %edi # sched: [1:0.50]
-; ATOM-NEXT:    cmovel %esi, %edi # sched: [1:0.50]
-; ATOM-NEXT:    cmovnel %esi, %edi # sched: [1:0.50]
-; ATOM-NEXT:    cmovnel %esi, %edi # sched: [1:0.50]
-; ATOM-NEXT:    cmovbel %esi, %edi # sched: [1:0.50]
-; ATOM-NEXT:    cmovbel %esi, %edi # sched: [1:0.50]
-; ATOM-NEXT:    cmoval %esi, %edi # sched: [1:0.50]
-; ATOM-NEXT:    cmoval %esi, %edi # sched: [1:0.50]
-; ATOM-NEXT:    cmovsl %esi, %edi # sched: [1:0.50]
-; ATOM-NEXT:    cmovnsl %esi, %edi # sched: [1:0.50]
-; ATOM-NEXT:    cmovpl %esi, %edi # sched: [1:0.50]
-; ATOM-NEXT:    cmovpl %esi, %edi # sched: [1:0.50]
-; ATOM-NEXT:    cmovnpl %esi, %edi # sched: [1:0.50]
-; ATOM-NEXT:    cmovnpl %esi, %edi # sched: [1:0.50]
-; ATOM-NEXT:    cmovll %esi, %edi # sched: [1:0.50]
-; ATOM-NEXT:    cmovll %esi, %edi # sched: [1:0.50]
-; ATOM-NEXT:    cmovgel %esi, %edi # sched: [1:0.50]
-; ATOM-NEXT:    cmovgel %esi, %edi # sched: [1:0.50]
-; ATOM-NEXT:    cmovlel %esi, %edi # sched: [1:0.50]
-; ATOM-NEXT:    cmovlel %esi, %edi # sched: [1:0.50]
-; ATOM-NEXT:    cmovgl %esi, %edi # sched: [1:0.50]
-; ATOM-NEXT:    cmovgl %esi, %edi # sched: [1:0.50]
-; ATOM-NEXT:    cmovol (%rdx), %edi # sched: [1:1.00]
-; ATOM-NEXT:    cmovnol (%rdx), %edi # sched: [1:1.00]
-; ATOM-NEXT:    cmovbl (%rdx), %edi # sched: [1:1.00]
-; ATOM-NEXT:    cmovbl (%rdx), %edi # sched: [1:1.00]
-; ATOM-NEXT:    cmovbl (%rdx), %edi # sched: [1:1.00]
-; ATOM-NEXT:    cmovael (%rdx), %edi # sched: [1:1.00]
-; ATOM-NEXT:    cmovael (%rdx), %edi # sched: [1:1.00]
-; ATOM-NEXT:    cmovael (%rdx), %edi # sched: [1:1.00]
-; ATOM-NEXT:    cmovel (%rdx), %edi # sched: [1:1.00]
-; ATOM-NEXT:    cmovel (%rdx), %edi # sched: [1:1.00]
-; ATOM-NEXT:    cmovnel (%rdx), %edi # sched: [1:1.00]
-; ATOM-NEXT:    cmovnel (%rdx), %edi # sched: [1:1.00]
-; ATOM-NEXT:    cmovbel (%rdx), %edi # sched: [1:1.00]
-; ATOM-NEXT:    cmovbel (%rdx), %edi # sched: [1:1.00]
-; ATOM-NEXT:    cmoval (%rdx), %edi # sched: [1:1.00]
-; ATOM-NEXT:    cmoval (%rdx), %edi # sched: [1:1.00]
-; ATOM-NEXT:    cmovsl (%rdx), %edi # sched: [1:1.00]
-; ATOM-NEXT:    cmovnsl (%rdx), %edi # sched: [1:1.00]
-; ATOM-NEXT:    cmovpl (%rdx), %edi # sched: [1:1.00]
-; ATOM-NEXT:    cmovpl (%rdx), %edi # sched: [1:1.00]
-; ATOM-NEXT:    cmovnpl (%rdx), %edi # sched: [1:1.00]
-; ATOM-NEXT:    cmovnpl (%rdx), %edi # sched: [1:1.00]
-; ATOM-NEXT:    cmovll (%rdx), %edi # sched: [1:1.00]
-; ATOM-NEXT:    cmovll (%rdx), %edi # sched: [1:1.00]
-; ATOM-NEXT:    cmovgel (%rdx), %edi # sched: [1:1.00]
-; ATOM-NEXT:    cmovgel (%rdx), %edi # sched: [1:1.00]
-; ATOM-NEXT:    cmovlel (%rdx), %edi # sched: [1:1.00]
-; ATOM-NEXT:    cmovlel (%rdx), %edi # sched: [1:1.00]
-; ATOM-NEXT:    cmovgl (%rdx), %edi # sched: [1:1.00]
-; ATOM-NEXT:    cmovgl (%rdx), %edi # sched: [1:1.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_cmov_32:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    cmovol %esi, %edi # sched: [2:1.00]
-; SLM-NEXT:    cmovnol %esi, %edi # sched: [2:1.00]
-; SLM-NEXT:    cmovbl %esi, %edi # sched: [2:1.00]
-; SLM-NEXT:    cmovbl %esi, %edi # sched: [2:1.00]
-; SLM-NEXT:    cmovbl %esi, %edi # sched: [2:1.00]
-; SLM-NEXT:    cmovael %esi, %edi # sched: [2:1.00]
-; SLM-NEXT:    cmovael %esi, %edi # sched: [2:1.00]
-; SLM-NEXT:    cmovael %esi, %edi # sched: [2:1.00]
-; SLM-NEXT:    cmovel %esi, %edi # sched: [2:1.00]
-; SLM-NEXT:    cmovel %esi, %edi # sched: [2:1.00]
-; SLM-NEXT:    cmovnel %esi, %edi # sched: [2:1.00]
-; SLM-NEXT:    cmovnel %esi, %edi # sched: [2:1.00]
-; SLM-NEXT:    cmovbel %esi, %edi # sched: [2:1.00]
-; SLM-NEXT:    cmovbel %esi, %edi # sched: [2:1.00]
-; SLM-NEXT:    cmoval %esi, %edi # sched: [2:1.00]
-; SLM-NEXT:    cmoval %esi, %edi # sched: [2:1.00]
-; SLM-NEXT:    cmovsl %esi, %edi # sched: [2:1.00]
-; SLM-NEXT:    cmovnsl %esi, %edi # sched: [2:1.00]
-; SLM-NEXT:    cmovpl %esi, %edi # sched: [2:1.00]
-; SLM-NEXT:    cmovpl %esi, %edi # sched: [2:1.00]
-; SLM-NEXT:    cmovnpl %esi, %edi # sched: [2:1.00]
-; SLM-NEXT:    cmovnpl %esi, %edi # sched: [2:1.00]
-; SLM-NEXT:    cmovll %esi, %edi # sched: [2:1.00]
-; SLM-NEXT:    cmovll %esi, %edi # sched: [2:1.00]
-; SLM-NEXT:    cmovgel %esi, %edi # sched: [2:1.00]
-; SLM-NEXT:    cmovgel %esi, %edi # sched: [2:1.00]
-; SLM-NEXT:    cmovlel %esi, %edi # sched: [2:1.00]
-; SLM-NEXT:    cmovlel %esi, %edi # sched: [2:1.00]
-; SLM-NEXT:    cmovgl %esi, %edi # sched: [2:1.00]
-; SLM-NEXT:    cmovgl %esi, %edi # sched: [2:1.00]
-; SLM-NEXT:    cmovol (%rdx), %edi # sched: [5:1.00]
-; SLM-NEXT:    cmovnol (%rdx), %edi # sched: [5:1.00]
-; SLM-NEXT:    cmovbl (%rdx), %edi # sched: [5:1.00]
-; SLM-NEXT:    cmovbl (%rdx), %edi # sched: [5:1.00]
-; SLM-NEXT:    cmovbl (%rdx), %edi # sched: [5:1.00]
-; SLM-NEXT:    cmovael (%rdx), %edi # sched: [5:1.00]
-; SLM-NEXT:    cmovael (%rdx), %edi # sched: [5:1.00]
-; SLM-NEXT:    cmovael (%rdx), %edi # sched: [5:1.00]
-; SLM-NEXT:    cmovel (%rdx), %edi # sched: [5:1.00]
-; SLM-NEXT:    cmovel (%rdx), %edi # sched: [5:1.00]
-; SLM-NEXT:    cmovnel (%rdx), %edi # sched: [5:1.00]
-; SLM-NEXT:    cmovnel (%rdx), %edi # sched: [5:1.00]
-; SLM-NEXT:    cmovbel (%rdx), %edi # sched: [5:1.00]
-; SLM-NEXT:    cmovbel (%rdx), %edi # sched: [5:1.00]
-; SLM-NEXT:    cmoval (%rdx), %edi # sched: [5:1.00]
-; SLM-NEXT:    cmoval (%rdx), %edi # sched: [5:1.00]
-; SLM-NEXT:    cmovsl (%rdx), %edi # sched: [5:1.00]
-; SLM-NEXT:    cmovnsl (%rdx), %edi # sched: [5:1.00]
-; SLM-NEXT:    cmovpl (%rdx), %edi # sched: [5:1.00]
-; SLM-NEXT:    cmovpl (%rdx), %edi # sched: [5:1.00]
-; SLM-NEXT:    cmovnpl (%rdx), %edi # sched: [5:1.00]
-; SLM-NEXT:    cmovnpl (%rdx), %edi # sched: [5:1.00]
-; SLM-NEXT:    cmovll (%rdx), %edi # sched: [5:1.00]
-; SLM-NEXT:    cmovll (%rdx), %edi # sched: [5:1.00]
-; SLM-NEXT:    cmovgel (%rdx), %edi # sched: [5:1.00]
-; SLM-NEXT:    cmovgel (%rdx), %edi # sched: [5:1.00]
-; SLM-NEXT:    cmovlel (%rdx), %edi # sched: [5:1.00]
-; SLM-NEXT:    cmovlel (%rdx), %edi # sched: [5:1.00]
-; SLM-NEXT:    cmovgl (%rdx), %edi # sched: [5:1.00]
-; SLM-NEXT:    cmovgl (%rdx), %edi # sched: [5:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_cmov_32:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    cmovol %esi, %edi # sched: [2:0.67]
-; SANDY-NEXT:    cmovnol %esi, %edi # sched: [2:0.67]
-; SANDY-NEXT:    cmovbl %esi, %edi # sched: [2:0.67]
-; SANDY-NEXT:    cmovbl %esi, %edi # sched: [2:0.67]
-; SANDY-NEXT:    cmovbl %esi, %edi # sched: [2:0.67]
-; SANDY-NEXT:    cmovael %esi, %edi # sched: [2:0.67]
-; SANDY-NEXT:    cmovael %esi, %edi # sched: [2:0.67]
-; SANDY-NEXT:    cmovael %esi, %edi # sched: [2:0.67]
-; SANDY-NEXT:    cmovel %esi, %edi # sched: [2:0.67]
-; SANDY-NEXT:    cmovel %esi, %edi # sched: [2:0.67]
-; SANDY-NEXT:    cmovnel %esi, %edi # sched: [2:0.67]
-; SANDY-NEXT:    cmovnel %esi, %edi # sched: [2:0.67]
-; SANDY-NEXT:    cmovbel %esi, %edi # sched: [3:1.00]
-; SANDY-NEXT:    cmovbel %esi, %edi # sched: [3:1.00]
-; SANDY-NEXT:    cmoval %esi, %edi # sched: [3:1.00]
-; SANDY-NEXT:    cmoval %esi, %edi # sched: [3:1.00]
-; SANDY-NEXT:    cmovsl %esi, %edi # sched: [2:0.67]
-; SANDY-NEXT:    cmovnsl %esi, %edi # sched: [2:0.67]
-; SANDY-NEXT:    cmovpl %esi, %edi # sched: [2:0.67]
-; SANDY-NEXT:    cmovpl %esi, %edi # sched: [2:0.67]
-; SANDY-NEXT:    cmovnpl %esi, %edi # sched: [2:0.67]
-; SANDY-NEXT:    cmovnpl %esi, %edi # sched: [2:0.67]
-; SANDY-NEXT:    cmovll %esi, %edi # sched: [2:0.67]
-; SANDY-NEXT:    cmovll %esi, %edi # sched: [2:0.67]
-; SANDY-NEXT:    cmovgel %esi, %edi # sched: [2:0.67]
-; SANDY-NEXT:    cmovgel %esi, %edi # sched: [2:0.67]
-; SANDY-NEXT:    cmovlel %esi, %edi # sched: [2:0.67]
-; SANDY-NEXT:    cmovlel %esi, %edi # sched: [2:0.67]
-; SANDY-NEXT:    cmovgl %esi, %edi # sched: [2:0.67]
-; SANDY-NEXT:    cmovgl %esi, %edi # sched: [2:0.67]
-; SANDY-NEXT:    cmovol (%rdx), %edi # sched: [7:0.67]
-; SANDY-NEXT:    cmovnol (%rdx), %edi # sched: [7:0.67]
-; SANDY-NEXT:    cmovbl (%rdx), %edi # sched: [7:0.67]
-; SANDY-NEXT:    cmovbl (%rdx), %edi # sched: [7:0.67]
-; SANDY-NEXT:    cmovbl (%rdx), %edi # sched: [7:0.67]
-; SANDY-NEXT:    cmovael (%rdx), %edi # sched: [7:0.67]
-; SANDY-NEXT:    cmovael (%rdx), %edi # sched: [7:0.67]
-; SANDY-NEXT:    cmovael (%rdx), %edi # sched: [7:0.67]
-; SANDY-NEXT:    cmovel (%rdx), %edi # sched: [7:0.67]
-; SANDY-NEXT:    cmovel (%rdx), %edi # sched: [7:0.67]
-; SANDY-NEXT:    cmovnel (%rdx), %edi # sched: [7:0.67]
-; SANDY-NEXT:    cmovnel (%rdx), %edi # sched: [7:0.67]
-; SANDY-NEXT:    cmovbel (%rdx), %edi # sched: [8:1.00]
-; SANDY-NEXT:    cmovbel (%rdx), %edi # sched: [8:1.00]
-; SANDY-NEXT:    cmoval (%rdx), %edi # sched: [8:1.00]
-; SANDY-NEXT:    cmoval (%rdx), %edi # sched: [8:1.00]
-; SANDY-NEXT:    cmovsl (%rdx), %edi # sched: [7:0.67]
-; SANDY-NEXT:    cmovnsl (%rdx), %edi # sched: [7:0.67]
-; SANDY-NEXT:    cmovpl (%rdx), %edi # sched: [7:0.67]
-; SANDY-NEXT:    cmovpl (%rdx), %edi # sched: [7:0.67]
-; SANDY-NEXT:    cmovnpl (%rdx), %edi # sched: [7:0.67]
-; SANDY-NEXT:    cmovnpl (%rdx), %edi # sched: [7:0.67]
-; SANDY-NEXT:    cmovll (%rdx), %edi # sched: [7:0.67]
-; SANDY-NEXT:    cmovll (%rdx), %edi # sched: [7:0.67]
-; SANDY-NEXT:    cmovgel (%rdx), %edi # sched: [7:0.67]
-; SANDY-NEXT:    cmovgel (%rdx), %edi # sched: [7:0.67]
-; SANDY-NEXT:    cmovlel (%rdx), %edi # sched: [7:0.67]
-; SANDY-NEXT:    cmovlel (%rdx), %edi # sched: [7:0.67]
-; SANDY-NEXT:    cmovgl (%rdx), %edi # sched: [7:0.67]
-; SANDY-NEXT:    cmovgl (%rdx), %edi # sched: [7:0.67]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_cmov_32:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    cmovol %esi, %edi # sched: [2:0.50]
-; HASWELL-NEXT:    cmovnol %esi, %edi # sched: [2:0.50]
-; HASWELL-NEXT:    cmovbl %esi, %edi # sched: [2:0.50]
-; HASWELL-NEXT:    cmovbl %esi, %edi # sched: [2:0.50]
-; HASWELL-NEXT:    cmovbl %esi, %edi # sched: [2:0.50]
-; HASWELL-NEXT:    cmovael %esi, %edi # sched: [2:0.50]
-; HASWELL-NEXT:    cmovael %esi, %edi # sched: [2:0.50]
-; HASWELL-NEXT:    cmovael %esi, %edi # sched: [2:0.50]
-; HASWELL-NEXT:    cmovel %esi, %edi # sched: [2:0.50]
-; HASWELL-NEXT:    cmovel %esi, %edi # sched: [2:0.50]
-; HASWELL-NEXT:    cmovnel %esi, %edi # sched: [2:0.50]
-; HASWELL-NEXT:    cmovnel %esi, %edi # sched: [2:0.50]
-; HASWELL-NEXT:    cmovbel %esi, %edi # sched: [3:0.75]
-; HASWELL-NEXT:    cmovbel %esi, %edi # sched: [3:0.75]
-; HASWELL-NEXT:    cmoval %esi, %edi # sched: [3:0.75]
-; HASWELL-NEXT:    cmoval %esi, %edi # sched: [3:0.75]
-; HASWELL-NEXT:    cmovsl %esi, %edi # sched: [2:0.50]
-; HASWELL-NEXT:    cmovnsl %esi, %edi # sched: [2:0.50]
-; HASWELL-NEXT:    cmovpl %esi, %edi # sched: [2:0.50]
-; HASWELL-NEXT:    cmovpl %esi, %edi # sched: [2:0.50]
-; HASWELL-NEXT:    cmovnpl %esi, %edi # sched: [2:0.50]
-; HASWELL-NEXT:    cmovnpl %esi, %edi # sched: [2:0.50]
-; HASWELL-NEXT:    cmovll %esi, %edi # sched: [2:0.50]
-; HASWELL-NEXT:    cmovll %esi, %edi # sched: [2:0.50]
-; HASWELL-NEXT:    cmovgel %esi, %edi # sched: [2:0.50]
-; HASWELL-NEXT:    cmovgel %esi, %edi # sched: [2:0.50]
-; HASWELL-NEXT:    cmovlel %esi, %edi # sched: [2:0.50]
-; HASWELL-NEXT:    cmovlel %esi, %edi # sched: [2:0.50]
-; HASWELL-NEXT:    cmovgl %esi, %edi # sched: [2:0.50]
-; HASWELL-NEXT:    cmovgl %esi, %edi # sched: [2:0.50]
-; HASWELL-NEXT:    cmovol (%rdx), %edi # sched: [7:0.50]
-; HASWELL-NEXT:    cmovnol (%rdx), %edi # sched: [7:0.50]
-; HASWELL-NEXT:    cmovbl (%rdx), %edi # sched: [7:0.50]
-; HASWELL-NEXT:    cmovbl (%rdx), %edi # sched: [7:0.50]
-; HASWELL-NEXT:    cmovbl (%rdx), %edi # sched: [7:0.50]
-; HASWELL-NEXT:    cmovael (%rdx), %edi # sched: [7:0.50]
-; HASWELL-NEXT:    cmovael (%rdx), %edi # sched: [7:0.50]
-; HASWELL-NEXT:    cmovael (%rdx), %edi # sched: [7:0.50]
-; HASWELL-NEXT:    cmovel (%rdx), %edi # sched: [7:0.50]
-; HASWELL-NEXT:    cmovel (%rdx), %edi # sched: [7:0.50]
-; HASWELL-NEXT:    cmovnel (%rdx), %edi # sched: [7:0.50]
-; HASWELL-NEXT:    cmovnel (%rdx), %edi # sched: [7:0.50]
-; HASWELL-NEXT:    cmovbel (%rdx), %edi # sched: [8:0.75]
-; HASWELL-NEXT:    cmovbel (%rdx), %edi # sched: [8:0.75]
-; HASWELL-NEXT:    cmoval (%rdx), %edi # sched: [8:0.75]
-; HASWELL-NEXT:    cmoval (%rdx), %edi # sched: [8:0.75]
-; HASWELL-NEXT:    cmovsl (%rdx), %edi # sched: [7:0.50]
-; HASWELL-NEXT:    cmovnsl (%rdx), %edi # sched: [7:0.50]
-; HASWELL-NEXT:    cmovpl (%rdx), %edi # sched: [7:0.50]
-; HASWELL-NEXT:    cmovpl (%rdx), %edi # sched: [7:0.50]
-; HASWELL-NEXT:    cmovnpl (%rdx), %edi # sched: [7:0.50]
-; HASWELL-NEXT:    cmovnpl (%rdx), %edi # sched: [7:0.50]
-; HASWELL-NEXT:    cmovll (%rdx), %edi # sched: [7:0.50]
-; HASWELL-NEXT:    cmovll (%rdx), %edi # sched: [7:0.50]
-; HASWELL-NEXT:    cmovgel (%rdx), %edi # sched: [7:0.50]
-; HASWELL-NEXT:    cmovgel (%rdx), %edi # sched: [7:0.50]
-; HASWELL-NEXT:    cmovlel (%rdx), %edi # sched: [7:0.50]
-; HASWELL-NEXT:    cmovlel (%rdx), %edi # sched: [7:0.50]
-; HASWELL-NEXT:    cmovgl (%rdx), %edi # sched: [7:0.50]
-; HASWELL-NEXT:    cmovgl (%rdx), %edi # sched: [7:0.50]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_cmov_32:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    cmovol %esi, %edi # sched: [1:0.50]
-; BROADWELL-NEXT:    cmovnol %esi, %edi # sched: [1:0.50]
-; BROADWELL-NEXT:    cmovbl %esi, %edi # sched: [1:0.50]
-; BROADWELL-NEXT:    cmovbl %esi, %edi # sched: [1:0.50]
-; BROADWELL-NEXT:    cmovbl %esi, %edi # sched: [1:0.50]
-; BROADWELL-NEXT:    cmovael %esi, %edi # sched: [1:0.50]
-; BROADWELL-NEXT:    cmovael %esi, %edi # sched: [1:0.50]
-; BROADWELL-NEXT:    cmovael %esi, %edi # sched: [1:0.50]
-; BROADWELL-NEXT:    cmovel %esi, %edi # sched: [1:0.50]
-; BROADWELL-NEXT:    cmovel %esi, %edi # sched: [1:0.50]
-; BROADWELL-NEXT:    cmovnel %esi, %edi # sched: [1:0.50]
-; BROADWELL-NEXT:    cmovnel %esi, %edi # sched: [1:0.50]
-; BROADWELL-NEXT:    cmovbel %esi, %edi # sched: [2:0.50]
-; BROADWELL-NEXT:    cmovbel %esi, %edi # sched: [2:0.50]
-; BROADWELL-NEXT:    cmoval %esi, %edi # sched: [2:0.50]
-; BROADWELL-NEXT:    cmoval %esi, %edi # sched: [2:0.50]
-; BROADWELL-NEXT:    cmovsl %esi, %edi # sched: [1:0.50]
-; BROADWELL-NEXT:    cmovnsl %esi, %edi # sched: [1:0.50]
-; BROADWELL-NEXT:    cmovpl %esi, %edi # sched: [1:0.50]
-; BROADWELL-NEXT:    cmovpl %esi, %edi # sched: [1:0.50]
-; BROADWELL-NEXT:    cmovnpl %esi, %edi # sched: [1:0.50]
-; BROADWELL-NEXT:    cmovnpl %esi, %edi # sched: [1:0.50]
-; BROADWELL-NEXT:    cmovll %esi, %edi # sched: [1:0.50]
-; BROADWELL-NEXT:    cmovll %esi, %edi # sched: [1:0.50]
-; BROADWELL-NEXT:    cmovgel %esi, %edi # sched: [1:0.50]
-; BROADWELL-NEXT:    cmovgel %esi, %edi # sched: [1:0.50]
-; BROADWELL-NEXT:    cmovlel %esi, %edi # sched: [1:0.50]
-; BROADWELL-NEXT:    cmovlel %esi, %edi # sched: [1:0.50]
-; BROADWELL-NEXT:    cmovgl %esi, %edi # sched: [1:0.50]
-; BROADWELL-NEXT:    cmovgl %esi, %edi # sched: [1:0.50]
-; BROADWELL-NEXT:    cmovol (%rdx), %edi # sched: [6:0.50]
-; BROADWELL-NEXT:    cmovnol (%rdx), %edi # sched: [6:0.50]
-; BROADWELL-NEXT:    cmovbl (%rdx), %edi # sched: [6:0.50]
-; BROADWELL-NEXT:    cmovbl (%rdx), %edi # sched: [6:0.50]
-; BROADWELL-NEXT:    cmovbl (%rdx), %edi # sched: [6:0.50]
-; BROADWELL-NEXT:    cmovael (%rdx), %edi # sched: [6:0.50]
-; BROADWELL-NEXT:    cmovael (%rdx), %edi # sched: [6:0.50]
-; BROADWELL-NEXT:    cmovael (%rdx), %edi # sched: [6:0.50]
-; BROADWELL-NEXT:    cmovel (%rdx), %edi # sched: [6:0.50]
-; BROADWELL-NEXT:    cmovel (%rdx), %edi # sched: [6:0.50]
-; BROADWELL-NEXT:    cmovnel (%rdx), %edi # sched: [6:0.50]
-; BROADWELL-NEXT:    cmovnel (%rdx), %edi # sched: [6:0.50]
-; BROADWELL-NEXT:    cmovbel (%rdx), %edi # sched: [7:0.50]
-; BROADWELL-NEXT:    cmovbel (%rdx), %edi # sched: [7:0.50]
-; BROADWELL-NEXT:    cmoval (%rdx), %edi # sched: [7:0.50]
-; BROADWELL-NEXT:    cmoval (%rdx), %edi # sched: [7:0.50]
-; BROADWELL-NEXT:    cmovsl (%rdx), %edi # sched: [6:0.50]
-; BROADWELL-NEXT:    cmovnsl (%rdx), %edi # sched: [6:0.50]
-; BROADWELL-NEXT:    cmovpl (%rdx), %edi # sched: [6:0.50]
-; BROADWELL-NEXT:    cmovpl (%rdx), %edi # sched: [6:0.50]
-; BROADWELL-NEXT:    cmovnpl (%rdx), %edi # sched: [6:0.50]
-; BROADWELL-NEXT:    cmovnpl (%rdx), %edi # sched: [6:0.50]
-; BROADWELL-NEXT:    cmovll (%rdx), %edi # sched: [6:0.50]
-; BROADWELL-NEXT:    cmovll (%rdx), %edi # sched: [6:0.50]
-; BROADWELL-NEXT:    cmovgel (%rdx), %edi # sched: [6:0.50]
-; BROADWELL-NEXT:    cmovgel (%rdx), %edi # sched: [6:0.50]
-; BROADWELL-NEXT:    cmovlel (%rdx), %edi # sched: [6:0.50]
-; BROADWELL-NEXT:    cmovlel (%rdx), %edi # sched: [6:0.50]
-; BROADWELL-NEXT:    cmovgl (%rdx), %edi # sched: [6:0.50]
-; BROADWELL-NEXT:    cmovgl (%rdx), %edi # sched: [6:0.50]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_cmov_32:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    cmovol %esi, %edi # sched: [1:0.50]
-; SKYLAKE-NEXT:    cmovnol %esi, %edi # sched: [1:0.50]
-; SKYLAKE-NEXT:    cmovbl %esi, %edi # sched: [1:0.50]
-; SKYLAKE-NEXT:    cmovbl %esi, %edi # sched: [1:0.50]
-; SKYLAKE-NEXT:    cmovbl %esi, %edi # sched: [1:0.50]
-; SKYLAKE-NEXT:    cmovael %esi, %edi # sched: [1:0.50]
-; SKYLAKE-NEXT:    cmovael %esi, %edi # sched: [1:0.50]
-; SKYLAKE-NEXT:    cmovael %esi, %edi # sched: [1:0.50]
-; SKYLAKE-NEXT:    cmovel %esi, %edi # sched: [1:0.50]
-; SKYLAKE-NEXT:    cmovel %esi, %edi # sched: [1:0.50]
-; SKYLAKE-NEXT:    cmovnel %esi, %edi # sched: [1:0.50]
-; SKYLAKE-NEXT:    cmovnel %esi, %edi # sched: [1:0.50]
-; SKYLAKE-NEXT:    cmovbel %esi, %edi # sched: [2:1.00]
-; SKYLAKE-NEXT:    cmovbel %esi, %edi # sched: [2:1.00]
-; SKYLAKE-NEXT:    cmoval %esi, %edi # sched: [2:1.00]
-; SKYLAKE-NEXT:    cmoval %esi, %edi # sched: [2:1.00]
-; SKYLAKE-NEXT:    cmovsl %esi, %edi # sched: [1:0.50]
-; SKYLAKE-NEXT:    cmovnsl %esi, %edi # sched: [1:0.50]
-; SKYLAKE-NEXT:    cmovpl %esi, %edi # sched: [1:0.50]
-; SKYLAKE-NEXT:    cmovpl %esi, %edi # sched: [1:0.50]
-; SKYLAKE-NEXT:    cmovnpl %esi, %edi # sched: [1:0.50]
-; SKYLAKE-NEXT:    cmovnpl %esi, %edi # sched: [1:0.50]
-; SKYLAKE-NEXT:    cmovll %esi, %edi # sched: [1:0.50]
-; SKYLAKE-NEXT:    cmovll %esi, %edi # sched: [1:0.50]
-; SKYLAKE-NEXT:    cmovgel %esi, %edi # sched: [1:0.50]
-; SKYLAKE-NEXT:    cmovgel %esi, %edi # sched: [1:0.50]
-; SKYLAKE-NEXT:    cmovlel %esi, %edi # sched: [1:0.50]
-; SKYLAKE-NEXT:    cmovlel %esi, %edi # sched: [1:0.50]
-; SKYLAKE-NEXT:    cmovgl %esi, %edi # sched: [1:0.50]
-; SKYLAKE-NEXT:    cmovgl %esi, %edi # sched: [1:0.50]
-; SKYLAKE-NEXT:    cmovol (%rdx), %edi # sched: [6:0.50]
-; SKYLAKE-NEXT:    cmovnol (%rdx), %edi # sched: [6:0.50]
-; SKYLAKE-NEXT:    cmovbl (%rdx), %edi # sched: [6:0.50]
-; SKYLAKE-NEXT:    cmovbl (%rdx), %edi # sched: [6:0.50]
-; SKYLAKE-NEXT:    cmovbl (%rdx), %edi # sched: [6:0.50]
-; SKYLAKE-NEXT:    cmovael (%rdx), %edi # sched: [6:0.50]
-; SKYLAKE-NEXT:    cmovael (%rdx), %edi # sched: [6:0.50]
-; SKYLAKE-NEXT:    cmovael (%rdx), %edi # sched: [6:0.50]
-; SKYLAKE-NEXT:    cmovel (%rdx), %edi # sched: [6:0.50]
-; SKYLAKE-NEXT:    cmovel (%rdx), %edi # sched: [6:0.50]
-; SKYLAKE-NEXT:    cmovnel (%rdx), %edi # sched: [6:0.50]
-; SKYLAKE-NEXT:    cmovnel (%rdx), %edi # sched: [6:0.50]
-; SKYLAKE-NEXT:    cmovbel (%rdx), %edi # sched: [7:1.00]
-; SKYLAKE-NEXT:    cmovbel (%rdx), %edi # sched: [7:1.00]
-; SKYLAKE-NEXT:    cmoval (%rdx), %edi # sched: [7:1.00]
-; SKYLAKE-NEXT:    cmoval (%rdx), %edi # sched: [7:1.00]
-; SKYLAKE-NEXT:    cmovsl (%rdx), %edi # sched: [6:0.50]
-; SKYLAKE-NEXT:    cmovnsl (%rdx), %edi # sched: [6:0.50]
-; SKYLAKE-NEXT:    cmovpl (%rdx), %edi # sched: [6:0.50]
-; SKYLAKE-NEXT:    cmovpl (%rdx), %edi # sched: [6:0.50]
-; SKYLAKE-NEXT:    cmovnpl (%rdx), %edi # sched: [6:0.50]
-; SKYLAKE-NEXT:    cmovnpl (%rdx), %edi # sched: [6:0.50]
-; SKYLAKE-NEXT:    cmovll (%rdx), %edi # sched: [6:0.50]
-; SKYLAKE-NEXT:    cmovll (%rdx), %edi # sched: [6:0.50]
-; SKYLAKE-NEXT:    cmovgel (%rdx), %edi # sched: [6:0.50]
-; SKYLAKE-NEXT:    cmovgel (%rdx), %edi # sched: [6:0.50]
-; SKYLAKE-NEXT:    cmovlel (%rdx), %edi # sched: [6:0.50]
-; SKYLAKE-NEXT:    cmovlel (%rdx), %edi # sched: [6:0.50]
-; SKYLAKE-NEXT:    cmovgl (%rdx), %edi # sched: [6:0.50]
-; SKYLAKE-NEXT:    cmovgl (%rdx), %edi # sched: [6:0.50]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_cmov_32:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    cmovol %esi, %edi # sched: [1:0.50]
-; SKX-NEXT:    cmovnol %esi, %edi # sched: [1:0.50]
-; SKX-NEXT:    cmovbl %esi, %edi # sched: [1:0.50]
-; SKX-NEXT:    cmovbl %esi, %edi # sched: [1:0.50]
-; SKX-NEXT:    cmovbl %esi, %edi # sched: [1:0.50]
-; SKX-NEXT:    cmovael %esi, %edi # sched: [1:0.50]
-; SKX-NEXT:    cmovael %esi, %edi # sched: [1:0.50]
-; SKX-NEXT:    cmovael %esi, %edi # sched: [1:0.50]
-; SKX-NEXT:    cmovel %esi, %edi # sched: [1:0.50]
-; SKX-NEXT:    cmovel %esi, %edi # sched: [1:0.50]
-; SKX-NEXT:    cmovnel %esi, %edi # sched: [1:0.50]
-; SKX-NEXT:    cmovnel %esi, %edi # sched: [1:0.50]
-; SKX-NEXT:    cmovbel %esi, %edi # sched: [2:1.00]
-; SKX-NEXT:    cmovbel %esi, %edi # sched: [2:1.00]
-; SKX-NEXT:    cmoval %esi, %edi # sched: [2:1.00]
-; SKX-NEXT:    cmoval %esi, %edi # sched: [2:1.00]
-; SKX-NEXT:    cmovsl %esi, %edi # sched: [1:0.50]
-; SKX-NEXT:    cmovnsl %esi, %edi # sched: [1:0.50]
-; SKX-NEXT:    cmovpl %esi, %edi # sched: [1:0.50]
-; SKX-NEXT:    cmovpl %esi, %edi # sched: [1:0.50]
-; SKX-NEXT:    cmovnpl %esi, %edi # sched: [1:0.50]
-; SKX-NEXT:    cmovnpl %esi, %edi # sched: [1:0.50]
-; SKX-NEXT:    cmovll %esi, %edi # sched: [1:0.50]
-; SKX-NEXT:    cmovll %esi, %edi # sched: [1:0.50]
-; SKX-NEXT:    cmovgel %esi, %edi # sched: [1:0.50]
-; SKX-NEXT:    cmovgel %esi, %edi # sched: [1:0.50]
-; SKX-NEXT:    cmovlel %esi, %edi # sched: [1:0.50]
-; SKX-NEXT:    cmovlel %esi, %edi # sched: [1:0.50]
-; SKX-NEXT:    cmovgl %esi, %edi # sched: [1:0.50]
-; SKX-NEXT:    cmovgl %esi, %edi # sched: [1:0.50]
-; SKX-NEXT:    cmovol (%rdx), %edi # sched: [6:0.50]
-; SKX-NEXT:    cmovnol (%rdx), %edi # sched: [6:0.50]
-; SKX-NEXT:    cmovbl (%rdx), %edi # sched: [6:0.50]
-; SKX-NEXT:    cmovbl (%rdx), %edi # sched: [6:0.50]
-; SKX-NEXT:    cmovbl (%rdx), %edi # sched: [6:0.50]
-; SKX-NEXT:    cmovael (%rdx), %edi # sched: [6:0.50]
-; SKX-NEXT:    cmovael (%rdx), %edi # sched: [6:0.50]
-; SKX-NEXT:    cmovael (%rdx), %edi # sched: [6:0.50]
-; SKX-NEXT:    cmovel (%rdx), %edi # sched: [6:0.50]
-; SKX-NEXT:    cmovel (%rdx), %edi # sched: [6:0.50]
-; SKX-NEXT:    cmovnel (%rdx), %edi # sched: [6:0.50]
-; SKX-NEXT:    cmovnel (%rdx), %edi # sched: [6:0.50]
-; SKX-NEXT:    cmovbel (%rdx), %edi # sched: [7:1.00]
-; SKX-NEXT:    cmovbel (%rdx), %edi # sched: [7:1.00]
-; SKX-NEXT:    cmoval (%rdx), %edi # sched: [7:1.00]
-; SKX-NEXT:    cmoval (%rdx), %edi # sched: [7:1.00]
-; SKX-NEXT:    cmovsl (%rdx), %edi # sched: [6:0.50]
-; SKX-NEXT:    cmovnsl (%rdx), %edi # sched: [6:0.50]
-; SKX-NEXT:    cmovpl (%rdx), %edi # sched: [6:0.50]
-; SKX-NEXT:    cmovpl (%rdx), %edi # sched: [6:0.50]
-; SKX-NEXT:    cmovnpl (%rdx), %edi # sched: [6:0.50]
-; SKX-NEXT:    cmovnpl (%rdx), %edi # sched: [6:0.50]
-; SKX-NEXT:    cmovll (%rdx), %edi # sched: [6:0.50]
-; SKX-NEXT:    cmovll (%rdx), %edi # sched: [6:0.50]
-; SKX-NEXT:    cmovgel (%rdx), %edi # sched: [6:0.50]
-; SKX-NEXT:    cmovgel (%rdx), %edi # sched: [6:0.50]
-; SKX-NEXT:    cmovlel (%rdx), %edi # sched: [6:0.50]
-; SKX-NEXT:    cmovlel (%rdx), %edi # sched: [6:0.50]
-; SKX-NEXT:    cmovgl (%rdx), %edi # sched: [6:0.50]
-; SKX-NEXT:    cmovgl (%rdx), %edi # sched: [6:0.50]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_cmov_32:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    cmovol %esi, %edi # sched: [1:0.50]
-; BDVER2-NEXT:    cmovnol %esi, %edi # sched: [1:0.50]
-; BDVER2-NEXT:    cmovbl %esi, %edi # sched: [1:0.50]
-; BDVER2-NEXT:    cmovbl %esi, %edi # sched: [1:0.50]
-; BDVER2-NEXT:    cmovbl %esi, %edi # sched: [1:0.50]
-; BDVER2-NEXT:    cmovael %esi, %edi # sched: [1:0.50]
-; BDVER2-NEXT:    cmovael %esi, %edi # sched: [1:0.50]
-; BDVER2-NEXT:    cmovael %esi, %edi # sched: [1:0.50]
-; BDVER2-NEXT:    cmovel %esi, %edi # sched: [1:0.50]
-; BDVER2-NEXT:    cmovel %esi, %edi # sched: [1:0.50]
-; BDVER2-NEXT:    cmovnel %esi, %edi # sched: [1:0.50]
-; BDVER2-NEXT:    cmovnel %esi, %edi # sched: [1:0.50]
-; BDVER2-NEXT:    cmovbel %esi, %edi # sched: [1:0.50]
-; BDVER2-NEXT:    cmovbel %esi, %edi # sched: [1:0.50]
-; BDVER2-NEXT:    cmoval %esi, %edi # sched: [1:0.50]
-; BDVER2-NEXT:    cmoval %esi, %edi # sched: [1:0.50]
-; BDVER2-NEXT:    cmovsl %esi, %edi # sched: [1:0.50]
-; BDVER2-NEXT:    cmovnsl %esi, %edi # sched: [1:0.50]
-; BDVER2-NEXT:    cmovpl %esi, %edi # sched: [1:0.50]
-; BDVER2-NEXT:    cmovpl %esi, %edi # sched: [1:0.50]
-; BDVER2-NEXT:    cmovnpl %esi, %edi # sched: [1:0.50]
-; BDVER2-NEXT:    cmovnpl %esi, %edi # sched: [1:0.50]
-; BDVER2-NEXT:    cmovll %esi, %edi # sched: [1:0.50]
-; BDVER2-NEXT:    cmovll %esi, %edi # sched: [1:0.50]
-; BDVER2-NEXT:    cmovgel %esi, %edi # sched: [1:0.50]
-; BDVER2-NEXT:    cmovgel %esi, %edi # sched: [1:0.50]
-; BDVER2-NEXT:    cmovlel %esi, %edi # sched: [1:0.50]
-; BDVER2-NEXT:    cmovlel %esi, %edi # sched: [1:0.50]
-; BDVER2-NEXT:    cmovgl %esi, %edi # sched: [1:0.50]
-; BDVER2-NEXT:    cmovgl %esi, %edi # sched: [1:0.50]
-; BDVER2-NEXT:    cmovol (%rdx), %edi # sched: [5:0.50]
-; BDVER2-NEXT:    cmovnol (%rdx), %edi # sched: [5:0.50]
-; BDVER2-NEXT:    cmovbl (%rdx), %edi # sched: [5:0.50]
-; BDVER2-NEXT:    cmovbl (%rdx), %edi # sched: [5:0.50]
-; BDVER2-NEXT:    cmovbl (%rdx), %edi # sched: [5:0.50]
-; BDVER2-NEXT:    cmovael (%rdx), %edi # sched: [5:0.50]
-; BDVER2-NEXT:    cmovael (%rdx), %edi # sched: [5:0.50]
-; BDVER2-NEXT:    cmovael (%rdx), %edi # sched: [5:0.50]
-; BDVER2-NEXT:    cmovel (%rdx), %edi # sched: [5:0.50]
-; BDVER2-NEXT:    cmovel (%rdx), %edi # sched: [5:0.50]
-; BDVER2-NEXT:    cmovnel (%rdx), %edi # sched: [5:0.50]
-; BDVER2-NEXT:    cmovnel (%rdx), %edi # sched: [5:0.50]
-; BDVER2-NEXT:    cmovbel (%rdx), %edi # sched: [5:0.50]
-; BDVER2-NEXT:    cmovbel (%rdx), %edi # sched: [5:0.50]
-; BDVER2-NEXT:    cmoval (%rdx), %edi # sched: [5:0.50]
-; BDVER2-NEXT:    cmoval (%rdx), %edi # sched: [5:0.50]
-; BDVER2-NEXT:    cmovsl (%rdx), %edi # sched: [5:0.50]
-; BDVER2-NEXT:    cmovnsl (%rdx), %edi # sched: [5:0.50]
-; BDVER2-NEXT:    cmovpl (%rdx), %edi # sched: [5:0.50]
-; BDVER2-NEXT:    cmovpl (%rdx), %edi # sched: [5:0.50]
-; BDVER2-NEXT:    cmovnpl (%rdx), %edi # sched: [5:0.50]
-; BDVER2-NEXT:    cmovnpl (%rdx), %edi # sched: [5:0.50]
-; BDVER2-NEXT:    cmovll (%rdx), %edi # sched: [5:0.50]
-; BDVER2-NEXT:    cmovll (%rdx), %edi # sched: [5:0.50]
-; BDVER2-NEXT:    cmovgel (%rdx), %edi # sched: [5:0.50]
-; BDVER2-NEXT:    cmovgel (%rdx), %edi # sched: [5:0.50]
-; BDVER2-NEXT:    cmovlel (%rdx), %edi # sched: [5:0.50]
-; BDVER2-NEXT:    cmovlel (%rdx), %edi # sched: [5:0.50]
-; BDVER2-NEXT:    cmovgl (%rdx), %edi # sched: [5:0.50]
-; BDVER2-NEXT:    cmovgl (%rdx), %edi # sched: [5:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_cmov_32:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    cmovol %esi, %edi # sched: [1:0.50]
-; BTVER2-NEXT:    cmovnol %esi, %edi # sched: [1:0.50]
-; BTVER2-NEXT:    cmovbl %esi, %edi # sched: [1:0.50]
-; BTVER2-NEXT:    cmovbl %esi, %edi # sched: [1:0.50]
-; BTVER2-NEXT:    cmovbl %esi, %edi # sched: [1:0.50]
-; BTVER2-NEXT:    cmovael %esi, %edi # sched: [1:0.50]
-; BTVER2-NEXT:    cmovael %esi, %edi # sched: [1:0.50]
-; BTVER2-NEXT:    cmovael %esi, %edi # sched: [1:0.50]
-; BTVER2-NEXT:    cmovel %esi, %edi # sched: [1:0.50]
-; BTVER2-NEXT:    cmovel %esi, %edi # sched: [1:0.50]
-; BTVER2-NEXT:    cmovnel %esi, %edi # sched: [1:0.50]
-; BTVER2-NEXT:    cmovnel %esi, %edi # sched: [1:0.50]
-; BTVER2-NEXT:    cmovbel %esi, %edi # sched: [1:0.50]
-; BTVER2-NEXT:    cmovbel %esi, %edi # sched: [1:0.50]
-; BTVER2-NEXT:    cmoval %esi, %edi # sched: [1:0.50]
-; BTVER2-NEXT:    cmoval %esi, %edi # sched: [1:0.50]
-; BTVER2-NEXT:    cmovsl %esi, %edi # sched: [1:0.50]
-; BTVER2-NEXT:    cmovnsl %esi, %edi # sched: [1:0.50]
-; BTVER2-NEXT:    cmovpl %esi, %edi # sched: [1:0.50]
-; BTVER2-NEXT:    cmovpl %esi, %edi # sched: [1:0.50]
-; BTVER2-NEXT:    cmovnpl %esi, %edi # sched: [1:0.50]
-; BTVER2-NEXT:    cmovnpl %esi, %edi # sched: [1:0.50]
-; BTVER2-NEXT:    cmovll %esi, %edi # sched: [1:0.50]
-; BTVER2-NEXT:    cmovll %esi, %edi # sched: [1:0.50]
-; BTVER2-NEXT:    cmovgel %esi, %edi # sched: [1:0.50]
-; BTVER2-NEXT:    cmovgel %esi, %edi # sched: [1:0.50]
-; BTVER2-NEXT:    cmovlel %esi, %edi # sched: [1:0.50]
-; BTVER2-NEXT:    cmovlel %esi, %edi # sched: [1:0.50]
-; BTVER2-NEXT:    cmovgl %esi, %edi # sched: [1:0.50]
-; BTVER2-NEXT:    cmovgl %esi, %edi # sched: [1:0.50]
-; BTVER2-NEXT:    cmovol (%rdx), %edi # sched: [4:1.00]
-; BTVER2-NEXT:    cmovnol (%rdx), %edi # sched: [4:1.00]
-; BTVER2-NEXT:    cmovbl (%rdx), %edi # sched: [4:1.00]
-; BTVER2-NEXT:    cmovbl (%rdx), %edi # sched: [4:1.00]
-; BTVER2-NEXT:    cmovbl (%rdx), %edi # sched: [4:1.00]
-; BTVER2-NEXT:    cmovael (%rdx), %edi # sched: [4:1.00]
-; BTVER2-NEXT:    cmovael (%rdx), %edi # sched: [4:1.00]
-; BTVER2-NEXT:    cmovael (%rdx), %edi # sched: [4:1.00]
-; BTVER2-NEXT:    cmovel (%rdx), %edi # sched: [4:1.00]
-; BTVER2-NEXT:    cmovel (%rdx), %edi # sched: [4:1.00]
-; BTVER2-NEXT:    cmovnel (%rdx), %edi # sched: [4:1.00]
-; BTVER2-NEXT:    cmovnel (%rdx), %edi # sched: [4:1.00]
-; BTVER2-NEXT:    cmovbel (%rdx), %edi # sched: [4:1.00]
-; BTVER2-NEXT:    cmovbel (%rdx), %edi # sched: [4:1.00]
-; BTVER2-NEXT:    cmoval (%rdx), %edi # sched: [4:1.00]
-; BTVER2-NEXT:    cmoval (%rdx), %edi # sched: [4:1.00]
-; BTVER2-NEXT:    cmovsl (%rdx), %edi # sched: [4:1.00]
-; BTVER2-NEXT:    cmovnsl (%rdx), %edi # sched: [4:1.00]
-; BTVER2-NEXT:    cmovpl (%rdx), %edi # sched: [4:1.00]
-; BTVER2-NEXT:    cmovpl (%rdx), %edi # sched: [4:1.00]
-; BTVER2-NEXT:    cmovnpl (%rdx), %edi # sched: [4:1.00]
-; BTVER2-NEXT:    cmovnpl (%rdx), %edi # sched: [4:1.00]
-; BTVER2-NEXT:    cmovll (%rdx), %edi # sched: [4:1.00]
-; BTVER2-NEXT:    cmovll (%rdx), %edi # sched: [4:1.00]
-; BTVER2-NEXT:    cmovgel (%rdx), %edi # sched: [4:1.00]
-; BTVER2-NEXT:    cmovgel (%rdx), %edi # sched: [4:1.00]
-; BTVER2-NEXT:    cmovlel (%rdx), %edi # sched: [4:1.00]
-; BTVER2-NEXT:    cmovlel (%rdx), %edi # sched: [4:1.00]
-; BTVER2-NEXT:    cmovgl (%rdx), %edi # sched: [4:1.00]
-; BTVER2-NEXT:    cmovgl (%rdx), %edi # sched: [4:1.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_cmov_32:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    cmovol %esi, %edi # sched: [1:0.25]
-; ZNVER1-NEXT:    cmovnol %esi, %edi # sched: [1:0.25]
-; ZNVER1-NEXT:    cmovbl %esi, %edi # sched: [1:0.25]
-; ZNVER1-NEXT:    cmovbl %esi, %edi # sched: [1:0.25]
-; ZNVER1-NEXT:    cmovbl %esi, %edi # sched: [1:0.25]
-; ZNVER1-NEXT:    cmovael %esi, %edi # sched: [1:0.25]
-; ZNVER1-NEXT:    cmovael %esi, %edi # sched: [1:0.25]
-; ZNVER1-NEXT:    cmovael %esi, %edi # sched: [1:0.25]
-; ZNVER1-NEXT:    cmovel %esi, %edi # sched: [1:0.25]
-; ZNVER1-NEXT:    cmovel %esi, %edi # sched: [1:0.25]
-; ZNVER1-NEXT:    cmovnel %esi, %edi # sched: [1:0.25]
-; ZNVER1-NEXT:    cmovnel %esi, %edi # sched: [1:0.25]
-; ZNVER1-NEXT:    cmovbel %esi, %edi # sched: [1:0.25]
-; ZNVER1-NEXT:    cmovbel %esi, %edi # sched: [1:0.25]
-; ZNVER1-NEXT:    cmoval %esi, %edi # sched: [1:0.25]
-; ZNVER1-NEXT:    cmoval %esi, %edi # sched: [1:0.25]
-; ZNVER1-NEXT:    cmovsl %esi, %edi # sched: [1:0.25]
-; ZNVER1-NEXT:    cmovnsl %esi, %edi # sched: [1:0.25]
-; ZNVER1-NEXT:    cmovpl %esi, %edi # sched: [1:0.25]
-; ZNVER1-NEXT:    cmovpl %esi, %edi # sched: [1:0.25]
-; ZNVER1-NEXT:    cmovnpl %esi, %edi # sched: [1:0.25]
-; ZNVER1-NEXT:    cmovnpl %esi, %edi # sched: [1:0.25]
-; ZNVER1-NEXT:    cmovll %esi, %edi # sched: [1:0.25]
-; ZNVER1-NEXT:    cmovll %esi, %edi # sched: [1:0.25]
-; ZNVER1-NEXT:    cmovgel %esi, %edi # sched: [1:0.25]
-; ZNVER1-NEXT:    cmovgel %esi, %edi # sched: [1:0.25]
-; ZNVER1-NEXT:    cmovlel %esi, %edi # sched: [1:0.25]
-; ZNVER1-NEXT:    cmovlel %esi, %edi # sched: [1:0.25]
-; ZNVER1-NEXT:    cmovgl %esi, %edi # sched: [1:0.25]
-; ZNVER1-NEXT:    cmovgl %esi, %edi # sched: [1:0.25]
-; ZNVER1-NEXT:    cmovol (%rdx), %edi # sched: [5:0.50]
-; ZNVER1-NEXT:    cmovnol (%rdx), %edi # sched: [5:0.50]
-; ZNVER1-NEXT:    cmovbl (%rdx), %edi # sched: [5:0.50]
-; ZNVER1-NEXT:    cmovbl (%rdx), %edi # sched: [5:0.50]
-; ZNVER1-NEXT:    cmovbl (%rdx), %edi # sched: [5:0.50]
-; ZNVER1-NEXT:    cmovael (%rdx), %edi # sched: [5:0.50]
-; ZNVER1-NEXT:    cmovael (%rdx), %edi # sched: [5:0.50]
-; ZNVER1-NEXT:    cmovael (%rdx), %edi # sched: [5:0.50]
-; ZNVER1-NEXT:    cmovel (%rdx), %edi # sched: [5:0.50]
-; ZNVER1-NEXT:    cmovel (%rdx), %edi # sched: [5:0.50]
-; ZNVER1-NEXT:    cmovnel (%rdx), %edi # sched: [5:0.50]
-; ZNVER1-NEXT:    cmovnel (%rdx), %edi # sched: [5:0.50]
-; ZNVER1-NEXT:    cmovbel (%rdx), %edi # sched: [5:0.50]
-; ZNVER1-NEXT:    cmovbel (%rdx), %edi # sched: [5:0.50]
-; ZNVER1-NEXT:    cmoval (%rdx), %edi # sched: [5:0.50]
-; ZNVER1-NEXT:    cmoval (%rdx), %edi # sched: [5:0.50]
-; ZNVER1-NEXT:    cmovsl (%rdx), %edi # sched: [5:0.50]
-; ZNVER1-NEXT:    cmovnsl (%rdx), %edi # sched: [5:0.50]
-; ZNVER1-NEXT:    cmovpl (%rdx), %edi # sched: [5:0.50]
-; ZNVER1-NEXT:    cmovpl (%rdx), %edi # sched: [5:0.50]
-; ZNVER1-NEXT:    cmovnpl (%rdx), %edi # sched: [5:0.50]
-; ZNVER1-NEXT:    cmovnpl (%rdx), %edi # sched: [5:0.50]
-; ZNVER1-NEXT:    cmovll (%rdx), %edi # sched: [5:0.50]
-; ZNVER1-NEXT:    cmovll (%rdx), %edi # sched: [5:0.50]
-; ZNVER1-NEXT:    cmovgel (%rdx), %edi # sched: [5:0.50]
-; ZNVER1-NEXT:    cmovgel (%rdx), %edi # sched: [5:0.50]
-; ZNVER1-NEXT:    cmovlel (%rdx), %edi # sched: [5:0.50]
-; ZNVER1-NEXT:    cmovlel (%rdx), %edi # sched: [5:0.50]
-; ZNVER1-NEXT:    cmovgl (%rdx), %edi # sched: [5:0.50]
-; ZNVER1-NEXT:    cmovgl (%rdx), %edi # sched: [5:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  call void asm sideeffect "cmovol $1, $0 \0A\09 cmovnol $1, $0 \0A\09 cmovbl $1, $0 \0A\09 cmovcl $1, $0 \0A\09 cmovnael $1, $0 \0A\09 cmovnbl $1, $0 \0A\09 cmovncl $1, $0 \0A\09 cmovael $1, $0 \0A\09 cmovzl $1, $0 \0A\09 cmovel $1, $0 \0A\09 cmovnzl $1, $0 \0A\09 cmovnel $1, $0 \0A\09 cmovbel $1, $0 \0A\09 cmovnal $1, $0 \0A\09 cmovnbel $1, $0 \0A\09 cmoval $1, $0 \0A\09 cmovsl $1, $0 \0A\09 cmovnsl $1, $0 \0A\09 cmovpl $1, $0 \0A\09 cmovpel $1, $0 \0A\09 cmovnpl $1, $0 \0A\09 cmovpol $1, $0 \0A\09 cmovll $1, $0 \0A\09 cmovngel $1, $0 \0A\09 cmovnll $1, $0 \0A\09 cmovgel $1, $0 \0A\09 cmovlel $1, $0 \0A\09 cmovngl $1, $0 \0A\09 cmovnlel $1, $0 \0A\09 cmovgl $1, $0 \0A\09 cmovol $2, $0 \0A\09 cmovnol $2, $0 \0A\09 cmovbl $2, $0 \0A\09 cmovcl $2, $0 \0A\09 cmovnael $2, $0 \0A\09 cmovnbl $2, $0 \0A\09 cmovncl $2, $0 \0A\09 cmovael $2, $0 \0A\09 cmovzl $2, $0 \0A\09 cmovel $2, $0 \0A\09 cmovnzl $2, $0 \0A\09 cmovnel $2, $0 \0A\09 cmovbel $2, $0 \0A\09 cmovnal $2, $0 \0A\09 cmovnbel $2, $0 \0A\09 cmoval $2, $0 \0A\09 cmovsl $2, $0 \0A\09 cmovnsl $2, $0 \0A\09 cmovpl $2, $0 \0A\09 cmovpel $2, $0 \0A\09 cmovnpl $2, $0 \0A\09 cmovpol $2, $0 \0A\09 cmovll $2, $0 \0A\09 cmovngel $2, $0 \0A\09 cmovnll $2, $0 \0A\09 cmovgel $2, $0 \0A\09 cmovlel $2, $0 \0A\09 cmovngl $2, $0 \0A\09 cmovnlel $2, $0 \0A\09 cmovgl $2, $0", "r,r,*m"(i32 %a0, i32 %a1, i32 *%a2)
-  ret void
-}
-
-define void @test_cmov_64(i64 %a0, i64 %a1, i64 *%a2) optsize {
-; GENERIC-LABEL: test_cmov_64:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    cmovoq %rsi, %rdi # sched: [2:0.67]
-; GENERIC-NEXT:    cmovnoq %rsi, %rdi # sched: [2:0.67]
-; GENERIC-NEXT:    cmovbq %rsi, %rdi # sched: [2:0.67]
-; GENERIC-NEXT:    cmovbq %rsi, %rdi # sched: [2:0.67]
-; GENERIC-NEXT:    cmovbq %rsi, %rdi # sched: [2:0.67]
-; GENERIC-NEXT:    cmovaeq %rsi, %rdi # sched: [2:0.67]
-; GENERIC-NEXT:    cmovaeq %rsi, %rdi # sched: [2:0.67]
-; GENERIC-NEXT:    cmovaeq %rsi, %rdi # sched: [2:0.67]
-; GENERIC-NEXT:    cmoveq %rsi, %rdi # sched: [2:0.67]
-; GENERIC-NEXT:    cmoveq %rsi, %rdi # sched: [2:0.67]
-; GENERIC-NEXT:    cmovneq %rsi, %rdi # sched: [2:0.67]
-; GENERIC-NEXT:    cmovneq %rsi, %rdi # sched: [2:0.67]
-; GENERIC-NEXT:    cmovbeq %rsi, %rdi # sched: [3:1.00]
-; GENERIC-NEXT:    cmovbeq %rsi, %rdi # sched: [3:1.00]
-; GENERIC-NEXT:    cmovaq %rsi, %rdi # sched: [3:1.00]
-; GENERIC-NEXT:    cmovaq %rsi, %rdi # sched: [3:1.00]
-; GENERIC-NEXT:    cmovsq %rsi, %rdi # sched: [2:0.67]
-; GENERIC-NEXT:    cmovnsq %rsi, %rdi # sched: [2:0.67]
-; GENERIC-NEXT:    cmovpq %rsi, %rdi # sched: [2:0.67]
-; GENERIC-NEXT:    cmovpq %rsi, %rdi # sched: [2:0.67]
-; GENERIC-NEXT:    cmovnpq %rsi, %rdi # sched: [2:0.67]
-; GENERIC-NEXT:    cmovnpq %rsi, %rdi # sched: [2:0.67]
-; GENERIC-NEXT:    cmovlq %rsi, %rdi # sched: [2:0.67]
-; GENERIC-NEXT:    cmovlq %rsi, %rdi # sched: [2:0.67]
-; GENERIC-NEXT:    cmovgeq %rsi, %rdi # sched: [2:0.67]
-; GENERIC-NEXT:    cmovgeq %rsi, %rdi # sched: [2:0.67]
-; GENERIC-NEXT:    cmovleq %rsi, %rdi # sched: [2:0.67]
-; GENERIC-NEXT:    cmovleq %rsi, %rdi # sched: [2:0.67]
-; GENERIC-NEXT:    cmovgq %rsi, %rdi # sched: [2:0.67]
-; GENERIC-NEXT:    cmovgq %rsi, %rdi # sched: [2:0.67]
-; GENERIC-NEXT:    cmovoq (%rdx), %rdi # sched: [7:0.67]
-; GENERIC-NEXT:    cmovnoq (%rdx), %rdi # sched: [7:0.67]
-; GENERIC-NEXT:    cmovbq (%rdx), %rdi # sched: [7:0.67]
-; GENERIC-NEXT:    cmovbq (%rdx), %rdi # sched: [7:0.67]
-; GENERIC-NEXT:    cmovbq (%rdx), %rdi # sched: [7:0.67]
-; GENERIC-NEXT:    cmovaeq (%rdx), %rdi # sched: [7:0.67]
-; GENERIC-NEXT:    cmovaeq (%rdx), %rdi # sched: [7:0.67]
-; GENERIC-NEXT:    cmovaeq (%rdx), %rdi # sched: [7:0.67]
-; GENERIC-NEXT:    cmoveq (%rdx), %rdi # sched: [7:0.67]
-; GENERIC-NEXT:    cmoveq (%rdx), %rdi # sched: [7:0.67]
-; GENERIC-NEXT:    cmovneq (%rdx), %rdi # sched: [7:0.67]
-; GENERIC-NEXT:    cmovneq (%rdx), %rdi # sched: [7:0.67]
-; GENERIC-NEXT:    cmovbeq (%rdx), %rdi # sched: [8:1.00]
-; GENERIC-NEXT:    cmovbeq (%rdx), %rdi # sched: [8:1.00]
-; GENERIC-NEXT:    cmovaq (%rdx), %rdi # sched: [8:1.00]
-; GENERIC-NEXT:    cmovaq (%rdx), %rdi # sched: [8:1.00]
-; GENERIC-NEXT:    cmovsq (%rdx), %rdi # sched: [7:0.67]
-; GENERIC-NEXT:    cmovnsq (%rdx), %rdi # sched: [7:0.67]
-; GENERIC-NEXT:    cmovpq (%rdx), %rdi # sched: [7:0.67]
-; GENERIC-NEXT:    cmovpq (%rdx), %rdi # sched: [7:0.67]
-; GENERIC-NEXT:    cmovnpq (%rdx), %rdi # sched: [7:0.67]
-; GENERIC-NEXT:    cmovnpq (%rdx), %rdi # sched: [7:0.67]
-; GENERIC-NEXT:    cmovlq (%rdx), %rdi # sched: [7:0.67]
-; GENERIC-NEXT:    cmovlq (%rdx), %rdi # sched: [7:0.67]
-; GENERIC-NEXT:    cmovgeq (%rdx), %rdi # sched: [7:0.67]
-; GENERIC-NEXT:    cmovgeq (%rdx), %rdi # sched: [7:0.67]
-; GENERIC-NEXT:    cmovleq (%rdx), %rdi # sched: [7:0.67]
-; GENERIC-NEXT:    cmovleq (%rdx), %rdi # sched: [7:0.67]
-; GENERIC-NEXT:    cmovgq (%rdx), %rdi # sched: [7:0.67]
-; GENERIC-NEXT:    cmovgq (%rdx), %rdi # sched: [7:0.67]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_cmov_64:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    cmovoq %rsi, %rdi # sched: [1:0.50]
-; ATOM-NEXT:    cmovnoq %rsi, %rdi # sched: [1:0.50]
-; ATOM-NEXT:    cmovbq %rsi, %rdi # sched: [1:0.50]
-; ATOM-NEXT:    cmovbq %rsi, %rdi # sched: [1:0.50]
-; ATOM-NEXT:    cmovbq %rsi, %rdi # sched: [1:0.50]
-; ATOM-NEXT:    cmovaeq %rsi, %rdi # sched: [1:0.50]
-; ATOM-NEXT:    cmovaeq %rsi, %rdi # sched: [1:0.50]
-; ATOM-NEXT:    cmovaeq %rsi, %rdi # sched: [1:0.50]
-; ATOM-NEXT:    cmoveq %rsi, %rdi # sched: [1:0.50]
-; ATOM-NEXT:    cmoveq %rsi, %rdi # sched: [1:0.50]
-; ATOM-NEXT:    cmovneq %rsi, %rdi # sched: [1:0.50]
-; ATOM-NEXT:    cmovneq %rsi, %rdi # sched: [1:0.50]
-; ATOM-NEXT:    cmovbeq %rsi, %rdi # sched: [1:0.50]
-; ATOM-NEXT:    cmovbeq %rsi, %rdi # sched: [1:0.50]
-; ATOM-NEXT:    cmovaq %rsi, %rdi # sched: [1:0.50]
-; ATOM-NEXT:    cmovaq %rsi, %rdi # sched: [1:0.50]
-; ATOM-NEXT:    cmovsq %rsi, %rdi # sched: [1:0.50]
-; ATOM-NEXT:    cmovnsq %rsi, %rdi # sched: [1:0.50]
-; ATOM-NEXT:    cmovpq %rsi, %rdi # sched: [1:0.50]
-; ATOM-NEXT:    cmovpq %rsi, %rdi # sched: [1:0.50]
-; ATOM-NEXT:    cmovnpq %rsi, %rdi # sched: [1:0.50]
-; ATOM-NEXT:    cmovnpq %rsi, %rdi # sched: [1:0.50]
-; ATOM-NEXT:    cmovlq %rsi, %rdi # sched: [1:0.50]
-; ATOM-NEXT:    cmovlq %rsi, %rdi # sched: [1:0.50]
-; ATOM-NEXT:    cmovgeq %rsi, %rdi # sched: [1:0.50]
-; ATOM-NEXT:    cmovgeq %rsi, %rdi # sched: [1:0.50]
-; ATOM-NEXT:    cmovleq %rsi, %rdi # sched: [1:0.50]
-; ATOM-NEXT:    cmovleq %rsi, %rdi # sched: [1:0.50]
-; ATOM-NEXT:    cmovgq %rsi, %rdi # sched: [1:0.50]
-; ATOM-NEXT:    cmovgq %rsi, %rdi # sched: [1:0.50]
-; ATOM-NEXT:    cmovoq (%rdx), %rdi # sched: [1:1.00]
-; ATOM-NEXT:    cmovnoq (%rdx), %rdi # sched: [1:1.00]
-; ATOM-NEXT:    cmovbq (%rdx), %rdi # sched: [1:1.00]
-; ATOM-NEXT:    cmovbq (%rdx), %rdi # sched: [1:1.00]
-; ATOM-NEXT:    cmovbq (%rdx), %rdi # sched: [1:1.00]
-; ATOM-NEXT:    cmovaeq (%rdx), %rdi # sched: [1:1.00]
-; ATOM-NEXT:    cmovaeq (%rdx), %rdi # sched: [1:1.00]
-; ATOM-NEXT:    cmovaeq (%rdx), %rdi # sched: [1:1.00]
-; ATOM-NEXT:    cmoveq (%rdx), %rdi # sched: [1:1.00]
-; ATOM-NEXT:    cmoveq (%rdx), %rdi # sched: [1:1.00]
-; ATOM-NEXT:    cmovneq (%rdx), %rdi # sched: [1:1.00]
-; ATOM-NEXT:    cmovneq (%rdx), %rdi # sched: [1:1.00]
-; ATOM-NEXT:    cmovbeq (%rdx), %rdi # sched: [1:1.00]
-; ATOM-NEXT:    cmovbeq (%rdx), %rdi # sched: [1:1.00]
-; ATOM-NEXT:    cmovaq (%rdx), %rdi # sched: [1:1.00]
-; ATOM-NEXT:    cmovaq (%rdx), %rdi # sched: [1:1.00]
-; ATOM-NEXT:    cmovsq (%rdx), %rdi # sched: [1:1.00]
-; ATOM-NEXT:    cmovnsq (%rdx), %rdi # sched: [1:1.00]
-; ATOM-NEXT:    cmovpq (%rdx), %rdi # sched: [1:1.00]
-; ATOM-NEXT:    cmovpq (%rdx), %rdi # sched: [1:1.00]
-; ATOM-NEXT:    cmovnpq (%rdx), %rdi # sched: [1:1.00]
-; ATOM-NEXT:    cmovnpq (%rdx), %rdi # sched: [1:1.00]
-; ATOM-NEXT:    cmovlq (%rdx), %rdi # sched: [1:1.00]
-; ATOM-NEXT:    cmovlq (%rdx), %rdi # sched: [1:1.00]
-; ATOM-NEXT:    cmovgeq (%rdx), %rdi # sched: [1:1.00]
-; ATOM-NEXT:    cmovgeq (%rdx), %rdi # sched: [1:1.00]
-; ATOM-NEXT:    cmovleq (%rdx), %rdi # sched: [1:1.00]
-; ATOM-NEXT:    cmovleq (%rdx), %rdi # sched: [1:1.00]
-; ATOM-NEXT:    cmovgq (%rdx), %rdi # sched: [1:1.00]
-; ATOM-NEXT:    cmovgq (%rdx), %rdi # sched: [1:1.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_cmov_64:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    cmovoq %rsi, %rdi # sched: [2:1.00]
-; SLM-NEXT:    cmovnoq %rsi, %rdi # sched: [2:1.00]
-; SLM-NEXT:    cmovbq %rsi, %rdi # sched: [2:1.00]
-; SLM-NEXT:    cmovbq %rsi, %rdi # sched: [2:1.00]
-; SLM-NEXT:    cmovbq %rsi, %rdi # sched: [2:1.00]
-; SLM-NEXT:    cmovaeq %rsi, %rdi # sched: [2:1.00]
-; SLM-NEXT:    cmovaeq %rsi, %rdi # sched: [2:1.00]
-; SLM-NEXT:    cmovaeq %rsi, %rdi # sched: [2:1.00]
-; SLM-NEXT:    cmoveq %rsi, %rdi # sched: [2:1.00]
-; SLM-NEXT:    cmoveq %rsi, %rdi # sched: [2:1.00]
-; SLM-NEXT:    cmovneq %rsi, %rdi # sched: [2:1.00]
-; SLM-NEXT:    cmovneq %rsi, %rdi # sched: [2:1.00]
-; SLM-NEXT:    cmovbeq %rsi, %rdi # sched: [2:1.00]
-; SLM-NEXT:    cmovbeq %rsi, %rdi # sched: [2:1.00]
-; SLM-NEXT:    cmovaq %rsi, %rdi # sched: [2:1.00]
-; SLM-NEXT:    cmovaq %rsi, %rdi # sched: [2:1.00]
-; SLM-NEXT:    cmovsq %rsi, %rdi # sched: [2:1.00]
-; SLM-NEXT:    cmovnsq %rsi, %rdi # sched: [2:1.00]
-; SLM-NEXT:    cmovpq %rsi, %rdi # sched: [2:1.00]
-; SLM-NEXT:    cmovpq %rsi, %rdi # sched: [2:1.00]
-; SLM-NEXT:    cmovnpq %rsi, %rdi # sched: [2:1.00]
-; SLM-NEXT:    cmovnpq %rsi, %rdi # sched: [2:1.00]
-; SLM-NEXT:    cmovlq %rsi, %rdi # sched: [2:1.00]
-; SLM-NEXT:    cmovlq %rsi, %rdi # sched: [2:1.00]
-; SLM-NEXT:    cmovgeq %rsi, %rdi # sched: [2:1.00]
-; SLM-NEXT:    cmovgeq %rsi, %rdi # sched: [2:1.00]
-; SLM-NEXT:    cmovleq %rsi, %rdi # sched: [2:1.00]
-; SLM-NEXT:    cmovleq %rsi, %rdi # sched: [2:1.00]
-; SLM-NEXT:    cmovgq %rsi, %rdi # sched: [2:1.00]
-; SLM-NEXT:    cmovgq %rsi, %rdi # sched: [2:1.00]
-; SLM-NEXT:    cmovoq (%rdx), %rdi # sched: [5:1.00]
-; SLM-NEXT:    cmovnoq (%rdx), %rdi # sched: [5:1.00]
-; SLM-NEXT:    cmovbq (%rdx), %rdi # sched: [5:1.00]
-; SLM-NEXT:    cmovbq (%rdx), %rdi # sched: [5:1.00]
-; SLM-NEXT:    cmovbq (%rdx), %rdi # sched: [5:1.00]
-; SLM-NEXT:    cmovaeq (%rdx), %rdi # sched: [5:1.00]
-; SLM-NEXT:    cmovaeq (%rdx), %rdi # sched: [5:1.00]
-; SLM-NEXT:    cmovaeq (%rdx), %rdi # sched: [5:1.00]
-; SLM-NEXT:    cmoveq (%rdx), %rdi # sched: [5:1.00]
-; SLM-NEXT:    cmoveq (%rdx), %rdi # sched: [5:1.00]
-; SLM-NEXT:    cmovneq (%rdx), %rdi # sched: [5:1.00]
-; SLM-NEXT:    cmovneq (%rdx), %rdi # sched: [5:1.00]
-; SLM-NEXT:    cmovbeq (%rdx), %rdi # sched: [5:1.00]
-; SLM-NEXT:    cmovbeq (%rdx), %rdi # sched: [5:1.00]
-; SLM-NEXT:    cmovaq (%rdx), %rdi # sched: [5:1.00]
-; SLM-NEXT:    cmovaq (%rdx), %rdi # sched: [5:1.00]
-; SLM-NEXT:    cmovsq (%rdx), %rdi # sched: [5:1.00]
-; SLM-NEXT:    cmovnsq (%rdx), %rdi # sched: [5:1.00]
-; SLM-NEXT:    cmovpq (%rdx), %rdi # sched: [5:1.00]
-; SLM-NEXT:    cmovpq (%rdx), %rdi # sched: [5:1.00]
-; SLM-NEXT:    cmovnpq (%rdx), %rdi # sched: [5:1.00]
-; SLM-NEXT:    cmovnpq (%rdx), %rdi # sched: [5:1.00]
-; SLM-NEXT:    cmovlq (%rdx), %rdi # sched: [5:1.00]
-; SLM-NEXT:    cmovlq (%rdx), %rdi # sched: [5:1.00]
-; SLM-NEXT:    cmovgeq (%rdx), %rdi # sched: [5:1.00]
-; SLM-NEXT:    cmovgeq (%rdx), %rdi # sched: [5:1.00]
-; SLM-NEXT:    cmovleq (%rdx), %rdi # sched: [5:1.00]
-; SLM-NEXT:    cmovleq (%rdx), %rdi # sched: [5:1.00]
-; SLM-NEXT:    cmovgq (%rdx), %rdi # sched: [5:1.00]
-; SLM-NEXT:    cmovgq (%rdx), %rdi # sched: [5:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_cmov_64:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    cmovoq %rsi, %rdi # sched: [2:0.67]
-; SANDY-NEXT:    cmovnoq %rsi, %rdi # sched: [2:0.67]
-; SANDY-NEXT:    cmovbq %rsi, %rdi # sched: [2:0.67]
-; SANDY-NEXT:    cmovbq %rsi, %rdi # sched: [2:0.67]
-; SANDY-NEXT:    cmovbq %rsi, %rdi # sched: [2:0.67]
-; SANDY-NEXT:    cmovaeq %rsi, %rdi # sched: [2:0.67]
-; SANDY-NEXT:    cmovaeq %rsi, %rdi # sched: [2:0.67]
-; SANDY-NEXT:    cmovaeq %rsi, %rdi # sched: [2:0.67]
-; SANDY-NEXT:    cmoveq %rsi, %rdi # sched: [2:0.67]
-; SANDY-NEXT:    cmoveq %rsi, %rdi # sched: [2:0.67]
-; SANDY-NEXT:    cmovneq %rsi, %rdi # sched: [2:0.67]
-; SANDY-NEXT:    cmovneq %rsi, %rdi # sched: [2:0.67]
-; SANDY-NEXT:    cmovbeq %rsi, %rdi # sched: [3:1.00]
-; SANDY-NEXT:    cmovbeq %rsi, %rdi # sched: [3:1.00]
-; SANDY-NEXT:    cmovaq %rsi, %rdi # sched: [3:1.00]
-; SANDY-NEXT:    cmovaq %rsi, %rdi # sched: [3:1.00]
-; SANDY-NEXT:    cmovsq %rsi, %rdi # sched: [2:0.67]
-; SANDY-NEXT:    cmovnsq %rsi, %rdi # sched: [2:0.67]
-; SANDY-NEXT:    cmovpq %rsi, %rdi # sched: [2:0.67]
-; SANDY-NEXT:    cmovpq %rsi, %rdi # sched: [2:0.67]
-; SANDY-NEXT:    cmovnpq %rsi, %rdi # sched: [2:0.67]
-; SANDY-NEXT:    cmovnpq %rsi, %rdi # sched: [2:0.67]
-; SANDY-NEXT:    cmovlq %rsi, %rdi # sched: [2:0.67]
-; SANDY-NEXT:    cmovlq %rsi, %rdi # sched: [2:0.67]
-; SANDY-NEXT:    cmovgeq %rsi, %rdi # sched: [2:0.67]
-; SANDY-NEXT:    cmovgeq %rsi, %rdi # sched: [2:0.67]
-; SANDY-NEXT:    cmovleq %rsi, %rdi # sched: [2:0.67]
-; SANDY-NEXT:    cmovleq %rsi, %rdi # sched: [2:0.67]
-; SANDY-NEXT:    cmovgq %rsi, %rdi # sched: [2:0.67]
-; SANDY-NEXT:    cmovgq %rsi, %rdi # sched: [2:0.67]
-; SANDY-NEXT:    cmovoq (%rdx), %rdi # sched: [7:0.67]
-; SANDY-NEXT:    cmovnoq (%rdx), %rdi # sched: [7:0.67]
-; SANDY-NEXT:    cmovbq (%rdx), %rdi # sched: [7:0.67]
-; SANDY-NEXT:    cmovbq (%rdx), %rdi # sched: [7:0.67]
-; SANDY-NEXT:    cmovbq (%rdx), %rdi # sched: [7:0.67]
-; SANDY-NEXT:    cmovaeq (%rdx), %rdi # sched: [7:0.67]
-; SANDY-NEXT:    cmovaeq (%rdx), %rdi # sched: [7:0.67]
-; SANDY-NEXT:    cmovaeq (%rdx), %rdi # sched: [7:0.67]
-; SANDY-NEXT:    cmoveq (%rdx), %rdi # sched: [7:0.67]
-; SANDY-NEXT:    cmoveq (%rdx), %rdi # sched: [7:0.67]
-; SANDY-NEXT:    cmovneq (%rdx), %rdi # sched: [7:0.67]
-; SANDY-NEXT:    cmovneq (%rdx), %rdi # sched: [7:0.67]
-; SANDY-NEXT:    cmovbeq (%rdx), %rdi # sched: [8:1.00]
-; SANDY-NEXT:    cmovbeq (%rdx), %rdi # sched: [8:1.00]
-; SANDY-NEXT:    cmovaq (%rdx), %rdi # sched: [8:1.00]
-; SANDY-NEXT:    cmovaq (%rdx), %rdi # sched: [8:1.00]
-; SANDY-NEXT:    cmovsq (%rdx), %rdi # sched: [7:0.67]
-; SANDY-NEXT:    cmovnsq (%rdx), %rdi # sched: [7:0.67]
-; SANDY-NEXT:    cmovpq (%rdx), %rdi # sched: [7:0.67]
-; SANDY-NEXT:    cmovpq (%rdx), %rdi # sched: [7:0.67]
-; SANDY-NEXT:    cmovnpq (%rdx), %rdi # sched: [7:0.67]
-; SANDY-NEXT:    cmovnpq (%rdx), %rdi # sched: [7:0.67]
-; SANDY-NEXT:    cmovlq (%rdx), %rdi # sched: [7:0.67]
-; SANDY-NEXT:    cmovlq (%rdx), %rdi # sched: [7:0.67]
-; SANDY-NEXT:    cmovgeq (%rdx), %rdi # sched: [7:0.67]
-; SANDY-NEXT:    cmovgeq (%rdx), %rdi # sched: [7:0.67]
-; SANDY-NEXT:    cmovleq (%rdx), %rdi # sched: [7:0.67]
-; SANDY-NEXT:    cmovleq (%rdx), %rdi # sched: [7:0.67]
-; SANDY-NEXT:    cmovgq (%rdx), %rdi # sched: [7:0.67]
-; SANDY-NEXT:    cmovgq (%rdx), %rdi # sched: [7:0.67]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_cmov_64:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    cmovoq %rsi, %rdi # sched: [2:0.50]
-; HASWELL-NEXT:    cmovnoq %rsi, %rdi # sched: [2:0.50]
-; HASWELL-NEXT:    cmovbq %rsi, %rdi # sched: [2:0.50]
-; HASWELL-NEXT:    cmovbq %rsi, %rdi # sched: [2:0.50]
-; HASWELL-NEXT:    cmovbq %rsi, %rdi # sched: [2:0.50]
-; HASWELL-NEXT:    cmovaeq %rsi, %rdi # sched: [2:0.50]
-; HASWELL-NEXT:    cmovaeq %rsi, %rdi # sched: [2:0.50]
-; HASWELL-NEXT:    cmovaeq %rsi, %rdi # sched: [2:0.50]
-; HASWELL-NEXT:    cmoveq %rsi, %rdi # sched: [2:0.50]
-; HASWELL-NEXT:    cmoveq %rsi, %rdi # sched: [2:0.50]
-; HASWELL-NEXT:    cmovneq %rsi, %rdi # sched: [2:0.50]
-; HASWELL-NEXT:    cmovneq %rsi, %rdi # sched: [2:0.50]
-; HASWELL-NEXT:    cmovbeq %rsi, %rdi # sched: [3:0.75]
-; HASWELL-NEXT:    cmovbeq %rsi, %rdi # sched: [3:0.75]
-; HASWELL-NEXT:    cmovaq %rsi, %rdi # sched: [3:0.75]
-; HASWELL-NEXT:    cmovaq %rsi, %rdi # sched: [3:0.75]
-; HASWELL-NEXT:    cmovsq %rsi, %rdi # sched: [2:0.50]
-; HASWELL-NEXT:    cmovnsq %rsi, %rdi # sched: [2:0.50]
-; HASWELL-NEXT:    cmovpq %rsi, %rdi # sched: [2:0.50]
-; HASWELL-NEXT:    cmovpq %rsi, %rdi # sched: [2:0.50]
-; HASWELL-NEXT:    cmovnpq %rsi, %rdi # sched: [2:0.50]
-; HASWELL-NEXT:    cmovnpq %rsi, %rdi # sched: [2:0.50]
-; HASWELL-NEXT:    cmovlq %rsi, %rdi # sched: [2:0.50]
-; HASWELL-NEXT:    cmovlq %rsi, %rdi # sched: [2:0.50]
-; HASWELL-NEXT:    cmovgeq %rsi, %rdi # sched: [2:0.50]
-; HASWELL-NEXT:    cmovgeq %rsi, %rdi # sched: [2:0.50]
-; HASWELL-NEXT:    cmovleq %rsi, %rdi # sched: [2:0.50]
-; HASWELL-NEXT:    cmovleq %rsi, %rdi # sched: [2:0.50]
-; HASWELL-NEXT:    cmovgq %rsi, %rdi # sched: [2:0.50]
-; HASWELL-NEXT:    cmovgq %rsi, %rdi # sched: [2:0.50]
-; HASWELL-NEXT:    cmovoq (%rdx), %rdi # sched: [7:0.50]
-; HASWELL-NEXT:    cmovnoq (%rdx), %rdi # sched: [7:0.50]
-; HASWELL-NEXT:    cmovbq (%rdx), %rdi # sched: [7:0.50]
-; HASWELL-NEXT:    cmovbq (%rdx), %rdi # sched: [7:0.50]
-; HASWELL-NEXT:    cmovbq (%rdx), %rdi # sched: [7:0.50]
-; HASWELL-NEXT:    cmovaeq (%rdx), %rdi # sched: [7:0.50]
-; HASWELL-NEXT:    cmovaeq (%rdx), %rdi # sched: [7:0.50]
-; HASWELL-NEXT:    cmovaeq (%rdx), %rdi # sched: [7:0.50]
-; HASWELL-NEXT:    cmoveq (%rdx), %rdi # sched: [7:0.50]
-; HASWELL-NEXT:    cmoveq (%rdx), %rdi # sched: [7:0.50]
-; HASWELL-NEXT:    cmovneq (%rdx), %rdi # sched: [7:0.50]
-; HASWELL-NEXT:    cmovneq (%rdx), %rdi # sched: [7:0.50]
-; HASWELL-NEXT:    cmovbeq (%rdx), %rdi # sched: [8:0.75]
-; HASWELL-NEXT:    cmovbeq (%rdx), %rdi # sched: [8:0.75]
-; HASWELL-NEXT:    cmovaq (%rdx), %rdi # sched: [8:0.75]
-; HASWELL-NEXT:    cmovaq (%rdx), %rdi # sched: [8:0.75]
-; HASWELL-NEXT:    cmovsq (%rdx), %rdi # sched: [7:0.50]
-; HASWELL-NEXT:    cmovnsq (%rdx), %rdi # sched: [7:0.50]
-; HASWELL-NEXT:    cmovpq (%rdx), %rdi # sched: [7:0.50]
-; HASWELL-NEXT:    cmovpq (%rdx), %rdi # sched: [7:0.50]
-; HASWELL-NEXT:    cmovnpq (%rdx), %rdi # sched: [7:0.50]
-; HASWELL-NEXT:    cmovnpq (%rdx), %rdi # sched: [7:0.50]
-; HASWELL-NEXT:    cmovlq (%rdx), %rdi # sched: [7:0.50]
-; HASWELL-NEXT:    cmovlq (%rdx), %rdi # sched: [7:0.50]
-; HASWELL-NEXT:    cmovgeq (%rdx), %rdi # sched: [7:0.50]
-; HASWELL-NEXT:    cmovgeq (%rdx), %rdi # sched: [7:0.50]
-; HASWELL-NEXT:    cmovleq (%rdx), %rdi # sched: [7:0.50]
-; HASWELL-NEXT:    cmovleq (%rdx), %rdi # sched: [7:0.50]
-; HASWELL-NEXT:    cmovgq (%rdx), %rdi # sched: [7:0.50]
-; HASWELL-NEXT:    cmovgq (%rdx), %rdi # sched: [7:0.50]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_cmov_64:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    cmovoq %rsi, %rdi # sched: [1:0.50]
-; BROADWELL-NEXT:    cmovnoq %rsi, %rdi # sched: [1:0.50]
-; BROADWELL-NEXT:    cmovbq %rsi, %rdi # sched: [1:0.50]
-; BROADWELL-NEXT:    cmovbq %rsi, %rdi # sched: [1:0.50]
-; BROADWELL-NEXT:    cmovbq %rsi, %rdi # sched: [1:0.50]
-; BROADWELL-NEXT:    cmovaeq %rsi, %rdi # sched: [1:0.50]
-; BROADWELL-NEXT:    cmovaeq %rsi, %rdi # sched: [1:0.50]
-; BROADWELL-NEXT:    cmovaeq %rsi, %rdi # sched: [1:0.50]
-; BROADWELL-NEXT:    cmoveq %rsi, %rdi # sched: [1:0.50]
-; BROADWELL-NEXT:    cmoveq %rsi, %rdi # sched: [1:0.50]
-; BROADWELL-NEXT:    cmovneq %rsi, %rdi # sched: [1:0.50]
-; BROADWELL-NEXT:    cmovneq %rsi, %rdi # sched: [1:0.50]
-; BROADWELL-NEXT:    cmovbeq %rsi, %rdi # sched: [2:0.50]
-; BROADWELL-NEXT:    cmovbeq %rsi, %rdi # sched: [2:0.50]
-; BROADWELL-NEXT:    cmovaq %rsi, %rdi # sched: [2:0.50]
-; BROADWELL-NEXT:    cmovaq %rsi, %rdi # sched: [2:0.50]
-; BROADWELL-NEXT:    cmovsq %rsi, %rdi # sched: [1:0.50]
-; BROADWELL-NEXT:    cmovnsq %rsi, %rdi # sched: [1:0.50]
-; BROADWELL-NEXT:    cmovpq %rsi, %rdi # sched: [1:0.50]
-; BROADWELL-NEXT:    cmovpq %rsi, %rdi # sched: [1:0.50]
-; BROADWELL-NEXT:    cmovnpq %rsi, %rdi # sched: [1:0.50]
-; BROADWELL-NEXT:    cmovnpq %rsi, %rdi # sched: [1:0.50]
-; BROADWELL-NEXT:    cmovlq %rsi, %rdi # sched: [1:0.50]
-; BROADWELL-NEXT:    cmovlq %rsi, %rdi # sched: [1:0.50]
-; BROADWELL-NEXT:    cmovgeq %rsi, %rdi # sched: [1:0.50]
-; BROADWELL-NEXT:    cmovgeq %rsi, %rdi # sched: [1:0.50]
-; BROADWELL-NEXT:    cmovleq %rsi, %rdi # sched: [1:0.50]
-; BROADWELL-NEXT:    cmovleq %rsi, %rdi # sched: [1:0.50]
-; BROADWELL-NEXT:    cmovgq %rsi, %rdi # sched: [1:0.50]
-; BROADWELL-NEXT:    cmovgq %rsi, %rdi # sched: [1:0.50]
-; BROADWELL-NEXT:    cmovoq (%rdx), %rdi # sched: [6:0.50]
-; BROADWELL-NEXT:    cmovnoq (%rdx), %rdi # sched: [6:0.50]
-; BROADWELL-NEXT:    cmovbq (%rdx), %rdi # sched: [6:0.50]
-; BROADWELL-NEXT:    cmovbq (%rdx), %rdi # sched: [6:0.50]
-; BROADWELL-NEXT:    cmovbq (%rdx), %rdi # sched: [6:0.50]
-; BROADWELL-NEXT:    cmovaeq (%rdx), %rdi # sched: [6:0.50]
-; BROADWELL-NEXT:    cmovaeq (%rdx), %rdi # sched: [6:0.50]
-; BROADWELL-NEXT:    cmovaeq (%rdx), %rdi # sched: [6:0.50]
-; BROADWELL-NEXT:    cmoveq (%rdx), %rdi # sched: [6:0.50]
-; BROADWELL-NEXT:    cmoveq (%rdx), %rdi # sched: [6:0.50]
-; BROADWELL-NEXT:    cmovneq (%rdx), %rdi # sched: [6:0.50]
-; BROADWELL-NEXT:    cmovneq (%rdx), %rdi # sched: [6:0.50]
-; BROADWELL-NEXT:    cmovbeq (%rdx), %rdi # sched: [7:0.50]
-; BROADWELL-NEXT:    cmovbeq (%rdx), %rdi # sched: [7:0.50]
-; BROADWELL-NEXT:    cmovaq (%rdx), %rdi # sched: [7:0.50]
-; BROADWELL-NEXT:    cmovaq (%rdx), %rdi # sched: [7:0.50]
-; BROADWELL-NEXT:    cmovsq (%rdx), %rdi # sched: [6:0.50]
-; BROADWELL-NEXT:    cmovnsq (%rdx), %rdi # sched: [6:0.50]
-; BROADWELL-NEXT:    cmovpq (%rdx), %rdi # sched: [6:0.50]
-; BROADWELL-NEXT:    cmovpq (%rdx), %rdi # sched: [6:0.50]
-; BROADWELL-NEXT:    cmovnpq (%rdx), %rdi # sched: [6:0.50]
-; BROADWELL-NEXT:    cmovnpq (%rdx), %rdi # sched: [6:0.50]
-; BROADWELL-NEXT:    cmovlq (%rdx), %rdi # sched: [6:0.50]
-; BROADWELL-NEXT:    cmovlq (%rdx), %rdi # sched: [6:0.50]
-; BROADWELL-NEXT:    cmovgeq (%rdx), %rdi # sched: [6:0.50]
-; BROADWELL-NEXT:    cmovgeq (%rdx), %rdi # sched: [6:0.50]
-; BROADWELL-NEXT:    cmovleq (%rdx), %rdi # sched: [6:0.50]
-; BROADWELL-NEXT:    cmovleq (%rdx), %rdi # sched: [6:0.50]
-; BROADWELL-NEXT:    cmovgq (%rdx), %rdi # sched: [6:0.50]
-; BROADWELL-NEXT:    cmovgq (%rdx), %rdi # sched: [6:0.50]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_cmov_64:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    cmovoq %rsi, %rdi # sched: [1:0.50]
-; SKYLAKE-NEXT:    cmovnoq %rsi, %rdi # sched: [1:0.50]
-; SKYLAKE-NEXT:    cmovbq %rsi, %rdi # sched: [1:0.50]
-; SKYLAKE-NEXT:    cmovbq %rsi, %rdi # sched: [1:0.50]
-; SKYLAKE-NEXT:    cmovbq %rsi, %rdi # sched: [1:0.50]
-; SKYLAKE-NEXT:    cmovaeq %rsi, %rdi # sched: [1:0.50]
-; SKYLAKE-NEXT:    cmovaeq %rsi, %rdi # sched: [1:0.50]
-; SKYLAKE-NEXT:    cmovaeq %rsi, %rdi # sched: [1:0.50]
-; SKYLAKE-NEXT:    cmoveq %rsi, %rdi # sched: [1:0.50]
-; SKYLAKE-NEXT:    cmoveq %rsi, %rdi # sched: [1:0.50]
-; SKYLAKE-NEXT:    cmovneq %rsi, %rdi # sched: [1:0.50]
-; SKYLAKE-NEXT:    cmovneq %rsi, %rdi # sched: [1:0.50]
-; SKYLAKE-NEXT:    cmovbeq %rsi, %rdi # sched: [2:1.00]
-; SKYLAKE-NEXT:    cmovbeq %rsi, %rdi # sched: [2:1.00]
-; SKYLAKE-NEXT:    cmovaq %rsi, %rdi # sched: [2:1.00]
-; SKYLAKE-NEXT:    cmovaq %rsi, %rdi # sched: [2:1.00]
-; SKYLAKE-NEXT:    cmovsq %rsi, %rdi # sched: [1:0.50]
-; SKYLAKE-NEXT:    cmovnsq %rsi, %rdi # sched: [1:0.50]
-; SKYLAKE-NEXT:    cmovpq %rsi, %rdi # sched: [1:0.50]
-; SKYLAKE-NEXT:    cmovpq %rsi, %rdi # sched: [1:0.50]
-; SKYLAKE-NEXT:    cmovnpq %rsi, %rdi # sched: [1:0.50]
-; SKYLAKE-NEXT:    cmovnpq %rsi, %rdi # sched: [1:0.50]
-; SKYLAKE-NEXT:    cmovlq %rsi, %rdi # sched: [1:0.50]
-; SKYLAKE-NEXT:    cmovlq %rsi, %rdi # sched: [1:0.50]
-; SKYLAKE-NEXT:    cmovgeq %rsi, %rdi # sched: [1:0.50]
-; SKYLAKE-NEXT:    cmovgeq %rsi, %rdi # sched: [1:0.50]
-; SKYLAKE-NEXT:    cmovleq %rsi, %rdi # sched: [1:0.50]
-; SKYLAKE-NEXT:    cmovleq %rsi, %rdi # sched: [1:0.50]
-; SKYLAKE-NEXT:    cmovgq %rsi, %rdi # sched: [1:0.50]
-; SKYLAKE-NEXT:    cmovgq %rsi, %rdi # sched: [1:0.50]
-; SKYLAKE-NEXT:    cmovoq (%rdx), %rdi # sched: [6:0.50]
-; SKYLAKE-NEXT:    cmovnoq (%rdx), %rdi # sched: [6:0.50]
-; SKYLAKE-NEXT:    cmovbq (%rdx), %rdi # sched: [6:0.50]
-; SKYLAKE-NEXT:    cmovbq (%rdx), %rdi # sched: [6:0.50]
-; SKYLAKE-NEXT:    cmovbq (%rdx), %rdi # sched: [6:0.50]
-; SKYLAKE-NEXT:    cmovaeq (%rdx), %rdi # sched: [6:0.50]
-; SKYLAKE-NEXT:    cmovaeq (%rdx), %rdi # sched: [6:0.50]
-; SKYLAKE-NEXT:    cmovaeq (%rdx), %rdi # sched: [6:0.50]
-; SKYLAKE-NEXT:    cmoveq (%rdx), %rdi # sched: [6:0.50]
-; SKYLAKE-NEXT:    cmoveq (%rdx), %rdi # sched: [6:0.50]
-; SKYLAKE-NEXT:    cmovneq (%rdx), %rdi # sched: [6:0.50]
-; SKYLAKE-NEXT:    cmovneq (%rdx), %rdi # sched: [6:0.50]
-; SKYLAKE-NEXT:    cmovbeq (%rdx), %rdi # sched: [7:1.00]
-; SKYLAKE-NEXT:    cmovbeq (%rdx), %rdi # sched: [7:1.00]
-; SKYLAKE-NEXT:    cmovaq (%rdx), %rdi # sched: [7:1.00]
-; SKYLAKE-NEXT:    cmovaq (%rdx), %rdi # sched: [7:1.00]
-; SKYLAKE-NEXT:    cmovsq (%rdx), %rdi # sched: [6:0.50]
-; SKYLAKE-NEXT:    cmovnsq (%rdx), %rdi # sched: [6:0.50]
-; SKYLAKE-NEXT:    cmovpq (%rdx), %rdi # sched: [6:0.50]
-; SKYLAKE-NEXT:    cmovpq (%rdx), %rdi # sched: [6:0.50]
-; SKYLAKE-NEXT:    cmovnpq (%rdx), %rdi # sched: [6:0.50]
-; SKYLAKE-NEXT:    cmovnpq (%rdx), %rdi # sched: [6:0.50]
-; SKYLAKE-NEXT:    cmovlq (%rdx), %rdi # sched: [6:0.50]
-; SKYLAKE-NEXT:    cmovlq (%rdx), %rdi # sched: [6:0.50]
-; SKYLAKE-NEXT:    cmovgeq (%rdx), %rdi # sched: [6:0.50]
-; SKYLAKE-NEXT:    cmovgeq (%rdx), %rdi # sched: [6:0.50]
-; SKYLAKE-NEXT:    cmovleq (%rdx), %rdi # sched: [6:0.50]
-; SKYLAKE-NEXT:    cmovleq (%rdx), %rdi # sched: [6:0.50]
-; SKYLAKE-NEXT:    cmovgq (%rdx), %rdi # sched: [6:0.50]
-; SKYLAKE-NEXT:    cmovgq (%rdx), %rdi # sched: [6:0.50]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_cmov_64:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    cmovoq %rsi, %rdi # sched: [1:0.50]
-; SKX-NEXT:    cmovnoq %rsi, %rdi # sched: [1:0.50]
-; SKX-NEXT:    cmovbq %rsi, %rdi # sched: [1:0.50]
-; SKX-NEXT:    cmovbq %rsi, %rdi # sched: [1:0.50]
-; SKX-NEXT:    cmovbq %rsi, %rdi # sched: [1:0.50]
-; SKX-NEXT:    cmovaeq %rsi, %rdi # sched: [1:0.50]
-; SKX-NEXT:    cmovaeq %rsi, %rdi # sched: [1:0.50]
-; SKX-NEXT:    cmovaeq %rsi, %rdi # sched: [1:0.50]
-; SKX-NEXT:    cmoveq %rsi, %rdi # sched: [1:0.50]
-; SKX-NEXT:    cmoveq %rsi, %rdi # sched: [1:0.50]
-; SKX-NEXT:    cmovneq %rsi, %rdi # sched: [1:0.50]
-; SKX-NEXT:    cmovneq %rsi, %rdi # sched: [1:0.50]
-; SKX-NEXT:    cmovbeq %rsi, %rdi # sched: [2:1.00]
-; SKX-NEXT:    cmovbeq %rsi, %rdi # sched: [2:1.00]
-; SKX-NEXT:    cmovaq %rsi, %rdi # sched: [2:1.00]
-; SKX-NEXT:    cmovaq %rsi, %rdi # sched: [2:1.00]
-; SKX-NEXT:    cmovsq %rsi, %rdi # sched: [1:0.50]
-; SKX-NEXT:    cmovnsq %rsi, %rdi # sched: [1:0.50]
-; SKX-NEXT:    cmovpq %rsi, %rdi # sched: [1:0.50]
-; SKX-NEXT:    cmovpq %rsi, %rdi # sched: [1:0.50]
-; SKX-NEXT:    cmovnpq %rsi, %rdi # sched: [1:0.50]
-; SKX-NEXT:    cmovnpq %rsi, %rdi # sched: [1:0.50]
-; SKX-NEXT:    cmovlq %rsi, %rdi # sched: [1:0.50]
-; SKX-NEXT:    cmovlq %rsi, %rdi # sched: [1:0.50]
-; SKX-NEXT:    cmovgeq %rsi, %rdi # sched: [1:0.50]
-; SKX-NEXT:    cmovgeq %rsi, %rdi # sched: [1:0.50]
-; SKX-NEXT:    cmovleq %rsi, %rdi # sched: [1:0.50]
-; SKX-NEXT:    cmovleq %rsi, %rdi # sched: [1:0.50]
-; SKX-NEXT:    cmovgq %rsi, %rdi # sched: [1:0.50]
-; SKX-NEXT:    cmovgq %rsi, %rdi # sched: [1:0.50]
-; SKX-NEXT:    cmovoq (%rdx), %rdi # sched: [6:0.50]
-; SKX-NEXT:    cmovnoq (%rdx), %rdi # sched: [6:0.50]
-; SKX-NEXT:    cmovbq (%rdx), %rdi # sched: [6:0.50]
-; SKX-NEXT:    cmovbq (%rdx), %rdi # sched: [6:0.50]
-; SKX-NEXT:    cmovbq (%rdx), %rdi # sched: [6:0.50]
-; SKX-NEXT:    cmovaeq (%rdx), %rdi # sched: [6:0.50]
-; SKX-NEXT:    cmovaeq (%rdx), %rdi # sched: [6:0.50]
-; SKX-NEXT:    cmovaeq (%rdx), %rdi # sched: [6:0.50]
-; SKX-NEXT:    cmoveq (%rdx), %rdi # sched: [6:0.50]
-; SKX-NEXT:    cmoveq (%rdx), %rdi # sched: [6:0.50]
-; SKX-NEXT:    cmovneq (%rdx), %rdi # sched: [6:0.50]
-; SKX-NEXT:    cmovneq (%rdx), %rdi # sched: [6:0.50]
-; SKX-NEXT:    cmovbeq (%rdx), %rdi # sched: [7:1.00]
-; SKX-NEXT:    cmovbeq (%rdx), %rdi # sched: [7:1.00]
-; SKX-NEXT:    cmovaq (%rdx), %rdi # sched: [7:1.00]
-; SKX-NEXT:    cmovaq (%rdx), %rdi # sched: [7:1.00]
-; SKX-NEXT:    cmovsq (%rdx), %rdi # sched: [6:0.50]
-; SKX-NEXT:    cmovnsq (%rdx), %rdi # sched: [6:0.50]
-; SKX-NEXT:    cmovpq (%rdx), %rdi # sched: [6:0.50]
-; SKX-NEXT:    cmovpq (%rdx), %rdi # sched: [6:0.50]
-; SKX-NEXT:    cmovnpq (%rdx), %rdi # sched: [6:0.50]
-; SKX-NEXT:    cmovnpq (%rdx), %rdi # sched: [6:0.50]
-; SKX-NEXT:    cmovlq (%rdx), %rdi # sched: [6:0.50]
-; SKX-NEXT:    cmovlq (%rdx), %rdi # sched: [6:0.50]
-; SKX-NEXT:    cmovgeq (%rdx), %rdi # sched: [6:0.50]
-; SKX-NEXT:    cmovgeq (%rdx), %rdi # sched: [6:0.50]
-; SKX-NEXT:    cmovleq (%rdx), %rdi # sched: [6:0.50]
-; SKX-NEXT:    cmovleq (%rdx), %rdi # sched: [6:0.50]
-; SKX-NEXT:    cmovgq (%rdx), %rdi # sched: [6:0.50]
-; SKX-NEXT:    cmovgq (%rdx), %rdi # sched: [6:0.50]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_cmov_64:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    cmovoq %rsi, %rdi # sched: [1:0.50]
-; BDVER2-NEXT:    cmovnoq %rsi, %rdi # sched: [1:0.50]
-; BDVER2-NEXT:    cmovbq %rsi, %rdi # sched: [1:0.50]
-; BDVER2-NEXT:    cmovbq %rsi, %rdi # sched: [1:0.50]
-; BDVER2-NEXT:    cmovbq %rsi, %rdi # sched: [1:0.50]
-; BDVER2-NEXT:    cmovaeq %rsi, %rdi # sched: [1:0.50]
-; BDVER2-NEXT:    cmovaeq %rsi, %rdi # sched: [1:0.50]
-; BDVER2-NEXT:    cmovaeq %rsi, %rdi # sched: [1:0.50]
-; BDVER2-NEXT:    cmoveq %rsi, %rdi # sched: [1:0.50]
-; BDVER2-NEXT:    cmoveq %rsi, %rdi # sched: [1:0.50]
-; BDVER2-NEXT:    cmovneq %rsi, %rdi # sched: [1:0.50]
-; BDVER2-NEXT:    cmovneq %rsi, %rdi # sched: [1:0.50]
-; BDVER2-NEXT:    cmovbeq %rsi, %rdi # sched: [1:0.50]
-; BDVER2-NEXT:    cmovbeq %rsi, %rdi # sched: [1:0.50]
-; BDVER2-NEXT:    cmovaq %rsi, %rdi # sched: [1:0.50]
-; BDVER2-NEXT:    cmovaq %rsi, %rdi # sched: [1:0.50]
-; BDVER2-NEXT:    cmovsq %rsi, %rdi # sched: [1:0.50]
-; BDVER2-NEXT:    cmovnsq %rsi, %rdi # sched: [1:0.50]
-; BDVER2-NEXT:    cmovpq %rsi, %rdi # sched: [1:0.50]
-; BDVER2-NEXT:    cmovpq %rsi, %rdi # sched: [1:0.50]
-; BDVER2-NEXT:    cmovnpq %rsi, %rdi # sched: [1:0.50]
-; BDVER2-NEXT:    cmovnpq %rsi, %rdi # sched: [1:0.50]
-; BDVER2-NEXT:    cmovlq %rsi, %rdi # sched: [1:0.50]
-; BDVER2-NEXT:    cmovlq %rsi, %rdi # sched: [1:0.50]
-; BDVER2-NEXT:    cmovgeq %rsi, %rdi # sched: [1:0.50]
-; BDVER2-NEXT:    cmovgeq %rsi, %rdi # sched: [1:0.50]
-; BDVER2-NEXT:    cmovleq %rsi, %rdi # sched: [1:0.50]
-; BDVER2-NEXT:    cmovleq %rsi, %rdi # sched: [1:0.50]
-; BDVER2-NEXT:    cmovgq %rsi, %rdi # sched: [1:0.50]
-; BDVER2-NEXT:    cmovgq %rsi, %rdi # sched: [1:0.50]
-; BDVER2-NEXT:    cmovoq (%rdx), %rdi # sched: [5:0.50]
-; BDVER2-NEXT:    cmovnoq (%rdx), %rdi # sched: [5:0.50]
-; BDVER2-NEXT:    cmovbq (%rdx), %rdi # sched: [5:0.50]
-; BDVER2-NEXT:    cmovbq (%rdx), %rdi # sched: [5:0.50]
-; BDVER2-NEXT:    cmovbq (%rdx), %rdi # sched: [5:0.50]
-; BDVER2-NEXT:    cmovaeq (%rdx), %rdi # sched: [5:0.50]
-; BDVER2-NEXT:    cmovaeq (%rdx), %rdi # sched: [5:0.50]
-; BDVER2-NEXT:    cmovaeq (%rdx), %rdi # sched: [5:0.50]
-; BDVER2-NEXT:    cmoveq (%rdx), %rdi # sched: [5:0.50]
-; BDVER2-NEXT:    cmoveq (%rdx), %rdi # sched: [5:0.50]
-; BDVER2-NEXT:    cmovneq (%rdx), %rdi # sched: [5:0.50]
-; BDVER2-NEXT:    cmovneq (%rdx), %rdi # sched: [5:0.50]
-; BDVER2-NEXT:    cmovbeq (%rdx), %rdi # sched: [5:0.50]
-; BDVER2-NEXT:    cmovbeq (%rdx), %rdi # sched: [5:0.50]
-; BDVER2-NEXT:    cmovaq (%rdx), %rdi # sched: [5:0.50]
-; BDVER2-NEXT:    cmovaq (%rdx), %rdi # sched: [5:0.50]
-; BDVER2-NEXT:    cmovsq (%rdx), %rdi # sched: [5:0.50]
-; BDVER2-NEXT:    cmovnsq (%rdx), %rdi # sched: [5:0.50]
-; BDVER2-NEXT:    cmovpq (%rdx), %rdi # sched: [5:0.50]
-; BDVER2-NEXT:    cmovpq (%rdx), %rdi # sched: [5:0.50]
-; BDVER2-NEXT:    cmovnpq (%rdx), %rdi # sched: [5:0.50]
-; BDVER2-NEXT:    cmovnpq (%rdx), %rdi # sched: [5:0.50]
-; BDVER2-NEXT:    cmovlq (%rdx), %rdi # sched: [5:0.50]
-; BDVER2-NEXT:    cmovlq (%rdx), %rdi # sched: [5:0.50]
-; BDVER2-NEXT:    cmovgeq (%rdx), %rdi # sched: [5:0.50]
-; BDVER2-NEXT:    cmovgeq (%rdx), %rdi # sched: [5:0.50]
-; BDVER2-NEXT:    cmovleq (%rdx), %rdi # sched: [5:0.50]
-; BDVER2-NEXT:    cmovleq (%rdx), %rdi # sched: [5:0.50]
-; BDVER2-NEXT:    cmovgq (%rdx), %rdi # sched: [5:0.50]
-; BDVER2-NEXT:    cmovgq (%rdx), %rdi # sched: [5:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_cmov_64:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    cmovoq %rsi, %rdi # sched: [1:0.50]
-; BTVER2-NEXT:    cmovnoq %rsi, %rdi # sched: [1:0.50]
-; BTVER2-NEXT:    cmovbq %rsi, %rdi # sched: [1:0.50]
-; BTVER2-NEXT:    cmovbq %rsi, %rdi # sched: [1:0.50]
-; BTVER2-NEXT:    cmovbq %rsi, %rdi # sched: [1:0.50]
-; BTVER2-NEXT:    cmovaeq %rsi, %rdi # sched: [1:0.50]
-; BTVER2-NEXT:    cmovaeq %rsi, %rdi # sched: [1:0.50]
-; BTVER2-NEXT:    cmovaeq %rsi, %rdi # sched: [1:0.50]
-; BTVER2-NEXT:    cmoveq %rsi, %rdi # sched: [1:0.50]
-; BTVER2-NEXT:    cmoveq %rsi, %rdi # sched: [1:0.50]
-; BTVER2-NEXT:    cmovneq %rsi, %rdi # sched: [1:0.50]
-; BTVER2-NEXT:    cmovneq %rsi, %rdi # sched: [1:0.50]
-; BTVER2-NEXT:    cmovbeq %rsi, %rdi # sched: [1:0.50]
-; BTVER2-NEXT:    cmovbeq %rsi, %rdi # sched: [1:0.50]
-; BTVER2-NEXT:    cmovaq %rsi, %rdi # sched: [1:0.50]
-; BTVER2-NEXT:    cmovaq %rsi, %rdi # sched: [1:0.50]
-; BTVER2-NEXT:    cmovsq %rsi, %rdi # sched: [1:0.50]
-; BTVER2-NEXT:    cmovnsq %rsi, %rdi # sched: [1:0.50]
-; BTVER2-NEXT:    cmovpq %rsi, %rdi # sched: [1:0.50]
-; BTVER2-NEXT:    cmovpq %rsi, %rdi # sched: [1:0.50]
-; BTVER2-NEXT:    cmovnpq %rsi, %rdi # sched: [1:0.50]
-; BTVER2-NEXT:    cmovnpq %rsi, %rdi # sched: [1:0.50]
-; BTVER2-NEXT:    cmovlq %rsi, %rdi # sched: [1:0.50]
-; BTVER2-NEXT:    cmovlq %rsi, %rdi # sched: [1:0.50]
-; BTVER2-NEXT:    cmovgeq %rsi, %rdi # sched: [1:0.50]
-; BTVER2-NEXT:    cmovgeq %rsi, %rdi # sched: [1:0.50]
-; BTVER2-NEXT:    cmovleq %rsi, %rdi # sched: [1:0.50]
-; BTVER2-NEXT:    cmovleq %rsi, %rdi # sched: [1:0.50]
-; BTVER2-NEXT:    cmovgq %rsi, %rdi # sched: [1:0.50]
-; BTVER2-NEXT:    cmovgq %rsi, %rdi # sched: [1:0.50]
-; BTVER2-NEXT:    cmovoq (%rdx), %rdi # sched: [4:1.00]
-; BTVER2-NEXT:    cmovnoq (%rdx), %rdi # sched: [4:1.00]
-; BTVER2-NEXT:    cmovbq (%rdx), %rdi # sched: [4:1.00]
-; BTVER2-NEXT:    cmovbq (%rdx), %rdi # sched: [4:1.00]
-; BTVER2-NEXT:    cmovbq (%rdx), %rdi # sched: [4:1.00]
-; BTVER2-NEXT:    cmovaeq (%rdx), %rdi # sched: [4:1.00]
-; BTVER2-NEXT:    cmovaeq (%rdx), %rdi # sched: [4:1.00]
-; BTVER2-NEXT:    cmovaeq (%rdx), %rdi # sched: [4:1.00]
-; BTVER2-NEXT:    cmoveq (%rdx), %rdi # sched: [4:1.00]
-; BTVER2-NEXT:    cmoveq (%rdx), %rdi # sched: [4:1.00]
-; BTVER2-NEXT:    cmovneq (%rdx), %rdi # sched: [4:1.00]
-; BTVER2-NEXT:    cmovneq (%rdx), %rdi # sched: [4:1.00]
-; BTVER2-NEXT:    cmovbeq (%rdx), %rdi # sched: [4:1.00]
-; BTVER2-NEXT:    cmovbeq (%rdx), %rdi # sched: [4:1.00]
-; BTVER2-NEXT:    cmovaq (%rdx), %rdi # sched: [4:1.00]
-; BTVER2-NEXT:    cmovaq (%rdx), %rdi # sched: [4:1.00]
-; BTVER2-NEXT:    cmovsq (%rdx), %rdi # sched: [4:1.00]
-; BTVER2-NEXT:    cmovnsq (%rdx), %rdi # sched: [4:1.00]
-; BTVER2-NEXT:    cmovpq (%rdx), %rdi # sched: [4:1.00]
-; BTVER2-NEXT:    cmovpq (%rdx), %rdi # sched: [4:1.00]
-; BTVER2-NEXT:    cmovnpq (%rdx), %rdi # sched: [4:1.00]
-; BTVER2-NEXT:    cmovnpq (%rdx), %rdi # sched: [4:1.00]
-; BTVER2-NEXT:    cmovlq (%rdx), %rdi # sched: [4:1.00]
-; BTVER2-NEXT:    cmovlq (%rdx), %rdi # sched: [4:1.00]
-; BTVER2-NEXT:    cmovgeq (%rdx), %rdi # sched: [4:1.00]
-; BTVER2-NEXT:    cmovgeq (%rdx), %rdi # sched: [4:1.00]
-; BTVER2-NEXT:    cmovleq (%rdx), %rdi # sched: [4:1.00]
-; BTVER2-NEXT:    cmovleq (%rdx), %rdi # sched: [4:1.00]
-; BTVER2-NEXT:    cmovgq (%rdx), %rdi # sched: [4:1.00]
-; BTVER2-NEXT:    cmovgq (%rdx), %rdi # sched: [4:1.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_cmov_64:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    cmovoq %rsi, %rdi # sched: [1:0.25]
-; ZNVER1-NEXT:    cmovnoq %rsi, %rdi # sched: [1:0.25]
-; ZNVER1-NEXT:    cmovbq %rsi, %rdi # sched: [1:0.25]
-; ZNVER1-NEXT:    cmovbq %rsi, %rdi # sched: [1:0.25]
-; ZNVER1-NEXT:    cmovbq %rsi, %rdi # sched: [1:0.25]
-; ZNVER1-NEXT:    cmovaeq %rsi, %rdi # sched: [1:0.25]
-; ZNVER1-NEXT:    cmovaeq %rsi, %rdi # sched: [1:0.25]
-; ZNVER1-NEXT:    cmovaeq %rsi, %rdi # sched: [1:0.25]
-; ZNVER1-NEXT:    cmoveq %rsi, %rdi # sched: [1:0.25]
-; ZNVER1-NEXT:    cmoveq %rsi, %rdi # sched: [1:0.25]
-; ZNVER1-NEXT:    cmovneq %rsi, %rdi # sched: [1:0.25]
-; ZNVER1-NEXT:    cmovneq %rsi, %rdi # sched: [1:0.25]
-; ZNVER1-NEXT:    cmovbeq %rsi, %rdi # sched: [1:0.25]
-; ZNVER1-NEXT:    cmovbeq %rsi, %rdi # sched: [1:0.25]
-; ZNVER1-NEXT:    cmovaq %rsi, %rdi # sched: [1:0.25]
-; ZNVER1-NEXT:    cmovaq %rsi, %rdi # sched: [1:0.25]
-; ZNVER1-NEXT:    cmovsq %rsi, %rdi # sched: [1:0.25]
-; ZNVER1-NEXT:    cmovnsq %rsi, %rdi # sched: [1:0.25]
-; ZNVER1-NEXT:    cmovpq %rsi, %rdi # sched: [1:0.25]
-; ZNVER1-NEXT:    cmovpq %rsi, %rdi # sched: [1:0.25]
-; ZNVER1-NEXT:    cmovnpq %rsi, %rdi # sched: [1:0.25]
-; ZNVER1-NEXT:    cmovnpq %rsi, %rdi # sched: [1:0.25]
-; ZNVER1-NEXT:    cmovlq %rsi, %rdi # sched: [1:0.25]
-; ZNVER1-NEXT:    cmovlq %rsi, %rdi # sched: [1:0.25]
-; ZNVER1-NEXT:    cmovgeq %rsi, %rdi # sched: [1:0.25]
-; ZNVER1-NEXT:    cmovgeq %rsi, %rdi # sched: [1:0.25]
-; ZNVER1-NEXT:    cmovleq %rsi, %rdi # sched: [1:0.25]
-; ZNVER1-NEXT:    cmovleq %rsi, %rdi # sched: [1:0.25]
-; ZNVER1-NEXT:    cmovgq %rsi, %rdi # sched: [1:0.25]
-; ZNVER1-NEXT:    cmovgq %rsi, %rdi # sched: [1:0.25]
-; ZNVER1-NEXT:    cmovoq (%rdx), %rdi # sched: [5:0.50]
-; ZNVER1-NEXT:    cmovnoq (%rdx), %rdi # sched: [5:0.50]
-; ZNVER1-NEXT:    cmovbq (%rdx), %rdi # sched: [5:0.50]
-; ZNVER1-NEXT:    cmovbq (%rdx), %rdi # sched: [5:0.50]
-; ZNVER1-NEXT:    cmovbq (%rdx), %rdi # sched: [5:0.50]
-; ZNVER1-NEXT:    cmovaeq (%rdx), %rdi # sched: [5:0.50]
-; ZNVER1-NEXT:    cmovaeq (%rdx), %rdi # sched: [5:0.50]
-; ZNVER1-NEXT:    cmovaeq (%rdx), %rdi # sched: [5:0.50]
-; ZNVER1-NEXT:    cmoveq (%rdx), %rdi # sched: [5:0.50]
-; ZNVER1-NEXT:    cmoveq (%rdx), %rdi # sched: [5:0.50]
-; ZNVER1-NEXT:    cmovneq (%rdx), %rdi # sched: [5:0.50]
-; ZNVER1-NEXT:    cmovneq (%rdx), %rdi # sched: [5:0.50]
-; ZNVER1-NEXT:    cmovbeq (%rdx), %rdi # sched: [5:0.50]
-; ZNVER1-NEXT:    cmovbeq (%rdx), %rdi # sched: [5:0.50]
-; ZNVER1-NEXT:    cmovaq (%rdx), %rdi # sched: [5:0.50]
-; ZNVER1-NEXT:    cmovaq (%rdx), %rdi # sched: [5:0.50]
-; ZNVER1-NEXT:    cmovsq (%rdx), %rdi # sched: [5:0.50]
-; ZNVER1-NEXT:    cmovnsq (%rdx), %rdi # sched: [5:0.50]
-; ZNVER1-NEXT:    cmovpq (%rdx), %rdi # sched: [5:0.50]
-; ZNVER1-NEXT:    cmovpq (%rdx), %rdi # sched: [5:0.50]
-; ZNVER1-NEXT:    cmovnpq (%rdx), %rdi # sched: [5:0.50]
-; ZNVER1-NEXT:    cmovnpq (%rdx), %rdi # sched: [5:0.50]
-; ZNVER1-NEXT:    cmovlq (%rdx), %rdi # sched: [5:0.50]
-; ZNVER1-NEXT:    cmovlq (%rdx), %rdi # sched: [5:0.50]
-; ZNVER1-NEXT:    cmovgeq (%rdx), %rdi # sched: [5:0.50]
-; ZNVER1-NEXT:    cmovgeq (%rdx), %rdi # sched: [5:0.50]
-; ZNVER1-NEXT:    cmovleq (%rdx), %rdi # sched: [5:0.50]
-; ZNVER1-NEXT:    cmovleq (%rdx), %rdi # sched: [5:0.50]
-; ZNVER1-NEXT:    cmovgq (%rdx), %rdi # sched: [5:0.50]
-; ZNVER1-NEXT:    cmovgq (%rdx), %rdi # sched: [5:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  call void asm sideeffect "cmovoq $1, $0 \0A\09 cmovnoq $1, $0 \0A\09 cmovbq $1, $0 \0A\09 cmovcq $1, $0 \0A\09 cmovnaeq $1, $0 \0A\09 cmovnbq $1, $0 \0A\09 cmovncq $1, $0 \0A\09 cmovaeq $1, $0 \0A\09 cmovzq $1, $0 \0A\09 cmoveq $1, $0 \0A\09 cmovnzq $1, $0 \0A\09 cmovneq $1, $0 \0A\09 cmovbeq $1, $0 \0A\09 cmovnaq $1, $0 \0A\09 cmovnbeq $1, $0 \0A\09 cmovaq $1, $0 \0A\09 cmovsq $1, $0 \0A\09 cmovnsq $1, $0 \0A\09 cmovpq $1, $0 \0A\09 cmovpeq $1, $0 \0A\09 cmovnpq $1, $0 \0A\09 cmovpoq $1, $0 \0A\09 cmovlq $1, $0 \0A\09 cmovngeq $1, $0 \0A\09 cmovnlq $1, $0 \0A\09 cmovgeq $1, $0 \0A\09 cmovleq $1, $0 \0A\09 cmovngq $1, $0 \0A\09 cmovnleq $1, $0 \0A\09 cmovgq $1, $0 \0A\09 cmovoq $2, $0 \0A\09 cmovnoq $2, $0 \0A\09 cmovbq $2, $0 \0A\09 cmovcq $2, $0 \0A\09 cmovnaeq $2, $0 \0A\09 cmovnbq $2, $0 \0A\09 cmovncq $2, $0 \0A\09 cmovaeq $2, $0 \0A\09 cmovzq $2, $0 \0A\09 cmoveq $2, $0 \0A\09 cmovnzq $2, $0 \0A\09 cmovneq $2, $0 \0A\09 cmovbeq $2, $0 \0A\09 cmovnaq $2, $0 \0A\09 cmovnbeq $2, $0 \0A\09 cmovaq $2, $0 \0A\09 cmovsq $2, $0 \0A\09 cmovnsq $2, $0 \0A\09 cmovpq $2, $0 \0A\09 cmovpeq $2, $0 \0A\09 cmovnpq $2, $0 \0A\09 cmovpoq $2, $0 \0A\09 cmovlq $2, $0 \0A\09 cmovngeq $2, $0 \0A\09 cmovnlq $2, $0 \0A\09 cmovgeq $2, $0 \0A\09 cmovleq $2, $0 \0A\09 cmovngq $2, $0 \0A\09 cmovnleq $2, $0 \0A\09 cmovgq $2, $0", "r,r,*m"(i64 %a0, i64 %a1, i64 *%a2)
-  ret void
-}
diff --git a/test/CodeGen/X86/coalescer-dce.ll b/test/CodeGen/X86/coalescer-dce.ll
index 90a07720e65f94cb282b438e607c1a7f6636ce89..7685526cef9b4067f3c7608455d8020c28b4e432 100644
--- a/test/CodeGen/X86/coalescer-dce.ll
+++ b/test/CodeGen/X86/coalescer-dce.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -verify-machineinstrs -disable-fp-elim -disable-machine-dce -verify-coalescing
+; RUN: llc < %s -verify-machineinstrs -frame-pointer=all -disable-machine-dce -verify-coalescing
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 target triple = "x86_64-apple-macosx10.7.0"
 
diff --git a/test/CodeGen/X86/code_placement_no_header_change.ll b/test/CodeGen/X86/code_placement_no_header_change.ll
new file mode 100644
index 0000000000000000000000000000000000000000..0275606568c18535bf7063acca6806eda72c5787
--- /dev/null
+++ b/test/CodeGen/X86/code_placement_no_header_change.ll
@@ -0,0 +1,36 @@
+; RUN: llc -mtriple=i686-linux < %s | FileCheck %s
+
+
+define i32 @bar(i32 %count) {
+; Test checks that basic block backedge2 is not moved before header,
+; because it can't reduce taken branches.
+; Later backedge1 and backedge2 is rotated before loop header.
+; CHECK-LABEL: bar
+; CHECK: %.entry
+; CHECK: %.backedge1
+; CHECK: %.backedge2
+; CHECK: %.header
+; CHECK: %.exit
+.entry:
+  %c = shl nsw i32 %count, 2
+  br label %.header
+
+.header:
+  %val1 = call i32 @foo()
+  %cond1 = icmp sgt i32 %val1, 1
+  br i1 %cond1, label %.exit, label %.backedge1
+
+.backedge1:
+  %val2 = call i32 @foo()
+  %cond2 = icmp sgt i32 %val2, 1
+  br i1 %cond2, label %.header, label %.backedge2
+
+.backedge2:
+  %val3 = call i32 @foo()
+  br label %.header
+
+.exit:
+  ret i32 %c
+}
+
+declare i32 @foo()
diff --git a/test/CodeGen/X86/codegen-prepare-uaddo.ll b/test/CodeGen/X86/codegen-prepare-uaddo.ll
new file mode 100644
index 0000000000000000000000000000000000000000..2bc13cc57d2cebdcf6acc544ed96f6cc9207d9a7
--- /dev/null
+++ b/test/CodeGen/X86/codegen-prepare-uaddo.ll
@@ -0,0 +1,270 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown | FileCheck %s
+
+; PR31754
+;
+; #include <x86intrin.h>
+; using u64 = unsigned long long;
+;
+; template<u64 K>
+; void test(u64& alo, u64& ahi)
+; {
+;     u64 blo = K;
+;     u64 bhi = 0;
+;     bool cf = (alo += blo) < blo;
+;     _addcarry_u64(cf, ahi, bhi, &ahi);
+; }
+;
+; template void test<0ull>(u64&, u64&);
+; template void test<1ull>(u64&, u64&);
+; template void test<2ull>(u64&, u64&);
+; template void test<3ull>(u64&, u64&);
+; template void test<4ull>(u64&, u64&);
+; template void test<0x7fffffffffffffffull>(u64&, u64&);
+; template void test<0x8000000000000000ull>(u64&, u64&);
+; template void test<0x8000000000000001ull>(u64&, u64&);
+; template void test<0xffffffff80000000ull>(u64&, u64&);
+; template void test<0xfffffffffffffffdull>(u64&, u64&);
+; template void test<0xfffffffffffffffeull>(u64&, u64&);
+; template void test<0xffffffffffffffffull>(u64&, u64&);
+
+define void @test_0(i64*, i64*) {
+; CHECK-LABEL: test_0:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    retq
+  %3 = load i64, i64* %1, align 8
+  %4 = tail call { i8, i64 } @llvm.x86.addcarry.64(i8 0, i64 %3, i64 0)
+  %5 = extractvalue { i8, i64 } %4, 1
+  store i64 %5, i64* %1, align 8
+  ret void
+}
+
+define void @test_1(i64*, i64*) {
+; CHECK-LABEL: test_1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addq $1, (%rdi)
+; CHECK-NEXT:    adcq $0, (%rsi)
+; CHECK-NEXT:    retq
+  %3 = load i64, i64* %0, align 8
+  %4 = add i64 %3, 1
+  store i64 %4, i64* %0, align 8
+  %5 = icmp eq i64 %4, 0
+  %6 = zext i1 %5 to i8
+  %7 = load i64, i64* %1, align 8
+  %8 = tail call { i8, i64 } @llvm.x86.addcarry.64(i8 %6, i64 %7, i64 0)
+  %9 = extractvalue { i8, i64 } %8, 1
+  store i64 %9, i64* %1, align 8
+  ret void
+}
+
+define void @test_2(i64*, i64*) {
+; CHECK-LABEL: test_2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addq $2, (%rdi)
+; CHECK-NEXT:    adcq $0, (%rsi)
+; CHECK-NEXT:    retq
+  %3 = load i64, i64* %0, align 8
+  %4 = add i64 %3, 2
+  store i64 %4, i64* %0, align 8
+  %5 = icmp ult i64 %4, 2
+  %6 = zext i1 %5 to i8
+  %7 = load i64, i64* %1, align 8
+  %8 = tail call { i8, i64 } @llvm.x86.addcarry.64(i8 %6, i64 %7, i64 0)
+  %9 = extractvalue { i8, i64 } %8, 1
+  store i64 %9, i64* %1, align 8
+  ret void
+}
+
+define void @test_3(i64*, i64*) {
+; CHECK-LABEL: test_3:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addq $3, (%rdi)
+; CHECK-NEXT:    adcq $0, (%rsi)
+; CHECK-NEXT:    retq
+  %3 = load i64, i64* %0, align 8
+  %4 = add i64 %3, 3
+  store i64 %4, i64* %0, align 8
+  %5 = icmp ult i64 %4, 3
+  %6 = zext i1 %5 to i8
+  %7 = load i64, i64* %1, align 8
+  %8 = tail call { i8, i64 } @llvm.x86.addcarry.64(i8 %6, i64 %7, i64 0)
+  %9 = extractvalue { i8, i64 } %8, 1
+  store i64 %9, i64* %1, align 8
+  ret void
+}
+
+define void @test_4(i64*, i64*) {
+; CHECK-LABEL: test_4:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addq $4, (%rdi)
+; CHECK-NEXT:    adcq $0, (%rsi)
+; CHECK-NEXT:    retq
+  %3 = load i64, i64* %0, align 8
+  %4 = add i64 %3, 4
+  store i64 %4, i64* %0, align 8
+  %5 = icmp ult i64 %4, 4
+  %6 = zext i1 %5 to i8
+  %7 = load i64, i64* %1, align 8
+  %8 = tail call { i8, i64 } @llvm.x86.addcarry.64(i8 %6, i64 %7, i64 0)
+  %9 = extractvalue { i8, i64 } %8, 1
+  store i64 %9, i64* %1, align 8
+  ret void
+}
+
+define void @test_9223372036854775807(i64*, i64*) {
+; CHECK-LABEL: test_9223372036854775807:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movabsq $9223372036854775807, %rax # imm = 0x7FFFFFFFFFFFFFFF
+; CHECK-NEXT:    addq %rax, (%rdi)
+; CHECK-NEXT:    adcq $0, (%rsi)
+; CHECK-NEXT:    retq
+  %3 = load i64, i64* %0, align 8
+  %4 = add i64 %3, 9223372036854775807
+  store i64 %4, i64* %0, align 8
+  %5 = icmp ult i64 %4, 9223372036854775807
+  %6 = zext i1 %5 to i8
+  %7 = load i64, i64* %1, align 8
+  %8 = tail call { i8, i64 } @llvm.x86.addcarry.64(i8 %6, i64 %7, i64 0)
+  %9 = extractvalue { i8, i64 } %8, 1
+  store i64 %9, i64* %1, align 8
+  ret void
+}
+
+define void @test_9223372036854775808(i64*, i64*) {
+; CHECK-LABEL: test_9223372036854775808:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq (%rdi), %rax
+; CHECK-NEXT:    movabsq $-9223372036854775808, %rcx # imm = 0x8000000000000000
+; CHECK-NEXT:    xorq %rax, %rcx
+; CHECK-NEXT:    movq %rcx, (%rdi)
+; CHECK-NEXT:    shrq $63, %rax
+; CHECK-NEXT:    addb $-1, %al
+; CHECK-NEXT:    adcq $0, (%rsi)
+; CHECK-NEXT:    retq
+  %3 = load i64, i64* %0, align 8
+  %4 = xor i64 %3, -9223372036854775808
+  store i64 %4, i64* %0, align 8
+  %5 = lshr i64 %3, 63
+  %6 = trunc i64 %5 to i8
+  %7 = load i64, i64* %1, align 8
+  %8 = tail call { i8, i64 } @llvm.x86.addcarry.64(i8 %6, i64 %7, i64 0)
+  %9 = extractvalue { i8, i64 } %8, 1
+  store i64 %9, i64* %1, align 8
+  ret void
+}
+
+define void @test_9223372036854775809(i64*, i64*) {
+; CHECK-LABEL: test_9223372036854775809:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movabsq $-9223372036854775807, %rax # imm = 0x8000000000000001
+; CHECK-NEXT:    addq %rax, (%rdi)
+; CHECK-NEXT:    adcq $0, (%rsi)
+; CHECK-NEXT:    retq
+  %3 = load i64, i64* %0, align 8
+  %4 = add i64 %3, -9223372036854775807
+  store i64 %4, i64* %0, align 8
+  %5 = icmp ult i64 %4, -9223372036854775807
+  %6 = zext i1 %5 to i8
+  %7 = load i64, i64* %1, align 8
+  %8 = tail call { i8, i64 } @llvm.x86.addcarry.64(i8 %6, i64 %7, i64 0)
+  %9 = extractvalue { i8, i64 } %8, 1
+  store i64 %9, i64* %1, align 8
+  ret void
+}
+
+define void @test_18446744071562067968(i64*, i64*) {
+; CHECK-LABEL: test_18446744071562067968:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addq $-2147483648, (%rdi) # imm = 0x80000000
+; CHECK-NEXT:    adcq $0, (%rsi)
+; CHECK-NEXT:    retq
+  %3 = load i64, i64* %0, align 8
+  %4 = add i64 %3, -2147483648
+  store i64 %4, i64* %0, align 8
+  %5 = icmp ult i64 %4, -2147483648
+  %6 = zext i1 %5 to i8
+  %7 = load i64, i64* %1, align 8
+  %8 = tail call { i8, i64 } @llvm.x86.addcarry.64(i8 %6, i64 %7, i64 0)
+  %9 = extractvalue { i8, i64 } %8, 1
+  store i64 %9, i64* %1, align 8
+  ret void
+}
+
+define void @test_18446744073709551613(i64*, i64*) {
+; CHECK-LABEL: test_18446744073709551613:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addq $-3, (%rdi)
+; CHECK-NEXT:    adcq $0, (%rsi)
+; CHECK-NEXT:    retq
+  %3 = load i64, i64* %0, align 8
+  %4 = add i64 %3, -3
+  store i64 %4, i64* %0, align 8
+  %5 = icmp ult i64 %4, -3
+  %6 = zext i1 %5 to i8
+  %7 = load i64, i64* %1, align 8
+  %8 = tail call { i8, i64 } @llvm.x86.addcarry.64(i8 %6, i64 %7, i64 0)
+  %9 = extractvalue { i8, i64 } %8, 1
+  store i64 %9, i64* %1, align 8
+  ret void
+}
+
+define void @test_18446744073709551614(i64*, i64*) {
+; CHECK-LABEL: test_18446744073709551614:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addq $-2, (%rdi)
+; CHECK-NEXT:    adcq $0, (%rsi)
+; CHECK-NEXT:    retq
+  %3 = load i64, i64* %0, align 8
+  %4 = add i64 %3, -2
+  store i64 %4, i64* %0, align 8
+  %5 = icmp ult i64 %4, -2
+  %6 = zext i1 %5 to i8
+  %7 = load i64, i64* %1, align 8
+  %8 = tail call { i8, i64 } @llvm.x86.addcarry.64(i8 %6, i64 %7, i64 0)
+  %9 = extractvalue { i8, i64 } %8, 1
+  store i64 %9, i64* %1, align 8
+  ret void
+}
+
+define void @test_18446744073709551615(i64*, i64*) {
+; CHECK-LABEL: test_18446744073709551615:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq (%rdi), %rax
+; CHECK-NEXT:    leaq -1(%rax), %rcx
+; CHECK-NEXT:    movq %rcx, (%rdi)
+; CHECK-NEXT:    testq %rax, %rax
+; CHECK-NEXT:    setne %al
+; CHECK-NEXT:    addb $-1, %al
+; CHECK-NEXT:    adcq $0, (%rsi)
+; CHECK-NEXT:    retq
+  %3 = load i64, i64* %0, align 8
+  %4 = add i64 %3, -1
+  store i64 %4, i64* %0, align 8
+  %5 = icmp ne i64 %3, 0
+  %6 = zext i1 %5 to i8
+  %7 = load i64, i64* %1, align 8
+  %8 = tail call { i8, i64 } @llvm.x86.addcarry.64(i8 %6, i64 %7, i64 0)
+  %9 = extractvalue { i8, i64 } %8, 1
+  store i64 %9, i64* %1, align 8
+  ret void
+}
+
+define i1 @illegal_type(i17 %x, i17* %p) {
+; CHECK-LABEL: illegal_type:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addl $29, %edi
+; CHECK-NEXT:    movw %di, (%rsi)
+; CHECK-NEXT:    andl $131071, %edi # imm = 0x1FFFF
+; CHECK-NEXT:    movl %edi, %eax
+; CHECK-NEXT:    shrl $16, %eax
+; CHECK-NEXT:    movb %al, 2(%rsi)
+; CHECK-NEXT:    cmpl $29, %edi
+; CHECK-NEXT:    setb %al
+; CHECK-NEXT:    retq
+  %a = add i17 %x, 29
+  store i17 %a, i17* %p
+  %ov = icmp ult i17 %a, 29
+  ret i1 %ov
+}
+
+declare { i8, i64 } @llvm.x86.addcarry.64(i8, i64, i64)
diff --git a/test/CodeGen/X86/combine-abs.ll b/test/CodeGen/X86/combine-abs.ll
index 30bba8d9e36a03efb38a19971314bfbd277f3102..95a2b7e392ba524faf001b0e77ee2f8fac7d1427 100644
--- a/test/CodeGen/X86/combine-abs.ll
+++ b/test/CodeGen/X86/combine-abs.ll
@@ -67,12 +67,8 @@ define <4 x i64> @combine_v4i64_abs_abs(<4 x i64> %a) {
 ; AVX2-LABEL: combine_v4i64_abs_abs:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT:    vpcmpgtq %ymm0, %ymm1, %ymm2
-; AVX2-NEXT:    vpaddq %ymm2, %ymm0, %ymm0
-; AVX2-NEXT:    vpxor %ymm2, %ymm0, %ymm0
-; AVX2-NEXT:    vpcmpgtq %ymm0, %ymm1, %ymm1
-; AVX2-NEXT:    vpaddq %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpxor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpsubq %ymm0, %ymm1, %ymm1
+; AVX2-NEXT:    vblendvpd %ymm0, %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    retq
 ;
 ; AVX512F-LABEL: combine_v4i64_abs_abs:
diff --git a/test/CodeGen/X86/combine-add-ssat.ll b/test/CodeGen/X86/combine-add-ssat.ll
new file mode 100644
index 0000000000000000000000000000000000000000..c261253055885fb2fcbaab0e6fca840785d5a847
--- /dev/null
+++ b/test/CodeGen/X86/combine-add-ssat.ll
@@ -0,0 +1,162 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefixes=CHECK,SSE,SSE2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefixes=CHECK,SSE,SSE41
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.2 | FileCheck %s --check-prefixes=CHECK,SSE,SSE42
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefixes=CHECK,AVX,AVX1
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefixes=CHECK,AVX,AVX2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f | FileCheck %s --check-prefixes=CHECK,AVX,AVX512,AVX512F
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw | FileCheck %s --check-prefixes=CHECK,AVX,AVX512,AVX512BW
+
+declare  i32 @llvm.sadd.sat.i32  (i32, i32)
+declare  i64 @llvm.sadd.sat.i64  (i64, i64)
+declare  <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16>, <8 x i16>)
+
+; fold (sadd_sat x, undef) -> -1
+define i32 @combine_undef_i32(i32 %a0) {
+; CHECK-LABEL: combine_undef_i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl $-1, %eax
+; CHECK-NEXT:    retq
+  %res = call i32 @llvm.sadd.sat.i32(i32 %a0, i32 undef)
+  ret i32 %res
+}
+
+define <8 x i16> @combine_undef_v8i16(<8 x i16> %a0) {
+; SSE-LABEL: combine_undef_v8i16:
+; SSE:       # %bb.0:
+; SSE-NEXT:    pcmpeqd %xmm0, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: combine_undef_v8i16:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    retq
+  %res = call <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16> undef, <8 x i16> %a0)
+  ret <8 x i16> %res
+}
+
+; fold (sadd_sat c1, c2) -> c3
+define i32 @combine_constfold_i32() {
+; CHECK-LABEL: combine_constfold_i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl $2147483647, %eax # imm = 0x7FFFFFFF
+; CHECK-NEXT:    retq
+  %res = call i32 @llvm.sadd.sat.i32(i32 2147483647, i32 100)
+  ret i32 %res
+}
+
+define <8 x i16> @combine_constfold_v8i16() {
+; SSE-LABEL: combine_constfold_v8i16:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movaps {{.*#+}} xmm0 = [1,0,256,65534,0,65280,32768,0]
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: combine_constfold_v8i16:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovaps {{.*#+}} xmm0 = [1,0,256,65534,0,65280,32768,0]
+; AVX-NEXT:    retq
+  %res = call <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16> <i16 0, i16 1, i16 255, i16 65535, i16 -1, i16 -255, i16 -32760, i16 1>, <8 x i16> <i16 1, i16 65535, i16 1, i16 65535, i16 1, i16 65535, i16 -10, i16 65535>)
+  ret <8 x i16> %res
+}
+
+define <8 x i16> @combine_constfold_undef_v8i16() {
+; SSE-LABEL: combine_constfold_undef_v8i16:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movaps {{.*#+}} xmm0 = [65535,65535,65535,65534,0,65280,32768,0]
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: combine_constfold_undef_v8i16:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovaps {{.*#+}} xmm0 = [65535,65535,65535,65534,0,65280,32768,0]
+; AVX-NEXT:    retq
+  %res = call <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16> <i16 undef, i16 1, i16 undef, i16 65535, i16 -1, i16 -255, i16 -32760, i16 1>, <8 x i16> <i16 1, i16 undef, i16 undef, i16 65535, i16 1, i16 65535, i16 -10, i16 65535>)
+  ret <8 x i16> %res
+}
+
+; fold (sadd_sat c, x) -> (sadd_sat x, c)
+define i32 @combine_constant_i32(i32 %a0) {
+; CHECK-LABEL: combine_constant_i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    movl %edi, %ecx
+; CHECK-NEXT:    incl %ecx
+; CHECK-NEXT:    setns %al
+; CHECK-NEXT:    addl $2147483647, %eax # imm = 0x7FFFFFFF
+; CHECK-NEXT:    incl %edi
+; CHECK-NEXT:    cmovnol %edi, %eax
+; CHECK-NEXT:    retq
+  %res = call i32 @llvm.sadd.sat.i32(i32 1, i32 %a0)
+  ret i32 %res
+}
+
+define <8 x i16> @combine_constant_v8i16(<8 x i16> %a0) {
+; SSE-LABEL: combine_constant_v8i16:
+; SSE:       # %bb.0:
+; SSE-NEXT:    paddsw {{.*}}(%rip), %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: combine_constant_v8i16:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpaddsw {{.*}}(%rip), %xmm0, %xmm0
+; AVX-NEXT:    retq
+  %res = call <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16> <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>, <8 x i16> %a0)
+  ret <8 x i16> %res
+}
+
+; fold (sadd_sat c, 0) -> x
+define i32 @combine_zero_i32(i32 %a0) {
+; CHECK-LABEL: combine_zero_i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl %edi, %eax
+; CHECK-NEXT:    retq
+  %1 = call i32 @llvm.sadd.sat.i32(i32 %a0, i32 0)
+  ret i32 %1
+}
+
+define <8 x i16> @combine_zero_v8i16(<8 x i16> %a0) {
+; CHECK-LABEL: combine_zero_v8i16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    retq
+  %1 = call <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16> %a0, <8 x i16> zeroinitializer)
+  ret <8 x i16> %1
+}
+
+; fold (sadd_sat x, y) -> (add x, y) iff no overflow
+define i32 @combine_no_overflow_i32(i32 %a0, i32 %a1) {
+; CHECK-LABEL: combine_no_overflow_i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    sarl $16, %edi
+; CHECK-NEXT:    shrl $16, %esi
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    movl %edi, %ecx
+; CHECK-NEXT:    addl %esi, %ecx
+; CHECK-NEXT:    setns %al
+; CHECK-NEXT:    addl $2147483647, %eax # imm = 0x7FFFFFFF
+; CHECK-NEXT:    addl %edi, %esi
+; CHECK-NEXT:    cmovnol %esi, %eax
+; CHECK-NEXT:    retq
+  %1 = ashr i32 %a0, 16
+  %2 = lshr i32 %a1, 16
+  %3 = call i32 @llvm.sadd.sat.i32(i32 %1, i32 %2)
+  ret i32 %3
+}
+
+define <8 x i16> @combine_no_overflow_v8i16(<8 x i16> %a0, <8 x i16> %a1) {
+; SSE-LABEL: combine_no_overflow_v8i16:
+; SSE:       # %bb.0:
+; SSE-NEXT:    psraw $10, %xmm0
+; SSE-NEXT:    psrlw $10, %xmm1
+; SSE-NEXT:    paddsw %xmm1, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: combine_no_overflow_v8i16:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpsraw $10, %xmm0, %xmm0
+; AVX-NEXT:    vpsrlw $10, %xmm1, %xmm1
+; AVX-NEXT:    vpaddsw %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    retq
+  %1 = ashr <8 x i16> %a0, <i16 10, i16 10, i16 10, i16 10, i16 10, i16 10, i16 10, i16 10>
+  %2 = lshr <8 x i16> %a1, <i16 10, i16 10, i16 10, i16 10, i16 10, i16 10, i16 10, i16 10>
+  %3 = call <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16> %1, <8 x i16> %2)
+  ret <8 x i16> %3
+}
diff --git a/test/CodeGen/X86/combine-add-usat.ll b/test/CodeGen/X86/combine-add-usat.ll
new file mode 100644
index 0000000000000000000000000000000000000000..478a4f388f89bdf1dab1a27d2c0762d8b07f075b
--- /dev/null
+++ b/test/CodeGen/X86/combine-add-usat.ll
@@ -0,0 +1,154 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefixes=CHECK,SSE,SSE2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefixes=CHECK,SSE,SSE41
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.2 | FileCheck %s --check-prefixes=CHECK,SSE,SSE42
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefixes=CHECK,AVX,AVX1
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefixes=CHECK,AVX,AVX2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f | FileCheck %s --check-prefixes=CHECK,AVX,AVX512,AVX512F
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw | FileCheck %s --check-prefixes=CHECK,AVX,AVX512,AVX512BW
+
+declare  i32 @llvm.uadd.sat.i32  (i32, i32)
+declare  i64 @llvm.uadd.sat.i64  (i64, i64)
+declare  <8 x i16> @llvm.uadd.sat.v8i16(<8 x i16>, <8 x i16>)
+
+; fold (uadd_sat x, undef) -> -1
+define i32 @combine_undef_i32(i32 %a0) {
+; CHECK-LABEL: combine_undef_i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl $-1, %eax
+; CHECK-NEXT:    retq
+  %res = call i32 @llvm.uadd.sat.i32(i32 %a0, i32 undef)
+  ret i32 %res
+}
+
+define <8 x i16> @combine_undef_v8i16(<8 x i16> %a0) {
+; SSE-LABEL: combine_undef_v8i16:
+; SSE:       # %bb.0:
+; SSE-NEXT:    pcmpeqd %xmm0, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: combine_undef_v8i16:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    retq
+  %res = call <8 x i16> @llvm.uadd.sat.v8i16(<8 x i16> undef, <8 x i16> %a0)
+  ret <8 x i16> %res
+}
+
+; fold (uadd_sat c1, c2) -> c3
+define i32 @combine_constfold_i32() {
+; CHECK-LABEL: combine_constfold_i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl $-1, %eax
+; CHECK-NEXT:    retq
+  %res = call i32 @llvm.uadd.sat.i32(i32 4294967295, i32 100)
+  ret i32 %res
+}
+
+define <8 x i16> @combine_constfold_v8i16() {
+; SSE-LABEL: combine_constfold_v8i16:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movaps {{.*#+}} xmm0 = [1,65535,256,65535,65535,65535,2,65535]
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: combine_constfold_v8i16:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovaps {{.*#+}} xmm0 = [1,65535,256,65535,65535,65535,2,65535]
+; AVX-NEXT:    retq
+  %res = call <8 x i16> @llvm.uadd.sat.v8i16(<8 x i16> <i16 0, i16 1, i16 255, i16 65535, i16 -1, i16 -255, i16 -65535, i16 1>, <8 x i16> <i16 1, i16 65535, i16 1, i16 65535, i16 1, i16 65535, i16 1, i16 65535>)
+  ret <8 x i16> %res
+}
+
+define <8 x i16> @combine_constfold_undef_v8i16() {
+; SSE-LABEL: combine_constfold_undef_v8i16:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movaps {{.*#+}} xmm0 = [65535,65535,65535,65535,65535,65535,2,65535]
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: combine_constfold_undef_v8i16:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovaps {{.*#+}} xmm0 = [65535,65535,65535,65535,65535,65535,2,65535]
+; AVX-NEXT:    retq
+  %res = call <8 x i16> @llvm.uadd.sat.v8i16(<8 x i16> <i16 undef, i16 1, i16 undef, i16 65535, i16 -1, i16 -255, i16 -65535, i16 1>, <8 x i16> <i16 1, i16 undef, i16 undef, i16 65535, i16 1, i16 65535, i16 1, i16 65535>)
+  ret <8 x i16> %res
+}
+
+; fold (uadd_sat c, x) -> (add_ssat x, c)
+define i32 @combine_constant_i32(i32 %a0) {
+; CHECK-LABEL: combine_constant_i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    incl %edi
+; CHECK-NEXT:    movl $-1, %eax
+; CHECK-NEXT:    cmovnel %edi, %eax
+; CHECK-NEXT:    retq
+  %1 = call i32 @llvm.uadd.sat.i32(i32 1, i32 %a0)
+  ret i32 %1
+}
+
+define <8 x i16> @combine_constant_v8i16(<8 x i16> %a0) {
+; SSE-LABEL: combine_constant_v8i16:
+; SSE:       # %bb.0:
+; SSE-NEXT:    paddusw {{.*}}(%rip), %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: combine_constant_v8i16:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpaddusw {{.*}}(%rip), %xmm0, %xmm0
+; AVX-NEXT:    retq
+  %1 = call <8 x i16> @llvm.uadd.sat.v8i16(<8 x i16> <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>, <8 x i16> %a0)
+  ret <8 x i16> %1
+}
+
+; fold (uadd_sat c, 0) -> x
+define i32 @combine_zero_i32(i32 %a0) {
+; CHECK-LABEL: combine_zero_i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl %edi, %eax
+; CHECK-NEXT:    retq
+  %1 = call i32 @llvm.uadd.sat.i32(i32 %a0, i32 0)
+  ret i32 %1
+}
+
+define <8 x i16> @combine_zero_v8i16(<8 x i16> %a0) {
+; CHECK-LABEL: combine_zero_v8i16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    retq
+  %1 = call <8 x i16> @llvm.uadd.sat.v8i16(<8 x i16> %a0, <8 x i16> zeroinitializer)
+  ret <8 x i16> %1
+}
+
+; fold (uadd_sat x, y) -> (add x, y) iff no overflow
+define i32 @combine_no_overflow_i32(i32 %a0, i32 %a1) {
+; CHECK-LABEL: combine_no_overflow_i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    # kill: def $esi killed $esi def $rsi
+; CHECK-NEXT:    # kill: def $edi killed $edi def $rdi
+; CHECK-NEXT:    shrl $16, %edi
+; CHECK-NEXT:    shrl $16, %esi
+; CHECK-NEXT:    leal (%rsi,%rdi), %eax
+; CHECK-NEXT:    retq
+  %1 = lshr i32 %a0, 16
+  %2 = lshr i32 %a1, 16
+  %3 = call i32 @llvm.uadd.sat.i32(i32 %1, i32 %2)
+  ret i32 %3
+}
+
+define <8 x i16> @combine_no_overflow_v8i16(<8 x i16> %a0, <8 x i16> %a1) {
+; SSE-LABEL: combine_no_overflow_v8i16:
+; SSE:       # %bb.0:
+; SSE-NEXT:    psrlw $10, %xmm0
+; SSE-NEXT:    psrlw $10, %xmm1
+; SSE-NEXT:    paddw %xmm1, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: combine_no_overflow_v8i16:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpsrlw $10, %xmm0, %xmm0
+; AVX-NEXT:    vpsrlw $10, %xmm1, %xmm1
+; AVX-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    retq
+  %1 = lshr <8 x i16> %a0, <i16 10, i16 10, i16 10, i16 10, i16 10, i16 10, i16 10, i16 10>
+  %2 = lshr <8 x i16> %a1, <i16 10, i16 10, i16 10, i16 10, i16 10, i16 10, i16 10, i16 10>
+  %3 = call <8 x i16> @llvm.uadd.sat.v8i16(<8 x i16> %1, <8 x i16> %2)
+  ret <8 x i16> %3
+}
diff --git a/test/CodeGen/X86/combine-add.ll b/test/CodeGen/X86/combine-add.ll
index 7e3b9961f8732336b4eea06291c57cf482e34398..8eb4a5d76b6d34167059e806636928b558fdd7a7 100644
--- a/test/CodeGen/X86/combine-add.ll
+++ b/test/CodeGen/X86/combine-add.ll
@@ -99,6 +99,41 @@ define <4 x i32> @combine_vec_add_sub1(<4 x i32> %a, <4 x i32> %b) {
   ret <4 x i32> %2
 }
 
+; fold ((A-B)+(C-A)) -> (C-B)
+define <4 x i32> @combine_vec_add_sub_sub0(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c) {
+; SSE-LABEL: combine_vec_add_sub_sub0:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movdqa %xmm2, %xmm0
+; SSE-NEXT:    psubd %xmm1, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: combine_vec_add_sub_sub0:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpsubd %xmm1, %xmm2, %xmm0
+; AVX-NEXT:    retq
+  %1 = sub <4 x i32> %a, %b
+  %2 = sub <4 x i32> %c, %a
+  %3 = add <4 x i32> %1, %2
+  ret <4 x i32> %3
+}
+
+; fold ((A-B)+(B-C)) -> (A-C)
+define <4 x i32> @combine_vec_add_sub_sub1(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c) {
+; SSE-LABEL: combine_vec_add_sub_sub1:
+; SSE:       # %bb.0:
+; SSE-NEXT:    psubd %xmm2, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: combine_vec_add_sub_sub1:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpsubd %xmm2, %xmm0, %xmm0
+; AVX-NEXT:    retq
+  %1 = sub <4 x i32> %a, %b
+  %2 = sub <4 x i32> %b, %c
+  %3 = add <4 x i32> %1, %2
+  ret <4 x i32> %3
+}
+
 ; fold (A+(B-(A+C))) to (B-C)
 define <4 x i32> @combine_vec_add_sub_add0(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c) {
 ; SSE-LABEL: combine_vec_add_sub_add0:
diff --git a/test/CodeGen/X86/combine-adx.ll b/test/CodeGen/X86/combine-adx.ll
new file mode 100644
index 0000000000000000000000000000000000000000..c2b11e78115a8d6886e026c6128e5ebdd9c98a25
--- /dev/null
+++ b/test/CodeGen/X86/combine-adx.ll
@@ -0,0 +1,68 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-unknown-unknown | FileCheck %s --check-prefixes=X86
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown | FileCheck %s --check-prefixes=X64
+
+define i32 @test_addcarry_32_x_0_false(i32 %a0)  {
+; X86-LABEL: test_addcarry_32_x_0_false:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: test_addcarry_32_x_0_false:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    retq
+  %1 = tail call { i8, i32 } @llvm.x86.addcarry.32(i8 0, i32 %a0, i32 0)
+  %2 = extractvalue { i8, i32 } %1, 1
+  ret i32 %2
+}
+
+define i32 @test_addcarry_32_0_x_false(i32 %a0)  {
+; X86-LABEL: test_addcarry_32_0_x_false:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: test_addcarry_32_0_x_false:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    retq
+  %1 = tail call { i8, i32 } @llvm.x86.addcarry.32(i8 0, i32 0, i32 %a0)
+  %2 = extractvalue { i8, i32 } %1, 1
+  ret i32 %2
+}
+
+define i32 @test_subborrow_32_x_0_false(i32 %a0)  {
+; X86-LABEL: test_subborrow_32_x_0_false:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: test_subborrow_32_x_0_false:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    retq
+  %1 = tail call { i8, i32 } @llvm.x86.subborrow.32(i8 0, i32 %a0, i32 0)
+  %2 = extractvalue { i8, i32 } %1, 1
+  ret i32 %2
+}
+
+define i32 @test_subborrow_32_0_x_false(i32 %a0)  {
+; X86-LABEL: test_subborrow_32_0_x_false:
+; X86:       # %bb.0:
+; X86-NEXT:    xorl %eax, %eax
+; X86-NEXT:    subl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: test_subborrow_32_0_x_false:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    negl %eax
+; X64-NEXT:    retq
+  %1 = tail call { i8, i32 } @llvm.x86.subborrow.32(i8 0, i32 0, i32 %a0)
+  %2 = extractvalue { i8, i32 } %1, 1
+  ret i32 %2
+}
+
+declare { i8, i32 } @llvm.x86.addcarry.32(i8, i32, i32)
+declare { i8, i32 } @llvm.x86.subborrow.32(i8, i32, i32)
diff --git a/test/CodeGen/X86/combine-bitreverse.ll b/test/CodeGen/X86/combine-bitreverse.ll
new file mode 100644
index 0000000000000000000000000000000000000000..4fb2ab4bb6a79c4009bf56478cc3028b94e83176
--- /dev/null
+++ b/test/CodeGen/X86/combine-bitreverse.ll
@@ -0,0 +1,99 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-unknown -mattr=+sse2 | FileCheck %s --check-prefix=X86
+; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx | FileCheck %s --check-prefix=X64
+
+; These tests just check that the plumbing is in place for @llvm.bitreverse. The
+; actual output is massive at the moment as llvm.bitreverse is not yet legal.
+
+declare i32 @llvm.bitreverse.i32(i32) readnone
+declare <4 x i32> @llvm.bitreverse.v4i32(<4 x i32>) readnone
+
+; fold (bitreverse undef) -> undef
+define i32 @test_undef() nounwind {
+; X86-LABEL: test_undef:
+; X86:       # %bb.0:
+; X86-NEXT:    retl
+;
+; X64-LABEL: test_undef:
+; X64:       # %bb.0:
+; X64-NEXT:    retq
+  %b = call i32 @llvm.bitreverse.i32(i32 undef)
+  ret i32 %b
+}
+
+; fold (bitreverse (bitreverse x)) -> x
+define i32 @test_bitreverse_bitreverse(i32 %a0) nounwind {
+; X86-LABEL: test_bitreverse_bitreverse:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: test_bitreverse_bitreverse:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    retq
+  %b = call i32 @llvm.bitreverse.i32(i32 %a0)
+  %c = call i32 @llvm.bitreverse.i32(i32 %b)
+  ret i32 %c
+}
+
+define <4 x i32> @test_demandedbits_bitreverse(<4 x i32> %a0) nounwind {
+; X86-LABEL: test_demandedbits_bitreverse:
+; X86:       # %bb.0:
+; X86-NEXT:    por {{\.LCPI.*}}, %xmm0
+; X86-NEXT:    pxor %xmm1, %xmm1
+; X86-NEXT:    movdqa %xmm0, %xmm2
+; X86-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm1[8],xmm2[9],xmm1[9],xmm2[10],xmm1[10],xmm2[11],xmm1[11],xmm2[12],xmm1[12],xmm2[13],xmm1[13],xmm2[14],xmm1[14],xmm2[15],xmm1[15]
+; X86-NEXT:    pshuflw {{.*#+}} xmm2 = xmm2[3,2,1,0,4,5,6,7]
+; X86-NEXT:    pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,7,6,5,4]
+; X86-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; X86-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[3,2,1,0,4,5,6,7]
+; X86-NEXT:    pshufhw {{.*#+}} xmm1 = xmm0[0,1,2,3,7,6,5,4]
+; X86-NEXT:    packuswb %xmm2, %xmm1
+; X86-NEXT:    movdqa {{.*#+}} xmm0 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; X86-NEXT:    movdqa %xmm1, %xmm2
+; X86-NEXT:    pand %xmm0, %xmm2
+; X86-NEXT:    psllw $4, %xmm2
+; X86-NEXT:    pand {{\.LCPI.*}}, %xmm1
+; X86-NEXT:    psrlw $4, %xmm1
+; X86-NEXT:    pand %xmm0, %xmm1
+; X86-NEXT:    pandn %xmm2, %xmm0
+; X86-NEXT:    por %xmm1, %xmm0
+; X86-NEXT:    movdqa {{.*#+}} xmm1 = [51,51,51,51,51,51,51,51,51,51,51,51,51,51,51,51]
+; X86-NEXT:    pand %xmm0, %xmm1
+; X86-NEXT:    psllw $2, %xmm1
+; X86-NEXT:    pand {{\.LCPI.*}}, %xmm1
+; X86-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-NEXT:    psrlw $2, %xmm0
+; X86-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-NEXT:    por %xmm1, %xmm0
+; X86-NEXT:    movdqa {{.*#+}} xmm1 = [85,85,85,85,85,85,85,85,85,85,85,85,85,85,85,85]
+; X86-NEXT:    pand %xmm0, %xmm1
+; X86-NEXT:    paddb %xmm1, %xmm1
+; X86-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-NEXT:    psrlw $1, %xmm0
+; X86-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-NEXT:    por %xmm1, %xmm0
+; X86-NEXT:    pand {{\.LCPI.*}}, %xmm0
+; X86-NEXT:    retl
+;
+; X64-LABEL: test_demandedbits_bitreverse:
+; X64:       # %bb.0:
+; X64-NEXT:    vpor {{.*}}(%rip), %xmm0, %xmm0
+; X64-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[3,2,1,0,7,6,5,4,11,10,9,8,15,14,13,12]
+; X64-NEXT:    vmovdqa {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; X64-NEXT:    vpand %xmm1, %xmm0, %xmm2
+; X64-NEXT:    vmovdqa {{.*#+}} xmm3 = [0,128,64,192,32,160,96,224,16,144,80,208,48,176,112,240]
+; X64-NEXT:    vpshufb %xmm2, %xmm3, %xmm2
+; X64-NEXT:    vpsrlw $4, %xmm0, %xmm0
+; X64-NEXT:    vpand %xmm1, %xmm0, %xmm0
+; X64-NEXT:    vmovdqa {{.*#+}} xmm1 = [0,8,4,12,2,10,6,14,1,9,5,13,3,11,7,15]
+; X64-NEXT:    vpshufb %xmm0, %xmm1, %xmm0
+; X64-NEXT:    vpor %xmm0, %xmm2, %xmm0
+; X64-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; X64-NEXT:    retq
+  %b = or <4 x i32> %a0, <i32 2147483648, i32 2147483648, i32 2147483648, i32 2147483648>
+  %c = call <4 x i32> @llvm.bitreverse.v4i32(<4 x i32> %b)
+  %d = and <4 x i32> %c, <i32 -2, i32 -2, i32 -2, i32 -2>
+  ret <4 x i32> %d
+}
diff --git a/test/CodeGen/X86/combine-bitselect.ll b/test/CodeGen/X86/combine-bitselect.ll
new file mode 100644
index 0000000000000000000000000000000000000000..3518c939b4f6803a8f9b35d892140638c232df9b
--- /dev/null
+++ b/test/CodeGen/X86/combine-bitselect.ll
@@ -0,0 +1,593 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefixes=SSE,SSE2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+xop  | FileCheck %s --check-prefixes=XOP
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx  | FileCheck %s --check-prefixes=AVX,AVX1
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefixes=AVX,AVX2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f | FileCheck %s --check-prefixes=AVX,AVX512F
+
+;
+; 128-bit vectors
+;
+
+define <2 x i64> @bitselect_v2i64_rr(<2 x i64>, <2 x i64>) {
+; SSE-LABEL: bitselect_v2i64_rr:
+; SSE:       # %bb.0:
+; SSE-NEXT:    andps {{.*}}(%rip), %xmm0
+; SSE-NEXT:    andps {{.*}}(%rip), %xmm1
+; SSE-NEXT:    orps %xmm1, %xmm0
+; SSE-NEXT:    retq
+;
+; XOP-LABEL: bitselect_v2i64_rr:
+; XOP:       # %bb.0:
+; XOP-NEXT:    vpcmov {{.*}}(%rip), %xmm0, %xmm1, %xmm0
+; XOP-NEXT:    retq
+;
+; AVX-LABEL: bitselect_v2i64_rr:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vandps {{.*}}(%rip), %xmm0, %xmm0
+; AVX-NEXT:    vandps {{.*}}(%rip), %xmm1, %xmm1
+; AVX-NEXT:    vorps %xmm0, %xmm1, %xmm0
+; AVX-NEXT:    retq
+  %3 = and <2 x i64> %0, <i64 4294967296, i64 12884901890>
+  %4 = and <2 x i64> %1, <i64 -4294967297, i64 -12884901891>
+  %5 = or <2 x i64> %4, %3
+  ret <2 x i64> %5
+}
+
+define <2 x i64> @bitselect_v2i64_rm(<2 x i64>, <2 x i64>* nocapture readonly) {
+; SSE-LABEL: bitselect_v2i64_rm:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movaps (%rdi), %xmm1
+; SSE-NEXT:    andps {{.*}}(%rip), %xmm0
+; SSE-NEXT:    andps {{.*}}(%rip), %xmm1
+; SSE-NEXT:    orps %xmm1, %xmm0
+; SSE-NEXT:    retq
+;
+; XOP-LABEL: bitselect_v2i64_rm:
+; XOP:       # %bb.0:
+; XOP-NEXT:    vmovdqa (%rdi), %xmm1
+; XOP-NEXT:    vpcmov {{.*}}(%rip), %xmm0, %xmm1, %xmm0
+; XOP-NEXT:    retq
+;
+; AVX-LABEL: bitselect_v2i64_rm:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovaps (%rdi), %xmm1
+; AVX-NEXT:    vandps {{.*}}(%rip), %xmm0, %xmm0
+; AVX-NEXT:    vandps {{.*}}(%rip), %xmm1, %xmm1
+; AVX-NEXT:    vorps %xmm0, %xmm1, %xmm0
+; AVX-NEXT:    retq
+  %3 = load <2 x i64>, <2 x i64>* %1
+  %4 = and <2 x i64> %0, <i64 8589934593, i64 3>
+  %5 = and <2 x i64> %3, <i64 -8589934594, i64 -4>
+  %6 = or <2 x i64> %5, %4
+  ret <2 x i64> %6
+}
+
+define <2 x i64> @bitselect_v2i64_mr(<2 x i64>* nocapture readonly, <2 x i64>) {
+; SSE-LABEL: bitselect_v2i64_mr:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movaps (%rdi), %xmm1
+; SSE-NEXT:    andps {{.*}}(%rip), %xmm1
+; SSE-NEXT:    andps {{.*}}(%rip), %xmm0
+; SSE-NEXT:    orps %xmm1, %xmm0
+; SSE-NEXT:    retq
+;
+; XOP-LABEL: bitselect_v2i64_mr:
+; XOP:       # %bb.0:
+; XOP-NEXT:    vmovdqa (%rdi), %xmm1
+; XOP-NEXT:    vpcmov {{.*}}(%rip), %xmm0, %xmm1, %xmm0
+; XOP-NEXT:    retq
+;
+; AVX-LABEL: bitselect_v2i64_mr:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovaps (%rdi), %xmm1
+; AVX-NEXT:    vandps {{.*}}(%rip), %xmm1, %xmm1
+; AVX-NEXT:    vandps {{.*}}(%rip), %xmm0, %xmm0
+; AVX-NEXT:    vorps %xmm0, %xmm1, %xmm0
+; AVX-NEXT:    retq
+  %3 = load <2 x i64>, <2 x i64>* %0
+  %4 = and <2 x i64> %3, <i64 12884901890, i64 4294967296>
+  %5 = and <2 x i64> %1, <i64 -12884901891, i64 -4294967297>
+  %6 = or <2 x i64> %4, %5
+  ret <2 x i64> %6
+}
+
+define <2 x i64> @bitselect_v2i64_mm(<2 x i64>* nocapture readonly, <2 x i64>* nocapture readonly) {
+; SSE-LABEL: bitselect_v2i64_mm:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movaps (%rdi), %xmm1
+; SSE-NEXT:    movaps (%rsi), %xmm0
+; SSE-NEXT:    andps {{.*}}(%rip), %xmm1
+; SSE-NEXT:    andps {{.*}}(%rip), %xmm0
+; SSE-NEXT:    orps %xmm1, %xmm0
+; SSE-NEXT:    retq
+;
+; XOP-LABEL: bitselect_v2i64_mm:
+; XOP:       # %bb.0:
+; XOP-NEXT:    vmovdqa (%rsi), %xmm0
+; XOP-NEXT:    vmovdqa {{.*#+}} xmm1 = [18446744073709551612,18446744065119617022]
+; XOP-NEXT:    vpcmov %xmm1, (%rdi), %xmm0, %xmm0
+; XOP-NEXT:    retq
+;
+; AVX-LABEL: bitselect_v2i64_mm:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovaps (%rdi), %xmm0
+; AVX-NEXT:    vmovaps (%rsi), %xmm1
+; AVX-NEXT:    vandps {{.*}}(%rip), %xmm0, %xmm0
+; AVX-NEXT:    vandps {{.*}}(%rip), %xmm1, %xmm1
+; AVX-NEXT:    vorps %xmm0, %xmm1, %xmm0
+; AVX-NEXT:    retq
+  %3 = load <2 x i64>, <2 x i64>* %0
+  %4 = load <2 x i64>, <2 x i64>* %1
+  %5 = and <2 x i64> %3, <i64 3, i64 8589934593>
+  %6 = and <2 x i64> %4, <i64 -4, i64 -8589934594>
+  %7 = or <2 x i64> %6, %5
+  ret <2 x i64> %7
+}
+
+;
+; 256-bit vectors
+;
+
+define <4 x i64> @bitselect_v4i64_rr(<4 x i64>, <4 x i64>) {
+; SSE-LABEL: bitselect_v4i64_rr:
+; SSE:       # %bb.0:
+; SSE-NEXT:    andps {{.*}}(%rip), %xmm1
+; SSE-NEXT:    andps {{.*}}(%rip), %xmm0
+; SSE-NEXT:    andps {{.*}}(%rip), %xmm3
+; SSE-NEXT:    orps %xmm3, %xmm1
+; SSE-NEXT:    andps {{.*}}(%rip), %xmm2
+; SSE-NEXT:    orps %xmm2, %xmm0
+; SSE-NEXT:    retq
+;
+; XOP-LABEL: bitselect_v4i64_rr:
+; XOP:       # %bb.0:
+; XOP-NEXT:    vpcmov {{.*}}(%rip), %ymm0, %ymm1, %ymm0
+; XOP-NEXT:    retq
+;
+; AVX-LABEL: bitselect_v4i64_rr:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vandps {{.*}}(%rip), %ymm0, %ymm0
+; AVX-NEXT:    vandps {{.*}}(%rip), %ymm1, %ymm1
+; AVX-NEXT:    vorps %ymm0, %ymm1, %ymm0
+; AVX-NEXT:    retq
+  %3 = and <4 x i64> %0, <i64 4294967296, i64 12884901890, i64 12884901890, i64 12884901890>
+  %4 = and <4 x i64> %1, <i64 -4294967297, i64 -12884901891, i64 -12884901891, i64 -12884901891>
+  %5 = or <4 x i64> %4, %3
+  ret <4 x i64> %5
+}
+
+define <4 x i64> @bitselect_v4i64_rm(<4 x i64>, <4 x i64>* nocapture readonly) {
+; SSE-LABEL: bitselect_v4i64_rm:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movaps {{.*#+}} xmm2 = [18446744065119617022,18446744073709551612]
+; SSE-NEXT:    movaps 16(%rdi), %xmm4
+; SSE-NEXT:    andps %xmm2, %xmm4
+; SSE-NEXT:    movaps (%rdi), %xmm5
+; SSE-NEXT:    andps %xmm2, %xmm5
+; SSE-NEXT:    movaps %xmm2, %xmm3
+; SSE-NEXT:    andnps %xmm0, %xmm3
+; SSE-NEXT:    orps %xmm5, %xmm3
+; SSE-NEXT:    andnps %xmm1, %xmm2
+; SSE-NEXT:    orps %xmm4, %xmm2
+; SSE-NEXT:    movaps %xmm3, %xmm0
+; SSE-NEXT:    movaps %xmm2, %xmm1
+; SSE-NEXT:    retq
+;
+; XOP-LABEL: bitselect_v4i64_rm:
+; XOP:       # %bb.0:
+; XOP-NEXT:    vmovdqa (%rdi), %ymm1
+; XOP-NEXT:    vpcmov {{.*}}(%rip), %ymm0, %ymm1, %ymm0
+; XOP-NEXT:    retq
+;
+; AVX-LABEL: bitselect_v4i64_rm:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovaps (%rdi), %ymm1
+; AVX-NEXT:    vandps {{.*}}(%rip), %ymm0, %ymm0
+; AVX-NEXT:    vandps {{.*}}(%rip), %ymm1, %ymm1
+; AVX-NEXT:    vorps %ymm0, %ymm1, %ymm0
+; AVX-NEXT:    retq
+  %3 = load <4 x i64>, <4 x i64>* %1
+  %4 = and <4 x i64> %0, <i64 8589934593, i64 3, i64 8589934593, i64 3>
+  %5 = and <4 x i64> %3, <i64 -8589934594, i64 -4, i64 -8589934594, i64 -4>
+  %6 = or <4 x i64> %5, %4
+  ret <4 x i64> %6
+}
+
+define <4 x i64> @bitselect_v4i64_mr(<4 x i64>* nocapture readonly, <4 x i64>) {
+; SSE-LABEL: bitselect_v4i64_mr:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movaps {{.*#+}} xmm2 = [12884901890,4294967296]
+; SSE-NEXT:    movaps 16(%rdi), %xmm4
+; SSE-NEXT:    andps %xmm2, %xmm4
+; SSE-NEXT:    movaps (%rdi), %xmm5
+; SSE-NEXT:    andps %xmm2, %xmm5
+; SSE-NEXT:    movaps %xmm2, %xmm3
+; SSE-NEXT:    andnps %xmm0, %xmm3
+; SSE-NEXT:    orps %xmm5, %xmm3
+; SSE-NEXT:    andnps %xmm1, %xmm2
+; SSE-NEXT:    orps %xmm4, %xmm2
+; SSE-NEXT:    movaps %xmm3, %xmm0
+; SSE-NEXT:    movaps %xmm2, %xmm1
+; SSE-NEXT:    retq
+;
+; XOP-LABEL: bitselect_v4i64_mr:
+; XOP:       # %bb.0:
+; XOP-NEXT:    vmovdqa (%rdi), %ymm1
+; XOP-NEXT:    vpcmov {{.*}}(%rip), %ymm0, %ymm1, %ymm0
+; XOP-NEXT:    retq
+;
+; AVX-LABEL: bitselect_v4i64_mr:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovaps (%rdi), %ymm1
+; AVX-NEXT:    vandps {{.*}}(%rip), %ymm1, %ymm1
+; AVX-NEXT:    vandps {{.*}}(%rip), %ymm0, %ymm0
+; AVX-NEXT:    vorps %ymm0, %ymm1, %ymm0
+; AVX-NEXT:    retq
+  %3 = load <4 x i64>, <4 x i64>* %0
+  %4 = and <4 x i64> %3, <i64 12884901890, i64 4294967296, i64 12884901890, i64 4294967296>
+  %5 = and <4 x i64> %1, <i64 -12884901891, i64 -4294967297, i64 -12884901891, i64 -4294967297>
+  %6 = or <4 x i64> %4, %5
+  ret <4 x i64> %6
+}
+
+define <4 x i64> @bitselect_v4i64_mm(<4 x i64>* nocapture readonly, <4 x i64>* nocapture readonly) {
+; SSE-LABEL: bitselect_v4i64_mm:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movaps {{.*#+}} xmm1 = [18446744073709551612,18446744065119617022]
+; SSE-NEXT:    movaps 16(%rsi), %xmm2
+; SSE-NEXT:    andps %xmm1, %xmm2
+; SSE-NEXT:    movaps (%rsi), %xmm3
+; SSE-NEXT:    andps %xmm1, %xmm3
+; SSE-NEXT:    movaps %xmm1, %xmm0
+; SSE-NEXT:    andnps (%rdi), %xmm0
+; SSE-NEXT:    orps %xmm3, %xmm0
+; SSE-NEXT:    andnps 16(%rdi), %xmm1
+; SSE-NEXT:    orps %xmm2, %xmm1
+; SSE-NEXT:    retq
+;
+; XOP-LABEL: bitselect_v4i64_mm:
+; XOP:       # %bb.0:
+; XOP-NEXT:    vmovdqa (%rsi), %ymm0
+; XOP-NEXT:    vmovdqa {{.*#+}} ymm1 = [18446744073709551612,18446744065119617022,18446744073709551612,18446744065119617022]
+; XOP-NEXT:    vpcmov %ymm1, (%rdi), %ymm0, %ymm0
+; XOP-NEXT:    retq
+;
+; AVX-LABEL: bitselect_v4i64_mm:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovaps (%rdi), %ymm0
+; AVX-NEXT:    vmovaps (%rsi), %ymm1
+; AVX-NEXT:    vandps {{.*}}(%rip), %ymm0, %ymm0
+; AVX-NEXT:    vandps {{.*}}(%rip), %ymm1, %ymm1
+; AVX-NEXT:    vorps %ymm0, %ymm1, %ymm0
+; AVX-NEXT:    retq
+  %3 = load <4 x i64>, <4 x i64>* %0
+  %4 = load <4 x i64>, <4 x i64>* %1
+  %5 = and <4 x i64> %3, <i64 3, i64 8589934593, i64 3, i64 8589934593>
+  %6 = and <4 x i64> %4, <i64 -4, i64 -8589934594, i64 -4, i64 -8589934594>
+  %7 = or <4 x i64> %6, %5
+  ret <4 x i64> %7
+}
+
+;
+; 512-bit vectors
+;
+
+define <8 x i64> @bitselect_v8i64_rr(<8 x i64>, <8 x i64>) {
+; SSE-LABEL: bitselect_v8i64_rr:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movaps {{.*#+}} xmm8 = [18446744060824649725,18446744060824649725]
+; SSE-NEXT:    andps %xmm8, %xmm7
+; SSE-NEXT:    movaps {{.*#+}} xmm9 = [18446744069414584319,18446744060824649725]
+; SSE-NEXT:    andps %xmm9, %xmm6
+; SSE-NEXT:    andps %xmm8, %xmm5
+; SSE-NEXT:    andps %xmm9, %xmm4
+; SSE-NEXT:    movaps %xmm9, %xmm10
+; SSE-NEXT:    andnps %xmm0, %xmm10
+; SSE-NEXT:    orps %xmm4, %xmm10
+; SSE-NEXT:    movaps %xmm8, %xmm4
+; SSE-NEXT:    andnps %xmm1, %xmm4
+; SSE-NEXT:    orps %xmm5, %xmm4
+; SSE-NEXT:    andnps %xmm2, %xmm9
+; SSE-NEXT:    orps %xmm6, %xmm9
+; SSE-NEXT:    andnps %xmm3, %xmm8
+; SSE-NEXT:    orps %xmm7, %xmm8
+; SSE-NEXT:    movaps %xmm10, %xmm0
+; SSE-NEXT:    movaps %xmm4, %xmm1
+; SSE-NEXT:    movaps %xmm9, %xmm2
+; SSE-NEXT:    movaps %xmm8, %xmm3
+; SSE-NEXT:    retq
+;
+; XOP-LABEL: bitselect_v8i64_rr:
+; XOP:       # %bb.0:
+; XOP-NEXT:    vmovdqa {{.*#+}} ymm4 = [18446744069414584319,18446744060824649725,18446744060824649725,18446744060824649725]
+; XOP-NEXT:    vpcmov %ymm4, %ymm0, %ymm2, %ymm0
+; XOP-NEXT:    vpcmov %ymm4, %ymm1, %ymm3, %ymm1
+; XOP-NEXT:    retq
+;
+; AVX1-LABEL: bitselect_v8i64_rr:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vmovaps {{.*#+}} ymm4 = [18446744069414584319,18446744060824649725,18446744060824649725,18446744060824649725]
+; AVX1-NEXT:    vandps %ymm4, %ymm3, %ymm3
+; AVX1-NEXT:    vandps %ymm4, %ymm2, %ymm2
+; AVX1-NEXT:    vandnps %ymm0, %ymm4, %ymm0
+; AVX1-NEXT:    vorps %ymm0, %ymm2, %ymm0
+; AVX1-NEXT:    vandnps %ymm1, %ymm4, %ymm1
+; AVX1-NEXT:    vorps %ymm1, %ymm3, %ymm1
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: bitselect_v8i64_rr:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vmovaps {{.*#+}} ymm4 = [18446744069414584319,18446744060824649725,18446744060824649725,18446744060824649725]
+; AVX2-NEXT:    vandps %ymm4, %ymm3, %ymm3
+; AVX2-NEXT:    vandps %ymm4, %ymm2, %ymm2
+; AVX2-NEXT:    vandnps %ymm0, %ymm4, %ymm0
+; AVX2-NEXT:    vorps %ymm0, %ymm2, %ymm0
+; AVX2-NEXT:    vandnps %ymm1, %ymm4, %ymm1
+; AVX2-NEXT:    vorps %ymm1, %ymm3, %ymm1
+; AVX2-NEXT:    retq
+;
+; AVX512F-LABEL: bitselect_v8i64_rr:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpandq {{.*}}(%rip), %zmm0, %zmm0
+; AVX512F-NEXT:    vpandq {{.*}}(%rip), %zmm1, %zmm1
+; AVX512F-NEXT:    vporq %zmm0, %zmm1, %zmm0
+; AVX512F-NEXT:    retq
+  %3 = and <8 x i64> %0, <i64 4294967296, i64 12884901890, i64 12884901890, i64 12884901890, i64 4294967296, i64 12884901890, i64 12884901890, i64 12884901890>
+  %4 = and <8 x i64> %1, <i64 -4294967297, i64 -12884901891, i64 -12884901891, i64 -12884901891, i64 -4294967297, i64 -12884901891, i64 -12884901891, i64 -12884901891>
+  %5 = or <8 x i64> %4, %3
+  ret <8 x i64> %5
+}
+
+define <8 x i64> @bitselect_v8i64_rm(<8 x i64>, <8 x i64>* nocapture readonly) {
+; SSE-LABEL: bitselect_v8i64_rm:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movaps {{.*#+}} xmm4 = [18446744065119617022,18446744073709551612]
+; SSE-NEXT:    movaps 48(%rdi), %xmm8
+; SSE-NEXT:    andps %xmm4, %xmm8
+; SSE-NEXT:    movaps 32(%rdi), %xmm9
+; SSE-NEXT:    andps %xmm4, %xmm9
+; SSE-NEXT:    movaps 16(%rdi), %xmm7
+; SSE-NEXT:    andps %xmm4, %xmm7
+; SSE-NEXT:    movaps (%rdi), %xmm6
+; SSE-NEXT:    andps %xmm4, %xmm6
+; SSE-NEXT:    movaps %xmm4, %xmm5
+; SSE-NEXT:    andnps %xmm0, %xmm5
+; SSE-NEXT:    orps %xmm6, %xmm5
+; SSE-NEXT:    movaps %xmm4, %xmm6
+; SSE-NEXT:    andnps %xmm1, %xmm6
+; SSE-NEXT:    orps %xmm7, %xmm6
+; SSE-NEXT:    movaps %xmm4, %xmm7
+; SSE-NEXT:    andnps %xmm2, %xmm7
+; SSE-NEXT:    orps %xmm9, %xmm7
+; SSE-NEXT:    andnps %xmm3, %xmm4
+; SSE-NEXT:    orps %xmm8, %xmm4
+; SSE-NEXT:    movaps %xmm5, %xmm0
+; SSE-NEXT:    movaps %xmm6, %xmm1
+; SSE-NEXT:    movaps %xmm7, %xmm2
+; SSE-NEXT:    movaps %xmm4, %xmm3
+; SSE-NEXT:    retq
+;
+; XOP-LABEL: bitselect_v8i64_rm:
+; XOP:       # %bb.0:
+; XOP-NEXT:    vbroadcastf128 {{.*#+}} ymm2 = [8589934593,3,8589934593,3]
+; XOP-NEXT:    # ymm2 = mem[0,1,0,1]
+; XOP-NEXT:    vandps %ymm2, %ymm1, %ymm1
+; XOP-NEXT:    vandps %ymm2, %ymm0, %ymm0
+; XOP-NEXT:    vbroadcastf128 {{.*#+}} ymm2 = [18446744065119617022,18446744073709551612,18446744065119617022,18446744073709551612]
+; XOP-NEXT:    # ymm2 = mem[0,1,0,1]
+; XOP-NEXT:    vandps 32(%rdi), %ymm2, %ymm3
+; XOP-NEXT:    vorps %ymm1, %ymm3, %ymm1
+; XOP-NEXT:    vandps (%rdi), %ymm2, %ymm2
+; XOP-NEXT:    vorps %ymm0, %ymm2, %ymm0
+; XOP-NEXT:    retq
+;
+; AVX1-LABEL: bitselect_v8i64_rm:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vbroadcastf128 {{.*#+}} ymm2 = [8589934593,3,8589934593,3]
+; AVX1-NEXT:    # ymm2 = mem[0,1,0,1]
+; AVX1-NEXT:    vandps %ymm2, %ymm1, %ymm1
+; AVX1-NEXT:    vandps %ymm2, %ymm0, %ymm0
+; AVX1-NEXT:    vbroadcastf128 {{.*#+}} ymm2 = [18446744065119617022,18446744073709551612,18446744065119617022,18446744073709551612]
+; AVX1-NEXT:    # ymm2 = mem[0,1,0,1]
+; AVX1-NEXT:    vandps 32(%rdi), %ymm2, %ymm3
+; AVX1-NEXT:    vorps %ymm1, %ymm3, %ymm1
+; AVX1-NEXT:    vandps (%rdi), %ymm2, %ymm2
+; AVX1-NEXT:    vorps %ymm0, %ymm2, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: bitselect_v8i64_rm:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vbroadcastf128 {{.*#+}} ymm2 = [8589934593,3,8589934593,3]
+; AVX2-NEXT:    # ymm2 = mem[0,1,0,1]
+; AVX2-NEXT:    vandps %ymm2, %ymm1, %ymm1
+; AVX2-NEXT:    vandps %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vbroadcastf128 {{.*#+}} ymm2 = [18446744065119617022,18446744073709551612,18446744065119617022,18446744073709551612]
+; AVX2-NEXT:    # ymm2 = mem[0,1,0,1]
+; AVX2-NEXT:    vandps 32(%rdi), %ymm2, %ymm3
+; AVX2-NEXT:    vorps %ymm1, %ymm3, %ymm1
+; AVX2-NEXT:    vandps (%rdi), %ymm2, %ymm2
+; AVX2-NEXT:    vorps %ymm0, %ymm2, %ymm0
+; AVX2-NEXT:    retq
+;
+; AVX512F-LABEL: bitselect_v8i64_rm:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vmovdqa64 (%rdi), %zmm1
+; AVX512F-NEXT:    vpandq {{.*}}(%rip), %zmm0, %zmm0
+; AVX512F-NEXT:    vpandq {{.*}}(%rip), %zmm1, %zmm1
+; AVX512F-NEXT:    vporq %zmm0, %zmm1, %zmm0
+; AVX512F-NEXT:    retq
+  %3 = load <8 x i64>, <8 x i64>* %1
+  %4 = and <8 x i64> %0, <i64 8589934593, i64 3, i64 8589934593, i64 3, i64 8589934593, i64 3, i64 8589934593, i64 3>
+  %5 = and <8 x i64> %3, <i64 -8589934594, i64 -4, i64 -8589934594, i64 -4, i64 -8589934594, i64 -4, i64 -8589934594, i64 -4>
+  %6 = or <8 x i64> %5, %4
+  ret <8 x i64> %6
+}
+
+define <8 x i64> @bitselect_v8i64_mr(<8 x i64>* nocapture readonly, <8 x i64>) {
+; SSE-LABEL: bitselect_v8i64_mr:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movaps {{.*#+}} xmm4 = [12884901890,4294967296]
+; SSE-NEXT:    movaps 48(%rdi), %xmm8
+; SSE-NEXT:    andps %xmm4, %xmm8
+; SSE-NEXT:    movaps 32(%rdi), %xmm9
+; SSE-NEXT:    andps %xmm4, %xmm9
+; SSE-NEXT:    movaps 16(%rdi), %xmm7
+; SSE-NEXT:    andps %xmm4, %xmm7
+; SSE-NEXT:    movaps (%rdi), %xmm6
+; SSE-NEXT:    andps %xmm4, %xmm6
+; SSE-NEXT:    movaps %xmm4, %xmm5
+; SSE-NEXT:    andnps %xmm0, %xmm5
+; SSE-NEXT:    orps %xmm6, %xmm5
+; SSE-NEXT:    movaps %xmm4, %xmm6
+; SSE-NEXT:    andnps %xmm1, %xmm6
+; SSE-NEXT:    orps %xmm7, %xmm6
+; SSE-NEXT:    movaps %xmm4, %xmm7
+; SSE-NEXT:    andnps %xmm2, %xmm7
+; SSE-NEXT:    orps %xmm9, %xmm7
+; SSE-NEXT:    andnps %xmm3, %xmm4
+; SSE-NEXT:    orps %xmm8, %xmm4
+; SSE-NEXT:    movaps %xmm5, %xmm0
+; SSE-NEXT:    movaps %xmm6, %xmm1
+; SSE-NEXT:    movaps %xmm7, %xmm2
+; SSE-NEXT:    movaps %xmm4, %xmm3
+; SSE-NEXT:    retq
+;
+; XOP-LABEL: bitselect_v8i64_mr:
+; XOP:       # %bb.0:
+; XOP-NEXT:    vbroadcastf128 {{.*#+}} ymm2 = [12884901890,4294967296,12884901890,4294967296]
+; XOP-NEXT:    # ymm2 = mem[0,1,0,1]
+; XOP-NEXT:    vandps 32(%rdi), %ymm2, %ymm3
+; XOP-NEXT:    vandps (%rdi), %ymm2, %ymm2
+; XOP-NEXT:    vbroadcastf128 {{.*#+}} ymm4 = [18446744060824649725,18446744069414584319,18446744060824649725,18446744069414584319]
+; XOP-NEXT:    # ymm4 = mem[0,1,0,1]
+; XOP-NEXT:    vandps %ymm4, %ymm1, %ymm1
+; XOP-NEXT:    vorps %ymm1, %ymm3, %ymm1
+; XOP-NEXT:    vandps %ymm4, %ymm0, %ymm0
+; XOP-NEXT:    vorps %ymm0, %ymm2, %ymm0
+; XOP-NEXT:    retq
+;
+; AVX1-LABEL: bitselect_v8i64_mr:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vbroadcastf128 {{.*#+}} ymm2 = [12884901890,4294967296,12884901890,4294967296]
+; AVX1-NEXT:    # ymm2 = mem[0,1,0,1]
+; AVX1-NEXT:    vandps 32(%rdi), %ymm2, %ymm3
+; AVX1-NEXT:    vandps (%rdi), %ymm2, %ymm2
+; AVX1-NEXT:    vbroadcastf128 {{.*#+}} ymm4 = [18446744060824649725,18446744069414584319,18446744060824649725,18446744069414584319]
+; AVX1-NEXT:    # ymm4 = mem[0,1,0,1]
+; AVX1-NEXT:    vandps %ymm4, %ymm1, %ymm1
+; AVX1-NEXT:    vorps %ymm1, %ymm3, %ymm1
+; AVX1-NEXT:    vandps %ymm4, %ymm0, %ymm0
+; AVX1-NEXT:    vorps %ymm0, %ymm2, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: bitselect_v8i64_mr:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vbroadcastf128 {{.*#+}} ymm2 = [12884901890,4294967296,12884901890,4294967296]
+; AVX2-NEXT:    # ymm2 = mem[0,1,0,1]
+; AVX2-NEXT:    vandps 32(%rdi), %ymm2, %ymm3
+; AVX2-NEXT:    vandps (%rdi), %ymm2, %ymm2
+; AVX2-NEXT:    vbroadcastf128 {{.*#+}} ymm4 = [18446744060824649725,18446744069414584319,18446744060824649725,18446744069414584319]
+; AVX2-NEXT:    # ymm4 = mem[0,1,0,1]
+; AVX2-NEXT:    vandps %ymm4, %ymm1, %ymm1
+; AVX2-NEXT:    vorps %ymm1, %ymm3, %ymm1
+; AVX2-NEXT:    vandps %ymm4, %ymm0, %ymm0
+; AVX2-NEXT:    vorps %ymm0, %ymm2, %ymm0
+; AVX2-NEXT:    retq
+;
+; AVX512F-LABEL: bitselect_v8i64_mr:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vmovdqa64 (%rdi), %zmm1
+; AVX512F-NEXT:    vpandq {{.*}}(%rip), %zmm1, %zmm1
+; AVX512F-NEXT:    vpandq {{.*}}(%rip), %zmm0, %zmm0
+; AVX512F-NEXT:    vporq %zmm0, %zmm1, %zmm0
+; AVX512F-NEXT:    retq
+  %3 = load <8 x i64>, <8 x i64>* %0
+  %4 = and <8 x i64> %3, <i64 12884901890, i64 4294967296, i64 12884901890, i64 4294967296, i64 12884901890, i64 4294967296, i64 12884901890, i64 4294967296>
+  %5 = and <8 x i64> %1, <i64 -12884901891, i64 -4294967297, i64 -12884901891, i64 -4294967297, i64 -12884901891, i64 -4294967297, i64 -12884901891, i64 -4294967297>
+  %6 = or <8 x i64> %4, %5
+  ret <8 x i64> %6
+}
+
+define <8 x i64> @bitselect_v8i64_mm(<8 x i64>* nocapture readonly, <8 x i64>* nocapture readonly) {
+; SSE-LABEL: bitselect_v8i64_mm:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movaps {{.*#+}} xmm3 = [18446744073709551612,18446744065119617022]
+; SSE-NEXT:    movaps 48(%rsi), %xmm4
+; SSE-NEXT:    andps %xmm3, %xmm4
+; SSE-NEXT:    movaps 32(%rsi), %xmm5
+; SSE-NEXT:    andps %xmm3, %xmm5
+; SSE-NEXT:    movaps 16(%rsi), %xmm2
+; SSE-NEXT:    andps %xmm3, %xmm2
+; SSE-NEXT:    movaps (%rsi), %xmm1
+; SSE-NEXT:    andps %xmm3, %xmm1
+; SSE-NEXT:    movaps %xmm3, %xmm0
+; SSE-NEXT:    andnps (%rdi), %xmm0
+; SSE-NEXT:    orps %xmm1, %xmm0
+; SSE-NEXT:    movaps %xmm3, %xmm1
+; SSE-NEXT:    andnps 16(%rdi), %xmm1
+; SSE-NEXT:    orps %xmm2, %xmm1
+; SSE-NEXT:    movaps %xmm3, %xmm2
+; SSE-NEXT:    andnps 32(%rdi), %xmm2
+; SSE-NEXT:    orps %xmm5, %xmm2
+; SSE-NEXT:    andnps 48(%rdi), %xmm3
+; SSE-NEXT:    orps %xmm4, %xmm3
+; SSE-NEXT:    retq
+;
+; XOP-LABEL: bitselect_v8i64_mm:
+; XOP:       # %bb.0:
+; XOP-NEXT:    vbroadcastf128 {{.*#+}} ymm0 = [3,8589934593,3,8589934593]
+; XOP-NEXT:    # ymm0 = mem[0,1,0,1]
+; XOP-NEXT:    vandps 32(%rdi), %ymm0, %ymm1
+; XOP-NEXT:    vandps (%rdi), %ymm0, %ymm0
+; XOP-NEXT:    vbroadcastf128 {{.*#+}} ymm2 = [18446744073709551612,18446744065119617022,18446744073709551612,18446744065119617022]
+; XOP-NEXT:    # ymm2 = mem[0,1,0,1]
+; XOP-NEXT:    vandps 32(%rsi), %ymm2, %ymm3
+; XOP-NEXT:    vorps %ymm1, %ymm3, %ymm1
+; XOP-NEXT:    vandps (%rsi), %ymm2, %ymm2
+; XOP-NEXT:    vorps %ymm0, %ymm2, %ymm0
+; XOP-NEXT:    retq
+;
+; AVX1-LABEL: bitselect_v8i64_mm:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vbroadcastf128 {{.*#+}} ymm0 = [3,8589934593,3,8589934593]
+; AVX1-NEXT:    # ymm0 = mem[0,1,0,1]
+; AVX1-NEXT:    vandps 32(%rdi), %ymm0, %ymm1
+; AVX1-NEXT:    vandps (%rdi), %ymm0, %ymm0
+; AVX1-NEXT:    vbroadcastf128 {{.*#+}} ymm2 = [18446744073709551612,18446744065119617022,18446744073709551612,18446744065119617022]
+; AVX1-NEXT:    # ymm2 = mem[0,1,0,1]
+; AVX1-NEXT:    vandps 32(%rsi), %ymm2, %ymm3
+; AVX1-NEXT:    vorps %ymm1, %ymm3, %ymm1
+; AVX1-NEXT:    vandps (%rsi), %ymm2, %ymm2
+; AVX1-NEXT:    vorps %ymm0, %ymm2, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: bitselect_v8i64_mm:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vbroadcastf128 {{.*#+}} ymm0 = [3,8589934593,3,8589934593]
+; AVX2-NEXT:    # ymm0 = mem[0,1,0,1]
+; AVX2-NEXT:    vandps 32(%rdi), %ymm0, %ymm1
+; AVX2-NEXT:    vandps (%rdi), %ymm0, %ymm0
+; AVX2-NEXT:    vbroadcastf128 {{.*#+}} ymm2 = [18446744073709551612,18446744065119617022,18446744073709551612,18446744065119617022]
+; AVX2-NEXT:    # ymm2 = mem[0,1,0,1]
+; AVX2-NEXT:    vandps 32(%rsi), %ymm2, %ymm3
+; AVX2-NEXT:    vorps %ymm1, %ymm3, %ymm1
+; AVX2-NEXT:    vandps (%rsi), %ymm2, %ymm2
+; AVX2-NEXT:    vorps %ymm0, %ymm2, %ymm0
+; AVX2-NEXT:    retq
+;
+; AVX512F-LABEL: bitselect_v8i64_mm:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vmovdqa64 (%rdi), %zmm0
+; AVX512F-NEXT:    vmovdqa64 (%rsi), %zmm1
+; AVX512F-NEXT:    vpandq {{.*}}(%rip), %zmm0, %zmm0
+; AVX512F-NEXT:    vpandq {{.*}}(%rip), %zmm1, %zmm1
+; AVX512F-NEXT:    vporq %zmm0, %zmm1, %zmm0
+; AVX512F-NEXT:    retq
+  %3 = load <8 x i64>, <8 x i64>* %0
+  %4 = load <8 x i64>, <8 x i64>* %1
+  %5 = and <8 x i64> %3, <i64 3, i64 8589934593, i64 3, i64 8589934593, i64 3, i64 8589934593, i64 3, i64 8589934593>
+  %6 = and <8 x i64> %4, <i64 -4, i64 -8589934594, i64 -4, i64 -8589934594, i64 -4, i64 -8589934594, i64 -4, i64 -8589934594>
+  %7 = or <8 x i64> %6, %5
+  ret <8 x i64> %7
+}
diff --git a/test/CodeGen/X86/combine-bswap.ll b/test/CodeGen/X86/combine-bswap.ll
new file mode 100644
index 0000000000000000000000000000000000000000..742ea2e02cda259c102094e3c03f3d73ef4d5705
--- /dev/null
+++ b/test/CodeGen/X86/combine-bswap.ll
@@ -0,0 +1,60 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-unknown -mattr=+sse2 | FileCheck %s --check-prefix=X86
+; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx | FileCheck %s --check-prefix=X64
+
+; These tests just check that the plumbing is in place for @llvm.bswap. The
+; actual output is massive at the moment as llvm.bswap is not yet legal.
+
+declare i32 @llvm.bswap.i32(i32) readnone
+declare i32 @llvm.bswap.v4i32(i32) readnone
+
+; fold (bswap undef) -> undef
+define i32 @test_undef() nounwind {
+; X86-LABEL: test_undef:
+; X86:       # %bb.0:
+; X86-NEXT:    retl
+;
+; X64-LABEL: test_undef:
+; X64:       # %bb.0:
+; X64-NEXT:    retq
+  %b = call i32 @llvm.bswap.i32(i32 undef)
+  ret i32 %b
+}
+
+; fold (bswap (bswap x)) -> x
+define i32 @test_bswap_bswap(i32 %a0) nounwind {
+; X86-LABEL: test_bswap_bswap:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: test_bswap_bswap:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    retq
+  %b = call i32 @llvm.bswap.i32(i32 %a0)
+  %c = call i32 @llvm.bswap.i32(i32 %b)
+  ret i32 %c
+}
+
+define i32 @test_demandedbits_bswap(i32 %a0) nounwind {
+; X86-LABEL: test_demandedbits_bswap:
+; X86:       # %bb.0:
+; X86-NEXT:    movl $-16777216, %eax # imm = 0xFF000000
+; X86-NEXT:    orl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    bswapl %eax
+; X86-NEXT:    andl $-65536, %eax # imm = 0xFFFF0000
+; X86-NEXT:    retl
+;
+; X64-LABEL: test_demandedbits_bswap:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    orl $-16777216, %eax # imm = 0xFF000000
+; X64-NEXT:    bswapl %eax
+; X64-NEXT:    andl $-65536, %eax # imm = 0xFFFF0000
+; X64-NEXT:    retq
+  %b = or i32 %a0, 4278190080
+  %c = call i32 @llvm.bswap.i32(i32 %b)
+  %d = and i32 %c, 4294901760
+  ret i32 %d
+}
diff --git a/test/CodeGen/X86/combine-fcopysign.ll b/test/CodeGen/X86/combine-fcopysign.ll
index 5d27fdfa889589b79578a84fda49ecd0dc2aba52..05c546c411f5b711c007840311566a68b26a3d8b 100644
--- a/test/CodeGen/X86/combine-fcopysign.ll
+++ b/test/CodeGen/X86/combine-fcopysign.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefix=SSE
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefix=AVX
 
@@ -15,7 +16,7 @@ define <4 x float> @combine_vec_fcopysign_pos_constant0(<4 x float> %x) {
 ;
 ; AVX-LABEL: combine_vec_fcopysign_pos_constant0:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vbroadcastss {{.*}}(%rip), %xmm1
+; AVX-NEXT:    vbroadcastss {{.*#+}} xmm1 = [NaN,NaN,NaN,NaN]
 ; AVX-NEXT:    vandps %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    retq
   %1 = call <4 x float> @llvm.copysign.v4f32(<4 x float> %x, <4 x float> <float 2.0, float 2.0, float 2.0, float 2.0>)
@@ -30,7 +31,7 @@ define <4 x float> @combine_vec_fcopysign_pos_constant1(<4 x float> %x) {
 ;
 ; AVX-LABEL: combine_vec_fcopysign_pos_constant1:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vbroadcastss {{.*}}(%rip), %xmm1
+; AVX-NEXT:    vbroadcastss {{.*#+}} xmm1 = [NaN,NaN,NaN,NaN]
 ; AVX-NEXT:    vandps %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    retq
   %1 = call <4 x float> @llvm.copysign.v4f32(<4 x float> %x, <4 x float> <float 0.0, float 2.0, float 4.0, float 8.0>)
@@ -45,7 +46,7 @@ define <4 x float> @combine_vec_fcopysign_fabs_sgn(<4 x float> %x, <4 x float> %
 ;
 ; AVX-LABEL: combine_vec_fcopysign_fabs_sgn:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vbroadcastss {{.*}}(%rip), %xmm1
+; AVX-NEXT:    vbroadcastss {{.*#+}} xmm1 = [NaN,NaN,NaN,NaN]
 ; AVX-NEXT:    vandps %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    retq
   %1 = call <4 x float> @llvm.fabs.v4f32(<4 x float> %y)
@@ -114,7 +115,7 @@ define <4 x float> @combine_vec_fcopysign_fabs_mag(<4 x float> %x, <4 x float> %
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vbroadcastss {{.*#+}} xmm2 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]
 ; AVX-NEXT:    vandps %xmm2, %xmm1, %xmm1
-; AVX-NEXT:    vbroadcastss {{.*}}(%rip), %xmm2
+; AVX-NEXT:    vbroadcastss {{.*#+}} xmm2 = [NaN,NaN,NaN,NaN]
 ; AVX-NEXT:    vandps %xmm2, %xmm0, %xmm0
 ; AVX-NEXT:    vorps %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    retq
@@ -136,7 +137,7 @@ define <4 x float> @combine_vec_fcopysign_fneg_mag(<4 x float> %x, <4 x float> %
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vbroadcastss {{.*#+}} xmm2 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]
 ; AVX-NEXT:    vandps %xmm2, %xmm1, %xmm1
-; AVX-NEXT:    vbroadcastss {{.*}}(%rip), %xmm2
+; AVX-NEXT:    vbroadcastss {{.*#+}} xmm2 = [NaN,NaN,NaN,NaN]
 ; AVX-NEXT:    vandps %xmm2, %xmm0, %xmm0
 ; AVX-NEXT:    vorps %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    retq
@@ -158,7 +159,7 @@ define <4 x float> @combine_vec_fcopysign_fcopysign_mag(<4 x float> %x, <4 x flo
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vbroadcastss {{.*#+}} xmm2 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]
 ; AVX-NEXT:    vandps %xmm2, %xmm1, %xmm1
-; AVX-NEXT:    vbroadcastss {{.*}}(%rip), %xmm2
+; AVX-NEXT:    vbroadcastss {{.*#+}} xmm2 = [NaN,NaN,NaN,NaN]
 ; AVX-NEXT:    vandps %xmm2, %xmm0, %xmm0
 ; AVX-NEXT:    vorps %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    retq
@@ -180,7 +181,7 @@ define <4 x float> @combine_vec_fcopysign_fcopysign_sgn(<4 x float> %x, <4 x flo
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vbroadcastss {{.*#+}} xmm1 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]
 ; AVX-NEXT:    vandps %xmm1, %xmm2, %xmm1
-; AVX-NEXT:    vbroadcastss {{.*}}(%rip), %xmm2
+; AVX-NEXT:    vbroadcastss {{.*#+}} xmm2 = [NaN,NaN,NaN,NaN]
 ; AVX-NEXT:    vandps %xmm2, %xmm0, %xmm0
 ; AVX-NEXT:    vorps %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    retq
@@ -194,42 +195,40 @@ define <4 x double> @combine_vec_fcopysign_fpext_sgn(<4 x double> %x, <4 x float
 ; SSE-LABEL: combine_vec_fcopysign_fpext_sgn:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    movaps %xmm2, %xmm3
-; SSE-NEXT:    cvtss2sd %xmm2, %xmm4
-; SSE-NEXT:    movshdup {{.*#+}} xmm5 = xmm2[1,1,3,3]
-; SSE-NEXT:    movaps %xmm2, %xmm6
-; SSE-NEXT:    unpckhpd {{.*#+}} xmm6 = xmm6[1],xmm2[1]
-; SSE-NEXT:    shufps {{.*#+}} xmm3 = xmm3[3,1],xmm2[2,3]
-; SSE-NEXT:    movaps {{.*#+}} xmm7
-; SSE-NEXT:    movaps %xmm0, %xmm2
-; SSE-NEXT:    andps %xmm7, %xmm2
-; SSE-NEXT:    movaps {{.*#+}} xmm8 = [-0.0E+0,-0.0E+0]
-; SSE-NEXT:    andps %xmm8, %xmm4
-; SSE-NEXT:    orps %xmm4, %xmm2
-; SSE-NEXT:    movhlps {{.*#+}} xmm0 = xmm0[1,1]
-; SSE-NEXT:    andps %xmm7, %xmm0
-; SSE-NEXT:    xorps %xmm4, %xmm4
-; SSE-NEXT:    cvtss2sd %xmm5, %xmm4
-; SSE-NEXT:    andps %xmm8, %xmm4
-; SSE-NEXT:    orps %xmm0, %xmm4
-; SSE-NEXT:    movlhps {{.*#+}} xmm2 = xmm2[0],xmm4[0]
-; SSE-NEXT:    movaps %xmm1, %xmm0
-; SSE-NEXT:    unpckhpd {{.*#+}} xmm0 = xmm0[1],xmm1[1]
-; SSE-NEXT:    andps %xmm7, %xmm0
+; SSE-NEXT:    movshdup {{.*#+}} xmm4 = xmm2[1,1,3,3]
+; SSE-NEXT:    cvtss2sd %xmm2, %xmm5
+; SSE-NEXT:    movhlps {{.*#+}} xmm2 = xmm2[1,1]
+; SSE-NEXT:    shufps {{.*#+}} xmm3 = xmm3[3,1,2,3]
+; SSE-NEXT:    movaps {{.*#+}} xmm6 = [NaN,NaN]
 ; SSE-NEXT:    cvtss2sd %xmm3, %xmm3
-; SSE-NEXT:    andps %xmm8, %xmm3
-; SSE-NEXT:    orps %xmm0, %xmm3
-; SSE-NEXT:    andps %xmm7, %xmm1
-; SSE-NEXT:    xorps %xmm0, %xmm0
-; SSE-NEXT:    cvtss2sd %xmm6, %xmm0
-; SSE-NEXT:    andps %xmm8, %xmm0
-; SSE-NEXT:    orps %xmm0, %xmm1
-; SSE-NEXT:    movlhps {{.*#+}} xmm1 = xmm1[0],xmm3[0]
-; SSE-NEXT:    movaps %xmm2, %xmm0
+; SSE-NEXT:    movaps %xmm6, %xmm7
+; SSE-NEXT:    andnps %xmm3, %xmm7
+; SSE-NEXT:    movaps %xmm1, %xmm3
+; SSE-NEXT:    unpckhpd {{.*#+}} xmm3 = xmm3[1],xmm1[1]
+; SSE-NEXT:    andps %xmm6, %xmm3
+; SSE-NEXT:    orps %xmm3, %xmm7
+; SSE-NEXT:    andps %xmm6, %xmm1
+; SSE-NEXT:    cvtss2sd %xmm2, %xmm2
+; SSE-NEXT:    movaps %xmm6, %xmm3
+; SSE-NEXT:    andnps %xmm2, %xmm3
+; SSE-NEXT:    orps %xmm3, %xmm1
+; SSE-NEXT:    movlhps {{.*#+}} xmm1 = xmm1[0],xmm7[0]
+; SSE-NEXT:    movaps %xmm0, %xmm2
+; SSE-NEXT:    unpckhpd {{.*#+}} xmm2 = xmm2[1],xmm0[1]
+; SSE-NEXT:    andps %xmm6, %xmm2
+; SSE-NEXT:    xorps %xmm3, %xmm3
+; SSE-NEXT:    cvtss2sd %xmm4, %xmm3
+; SSE-NEXT:    andps %xmm6, %xmm0
+; SSE-NEXT:    andnps %xmm3, %xmm6
+; SSE-NEXT:    orps %xmm2, %xmm6
+; SSE-NEXT:    andps {{.*}}(%rip), %xmm5
+; SSE-NEXT:    orps %xmm5, %xmm0
+; SSE-NEXT:    movlhps {{.*#+}} xmm0 = xmm0[0],xmm6[0]
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: combine_vec_fcopysign_fpext_sgn:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vbroadcastsd {{.*}}(%rip), %ymm2
+; AVX-NEXT:    vbroadcastsd {{.*#+}} ymm2 = [NaN,NaN,NaN,NaN]
 ; AVX-NEXT:    vandps %ymm2, %ymm0, %ymm0
 ; AVX-NEXT:    vcvtps2pd %xmm1, %ymm1
 ; AVX-NEXT:    vbroadcastsd {{.*#+}} ymm2 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]
@@ -246,40 +245,42 @@ define <4 x float> @combine_vec_fcopysign_fptrunc_sgn(<4 x float> %x, <4 x doubl
 ; SSE-LABEL: combine_vec_fcopysign_fptrunc_sgn:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    movaps %xmm0, %xmm3
-; SSE-NEXT:    movaps {{.*#+}} xmm5
-; SSE-NEXT:    andps %xmm5, %xmm0
-; SSE-NEXT:    cvtsd2ss %xmm1, %xmm6
-; SSE-NEXT:    movaps {{.*#+}} xmm4 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]
-; SSE-NEXT:    andps %xmm4, %xmm6
-; SSE-NEXT:    orps %xmm6, %xmm0
-; SSE-NEXT:    movshdup {{.*#+}} xmm6 = xmm3[1,1,3,3]
-; SSE-NEXT:    andps %xmm5, %xmm6
+; SSE-NEXT:    unpckhpd {{.*#+}} xmm3 = xmm3[1],xmm0[1]
+; SSE-NEXT:    movaps {{.*#+}} xmm4 = [NaN,NaN,NaN,NaN]
+; SSE-NEXT:    andps %xmm4, %xmm3
+; SSE-NEXT:    cvtsd2ss %xmm2, %xmm5
+; SSE-NEXT:    movaps %xmm4, %xmm6
+; SSE-NEXT:    andnps %xmm5, %xmm6
+; SSE-NEXT:    orps %xmm3, %xmm6
+; SSE-NEXT:    movaps %xmm0, %xmm3
+; SSE-NEXT:    andps %xmm4, %xmm3
+; SSE-NEXT:    xorps %xmm5, %xmm5
+; SSE-NEXT:    cvtsd2ss %xmm1, %xmm5
+; SSE-NEXT:    movaps %xmm4, %xmm7
+; SSE-NEXT:    andnps %xmm5, %xmm7
+; SSE-NEXT:    orps %xmm7, %xmm3
+; SSE-NEXT:    movshdup {{.*#+}} xmm5 = xmm0[1,1,3,3]
+; SSE-NEXT:    andps %xmm4, %xmm5
 ; SSE-NEXT:    movhlps {{.*#+}} xmm1 = xmm1[1,1]
 ; SSE-NEXT:    cvtsd2ss %xmm1, %xmm1
-; SSE-NEXT:    andps %xmm4, %xmm1
-; SSE-NEXT:    orps %xmm6, %xmm1
-; SSE-NEXT:    unpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; SSE-NEXT:    movaps %xmm3, %xmm1
-; SSE-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm3[1]
-; SSE-NEXT:    andps %xmm5, %xmm1
-; SSE-NEXT:    xorps %xmm6, %xmm6
-; SSE-NEXT:    cvtsd2ss %xmm2, %xmm6
-; SSE-NEXT:    andps %xmm4, %xmm6
-; SSE-NEXT:    orps %xmm1, %xmm6
-; SSE-NEXT:    insertps {{.*#+}} xmm0 = xmm0[0,1],xmm6[0],xmm0[3]
-; SSE-NEXT:    shufps {{.*#+}} xmm3 = xmm3[3,1,2,3]
-; SSE-NEXT:    andps %xmm5, %xmm3
+; SSE-NEXT:    andps {{.*}}(%rip), %xmm1
+; SSE-NEXT:    orps %xmm5, %xmm1
+; SSE-NEXT:    unpcklps {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
+; SSE-NEXT:    insertps {{.*#+}} xmm3 = xmm3[0,1],xmm6[0],xmm3[3]
+; SSE-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE-NEXT:    andps %xmm4, %xmm0
 ; SSE-NEXT:    movhlps {{.*#+}} xmm2 = xmm2[1,1]
 ; SSE-NEXT:    xorps %xmm1, %xmm1
 ; SSE-NEXT:    cvtsd2ss %xmm2, %xmm1
-; SSE-NEXT:    andps %xmm4, %xmm1
-; SSE-NEXT:    orps %xmm3, %xmm1
-; SSE-NEXT:    insertps {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[0]
+; SSE-NEXT:    andnps %xmm1, %xmm4
+; SSE-NEXT:    orps %xmm0, %xmm4
+; SSE-NEXT:    insertps {{.*#+}} xmm3 = xmm3[0,1,2],xmm4[0]
+; SSE-NEXT:    movaps %xmm3, %xmm0
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: combine_vec_fcopysign_fptrunc_sgn:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vbroadcastss {{.*}}(%rip), %xmm2
+; AVX-NEXT:    vbroadcastss {{.*#+}} xmm2 = [NaN,NaN,NaN,NaN]
 ; AVX-NEXT:    vandpd %xmm2, %xmm0, %xmm0
 ; AVX-NEXT:    vcvtpd2ps %ymm1, %xmm1
 ; AVX-NEXT:    vbroadcastss {{.*#+}} xmm2 = [-0.0E+0,-0.0E+0,-0.0E+0,-0.0E+0]
diff --git a/test/CodeGen/X86/combine-sbb.ll b/test/CodeGen/X86/combine-sbb.ll
new file mode 100644
index 0000000000000000000000000000000000000000..6ef26f0db0d6e96bc7ceeaa89ea701c57568268e
--- /dev/null
+++ b/test/CodeGen/X86/combine-sbb.ll
@@ -0,0 +1,199 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-unknown-unknown | FileCheck %s --check-prefixes=X86
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown | FileCheck %s --check-prefixes=X64
+
+%WideUInt32 = type { i32, i32 }
+
+define void @PR25858_i32(%WideUInt32* sret, %WideUInt32*, %WideUInt32*) nounwind {
+; X86-LABEL: PR25858_i32:
+; X86:       # %bb.0: # %top
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl (%ecx), %esi
+; X86-NEXT:    movl 4(%ecx), %ecx
+; X86-NEXT:    subl (%edx), %esi
+; X86-NEXT:    sbbl 4(%edx), %ecx
+; X86-NEXT:    movl %ecx, 4(%eax)
+; X86-NEXT:    movl %esi, (%eax)
+; X86-NEXT:    popl %esi
+; X86-NEXT:    retl $4
+;
+; X64-LABEL: PR25858_i32:
+; X64:       # %bb.0: # %top
+; X64-NEXT:    movq %rdi, %rax
+; X64-NEXT:    movl (%rsi), %ecx
+; X64-NEXT:    movl 4(%rsi), %esi
+; X64-NEXT:    subl (%rdx), %ecx
+; X64-NEXT:    sbbl 4(%rdx), %esi
+; X64-NEXT:    movl %esi, 4(%rdi)
+; X64-NEXT:    movl %ecx, (%rdi)
+; X64-NEXT:    retq
+top:
+  %3 = bitcast %WideUInt32* %1 to i32*
+  %4 = load i32, i32* %3, align 4
+  %5 = bitcast %WideUInt32* %2 to i32*
+  %6 = load i32, i32* %5, align 4
+  %7 = sub i32 %4, %6
+  %8 = call { i32, i1 } @llvm.usub.with.overflow.i32(i32 %4, i32 %6)
+  %9 = extractvalue { i32, i1 } %8, 1
+  %10 = getelementptr inbounds %WideUInt32, %WideUInt32* %1, i32 0, i32 1
+  %11 = load i32, i32* %10, align 8
+  %12 = getelementptr inbounds %WideUInt32, %WideUInt32* %2, i32 0, i32 1
+  %13 = load i32, i32* %12, align 8
+  %14 = sub i32 %11, %13
+  %.neg1 = sext i1 %9 to i32
+  %15 = add i32 %14, %.neg1
+  %16 = insertvalue %WideUInt32 undef, i32 %7, 0
+  %17 = insertvalue %WideUInt32 %16, i32 %15, 1
+  store %WideUInt32 %17, %WideUInt32* %0, align 4
+  ret void
+}
+
+declare  { i32, i1 } @llvm.usub.with.overflow.i32(i32, i32)
+
+%WideUInt64 = type { i64, i64 }
+
+define void @PR25858_i64(%WideUInt64* sret, %WideUInt64*, %WideUInt64*) nounwind {
+; X86-LABEL: PR25858_i64:
+; X86:       # %bb.0: # %top
+; X86-NEXT:    pushl %ebp
+; X86-NEXT:    pushl %ebx
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl (%edx), %esi
+; X86-NEXT:    movl 4(%edx), %edi
+; X86-NEXT:    subl (%ecx), %esi
+; X86-NEXT:    sbbl 4(%ecx), %edi
+; X86-NEXT:    setb %bl
+; X86-NEXT:    movl 12(%edx), %ebp
+; X86-NEXT:    movl 8(%edx), %edx
+; X86-NEXT:    subl 8(%ecx), %edx
+; X86-NEXT:    sbbl 12(%ecx), %ebp
+; X86-NEXT:    movzbl %bl, %ecx
+; X86-NEXT:    subl %ecx, %edx
+; X86-NEXT:    sbbl $0, %ebp
+; X86-NEXT:    movl %edi, 4(%eax)
+; X86-NEXT:    movl %esi, (%eax)
+; X86-NEXT:    movl %edx, 8(%eax)
+; X86-NEXT:    movl %ebp, 12(%eax)
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    popl %ebx
+; X86-NEXT:    popl %ebp
+; X86-NEXT:    retl $4
+;
+; X64-LABEL: PR25858_i64:
+; X64:       # %bb.0: # %top
+; X64-NEXT:    movq %rdi, %rax
+; X64-NEXT:    movq (%rsi), %rcx
+; X64-NEXT:    movq 8(%rsi), %rsi
+; X64-NEXT:    subq (%rdx), %rcx
+; X64-NEXT:    sbbq 8(%rdx), %rsi
+; X64-NEXT:    movq %rsi, 8(%rdi)
+; X64-NEXT:    movq %rcx, (%rdi)
+; X64-NEXT:    retq
+top:
+  %3 = bitcast %WideUInt64* %1 to i64*
+  %4 = load i64, i64* %3, align 8
+  %5 = bitcast %WideUInt64* %2 to i64*
+  %6 = load i64, i64* %5, align 8
+  %7 = sub i64 %4, %6
+  %8 = call { i64, i1 } @llvm.usub.with.overflow.i64(i64 %4, i64 %6)
+  %9 = extractvalue { i64, i1 } %8, 1
+  %10 = getelementptr inbounds %WideUInt64, %WideUInt64* %1, i64 0, i32 1
+  %11 = load i64, i64* %10, align 8
+  %12 = getelementptr inbounds %WideUInt64, %WideUInt64* %2, i64 0, i32 1
+  %13 = load i64, i64* %12, align 8
+  %14 = sub i64 %11, %13
+  %.neg1 = sext i1 %9 to i64
+  %15 = add i64 %14, %.neg1
+  %16 = insertvalue %WideUInt64 undef, i64 %7, 0
+  %17 = insertvalue %WideUInt64 %16, i64 %15, 1
+  store %WideUInt64 %17, %WideUInt64* %0, align 8
+  ret void
+}
+
+declare  { i64, i1 } @llvm.usub.with.overflow.i64(i64, i64)
+
+; PR24545 less_than_ideal()
+define i8 @PR24545(i32, i32, i32* nocapture readonly) {
+; X86-LABEL: PR24545:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    cmpl (%ecx), %edx
+; X86-NEXT:    sbbl 4(%ecx), %eax
+; X86-NEXT:    setb %al
+; X86-NEXT:    retl
+;
+; X64-LABEL: PR24545:
+; X64:       # %bb.0:
+; X64-NEXT:    cmpl (%rdx), %edi
+; X64-NEXT:    sbbl 4(%rdx), %esi
+; X64-NEXT:    setb %al
+; X64-NEXT:    retq
+  %4 = load i32, i32* %2
+  %5 = icmp ugt i32 %4, %0
+  %6 = zext i1 %5 to i8
+  %7 = getelementptr inbounds i32, i32* %2, i32 1
+  %8 = load i32, i32* %7
+  %9 = tail call { i8, i32 } @llvm.x86.subborrow.32(i8 %6, i32 %1, i32 %8)
+  %10 = extractvalue { i8, i32 } %9, 0
+  %11 = icmp ne i8 %10, 0
+  %12 = zext i1 %11 to i8
+  ret i8 %12
+}
+
+define i32 @PR40483_sub1(i32*, i32) nounwind {
+; X86-LABEL: PR40483_sub1:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    subl %eax, (%ecx)
+; X86-NEXT:    xorl %eax, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: PR40483_sub1:
+; X64:       # %bb.0:
+; X64-NEXT:    subl %esi, (%rdi)
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    retq
+  %3 = load i32, i32* %0, align 4
+  %4 = tail call { i8, i32 } @llvm.x86.subborrow.32(i8 0, i32 %3, i32 %1)
+  %5 = extractvalue { i8, i32 } %4, 1
+  store i32 %5, i32* %0, align 4
+  %6 = sub i32 %1, %3
+  %7 = add i32 %6, %5
+  ret i32 %7
+}
+
+define i32 @PR40483_sub2(i32*, i32) nounwind {
+; X86-LABEL: PR40483_sub2:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    subl %eax, (%ecx)
+; X86-NEXT:    xorl %eax, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: PR40483_sub2:
+; X64:       # %bb.0:
+; X64-NEXT:    subl %esi, (%rdi)
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    retq
+  %3 = load i32, i32* %0, align 4
+  %4 = sub i32 %3, %1
+  %5 = tail call { i8, i32 } @llvm.x86.subborrow.32(i8 0, i32 %3, i32 %1)
+  %6 = extractvalue { i8, i32 } %5, 1
+  store i32 %6, i32* %0, align 4
+  %7 = sub i32 %4, %6
+  ret i32 %7
+}
+
+declare { i8, i32 } @llvm.x86.subborrow.32(i8, i32, i32)
diff --git a/test/CodeGen/X86/combine-sdiv.ll b/test/CodeGen/X86/combine-sdiv.ll
index 6fff298dbb3e3011bb8a20227f1b66a1f806d522..14627e6e8133234079db85efd36717d829deab65 100644
--- a/test/CodeGen/X86/combine-sdiv.ll
+++ b/test/CodeGen/X86/combine-sdiv.ll
@@ -1198,8 +1198,7 @@ define <8 x i32> @combine_vec_sdiv_by_pow2b_v8i32(<8 x i32> %x) {
 ; AVX1-NEXT:    vpsrad $4, %xmm2, %xmm3
 ; AVX1-NEXT:    vpsrad $2, %xmm2, %xmm4
 ; AVX1-NEXT:    vpblendw {{.*#+}} xmm3 = xmm4[0,1,2,3],xmm3[4,5,6,7]
-; AVX1-NEXT:    vpsrad $3, %xmm2, %xmm4
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0,1,2,3],xmm4[4,5,6,7]
+; AVX1-NEXT:    vpsrad $3, %xmm2, %xmm2
 ; AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0,1],xmm3[2,3],xmm2[4,5],xmm3[6,7]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm1
 ; AVX1-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm1[1,2,3],ymm0[4],ymm1[5,6,7]
@@ -1419,8 +1418,7 @@ define <16 x i32> @combine_vec_sdiv_by_pow2b_v16i32(<16 x i32> %x) {
 ; AVX1-NEXT:    vpsrad $4, %xmm3, %xmm4
 ; AVX1-NEXT:    vpsrad $2, %xmm3, %xmm5
 ; AVX1-NEXT:    vpblendw {{.*#+}} xmm4 = xmm5[0,1,2,3],xmm4[4,5,6,7]
-; AVX1-NEXT:    vpsrad $3, %xmm3, %xmm5
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm3 = xmm3[0,1,2,3],xmm5[4,5,6,7]
+; AVX1-NEXT:    vpsrad $3, %xmm3, %xmm3
 ; AVX1-NEXT:    vpblendw {{.*#+}} xmm3 = xmm3[0,1],xmm4[2,3],xmm3[4,5],xmm4[6,7]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm3, %ymm2
 ; AVX1-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm2[1,2,3],ymm0[4],ymm2[5,6,7]
@@ -1448,8 +1446,7 @@ define <16 x i32> @combine_vec_sdiv_by_pow2b_v16i32(<16 x i32> %x) {
 ; AVX1-NEXT:    vpsrad $4, %xmm3, %xmm4
 ; AVX1-NEXT:    vpsrad $2, %xmm3, %xmm5
 ; AVX1-NEXT:    vpblendw {{.*#+}} xmm4 = xmm5[0,1,2,3],xmm4[4,5,6,7]
-; AVX1-NEXT:    vpsrad $3, %xmm3, %xmm5
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm3 = xmm3[0,1,2,3],xmm5[4,5,6,7]
+; AVX1-NEXT:    vpsrad $3, %xmm3, %xmm3
 ; AVX1-NEXT:    vpblendw {{.*#+}} xmm3 = xmm3[0,1],xmm4[2,3],xmm3[4,5],xmm4[6,7]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm3, %ymm2
 ; AVX1-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0],ymm2[1,2,3],ymm1[4],ymm2[5,6,7]
diff --git a/test/CodeGen/X86/combine-shl.ll b/test/CodeGen/X86/combine-shl.ll
index 3cb1dd65e9ce925dced13c6789851dd027d10e53..64f9f10c4c64cf9bcd358c265077eab487ff8efc 100644
--- a/test/CodeGen/X86/combine-shl.ll
+++ b/test/CodeGen/X86/combine-shl.ll
@@ -46,15 +46,9 @@ define <4 x i32> @combine_vec_shl_outofrange2(<4 x i32> %a0) {
 }
 
 define <4 x i32> @combine_vec_shl_outofrange3(<4 x i32> %a0) {
-; SSE-LABEL: combine_vec_shl_outofrange3:
-; SSE:       # %bb.0:
-; SSE-NEXT:    xorps %xmm0, %xmm0
-; SSE-NEXT:    retq
-;
-; AVX-LABEL: combine_vec_shl_outofrange3:
-; AVX:       # %bb.0:
-; AVX-NEXT:    vpsllvd {{.*}}(%rip), %xmm0, %xmm0
-; AVX-NEXT:    retq
+; CHECK-LABEL: combine_vec_shl_outofrange3:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    retq
   %1 = shl <4 x i32> %a0, <i32 33, i32 34, i32 35, i32 undef>
   ret <4 x i32> %1
 }
@@ -275,33 +269,25 @@ define <8 x i32> @combine_vec_shl_ext_shl1(<8 x i16> %x) {
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    pmullw {{.*}}(%rip), %xmm0
 ; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
-; SSE2-NEXT:    psrad $16, %xmm1
 ; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
-; SSE2-NEXT:    psrad $16, %xmm0
-; SSE2-NEXT:    movdqa %xmm0, %xmm2
-; SSE2-NEXT:    pslld $31, %xmm2
-; SSE2-NEXT:    pslld $30, %xmm0
-; SSE2-NEXT:    movsd {{.*#+}} xmm0 = xmm2[0],xmm0[1]
+; SSE2-NEXT:    psrad $16, %xmm1
 ; SSE2-NEXT:    movdqa %xmm1, %xmm2
 ; SSE2-NEXT:    pslld $29, %xmm2
 ; SSE2-NEXT:    pslld $28, %xmm1
 ; SSE2-NEXT:    movsd {{.*#+}} xmm1 = xmm2[0],xmm1[1]
+; SSE2-NEXT:    pslld $30, %xmm0
+; SSE2-NEXT:    xorpd %xmm2, %xmm2
+; SSE2-NEXT:    movsd {{.*#+}} xmm0 = xmm2[0],xmm0[1]
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: combine_vec_shl_ext_shl1:
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    pmullw {{.*}}(%rip), %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; SSE41-NEXT:    pmovsxwd %xmm1, %xmm1
 ; SSE41-NEXT:    pmovsxwd %xmm0, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm2
-; SSE41-NEXT:    pslld $30, %xmm2
-; SSE41-NEXT:    pslld $31, %xmm0
-; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm2[4,5,6,7]
-; SSE41-NEXT:    movdqa %xmm1, %xmm2
-; SSE41-NEXT:    pslld $28, %xmm2
-; SSE41-NEXT:    pslld $29, %xmm1
-; SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm2[4,5,6,7]
+; SSE41-NEXT:    pslld $30, %xmm0
+; SSE41-NEXT:    pxor %xmm1, %xmm1
+; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm1[0,1,2,3],xmm0[4,5,6,7]
+; SSE41-NEXT:    pxor %xmm1, %xmm1
 ; SSE41-NEXT:    retq
 ;
 ; AVX-LABEL: combine_vec_shl_ext_shl1:
diff --git a/test/CodeGen/X86/combine-sra.ll b/test/CodeGen/X86/combine-sra.ll
index f8b536ac553ae8569167be8f942d100c3c96fd75..4a14991c5eb5e89b9045b02b47cf9f71eb861157 100644
--- a/test/CodeGen/X86/combine-sra.ll
+++ b/test/CodeGen/X86/combine-sra.ll
@@ -51,14 +51,9 @@ define <4 x i32> @combine_vec_ashr_outofrange1(<4 x i32> %x) {
 }
 
 define <4 x i32> @combine_vec_ashr_outofrange2(<4 x i32> %x) {
-; SSE-LABEL: combine_vec_ashr_outofrange2:
-; SSE:       # %bb.0:
-; SSE-NEXT:    retq
-;
-; AVX-LABEL: combine_vec_ashr_outofrange2:
-; AVX:       # %bb.0:
-; AVX-NEXT:    vpsravd {{.*}}(%rip), %xmm0, %xmm0
-; AVX-NEXT:    retq
+; CHECK-LABEL: combine_vec_ashr_outofrange2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    retq
   %1 = ashr <4 x i32> %x, <i32 33, i32 34, i32 35, i32 undef>
   ret <4 x i32> %1
 }
diff --git a/test/CodeGen/X86/combine-srl.ll b/test/CodeGen/X86/combine-srl.ll
index 2dd1fa74871af05e2952b0e098cccb1550149795..960aa079e4c0b3b3813a1d51b33faaac4de55f85 100644
--- a/test/CodeGen/X86/combine-srl.ll
+++ b/test/CodeGen/X86/combine-srl.ll
@@ -36,14 +36,9 @@ define <4 x i32> @combine_vec_lshr_outofrange1(<4 x i32> %x) {
 }
 
 define <4 x i32> @combine_vec_lshr_outofrange2(<4 x i32> %x) {
-; SSE-LABEL: combine_vec_lshr_outofrange2:
-; SSE:       # %bb.0:
-; SSE-NEXT:    retq
-;
-; AVX-LABEL: combine_vec_lshr_outofrange2:
-; AVX:       # %bb.0:
-; AVX-NEXT:    vpsrlvd {{.*}}(%rip), %xmm0, %xmm0
-; AVX-NEXT:    retq
+; CHECK-LABEL: combine_vec_lshr_outofrange2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    retq
   %1 = lshr <4 x i32> %x, <i32 33, i32 34, i32 35, i32 undef>
   ret <4 x i32> %1
 }
diff --git a/test/CodeGen/X86/combine-sse41-intrinsics.ll b/test/CodeGen/X86/combine-sse41-intrinsics.ll
index 698e5bc423c1fc61c744704cdadca7bae3693ab1..7944aec30e1a4a62d2ac4d9fb6e1eaa73c17040b 100644
--- a/test/CodeGen/X86/combine-sse41-intrinsics.ll
+++ b/test/CodeGen/X86/combine-sse41-intrinsics.ll
@@ -77,7 +77,86 @@ define <8 x i16> @test3_x86_sse41_pblend_w(<8 x i16> %a0) {
   ret <8 x i16> %1
 }
 
+define double @demandedelts_blendvpd(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2) {
+; CHECK-LABEL: demandedelts_blendvpd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movapd %xmm0, %xmm3
+; CHECK-NEXT:    movaps %xmm2, %xmm0
+; CHECK-NEXT:    blendvpd %xmm0, %xmm1, %xmm3
+; CHECK-NEXT:    movapd %xmm3, %xmm0
+; CHECK-NEXT:    retq
+  %1 = shufflevector <2 x double> %a0, <2 x double> undef, <2 x i32> zeroinitializer
+  %2 = shufflevector <2 x double> %a1, <2 x double> undef, <2 x i32> zeroinitializer
+  %3 = shufflevector <2 x double> %a2, <2 x double> undef, <2 x i32> zeroinitializer
+  %4 = tail call <2 x double> @llvm.x86.sse41.blendvpd(<2 x double> %1, <2 x double> %2, <2 x double> %3)
+  %5 = extractelement <2 x double> %4, i32 0
+  ret double %5
+}
+
+define float @demandedelts_blendvps(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2) {
+; CHECK-LABEL: demandedelts_blendvps:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movaps %xmm0, %xmm3
+; CHECK-NEXT:    movaps %xmm2, %xmm0
+; CHECK-NEXT:    blendvps %xmm0, %xmm1, %xmm3
+; CHECK-NEXT:    movaps %xmm3, %xmm0
+; CHECK-NEXT:    retq
+  %1 = shufflevector <4 x float> %a0, <4 x float> undef, <4 x i32> zeroinitializer
+  %2 = shufflevector <4 x float> %a1, <4 x float> undef, <4 x i32> zeroinitializer
+  %3 = shufflevector <4 x float> %a2, <4 x float> undef, <4 x i32> zeroinitializer
+  %4 = tail call <4 x float> @llvm.x86.sse41.blendvps(<4 x float> %1, <4 x float> %2, <4 x float> %3)
+  %5 = extractelement <4 x float> %4, i32 0
+  ret float %5
+}
+
+define <16 x i8> @demandedelts_pblendvb(<16 x i8> %a0, <16 x i8> %a1, <16 x i8> %a2) {
+; CHECK-LABEL: demandedelts_pblendvb:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movdqa %xmm0, %xmm3
+; CHECK-NEXT:    movdqa %xmm2, %xmm0
+; CHECK-NEXT:    pblendvb %xmm0, %xmm1, %xmm3
+; CHECK-NEXT:    pxor %xmm0, %xmm0
+; CHECK-NEXT:    pshufb %xmm0, %xmm3
+; CHECK-NEXT:    movdqa %xmm3, %xmm0
+; CHECK-NEXT:    retq
+  %1 = shufflevector <16 x i8> %a0, <16 x i8> undef, <16 x i32> zeroinitializer
+  %2 = shufflevector <16 x i8> %a1, <16 x i8> undef, <16 x i32> zeroinitializer
+  %3 = shufflevector <16 x i8> %a2, <16 x i8> undef, <16 x i32> zeroinitializer
+  %4 = tail call <16 x i8> @llvm.x86.sse41.pblendvb(<16 x i8> %1, <16 x i8> %2, <16 x i8> %3)
+  %5 = shufflevector <16 x i8> %4, <16 x i8> undef, <16 x i32> zeroinitializer
+  ret <16 x i8> %5
+}
+
+define <2 x i64> @demandedbits_blendpd(i64 %a0, i64 %a2, <2 x double> %a3) {
+; CHECK-LABEL: demandedbits_blendpd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %rdi, %rax
+; CHECK-NEXT:    orq $1, %rax
+; CHECK-NEXT:    orq $4, %rdi
+; CHECK-NEXT:    movq %rax, %xmm1
+; CHECK-NEXT:    movq %rdi, %xmm2
+; CHECK-NEXT:    movq {{.*#+}} xmm1 = xmm1[0],zero
+; CHECK-NEXT:    movq {{.*#+}} xmm2 = xmm2[0],zero
+; CHECK-NEXT:    blendvpd %xmm0, %xmm2, %xmm1
+; CHECK-NEXT:    psrlq $11, %xmm1
+; CHECK-NEXT:    movdqa %xmm1, %xmm0
+; CHECK-NEXT:    retq
+  %1  = or i64 %a0, 1
+  %2  = or i64 %a0, 4
+  %3  = bitcast i64 %1 to double
+  %4  = bitcast i64 %2 to double
+  %5  = insertelement <2 x double> zeroinitializer, double %3, i32 0
+  %6  = insertelement <2 x double> zeroinitializer, double %4, i32 0
+  %7  = tail call <2 x double> @llvm.x86.sse41.blendvpd(<2 x double> %5, <2 x double> %6, <2 x double> %a3)
+  %8  = bitcast <2 x double> %7 to <2 x i64>
+  %9  = lshr <2 x i64> %8, <i64 11, i64 11>
+  ret <2 x i64> %9
+}
+
 declare <2 x double> @llvm.x86.sse41.blendpd(<2 x double>, <2 x double>, i32)
 declare <4 x float> @llvm.x86.sse41.blendps(<4 x float>, <4 x float>, i32)
 declare <8 x i16> @llvm.x86.sse41.pblendw(<8 x i16>, <8 x i16>, i32)
 
+declare <2 x double> @llvm.x86.sse41.blendvpd(<2 x double>, <2 x double>, <2 x double>)
+declare <4 x float> @llvm.x86.sse41.blendvps(<4 x float>, <4 x float>, <4 x float>)
+declare <16 x i8> @llvm.x86.sse41.pblendvb(<16 x i8>, <16 x i8>, <16 x i8>)
diff --git a/test/CodeGen/X86/combine-sub-ssat.ll b/test/CodeGen/X86/combine-sub-ssat.ll
new file mode 100644
index 0000000000000000000000000000000000000000..4cce35552ec2a279b4cf853488bcef79eca26dda
--- /dev/null
+++ b/test/CodeGen/X86/combine-sub-ssat.ll
@@ -0,0 +1,116 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefixes=CHECK,SSE,SSE2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefixes=CHECK,SSE,SSE41
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.2 | FileCheck %s --check-prefixes=CHECK,SSE,SSE42
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefixes=CHECK,AVX,AVX1
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefixes=CHECK,AVX,AVX2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f | FileCheck %s --check-prefixes=CHECK,AVX,AVX512,AVX512F
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw | FileCheck %s --check-prefixes=CHECK,AVX,AVX512,AVX512BW
+
+declare  i32 @llvm.ssub.sat.i32  (i32, i32)
+declare  i64 @llvm.ssub.sat.i64  (i64, i64)
+declare  <8 x i16> @llvm.ssub.sat.v8i16(<8 x i16>, <8 x i16>)
+
+; fold (ssub_sat x, undef) -> 0
+define i32 @combine_undef_i32(i32 %a0) {
+; CHECK-LABEL: combine_undef_i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    retq
+  %res = call i32 @llvm.ssub.sat.i32(i32 %a0, i32 undef)
+  ret i32 %res
+}
+
+define <8 x i16> @combine_undef_v8i16(<8 x i16> %a0) {
+; SSE-LABEL: combine_undef_v8i16:
+; SSE:       # %bb.0:
+; SSE-NEXT:    xorps %xmm0, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: combine_undef_v8i16:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vxorps %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    retq
+  %res = call <8 x i16> @llvm.ssub.sat.v8i16(<8 x i16> undef, <8 x i16> %a0)
+  ret <8 x i16> %res
+}
+
+; fold (ssub_sat c1, c2) -> c3
+define i32 @combine_constfold_i32() {
+; CHECK-LABEL: combine_constfold_i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl $-2147483547, %eax # imm = 0x80000065
+; CHECK-NEXT:    retq
+  %res = call i32 @llvm.ssub.sat.i32(i32 100, i32 2147483647)
+  ret i32 %res
+}
+
+define <8 x i16> @combine_constfold_v8i16() {
+; SSE-LABEL: combine_constfold_v8i16:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movaps {{.*#+}} xmm0 = [65535,2,254,0,65534,65282,32786,2]
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: combine_constfold_v8i16:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovaps {{.*#+}} xmm0 = [65535,2,254,0,65534,65282,32786,2]
+; AVX-NEXT:    retq
+  %res = call <8 x i16> @llvm.ssub.sat.v8i16(<8 x i16> <i16 0, i16 1, i16 255, i16 65535, i16 -1, i16 -255, i16 -32760, i16 1>, <8 x i16> <i16 1, i16 65535, i16 1, i16 65535, i16 1, i16 65535, i16 -10, i16 65535>)
+  ret <8 x i16> %res
+}
+
+define <8 x i16> @combine_constfold_undef_v8i16() {
+; SSE-LABEL: combine_constfold_undef_v8i16:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movaps {{.*#+}} xmm0 = [0,0,0,0,65534,65282,32786,2]
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: combine_constfold_undef_v8i16:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovaps {{.*#+}} xmm0 = [0,0,0,0,65534,65282,32786,2]
+; AVX-NEXT:    retq
+  %res = call <8 x i16> @llvm.ssub.sat.v8i16(<8 x i16> <i16 undef, i16 1, i16 undef, i16 65535, i16 -1, i16 -255, i16 -32760, i16 1>, <8 x i16> <i16 1, i16 undef, i16 undef, i16 65535, i16 1, i16 65535, i16 -10, i16 65535>)
+  ret <8 x i16> %res
+}
+
+; fold (ssub_sat x, 0) -> x
+define i32 @combine_zero_i32(i32 %a0) {
+; CHECK-LABEL: combine_zero_i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl %edi, %eax
+; CHECK-NEXT:    retq
+  %1 = call i32 @llvm.ssub.sat.i32(i32 %a0, i32 0)
+  ret i32 %1
+}
+
+define <8 x i16> @combine_zero_v8i16(<8 x i16> %a0) {
+; CHECK-LABEL: combine_zero_v8i16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    retq
+  %1 = call <8 x i16> @llvm.ssub.sat.v8i16(<8 x i16> %a0, <8 x i16> zeroinitializer)
+  ret <8 x i16> %1
+}
+
+; fold (ssub_sat x, x) -> 0
+define i32 @combine_self_i32(i32 %a0) {
+; CHECK-LABEL: combine_self_i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    retq
+  %1 = call i32 @llvm.ssub.sat.i32(i32 %a0, i32 %a0)
+  ret i32 %1
+}
+
+define <8 x i16> @combine_self_v8i16(<8 x i16> %a0) {
+; SSE-LABEL: combine_self_v8i16:
+; SSE:       # %bb.0:
+; SSE-NEXT:    xorps %xmm0, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: combine_self_v8i16:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vxorps %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    retq
+  %1 = call <8 x i16> @llvm.ssub.sat.v8i16(<8 x i16> %a0, <8 x i16> %a0)
+  ret <8 x i16> %1
+}
diff --git a/test/CodeGen/X86/combine-sub-usat.ll b/test/CodeGen/X86/combine-sub-usat.ll
new file mode 100644
index 0000000000000000000000000000000000000000..e26d0d667996f45cca17047d78edbc3e17c454e6
--- /dev/null
+++ b/test/CodeGen/X86/combine-sub-usat.ll
@@ -0,0 +1,116 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefixes=CHECK,SSE,SSE2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefixes=CHECK,SSE,SSE41
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.2 | FileCheck %s --check-prefixes=CHECK,SSE,SSE42
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefixes=CHECK,AVX,AVX1
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefixes=CHECK,AVX,AVX2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f | FileCheck %s --check-prefixes=CHECK,AVX,AVX512,AVX512F
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw | FileCheck %s --check-prefixes=CHECK,AVX,AVX512,AVX512BW
+
+declare  i32 @llvm.usub.sat.i32  (i32, i32)
+declare  i64 @llvm.usub.sat.i64  (i64, i64)
+declare  <8 x i16> @llvm.usub.sat.v8i16(<8 x i16>, <8 x i16>)
+
+; fold (usub_sat x, undef) -> 0
+define i32 @combine_undef_i32(i32 %a0) {
+; CHECK-LABEL: combine_undef_i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    retq
+  %res = call i32 @llvm.usub.sat.i32(i32 %a0, i32 undef)
+  ret i32 %res
+}
+
+define <8 x i16> @combine_undef_v8i16(<8 x i16> %a0) {
+; SSE-LABEL: combine_undef_v8i16:
+; SSE:       # %bb.0:
+; SSE-NEXT:    xorps %xmm0, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: combine_undef_v8i16:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vxorps %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    retq
+  %res = call <8 x i16> @llvm.usub.sat.v8i16(<8 x i16> undef, <8 x i16> %a0)
+  ret <8 x i16> %res
+}
+
+; fold (usub_sat c1, c2) -> c3
+define i32 @combine_constfold_i32() {
+; CHECK-LABEL: combine_constfold_i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    retq
+  %res = call i32 @llvm.usub.sat.i32(i32 100, i32 4294967295)
+  ret i32 %res
+}
+
+define <8 x i16> @combine_constfold_v8i16() {
+; SSE-LABEL: combine_constfold_v8i16:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movaps {{.*#+}} xmm0 = [0,0,254,0,65534,0,0,0]
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: combine_constfold_v8i16:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovaps {{.*#+}} xmm0 = [0,0,254,0,65534,0,0,0]
+; AVX-NEXT:    retq
+  %res = call <8 x i16> @llvm.usub.sat.v8i16(<8 x i16> <i16 0, i16 1, i16 255, i16 65535, i16 -1, i16 -255, i16 -65535, i16 1>, <8 x i16> <i16 1, i16 65535, i16 1, i16 65535, i16 1, i16 65535, i16 1, i16 65535>)
+  ret <8 x i16> %res
+}
+
+define <8 x i16> @combine_constfold_undef_v8i16() {
+; SSE-LABEL: combine_constfold_undef_v8i16:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movaps {{.*#+}} xmm0 = [0,0,0,0,65534,0,0,0]
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: combine_constfold_undef_v8i16:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovaps {{.*#+}} xmm0 = [0,0,0,0,65534,0,0,0]
+; AVX-NEXT:    retq
+  %res = call <8 x i16> @llvm.usub.sat.v8i16(<8 x i16> <i16 undef, i16 1, i16 undef, i16 65535, i16 -1, i16 -255, i16 -65535, i16 1>, <8 x i16> <i16 1, i16 undef, i16 undef, i16 65535, i16 1, i16 65535, i16 1, i16 65535>)
+  ret <8 x i16> %res
+}
+
+; fold (usub_sat x, 0) -> x
+define i32 @combine_zero_i32(i32 %a0) {
+; CHECK-LABEL: combine_zero_i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl %edi, %eax
+; CHECK-NEXT:    retq
+  %1 = call i32 @llvm.usub.sat.i32(i32 %a0, i32 0)
+  ret i32 %1
+}
+
+define <8 x i16> @combine_zero_v8i16(<8 x i16> %a0) {
+; CHECK-LABEL: combine_zero_v8i16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    retq
+  %1 = call <8 x i16> @llvm.usub.sat.v8i16(<8 x i16> %a0, <8 x i16> zeroinitializer)
+  ret <8 x i16> %1
+}
+
+; fold (usub_sat x, x) -> 0
+define i32 @combine_self_i32(i32 %a0) {
+; CHECK-LABEL: combine_self_i32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    retq
+  %1 = call i32 @llvm.usub.sat.i32(i32 %a0, i32 %a0)
+  ret i32 %1
+}
+
+define <8 x i16> @combine_self_v8i16(<8 x i16> %a0) {
+; SSE-LABEL: combine_self_v8i16:
+; SSE:       # %bb.0:
+; SSE-NEXT:    xorps %xmm0, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: combine_self_v8i16:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vxorps %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    retq
+  %1 = call <8 x i16> @llvm.usub.sat.v8i16(<8 x i16> %a0, <8 x i16> %a0)
+  ret <8 x i16> %1
+}
diff --git a/test/CodeGen/X86/commute-xop.ll b/test/CodeGen/X86/commute-xop.ll
index 606ff126dad720a7c774133493fcbbe3270c209d..789afbb99bc618833a5e239d4d903ece0d0951c6 100644
--- a/test/CodeGen/X86/commute-xop.ll
+++ b/test/CodeGen/X86/commute-xop.ll
@@ -74,12 +74,12 @@ define <4 x i32> @commute_fold_vpcomud(<4 x i32>* %a0, <4 x i32> %a1) {
 ; X32-LABEL: commute_fold_vpcomud:
 ; X32:       # %bb.0:
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    vpcomequd (%eax), %xmm0, %xmm0
+; X32-NEXT:    vpcomeqd (%eax), %xmm0, %xmm0
 ; X32-NEXT:    retl
 ;
 ; X64-LABEL: commute_fold_vpcomud:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpcomequd (%rdi), %xmm0, %xmm0
+; X64-NEXT:    vpcomeqd (%rdi), %xmm0, %xmm0
 ; X64-NEXT:    retq
   %1 = load <4 x i32>, <4 x i32>* %a0
   %2 = call <4 x i32> @llvm.x86.xop.vpcomud(<4 x i32> %1, <4 x i32> %a1, i8 4) ; vpcomequd
@@ -91,12 +91,12 @@ define <2 x i64> @commute_fold_vpcomuq(<2 x i64>* %a0, <2 x i64> %a1) {
 ; X32-LABEL: commute_fold_vpcomuq:
 ; X32:       # %bb.0:
 ; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    vpcomnequq (%eax), %xmm0, %xmm0
+; X32-NEXT:    vpcomneqq (%eax), %xmm0, %xmm0
 ; X32-NEXT:    retl
 ;
 ; X64-LABEL: commute_fold_vpcomuq:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpcomnequq (%rdi), %xmm0, %xmm0
+; X64-NEXT:    vpcomneqq (%rdi), %xmm0, %xmm0
 ; X64-NEXT:    retq
   %1 = load <2 x i64>, <2 x i64>* %a0
   %2 = call <2 x i64> @llvm.x86.xop.vpcomuq(<2 x i64> %1, <2 x i64> %a1, i8 5) ; vpcomnequq
@@ -107,13 +107,12 @@ declare <2 x i64> @llvm.x86.xop.vpcomuq(<2 x i64>, <2 x i64>, i8) nounwind readn
 define <8 x i16> @commute_fold_vpcomuw(<8 x i16>* %a0, <8 x i16> %a1) {
 ; X32-LABEL: commute_fold_vpcomuw:
 ; X32:       # %bb.0:
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    vpcomfalseuw (%eax), %xmm0, %xmm0
+; X32-NEXT:    vxorps %xmm0, %xmm0, %xmm0
 ; X32-NEXT:    retl
 ;
 ; X64-LABEL: commute_fold_vpcomuw:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpcomfalseuw (%rdi), %xmm0, %xmm0
+; X64-NEXT:    vxorps %xmm0, %xmm0, %xmm0
 ; X64-NEXT:    retq
   %1 = load <8 x i16>, <8 x i16>* %a0
   %2 = call <8 x i16> @llvm.x86.xop.vpcomuw(<8 x i16> %1, <8 x i16> %a1, i8 6) ; vpcomfalseuw
@@ -124,13 +123,12 @@ declare <8 x i16> @llvm.x86.xop.vpcomuw(<8 x i16>, <8 x i16>, i8) nounwind readn
 define <8 x i16> @commute_fold_vpcomw(<8 x i16>* %a0, <8 x i16> %a1) {
 ; X32-LABEL: commute_fold_vpcomw:
 ; X32:       # %bb.0:
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    vpcomtruew (%eax), %xmm0, %xmm0
+; X32-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
 ; X32-NEXT:    retl
 ;
 ; X64-LABEL: commute_fold_vpcomw:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpcomtruew (%rdi), %xmm0, %xmm0
+; X64-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
 ; X64-NEXT:    retq
   %1 = load <8 x i16>, <8 x i16>* %a0
   %2 = call <8 x i16> @llvm.x86.xop.vpcomw(<8 x i16> %1, <8 x i16> %a1, i8 7) ; vpcomtruew
diff --git a/test/CodeGen/X86/compact-unwind.ll b/test/CodeGen/X86/compact-unwind.ll
index f8266a10cfb5ab052be376997ed36f9aed7f16c1..ab85f599b3ae557f3f35f5bbe5980abbe0670916 100644
--- a/test/CodeGen/X86/compact-unwind.ll
+++ b/test/CodeGen/X86/compact-unwind.ll
@@ -1,8 +1,8 @@
-; RUN: llc < %s -disable-fp-elim -mtriple x86_64-apple-darwin11 -mcpu corei7 | FileCheck -check-prefix=ASM %s
-; RUN: llc < %s -disable-fp-elim -mtriple x86_64-apple-darwin11 -mcpu corei7 -filetype=obj -o - \
+; RUN: llc < %s -frame-pointer=all -mtriple x86_64-apple-darwin11 -mcpu corei7 | FileCheck -check-prefix=ASM %s
+; RUN: llc < %s -frame-pointer=all -mtriple x86_64-apple-darwin11 -mcpu corei7 -filetype=obj -o - \
 ; RUN:  | llvm-objdump -triple x86_64-apple-darwin11 -unwind-info - \
 ; RUN:  | FileCheck -check-prefix=CU %s
-; RUN: llc < %s -disable-fp-elim -mtriple x86_64-apple-darwin11 -mcpu corei7 \
+; RUN: llc < %s -frame-pointer=all -mtriple x86_64-apple-darwin11 -mcpu corei7 \
 ; RUN:  | llvm-mc -triple x86_64-apple-darwin11 -filetype=obj -o - \
 ; RUN:  | llvm-objdump -triple x86_64-apple-darwin11 -unwind-info - \
 ; RUN:  | FileCheck -check-prefix=FROM-ASM %s
diff --git a/test/CodeGen/X86/copy-eflags.ll b/test/CodeGen/X86/copy-eflags.ll
index 1e9a598c6511198e07010d0e6fc256e00870fb89..018ea8bbee29bdb1c66dbbac950d4c7902a1a773 100644
--- a/test/CodeGen/X86/copy-eflags.ll
+++ b/test/CodeGen/X86/copy-eflags.ll
@@ -102,13 +102,13 @@ define i32 @test2(i32* %ptr) nounwind {
 ; X32-NEXT:    calll external
 ; X32-NEXT:    addl $4, %esp
 ; X32-NEXT:    testb %bl, %bl
-; X32-NEXT:    je .LBB1_1
-; X32-NEXT:  # %bb.2: # %else
-; X32-NEXT:    xorl %eax, %eax
+; X32-NEXT:    jne .LBB1_2
+; X32-NEXT:  # %bb.1: # %then
+; X32-NEXT:    movl $64, %eax
 ; X32-NEXT:    popl %ebx
 ; X32-NEXT:    retl
-; X32-NEXT:  .LBB1_1: # %then
-; X32-NEXT:    movl $64, %eax
+; X32-NEXT:  .LBB1_2: # %else
+; X32-NEXT:    xorl %eax, %eax
 ; X32-NEXT:    popl %ebx
 ; X32-NEXT:    retl
 ;
@@ -120,13 +120,13 @@ define i32 @test2(i32* %ptr) nounwind {
 ; X64-NEXT:    movl $42, %edi
 ; X64-NEXT:    callq external
 ; X64-NEXT:    testb %bl, %bl
-; X64-NEXT:    je .LBB1_1
-; X64-NEXT:  # %bb.2: # %else
-; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    jne .LBB1_2
+; X64-NEXT:  # %bb.1: # %then
+; X64-NEXT:    movl $64, %eax
 ; X64-NEXT:    popq %rbx
 ; X64-NEXT:    retq
-; X64-NEXT:  .LBB1_1: # %then
-; X64-NEXT:    movl $64, %eax
+; X64-NEXT:  .LBB1_2: # %else
+; X64-NEXT:    xorl %eax, %eax
 ; X64-NEXT:    popq %rbx
 ; X64-NEXT:    retq
 entry:
diff --git a/test/CodeGen/X86/crash-O0.ll b/test/CodeGen/X86/crash-O0.ll
index 1a234d45cb21904940233ffb55c2e5f3a42179d3..d23a996f64e0d692d4b5817d3cc0618f7ac8851e 100644
--- a/test/CodeGen/X86/crash-O0.ll
+++ b/test/CodeGen/X86/crash-O0.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -O0 -relocation-model=pic -disable-fp-elim < %s | FileCheck %s
+; RUN: llc -O0 -relocation-model=pic -frame-pointer=all < %s | FileCheck %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 target triple = "x86_64-apple-darwin10"
 
diff --git a/test/CodeGen/X86/debug-loclists.ll b/test/CodeGen/X86/debug-loclists.ll
index 874cdc196e4d04e83a77b9cbde31ca67df099bd3..0c2ab3dfad5a9077747f9eb623b82110e8c69ffc 100644
--- a/test/CodeGen/X86/debug-loclists.ll
+++ b/test/CodeGen/X86/debug-loclists.ll
@@ -5,13 +5,13 @@
 ; CHECK-NEXT:               DW_AT_location [DW_FORM_sec_offset]   (0x0000000c
 ; CHECK-NEXT:                  [0x0000000000000000, 0x0000000000000004): DW_OP_breg5 RDI+0
 ; CHECK-NEXT:                  [0x0000000000000004, 0x0000000000000012): DW_OP_breg3 RBX+0)
-; CHECK-NEXT:               DW_AT_name [DW_FORM_strx1]    ( indexed (0000000e) string = "a")
+; CHECK-NEXT:               DW_AT_name [DW_FORM_strx1]    (indexed (0000000e) string = "a")
 ; CHECK-NEXT:               DW_AT_decl_file [DW_FORM_data1]       ("/home/folder{{\\|\/}}test.cc")
 ; CHECK-NEXT:               DW_AT_decl_line [DW_FORM_data1]       (6)
 ; CHECK-NEXT:               DW_AT_type [DW_FORM_ref4]     (cu + 0x0040 => {0x00000040} "A")
 
 ; CHECK:      .debug_loclists contents:
-; CHECK-NEXT: 0x00000000: locations list header: length = 0x00000017, version = 0x0005, addr_size = 0x08, seg_size = 0x00, offset_entry_count = 0x00000000
+; CHECK-NEXT: 0x00000000: locations list header: length = 0x00000015, version = 0x0005, addr_size = 0x08, seg_size = 0x00, offset_entry_count = 0x00000000
 ; CHECK-NEXT: 0x00000000:
 ; CHECK-NEXT:  [0x0000000000000000, 0x0000000000000004): DW_OP_breg5 RDI+0
 ; CHECK-NEXT:  [0x0000000000000004, 0x0000000000000012): DW_OP_breg3 RBX+0
@@ -32,13 +32,13 @@
 ; ASM-NEXT:  .byte 4                               # DW_LLE_offset_pair
 ; ASM-NEXT:  .uleb128 .Lfunc_begin0-.Lfunc_begin0  # starting offset
 ; ASM-NEXT:  .uleb128 .Ltmp0-.Lfunc_begin0         # ending offset
-; ASM-NEXT:  .short 2                              # Loc expr size
+; ASM-NEXT:  .byte 2                               # Loc expr size
 ; ASM-NEXT:  .byte 117                             # DW_OP_breg5
 ; ASM-NEXT:  .byte 0                               # 0
 ; ASM-NEXT:  .byte 4                               # DW_LLE_offset_pair
 ; ASM-NEXT:  .uleb128 .Ltmp0-.Lfunc_begin0         # starting offset
 ; ASM-NEXT:  .uleb128 .Ltmp1-.Lfunc_begin0         # ending offset
-; ASM-NEXT:  .short 2                              # Loc expr size
+; ASM-NEXT:  .byte 2                               # Loc expr size
 ; ASM-NEXT:  .byte 115                             # DW_OP_breg3
 ; ASM-NEXT:  .byte 0                               # 0
 ; ASM-NEXT:  .byte 0                               # DW_LLE_end_of_list
diff --git a/test/CodeGen/X86/discriminate-mem-ops.ll b/test/CodeGen/X86/discriminate-mem-ops.ll
index b77a91fafd2cd8681ed64d8882f4b4544f77f209..a30dc22a0d9c02dc26193303142e169bd8e57361 100644
--- a/test/CodeGen/X86/discriminate-mem-ops.ll
+++ b/test/CodeGen/X86/discriminate-mem-ops.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s | FileCheck %s
+; RUN: llc -x86-discriminate-memops  < %s | FileCheck %s
 ;
 ; original source, compiled with -O3 -gmlt -fdebug-info-for-profiling:
 ; int sum(int* arr, int pos1, int pos2) {
diff --git a/test/CodeGen/X86/divide-by-constant.ll b/test/CodeGen/X86/divide-by-constant.ll
index 9fbef11f87bf825c5ccf2521dd1dc1d68a5308a9..23a3d1e095ca793436ab7059eb1e70d9d414bc88 100644
--- a/test/CodeGen/X86/divide-by-constant.ll
+++ b/test/CodeGen/X86/divide-by-constant.ll
@@ -441,12 +441,12 @@ define { i64, i32 } @PR38622_signed(i64) nounwind {
 ; X64-NEXT:    movabsq $1237940039285380275, %rcx # imm = 0x112E0BE826D694B3
 ; X64-NEXT:    movq %rdi, %rax
 ; X64-NEXT:    imulq %rcx
+; X64-NEXT:    movq %rdx, %rax
 ; X64-NEXT:    movq %rdx, %rcx
 ; X64-NEXT:    shrq $63, %rcx
-; X64-NEXT:    sarq $28, %rdx
-; X64-NEXT:    leaq (%rdx,%rcx), %rax
-; X64-NEXT:    addl %ecx, %edx
-; X64-NEXT:    imull $-294967296, %edx, %ecx # imm = 0xEE6B2800
+; X64-NEXT:    sarq $28, %rax
+; X64-NEXT:    addq %rcx, %rax
+; X64-NEXT:    imull $-294967296, %eax, %ecx # imm = 0xEE6B2800
 ; X64-NEXT:    subl %ecx, %edi
 ; X64-NEXT:    movl %edi, %edx
 ; X64-NEXT:    retq
diff --git a/test/CodeGen/X86/empty-functions.ll b/test/CodeGen/X86/empty-functions.ll
index a310fb3407866bdf0b37878db333c939d5ebcd1b..faded65ac6dc5f31e3dd8bca6712b07978dd7eb8 100644
--- a/test/CodeGen/X86/empty-functions.ll
+++ b/test/CodeGen/X86/empty-functions.ll
@@ -1,7 +1,7 @@
 ; RUN: llc < %s -mtriple=x86_64-apple-darwin | FileCheck -check-prefix=CHECK-NO-FP %s
-; RUN: llc < %s -mtriple=x86_64-apple-darwin -disable-fp-elim | FileCheck -check-prefix=CHECK-FP %s
+; RUN: llc < %s -mtriple=x86_64-apple-darwin -frame-pointer=all | FileCheck -check-prefix=CHECK-FP %s
 ; RUN: llc < %s -mtriple=x86_64-linux-gnu | FileCheck -check-prefix=LINUX-NO-FP %s
-; RUN: llc < %s -mtriple=x86_64-linux-gnu -disable-fp-elim | FileCheck -check-prefix=LINUX-FP %s
+; RUN: llc < %s -mtriple=x86_64-linux-gnu -frame-pointer=all | FileCheck -check-prefix=LINUX-FP %s
 
 define void @func() {
 entry:
diff --git a/test/CodeGen/X86/expand-integer-x86_64-intrinsic-error.ll b/test/CodeGen/X86/expand-integer-x86_64-intrinsic-error.ll
new file mode 100644
index 0000000000000000000000000000000000000000..34f3c258f73873eabe51019285f09e8cf02e1247
--- /dev/null
+++ b/test/CodeGen/X86/expand-integer-x86_64-intrinsic-error.ll
@@ -0,0 +1,11 @@
+;RUN: not llc < %s -mtriple=i686-unknown-unknown -mattr=sse2 2>&1 | FileCheck %s --check-prefix=CHECK
+
+; Make sure we generate fatal error from the type legalizer for using a 64-bit
+; mode intrinsics in 32-bit mode. We used to use an llvm_unreachable.
+
+; CHECK: LLVM ERROR: Do not know how to expand the result of this operator!
+define i64 @test_x86_sse2_cvtsd2si64(<2 x double> %a0) {
+  %res = call i64 @llvm.x86.sse2.cvtsd2si64(<2 x double> %a0) ; <i64> [#uses=1]
+  ret i64 %res
+}
+declare i64 @llvm.x86.sse2.cvtsd2si64(<2 x double>) nounwind readnone
diff --git a/test/CodeGen/X86/extract-bits.ll b/test/CodeGen/X86/extract-bits.ll
index b69c0c1f48b7572e2a46b77430768bed2c9c04ba..7921e9685cefcd02008afeb794326d91203f56a0 100644
--- a/test/CodeGen/X86/extract-bits.ll
+++ b/test/CodeGen/X86/extract-bits.ll
@@ -665,9 +665,9 @@ define i64 @bextr64_a0(i64 %val, i64 %numskipbits, i64 %numlowbits) nounwind {
 ;
 ; X64-BMI1NOTBM-LABEL: bextr64_a0:
 ; X64-BMI1NOTBM:       # %bb.0:
-; X64-BMI1NOTBM-NEXT:    shlq $8, %rdx
+; X64-BMI1NOTBM-NEXT:    shll $8, %edx
 ; X64-BMI1NOTBM-NEXT:    movzbl %sil, %eax
-; X64-BMI1NOTBM-NEXT:    orq %rdx, %rax
+; X64-BMI1NOTBM-NEXT:    orl %edx, %eax
 ; X64-BMI1NOTBM-NEXT:    bextrq %rax, %rdi, %rax
 ; X64-BMI1NOTBM-NEXT:    retq
 ;
@@ -813,7 +813,7 @@ define i64 @bextr64_a0_arithmetic(i64 %val, i64 %numskipbits, i64 %numlowbits) n
 ; X64-BMI1NOTBM-NEXT:    movq %rsi, %rcx
 ; X64-BMI1NOTBM-NEXT:    # kill: def $cl killed $cl killed $rcx
 ; X64-BMI1NOTBM-NEXT:    sarq %cl, %rdi
-; X64-BMI1NOTBM-NEXT:    shlq $8, %rdx
+; X64-BMI1NOTBM-NEXT:    shll $8, %edx
 ; X64-BMI1NOTBM-NEXT:    bextrq %rdx, %rdi, %rax
 ; X64-BMI1NOTBM-NEXT:    retq
 ;
@@ -953,10 +953,9 @@ define i64 @bextr64_a1_indexzext(i64 %val, i8 zeroext %numskipbits, i8 zeroext %
 ;
 ; X64-BMI1NOTBM-LABEL: bextr64_a1_indexzext:
 ; X64-BMI1NOTBM:       # %bb.0:
-; X64-BMI1NOTBM-NEXT:    # kill: def $edx killed $edx def $rdx
-; X64-BMI1NOTBM-NEXT:    shlq $8, %rdx
+; X64-BMI1NOTBM-NEXT:    shll $8, %edx
 ; X64-BMI1NOTBM-NEXT:    movzbl %sil, %eax
-; X64-BMI1NOTBM-NEXT:    orq %rdx, %rax
+; X64-BMI1NOTBM-NEXT:    orl %edx, %eax
 ; X64-BMI1NOTBM-NEXT:    bextrq %rax, %rdi, %rax
 ; X64-BMI1NOTBM-NEXT:    retq
 ;
@@ -1104,9 +1103,9 @@ define i64 @bextr64_a2_load(i64* %w, i64 %numskipbits, i64 %numlowbits) nounwind
 ;
 ; X64-BMI1NOTBM-LABEL: bextr64_a2_load:
 ; X64-BMI1NOTBM:       # %bb.0:
-; X64-BMI1NOTBM-NEXT:    shlq $8, %rdx
+; X64-BMI1NOTBM-NEXT:    shll $8, %edx
 ; X64-BMI1NOTBM-NEXT:    movzbl %sil, %eax
-; X64-BMI1NOTBM-NEXT:    orq %rdx, %rax
+; X64-BMI1NOTBM-NEXT:    orl %edx, %eax
 ; X64-BMI1NOTBM-NEXT:    bextrq %rax, (%rdi), %rax
 ; X64-BMI1NOTBM-NEXT:    retq
 ;
@@ -1251,10 +1250,9 @@ define i64 @bextr64_a3_load_indexzext(i64* %w, i8 zeroext %numskipbits, i8 zeroe
 ;
 ; X64-BMI1NOTBM-LABEL: bextr64_a3_load_indexzext:
 ; X64-BMI1NOTBM:       # %bb.0:
-; X64-BMI1NOTBM-NEXT:    # kill: def $edx killed $edx def $rdx
-; X64-BMI1NOTBM-NEXT:    shlq $8, %rdx
+; X64-BMI1NOTBM-NEXT:    shll $8, %edx
 ; X64-BMI1NOTBM-NEXT:    movzbl %sil, %eax
-; X64-BMI1NOTBM-NEXT:    orq %rdx, %rax
+; X64-BMI1NOTBM-NEXT:    orl %edx, %eax
 ; X64-BMI1NOTBM-NEXT:    bextrq %rax, (%rdi), %rax
 ; X64-BMI1NOTBM-NEXT:    retq
 ;
@@ -1399,9 +1397,9 @@ define i64 @bextr64_a4_commutative(i64 %val, i64 %numskipbits, i64 %numlowbits)
 ;
 ; X64-BMI1NOTBM-LABEL: bextr64_a4_commutative:
 ; X64-BMI1NOTBM:       # %bb.0:
-; X64-BMI1NOTBM-NEXT:    shlq $8, %rdx
+; X64-BMI1NOTBM-NEXT:    shll $8, %edx
 ; X64-BMI1NOTBM-NEXT:    movzbl %sil, %eax
-; X64-BMI1NOTBM-NEXT:    orq %rdx, %rax
+; X64-BMI1NOTBM-NEXT:    orl %edx, %eax
 ; X64-BMI1NOTBM-NEXT:    bextrq %rax, %rdi, %rax
 ; X64-BMI1NOTBM-NEXT:    retq
 ;
@@ -1586,9 +1584,9 @@ define i64 @bextr64_a5_skipextrauses(i64 %val, i64 %numskipbits, i64 %numlowbits
 ; X64-BMI1NOTBM-LABEL: bextr64_a5_skipextrauses:
 ; X64-BMI1NOTBM:       # %bb.0:
 ; X64-BMI1NOTBM-NEXT:    pushq %rbx
-; X64-BMI1NOTBM-NEXT:    shlq $8, %rdx
+; X64-BMI1NOTBM-NEXT:    shll $8, %edx
 ; X64-BMI1NOTBM-NEXT:    movzbl %sil, %eax
-; X64-BMI1NOTBM-NEXT:    orq %rdx, %rax
+; X64-BMI1NOTBM-NEXT:    orl %edx, %eax
 ; X64-BMI1NOTBM-NEXT:    bextrq %rax, %rdi, %rbx
 ; X64-BMI1NOTBM-NEXT:    movq %rsi, %rdi
 ; X64-BMI1NOTBM-NEXT:    callq use64
@@ -1614,286 +1612,493 @@ define i64 @bextr64_a5_skipextrauses(i64 %val, i64 %numskipbits, i64 %numlowbits
   ret i64 %masked
 }
 
-; ---------------------------------------------------------------------------- ;
-; Pattern b. 32-bit
-; ---------------------------------------------------------------------------- ;
+; 64-bit, but with 32-bit output
 
-define i32 @bextr32_b0(i32 %val, i32 %numskipbits, i32 %numlowbits) nounwind {
-; X86-NOBMI-LABEL: bextr32_b0:
+; Everything done in 64-bit, truncation happens last.
+define i32 @bextr64_32_a0(i64 %val, i64 %numskipbits, i64 %numlowbits) nounwind {
+; X86-NOBMI-LABEL: bextr64_32_a0:
 ; X86-NOBMI:       # %bb.0:
+; X86-NOBMI-NEXT:    pushl %edi
 ; X86-NOBMI-NEXT:    pushl %esi
 ; X86-NOBMI-NEXT:    movb {{[0-9]+}}(%esp), %dl
 ; X86-NOBMI-NEXT:    movb {{[0-9]+}}(%esp), %cl
-; X86-NOBMI-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NOBMI-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NOBMI-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NOBMI-NEXT:    movl %edi, %esi
 ; X86-NOBMI-NEXT:    shrl %cl, %esi
-; X86-NOBMI-NEXT:    movl $-1, %eax
+; X86-NOBMI-NEXT:    shrdl %cl, %edi, %eax
+; X86-NOBMI-NEXT:    testb $32, %cl
+; X86-NOBMI-NEXT:    jne .LBB14_2
+; X86-NOBMI-NEXT:  # %bb.1:
+; X86-NOBMI-NEXT:    movl %eax, %esi
+; X86-NOBMI-NEXT:  .LBB14_2:
+; X86-NOBMI-NEXT:    movl $1, %edi
 ; X86-NOBMI-NEXT:    movl %edx, %ecx
-; X86-NOBMI-NEXT:    shll %cl, %eax
-; X86-NOBMI-NEXT:    notl %eax
+; X86-NOBMI-NEXT:    shll %cl, %edi
+; X86-NOBMI-NEXT:    xorl %eax, %eax
+; X86-NOBMI-NEXT:    testb $32, %dl
+; X86-NOBMI-NEXT:    jne .LBB14_4
+; X86-NOBMI-NEXT:  # %bb.3:
+; X86-NOBMI-NEXT:    movl %edi, %eax
+; X86-NOBMI-NEXT:  .LBB14_4:
+; X86-NOBMI-NEXT:    decl %eax
 ; X86-NOBMI-NEXT:    andl %esi, %eax
 ; X86-NOBMI-NEXT:    popl %esi
+; X86-NOBMI-NEXT:    popl %edi
 ; X86-NOBMI-NEXT:    retl
 ;
-; X86-BMI1NOTBM-LABEL: bextr32_b0:
+; X86-BMI1NOTBM-LABEL: bextr64_32_a0:
 ; X86-BMI1NOTBM:       # %bb.0:
-; X86-BMI1NOTBM-NEXT:    movb {{[0-9]+}}(%esp), %al
-; X86-BMI1NOTBM-NEXT:    shll $8, %eax
-; X86-BMI1NOTBM-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
-; X86-BMI1NOTBM-NEXT:    orl %eax, %ecx
-; X86-BMI1NOTBM-NEXT:    bextrl %ecx, {{[0-9]+}}(%esp), %eax
+; X86-BMI1NOTBM-NEXT:    pushl %edi
+; X86-BMI1NOTBM-NEXT:    pushl %esi
+; X86-BMI1NOTBM-NEXT:    movb {{[0-9]+}}(%esp), %dl
+; X86-BMI1NOTBM-NEXT:    movb {{[0-9]+}}(%esp), %cl
+; X86-BMI1NOTBM-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-BMI1NOTBM-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-BMI1NOTBM-NEXT:    movl %edi, %esi
+; X86-BMI1NOTBM-NEXT:    shrl %cl, %esi
+; X86-BMI1NOTBM-NEXT:    shrdl %cl, %edi, %eax
+; X86-BMI1NOTBM-NEXT:    testb $32, %cl
+; X86-BMI1NOTBM-NEXT:    jne .LBB14_2
+; X86-BMI1NOTBM-NEXT:  # %bb.1:
+; X86-BMI1NOTBM-NEXT:    movl %eax, %esi
+; X86-BMI1NOTBM-NEXT:  .LBB14_2:
+; X86-BMI1NOTBM-NEXT:    movl $1, %edi
+; X86-BMI1NOTBM-NEXT:    movl %edx, %ecx
+; X86-BMI1NOTBM-NEXT:    shll %cl, %edi
+; X86-BMI1NOTBM-NEXT:    xorl %eax, %eax
+; X86-BMI1NOTBM-NEXT:    testb $32, %dl
+; X86-BMI1NOTBM-NEXT:    jne .LBB14_4
+; X86-BMI1NOTBM-NEXT:  # %bb.3:
+; X86-BMI1NOTBM-NEXT:    movl %edi, %eax
+; X86-BMI1NOTBM-NEXT:  .LBB14_4:
+; X86-BMI1NOTBM-NEXT:    decl %eax
+; X86-BMI1NOTBM-NEXT:    andl %esi, %eax
+; X86-BMI1NOTBM-NEXT:    popl %esi
+; X86-BMI1NOTBM-NEXT:    popl %edi
 ; X86-BMI1NOTBM-NEXT:    retl
 ;
-; X86-BMI1BMI2-LABEL: bextr32_b0:
+; X86-BMI1BMI2-LABEL: bextr64_32_a0:
 ; X86-BMI1BMI2:       # %bb.0:
-; X86-BMI1BMI2-NEXT:    movb {{[0-9]+}}(%esp), %al
+; X86-BMI1BMI2-NEXT:    pushl %ebx
+; X86-BMI1BMI2-NEXT:    movb {{[0-9]+}}(%esp), %bl
 ; X86-BMI1BMI2-NEXT:    movb {{[0-9]+}}(%esp), %cl
-; X86-BMI1BMI2-NEXT:    shrxl %ecx, {{[0-9]+}}(%esp), %ecx
-; X86-BMI1BMI2-NEXT:    bzhil %eax, %ecx, %eax
+; X86-BMI1BMI2-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-BMI1BMI2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-BMI1BMI2-NEXT:    shrdl %cl, %eax, %edx
+; X86-BMI1BMI2-NEXT:    testb $32, %cl
+; X86-BMI1BMI2-NEXT:    je .LBB14_2
+; X86-BMI1BMI2-NEXT:  # %bb.1:
+; X86-BMI1BMI2-NEXT:    shrxl %ecx, %eax, %edx
+; X86-BMI1BMI2-NEXT:  .LBB14_2:
+; X86-BMI1BMI2-NEXT:    xorl %eax, %eax
+; X86-BMI1BMI2-NEXT:    testb $32, %bl
+; X86-BMI1BMI2-NEXT:    jne .LBB14_4
+; X86-BMI1BMI2-NEXT:  # %bb.3:
+; X86-BMI1BMI2-NEXT:    movl $1, %eax
+; X86-BMI1BMI2-NEXT:    shlxl %ebx, %eax, %eax
+; X86-BMI1BMI2-NEXT:  .LBB14_4:
+; X86-BMI1BMI2-NEXT:    decl %eax
+; X86-BMI1BMI2-NEXT:    andl %edx, %eax
+; X86-BMI1BMI2-NEXT:    popl %ebx
 ; X86-BMI1BMI2-NEXT:    retl
 ;
-; X64-NOBMI-LABEL: bextr32_b0:
+; X64-NOBMI-LABEL: bextr64_32_a0:
 ; X64-NOBMI:       # %bb.0:
-; X64-NOBMI-NEXT:    movl %esi, %ecx
-; X64-NOBMI-NEXT:    # kill: def $cl killed $cl killed $ecx
-; X64-NOBMI-NEXT:    shrl %cl, %edi
-; X64-NOBMI-NEXT:    movl $-1, %eax
+; X64-NOBMI-NEXT:    movq %rsi, %rcx
+; X64-NOBMI-NEXT:    # kill: def $cl killed $cl killed $rcx
+; X64-NOBMI-NEXT:    shrq %cl, %rdi
+; X64-NOBMI-NEXT:    movl $1, %eax
 ; X64-NOBMI-NEXT:    movl %edx, %ecx
-; X64-NOBMI-NEXT:    shll %cl, %eax
-; X64-NOBMI-NEXT:    notl %eax
+; X64-NOBMI-NEXT:    shlq %cl, %rax
+; X64-NOBMI-NEXT:    decl %eax
 ; X64-NOBMI-NEXT:    andl %edi, %eax
+; X64-NOBMI-NEXT:    # kill: def $eax killed $eax killed $rax
 ; X64-NOBMI-NEXT:    retq
 ;
-; X64-BMI1NOTBM-LABEL: bextr32_b0:
+; X64-BMI1NOTBM-LABEL: bextr64_32_a0:
 ; X64-BMI1NOTBM:       # %bb.0:
-; X64-BMI1NOTBM-NEXT:    shll $8, %edx
-; X64-BMI1NOTBM-NEXT:    movzbl %sil, %eax
-; X64-BMI1NOTBM-NEXT:    orl %edx, %eax
-; X64-BMI1NOTBM-NEXT:    bextrl %eax, %edi, %eax
+; X64-BMI1NOTBM-NEXT:    movq %rsi, %rcx
+; X64-BMI1NOTBM-NEXT:    # kill: def $cl killed $cl killed $rcx
+; X64-BMI1NOTBM-NEXT:    shrq %cl, %rdi
+; X64-BMI1NOTBM-NEXT:    movl $1, %eax
+; X64-BMI1NOTBM-NEXT:    movl %edx, %ecx
+; X64-BMI1NOTBM-NEXT:    shlq %cl, %rax
+; X64-BMI1NOTBM-NEXT:    decl %eax
+; X64-BMI1NOTBM-NEXT:    andl %edi, %eax
+; X64-BMI1NOTBM-NEXT:    # kill: def $eax killed $eax killed $rax
 ; X64-BMI1NOTBM-NEXT:    retq
 ;
-; X64-BMI1BMI2-LABEL: bextr32_b0:
+; X64-BMI1BMI2-LABEL: bextr64_32_a0:
 ; X64-BMI1BMI2:       # %bb.0:
-; X64-BMI1BMI2-NEXT:    shrxl %esi, %edi, %eax
-; X64-BMI1BMI2-NEXT:    bzhil %edx, %eax, %eax
+; X64-BMI1BMI2-NEXT:    shrxq %rsi, %rdi, %rax
+; X64-BMI1BMI2-NEXT:    movl $1, %ecx
+; X64-BMI1BMI2-NEXT:    shlxq %rdx, %rcx, %rcx
+; X64-BMI1BMI2-NEXT:    decl %ecx
+; X64-BMI1BMI2-NEXT:    andl %ecx, %eax
+; X64-BMI1BMI2-NEXT:    # kill: def $eax killed $eax killed $rax
 ; X64-BMI1BMI2-NEXT:    retq
-  %shifted = lshr i32 %val, %numskipbits
-  %notmask = shl i32 -1, %numlowbits
-  %mask = xor i32 %notmask, -1
-  %masked = and i32 %mask, %shifted
-  ret i32 %masked
+  %shifted = lshr i64 %val, %numskipbits
+  %onebit = shl i64 1, %numlowbits
+  %mask = add nsw i64 %onebit, -1
+  %masked = and i64 %mask, %shifted
+  %res = trunc i64 %masked to i32
+  ret i32 %res
 }
 
-define i32 @bextr32_b1_indexzext(i32 %val, i8 zeroext %numskipbits, i8 zeroext %numlowbits) nounwind {
-; X86-NOBMI-LABEL: bextr32_b1_indexzext:
+; Shifting happens in 64-bit, then truncation. Masking is 32-bit.
+define i32 @bextr64_32_a1(i64 %val, i64 %numskipbits, i32 %numlowbits) nounwind {
+; X86-NOBMI-LABEL: bextr64_32_a1:
 ; X86-NOBMI:       # %bb.0:
+; X86-NOBMI-NEXT:    pushl %edi
 ; X86-NOBMI-NEXT:    pushl %esi
 ; X86-NOBMI-NEXT:    movb {{[0-9]+}}(%esp), %dl
 ; X86-NOBMI-NEXT:    movb {{[0-9]+}}(%esp), %cl
-; X86-NOBMI-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NOBMI-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NOBMI-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NOBMI-NEXT:    movl %edi, %esi
 ; X86-NOBMI-NEXT:    shrl %cl, %esi
-; X86-NOBMI-NEXT:    movl $-1, %eax
+; X86-NOBMI-NEXT:    shrdl %cl, %edi, %eax
+; X86-NOBMI-NEXT:    testb $32, %cl
+; X86-NOBMI-NEXT:    jne .LBB15_2
+; X86-NOBMI-NEXT:  # %bb.1:
+; X86-NOBMI-NEXT:    movl %eax, %esi
+; X86-NOBMI-NEXT:  .LBB15_2:
+; X86-NOBMI-NEXT:    movl $1, %eax
 ; X86-NOBMI-NEXT:    movl %edx, %ecx
 ; X86-NOBMI-NEXT:    shll %cl, %eax
-; X86-NOBMI-NEXT:    notl %eax
+; X86-NOBMI-NEXT:    decl %eax
 ; X86-NOBMI-NEXT:    andl %esi, %eax
 ; X86-NOBMI-NEXT:    popl %esi
+; X86-NOBMI-NEXT:    popl %edi
 ; X86-NOBMI-NEXT:    retl
 ;
-; X86-BMI1NOTBM-LABEL: bextr32_b1_indexzext:
+; X86-BMI1NOTBM-LABEL: bextr64_32_a1:
 ; X86-BMI1NOTBM:       # %bb.0:
+; X86-BMI1NOTBM-NEXT:    pushl %edi
+; X86-BMI1NOTBM-NEXT:    pushl %esi
 ; X86-BMI1NOTBM-NEXT:    movb {{[0-9]+}}(%esp), %al
+; X86-BMI1NOTBM-NEXT:    movb {{[0-9]+}}(%esp), %cl
+; X86-BMI1NOTBM-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-BMI1NOTBM-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-BMI1NOTBM-NEXT:    movl %edi, %edx
+; X86-BMI1NOTBM-NEXT:    shrl %cl, %edx
+; X86-BMI1NOTBM-NEXT:    shrdl %cl, %edi, %esi
+; X86-BMI1NOTBM-NEXT:    testb $32, %cl
+; X86-BMI1NOTBM-NEXT:    jne .LBB15_2
+; X86-BMI1NOTBM-NEXT:  # %bb.1:
+; X86-BMI1NOTBM-NEXT:    movl %esi, %edx
+; X86-BMI1NOTBM-NEXT:  .LBB15_2:
 ; X86-BMI1NOTBM-NEXT:    shll $8, %eax
-; X86-BMI1NOTBM-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
-; X86-BMI1NOTBM-NEXT:    orl %eax, %ecx
-; X86-BMI1NOTBM-NEXT:    bextrl %ecx, {{[0-9]+}}(%esp), %eax
+; X86-BMI1NOTBM-NEXT:    bextrl %eax, %edx, %eax
+; X86-BMI1NOTBM-NEXT:    popl %esi
+; X86-BMI1NOTBM-NEXT:    popl %edi
 ; X86-BMI1NOTBM-NEXT:    retl
 ;
-; X86-BMI1BMI2-LABEL: bextr32_b1_indexzext:
+; X86-BMI1BMI2-LABEL: bextr64_32_a1:
 ; X86-BMI1BMI2:       # %bb.0:
+; X86-BMI1BMI2-NEXT:    pushl %esi
 ; X86-BMI1BMI2-NEXT:    movb {{[0-9]+}}(%esp), %al
 ; X86-BMI1BMI2-NEXT:    movb {{[0-9]+}}(%esp), %cl
-; X86-BMI1BMI2-NEXT:    shrxl %ecx, {{[0-9]+}}(%esp), %ecx
-; X86-BMI1BMI2-NEXT:    bzhil %eax, %ecx, %eax
+; X86-BMI1BMI2-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-BMI1BMI2-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-BMI1BMI2-NEXT:    shrdl %cl, %esi, %edx
+; X86-BMI1BMI2-NEXT:    testb $32, %cl
+; X86-BMI1BMI2-NEXT:    je .LBB15_2
+; X86-BMI1BMI2-NEXT:  # %bb.1:
+; X86-BMI1BMI2-NEXT:    shrxl %ecx, %esi, %edx
+; X86-BMI1BMI2-NEXT:  .LBB15_2:
+; X86-BMI1BMI2-NEXT:    bzhil %eax, %edx, %eax
+; X86-BMI1BMI2-NEXT:    popl %esi
 ; X86-BMI1BMI2-NEXT:    retl
 ;
-; X64-NOBMI-LABEL: bextr32_b1_indexzext:
+; X64-NOBMI-LABEL: bextr64_32_a1:
 ; X64-NOBMI:       # %bb.0:
-; X64-NOBMI-NEXT:    movl %esi, %ecx
-; X64-NOBMI-NEXT:    # kill: def $cl killed $cl killed $ecx
-; X64-NOBMI-NEXT:    shrl %cl, %edi
-; X64-NOBMI-NEXT:    movl $-1, %eax
+; X64-NOBMI-NEXT:    movq %rsi, %rcx
+; X64-NOBMI-NEXT:    # kill: def $cl killed $cl killed $rcx
+; X64-NOBMI-NEXT:    shrq %cl, %rdi
+; X64-NOBMI-NEXT:    movl $1, %eax
 ; X64-NOBMI-NEXT:    movl %edx, %ecx
 ; X64-NOBMI-NEXT:    shll %cl, %eax
-; X64-NOBMI-NEXT:    notl %eax
+; X64-NOBMI-NEXT:    decl %eax
 ; X64-NOBMI-NEXT:    andl %edi, %eax
 ; X64-NOBMI-NEXT:    retq
 ;
-; X64-BMI1NOTBM-LABEL: bextr32_b1_indexzext:
+; X64-BMI1NOTBM-LABEL: bextr64_32_a1:
 ; X64-BMI1NOTBM:       # %bb.0:
 ; X64-BMI1NOTBM-NEXT:    shll $8, %edx
 ; X64-BMI1NOTBM-NEXT:    movzbl %sil, %eax
 ; X64-BMI1NOTBM-NEXT:    orl %edx, %eax
-; X64-BMI1NOTBM-NEXT:    bextrl %eax, %edi, %eax
+; X64-BMI1NOTBM-NEXT:    bextrq %rax, %rdi, %rax
+; X64-BMI1NOTBM-NEXT:    # kill: def $eax killed $eax killed $rax
 ; X64-BMI1NOTBM-NEXT:    retq
 ;
-; X64-BMI1BMI2-LABEL: bextr32_b1_indexzext:
+; X64-BMI1BMI2-LABEL: bextr64_32_a1:
 ; X64-BMI1BMI2:       # %bb.0:
-; X64-BMI1BMI2-NEXT:    shrxl %esi, %edi, %eax
+; X64-BMI1BMI2-NEXT:    shrxq %rsi, %rdi, %rax
 ; X64-BMI1BMI2-NEXT:    bzhil %edx, %eax, %eax
 ; X64-BMI1BMI2-NEXT:    retq
-  %skip = zext i8 %numskipbits to i32
-  %shifted = lshr i32 %val, %skip
-  %conv = zext i8 %numlowbits to i32
-  %notmask = shl i32 -1, %conv
-  %mask = xor i32 %notmask, -1
-  %masked = and i32 %mask, %shifted
+  %shifted = lshr i64 %val, %numskipbits
+  %truncshifted = trunc i64 %shifted to i32
+  %onebit = shl i32 1, %numlowbits
+  %mask = add nsw i32 %onebit, -1
+  %masked = and i32 %mask, %truncshifted
   ret i32 %masked
 }
 
-define i32 @bextr32_b2_load(i32* %w, i32 %numskipbits, i32 %numlowbits) nounwind {
-; X86-NOBMI-LABEL: bextr32_b2_load:
+; Shifting happens in 64-bit, then truncation (with extra use).
+; Masking is 32-bit.
+define i32 @bextr64_32_a1_trunc_extrause(i64 %val, i64 %numskipbits, i32 %numlowbits) nounwind {
+; X86-NOBMI-LABEL: bextr64_32_a1_trunc_extrause:
 ; X86-NOBMI:       # %bb.0:
+; X86-NOBMI-NEXT:    pushl %ebx
 ; X86-NOBMI-NEXT:    pushl %esi
-; X86-NOBMI-NEXT:    movb {{[0-9]+}}(%esp), %dl
+; X86-NOBMI-NEXT:    pushl %eax
+; X86-NOBMI-NEXT:    movb {{[0-9]+}}(%esp), %bl
 ; X86-NOBMI-NEXT:    movb {{[0-9]+}}(%esp), %cl
 ; X86-NOBMI-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NOBMI-NEXT:    movl (%eax), %esi
+; X86-NOBMI-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NOBMI-NEXT:    movl %edx, %esi
 ; X86-NOBMI-NEXT:    shrl %cl, %esi
-; X86-NOBMI-NEXT:    movl $-1, %eax
-; X86-NOBMI-NEXT:    movl %edx, %ecx
+; X86-NOBMI-NEXT:    shrdl %cl, %edx, %eax
+; X86-NOBMI-NEXT:    testb $32, %cl
+; X86-NOBMI-NEXT:    jne .LBB16_2
+; X86-NOBMI-NEXT:  # %bb.1:
+; X86-NOBMI-NEXT:    movl %eax, %esi
+; X86-NOBMI-NEXT:  .LBB16_2:
+; X86-NOBMI-NEXT:    movl %esi, (%esp)
+; X86-NOBMI-NEXT:    calll use32
+; X86-NOBMI-NEXT:    movl $1, %eax
+; X86-NOBMI-NEXT:    movl %ebx, %ecx
 ; X86-NOBMI-NEXT:    shll %cl, %eax
-; X86-NOBMI-NEXT:    notl %eax
+; X86-NOBMI-NEXT:    decl %eax
 ; X86-NOBMI-NEXT:    andl %esi, %eax
+; X86-NOBMI-NEXT:    addl $4, %esp
 ; X86-NOBMI-NEXT:    popl %esi
+; X86-NOBMI-NEXT:    popl %ebx
 ; X86-NOBMI-NEXT:    retl
 ;
-; X86-BMI1NOTBM-LABEL: bextr32_b2_load:
+; X86-BMI1NOTBM-LABEL: bextr64_32_a1_trunc_extrause:
 ; X86-BMI1NOTBM:       # %bb.0:
-; X86-BMI1NOTBM-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-BMI1NOTBM-NEXT:    pushl %ebx
+; X86-BMI1NOTBM-NEXT:    pushl %esi
+; X86-BMI1NOTBM-NEXT:    pushl %eax
+; X86-BMI1NOTBM-NEXT:    movb {{[0-9]+}}(%esp), %bl
 ; X86-BMI1NOTBM-NEXT:    movb {{[0-9]+}}(%esp), %cl
-; X86-BMI1NOTBM-NEXT:    shll $8, %ecx
-; X86-BMI1NOTBM-NEXT:    movzbl {{[0-9]+}}(%esp), %edx
-; X86-BMI1NOTBM-NEXT:    orl %ecx, %edx
-; X86-BMI1NOTBM-NEXT:    bextrl %edx, (%eax), %eax
+; X86-BMI1NOTBM-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-BMI1NOTBM-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-BMI1NOTBM-NEXT:    movl %edx, %esi
+; X86-BMI1NOTBM-NEXT:    shrl %cl, %esi
+; X86-BMI1NOTBM-NEXT:    shrdl %cl, %edx, %eax
+; X86-BMI1NOTBM-NEXT:    testb $32, %cl
+; X86-BMI1NOTBM-NEXT:    jne .LBB16_2
+; X86-BMI1NOTBM-NEXT:  # %bb.1:
+; X86-BMI1NOTBM-NEXT:    movl %eax, %esi
+; X86-BMI1NOTBM-NEXT:  .LBB16_2:
+; X86-BMI1NOTBM-NEXT:    movl %esi, (%esp)
+; X86-BMI1NOTBM-NEXT:    calll use32
+; X86-BMI1NOTBM-NEXT:    shll $8, %ebx
+; X86-BMI1NOTBM-NEXT:    bextrl %ebx, %esi, %eax
+; X86-BMI1NOTBM-NEXT:    addl $4, %esp
+; X86-BMI1NOTBM-NEXT:    popl %esi
+; X86-BMI1NOTBM-NEXT:    popl %ebx
 ; X86-BMI1NOTBM-NEXT:    retl
 ;
-; X86-BMI1BMI2-LABEL: bextr32_b2_load:
+; X86-BMI1BMI2-LABEL: bextr64_32_a1_trunc_extrause:
 ; X86-BMI1BMI2:       # %bb.0:
-; X86-BMI1BMI2-NEXT:    movb {{[0-9]+}}(%esp), %al
-; X86-BMI1BMI2-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-BMI1BMI2-NEXT:    movb {{[0-9]+}}(%esp), %dl
-; X86-BMI1BMI2-NEXT:    shrxl %edx, (%ecx), %ecx
-; X86-BMI1BMI2-NEXT:    bzhil %eax, %ecx, %eax
+; X86-BMI1BMI2-NEXT:    pushl %ebx
+; X86-BMI1BMI2-NEXT:    pushl %esi
+; X86-BMI1BMI2-NEXT:    pushl %eax
+; X86-BMI1BMI2-NEXT:    movb {{[0-9]+}}(%esp), %bl
+; X86-BMI1BMI2-NEXT:    movb {{[0-9]+}}(%esp), %cl
+; X86-BMI1BMI2-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-BMI1BMI2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-BMI1BMI2-NEXT:    shrdl %cl, %eax, %esi
+; X86-BMI1BMI2-NEXT:    testb $32, %cl
+; X86-BMI1BMI2-NEXT:    je .LBB16_2
+; X86-BMI1BMI2-NEXT:  # %bb.1:
+; X86-BMI1BMI2-NEXT:    shrxl %ecx, %eax, %esi
+; X86-BMI1BMI2-NEXT:  .LBB16_2:
+; X86-BMI1BMI2-NEXT:    movl %esi, (%esp)
+; X86-BMI1BMI2-NEXT:    calll use32
+; X86-BMI1BMI2-NEXT:    bzhil %ebx, %esi, %eax
+; X86-BMI1BMI2-NEXT:    addl $4, %esp
+; X86-BMI1BMI2-NEXT:    popl %esi
+; X86-BMI1BMI2-NEXT:    popl %ebx
 ; X86-BMI1BMI2-NEXT:    retl
 ;
-; X64-NOBMI-LABEL: bextr32_b2_load:
+; X64-NOBMI-LABEL: bextr64_32_a1_trunc_extrause:
 ; X64-NOBMI:       # %bb.0:
-; X64-NOBMI-NEXT:    movl %esi, %ecx
-; X64-NOBMI-NEXT:    movl (%rdi), %esi
-; X64-NOBMI-NEXT:    # kill: def $cl killed $cl killed $ecx
-; X64-NOBMI-NEXT:    shrl %cl, %esi
-; X64-NOBMI-NEXT:    movl $-1, %eax
-; X64-NOBMI-NEXT:    movl %edx, %ecx
-; X64-NOBMI-NEXT:    shll %cl, %eax
-; X64-NOBMI-NEXT:    notl %eax
-; X64-NOBMI-NEXT:    andl %esi, %eax
-; X64-NOBMI-NEXT:    retq
-;
-; X64-BMI1NOTBM-LABEL: bextr32_b2_load:
+; X64-NOBMI-NEXT:    pushq %rbp
+; X64-NOBMI-NEXT:    pushq %rbx
+; X64-NOBMI-NEXT:    pushq %rax
+; X64-NOBMI-NEXT:    movl %edx, %ebp
+; X64-NOBMI-NEXT:    movq %rsi, %rcx
+; X64-NOBMI-NEXT:    movq %rdi, %rbx
+; X64-NOBMI-NEXT:    # kill: def $cl killed $cl killed $rcx
+; X64-NOBMI-NEXT:    shrq %cl, %rbx
+; X64-NOBMI-NEXT:    movl %ebx, %edi
+; X64-NOBMI-NEXT:    callq use32
+; X64-NOBMI-NEXT:    movl $1, %eax
+; X64-NOBMI-NEXT:    movl %ebp, %ecx
+; X64-NOBMI-NEXT:    shll %cl, %eax
+; X64-NOBMI-NEXT:    decl %eax
+; X64-NOBMI-NEXT:    andl %ebx, %eax
+; X64-NOBMI-NEXT:    addq $8, %rsp
+; X64-NOBMI-NEXT:    popq %rbx
+; X64-NOBMI-NEXT:    popq %rbp
+; X64-NOBMI-NEXT:    retq
+;
+; X64-BMI1NOTBM-LABEL: bextr64_32_a1_trunc_extrause:
 ; X64-BMI1NOTBM:       # %bb.0:
-; X64-BMI1NOTBM-NEXT:    shll $8, %edx
-; X64-BMI1NOTBM-NEXT:    movzbl %sil, %eax
-; X64-BMI1NOTBM-NEXT:    orl %edx, %eax
-; X64-BMI1NOTBM-NEXT:    bextrl %eax, (%rdi), %eax
+; X64-BMI1NOTBM-NEXT:    pushq %rbp
+; X64-BMI1NOTBM-NEXT:    pushq %rbx
+; X64-BMI1NOTBM-NEXT:    pushq %rax
+; X64-BMI1NOTBM-NEXT:    movl %edx, %ebp
+; X64-BMI1NOTBM-NEXT:    movq %rsi, %rcx
+; X64-BMI1NOTBM-NEXT:    movq %rdi, %rbx
+; X64-BMI1NOTBM-NEXT:    # kill: def $cl killed $cl killed $rcx
+; X64-BMI1NOTBM-NEXT:    shrq %cl, %rbx
+; X64-BMI1NOTBM-NEXT:    movl %ebx, %edi
+; X64-BMI1NOTBM-NEXT:    callq use32
+; X64-BMI1NOTBM-NEXT:    shll $8, %ebp
+; X64-BMI1NOTBM-NEXT:    bextrl %ebp, %ebx, %eax
+; X64-BMI1NOTBM-NEXT:    addq $8, %rsp
+; X64-BMI1NOTBM-NEXT:    popq %rbx
+; X64-BMI1NOTBM-NEXT:    popq %rbp
 ; X64-BMI1NOTBM-NEXT:    retq
 ;
-; X64-BMI1BMI2-LABEL: bextr32_b2_load:
+; X64-BMI1BMI2-LABEL: bextr64_32_a1_trunc_extrause:
 ; X64-BMI1BMI2:       # %bb.0:
-; X64-BMI1BMI2-NEXT:    shrxl %esi, (%rdi), %eax
-; X64-BMI1BMI2-NEXT:    bzhil %edx, %eax, %eax
+; X64-BMI1BMI2-NEXT:    pushq %rbp
+; X64-BMI1BMI2-NEXT:    pushq %rbx
+; X64-BMI1BMI2-NEXT:    pushq %rax
+; X64-BMI1BMI2-NEXT:    movl %edx, %ebp
+; X64-BMI1BMI2-NEXT:    shrxq %rsi, %rdi, %rbx
+; X64-BMI1BMI2-NEXT:    movl %ebx, %edi
+; X64-BMI1BMI2-NEXT:    callq use32
+; X64-BMI1BMI2-NEXT:    bzhil %ebp, %ebx, %eax
+; X64-BMI1BMI2-NEXT:    addq $8, %rsp
+; X64-BMI1BMI2-NEXT:    popq %rbx
+; X64-BMI1BMI2-NEXT:    popq %rbp
 ; X64-BMI1BMI2-NEXT:    retq
-  %val = load i32, i32* %w
-  %shifted = lshr i32 %val, %numskipbits
-  %notmask = shl i32 -1, %numlowbits
-  %mask = xor i32 %notmask, -1
-  %masked = and i32 %mask, %shifted
+  %shifted = lshr i64 %val, %numskipbits
+  %truncshifted = trunc i64 %shifted to i32
+  call void @use32(i32 %truncshifted)
+  %onebit = shl i32 1, %numlowbits
+  %mask = add nsw i32 %onebit, -1
+  %masked = and i32 %mask, %truncshifted
   ret i32 %masked
 }
 
-define i32 @bextr32_b3_load_indexzext(i32* %w, i8 zeroext %numskipbits, i8 zeroext %numlowbits) nounwind {
-; X86-NOBMI-LABEL: bextr32_b3_load_indexzext:
+; Shifting happens in 64-bit. Mask is 32-bit, but extended to 64-bit.
+; Masking is 64-bit. Then truncation.
+define i32 @bextr64_32_a2(i64 %val, i64 %numskipbits, i32 %numlowbits) nounwind {
+; X86-NOBMI-LABEL: bextr64_32_a2:
 ; X86-NOBMI:       # %bb.0:
+; X86-NOBMI-NEXT:    pushl %edi
 ; X86-NOBMI-NEXT:    pushl %esi
 ; X86-NOBMI-NEXT:    movb {{[0-9]+}}(%esp), %dl
 ; X86-NOBMI-NEXT:    movb {{[0-9]+}}(%esp), %cl
 ; X86-NOBMI-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NOBMI-NEXT:    movl (%eax), %esi
+; X86-NOBMI-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NOBMI-NEXT:    movl %edi, %esi
 ; X86-NOBMI-NEXT:    shrl %cl, %esi
-; X86-NOBMI-NEXT:    movl $-1, %eax
+; X86-NOBMI-NEXT:    shrdl %cl, %edi, %eax
+; X86-NOBMI-NEXT:    testb $32, %cl
+; X86-NOBMI-NEXT:    jne .LBB17_2
+; X86-NOBMI-NEXT:  # %bb.1:
+; X86-NOBMI-NEXT:    movl %eax, %esi
+; X86-NOBMI-NEXT:  .LBB17_2:
+; X86-NOBMI-NEXT:    movl $1, %eax
 ; X86-NOBMI-NEXT:    movl %edx, %ecx
 ; X86-NOBMI-NEXT:    shll %cl, %eax
-; X86-NOBMI-NEXT:    notl %eax
+; X86-NOBMI-NEXT:    decl %eax
 ; X86-NOBMI-NEXT:    andl %esi, %eax
 ; X86-NOBMI-NEXT:    popl %esi
+; X86-NOBMI-NEXT:    popl %edi
 ; X86-NOBMI-NEXT:    retl
 ;
-; X86-BMI1NOTBM-LABEL: bextr32_b3_load_indexzext:
+; X86-BMI1NOTBM-LABEL: bextr64_32_a2:
 ; X86-BMI1NOTBM:       # %bb.0:
-; X86-BMI1NOTBM-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-BMI1NOTBM-NEXT:    pushl %edi
+; X86-BMI1NOTBM-NEXT:    pushl %esi
+; X86-BMI1NOTBM-NEXT:    movb {{[0-9]+}}(%esp), %al
 ; X86-BMI1NOTBM-NEXT:    movb {{[0-9]+}}(%esp), %cl
-; X86-BMI1NOTBM-NEXT:    shll $8, %ecx
-; X86-BMI1NOTBM-NEXT:    movzbl {{[0-9]+}}(%esp), %edx
-; X86-BMI1NOTBM-NEXT:    orl %ecx, %edx
-; X86-BMI1NOTBM-NEXT:    bextrl %edx, (%eax), %eax
+; X86-BMI1NOTBM-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-BMI1NOTBM-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-BMI1NOTBM-NEXT:    movl %edi, %edx
+; X86-BMI1NOTBM-NEXT:    shrl %cl, %edx
+; X86-BMI1NOTBM-NEXT:    shrdl %cl, %edi, %esi
+; X86-BMI1NOTBM-NEXT:    testb $32, %cl
+; X86-BMI1NOTBM-NEXT:    jne .LBB17_2
+; X86-BMI1NOTBM-NEXT:  # %bb.1:
+; X86-BMI1NOTBM-NEXT:    movl %esi, %edx
+; X86-BMI1NOTBM-NEXT:  .LBB17_2:
+; X86-BMI1NOTBM-NEXT:    shll $8, %eax
+; X86-BMI1NOTBM-NEXT:    bextrl %eax, %edx, %eax
+; X86-BMI1NOTBM-NEXT:    popl %esi
+; X86-BMI1NOTBM-NEXT:    popl %edi
 ; X86-BMI1NOTBM-NEXT:    retl
 ;
-; X86-BMI1BMI2-LABEL: bextr32_b3_load_indexzext:
+; X86-BMI1BMI2-LABEL: bextr64_32_a2:
 ; X86-BMI1BMI2:       # %bb.0:
+; X86-BMI1BMI2-NEXT:    pushl %esi
 ; X86-BMI1BMI2-NEXT:    movb {{[0-9]+}}(%esp), %al
-; X86-BMI1BMI2-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-BMI1BMI2-NEXT:    movb {{[0-9]+}}(%esp), %dl
-; X86-BMI1BMI2-NEXT:    shrxl %edx, (%ecx), %ecx
-; X86-BMI1BMI2-NEXT:    bzhil %eax, %ecx, %eax
+; X86-BMI1BMI2-NEXT:    movb {{[0-9]+}}(%esp), %cl
+; X86-BMI1BMI2-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-BMI1BMI2-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-BMI1BMI2-NEXT:    shrdl %cl, %esi, %edx
+; X86-BMI1BMI2-NEXT:    testb $32, %cl
+; X86-BMI1BMI2-NEXT:    je .LBB17_2
+; X86-BMI1BMI2-NEXT:  # %bb.1:
+; X86-BMI1BMI2-NEXT:    shrxl %ecx, %esi, %edx
+; X86-BMI1BMI2-NEXT:  .LBB17_2:
+; X86-BMI1BMI2-NEXT:    bzhil %eax, %edx, %eax
+; X86-BMI1BMI2-NEXT:    popl %esi
 ; X86-BMI1BMI2-NEXT:    retl
 ;
-; X64-NOBMI-LABEL: bextr32_b3_load_indexzext:
+; X64-NOBMI-LABEL: bextr64_32_a2:
 ; X64-NOBMI:       # %bb.0:
-; X64-NOBMI-NEXT:    movl %esi, %ecx
-; X64-NOBMI-NEXT:    movl (%rdi), %esi
-; X64-NOBMI-NEXT:    # kill: def $cl killed $cl killed $ecx
-; X64-NOBMI-NEXT:    shrl %cl, %esi
-; X64-NOBMI-NEXT:    movl $-1, %eax
+; X64-NOBMI-NEXT:    movq %rsi, %rcx
+; X64-NOBMI-NEXT:    # kill: def $cl killed $cl killed $rcx
+; X64-NOBMI-NEXT:    shrq %cl, %rdi
+; X64-NOBMI-NEXT:    movl $1, %eax
 ; X64-NOBMI-NEXT:    movl %edx, %ecx
 ; X64-NOBMI-NEXT:    shll %cl, %eax
-; X64-NOBMI-NEXT:    notl %eax
-; X64-NOBMI-NEXT:    andl %esi, %eax
+; X64-NOBMI-NEXT:    decl %eax
+; X64-NOBMI-NEXT:    andl %edi, %eax
 ; X64-NOBMI-NEXT:    retq
 ;
-; X64-BMI1NOTBM-LABEL: bextr32_b3_load_indexzext:
+; X64-BMI1NOTBM-LABEL: bextr64_32_a2:
 ; X64-BMI1NOTBM:       # %bb.0:
 ; X64-BMI1NOTBM-NEXT:    shll $8, %edx
 ; X64-BMI1NOTBM-NEXT:    movzbl %sil, %eax
 ; X64-BMI1NOTBM-NEXT:    orl %edx, %eax
-; X64-BMI1NOTBM-NEXT:    bextrl %eax, (%rdi), %eax
+; X64-BMI1NOTBM-NEXT:    bextrq %rax, %rdi, %rax
+; X64-BMI1NOTBM-NEXT:    # kill: def $eax killed $eax killed $rax
 ; X64-BMI1NOTBM-NEXT:    retq
 ;
-; X64-BMI1BMI2-LABEL: bextr32_b3_load_indexzext:
+; X64-BMI1BMI2-LABEL: bextr64_32_a2:
 ; X64-BMI1BMI2:       # %bb.0:
-; X64-BMI1BMI2-NEXT:    shrxl %esi, (%rdi), %eax
+; X64-BMI1BMI2-NEXT:    shrxq %rsi, %rdi, %rax
 ; X64-BMI1BMI2-NEXT:    bzhil %edx, %eax, %eax
 ; X64-BMI1BMI2-NEXT:    retq
-  %val = load i32, i32* %w
-  %skip = zext i8 %numskipbits to i32
-  %shifted = lshr i32 %val, %skip
-  %conv = zext i8 %numlowbits to i32
-  %notmask = shl i32 -1, %conv
-  %mask = xor i32 %notmask, -1
-  %masked = and i32 %mask, %shifted
-  ret i32 %masked
+  %shifted = lshr i64 %val, %numskipbits
+  %onebit = shl i32 1, %numlowbits
+  %mask = add nsw i32 %onebit, -1
+  %zextmask = zext i32 %mask to i64
+  %masked = and i64 %zextmask, %shifted
+  %truncmasked = trunc i64 %masked to i32
+  ret i32 %truncmasked
 }
 
-define i32 @bextr32_b4_commutative(i32 %val, i32 %numskipbits, i32 %numlowbits) nounwind {
-; X86-NOBMI-LABEL: bextr32_b4_commutative:
+; ---------------------------------------------------------------------------- ;
+; Pattern b. 32-bit
+; ---------------------------------------------------------------------------- ;
+
+define i32 @bextr32_b0(i32 %val, i32 %numskipbits, i32 %numlowbits) nounwind {
+; X86-NOBMI-LABEL: bextr32_b0:
 ; X86-NOBMI:       # %bb.0:
 ; X86-NOBMI-NEXT:    pushl %esi
 ; X86-NOBMI-NEXT:    movb {{[0-9]+}}(%esp), %dl
@@ -1908,7 +2113,7 @@ define i32 @bextr32_b4_commutative(i32 %val, i32 %numskipbits, i32 %numlowbits)
 ; X86-NOBMI-NEXT:    popl %esi
 ; X86-NOBMI-NEXT:    retl
 ;
-; X86-BMI1NOTBM-LABEL: bextr32_b4_commutative:
+; X86-BMI1NOTBM-LABEL: bextr32_b0:
 ; X86-BMI1NOTBM:       # %bb.0:
 ; X86-BMI1NOTBM-NEXT:    movb {{[0-9]+}}(%esp), %al
 ; X86-BMI1NOTBM-NEXT:    shll $8, %eax
@@ -1917,7 +2122,7 @@ define i32 @bextr32_b4_commutative(i32 %val, i32 %numskipbits, i32 %numlowbits)
 ; X86-BMI1NOTBM-NEXT:    bextrl %ecx, {{[0-9]+}}(%esp), %eax
 ; X86-BMI1NOTBM-NEXT:    retl
 ;
-; X86-BMI1BMI2-LABEL: bextr32_b4_commutative:
+; X86-BMI1BMI2-LABEL: bextr32_b0:
 ; X86-BMI1BMI2:       # %bb.0:
 ; X86-BMI1BMI2-NEXT:    movb {{[0-9]+}}(%esp), %al
 ; X86-BMI1BMI2-NEXT:    movb {{[0-9]+}}(%esp), %cl
@@ -1925,7 +2130,7 @@ define i32 @bextr32_b4_commutative(i32 %val, i32 %numskipbits, i32 %numlowbits)
 ; X86-BMI1BMI2-NEXT:    bzhil %eax, %ecx, %eax
 ; X86-BMI1BMI2-NEXT:    retl
 ;
-; X64-NOBMI-LABEL: bextr32_b4_commutative:
+; X64-NOBMI-LABEL: bextr32_b0:
 ; X64-NOBMI:       # %bb.0:
 ; X64-NOBMI-NEXT:    movl %esi, %ecx
 ; X64-NOBMI-NEXT:    # kill: def $cl killed $cl killed $ecx
@@ -1937,7 +2142,7 @@ define i32 @bextr32_b4_commutative(i32 %val, i32 %numskipbits, i32 %numlowbits)
 ; X64-NOBMI-NEXT:    andl %edi, %eax
 ; X64-NOBMI-NEXT:    retq
 ;
-; X64-BMI1NOTBM-LABEL: bextr32_b4_commutative:
+; X64-BMI1NOTBM-LABEL: bextr32_b0:
 ; X64-BMI1NOTBM:       # %bb.0:
 ; X64-BMI1NOTBM-NEXT:    shll $8, %edx
 ; X64-BMI1NOTBM-NEXT:    movzbl %sil, %eax
@@ -1945,7 +2150,7 @@ define i32 @bextr32_b4_commutative(i32 %val, i32 %numskipbits, i32 %numlowbits)
 ; X64-BMI1NOTBM-NEXT:    bextrl %eax, %edi, %eax
 ; X64-BMI1NOTBM-NEXT:    retq
 ;
-; X64-BMI1BMI2-LABEL: bextr32_b4_commutative:
+; X64-BMI1BMI2-LABEL: bextr32_b0:
 ; X64-BMI1BMI2:       # %bb.0:
 ; X64-BMI1BMI2-NEXT:    shrxl %esi, %edi, %eax
 ; X64-BMI1BMI2-NEXT:    bzhil %edx, %eax, %eax
@@ -1953,428 +2158,418 @@ define i32 @bextr32_b4_commutative(i32 %val, i32 %numskipbits, i32 %numlowbits)
   %shifted = lshr i32 %val, %numskipbits
   %notmask = shl i32 -1, %numlowbits
   %mask = xor i32 %notmask, -1
-  %masked = and i32 %shifted, %mask ; swapped order
+  %masked = and i32 %mask, %shifted
   ret i32 %masked
 }
 
-define i32 @bextr32_b5_skipextrauses(i32 %val, i32 %numskipbits, i32 %numlowbits) nounwind {
-; X86-NOBMI-LABEL: bextr32_b5_skipextrauses:
+define i32 @bextr32_b1_indexzext(i32 %val, i8 zeroext %numskipbits, i8 zeroext %numlowbits) nounwind {
+; X86-NOBMI-LABEL: bextr32_b1_indexzext:
 ; X86-NOBMI:       # %bb.0:
-; X86-NOBMI-NEXT:    pushl %edi
 ; X86-NOBMI-NEXT:    pushl %esi
-; X86-NOBMI-NEXT:    pushl %eax
 ; X86-NOBMI-NEXT:    movb {{[0-9]+}}(%esp), %dl
-; X86-NOBMI-NEXT:    movl {{[0-9]+}}(%esp), %edi
-; X86-NOBMI-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NOBMI-NEXT:    movl %eax, %ecx
-; X86-NOBMI-NEXT:    shrl %cl, %edi
-; X86-NOBMI-NEXT:    movl $-1, %esi
+; X86-NOBMI-NEXT:    movb {{[0-9]+}}(%esp), %cl
+; X86-NOBMI-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NOBMI-NEXT:    shrl %cl, %esi
+; X86-NOBMI-NEXT:    movl $-1, %eax
 ; X86-NOBMI-NEXT:    movl %edx, %ecx
-; X86-NOBMI-NEXT:    shll %cl, %esi
-; X86-NOBMI-NEXT:    notl %esi
-; X86-NOBMI-NEXT:    andl %edi, %esi
-; X86-NOBMI-NEXT:    movl %eax, (%esp)
-; X86-NOBMI-NEXT:    calll use32
-; X86-NOBMI-NEXT:    movl %esi, %eax
-; X86-NOBMI-NEXT:    addl $4, %esp
+; X86-NOBMI-NEXT:    shll %cl, %eax
+; X86-NOBMI-NEXT:    notl %eax
+; X86-NOBMI-NEXT:    andl %esi, %eax
 ; X86-NOBMI-NEXT:    popl %esi
-; X86-NOBMI-NEXT:    popl %edi
 ; X86-NOBMI-NEXT:    retl
 ;
-; X86-BMI1NOTBM-LABEL: bextr32_b5_skipextrauses:
+; X86-BMI1NOTBM-LABEL: bextr32_b1_indexzext:
 ; X86-BMI1NOTBM:       # %bb.0:
-; X86-BMI1NOTBM-NEXT:    pushl %esi
-; X86-BMI1NOTBM-NEXT:    subl $8, %esp
-; X86-BMI1NOTBM-NEXT:    movb {{[0-9]+}}(%esp), %cl
-; X86-BMI1NOTBM-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-BMI1NOTBM-NEXT:    shll $8, %ecx
-; X86-BMI1NOTBM-NEXT:    movzbl %al, %edx
-; X86-BMI1NOTBM-NEXT:    orl %ecx, %edx
-; X86-BMI1NOTBM-NEXT:    bextrl %edx, {{[0-9]+}}(%esp), %esi
-; X86-BMI1NOTBM-NEXT:    movl %eax, (%esp)
-; X86-BMI1NOTBM-NEXT:    calll use32
-; X86-BMI1NOTBM-NEXT:    movl %esi, %eax
-; X86-BMI1NOTBM-NEXT:    addl $8, %esp
-; X86-BMI1NOTBM-NEXT:    popl %esi
+; X86-BMI1NOTBM-NEXT:    movb {{[0-9]+}}(%esp), %al
+; X86-BMI1NOTBM-NEXT:    shll $8, %eax
+; X86-BMI1NOTBM-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
+; X86-BMI1NOTBM-NEXT:    orl %eax, %ecx
+; X86-BMI1NOTBM-NEXT:    bextrl %ecx, {{[0-9]+}}(%esp), %eax
 ; X86-BMI1NOTBM-NEXT:    retl
 ;
-; X86-BMI1BMI2-LABEL: bextr32_b5_skipextrauses:
+; X86-BMI1BMI2-LABEL: bextr32_b1_indexzext:
 ; X86-BMI1BMI2:       # %bb.0:
-; X86-BMI1BMI2-NEXT:    pushl %esi
-; X86-BMI1BMI2-NEXT:    subl $8, %esp
 ; X86-BMI1BMI2-NEXT:    movb {{[0-9]+}}(%esp), %al
-; X86-BMI1BMI2-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-BMI1BMI2-NEXT:    shrxl %ecx, {{[0-9]+}}(%esp), %edx
-; X86-BMI1BMI2-NEXT:    bzhil %eax, %edx, %esi
-; X86-BMI1BMI2-NEXT:    movl %ecx, (%esp)
-; X86-BMI1BMI2-NEXT:    calll use32
-; X86-BMI1BMI2-NEXT:    movl %esi, %eax
-; X86-BMI1BMI2-NEXT:    addl $8, %esp
-; X86-BMI1BMI2-NEXT:    popl %esi
+; X86-BMI1BMI2-NEXT:    movb {{[0-9]+}}(%esp), %cl
+; X86-BMI1BMI2-NEXT:    shrxl %ecx, {{[0-9]+}}(%esp), %ecx
+; X86-BMI1BMI2-NEXT:    bzhil %eax, %ecx, %eax
 ; X86-BMI1BMI2-NEXT:    retl
 ;
-; X64-NOBMI-LABEL: bextr32_b5_skipextrauses:
+; X64-NOBMI-LABEL: bextr32_b1_indexzext:
 ; X64-NOBMI:       # %bb.0:
-; X64-NOBMI-NEXT:    pushq %rbx
 ; X64-NOBMI-NEXT:    movl %esi, %ecx
+; X64-NOBMI-NEXT:    # kill: def $cl killed $cl killed $ecx
 ; X64-NOBMI-NEXT:    shrl %cl, %edi
-; X64-NOBMI-NEXT:    movl $-1, %ebx
+; X64-NOBMI-NEXT:    movl $-1, %eax
 ; X64-NOBMI-NEXT:    movl %edx, %ecx
-; X64-NOBMI-NEXT:    shll %cl, %ebx
-; X64-NOBMI-NEXT:    notl %ebx
-; X64-NOBMI-NEXT:    andl %edi, %ebx
-; X64-NOBMI-NEXT:    movl %esi, %edi
-; X64-NOBMI-NEXT:    callq use32
-; X64-NOBMI-NEXT:    movl %ebx, %eax
-; X64-NOBMI-NEXT:    popq %rbx
+; X64-NOBMI-NEXT:    shll %cl, %eax
+; X64-NOBMI-NEXT:    notl %eax
+; X64-NOBMI-NEXT:    andl %edi, %eax
 ; X64-NOBMI-NEXT:    retq
 ;
-; X64-BMI1NOTBM-LABEL: bextr32_b5_skipextrauses:
+; X64-BMI1NOTBM-LABEL: bextr32_b1_indexzext:
 ; X64-BMI1NOTBM:       # %bb.0:
-; X64-BMI1NOTBM-NEXT:    pushq %rbx
 ; X64-BMI1NOTBM-NEXT:    shll $8, %edx
 ; X64-BMI1NOTBM-NEXT:    movzbl %sil, %eax
 ; X64-BMI1NOTBM-NEXT:    orl %edx, %eax
-; X64-BMI1NOTBM-NEXT:    bextrl %eax, %edi, %ebx
-; X64-BMI1NOTBM-NEXT:    movl %esi, %edi
-; X64-BMI1NOTBM-NEXT:    callq use32
-; X64-BMI1NOTBM-NEXT:    movl %ebx, %eax
-; X64-BMI1NOTBM-NEXT:    popq %rbx
+; X64-BMI1NOTBM-NEXT:    bextrl %eax, %edi, %eax
 ; X64-BMI1NOTBM-NEXT:    retq
 ;
-; X64-BMI1BMI2-LABEL: bextr32_b5_skipextrauses:
+; X64-BMI1BMI2-LABEL: bextr32_b1_indexzext:
 ; X64-BMI1BMI2:       # %bb.0:
-; X64-BMI1BMI2-NEXT:    pushq %rbx
 ; X64-BMI1BMI2-NEXT:    shrxl %esi, %edi, %eax
-; X64-BMI1BMI2-NEXT:    bzhil %edx, %eax, %ebx
-; X64-BMI1BMI2-NEXT:    movl %esi, %edi
-; X64-BMI1BMI2-NEXT:    callq use32
-; X64-BMI1BMI2-NEXT:    movl %ebx, %eax
-; X64-BMI1BMI2-NEXT:    popq %rbx
+; X64-BMI1BMI2-NEXT:    bzhil %edx, %eax, %eax
 ; X64-BMI1BMI2-NEXT:    retq
-  %shifted = lshr i32 %val, %numskipbits
-  %notmask = shl i32 -1, %numlowbits
+  %skip = zext i8 %numskipbits to i32
+  %shifted = lshr i32 %val, %skip
+  %conv = zext i8 %numlowbits to i32
+  %notmask = shl i32 -1, %conv
   %mask = xor i32 %notmask, -1
   %masked = and i32 %mask, %shifted
-  call void @use32(i32 %numskipbits)
   ret i32 %masked
 }
 
-; 64-bit
-
-define i64 @bextr64_b0(i64 %val, i64 %numskipbits, i64 %numlowbits) nounwind {
-; X86-NOBMI-LABEL: bextr64_b0:
+define i32 @bextr32_b2_load(i32* %w, i32 %numskipbits, i32 %numlowbits) nounwind {
+; X86-NOBMI-LABEL: bextr32_b2_load:
 ; X86-NOBMI:       # %bb.0:
-; X86-NOBMI-NEXT:    pushl %edi
 ; X86-NOBMI-NEXT:    pushl %esi
-; X86-NOBMI-NEXT:    movb {{[0-9]+}}(%esp), %ch
+; X86-NOBMI-NEXT:    movb {{[0-9]+}}(%esp), %dl
 ; X86-NOBMI-NEXT:    movb {{[0-9]+}}(%esp), %cl
-; X86-NOBMI-NEXT:    movl {{[0-9]+}}(%esp), %esi
 ; X86-NOBMI-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NOBMI-NEXT:    movl %eax, %edi
-; X86-NOBMI-NEXT:    shrl %cl, %edi
-; X86-NOBMI-NEXT:    shrdl %cl, %eax, %esi
-; X86-NOBMI-NEXT:    testb $32, %cl
-; X86-NOBMI-NEXT:    je .LBB20_2
-; X86-NOBMI-NEXT:  # %bb.1:
-; X86-NOBMI-NEXT:    movl %edi, %esi
-; X86-NOBMI-NEXT:    xorl %edi, %edi
-; X86-NOBMI-NEXT:  .LBB20_2:
-; X86-NOBMI-NEXT:    movl $-1, %edx
+; X86-NOBMI-NEXT:    movl (%eax), %esi
+; X86-NOBMI-NEXT:    shrl %cl, %esi
 ; X86-NOBMI-NEXT:    movl $-1, %eax
-; X86-NOBMI-NEXT:    movb %ch, %cl
+; X86-NOBMI-NEXT:    movl %edx, %ecx
 ; X86-NOBMI-NEXT:    shll %cl, %eax
-; X86-NOBMI-NEXT:    shldl %cl, %edx, %edx
-; X86-NOBMI-NEXT:    testb $32, %ch
-; X86-NOBMI-NEXT:    je .LBB20_4
-; X86-NOBMI-NEXT:  # %bb.3:
-; X86-NOBMI-NEXT:    movl %eax, %edx
-; X86-NOBMI-NEXT:    xorl %eax, %eax
-; X86-NOBMI-NEXT:  .LBB20_4:
-; X86-NOBMI-NEXT:    notl %edx
-; X86-NOBMI-NEXT:    andl %edi, %edx
 ; X86-NOBMI-NEXT:    notl %eax
 ; X86-NOBMI-NEXT:    andl %esi, %eax
 ; X86-NOBMI-NEXT:    popl %esi
-; X86-NOBMI-NEXT:    popl %edi
 ; X86-NOBMI-NEXT:    retl
 ;
-; X86-BMI1NOTBM-LABEL: bextr64_b0:
+; X86-BMI1NOTBM-LABEL: bextr32_b2_load:
 ; X86-BMI1NOTBM:       # %bb.0:
-; X86-BMI1NOTBM-NEXT:    pushl %ebx
-; X86-BMI1NOTBM-NEXT:    pushl %edi
-; X86-BMI1NOTBM-NEXT:    pushl %esi
-; X86-BMI1NOTBM-NEXT:    movb {{[0-9]+}}(%esp), %al
+; X86-BMI1NOTBM-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-BMI1NOTBM-NEXT:    movb {{[0-9]+}}(%esp), %cl
-; X86-BMI1NOTBM-NEXT:    movl {{[0-9]+}}(%esp), %esi
-; X86-BMI1NOTBM-NEXT:    movl {{[0-9]+}}(%esp), %edi
-; X86-BMI1NOTBM-NEXT:    movl %edi, %edx
-; X86-BMI1NOTBM-NEXT:    shrl %cl, %edx
-; X86-BMI1NOTBM-NEXT:    shrdl %cl, %edi, %esi
-; X86-BMI1NOTBM-NEXT:    testb $32, %cl
-; X86-BMI1NOTBM-NEXT:    je .LBB20_2
-; X86-BMI1NOTBM-NEXT:  # %bb.1:
-; X86-BMI1NOTBM-NEXT:    movl %edx, %esi
-; X86-BMI1NOTBM-NEXT:    xorl %edx, %edx
-; X86-BMI1NOTBM-NEXT:  .LBB20_2:
-; X86-BMI1NOTBM-NEXT:    movl $-1, %edi
-; X86-BMI1NOTBM-NEXT:    movl $-1, %ebx
-; X86-BMI1NOTBM-NEXT:    movl %eax, %ecx
-; X86-BMI1NOTBM-NEXT:    shll %cl, %ebx
-; X86-BMI1NOTBM-NEXT:    shldl %cl, %edi, %edi
-; X86-BMI1NOTBM-NEXT:    testb $32, %al
-; X86-BMI1NOTBM-NEXT:    je .LBB20_4
-; X86-BMI1NOTBM-NEXT:  # %bb.3:
-; X86-BMI1NOTBM-NEXT:    movl %ebx, %edi
-; X86-BMI1NOTBM-NEXT:    xorl %ebx, %ebx
-; X86-BMI1NOTBM-NEXT:  .LBB20_4:
-; X86-BMI1NOTBM-NEXT:    andnl %edx, %edi, %edx
-; X86-BMI1NOTBM-NEXT:    andnl %esi, %ebx, %eax
-; X86-BMI1NOTBM-NEXT:    popl %esi
-; X86-BMI1NOTBM-NEXT:    popl %edi
-; X86-BMI1NOTBM-NEXT:    popl %ebx
+; X86-BMI1NOTBM-NEXT:    shll $8, %ecx
+; X86-BMI1NOTBM-NEXT:    movzbl {{[0-9]+}}(%esp), %edx
+; X86-BMI1NOTBM-NEXT:    orl %ecx, %edx
+; X86-BMI1NOTBM-NEXT:    bextrl %edx, (%eax), %eax
 ; X86-BMI1NOTBM-NEXT:    retl
 ;
-; X86-BMI1BMI2-LABEL: bextr64_b0:
+; X86-BMI1BMI2-LABEL: bextr32_b2_load:
 ; X86-BMI1BMI2:       # %bb.0:
-; X86-BMI1BMI2-NEXT:    pushl %ebx
-; X86-BMI1BMI2-NEXT:    pushl %edi
-; X86-BMI1BMI2-NEXT:    pushl %esi
 ; X86-BMI1BMI2-NEXT:    movb {{[0-9]+}}(%esp), %al
-; X86-BMI1BMI2-NEXT:    movb {{[0-9]+}}(%esp), %cl
-; X86-BMI1BMI2-NEXT:    movl {{[0-9]+}}(%esp), %esi
-; X86-BMI1BMI2-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; X86-BMI1BMI2-NEXT:    shrdl %cl, %edx, %esi
-; X86-BMI1BMI2-NEXT:    shrxl %ecx, %edx, %edx
-; X86-BMI1BMI2-NEXT:    testb $32, %cl
-; X86-BMI1BMI2-NEXT:    je .LBB20_2
-; X86-BMI1BMI2-NEXT:  # %bb.1:
-; X86-BMI1BMI2-NEXT:    movl %edx, %esi
-; X86-BMI1BMI2-NEXT:    xorl %edx, %edx
-; X86-BMI1BMI2-NEXT:  .LBB20_2:
-; X86-BMI1BMI2-NEXT:    movl $-1, %edi
-; X86-BMI1BMI2-NEXT:    shlxl %eax, %edi, %ebx
-; X86-BMI1BMI2-NEXT:    movl %eax, %ecx
-; X86-BMI1BMI2-NEXT:    shldl %cl, %edi, %edi
-; X86-BMI1BMI2-NEXT:    testb $32, %al
-; X86-BMI1BMI2-NEXT:    je .LBB20_4
-; X86-BMI1BMI2-NEXT:  # %bb.3:
-; X86-BMI1BMI2-NEXT:    movl %ebx, %edi
-; X86-BMI1BMI2-NEXT:    xorl %ebx, %ebx
-; X86-BMI1BMI2-NEXT:  .LBB20_4:
-; X86-BMI1BMI2-NEXT:    andnl %edx, %edi, %edx
-; X86-BMI1BMI2-NEXT:    andnl %esi, %ebx, %eax
-; X86-BMI1BMI2-NEXT:    popl %esi
-; X86-BMI1BMI2-NEXT:    popl %edi
-; X86-BMI1BMI2-NEXT:    popl %ebx
+; X86-BMI1BMI2-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-BMI1BMI2-NEXT:    movb {{[0-9]+}}(%esp), %dl
+; X86-BMI1BMI2-NEXT:    shrxl %edx, (%ecx), %ecx
+; X86-BMI1BMI2-NEXT:    bzhil %eax, %ecx, %eax
 ; X86-BMI1BMI2-NEXT:    retl
 ;
-; X64-NOBMI-LABEL: bextr64_b0:
+; X64-NOBMI-LABEL: bextr32_b2_load:
 ; X64-NOBMI:       # %bb.0:
-; X64-NOBMI-NEXT:    movq %rsi, %rcx
-; X64-NOBMI-NEXT:    # kill: def $cl killed $cl killed $rcx
-; X64-NOBMI-NEXT:    shrq %cl, %rdi
-; X64-NOBMI-NEXT:    movq $-1, %rax
+; X64-NOBMI-NEXT:    movl %esi, %ecx
+; X64-NOBMI-NEXT:    movl (%rdi), %esi
+; X64-NOBMI-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X64-NOBMI-NEXT:    shrl %cl, %esi
+; X64-NOBMI-NEXT:    movl $-1, %eax
 ; X64-NOBMI-NEXT:    movl %edx, %ecx
-; X64-NOBMI-NEXT:    shlq %cl, %rax
-; X64-NOBMI-NEXT:    notq %rax
-; X64-NOBMI-NEXT:    andq %rdi, %rax
+; X64-NOBMI-NEXT:    shll %cl, %eax
+; X64-NOBMI-NEXT:    notl %eax
+; X64-NOBMI-NEXT:    andl %esi, %eax
 ; X64-NOBMI-NEXT:    retq
 ;
-; X64-BMI1NOTBM-LABEL: bextr64_b0:
+; X64-BMI1NOTBM-LABEL: bextr32_b2_load:
 ; X64-BMI1NOTBM:       # %bb.0:
-; X64-BMI1NOTBM-NEXT:    shlq $8, %rdx
+; X64-BMI1NOTBM-NEXT:    shll $8, %edx
 ; X64-BMI1NOTBM-NEXT:    movzbl %sil, %eax
-; X64-BMI1NOTBM-NEXT:    orq %rdx, %rax
-; X64-BMI1NOTBM-NEXT:    bextrq %rax, %rdi, %rax
+; X64-BMI1NOTBM-NEXT:    orl %edx, %eax
+; X64-BMI1NOTBM-NEXT:    bextrl %eax, (%rdi), %eax
 ; X64-BMI1NOTBM-NEXT:    retq
 ;
-; X64-BMI1BMI2-LABEL: bextr64_b0:
+; X64-BMI1BMI2-LABEL: bextr32_b2_load:
 ; X64-BMI1BMI2:       # %bb.0:
-; X64-BMI1BMI2-NEXT:    shrxq %rsi, %rdi, %rax
-; X64-BMI1BMI2-NEXT:    bzhiq %rdx, %rax, %rax
+; X64-BMI1BMI2-NEXT:    shrxl %esi, (%rdi), %eax
+; X64-BMI1BMI2-NEXT:    bzhil %edx, %eax, %eax
 ; X64-BMI1BMI2-NEXT:    retq
-  %shifted = lshr i64 %val, %numskipbits
-  %notmask = shl i64 -1, %numlowbits
-  %mask = xor i64 %notmask, -1
-  %masked = and i64 %mask, %shifted
-  ret i64 %masked
+  %val = load i32, i32* %w
+  %shifted = lshr i32 %val, %numskipbits
+  %notmask = shl i32 -1, %numlowbits
+  %mask = xor i32 %notmask, -1
+  %masked = and i32 %mask, %shifted
+  ret i32 %masked
 }
 
-define i64 @bextr64_b1_indexzext(i64 %val, i8 zeroext %numskipbits, i8 zeroext %numlowbits) nounwind {
-; X86-NOBMI-LABEL: bextr64_b1_indexzext:
+define i32 @bextr32_b3_load_indexzext(i32* %w, i8 zeroext %numskipbits, i8 zeroext %numlowbits) nounwind {
+; X86-NOBMI-LABEL: bextr32_b3_load_indexzext:
 ; X86-NOBMI:       # %bb.0:
-; X86-NOBMI-NEXT:    pushl %edi
 ; X86-NOBMI-NEXT:    pushl %esi
-; X86-NOBMI-NEXT:    movb {{[0-9]+}}(%esp), %ch
+; X86-NOBMI-NEXT:    movb {{[0-9]+}}(%esp), %dl
 ; X86-NOBMI-NEXT:    movb {{[0-9]+}}(%esp), %cl
-; X86-NOBMI-NEXT:    movl {{[0-9]+}}(%esp), %esi
 ; X86-NOBMI-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NOBMI-NEXT:    movl %eax, %edi
-; X86-NOBMI-NEXT:    shrl %cl, %edi
-; X86-NOBMI-NEXT:    shrdl %cl, %eax, %esi
-; X86-NOBMI-NEXT:    testb $32, %cl
-; X86-NOBMI-NEXT:    je .LBB21_2
-; X86-NOBMI-NEXT:  # %bb.1:
-; X86-NOBMI-NEXT:    movl %edi, %esi
-; X86-NOBMI-NEXT:    xorl %edi, %edi
-; X86-NOBMI-NEXT:  .LBB21_2:
-; X86-NOBMI-NEXT:    movl $-1, %edx
+; X86-NOBMI-NEXT:    movl (%eax), %esi
+; X86-NOBMI-NEXT:    shrl %cl, %esi
 ; X86-NOBMI-NEXT:    movl $-1, %eax
-; X86-NOBMI-NEXT:    movb %ch, %cl
+; X86-NOBMI-NEXT:    movl %edx, %ecx
 ; X86-NOBMI-NEXT:    shll %cl, %eax
-; X86-NOBMI-NEXT:    shldl %cl, %edx, %edx
-; X86-NOBMI-NEXT:    testb $32, %ch
-; X86-NOBMI-NEXT:    je .LBB21_4
-; X86-NOBMI-NEXT:  # %bb.3:
-; X86-NOBMI-NEXT:    movl %eax, %edx
-; X86-NOBMI-NEXT:    xorl %eax, %eax
-; X86-NOBMI-NEXT:  .LBB21_4:
-; X86-NOBMI-NEXT:    notl %edx
-; X86-NOBMI-NEXT:    andl %edi, %edx
 ; X86-NOBMI-NEXT:    notl %eax
 ; X86-NOBMI-NEXT:    andl %esi, %eax
 ; X86-NOBMI-NEXT:    popl %esi
-; X86-NOBMI-NEXT:    popl %edi
 ; X86-NOBMI-NEXT:    retl
 ;
-; X86-BMI1NOTBM-LABEL: bextr64_b1_indexzext:
+; X86-BMI1NOTBM-LABEL: bextr32_b3_load_indexzext:
 ; X86-BMI1NOTBM:       # %bb.0:
-; X86-BMI1NOTBM-NEXT:    pushl %ebx
-; X86-BMI1NOTBM-NEXT:    pushl %edi
-; X86-BMI1NOTBM-NEXT:    pushl %esi
-; X86-BMI1NOTBM-NEXT:    movb {{[0-9]+}}(%esp), %al
+; X86-BMI1NOTBM-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-BMI1NOTBM-NEXT:    movb {{[0-9]+}}(%esp), %cl
-; X86-BMI1NOTBM-NEXT:    movl {{[0-9]+}}(%esp), %esi
-; X86-BMI1NOTBM-NEXT:    movl {{[0-9]+}}(%esp), %edi
-; X86-BMI1NOTBM-NEXT:    movl %edi, %edx
-; X86-BMI1NOTBM-NEXT:    shrl %cl, %edx
-; X86-BMI1NOTBM-NEXT:    shrdl %cl, %edi, %esi
-; X86-BMI1NOTBM-NEXT:    testb $32, %cl
-; X86-BMI1NOTBM-NEXT:    je .LBB21_2
-; X86-BMI1NOTBM-NEXT:  # %bb.1:
-; X86-BMI1NOTBM-NEXT:    movl %edx, %esi
-; X86-BMI1NOTBM-NEXT:    xorl %edx, %edx
-; X86-BMI1NOTBM-NEXT:  .LBB21_2:
-; X86-BMI1NOTBM-NEXT:    movl $-1, %edi
-; X86-BMI1NOTBM-NEXT:    movl $-1, %ebx
-; X86-BMI1NOTBM-NEXT:    movl %eax, %ecx
-; X86-BMI1NOTBM-NEXT:    shll %cl, %ebx
-; X86-BMI1NOTBM-NEXT:    shldl %cl, %edi, %edi
-; X86-BMI1NOTBM-NEXT:    testb $32, %al
-; X86-BMI1NOTBM-NEXT:    je .LBB21_4
-; X86-BMI1NOTBM-NEXT:  # %bb.3:
-; X86-BMI1NOTBM-NEXT:    movl %ebx, %edi
-; X86-BMI1NOTBM-NEXT:    xorl %ebx, %ebx
-; X86-BMI1NOTBM-NEXT:  .LBB21_4:
-; X86-BMI1NOTBM-NEXT:    andnl %edx, %edi, %edx
-; X86-BMI1NOTBM-NEXT:    andnl %esi, %ebx, %eax
-; X86-BMI1NOTBM-NEXT:    popl %esi
-; X86-BMI1NOTBM-NEXT:    popl %edi
-; X86-BMI1NOTBM-NEXT:    popl %ebx
+; X86-BMI1NOTBM-NEXT:    shll $8, %ecx
+; X86-BMI1NOTBM-NEXT:    movzbl {{[0-9]+}}(%esp), %edx
+; X86-BMI1NOTBM-NEXT:    orl %ecx, %edx
+; X86-BMI1NOTBM-NEXT:    bextrl %edx, (%eax), %eax
 ; X86-BMI1NOTBM-NEXT:    retl
 ;
-; X86-BMI1BMI2-LABEL: bextr64_b1_indexzext:
+; X86-BMI1BMI2-LABEL: bextr32_b3_load_indexzext:
 ; X86-BMI1BMI2:       # %bb.0:
-; X86-BMI1BMI2-NEXT:    pushl %ebx
-; X86-BMI1BMI2-NEXT:    pushl %edi
-; X86-BMI1BMI2-NEXT:    pushl %esi
 ; X86-BMI1BMI2-NEXT:    movb {{[0-9]+}}(%esp), %al
-; X86-BMI1BMI2-NEXT:    movb {{[0-9]+}}(%esp), %cl
-; X86-BMI1BMI2-NEXT:    movl {{[0-9]+}}(%esp), %esi
-; X86-BMI1BMI2-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; X86-BMI1BMI2-NEXT:    shrdl %cl, %edx, %esi
-; X86-BMI1BMI2-NEXT:    shrxl %ecx, %edx, %edx
-; X86-BMI1BMI2-NEXT:    testb $32, %cl
-; X86-BMI1BMI2-NEXT:    je .LBB21_2
-; X86-BMI1BMI2-NEXT:  # %bb.1:
-; X86-BMI1BMI2-NEXT:    movl %edx, %esi
-; X86-BMI1BMI2-NEXT:    xorl %edx, %edx
-; X86-BMI1BMI2-NEXT:  .LBB21_2:
-; X86-BMI1BMI2-NEXT:    movl $-1, %edi
-; X86-BMI1BMI2-NEXT:    shlxl %eax, %edi, %ebx
-; X86-BMI1BMI2-NEXT:    movl %eax, %ecx
-; X86-BMI1BMI2-NEXT:    shldl %cl, %edi, %edi
-; X86-BMI1BMI2-NEXT:    testb $32, %al
-; X86-BMI1BMI2-NEXT:    je .LBB21_4
-; X86-BMI1BMI2-NEXT:  # %bb.3:
-; X86-BMI1BMI2-NEXT:    movl %ebx, %edi
-; X86-BMI1BMI2-NEXT:    xorl %ebx, %ebx
-; X86-BMI1BMI2-NEXT:  .LBB21_4:
-; X86-BMI1BMI2-NEXT:    andnl %edx, %edi, %edx
-; X86-BMI1BMI2-NEXT:    andnl %esi, %ebx, %eax
-; X86-BMI1BMI2-NEXT:    popl %esi
-; X86-BMI1BMI2-NEXT:    popl %edi
-; X86-BMI1BMI2-NEXT:    popl %ebx
+; X86-BMI1BMI2-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-BMI1BMI2-NEXT:    movb {{[0-9]+}}(%esp), %dl
+; X86-BMI1BMI2-NEXT:    shrxl %edx, (%ecx), %ecx
+; X86-BMI1BMI2-NEXT:    bzhil %eax, %ecx, %eax
 ; X86-BMI1BMI2-NEXT:    retl
 ;
-; X64-NOBMI-LABEL: bextr64_b1_indexzext:
+; X64-NOBMI-LABEL: bextr32_b3_load_indexzext:
 ; X64-NOBMI:       # %bb.0:
 ; X64-NOBMI-NEXT:    movl %esi, %ecx
+; X64-NOBMI-NEXT:    movl (%rdi), %esi
 ; X64-NOBMI-NEXT:    # kill: def $cl killed $cl killed $ecx
-; X64-NOBMI-NEXT:    shrq %cl, %rdi
-; X64-NOBMI-NEXT:    movq $-1, %rax
+; X64-NOBMI-NEXT:    shrl %cl, %esi
+; X64-NOBMI-NEXT:    movl $-1, %eax
 ; X64-NOBMI-NEXT:    movl %edx, %ecx
-; X64-NOBMI-NEXT:    shlq %cl, %rax
-; X64-NOBMI-NEXT:    notq %rax
-; X64-NOBMI-NEXT:    andq %rdi, %rax
+; X64-NOBMI-NEXT:    shll %cl, %eax
+; X64-NOBMI-NEXT:    notl %eax
+; X64-NOBMI-NEXT:    andl %esi, %eax
 ; X64-NOBMI-NEXT:    retq
 ;
-; X64-BMI1NOTBM-LABEL: bextr64_b1_indexzext:
+; X64-BMI1NOTBM-LABEL: bextr32_b3_load_indexzext:
 ; X64-BMI1NOTBM:       # %bb.0:
-; X64-BMI1NOTBM-NEXT:    # kill: def $edx killed $edx def $rdx
-; X64-BMI1NOTBM-NEXT:    shlq $8, %rdx
+; X64-BMI1NOTBM-NEXT:    shll $8, %edx
 ; X64-BMI1NOTBM-NEXT:    movzbl %sil, %eax
-; X64-BMI1NOTBM-NEXT:    orq %rdx, %rax
-; X64-BMI1NOTBM-NEXT:    bextrq %rax, %rdi, %rax
+; X64-BMI1NOTBM-NEXT:    orl %edx, %eax
+; X64-BMI1NOTBM-NEXT:    bextrl %eax, (%rdi), %eax
 ; X64-BMI1NOTBM-NEXT:    retq
 ;
-; X64-BMI1BMI2-LABEL: bextr64_b1_indexzext:
+; X64-BMI1BMI2-LABEL: bextr32_b3_load_indexzext:
 ; X64-BMI1BMI2:       # %bb.0:
-; X64-BMI1BMI2-NEXT:    # kill: def $edx killed $edx def $rdx
-; X64-BMI1BMI2-NEXT:    # kill: def $esi killed $esi def $rsi
-; X64-BMI1BMI2-NEXT:    shrxq %rsi, %rdi, %rax
-; X64-BMI1BMI2-NEXT:    bzhiq %rdx, %rax, %rax
+; X64-BMI1BMI2-NEXT:    shrxl %esi, (%rdi), %eax
+; X64-BMI1BMI2-NEXT:    bzhil %edx, %eax, %eax
 ; X64-BMI1BMI2-NEXT:    retq
-  %skip = zext i8 %numskipbits to i64
-  %shifted = lshr i64 %val, %skip
-  %conv = zext i8 %numlowbits to i64
-  %notmask = shl i64 -1, %conv
-  %mask = xor i64 %notmask, -1
-  %masked = and i64 %mask, %shifted
-  ret i64 %masked
+  %val = load i32, i32* %w
+  %skip = zext i8 %numskipbits to i32
+  %shifted = lshr i32 %val, %skip
+  %conv = zext i8 %numlowbits to i32
+  %notmask = shl i32 -1, %conv
+  %mask = xor i32 %notmask, -1
+  %masked = and i32 %mask, %shifted
+  ret i32 %masked
 }
 
-define i64 @bextr64_b2_load(i64* %w, i64 %numskipbits, i64 %numlowbits) nounwind {
-; X86-NOBMI-LABEL: bextr64_b2_load:
+define i32 @bextr32_b4_commutative(i32 %val, i32 %numskipbits, i32 %numlowbits) nounwind {
+; X86-NOBMI-LABEL: bextr32_b4_commutative:
+; X86-NOBMI:       # %bb.0:
+; X86-NOBMI-NEXT:    pushl %esi
+; X86-NOBMI-NEXT:    movb {{[0-9]+}}(%esp), %dl
+; X86-NOBMI-NEXT:    movb {{[0-9]+}}(%esp), %cl
+; X86-NOBMI-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NOBMI-NEXT:    shrl %cl, %esi
+; X86-NOBMI-NEXT:    movl $-1, %eax
+; X86-NOBMI-NEXT:    movl %edx, %ecx
+; X86-NOBMI-NEXT:    shll %cl, %eax
+; X86-NOBMI-NEXT:    notl %eax
+; X86-NOBMI-NEXT:    andl %esi, %eax
+; X86-NOBMI-NEXT:    popl %esi
+; X86-NOBMI-NEXT:    retl
+;
+; X86-BMI1NOTBM-LABEL: bextr32_b4_commutative:
+; X86-BMI1NOTBM:       # %bb.0:
+; X86-BMI1NOTBM-NEXT:    movb {{[0-9]+}}(%esp), %al
+; X86-BMI1NOTBM-NEXT:    shll $8, %eax
+; X86-BMI1NOTBM-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx
+; X86-BMI1NOTBM-NEXT:    orl %eax, %ecx
+; X86-BMI1NOTBM-NEXT:    bextrl %ecx, {{[0-9]+}}(%esp), %eax
+; X86-BMI1NOTBM-NEXT:    retl
+;
+; X86-BMI1BMI2-LABEL: bextr32_b4_commutative:
+; X86-BMI1BMI2:       # %bb.0:
+; X86-BMI1BMI2-NEXT:    movb {{[0-9]+}}(%esp), %al
+; X86-BMI1BMI2-NEXT:    movb {{[0-9]+}}(%esp), %cl
+; X86-BMI1BMI2-NEXT:    shrxl %ecx, {{[0-9]+}}(%esp), %ecx
+; X86-BMI1BMI2-NEXT:    bzhil %eax, %ecx, %eax
+; X86-BMI1BMI2-NEXT:    retl
+;
+; X64-NOBMI-LABEL: bextr32_b4_commutative:
+; X64-NOBMI:       # %bb.0:
+; X64-NOBMI-NEXT:    movl %esi, %ecx
+; X64-NOBMI-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X64-NOBMI-NEXT:    shrl %cl, %edi
+; X64-NOBMI-NEXT:    movl $-1, %eax
+; X64-NOBMI-NEXT:    movl %edx, %ecx
+; X64-NOBMI-NEXT:    shll %cl, %eax
+; X64-NOBMI-NEXT:    notl %eax
+; X64-NOBMI-NEXT:    andl %edi, %eax
+; X64-NOBMI-NEXT:    retq
+;
+; X64-BMI1NOTBM-LABEL: bextr32_b4_commutative:
+; X64-BMI1NOTBM:       # %bb.0:
+; X64-BMI1NOTBM-NEXT:    shll $8, %edx
+; X64-BMI1NOTBM-NEXT:    movzbl %sil, %eax
+; X64-BMI1NOTBM-NEXT:    orl %edx, %eax
+; X64-BMI1NOTBM-NEXT:    bextrl %eax, %edi, %eax
+; X64-BMI1NOTBM-NEXT:    retq
+;
+; X64-BMI1BMI2-LABEL: bextr32_b4_commutative:
+; X64-BMI1BMI2:       # %bb.0:
+; X64-BMI1BMI2-NEXT:    shrxl %esi, %edi, %eax
+; X64-BMI1BMI2-NEXT:    bzhil %edx, %eax, %eax
+; X64-BMI1BMI2-NEXT:    retq
+  %shifted = lshr i32 %val, %numskipbits
+  %notmask = shl i32 -1, %numlowbits
+  %mask = xor i32 %notmask, -1
+  %masked = and i32 %shifted, %mask ; swapped order
+  ret i32 %masked
+}
+
+define i32 @bextr32_b5_skipextrauses(i32 %val, i32 %numskipbits, i32 %numlowbits) nounwind {
+; X86-NOBMI-LABEL: bextr32_b5_skipextrauses:
+; X86-NOBMI:       # %bb.0:
+; X86-NOBMI-NEXT:    pushl %edi
+; X86-NOBMI-NEXT:    pushl %esi
+; X86-NOBMI-NEXT:    pushl %eax
+; X86-NOBMI-NEXT:    movb {{[0-9]+}}(%esp), %dl
+; X86-NOBMI-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NOBMI-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NOBMI-NEXT:    movl %eax, %ecx
+; X86-NOBMI-NEXT:    shrl %cl, %edi
+; X86-NOBMI-NEXT:    movl $-1, %esi
+; X86-NOBMI-NEXT:    movl %edx, %ecx
+; X86-NOBMI-NEXT:    shll %cl, %esi
+; X86-NOBMI-NEXT:    notl %esi
+; X86-NOBMI-NEXT:    andl %edi, %esi
+; X86-NOBMI-NEXT:    movl %eax, (%esp)
+; X86-NOBMI-NEXT:    calll use32
+; X86-NOBMI-NEXT:    movl %esi, %eax
+; X86-NOBMI-NEXT:    addl $4, %esp
+; X86-NOBMI-NEXT:    popl %esi
+; X86-NOBMI-NEXT:    popl %edi
+; X86-NOBMI-NEXT:    retl
+;
+; X86-BMI1NOTBM-LABEL: bextr32_b5_skipextrauses:
+; X86-BMI1NOTBM:       # %bb.0:
+; X86-BMI1NOTBM-NEXT:    pushl %esi
+; X86-BMI1NOTBM-NEXT:    subl $8, %esp
+; X86-BMI1NOTBM-NEXT:    movb {{[0-9]+}}(%esp), %cl
+; X86-BMI1NOTBM-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-BMI1NOTBM-NEXT:    shll $8, %ecx
+; X86-BMI1NOTBM-NEXT:    movzbl %al, %edx
+; X86-BMI1NOTBM-NEXT:    orl %ecx, %edx
+; X86-BMI1NOTBM-NEXT:    bextrl %edx, {{[0-9]+}}(%esp), %esi
+; X86-BMI1NOTBM-NEXT:    movl %eax, (%esp)
+; X86-BMI1NOTBM-NEXT:    calll use32
+; X86-BMI1NOTBM-NEXT:    movl %esi, %eax
+; X86-BMI1NOTBM-NEXT:    addl $8, %esp
+; X86-BMI1NOTBM-NEXT:    popl %esi
+; X86-BMI1NOTBM-NEXT:    retl
+;
+; X86-BMI1BMI2-LABEL: bextr32_b5_skipextrauses:
+; X86-BMI1BMI2:       # %bb.0:
+; X86-BMI1BMI2-NEXT:    pushl %esi
+; X86-BMI1BMI2-NEXT:    subl $8, %esp
+; X86-BMI1BMI2-NEXT:    movb {{[0-9]+}}(%esp), %al
+; X86-BMI1BMI2-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-BMI1BMI2-NEXT:    shrxl %ecx, {{[0-9]+}}(%esp), %edx
+; X86-BMI1BMI2-NEXT:    bzhil %eax, %edx, %esi
+; X86-BMI1BMI2-NEXT:    movl %ecx, (%esp)
+; X86-BMI1BMI2-NEXT:    calll use32
+; X86-BMI1BMI2-NEXT:    movl %esi, %eax
+; X86-BMI1BMI2-NEXT:    addl $8, %esp
+; X86-BMI1BMI2-NEXT:    popl %esi
+; X86-BMI1BMI2-NEXT:    retl
+;
+; X64-NOBMI-LABEL: bextr32_b5_skipextrauses:
+; X64-NOBMI:       # %bb.0:
+; X64-NOBMI-NEXT:    pushq %rbx
+; X64-NOBMI-NEXT:    movl %esi, %ecx
+; X64-NOBMI-NEXT:    shrl %cl, %edi
+; X64-NOBMI-NEXT:    movl $-1, %ebx
+; X64-NOBMI-NEXT:    movl %edx, %ecx
+; X64-NOBMI-NEXT:    shll %cl, %ebx
+; X64-NOBMI-NEXT:    notl %ebx
+; X64-NOBMI-NEXT:    andl %edi, %ebx
+; X64-NOBMI-NEXT:    movl %esi, %edi
+; X64-NOBMI-NEXT:    callq use32
+; X64-NOBMI-NEXT:    movl %ebx, %eax
+; X64-NOBMI-NEXT:    popq %rbx
+; X64-NOBMI-NEXT:    retq
+;
+; X64-BMI1NOTBM-LABEL: bextr32_b5_skipextrauses:
+; X64-BMI1NOTBM:       # %bb.0:
+; X64-BMI1NOTBM-NEXT:    pushq %rbx
+; X64-BMI1NOTBM-NEXT:    shll $8, %edx
+; X64-BMI1NOTBM-NEXT:    movzbl %sil, %eax
+; X64-BMI1NOTBM-NEXT:    orl %edx, %eax
+; X64-BMI1NOTBM-NEXT:    bextrl %eax, %edi, %ebx
+; X64-BMI1NOTBM-NEXT:    movl %esi, %edi
+; X64-BMI1NOTBM-NEXT:    callq use32
+; X64-BMI1NOTBM-NEXT:    movl %ebx, %eax
+; X64-BMI1NOTBM-NEXT:    popq %rbx
+; X64-BMI1NOTBM-NEXT:    retq
+;
+; X64-BMI1BMI2-LABEL: bextr32_b5_skipextrauses:
+; X64-BMI1BMI2:       # %bb.0:
+; X64-BMI1BMI2-NEXT:    pushq %rbx
+; X64-BMI1BMI2-NEXT:    shrxl %esi, %edi, %eax
+; X64-BMI1BMI2-NEXT:    bzhil %edx, %eax, %ebx
+; X64-BMI1BMI2-NEXT:    movl %esi, %edi
+; X64-BMI1BMI2-NEXT:    callq use32
+; X64-BMI1BMI2-NEXT:    movl %ebx, %eax
+; X64-BMI1BMI2-NEXT:    popq %rbx
+; X64-BMI1BMI2-NEXT:    retq
+  %shifted = lshr i32 %val, %numskipbits
+  %notmask = shl i32 -1, %numlowbits
+  %mask = xor i32 %notmask, -1
+  %masked = and i32 %mask, %shifted
+  call void @use32(i32 %numskipbits)
+  ret i32 %masked
+}
+
+; 64-bit
+
+define i64 @bextr64_b0(i64 %val, i64 %numskipbits, i64 %numlowbits) nounwind {
+; X86-NOBMI-LABEL: bextr64_b0:
 ; X86-NOBMI:       # %bb.0:
 ; X86-NOBMI-NEXT:    pushl %edi
 ; X86-NOBMI-NEXT:    pushl %esi
 ; X86-NOBMI-NEXT:    movb {{[0-9]+}}(%esp), %ch
 ; X86-NOBMI-NEXT:    movb {{[0-9]+}}(%esp), %cl
+; X86-NOBMI-NEXT:    movl {{[0-9]+}}(%esp), %esi
 ; X86-NOBMI-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NOBMI-NEXT:    movl (%eax), %esi
-; X86-NOBMI-NEXT:    movl 4(%eax), %eax
 ; X86-NOBMI-NEXT:    movl %eax, %edi
 ; X86-NOBMI-NEXT:    shrl %cl, %edi
 ; X86-NOBMI-NEXT:    shrdl %cl, %eax, %esi
 ; X86-NOBMI-NEXT:    testb $32, %cl
-; X86-NOBMI-NEXT:    je .LBB22_2
+; X86-NOBMI-NEXT:    je .LBB24_2
 ; X86-NOBMI-NEXT:  # %bb.1:
 ; X86-NOBMI-NEXT:    movl %edi, %esi
 ; X86-NOBMI-NEXT:    xorl %edi, %edi
-; X86-NOBMI-NEXT:  .LBB22_2:
+; X86-NOBMI-NEXT:  .LBB24_2:
 ; X86-NOBMI-NEXT:    movl $-1, %edx
 ; X86-NOBMI-NEXT:    movl $-1, %eax
 ; X86-NOBMI-NEXT:    movb %ch, %cl
 ; X86-NOBMI-NEXT:    shll %cl, %eax
 ; X86-NOBMI-NEXT:    shldl %cl, %edx, %edx
 ; X86-NOBMI-NEXT:    testb $32, %ch
-; X86-NOBMI-NEXT:    je .LBB22_4
+; X86-NOBMI-NEXT:    je .LBB24_4
 ; X86-NOBMI-NEXT:  # %bb.3:
 ; X86-NOBMI-NEXT:    movl %eax, %edx
 ; X86-NOBMI-NEXT:    xorl %eax, %eax
-; X86-NOBMI-NEXT:  .LBB22_4:
+; X86-NOBMI-NEXT:  .LBB24_4:
 ; X86-NOBMI-NEXT:    notl %edx
 ; X86-NOBMI-NEXT:    andl %edi, %edx
 ; X86-NOBMI-NEXT:    notl %eax
@@ -2383,36 +2578,35 @@ define i64 @bextr64_b2_load(i64* %w, i64 %numskipbits, i64 %numlowbits) nounwind
 ; X86-NOBMI-NEXT:    popl %edi
 ; X86-NOBMI-NEXT:    retl
 ;
-; X86-BMI1NOTBM-LABEL: bextr64_b2_load:
+; X86-BMI1NOTBM-LABEL: bextr64_b0:
 ; X86-BMI1NOTBM:       # %bb.0:
 ; X86-BMI1NOTBM-NEXT:    pushl %ebx
 ; X86-BMI1NOTBM-NEXT:    pushl %edi
 ; X86-BMI1NOTBM-NEXT:    pushl %esi
 ; X86-BMI1NOTBM-NEXT:    movb {{[0-9]+}}(%esp), %al
 ; X86-BMI1NOTBM-NEXT:    movb {{[0-9]+}}(%esp), %cl
-; X86-BMI1NOTBM-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; X86-BMI1NOTBM-NEXT:    movl (%edx), %esi
-; X86-BMI1NOTBM-NEXT:    movl 4(%edx), %edi
+; X86-BMI1NOTBM-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-BMI1NOTBM-NEXT:    movl {{[0-9]+}}(%esp), %edi
 ; X86-BMI1NOTBM-NEXT:    movl %edi, %edx
 ; X86-BMI1NOTBM-NEXT:    shrl %cl, %edx
 ; X86-BMI1NOTBM-NEXT:    shrdl %cl, %edi, %esi
 ; X86-BMI1NOTBM-NEXT:    testb $32, %cl
-; X86-BMI1NOTBM-NEXT:    je .LBB22_2
+; X86-BMI1NOTBM-NEXT:    je .LBB24_2
 ; X86-BMI1NOTBM-NEXT:  # %bb.1:
 ; X86-BMI1NOTBM-NEXT:    movl %edx, %esi
 ; X86-BMI1NOTBM-NEXT:    xorl %edx, %edx
-; X86-BMI1NOTBM-NEXT:  .LBB22_2:
+; X86-BMI1NOTBM-NEXT:  .LBB24_2:
 ; X86-BMI1NOTBM-NEXT:    movl $-1, %edi
 ; X86-BMI1NOTBM-NEXT:    movl $-1, %ebx
 ; X86-BMI1NOTBM-NEXT:    movl %eax, %ecx
 ; X86-BMI1NOTBM-NEXT:    shll %cl, %ebx
 ; X86-BMI1NOTBM-NEXT:    shldl %cl, %edi, %edi
 ; X86-BMI1NOTBM-NEXT:    testb $32, %al
-; X86-BMI1NOTBM-NEXT:    je .LBB22_4
+; X86-BMI1NOTBM-NEXT:    je .LBB24_4
 ; X86-BMI1NOTBM-NEXT:  # %bb.3:
 ; X86-BMI1NOTBM-NEXT:    movl %ebx, %edi
 ; X86-BMI1NOTBM-NEXT:    xorl %ebx, %ebx
-; X86-BMI1NOTBM-NEXT:  .LBB22_4:
+; X86-BMI1NOTBM-NEXT:  .LBB24_4:
 ; X86-BMI1NOTBM-NEXT:    andnl %edx, %edi, %edx
 ; X86-BMI1NOTBM-NEXT:    andnl %esi, %ebx, %eax
 ; X86-BMI1NOTBM-NEXT:    popl %esi
@@ -2420,34 +2614,33 @@ define i64 @bextr64_b2_load(i64* %w, i64 %numskipbits, i64 %numlowbits) nounwind
 ; X86-BMI1NOTBM-NEXT:    popl %ebx
 ; X86-BMI1NOTBM-NEXT:    retl
 ;
-; X86-BMI1BMI2-LABEL: bextr64_b2_load:
+; X86-BMI1BMI2-LABEL: bextr64_b0:
 ; X86-BMI1BMI2:       # %bb.0:
 ; X86-BMI1BMI2-NEXT:    pushl %ebx
 ; X86-BMI1BMI2-NEXT:    pushl %edi
 ; X86-BMI1BMI2-NEXT:    pushl %esi
 ; X86-BMI1BMI2-NEXT:    movb {{[0-9]+}}(%esp), %al
 ; X86-BMI1BMI2-NEXT:    movb {{[0-9]+}}(%esp), %cl
+; X86-BMI1BMI2-NEXT:    movl {{[0-9]+}}(%esp), %esi
 ; X86-BMI1BMI2-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; X86-BMI1BMI2-NEXT:    movl (%edx), %esi
-; X86-BMI1BMI2-NEXT:    movl 4(%edx), %edi
-; X86-BMI1BMI2-NEXT:    shrxl %ecx, %edi, %edx
-; X86-BMI1BMI2-NEXT:    shrdl %cl, %edi, %esi
+; X86-BMI1BMI2-NEXT:    shrdl %cl, %edx, %esi
+; X86-BMI1BMI2-NEXT:    shrxl %ecx, %edx, %edx
 ; X86-BMI1BMI2-NEXT:    testb $32, %cl
-; X86-BMI1BMI2-NEXT:    je .LBB22_2
+; X86-BMI1BMI2-NEXT:    je .LBB24_2
 ; X86-BMI1BMI2-NEXT:  # %bb.1:
 ; X86-BMI1BMI2-NEXT:    movl %edx, %esi
 ; X86-BMI1BMI2-NEXT:    xorl %edx, %edx
-; X86-BMI1BMI2-NEXT:  .LBB22_2:
+; X86-BMI1BMI2-NEXT:  .LBB24_2:
 ; X86-BMI1BMI2-NEXT:    movl $-1, %edi
 ; X86-BMI1BMI2-NEXT:    shlxl %eax, %edi, %ebx
 ; X86-BMI1BMI2-NEXT:    movl %eax, %ecx
 ; X86-BMI1BMI2-NEXT:    shldl %cl, %edi, %edi
 ; X86-BMI1BMI2-NEXT:    testb $32, %al
-; X86-BMI1BMI2-NEXT:    je .LBB22_4
+; X86-BMI1BMI2-NEXT:    je .LBB24_4
 ; X86-BMI1BMI2-NEXT:  # %bb.3:
 ; X86-BMI1BMI2-NEXT:    movl %ebx, %edi
 ; X86-BMI1BMI2-NEXT:    xorl %ebx, %ebx
-; X86-BMI1BMI2-NEXT:  .LBB22_4:
+; X86-BMI1BMI2-NEXT:  .LBB24_4:
 ; X86-BMI1BMI2-NEXT:    andnl %edx, %edi, %edx
 ; X86-BMI1BMI2-NEXT:    andnl %esi, %ebx, %eax
 ; X86-BMI1BMI2-NEXT:    popl %esi
@@ -2455,33 +2648,31 @@ define i64 @bextr64_b2_load(i64* %w, i64 %numskipbits, i64 %numlowbits) nounwind
 ; X86-BMI1BMI2-NEXT:    popl %ebx
 ; X86-BMI1BMI2-NEXT:    retl
 ;
-; X64-NOBMI-LABEL: bextr64_b2_load:
+; X64-NOBMI-LABEL: bextr64_b0:
 ; X64-NOBMI:       # %bb.0:
 ; X64-NOBMI-NEXT:    movq %rsi, %rcx
-; X64-NOBMI-NEXT:    movq (%rdi), %rsi
 ; X64-NOBMI-NEXT:    # kill: def $cl killed $cl killed $rcx
-; X64-NOBMI-NEXT:    shrq %cl, %rsi
+; X64-NOBMI-NEXT:    shrq %cl, %rdi
 ; X64-NOBMI-NEXT:    movq $-1, %rax
 ; X64-NOBMI-NEXT:    movl %edx, %ecx
 ; X64-NOBMI-NEXT:    shlq %cl, %rax
 ; X64-NOBMI-NEXT:    notq %rax
-; X64-NOBMI-NEXT:    andq %rsi, %rax
+; X64-NOBMI-NEXT:    andq %rdi, %rax
 ; X64-NOBMI-NEXT:    retq
 ;
-; X64-BMI1NOTBM-LABEL: bextr64_b2_load:
+; X64-BMI1NOTBM-LABEL: bextr64_b0:
 ; X64-BMI1NOTBM:       # %bb.0:
-; X64-BMI1NOTBM-NEXT:    shlq $8, %rdx
+; X64-BMI1NOTBM-NEXT:    shll $8, %edx
 ; X64-BMI1NOTBM-NEXT:    movzbl %sil, %eax
-; X64-BMI1NOTBM-NEXT:    orq %rdx, %rax
-; X64-BMI1NOTBM-NEXT:    bextrq %rax, (%rdi), %rax
+; X64-BMI1NOTBM-NEXT:    orl %edx, %eax
+; X64-BMI1NOTBM-NEXT:    bextrq %rax, %rdi, %rax
 ; X64-BMI1NOTBM-NEXT:    retq
 ;
-; X64-BMI1BMI2-LABEL: bextr64_b2_load:
+; X64-BMI1BMI2-LABEL: bextr64_b0:
 ; X64-BMI1BMI2:       # %bb.0:
-; X64-BMI1BMI2-NEXT:    shrxq %rsi, (%rdi), %rax
+; X64-BMI1BMI2-NEXT:    shrxq %rsi, %rdi, %rax
 ; X64-BMI1BMI2-NEXT:    bzhiq %rdx, %rax, %rax
 ; X64-BMI1BMI2-NEXT:    retq
-  %val = load i64, i64* %w
   %shifted = lshr i64 %val, %numskipbits
   %notmask = shl i64 -1, %numlowbits
   %mask = xor i64 %notmask, -1
@@ -2489,36 +2680,35 @@ define i64 @bextr64_b2_load(i64* %w, i64 %numskipbits, i64 %numlowbits) nounwind
   ret i64 %masked
 }
 
-define i64 @bextr64_b3_load_indexzext(i64* %w, i8 zeroext %numskipbits, i8 zeroext %numlowbits) nounwind {
-; X86-NOBMI-LABEL: bextr64_b3_load_indexzext:
+define i64 @bextr64_b1_indexzext(i64 %val, i8 zeroext %numskipbits, i8 zeroext %numlowbits) nounwind {
+; X86-NOBMI-LABEL: bextr64_b1_indexzext:
 ; X86-NOBMI:       # %bb.0:
 ; X86-NOBMI-NEXT:    pushl %edi
 ; X86-NOBMI-NEXT:    pushl %esi
 ; X86-NOBMI-NEXT:    movb {{[0-9]+}}(%esp), %ch
 ; X86-NOBMI-NEXT:    movb {{[0-9]+}}(%esp), %cl
+; X86-NOBMI-NEXT:    movl {{[0-9]+}}(%esp), %esi
 ; X86-NOBMI-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NOBMI-NEXT:    movl (%eax), %esi
-; X86-NOBMI-NEXT:    movl 4(%eax), %eax
 ; X86-NOBMI-NEXT:    movl %eax, %edi
 ; X86-NOBMI-NEXT:    shrl %cl, %edi
 ; X86-NOBMI-NEXT:    shrdl %cl, %eax, %esi
 ; X86-NOBMI-NEXT:    testb $32, %cl
-; X86-NOBMI-NEXT:    je .LBB23_2
+; X86-NOBMI-NEXT:    je .LBB25_2
 ; X86-NOBMI-NEXT:  # %bb.1:
 ; X86-NOBMI-NEXT:    movl %edi, %esi
 ; X86-NOBMI-NEXT:    xorl %edi, %edi
-; X86-NOBMI-NEXT:  .LBB23_2:
+; X86-NOBMI-NEXT:  .LBB25_2:
 ; X86-NOBMI-NEXT:    movl $-1, %edx
 ; X86-NOBMI-NEXT:    movl $-1, %eax
 ; X86-NOBMI-NEXT:    movb %ch, %cl
 ; X86-NOBMI-NEXT:    shll %cl, %eax
 ; X86-NOBMI-NEXT:    shldl %cl, %edx, %edx
 ; X86-NOBMI-NEXT:    testb $32, %ch
-; X86-NOBMI-NEXT:    je .LBB23_4
+; X86-NOBMI-NEXT:    je .LBB25_4
 ; X86-NOBMI-NEXT:  # %bb.3:
 ; X86-NOBMI-NEXT:    movl %eax, %edx
 ; X86-NOBMI-NEXT:    xorl %eax, %eax
-; X86-NOBMI-NEXT:  .LBB23_4:
+; X86-NOBMI-NEXT:  .LBB25_4:
 ; X86-NOBMI-NEXT:    notl %edx
 ; X86-NOBMI-NEXT:    andl %edi, %edx
 ; X86-NOBMI-NEXT:    notl %eax
@@ -2527,36 +2717,35 @@ define i64 @bextr64_b3_load_indexzext(i64* %w, i8 zeroext %numskipbits, i8 zeroe
 ; X86-NOBMI-NEXT:    popl %edi
 ; X86-NOBMI-NEXT:    retl
 ;
-; X86-BMI1NOTBM-LABEL: bextr64_b3_load_indexzext:
+; X86-BMI1NOTBM-LABEL: bextr64_b1_indexzext:
 ; X86-BMI1NOTBM:       # %bb.0:
 ; X86-BMI1NOTBM-NEXT:    pushl %ebx
 ; X86-BMI1NOTBM-NEXT:    pushl %edi
 ; X86-BMI1NOTBM-NEXT:    pushl %esi
 ; X86-BMI1NOTBM-NEXT:    movb {{[0-9]+}}(%esp), %al
 ; X86-BMI1NOTBM-NEXT:    movb {{[0-9]+}}(%esp), %cl
-; X86-BMI1NOTBM-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; X86-BMI1NOTBM-NEXT:    movl (%edx), %esi
-; X86-BMI1NOTBM-NEXT:    movl 4(%edx), %edi
+; X86-BMI1NOTBM-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-BMI1NOTBM-NEXT:    movl {{[0-9]+}}(%esp), %edi
 ; X86-BMI1NOTBM-NEXT:    movl %edi, %edx
 ; X86-BMI1NOTBM-NEXT:    shrl %cl, %edx
 ; X86-BMI1NOTBM-NEXT:    shrdl %cl, %edi, %esi
 ; X86-BMI1NOTBM-NEXT:    testb $32, %cl
-; X86-BMI1NOTBM-NEXT:    je .LBB23_2
+; X86-BMI1NOTBM-NEXT:    je .LBB25_2
 ; X86-BMI1NOTBM-NEXT:  # %bb.1:
 ; X86-BMI1NOTBM-NEXT:    movl %edx, %esi
 ; X86-BMI1NOTBM-NEXT:    xorl %edx, %edx
-; X86-BMI1NOTBM-NEXT:  .LBB23_2:
+; X86-BMI1NOTBM-NEXT:  .LBB25_2:
 ; X86-BMI1NOTBM-NEXT:    movl $-1, %edi
 ; X86-BMI1NOTBM-NEXT:    movl $-1, %ebx
 ; X86-BMI1NOTBM-NEXT:    movl %eax, %ecx
 ; X86-BMI1NOTBM-NEXT:    shll %cl, %ebx
 ; X86-BMI1NOTBM-NEXT:    shldl %cl, %edi, %edi
 ; X86-BMI1NOTBM-NEXT:    testb $32, %al
-; X86-BMI1NOTBM-NEXT:    je .LBB23_4
+; X86-BMI1NOTBM-NEXT:    je .LBB25_4
 ; X86-BMI1NOTBM-NEXT:  # %bb.3:
 ; X86-BMI1NOTBM-NEXT:    movl %ebx, %edi
 ; X86-BMI1NOTBM-NEXT:    xorl %ebx, %ebx
-; X86-BMI1NOTBM-NEXT:  .LBB23_4:
+; X86-BMI1NOTBM-NEXT:  .LBB25_4:
 ; X86-BMI1NOTBM-NEXT:    andnl %edx, %edi, %edx
 ; X86-BMI1NOTBM-NEXT:    andnl %esi, %ebx, %eax
 ; X86-BMI1NOTBM-NEXT:    popl %esi
@@ -2564,34 +2753,33 @@ define i64 @bextr64_b3_load_indexzext(i64* %w, i8 zeroext %numskipbits, i8 zeroe
 ; X86-BMI1NOTBM-NEXT:    popl %ebx
 ; X86-BMI1NOTBM-NEXT:    retl
 ;
-; X86-BMI1BMI2-LABEL: bextr64_b3_load_indexzext:
+; X86-BMI1BMI2-LABEL: bextr64_b1_indexzext:
 ; X86-BMI1BMI2:       # %bb.0:
 ; X86-BMI1BMI2-NEXT:    pushl %ebx
 ; X86-BMI1BMI2-NEXT:    pushl %edi
 ; X86-BMI1BMI2-NEXT:    pushl %esi
 ; X86-BMI1BMI2-NEXT:    movb {{[0-9]+}}(%esp), %al
 ; X86-BMI1BMI2-NEXT:    movb {{[0-9]+}}(%esp), %cl
+; X86-BMI1BMI2-NEXT:    movl {{[0-9]+}}(%esp), %esi
 ; X86-BMI1BMI2-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; X86-BMI1BMI2-NEXT:    movl (%edx), %esi
-; X86-BMI1BMI2-NEXT:    movl 4(%edx), %edi
-; X86-BMI1BMI2-NEXT:    shrxl %ecx, %edi, %edx
-; X86-BMI1BMI2-NEXT:    shrdl %cl, %edi, %esi
+; X86-BMI1BMI2-NEXT:    shrdl %cl, %edx, %esi
+; X86-BMI1BMI2-NEXT:    shrxl %ecx, %edx, %edx
 ; X86-BMI1BMI2-NEXT:    testb $32, %cl
-; X86-BMI1BMI2-NEXT:    je .LBB23_2
+; X86-BMI1BMI2-NEXT:    je .LBB25_2
 ; X86-BMI1BMI2-NEXT:  # %bb.1:
 ; X86-BMI1BMI2-NEXT:    movl %edx, %esi
 ; X86-BMI1BMI2-NEXT:    xorl %edx, %edx
-; X86-BMI1BMI2-NEXT:  .LBB23_2:
+; X86-BMI1BMI2-NEXT:  .LBB25_2:
 ; X86-BMI1BMI2-NEXT:    movl $-1, %edi
 ; X86-BMI1BMI2-NEXT:    shlxl %eax, %edi, %ebx
 ; X86-BMI1BMI2-NEXT:    movl %eax, %ecx
 ; X86-BMI1BMI2-NEXT:    shldl %cl, %edi, %edi
 ; X86-BMI1BMI2-NEXT:    testb $32, %al
-; X86-BMI1BMI2-NEXT:    je .LBB23_4
+; X86-BMI1BMI2-NEXT:    je .LBB25_4
 ; X86-BMI1BMI2-NEXT:  # %bb.3:
 ; X86-BMI1BMI2-NEXT:    movl %ebx, %edi
 ; X86-BMI1BMI2-NEXT:    xorl %ebx, %ebx
-; X86-BMI1BMI2-NEXT:  .LBB23_4:
+; X86-BMI1BMI2-NEXT:  .LBB25_4:
 ; X86-BMI1BMI2-NEXT:    andnl %edx, %edi, %edx
 ; X86-BMI1BMI2-NEXT:    andnl %esi, %ebx, %eax
 ; X86-BMI1BMI2-NEXT:    popl %esi
@@ -2599,36 +2787,33 @@ define i64 @bextr64_b3_load_indexzext(i64* %w, i8 zeroext %numskipbits, i8 zeroe
 ; X86-BMI1BMI2-NEXT:    popl %ebx
 ; X86-BMI1BMI2-NEXT:    retl
 ;
-; X64-NOBMI-LABEL: bextr64_b3_load_indexzext:
+; X64-NOBMI-LABEL: bextr64_b1_indexzext:
 ; X64-NOBMI:       # %bb.0:
 ; X64-NOBMI-NEXT:    movl %esi, %ecx
-; X64-NOBMI-NEXT:    movq (%rdi), %rsi
 ; X64-NOBMI-NEXT:    # kill: def $cl killed $cl killed $ecx
-; X64-NOBMI-NEXT:    shrq %cl, %rsi
+; X64-NOBMI-NEXT:    shrq %cl, %rdi
 ; X64-NOBMI-NEXT:    movq $-1, %rax
 ; X64-NOBMI-NEXT:    movl %edx, %ecx
 ; X64-NOBMI-NEXT:    shlq %cl, %rax
 ; X64-NOBMI-NEXT:    notq %rax
-; X64-NOBMI-NEXT:    andq %rsi, %rax
+; X64-NOBMI-NEXT:    andq %rdi, %rax
 ; X64-NOBMI-NEXT:    retq
 ;
-; X64-BMI1NOTBM-LABEL: bextr64_b3_load_indexzext:
+; X64-BMI1NOTBM-LABEL: bextr64_b1_indexzext:
 ; X64-BMI1NOTBM:       # %bb.0:
-; X64-BMI1NOTBM-NEXT:    # kill: def $edx killed $edx def $rdx
-; X64-BMI1NOTBM-NEXT:    shlq $8, %rdx
+; X64-BMI1NOTBM-NEXT:    shll $8, %edx
 ; X64-BMI1NOTBM-NEXT:    movzbl %sil, %eax
-; X64-BMI1NOTBM-NEXT:    orq %rdx, %rax
-; X64-BMI1NOTBM-NEXT:    bextrq %rax, (%rdi), %rax
+; X64-BMI1NOTBM-NEXT:    orl %edx, %eax
+; X64-BMI1NOTBM-NEXT:    bextrq %rax, %rdi, %rax
 ; X64-BMI1NOTBM-NEXT:    retq
 ;
-; X64-BMI1BMI2-LABEL: bextr64_b3_load_indexzext:
+; X64-BMI1BMI2-LABEL: bextr64_b1_indexzext:
 ; X64-BMI1BMI2:       # %bb.0:
 ; X64-BMI1BMI2-NEXT:    # kill: def $edx killed $edx def $rdx
 ; X64-BMI1BMI2-NEXT:    # kill: def $esi killed $esi def $rsi
-; X64-BMI1BMI2-NEXT:    shrxq %rsi, (%rdi), %rax
+; X64-BMI1BMI2-NEXT:    shrxq %rsi, %rdi, %rax
 ; X64-BMI1BMI2-NEXT:    bzhiq %rdx, %rax, %rax
 ; X64-BMI1BMI2-NEXT:    retq
-  %val = load i64, i64* %w
   %skip = zext i8 %numskipbits to i64
   %shifted = lshr i64 %val, %skip
   %conv = zext i8 %numlowbits to i64
@@ -2638,72 +2823,74 @@ define i64 @bextr64_b3_load_indexzext(i64* %w, i8 zeroext %numskipbits, i8 zeroe
   ret i64 %masked
 }
 
-define i64 @bextr64_b4_commutative(i64 %val, i64 %numskipbits, i64 %numlowbits) nounwind {
-; X86-NOBMI-LABEL: bextr64_b4_commutative:
+define i64 @bextr64_b2_load(i64* %w, i64 %numskipbits, i64 %numlowbits) nounwind {
+; X86-NOBMI-LABEL: bextr64_b2_load:
 ; X86-NOBMI:       # %bb.0:
 ; X86-NOBMI-NEXT:    pushl %edi
 ; X86-NOBMI-NEXT:    pushl %esi
 ; X86-NOBMI-NEXT:    movb {{[0-9]+}}(%esp), %ch
 ; X86-NOBMI-NEXT:    movb {{[0-9]+}}(%esp), %cl
 ; X86-NOBMI-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NOBMI-NEXT:    movl {{[0-9]+}}(%esp), %esi
-; X86-NOBMI-NEXT:    movl %esi, %edx
-; X86-NOBMI-NEXT:    shrl %cl, %edx
-; X86-NOBMI-NEXT:    shrdl %cl, %esi, %eax
+; X86-NOBMI-NEXT:    movl (%eax), %esi
+; X86-NOBMI-NEXT:    movl 4(%eax), %eax
+; X86-NOBMI-NEXT:    movl %eax, %edi
+; X86-NOBMI-NEXT:    shrl %cl, %edi
+; X86-NOBMI-NEXT:    shrdl %cl, %eax, %esi
 ; X86-NOBMI-NEXT:    testb $32, %cl
-; X86-NOBMI-NEXT:    je .LBB24_2
+; X86-NOBMI-NEXT:    je .LBB26_2
 ; X86-NOBMI-NEXT:  # %bb.1:
-; X86-NOBMI-NEXT:    movl %edx, %eax
-; X86-NOBMI-NEXT:    xorl %edx, %edx
-; X86-NOBMI-NEXT:  .LBB24_2:
-; X86-NOBMI-NEXT:    movl $-1, %edi
-; X86-NOBMI-NEXT:    movl $-1, %esi
+; X86-NOBMI-NEXT:    movl %edi, %esi
+; X86-NOBMI-NEXT:    xorl %edi, %edi
+; X86-NOBMI-NEXT:  .LBB26_2:
+; X86-NOBMI-NEXT:    movl $-1, %edx
+; X86-NOBMI-NEXT:    movl $-1, %eax
 ; X86-NOBMI-NEXT:    movb %ch, %cl
-; X86-NOBMI-NEXT:    shll %cl, %esi
-; X86-NOBMI-NEXT:    shldl %cl, %edi, %edi
+; X86-NOBMI-NEXT:    shll %cl, %eax
+; X86-NOBMI-NEXT:    shldl %cl, %edx, %edx
 ; X86-NOBMI-NEXT:    testb $32, %ch
-; X86-NOBMI-NEXT:    je .LBB24_4
+; X86-NOBMI-NEXT:    je .LBB26_4
 ; X86-NOBMI-NEXT:  # %bb.3:
-; X86-NOBMI-NEXT:    movl %esi, %edi
-; X86-NOBMI-NEXT:    xorl %esi, %esi
-; X86-NOBMI-NEXT:  .LBB24_4:
-; X86-NOBMI-NEXT:    notl %edi
+; X86-NOBMI-NEXT:    movl %eax, %edx
+; X86-NOBMI-NEXT:    xorl %eax, %eax
+; X86-NOBMI-NEXT:  .LBB26_4:
+; X86-NOBMI-NEXT:    notl %edx
 ; X86-NOBMI-NEXT:    andl %edi, %edx
-; X86-NOBMI-NEXT:    notl %esi
+; X86-NOBMI-NEXT:    notl %eax
 ; X86-NOBMI-NEXT:    andl %esi, %eax
 ; X86-NOBMI-NEXT:    popl %esi
 ; X86-NOBMI-NEXT:    popl %edi
 ; X86-NOBMI-NEXT:    retl
 ;
-; X86-BMI1NOTBM-LABEL: bextr64_b4_commutative:
+; X86-BMI1NOTBM-LABEL: bextr64_b2_load:
 ; X86-BMI1NOTBM:       # %bb.0:
 ; X86-BMI1NOTBM-NEXT:    pushl %ebx
 ; X86-BMI1NOTBM-NEXT:    pushl %edi
 ; X86-BMI1NOTBM-NEXT:    pushl %esi
 ; X86-BMI1NOTBM-NEXT:    movb {{[0-9]+}}(%esp), %al
 ; X86-BMI1NOTBM-NEXT:    movb {{[0-9]+}}(%esp), %cl
-; X86-BMI1NOTBM-NEXT:    movl {{[0-9]+}}(%esp), %esi
-; X86-BMI1NOTBM-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-BMI1NOTBM-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-BMI1NOTBM-NEXT:    movl (%edx), %esi
+; X86-BMI1NOTBM-NEXT:    movl 4(%edx), %edi
 ; X86-BMI1NOTBM-NEXT:    movl %edi, %edx
 ; X86-BMI1NOTBM-NEXT:    shrl %cl, %edx
 ; X86-BMI1NOTBM-NEXT:    shrdl %cl, %edi, %esi
 ; X86-BMI1NOTBM-NEXT:    testb $32, %cl
-; X86-BMI1NOTBM-NEXT:    je .LBB24_2
+; X86-BMI1NOTBM-NEXT:    je .LBB26_2
 ; X86-BMI1NOTBM-NEXT:  # %bb.1:
 ; X86-BMI1NOTBM-NEXT:    movl %edx, %esi
 ; X86-BMI1NOTBM-NEXT:    xorl %edx, %edx
-; X86-BMI1NOTBM-NEXT:  .LBB24_2:
+; X86-BMI1NOTBM-NEXT:  .LBB26_2:
 ; X86-BMI1NOTBM-NEXT:    movl $-1, %edi
 ; X86-BMI1NOTBM-NEXT:    movl $-1, %ebx
 ; X86-BMI1NOTBM-NEXT:    movl %eax, %ecx
 ; X86-BMI1NOTBM-NEXT:    shll %cl, %ebx
 ; X86-BMI1NOTBM-NEXT:    shldl %cl, %edi, %edi
 ; X86-BMI1NOTBM-NEXT:    testb $32, %al
-; X86-BMI1NOTBM-NEXT:    je .LBB24_4
+; X86-BMI1NOTBM-NEXT:    je .LBB26_4
 ; X86-BMI1NOTBM-NEXT:  # %bb.3:
 ; X86-BMI1NOTBM-NEXT:    movl %ebx, %edi
 ; X86-BMI1NOTBM-NEXT:    xorl %ebx, %ebx
-; X86-BMI1NOTBM-NEXT:  .LBB24_4:
+; X86-BMI1NOTBM-NEXT:  .LBB26_4:
 ; X86-BMI1NOTBM-NEXT:    andnl %edx, %edi, %edx
 ; X86-BMI1NOTBM-NEXT:    andnl %esi, %ebx, %eax
 ; X86-BMI1NOTBM-NEXT:    popl %esi
@@ -2711,33 +2898,34 @@ define i64 @bextr64_b4_commutative(i64 %val, i64 %numskipbits, i64 %numlowbits)
 ; X86-BMI1NOTBM-NEXT:    popl %ebx
 ; X86-BMI1NOTBM-NEXT:    retl
 ;
-; X86-BMI1BMI2-LABEL: bextr64_b4_commutative:
+; X86-BMI1BMI2-LABEL: bextr64_b2_load:
 ; X86-BMI1BMI2:       # %bb.0:
 ; X86-BMI1BMI2-NEXT:    pushl %ebx
 ; X86-BMI1BMI2-NEXT:    pushl %edi
 ; X86-BMI1BMI2-NEXT:    pushl %esi
 ; X86-BMI1BMI2-NEXT:    movb {{[0-9]+}}(%esp), %al
 ; X86-BMI1BMI2-NEXT:    movb {{[0-9]+}}(%esp), %cl
-; X86-BMI1BMI2-NEXT:    movl {{[0-9]+}}(%esp), %esi
 ; X86-BMI1BMI2-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; X86-BMI1BMI2-NEXT:    shrdl %cl, %edx, %esi
-; X86-BMI1BMI2-NEXT:    shrxl %ecx, %edx, %edx
+; X86-BMI1BMI2-NEXT:    movl (%edx), %esi
+; X86-BMI1BMI2-NEXT:    movl 4(%edx), %edi
+; X86-BMI1BMI2-NEXT:    shrxl %ecx, %edi, %edx
+; X86-BMI1BMI2-NEXT:    shrdl %cl, %edi, %esi
 ; X86-BMI1BMI2-NEXT:    testb $32, %cl
-; X86-BMI1BMI2-NEXT:    je .LBB24_2
+; X86-BMI1BMI2-NEXT:    je .LBB26_2
 ; X86-BMI1BMI2-NEXT:  # %bb.1:
 ; X86-BMI1BMI2-NEXT:    movl %edx, %esi
 ; X86-BMI1BMI2-NEXT:    xorl %edx, %edx
-; X86-BMI1BMI2-NEXT:  .LBB24_2:
+; X86-BMI1BMI2-NEXT:  .LBB26_2:
 ; X86-BMI1BMI2-NEXT:    movl $-1, %edi
 ; X86-BMI1BMI2-NEXT:    shlxl %eax, %edi, %ebx
 ; X86-BMI1BMI2-NEXT:    movl %eax, %ecx
 ; X86-BMI1BMI2-NEXT:    shldl %cl, %edi, %edi
 ; X86-BMI1BMI2-NEXT:    testb $32, %al
-; X86-BMI1BMI2-NEXT:    je .LBB24_4
+; X86-BMI1BMI2-NEXT:    je .LBB26_4
 ; X86-BMI1BMI2-NEXT:  # %bb.3:
 ; X86-BMI1BMI2-NEXT:    movl %ebx, %edi
 ; X86-BMI1BMI2-NEXT:    xorl %ebx, %ebx
-; X86-BMI1BMI2-NEXT:  .LBB24_4:
+; X86-BMI1BMI2-NEXT:  .LBB26_4:
 ; X86-BMI1BMI2-NEXT:    andnl %edx, %edi, %edx
 ; X86-BMI1BMI2-NEXT:    andnl %esi, %ebx, %eax
 ; X86-BMI1BMI2-NEXT:    popl %esi
@@ -2745,361 +2933,984 @@ define i64 @bextr64_b4_commutative(i64 %val, i64 %numskipbits, i64 %numlowbits)
 ; X86-BMI1BMI2-NEXT:    popl %ebx
 ; X86-BMI1BMI2-NEXT:    retl
 ;
-; X64-NOBMI-LABEL: bextr64_b4_commutative:
+; X64-NOBMI-LABEL: bextr64_b2_load:
 ; X64-NOBMI:       # %bb.0:
 ; X64-NOBMI-NEXT:    movq %rsi, %rcx
+; X64-NOBMI-NEXT:    movq (%rdi), %rsi
 ; X64-NOBMI-NEXT:    # kill: def $cl killed $cl killed $rcx
-; X64-NOBMI-NEXT:    shrq %cl, %rdi
+; X64-NOBMI-NEXT:    shrq %cl, %rsi
 ; X64-NOBMI-NEXT:    movq $-1, %rax
 ; X64-NOBMI-NEXT:    movl %edx, %ecx
 ; X64-NOBMI-NEXT:    shlq %cl, %rax
 ; X64-NOBMI-NEXT:    notq %rax
-; X64-NOBMI-NEXT:    andq %rdi, %rax
+; X64-NOBMI-NEXT:    andq %rsi, %rax
 ; X64-NOBMI-NEXT:    retq
 ;
-; X64-BMI1NOTBM-LABEL: bextr64_b4_commutative:
+; X64-BMI1NOTBM-LABEL: bextr64_b2_load:
 ; X64-BMI1NOTBM:       # %bb.0:
-; X64-BMI1NOTBM-NEXT:    shlq $8, %rdx
+; X64-BMI1NOTBM-NEXT:    shll $8, %edx
 ; X64-BMI1NOTBM-NEXT:    movzbl %sil, %eax
-; X64-BMI1NOTBM-NEXT:    orq %rdx, %rax
-; X64-BMI1NOTBM-NEXT:    bextrq %rax, %rdi, %rax
+; X64-BMI1NOTBM-NEXT:    orl %edx, %eax
+; X64-BMI1NOTBM-NEXT:    bextrq %rax, (%rdi), %rax
 ; X64-BMI1NOTBM-NEXT:    retq
 ;
-; X64-BMI1BMI2-LABEL: bextr64_b4_commutative:
+; X64-BMI1BMI2-LABEL: bextr64_b2_load:
 ; X64-BMI1BMI2:       # %bb.0:
-; X64-BMI1BMI2-NEXT:    shrxq %rsi, %rdi, %rax
+; X64-BMI1BMI2-NEXT:    shrxq %rsi, (%rdi), %rax
 ; X64-BMI1BMI2-NEXT:    bzhiq %rdx, %rax, %rax
 ; X64-BMI1BMI2-NEXT:    retq
+  %val = load i64, i64* %w
   %shifted = lshr i64 %val, %numskipbits
   %notmask = shl i64 -1, %numlowbits
   %mask = xor i64 %notmask, -1
-  %masked = and i64 %shifted, %mask ; swapped order
+  %masked = and i64 %mask, %shifted
   ret i64 %masked
 }
 
-define i64 @bextr64_b5_skipextrauses(i64 %val, i64 %numskipbits, i64 %numlowbits) nounwind {
-; X86-NOBMI-LABEL: bextr64_b5_skipextrauses:
+define i64 @bextr64_b3_load_indexzext(i64* %w, i8 zeroext %numskipbits, i8 zeroext %numlowbits) nounwind {
+; X86-NOBMI-LABEL: bextr64_b3_load_indexzext:
 ; X86-NOBMI:       # %bb.0:
-; X86-NOBMI-NEXT:    pushl %ebp
-; X86-NOBMI-NEXT:    pushl %ebx
 ; X86-NOBMI-NEXT:    pushl %edi
 ; X86-NOBMI-NEXT:    pushl %esi
-; X86-NOBMI-NEXT:    subl $12, %esp
-; X86-NOBMI-NEXT:    movb {{[0-9]+}}(%esp), %dl
-; X86-NOBMI-NEXT:    movl {{[0-9]+}}(%esp), %ebx
-; X86-NOBMI-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NOBMI-NEXT:    movb {{[0-9]+}}(%esp), %ch
+; X86-NOBMI-NEXT:    movb {{[0-9]+}}(%esp), %cl
 ; X86-NOBMI-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NOBMI-NEXT:    movl %esi, %ebp
-; X86-NOBMI-NEXT:    movl %eax, %ecx
-; X86-NOBMI-NEXT:    shrl %cl, %ebp
-; X86-NOBMI-NEXT:    shrdl %cl, %esi, %ebx
-; X86-NOBMI-NEXT:    testb $32, %al
-; X86-NOBMI-NEXT:    je .LBB25_2
+; X86-NOBMI-NEXT:    movl (%eax), %esi
+; X86-NOBMI-NEXT:    movl 4(%eax), %eax
+; X86-NOBMI-NEXT:    movl %eax, %edi
+; X86-NOBMI-NEXT:    shrl %cl, %edi
+; X86-NOBMI-NEXT:    shrdl %cl, %eax, %esi
+; X86-NOBMI-NEXT:    testb $32, %cl
+; X86-NOBMI-NEXT:    je .LBB27_2
 ; X86-NOBMI-NEXT:  # %bb.1:
-; X86-NOBMI-NEXT:    movl %ebp, %ebx
-; X86-NOBMI-NEXT:    xorl %ebp, %ebp
-; X86-NOBMI-NEXT:  .LBB25_2:
-; X86-NOBMI-NEXT:    movl $-1, %esi
-; X86-NOBMI-NEXT:    movl $-1, %edi
-; X86-NOBMI-NEXT:    movl %edx, %ecx
-; X86-NOBMI-NEXT:    shll %cl, %edi
-; X86-NOBMI-NEXT:    shldl %cl, %esi, %esi
-; X86-NOBMI-NEXT:    testb $32, %dl
-; X86-NOBMI-NEXT:    je .LBB25_4
-; X86-NOBMI-NEXT:  # %bb.3:
 ; X86-NOBMI-NEXT:    movl %edi, %esi
 ; X86-NOBMI-NEXT:    xorl %edi, %edi
-; X86-NOBMI-NEXT:  .LBB25_4:
-; X86-NOBMI-NEXT:    notl %esi
-; X86-NOBMI-NEXT:    andl %ebp, %esi
-; X86-NOBMI-NEXT:    notl %edi
-; X86-NOBMI-NEXT:    andl %ebx, %edi
-; X86-NOBMI-NEXT:    subl $8, %esp
-; X86-NOBMI-NEXT:    pushl {{[0-9]+}}(%esp)
-; X86-NOBMI-NEXT:    pushl %eax
-; X86-NOBMI-NEXT:    calll use64
-; X86-NOBMI-NEXT:    addl $16, %esp
-; X86-NOBMI-NEXT:    movl %edi, %eax
-; X86-NOBMI-NEXT:    movl %esi, %edx
-; X86-NOBMI-NEXT:    addl $12, %esp
+; X86-NOBMI-NEXT:  .LBB27_2:
+; X86-NOBMI-NEXT:    movl $-1, %edx
+; X86-NOBMI-NEXT:    movl $-1, %eax
+; X86-NOBMI-NEXT:    movb %ch, %cl
+; X86-NOBMI-NEXT:    shll %cl, %eax
+; X86-NOBMI-NEXT:    shldl %cl, %edx, %edx
+; X86-NOBMI-NEXT:    testb $32, %ch
+; X86-NOBMI-NEXT:    je .LBB27_4
+; X86-NOBMI-NEXT:  # %bb.3:
+; X86-NOBMI-NEXT:    movl %eax, %edx
+; X86-NOBMI-NEXT:    xorl %eax, %eax
+; X86-NOBMI-NEXT:  .LBB27_4:
+; X86-NOBMI-NEXT:    notl %edx
+; X86-NOBMI-NEXT:    andl %edi, %edx
+; X86-NOBMI-NEXT:    notl %eax
+; X86-NOBMI-NEXT:    andl %esi, %eax
 ; X86-NOBMI-NEXT:    popl %esi
 ; X86-NOBMI-NEXT:    popl %edi
-; X86-NOBMI-NEXT:    popl %ebx
-; X86-NOBMI-NEXT:    popl %ebp
 ; X86-NOBMI-NEXT:    retl
 ;
-; X86-BMI1NOTBM-LABEL: bextr64_b5_skipextrauses:
+; X86-BMI1NOTBM-LABEL: bextr64_b3_load_indexzext:
 ; X86-BMI1NOTBM:       # %bb.0:
-; X86-BMI1NOTBM-NEXT:    pushl %ebp
 ; X86-BMI1NOTBM-NEXT:    pushl %ebx
 ; X86-BMI1NOTBM-NEXT:    pushl %edi
 ; X86-BMI1NOTBM-NEXT:    pushl %esi
-; X86-BMI1NOTBM-NEXT:    subl $12, %esp
-; X86-BMI1NOTBM-NEXT:    movb {{[0-9]+}}(%esp), %dl
-; X86-BMI1NOTBM-NEXT:    movl {{[0-9]+}}(%esp), %edi
-; X86-BMI1NOTBM-NEXT:    movl {{[0-9]+}}(%esp), %ebx
-; X86-BMI1NOTBM-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-BMI1NOTBM-NEXT:    movl %ebx, %esi
-; X86-BMI1NOTBM-NEXT:    movl %eax, %ecx
-; X86-BMI1NOTBM-NEXT:    shrl %cl, %esi
-; X86-BMI1NOTBM-NEXT:    shrdl %cl, %ebx, %edi
-; X86-BMI1NOTBM-NEXT:    testb $32, %al
-; X86-BMI1NOTBM-NEXT:    je .LBB25_2
+; X86-BMI1NOTBM-NEXT:    movb {{[0-9]+}}(%esp), %al
+; X86-BMI1NOTBM-NEXT:    movb {{[0-9]+}}(%esp), %cl
+; X86-BMI1NOTBM-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-BMI1NOTBM-NEXT:    movl (%edx), %esi
+; X86-BMI1NOTBM-NEXT:    movl 4(%edx), %edi
+; X86-BMI1NOTBM-NEXT:    movl %edi, %edx
+; X86-BMI1NOTBM-NEXT:    shrl %cl, %edx
+; X86-BMI1NOTBM-NEXT:    shrdl %cl, %edi, %esi
+; X86-BMI1NOTBM-NEXT:    testb $32, %cl
+; X86-BMI1NOTBM-NEXT:    je .LBB27_2
 ; X86-BMI1NOTBM-NEXT:  # %bb.1:
-; X86-BMI1NOTBM-NEXT:    movl %esi, %edi
-; X86-BMI1NOTBM-NEXT:    xorl %esi, %esi
-; X86-BMI1NOTBM-NEXT:  .LBB25_2:
+; X86-BMI1NOTBM-NEXT:    movl %edx, %esi
+; X86-BMI1NOTBM-NEXT:    xorl %edx, %edx
+; X86-BMI1NOTBM-NEXT:  .LBB27_2:
+; X86-BMI1NOTBM-NEXT:    movl $-1, %edi
 ; X86-BMI1NOTBM-NEXT:    movl $-1, %ebx
-; X86-BMI1NOTBM-NEXT:    movl $-1, %ebp
-; X86-BMI1NOTBM-NEXT:    movl %edx, %ecx
-; X86-BMI1NOTBM-NEXT:    shll %cl, %ebp
-; X86-BMI1NOTBM-NEXT:    shldl %cl, %ebx, %ebx
-; X86-BMI1NOTBM-NEXT:    testb $32, %dl
-; X86-BMI1NOTBM-NEXT:    je .LBB25_4
+; X86-BMI1NOTBM-NEXT:    movl %eax, %ecx
+; X86-BMI1NOTBM-NEXT:    shll %cl, %ebx
+; X86-BMI1NOTBM-NEXT:    shldl %cl, %edi, %edi
+; X86-BMI1NOTBM-NEXT:    testb $32, %al
+; X86-BMI1NOTBM-NEXT:    je .LBB27_4
 ; X86-BMI1NOTBM-NEXT:  # %bb.3:
-; X86-BMI1NOTBM-NEXT:    movl %ebp, %ebx
-; X86-BMI1NOTBM-NEXT:    xorl %ebp, %ebp
-; X86-BMI1NOTBM-NEXT:  .LBB25_4:
-; X86-BMI1NOTBM-NEXT:    andnl %esi, %ebx, %esi
-; X86-BMI1NOTBM-NEXT:    andnl %edi, %ebp, %edi
-; X86-BMI1NOTBM-NEXT:    subl $8, %esp
-; X86-BMI1NOTBM-NEXT:    pushl {{[0-9]+}}(%esp)
-; X86-BMI1NOTBM-NEXT:    pushl %eax
-; X86-BMI1NOTBM-NEXT:    calll use64
-; X86-BMI1NOTBM-NEXT:    addl $16, %esp
-; X86-BMI1NOTBM-NEXT:    movl %edi, %eax
-; X86-BMI1NOTBM-NEXT:    movl %esi, %edx
-; X86-BMI1NOTBM-NEXT:    addl $12, %esp
+; X86-BMI1NOTBM-NEXT:    movl %ebx, %edi
+; X86-BMI1NOTBM-NEXT:    xorl %ebx, %ebx
+; X86-BMI1NOTBM-NEXT:  .LBB27_4:
+; X86-BMI1NOTBM-NEXT:    andnl %edx, %edi, %edx
+; X86-BMI1NOTBM-NEXT:    andnl %esi, %ebx, %eax
 ; X86-BMI1NOTBM-NEXT:    popl %esi
 ; X86-BMI1NOTBM-NEXT:    popl %edi
 ; X86-BMI1NOTBM-NEXT:    popl %ebx
-; X86-BMI1NOTBM-NEXT:    popl %ebp
 ; X86-BMI1NOTBM-NEXT:    retl
 ;
-; X86-BMI1BMI2-LABEL: bextr64_b5_skipextrauses:
+; X86-BMI1BMI2-LABEL: bextr64_b3_load_indexzext:
 ; X86-BMI1BMI2:       # %bb.0:
-; X86-BMI1BMI2-NEXT:    pushl %ebp
 ; X86-BMI1BMI2-NEXT:    pushl %ebx
 ; X86-BMI1BMI2-NEXT:    pushl %edi
 ; X86-BMI1BMI2-NEXT:    pushl %esi
-; X86-BMI1BMI2-NEXT:    subl $12, %esp
-; X86-BMI1BMI2-NEXT:    movb {{[0-9]+}}(%esp), %dl
-; X86-BMI1BMI2-NEXT:    movl {{[0-9]+}}(%esp), %edi
-; X86-BMI1BMI2-NEXT:    movl {{[0-9]+}}(%esp), %esi
-; X86-BMI1BMI2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-BMI1BMI2-NEXT:    movb {{[0-9]+}}(%esp), %al
+; X86-BMI1BMI2-NEXT:    movb {{[0-9]+}}(%esp), %cl
+; X86-BMI1BMI2-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-BMI1BMI2-NEXT:    movl (%edx), %esi
+; X86-BMI1BMI2-NEXT:    movl 4(%edx), %edi
+; X86-BMI1BMI2-NEXT:    shrxl %ecx, %edi, %edx
+; X86-BMI1BMI2-NEXT:    shrdl %cl, %edi, %esi
+; X86-BMI1BMI2-NEXT:    testb $32, %cl
+; X86-BMI1BMI2-NEXT:    je .LBB27_2
+; X86-BMI1BMI2-NEXT:  # %bb.1:
+; X86-BMI1BMI2-NEXT:    movl %edx, %esi
+; X86-BMI1BMI2-NEXT:    xorl %edx, %edx
+; X86-BMI1BMI2-NEXT:  .LBB27_2:
+; X86-BMI1BMI2-NEXT:    movl $-1, %edi
+; X86-BMI1BMI2-NEXT:    shlxl %eax, %edi, %ebx
 ; X86-BMI1BMI2-NEXT:    movl %eax, %ecx
-; X86-BMI1BMI2-NEXT:    shrdl %cl, %esi, %edi
-; X86-BMI1BMI2-NEXT:    shrxl %eax, %esi, %esi
+; X86-BMI1BMI2-NEXT:    shldl %cl, %edi, %edi
 ; X86-BMI1BMI2-NEXT:    testb $32, %al
-; X86-BMI1BMI2-NEXT:    je .LBB25_2
-; X86-BMI1BMI2-NEXT:  # %bb.1:
-; X86-BMI1BMI2-NEXT:    movl %esi, %edi
-; X86-BMI1BMI2-NEXT:    xorl %esi, %esi
-; X86-BMI1BMI2-NEXT:  .LBB25_2:
-; X86-BMI1BMI2-NEXT:    movl $-1, %ebp
-; X86-BMI1BMI2-NEXT:    shlxl %edx, %ebp, %ebx
-; X86-BMI1BMI2-NEXT:    movl %edx, %ecx
-; X86-BMI1BMI2-NEXT:    shldl %cl, %ebp, %ebp
-; X86-BMI1BMI2-NEXT:    testb $32, %dl
-; X86-BMI1BMI2-NEXT:    je .LBB25_4
+; X86-BMI1BMI2-NEXT:    je .LBB27_4
 ; X86-BMI1BMI2-NEXT:  # %bb.3:
-; X86-BMI1BMI2-NEXT:    movl %ebx, %ebp
+; X86-BMI1BMI2-NEXT:    movl %ebx, %edi
 ; X86-BMI1BMI2-NEXT:    xorl %ebx, %ebx
-; X86-BMI1BMI2-NEXT:  .LBB25_4:
-; X86-BMI1BMI2-NEXT:    andnl %esi, %ebp, %esi
-; X86-BMI1BMI2-NEXT:    andnl %edi, %ebx, %edi
-; X86-BMI1BMI2-NEXT:    subl $8, %esp
-; X86-BMI1BMI2-NEXT:    pushl {{[0-9]+}}(%esp)
-; X86-BMI1BMI2-NEXT:    pushl %eax
-; X86-BMI1BMI2-NEXT:    calll use64
-; X86-BMI1BMI2-NEXT:    addl $16, %esp
-; X86-BMI1BMI2-NEXT:    movl %edi, %eax
-; X86-BMI1BMI2-NEXT:    movl %esi, %edx
-; X86-BMI1BMI2-NEXT:    addl $12, %esp
+; X86-BMI1BMI2-NEXT:  .LBB27_4:
+; X86-BMI1BMI2-NEXT:    andnl %edx, %edi, %edx
+; X86-BMI1BMI2-NEXT:    andnl %esi, %ebx, %eax
 ; X86-BMI1BMI2-NEXT:    popl %esi
 ; X86-BMI1BMI2-NEXT:    popl %edi
 ; X86-BMI1BMI2-NEXT:    popl %ebx
-; X86-BMI1BMI2-NEXT:    popl %ebp
 ; X86-BMI1BMI2-NEXT:    retl
 ;
-; X64-NOBMI-LABEL: bextr64_b5_skipextrauses:
+; X64-NOBMI-LABEL: bextr64_b3_load_indexzext:
 ; X64-NOBMI:       # %bb.0:
-; X64-NOBMI-NEXT:    pushq %rbx
 ; X64-NOBMI-NEXT:    movl %esi, %ecx
-; X64-NOBMI-NEXT:    shrq %cl, %rdi
-; X64-NOBMI-NEXT:    movq $-1, %rbx
+; X64-NOBMI-NEXT:    movq (%rdi), %rsi
+; X64-NOBMI-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X64-NOBMI-NEXT:    shrq %cl, %rsi
+; X64-NOBMI-NEXT:    movq $-1, %rax
 ; X64-NOBMI-NEXT:    movl %edx, %ecx
-; X64-NOBMI-NEXT:    shlq %cl, %rbx
-; X64-NOBMI-NEXT:    notq %rbx
-; X64-NOBMI-NEXT:    andq %rdi, %rbx
-; X64-NOBMI-NEXT:    movq %rsi, %rdi
-; X64-NOBMI-NEXT:    callq use64
-; X64-NOBMI-NEXT:    movq %rbx, %rax
-; X64-NOBMI-NEXT:    popq %rbx
+; X64-NOBMI-NEXT:    shlq %cl, %rax
+; X64-NOBMI-NEXT:    notq %rax
+; X64-NOBMI-NEXT:    andq %rsi, %rax
 ; X64-NOBMI-NEXT:    retq
 ;
-; X64-BMI1NOTBM-LABEL: bextr64_b5_skipextrauses:
+; X64-BMI1NOTBM-LABEL: bextr64_b3_load_indexzext:
 ; X64-BMI1NOTBM:       # %bb.0:
-; X64-BMI1NOTBM-NEXT:    pushq %rbx
-; X64-BMI1NOTBM-NEXT:    shlq $8, %rdx
+; X64-BMI1NOTBM-NEXT:    shll $8, %edx
 ; X64-BMI1NOTBM-NEXT:    movzbl %sil, %eax
-; X64-BMI1NOTBM-NEXT:    orq %rdx, %rax
-; X64-BMI1NOTBM-NEXT:    bextrq %rax, %rdi, %rbx
-; X64-BMI1NOTBM-NEXT:    movq %rsi, %rdi
-; X64-BMI1NOTBM-NEXT:    callq use64
-; X64-BMI1NOTBM-NEXT:    movq %rbx, %rax
-; X64-BMI1NOTBM-NEXT:    popq %rbx
+; X64-BMI1NOTBM-NEXT:    orl %edx, %eax
+; X64-BMI1NOTBM-NEXT:    bextrq %rax, (%rdi), %rax
 ; X64-BMI1NOTBM-NEXT:    retq
 ;
-; X64-BMI1BMI2-LABEL: bextr64_b5_skipextrauses:
+; X64-BMI1BMI2-LABEL: bextr64_b3_load_indexzext:
 ; X64-BMI1BMI2:       # %bb.0:
-; X64-BMI1BMI2-NEXT:    pushq %rbx
-; X64-BMI1BMI2-NEXT:    shrxq %rsi, %rdi, %rax
-; X64-BMI1BMI2-NEXT:    bzhiq %rdx, %rax, %rbx
-; X64-BMI1BMI2-NEXT:    movq %rsi, %rdi
-; X64-BMI1BMI2-NEXT:    callq use64
-; X64-BMI1BMI2-NEXT:    movq %rbx, %rax
-; X64-BMI1BMI2-NEXT:    popq %rbx
+; X64-BMI1BMI2-NEXT:    # kill: def $edx killed $edx def $rdx
+; X64-BMI1BMI2-NEXT:    # kill: def $esi killed $esi def $rsi
+; X64-BMI1BMI2-NEXT:    shrxq %rsi, (%rdi), %rax
+; X64-BMI1BMI2-NEXT:    bzhiq %rdx, %rax, %rax
 ; X64-BMI1BMI2-NEXT:    retq
-  %shifted = lshr i64 %val, %numskipbits
-  %notmask = shl i64 -1, %numlowbits
+  %val = load i64, i64* %w
+  %skip = zext i8 %numskipbits to i64
+  %shifted = lshr i64 %val, %skip
+  %conv = zext i8 %numlowbits to i64
+  %notmask = shl i64 -1, %conv
   %mask = xor i64 %notmask, -1
   %masked = and i64 %mask, %shifted
-  call void @use64(i64 %numskipbits)
   ret i64 %masked
 }
 
-; ---------------------------------------------------------------------------- ;
-; Pattern c. 32-bit
-; ---------------------------------------------------------------------------- ;
-
-define i32 @bextr32_c0(i32 %val, i32 %numskipbits, i32 %numlowbits) nounwind {
-; X86-NOBMI-LABEL: bextr32_c0:
+define i64 @bextr64_b4_commutative(i64 %val, i64 %numskipbits, i64 %numlowbits) nounwind {
+; X86-NOBMI-LABEL: bextr64_b4_commutative:
 ; X86-NOBMI:       # %bb.0:
 ; X86-NOBMI-NEXT:    pushl %edi
 ; X86-NOBMI-NEXT:    pushl %esi
-; X86-NOBMI-NEXT:    pushl %eax
+; X86-NOBMI-NEXT:    movb {{[0-9]+}}(%esp), %ch
 ; X86-NOBMI-NEXT:    movb {{[0-9]+}}(%esp), %cl
-; X86-NOBMI-NEXT:    movl {{[0-9]+}}(%esp), %edi
-; X86-NOBMI-NEXT:    shrl %cl, %edi
-; X86-NOBMI-NEXT:    xorl %ecx, %ecx
-; X86-NOBMI-NEXT:    subb {{[0-9]+}}(%esp), %cl
+; X86-NOBMI-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NOBMI-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NOBMI-NEXT:    movl %esi, %edx
+; X86-NOBMI-NEXT:    shrl %cl, %edx
+; X86-NOBMI-NEXT:    shrdl %cl, %esi, %eax
+; X86-NOBMI-NEXT:    testb $32, %cl
+; X86-NOBMI-NEXT:    je .LBB28_2
+; X86-NOBMI-NEXT:  # %bb.1:
+; X86-NOBMI-NEXT:    movl %edx, %eax
+; X86-NOBMI-NEXT:    xorl %edx, %edx
+; X86-NOBMI-NEXT:  .LBB28_2:
+; X86-NOBMI-NEXT:    movl $-1, %edi
 ; X86-NOBMI-NEXT:    movl $-1, %esi
-; X86-NOBMI-NEXT:    # kill: def $cl killed $cl killed $ecx
-; X86-NOBMI-NEXT:    shrl %cl, %esi
-; X86-NOBMI-NEXT:    movl %esi, (%esp)
-; X86-NOBMI-NEXT:    calll use32
-; X86-NOBMI-NEXT:    andl %edi, %esi
-; X86-NOBMI-NEXT:    movl %esi, %eax
-; X86-NOBMI-NEXT:    addl $4, %esp
+; X86-NOBMI-NEXT:    movb %ch, %cl
+; X86-NOBMI-NEXT:    shll %cl, %esi
+; X86-NOBMI-NEXT:    shldl %cl, %edi, %edi
+; X86-NOBMI-NEXT:    testb $32, %ch
+; X86-NOBMI-NEXT:    je .LBB28_4
+; X86-NOBMI-NEXT:  # %bb.3:
+; X86-NOBMI-NEXT:    movl %esi, %edi
+; X86-NOBMI-NEXT:    xorl %esi, %esi
+; X86-NOBMI-NEXT:  .LBB28_4:
+; X86-NOBMI-NEXT:    notl %edi
+; X86-NOBMI-NEXT:    andl %edi, %edx
+; X86-NOBMI-NEXT:    notl %esi
+; X86-NOBMI-NEXT:    andl %esi, %eax
 ; X86-NOBMI-NEXT:    popl %esi
 ; X86-NOBMI-NEXT:    popl %edi
 ; X86-NOBMI-NEXT:    retl
 ;
-; X86-BMI1NOTBM-LABEL: bextr32_c0:
+; X86-BMI1NOTBM-LABEL: bextr64_b4_commutative:
 ; X86-BMI1NOTBM:       # %bb.0:
+; X86-BMI1NOTBM-NEXT:    pushl %ebx
 ; X86-BMI1NOTBM-NEXT:    pushl %edi
 ; X86-BMI1NOTBM-NEXT:    pushl %esi
-; X86-BMI1NOTBM-NEXT:    pushl %eax
+; X86-BMI1NOTBM-NEXT:    movb {{[0-9]+}}(%esp), %al
 ; X86-BMI1NOTBM-NEXT:    movb {{[0-9]+}}(%esp), %cl
+; X86-BMI1NOTBM-NEXT:    movl {{[0-9]+}}(%esp), %esi
 ; X86-BMI1NOTBM-NEXT:    movl {{[0-9]+}}(%esp), %edi
-; X86-BMI1NOTBM-NEXT:    shrl %cl, %edi
-; X86-BMI1NOTBM-NEXT:    xorl %ecx, %ecx
-; X86-BMI1NOTBM-NEXT:    subb {{[0-9]+}}(%esp), %cl
-; X86-BMI1NOTBM-NEXT:    movl $-1, %esi
-; X86-BMI1NOTBM-NEXT:    # kill: def $cl killed $cl killed $ecx
-; X86-BMI1NOTBM-NEXT:    shrl %cl, %esi
-; X86-BMI1NOTBM-NEXT:    movl %esi, (%esp)
-; X86-BMI1NOTBM-NEXT:    calll use32
-; X86-BMI1NOTBM-NEXT:    andl %edi, %esi
-; X86-BMI1NOTBM-NEXT:    movl %esi, %eax
-; X86-BMI1NOTBM-NEXT:    addl $4, %esp
+; X86-BMI1NOTBM-NEXT:    movl %edi, %edx
+; X86-BMI1NOTBM-NEXT:    shrl %cl, %edx
+; X86-BMI1NOTBM-NEXT:    shrdl %cl, %edi, %esi
+; X86-BMI1NOTBM-NEXT:    testb $32, %cl
+; X86-BMI1NOTBM-NEXT:    je .LBB28_2
+; X86-BMI1NOTBM-NEXT:  # %bb.1:
+; X86-BMI1NOTBM-NEXT:    movl %edx, %esi
+; X86-BMI1NOTBM-NEXT:    xorl %edx, %edx
+; X86-BMI1NOTBM-NEXT:  .LBB28_2:
+; X86-BMI1NOTBM-NEXT:    movl $-1, %edi
+; X86-BMI1NOTBM-NEXT:    movl $-1, %ebx
+; X86-BMI1NOTBM-NEXT:    movl %eax, %ecx
+; X86-BMI1NOTBM-NEXT:    shll %cl, %ebx
+; X86-BMI1NOTBM-NEXT:    shldl %cl, %edi, %edi
+; X86-BMI1NOTBM-NEXT:    testb $32, %al
+; X86-BMI1NOTBM-NEXT:    je .LBB28_4
+; X86-BMI1NOTBM-NEXT:  # %bb.3:
+; X86-BMI1NOTBM-NEXT:    movl %ebx, %edi
+; X86-BMI1NOTBM-NEXT:    xorl %ebx, %ebx
+; X86-BMI1NOTBM-NEXT:  .LBB28_4:
+; X86-BMI1NOTBM-NEXT:    andnl %edx, %edi, %edx
+; X86-BMI1NOTBM-NEXT:    andnl %esi, %ebx, %eax
 ; X86-BMI1NOTBM-NEXT:    popl %esi
 ; X86-BMI1NOTBM-NEXT:    popl %edi
+; X86-BMI1NOTBM-NEXT:    popl %ebx
 ; X86-BMI1NOTBM-NEXT:    retl
 ;
-; X86-BMI1BMI2-LABEL: bextr32_c0:
+; X86-BMI1BMI2-LABEL: bextr64_b4_commutative:
 ; X86-BMI1BMI2:       # %bb.0:
 ; X86-BMI1BMI2-NEXT:    pushl %ebx
+; X86-BMI1BMI2-NEXT:    pushl %edi
 ; X86-BMI1BMI2-NEXT:    pushl %esi
-; X86-BMI1BMI2-NEXT:    pushl %eax
-; X86-BMI1BMI2-NEXT:    movb {{[0-9]+}}(%esp), %bl
 ; X86-BMI1BMI2-NEXT:    movb {{[0-9]+}}(%esp), %al
-; X86-BMI1BMI2-NEXT:    shrxl %eax, {{[0-9]+}}(%esp), %esi
-; X86-BMI1BMI2-NEXT:    movl %ebx, %eax
-; X86-BMI1BMI2-NEXT:    negb %al
-; X86-BMI1BMI2-NEXT:    movl $-1, %ecx
-; X86-BMI1BMI2-NEXT:    shrxl %eax, %ecx, %eax
-; X86-BMI1BMI2-NEXT:    movl %eax, (%esp)
-; X86-BMI1BMI2-NEXT:    calll use32
-; X86-BMI1BMI2-NEXT:    bzhil %ebx, %esi, %eax
-; X86-BMI1BMI2-NEXT:    addl $4, %esp
+; X86-BMI1BMI2-NEXT:    movb {{[0-9]+}}(%esp), %cl
+; X86-BMI1BMI2-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-BMI1BMI2-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-BMI1BMI2-NEXT:    shrdl %cl, %edx, %esi
+; X86-BMI1BMI2-NEXT:    shrxl %ecx, %edx, %edx
+; X86-BMI1BMI2-NEXT:    testb $32, %cl
+; X86-BMI1BMI2-NEXT:    je .LBB28_2
+; X86-BMI1BMI2-NEXT:  # %bb.1:
+; X86-BMI1BMI2-NEXT:    movl %edx, %esi
+; X86-BMI1BMI2-NEXT:    xorl %edx, %edx
+; X86-BMI1BMI2-NEXT:  .LBB28_2:
+; X86-BMI1BMI2-NEXT:    movl $-1, %edi
+; X86-BMI1BMI2-NEXT:    shlxl %eax, %edi, %ebx
+; X86-BMI1BMI2-NEXT:    movl %eax, %ecx
+; X86-BMI1BMI2-NEXT:    shldl %cl, %edi, %edi
+; X86-BMI1BMI2-NEXT:    testb $32, %al
+; X86-BMI1BMI2-NEXT:    je .LBB28_4
+; X86-BMI1BMI2-NEXT:  # %bb.3:
+; X86-BMI1BMI2-NEXT:    movl %ebx, %edi
+; X86-BMI1BMI2-NEXT:    xorl %ebx, %ebx
+; X86-BMI1BMI2-NEXT:  .LBB28_4:
+; X86-BMI1BMI2-NEXT:    andnl %edx, %edi, %edx
+; X86-BMI1BMI2-NEXT:    andnl %esi, %ebx, %eax
 ; X86-BMI1BMI2-NEXT:    popl %esi
+; X86-BMI1BMI2-NEXT:    popl %edi
 ; X86-BMI1BMI2-NEXT:    popl %ebx
 ; X86-BMI1BMI2-NEXT:    retl
 ;
-; X64-NOBMI-LABEL: bextr32_c0:
+; X64-NOBMI-LABEL: bextr64_b4_commutative:
 ; X64-NOBMI:       # %bb.0:
-; X64-NOBMI-NEXT:    pushq %rbp
-; X64-NOBMI-NEXT:    pushq %rbx
-; X64-NOBMI-NEXT:    pushq %rax
-; X64-NOBMI-NEXT:    movl %esi, %ecx
-; X64-NOBMI-NEXT:    movl %edi, %ebx
-; X64-NOBMI-NEXT:    # kill: def $cl killed $cl killed $ecx
-; X64-NOBMI-NEXT:    shrl %cl, %ebx
-; X64-NOBMI-NEXT:    negb %dl
-; X64-NOBMI-NEXT:    movl $-1, %ebp
+; X64-NOBMI-NEXT:    movq %rsi, %rcx
+; X64-NOBMI-NEXT:    # kill: def $cl killed $cl killed $rcx
+; X64-NOBMI-NEXT:    shrq %cl, %rdi
+; X64-NOBMI-NEXT:    movq $-1, %rax
 ; X64-NOBMI-NEXT:    movl %edx, %ecx
-; X64-NOBMI-NEXT:    shrl %cl, %ebp
-; X64-NOBMI-NEXT:    movl %ebp, %edi
-; X64-NOBMI-NEXT:    callq use32
-; X64-NOBMI-NEXT:    andl %ebx, %ebp
-; X64-NOBMI-NEXT:    movl %ebp, %eax
-; X64-NOBMI-NEXT:    addq $8, %rsp
-; X64-NOBMI-NEXT:    popq %rbx
-; X64-NOBMI-NEXT:    popq %rbp
+; X64-NOBMI-NEXT:    shlq %cl, %rax
+; X64-NOBMI-NEXT:    notq %rax
+; X64-NOBMI-NEXT:    andq %rdi, %rax
 ; X64-NOBMI-NEXT:    retq
 ;
-; X64-BMI1NOTBM-LABEL: bextr32_c0:
+; X64-BMI1NOTBM-LABEL: bextr64_b4_commutative:
 ; X64-BMI1NOTBM:       # %bb.0:
-; X64-BMI1NOTBM-NEXT:    pushq %rbp
-; X64-BMI1NOTBM-NEXT:    pushq %rbx
-; X64-BMI1NOTBM-NEXT:    pushq %rax
-; X64-BMI1NOTBM-NEXT:    movl %esi, %ecx
-; X64-BMI1NOTBM-NEXT:    movl %edi, %ebx
-; X64-BMI1NOTBM-NEXT:    # kill: def $cl killed $cl killed $ecx
-; X64-BMI1NOTBM-NEXT:    shrl %cl, %ebx
-; X64-BMI1NOTBM-NEXT:    negb %dl
-; X64-BMI1NOTBM-NEXT:    movl $-1, %ebp
-; X64-BMI1NOTBM-NEXT:    movl %edx, %ecx
-; X64-BMI1NOTBM-NEXT:    shrl %cl, %ebp
-; X64-BMI1NOTBM-NEXT:    movl %ebp, %edi
-; X64-BMI1NOTBM-NEXT:    callq use32
-; X64-BMI1NOTBM-NEXT:    andl %ebx, %ebp
-; X64-BMI1NOTBM-NEXT:    movl %ebp, %eax
-; X64-BMI1NOTBM-NEXT:    addq $8, %rsp
-; X64-BMI1NOTBM-NEXT:    popq %rbx
-; X64-BMI1NOTBM-NEXT:    popq %rbp
+; X64-BMI1NOTBM-NEXT:    shll $8, %edx
+; X64-BMI1NOTBM-NEXT:    movzbl %sil, %eax
+; X64-BMI1NOTBM-NEXT:    orl %edx, %eax
+; X64-BMI1NOTBM-NEXT:    bextrq %rax, %rdi, %rax
 ; X64-BMI1NOTBM-NEXT:    retq
 ;
-; X64-BMI1BMI2-LABEL: bextr32_c0:
+; X64-BMI1BMI2-LABEL: bextr64_b4_commutative:
 ; X64-BMI1BMI2:       # %bb.0:
-; X64-BMI1BMI2-NEXT:    pushq %rbp
-; X64-BMI1BMI2-NEXT:    pushq %rbx
-; X64-BMI1BMI2-NEXT:    pushq %rax
-; X64-BMI1BMI2-NEXT:    movl %edx, %ebx
-; X64-BMI1BMI2-NEXT:    shrxl %esi, %edi, %ebp
-; X64-BMI1BMI2-NEXT:    movl %ebx, %eax
-; X64-BMI1BMI2-NEXT:    negb %al
-; X64-BMI1BMI2-NEXT:    movl $-1, %ecx
-; X64-BMI1BMI2-NEXT:    shrxl %eax, %ecx, %edi
-; X64-BMI1BMI2-NEXT:    callq use32
-; X64-BMI1BMI2-NEXT:    bzhil %ebx, %ebp, %eax
-; X64-BMI1BMI2-NEXT:    addq $8, %rsp
-; X64-BMI1BMI2-NEXT:    popq %rbx
-; X64-BMI1BMI2-NEXT:    popq %rbp
+; X64-BMI1BMI2-NEXT:    shrxq %rsi, %rdi, %rax
+; X64-BMI1BMI2-NEXT:    bzhiq %rdx, %rax, %rax
 ; X64-BMI1BMI2-NEXT:    retq
-  %shifted = lshr i32 %val, %numskipbits
+  %shifted = lshr i64 %val, %numskipbits
+  %notmask = shl i64 -1, %numlowbits
+  %mask = xor i64 %notmask, -1
+  %masked = and i64 %shifted, %mask ; swapped order
+  ret i64 %masked
+}
+
+define i64 @bextr64_b5_skipextrauses(i64 %val, i64 %numskipbits, i64 %numlowbits) nounwind {
+; X86-NOBMI-LABEL: bextr64_b5_skipextrauses:
+; X86-NOBMI:       # %bb.0:
+; X86-NOBMI-NEXT:    pushl %ebp
+; X86-NOBMI-NEXT:    pushl %ebx
+; X86-NOBMI-NEXT:    pushl %edi
+; X86-NOBMI-NEXT:    pushl %esi
+; X86-NOBMI-NEXT:    subl $12, %esp
+; X86-NOBMI-NEXT:    movb {{[0-9]+}}(%esp), %dl
+; X86-NOBMI-NEXT:    movl {{[0-9]+}}(%esp), %ebx
+; X86-NOBMI-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NOBMI-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NOBMI-NEXT:    movl %esi, %ebp
+; X86-NOBMI-NEXT:    movl %eax, %ecx
+; X86-NOBMI-NEXT:    shrl %cl, %ebp
+; X86-NOBMI-NEXT:    shrdl %cl, %esi, %ebx
+; X86-NOBMI-NEXT:    testb $32, %al
+; X86-NOBMI-NEXT:    je .LBB29_2
+; X86-NOBMI-NEXT:  # %bb.1:
+; X86-NOBMI-NEXT:    movl %ebp, %ebx
+; X86-NOBMI-NEXT:    xorl %ebp, %ebp
+; X86-NOBMI-NEXT:  .LBB29_2:
+; X86-NOBMI-NEXT:    movl $-1, %esi
+; X86-NOBMI-NEXT:    movl $-1, %edi
+; X86-NOBMI-NEXT:    movl %edx, %ecx
+; X86-NOBMI-NEXT:    shll %cl, %edi
+; X86-NOBMI-NEXT:    shldl %cl, %esi, %esi
+; X86-NOBMI-NEXT:    testb $32, %dl
+; X86-NOBMI-NEXT:    je .LBB29_4
+; X86-NOBMI-NEXT:  # %bb.3:
+; X86-NOBMI-NEXT:    movl %edi, %esi
+; X86-NOBMI-NEXT:    xorl %edi, %edi
+; X86-NOBMI-NEXT:  .LBB29_4:
+; X86-NOBMI-NEXT:    notl %esi
+; X86-NOBMI-NEXT:    andl %ebp, %esi
+; X86-NOBMI-NEXT:    notl %edi
+; X86-NOBMI-NEXT:    andl %ebx, %edi
+; X86-NOBMI-NEXT:    subl $8, %esp
+; X86-NOBMI-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NOBMI-NEXT:    pushl %eax
+; X86-NOBMI-NEXT:    calll use64
+; X86-NOBMI-NEXT:    addl $16, %esp
+; X86-NOBMI-NEXT:    movl %edi, %eax
+; X86-NOBMI-NEXT:    movl %esi, %edx
+; X86-NOBMI-NEXT:    addl $12, %esp
+; X86-NOBMI-NEXT:    popl %esi
+; X86-NOBMI-NEXT:    popl %edi
+; X86-NOBMI-NEXT:    popl %ebx
+; X86-NOBMI-NEXT:    popl %ebp
+; X86-NOBMI-NEXT:    retl
+;
+; X86-BMI1NOTBM-LABEL: bextr64_b5_skipextrauses:
+; X86-BMI1NOTBM:       # %bb.0:
+; X86-BMI1NOTBM-NEXT:    pushl %ebp
+; X86-BMI1NOTBM-NEXT:    pushl %ebx
+; X86-BMI1NOTBM-NEXT:    pushl %edi
+; X86-BMI1NOTBM-NEXT:    pushl %esi
+; X86-BMI1NOTBM-NEXT:    subl $12, %esp
+; X86-BMI1NOTBM-NEXT:    movb {{[0-9]+}}(%esp), %dl
+; X86-BMI1NOTBM-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-BMI1NOTBM-NEXT:    movl {{[0-9]+}}(%esp), %ebx
+; X86-BMI1NOTBM-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-BMI1NOTBM-NEXT:    movl %ebx, %esi
+; X86-BMI1NOTBM-NEXT:    movl %eax, %ecx
+; X86-BMI1NOTBM-NEXT:    shrl %cl, %esi
+; X86-BMI1NOTBM-NEXT:    shrdl %cl, %ebx, %edi
+; X86-BMI1NOTBM-NEXT:    testb $32, %al
+; X86-BMI1NOTBM-NEXT:    je .LBB29_2
+; X86-BMI1NOTBM-NEXT:  # %bb.1:
+; X86-BMI1NOTBM-NEXT:    movl %esi, %edi
+; X86-BMI1NOTBM-NEXT:    xorl %esi, %esi
+; X86-BMI1NOTBM-NEXT:  .LBB29_2:
+; X86-BMI1NOTBM-NEXT:    movl $-1, %ebx
+; X86-BMI1NOTBM-NEXT:    movl $-1, %ebp
+; X86-BMI1NOTBM-NEXT:    movl %edx, %ecx
+; X86-BMI1NOTBM-NEXT:    shll %cl, %ebp
+; X86-BMI1NOTBM-NEXT:    shldl %cl, %ebx, %ebx
+; X86-BMI1NOTBM-NEXT:    testb $32, %dl
+; X86-BMI1NOTBM-NEXT:    je .LBB29_4
+; X86-BMI1NOTBM-NEXT:  # %bb.3:
+; X86-BMI1NOTBM-NEXT:    movl %ebp, %ebx
+; X86-BMI1NOTBM-NEXT:    xorl %ebp, %ebp
+; X86-BMI1NOTBM-NEXT:  .LBB29_4:
+; X86-BMI1NOTBM-NEXT:    andnl %esi, %ebx, %esi
+; X86-BMI1NOTBM-NEXT:    andnl %edi, %ebp, %edi
+; X86-BMI1NOTBM-NEXT:    subl $8, %esp
+; X86-BMI1NOTBM-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-BMI1NOTBM-NEXT:    pushl %eax
+; X86-BMI1NOTBM-NEXT:    calll use64
+; X86-BMI1NOTBM-NEXT:    addl $16, %esp
+; X86-BMI1NOTBM-NEXT:    movl %edi, %eax
+; X86-BMI1NOTBM-NEXT:    movl %esi, %edx
+; X86-BMI1NOTBM-NEXT:    addl $12, %esp
+; X86-BMI1NOTBM-NEXT:    popl %esi
+; X86-BMI1NOTBM-NEXT:    popl %edi
+; X86-BMI1NOTBM-NEXT:    popl %ebx
+; X86-BMI1NOTBM-NEXT:    popl %ebp
+; X86-BMI1NOTBM-NEXT:    retl
+;
+; X86-BMI1BMI2-LABEL: bextr64_b5_skipextrauses:
+; X86-BMI1BMI2:       # %bb.0:
+; X86-BMI1BMI2-NEXT:    pushl %ebp
+; X86-BMI1BMI2-NEXT:    pushl %ebx
+; X86-BMI1BMI2-NEXT:    pushl %edi
+; X86-BMI1BMI2-NEXT:    pushl %esi
+; X86-BMI1BMI2-NEXT:    subl $12, %esp
+; X86-BMI1BMI2-NEXT:    movb {{[0-9]+}}(%esp), %dl
+; X86-BMI1BMI2-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-BMI1BMI2-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-BMI1BMI2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-BMI1BMI2-NEXT:    movl %eax, %ecx
+; X86-BMI1BMI2-NEXT:    shrdl %cl, %esi, %edi
+; X86-BMI1BMI2-NEXT:    shrxl %eax, %esi, %esi
+; X86-BMI1BMI2-NEXT:    testb $32, %al
+; X86-BMI1BMI2-NEXT:    je .LBB29_2
+; X86-BMI1BMI2-NEXT:  # %bb.1:
+; X86-BMI1BMI2-NEXT:    movl %esi, %edi
+; X86-BMI1BMI2-NEXT:    xorl %esi, %esi
+; X86-BMI1BMI2-NEXT:  .LBB29_2:
+; X86-BMI1BMI2-NEXT:    movl $-1, %ebp
+; X86-BMI1BMI2-NEXT:    shlxl %edx, %ebp, %ebx
+; X86-BMI1BMI2-NEXT:    movl %edx, %ecx
+; X86-BMI1BMI2-NEXT:    shldl %cl, %ebp, %ebp
+; X86-BMI1BMI2-NEXT:    testb $32, %dl
+; X86-BMI1BMI2-NEXT:    je .LBB29_4
+; X86-BMI1BMI2-NEXT:  # %bb.3:
+; X86-BMI1BMI2-NEXT:    movl %ebx, %ebp
+; X86-BMI1BMI2-NEXT:    xorl %ebx, %ebx
+; X86-BMI1BMI2-NEXT:  .LBB29_4:
+; X86-BMI1BMI2-NEXT:    andnl %esi, %ebp, %esi
+; X86-BMI1BMI2-NEXT:    andnl %edi, %ebx, %edi
+; X86-BMI1BMI2-NEXT:    subl $8, %esp
+; X86-BMI1BMI2-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-BMI1BMI2-NEXT:    pushl %eax
+; X86-BMI1BMI2-NEXT:    calll use64
+; X86-BMI1BMI2-NEXT:    addl $16, %esp
+; X86-BMI1BMI2-NEXT:    movl %edi, %eax
+; X86-BMI1BMI2-NEXT:    movl %esi, %edx
+; X86-BMI1BMI2-NEXT:    addl $12, %esp
+; X86-BMI1BMI2-NEXT:    popl %esi
+; X86-BMI1BMI2-NEXT:    popl %edi
+; X86-BMI1BMI2-NEXT:    popl %ebx
+; X86-BMI1BMI2-NEXT:    popl %ebp
+; X86-BMI1BMI2-NEXT:    retl
+;
+; X64-NOBMI-LABEL: bextr64_b5_skipextrauses:
+; X64-NOBMI:       # %bb.0:
+; X64-NOBMI-NEXT:    pushq %rbx
+; X64-NOBMI-NEXT:    movl %esi, %ecx
+; X64-NOBMI-NEXT:    shrq %cl, %rdi
+; X64-NOBMI-NEXT:    movq $-1, %rbx
+; X64-NOBMI-NEXT:    movl %edx, %ecx
+; X64-NOBMI-NEXT:    shlq %cl, %rbx
+; X64-NOBMI-NEXT:    notq %rbx
+; X64-NOBMI-NEXT:    andq %rdi, %rbx
+; X64-NOBMI-NEXT:    movq %rsi, %rdi
+; X64-NOBMI-NEXT:    callq use64
+; X64-NOBMI-NEXT:    movq %rbx, %rax
+; X64-NOBMI-NEXT:    popq %rbx
+; X64-NOBMI-NEXT:    retq
+;
+; X64-BMI1NOTBM-LABEL: bextr64_b5_skipextrauses:
+; X64-BMI1NOTBM:       # %bb.0:
+; X64-BMI1NOTBM-NEXT:    pushq %rbx
+; X64-BMI1NOTBM-NEXT:    shll $8, %edx
+; X64-BMI1NOTBM-NEXT:    movzbl %sil, %eax
+; X64-BMI1NOTBM-NEXT:    orl %edx, %eax
+; X64-BMI1NOTBM-NEXT:    bextrq %rax, %rdi, %rbx
+; X64-BMI1NOTBM-NEXT:    movq %rsi, %rdi
+; X64-BMI1NOTBM-NEXT:    callq use64
+; X64-BMI1NOTBM-NEXT:    movq %rbx, %rax
+; X64-BMI1NOTBM-NEXT:    popq %rbx
+; X64-BMI1NOTBM-NEXT:    retq
+;
+; X64-BMI1BMI2-LABEL: bextr64_b5_skipextrauses:
+; X64-BMI1BMI2:       # %bb.0:
+; X64-BMI1BMI2-NEXT:    pushq %rbx
+; X64-BMI1BMI2-NEXT:    shrxq %rsi, %rdi, %rax
+; X64-BMI1BMI2-NEXT:    bzhiq %rdx, %rax, %rbx
+; X64-BMI1BMI2-NEXT:    movq %rsi, %rdi
+; X64-BMI1BMI2-NEXT:    callq use64
+; X64-BMI1BMI2-NEXT:    movq %rbx, %rax
+; X64-BMI1BMI2-NEXT:    popq %rbx
+; X64-BMI1BMI2-NEXT:    retq
+  %shifted = lshr i64 %val, %numskipbits
+  %notmask = shl i64 -1, %numlowbits
+  %mask = xor i64 %notmask, -1
+  %masked = and i64 %mask, %shifted
+  call void @use64(i64 %numskipbits)
+  ret i64 %masked
+}
+
+; 64-bit, but with 32-bit output
+
+; Everything done in 64-bit, truncation happens last.
+define i32 @bextr64_32_b0(i64 %val, i64 %numskipbits, i8 %numlowbits) nounwind {
+; X86-NOBMI-LABEL: bextr64_32_b0:
+; X86-NOBMI:       # %bb.0:
+; X86-NOBMI-NEXT:    pushl %edi
+; X86-NOBMI-NEXT:    pushl %esi
+; X86-NOBMI-NEXT:    movb {{[0-9]+}}(%esp), %dl
+; X86-NOBMI-NEXT:    movb {{[0-9]+}}(%esp), %cl
+; X86-NOBMI-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NOBMI-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NOBMI-NEXT:    movl %edi, %eax
+; X86-NOBMI-NEXT:    shrl %cl, %eax
+; X86-NOBMI-NEXT:    shrdl %cl, %edi, %esi
+; X86-NOBMI-NEXT:    testb $32, %cl
+; X86-NOBMI-NEXT:    jne .LBB30_2
+; X86-NOBMI-NEXT:  # %bb.1:
+; X86-NOBMI-NEXT:    movl %esi, %eax
+; X86-NOBMI-NEXT:  .LBB30_2:
+; X86-NOBMI-NEXT:    movl $-1, %esi
+; X86-NOBMI-NEXT:    movl %edx, %ecx
+; X86-NOBMI-NEXT:    shll %cl, %esi
+; X86-NOBMI-NEXT:    xorl %ecx, %ecx
+; X86-NOBMI-NEXT:    testb $32, %dl
+; X86-NOBMI-NEXT:    jne .LBB30_4
+; X86-NOBMI-NEXT:  # %bb.3:
+; X86-NOBMI-NEXT:    movl %esi, %ecx
+; X86-NOBMI-NEXT:  .LBB30_4:
+; X86-NOBMI-NEXT:    notl %ecx
+; X86-NOBMI-NEXT:    andl %ecx, %eax
+; X86-NOBMI-NEXT:    popl %esi
+; X86-NOBMI-NEXT:    popl %edi
+; X86-NOBMI-NEXT:    retl
+;
+; X86-BMI1NOTBM-LABEL: bextr64_32_b0:
+; X86-BMI1NOTBM:       # %bb.0:
+; X86-BMI1NOTBM-NEXT:    pushl %edi
+; X86-BMI1NOTBM-NEXT:    pushl %esi
+; X86-BMI1NOTBM-NEXT:    movb {{[0-9]+}}(%esp), %al
+; X86-BMI1NOTBM-NEXT:    movb {{[0-9]+}}(%esp), %cl
+; X86-BMI1NOTBM-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-BMI1NOTBM-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-BMI1NOTBM-NEXT:    movl %edi, %edx
+; X86-BMI1NOTBM-NEXT:    shrl %cl, %edx
+; X86-BMI1NOTBM-NEXT:    shrdl %cl, %edi, %esi
+; X86-BMI1NOTBM-NEXT:    testb $32, %cl
+; X86-BMI1NOTBM-NEXT:    jne .LBB30_2
+; X86-BMI1NOTBM-NEXT:  # %bb.1:
+; X86-BMI1NOTBM-NEXT:    movl %esi, %edx
+; X86-BMI1NOTBM-NEXT:  .LBB30_2:
+; X86-BMI1NOTBM-NEXT:    movl $-1, %esi
+; X86-BMI1NOTBM-NEXT:    movl %eax, %ecx
+; X86-BMI1NOTBM-NEXT:    shll %cl, %esi
+; X86-BMI1NOTBM-NEXT:    xorl %ecx, %ecx
+; X86-BMI1NOTBM-NEXT:    testb $32, %al
+; X86-BMI1NOTBM-NEXT:    jne .LBB30_4
+; X86-BMI1NOTBM-NEXT:  # %bb.3:
+; X86-BMI1NOTBM-NEXT:    movl %esi, %ecx
+; X86-BMI1NOTBM-NEXT:  .LBB30_4:
+; X86-BMI1NOTBM-NEXT:    andnl %edx, %ecx, %eax
+; X86-BMI1NOTBM-NEXT:    popl %esi
+; X86-BMI1NOTBM-NEXT:    popl %edi
+; X86-BMI1NOTBM-NEXT:    retl
+;
+; X86-BMI1BMI2-LABEL: bextr64_32_b0:
+; X86-BMI1BMI2:       # %bb.0:
+; X86-BMI1BMI2-NEXT:    pushl %esi
+; X86-BMI1BMI2-NEXT:    movb {{[0-9]+}}(%esp), %al
+; X86-BMI1BMI2-NEXT:    movb {{[0-9]+}}(%esp), %cl
+; X86-BMI1BMI2-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-BMI1BMI2-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-BMI1BMI2-NEXT:    shrdl %cl, %esi, %edx
+; X86-BMI1BMI2-NEXT:    testb $32, %cl
+; X86-BMI1BMI2-NEXT:    je .LBB30_2
+; X86-BMI1BMI2-NEXT:  # %bb.1:
+; X86-BMI1BMI2-NEXT:    shrxl %ecx, %esi, %edx
+; X86-BMI1BMI2-NEXT:  .LBB30_2:
+; X86-BMI1BMI2-NEXT:    xorl %ecx, %ecx
+; X86-BMI1BMI2-NEXT:    testb $32, %al
+; X86-BMI1BMI2-NEXT:    jne .LBB30_4
+; X86-BMI1BMI2-NEXT:  # %bb.3:
+; X86-BMI1BMI2-NEXT:    movl $-1, %ecx
+; X86-BMI1BMI2-NEXT:    shlxl %eax, %ecx, %ecx
+; X86-BMI1BMI2-NEXT:  .LBB30_4:
+; X86-BMI1BMI2-NEXT:    andnl %edx, %ecx, %eax
+; X86-BMI1BMI2-NEXT:    popl %esi
+; X86-BMI1BMI2-NEXT:    retl
+;
+; X64-NOBMI-LABEL: bextr64_32_b0:
+; X64-NOBMI:       # %bb.0:
+; X64-NOBMI-NEXT:    movq %rsi, %rcx
+; X64-NOBMI-NEXT:    # kill: def $cl killed $cl killed $rcx
+; X64-NOBMI-NEXT:    shrq %cl, %rdi
+; X64-NOBMI-NEXT:    movq $-1, %rax
+; X64-NOBMI-NEXT:    movl %edx, %ecx
+; X64-NOBMI-NEXT:    shlq %cl, %rax
+; X64-NOBMI-NEXT:    notl %eax
+; X64-NOBMI-NEXT:    andl %edi, %eax
+; X64-NOBMI-NEXT:    # kill: def $eax killed $eax killed $rax
+; X64-NOBMI-NEXT:    retq
+;
+; X64-BMI1NOTBM-LABEL: bextr64_32_b0:
+; X64-BMI1NOTBM:       # %bb.0:
+; X64-BMI1NOTBM-NEXT:    movq %rsi, %rcx
+; X64-BMI1NOTBM-NEXT:    # kill: def $cl killed $cl killed $rcx
+; X64-BMI1NOTBM-NEXT:    shrq %cl, %rdi
+; X64-BMI1NOTBM-NEXT:    movq $-1, %rax
+; X64-BMI1NOTBM-NEXT:    movl %edx, %ecx
+; X64-BMI1NOTBM-NEXT:    shlq %cl, %rax
+; X64-BMI1NOTBM-NEXT:    andnl %edi, %eax, %eax
+; X64-BMI1NOTBM-NEXT:    retq
+;
+; X64-BMI1BMI2-LABEL: bextr64_32_b0:
+; X64-BMI1BMI2:       # %bb.0:
+; X64-BMI1BMI2-NEXT:    # kill: def $edx killed $edx def $rdx
+; X64-BMI1BMI2-NEXT:    shrxq %rsi, %rdi, %rax
+; X64-BMI1BMI2-NEXT:    movq $-1, %rcx
+; X64-BMI1BMI2-NEXT:    shlxq %rdx, %rcx, %rcx
+; X64-BMI1BMI2-NEXT:    andnl %eax, %ecx, %eax
+; X64-BMI1BMI2-NEXT:    retq
+  %shiftedval = lshr i64 %val, %numskipbits
+  %widenumlowbits = zext i8 %numlowbits to i64
+  %notmask = shl nsw i64 -1, %widenumlowbits
+  %mask = xor i64 %notmask, -1
+  %wideres = and i64 %shiftedval, %mask
+  %res = trunc i64 %wideres to i32
+  ret i32 %res
+}
+
+; Shifting happens in 64-bit, then truncation. Masking is 32-bit.
+define i32 @bextr64_32_b1(i64 %val, i64 %numskipbits, i8 %numlowbits) nounwind {
+; X86-NOBMI-LABEL: bextr64_32_b1:
+; X86-NOBMI:       # %bb.0:
+; X86-NOBMI-NEXT:    pushl %edi
+; X86-NOBMI-NEXT:    pushl %esi
+; X86-NOBMI-NEXT:    movb {{[0-9]+}}(%esp), %dl
+; X86-NOBMI-NEXT:    movb {{[0-9]+}}(%esp), %cl
+; X86-NOBMI-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NOBMI-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NOBMI-NEXT:    movl %edi, %esi
+; X86-NOBMI-NEXT:    shrl %cl, %esi
+; X86-NOBMI-NEXT:    shrdl %cl, %edi, %eax
+; X86-NOBMI-NEXT:    testb $32, %cl
+; X86-NOBMI-NEXT:    jne .LBB31_2
+; X86-NOBMI-NEXT:  # %bb.1:
+; X86-NOBMI-NEXT:    movl %eax, %esi
+; X86-NOBMI-NEXT:  .LBB31_2:
+; X86-NOBMI-NEXT:    movl $-1, %eax
+; X86-NOBMI-NEXT:    movl %edx, %ecx
+; X86-NOBMI-NEXT:    shll %cl, %eax
+; X86-NOBMI-NEXT:    notl %eax
+; X86-NOBMI-NEXT:    andl %esi, %eax
+; X86-NOBMI-NEXT:    popl %esi
+; X86-NOBMI-NEXT:    popl %edi
+; X86-NOBMI-NEXT:    retl
+;
+; X86-BMI1NOTBM-LABEL: bextr64_32_b1:
+; X86-BMI1NOTBM:       # %bb.0:
+; X86-BMI1NOTBM-NEXT:    pushl %edi
+; X86-BMI1NOTBM-NEXT:    pushl %esi
+; X86-BMI1NOTBM-NEXT:    movb {{[0-9]+}}(%esp), %al
+; X86-BMI1NOTBM-NEXT:    movb {{[0-9]+}}(%esp), %cl
+; X86-BMI1NOTBM-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-BMI1NOTBM-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-BMI1NOTBM-NEXT:    movl %edi, %edx
+; X86-BMI1NOTBM-NEXT:    shrl %cl, %edx
+; X86-BMI1NOTBM-NEXT:    shrdl %cl, %edi, %esi
+; X86-BMI1NOTBM-NEXT:    testb $32, %cl
+; X86-BMI1NOTBM-NEXT:    jne .LBB31_2
+; X86-BMI1NOTBM-NEXT:  # %bb.1:
+; X86-BMI1NOTBM-NEXT:    movl %esi, %edx
+; X86-BMI1NOTBM-NEXT:  .LBB31_2:
+; X86-BMI1NOTBM-NEXT:    shll $8, %eax
+; X86-BMI1NOTBM-NEXT:    bextrl %eax, %edx, %eax
+; X86-BMI1NOTBM-NEXT:    popl %esi
+; X86-BMI1NOTBM-NEXT:    popl %edi
+; X86-BMI1NOTBM-NEXT:    retl
+;
+; X86-BMI1BMI2-LABEL: bextr64_32_b1:
+; X86-BMI1BMI2:       # %bb.0:
+; X86-BMI1BMI2-NEXT:    pushl %esi
+; X86-BMI1BMI2-NEXT:    movb {{[0-9]+}}(%esp), %al
+; X86-BMI1BMI2-NEXT:    movb {{[0-9]+}}(%esp), %cl
+; X86-BMI1BMI2-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-BMI1BMI2-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-BMI1BMI2-NEXT:    shrdl %cl, %esi, %edx
+; X86-BMI1BMI2-NEXT:    testb $32, %cl
+; X86-BMI1BMI2-NEXT:    je .LBB31_2
+; X86-BMI1BMI2-NEXT:  # %bb.1:
+; X86-BMI1BMI2-NEXT:    shrxl %ecx, %esi, %edx
+; X86-BMI1BMI2-NEXT:  .LBB31_2:
+; X86-BMI1BMI2-NEXT:    bzhil %eax, %edx, %eax
+; X86-BMI1BMI2-NEXT:    popl %esi
+; X86-BMI1BMI2-NEXT:    retl
+;
+; X64-NOBMI-LABEL: bextr64_32_b1:
+; X64-NOBMI:       # %bb.0:
+; X64-NOBMI-NEXT:    movq %rsi, %rcx
+; X64-NOBMI-NEXT:    # kill: def $cl killed $cl killed $rcx
+; X64-NOBMI-NEXT:    shrq %cl, %rdi
+; X64-NOBMI-NEXT:    movl $-1, %eax
+; X64-NOBMI-NEXT:    movl %edx, %ecx
+; X64-NOBMI-NEXT:    shll %cl, %eax
+; X64-NOBMI-NEXT:    notl %eax
+; X64-NOBMI-NEXT:    andl %edi, %eax
+; X64-NOBMI-NEXT:    retq
+;
+; X64-BMI1NOTBM-LABEL: bextr64_32_b1:
+; X64-BMI1NOTBM:       # %bb.0:
+; X64-BMI1NOTBM-NEXT:    shll $8, %edx
+; X64-BMI1NOTBM-NEXT:    movzbl %sil, %eax
+; X64-BMI1NOTBM-NEXT:    orl %edx, %eax
+; X64-BMI1NOTBM-NEXT:    bextrq %rax, %rdi, %rax
+; X64-BMI1NOTBM-NEXT:    # kill: def $eax killed $eax killed $rax
+; X64-BMI1NOTBM-NEXT:    retq
+;
+; X64-BMI1BMI2-LABEL: bextr64_32_b1:
+; X64-BMI1BMI2:       # %bb.0:
+; X64-BMI1BMI2-NEXT:    shrxq %rsi, %rdi, %rax
+; X64-BMI1BMI2-NEXT:    bzhil %edx, %eax, %eax
+; X64-BMI1BMI2-NEXT:    retq
+  %shiftedval = lshr i64 %val, %numskipbits
+  %truncshiftedval = trunc i64 %shiftedval to i32
+  %widenumlowbits = zext i8 %numlowbits to i32
+  %notmask = shl nsw i32 -1, %widenumlowbits
+  %mask = xor i32 %notmask, -1
+  %res = and i32 %truncshiftedval, %mask
+  ret i32 %res
+}
+
+; Shifting happens in 64-bit. Mask is 32-bit, but extended to 64-bit.
+; Masking is 64-bit. Then truncation.
+define i32 @bextr64_32_b2(i64 %val, i64 %numskipbits, i8 %numlowbits) nounwind {
+; X86-NOBMI-LABEL: bextr64_32_b2:
+; X86-NOBMI:       # %bb.0:
+; X86-NOBMI-NEXT:    pushl %edi
+; X86-NOBMI-NEXT:    pushl %esi
+; X86-NOBMI-NEXT:    movb {{[0-9]+}}(%esp), %dl
+; X86-NOBMI-NEXT:    movb {{[0-9]+}}(%esp), %cl
+; X86-NOBMI-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NOBMI-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NOBMI-NEXT:    movl %edi, %esi
+; X86-NOBMI-NEXT:    shrl %cl, %esi
+; X86-NOBMI-NEXT:    shrdl %cl, %edi, %eax
+; X86-NOBMI-NEXT:    testb $32, %cl
+; X86-NOBMI-NEXT:    jne .LBB32_2
+; X86-NOBMI-NEXT:  # %bb.1:
+; X86-NOBMI-NEXT:    movl %eax, %esi
+; X86-NOBMI-NEXT:  .LBB32_2:
+; X86-NOBMI-NEXT:    movl $-1, %eax
+; X86-NOBMI-NEXT:    movl %edx, %ecx
+; X86-NOBMI-NEXT:    shll %cl, %eax
+; X86-NOBMI-NEXT:    notl %eax
+; X86-NOBMI-NEXT:    andl %esi, %eax
+; X86-NOBMI-NEXT:    popl %esi
+; X86-NOBMI-NEXT:    popl %edi
+; X86-NOBMI-NEXT:    retl
+;
+; X86-BMI1NOTBM-LABEL: bextr64_32_b2:
+; X86-BMI1NOTBM:       # %bb.0:
+; X86-BMI1NOTBM-NEXT:    pushl %edi
+; X86-BMI1NOTBM-NEXT:    pushl %esi
+; X86-BMI1NOTBM-NEXT:    movb {{[0-9]+}}(%esp), %al
+; X86-BMI1NOTBM-NEXT:    movb {{[0-9]+}}(%esp), %cl
+; X86-BMI1NOTBM-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-BMI1NOTBM-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-BMI1NOTBM-NEXT:    movl %edi, %edx
+; X86-BMI1NOTBM-NEXT:    shrl %cl, %edx
+; X86-BMI1NOTBM-NEXT:    shrdl %cl, %edi, %esi
+; X86-BMI1NOTBM-NEXT:    testb $32, %cl
+; X86-BMI1NOTBM-NEXT:    jne .LBB32_2
+; X86-BMI1NOTBM-NEXT:  # %bb.1:
+; X86-BMI1NOTBM-NEXT:    movl %esi, %edx
+; X86-BMI1NOTBM-NEXT:  .LBB32_2:
+; X86-BMI1NOTBM-NEXT:    shll $8, %eax
+; X86-BMI1NOTBM-NEXT:    bextrl %eax, %edx, %eax
+; X86-BMI1NOTBM-NEXT:    popl %esi
+; X86-BMI1NOTBM-NEXT:    popl %edi
+; X86-BMI1NOTBM-NEXT:    retl
+;
+; X86-BMI1BMI2-LABEL: bextr64_32_b2:
+; X86-BMI1BMI2:       # %bb.0:
+; X86-BMI1BMI2-NEXT:    pushl %esi
+; X86-BMI1BMI2-NEXT:    movb {{[0-9]+}}(%esp), %al
+; X86-BMI1BMI2-NEXT:    movb {{[0-9]+}}(%esp), %cl
+; X86-BMI1BMI2-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-BMI1BMI2-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-BMI1BMI2-NEXT:    shrdl %cl, %esi, %edx
+; X86-BMI1BMI2-NEXT:    testb $32, %cl
+; X86-BMI1BMI2-NEXT:    je .LBB32_2
+; X86-BMI1BMI2-NEXT:  # %bb.1:
+; X86-BMI1BMI2-NEXT:    shrxl %ecx, %esi, %edx
+; X86-BMI1BMI2-NEXT:  .LBB32_2:
+; X86-BMI1BMI2-NEXT:    bzhil %eax, %edx, %eax
+; X86-BMI1BMI2-NEXT:    popl %esi
+; X86-BMI1BMI2-NEXT:    retl
+;
+; X64-NOBMI-LABEL: bextr64_32_b2:
+; X64-NOBMI:       # %bb.0:
+; X64-NOBMI-NEXT:    movq %rsi, %rcx
+; X64-NOBMI-NEXT:    # kill: def $cl killed $cl killed $rcx
+; X64-NOBMI-NEXT:    shrq %cl, %rdi
+; X64-NOBMI-NEXT:    movl $-1, %eax
+; X64-NOBMI-NEXT:    movl %edx, %ecx
+; X64-NOBMI-NEXT:    shll %cl, %eax
+; X64-NOBMI-NEXT:    notl %eax
+; X64-NOBMI-NEXT:    andl %edi, %eax
+; X64-NOBMI-NEXT:    retq
+;
+; X64-BMI1NOTBM-LABEL: bextr64_32_b2:
+; X64-BMI1NOTBM:       # %bb.0:
+; X64-BMI1NOTBM-NEXT:    shll $8, %edx
+; X64-BMI1NOTBM-NEXT:    movzbl %sil, %eax
+; X64-BMI1NOTBM-NEXT:    orl %edx, %eax
+; X64-BMI1NOTBM-NEXT:    bextrq %rax, %rdi, %rax
+; X64-BMI1NOTBM-NEXT:    # kill: def $eax killed $eax killed $rax
+; X64-BMI1NOTBM-NEXT:    retq
+;
+; X64-BMI1BMI2-LABEL: bextr64_32_b2:
+; X64-BMI1BMI2:       # %bb.0:
+; X64-BMI1BMI2-NEXT:    shrxq %rsi, %rdi, %rax
+; X64-BMI1BMI2-NEXT:    bzhil %edx, %eax, %eax
+; X64-BMI1BMI2-NEXT:    retq
+  %shiftedval = lshr i64 %val, %numskipbits
+  %widenumlowbits = zext i8 %numlowbits to i32
+  %notmask = shl nsw i32 -1, %widenumlowbits
+  %mask = xor i32 %notmask, -1
+  %zextmask = zext i32 %mask to i64
+  %wideres = and i64 %shiftedval, %zextmask
+  %res = trunc i64 %wideres to i32
+  ret i32 %res
+}
+
+; ---------------------------------------------------------------------------- ;
+; Pattern c. 32-bit
+; ---------------------------------------------------------------------------- ;
+
+define i32 @bextr32_c0(i32 %val, i32 %numskipbits, i32 %numlowbits) nounwind {
+; X86-NOBMI-LABEL: bextr32_c0:
+; X86-NOBMI:       # %bb.0:
+; X86-NOBMI-NEXT:    pushl %edi
+; X86-NOBMI-NEXT:    pushl %esi
+; X86-NOBMI-NEXT:    pushl %eax
+; X86-NOBMI-NEXT:    movb {{[0-9]+}}(%esp), %cl
+; X86-NOBMI-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NOBMI-NEXT:    shrl %cl, %edi
+; X86-NOBMI-NEXT:    xorl %ecx, %ecx
+; X86-NOBMI-NEXT:    subb {{[0-9]+}}(%esp), %cl
+; X86-NOBMI-NEXT:    movl $-1, %esi
+; X86-NOBMI-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X86-NOBMI-NEXT:    shrl %cl, %esi
+; X86-NOBMI-NEXT:    movl %esi, (%esp)
+; X86-NOBMI-NEXT:    calll use32
+; X86-NOBMI-NEXT:    andl %edi, %esi
+; X86-NOBMI-NEXT:    movl %esi, %eax
+; X86-NOBMI-NEXT:    addl $4, %esp
+; X86-NOBMI-NEXT:    popl %esi
+; X86-NOBMI-NEXT:    popl %edi
+; X86-NOBMI-NEXT:    retl
+;
+; X86-BMI1NOTBM-LABEL: bextr32_c0:
+; X86-BMI1NOTBM:       # %bb.0:
+; X86-BMI1NOTBM-NEXT:    pushl %edi
+; X86-BMI1NOTBM-NEXT:    pushl %esi
+; X86-BMI1NOTBM-NEXT:    pushl %eax
+; X86-BMI1NOTBM-NEXT:    movb {{[0-9]+}}(%esp), %cl
+; X86-BMI1NOTBM-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-BMI1NOTBM-NEXT:    shrl %cl, %edi
+; X86-BMI1NOTBM-NEXT:    xorl %ecx, %ecx
+; X86-BMI1NOTBM-NEXT:    subb {{[0-9]+}}(%esp), %cl
+; X86-BMI1NOTBM-NEXT:    movl $-1, %esi
+; X86-BMI1NOTBM-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X86-BMI1NOTBM-NEXT:    shrl %cl, %esi
+; X86-BMI1NOTBM-NEXT:    movl %esi, (%esp)
+; X86-BMI1NOTBM-NEXT:    calll use32
+; X86-BMI1NOTBM-NEXT:    andl %edi, %esi
+; X86-BMI1NOTBM-NEXT:    movl %esi, %eax
+; X86-BMI1NOTBM-NEXT:    addl $4, %esp
+; X86-BMI1NOTBM-NEXT:    popl %esi
+; X86-BMI1NOTBM-NEXT:    popl %edi
+; X86-BMI1NOTBM-NEXT:    retl
+;
+; X86-BMI1BMI2-LABEL: bextr32_c0:
+; X86-BMI1BMI2:       # %bb.0:
+; X86-BMI1BMI2-NEXT:    pushl %ebx
+; X86-BMI1BMI2-NEXT:    pushl %esi
+; X86-BMI1BMI2-NEXT:    pushl %eax
+; X86-BMI1BMI2-NEXT:    movb {{[0-9]+}}(%esp), %bl
+; X86-BMI1BMI2-NEXT:    movb {{[0-9]+}}(%esp), %al
+; X86-BMI1BMI2-NEXT:    shrxl %eax, {{[0-9]+}}(%esp), %esi
+; X86-BMI1BMI2-NEXT:    movl %ebx, %eax
+; X86-BMI1BMI2-NEXT:    negb %al
+; X86-BMI1BMI2-NEXT:    movl $-1, %ecx
+; X86-BMI1BMI2-NEXT:    shrxl %eax, %ecx, %eax
+; X86-BMI1BMI2-NEXT:    movl %eax, (%esp)
+; X86-BMI1BMI2-NEXT:    calll use32
+; X86-BMI1BMI2-NEXT:    bzhil %ebx, %esi, %eax
+; X86-BMI1BMI2-NEXT:    addl $4, %esp
+; X86-BMI1BMI2-NEXT:    popl %esi
+; X86-BMI1BMI2-NEXT:    popl %ebx
+; X86-BMI1BMI2-NEXT:    retl
+;
+; X64-NOBMI-LABEL: bextr32_c0:
+; X64-NOBMI:       # %bb.0:
+; X64-NOBMI-NEXT:    pushq %rbp
+; X64-NOBMI-NEXT:    pushq %rbx
+; X64-NOBMI-NEXT:    pushq %rax
+; X64-NOBMI-NEXT:    movl %esi, %ecx
+; X64-NOBMI-NEXT:    movl %edi, %ebx
+; X64-NOBMI-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X64-NOBMI-NEXT:    shrl %cl, %ebx
+; X64-NOBMI-NEXT:    negb %dl
+; X64-NOBMI-NEXT:    movl $-1, %ebp
+; X64-NOBMI-NEXT:    movl %edx, %ecx
+; X64-NOBMI-NEXT:    shrl %cl, %ebp
+; X64-NOBMI-NEXT:    movl %ebp, %edi
+; X64-NOBMI-NEXT:    callq use32
+; X64-NOBMI-NEXT:    andl %ebx, %ebp
+; X64-NOBMI-NEXT:    movl %ebp, %eax
+; X64-NOBMI-NEXT:    addq $8, %rsp
+; X64-NOBMI-NEXT:    popq %rbx
+; X64-NOBMI-NEXT:    popq %rbp
+; X64-NOBMI-NEXT:    retq
+;
+; X64-BMI1NOTBM-LABEL: bextr32_c0:
+; X64-BMI1NOTBM:       # %bb.0:
+; X64-BMI1NOTBM-NEXT:    pushq %rbp
+; X64-BMI1NOTBM-NEXT:    pushq %rbx
+; X64-BMI1NOTBM-NEXT:    pushq %rax
+; X64-BMI1NOTBM-NEXT:    movl %esi, %ecx
+; X64-BMI1NOTBM-NEXT:    movl %edi, %ebx
+; X64-BMI1NOTBM-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X64-BMI1NOTBM-NEXT:    shrl %cl, %ebx
+; X64-BMI1NOTBM-NEXT:    negb %dl
+; X64-BMI1NOTBM-NEXT:    movl $-1, %ebp
+; X64-BMI1NOTBM-NEXT:    movl %edx, %ecx
+; X64-BMI1NOTBM-NEXT:    shrl %cl, %ebp
+; X64-BMI1NOTBM-NEXT:    movl %ebp, %edi
+; X64-BMI1NOTBM-NEXT:    callq use32
+; X64-BMI1NOTBM-NEXT:    andl %ebx, %ebp
+; X64-BMI1NOTBM-NEXT:    movl %ebp, %eax
+; X64-BMI1NOTBM-NEXT:    addq $8, %rsp
+; X64-BMI1NOTBM-NEXT:    popq %rbx
+; X64-BMI1NOTBM-NEXT:    popq %rbp
+; X64-BMI1NOTBM-NEXT:    retq
+;
+; X64-BMI1BMI2-LABEL: bextr32_c0:
+; X64-BMI1BMI2:       # %bb.0:
+; X64-BMI1BMI2-NEXT:    pushq %rbp
+; X64-BMI1BMI2-NEXT:    pushq %rbx
+; X64-BMI1BMI2-NEXT:    pushq %rax
+; X64-BMI1BMI2-NEXT:    movl %edx, %ebx
+; X64-BMI1BMI2-NEXT:    shrxl %esi, %edi, %ebp
+; X64-BMI1BMI2-NEXT:    movl %ebx, %eax
+; X64-BMI1BMI2-NEXT:    negb %al
+; X64-BMI1BMI2-NEXT:    movl $-1, %ecx
+; X64-BMI1BMI2-NEXT:    shrxl %eax, %ecx, %edi
+; X64-BMI1BMI2-NEXT:    callq use32
+; X64-BMI1BMI2-NEXT:    bzhil %ebx, %ebp, %eax
+; X64-BMI1BMI2-NEXT:    addq $8, %rsp
+; X64-BMI1BMI2-NEXT:    popq %rbx
+; X64-BMI1BMI2-NEXT:    popq %rbp
+; X64-BMI1BMI2-NEXT:    retq
+  %shifted = lshr i32 %val, %numskipbits
   %numhighbits = sub i32 32, %numlowbits
   %mask = lshr i32 -1, %numhighbits
   call void @use32(i32 %mask)
@@ -3107,14 +3918,151 @@ define i32 @bextr32_c0(i32 %val, i32 %numskipbits, i32 %numlowbits) nounwind {
   ret i32 %masked
 }
 
-define i32 @bextr32_c1_indexzext(i32 %val, i8 %numskipbits, i8 %numlowbits) nounwind {
-; X86-NOBMI-LABEL: bextr32_c1_indexzext:
+define i32 @bextr32_c1_indexzext(i32 %val, i8 %numskipbits, i8 %numlowbits) nounwind {
+; X86-NOBMI-LABEL: bextr32_c1_indexzext:
+; X86-NOBMI:       # %bb.0:
+; X86-NOBMI-NEXT:    pushl %edi
+; X86-NOBMI-NEXT:    pushl %esi
+; X86-NOBMI-NEXT:    pushl %eax
+; X86-NOBMI-NEXT:    movb {{[0-9]+}}(%esp), %cl
+; X86-NOBMI-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NOBMI-NEXT:    shrl %cl, %edi
+; X86-NOBMI-NEXT:    xorl %ecx, %ecx
+; X86-NOBMI-NEXT:    subb {{[0-9]+}}(%esp), %cl
+; X86-NOBMI-NEXT:    movl $-1, %esi
+; X86-NOBMI-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X86-NOBMI-NEXT:    shrl %cl, %esi
+; X86-NOBMI-NEXT:    movl %esi, (%esp)
+; X86-NOBMI-NEXT:    calll use32
+; X86-NOBMI-NEXT:    andl %edi, %esi
+; X86-NOBMI-NEXT:    movl %esi, %eax
+; X86-NOBMI-NEXT:    addl $4, %esp
+; X86-NOBMI-NEXT:    popl %esi
+; X86-NOBMI-NEXT:    popl %edi
+; X86-NOBMI-NEXT:    retl
+;
+; X86-BMI1NOTBM-LABEL: bextr32_c1_indexzext:
+; X86-BMI1NOTBM:       # %bb.0:
+; X86-BMI1NOTBM-NEXT:    pushl %edi
+; X86-BMI1NOTBM-NEXT:    pushl %esi
+; X86-BMI1NOTBM-NEXT:    pushl %eax
+; X86-BMI1NOTBM-NEXT:    movb {{[0-9]+}}(%esp), %cl
+; X86-BMI1NOTBM-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-BMI1NOTBM-NEXT:    shrl %cl, %edi
+; X86-BMI1NOTBM-NEXT:    xorl %ecx, %ecx
+; X86-BMI1NOTBM-NEXT:    subb {{[0-9]+}}(%esp), %cl
+; X86-BMI1NOTBM-NEXT:    movl $-1, %esi
+; X86-BMI1NOTBM-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X86-BMI1NOTBM-NEXT:    shrl %cl, %esi
+; X86-BMI1NOTBM-NEXT:    movl %esi, (%esp)
+; X86-BMI1NOTBM-NEXT:    calll use32
+; X86-BMI1NOTBM-NEXT:    andl %edi, %esi
+; X86-BMI1NOTBM-NEXT:    movl %esi, %eax
+; X86-BMI1NOTBM-NEXT:    addl $4, %esp
+; X86-BMI1NOTBM-NEXT:    popl %esi
+; X86-BMI1NOTBM-NEXT:    popl %edi
+; X86-BMI1NOTBM-NEXT:    retl
+;
+; X86-BMI1BMI2-LABEL: bextr32_c1_indexzext:
+; X86-BMI1BMI2:       # %bb.0:
+; X86-BMI1BMI2-NEXT:    pushl %ebx
+; X86-BMI1BMI2-NEXT:    pushl %esi
+; X86-BMI1BMI2-NEXT:    pushl %eax
+; X86-BMI1BMI2-NEXT:    movb {{[0-9]+}}(%esp), %bl
+; X86-BMI1BMI2-NEXT:    movb {{[0-9]+}}(%esp), %al
+; X86-BMI1BMI2-NEXT:    shrxl %eax, {{[0-9]+}}(%esp), %esi
+; X86-BMI1BMI2-NEXT:    movl %ebx, %eax
+; X86-BMI1BMI2-NEXT:    negb %al
+; X86-BMI1BMI2-NEXT:    movl $-1, %ecx
+; X86-BMI1BMI2-NEXT:    shrxl %eax, %ecx, %eax
+; X86-BMI1BMI2-NEXT:    movl %eax, (%esp)
+; X86-BMI1BMI2-NEXT:    calll use32
+; X86-BMI1BMI2-NEXT:    bzhil %ebx, %esi, %eax
+; X86-BMI1BMI2-NEXT:    addl $4, %esp
+; X86-BMI1BMI2-NEXT:    popl %esi
+; X86-BMI1BMI2-NEXT:    popl %ebx
+; X86-BMI1BMI2-NEXT:    retl
+;
+; X64-NOBMI-LABEL: bextr32_c1_indexzext:
+; X64-NOBMI:       # %bb.0:
+; X64-NOBMI-NEXT:    pushq %rbp
+; X64-NOBMI-NEXT:    pushq %rbx
+; X64-NOBMI-NEXT:    pushq %rax
+; X64-NOBMI-NEXT:    movl %esi, %ecx
+; X64-NOBMI-NEXT:    movl %edi, %ebx
+; X64-NOBMI-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X64-NOBMI-NEXT:    shrl %cl, %ebx
+; X64-NOBMI-NEXT:    negb %dl
+; X64-NOBMI-NEXT:    movl $-1, %ebp
+; X64-NOBMI-NEXT:    movl %edx, %ecx
+; X64-NOBMI-NEXT:    shrl %cl, %ebp
+; X64-NOBMI-NEXT:    movl %ebp, %edi
+; X64-NOBMI-NEXT:    callq use32
+; X64-NOBMI-NEXT:    andl %ebx, %ebp
+; X64-NOBMI-NEXT:    movl %ebp, %eax
+; X64-NOBMI-NEXT:    addq $8, %rsp
+; X64-NOBMI-NEXT:    popq %rbx
+; X64-NOBMI-NEXT:    popq %rbp
+; X64-NOBMI-NEXT:    retq
+;
+; X64-BMI1NOTBM-LABEL: bextr32_c1_indexzext:
+; X64-BMI1NOTBM:       # %bb.0:
+; X64-BMI1NOTBM-NEXT:    pushq %rbp
+; X64-BMI1NOTBM-NEXT:    pushq %rbx
+; X64-BMI1NOTBM-NEXT:    pushq %rax
+; X64-BMI1NOTBM-NEXT:    movl %esi, %ecx
+; X64-BMI1NOTBM-NEXT:    movl %edi, %ebx
+; X64-BMI1NOTBM-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X64-BMI1NOTBM-NEXT:    shrl %cl, %ebx
+; X64-BMI1NOTBM-NEXT:    negb %dl
+; X64-BMI1NOTBM-NEXT:    movl $-1, %ebp
+; X64-BMI1NOTBM-NEXT:    movl %edx, %ecx
+; X64-BMI1NOTBM-NEXT:    shrl %cl, %ebp
+; X64-BMI1NOTBM-NEXT:    movl %ebp, %edi
+; X64-BMI1NOTBM-NEXT:    callq use32
+; X64-BMI1NOTBM-NEXT:    andl %ebx, %ebp
+; X64-BMI1NOTBM-NEXT:    movl %ebp, %eax
+; X64-BMI1NOTBM-NEXT:    addq $8, %rsp
+; X64-BMI1NOTBM-NEXT:    popq %rbx
+; X64-BMI1NOTBM-NEXT:    popq %rbp
+; X64-BMI1NOTBM-NEXT:    retq
+;
+; X64-BMI1BMI2-LABEL: bextr32_c1_indexzext:
+; X64-BMI1BMI2:       # %bb.0:
+; X64-BMI1BMI2-NEXT:    pushq %rbp
+; X64-BMI1BMI2-NEXT:    pushq %rbx
+; X64-BMI1BMI2-NEXT:    pushq %rax
+; X64-BMI1BMI2-NEXT:    movl %edx, %ebx
+; X64-BMI1BMI2-NEXT:    shrxl %esi, %edi, %ebp
+; X64-BMI1BMI2-NEXT:    movl %ebx, %eax
+; X64-BMI1BMI2-NEXT:    negb %al
+; X64-BMI1BMI2-NEXT:    movl $-1, %ecx
+; X64-BMI1BMI2-NEXT:    shrxl %eax, %ecx, %edi
+; X64-BMI1BMI2-NEXT:    callq use32
+; X64-BMI1BMI2-NEXT:    bzhil %ebx, %ebp, %eax
+; X64-BMI1BMI2-NEXT:    addq $8, %rsp
+; X64-BMI1BMI2-NEXT:    popq %rbx
+; X64-BMI1BMI2-NEXT:    popq %rbp
+; X64-BMI1BMI2-NEXT:    retq
+  %skip = zext i8 %numskipbits to i32
+  %shifted = lshr i32 %val, %skip
+  %numhighbits = sub i8 32, %numlowbits
+  %sh_prom = zext i8 %numhighbits to i32
+  %mask = lshr i32 -1, %sh_prom
+  call void @use32(i32 %mask)
+  %masked = and i32 %mask, %shifted
+  ret i32 %masked
+}
+
+define i32 @bextr32_c2_load(i32* %w, i32 %numskipbits, i32 %numlowbits) nounwind {
+; X86-NOBMI-LABEL: bextr32_c2_load:
 ; X86-NOBMI:       # %bb.0:
 ; X86-NOBMI-NEXT:    pushl %edi
 ; X86-NOBMI-NEXT:    pushl %esi
 ; X86-NOBMI-NEXT:    pushl %eax
 ; X86-NOBMI-NEXT:    movb {{[0-9]+}}(%esp), %cl
-; X86-NOBMI-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NOBMI-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NOBMI-NEXT:    movl (%eax), %edi
 ; X86-NOBMI-NEXT:    shrl %cl, %edi
 ; X86-NOBMI-NEXT:    xorl %ecx, %ecx
 ; X86-NOBMI-NEXT:    subb {{[0-9]+}}(%esp), %cl
@@ -3130,13 +4078,14 @@ define i32 @bextr32_c1_indexzext(i32 %val, i8 %numskipbits, i8 %numlowbits) noun
 ; X86-NOBMI-NEXT:    popl %edi
 ; X86-NOBMI-NEXT:    retl
 ;
-; X86-BMI1NOTBM-LABEL: bextr32_c1_indexzext:
+; X86-BMI1NOTBM-LABEL: bextr32_c2_load:
 ; X86-BMI1NOTBM:       # %bb.0:
 ; X86-BMI1NOTBM-NEXT:    pushl %edi
 ; X86-BMI1NOTBM-NEXT:    pushl %esi
 ; X86-BMI1NOTBM-NEXT:    pushl %eax
 ; X86-BMI1NOTBM-NEXT:    movb {{[0-9]+}}(%esp), %cl
-; X86-BMI1NOTBM-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-BMI1NOTBM-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-BMI1NOTBM-NEXT:    movl (%eax), %edi
 ; X86-BMI1NOTBM-NEXT:    shrl %cl, %edi
 ; X86-BMI1NOTBM-NEXT:    xorl %ecx, %ecx
 ; X86-BMI1NOTBM-NEXT:    subb {{[0-9]+}}(%esp), %cl
@@ -3152,14 +4101,15 @@ define i32 @bextr32_c1_indexzext(i32 %val, i8 %numskipbits, i8 %numlowbits) noun
 ; X86-BMI1NOTBM-NEXT:    popl %edi
 ; X86-BMI1NOTBM-NEXT:    retl
 ;
-; X86-BMI1BMI2-LABEL: bextr32_c1_indexzext:
+; X86-BMI1BMI2-LABEL: bextr32_c2_load:
 ; X86-BMI1BMI2:       # %bb.0:
 ; X86-BMI1BMI2-NEXT:    pushl %ebx
 ; X86-BMI1BMI2-NEXT:    pushl %esi
 ; X86-BMI1BMI2-NEXT:    pushl %eax
 ; X86-BMI1BMI2-NEXT:    movb {{[0-9]+}}(%esp), %bl
-; X86-BMI1BMI2-NEXT:    movb {{[0-9]+}}(%esp), %al
-; X86-BMI1BMI2-NEXT:    shrxl %eax, {{[0-9]+}}(%esp), %esi
+; X86-BMI1BMI2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-BMI1BMI2-NEXT:    movb {{[0-9]+}}(%esp), %cl
+; X86-BMI1BMI2-NEXT:    shrxl %ecx, (%eax), %esi
 ; X86-BMI1BMI2-NEXT:    movl %ebx, %eax
 ; X86-BMI1BMI2-NEXT:    negb %al
 ; X86-BMI1BMI2-NEXT:    movl $-1, %ecx
@@ -3172,57 +4122,57 @@ define i32 @bextr32_c1_indexzext(i32 %val, i8 %numskipbits, i8 %numlowbits) noun
 ; X86-BMI1BMI2-NEXT:    popl %ebx
 ; X86-BMI1BMI2-NEXT:    retl
 ;
-; X64-NOBMI-LABEL: bextr32_c1_indexzext:
+; X64-NOBMI-LABEL: bextr32_c2_load:
 ; X64-NOBMI:       # %bb.0:
 ; X64-NOBMI-NEXT:    pushq %rbp
 ; X64-NOBMI-NEXT:    pushq %rbx
 ; X64-NOBMI-NEXT:    pushq %rax
 ; X64-NOBMI-NEXT:    movl %esi, %ecx
-; X64-NOBMI-NEXT:    movl %edi, %ebx
+; X64-NOBMI-NEXT:    movl (%rdi), %ebp
 ; X64-NOBMI-NEXT:    # kill: def $cl killed $cl killed $ecx
-; X64-NOBMI-NEXT:    shrl %cl, %ebx
+; X64-NOBMI-NEXT:    shrl %cl, %ebp
 ; X64-NOBMI-NEXT:    negb %dl
-; X64-NOBMI-NEXT:    movl $-1, %ebp
+; X64-NOBMI-NEXT:    movl $-1, %ebx
 ; X64-NOBMI-NEXT:    movl %edx, %ecx
-; X64-NOBMI-NEXT:    shrl %cl, %ebp
-; X64-NOBMI-NEXT:    movl %ebp, %edi
+; X64-NOBMI-NEXT:    shrl %cl, %ebx
+; X64-NOBMI-NEXT:    movl %ebx, %edi
 ; X64-NOBMI-NEXT:    callq use32
-; X64-NOBMI-NEXT:    andl %ebx, %ebp
-; X64-NOBMI-NEXT:    movl %ebp, %eax
+; X64-NOBMI-NEXT:    andl %ebp, %ebx
+; X64-NOBMI-NEXT:    movl %ebx, %eax
 ; X64-NOBMI-NEXT:    addq $8, %rsp
 ; X64-NOBMI-NEXT:    popq %rbx
 ; X64-NOBMI-NEXT:    popq %rbp
 ; X64-NOBMI-NEXT:    retq
 ;
-; X64-BMI1NOTBM-LABEL: bextr32_c1_indexzext:
+; X64-BMI1NOTBM-LABEL: bextr32_c2_load:
 ; X64-BMI1NOTBM:       # %bb.0:
 ; X64-BMI1NOTBM-NEXT:    pushq %rbp
 ; X64-BMI1NOTBM-NEXT:    pushq %rbx
 ; X64-BMI1NOTBM-NEXT:    pushq %rax
 ; X64-BMI1NOTBM-NEXT:    movl %esi, %ecx
-; X64-BMI1NOTBM-NEXT:    movl %edi, %ebx
+; X64-BMI1NOTBM-NEXT:    movl (%rdi), %ebp
 ; X64-BMI1NOTBM-NEXT:    # kill: def $cl killed $cl killed $ecx
-; X64-BMI1NOTBM-NEXT:    shrl %cl, %ebx
+; X64-BMI1NOTBM-NEXT:    shrl %cl, %ebp
 ; X64-BMI1NOTBM-NEXT:    negb %dl
-; X64-BMI1NOTBM-NEXT:    movl $-1, %ebp
+; X64-BMI1NOTBM-NEXT:    movl $-1, %ebx
 ; X64-BMI1NOTBM-NEXT:    movl %edx, %ecx
-; X64-BMI1NOTBM-NEXT:    shrl %cl, %ebp
-; X64-BMI1NOTBM-NEXT:    movl %ebp, %edi
+; X64-BMI1NOTBM-NEXT:    shrl %cl, %ebx
+; X64-BMI1NOTBM-NEXT:    movl %ebx, %edi
 ; X64-BMI1NOTBM-NEXT:    callq use32
-; X64-BMI1NOTBM-NEXT:    andl %ebx, %ebp
-; X64-BMI1NOTBM-NEXT:    movl %ebp, %eax
+; X64-BMI1NOTBM-NEXT:    andl %ebp, %ebx
+; X64-BMI1NOTBM-NEXT:    movl %ebx, %eax
 ; X64-BMI1NOTBM-NEXT:    addq $8, %rsp
 ; X64-BMI1NOTBM-NEXT:    popq %rbx
 ; X64-BMI1NOTBM-NEXT:    popq %rbp
 ; X64-BMI1NOTBM-NEXT:    retq
 ;
-; X64-BMI1BMI2-LABEL: bextr32_c1_indexzext:
+; X64-BMI1BMI2-LABEL: bextr32_c2_load:
 ; X64-BMI1BMI2:       # %bb.0:
 ; X64-BMI1BMI2-NEXT:    pushq %rbp
 ; X64-BMI1BMI2-NEXT:    pushq %rbx
 ; X64-BMI1BMI2-NEXT:    pushq %rax
 ; X64-BMI1BMI2-NEXT:    movl %edx, %ebx
-; X64-BMI1BMI2-NEXT:    shrxl %esi, %edi, %ebp
+; X64-BMI1BMI2-NEXT:    shrxl %esi, (%rdi), %ebp
 ; X64-BMI1BMI2-NEXT:    movl %ebx, %eax
 ; X64-BMI1BMI2-NEXT:    negb %al
 ; X64-BMI1BMI2-NEXT:    movl $-1, %ecx
@@ -3233,18 +4183,17 @@ define i32 @bextr32_c1_indexzext(i32 %val, i8 %numskipbits, i8 %numlowbits) noun
 ; X64-BMI1BMI2-NEXT:    popq %rbx
 ; X64-BMI1BMI2-NEXT:    popq %rbp
 ; X64-BMI1BMI2-NEXT:    retq
-  %skip = zext i8 %numskipbits to i32
-  %shifted = lshr i32 %val, %skip
-  %numhighbits = sub i8 32, %numlowbits
-  %sh_prom = zext i8 %numhighbits to i32
-  %mask = lshr i32 -1, %sh_prom
+  %val = load i32, i32* %w
+  %shifted = lshr i32 %val, %numskipbits
+  %numhighbits = sub i32 32, %numlowbits
+  %mask = lshr i32 -1, %numhighbits
   call void @use32(i32 %mask)
   %masked = and i32 %mask, %shifted
   ret i32 %masked
 }
 
-define i32 @bextr32_c2_load(i32* %w, i32 %numskipbits, i32 %numlowbits) nounwind {
-; X86-NOBMI-LABEL: bextr32_c2_load:
+define i32 @bextr32_c3_load_indexzext(i32* %w, i8 %numskipbits, i8 %numlowbits) nounwind {
+; X86-NOBMI-LABEL: bextr32_c3_load_indexzext:
 ; X86-NOBMI:       # %bb.0:
 ; X86-NOBMI-NEXT:    pushl %edi
 ; X86-NOBMI-NEXT:    pushl %esi
@@ -3267,7 +4216,7 @@ define i32 @bextr32_c2_load(i32* %w, i32 %numskipbits, i32 %numlowbits) nounwind
 ; X86-NOBMI-NEXT:    popl %edi
 ; X86-NOBMI-NEXT:    retl
 ;
-; X86-BMI1NOTBM-LABEL: bextr32_c2_load:
+; X86-BMI1NOTBM-LABEL: bextr32_c3_load_indexzext:
 ; X86-BMI1NOTBM:       # %bb.0:
 ; X86-BMI1NOTBM-NEXT:    pushl %edi
 ; X86-BMI1NOTBM-NEXT:    pushl %esi
@@ -3290,7 +4239,7 @@ define i32 @bextr32_c2_load(i32* %w, i32 %numskipbits, i32 %numlowbits) nounwind
 ; X86-BMI1NOTBM-NEXT:    popl %edi
 ; X86-BMI1NOTBM-NEXT:    retl
 ;
-; X86-BMI1BMI2-LABEL: bextr32_c2_load:
+; X86-BMI1BMI2-LABEL: bextr32_c3_load_indexzext:
 ; X86-BMI1BMI2:       # %bb.0:
 ; X86-BMI1BMI2-NEXT:    pushl %ebx
 ; X86-BMI1BMI2-NEXT:    pushl %esi
@@ -3311,7 +4260,7 @@ define i32 @bextr32_c2_load(i32* %w, i32 %numskipbits, i32 %numlowbits) nounwind
 ; X86-BMI1BMI2-NEXT:    popl %ebx
 ; X86-BMI1BMI2-NEXT:    retl
 ;
-; X64-NOBMI-LABEL: bextr32_c2_load:
+; X64-NOBMI-LABEL: bextr32_c3_load_indexzext:
 ; X64-NOBMI:       # %bb.0:
 ; X64-NOBMI-NEXT:    pushq %rbp
 ; X64-NOBMI-NEXT:    pushq %rbx
@@ -3333,7 +4282,7 @@ define i32 @bextr32_c2_load(i32* %w, i32 %numskipbits, i32 %numlowbits) nounwind
 ; X64-NOBMI-NEXT:    popq %rbp
 ; X64-NOBMI-NEXT:    retq
 ;
-; X64-BMI1NOTBM-LABEL: bextr32_c2_load:
+; X64-BMI1NOTBM-LABEL: bextr32_c3_load_indexzext:
 ; X64-BMI1NOTBM:       # %bb.0:
 ; X64-BMI1NOTBM-NEXT:    pushq %rbp
 ; X64-BMI1NOTBM-NEXT:    pushq %rbx
@@ -3355,7 +4304,7 @@ define i32 @bextr32_c2_load(i32* %w, i32 %numskipbits, i32 %numlowbits) nounwind
 ; X64-BMI1NOTBM-NEXT:    popq %rbp
 ; X64-BMI1NOTBM-NEXT:    retq
 ;
-; X64-BMI1BMI2-LABEL: bextr32_c2_load:
+; X64-BMI1BMI2-LABEL: bextr32_c3_load_indexzext:
 ; X64-BMI1BMI2:       # %bb.0:
 ; X64-BMI1BMI2-NEXT:    pushq %rbp
 ; X64-BMI1BMI2-NEXT:    pushq %rbx
@@ -3373,23 +4322,160 @@ define i32 @bextr32_c2_load(i32* %w, i32 %numskipbits, i32 %numlowbits) nounwind
 ; X64-BMI1BMI2-NEXT:    popq %rbp
 ; X64-BMI1BMI2-NEXT:    retq
   %val = load i32, i32* %w
+  %skip = zext i8 %numskipbits to i32
+  %shifted = lshr i32 %val, %skip
+  %numhighbits = sub i8 32, %numlowbits
+  %sh_prom = zext i8 %numhighbits to i32
+  %mask = lshr i32 -1, %sh_prom
+  call void @use32(i32 %mask)
+  %masked = and i32 %mask, %shifted
+  ret i32 %masked
+}
+
+define i32 @bextr32_c4_commutative(i32 %val, i32 %numskipbits, i32 %numlowbits) nounwind {
+; X86-NOBMI-LABEL: bextr32_c4_commutative:
+; X86-NOBMI:       # %bb.0:
+; X86-NOBMI-NEXT:    pushl %edi
+; X86-NOBMI-NEXT:    pushl %esi
+; X86-NOBMI-NEXT:    pushl %eax
+; X86-NOBMI-NEXT:    movb {{[0-9]+}}(%esp), %cl
+; X86-NOBMI-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NOBMI-NEXT:    shrl %cl, %edi
+; X86-NOBMI-NEXT:    xorl %ecx, %ecx
+; X86-NOBMI-NEXT:    subb {{[0-9]+}}(%esp), %cl
+; X86-NOBMI-NEXT:    movl $-1, %esi
+; X86-NOBMI-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X86-NOBMI-NEXT:    shrl %cl, %esi
+; X86-NOBMI-NEXT:    movl %esi, (%esp)
+; X86-NOBMI-NEXT:    calll use32
+; X86-NOBMI-NEXT:    andl %edi, %esi
+; X86-NOBMI-NEXT:    movl %esi, %eax
+; X86-NOBMI-NEXT:    addl $4, %esp
+; X86-NOBMI-NEXT:    popl %esi
+; X86-NOBMI-NEXT:    popl %edi
+; X86-NOBMI-NEXT:    retl
+;
+; X86-BMI1NOTBM-LABEL: bextr32_c4_commutative:
+; X86-BMI1NOTBM:       # %bb.0:
+; X86-BMI1NOTBM-NEXT:    pushl %edi
+; X86-BMI1NOTBM-NEXT:    pushl %esi
+; X86-BMI1NOTBM-NEXT:    pushl %eax
+; X86-BMI1NOTBM-NEXT:    movb {{[0-9]+}}(%esp), %cl
+; X86-BMI1NOTBM-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-BMI1NOTBM-NEXT:    shrl %cl, %edi
+; X86-BMI1NOTBM-NEXT:    xorl %ecx, %ecx
+; X86-BMI1NOTBM-NEXT:    subb {{[0-9]+}}(%esp), %cl
+; X86-BMI1NOTBM-NEXT:    movl $-1, %esi
+; X86-BMI1NOTBM-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X86-BMI1NOTBM-NEXT:    shrl %cl, %esi
+; X86-BMI1NOTBM-NEXT:    movl %esi, (%esp)
+; X86-BMI1NOTBM-NEXT:    calll use32
+; X86-BMI1NOTBM-NEXT:    andl %edi, %esi
+; X86-BMI1NOTBM-NEXT:    movl %esi, %eax
+; X86-BMI1NOTBM-NEXT:    addl $4, %esp
+; X86-BMI1NOTBM-NEXT:    popl %esi
+; X86-BMI1NOTBM-NEXT:    popl %edi
+; X86-BMI1NOTBM-NEXT:    retl
+;
+; X86-BMI1BMI2-LABEL: bextr32_c4_commutative:
+; X86-BMI1BMI2:       # %bb.0:
+; X86-BMI1BMI2-NEXT:    pushl %ebx
+; X86-BMI1BMI2-NEXT:    pushl %esi
+; X86-BMI1BMI2-NEXT:    pushl %eax
+; X86-BMI1BMI2-NEXT:    movb {{[0-9]+}}(%esp), %bl
+; X86-BMI1BMI2-NEXT:    movb {{[0-9]+}}(%esp), %al
+; X86-BMI1BMI2-NEXT:    shrxl %eax, {{[0-9]+}}(%esp), %esi
+; X86-BMI1BMI2-NEXT:    movl %ebx, %eax
+; X86-BMI1BMI2-NEXT:    negb %al
+; X86-BMI1BMI2-NEXT:    movl $-1, %ecx
+; X86-BMI1BMI2-NEXT:    shrxl %eax, %ecx, %eax
+; X86-BMI1BMI2-NEXT:    movl %eax, (%esp)
+; X86-BMI1BMI2-NEXT:    calll use32
+; X86-BMI1BMI2-NEXT:    bzhil %ebx, %esi, %eax
+; X86-BMI1BMI2-NEXT:    addl $4, %esp
+; X86-BMI1BMI2-NEXT:    popl %esi
+; X86-BMI1BMI2-NEXT:    popl %ebx
+; X86-BMI1BMI2-NEXT:    retl
+;
+; X64-NOBMI-LABEL: bextr32_c4_commutative:
+; X64-NOBMI:       # %bb.0:
+; X64-NOBMI-NEXT:    pushq %rbp
+; X64-NOBMI-NEXT:    pushq %rbx
+; X64-NOBMI-NEXT:    pushq %rax
+; X64-NOBMI-NEXT:    movl %esi, %ecx
+; X64-NOBMI-NEXT:    movl %edi, %ebx
+; X64-NOBMI-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X64-NOBMI-NEXT:    shrl %cl, %ebx
+; X64-NOBMI-NEXT:    negb %dl
+; X64-NOBMI-NEXT:    movl $-1, %ebp
+; X64-NOBMI-NEXT:    movl %edx, %ecx
+; X64-NOBMI-NEXT:    shrl %cl, %ebp
+; X64-NOBMI-NEXT:    movl %ebp, %edi
+; X64-NOBMI-NEXT:    callq use32
+; X64-NOBMI-NEXT:    andl %ebx, %ebp
+; X64-NOBMI-NEXT:    movl %ebp, %eax
+; X64-NOBMI-NEXT:    addq $8, %rsp
+; X64-NOBMI-NEXT:    popq %rbx
+; X64-NOBMI-NEXT:    popq %rbp
+; X64-NOBMI-NEXT:    retq
+;
+; X64-BMI1NOTBM-LABEL: bextr32_c4_commutative:
+; X64-BMI1NOTBM:       # %bb.0:
+; X64-BMI1NOTBM-NEXT:    pushq %rbp
+; X64-BMI1NOTBM-NEXT:    pushq %rbx
+; X64-BMI1NOTBM-NEXT:    pushq %rax
+; X64-BMI1NOTBM-NEXT:    movl %esi, %ecx
+; X64-BMI1NOTBM-NEXT:    movl %edi, %ebx
+; X64-BMI1NOTBM-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X64-BMI1NOTBM-NEXT:    shrl %cl, %ebx
+; X64-BMI1NOTBM-NEXT:    negb %dl
+; X64-BMI1NOTBM-NEXT:    movl $-1, %ebp
+; X64-BMI1NOTBM-NEXT:    movl %edx, %ecx
+; X64-BMI1NOTBM-NEXT:    shrl %cl, %ebp
+; X64-BMI1NOTBM-NEXT:    movl %ebp, %edi
+; X64-BMI1NOTBM-NEXT:    callq use32
+; X64-BMI1NOTBM-NEXT:    andl %ebx, %ebp
+; X64-BMI1NOTBM-NEXT:    movl %ebp, %eax
+; X64-BMI1NOTBM-NEXT:    addq $8, %rsp
+; X64-BMI1NOTBM-NEXT:    popq %rbx
+; X64-BMI1NOTBM-NEXT:    popq %rbp
+; X64-BMI1NOTBM-NEXT:    retq
+;
+; X64-BMI1BMI2-LABEL: bextr32_c4_commutative:
+; X64-BMI1BMI2:       # %bb.0:
+; X64-BMI1BMI2-NEXT:    pushq %rbp
+; X64-BMI1BMI2-NEXT:    pushq %rbx
+; X64-BMI1BMI2-NEXT:    pushq %rax
+; X64-BMI1BMI2-NEXT:    movl %edx, %ebx
+; X64-BMI1BMI2-NEXT:    shrxl %esi, %edi, %ebp
+; X64-BMI1BMI2-NEXT:    movl %ebx, %eax
+; X64-BMI1BMI2-NEXT:    negb %al
+; X64-BMI1BMI2-NEXT:    movl $-1, %ecx
+; X64-BMI1BMI2-NEXT:    shrxl %eax, %ecx, %edi
+; X64-BMI1BMI2-NEXT:    callq use32
+; X64-BMI1BMI2-NEXT:    bzhil %ebx, %ebp, %eax
+; X64-BMI1BMI2-NEXT:    addq $8, %rsp
+; X64-BMI1BMI2-NEXT:    popq %rbx
+; X64-BMI1BMI2-NEXT:    popq %rbp
+; X64-BMI1BMI2-NEXT:    retq
   %shifted = lshr i32 %val, %numskipbits
   %numhighbits = sub i32 32, %numlowbits
   %mask = lshr i32 -1, %numhighbits
   call void @use32(i32 %mask)
-  %masked = and i32 %mask, %shifted
+  %masked = and i32 %shifted, %mask ; swapped order
   ret i32 %masked
 }
 
-define i32 @bextr32_c3_load_indexzext(i32* %w, i8 %numskipbits, i8 %numlowbits) nounwind {
-; X86-NOBMI-LABEL: bextr32_c3_load_indexzext:
+define i32 @bextr32_c5_skipextrauses(i32 %val, i32 %numskipbits, i32 %numlowbits) nounwind {
+; X86-NOBMI-LABEL: bextr32_c5_skipextrauses:
 ; X86-NOBMI:       # %bb.0:
+; X86-NOBMI-NEXT:    pushl %ebx
 ; X86-NOBMI-NEXT:    pushl %edi
 ; X86-NOBMI-NEXT:    pushl %esi
-; X86-NOBMI-NEXT:    pushl %eax
-; X86-NOBMI-NEXT:    movb {{[0-9]+}}(%esp), %cl
-; X86-NOBMI-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NOBMI-NEXT:    movl (%eax), %edi
+; X86-NOBMI-NEXT:    subl $16, %esp
+; X86-NOBMI-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NOBMI-NEXT:    movl {{[0-9]+}}(%esp), %ebx
+; X86-NOBMI-NEXT:    movl %ebx, %ecx
 ; X86-NOBMI-NEXT:    shrl %cl, %edi
 ; X86-NOBMI-NEXT:    xorl %ecx, %ecx
 ; X86-NOBMI-NEXT:    subb {{[0-9]+}}(%esp), %cl
@@ -3399,20 +4485,24 @@ define i32 @bextr32_c3_load_indexzext(i32* %w, i8 %numskipbits, i8 %numlowbits)
 ; X86-NOBMI-NEXT:    movl %esi, (%esp)
 ; X86-NOBMI-NEXT:    calll use32
 ; X86-NOBMI-NEXT:    andl %edi, %esi
+; X86-NOBMI-NEXT:    movl %ebx, (%esp)
+; X86-NOBMI-NEXT:    calll use32
 ; X86-NOBMI-NEXT:    movl %esi, %eax
-; X86-NOBMI-NEXT:    addl $4, %esp
+; X86-NOBMI-NEXT:    addl $16, %esp
 ; X86-NOBMI-NEXT:    popl %esi
 ; X86-NOBMI-NEXT:    popl %edi
+; X86-NOBMI-NEXT:    popl %ebx
 ; X86-NOBMI-NEXT:    retl
 ;
-; X86-BMI1NOTBM-LABEL: bextr32_c3_load_indexzext:
+; X86-BMI1NOTBM-LABEL: bextr32_c5_skipextrauses:
 ; X86-BMI1NOTBM:       # %bb.0:
+; X86-BMI1NOTBM-NEXT:    pushl %ebx
 ; X86-BMI1NOTBM-NEXT:    pushl %edi
 ; X86-BMI1NOTBM-NEXT:    pushl %esi
-; X86-BMI1NOTBM-NEXT:    pushl %eax
-; X86-BMI1NOTBM-NEXT:    movb {{[0-9]+}}(%esp), %cl
-; X86-BMI1NOTBM-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-BMI1NOTBM-NEXT:    movl (%eax), %edi
+; X86-BMI1NOTBM-NEXT:    subl $16, %esp
+; X86-BMI1NOTBM-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-BMI1NOTBM-NEXT:    movl {{[0-9]+}}(%esp), %ebx
+; X86-BMI1NOTBM-NEXT:    movl %ebx, %ecx
 ; X86-BMI1NOTBM-NEXT:    shrl %cl, %edi
 ; X86-BMI1NOTBM-NEXT:    xorl %ecx, %ecx
 ; X86-BMI1NOTBM-NEXT:    subb {{[0-9]+}}(%esp), %cl
@@ -3422,41 +4512,48 @@ define i32 @bextr32_c3_load_indexzext(i32* %w, i8 %numskipbits, i8 %numlowbits)
 ; X86-BMI1NOTBM-NEXT:    movl %esi, (%esp)
 ; X86-BMI1NOTBM-NEXT:    calll use32
 ; X86-BMI1NOTBM-NEXT:    andl %edi, %esi
+; X86-BMI1NOTBM-NEXT:    movl %ebx, (%esp)
+; X86-BMI1NOTBM-NEXT:    calll use32
 ; X86-BMI1NOTBM-NEXT:    movl %esi, %eax
-; X86-BMI1NOTBM-NEXT:    addl $4, %esp
+; X86-BMI1NOTBM-NEXT:    addl $16, %esp
 ; X86-BMI1NOTBM-NEXT:    popl %esi
 ; X86-BMI1NOTBM-NEXT:    popl %edi
+; X86-BMI1NOTBM-NEXT:    popl %ebx
 ; X86-BMI1NOTBM-NEXT:    retl
 ;
-; X86-BMI1BMI2-LABEL: bextr32_c3_load_indexzext:
+; X86-BMI1BMI2-LABEL: bextr32_c5_skipextrauses:
 ; X86-BMI1BMI2:       # %bb.0:
 ; X86-BMI1BMI2-NEXT:    pushl %ebx
+; X86-BMI1BMI2-NEXT:    pushl %edi
 ; X86-BMI1BMI2-NEXT:    pushl %esi
-; X86-BMI1BMI2-NEXT:    pushl %eax
+; X86-BMI1BMI2-NEXT:    subl $16, %esp
 ; X86-BMI1BMI2-NEXT:    movb {{[0-9]+}}(%esp), %bl
-; X86-BMI1BMI2-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-BMI1BMI2-NEXT:    movb {{[0-9]+}}(%esp), %cl
-; X86-BMI1BMI2-NEXT:    shrxl %ecx, (%eax), %esi
+; X86-BMI1BMI2-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-BMI1BMI2-NEXT:    shrxl %edi, {{[0-9]+}}(%esp), %esi
 ; X86-BMI1BMI2-NEXT:    movl %ebx, %eax
 ; X86-BMI1BMI2-NEXT:    negb %al
 ; X86-BMI1BMI2-NEXT:    movl $-1, %ecx
 ; X86-BMI1BMI2-NEXT:    shrxl %eax, %ecx, %eax
 ; X86-BMI1BMI2-NEXT:    movl %eax, (%esp)
 ; X86-BMI1BMI2-NEXT:    calll use32
-; X86-BMI1BMI2-NEXT:    bzhil %ebx, %esi, %eax
-; X86-BMI1BMI2-NEXT:    addl $4, %esp
+; X86-BMI1BMI2-NEXT:    bzhil %ebx, %esi, %esi
+; X86-BMI1BMI2-NEXT:    movl %edi, (%esp)
+; X86-BMI1BMI2-NEXT:    calll use32
+; X86-BMI1BMI2-NEXT:    movl %esi, %eax
+; X86-BMI1BMI2-NEXT:    addl $16, %esp
 ; X86-BMI1BMI2-NEXT:    popl %esi
+; X86-BMI1BMI2-NEXT:    popl %edi
 ; X86-BMI1BMI2-NEXT:    popl %ebx
 ; X86-BMI1BMI2-NEXT:    retl
 ;
-; X64-NOBMI-LABEL: bextr32_c3_load_indexzext:
+; X64-NOBMI-LABEL: bextr32_c5_skipextrauses:
 ; X64-NOBMI:       # %bb.0:
 ; X64-NOBMI-NEXT:    pushq %rbp
+; X64-NOBMI-NEXT:    pushq %r14
 ; X64-NOBMI-NEXT:    pushq %rbx
-; X64-NOBMI-NEXT:    pushq %rax
-; X64-NOBMI-NEXT:    movl %esi, %ecx
-; X64-NOBMI-NEXT:    movl (%rdi), %ebp
-; X64-NOBMI-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X64-NOBMI-NEXT:    movl %esi, %r14d
+; X64-NOBMI-NEXT:    movl %edi, %ebp
+; X64-NOBMI-NEXT:    movl %r14d, %ecx
 ; X64-NOBMI-NEXT:    shrl %cl, %ebp
 ; X64-NOBMI-NEXT:    negb %dl
 ; X64-NOBMI-NEXT:    movl $-1, %ebx
@@ -3465,20 +4562,22 @@ define i32 @bextr32_c3_load_indexzext(i32* %w, i8 %numskipbits, i8 %numlowbits)
 ; X64-NOBMI-NEXT:    movl %ebx, %edi
 ; X64-NOBMI-NEXT:    callq use32
 ; X64-NOBMI-NEXT:    andl %ebp, %ebx
+; X64-NOBMI-NEXT:    movl %r14d, %edi
+; X64-NOBMI-NEXT:    callq use32
 ; X64-NOBMI-NEXT:    movl %ebx, %eax
-; X64-NOBMI-NEXT:    addq $8, %rsp
 ; X64-NOBMI-NEXT:    popq %rbx
+; X64-NOBMI-NEXT:    popq %r14
 ; X64-NOBMI-NEXT:    popq %rbp
 ; X64-NOBMI-NEXT:    retq
 ;
-; X64-BMI1NOTBM-LABEL: bextr32_c3_load_indexzext:
+; X64-BMI1NOTBM-LABEL: bextr32_c5_skipextrauses:
 ; X64-BMI1NOTBM:       # %bb.0:
 ; X64-BMI1NOTBM-NEXT:    pushq %rbp
+; X64-BMI1NOTBM-NEXT:    pushq %r14
 ; X64-BMI1NOTBM-NEXT:    pushq %rbx
-; X64-BMI1NOTBM-NEXT:    pushq %rax
-; X64-BMI1NOTBM-NEXT:    movl %esi, %ecx
-; X64-BMI1NOTBM-NEXT:    movl (%rdi), %ebp
-; X64-BMI1NOTBM-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X64-BMI1NOTBM-NEXT:    movl %esi, %r14d
+; X64-BMI1NOTBM-NEXT:    movl %edi, %ebp
+; X64-BMI1NOTBM-NEXT:    movl %r14d, %ecx
 ; X64-BMI1NOTBM-NEXT:    shrl %cl, %ebp
 ; X64-BMI1NOTBM-NEXT:    negb %dl
 ; X64-BMI1NOTBM-NEXT:    movl $-1, %ebx
@@ -3487,336 +4586,469 @@ define i32 @bextr32_c3_load_indexzext(i32* %w, i8 %numskipbits, i8 %numlowbits)
 ; X64-BMI1NOTBM-NEXT:    movl %ebx, %edi
 ; X64-BMI1NOTBM-NEXT:    callq use32
 ; X64-BMI1NOTBM-NEXT:    andl %ebp, %ebx
+; X64-BMI1NOTBM-NEXT:    movl %r14d, %edi
+; X64-BMI1NOTBM-NEXT:    callq use32
 ; X64-BMI1NOTBM-NEXT:    movl %ebx, %eax
-; X64-BMI1NOTBM-NEXT:    addq $8, %rsp
 ; X64-BMI1NOTBM-NEXT:    popq %rbx
+; X64-BMI1NOTBM-NEXT:    popq %r14
 ; X64-BMI1NOTBM-NEXT:    popq %rbp
 ; X64-BMI1NOTBM-NEXT:    retq
 ;
-; X64-BMI1BMI2-LABEL: bextr32_c3_load_indexzext:
+; X64-BMI1BMI2-LABEL: bextr32_c5_skipextrauses:
 ; X64-BMI1BMI2:       # %bb.0:
 ; X64-BMI1BMI2-NEXT:    pushq %rbp
+; X64-BMI1BMI2-NEXT:    pushq %r14
 ; X64-BMI1BMI2-NEXT:    pushq %rbx
-; X64-BMI1BMI2-NEXT:    pushq %rax
 ; X64-BMI1BMI2-NEXT:    movl %edx, %ebx
-; X64-BMI1BMI2-NEXT:    shrxl %esi, (%rdi), %ebp
+; X64-BMI1BMI2-NEXT:    movl %esi, %ebp
+; X64-BMI1BMI2-NEXT:    shrxl %esi, %edi, %r14d
 ; X64-BMI1BMI2-NEXT:    movl %ebx, %eax
 ; X64-BMI1BMI2-NEXT:    negb %al
 ; X64-BMI1BMI2-NEXT:    movl $-1, %ecx
 ; X64-BMI1BMI2-NEXT:    shrxl %eax, %ecx, %edi
 ; X64-BMI1BMI2-NEXT:    callq use32
-; X64-BMI1BMI2-NEXT:    bzhil %ebx, %ebp, %eax
-; X64-BMI1BMI2-NEXT:    addq $8, %rsp
+; X64-BMI1BMI2-NEXT:    bzhil %ebx, %r14d, %ebx
+; X64-BMI1BMI2-NEXT:    movl %ebp, %edi
+; X64-BMI1BMI2-NEXT:    callq use32
+; X64-BMI1BMI2-NEXT:    movl %ebx, %eax
 ; X64-BMI1BMI2-NEXT:    popq %rbx
+; X64-BMI1BMI2-NEXT:    popq %r14
 ; X64-BMI1BMI2-NEXT:    popq %rbp
 ; X64-BMI1BMI2-NEXT:    retq
-  %val = load i32, i32* %w
-  %skip = zext i8 %numskipbits to i32
-  %shifted = lshr i32 %val, %skip
-  %numhighbits = sub i8 32, %numlowbits
-  %sh_prom = zext i8 %numhighbits to i32
-  %mask = lshr i32 -1, %sh_prom
+  %shifted = lshr i32 %val, %numskipbits
+  %numhighbits = sub i32 32, %numlowbits
+  %mask = lshr i32 -1, %numhighbits
   call void @use32(i32 %mask)
   %masked = and i32 %mask, %shifted
+  call void @use32(i32 %numskipbits)
   ret i32 %masked
 }
 
-define i32 @bextr32_c4_commutative(i32 %val, i32 %numskipbits, i32 %numlowbits) nounwind {
-; X86-NOBMI-LABEL: bextr32_c4_commutative:
+; 64-bit
+
+define i64 @bextr64_c0(i64 %val, i64 %numskipbits, i64 %numlowbits) nounwind {
+; X86-NOBMI-LABEL: bextr64_c0:
 ; X86-NOBMI:       # %bb.0:
+; X86-NOBMI-NEXT:    pushl %ebp
+; X86-NOBMI-NEXT:    pushl %ebx
 ; X86-NOBMI-NEXT:    pushl %edi
 ; X86-NOBMI-NEXT:    pushl %esi
-; X86-NOBMI-NEXT:    pushl %eax
+; X86-NOBMI-NEXT:    subl $12, %esp
 ; X86-NOBMI-NEXT:    movb {{[0-9]+}}(%esp), %cl
-; X86-NOBMI-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NOBMI-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NOBMI-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NOBMI-NEXT:    movl %eax, %edi
 ; X86-NOBMI-NEXT:    shrl %cl, %edi
-; X86-NOBMI-NEXT:    xorl %ecx, %ecx
+; X86-NOBMI-NEXT:    shrdl %cl, %eax, %esi
+; X86-NOBMI-NEXT:    testb $32, %cl
+; X86-NOBMI-NEXT:    je .LBB39_2
+; X86-NOBMI-NEXT:  # %bb.1:
+; X86-NOBMI-NEXT:    movl %edi, %esi
+; X86-NOBMI-NEXT:    xorl %edi, %edi
+; X86-NOBMI-NEXT:  .LBB39_2:
+; X86-NOBMI-NEXT:    movb $64, %cl
 ; X86-NOBMI-NEXT:    subb {{[0-9]+}}(%esp), %cl
-; X86-NOBMI-NEXT:    movl $-1, %esi
-; X86-NOBMI-NEXT:    # kill: def $cl killed $cl killed $ecx
-; X86-NOBMI-NEXT:    shrl %cl, %esi
-; X86-NOBMI-NEXT:    movl %esi, (%esp)
-; X86-NOBMI-NEXT:    calll use32
-; X86-NOBMI-NEXT:    andl %edi, %esi
+; X86-NOBMI-NEXT:    movl $-1, %ebp
+; X86-NOBMI-NEXT:    movl $-1, %ebx
+; X86-NOBMI-NEXT:    shrl %cl, %ebx
+; X86-NOBMI-NEXT:    shrdl %cl, %ebp, %ebp
+; X86-NOBMI-NEXT:    testb $32, %cl
+; X86-NOBMI-NEXT:    je .LBB39_4
+; X86-NOBMI-NEXT:  # %bb.3:
+; X86-NOBMI-NEXT:    movl %ebx, %ebp
+; X86-NOBMI-NEXT:    xorl %ebx, %ebx
+; X86-NOBMI-NEXT:  .LBB39_4:
+; X86-NOBMI-NEXT:    subl $8, %esp
+; X86-NOBMI-NEXT:    pushl %ebx
+; X86-NOBMI-NEXT:    pushl %ebp
+; X86-NOBMI-NEXT:    calll use64
+; X86-NOBMI-NEXT:    addl $16, %esp
+; X86-NOBMI-NEXT:    andl %ebp, %esi
+; X86-NOBMI-NEXT:    andl %ebx, %edi
 ; X86-NOBMI-NEXT:    movl %esi, %eax
-; X86-NOBMI-NEXT:    addl $4, %esp
+; X86-NOBMI-NEXT:    movl %edi, %edx
+; X86-NOBMI-NEXT:    addl $12, %esp
 ; X86-NOBMI-NEXT:    popl %esi
 ; X86-NOBMI-NEXT:    popl %edi
+; X86-NOBMI-NEXT:    popl %ebx
+; X86-NOBMI-NEXT:    popl %ebp
 ; X86-NOBMI-NEXT:    retl
 ;
-; X86-BMI1NOTBM-LABEL: bextr32_c4_commutative:
+; X86-BMI1NOTBM-LABEL: bextr64_c0:
 ; X86-BMI1NOTBM:       # %bb.0:
+; X86-BMI1NOTBM-NEXT:    pushl %ebp
+; X86-BMI1NOTBM-NEXT:    pushl %ebx
 ; X86-BMI1NOTBM-NEXT:    pushl %edi
 ; X86-BMI1NOTBM-NEXT:    pushl %esi
-; X86-BMI1NOTBM-NEXT:    pushl %eax
+; X86-BMI1NOTBM-NEXT:    subl $12, %esp
 ; X86-BMI1NOTBM-NEXT:    movb {{[0-9]+}}(%esp), %cl
-; X86-BMI1NOTBM-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-BMI1NOTBM-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-BMI1NOTBM-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-BMI1NOTBM-NEXT:    movl %eax, %edi
 ; X86-BMI1NOTBM-NEXT:    shrl %cl, %edi
-; X86-BMI1NOTBM-NEXT:    xorl %ecx, %ecx
+; X86-BMI1NOTBM-NEXT:    shrdl %cl, %eax, %esi
+; X86-BMI1NOTBM-NEXT:    testb $32, %cl
+; X86-BMI1NOTBM-NEXT:    je .LBB39_2
+; X86-BMI1NOTBM-NEXT:  # %bb.1:
+; X86-BMI1NOTBM-NEXT:    movl %edi, %esi
+; X86-BMI1NOTBM-NEXT:    xorl %edi, %edi
+; X86-BMI1NOTBM-NEXT:  .LBB39_2:
+; X86-BMI1NOTBM-NEXT:    movb $64, %cl
 ; X86-BMI1NOTBM-NEXT:    subb {{[0-9]+}}(%esp), %cl
-; X86-BMI1NOTBM-NEXT:    movl $-1, %esi
-; X86-BMI1NOTBM-NEXT:    # kill: def $cl killed $cl killed $ecx
-; X86-BMI1NOTBM-NEXT:    shrl %cl, %esi
-; X86-BMI1NOTBM-NEXT:    movl %esi, (%esp)
-; X86-BMI1NOTBM-NEXT:    calll use32
-; X86-BMI1NOTBM-NEXT:    andl %edi, %esi
+; X86-BMI1NOTBM-NEXT:    movl $-1, %ebp
+; X86-BMI1NOTBM-NEXT:    movl $-1, %ebx
+; X86-BMI1NOTBM-NEXT:    shrl %cl, %ebx
+; X86-BMI1NOTBM-NEXT:    shrdl %cl, %ebp, %ebp
+; X86-BMI1NOTBM-NEXT:    testb $32, %cl
+; X86-BMI1NOTBM-NEXT:    je .LBB39_4
+; X86-BMI1NOTBM-NEXT:  # %bb.3:
+; X86-BMI1NOTBM-NEXT:    movl %ebx, %ebp
+; X86-BMI1NOTBM-NEXT:    xorl %ebx, %ebx
+; X86-BMI1NOTBM-NEXT:  .LBB39_4:
+; X86-BMI1NOTBM-NEXT:    subl $8, %esp
+; X86-BMI1NOTBM-NEXT:    pushl %ebx
+; X86-BMI1NOTBM-NEXT:    pushl %ebp
+; X86-BMI1NOTBM-NEXT:    calll use64
+; X86-BMI1NOTBM-NEXT:    addl $16, %esp
+; X86-BMI1NOTBM-NEXT:    andl %ebp, %esi
+; X86-BMI1NOTBM-NEXT:    andl %ebx, %edi
 ; X86-BMI1NOTBM-NEXT:    movl %esi, %eax
-; X86-BMI1NOTBM-NEXT:    addl $4, %esp
+; X86-BMI1NOTBM-NEXT:    movl %edi, %edx
+; X86-BMI1NOTBM-NEXT:    addl $12, %esp
 ; X86-BMI1NOTBM-NEXT:    popl %esi
 ; X86-BMI1NOTBM-NEXT:    popl %edi
+; X86-BMI1NOTBM-NEXT:    popl %ebx
+; X86-BMI1NOTBM-NEXT:    popl %ebp
 ; X86-BMI1NOTBM-NEXT:    retl
 ;
-; X86-BMI1BMI2-LABEL: bextr32_c4_commutative:
+; X86-BMI1BMI2-LABEL: bextr64_c0:
 ; X86-BMI1BMI2:       # %bb.0:
+; X86-BMI1BMI2-NEXT:    pushl %ebp
 ; X86-BMI1BMI2-NEXT:    pushl %ebx
+; X86-BMI1BMI2-NEXT:    pushl %edi
 ; X86-BMI1BMI2-NEXT:    pushl %esi
-; X86-BMI1BMI2-NEXT:    pushl %eax
-; X86-BMI1BMI2-NEXT:    movb {{[0-9]+}}(%esp), %bl
-; X86-BMI1BMI2-NEXT:    movb {{[0-9]+}}(%esp), %al
-; X86-BMI1BMI2-NEXT:    shrxl %eax, {{[0-9]+}}(%esp), %esi
-; X86-BMI1BMI2-NEXT:    movl %ebx, %eax
-; X86-BMI1BMI2-NEXT:    negb %al
-; X86-BMI1BMI2-NEXT:    movl $-1, %ecx
-; X86-BMI1BMI2-NEXT:    shrxl %eax, %ecx, %eax
-; X86-BMI1BMI2-NEXT:    movl %eax, (%esp)
-; X86-BMI1BMI2-NEXT:    calll use32
-; X86-BMI1BMI2-NEXT:    bzhil %ebx, %esi, %eax
-; X86-BMI1BMI2-NEXT:    addl $4, %esp
+; X86-BMI1BMI2-NEXT:    subl $12, %esp
+; X86-BMI1BMI2-NEXT:    movb {{[0-9]+}}(%esp), %cl
+; X86-BMI1BMI2-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-BMI1BMI2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-BMI1BMI2-NEXT:    shrdl %cl, %eax, %esi
+; X86-BMI1BMI2-NEXT:    shrxl %ecx, %eax, %edi
+; X86-BMI1BMI2-NEXT:    testb $32, %cl
+; X86-BMI1BMI2-NEXT:    je .LBB39_2
+; X86-BMI1BMI2-NEXT:  # %bb.1:
+; X86-BMI1BMI2-NEXT:    movl %edi, %esi
+; X86-BMI1BMI2-NEXT:    xorl %edi, %edi
+; X86-BMI1BMI2-NEXT:  .LBB39_2:
+; X86-BMI1BMI2-NEXT:    movb $64, %cl
+; X86-BMI1BMI2-NEXT:    subb {{[0-9]+}}(%esp), %cl
+; X86-BMI1BMI2-NEXT:    movl $-1, %ebx
+; X86-BMI1BMI2-NEXT:    shrxl %ecx, %ebx, %ebp
+; X86-BMI1BMI2-NEXT:    shrdl %cl, %ebx, %ebx
+; X86-BMI1BMI2-NEXT:    testb $32, %cl
+; X86-BMI1BMI2-NEXT:    je .LBB39_4
+; X86-BMI1BMI2-NEXT:  # %bb.3:
+; X86-BMI1BMI2-NEXT:    movl %ebp, %ebx
+; X86-BMI1BMI2-NEXT:    xorl %ebp, %ebp
+; X86-BMI1BMI2-NEXT:  .LBB39_4:
+; X86-BMI1BMI2-NEXT:    subl $8, %esp
+; X86-BMI1BMI2-NEXT:    pushl %ebp
+; X86-BMI1BMI2-NEXT:    pushl %ebx
+; X86-BMI1BMI2-NEXT:    calll use64
+; X86-BMI1BMI2-NEXT:    addl $16, %esp
+; X86-BMI1BMI2-NEXT:    andl %ebx, %esi
+; X86-BMI1BMI2-NEXT:    andl %ebp, %edi
+; X86-BMI1BMI2-NEXT:    movl %esi, %eax
+; X86-BMI1BMI2-NEXT:    movl %edi, %edx
+; X86-BMI1BMI2-NEXT:    addl $12, %esp
 ; X86-BMI1BMI2-NEXT:    popl %esi
+; X86-BMI1BMI2-NEXT:    popl %edi
 ; X86-BMI1BMI2-NEXT:    popl %ebx
+; X86-BMI1BMI2-NEXT:    popl %ebp
 ; X86-BMI1BMI2-NEXT:    retl
 ;
-; X64-NOBMI-LABEL: bextr32_c4_commutative:
+; X64-NOBMI-LABEL: bextr64_c0:
 ; X64-NOBMI:       # %bb.0:
-; X64-NOBMI-NEXT:    pushq %rbp
+; X64-NOBMI-NEXT:    pushq %r14
 ; X64-NOBMI-NEXT:    pushq %rbx
 ; X64-NOBMI-NEXT:    pushq %rax
-; X64-NOBMI-NEXT:    movl %esi, %ecx
-; X64-NOBMI-NEXT:    movl %edi, %ebx
-; X64-NOBMI-NEXT:    # kill: def $cl killed $cl killed $ecx
-; X64-NOBMI-NEXT:    shrl %cl, %ebx
-; X64-NOBMI-NEXT:    negb %dl
-; X64-NOBMI-NEXT:    movl $-1, %ebp
-; X64-NOBMI-NEXT:    movl %edx, %ecx
-; X64-NOBMI-NEXT:    shrl %cl, %ebp
-; X64-NOBMI-NEXT:    movl %ebp, %edi
-; X64-NOBMI-NEXT:    callq use32
-; X64-NOBMI-NEXT:    andl %ebx, %ebp
-; X64-NOBMI-NEXT:    movl %ebp, %eax
+; X64-NOBMI-NEXT:    movq %rsi, %rcx
+; X64-NOBMI-NEXT:    movq %rdi, %r14
+; X64-NOBMI-NEXT:    # kill: def $cl killed $cl killed $rcx
+; X64-NOBMI-NEXT:    shrq %cl, %r14
+; X64-NOBMI-NEXT:    negb %dl
+; X64-NOBMI-NEXT:    movq $-1, %rbx
+; X64-NOBMI-NEXT:    movl %edx, %ecx
+; X64-NOBMI-NEXT:    shrq %cl, %rbx
+; X64-NOBMI-NEXT:    movq %rbx, %rdi
+; X64-NOBMI-NEXT:    callq use64
+; X64-NOBMI-NEXT:    andq %r14, %rbx
+; X64-NOBMI-NEXT:    movq %rbx, %rax
 ; X64-NOBMI-NEXT:    addq $8, %rsp
 ; X64-NOBMI-NEXT:    popq %rbx
-; X64-NOBMI-NEXT:    popq %rbp
+; X64-NOBMI-NEXT:    popq %r14
 ; X64-NOBMI-NEXT:    retq
 ;
-; X64-BMI1NOTBM-LABEL: bextr32_c4_commutative:
+; X64-BMI1NOTBM-LABEL: bextr64_c0:
 ; X64-BMI1NOTBM:       # %bb.0:
-; X64-BMI1NOTBM-NEXT:    pushq %rbp
+; X64-BMI1NOTBM-NEXT:    pushq %r14
 ; X64-BMI1NOTBM-NEXT:    pushq %rbx
 ; X64-BMI1NOTBM-NEXT:    pushq %rax
-; X64-BMI1NOTBM-NEXT:    movl %esi, %ecx
-; X64-BMI1NOTBM-NEXT:    movl %edi, %ebx
-; X64-BMI1NOTBM-NEXT:    # kill: def $cl killed $cl killed $ecx
-; X64-BMI1NOTBM-NEXT:    shrl %cl, %ebx
+; X64-BMI1NOTBM-NEXT:    movq %rsi, %rcx
+; X64-BMI1NOTBM-NEXT:    movq %rdi, %r14
+; X64-BMI1NOTBM-NEXT:    # kill: def $cl killed $cl killed $rcx
+; X64-BMI1NOTBM-NEXT:    shrq %cl, %r14
 ; X64-BMI1NOTBM-NEXT:    negb %dl
-; X64-BMI1NOTBM-NEXT:    movl $-1, %ebp
+; X64-BMI1NOTBM-NEXT:    movq $-1, %rbx
 ; X64-BMI1NOTBM-NEXT:    movl %edx, %ecx
-; X64-BMI1NOTBM-NEXT:    shrl %cl, %ebp
-; X64-BMI1NOTBM-NEXT:    movl %ebp, %edi
-; X64-BMI1NOTBM-NEXT:    callq use32
-; X64-BMI1NOTBM-NEXT:    andl %ebx, %ebp
-; X64-BMI1NOTBM-NEXT:    movl %ebp, %eax
+; X64-BMI1NOTBM-NEXT:    shrq %cl, %rbx
+; X64-BMI1NOTBM-NEXT:    movq %rbx, %rdi
+; X64-BMI1NOTBM-NEXT:    callq use64
+; X64-BMI1NOTBM-NEXT:    andq %r14, %rbx
+; X64-BMI1NOTBM-NEXT:    movq %rbx, %rax
 ; X64-BMI1NOTBM-NEXT:    addq $8, %rsp
 ; X64-BMI1NOTBM-NEXT:    popq %rbx
-; X64-BMI1NOTBM-NEXT:    popq %rbp
+; X64-BMI1NOTBM-NEXT:    popq %r14
 ; X64-BMI1NOTBM-NEXT:    retq
 ;
-; X64-BMI1BMI2-LABEL: bextr32_c4_commutative:
+; X64-BMI1BMI2-LABEL: bextr64_c0:
 ; X64-BMI1BMI2:       # %bb.0:
-; X64-BMI1BMI2-NEXT:    pushq %rbp
+; X64-BMI1BMI2-NEXT:    pushq %r14
 ; X64-BMI1BMI2-NEXT:    pushq %rbx
 ; X64-BMI1BMI2-NEXT:    pushq %rax
-; X64-BMI1BMI2-NEXT:    movl %edx, %ebx
-; X64-BMI1BMI2-NEXT:    shrxl %esi, %edi, %ebp
+; X64-BMI1BMI2-NEXT:    movq %rdx, %rbx
+; X64-BMI1BMI2-NEXT:    shrxq %rsi, %rdi, %r14
 ; X64-BMI1BMI2-NEXT:    movl %ebx, %eax
 ; X64-BMI1BMI2-NEXT:    negb %al
-; X64-BMI1BMI2-NEXT:    movl $-1, %ecx
-; X64-BMI1BMI2-NEXT:    shrxl %eax, %ecx, %edi
-; X64-BMI1BMI2-NEXT:    callq use32
-; X64-BMI1BMI2-NEXT:    bzhil %ebx, %ebp, %eax
+; X64-BMI1BMI2-NEXT:    movq $-1, %rcx
+; X64-BMI1BMI2-NEXT:    shrxq %rax, %rcx, %rdi
+; X64-BMI1BMI2-NEXT:    callq use64
+; X64-BMI1BMI2-NEXT:    bzhiq %rbx, %r14, %rax
 ; X64-BMI1BMI2-NEXT:    addq $8, %rsp
 ; X64-BMI1BMI2-NEXT:    popq %rbx
-; X64-BMI1BMI2-NEXT:    popq %rbp
+; X64-BMI1BMI2-NEXT:    popq %r14
 ; X64-BMI1BMI2-NEXT:    retq
-  %shifted = lshr i32 %val, %numskipbits
-  %numhighbits = sub i32 32, %numlowbits
-  %mask = lshr i32 -1, %numhighbits
-  call void @use32(i32 %mask)
-  %masked = and i32 %shifted, %mask ; swapped order
-  ret i32 %masked
+  %shifted = lshr i64 %val, %numskipbits
+  %numhighbits = sub i64 64, %numlowbits
+  %mask = lshr i64 -1, %numhighbits
+  call void @use64(i64 %mask)
+  %masked = and i64 %mask, %shifted
+  ret i64 %masked
 }
 
-define i32 @bextr32_c5_skipextrauses(i32 %val, i32 %numskipbits, i32 %numlowbits) nounwind {
-; X86-NOBMI-LABEL: bextr32_c5_skipextrauses:
+define i64 @bextr64_c1_indexzext(i64 %val, i8 %numskipbits, i8 %numlowbits) nounwind {
+; X86-NOBMI-LABEL: bextr64_c1_indexzext:
 ; X86-NOBMI:       # %bb.0:
+; X86-NOBMI-NEXT:    pushl %ebp
 ; X86-NOBMI-NEXT:    pushl %ebx
 ; X86-NOBMI-NEXT:    pushl %edi
 ; X86-NOBMI-NEXT:    pushl %esi
-; X86-NOBMI-NEXT:    subl $16, %esp
-; X86-NOBMI-NEXT:    movl {{[0-9]+}}(%esp), %edi
-; X86-NOBMI-NEXT:    movl {{[0-9]+}}(%esp), %ebx
-; X86-NOBMI-NEXT:    movl %ebx, %ecx
+; X86-NOBMI-NEXT:    subl $12, %esp
+; X86-NOBMI-NEXT:    movb {{[0-9]+}}(%esp), %cl
+; X86-NOBMI-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NOBMI-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NOBMI-NEXT:    movl %eax, %edi
 ; X86-NOBMI-NEXT:    shrl %cl, %edi
-; X86-NOBMI-NEXT:    xorl %ecx, %ecx
+; X86-NOBMI-NEXT:    shrdl %cl, %eax, %esi
+; X86-NOBMI-NEXT:    testb $32, %cl
+; X86-NOBMI-NEXT:    je .LBB40_2
+; X86-NOBMI-NEXT:  # %bb.1:
+; X86-NOBMI-NEXT:    movl %edi, %esi
+; X86-NOBMI-NEXT:    xorl %edi, %edi
+; X86-NOBMI-NEXT:  .LBB40_2:
+; X86-NOBMI-NEXT:    movb $64, %cl
 ; X86-NOBMI-NEXT:    subb {{[0-9]+}}(%esp), %cl
-; X86-NOBMI-NEXT:    movl $-1, %esi
-; X86-NOBMI-NEXT:    # kill: def $cl killed $cl killed $ecx
-; X86-NOBMI-NEXT:    shrl %cl, %esi
-; X86-NOBMI-NEXT:    movl %esi, (%esp)
-; X86-NOBMI-NEXT:    calll use32
-; X86-NOBMI-NEXT:    andl %edi, %esi
-; X86-NOBMI-NEXT:    movl %ebx, (%esp)
-; X86-NOBMI-NEXT:    calll use32
-; X86-NOBMI-NEXT:    movl %esi, %eax
+; X86-NOBMI-NEXT:    movl $-1, %ebp
+; X86-NOBMI-NEXT:    movl $-1, %ebx
+; X86-NOBMI-NEXT:    shrl %cl, %ebx
+; X86-NOBMI-NEXT:    shrdl %cl, %ebp, %ebp
+; X86-NOBMI-NEXT:    testb $32, %cl
+; X86-NOBMI-NEXT:    je .LBB40_4
+; X86-NOBMI-NEXT:  # %bb.3:
+; X86-NOBMI-NEXT:    movl %ebx, %ebp
+; X86-NOBMI-NEXT:    xorl %ebx, %ebx
+; X86-NOBMI-NEXT:  .LBB40_4:
+; X86-NOBMI-NEXT:    subl $8, %esp
+; X86-NOBMI-NEXT:    pushl %ebx
+; X86-NOBMI-NEXT:    pushl %ebp
+; X86-NOBMI-NEXT:    calll use64
 ; X86-NOBMI-NEXT:    addl $16, %esp
+; X86-NOBMI-NEXT:    andl %ebp, %esi
+; X86-NOBMI-NEXT:    andl %ebx, %edi
+; X86-NOBMI-NEXT:    movl %esi, %eax
+; X86-NOBMI-NEXT:    movl %edi, %edx
+; X86-NOBMI-NEXT:    addl $12, %esp
 ; X86-NOBMI-NEXT:    popl %esi
 ; X86-NOBMI-NEXT:    popl %edi
 ; X86-NOBMI-NEXT:    popl %ebx
+; X86-NOBMI-NEXT:    popl %ebp
 ; X86-NOBMI-NEXT:    retl
 ;
-; X86-BMI1NOTBM-LABEL: bextr32_c5_skipextrauses:
+; X86-BMI1NOTBM-LABEL: bextr64_c1_indexzext:
 ; X86-BMI1NOTBM:       # %bb.0:
+; X86-BMI1NOTBM-NEXT:    pushl %ebp
 ; X86-BMI1NOTBM-NEXT:    pushl %ebx
 ; X86-BMI1NOTBM-NEXT:    pushl %edi
 ; X86-BMI1NOTBM-NEXT:    pushl %esi
-; X86-BMI1NOTBM-NEXT:    subl $16, %esp
-; X86-BMI1NOTBM-NEXT:    movl {{[0-9]+}}(%esp), %edi
-; X86-BMI1NOTBM-NEXT:    movl {{[0-9]+}}(%esp), %ebx
-; X86-BMI1NOTBM-NEXT:    movl %ebx, %ecx
+; X86-BMI1NOTBM-NEXT:    subl $12, %esp
+; X86-BMI1NOTBM-NEXT:    movb {{[0-9]+}}(%esp), %cl
+; X86-BMI1NOTBM-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-BMI1NOTBM-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-BMI1NOTBM-NEXT:    movl %eax, %edi
 ; X86-BMI1NOTBM-NEXT:    shrl %cl, %edi
-; X86-BMI1NOTBM-NEXT:    xorl %ecx, %ecx
+; X86-BMI1NOTBM-NEXT:    shrdl %cl, %eax, %esi
+; X86-BMI1NOTBM-NEXT:    testb $32, %cl
+; X86-BMI1NOTBM-NEXT:    je .LBB40_2
+; X86-BMI1NOTBM-NEXT:  # %bb.1:
+; X86-BMI1NOTBM-NEXT:    movl %edi, %esi
+; X86-BMI1NOTBM-NEXT:    xorl %edi, %edi
+; X86-BMI1NOTBM-NEXT:  .LBB40_2:
+; X86-BMI1NOTBM-NEXT:    movb $64, %cl
 ; X86-BMI1NOTBM-NEXT:    subb {{[0-9]+}}(%esp), %cl
-; X86-BMI1NOTBM-NEXT:    movl $-1, %esi
-; X86-BMI1NOTBM-NEXT:    # kill: def $cl killed $cl killed $ecx
-; X86-BMI1NOTBM-NEXT:    shrl %cl, %esi
-; X86-BMI1NOTBM-NEXT:    movl %esi, (%esp)
-; X86-BMI1NOTBM-NEXT:    calll use32
-; X86-BMI1NOTBM-NEXT:    andl %edi, %esi
-; X86-BMI1NOTBM-NEXT:    movl %ebx, (%esp)
-; X86-BMI1NOTBM-NEXT:    calll use32
-; X86-BMI1NOTBM-NEXT:    movl %esi, %eax
+; X86-BMI1NOTBM-NEXT:    movl $-1, %ebp
+; X86-BMI1NOTBM-NEXT:    movl $-1, %ebx
+; X86-BMI1NOTBM-NEXT:    shrl %cl, %ebx
+; X86-BMI1NOTBM-NEXT:    shrdl %cl, %ebp, %ebp
+; X86-BMI1NOTBM-NEXT:    testb $32, %cl
+; X86-BMI1NOTBM-NEXT:    je .LBB40_4
+; X86-BMI1NOTBM-NEXT:  # %bb.3:
+; X86-BMI1NOTBM-NEXT:    movl %ebx, %ebp
+; X86-BMI1NOTBM-NEXT:    xorl %ebx, %ebx
+; X86-BMI1NOTBM-NEXT:  .LBB40_4:
+; X86-BMI1NOTBM-NEXT:    subl $8, %esp
+; X86-BMI1NOTBM-NEXT:    pushl %ebx
+; X86-BMI1NOTBM-NEXT:    pushl %ebp
+; X86-BMI1NOTBM-NEXT:    calll use64
 ; X86-BMI1NOTBM-NEXT:    addl $16, %esp
+; X86-BMI1NOTBM-NEXT:    andl %ebp, %esi
+; X86-BMI1NOTBM-NEXT:    andl %ebx, %edi
+; X86-BMI1NOTBM-NEXT:    movl %esi, %eax
+; X86-BMI1NOTBM-NEXT:    movl %edi, %edx
+; X86-BMI1NOTBM-NEXT:    addl $12, %esp
 ; X86-BMI1NOTBM-NEXT:    popl %esi
 ; X86-BMI1NOTBM-NEXT:    popl %edi
 ; X86-BMI1NOTBM-NEXT:    popl %ebx
+; X86-BMI1NOTBM-NEXT:    popl %ebp
 ; X86-BMI1NOTBM-NEXT:    retl
 ;
-; X86-BMI1BMI2-LABEL: bextr32_c5_skipextrauses:
+; X86-BMI1BMI2-LABEL: bextr64_c1_indexzext:
 ; X86-BMI1BMI2:       # %bb.0:
+; X86-BMI1BMI2-NEXT:    pushl %ebp
 ; X86-BMI1BMI2-NEXT:    pushl %ebx
 ; X86-BMI1BMI2-NEXT:    pushl %edi
 ; X86-BMI1BMI2-NEXT:    pushl %esi
-; X86-BMI1BMI2-NEXT:    subl $16, %esp
-; X86-BMI1BMI2-NEXT:    movb {{[0-9]+}}(%esp), %bl
-; X86-BMI1BMI2-NEXT:    movl {{[0-9]+}}(%esp), %edi
-; X86-BMI1BMI2-NEXT:    shrxl %edi, {{[0-9]+}}(%esp), %esi
-; X86-BMI1BMI2-NEXT:    movl %ebx, %eax
-; X86-BMI1BMI2-NEXT:    negb %al
-; X86-BMI1BMI2-NEXT:    movl $-1, %ecx
-; X86-BMI1BMI2-NEXT:    shrxl %eax, %ecx, %eax
-; X86-BMI1BMI2-NEXT:    movl %eax, (%esp)
-; X86-BMI1BMI2-NEXT:    calll use32
-; X86-BMI1BMI2-NEXT:    bzhil %ebx, %esi, %esi
-; X86-BMI1BMI2-NEXT:    movl %edi, (%esp)
-; X86-BMI1BMI2-NEXT:    calll use32
-; X86-BMI1BMI2-NEXT:    movl %esi, %eax
+; X86-BMI1BMI2-NEXT:    subl $12, %esp
+; X86-BMI1BMI2-NEXT:    movb {{[0-9]+}}(%esp), %cl
+; X86-BMI1BMI2-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-BMI1BMI2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-BMI1BMI2-NEXT:    shrdl %cl, %eax, %esi
+; X86-BMI1BMI2-NEXT:    shrxl %ecx, %eax, %edi
+; X86-BMI1BMI2-NEXT:    testb $32, %cl
+; X86-BMI1BMI2-NEXT:    je .LBB40_2
+; X86-BMI1BMI2-NEXT:  # %bb.1:
+; X86-BMI1BMI2-NEXT:    movl %edi, %esi
+; X86-BMI1BMI2-NEXT:    xorl %edi, %edi
+; X86-BMI1BMI2-NEXT:  .LBB40_2:
+; X86-BMI1BMI2-NEXT:    movb $64, %cl
+; X86-BMI1BMI2-NEXT:    subb {{[0-9]+}}(%esp), %cl
+; X86-BMI1BMI2-NEXT:    movl $-1, %ebx
+; X86-BMI1BMI2-NEXT:    shrxl %ecx, %ebx, %ebp
+; X86-BMI1BMI2-NEXT:    shrdl %cl, %ebx, %ebx
+; X86-BMI1BMI2-NEXT:    testb $32, %cl
+; X86-BMI1BMI2-NEXT:    je .LBB40_4
+; X86-BMI1BMI2-NEXT:  # %bb.3:
+; X86-BMI1BMI2-NEXT:    movl %ebp, %ebx
+; X86-BMI1BMI2-NEXT:    xorl %ebp, %ebp
+; X86-BMI1BMI2-NEXT:  .LBB40_4:
+; X86-BMI1BMI2-NEXT:    subl $8, %esp
+; X86-BMI1BMI2-NEXT:    pushl %ebp
+; X86-BMI1BMI2-NEXT:    pushl %ebx
+; X86-BMI1BMI2-NEXT:    calll use64
 ; X86-BMI1BMI2-NEXT:    addl $16, %esp
+; X86-BMI1BMI2-NEXT:    andl %ebx, %esi
+; X86-BMI1BMI2-NEXT:    andl %ebp, %edi
+; X86-BMI1BMI2-NEXT:    movl %esi, %eax
+; X86-BMI1BMI2-NEXT:    movl %edi, %edx
+; X86-BMI1BMI2-NEXT:    addl $12, %esp
 ; X86-BMI1BMI2-NEXT:    popl %esi
 ; X86-BMI1BMI2-NEXT:    popl %edi
 ; X86-BMI1BMI2-NEXT:    popl %ebx
+; X86-BMI1BMI2-NEXT:    popl %ebp
 ; X86-BMI1BMI2-NEXT:    retl
 ;
-; X64-NOBMI-LABEL: bextr32_c5_skipextrauses:
+; X64-NOBMI-LABEL: bextr64_c1_indexzext:
 ; X64-NOBMI:       # %bb.0:
-; X64-NOBMI-NEXT:    pushq %rbp
 ; X64-NOBMI-NEXT:    pushq %r14
 ; X64-NOBMI-NEXT:    pushq %rbx
-; X64-NOBMI-NEXT:    movl %esi, %r14d
-; X64-NOBMI-NEXT:    movl %edi, %ebp
-; X64-NOBMI-NEXT:    movl %r14d, %ecx
-; X64-NOBMI-NEXT:    shrl %cl, %ebp
+; X64-NOBMI-NEXT:    pushq %rax
+; X64-NOBMI-NEXT:    movl %esi, %ecx
+; X64-NOBMI-NEXT:    movq %rdi, %r14
+; X64-NOBMI-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X64-NOBMI-NEXT:    shrq %cl, %r14
 ; X64-NOBMI-NEXT:    negb %dl
-; X64-NOBMI-NEXT:    movl $-1, %ebx
+; X64-NOBMI-NEXT:    movq $-1, %rbx
 ; X64-NOBMI-NEXT:    movl %edx, %ecx
-; X64-NOBMI-NEXT:    shrl %cl, %ebx
-; X64-NOBMI-NEXT:    movl %ebx, %edi
-; X64-NOBMI-NEXT:    callq use32
-; X64-NOBMI-NEXT:    andl %ebp, %ebx
-; X64-NOBMI-NEXT:    movl %r14d, %edi
-; X64-NOBMI-NEXT:    callq use32
-; X64-NOBMI-NEXT:    movl %ebx, %eax
+; X64-NOBMI-NEXT:    shrq %cl, %rbx
+; X64-NOBMI-NEXT:    movq %rbx, %rdi
+; X64-NOBMI-NEXT:    callq use64
+; X64-NOBMI-NEXT:    andq %r14, %rbx
+; X64-NOBMI-NEXT:    movq %rbx, %rax
+; X64-NOBMI-NEXT:    addq $8, %rsp
 ; X64-NOBMI-NEXT:    popq %rbx
 ; X64-NOBMI-NEXT:    popq %r14
-; X64-NOBMI-NEXT:    popq %rbp
 ; X64-NOBMI-NEXT:    retq
 ;
-; X64-BMI1NOTBM-LABEL: bextr32_c5_skipextrauses:
+; X64-BMI1NOTBM-LABEL: bextr64_c1_indexzext:
 ; X64-BMI1NOTBM:       # %bb.0:
-; X64-BMI1NOTBM-NEXT:    pushq %rbp
 ; X64-BMI1NOTBM-NEXT:    pushq %r14
 ; X64-BMI1NOTBM-NEXT:    pushq %rbx
-; X64-BMI1NOTBM-NEXT:    movl %esi, %r14d
-; X64-BMI1NOTBM-NEXT:    movl %edi, %ebp
-; X64-BMI1NOTBM-NEXT:    movl %r14d, %ecx
-; X64-BMI1NOTBM-NEXT:    shrl %cl, %ebp
-; X64-BMI1NOTBM-NEXT:    negb %dl
-; X64-BMI1NOTBM-NEXT:    movl $-1, %ebx
-; X64-BMI1NOTBM-NEXT:    movl %edx, %ecx
-; X64-BMI1NOTBM-NEXT:    shrl %cl, %ebx
-; X64-BMI1NOTBM-NEXT:    movl %ebx, %edi
-; X64-BMI1NOTBM-NEXT:    callq use32
-; X64-BMI1NOTBM-NEXT:    andl %ebp, %ebx
-; X64-BMI1NOTBM-NEXT:    movl %r14d, %edi
-; X64-BMI1NOTBM-NEXT:    callq use32
-; X64-BMI1NOTBM-NEXT:    movl %ebx, %eax
+; X64-BMI1NOTBM-NEXT:    pushq %rax
+; X64-BMI1NOTBM-NEXT:    movl %esi, %ecx
+; X64-BMI1NOTBM-NEXT:    movq %rdi, %r14
+; X64-BMI1NOTBM-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X64-BMI1NOTBM-NEXT:    shrq %cl, %r14
+; X64-BMI1NOTBM-NEXT:    negb %dl
+; X64-BMI1NOTBM-NEXT:    movq $-1, %rbx
+; X64-BMI1NOTBM-NEXT:    movl %edx, %ecx
+; X64-BMI1NOTBM-NEXT:    shrq %cl, %rbx
+; X64-BMI1NOTBM-NEXT:    movq %rbx, %rdi
+; X64-BMI1NOTBM-NEXT:    callq use64
+; X64-BMI1NOTBM-NEXT:    andq %r14, %rbx
+; X64-BMI1NOTBM-NEXT:    movq %rbx, %rax
+; X64-BMI1NOTBM-NEXT:    addq $8, %rsp
 ; X64-BMI1NOTBM-NEXT:    popq %rbx
 ; X64-BMI1NOTBM-NEXT:    popq %r14
-; X64-BMI1NOTBM-NEXT:    popq %rbp
 ; X64-BMI1NOTBM-NEXT:    retq
 ;
-; X64-BMI1BMI2-LABEL: bextr32_c5_skipextrauses:
+; X64-BMI1BMI2-LABEL: bextr64_c1_indexzext:
 ; X64-BMI1BMI2:       # %bb.0:
-; X64-BMI1BMI2-NEXT:    pushq %rbp
 ; X64-BMI1BMI2-NEXT:    pushq %r14
 ; X64-BMI1BMI2-NEXT:    pushq %rbx
+; X64-BMI1BMI2-NEXT:    pushq %rax
 ; X64-BMI1BMI2-NEXT:    movl %edx, %ebx
-; X64-BMI1BMI2-NEXT:    movl %esi, %ebp
-; X64-BMI1BMI2-NEXT:    shrxl %esi, %edi, %r14d
+; X64-BMI1BMI2-NEXT:    # kill: def $esi killed $esi def $rsi
+; X64-BMI1BMI2-NEXT:    shrxq %rsi, %rdi, %r14
 ; X64-BMI1BMI2-NEXT:    movl %ebx, %eax
 ; X64-BMI1BMI2-NEXT:    negb %al
-; X64-BMI1BMI2-NEXT:    movl $-1, %ecx
-; X64-BMI1BMI2-NEXT:    shrxl %eax, %ecx, %edi
-; X64-BMI1BMI2-NEXT:    callq use32
-; X64-BMI1BMI2-NEXT:    bzhil %ebx, %r14d, %ebx
-; X64-BMI1BMI2-NEXT:    movl %ebp, %edi
-; X64-BMI1BMI2-NEXT:    callq use32
-; X64-BMI1BMI2-NEXT:    movl %ebx, %eax
+; X64-BMI1BMI2-NEXT:    movq $-1, %rcx
+; X64-BMI1BMI2-NEXT:    shrxq %rax, %rcx, %rdi
+; X64-BMI1BMI2-NEXT:    callq use64
+; X64-BMI1BMI2-NEXT:    bzhiq %rbx, %r14, %rax
+; X64-BMI1BMI2-NEXT:    addq $8, %rsp
 ; X64-BMI1BMI2-NEXT:    popq %rbx
 ; X64-BMI1BMI2-NEXT:    popq %r14
-; X64-BMI1BMI2-NEXT:    popq %rbp
 ; X64-BMI1BMI2-NEXT:    retq
-  %shifted = lshr i32 %val, %numskipbits
-  %numhighbits = sub i32 32, %numlowbits
-  %mask = lshr i32 -1, %numhighbits
-  call void @use32(i32 %mask)
-  %masked = and i32 %mask, %shifted
-  call void @use32(i32 %numskipbits)
-  ret i32 %masked
+  %skip = zext i8 %numskipbits to i64
+  %shifted = lshr i64 %val, %skip
+  %numhighbits = sub i8 64, %numlowbits
+  %sh_prom = zext i8 %numhighbits to i64
+  %mask = lshr i64 -1, %sh_prom
+  call void @use64(i64 %mask)
+  %masked = and i64 %mask, %shifted
+  ret i64 %masked
 }
 
-; 64-bit
-
-define i64 @bextr64_c0(i64 %val, i64 %numskipbits, i64 %numlowbits) nounwind {
-; X86-NOBMI-LABEL: bextr64_c0:
+define i64 @bextr64_c2_load(i64* %w, i64 %numskipbits, i64 %numlowbits) nounwind {
+; X86-NOBMI-LABEL: bextr64_c2_load:
 ; X86-NOBMI:       # %bb.0:
 ; X86-NOBMI-NEXT:    pushl %ebp
 ; X86-NOBMI-NEXT:    pushl %ebx
@@ -3824,17 +5056,18 @@ define i64 @bextr64_c0(i64 %val, i64 %numskipbits, i64 %numlowbits) nounwind {
 ; X86-NOBMI-NEXT:    pushl %esi
 ; X86-NOBMI-NEXT:    subl $12, %esp
 ; X86-NOBMI-NEXT:    movb {{[0-9]+}}(%esp), %cl
-; X86-NOBMI-NEXT:    movl {{[0-9]+}}(%esp), %esi
 ; X86-NOBMI-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NOBMI-NEXT:    movl (%eax), %esi
+; X86-NOBMI-NEXT:    movl 4(%eax), %eax
 ; X86-NOBMI-NEXT:    movl %eax, %edi
 ; X86-NOBMI-NEXT:    shrl %cl, %edi
 ; X86-NOBMI-NEXT:    shrdl %cl, %eax, %esi
 ; X86-NOBMI-NEXT:    testb $32, %cl
-; X86-NOBMI-NEXT:    je .LBB32_2
+; X86-NOBMI-NEXT:    je .LBB41_2
 ; X86-NOBMI-NEXT:  # %bb.1:
 ; X86-NOBMI-NEXT:    movl %edi, %esi
 ; X86-NOBMI-NEXT:    xorl %edi, %edi
-; X86-NOBMI-NEXT:  .LBB32_2:
+; X86-NOBMI-NEXT:  .LBB41_2:
 ; X86-NOBMI-NEXT:    movb $64, %cl
 ; X86-NOBMI-NEXT:    subb {{[0-9]+}}(%esp), %cl
 ; X86-NOBMI-NEXT:    movl $-1, %ebp
@@ -3842,11 +5075,11 @@ define i64 @bextr64_c0(i64 %val, i64 %numskipbits, i64 %numlowbits) nounwind {
 ; X86-NOBMI-NEXT:    shrl %cl, %ebx
 ; X86-NOBMI-NEXT:    shrdl %cl, %ebp, %ebp
 ; X86-NOBMI-NEXT:    testb $32, %cl
-; X86-NOBMI-NEXT:    je .LBB32_4
+; X86-NOBMI-NEXT:    je .LBB41_4
 ; X86-NOBMI-NEXT:  # %bb.3:
 ; X86-NOBMI-NEXT:    movl %ebx, %ebp
 ; X86-NOBMI-NEXT:    xorl %ebx, %ebx
-; X86-NOBMI-NEXT:  .LBB32_4:
+; X86-NOBMI-NEXT:  .LBB41_4:
 ; X86-NOBMI-NEXT:    subl $8, %esp
 ; X86-NOBMI-NEXT:    pushl %ebx
 ; X86-NOBMI-NEXT:    pushl %ebp
@@ -3863,7 +5096,7 @@ define i64 @bextr64_c0(i64 %val, i64 %numskipbits, i64 %numlowbits) nounwind {
 ; X86-NOBMI-NEXT:    popl %ebp
 ; X86-NOBMI-NEXT:    retl
 ;
-; X86-BMI1NOTBM-LABEL: bextr64_c0:
+; X86-BMI1NOTBM-LABEL: bextr64_c2_load:
 ; X86-BMI1NOTBM:       # %bb.0:
 ; X86-BMI1NOTBM-NEXT:    pushl %ebp
 ; X86-BMI1NOTBM-NEXT:    pushl %ebx
@@ -3871,17 +5104,18 @@ define i64 @bextr64_c0(i64 %val, i64 %numskipbits, i64 %numlowbits) nounwind {
 ; X86-BMI1NOTBM-NEXT:    pushl %esi
 ; X86-BMI1NOTBM-NEXT:    subl $12, %esp
 ; X86-BMI1NOTBM-NEXT:    movb {{[0-9]+}}(%esp), %cl
-; X86-BMI1NOTBM-NEXT:    movl {{[0-9]+}}(%esp), %esi
 ; X86-BMI1NOTBM-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-BMI1NOTBM-NEXT:    movl (%eax), %esi
+; X86-BMI1NOTBM-NEXT:    movl 4(%eax), %eax
 ; X86-BMI1NOTBM-NEXT:    movl %eax, %edi
 ; X86-BMI1NOTBM-NEXT:    shrl %cl, %edi
 ; X86-BMI1NOTBM-NEXT:    shrdl %cl, %eax, %esi
 ; X86-BMI1NOTBM-NEXT:    testb $32, %cl
-; X86-BMI1NOTBM-NEXT:    je .LBB32_2
+; X86-BMI1NOTBM-NEXT:    je .LBB41_2
 ; X86-BMI1NOTBM-NEXT:  # %bb.1:
 ; X86-BMI1NOTBM-NEXT:    movl %edi, %esi
 ; X86-BMI1NOTBM-NEXT:    xorl %edi, %edi
-; X86-BMI1NOTBM-NEXT:  .LBB32_2:
+; X86-BMI1NOTBM-NEXT:  .LBB41_2:
 ; X86-BMI1NOTBM-NEXT:    movb $64, %cl
 ; X86-BMI1NOTBM-NEXT:    subb {{[0-9]+}}(%esp), %cl
 ; X86-BMI1NOTBM-NEXT:    movl $-1, %ebp
@@ -3889,11 +5123,11 @@ define i64 @bextr64_c0(i64 %val, i64 %numskipbits, i64 %numlowbits) nounwind {
 ; X86-BMI1NOTBM-NEXT:    shrl %cl, %ebx
 ; X86-BMI1NOTBM-NEXT:    shrdl %cl, %ebp, %ebp
 ; X86-BMI1NOTBM-NEXT:    testb $32, %cl
-; X86-BMI1NOTBM-NEXT:    je .LBB32_4
+; X86-BMI1NOTBM-NEXT:    je .LBB41_4
 ; X86-BMI1NOTBM-NEXT:  # %bb.3:
 ; X86-BMI1NOTBM-NEXT:    movl %ebx, %ebp
 ; X86-BMI1NOTBM-NEXT:    xorl %ebx, %ebx
-; X86-BMI1NOTBM-NEXT:  .LBB32_4:
+; X86-BMI1NOTBM-NEXT:  .LBB41_4:
 ; X86-BMI1NOTBM-NEXT:    subl $8, %esp
 ; X86-BMI1NOTBM-NEXT:    pushl %ebx
 ; X86-BMI1NOTBM-NEXT:    pushl %ebp
@@ -3910,7 +5144,7 @@ define i64 @bextr64_c0(i64 %val, i64 %numskipbits, i64 %numlowbits) nounwind {
 ; X86-BMI1NOTBM-NEXT:    popl %ebp
 ; X86-BMI1NOTBM-NEXT:    retl
 ;
-; X86-BMI1BMI2-LABEL: bextr64_c0:
+; X86-BMI1BMI2-LABEL: bextr64_c2_load:
 ; X86-BMI1BMI2:       # %bb.0:
 ; X86-BMI1BMI2-NEXT:    pushl %ebp
 ; X86-BMI1BMI2-NEXT:    pushl %ebx
@@ -3918,27 +5152,28 @@ define i64 @bextr64_c0(i64 %val, i64 %numskipbits, i64 %numlowbits) nounwind {
 ; X86-BMI1BMI2-NEXT:    pushl %esi
 ; X86-BMI1BMI2-NEXT:    subl $12, %esp
 ; X86-BMI1BMI2-NEXT:    movb {{[0-9]+}}(%esp), %cl
-; X86-BMI1BMI2-NEXT:    movl {{[0-9]+}}(%esp), %esi
 ; X86-BMI1BMI2-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-BMI1BMI2-NEXT:    shrdl %cl, %eax, %esi
+; X86-BMI1BMI2-NEXT:    movl (%eax), %esi
+; X86-BMI1BMI2-NEXT:    movl 4(%eax), %eax
 ; X86-BMI1BMI2-NEXT:    shrxl %ecx, %eax, %edi
+; X86-BMI1BMI2-NEXT:    shrdl %cl, %eax, %esi
 ; X86-BMI1BMI2-NEXT:    testb $32, %cl
-; X86-BMI1BMI2-NEXT:    je .LBB32_2
+; X86-BMI1BMI2-NEXT:    je .LBB41_2
 ; X86-BMI1BMI2-NEXT:  # %bb.1:
 ; X86-BMI1BMI2-NEXT:    movl %edi, %esi
 ; X86-BMI1BMI2-NEXT:    xorl %edi, %edi
-; X86-BMI1BMI2-NEXT:  .LBB32_2:
+; X86-BMI1BMI2-NEXT:  .LBB41_2:
 ; X86-BMI1BMI2-NEXT:    movb $64, %cl
 ; X86-BMI1BMI2-NEXT:    subb {{[0-9]+}}(%esp), %cl
 ; X86-BMI1BMI2-NEXT:    movl $-1, %ebx
 ; X86-BMI1BMI2-NEXT:    shrxl %ecx, %ebx, %ebp
 ; X86-BMI1BMI2-NEXT:    shrdl %cl, %ebx, %ebx
 ; X86-BMI1BMI2-NEXT:    testb $32, %cl
-; X86-BMI1BMI2-NEXT:    je .LBB32_4
+; X86-BMI1BMI2-NEXT:    je .LBB41_4
 ; X86-BMI1BMI2-NEXT:  # %bb.3:
 ; X86-BMI1BMI2-NEXT:    movl %ebp, %ebx
 ; X86-BMI1BMI2-NEXT:    xorl %ebp, %ebp
-; X86-BMI1BMI2-NEXT:  .LBB32_4:
+; X86-BMI1BMI2-NEXT:  .LBB41_4:
 ; X86-BMI1BMI2-NEXT:    subl $8, %esp
 ; X86-BMI1BMI2-NEXT:    pushl %ebp
 ; X86-BMI1BMI2-NEXT:    pushl %ebx
@@ -3955,13 +5190,13 @@ define i64 @bextr64_c0(i64 %val, i64 %numskipbits, i64 %numlowbits) nounwind {
 ; X86-BMI1BMI2-NEXT:    popl %ebp
 ; X86-BMI1BMI2-NEXT:    retl
 ;
-; X64-NOBMI-LABEL: bextr64_c0:
+; X64-NOBMI-LABEL: bextr64_c2_load:
 ; X64-NOBMI:       # %bb.0:
 ; X64-NOBMI-NEXT:    pushq %r14
 ; X64-NOBMI-NEXT:    pushq %rbx
 ; X64-NOBMI-NEXT:    pushq %rax
 ; X64-NOBMI-NEXT:    movq %rsi, %rcx
-; X64-NOBMI-NEXT:    movq %rdi, %r14
+; X64-NOBMI-NEXT:    movq (%rdi), %r14
 ; X64-NOBMI-NEXT:    # kill: def $cl killed $cl killed $rcx
 ; X64-NOBMI-NEXT:    shrq %cl, %r14
 ; X64-NOBMI-NEXT:    negb %dl
@@ -3977,13 +5212,13 @@ define i64 @bextr64_c0(i64 %val, i64 %numskipbits, i64 %numlowbits) nounwind {
 ; X64-NOBMI-NEXT:    popq %r14
 ; X64-NOBMI-NEXT:    retq
 ;
-; X64-BMI1NOTBM-LABEL: bextr64_c0:
+; X64-BMI1NOTBM-LABEL: bextr64_c2_load:
 ; X64-BMI1NOTBM:       # %bb.0:
 ; X64-BMI1NOTBM-NEXT:    pushq %r14
 ; X64-BMI1NOTBM-NEXT:    pushq %rbx
 ; X64-BMI1NOTBM-NEXT:    pushq %rax
 ; X64-BMI1NOTBM-NEXT:    movq %rsi, %rcx
-; X64-BMI1NOTBM-NEXT:    movq %rdi, %r14
+; X64-BMI1NOTBM-NEXT:    movq (%rdi), %r14
 ; X64-BMI1NOTBM-NEXT:    # kill: def $cl killed $cl killed $rcx
 ; X64-BMI1NOTBM-NEXT:    shrq %cl, %r14
 ; X64-BMI1NOTBM-NEXT:    negb %dl
@@ -3999,13 +5234,13 @@ define i64 @bextr64_c0(i64 %val, i64 %numskipbits, i64 %numlowbits) nounwind {
 ; X64-BMI1NOTBM-NEXT:    popq %r14
 ; X64-BMI1NOTBM-NEXT:    retq
 ;
-; X64-BMI1BMI2-LABEL: bextr64_c0:
+; X64-BMI1BMI2-LABEL: bextr64_c2_load:
 ; X64-BMI1BMI2:       # %bb.0:
 ; X64-BMI1BMI2-NEXT:    pushq %r14
 ; X64-BMI1BMI2-NEXT:    pushq %rbx
 ; X64-BMI1BMI2-NEXT:    pushq %rax
 ; X64-BMI1BMI2-NEXT:    movq %rdx, %rbx
-; X64-BMI1BMI2-NEXT:    shrxq %rsi, %rdi, %r14
+; X64-BMI1BMI2-NEXT:    shrxq %rsi, (%rdi), %r14
 ; X64-BMI1BMI2-NEXT:    movl %ebx, %eax
 ; X64-BMI1BMI2-NEXT:    negb %al
 ; X64-BMI1BMI2-NEXT:    movq $-1, %rcx
@@ -4016,6 +5251,7 @@ define i64 @bextr64_c0(i64 %val, i64 %numskipbits, i64 %numlowbits) nounwind {
 ; X64-BMI1BMI2-NEXT:    popq %rbx
 ; X64-BMI1BMI2-NEXT:    popq %r14
 ; X64-BMI1BMI2-NEXT:    retq
+  %val = load i64, i64* %w
   %shifted = lshr i64 %val, %numskipbits
   %numhighbits = sub i64 64, %numlowbits
   %mask = lshr i64 -1, %numhighbits
@@ -4024,8 +5260,8 @@ define i64 @bextr64_c0(i64 %val, i64 %numskipbits, i64 %numlowbits) nounwind {
   ret i64 %masked
 }
 
-define i64 @bextr64_c1_indexzext(i64 %val, i8 %numskipbits, i8 %numlowbits) nounwind {
-; X86-NOBMI-LABEL: bextr64_c1_indexzext:
+define i64 @bextr64_c3_load_indexzext(i64* %w, i8 %numskipbits, i8 %numlowbits) nounwind {
+; X86-NOBMI-LABEL: bextr64_c3_load_indexzext:
 ; X86-NOBMI:       # %bb.0:
 ; X86-NOBMI-NEXT:    pushl %ebp
 ; X86-NOBMI-NEXT:    pushl %ebx
@@ -4033,17 +5269,18 @@ define i64 @bextr64_c1_indexzext(i64 %val, i8 %numskipbits, i8 %numlowbits) noun
 ; X86-NOBMI-NEXT:    pushl %esi
 ; X86-NOBMI-NEXT:    subl $12, %esp
 ; X86-NOBMI-NEXT:    movb {{[0-9]+}}(%esp), %cl
-; X86-NOBMI-NEXT:    movl {{[0-9]+}}(%esp), %esi
 ; X86-NOBMI-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NOBMI-NEXT:    movl (%eax), %esi
+; X86-NOBMI-NEXT:    movl 4(%eax), %eax
 ; X86-NOBMI-NEXT:    movl %eax, %edi
 ; X86-NOBMI-NEXT:    shrl %cl, %edi
 ; X86-NOBMI-NEXT:    shrdl %cl, %eax, %esi
 ; X86-NOBMI-NEXT:    testb $32, %cl
-; X86-NOBMI-NEXT:    je .LBB33_2
+; X86-NOBMI-NEXT:    je .LBB42_2
 ; X86-NOBMI-NEXT:  # %bb.1:
 ; X86-NOBMI-NEXT:    movl %edi, %esi
 ; X86-NOBMI-NEXT:    xorl %edi, %edi
-; X86-NOBMI-NEXT:  .LBB33_2:
+; X86-NOBMI-NEXT:  .LBB42_2:
 ; X86-NOBMI-NEXT:    movb $64, %cl
 ; X86-NOBMI-NEXT:    subb {{[0-9]+}}(%esp), %cl
 ; X86-NOBMI-NEXT:    movl $-1, %ebp
@@ -4051,11 +5288,11 @@ define i64 @bextr64_c1_indexzext(i64 %val, i8 %numskipbits, i8 %numlowbits) noun
 ; X86-NOBMI-NEXT:    shrl %cl, %ebx
 ; X86-NOBMI-NEXT:    shrdl %cl, %ebp, %ebp
 ; X86-NOBMI-NEXT:    testb $32, %cl
-; X86-NOBMI-NEXT:    je .LBB33_4
+; X86-NOBMI-NEXT:    je .LBB42_4
 ; X86-NOBMI-NEXT:  # %bb.3:
 ; X86-NOBMI-NEXT:    movl %ebx, %ebp
 ; X86-NOBMI-NEXT:    xorl %ebx, %ebx
-; X86-NOBMI-NEXT:  .LBB33_4:
+; X86-NOBMI-NEXT:  .LBB42_4:
 ; X86-NOBMI-NEXT:    subl $8, %esp
 ; X86-NOBMI-NEXT:    pushl %ebx
 ; X86-NOBMI-NEXT:    pushl %ebp
@@ -4072,7 +5309,7 @@ define i64 @bextr64_c1_indexzext(i64 %val, i8 %numskipbits, i8 %numlowbits) noun
 ; X86-NOBMI-NEXT:    popl %ebp
 ; X86-NOBMI-NEXT:    retl
 ;
-; X86-BMI1NOTBM-LABEL: bextr64_c1_indexzext:
+; X86-BMI1NOTBM-LABEL: bextr64_c3_load_indexzext:
 ; X86-BMI1NOTBM:       # %bb.0:
 ; X86-BMI1NOTBM-NEXT:    pushl %ebp
 ; X86-BMI1NOTBM-NEXT:    pushl %ebx
@@ -4080,17 +5317,18 @@ define i64 @bextr64_c1_indexzext(i64 %val, i8 %numskipbits, i8 %numlowbits) noun
 ; X86-BMI1NOTBM-NEXT:    pushl %esi
 ; X86-BMI1NOTBM-NEXT:    subl $12, %esp
 ; X86-BMI1NOTBM-NEXT:    movb {{[0-9]+}}(%esp), %cl
-; X86-BMI1NOTBM-NEXT:    movl {{[0-9]+}}(%esp), %esi
 ; X86-BMI1NOTBM-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-BMI1NOTBM-NEXT:    movl (%eax), %esi
+; X86-BMI1NOTBM-NEXT:    movl 4(%eax), %eax
 ; X86-BMI1NOTBM-NEXT:    movl %eax, %edi
 ; X86-BMI1NOTBM-NEXT:    shrl %cl, %edi
 ; X86-BMI1NOTBM-NEXT:    shrdl %cl, %eax, %esi
 ; X86-BMI1NOTBM-NEXT:    testb $32, %cl
-; X86-BMI1NOTBM-NEXT:    je .LBB33_2
+; X86-BMI1NOTBM-NEXT:    je .LBB42_2
 ; X86-BMI1NOTBM-NEXT:  # %bb.1:
 ; X86-BMI1NOTBM-NEXT:    movl %edi, %esi
 ; X86-BMI1NOTBM-NEXT:    xorl %edi, %edi
-; X86-BMI1NOTBM-NEXT:  .LBB33_2:
+; X86-BMI1NOTBM-NEXT:  .LBB42_2:
 ; X86-BMI1NOTBM-NEXT:    movb $64, %cl
 ; X86-BMI1NOTBM-NEXT:    subb {{[0-9]+}}(%esp), %cl
 ; X86-BMI1NOTBM-NEXT:    movl $-1, %ebp
@@ -4098,11 +5336,11 @@ define i64 @bextr64_c1_indexzext(i64 %val, i8 %numskipbits, i8 %numlowbits) noun
 ; X86-BMI1NOTBM-NEXT:    shrl %cl, %ebx
 ; X86-BMI1NOTBM-NEXT:    shrdl %cl, %ebp, %ebp
 ; X86-BMI1NOTBM-NEXT:    testb $32, %cl
-; X86-BMI1NOTBM-NEXT:    je .LBB33_4
+; X86-BMI1NOTBM-NEXT:    je .LBB42_4
 ; X86-BMI1NOTBM-NEXT:  # %bb.3:
 ; X86-BMI1NOTBM-NEXT:    movl %ebx, %ebp
 ; X86-BMI1NOTBM-NEXT:    xorl %ebx, %ebx
-; X86-BMI1NOTBM-NEXT:  .LBB33_4:
+; X86-BMI1NOTBM-NEXT:  .LBB42_4:
 ; X86-BMI1NOTBM-NEXT:    subl $8, %esp
 ; X86-BMI1NOTBM-NEXT:    pushl %ebx
 ; X86-BMI1NOTBM-NEXT:    pushl %ebp
@@ -4119,7 +5357,7 @@ define i64 @bextr64_c1_indexzext(i64 %val, i8 %numskipbits, i8 %numlowbits) noun
 ; X86-BMI1NOTBM-NEXT:    popl %ebp
 ; X86-BMI1NOTBM-NEXT:    retl
 ;
-; X86-BMI1BMI2-LABEL: bextr64_c1_indexzext:
+; X86-BMI1BMI2-LABEL: bextr64_c3_load_indexzext:
 ; X86-BMI1BMI2:       # %bb.0:
 ; X86-BMI1BMI2-NEXT:    pushl %ebp
 ; X86-BMI1BMI2-NEXT:    pushl %ebx
@@ -4127,27 +5365,28 @@ define i64 @bextr64_c1_indexzext(i64 %val, i8 %numskipbits, i8 %numlowbits) noun
 ; X86-BMI1BMI2-NEXT:    pushl %esi
 ; X86-BMI1BMI2-NEXT:    subl $12, %esp
 ; X86-BMI1BMI2-NEXT:    movb {{[0-9]+}}(%esp), %cl
-; X86-BMI1BMI2-NEXT:    movl {{[0-9]+}}(%esp), %esi
 ; X86-BMI1BMI2-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-BMI1BMI2-NEXT:    shrdl %cl, %eax, %esi
+; X86-BMI1BMI2-NEXT:    movl (%eax), %esi
+; X86-BMI1BMI2-NEXT:    movl 4(%eax), %eax
 ; X86-BMI1BMI2-NEXT:    shrxl %ecx, %eax, %edi
+; X86-BMI1BMI2-NEXT:    shrdl %cl, %eax, %esi
 ; X86-BMI1BMI2-NEXT:    testb $32, %cl
-; X86-BMI1BMI2-NEXT:    je .LBB33_2
+; X86-BMI1BMI2-NEXT:    je .LBB42_2
 ; X86-BMI1BMI2-NEXT:  # %bb.1:
 ; X86-BMI1BMI2-NEXT:    movl %edi, %esi
 ; X86-BMI1BMI2-NEXT:    xorl %edi, %edi
-; X86-BMI1BMI2-NEXT:  .LBB33_2:
+; X86-BMI1BMI2-NEXT:  .LBB42_2:
 ; X86-BMI1BMI2-NEXT:    movb $64, %cl
 ; X86-BMI1BMI2-NEXT:    subb {{[0-9]+}}(%esp), %cl
 ; X86-BMI1BMI2-NEXT:    movl $-1, %ebx
 ; X86-BMI1BMI2-NEXT:    shrxl %ecx, %ebx, %ebp
 ; X86-BMI1BMI2-NEXT:    shrdl %cl, %ebx, %ebx
 ; X86-BMI1BMI2-NEXT:    testb $32, %cl
-; X86-BMI1BMI2-NEXT:    je .LBB33_4
+; X86-BMI1BMI2-NEXT:    je .LBB42_4
 ; X86-BMI1BMI2-NEXT:  # %bb.3:
 ; X86-BMI1BMI2-NEXT:    movl %ebp, %ebx
 ; X86-BMI1BMI2-NEXT:    xorl %ebp, %ebp
-; X86-BMI1BMI2-NEXT:  .LBB33_4:
+; X86-BMI1BMI2-NEXT:  .LBB42_4:
 ; X86-BMI1BMI2-NEXT:    subl $8, %esp
 ; X86-BMI1BMI2-NEXT:    pushl %ebp
 ; X86-BMI1BMI2-NEXT:    pushl %ebx
@@ -4164,13 +5403,13 @@ define i64 @bextr64_c1_indexzext(i64 %val, i8 %numskipbits, i8 %numlowbits) noun
 ; X86-BMI1BMI2-NEXT:    popl %ebp
 ; X86-BMI1BMI2-NEXT:    retl
 ;
-; X64-NOBMI-LABEL: bextr64_c1_indexzext:
+; X64-NOBMI-LABEL: bextr64_c3_load_indexzext:
 ; X64-NOBMI:       # %bb.0:
 ; X64-NOBMI-NEXT:    pushq %r14
 ; X64-NOBMI-NEXT:    pushq %rbx
 ; X64-NOBMI-NEXT:    pushq %rax
 ; X64-NOBMI-NEXT:    movl %esi, %ecx
-; X64-NOBMI-NEXT:    movq %rdi, %r14
+; X64-NOBMI-NEXT:    movq (%rdi), %r14
 ; X64-NOBMI-NEXT:    # kill: def $cl killed $cl killed $ecx
 ; X64-NOBMI-NEXT:    shrq %cl, %r14
 ; X64-NOBMI-NEXT:    negb %dl
@@ -4186,13 +5425,13 @@ define i64 @bextr64_c1_indexzext(i64 %val, i8 %numskipbits, i8 %numlowbits) noun
 ; X64-NOBMI-NEXT:    popq %r14
 ; X64-NOBMI-NEXT:    retq
 ;
-; X64-BMI1NOTBM-LABEL: bextr64_c1_indexzext:
+; X64-BMI1NOTBM-LABEL: bextr64_c3_load_indexzext:
 ; X64-BMI1NOTBM:       # %bb.0:
 ; X64-BMI1NOTBM-NEXT:    pushq %r14
 ; X64-BMI1NOTBM-NEXT:    pushq %rbx
 ; X64-BMI1NOTBM-NEXT:    pushq %rax
 ; X64-BMI1NOTBM-NEXT:    movl %esi, %ecx
-; X64-BMI1NOTBM-NEXT:    movq %rdi, %r14
+; X64-BMI1NOTBM-NEXT:    movq (%rdi), %r14
 ; X64-BMI1NOTBM-NEXT:    # kill: def $cl killed $cl killed $ecx
 ; X64-BMI1NOTBM-NEXT:    shrq %cl, %r14
 ; X64-BMI1NOTBM-NEXT:    negb %dl
@@ -4208,14 +5447,14 @@ define i64 @bextr64_c1_indexzext(i64 %val, i8 %numskipbits, i8 %numlowbits) noun
 ; X64-BMI1NOTBM-NEXT:    popq %r14
 ; X64-BMI1NOTBM-NEXT:    retq
 ;
-; X64-BMI1BMI2-LABEL: bextr64_c1_indexzext:
+; X64-BMI1BMI2-LABEL: bextr64_c3_load_indexzext:
 ; X64-BMI1BMI2:       # %bb.0:
 ; X64-BMI1BMI2-NEXT:    pushq %r14
 ; X64-BMI1BMI2-NEXT:    pushq %rbx
 ; X64-BMI1BMI2-NEXT:    pushq %rax
 ; X64-BMI1BMI2-NEXT:    movl %edx, %ebx
 ; X64-BMI1BMI2-NEXT:    # kill: def $esi killed $esi def $rsi
-; X64-BMI1BMI2-NEXT:    shrxq %rsi, %rdi, %r14
+; X64-BMI1BMI2-NEXT:    shrxq %rsi, (%rdi), %r14
 ; X64-BMI1BMI2-NEXT:    movl %ebx, %eax
 ; X64-BMI1BMI2-NEXT:    negb %al
 ; X64-BMI1BMI2-NEXT:    movq $-1, %rcx
@@ -4226,6 +5465,7 @@ define i64 @bextr64_c1_indexzext(i64 %val, i8 %numskipbits, i8 %numlowbits) noun
 ; X64-BMI1BMI2-NEXT:    popq %rbx
 ; X64-BMI1BMI2-NEXT:    popq %r14
 ; X64-BMI1BMI2-NEXT:    retq
+  %val = load i64, i64* %w
   %skip = zext i8 %numskipbits to i64
   %shifted = lshr i64 %val, %skip
   %numhighbits = sub i8 64, %numlowbits
@@ -4236,8 +5476,8 @@ define i64 @bextr64_c1_indexzext(i64 %val, i8 %numskipbits, i8 %numlowbits) noun
   ret i64 %masked
 }
 
-define i64 @bextr64_c2_load(i64* %w, i64 %numskipbits, i64 %numlowbits) nounwind {
-; X86-NOBMI-LABEL: bextr64_c2_load:
+define i64 @bextr64_c4_commutative(i64 %val, i64 %numskipbits, i64 %numlowbits) nounwind {
+; X86-NOBMI-LABEL: bextr64_c4_commutative:
 ; X86-NOBMI:       # %bb.0:
 ; X86-NOBMI-NEXT:    pushl %ebp
 ; X86-NOBMI-NEXT:    pushl %ebx
@@ -4245,18 +5485,17 @@ define i64 @bextr64_c2_load(i64* %w, i64 %numskipbits, i64 %numlowbits) nounwind
 ; X86-NOBMI-NEXT:    pushl %esi
 ; X86-NOBMI-NEXT:    subl $12, %esp
 ; X86-NOBMI-NEXT:    movb {{[0-9]+}}(%esp), %cl
+; X86-NOBMI-NEXT:    movl {{[0-9]+}}(%esp), %esi
 ; X86-NOBMI-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NOBMI-NEXT:    movl (%eax), %esi
-; X86-NOBMI-NEXT:    movl 4(%eax), %eax
 ; X86-NOBMI-NEXT:    movl %eax, %edi
 ; X86-NOBMI-NEXT:    shrl %cl, %edi
 ; X86-NOBMI-NEXT:    shrdl %cl, %eax, %esi
 ; X86-NOBMI-NEXT:    testb $32, %cl
-; X86-NOBMI-NEXT:    je .LBB34_2
+; X86-NOBMI-NEXT:    je .LBB43_2
 ; X86-NOBMI-NEXT:  # %bb.1:
 ; X86-NOBMI-NEXT:    movl %edi, %esi
 ; X86-NOBMI-NEXT:    xorl %edi, %edi
-; X86-NOBMI-NEXT:  .LBB34_2:
+; X86-NOBMI-NEXT:  .LBB43_2:
 ; X86-NOBMI-NEXT:    movb $64, %cl
 ; X86-NOBMI-NEXT:    subb {{[0-9]+}}(%esp), %cl
 ; X86-NOBMI-NEXT:    movl $-1, %ebp
@@ -4264,11 +5503,11 @@ define i64 @bextr64_c2_load(i64* %w, i64 %numskipbits, i64 %numlowbits) nounwind
 ; X86-NOBMI-NEXT:    shrl %cl, %ebx
 ; X86-NOBMI-NEXT:    shrdl %cl, %ebp, %ebp
 ; X86-NOBMI-NEXT:    testb $32, %cl
-; X86-NOBMI-NEXT:    je .LBB34_4
+; X86-NOBMI-NEXT:    je .LBB43_4
 ; X86-NOBMI-NEXT:  # %bb.3:
 ; X86-NOBMI-NEXT:    movl %ebx, %ebp
 ; X86-NOBMI-NEXT:    xorl %ebx, %ebx
-; X86-NOBMI-NEXT:  .LBB34_4:
+; X86-NOBMI-NEXT:  .LBB43_4:
 ; X86-NOBMI-NEXT:    subl $8, %esp
 ; X86-NOBMI-NEXT:    pushl %ebx
 ; X86-NOBMI-NEXT:    pushl %ebp
@@ -4285,7 +5524,7 @@ define i64 @bextr64_c2_load(i64* %w, i64 %numskipbits, i64 %numlowbits) nounwind
 ; X86-NOBMI-NEXT:    popl %ebp
 ; X86-NOBMI-NEXT:    retl
 ;
-; X86-BMI1NOTBM-LABEL: bextr64_c2_load:
+; X86-BMI1NOTBM-LABEL: bextr64_c4_commutative:
 ; X86-BMI1NOTBM:       # %bb.0:
 ; X86-BMI1NOTBM-NEXT:    pushl %ebp
 ; X86-BMI1NOTBM-NEXT:    pushl %ebx
@@ -4293,18 +5532,17 @@ define i64 @bextr64_c2_load(i64* %w, i64 %numskipbits, i64 %numlowbits) nounwind
 ; X86-BMI1NOTBM-NEXT:    pushl %esi
 ; X86-BMI1NOTBM-NEXT:    subl $12, %esp
 ; X86-BMI1NOTBM-NEXT:    movb {{[0-9]+}}(%esp), %cl
+; X86-BMI1NOTBM-NEXT:    movl {{[0-9]+}}(%esp), %esi
 ; X86-BMI1NOTBM-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-BMI1NOTBM-NEXT:    movl (%eax), %esi
-; X86-BMI1NOTBM-NEXT:    movl 4(%eax), %eax
 ; X86-BMI1NOTBM-NEXT:    movl %eax, %edi
 ; X86-BMI1NOTBM-NEXT:    shrl %cl, %edi
 ; X86-BMI1NOTBM-NEXT:    shrdl %cl, %eax, %esi
 ; X86-BMI1NOTBM-NEXT:    testb $32, %cl
-; X86-BMI1NOTBM-NEXT:    je .LBB34_2
+; X86-BMI1NOTBM-NEXT:    je .LBB43_2
 ; X86-BMI1NOTBM-NEXT:  # %bb.1:
 ; X86-BMI1NOTBM-NEXT:    movl %edi, %esi
 ; X86-BMI1NOTBM-NEXT:    xorl %edi, %edi
-; X86-BMI1NOTBM-NEXT:  .LBB34_2:
+; X86-BMI1NOTBM-NEXT:  .LBB43_2:
 ; X86-BMI1NOTBM-NEXT:    movb $64, %cl
 ; X86-BMI1NOTBM-NEXT:    subb {{[0-9]+}}(%esp), %cl
 ; X86-BMI1NOTBM-NEXT:    movl $-1, %ebp
@@ -4312,11 +5550,11 @@ define i64 @bextr64_c2_load(i64* %w, i64 %numskipbits, i64 %numlowbits) nounwind
 ; X86-BMI1NOTBM-NEXT:    shrl %cl, %ebx
 ; X86-BMI1NOTBM-NEXT:    shrdl %cl, %ebp, %ebp
 ; X86-BMI1NOTBM-NEXT:    testb $32, %cl
-; X86-BMI1NOTBM-NEXT:    je .LBB34_4
+; X86-BMI1NOTBM-NEXT:    je .LBB43_4
 ; X86-BMI1NOTBM-NEXT:  # %bb.3:
 ; X86-BMI1NOTBM-NEXT:    movl %ebx, %ebp
 ; X86-BMI1NOTBM-NEXT:    xorl %ebx, %ebx
-; X86-BMI1NOTBM-NEXT:  .LBB34_4:
+; X86-BMI1NOTBM-NEXT:  .LBB43_4:
 ; X86-BMI1NOTBM-NEXT:    subl $8, %esp
 ; X86-BMI1NOTBM-NEXT:    pushl %ebx
 ; X86-BMI1NOTBM-NEXT:    pushl %ebp
@@ -4333,7 +5571,7 @@ define i64 @bextr64_c2_load(i64* %w, i64 %numskipbits, i64 %numlowbits) nounwind
 ; X86-BMI1NOTBM-NEXT:    popl %ebp
 ; X86-BMI1NOTBM-NEXT:    retl
 ;
-; X86-BMI1BMI2-LABEL: bextr64_c2_load:
+; X86-BMI1BMI2-LABEL: bextr64_c4_commutative:
 ; X86-BMI1BMI2:       # %bb.0:
 ; X86-BMI1BMI2-NEXT:    pushl %ebp
 ; X86-BMI1BMI2-NEXT:    pushl %ebx
@@ -4341,28 +5579,27 @@ define i64 @bextr64_c2_load(i64* %w, i64 %numskipbits, i64 %numlowbits) nounwind
 ; X86-BMI1BMI2-NEXT:    pushl %esi
 ; X86-BMI1BMI2-NEXT:    subl $12, %esp
 ; X86-BMI1BMI2-NEXT:    movb {{[0-9]+}}(%esp), %cl
+; X86-BMI1BMI2-NEXT:    movl {{[0-9]+}}(%esp), %esi
 ; X86-BMI1BMI2-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-BMI1BMI2-NEXT:    movl (%eax), %esi
-; X86-BMI1BMI2-NEXT:    movl 4(%eax), %eax
-; X86-BMI1BMI2-NEXT:    shrxl %ecx, %eax, %edi
 ; X86-BMI1BMI2-NEXT:    shrdl %cl, %eax, %esi
+; X86-BMI1BMI2-NEXT:    shrxl %ecx, %eax, %edi
 ; X86-BMI1BMI2-NEXT:    testb $32, %cl
-; X86-BMI1BMI2-NEXT:    je .LBB34_2
+; X86-BMI1BMI2-NEXT:    je .LBB43_2
 ; X86-BMI1BMI2-NEXT:  # %bb.1:
 ; X86-BMI1BMI2-NEXT:    movl %edi, %esi
 ; X86-BMI1BMI2-NEXT:    xorl %edi, %edi
-; X86-BMI1BMI2-NEXT:  .LBB34_2:
+; X86-BMI1BMI2-NEXT:  .LBB43_2:
 ; X86-BMI1BMI2-NEXT:    movb $64, %cl
 ; X86-BMI1BMI2-NEXT:    subb {{[0-9]+}}(%esp), %cl
 ; X86-BMI1BMI2-NEXT:    movl $-1, %ebx
 ; X86-BMI1BMI2-NEXT:    shrxl %ecx, %ebx, %ebp
 ; X86-BMI1BMI2-NEXT:    shrdl %cl, %ebx, %ebx
 ; X86-BMI1BMI2-NEXT:    testb $32, %cl
-; X86-BMI1BMI2-NEXT:    je .LBB34_4
+; X86-BMI1BMI2-NEXT:    je .LBB43_4
 ; X86-BMI1BMI2-NEXT:  # %bb.3:
 ; X86-BMI1BMI2-NEXT:    movl %ebp, %ebx
 ; X86-BMI1BMI2-NEXT:    xorl %ebp, %ebp
-; X86-BMI1BMI2-NEXT:  .LBB34_4:
+; X86-BMI1BMI2-NEXT:  .LBB43_4:
 ; X86-BMI1BMI2-NEXT:    subl $8, %esp
 ; X86-BMI1BMI2-NEXT:    pushl %ebp
 ; X86-BMI1BMI2-NEXT:    pushl %ebx
@@ -4379,13 +5616,13 @@ define i64 @bextr64_c2_load(i64* %w, i64 %numskipbits, i64 %numlowbits) nounwind
 ; X86-BMI1BMI2-NEXT:    popl %ebp
 ; X86-BMI1BMI2-NEXT:    retl
 ;
-; X64-NOBMI-LABEL: bextr64_c2_load:
+; X64-NOBMI-LABEL: bextr64_c4_commutative:
 ; X64-NOBMI:       # %bb.0:
 ; X64-NOBMI-NEXT:    pushq %r14
 ; X64-NOBMI-NEXT:    pushq %rbx
 ; X64-NOBMI-NEXT:    pushq %rax
 ; X64-NOBMI-NEXT:    movq %rsi, %rcx
-; X64-NOBMI-NEXT:    movq (%rdi), %r14
+; X64-NOBMI-NEXT:    movq %rdi, %r14
 ; X64-NOBMI-NEXT:    # kill: def $cl killed $cl killed $rcx
 ; X64-NOBMI-NEXT:    shrq %cl, %r14
 ; X64-NOBMI-NEXT:    negb %dl
@@ -4401,13 +5638,13 @@ define i64 @bextr64_c2_load(i64* %w, i64 %numskipbits, i64 %numlowbits) nounwind
 ; X64-NOBMI-NEXT:    popq %r14
 ; X64-NOBMI-NEXT:    retq
 ;
-; X64-BMI1NOTBM-LABEL: bextr64_c2_load:
+; X64-BMI1NOTBM-LABEL: bextr64_c4_commutative:
 ; X64-BMI1NOTBM:       # %bb.0:
 ; X64-BMI1NOTBM-NEXT:    pushq %r14
 ; X64-BMI1NOTBM-NEXT:    pushq %rbx
 ; X64-BMI1NOTBM-NEXT:    pushq %rax
 ; X64-BMI1NOTBM-NEXT:    movq %rsi, %rcx
-; X64-BMI1NOTBM-NEXT:    movq (%rdi), %r14
+; X64-BMI1NOTBM-NEXT:    movq %rdi, %r14
 ; X64-BMI1NOTBM-NEXT:    # kill: def $cl killed $cl killed $rcx
 ; X64-BMI1NOTBM-NEXT:    shrq %cl, %r14
 ; X64-BMI1NOTBM-NEXT:    negb %dl
@@ -4423,13 +5660,13 @@ define i64 @bextr64_c2_load(i64* %w, i64 %numskipbits, i64 %numlowbits) nounwind
 ; X64-BMI1NOTBM-NEXT:    popq %r14
 ; X64-BMI1NOTBM-NEXT:    retq
 ;
-; X64-BMI1BMI2-LABEL: bextr64_c2_load:
+; X64-BMI1BMI2-LABEL: bextr64_c4_commutative:
 ; X64-BMI1BMI2:       # %bb.0:
 ; X64-BMI1BMI2-NEXT:    pushq %r14
 ; X64-BMI1BMI2-NEXT:    pushq %rbx
 ; X64-BMI1BMI2-NEXT:    pushq %rax
 ; X64-BMI1BMI2-NEXT:    movq %rdx, %rbx
-; X64-BMI1BMI2-NEXT:    shrxq %rsi, (%rdi), %r14
+; X64-BMI1BMI2-NEXT:    shrxq %rsi, %rdi, %r14
 ; X64-BMI1BMI2-NEXT:    movl %ebx, %eax
 ; X64-BMI1BMI2-NEXT:    negb %al
 ; X64-BMI1BMI2-NEXT:    movq $-1, %rcx
@@ -4440,55 +5677,58 @@ define i64 @bextr64_c2_load(i64* %w, i64 %numskipbits, i64 %numlowbits) nounwind
 ; X64-BMI1BMI2-NEXT:    popq %rbx
 ; X64-BMI1BMI2-NEXT:    popq %r14
 ; X64-BMI1BMI2-NEXT:    retq
-  %val = load i64, i64* %w
   %shifted = lshr i64 %val, %numskipbits
   %numhighbits = sub i64 64, %numlowbits
   %mask = lshr i64 -1, %numhighbits
   call void @use64(i64 %mask)
-  %masked = and i64 %mask, %shifted
+  %masked = and i64 %shifted, %mask ; swapped order
   ret i64 %masked
 }
 
-define i64 @bextr64_c3_load_indexzext(i64* %w, i8 %numskipbits, i8 %numlowbits) nounwind {
-; X86-NOBMI-LABEL: bextr64_c3_load_indexzext:
+define i64 @bextr64_c5_skipextrauses(i64 %val, i64 %numskipbits, i64 %numlowbits) nounwind {
+; X86-NOBMI-LABEL: bextr64_c5_skipextrauses:
 ; X86-NOBMI:       # %bb.0:
 ; X86-NOBMI-NEXT:    pushl %ebp
 ; X86-NOBMI-NEXT:    pushl %ebx
 ; X86-NOBMI-NEXT:    pushl %edi
 ; X86-NOBMI-NEXT:    pushl %esi
 ; X86-NOBMI-NEXT:    subl $12, %esp
-; X86-NOBMI-NEXT:    movb {{[0-9]+}}(%esp), %cl
+; X86-NOBMI-NEXT:    movl {{[0-9]+}}(%esp), %esi
 ; X86-NOBMI-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NOBMI-NEXT:    movl (%eax), %esi
-; X86-NOBMI-NEXT:    movl 4(%eax), %eax
+; X86-NOBMI-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-NOBMI-NEXT:    movl %eax, %edi
 ; X86-NOBMI-NEXT:    shrl %cl, %edi
 ; X86-NOBMI-NEXT:    shrdl %cl, %eax, %esi
 ; X86-NOBMI-NEXT:    testb $32, %cl
-; X86-NOBMI-NEXT:    je .LBB35_2
+; X86-NOBMI-NEXT:    je .LBB44_2
 ; X86-NOBMI-NEXT:  # %bb.1:
 ; X86-NOBMI-NEXT:    movl %edi, %esi
 ; X86-NOBMI-NEXT:    xorl %edi, %edi
-; X86-NOBMI-NEXT:  .LBB35_2:
+; X86-NOBMI-NEXT:  .LBB44_2:
 ; X86-NOBMI-NEXT:    movb $64, %cl
 ; X86-NOBMI-NEXT:    subb {{[0-9]+}}(%esp), %cl
-; X86-NOBMI-NEXT:    movl $-1, %ebp
 ; X86-NOBMI-NEXT:    movl $-1, %ebx
-; X86-NOBMI-NEXT:    shrl %cl, %ebx
-; X86-NOBMI-NEXT:    shrdl %cl, %ebp, %ebp
+; X86-NOBMI-NEXT:    movl $-1, %ebp
+; X86-NOBMI-NEXT:    shrl %cl, %ebp
+; X86-NOBMI-NEXT:    shrdl %cl, %ebx, %ebx
 ; X86-NOBMI-NEXT:    testb $32, %cl
-; X86-NOBMI-NEXT:    je .LBB35_4
+; X86-NOBMI-NEXT:    je .LBB44_4
 ; X86-NOBMI-NEXT:  # %bb.3:
-; X86-NOBMI-NEXT:    movl %ebx, %ebp
-; X86-NOBMI-NEXT:    xorl %ebx, %ebx
-; X86-NOBMI-NEXT:  .LBB35_4:
+; X86-NOBMI-NEXT:    movl %ebp, %ebx
+; X86-NOBMI-NEXT:    xorl %ebp, %ebp
+; X86-NOBMI-NEXT:  .LBB44_4:
 ; X86-NOBMI-NEXT:    subl $8, %esp
-; X86-NOBMI-NEXT:    pushl %ebx
 ; X86-NOBMI-NEXT:    pushl %ebp
+; X86-NOBMI-NEXT:    pushl %ebx
+; X86-NOBMI-NEXT:    calll use64
+; X86-NOBMI-NEXT:    addl $16, %esp
+; X86-NOBMI-NEXT:    andl %ebx, %esi
+; X86-NOBMI-NEXT:    andl %ebp, %edi
+; X86-NOBMI-NEXT:    subl $8, %esp
+; X86-NOBMI-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NOBMI-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-NOBMI-NEXT:    calll use64
 ; X86-NOBMI-NEXT:    addl $16, %esp
-; X86-NOBMI-NEXT:    andl %ebp, %esi
-; X86-NOBMI-NEXT:    andl %ebx, %edi
 ; X86-NOBMI-NEXT:    movl %esi, %eax
 ; X86-NOBMI-NEXT:    movl %edi, %edx
 ; X86-NOBMI-NEXT:    addl $12, %esp
@@ -4498,45 +5738,49 @@ define i64 @bextr64_c3_load_indexzext(i64* %w, i8 %numskipbits, i8 %numlowbits)
 ; X86-NOBMI-NEXT:    popl %ebp
 ; X86-NOBMI-NEXT:    retl
 ;
-; X86-BMI1NOTBM-LABEL: bextr64_c3_load_indexzext:
+; X86-BMI1NOTBM-LABEL: bextr64_c5_skipextrauses:
 ; X86-BMI1NOTBM:       # %bb.0:
 ; X86-BMI1NOTBM-NEXT:    pushl %ebp
 ; X86-BMI1NOTBM-NEXT:    pushl %ebx
 ; X86-BMI1NOTBM-NEXT:    pushl %edi
 ; X86-BMI1NOTBM-NEXT:    pushl %esi
 ; X86-BMI1NOTBM-NEXT:    subl $12, %esp
-; X86-BMI1NOTBM-NEXT:    movb {{[0-9]+}}(%esp), %cl
+; X86-BMI1NOTBM-NEXT:    movl {{[0-9]+}}(%esp), %esi
 ; X86-BMI1NOTBM-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-BMI1NOTBM-NEXT:    movl (%eax), %esi
-; X86-BMI1NOTBM-NEXT:    movl 4(%eax), %eax
+; X86-BMI1NOTBM-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-BMI1NOTBM-NEXT:    movl %eax, %edi
 ; X86-BMI1NOTBM-NEXT:    shrl %cl, %edi
 ; X86-BMI1NOTBM-NEXT:    shrdl %cl, %eax, %esi
 ; X86-BMI1NOTBM-NEXT:    testb $32, %cl
-; X86-BMI1NOTBM-NEXT:    je .LBB35_2
+; X86-BMI1NOTBM-NEXT:    je .LBB44_2
 ; X86-BMI1NOTBM-NEXT:  # %bb.1:
 ; X86-BMI1NOTBM-NEXT:    movl %edi, %esi
 ; X86-BMI1NOTBM-NEXT:    xorl %edi, %edi
-; X86-BMI1NOTBM-NEXT:  .LBB35_2:
+; X86-BMI1NOTBM-NEXT:  .LBB44_2:
 ; X86-BMI1NOTBM-NEXT:    movb $64, %cl
 ; X86-BMI1NOTBM-NEXT:    subb {{[0-9]+}}(%esp), %cl
-; X86-BMI1NOTBM-NEXT:    movl $-1, %ebp
 ; X86-BMI1NOTBM-NEXT:    movl $-1, %ebx
-; X86-BMI1NOTBM-NEXT:    shrl %cl, %ebx
-; X86-BMI1NOTBM-NEXT:    shrdl %cl, %ebp, %ebp
+; X86-BMI1NOTBM-NEXT:    movl $-1, %ebp
+; X86-BMI1NOTBM-NEXT:    shrl %cl, %ebp
+; X86-BMI1NOTBM-NEXT:    shrdl %cl, %ebx, %ebx
 ; X86-BMI1NOTBM-NEXT:    testb $32, %cl
-; X86-BMI1NOTBM-NEXT:    je .LBB35_4
+; X86-BMI1NOTBM-NEXT:    je .LBB44_4
 ; X86-BMI1NOTBM-NEXT:  # %bb.3:
-; X86-BMI1NOTBM-NEXT:    movl %ebx, %ebp
-; X86-BMI1NOTBM-NEXT:    xorl %ebx, %ebx
-; X86-BMI1NOTBM-NEXT:  .LBB35_4:
+; X86-BMI1NOTBM-NEXT:    movl %ebp, %ebx
+; X86-BMI1NOTBM-NEXT:    xorl %ebp, %ebp
+; X86-BMI1NOTBM-NEXT:  .LBB44_4:
 ; X86-BMI1NOTBM-NEXT:    subl $8, %esp
-; X86-BMI1NOTBM-NEXT:    pushl %ebx
 ; X86-BMI1NOTBM-NEXT:    pushl %ebp
+; X86-BMI1NOTBM-NEXT:    pushl %ebx
+; X86-BMI1NOTBM-NEXT:    calll use64
+; X86-BMI1NOTBM-NEXT:    addl $16, %esp
+; X86-BMI1NOTBM-NEXT:    andl %ebx, %esi
+; X86-BMI1NOTBM-NEXT:    andl %ebp, %edi
+; X86-BMI1NOTBM-NEXT:    subl $8, %esp
+; X86-BMI1NOTBM-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-BMI1NOTBM-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-BMI1NOTBM-NEXT:    calll use64
 ; X86-BMI1NOTBM-NEXT:    addl $16, %esp
-; X86-BMI1NOTBM-NEXT:    andl %ebp, %esi
-; X86-BMI1NOTBM-NEXT:    andl %ebx, %edi
 ; X86-BMI1NOTBM-NEXT:    movl %esi, %eax
 ; X86-BMI1NOTBM-NEXT:    movl %edi, %edx
 ; X86-BMI1NOTBM-NEXT:    addl $12, %esp
@@ -4546,43 +5790,47 @@ define i64 @bextr64_c3_load_indexzext(i64* %w, i8 %numskipbits, i8 %numlowbits)
 ; X86-BMI1NOTBM-NEXT:    popl %ebp
 ; X86-BMI1NOTBM-NEXT:    retl
 ;
-; X86-BMI1BMI2-LABEL: bextr64_c3_load_indexzext:
+; X86-BMI1BMI2-LABEL: bextr64_c5_skipextrauses:
 ; X86-BMI1BMI2:       # %bb.0:
 ; X86-BMI1BMI2-NEXT:    pushl %ebp
 ; X86-BMI1BMI2-NEXT:    pushl %ebx
 ; X86-BMI1BMI2-NEXT:    pushl %edi
 ; X86-BMI1BMI2-NEXT:    pushl %esi
 ; X86-BMI1BMI2-NEXT:    subl $12, %esp
-; X86-BMI1BMI2-NEXT:    movb {{[0-9]+}}(%esp), %cl
+; X86-BMI1BMI2-NEXT:    movl {{[0-9]+}}(%esp), %esi
 ; X86-BMI1BMI2-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-BMI1BMI2-NEXT:    movl (%eax), %esi
-; X86-BMI1BMI2-NEXT:    movl 4(%eax), %eax
-; X86-BMI1BMI2-NEXT:    shrxl %ecx, %eax, %edi
+; X86-BMI1BMI2-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-BMI1BMI2-NEXT:    shrdl %cl, %eax, %esi
+; X86-BMI1BMI2-NEXT:    shrxl %ecx, %eax, %edi
 ; X86-BMI1BMI2-NEXT:    testb $32, %cl
-; X86-BMI1BMI2-NEXT:    je .LBB35_2
+; X86-BMI1BMI2-NEXT:    je .LBB44_2
 ; X86-BMI1BMI2-NEXT:  # %bb.1:
 ; X86-BMI1BMI2-NEXT:    movl %edi, %esi
 ; X86-BMI1BMI2-NEXT:    xorl %edi, %edi
-; X86-BMI1BMI2-NEXT:  .LBB35_2:
+; X86-BMI1BMI2-NEXT:  .LBB44_2:
 ; X86-BMI1BMI2-NEXT:    movb $64, %cl
 ; X86-BMI1BMI2-NEXT:    subb {{[0-9]+}}(%esp), %cl
-; X86-BMI1BMI2-NEXT:    movl $-1, %ebx
-; X86-BMI1BMI2-NEXT:    shrxl %ecx, %ebx, %ebp
-; X86-BMI1BMI2-NEXT:    shrdl %cl, %ebx, %ebx
+; X86-BMI1BMI2-NEXT:    movl $-1, %ebp
+; X86-BMI1BMI2-NEXT:    shrxl %ecx, %ebp, %ebx
+; X86-BMI1BMI2-NEXT:    shrdl %cl, %ebp, %ebp
 ; X86-BMI1BMI2-NEXT:    testb $32, %cl
-; X86-BMI1BMI2-NEXT:    je .LBB35_4
+; X86-BMI1BMI2-NEXT:    je .LBB44_4
 ; X86-BMI1BMI2-NEXT:  # %bb.3:
-; X86-BMI1BMI2-NEXT:    movl %ebp, %ebx
-; X86-BMI1BMI2-NEXT:    xorl %ebp, %ebp
-; X86-BMI1BMI2-NEXT:  .LBB35_4:
+; X86-BMI1BMI2-NEXT:    movl %ebx, %ebp
+; X86-BMI1BMI2-NEXT:    xorl %ebx, %ebx
+; X86-BMI1BMI2-NEXT:  .LBB44_4:
 ; X86-BMI1BMI2-NEXT:    subl $8, %esp
-; X86-BMI1BMI2-NEXT:    pushl %ebp
 ; X86-BMI1BMI2-NEXT:    pushl %ebx
+; X86-BMI1BMI2-NEXT:    pushl %ebp
+; X86-BMI1BMI2-NEXT:    calll use64
+; X86-BMI1BMI2-NEXT:    addl $16, %esp
+; X86-BMI1BMI2-NEXT:    andl %ebp, %esi
+; X86-BMI1BMI2-NEXT:    andl %ebx, %edi
+; X86-BMI1BMI2-NEXT:    subl $8, %esp
+; X86-BMI1BMI2-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-BMI1BMI2-NEXT:    pushl {{[0-9]+}}(%esp)
 ; X86-BMI1BMI2-NEXT:    calll use64
 ; X86-BMI1BMI2-NEXT:    addl $16, %esp
-; X86-BMI1BMI2-NEXT:    andl %ebx, %esi
-; X86-BMI1BMI2-NEXT:    andl %ebp, %edi
 ; X86-BMI1BMI2-NEXT:    movl %esi, %eax
 ; X86-BMI1BMI2-NEXT:    movl %edi, %edx
 ; X86-BMI1BMI2-NEXT:    addl $12, %esp
@@ -4592,519 +5840,411 @@ define i64 @bextr64_c3_load_indexzext(i64* %w, i8 %numskipbits, i8 %numlowbits)
 ; X86-BMI1BMI2-NEXT:    popl %ebp
 ; X86-BMI1BMI2-NEXT:    retl
 ;
-; X64-NOBMI-LABEL: bextr64_c3_load_indexzext:
+; X64-NOBMI-LABEL: bextr64_c5_skipextrauses:
 ; X64-NOBMI:       # %bb.0:
+; X64-NOBMI-NEXT:    pushq %r15
 ; X64-NOBMI-NEXT:    pushq %r14
 ; X64-NOBMI-NEXT:    pushq %rbx
-; X64-NOBMI-NEXT:    pushq %rax
-; X64-NOBMI-NEXT:    movl %esi, %ecx
-; X64-NOBMI-NEXT:    movq (%rdi), %r14
-; X64-NOBMI-NEXT:    # kill: def $cl killed $cl killed $ecx
-; X64-NOBMI-NEXT:    shrq %cl, %r14
+; X64-NOBMI-NEXT:    movq %rsi, %r14
+; X64-NOBMI-NEXT:    movq %rdi, %r15
+; X64-NOBMI-NEXT:    movl %r14d, %ecx
+; X64-NOBMI-NEXT:    shrq %cl, %r15
 ; X64-NOBMI-NEXT:    negb %dl
 ; X64-NOBMI-NEXT:    movq $-1, %rbx
 ; X64-NOBMI-NEXT:    movl %edx, %ecx
 ; X64-NOBMI-NEXT:    shrq %cl, %rbx
 ; X64-NOBMI-NEXT:    movq %rbx, %rdi
 ; X64-NOBMI-NEXT:    callq use64
-; X64-NOBMI-NEXT:    andq %r14, %rbx
+; X64-NOBMI-NEXT:    andq %r15, %rbx
+; X64-NOBMI-NEXT:    movq %r14, %rdi
+; X64-NOBMI-NEXT:    callq use64
 ; X64-NOBMI-NEXT:    movq %rbx, %rax
-; X64-NOBMI-NEXT:    addq $8, %rsp
 ; X64-NOBMI-NEXT:    popq %rbx
 ; X64-NOBMI-NEXT:    popq %r14
+; X64-NOBMI-NEXT:    popq %r15
 ; X64-NOBMI-NEXT:    retq
 ;
-; X64-BMI1NOTBM-LABEL: bextr64_c3_load_indexzext:
+; X64-BMI1NOTBM-LABEL: bextr64_c5_skipextrauses:
 ; X64-BMI1NOTBM:       # %bb.0:
+; X64-BMI1NOTBM-NEXT:    pushq %r15
 ; X64-BMI1NOTBM-NEXT:    pushq %r14
 ; X64-BMI1NOTBM-NEXT:    pushq %rbx
-; X64-BMI1NOTBM-NEXT:    pushq %rax
-; X64-BMI1NOTBM-NEXT:    movl %esi, %ecx
-; X64-BMI1NOTBM-NEXT:    movq (%rdi), %r14
-; X64-BMI1NOTBM-NEXT:    # kill: def $cl killed $cl killed $ecx
-; X64-BMI1NOTBM-NEXT:    shrq %cl, %r14
+; X64-BMI1NOTBM-NEXT:    movq %rsi, %r14
+; X64-BMI1NOTBM-NEXT:    movq %rdi, %r15
+; X64-BMI1NOTBM-NEXT:    movl %r14d, %ecx
+; X64-BMI1NOTBM-NEXT:    shrq %cl, %r15
 ; X64-BMI1NOTBM-NEXT:    negb %dl
 ; X64-BMI1NOTBM-NEXT:    movq $-1, %rbx
 ; X64-BMI1NOTBM-NEXT:    movl %edx, %ecx
 ; X64-BMI1NOTBM-NEXT:    shrq %cl, %rbx
 ; X64-BMI1NOTBM-NEXT:    movq %rbx, %rdi
 ; X64-BMI1NOTBM-NEXT:    callq use64
-; X64-BMI1NOTBM-NEXT:    andq %r14, %rbx
+; X64-BMI1NOTBM-NEXT:    andq %r15, %rbx
+; X64-BMI1NOTBM-NEXT:    movq %r14, %rdi
+; X64-BMI1NOTBM-NEXT:    callq use64
 ; X64-BMI1NOTBM-NEXT:    movq %rbx, %rax
-; X64-BMI1NOTBM-NEXT:    addq $8, %rsp
 ; X64-BMI1NOTBM-NEXT:    popq %rbx
 ; X64-BMI1NOTBM-NEXT:    popq %r14
+; X64-BMI1NOTBM-NEXT:    popq %r15
 ; X64-BMI1NOTBM-NEXT:    retq
 ;
-; X64-BMI1BMI2-LABEL: bextr64_c3_load_indexzext:
+; X64-BMI1BMI2-LABEL: bextr64_c5_skipextrauses:
 ; X64-BMI1BMI2:       # %bb.0:
+; X64-BMI1BMI2-NEXT:    pushq %r15
 ; X64-BMI1BMI2-NEXT:    pushq %r14
 ; X64-BMI1BMI2-NEXT:    pushq %rbx
-; X64-BMI1BMI2-NEXT:    pushq %rax
-; X64-BMI1BMI2-NEXT:    movl %edx, %ebx
-; X64-BMI1BMI2-NEXT:    # kill: def $esi killed $esi def $rsi
-; X64-BMI1BMI2-NEXT:    shrxq %rsi, (%rdi), %r14
+; X64-BMI1BMI2-NEXT:    movq %rdx, %rbx
+; X64-BMI1BMI2-NEXT:    movq %rsi, %r14
+; X64-BMI1BMI2-NEXT:    shrxq %rsi, %rdi, %r15
 ; X64-BMI1BMI2-NEXT:    movl %ebx, %eax
 ; X64-BMI1BMI2-NEXT:    negb %al
 ; X64-BMI1BMI2-NEXT:    movq $-1, %rcx
 ; X64-BMI1BMI2-NEXT:    shrxq %rax, %rcx, %rdi
 ; X64-BMI1BMI2-NEXT:    callq use64
-; X64-BMI1BMI2-NEXT:    bzhiq %rbx, %r14, %rax
-; X64-BMI1BMI2-NEXT:    addq $8, %rsp
+; X64-BMI1BMI2-NEXT:    bzhiq %rbx, %r15, %rbx
+; X64-BMI1BMI2-NEXT:    movq %r14, %rdi
+; X64-BMI1BMI2-NEXT:    callq use64
+; X64-BMI1BMI2-NEXT:    movq %rbx, %rax
 ; X64-BMI1BMI2-NEXT:    popq %rbx
 ; X64-BMI1BMI2-NEXT:    popq %r14
+; X64-BMI1BMI2-NEXT:    popq %r15
 ; X64-BMI1BMI2-NEXT:    retq
-  %val = load i64, i64* %w
-  %skip = zext i8 %numskipbits to i64
-  %shifted = lshr i64 %val, %skip
-  %numhighbits = sub i8 64, %numlowbits
-  %sh_prom = zext i8 %numhighbits to i64
-  %mask = lshr i64 -1, %sh_prom
+  %shifted = lshr i64 %val, %numskipbits
+  %numhighbits = sub i64 64, %numlowbits
+  %mask = lshr i64 -1, %numhighbits
   call void @use64(i64 %mask)
   %masked = and i64 %mask, %shifted
+  call void @use64(i64 %numskipbits)
   ret i64 %masked
 }
 
-define i64 @bextr64_c4_commutative(i64 %val, i64 %numskipbits, i64 %numlowbits) nounwind {
-; X86-NOBMI-LABEL: bextr64_c4_commutative:
+; 64-bit, but with 32-bit output
+
+; Everything done in 64-bit, truncation happens last.
+define i32 @bextr64_32_c0(i64 %val, i64 %numskipbits, i64 %numlowbits) nounwind {
+; X86-NOBMI-LABEL: bextr64_32_c0:
 ; X86-NOBMI:       # %bb.0:
-; X86-NOBMI-NEXT:    pushl %ebp
-; X86-NOBMI-NEXT:    pushl %ebx
-; X86-NOBMI-NEXT:    pushl %edi
 ; X86-NOBMI-NEXT:    pushl %esi
-; X86-NOBMI-NEXT:    subl $12, %esp
 ; X86-NOBMI-NEXT:    movb {{[0-9]+}}(%esp), %cl
-; X86-NOBMI-NEXT:    movl {{[0-9]+}}(%esp), %esi
 ; X86-NOBMI-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NOBMI-NEXT:    movl %eax, %edi
-; X86-NOBMI-NEXT:    shrl %cl, %edi
-; X86-NOBMI-NEXT:    shrdl %cl, %eax, %esi
+; X86-NOBMI-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NOBMI-NEXT:    movl %esi, %edx
+; X86-NOBMI-NEXT:    shrl %cl, %edx
+; X86-NOBMI-NEXT:    shrdl %cl, %esi, %eax
 ; X86-NOBMI-NEXT:    testb $32, %cl
-; X86-NOBMI-NEXT:    je .LBB36_2
+; X86-NOBMI-NEXT:    jne .LBB45_2
 ; X86-NOBMI-NEXT:  # %bb.1:
-; X86-NOBMI-NEXT:    movl %edi, %esi
-; X86-NOBMI-NEXT:    xorl %edi, %edi
-; X86-NOBMI-NEXT:  .LBB36_2:
+; X86-NOBMI-NEXT:    movl %eax, %edx
+; X86-NOBMI-NEXT:  .LBB45_2:
 ; X86-NOBMI-NEXT:    movb $64, %cl
 ; X86-NOBMI-NEXT:    subb {{[0-9]+}}(%esp), %cl
-; X86-NOBMI-NEXT:    movl $-1, %ebp
-; X86-NOBMI-NEXT:    movl $-1, %ebx
-; X86-NOBMI-NEXT:    shrl %cl, %ebx
-; X86-NOBMI-NEXT:    shrdl %cl, %ebp, %ebp
+; X86-NOBMI-NEXT:    movl $-1, %esi
+; X86-NOBMI-NEXT:    movl $-1, %eax
+; X86-NOBMI-NEXT:    shrl %cl, %eax
+; X86-NOBMI-NEXT:    shrdl %cl, %esi, %esi
 ; X86-NOBMI-NEXT:    testb $32, %cl
-; X86-NOBMI-NEXT:    je .LBB36_4
+; X86-NOBMI-NEXT:    jne .LBB45_4
 ; X86-NOBMI-NEXT:  # %bb.3:
-; X86-NOBMI-NEXT:    movl %ebx, %ebp
-; X86-NOBMI-NEXT:    xorl %ebx, %ebx
-; X86-NOBMI-NEXT:  .LBB36_4:
-; X86-NOBMI-NEXT:    subl $8, %esp
-; X86-NOBMI-NEXT:    pushl %ebx
-; X86-NOBMI-NEXT:    pushl %ebp
-; X86-NOBMI-NEXT:    calll use64
-; X86-NOBMI-NEXT:    addl $16, %esp
-; X86-NOBMI-NEXT:    andl %ebp, %esi
-; X86-NOBMI-NEXT:    andl %ebx, %edi
 ; X86-NOBMI-NEXT:    movl %esi, %eax
-; X86-NOBMI-NEXT:    movl %edi, %edx
-; X86-NOBMI-NEXT:    addl $12, %esp
+; X86-NOBMI-NEXT:  .LBB45_4:
+; X86-NOBMI-NEXT:    andl %edx, %eax
 ; X86-NOBMI-NEXT:    popl %esi
-; X86-NOBMI-NEXT:    popl %edi
-; X86-NOBMI-NEXT:    popl %ebx
-; X86-NOBMI-NEXT:    popl %ebp
 ; X86-NOBMI-NEXT:    retl
 ;
-; X86-BMI1NOTBM-LABEL: bextr64_c4_commutative:
+; X86-BMI1NOTBM-LABEL: bextr64_32_c0:
 ; X86-BMI1NOTBM:       # %bb.0:
-; X86-BMI1NOTBM-NEXT:    pushl %ebp
-; X86-BMI1NOTBM-NEXT:    pushl %ebx
-; X86-BMI1NOTBM-NEXT:    pushl %edi
 ; X86-BMI1NOTBM-NEXT:    pushl %esi
-; X86-BMI1NOTBM-NEXT:    subl $12, %esp
 ; X86-BMI1NOTBM-NEXT:    movb {{[0-9]+}}(%esp), %cl
-; X86-BMI1NOTBM-NEXT:    movl {{[0-9]+}}(%esp), %esi
 ; X86-BMI1NOTBM-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-BMI1NOTBM-NEXT:    movl %eax, %edi
-; X86-BMI1NOTBM-NEXT:    shrl %cl, %edi
-; X86-BMI1NOTBM-NEXT:    shrdl %cl, %eax, %esi
+; X86-BMI1NOTBM-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-BMI1NOTBM-NEXT:    movl %esi, %edx
+; X86-BMI1NOTBM-NEXT:    shrl %cl, %edx
+; X86-BMI1NOTBM-NEXT:    shrdl %cl, %esi, %eax
 ; X86-BMI1NOTBM-NEXT:    testb $32, %cl
-; X86-BMI1NOTBM-NEXT:    je .LBB36_2
+; X86-BMI1NOTBM-NEXT:    jne .LBB45_2
 ; X86-BMI1NOTBM-NEXT:  # %bb.1:
-; X86-BMI1NOTBM-NEXT:    movl %edi, %esi
-; X86-BMI1NOTBM-NEXT:    xorl %edi, %edi
-; X86-BMI1NOTBM-NEXT:  .LBB36_2:
+; X86-BMI1NOTBM-NEXT:    movl %eax, %edx
+; X86-BMI1NOTBM-NEXT:  .LBB45_2:
 ; X86-BMI1NOTBM-NEXT:    movb $64, %cl
 ; X86-BMI1NOTBM-NEXT:    subb {{[0-9]+}}(%esp), %cl
-; X86-BMI1NOTBM-NEXT:    movl $-1, %ebp
-; X86-BMI1NOTBM-NEXT:    movl $-1, %ebx
-; X86-BMI1NOTBM-NEXT:    shrl %cl, %ebx
-; X86-BMI1NOTBM-NEXT:    shrdl %cl, %ebp, %ebp
+; X86-BMI1NOTBM-NEXT:    movl $-1, %esi
+; X86-BMI1NOTBM-NEXT:    movl $-1, %eax
+; X86-BMI1NOTBM-NEXT:    shrl %cl, %eax
+; X86-BMI1NOTBM-NEXT:    shrdl %cl, %esi, %esi
 ; X86-BMI1NOTBM-NEXT:    testb $32, %cl
-; X86-BMI1NOTBM-NEXT:    je .LBB36_4
+; X86-BMI1NOTBM-NEXT:    jne .LBB45_4
 ; X86-BMI1NOTBM-NEXT:  # %bb.3:
-; X86-BMI1NOTBM-NEXT:    movl %ebx, %ebp
-; X86-BMI1NOTBM-NEXT:    xorl %ebx, %ebx
-; X86-BMI1NOTBM-NEXT:  .LBB36_4:
-; X86-BMI1NOTBM-NEXT:    subl $8, %esp
-; X86-BMI1NOTBM-NEXT:    pushl %ebx
-; X86-BMI1NOTBM-NEXT:    pushl %ebp
-; X86-BMI1NOTBM-NEXT:    calll use64
-; X86-BMI1NOTBM-NEXT:    addl $16, %esp
-; X86-BMI1NOTBM-NEXT:    andl %ebp, %esi
-; X86-BMI1NOTBM-NEXT:    andl %ebx, %edi
 ; X86-BMI1NOTBM-NEXT:    movl %esi, %eax
-; X86-BMI1NOTBM-NEXT:    movl %edi, %edx
-; X86-BMI1NOTBM-NEXT:    addl $12, %esp
+; X86-BMI1NOTBM-NEXT:  .LBB45_4:
+; X86-BMI1NOTBM-NEXT:    andl %edx, %eax
 ; X86-BMI1NOTBM-NEXT:    popl %esi
-; X86-BMI1NOTBM-NEXT:    popl %edi
-; X86-BMI1NOTBM-NEXT:    popl %ebx
-; X86-BMI1NOTBM-NEXT:    popl %ebp
 ; X86-BMI1NOTBM-NEXT:    retl
 ;
-; X86-BMI1BMI2-LABEL: bextr64_c4_commutative:
+; X86-BMI1BMI2-LABEL: bextr64_32_c0:
 ; X86-BMI1BMI2:       # %bb.0:
-; X86-BMI1BMI2-NEXT:    pushl %ebp
-; X86-BMI1BMI2-NEXT:    pushl %ebx
-; X86-BMI1BMI2-NEXT:    pushl %edi
 ; X86-BMI1BMI2-NEXT:    pushl %esi
-; X86-BMI1BMI2-NEXT:    subl $12, %esp
 ; X86-BMI1BMI2-NEXT:    movb {{[0-9]+}}(%esp), %cl
-; X86-BMI1BMI2-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-BMI1BMI2-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; X86-BMI1BMI2-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-BMI1BMI2-NEXT:    shrdl %cl, %eax, %esi
-; X86-BMI1BMI2-NEXT:    shrxl %ecx, %eax, %edi
+; X86-BMI1BMI2-NEXT:    shrdl %cl, %eax, %edx
 ; X86-BMI1BMI2-NEXT:    testb $32, %cl
-; X86-BMI1BMI2-NEXT:    je .LBB36_2
+; X86-BMI1BMI2-NEXT:    je .LBB45_2
 ; X86-BMI1BMI2-NEXT:  # %bb.1:
-; X86-BMI1BMI2-NEXT:    movl %edi, %esi
-; X86-BMI1BMI2-NEXT:    xorl %edi, %edi
-; X86-BMI1BMI2-NEXT:  .LBB36_2:
+; X86-BMI1BMI2-NEXT:    shrxl %ecx, %eax, %edx
+; X86-BMI1BMI2-NEXT:  .LBB45_2:
 ; X86-BMI1BMI2-NEXT:    movb $64, %cl
 ; X86-BMI1BMI2-NEXT:    subb {{[0-9]+}}(%esp), %cl
-; X86-BMI1BMI2-NEXT:    movl $-1, %ebx
-; X86-BMI1BMI2-NEXT:    shrxl %ecx, %ebx, %ebp
-; X86-BMI1BMI2-NEXT:    shrdl %cl, %ebx, %ebx
+; X86-BMI1BMI2-NEXT:    movl $-1, %esi
+; X86-BMI1BMI2-NEXT:    movl $-1, %eax
+; X86-BMI1BMI2-NEXT:    shrdl %cl, %eax, %eax
 ; X86-BMI1BMI2-NEXT:    testb $32, %cl
-; X86-BMI1BMI2-NEXT:    je .LBB36_4
+; X86-BMI1BMI2-NEXT:    je .LBB45_4
 ; X86-BMI1BMI2-NEXT:  # %bb.3:
-; X86-BMI1BMI2-NEXT:    movl %ebp, %ebx
-; X86-BMI1BMI2-NEXT:    xorl %ebp, %ebp
-; X86-BMI1BMI2-NEXT:  .LBB36_4:
-; X86-BMI1BMI2-NEXT:    subl $8, %esp
-; X86-BMI1BMI2-NEXT:    pushl %ebp
-; X86-BMI1BMI2-NEXT:    pushl %ebx
-; X86-BMI1BMI2-NEXT:    calll use64
-; X86-BMI1BMI2-NEXT:    addl $16, %esp
-; X86-BMI1BMI2-NEXT:    andl %ebx, %esi
-; X86-BMI1BMI2-NEXT:    andl %ebp, %edi
-; X86-BMI1BMI2-NEXT:    movl %esi, %eax
-; X86-BMI1BMI2-NEXT:    movl %edi, %edx
-; X86-BMI1BMI2-NEXT:    addl $12, %esp
+; X86-BMI1BMI2-NEXT:    shrxl %ecx, %esi, %eax
+; X86-BMI1BMI2-NEXT:  .LBB45_4:
+; X86-BMI1BMI2-NEXT:    andl %edx, %eax
+; X86-BMI1BMI2-NEXT:    popl %esi
+; X86-BMI1BMI2-NEXT:    retl
+;
+; X64-NOBMI-LABEL: bextr64_32_c0:
+; X64-NOBMI:       # %bb.0:
+; X64-NOBMI-NEXT:    movq %rsi, %rcx
+; X64-NOBMI-NEXT:    # kill: def $cl killed $cl killed $rcx
+; X64-NOBMI-NEXT:    shrq %cl, %rdi
+; X64-NOBMI-NEXT:    negb %dl
+; X64-NOBMI-NEXT:    movq $-1, %rax
+; X64-NOBMI-NEXT:    movl %edx, %ecx
+; X64-NOBMI-NEXT:    shrq %cl, %rax
+; X64-NOBMI-NEXT:    andl %edi, %eax
+; X64-NOBMI-NEXT:    # kill: def $eax killed $eax killed $rax
+; X64-NOBMI-NEXT:    retq
+;
+; X64-BMI1NOTBM-LABEL: bextr64_32_c0:
+; X64-BMI1NOTBM:       # %bb.0:
+; X64-BMI1NOTBM-NEXT:    movq %rsi, %rcx
+; X64-BMI1NOTBM-NEXT:    # kill: def $cl killed $cl killed $rcx
+; X64-BMI1NOTBM-NEXT:    shrq %cl, %rdi
+; X64-BMI1NOTBM-NEXT:    negb %dl
+; X64-BMI1NOTBM-NEXT:    movq $-1, %rax
+; X64-BMI1NOTBM-NEXT:    movl %edx, %ecx
+; X64-BMI1NOTBM-NEXT:    shrq %cl, %rax
+; X64-BMI1NOTBM-NEXT:    andl %edi, %eax
+; X64-BMI1NOTBM-NEXT:    # kill: def $eax killed $eax killed $rax
+; X64-BMI1NOTBM-NEXT:    retq
+;
+; X64-BMI1BMI2-LABEL: bextr64_32_c0:
+; X64-BMI1BMI2:       # %bb.0:
+; X64-BMI1BMI2-NEXT:    shrxq %rsi, %rdi, %rcx
+; X64-BMI1BMI2-NEXT:    negb %dl
+; X64-BMI1BMI2-NEXT:    movq $-1, %rax
+; X64-BMI1BMI2-NEXT:    shrxq %rdx, %rax, %rax
+; X64-BMI1BMI2-NEXT:    andl %ecx, %eax
+; X64-BMI1BMI2-NEXT:    # kill: def $eax killed $eax killed $rax
+; X64-BMI1BMI2-NEXT:    retq
+  %shifted = lshr i64 %val, %numskipbits
+  %numhighbits = sub i64 64, %numlowbits
+  %mask = lshr i64 -1, %numhighbits
+  %masked = and i64 %mask, %shifted
+  %res = trunc i64 %masked to i32
+  ret i32 %res
+}
+
+; Shifting happens in 64-bit, then truncation. Masking is 32-bit.
+define i32 @bextr64_32_c1(i64 %val, i64 %numskipbits, i32 %numlowbits) nounwind {
+; X86-NOBMI-LABEL: bextr64_32_c1:
+; X86-NOBMI:       # %bb.0:
+; X86-NOBMI-NEXT:    pushl %esi
+; X86-NOBMI-NEXT:    movb {{[0-9]+}}(%esp), %cl
+; X86-NOBMI-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NOBMI-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NOBMI-NEXT:    movl %esi, %eax
+; X86-NOBMI-NEXT:    shrl %cl, %eax
+; X86-NOBMI-NEXT:    shrdl %cl, %esi, %edx
+; X86-NOBMI-NEXT:    testb $32, %cl
+; X86-NOBMI-NEXT:    jne .LBB46_2
+; X86-NOBMI-NEXT:  # %bb.1:
+; X86-NOBMI-NEXT:    movl %edx, %eax
+; X86-NOBMI-NEXT:  .LBB46_2:
+; X86-NOBMI-NEXT:    xorl %ecx, %ecx
+; X86-NOBMI-NEXT:    subb {{[0-9]+}}(%esp), %cl
+; X86-NOBMI-NEXT:    shll %cl, %eax
+; X86-NOBMI-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X86-NOBMI-NEXT:    shrl %cl, %eax
+; X86-NOBMI-NEXT:    popl %esi
+; X86-NOBMI-NEXT:    retl
+;
+; X86-BMI1NOTBM-LABEL: bextr64_32_c1:
+; X86-BMI1NOTBM:       # %bb.0:
+; X86-BMI1NOTBM-NEXT:    pushl %edi
+; X86-BMI1NOTBM-NEXT:    pushl %esi
+; X86-BMI1NOTBM-NEXT:    movb {{[0-9]+}}(%esp), %al
+; X86-BMI1NOTBM-NEXT:    movb {{[0-9]+}}(%esp), %cl
+; X86-BMI1NOTBM-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-BMI1NOTBM-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-BMI1NOTBM-NEXT:    movl %edi, %edx
+; X86-BMI1NOTBM-NEXT:    shrl %cl, %edx
+; X86-BMI1NOTBM-NEXT:    shrdl %cl, %edi, %esi
+; X86-BMI1NOTBM-NEXT:    testb $32, %cl
+; X86-BMI1NOTBM-NEXT:    jne .LBB46_2
+; X86-BMI1NOTBM-NEXT:  # %bb.1:
+; X86-BMI1NOTBM-NEXT:    movl %esi, %edx
+; X86-BMI1NOTBM-NEXT:  .LBB46_2:
+; X86-BMI1NOTBM-NEXT:    shll $8, %eax
+; X86-BMI1NOTBM-NEXT:    bextrl %eax, %edx, %eax
+; X86-BMI1NOTBM-NEXT:    popl %esi
+; X86-BMI1NOTBM-NEXT:    popl %edi
+; X86-BMI1NOTBM-NEXT:    retl
+;
+; X86-BMI1BMI2-LABEL: bextr64_32_c1:
+; X86-BMI1BMI2:       # %bb.0:
+; X86-BMI1BMI2-NEXT:    pushl %esi
+; X86-BMI1BMI2-NEXT:    movb {{[0-9]+}}(%esp), %al
+; X86-BMI1BMI2-NEXT:    movb {{[0-9]+}}(%esp), %cl
+; X86-BMI1BMI2-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-BMI1BMI2-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-BMI1BMI2-NEXT:    shrdl %cl, %esi, %edx
+; X86-BMI1BMI2-NEXT:    testb $32, %cl
+; X86-BMI1BMI2-NEXT:    je .LBB46_2
+; X86-BMI1BMI2-NEXT:  # %bb.1:
+; X86-BMI1BMI2-NEXT:    shrxl %ecx, %esi, %edx
+; X86-BMI1BMI2-NEXT:  .LBB46_2:
+; X86-BMI1BMI2-NEXT:    bzhil %eax, %edx, %eax
 ; X86-BMI1BMI2-NEXT:    popl %esi
-; X86-BMI1BMI2-NEXT:    popl %edi
-; X86-BMI1BMI2-NEXT:    popl %ebx
-; X86-BMI1BMI2-NEXT:    popl %ebp
 ; X86-BMI1BMI2-NEXT:    retl
 ;
-; X64-NOBMI-LABEL: bextr64_c4_commutative:
+; X64-NOBMI-LABEL: bextr64_32_c1:
 ; X64-NOBMI:       # %bb.0:
-; X64-NOBMI-NEXT:    pushq %r14
-; X64-NOBMI-NEXT:    pushq %rbx
-; X64-NOBMI-NEXT:    pushq %rax
 ; X64-NOBMI-NEXT:    movq %rsi, %rcx
-; X64-NOBMI-NEXT:    movq %rdi, %r14
+; X64-NOBMI-NEXT:    movq %rdi, %rax
 ; X64-NOBMI-NEXT:    # kill: def $cl killed $cl killed $rcx
-; X64-NOBMI-NEXT:    shrq %cl, %r14
+; X64-NOBMI-NEXT:    shrq %cl, %rax
 ; X64-NOBMI-NEXT:    negb %dl
-; X64-NOBMI-NEXT:    movq $-1, %rbx
 ; X64-NOBMI-NEXT:    movl %edx, %ecx
-; X64-NOBMI-NEXT:    shrq %cl, %rbx
-; X64-NOBMI-NEXT:    movq %rbx, %rdi
-; X64-NOBMI-NEXT:    callq use64
-; X64-NOBMI-NEXT:    andq %r14, %rbx
-; X64-NOBMI-NEXT:    movq %rbx, %rax
-; X64-NOBMI-NEXT:    addq $8, %rsp
-; X64-NOBMI-NEXT:    popq %rbx
-; X64-NOBMI-NEXT:    popq %r14
+; X64-NOBMI-NEXT:    shll %cl, %eax
+; X64-NOBMI-NEXT:    shrl %cl, %eax
+; X64-NOBMI-NEXT:    # kill: def $eax killed $eax killed $rax
 ; X64-NOBMI-NEXT:    retq
 ;
-; X64-BMI1NOTBM-LABEL: bextr64_c4_commutative:
+; X64-BMI1NOTBM-LABEL: bextr64_32_c1:
 ; X64-BMI1NOTBM:       # %bb.0:
-; X64-BMI1NOTBM-NEXT:    pushq %r14
-; X64-BMI1NOTBM-NEXT:    pushq %rbx
-; X64-BMI1NOTBM-NEXT:    pushq %rax
-; X64-BMI1NOTBM-NEXT:    movq %rsi, %rcx
-; X64-BMI1NOTBM-NEXT:    movq %rdi, %r14
-; X64-BMI1NOTBM-NEXT:    # kill: def $cl killed $cl killed $rcx
-; X64-BMI1NOTBM-NEXT:    shrq %cl, %r14
-; X64-BMI1NOTBM-NEXT:    negb %dl
-; X64-BMI1NOTBM-NEXT:    movq $-1, %rbx
-; X64-BMI1NOTBM-NEXT:    movl %edx, %ecx
-; X64-BMI1NOTBM-NEXT:    shrq %cl, %rbx
-; X64-BMI1NOTBM-NEXT:    movq %rbx, %rdi
-; X64-BMI1NOTBM-NEXT:    callq use64
-; X64-BMI1NOTBM-NEXT:    andq %r14, %rbx
-; X64-BMI1NOTBM-NEXT:    movq %rbx, %rax
-; X64-BMI1NOTBM-NEXT:    addq $8, %rsp
-; X64-BMI1NOTBM-NEXT:    popq %rbx
-; X64-BMI1NOTBM-NEXT:    popq %r14
+; X64-BMI1NOTBM-NEXT:    shll $8, %edx
+; X64-BMI1NOTBM-NEXT:    movzbl %sil, %eax
+; X64-BMI1NOTBM-NEXT:    orl %edx, %eax
+; X64-BMI1NOTBM-NEXT:    bextrq %rax, %rdi, %rax
+; X64-BMI1NOTBM-NEXT:    # kill: def $eax killed $eax killed $rax
 ; X64-BMI1NOTBM-NEXT:    retq
 ;
-; X64-BMI1BMI2-LABEL: bextr64_c4_commutative:
+; X64-BMI1BMI2-LABEL: bextr64_32_c1:
 ; X64-BMI1BMI2:       # %bb.0:
-; X64-BMI1BMI2-NEXT:    pushq %r14
-; X64-BMI1BMI2-NEXT:    pushq %rbx
-; X64-BMI1BMI2-NEXT:    pushq %rax
-; X64-BMI1BMI2-NEXT:    movq %rdx, %rbx
-; X64-BMI1BMI2-NEXT:    shrxq %rsi, %rdi, %r14
-; X64-BMI1BMI2-NEXT:    movl %ebx, %eax
-; X64-BMI1BMI2-NEXT:    negb %al
-; X64-BMI1BMI2-NEXT:    movq $-1, %rcx
-; X64-BMI1BMI2-NEXT:    shrxq %rax, %rcx, %rdi
-; X64-BMI1BMI2-NEXT:    callq use64
-; X64-BMI1BMI2-NEXT:    bzhiq %rbx, %r14, %rax
-; X64-BMI1BMI2-NEXT:    addq $8, %rsp
-; X64-BMI1BMI2-NEXT:    popq %rbx
-; X64-BMI1BMI2-NEXT:    popq %r14
+; X64-BMI1BMI2-NEXT:    shrxq %rsi, %rdi, %rax
+; X64-BMI1BMI2-NEXT:    bzhil %edx, %eax, %eax
 ; X64-BMI1BMI2-NEXT:    retq
   %shifted = lshr i64 %val, %numskipbits
-  %numhighbits = sub i64 64, %numlowbits
-  %mask = lshr i64 -1, %numhighbits
-  call void @use64(i64 %mask)
-  %masked = and i64 %shifted, %mask ; swapped order
-  ret i64 %masked
+  %truncshifted = trunc i64 %shifted to i32
+  %numhighbits = sub i32 32, %numlowbits
+  %mask = lshr i32 -1, %numhighbits
+  %masked = and i32 %mask, %truncshifted
+  ret i32 %masked
 }
 
-define i64 @bextr64_c5_skipextrauses(i64 %val, i64 %numskipbits, i64 %numlowbits) nounwind {
-; X86-NOBMI-LABEL: bextr64_c5_skipextrauses:
+; Shifting happens in 64-bit. Mask is 32-bit, but extended to 64-bit.
+; Masking is 64-bit. Then truncation.
+define i32 @bextr64_32_c2(i64 %val, i64 %numskipbits, i32 %numlowbits) nounwind {
+; X86-NOBMI-LABEL: bextr64_32_c2:
 ; X86-NOBMI:       # %bb.0:
-; X86-NOBMI-NEXT:    pushl %ebp
-; X86-NOBMI-NEXT:    pushl %ebx
-; X86-NOBMI-NEXT:    pushl %edi
 ; X86-NOBMI-NEXT:    pushl %esi
-; X86-NOBMI-NEXT:    subl $12, %esp
+; X86-NOBMI-NEXT:    movb {{[0-9]+}}(%esp), %cl
+; X86-NOBMI-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; X86-NOBMI-NEXT:    movl {{[0-9]+}}(%esp), %esi
-; X86-NOBMI-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NOBMI-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-NOBMI-NEXT:    movl %eax, %edi
-; X86-NOBMI-NEXT:    shrl %cl, %edi
-; X86-NOBMI-NEXT:    shrdl %cl, %eax, %esi
+; X86-NOBMI-NEXT:    movl %esi, %eax
+; X86-NOBMI-NEXT:    shrl %cl, %eax
+; X86-NOBMI-NEXT:    shrdl %cl, %esi, %edx
 ; X86-NOBMI-NEXT:    testb $32, %cl
-; X86-NOBMI-NEXT:    je .LBB37_2
+; X86-NOBMI-NEXT:    jne .LBB47_2
 ; X86-NOBMI-NEXT:  # %bb.1:
-; X86-NOBMI-NEXT:    movl %edi, %esi
-; X86-NOBMI-NEXT:    xorl %edi, %edi
-; X86-NOBMI-NEXT:  .LBB37_2:
-; X86-NOBMI-NEXT:    movb $64, %cl
+; X86-NOBMI-NEXT:    movl %edx, %eax
+; X86-NOBMI-NEXT:  .LBB47_2:
+; X86-NOBMI-NEXT:    xorl %ecx, %ecx
 ; X86-NOBMI-NEXT:    subb {{[0-9]+}}(%esp), %cl
-; X86-NOBMI-NEXT:    movl $-1, %ebx
-; X86-NOBMI-NEXT:    movl $-1, %ebp
-; X86-NOBMI-NEXT:    shrl %cl, %ebp
-; X86-NOBMI-NEXT:    shrdl %cl, %ebx, %ebx
-; X86-NOBMI-NEXT:    testb $32, %cl
-; X86-NOBMI-NEXT:    je .LBB37_4
-; X86-NOBMI-NEXT:  # %bb.3:
-; X86-NOBMI-NEXT:    movl %ebp, %ebx
-; X86-NOBMI-NEXT:    xorl %ebp, %ebp
-; X86-NOBMI-NEXT:  .LBB37_4:
-; X86-NOBMI-NEXT:    subl $8, %esp
-; X86-NOBMI-NEXT:    pushl %ebp
-; X86-NOBMI-NEXT:    pushl %ebx
-; X86-NOBMI-NEXT:    calll use64
-; X86-NOBMI-NEXT:    addl $16, %esp
-; X86-NOBMI-NEXT:    andl %ebx, %esi
-; X86-NOBMI-NEXT:    andl %ebp, %edi
-; X86-NOBMI-NEXT:    subl $8, %esp
-; X86-NOBMI-NEXT:    pushl {{[0-9]+}}(%esp)
-; X86-NOBMI-NEXT:    pushl {{[0-9]+}}(%esp)
-; X86-NOBMI-NEXT:    calll use64
-; X86-NOBMI-NEXT:    addl $16, %esp
-; X86-NOBMI-NEXT:    movl %esi, %eax
-; X86-NOBMI-NEXT:    movl %edi, %edx
-; X86-NOBMI-NEXT:    addl $12, %esp
+; X86-NOBMI-NEXT:    shll %cl, %eax
+; X86-NOBMI-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X86-NOBMI-NEXT:    shrl %cl, %eax
 ; X86-NOBMI-NEXT:    popl %esi
-; X86-NOBMI-NEXT:    popl %edi
-; X86-NOBMI-NEXT:    popl %ebx
-; X86-NOBMI-NEXT:    popl %ebp
 ; X86-NOBMI-NEXT:    retl
 ;
-; X86-BMI1NOTBM-LABEL: bextr64_c5_skipextrauses:
+; X86-BMI1NOTBM-LABEL: bextr64_32_c2:
 ; X86-BMI1NOTBM:       # %bb.0:
-; X86-BMI1NOTBM-NEXT:    pushl %ebp
-; X86-BMI1NOTBM-NEXT:    pushl %ebx
 ; X86-BMI1NOTBM-NEXT:    pushl %edi
 ; X86-BMI1NOTBM-NEXT:    pushl %esi
-; X86-BMI1NOTBM-NEXT:    subl $12, %esp
+; X86-BMI1NOTBM-NEXT:    movb {{[0-9]+}}(%esp), %al
+; X86-BMI1NOTBM-NEXT:    movb {{[0-9]+}}(%esp), %cl
 ; X86-BMI1NOTBM-NEXT:    movl {{[0-9]+}}(%esp), %esi
-; X86-BMI1NOTBM-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-BMI1NOTBM-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-BMI1NOTBM-NEXT:    movl %eax, %edi
-; X86-BMI1NOTBM-NEXT:    shrl %cl, %edi
-; X86-BMI1NOTBM-NEXT:    shrdl %cl, %eax, %esi
+; X86-BMI1NOTBM-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-BMI1NOTBM-NEXT:    movl %edi, %edx
+; X86-BMI1NOTBM-NEXT:    shrl %cl, %edx
+; X86-BMI1NOTBM-NEXT:    shrdl %cl, %edi, %esi
 ; X86-BMI1NOTBM-NEXT:    testb $32, %cl
-; X86-BMI1NOTBM-NEXT:    je .LBB37_2
+; X86-BMI1NOTBM-NEXT:    jne .LBB47_2
 ; X86-BMI1NOTBM-NEXT:  # %bb.1:
-; X86-BMI1NOTBM-NEXT:    movl %edi, %esi
-; X86-BMI1NOTBM-NEXT:    xorl %edi, %edi
-; X86-BMI1NOTBM-NEXT:  .LBB37_2:
-; X86-BMI1NOTBM-NEXT:    movb $64, %cl
-; X86-BMI1NOTBM-NEXT:    subb {{[0-9]+}}(%esp), %cl
-; X86-BMI1NOTBM-NEXT:    movl $-1, %ebx
-; X86-BMI1NOTBM-NEXT:    movl $-1, %ebp
-; X86-BMI1NOTBM-NEXT:    shrl %cl, %ebp
-; X86-BMI1NOTBM-NEXT:    shrdl %cl, %ebx, %ebx
-; X86-BMI1NOTBM-NEXT:    testb $32, %cl
-; X86-BMI1NOTBM-NEXT:    je .LBB37_4
-; X86-BMI1NOTBM-NEXT:  # %bb.3:
-; X86-BMI1NOTBM-NEXT:    movl %ebp, %ebx
-; X86-BMI1NOTBM-NEXT:    xorl %ebp, %ebp
-; X86-BMI1NOTBM-NEXT:  .LBB37_4:
-; X86-BMI1NOTBM-NEXT:    subl $8, %esp
-; X86-BMI1NOTBM-NEXT:    pushl %ebp
-; X86-BMI1NOTBM-NEXT:    pushl %ebx
-; X86-BMI1NOTBM-NEXT:    calll use64
-; X86-BMI1NOTBM-NEXT:    addl $16, %esp
-; X86-BMI1NOTBM-NEXT:    andl %ebx, %esi
-; X86-BMI1NOTBM-NEXT:    andl %ebp, %edi
-; X86-BMI1NOTBM-NEXT:    subl $8, %esp
-; X86-BMI1NOTBM-NEXT:    pushl {{[0-9]+}}(%esp)
-; X86-BMI1NOTBM-NEXT:    pushl {{[0-9]+}}(%esp)
-; X86-BMI1NOTBM-NEXT:    calll use64
-; X86-BMI1NOTBM-NEXT:    addl $16, %esp
-; X86-BMI1NOTBM-NEXT:    movl %esi, %eax
-; X86-BMI1NOTBM-NEXT:    movl %edi, %edx
-; X86-BMI1NOTBM-NEXT:    addl $12, %esp
+; X86-BMI1NOTBM-NEXT:    movl %esi, %edx
+; X86-BMI1NOTBM-NEXT:  .LBB47_2:
+; X86-BMI1NOTBM-NEXT:    shll $8, %eax
+; X86-BMI1NOTBM-NEXT:    bextrl %eax, %edx, %eax
 ; X86-BMI1NOTBM-NEXT:    popl %esi
 ; X86-BMI1NOTBM-NEXT:    popl %edi
-; X86-BMI1NOTBM-NEXT:    popl %ebx
-; X86-BMI1NOTBM-NEXT:    popl %ebp
 ; X86-BMI1NOTBM-NEXT:    retl
 ;
-; X86-BMI1BMI2-LABEL: bextr64_c5_skipextrauses:
+; X86-BMI1BMI2-LABEL: bextr64_32_c2:
 ; X86-BMI1BMI2:       # %bb.0:
-; X86-BMI1BMI2-NEXT:    pushl %ebp
-; X86-BMI1BMI2-NEXT:    pushl %ebx
-; X86-BMI1BMI2-NEXT:    pushl %edi
 ; X86-BMI1BMI2-NEXT:    pushl %esi
-; X86-BMI1BMI2-NEXT:    subl $12, %esp
+; X86-BMI1BMI2-NEXT:    movb {{[0-9]+}}(%esp), %al
+; X86-BMI1BMI2-NEXT:    movb {{[0-9]+}}(%esp), %cl
+; X86-BMI1BMI2-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; X86-BMI1BMI2-NEXT:    movl {{[0-9]+}}(%esp), %esi
-; X86-BMI1BMI2-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-BMI1BMI2-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-BMI1BMI2-NEXT:    shrdl %cl, %eax, %esi
-; X86-BMI1BMI2-NEXT:    shrxl %ecx, %eax, %edi
+; X86-BMI1BMI2-NEXT:    shrdl %cl, %esi, %edx
 ; X86-BMI1BMI2-NEXT:    testb $32, %cl
-; X86-BMI1BMI2-NEXT:    je .LBB37_2
+; X86-BMI1BMI2-NEXT:    je .LBB47_2
 ; X86-BMI1BMI2-NEXT:  # %bb.1:
-; X86-BMI1BMI2-NEXT:    movl %edi, %esi
-; X86-BMI1BMI2-NEXT:    xorl %edi, %edi
-; X86-BMI1BMI2-NEXT:  .LBB37_2:
-; X86-BMI1BMI2-NEXT:    movb $64, %cl
-; X86-BMI1BMI2-NEXT:    subb {{[0-9]+}}(%esp), %cl
-; X86-BMI1BMI2-NEXT:    movl $-1, %ebp
-; X86-BMI1BMI2-NEXT:    shrxl %ecx, %ebp, %ebx
-; X86-BMI1BMI2-NEXT:    shrdl %cl, %ebp, %ebp
-; X86-BMI1BMI2-NEXT:    testb $32, %cl
-; X86-BMI1BMI2-NEXT:    je .LBB37_4
-; X86-BMI1BMI2-NEXT:  # %bb.3:
-; X86-BMI1BMI2-NEXT:    movl %ebx, %ebp
-; X86-BMI1BMI2-NEXT:    xorl %ebx, %ebx
-; X86-BMI1BMI2-NEXT:  .LBB37_4:
-; X86-BMI1BMI2-NEXT:    subl $8, %esp
-; X86-BMI1BMI2-NEXT:    pushl %ebx
-; X86-BMI1BMI2-NEXT:    pushl %ebp
-; X86-BMI1BMI2-NEXT:    calll use64
-; X86-BMI1BMI2-NEXT:    addl $16, %esp
-; X86-BMI1BMI2-NEXT:    andl %ebp, %esi
-; X86-BMI1BMI2-NEXT:    andl %ebx, %edi
-; X86-BMI1BMI2-NEXT:    subl $8, %esp
-; X86-BMI1BMI2-NEXT:    pushl {{[0-9]+}}(%esp)
-; X86-BMI1BMI2-NEXT:    pushl {{[0-9]+}}(%esp)
-; X86-BMI1BMI2-NEXT:    calll use64
-; X86-BMI1BMI2-NEXT:    addl $16, %esp
-; X86-BMI1BMI2-NEXT:    movl %esi, %eax
-; X86-BMI1BMI2-NEXT:    movl %edi, %edx
-; X86-BMI1BMI2-NEXT:    addl $12, %esp
+; X86-BMI1BMI2-NEXT:    shrxl %ecx, %esi, %edx
+; X86-BMI1BMI2-NEXT:  .LBB47_2:
+; X86-BMI1BMI2-NEXT:    bzhil %eax, %edx, %eax
 ; X86-BMI1BMI2-NEXT:    popl %esi
-; X86-BMI1BMI2-NEXT:    popl %edi
-; X86-BMI1BMI2-NEXT:    popl %ebx
-; X86-BMI1BMI2-NEXT:    popl %ebp
 ; X86-BMI1BMI2-NEXT:    retl
 ;
-; X64-NOBMI-LABEL: bextr64_c5_skipextrauses:
+; X64-NOBMI-LABEL: bextr64_32_c2:
 ; X64-NOBMI:       # %bb.0:
-; X64-NOBMI-NEXT:    pushq %r15
-; X64-NOBMI-NEXT:    pushq %r14
-; X64-NOBMI-NEXT:    pushq %rbx
-; X64-NOBMI-NEXT:    movq %rsi, %r14
-; X64-NOBMI-NEXT:    movq %rdi, %r15
-; X64-NOBMI-NEXT:    movl %r14d, %ecx
-; X64-NOBMI-NEXT:    shrq %cl, %r15
+; X64-NOBMI-NEXT:    movq %rsi, %rcx
+; X64-NOBMI-NEXT:    movq %rdi, %rax
+; X64-NOBMI-NEXT:    # kill: def $cl killed $cl killed $rcx
+; X64-NOBMI-NEXT:    shrq %cl, %rax
 ; X64-NOBMI-NEXT:    negb %dl
-; X64-NOBMI-NEXT:    movq $-1, %rbx
 ; X64-NOBMI-NEXT:    movl %edx, %ecx
-; X64-NOBMI-NEXT:    shrq %cl, %rbx
-; X64-NOBMI-NEXT:    movq %rbx, %rdi
-; X64-NOBMI-NEXT:    callq use64
-; X64-NOBMI-NEXT:    andq %r15, %rbx
-; X64-NOBMI-NEXT:    movq %r14, %rdi
-; X64-NOBMI-NEXT:    callq use64
-; X64-NOBMI-NEXT:    movq %rbx, %rax
-; X64-NOBMI-NEXT:    popq %rbx
-; X64-NOBMI-NEXT:    popq %r14
-; X64-NOBMI-NEXT:    popq %r15
+; X64-NOBMI-NEXT:    shll %cl, %eax
+; X64-NOBMI-NEXT:    shrl %cl, %eax
+; X64-NOBMI-NEXT:    # kill: def $eax killed $eax killed $rax
 ; X64-NOBMI-NEXT:    retq
 ;
-; X64-BMI1NOTBM-LABEL: bextr64_c5_skipextrauses:
+; X64-BMI1NOTBM-LABEL: bextr64_32_c2:
 ; X64-BMI1NOTBM:       # %bb.0:
-; X64-BMI1NOTBM-NEXT:    pushq %r15
-; X64-BMI1NOTBM-NEXT:    pushq %r14
-; X64-BMI1NOTBM-NEXT:    pushq %rbx
-; X64-BMI1NOTBM-NEXT:    movq %rsi, %r14
-; X64-BMI1NOTBM-NEXT:    movq %rdi, %r15
-; X64-BMI1NOTBM-NEXT:    movl %r14d, %ecx
-; X64-BMI1NOTBM-NEXT:    shrq %cl, %r15
-; X64-BMI1NOTBM-NEXT:    negb %dl
-; X64-BMI1NOTBM-NEXT:    movq $-1, %rbx
-; X64-BMI1NOTBM-NEXT:    movl %edx, %ecx
-; X64-BMI1NOTBM-NEXT:    shrq %cl, %rbx
-; X64-BMI1NOTBM-NEXT:    movq %rbx, %rdi
-; X64-BMI1NOTBM-NEXT:    callq use64
-; X64-BMI1NOTBM-NEXT:    andq %r15, %rbx
-; X64-BMI1NOTBM-NEXT:    movq %r14, %rdi
-; X64-BMI1NOTBM-NEXT:    callq use64
-; X64-BMI1NOTBM-NEXT:    movq %rbx, %rax
-; X64-BMI1NOTBM-NEXT:    popq %rbx
-; X64-BMI1NOTBM-NEXT:    popq %r14
-; X64-BMI1NOTBM-NEXT:    popq %r15
+; X64-BMI1NOTBM-NEXT:    shll $8, %edx
+; X64-BMI1NOTBM-NEXT:    movzbl %sil, %eax
+; X64-BMI1NOTBM-NEXT:    orl %edx, %eax
+; X64-BMI1NOTBM-NEXT:    bextrq %rax, %rdi, %rax
+; X64-BMI1NOTBM-NEXT:    # kill: def $eax killed $eax killed $rax
 ; X64-BMI1NOTBM-NEXT:    retq
 ;
-; X64-BMI1BMI2-LABEL: bextr64_c5_skipextrauses:
+; X64-BMI1BMI2-LABEL: bextr64_32_c2:
 ; X64-BMI1BMI2:       # %bb.0:
-; X64-BMI1BMI2-NEXT:    pushq %r15
-; X64-BMI1BMI2-NEXT:    pushq %r14
-; X64-BMI1BMI2-NEXT:    pushq %rbx
-; X64-BMI1BMI2-NEXT:    movq %rdx, %rbx
-; X64-BMI1BMI2-NEXT:    movq %rsi, %r14
-; X64-BMI1BMI2-NEXT:    shrxq %rsi, %rdi, %r15
-; X64-BMI1BMI2-NEXT:    movl %ebx, %eax
-; X64-BMI1BMI2-NEXT:    negb %al
-; X64-BMI1BMI2-NEXT:    movq $-1, %rcx
-; X64-BMI1BMI2-NEXT:    shrxq %rax, %rcx, %rdi
-; X64-BMI1BMI2-NEXT:    callq use64
-; X64-BMI1BMI2-NEXT:    bzhiq %rbx, %r15, %rbx
-; X64-BMI1BMI2-NEXT:    movq %r14, %rdi
-; X64-BMI1BMI2-NEXT:    callq use64
-; X64-BMI1BMI2-NEXT:    movq %rbx, %rax
-; X64-BMI1BMI2-NEXT:    popq %rbx
-; X64-BMI1BMI2-NEXT:    popq %r14
-; X64-BMI1BMI2-NEXT:    popq %r15
+; X64-BMI1BMI2-NEXT:    shrxq %rsi, %rdi, %rax
+; X64-BMI1BMI2-NEXT:    bzhil %edx, %eax, %eax
 ; X64-BMI1BMI2-NEXT:    retq
   %shifted = lshr i64 %val, %numskipbits
-  %numhighbits = sub i64 64, %numlowbits
-  %mask = lshr i64 -1, %numhighbits
-  call void @use64(i64 %mask)
-  %masked = and i64 %mask, %shifted
-  call void @use64(i64 %numskipbits)
-  ret i64 %masked
+  %numhighbits = sub i32 32, %numlowbits
+  %mask = lshr i32 -1, %numhighbits
+  %zextmask = zext i32 %mask to i64
+  %masked = and i64 %zextmask, %shifted
+  %truncmasked = trunc i64 %masked to i32
+  ret i32 %truncmasked
 }
 
 ; ---------------------------------------------------------------------------- ;
@@ -5487,36 +6627,36 @@ define i64 @bextr64_d0(i64 %val, i64 %numskipbits, i64 %numlowbits) nounwind {
 ; X86-NOBMI-NEXT:    shrdl %cl, %edx, %edi
 ; X86-NOBMI-NEXT:    xorl %esi, %esi
 ; X86-NOBMI-NEXT:    testb $32, %cl
-; X86-NOBMI-NEXT:    je .LBB43_2
+; X86-NOBMI-NEXT:    je .LBB53_2
 ; X86-NOBMI-NEXT:  # %bb.1:
 ; X86-NOBMI-NEXT:    movl %eax, %edi
 ; X86-NOBMI-NEXT:    xorl %eax, %eax
-; X86-NOBMI-NEXT:  .LBB43_2:
+; X86-NOBMI-NEXT:  .LBB53_2:
 ; X86-NOBMI-NEXT:    movb $64, %cl
 ; X86-NOBMI-NEXT:    subb {{[0-9]+}}(%esp), %cl
 ; X86-NOBMI-NEXT:    shldl %cl, %edi, %eax
 ; X86-NOBMI-NEXT:    shll %cl, %edi
 ; X86-NOBMI-NEXT:    testb $32, %cl
 ; X86-NOBMI-NEXT:    movl %edi, %ebx
-; X86-NOBMI-NEXT:    jne .LBB43_4
+; X86-NOBMI-NEXT:    jne .LBB53_4
 ; X86-NOBMI-NEXT:  # %bb.3:
 ; X86-NOBMI-NEXT:    movl %eax, %ebx
-; X86-NOBMI-NEXT:  .LBB43_4:
+; X86-NOBMI-NEXT:  .LBB53_4:
 ; X86-NOBMI-NEXT:    movl %ebx, %eax
 ; X86-NOBMI-NEXT:    shrl %cl, %eax
 ; X86-NOBMI-NEXT:    testb $32, %cl
 ; X86-NOBMI-NEXT:    movl $0, %edx
-; X86-NOBMI-NEXT:    jne .LBB43_6
+; X86-NOBMI-NEXT:    jne .LBB53_6
 ; X86-NOBMI-NEXT:  # %bb.5:
 ; X86-NOBMI-NEXT:    movl %edi, %esi
 ; X86-NOBMI-NEXT:    movl %eax, %edx
-; X86-NOBMI-NEXT:  .LBB43_6:
+; X86-NOBMI-NEXT:  .LBB53_6:
 ; X86-NOBMI-NEXT:    shrdl %cl, %ebx, %esi
 ; X86-NOBMI-NEXT:    testb $32, %cl
-; X86-NOBMI-NEXT:    jne .LBB43_8
+; X86-NOBMI-NEXT:    jne .LBB53_8
 ; X86-NOBMI-NEXT:  # %bb.7:
 ; X86-NOBMI-NEXT:    movl %esi, %eax
-; X86-NOBMI-NEXT:  .LBB43_8:
+; X86-NOBMI-NEXT:  .LBB53_8:
 ; X86-NOBMI-NEXT:    popl %esi
 ; X86-NOBMI-NEXT:    popl %edi
 ; X86-NOBMI-NEXT:    popl %ebx
@@ -5535,36 +6675,36 @@ define i64 @bextr64_d0(i64 %val, i64 %numskipbits, i64 %numlowbits) nounwind {
 ; X86-BMI1NOTBM-NEXT:    shrdl %cl, %edx, %edi
 ; X86-BMI1NOTBM-NEXT:    xorl %esi, %esi
 ; X86-BMI1NOTBM-NEXT:    testb $32, %cl
-; X86-BMI1NOTBM-NEXT:    je .LBB43_2
+; X86-BMI1NOTBM-NEXT:    je .LBB53_2
 ; X86-BMI1NOTBM-NEXT:  # %bb.1:
 ; X86-BMI1NOTBM-NEXT:    movl %eax, %edi
 ; X86-BMI1NOTBM-NEXT:    xorl %eax, %eax
-; X86-BMI1NOTBM-NEXT:  .LBB43_2:
+; X86-BMI1NOTBM-NEXT:  .LBB53_2:
 ; X86-BMI1NOTBM-NEXT:    movb $64, %cl
 ; X86-BMI1NOTBM-NEXT:    subb {{[0-9]+}}(%esp), %cl
 ; X86-BMI1NOTBM-NEXT:    shldl %cl, %edi, %eax
 ; X86-BMI1NOTBM-NEXT:    shll %cl, %edi
 ; X86-BMI1NOTBM-NEXT:    testb $32, %cl
 ; X86-BMI1NOTBM-NEXT:    movl %edi, %ebx
-; X86-BMI1NOTBM-NEXT:    jne .LBB43_4
+; X86-BMI1NOTBM-NEXT:    jne .LBB53_4
 ; X86-BMI1NOTBM-NEXT:  # %bb.3:
 ; X86-BMI1NOTBM-NEXT:    movl %eax, %ebx
-; X86-BMI1NOTBM-NEXT:  .LBB43_4:
+; X86-BMI1NOTBM-NEXT:  .LBB53_4:
 ; X86-BMI1NOTBM-NEXT:    movl %ebx, %eax
 ; X86-BMI1NOTBM-NEXT:    shrl %cl, %eax
 ; X86-BMI1NOTBM-NEXT:    testb $32, %cl
 ; X86-BMI1NOTBM-NEXT:    movl $0, %edx
-; X86-BMI1NOTBM-NEXT:    jne .LBB43_6
+; X86-BMI1NOTBM-NEXT:    jne .LBB53_6
 ; X86-BMI1NOTBM-NEXT:  # %bb.5:
 ; X86-BMI1NOTBM-NEXT:    movl %edi, %esi
 ; X86-BMI1NOTBM-NEXT:    movl %eax, %edx
-; X86-BMI1NOTBM-NEXT:  .LBB43_6:
+; X86-BMI1NOTBM-NEXT:  .LBB53_6:
 ; X86-BMI1NOTBM-NEXT:    shrdl %cl, %ebx, %esi
 ; X86-BMI1NOTBM-NEXT:    testb $32, %cl
-; X86-BMI1NOTBM-NEXT:    jne .LBB43_8
+; X86-BMI1NOTBM-NEXT:    jne .LBB53_8
 ; X86-BMI1NOTBM-NEXT:  # %bb.7:
 ; X86-BMI1NOTBM-NEXT:    movl %esi, %eax
-; X86-BMI1NOTBM-NEXT:  .LBB43_8:
+; X86-BMI1NOTBM-NEXT:  .LBB53_8:
 ; X86-BMI1NOTBM-NEXT:    popl %esi
 ; X86-BMI1NOTBM-NEXT:    popl %edi
 ; X86-BMI1NOTBM-NEXT:    popl %ebx
@@ -5581,32 +6721,32 @@ define i64 @bextr64_d0(i64 %val, i64 %numskipbits, i64 %numlowbits) nounwind {
 ; X86-BMI1BMI2-NEXT:    shrxl %ecx, %edx, %esi
 ; X86-BMI1BMI2-NEXT:    xorl %edx, %edx
 ; X86-BMI1BMI2-NEXT:    testb $32, %cl
-; X86-BMI1BMI2-NEXT:    je .LBB43_2
+; X86-BMI1BMI2-NEXT:    je .LBB53_2
 ; X86-BMI1BMI2-NEXT:  # %bb.1:
 ; X86-BMI1BMI2-NEXT:    movl %esi, %eax
 ; X86-BMI1BMI2-NEXT:    xorl %esi, %esi
-; X86-BMI1BMI2-NEXT:  .LBB43_2:
+; X86-BMI1BMI2-NEXT:  .LBB53_2:
 ; X86-BMI1BMI2-NEXT:    movb $64, %cl
 ; X86-BMI1BMI2-NEXT:    subb {{[0-9]+}}(%esp), %cl
 ; X86-BMI1BMI2-NEXT:    shldl %cl, %eax, %esi
 ; X86-BMI1BMI2-NEXT:    shlxl %ecx, %eax, %edi
 ; X86-BMI1BMI2-NEXT:    testb $32, %cl
-; X86-BMI1BMI2-NEXT:    je .LBB43_4
+; X86-BMI1BMI2-NEXT:    je .LBB53_4
 ; X86-BMI1BMI2-NEXT:  # %bb.3:
 ; X86-BMI1BMI2-NEXT:    movl %edi, %esi
 ; X86-BMI1BMI2-NEXT:    movl $0, %edi
-; X86-BMI1BMI2-NEXT:  .LBB43_4:
+; X86-BMI1BMI2-NEXT:  .LBB53_4:
 ; X86-BMI1BMI2-NEXT:    shrxl %ecx, %esi, %eax
-; X86-BMI1BMI2-NEXT:    jne .LBB43_6
+; X86-BMI1BMI2-NEXT:    jne .LBB53_6
 ; X86-BMI1BMI2-NEXT:  # %bb.5:
 ; X86-BMI1BMI2-NEXT:    movl %eax, %edx
-; X86-BMI1BMI2-NEXT:  .LBB43_6:
+; X86-BMI1BMI2-NEXT:  .LBB53_6:
 ; X86-BMI1BMI2-NEXT:    shrdl %cl, %esi, %edi
 ; X86-BMI1BMI2-NEXT:    testb $32, %cl
-; X86-BMI1BMI2-NEXT:    jne .LBB43_8
+; X86-BMI1BMI2-NEXT:    jne .LBB53_8
 ; X86-BMI1BMI2-NEXT:  # %bb.7:
 ; X86-BMI1BMI2-NEXT:    movl %edi, %eax
-; X86-BMI1BMI2-NEXT:  .LBB43_8:
+; X86-BMI1BMI2-NEXT:  .LBB53_8:
 ; X86-BMI1BMI2-NEXT:    popl %esi
 ; X86-BMI1BMI2-NEXT:    popl %edi
 ; X86-BMI1BMI2-NEXT:    retl
@@ -5625,9 +6765,9 @@ define i64 @bextr64_d0(i64 %val, i64 %numskipbits, i64 %numlowbits) nounwind {
 ;
 ; X64-BMI1NOTBM-LABEL: bextr64_d0:
 ; X64-BMI1NOTBM:       # %bb.0:
-; X64-BMI1NOTBM-NEXT:    shlq $8, %rdx
+; X64-BMI1NOTBM-NEXT:    shll $8, %edx
 ; X64-BMI1NOTBM-NEXT:    movzbl %sil, %eax
-; X64-BMI1NOTBM-NEXT:    orq %rdx, %rax
+; X64-BMI1NOTBM-NEXT:    orl %edx, %eax
 ; X64-BMI1NOTBM-NEXT:    bextrq %rax, %rdi, %rax
 ; X64-BMI1NOTBM-NEXT:    retq
 ;
@@ -5657,36 +6797,36 @@ define i64 @bextr64_d1_indexzext(i64 %val, i8 %numskipbits, i8 %numlowbits) noun
 ; X86-NOBMI-NEXT:    shrdl %cl, %edx, %edi
 ; X86-NOBMI-NEXT:    xorl %esi, %esi
 ; X86-NOBMI-NEXT:    testb $32, %cl
-; X86-NOBMI-NEXT:    je .LBB44_2
+; X86-NOBMI-NEXT:    je .LBB54_2
 ; X86-NOBMI-NEXT:  # %bb.1:
 ; X86-NOBMI-NEXT:    movl %eax, %edi
 ; X86-NOBMI-NEXT:    xorl %eax, %eax
-; X86-NOBMI-NEXT:  .LBB44_2:
+; X86-NOBMI-NEXT:  .LBB54_2:
 ; X86-NOBMI-NEXT:    movb $64, %cl
 ; X86-NOBMI-NEXT:    subb {{[0-9]+}}(%esp), %cl
 ; X86-NOBMI-NEXT:    shldl %cl, %edi, %eax
 ; X86-NOBMI-NEXT:    shll %cl, %edi
 ; X86-NOBMI-NEXT:    testb $32, %cl
 ; X86-NOBMI-NEXT:    movl %edi, %ebx
-; X86-NOBMI-NEXT:    jne .LBB44_4
+; X86-NOBMI-NEXT:    jne .LBB54_4
 ; X86-NOBMI-NEXT:  # %bb.3:
 ; X86-NOBMI-NEXT:    movl %eax, %ebx
-; X86-NOBMI-NEXT:  .LBB44_4:
+; X86-NOBMI-NEXT:  .LBB54_4:
 ; X86-NOBMI-NEXT:    movl %ebx, %eax
 ; X86-NOBMI-NEXT:    shrl %cl, %eax
 ; X86-NOBMI-NEXT:    testb $32, %cl
 ; X86-NOBMI-NEXT:    movl $0, %edx
-; X86-NOBMI-NEXT:    jne .LBB44_6
+; X86-NOBMI-NEXT:    jne .LBB54_6
 ; X86-NOBMI-NEXT:  # %bb.5:
 ; X86-NOBMI-NEXT:    movl %edi, %esi
 ; X86-NOBMI-NEXT:    movl %eax, %edx
-; X86-NOBMI-NEXT:  .LBB44_6:
+; X86-NOBMI-NEXT:  .LBB54_6:
 ; X86-NOBMI-NEXT:    shrdl %cl, %ebx, %esi
 ; X86-NOBMI-NEXT:    testb $32, %cl
-; X86-NOBMI-NEXT:    jne .LBB44_8
+; X86-NOBMI-NEXT:    jne .LBB54_8
 ; X86-NOBMI-NEXT:  # %bb.7:
 ; X86-NOBMI-NEXT:    movl %esi, %eax
-; X86-NOBMI-NEXT:  .LBB44_8:
+; X86-NOBMI-NEXT:  .LBB54_8:
 ; X86-NOBMI-NEXT:    popl %esi
 ; X86-NOBMI-NEXT:    popl %edi
 ; X86-NOBMI-NEXT:    popl %ebx
@@ -5705,36 +6845,36 @@ define i64 @bextr64_d1_indexzext(i64 %val, i8 %numskipbits, i8 %numlowbits) noun
 ; X86-BMI1NOTBM-NEXT:    shrdl %cl, %edx, %edi
 ; X86-BMI1NOTBM-NEXT:    xorl %esi, %esi
 ; X86-BMI1NOTBM-NEXT:    testb $32, %cl
-; X86-BMI1NOTBM-NEXT:    je .LBB44_2
+; X86-BMI1NOTBM-NEXT:    je .LBB54_2
 ; X86-BMI1NOTBM-NEXT:  # %bb.1:
 ; X86-BMI1NOTBM-NEXT:    movl %eax, %edi
 ; X86-BMI1NOTBM-NEXT:    xorl %eax, %eax
-; X86-BMI1NOTBM-NEXT:  .LBB44_2:
+; X86-BMI1NOTBM-NEXT:  .LBB54_2:
 ; X86-BMI1NOTBM-NEXT:    movb $64, %cl
 ; X86-BMI1NOTBM-NEXT:    subb {{[0-9]+}}(%esp), %cl
 ; X86-BMI1NOTBM-NEXT:    shldl %cl, %edi, %eax
 ; X86-BMI1NOTBM-NEXT:    shll %cl, %edi
 ; X86-BMI1NOTBM-NEXT:    testb $32, %cl
 ; X86-BMI1NOTBM-NEXT:    movl %edi, %ebx
-; X86-BMI1NOTBM-NEXT:    jne .LBB44_4
+; X86-BMI1NOTBM-NEXT:    jne .LBB54_4
 ; X86-BMI1NOTBM-NEXT:  # %bb.3:
 ; X86-BMI1NOTBM-NEXT:    movl %eax, %ebx
-; X86-BMI1NOTBM-NEXT:  .LBB44_4:
+; X86-BMI1NOTBM-NEXT:  .LBB54_4:
 ; X86-BMI1NOTBM-NEXT:    movl %ebx, %eax
 ; X86-BMI1NOTBM-NEXT:    shrl %cl, %eax
 ; X86-BMI1NOTBM-NEXT:    testb $32, %cl
 ; X86-BMI1NOTBM-NEXT:    movl $0, %edx
-; X86-BMI1NOTBM-NEXT:    jne .LBB44_6
+; X86-BMI1NOTBM-NEXT:    jne .LBB54_6
 ; X86-BMI1NOTBM-NEXT:  # %bb.5:
 ; X86-BMI1NOTBM-NEXT:    movl %edi, %esi
 ; X86-BMI1NOTBM-NEXT:    movl %eax, %edx
-; X86-BMI1NOTBM-NEXT:  .LBB44_6:
+; X86-BMI1NOTBM-NEXT:  .LBB54_6:
 ; X86-BMI1NOTBM-NEXT:    shrdl %cl, %ebx, %esi
 ; X86-BMI1NOTBM-NEXT:    testb $32, %cl
-; X86-BMI1NOTBM-NEXT:    jne .LBB44_8
+; X86-BMI1NOTBM-NEXT:    jne .LBB54_8
 ; X86-BMI1NOTBM-NEXT:  # %bb.7:
 ; X86-BMI1NOTBM-NEXT:    movl %esi, %eax
-; X86-BMI1NOTBM-NEXT:  .LBB44_8:
+; X86-BMI1NOTBM-NEXT:  .LBB54_8:
 ; X86-BMI1NOTBM-NEXT:    popl %esi
 ; X86-BMI1NOTBM-NEXT:    popl %edi
 ; X86-BMI1NOTBM-NEXT:    popl %ebx
@@ -5751,32 +6891,32 @@ define i64 @bextr64_d1_indexzext(i64 %val, i8 %numskipbits, i8 %numlowbits) noun
 ; X86-BMI1BMI2-NEXT:    shrxl %ecx, %edx, %esi
 ; X86-BMI1BMI2-NEXT:    xorl %edx, %edx
 ; X86-BMI1BMI2-NEXT:    testb $32, %cl
-; X86-BMI1BMI2-NEXT:    je .LBB44_2
+; X86-BMI1BMI2-NEXT:    je .LBB54_2
 ; X86-BMI1BMI2-NEXT:  # %bb.1:
 ; X86-BMI1BMI2-NEXT:    movl %esi, %eax
 ; X86-BMI1BMI2-NEXT:    xorl %esi, %esi
-; X86-BMI1BMI2-NEXT:  .LBB44_2:
+; X86-BMI1BMI2-NEXT:  .LBB54_2:
 ; X86-BMI1BMI2-NEXT:    movb $64, %cl
 ; X86-BMI1BMI2-NEXT:    subb {{[0-9]+}}(%esp), %cl
 ; X86-BMI1BMI2-NEXT:    shldl %cl, %eax, %esi
 ; X86-BMI1BMI2-NEXT:    shlxl %ecx, %eax, %edi
 ; X86-BMI1BMI2-NEXT:    testb $32, %cl
-; X86-BMI1BMI2-NEXT:    je .LBB44_4
+; X86-BMI1BMI2-NEXT:    je .LBB54_4
 ; X86-BMI1BMI2-NEXT:  # %bb.3:
 ; X86-BMI1BMI2-NEXT:    movl %edi, %esi
 ; X86-BMI1BMI2-NEXT:    movl $0, %edi
-; X86-BMI1BMI2-NEXT:  .LBB44_4:
+; X86-BMI1BMI2-NEXT:  .LBB54_4:
 ; X86-BMI1BMI2-NEXT:    shrxl %ecx, %esi, %eax
-; X86-BMI1BMI2-NEXT:    jne .LBB44_6
+; X86-BMI1BMI2-NEXT:    jne .LBB54_6
 ; X86-BMI1BMI2-NEXT:  # %bb.5:
 ; X86-BMI1BMI2-NEXT:    movl %eax, %edx
-; X86-BMI1BMI2-NEXT:  .LBB44_6:
+; X86-BMI1BMI2-NEXT:  .LBB54_6:
 ; X86-BMI1BMI2-NEXT:    shrdl %cl, %esi, %edi
 ; X86-BMI1BMI2-NEXT:    testb $32, %cl
-; X86-BMI1BMI2-NEXT:    jne .LBB44_8
+; X86-BMI1BMI2-NEXT:    jne .LBB54_8
 ; X86-BMI1BMI2-NEXT:  # %bb.7:
 ; X86-BMI1BMI2-NEXT:    movl %edi, %eax
-; X86-BMI1BMI2-NEXT:  .LBB44_8:
+; X86-BMI1BMI2-NEXT:  .LBB54_8:
 ; X86-BMI1BMI2-NEXT:    popl %esi
 ; X86-BMI1BMI2-NEXT:    popl %edi
 ; X86-BMI1BMI2-NEXT:    retl
@@ -5795,10 +6935,9 @@ define i64 @bextr64_d1_indexzext(i64 %val, i8 %numskipbits, i8 %numlowbits) noun
 ;
 ; X64-BMI1NOTBM-LABEL: bextr64_d1_indexzext:
 ; X64-BMI1NOTBM:       # %bb.0:
-; X64-BMI1NOTBM-NEXT:    # kill: def $edx killed $edx def $rdx
-; X64-BMI1NOTBM-NEXT:    shlq $8, %rdx
+; X64-BMI1NOTBM-NEXT:    shll $8, %edx
 ; X64-BMI1NOTBM-NEXT:    movzbl %sil, %eax
-; X64-BMI1NOTBM-NEXT:    orq %rdx, %rax
+; X64-BMI1NOTBM-NEXT:    orl %edx, %eax
 ; X64-BMI1NOTBM-NEXT:    bextrq %rax, %rdi, %rax
 ; X64-BMI1NOTBM-NEXT:    retq
 ;
@@ -5833,36 +6972,36 @@ define i64 @bextr64_d2_load(i64* %w, i64 %numskipbits, i64 %numlowbits) nounwind
 ; X86-NOBMI-NEXT:    shrdl %cl, %edx, %edi
 ; X86-NOBMI-NEXT:    xorl %esi, %esi
 ; X86-NOBMI-NEXT:    testb $32, %cl
-; X86-NOBMI-NEXT:    je .LBB45_2
+; X86-NOBMI-NEXT:    je .LBB55_2
 ; X86-NOBMI-NEXT:  # %bb.1:
 ; X86-NOBMI-NEXT:    movl %eax, %edi
 ; X86-NOBMI-NEXT:    xorl %eax, %eax
-; X86-NOBMI-NEXT:  .LBB45_2:
+; X86-NOBMI-NEXT:  .LBB55_2:
 ; X86-NOBMI-NEXT:    movb $64, %cl
 ; X86-NOBMI-NEXT:    subb {{[0-9]+}}(%esp), %cl
 ; X86-NOBMI-NEXT:    shldl %cl, %edi, %eax
 ; X86-NOBMI-NEXT:    shll %cl, %edi
 ; X86-NOBMI-NEXT:    testb $32, %cl
 ; X86-NOBMI-NEXT:    movl %edi, %ebx
-; X86-NOBMI-NEXT:    jne .LBB45_4
+; X86-NOBMI-NEXT:    jne .LBB55_4
 ; X86-NOBMI-NEXT:  # %bb.3:
 ; X86-NOBMI-NEXT:    movl %eax, %ebx
-; X86-NOBMI-NEXT:  .LBB45_4:
+; X86-NOBMI-NEXT:  .LBB55_4:
 ; X86-NOBMI-NEXT:    movl %ebx, %eax
 ; X86-NOBMI-NEXT:    shrl %cl, %eax
 ; X86-NOBMI-NEXT:    testb $32, %cl
 ; X86-NOBMI-NEXT:    movl $0, %edx
-; X86-NOBMI-NEXT:    jne .LBB45_6
+; X86-NOBMI-NEXT:    jne .LBB55_6
 ; X86-NOBMI-NEXT:  # %bb.5:
 ; X86-NOBMI-NEXT:    movl %edi, %esi
 ; X86-NOBMI-NEXT:    movl %eax, %edx
-; X86-NOBMI-NEXT:  .LBB45_6:
+; X86-NOBMI-NEXT:  .LBB55_6:
 ; X86-NOBMI-NEXT:    shrdl %cl, %ebx, %esi
 ; X86-NOBMI-NEXT:    testb $32, %cl
-; X86-NOBMI-NEXT:    jne .LBB45_8
+; X86-NOBMI-NEXT:    jne .LBB55_8
 ; X86-NOBMI-NEXT:  # %bb.7:
 ; X86-NOBMI-NEXT:    movl %esi, %eax
-; X86-NOBMI-NEXT:  .LBB45_8:
+; X86-NOBMI-NEXT:  .LBB55_8:
 ; X86-NOBMI-NEXT:    popl %esi
 ; X86-NOBMI-NEXT:    popl %edi
 ; X86-NOBMI-NEXT:    popl %ebx
@@ -5882,36 +7021,36 @@ define i64 @bextr64_d2_load(i64* %w, i64 %numskipbits, i64 %numlowbits) nounwind
 ; X86-BMI1NOTBM-NEXT:    shrdl %cl, %edx, %edi
 ; X86-BMI1NOTBM-NEXT:    xorl %esi, %esi
 ; X86-BMI1NOTBM-NEXT:    testb $32, %cl
-; X86-BMI1NOTBM-NEXT:    je .LBB45_2
+; X86-BMI1NOTBM-NEXT:    je .LBB55_2
 ; X86-BMI1NOTBM-NEXT:  # %bb.1:
 ; X86-BMI1NOTBM-NEXT:    movl %eax, %edi
 ; X86-BMI1NOTBM-NEXT:    xorl %eax, %eax
-; X86-BMI1NOTBM-NEXT:  .LBB45_2:
+; X86-BMI1NOTBM-NEXT:  .LBB55_2:
 ; X86-BMI1NOTBM-NEXT:    movb $64, %cl
 ; X86-BMI1NOTBM-NEXT:    subb {{[0-9]+}}(%esp), %cl
 ; X86-BMI1NOTBM-NEXT:    shldl %cl, %edi, %eax
 ; X86-BMI1NOTBM-NEXT:    shll %cl, %edi
 ; X86-BMI1NOTBM-NEXT:    testb $32, %cl
 ; X86-BMI1NOTBM-NEXT:    movl %edi, %ebx
-; X86-BMI1NOTBM-NEXT:    jne .LBB45_4
+; X86-BMI1NOTBM-NEXT:    jne .LBB55_4
 ; X86-BMI1NOTBM-NEXT:  # %bb.3:
 ; X86-BMI1NOTBM-NEXT:    movl %eax, %ebx
-; X86-BMI1NOTBM-NEXT:  .LBB45_4:
+; X86-BMI1NOTBM-NEXT:  .LBB55_4:
 ; X86-BMI1NOTBM-NEXT:    movl %ebx, %eax
 ; X86-BMI1NOTBM-NEXT:    shrl %cl, %eax
 ; X86-BMI1NOTBM-NEXT:    testb $32, %cl
 ; X86-BMI1NOTBM-NEXT:    movl $0, %edx
-; X86-BMI1NOTBM-NEXT:    jne .LBB45_6
+; X86-BMI1NOTBM-NEXT:    jne .LBB55_6
 ; X86-BMI1NOTBM-NEXT:  # %bb.5:
 ; X86-BMI1NOTBM-NEXT:    movl %edi, %esi
 ; X86-BMI1NOTBM-NEXT:    movl %eax, %edx
-; X86-BMI1NOTBM-NEXT:  .LBB45_6:
+; X86-BMI1NOTBM-NEXT:  .LBB55_6:
 ; X86-BMI1NOTBM-NEXT:    shrdl %cl, %ebx, %esi
 ; X86-BMI1NOTBM-NEXT:    testb $32, %cl
-; X86-BMI1NOTBM-NEXT:    jne .LBB45_8
+; X86-BMI1NOTBM-NEXT:    jne .LBB55_8
 ; X86-BMI1NOTBM-NEXT:  # %bb.7:
 ; X86-BMI1NOTBM-NEXT:    movl %esi, %eax
-; X86-BMI1NOTBM-NEXT:  .LBB45_8:
+; X86-BMI1NOTBM-NEXT:  .LBB55_8:
 ; X86-BMI1NOTBM-NEXT:    popl %esi
 ; X86-BMI1NOTBM-NEXT:    popl %edi
 ; X86-BMI1NOTBM-NEXT:    popl %ebx
@@ -5929,32 +7068,32 @@ define i64 @bextr64_d2_load(i64* %w, i64 %numskipbits, i64 %numlowbits) nounwind
 ; X86-BMI1BMI2-NEXT:    shrdl %cl, %edx, %eax
 ; X86-BMI1BMI2-NEXT:    xorl %edx, %edx
 ; X86-BMI1BMI2-NEXT:    testb $32, %cl
-; X86-BMI1BMI2-NEXT:    je .LBB45_2
+; X86-BMI1BMI2-NEXT:    je .LBB55_2
 ; X86-BMI1BMI2-NEXT:  # %bb.1:
 ; X86-BMI1BMI2-NEXT:    movl %esi, %eax
 ; X86-BMI1BMI2-NEXT:    xorl %esi, %esi
-; X86-BMI1BMI2-NEXT:  .LBB45_2:
+; X86-BMI1BMI2-NEXT:  .LBB55_2:
 ; X86-BMI1BMI2-NEXT:    movb $64, %cl
 ; X86-BMI1BMI2-NEXT:    subb {{[0-9]+}}(%esp), %cl
 ; X86-BMI1BMI2-NEXT:    shldl %cl, %eax, %esi
 ; X86-BMI1BMI2-NEXT:    shlxl %ecx, %eax, %edi
 ; X86-BMI1BMI2-NEXT:    testb $32, %cl
-; X86-BMI1BMI2-NEXT:    je .LBB45_4
+; X86-BMI1BMI2-NEXT:    je .LBB55_4
 ; X86-BMI1BMI2-NEXT:  # %bb.3:
 ; X86-BMI1BMI2-NEXT:    movl %edi, %esi
 ; X86-BMI1BMI2-NEXT:    movl $0, %edi
-; X86-BMI1BMI2-NEXT:  .LBB45_4:
+; X86-BMI1BMI2-NEXT:  .LBB55_4:
 ; X86-BMI1BMI2-NEXT:    shrxl %ecx, %esi, %eax
-; X86-BMI1BMI2-NEXT:    jne .LBB45_6
+; X86-BMI1BMI2-NEXT:    jne .LBB55_6
 ; X86-BMI1BMI2-NEXT:  # %bb.5:
 ; X86-BMI1BMI2-NEXT:    movl %eax, %edx
-; X86-BMI1BMI2-NEXT:  .LBB45_6:
+; X86-BMI1BMI2-NEXT:  .LBB55_6:
 ; X86-BMI1BMI2-NEXT:    shrdl %cl, %esi, %edi
 ; X86-BMI1BMI2-NEXT:    testb $32, %cl
-; X86-BMI1BMI2-NEXT:    jne .LBB45_8
+; X86-BMI1BMI2-NEXT:    jne .LBB55_8
 ; X86-BMI1BMI2-NEXT:  # %bb.7:
 ; X86-BMI1BMI2-NEXT:    movl %edi, %eax
-; X86-BMI1BMI2-NEXT:  .LBB45_8:
+; X86-BMI1BMI2-NEXT:  .LBB55_8:
 ; X86-BMI1BMI2-NEXT:    popl %esi
 ; X86-BMI1BMI2-NEXT:    popl %edi
 ; X86-BMI1BMI2-NEXT:    retl
@@ -5973,9 +7112,9 @@ define i64 @bextr64_d2_load(i64* %w, i64 %numskipbits, i64 %numlowbits) nounwind
 ;
 ; X64-BMI1NOTBM-LABEL: bextr64_d2_load:
 ; X64-BMI1NOTBM:       # %bb.0:
-; X64-BMI1NOTBM-NEXT:    shlq $8, %rdx
+; X64-BMI1NOTBM-NEXT:    shll $8, %edx
 ; X64-BMI1NOTBM-NEXT:    movzbl %sil, %eax
-; X64-BMI1NOTBM-NEXT:    orq %rdx, %rax
+; X64-BMI1NOTBM-NEXT:    orl %edx, %eax
 ; X64-BMI1NOTBM-NEXT:    bextrq %rax, (%rdi), %rax
 ; X64-BMI1NOTBM-NEXT:    retq
 ;
@@ -6007,36 +7146,36 @@ define i64 @bextr64_d3_load_indexzext(i64* %w, i8 %numskipbits, i8 %numlowbits)
 ; X86-NOBMI-NEXT:    shrdl %cl, %edx, %edi
 ; X86-NOBMI-NEXT:    xorl %esi, %esi
 ; X86-NOBMI-NEXT:    testb $32, %cl
-; X86-NOBMI-NEXT:    je .LBB46_2
+; X86-NOBMI-NEXT:    je .LBB56_2
 ; X86-NOBMI-NEXT:  # %bb.1:
 ; X86-NOBMI-NEXT:    movl %eax, %edi
 ; X86-NOBMI-NEXT:    xorl %eax, %eax
-; X86-NOBMI-NEXT:  .LBB46_2:
+; X86-NOBMI-NEXT:  .LBB56_2:
 ; X86-NOBMI-NEXT:    movb $64, %cl
 ; X86-NOBMI-NEXT:    subb {{[0-9]+}}(%esp), %cl
 ; X86-NOBMI-NEXT:    shldl %cl, %edi, %eax
 ; X86-NOBMI-NEXT:    shll %cl, %edi
 ; X86-NOBMI-NEXT:    testb $32, %cl
 ; X86-NOBMI-NEXT:    movl %edi, %ebx
-; X86-NOBMI-NEXT:    jne .LBB46_4
+; X86-NOBMI-NEXT:    jne .LBB56_4
 ; X86-NOBMI-NEXT:  # %bb.3:
 ; X86-NOBMI-NEXT:    movl %eax, %ebx
-; X86-NOBMI-NEXT:  .LBB46_4:
+; X86-NOBMI-NEXT:  .LBB56_4:
 ; X86-NOBMI-NEXT:    movl %ebx, %eax
 ; X86-NOBMI-NEXT:    shrl %cl, %eax
 ; X86-NOBMI-NEXT:    testb $32, %cl
 ; X86-NOBMI-NEXT:    movl $0, %edx
-; X86-NOBMI-NEXT:    jne .LBB46_6
+; X86-NOBMI-NEXT:    jne .LBB56_6
 ; X86-NOBMI-NEXT:  # %bb.5:
 ; X86-NOBMI-NEXT:    movl %edi, %esi
 ; X86-NOBMI-NEXT:    movl %eax, %edx
-; X86-NOBMI-NEXT:  .LBB46_6:
+; X86-NOBMI-NEXT:  .LBB56_6:
 ; X86-NOBMI-NEXT:    shrdl %cl, %ebx, %esi
 ; X86-NOBMI-NEXT:    testb $32, %cl
-; X86-NOBMI-NEXT:    jne .LBB46_8
+; X86-NOBMI-NEXT:    jne .LBB56_8
 ; X86-NOBMI-NEXT:  # %bb.7:
 ; X86-NOBMI-NEXT:    movl %esi, %eax
-; X86-NOBMI-NEXT:  .LBB46_8:
+; X86-NOBMI-NEXT:  .LBB56_8:
 ; X86-NOBMI-NEXT:    popl %esi
 ; X86-NOBMI-NEXT:    popl %edi
 ; X86-NOBMI-NEXT:    popl %ebx
@@ -6056,36 +7195,36 @@ define i64 @bextr64_d3_load_indexzext(i64* %w, i8 %numskipbits, i8 %numlowbits)
 ; X86-BMI1NOTBM-NEXT:    shrdl %cl, %edx, %edi
 ; X86-BMI1NOTBM-NEXT:    xorl %esi, %esi
 ; X86-BMI1NOTBM-NEXT:    testb $32, %cl
-; X86-BMI1NOTBM-NEXT:    je .LBB46_2
+; X86-BMI1NOTBM-NEXT:    je .LBB56_2
 ; X86-BMI1NOTBM-NEXT:  # %bb.1:
 ; X86-BMI1NOTBM-NEXT:    movl %eax, %edi
 ; X86-BMI1NOTBM-NEXT:    xorl %eax, %eax
-; X86-BMI1NOTBM-NEXT:  .LBB46_2:
+; X86-BMI1NOTBM-NEXT:  .LBB56_2:
 ; X86-BMI1NOTBM-NEXT:    movb $64, %cl
 ; X86-BMI1NOTBM-NEXT:    subb {{[0-9]+}}(%esp), %cl
 ; X86-BMI1NOTBM-NEXT:    shldl %cl, %edi, %eax
 ; X86-BMI1NOTBM-NEXT:    shll %cl, %edi
 ; X86-BMI1NOTBM-NEXT:    testb $32, %cl
 ; X86-BMI1NOTBM-NEXT:    movl %edi, %ebx
-; X86-BMI1NOTBM-NEXT:    jne .LBB46_4
+; X86-BMI1NOTBM-NEXT:    jne .LBB56_4
 ; X86-BMI1NOTBM-NEXT:  # %bb.3:
 ; X86-BMI1NOTBM-NEXT:    movl %eax, %ebx
-; X86-BMI1NOTBM-NEXT:  .LBB46_4:
+; X86-BMI1NOTBM-NEXT:  .LBB56_4:
 ; X86-BMI1NOTBM-NEXT:    movl %ebx, %eax
 ; X86-BMI1NOTBM-NEXT:    shrl %cl, %eax
 ; X86-BMI1NOTBM-NEXT:    testb $32, %cl
 ; X86-BMI1NOTBM-NEXT:    movl $0, %edx
-; X86-BMI1NOTBM-NEXT:    jne .LBB46_6
+; X86-BMI1NOTBM-NEXT:    jne .LBB56_6
 ; X86-BMI1NOTBM-NEXT:  # %bb.5:
 ; X86-BMI1NOTBM-NEXT:    movl %edi, %esi
 ; X86-BMI1NOTBM-NEXT:    movl %eax, %edx
-; X86-BMI1NOTBM-NEXT:  .LBB46_6:
+; X86-BMI1NOTBM-NEXT:  .LBB56_6:
 ; X86-BMI1NOTBM-NEXT:    shrdl %cl, %ebx, %esi
 ; X86-BMI1NOTBM-NEXT:    testb $32, %cl
-; X86-BMI1NOTBM-NEXT:    jne .LBB46_8
+; X86-BMI1NOTBM-NEXT:    jne .LBB56_8
 ; X86-BMI1NOTBM-NEXT:  # %bb.7:
 ; X86-BMI1NOTBM-NEXT:    movl %esi, %eax
-; X86-BMI1NOTBM-NEXT:  .LBB46_8:
+; X86-BMI1NOTBM-NEXT:  .LBB56_8:
 ; X86-BMI1NOTBM-NEXT:    popl %esi
 ; X86-BMI1NOTBM-NEXT:    popl %edi
 ; X86-BMI1NOTBM-NEXT:    popl %ebx
@@ -6103,32 +7242,32 @@ define i64 @bextr64_d3_load_indexzext(i64* %w, i8 %numskipbits, i8 %numlowbits)
 ; X86-BMI1BMI2-NEXT:    shrdl %cl, %edx, %eax
 ; X86-BMI1BMI2-NEXT:    xorl %edx, %edx
 ; X86-BMI1BMI2-NEXT:    testb $32, %cl
-; X86-BMI1BMI2-NEXT:    je .LBB46_2
+; X86-BMI1BMI2-NEXT:    je .LBB56_2
 ; X86-BMI1BMI2-NEXT:  # %bb.1:
 ; X86-BMI1BMI2-NEXT:    movl %esi, %eax
 ; X86-BMI1BMI2-NEXT:    xorl %esi, %esi
-; X86-BMI1BMI2-NEXT:  .LBB46_2:
+; X86-BMI1BMI2-NEXT:  .LBB56_2:
 ; X86-BMI1BMI2-NEXT:    movb $64, %cl
 ; X86-BMI1BMI2-NEXT:    subb {{[0-9]+}}(%esp), %cl
 ; X86-BMI1BMI2-NEXT:    shldl %cl, %eax, %esi
 ; X86-BMI1BMI2-NEXT:    shlxl %ecx, %eax, %edi
 ; X86-BMI1BMI2-NEXT:    testb $32, %cl
-; X86-BMI1BMI2-NEXT:    je .LBB46_4
+; X86-BMI1BMI2-NEXT:    je .LBB56_4
 ; X86-BMI1BMI2-NEXT:  # %bb.3:
 ; X86-BMI1BMI2-NEXT:    movl %edi, %esi
 ; X86-BMI1BMI2-NEXT:    movl $0, %edi
-; X86-BMI1BMI2-NEXT:  .LBB46_4:
+; X86-BMI1BMI2-NEXT:  .LBB56_4:
 ; X86-BMI1BMI2-NEXT:    shrxl %ecx, %esi, %eax
-; X86-BMI1BMI2-NEXT:    jne .LBB46_6
+; X86-BMI1BMI2-NEXT:    jne .LBB56_6
 ; X86-BMI1BMI2-NEXT:  # %bb.5:
 ; X86-BMI1BMI2-NEXT:    movl %eax, %edx
-; X86-BMI1BMI2-NEXT:  .LBB46_6:
+; X86-BMI1BMI2-NEXT:  .LBB56_6:
 ; X86-BMI1BMI2-NEXT:    shrdl %cl, %esi, %edi
 ; X86-BMI1BMI2-NEXT:    testb $32, %cl
-; X86-BMI1BMI2-NEXT:    jne .LBB46_8
+; X86-BMI1BMI2-NEXT:    jne .LBB56_8
 ; X86-BMI1BMI2-NEXT:  # %bb.7:
 ; X86-BMI1BMI2-NEXT:    movl %edi, %eax
-; X86-BMI1BMI2-NEXT:  .LBB46_8:
+; X86-BMI1BMI2-NEXT:  .LBB56_8:
 ; X86-BMI1BMI2-NEXT:    popl %esi
 ; X86-BMI1BMI2-NEXT:    popl %edi
 ; X86-BMI1BMI2-NEXT:    retl
@@ -6147,10 +7286,9 @@ define i64 @bextr64_d3_load_indexzext(i64* %w, i8 %numskipbits, i8 %numlowbits)
 ;
 ; X64-BMI1NOTBM-LABEL: bextr64_d3_load_indexzext:
 ; X64-BMI1NOTBM:       # %bb.0:
-; X64-BMI1NOTBM-NEXT:    # kill: def $edx killed $edx def $rdx
-; X64-BMI1NOTBM-NEXT:    shlq $8, %rdx
+; X64-BMI1NOTBM-NEXT:    shll $8, %edx
 ; X64-BMI1NOTBM-NEXT:    movzbl %sil, %eax
-; X64-BMI1NOTBM-NEXT:    orq %rdx, %rax
+; X64-BMI1NOTBM-NEXT:    orl %edx, %eax
 ; X64-BMI1NOTBM-NEXT:    bextrq %rax, (%rdi), %rax
 ; X64-BMI1NOTBM-NEXT:    retq
 ;
@@ -6188,37 +7326,37 @@ define i64 @bextr64_d5_skipextrauses(i64 %val, i64 %numskipbits, i64 %numlowbits
 ; X86-NOBMI-NEXT:    shrdl %cl, %edx, %ebx
 ; X86-NOBMI-NEXT:    xorl %edx, %edx
 ; X86-NOBMI-NEXT:    testb $32, %al
-; X86-NOBMI-NEXT:    je .LBB47_2
+; X86-NOBMI-NEXT:    je .LBB57_2
 ; X86-NOBMI-NEXT:  # %bb.1:
 ; X86-NOBMI-NEXT:    movl %esi, %ebx
 ; X86-NOBMI-NEXT:    xorl %esi, %esi
-; X86-NOBMI-NEXT:  .LBB47_2:
+; X86-NOBMI-NEXT:  .LBB57_2:
 ; X86-NOBMI-NEXT:    movb $64, %cl
 ; X86-NOBMI-NEXT:    subb {{[0-9]+}}(%esp), %cl
 ; X86-NOBMI-NEXT:    shldl %cl, %ebx, %esi
 ; X86-NOBMI-NEXT:    shll %cl, %ebx
 ; X86-NOBMI-NEXT:    testb $32, %cl
 ; X86-NOBMI-NEXT:    movl %ebx, %ebp
-; X86-NOBMI-NEXT:    jne .LBB47_4
+; X86-NOBMI-NEXT:    jne .LBB57_4
 ; X86-NOBMI-NEXT:  # %bb.3:
 ; X86-NOBMI-NEXT:    movl %esi, %ebp
-; X86-NOBMI-NEXT:  .LBB47_4:
+; X86-NOBMI-NEXT:  .LBB57_4:
 ; X86-NOBMI-NEXT:    movl %ebp, %esi
 ; X86-NOBMI-NEXT:    shrl %cl, %esi
 ; X86-NOBMI-NEXT:    testb $32, %cl
 ; X86-NOBMI-NEXT:    movl $0, %edi
-; X86-NOBMI-NEXT:    jne .LBB47_6
+; X86-NOBMI-NEXT:    jne .LBB57_6
 ; X86-NOBMI-NEXT:  # %bb.5:
 ; X86-NOBMI-NEXT:    movl %ebx, %edx
 ; X86-NOBMI-NEXT:    movl %esi, %edi
-; X86-NOBMI-NEXT:  .LBB47_6:
+; X86-NOBMI-NEXT:  .LBB57_6:
 ; X86-NOBMI-NEXT:    shrdl %cl, %ebp, %edx
 ; X86-NOBMI-NEXT:    testb $32, %cl
 ; X86-NOBMI-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-NOBMI-NEXT:    jne .LBB47_8
+; X86-NOBMI-NEXT:    jne .LBB57_8
 ; X86-NOBMI-NEXT:  # %bb.7:
 ; X86-NOBMI-NEXT:    movl %edx, %esi
-; X86-NOBMI-NEXT:  .LBB47_8:
+; X86-NOBMI-NEXT:  .LBB57_8:
 ; X86-NOBMI-NEXT:    subl $8, %esp
 ; X86-NOBMI-NEXT:    pushl %ecx
 ; X86-NOBMI-NEXT:    pushl %eax
@@ -6249,37 +7387,37 @@ define i64 @bextr64_d5_skipextrauses(i64 %val, i64 %numskipbits, i64 %numlowbits
 ; X86-BMI1NOTBM-NEXT:    shrdl %cl, %edx, %ebx
 ; X86-BMI1NOTBM-NEXT:    xorl %edx, %edx
 ; X86-BMI1NOTBM-NEXT:    testb $32, %al
-; X86-BMI1NOTBM-NEXT:    je .LBB47_2
+; X86-BMI1NOTBM-NEXT:    je .LBB57_2
 ; X86-BMI1NOTBM-NEXT:  # %bb.1:
 ; X86-BMI1NOTBM-NEXT:    movl %esi, %ebx
 ; X86-BMI1NOTBM-NEXT:    xorl %esi, %esi
-; X86-BMI1NOTBM-NEXT:  .LBB47_2:
+; X86-BMI1NOTBM-NEXT:  .LBB57_2:
 ; X86-BMI1NOTBM-NEXT:    movb $64, %cl
 ; X86-BMI1NOTBM-NEXT:    subb {{[0-9]+}}(%esp), %cl
 ; X86-BMI1NOTBM-NEXT:    shldl %cl, %ebx, %esi
 ; X86-BMI1NOTBM-NEXT:    shll %cl, %ebx
 ; X86-BMI1NOTBM-NEXT:    testb $32, %cl
 ; X86-BMI1NOTBM-NEXT:    movl %ebx, %ebp
-; X86-BMI1NOTBM-NEXT:    jne .LBB47_4
+; X86-BMI1NOTBM-NEXT:    jne .LBB57_4
 ; X86-BMI1NOTBM-NEXT:  # %bb.3:
 ; X86-BMI1NOTBM-NEXT:    movl %esi, %ebp
-; X86-BMI1NOTBM-NEXT:  .LBB47_4:
+; X86-BMI1NOTBM-NEXT:  .LBB57_4:
 ; X86-BMI1NOTBM-NEXT:    movl %ebp, %esi
 ; X86-BMI1NOTBM-NEXT:    shrl %cl, %esi
 ; X86-BMI1NOTBM-NEXT:    testb $32, %cl
 ; X86-BMI1NOTBM-NEXT:    movl $0, %edi
-; X86-BMI1NOTBM-NEXT:    jne .LBB47_6
+; X86-BMI1NOTBM-NEXT:    jne .LBB57_6
 ; X86-BMI1NOTBM-NEXT:  # %bb.5:
 ; X86-BMI1NOTBM-NEXT:    movl %ebx, %edx
 ; X86-BMI1NOTBM-NEXT:    movl %esi, %edi
-; X86-BMI1NOTBM-NEXT:  .LBB47_6:
+; X86-BMI1NOTBM-NEXT:  .LBB57_6:
 ; X86-BMI1NOTBM-NEXT:    shrdl %cl, %ebp, %edx
 ; X86-BMI1NOTBM-NEXT:    testb $32, %cl
 ; X86-BMI1NOTBM-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-BMI1NOTBM-NEXT:    jne .LBB47_8
+; X86-BMI1NOTBM-NEXT:    jne .LBB57_8
 ; X86-BMI1NOTBM-NEXT:  # %bb.7:
 ; X86-BMI1NOTBM-NEXT:    movl %edx, %esi
-; X86-BMI1NOTBM-NEXT:  .LBB47_8:
+; X86-BMI1NOTBM-NEXT:  .LBB57_8:
 ; X86-BMI1NOTBM-NEXT:    subl $8, %esp
 ; X86-BMI1NOTBM-NEXT:    pushl %ecx
 ; X86-BMI1NOTBM-NEXT:    pushl %eax
@@ -6307,33 +7445,33 @@ define i64 @bextr64_d5_skipextrauses(i64 %val, i64 %numskipbits, i64 %numlowbits
 ; X86-BMI1BMI2-NEXT:    shrxl %eax, %edx, %edx
 ; X86-BMI1BMI2-NEXT:    xorl %esi, %esi
 ; X86-BMI1BMI2-NEXT:    testb $32, %al
-; X86-BMI1BMI2-NEXT:    je .LBB47_2
+; X86-BMI1BMI2-NEXT:    je .LBB57_2
 ; X86-BMI1BMI2-NEXT:  # %bb.1:
 ; X86-BMI1BMI2-NEXT:    movl %edx, %edi
 ; X86-BMI1BMI2-NEXT:    xorl %edx, %edx
-; X86-BMI1BMI2-NEXT:  .LBB47_2:
+; X86-BMI1BMI2-NEXT:  .LBB57_2:
 ; X86-BMI1BMI2-NEXT:    movb $64, %cl
 ; X86-BMI1BMI2-NEXT:    subb {{[0-9]+}}(%esp), %cl
 ; X86-BMI1BMI2-NEXT:    shldl %cl, %edi, %edx
 ; X86-BMI1BMI2-NEXT:    shlxl %ecx, %edi, %ebx
 ; X86-BMI1BMI2-NEXT:    testb $32, %cl
-; X86-BMI1BMI2-NEXT:    je .LBB47_4
+; X86-BMI1BMI2-NEXT:    je .LBB57_4
 ; X86-BMI1BMI2-NEXT:  # %bb.3:
 ; X86-BMI1BMI2-NEXT:    movl %ebx, %edx
 ; X86-BMI1BMI2-NEXT:    movl $0, %ebx
-; X86-BMI1BMI2-NEXT:  .LBB47_4:
+; X86-BMI1BMI2-NEXT:  .LBB57_4:
 ; X86-BMI1BMI2-NEXT:    shrxl %ecx, %edx, %edi
-; X86-BMI1BMI2-NEXT:    jne .LBB47_6
+; X86-BMI1BMI2-NEXT:    jne .LBB57_6
 ; X86-BMI1BMI2-NEXT:  # %bb.5:
 ; X86-BMI1BMI2-NEXT:    movl %edi, %esi
-; X86-BMI1BMI2-NEXT:  .LBB47_6:
+; X86-BMI1BMI2-NEXT:  .LBB57_6:
 ; X86-BMI1BMI2-NEXT:    shrdl %cl, %edx, %ebx
 ; X86-BMI1BMI2-NEXT:    testb $32, %cl
 ; X86-BMI1BMI2-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X86-BMI1BMI2-NEXT:    jne .LBB47_8
+; X86-BMI1BMI2-NEXT:    jne .LBB57_8
 ; X86-BMI1BMI2-NEXT:  # %bb.7:
 ; X86-BMI1BMI2-NEXT:    movl %ebx, %edi
-; X86-BMI1BMI2-NEXT:  .LBB47_8:
+; X86-BMI1BMI2-NEXT:  .LBB57_8:
 ; X86-BMI1BMI2-NEXT:    subl $8, %esp
 ; X86-BMI1BMI2-NEXT:    pushl %ecx
 ; X86-BMI1BMI2-NEXT:    pushl %eax
@@ -6365,9 +7503,9 @@ define i64 @bextr64_d5_skipextrauses(i64 %val, i64 %numskipbits, i64 %numlowbits
 ; X64-BMI1NOTBM-LABEL: bextr64_d5_skipextrauses:
 ; X64-BMI1NOTBM:       # %bb.0:
 ; X64-BMI1NOTBM-NEXT:    pushq %rbx
-; X64-BMI1NOTBM-NEXT:    shlq $8, %rdx
+; X64-BMI1NOTBM-NEXT:    shll $8, %edx
 ; X64-BMI1NOTBM-NEXT:    movzbl %sil, %eax
-; X64-BMI1NOTBM-NEXT:    orq %rdx, %rax
+; X64-BMI1NOTBM-NEXT:    orl %edx, %eax
 ; X64-BMI1NOTBM-NEXT:    bextrq %rax, %rdi, %rbx
 ; X64-BMI1NOTBM-NEXT:    movq %rsi, %rdi
 ; X64-BMI1NOTBM-NEXT:    callq use64
@@ -6393,12 +7531,251 @@ define i64 @bextr64_d5_skipextrauses(i64 %val, i64 %numskipbits, i64 %numlowbits
   ret i64 %masked
 }
 
+; 64-bit, but with 32-bit output
+
+; Everything done in 64-bit, truncation happens last.
+define i32 @bextr64_32_d0(i64 %val, i64 %numskipbits, i64 %numlowbits) nounwind {
+; X86-NOBMI-LABEL: bextr64_32_d0:
+; X86-NOBMI:       # %bb.0:
+; X86-NOBMI-NEXT:    pushl %esi
+; X86-NOBMI-NEXT:    movb {{[0-9]+}}(%esp), %cl
+; X86-NOBMI-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NOBMI-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NOBMI-NEXT:    movl %esi, %eax
+; X86-NOBMI-NEXT:    shrl %cl, %eax
+; X86-NOBMI-NEXT:    shrdl %cl, %esi, %edx
+; X86-NOBMI-NEXT:    testb $32, %cl
+; X86-NOBMI-NEXT:    je .LBB58_2
+; X86-NOBMI-NEXT:  # %bb.1:
+; X86-NOBMI-NEXT:    movl %eax, %edx
+; X86-NOBMI-NEXT:    xorl %eax, %eax
+; X86-NOBMI-NEXT:  .LBB58_2:
+; X86-NOBMI-NEXT:    movb $64, %cl
+; X86-NOBMI-NEXT:    subb {{[0-9]+}}(%esp), %cl
+; X86-NOBMI-NEXT:    shldl %cl, %edx, %eax
+; X86-NOBMI-NEXT:    shll %cl, %edx
+; X86-NOBMI-NEXT:    testb $32, %cl
+; X86-NOBMI-NEXT:    je .LBB58_4
+; X86-NOBMI-NEXT:  # %bb.3:
+; X86-NOBMI-NEXT:    movl %edx, %eax
+; X86-NOBMI-NEXT:    xorl %edx, %edx
+; X86-NOBMI-NEXT:  .LBB58_4:
+; X86-NOBMI-NEXT:    shrdl %cl, %eax, %edx
+; X86-NOBMI-NEXT:    shrl %cl, %eax
+; X86-NOBMI-NEXT:    testb $32, %cl
+; X86-NOBMI-NEXT:    jne .LBB58_6
+; X86-NOBMI-NEXT:  # %bb.5:
+; X86-NOBMI-NEXT:    movl %edx, %eax
+; X86-NOBMI-NEXT:  .LBB58_6:
+; X86-NOBMI-NEXT:    popl %esi
+; X86-NOBMI-NEXT:    retl
+;
+; X86-BMI1NOTBM-LABEL: bextr64_32_d0:
+; X86-BMI1NOTBM:       # %bb.0:
+; X86-BMI1NOTBM-NEXT:    pushl %esi
+; X86-BMI1NOTBM-NEXT:    movb {{[0-9]+}}(%esp), %cl
+; X86-BMI1NOTBM-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-BMI1NOTBM-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-BMI1NOTBM-NEXT:    movl %esi, %eax
+; X86-BMI1NOTBM-NEXT:    shrl %cl, %eax
+; X86-BMI1NOTBM-NEXT:    shrdl %cl, %esi, %edx
+; X86-BMI1NOTBM-NEXT:    testb $32, %cl
+; X86-BMI1NOTBM-NEXT:    je .LBB58_2
+; X86-BMI1NOTBM-NEXT:  # %bb.1:
+; X86-BMI1NOTBM-NEXT:    movl %eax, %edx
+; X86-BMI1NOTBM-NEXT:    xorl %eax, %eax
+; X86-BMI1NOTBM-NEXT:  .LBB58_2:
+; X86-BMI1NOTBM-NEXT:    movb $64, %cl
+; X86-BMI1NOTBM-NEXT:    subb {{[0-9]+}}(%esp), %cl
+; X86-BMI1NOTBM-NEXT:    shldl %cl, %edx, %eax
+; X86-BMI1NOTBM-NEXT:    shll %cl, %edx
+; X86-BMI1NOTBM-NEXT:    testb $32, %cl
+; X86-BMI1NOTBM-NEXT:    je .LBB58_4
+; X86-BMI1NOTBM-NEXT:  # %bb.3:
+; X86-BMI1NOTBM-NEXT:    movl %edx, %eax
+; X86-BMI1NOTBM-NEXT:    xorl %edx, %edx
+; X86-BMI1NOTBM-NEXT:  .LBB58_4:
+; X86-BMI1NOTBM-NEXT:    shrdl %cl, %eax, %edx
+; X86-BMI1NOTBM-NEXT:    shrl %cl, %eax
+; X86-BMI1NOTBM-NEXT:    testb $32, %cl
+; X86-BMI1NOTBM-NEXT:    jne .LBB58_6
+; X86-BMI1NOTBM-NEXT:  # %bb.5:
+; X86-BMI1NOTBM-NEXT:    movl %edx, %eax
+; X86-BMI1NOTBM-NEXT:  .LBB58_6:
+; X86-BMI1NOTBM-NEXT:    popl %esi
+; X86-BMI1NOTBM-NEXT:    retl
+;
+; X86-BMI1BMI2-LABEL: bextr64_32_d0:
+; X86-BMI1BMI2:       # %bb.0:
+; X86-BMI1BMI2-NEXT:    movb {{[0-9]+}}(%esp), %cl
+; X86-BMI1BMI2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-BMI1BMI2-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-BMI1BMI2-NEXT:    shrdl %cl, %edx, %eax
+; X86-BMI1BMI2-NEXT:    shrxl %ecx, %edx, %edx
+; X86-BMI1BMI2-NEXT:    testb $32, %cl
+; X86-BMI1BMI2-NEXT:    je .LBB58_2
+; X86-BMI1BMI2-NEXT:  # %bb.1:
+; X86-BMI1BMI2-NEXT:    movl %edx, %eax
+; X86-BMI1BMI2-NEXT:    xorl %edx, %edx
+; X86-BMI1BMI2-NEXT:  .LBB58_2:
+; X86-BMI1BMI2-NEXT:    movb $64, %cl
+; X86-BMI1BMI2-NEXT:    subb {{[0-9]+}}(%esp), %cl
+; X86-BMI1BMI2-NEXT:    shldl %cl, %eax, %edx
+; X86-BMI1BMI2-NEXT:    shlxl %ecx, %eax, %eax
+; X86-BMI1BMI2-NEXT:    testb $32, %cl
+; X86-BMI1BMI2-NEXT:    je .LBB58_4
+; X86-BMI1BMI2-NEXT:  # %bb.3:
+; X86-BMI1BMI2-NEXT:    movl %eax, %edx
+; X86-BMI1BMI2-NEXT:    xorl %eax, %eax
+; X86-BMI1BMI2-NEXT:  .LBB58_4:
+; X86-BMI1BMI2-NEXT:    shrdl %cl, %edx, %eax
+; X86-BMI1BMI2-NEXT:    testb $32, %cl
+; X86-BMI1BMI2-NEXT:    je .LBB58_6
+; X86-BMI1BMI2-NEXT:  # %bb.5:
+; X86-BMI1BMI2-NEXT:    shrxl %ecx, %edx, %eax
+; X86-BMI1BMI2-NEXT:  .LBB58_6:
+; X86-BMI1BMI2-NEXT:    retl
+;
+; X64-NOBMI-LABEL: bextr64_32_d0:
+; X64-NOBMI:       # %bb.0:
+; X64-NOBMI-NEXT:    movq %rsi, %rcx
+; X64-NOBMI-NEXT:    movq %rdi, %rax
+; X64-NOBMI-NEXT:    # kill: def $cl killed $cl killed $rcx
+; X64-NOBMI-NEXT:    shrq %cl, %rax
+; X64-NOBMI-NEXT:    negb %dl
+; X64-NOBMI-NEXT:    movl %edx, %ecx
+; X64-NOBMI-NEXT:    shlq %cl, %rax
+; X64-NOBMI-NEXT:    shrq %cl, %rax
+; X64-NOBMI-NEXT:    # kill: def $eax killed $eax killed $rax
+; X64-NOBMI-NEXT:    retq
+;
+; X64-BMI1NOTBM-LABEL: bextr64_32_d0:
+; X64-BMI1NOTBM:       # %bb.0:
+; X64-BMI1NOTBM-NEXT:    shll $8, %edx
+; X64-BMI1NOTBM-NEXT:    movzbl %sil, %eax
+; X64-BMI1NOTBM-NEXT:    orl %edx, %eax
+; X64-BMI1NOTBM-NEXT:    bextrq %rax, %rdi, %rax
+; X64-BMI1NOTBM-NEXT:    # kill: def $eax killed $eax killed $rax
+; X64-BMI1NOTBM-NEXT:    retq
+;
+; X64-BMI1BMI2-LABEL: bextr64_32_d0:
+; X64-BMI1BMI2:       # %bb.0:
+; X64-BMI1BMI2-NEXT:    shrxq %rsi, %rdi, %rax
+; X64-BMI1BMI2-NEXT:    bzhiq %rdx, %rax, %rax
+; X64-BMI1BMI2-NEXT:    # kill: def $eax killed $eax killed $rax
+; X64-BMI1BMI2-NEXT:    retq
+  %shifted = lshr i64 %val, %numskipbits
+  %numhighbits = sub i64 64, %numlowbits
+  %highbitscleared = shl i64 %shifted, %numhighbits
+  %masked = lshr i64 %highbitscleared, %numhighbits
+  %res = trunc i64 %masked to i32
+  ret i32 %res
+}
+
+; Shifting happens in 64-bit, then truncation. Masking is 32-bit.
+define i32 @bextr64_32_d1(i64 %val, i64 %numskipbits, i32 %numlowbits) nounwind {
+; X86-NOBMI-LABEL: bextr64_32_d1:
+; X86-NOBMI:       # %bb.0:
+; X86-NOBMI-NEXT:    pushl %esi
+; X86-NOBMI-NEXT:    movb {{[0-9]+}}(%esp), %cl
+; X86-NOBMI-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NOBMI-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NOBMI-NEXT:    movl %esi, %eax
+; X86-NOBMI-NEXT:    shrl %cl, %eax
+; X86-NOBMI-NEXT:    shrdl %cl, %esi, %edx
+; X86-NOBMI-NEXT:    testb $32, %cl
+; X86-NOBMI-NEXT:    jne .LBB59_2
+; X86-NOBMI-NEXT:  # %bb.1:
+; X86-NOBMI-NEXT:    movl %edx, %eax
+; X86-NOBMI-NEXT:  .LBB59_2:
+; X86-NOBMI-NEXT:    xorl %ecx, %ecx
+; X86-NOBMI-NEXT:    subb {{[0-9]+}}(%esp), %cl
+; X86-NOBMI-NEXT:    shll %cl, %eax
+; X86-NOBMI-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X86-NOBMI-NEXT:    shrl %cl, %eax
+; X86-NOBMI-NEXT:    popl %esi
+; X86-NOBMI-NEXT:    retl
+;
+; X86-BMI1NOTBM-LABEL: bextr64_32_d1:
+; X86-BMI1NOTBM:       # %bb.0:
+; X86-BMI1NOTBM-NEXT:    pushl %edi
+; X86-BMI1NOTBM-NEXT:    pushl %esi
+; X86-BMI1NOTBM-NEXT:    movb {{[0-9]+}}(%esp), %al
+; X86-BMI1NOTBM-NEXT:    movb {{[0-9]+}}(%esp), %cl
+; X86-BMI1NOTBM-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-BMI1NOTBM-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-BMI1NOTBM-NEXT:    movl %edi, %edx
+; X86-BMI1NOTBM-NEXT:    shrl %cl, %edx
+; X86-BMI1NOTBM-NEXT:    shrdl %cl, %edi, %esi
+; X86-BMI1NOTBM-NEXT:    testb $32, %cl
+; X86-BMI1NOTBM-NEXT:    jne .LBB59_2
+; X86-BMI1NOTBM-NEXT:  # %bb.1:
+; X86-BMI1NOTBM-NEXT:    movl %esi, %edx
+; X86-BMI1NOTBM-NEXT:  .LBB59_2:
+; X86-BMI1NOTBM-NEXT:    shll $8, %eax
+; X86-BMI1NOTBM-NEXT:    bextrl %eax, %edx, %eax
+; X86-BMI1NOTBM-NEXT:    popl %esi
+; X86-BMI1NOTBM-NEXT:    popl %edi
+; X86-BMI1NOTBM-NEXT:    retl
+;
+; X86-BMI1BMI2-LABEL: bextr64_32_d1:
+; X86-BMI1BMI2:       # %bb.0:
+; X86-BMI1BMI2-NEXT:    pushl %esi
+; X86-BMI1BMI2-NEXT:    movb {{[0-9]+}}(%esp), %al
+; X86-BMI1BMI2-NEXT:    movb {{[0-9]+}}(%esp), %cl
+; X86-BMI1BMI2-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-BMI1BMI2-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-BMI1BMI2-NEXT:    shrdl %cl, %esi, %edx
+; X86-BMI1BMI2-NEXT:    testb $32, %cl
+; X86-BMI1BMI2-NEXT:    je .LBB59_2
+; X86-BMI1BMI2-NEXT:  # %bb.1:
+; X86-BMI1BMI2-NEXT:    shrxl %ecx, %esi, %edx
+; X86-BMI1BMI2-NEXT:  .LBB59_2:
+; X86-BMI1BMI2-NEXT:    bzhil %eax, %edx, %eax
+; X86-BMI1BMI2-NEXT:    popl %esi
+; X86-BMI1BMI2-NEXT:    retl
+;
+; X64-NOBMI-LABEL: bextr64_32_d1:
+; X64-NOBMI:       # %bb.0:
+; X64-NOBMI-NEXT:    movq %rsi, %rcx
+; X64-NOBMI-NEXT:    movq %rdi, %rax
+; X64-NOBMI-NEXT:    # kill: def $cl killed $cl killed $rcx
+; X64-NOBMI-NEXT:    shrq %cl, %rax
+; X64-NOBMI-NEXT:    negb %dl
+; X64-NOBMI-NEXT:    movl %edx, %ecx
+; X64-NOBMI-NEXT:    shll %cl, %eax
+; X64-NOBMI-NEXT:    shrl %cl, %eax
+; X64-NOBMI-NEXT:    # kill: def $eax killed $eax killed $rax
+; X64-NOBMI-NEXT:    retq
+;
+; X64-BMI1NOTBM-LABEL: bextr64_32_d1:
+; X64-BMI1NOTBM:       # %bb.0:
+; X64-BMI1NOTBM-NEXT:    shll $8, %edx
+; X64-BMI1NOTBM-NEXT:    movzbl %sil, %eax
+; X64-BMI1NOTBM-NEXT:    orl %edx, %eax
+; X64-BMI1NOTBM-NEXT:    bextrq %rax, %rdi, %rax
+; X64-BMI1NOTBM-NEXT:    # kill: def $eax killed $eax killed $rax
+; X64-BMI1NOTBM-NEXT:    retq
+;
+; X64-BMI1BMI2-LABEL: bextr64_32_d1:
+; X64-BMI1BMI2:       # %bb.0:
+; X64-BMI1BMI2-NEXT:    shrxq %rsi, %rdi, %rax
+; X64-BMI1BMI2-NEXT:    bzhil %edx, %eax, %eax
+; X64-BMI1BMI2-NEXT:    retq
+  %shifted = lshr i64 %val, %numskipbits
+  %truncshifted = trunc i64 %shifted to i32
+  %numhighbits = sub i32 32, %numlowbits
+  %highbitscleared = shl i32 %truncshifted, %numhighbits
+  %masked = lshr i32 %highbitscleared, %numhighbits
+  ret i32 %masked
+}
+
 ; ---------------------------------------------------------------------------- ;
 ; Constant
 ; ---------------------------------------------------------------------------- ;
 
 ; https://bugs.llvm.org/show_bug.cgi?id=38938
-define void @pr38938(i32* %a0, i64* %a1) {
+define void @pr38938(i32* %a0, i64* %a1) nounwind {
 ; X86-NOBMI-LABEL: pr38938:
 ; X86-NOBMI:       # %bb.0:
 ; X86-NOBMI-NEXT:    movl {{[0-9]+}}(%esp), %eax
@@ -6473,7 +7850,7 @@ define void @pr38938(i32* %a0, i64* %a1) {
 }
 
 ; The most canonical variant
-define i32 @c0_i32(i32 %arg) {
+define i32 @c0_i32(i32 %arg) nounwind {
 ; X86-NOBMI-LABEL: c0_i32:
 ; X86-NOBMI:       # %bb.0:
 ; X86-NOBMI-NEXT:    movl {{[0-9]+}}(%esp), %eax
@@ -6527,7 +7904,7 @@ define i32 @c0_i32(i32 %arg) {
 }
 
 ; Should be still fine, but the mask is shifted
-define i32 @c1_i32(i32 %arg) {
+define i32 @c1_i32(i32 %arg) nounwind {
 ; X86-LABEL: c1_i32:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
@@ -6547,7 +7924,7 @@ define i32 @c1_i32(i32 %arg) {
 }
 
 ; Should be still fine, but the result is shifted left afterwards
-define i32 @c2_i32(i32 %arg) {
+define i32 @c2_i32(i32 %arg) nounwind {
 ; X86-LABEL: c2_i32:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
@@ -6568,7 +7945,7 @@ define i32 @c2_i32(i32 %arg) {
 }
 
 ; The mask covers newly shifted-in bit
-define i32 @c4_i32_bad(i32 %arg) {
+define i32 @c4_i32_bad(i32 %arg) nounwind {
 ; X86-LABEL: c4_i32_bad:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
@@ -6590,7 +7967,7 @@ define i32 @c4_i32_bad(i32 %arg) {
 ; i64
 
 ; The most canonical variant
-define i64 @c0_i64(i64 %arg) {
+define i64 @c0_i64(i64 %arg) nounwind {
 ; X86-NOBMI-LABEL: c0_i64:
 ; X86-NOBMI:       # %bb.0:
 ; X86-NOBMI-NEXT:    movl {{[0-9]+}}(%esp), %eax
@@ -6648,7 +8025,7 @@ define i64 @c0_i64(i64 %arg) {
 }
 
 ; Should be still fine, but the mask is shifted
-define i64 @c1_i64(i64 %arg) {
+define i64 @c1_i64(i64 %arg) nounwind {
 ; X86-LABEL: c1_i64:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
@@ -6669,7 +8046,7 @@ define i64 @c1_i64(i64 %arg) {
 }
 
 ; Should be still fine, but the result is shifted left afterwards
-define i64 @c2_i64(i64 %arg) {
+define i64 @c2_i64(i64 %arg) nounwind {
 ; X86-LABEL: c2_i64:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
@@ -6691,7 +8068,7 @@ define i64 @c2_i64(i64 %arg) {
 }
 
 ; The mask covers newly shifted-in bit
-define i64 @c4_i64_bad(i64 %arg) {
+define i64 @c4_i64_bad(i64 %arg) nounwind {
 ; X86-LABEL: c4_i64_bad:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
@@ -6718,7 +8095,7 @@ define i64 @c4_i64_bad(i64 %arg) {
 ; i32
 
 ; The most canonical variant
-define void @c5_i32(i32 %arg, i32* %ptr) {
+define void @c5_i32(i32 %arg, i32* %ptr) nounwind {
 ; X86-NOBMI-LABEL: c5_i32:
 ; X86-NOBMI:       # %bb.0:
 ; X86-NOBMI-NEXT:    movl {{[0-9]+}}(%esp), %eax
@@ -6784,7 +8161,7 @@ define void @c5_i32(i32 %arg, i32* %ptr) {
 }
 
 ; Should be still fine, but the mask is shifted
-define void @c6_i32(i32 %arg, i32* %ptr) {
+define void @c6_i32(i32 %arg, i32* %ptr) nounwind {
 ; X86-NOBMI-LABEL: c6_i32:
 ; X86-NOBMI:       # %bb.0:
 ; X86-NOBMI-NEXT:    movl {{[0-9]+}}(%esp), %eax
@@ -6850,7 +8227,7 @@ define void @c6_i32(i32 %arg, i32* %ptr) {
 }
 
 ; Should be still fine, but the result is shifted left afterwards
-define void @c7_i32(i32 %arg, i32* %ptr) {
+define void @c7_i32(i32 %arg, i32* %ptr) nounwind {
 ; X86-LABEL: c7_i32:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
@@ -6876,7 +8253,7 @@ define void @c7_i32(i32 %arg, i32* %ptr) {
 ; i64
 
 ; The most canonical variant
-define void @c5_i64(i64 %arg, i64* %ptr) {
+define void @c5_i64(i64 %arg, i64* %ptr) nounwind {
 ; X86-NOBMI-LABEL: c5_i64:
 ; X86-NOBMI:       # %bb.0:
 ; X86-NOBMI-NEXT:    movl {{[0-9]+}}(%esp), %eax
@@ -6946,7 +8323,7 @@ define void @c5_i64(i64 %arg, i64* %ptr) {
 }
 
 ; Should be still fine, but the mask is shifted
-define void @c6_i64(i64 %arg, i64* %ptr) {
+define void @c6_i64(i64 %arg, i64* %ptr) nounwind {
 ; X86-NOBMI-LABEL: c6_i64:
 ; X86-NOBMI:       # %bb.0:
 ; X86-NOBMI-NEXT:    movl {{[0-9]+}}(%esp), %eax
@@ -7016,7 +8393,7 @@ define void @c6_i64(i64 %arg, i64* %ptr) {
 }
 
 ; Should be still fine, but the result is shifted left afterwards
-define void @c7_i64(i64 %arg, i64* %ptr) {
+define void @c7_i64(i64 %arg, i64* %ptr) nounwind {
 ; X86-LABEL: c7_i64:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
diff --git a/test/CodeGen/X86/extract-fp.ll b/test/CodeGen/X86/extract-fp.ll
index f72764c044c10b3ceb3cb4870290b536b69e71ea..06ba30b603e50b7679b085074c4e13f944d63573 100644
--- a/test/CodeGen/X86/extract-fp.ll
+++ b/test/CodeGen/X86/extract-fp.ll
@@ -4,8 +4,8 @@
 define float @ext_fadd_v4f32(<4 x float> %x) {
 ; CHECK-LABEL: ext_fadd_v4f32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    addps {{.*}}(%rip), %xmm0
 ; CHECK-NEXT:    movhlps {{.*#+}} xmm0 = xmm0[1,1]
+; CHECK-NEXT:    addss {{.*}}(%rip), %xmm0
 ; CHECK-NEXT:    retq
   %bo = fadd <4 x float> %x, <float 1.0, float 2.0, float 3.0, float 42.0>
   %ext = extractelement <4 x float> %bo, i32 2
@@ -15,9 +15,9 @@ define float @ext_fadd_v4f32(<4 x float> %x) {
 define float @ext_fsub_v4f32(<4 x float> %x) {
 ; CHECK-LABEL: ext_fsub_v4f32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    movaps {{.*#+}} xmm1 = <u,2.0E+0,u,u>
-; CHECK-NEXT:    subps %xmm0, %xmm1
-; CHECK-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1,2,3]
+; CHECK-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; CHECK-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; CHECK-NEXT:    subss %xmm0, %xmm1
 ; CHECK-NEXT:    movaps %xmm1, %xmm0
 ; CHECK-NEXT:    retq
   %bo = fsub <4 x float> <float 1.0, float 2.0, float 3.0, float 42.0>, %x
@@ -28,19 +28,20 @@ define float @ext_fsub_v4f32(<4 x float> %x) {
 define float @ext_fmul_v4f32(<4 x float> %x) {
 ; CHECK-LABEL: ext_fmul_v4f32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    mulps {{.*}}(%rip), %xmm0
 ; CHECK-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; CHECK-NEXT:    mulss {{.*}}(%rip), %xmm0
 ; CHECK-NEXT:    retq
   %bo = fmul <4 x float> %x, <float 1.0, float 2.0, float 3.0, float 42.0>
   %ext = extractelement <4 x float> %bo, i32 3
   ret float %ext
 }
 
+; TODO: X / 1.0 --> X
+
 define float @ext_fdiv_v4f32(<4 x float> %x) {
 ; CHECK-LABEL: ext_fdiv_v4f32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; CHECK-NEXT:    divps %xmm1, %xmm0
+; CHECK-NEXT:    divss {{.*}}(%rip), %xmm0
 ; CHECK-NEXT:    retq
   %bo = fdiv <4 x float> %x, <float 1.0, float 2.0, float 3.0, float 42.0>
   %ext = extractelement <4 x float> %bo, i32 0
@@ -50,9 +51,9 @@ define float @ext_fdiv_v4f32(<4 x float> %x) {
 define float @ext_fdiv_v4f32_constant_op0(<4 x float> %x) {
 ; CHECK-LABEL: ext_fdiv_v4f32_constant_op0:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    movaps {{.*#+}} xmm1 = <u,2.0E+0,u,u>
-; CHECK-NEXT:    divps %xmm0, %xmm1
-; CHECK-NEXT:    shufps {{.*#+}} xmm1 = xmm1[1,1,2,3]
+; CHECK-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; CHECK-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; CHECK-NEXT:    divss %xmm0, %xmm1
 ; CHECK-NEXT:    movaps %xmm1, %xmm0
 ; CHECK-NEXT:    retq
   %bo = fdiv <4 x float> <float 1.0, float 2.0, float 3.0, float 42.0>, %x
diff --git a/test/CodeGen/X86/extract-lowbits.ll b/test/CodeGen/X86/extract-lowbits.ll
index ac85edd16f2ac306de2cff31a25f3cf75f85e05d..47c611ebef69616f8fb214603cf7d803daaf797d 100644
--- a/test/CodeGen/X86/extract-lowbits.ll
+++ b/test/CodeGen/X86/extract-lowbits.ll
@@ -354,7 +354,7 @@ define i64 @bzhi64_a0(i64 %val, i64 %numlowbits) nounwind {
 ;
 ; X64-BMI1NOTBM-LABEL: bzhi64_a0:
 ; X64-BMI1NOTBM:       # %bb.0:
-; X64-BMI1NOTBM-NEXT:    shlq $8, %rsi
+; X64-BMI1NOTBM-NEXT:    shll $8, %esi
 ; X64-BMI1NOTBM-NEXT:    bextrq %rsi, %rdi, %rax
 ; X64-BMI1NOTBM-NEXT:    retq
 ;
@@ -439,7 +439,7 @@ define i64 @bzhi64_a1_indexzext(i64 %val, i8 zeroext %numlowbits) nounwind {
 ; X64-BMI1NOTBM-LABEL: bzhi64_a1_indexzext:
 ; X64-BMI1NOTBM:       # %bb.0:
 ; X64-BMI1NOTBM-NEXT:    # kill: def $esi killed $esi def $rsi
-; X64-BMI1NOTBM-NEXT:    shlq $8, %rsi
+; X64-BMI1NOTBM-NEXT:    shll $8, %esi
 ; X64-BMI1NOTBM-NEXT:    bextrq %rsi, %rdi, %rax
 ; X64-BMI1NOTBM-NEXT:    retq
 ;
@@ -534,7 +534,7 @@ define i64 @bzhi64_a2_load(i64* %w, i64 %numlowbits) nounwind {
 ;
 ; X64-BMI1NOTBM-LABEL: bzhi64_a2_load:
 ; X64-BMI1NOTBM:       # %bb.0:
-; X64-BMI1NOTBM-NEXT:    shlq $8, %rsi
+; X64-BMI1NOTBM-NEXT:    shll $8, %esi
 ; X64-BMI1NOTBM-NEXT:    bextrq %rsi, (%rdi), %rax
 ; X64-BMI1NOTBM-NEXT:    retq
 ;
@@ -629,7 +629,7 @@ define i64 @bzhi64_a3_load_indexzext(i64* %w, i8 zeroext %numlowbits) nounwind {
 ; X64-BMI1NOTBM-LABEL: bzhi64_a3_load_indexzext:
 ; X64-BMI1NOTBM:       # %bb.0:
 ; X64-BMI1NOTBM-NEXT:    # kill: def $esi killed $esi def $rsi
-; X64-BMI1NOTBM-NEXT:    shlq $8, %rsi
+; X64-BMI1NOTBM-NEXT:    shll $8, %esi
 ; X64-BMI1NOTBM-NEXT:    bextrq %rsi, (%rdi), %rax
 ; X64-BMI1NOTBM-NEXT:    retq
 ;
@@ -716,7 +716,7 @@ define i64 @bzhi64_a4_commutative(i64 %val, i64 %numlowbits) nounwind {
 ;
 ; X64-BMI1NOTBM-LABEL: bzhi64_a4_commutative:
 ; X64-BMI1NOTBM:       # %bb.0:
-; X64-BMI1NOTBM-NEXT:    shlq $8, %rsi
+; X64-BMI1NOTBM-NEXT:    shll $8, %esi
 ; X64-BMI1NOTBM-NEXT:    bextrq %rsi, %rdi, %rax
 ; X64-BMI1NOTBM-NEXT:    retq
 ;
@@ -1060,7 +1060,7 @@ define i64 @bzhi64_b0(i64 %val, i64 %numlowbits) nounwind {
 ;
 ; X64-BMI1NOTBM-LABEL: bzhi64_b0:
 ; X64-BMI1NOTBM:       # %bb.0:
-; X64-BMI1NOTBM-NEXT:    shlq $8, %rsi
+; X64-BMI1NOTBM-NEXT:    shll $8, %esi
 ; X64-BMI1NOTBM-NEXT:    bextrq %rsi, %rdi, %rax
 ; X64-BMI1NOTBM-NEXT:    retq
 ;
@@ -1144,7 +1144,7 @@ define i64 @bzhi64_b1_indexzext(i64 %val, i8 zeroext %numlowbits) nounwind {
 ; X64-BMI1NOTBM-LABEL: bzhi64_b1_indexzext:
 ; X64-BMI1NOTBM:       # %bb.0:
 ; X64-BMI1NOTBM-NEXT:    # kill: def $esi killed $esi def $rsi
-; X64-BMI1NOTBM-NEXT:    shlq $8, %rsi
+; X64-BMI1NOTBM-NEXT:    shll $8, %esi
 ; X64-BMI1NOTBM-NEXT:    bextrq %rsi, %rdi, %rax
 ; X64-BMI1NOTBM-NEXT:    retq
 ;
@@ -1234,7 +1234,7 @@ define i64 @bzhi64_b2_load(i64* %w, i64 %numlowbits) nounwind {
 ;
 ; X64-BMI1NOTBM-LABEL: bzhi64_b2_load:
 ; X64-BMI1NOTBM:       # %bb.0:
-; X64-BMI1NOTBM-NEXT:    shlq $8, %rsi
+; X64-BMI1NOTBM-NEXT:    shll $8, %esi
 ; X64-BMI1NOTBM-NEXT:    bextrq %rsi, (%rdi), %rax
 ; X64-BMI1NOTBM-NEXT:    retq
 ;
@@ -1324,7 +1324,7 @@ define i64 @bzhi64_b3_load_indexzext(i64* %w, i8 zeroext %numlowbits) nounwind {
 ; X64-BMI1NOTBM-LABEL: bzhi64_b3_load_indexzext:
 ; X64-BMI1NOTBM:       # %bb.0:
 ; X64-BMI1NOTBM-NEXT:    # kill: def $esi killed $esi def $rsi
-; X64-BMI1NOTBM-NEXT:    shlq $8, %rsi
+; X64-BMI1NOTBM-NEXT:    shll $8, %esi
 ; X64-BMI1NOTBM-NEXT:    bextrq %rsi, (%rdi), %rax
 ; X64-BMI1NOTBM-NEXT:    retq
 ;
@@ -1410,7 +1410,7 @@ define i64 @bzhi64_b4_commutative(i64 %val, i64 %numlowbits) nounwind {
 ;
 ; X64-BMI1NOTBM-LABEL: bzhi64_b4_commutative:
 ; X64-BMI1NOTBM:       # %bb.0:
-; X64-BMI1NOTBM-NEXT:    shlq $8, %rsi
+; X64-BMI1NOTBM-NEXT:    shll $8, %esi
 ; X64-BMI1NOTBM-NEXT:    bextrq %rsi, %rdi, %rax
 ; X64-BMI1NOTBM-NEXT:    retq
 ;
@@ -3120,7 +3120,7 @@ define i64 @bzhi64_d0(i64 %val, i64 %numlowbits) nounwind {
 ;
 ; X64-BMI1NOTBM-LABEL: bzhi64_d0:
 ; X64-BMI1NOTBM:       # %bb.0:
-; X64-BMI1NOTBM-NEXT:    shlq $8, %rsi
+; X64-BMI1NOTBM-NEXT:    shll $8, %esi
 ; X64-BMI1NOTBM-NEXT:    bextrq %rsi, %rdi, %rax
 ; X64-BMI1NOTBM-NEXT:    retq
 ;
@@ -3258,7 +3258,7 @@ define i64 @bzhi64_d1_indexzext(i64 %val, i8 %numlowbits) nounwind {
 ; X64-BMI1NOTBM-LABEL: bzhi64_d1_indexzext:
 ; X64-BMI1NOTBM:       # %bb.0:
 ; X64-BMI1NOTBM-NEXT:    # kill: def $esi killed $esi def $rsi
-; X64-BMI1NOTBM-NEXT:    shlq $8, %rsi
+; X64-BMI1NOTBM-NEXT:    shll $8, %esi
 ; X64-BMI1NOTBM-NEXT:    bextrq %rsi, %rdi, %rax
 ; X64-BMI1NOTBM-NEXT:    retq
 ;
@@ -3400,7 +3400,7 @@ define i64 @bzhi64_d2_load(i64* %w, i64 %numlowbits) nounwind {
 ;
 ; X64-BMI1NOTBM-LABEL: bzhi64_d2_load:
 ; X64-BMI1NOTBM:       # %bb.0:
-; X64-BMI1NOTBM-NEXT:    shlq $8, %rsi
+; X64-BMI1NOTBM-NEXT:    shll $8, %esi
 ; X64-BMI1NOTBM-NEXT:    bextrq %rsi, (%rdi), %rax
 ; X64-BMI1NOTBM-NEXT:    retq
 ;
@@ -3542,7 +3542,7 @@ define i64 @bzhi64_d3_load_indexzext(i64* %w, i8 %numlowbits) nounwind {
 ; X64-BMI1NOTBM-LABEL: bzhi64_d3_load_indexzext:
 ; X64-BMI1NOTBM:       # %bb.0:
 ; X64-BMI1NOTBM-NEXT:    # kill: def $esi killed $esi def $rsi
-; X64-BMI1NOTBM-NEXT:    shlq $8, %rsi
+; X64-BMI1NOTBM-NEXT:    shll $8, %esi
 ; X64-BMI1NOTBM-NEXT:    bextrq %rsi, (%rdi), %rax
 ; X64-BMI1NOTBM-NEXT:    retq
 ;
diff --git a/test/CodeGen/X86/extractelement-load.ll b/test/CodeGen/X86/extractelement-load.ll
index c4a597b90ed6b299ab256e8d9efb9ef368517a98..f1d31b83a772dede3aa5d536baece13b71aac554 100644
--- a/test/CodeGen/X86/extractelement-load.ll
+++ b/test/CodeGen/X86/extractelement-load.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=i686-unknown -mattr=+sse2 | FileCheck %s --check-prefix=X32-SSE2
-; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+ssse3 | FileCheck %s --check-prefix=X64-SSSE3
-; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx | FileCheck %s --check-prefix=X64-AVX
+; RUN: llc < %s -mtriple=i686-unknown -mattr=+sse2 | FileCheck %s --check-prefixes=X32-SSE2
+; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+ssse3 | FileCheck %s --check-prefixes=X64,X64-SSSE3
+; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx | FileCheck %s --check-prefixes=X64,X64-AVX
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
@@ -12,15 +12,10 @@ define i32 @t(<2 x i64>* %val) nounwind  {
 ; X32-SSE2-NEXT:    movl 8(%eax), %eax
 ; X32-SSE2-NEXT:    retl
 ;
-; X64-SSSE3-LABEL: t:
-; X64-SSSE3:       # %bb.0:
-; X64-SSSE3-NEXT:    movl 8(%rdi), %eax
-; X64-SSSE3-NEXT:    retq
-;
-; X64-AVX-LABEL: t:
-; X64-AVX:       # %bb.0:
-; X64-AVX-NEXT:    movl 8(%rdi), %eax
-; X64-AVX-NEXT:    retq
+; X64-LABEL: t:
+; X64:       # %bb.0:
+; X64-NEXT:    movl 8(%rdi), %eax
+; X64-NEXT:    retq
   %tmp2 = load <2 x i64>, <2 x i64>* %val, align 16		; <<2 x i64>> [#uses=1]
   %tmp3 = bitcast <2 x i64> %tmp2 to <4 x i32>		; <<4 x i32>> [#uses=1]
   %tmp4 = extractelement <4 x i32> %tmp3, i32 2		; <i32> [#uses=1]
@@ -34,13 +29,9 @@ define i32 @t2(<8 x i32>* %xp) {
 ; X32-SSE2:       # %bb.0:
 ; X32-SSE2-NEXT:    retl
 ;
-; X64-SSSE3-LABEL: t2:
-; X64-SSSE3:       # %bb.0:
-; X64-SSSE3-NEXT:    retq
-;
-; X64-AVX-LABEL: t2:
-; X64-AVX:       # %bb.0:
-; X64-AVX-NEXT:    retq
+; X64-LABEL: t2:
+; X64:       # %bb.0:
+; X64-NEXT:    retq
   %x = load <8 x i32>, <8 x i32>* %xp
   %Shuff68 = shufflevector <8 x i32> %x, <8 x i32> undef, <8 x i32> <i32 undef, i32 7, i32 9, i32 undef, i32 13, i32 15, i32 1, i32 3>
   %y = extractelement <8 x i32> %Shuff68, i32 0
@@ -91,15 +82,10 @@ define i64 @t4(<2 x double>* %a) {
 ; X32-SSE2-NEXT:    movd %xmm0, %edx
 ; X32-SSE2-NEXT:    retl
 ;
-; X64-SSSE3-LABEL: t4:
-; X64-SSSE3:       # %bb.0:
-; X64-SSSE3-NEXT:    movq (%rdi), %rax
-; X64-SSSE3-NEXT:    retq
-;
-; X64-AVX-LABEL: t4:
-; X64-AVX:       # %bb.0:
-; X64-AVX-NEXT:    movq (%rdi), %rax
-; X64-AVX-NEXT:    retq
+; X64-LABEL: t4:
+; X64:       # %bb.0:
+; X64-NEXT:    movq (%rdi), %rax
+; X64-NEXT:    retq
   %b = load <2 x double>, <2 x double>* %a, align 16
   %c = shufflevector <2 x double> %b, <2 x double> %b, <2 x i32> <i32 1, i32 0>
   %d = bitcast <2 x double> %c to <2 x i64>
diff --git a/test/CodeGen/X86/f16c-schedule.ll b/test/CodeGen/X86/f16c-schedule.ll
deleted file mode 100644
index b4d73ce9e5abca40cbb3c095106b3d47d1b0a6d3..0000000000000000000000000000000000000000
--- a/test/CodeGen/X86/f16c-schedule.ll
+++ /dev/null
@@ -1,255 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=x86-64 -mattr=+f16c | FileCheck %s --check-prefix=CHECK --check-prefix=GENERIC
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=ivybridge | FileCheck %s --check-prefix=CHECK --check-prefix=IVY
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=haswell | FileCheck %s --check-prefix=CHECK --check-prefix=HASWELL
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=broadwell | FileCheck %s --check-prefix=CHECK --check-prefix=BROADWELL
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=skylake | FileCheck %s --check-prefix=CHECK --check-prefix=SKYLAKE
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=bdver2 | FileCheck %s --check-prefix=CHECK --check-prefix=BDVER2
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=btver2 | FileCheck %s --check-prefix=CHECK --check-prefix=BTVER2
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=znver1 | FileCheck %s --check-prefix=CHECK --check-prefix=ZNVER1
-
-define <4 x float> @test_vcvtph2ps_128(<8 x i16> %a0, <8 x i16> *%a1) {
-; GENERIC-LABEL: test_vcvtph2ps_128:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vcvtph2ps (%rdi), %xmm1 # sched: [8:1.00]
-; GENERIC-NEXT:    vcvtph2ps %xmm0, %xmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    vaddps %xmm0, %xmm1, %xmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; IVY-LABEL: test_vcvtph2ps_128:
-; IVY:       # %bb.0:
-; IVY-NEXT:    vcvtph2ps (%rdi), %xmm1 # sched: [8:1.00]
-; IVY-NEXT:    vcvtph2ps %xmm0, %xmm0 # sched: [3:1.00]
-; IVY-NEXT:    vaddps %xmm0, %xmm1, %xmm0 # sched: [3:1.00]
-; IVY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_vcvtph2ps_128:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vcvtph2ps (%rdi), %xmm1 # sched: [6:1.00]
-; HASWELL-NEXT:    vcvtph2ps %xmm0, %xmm0 # sched: [2:1.00]
-; HASWELL-NEXT:    vaddps %xmm0, %xmm1, %xmm0 # sched: [3:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_vcvtph2ps_128:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vcvtph2ps (%rdi), %xmm1 # sched: [6:1.00]
-; BROADWELL-NEXT:    vcvtph2ps %xmm0, %xmm0 # sched: [2:1.00]
-; BROADWELL-NEXT:    vaddps %xmm0, %xmm1, %xmm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_vcvtph2ps_128:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vcvtph2ps (%rdi), %xmm1 # sched: [9:0.50]
-; SKYLAKE-NEXT:    vcvtph2ps %xmm0, %xmm0 # sched: [5:1.00]
-; SKYLAKE-NEXT:    vaddps %xmm0, %xmm1, %xmm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_vcvtph2ps_128:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vcvtph2ps (%rdi), %xmm1 # sched: [13:1.00]
-; BDVER2-NEXT:    vcvtph2ps %xmm0, %xmm0 # sched: [8:1.00]
-; BDVER2-NEXT:    vaddps %xmm0, %xmm1, %xmm0 # sched: [5:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_vcvtph2ps_128:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vcvtph2ps (%rdi), %xmm1 # sched: [8:1.00]
-; BTVER2-NEXT:    vcvtph2ps %xmm0, %xmm0 # sched: [3:1.00]
-; BTVER2-NEXT:    vaddps %xmm0, %xmm1, %xmm0 # sched: [3:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_vcvtph2ps_128:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vcvtph2ps (%rdi), %xmm1 # sched: [100:0.25]
-; ZNVER1-NEXT:    vcvtph2ps %xmm0, %xmm0 # sched: [100:0.25]
-; ZNVER1-NEXT:    vaddps %xmm0, %xmm1, %xmm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = load <8 x i16>, <8 x i16> *%a1
-  %2 = call <4 x float> @llvm.x86.vcvtph2ps.128(<8 x i16> %1)
-  %3 = call <4 x float> @llvm.x86.vcvtph2ps.128(<8 x i16> %a0)
-  %4 = fadd <4 x float> %2, %3
-  ret <4 x float> %4
-}
-declare <4 x float> @llvm.x86.vcvtph2ps.128(<8 x i16>)
-
-define <8 x float> @test_vcvtph2ps_256(<8 x i16> %a0, <8 x i16> *%a1) {
-; GENERIC-LABEL: test_vcvtph2ps_256:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vcvtph2ps (%rdi), %ymm1 # sched: [8:1.00]
-; GENERIC-NEXT:    vcvtph2ps %xmm0, %ymm0 # sched: [3:1.00]
-; GENERIC-NEXT:    vaddps %ymm0, %ymm1, %ymm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; IVY-LABEL: test_vcvtph2ps_256:
-; IVY:       # %bb.0:
-; IVY-NEXT:    vcvtph2ps (%rdi), %ymm1 # sched: [8:1.00]
-; IVY-NEXT:    vcvtph2ps %xmm0, %ymm0 # sched: [3:1.00]
-; IVY-NEXT:    vaddps %ymm0, %ymm1, %ymm0 # sched: [3:1.00]
-; IVY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_vcvtph2ps_256:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vcvtph2ps (%rdi), %ymm1 # sched: [7:1.00]
-; HASWELL-NEXT:    vcvtph2ps %xmm0, %ymm0 # sched: [2:1.00]
-; HASWELL-NEXT:    vaddps %ymm0, %ymm1, %ymm0 # sched: [3:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_vcvtph2ps_256:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vcvtph2ps (%rdi), %ymm1 # sched: [6:1.00]
-; BROADWELL-NEXT:    vcvtph2ps %xmm0, %ymm0 # sched: [2:1.00]
-; BROADWELL-NEXT:    vaddps %ymm0, %ymm1, %ymm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_vcvtph2ps_256:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vcvtph2ps (%rdi), %ymm1 # sched: [10:0.50]
-; SKYLAKE-NEXT:    vcvtph2ps %xmm0, %ymm0 # sched: [7:1.00]
-; SKYLAKE-NEXT:    vaddps %ymm0, %ymm1, %ymm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_vcvtph2ps_256:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vcvtph2ps (%rdi), %ymm1 # sched: [13:2.00]
-; BDVER2-NEXT:    vcvtph2ps %xmm0, %ymm0 # sched: [8:2.00]
-; BDVER2-NEXT:    vaddps %ymm0, %ymm1, %ymm0 # sched: [5:2.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_vcvtph2ps_256:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vcvtph2ps (%rdi), %ymm1 # sched: [8:2.00]
-; BTVER2-NEXT:    vcvtph2ps %xmm0, %ymm0 # sched: [3:2.00]
-; BTVER2-NEXT:    vaddps %ymm0, %ymm1, %ymm0 # sched: [3:2.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_vcvtph2ps_256:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vcvtph2ps (%rdi), %ymm1 # sched: [100:0.25]
-; ZNVER1-NEXT:    vcvtph2ps %xmm0, %ymm0 # sched: [100:0.25]
-; ZNVER1-NEXT:    vaddps %ymm0, %ymm1, %ymm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = load <8 x i16>, <8 x i16> *%a1
-  %2 = call <8 x float> @llvm.x86.vcvtph2ps.256(<8 x i16> %1)
-  %3 = call <8 x float> @llvm.x86.vcvtph2ps.256(<8 x i16> %a0)
-  %4 = fadd <8 x float> %2, %3
-  ret <8 x float> %4
-}
-declare <8 x float> @llvm.x86.vcvtph2ps.256(<8 x i16>)
-
-define <8 x i16> @test_vcvtps2ph_128(<4 x float> %a0, <4 x float> %a1, <4 x i16> *%a2) {
-; GENERIC-LABEL: test_vcvtps2ph_128:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vcvtps2ph $0, %xmm0, %xmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    vcvtps2ph $0, %xmm1, (%rdi) # sched: [4:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; IVY-LABEL: test_vcvtps2ph_128:
-; IVY:       # %bb.0:
-; IVY-NEXT:    vcvtps2ph $0, %xmm0, %xmm0 # sched: [3:1.00]
-; IVY-NEXT:    vcvtps2ph $0, %xmm1, (%rdi) # sched: [4:1.00]
-; IVY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_vcvtps2ph_128:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vcvtps2ph $0, %xmm0, %xmm0 # sched: [4:1.00]
-; HASWELL-NEXT:    vcvtps2ph $0, %xmm1, (%rdi) # sched: [5:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_vcvtps2ph_128:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vcvtps2ph $0, %xmm0, %xmm0 # sched: [4:1.00]
-; BROADWELL-NEXT:    vcvtps2ph $0, %xmm1, (%rdi) # sched: [5:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_vcvtps2ph_128:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vcvtps2ph $0, %xmm0, %xmm0 # sched: [5:1.00]
-; SKYLAKE-NEXT:    vcvtps2ph $0, %xmm1, (%rdi) # sched: [6:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_vcvtps2ph_128:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vcvtps2ph $0, %xmm0, %xmm0 # sched: [8:1.00]
-; BDVER2-NEXT:    vcvtps2ph $0, %xmm1, (%rdi) # sched: [4:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_vcvtps2ph_128:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vcvtps2ph $0, %xmm0, %xmm0 # sched: [3:1.00]
-; BTVER2-NEXT:    vcvtps2ph $0, %xmm1, (%rdi) # sched: [4:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_vcvtps2ph_128:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vcvtps2ph $0, %xmm0, %xmm0 # sched: [100:0.25]
-; ZNVER1-NEXT:    vcvtps2ph $0, %xmm1, (%rdi) # sched: [100:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <8 x i16> @llvm.x86.vcvtps2ph.128(<4 x float> %a0, i32 0)
-  %2 = call <8 x i16> @llvm.x86.vcvtps2ph.128(<4 x float> %a1, i32 0)
-  %3 = shufflevector <8 x i16> %2, <8 x i16> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-  store <4 x i16> %3, <4 x i16> *%a2
-  ret <8 x i16> %1
-}
-declare <8 x i16> @llvm.x86.vcvtps2ph.128(<4 x float>, i32)
-
-define <8 x i16> @test_vcvtps2ph_256(<8 x float> %a0, <8 x float> %a1, <8 x i16> *%a2) {
-; GENERIC-LABEL: test_vcvtps2ph_256:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    vcvtps2ph $0, %ymm0, %xmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    vcvtps2ph $0, %ymm1, (%rdi) # sched: [4:1.00]
-; GENERIC-NEXT:    vzeroupper # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; IVY-LABEL: test_vcvtps2ph_256:
-; IVY:       # %bb.0:
-; IVY-NEXT:    vcvtps2ph $0, %ymm0, %xmm0 # sched: [3:1.00]
-; IVY-NEXT:    vcvtps2ph $0, %ymm1, (%rdi) # sched: [4:1.00]
-; IVY-NEXT:    vzeroupper # sched: [1:1.00]
-; IVY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_vcvtps2ph_256:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vcvtps2ph $0, %ymm0, %xmm0 # sched: [6:1.00]
-; HASWELL-NEXT:    vcvtps2ph $0, %ymm1, (%rdi) # sched: [7:1.00]
-; HASWELL-NEXT:    vzeroupper # sched: [0:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_vcvtps2ph_256:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vcvtps2ph $0, %ymm0, %xmm0 # sched: [6:1.00]
-; BROADWELL-NEXT:    vcvtps2ph $0, %ymm1, (%rdi) # sched: [7:1.00]
-; BROADWELL-NEXT:    vzeroupper # sched: [0:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_vcvtps2ph_256:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vcvtps2ph $0, %ymm0, %xmm0 # sched: [7:1.00]
-; SKYLAKE-NEXT:    vcvtps2ph $0, %ymm1, (%rdi) # sched: [8:1.00]
-; SKYLAKE-NEXT:    vzeroupper # sched: [0:0.67]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_vcvtps2ph_256:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vcvtps2ph $0, %ymm0, %xmm0 # sched: [8:2.00]
-; BDVER2-NEXT:    vcvtps2ph $0, %ymm1, (%rdi) # sched: [4:2.00]
-; BDVER2-NEXT:    vzeroupper # sched: [46:4.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_vcvtps2ph_256:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vcvtps2ph $0, %ymm0, %xmm0 # sched: [6:2.00]
-; BTVER2-NEXT:    vcvtps2ph $0, %ymm1, (%rdi) # sched: [7:2.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_vcvtps2ph_256:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vcvtps2ph $0, %ymm0, %xmm0 # sched: [100:0.25]
-; ZNVER1-NEXT:    vcvtps2ph $0, %ymm1, (%rdi) # sched: [100:0.25]
-; ZNVER1-NEXT:    vzeroupper # sched: [100:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <8 x i16> @llvm.x86.vcvtps2ph.256(<8 x float> %a0, i32 0)
-  %2 = call <8 x i16> @llvm.x86.vcvtps2ph.256(<8 x float> %a1, i32 0)
-  store <8 x i16> %2, <8 x i16> *%a2
-  ret <8 x i16> %1
-}
-declare <8 x i16> @llvm.x86.vcvtps2ph.256(<8 x float>, i32)
diff --git a/test/CodeGen/X86/fcmove.ll b/test/CodeGen/X86/fcmove.ll
index 35dbb68117ba27e926029699ba21eaa45431b654..6bb014858d04859b187a43f230abb4a340e924e5 100644
--- a/test/CodeGen/X86/fcmove.ll
+++ b/test/CodeGen/X86/fcmove.ll
@@ -6,7 +6,7 @@ target triple = "x86_64-unknown-unknown"
 ; Test that we can generate an fcmove, and also that it passes verification.
 
 ; CHECK-LABEL: cmove_f
-; CHECK: fcmove %st({{[0-7]}}), %st(0)
+; CHECK: fcmove %st({{[0-7]}}), %st
 define x86_fp80 @cmove_f(x86_fp80 %a, x86_fp80 %b, i32 %c) {
   %test = icmp eq i32 %c, 0
   %add = fadd x86_fp80 %a, %b
diff --git a/test/CodeGen/X86/fltused.ll b/test/CodeGen/X86/fltused.ll
index f1819bbaaea6ddb47938b83c724e3cf4de274a2a..0d020c75bbb29f96f1d8fa6d1d533f641b2b8545 100644
--- a/test/CodeGen/X86/fltused.ll
+++ b/test/CodeGen/X86/fltused.ll
@@ -1,6 +1,5 @@
-; The purpose of this test to verify that the fltused symbol is emitted when
-; any function is called with floating point arguments on Windows. And that it
-; is not emitted otherwise.
+; The purpose of this test to verify that the fltused symbol is
+; emitted when a floating point call is made on Windows.
 
 ; RUN: llc < %s -mtriple i686-pc-win32 | FileCheck %s --check-prefix WIN32
 ; RUN: llc < %s -mtriple x86_64-pc-win32 | FileCheck %s --check-prefix WIN64
diff --git a/test/CodeGen/X86/fltused_function_pointer.ll b/test/CodeGen/X86/fltused_function_pointer.ll
index 218b28f298fc5a746774fd3e3d2a31850e73c7f1..b99ff1c35accc24b39cfab119b7bbed652af8455 100644
--- a/test/CodeGen/X86/fltused_function_pointer.ll
+++ b/test/CodeGen/X86/fltused_function_pointer.ll
@@ -1,6 +1,5 @@
-; The purpose of this test to verify that the fltused symbol is emitted when
-; any function is called with floating point arguments on Windows. And that it
-; is not emitted otherwise.
+; The purpose of this test to verify that the fltused symbol is
+; emitted when a floating point call is made on Windows.
 
 ; RUN: llc < %s -mtriple i686-pc-win32 | FileCheck %s --check-prefix WIN32
 ; RUN: llc < %s -mtriple x86_64-pc-win32 | FileCheck %s --check-prefix WIN64
diff --git a/test/CodeGen/X86/fltused_math.ll b/test/CodeGen/X86/fltused_math.ll
new file mode 100644
index 0000000000000000000000000000000000000000..6f0348e9d308506407aaff6a00040515e640ecdf
--- /dev/null
+++ b/test/CodeGen/X86/fltused_math.ll
@@ -0,0 +1,18 @@
+; The purpose of this test to verify that the fltused symbol is
+; emitted when floating point operations are used on Windows.
+
+; RUN: llc < %s -mtriple i686-pc-win32 | FileCheck %s --check-prefix WIN32
+; RUN: llc < %s -mtriple x86_64-pc-win32 | FileCheck %s --check-prefix WIN64
+; RUN: llc < %s -O0 -mtriple i686-pc-win32 | FileCheck %s --check-prefix WIN32
+; RUN: llc < %s -O0 -mtriple x86_64-pc-win32 | FileCheck %s --check-prefix WIN64
+
+define i32 @foo(i32 %a) nounwind {
+entry:
+  %da = sitofp i32 %a to double
+  %div = fdiv double %da, 3.100000e+00
+  %res = fptosi double %div to i32
+  ret i32 %res
+}
+
+; WIN32: .globl __fltused
+; WIN64: .globl _fltused
diff --git a/test/CodeGen/X86/fma-schedule.ll b/test/CodeGen/X86/fma-schedule.ll
deleted file mode 100644
index ce8e00e5bc00df00e5e08212bbcd7102a218d78f..0000000000000000000000000000000000000000
--- a/test/CodeGen/X86/fma-schedule.ll
+++ /dev/null
@@ -1,3317 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=x86-64 -mattr=+fma | FileCheck %s --check-prefix=CHECK --check-prefix=GENERIC
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=bdver2 | FileCheck %s --check-prefix=CHECK --check-prefix=BDVER2
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=haswell | FileCheck %s --check-prefix=CHECK --check-prefix=HASWELL
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=broadwell | FileCheck %s --check-prefix=CHECK --check-prefix=BROADWELL
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=skylake | FileCheck %s --check-prefix=CHECK --check-prefix=SKYLAKE
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=knl | FileCheck %s --check-prefix=CHECK --check-prefix=KNL
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=skx | FileCheck %s --check-prefix=CHECK --check-prefix=SKX
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=znver1 | FileCheck %s --check-prefix=CHECK --check-prefix=ZNVER1
-
-;
-; VFMADD
-;
-
-define void @test_vfmaddpd_128(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, <2 x double> *%a3) optsize {
-; GENERIC-LABEL: test_vfmaddpd_128:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    vfmadd132pd {{.*#+}} xmm0 = (xmm0 * xmm2) + xmm1 sched: [5:0.50]
-; GENERIC-NEXT:    vfmadd213pd {{.*#+}} xmm0 = (xmm1 * xmm0) + xmm2 sched: [5:0.50]
-; GENERIC-NEXT:    vfmadd231pd {{.*#+}} xmm0 = (xmm1 * xmm2) + xmm0 sched: [5:0.50]
-; GENERIC-NEXT:    vfmadd132pd {{.*#+}} xmm0 = (xmm0 * mem) + xmm1 sched: [10:0.50]
-; GENERIC-NEXT:    vfmadd213pd {{.*#+}} xmm0 = (xmm1 * xmm0) + mem sched: [10:0.50]
-; GENERIC-NEXT:    vfmadd231pd {{.*#+}} xmm0 = (xmm1 * mem) + xmm0 sched: [10:0.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER2-LABEL: test_vfmaddpd_128:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    vfmadd132pd {{.*#+}} xmm0 = (xmm0 * xmm2) + xmm1 sched: [5:0.50]
-; BDVER2-NEXT:    vfmadd213pd {{.*#+}} xmm0 = (xmm1 * xmm0) + xmm2 sched: [5:0.50]
-; BDVER2-NEXT:    vfmadd231pd {{.*#+}} xmm0 = (xmm1 * xmm2) + xmm0 sched: [5:0.50]
-; BDVER2-NEXT:    vfmadd132pd {{.*#+}} xmm0 = (xmm0 * mem) + xmm1 sched: [10:0.50]
-; BDVER2-NEXT:    vfmadd213pd {{.*#+}} xmm0 = (xmm1 * xmm0) + mem sched: [10:0.50]
-; BDVER2-NEXT:    vfmadd231pd {{.*#+}} xmm0 = (xmm1 * mem) + xmm0 sched: [10:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; HASWELL-LABEL: test_vfmaddpd_128:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    vfmadd132pd {{.*#+}} xmm0 = (xmm0 * xmm2) + xmm1 sched: [5:0.50]
-; HASWELL-NEXT:    vfmadd213pd {{.*#+}} xmm0 = (xmm1 * xmm0) + xmm2 sched: [5:0.50]
-; HASWELL-NEXT:    vfmadd231pd {{.*#+}} xmm0 = (xmm1 * xmm2) + xmm0 sched: [5:0.50]
-; HASWELL-NEXT:    vfmadd132pd {{.*#+}} xmm0 = (xmm0 * mem) + xmm1 sched: [11:0.50]
-; HASWELL-NEXT:    vfmadd213pd {{.*#+}} xmm0 = (xmm1 * xmm0) + mem sched: [11:0.50]
-; HASWELL-NEXT:    vfmadd231pd {{.*#+}} xmm0 = (xmm1 * mem) + xmm0 sched: [11:0.50]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_vfmaddpd_128:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    vfmadd132pd {{.*#+}} xmm0 = (xmm0 * xmm2) + xmm1 sched: [5:0.50]
-; BROADWELL-NEXT:    vfmadd213pd {{.*#+}} xmm0 = (xmm1 * xmm0) + xmm2 sched: [5:0.50]
-; BROADWELL-NEXT:    vfmadd231pd {{.*#+}} xmm0 = (xmm1 * xmm2) + xmm0 sched: [5:0.50]
-; BROADWELL-NEXT:    vfmadd132pd {{.*#+}} xmm0 = (xmm0 * mem) + xmm1 sched: [10:0.50]
-; BROADWELL-NEXT:    vfmadd213pd {{.*#+}} xmm0 = (xmm1 * xmm0) + mem sched: [10:0.50]
-; BROADWELL-NEXT:    vfmadd231pd {{.*#+}} xmm0 = (xmm1 * mem) + xmm0 sched: [10:0.50]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_vfmaddpd_128:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    vfmadd132pd {{.*#+}} xmm0 = (xmm0 * xmm2) + xmm1 sched: [4:0.50]
-; SKYLAKE-NEXT:    vfmadd213pd {{.*#+}} xmm0 = (xmm1 * xmm0) + xmm2 sched: [4:0.50]
-; SKYLAKE-NEXT:    vfmadd231pd {{.*#+}} xmm0 = (xmm1 * xmm2) + xmm0 sched: [4:0.50]
-; SKYLAKE-NEXT:    vfmadd132pd {{.*#+}} xmm0 = (xmm0 * mem) + xmm1 sched: [10:0.50]
-; SKYLAKE-NEXT:    vfmadd213pd {{.*#+}} xmm0 = (xmm1 * xmm0) + mem sched: [10:0.50]
-; SKYLAKE-NEXT:    vfmadd231pd {{.*#+}} xmm0 = (xmm1 * mem) + xmm0 sched: [10:0.50]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; KNL-LABEL: test_vfmaddpd_128:
-; KNL:       # %bb.0:
-; KNL-NEXT:    #APP
-; KNL-NEXT:    vfmadd132pd {{.*#+}} xmm0 = (xmm0 * xmm2) + xmm1 sched: [5:0.50]
-; KNL-NEXT:    vfmadd213pd {{.*#+}} xmm0 = (xmm1 * xmm0) + xmm2 sched: [5:0.50]
-; KNL-NEXT:    vfmadd231pd {{.*#+}} xmm0 = (xmm1 * xmm2) + xmm0 sched: [5:0.50]
-; KNL-NEXT:    vfmadd132pd {{.*#+}} xmm0 = (xmm0 * mem) + xmm1 sched: [11:0.50]
-; KNL-NEXT:    vfmadd213pd {{.*#+}} xmm0 = (xmm1 * xmm0) + mem sched: [11:0.50]
-; KNL-NEXT:    vfmadd231pd {{.*#+}} xmm0 = (xmm1 * mem) + xmm0 sched: [11:0.50]
-; KNL-NEXT:    #NO_APP
-; KNL-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_vfmaddpd_128:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    vfmadd132pd {{.*#+}} xmm0 = (xmm0 * xmm2) + xmm1 sched: [4:0.50]
-; SKX-NEXT:    vfmadd213pd {{.*#+}} xmm0 = (xmm1 * xmm0) + xmm2 sched: [4:0.50]
-; SKX-NEXT:    vfmadd231pd {{.*#+}} xmm0 = (xmm1 * xmm2) + xmm0 sched: [4:0.50]
-; SKX-NEXT:    vfmadd132pd {{.*#+}} xmm0 = (xmm0 * mem) + xmm1 sched: [10:0.50]
-; SKX-NEXT:    vfmadd213pd {{.*#+}} xmm0 = (xmm1 * xmm0) + mem sched: [10:0.50]
-; SKX-NEXT:    vfmadd231pd {{.*#+}} xmm0 = (xmm1 * mem) + xmm0 sched: [10:0.50]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_vfmaddpd_128:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    vfmadd132pd {{.*#+}} xmm0 = (xmm0 * xmm2) + xmm1 sched: [5:0.50]
-; ZNVER1-NEXT:    vfmadd213pd {{.*#+}} xmm0 = (xmm1 * xmm0) + xmm2 sched: [5:0.50]
-; ZNVER1-NEXT:    vfmadd231pd {{.*#+}} xmm0 = (xmm1 * xmm2) + xmm0 sched: [5:0.50]
-; ZNVER1-NEXT:    vfmadd132pd {{.*#+}} xmm0 = (xmm0 * mem) + xmm1 sched: [12:0.50]
-; ZNVER1-NEXT:    vfmadd213pd {{.*#+}} xmm0 = (xmm1 * xmm0) + mem sched: [12:0.50]
-; ZNVER1-NEXT:    vfmadd231pd {{.*#+}} xmm0 = (xmm1 * mem) + xmm0 sched: [12:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  tail call void asm "vfmadd132pd $2, $1, $0 \0A\09 vfmadd213pd $2, $1, $0 \0A\09 vfmadd231pd $2, $1, $0 \0A\09 vfmadd132pd $3, $1, $0 \0A\09 vfmadd213pd $3, $1, $0 \0A\09 vfmadd231pd $3, $1, $0", "x,x,x,*m"(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, <2 x double> *%a3) nounwind
-  ret void
-}
-
-define void @test_vfmaddpd_256(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2, <4 x double> *%a3) optsize {
-; GENERIC-LABEL: test_vfmaddpd_256:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    vfmadd132pd {{.*#+}} ymm0 = (ymm0 * ymm2) + ymm1 sched: [5:0.50]
-; GENERIC-NEXT:    vfmadd213pd {{.*#+}} ymm0 = (ymm1 * ymm0) + ymm2 sched: [5:0.50]
-; GENERIC-NEXT:    vfmadd231pd {{.*#+}} ymm0 = (ymm1 * ymm2) + ymm0 sched: [5:0.50]
-; GENERIC-NEXT:    vfmadd132pd {{.*#+}} ymm0 = (ymm0 * mem) + ymm1 sched: [10:0.50]
-; GENERIC-NEXT:    vfmadd213pd {{.*#+}} ymm0 = (ymm1 * ymm0) + mem sched: [10:0.50]
-; GENERIC-NEXT:    vfmadd231pd {{.*#+}} ymm0 = (ymm1 * mem) + ymm0 sched: [10:0.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    vzeroupper # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER2-LABEL: test_vfmaddpd_256:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    vfmadd132pd {{.*#+}} ymm0 = (ymm0 * ymm2) + ymm1 sched: [5:0.50]
-; BDVER2-NEXT:    vfmadd213pd {{.*#+}} ymm0 = (ymm1 * ymm0) + ymm2 sched: [5:0.50]
-; BDVER2-NEXT:    vfmadd231pd {{.*#+}} ymm0 = (ymm1 * ymm2) + ymm0 sched: [5:0.50]
-; BDVER2-NEXT:    vfmadd132pd {{.*#+}} ymm0 = (ymm0 * mem) + ymm1 sched: [10:1.00]
-; BDVER2-NEXT:    vfmadd213pd {{.*#+}} ymm0 = (ymm1 * ymm0) + mem sched: [10:1.00]
-; BDVER2-NEXT:    vfmadd231pd {{.*#+}} ymm0 = (ymm1 * mem) + ymm0 sched: [10:1.00]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    vzeroupper # sched: [46:4.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; HASWELL-LABEL: test_vfmaddpd_256:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    vfmadd132pd {{.*#+}} ymm0 = (ymm0 * ymm2) + ymm1 sched: [5:0.50]
-; HASWELL-NEXT:    vfmadd213pd {{.*#+}} ymm0 = (ymm1 * ymm0) + ymm2 sched: [5:0.50]
-; HASWELL-NEXT:    vfmadd231pd {{.*#+}} ymm0 = (ymm1 * ymm2) + ymm0 sched: [5:0.50]
-; HASWELL-NEXT:    vfmadd132pd {{.*#+}} ymm0 = (ymm0 * mem) + ymm1 sched: [12:0.50]
-; HASWELL-NEXT:    vfmadd213pd {{.*#+}} ymm0 = (ymm1 * ymm0) + mem sched: [12:0.50]
-; HASWELL-NEXT:    vfmadd231pd {{.*#+}} ymm0 = (ymm1 * mem) + ymm0 sched: [12:0.50]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    vzeroupper # sched: [0:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_vfmaddpd_256:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    vfmadd132pd {{.*#+}} ymm0 = (ymm0 * ymm2) + ymm1 sched: [5:0.50]
-; BROADWELL-NEXT:    vfmadd213pd {{.*#+}} ymm0 = (ymm1 * ymm0) + ymm2 sched: [5:0.50]
-; BROADWELL-NEXT:    vfmadd231pd {{.*#+}} ymm0 = (ymm1 * ymm2) + ymm0 sched: [5:0.50]
-; BROADWELL-NEXT:    vfmadd132pd {{.*#+}} ymm0 = (ymm0 * mem) + ymm1 sched: [11:0.50]
-; BROADWELL-NEXT:    vfmadd213pd {{.*#+}} ymm0 = (ymm1 * ymm0) + mem sched: [11:0.50]
-; BROADWELL-NEXT:    vfmadd231pd {{.*#+}} ymm0 = (ymm1 * mem) + ymm0 sched: [11:0.50]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    vzeroupper # sched: [0:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_vfmaddpd_256:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    vfmadd132pd {{.*#+}} ymm0 = (ymm0 * ymm2) + ymm1 sched: [4:0.50]
-; SKYLAKE-NEXT:    vfmadd213pd {{.*#+}} ymm0 = (ymm1 * ymm0) + ymm2 sched: [4:0.50]
-; SKYLAKE-NEXT:    vfmadd231pd {{.*#+}} ymm0 = (ymm1 * ymm2) + ymm0 sched: [4:0.50]
-; SKYLAKE-NEXT:    vfmadd132pd {{.*#+}} ymm0 = (ymm0 * mem) + ymm1 sched: [11:0.50]
-; SKYLAKE-NEXT:    vfmadd213pd {{.*#+}} ymm0 = (ymm1 * ymm0) + mem sched: [11:0.50]
-; SKYLAKE-NEXT:    vfmadd231pd {{.*#+}} ymm0 = (ymm1 * mem) + ymm0 sched: [11:0.50]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    vzeroupper # sched: [0:0.67]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; KNL-LABEL: test_vfmaddpd_256:
-; KNL:       # %bb.0:
-; KNL-NEXT:    #APP
-; KNL-NEXT:    vfmadd132pd {{.*#+}} ymm0 = (ymm0 * ymm2) + ymm1 sched: [5:0.50]
-; KNL-NEXT:    vfmadd213pd {{.*#+}} ymm0 = (ymm1 * ymm0) + ymm2 sched: [5:0.50]
-; KNL-NEXT:    vfmadd231pd {{.*#+}} ymm0 = (ymm1 * ymm2) + ymm0 sched: [5:0.50]
-; KNL-NEXT:    vfmadd132pd {{.*#+}} ymm0 = (ymm0 * mem) + ymm1 sched: [12:0.50]
-; KNL-NEXT:    vfmadd213pd {{.*#+}} ymm0 = (ymm1 * ymm0) + mem sched: [12:0.50]
-; KNL-NEXT:    vfmadd231pd {{.*#+}} ymm0 = (ymm1 * mem) + ymm0 sched: [12:0.50]
-; KNL-NEXT:    #NO_APP
-; KNL-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_vfmaddpd_256:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    vfmadd132pd {{.*#+}} ymm0 = (ymm0 * ymm2) + ymm1 sched: [4:0.50]
-; SKX-NEXT:    vfmadd213pd {{.*#+}} ymm0 = (ymm1 * ymm0) + ymm2 sched: [4:0.50]
-; SKX-NEXT:    vfmadd231pd {{.*#+}} ymm0 = (ymm1 * ymm2) + ymm0 sched: [4:0.50]
-; SKX-NEXT:    vfmadd132pd {{.*#+}} ymm0 = (ymm0 * mem) + ymm1 sched: [11:0.50]
-; SKX-NEXT:    vfmadd213pd {{.*#+}} ymm0 = (ymm1 * ymm0) + mem sched: [11:0.50]
-; SKX-NEXT:    vfmadd231pd {{.*#+}} ymm0 = (ymm1 * mem) + ymm0 sched: [11:0.50]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    vzeroupper # sched: [0:0.67]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_vfmaddpd_256:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    vfmadd132pd {{.*#+}} ymm0 = (ymm0 * ymm2) + ymm1 sched: [5:0.50]
-; ZNVER1-NEXT:    vfmadd213pd {{.*#+}} ymm0 = (ymm1 * ymm0) + ymm2 sched: [5:0.50]
-; ZNVER1-NEXT:    vfmadd231pd {{.*#+}} ymm0 = (ymm1 * ymm2) + ymm0 sched: [5:0.50]
-; ZNVER1-NEXT:    vfmadd132pd {{.*#+}} ymm0 = (ymm0 * mem) + ymm1 sched: [12:0.50]
-; ZNVER1-NEXT:    vfmadd213pd {{.*#+}} ymm0 = (ymm1 * ymm0) + mem sched: [12:0.50]
-; ZNVER1-NEXT:    vfmadd231pd {{.*#+}} ymm0 = (ymm1 * mem) + ymm0 sched: [12:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    vzeroupper # sched: [100:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  tail call void asm "vfmadd132pd $2, $1, $0 \0A\09 vfmadd213pd $2, $1, $0 \0A\09 vfmadd231pd $2, $1, $0 \0A\09 vfmadd132pd $3, $1, $0 \0A\09 vfmadd213pd $3, $1, $0 \0A\09 vfmadd231pd $3, $1, $0", "x,x,x,*m"(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2, <4 x double> *%a3) nounwind
-  ret void
-}
-
-define void @test_vfmaddps_128(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, <4 x float> *%a3) optsize {
-; GENERIC-LABEL: test_vfmaddps_128:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    vfmadd132ps {{.*#+}} xmm0 = (xmm0 * xmm2) + xmm1 sched: [5:0.50]
-; GENERIC-NEXT:    vfmadd213ps {{.*#+}} xmm0 = (xmm1 * xmm0) + xmm2 sched: [5:0.50]
-; GENERIC-NEXT:    vfmadd231ps {{.*#+}} xmm0 = (xmm1 * xmm2) + xmm0 sched: [5:0.50]
-; GENERIC-NEXT:    vfmadd132ps {{.*#+}} xmm0 = (xmm0 * mem) + xmm1 sched: [10:0.50]
-; GENERIC-NEXT:    vfmadd213ps {{.*#+}} xmm0 = (xmm1 * xmm0) + mem sched: [10:0.50]
-; GENERIC-NEXT:    vfmadd231ps {{.*#+}} xmm0 = (xmm1 * mem) + xmm0 sched: [10:0.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER2-LABEL: test_vfmaddps_128:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    vfmadd132ps {{.*#+}} xmm0 = (xmm0 * xmm2) + xmm1 sched: [5:0.50]
-; BDVER2-NEXT:    vfmadd213ps {{.*#+}} xmm0 = (xmm1 * xmm0) + xmm2 sched: [5:0.50]
-; BDVER2-NEXT:    vfmadd231ps {{.*#+}} xmm0 = (xmm1 * xmm2) + xmm0 sched: [5:0.50]
-; BDVER2-NEXT:    vfmadd132ps {{.*#+}} xmm0 = (xmm0 * mem) + xmm1 sched: [10:0.50]
-; BDVER2-NEXT:    vfmadd213ps {{.*#+}} xmm0 = (xmm1 * xmm0) + mem sched: [10:0.50]
-; BDVER2-NEXT:    vfmadd231ps {{.*#+}} xmm0 = (xmm1 * mem) + xmm0 sched: [10:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; HASWELL-LABEL: test_vfmaddps_128:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    vfmadd132ps {{.*#+}} xmm0 = (xmm0 * xmm2) + xmm1 sched: [5:0.50]
-; HASWELL-NEXT:    vfmadd213ps {{.*#+}} xmm0 = (xmm1 * xmm0) + xmm2 sched: [5:0.50]
-; HASWELL-NEXT:    vfmadd231ps {{.*#+}} xmm0 = (xmm1 * xmm2) + xmm0 sched: [5:0.50]
-; HASWELL-NEXT:    vfmadd132ps {{.*#+}} xmm0 = (xmm0 * mem) + xmm1 sched: [11:0.50]
-; HASWELL-NEXT:    vfmadd213ps {{.*#+}} xmm0 = (xmm1 * xmm0) + mem sched: [11:0.50]
-; HASWELL-NEXT:    vfmadd231ps {{.*#+}} xmm0 = (xmm1 * mem) + xmm0 sched: [11:0.50]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_vfmaddps_128:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    vfmadd132ps {{.*#+}} xmm0 = (xmm0 * xmm2) + xmm1 sched: [5:0.50]
-; BROADWELL-NEXT:    vfmadd213ps {{.*#+}} xmm0 = (xmm1 * xmm0) + xmm2 sched: [5:0.50]
-; BROADWELL-NEXT:    vfmadd231ps {{.*#+}} xmm0 = (xmm1 * xmm2) + xmm0 sched: [5:0.50]
-; BROADWELL-NEXT:    vfmadd132ps {{.*#+}} xmm0 = (xmm0 * mem) + xmm1 sched: [10:0.50]
-; BROADWELL-NEXT:    vfmadd213ps {{.*#+}} xmm0 = (xmm1 * xmm0) + mem sched: [10:0.50]
-; BROADWELL-NEXT:    vfmadd231ps {{.*#+}} xmm0 = (xmm1 * mem) + xmm0 sched: [10:0.50]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_vfmaddps_128:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    vfmadd132ps {{.*#+}} xmm0 = (xmm0 * xmm2) + xmm1 sched: [4:0.50]
-; SKYLAKE-NEXT:    vfmadd213ps {{.*#+}} xmm0 = (xmm1 * xmm0) + xmm2 sched: [4:0.50]
-; SKYLAKE-NEXT:    vfmadd231ps {{.*#+}} xmm0 = (xmm1 * xmm2) + xmm0 sched: [4:0.50]
-; SKYLAKE-NEXT:    vfmadd132ps {{.*#+}} xmm0 = (xmm0 * mem) + xmm1 sched: [10:0.50]
-; SKYLAKE-NEXT:    vfmadd213ps {{.*#+}} xmm0 = (xmm1 * xmm0) + mem sched: [10:0.50]
-; SKYLAKE-NEXT:    vfmadd231ps {{.*#+}} xmm0 = (xmm1 * mem) + xmm0 sched: [10:0.50]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; KNL-LABEL: test_vfmaddps_128:
-; KNL:       # %bb.0:
-; KNL-NEXT:    #APP
-; KNL-NEXT:    vfmadd132ps {{.*#+}} xmm0 = (xmm0 * xmm2) + xmm1 sched: [5:0.50]
-; KNL-NEXT:    vfmadd213ps {{.*#+}} xmm0 = (xmm1 * xmm0) + xmm2 sched: [5:0.50]
-; KNL-NEXT:    vfmadd231ps {{.*#+}} xmm0 = (xmm1 * xmm2) + xmm0 sched: [5:0.50]
-; KNL-NEXT:    vfmadd132ps {{.*#+}} xmm0 = (xmm0 * mem) + xmm1 sched: [11:0.50]
-; KNL-NEXT:    vfmadd213ps {{.*#+}} xmm0 = (xmm1 * xmm0) + mem sched: [11:0.50]
-; KNL-NEXT:    vfmadd231ps {{.*#+}} xmm0 = (xmm1 * mem) + xmm0 sched: [11:0.50]
-; KNL-NEXT:    #NO_APP
-; KNL-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_vfmaddps_128:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    vfmadd132ps {{.*#+}} xmm0 = (xmm0 * xmm2) + xmm1 sched: [4:0.50]
-; SKX-NEXT:    vfmadd213ps {{.*#+}} xmm0 = (xmm1 * xmm0) + xmm2 sched: [4:0.50]
-; SKX-NEXT:    vfmadd231ps {{.*#+}} xmm0 = (xmm1 * xmm2) + xmm0 sched: [4:0.50]
-; SKX-NEXT:    vfmadd132ps {{.*#+}} xmm0 = (xmm0 * mem) + xmm1 sched: [10:0.50]
-; SKX-NEXT:    vfmadd213ps {{.*#+}} xmm0 = (xmm1 * xmm0) + mem sched: [10:0.50]
-; SKX-NEXT:    vfmadd231ps {{.*#+}} xmm0 = (xmm1 * mem) + xmm0 sched: [10:0.50]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_vfmaddps_128:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    vfmadd132ps {{.*#+}} xmm0 = (xmm0 * xmm2) + xmm1 sched: [5:0.50]
-; ZNVER1-NEXT:    vfmadd213ps {{.*#+}} xmm0 = (xmm1 * xmm0) + xmm2 sched: [5:0.50]
-; ZNVER1-NEXT:    vfmadd231ps {{.*#+}} xmm0 = (xmm1 * xmm2) + xmm0 sched: [5:0.50]
-; ZNVER1-NEXT:    vfmadd132ps {{.*#+}} xmm0 = (xmm0 * mem) + xmm1 sched: [12:0.50]
-; ZNVER1-NEXT:    vfmadd213ps {{.*#+}} xmm0 = (xmm1 * xmm0) + mem sched: [12:0.50]
-; ZNVER1-NEXT:    vfmadd231ps {{.*#+}} xmm0 = (xmm1 * mem) + xmm0 sched: [12:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  tail call void asm "vfmadd132ps $2, $1, $0 \0A\09 vfmadd213ps $2, $1, $0 \0A\09 vfmadd231ps $2, $1, $0 \0A\09 vfmadd132ps $3, $1, $0 \0A\09 vfmadd213ps $3, $1, $0 \0A\09 vfmadd231ps $3, $1, $0", "x,x,x,*m"(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, <4 x float> *%a3) nounwind
-  ret void
-}
-
-define void @test_vfmaddps_256(<8 x float> %a0, <8 x float> %a1, <8 x float> %a2, <8 x float> *%a3) optsize {
-; GENERIC-LABEL: test_vfmaddps_256:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    vfmadd132ps {{.*#+}} ymm0 = (ymm0 * ymm2) + ymm1 sched: [5:0.50]
-; GENERIC-NEXT:    vfmadd213ps {{.*#+}} ymm0 = (ymm1 * ymm0) + ymm2 sched: [5:0.50]
-; GENERIC-NEXT:    vfmadd231ps {{.*#+}} ymm0 = (ymm1 * ymm2) + ymm0 sched: [5:0.50]
-; GENERIC-NEXT:    vfmadd132ps {{.*#+}} ymm0 = (ymm0 * mem) + ymm1 sched: [10:0.50]
-; GENERIC-NEXT:    vfmadd213ps {{.*#+}} ymm0 = (ymm1 * ymm0) + mem sched: [10:0.50]
-; GENERIC-NEXT:    vfmadd231ps {{.*#+}} ymm0 = (ymm1 * mem) + ymm0 sched: [10:0.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    vzeroupper # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER2-LABEL: test_vfmaddps_256:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    vfmadd132ps {{.*#+}} ymm0 = (ymm0 * ymm2) + ymm1 sched: [5:0.50]
-; BDVER2-NEXT:    vfmadd213ps {{.*#+}} ymm0 = (ymm1 * ymm0) + ymm2 sched: [5:0.50]
-; BDVER2-NEXT:    vfmadd231ps {{.*#+}} ymm0 = (ymm1 * ymm2) + ymm0 sched: [5:0.50]
-; BDVER2-NEXT:    vfmadd132ps {{.*#+}} ymm0 = (ymm0 * mem) + ymm1 sched: [10:1.00]
-; BDVER2-NEXT:    vfmadd213ps {{.*#+}} ymm0 = (ymm1 * ymm0) + mem sched: [10:1.00]
-; BDVER2-NEXT:    vfmadd231ps {{.*#+}} ymm0 = (ymm1 * mem) + ymm0 sched: [10:1.00]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    vzeroupper # sched: [46:4.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; HASWELL-LABEL: test_vfmaddps_256:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    vfmadd132ps {{.*#+}} ymm0 = (ymm0 * ymm2) + ymm1 sched: [5:0.50]
-; HASWELL-NEXT:    vfmadd213ps {{.*#+}} ymm0 = (ymm1 * ymm0) + ymm2 sched: [5:0.50]
-; HASWELL-NEXT:    vfmadd231ps {{.*#+}} ymm0 = (ymm1 * ymm2) + ymm0 sched: [5:0.50]
-; HASWELL-NEXT:    vfmadd132ps {{.*#+}} ymm0 = (ymm0 * mem) + ymm1 sched: [12:0.50]
-; HASWELL-NEXT:    vfmadd213ps {{.*#+}} ymm0 = (ymm1 * ymm0) + mem sched: [12:0.50]
-; HASWELL-NEXT:    vfmadd231ps {{.*#+}} ymm0 = (ymm1 * mem) + ymm0 sched: [12:0.50]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    vzeroupper # sched: [0:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_vfmaddps_256:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    vfmadd132ps {{.*#+}} ymm0 = (ymm0 * ymm2) + ymm1 sched: [5:0.50]
-; BROADWELL-NEXT:    vfmadd213ps {{.*#+}} ymm0 = (ymm1 * ymm0) + ymm2 sched: [5:0.50]
-; BROADWELL-NEXT:    vfmadd231ps {{.*#+}} ymm0 = (ymm1 * ymm2) + ymm0 sched: [5:0.50]
-; BROADWELL-NEXT:    vfmadd132ps {{.*#+}} ymm0 = (ymm0 * mem) + ymm1 sched: [11:0.50]
-; BROADWELL-NEXT:    vfmadd213ps {{.*#+}} ymm0 = (ymm1 * ymm0) + mem sched: [11:0.50]
-; BROADWELL-NEXT:    vfmadd231ps {{.*#+}} ymm0 = (ymm1 * mem) + ymm0 sched: [11:0.50]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    vzeroupper # sched: [0:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_vfmaddps_256:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    vfmadd132ps {{.*#+}} ymm0 = (ymm0 * ymm2) + ymm1 sched: [4:0.50]
-; SKYLAKE-NEXT:    vfmadd213ps {{.*#+}} ymm0 = (ymm1 * ymm0) + ymm2 sched: [4:0.50]
-; SKYLAKE-NEXT:    vfmadd231ps {{.*#+}} ymm0 = (ymm1 * ymm2) + ymm0 sched: [4:0.50]
-; SKYLAKE-NEXT:    vfmadd132ps {{.*#+}} ymm0 = (ymm0 * mem) + ymm1 sched: [11:0.50]
-; SKYLAKE-NEXT:    vfmadd213ps {{.*#+}} ymm0 = (ymm1 * ymm0) + mem sched: [11:0.50]
-; SKYLAKE-NEXT:    vfmadd231ps {{.*#+}} ymm0 = (ymm1 * mem) + ymm0 sched: [11:0.50]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    vzeroupper # sched: [0:0.67]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; KNL-LABEL: test_vfmaddps_256:
-; KNL:       # %bb.0:
-; KNL-NEXT:    #APP
-; KNL-NEXT:    vfmadd132ps {{.*#+}} ymm0 = (ymm0 * ymm2) + ymm1 sched: [5:0.50]
-; KNL-NEXT:    vfmadd213ps {{.*#+}} ymm0 = (ymm1 * ymm0) + ymm2 sched: [5:0.50]
-; KNL-NEXT:    vfmadd231ps {{.*#+}} ymm0 = (ymm1 * ymm2) + ymm0 sched: [5:0.50]
-; KNL-NEXT:    vfmadd132ps {{.*#+}} ymm0 = (ymm0 * mem) + ymm1 sched: [12:0.50]
-; KNL-NEXT:    vfmadd213ps {{.*#+}} ymm0 = (ymm1 * ymm0) + mem sched: [12:0.50]
-; KNL-NEXT:    vfmadd231ps {{.*#+}} ymm0 = (ymm1 * mem) + ymm0 sched: [12:0.50]
-; KNL-NEXT:    #NO_APP
-; KNL-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_vfmaddps_256:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    vfmadd132ps {{.*#+}} ymm0 = (ymm0 * ymm2) + ymm1 sched: [4:0.50]
-; SKX-NEXT:    vfmadd213ps {{.*#+}} ymm0 = (ymm1 * ymm0) + ymm2 sched: [4:0.50]
-; SKX-NEXT:    vfmadd231ps {{.*#+}} ymm0 = (ymm1 * ymm2) + ymm0 sched: [4:0.50]
-; SKX-NEXT:    vfmadd132ps {{.*#+}} ymm0 = (ymm0 * mem) + ymm1 sched: [11:0.50]
-; SKX-NEXT:    vfmadd213ps {{.*#+}} ymm0 = (ymm1 * ymm0) + mem sched: [11:0.50]
-; SKX-NEXT:    vfmadd231ps {{.*#+}} ymm0 = (ymm1 * mem) + ymm0 sched: [11:0.50]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    vzeroupper # sched: [0:0.67]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_vfmaddps_256:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    vfmadd132ps {{.*#+}} ymm0 = (ymm0 * ymm2) + ymm1 sched: [5:0.50]
-; ZNVER1-NEXT:    vfmadd213ps {{.*#+}} ymm0 = (ymm1 * ymm0) + ymm2 sched: [5:0.50]
-; ZNVER1-NEXT:    vfmadd231ps {{.*#+}} ymm0 = (ymm1 * ymm2) + ymm0 sched: [5:0.50]
-; ZNVER1-NEXT:    vfmadd132ps {{.*#+}} ymm0 = (ymm0 * mem) + ymm1 sched: [12:0.50]
-; ZNVER1-NEXT:    vfmadd213ps {{.*#+}} ymm0 = (ymm1 * ymm0) + mem sched: [12:0.50]
-; ZNVER1-NEXT:    vfmadd231ps {{.*#+}} ymm0 = (ymm1 * mem) + ymm0 sched: [12:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    vzeroupper # sched: [100:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  tail call void asm "vfmadd132ps $2, $1, $0 \0A\09 vfmadd213ps $2, $1, $0 \0A\09 vfmadd231ps $2, $1, $0 \0A\09 vfmadd132ps $3, $1, $0 \0A\09 vfmadd213ps $3, $1, $0 \0A\09 vfmadd231ps $3, $1, $0", "x,x,x,*m"(<8 x float> %a0, <8 x float> %a1, <8 x float> %a2, <8 x float> *%a3) nounwind
-  ret void
-}
-
-define void @test_vfmaddsd_128(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, <2 x double> *%a3) optsize {
-; GENERIC-LABEL: test_vfmaddsd_128:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    vfmadd132sd {{.*#+}} xmm0 = (xmm0 * xmm2) + xmm1 sched: [5:0.50]
-; GENERIC-NEXT:    vfmadd213sd {{.*#+}} xmm0 = (xmm1 * xmm0) + xmm2 sched: [5:0.50]
-; GENERIC-NEXT:    vfmadd231sd {{.*#+}} xmm0 = (xmm1 * xmm2) + xmm0 sched: [5:0.50]
-; GENERIC-NEXT:    vfmadd132sd {{.*#+}} xmm0 = (xmm0 * mem) + xmm1 sched: [10:0.50]
-; GENERIC-NEXT:    vfmadd213sd {{.*#+}} xmm0 = (xmm1 * xmm0) + mem sched: [10:0.50]
-; GENERIC-NEXT:    vfmadd231sd {{.*#+}} xmm0 = (xmm1 * mem) + xmm0 sched: [10:0.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER2-LABEL: test_vfmaddsd_128:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    vfmadd132sd {{.*#+}} xmm0 = (xmm0 * xmm2) + xmm1 sched: [5:0.50]
-; BDVER2-NEXT:    vfmadd213sd {{.*#+}} xmm0 = (xmm1 * xmm0) + xmm2 sched: [5:0.50]
-; BDVER2-NEXT:    vfmadd231sd {{.*#+}} xmm0 = (xmm1 * xmm2) + xmm0 sched: [5:0.50]
-; BDVER2-NEXT:    vfmadd132sd {{.*#+}} xmm0 = (xmm0 * mem) + xmm1 sched: [10:0.50]
-; BDVER2-NEXT:    vfmadd213sd {{.*#+}} xmm0 = (xmm1 * xmm0) + mem sched: [10:0.50]
-; BDVER2-NEXT:    vfmadd231sd {{.*#+}} xmm0 = (xmm1 * mem) + xmm0 sched: [10:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; HASWELL-LABEL: test_vfmaddsd_128:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    vfmadd132sd {{.*#+}} xmm0 = (xmm0 * xmm2) + xmm1 sched: [5:0.50]
-; HASWELL-NEXT:    vfmadd213sd {{.*#+}} xmm0 = (xmm1 * xmm0) + xmm2 sched: [5:0.50]
-; HASWELL-NEXT:    vfmadd231sd {{.*#+}} xmm0 = (xmm1 * xmm2) + xmm0 sched: [5:0.50]
-; HASWELL-NEXT:    vfmadd132sd {{.*#+}} xmm0 = (xmm0 * mem) + xmm1 sched: [10:0.50]
-; HASWELL-NEXT:    vfmadd213sd {{.*#+}} xmm0 = (xmm1 * xmm0) + mem sched: [10:0.50]
-; HASWELL-NEXT:    vfmadd231sd {{.*#+}} xmm0 = (xmm1 * mem) + xmm0 sched: [10:0.50]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_vfmaddsd_128:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    vfmadd132sd {{.*#+}} xmm0 = (xmm0 * xmm2) + xmm1 sched: [5:0.50]
-; BROADWELL-NEXT:    vfmadd213sd {{.*#+}} xmm0 = (xmm1 * xmm0) + xmm2 sched: [5:0.50]
-; BROADWELL-NEXT:    vfmadd231sd {{.*#+}} xmm0 = (xmm1 * xmm2) + xmm0 sched: [5:0.50]
-; BROADWELL-NEXT:    vfmadd132sd {{.*#+}} xmm0 = (xmm0 * mem) + xmm1 sched: [10:0.50]
-; BROADWELL-NEXT:    vfmadd213sd {{.*#+}} xmm0 = (xmm1 * xmm0) + mem sched: [10:0.50]
-; BROADWELL-NEXT:    vfmadd231sd {{.*#+}} xmm0 = (xmm1 * mem) + xmm0 sched: [10:0.50]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_vfmaddsd_128:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    vfmadd132sd {{.*#+}} xmm0 = (xmm0 * xmm2) + xmm1 sched: [4:0.50]
-; SKYLAKE-NEXT:    vfmadd213sd {{.*#+}} xmm0 = (xmm1 * xmm0) + xmm2 sched: [4:0.50]
-; SKYLAKE-NEXT:    vfmadd231sd {{.*#+}} xmm0 = (xmm1 * xmm2) + xmm0 sched: [4:0.50]
-; SKYLAKE-NEXT:    vfmadd132sd {{.*#+}} xmm0 = (xmm0 * mem) + xmm1 sched: [9:0.50]
-; SKYLAKE-NEXT:    vfmadd213sd {{.*#+}} xmm0 = (xmm1 * xmm0) + mem sched: [9:0.50]
-; SKYLAKE-NEXT:    vfmadd231sd {{.*#+}} xmm0 = (xmm1 * mem) + xmm0 sched: [9:0.50]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; KNL-LABEL: test_vfmaddsd_128:
-; KNL:       # %bb.0:
-; KNL-NEXT:    #APP
-; KNL-NEXT:    vfmadd132sd {{.*#+}} xmm0 = (xmm0 * xmm2) + xmm1 sched: [5:0.50]
-; KNL-NEXT:    vfmadd213sd {{.*#+}} xmm0 = (xmm1 * xmm0) + xmm2 sched: [5:0.50]
-; KNL-NEXT:    vfmadd231sd {{.*#+}} xmm0 = (xmm1 * xmm2) + xmm0 sched: [5:0.50]
-; KNL-NEXT:    vfmadd132sd {{.*#+}} xmm0 = (xmm0 * mem) + xmm1 sched: [10:0.50]
-; KNL-NEXT:    vfmadd213sd {{.*#+}} xmm0 = (xmm1 * xmm0) + mem sched: [10:0.50]
-; KNL-NEXT:    vfmadd231sd {{.*#+}} xmm0 = (xmm1 * mem) + xmm0 sched: [10:0.50]
-; KNL-NEXT:    #NO_APP
-; KNL-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_vfmaddsd_128:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    vfmadd132sd {{.*#+}} xmm0 = (xmm0 * xmm2) + xmm1 sched: [4:0.50]
-; SKX-NEXT:    vfmadd213sd {{.*#+}} xmm0 = (xmm1 * xmm0) + xmm2 sched: [4:0.50]
-; SKX-NEXT:    vfmadd231sd {{.*#+}} xmm0 = (xmm1 * xmm2) + xmm0 sched: [4:0.50]
-; SKX-NEXT:    vfmadd132sd {{.*#+}} xmm0 = (xmm0 * mem) + xmm1 sched: [9:0.50]
-; SKX-NEXT:    vfmadd213sd {{.*#+}} xmm0 = (xmm1 * xmm0) + mem sched: [9:0.50]
-; SKX-NEXT:    vfmadd231sd {{.*#+}} xmm0 = (xmm1 * mem) + xmm0 sched: [9:0.50]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_vfmaddsd_128:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    vfmadd132sd {{.*#+}} xmm0 = (xmm0 * xmm2) + xmm1 sched: [5:0.50]
-; ZNVER1-NEXT:    vfmadd213sd {{.*#+}} xmm0 = (xmm1 * xmm0) + xmm2 sched: [5:0.50]
-; ZNVER1-NEXT:    vfmadd231sd {{.*#+}} xmm0 = (xmm1 * xmm2) + xmm0 sched: [5:0.50]
-; ZNVER1-NEXT:    vfmadd132sd {{.*#+}} xmm0 = (xmm0 * mem) + xmm1 sched: [12:0.50]
-; ZNVER1-NEXT:    vfmadd213sd {{.*#+}} xmm0 = (xmm1 * xmm0) + mem sched: [12:0.50]
-; ZNVER1-NEXT:    vfmadd231sd {{.*#+}} xmm0 = (xmm1 * mem) + xmm0 sched: [12:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  tail call void asm "vfmadd132sd $2, $1, $0 \0A\09 vfmadd213sd $2, $1, $0 \0A\09 vfmadd231sd $2, $1, $0 \0A\09 vfmadd132sd $3, $1, $0 \0A\09 vfmadd213sd $3, $1, $0 \0A\09 vfmadd231sd $3, $1, $0", "x,x,x,*m"(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, <2 x double> *%a3) nounwind
-  ret void
-}
-
-define void @test_vfmaddss_128(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, <4 x float> *%a3) optsize {
-; GENERIC-LABEL: test_vfmaddss_128:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    vfmadd132ss {{.*#+}} xmm0 = (xmm0 * xmm2) + xmm1 sched: [5:0.50]
-; GENERIC-NEXT:    vfmadd213ss {{.*#+}} xmm0 = (xmm1 * xmm0) + xmm2 sched: [5:0.50]
-; GENERIC-NEXT:    vfmadd231ss {{.*#+}} xmm0 = (xmm1 * xmm2) + xmm0 sched: [5:0.50]
-; GENERIC-NEXT:    vfmadd132ss {{.*#+}} xmm0 = (xmm0 * mem) + xmm1 sched: [10:0.50]
-; GENERIC-NEXT:    vfmadd213ss {{.*#+}} xmm0 = (xmm1 * xmm0) + mem sched: [10:0.50]
-; GENERIC-NEXT:    vfmadd231ss {{.*#+}} xmm0 = (xmm1 * mem) + xmm0 sched: [10:0.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER2-LABEL: test_vfmaddss_128:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    vfmadd132ss {{.*#+}} xmm0 = (xmm0 * xmm2) + xmm1 sched: [5:0.50]
-; BDVER2-NEXT:    vfmadd213ss {{.*#+}} xmm0 = (xmm1 * xmm0) + xmm2 sched: [5:0.50]
-; BDVER2-NEXT:    vfmadd231ss {{.*#+}} xmm0 = (xmm1 * xmm2) + xmm0 sched: [5:0.50]
-; BDVER2-NEXT:    vfmadd132ss {{.*#+}} xmm0 = (xmm0 * mem) + xmm1 sched: [10:0.50]
-; BDVER2-NEXT:    vfmadd213ss {{.*#+}} xmm0 = (xmm1 * xmm0) + mem sched: [10:0.50]
-; BDVER2-NEXT:    vfmadd231ss {{.*#+}} xmm0 = (xmm1 * mem) + xmm0 sched: [10:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; HASWELL-LABEL: test_vfmaddss_128:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    vfmadd132ss {{.*#+}} xmm0 = (xmm0 * xmm2) + xmm1 sched: [5:0.50]
-; HASWELL-NEXT:    vfmadd213ss {{.*#+}} xmm0 = (xmm1 * xmm0) + xmm2 sched: [5:0.50]
-; HASWELL-NEXT:    vfmadd231ss {{.*#+}} xmm0 = (xmm1 * xmm2) + xmm0 sched: [5:0.50]
-; HASWELL-NEXT:    vfmadd132ss {{.*#+}} xmm0 = (xmm0 * mem) + xmm1 sched: [10:0.50]
-; HASWELL-NEXT:    vfmadd213ss {{.*#+}} xmm0 = (xmm1 * xmm0) + mem sched: [10:0.50]
-; HASWELL-NEXT:    vfmadd231ss {{.*#+}} xmm0 = (xmm1 * mem) + xmm0 sched: [10:0.50]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_vfmaddss_128:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    vfmadd132ss {{.*#+}} xmm0 = (xmm0 * xmm2) + xmm1 sched: [5:0.50]
-; BROADWELL-NEXT:    vfmadd213ss {{.*#+}} xmm0 = (xmm1 * xmm0) + xmm2 sched: [5:0.50]
-; BROADWELL-NEXT:    vfmadd231ss {{.*#+}} xmm0 = (xmm1 * xmm2) + xmm0 sched: [5:0.50]
-; BROADWELL-NEXT:    vfmadd132ss {{.*#+}} xmm0 = (xmm0 * mem) + xmm1 sched: [10:0.50]
-; BROADWELL-NEXT:    vfmadd213ss {{.*#+}} xmm0 = (xmm1 * xmm0) + mem sched: [10:0.50]
-; BROADWELL-NEXT:    vfmadd231ss {{.*#+}} xmm0 = (xmm1 * mem) + xmm0 sched: [10:0.50]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_vfmaddss_128:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    vfmadd132ss {{.*#+}} xmm0 = (xmm0 * xmm2) + xmm1 sched: [4:0.50]
-; SKYLAKE-NEXT:    vfmadd213ss {{.*#+}} xmm0 = (xmm1 * xmm0) + xmm2 sched: [4:0.50]
-; SKYLAKE-NEXT:    vfmadd231ss {{.*#+}} xmm0 = (xmm1 * xmm2) + xmm0 sched: [4:0.50]
-; SKYLAKE-NEXT:    vfmadd132ss {{.*#+}} xmm0 = (xmm0 * mem) + xmm1 sched: [9:0.50]
-; SKYLAKE-NEXT:    vfmadd213ss {{.*#+}} xmm0 = (xmm1 * xmm0) + mem sched: [9:0.50]
-; SKYLAKE-NEXT:    vfmadd231ss {{.*#+}} xmm0 = (xmm1 * mem) + xmm0 sched: [9:0.50]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; KNL-LABEL: test_vfmaddss_128:
-; KNL:       # %bb.0:
-; KNL-NEXT:    #APP
-; KNL-NEXT:    vfmadd132ss {{.*#+}} xmm0 = (xmm0 * xmm2) + xmm1 sched: [5:0.50]
-; KNL-NEXT:    vfmadd213ss {{.*#+}} xmm0 = (xmm1 * xmm0) + xmm2 sched: [5:0.50]
-; KNL-NEXT:    vfmadd231ss {{.*#+}} xmm0 = (xmm1 * xmm2) + xmm0 sched: [5:0.50]
-; KNL-NEXT:    vfmadd132ss {{.*#+}} xmm0 = (xmm0 * mem) + xmm1 sched: [10:0.50]
-; KNL-NEXT:    vfmadd213ss {{.*#+}} xmm0 = (xmm1 * xmm0) + mem sched: [10:0.50]
-; KNL-NEXT:    vfmadd231ss {{.*#+}} xmm0 = (xmm1 * mem) + xmm0 sched: [10:0.50]
-; KNL-NEXT:    #NO_APP
-; KNL-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_vfmaddss_128:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    vfmadd132ss {{.*#+}} xmm0 = (xmm0 * xmm2) + xmm1 sched: [4:0.50]
-; SKX-NEXT:    vfmadd213ss {{.*#+}} xmm0 = (xmm1 * xmm0) + xmm2 sched: [4:0.50]
-; SKX-NEXT:    vfmadd231ss {{.*#+}} xmm0 = (xmm1 * xmm2) + xmm0 sched: [4:0.50]
-; SKX-NEXT:    vfmadd132ss {{.*#+}} xmm0 = (xmm0 * mem) + xmm1 sched: [9:0.50]
-; SKX-NEXT:    vfmadd213ss {{.*#+}} xmm0 = (xmm1 * xmm0) + mem sched: [9:0.50]
-; SKX-NEXT:    vfmadd231ss {{.*#+}} xmm0 = (xmm1 * mem) + xmm0 sched: [9:0.50]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_vfmaddss_128:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    vfmadd132ss {{.*#+}} xmm0 = (xmm0 * xmm2) + xmm1 sched: [5:0.50]
-; ZNVER1-NEXT:    vfmadd213ss {{.*#+}} xmm0 = (xmm1 * xmm0) + xmm2 sched: [5:0.50]
-; ZNVER1-NEXT:    vfmadd231ss {{.*#+}} xmm0 = (xmm1 * xmm2) + xmm0 sched: [5:0.50]
-; ZNVER1-NEXT:    vfmadd132ss {{.*#+}} xmm0 = (xmm0 * mem) + xmm1 sched: [12:0.50]
-; ZNVER1-NEXT:    vfmadd213ss {{.*#+}} xmm0 = (xmm1 * xmm0) + mem sched: [12:0.50]
-; ZNVER1-NEXT:    vfmadd231ss {{.*#+}} xmm0 = (xmm1 * mem) + xmm0 sched: [12:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  tail call void asm "vfmadd132ss $2, $1, $0 \0A\09 vfmadd213ss $2, $1, $0 \0A\09 vfmadd231ss $2, $1, $0 \0A\09 vfmadd132ss $3, $1, $0 \0A\09 vfmadd213ss $3, $1, $0 \0A\09 vfmadd231ss $3, $1, $0", "x,x,x,*m"(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, <4 x float> *%a3) nounwind
-  ret void
-}
-
-;
-; VFMADDSUB
-;
-
-define void @test_vfmaddsubpd_128(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, <2 x double> *%a3) optsize {
-; GENERIC-LABEL: test_vfmaddsubpd_128:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    vfmaddsub132pd {{.*#+}} xmm0 = (xmm0 * xmm2) +/- xmm1 sched: [5:0.50]
-; GENERIC-NEXT:    vfmaddsub213pd {{.*#+}} xmm0 = (xmm1 * xmm0) +/- xmm2 sched: [5:0.50]
-; GENERIC-NEXT:    vfmaddsub231pd {{.*#+}} xmm0 = (xmm1 * xmm2) +/- xmm0 sched: [5:0.50]
-; GENERIC-NEXT:    vfmaddsub132pd {{.*#+}} xmm0 = (xmm0 * mem) +/- xmm1 sched: [10:0.50]
-; GENERIC-NEXT:    vfmaddsub213pd {{.*#+}} xmm0 = (xmm1 * xmm0) +/- mem sched: [10:0.50]
-; GENERIC-NEXT:    vfmaddsub231pd {{.*#+}} xmm0 = (xmm1 * mem) +/- xmm0 sched: [10:0.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER2-LABEL: test_vfmaddsubpd_128:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    vfmaddsub132pd {{.*#+}} xmm0 = (xmm0 * xmm2) +/- xmm1 sched: [5:0.50]
-; BDVER2-NEXT:    vfmaddsub213pd {{.*#+}} xmm0 = (xmm1 * xmm0) +/- xmm2 sched: [5:0.50]
-; BDVER2-NEXT:    vfmaddsub231pd {{.*#+}} xmm0 = (xmm1 * xmm2) +/- xmm0 sched: [5:0.50]
-; BDVER2-NEXT:    vfmaddsub132pd {{.*#+}} xmm0 = (xmm0 * mem) +/- xmm1 sched: [10:0.50]
-; BDVER2-NEXT:    vfmaddsub213pd {{.*#+}} xmm0 = (xmm1 * xmm0) +/- mem sched: [10:0.50]
-; BDVER2-NEXT:    vfmaddsub231pd {{.*#+}} xmm0 = (xmm1 * mem) +/- xmm0 sched: [10:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; HASWELL-LABEL: test_vfmaddsubpd_128:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    vfmaddsub132pd {{.*#+}} xmm0 = (xmm0 * xmm2) +/- xmm1 sched: [5:0.50]
-; HASWELL-NEXT:    vfmaddsub213pd {{.*#+}} xmm0 = (xmm1 * xmm0) +/- xmm2 sched: [5:0.50]
-; HASWELL-NEXT:    vfmaddsub231pd {{.*#+}} xmm0 = (xmm1 * xmm2) +/- xmm0 sched: [5:0.50]
-; HASWELL-NEXT:    vfmaddsub132pd {{.*#+}} xmm0 = (xmm0 * mem) +/- xmm1 sched: [11:0.50]
-; HASWELL-NEXT:    vfmaddsub213pd {{.*#+}} xmm0 = (xmm1 * xmm0) +/- mem sched: [11:0.50]
-; HASWELL-NEXT:    vfmaddsub231pd {{.*#+}} xmm0 = (xmm1 * mem) +/- xmm0 sched: [11:0.50]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_vfmaddsubpd_128:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    vfmaddsub132pd {{.*#+}} xmm0 = (xmm0 * xmm2) +/- xmm1 sched: [5:0.50]
-; BROADWELL-NEXT:    vfmaddsub213pd {{.*#+}} xmm0 = (xmm1 * xmm0) +/- xmm2 sched: [5:0.50]
-; BROADWELL-NEXT:    vfmaddsub231pd {{.*#+}} xmm0 = (xmm1 * xmm2) +/- xmm0 sched: [5:0.50]
-; BROADWELL-NEXT:    vfmaddsub132pd {{.*#+}} xmm0 = (xmm0 * mem) +/- xmm1 sched: [10:0.50]
-; BROADWELL-NEXT:    vfmaddsub213pd {{.*#+}} xmm0 = (xmm1 * xmm0) +/- mem sched: [10:0.50]
-; BROADWELL-NEXT:    vfmaddsub231pd {{.*#+}} xmm0 = (xmm1 * mem) +/- xmm0 sched: [10:0.50]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_vfmaddsubpd_128:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    vfmaddsub132pd {{.*#+}} xmm0 = (xmm0 * xmm2) +/- xmm1 sched: [4:0.50]
-; SKYLAKE-NEXT:    vfmaddsub213pd {{.*#+}} xmm0 = (xmm1 * xmm0) +/- xmm2 sched: [4:0.50]
-; SKYLAKE-NEXT:    vfmaddsub231pd {{.*#+}} xmm0 = (xmm1 * xmm2) +/- xmm0 sched: [4:0.50]
-; SKYLAKE-NEXT:    vfmaddsub132pd {{.*#+}} xmm0 = (xmm0 * mem) +/- xmm1 sched: [10:0.50]
-; SKYLAKE-NEXT:    vfmaddsub213pd {{.*#+}} xmm0 = (xmm1 * xmm0) +/- mem sched: [10:0.50]
-; SKYLAKE-NEXT:    vfmaddsub231pd {{.*#+}} xmm0 = (xmm1 * mem) +/- xmm0 sched: [10:0.50]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; KNL-LABEL: test_vfmaddsubpd_128:
-; KNL:       # %bb.0:
-; KNL-NEXT:    #APP
-; KNL-NEXT:    vfmaddsub132pd {{.*#+}} xmm0 = (xmm0 * xmm2) +/- xmm1 sched: [5:0.50]
-; KNL-NEXT:    vfmaddsub213pd {{.*#+}} xmm0 = (xmm1 * xmm0) +/- xmm2 sched: [5:0.50]
-; KNL-NEXT:    vfmaddsub231pd {{.*#+}} xmm0 = (xmm1 * xmm2) +/- xmm0 sched: [5:0.50]
-; KNL-NEXT:    vfmaddsub132pd {{.*#+}} xmm0 = (xmm0 * mem) +/- xmm1 sched: [11:0.50]
-; KNL-NEXT:    vfmaddsub213pd {{.*#+}} xmm0 = (xmm1 * xmm0) +/- mem sched: [11:0.50]
-; KNL-NEXT:    vfmaddsub231pd {{.*#+}} xmm0 = (xmm1 * mem) +/- xmm0 sched: [11:0.50]
-; KNL-NEXT:    #NO_APP
-; KNL-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_vfmaddsubpd_128:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    vfmaddsub132pd {{.*#+}} xmm0 = (xmm0 * xmm2) +/- xmm1 sched: [4:0.50]
-; SKX-NEXT:    vfmaddsub213pd {{.*#+}} xmm0 = (xmm1 * xmm0) +/- xmm2 sched: [4:0.50]
-; SKX-NEXT:    vfmaddsub231pd {{.*#+}} xmm0 = (xmm1 * xmm2) +/- xmm0 sched: [4:0.50]
-; SKX-NEXT:    vfmaddsub132pd {{.*#+}} xmm0 = (xmm0 * mem) +/- xmm1 sched: [10:0.50]
-; SKX-NEXT:    vfmaddsub213pd {{.*#+}} xmm0 = (xmm1 * xmm0) +/- mem sched: [10:0.50]
-; SKX-NEXT:    vfmaddsub231pd {{.*#+}} xmm0 = (xmm1 * mem) +/- xmm0 sched: [10:0.50]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_vfmaddsubpd_128:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    vfmaddsub132pd {{.*#+}} xmm0 = (xmm0 * xmm2) +/- xmm1 sched: [5:0.50]
-; ZNVER1-NEXT:    vfmaddsub213pd {{.*#+}} xmm0 = (xmm1 * xmm0) +/- xmm2 sched: [5:0.50]
-; ZNVER1-NEXT:    vfmaddsub231pd {{.*#+}} xmm0 = (xmm1 * xmm2) +/- xmm0 sched: [5:0.50]
-; ZNVER1-NEXT:    vfmaddsub132pd {{.*#+}} xmm0 = (xmm0 * mem) +/- xmm1 sched: [12:0.50]
-; ZNVER1-NEXT:    vfmaddsub213pd {{.*#+}} xmm0 = (xmm1 * xmm0) +/- mem sched: [12:0.50]
-; ZNVER1-NEXT:    vfmaddsub231pd {{.*#+}} xmm0 = (xmm1 * mem) +/- xmm0 sched: [12:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  tail call void asm "vfmaddsub132pd $2, $1, $0 \0A\09 vfmaddsub213pd $2, $1, $0 \0A\09 vfmaddsub231pd $2, $1, $0 \0A\09 vfmaddsub132pd $3, $1, $0 \0A\09 vfmaddsub213pd $3, $1, $0 \0A\09 vfmaddsub231pd $3, $1, $0", "x,x,x,*m"(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, <2 x double> *%a3) nounwind
-  ret void
-}
-
-define void @test_vfmaddsubpd_256(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2, <4 x double> *%a3) optsize {
-; GENERIC-LABEL: test_vfmaddsubpd_256:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    vfmaddsub132pd {{.*#+}} ymm0 = (ymm0 * ymm2) +/- ymm1 sched: [5:0.50]
-; GENERIC-NEXT:    vfmaddsub213pd {{.*#+}} ymm0 = (ymm1 * ymm0) +/- ymm2 sched: [5:0.50]
-; GENERIC-NEXT:    vfmaddsub231pd {{.*#+}} ymm0 = (ymm1 * ymm2) +/- ymm0 sched: [5:0.50]
-; GENERIC-NEXT:    vfmaddsub132pd {{.*#+}} ymm0 = (ymm0 * mem) +/- ymm1 sched: [10:0.50]
-; GENERIC-NEXT:    vfmaddsub213pd {{.*#+}} ymm0 = (ymm1 * ymm0) +/- mem sched: [10:0.50]
-; GENERIC-NEXT:    vfmaddsub231pd {{.*#+}} ymm0 = (ymm1 * mem) +/- ymm0 sched: [10:0.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    vzeroupper # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER2-LABEL: test_vfmaddsubpd_256:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    vfmaddsub132pd {{.*#+}} ymm0 = (ymm0 * ymm2) +/- ymm1 sched: [5:0.50]
-; BDVER2-NEXT:    vfmaddsub213pd {{.*#+}} ymm0 = (ymm1 * ymm0) +/- ymm2 sched: [5:0.50]
-; BDVER2-NEXT:    vfmaddsub231pd {{.*#+}} ymm0 = (ymm1 * ymm2) +/- ymm0 sched: [5:0.50]
-; BDVER2-NEXT:    vfmaddsub132pd {{.*#+}} ymm0 = (ymm0 * mem) +/- ymm1 sched: [10:1.00]
-; BDVER2-NEXT:    vfmaddsub213pd {{.*#+}} ymm0 = (ymm1 * ymm0) +/- mem sched: [10:1.00]
-; BDVER2-NEXT:    vfmaddsub231pd {{.*#+}} ymm0 = (ymm1 * mem) +/- ymm0 sched: [10:1.00]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    vzeroupper # sched: [46:4.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; HASWELL-LABEL: test_vfmaddsubpd_256:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    vfmaddsub132pd {{.*#+}} ymm0 = (ymm0 * ymm2) +/- ymm1 sched: [5:0.50]
-; HASWELL-NEXT:    vfmaddsub213pd {{.*#+}} ymm0 = (ymm1 * ymm0) +/- ymm2 sched: [5:0.50]
-; HASWELL-NEXT:    vfmaddsub231pd {{.*#+}} ymm0 = (ymm1 * ymm2) +/- ymm0 sched: [5:0.50]
-; HASWELL-NEXT:    vfmaddsub132pd {{.*#+}} ymm0 = (ymm0 * mem) +/- ymm1 sched: [12:0.50]
-; HASWELL-NEXT:    vfmaddsub213pd {{.*#+}} ymm0 = (ymm1 * ymm0) +/- mem sched: [12:0.50]
-; HASWELL-NEXT:    vfmaddsub231pd {{.*#+}} ymm0 = (ymm1 * mem) +/- ymm0 sched: [12:0.50]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    vzeroupper # sched: [0:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_vfmaddsubpd_256:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    vfmaddsub132pd {{.*#+}} ymm0 = (ymm0 * ymm2) +/- ymm1 sched: [5:0.50]
-; BROADWELL-NEXT:    vfmaddsub213pd {{.*#+}} ymm0 = (ymm1 * ymm0) +/- ymm2 sched: [5:0.50]
-; BROADWELL-NEXT:    vfmaddsub231pd {{.*#+}} ymm0 = (ymm1 * ymm2) +/- ymm0 sched: [5:0.50]
-; BROADWELL-NEXT:    vfmaddsub132pd {{.*#+}} ymm0 = (ymm0 * mem) +/- ymm1 sched: [11:0.50]
-; BROADWELL-NEXT:    vfmaddsub213pd {{.*#+}} ymm0 = (ymm1 * ymm0) +/- mem sched: [11:0.50]
-; BROADWELL-NEXT:    vfmaddsub231pd {{.*#+}} ymm0 = (ymm1 * mem) +/- ymm0 sched: [11:0.50]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    vzeroupper # sched: [0:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_vfmaddsubpd_256:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    vfmaddsub132pd {{.*#+}} ymm0 = (ymm0 * ymm2) +/- ymm1 sched: [4:0.50]
-; SKYLAKE-NEXT:    vfmaddsub213pd {{.*#+}} ymm0 = (ymm1 * ymm0) +/- ymm2 sched: [4:0.50]
-; SKYLAKE-NEXT:    vfmaddsub231pd {{.*#+}} ymm0 = (ymm1 * ymm2) +/- ymm0 sched: [4:0.50]
-; SKYLAKE-NEXT:    vfmaddsub132pd {{.*#+}} ymm0 = (ymm0 * mem) +/- ymm1 sched: [11:0.50]
-; SKYLAKE-NEXT:    vfmaddsub213pd {{.*#+}} ymm0 = (ymm1 * ymm0) +/- mem sched: [11:0.50]
-; SKYLAKE-NEXT:    vfmaddsub231pd {{.*#+}} ymm0 = (ymm1 * mem) +/- ymm0 sched: [11:0.50]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    vzeroupper # sched: [0:0.67]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; KNL-LABEL: test_vfmaddsubpd_256:
-; KNL:       # %bb.0:
-; KNL-NEXT:    #APP
-; KNL-NEXT:    vfmaddsub132pd {{.*#+}} ymm0 = (ymm0 * ymm2) +/- ymm1 sched: [5:0.50]
-; KNL-NEXT:    vfmaddsub213pd {{.*#+}} ymm0 = (ymm1 * ymm0) +/- ymm2 sched: [5:0.50]
-; KNL-NEXT:    vfmaddsub231pd {{.*#+}} ymm0 = (ymm1 * ymm2) +/- ymm0 sched: [5:0.50]
-; KNL-NEXT:    vfmaddsub132pd {{.*#+}} ymm0 = (ymm0 * mem) +/- ymm1 sched: [12:0.50]
-; KNL-NEXT:    vfmaddsub213pd {{.*#+}} ymm0 = (ymm1 * ymm0) +/- mem sched: [12:0.50]
-; KNL-NEXT:    vfmaddsub231pd {{.*#+}} ymm0 = (ymm1 * mem) +/- ymm0 sched: [12:0.50]
-; KNL-NEXT:    #NO_APP
-; KNL-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_vfmaddsubpd_256:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    vfmaddsub132pd {{.*#+}} ymm0 = (ymm0 * ymm2) +/- ymm1 sched: [4:0.50]
-; SKX-NEXT:    vfmaddsub213pd {{.*#+}} ymm0 = (ymm1 * ymm0) +/- ymm2 sched: [4:0.50]
-; SKX-NEXT:    vfmaddsub231pd {{.*#+}} ymm0 = (ymm1 * ymm2) +/- ymm0 sched: [4:0.50]
-; SKX-NEXT:    vfmaddsub132pd {{.*#+}} ymm0 = (ymm0 * mem) +/- ymm1 sched: [11:0.50]
-; SKX-NEXT:    vfmaddsub213pd {{.*#+}} ymm0 = (ymm1 * ymm0) +/- mem sched: [11:0.50]
-; SKX-NEXT:    vfmaddsub231pd {{.*#+}} ymm0 = (ymm1 * mem) +/- ymm0 sched: [11:0.50]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    vzeroupper # sched: [0:0.67]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_vfmaddsubpd_256:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    vfmaddsub132pd {{.*#+}} ymm0 = (ymm0 * ymm2) +/- ymm1 sched: [5:0.50]
-; ZNVER1-NEXT:    vfmaddsub213pd {{.*#+}} ymm0 = (ymm1 * ymm0) +/- ymm2 sched: [5:0.50]
-; ZNVER1-NEXT:    vfmaddsub231pd {{.*#+}} ymm0 = (ymm1 * ymm2) +/- ymm0 sched: [5:0.50]
-; ZNVER1-NEXT:    vfmaddsub132pd {{.*#+}} ymm0 = (ymm0 * mem) +/- ymm1 sched: [12:0.50]
-; ZNVER1-NEXT:    vfmaddsub213pd {{.*#+}} ymm0 = (ymm1 * ymm0) +/- mem sched: [12:0.50]
-; ZNVER1-NEXT:    vfmaddsub231pd {{.*#+}} ymm0 = (ymm1 * mem) +/- ymm0 sched: [12:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    vzeroupper # sched: [100:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  tail call void asm "vfmaddsub132pd $2, $1, $0 \0A\09 vfmaddsub213pd $2, $1, $0 \0A\09 vfmaddsub231pd $2, $1, $0 \0A\09 vfmaddsub132pd $3, $1, $0 \0A\09 vfmaddsub213pd $3, $1, $0 \0A\09 vfmaddsub231pd $3, $1, $0", "x,x,x,*m"(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2, <4 x double> *%a3) nounwind
-  ret void
-}
-
-define void @test_vfmaddsubps_128(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, <4 x float> *%a3) optsize {
-; GENERIC-LABEL: test_vfmaddsubps_128:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    vfmaddsub132ps {{.*#+}} xmm0 = (xmm0 * xmm2) +/- xmm1 sched: [5:0.50]
-; GENERIC-NEXT:    vfmaddsub213ps {{.*#+}} xmm0 = (xmm1 * xmm0) +/- xmm2 sched: [5:0.50]
-; GENERIC-NEXT:    vfmaddsub231ps {{.*#+}} xmm0 = (xmm1 * xmm2) +/- xmm0 sched: [5:0.50]
-; GENERIC-NEXT:    vfmaddsub132ps {{.*#+}} xmm0 = (xmm0 * mem) +/- xmm1 sched: [10:0.50]
-; GENERIC-NEXT:    vfmaddsub213ps {{.*#+}} xmm0 = (xmm1 * xmm0) +/- mem sched: [10:0.50]
-; GENERIC-NEXT:    vfmaddsub231ps {{.*#+}} xmm0 = (xmm1 * mem) +/- xmm0 sched: [10:0.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER2-LABEL: test_vfmaddsubps_128:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    vfmaddsub132ps {{.*#+}} xmm0 = (xmm0 * xmm2) +/- xmm1 sched: [5:0.50]
-; BDVER2-NEXT:    vfmaddsub213ps {{.*#+}} xmm0 = (xmm1 * xmm0) +/- xmm2 sched: [5:0.50]
-; BDVER2-NEXT:    vfmaddsub231ps {{.*#+}} xmm0 = (xmm1 * xmm2) +/- xmm0 sched: [5:0.50]
-; BDVER2-NEXT:    vfmaddsub132ps {{.*#+}} xmm0 = (xmm0 * mem) +/- xmm1 sched: [10:0.50]
-; BDVER2-NEXT:    vfmaddsub213ps {{.*#+}} xmm0 = (xmm1 * xmm0) +/- mem sched: [10:0.50]
-; BDVER2-NEXT:    vfmaddsub231ps {{.*#+}} xmm0 = (xmm1 * mem) +/- xmm0 sched: [10:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; HASWELL-LABEL: test_vfmaddsubps_128:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    vfmaddsub132ps {{.*#+}} xmm0 = (xmm0 * xmm2) +/- xmm1 sched: [5:0.50]
-; HASWELL-NEXT:    vfmaddsub213ps {{.*#+}} xmm0 = (xmm1 * xmm0) +/- xmm2 sched: [5:0.50]
-; HASWELL-NEXT:    vfmaddsub231ps {{.*#+}} xmm0 = (xmm1 * xmm2) +/- xmm0 sched: [5:0.50]
-; HASWELL-NEXT:    vfmaddsub132ps {{.*#+}} xmm0 = (xmm0 * mem) +/- xmm1 sched: [11:0.50]
-; HASWELL-NEXT:    vfmaddsub213ps {{.*#+}} xmm0 = (xmm1 * xmm0) +/- mem sched: [11:0.50]
-; HASWELL-NEXT:    vfmaddsub231ps {{.*#+}} xmm0 = (xmm1 * mem) +/- xmm0 sched: [11:0.50]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_vfmaddsubps_128:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    vfmaddsub132ps {{.*#+}} xmm0 = (xmm0 * xmm2) +/- xmm1 sched: [5:0.50]
-; BROADWELL-NEXT:    vfmaddsub213ps {{.*#+}} xmm0 = (xmm1 * xmm0) +/- xmm2 sched: [5:0.50]
-; BROADWELL-NEXT:    vfmaddsub231ps {{.*#+}} xmm0 = (xmm1 * xmm2) +/- xmm0 sched: [5:0.50]
-; BROADWELL-NEXT:    vfmaddsub132ps {{.*#+}} xmm0 = (xmm0 * mem) +/- xmm1 sched: [10:0.50]
-; BROADWELL-NEXT:    vfmaddsub213ps {{.*#+}} xmm0 = (xmm1 * xmm0) +/- mem sched: [10:0.50]
-; BROADWELL-NEXT:    vfmaddsub231ps {{.*#+}} xmm0 = (xmm1 * mem) +/- xmm0 sched: [10:0.50]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_vfmaddsubps_128:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    vfmaddsub132ps {{.*#+}} xmm0 = (xmm0 * xmm2) +/- xmm1 sched: [4:0.50]
-; SKYLAKE-NEXT:    vfmaddsub213ps {{.*#+}} xmm0 = (xmm1 * xmm0) +/- xmm2 sched: [4:0.50]
-; SKYLAKE-NEXT:    vfmaddsub231ps {{.*#+}} xmm0 = (xmm1 * xmm2) +/- xmm0 sched: [4:0.50]
-; SKYLAKE-NEXT:    vfmaddsub132ps {{.*#+}} xmm0 = (xmm0 * mem) +/- xmm1 sched: [10:0.50]
-; SKYLAKE-NEXT:    vfmaddsub213ps {{.*#+}} xmm0 = (xmm1 * xmm0) +/- mem sched: [10:0.50]
-; SKYLAKE-NEXT:    vfmaddsub231ps {{.*#+}} xmm0 = (xmm1 * mem) +/- xmm0 sched: [10:0.50]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; KNL-LABEL: test_vfmaddsubps_128:
-; KNL:       # %bb.0:
-; KNL-NEXT:    #APP
-; KNL-NEXT:    vfmaddsub132ps {{.*#+}} xmm0 = (xmm0 * xmm2) +/- xmm1 sched: [5:0.50]
-; KNL-NEXT:    vfmaddsub213ps {{.*#+}} xmm0 = (xmm1 * xmm0) +/- xmm2 sched: [5:0.50]
-; KNL-NEXT:    vfmaddsub231ps {{.*#+}} xmm0 = (xmm1 * xmm2) +/- xmm0 sched: [5:0.50]
-; KNL-NEXT:    vfmaddsub132ps {{.*#+}} xmm0 = (xmm0 * mem) +/- xmm1 sched: [11:0.50]
-; KNL-NEXT:    vfmaddsub213ps {{.*#+}} xmm0 = (xmm1 * xmm0) +/- mem sched: [11:0.50]
-; KNL-NEXT:    vfmaddsub231ps {{.*#+}} xmm0 = (xmm1 * mem) +/- xmm0 sched: [11:0.50]
-; KNL-NEXT:    #NO_APP
-; KNL-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_vfmaddsubps_128:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    vfmaddsub132ps {{.*#+}} xmm0 = (xmm0 * xmm2) +/- xmm1 sched: [4:0.50]
-; SKX-NEXT:    vfmaddsub213ps {{.*#+}} xmm0 = (xmm1 * xmm0) +/- xmm2 sched: [4:0.50]
-; SKX-NEXT:    vfmaddsub231ps {{.*#+}} xmm0 = (xmm1 * xmm2) +/- xmm0 sched: [4:0.50]
-; SKX-NEXT:    vfmaddsub132ps {{.*#+}} xmm0 = (xmm0 * mem) +/- xmm1 sched: [10:0.50]
-; SKX-NEXT:    vfmaddsub213ps {{.*#+}} xmm0 = (xmm1 * xmm0) +/- mem sched: [10:0.50]
-; SKX-NEXT:    vfmaddsub231ps {{.*#+}} xmm0 = (xmm1 * mem) +/- xmm0 sched: [10:0.50]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_vfmaddsubps_128:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    vfmaddsub132ps {{.*#+}} xmm0 = (xmm0 * xmm2) +/- xmm1 sched: [5:0.50]
-; ZNVER1-NEXT:    vfmaddsub213ps {{.*#+}} xmm0 = (xmm1 * xmm0) +/- xmm2 sched: [5:0.50]
-; ZNVER1-NEXT:    vfmaddsub231ps {{.*#+}} xmm0 = (xmm1 * xmm2) +/- xmm0 sched: [5:0.50]
-; ZNVER1-NEXT:    vfmaddsub132ps {{.*#+}} xmm0 = (xmm0 * mem) +/- xmm1 sched: [12:0.50]
-; ZNVER1-NEXT:    vfmaddsub213ps {{.*#+}} xmm0 = (xmm1 * xmm0) +/- mem sched: [12:0.50]
-; ZNVER1-NEXT:    vfmaddsub231ps {{.*#+}} xmm0 = (xmm1 * mem) +/- xmm0 sched: [12:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  tail call void asm "vfmaddsub132ps $2, $1, $0 \0A\09 vfmaddsub213ps $2, $1, $0 \0A\09 vfmaddsub231ps $2, $1, $0 \0A\09 vfmaddsub132ps $3, $1, $0 \0A\09 vfmaddsub213ps $3, $1, $0 \0A\09 vfmaddsub231ps $3, $1, $0", "x,x,x,*m"(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, <4 x float> *%a3) nounwind
-  ret void
-}
-
-define void @test_vfmaddsubps_256(<8 x float> %a0, <8 x float> %a1, <8 x float> %a2, <8 x float> *%a3) optsize {
-; GENERIC-LABEL: test_vfmaddsubps_256:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    vfmaddsub132ps {{.*#+}} ymm0 = (ymm0 * ymm2) +/- ymm1 sched: [5:0.50]
-; GENERIC-NEXT:    vfmaddsub213ps {{.*#+}} ymm0 = (ymm1 * ymm0) +/- ymm2 sched: [5:0.50]
-; GENERIC-NEXT:    vfmaddsub231ps {{.*#+}} ymm0 = (ymm1 * ymm2) +/- ymm0 sched: [5:0.50]
-; GENERIC-NEXT:    vfmaddsub132ps {{.*#+}} ymm0 = (ymm0 * mem) +/- ymm1 sched: [10:0.50]
-; GENERIC-NEXT:    vfmaddsub213ps {{.*#+}} ymm0 = (ymm1 * ymm0) +/- mem sched: [10:0.50]
-; GENERIC-NEXT:    vfmaddsub231ps {{.*#+}} ymm0 = (ymm1 * mem) +/- ymm0 sched: [10:0.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    vzeroupper # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER2-LABEL: test_vfmaddsubps_256:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    vfmaddsub132ps {{.*#+}} ymm0 = (ymm0 * ymm2) +/- ymm1 sched: [5:0.50]
-; BDVER2-NEXT:    vfmaddsub213ps {{.*#+}} ymm0 = (ymm1 * ymm0) +/- ymm2 sched: [5:0.50]
-; BDVER2-NEXT:    vfmaddsub231ps {{.*#+}} ymm0 = (ymm1 * ymm2) +/- ymm0 sched: [5:0.50]
-; BDVER2-NEXT:    vfmaddsub132ps {{.*#+}} ymm0 = (ymm0 * mem) +/- ymm1 sched: [10:1.00]
-; BDVER2-NEXT:    vfmaddsub213ps {{.*#+}} ymm0 = (ymm1 * ymm0) +/- mem sched: [10:1.00]
-; BDVER2-NEXT:    vfmaddsub231ps {{.*#+}} ymm0 = (ymm1 * mem) +/- ymm0 sched: [10:1.00]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    vzeroupper # sched: [46:4.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; HASWELL-LABEL: test_vfmaddsubps_256:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    vfmaddsub132ps {{.*#+}} ymm0 = (ymm0 * ymm2) +/- ymm1 sched: [5:0.50]
-; HASWELL-NEXT:    vfmaddsub213ps {{.*#+}} ymm0 = (ymm1 * ymm0) +/- ymm2 sched: [5:0.50]
-; HASWELL-NEXT:    vfmaddsub231ps {{.*#+}} ymm0 = (ymm1 * ymm2) +/- ymm0 sched: [5:0.50]
-; HASWELL-NEXT:    vfmaddsub132ps {{.*#+}} ymm0 = (ymm0 * mem) +/- ymm1 sched: [12:0.50]
-; HASWELL-NEXT:    vfmaddsub213ps {{.*#+}} ymm0 = (ymm1 * ymm0) +/- mem sched: [12:0.50]
-; HASWELL-NEXT:    vfmaddsub231ps {{.*#+}} ymm0 = (ymm1 * mem) +/- ymm0 sched: [12:0.50]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    vzeroupper # sched: [0:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_vfmaddsubps_256:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    vfmaddsub132ps {{.*#+}} ymm0 = (ymm0 * ymm2) +/- ymm1 sched: [5:0.50]
-; BROADWELL-NEXT:    vfmaddsub213ps {{.*#+}} ymm0 = (ymm1 * ymm0) +/- ymm2 sched: [5:0.50]
-; BROADWELL-NEXT:    vfmaddsub231ps {{.*#+}} ymm0 = (ymm1 * ymm2) +/- ymm0 sched: [5:0.50]
-; BROADWELL-NEXT:    vfmaddsub132ps {{.*#+}} ymm0 = (ymm0 * mem) +/- ymm1 sched: [11:0.50]
-; BROADWELL-NEXT:    vfmaddsub213ps {{.*#+}} ymm0 = (ymm1 * ymm0) +/- mem sched: [11:0.50]
-; BROADWELL-NEXT:    vfmaddsub231ps {{.*#+}} ymm0 = (ymm1 * mem) +/- ymm0 sched: [11:0.50]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    vzeroupper # sched: [0:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_vfmaddsubps_256:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    vfmaddsub132ps {{.*#+}} ymm0 = (ymm0 * ymm2) +/- ymm1 sched: [4:0.50]
-; SKYLAKE-NEXT:    vfmaddsub213ps {{.*#+}} ymm0 = (ymm1 * ymm0) +/- ymm2 sched: [4:0.50]
-; SKYLAKE-NEXT:    vfmaddsub231ps {{.*#+}} ymm0 = (ymm1 * ymm2) +/- ymm0 sched: [4:0.50]
-; SKYLAKE-NEXT:    vfmaddsub132ps {{.*#+}} ymm0 = (ymm0 * mem) +/- ymm1 sched: [11:0.50]
-; SKYLAKE-NEXT:    vfmaddsub213ps {{.*#+}} ymm0 = (ymm1 * ymm0) +/- mem sched: [11:0.50]
-; SKYLAKE-NEXT:    vfmaddsub231ps {{.*#+}} ymm0 = (ymm1 * mem) +/- ymm0 sched: [11:0.50]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    vzeroupper # sched: [0:0.67]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; KNL-LABEL: test_vfmaddsubps_256:
-; KNL:       # %bb.0:
-; KNL-NEXT:    #APP
-; KNL-NEXT:    vfmaddsub132ps {{.*#+}} ymm0 = (ymm0 * ymm2) +/- ymm1 sched: [5:0.50]
-; KNL-NEXT:    vfmaddsub213ps {{.*#+}} ymm0 = (ymm1 * ymm0) +/- ymm2 sched: [5:0.50]
-; KNL-NEXT:    vfmaddsub231ps {{.*#+}} ymm0 = (ymm1 * ymm2) +/- ymm0 sched: [5:0.50]
-; KNL-NEXT:    vfmaddsub132ps {{.*#+}} ymm0 = (ymm0 * mem) +/- ymm1 sched: [12:0.50]
-; KNL-NEXT:    vfmaddsub213ps {{.*#+}} ymm0 = (ymm1 * ymm0) +/- mem sched: [12:0.50]
-; KNL-NEXT:    vfmaddsub231ps {{.*#+}} ymm0 = (ymm1 * mem) +/- ymm0 sched: [12:0.50]
-; KNL-NEXT:    #NO_APP
-; KNL-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_vfmaddsubps_256:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    vfmaddsub132ps {{.*#+}} ymm0 = (ymm0 * ymm2) +/- ymm1 sched: [4:0.50]
-; SKX-NEXT:    vfmaddsub213ps {{.*#+}} ymm0 = (ymm1 * ymm0) +/- ymm2 sched: [4:0.50]
-; SKX-NEXT:    vfmaddsub231ps {{.*#+}} ymm0 = (ymm1 * ymm2) +/- ymm0 sched: [4:0.50]
-; SKX-NEXT:    vfmaddsub132ps {{.*#+}} ymm0 = (ymm0 * mem) +/- ymm1 sched: [11:0.50]
-; SKX-NEXT:    vfmaddsub213ps {{.*#+}} ymm0 = (ymm1 * ymm0) +/- mem sched: [11:0.50]
-; SKX-NEXT:    vfmaddsub231ps {{.*#+}} ymm0 = (ymm1 * mem) +/- ymm0 sched: [11:0.50]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    vzeroupper # sched: [0:0.67]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_vfmaddsubps_256:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    vfmaddsub132ps {{.*#+}} ymm0 = (ymm0 * ymm2) +/- ymm1 sched: [5:0.50]
-; ZNVER1-NEXT:    vfmaddsub213ps {{.*#+}} ymm0 = (ymm1 * ymm0) +/- ymm2 sched: [5:0.50]
-; ZNVER1-NEXT:    vfmaddsub231ps {{.*#+}} ymm0 = (ymm1 * ymm2) +/- ymm0 sched: [5:0.50]
-; ZNVER1-NEXT:    vfmaddsub132ps {{.*#+}} ymm0 = (ymm0 * mem) +/- ymm1 sched: [12:0.50]
-; ZNVER1-NEXT:    vfmaddsub213ps {{.*#+}} ymm0 = (ymm1 * ymm0) +/- mem sched: [12:0.50]
-; ZNVER1-NEXT:    vfmaddsub231ps {{.*#+}} ymm0 = (ymm1 * mem) +/- ymm0 sched: [12:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    vzeroupper # sched: [100:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  tail call void asm "vfmaddsub132ps $2, $1, $0 \0A\09 vfmaddsub213ps $2, $1, $0 \0A\09 vfmaddsub231ps $2, $1, $0 \0A\09 vfmaddsub132ps $3, $1, $0 \0A\09 vfmaddsub213ps $3, $1, $0 \0A\09 vfmaddsub231ps $3, $1, $0", "x,x,x,*m"(<8 x float> %a0, <8 x float> %a1, <8 x float> %a2, <8 x float> *%a3) nounwind
-  ret void
-}
-
-;
-; VFMSUBADD
-;
-
-define void @test_vfmsubaddpd_128(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, <2 x double> *%a3) optsize {
-; GENERIC-LABEL: test_vfmsubaddpd_128:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    vfmsubadd132pd {{.*#+}} xmm0 = (xmm0 * xmm2) -/+ xmm1 sched: [5:0.50]
-; GENERIC-NEXT:    vfmsubadd213pd {{.*#+}} xmm0 = (xmm1 * xmm0) -/+ xmm2 sched: [5:0.50]
-; GENERIC-NEXT:    vfmsubadd231pd {{.*#+}} xmm0 = (xmm1 * xmm2) -/+ xmm0 sched: [5:0.50]
-; GENERIC-NEXT:    vfmsubadd132pd {{.*#+}} xmm0 = (xmm0 * mem) -/+ xmm1 sched: [10:0.50]
-; GENERIC-NEXT:    vfmsubadd213pd {{.*#+}} xmm0 = (xmm1 * xmm0) -/+ mem sched: [10:0.50]
-; GENERIC-NEXT:    vfmsubadd231pd {{.*#+}} xmm0 = (xmm1 * mem) -/+ xmm0 sched: [10:0.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER2-LABEL: test_vfmsubaddpd_128:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    vfmsubadd132pd {{.*#+}} xmm0 = (xmm0 * xmm2) -/+ xmm1 sched: [5:0.50]
-; BDVER2-NEXT:    vfmsubadd213pd {{.*#+}} xmm0 = (xmm1 * xmm0) -/+ xmm2 sched: [5:0.50]
-; BDVER2-NEXT:    vfmsubadd231pd {{.*#+}} xmm0 = (xmm1 * xmm2) -/+ xmm0 sched: [5:0.50]
-; BDVER2-NEXT:    vfmsubadd132pd {{.*#+}} xmm0 = (xmm0 * mem) -/+ xmm1 sched: [10:0.50]
-; BDVER2-NEXT:    vfmsubadd213pd {{.*#+}} xmm0 = (xmm1 * xmm0) -/+ mem sched: [10:0.50]
-; BDVER2-NEXT:    vfmsubadd231pd {{.*#+}} xmm0 = (xmm1 * mem) -/+ xmm0 sched: [10:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; HASWELL-LABEL: test_vfmsubaddpd_128:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    vfmsubadd132pd {{.*#+}} xmm0 = (xmm0 * xmm2) -/+ xmm1 sched: [5:0.50]
-; HASWELL-NEXT:    vfmsubadd213pd {{.*#+}} xmm0 = (xmm1 * xmm0) -/+ xmm2 sched: [5:0.50]
-; HASWELL-NEXT:    vfmsubadd231pd {{.*#+}} xmm0 = (xmm1 * xmm2) -/+ xmm0 sched: [5:0.50]
-; HASWELL-NEXT:    vfmsubadd132pd {{.*#+}} xmm0 = (xmm0 * mem) -/+ xmm1 sched: [11:0.50]
-; HASWELL-NEXT:    vfmsubadd213pd {{.*#+}} xmm0 = (xmm1 * xmm0) -/+ mem sched: [11:0.50]
-; HASWELL-NEXT:    vfmsubadd231pd {{.*#+}} xmm0 = (xmm1 * mem) -/+ xmm0 sched: [11:0.50]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_vfmsubaddpd_128:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    vfmsubadd132pd {{.*#+}} xmm0 = (xmm0 * xmm2) -/+ xmm1 sched: [5:0.50]
-; BROADWELL-NEXT:    vfmsubadd213pd {{.*#+}} xmm0 = (xmm1 * xmm0) -/+ xmm2 sched: [5:0.50]
-; BROADWELL-NEXT:    vfmsubadd231pd {{.*#+}} xmm0 = (xmm1 * xmm2) -/+ xmm0 sched: [5:0.50]
-; BROADWELL-NEXT:    vfmsubadd132pd {{.*#+}} xmm0 = (xmm0 * mem) -/+ xmm1 sched: [10:0.50]
-; BROADWELL-NEXT:    vfmsubadd213pd {{.*#+}} xmm0 = (xmm1 * xmm0) -/+ mem sched: [10:0.50]
-; BROADWELL-NEXT:    vfmsubadd231pd {{.*#+}} xmm0 = (xmm1 * mem) -/+ xmm0 sched: [10:0.50]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_vfmsubaddpd_128:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    vfmsubadd132pd {{.*#+}} xmm0 = (xmm0 * xmm2) -/+ xmm1 sched: [4:0.50]
-; SKYLAKE-NEXT:    vfmsubadd213pd {{.*#+}} xmm0 = (xmm1 * xmm0) -/+ xmm2 sched: [4:0.50]
-; SKYLAKE-NEXT:    vfmsubadd231pd {{.*#+}} xmm0 = (xmm1 * xmm2) -/+ xmm0 sched: [4:0.50]
-; SKYLAKE-NEXT:    vfmsubadd132pd {{.*#+}} xmm0 = (xmm0 * mem) -/+ xmm1 sched: [10:0.50]
-; SKYLAKE-NEXT:    vfmsubadd213pd {{.*#+}} xmm0 = (xmm1 * xmm0) -/+ mem sched: [10:0.50]
-; SKYLAKE-NEXT:    vfmsubadd231pd {{.*#+}} xmm0 = (xmm1 * mem) -/+ xmm0 sched: [10:0.50]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; KNL-LABEL: test_vfmsubaddpd_128:
-; KNL:       # %bb.0:
-; KNL-NEXT:    #APP
-; KNL-NEXT:    vfmsubadd132pd {{.*#+}} xmm0 = (xmm0 * xmm2) -/+ xmm1 sched: [5:0.50]
-; KNL-NEXT:    vfmsubadd213pd {{.*#+}} xmm0 = (xmm1 * xmm0) -/+ xmm2 sched: [5:0.50]
-; KNL-NEXT:    vfmsubadd231pd {{.*#+}} xmm0 = (xmm1 * xmm2) -/+ xmm0 sched: [5:0.50]
-; KNL-NEXT:    vfmsubadd132pd {{.*#+}} xmm0 = (xmm0 * mem) -/+ xmm1 sched: [11:0.50]
-; KNL-NEXT:    vfmsubadd213pd {{.*#+}} xmm0 = (xmm1 * xmm0) -/+ mem sched: [11:0.50]
-; KNL-NEXT:    vfmsubadd231pd {{.*#+}} xmm0 = (xmm1 * mem) -/+ xmm0 sched: [11:0.50]
-; KNL-NEXT:    #NO_APP
-; KNL-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_vfmsubaddpd_128:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    vfmsubadd132pd {{.*#+}} xmm0 = (xmm0 * xmm2) -/+ xmm1 sched: [4:0.50]
-; SKX-NEXT:    vfmsubadd213pd {{.*#+}} xmm0 = (xmm1 * xmm0) -/+ xmm2 sched: [4:0.50]
-; SKX-NEXT:    vfmsubadd231pd {{.*#+}} xmm0 = (xmm1 * xmm2) -/+ xmm0 sched: [4:0.50]
-; SKX-NEXT:    vfmsubadd132pd {{.*#+}} xmm0 = (xmm0 * mem) -/+ xmm1 sched: [10:0.50]
-; SKX-NEXT:    vfmsubadd213pd {{.*#+}} xmm0 = (xmm1 * xmm0) -/+ mem sched: [10:0.50]
-; SKX-NEXT:    vfmsubadd231pd {{.*#+}} xmm0 = (xmm1 * mem) -/+ xmm0 sched: [10:0.50]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_vfmsubaddpd_128:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    vfmsubadd132pd {{.*#+}} xmm0 = (xmm0 * xmm2) -/+ xmm1 sched: [5:0.50]
-; ZNVER1-NEXT:    vfmsubadd213pd {{.*#+}} xmm0 = (xmm1 * xmm0) -/+ xmm2 sched: [5:0.50]
-; ZNVER1-NEXT:    vfmsubadd231pd {{.*#+}} xmm0 = (xmm1 * xmm2) -/+ xmm0 sched: [5:0.50]
-; ZNVER1-NEXT:    vfmsubadd132pd {{.*#+}} xmm0 = (xmm0 * mem) -/+ xmm1 sched: [12:0.50]
-; ZNVER1-NEXT:    vfmsubadd213pd {{.*#+}} xmm0 = (xmm1 * xmm0) -/+ mem sched: [12:0.50]
-; ZNVER1-NEXT:    vfmsubadd231pd {{.*#+}} xmm0 = (xmm1 * mem) -/+ xmm0 sched: [12:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  tail call void asm "vfmsubadd132pd $2, $1, $0 \0A\09 vfmsubadd213pd $2, $1, $0 \0A\09 vfmsubadd231pd $2, $1, $0 \0A\09 vfmsubadd132pd $3, $1, $0 \0A\09 vfmsubadd213pd $3, $1, $0 \0A\09 vfmsubadd231pd $3, $1, $0", "x,x,x,*m"(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, <2 x double> *%a3) nounwind
-  ret void
-}
-
-define void @test_vfmsubaddpd_256(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2, <4 x double> *%a3) optsize {
-; GENERIC-LABEL: test_vfmsubaddpd_256:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    vfmsubadd132pd {{.*#+}} ymm0 = (ymm0 * ymm2) -/+ ymm1 sched: [5:0.50]
-; GENERIC-NEXT:    vfmsubadd213pd {{.*#+}} ymm0 = (ymm1 * ymm0) -/+ ymm2 sched: [5:0.50]
-; GENERIC-NEXT:    vfmsubadd231pd {{.*#+}} ymm0 = (ymm1 * ymm2) -/+ ymm0 sched: [5:0.50]
-; GENERIC-NEXT:    vfmsubadd132pd {{.*#+}} ymm0 = (ymm0 * mem) -/+ ymm1 sched: [10:0.50]
-; GENERIC-NEXT:    vfmsubadd213pd {{.*#+}} ymm0 = (ymm1 * ymm0) -/+ mem sched: [10:0.50]
-; GENERIC-NEXT:    vfmsubadd231pd {{.*#+}} ymm0 = (ymm1 * mem) -/+ ymm0 sched: [10:0.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    vzeroupper # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER2-LABEL: test_vfmsubaddpd_256:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    vfmsubadd132pd {{.*#+}} ymm0 = (ymm0 * ymm2) -/+ ymm1 sched: [5:0.50]
-; BDVER2-NEXT:    vfmsubadd213pd {{.*#+}} ymm0 = (ymm1 * ymm0) -/+ ymm2 sched: [5:0.50]
-; BDVER2-NEXT:    vfmsubadd231pd {{.*#+}} ymm0 = (ymm1 * ymm2) -/+ ymm0 sched: [5:0.50]
-; BDVER2-NEXT:    vfmsubadd132pd {{.*#+}} ymm0 = (ymm0 * mem) -/+ ymm1 sched: [10:1.00]
-; BDVER2-NEXT:    vfmsubadd213pd {{.*#+}} ymm0 = (ymm1 * ymm0) -/+ mem sched: [10:1.00]
-; BDVER2-NEXT:    vfmsubadd231pd {{.*#+}} ymm0 = (ymm1 * mem) -/+ ymm0 sched: [10:1.00]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    vzeroupper # sched: [46:4.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; HASWELL-LABEL: test_vfmsubaddpd_256:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    vfmsubadd132pd {{.*#+}} ymm0 = (ymm0 * ymm2) -/+ ymm1 sched: [5:0.50]
-; HASWELL-NEXT:    vfmsubadd213pd {{.*#+}} ymm0 = (ymm1 * ymm0) -/+ ymm2 sched: [5:0.50]
-; HASWELL-NEXT:    vfmsubadd231pd {{.*#+}} ymm0 = (ymm1 * ymm2) -/+ ymm0 sched: [5:0.50]
-; HASWELL-NEXT:    vfmsubadd132pd {{.*#+}} ymm0 = (ymm0 * mem) -/+ ymm1 sched: [12:0.50]
-; HASWELL-NEXT:    vfmsubadd213pd {{.*#+}} ymm0 = (ymm1 * ymm0) -/+ mem sched: [12:0.50]
-; HASWELL-NEXT:    vfmsubadd231pd {{.*#+}} ymm0 = (ymm1 * mem) -/+ ymm0 sched: [12:0.50]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    vzeroupper # sched: [0:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_vfmsubaddpd_256:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    vfmsubadd132pd {{.*#+}} ymm0 = (ymm0 * ymm2) -/+ ymm1 sched: [5:0.50]
-; BROADWELL-NEXT:    vfmsubadd213pd {{.*#+}} ymm0 = (ymm1 * ymm0) -/+ ymm2 sched: [5:0.50]
-; BROADWELL-NEXT:    vfmsubadd231pd {{.*#+}} ymm0 = (ymm1 * ymm2) -/+ ymm0 sched: [5:0.50]
-; BROADWELL-NEXT:    vfmsubadd132pd {{.*#+}} ymm0 = (ymm0 * mem) -/+ ymm1 sched: [11:0.50]
-; BROADWELL-NEXT:    vfmsubadd213pd {{.*#+}} ymm0 = (ymm1 * ymm0) -/+ mem sched: [11:0.50]
-; BROADWELL-NEXT:    vfmsubadd231pd {{.*#+}} ymm0 = (ymm1 * mem) -/+ ymm0 sched: [11:0.50]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    vzeroupper # sched: [0:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_vfmsubaddpd_256:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    vfmsubadd132pd {{.*#+}} ymm0 = (ymm0 * ymm2) -/+ ymm1 sched: [4:0.50]
-; SKYLAKE-NEXT:    vfmsubadd213pd {{.*#+}} ymm0 = (ymm1 * ymm0) -/+ ymm2 sched: [4:0.50]
-; SKYLAKE-NEXT:    vfmsubadd231pd {{.*#+}} ymm0 = (ymm1 * ymm2) -/+ ymm0 sched: [4:0.50]
-; SKYLAKE-NEXT:    vfmsubadd132pd {{.*#+}} ymm0 = (ymm0 * mem) -/+ ymm1 sched: [11:0.50]
-; SKYLAKE-NEXT:    vfmsubadd213pd {{.*#+}} ymm0 = (ymm1 * ymm0) -/+ mem sched: [11:0.50]
-; SKYLAKE-NEXT:    vfmsubadd231pd {{.*#+}} ymm0 = (ymm1 * mem) -/+ ymm0 sched: [11:0.50]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    vzeroupper # sched: [0:0.67]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; KNL-LABEL: test_vfmsubaddpd_256:
-; KNL:       # %bb.0:
-; KNL-NEXT:    #APP
-; KNL-NEXT:    vfmsubadd132pd {{.*#+}} ymm0 = (ymm0 * ymm2) -/+ ymm1 sched: [5:0.50]
-; KNL-NEXT:    vfmsubadd213pd {{.*#+}} ymm0 = (ymm1 * ymm0) -/+ ymm2 sched: [5:0.50]
-; KNL-NEXT:    vfmsubadd231pd {{.*#+}} ymm0 = (ymm1 * ymm2) -/+ ymm0 sched: [5:0.50]
-; KNL-NEXT:    vfmsubadd132pd {{.*#+}} ymm0 = (ymm0 * mem) -/+ ymm1 sched: [12:0.50]
-; KNL-NEXT:    vfmsubadd213pd {{.*#+}} ymm0 = (ymm1 * ymm0) -/+ mem sched: [12:0.50]
-; KNL-NEXT:    vfmsubadd231pd {{.*#+}} ymm0 = (ymm1 * mem) -/+ ymm0 sched: [12:0.50]
-; KNL-NEXT:    #NO_APP
-; KNL-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_vfmsubaddpd_256:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    vfmsubadd132pd {{.*#+}} ymm0 = (ymm0 * ymm2) -/+ ymm1 sched: [4:0.50]
-; SKX-NEXT:    vfmsubadd213pd {{.*#+}} ymm0 = (ymm1 * ymm0) -/+ ymm2 sched: [4:0.50]
-; SKX-NEXT:    vfmsubadd231pd {{.*#+}} ymm0 = (ymm1 * ymm2) -/+ ymm0 sched: [4:0.50]
-; SKX-NEXT:    vfmsubadd132pd {{.*#+}} ymm0 = (ymm0 * mem) -/+ ymm1 sched: [11:0.50]
-; SKX-NEXT:    vfmsubadd213pd {{.*#+}} ymm0 = (ymm1 * ymm0) -/+ mem sched: [11:0.50]
-; SKX-NEXT:    vfmsubadd231pd {{.*#+}} ymm0 = (ymm1 * mem) -/+ ymm0 sched: [11:0.50]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    vzeroupper # sched: [0:0.67]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_vfmsubaddpd_256:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    vfmsubadd132pd {{.*#+}} ymm0 = (ymm0 * ymm2) -/+ ymm1 sched: [5:0.50]
-; ZNVER1-NEXT:    vfmsubadd213pd {{.*#+}} ymm0 = (ymm1 * ymm0) -/+ ymm2 sched: [5:0.50]
-; ZNVER1-NEXT:    vfmsubadd231pd {{.*#+}} ymm0 = (ymm1 * ymm2) -/+ ymm0 sched: [5:0.50]
-; ZNVER1-NEXT:    vfmsubadd132pd {{.*#+}} ymm0 = (ymm0 * mem) -/+ ymm1 sched: [12:0.50]
-; ZNVER1-NEXT:    vfmsubadd213pd {{.*#+}} ymm0 = (ymm1 * ymm0) -/+ mem sched: [12:0.50]
-; ZNVER1-NEXT:    vfmsubadd231pd {{.*#+}} ymm0 = (ymm1 * mem) -/+ ymm0 sched: [12:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    vzeroupper # sched: [100:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  tail call void asm "vfmsubadd132pd $2, $1, $0 \0A\09 vfmsubadd213pd $2, $1, $0 \0A\09 vfmsubadd231pd $2, $1, $0 \0A\09 vfmsubadd132pd $3, $1, $0 \0A\09 vfmsubadd213pd $3, $1, $0 \0A\09 vfmsubadd231pd $3, $1, $0", "x,x,x,*m"(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2, <4 x double> *%a3) nounwind
-  ret void
-}
-
-define void @test_vfmsubaddps_128(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, <4 x float> *%a3) optsize {
-; GENERIC-LABEL: test_vfmsubaddps_128:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    vfmsubadd132ps {{.*#+}} xmm0 = (xmm0 * xmm2) -/+ xmm1 sched: [5:0.50]
-; GENERIC-NEXT:    vfmsubadd213ps {{.*#+}} xmm0 = (xmm1 * xmm0) -/+ xmm2 sched: [5:0.50]
-; GENERIC-NEXT:    vfmsubadd231ps {{.*#+}} xmm0 = (xmm1 * xmm2) -/+ xmm0 sched: [5:0.50]
-; GENERIC-NEXT:    vfmsubadd132ps {{.*#+}} xmm0 = (xmm0 * mem) -/+ xmm1 sched: [10:0.50]
-; GENERIC-NEXT:    vfmsubadd213ps {{.*#+}} xmm0 = (xmm1 * xmm0) -/+ mem sched: [10:0.50]
-; GENERIC-NEXT:    vfmsubadd231ps {{.*#+}} xmm0 = (xmm1 * mem) -/+ xmm0 sched: [10:0.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER2-LABEL: test_vfmsubaddps_128:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    vfmsubadd132ps {{.*#+}} xmm0 = (xmm0 * xmm2) -/+ xmm1 sched: [5:0.50]
-; BDVER2-NEXT:    vfmsubadd213ps {{.*#+}} xmm0 = (xmm1 * xmm0) -/+ xmm2 sched: [5:0.50]
-; BDVER2-NEXT:    vfmsubadd231ps {{.*#+}} xmm0 = (xmm1 * xmm2) -/+ xmm0 sched: [5:0.50]
-; BDVER2-NEXT:    vfmsubadd132ps {{.*#+}} xmm0 = (xmm0 * mem) -/+ xmm1 sched: [10:0.50]
-; BDVER2-NEXT:    vfmsubadd213ps {{.*#+}} xmm0 = (xmm1 * xmm0) -/+ mem sched: [10:0.50]
-; BDVER2-NEXT:    vfmsubadd231ps {{.*#+}} xmm0 = (xmm1 * mem) -/+ xmm0 sched: [10:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; HASWELL-LABEL: test_vfmsubaddps_128:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    vfmsubadd132ps {{.*#+}} xmm0 = (xmm0 * xmm2) -/+ xmm1 sched: [5:0.50]
-; HASWELL-NEXT:    vfmsubadd213ps {{.*#+}} xmm0 = (xmm1 * xmm0) -/+ xmm2 sched: [5:0.50]
-; HASWELL-NEXT:    vfmsubadd231ps {{.*#+}} xmm0 = (xmm1 * xmm2) -/+ xmm0 sched: [5:0.50]
-; HASWELL-NEXT:    vfmsubadd132ps {{.*#+}} xmm0 = (xmm0 * mem) -/+ xmm1 sched: [11:0.50]
-; HASWELL-NEXT:    vfmsubadd213ps {{.*#+}} xmm0 = (xmm1 * xmm0) -/+ mem sched: [11:0.50]
-; HASWELL-NEXT:    vfmsubadd231ps {{.*#+}} xmm0 = (xmm1 * mem) -/+ xmm0 sched: [11:0.50]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_vfmsubaddps_128:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    vfmsubadd132ps {{.*#+}} xmm0 = (xmm0 * xmm2) -/+ xmm1 sched: [5:0.50]
-; BROADWELL-NEXT:    vfmsubadd213ps {{.*#+}} xmm0 = (xmm1 * xmm0) -/+ xmm2 sched: [5:0.50]
-; BROADWELL-NEXT:    vfmsubadd231ps {{.*#+}} xmm0 = (xmm1 * xmm2) -/+ xmm0 sched: [5:0.50]
-; BROADWELL-NEXT:    vfmsubadd132ps {{.*#+}} xmm0 = (xmm0 * mem) -/+ xmm1 sched: [10:0.50]
-; BROADWELL-NEXT:    vfmsubadd213ps {{.*#+}} xmm0 = (xmm1 * xmm0) -/+ mem sched: [10:0.50]
-; BROADWELL-NEXT:    vfmsubadd231ps {{.*#+}} xmm0 = (xmm1 * mem) -/+ xmm0 sched: [10:0.50]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_vfmsubaddps_128:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    vfmsubadd132ps {{.*#+}} xmm0 = (xmm0 * xmm2) -/+ xmm1 sched: [4:0.50]
-; SKYLAKE-NEXT:    vfmsubadd213ps {{.*#+}} xmm0 = (xmm1 * xmm0) -/+ xmm2 sched: [4:0.50]
-; SKYLAKE-NEXT:    vfmsubadd231ps {{.*#+}} xmm0 = (xmm1 * xmm2) -/+ xmm0 sched: [4:0.50]
-; SKYLAKE-NEXT:    vfmsubadd132ps {{.*#+}} xmm0 = (xmm0 * mem) -/+ xmm1 sched: [10:0.50]
-; SKYLAKE-NEXT:    vfmsubadd213ps {{.*#+}} xmm0 = (xmm1 * xmm0) -/+ mem sched: [10:0.50]
-; SKYLAKE-NEXT:    vfmsubadd231ps {{.*#+}} xmm0 = (xmm1 * mem) -/+ xmm0 sched: [10:0.50]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; KNL-LABEL: test_vfmsubaddps_128:
-; KNL:       # %bb.0:
-; KNL-NEXT:    #APP
-; KNL-NEXT:    vfmsubadd132ps {{.*#+}} xmm0 = (xmm0 * xmm2) -/+ xmm1 sched: [5:0.50]
-; KNL-NEXT:    vfmsubadd213ps {{.*#+}} xmm0 = (xmm1 * xmm0) -/+ xmm2 sched: [5:0.50]
-; KNL-NEXT:    vfmsubadd231ps {{.*#+}} xmm0 = (xmm1 * xmm2) -/+ xmm0 sched: [5:0.50]
-; KNL-NEXT:    vfmsubadd132ps {{.*#+}} xmm0 = (xmm0 * mem) -/+ xmm1 sched: [11:0.50]
-; KNL-NEXT:    vfmsubadd213ps {{.*#+}} xmm0 = (xmm1 * xmm0) -/+ mem sched: [11:0.50]
-; KNL-NEXT:    vfmsubadd231ps {{.*#+}} xmm0 = (xmm1 * mem) -/+ xmm0 sched: [11:0.50]
-; KNL-NEXT:    #NO_APP
-; KNL-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_vfmsubaddps_128:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    vfmsubadd132ps {{.*#+}} xmm0 = (xmm0 * xmm2) -/+ xmm1 sched: [4:0.50]
-; SKX-NEXT:    vfmsubadd213ps {{.*#+}} xmm0 = (xmm1 * xmm0) -/+ xmm2 sched: [4:0.50]
-; SKX-NEXT:    vfmsubadd231ps {{.*#+}} xmm0 = (xmm1 * xmm2) -/+ xmm0 sched: [4:0.50]
-; SKX-NEXT:    vfmsubadd132ps {{.*#+}} xmm0 = (xmm0 * mem) -/+ xmm1 sched: [10:0.50]
-; SKX-NEXT:    vfmsubadd213ps {{.*#+}} xmm0 = (xmm1 * xmm0) -/+ mem sched: [10:0.50]
-; SKX-NEXT:    vfmsubadd231ps {{.*#+}} xmm0 = (xmm1 * mem) -/+ xmm0 sched: [10:0.50]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_vfmsubaddps_128:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    vfmsubadd132ps {{.*#+}} xmm0 = (xmm0 * xmm2) -/+ xmm1 sched: [5:0.50]
-; ZNVER1-NEXT:    vfmsubadd213ps {{.*#+}} xmm0 = (xmm1 * xmm0) -/+ xmm2 sched: [5:0.50]
-; ZNVER1-NEXT:    vfmsubadd231ps {{.*#+}} xmm0 = (xmm1 * xmm2) -/+ xmm0 sched: [5:0.50]
-; ZNVER1-NEXT:    vfmsubadd132ps {{.*#+}} xmm0 = (xmm0 * mem) -/+ xmm1 sched: [12:0.50]
-; ZNVER1-NEXT:    vfmsubadd213ps {{.*#+}} xmm0 = (xmm1 * xmm0) -/+ mem sched: [12:0.50]
-; ZNVER1-NEXT:    vfmsubadd231ps {{.*#+}} xmm0 = (xmm1 * mem) -/+ xmm0 sched: [12:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  tail call void asm "vfmsubadd132ps $2, $1, $0 \0A\09 vfmsubadd213ps $2, $1, $0 \0A\09 vfmsubadd231ps $2, $1, $0 \0A\09 vfmsubadd132ps $3, $1, $0 \0A\09 vfmsubadd213ps $3, $1, $0 \0A\09 vfmsubadd231ps $3, $1, $0", "x,x,x,*m"(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, <4 x float> *%a3) nounwind
-  ret void
-}
-
-define void @test_vfmsubaddps_256(<8 x float> %a0, <8 x float> %a1, <8 x float> %a2, <8 x float> *%a3) optsize {
-; GENERIC-LABEL: test_vfmsubaddps_256:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    vfmsubadd132ps {{.*#+}} ymm0 = (ymm0 * ymm2) -/+ ymm1 sched: [5:0.50]
-; GENERIC-NEXT:    vfmsubadd213ps {{.*#+}} ymm0 = (ymm1 * ymm0) -/+ ymm2 sched: [5:0.50]
-; GENERIC-NEXT:    vfmsubadd231ps {{.*#+}} ymm0 = (ymm1 * ymm2) -/+ ymm0 sched: [5:0.50]
-; GENERIC-NEXT:    vfmsubadd132ps {{.*#+}} ymm0 = (ymm0 * mem) -/+ ymm1 sched: [10:0.50]
-; GENERIC-NEXT:    vfmsubadd213ps {{.*#+}} ymm0 = (ymm1 * ymm0) -/+ mem sched: [10:0.50]
-; GENERIC-NEXT:    vfmsubadd231ps {{.*#+}} ymm0 = (ymm1 * mem) -/+ ymm0 sched: [10:0.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    vzeroupper # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER2-LABEL: test_vfmsubaddps_256:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    vfmsubadd132ps {{.*#+}} ymm0 = (ymm0 * ymm2) -/+ ymm1 sched: [5:0.50]
-; BDVER2-NEXT:    vfmsubadd213ps {{.*#+}} ymm0 = (ymm1 * ymm0) -/+ ymm2 sched: [5:0.50]
-; BDVER2-NEXT:    vfmsubadd231ps {{.*#+}} ymm0 = (ymm1 * ymm2) -/+ ymm0 sched: [5:0.50]
-; BDVER2-NEXT:    vfmsubadd132ps {{.*#+}} ymm0 = (ymm0 * mem) -/+ ymm1 sched: [10:1.00]
-; BDVER2-NEXT:    vfmsubadd213ps {{.*#+}} ymm0 = (ymm1 * ymm0) -/+ mem sched: [10:1.00]
-; BDVER2-NEXT:    vfmsubadd231ps {{.*#+}} ymm0 = (ymm1 * mem) -/+ ymm0 sched: [10:1.00]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    vzeroupper # sched: [46:4.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; HASWELL-LABEL: test_vfmsubaddps_256:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    vfmsubadd132ps {{.*#+}} ymm0 = (ymm0 * ymm2) -/+ ymm1 sched: [5:0.50]
-; HASWELL-NEXT:    vfmsubadd213ps {{.*#+}} ymm0 = (ymm1 * ymm0) -/+ ymm2 sched: [5:0.50]
-; HASWELL-NEXT:    vfmsubadd231ps {{.*#+}} ymm0 = (ymm1 * ymm2) -/+ ymm0 sched: [5:0.50]
-; HASWELL-NEXT:    vfmsubadd132ps {{.*#+}} ymm0 = (ymm0 * mem) -/+ ymm1 sched: [12:0.50]
-; HASWELL-NEXT:    vfmsubadd213ps {{.*#+}} ymm0 = (ymm1 * ymm0) -/+ mem sched: [12:0.50]
-; HASWELL-NEXT:    vfmsubadd231ps {{.*#+}} ymm0 = (ymm1 * mem) -/+ ymm0 sched: [12:0.50]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    vzeroupper # sched: [0:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_vfmsubaddps_256:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    vfmsubadd132ps {{.*#+}} ymm0 = (ymm0 * ymm2) -/+ ymm1 sched: [5:0.50]
-; BROADWELL-NEXT:    vfmsubadd213ps {{.*#+}} ymm0 = (ymm1 * ymm0) -/+ ymm2 sched: [5:0.50]
-; BROADWELL-NEXT:    vfmsubadd231ps {{.*#+}} ymm0 = (ymm1 * ymm2) -/+ ymm0 sched: [5:0.50]
-; BROADWELL-NEXT:    vfmsubadd132ps {{.*#+}} ymm0 = (ymm0 * mem) -/+ ymm1 sched: [11:0.50]
-; BROADWELL-NEXT:    vfmsubadd213ps {{.*#+}} ymm0 = (ymm1 * ymm0) -/+ mem sched: [11:0.50]
-; BROADWELL-NEXT:    vfmsubadd231ps {{.*#+}} ymm0 = (ymm1 * mem) -/+ ymm0 sched: [11:0.50]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    vzeroupper # sched: [0:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_vfmsubaddps_256:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    vfmsubadd132ps {{.*#+}} ymm0 = (ymm0 * ymm2) -/+ ymm1 sched: [4:0.50]
-; SKYLAKE-NEXT:    vfmsubadd213ps {{.*#+}} ymm0 = (ymm1 * ymm0) -/+ ymm2 sched: [4:0.50]
-; SKYLAKE-NEXT:    vfmsubadd231ps {{.*#+}} ymm0 = (ymm1 * ymm2) -/+ ymm0 sched: [4:0.50]
-; SKYLAKE-NEXT:    vfmsubadd132ps {{.*#+}} ymm0 = (ymm0 * mem) -/+ ymm1 sched: [11:0.50]
-; SKYLAKE-NEXT:    vfmsubadd213ps {{.*#+}} ymm0 = (ymm1 * ymm0) -/+ mem sched: [11:0.50]
-; SKYLAKE-NEXT:    vfmsubadd231ps {{.*#+}} ymm0 = (ymm1 * mem) -/+ ymm0 sched: [11:0.50]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    vzeroupper # sched: [0:0.67]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; KNL-LABEL: test_vfmsubaddps_256:
-; KNL:       # %bb.0:
-; KNL-NEXT:    #APP
-; KNL-NEXT:    vfmsubadd132ps {{.*#+}} ymm0 = (ymm0 * ymm2) -/+ ymm1 sched: [5:0.50]
-; KNL-NEXT:    vfmsubadd213ps {{.*#+}} ymm0 = (ymm1 * ymm0) -/+ ymm2 sched: [5:0.50]
-; KNL-NEXT:    vfmsubadd231ps {{.*#+}} ymm0 = (ymm1 * ymm2) -/+ ymm0 sched: [5:0.50]
-; KNL-NEXT:    vfmsubadd132ps {{.*#+}} ymm0 = (ymm0 * mem) -/+ ymm1 sched: [12:0.50]
-; KNL-NEXT:    vfmsubadd213ps {{.*#+}} ymm0 = (ymm1 * ymm0) -/+ mem sched: [12:0.50]
-; KNL-NEXT:    vfmsubadd231ps {{.*#+}} ymm0 = (ymm1 * mem) -/+ ymm0 sched: [12:0.50]
-; KNL-NEXT:    #NO_APP
-; KNL-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_vfmsubaddps_256:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    vfmsubadd132ps {{.*#+}} ymm0 = (ymm0 * ymm2) -/+ ymm1 sched: [4:0.50]
-; SKX-NEXT:    vfmsubadd213ps {{.*#+}} ymm0 = (ymm1 * ymm0) -/+ ymm2 sched: [4:0.50]
-; SKX-NEXT:    vfmsubadd231ps {{.*#+}} ymm0 = (ymm1 * ymm2) -/+ ymm0 sched: [4:0.50]
-; SKX-NEXT:    vfmsubadd132ps {{.*#+}} ymm0 = (ymm0 * mem) -/+ ymm1 sched: [11:0.50]
-; SKX-NEXT:    vfmsubadd213ps {{.*#+}} ymm0 = (ymm1 * ymm0) -/+ mem sched: [11:0.50]
-; SKX-NEXT:    vfmsubadd231ps {{.*#+}} ymm0 = (ymm1 * mem) -/+ ymm0 sched: [11:0.50]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    vzeroupper # sched: [0:0.67]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_vfmsubaddps_256:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    vfmsubadd132ps {{.*#+}} ymm0 = (ymm0 * ymm2) -/+ ymm1 sched: [5:0.50]
-; ZNVER1-NEXT:    vfmsubadd213ps {{.*#+}} ymm0 = (ymm1 * ymm0) -/+ ymm2 sched: [5:0.50]
-; ZNVER1-NEXT:    vfmsubadd231ps {{.*#+}} ymm0 = (ymm1 * ymm2) -/+ ymm0 sched: [5:0.50]
-; ZNVER1-NEXT:    vfmsubadd132ps {{.*#+}} ymm0 = (ymm0 * mem) -/+ ymm1 sched: [12:0.50]
-; ZNVER1-NEXT:    vfmsubadd213ps {{.*#+}} ymm0 = (ymm1 * ymm0) -/+ mem sched: [12:0.50]
-; ZNVER1-NEXT:    vfmsubadd231ps {{.*#+}} ymm0 = (ymm1 * mem) -/+ ymm0 sched: [12:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    vzeroupper # sched: [100:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  tail call void asm "vfmsubadd132ps $2, $1, $0 \0A\09 vfmsubadd213ps $2, $1, $0 \0A\09 vfmsubadd231ps $2, $1, $0 \0A\09 vfmsubadd132ps $3, $1, $0 \0A\09 vfmsubadd213ps $3, $1, $0 \0A\09 vfmsubadd231ps $3, $1, $0", "x,x,x,*m"(<8 x float> %a0, <8 x float> %a1, <8 x float> %a2, <8 x float> *%a3) nounwind
-  ret void
-}
-
-;
-; VFMSUB
-;
-
-define void @test_vfmsubpd_128(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, <2 x double> *%a3) optsize {
-; GENERIC-LABEL: test_vfmsubpd_128:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    vfmsub132pd {{.*#+}} xmm0 = (xmm0 * xmm2) - xmm1 sched: [5:0.50]
-; GENERIC-NEXT:    vfmsub213pd {{.*#+}} xmm0 = (xmm1 * xmm0) - xmm2 sched: [5:0.50]
-; GENERIC-NEXT:    vfmsub231pd {{.*#+}} xmm0 = (xmm1 * xmm2) - xmm0 sched: [5:0.50]
-; GENERIC-NEXT:    vfmsub132pd {{.*#+}} xmm0 = (xmm0 * mem) - xmm1 sched: [10:0.50]
-; GENERIC-NEXT:    vfmsub213pd {{.*#+}} xmm0 = (xmm1 * xmm0) - mem sched: [10:0.50]
-; GENERIC-NEXT:    vfmsub231pd {{.*#+}} xmm0 = (xmm1 * mem) - xmm0 sched: [10:0.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER2-LABEL: test_vfmsubpd_128:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    vfmsub132pd {{.*#+}} xmm0 = (xmm0 * xmm2) - xmm1 sched: [5:0.50]
-; BDVER2-NEXT:    vfmsub213pd {{.*#+}} xmm0 = (xmm1 * xmm0) - xmm2 sched: [5:0.50]
-; BDVER2-NEXT:    vfmsub231pd {{.*#+}} xmm0 = (xmm1 * xmm2) - xmm0 sched: [5:0.50]
-; BDVER2-NEXT:    vfmsub132pd {{.*#+}} xmm0 = (xmm0 * mem) - xmm1 sched: [10:0.50]
-; BDVER2-NEXT:    vfmsub213pd {{.*#+}} xmm0 = (xmm1 * xmm0) - mem sched: [10:0.50]
-; BDVER2-NEXT:    vfmsub231pd {{.*#+}} xmm0 = (xmm1 * mem) - xmm0 sched: [10:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; HASWELL-LABEL: test_vfmsubpd_128:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    vfmsub132pd {{.*#+}} xmm0 = (xmm0 * xmm2) - xmm1 sched: [5:0.50]
-; HASWELL-NEXT:    vfmsub213pd {{.*#+}} xmm0 = (xmm1 * xmm0) - xmm2 sched: [5:0.50]
-; HASWELL-NEXT:    vfmsub231pd {{.*#+}} xmm0 = (xmm1 * xmm2) - xmm0 sched: [5:0.50]
-; HASWELL-NEXT:    vfmsub132pd {{.*#+}} xmm0 = (xmm0 * mem) - xmm1 sched: [11:0.50]
-; HASWELL-NEXT:    vfmsub213pd {{.*#+}} xmm0 = (xmm1 * xmm0) - mem sched: [11:0.50]
-; HASWELL-NEXT:    vfmsub231pd {{.*#+}} xmm0 = (xmm1 * mem) - xmm0 sched: [11:0.50]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_vfmsubpd_128:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    vfmsub132pd {{.*#+}} xmm0 = (xmm0 * xmm2) - xmm1 sched: [5:0.50]
-; BROADWELL-NEXT:    vfmsub213pd {{.*#+}} xmm0 = (xmm1 * xmm0) - xmm2 sched: [5:0.50]
-; BROADWELL-NEXT:    vfmsub231pd {{.*#+}} xmm0 = (xmm1 * xmm2) - xmm0 sched: [5:0.50]
-; BROADWELL-NEXT:    vfmsub132pd {{.*#+}} xmm0 = (xmm0 * mem) - xmm1 sched: [10:0.50]
-; BROADWELL-NEXT:    vfmsub213pd {{.*#+}} xmm0 = (xmm1 * xmm0) - mem sched: [10:0.50]
-; BROADWELL-NEXT:    vfmsub231pd {{.*#+}} xmm0 = (xmm1 * mem) - xmm0 sched: [10:0.50]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_vfmsubpd_128:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    vfmsub132pd {{.*#+}} xmm0 = (xmm0 * xmm2) - xmm1 sched: [4:0.50]
-; SKYLAKE-NEXT:    vfmsub213pd {{.*#+}} xmm0 = (xmm1 * xmm0) - xmm2 sched: [4:0.50]
-; SKYLAKE-NEXT:    vfmsub231pd {{.*#+}} xmm0 = (xmm1 * xmm2) - xmm0 sched: [4:0.50]
-; SKYLAKE-NEXT:    vfmsub132pd {{.*#+}} xmm0 = (xmm0 * mem) - xmm1 sched: [10:0.50]
-; SKYLAKE-NEXT:    vfmsub213pd {{.*#+}} xmm0 = (xmm1 * xmm0) - mem sched: [10:0.50]
-; SKYLAKE-NEXT:    vfmsub231pd {{.*#+}} xmm0 = (xmm1 * mem) - xmm0 sched: [10:0.50]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; KNL-LABEL: test_vfmsubpd_128:
-; KNL:       # %bb.0:
-; KNL-NEXT:    #APP
-; KNL-NEXT:    vfmsub132pd {{.*#+}} xmm0 = (xmm0 * xmm2) - xmm1 sched: [5:0.50]
-; KNL-NEXT:    vfmsub213pd {{.*#+}} xmm0 = (xmm1 * xmm0) - xmm2 sched: [5:0.50]
-; KNL-NEXT:    vfmsub231pd {{.*#+}} xmm0 = (xmm1 * xmm2) - xmm0 sched: [5:0.50]
-; KNL-NEXT:    vfmsub132pd {{.*#+}} xmm0 = (xmm0 * mem) - xmm1 sched: [11:0.50]
-; KNL-NEXT:    vfmsub213pd {{.*#+}} xmm0 = (xmm1 * xmm0) - mem sched: [11:0.50]
-; KNL-NEXT:    vfmsub231pd {{.*#+}} xmm0 = (xmm1 * mem) - xmm0 sched: [11:0.50]
-; KNL-NEXT:    #NO_APP
-; KNL-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_vfmsubpd_128:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    vfmsub132pd {{.*#+}} xmm0 = (xmm0 * xmm2) - xmm1 sched: [4:0.50]
-; SKX-NEXT:    vfmsub213pd {{.*#+}} xmm0 = (xmm1 * xmm0) - xmm2 sched: [4:0.50]
-; SKX-NEXT:    vfmsub231pd {{.*#+}} xmm0 = (xmm1 * xmm2) - xmm0 sched: [4:0.50]
-; SKX-NEXT:    vfmsub132pd {{.*#+}} xmm0 = (xmm0 * mem) - xmm1 sched: [10:0.50]
-; SKX-NEXT:    vfmsub213pd {{.*#+}} xmm0 = (xmm1 * xmm0) - mem sched: [10:0.50]
-; SKX-NEXT:    vfmsub231pd {{.*#+}} xmm0 = (xmm1 * mem) - xmm0 sched: [10:0.50]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_vfmsubpd_128:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    vfmsub132pd {{.*#+}} xmm0 = (xmm0 * xmm2) - xmm1 sched: [5:0.50]
-; ZNVER1-NEXT:    vfmsub213pd {{.*#+}} xmm0 = (xmm1 * xmm0) - xmm2 sched: [5:0.50]
-; ZNVER1-NEXT:    vfmsub231pd {{.*#+}} xmm0 = (xmm1 * xmm2) - xmm0 sched: [5:0.50]
-; ZNVER1-NEXT:    vfmsub132pd {{.*#+}} xmm0 = (xmm0 * mem) - xmm1 sched: [12:0.50]
-; ZNVER1-NEXT:    vfmsub213pd {{.*#+}} xmm0 = (xmm1 * xmm0) - mem sched: [12:0.50]
-; ZNVER1-NEXT:    vfmsub231pd {{.*#+}} xmm0 = (xmm1 * mem) - xmm0 sched: [12:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  tail call void asm "vfmsub132pd $2, $1, $0 \0A\09 vfmsub213pd $2, $1, $0 \0A\09 vfmsub231pd $2, $1, $0 \0A\09 vfmsub132pd $3, $1, $0 \0A\09 vfmsub213pd $3, $1, $0 \0A\09 vfmsub231pd $3, $1, $0", "x,x,x,*m"(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, <2 x double> *%a3) nounwind
-  ret void
-}
-
-define void @test_vfmsubpd_256(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2, <4 x double> *%a3) optsize {
-; GENERIC-LABEL: test_vfmsubpd_256:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    vfmsub132pd {{.*#+}} ymm0 = (ymm0 * ymm2) - ymm1 sched: [5:0.50]
-; GENERIC-NEXT:    vfmsub213pd {{.*#+}} ymm0 = (ymm1 * ymm0) - ymm2 sched: [5:0.50]
-; GENERIC-NEXT:    vfmsub231pd {{.*#+}} ymm0 = (ymm1 * ymm2) - ymm0 sched: [5:0.50]
-; GENERIC-NEXT:    vfmsub132pd {{.*#+}} ymm0 = (ymm0 * mem) - ymm1 sched: [10:0.50]
-; GENERIC-NEXT:    vfmsub213pd {{.*#+}} ymm0 = (ymm1 * ymm0) - mem sched: [10:0.50]
-; GENERIC-NEXT:    vfmsub231pd {{.*#+}} ymm0 = (ymm1 * mem) - ymm0 sched: [10:0.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    vzeroupper # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER2-LABEL: test_vfmsubpd_256:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    vfmsub132pd {{.*#+}} ymm0 = (ymm0 * ymm2) - ymm1 sched: [5:0.50]
-; BDVER2-NEXT:    vfmsub213pd {{.*#+}} ymm0 = (ymm1 * ymm0) - ymm2 sched: [5:0.50]
-; BDVER2-NEXT:    vfmsub231pd {{.*#+}} ymm0 = (ymm1 * ymm2) - ymm0 sched: [5:0.50]
-; BDVER2-NEXT:    vfmsub132pd {{.*#+}} ymm0 = (ymm0 * mem) - ymm1 sched: [10:1.00]
-; BDVER2-NEXT:    vfmsub213pd {{.*#+}} ymm0 = (ymm1 * ymm0) - mem sched: [10:1.00]
-; BDVER2-NEXT:    vfmsub231pd {{.*#+}} ymm0 = (ymm1 * mem) - ymm0 sched: [10:1.00]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    vzeroupper # sched: [46:4.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; HASWELL-LABEL: test_vfmsubpd_256:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    vfmsub132pd {{.*#+}} ymm0 = (ymm0 * ymm2) - ymm1 sched: [5:0.50]
-; HASWELL-NEXT:    vfmsub213pd {{.*#+}} ymm0 = (ymm1 * ymm0) - ymm2 sched: [5:0.50]
-; HASWELL-NEXT:    vfmsub231pd {{.*#+}} ymm0 = (ymm1 * ymm2) - ymm0 sched: [5:0.50]
-; HASWELL-NEXT:    vfmsub132pd {{.*#+}} ymm0 = (ymm0 * mem) - ymm1 sched: [12:0.50]
-; HASWELL-NEXT:    vfmsub213pd {{.*#+}} ymm0 = (ymm1 * ymm0) - mem sched: [12:0.50]
-; HASWELL-NEXT:    vfmsub231pd {{.*#+}} ymm0 = (ymm1 * mem) - ymm0 sched: [12:0.50]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    vzeroupper # sched: [0:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_vfmsubpd_256:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    vfmsub132pd {{.*#+}} ymm0 = (ymm0 * ymm2) - ymm1 sched: [5:0.50]
-; BROADWELL-NEXT:    vfmsub213pd {{.*#+}} ymm0 = (ymm1 * ymm0) - ymm2 sched: [5:0.50]
-; BROADWELL-NEXT:    vfmsub231pd {{.*#+}} ymm0 = (ymm1 * ymm2) - ymm0 sched: [5:0.50]
-; BROADWELL-NEXT:    vfmsub132pd {{.*#+}} ymm0 = (ymm0 * mem) - ymm1 sched: [11:0.50]
-; BROADWELL-NEXT:    vfmsub213pd {{.*#+}} ymm0 = (ymm1 * ymm0) - mem sched: [11:0.50]
-; BROADWELL-NEXT:    vfmsub231pd {{.*#+}} ymm0 = (ymm1 * mem) - ymm0 sched: [11:0.50]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    vzeroupper # sched: [0:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_vfmsubpd_256:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    vfmsub132pd {{.*#+}} ymm0 = (ymm0 * ymm2) - ymm1 sched: [4:0.50]
-; SKYLAKE-NEXT:    vfmsub213pd {{.*#+}} ymm0 = (ymm1 * ymm0) - ymm2 sched: [4:0.50]
-; SKYLAKE-NEXT:    vfmsub231pd {{.*#+}} ymm0 = (ymm1 * ymm2) - ymm0 sched: [4:0.50]
-; SKYLAKE-NEXT:    vfmsub132pd {{.*#+}} ymm0 = (ymm0 * mem) - ymm1 sched: [11:0.50]
-; SKYLAKE-NEXT:    vfmsub213pd {{.*#+}} ymm0 = (ymm1 * ymm0) - mem sched: [11:0.50]
-; SKYLAKE-NEXT:    vfmsub231pd {{.*#+}} ymm0 = (ymm1 * mem) - ymm0 sched: [11:0.50]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    vzeroupper # sched: [0:0.67]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; KNL-LABEL: test_vfmsubpd_256:
-; KNL:       # %bb.0:
-; KNL-NEXT:    #APP
-; KNL-NEXT:    vfmsub132pd {{.*#+}} ymm0 = (ymm0 * ymm2) - ymm1 sched: [5:0.50]
-; KNL-NEXT:    vfmsub213pd {{.*#+}} ymm0 = (ymm1 * ymm0) - ymm2 sched: [5:0.50]
-; KNL-NEXT:    vfmsub231pd {{.*#+}} ymm0 = (ymm1 * ymm2) - ymm0 sched: [5:0.50]
-; KNL-NEXT:    vfmsub132pd {{.*#+}} ymm0 = (ymm0 * mem) - ymm1 sched: [12:0.50]
-; KNL-NEXT:    vfmsub213pd {{.*#+}} ymm0 = (ymm1 * ymm0) - mem sched: [12:0.50]
-; KNL-NEXT:    vfmsub231pd {{.*#+}} ymm0 = (ymm1 * mem) - ymm0 sched: [12:0.50]
-; KNL-NEXT:    #NO_APP
-; KNL-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_vfmsubpd_256:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    vfmsub132pd {{.*#+}} ymm0 = (ymm0 * ymm2) - ymm1 sched: [4:0.50]
-; SKX-NEXT:    vfmsub213pd {{.*#+}} ymm0 = (ymm1 * ymm0) - ymm2 sched: [4:0.50]
-; SKX-NEXT:    vfmsub231pd {{.*#+}} ymm0 = (ymm1 * ymm2) - ymm0 sched: [4:0.50]
-; SKX-NEXT:    vfmsub132pd {{.*#+}} ymm0 = (ymm0 * mem) - ymm1 sched: [11:0.50]
-; SKX-NEXT:    vfmsub213pd {{.*#+}} ymm0 = (ymm1 * ymm0) - mem sched: [11:0.50]
-; SKX-NEXT:    vfmsub231pd {{.*#+}} ymm0 = (ymm1 * mem) - ymm0 sched: [11:0.50]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    vzeroupper # sched: [0:0.67]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_vfmsubpd_256:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    vfmsub132pd {{.*#+}} ymm0 = (ymm0 * ymm2) - ymm1 sched: [5:0.50]
-; ZNVER1-NEXT:    vfmsub213pd {{.*#+}} ymm0 = (ymm1 * ymm0) - ymm2 sched: [5:0.50]
-; ZNVER1-NEXT:    vfmsub231pd {{.*#+}} ymm0 = (ymm1 * ymm2) - ymm0 sched: [5:0.50]
-; ZNVER1-NEXT:    vfmsub132pd {{.*#+}} ymm0 = (ymm0 * mem) - ymm1 sched: [12:0.50]
-; ZNVER1-NEXT:    vfmsub213pd {{.*#+}} ymm0 = (ymm1 * ymm0) - mem sched: [12:0.50]
-; ZNVER1-NEXT:    vfmsub231pd {{.*#+}} ymm0 = (ymm1 * mem) - ymm0 sched: [12:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    vzeroupper # sched: [100:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  tail call void asm "vfmsub132pd $2, $1, $0 \0A\09 vfmsub213pd $2, $1, $0 \0A\09 vfmsub231pd $2, $1, $0 \0A\09 vfmsub132pd $3, $1, $0 \0A\09 vfmsub213pd $3, $1, $0 \0A\09 vfmsub231pd $3, $1, $0", "x,x,x,*m"(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2, <4 x double> *%a3) nounwind
-  ret void
-}
-
-define void @test_vfmsubps_128(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, <4 x float> *%a3) optsize {
-; GENERIC-LABEL: test_vfmsubps_128:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    vfmsub132ps {{.*#+}} xmm0 = (xmm0 * xmm2) - xmm1 sched: [5:0.50]
-; GENERIC-NEXT:    vfmsub213ps {{.*#+}} xmm0 = (xmm1 * xmm0) - xmm2 sched: [5:0.50]
-; GENERIC-NEXT:    vfmsub231ps {{.*#+}} xmm0 = (xmm1 * xmm2) - xmm0 sched: [5:0.50]
-; GENERIC-NEXT:    vfmsub132ps {{.*#+}} xmm0 = (xmm0 * mem) - xmm1 sched: [10:0.50]
-; GENERIC-NEXT:    vfmsub213ps {{.*#+}} xmm0 = (xmm1 * xmm0) - mem sched: [10:0.50]
-; GENERIC-NEXT:    vfmsub231ps {{.*#+}} xmm0 = (xmm1 * mem) - xmm0 sched: [10:0.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER2-LABEL: test_vfmsubps_128:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    vfmsub132ps {{.*#+}} xmm0 = (xmm0 * xmm2) - xmm1 sched: [5:0.50]
-; BDVER2-NEXT:    vfmsub213ps {{.*#+}} xmm0 = (xmm1 * xmm0) - xmm2 sched: [5:0.50]
-; BDVER2-NEXT:    vfmsub231ps {{.*#+}} xmm0 = (xmm1 * xmm2) - xmm0 sched: [5:0.50]
-; BDVER2-NEXT:    vfmsub132ps {{.*#+}} xmm0 = (xmm0 * mem) - xmm1 sched: [10:0.50]
-; BDVER2-NEXT:    vfmsub213ps {{.*#+}} xmm0 = (xmm1 * xmm0) - mem sched: [10:0.50]
-; BDVER2-NEXT:    vfmsub231ps {{.*#+}} xmm0 = (xmm1 * mem) - xmm0 sched: [10:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; HASWELL-LABEL: test_vfmsubps_128:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    vfmsub132ps {{.*#+}} xmm0 = (xmm0 * xmm2) - xmm1 sched: [5:0.50]
-; HASWELL-NEXT:    vfmsub213ps {{.*#+}} xmm0 = (xmm1 * xmm0) - xmm2 sched: [5:0.50]
-; HASWELL-NEXT:    vfmsub231ps {{.*#+}} xmm0 = (xmm1 * xmm2) - xmm0 sched: [5:0.50]
-; HASWELL-NEXT:    vfmsub132ps {{.*#+}} xmm0 = (xmm0 * mem) - xmm1 sched: [11:0.50]
-; HASWELL-NEXT:    vfmsub213ps {{.*#+}} xmm0 = (xmm1 * xmm0) - mem sched: [11:0.50]
-; HASWELL-NEXT:    vfmsub231ps {{.*#+}} xmm0 = (xmm1 * mem) - xmm0 sched: [11:0.50]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_vfmsubps_128:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    vfmsub132ps {{.*#+}} xmm0 = (xmm0 * xmm2) - xmm1 sched: [5:0.50]
-; BROADWELL-NEXT:    vfmsub213ps {{.*#+}} xmm0 = (xmm1 * xmm0) - xmm2 sched: [5:0.50]
-; BROADWELL-NEXT:    vfmsub231ps {{.*#+}} xmm0 = (xmm1 * xmm2) - xmm0 sched: [5:0.50]
-; BROADWELL-NEXT:    vfmsub132ps {{.*#+}} xmm0 = (xmm0 * mem) - xmm1 sched: [10:0.50]
-; BROADWELL-NEXT:    vfmsub213ps {{.*#+}} xmm0 = (xmm1 * xmm0) - mem sched: [10:0.50]
-; BROADWELL-NEXT:    vfmsub231ps {{.*#+}} xmm0 = (xmm1 * mem) - xmm0 sched: [10:0.50]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_vfmsubps_128:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    vfmsub132ps {{.*#+}} xmm0 = (xmm0 * xmm2) - xmm1 sched: [4:0.50]
-; SKYLAKE-NEXT:    vfmsub213ps {{.*#+}} xmm0 = (xmm1 * xmm0) - xmm2 sched: [4:0.50]
-; SKYLAKE-NEXT:    vfmsub231ps {{.*#+}} xmm0 = (xmm1 * xmm2) - xmm0 sched: [4:0.50]
-; SKYLAKE-NEXT:    vfmsub132ps {{.*#+}} xmm0 = (xmm0 * mem) - xmm1 sched: [10:0.50]
-; SKYLAKE-NEXT:    vfmsub213ps {{.*#+}} xmm0 = (xmm1 * xmm0) - mem sched: [10:0.50]
-; SKYLAKE-NEXT:    vfmsub231ps {{.*#+}} xmm0 = (xmm1 * mem) - xmm0 sched: [10:0.50]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; KNL-LABEL: test_vfmsubps_128:
-; KNL:       # %bb.0:
-; KNL-NEXT:    #APP
-; KNL-NEXT:    vfmsub132ps {{.*#+}} xmm0 = (xmm0 * xmm2) - xmm1 sched: [5:0.50]
-; KNL-NEXT:    vfmsub213ps {{.*#+}} xmm0 = (xmm1 * xmm0) - xmm2 sched: [5:0.50]
-; KNL-NEXT:    vfmsub231ps {{.*#+}} xmm0 = (xmm1 * xmm2) - xmm0 sched: [5:0.50]
-; KNL-NEXT:    vfmsub132ps {{.*#+}} xmm0 = (xmm0 * mem) - xmm1 sched: [11:0.50]
-; KNL-NEXT:    vfmsub213ps {{.*#+}} xmm0 = (xmm1 * xmm0) - mem sched: [11:0.50]
-; KNL-NEXT:    vfmsub231ps {{.*#+}} xmm0 = (xmm1 * mem) - xmm0 sched: [11:0.50]
-; KNL-NEXT:    #NO_APP
-; KNL-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_vfmsubps_128:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    vfmsub132ps {{.*#+}} xmm0 = (xmm0 * xmm2) - xmm1 sched: [4:0.50]
-; SKX-NEXT:    vfmsub213ps {{.*#+}} xmm0 = (xmm1 * xmm0) - xmm2 sched: [4:0.50]
-; SKX-NEXT:    vfmsub231ps {{.*#+}} xmm0 = (xmm1 * xmm2) - xmm0 sched: [4:0.50]
-; SKX-NEXT:    vfmsub132ps {{.*#+}} xmm0 = (xmm0 * mem) - xmm1 sched: [10:0.50]
-; SKX-NEXT:    vfmsub213ps {{.*#+}} xmm0 = (xmm1 * xmm0) - mem sched: [10:0.50]
-; SKX-NEXT:    vfmsub231ps {{.*#+}} xmm0 = (xmm1 * mem) - xmm0 sched: [10:0.50]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_vfmsubps_128:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    vfmsub132ps {{.*#+}} xmm0 = (xmm0 * xmm2) - xmm1 sched: [5:0.50]
-; ZNVER1-NEXT:    vfmsub213ps {{.*#+}} xmm0 = (xmm1 * xmm0) - xmm2 sched: [5:0.50]
-; ZNVER1-NEXT:    vfmsub231ps {{.*#+}} xmm0 = (xmm1 * xmm2) - xmm0 sched: [5:0.50]
-; ZNVER1-NEXT:    vfmsub132ps {{.*#+}} xmm0 = (xmm0 * mem) - xmm1 sched: [12:0.50]
-; ZNVER1-NEXT:    vfmsub213ps {{.*#+}} xmm0 = (xmm1 * xmm0) - mem sched: [12:0.50]
-; ZNVER1-NEXT:    vfmsub231ps {{.*#+}} xmm0 = (xmm1 * mem) - xmm0 sched: [12:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  tail call void asm "vfmsub132ps $2, $1, $0 \0A\09 vfmsub213ps $2, $1, $0 \0A\09 vfmsub231ps $2, $1, $0 \0A\09 vfmsub132ps $3, $1, $0 \0A\09 vfmsub213ps $3, $1, $0 \0A\09 vfmsub231ps $3, $1, $0", "x,x,x,*m"(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, <4 x float> *%a3) nounwind
-  ret void
-}
-
-define void @test_vfmsubps_256(<8 x float> %a0, <8 x float> %a1, <8 x float> %a2, <8 x float> *%a3) optsize {
-; GENERIC-LABEL: test_vfmsubps_256:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    vfmsub132ps {{.*#+}} ymm0 = (ymm0 * ymm2) - ymm1 sched: [5:0.50]
-; GENERIC-NEXT:    vfmsub213ps {{.*#+}} ymm0 = (ymm1 * ymm0) - ymm2 sched: [5:0.50]
-; GENERIC-NEXT:    vfmsub231ps {{.*#+}} ymm0 = (ymm1 * ymm2) - ymm0 sched: [5:0.50]
-; GENERIC-NEXT:    vfmsub132ps {{.*#+}} ymm0 = (ymm0 * mem) - ymm1 sched: [10:0.50]
-; GENERIC-NEXT:    vfmsub213ps {{.*#+}} ymm0 = (ymm1 * ymm0) - mem sched: [10:0.50]
-; GENERIC-NEXT:    vfmsub231ps {{.*#+}} ymm0 = (ymm1 * mem) - ymm0 sched: [10:0.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    vzeroupper # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER2-LABEL: test_vfmsubps_256:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    vfmsub132ps {{.*#+}} ymm0 = (ymm0 * ymm2) - ymm1 sched: [5:0.50]
-; BDVER2-NEXT:    vfmsub213ps {{.*#+}} ymm0 = (ymm1 * ymm0) - ymm2 sched: [5:0.50]
-; BDVER2-NEXT:    vfmsub231ps {{.*#+}} ymm0 = (ymm1 * ymm2) - ymm0 sched: [5:0.50]
-; BDVER2-NEXT:    vfmsub132ps {{.*#+}} ymm0 = (ymm0 * mem) - ymm1 sched: [10:1.00]
-; BDVER2-NEXT:    vfmsub213ps {{.*#+}} ymm0 = (ymm1 * ymm0) - mem sched: [10:1.00]
-; BDVER2-NEXT:    vfmsub231ps {{.*#+}} ymm0 = (ymm1 * mem) - ymm0 sched: [10:1.00]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    vzeroupper # sched: [46:4.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; HASWELL-LABEL: test_vfmsubps_256:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    vfmsub132ps {{.*#+}} ymm0 = (ymm0 * ymm2) - ymm1 sched: [5:0.50]
-; HASWELL-NEXT:    vfmsub213ps {{.*#+}} ymm0 = (ymm1 * ymm0) - ymm2 sched: [5:0.50]
-; HASWELL-NEXT:    vfmsub231ps {{.*#+}} ymm0 = (ymm1 * ymm2) - ymm0 sched: [5:0.50]
-; HASWELL-NEXT:    vfmsub132ps {{.*#+}} ymm0 = (ymm0 * mem) - ymm1 sched: [12:0.50]
-; HASWELL-NEXT:    vfmsub213ps {{.*#+}} ymm0 = (ymm1 * ymm0) - mem sched: [12:0.50]
-; HASWELL-NEXT:    vfmsub231ps {{.*#+}} ymm0 = (ymm1 * mem) - ymm0 sched: [12:0.50]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    vzeroupper # sched: [0:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_vfmsubps_256:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    vfmsub132ps {{.*#+}} ymm0 = (ymm0 * ymm2) - ymm1 sched: [5:0.50]
-; BROADWELL-NEXT:    vfmsub213ps {{.*#+}} ymm0 = (ymm1 * ymm0) - ymm2 sched: [5:0.50]
-; BROADWELL-NEXT:    vfmsub231ps {{.*#+}} ymm0 = (ymm1 * ymm2) - ymm0 sched: [5:0.50]
-; BROADWELL-NEXT:    vfmsub132ps {{.*#+}} ymm0 = (ymm0 * mem) - ymm1 sched: [11:0.50]
-; BROADWELL-NEXT:    vfmsub213ps {{.*#+}} ymm0 = (ymm1 * ymm0) - mem sched: [11:0.50]
-; BROADWELL-NEXT:    vfmsub231ps {{.*#+}} ymm0 = (ymm1 * mem) - ymm0 sched: [11:0.50]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    vzeroupper # sched: [0:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_vfmsubps_256:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    vfmsub132ps {{.*#+}} ymm0 = (ymm0 * ymm2) - ymm1 sched: [4:0.50]
-; SKYLAKE-NEXT:    vfmsub213ps {{.*#+}} ymm0 = (ymm1 * ymm0) - ymm2 sched: [4:0.50]
-; SKYLAKE-NEXT:    vfmsub231ps {{.*#+}} ymm0 = (ymm1 * ymm2) - ymm0 sched: [4:0.50]
-; SKYLAKE-NEXT:    vfmsub132ps {{.*#+}} ymm0 = (ymm0 * mem) - ymm1 sched: [11:0.50]
-; SKYLAKE-NEXT:    vfmsub213ps {{.*#+}} ymm0 = (ymm1 * ymm0) - mem sched: [11:0.50]
-; SKYLAKE-NEXT:    vfmsub231ps {{.*#+}} ymm0 = (ymm1 * mem) - ymm0 sched: [11:0.50]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    vzeroupper # sched: [0:0.67]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; KNL-LABEL: test_vfmsubps_256:
-; KNL:       # %bb.0:
-; KNL-NEXT:    #APP
-; KNL-NEXT:    vfmsub132ps {{.*#+}} ymm0 = (ymm0 * ymm2) - ymm1 sched: [5:0.50]
-; KNL-NEXT:    vfmsub213ps {{.*#+}} ymm0 = (ymm1 * ymm0) - ymm2 sched: [5:0.50]
-; KNL-NEXT:    vfmsub231ps {{.*#+}} ymm0 = (ymm1 * ymm2) - ymm0 sched: [5:0.50]
-; KNL-NEXT:    vfmsub132ps {{.*#+}} ymm0 = (ymm0 * mem) - ymm1 sched: [12:0.50]
-; KNL-NEXT:    vfmsub213ps {{.*#+}} ymm0 = (ymm1 * ymm0) - mem sched: [12:0.50]
-; KNL-NEXT:    vfmsub231ps {{.*#+}} ymm0 = (ymm1 * mem) - ymm0 sched: [12:0.50]
-; KNL-NEXT:    #NO_APP
-; KNL-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_vfmsubps_256:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    vfmsub132ps {{.*#+}} ymm0 = (ymm0 * ymm2) - ymm1 sched: [4:0.50]
-; SKX-NEXT:    vfmsub213ps {{.*#+}} ymm0 = (ymm1 * ymm0) - ymm2 sched: [4:0.50]
-; SKX-NEXT:    vfmsub231ps {{.*#+}} ymm0 = (ymm1 * ymm2) - ymm0 sched: [4:0.50]
-; SKX-NEXT:    vfmsub132ps {{.*#+}} ymm0 = (ymm0 * mem) - ymm1 sched: [11:0.50]
-; SKX-NEXT:    vfmsub213ps {{.*#+}} ymm0 = (ymm1 * ymm0) - mem sched: [11:0.50]
-; SKX-NEXT:    vfmsub231ps {{.*#+}} ymm0 = (ymm1 * mem) - ymm0 sched: [11:0.50]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    vzeroupper # sched: [0:0.67]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_vfmsubps_256:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    vfmsub132ps {{.*#+}} ymm0 = (ymm0 * ymm2) - ymm1 sched: [5:0.50]
-; ZNVER1-NEXT:    vfmsub213ps {{.*#+}} ymm0 = (ymm1 * ymm0) - ymm2 sched: [5:0.50]
-; ZNVER1-NEXT:    vfmsub231ps {{.*#+}} ymm0 = (ymm1 * ymm2) - ymm0 sched: [5:0.50]
-; ZNVER1-NEXT:    vfmsub132ps {{.*#+}} ymm0 = (ymm0 * mem) - ymm1 sched: [12:0.50]
-; ZNVER1-NEXT:    vfmsub213ps {{.*#+}} ymm0 = (ymm1 * ymm0) - mem sched: [12:0.50]
-; ZNVER1-NEXT:    vfmsub231ps {{.*#+}} ymm0 = (ymm1 * mem) - ymm0 sched: [12:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    vzeroupper # sched: [100:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  tail call void asm "vfmsub132ps $2, $1, $0 \0A\09 vfmsub213ps $2, $1, $0 \0A\09 vfmsub231ps $2, $1, $0 \0A\09 vfmsub132ps $3, $1, $0 \0A\09 vfmsub213ps $3, $1, $0 \0A\09 vfmsub231ps $3, $1, $0", "x,x,x,*m"(<8 x float> %a0, <8 x float> %a1, <8 x float> %a2, <8 x float> *%a3) nounwind
-  ret void
-}
-
-define void @test_vfmsubsd_128(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, <2 x double> *%a3) optsize {
-; GENERIC-LABEL: test_vfmsubsd_128:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    vfmsub132sd {{.*#+}} xmm0 = (xmm0 * xmm2) - xmm1 sched: [5:0.50]
-; GENERIC-NEXT:    vfmsub213sd {{.*#+}} xmm0 = (xmm1 * xmm0) - xmm2 sched: [5:0.50]
-; GENERIC-NEXT:    vfmsub231sd {{.*#+}} xmm0 = (xmm1 * xmm2) - xmm0 sched: [5:0.50]
-; GENERIC-NEXT:    vfmsub132sd {{.*#+}} xmm0 = (xmm0 * mem) - xmm1 sched: [10:0.50]
-; GENERIC-NEXT:    vfmsub213sd {{.*#+}} xmm0 = (xmm1 * xmm0) - mem sched: [10:0.50]
-; GENERIC-NEXT:    vfmsub231sd {{.*#+}} xmm0 = (xmm1 * mem) - xmm0 sched: [10:0.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER2-LABEL: test_vfmsubsd_128:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    vfmsub132sd {{.*#+}} xmm0 = (xmm0 * xmm2) - xmm1 sched: [5:0.50]
-; BDVER2-NEXT:    vfmsub213sd {{.*#+}} xmm0 = (xmm1 * xmm0) - xmm2 sched: [5:0.50]
-; BDVER2-NEXT:    vfmsub231sd {{.*#+}} xmm0 = (xmm1 * xmm2) - xmm0 sched: [5:0.50]
-; BDVER2-NEXT:    vfmsub132sd {{.*#+}} xmm0 = (xmm0 * mem) - xmm1 sched: [10:0.50]
-; BDVER2-NEXT:    vfmsub213sd {{.*#+}} xmm0 = (xmm1 * xmm0) - mem sched: [10:0.50]
-; BDVER2-NEXT:    vfmsub231sd {{.*#+}} xmm0 = (xmm1 * mem) - xmm0 sched: [10:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; HASWELL-LABEL: test_vfmsubsd_128:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    vfmsub132sd {{.*#+}} xmm0 = (xmm0 * xmm2) - xmm1 sched: [5:0.50]
-; HASWELL-NEXT:    vfmsub213sd {{.*#+}} xmm0 = (xmm1 * xmm0) - xmm2 sched: [5:0.50]
-; HASWELL-NEXT:    vfmsub231sd {{.*#+}} xmm0 = (xmm1 * xmm2) - xmm0 sched: [5:0.50]
-; HASWELL-NEXT:    vfmsub132sd {{.*#+}} xmm0 = (xmm0 * mem) - xmm1 sched: [10:0.50]
-; HASWELL-NEXT:    vfmsub213sd {{.*#+}} xmm0 = (xmm1 * xmm0) - mem sched: [10:0.50]
-; HASWELL-NEXT:    vfmsub231sd {{.*#+}} xmm0 = (xmm1 * mem) - xmm0 sched: [10:0.50]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_vfmsubsd_128:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    vfmsub132sd {{.*#+}} xmm0 = (xmm0 * xmm2) - xmm1 sched: [5:0.50]
-; BROADWELL-NEXT:    vfmsub213sd {{.*#+}} xmm0 = (xmm1 * xmm0) - xmm2 sched: [5:0.50]
-; BROADWELL-NEXT:    vfmsub231sd {{.*#+}} xmm0 = (xmm1 * xmm2) - xmm0 sched: [5:0.50]
-; BROADWELL-NEXT:    vfmsub132sd {{.*#+}} xmm0 = (xmm0 * mem) - xmm1 sched: [10:0.50]
-; BROADWELL-NEXT:    vfmsub213sd {{.*#+}} xmm0 = (xmm1 * xmm0) - mem sched: [10:0.50]
-; BROADWELL-NEXT:    vfmsub231sd {{.*#+}} xmm0 = (xmm1 * mem) - xmm0 sched: [10:0.50]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_vfmsubsd_128:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    vfmsub132sd {{.*#+}} xmm0 = (xmm0 * xmm2) - xmm1 sched: [4:0.50]
-; SKYLAKE-NEXT:    vfmsub213sd {{.*#+}} xmm0 = (xmm1 * xmm0) - xmm2 sched: [4:0.50]
-; SKYLAKE-NEXT:    vfmsub231sd {{.*#+}} xmm0 = (xmm1 * xmm2) - xmm0 sched: [4:0.50]
-; SKYLAKE-NEXT:    vfmsub132sd {{.*#+}} xmm0 = (xmm0 * mem) - xmm1 sched: [9:0.50]
-; SKYLAKE-NEXT:    vfmsub213sd {{.*#+}} xmm0 = (xmm1 * xmm0) - mem sched: [9:0.50]
-; SKYLAKE-NEXT:    vfmsub231sd {{.*#+}} xmm0 = (xmm1 * mem) - xmm0 sched: [9:0.50]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; KNL-LABEL: test_vfmsubsd_128:
-; KNL:       # %bb.0:
-; KNL-NEXT:    #APP
-; KNL-NEXT:    vfmsub132sd {{.*#+}} xmm0 = (xmm0 * xmm2) - xmm1 sched: [5:0.50]
-; KNL-NEXT:    vfmsub213sd {{.*#+}} xmm0 = (xmm1 * xmm0) - xmm2 sched: [5:0.50]
-; KNL-NEXT:    vfmsub231sd {{.*#+}} xmm0 = (xmm1 * xmm2) - xmm0 sched: [5:0.50]
-; KNL-NEXT:    vfmsub132sd {{.*#+}} xmm0 = (xmm0 * mem) - xmm1 sched: [10:0.50]
-; KNL-NEXT:    vfmsub213sd {{.*#+}} xmm0 = (xmm1 * xmm0) - mem sched: [10:0.50]
-; KNL-NEXT:    vfmsub231sd {{.*#+}} xmm0 = (xmm1 * mem) - xmm0 sched: [10:0.50]
-; KNL-NEXT:    #NO_APP
-; KNL-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_vfmsubsd_128:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    vfmsub132sd {{.*#+}} xmm0 = (xmm0 * xmm2) - xmm1 sched: [4:0.50]
-; SKX-NEXT:    vfmsub213sd {{.*#+}} xmm0 = (xmm1 * xmm0) - xmm2 sched: [4:0.50]
-; SKX-NEXT:    vfmsub231sd {{.*#+}} xmm0 = (xmm1 * xmm2) - xmm0 sched: [4:0.50]
-; SKX-NEXT:    vfmsub132sd {{.*#+}} xmm0 = (xmm0 * mem) - xmm1 sched: [9:0.50]
-; SKX-NEXT:    vfmsub213sd {{.*#+}} xmm0 = (xmm1 * xmm0) - mem sched: [9:0.50]
-; SKX-NEXT:    vfmsub231sd {{.*#+}} xmm0 = (xmm1 * mem) - xmm0 sched: [9:0.50]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_vfmsubsd_128:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    vfmsub132sd {{.*#+}} xmm0 = (xmm0 * xmm2) - xmm1 sched: [5:0.50]
-; ZNVER1-NEXT:    vfmsub213sd {{.*#+}} xmm0 = (xmm1 * xmm0) - xmm2 sched: [5:0.50]
-; ZNVER1-NEXT:    vfmsub231sd {{.*#+}} xmm0 = (xmm1 * xmm2) - xmm0 sched: [5:0.50]
-; ZNVER1-NEXT:    vfmsub132sd {{.*#+}} xmm0 = (xmm0 * mem) - xmm1 sched: [12:0.50]
-; ZNVER1-NEXT:    vfmsub213sd {{.*#+}} xmm0 = (xmm1 * xmm0) - mem sched: [12:0.50]
-; ZNVER1-NEXT:    vfmsub231sd {{.*#+}} xmm0 = (xmm1 * mem) - xmm0 sched: [12:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  tail call void asm "vfmsub132sd $2, $1, $0 \0A\09 vfmsub213sd $2, $1, $0 \0A\09 vfmsub231sd $2, $1, $0 \0A\09 vfmsub132sd $3, $1, $0 \0A\09 vfmsub213sd $3, $1, $0 \0A\09 vfmsub231sd $3, $1, $0", "x,x,x,*m"(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, <2 x double> *%a3) nounwind
-  ret void
-}
-
-define void @test_vfmsubss_128(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, <4 x float> *%a3) optsize {
-; GENERIC-LABEL: test_vfmsubss_128:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    vfmsub132ss {{.*#+}} xmm0 = (xmm0 * xmm2) - xmm1 sched: [5:0.50]
-; GENERIC-NEXT:    vfmsub213ss {{.*#+}} xmm0 = (xmm1 * xmm0) - xmm2 sched: [5:0.50]
-; GENERIC-NEXT:    vfmsub231ss {{.*#+}} xmm0 = (xmm1 * xmm2) - xmm0 sched: [5:0.50]
-; GENERIC-NEXT:    vfmsub132ss {{.*#+}} xmm0 = (xmm0 * mem) - xmm1 sched: [10:0.50]
-; GENERIC-NEXT:    vfmsub213ss {{.*#+}} xmm0 = (xmm1 * xmm0) - mem sched: [10:0.50]
-; GENERIC-NEXT:    vfmsub231ss {{.*#+}} xmm0 = (xmm1 * mem) - xmm0 sched: [10:0.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER2-LABEL: test_vfmsubss_128:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    vfmsub132ss {{.*#+}} xmm0 = (xmm0 * xmm2) - xmm1 sched: [5:0.50]
-; BDVER2-NEXT:    vfmsub213ss {{.*#+}} xmm0 = (xmm1 * xmm0) - xmm2 sched: [5:0.50]
-; BDVER2-NEXT:    vfmsub231ss {{.*#+}} xmm0 = (xmm1 * xmm2) - xmm0 sched: [5:0.50]
-; BDVER2-NEXT:    vfmsub132ss {{.*#+}} xmm0 = (xmm0 * mem) - xmm1 sched: [10:0.50]
-; BDVER2-NEXT:    vfmsub213ss {{.*#+}} xmm0 = (xmm1 * xmm0) - mem sched: [10:0.50]
-; BDVER2-NEXT:    vfmsub231ss {{.*#+}} xmm0 = (xmm1 * mem) - xmm0 sched: [10:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; HASWELL-LABEL: test_vfmsubss_128:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    vfmsub132ss {{.*#+}} xmm0 = (xmm0 * xmm2) - xmm1 sched: [5:0.50]
-; HASWELL-NEXT:    vfmsub213ss {{.*#+}} xmm0 = (xmm1 * xmm0) - xmm2 sched: [5:0.50]
-; HASWELL-NEXT:    vfmsub231ss {{.*#+}} xmm0 = (xmm1 * xmm2) - xmm0 sched: [5:0.50]
-; HASWELL-NEXT:    vfmsub132ss {{.*#+}} xmm0 = (xmm0 * mem) - xmm1 sched: [10:0.50]
-; HASWELL-NEXT:    vfmsub213ss {{.*#+}} xmm0 = (xmm1 * xmm0) - mem sched: [10:0.50]
-; HASWELL-NEXT:    vfmsub231ss {{.*#+}} xmm0 = (xmm1 * mem) - xmm0 sched: [10:0.50]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_vfmsubss_128:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    vfmsub132ss {{.*#+}} xmm0 = (xmm0 * xmm2) - xmm1 sched: [5:0.50]
-; BROADWELL-NEXT:    vfmsub213ss {{.*#+}} xmm0 = (xmm1 * xmm0) - xmm2 sched: [5:0.50]
-; BROADWELL-NEXT:    vfmsub231ss {{.*#+}} xmm0 = (xmm1 * xmm2) - xmm0 sched: [5:0.50]
-; BROADWELL-NEXT:    vfmsub132ss {{.*#+}} xmm0 = (xmm0 * mem) - xmm1 sched: [10:0.50]
-; BROADWELL-NEXT:    vfmsub213ss {{.*#+}} xmm0 = (xmm1 * xmm0) - mem sched: [10:0.50]
-; BROADWELL-NEXT:    vfmsub231ss {{.*#+}} xmm0 = (xmm1 * mem) - xmm0 sched: [10:0.50]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_vfmsubss_128:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    vfmsub132ss {{.*#+}} xmm0 = (xmm0 * xmm2) - xmm1 sched: [4:0.50]
-; SKYLAKE-NEXT:    vfmsub213ss {{.*#+}} xmm0 = (xmm1 * xmm0) - xmm2 sched: [4:0.50]
-; SKYLAKE-NEXT:    vfmsub231ss {{.*#+}} xmm0 = (xmm1 * xmm2) - xmm0 sched: [4:0.50]
-; SKYLAKE-NEXT:    vfmsub132ss {{.*#+}} xmm0 = (xmm0 * mem) - xmm1 sched: [9:0.50]
-; SKYLAKE-NEXT:    vfmsub213ss {{.*#+}} xmm0 = (xmm1 * xmm0) - mem sched: [9:0.50]
-; SKYLAKE-NEXT:    vfmsub231ss {{.*#+}} xmm0 = (xmm1 * mem) - xmm0 sched: [9:0.50]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; KNL-LABEL: test_vfmsubss_128:
-; KNL:       # %bb.0:
-; KNL-NEXT:    #APP
-; KNL-NEXT:    vfmsub132ss {{.*#+}} xmm0 = (xmm0 * xmm2) - xmm1 sched: [5:0.50]
-; KNL-NEXT:    vfmsub213ss {{.*#+}} xmm0 = (xmm1 * xmm0) - xmm2 sched: [5:0.50]
-; KNL-NEXT:    vfmsub231ss {{.*#+}} xmm0 = (xmm1 * xmm2) - xmm0 sched: [5:0.50]
-; KNL-NEXT:    vfmsub132ss {{.*#+}} xmm0 = (xmm0 * mem) - xmm1 sched: [10:0.50]
-; KNL-NEXT:    vfmsub213ss {{.*#+}} xmm0 = (xmm1 * xmm0) - mem sched: [10:0.50]
-; KNL-NEXT:    vfmsub231ss {{.*#+}} xmm0 = (xmm1 * mem) - xmm0 sched: [10:0.50]
-; KNL-NEXT:    #NO_APP
-; KNL-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_vfmsubss_128:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    vfmsub132ss {{.*#+}} xmm0 = (xmm0 * xmm2) - xmm1 sched: [4:0.50]
-; SKX-NEXT:    vfmsub213ss {{.*#+}} xmm0 = (xmm1 * xmm0) - xmm2 sched: [4:0.50]
-; SKX-NEXT:    vfmsub231ss {{.*#+}} xmm0 = (xmm1 * xmm2) - xmm0 sched: [4:0.50]
-; SKX-NEXT:    vfmsub132ss {{.*#+}} xmm0 = (xmm0 * mem) - xmm1 sched: [9:0.50]
-; SKX-NEXT:    vfmsub213ss {{.*#+}} xmm0 = (xmm1 * xmm0) - mem sched: [9:0.50]
-; SKX-NEXT:    vfmsub231ss {{.*#+}} xmm0 = (xmm1 * mem) - xmm0 sched: [9:0.50]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_vfmsubss_128:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    vfmsub132ss {{.*#+}} xmm0 = (xmm0 * xmm2) - xmm1 sched: [5:0.50]
-; ZNVER1-NEXT:    vfmsub213ss {{.*#+}} xmm0 = (xmm1 * xmm0) - xmm2 sched: [5:0.50]
-; ZNVER1-NEXT:    vfmsub231ss {{.*#+}} xmm0 = (xmm1 * xmm2) - xmm0 sched: [5:0.50]
-; ZNVER1-NEXT:    vfmsub132ss {{.*#+}} xmm0 = (xmm0 * mem) - xmm1 sched: [12:0.50]
-; ZNVER1-NEXT:    vfmsub213ss {{.*#+}} xmm0 = (xmm1 * xmm0) - mem sched: [12:0.50]
-; ZNVER1-NEXT:    vfmsub231ss {{.*#+}} xmm0 = (xmm1 * mem) - xmm0 sched: [12:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  tail call void asm "vfmsub132ss $2, $1, $0 \0A\09 vfmsub213ss $2, $1, $0 \0A\09 vfmsub231ss $2, $1, $0 \0A\09 vfmsub132ss $3, $1, $0 \0A\09 vfmsub213ss $3, $1, $0 \0A\09 vfmsub231ss $3, $1, $0", "x,x,x,*m"(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, <4 x float> *%a3) nounwind
-  ret void
-}
-
-;
-; VFNMADD
-;
-
-define void @test_vfnmaddpd_128(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, <2 x double> *%a3) optsize {
-; GENERIC-LABEL: test_vfnmaddpd_128:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    vfnmadd132pd {{.*#+}} xmm0 = -(xmm0 * xmm2) + xmm1 sched: [5:0.50]
-; GENERIC-NEXT:    vfnmadd213pd {{.*#+}} xmm0 = -(xmm1 * xmm0) + xmm2 sched: [5:0.50]
-; GENERIC-NEXT:    vfnmadd231pd {{.*#+}} xmm0 = -(xmm1 * xmm2) + xmm0 sched: [5:0.50]
-; GENERIC-NEXT:    vfnmadd132pd {{.*#+}} xmm0 = -(xmm0 * mem) + xmm1 sched: [10:0.50]
-; GENERIC-NEXT:    vfnmadd213pd {{.*#+}} xmm0 = -(xmm1 * xmm0) + mem sched: [10:0.50]
-; GENERIC-NEXT:    vfnmadd231pd {{.*#+}} xmm0 = -(xmm1 * mem) + xmm0 sched: [10:0.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER2-LABEL: test_vfnmaddpd_128:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    vfnmadd132pd {{.*#+}} xmm0 = -(xmm0 * xmm2) + xmm1 sched: [5:0.50]
-; BDVER2-NEXT:    vfnmadd213pd {{.*#+}} xmm0 = -(xmm1 * xmm0) + xmm2 sched: [5:0.50]
-; BDVER2-NEXT:    vfnmadd231pd {{.*#+}} xmm0 = -(xmm1 * xmm2) + xmm0 sched: [5:0.50]
-; BDVER2-NEXT:    vfnmadd132pd {{.*#+}} xmm0 = -(xmm0 * mem) + xmm1 sched: [10:0.50]
-; BDVER2-NEXT:    vfnmadd213pd {{.*#+}} xmm0 = -(xmm1 * xmm0) + mem sched: [10:0.50]
-; BDVER2-NEXT:    vfnmadd231pd {{.*#+}} xmm0 = -(xmm1 * mem) + xmm0 sched: [10:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; HASWELL-LABEL: test_vfnmaddpd_128:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    vfnmadd132pd {{.*#+}} xmm0 = -(xmm0 * xmm2) + xmm1 sched: [5:0.50]
-; HASWELL-NEXT:    vfnmadd213pd {{.*#+}} xmm0 = -(xmm1 * xmm0) + xmm2 sched: [5:0.50]
-; HASWELL-NEXT:    vfnmadd231pd {{.*#+}} xmm0 = -(xmm1 * xmm2) + xmm0 sched: [5:0.50]
-; HASWELL-NEXT:    vfnmadd132pd {{.*#+}} xmm0 = -(xmm0 * mem) + xmm1 sched: [11:0.50]
-; HASWELL-NEXT:    vfnmadd213pd {{.*#+}} xmm0 = -(xmm1 * xmm0) + mem sched: [11:0.50]
-; HASWELL-NEXT:    vfnmadd231pd {{.*#+}} xmm0 = -(xmm1 * mem) + xmm0 sched: [11:0.50]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_vfnmaddpd_128:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    vfnmadd132pd {{.*#+}} xmm0 = -(xmm0 * xmm2) + xmm1 sched: [5:0.50]
-; BROADWELL-NEXT:    vfnmadd213pd {{.*#+}} xmm0 = -(xmm1 * xmm0) + xmm2 sched: [5:0.50]
-; BROADWELL-NEXT:    vfnmadd231pd {{.*#+}} xmm0 = -(xmm1 * xmm2) + xmm0 sched: [5:0.50]
-; BROADWELL-NEXT:    vfnmadd132pd {{.*#+}} xmm0 = -(xmm0 * mem) + xmm1 sched: [10:0.50]
-; BROADWELL-NEXT:    vfnmadd213pd {{.*#+}} xmm0 = -(xmm1 * xmm0) + mem sched: [10:0.50]
-; BROADWELL-NEXT:    vfnmadd231pd {{.*#+}} xmm0 = -(xmm1 * mem) + xmm0 sched: [10:0.50]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_vfnmaddpd_128:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    vfnmadd132pd {{.*#+}} xmm0 = -(xmm0 * xmm2) + xmm1 sched: [4:0.50]
-; SKYLAKE-NEXT:    vfnmadd213pd {{.*#+}} xmm0 = -(xmm1 * xmm0) + xmm2 sched: [4:0.50]
-; SKYLAKE-NEXT:    vfnmadd231pd {{.*#+}} xmm0 = -(xmm1 * xmm2) + xmm0 sched: [4:0.50]
-; SKYLAKE-NEXT:    vfnmadd132pd {{.*#+}} xmm0 = -(xmm0 * mem) + xmm1 sched: [10:0.50]
-; SKYLAKE-NEXT:    vfnmadd213pd {{.*#+}} xmm0 = -(xmm1 * xmm0) + mem sched: [10:0.50]
-; SKYLAKE-NEXT:    vfnmadd231pd {{.*#+}} xmm0 = -(xmm1 * mem) + xmm0 sched: [10:0.50]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; KNL-LABEL: test_vfnmaddpd_128:
-; KNL:       # %bb.0:
-; KNL-NEXT:    #APP
-; KNL-NEXT:    vfnmadd132pd {{.*#+}} xmm0 = -(xmm0 * xmm2) + xmm1 sched: [5:0.50]
-; KNL-NEXT:    vfnmadd213pd {{.*#+}} xmm0 = -(xmm1 * xmm0) + xmm2 sched: [5:0.50]
-; KNL-NEXT:    vfnmadd231pd {{.*#+}} xmm0 = -(xmm1 * xmm2) + xmm0 sched: [5:0.50]
-; KNL-NEXT:    vfnmadd132pd {{.*#+}} xmm0 = -(xmm0 * mem) + xmm1 sched: [11:0.50]
-; KNL-NEXT:    vfnmadd213pd {{.*#+}} xmm0 = -(xmm1 * xmm0) + mem sched: [11:0.50]
-; KNL-NEXT:    vfnmadd231pd {{.*#+}} xmm0 = -(xmm1 * mem) + xmm0 sched: [11:0.50]
-; KNL-NEXT:    #NO_APP
-; KNL-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_vfnmaddpd_128:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    vfnmadd132pd {{.*#+}} xmm0 = -(xmm0 * xmm2) + xmm1 sched: [4:0.50]
-; SKX-NEXT:    vfnmadd213pd {{.*#+}} xmm0 = -(xmm1 * xmm0) + xmm2 sched: [4:0.50]
-; SKX-NEXT:    vfnmadd231pd {{.*#+}} xmm0 = -(xmm1 * xmm2) + xmm0 sched: [4:0.50]
-; SKX-NEXT:    vfnmadd132pd {{.*#+}} xmm0 = -(xmm0 * mem) + xmm1 sched: [10:0.50]
-; SKX-NEXT:    vfnmadd213pd {{.*#+}} xmm0 = -(xmm1 * xmm0) + mem sched: [10:0.50]
-; SKX-NEXT:    vfnmadd231pd {{.*#+}} xmm0 = -(xmm1 * mem) + xmm0 sched: [10:0.50]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_vfnmaddpd_128:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    vfnmadd132pd {{.*#+}} xmm0 = -(xmm0 * xmm2) + xmm1 sched: [5:0.50]
-; ZNVER1-NEXT:    vfnmadd213pd {{.*#+}} xmm0 = -(xmm1 * xmm0) + xmm2 sched: [5:0.50]
-; ZNVER1-NEXT:    vfnmadd231pd {{.*#+}} xmm0 = -(xmm1 * xmm2) + xmm0 sched: [5:0.50]
-; ZNVER1-NEXT:    vfnmadd132pd {{.*#+}} xmm0 = -(xmm0 * mem) + xmm1 sched: [12:0.50]
-; ZNVER1-NEXT:    vfnmadd213pd {{.*#+}} xmm0 = -(xmm1 * xmm0) + mem sched: [12:0.50]
-; ZNVER1-NEXT:    vfnmadd231pd {{.*#+}} xmm0 = -(xmm1 * mem) + xmm0 sched: [12:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  tail call void asm "vfnmadd132pd $2, $1, $0 \0A\09 vfnmadd213pd $2, $1, $0 \0A\09 vfnmadd231pd $2, $1, $0 \0A\09 vfnmadd132pd $3, $1, $0 \0A\09 vfnmadd213pd $3, $1, $0 \0A\09 vfnmadd231pd $3, $1, $0", "x,x,x,*m"(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, <2 x double> *%a3) nounwind
-  ret void
-}
-
-define void @test_vfnmaddpd_256(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2, <4 x double> *%a3) optsize {
-; GENERIC-LABEL: test_vfnmaddpd_256:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    vfnmadd132pd {{.*#+}} ymm0 = -(ymm0 * ymm2) + ymm1 sched: [5:0.50]
-; GENERIC-NEXT:    vfnmadd213pd {{.*#+}} ymm0 = -(ymm1 * ymm0) + ymm2 sched: [5:0.50]
-; GENERIC-NEXT:    vfnmadd231pd {{.*#+}} ymm0 = -(ymm1 * ymm2) + ymm0 sched: [5:0.50]
-; GENERIC-NEXT:    vfnmadd132pd {{.*#+}} ymm0 = -(ymm0 * mem) + ymm1 sched: [10:0.50]
-; GENERIC-NEXT:    vfnmadd213pd {{.*#+}} ymm0 = -(ymm1 * ymm0) + mem sched: [10:0.50]
-; GENERIC-NEXT:    vfnmadd231pd {{.*#+}} ymm0 = -(ymm1 * mem) + ymm0 sched: [10:0.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    vzeroupper # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER2-LABEL: test_vfnmaddpd_256:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    vfnmadd132pd {{.*#+}} ymm0 = -(ymm0 * ymm2) + ymm1 sched: [5:0.50]
-; BDVER2-NEXT:    vfnmadd213pd {{.*#+}} ymm0 = -(ymm1 * ymm0) + ymm2 sched: [5:0.50]
-; BDVER2-NEXT:    vfnmadd231pd {{.*#+}} ymm0 = -(ymm1 * ymm2) + ymm0 sched: [5:0.50]
-; BDVER2-NEXT:    vfnmadd132pd {{.*#+}} ymm0 = -(ymm0 * mem) + ymm1 sched: [10:1.00]
-; BDVER2-NEXT:    vfnmadd213pd {{.*#+}} ymm0 = -(ymm1 * ymm0) + mem sched: [10:1.00]
-; BDVER2-NEXT:    vfnmadd231pd {{.*#+}} ymm0 = -(ymm1 * mem) + ymm0 sched: [10:1.00]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    vzeroupper # sched: [46:4.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; HASWELL-LABEL: test_vfnmaddpd_256:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    vfnmadd132pd {{.*#+}} ymm0 = -(ymm0 * ymm2) + ymm1 sched: [5:0.50]
-; HASWELL-NEXT:    vfnmadd213pd {{.*#+}} ymm0 = -(ymm1 * ymm0) + ymm2 sched: [5:0.50]
-; HASWELL-NEXT:    vfnmadd231pd {{.*#+}} ymm0 = -(ymm1 * ymm2) + ymm0 sched: [5:0.50]
-; HASWELL-NEXT:    vfnmadd132pd {{.*#+}} ymm0 = -(ymm0 * mem) + ymm1 sched: [12:0.50]
-; HASWELL-NEXT:    vfnmadd213pd {{.*#+}} ymm0 = -(ymm1 * ymm0) + mem sched: [12:0.50]
-; HASWELL-NEXT:    vfnmadd231pd {{.*#+}} ymm0 = -(ymm1 * mem) + ymm0 sched: [12:0.50]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    vzeroupper # sched: [0:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_vfnmaddpd_256:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    vfnmadd132pd {{.*#+}} ymm0 = -(ymm0 * ymm2) + ymm1 sched: [5:0.50]
-; BROADWELL-NEXT:    vfnmadd213pd {{.*#+}} ymm0 = -(ymm1 * ymm0) + ymm2 sched: [5:0.50]
-; BROADWELL-NEXT:    vfnmadd231pd {{.*#+}} ymm0 = -(ymm1 * ymm2) + ymm0 sched: [5:0.50]
-; BROADWELL-NEXT:    vfnmadd132pd {{.*#+}} ymm0 = -(ymm0 * mem) + ymm1 sched: [11:0.50]
-; BROADWELL-NEXT:    vfnmadd213pd {{.*#+}} ymm0 = -(ymm1 * ymm0) + mem sched: [11:0.50]
-; BROADWELL-NEXT:    vfnmadd231pd {{.*#+}} ymm0 = -(ymm1 * mem) + ymm0 sched: [11:0.50]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    vzeroupper # sched: [0:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_vfnmaddpd_256:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    vfnmadd132pd {{.*#+}} ymm0 = -(ymm0 * ymm2) + ymm1 sched: [4:0.50]
-; SKYLAKE-NEXT:    vfnmadd213pd {{.*#+}} ymm0 = -(ymm1 * ymm0) + ymm2 sched: [4:0.50]
-; SKYLAKE-NEXT:    vfnmadd231pd {{.*#+}} ymm0 = -(ymm1 * ymm2) + ymm0 sched: [4:0.50]
-; SKYLAKE-NEXT:    vfnmadd132pd {{.*#+}} ymm0 = -(ymm0 * mem) + ymm1 sched: [11:0.50]
-; SKYLAKE-NEXT:    vfnmadd213pd {{.*#+}} ymm0 = -(ymm1 * ymm0) + mem sched: [11:0.50]
-; SKYLAKE-NEXT:    vfnmadd231pd {{.*#+}} ymm0 = -(ymm1 * mem) + ymm0 sched: [11:0.50]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    vzeroupper # sched: [0:0.67]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; KNL-LABEL: test_vfnmaddpd_256:
-; KNL:       # %bb.0:
-; KNL-NEXT:    #APP
-; KNL-NEXT:    vfnmadd132pd {{.*#+}} ymm0 = -(ymm0 * ymm2) + ymm1 sched: [5:0.50]
-; KNL-NEXT:    vfnmadd213pd {{.*#+}} ymm0 = -(ymm1 * ymm0) + ymm2 sched: [5:0.50]
-; KNL-NEXT:    vfnmadd231pd {{.*#+}} ymm0 = -(ymm1 * ymm2) + ymm0 sched: [5:0.50]
-; KNL-NEXT:    vfnmadd132pd {{.*#+}} ymm0 = -(ymm0 * mem) + ymm1 sched: [12:0.50]
-; KNL-NEXT:    vfnmadd213pd {{.*#+}} ymm0 = -(ymm1 * ymm0) + mem sched: [12:0.50]
-; KNL-NEXT:    vfnmadd231pd {{.*#+}} ymm0 = -(ymm1 * mem) + ymm0 sched: [12:0.50]
-; KNL-NEXT:    #NO_APP
-; KNL-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_vfnmaddpd_256:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    vfnmadd132pd {{.*#+}} ymm0 = -(ymm0 * ymm2) + ymm1 sched: [4:0.50]
-; SKX-NEXT:    vfnmadd213pd {{.*#+}} ymm0 = -(ymm1 * ymm0) + ymm2 sched: [4:0.50]
-; SKX-NEXT:    vfnmadd231pd {{.*#+}} ymm0 = -(ymm1 * ymm2) + ymm0 sched: [4:0.50]
-; SKX-NEXT:    vfnmadd132pd {{.*#+}} ymm0 = -(ymm0 * mem) + ymm1 sched: [11:0.50]
-; SKX-NEXT:    vfnmadd213pd {{.*#+}} ymm0 = -(ymm1 * ymm0) + mem sched: [11:0.50]
-; SKX-NEXT:    vfnmadd231pd {{.*#+}} ymm0 = -(ymm1 * mem) + ymm0 sched: [11:0.50]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    vzeroupper # sched: [0:0.67]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_vfnmaddpd_256:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    vfnmadd132pd {{.*#+}} ymm0 = -(ymm0 * ymm2) + ymm1 sched: [5:0.50]
-; ZNVER1-NEXT:    vfnmadd213pd {{.*#+}} ymm0 = -(ymm1 * ymm0) + ymm2 sched: [5:0.50]
-; ZNVER1-NEXT:    vfnmadd231pd {{.*#+}} ymm0 = -(ymm1 * ymm2) + ymm0 sched: [5:0.50]
-; ZNVER1-NEXT:    vfnmadd132pd {{.*#+}} ymm0 = -(ymm0 * mem) + ymm1 sched: [12:0.50]
-; ZNVER1-NEXT:    vfnmadd213pd {{.*#+}} ymm0 = -(ymm1 * ymm0) + mem sched: [12:0.50]
-; ZNVER1-NEXT:    vfnmadd231pd {{.*#+}} ymm0 = -(ymm1 * mem) + ymm0 sched: [12:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    vzeroupper # sched: [100:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  tail call void asm "vfnmadd132pd $2, $1, $0 \0A\09 vfnmadd213pd $2, $1, $0 \0A\09 vfnmadd231pd $2, $1, $0 \0A\09 vfnmadd132pd $3, $1, $0 \0A\09 vfnmadd213pd $3, $1, $0 \0A\09 vfnmadd231pd $3, $1, $0", "x,x,x,*m"(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2, <4 x double> *%a3) nounwind
-  ret void
-}
-
-define void @test_vfnmaddps_128(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, <4 x float> *%a3) optsize {
-; GENERIC-LABEL: test_vfnmaddps_128:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    vfnmadd132ps {{.*#+}} xmm0 = -(xmm0 * xmm2) + xmm1 sched: [5:0.50]
-; GENERIC-NEXT:    vfnmadd213ps {{.*#+}} xmm0 = -(xmm1 * xmm0) + xmm2 sched: [5:0.50]
-; GENERIC-NEXT:    vfnmadd231ps {{.*#+}} xmm0 = -(xmm1 * xmm2) + xmm0 sched: [5:0.50]
-; GENERIC-NEXT:    vfnmadd132ps {{.*#+}} xmm0 = -(xmm0 * mem) + xmm1 sched: [10:0.50]
-; GENERIC-NEXT:    vfnmadd213ps {{.*#+}} xmm0 = -(xmm1 * xmm0) + mem sched: [10:0.50]
-; GENERIC-NEXT:    vfnmadd231ps {{.*#+}} xmm0 = -(xmm1 * mem) + xmm0 sched: [10:0.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER2-LABEL: test_vfnmaddps_128:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    vfnmadd132ps {{.*#+}} xmm0 = -(xmm0 * xmm2) + xmm1 sched: [5:0.50]
-; BDVER2-NEXT:    vfnmadd213ps {{.*#+}} xmm0 = -(xmm1 * xmm0) + xmm2 sched: [5:0.50]
-; BDVER2-NEXT:    vfnmadd231ps {{.*#+}} xmm0 = -(xmm1 * xmm2) + xmm0 sched: [5:0.50]
-; BDVER2-NEXT:    vfnmadd132ps {{.*#+}} xmm0 = -(xmm0 * mem) + xmm1 sched: [10:0.50]
-; BDVER2-NEXT:    vfnmadd213ps {{.*#+}} xmm0 = -(xmm1 * xmm0) + mem sched: [10:0.50]
-; BDVER2-NEXT:    vfnmadd231ps {{.*#+}} xmm0 = -(xmm1 * mem) + xmm0 sched: [10:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; HASWELL-LABEL: test_vfnmaddps_128:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    vfnmadd132ps {{.*#+}} xmm0 = -(xmm0 * xmm2) + xmm1 sched: [5:0.50]
-; HASWELL-NEXT:    vfnmadd213ps {{.*#+}} xmm0 = -(xmm1 * xmm0) + xmm2 sched: [5:0.50]
-; HASWELL-NEXT:    vfnmadd231ps {{.*#+}} xmm0 = -(xmm1 * xmm2) + xmm0 sched: [5:0.50]
-; HASWELL-NEXT:    vfnmadd132ps {{.*#+}} xmm0 = -(xmm0 * mem) + xmm1 sched: [11:0.50]
-; HASWELL-NEXT:    vfnmadd213ps {{.*#+}} xmm0 = -(xmm1 * xmm0) + mem sched: [11:0.50]
-; HASWELL-NEXT:    vfnmadd231ps {{.*#+}} xmm0 = -(xmm1 * mem) + xmm0 sched: [11:0.50]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_vfnmaddps_128:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    vfnmadd132ps {{.*#+}} xmm0 = -(xmm0 * xmm2) + xmm1 sched: [5:0.50]
-; BROADWELL-NEXT:    vfnmadd213ps {{.*#+}} xmm0 = -(xmm1 * xmm0) + xmm2 sched: [5:0.50]
-; BROADWELL-NEXT:    vfnmadd231ps {{.*#+}} xmm0 = -(xmm1 * xmm2) + xmm0 sched: [5:0.50]
-; BROADWELL-NEXT:    vfnmadd132ps {{.*#+}} xmm0 = -(xmm0 * mem) + xmm1 sched: [10:0.50]
-; BROADWELL-NEXT:    vfnmadd213ps {{.*#+}} xmm0 = -(xmm1 * xmm0) + mem sched: [10:0.50]
-; BROADWELL-NEXT:    vfnmadd231ps {{.*#+}} xmm0 = -(xmm1 * mem) + xmm0 sched: [10:0.50]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_vfnmaddps_128:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    vfnmadd132ps {{.*#+}} xmm0 = -(xmm0 * xmm2) + xmm1 sched: [4:0.50]
-; SKYLAKE-NEXT:    vfnmadd213ps {{.*#+}} xmm0 = -(xmm1 * xmm0) + xmm2 sched: [4:0.50]
-; SKYLAKE-NEXT:    vfnmadd231ps {{.*#+}} xmm0 = -(xmm1 * xmm2) + xmm0 sched: [4:0.50]
-; SKYLAKE-NEXT:    vfnmadd132ps {{.*#+}} xmm0 = -(xmm0 * mem) + xmm1 sched: [10:0.50]
-; SKYLAKE-NEXT:    vfnmadd213ps {{.*#+}} xmm0 = -(xmm1 * xmm0) + mem sched: [10:0.50]
-; SKYLAKE-NEXT:    vfnmadd231ps {{.*#+}} xmm0 = -(xmm1 * mem) + xmm0 sched: [10:0.50]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; KNL-LABEL: test_vfnmaddps_128:
-; KNL:       # %bb.0:
-; KNL-NEXT:    #APP
-; KNL-NEXT:    vfnmadd132ps {{.*#+}} xmm0 = -(xmm0 * xmm2) + xmm1 sched: [5:0.50]
-; KNL-NEXT:    vfnmadd213ps {{.*#+}} xmm0 = -(xmm1 * xmm0) + xmm2 sched: [5:0.50]
-; KNL-NEXT:    vfnmadd231ps {{.*#+}} xmm0 = -(xmm1 * xmm2) + xmm0 sched: [5:0.50]
-; KNL-NEXT:    vfnmadd132ps {{.*#+}} xmm0 = -(xmm0 * mem) + xmm1 sched: [11:0.50]
-; KNL-NEXT:    vfnmadd213ps {{.*#+}} xmm0 = -(xmm1 * xmm0) + mem sched: [11:0.50]
-; KNL-NEXT:    vfnmadd231ps {{.*#+}} xmm0 = -(xmm1 * mem) + xmm0 sched: [11:0.50]
-; KNL-NEXT:    #NO_APP
-; KNL-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_vfnmaddps_128:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    vfnmadd132ps {{.*#+}} xmm0 = -(xmm0 * xmm2) + xmm1 sched: [4:0.50]
-; SKX-NEXT:    vfnmadd213ps {{.*#+}} xmm0 = -(xmm1 * xmm0) + xmm2 sched: [4:0.50]
-; SKX-NEXT:    vfnmadd231ps {{.*#+}} xmm0 = -(xmm1 * xmm2) + xmm0 sched: [4:0.50]
-; SKX-NEXT:    vfnmadd132ps {{.*#+}} xmm0 = -(xmm0 * mem) + xmm1 sched: [10:0.50]
-; SKX-NEXT:    vfnmadd213ps {{.*#+}} xmm0 = -(xmm1 * xmm0) + mem sched: [10:0.50]
-; SKX-NEXT:    vfnmadd231ps {{.*#+}} xmm0 = -(xmm1 * mem) + xmm0 sched: [10:0.50]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_vfnmaddps_128:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    vfnmadd132ps {{.*#+}} xmm0 = -(xmm0 * xmm2) + xmm1 sched: [5:0.50]
-; ZNVER1-NEXT:    vfnmadd213ps {{.*#+}} xmm0 = -(xmm1 * xmm0) + xmm2 sched: [5:0.50]
-; ZNVER1-NEXT:    vfnmadd231ps {{.*#+}} xmm0 = -(xmm1 * xmm2) + xmm0 sched: [5:0.50]
-; ZNVER1-NEXT:    vfnmadd132ps {{.*#+}} xmm0 = -(xmm0 * mem) + xmm1 sched: [12:0.50]
-; ZNVER1-NEXT:    vfnmadd213ps {{.*#+}} xmm0 = -(xmm1 * xmm0) + mem sched: [12:0.50]
-; ZNVER1-NEXT:    vfnmadd231ps {{.*#+}} xmm0 = -(xmm1 * mem) + xmm0 sched: [12:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  tail call void asm "vfnmadd132ps $2, $1, $0 \0A\09 vfnmadd213ps $2, $1, $0 \0A\09 vfnmadd231ps $2, $1, $0 \0A\09 vfnmadd132ps $3, $1, $0 \0A\09 vfnmadd213ps $3, $1, $0 \0A\09 vfnmadd231ps $3, $1, $0", "x,x,x,*m"(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, <4 x float> *%a3) nounwind
-  ret void
-}
-
-define void @test_vfnmaddps_256(<8 x float> %a0, <8 x float> %a1, <8 x float> %a2, <8 x float> *%a3) optsize {
-; GENERIC-LABEL: test_vfnmaddps_256:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    vfnmadd132ps {{.*#+}} ymm0 = -(ymm0 * ymm2) + ymm1 sched: [5:0.50]
-; GENERIC-NEXT:    vfnmadd213ps {{.*#+}} ymm0 = -(ymm1 * ymm0) + ymm2 sched: [5:0.50]
-; GENERIC-NEXT:    vfnmadd231ps {{.*#+}} ymm0 = -(ymm1 * ymm2) + ymm0 sched: [5:0.50]
-; GENERIC-NEXT:    vfnmadd132ps {{.*#+}} ymm0 = -(ymm0 * mem) + ymm1 sched: [10:0.50]
-; GENERIC-NEXT:    vfnmadd213ps {{.*#+}} ymm0 = -(ymm1 * ymm0) + mem sched: [10:0.50]
-; GENERIC-NEXT:    vfnmadd231ps {{.*#+}} ymm0 = -(ymm1 * mem) + ymm0 sched: [10:0.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    vzeroupper # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER2-LABEL: test_vfnmaddps_256:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    vfnmadd132ps {{.*#+}} ymm0 = -(ymm0 * ymm2) + ymm1 sched: [5:0.50]
-; BDVER2-NEXT:    vfnmadd213ps {{.*#+}} ymm0 = -(ymm1 * ymm0) + ymm2 sched: [5:0.50]
-; BDVER2-NEXT:    vfnmadd231ps {{.*#+}} ymm0 = -(ymm1 * ymm2) + ymm0 sched: [5:0.50]
-; BDVER2-NEXT:    vfnmadd132ps {{.*#+}} ymm0 = -(ymm0 * mem) + ymm1 sched: [10:1.00]
-; BDVER2-NEXT:    vfnmadd213ps {{.*#+}} ymm0 = -(ymm1 * ymm0) + mem sched: [10:1.00]
-; BDVER2-NEXT:    vfnmadd231ps {{.*#+}} ymm0 = -(ymm1 * mem) + ymm0 sched: [10:1.00]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    vzeroupper # sched: [46:4.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; HASWELL-LABEL: test_vfnmaddps_256:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    vfnmadd132ps {{.*#+}} ymm0 = -(ymm0 * ymm2) + ymm1 sched: [5:0.50]
-; HASWELL-NEXT:    vfnmadd213ps {{.*#+}} ymm0 = -(ymm1 * ymm0) + ymm2 sched: [5:0.50]
-; HASWELL-NEXT:    vfnmadd231ps {{.*#+}} ymm0 = -(ymm1 * ymm2) + ymm0 sched: [5:0.50]
-; HASWELL-NEXT:    vfnmadd132ps {{.*#+}} ymm0 = -(ymm0 * mem) + ymm1 sched: [12:0.50]
-; HASWELL-NEXT:    vfnmadd213ps {{.*#+}} ymm0 = -(ymm1 * ymm0) + mem sched: [12:0.50]
-; HASWELL-NEXT:    vfnmadd231ps {{.*#+}} ymm0 = -(ymm1 * mem) + ymm0 sched: [12:0.50]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    vzeroupper # sched: [0:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_vfnmaddps_256:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    vfnmadd132ps {{.*#+}} ymm0 = -(ymm0 * ymm2) + ymm1 sched: [5:0.50]
-; BROADWELL-NEXT:    vfnmadd213ps {{.*#+}} ymm0 = -(ymm1 * ymm0) + ymm2 sched: [5:0.50]
-; BROADWELL-NEXT:    vfnmadd231ps {{.*#+}} ymm0 = -(ymm1 * ymm2) + ymm0 sched: [5:0.50]
-; BROADWELL-NEXT:    vfnmadd132ps {{.*#+}} ymm0 = -(ymm0 * mem) + ymm1 sched: [11:0.50]
-; BROADWELL-NEXT:    vfnmadd213ps {{.*#+}} ymm0 = -(ymm1 * ymm0) + mem sched: [11:0.50]
-; BROADWELL-NEXT:    vfnmadd231ps {{.*#+}} ymm0 = -(ymm1 * mem) + ymm0 sched: [11:0.50]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    vzeroupper # sched: [0:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_vfnmaddps_256:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    vfnmadd132ps {{.*#+}} ymm0 = -(ymm0 * ymm2) + ymm1 sched: [4:0.50]
-; SKYLAKE-NEXT:    vfnmadd213ps {{.*#+}} ymm0 = -(ymm1 * ymm0) + ymm2 sched: [4:0.50]
-; SKYLAKE-NEXT:    vfnmadd231ps {{.*#+}} ymm0 = -(ymm1 * ymm2) + ymm0 sched: [4:0.50]
-; SKYLAKE-NEXT:    vfnmadd132ps {{.*#+}} ymm0 = -(ymm0 * mem) + ymm1 sched: [11:0.50]
-; SKYLAKE-NEXT:    vfnmadd213ps {{.*#+}} ymm0 = -(ymm1 * ymm0) + mem sched: [11:0.50]
-; SKYLAKE-NEXT:    vfnmadd231ps {{.*#+}} ymm0 = -(ymm1 * mem) + ymm0 sched: [11:0.50]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    vzeroupper # sched: [0:0.67]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; KNL-LABEL: test_vfnmaddps_256:
-; KNL:       # %bb.0:
-; KNL-NEXT:    #APP
-; KNL-NEXT:    vfnmadd132ps {{.*#+}} ymm0 = -(ymm0 * ymm2) + ymm1 sched: [5:0.50]
-; KNL-NEXT:    vfnmadd213ps {{.*#+}} ymm0 = -(ymm1 * ymm0) + ymm2 sched: [5:0.50]
-; KNL-NEXT:    vfnmadd231ps {{.*#+}} ymm0 = -(ymm1 * ymm2) + ymm0 sched: [5:0.50]
-; KNL-NEXT:    vfnmadd132ps {{.*#+}} ymm0 = -(ymm0 * mem) + ymm1 sched: [12:0.50]
-; KNL-NEXT:    vfnmadd213ps {{.*#+}} ymm0 = -(ymm1 * ymm0) + mem sched: [12:0.50]
-; KNL-NEXT:    vfnmadd231ps {{.*#+}} ymm0 = -(ymm1 * mem) + ymm0 sched: [12:0.50]
-; KNL-NEXT:    #NO_APP
-; KNL-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_vfnmaddps_256:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    vfnmadd132ps {{.*#+}} ymm0 = -(ymm0 * ymm2) + ymm1 sched: [4:0.50]
-; SKX-NEXT:    vfnmadd213ps {{.*#+}} ymm0 = -(ymm1 * ymm0) + ymm2 sched: [4:0.50]
-; SKX-NEXT:    vfnmadd231ps {{.*#+}} ymm0 = -(ymm1 * ymm2) + ymm0 sched: [4:0.50]
-; SKX-NEXT:    vfnmadd132ps {{.*#+}} ymm0 = -(ymm0 * mem) + ymm1 sched: [11:0.50]
-; SKX-NEXT:    vfnmadd213ps {{.*#+}} ymm0 = -(ymm1 * ymm0) + mem sched: [11:0.50]
-; SKX-NEXT:    vfnmadd231ps {{.*#+}} ymm0 = -(ymm1 * mem) + ymm0 sched: [11:0.50]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    vzeroupper # sched: [0:0.67]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_vfnmaddps_256:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    vfnmadd132ps {{.*#+}} ymm0 = -(ymm0 * ymm2) + ymm1 sched: [5:0.50]
-; ZNVER1-NEXT:    vfnmadd213ps {{.*#+}} ymm0 = -(ymm1 * ymm0) + ymm2 sched: [5:0.50]
-; ZNVER1-NEXT:    vfnmadd231ps {{.*#+}} ymm0 = -(ymm1 * ymm2) + ymm0 sched: [5:0.50]
-; ZNVER1-NEXT:    vfnmadd132ps {{.*#+}} ymm0 = -(ymm0 * mem) + ymm1 sched: [12:0.50]
-; ZNVER1-NEXT:    vfnmadd213ps {{.*#+}} ymm0 = -(ymm1 * ymm0) + mem sched: [12:0.50]
-; ZNVER1-NEXT:    vfnmadd231ps {{.*#+}} ymm0 = -(ymm1 * mem) + ymm0 sched: [12:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    vzeroupper # sched: [100:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  tail call void asm "vfnmadd132ps $2, $1, $0 \0A\09 vfnmadd213ps $2, $1, $0 \0A\09 vfnmadd231ps $2, $1, $0 \0A\09 vfnmadd132ps $3, $1, $0 \0A\09 vfnmadd213ps $3, $1, $0 \0A\09 vfnmadd231ps $3, $1, $0", "x,x,x,*m"(<8 x float> %a0, <8 x float> %a1, <8 x float> %a2, <8 x float> *%a3) nounwind
-  ret void
-}
-
-define void @test_vfnmaddsd_128(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, <2 x double> *%a3) optsize {
-; GENERIC-LABEL: test_vfnmaddsd_128:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    vfnmadd132sd {{.*#+}} xmm0 = -(xmm0 * xmm2) + xmm1 sched: [5:0.50]
-; GENERIC-NEXT:    vfnmadd213sd {{.*#+}} xmm0 = -(xmm1 * xmm0) + xmm2 sched: [5:0.50]
-; GENERIC-NEXT:    vfnmadd231sd {{.*#+}} xmm0 = -(xmm1 * xmm2) + xmm0 sched: [5:0.50]
-; GENERIC-NEXT:    vfnmadd132sd {{.*#+}} xmm0 = -(xmm0 * mem) + xmm1 sched: [10:0.50]
-; GENERIC-NEXT:    vfnmadd213sd {{.*#+}} xmm0 = -(xmm1 * xmm0) + mem sched: [10:0.50]
-; GENERIC-NEXT:    vfnmadd231sd {{.*#+}} xmm0 = -(xmm1 * mem) + xmm0 sched: [10:0.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER2-LABEL: test_vfnmaddsd_128:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    vfnmadd132sd {{.*#+}} xmm0 = -(xmm0 * xmm2) + xmm1 sched: [5:0.50]
-; BDVER2-NEXT:    vfnmadd213sd {{.*#+}} xmm0 = -(xmm1 * xmm0) + xmm2 sched: [5:0.50]
-; BDVER2-NEXT:    vfnmadd231sd {{.*#+}} xmm0 = -(xmm1 * xmm2) + xmm0 sched: [5:0.50]
-; BDVER2-NEXT:    vfnmadd132sd {{.*#+}} xmm0 = -(xmm0 * mem) + xmm1 sched: [10:0.50]
-; BDVER2-NEXT:    vfnmadd213sd {{.*#+}} xmm0 = -(xmm1 * xmm0) + mem sched: [10:0.50]
-; BDVER2-NEXT:    vfnmadd231sd {{.*#+}} xmm0 = -(xmm1 * mem) + xmm0 sched: [10:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; HASWELL-LABEL: test_vfnmaddsd_128:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    vfnmadd132sd {{.*#+}} xmm0 = -(xmm0 * xmm2) + xmm1 sched: [5:0.50]
-; HASWELL-NEXT:    vfnmadd213sd {{.*#+}} xmm0 = -(xmm1 * xmm0) + xmm2 sched: [5:0.50]
-; HASWELL-NEXT:    vfnmadd231sd {{.*#+}} xmm0 = -(xmm1 * xmm2) + xmm0 sched: [5:0.50]
-; HASWELL-NEXT:    vfnmadd132sd {{.*#+}} xmm0 = -(xmm0 * mem) + xmm1 sched: [10:0.50]
-; HASWELL-NEXT:    vfnmadd213sd {{.*#+}} xmm0 = -(xmm1 * xmm0) + mem sched: [10:0.50]
-; HASWELL-NEXT:    vfnmadd231sd {{.*#+}} xmm0 = -(xmm1 * mem) + xmm0 sched: [10:0.50]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_vfnmaddsd_128:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    vfnmadd132sd {{.*#+}} xmm0 = -(xmm0 * xmm2) + xmm1 sched: [5:0.50]
-; BROADWELL-NEXT:    vfnmadd213sd {{.*#+}} xmm0 = -(xmm1 * xmm0) + xmm2 sched: [5:0.50]
-; BROADWELL-NEXT:    vfnmadd231sd {{.*#+}} xmm0 = -(xmm1 * xmm2) + xmm0 sched: [5:0.50]
-; BROADWELL-NEXT:    vfnmadd132sd {{.*#+}} xmm0 = -(xmm0 * mem) + xmm1 sched: [10:0.50]
-; BROADWELL-NEXT:    vfnmadd213sd {{.*#+}} xmm0 = -(xmm1 * xmm0) + mem sched: [10:0.50]
-; BROADWELL-NEXT:    vfnmadd231sd {{.*#+}} xmm0 = -(xmm1 * mem) + xmm0 sched: [10:0.50]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_vfnmaddsd_128:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    vfnmadd132sd {{.*#+}} xmm0 = -(xmm0 * xmm2) + xmm1 sched: [4:0.50]
-; SKYLAKE-NEXT:    vfnmadd213sd {{.*#+}} xmm0 = -(xmm1 * xmm0) + xmm2 sched: [4:0.50]
-; SKYLAKE-NEXT:    vfnmadd231sd {{.*#+}} xmm0 = -(xmm1 * xmm2) + xmm0 sched: [4:0.50]
-; SKYLAKE-NEXT:    vfnmadd132sd {{.*#+}} xmm0 = -(xmm0 * mem) + xmm1 sched: [9:0.50]
-; SKYLAKE-NEXT:    vfnmadd213sd {{.*#+}} xmm0 = -(xmm1 * xmm0) + mem sched: [9:0.50]
-; SKYLAKE-NEXT:    vfnmadd231sd {{.*#+}} xmm0 = -(xmm1 * mem) + xmm0 sched: [9:0.50]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; KNL-LABEL: test_vfnmaddsd_128:
-; KNL:       # %bb.0:
-; KNL-NEXT:    #APP
-; KNL-NEXT:    vfnmadd132sd {{.*#+}} xmm0 = -(xmm0 * xmm2) + xmm1 sched: [5:0.50]
-; KNL-NEXT:    vfnmadd213sd {{.*#+}} xmm0 = -(xmm1 * xmm0) + xmm2 sched: [5:0.50]
-; KNL-NEXT:    vfnmadd231sd {{.*#+}} xmm0 = -(xmm1 * xmm2) + xmm0 sched: [5:0.50]
-; KNL-NEXT:    vfnmadd132sd {{.*#+}} xmm0 = -(xmm0 * mem) + xmm1 sched: [10:0.50]
-; KNL-NEXT:    vfnmadd213sd {{.*#+}} xmm0 = -(xmm1 * xmm0) + mem sched: [10:0.50]
-; KNL-NEXT:    vfnmadd231sd {{.*#+}} xmm0 = -(xmm1 * mem) + xmm0 sched: [10:0.50]
-; KNL-NEXT:    #NO_APP
-; KNL-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_vfnmaddsd_128:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    vfnmadd132sd {{.*#+}} xmm0 = -(xmm0 * xmm2) + xmm1 sched: [4:0.50]
-; SKX-NEXT:    vfnmadd213sd {{.*#+}} xmm0 = -(xmm1 * xmm0) + xmm2 sched: [4:0.50]
-; SKX-NEXT:    vfnmadd231sd {{.*#+}} xmm0 = -(xmm1 * xmm2) + xmm0 sched: [4:0.50]
-; SKX-NEXT:    vfnmadd132sd {{.*#+}} xmm0 = -(xmm0 * mem) + xmm1 sched: [9:0.50]
-; SKX-NEXT:    vfnmadd213sd {{.*#+}} xmm0 = -(xmm1 * xmm0) + mem sched: [9:0.50]
-; SKX-NEXT:    vfnmadd231sd {{.*#+}} xmm0 = -(xmm1 * mem) + xmm0 sched: [9:0.50]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_vfnmaddsd_128:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    vfnmadd132sd {{.*#+}} xmm0 = -(xmm0 * xmm2) + xmm1 sched: [5:0.50]
-; ZNVER1-NEXT:    vfnmadd213sd {{.*#+}} xmm0 = -(xmm1 * xmm0) + xmm2 sched: [5:0.50]
-; ZNVER1-NEXT:    vfnmadd231sd {{.*#+}} xmm0 = -(xmm1 * xmm2) + xmm0 sched: [5:0.50]
-; ZNVER1-NEXT:    vfnmadd132sd {{.*#+}} xmm0 = -(xmm0 * mem) + xmm1 sched: [12:0.50]
-; ZNVER1-NEXT:    vfnmadd213sd {{.*#+}} xmm0 = -(xmm1 * xmm0) + mem sched: [12:0.50]
-; ZNVER1-NEXT:    vfnmadd231sd {{.*#+}} xmm0 = -(xmm1 * mem) + xmm0 sched: [12:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  tail call void asm "vfnmadd132sd $2, $1, $0 \0A\09 vfnmadd213sd $2, $1, $0 \0A\09 vfnmadd231sd $2, $1, $0 \0A\09 vfnmadd132sd $3, $1, $0 \0A\09 vfnmadd213sd $3, $1, $0 \0A\09 vfnmadd231sd $3, $1, $0", "x,x,x,*m"(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, <2 x double> *%a3) nounwind
-  ret void
-}
-
-define void @test_vfnmaddss_128(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, <4 x float> *%a3) optsize {
-; GENERIC-LABEL: test_vfnmaddss_128:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    vfnmadd132ss {{.*#+}} xmm0 = -(xmm0 * xmm2) + xmm1 sched: [5:0.50]
-; GENERIC-NEXT:    vfnmadd213ss {{.*#+}} xmm0 = -(xmm1 * xmm0) + xmm2 sched: [5:0.50]
-; GENERIC-NEXT:    vfnmadd231ss {{.*#+}} xmm0 = -(xmm1 * xmm2) + xmm0 sched: [5:0.50]
-; GENERIC-NEXT:    vfnmadd132ss {{.*#+}} xmm0 = -(xmm0 * mem) + xmm1 sched: [10:0.50]
-; GENERIC-NEXT:    vfnmadd213ss {{.*#+}} xmm0 = -(xmm1 * xmm0) + mem sched: [10:0.50]
-; GENERIC-NEXT:    vfnmadd231ss {{.*#+}} xmm0 = -(xmm1 * mem) + xmm0 sched: [10:0.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER2-LABEL: test_vfnmaddss_128:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    vfnmadd132ss {{.*#+}} xmm0 = -(xmm0 * xmm2) + xmm1 sched: [5:0.50]
-; BDVER2-NEXT:    vfnmadd213ss {{.*#+}} xmm0 = -(xmm1 * xmm0) + xmm2 sched: [5:0.50]
-; BDVER2-NEXT:    vfnmadd231ss {{.*#+}} xmm0 = -(xmm1 * xmm2) + xmm0 sched: [5:0.50]
-; BDVER2-NEXT:    vfnmadd132ss {{.*#+}} xmm0 = -(xmm0 * mem) + xmm1 sched: [10:0.50]
-; BDVER2-NEXT:    vfnmadd213ss {{.*#+}} xmm0 = -(xmm1 * xmm0) + mem sched: [10:0.50]
-; BDVER2-NEXT:    vfnmadd231ss {{.*#+}} xmm0 = -(xmm1 * mem) + xmm0 sched: [10:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; HASWELL-LABEL: test_vfnmaddss_128:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    vfnmadd132ss {{.*#+}} xmm0 = -(xmm0 * xmm2) + xmm1 sched: [5:0.50]
-; HASWELL-NEXT:    vfnmadd213ss {{.*#+}} xmm0 = -(xmm1 * xmm0) + xmm2 sched: [5:0.50]
-; HASWELL-NEXT:    vfnmadd231ss {{.*#+}} xmm0 = -(xmm1 * xmm2) + xmm0 sched: [5:0.50]
-; HASWELL-NEXT:    vfnmadd132ss {{.*#+}} xmm0 = -(xmm0 * mem) + xmm1 sched: [10:0.50]
-; HASWELL-NEXT:    vfnmadd213ss {{.*#+}} xmm0 = -(xmm1 * xmm0) + mem sched: [10:0.50]
-; HASWELL-NEXT:    vfnmadd231ss {{.*#+}} xmm0 = -(xmm1 * mem) + xmm0 sched: [10:0.50]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_vfnmaddss_128:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    vfnmadd132ss {{.*#+}} xmm0 = -(xmm0 * xmm2) + xmm1 sched: [5:0.50]
-; BROADWELL-NEXT:    vfnmadd213ss {{.*#+}} xmm0 = -(xmm1 * xmm0) + xmm2 sched: [5:0.50]
-; BROADWELL-NEXT:    vfnmadd231ss {{.*#+}} xmm0 = -(xmm1 * xmm2) + xmm0 sched: [5:0.50]
-; BROADWELL-NEXT:    vfnmadd132ss {{.*#+}} xmm0 = -(xmm0 * mem) + xmm1 sched: [10:0.50]
-; BROADWELL-NEXT:    vfnmadd213ss {{.*#+}} xmm0 = -(xmm1 * xmm0) + mem sched: [10:0.50]
-; BROADWELL-NEXT:    vfnmadd231ss {{.*#+}} xmm0 = -(xmm1 * mem) + xmm0 sched: [10:0.50]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_vfnmaddss_128:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    vfnmadd132ss {{.*#+}} xmm0 = -(xmm0 * xmm2) + xmm1 sched: [4:0.50]
-; SKYLAKE-NEXT:    vfnmadd213ss {{.*#+}} xmm0 = -(xmm1 * xmm0) + xmm2 sched: [4:0.50]
-; SKYLAKE-NEXT:    vfnmadd231ss {{.*#+}} xmm0 = -(xmm1 * xmm2) + xmm0 sched: [4:0.50]
-; SKYLAKE-NEXT:    vfnmadd132ss {{.*#+}} xmm0 = -(xmm0 * mem) + xmm1 sched: [9:0.50]
-; SKYLAKE-NEXT:    vfnmadd213ss {{.*#+}} xmm0 = -(xmm1 * xmm0) + mem sched: [9:0.50]
-; SKYLAKE-NEXT:    vfnmadd231ss {{.*#+}} xmm0 = -(xmm1 * mem) + xmm0 sched: [9:0.50]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; KNL-LABEL: test_vfnmaddss_128:
-; KNL:       # %bb.0:
-; KNL-NEXT:    #APP
-; KNL-NEXT:    vfnmadd132ss {{.*#+}} xmm0 = -(xmm0 * xmm2) + xmm1 sched: [5:0.50]
-; KNL-NEXT:    vfnmadd213ss {{.*#+}} xmm0 = -(xmm1 * xmm0) + xmm2 sched: [5:0.50]
-; KNL-NEXT:    vfnmadd231ss {{.*#+}} xmm0 = -(xmm1 * xmm2) + xmm0 sched: [5:0.50]
-; KNL-NEXT:    vfnmadd132ss {{.*#+}} xmm0 = -(xmm0 * mem) + xmm1 sched: [10:0.50]
-; KNL-NEXT:    vfnmadd213ss {{.*#+}} xmm0 = -(xmm1 * xmm0) + mem sched: [10:0.50]
-; KNL-NEXT:    vfnmadd231ss {{.*#+}} xmm0 = -(xmm1 * mem) + xmm0 sched: [10:0.50]
-; KNL-NEXT:    #NO_APP
-; KNL-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_vfnmaddss_128:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    vfnmadd132ss {{.*#+}} xmm0 = -(xmm0 * xmm2) + xmm1 sched: [4:0.50]
-; SKX-NEXT:    vfnmadd213ss {{.*#+}} xmm0 = -(xmm1 * xmm0) + xmm2 sched: [4:0.50]
-; SKX-NEXT:    vfnmadd231ss {{.*#+}} xmm0 = -(xmm1 * xmm2) + xmm0 sched: [4:0.50]
-; SKX-NEXT:    vfnmadd132ss {{.*#+}} xmm0 = -(xmm0 * mem) + xmm1 sched: [9:0.50]
-; SKX-NEXT:    vfnmadd213ss {{.*#+}} xmm0 = -(xmm1 * xmm0) + mem sched: [9:0.50]
-; SKX-NEXT:    vfnmadd231ss {{.*#+}} xmm0 = -(xmm1 * mem) + xmm0 sched: [9:0.50]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_vfnmaddss_128:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    vfnmadd132ss {{.*#+}} xmm0 = -(xmm0 * xmm2) + xmm1 sched: [5:0.50]
-; ZNVER1-NEXT:    vfnmadd213ss {{.*#+}} xmm0 = -(xmm1 * xmm0) + xmm2 sched: [5:0.50]
-; ZNVER1-NEXT:    vfnmadd231ss {{.*#+}} xmm0 = -(xmm1 * xmm2) + xmm0 sched: [5:0.50]
-; ZNVER1-NEXT:    vfnmadd132ss {{.*#+}} xmm0 = -(xmm0 * mem) + xmm1 sched: [12:0.50]
-; ZNVER1-NEXT:    vfnmadd213ss {{.*#+}} xmm0 = -(xmm1 * xmm0) + mem sched: [12:0.50]
-; ZNVER1-NEXT:    vfnmadd231ss {{.*#+}} xmm0 = -(xmm1 * mem) + xmm0 sched: [12:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  tail call void asm "vfnmadd132ss $2, $1, $0 \0A\09 vfnmadd213ss $2, $1, $0 \0A\09 vfnmadd231ss $2, $1, $0 \0A\09 vfnmadd132ss $3, $1, $0 \0A\09 vfnmadd213ss $3, $1, $0 \0A\09 vfnmadd231ss $3, $1, $0", "x,x,x,*m"(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, <4 x float> *%a3) nounwind
-  ret void
-}
-
-;
-; VFNMSUB
-;
-
-define void @test_vfnmsubpd_128(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, <2 x double> *%a3) optsize {
-; GENERIC-LABEL: test_vfnmsubpd_128:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    vfnmsub132pd {{.*#+}} xmm0 = -(xmm0 * xmm2) - xmm1 sched: [5:0.50]
-; GENERIC-NEXT:    vfnmsub213pd {{.*#+}} xmm0 = -(xmm1 * xmm0) - xmm2 sched: [5:0.50]
-; GENERIC-NEXT:    vfnmsub231pd {{.*#+}} xmm0 = -(xmm1 * xmm2) - xmm0 sched: [5:0.50]
-; GENERIC-NEXT:    vfnmsub132pd {{.*#+}} xmm0 = -(xmm0 * mem) - xmm1 sched: [10:0.50]
-; GENERIC-NEXT:    vfnmsub213pd {{.*#+}} xmm0 = -(xmm1 * xmm0) - mem sched: [10:0.50]
-; GENERIC-NEXT:    vfnmsub231pd {{.*#+}} xmm0 = -(xmm1 * mem) - xmm0 sched: [10:0.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER2-LABEL: test_vfnmsubpd_128:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    vfnmsub132pd {{.*#+}} xmm0 = -(xmm0 * xmm2) - xmm1 sched: [5:0.50]
-; BDVER2-NEXT:    vfnmsub213pd {{.*#+}} xmm0 = -(xmm1 * xmm0) - xmm2 sched: [5:0.50]
-; BDVER2-NEXT:    vfnmsub231pd {{.*#+}} xmm0 = -(xmm1 * xmm2) - xmm0 sched: [5:0.50]
-; BDVER2-NEXT:    vfnmsub132pd {{.*#+}} xmm0 = -(xmm0 * mem) - xmm1 sched: [10:0.50]
-; BDVER2-NEXT:    vfnmsub213pd {{.*#+}} xmm0 = -(xmm1 * xmm0) - mem sched: [10:0.50]
-; BDVER2-NEXT:    vfnmsub231pd {{.*#+}} xmm0 = -(xmm1 * mem) - xmm0 sched: [10:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; HASWELL-LABEL: test_vfnmsubpd_128:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    vfnmsub132pd {{.*#+}} xmm0 = -(xmm0 * xmm2) - xmm1 sched: [5:0.50]
-; HASWELL-NEXT:    vfnmsub213pd {{.*#+}} xmm0 = -(xmm1 * xmm0) - xmm2 sched: [5:0.50]
-; HASWELL-NEXT:    vfnmsub231pd {{.*#+}} xmm0 = -(xmm1 * xmm2) - xmm0 sched: [5:0.50]
-; HASWELL-NEXT:    vfnmsub132pd {{.*#+}} xmm0 = -(xmm0 * mem) - xmm1 sched: [11:0.50]
-; HASWELL-NEXT:    vfnmsub213pd {{.*#+}} xmm0 = -(xmm1 * xmm0) - mem sched: [11:0.50]
-; HASWELL-NEXT:    vfnmsub231pd {{.*#+}} xmm0 = -(xmm1 * mem) - xmm0 sched: [11:0.50]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_vfnmsubpd_128:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    vfnmsub132pd {{.*#+}} xmm0 = -(xmm0 * xmm2) - xmm1 sched: [5:0.50]
-; BROADWELL-NEXT:    vfnmsub213pd {{.*#+}} xmm0 = -(xmm1 * xmm0) - xmm2 sched: [5:0.50]
-; BROADWELL-NEXT:    vfnmsub231pd {{.*#+}} xmm0 = -(xmm1 * xmm2) - xmm0 sched: [5:0.50]
-; BROADWELL-NEXT:    vfnmsub132pd {{.*#+}} xmm0 = -(xmm0 * mem) - xmm1 sched: [10:0.50]
-; BROADWELL-NEXT:    vfnmsub213pd {{.*#+}} xmm0 = -(xmm1 * xmm0) - mem sched: [10:0.50]
-; BROADWELL-NEXT:    vfnmsub231pd {{.*#+}} xmm0 = -(xmm1 * mem) - xmm0 sched: [10:0.50]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_vfnmsubpd_128:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    vfnmsub132pd {{.*#+}} xmm0 = -(xmm0 * xmm2) - xmm1 sched: [4:0.50]
-; SKYLAKE-NEXT:    vfnmsub213pd {{.*#+}} xmm0 = -(xmm1 * xmm0) - xmm2 sched: [4:0.50]
-; SKYLAKE-NEXT:    vfnmsub231pd {{.*#+}} xmm0 = -(xmm1 * xmm2) - xmm0 sched: [4:0.50]
-; SKYLAKE-NEXT:    vfnmsub132pd {{.*#+}} xmm0 = -(xmm0 * mem) - xmm1 sched: [10:0.50]
-; SKYLAKE-NEXT:    vfnmsub213pd {{.*#+}} xmm0 = -(xmm1 * xmm0) - mem sched: [10:0.50]
-; SKYLAKE-NEXT:    vfnmsub231pd {{.*#+}} xmm0 = -(xmm1 * mem) - xmm0 sched: [10:0.50]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; KNL-LABEL: test_vfnmsubpd_128:
-; KNL:       # %bb.0:
-; KNL-NEXT:    #APP
-; KNL-NEXT:    vfnmsub132pd {{.*#+}} xmm0 = -(xmm0 * xmm2) - xmm1 sched: [5:0.50]
-; KNL-NEXT:    vfnmsub213pd {{.*#+}} xmm0 = -(xmm1 * xmm0) - xmm2 sched: [5:0.50]
-; KNL-NEXT:    vfnmsub231pd {{.*#+}} xmm0 = -(xmm1 * xmm2) - xmm0 sched: [5:0.50]
-; KNL-NEXT:    vfnmsub132pd {{.*#+}} xmm0 = -(xmm0 * mem) - xmm1 sched: [11:0.50]
-; KNL-NEXT:    vfnmsub213pd {{.*#+}} xmm0 = -(xmm1 * xmm0) - mem sched: [11:0.50]
-; KNL-NEXT:    vfnmsub231pd {{.*#+}} xmm0 = -(xmm1 * mem) - xmm0 sched: [11:0.50]
-; KNL-NEXT:    #NO_APP
-; KNL-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_vfnmsubpd_128:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    vfnmsub132pd {{.*#+}} xmm0 = -(xmm0 * xmm2) - xmm1 sched: [4:0.50]
-; SKX-NEXT:    vfnmsub213pd {{.*#+}} xmm0 = -(xmm1 * xmm0) - xmm2 sched: [4:0.50]
-; SKX-NEXT:    vfnmsub231pd {{.*#+}} xmm0 = -(xmm1 * xmm2) - xmm0 sched: [4:0.50]
-; SKX-NEXT:    vfnmsub132pd {{.*#+}} xmm0 = -(xmm0 * mem) - xmm1 sched: [10:0.50]
-; SKX-NEXT:    vfnmsub213pd {{.*#+}} xmm0 = -(xmm1 * xmm0) - mem sched: [10:0.50]
-; SKX-NEXT:    vfnmsub231pd {{.*#+}} xmm0 = -(xmm1 * mem) - xmm0 sched: [10:0.50]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_vfnmsubpd_128:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    vfnmsub132pd {{.*#+}} xmm0 = -(xmm0 * xmm2) - xmm1 sched: [5:0.50]
-; ZNVER1-NEXT:    vfnmsub213pd {{.*#+}} xmm0 = -(xmm1 * xmm0) - xmm2 sched: [5:0.50]
-; ZNVER1-NEXT:    vfnmsub231pd {{.*#+}} xmm0 = -(xmm1 * xmm2) - xmm0 sched: [5:0.50]
-; ZNVER1-NEXT:    vfnmsub132pd {{.*#+}} xmm0 = -(xmm0 * mem) - xmm1 sched: [12:0.50]
-; ZNVER1-NEXT:    vfnmsub213pd {{.*#+}} xmm0 = -(xmm1 * xmm0) - mem sched: [12:0.50]
-; ZNVER1-NEXT:    vfnmsub231pd {{.*#+}} xmm0 = -(xmm1 * mem) - xmm0 sched: [12:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  tail call void asm "vfnmsub132pd $2, $1, $0 \0A\09 vfnmsub213pd $2, $1, $0 \0A\09 vfnmsub231pd $2, $1, $0 \0A\09 vfnmsub132pd $3, $1, $0 \0A\09 vfnmsub213pd $3, $1, $0 \0A\09 vfnmsub231pd $3, $1, $0", "x,x,x,*m"(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, <2 x double> *%a3) nounwind
-  ret void
-}
-
-define void @test_vfnmsubpd_256(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2, <4 x double> *%a3) optsize {
-; GENERIC-LABEL: test_vfnmsubpd_256:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    vfnmsub132pd {{.*#+}} ymm0 = -(ymm0 * ymm2) - ymm1 sched: [5:0.50]
-; GENERIC-NEXT:    vfnmsub213pd {{.*#+}} ymm0 = -(ymm1 * ymm0) - ymm2 sched: [5:0.50]
-; GENERIC-NEXT:    vfnmsub231pd {{.*#+}} ymm0 = -(ymm1 * ymm2) - ymm0 sched: [5:0.50]
-; GENERIC-NEXT:    vfnmsub132pd {{.*#+}} ymm0 = -(ymm0 * mem) - ymm1 sched: [10:0.50]
-; GENERIC-NEXT:    vfnmsub213pd {{.*#+}} ymm0 = -(ymm1 * ymm0) - mem sched: [10:0.50]
-; GENERIC-NEXT:    vfnmsub231pd {{.*#+}} ymm0 = -(ymm1 * mem) - ymm0 sched: [10:0.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    vzeroupper # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER2-LABEL: test_vfnmsubpd_256:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    vfnmsub132pd {{.*#+}} ymm0 = -(ymm0 * ymm2) - ymm1 sched: [5:0.50]
-; BDVER2-NEXT:    vfnmsub213pd {{.*#+}} ymm0 = -(ymm1 * ymm0) - ymm2 sched: [5:0.50]
-; BDVER2-NEXT:    vfnmsub231pd {{.*#+}} ymm0 = -(ymm1 * ymm2) - ymm0 sched: [5:0.50]
-; BDVER2-NEXT:    vfnmsub132pd {{.*#+}} ymm0 = -(ymm0 * mem) - ymm1 sched: [10:1.00]
-; BDVER2-NEXT:    vfnmsub213pd {{.*#+}} ymm0 = -(ymm1 * ymm0) - mem sched: [10:1.00]
-; BDVER2-NEXT:    vfnmsub231pd {{.*#+}} ymm0 = -(ymm1 * mem) - ymm0 sched: [10:1.00]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    vzeroupper # sched: [46:4.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; HASWELL-LABEL: test_vfnmsubpd_256:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    vfnmsub132pd {{.*#+}} ymm0 = -(ymm0 * ymm2) - ymm1 sched: [5:0.50]
-; HASWELL-NEXT:    vfnmsub213pd {{.*#+}} ymm0 = -(ymm1 * ymm0) - ymm2 sched: [5:0.50]
-; HASWELL-NEXT:    vfnmsub231pd {{.*#+}} ymm0 = -(ymm1 * ymm2) - ymm0 sched: [5:0.50]
-; HASWELL-NEXT:    vfnmsub132pd {{.*#+}} ymm0 = -(ymm0 * mem) - ymm1 sched: [12:0.50]
-; HASWELL-NEXT:    vfnmsub213pd {{.*#+}} ymm0 = -(ymm1 * ymm0) - mem sched: [12:0.50]
-; HASWELL-NEXT:    vfnmsub231pd {{.*#+}} ymm0 = -(ymm1 * mem) - ymm0 sched: [12:0.50]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    vzeroupper # sched: [0:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_vfnmsubpd_256:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    vfnmsub132pd {{.*#+}} ymm0 = -(ymm0 * ymm2) - ymm1 sched: [5:0.50]
-; BROADWELL-NEXT:    vfnmsub213pd {{.*#+}} ymm0 = -(ymm1 * ymm0) - ymm2 sched: [5:0.50]
-; BROADWELL-NEXT:    vfnmsub231pd {{.*#+}} ymm0 = -(ymm1 * ymm2) - ymm0 sched: [5:0.50]
-; BROADWELL-NEXT:    vfnmsub132pd {{.*#+}} ymm0 = -(ymm0 * mem) - ymm1 sched: [11:0.50]
-; BROADWELL-NEXT:    vfnmsub213pd {{.*#+}} ymm0 = -(ymm1 * ymm0) - mem sched: [11:0.50]
-; BROADWELL-NEXT:    vfnmsub231pd {{.*#+}} ymm0 = -(ymm1 * mem) - ymm0 sched: [11:0.50]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    vzeroupper # sched: [0:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_vfnmsubpd_256:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    vfnmsub132pd {{.*#+}} ymm0 = -(ymm0 * ymm2) - ymm1 sched: [4:0.50]
-; SKYLAKE-NEXT:    vfnmsub213pd {{.*#+}} ymm0 = -(ymm1 * ymm0) - ymm2 sched: [4:0.50]
-; SKYLAKE-NEXT:    vfnmsub231pd {{.*#+}} ymm0 = -(ymm1 * ymm2) - ymm0 sched: [4:0.50]
-; SKYLAKE-NEXT:    vfnmsub132pd {{.*#+}} ymm0 = -(ymm0 * mem) - ymm1 sched: [11:0.50]
-; SKYLAKE-NEXT:    vfnmsub213pd {{.*#+}} ymm0 = -(ymm1 * ymm0) - mem sched: [11:0.50]
-; SKYLAKE-NEXT:    vfnmsub231pd {{.*#+}} ymm0 = -(ymm1 * mem) - ymm0 sched: [11:0.50]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    vzeroupper # sched: [0:0.67]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; KNL-LABEL: test_vfnmsubpd_256:
-; KNL:       # %bb.0:
-; KNL-NEXT:    #APP
-; KNL-NEXT:    vfnmsub132pd {{.*#+}} ymm0 = -(ymm0 * ymm2) - ymm1 sched: [5:0.50]
-; KNL-NEXT:    vfnmsub213pd {{.*#+}} ymm0 = -(ymm1 * ymm0) - ymm2 sched: [5:0.50]
-; KNL-NEXT:    vfnmsub231pd {{.*#+}} ymm0 = -(ymm1 * ymm2) - ymm0 sched: [5:0.50]
-; KNL-NEXT:    vfnmsub132pd {{.*#+}} ymm0 = -(ymm0 * mem) - ymm1 sched: [12:0.50]
-; KNL-NEXT:    vfnmsub213pd {{.*#+}} ymm0 = -(ymm1 * ymm0) - mem sched: [12:0.50]
-; KNL-NEXT:    vfnmsub231pd {{.*#+}} ymm0 = -(ymm1 * mem) - ymm0 sched: [12:0.50]
-; KNL-NEXT:    #NO_APP
-; KNL-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_vfnmsubpd_256:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    vfnmsub132pd {{.*#+}} ymm0 = -(ymm0 * ymm2) - ymm1 sched: [4:0.50]
-; SKX-NEXT:    vfnmsub213pd {{.*#+}} ymm0 = -(ymm1 * ymm0) - ymm2 sched: [4:0.50]
-; SKX-NEXT:    vfnmsub231pd {{.*#+}} ymm0 = -(ymm1 * ymm2) - ymm0 sched: [4:0.50]
-; SKX-NEXT:    vfnmsub132pd {{.*#+}} ymm0 = -(ymm0 * mem) - ymm1 sched: [11:0.50]
-; SKX-NEXT:    vfnmsub213pd {{.*#+}} ymm0 = -(ymm1 * ymm0) - mem sched: [11:0.50]
-; SKX-NEXT:    vfnmsub231pd {{.*#+}} ymm0 = -(ymm1 * mem) - ymm0 sched: [11:0.50]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    vzeroupper # sched: [0:0.67]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_vfnmsubpd_256:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    vfnmsub132pd {{.*#+}} ymm0 = -(ymm0 * ymm2) - ymm1 sched: [5:0.50]
-; ZNVER1-NEXT:    vfnmsub213pd {{.*#+}} ymm0 = -(ymm1 * ymm0) - ymm2 sched: [5:0.50]
-; ZNVER1-NEXT:    vfnmsub231pd {{.*#+}} ymm0 = -(ymm1 * ymm2) - ymm0 sched: [5:0.50]
-; ZNVER1-NEXT:    vfnmsub132pd {{.*#+}} ymm0 = -(ymm0 * mem) - ymm1 sched: [12:0.50]
-; ZNVER1-NEXT:    vfnmsub213pd {{.*#+}} ymm0 = -(ymm1 * ymm0) - mem sched: [12:0.50]
-; ZNVER1-NEXT:    vfnmsub231pd {{.*#+}} ymm0 = -(ymm1 * mem) - ymm0 sched: [12:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    vzeroupper # sched: [100:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  tail call void asm "vfnmsub132pd $2, $1, $0 \0A\09 vfnmsub213pd $2, $1, $0 \0A\09 vfnmsub231pd $2, $1, $0 \0A\09 vfnmsub132pd $3, $1, $0 \0A\09 vfnmsub213pd $3, $1, $0 \0A\09 vfnmsub231pd $3, $1, $0", "x,x,x,*m"(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2, <4 x double> *%a3) nounwind
-  ret void
-}
-
-define void @test_vfnmsubps_128(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, <4 x float> *%a3) optsize {
-; GENERIC-LABEL: test_vfnmsubps_128:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    vfnmsub132ps {{.*#+}} xmm0 = -(xmm0 * xmm2) - xmm1 sched: [5:0.50]
-; GENERIC-NEXT:    vfnmsub213ps {{.*#+}} xmm0 = -(xmm1 * xmm0) - xmm2 sched: [5:0.50]
-; GENERIC-NEXT:    vfnmsub231ps {{.*#+}} xmm0 = -(xmm1 * xmm2) - xmm0 sched: [5:0.50]
-; GENERIC-NEXT:    vfnmsub132ps {{.*#+}} xmm0 = -(xmm0 * mem) - xmm1 sched: [10:0.50]
-; GENERIC-NEXT:    vfnmsub213ps {{.*#+}} xmm0 = -(xmm1 * xmm0) - mem sched: [10:0.50]
-; GENERIC-NEXT:    vfnmsub231ps {{.*#+}} xmm0 = -(xmm1 * mem) - xmm0 sched: [10:0.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER2-LABEL: test_vfnmsubps_128:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    vfnmsub132ps {{.*#+}} xmm0 = -(xmm0 * xmm2) - xmm1 sched: [5:0.50]
-; BDVER2-NEXT:    vfnmsub213ps {{.*#+}} xmm0 = -(xmm1 * xmm0) - xmm2 sched: [5:0.50]
-; BDVER2-NEXT:    vfnmsub231ps {{.*#+}} xmm0 = -(xmm1 * xmm2) - xmm0 sched: [5:0.50]
-; BDVER2-NEXT:    vfnmsub132ps {{.*#+}} xmm0 = -(xmm0 * mem) - xmm1 sched: [10:0.50]
-; BDVER2-NEXT:    vfnmsub213ps {{.*#+}} xmm0 = -(xmm1 * xmm0) - mem sched: [10:0.50]
-; BDVER2-NEXT:    vfnmsub231ps {{.*#+}} xmm0 = -(xmm1 * mem) - xmm0 sched: [10:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; HASWELL-LABEL: test_vfnmsubps_128:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    vfnmsub132ps {{.*#+}} xmm0 = -(xmm0 * xmm2) - xmm1 sched: [5:0.50]
-; HASWELL-NEXT:    vfnmsub213ps {{.*#+}} xmm0 = -(xmm1 * xmm0) - xmm2 sched: [5:0.50]
-; HASWELL-NEXT:    vfnmsub231ps {{.*#+}} xmm0 = -(xmm1 * xmm2) - xmm0 sched: [5:0.50]
-; HASWELL-NEXT:    vfnmsub132ps {{.*#+}} xmm0 = -(xmm0 * mem) - xmm1 sched: [11:0.50]
-; HASWELL-NEXT:    vfnmsub213ps {{.*#+}} xmm0 = -(xmm1 * xmm0) - mem sched: [11:0.50]
-; HASWELL-NEXT:    vfnmsub231ps {{.*#+}} xmm0 = -(xmm1 * mem) - xmm0 sched: [11:0.50]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_vfnmsubps_128:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    vfnmsub132ps {{.*#+}} xmm0 = -(xmm0 * xmm2) - xmm1 sched: [5:0.50]
-; BROADWELL-NEXT:    vfnmsub213ps {{.*#+}} xmm0 = -(xmm1 * xmm0) - xmm2 sched: [5:0.50]
-; BROADWELL-NEXT:    vfnmsub231ps {{.*#+}} xmm0 = -(xmm1 * xmm2) - xmm0 sched: [5:0.50]
-; BROADWELL-NEXT:    vfnmsub132ps {{.*#+}} xmm0 = -(xmm0 * mem) - xmm1 sched: [10:0.50]
-; BROADWELL-NEXT:    vfnmsub213ps {{.*#+}} xmm0 = -(xmm1 * xmm0) - mem sched: [10:0.50]
-; BROADWELL-NEXT:    vfnmsub231ps {{.*#+}} xmm0 = -(xmm1 * mem) - xmm0 sched: [10:0.50]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_vfnmsubps_128:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    vfnmsub132ps {{.*#+}} xmm0 = -(xmm0 * xmm2) - xmm1 sched: [4:0.50]
-; SKYLAKE-NEXT:    vfnmsub213ps {{.*#+}} xmm0 = -(xmm1 * xmm0) - xmm2 sched: [4:0.50]
-; SKYLAKE-NEXT:    vfnmsub231ps {{.*#+}} xmm0 = -(xmm1 * xmm2) - xmm0 sched: [4:0.50]
-; SKYLAKE-NEXT:    vfnmsub132ps {{.*#+}} xmm0 = -(xmm0 * mem) - xmm1 sched: [10:0.50]
-; SKYLAKE-NEXT:    vfnmsub213ps {{.*#+}} xmm0 = -(xmm1 * xmm0) - mem sched: [10:0.50]
-; SKYLAKE-NEXT:    vfnmsub231ps {{.*#+}} xmm0 = -(xmm1 * mem) - xmm0 sched: [10:0.50]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; KNL-LABEL: test_vfnmsubps_128:
-; KNL:       # %bb.0:
-; KNL-NEXT:    #APP
-; KNL-NEXT:    vfnmsub132ps {{.*#+}} xmm0 = -(xmm0 * xmm2) - xmm1 sched: [5:0.50]
-; KNL-NEXT:    vfnmsub213ps {{.*#+}} xmm0 = -(xmm1 * xmm0) - xmm2 sched: [5:0.50]
-; KNL-NEXT:    vfnmsub231ps {{.*#+}} xmm0 = -(xmm1 * xmm2) - xmm0 sched: [5:0.50]
-; KNL-NEXT:    vfnmsub132ps {{.*#+}} xmm0 = -(xmm0 * mem) - xmm1 sched: [11:0.50]
-; KNL-NEXT:    vfnmsub213ps {{.*#+}} xmm0 = -(xmm1 * xmm0) - mem sched: [11:0.50]
-; KNL-NEXT:    vfnmsub231ps {{.*#+}} xmm0 = -(xmm1 * mem) - xmm0 sched: [11:0.50]
-; KNL-NEXT:    #NO_APP
-; KNL-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_vfnmsubps_128:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    vfnmsub132ps {{.*#+}} xmm0 = -(xmm0 * xmm2) - xmm1 sched: [4:0.50]
-; SKX-NEXT:    vfnmsub213ps {{.*#+}} xmm0 = -(xmm1 * xmm0) - xmm2 sched: [4:0.50]
-; SKX-NEXT:    vfnmsub231ps {{.*#+}} xmm0 = -(xmm1 * xmm2) - xmm0 sched: [4:0.50]
-; SKX-NEXT:    vfnmsub132ps {{.*#+}} xmm0 = -(xmm0 * mem) - xmm1 sched: [10:0.50]
-; SKX-NEXT:    vfnmsub213ps {{.*#+}} xmm0 = -(xmm1 * xmm0) - mem sched: [10:0.50]
-; SKX-NEXT:    vfnmsub231ps {{.*#+}} xmm0 = -(xmm1 * mem) - xmm0 sched: [10:0.50]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_vfnmsubps_128:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    vfnmsub132ps {{.*#+}} xmm0 = -(xmm0 * xmm2) - xmm1 sched: [5:0.50]
-; ZNVER1-NEXT:    vfnmsub213ps {{.*#+}} xmm0 = -(xmm1 * xmm0) - xmm2 sched: [5:0.50]
-; ZNVER1-NEXT:    vfnmsub231ps {{.*#+}} xmm0 = -(xmm1 * xmm2) - xmm0 sched: [5:0.50]
-; ZNVER1-NEXT:    vfnmsub132ps {{.*#+}} xmm0 = -(xmm0 * mem) - xmm1 sched: [12:0.50]
-; ZNVER1-NEXT:    vfnmsub213ps {{.*#+}} xmm0 = -(xmm1 * xmm0) - mem sched: [12:0.50]
-; ZNVER1-NEXT:    vfnmsub231ps {{.*#+}} xmm0 = -(xmm1 * mem) - xmm0 sched: [12:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  tail call void asm "vfnmsub132ps $2, $1, $0 \0A\09 vfnmsub213ps $2, $1, $0 \0A\09 vfnmsub231ps $2, $1, $0 \0A\09 vfnmsub132ps $3, $1, $0 \0A\09 vfnmsub213ps $3, $1, $0 \0A\09 vfnmsub231ps $3, $1, $0", "x,x,x,*m"(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, <4 x float> *%a3) nounwind
-  ret void
-}
-
-define void @test_vfnmsubps_256(<8 x float> %a0, <8 x float> %a1, <8 x float> %a2, <8 x float> *%a3) optsize {
-; GENERIC-LABEL: test_vfnmsubps_256:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    vfnmsub132ps {{.*#+}} ymm0 = -(ymm0 * ymm2) - ymm1 sched: [5:0.50]
-; GENERIC-NEXT:    vfnmsub213ps {{.*#+}} ymm0 = -(ymm1 * ymm0) - ymm2 sched: [5:0.50]
-; GENERIC-NEXT:    vfnmsub231ps {{.*#+}} ymm0 = -(ymm1 * ymm2) - ymm0 sched: [5:0.50]
-; GENERIC-NEXT:    vfnmsub132ps {{.*#+}} ymm0 = -(ymm0 * mem) - ymm1 sched: [10:0.50]
-; GENERIC-NEXT:    vfnmsub213ps {{.*#+}} ymm0 = -(ymm1 * ymm0) - mem sched: [10:0.50]
-; GENERIC-NEXT:    vfnmsub231ps {{.*#+}} ymm0 = -(ymm1 * mem) - ymm0 sched: [10:0.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    vzeroupper # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER2-LABEL: test_vfnmsubps_256:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    vfnmsub132ps {{.*#+}} ymm0 = -(ymm0 * ymm2) - ymm1 sched: [5:0.50]
-; BDVER2-NEXT:    vfnmsub213ps {{.*#+}} ymm0 = -(ymm1 * ymm0) - ymm2 sched: [5:0.50]
-; BDVER2-NEXT:    vfnmsub231ps {{.*#+}} ymm0 = -(ymm1 * ymm2) - ymm0 sched: [5:0.50]
-; BDVER2-NEXT:    vfnmsub132ps {{.*#+}} ymm0 = -(ymm0 * mem) - ymm1 sched: [10:1.00]
-; BDVER2-NEXT:    vfnmsub213ps {{.*#+}} ymm0 = -(ymm1 * ymm0) - mem sched: [10:1.00]
-; BDVER2-NEXT:    vfnmsub231ps {{.*#+}} ymm0 = -(ymm1 * mem) - ymm0 sched: [10:1.00]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    vzeroupper # sched: [46:4.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; HASWELL-LABEL: test_vfnmsubps_256:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    vfnmsub132ps {{.*#+}} ymm0 = -(ymm0 * ymm2) - ymm1 sched: [5:0.50]
-; HASWELL-NEXT:    vfnmsub213ps {{.*#+}} ymm0 = -(ymm1 * ymm0) - ymm2 sched: [5:0.50]
-; HASWELL-NEXT:    vfnmsub231ps {{.*#+}} ymm0 = -(ymm1 * ymm2) - ymm0 sched: [5:0.50]
-; HASWELL-NEXT:    vfnmsub132ps {{.*#+}} ymm0 = -(ymm0 * mem) - ymm1 sched: [12:0.50]
-; HASWELL-NEXT:    vfnmsub213ps {{.*#+}} ymm0 = -(ymm1 * ymm0) - mem sched: [12:0.50]
-; HASWELL-NEXT:    vfnmsub231ps {{.*#+}} ymm0 = -(ymm1 * mem) - ymm0 sched: [12:0.50]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    vzeroupper # sched: [0:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_vfnmsubps_256:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    vfnmsub132ps {{.*#+}} ymm0 = -(ymm0 * ymm2) - ymm1 sched: [5:0.50]
-; BROADWELL-NEXT:    vfnmsub213ps {{.*#+}} ymm0 = -(ymm1 * ymm0) - ymm2 sched: [5:0.50]
-; BROADWELL-NEXT:    vfnmsub231ps {{.*#+}} ymm0 = -(ymm1 * ymm2) - ymm0 sched: [5:0.50]
-; BROADWELL-NEXT:    vfnmsub132ps {{.*#+}} ymm0 = -(ymm0 * mem) - ymm1 sched: [11:0.50]
-; BROADWELL-NEXT:    vfnmsub213ps {{.*#+}} ymm0 = -(ymm1 * ymm0) - mem sched: [11:0.50]
-; BROADWELL-NEXT:    vfnmsub231ps {{.*#+}} ymm0 = -(ymm1 * mem) - ymm0 sched: [11:0.50]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    vzeroupper # sched: [0:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_vfnmsubps_256:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    vfnmsub132ps {{.*#+}} ymm0 = -(ymm0 * ymm2) - ymm1 sched: [4:0.50]
-; SKYLAKE-NEXT:    vfnmsub213ps {{.*#+}} ymm0 = -(ymm1 * ymm0) - ymm2 sched: [4:0.50]
-; SKYLAKE-NEXT:    vfnmsub231ps {{.*#+}} ymm0 = -(ymm1 * ymm2) - ymm0 sched: [4:0.50]
-; SKYLAKE-NEXT:    vfnmsub132ps {{.*#+}} ymm0 = -(ymm0 * mem) - ymm1 sched: [11:0.50]
-; SKYLAKE-NEXT:    vfnmsub213ps {{.*#+}} ymm0 = -(ymm1 * ymm0) - mem sched: [11:0.50]
-; SKYLAKE-NEXT:    vfnmsub231ps {{.*#+}} ymm0 = -(ymm1 * mem) - ymm0 sched: [11:0.50]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    vzeroupper # sched: [0:0.67]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; KNL-LABEL: test_vfnmsubps_256:
-; KNL:       # %bb.0:
-; KNL-NEXT:    #APP
-; KNL-NEXT:    vfnmsub132ps {{.*#+}} ymm0 = -(ymm0 * ymm2) - ymm1 sched: [5:0.50]
-; KNL-NEXT:    vfnmsub213ps {{.*#+}} ymm0 = -(ymm1 * ymm0) - ymm2 sched: [5:0.50]
-; KNL-NEXT:    vfnmsub231ps {{.*#+}} ymm0 = -(ymm1 * ymm2) - ymm0 sched: [5:0.50]
-; KNL-NEXT:    vfnmsub132ps {{.*#+}} ymm0 = -(ymm0 * mem) - ymm1 sched: [12:0.50]
-; KNL-NEXT:    vfnmsub213ps {{.*#+}} ymm0 = -(ymm1 * ymm0) - mem sched: [12:0.50]
-; KNL-NEXT:    vfnmsub231ps {{.*#+}} ymm0 = -(ymm1 * mem) - ymm0 sched: [12:0.50]
-; KNL-NEXT:    #NO_APP
-; KNL-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_vfnmsubps_256:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    vfnmsub132ps {{.*#+}} ymm0 = -(ymm0 * ymm2) - ymm1 sched: [4:0.50]
-; SKX-NEXT:    vfnmsub213ps {{.*#+}} ymm0 = -(ymm1 * ymm0) - ymm2 sched: [4:0.50]
-; SKX-NEXT:    vfnmsub231ps {{.*#+}} ymm0 = -(ymm1 * ymm2) - ymm0 sched: [4:0.50]
-; SKX-NEXT:    vfnmsub132ps {{.*#+}} ymm0 = -(ymm0 * mem) - ymm1 sched: [11:0.50]
-; SKX-NEXT:    vfnmsub213ps {{.*#+}} ymm0 = -(ymm1 * ymm0) - mem sched: [11:0.50]
-; SKX-NEXT:    vfnmsub231ps {{.*#+}} ymm0 = -(ymm1 * mem) - ymm0 sched: [11:0.50]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    vzeroupper # sched: [0:0.67]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_vfnmsubps_256:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    vfnmsub132ps {{.*#+}} ymm0 = -(ymm0 * ymm2) - ymm1 sched: [5:0.50]
-; ZNVER1-NEXT:    vfnmsub213ps {{.*#+}} ymm0 = -(ymm1 * ymm0) - ymm2 sched: [5:0.50]
-; ZNVER1-NEXT:    vfnmsub231ps {{.*#+}} ymm0 = -(ymm1 * ymm2) - ymm0 sched: [5:0.50]
-; ZNVER1-NEXT:    vfnmsub132ps {{.*#+}} ymm0 = -(ymm0 * mem) - ymm1 sched: [12:0.50]
-; ZNVER1-NEXT:    vfnmsub213ps {{.*#+}} ymm0 = -(ymm1 * ymm0) - mem sched: [12:0.50]
-; ZNVER1-NEXT:    vfnmsub231ps {{.*#+}} ymm0 = -(ymm1 * mem) - ymm0 sched: [12:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    vzeroupper # sched: [100:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  tail call void asm "vfnmsub132ps $2, $1, $0 \0A\09 vfnmsub213ps $2, $1, $0 \0A\09 vfnmsub231ps $2, $1, $0 \0A\09 vfnmsub132ps $3, $1, $0 \0A\09 vfnmsub213ps $3, $1, $0 \0A\09 vfnmsub231ps $3, $1, $0", "x,x,x,*m"(<8 x float> %a0, <8 x float> %a1, <8 x float> %a2, <8 x float> *%a3) nounwind
-  ret void
-}
-
-define void @test_vfnmsubsd_128(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, <2 x double> *%a3) optsize {
-; GENERIC-LABEL: test_vfnmsubsd_128:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    vfnmsub132sd {{.*#+}} xmm0 = -(xmm0 * xmm2) - xmm1 sched: [5:0.50]
-; GENERIC-NEXT:    vfnmsub213sd {{.*#+}} xmm0 = -(xmm1 * xmm0) - xmm2 sched: [5:0.50]
-; GENERIC-NEXT:    vfnmsub231sd {{.*#+}} xmm0 = -(xmm1 * xmm2) - xmm0 sched: [5:0.50]
-; GENERIC-NEXT:    vfnmsub132sd {{.*#+}} xmm0 = -(xmm0 * mem) - xmm1 sched: [10:0.50]
-; GENERIC-NEXT:    vfnmsub213sd {{.*#+}} xmm0 = -(xmm1 * xmm0) - mem sched: [10:0.50]
-; GENERIC-NEXT:    vfnmsub231sd {{.*#+}} xmm0 = -(xmm1 * mem) - xmm0 sched: [10:0.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER2-LABEL: test_vfnmsubsd_128:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    vfnmsub132sd {{.*#+}} xmm0 = -(xmm0 * xmm2) - xmm1 sched: [5:0.50]
-; BDVER2-NEXT:    vfnmsub213sd {{.*#+}} xmm0 = -(xmm1 * xmm0) - xmm2 sched: [5:0.50]
-; BDVER2-NEXT:    vfnmsub231sd {{.*#+}} xmm0 = -(xmm1 * xmm2) - xmm0 sched: [5:0.50]
-; BDVER2-NEXT:    vfnmsub132sd {{.*#+}} xmm0 = -(xmm0 * mem) - xmm1 sched: [10:0.50]
-; BDVER2-NEXT:    vfnmsub213sd {{.*#+}} xmm0 = -(xmm1 * xmm0) - mem sched: [10:0.50]
-; BDVER2-NEXT:    vfnmsub231sd {{.*#+}} xmm0 = -(xmm1 * mem) - xmm0 sched: [10:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; HASWELL-LABEL: test_vfnmsubsd_128:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    vfnmsub132sd {{.*#+}} xmm0 = -(xmm0 * xmm2) - xmm1 sched: [5:0.50]
-; HASWELL-NEXT:    vfnmsub213sd {{.*#+}} xmm0 = -(xmm1 * xmm0) - xmm2 sched: [5:0.50]
-; HASWELL-NEXT:    vfnmsub231sd {{.*#+}} xmm0 = -(xmm1 * xmm2) - xmm0 sched: [5:0.50]
-; HASWELL-NEXT:    vfnmsub132sd {{.*#+}} xmm0 = -(xmm0 * mem) - xmm1 sched: [10:0.50]
-; HASWELL-NEXT:    vfnmsub213sd {{.*#+}} xmm0 = -(xmm1 * xmm0) - mem sched: [10:0.50]
-; HASWELL-NEXT:    vfnmsub231sd {{.*#+}} xmm0 = -(xmm1 * mem) - xmm0 sched: [10:0.50]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_vfnmsubsd_128:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    vfnmsub132sd {{.*#+}} xmm0 = -(xmm0 * xmm2) - xmm1 sched: [5:0.50]
-; BROADWELL-NEXT:    vfnmsub213sd {{.*#+}} xmm0 = -(xmm1 * xmm0) - xmm2 sched: [5:0.50]
-; BROADWELL-NEXT:    vfnmsub231sd {{.*#+}} xmm0 = -(xmm1 * xmm2) - xmm0 sched: [5:0.50]
-; BROADWELL-NEXT:    vfnmsub132sd {{.*#+}} xmm0 = -(xmm0 * mem) - xmm1 sched: [10:0.50]
-; BROADWELL-NEXT:    vfnmsub213sd {{.*#+}} xmm0 = -(xmm1 * xmm0) - mem sched: [10:0.50]
-; BROADWELL-NEXT:    vfnmsub231sd {{.*#+}} xmm0 = -(xmm1 * mem) - xmm0 sched: [10:0.50]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_vfnmsubsd_128:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    vfnmsub132sd {{.*#+}} xmm0 = -(xmm0 * xmm2) - xmm1 sched: [4:0.50]
-; SKYLAKE-NEXT:    vfnmsub213sd {{.*#+}} xmm0 = -(xmm1 * xmm0) - xmm2 sched: [4:0.50]
-; SKYLAKE-NEXT:    vfnmsub231sd {{.*#+}} xmm0 = -(xmm1 * xmm2) - xmm0 sched: [4:0.50]
-; SKYLAKE-NEXT:    vfnmsub132sd {{.*#+}} xmm0 = -(xmm0 * mem) - xmm1 sched: [9:0.50]
-; SKYLAKE-NEXT:    vfnmsub213sd {{.*#+}} xmm0 = -(xmm1 * xmm0) - mem sched: [9:0.50]
-; SKYLAKE-NEXT:    vfnmsub231sd {{.*#+}} xmm0 = -(xmm1 * mem) - xmm0 sched: [9:0.50]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; KNL-LABEL: test_vfnmsubsd_128:
-; KNL:       # %bb.0:
-; KNL-NEXT:    #APP
-; KNL-NEXT:    vfnmsub132sd {{.*#+}} xmm0 = -(xmm0 * xmm2) - xmm1 sched: [5:0.50]
-; KNL-NEXT:    vfnmsub213sd {{.*#+}} xmm0 = -(xmm1 * xmm0) - xmm2 sched: [5:0.50]
-; KNL-NEXT:    vfnmsub231sd {{.*#+}} xmm0 = -(xmm1 * xmm2) - xmm0 sched: [5:0.50]
-; KNL-NEXT:    vfnmsub132sd {{.*#+}} xmm0 = -(xmm0 * mem) - xmm1 sched: [10:0.50]
-; KNL-NEXT:    vfnmsub213sd {{.*#+}} xmm0 = -(xmm1 * xmm0) - mem sched: [10:0.50]
-; KNL-NEXT:    vfnmsub231sd {{.*#+}} xmm0 = -(xmm1 * mem) - xmm0 sched: [10:0.50]
-; KNL-NEXT:    #NO_APP
-; KNL-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_vfnmsubsd_128:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    vfnmsub132sd {{.*#+}} xmm0 = -(xmm0 * xmm2) - xmm1 sched: [4:0.50]
-; SKX-NEXT:    vfnmsub213sd {{.*#+}} xmm0 = -(xmm1 * xmm0) - xmm2 sched: [4:0.50]
-; SKX-NEXT:    vfnmsub231sd {{.*#+}} xmm0 = -(xmm1 * xmm2) - xmm0 sched: [4:0.50]
-; SKX-NEXT:    vfnmsub132sd {{.*#+}} xmm0 = -(xmm0 * mem) - xmm1 sched: [9:0.50]
-; SKX-NEXT:    vfnmsub213sd {{.*#+}} xmm0 = -(xmm1 * xmm0) - mem sched: [9:0.50]
-; SKX-NEXT:    vfnmsub231sd {{.*#+}} xmm0 = -(xmm1 * mem) - xmm0 sched: [9:0.50]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_vfnmsubsd_128:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    vfnmsub132sd {{.*#+}} xmm0 = -(xmm0 * xmm2) - xmm1 sched: [5:0.50]
-; ZNVER1-NEXT:    vfnmsub213sd {{.*#+}} xmm0 = -(xmm1 * xmm0) - xmm2 sched: [5:0.50]
-; ZNVER1-NEXT:    vfnmsub231sd {{.*#+}} xmm0 = -(xmm1 * xmm2) - xmm0 sched: [5:0.50]
-; ZNVER1-NEXT:    vfnmsub132sd {{.*#+}} xmm0 = -(xmm0 * mem) - xmm1 sched: [12:0.50]
-; ZNVER1-NEXT:    vfnmsub213sd {{.*#+}} xmm0 = -(xmm1 * xmm0) - mem sched: [12:0.50]
-; ZNVER1-NEXT:    vfnmsub231sd {{.*#+}} xmm0 = -(xmm1 * mem) - xmm0 sched: [12:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  tail call void asm "vfnmsub132sd $2, $1, $0 \0A\09 vfnmsub213sd $2, $1, $0 \0A\09 vfnmsub231sd $2, $1, $0 \0A\09 vfnmsub132sd $3, $1, $0 \0A\09 vfnmsub213sd $3, $1, $0 \0A\09 vfnmsub231sd $3, $1, $0", "x,x,x,*m"(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, <2 x double> *%a3) nounwind
-  ret void
-}
-
-define void @test_vfnmsubss_128(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, <4 x float> *%a3) optsize {
-; GENERIC-LABEL: test_vfnmsubss_128:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    vfnmsub132ss {{.*#+}} xmm0 = -(xmm0 * xmm2) - xmm1 sched: [5:0.50]
-; GENERIC-NEXT:    vfnmsub213ss {{.*#+}} xmm0 = -(xmm1 * xmm0) - xmm2 sched: [5:0.50]
-; GENERIC-NEXT:    vfnmsub231ss {{.*#+}} xmm0 = -(xmm1 * xmm2) - xmm0 sched: [5:0.50]
-; GENERIC-NEXT:    vfnmsub132ss {{.*#+}} xmm0 = -(xmm0 * mem) - xmm1 sched: [10:0.50]
-; GENERIC-NEXT:    vfnmsub213ss {{.*#+}} xmm0 = -(xmm1 * xmm0) - mem sched: [10:0.50]
-; GENERIC-NEXT:    vfnmsub231ss {{.*#+}} xmm0 = -(xmm1 * mem) - xmm0 sched: [10:0.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER2-LABEL: test_vfnmsubss_128:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    vfnmsub132ss {{.*#+}} xmm0 = -(xmm0 * xmm2) - xmm1 sched: [5:0.50]
-; BDVER2-NEXT:    vfnmsub213ss {{.*#+}} xmm0 = -(xmm1 * xmm0) - xmm2 sched: [5:0.50]
-; BDVER2-NEXT:    vfnmsub231ss {{.*#+}} xmm0 = -(xmm1 * xmm2) - xmm0 sched: [5:0.50]
-; BDVER2-NEXT:    vfnmsub132ss {{.*#+}} xmm0 = -(xmm0 * mem) - xmm1 sched: [10:0.50]
-; BDVER2-NEXT:    vfnmsub213ss {{.*#+}} xmm0 = -(xmm1 * xmm0) - mem sched: [10:0.50]
-; BDVER2-NEXT:    vfnmsub231ss {{.*#+}} xmm0 = -(xmm1 * mem) - xmm0 sched: [10:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; HASWELL-LABEL: test_vfnmsubss_128:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    vfnmsub132ss {{.*#+}} xmm0 = -(xmm0 * xmm2) - xmm1 sched: [5:0.50]
-; HASWELL-NEXT:    vfnmsub213ss {{.*#+}} xmm0 = -(xmm1 * xmm0) - xmm2 sched: [5:0.50]
-; HASWELL-NEXT:    vfnmsub231ss {{.*#+}} xmm0 = -(xmm1 * xmm2) - xmm0 sched: [5:0.50]
-; HASWELL-NEXT:    vfnmsub132ss {{.*#+}} xmm0 = -(xmm0 * mem) - xmm1 sched: [10:0.50]
-; HASWELL-NEXT:    vfnmsub213ss {{.*#+}} xmm0 = -(xmm1 * xmm0) - mem sched: [10:0.50]
-; HASWELL-NEXT:    vfnmsub231ss {{.*#+}} xmm0 = -(xmm1 * mem) - xmm0 sched: [10:0.50]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_vfnmsubss_128:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    vfnmsub132ss {{.*#+}} xmm0 = -(xmm0 * xmm2) - xmm1 sched: [5:0.50]
-; BROADWELL-NEXT:    vfnmsub213ss {{.*#+}} xmm0 = -(xmm1 * xmm0) - xmm2 sched: [5:0.50]
-; BROADWELL-NEXT:    vfnmsub231ss {{.*#+}} xmm0 = -(xmm1 * xmm2) - xmm0 sched: [5:0.50]
-; BROADWELL-NEXT:    vfnmsub132ss {{.*#+}} xmm0 = -(xmm0 * mem) - xmm1 sched: [10:0.50]
-; BROADWELL-NEXT:    vfnmsub213ss {{.*#+}} xmm0 = -(xmm1 * xmm0) - mem sched: [10:0.50]
-; BROADWELL-NEXT:    vfnmsub231ss {{.*#+}} xmm0 = -(xmm1 * mem) - xmm0 sched: [10:0.50]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_vfnmsubss_128:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    vfnmsub132ss {{.*#+}} xmm0 = -(xmm0 * xmm2) - xmm1 sched: [4:0.50]
-; SKYLAKE-NEXT:    vfnmsub213ss {{.*#+}} xmm0 = -(xmm1 * xmm0) - xmm2 sched: [4:0.50]
-; SKYLAKE-NEXT:    vfnmsub231ss {{.*#+}} xmm0 = -(xmm1 * xmm2) - xmm0 sched: [4:0.50]
-; SKYLAKE-NEXT:    vfnmsub132ss {{.*#+}} xmm0 = -(xmm0 * mem) - xmm1 sched: [9:0.50]
-; SKYLAKE-NEXT:    vfnmsub213ss {{.*#+}} xmm0 = -(xmm1 * xmm0) - mem sched: [9:0.50]
-; SKYLAKE-NEXT:    vfnmsub231ss {{.*#+}} xmm0 = -(xmm1 * mem) - xmm0 sched: [9:0.50]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; KNL-LABEL: test_vfnmsubss_128:
-; KNL:       # %bb.0:
-; KNL-NEXT:    #APP
-; KNL-NEXT:    vfnmsub132ss {{.*#+}} xmm0 = -(xmm0 * xmm2) - xmm1 sched: [5:0.50]
-; KNL-NEXT:    vfnmsub213ss {{.*#+}} xmm0 = -(xmm1 * xmm0) - xmm2 sched: [5:0.50]
-; KNL-NEXT:    vfnmsub231ss {{.*#+}} xmm0 = -(xmm1 * xmm2) - xmm0 sched: [5:0.50]
-; KNL-NEXT:    vfnmsub132ss {{.*#+}} xmm0 = -(xmm0 * mem) - xmm1 sched: [10:0.50]
-; KNL-NEXT:    vfnmsub213ss {{.*#+}} xmm0 = -(xmm1 * xmm0) - mem sched: [10:0.50]
-; KNL-NEXT:    vfnmsub231ss {{.*#+}} xmm0 = -(xmm1 * mem) - xmm0 sched: [10:0.50]
-; KNL-NEXT:    #NO_APP
-; KNL-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_vfnmsubss_128:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    vfnmsub132ss {{.*#+}} xmm0 = -(xmm0 * xmm2) - xmm1 sched: [4:0.50]
-; SKX-NEXT:    vfnmsub213ss {{.*#+}} xmm0 = -(xmm1 * xmm0) - xmm2 sched: [4:0.50]
-; SKX-NEXT:    vfnmsub231ss {{.*#+}} xmm0 = -(xmm1 * xmm2) - xmm0 sched: [4:0.50]
-; SKX-NEXT:    vfnmsub132ss {{.*#+}} xmm0 = -(xmm0 * mem) - xmm1 sched: [9:0.50]
-; SKX-NEXT:    vfnmsub213ss {{.*#+}} xmm0 = -(xmm1 * xmm0) - mem sched: [9:0.50]
-; SKX-NEXT:    vfnmsub231ss {{.*#+}} xmm0 = -(xmm1 * mem) - xmm0 sched: [9:0.50]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_vfnmsubss_128:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    vfnmsub132ss {{.*#+}} xmm0 = -(xmm0 * xmm2) - xmm1 sched: [5:0.50]
-; ZNVER1-NEXT:    vfnmsub213ss {{.*#+}} xmm0 = -(xmm1 * xmm0) - xmm2 sched: [5:0.50]
-; ZNVER1-NEXT:    vfnmsub231ss {{.*#+}} xmm0 = -(xmm1 * xmm2) - xmm0 sched: [5:0.50]
-; ZNVER1-NEXT:    vfnmsub132ss {{.*#+}} xmm0 = -(xmm0 * mem) - xmm1 sched: [12:0.50]
-; ZNVER1-NEXT:    vfnmsub213ss {{.*#+}} xmm0 = -(xmm1 * xmm0) - mem sched: [12:0.50]
-; ZNVER1-NEXT:    vfnmsub231ss {{.*#+}} xmm0 = -(xmm1 * mem) - xmm0 sched: [12:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  tail call void asm "vfnmsub132ss $2, $1, $0 \0A\09 vfnmsub213ss $2, $1, $0 \0A\09 vfnmsub231ss $2, $1, $0 \0A\09 vfnmsub132ss $3, $1, $0 \0A\09 vfnmsub213ss $3, $1, $0 \0A\09 vfnmsub231ss $3, $1, $0", "x,x,x,*m"(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, <4 x float> *%a3) nounwind
-  ret void
-}
diff --git a/test/CodeGen/X86/fma4-schedule.ll b/test/CodeGen/X86/fma4-schedule.ll
deleted file mode 100644
index 118ad2b10be09413ec7b08e43252fb6795b79d7f..0000000000000000000000000000000000000000
--- a/test/CodeGen/X86/fma4-schedule.ll
+++ /dev/null
@@ -1,1058 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=x86-64 -mattr=+fma4 | FileCheck %s --check-prefixes=CHECK,GENERIC
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=bdver1              | FileCheck %s --check-prefixes=CHECK,BDVER,BDVER12,BDVER1
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=bdver2 -mattr=-fma  | FileCheck %s --check-prefixes=CHECK,BDVER,BDVER12,BDVER2
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=bdver3 -mattr=-fma  | FileCheck %s --check-prefixes=CHECK,BDVER,BDVER34,BDVER3
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=bdver4 -mattr=-fma  | FileCheck %s --check-prefixes=CHECK,BDVER,BDVER34,BDVER4
-
-;
-; VFMADD
-;
-
-define void @test_vfmaddpd_128(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, <2 x double> *%a3) optsize {
-; GENERIC-LABEL: test_vfmaddpd_128:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    vfmaddpd %xmm2, %xmm1, %xmm0, %xmm0 # sched: [5:0.50]
-; GENERIC-NEXT:    vfmaddpd (%rdi), %xmm1, %xmm0, %xmm0 # sched: [10:0.50]
-; GENERIC-NEXT:    vfmaddpd %xmm1, (%rdi), %xmm0, %xmm0 # sched: [10:0.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER12-LABEL: test_vfmaddpd_128:
-; BDVER12:       # %bb.0:
-; BDVER12-NEXT:    #APP
-; BDVER12-NEXT:    vfmaddpd %xmm2, %xmm1, %xmm0, %xmm0 # sched: [5:0.50]
-; BDVER12-NEXT:    vfmaddpd (%rdi), %xmm1, %xmm0, %xmm0 # sched: [10:0.50]
-; BDVER12-NEXT:    vfmaddpd %xmm1, (%rdi), %xmm0, %xmm0 # sched: [10:0.50]
-; BDVER12-NEXT:    #NO_APP
-; BDVER12-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER34-LABEL: test_vfmaddpd_128:
-; BDVER34:       # %bb.0:
-; BDVER34-NEXT:    #APP
-; BDVER34-NEXT:    vfmaddpd %xmm2, %xmm1, %xmm0, %xmm0
-; BDVER34-NEXT:    vfmaddpd (%rdi), %xmm1, %xmm0, %xmm0
-; BDVER34-NEXT:    vfmaddpd %xmm1, (%rdi), %xmm0, %xmm0
-; BDVER34-NEXT:    #NO_APP
-; BDVER34-NEXT:    retq
-  tail call void asm "vfmaddpd $2, $1, $0, $0 \0A\09 vfmaddpd $3, $1, $0, $0 \0A\09 vfmaddpd $1, $3, $0, $0", "x,x,x,*m"(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, <2 x double> *%a3) nounwind
-  ret void
-}
-
-define void @test_vfmaddpd_256(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2, <4 x double> *%a3) optsize {
-; GENERIC-LABEL: test_vfmaddpd_256:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    vfmaddpd %ymm2, %ymm1, %ymm0, %ymm0 # sched: [5:0.50]
-; GENERIC-NEXT:    vfmaddpd (%rdi), %ymm1, %ymm0, %ymm0 # sched: [10:0.50]
-; GENERIC-NEXT:    vfmaddpd %ymm1, (%rdi), %ymm0, %ymm0 # sched: [10:0.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    vzeroupper # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER12-LABEL: test_vfmaddpd_256:
-; BDVER12:       # %bb.0:
-; BDVER12-NEXT:    #APP
-; BDVER12-NEXT:    vfmaddpd %ymm2, %ymm1, %ymm0, %ymm0 # sched: [5:0.50]
-; BDVER12-NEXT:    vfmaddpd (%rdi), %ymm1, %ymm0, %ymm0 # sched: [10:1.00]
-; BDVER12-NEXT:    vfmaddpd %ymm1, (%rdi), %ymm0, %ymm0 # sched: [10:1.00]
-; BDVER12-NEXT:    #NO_APP
-; BDVER12-NEXT:    vzeroupper # sched: [46:4.00]
-; BDVER12-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER34-LABEL: test_vfmaddpd_256:
-; BDVER34:       # %bb.0:
-; BDVER34-NEXT:    #APP
-; BDVER34-NEXT:    vfmaddpd %ymm2, %ymm1, %ymm0, %ymm0
-; BDVER34-NEXT:    vfmaddpd (%rdi), %ymm1, %ymm0, %ymm0
-; BDVER34-NEXT:    vfmaddpd %ymm1, (%rdi), %ymm0, %ymm0
-; BDVER34-NEXT:    #NO_APP
-; BDVER34-NEXT:    vzeroupper
-; BDVER34-NEXT:    retq
-  tail call void asm "vfmaddpd $2, $1, $0, $0 \0A\09 vfmaddpd $3, $1, $0, $0 \0A\09 vfmaddpd $1, $3, $0, $0", "x,x,x,*m"(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2, <4 x double> *%a3) nounwind
-  ret void
-}
-
-define void @test_vfmaddps_128(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, <4 x float> *%a3) optsize {
-; GENERIC-LABEL: test_vfmaddps_128:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    vfmaddps %xmm2, %xmm1, %xmm0, %xmm0 # sched: [5:0.50]
-; GENERIC-NEXT:    vfmaddps (%rdi), %xmm1, %xmm0, %xmm0 # sched: [10:0.50]
-; GENERIC-NEXT:    vfmaddps %xmm1, (%rdi), %xmm0, %xmm0 # sched: [10:0.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER12-LABEL: test_vfmaddps_128:
-; BDVER12:       # %bb.0:
-; BDVER12-NEXT:    #APP
-; BDVER12-NEXT:    vfmaddps %xmm2, %xmm1, %xmm0, %xmm0 # sched: [5:0.50]
-; BDVER12-NEXT:    vfmaddps (%rdi), %xmm1, %xmm0, %xmm0 # sched: [10:0.50]
-; BDVER12-NEXT:    vfmaddps %xmm1, (%rdi), %xmm0, %xmm0 # sched: [10:0.50]
-; BDVER12-NEXT:    #NO_APP
-; BDVER12-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER34-LABEL: test_vfmaddps_128:
-; BDVER34:       # %bb.0:
-; BDVER34-NEXT:    #APP
-; BDVER34-NEXT:    vfmaddps %xmm2, %xmm1, %xmm0, %xmm0
-; BDVER34-NEXT:    vfmaddps (%rdi), %xmm1, %xmm0, %xmm0
-; BDVER34-NEXT:    vfmaddps %xmm1, (%rdi), %xmm0, %xmm0
-; BDVER34-NEXT:    #NO_APP
-; BDVER34-NEXT:    retq
-  tail call void asm "vfmaddps $2, $1, $0, $0 \0A\09 vfmaddps $3, $1, $0, $0 \0A\09 vfmaddps $1, $3, $0, $0", "x,x,x,*m"(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, <4 x float> *%a3) nounwind
-  ret void
-}
-
-define void @test_vfmaddps_256(<8 x float> %a0, <8 x float> %a1, <8 x float> %a2, <8 x float> *%a3) optsize {
-; GENERIC-LABEL: test_vfmaddps_256:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    vfmaddps %ymm2, %ymm1, %ymm0, %ymm0 # sched: [5:0.50]
-; GENERIC-NEXT:    vfmaddps (%rdi), %ymm1, %ymm0, %ymm0 # sched: [10:0.50]
-; GENERIC-NEXT:    vfmaddps %ymm1, (%rdi), %ymm0, %ymm0 # sched: [10:0.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    vzeroupper # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER12-LABEL: test_vfmaddps_256:
-; BDVER12:       # %bb.0:
-; BDVER12-NEXT:    #APP
-; BDVER12-NEXT:    vfmaddps %ymm2, %ymm1, %ymm0, %ymm0 # sched: [5:0.50]
-; BDVER12-NEXT:    vfmaddps (%rdi), %ymm1, %ymm0, %ymm0 # sched: [10:1.00]
-; BDVER12-NEXT:    vfmaddps %ymm1, (%rdi), %ymm0, %ymm0 # sched: [10:1.00]
-; BDVER12-NEXT:    #NO_APP
-; BDVER12-NEXT:    vzeroupper # sched: [46:4.00]
-; BDVER12-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER34-LABEL: test_vfmaddps_256:
-; BDVER34:       # %bb.0:
-; BDVER34-NEXT:    #APP
-; BDVER34-NEXT:    vfmaddps %ymm2, %ymm1, %ymm0, %ymm0
-; BDVER34-NEXT:    vfmaddps (%rdi), %ymm1, %ymm0, %ymm0
-; BDVER34-NEXT:    vfmaddps %ymm1, (%rdi), %ymm0, %ymm0
-; BDVER34-NEXT:    #NO_APP
-; BDVER34-NEXT:    vzeroupper
-; BDVER34-NEXT:    retq
-  tail call void asm "vfmaddps $2, $1, $0, $0 \0A\09 vfmaddps $3, $1, $0, $0 \0A\09 vfmaddps $1, $3, $0, $0", "x,x,x,*m"(<8 x float> %a0, <8 x float> %a1, <8 x float> %a2, <8 x float> *%a3) nounwind
-  ret void
-}
-
-define void @test_vfmaddsd_128(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, <2 x double> *%a3) optsize {
-; GENERIC-LABEL: test_vfmaddsd_128:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    vfmaddsd %xmm2, %xmm1, %xmm0, %xmm0 # sched: [5:0.50]
-; GENERIC-NEXT:    vfmaddsd (%rdi), %xmm1, %xmm0, %xmm0 # sched: [10:0.50]
-; GENERIC-NEXT:    vfmaddsd %xmm1, (%rdi), %xmm0, %xmm0 # sched: [10:0.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER12-LABEL: test_vfmaddsd_128:
-; BDVER12:       # %bb.0:
-; BDVER12-NEXT:    #APP
-; BDVER12-NEXT:    vfmaddsd %xmm2, %xmm1, %xmm0, %xmm0 # sched: [5:0.50]
-; BDVER12-NEXT:    vfmaddsd (%rdi), %xmm1, %xmm0, %xmm0 # sched: [10:0.50]
-; BDVER12-NEXT:    vfmaddsd %xmm1, (%rdi), %xmm0, %xmm0 # sched: [10:0.50]
-; BDVER12-NEXT:    #NO_APP
-; BDVER12-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER34-LABEL: test_vfmaddsd_128:
-; BDVER34:       # %bb.0:
-; BDVER34-NEXT:    #APP
-; BDVER34-NEXT:    vfmaddsd %xmm2, %xmm1, %xmm0, %xmm0
-; BDVER34-NEXT:    vfmaddsd (%rdi), %xmm1, %xmm0, %xmm0
-; BDVER34-NEXT:    vfmaddsd %xmm1, (%rdi), %xmm0, %xmm0
-; BDVER34-NEXT:    #NO_APP
-; BDVER34-NEXT:    retq
-  tail call void asm "vfmaddsd $2, $1, $0, $0 \0A\09 vfmaddsd $3, $1, $0, $0 \0A\09 vfmaddsd $1, $3, $0, $0", "x,x,x,*m"(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, <2 x double> *%a3) nounwind
-  ret void
-}
-
-define void @test_vfmaddss_128(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, <4 x float> *%a3) optsize {
-; GENERIC-LABEL: test_vfmaddss_128:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    vfmaddss %xmm2, %xmm1, %xmm0, %xmm0 # sched: [5:0.50]
-; GENERIC-NEXT:    vfmaddss (%rdi), %xmm1, %xmm0, %xmm0 # sched: [10:0.50]
-; GENERIC-NEXT:    vfmaddss %xmm1, (%rdi), %xmm0, %xmm0 # sched: [10:0.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER12-LABEL: test_vfmaddss_128:
-; BDVER12:       # %bb.0:
-; BDVER12-NEXT:    #APP
-; BDVER12-NEXT:    vfmaddss %xmm2, %xmm1, %xmm0, %xmm0 # sched: [5:0.50]
-; BDVER12-NEXT:    vfmaddss (%rdi), %xmm1, %xmm0, %xmm0 # sched: [10:0.50]
-; BDVER12-NEXT:    vfmaddss %xmm1, (%rdi), %xmm0, %xmm0 # sched: [10:0.50]
-; BDVER12-NEXT:    #NO_APP
-; BDVER12-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER34-LABEL: test_vfmaddss_128:
-; BDVER34:       # %bb.0:
-; BDVER34-NEXT:    #APP
-; BDVER34-NEXT:    vfmaddss %xmm2, %xmm1, %xmm0, %xmm0
-; BDVER34-NEXT:    vfmaddss (%rdi), %xmm1, %xmm0, %xmm0
-; BDVER34-NEXT:    vfmaddss %xmm1, (%rdi), %xmm0, %xmm0
-; BDVER34-NEXT:    #NO_APP
-; BDVER34-NEXT:    retq
-  tail call void asm "vfmaddss $2, $1, $0, $0 \0A\09 vfmaddss $3, $1, $0, $0 \0A\09 vfmaddss $1, $3, $0, $0", "x,x,x,*m"(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, <4 x float> *%a3) nounwind
-  ret void
-}
-
-;
-; VFMADDSUB
-;
-
-define void @test_vfmaddsubpd_128(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, <2 x double> *%a3) optsize {
-; GENERIC-LABEL: test_vfmaddsubpd_128:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    vfmaddsubpd %xmm2, %xmm1, %xmm0, %xmm0 # sched: [5:0.50]
-; GENERIC-NEXT:    vfmaddsubpd (%rdi), %xmm1, %xmm0, %xmm0 # sched: [10:0.50]
-; GENERIC-NEXT:    vfmaddsubpd %xmm1, (%rdi), %xmm0, %xmm0 # sched: [10:0.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER12-LABEL: test_vfmaddsubpd_128:
-; BDVER12:       # %bb.0:
-; BDVER12-NEXT:    #APP
-; BDVER12-NEXT:    vfmaddsubpd %xmm2, %xmm1, %xmm0, %xmm0 # sched: [5:0.50]
-; BDVER12-NEXT:    vfmaddsubpd (%rdi), %xmm1, %xmm0, %xmm0 # sched: [10:0.50]
-; BDVER12-NEXT:    vfmaddsubpd %xmm1, (%rdi), %xmm0, %xmm0 # sched: [10:0.50]
-; BDVER12-NEXT:    #NO_APP
-; BDVER12-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER34-LABEL: test_vfmaddsubpd_128:
-; BDVER34:       # %bb.0:
-; BDVER34-NEXT:    #APP
-; BDVER34-NEXT:    vfmaddsubpd %xmm2, %xmm1, %xmm0, %xmm0
-; BDVER34-NEXT:    vfmaddsubpd (%rdi), %xmm1, %xmm0, %xmm0
-; BDVER34-NEXT:    vfmaddsubpd %xmm1, (%rdi), %xmm0, %xmm0
-; BDVER34-NEXT:    #NO_APP
-; BDVER34-NEXT:    retq
-  tail call void asm "vfmaddsubpd $2, $1, $0, $0 \0A\09 vfmaddsubpd $3, $1, $0, $0 \0A\09 vfmaddsubpd $1, $3, $0, $0", "x,x,x,*m"(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, <2 x double> *%a3) nounwind
-  ret void
-}
-
-define void @test_vfmaddsubpd_256(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2, <4 x double> *%a3) optsize {
-; GENERIC-LABEL: test_vfmaddsubpd_256:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    vfmaddsubpd %ymm2, %ymm1, %ymm0, %ymm0 # sched: [5:0.50]
-; GENERIC-NEXT:    vfmaddsubpd (%rdi), %ymm1, %ymm0, %ymm0 # sched: [10:0.50]
-; GENERIC-NEXT:    vfmaddsubpd %ymm1, (%rdi), %ymm0, %ymm0 # sched: [10:0.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    vzeroupper # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER12-LABEL: test_vfmaddsubpd_256:
-; BDVER12:       # %bb.0:
-; BDVER12-NEXT:    #APP
-; BDVER12-NEXT:    vfmaddsubpd %ymm2, %ymm1, %ymm0, %ymm0 # sched: [5:0.50]
-; BDVER12-NEXT:    vfmaddsubpd (%rdi), %ymm1, %ymm0, %ymm0 # sched: [10:1.00]
-; BDVER12-NEXT:    vfmaddsubpd %ymm1, (%rdi), %ymm0, %ymm0 # sched: [10:1.00]
-; BDVER12-NEXT:    #NO_APP
-; BDVER12-NEXT:    vzeroupper # sched: [46:4.00]
-; BDVER12-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER34-LABEL: test_vfmaddsubpd_256:
-; BDVER34:       # %bb.0:
-; BDVER34-NEXT:    #APP
-; BDVER34-NEXT:    vfmaddsubpd %ymm2, %ymm1, %ymm0, %ymm0
-; BDVER34-NEXT:    vfmaddsubpd (%rdi), %ymm1, %ymm0, %ymm0
-; BDVER34-NEXT:    vfmaddsubpd %ymm1, (%rdi), %ymm0, %ymm0
-; BDVER34-NEXT:    #NO_APP
-; BDVER34-NEXT:    vzeroupper
-; BDVER34-NEXT:    retq
-  tail call void asm "vfmaddsubpd $2, $1, $0, $0 \0A\09 vfmaddsubpd $3, $1, $0, $0 \0A\09 vfmaddsubpd $1, $3, $0, $0", "x,x,x,*m"(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2, <4 x double> *%a3) nounwind
-  ret void
-}
-
-define void @test_vfmaddsubps_128(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, <4 x float> *%a3) optsize {
-; GENERIC-LABEL: test_vfmaddsubps_128:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    vfmaddsubps %xmm2, %xmm1, %xmm0, %xmm0 # sched: [5:0.50]
-; GENERIC-NEXT:    vfmaddsubps (%rdi), %xmm1, %xmm0, %xmm0 # sched: [10:0.50]
-; GENERIC-NEXT:    vfmaddsubps %xmm1, (%rdi), %xmm0, %xmm0 # sched: [10:0.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER12-LABEL: test_vfmaddsubps_128:
-; BDVER12:       # %bb.0:
-; BDVER12-NEXT:    #APP
-; BDVER12-NEXT:    vfmaddsubps %xmm2, %xmm1, %xmm0, %xmm0 # sched: [5:0.50]
-; BDVER12-NEXT:    vfmaddsubps (%rdi), %xmm1, %xmm0, %xmm0 # sched: [10:0.50]
-; BDVER12-NEXT:    vfmaddsubps %xmm1, (%rdi), %xmm0, %xmm0 # sched: [10:0.50]
-; BDVER12-NEXT:    #NO_APP
-; BDVER12-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER34-LABEL: test_vfmaddsubps_128:
-; BDVER34:       # %bb.0:
-; BDVER34-NEXT:    #APP
-; BDVER34-NEXT:    vfmaddsubps %xmm2, %xmm1, %xmm0, %xmm0
-; BDVER34-NEXT:    vfmaddsubps (%rdi), %xmm1, %xmm0, %xmm0
-; BDVER34-NEXT:    vfmaddsubps %xmm1, (%rdi), %xmm0, %xmm0
-; BDVER34-NEXT:    #NO_APP
-; BDVER34-NEXT:    retq
-  tail call void asm "vfmaddsubps $2, $1, $0, $0 \0A\09 vfmaddsubps $3, $1, $0, $0 \0A\09 vfmaddsubps $1, $3, $0, $0", "x,x,x,*m"(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, <4 x float> *%a3) nounwind
-  ret void
-}
-
-define void @test_vfmaddsubps_256(<8 x float> %a0, <8 x float> %a1, <8 x float> %a2, <8 x float> *%a3) optsize {
-; GENERIC-LABEL: test_vfmaddsubps_256:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    vfmaddsubps %ymm2, %ymm1, %ymm0, %ymm0 # sched: [5:0.50]
-; GENERIC-NEXT:    vfmaddsubps (%rdi), %ymm1, %ymm0, %ymm0 # sched: [10:0.50]
-; GENERIC-NEXT:    vfmaddsubps %ymm1, (%rdi), %ymm0, %ymm0 # sched: [10:0.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    vzeroupper # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER12-LABEL: test_vfmaddsubps_256:
-; BDVER12:       # %bb.0:
-; BDVER12-NEXT:    #APP
-; BDVER12-NEXT:    vfmaddsubps %ymm2, %ymm1, %ymm0, %ymm0 # sched: [5:0.50]
-; BDVER12-NEXT:    vfmaddsubps (%rdi), %ymm1, %ymm0, %ymm0 # sched: [10:1.00]
-; BDVER12-NEXT:    vfmaddsubps %ymm1, (%rdi), %ymm0, %ymm0 # sched: [10:1.00]
-; BDVER12-NEXT:    #NO_APP
-; BDVER12-NEXT:    vzeroupper # sched: [46:4.00]
-; BDVER12-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER34-LABEL: test_vfmaddsubps_256:
-; BDVER34:       # %bb.0:
-; BDVER34-NEXT:    #APP
-; BDVER34-NEXT:    vfmaddsubps %ymm2, %ymm1, %ymm0, %ymm0
-; BDVER34-NEXT:    vfmaddsubps (%rdi), %ymm1, %ymm0, %ymm0
-; BDVER34-NEXT:    vfmaddsubps %ymm1, (%rdi), %ymm0, %ymm0
-; BDVER34-NEXT:    #NO_APP
-; BDVER34-NEXT:    vzeroupper
-; BDVER34-NEXT:    retq
-  tail call void asm "vfmaddsubps $2, $1, $0, $0 \0A\09 vfmaddsubps $3, $1, $0, $0 \0A\09 vfmaddsubps $1, $3, $0, $0", "x,x,x,*m"(<8 x float> %a0, <8 x float> %a1, <8 x float> %a2, <8 x float> *%a3) nounwind
-  ret void
-}
-
-;
-; VFMSUBADD
-;
-
-define void @test_vfmsubaddpd_128(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, <2 x double> *%a3) optsize {
-; GENERIC-LABEL: test_vfmsubaddpd_128:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    vfmsubaddpd %xmm2, %xmm1, %xmm0, %xmm0 # sched: [5:0.50]
-; GENERIC-NEXT:    vfmsubaddpd (%rdi), %xmm1, %xmm0, %xmm0 # sched: [10:0.50]
-; GENERIC-NEXT:    vfmsubaddpd %xmm1, (%rdi), %xmm0, %xmm0 # sched: [10:0.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER12-LABEL: test_vfmsubaddpd_128:
-; BDVER12:       # %bb.0:
-; BDVER12-NEXT:    #APP
-; BDVER12-NEXT:    vfmsubaddpd %xmm2, %xmm1, %xmm0, %xmm0 # sched: [5:0.50]
-; BDVER12-NEXT:    vfmsubaddpd (%rdi), %xmm1, %xmm0, %xmm0 # sched: [10:0.50]
-; BDVER12-NEXT:    vfmsubaddpd %xmm1, (%rdi), %xmm0, %xmm0 # sched: [10:0.50]
-; BDVER12-NEXT:    #NO_APP
-; BDVER12-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER34-LABEL: test_vfmsubaddpd_128:
-; BDVER34:       # %bb.0:
-; BDVER34-NEXT:    #APP
-; BDVER34-NEXT:    vfmsubaddpd %xmm2, %xmm1, %xmm0, %xmm0
-; BDVER34-NEXT:    vfmsubaddpd (%rdi), %xmm1, %xmm0, %xmm0
-; BDVER34-NEXT:    vfmsubaddpd %xmm1, (%rdi), %xmm0, %xmm0
-; BDVER34-NEXT:    #NO_APP
-; BDVER34-NEXT:    retq
-  tail call void asm "vfmsubaddpd $2, $1, $0, $0 \0A\09 vfmsubaddpd $3, $1, $0, $0 \0A\09 vfmsubaddpd $1, $3, $0, $0", "x,x,x,*m"(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, <2 x double> *%a3) nounwind
-  ret void
-}
-
-define void @test_vfmsubaddpd_256(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2, <4 x double> *%a3) optsize {
-; GENERIC-LABEL: test_vfmsubaddpd_256:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    vfmsubaddpd %ymm2, %ymm1, %ymm0, %ymm0 # sched: [5:0.50]
-; GENERIC-NEXT:    vfmsubaddpd (%rdi), %ymm1, %ymm0, %ymm0 # sched: [10:0.50]
-; GENERIC-NEXT:    vfmsubaddpd %ymm1, (%rdi), %ymm0, %ymm0 # sched: [10:0.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    vzeroupper # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER12-LABEL: test_vfmsubaddpd_256:
-; BDVER12:       # %bb.0:
-; BDVER12-NEXT:    #APP
-; BDVER12-NEXT:    vfmsubaddpd %ymm2, %ymm1, %ymm0, %ymm0 # sched: [5:0.50]
-; BDVER12-NEXT:    vfmsubaddpd (%rdi), %ymm1, %ymm0, %ymm0 # sched: [10:1.00]
-; BDVER12-NEXT:    vfmsubaddpd %ymm1, (%rdi), %ymm0, %ymm0 # sched: [10:1.00]
-; BDVER12-NEXT:    #NO_APP
-; BDVER12-NEXT:    vzeroupper # sched: [46:4.00]
-; BDVER12-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER34-LABEL: test_vfmsubaddpd_256:
-; BDVER34:       # %bb.0:
-; BDVER34-NEXT:    #APP
-; BDVER34-NEXT:    vfmsubaddpd %ymm2, %ymm1, %ymm0, %ymm0
-; BDVER34-NEXT:    vfmsubaddpd (%rdi), %ymm1, %ymm0, %ymm0
-; BDVER34-NEXT:    vfmsubaddpd %ymm1, (%rdi), %ymm0, %ymm0
-; BDVER34-NEXT:    #NO_APP
-; BDVER34-NEXT:    vzeroupper
-; BDVER34-NEXT:    retq
-  tail call void asm "vfmsubaddpd $2, $1, $0, $0 \0A\09 vfmsubaddpd $3, $1, $0, $0 \0A\09 vfmsubaddpd $1, $3, $0, $0", "x,x,x,*m"(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2, <4 x double> *%a3) nounwind
-  ret void
-}
-
-define void @test_vfmsubaddps_128(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, <4 x float> *%a3) optsize {
-; GENERIC-LABEL: test_vfmsubaddps_128:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    vfmsubaddps %xmm2, %xmm1, %xmm0, %xmm0 # sched: [5:0.50]
-; GENERIC-NEXT:    vfmsubaddps (%rdi), %xmm1, %xmm0, %xmm0 # sched: [10:0.50]
-; GENERIC-NEXT:    vfmsubaddps %xmm1, (%rdi), %xmm0, %xmm0 # sched: [10:0.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER12-LABEL: test_vfmsubaddps_128:
-; BDVER12:       # %bb.0:
-; BDVER12-NEXT:    #APP
-; BDVER12-NEXT:    vfmsubaddps %xmm2, %xmm1, %xmm0, %xmm0 # sched: [5:0.50]
-; BDVER12-NEXT:    vfmsubaddps (%rdi), %xmm1, %xmm0, %xmm0 # sched: [10:0.50]
-; BDVER12-NEXT:    vfmsubaddps %xmm1, (%rdi), %xmm0, %xmm0 # sched: [10:0.50]
-; BDVER12-NEXT:    #NO_APP
-; BDVER12-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER34-LABEL: test_vfmsubaddps_128:
-; BDVER34:       # %bb.0:
-; BDVER34-NEXT:    #APP
-; BDVER34-NEXT:    vfmsubaddps %xmm2, %xmm1, %xmm0, %xmm0
-; BDVER34-NEXT:    vfmsubaddps (%rdi), %xmm1, %xmm0, %xmm0
-; BDVER34-NEXT:    vfmsubaddps %xmm1, (%rdi), %xmm0, %xmm0
-; BDVER34-NEXT:    #NO_APP
-; BDVER34-NEXT:    retq
-  tail call void asm "vfmsubaddps $2, $1, $0, $0 \0A\09 vfmsubaddps $3, $1, $0, $0 \0A\09 vfmsubaddps $1, $3, $0, $0", "x,x,x,*m"(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, <4 x float> *%a3) nounwind
-  ret void
-}
-
-define void @test_vfmsubaddps_256(<8 x float> %a0, <8 x float> %a1, <8 x float> %a2, <8 x float> *%a3) optsize {
-; GENERIC-LABEL: test_vfmsubaddps_256:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    vfmsubaddps %ymm2, %ymm1, %ymm0, %ymm0 # sched: [5:0.50]
-; GENERIC-NEXT:    vfmsubaddps (%rdi), %ymm1, %ymm0, %ymm0 # sched: [10:0.50]
-; GENERIC-NEXT:    vfmsubaddps %ymm1, (%rdi), %ymm0, %ymm0 # sched: [10:0.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    vzeroupper # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER12-LABEL: test_vfmsubaddps_256:
-; BDVER12:       # %bb.0:
-; BDVER12-NEXT:    #APP
-; BDVER12-NEXT:    vfmsubaddps %ymm2, %ymm1, %ymm0, %ymm0 # sched: [5:0.50]
-; BDVER12-NEXT:    vfmsubaddps (%rdi), %ymm1, %ymm0, %ymm0 # sched: [10:1.00]
-; BDVER12-NEXT:    vfmsubaddps %ymm1, (%rdi), %ymm0, %ymm0 # sched: [10:1.00]
-; BDVER12-NEXT:    #NO_APP
-; BDVER12-NEXT:    vzeroupper # sched: [46:4.00]
-; BDVER12-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER34-LABEL: test_vfmsubaddps_256:
-; BDVER34:       # %bb.0:
-; BDVER34-NEXT:    #APP
-; BDVER34-NEXT:    vfmsubaddps %ymm2, %ymm1, %ymm0, %ymm0
-; BDVER34-NEXT:    vfmsubaddps (%rdi), %ymm1, %ymm0, %ymm0
-; BDVER34-NEXT:    vfmsubaddps %ymm1, (%rdi), %ymm0, %ymm0
-; BDVER34-NEXT:    #NO_APP
-; BDVER34-NEXT:    vzeroupper
-; BDVER34-NEXT:    retq
-  tail call void asm "vfmsubaddps $2, $1, $0, $0 \0A\09 vfmsubaddps $3, $1, $0, $0 \0A\09 vfmsubaddps $1, $3, $0, $0", "x,x,x,*m"(<8 x float> %a0, <8 x float> %a1, <8 x float> %a2, <8 x float> *%a3) nounwind
-  ret void
-}
-
-;
-; VFMSUB
-;
-
-define void @test_vfmsubpd_128(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, <2 x double> *%a3) optsize {
-; GENERIC-LABEL: test_vfmsubpd_128:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    vfmsubpd %xmm2, %xmm1, %xmm0, %xmm0 # sched: [5:0.50]
-; GENERIC-NEXT:    vfmsubpd (%rdi), %xmm1, %xmm0, %xmm0 # sched: [10:0.50]
-; GENERIC-NEXT:    vfmsubpd %xmm1, (%rdi), %xmm0, %xmm0 # sched: [10:0.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER12-LABEL: test_vfmsubpd_128:
-; BDVER12:       # %bb.0:
-; BDVER12-NEXT:    #APP
-; BDVER12-NEXT:    vfmsubpd %xmm2, %xmm1, %xmm0, %xmm0 # sched: [5:0.50]
-; BDVER12-NEXT:    vfmsubpd (%rdi), %xmm1, %xmm0, %xmm0 # sched: [10:0.50]
-; BDVER12-NEXT:    vfmsubpd %xmm1, (%rdi), %xmm0, %xmm0 # sched: [10:0.50]
-; BDVER12-NEXT:    #NO_APP
-; BDVER12-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER34-LABEL: test_vfmsubpd_128:
-; BDVER34:       # %bb.0:
-; BDVER34-NEXT:    #APP
-; BDVER34-NEXT:    vfmsubpd %xmm2, %xmm1, %xmm0, %xmm0
-; BDVER34-NEXT:    vfmsubpd (%rdi), %xmm1, %xmm0, %xmm0
-; BDVER34-NEXT:    vfmsubpd %xmm1, (%rdi), %xmm0, %xmm0
-; BDVER34-NEXT:    #NO_APP
-; BDVER34-NEXT:    retq
-  tail call void asm "vfmsubpd $2, $1, $0, $0 \0A\09 vfmsubpd $3, $1, $0, $0 \0A\09 vfmsubpd $1, $3, $0, $0", "x,x,x,*m"(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, <2 x double> *%a3) nounwind
-  ret void
-}
-
-define void @test_vfmsubpd_256(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2, <4 x double> *%a3) optsize {
-; GENERIC-LABEL: test_vfmsubpd_256:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    vfmsubpd %ymm2, %ymm1, %ymm0, %ymm0 # sched: [5:0.50]
-; GENERIC-NEXT:    vfmsubpd (%rdi), %ymm1, %ymm0, %ymm0 # sched: [10:0.50]
-; GENERIC-NEXT:    vfmsubpd %ymm1, (%rdi), %ymm0, %ymm0 # sched: [10:0.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    vzeroupper # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER12-LABEL: test_vfmsubpd_256:
-; BDVER12:       # %bb.0:
-; BDVER12-NEXT:    #APP
-; BDVER12-NEXT:    vfmsubpd %ymm2, %ymm1, %ymm0, %ymm0 # sched: [5:0.50]
-; BDVER12-NEXT:    vfmsubpd (%rdi), %ymm1, %ymm0, %ymm0 # sched: [10:1.00]
-; BDVER12-NEXT:    vfmsubpd %ymm1, (%rdi), %ymm0, %ymm0 # sched: [10:1.00]
-; BDVER12-NEXT:    #NO_APP
-; BDVER12-NEXT:    vzeroupper # sched: [46:4.00]
-; BDVER12-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER34-LABEL: test_vfmsubpd_256:
-; BDVER34:       # %bb.0:
-; BDVER34-NEXT:    #APP
-; BDVER34-NEXT:    vfmsubpd %ymm2, %ymm1, %ymm0, %ymm0
-; BDVER34-NEXT:    vfmsubpd (%rdi), %ymm1, %ymm0, %ymm0
-; BDVER34-NEXT:    vfmsubpd %ymm1, (%rdi), %ymm0, %ymm0
-; BDVER34-NEXT:    #NO_APP
-; BDVER34-NEXT:    vzeroupper
-; BDVER34-NEXT:    retq
-  tail call void asm "vfmsubpd $2, $1, $0, $0 \0A\09 vfmsubpd $3, $1, $0, $0 \0A\09 vfmsubpd $1, $3, $0, $0", "x,x,x,*m"(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2, <4 x double> *%a3) nounwind
-  ret void
-}
-
-define void @test_vfmsubps_128(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, <4 x float> *%a3) optsize {
-; GENERIC-LABEL: test_vfmsubps_128:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    vfmsubps %xmm2, %xmm1, %xmm0, %xmm0 # sched: [5:0.50]
-; GENERIC-NEXT:    vfmsubps (%rdi), %xmm1, %xmm0, %xmm0 # sched: [10:0.50]
-; GENERIC-NEXT:    vfmsubps %xmm1, (%rdi), %xmm0, %xmm0 # sched: [10:0.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER12-LABEL: test_vfmsubps_128:
-; BDVER12:       # %bb.0:
-; BDVER12-NEXT:    #APP
-; BDVER12-NEXT:    vfmsubps %xmm2, %xmm1, %xmm0, %xmm0 # sched: [5:0.50]
-; BDVER12-NEXT:    vfmsubps (%rdi), %xmm1, %xmm0, %xmm0 # sched: [10:0.50]
-; BDVER12-NEXT:    vfmsubps %xmm1, (%rdi), %xmm0, %xmm0 # sched: [10:0.50]
-; BDVER12-NEXT:    #NO_APP
-; BDVER12-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER34-LABEL: test_vfmsubps_128:
-; BDVER34:       # %bb.0:
-; BDVER34-NEXT:    #APP
-; BDVER34-NEXT:    vfmsubps %xmm2, %xmm1, %xmm0, %xmm0
-; BDVER34-NEXT:    vfmsubps (%rdi), %xmm1, %xmm0, %xmm0
-; BDVER34-NEXT:    vfmsubps %xmm1, (%rdi), %xmm0, %xmm0
-; BDVER34-NEXT:    #NO_APP
-; BDVER34-NEXT:    retq
-  tail call void asm "vfmsubps $2, $1, $0, $0 \0A\09 vfmsubps $3, $1, $0, $0 \0A\09 vfmsubps $1, $3, $0, $0", "x,x,x,*m"(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, <4 x float> *%a3) nounwind
-  ret void
-}
-
-define void @test_vfmsubps_256(<8 x float> %a0, <8 x float> %a1, <8 x float> %a2, <8 x float> *%a3) optsize {
-; GENERIC-LABEL: test_vfmsubps_256:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    vfmsubps %ymm2, %ymm1, %ymm0, %ymm0 # sched: [5:0.50]
-; GENERIC-NEXT:    vfmsubps (%rdi), %ymm1, %ymm0, %ymm0 # sched: [10:0.50]
-; GENERIC-NEXT:    vfmsubps %ymm1, (%rdi), %ymm0, %ymm0 # sched: [10:0.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    vzeroupper # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER12-LABEL: test_vfmsubps_256:
-; BDVER12:       # %bb.0:
-; BDVER12-NEXT:    #APP
-; BDVER12-NEXT:    vfmsubps %ymm2, %ymm1, %ymm0, %ymm0 # sched: [5:0.50]
-; BDVER12-NEXT:    vfmsubps (%rdi), %ymm1, %ymm0, %ymm0 # sched: [10:1.00]
-; BDVER12-NEXT:    vfmsubps %ymm1, (%rdi), %ymm0, %ymm0 # sched: [10:1.00]
-; BDVER12-NEXT:    #NO_APP
-; BDVER12-NEXT:    vzeroupper # sched: [46:4.00]
-; BDVER12-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER34-LABEL: test_vfmsubps_256:
-; BDVER34:       # %bb.0:
-; BDVER34-NEXT:    #APP
-; BDVER34-NEXT:    vfmsubps %ymm2, %ymm1, %ymm0, %ymm0
-; BDVER34-NEXT:    vfmsubps (%rdi), %ymm1, %ymm0, %ymm0
-; BDVER34-NEXT:    vfmsubps %ymm1, (%rdi), %ymm0, %ymm0
-; BDVER34-NEXT:    #NO_APP
-; BDVER34-NEXT:    vzeroupper
-; BDVER34-NEXT:    retq
-  tail call void asm "vfmsubps $2, $1, $0, $0 \0A\09 vfmsubps $3, $1, $0, $0 \0A\09 vfmsubps $1, $3, $0, $0", "x,x,x,*m"(<8 x float> %a0, <8 x float> %a1, <8 x float> %a2, <8 x float> *%a3) nounwind
-  ret void
-}
-
-define void @test_vfmsubsd_128(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, <2 x double> *%a3) optsize {
-; GENERIC-LABEL: test_vfmsubsd_128:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    vfmsubsd %xmm2, %xmm1, %xmm0, %xmm0 # sched: [5:0.50]
-; GENERIC-NEXT:    vfmsubsd (%rdi), %xmm1, %xmm0, %xmm0 # sched: [10:0.50]
-; GENERIC-NEXT:    vfmsubsd %xmm1, (%rdi), %xmm0, %xmm0 # sched: [10:0.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER12-LABEL: test_vfmsubsd_128:
-; BDVER12:       # %bb.0:
-; BDVER12-NEXT:    #APP
-; BDVER12-NEXT:    vfmsubsd %xmm2, %xmm1, %xmm0, %xmm0 # sched: [5:0.50]
-; BDVER12-NEXT:    vfmsubsd (%rdi), %xmm1, %xmm0, %xmm0 # sched: [10:0.50]
-; BDVER12-NEXT:    vfmsubsd %xmm1, (%rdi), %xmm0, %xmm0 # sched: [10:0.50]
-; BDVER12-NEXT:    #NO_APP
-; BDVER12-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER34-LABEL: test_vfmsubsd_128:
-; BDVER34:       # %bb.0:
-; BDVER34-NEXT:    #APP
-; BDVER34-NEXT:    vfmsubsd %xmm2, %xmm1, %xmm0, %xmm0
-; BDVER34-NEXT:    vfmsubsd (%rdi), %xmm1, %xmm0, %xmm0
-; BDVER34-NEXT:    vfmsubsd %xmm1, (%rdi), %xmm0, %xmm0
-; BDVER34-NEXT:    #NO_APP
-; BDVER34-NEXT:    retq
-  tail call void asm "vfmsubsd $2, $1, $0, $0 \0A\09 vfmsubsd $3, $1, $0, $0 \0A\09 vfmsubsd $1, $3, $0, $0", "x,x,x,*m"(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, <2 x double> *%a3) nounwind
-  ret void
-}
-
-define void @test_vfmsubss_128(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, <4 x float> *%a3) optsize {
-; GENERIC-LABEL: test_vfmsubss_128:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    vfmsubss %xmm2, %xmm1, %xmm0, %xmm0 # sched: [5:0.50]
-; GENERIC-NEXT:    vfmsubss (%rdi), %xmm1, %xmm0, %xmm0 # sched: [10:0.50]
-; GENERIC-NEXT:    vfmsubss %xmm1, (%rdi), %xmm0, %xmm0 # sched: [10:0.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER12-LABEL: test_vfmsubss_128:
-; BDVER12:       # %bb.0:
-; BDVER12-NEXT:    #APP
-; BDVER12-NEXT:    vfmsubss %xmm2, %xmm1, %xmm0, %xmm0 # sched: [5:0.50]
-; BDVER12-NEXT:    vfmsubss (%rdi), %xmm1, %xmm0, %xmm0 # sched: [10:0.50]
-; BDVER12-NEXT:    vfmsubss %xmm1, (%rdi), %xmm0, %xmm0 # sched: [10:0.50]
-; BDVER12-NEXT:    #NO_APP
-; BDVER12-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER34-LABEL: test_vfmsubss_128:
-; BDVER34:       # %bb.0:
-; BDVER34-NEXT:    #APP
-; BDVER34-NEXT:    vfmsubss %xmm2, %xmm1, %xmm0, %xmm0
-; BDVER34-NEXT:    vfmsubss (%rdi), %xmm1, %xmm0, %xmm0
-; BDVER34-NEXT:    vfmsubss %xmm1, (%rdi), %xmm0, %xmm0
-; BDVER34-NEXT:    #NO_APP
-; BDVER34-NEXT:    retq
-  tail call void asm "vfmsubss $2, $1, $0, $0 \0A\09 vfmsubss $3, $1, $0, $0 \0A\09 vfmsubss $1, $3, $0, $0", "x,x,x,*m"(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, <4 x float> *%a3) nounwind
-  ret void
-}
-
-;
-; VFNMADD
-;
-
-define void @test_vfnmaddpd_128(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, <2 x double> *%a3) optsize {
-; GENERIC-LABEL: test_vfnmaddpd_128:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    vfnmaddpd %xmm2, %xmm1, %xmm0, %xmm0 # sched: [5:0.50]
-; GENERIC-NEXT:    vfnmaddpd (%rdi), %xmm1, %xmm0, %xmm0 # sched: [10:0.50]
-; GENERIC-NEXT:    vfnmaddpd %xmm1, (%rdi), %xmm0, %xmm0 # sched: [10:0.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER12-LABEL: test_vfnmaddpd_128:
-; BDVER12:       # %bb.0:
-; BDVER12-NEXT:    #APP
-; BDVER12-NEXT:    vfnmaddpd %xmm2, %xmm1, %xmm0, %xmm0 # sched: [5:0.50]
-; BDVER12-NEXT:    vfnmaddpd (%rdi), %xmm1, %xmm0, %xmm0 # sched: [10:0.50]
-; BDVER12-NEXT:    vfnmaddpd %xmm1, (%rdi), %xmm0, %xmm0 # sched: [10:0.50]
-; BDVER12-NEXT:    #NO_APP
-; BDVER12-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER34-LABEL: test_vfnmaddpd_128:
-; BDVER34:       # %bb.0:
-; BDVER34-NEXT:    #APP
-; BDVER34-NEXT:    vfnmaddpd %xmm2, %xmm1, %xmm0, %xmm0
-; BDVER34-NEXT:    vfnmaddpd (%rdi), %xmm1, %xmm0, %xmm0
-; BDVER34-NEXT:    vfnmaddpd %xmm1, (%rdi), %xmm0, %xmm0
-; BDVER34-NEXT:    #NO_APP
-; BDVER34-NEXT:    retq
-  tail call void asm "vfnmaddpd $2, $1, $0, $0 \0A\09 vfnmaddpd $3, $1, $0, $0 \0A\09 vfnmaddpd $1, $3, $0, $0", "x,x,x,*m"(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, <2 x double> *%a3) nounwind
-  ret void
-}
-
-define void @test_vfnmaddpd_256(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2, <4 x double> *%a3) optsize {
-; GENERIC-LABEL: test_vfnmaddpd_256:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    vfnmaddpd %ymm2, %ymm1, %ymm0, %ymm0 # sched: [5:0.50]
-; GENERIC-NEXT:    vfnmaddpd (%rdi), %ymm1, %ymm0, %ymm0 # sched: [10:0.50]
-; GENERIC-NEXT:    vfnmaddpd %ymm1, (%rdi), %ymm0, %ymm0 # sched: [10:0.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    vzeroupper # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER12-LABEL: test_vfnmaddpd_256:
-; BDVER12:       # %bb.0:
-; BDVER12-NEXT:    #APP
-; BDVER12-NEXT:    vfnmaddpd %ymm2, %ymm1, %ymm0, %ymm0 # sched: [5:0.50]
-; BDVER12-NEXT:    vfnmaddpd (%rdi), %ymm1, %ymm0, %ymm0 # sched: [10:1.00]
-; BDVER12-NEXT:    vfnmaddpd %ymm1, (%rdi), %ymm0, %ymm0 # sched: [10:1.00]
-; BDVER12-NEXT:    #NO_APP
-; BDVER12-NEXT:    vzeroupper # sched: [46:4.00]
-; BDVER12-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER34-LABEL: test_vfnmaddpd_256:
-; BDVER34:       # %bb.0:
-; BDVER34-NEXT:    #APP
-; BDVER34-NEXT:    vfnmaddpd %ymm2, %ymm1, %ymm0, %ymm0
-; BDVER34-NEXT:    vfnmaddpd (%rdi), %ymm1, %ymm0, %ymm0
-; BDVER34-NEXT:    vfnmaddpd %ymm1, (%rdi), %ymm0, %ymm0
-; BDVER34-NEXT:    #NO_APP
-; BDVER34-NEXT:    vzeroupper
-; BDVER34-NEXT:    retq
-  tail call void asm "vfnmaddpd $2, $1, $0, $0 \0A\09 vfnmaddpd $3, $1, $0, $0 \0A\09 vfnmaddpd $1, $3, $0, $0", "x,x,x,*m"(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2, <4 x double> *%a3) nounwind
-  ret void
-}
-
-define void @test_vfnmaddps_128(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, <4 x float> *%a3) optsize {
-; GENERIC-LABEL: test_vfnmaddps_128:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    vfnmaddps %xmm2, %xmm1, %xmm0, %xmm0 # sched: [5:0.50]
-; GENERIC-NEXT:    vfnmaddps (%rdi), %xmm1, %xmm0, %xmm0 # sched: [10:0.50]
-; GENERIC-NEXT:    vfnmaddps %xmm1, (%rdi), %xmm0, %xmm0 # sched: [10:0.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER12-LABEL: test_vfnmaddps_128:
-; BDVER12:       # %bb.0:
-; BDVER12-NEXT:    #APP
-; BDVER12-NEXT:    vfnmaddps %xmm2, %xmm1, %xmm0, %xmm0 # sched: [5:0.50]
-; BDVER12-NEXT:    vfnmaddps (%rdi), %xmm1, %xmm0, %xmm0 # sched: [10:0.50]
-; BDVER12-NEXT:    vfnmaddps %xmm1, (%rdi), %xmm0, %xmm0 # sched: [10:0.50]
-; BDVER12-NEXT:    #NO_APP
-; BDVER12-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER34-LABEL: test_vfnmaddps_128:
-; BDVER34:       # %bb.0:
-; BDVER34-NEXT:    #APP
-; BDVER34-NEXT:    vfnmaddps %xmm2, %xmm1, %xmm0, %xmm0
-; BDVER34-NEXT:    vfnmaddps (%rdi), %xmm1, %xmm0, %xmm0
-; BDVER34-NEXT:    vfnmaddps %xmm1, (%rdi), %xmm0, %xmm0
-; BDVER34-NEXT:    #NO_APP
-; BDVER34-NEXT:    retq
-  tail call void asm "vfnmaddps $2, $1, $0, $0 \0A\09 vfnmaddps $3, $1, $0, $0 \0A\09 vfnmaddps $1, $3, $0, $0", "x,x,x,*m"(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, <4 x float> *%a3) nounwind
-  ret void
-}
-
-define void @test_vfnmaddps_256(<8 x float> %a0, <8 x float> %a1, <8 x float> %a2, <8 x float> *%a3) optsize {
-; GENERIC-LABEL: test_vfnmaddps_256:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    vfnmaddps %ymm2, %ymm1, %ymm0, %ymm0 # sched: [5:0.50]
-; GENERIC-NEXT:    vfnmaddps (%rdi), %ymm1, %ymm0, %ymm0 # sched: [10:0.50]
-; GENERIC-NEXT:    vfnmaddps %ymm1, (%rdi), %ymm0, %ymm0 # sched: [10:0.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    vzeroupper # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER12-LABEL: test_vfnmaddps_256:
-; BDVER12:       # %bb.0:
-; BDVER12-NEXT:    #APP
-; BDVER12-NEXT:    vfnmaddps %ymm2, %ymm1, %ymm0, %ymm0 # sched: [5:0.50]
-; BDVER12-NEXT:    vfnmaddps (%rdi), %ymm1, %ymm0, %ymm0 # sched: [10:1.00]
-; BDVER12-NEXT:    vfnmaddps %ymm1, (%rdi), %ymm0, %ymm0 # sched: [10:1.00]
-; BDVER12-NEXT:    #NO_APP
-; BDVER12-NEXT:    vzeroupper # sched: [46:4.00]
-; BDVER12-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER34-LABEL: test_vfnmaddps_256:
-; BDVER34:       # %bb.0:
-; BDVER34-NEXT:    #APP
-; BDVER34-NEXT:    vfnmaddps %ymm2, %ymm1, %ymm0, %ymm0
-; BDVER34-NEXT:    vfnmaddps (%rdi), %ymm1, %ymm0, %ymm0
-; BDVER34-NEXT:    vfnmaddps %ymm1, (%rdi), %ymm0, %ymm0
-; BDVER34-NEXT:    #NO_APP
-; BDVER34-NEXT:    vzeroupper
-; BDVER34-NEXT:    retq
-  tail call void asm "vfnmaddps $2, $1, $0, $0 \0A\09 vfnmaddps $3, $1, $0, $0 \0A\09 vfnmaddps $1, $3, $0, $0", "x,x,x,*m"(<8 x float> %a0, <8 x float> %a1, <8 x float> %a2, <8 x float> *%a3) nounwind
-  ret void
-}
-
-define void @test_vfnmaddsd_128(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, <2 x double> *%a3) optsize {
-; GENERIC-LABEL: test_vfnmaddsd_128:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    vfnmaddsd %xmm2, %xmm1, %xmm0, %xmm0 # sched: [5:0.50]
-; GENERIC-NEXT:    vfnmaddsd (%rdi), %xmm1, %xmm0, %xmm0 # sched: [10:0.50]
-; GENERIC-NEXT:    vfnmaddsd %xmm1, (%rdi), %xmm0, %xmm0 # sched: [10:0.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER12-LABEL: test_vfnmaddsd_128:
-; BDVER12:       # %bb.0:
-; BDVER12-NEXT:    #APP
-; BDVER12-NEXT:    vfnmaddsd %xmm2, %xmm1, %xmm0, %xmm0 # sched: [5:0.50]
-; BDVER12-NEXT:    vfnmaddsd (%rdi), %xmm1, %xmm0, %xmm0 # sched: [10:0.50]
-; BDVER12-NEXT:    vfnmaddsd %xmm1, (%rdi), %xmm0, %xmm0 # sched: [10:0.50]
-; BDVER12-NEXT:    #NO_APP
-; BDVER12-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER34-LABEL: test_vfnmaddsd_128:
-; BDVER34:       # %bb.0:
-; BDVER34-NEXT:    #APP
-; BDVER34-NEXT:    vfnmaddsd %xmm2, %xmm1, %xmm0, %xmm0
-; BDVER34-NEXT:    vfnmaddsd (%rdi), %xmm1, %xmm0, %xmm0
-; BDVER34-NEXT:    vfnmaddsd %xmm1, (%rdi), %xmm0, %xmm0
-; BDVER34-NEXT:    #NO_APP
-; BDVER34-NEXT:    retq
-  tail call void asm "vfnmaddsd $2, $1, $0, $0 \0A\09 vfnmaddsd $3, $1, $0, $0 \0A\09 vfnmaddsd $1, $3, $0, $0", "x,x,x,*m"(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, <2 x double> *%a3) nounwind
-  ret void
-}
-
-define void @test_vfnmaddss_128(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, <4 x float> *%a3) optsize {
-; GENERIC-LABEL: test_vfnmaddss_128:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    vfnmaddss %xmm2, %xmm1, %xmm0, %xmm0 # sched: [5:0.50]
-; GENERIC-NEXT:    vfnmaddss (%rdi), %xmm1, %xmm0, %xmm0 # sched: [10:0.50]
-; GENERIC-NEXT:    vfnmaddss %xmm1, (%rdi), %xmm0, %xmm0 # sched: [10:0.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER12-LABEL: test_vfnmaddss_128:
-; BDVER12:       # %bb.0:
-; BDVER12-NEXT:    #APP
-; BDVER12-NEXT:    vfnmaddss %xmm2, %xmm1, %xmm0, %xmm0 # sched: [5:0.50]
-; BDVER12-NEXT:    vfnmaddss (%rdi), %xmm1, %xmm0, %xmm0 # sched: [10:0.50]
-; BDVER12-NEXT:    vfnmaddss %xmm1, (%rdi), %xmm0, %xmm0 # sched: [10:0.50]
-; BDVER12-NEXT:    #NO_APP
-; BDVER12-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER34-LABEL: test_vfnmaddss_128:
-; BDVER34:       # %bb.0:
-; BDVER34-NEXT:    #APP
-; BDVER34-NEXT:    vfnmaddss %xmm2, %xmm1, %xmm0, %xmm0
-; BDVER34-NEXT:    vfnmaddss (%rdi), %xmm1, %xmm0, %xmm0
-; BDVER34-NEXT:    vfnmaddss %xmm1, (%rdi), %xmm0, %xmm0
-; BDVER34-NEXT:    #NO_APP
-; BDVER34-NEXT:    retq
-  tail call void asm "vfnmaddss $2, $1, $0, $0 \0A\09 vfnmaddss $3, $1, $0, $0 \0A\09 vfnmaddss $1, $3, $0, $0", "x,x,x,*m"(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, <4 x float> *%a3) nounwind
-  ret void
-}
-
-;
-; VFNMSUB
-;
-
-define void @test_vfnmsubpd_128(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, <2 x double> *%a3) optsize {
-; GENERIC-LABEL: test_vfnmsubpd_128:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    vfnmsubpd %xmm2, %xmm1, %xmm0, %xmm0 # sched: [5:0.50]
-; GENERIC-NEXT:    vfnmsubpd (%rdi), %xmm1, %xmm0, %xmm0 # sched: [10:0.50]
-; GENERIC-NEXT:    vfnmsubpd %xmm1, (%rdi), %xmm0, %xmm0 # sched: [10:0.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER12-LABEL: test_vfnmsubpd_128:
-; BDVER12:       # %bb.0:
-; BDVER12-NEXT:    #APP
-; BDVER12-NEXT:    vfnmsubpd %xmm2, %xmm1, %xmm0, %xmm0 # sched: [5:0.50]
-; BDVER12-NEXT:    vfnmsubpd (%rdi), %xmm1, %xmm0, %xmm0 # sched: [10:0.50]
-; BDVER12-NEXT:    vfnmsubpd %xmm1, (%rdi), %xmm0, %xmm0 # sched: [10:0.50]
-; BDVER12-NEXT:    #NO_APP
-; BDVER12-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER34-LABEL: test_vfnmsubpd_128:
-; BDVER34:       # %bb.0:
-; BDVER34-NEXT:    #APP
-; BDVER34-NEXT:    vfnmsubpd %xmm2, %xmm1, %xmm0, %xmm0
-; BDVER34-NEXT:    vfnmsubpd (%rdi), %xmm1, %xmm0, %xmm0
-; BDVER34-NEXT:    vfnmsubpd %xmm1, (%rdi), %xmm0, %xmm0
-; BDVER34-NEXT:    #NO_APP
-; BDVER34-NEXT:    retq
-  tail call void asm "vfnmsubpd $2, $1, $0, $0 \0A\09 vfnmsubpd $3, $1, $0, $0 \0A\09 vfnmsubpd $1, $3, $0, $0", "x,x,x,*m"(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, <2 x double> *%a3) nounwind
-  ret void
-}
-
-define void @test_vfnmsubpd_256(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2, <4 x double> *%a3) optsize {
-; GENERIC-LABEL: test_vfnmsubpd_256:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    vfnmsubpd %ymm2, %ymm1, %ymm0, %ymm0 # sched: [5:0.50]
-; GENERIC-NEXT:    vfnmsubpd (%rdi), %ymm1, %ymm0, %ymm0 # sched: [10:0.50]
-; GENERIC-NEXT:    vfnmsubpd %ymm1, (%rdi), %ymm0, %ymm0 # sched: [10:0.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    vzeroupper # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER12-LABEL: test_vfnmsubpd_256:
-; BDVER12:       # %bb.0:
-; BDVER12-NEXT:    #APP
-; BDVER12-NEXT:    vfnmsubpd %ymm2, %ymm1, %ymm0, %ymm0 # sched: [5:0.50]
-; BDVER12-NEXT:    vfnmsubpd (%rdi), %ymm1, %ymm0, %ymm0 # sched: [10:1.00]
-; BDVER12-NEXT:    vfnmsubpd %ymm1, (%rdi), %ymm0, %ymm0 # sched: [10:1.00]
-; BDVER12-NEXT:    #NO_APP
-; BDVER12-NEXT:    vzeroupper # sched: [46:4.00]
-; BDVER12-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER34-LABEL: test_vfnmsubpd_256:
-; BDVER34:       # %bb.0:
-; BDVER34-NEXT:    #APP
-; BDVER34-NEXT:    vfnmsubpd %ymm2, %ymm1, %ymm0, %ymm0
-; BDVER34-NEXT:    vfnmsubpd (%rdi), %ymm1, %ymm0, %ymm0
-; BDVER34-NEXT:    vfnmsubpd %ymm1, (%rdi), %ymm0, %ymm0
-; BDVER34-NEXT:    #NO_APP
-; BDVER34-NEXT:    vzeroupper
-; BDVER34-NEXT:    retq
-  tail call void asm "vfnmsubpd $2, $1, $0, $0 \0A\09 vfnmsubpd $3, $1, $0, $0 \0A\09 vfnmsubpd $1, $3, $0, $0", "x,x,x,*m"(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2, <4 x double> *%a3) nounwind
-  ret void
-}
-
-define void @test_vfnmsubps_128(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, <4 x float> *%a3) optsize {
-; GENERIC-LABEL: test_vfnmsubps_128:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    vfnmsubps %xmm2, %xmm1, %xmm0, %xmm0 # sched: [5:0.50]
-; GENERIC-NEXT:    vfnmsubps (%rdi), %xmm1, %xmm0, %xmm0 # sched: [10:0.50]
-; GENERIC-NEXT:    vfnmsubps %xmm1, (%rdi), %xmm0, %xmm0 # sched: [10:0.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER12-LABEL: test_vfnmsubps_128:
-; BDVER12:       # %bb.0:
-; BDVER12-NEXT:    #APP
-; BDVER12-NEXT:    vfnmsubps %xmm2, %xmm1, %xmm0, %xmm0 # sched: [5:0.50]
-; BDVER12-NEXT:    vfnmsubps (%rdi), %xmm1, %xmm0, %xmm0 # sched: [10:0.50]
-; BDVER12-NEXT:    vfnmsubps %xmm1, (%rdi), %xmm0, %xmm0 # sched: [10:0.50]
-; BDVER12-NEXT:    #NO_APP
-; BDVER12-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER34-LABEL: test_vfnmsubps_128:
-; BDVER34:       # %bb.0:
-; BDVER34-NEXT:    #APP
-; BDVER34-NEXT:    vfnmsubps %xmm2, %xmm1, %xmm0, %xmm0
-; BDVER34-NEXT:    vfnmsubps (%rdi), %xmm1, %xmm0, %xmm0
-; BDVER34-NEXT:    vfnmsubps %xmm1, (%rdi), %xmm0, %xmm0
-; BDVER34-NEXT:    #NO_APP
-; BDVER34-NEXT:    retq
-  tail call void asm "vfnmsubps $2, $1, $0, $0 \0A\09 vfnmsubps $3, $1, $0, $0 \0A\09 vfnmsubps $1, $3, $0, $0", "x,x,x,*m"(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, <4 x float> *%a3) nounwind
-  ret void
-}
-
-define void @test_vfnmsubps_256(<8 x float> %a0, <8 x float> %a1, <8 x float> %a2, <8 x float> *%a3) optsize {
-; GENERIC-LABEL: test_vfnmsubps_256:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    vfnmsubps %ymm2, %ymm1, %ymm0, %ymm0 # sched: [5:0.50]
-; GENERIC-NEXT:    vfnmsubps (%rdi), %ymm1, %ymm0, %ymm0 # sched: [10:0.50]
-; GENERIC-NEXT:    vfnmsubps %ymm1, (%rdi), %ymm0, %ymm0 # sched: [10:0.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    vzeroupper # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER12-LABEL: test_vfnmsubps_256:
-; BDVER12:       # %bb.0:
-; BDVER12-NEXT:    #APP
-; BDVER12-NEXT:    vfnmsubps %ymm2, %ymm1, %ymm0, %ymm0 # sched: [5:0.50]
-; BDVER12-NEXT:    vfnmsubps (%rdi), %ymm1, %ymm0, %ymm0 # sched: [10:1.00]
-; BDVER12-NEXT:    vfnmsubps %ymm1, (%rdi), %ymm0, %ymm0 # sched: [10:1.00]
-; BDVER12-NEXT:    #NO_APP
-; BDVER12-NEXT:    vzeroupper # sched: [46:4.00]
-; BDVER12-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER34-LABEL: test_vfnmsubps_256:
-; BDVER34:       # %bb.0:
-; BDVER34-NEXT:    #APP
-; BDVER34-NEXT:    vfnmsubps %ymm2, %ymm1, %ymm0, %ymm0
-; BDVER34-NEXT:    vfnmsubps (%rdi), %ymm1, %ymm0, %ymm0
-; BDVER34-NEXT:    vfnmsubps %ymm1, (%rdi), %ymm0, %ymm0
-; BDVER34-NEXT:    #NO_APP
-; BDVER34-NEXT:    vzeroupper
-; BDVER34-NEXT:    retq
-  tail call void asm "vfnmsubps $2, $1, $0, $0 \0A\09 vfnmsubps $3, $1, $0, $0 \0A\09 vfnmsubps $1, $3, $0, $0", "x,x,x,*m"(<8 x float> %a0, <8 x float> %a1, <8 x float> %a2, <8 x float> *%a3) nounwind
-  ret void
-}
-
-define void @test_vfnmsubsd_128(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, <2 x double> *%a3) optsize {
-; GENERIC-LABEL: test_vfnmsubsd_128:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    vfnmsubsd %xmm2, %xmm1, %xmm0, %xmm0 # sched: [5:0.50]
-; GENERIC-NEXT:    vfnmsubsd (%rdi), %xmm1, %xmm0, %xmm0 # sched: [10:0.50]
-; GENERIC-NEXT:    vfnmsubsd %xmm1, (%rdi), %xmm0, %xmm0 # sched: [10:0.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER12-LABEL: test_vfnmsubsd_128:
-; BDVER12:       # %bb.0:
-; BDVER12-NEXT:    #APP
-; BDVER12-NEXT:    vfnmsubsd %xmm2, %xmm1, %xmm0, %xmm0 # sched: [5:0.50]
-; BDVER12-NEXT:    vfnmsubsd (%rdi), %xmm1, %xmm0, %xmm0 # sched: [10:0.50]
-; BDVER12-NEXT:    vfnmsubsd %xmm1, (%rdi), %xmm0, %xmm0 # sched: [10:0.50]
-; BDVER12-NEXT:    #NO_APP
-; BDVER12-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER34-LABEL: test_vfnmsubsd_128:
-; BDVER34:       # %bb.0:
-; BDVER34-NEXT:    #APP
-; BDVER34-NEXT:    vfnmsubsd %xmm2, %xmm1, %xmm0, %xmm0
-; BDVER34-NEXT:    vfnmsubsd (%rdi), %xmm1, %xmm0, %xmm0
-; BDVER34-NEXT:    vfnmsubsd %xmm1, (%rdi), %xmm0, %xmm0
-; BDVER34-NEXT:    #NO_APP
-; BDVER34-NEXT:    retq
-  tail call void asm "vfnmsubsd $2, $1, $0, $0 \0A\09 vfnmsubsd $3, $1, $0, $0 \0A\09 vfnmsubsd $1, $3, $0, $0", "x,x,x,*m"(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, <2 x double> *%a3) nounwind
-  ret void
-}
-
-define void @test_vfnmsubss_128(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, <4 x float> *%a3) optsize {
-; GENERIC-LABEL: test_vfnmsubss_128:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    vfnmsubss %xmm2, %xmm1, %xmm0, %xmm0 # sched: [5:0.50]
-; GENERIC-NEXT:    vfnmsubss (%rdi), %xmm1, %xmm0, %xmm0 # sched: [10:0.50]
-; GENERIC-NEXT:    vfnmsubss %xmm1, (%rdi), %xmm0, %xmm0 # sched: [10:0.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER12-LABEL: test_vfnmsubss_128:
-; BDVER12:       # %bb.0:
-; BDVER12-NEXT:    #APP
-; BDVER12-NEXT:    vfnmsubss %xmm2, %xmm1, %xmm0, %xmm0 # sched: [5:0.50]
-; BDVER12-NEXT:    vfnmsubss (%rdi), %xmm1, %xmm0, %xmm0 # sched: [10:0.50]
-; BDVER12-NEXT:    vfnmsubss %xmm1, (%rdi), %xmm0, %xmm0 # sched: [10:0.50]
-; BDVER12-NEXT:    #NO_APP
-; BDVER12-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER34-LABEL: test_vfnmsubss_128:
-; BDVER34:       # %bb.0:
-; BDVER34-NEXT:    #APP
-; BDVER34-NEXT:    vfnmsubss %xmm2, %xmm1, %xmm0, %xmm0
-; BDVER34-NEXT:    vfnmsubss (%rdi), %xmm1, %xmm0, %xmm0
-; BDVER34-NEXT:    vfnmsubss %xmm1, (%rdi), %xmm0, %xmm0
-; BDVER34-NEXT:    #NO_APP
-; BDVER34-NEXT:    retq
-  tail call void asm "vfnmsubss $2, $1, $0, $0 \0A\09 vfnmsubss $3, $1, $0, $0 \0A\09 vfnmsubss $1, $3, $0, $0", "x,x,x,*m"(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, <4 x float> *%a3) nounwind
-  ret void
-}
diff --git a/test/CodeGen/X86/fmf-flags.ll b/test/CodeGen/X86/fmf-flags.ll
index 4fb2040b338d93fdafc7ceff8a19afc0c2cb2410..c0ee182b64e94d369fa7a9f957c79555dfbf3188 100644
--- a/test/CodeGen/X86/fmf-flags.ll
+++ b/test/CodeGen/X86/fmf-flags.ll
@@ -8,11 +8,11 @@ define float @fast_recip_sqrt(float %x) {
 ; X64-LABEL: fast_recip_sqrt:
 ; X64:       # %bb.0:
 ; X64-NEXT:    rsqrtss %xmm0, %xmm1
-; X64-NEXT:    mulss   %xmm1, %xmm0
-; X64-NEXT:    mulss   %xmm1, %xmm0
-; X64-NEXT:    addss   {{.*}}(%rip), %xmm0
-; X64-NEXT:    mulss   {{.*}}(%rip), %xmm1
-; X64-NEXT:    mulss   %xmm1, %xmm0
+; X64-NEXT:    mulss %xmm1, %xmm0
+; X64-NEXT:    mulss %xmm1, %xmm0
+; X64-NEXT:    addss {{.*}}(%rip), %xmm0
+; X64-NEXT:    mulss {{.*}}(%rip), %xmm1
+; X64-NEXT:    mulss %xmm1, %xmm0
 ; X64-NEXT:    retq
 ;
 ; X86-LABEL: fast_recip_sqrt:
@@ -20,7 +20,7 @@ define float @fast_recip_sqrt(float %x) {
 ; X86-NEXT:    flds {{[0-9]+}}(%esp)
 ; X86-NEXT:    fsqrt
 ; X86-NEXT:    fld1
-; X86-NEXT:    fdivp %st(1)
+; X86-NEXT:    fdivp %st, %st(1)
 ; X86-NEXT:    retl
   %y = call fast float @llvm.sqrt.f32(float %x)
   %z = fdiv fast float 1.0,  %y
@@ -38,7 +38,7 @@ define float @fast_fmuladd_opts(float %a , float %b , float %c) {
 ; X86-LABEL: fast_fmuladd_opts:
 ; X86:       # %bb.0:
 ; X86-NEXT:    flds {{[0-9]+}}(%esp)
-; X86-NEXT:    fmuls {{.*}}
+; X86-NEXT:    fmuls {{\.LCPI.*}}
 ; X86-NEXT:    retl
   %res = call fast float @llvm.fmuladd.f32(float %a, float 2.0, float %a)
   ret float %res
@@ -51,7 +51,7 @@ define float @fast_fmuladd_opts(float %a , float %b , float %c) {
 define double @not_so_fast_mul_add(double %x) {
 ; X64-LABEL: not_so_fast_mul_add:
 ; X64:       # %bb.0:
-; X64-NEXT:    movsd {{.*}}(%rip), %xmm1
+; X64-NEXT:    movsd {{.*#+}} xmm1 = mem[0],zero
 ; X64-NEXT:    mulsd %xmm0, %xmm1
 ; X64-NEXT:    mulsd {{.*}}(%rip), %xmm0
 ; X64-NEXT:    movsd %xmm1, {{.*}}(%rip)
@@ -81,13 +81,13 @@ define float @not_so_fast_recip_sqrt(float %x) {
 ; X64-LABEL: not_so_fast_recip_sqrt:
 ; X64:       # %bb.0:
 ; X64-NEXT:    rsqrtss %xmm0, %xmm1
-; X64-NEXT:    sqrtss  %xmm0, %xmm2
-; X64-NEXT:    mulss   %xmm1, %xmm0
-; X64-NEXT:    mulss   %xmm1, %xmm0
-; X64-NEXT:    addss   {{.*}}(%rip), %xmm0
-; X64-NEXT:    mulss   {{.*}}(%rip), %xmm1
-; X64-NEXT:    mulss   %xmm1, %xmm0
-; X64-NEXT:    movss   %xmm2, sqrt1(%rip)
+; X64-NEXT:    sqrtss %xmm0, %xmm2
+; X64-NEXT:    mulss %xmm1, %xmm0
+; X64-NEXT:    mulss %xmm1, %xmm0
+; X64-NEXT:    addss {{.*}}(%rip), %xmm0
+; X64-NEXT:    mulss {{.*}}(%rip), %xmm1
+; X64-NEXT:    mulss %xmm1, %xmm0
+; X64-NEXT:    movss %xmm2, {{.*}}(%rip)
 ; X64-NEXT:    retq
 ;
 ; X86-LABEL: not_so_fast_recip_sqrt:
@@ -95,7 +95,7 @@ define float @not_so_fast_recip_sqrt(float %x) {
 ; X86-NEXT:    flds {{[0-9]+}}(%esp)
 ; X86-NEXT:    fsqrt
 ; X86-NEXT:    fld1
-; X86-NEXT:    fdiv %st(1)
+; X86-NEXT:    fdiv %st(1), %st
 ; X86-NEXT:    fxch %st(1)
 ; X86-NEXT:    fstps sqrt1
 ; X86-NEXT:    retl
@@ -107,17 +107,32 @@ define float @not_so_fast_recip_sqrt(float %x) {
 }
 
 define float @div_arcp_by_const(half %x) {
-; X64-LABEL: .LCPI4_0:
-; X64-NEXT:  .long 1036828672
 ; X64-LABEL: div_arcp_by_const:
-; X64:       movzwl %ax, %edi
-; X64:       mulss .LCPI4_0(%rip), %xmm0 
+; X64:       # %bb.0:
+; X64-NEXT:    pushq %rax
+; X64-NEXT:    .cfi_def_cfa_offset 16
+; X64-NEXT:    callq __gnu_f2h_ieee
+; X64-NEXT:    movzwl %ax, %edi
+; X64-NEXT:    callq __gnu_h2f_ieee
+; X64-NEXT:    mulss {{.*}}(%rip), %xmm0
+; X64-NEXT:    popq %rax
+; X64-NEXT:    .cfi_def_cfa_offset 8
+; X64-NEXT:    retq
 ;
-; X86-LABEL: .LCPI4_0:
-; X86-NEXT:  .long 1036828672
 ; X86-LABEL: div_arcp_by_const:
-; X86:       movzwl %ax, %eax
-; X86:       fmuls .LCPI4_0
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %eax
+; X86-NEXT:    .cfi_def_cfa_offset 8
+; X86-NEXT:    flds {{[0-9]+}}(%esp)
+; X86-NEXT:    fstps (%esp)
+; X86-NEXT:    calll __gnu_f2h_ieee
+; X86-NEXT:    movzwl %ax, %eax
+; X86-NEXT:    movl %eax, (%esp)
+; X86-NEXT:    calll __gnu_h2f_ieee
+; X86-NEXT:    fmuls {{\.LCPI.*}}
+; X86-NEXT:    popl %eax
+; X86-NEXT:    .cfi_def_cfa_offset 4
+; X86-NEXT:    retl
   %rcp = fdiv arcp half %x, 10.0
   %z = fpext half %rcp to float
   ret float %z
diff --git a/test/CodeGen/X86/fold-vector-sext-zext.ll b/test/CodeGen/X86/fold-vector-sext-zext.ll
index a8e78cc5565fe91aea42d879234ee5bb233fe3ce..465c7cebf5408379e3114687bf9c65eae6ab9956 100644
--- a/test/CodeGen/X86/fold-vector-sext-zext.ll
+++ b/test/CodeGen/X86/fold-vector-sext-zext.ll
@@ -261,12 +261,12 @@ define <4 x i64> @test_zext_4i8_4i64() {
 define <4 x i16> @test_zext_4i8_4i16_undef() {
 ; X32-LABEL: test_zext_4i8_4i16_undef:
 ; X32:       # %bb.0:
-; X32-NEXT:    vmovaps {{.*#+}} xmm0 = <u,255,u,253>
+; X32-NEXT:    vmovaps {{.*#+}} xmm0 = [0,255,0,253]
 ; X32-NEXT:    retl
 ;
 ; X64-LABEL: test_zext_4i8_4i16_undef:
 ; X64:       # %bb.0:
-; X64-NEXT:    vmovaps {{.*#+}} xmm0 = <u,255,u,253>
+; X64-NEXT:    vmovaps {{.*#+}} xmm0 = [0,255,0,253]
 ; X64-NEXT:    retq
   %1 = insertelement <4 x i8> undef, i8 undef, i32 0
   %2 = insertelement <4 x i8> %1, i8 -1, i32 1
@@ -279,12 +279,12 @@ define <4 x i16> @test_zext_4i8_4i16_undef() {
 define <4 x i32> @test_zext_4i8_4i32_undef() {
 ; X32-LABEL: test_zext_4i8_4i32_undef:
 ; X32:       # %bb.0:
-; X32-NEXT:    vmovaps {{.*#+}} xmm0 = <0,u,2,u>
+; X32-NEXT:    vmovaps {{.*#+}} xmm0 = [0,0,2,0]
 ; X32-NEXT:    retl
 ;
 ; X64-LABEL: test_zext_4i8_4i32_undef:
 ; X64:       # %bb.0:
-; X64-NEXT:    vmovaps {{.*#+}} xmm0 = <0,u,2,u>
+; X64-NEXT:    vmovaps {{.*#+}} xmm0 = [0,0,2,0]
 ; X64-NEXT:    retq
   %1 = insertelement <4 x i8> undef, i8 0, i32 0
   %2 = insertelement <4 x i8> %1, i8 undef, i32 1
@@ -297,12 +297,12 @@ define <4 x i32> @test_zext_4i8_4i32_undef() {
 define <4 x i64> @test_zext_4i8_4i64_undef() {
 ; X32-LABEL: test_zext_4i8_4i64_undef:
 ; X32:       # %bb.0:
-; X32-NEXT:    vmovaps {{.*#+}} ymm0 = <u,u,255,0,2,0,u,u>
+; X32-NEXT:    vmovaps {{.*#+}} ymm0 = [0,0,255,0,2,0,0,0]
 ; X32-NEXT:    retl
 ;
 ; X64-LABEL: test_zext_4i8_4i64_undef:
 ; X64:       # %bb.0:
-; X64-NEXT:    vmovaps {{.*#+}} ymm0 = <u,255,2,u>
+; X64-NEXT:    vmovaps {{.*#+}} ymm0 = [0,255,2,0]
 ; X64-NEXT:    retq
   %1 = insertelement <4 x i8> undef, i8 undef, i32 0
   %2 = insertelement <4 x i8> %1, i8 -1, i32 1
@@ -359,12 +359,12 @@ define <8 x i32> @test_zext_8i8_8i32() {
 define <8 x i16> @test_zext_8i8_8i16_undef() {
 ; X32-LABEL: test_zext_8i8_8i16_undef:
 ; X32:       # %bb.0:
-; X32-NEXT:    vmovaps {{.*#+}} xmm0 = <u,255,u,253,u,251,u,249>
+; X32-NEXT:    vmovaps {{.*#+}} xmm0 = [0,255,0,253,0,251,0,249]
 ; X32-NEXT:    retl
 ;
 ; X64-LABEL: test_zext_8i8_8i16_undef:
 ; X64:       # %bb.0:
-; X64-NEXT:    vmovaps {{.*#+}} xmm0 = <u,255,u,253,u,251,u,249>
+; X64-NEXT:    vmovaps {{.*#+}} xmm0 = [0,255,0,253,0,251,0,249]
 ; X64-NEXT:    retq
   %1 = insertelement <8 x i8> undef, i8 undef, i32 0
   %2 = insertelement <8 x i8> %1, i8 -1, i32 1
@@ -381,12 +381,12 @@ define <8 x i16> @test_zext_8i8_8i16_undef() {
 define <8 x i32> @test_zext_8i8_8i32_undef() {
 ; X32-LABEL: test_zext_8i8_8i32_undef:
 ; X32:       # %bb.0:
-; X32-NEXT:    vmovaps {{.*#+}} ymm0 = <0,u,2,253,4,u,6,u>
+; X32-NEXT:    vmovaps {{.*#+}} ymm0 = [0,0,2,253,4,0,6,0]
 ; X32-NEXT:    retl
 ;
 ; X64-LABEL: test_zext_8i8_8i32_undef:
 ; X64:       # %bb.0:
-; X64-NEXT:    vmovaps {{.*#+}} ymm0 = <0,u,2,253,4,u,6,u>
+; X64-NEXT:    vmovaps {{.*#+}} ymm0 = [0,0,2,253,4,0,6,0]
 ; X64-NEXT:    retq
   %1 = insertelement <8 x i8> undef, i8 0, i32 0
   %2 = insertelement <8 x i8> %1, i8 undef, i32 1
diff --git a/test/CodeGen/X86/fp-cvt.ll b/test/CodeGen/X86/fp-cvt.ll
index ab3d40ddcaa527bb038672b22918f396d5a8cbae..92bff0333be16fea4219580572a0b271633ee6f4 100644
--- a/test/CodeGen/X86/fp-cvt.ll
+++ b/test/CodeGen/X86/fp-cvt.ll
@@ -449,7 +449,7 @@ define i64 @fptoui_i64_fp80(x86_fp80 %a0) nounwind {
 ; X86-NEXT:    fldt 8(%ebp)
 ; X86-NEXT:    flds {{\.LCPI.*}}
 ; X86-NEXT:    fld %st(1)
-; X86-NEXT:    fsub %st(1)
+; X86-NEXT:    fsub %st(1), %st
 ; X86-NEXT:    fxch %st(1)
 ; X86-NEXT:    fucomp %st(2)
 ; X86-NEXT:    fnstsw %ax
@@ -482,11 +482,11 @@ define i64 @fptoui_i64_fp80(x86_fp80 %a0) nounwind {
 ; X64-X87-NEXT:    fldt {{[0-9]+}}(%rsp)
 ; X64-X87-NEXT:    flds {{.*}}(%rip)
 ; X64-X87-NEXT:    fld %st(1)
-; X64-X87-NEXT:    fsub %st(1)
+; X64-X87-NEXT:    fsub %st(1), %st
 ; X64-X87-NEXT:    xorl %eax, %eax
 ; X64-X87-NEXT:    fxch %st(1)
-; X64-X87-NEXT:    fucompi %st(2)
-; X64-X87-NEXT:    fcmovnbe %st(1), %st(0)
+; X64-X87-NEXT:    fucompi %st(2), %st
+; X64-X87-NEXT:    fcmovnbe %st(1), %st
 ; X64-X87-NEXT:    fstp %st(1)
 ; X64-X87-NEXT:    fnstcw -{{[0-9]+}}(%rsp)
 ; X64-X87-NEXT:    movzwl -{{[0-9]+}}(%rsp), %ecx
@@ -505,11 +505,11 @@ define i64 @fptoui_i64_fp80(x86_fp80 %a0) nounwind {
 ; X64-SSSE3-NEXT:    fldt {{[0-9]+}}(%rsp)
 ; X64-SSSE3-NEXT:    flds {{.*}}(%rip)
 ; X64-SSSE3-NEXT:    fld %st(1)
-; X64-SSSE3-NEXT:    fsub %st(1)
+; X64-SSSE3-NEXT:    fsub %st(1), %st
 ; X64-SSSE3-NEXT:    xorl %eax, %eax
 ; X64-SSSE3-NEXT:    fxch %st(1)
-; X64-SSSE3-NEXT:    fucompi %st(2)
-; X64-SSSE3-NEXT:    fcmovnbe %st(1), %st(0)
+; X64-SSSE3-NEXT:    fucompi %st(2), %st
+; X64-SSSE3-NEXT:    fcmovnbe %st(1), %st
 ; X64-SSSE3-NEXT:    fstp %st(1)
 ; X64-SSSE3-NEXT:    fisttpll -{{[0-9]+}}(%rsp)
 ; X64-SSSE3-NEXT:    setbe %al
@@ -531,7 +531,7 @@ define i64 @fptoui_i64_fp80_ld(x86_fp80 *%a0) nounwind {
 ; X86-NEXT:    fldt (%eax)
 ; X86-NEXT:    flds {{\.LCPI.*}}
 ; X86-NEXT:    fld %st(1)
-; X86-NEXT:    fsub %st(1)
+; X86-NEXT:    fsub %st(1), %st
 ; X86-NEXT:    fxch %st(1)
 ; X86-NEXT:    fucomp %st(2)
 ; X86-NEXT:    fnstsw %ax
@@ -564,11 +564,11 @@ define i64 @fptoui_i64_fp80_ld(x86_fp80 *%a0) nounwind {
 ; X64-X87-NEXT:    fldt (%rdi)
 ; X64-X87-NEXT:    flds {{.*}}(%rip)
 ; X64-X87-NEXT:    fld %st(1)
-; X64-X87-NEXT:    fsub %st(1)
+; X64-X87-NEXT:    fsub %st(1), %st
 ; X64-X87-NEXT:    xorl %eax, %eax
 ; X64-X87-NEXT:    fxch %st(1)
-; X64-X87-NEXT:    fucompi %st(2)
-; X64-X87-NEXT:    fcmovnbe %st(1), %st(0)
+; X64-X87-NEXT:    fucompi %st(2), %st
+; X64-X87-NEXT:    fcmovnbe %st(1), %st
 ; X64-X87-NEXT:    fstp %st(1)
 ; X64-X87-NEXT:    fnstcw -{{[0-9]+}}(%rsp)
 ; X64-X87-NEXT:    movzwl -{{[0-9]+}}(%rsp), %ecx
@@ -587,11 +587,11 @@ define i64 @fptoui_i64_fp80_ld(x86_fp80 *%a0) nounwind {
 ; X64-SSSE3-NEXT:    fldt (%rdi)
 ; X64-SSSE3-NEXT:    flds {{.*}}(%rip)
 ; X64-SSSE3-NEXT:    fld %st(1)
-; X64-SSSE3-NEXT:    fsub %st(1)
+; X64-SSSE3-NEXT:    fsub %st(1), %st
 ; X64-SSSE3-NEXT:    xorl %eax, %eax
 ; X64-SSSE3-NEXT:    fxch %st(1)
-; X64-SSSE3-NEXT:    fucompi %st(2)
-; X64-SSSE3-NEXT:    fcmovnbe %st(1), %st(0)
+; X64-SSSE3-NEXT:    fucompi %st(2), %st
+; X64-SSSE3-NEXT:    fcmovnbe %st(1), %st
 ; X64-SSSE3-NEXT:    fstp %st(1)
 ; X64-SSSE3-NEXT:    fisttpll -{{[0-9]+}}(%rsp)
 ; X64-SSSE3-NEXT:    setbe %al
diff --git a/test/CodeGen/X86/fp-elim.ll b/test/CodeGen/X86/fp-elim.ll
index 625c16ef7034a2eece157b556f937341be40ee1e..65764d144a2ca0da21613660dd32aeb118323ca2 100644
--- a/test/CodeGen/X86/fp-elim.ll
+++ b/test/CodeGen/X86/fp-elim.ll
@@ -1,5 +1,5 @@
 ; RUN: llc < %s -mtriple=i686-- -asm-verbose=false                           | FileCheck %s -check-prefix=FP-ELIM
-; RUN: llc < %s -mtriple=i686-- -asm-verbose=false -disable-fp-elim          | FileCheck %s -check-prefix=NO-ELIM
+; RUN: llc < %s -mtriple=i686-- -asm-verbose=false -frame-pointer=all          | FileCheck %s -check-prefix=NO-ELIM
 
 ; Implement -momit-leaf-frame-pointer
 ; rdar://7886181
diff --git a/test/CodeGen/X86/fpstack-debuginstr-kill.ll b/test/CodeGen/X86/fpstack-debuginstr-kill.ll
index 93d2dfd5b8a8326d19521f7cd8e61f8d5cd7ce85..3bf363b2a009a3ab135e8c9457fc045e72bbbb95 100644
--- a/test/CodeGen/X86/fpstack-debuginstr-kill.ll
+++ b/test/CodeGen/X86/fpstack-debuginstr-kill.ll
@@ -59,7 +59,7 @@ attributes #0 = { nounwind readnone }
 !7 = !DIGlobalVariableExpression(var: !8, expr: !DIExpression())
 !8 = !DIGlobalVariable(name: "g2", scope: null, file: !2, line: 6, type: !9, isLocal: false, isDefinition: true)
 !9 = !DIBasicType(name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
-!10 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !11, producer: "clang version 3.6.0 (http://llvm.org/git/clang 8444ae7cfeaefae031f8fedf0d1435ca3b14d90b) (http://llvm.org/git/llvm 886f0101a7d176543b831f5efb74c03427244a55)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !12, retainedTypes: !12, globals: !13, imports: !12)
+!10 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !11, producer: "clang version 3.6.0", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !12, retainedTypes: !12, globals: !13, imports: !12)
 !11 = !DIFile(filename: "fpu_ieee.cpp", directory: "x87stackifier")
 !12 = !{}
 !13 = !{!0, !7}
diff --git a/test/CodeGen/X86/fsgsbase-schedule.ll b/test/CodeGen/X86/fsgsbase-schedule.ll
deleted file mode 100644
index 653df60f8029b523d40e9d5051db62fcf0c1fd57..0000000000000000000000000000000000000000
--- a/test/CodeGen/X86/fsgsbase-schedule.ll
+++ /dev/null
@@ -1,411 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=x86-64 -mattr=fsgsbase | FileCheck %s --check-prefix=GENERIC
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=goldmont | FileCheck %s --check-prefix=GLM
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=ivybridge | FileCheck %s --check-prefix=IVY
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=haswell | FileCheck %s --check-prefix=HASWELL
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=skylake | FileCheck %s --check-prefix=SKYLAKE
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=skx | FileCheck %s --check-prefix=SKX
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=knl     | FileCheck %s --check-prefix=KNL
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=bdver3 | FileCheck %s --check-prefix=BDVER --check-prefix=BDVER3
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=bdver4 | FileCheck %s --check-prefix=BDVER --check-prefix=BDVER4
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=znver1 | FileCheck %s --check-prefix=ZNVER1
-
-define i32 @test_x86_rdfsbase_32() {
-; GENERIC-LABEL: test_x86_rdfsbase_32:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    rdfsbasel %eax # sched: [100:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; GLM-LABEL: test_x86_rdfsbase_32:
-; GLM:       # %bb.0:
-; GLM-NEXT:    rdfsbasel %eax # sched: [100:1.00]
-; GLM-NEXT:    retq # sched: [4:1.00]
-;
-; IVY-LABEL: test_x86_rdfsbase_32:
-; IVY:       # %bb.0:
-; IVY-NEXT:    rdfsbasel %eax # sched: [100:0.33]
-; IVY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_x86_rdfsbase_32:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    rdfsbasel %eax # sched: [100:0.25]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_x86_rdfsbase_32:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    rdfsbasel %eax # sched: [100:0.25]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_x86_rdfsbase_32:
-; SKX:       # %bb.0:
-; SKX-NEXT:    rdfsbasel %eax # sched: [100:0.25]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; KNL-LABEL: test_x86_rdfsbase_32:
-; KNL:       # %bb.0:
-; KNL-NEXT:    rdfsbasel %eax # sched: [100:0.25]
-; KNL-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER-LABEL: test_x86_rdfsbase_32:
-; BDVER:       # %bb.0:
-; BDVER-NEXT:    rdfsbasel %eax
-; BDVER-NEXT:    retq
-;
-; ZNVER1-LABEL: test_x86_rdfsbase_32:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    rdfsbasel %eax # sched: [100:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %res = call i32 @llvm.x86.rdfsbase.32()
-  ret i32 %res
-}
-declare i32 @llvm.x86.rdfsbase.32() nounwind readnone
-
-define i32 @test_x86_rdgsbase_32() {
-; GENERIC-LABEL: test_x86_rdgsbase_32:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    rdgsbasel %eax # sched: [100:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; GLM-LABEL: test_x86_rdgsbase_32:
-; GLM:       # %bb.0:
-; GLM-NEXT:    rdgsbasel %eax # sched: [100:1.00]
-; GLM-NEXT:    retq # sched: [4:1.00]
-;
-; IVY-LABEL: test_x86_rdgsbase_32:
-; IVY:       # %bb.0:
-; IVY-NEXT:    rdgsbasel %eax # sched: [100:0.33]
-; IVY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_x86_rdgsbase_32:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    rdgsbasel %eax # sched: [100:0.25]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_x86_rdgsbase_32:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    rdgsbasel %eax # sched: [100:0.25]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_x86_rdgsbase_32:
-; SKX:       # %bb.0:
-; SKX-NEXT:    rdgsbasel %eax # sched: [100:0.25]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; KNL-LABEL: test_x86_rdgsbase_32:
-; KNL:       # %bb.0:
-; KNL-NEXT:    rdgsbasel %eax # sched: [100:0.25]
-; KNL-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER-LABEL: test_x86_rdgsbase_32:
-; BDVER:       # %bb.0:
-; BDVER-NEXT:    rdgsbasel %eax
-; BDVER-NEXT:    retq
-;
-; ZNVER1-LABEL: test_x86_rdgsbase_32:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    rdgsbasel %eax # sched: [100:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %res = call i32 @llvm.x86.rdgsbase.32()
-  ret i32 %res
-}
-declare i32 @llvm.x86.rdgsbase.32() nounwind readnone
-
-define i64 @test_x86_rdfsbase_64() {
-; GENERIC-LABEL: test_x86_rdfsbase_64:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    rdfsbaseq %rax # sched: [100:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; GLM-LABEL: test_x86_rdfsbase_64:
-; GLM:       # %bb.0:
-; GLM-NEXT:    rdfsbaseq %rax # sched: [100:1.00]
-; GLM-NEXT:    retq # sched: [4:1.00]
-;
-; IVY-LABEL: test_x86_rdfsbase_64:
-; IVY:       # %bb.0:
-; IVY-NEXT:    rdfsbaseq %rax # sched: [100:0.33]
-; IVY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_x86_rdfsbase_64:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    rdfsbaseq %rax # sched: [100:0.25]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_x86_rdfsbase_64:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    rdfsbaseq %rax # sched: [100:0.25]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_x86_rdfsbase_64:
-; SKX:       # %bb.0:
-; SKX-NEXT:    rdfsbaseq %rax # sched: [100:0.25]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; KNL-LABEL: test_x86_rdfsbase_64:
-; KNL:       # %bb.0:
-; KNL-NEXT:    rdfsbaseq %rax # sched: [100:0.25]
-; KNL-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER-LABEL: test_x86_rdfsbase_64:
-; BDVER:       # %bb.0:
-; BDVER-NEXT:    rdfsbaseq %rax
-; BDVER-NEXT:    retq
-;
-; ZNVER1-LABEL: test_x86_rdfsbase_64:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    rdfsbaseq %rax # sched: [100:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %res = call i64 @llvm.x86.rdfsbase.64()
-  ret i64 %res
-}
-declare i64 @llvm.x86.rdfsbase.64() nounwind readnone
-
-define i64 @test_x86_rdgsbase_64() {
-; GENERIC-LABEL: test_x86_rdgsbase_64:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    rdgsbaseq %rax # sched: [100:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; GLM-LABEL: test_x86_rdgsbase_64:
-; GLM:       # %bb.0:
-; GLM-NEXT:    rdgsbaseq %rax # sched: [100:1.00]
-; GLM-NEXT:    retq # sched: [4:1.00]
-;
-; IVY-LABEL: test_x86_rdgsbase_64:
-; IVY:       # %bb.0:
-; IVY-NEXT:    rdgsbaseq %rax # sched: [100:0.33]
-; IVY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_x86_rdgsbase_64:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    rdgsbaseq %rax # sched: [100:0.25]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_x86_rdgsbase_64:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    rdgsbaseq %rax # sched: [100:0.25]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_x86_rdgsbase_64:
-; SKX:       # %bb.0:
-; SKX-NEXT:    rdgsbaseq %rax # sched: [100:0.25]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; KNL-LABEL: test_x86_rdgsbase_64:
-; KNL:       # %bb.0:
-; KNL-NEXT:    rdgsbaseq %rax # sched: [100:0.25]
-; KNL-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER-LABEL: test_x86_rdgsbase_64:
-; BDVER:       # %bb.0:
-; BDVER-NEXT:    rdgsbaseq %rax
-; BDVER-NEXT:    retq
-;
-; ZNVER1-LABEL: test_x86_rdgsbase_64:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    rdgsbaseq %rax # sched: [100:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %res = call i64 @llvm.x86.rdgsbase.64()
-  ret i64 %res
-}
-declare i64 @llvm.x86.rdgsbase.64() nounwind readnone
-
-define void @test_x86_wrfsbase_32(i32 %x) {
-; GENERIC-LABEL: test_x86_wrfsbase_32:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    wrfsbasel %edi # sched: [100:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; GLM-LABEL: test_x86_wrfsbase_32:
-; GLM:       # %bb.0:
-; GLM-NEXT:    wrfsbasel %edi # sched: [100:1.00]
-; GLM-NEXT:    retq # sched: [4:1.00]
-;
-; IVY-LABEL: test_x86_wrfsbase_32:
-; IVY:       # %bb.0:
-; IVY-NEXT:    wrfsbasel %edi # sched: [100:0.33]
-; IVY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_x86_wrfsbase_32:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    wrfsbasel %edi # sched: [100:0.25]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_x86_wrfsbase_32:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    wrfsbasel %edi # sched: [100:0.25]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_x86_wrfsbase_32:
-; SKX:       # %bb.0:
-; SKX-NEXT:    wrfsbasel %edi # sched: [100:0.25]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; KNL-LABEL: test_x86_wrfsbase_32:
-; KNL:       # %bb.0:
-; KNL-NEXT:    wrfsbasel %edi # sched: [100:0.25]
-; KNL-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER-LABEL: test_x86_wrfsbase_32:
-; BDVER:       # %bb.0:
-; BDVER-NEXT:    wrfsbasel %edi
-; BDVER-NEXT:    retq
-;
-; ZNVER1-LABEL: test_x86_wrfsbase_32:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    wrfsbasel %edi # sched: [100:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  call void @llvm.x86.wrfsbase.32(i32 %x)
-  ret void
-}
-declare void @llvm.x86.wrfsbase.32(i32) nounwind readnone
-
-define void @test_x86_wrgsbase_32(i32 %x) {
-; GENERIC-LABEL: test_x86_wrgsbase_32:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    wrgsbasel %edi # sched: [100:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; GLM-LABEL: test_x86_wrgsbase_32:
-; GLM:       # %bb.0:
-; GLM-NEXT:    wrgsbasel %edi # sched: [100:1.00]
-; GLM-NEXT:    retq # sched: [4:1.00]
-;
-; IVY-LABEL: test_x86_wrgsbase_32:
-; IVY:       # %bb.0:
-; IVY-NEXT:    wrgsbasel %edi # sched: [100:0.33]
-; IVY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_x86_wrgsbase_32:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    wrgsbasel %edi # sched: [100:0.25]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_x86_wrgsbase_32:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    wrgsbasel %edi # sched: [100:0.25]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_x86_wrgsbase_32:
-; SKX:       # %bb.0:
-; SKX-NEXT:    wrgsbasel %edi # sched: [100:0.25]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; KNL-LABEL: test_x86_wrgsbase_32:
-; KNL:       # %bb.0:
-; KNL-NEXT:    wrgsbasel %edi # sched: [100:0.25]
-; KNL-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER-LABEL: test_x86_wrgsbase_32:
-; BDVER:       # %bb.0:
-; BDVER-NEXT:    wrgsbasel %edi
-; BDVER-NEXT:    retq
-;
-; ZNVER1-LABEL: test_x86_wrgsbase_32:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    wrgsbasel %edi # sched: [100:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  call void @llvm.x86.wrgsbase.32(i32 %x)
-  ret void
-}
-declare void @llvm.x86.wrgsbase.32(i32) nounwind readnone
-
-define void @test_x86_wrfsbase_64(i64 %x) {
-; GENERIC-LABEL: test_x86_wrfsbase_64:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    wrfsbaseq %rdi # sched: [100:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; GLM-LABEL: test_x86_wrfsbase_64:
-; GLM:       # %bb.0:
-; GLM-NEXT:    wrfsbaseq %rdi # sched: [100:1.00]
-; GLM-NEXT:    retq # sched: [4:1.00]
-;
-; IVY-LABEL: test_x86_wrfsbase_64:
-; IVY:       # %bb.0:
-; IVY-NEXT:    wrfsbaseq %rdi # sched: [100:0.33]
-; IVY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_x86_wrfsbase_64:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    wrfsbaseq %rdi # sched: [100:0.25]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_x86_wrfsbase_64:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    wrfsbaseq %rdi # sched: [100:0.25]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_x86_wrfsbase_64:
-; SKX:       # %bb.0:
-; SKX-NEXT:    wrfsbaseq %rdi # sched: [100:0.25]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; KNL-LABEL: test_x86_wrfsbase_64:
-; KNL:       # %bb.0:
-; KNL-NEXT:    wrfsbaseq %rdi # sched: [100:0.25]
-; KNL-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER-LABEL: test_x86_wrfsbase_64:
-; BDVER:       # %bb.0:
-; BDVER-NEXT:    wrfsbaseq %rdi
-; BDVER-NEXT:    retq
-;
-; ZNVER1-LABEL: test_x86_wrfsbase_64:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    wrfsbaseq %rdi # sched: [100:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  call void @llvm.x86.wrfsbase.64(i64 %x)
-  ret void
-}
-declare void @llvm.x86.wrfsbase.64(i64) nounwind readnone
-
-define void @test_x86_wrgsbase_64(i64 %x) {
-; GENERIC-LABEL: test_x86_wrgsbase_64:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    wrgsbaseq %rdi # sched: [100:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; GLM-LABEL: test_x86_wrgsbase_64:
-; GLM:       # %bb.0:
-; GLM-NEXT:    wrgsbaseq %rdi # sched: [100:1.00]
-; GLM-NEXT:    retq # sched: [4:1.00]
-;
-; IVY-LABEL: test_x86_wrgsbase_64:
-; IVY:       # %bb.0:
-; IVY-NEXT:    wrgsbaseq %rdi # sched: [100:0.33]
-; IVY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_x86_wrgsbase_64:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    wrgsbaseq %rdi # sched: [100:0.25]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_x86_wrgsbase_64:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    wrgsbaseq %rdi # sched: [100:0.25]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_x86_wrgsbase_64:
-; SKX:       # %bb.0:
-; SKX-NEXT:    wrgsbaseq %rdi # sched: [100:0.25]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; KNL-LABEL: test_x86_wrgsbase_64:
-; KNL:       # %bb.0:
-; KNL-NEXT:    wrgsbaseq %rdi # sched: [100:0.25]
-; KNL-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER-LABEL: test_x86_wrgsbase_64:
-; BDVER:       # %bb.0:
-; BDVER-NEXT:    wrgsbaseq %rdi
-; BDVER-NEXT:    retq
-;
-; ZNVER1-LABEL: test_x86_wrgsbase_64:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    wrgsbaseq %rdi # sched: [100:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  call void @llvm.x86.wrgsbase.64(i64 %x)
-  ret void
-}
-declare void @llvm.x86.wrgsbase.64(i64) nounwind readnone
diff --git a/test/CodeGen/X86/funnel-shift-rot.ll b/test/CodeGen/X86/funnel-shift-rot.ll
index e07d34f9e6421654d9ec2aaf90974f5355b604de..fa1679902ecaf8c5a9785b1e1c1722e914d56114 100644
--- a/test/CodeGen/X86/funnel-shift-rot.ll
+++ b/test/CodeGen/X86/funnel-shift-rot.ll
@@ -96,51 +96,32 @@ define i32 @rotl_i32(i32 %x, i32 %z) nounwind {
 define <4 x i32> @rotl_v4i32(<4 x i32> %x, <4 x i32> %z) nounwind {
 ; X32-SSE2-LABEL: rotl_v4i32:
 ; X32-SSE2:       # %bb.0:
-; X32-SSE2-NEXT:    pxor %xmm3, %xmm3
-; X32-SSE2-NEXT:    psubd %xmm1, %xmm3
-; X32-SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [31,31,31,31]
-; X32-SSE2-NEXT:    pand %xmm4, %xmm3
-; X32-SSE2-NEXT:    pshuflw {{.*#+}} xmm2 = xmm3[2,3,3,3,4,5,6,7]
-; X32-SSE2-NEXT:    movdqa %xmm0, %xmm5
-; X32-SSE2-NEXT:    psrld %xmm2, %xmm5
-; X32-SSE2-NEXT:    pshuflw {{.*#+}} xmm6 = xmm3[0,1,1,1,4,5,6,7]
-; X32-SSE2-NEXT:    movdqa %xmm0, %xmm2
-; X32-SSE2-NEXT:    psrld %xmm6, %xmm2
-; X32-SSE2-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm5[0]
-; X32-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[2,3,0,1]
-; X32-SSE2-NEXT:    pshuflw {{.*#+}} xmm5 = xmm3[2,3,3,3,4,5,6,7]
-; X32-SSE2-NEXT:    movdqa %xmm0, %xmm6
-; X32-SSE2-NEXT:    psrld %xmm5, %xmm6
-; X32-SSE2-NEXT:    pshuflw {{.*#+}} xmm3 = xmm3[0,1,1,1,4,5,6,7]
-; X32-SSE2-NEXT:    movdqa %xmm0, %xmm5
-; X32-SSE2-NEXT:    psrld %xmm3, %xmm5
-; X32-SSE2-NEXT:    punpckhqdq {{.*#+}} xmm5 = xmm5[1],xmm6[1]
-; X32-SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,3],xmm5[0,3]
-; X32-SSE2-NEXT:    pand %xmm4, %xmm1
+; X32-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm1
 ; X32-SSE2-NEXT:    pslld $23, %xmm1
 ; X32-SSE2-NEXT:    paddd {{\.LCPI.*}}, %xmm1
 ; X32-SSE2-NEXT:    cvttps2dq %xmm1, %xmm1
-; X32-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; X32-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
 ; X32-SSE2-NEXT:    pmuludq %xmm1, %xmm0
-; X32-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; X32-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,3,2,3]
 ; X32-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; X32-SSE2-NEXT:    pmuludq %xmm3, %xmm1
+; X32-SSE2-NEXT:    pmuludq %xmm2, %xmm1
+; X32-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,3,2,3]
+; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
+; X32-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
 ; X32-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
 ; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; X32-SSE2-NEXT:    orps %xmm0, %xmm2
-; X32-SSE2-NEXT:    movaps %xmm2, %xmm0
+; X32-SSE2-NEXT:    por %xmm3, %xmm0
 ; X32-SSE2-NEXT:    retl
 ;
 ; X64-AVX2-LABEL: rotl_v4i32:
 ; X64-AVX2:       # %bb.0:
 ; X64-AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [31,31,31,31]
-; X64-AVX2-NEXT:    vpand %xmm2, %xmm1, %xmm3
-; X64-AVX2-NEXT:    vpsllvd %xmm3, %xmm0, %xmm3
-; X64-AVX2-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; X64-AVX2-NEXT:    vpsubd %xmm1, %xmm4, %xmm1
 ; X64-AVX2-NEXT:    vpand %xmm2, %xmm1, %xmm1
+; X64-AVX2-NEXT:    vpsllvd %xmm1, %xmm0, %xmm2
+; X64-AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm3 = [32,32,32,32]
+; X64-AVX2-NEXT:    vpsubd %xmm1, %xmm3, %xmm1
 ; X64-AVX2-NEXT:    vpsrlvd %xmm1, %xmm0, %xmm0
-; X64-AVX2-NEXT:    vpor %xmm0, %xmm3, %xmm0
+; X64-AVX2-NEXT:    vpor %xmm0, %xmm2, %xmm0
 ; X64-AVX2-NEXT:    retq
   %f = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %x, <4 x i32> %x, <4 x i32> %z)
   ret <4 x i32> %f
@@ -273,52 +254,36 @@ define i64 @rotr_i64(i64 %x, i64 %z) nounwind {
 define <4 x i32> @rotr_v4i32(<4 x i32> %x, <4 x i32> %z) nounwind {
 ; X32-SSE2-LABEL: rotr_v4i32:
 ; X32-SSE2:       # %bb.0:
-; X32-SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [31,31,31,31]
-; X32-SSE2-NEXT:    pxor %xmm3, %xmm3
-; X32-SSE2-NEXT:    psubd %xmm1, %xmm3
-; X32-SSE2-NEXT:    movdqa %xmm1, %xmm4
-; X32-SSE2-NEXT:    pand %xmm2, %xmm4
-; X32-SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm4[2,3,3,3,4,5,6,7]
-; X32-SSE2-NEXT:    movdqa %xmm0, %xmm5
-; X32-SSE2-NEXT:    psrld %xmm1, %xmm5
-; X32-SSE2-NEXT:    pshuflw {{.*#+}} xmm6 = xmm4[0,1,1,1,4,5,6,7]
-; X32-SSE2-NEXT:    movdqa %xmm0, %xmm1
-; X32-SSE2-NEXT:    psrld %xmm6, %xmm1
-; X32-SSE2-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm5[0]
-; X32-SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[2,3,0,1]
-; X32-SSE2-NEXT:    pshuflw {{.*#+}} xmm5 = xmm4[2,3,3,3,4,5,6,7]
-; X32-SSE2-NEXT:    movdqa %xmm0, %xmm6
-; X32-SSE2-NEXT:    psrld %xmm5, %xmm6
-; X32-SSE2-NEXT:    pshuflw {{.*#+}} xmm4 = xmm4[0,1,1,1,4,5,6,7]
-; X32-SSE2-NEXT:    movdqa %xmm0, %xmm5
-; X32-SSE2-NEXT:    psrld %xmm4, %xmm5
-; X32-SSE2-NEXT:    punpckhqdq {{.*#+}} xmm5 = xmm5[1],xmm6[1]
-; X32-SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,3],xmm5[0,3]
-; X32-SSE2-NEXT:    pand %xmm2, %xmm3
-; X32-SSE2-NEXT:    pslld $23, %xmm3
-; X32-SSE2-NEXT:    paddd {{\.LCPI.*}}, %xmm3
-; X32-SSE2-NEXT:    cvttps2dq %xmm3, %xmm2
-; X32-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
-; X32-SSE2-NEXT:    pmuludq %xmm2, %xmm0
+; X32-SSE2-NEXT:    pxor %xmm2, %xmm2
+; X32-SSE2-NEXT:    psubd %xmm1, %xmm2
+; X32-SSE2-NEXT:    pand {{\.LCPI.*}}, %xmm2
+; X32-SSE2-NEXT:    pslld $23, %xmm2
+; X32-SSE2-NEXT:    paddd {{\.LCPI.*}}, %xmm2
+; X32-SSE2-NEXT:    cvttps2dq %xmm2, %xmm1
+; X32-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
+; X32-SSE2-NEXT:    pmuludq %xmm1, %xmm0
+; X32-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,3,2,3]
+; X32-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; X32-SSE2-NEXT:    pmuludq %xmm2, %xmm1
+; X32-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,3,2,3]
+; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
 ; X32-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
-; X32-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; X32-SSE2-NEXT:    pmuludq %xmm3, %xmm2
-; X32-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
-; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
-; X32-SSE2-NEXT:    orps %xmm0, %xmm1
-; X32-SSE2-NEXT:    movaps %xmm1, %xmm0
+; X32-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; X32-SSE2-NEXT:    por %xmm3, %xmm0
 ; X32-SSE2-NEXT:    retl
 ;
 ; X64-AVX2-LABEL: rotr_v4i32:
 ; X64-AVX2:       # %bb.0:
+; X64-AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; X64-AVX2-NEXT:    vpsubd %xmm1, %xmm2, %xmm1
 ; X64-AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [31,31,31,31]
-; X64-AVX2-NEXT:    vpand %xmm2, %xmm1, %xmm3
-; X64-AVX2-NEXT:    vpsrlvd %xmm3, %xmm0, %xmm3
-; X64-AVX2-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; X64-AVX2-NEXT:    vpsubd %xmm1, %xmm4, %xmm1
 ; X64-AVX2-NEXT:    vpand %xmm2, %xmm1, %xmm1
-; X64-AVX2-NEXT:    vpsllvd %xmm1, %xmm0, %xmm0
-; X64-AVX2-NEXT:    vpor %xmm3, %xmm0, %xmm0
+; X64-AVX2-NEXT:    vpsllvd %xmm1, %xmm0, %xmm2
+; X64-AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm3 = [32,32,32,32]
+; X64-AVX2-NEXT:    vpsubd %xmm1, %xmm3, %xmm1
+; X64-AVX2-NEXT:    vpsrlvd %xmm1, %xmm0, %xmm0
+; X64-AVX2-NEXT:    vpor %xmm0, %xmm2, %xmm0
 ; X64-AVX2-NEXT:    retq
   %f = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %x, <4 x i32> %x, <4 x i32> %z)
   ret <4 x i32> %f
diff --git a/test/CodeGen/X86/funnel-shift.ll b/test/CodeGen/X86/funnel-shift.ll
index 0969d6d190c244ea3a5ec0578797fd632c5069b9..517880fb88e571d6f6436e9d7106ba9791530a45 100644
--- a/test/CodeGen/X86/funnel-shift.ll
+++ b/test/CodeGen/X86/funnel-shift.ll
@@ -318,6 +318,486 @@ define i7 @fshr_i7_const_fold() nounwind {
   ret i7 %f
 }
 
+; demanded bits tests
+
+define i32 @fshl_i32_demandedbits(i32 %a0, i32 %a1) nounwind {
+; X32-SSE2-LABEL: fshl_i32_demandedbits:
+; X32-SSE2:       # %bb.0:
+; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-SSE2-NEXT:    shldl $9, %ecx, %eax
+; X32-SSE2-NEXT:    retl
+;
+; X64-AVX2-LABEL: fshl_i32_demandedbits:
+; X64-AVX2:       # %bb.0:
+; X64-AVX2-NEXT:    movl %edi, %eax
+; X64-AVX2-NEXT:    shldl $9, %esi, %eax
+; X64-AVX2-NEXT:    retq
+  %x = or i32 %a0, 2147483648
+  %y = or i32 %a1, 1
+  %res = call i32 @llvm.fshl.i32(i32 %x, i32 %y, i32 9)
+  ret i32 %res
+}
+
+define i32 @fshr_i32_demandedbits(i32 %a0, i32 %a1) nounwind {
+; X32-SSE2-LABEL: fshr_i32_demandedbits:
+; X32-SSE2:       # %bb.0:
+; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-SSE2-NEXT:    shrdl $9, %ecx, %eax
+; X32-SSE2-NEXT:    retl
+;
+; X64-AVX2-LABEL: fshr_i32_demandedbits:
+; X64-AVX2:       # %bb.0:
+; X64-AVX2-NEXT:    movl %edi, %eax
+; X64-AVX2-NEXT:    shldl $23, %esi, %eax
+; X64-AVX2-NEXT:    retq
+  %x = or i32 %a0, 2147483648
+  %y = or i32 %a1, 1
+  %res = call i32 @llvm.fshr.i32(i32 %x, i32 %y, i32 9)
+  ret i32 %res
+}
+
+; undef handling
+
+define i32 @fshl_i32_undef0(i32 %a0, i32 %a1) nounwind {
+; X32-SSE2-LABEL: fshl_i32_undef0:
+; X32-SSE2:       # %bb.0:
+; X32-SSE2-NEXT:    movb {{[0-9]+}}(%esp), %cl
+; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-SSE2-NEXT:    shldl %cl, %eax, %eax
+; X32-SSE2-NEXT:    retl
+;
+; X64-AVX2-LABEL: fshl_i32_undef0:
+; X64-AVX2:       # %bb.0:
+; X64-AVX2-NEXT:    movl %esi, %ecx
+; X64-AVX2-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X64-AVX2-NEXT:    shldl %cl, %edi, %eax
+; X64-AVX2-NEXT:    retq
+  %res = call i32 @llvm.fshl.i32(i32 undef, i32 %a0, i32 %a1)
+  ret i32 %res
+}
+
+define i32 @fshl_i32_undef0_msk(i32 %a0, i32 %a1) nounwind {
+; X32-SSE2-LABEL: fshl_i32_undef0_msk:
+; X32-SSE2:       # %bb.0:
+; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X32-SSE2-NEXT:    andl $7, %ecx
+; X32-SSE2-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X32-SSE2-NEXT:    shldl %cl, %eax, %eax
+; X32-SSE2-NEXT:    retl
+;
+; X64-AVX2-LABEL: fshl_i32_undef0_msk:
+; X64-AVX2:       # %bb.0:
+; X64-AVX2-NEXT:    movl %esi, %ecx
+; X64-AVX2-NEXT:    andl $7, %ecx
+; X64-AVX2-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X64-AVX2-NEXT:    shldl %cl, %edi, %eax
+; X64-AVX2-NEXT:    retq
+  %m = and i32 %a1, 7
+  %res = call i32 @llvm.fshl.i32(i32 undef, i32 %a0, i32 %m)
+  ret i32 %res
+}
+
+define i32 @fshl_i32_undef0_cst(i32 %a0) nounwind {
+; X32-SSE2-LABEL: fshl_i32_undef0_cst:
+; X32-SSE2:       # %bb.0:
+; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-SSE2-NEXT:    shrl $23, %eax
+; X32-SSE2-NEXT:    retl
+;
+; X64-AVX2-LABEL: fshl_i32_undef0_cst:
+; X64-AVX2:       # %bb.0:
+; X64-AVX2-NEXT:    movl %edi, %eax
+; X64-AVX2-NEXT:    shrl $23, %eax
+; X64-AVX2-NEXT:    retq
+  %res = call i32 @llvm.fshl.i32(i32 undef, i32 %a0, i32 9)
+  ret i32 %res
+}
+
+define i32 @fshl_i32_undef1(i32 %a0, i32 %a1) nounwind {
+; X32-SSE2-LABEL: fshl_i32_undef1:
+; X32-SSE2:       # %bb.0:
+; X32-SSE2-NEXT:    movb {{[0-9]+}}(%esp), %cl
+; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-SSE2-NEXT:    shldl %cl, %eax, %eax
+; X32-SSE2-NEXT:    retl
+;
+; X64-AVX2-LABEL: fshl_i32_undef1:
+; X64-AVX2:       # %bb.0:
+; X64-AVX2-NEXT:    movl %esi, %ecx
+; X64-AVX2-NEXT:    movl %edi, %eax
+; X64-AVX2-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X64-AVX2-NEXT:    shldl %cl, %eax, %eax
+; X64-AVX2-NEXT:    retq
+  %res = call i32 @llvm.fshl.i32(i32 %a0, i32 undef, i32 %a1)
+  ret i32 %res
+}
+
+define i32 @fshl_i32_undef1_msk(i32 %a0, i32 %a1) nounwind {
+; X32-SSE2-LABEL: fshl_i32_undef1_msk:
+; X32-SSE2:       # %bb.0:
+; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-SSE2-NEXT:    movb {{[0-9]+}}(%esp), %cl
+; X32-SSE2-NEXT:    andb $7, %cl
+; X32-SSE2-NEXT:    shll %cl, %eax
+; X32-SSE2-NEXT:    retl
+;
+; X64-AVX2-LABEL: fshl_i32_undef1_msk:
+; X64-AVX2:       # %bb.0:
+; X64-AVX2-NEXT:    movl %esi, %ecx
+; X64-AVX2-NEXT:    movl %edi, %eax
+; X64-AVX2-NEXT:    andb $7, %cl
+; X64-AVX2-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X64-AVX2-NEXT:    shll %cl, %eax
+; X64-AVX2-NEXT:    retq
+  %m = and i32 %a1, 7
+  %res = call i32 @llvm.fshl.i32(i32 %a0, i32 undef, i32 %m)
+  ret i32 %res
+}
+
+define i32 @fshl_i32_undef1_cst(i32 %a0) nounwind {
+; X32-SSE2-LABEL: fshl_i32_undef1_cst:
+; X32-SSE2:       # %bb.0:
+; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-SSE2-NEXT:    shll $9, %eax
+; X32-SSE2-NEXT:    retl
+;
+; X64-AVX2-LABEL: fshl_i32_undef1_cst:
+; X64-AVX2:       # %bb.0:
+; X64-AVX2-NEXT:    movl %edi, %eax
+; X64-AVX2-NEXT:    shll $9, %eax
+; X64-AVX2-NEXT:    retq
+  %res = call i32 @llvm.fshl.i32(i32 %a0, i32 undef, i32 9)
+  ret i32 %res
+}
+
+define i32 @fshl_i32_undef2(i32 %a0, i32 %a1) nounwind {
+; X32-SSE2-LABEL: fshl_i32_undef2:
+; X32-SSE2:       # %bb.0:
+; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-SSE2-NEXT:    shldl %cl, %ecx, %eax
+; X32-SSE2-NEXT:    retl
+;
+; X64-AVX2-LABEL: fshl_i32_undef2:
+; X64-AVX2:       # %bb.0:
+; X64-AVX2-NEXT:    movl %edi, %eax
+; X64-AVX2-NEXT:    shldl %cl, %esi, %eax
+; X64-AVX2-NEXT:    retq
+  %res = call i32 @llvm.fshl.i32(i32 %a0, i32 %a1, i32 undef)
+  ret i32 %res
+}
+
+define i32 @fshr_i32_undef0(i32 %a0, i32 %a1) nounwind {
+; X32-SSE2-LABEL: fshr_i32_undef0:
+; X32-SSE2:       # %bb.0:
+; X32-SSE2-NEXT:    movb {{[0-9]+}}(%esp), %cl
+; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-SSE2-NEXT:    shrdl %cl, %eax, %eax
+; X32-SSE2-NEXT:    retl
+;
+; X64-AVX2-LABEL: fshr_i32_undef0:
+; X64-AVX2:       # %bb.0:
+; X64-AVX2-NEXT:    movl %esi, %ecx
+; X64-AVX2-NEXT:    movl %edi, %eax
+; X64-AVX2-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X64-AVX2-NEXT:    shrdl %cl, %eax, %eax
+; X64-AVX2-NEXT:    retq
+  %res = call i32 @llvm.fshr.i32(i32 undef, i32 %a0, i32 %a1)
+  ret i32 %res
+}
+
+define i32 @fshr_i32_undef0_msk(i32 %a0, i32 %a1) nounwind {
+; X32-SSE2-LABEL: fshr_i32_undef0_msk:
+; X32-SSE2:       # %bb.0:
+; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-SSE2-NEXT:    movb {{[0-9]+}}(%esp), %cl
+; X32-SSE2-NEXT:    andb $7, %cl
+; X32-SSE2-NEXT:    shrl %cl, %eax
+; X32-SSE2-NEXT:    retl
+;
+; X64-AVX2-LABEL: fshr_i32_undef0_msk:
+; X64-AVX2:       # %bb.0:
+; X64-AVX2-NEXT:    movl %esi, %ecx
+; X64-AVX2-NEXT:    movl %edi, %eax
+; X64-AVX2-NEXT:    andb $7, %cl
+; X64-AVX2-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X64-AVX2-NEXT:    shrl %cl, %eax
+; X64-AVX2-NEXT:    retq
+  %m = and i32 %a1, 7
+  %res = call i32 @llvm.fshr.i32(i32 undef, i32 %a0, i32 %m)
+  ret i32 %res
+}
+
+define i32 @fshr_i32_undef0_cst(i32 %a0) nounwind {
+; X32-SSE2-LABEL: fshr_i32_undef0_cst:
+; X32-SSE2:       # %bb.0:
+; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-SSE2-NEXT:    shrl $9, %eax
+; X32-SSE2-NEXT:    retl
+;
+; X64-AVX2-LABEL: fshr_i32_undef0_cst:
+; X64-AVX2:       # %bb.0:
+; X64-AVX2-NEXT:    movl %edi, %eax
+; X64-AVX2-NEXT:    shrl $9, %eax
+; X64-AVX2-NEXT:    retq
+  %res = call i32 @llvm.fshr.i32(i32 undef, i32 %a0, i32 9)
+  ret i32 %res
+}
+
+define i32 @fshr_i32_undef1(i32 %a0, i32 %a1) nounwind {
+; X32-SSE2-LABEL: fshr_i32_undef1:
+; X32-SSE2:       # %bb.0:
+; X32-SSE2-NEXT:    movb {{[0-9]+}}(%esp), %cl
+; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-SSE2-NEXT:    shrdl %cl, %eax, %eax
+; X32-SSE2-NEXT:    retl
+;
+; X64-AVX2-LABEL: fshr_i32_undef1:
+; X64-AVX2:       # %bb.0:
+; X64-AVX2-NEXT:    movl %esi, %ecx
+; X64-AVX2-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X64-AVX2-NEXT:    shrdl %cl, %edi, %eax
+; X64-AVX2-NEXT:    retq
+  %res = call i32 @llvm.fshr.i32(i32 %a0, i32 undef, i32 %a1)
+  ret i32 %res
+}
+
+define i32 @fshr_i32_undef1_msk(i32 %a0, i32 %a1) nounwind {
+; X32-SSE2-LABEL: fshr_i32_undef1_msk:
+; X32-SSE2:       # %bb.0:
+; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X32-SSE2-NEXT:    andl $7, %ecx
+; X32-SSE2-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X32-SSE2-NEXT:    shrdl %cl, %eax, %eax
+; X32-SSE2-NEXT:    retl
+;
+; X64-AVX2-LABEL: fshr_i32_undef1_msk:
+; X64-AVX2:       # %bb.0:
+; X64-AVX2-NEXT:    movl %esi, %ecx
+; X64-AVX2-NEXT:    andl $7, %ecx
+; X64-AVX2-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X64-AVX2-NEXT:    shrdl %cl, %edi, %eax
+; X64-AVX2-NEXT:    retq
+  %m = and i32 %a1, 7
+  %res = call i32 @llvm.fshr.i32(i32 %a0, i32 undef, i32 %m)
+  ret i32 %res
+}
+
+define i32 @fshr_i32_undef1_cst(i32 %a0) nounwind {
+; X32-SSE2-LABEL: fshr_i32_undef1_cst:
+; X32-SSE2:       # %bb.0:
+; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-SSE2-NEXT:    shll $23, %eax
+; X32-SSE2-NEXT:    retl
+;
+; X64-AVX2-LABEL: fshr_i32_undef1_cst:
+; X64-AVX2:       # %bb.0:
+; X64-AVX2-NEXT:    movl %edi, %eax
+; X64-AVX2-NEXT:    shll $23, %eax
+; X64-AVX2-NEXT:    retq
+  %res = call i32 @llvm.fshr.i32(i32 %a0, i32 undef, i32 9)
+  ret i32 %res
+}
+
+define i32 @fshr_i32_undef2(i32 %a0, i32 %a1) nounwind {
+; X32-SSE2-LABEL: fshr_i32_undef2:
+; X32-SSE2:       # %bb.0:
+; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-SSE2-NEXT:    shrdl %cl, %ecx, %eax
+; X32-SSE2-NEXT:    retl
+;
+; X64-AVX2-LABEL: fshr_i32_undef2:
+; X64-AVX2:       # %bb.0:
+; X64-AVX2-NEXT:    movl %esi, %eax
+; X64-AVX2-NEXT:    shrdl %cl, %edi, %eax
+; X64-AVX2-NEXT:    retq
+  %res = call i32 @llvm.fshr.i32(i32 %a0, i32 %a1, i32 undef)
+  ret i32 %res
+}
+
+; shift zero args
+
+define i32 @fshl_i32_zero0(i32 %a0, i32 %a1) nounwind {
+; X32-SSE2-LABEL: fshl_i32_zero0:
+; X32-SSE2:       # %bb.0:
+; X32-SSE2-NEXT:    movb {{[0-9]+}}(%esp), %cl
+; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X32-SSE2-NEXT:    xorl %eax, %eax
+; X32-SSE2-NEXT:    shldl %cl, %edx, %eax
+; X32-SSE2-NEXT:    retl
+;
+; X64-AVX2-LABEL: fshl_i32_zero0:
+; X64-AVX2:       # %bb.0:
+; X64-AVX2-NEXT:    movl %esi, %ecx
+; X64-AVX2-NEXT:    xorl %eax, %eax
+; X64-AVX2-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X64-AVX2-NEXT:    shldl %cl, %edi, %eax
+; X64-AVX2-NEXT:    retq
+  %res = call i32 @llvm.fshl.i32(i32 0, i32 %a0, i32 %a1)
+  ret i32 %res
+}
+
+define i32 @fshl_i32_zero0_cst(i32 %a0) nounwind {
+; X32-SSE2-LABEL: fshl_i32_zero0_cst:
+; X32-SSE2:       # %bb.0:
+; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-SSE2-NEXT:    shrl $23, %eax
+; X32-SSE2-NEXT:    retl
+;
+; X64-AVX2-LABEL: fshl_i32_zero0_cst:
+; X64-AVX2:       # %bb.0:
+; X64-AVX2-NEXT:    movl %edi, %eax
+; X64-AVX2-NEXT:    shrl $23, %eax
+; X64-AVX2-NEXT:    retq
+  %res = call i32 @llvm.fshl.i32(i32 0, i32 %a0, i32 9)
+  ret i32 %res
+}
+
+define i32 @fshl_i32_zero1(i32 %a0, i32 %a1) nounwind {
+; X32-SSE2-LABEL: fshl_i32_zero1:
+; X32-SSE2:       # %bb.0:
+; X32-SSE2-NEXT:    movb {{[0-9]+}}(%esp), %cl
+; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-SSE2-NEXT:    xorl %edx, %edx
+; X32-SSE2-NEXT:    shldl %cl, %edx, %eax
+; X32-SSE2-NEXT:    retl
+;
+; X64-AVX2-LABEL: fshl_i32_zero1:
+; X64-AVX2:       # %bb.0:
+; X64-AVX2-NEXT:    movl %esi, %ecx
+; X64-AVX2-NEXT:    movl %edi, %eax
+; X64-AVX2-NEXT:    xorl %edx, %edx
+; X64-AVX2-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X64-AVX2-NEXT:    shldl %cl, %edx, %eax
+; X64-AVX2-NEXT:    retq
+  %res = call i32 @llvm.fshl.i32(i32 %a0, i32 0, i32 %a1)
+  ret i32 %res
+}
+
+define i32 @fshl_i32_zero1_cst(i32 %a0) nounwind {
+; X32-SSE2-LABEL: fshl_i32_zero1_cst:
+; X32-SSE2:       # %bb.0:
+; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-SSE2-NEXT:    shll $9, %eax
+; X32-SSE2-NEXT:    retl
+;
+; X64-AVX2-LABEL: fshl_i32_zero1_cst:
+; X64-AVX2:       # %bb.0:
+; X64-AVX2-NEXT:    movl %edi, %eax
+; X64-AVX2-NEXT:    shll $9, %eax
+; X64-AVX2-NEXT:    retq
+  %res = call i32 @llvm.fshl.i32(i32 %a0, i32 0, i32 9)
+  ret i32 %res
+}
+
+define i32 @fshr_i32_zero0(i32 %a0, i32 %a1) nounwind {
+; X32-SSE2-LABEL: fshr_i32_zero0:
+; X32-SSE2:       # %bb.0:
+; X32-SSE2-NEXT:    movb {{[0-9]+}}(%esp), %cl
+; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-SSE2-NEXT:    xorl %edx, %edx
+; X32-SSE2-NEXT:    shrdl %cl, %edx, %eax
+; X32-SSE2-NEXT:    retl
+;
+; X64-AVX2-LABEL: fshr_i32_zero0:
+; X64-AVX2:       # %bb.0:
+; X64-AVX2-NEXT:    movl %esi, %ecx
+; X64-AVX2-NEXT:    movl %edi, %eax
+; X64-AVX2-NEXT:    xorl %edx, %edx
+; X64-AVX2-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X64-AVX2-NEXT:    shrdl %cl, %edx, %eax
+; X64-AVX2-NEXT:    retq
+  %res = call i32 @llvm.fshr.i32(i32 0, i32 %a0, i32 %a1)
+  ret i32 %res
+}
+
+define i32 @fshr_i32_zero0_cst(i32 %a0) nounwind {
+; X32-SSE2-LABEL: fshr_i32_zero0_cst:
+; X32-SSE2:       # %bb.0:
+; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-SSE2-NEXT:    shrl $9, %eax
+; X32-SSE2-NEXT:    retl
+;
+; X64-AVX2-LABEL: fshr_i32_zero0_cst:
+; X64-AVX2:       # %bb.0:
+; X64-AVX2-NEXT:    movl %edi, %eax
+; X64-AVX2-NEXT:    shrl $9, %eax
+; X64-AVX2-NEXT:    retq
+  %res = call i32 @llvm.fshr.i32(i32 0, i32 %a0, i32 9)
+  ret i32 %res
+}
+
+define i32 @fshr_i32_zero1(i32 %a0, i32 %a1) nounwind {
+; X32-SSE2-LABEL: fshr_i32_zero1:
+; X32-SSE2:       # %bb.0:
+; X32-SSE2-NEXT:    movb {{[0-9]+}}(%esp), %cl
+; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X32-SSE2-NEXT:    xorl %eax, %eax
+; X32-SSE2-NEXT:    shrdl %cl, %edx, %eax
+; X32-SSE2-NEXT:    retl
+;
+; X64-AVX2-LABEL: fshr_i32_zero1:
+; X64-AVX2:       # %bb.0:
+; X64-AVX2-NEXT:    movl %esi, %ecx
+; X64-AVX2-NEXT:    xorl %eax, %eax
+; X64-AVX2-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X64-AVX2-NEXT:    shrdl %cl, %edi, %eax
+; X64-AVX2-NEXT:    retq
+  %res = call i32 @llvm.fshr.i32(i32 %a0, i32 0, i32 %a1)
+  ret i32 %res
+}
+
+define i32 @fshr_i32_zero1_cst(i32 %a0) nounwind {
+; X32-SSE2-LABEL: fshr_i32_zero1_cst:
+; X32-SSE2:       # %bb.0:
+; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-SSE2-NEXT:    shll $23, %eax
+; X32-SSE2-NEXT:    retl
+;
+; X64-AVX2-LABEL: fshr_i32_zero1_cst:
+; X64-AVX2:       # %bb.0:
+; X64-AVX2-NEXT:    movl %edi, %eax
+; X64-AVX2-NEXT:    shll $23, %eax
+; X64-AVX2-NEXT:    retq
+  %res = call i32 @llvm.fshr.i32(i32 %a0, i32 0, i32 9)
+  ret i32 %res
+}
+
+; shift by zero
+
+define i32 @fshl_i32_zero2(i32 %a0, i32 %a1) nounwind {
+; X32-SSE2-LABEL: fshl_i32_zero2:
+; X32-SSE2:       # %bb.0:
+; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-SSE2-NEXT:    retl
+;
+; X64-AVX2-LABEL: fshl_i32_zero2:
+; X64-AVX2:       # %bb.0:
+; X64-AVX2-NEXT:    movl %edi, %eax
+; X64-AVX2-NEXT:    retq
+  %res = call i32 @llvm.fshl.i32(i32 %a0, i32 %a1, i32 0)
+  ret i32 %res
+}
+
+define i32 @fshr_i32_zero2(i32 %a0, i32 %a1) nounwind {
+; X32-SSE2-LABEL: fshr_i32_zero2:
+; X32-SSE2:       # %bb.0:
+; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-SSE2-NEXT:    retl
+;
+; X64-AVX2-LABEL: fshr_i32_zero2:
+; X64-AVX2:       # %bb.0:
+; X64-AVX2-NEXT:    movl %esi, %eax
+; X64-AVX2-NEXT:    retq
+  %res = call i32 @llvm.fshr.i32(i32 %a0, i32 %a1, i32 0)
+  ret i32 %res
+}
+
 ; With constant shift amount, this is 'shrd' or 'shld'.
 
 define i32 @fshr_i32_const_shift(i32 %x, i32 %y) nounwind {
diff --git a/test/CodeGen/X86/haddsub-2.ll b/test/CodeGen/X86/haddsub-2.ll
index a75fe1243a77472dc183b5597e13015370f4f49e..c4d470a6cd69ef975eb86fc35840f227317a5296 100644
--- a/test/CodeGen/X86/haddsub-2.ll
+++ b/test/CodeGen/X86/haddsub-2.ll
@@ -620,27 +620,15 @@ define <8 x i32> @avx2_vphadd_d_test(<8 x i32> %A, <8 x i32> %B) {
   ret <8 x i32> %vecinit29
 }
 
-define <16 x i16> @avx2_vphadd_w_test(<16 x i16> %a, <16 x i16> %b) {
+define <16 x i16> @avx2_vphadd_w_test(<16 x i16> %a, <16 x i16> %b) nounwind {
 ; SSE3-LABEL: avx2_vphadd_w_test:
 ; SSE3:       # %bb.0:
 ; SSE3-NEXT:    pushq %rbp
-; SSE3-NEXT:    .cfi_def_cfa_offset 16
 ; SSE3-NEXT:    pushq %r15
-; SSE3-NEXT:    .cfi_def_cfa_offset 24
 ; SSE3-NEXT:    pushq %r14
-; SSE3-NEXT:    .cfi_def_cfa_offset 32
 ; SSE3-NEXT:    pushq %r13
-; SSE3-NEXT:    .cfi_def_cfa_offset 40
 ; SSE3-NEXT:    pushq %r12
-; SSE3-NEXT:    .cfi_def_cfa_offset 48
 ; SSE3-NEXT:    pushq %rbx
-; SSE3-NEXT:    .cfi_def_cfa_offset 56
-; SSE3-NEXT:    .cfi_offset %rbx, -56
-; SSE3-NEXT:    .cfi_offset %r12, -48
-; SSE3-NEXT:    .cfi_offset %r13, -40
-; SSE3-NEXT:    .cfi_offset %r14, -32
-; SSE3-NEXT:    .cfi_offset %r15, -24
-; SSE3-NEXT:    .cfi_offset %rbp, -16
 ; SSE3-NEXT:    movd %xmm0, %eax
 ; SSE3-NEXT:    pextrw $1, %xmm0, %ecx
 ; SSE3-NEXT:    addl %eax, %ecx
@@ -724,17 +712,11 @@ define <16 x i16> @avx2_vphadd_w_test(<16 x i16> %a, <16 x i16> %b) {
 ; SSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
 ; SSE3-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm5[0]
 ; SSE3-NEXT:    popq %rbx
-; SSE3-NEXT:    .cfi_def_cfa_offset 48
 ; SSE3-NEXT:    popq %r12
-; SSE3-NEXT:    .cfi_def_cfa_offset 40
 ; SSE3-NEXT:    popq %r13
-; SSE3-NEXT:    .cfi_def_cfa_offset 32
 ; SSE3-NEXT:    popq %r14
-; SSE3-NEXT:    .cfi_def_cfa_offset 24
 ; SSE3-NEXT:    popq %r15
-; SSE3-NEXT:    .cfi_def_cfa_offset 16
 ; SSE3-NEXT:    popq %rbp
-; SSE3-NEXT:    .cfi_def_cfa_offset 8
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: avx2_vphadd_w_test:
@@ -1253,27 +1235,15 @@ define <8 x i32> @avx2_hadd_d(<8 x i32> %a, <8 x i32> %b) {
   ret <8 x i32> %vecinit29
 }
 
-define <16 x i16> @avx2_hadd_w(<16 x i16> %a, <16 x i16> %b) {
+define <16 x i16> @avx2_hadd_w(<16 x i16> %a, <16 x i16> %b) nounwind {
 ; SSE3-LABEL: avx2_hadd_w:
 ; SSE3:       # %bb.0:
 ; SSE3-NEXT:    pushq %rbp
-; SSE3-NEXT:    .cfi_def_cfa_offset 16
 ; SSE3-NEXT:    pushq %r15
-; SSE3-NEXT:    .cfi_def_cfa_offset 24
 ; SSE3-NEXT:    pushq %r14
-; SSE3-NEXT:    .cfi_def_cfa_offset 32
 ; SSE3-NEXT:    pushq %r13
-; SSE3-NEXT:    .cfi_def_cfa_offset 40
 ; SSE3-NEXT:    pushq %r12
-; SSE3-NEXT:    .cfi_def_cfa_offset 48
 ; SSE3-NEXT:    pushq %rbx
-; SSE3-NEXT:    .cfi_def_cfa_offset 56
-; SSE3-NEXT:    .cfi_offset %rbx, -56
-; SSE3-NEXT:    .cfi_offset %r12, -48
-; SSE3-NEXT:    .cfi_offset %r13, -40
-; SSE3-NEXT:    .cfi_offset %r14, -32
-; SSE3-NEXT:    .cfi_offset %r15, -24
-; SSE3-NEXT:    .cfi_offset %rbp, -16
 ; SSE3-NEXT:    movd %xmm0, %eax
 ; SSE3-NEXT:    pextrw $1, %xmm0, %r10d
 ; SSE3-NEXT:    addl %eax, %r10d
@@ -1357,17 +1327,11 @@ define <16 x i16> @avx2_hadd_w(<16 x i16> %a, <16 x i16> %b) {
 ; SSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
 ; SSE3-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm5[0]
 ; SSE3-NEXT:    popq %rbx
-; SSE3-NEXT:    .cfi_def_cfa_offset 48
 ; SSE3-NEXT:    popq %r12
-; SSE3-NEXT:    .cfi_def_cfa_offset 40
 ; SSE3-NEXT:    popq %r13
-; SSE3-NEXT:    .cfi_def_cfa_offset 32
 ; SSE3-NEXT:    popq %r14
-; SSE3-NEXT:    .cfi_def_cfa_offset 24
 ; SSE3-NEXT:    popq %r15
-; SSE3-NEXT:    .cfi_def_cfa_offset 16
 ; SSE3-NEXT:    popq %rbp
-; SSE3-NEXT:    .cfi_def_cfa_offset 8
 ; SSE3-NEXT:    retq
 ;
 ; SSSE3-LABEL: avx2_hadd_w:
diff --git a/test/CodeGen/X86/haddsub-undef.ll b/test/CodeGen/X86/haddsub-undef.ll
index 166528b08ef66b298b3b4233f869f758a10475c8..c27be13104fe61a64ff8ccb9ceaae9239d881bf3 100644
--- a/test/CodeGen/X86/haddsub-undef.ll
+++ b/test/CodeGen/X86/haddsub-undef.ll
@@ -1,10 +1,10 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+ssse3           | FileCheck %s --check-prefixes=SSE,SSE-SLOW
-; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+ssse3,fast-hops | FileCheck %s --check-prefixes=SSE,SSE-FAST
-; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx             | FileCheck %s --check-prefixes=AVX,AVX1,AVX1-SLOW
-; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx,fast-hops   | FileCheck %s --check-prefixes=AVX,AVX1,AVX1-FAST
-; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx2            | FileCheck %s --check-prefixes=AVX,AVX2,AVX2-SLOW
-; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx2,fast-hops  | FileCheck %s --check-prefixes=AVX,AVX2,AVX2-FAST
+; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+sse3               | FileCheck %s --check-prefixes=SSE,SSE-SLOW
+; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+sse3,fast-hops     | FileCheck %s --check-prefixes=SSE,SSE-FAST
+; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx                | FileCheck %s --check-prefixes=AVX,AVX-SLOW,AVX1-SLOW
+; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx,fast-hops      | FileCheck %s --check-prefixes=AVX,AVX-FAST,AVX1-FAST
+; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx512f            | FileCheck %s --check-prefixes=AVX,AVX-SLOW,AVX512,AVX512-SLOW
+; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx512f,fast-hops  | FileCheck %s --check-prefixes=AVX,AVX-FAST,AVX512,AVX512-FAST
 
 ; Verify that we correctly fold horizontal binop even in the presence of UNDEFs.
 
@@ -84,17 +84,27 @@ define <4 x float> @test3_undef(<4 x float> %a, <4 x float> %b) {
 }
 
 define <4 x float> @test4_undef(<4 x float> %a, <4 x float> %b) {
-; SSE-LABEL: test4_undef:
-; SSE:       # %bb.0:
-; SSE-NEXT:    movshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
-; SSE-NEXT:    addss %xmm1, %xmm0
-; SSE-NEXT:    retq
+; SSE-SLOW-LABEL: test4_undef:
+; SSE-SLOW:       # %bb.0:
+; SSE-SLOW-NEXT:    movshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; SSE-SLOW-NEXT:    addss %xmm1, %xmm0
+; SSE-SLOW-NEXT:    retq
 ;
-; AVX-LABEL: test4_undef:
-; AVX:       # %bb.0:
-; AVX-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
-; AVX-NEXT:    vaddss %xmm1, %xmm0, %xmm0
-; AVX-NEXT:    retq
+; SSE-FAST-LABEL: test4_undef:
+; SSE-FAST:       # %bb.0:
+; SSE-FAST-NEXT:    haddps %xmm0, %xmm0
+; SSE-FAST-NEXT:    retq
+;
+; AVX-SLOW-LABEL: test4_undef:
+; AVX-SLOW:       # %bb.0:
+; AVX-SLOW-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; AVX-SLOW-NEXT:    vaddss %xmm1, %xmm0, %xmm0
+; AVX-SLOW-NEXT:    retq
+;
+; AVX-FAST-LABEL: test4_undef:
+; AVX-FAST:       # %bb.0:
+; AVX-FAST-NEXT:    vhaddps %xmm0, %xmm0, %xmm0
+; AVX-FAST-NEXT:    retq
   %vecext = extractelement <4 x float> %a, i32 0
   %vecext1 = extractelement <4 x float> %a, i32 1
   %add = fadd float %vecext, %vecext1
@@ -103,19 +113,29 @@ define <4 x float> @test4_undef(<4 x float> %a, <4 x float> %b) {
 }
 
 define <2 x double> @test5_undef(<2 x double> %a, <2 x double> %b) {
-; SSE-LABEL: test5_undef:
-; SSE:       # %bb.0:
-; SSE-NEXT:    movapd %xmm0, %xmm1
-; SSE-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm0[1]
-; SSE-NEXT:    addsd %xmm0, %xmm1
-; SSE-NEXT:    movapd %xmm1, %xmm0
-; SSE-NEXT:    retq
+; SSE-SLOW-LABEL: test5_undef:
+; SSE-SLOW:       # %bb.0:
+; SSE-SLOW-NEXT:    movapd %xmm0, %xmm1
+; SSE-SLOW-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm0[1]
+; SSE-SLOW-NEXT:    addsd %xmm0, %xmm1
+; SSE-SLOW-NEXT:    movapd %xmm1, %xmm0
+; SSE-SLOW-NEXT:    retq
 ;
-; AVX-LABEL: test5_undef:
-; AVX:       # %bb.0:
-; AVX-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
-; AVX-NEXT:    vaddsd %xmm1, %xmm0, %xmm0
-; AVX-NEXT:    retq
+; SSE-FAST-LABEL: test5_undef:
+; SSE-FAST:       # %bb.0:
+; SSE-FAST-NEXT:    haddpd %xmm0, %xmm0
+; SSE-FAST-NEXT:    retq
+;
+; AVX-SLOW-LABEL: test5_undef:
+; AVX-SLOW:       # %bb.0:
+; AVX-SLOW-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
+; AVX-SLOW-NEXT:    vaddsd %xmm1, %xmm0, %xmm0
+; AVX-SLOW-NEXT:    retq
+;
+; AVX-FAST-LABEL: test5_undef:
+; AVX-FAST:       # %bb.0:
+; AVX-FAST-NEXT:    vhaddpd %xmm0, %xmm0, %xmm0
+; AVX-FAST-NEXT:    retq
   %vecext = extractelement <2 x double> %a, i32 0
   %vecext1 = extractelement <2 x double> %a, i32 1
   %add = fadd double %vecext, %vecext1
@@ -241,18 +261,25 @@ define <8 x float> @test10_undef(<8 x float> %a, <8 x float> %b) {
 }
 
 define <8 x float> @test11_undef(<8 x float> %a, <8 x float> %b) {
-; SSE-LABEL: test11_undef:
-; SSE:       # %bb.0:
-; SSE-NEXT:    movshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
-; SSE-NEXT:    addss %xmm1, %xmm0
-; SSE-NEXT:    movshdup {{.*#+}} xmm1 = xmm3[1,1,3,3]
-; SSE-NEXT:    addss %xmm3, %xmm1
-; SSE-NEXT:    movddup {{.*#+}} xmm1 = xmm1[0,0]
-; SSE-NEXT:    retq
+; SSE-SLOW-LABEL: test11_undef:
+; SSE-SLOW:       # %bb.0:
+; SSE-SLOW-NEXT:    movshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; SSE-SLOW-NEXT:    addss %xmm1, %xmm0
+; SSE-SLOW-NEXT:    movshdup {{.*#+}} xmm1 = xmm3[1,1,3,3]
+; SSE-SLOW-NEXT:    addss %xmm3, %xmm1
+; SSE-SLOW-NEXT:    movddup {{.*#+}} xmm1 = xmm1[0,0]
+; SSE-SLOW-NEXT:    retq
+;
+; SSE-FAST-LABEL: test11_undef:
+; SSE-FAST:       # %bb.0:
+; SSE-FAST-NEXT:    haddps %xmm0, %xmm0
+; SSE-FAST-NEXT:    haddps %xmm3, %xmm3
+; SSE-FAST-NEXT:    movddup {{.*#+}} xmm1 = xmm3[0,0]
+; SSE-FAST-NEXT:    retq
 ;
 ; AVX-LABEL: test11_undef:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vhaddps %ymm0, %ymm0, %ymm0
+; AVX-NEXT:    vhaddps %ymm1, %ymm0, %ymm0
 ; AVX-NEXT:    retq
   %vecext = extractelement <8 x float> %a, i32 0
   %vecext1 = extractelement <8 x float> %a, i32 1
@@ -316,141 +343,59 @@ define <8 x float> @test13_undef(<8 x float> %a, <8 x float> %b) {
   ret <8 x float> %vecinit4
 }
 
-define <8 x i32> @test14_undef(<8 x i32> %a, <8 x i32> %b) {
-; SSE-LABEL: test14_undef:
+define <16 x float> @test13_v16f32_undef(<16 x float> %a, <16 x float> %b) {
+; SSE-LABEL: test13_v16f32_undef:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    phaddd %xmm2, %xmm0
-; SSE-NEXT:    retq
-;
-; AVX1-LABEL: test14_undef:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vphaddd %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    retq
-;
-; AVX2-LABEL: test14_undef:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vphaddd %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    retq
-  %vecext = extractelement <8 x i32> %a, i32 0
-  %vecext1 = extractelement <8 x i32> %a, i32 1
-  %add = add i32 %vecext, %vecext1
-  %vecinit = insertelement <8 x i32> undef, i32 %add, i32 0
-  %vecext2 = extractelement <8 x i32> %b, i32 2
-  %vecext3 = extractelement <8 x i32> %b, i32 3
-  %add4 = add i32 %vecext2, %vecext3
-  %vecinit5 = insertelement <8 x i32> %vecinit, i32 %add4, i32 3
-  ret <8 x i32> %vecinit5
-}
-
-; integer horizontal adds instead of two scalar adds followed by vector inserts.
-define <8 x i32> @test15_undef(<8 x i32> %a, <8 x i32> %b) {
-; SSE-LABEL: test15_undef:
-; SSE:       # %bb.0:
-; SSE-NEXT:    movd %xmm0, %eax
-; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
-; SSE-NEXT:    movd %xmm0, %ecx
-; SSE-NEXT:    addl %eax, %ecx
-; SSE-NEXT:    movd %xmm3, %eax
-; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,2,3]
-; SSE-NEXT:    movd %xmm0, %edx
-; SSE-NEXT:    addl %eax, %edx
-; SSE-NEXT:    movd %ecx, %xmm0
-; SSE-NEXT:    movd %edx, %xmm1
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,1,0,1]
+; SSE-NEXT:    haddps %xmm1, %xmm0
 ; SSE-NEXT:    retq
 ;
-; AVX1-LABEL: test15_undef:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vmovd %xmm0, %eax
-; AVX1-NEXT:    vpextrd $1, %xmm0, %ecx
-; AVX1-NEXT:    addl %eax, %ecx
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm0
-; AVX1-NEXT:    vmovd %xmm0, %eax
-; AVX1-NEXT:    vpextrd $1, %xmm0, %edx
-; AVX1-NEXT:    addl %eax, %edx
-; AVX1-NEXT:    vmovd %ecx, %xmm0
-; AVX1-NEXT:    vmovd %edx, %xmm1
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,1,0,1]
-; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
-; AVX1-NEXT:    retq
-;
-; AVX2-LABEL: test15_undef:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vphaddd %ymm0, %ymm0, %ymm0
-; AVX2-NEXT:    retq
-  %vecext = extractelement <8 x i32> %a, i32 0
-  %vecext1 = extractelement <8 x i32> %a, i32 1
-  %add = add i32 %vecext, %vecext1
-  %vecinit = insertelement <8 x i32> undef, i32 %add, i32 0
-  %vecext2 = extractelement <8 x i32> %b, i32 4
-  %vecext3 = extractelement <8 x i32> %b, i32 5
-  %add4 = add i32 %vecext2, %vecext3
-  %vecinit5 = insertelement <8 x i32> %vecinit, i32 %add4, i32 6
-  ret <8 x i32> %vecinit5
-}
-
-define <8 x i32> @test16_undef(<8 x i32> %a, <8 x i32> %b) {
-; SSE-LABEL: test16_undef:
-; SSE:       # %bb.0:
-; SSE-NEXT:    phaddd %xmm0, %xmm0
-; SSE-NEXT:    retq
+; AVX1-SLOW-LABEL: test13_v16f32_undef:
+; AVX1-SLOW:       # %bb.0:
+; AVX1-SLOW-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX1-SLOW-NEXT:    vhaddps %xmm1, %xmm0, %xmm0
+; AVX1-SLOW-NEXT:    retq
 ;
-; AVX1-LABEL: test16_undef:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vphaddd %xmm0, %xmm0, %xmm0
-; AVX1-NEXT:    retq
-;
-; AVX2-LABEL: test16_undef:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vphaddd %ymm0, %ymm0, %ymm0
-; AVX2-NEXT:    retq
-  %vecext = extractelement <8 x i32> %a, i32 0
-  %vecext1 = extractelement <8 x i32> %a, i32 1
-  %add = add i32 %vecext, %vecext1
-  %vecinit = insertelement <8 x i32> undef, i32 %add, i32 0
-  %vecext2 = extractelement <8 x i32> %a, i32 2
-  %vecext3 = extractelement <8 x i32> %a, i32 3
-  %add4 = add i32 %vecext2, %vecext3
-  %vecinit5 = insertelement <8 x i32> %vecinit, i32 %add4, i32 1
-  ret <8 x i32> %vecinit5
-}
-
-define <8 x i32> @test17_undef(<8 x i32> %a, <8 x i32> %b) {
-; SSE-LABEL: test17_undef:
-; SSE:       # %bb.0:
-; SSE-NEXT:    phaddd %xmm1, %xmm0
-; SSE-NEXT:    retq
+; AVX1-FAST-LABEL: test13_v16f32_undef:
+; AVX1-FAST:       # %bb.0:
+; AVX1-FAST-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX1-FAST-NEXT:    vhaddps %xmm1, %xmm0, %xmm0
+; AVX1-FAST-NEXT:    retq
 ;
-; AVX1-LABEL: test17_undef:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vphaddd %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    retq
-;
-; AVX2-LABEL: test17_undef:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vphaddd %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    retq
-  %vecext = extractelement <8 x i32> %a, i32 0
-  %vecext1 = extractelement <8 x i32> %a, i32 1
-  %add1 = add i32 %vecext, %vecext1
-  %vecinit1 = insertelement <8 x i32> undef, i32 %add1, i32 0
-  %vecext2 = extractelement <8 x i32> %a, i32 2
-  %vecext3 = extractelement <8 x i32> %a, i32 3
-  %add2 = add i32 %vecext2, %vecext3
-  %vecinit2 = insertelement <8 x i32> %vecinit1, i32 %add2, i32 1
-  %vecext4 = extractelement <8 x i32> %a, i32 4
-  %vecext5 = extractelement <8 x i32> %a, i32 5
-  %add3 = add i32 %vecext4, %vecext5
-  %vecinit3 = insertelement <8 x i32> %vecinit2, i32 %add3, i32 2
-  %vecext6 = extractelement <8 x i32> %a, i32 6
-  %vecext7 = extractelement <8 x i32> %a, i32 7
-  %add4 = add i32 %vecext6, %vecext7
-  %vecinit4 = insertelement <8 x i32> %vecinit3, i32 %add4, i32 3
-  ret <8 x i32> %vecinit4
+; AVX512-LABEL: test13_v16f32_undef:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; AVX512-NEXT:    vaddss %xmm1, %xmm0, %xmm1
+; AVX512-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm0[1,0]
+; AVX512-NEXT:    vpermilps {{.*#+}} xmm3 = xmm0[3,1,2,3]
+; AVX512-NEXT:    vaddss %xmm3, %xmm2, %xmm2
+; AVX512-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[2,3]
+; AVX512-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; AVX512-NEXT:    vmovshdup {{.*#+}} xmm2 = xmm0[1,1,3,3]
+; AVX512-NEXT:    vaddss %xmm2, %xmm0, %xmm2
+; AVX512-NEXT:    vinsertps {{.*#+}} xmm1 = xmm1[0,1],xmm2[0],xmm1[3]
+; AVX512-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm0[1,0]
+; AVX512-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; AVX512-NEXT:    vaddss %xmm0, %xmm2, %xmm0
+; AVX512-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
+; AVX512-NEXT:    retq
+  %vecext = extractelement <16 x float> %a, i32 0
+  %vecext1 = extractelement <16 x float> %a, i32 1
+  %add1 = fadd float %vecext, %vecext1
+  %vecinit1 = insertelement <16 x float> undef, float %add1, i32 0
+  %vecext2 = extractelement <16 x float> %a, i32 2
+  %vecext3 = extractelement <16 x float> %a, i32 3
+  %add2 = fadd float %vecext2, %vecext3
+  %vecinit2 = insertelement <16 x float> %vecinit1, float %add2, i32 1
+  %vecext4 = extractelement <16 x float> %a, i32 4
+  %vecext5 = extractelement <16 x float> %a, i32 5
+  %add3 = fadd float %vecext4, %vecext5
+  %vecinit3 = insertelement <16 x float> %vecinit2, float %add3, i32 2
+  %vecext6 = extractelement <16 x float> %a, i32 6
+  %vecext7 = extractelement <16 x float> %a, i32 7
+  %add4 = fadd float %vecext6, %vecext7
+  %vecinit4 = insertelement <16 x float> %vecinit3, float %add4, i32 3
+  ret <16 x float> %vecinit4
 }
-
 define <2 x double> @add_pd_003(<2 x double> %x) {
 ; SSE-SLOW-LABEL: add_pd_003:
 ; SSE-SLOW:       # %bb.0:
@@ -463,27 +408,16 @@ define <2 x double> @add_pd_003(<2 x double> %x) {
 ; SSE-FAST-NEXT:    haddpd %xmm0, %xmm0
 ; SSE-FAST-NEXT:    retq
 ;
-; AVX1-SLOW-LABEL: add_pd_003:
-; AVX1-SLOW:       # %bb.0:
-; AVX1-SLOW-NEXT:    vmovddup {{.*#+}} xmm1 = xmm0[0,0]
-; AVX1-SLOW-NEXT:    vaddpd %xmm0, %xmm1, %xmm0
-; AVX1-SLOW-NEXT:    retq
+; AVX-SLOW-LABEL: add_pd_003:
+; AVX-SLOW:       # %bb.0:
+; AVX-SLOW-NEXT:    vmovddup {{.*#+}} xmm1 = xmm0[0,0]
+; AVX-SLOW-NEXT:    vaddpd %xmm0, %xmm1, %xmm0
+; AVX-SLOW-NEXT:    retq
 ;
-; AVX1-FAST-LABEL: add_pd_003:
-; AVX1-FAST:       # %bb.0:
-; AVX1-FAST-NEXT:    vhaddpd %xmm0, %xmm0, %xmm0
-; AVX1-FAST-NEXT:    retq
-;
-; AVX2-SLOW-LABEL: add_pd_003:
-; AVX2-SLOW:       # %bb.0:
-; AVX2-SLOW-NEXT:    vmovddup {{.*#+}} xmm1 = xmm0[0,0]
-; AVX2-SLOW-NEXT:    vaddpd %xmm0, %xmm1, %xmm0
-; AVX2-SLOW-NEXT:    retq
-;
-; AVX2-FAST-LABEL: add_pd_003:
-; AVX2-FAST:       # %bb.0:
-; AVX2-FAST-NEXT:    vhaddpd %xmm0, %xmm0, %xmm0
-; AVX2-FAST-NEXT:    retq
+; AVX-FAST-LABEL: add_pd_003:
+; AVX-FAST:       # %bb.0:
+; AVX-FAST-NEXT:    vhaddpd %xmm0, %xmm0, %xmm0
+; AVX-FAST-NEXT:    retq
   %l = shufflevector <2 x double> %x, <2 x double> undef, <2 x i32> <i32 undef, i32 0>
   %add = fadd <2 x double> %l, %x
   ret <2 x double> %add
@@ -505,27 +439,16 @@ define <2 x double> @add_pd_003_2(<2 x double> %x) {
 ; SSE-FAST-NEXT:    haddpd %xmm0, %xmm0
 ; SSE-FAST-NEXT:    retq
 ;
-; AVX1-SLOW-LABEL: add_pd_003_2:
-; AVX1-SLOW:       # %bb.0:
-; AVX1-SLOW-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
-; AVX1-SLOW-NEXT:    vaddpd %xmm0, %xmm1, %xmm0
-; AVX1-SLOW-NEXT:    retq
+; AVX-SLOW-LABEL: add_pd_003_2:
+; AVX-SLOW:       # %bb.0:
+; AVX-SLOW-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
+; AVX-SLOW-NEXT:    vaddpd %xmm0, %xmm1, %xmm0
+; AVX-SLOW-NEXT:    retq
 ;
-; AVX1-FAST-LABEL: add_pd_003_2:
-; AVX1-FAST:       # %bb.0:
-; AVX1-FAST-NEXT:    vhaddpd %xmm0, %xmm0, %xmm0
-; AVX1-FAST-NEXT:    retq
-;
-; AVX2-SLOW-LABEL: add_pd_003_2:
-; AVX2-SLOW:       # %bb.0:
-; AVX2-SLOW-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
-; AVX2-SLOW-NEXT:    vaddpd %xmm0, %xmm1, %xmm0
-; AVX2-SLOW-NEXT:    retq
-;
-; AVX2-FAST-LABEL: add_pd_003_2:
-; AVX2-FAST:       # %bb.0:
-; AVX2-FAST-NEXT:    vhaddpd %xmm0, %xmm0, %xmm0
-; AVX2-FAST-NEXT:    retq
+; AVX-FAST-LABEL: add_pd_003_2:
+; AVX-FAST:       # %bb.0:
+; AVX-FAST-NEXT:    vhaddpd %xmm0, %xmm0, %xmm0
+; AVX-FAST-NEXT:    retq
   %l = shufflevector <2 x double> %x, <2 x double> undef, <2 x i32> <i32 1, i32 0>
   %add = fadd <2 x double> %l, %x
   ret <2 x double> %add
@@ -545,31 +468,18 @@ define <2 x double> @add_pd_010(<2 x double> %x) {
 ; SSE-FAST-NEXT:    haddpd %xmm0, %xmm0
 ; SSE-FAST-NEXT:    retq
 ;
-; AVX1-SLOW-LABEL: add_pd_010:
-; AVX1-SLOW:       # %bb.0:
-; AVX1-SLOW-NEXT:    vmovddup {{.*#+}} xmm1 = xmm0[0,0]
-; AVX1-SLOW-NEXT:    vaddpd %xmm0, %xmm1, %xmm0
-; AVX1-SLOW-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
-; AVX1-SLOW-NEXT:    retq
-;
-; AVX1-FAST-LABEL: add_pd_010:
-; AVX1-FAST:       # %bb.0:
-; AVX1-FAST-NEXT:    vhaddpd %xmm0, %xmm0, %xmm0
-; AVX1-FAST-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
-; AVX1-FAST-NEXT:    retq
-;
-; AVX2-SLOW-LABEL: add_pd_010:
-; AVX2-SLOW:       # %bb.0:
-; AVX2-SLOW-NEXT:    vmovddup {{.*#+}} xmm1 = xmm0[0,0]
-; AVX2-SLOW-NEXT:    vaddpd %xmm0, %xmm1, %xmm0
-; AVX2-SLOW-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
-; AVX2-SLOW-NEXT:    retq
-;
-; AVX2-FAST-LABEL: add_pd_010:
-; AVX2-FAST:       # %bb.0:
-; AVX2-FAST-NEXT:    vhaddpd %xmm0, %xmm0, %xmm0
-; AVX2-FAST-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
-; AVX2-FAST-NEXT:    retq
+; AVX-SLOW-LABEL: add_pd_010:
+; AVX-SLOW:       # %bb.0:
+; AVX-SLOW-NEXT:    vmovddup {{.*#+}} xmm1 = xmm0[0,0]
+; AVX-SLOW-NEXT:    vaddpd %xmm0, %xmm1, %xmm0
+; AVX-SLOW-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
+; AVX-SLOW-NEXT:    retq
+;
+; AVX-FAST-LABEL: add_pd_010:
+; AVX-FAST:       # %bb.0:
+; AVX-FAST-NEXT:    vhaddpd %xmm0, %xmm0, %xmm0
+; AVX-FAST-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
+; AVX-FAST-NEXT:    retq
   %l = shufflevector <2 x double> %x, <2 x double> undef, <2 x i32> <i32 undef, i32 0>
   %add = fadd <2 x double> %l, %x
   %shuffle2 = shufflevector <2 x double> %add, <2 x double> undef, <2 x i32> <i32 1, i32 undef>
@@ -590,29 +500,17 @@ define <4 x float> @add_ps_007(<4 x float> %x) {
 ; SSE-FAST-NEXT:    haddps %xmm0, %xmm0
 ; SSE-FAST-NEXT:    retq
 ;
-; AVX1-SLOW-LABEL: add_ps_007:
-; AVX1-SLOW:       # %bb.0:
-; AVX1-SLOW-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[0,1,0,2]
-; AVX1-SLOW-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,1,1,3]
-; AVX1-SLOW-NEXT:    vaddps %xmm0, %xmm1, %xmm0
-; AVX1-SLOW-NEXT:    retq
-;
-; AVX1-FAST-LABEL: add_ps_007:
-; AVX1-FAST:       # %bb.0:
-; AVX1-FAST-NEXT:    vhaddps %xmm0, %xmm0, %xmm0
-; AVX1-FAST-NEXT:    retq
-;
-; AVX2-SLOW-LABEL: add_ps_007:
-; AVX2-SLOW:       # %bb.0:
-; AVX2-SLOW-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[0,1,0,2]
-; AVX2-SLOW-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,1,1,3]
-; AVX2-SLOW-NEXT:    vaddps %xmm0, %xmm1, %xmm0
-; AVX2-SLOW-NEXT:    retq
-;
-; AVX2-FAST-LABEL: add_ps_007:
-; AVX2-FAST:       # %bb.0:
-; AVX2-FAST-NEXT:    vhaddps %xmm0, %xmm0, %xmm0
-; AVX2-FAST-NEXT:    retq
+; AVX-SLOW-LABEL: add_ps_007:
+; AVX-SLOW:       # %bb.0:
+; AVX-SLOW-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[0,1,0,2]
+; AVX-SLOW-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,1,1,3]
+; AVX-SLOW-NEXT:    vaddps %xmm0, %xmm1, %xmm0
+; AVX-SLOW-NEXT:    retq
+;
+; AVX-FAST-LABEL: add_ps_007:
+; AVX-FAST:       # %bb.0:
+; AVX-FAST-NEXT:    vhaddps %xmm0, %xmm0, %xmm0
+; AVX-FAST-NEXT:    retq
   %l = shufflevector <4 x float> %x, <4 x float> undef, <4 x i32> <i32 undef, i32 undef, i32 0, i32 2>
   %r = shufflevector <4 x float> %x, <4 x float> undef, <4 x i32> <i32 undef, i32 undef, i32 1, i32 3>
   %add = fadd <4 x float> %l, %r
@@ -635,33 +533,19 @@ define <4 x float> @add_ps_030(<4 x float> %x) {
 ; SSE-FAST-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,2,2,3]
 ; SSE-FAST-NEXT:    retq
 ;
-; AVX1-SLOW-LABEL: add_ps_030:
-; AVX1-SLOW:       # %bb.0:
-; AVX1-SLOW-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[0,1,0,2]
-; AVX1-SLOW-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,1,1,3]
-; AVX1-SLOW-NEXT:    vaddps %xmm0, %xmm1, %xmm0
-; AVX1-SLOW-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,2,2,3]
-; AVX1-SLOW-NEXT:    retq
-;
-; AVX1-FAST-LABEL: add_ps_030:
-; AVX1-FAST:       # %bb.0:
-; AVX1-FAST-NEXT:    vhaddps %xmm0, %xmm0, %xmm0
-; AVX1-FAST-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,2,2,3]
-; AVX1-FAST-NEXT:    retq
-;
-; AVX2-SLOW-LABEL: add_ps_030:
-; AVX2-SLOW:       # %bb.0:
-; AVX2-SLOW-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[0,1,0,2]
-; AVX2-SLOW-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,1,1,3]
-; AVX2-SLOW-NEXT:    vaddps %xmm0, %xmm1, %xmm0
-; AVX2-SLOW-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,2,2,3]
-; AVX2-SLOW-NEXT:    retq
-;
-; AVX2-FAST-LABEL: add_ps_030:
-; AVX2-FAST:       # %bb.0:
-; AVX2-FAST-NEXT:    vhaddps %xmm0, %xmm0, %xmm0
-; AVX2-FAST-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,2,2,3]
-; AVX2-FAST-NEXT:    retq
+; AVX-SLOW-LABEL: add_ps_030:
+; AVX-SLOW:       # %bb.0:
+; AVX-SLOW-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[0,1,0,2]
+; AVX-SLOW-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,1,1,3]
+; AVX-SLOW-NEXT:    vaddps %xmm0, %xmm1, %xmm0
+; AVX-SLOW-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,2,2,3]
+; AVX-SLOW-NEXT:    retq
+;
+; AVX-FAST-LABEL: add_ps_030:
+; AVX-FAST:       # %bb.0:
+; AVX-FAST-NEXT:    vhaddps %xmm0, %xmm0, %xmm0
+; AVX-FAST-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,2,2,3]
+; AVX-FAST-NEXT:    retq
   %l = shufflevector <4 x float> %x, <4 x float> undef, <4 x i32> <i32 undef, i32 undef, i32 0, i32 2>
   %r = shufflevector <4 x float> %x, <4 x float> undef, <4 x i32> <i32 undef, i32 undef, i32 1, i32 3>
   %add = fadd <4 x float> %l, %r
@@ -689,22 +573,17 @@ define <4 x float> @add_ps_007_2(<4 x float> %x) {
 ; AVX1-SLOW-NEXT:    vaddps %xmm0, %xmm1, %xmm0
 ; AVX1-SLOW-NEXT:    retq
 ;
-; AVX1-FAST-LABEL: add_ps_007_2:
-; AVX1-FAST:       # %bb.0:
-; AVX1-FAST-NEXT:    vhaddps %xmm0, %xmm0, %xmm0
-; AVX1-FAST-NEXT:    retq
-;
-; AVX2-SLOW-LABEL: add_ps_007_2:
-; AVX2-SLOW:       # %bb.0:
-; AVX2-SLOW-NEXT:    vbroadcastss %xmm0, %xmm1
-; AVX2-SLOW-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,1,1,3]
-; AVX2-SLOW-NEXT:    vaddps %xmm0, %xmm1, %xmm0
-; AVX2-SLOW-NEXT:    retq
-;
-; AVX2-FAST-LABEL: add_ps_007_2:
-; AVX2-FAST:       # %bb.0:
-; AVX2-FAST-NEXT:    vhaddps %xmm0, %xmm0, %xmm0
-; AVX2-FAST-NEXT:    retq
+; AVX-FAST-LABEL: add_ps_007_2:
+; AVX-FAST:       # %bb.0:
+; AVX-FAST-NEXT:    vhaddps %xmm0, %xmm0, %xmm0
+; AVX-FAST-NEXT:    retq
+;
+; AVX512-SLOW-LABEL: add_ps_007_2:
+; AVX512-SLOW:       # %bb.0:
+; AVX512-SLOW-NEXT:    vbroadcastss %xmm0, %xmm1
+; AVX512-SLOW-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,1,1,3]
+; AVX512-SLOW-NEXT:    vaddps %xmm0, %xmm1, %xmm0
+; AVX512-SLOW-NEXT:    retq
   %l = shufflevector <4 x float> %x, <4 x float> undef, <4 x i32> <i32 undef, i32 undef, i32 0, i32 undef>
   %r = shufflevector <4 x float> %x, <4 x float> undef, <4 x i32> <i32 undef, i32 undef, i32 1, i32 undef>
   %add = fadd <4 x float> %l, %r
@@ -723,27 +602,16 @@ define <4 x float> @add_ps_008(<4 x float> %x) {
 ; SSE-FAST-NEXT:    haddps %xmm0, %xmm0
 ; SSE-FAST-NEXT:    retq
 ;
-; AVX1-SLOW-LABEL: add_ps_008:
-; AVX1-SLOW:       # %bb.0:
-; AVX1-SLOW-NEXT:    vmovsldup {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; AVX1-SLOW-NEXT:    vaddps %xmm0, %xmm1, %xmm0
-; AVX1-SLOW-NEXT:    retq
+; AVX-SLOW-LABEL: add_ps_008:
+; AVX-SLOW:       # %bb.0:
+; AVX-SLOW-NEXT:    vmovsldup {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; AVX-SLOW-NEXT:    vaddps %xmm0, %xmm1, %xmm0
+; AVX-SLOW-NEXT:    retq
 ;
-; AVX1-FAST-LABEL: add_ps_008:
-; AVX1-FAST:       # %bb.0:
-; AVX1-FAST-NEXT:    vhaddps %xmm0, %xmm0, %xmm0
-; AVX1-FAST-NEXT:    retq
-;
-; AVX2-SLOW-LABEL: add_ps_008:
-; AVX2-SLOW:       # %bb.0:
-; AVX2-SLOW-NEXT:    vmovsldup {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; AVX2-SLOW-NEXT:    vaddps %xmm0, %xmm1, %xmm0
-; AVX2-SLOW-NEXT:    retq
-;
-; AVX2-FAST-LABEL: add_ps_008:
-; AVX2-FAST:       # %bb.0:
-; AVX2-FAST-NEXT:    vhaddps %xmm0, %xmm0, %xmm0
-; AVX2-FAST-NEXT:    retq
+; AVX-FAST-LABEL: add_ps_008:
+; AVX-FAST:       # %bb.0:
+; AVX-FAST-NEXT:    vhaddps %xmm0, %xmm0, %xmm0
+; AVX-FAST-NEXT:    retq
   %l = shufflevector <4 x float> %x, <4 x float> undef, <4 x i32> <i32 undef, i32 undef, i32 undef, i32 2>
   %add = fadd <4 x float> %l, %x
   ret <4 x float> %add
@@ -764,31 +632,18 @@ define <4 x float> @add_ps_017(<4 x float> %x) {
 ; SSE-FAST-NEXT:    shufps {{.*#+}} xmm0 = xmm0[3,1,2,3]
 ; SSE-FAST-NEXT:    retq
 ;
-; AVX1-SLOW-LABEL: add_ps_017:
-; AVX1-SLOW:       # %bb.0:
-; AVX1-SLOW-NEXT:    vmovsldup {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; AVX1-SLOW-NEXT:    vaddps %xmm0, %xmm1, %xmm0
-; AVX1-SLOW-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,1,2,3]
-; AVX1-SLOW-NEXT:    retq
-;
-; AVX1-FAST-LABEL: add_ps_017:
-; AVX1-FAST:       # %bb.0:
-; AVX1-FAST-NEXT:    vhaddps %xmm0, %xmm0, %xmm0
-; AVX1-FAST-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,1,2,3]
-; AVX1-FAST-NEXT:    retq
-;
-; AVX2-SLOW-LABEL: add_ps_017:
-; AVX2-SLOW:       # %bb.0:
-; AVX2-SLOW-NEXT:    vmovsldup {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; AVX2-SLOW-NEXT:    vaddps %xmm0, %xmm1, %xmm0
-; AVX2-SLOW-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,1,2,3]
-; AVX2-SLOW-NEXT:    retq
-;
-; AVX2-FAST-LABEL: add_ps_017:
-; AVX2-FAST:       # %bb.0:
-; AVX2-FAST-NEXT:    vhaddps %xmm0, %xmm0, %xmm0
-; AVX2-FAST-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,1,2,3]
-; AVX2-FAST-NEXT:    retq
+; AVX-SLOW-LABEL: add_ps_017:
+; AVX-SLOW:       # %bb.0:
+; AVX-SLOW-NEXT:    vmovsldup {{.*#+}} xmm1 = xmm0[0,0,2,2]
+; AVX-SLOW-NEXT:    vaddps %xmm0, %xmm1, %xmm0
+; AVX-SLOW-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; AVX-SLOW-NEXT:    retq
+;
+; AVX-FAST-LABEL: add_ps_017:
+; AVX-FAST:       # %bb.0:
+; AVX-FAST-NEXT:    vhaddps %xmm0, %xmm0, %xmm0
+; AVX-FAST-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; AVX-FAST-NEXT:    retq
   %l = shufflevector <4 x float> %x, <4 x float> undef, <4 x i32> <i32 undef, i32 undef, i32 undef, i32 2>
   %add = fadd <4 x float> %l, %x
   %shuffle2 = shufflevector <4 x float> %add, <4 x float> undef, <4 x i32> <i32 3, i32 undef, i32 undef, i32 undef>
@@ -818,25 +673,19 @@ define <4 x float> @add_ps_018(<4 x float> %x) {
 ; AVX1-SLOW-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,2,2,3]
 ; AVX1-SLOW-NEXT:    retq
 ;
-; AVX1-FAST-LABEL: add_ps_018:
-; AVX1-FAST:       # %bb.0:
-; AVX1-FAST-NEXT:    vhaddps %xmm0, %xmm0, %xmm0
-; AVX1-FAST-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,2,2,3]
-; AVX1-FAST-NEXT:    retq
-;
-; AVX2-SLOW-LABEL: add_ps_018:
-; AVX2-SLOW:       # %bb.0:
-; AVX2-SLOW-NEXT:    vbroadcastss %xmm0, %xmm1
-; AVX2-SLOW-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,1,1,3]
-; AVX2-SLOW-NEXT:    vaddps %xmm0, %xmm1, %xmm0
-; AVX2-SLOW-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,2,2,3]
-; AVX2-SLOW-NEXT:    retq
-;
-; AVX2-FAST-LABEL: add_ps_018:
-; AVX2-FAST:       # %bb.0:
-; AVX2-FAST-NEXT:    vhaddps %xmm0, %xmm0, %xmm0
-; AVX2-FAST-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,2,2,3]
-; AVX2-FAST-NEXT:    retq
+; AVX-FAST-LABEL: add_ps_018:
+; AVX-FAST:       # %bb.0:
+; AVX-FAST-NEXT:    vhaddps %xmm0, %xmm0, %xmm0
+; AVX-FAST-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; AVX-FAST-NEXT:    retq
+;
+; AVX512-SLOW-LABEL: add_ps_018:
+; AVX512-SLOW:       # %bb.0:
+; AVX512-SLOW-NEXT:    vbroadcastss %xmm0, %xmm1
+; AVX512-SLOW-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,1,1,3]
+; AVX512-SLOW-NEXT:    vaddps %xmm0, %xmm1, %xmm0
+; AVX512-SLOW-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; AVX512-SLOW-NEXT:    retq
   %l = shufflevector <4 x float> %x, <4 x float> undef, <4 x i32> <i32 undef, i32 undef, i32 0, i32 undef>
   %r = shufflevector <4 x float> %x, <4 x float> undef, <4 x i32> <i32 undef, i32 undef, i32 1, i32 undef>
   %add = fadd <4 x float> %l, %r
@@ -844,3 +693,177 @@ define <4 x float> @add_ps_018(<4 x float> %x) {
   ret <4 x float> %shuffle2
 }
 
+define <4 x float> @v8f32_inputs_v4f32_output_0101(<8 x float> %a, <8 x float> %b) {
+; SSE-LABEL: v8f32_inputs_v4f32_output_0101:
+; SSE:       # %bb.0:
+; SSE-NEXT:    haddps %xmm2, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: v8f32_inputs_v4f32_output_0101:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vhaddps %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vzeroupper
+; AVX-NEXT:    retq
+  %a0 = extractelement <8 x float> %a, i32 0
+  %a1 = extractelement <8 x float> %a, i32 1
+  %b0 = extractelement <8 x float> %b, i32 0
+  %b1 = extractelement <8 x float> %b, i32 1
+  %add0 = fadd float %a0, %a1
+  %add2 = fadd float %b0, %b1
+  %r0 = insertelement <4 x float> undef, float %add0, i32 0
+  %r = insertelement <4 x float> %r0, float %add2, i32 2
+  ret <4 x float> %r
+}
+
+define <4 x float> @v8f32_input0_v4f32_output_0123(<8 x float> %a, <4 x float> %b) {
+; SSE-LABEL: v8f32_input0_v4f32_output_0123:
+; SSE:       # %bb.0:
+; SSE-NEXT:    haddps %xmm2, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: v8f32_input0_v4f32_output_0123:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vhaddps %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vzeroupper
+; AVX-NEXT:    retq
+  %a0 = extractelement <8 x float> %a, i32 0
+  %a1 = extractelement <8 x float> %a, i32 1
+  %b2 = extractelement <4 x float> %b, i32 2
+  %b3 = extractelement <4 x float> %b, i32 3
+  %add0 = fadd float %a0, %a1
+  %add3 = fadd float %b2, %b3
+  %r0 = insertelement <4 x float> undef, float %add0, i32 0
+  %r = insertelement <4 x float> %r0, float %add3, i32 3
+  ret <4 x float> %r
+}
+
+define <4 x float> @v8f32_input1_v4f32_output_2301(<4 x float> %a, <8 x float> %b) {
+; SSE-LABEL: v8f32_input1_v4f32_output_2301:
+; SSE:       # %bb.0:
+; SSE-NEXT:    haddps %xmm1, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: v8f32_input1_v4f32_output_2301:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vhaddps %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vzeroupper
+; AVX-NEXT:    retq
+  %a2 = extractelement <4 x float> %a, i32 2
+  %a3 = extractelement <4 x float> %a, i32 3
+  %b0 = extractelement <8 x float> %b, i32 0
+  %b1 = extractelement <8 x float> %b, i32 1
+  %add1 = fadd float %a2, %a3
+  %add2 = fadd float %b0, %b1
+  %r1 = insertelement <4 x float> undef, float %add1, i32 1
+  %r = insertelement <4 x float> %r1, float %add2, i32 2
+  ret <4 x float> %r
+}
+
+define <4 x float> @v8f32_inputs_v4f32_output_2323(<8 x float> %a, <8 x float> %b) {
+; SSE-LABEL: v8f32_inputs_v4f32_output_2323:
+; SSE:       # %bb.0:
+; SSE-NEXT:    haddps %xmm2, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: v8f32_inputs_v4f32_output_2323:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vhaddps %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vzeroupper
+; AVX-NEXT:    retq
+  %a2 = extractelement <8 x float> %a, i32 2
+  %a3 = extractelement <8 x float> %a, i32 3
+  %b2 = extractelement <8 x float> %b, i32 2
+  %b3 = extractelement <8 x float> %b, i32 3
+  %add1 = fadd float %a2, %a3
+  %add3 = fadd float %b2, %b3
+  %r1 = insertelement <4 x float> undef, float %add1, i32 1
+  %r = insertelement <4 x float> %r1, float %add3, i32 3
+  ret <4 x float> %r
+}
+
+define <4 x float> @v16f32_inputs_v4f32_output_0123(<16 x float> %a, <16 x float> %b) {
+; SSE-LABEL: v16f32_inputs_v4f32_output_0123:
+; SSE:       # %bb.0:
+; SSE-NEXT:    haddps %xmm4, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX1-SLOW-LABEL: v16f32_inputs_v4f32_output_0123:
+; AVX1-SLOW:       # %bb.0:
+; AVX1-SLOW-NEXT:    vhaddps %xmm2, %xmm0, %xmm0
+; AVX1-SLOW-NEXT:    vzeroupper
+; AVX1-SLOW-NEXT:    retq
+;
+; AVX1-FAST-LABEL: v16f32_inputs_v4f32_output_0123:
+; AVX1-FAST:       # %bb.0:
+; AVX1-FAST-NEXT:    vhaddps %xmm2, %xmm0, %xmm0
+; AVX1-FAST-NEXT:    vzeroupper
+; AVX1-FAST-NEXT:    retq
+;
+; AVX512-LABEL: v16f32_inputs_v4f32_output_0123:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vhaddps %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vzeroupper
+; AVX512-NEXT:    retq
+  %a0 = extractelement <16 x float> %a, i32 0
+  %a1 = extractelement <16 x float> %a, i32 1
+  %b2 = extractelement <16 x float> %b, i32 2
+  %b3 = extractelement <16 x float> %b, i32 3
+  %add0 = fadd float %a0, %a1
+  %add3 = fadd float %b2, %b3
+  %r0 = insertelement <4 x float> undef, float %add0, i32 0
+  %r = insertelement <4 x float> %r0, float %add3, i32 3
+  ret <4 x float> %r
+}
+
+define <8 x float> @v16f32_inputs_v8f32_output_4567(<16 x float> %a, <16 x float> %b) {
+; SSE-LABEL: v16f32_inputs_v8f32_output_4567:
+; SSE:       # %bb.0:
+; SSE-NEXT:    haddps %xmm5, %xmm1
+; SSE-NEXT:    retq
+;
+; AVX1-SLOW-LABEL: v16f32_inputs_v8f32_output_4567:
+; AVX1-SLOW:       # %bb.0:
+; AVX1-SLOW-NEXT:    vhaddps %ymm2, %ymm0, %ymm0
+; AVX1-SLOW-NEXT:    retq
+;
+; AVX1-FAST-LABEL: v16f32_inputs_v8f32_output_4567:
+; AVX1-FAST:       # %bb.0:
+; AVX1-FAST-NEXT:    vhaddps %ymm2, %ymm0, %ymm0
+; AVX1-FAST-NEXT:    retq
+;
+; AVX512-LABEL: v16f32_inputs_v8f32_output_4567:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vhaddps %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    retq
+  %a4 = extractelement <16 x float> %a, i32 4
+  %a5 = extractelement <16 x float> %a, i32 5
+  %b6 = extractelement <16 x float> %b, i32 6
+  %b7 = extractelement <16 x float> %b, i32 7
+  %add4 = fadd float %a4, %a5
+  %add7 = fadd float %b6, %b7
+  %r4 = insertelement <8 x float> undef, float %add4, i32 4
+  %r = insertelement <8 x float> %r4, float %add7, i32 7
+  ret <8 x float> %r
+}
+
+define <8 x float> @PR40243(<8 x float> %a, <8 x float> %b) {
+; SSE-LABEL: PR40243:
+; SSE:       # %bb.0:
+; SSE-NEXT:    haddps %xmm3, %xmm1
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: PR40243:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vhaddps %ymm1, %ymm0, %ymm0
+; AVX-NEXT:    retq
+  %a4 = extractelement <8 x float> %a, i32 4
+  %a5 = extractelement <8 x float> %a, i32 5
+  %add4 = fadd float %a4, %a5
+  %b6 = extractelement <8 x float> %b, i32 6
+  %b7 = extractelement <8 x float> %b, i32 7
+  %add7 = fadd float %b6, %b7
+  %r4 = insertelement <8 x float> undef, float %add4, i32 4
+  %r = insertelement <8 x float> %r4, float %add7, i32 7
+  ret <8 x float> %r
+}
+
diff --git a/test/CodeGen/X86/haddsub.ll b/test/CodeGen/X86/haddsub.ll
index 6221d4e43bcfe469d89e3145b9b1883afa104b26..305da125a4d8f15049b84aa8633a5e309ce3a55d 100644
--- a/test/CodeGen/X86/haddsub.ll
+++ b/test/CodeGen/X86/haddsub.ll
@@ -1,8 +1,10 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+sse3           | FileCheck %s --check-prefixes=SSE3,SSE3-SLOW
-; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+sse3,fast-hops | FileCheck %s --check-prefixes=SSE3,SSE3-FAST
-; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx            | FileCheck %s --check-prefixes=AVX,AVX-SLOW
-; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx,fast-hops  | FileCheck %s --check-prefixes=AVX,AVX-FAST
+; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+sse3               | FileCheck %s --check-prefixes=SSE3,SSE3-SLOW
+; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+sse3,fast-hops     | FileCheck %s --check-prefixes=SSE3,SSE3-FAST
+; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx                | FileCheck %s --check-prefixes=AVX,AVX-SLOW,AVX1-SLOW
+; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx,fast-hops      | FileCheck %s --check-prefixes=AVX,AVX-FAST,AVX1-FAST
+; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx512f            | FileCheck %s --check-prefixes=AVX,AVX-SLOW,AVX512-SLOW
+; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx512f,fast-hops  | FileCheck %s --check-prefixes=AVX,AVX-FAST,AVX512-FAST
 
 define <2 x double> @haddpd1(<2 x double> %x, <2 x double> %y) {
 ; SSE3-LABEL: haddpd1:
@@ -583,3 +585,856 @@ define <2 x float> @haddps_v2f32(<4 x float> %v0) {
   ret <2 x float> %res1
 }
 
+; 128-bit vectors, float/double, fadd/fsub
+
+define float @extract_extract_v4f32_fadd_f32(<4 x float> %x) {
+; SSE3-SLOW-LABEL: extract_extract_v4f32_fadd_f32:
+; SSE3-SLOW:       # %bb.0:
+; SSE3-SLOW-NEXT:    movshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; SSE3-SLOW-NEXT:    addss %xmm1, %xmm0
+; SSE3-SLOW-NEXT:    retq
+;
+; SSE3-FAST-LABEL: extract_extract_v4f32_fadd_f32:
+; SSE3-FAST:       # %bb.0:
+; SSE3-FAST-NEXT:    haddps %xmm0, %xmm0
+; SSE3-FAST-NEXT:    retq
+;
+; AVX-SLOW-LABEL: extract_extract_v4f32_fadd_f32:
+; AVX-SLOW:       # %bb.0:
+; AVX-SLOW-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; AVX-SLOW-NEXT:    vaddss %xmm1, %xmm0, %xmm0
+; AVX-SLOW-NEXT:    retq
+;
+; AVX-FAST-LABEL: extract_extract_v4f32_fadd_f32:
+; AVX-FAST:       # %bb.0:
+; AVX-FAST-NEXT:    vhaddps %xmm0, %xmm0, %xmm0
+; AVX-FAST-NEXT:    retq
+  %x0 = extractelement <4 x float> %x, i32 0
+  %x1 = extractelement <4 x float> %x, i32 1
+  %x01 = fadd float %x0, %x1
+  ret float %x01
+}
+
+define float @extract_extract_v4f32_fadd_f32_commute(<4 x float> %x) {
+; SSE3-SLOW-LABEL: extract_extract_v4f32_fadd_f32_commute:
+; SSE3-SLOW:       # %bb.0:
+; SSE3-SLOW-NEXT:    movshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; SSE3-SLOW-NEXT:    addss %xmm1, %xmm0
+; SSE3-SLOW-NEXT:    retq
+;
+; SSE3-FAST-LABEL: extract_extract_v4f32_fadd_f32_commute:
+; SSE3-FAST:       # %bb.0:
+; SSE3-FAST-NEXT:    haddps %xmm0, %xmm0
+; SSE3-FAST-NEXT:    retq
+;
+; AVX-SLOW-LABEL: extract_extract_v4f32_fadd_f32_commute:
+; AVX-SLOW:       # %bb.0:
+; AVX-SLOW-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; AVX-SLOW-NEXT:    vaddss %xmm0, %xmm1, %xmm0
+; AVX-SLOW-NEXT:    retq
+;
+; AVX-FAST-LABEL: extract_extract_v4f32_fadd_f32_commute:
+; AVX-FAST:       # %bb.0:
+; AVX-FAST-NEXT:    vhaddps %xmm0, %xmm0, %xmm0
+; AVX-FAST-NEXT:    retq
+  %x0 = extractelement <4 x float> %x, i32 0
+  %x1 = extractelement <4 x float> %x, i32 1
+  %x01 = fadd float %x1, %x0
+  ret float %x01
+}
+
+define double @extract_extract_v2f64_fadd_f64(<2 x double> %x) {
+; SSE3-SLOW-LABEL: extract_extract_v2f64_fadd_f64:
+; SSE3-SLOW:       # %bb.0:
+; SSE3-SLOW-NEXT:    movapd %xmm0, %xmm1
+; SSE3-SLOW-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm0[1]
+; SSE3-SLOW-NEXT:    addsd %xmm0, %xmm1
+; SSE3-SLOW-NEXT:    movapd %xmm1, %xmm0
+; SSE3-SLOW-NEXT:    retq
+;
+; SSE3-FAST-LABEL: extract_extract_v2f64_fadd_f64:
+; SSE3-FAST:       # %bb.0:
+; SSE3-FAST-NEXT:    haddpd %xmm0, %xmm0
+; SSE3-FAST-NEXT:    retq
+;
+; AVX-SLOW-LABEL: extract_extract_v2f64_fadd_f64:
+; AVX-SLOW:       # %bb.0:
+; AVX-SLOW-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
+; AVX-SLOW-NEXT:    vaddsd %xmm1, %xmm0, %xmm0
+; AVX-SLOW-NEXT:    retq
+;
+; AVX-FAST-LABEL: extract_extract_v2f64_fadd_f64:
+; AVX-FAST:       # %bb.0:
+; AVX-FAST-NEXT:    vhaddpd %xmm0, %xmm0, %xmm0
+; AVX-FAST-NEXT:    retq
+  %x0 = extractelement <2 x double> %x, i32 0
+  %x1 = extractelement <2 x double> %x, i32 1
+  %x01 = fadd double %x0, %x1
+  ret double %x01
+}
+
+define double @extract_extract_v2f64_fadd_f64_commute(<2 x double> %x) {
+; SSE3-SLOW-LABEL: extract_extract_v2f64_fadd_f64_commute:
+; SSE3-SLOW:       # %bb.0:
+; SSE3-SLOW-NEXT:    movapd %xmm0, %xmm1
+; SSE3-SLOW-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm0[1]
+; SSE3-SLOW-NEXT:    addsd %xmm0, %xmm1
+; SSE3-SLOW-NEXT:    movapd %xmm1, %xmm0
+; SSE3-SLOW-NEXT:    retq
+;
+; SSE3-FAST-LABEL: extract_extract_v2f64_fadd_f64_commute:
+; SSE3-FAST:       # %bb.0:
+; SSE3-FAST-NEXT:    haddpd %xmm0, %xmm0
+; SSE3-FAST-NEXT:    retq
+;
+; AVX-SLOW-LABEL: extract_extract_v2f64_fadd_f64_commute:
+; AVX-SLOW:       # %bb.0:
+; AVX-SLOW-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
+; AVX-SLOW-NEXT:    vaddsd %xmm0, %xmm1, %xmm0
+; AVX-SLOW-NEXT:    retq
+;
+; AVX-FAST-LABEL: extract_extract_v2f64_fadd_f64_commute:
+; AVX-FAST:       # %bb.0:
+; AVX-FAST-NEXT:    vhaddpd %xmm0, %xmm0, %xmm0
+; AVX-FAST-NEXT:    retq
+  %x0 = extractelement <2 x double> %x, i32 0
+  %x1 = extractelement <2 x double> %x, i32 1
+  %x01 = fadd double %x1, %x0
+  ret double %x01
+}
+
+define float @extract_extract_v4f32_fsub_f32(<4 x float> %x) {
+; SSE3-SLOW-LABEL: extract_extract_v4f32_fsub_f32:
+; SSE3-SLOW:       # %bb.0:
+; SSE3-SLOW-NEXT:    movshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; SSE3-SLOW-NEXT:    subss %xmm1, %xmm0
+; SSE3-SLOW-NEXT:    retq
+;
+; SSE3-FAST-LABEL: extract_extract_v4f32_fsub_f32:
+; SSE3-FAST:       # %bb.0:
+; SSE3-FAST-NEXT:    hsubps %xmm0, %xmm0
+; SSE3-FAST-NEXT:    retq
+;
+; AVX-SLOW-LABEL: extract_extract_v4f32_fsub_f32:
+; AVX-SLOW:       # %bb.0:
+; AVX-SLOW-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; AVX-SLOW-NEXT:    vsubss %xmm1, %xmm0, %xmm0
+; AVX-SLOW-NEXT:    retq
+;
+; AVX-FAST-LABEL: extract_extract_v4f32_fsub_f32:
+; AVX-FAST:       # %bb.0:
+; AVX-FAST-NEXT:    vhsubps %xmm0, %xmm0, %xmm0
+; AVX-FAST-NEXT:    retq
+  %x0 = extractelement <4 x float> %x, i32 0
+  %x1 = extractelement <4 x float> %x, i32 1
+  %x01 = fsub float %x0, %x1
+  ret float %x01
+}
+
+define float @extract_extract_v4f32_fsub_f32_commute(<4 x float> %x) {
+; SSE3-LABEL: extract_extract_v4f32_fsub_f32_commute:
+; SSE3:       # %bb.0:
+; SSE3-NEXT:    movshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; SSE3-NEXT:    subss %xmm0, %xmm1
+; SSE3-NEXT:    movaps %xmm1, %xmm0
+; SSE3-NEXT:    retq
+;
+; AVX-LABEL: extract_extract_v4f32_fsub_f32_commute:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; AVX-NEXT:    vsubss %xmm0, %xmm1, %xmm0
+; AVX-NEXT:    retq
+  %x0 = extractelement <4 x float> %x, i32 0
+  %x1 = extractelement <4 x float> %x, i32 1
+  %x01 = fsub float %x1, %x0
+  ret float %x01
+}
+
+define double @extract_extract_v2f64_fsub_f64(<2 x double> %x) {
+; SSE3-SLOW-LABEL: extract_extract_v2f64_fsub_f64:
+; SSE3-SLOW:       # %bb.0:
+; SSE3-SLOW-NEXT:    movapd %xmm0, %xmm1
+; SSE3-SLOW-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm0[1]
+; SSE3-SLOW-NEXT:    subsd %xmm1, %xmm0
+; SSE3-SLOW-NEXT:    retq
+;
+; SSE3-FAST-LABEL: extract_extract_v2f64_fsub_f64:
+; SSE3-FAST:       # %bb.0:
+; SSE3-FAST-NEXT:    hsubpd %xmm0, %xmm0
+; SSE3-FAST-NEXT:    retq
+;
+; AVX-SLOW-LABEL: extract_extract_v2f64_fsub_f64:
+; AVX-SLOW:       # %bb.0:
+; AVX-SLOW-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
+; AVX-SLOW-NEXT:    vsubsd %xmm1, %xmm0, %xmm0
+; AVX-SLOW-NEXT:    retq
+;
+; AVX-FAST-LABEL: extract_extract_v2f64_fsub_f64:
+; AVX-FAST:       # %bb.0:
+; AVX-FAST-NEXT:    vhsubpd %xmm0, %xmm0, %xmm0
+; AVX-FAST-NEXT:    retq
+  %x0 = extractelement <2 x double> %x, i32 0
+  %x1 = extractelement <2 x double> %x, i32 1
+  %x01 = fsub double %x0, %x1
+  ret double %x01
+}
+
+define double @extract_extract_v2f64_fsub_f64_commute(<2 x double> %x) {
+; SSE3-LABEL: extract_extract_v2f64_fsub_f64_commute:
+; SSE3:       # %bb.0:
+; SSE3-NEXT:    movapd %xmm0, %xmm1
+; SSE3-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm0[1]
+; SSE3-NEXT:    subsd %xmm0, %xmm1
+; SSE3-NEXT:    movapd %xmm1, %xmm0
+; SSE3-NEXT:    retq
+;
+; AVX-LABEL: extract_extract_v2f64_fsub_f64_commute:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
+; AVX-NEXT:    vsubsd %xmm0, %xmm1, %xmm0
+; AVX-NEXT:    retq
+  %x0 = extractelement <2 x double> %x, i32 0
+  %x1 = extractelement <2 x double> %x, i32 1
+  %x01 = fsub double %x1, %x0
+  ret double %x01
+}
+
+; 256-bit vectors, float/double, fadd/fsub
+
+define float @extract_extract_v8f32_fadd_f32(<8 x float> %x) {
+; SSE3-SLOW-LABEL: extract_extract_v8f32_fadd_f32:
+; SSE3-SLOW:       # %bb.0:
+; SSE3-SLOW-NEXT:    movshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; SSE3-SLOW-NEXT:    addss %xmm1, %xmm0
+; SSE3-SLOW-NEXT:    retq
+;
+; SSE3-FAST-LABEL: extract_extract_v8f32_fadd_f32:
+; SSE3-FAST:       # %bb.0:
+; SSE3-FAST-NEXT:    haddps %xmm0, %xmm0
+; SSE3-FAST-NEXT:    retq
+;
+; AVX-SLOW-LABEL: extract_extract_v8f32_fadd_f32:
+; AVX-SLOW:       # %bb.0:
+; AVX-SLOW-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; AVX-SLOW-NEXT:    vaddss %xmm1, %xmm0, %xmm0
+; AVX-SLOW-NEXT:    vzeroupper
+; AVX-SLOW-NEXT:    retq
+;
+; AVX-FAST-LABEL: extract_extract_v8f32_fadd_f32:
+; AVX-FAST:       # %bb.0:
+; AVX-FAST-NEXT:    vhaddps %xmm0, %xmm0, %xmm0
+; AVX-FAST-NEXT:    vzeroupper
+; AVX-FAST-NEXT:    retq
+  %x0 = extractelement <8 x float> %x, i32 0
+  %x1 = extractelement <8 x float> %x, i32 1
+  %x01 = fadd float %x0, %x1
+  ret float %x01
+}
+
+define float @extract_extract_v8f32_fadd_f32_commute(<8 x float> %x) {
+; SSE3-SLOW-LABEL: extract_extract_v8f32_fadd_f32_commute:
+; SSE3-SLOW:       # %bb.0:
+; SSE3-SLOW-NEXT:    movshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; SSE3-SLOW-NEXT:    addss %xmm1, %xmm0
+; SSE3-SLOW-NEXT:    retq
+;
+; SSE3-FAST-LABEL: extract_extract_v8f32_fadd_f32_commute:
+; SSE3-FAST:       # %bb.0:
+; SSE3-FAST-NEXT:    haddps %xmm0, %xmm0
+; SSE3-FAST-NEXT:    retq
+;
+; AVX-SLOW-LABEL: extract_extract_v8f32_fadd_f32_commute:
+; AVX-SLOW:       # %bb.0:
+; AVX-SLOW-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; AVX-SLOW-NEXT:    vaddss %xmm0, %xmm1, %xmm0
+; AVX-SLOW-NEXT:    vzeroupper
+; AVX-SLOW-NEXT:    retq
+;
+; AVX-FAST-LABEL: extract_extract_v8f32_fadd_f32_commute:
+; AVX-FAST:       # %bb.0:
+; AVX-FAST-NEXT:    vhaddps %xmm0, %xmm0, %xmm0
+; AVX-FAST-NEXT:    vzeroupper
+; AVX-FAST-NEXT:    retq
+  %x0 = extractelement <8 x float> %x, i32 0
+  %x1 = extractelement <8 x float> %x, i32 1
+  %x01 = fadd float %x1, %x0
+  ret float %x01
+}
+
+define double @extract_extract_v4f64_fadd_f64(<4 x double> %x) {
+; SSE3-SLOW-LABEL: extract_extract_v4f64_fadd_f64:
+; SSE3-SLOW:       # %bb.0:
+; SSE3-SLOW-NEXT:    movapd %xmm0, %xmm1
+; SSE3-SLOW-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm0[1]
+; SSE3-SLOW-NEXT:    addsd %xmm0, %xmm1
+; SSE3-SLOW-NEXT:    movapd %xmm1, %xmm0
+; SSE3-SLOW-NEXT:    retq
+;
+; SSE3-FAST-LABEL: extract_extract_v4f64_fadd_f64:
+; SSE3-FAST:       # %bb.0:
+; SSE3-FAST-NEXT:    haddpd %xmm0, %xmm0
+; SSE3-FAST-NEXT:    retq
+;
+; AVX-SLOW-LABEL: extract_extract_v4f64_fadd_f64:
+; AVX-SLOW:       # %bb.0:
+; AVX-SLOW-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
+; AVX-SLOW-NEXT:    vaddsd %xmm1, %xmm0, %xmm0
+; AVX-SLOW-NEXT:    vzeroupper
+; AVX-SLOW-NEXT:    retq
+;
+; AVX-FAST-LABEL: extract_extract_v4f64_fadd_f64:
+; AVX-FAST:       # %bb.0:
+; AVX-FAST-NEXT:    vhaddpd %xmm0, %xmm0, %xmm0
+; AVX-FAST-NEXT:    vzeroupper
+; AVX-FAST-NEXT:    retq
+  %x0 = extractelement <4 x double> %x, i32 0
+  %x1 = extractelement <4 x double> %x, i32 1
+  %x01 = fadd double %x0, %x1
+  ret double %x01
+}
+
+define double @extract_extract_v4f64_fadd_f64_commute(<4 x double> %x) {
+; SSE3-SLOW-LABEL: extract_extract_v4f64_fadd_f64_commute:
+; SSE3-SLOW:       # %bb.0:
+; SSE3-SLOW-NEXT:    movapd %xmm0, %xmm1
+; SSE3-SLOW-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm0[1]
+; SSE3-SLOW-NEXT:    addsd %xmm0, %xmm1
+; SSE3-SLOW-NEXT:    movapd %xmm1, %xmm0
+; SSE3-SLOW-NEXT:    retq
+;
+; SSE3-FAST-LABEL: extract_extract_v4f64_fadd_f64_commute:
+; SSE3-FAST:       # %bb.0:
+; SSE3-FAST-NEXT:    haddpd %xmm0, %xmm0
+; SSE3-FAST-NEXT:    retq
+;
+; AVX-SLOW-LABEL: extract_extract_v4f64_fadd_f64_commute:
+; AVX-SLOW:       # %bb.0:
+; AVX-SLOW-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
+; AVX-SLOW-NEXT:    vaddsd %xmm0, %xmm1, %xmm0
+; AVX-SLOW-NEXT:    vzeroupper
+; AVX-SLOW-NEXT:    retq
+;
+; AVX-FAST-LABEL: extract_extract_v4f64_fadd_f64_commute:
+; AVX-FAST:       # %bb.0:
+; AVX-FAST-NEXT:    vhaddpd %xmm0, %xmm0, %xmm0
+; AVX-FAST-NEXT:    vzeroupper
+; AVX-FAST-NEXT:    retq
+  %x0 = extractelement <4 x double> %x, i32 0
+  %x1 = extractelement <4 x double> %x, i32 1
+  %x01 = fadd double %x1, %x0
+  ret double %x01
+}
+
+define float @extract_extract_v8f32_fsub_f32(<8 x float> %x) {
+; SSE3-SLOW-LABEL: extract_extract_v8f32_fsub_f32:
+; SSE3-SLOW:       # %bb.0:
+; SSE3-SLOW-NEXT:    movshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; SSE3-SLOW-NEXT:    subss %xmm1, %xmm0
+; SSE3-SLOW-NEXT:    retq
+;
+; SSE3-FAST-LABEL: extract_extract_v8f32_fsub_f32:
+; SSE3-FAST:       # %bb.0:
+; SSE3-FAST-NEXT:    hsubps %xmm0, %xmm0
+; SSE3-FAST-NEXT:    retq
+;
+; AVX-SLOW-LABEL: extract_extract_v8f32_fsub_f32:
+; AVX-SLOW:       # %bb.0:
+; AVX-SLOW-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; AVX-SLOW-NEXT:    vsubss %xmm1, %xmm0, %xmm0
+; AVX-SLOW-NEXT:    vzeroupper
+; AVX-SLOW-NEXT:    retq
+;
+; AVX-FAST-LABEL: extract_extract_v8f32_fsub_f32:
+; AVX-FAST:       # %bb.0:
+; AVX-FAST-NEXT:    vhsubps %xmm0, %xmm0, %xmm0
+; AVX-FAST-NEXT:    vzeroupper
+; AVX-FAST-NEXT:    retq
+  %x0 = extractelement <8 x float> %x, i32 0
+  %x1 = extractelement <8 x float> %x, i32 1
+  %x01 = fsub float %x0, %x1
+  ret float %x01
+}
+
+; Negative test...or get hoppy and negate?
+
+define float @extract_extract_v8f32_fsub_f32_commute(<8 x float> %x) {
+; SSE3-LABEL: extract_extract_v8f32_fsub_f32_commute:
+; SSE3:       # %bb.0:
+; SSE3-NEXT:    movshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; SSE3-NEXT:    subss %xmm0, %xmm1
+; SSE3-NEXT:    movaps %xmm1, %xmm0
+; SSE3-NEXT:    retq
+;
+; AVX-LABEL: extract_extract_v8f32_fsub_f32_commute:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; AVX-NEXT:    vsubss %xmm0, %xmm1, %xmm0
+; AVX-NEXT:    vzeroupper
+; AVX-NEXT:    retq
+  %x0 = extractelement <8 x float> %x, i32 0
+  %x1 = extractelement <8 x float> %x, i32 1
+  %x01 = fsub float %x1, %x0
+  ret float %x01
+}
+
+define double @extract_extract_v4f64_fsub_f64(<4 x double> %x) {
+; SSE3-SLOW-LABEL: extract_extract_v4f64_fsub_f64:
+; SSE3-SLOW:       # %bb.0:
+; SSE3-SLOW-NEXT:    movapd %xmm0, %xmm1
+; SSE3-SLOW-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm0[1]
+; SSE3-SLOW-NEXT:    subsd %xmm1, %xmm0
+; SSE3-SLOW-NEXT:    retq
+;
+; SSE3-FAST-LABEL: extract_extract_v4f64_fsub_f64:
+; SSE3-FAST:       # %bb.0:
+; SSE3-FAST-NEXT:    hsubpd %xmm0, %xmm0
+; SSE3-FAST-NEXT:    retq
+;
+; AVX-SLOW-LABEL: extract_extract_v4f64_fsub_f64:
+; AVX-SLOW:       # %bb.0:
+; AVX-SLOW-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
+; AVX-SLOW-NEXT:    vsubsd %xmm1, %xmm0, %xmm0
+; AVX-SLOW-NEXT:    vzeroupper
+; AVX-SLOW-NEXT:    retq
+;
+; AVX-FAST-LABEL: extract_extract_v4f64_fsub_f64:
+; AVX-FAST:       # %bb.0:
+; AVX-FAST-NEXT:    vhsubpd %xmm0, %xmm0, %xmm0
+; AVX-FAST-NEXT:    vzeroupper
+; AVX-FAST-NEXT:    retq
+  %x0 = extractelement <4 x double> %x, i32 0
+  %x1 = extractelement <4 x double> %x, i32 1
+  %x01 = fsub double %x0, %x1
+  ret double %x01
+}
+
+; Negative test...or get hoppy and negate?
+
+define double @extract_extract_v4f64_fsub_f64_commute(<4 x double> %x) {
+; SSE3-LABEL: extract_extract_v4f64_fsub_f64_commute:
+; SSE3:       # %bb.0:
+; SSE3-NEXT:    movapd %xmm0, %xmm1
+; SSE3-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm0[1]
+; SSE3-NEXT:    subsd %xmm0, %xmm1
+; SSE3-NEXT:    movapd %xmm1, %xmm0
+; SSE3-NEXT:    retq
+;
+; AVX-LABEL: extract_extract_v4f64_fsub_f64_commute:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
+; AVX-NEXT:    vsubsd %xmm0, %xmm1, %xmm0
+; AVX-NEXT:    vzeroupper
+; AVX-NEXT:    retq
+  %x0 = extractelement <4 x double> %x, i32 0
+  %x1 = extractelement <4 x double> %x, i32 1
+  %x01 = fsub double %x1, %x0
+  ret double %x01
+}
+
+; 512-bit vectors, float/double, fadd/fsub
+
+define float @extract_extract_v16f32_fadd_f32(<16 x float> %x) {
+; SSE3-SLOW-LABEL: extract_extract_v16f32_fadd_f32:
+; SSE3-SLOW:       # %bb.0:
+; SSE3-SLOW-NEXT:    movshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; SSE3-SLOW-NEXT:    addss %xmm1, %xmm0
+; SSE3-SLOW-NEXT:    retq
+;
+; SSE3-FAST-LABEL: extract_extract_v16f32_fadd_f32:
+; SSE3-FAST:       # %bb.0:
+; SSE3-FAST-NEXT:    haddps %xmm0, %xmm0
+; SSE3-FAST-NEXT:    retq
+;
+; AVX-SLOW-LABEL: extract_extract_v16f32_fadd_f32:
+; AVX-SLOW:       # %bb.0:
+; AVX-SLOW-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; AVX-SLOW-NEXT:    vaddss %xmm1, %xmm0, %xmm0
+; AVX-SLOW-NEXT:    vzeroupper
+; AVX-SLOW-NEXT:    retq
+;
+; AVX-FAST-LABEL: extract_extract_v16f32_fadd_f32:
+; AVX-FAST:       # %bb.0:
+; AVX-FAST-NEXT:    vhaddps %xmm0, %xmm0, %xmm0
+; AVX-FAST-NEXT:    vzeroupper
+; AVX-FAST-NEXT:    retq
+  %x0 = extractelement <16 x float> %x, i32 0
+  %x1 = extractelement <16 x float> %x, i32 1
+  %x01 = fadd float %x0, %x1
+  ret float %x01
+}
+
+define float @extract_extract_v16f32_fadd_f32_commute(<16 x float> %x) {
+; SSE3-SLOW-LABEL: extract_extract_v16f32_fadd_f32_commute:
+; SSE3-SLOW:       # %bb.0:
+; SSE3-SLOW-NEXT:    movshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; SSE3-SLOW-NEXT:    addss %xmm1, %xmm0
+; SSE3-SLOW-NEXT:    retq
+;
+; SSE3-FAST-LABEL: extract_extract_v16f32_fadd_f32_commute:
+; SSE3-FAST:       # %bb.0:
+; SSE3-FAST-NEXT:    haddps %xmm0, %xmm0
+; SSE3-FAST-NEXT:    retq
+;
+; AVX-SLOW-LABEL: extract_extract_v16f32_fadd_f32_commute:
+; AVX-SLOW:       # %bb.0:
+; AVX-SLOW-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; AVX-SLOW-NEXT:    vaddss %xmm0, %xmm1, %xmm0
+; AVX-SLOW-NEXT:    vzeroupper
+; AVX-SLOW-NEXT:    retq
+;
+; AVX-FAST-LABEL: extract_extract_v16f32_fadd_f32_commute:
+; AVX-FAST:       # %bb.0:
+; AVX-FAST-NEXT:    vhaddps %xmm0, %xmm0, %xmm0
+; AVX-FAST-NEXT:    vzeroupper
+; AVX-FAST-NEXT:    retq
+  %x0 = extractelement <16 x float> %x, i32 0
+  %x1 = extractelement <16 x float> %x, i32 1
+  %x01 = fadd float %x1, %x0
+  ret float %x01
+}
+
+define double @extract_extract_v8f64_fadd_f64(<8 x double> %x) {
+; SSE3-SLOW-LABEL: extract_extract_v8f64_fadd_f64:
+; SSE3-SLOW:       # %bb.0:
+; SSE3-SLOW-NEXT:    movapd %xmm0, %xmm1
+; SSE3-SLOW-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm0[1]
+; SSE3-SLOW-NEXT:    addsd %xmm0, %xmm1
+; SSE3-SLOW-NEXT:    movapd %xmm1, %xmm0
+; SSE3-SLOW-NEXT:    retq
+;
+; SSE3-FAST-LABEL: extract_extract_v8f64_fadd_f64:
+; SSE3-FAST:       # %bb.0:
+; SSE3-FAST-NEXT:    haddpd %xmm0, %xmm0
+; SSE3-FAST-NEXT:    retq
+;
+; AVX-SLOW-LABEL: extract_extract_v8f64_fadd_f64:
+; AVX-SLOW:       # %bb.0:
+; AVX-SLOW-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
+; AVX-SLOW-NEXT:    vaddsd %xmm1, %xmm0, %xmm0
+; AVX-SLOW-NEXT:    vzeroupper
+; AVX-SLOW-NEXT:    retq
+;
+; AVX-FAST-LABEL: extract_extract_v8f64_fadd_f64:
+; AVX-FAST:       # %bb.0:
+; AVX-FAST-NEXT:    vhaddpd %xmm0, %xmm0, %xmm0
+; AVX-FAST-NEXT:    vzeroupper
+; AVX-FAST-NEXT:    retq
+  %x0 = extractelement <8 x double> %x, i32 0
+  %x1 = extractelement <8 x double> %x, i32 1
+  %x01 = fadd double %x0, %x1
+  ret double %x01
+}
+
+define double @extract_extract_v8f64_fadd_f64_commute(<8 x double> %x) {
+; SSE3-SLOW-LABEL: extract_extract_v8f64_fadd_f64_commute:
+; SSE3-SLOW:       # %bb.0:
+; SSE3-SLOW-NEXT:    movapd %xmm0, %xmm1
+; SSE3-SLOW-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm0[1]
+; SSE3-SLOW-NEXT:    addsd %xmm0, %xmm1
+; SSE3-SLOW-NEXT:    movapd %xmm1, %xmm0
+; SSE3-SLOW-NEXT:    retq
+;
+; SSE3-FAST-LABEL: extract_extract_v8f64_fadd_f64_commute:
+; SSE3-FAST:       # %bb.0:
+; SSE3-FAST-NEXT:    haddpd %xmm0, %xmm0
+; SSE3-FAST-NEXT:    retq
+;
+; AVX-SLOW-LABEL: extract_extract_v8f64_fadd_f64_commute:
+; AVX-SLOW:       # %bb.0:
+; AVX-SLOW-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
+; AVX-SLOW-NEXT:    vaddsd %xmm0, %xmm1, %xmm0
+; AVX-SLOW-NEXT:    vzeroupper
+; AVX-SLOW-NEXT:    retq
+;
+; AVX-FAST-LABEL: extract_extract_v8f64_fadd_f64_commute:
+; AVX-FAST:       # %bb.0:
+; AVX-FAST-NEXT:    vhaddpd %xmm0, %xmm0, %xmm0
+; AVX-FAST-NEXT:    vzeroupper
+; AVX-FAST-NEXT:    retq
+  %x0 = extractelement <8 x double> %x, i32 0
+  %x1 = extractelement <8 x double> %x, i32 1
+  %x01 = fadd double %x1, %x0
+  ret double %x01
+}
+
+define float @extract_extract_v16f32_fsub_f32(<16 x float> %x) {
+; SSE3-SLOW-LABEL: extract_extract_v16f32_fsub_f32:
+; SSE3-SLOW:       # %bb.0:
+; SSE3-SLOW-NEXT:    movshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; SSE3-SLOW-NEXT:    subss %xmm1, %xmm0
+; SSE3-SLOW-NEXT:    retq
+;
+; SSE3-FAST-LABEL: extract_extract_v16f32_fsub_f32:
+; SSE3-FAST:       # %bb.0:
+; SSE3-FAST-NEXT:    hsubps %xmm0, %xmm0
+; SSE3-FAST-NEXT:    retq
+;
+; AVX-SLOW-LABEL: extract_extract_v16f32_fsub_f32:
+; AVX-SLOW:       # %bb.0:
+; AVX-SLOW-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; AVX-SLOW-NEXT:    vsubss %xmm1, %xmm0, %xmm0
+; AVX-SLOW-NEXT:    vzeroupper
+; AVX-SLOW-NEXT:    retq
+;
+; AVX-FAST-LABEL: extract_extract_v16f32_fsub_f32:
+; AVX-FAST:       # %bb.0:
+; AVX-FAST-NEXT:    vhsubps %xmm0, %xmm0, %xmm0
+; AVX-FAST-NEXT:    vzeroupper
+; AVX-FAST-NEXT:    retq
+  %x0 = extractelement <16 x float> %x, i32 0
+  %x1 = extractelement <16 x float> %x, i32 1
+  %x01 = fsub float %x0, %x1
+  ret float %x01
+}
+
+define float @extract_extract_v16f32_fsub_f32_commute(<16 x float> %x) {
+; SSE3-LABEL: extract_extract_v16f32_fsub_f32_commute:
+; SSE3:       # %bb.0:
+; SSE3-NEXT:    movshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; SSE3-NEXT:    subss %xmm0, %xmm1
+; SSE3-NEXT:    movaps %xmm1, %xmm0
+; SSE3-NEXT:    retq
+;
+; AVX-LABEL: extract_extract_v16f32_fsub_f32_commute:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; AVX-NEXT:    vsubss %xmm0, %xmm1, %xmm0
+; AVX-NEXT:    vzeroupper
+; AVX-NEXT:    retq
+  %x0 = extractelement <16 x float> %x, i32 0
+  %x1 = extractelement <16 x float> %x, i32 1
+  %x01 = fsub float %x1, %x0
+  ret float %x01
+}
+
+define double @extract_extract_v8f64_fsub_f64(<8 x double> %x) {
+; SSE3-SLOW-LABEL: extract_extract_v8f64_fsub_f64:
+; SSE3-SLOW:       # %bb.0:
+; SSE3-SLOW-NEXT:    movapd %xmm0, %xmm1
+; SSE3-SLOW-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm0[1]
+; SSE3-SLOW-NEXT:    subsd %xmm1, %xmm0
+; SSE3-SLOW-NEXT:    retq
+;
+; SSE3-FAST-LABEL: extract_extract_v8f64_fsub_f64:
+; SSE3-FAST:       # %bb.0:
+; SSE3-FAST-NEXT:    hsubpd %xmm0, %xmm0
+; SSE3-FAST-NEXT:    retq
+;
+; AVX-SLOW-LABEL: extract_extract_v8f64_fsub_f64:
+; AVX-SLOW:       # %bb.0:
+; AVX-SLOW-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
+; AVX-SLOW-NEXT:    vsubsd %xmm1, %xmm0, %xmm0
+; AVX-SLOW-NEXT:    vzeroupper
+; AVX-SLOW-NEXT:    retq
+;
+; AVX-FAST-LABEL: extract_extract_v8f64_fsub_f64:
+; AVX-FAST:       # %bb.0:
+; AVX-FAST-NEXT:    vhsubpd %xmm0, %xmm0, %xmm0
+; AVX-FAST-NEXT:    vzeroupper
+; AVX-FAST-NEXT:    retq
+  %x0 = extractelement <8 x double> %x, i32 0
+  %x1 = extractelement <8 x double> %x, i32 1
+  %x01 = fsub double %x0, %x1
+  ret double %x01
+}
+
+define double @extract_extract_v8f64_fsub_f64_commute(<8 x double> %x) {
+; SSE3-LABEL: extract_extract_v8f64_fsub_f64_commute:
+; SSE3:       # %bb.0:
+; SSE3-NEXT:    movapd %xmm0, %xmm1
+; SSE3-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm0[1]
+; SSE3-NEXT:    subsd %xmm0, %xmm1
+; SSE3-NEXT:    movapd %xmm1, %xmm0
+; SSE3-NEXT:    retq
+;
+; AVX-LABEL: extract_extract_v8f64_fsub_f64_commute:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
+; AVX-NEXT:    vsubsd %xmm0, %xmm1, %xmm0
+; AVX-NEXT:    vzeroupper
+; AVX-NEXT:    retq
+  %x0 = extractelement <8 x double> %x, i32 0
+  %x1 = extractelement <8 x double> %x, i32 1
+  %x01 = fsub double %x1, %x0
+  ret double %x01
+}
+
+; Check output when 1 or both extracts have extra uses.
+
+define float @extract_extract_v4f32_fadd_f32_uses1(<4 x float> %x, float* %p) {
+; SSE3-SLOW-LABEL: extract_extract_v4f32_fadd_f32_uses1:
+; SSE3-SLOW:       # %bb.0:
+; SSE3-SLOW-NEXT:    movss %xmm0, (%rdi)
+; SSE3-SLOW-NEXT:    movshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; SSE3-SLOW-NEXT:    addss %xmm1, %xmm0
+; SSE3-SLOW-NEXT:    retq
+;
+; SSE3-FAST-LABEL: extract_extract_v4f32_fadd_f32_uses1:
+; SSE3-FAST:       # %bb.0:
+; SSE3-FAST-NEXT:    movss %xmm0, (%rdi)
+; SSE3-FAST-NEXT:    haddps %xmm0, %xmm0
+; SSE3-FAST-NEXT:    retq
+;
+; AVX-SLOW-LABEL: extract_extract_v4f32_fadd_f32_uses1:
+; AVX-SLOW:       # %bb.0:
+; AVX-SLOW-NEXT:    vmovss %xmm0, (%rdi)
+; AVX-SLOW-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; AVX-SLOW-NEXT:    vaddss %xmm1, %xmm0, %xmm0
+; AVX-SLOW-NEXT:    retq
+;
+; AVX-FAST-LABEL: extract_extract_v4f32_fadd_f32_uses1:
+; AVX-FAST:       # %bb.0:
+; AVX-FAST-NEXT:    vmovss %xmm0, (%rdi)
+; AVX-FAST-NEXT:    vhaddps %xmm0, %xmm0, %xmm0
+; AVX-FAST-NEXT:    retq
+  %x0 = extractelement <4 x float> %x, i32 0
+  store float %x0, float* %p
+  %x1 = extractelement <4 x float> %x, i32 1
+  %x01 = fadd float %x0, %x1
+  ret float %x01
+}
+
+define float @extract_extract_v4f32_fadd_f32_uses2(<4 x float> %x, float* %p) {
+; SSE3-SLOW-LABEL: extract_extract_v4f32_fadd_f32_uses2:
+; SSE3-SLOW:       # %bb.0:
+; SSE3-SLOW-NEXT:    movshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; SSE3-SLOW-NEXT:    movss %xmm1, (%rdi)
+; SSE3-SLOW-NEXT:    addss %xmm1, %xmm0
+; SSE3-SLOW-NEXT:    retq
+;
+; SSE3-FAST-LABEL: extract_extract_v4f32_fadd_f32_uses2:
+; SSE3-FAST:       # %bb.0:
+; SSE3-FAST-NEXT:    movshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; SSE3-FAST-NEXT:    movss %xmm1, (%rdi)
+; SSE3-FAST-NEXT:    haddps %xmm0, %xmm0
+; SSE3-FAST-NEXT:    retq
+;
+; AVX-SLOW-LABEL: extract_extract_v4f32_fadd_f32_uses2:
+; AVX-SLOW:       # %bb.0:
+; AVX-SLOW-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; AVX-SLOW-NEXT:    vmovss %xmm1, (%rdi)
+; AVX-SLOW-NEXT:    vaddss %xmm1, %xmm0, %xmm0
+; AVX-SLOW-NEXT:    retq
+;
+; AVX-FAST-LABEL: extract_extract_v4f32_fadd_f32_uses2:
+; AVX-FAST:       # %bb.0:
+; AVX-FAST-NEXT:    vextractps $1, %xmm0, (%rdi)
+; AVX-FAST-NEXT:    vhaddps %xmm0, %xmm0, %xmm0
+; AVX-FAST-NEXT:    retq
+  %x0 = extractelement <4 x float> %x, i32 0
+  %x1 = extractelement <4 x float> %x, i32 1
+  store float %x1, float* %p
+  %x01 = fadd float %x0, %x1
+  ret float %x01
+}
+
+define float @extract_extract_v4f32_fadd_f32_uses3(<4 x float> %x, float* %p1, float* %p2) {
+; SSE3-LABEL: extract_extract_v4f32_fadd_f32_uses3:
+; SSE3:       # %bb.0:
+; SSE3-NEXT:    movss %xmm0, (%rdi)
+; SSE3-NEXT:    movshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; SSE3-NEXT:    movss %xmm1, (%rsi)
+; SSE3-NEXT:    addss %xmm1, %xmm0
+; SSE3-NEXT:    retq
+;
+; AVX-LABEL: extract_extract_v4f32_fadd_f32_uses3:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovss %xmm0, (%rdi)
+; AVX-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; AVX-NEXT:    vmovss %xmm1, (%rsi)
+; AVX-NEXT:    vaddss %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    retq
+  %x0 = extractelement <4 x float> %x, i32 0
+  store float %x0, float* %p1
+  %x1 = extractelement <4 x float> %x, i32 1
+  store float %x1, float* %p2
+  %x01 = fadd float %x0, %x1
+  ret float %x01
+}
+
+; Repeat tests from general reductions to verify output for hoppy targets:
+; PR38971: https://bugs.llvm.org/show_bug.cgi?id=38971
+
+declare float @llvm.experimental.vector.reduce.fadd.f32.f32.v8f32(float, <8 x float>)
+declare double @llvm.experimental.vector.reduce.fadd.f64.f64.v4f64(double, <4 x double>)
+
+define float @fadd_reduce_v8f32(float %a0, <8 x float> %a1) {
+; SSE3-SLOW-LABEL: fadd_reduce_v8f32:
+; SSE3-SLOW:       # %bb.0:
+; SSE3-SLOW-NEXT:    addps %xmm2, %xmm1
+; SSE3-SLOW-NEXT:    movaps %xmm1, %xmm2
+; SSE3-SLOW-NEXT:    unpckhpd {{.*#+}} xmm2 = xmm2[1],xmm1[1]
+; SSE3-SLOW-NEXT:    addps %xmm1, %xmm2
+; SSE3-SLOW-NEXT:    movshdup {{.*#+}} xmm0 = xmm2[1,1,3,3]
+; SSE3-SLOW-NEXT:    addps %xmm2, %xmm0
+; SSE3-SLOW-NEXT:    retq
+;
+; SSE3-FAST-LABEL: fadd_reduce_v8f32:
+; SSE3-FAST:       # %bb.0:
+; SSE3-FAST-NEXT:    addps %xmm2, %xmm1
+; SSE3-FAST-NEXT:    movaps %xmm1, %xmm0
+; SSE3-FAST-NEXT:    unpckhpd {{.*#+}} xmm0 = xmm0[1],xmm1[1]
+; SSE3-FAST-NEXT:    addps %xmm1, %xmm0
+; SSE3-FAST-NEXT:    haddps %xmm0, %xmm0
+; SSE3-FAST-NEXT:    retq
+;
+; AVX-SLOW-LABEL: fadd_reduce_v8f32:
+; AVX-SLOW:       # %bb.0:
+; AVX-SLOW-NEXT:    vextractf128 $1, %ymm1, %xmm0
+; AVX-SLOW-NEXT:    vaddps %xmm0, %xmm1, %xmm0
+; AVX-SLOW-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
+; AVX-SLOW-NEXT:    vaddps %xmm1, %xmm0, %xmm0
+; AVX-SLOW-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; AVX-SLOW-NEXT:    vaddps %xmm1, %xmm0, %xmm0
+; AVX-SLOW-NEXT:    vzeroupper
+; AVX-SLOW-NEXT:    retq
+;
+; AVX-FAST-LABEL: fadd_reduce_v8f32:
+; AVX-FAST:       # %bb.0:
+; AVX-FAST-NEXT:    vextractf128 $1, %ymm1, %xmm0
+; AVX-FAST-NEXT:    vaddps %xmm0, %xmm1, %xmm0
+; AVX-FAST-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
+; AVX-FAST-NEXT:    vaddps %xmm1, %xmm0, %xmm0
+; AVX-FAST-NEXT:    vhaddps %xmm0, %xmm0, %xmm0
+; AVX-FAST-NEXT:    vzeroupper
+; AVX-FAST-NEXT:    retq
+  %r = call fast float @llvm.experimental.vector.reduce.fadd.f32.f32.v8f32(float %a0, <8 x float> %a1)
+  ret float %r
+}
+
+define double @fadd_reduce_v4f64(double %a0, <4 x double> %a1) {
+; SSE3-SLOW-LABEL: fadd_reduce_v4f64:
+; SSE3-SLOW:       # %bb.0:
+; SSE3-SLOW-NEXT:    addpd %xmm2, %xmm1
+; SSE3-SLOW-NEXT:    movapd %xmm1, %xmm0
+; SSE3-SLOW-NEXT:    unpckhpd {{.*#+}} xmm0 = xmm0[1],xmm1[1]
+; SSE3-SLOW-NEXT:    addpd %xmm1, %xmm0
+; SSE3-SLOW-NEXT:    retq
+;
+; SSE3-FAST-LABEL: fadd_reduce_v4f64:
+; SSE3-FAST:       # %bb.0:
+; SSE3-FAST-NEXT:    movapd %xmm1, %xmm0
+; SSE3-FAST-NEXT:    addpd %xmm2, %xmm0
+; SSE3-FAST-NEXT:    haddpd %xmm0, %xmm0
+; SSE3-FAST-NEXT:    retq
+;
+; AVX-SLOW-LABEL: fadd_reduce_v4f64:
+; AVX-SLOW:       # %bb.0:
+; AVX-SLOW-NEXT:    vextractf128 $1, %ymm1, %xmm0
+; AVX-SLOW-NEXT:    vaddpd %xmm0, %xmm1, %xmm0
+; AVX-SLOW-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
+; AVX-SLOW-NEXT:    vaddpd %xmm1, %xmm0, %xmm0
+; AVX-SLOW-NEXT:    vzeroupper
+; AVX-SLOW-NEXT:    retq
+;
+; AVX-FAST-LABEL: fadd_reduce_v4f64:
+; AVX-FAST:       # %bb.0:
+; AVX-FAST-NEXT:    vextractf128 $1, %ymm1, %xmm0
+; AVX-FAST-NEXT:    vaddpd %xmm0, %xmm1, %xmm0
+; AVX-FAST-NEXT:    vhaddpd %xmm0, %xmm0, %xmm0
+; AVX-FAST-NEXT:    vzeroupper
+; AVX-FAST-NEXT:    retq
+  %r = call fast double @llvm.experimental.vector.reduce.fadd.f64.f64.v4f64(double %a0, <4 x double> %a1)
+  ret double %r
+}
+
diff --git a/test/CodeGen/X86/half.ll b/test/CodeGen/X86/half.ll
index 2bdd537c651d3ee4ff4eae896be665cfeb5ca07f..f8688de5b4e4648b974c58b21e906bfc4666ed29 100644
--- a/test/CodeGen/X86/half.ll
+++ b/test/CodeGen/X86/half.ll
@@ -938,4 +938,67 @@ define float @test_sitofp_fadd_i32(i32 %a, half* %b) #0 {
   ret float %tmp3
 }
 
+define half @PR40273(half) #0 {
+; CHECK-LIBCALL-LABEL: PR40273:
+; CHECK-LIBCALL:       # %bb.0:
+; CHECK-LIBCALL-NEXT:    pushq %rax
+; CHECK-LIBCALL-NEXT:    callq __gnu_f2h_ieee
+; CHECK-LIBCALL-NEXT:    movzwl %ax, %edi
+; CHECK-LIBCALL-NEXT:    callq __gnu_h2f_ieee
+; CHECK-LIBCALL-NEXT:    xorps %xmm1, %xmm1
+; CHECK-LIBCALL-NEXT:    ucomiss %xmm1, %xmm0
+; CHECK-LIBCALL-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; CHECK-LIBCALL-NEXT:    jne .LBB17_3
+; CHECK-LIBCALL-NEXT:  # %bb.1:
+; CHECK-LIBCALL-NEXT:    jp .LBB17_3
+; CHECK-LIBCALL-NEXT:  # %bb.2:
+; CHECK-LIBCALL-NEXT:    xorps %xmm0, %xmm0
+; CHECK-LIBCALL-NEXT:  .LBB17_3:
+; CHECK-LIBCALL-NEXT:    popq %rax
+; CHECK-LIBCALL-NEXT:    retq
+;
+; BWON-F16C-LABEL: PR40273:
+; BWON-F16C:       # %bb.0:
+; BWON-F16C-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
+; BWON-F16C-NEXT:    vcvtph2ps %xmm0, %xmm0
+; BWON-F16C-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; BWON-F16C-NEXT:    vucomiss %xmm1, %xmm0
+; BWON-F16C-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; BWON-F16C-NEXT:    jne .LBB17_3
+; BWON-F16C-NEXT:  # %bb.1:
+; BWON-F16C-NEXT:    jp .LBB17_3
+; BWON-F16C-NEXT:  # %bb.2:
+; BWON-F16C-NEXT:    vxorps %xmm0, %xmm0, %xmm0
+; BWON-F16C-NEXT:  .LBB17_3:
+; BWON-F16C-NEXT:    retq
+;
+; CHECK-I686-LABEL: PR40273:
+; CHECK-I686:       # %bb.0:
+; CHECK-I686-NEXT:    subl $12, %esp
+; CHECK-I686-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; CHECK-I686-NEXT:    movss %xmm0, (%esp)
+; CHECK-I686-NEXT:    calll __gnu_f2h_ieee
+; CHECK-I686-NEXT:    movzwl %ax, %eax
+; CHECK-I686-NEXT:    movl %eax, (%esp)
+; CHECK-I686-NEXT:    calll __gnu_h2f_ieee
+; CHECK-I686-NEXT:    fstps {{[0-9]+}}(%esp)
+; CHECK-I686-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; CHECK-I686-NEXT:    xorps %xmm1, %xmm1
+; CHECK-I686-NEXT:    ucomiss %xmm1, %xmm0
+; CHECK-I686-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; CHECK-I686-NEXT:    jne .LBB17_3
+; CHECK-I686-NEXT:  # %bb.1:
+; CHECK-I686-NEXT:    jp .LBB17_3
+; CHECK-I686-NEXT:  # %bb.2:
+; CHECK-I686-NEXT:    xorps %xmm0, %xmm0
+; CHECK-I686-NEXT:  .LBB17_3:
+; CHECK-I686-NEXT:    movss %xmm0, {{[0-9]+}}(%esp)
+; CHECK-I686-NEXT:    flds {{[0-9]+}}(%esp)
+; CHECK-I686-NEXT:    addl $12, %esp
+; CHECK-I686-NEXT:    retl
+  %2 = fcmp une half %0, 0xH0000
+  %3 = uitofp i1 %2 to half
+  ret half %3
+}
+
 attributes #0 = { nounwind }
diff --git a/test/CodeGen/X86/hidden-vis-pic.ll b/test/CodeGen/X86/hidden-vis-pic.ll
index 23bdb843f5ccf2d091b96fbab62f940cef5f8db3..6cc41b88404b0cafaaf645178dd14451811377d1 100644
--- a/test/CodeGen/X86/hidden-vis-pic.ll
+++ b/test/CodeGen/X86/hidden-vis-pic.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -mtriple=i386-apple-darwin9 -relocation-model=pic -disable-fp-elim | FileCheck %s
+; RUN: llc < %s -mtriple=i386-apple-darwin9 -relocation-model=pic -frame-pointer=all | FileCheck %s
 
 
 
diff --git a/test/CodeGen/X86/horizontal-reduce-smax.ll b/test/CodeGen/X86/horizontal-reduce-smax.ll
index 98fbd3620face30bed88f6dcf5b1239f2e9f2fe8..55bac985d6874520a4bb64742551ce6fff48d5b3 100644
--- a/test/CodeGen/X86/horizontal-reduce-smax.ll
+++ b/test/CodeGen/X86/horizontal-reduce-smax.ll
@@ -211,21 +211,19 @@ define i16 @test_reduce_v8i16(<8 x i16> %a0) {
 ;
 ; X86-SSE42-LABEL: test_reduce_v8i16:
 ; X86-SSE42:       ## %bb.0:
-; X86-SSE42-NEXT:    movdqa {{.*#+}} xmm1 = [32767,32767,32767,32767,32767,32767,32767,32767]
-; X86-SSE42-NEXT:    pxor %xmm1, %xmm0
+; X86-SSE42-NEXT:    pxor LCPI2_0, %xmm0
 ; X86-SSE42-NEXT:    phminposuw %xmm0, %xmm0
-; X86-SSE42-NEXT:    pxor %xmm1, %xmm0
 ; X86-SSE42-NEXT:    movd %xmm0, %eax
+; X86-SSE42-NEXT:    xorl $32767, %eax ## imm = 0x7FFF
 ; X86-SSE42-NEXT:    ## kill: def $ax killed $ax killed $eax
 ; X86-SSE42-NEXT:    retl
 ;
 ; X86-AVX-LABEL: test_reduce_v8i16:
 ; X86-AVX:       ## %bb.0:
-; X86-AVX-NEXT:    vmovdqa {{.*#+}} xmm1 = [32767,32767,32767,32767,32767,32767,32767,32767]
-; X86-AVX-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; X86-AVX-NEXT:    vpxor LCPI2_0, %xmm0, %xmm0
 ; X86-AVX-NEXT:    vphminposuw %xmm0, %xmm0
-; X86-AVX-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; X86-AVX-NEXT:    vmovd %xmm0, %eax
+; X86-AVX-NEXT:    xorl $32767, %eax ## imm = 0x7FFF
 ; X86-AVX-NEXT:    ## kill: def $ax killed $ax killed $eax
 ; X86-AVX-NEXT:    retl
 ;
@@ -244,21 +242,19 @@ define i16 @test_reduce_v8i16(<8 x i16> %a0) {
 ;
 ; X64-SSE42-LABEL: test_reduce_v8i16:
 ; X64-SSE42:       ## %bb.0:
-; X64-SSE42-NEXT:    movdqa {{.*#+}} xmm1 = [32767,32767,32767,32767,32767,32767,32767,32767]
-; X64-SSE42-NEXT:    pxor %xmm1, %xmm0
+; X64-SSE42-NEXT:    pxor {{.*}}(%rip), %xmm0
 ; X64-SSE42-NEXT:    phminposuw %xmm0, %xmm0
-; X64-SSE42-NEXT:    pxor %xmm1, %xmm0
 ; X64-SSE42-NEXT:    movd %xmm0, %eax
+; X64-SSE42-NEXT:    xorl $32767, %eax ## imm = 0x7FFF
 ; X64-SSE42-NEXT:    ## kill: def $ax killed $ax killed $eax
 ; X64-SSE42-NEXT:    retq
 ;
 ; X64-AVX-LABEL: test_reduce_v8i16:
 ; X64-AVX:       ## %bb.0:
-; X64-AVX-NEXT:    vmovdqa {{.*#+}} xmm1 = [32767,32767,32767,32767,32767,32767,32767,32767]
-; X64-AVX-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; X64-AVX-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
 ; X64-AVX-NEXT:    vphminposuw %xmm0, %xmm0
-; X64-AVX-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; X64-AVX-NEXT:    vmovd %xmm0, %eax
+; X64-AVX-NEXT:    xorl $32767, %eax ## imm = 0x7FFF
 ; X64-AVX-NEXT:    ## kill: def $ax killed $ax killed $eax
 ; X64-AVX-NEXT:    retq
   %1  = shufflevector <8 x i16> %a0, <8 x i16> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
@@ -309,26 +305,24 @@ define i8 @test_reduce_v16i8(<16 x i8> %a0) {
 ;
 ; X86-SSE42-LABEL: test_reduce_v16i8:
 ; X86-SSE42:       ## %bb.0:
-; X86-SSE42-NEXT:    movdqa {{.*#+}} xmm1 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
-; X86-SSE42-NEXT:    pxor %xmm1, %xmm0
-; X86-SSE42-NEXT:    movdqa %xmm0, %xmm2
-; X86-SSE42-NEXT:    psrlw $8, %xmm2
-; X86-SSE42-NEXT:    pminub %xmm0, %xmm2
-; X86-SSE42-NEXT:    phminposuw %xmm2, %xmm0
-; X86-SSE42-NEXT:    pxor %xmm1, %xmm0
+; X86-SSE42-NEXT:    pxor LCPI3_0, %xmm0
+; X86-SSE42-NEXT:    movdqa %xmm0, %xmm1
+; X86-SSE42-NEXT:    psrlw $8, %xmm1
+; X86-SSE42-NEXT:    pminub %xmm0, %xmm1
+; X86-SSE42-NEXT:    phminposuw %xmm1, %xmm0
 ; X86-SSE42-NEXT:    pextrb $0, %xmm0, %eax
+; X86-SSE42-NEXT:    xorb $127, %al
 ; X86-SSE42-NEXT:    ## kill: def $al killed $al killed $eax
 ; X86-SSE42-NEXT:    retl
 ;
 ; X86-AVX-LABEL: test_reduce_v16i8:
 ; X86-AVX:       ## %bb.0:
-; X86-AVX-NEXT:    vmovdqa {{.*#+}} xmm1 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
-; X86-AVX-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; X86-AVX-NEXT:    vpsrlw $8, %xmm0, %xmm2
-; X86-AVX-NEXT:    vpminub %xmm2, %xmm0, %xmm0
+; X86-AVX-NEXT:    vpxor LCPI3_0, %xmm0, %xmm0
+; X86-AVX-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; X86-AVX-NEXT:    vpminub %xmm1, %xmm0, %xmm0
 ; X86-AVX-NEXT:    vphminposuw %xmm0, %xmm0
-; X86-AVX-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; X86-AVX-NEXT:    vpextrb $0, %xmm0, %eax
+; X86-AVX-NEXT:    xorb $127, %al
 ; X86-AVX-NEXT:    ## kill: def $al killed $al killed $eax
 ; X86-AVX-NEXT:    retl
 ;
@@ -366,26 +360,24 @@ define i8 @test_reduce_v16i8(<16 x i8> %a0) {
 ;
 ; X64-SSE42-LABEL: test_reduce_v16i8:
 ; X64-SSE42:       ## %bb.0:
-; X64-SSE42-NEXT:    movdqa {{.*#+}} xmm1 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
-; X64-SSE42-NEXT:    pxor %xmm1, %xmm0
-; X64-SSE42-NEXT:    movdqa %xmm0, %xmm2
-; X64-SSE42-NEXT:    psrlw $8, %xmm2
-; X64-SSE42-NEXT:    pminub %xmm0, %xmm2
-; X64-SSE42-NEXT:    phminposuw %xmm2, %xmm0
-; X64-SSE42-NEXT:    pxor %xmm1, %xmm0
+; X64-SSE42-NEXT:    pxor {{.*}}(%rip), %xmm0
+; X64-SSE42-NEXT:    movdqa %xmm0, %xmm1
+; X64-SSE42-NEXT:    psrlw $8, %xmm1
+; X64-SSE42-NEXT:    pminub %xmm0, %xmm1
+; X64-SSE42-NEXT:    phminposuw %xmm1, %xmm0
 ; X64-SSE42-NEXT:    pextrb $0, %xmm0, %eax
+; X64-SSE42-NEXT:    xorb $127, %al
 ; X64-SSE42-NEXT:    ## kill: def $al killed $al killed $eax
 ; X64-SSE42-NEXT:    retq
 ;
 ; X64-AVX-LABEL: test_reduce_v16i8:
 ; X64-AVX:       ## %bb.0:
-; X64-AVX-NEXT:    vmovdqa {{.*#+}} xmm1 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
-; X64-AVX-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; X64-AVX-NEXT:    vpsrlw $8, %xmm0, %xmm2
-; X64-AVX-NEXT:    vpminub %xmm2, %xmm0, %xmm0
+; X64-AVX-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
+; X64-AVX-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; X64-AVX-NEXT:    vpminub %xmm1, %xmm0, %xmm0
 ; X64-AVX-NEXT:    vphminposuw %xmm0, %xmm0
-; X64-AVX-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; X64-AVX-NEXT:    vpextrb $0, %xmm0, %eax
+; X64-AVX-NEXT:    xorb $127, %al
 ; X64-AVX-NEXT:    ## kill: def $al killed $al killed $eax
 ; X64-AVX-NEXT:    retq
   %1  = shufflevector <16 x i8> %a0, <16 x i8> undef, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
@@ -736,11 +728,10 @@ define i16 @test_reduce_v16i16(<16 x i16> %a0) {
 ; X86-SSE42-LABEL: test_reduce_v16i16:
 ; X86-SSE42:       ## %bb.0:
 ; X86-SSE42-NEXT:    pmaxsw %xmm1, %xmm0
-; X86-SSE42-NEXT:    movdqa {{.*#+}} xmm1 = [32767,32767,32767,32767,32767,32767,32767,32767]
-; X86-SSE42-NEXT:    pxor %xmm1, %xmm0
+; X86-SSE42-NEXT:    pxor LCPI6_0, %xmm0
 ; X86-SSE42-NEXT:    phminposuw %xmm0, %xmm0
-; X86-SSE42-NEXT:    pxor %xmm1, %xmm0
 ; X86-SSE42-NEXT:    movd %xmm0, %eax
+; X86-SSE42-NEXT:    xorl $32767, %eax ## imm = 0x7FFF
 ; X86-SSE42-NEXT:    ## kill: def $ax killed $ax killed $eax
 ; X86-SSE42-NEXT:    retl
 ;
@@ -748,11 +739,10 @@ define i16 @test_reduce_v16i16(<16 x i16> %a0) {
 ; X86-AVX1:       ## %bb.0:
 ; X86-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; X86-AVX1-NEXT:    vpmaxsw %xmm1, %xmm0, %xmm0
-; X86-AVX1-NEXT:    vmovdqa {{.*#+}} xmm1 = [32767,32767,32767,32767,32767,32767,32767,32767]
-; X86-AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vpxor LCPI6_0, %xmm0, %xmm0
 ; X86-AVX1-NEXT:    vphminposuw %xmm0, %xmm0
-; X86-AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; X86-AVX1-NEXT:    vmovd %xmm0, %eax
+; X86-AVX1-NEXT:    xorl $32767, %eax ## imm = 0x7FFF
 ; X86-AVX1-NEXT:    ## kill: def $ax killed $ax killed $eax
 ; X86-AVX1-NEXT:    vzeroupper
 ; X86-AVX1-NEXT:    retl
@@ -761,11 +751,10 @@ define i16 @test_reduce_v16i16(<16 x i16> %a0) {
 ; X86-AVX2:       ## %bb.0:
 ; X86-AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; X86-AVX2-NEXT:    vpmaxsw %xmm1, %xmm0, %xmm0
-; X86-AVX2-NEXT:    vmovdqa {{.*#+}} xmm1 = [32767,32767,32767,32767,32767,32767,32767,32767]
-; X86-AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; X86-AVX2-NEXT:    vpxor LCPI6_0, %xmm0, %xmm0
 ; X86-AVX2-NEXT:    vphminposuw %xmm0, %xmm0
-; X86-AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; X86-AVX2-NEXT:    vmovd %xmm0, %eax
+; X86-AVX2-NEXT:    xorl $32767, %eax ## imm = 0x7FFF
 ; X86-AVX2-NEXT:    ## kill: def $ax killed $ax killed $eax
 ; X86-AVX2-NEXT:    vzeroupper
 ; X86-AVX2-NEXT:    retl
@@ -787,11 +776,10 @@ define i16 @test_reduce_v16i16(<16 x i16> %a0) {
 ; X64-SSE42-LABEL: test_reduce_v16i16:
 ; X64-SSE42:       ## %bb.0:
 ; X64-SSE42-NEXT:    pmaxsw %xmm1, %xmm0
-; X64-SSE42-NEXT:    movdqa {{.*#+}} xmm1 = [32767,32767,32767,32767,32767,32767,32767,32767]
-; X64-SSE42-NEXT:    pxor %xmm1, %xmm0
+; X64-SSE42-NEXT:    pxor {{.*}}(%rip), %xmm0
 ; X64-SSE42-NEXT:    phminposuw %xmm0, %xmm0
-; X64-SSE42-NEXT:    pxor %xmm1, %xmm0
 ; X64-SSE42-NEXT:    movd %xmm0, %eax
+; X64-SSE42-NEXT:    xorl $32767, %eax ## imm = 0x7FFF
 ; X64-SSE42-NEXT:    ## kill: def $ax killed $ax killed $eax
 ; X64-SSE42-NEXT:    retq
 ;
@@ -799,11 +787,10 @@ define i16 @test_reduce_v16i16(<16 x i16> %a0) {
 ; X64-AVX1:       ## %bb.0:
 ; X64-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; X64-AVX1-NEXT:    vpmaxsw %xmm1, %xmm0, %xmm0
-; X64-AVX1-NEXT:    vmovdqa {{.*#+}} xmm1 = [32767,32767,32767,32767,32767,32767,32767,32767]
-; X64-AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; X64-AVX1-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
 ; X64-AVX1-NEXT:    vphminposuw %xmm0, %xmm0
-; X64-AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; X64-AVX1-NEXT:    vmovd %xmm0, %eax
+; X64-AVX1-NEXT:    xorl $32767, %eax ## imm = 0x7FFF
 ; X64-AVX1-NEXT:    ## kill: def $ax killed $ax killed $eax
 ; X64-AVX1-NEXT:    vzeroupper
 ; X64-AVX1-NEXT:    retq
@@ -812,11 +799,10 @@ define i16 @test_reduce_v16i16(<16 x i16> %a0) {
 ; X64-AVX2:       ## %bb.0:
 ; X64-AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; X64-AVX2-NEXT:    vpmaxsw %xmm1, %xmm0, %xmm0
-; X64-AVX2-NEXT:    vmovdqa {{.*#+}} xmm1 = [32767,32767,32767,32767,32767,32767,32767,32767]
-; X64-AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; X64-AVX2-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
 ; X64-AVX2-NEXT:    vphminposuw %xmm0, %xmm0
-; X64-AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; X64-AVX2-NEXT:    vmovd %xmm0, %eax
+; X64-AVX2-NEXT:    xorl $32767, %eax ## imm = 0x7FFF
 ; X64-AVX2-NEXT:    ## kill: def $ax killed $ax killed $eax
 ; X64-AVX2-NEXT:    vzeroupper
 ; X64-AVX2-NEXT:    retq
@@ -825,11 +811,10 @@ define i16 @test_reduce_v16i16(<16 x i16> %a0) {
 ; X64-AVX512:       ## %bb.0:
 ; X64-AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; X64-AVX512-NEXT:    vpmaxsw %xmm1, %xmm0, %xmm0
-; X64-AVX512-NEXT:    vmovdqa {{.*#+}} xmm1 = [32767,32767,32767,32767,32767,32767,32767,32767]
-; X64-AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; X64-AVX512-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
 ; X64-AVX512-NEXT:    vphminposuw %xmm0, %xmm0
-; X64-AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; X64-AVX512-NEXT:    vmovd %xmm0, %eax
+; X64-AVX512-NEXT:    xorl $32767, %eax ## imm = 0x7FFF
 ; X64-AVX512-NEXT:    ## kill: def $ax killed $ax killed $eax
 ; X64-AVX512-NEXT:    vzeroupper
 ; X64-AVX512-NEXT:    retq
@@ -890,14 +875,13 @@ define i8 @test_reduce_v32i8(<32 x i8> %a0) {
 ; X86-SSE42-LABEL: test_reduce_v32i8:
 ; X86-SSE42:       ## %bb.0:
 ; X86-SSE42-NEXT:    pmaxsb %xmm1, %xmm0
-; X86-SSE42-NEXT:    movdqa {{.*#+}} xmm1 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
-; X86-SSE42-NEXT:    pxor %xmm1, %xmm0
-; X86-SSE42-NEXT:    movdqa %xmm0, %xmm2
-; X86-SSE42-NEXT:    psrlw $8, %xmm2
-; X86-SSE42-NEXT:    pminub %xmm0, %xmm2
-; X86-SSE42-NEXT:    phminposuw %xmm2, %xmm0
-; X86-SSE42-NEXT:    pxor %xmm1, %xmm0
+; X86-SSE42-NEXT:    pxor LCPI7_0, %xmm0
+; X86-SSE42-NEXT:    movdqa %xmm0, %xmm1
+; X86-SSE42-NEXT:    psrlw $8, %xmm1
+; X86-SSE42-NEXT:    pminub %xmm0, %xmm1
+; X86-SSE42-NEXT:    phminposuw %xmm1, %xmm0
 ; X86-SSE42-NEXT:    pextrb $0, %xmm0, %eax
+; X86-SSE42-NEXT:    xorb $127, %al
 ; X86-SSE42-NEXT:    ## kill: def $al killed $al killed $eax
 ; X86-SSE42-NEXT:    retl
 ;
@@ -905,13 +889,12 @@ define i8 @test_reduce_v32i8(<32 x i8> %a0) {
 ; X86-AVX1:       ## %bb.0:
 ; X86-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; X86-AVX1-NEXT:    vpmaxsb %xmm1, %xmm0, %xmm0
-; X86-AVX1-NEXT:    vmovdqa {{.*#+}} xmm1 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
-; X86-AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; X86-AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm2
-; X86-AVX1-NEXT:    vpminub %xmm2, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vpxor LCPI7_0, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; X86-AVX1-NEXT:    vpminub %xmm1, %xmm0, %xmm0
 ; X86-AVX1-NEXT:    vphminposuw %xmm0, %xmm0
-; X86-AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; X86-AVX1-NEXT:    vpextrb $0, %xmm0, %eax
+; X86-AVX1-NEXT:    xorb $127, %al
 ; X86-AVX1-NEXT:    ## kill: def $al killed $al killed $eax
 ; X86-AVX1-NEXT:    vzeroupper
 ; X86-AVX1-NEXT:    retl
@@ -920,13 +903,12 @@ define i8 @test_reduce_v32i8(<32 x i8> %a0) {
 ; X86-AVX2:       ## %bb.0:
 ; X86-AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; X86-AVX2-NEXT:    vpmaxsb %xmm1, %xmm0, %xmm0
-; X86-AVX2-NEXT:    vmovdqa {{.*#+}} xmm1 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
-; X86-AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; X86-AVX2-NEXT:    vpsrlw $8, %xmm0, %xmm2
-; X86-AVX2-NEXT:    vpminub %xmm2, %xmm0, %xmm0
+; X86-AVX2-NEXT:    vpxor LCPI7_0, %xmm0, %xmm0
+; X86-AVX2-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; X86-AVX2-NEXT:    vpminub %xmm1, %xmm0, %xmm0
 ; X86-AVX2-NEXT:    vphminposuw %xmm0, %xmm0
-; X86-AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; X86-AVX2-NEXT:    vpextrb $0, %xmm0, %eax
+; X86-AVX2-NEXT:    xorb $127, %al
 ; X86-AVX2-NEXT:    ## kill: def $al killed $al killed $eax
 ; X86-AVX2-NEXT:    vzeroupper
 ; X86-AVX2-NEXT:    retl
@@ -971,14 +953,13 @@ define i8 @test_reduce_v32i8(<32 x i8> %a0) {
 ; X64-SSE42-LABEL: test_reduce_v32i8:
 ; X64-SSE42:       ## %bb.0:
 ; X64-SSE42-NEXT:    pmaxsb %xmm1, %xmm0
-; X64-SSE42-NEXT:    movdqa {{.*#+}} xmm1 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
-; X64-SSE42-NEXT:    pxor %xmm1, %xmm0
-; X64-SSE42-NEXT:    movdqa %xmm0, %xmm2
-; X64-SSE42-NEXT:    psrlw $8, %xmm2
-; X64-SSE42-NEXT:    pminub %xmm0, %xmm2
-; X64-SSE42-NEXT:    phminposuw %xmm2, %xmm0
-; X64-SSE42-NEXT:    pxor %xmm1, %xmm0
+; X64-SSE42-NEXT:    pxor {{.*}}(%rip), %xmm0
+; X64-SSE42-NEXT:    movdqa %xmm0, %xmm1
+; X64-SSE42-NEXT:    psrlw $8, %xmm1
+; X64-SSE42-NEXT:    pminub %xmm0, %xmm1
+; X64-SSE42-NEXT:    phminposuw %xmm1, %xmm0
 ; X64-SSE42-NEXT:    pextrb $0, %xmm0, %eax
+; X64-SSE42-NEXT:    xorb $127, %al
 ; X64-SSE42-NEXT:    ## kill: def $al killed $al killed $eax
 ; X64-SSE42-NEXT:    retq
 ;
@@ -986,13 +967,12 @@ define i8 @test_reduce_v32i8(<32 x i8> %a0) {
 ; X64-AVX1:       ## %bb.0:
 ; X64-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; X64-AVX1-NEXT:    vpmaxsb %xmm1, %xmm0, %xmm0
-; X64-AVX1-NEXT:    vmovdqa {{.*#+}} xmm1 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
-; X64-AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; X64-AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm2
-; X64-AVX1-NEXT:    vpminub %xmm2, %xmm0, %xmm0
+; X64-AVX1-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
+; X64-AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; X64-AVX1-NEXT:    vpminub %xmm1, %xmm0, %xmm0
 ; X64-AVX1-NEXT:    vphminposuw %xmm0, %xmm0
-; X64-AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; X64-AVX1-NEXT:    vpextrb $0, %xmm0, %eax
+; X64-AVX1-NEXT:    xorb $127, %al
 ; X64-AVX1-NEXT:    ## kill: def $al killed $al killed $eax
 ; X64-AVX1-NEXT:    vzeroupper
 ; X64-AVX1-NEXT:    retq
@@ -1001,13 +981,12 @@ define i8 @test_reduce_v32i8(<32 x i8> %a0) {
 ; X64-AVX2:       ## %bb.0:
 ; X64-AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; X64-AVX2-NEXT:    vpmaxsb %xmm1, %xmm0, %xmm0
-; X64-AVX2-NEXT:    vmovdqa {{.*#+}} xmm1 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
-; X64-AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; X64-AVX2-NEXT:    vpsrlw $8, %xmm0, %xmm2
-; X64-AVX2-NEXT:    vpminub %xmm2, %xmm0, %xmm0
+; X64-AVX2-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
+; X64-AVX2-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; X64-AVX2-NEXT:    vpminub %xmm1, %xmm0, %xmm0
 ; X64-AVX2-NEXT:    vphminposuw %xmm0, %xmm0
-; X64-AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; X64-AVX2-NEXT:    vpextrb $0, %xmm0, %eax
+; X64-AVX2-NEXT:    xorb $127, %al
 ; X64-AVX2-NEXT:    ## kill: def $al killed $al killed $eax
 ; X64-AVX2-NEXT:    vzeroupper
 ; X64-AVX2-NEXT:    retq
@@ -1016,13 +995,12 @@ define i8 @test_reduce_v32i8(<32 x i8> %a0) {
 ; X64-AVX512:       ## %bb.0:
 ; X64-AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; X64-AVX512-NEXT:    vpmaxsb %xmm1, %xmm0, %xmm0
-; X64-AVX512-NEXT:    vmovdqa {{.*#+}} xmm1 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
-; X64-AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; X64-AVX512-NEXT:    vpsrlw $8, %xmm0, %xmm2
-; X64-AVX512-NEXT:    vpminub %xmm2, %xmm0, %xmm0
+; X64-AVX512-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
+; X64-AVX512-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; X64-AVX512-NEXT:    vpminub %xmm1, %xmm0, %xmm0
 ; X64-AVX512-NEXT:    vphminposuw %xmm0, %xmm0
-; X64-AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; X64-AVX512-NEXT:    vpextrb $0, %xmm0, %eax
+; X64-AVX512-NEXT:    xorb $127, %al
 ; X64-AVX512-NEXT:    ## kill: def $al killed $al killed $eax
 ; X64-AVX512-NEXT:    vzeroupper
 ; X64-AVX512-NEXT:    retq
@@ -1513,11 +1491,10 @@ define i16 @test_reduce_v32i16(<32 x i16> %a0) {
 ; X86-SSE42-NEXT:    pmaxsw %xmm3, %xmm1
 ; X86-SSE42-NEXT:    pmaxsw %xmm2, %xmm0
 ; X86-SSE42-NEXT:    pmaxsw %xmm1, %xmm0
-; X86-SSE42-NEXT:    movdqa {{.*#+}} xmm1 = [32767,32767,32767,32767,32767,32767,32767,32767]
-; X86-SSE42-NEXT:    pxor %xmm1, %xmm0
+; X86-SSE42-NEXT:    pxor LCPI10_0, %xmm0
 ; X86-SSE42-NEXT:    phminposuw %xmm0, %xmm0
-; X86-SSE42-NEXT:    pxor %xmm1, %xmm0
 ; X86-SSE42-NEXT:    movd %xmm0, %eax
+; X86-SSE42-NEXT:    xorl $32767, %eax ## imm = 0x7FFF
 ; X86-SSE42-NEXT:    ## kill: def $ax killed $ax killed $eax
 ; X86-SSE42-NEXT:    retl
 ;
@@ -1528,11 +1505,10 @@ define i16 @test_reduce_v32i16(<32 x i16> %a0) {
 ; X86-AVX1-NEXT:    vpmaxsw %xmm2, %xmm3, %xmm2
 ; X86-AVX1-NEXT:    vpmaxsw %xmm1, %xmm0, %xmm0
 ; X86-AVX1-NEXT:    vpmaxsw %xmm2, %xmm0, %xmm0
-; X86-AVX1-NEXT:    vmovdqa {{.*#+}} xmm1 = [32767,32767,32767,32767,32767,32767,32767,32767]
-; X86-AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vpxor LCPI10_0, %xmm0, %xmm0
 ; X86-AVX1-NEXT:    vphminposuw %xmm0, %xmm0
-; X86-AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; X86-AVX1-NEXT:    vmovd %xmm0, %eax
+; X86-AVX1-NEXT:    xorl $32767, %eax ## imm = 0x7FFF
 ; X86-AVX1-NEXT:    ## kill: def $ax killed $ax killed $eax
 ; X86-AVX1-NEXT:    vzeroupper
 ; X86-AVX1-NEXT:    retl
@@ -1542,11 +1518,10 @@ define i16 @test_reduce_v32i16(<32 x i16> %a0) {
 ; X86-AVX2-NEXT:    vpmaxsw %ymm1, %ymm0, %ymm0
 ; X86-AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; X86-AVX2-NEXT:    vpmaxsw %xmm1, %xmm0, %xmm0
-; X86-AVX2-NEXT:    vmovdqa {{.*#+}} xmm1 = [32767,32767,32767,32767,32767,32767,32767,32767]
-; X86-AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; X86-AVX2-NEXT:    vpxor LCPI10_0, %xmm0, %xmm0
 ; X86-AVX2-NEXT:    vphminposuw %xmm0, %xmm0
-; X86-AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; X86-AVX2-NEXT:    vmovd %xmm0, %eax
+; X86-AVX2-NEXT:    xorl $32767, %eax ## imm = 0x7FFF
 ; X86-AVX2-NEXT:    ## kill: def $ax killed $ax killed $eax
 ; X86-AVX2-NEXT:    vzeroupper
 ; X86-AVX2-NEXT:    retl
@@ -1572,11 +1547,10 @@ define i16 @test_reduce_v32i16(<32 x i16> %a0) {
 ; X64-SSE42-NEXT:    pmaxsw %xmm3, %xmm1
 ; X64-SSE42-NEXT:    pmaxsw %xmm2, %xmm0
 ; X64-SSE42-NEXT:    pmaxsw %xmm1, %xmm0
-; X64-SSE42-NEXT:    movdqa {{.*#+}} xmm1 = [32767,32767,32767,32767,32767,32767,32767,32767]
-; X64-SSE42-NEXT:    pxor %xmm1, %xmm0
+; X64-SSE42-NEXT:    pxor {{.*}}(%rip), %xmm0
 ; X64-SSE42-NEXT:    phminposuw %xmm0, %xmm0
-; X64-SSE42-NEXT:    pxor %xmm1, %xmm0
 ; X64-SSE42-NEXT:    movd %xmm0, %eax
+; X64-SSE42-NEXT:    xorl $32767, %eax ## imm = 0x7FFF
 ; X64-SSE42-NEXT:    ## kill: def $ax killed $ax killed $eax
 ; X64-SSE42-NEXT:    retq
 ;
@@ -1587,11 +1561,10 @@ define i16 @test_reduce_v32i16(<32 x i16> %a0) {
 ; X64-AVX1-NEXT:    vpmaxsw %xmm2, %xmm3, %xmm2
 ; X64-AVX1-NEXT:    vpmaxsw %xmm1, %xmm0, %xmm0
 ; X64-AVX1-NEXT:    vpmaxsw %xmm2, %xmm0, %xmm0
-; X64-AVX1-NEXT:    vmovdqa {{.*#+}} xmm1 = [32767,32767,32767,32767,32767,32767,32767,32767]
-; X64-AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; X64-AVX1-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
 ; X64-AVX1-NEXT:    vphminposuw %xmm0, %xmm0
-; X64-AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; X64-AVX1-NEXT:    vmovd %xmm0, %eax
+; X64-AVX1-NEXT:    xorl $32767, %eax ## imm = 0x7FFF
 ; X64-AVX1-NEXT:    ## kill: def $ax killed $ax killed $eax
 ; X64-AVX1-NEXT:    vzeroupper
 ; X64-AVX1-NEXT:    retq
@@ -1601,11 +1574,10 @@ define i16 @test_reduce_v32i16(<32 x i16> %a0) {
 ; X64-AVX2-NEXT:    vpmaxsw %ymm1, %ymm0, %ymm0
 ; X64-AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; X64-AVX2-NEXT:    vpmaxsw %xmm1, %xmm0, %xmm0
-; X64-AVX2-NEXT:    vmovdqa {{.*#+}} xmm1 = [32767,32767,32767,32767,32767,32767,32767,32767]
-; X64-AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; X64-AVX2-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
 ; X64-AVX2-NEXT:    vphminposuw %xmm0, %xmm0
-; X64-AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; X64-AVX2-NEXT:    vmovd %xmm0, %eax
+; X64-AVX2-NEXT:    xorl $32767, %eax ## imm = 0x7FFF
 ; X64-AVX2-NEXT:    ## kill: def $ax killed $ax killed $eax
 ; X64-AVX2-NEXT:    vzeroupper
 ; X64-AVX2-NEXT:    retq
@@ -1616,11 +1588,10 @@ define i16 @test_reduce_v32i16(<32 x i16> %a0) {
 ; X64-AVX512-NEXT:    vpmaxsw %ymm1, %ymm0, %ymm0
 ; X64-AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; X64-AVX512-NEXT:    vpmaxsw %xmm1, %xmm0, %xmm0
-; X64-AVX512-NEXT:    vmovdqa {{.*#+}} xmm1 = [32767,32767,32767,32767,32767,32767,32767,32767]
-; X64-AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; X64-AVX512-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
 ; X64-AVX512-NEXT:    vphminposuw %xmm0, %xmm0
-; X64-AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; X64-AVX512-NEXT:    vmovd %xmm0, %eax
+; X64-AVX512-NEXT:    xorl $32767, %eax ## imm = 0x7FFF
 ; X64-AVX512-NEXT:    ## kill: def $ax killed $ax killed $eax
 ; X64-AVX512-NEXT:    vzeroupper
 ; X64-AVX512-NEXT:    retq
@@ -1696,14 +1667,13 @@ define i8 @test_reduce_v64i8(<64 x i8> %a0) {
 ; X86-SSE42-NEXT:    pmaxsb %xmm3, %xmm1
 ; X86-SSE42-NEXT:    pmaxsb %xmm2, %xmm0
 ; X86-SSE42-NEXT:    pmaxsb %xmm1, %xmm0
-; X86-SSE42-NEXT:    movdqa {{.*#+}} xmm1 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
-; X86-SSE42-NEXT:    pxor %xmm1, %xmm0
-; X86-SSE42-NEXT:    movdqa %xmm0, %xmm2
-; X86-SSE42-NEXT:    psrlw $8, %xmm2
-; X86-SSE42-NEXT:    pminub %xmm0, %xmm2
-; X86-SSE42-NEXT:    phminposuw %xmm2, %xmm0
-; X86-SSE42-NEXT:    pxor %xmm1, %xmm0
+; X86-SSE42-NEXT:    pxor LCPI11_0, %xmm0
+; X86-SSE42-NEXT:    movdqa %xmm0, %xmm1
+; X86-SSE42-NEXT:    psrlw $8, %xmm1
+; X86-SSE42-NEXT:    pminub %xmm0, %xmm1
+; X86-SSE42-NEXT:    phminposuw %xmm1, %xmm0
 ; X86-SSE42-NEXT:    pextrb $0, %xmm0, %eax
+; X86-SSE42-NEXT:    xorb $127, %al
 ; X86-SSE42-NEXT:    ## kill: def $al killed $al killed $eax
 ; X86-SSE42-NEXT:    retl
 ;
@@ -1714,13 +1684,12 @@ define i8 @test_reduce_v64i8(<64 x i8> %a0) {
 ; X86-AVX1-NEXT:    vpmaxsb %xmm2, %xmm3, %xmm2
 ; X86-AVX1-NEXT:    vpmaxsb %xmm1, %xmm0, %xmm0
 ; X86-AVX1-NEXT:    vpmaxsb %xmm2, %xmm0, %xmm0
-; X86-AVX1-NEXT:    vmovdqa {{.*#+}} xmm1 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
-; X86-AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; X86-AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm2
-; X86-AVX1-NEXT:    vpminub %xmm2, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vpxor LCPI11_0, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; X86-AVX1-NEXT:    vpminub %xmm1, %xmm0, %xmm0
 ; X86-AVX1-NEXT:    vphminposuw %xmm0, %xmm0
-; X86-AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; X86-AVX1-NEXT:    vpextrb $0, %xmm0, %eax
+; X86-AVX1-NEXT:    xorb $127, %al
 ; X86-AVX1-NEXT:    ## kill: def $al killed $al killed $eax
 ; X86-AVX1-NEXT:    vzeroupper
 ; X86-AVX1-NEXT:    retl
@@ -1730,13 +1699,12 @@ define i8 @test_reduce_v64i8(<64 x i8> %a0) {
 ; X86-AVX2-NEXT:    vpmaxsb %ymm1, %ymm0, %ymm0
 ; X86-AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; X86-AVX2-NEXT:    vpmaxsb %xmm1, %xmm0, %xmm0
-; X86-AVX2-NEXT:    vmovdqa {{.*#+}} xmm1 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
-; X86-AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; X86-AVX2-NEXT:    vpsrlw $8, %xmm0, %xmm2
-; X86-AVX2-NEXT:    vpminub %xmm2, %xmm0, %xmm0
+; X86-AVX2-NEXT:    vpxor LCPI11_0, %xmm0, %xmm0
+; X86-AVX2-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; X86-AVX2-NEXT:    vpminub %xmm1, %xmm0, %xmm0
 ; X86-AVX2-NEXT:    vphminposuw %xmm0, %xmm0
-; X86-AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; X86-AVX2-NEXT:    vpextrb $0, %xmm0, %eax
+; X86-AVX2-NEXT:    xorb $127, %al
 ; X86-AVX2-NEXT:    ## kill: def $al killed $al killed $eax
 ; X86-AVX2-NEXT:    vzeroupper
 ; X86-AVX2-NEXT:    retl
@@ -1793,14 +1761,13 @@ define i8 @test_reduce_v64i8(<64 x i8> %a0) {
 ; X64-SSE42-NEXT:    pmaxsb %xmm3, %xmm1
 ; X64-SSE42-NEXT:    pmaxsb %xmm2, %xmm0
 ; X64-SSE42-NEXT:    pmaxsb %xmm1, %xmm0
-; X64-SSE42-NEXT:    movdqa {{.*#+}} xmm1 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
-; X64-SSE42-NEXT:    pxor %xmm1, %xmm0
-; X64-SSE42-NEXT:    movdqa %xmm0, %xmm2
-; X64-SSE42-NEXT:    psrlw $8, %xmm2
-; X64-SSE42-NEXT:    pminub %xmm0, %xmm2
-; X64-SSE42-NEXT:    phminposuw %xmm2, %xmm0
-; X64-SSE42-NEXT:    pxor %xmm1, %xmm0
+; X64-SSE42-NEXT:    pxor {{.*}}(%rip), %xmm0
+; X64-SSE42-NEXT:    movdqa %xmm0, %xmm1
+; X64-SSE42-NEXT:    psrlw $8, %xmm1
+; X64-SSE42-NEXT:    pminub %xmm0, %xmm1
+; X64-SSE42-NEXT:    phminposuw %xmm1, %xmm0
 ; X64-SSE42-NEXT:    pextrb $0, %xmm0, %eax
+; X64-SSE42-NEXT:    xorb $127, %al
 ; X64-SSE42-NEXT:    ## kill: def $al killed $al killed $eax
 ; X64-SSE42-NEXT:    retq
 ;
@@ -1811,13 +1778,12 @@ define i8 @test_reduce_v64i8(<64 x i8> %a0) {
 ; X64-AVX1-NEXT:    vpmaxsb %xmm2, %xmm3, %xmm2
 ; X64-AVX1-NEXT:    vpmaxsb %xmm1, %xmm0, %xmm0
 ; X64-AVX1-NEXT:    vpmaxsb %xmm2, %xmm0, %xmm0
-; X64-AVX1-NEXT:    vmovdqa {{.*#+}} xmm1 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
-; X64-AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; X64-AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm2
-; X64-AVX1-NEXT:    vpminub %xmm2, %xmm0, %xmm0
+; X64-AVX1-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
+; X64-AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; X64-AVX1-NEXT:    vpminub %xmm1, %xmm0, %xmm0
 ; X64-AVX1-NEXT:    vphminposuw %xmm0, %xmm0
-; X64-AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; X64-AVX1-NEXT:    vpextrb $0, %xmm0, %eax
+; X64-AVX1-NEXT:    xorb $127, %al
 ; X64-AVX1-NEXT:    ## kill: def $al killed $al killed $eax
 ; X64-AVX1-NEXT:    vzeroupper
 ; X64-AVX1-NEXT:    retq
@@ -1827,13 +1793,12 @@ define i8 @test_reduce_v64i8(<64 x i8> %a0) {
 ; X64-AVX2-NEXT:    vpmaxsb %ymm1, %ymm0, %ymm0
 ; X64-AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; X64-AVX2-NEXT:    vpmaxsb %xmm1, %xmm0, %xmm0
-; X64-AVX2-NEXT:    vmovdqa {{.*#+}} xmm1 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
-; X64-AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; X64-AVX2-NEXT:    vpsrlw $8, %xmm0, %xmm2
-; X64-AVX2-NEXT:    vpminub %xmm2, %xmm0, %xmm0
+; X64-AVX2-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
+; X64-AVX2-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; X64-AVX2-NEXT:    vpminub %xmm1, %xmm0, %xmm0
 ; X64-AVX2-NEXT:    vphminposuw %xmm0, %xmm0
-; X64-AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; X64-AVX2-NEXT:    vpextrb $0, %xmm0, %eax
+; X64-AVX2-NEXT:    xorb $127, %al
 ; X64-AVX2-NEXT:    ## kill: def $al killed $al killed $eax
 ; X64-AVX2-NEXT:    vzeroupper
 ; X64-AVX2-NEXT:    retq
@@ -1844,13 +1809,12 @@ define i8 @test_reduce_v64i8(<64 x i8> %a0) {
 ; X64-AVX512-NEXT:    vpmaxsb %ymm1, %ymm0, %ymm0
 ; X64-AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; X64-AVX512-NEXT:    vpmaxsb %xmm1, %xmm0, %xmm0
-; X64-AVX512-NEXT:    vmovdqa {{.*#+}} xmm1 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
-; X64-AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; X64-AVX512-NEXT:    vpsrlw $8, %xmm0, %xmm2
-; X64-AVX512-NEXT:    vpminub %xmm2, %xmm0, %xmm0
+; X64-AVX512-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
+; X64-AVX512-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; X64-AVX512-NEXT:    vpminub %xmm1, %xmm0, %xmm0
 ; X64-AVX512-NEXT:    vphminposuw %xmm0, %xmm0
-; X64-AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; X64-AVX512-NEXT:    vpextrb $0, %xmm0, %eax
+; X64-AVX512-NEXT:    xorb $127, %al
 ; X64-AVX512-NEXT:    ## kill: def $al killed $al killed $eax
 ; X64-AVX512-NEXT:    vzeroupper
 ; X64-AVX512-NEXT:    retq
diff --git a/test/CodeGen/X86/horizontal-reduce-smin.ll b/test/CodeGen/X86/horizontal-reduce-smin.ll
index 3ac86148e85228fd8a79d5e294b871b793e59c37..7036d9375912317774131dc614fcc0bcb6020026 100644
--- a/test/CodeGen/X86/horizontal-reduce-smin.ll
+++ b/test/CodeGen/X86/horizontal-reduce-smin.ll
@@ -213,21 +213,19 @@ define i16 @test_reduce_v8i16(<8 x i16> %a0) {
 ;
 ; X86-SSE42-LABEL: test_reduce_v8i16:
 ; X86-SSE42:       ## %bb.0:
-; X86-SSE42-NEXT:    movdqa {{.*#+}} xmm1 = [32768,32768,32768,32768,32768,32768,32768,32768]
-; X86-SSE42-NEXT:    pxor %xmm1, %xmm0
+; X86-SSE42-NEXT:    pxor LCPI2_0, %xmm0
 ; X86-SSE42-NEXT:    phminposuw %xmm0, %xmm0
-; X86-SSE42-NEXT:    pxor %xmm1, %xmm0
 ; X86-SSE42-NEXT:    movd %xmm0, %eax
+; X86-SSE42-NEXT:    xorl $32768, %eax ## imm = 0x8000
 ; X86-SSE42-NEXT:    ## kill: def $ax killed $ax killed $eax
 ; X86-SSE42-NEXT:    retl
 ;
 ; X86-AVX-LABEL: test_reduce_v8i16:
 ; X86-AVX:       ## %bb.0:
-; X86-AVX-NEXT:    vmovdqa {{.*#+}} xmm1 = [32768,32768,32768,32768,32768,32768,32768,32768]
-; X86-AVX-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; X86-AVX-NEXT:    vpxor LCPI2_0, %xmm0, %xmm0
 ; X86-AVX-NEXT:    vphminposuw %xmm0, %xmm0
-; X86-AVX-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; X86-AVX-NEXT:    vmovd %xmm0, %eax
+; X86-AVX-NEXT:    xorl $32768, %eax ## imm = 0x8000
 ; X86-AVX-NEXT:    ## kill: def $ax killed $ax killed $eax
 ; X86-AVX-NEXT:    retl
 ;
@@ -246,21 +244,19 @@ define i16 @test_reduce_v8i16(<8 x i16> %a0) {
 ;
 ; X64-SSE42-LABEL: test_reduce_v8i16:
 ; X64-SSE42:       ## %bb.0:
-; X64-SSE42-NEXT:    movdqa {{.*#+}} xmm1 = [32768,32768,32768,32768,32768,32768,32768,32768]
-; X64-SSE42-NEXT:    pxor %xmm1, %xmm0
+; X64-SSE42-NEXT:    pxor {{.*}}(%rip), %xmm0
 ; X64-SSE42-NEXT:    phminposuw %xmm0, %xmm0
-; X64-SSE42-NEXT:    pxor %xmm1, %xmm0
 ; X64-SSE42-NEXT:    movd %xmm0, %eax
+; X64-SSE42-NEXT:    xorl $32768, %eax ## imm = 0x8000
 ; X64-SSE42-NEXT:    ## kill: def $ax killed $ax killed $eax
 ; X64-SSE42-NEXT:    retq
 ;
 ; X64-AVX-LABEL: test_reduce_v8i16:
 ; X64-AVX:       ## %bb.0:
-; X64-AVX-NEXT:    vmovdqa {{.*#+}} xmm1 = [32768,32768,32768,32768,32768,32768,32768,32768]
-; X64-AVX-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; X64-AVX-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
 ; X64-AVX-NEXT:    vphminposuw %xmm0, %xmm0
-; X64-AVX-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; X64-AVX-NEXT:    vmovd %xmm0, %eax
+; X64-AVX-NEXT:    xorl $32768, %eax ## imm = 0x8000
 ; X64-AVX-NEXT:    ## kill: def $ax killed $ax killed $eax
 ; X64-AVX-NEXT:    retq
   %1  = shufflevector <8 x i16> %a0, <8 x i16> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
@@ -311,26 +307,24 @@ define i8 @test_reduce_v16i8(<16 x i8> %a0) {
 ;
 ; X86-SSE42-LABEL: test_reduce_v16i8:
 ; X86-SSE42:       ## %bb.0:
-; X86-SSE42-NEXT:    movdqa {{.*#+}} xmm1 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128]
-; X86-SSE42-NEXT:    pxor %xmm1, %xmm0
-; X86-SSE42-NEXT:    movdqa %xmm0, %xmm2
-; X86-SSE42-NEXT:    psrlw $8, %xmm2
-; X86-SSE42-NEXT:    pminub %xmm0, %xmm2
-; X86-SSE42-NEXT:    phminposuw %xmm2, %xmm0
-; X86-SSE42-NEXT:    pxor %xmm1, %xmm0
+; X86-SSE42-NEXT:    pxor LCPI3_0, %xmm0
+; X86-SSE42-NEXT:    movdqa %xmm0, %xmm1
+; X86-SSE42-NEXT:    psrlw $8, %xmm1
+; X86-SSE42-NEXT:    pminub %xmm0, %xmm1
+; X86-SSE42-NEXT:    phminposuw %xmm1, %xmm0
 ; X86-SSE42-NEXT:    pextrb $0, %xmm0, %eax
+; X86-SSE42-NEXT:    xorb $-128, %al
 ; X86-SSE42-NEXT:    ## kill: def $al killed $al killed $eax
 ; X86-SSE42-NEXT:    retl
 ;
 ; X86-AVX-LABEL: test_reduce_v16i8:
 ; X86-AVX:       ## %bb.0:
-; X86-AVX-NEXT:    vmovdqa {{.*#+}} xmm1 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128]
-; X86-AVX-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; X86-AVX-NEXT:    vpsrlw $8, %xmm0, %xmm2
-; X86-AVX-NEXT:    vpminub %xmm2, %xmm0, %xmm0
+; X86-AVX-NEXT:    vpxor LCPI3_0, %xmm0, %xmm0
+; X86-AVX-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; X86-AVX-NEXT:    vpminub %xmm1, %xmm0, %xmm0
 ; X86-AVX-NEXT:    vphminposuw %xmm0, %xmm0
-; X86-AVX-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; X86-AVX-NEXT:    vpextrb $0, %xmm0, %eax
+; X86-AVX-NEXT:    xorb $-128, %al
 ; X86-AVX-NEXT:    ## kill: def $al killed $al killed $eax
 ; X86-AVX-NEXT:    retl
 ;
@@ -368,26 +362,24 @@ define i8 @test_reduce_v16i8(<16 x i8> %a0) {
 ;
 ; X64-SSE42-LABEL: test_reduce_v16i8:
 ; X64-SSE42:       ## %bb.0:
-; X64-SSE42-NEXT:    movdqa {{.*#+}} xmm1 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128]
-; X64-SSE42-NEXT:    pxor %xmm1, %xmm0
-; X64-SSE42-NEXT:    movdqa %xmm0, %xmm2
-; X64-SSE42-NEXT:    psrlw $8, %xmm2
-; X64-SSE42-NEXT:    pminub %xmm0, %xmm2
-; X64-SSE42-NEXT:    phminposuw %xmm2, %xmm0
-; X64-SSE42-NEXT:    pxor %xmm1, %xmm0
+; X64-SSE42-NEXT:    pxor {{.*}}(%rip), %xmm0
+; X64-SSE42-NEXT:    movdqa %xmm0, %xmm1
+; X64-SSE42-NEXT:    psrlw $8, %xmm1
+; X64-SSE42-NEXT:    pminub %xmm0, %xmm1
+; X64-SSE42-NEXT:    phminposuw %xmm1, %xmm0
 ; X64-SSE42-NEXT:    pextrb $0, %xmm0, %eax
+; X64-SSE42-NEXT:    xorb $-128, %al
 ; X64-SSE42-NEXT:    ## kill: def $al killed $al killed $eax
 ; X64-SSE42-NEXT:    retq
 ;
 ; X64-AVX-LABEL: test_reduce_v16i8:
 ; X64-AVX:       ## %bb.0:
-; X64-AVX-NEXT:    vmovdqa {{.*#+}} xmm1 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128]
-; X64-AVX-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; X64-AVX-NEXT:    vpsrlw $8, %xmm0, %xmm2
-; X64-AVX-NEXT:    vpminub %xmm2, %xmm0, %xmm0
+; X64-AVX-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
+; X64-AVX-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; X64-AVX-NEXT:    vpminub %xmm1, %xmm0, %xmm0
 ; X64-AVX-NEXT:    vphminposuw %xmm0, %xmm0
-; X64-AVX-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; X64-AVX-NEXT:    vpextrb $0, %xmm0, %eax
+; X64-AVX-NEXT:    xorb $-128, %al
 ; X64-AVX-NEXT:    ## kill: def $al killed $al killed $eax
 ; X64-AVX-NEXT:    retq
   %1  = shufflevector <16 x i8> %a0, <16 x i8> undef, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
@@ -740,11 +732,10 @@ define i16 @test_reduce_v16i16(<16 x i16> %a0) {
 ; X86-SSE42-LABEL: test_reduce_v16i16:
 ; X86-SSE42:       ## %bb.0:
 ; X86-SSE42-NEXT:    pminsw %xmm1, %xmm0
-; X86-SSE42-NEXT:    movdqa {{.*#+}} xmm1 = [32768,32768,32768,32768,32768,32768,32768,32768]
-; X86-SSE42-NEXT:    pxor %xmm1, %xmm0
+; X86-SSE42-NEXT:    pxor LCPI6_0, %xmm0
 ; X86-SSE42-NEXT:    phminposuw %xmm0, %xmm0
-; X86-SSE42-NEXT:    pxor %xmm1, %xmm0
 ; X86-SSE42-NEXT:    movd %xmm0, %eax
+; X86-SSE42-NEXT:    xorl $32768, %eax ## imm = 0x8000
 ; X86-SSE42-NEXT:    ## kill: def $ax killed $ax killed $eax
 ; X86-SSE42-NEXT:    retl
 ;
@@ -752,11 +743,10 @@ define i16 @test_reduce_v16i16(<16 x i16> %a0) {
 ; X86-AVX1:       ## %bb.0:
 ; X86-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; X86-AVX1-NEXT:    vpminsw %xmm1, %xmm0, %xmm0
-; X86-AVX1-NEXT:    vmovdqa {{.*#+}} xmm1 = [32768,32768,32768,32768,32768,32768,32768,32768]
-; X86-AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vpxor LCPI6_0, %xmm0, %xmm0
 ; X86-AVX1-NEXT:    vphminposuw %xmm0, %xmm0
-; X86-AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; X86-AVX1-NEXT:    vmovd %xmm0, %eax
+; X86-AVX1-NEXT:    xorl $32768, %eax ## imm = 0x8000
 ; X86-AVX1-NEXT:    ## kill: def $ax killed $ax killed $eax
 ; X86-AVX1-NEXT:    vzeroupper
 ; X86-AVX1-NEXT:    retl
@@ -765,11 +755,10 @@ define i16 @test_reduce_v16i16(<16 x i16> %a0) {
 ; X86-AVX2:       ## %bb.0:
 ; X86-AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; X86-AVX2-NEXT:    vpminsw %xmm1, %xmm0, %xmm0
-; X86-AVX2-NEXT:    vmovdqa {{.*#+}} xmm1 = [32768,32768,32768,32768,32768,32768,32768,32768]
-; X86-AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; X86-AVX2-NEXT:    vpxor LCPI6_0, %xmm0, %xmm0
 ; X86-AVX2-NEXT:    vphminposuw %xmm0, %xmm0
-; X86-AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; X86-AVX2-NEXT:    vmovd %xmm0, %eax
+; X86-AVX2-NEXT:    xorl $32768, %eax ## imm = 0x8000
 ; X86-AVX2-NEXT:    ## kill: def $ax killed $ax killed $eax
 ; X86-AVX2-NEXT:    vzeroupper
 ; X86-AVX2-NEXT:    retl
@@ -791,11 +780,10 @@ define i16 @test_reduce_v16i16(<16 x i16> %a0) {
 ; X64-SSE42-LABEL: test_reduce_v16i16:
 ; X64-SSE42:       ## %bb.0:
 ; X64-SSE42-NEXT:    pminsw %xmm1, %xmm0
-; X64-SSE42-NEXT:    movdqa {{.*#+}} xmm1 = [32768,32768,32768,32768,32768,32768,32768,32768]
-; X64-SSE42-NEXT:    pxor %xmm1, %xmm0
+; X64-SSE42-NEXT:    pxor {{.*}}(%rip), %xmm0
 ; X64-SSE42-NEXT:    phminposuw %xmm0, %xmm0
-; X64-SSE42-NEXT:    pxor %xmm1, %xmm0
 ; X64-SSE42-NEXT:    movd %xmm0, %eax
+; X64-SSE42-NEXT:    xorl $32768, %eax ## imm = 0x8000
 ; X64-SSE42-NEXT:    ## kill: def $ax killed $ax killed $eax
 ; X64-SSE42-NEXT:    retq
 ;
@@ -803,11 +791,10 @@ define i16 @test_reduce_v16i16(<16 x i16> %a0) {
 ; X64-AVX1:       ## %bb.0:
 ; X64-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; X64-AVX1-NEXT:    vpminsw %xmm1, %xmm0, %xmm0
-; X64-AVX1-NEXT:    vmovdqa {{.*#+}} xmm1 = [32768,32768,32768,32768,32768,32768,32768,32768]
-; X64-AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; X64-AVX1-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
 ; X64-AVX1-NEXT:    vphminposuw %xmm0, %xmm0
-; X64-AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; X64-AVX1-NEXT:    vmovd %xmm0, %eax
+; X64-AVX1-NEXT:    xorl $32768, %eax ## imm = 0x8000
 ; X64-AVX1-NEXT:    ## kill: def $ax killed $ax killed $eax
 ; X64-AVX1-NEXT:    vzeroupper
 ; X64-AVX1-NEXT:    retq
@@ -816,11 +803,10 @@ define i16 @test_reduce_v16i16(<16 x i16> %a0) {
 ; X64-AVX2:       ## %bb.0:
 ; X64-AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; X64-AVX2-NEXT:    vpminsw %xmm1, %xmm0, %xmm0
-; X64-AVX2-NEXT:    vmovdqa {{.*#+}} xmm1 = [32768,32768,32768,32768,32768,32768,32768,32768]
-; X64-AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; X64-AVX2-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
 ; X64-AVX2-NEXT:    vphminposuw %xmm0, %xmm0
-; X64-AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; X64-AVX2-NEXT:    vmovd %xmm0, %eax
+; X64-AVX2-NEXT:    xorl $32768, %eax ## imm = 0x8000
 ; X64-AVX2-NEXT:    ## kill: def $ax killed $ax killed $eax
 ; X64-AVX2-NEXT:    vzeroupper
 ; X64-AVX2-NEXT:    retq
@@ -829,11 +815,10 @@ define i16 @test_reduce_v16i16(<16 x i16> %a0) {
 ; X64-AVX512:       ## %bb.0:
 ; X64-AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; X64-AVX512-NEXT:    vpminsw %xmm1, %xmm0, %xmm0
-; X64-AVX512-NEXT:    vmovdqa {{.*#+}} xmm1 = [32768,32768,32768,32768,32768,32768,32768,32768]
-; X64-AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; X64-AVX512-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
 ; X64-AVX512-NEXT:    vphminposuw %xmm0, %xmm0
-; X64-AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; X64-AVX512-NEXT:    vmovd %xmm0, %eax
+; X64-AVX512-NEXT:    xorl $32768, %eax ## imm = 0x8000
 ; X64-AVX512-NEXT:    ## kill: def $ax killed $ax killed $eax
 ; X64-AVX512-NEXT:    vzeroupper
 ; X64-AVX512-NEXT:    retq
@@ -894,14 +879,13 @@ define i8 @test_reduce_v32i8(<32 x i8> %a0) {
 ; X86-SSE42-LABEL: test_reduce_v32i8:
 ; X86-SSE42:       ## %bb.0:
 ; X86-SSE42-NEXT:    pminsb %xmm1, %xmm0
-; X86-SSE42-NEXT:    movdqa {{.*#+}} xmm1 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128]
-; X86-SSE42-NEXT:    pxor %xmm1, %xmm0
-; X86-SSE42-NEXT:    movdqa %xmm0, %xmm2
-; X86-SSE42-NEXT:    psrlw $8, %xmm2
-; X86-SSE42-NEXT:    pminub %xmm0, %xmm2
-; X86-SSE42-NEXT:    phminposuw %xmm2, %xmm0
-; X86-SSE42-NEXT:    pxor %xmm1, %xmm0
+; X86-SSE42-NEXT:    pxor LCPI7_0, %xmm0
+; X86-SSE42-NEXT:    movdqa %xmm0, %xmm1
+; X86-SSE42-NEXT:    psrlw $8, %xmm1
+; X86-SSE42-NEXT:    pminub %xmm0, %xmm1
+; X86-SSE42-NEXT:    phminposuw %xmm1, %xmm0
 ; X86-SSE42-NEXT:    pextrb $0, %xmm0, %eax
+; X86-SSE42-NEXT:    xorb $-128, %al
 ; X86-SSE42-NEXT:    ## kill: def $al killed $al killed $eax
 ; X86-SSE42-NEXT:    retl
 ;
@@ -909,13 +893,12 @@ define i8 @test_reduce_v32i8(<32 x i8> %a0) {
 ; X86-AVX1:       ## %bb.0:
 ; X86-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; X86-AVX1-NEXT:    vpminsb %xmm1, %xmm0, %xmm0
-; X86-AVX1-NEXT:    vmovdqa {{.*#+}} xmm1 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128]
-; X86-AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; X86-AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm2
-; X86-AVX1-NEXT:    vpminub %xmm2, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vpxor LCPI7_0, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; X86-AVX1-NEXT:    vpminub %xmm1, %xmm0, %xmm0
 ; X86-AVX1-NEXT:    vphminposuw %xmm0, %xmm0
-; X86-AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; X86-AVX1-NEXT:    vpextrb $0, %xmm0, %eax
+; X86-AVX1-NEXT:    xorb $-128, %al
 ; X86-AVX1-NEXT:    ## kill: def $al killed $al killed $eax
 ; X86-AVX1-NEXT:    vzeroupper
 ; X86-AVX1-NEXT:    retl
@@ -924,13 +907,12 @@ define i8 @test_reduce_v32i8(<32 x i8> %a0) {
 ; X86-AVX2:       ## %bb.0:
 ; X86-AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; X86-AVX2-NEXT:    vpminsb %xmm1, %xmm0, %xmm0
-; X86-AVX2-NEXT:    vmovdqa {{.*#+}} xmm1 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128]
-; X86-AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; X86-AVX2-NEXT:    vpsrlw $8, %xmm0, %xmm2
-; X86-AVX2-NEXT:    vpminub %xmm2, %xmm0, %xmm0
+; X86-AVX2-NEXT:    vpxor LCPI7_0, %xmm0, %xmm0
+; X86-AVX2-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; X86-AVX2-NEXT:    vpminub %xmm1, %xmm0, %xmm0
 ; X86-AVX2-NEXT:    vphminposuw %xmm0, %xmm0
-; X86-AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; X86-AVX2-NEXT:    vpextrb $0, %xmm0, %eax
+; X86-AVX2-NEXT:    xorb $-128, %al
 ; X86-AVX2-NEXT:    ## kill: def $al killed $al killed $eax
 ; X86-AVX2-NEXT:    vzeroupper
 ; X86-AVX2-NEXT:    retl
@@ -975,14 +957,13 @@ define i8 @test_reduce_v32i8(<32 x i8> %a0) {
 ; X64-SSE42-LABEL: test_reduce_v32i8:
 ; X64-SSE42:       ## %bb.0:
 ; X64-SSE42-NEXT:    pminsb %xmm1, %xmm0
-; X64-SSE42-NEXT:    movdqa {{.*#+}} xmm1 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128]
-; X64-SSE42-NEXT:    pxor %xmm1, %xmm0
-; X64-SSE42-NEXT:    movdqa %xmm0, %xmm2
-; X64-SSE42-NEXT:    psrlw $8, %xmm2
-; X64-SSE42-NEXT:    pminub %xmm0, %xmm2
-; X64-SSE42-NEXT:    phminposuw %xmm2, %xmm0
-; X64-SSE42-NEXT:    pxor %xmm1, %xmm0
+; X64-SSE42-NEXT:    pxor {{.*}}(%rip), %xmm0
+; X64-SSE42-NEXT:    movdqa %xmm0, %xmm1
+; X64-SSE42-NEXT:    psrlw $8, %xmm1
+; X64-SSE42-NEXT:    pminub %xmm0, %xmm1
+; X64-SSE42-NEXT:    phminposuw %xmm1, %xmm0
 ; X64-SSE42-NEXT:    pextrb $0, %xmm0, %eax
+; X64-SSE42-NEXT:    xorb $-128, %al
 ; X64-SSE42-NEXT:    ## kill: def $al killed $al killed $eax
 ; X64-SSE42-NEXT:    retq
 ;
@@ -990,13 +971,12 @@ define i8 @test_reduce_v32i8(<32 x i8> %a0) {
 ; X64-AVX1:       ## %bb.0:
 ; X64-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; X64-AVX1-NEXT:    vpminsb %xmm1, %xmm0, %xmm0
-; X64-AVX1-NEXT:    vmovdqa {{.*#+}} xmm1 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128]
-; X64-AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; X64-AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm2
-; X64-AVX1-NEXT:    vpminub %xmm2, %xmm0, %xmm0
+; X64-AVX1-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
+; X64-AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; X64-AVX1-NEXT:    vpminub %xmm1, %xmm0, %xmm0
 ; X64-AVX1-NEXT:    vphminposuw %xmm0, %xmm0
-; X64-AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; X64-AVX1-NEXT:    vpextrb $0, %xmm0, %eax
+; X64-AVX1-NEXT:    xorb $-128, %al
 ; X64-AVX1-NEXT:    ## kill: def $al killed $al killed $eax
 ; X64-AVX1-NEXT:    vzeroupper
 ; X64-AVX1-NEXT:    retq
@@ -1005,13 +985,12 @@ define i8 @test_reduce_v32i8(<32 x i8> %a0) {
 ; X64-AVX2:       ## %bb.0:
 ; X64-AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; X64-AVX2-NEXT:    vpminsb %xmm1, %xmm0, %xmm0
-; X64-AVX2-NEXT:    vmovdqa {{.*#+}} xmm1 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128]
-; X64-AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; X64-AVX2-NEXT:    vpsrlw $8, %xmm0, %xmm2
-; X64-AVX2-NEXT:    vpminub %xmm2, %xmm0, %xmm0
+; X64-AVX2-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
+; X64-AVX2-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; X64-AVX2-NEXT:    vpminub %xmm1, %xmm0, %xmm0
 ; X64-AVX2-NEXT:    vphminposuw %xmm0, %xmm0
-; X64-AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; X64-AVX2-NEXT:    vpextrb $0, %xmm0, %eax
+; X64-AVX2-NEXT:    xorb $-128, %al
 ; X64-AVX2-NEXT:    ## kill: def $al killed $al killed $eax
 ; X64-AVX2-NEXT:    vzeroupper
 ; X64-AVX2-NEXT:    retq
@@ -1020,13 +999,12 @@ define i8 @test_reduce_v32i8(<32 x i8> %a0) {
 ; X64-AVX512:       ## %bb.0:
 ; X64-AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; X64-AVX512-NEXT:    vpminsb %xmm1, %xmm0, %xmm0
-; X64-AVX512-NEXT:    vmovdqa {{.*#+}} xmm1 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128]
-; X64-AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; X64-AVX512-NEXT:    vpsrlw $8, %xmm0, %xmm2
-; X64-AVX512-NEXT:    vpminub %xmm2, %xmm0, %xmm0
+; X64-AVX512-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
+; X64-AVX512-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; X64-AVX512-NEXT:    vpminub %xmm1, %xmm0, %xmm0
 ; X64-AVX512-NEXT:    vphminposuw %xmm0, %xmm0
-; X64-AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; X64-AVX512-NEXT:    vpextrb $0, %xmm0, %eax
+; X64-AVX512-NEXT:    xorb $-128, %al
 ; X64-AVX512-NEXT:    ## kill: def $al killed $al killed $eax
 ; X64-AVX512-NEXT:    vzeroupper
 ; X64-AVX512-NEXT:    retq
@@ -1517,11 +1495,10 @@ define i16 @test_reduce_v32i16(<32 x i16> %a0) {
 ; X86-SSE42-NEXT:    pminsw %xmm3, %xmm1
 ; X86-SSE42-NEXT:    pminsw %xmm2, %xmm0
 ; X86-SSE42-NEXT:    pminsw %xmm1, %xmm0
-; X86-SSE42-NEXT:    movdqa {{.*#+}} xmm1 = [32768,32768,32768,32768,32768,32768,32768,32768]
-; X86-SSE42-NEXT:    pxor %xmm1, %xmm0
+; X86-SSE42-NEXT:    pxor LCPI10_0, %xmm0
 ; X86-SSE42-NEXT:    phminposuw %xmm0, %xmm0
-; X86-SSE42-NEXT:    pxor %xmm1, %xmm0
 ; X86-SSE42-NEXT:    movd %xmm0, %eax
+; X86-SSE42-NEXT:    xorl $32768, %eax ## imm = 0x8000
 ; X86-SSE42-NEXT:    ## kill: def $ax killed $ax killed $eax
 ; X86-SSE42-NEXT:    retl
 ;
@@ -1532,11 +1509,10 @@ define i16 @test_reduce_v32i16(<32 x i16> %a0) {
 ; X86-AVX1-NEXT:    vpminsw %xmm2, %xmm3, %xmm2
 ; X86-AVX1-NEXT:    vpminsw %xmm1, %xmm0, %xmm0
 ; X86-AVX1-NEXT:    vpminsw %xmm2, %xmm0, %xmm0
-; X86-AVX1-NEXT:    vmovdqa {{.*#+}} xmm1 = [32768,32768,32768,32768,32768,32768,32768,32768]
-; X86-AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vpxor LCPI10_0, %xmm0, %xmm0
 ; X86-AVX1-NEXT:    vphminposuw %xmm0, %xmm0
-; X86-AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; X86-AVX1-NEXT:    vmovd %xmm0, %eax
+; X86-AVX1-NEXT:    xorl $32768, %eax ## imm = 0x8000
 ; X86-AVX1-NEXT:    ## kill: def $ax killed $ax killed $eax
 ; X86-AVX1-NEXT:    vzeroupper
 ; X86-AVX1-NEXT:    retl
@@ -1546,11 +1522,10 @@ define i16 @test_reduce_v32i16(<32 x i16> %a0) {
 ; X86-AVX2-NEXT:    vpminsw %ymm1, %ymm0, %ymm0
 ; X86-AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; X86-AVX2-NEXT:    vpminsw %xmm1, %xmm0, %xmm0
-; X86-AVX2-NEXT:    vmovdqa {{.*#+}} xmm1 = [32768,32768,32768,32768,32768,32768,32768,32768]
-; X86-AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; X86-AVX2-NEXT:    vpxor LCPI10_0, %xmm0, %xmm0
 ; X86-AVX2-NEXT:    vphminposuw %xmm0, %xmm0
-; X86-AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; X86-AVX2-NEXT:    vmovd %xmm0, %eax
+; X86-AVX2-NEXT:    xorl $32768, %eax ## imm = 0x8000
 ; X86-AVX2-NEXT:    ## kill: def $ax killed $ax killed $eax
 ; X86-AVX2-NEXT:    vzeroupper
 ; X86-AVX2-NEXT:    retl
@@ -1576,11 +1551,10 @@ define i16 @test_reduce_v32i16(<32 x i16> %a0) {
 ; X64-SSE42-NEXT:    pminsw %xmm3, %xmm1
 ; X64-SSE42-NEXT:    pminsw %xmm2, %xmm0
 ; X64-SSE42-NEXT:    pminsw %xmm1, %xmm0
-; X64-SSE42-NEXT:    movdqa {{.*#+}} xmm1 = [32768,32768,32768,32768,32768,32768,32768,32768]
-; X64-SSE42-NEXT:    pxor %xmm1, %xmm0
+; X64-SSE42-NEXT:    pxor {{.*}}(%rip), %xmm0
 ; X64-SSE42-NEXT:    phminposuw %xmm0, %xmm0
-; X64-SSE42-NEXT:    pxor %xmm1, %xmm0
 ; X64-SSE42-NEXT:    movd %xmm0, %eax
+; X64-SSE42-NEXT:    xorl $32768, %eax ## imm = 0x8000
 ; X64-SSE42-NEXT:    ## kill: def $ax killed $ax killed $eax
 ; X64-SSE42-NEXT:    retq
 ;
@@ -1591,11 +1565,10 @@ define i16 @test_reduce_v32i16(<32 x i16> %a0) {
 ; X64-AVX1-NEXT:    vpminsw %xmm2, %xmm3, %xmm2
 ; X64-AVX1-NEXT:    vpminsw %xmm1, %xmm0, %xmm0
 ; X64-AVX1-NEXT:    vpminsw %xmm2, %xmm0, %xmm0
-; X64-AVX1-NEXT:    vmovdqa {{.*#+}} xmm1 = [32768,32768,32768,32768,32768,32768,32768,32768]
-; X64-AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; X64-AVX1-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
 ; X64-AVX1-NEXT:    vphminposuw %xmm0, %xmm0
-; X64-AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; X64-AVX1-NEXT:    vmovd %xmm0, %eax
+; X64-AVX1-NEXT:    xorl $32768, %eax ## imm = 0x8000
 ; X64-AVX1-NEXT:    ## kill: def $ax killed $ax killed $eax
 ; X64-AVX1-NEXT:    vzeroupper
 ; X64-AVX1-NEXT:    retq
@@ -1605,11 +1578,10 @@ define i16 @test_reduce_v32i16(<32 x i16> %a0) {
 ; X64-AVX2-NEXT:    vpminsw %ymm1, %ymm0, %ymm0
 ; X64-AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; X64-AVX2-NEXT:    vpminsw %xmm1, %xmm0, %xmm0
-; X64-AVX2-NEXT:    vmovdqa {{.*#+}} xmm1 = [32768,32768,32768,32768,32768,32768,32768,32768]
-; X64-AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; X64-AVX2-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
 ; X64-AVX2-NEXT:    vphminposuw %xmm0, %xmm0
-; X64-AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; X64-AVX2-NEXT:    vmovd %xmm0, %eax
+; X64-AVX2-NEXT:    xorl $32768, %eax ## imm = 0x8000
 ; X64-AVX2-NEXT:    ## kill: def $ax killed $ax killed $eax
 ; X64-AVX2-NEXT:    vzeroupper
 ; X64-AVX2-NEXT:    retq
@@ -1620,11 +1592,10 @@ define i16 @test_reduce_v32i16(<32 x i16> %a0) {
 ; X64-AVX512-NEXT:    vpminsw %ymm1, %ymm0, %ymm0
 ; X64-AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; X64-AVX512-NEXT:    vpminsw %xmm1, %xmm0, %xmm0
-; X64-AVX512-NEXT:    vmovdqa {{.*#+}} xmm1 = [32768,32768,32768,32768,32768,32768,32768,32768]
-; X64-AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; X64-AVX512-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
 ; X64-AVX512-NEXT:    vphminposuw %xmm0, %xmm0
-; X64-AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; X64-AVX512-NEXT:    vmovd %xmm0, %eax
+; X64-AVX512-NEXT:    xorl $32768, %eax ## imm = 0x8000
 ; X64-AVX512-NEXT:    ## kill: def $ax killed $ax killed $eax
 ; X64-AVX512-NEXT:    vzeroupper
 ; X64-AVX512-NEXT:    retq
@@ -1700,14 +1671,13 @@ define i8 @test_reduce_v64i8(<64 x i8> %a0) {
 ; X86-SSE42-NEXT:    pminsb %xmm3, %xmm1
 ; X86-SSE42-NEXT:    pminsb %xmm2, %xmm0
 ; X86-SSE42-NEXT:    pminsb %xmm1, %xmm0
-; X86-SSE42-NEXT:    movdqa {{.*#+}} xmm1 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128]
-; X86-SSE42-NEXT:    pxor %xmm1, %xmm0
-; X86-SSE42-NEXT:    movdqa %xmm0, %xmm2
-; X86-SSE42-NEXT:    psrlw $8, %xmm2
-; X86-SSE42-NEXT:    pminub %xmm0, %xmm2
-; X86-SSE42-NEXT:    phminposuw %xmm2, %xmm0
-; X86-SSE42-NEXT:    pxor %xmm1, %xmm0
+; X86-SSE42-NEXT:    pxor LCPI11_0, %xmm0
+; X86-SSE42-NEXT:    movdqa %xmm0, %xmm1
+; X86-SSE42-NEXT:    psrlw $8, %xmm1
+; X86-SSE42-NEXT:    pminub %xmm0, %xmm1
+; X86-SSE42-NEXT:    phminposuw %xmm1, %xmm0
 ; X86-SSE42-NEXT:    pextrb $0, %xmm0, %eax
+; X86-SSE42-NEXT:    xorb $-128, %al
 ; X86-SSE42-NEXT:    ## kill: def $al killed $al killed $eax
 ; X86-SSE42-NEXT:    retl
 ;
@@ -1718,13 +1688,12 @@ define i8 @test_reduce_v64i8(<64 x i8> %a0) {
 ; X86-AVX1-NEXT:    vpminsb %xmm2, %xmm3, %xmm2
 ; X86-AVX1-NEXT:    vpminsb %xmm1, %xmm0, %xmm0
 ; X86-AVX1-NEXT:    vpminsb %xmm2, %xmm0, %xmm0
-; X86-AVX1-NEXT:    vmovdqa {{.*#+}} xmm1 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128]
-; X86-AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; X86-AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm2
-; X86-AVX1-NEXT:    vpminub %xmm2, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vpxor LCPI11_0, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; X86-AVX1-NEXT:    vpminub %xmm1, %xmm0, %xmm0
 ; X86-AVX1-NEXT:    vphminposuw %xmm0, %xmm0
-; X86-AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; X86-AVX1-NEXT:    vpextrb $0, %xmm0, %eax
+; X86-AVX1-NEXT:    xorb $-128, %al
 ; X86-AVX1-NEXT:    ## kill: def $al killed $al killed $eax
 ; X86-AVX1-NEXT:    vzeroupper
 ; X86-AVX1-NEXT:    retl
@@ -1734,13 +1703,12 @@ define i8 @test_reduce_v64i8(<64 x i8> %a0) {
 ; X86-AVX2-NEXT:    vpminsb %ymm1, %ymm0, %ymm0
 ; X86-AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; X86-AVX2-NEXT:    vpminsb %xmm1, %xmm0, %xmm0
-; X86-AVX2-NEXT:    vmovdqa {{.*#+}} xmm1 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128]
-; X86-AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; X86-AVX2-NEXT:    vpsrlw $8, %xmm0, %xmm2
-; X86-AVX2-NEXT:    vpminub %xmm2, %xmm0, %xmm0
+; X86-AVX2-NEXT:    vpxor LCPI11_0, %xmm0, %xmm0
+; X86-AVX2-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; X86-AVX2-NEXT:    vpminub %xmm1, %xmm0, %xmm0
 ; X86-AVX2-NEXT:    vphminposuw %xmm0, %xmm0
-; X86-AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; X86-AVX2-NEXT:    vpextrb $0, %xmm0, %eax
+; X86-AVX2-NEXT:    xorb $-128, %al
 ; X86-AVX2-NEXT:    ## kill: def $al killed $al killed $eax
 ; X86-AVX2-NEXT:    vzeroupper
 ; X86-AVX2-NEXT:    retl
@@ -1797,14 +1765,13 @@ define i8 @test_reduce_v64i8(<64 x i8> %a0) {
 ; X64-SSE42-NEXT:    pminsb %xmm3, %xmm1
 ; X64-SSE42-NEXT:    pminsb %xmm2, %xmm0
 ; X64-SSE42-NEXT:    pminsb %xmm1, %xmm0
-; X64-SSE42-NEXT:    movdqa {{.*#+}} xmm1 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128]
-; X64-SSE42-NEXT:    pxor %xmm1, %xmm0
-; X64-SSE42-NEXT:    movdqa %xmm0, %xmm2
-; X64-SSE42-NEXT:    psrlw $8, %xmm2
-; X64-SSE42-NEXT:    pminub %xmm0, %xmm2
-; X64-SSE42-NEXT:    phminposuw %xmm2, %xmm0
-; X64-SSE42-NEXT:    pxor %xmm1, %xmm0
+; X64-SSE42-NEXT:    pxor {{.*}}(%rip), %xmm0
+; X64-SSE42-NEXT:    movdqa %xmm0, %xmm1
+; X64-SSE42-NEXT:    psrlw $8, %xmm1
+; X64-SSE42-NEXT:    pminub %xmm0, %xmm1
+; X64-SSE42-NEXT:    phminposuw %xmm1, %xmm0
 ; X64-SSE42-NEXT:    pextrb $0, %xmm0, %eax
+; X64-SSE42-NEXT:    xorb $-128, %al
 ; X64-SSE42-NEXT:    ## kill: def $al killed $al killed $eax
 ; X64-SSE42-NEXT:    retq
 ;
@@ -1815,13 +1782,12 @@ define i8 @test_reduce_v64i8(<64 x i8> %a0) {
 ; X64-AVX1-NEXT:    vpminsb %xmm2, %xmm3, %xmm2
 ; X64-AVX1-NEXT:    vpminsb %xmm1, %xmm0, %xmm0
 ; X64-AVX1-NEXT:    vpminsb %xmm2, %xmm0, %xmm0
-; X64-AVX1-NEXT:    vmovdqa {{.*#+}} xmm1 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128]
-; X64-AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; X64-AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm2
-; X64-AVX1-NEXT:    vpminub %xmm2, %xmm0, %xmm0
+; X64-AVX1-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
+; X64-AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; X64-AVX1-NEXT:    vpminub %xmm1, %xmm0, %xmm0
 ; X64-AVX1-NEXT:    vphminposuw %xmm0, %xmm0
-; X64-AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; X64-AVX1-NEXT:    vpextrb $0, %xmm0, %eax
+; X64-AVX1-NEXT:    xorb $-128, %al
 ; X64-AVX1-NEXT:    ## kill: def $al killed $al killed $eax
 ; X64-AVX1-NEXT:    vzeroupper
 ; X64-AVX1-NEXT:    retq
@@ -1831,13 +1797,12 @@ define i8 @test_reduce_v64i8(<64 x i8> %a0) {
 ; X64-AVX2-NEXT:    vpminsb %ymm1, %ymm0, %ymm0
 ; X64-AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; X64-AVX2-NEXT:    vpminsb %xmm1, %xmm0, %xmm0
-; X64-AVX2-NEXT:    vmovdqa {{.*#+}} xmm1 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128]
-; X64-AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; X64-AVX2-NEXT:    vpsrlw $8, %xmm0, %xmm2
-; X64-AVX2-NEXT:    vpminub %xmm2, %xmm0, %xmm0
+; X64-AVX2-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
+; X64-AVX2-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; X64-AVX2-NEXT:    vpminub %xmm1, %xmm0, %xmm0
 ; X64-AVX2-NEXT:    vphminposuw %xmm0, %xmm0
-; X64-AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; X64-AVX2-NEXT:    vpextrb $0, %xmm0, %eax
+; X64-AVX2-NEXT:    xorb $-128, %al
 ; X64-AVX2-NEXT:    ## kill: def $al killed $al killed $eax
 ; X64-AVX2-NEXT:    vzeroupper
 ; X64-AVX2-NEXT:    retq
@@ -1848,13 +1813,12 @@ define i8 @test_reduce_v64i8(<64 x i8> %a0) {
 ; X64-AVX512-NEXT:    vpminsb %ymm1, %ymm0, %ymm0
 ; X64-AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; X64-AVX512-NEXT:    vpminsb %xmm1, %xmm0, %xmm0
-; X64-AVX512-NEXT:    vmovdqa {{.*#+}} xmm1 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128]
-; X64-AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; X64-AVX512-NEXT:    vpsrlw $8, %xmm0, %xmm2
-; X64-AVX512-NEXT:    vpminub %xmm2, %xmm0, %xmm0
+; X64-AVX512-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
+; X64-AVX512-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; X64-AVX512-NEXT:    vpminub %xmm1, %xmm0, %xmm0
 ; X64-AVX512-NEXT:    vphminposuw %xmm0, %xmm0
-; X64-AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; X64-AVX512-NEXT:    vpextrb $0, %xmm0, %eax
+; X64-AVX512-NEXT:    xorb $-128, %al
 ; X64-AVX512-NEXT:    ## kill: def $al killed $al killed $eax
 ; X64-AVX512-NEXT:    vzeroupper
 ; X64-AVX512-NEXT:    retq
diff --git a/test/CodeGen/X86/horizontal-reduce-umax.ll b/test/CodeGen/X86/horizontal-reduce-umax.ll
index 88f6b01131a2c8e32f528099284af229c037487c..2bc72158ff388a9e4937da09bb05fbcee7a8a77a 100644
--- a/test/CodeGen/X86/horizontal-reduce-umax.ll
+++ b/test/CodeGen/X86/horizontal-reduce-umax.ll
@@ -240,18 +240,18 @@ define i16 @test_reduce_v8i16(<8 x i16> %a0) {
 ; X86-SSE2-NEXT:    psrld $16, %xmm1
 ; X86-SSE2-NEXT:    pxor %xmm2, %xmm1
 ; X86-SSE2-NEXT:    pmaxsw %xmm0, %xmm1
-; X86-SSE2-NEXT:    pxor %xmm2, %xmm1
 ; X86-SSE2-NEXT:    movd %xmm1, %eax
+; X86-SSE2-NEXT:    xorl $32768, %eax ## imm = 0x8000
 ; X86-SSE2-NEXT:    ## kill: def $ax killed $ax killed $eax
 ; X86-SSE2-NEXT:    retl
 ;
 ; X86-SSE42-LABEL: test_reduce_v8i16:
 ; X86-SSE42:       ## %bb.0:
 ; X86-SSE42-NEXT:    pcmpeqd %xmm1, %xmm1
-; X86-SSE42-NEXT:    pxor %xmm1, %xmm0
-; X86-SSE42-NEXT:    phminposuw %xmm0, %xmm0
-; X86-SSE42-NEXT:    pxor %xmm1, %xmm0
+; X86-SSE42-NEXT:    pxor %xmm0, %xmm1
+; X86-SSE42-NEXT:    phminposuw %xmm1, %xmm0
 ; X86-SSE42-NEXT:    movd %xmm0, %eax
+; X86-SSE42-NEXT:    notl %eax
 ; X86-SSE42-NEXT:    ## kill: def $ax killed $ax killed $eax
 ; X86-SSE42-NEXT:    retl
 ;
@@ -260,8 +260,8 @@ define i16 @test_reduce_v8i16(<8 x i16> %a0) {
 ; X86-AVX-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
 ; X86-AVX-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; X86-AVX-NEXT:    vphminposuw %xmm0, %xmm0
-; X86-AVX-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; X86-AVX-NEXT:    vmovd %xmm0, %eax
+; X86-AVX-NEXT:    notl %eax
 ; X86-AVX-NEXT:    ## kill: def $ax killed $ax killed $eax
 ; X86-AVX-NEXT:    retl
 ;
@@ -282,18 +282,18 @@ define i16 @test_reduce_v8i16(<8 x i16> %a0) {
 ; X64-SSE2-NEXT:    psrld $16, %xmm1
 ; X64-SSE2-NEXT:    pxor %xmm2, %xmm1
 ; X64-SSE2-NEXT:    pmaxsw %xmm0, %xmm1
-; X64-SSE2-NEXT:    pxor %xmm2, %xmm1
 ; X64-SSE2-NEXT:    movd %xmm1, %eax
+; X64-SSE2-NEXT:    xorl $32768, %eax ## imm = 0x8000
 ; X64-SSE2-NEXT:    ## kill: def $ax killed $ax killed $eax
 ; X64-SSE2-NEXT:    retq
 ;
 ; X64-SSE42-LABEL: test_reduce_v8i16:
 ; X64-SSE42:       ## %bb.0:
 ; X64-SSE42-NEXT:    pcmpeqd %xmm1, %xmm1
-; X64-SSE42-NEXT:    pxor %xmm1, %xmm0
-; X64-SSE42-NEXT:    phminposuw %xmm0, %xmm0
-; X64-SSE42-NEXT:    pxor %xmm1, %xmm0
+; X64-SSE42-NEXT:    pxor %xmm0, %xmm1
+; X64-SSE42-NEXT:    phminposuw %xmm1, %xmm0
 ; X64-SSE42-NEXT:    movd %xmm0, %eax
+; X64-SSE42-NEXT:    notl %eax
 ; X64-SSE42-NEXT:    ## kill: def $ax killed $ax killed $eax
 ; X64-SSE42-NEXT:    retq
 ;
@@ -302,8 +302,8 @@ define i16 @test_reduce_v8i16(<8 x i16> %a0) {
 ; X64-AVX1-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
 ; X64-AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; X64-AVX1-NEXT:    vphminposuw %xmm0, %xmm0
-; X64-AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; X64-AVX1-NEXT:    vmovd %xmm0, %eax
+; X64-AVX1-NEXT:    notl %eax
 ; X64-AVX1-NEXT:    ## kill: def $ax killed $ax killed $eax
 ; X64-AVX1-NEXT:    retq
 ;
@@ -312,8 +312,8 @@ define i16 @test_reduce_v8i16(<8 x i16> %a0) {
 ; X64-AVX2-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
 ; X64-AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; X64-AVX2-NEXT:    vphminposuw %xmm0, %xmm0
-; X64-AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; X64-AVX2-NEXT:    vmovd %xmm0, %eax
+; X64-AVX2-NEXT:    notl %eax
 ; X64-AVX2-NEXT:    ## kill: def $ax killed $ax killed $eax
 ; X64-AVX2-NEXT:    retq
 ;
@@ -321,8 +321,8 @@ define i16 @test_reduce_v8i16(<8 x i16> %a0) {
 ; X64-AVX512:       ## %bb.0:
 ; X64-AVX512-NEXT:    vpternlogq $15, %xmm0, %xmm0, %xmm0
 ; X64-AVX512-NEXT:    vphminposuw %xmm0, %xmm0
-; X64-AVX512-NEXT:    vpternlogq $15, %xmm0, %xmm0, %xmm0
 ; X64-AVX512-NEXT:    vmovd %xmm0, %eax
+; X64-AVX512-NEXT:    notl %eax
 ; X64-AVX512-NEXT:    ## kill: def $ax killed $ax killed $eax
 ; X64-AVX512-NEXT:    retq
   %1  = shufflevector <8 x i16> %a0, <8 x i16> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
@@ -358,13 +358,13 @@ define i8 @test_reduce_v16i8(<16 x i8> %a0) {
 ; X86-SSE42-LABEL: test_reduce_v16i8:
 ; X86-SSE42:       ## %bb.0:
 ; X86-SSE42-NEXT:    pcmpeqd %xmm1, %xmm1
-; X86-SSE42-NEXT:    pxor %xmm1, %xmm0
-; X86-SSE42-NEXT:    movdqa %xmm0, %xmm2
-; X86-SSE42-NEXT:    psrlw $8, %xmm2
-; X86-SSE42-NEXT:    pminub %xmm0, %xmm2
-; X86-SSE42-NEXT:    phminposuw %xmm2, %xmm0
-; X86-SSE42-NEXT:    pxor %xmm1, %xmm0
+; X86-SSE42-NEXT:    pxor %xmm0, %xmm1
+; X86-SSE42-NEXT:    movdqa %xmm1, %xmm0
+; X86-SSE42-NEXT:    psrlw $8, %xmm0
+; X86-SSE42-NEXT:    pminub %xmm1, %xmm0
+; X86-SSE42-NEXT:    phminposuw %xmm0, %xmm0
 ; X86-SSE42-NEXT:    pextrb $0, %xmm0, %eax
+; X86-SSE42-NEXT:    notb %al
 ; X86-SSE42-NEXT:    ## kill: def $al killed $al killed $eax
 ; X86-SSE42-NEXT:    retl
 ;
@@ -372,11 +372,11 @@ define i8 @test_reduce_v16i8(<16 x i8> %a0) {
 ; X86-AVX:       ## %bb.0:
 ; X86-AVX-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
 ; X86-AVX-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; X86-AVX-NEXT:    vpsrlw $8, %xmm0, %xmm2
-; X86-AVX-NEXT:    vpminub %xmm2, %xmm0, %xmm0
+; X86-AVX-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; X86-AVX-NEXT:    vpminub %xmm1, %xmm0, %xmm0
 ; X86-AVX-NEXT:    vphminposuw %xmm0, %xmm0
-; X86-AVX-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; X86-AVX-NEXT:    vpextrb $0, %xmm0, %eax
+; X86-AVX-NEXT:    notb %al
 ; X86-AVX-NEXT:    ## kill: def $al killed $al killed $eax
 ; X86-AVX-NEXT:    retl
 ;
@@ -399,13 +399,13 @@ define i8 @test_reduce_v16i8(<16 x i8> %a0) {
 ; X64-SSE42-LABEL: test_reduce_v16i8:
 ; X64-SSE42:       ## %bb.0:
 ; X64-SSE42-NEXT:    pcmpeqd %xmm1, %xmm1
-; X64-SSE42-NEXT:    pxor %xmm1, %xmm0
-; X64-SSE42-NEXT:    movdqa %xmm0, %xmm2
-; X64-SSE42-NEXT:    psrlw $8, %xmm2
-; X64-SSE42-NEXT:    pminub %xmm0, %xmm2
-; X64-SSE42-NEXT:    phminposuw %xmm2, %xmm0
-; X64-SSE42-NEXT:    pxor %xmm1, %xmm0
+; X64-SSE42-NEXT:    pxor %xmm0, %xmm1
+; X64-SSE42-NEXT:    movdqa %xmm1, %xmm0
+; X64-SSE42-NEXT:    psrlw $8, %xmm0
+; X64-SSE42-NEXT:    pminub %xmm1, %xmm0
+; X64-SSE42-NEXT:    phminposuw %xmm0, %xmm0
 ; X64-SSE42-NEXT:    pextrb $0, %xmm0, %eax
+; X64-SSE42-NEXT:    notb %al
 ; X64-SSE42-NEXT:    ## kill: def $al killed $al killed $eax
 ; X64-SSE42-NEXT:    retq
 ;
@@ -413,11 +413,11 @@ define i8 @test_reduce_v16i8(<16 x i8> %a0) {
 ; X64-AVX1:       ## %bb.0:
 ; X64-AVX1-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
 ; X64-AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; X64-AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm2
-; X64-AVX1-NEXT:    vpminub %xmm2, %xmm0, %xmm0
+; X64-AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; X64-AVX1-NEXT:    vpminub %xmm1, %xmm0, %xmm0
 ; X64-AVX1-NEXT:    vphminposuw %xmm0, %xmm0
-; X64-AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; X64-AVX1-NEXT:    vpextrb $0, %xmm0, %eax
+; X64-AVX1-NEXT:    notb %al
 ; X64-AVX1-NEXT:    ## kill: def $al killed $al killed $eax
 ; X64-AVX1-NEXT:    retq
 ;
@@ -425,11 +425,11 @@ define i8 @test_reduce_v16i8(<16 x i8> %a0) {
 ; X64-AVX2:       ## %bb.0:
 ; X64-AVX2-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
 ; X64-AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; X64-AVX2-NEXT:    vpsrlw $8, %xmm0, %xmm2
-; X64-AVX2-NEXT:    vpminub %xmm2, %xmm0, %xmm0
+; X64-AVX2-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; X64-AVX2-NEXT:    vpminub %xmm1, %xmm0, %xmm0
 ; X64-AVX2-NEXT:    vphminposuw %xmm0, %xmm0
-; X64-AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; X64-AVX2-NEXT:    vpextrb $0, %xmm0, %eax
+; X64-AVX2-NEXT:    notb %al
 ; X64-AVX2-NEXT:    ## kill: def $al killed $al killed $eax
 ; X64-AVX2-NEXT:    retq
 ;
@@ -439,8 +439,8 @@ define i8 @test_reduce_v16i8(<16 x i8> %a0) {
 ; X64-AVX512-NEXT:    vpsrlw $8, %xmm0, %xmm1
 ; X64-AVX512-NEXT:    vpminub %xmm1, %xmm0, %xmm0
 ; X64-AVX512-NEXT:    vphminposuw %xmm0, %xmm0
-; X64-AVX512-NEXT:    vpternlogq $15, %xmm0, %xmm0, %xmm0
 ; X64-AVX512-NEXT:    vpextrb $0, %xmm0, %eax
+; X64-AVX512-NEXT:    notb %al
 ; X64-AVX512-NEXT:    ## kill: def $al killed $al killed $eax
 ; X64-AVX512-NEXT:    retq
   %1  = shufflevector <16 x i8> %a0, <16 x i8> undef, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
@@ -845,8 +845,8 @@ define i16 @test_reduce_v16i16(<16 x i16> %a0) {
 ; X86-SSE2-NEXT:    psrld $16, %xmm1
 ; X86-SSE2-NEXT:    pxor %xmm2, %xmm1
 ; X86-SSE2-NEXT:    pmaxsw %xmm0, %xmm1
-; X86-SSE2-NEXT:    pxor %xmm2, %xmm1
 ; X86-SSE2-NEXT:    movd %xmm1, %eax
+; X86-SSE2-NEXT:    xorl $32768, %eax ## imm = 0x8000
 ; X86-SSE2-NEXT:    ## kill: def $ax killed $ax killed $eax
 ; X86-SSE2-NEXT:    retl
 ;
@@ -854,10 +854,10 @@ define i16 @test_reduce_v16i16(<16 x i16> %a0) {
 ; X86-SSE42:       ## %bb.0:
 ; X86-SSE42-NEXT:    pmaxuw %xmm1, %xmm0
 ; X86-SSE42-NEXT:    pcmpeqd %xmm1, %xmm1
-; X86-SSE42-NEXT:    pxor %xmm1, %xmm0
-; X86-SSE42-NEXT:    phminposuw %xmm0, %xmm0
-; X86-SSE42-NEXT:    pxor %xmm1, %xmm0
+; X86-SSE42-NEXT:    pxor %xmm0, %xmm1
+; X86-SSE42-NEXT:    phminposuw %xmm1, %xmm0
 ; X86-SSE42-NEXT:    movd %xmm0, %eax
+; X86-SSE42-NEXT:    notl %eax
 ; X86-SSE42-NEXT:    ## kill: def $ax killed $ax killed $eax
 ; X86-SSE42-NEXT:    retl
 ;
@@ -868,8 +868,8 @@ define i16 @test_reduce_v16i16(<16 x i16> %a0) {
 ; X86-AVX1-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
 ; X86-AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; X86-AVX1-NEXT:    vphminposuw %xmm0, %xmm0
-; X86-AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; X86-AVX1-NEXT:    vmovd %xmm0, %eax
+; X86-AVX1-NEXT:    notl %eax
 ; X86-AVX1-NEXT:    ## kill: def $ax killed $ax killed $eax
 ; X86-AVX1-NEXT:    vzeroupper
 ; X86-AVX1-NEXT:    retl
@@ -881,8 +881,8 @@ define i16 @test_reduce_v16i16(<16 x i16> %a0) {
 ; X86-AVX2-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
 ; X86-AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; X86-AVX2-NEXT:    vphminposuw %xmm0, %xmm0
-; X86-AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; X86-AVX2-NEXT:    vmovd %xmm0, %eax
+; X86-AVX2-NEXT:    notl %eax
 ; X86-AVX2-NEXT:    ## kill: def $ax killed $ax killed $eax
 ; X86-AVX2-NEXT:    vzeroupper
 ; X86-AVX2-NEXT:    retl
@@ -908,8 +908,8 @@ define i16 @test_reduce_v16i16(<16 x i16> %a0) {
 ; X64-SSE2-NEXT:    psrld $16, %xmm1
 ; X64-SSE2-NEXT:    pxor %xmm2, %xmm1
 ; X64-SSE2-NEXT:    pmaxsw %xmm0, %xmm1
-; X64-SSE2-NEXT:    pxor %xmm2, %xmm1
 ; X64-SSE2-NEXT:    movd %xmm1, %eax
+; X64-SSE2-NEXT:    xorl $32768, %eax ## imm = 0x8000
 ; X64-SSE2-NEXT:    ## kill: def $ax killed $ax killed $eax
 ; X64-SSE2-NEXT:    retq
 ;
@@ -917,10 +917,10 @@ define i16 @test_reduce_v16i16(<16 x i16> %a0) {
 ; X64-SSE42:       ## %bb.0:
 ; X64-SSE42-NEXT:    pmaxuw %xmm1, %xmm0
 ; X64-SSE42-NEXT:    pcmpeqd %xmm1, %xmm1
-; X64-SSE42-NEXT:    pxor %xmm1, %xmm0
-; X64-SSE42-NEXT:    phminposuw %xmm0, %xmm0
-; X64-SSE42-NEXT:    pxor %xmm1, %xmm0
+; X64-SSE42-NEXT:    pxor %xmm0, %xmm1
+; X64-SSE42-NEXT:    phminposuw %xmm1, %xmm0
 ; X64-SSE42-NEXT:    movd %xmm0, %eax
+; X64-SSE42-NEXT:    notl %eax
 ; X64-SSE42-NEXT:    ## kill: def $ax killed $ax killed $eax
 ; X64-SSE42-NEXT:    retq
 ;
@@ -931,8 +931,8 @@ define i16 @test_reduce_v16i16(<16 x i16> %a0) {
 ; X64-AVX1-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
 ; X64-AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; X64-AVX1-NEXT:    vphminposuw %xmm0, %xmm0
-; X64-AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; X64-AVX1-NEXT:    vmovd %xmm0, %eax
+; X64-AVX1-NEXT:    notl %eax
 ; X64-AVX1-NEXT:    ## kill: def $ax killed $ax killed $eax
 ; X64-AVX1-NEXT:    vzeroupper
 ; X64-AVX1-NEXT:    retq
@@ -944,8 +944,8 @@ define i16 @test_reduce_v16i16(<16 x i16> %a0) {
 ; X64-AVX2-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
 ; X64-AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; X64-AVX2-NEXT:    vphminposuw %xmm0, %xmm0
-; X64-AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; X64-AVX2-NEXT:    vmovd %xmm0, %eax
+; X64-AVX2-NEXT:    notl %eax
 ; X64-AVX2-NEXT:    ## kill: def $ax killed $ax killed $eax
 ; X64-AVX2-NEXT:    vzeroupper
 ; X64-AVX2-NEXT:    retq
@@ -956,8 +956,8 @@ define i16 @test_reduce_v16i16(<16 x i16> %a0) {
 ; X64-AVX512-NEXT:    vpmaxuw %xmm1, %xmm0, %xmm0
 ; X64-AVX512-NEXT:    vpternlogq $15, %xmm0, %xmm0, %xmm0
 ; X64-AVX512-NEXT:    vphminposuw %xmm0, %xmm0
-; X64-AVX512-NEXT:    vpternlogq $15, %xmm0, %xmm0, %xmm0
 ; X64-AVX512-NEXT:    vmovd %xmm0, %eax
+; X64-AVX512-NEXT:    notl %eax
 ; X64-AVX512-NEXT:    ## kill: def $ax killed $ax killed $eax
 ; X64-AVX512-NEXT:    vzeroupper
 ; X64-AVX512-NEXT:    retq
@@ -999,13 +999,13 @@ define i8 @test_reduce_v32i8(<32 x i8> %a0) {
 ; X86-SSE42:       ## %bb.0:
 ; X86-SSE42-NEXT:    pmaxub %xmm1, %xmm0
 ; X86-SSE42-NEXT:    pcmpeqd %xmm1, %xmm1
-; X86-SSE42-NEXT:    pxor %xmm1, %xmm0
-; X86-SSE42-NEXT:    movdqa %xmm0, %xmm2
-; X86-SSE42-NEXT:    psrlw $8, %xmm2
-; X86-SSE42-NEXT:    pminub %xmm0, %xmm2
-; X86-SSE42-NEXT:    phminposuw %xmm2, %xmm0
-; X86-SSE42-NEXT:    pxor %xmm1, %xmm0
+; X86-SSE42-NEXT:    pxor %xmm0, %xmm1
+; X86-SSE42-NEXT:    movdqa %xmm1, %xmm0
+; X86-SSE42-NEXT:    psrlw $8, %xmm0
+; X86-SSE42-NEXT:    pminub %xmm1, %xmm0
+; X86-SSE42-NEXT:    phminposuw %xmm0, %xmm0
 ; X86-SSE42-NEXT:    pextrb $0, %xmm0, %eax
+; X86-SSE42-NEXT:    notb %al
 ; X86-SSE42-NEXT:    ## kill: def $al killed $al killed $eax
 ; X86-SSE42-NEXT:    retl
 ;
@@ -1015,11 +1015,11 @@ define i8 @test_reduce_v32i8(<32 x i8> %a0) {
 ; X86-AVX1-NEXT:    vpmaxub %xmm1, %xmm0, %xmm0
 ; X86-AVX1-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
 ; X86-AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; X86-AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm2
-; X86-AVX1-NEXT:    vpminub %xmm2, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; X86-AVX1-NEXT:    vpminub %xmm1, %xmm0, %xmm0
 ; X86-AVX1-NEXT:    vphminposuw %xmm0, %xmm0
-; X86-AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; X86-AVX1-NEXT:    vpextrb $0, %xmm0, %eax
+; X86-AVX1-NEXT:    notb %al
 ; X86-AVX1-NEXT:    ## kill: def $al killed $al killed $eax
 ; X86-AVX1-NEXT:    vzeroupper
 ; X86-AVX1-NEXT:    retl
@@ -1030,11 +1030,11 @@ define i8 @test_reduce_v32i8(<32 x i8> %a0) {
 ; X86-AVX2-NEXT:    vpmaxub %xmm1, %xmm0, %xmm0
 ; X86-AVX2-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
 ; X86-AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; X86-AVX2-NEXT:    vpsrlw $8, %xmm0, %xmm2
-; X86-AVX2-NEXT:    vpminub %xmm2, %xmm0, %xmm0
+; X86-AVX2-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; X86-AVX2-NEXT:    vpminub %xmm1, %xmm0, %xmm0
 ; X86-AVX2-NEXT:    vphminposuw %xmm0, %xmm0
-; X86-AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; X86-AVX2-NEXT:    vpextrb $0, %xmm0, %eax
+; X86-AVX2-NEXT:    notb %al
 ; X86-AVX2-NEXT:    ## kill: def $al killed $al killed $eax
 ; X86-AVX2-NEXT:    vzeroupper
 ; X86-AVX2-NEXT:    retl
@@ -1060,13 +1060,13 @@ define i8 @test_reduce_v32i8(<32 x i8> %a0) {
 ; X64-SSE42:       ## %bb.0:
 ; X64-SSE42-NEXT:    pmaxub %xmm1, %xmm0
 ; X64-SSE42-NEXT:    pcmpeqd %xmm1, %xmm1
-; X64-SSE42-NEXT:    pxor %xmm1, %xmm0
-; X64-SSE42-NEXT:    movdqa %xmm0, %xmm2
-; X64-SSE42-NEXT:    psrlw $8, %xmm2
-; X64-SSE42-NEXT:    pminub %xmm0, %xmm2
-; X64-SSE42-NEXT:    phminposuw %xmm2, %xmm0
-; X64-SSE42-NEXT:    pxor %xmm1, %xmm0
+; X64-SSE42-NEXT:    pxor %xmm0, %xmm1
+; X64-SSE42-NEXT:    movdqa %xmm1, %xmm0
+; X64-SSE42-NEXT:    psrlw $8, %xmm0
+; X64-SSE42-NEXT:    pminub %xmm1, %xmm0
+; X64-SSE42-NEXT:    phminposuw %xmm0, %xmm0
 ; X64-SSE42-NEXT:    pextrb $0, %xmm0, %eax
+; X64-SSE42-NEXT:    notb %al
 ; X64-SSE42-NEXT:    ## kill: def $al killed $al killed $eax
 ; X64-SSE42-NEXT:    retq
 ;
@@ -1076,11 +1076,11 @@ define i8 @test_reduce_v32i8(<32 x i8> %a0) {
 ; X64-AVX1-NEXT:    vpmaxub %xmm1, %xmm0, %xmm0
 ; X64-AVX1-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
 ; X64-AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; X64-AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm2
-; X64-AVX1-NEXT:    vpminub %xmm2, %xmm0, %xmm0
+; X64-AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; X64-AVX1-NEXT:    vpminub %xmm1, %xmm0, %xmm0
 ; X64-AVX1-NEXT:    vphminposuw %xmm0, %xmm0
-; X64-AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; X64-AVX1-NEXT:    vpextrb $0, %xmm0, %eax
+; X64-AVX1-NEXT:    notb %al
 ; X64-AVX1-NEXT:    ## kill: def $al killed $al killed $eax
 ; X64-AVX1-NEXT:    vzeroupper
 ; X64-AVX1-NEXT:    retq
@@ -1091,11 +1091,11 @@ define i8 @test_reduce_v32i8(<32 x i8> %a0) {
 ; X64-AVX2-NEXT:    vpmaxub %xmm1, %xmm0, %xmm0
 ; X64-AVX2-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
 ; X64-AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; X64-AVX2-NEXT:    vpsrlw $8, %xmm0, %xmm2
-; X64-AVX2-NEXT:    vpminub %xmm2, %xmm0, %xmm0
+; X64-AVX2-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; X64-AVX2-NEXT:    vpminub %xmm1, %xmm0, %xmm0
 ; X64-AVX2-NEXT:    vphminposuw %xmm0, %xmm0
-; X64-AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; X64-AVX2-NEXT:    vpextrb $0, %xmm0, %eax
+; X64-AVX2-NEXT:    notb %al
 ; X64-AVX2-NEXT:    ## kill: def $al killed $al killed $eax
 ; X64-AVX2-NEXT:    vzeroupper
 ; X64-AVX2-NEXT:    retq
@@ -1108,8 +1108,8 @@ define i8 @test_reduce_v32i8(<32 x i8> %a0) {
 ; X64-AVX512-NEXT:    vpsrlw $8, %xmm0, %xmm1
 ; X64-AVX512-NEXT:    vpminub %xmm1, %xmm0, %xmm0
 ; X64-AVX512-NEXT:    vphminposuw %xmm0, %xmm0
-; X64-AVX512-NEXT:    vpternlogq $15, %xmm0, %xmm0, %xmm0
 ; X64-AVX512-NEXT:    vpextrb $0, %xmm0, %eax
+; X64-AVX512-NEXT:    notb %al
 ; X64-AVX512-NEXT:    ## kill: def $al killed $al killed $eax
 ; X64-AVX512-NEXT:    vzeroupper
 ; X64-AVX512-NEXT:    retq
@@ -1688,8 +1688,8 @@ define i16 @test_reduce_v32i16(<32 x i16> %a0) {
 ; X86-SSE2-NEXT:    psrld $16, %xmm1
 ; X86-SSE2-NEXT:    pxor %xmm4, %xmm1
 ; X86-SSE2-NEXT:    pmaxsw %xmm0, %xmm1
-; X86-SSE2-NEXT:    pxor %xmm4, %xmm1
 ; X86-SSE2-NEXT:    movd %xmm1, %eax
+; X86-SSE2-NEXT:    xorl $32768, %eax ## imm = 0x8000
 ; X86-SSE2-NEXT:    ## kill: def $ax killed $ax killed $eax
 ; X86-SSE2-NEXT:    retl
 ;
@@ -1699,10 +1699,10 @@ define i16 @test_reduce_v32i16(<32 x i16> %a0) {
 ; X86-SSE42-NEXT:    pmaxuw %xmm2, %xmm0
 ; X86-SSE42-NEXT:    pmaxuw %xmm1, %xmm0
 ; X86-SSE42-NEXT:    pcmpeqd %xmm1, %xmm1
-; X86-SSE42-NEXT:    pxor %xmm1, %xmm0
-; X86-SSE42-NEXT:    phminposuw %xmm0, %xmm0
-; X86-SSE42-NEXT:    pxor %xmm1, %xmm0
+; X86-SSE42-NEXT:    pxor %xmm0, %xmm1
+; X86-SSE42-NEXT:    phminposuw %xmm1, %xmm0
 ; X86-SSE42-NEXT:    movd %xmm0, %eax
+; X86-SSE42-NEXT:    notl %eax
 ; X86-SSE42-NEXT:    ## kill: def $ax killed $ax killed $eax
 ; X86-SSE42-NEXT:    retl
 ;
@@ -1716,8 +1716,8 @@ define i16 @test_reduce_v32i16(<32 x i16> %a0) {
 ; X86-AVX1-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
 ; X86-AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; X86-AVX1-NEXT:    vphminposuw %xmm0, %xmm0
-; X86-AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; X86-AVX1-NEXT:    vmovd %xmm0, %eax
+; X86-AVX1-NEXT:    notl %eax
 ; X86-AVX1-NEXT:    ## kill: def $ax killed $ax killed $eax
 ; X86-AVX1-NEXT:    vzeroupper
 ; X86-AVX1-NEXT:    retl
@@ -1730,8 +1730,8 @@ define i16 @test_reduce_v32i16(<32 x i16> %a0) {
 ; X86-AVX2-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
 ; X86-AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; X86-AVX2-NEXT:    vphminposuw %xmm0, %xmm0
-; X86-AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; X86-AVX2-NEXT:    vmovd %xmm0, %eax
+; X86-AVX2-NEXT:    notl %eax
 ; X86-AVX2-NEXT:    ## kill: def $ax killed $ax killed $eax
 ; X86-AVX2-NEXT:    vzeroupper
 ; X86-AVX2-NEXT:    retl
@@ -1761,8 +1761,8 @@ define i16 @test_reduce_v32i16(<32 x i16> %a0) {
 ; X64-SSE2-NEXT:    psrld $16, %xmm1
 ; X64-SSE2-NEXT:    pxor %xmm4, %xmm1
 ; X64-SSE2-NEXT:    pmaxsw %xmm0, %xmm1
-; X64-SSE2-NEXT:    pxor %xmm4, %xmm1
 ; X64-SSE2-NEXT:    movd %xmm1, %eax
+; X64-SSE2-NEXT:    xorl $32768, %eax ## imm = 0x8000
 ; X64-SSE2-NEXT:    ## kill: def $ax killed $ax killed $eax
 ; X64-SSE2-NEXT:    retq
 ;
@@ -1772,10 +1772,10 @@ define i16 @test_reduce_v32i16(<32 x i16> %a0) {
 ; X64-SSE42-NEXT:    pmaxuw %xmm2, %xmm0
 ; X64-SSE42-NEXT:    pmaxuw %xmm1, %xmm0
 ; X64-SSE42-NEXT:    pcmpeqd %xmm1, %xmm1
-; X64-SSE42-NEXT:    pxor %xmm1, %xmm0
-; X64-SSE42-NEXT:    phminposuw %xmm0, %xmm0
-; X64-SSE42-NEXT:    pxor %xmm1, %xmm0
+; X64-SSE42-NEXT:    pxor %xmm0, %xmm1
+; X64-SSE42-NEXT:    phminposuw %xmm1, %xmm0
 ; X64-SSE42-NEXT:    movd %xmm0, %eax
+; X64-SSE42-NEXT:    notl %eax
 ; X64-SSE42-NEXT:    ## kill: def $ax killed $ax killed $eax
 ; X64-SSE42-NEXT:    retq
 ;
@@ -1789,8 +1789,8 @@ define i16 @test_reduce_v32i16(<32 x i16> %a0) {
 ; X64-AVX1-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
 ; X64-AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; X64-AVX1-NEXT:    vphminposuw %xmm0, %xmm0
-; X64-AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; X64-AVX1-NEXT:    vmovd %xmm0, %eax
+; X64-AVX1-NEXT:    notl %eax
 ; X64-AVX1-NEXT:    ## kill: def $ax killed $ax killed $eax
 ; X64-AVX1-NEXT:    vzeroupper
 ; X64-AVX1-NEXT:    retq
@@ -1803,8 +1803,8 @@ define i16 @test_reduce_v32i16(<32 x i16> %a0) {
 ; X64-AVX2-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
 ; X64-AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; X64-AVX2-NEXT:    vphminposuw %xmm0, %xmm0
-; X64-AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; X64-AVX2-NEXT:    vmovd %xmm0, %eax
+; X64-AVX2-NEXT:    notl %eax
 ; X64-AVX2-NEXT:    ## kill: def $ax killed $ax killed $eax
 ; X64-AVX2-NEXT:    vzeroupper
 ; X64-AVX2-NEXT:    retq
@@ -1817,8 +1817,8 @@ define i16 @test_reduce_v32i16(<32 x i16> %a0) {
 ; X64-AVX512-NEXT:    vpmaxuw %xmm1, %xmm0, %xmm0
 ; X64-AVX512-NEXT:    vpternlogq $15, %xmm0, %xmm0, %xmm0
 ; X64-AVX512-NEXT:    vphminposuw %xmm0, %xmm0
-; X64-AVX512-NEXT:    vpternlogq $15, %xmm0, %xmm0, %xmm0
 ; X64-AVX512-NEXT:    vmovd %xmm0, %eax
+; X64-AVX512-NEXT:    notl %eax
 ; X64-AVX512-NEXT:    ## kill: def $ax killed $ax killed $eax
 ; X64-AVX512-NEXT:    vzeroupper
 ; X64-AVX512-NEXT:    retq
@@ -1867,13 +1867,13 @@ define i8 @test_reduce_v64i8(<64 x i8> %a0) {
 ; X86-SSE42-NEXT:    pmaxub %xmm2, %xmm0
 ; X86-SSE42-NEXT:    pmaxub %xmm1, %xmm0
 ; X86-SSE42-NEXT:    pcmpeqd %xmm1, %xmm1
-; X86-SSE42-NEXT:    pxor %xmm1, %xmm0
-; X86-SSE42-NEXT:    movdqa %xmm0, %xmm2
-; X86-SSE42-NEXT:    psrlw $8, %xmm2
-; X86-SSE42-NEXT:    pminub %xmm0, %xmm2
-; X86-SSE42-NEXT:    phminposuw %xmm2, %xmm0
-; X86-SSE42-NEXT:    pxor %xmm1, %xmm0
+; X86-SSE42-NEXT:    pxor %xmm0, %xmm1
+; X86-SSE42-NEXT:    movdqa %xmm1, %xmm0
+; X86-SSE42-NEXT:    psrlw $8, %xmm0
+; X86-SSE42-NEXT:    pminub %xmm1, %xmm0
+; X86-SSE42-NEXT:    phminposuw %xmm0, %xmm0
 ; X86-SSE42-NEXT:    pextrb $0, %xmm0, %eax
+; X86-SSE42-NEXT:    notb %al
 ; X86-SSE42-NEXT:    ## kill: def $al killed $al killed $eax
 ; X86-SSE42-NEXT:    retl
 ;
@@ -1886,11 +1886,11 @@ define i8 @test_reduce_v64i8(<64 x i8> %a0) {
 ; X86-AVX1-NEXT:    vpmaxub %xmm2, %xmm0, %xmm0
 ; X86-AVX1-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
 ; X86-AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; X86-AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm2
-; X86-AVX1-NEXT:    vpminub %xmm2, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; X86-AVX1-NEXT:    vpminub %xmm1, %xmm0, %xmm0
 ; X86-AVX1-NEXT:    vphminposuw %xmm0, %xmm0
-; X86-AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; X86-AVX1-NEXT:    vpextrb $0, %xmm0, %eax
+; X86-AVX1-NEXT:    notb %al
 ; X86-AVX1-NEXT:    ## kill: def $al killed $al killed $eax
 ; X86-AVX1-NEXT:    vzeroupper
 ; X86-AVX1-NEXT:    retl
@@ -1902,11 +1902,11 @@ define i8 @test_reduce_v64i8(<64 x i8> %a0) {
 ; X86-AVX2-NEXT:    vpmaxub %xmm1, %xmm0, %xmm0
 ; X86-AVX2-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
 ; X86-AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; X86-AVX2-NEXT:    vpsrlw $8, %xmm0, %xmm2
-; X86-AVX2-NEXT:    vpminub %xmm2, %xmm0, %xmm0
+; X86-AVX2-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; X86-AVX2-NEXT:    vpminub %xmm1, %xmm0, %xmm0
 ; X86-AVX2-NEXT:    vphminposuw %xmm0, %xmm0
-; X86-AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; X86-AVX2-NEXT:    vpextrb $0, %xmm0, %eax
+; X86-AVX2-NEXT:    notb %al
 ; X86-AVX2-NEXT:    ## kill: def $al killed $al killed $eax
 ; X86-AVX2-NEXT:    vzeroupper
 ; X86-AVX2-NEXT:    retl
@@ -1936,13 +1936,13 @@ define i8 @test_reduce_v64i8(<64 x i8> %a0) {
 ; X64-SSE42-NEXT:    pmaxub %xmm2, %xmm0
 ; X64-SSE42-NEXT:    pmaxub %xmm1, %xmm0
 ; X64-SSE42-NEXT:    pcmpeqd %xmm1, %xmm1
-; X64-SSE42-NEXT:    pxor %xmm1, %xmm0
-; X64-SSE42-NEXT:    movdqa %xmm0, %xmm2
-; X64-SSE42-NEXT:    psrlw $8, %xmm2
-; X64-SSE42-NEXT:    pminub %xmm0, %xmm2
-; X64-SSE42-NEXT:    phminposuw %xmm2, %xmm0
-; X64-SSE42-NEXT:    pxor %xmm1, %xmm0
+; X64-SSE42-NEXT:    pxor %xmm0, %xmm1
+; X64-SSE42-NEXT:    movdqa %xmm1, %xmm0
+; X64-SSE42-NEXT:    psrlw $8, %xmm0
+; X64-SSE42-NEXT:    pminub %xmm1, %xmm0
+; X64-SSE42-NEXT:    phminposuw %xmm0, %xmm0
 ; X64-SSE42-NEXT:    pextrb $0, %xmm0, %eax
+; X64-SSE42-NEXT:    notb %al
 ; X64-SSE42-NEXT:    ## kill: def $al killed $al killed $eax
 ; X64-SSE42-NEXT:    retq
 ;
@@ -1955,11 +1955,11 @@ define i8 @test_reduce_v64i8(<64 x i8> %a0) {
 ; X64-AVX1-NEXT:    vpmaxub %xmm2, %xmm0, %xmm0
 ; X64-AVX1-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
 ; X64-AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; X64-AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm2
-; X64-AVX1-NEXT:    vpminub %xmm2, %xmm0, %xmm0
+; X64-AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; X64-AVX1-NEXT:    vpminub %xmm1, %xmm0, %xmm0
 ; X64-AVX1-NEXT:    vphminposuw %xmm0, %xmm0
-; X64-AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; X64-AVX1-NEXT:    vpextrb $0, %xmm0, %eax
+; X64-AVX1-NEXT:    notb %al
 ; X64-AVX1-NEXT:    ## kill: def $al killed $al killed $eax
 ; X64-AVX1-NEXT:    vzeroupper
 ; X64-AVX1-NEXT:    retq
@@ -1971,11 +1971,11 @@ define i8 @test_reduce_v64i8(<64 x i8> %a0) {
 ; X64-AVX2-NEXT:    vpmaxub %xmm1, %xmm0, %xmm0
 ; X64-AVX2-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
 ; X64-AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; X64-AVX2-NEXT:    vpsrlw $8, %xmm0, %xmm2
-; X64-AVX2-NEXT:    vpminub %xmm2, %xmm0, %xmm0
+; X64-AVX2-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; X64-AVX2-NEXT:    vpminub %xmm1, %xmm0, %xmm0
 ; X64-AVX2-NEXT:    vphminposuw %xmm0, %xmm0
-; X64-AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; X64-AVX2-NEXT:    vpextrb $0, %xmm0, %eax
+; X64-AVX2-NEXT:    notb %al
 ; X64-AVX2-NEXT:    ## kill: def $al killed $al killed $eax
 ; X64-AVX2-NEXT:    vzeroupper
 ; X64-AVX2-NEXT:    retq
@@ -1990,8 +1990,8 @@ define i8 @test_reduce_v64i8(<64 x i8> %a0) {
 ; X64-AVX512-NEXT:    vpsrlw $8, %xmm0, %xmm1
 ; X64-AVX512-NEXT:    vpminub %xmm1, %xmm0, %xmm0
 ; X64-AVX512-NEXT:    vphminposuw %xmm0, %xmm0
-; X64-AVX512-NEXT:    vpternlogq $15, %xmm0, %xmm0, %xmm0
 ; X64-AVX512-NEXT:    vpextrb $0, %xmm0, %eax
+; X64-AVX512-NEXT:    notb %al
 ; X64-AVX512-NEXT:    ## kill: def $al killed $al killed $eax
 ; X64-AVX512-NEXT:    vzeroupper
 ; X64-AVX512-NEXT:    retq
diff --git a/test/CodeGen/X86/horizontal-reduce-umin.ll b/test/CodeGen/X86/horizontal-reduce-umin.ll
index 482d08260374e4c94426b7a744961e6f32abb09f..e4b92239c23f982b8e600044b95080cd108635d7 100644
--- a/test/CodeGen/X86/horizontal-reduce-umin.ll
+++ b/test/CodeGen/X86/horizontal-reduce-umin.ll
@@ -242,8 +242,8 @@ define i16 @test_reduce_v8i16(<8 x i16> %a0) {
 ; X86-SSE2-NEXT:    psrld $16, %xmm1
 ; X86-SSE2-NEXT:    pxor %xmm2, %xmm1
 ; X86-SSE2-NEXT:    pminsw %xmm0, %xmm1
-; X86-SSE2-NEXT:    pxor %xmm2, %xmm1
 ; X86-SSE2-NEXT:    movd %xmm1, %eax
+; X86-SSE2-NEXT:    xorl $32768, %eax ## imm = 0x8000
 ; X86-SSE2-NEXT:    ## kill: def $ax killed $ax killed $eax
 ; X86-SSE2-NEXT:    retl
 ;
@@ -278,8 +278,8 @@ define i16 @test_reduce_v8i16(<8 x i16> %a0) {
 ; X64-SSE2-NEXT:    psrld $16, %xmm1
 ; X64-SSE2-NEXT:    pxor %xmm2, %xmm1
 ; X64-SSE2-NEXT:    pminsw %xmm0, %xmm1
-; X64-SSE2-NEXT:    pxor %xmm2, %xmm1
 ; X64-SSE2-NEXT:    movd %xmm1, %eax
+; X64-SSE2-NEXT:    xorl $32768, %eax ## imm = 0x8000
 ; X64-SSE2-NEXT:    ## kill: def $ax killed $ax killed $eax
 ; X64-SSE2-NEXT:    retq
 ;
@@ -785,8 +785,8 @@ define i16 @test_reduce_v16i16(<16 x i16> %a0) {
 ; X86-SSE2-NEXT:    psrld $16, %xmm1
 ; X86-SSE2-NEXT:    pxor %xmm2, %xmm1
 ; X86-SSE2-NEXT:    pminsw %xmm0, %xmm1
-; X86-SSE2-NEXT:    pxor %xmm2, %xmm1
 ; X86-SSE2-NEXT:    movd %xmm1, %eax
+; X86-SSE2-NEXT:    xorl $32768, %eax ## imm = 0x8000
 ; X86-SSE2-NEXT:    ## kill: def $ax killed $ax killed $eax
 ; X86-SSE2-NEXT:    retl
 ;
@@ -839,8 +839,8 @@ define i16 @test_reduce_v16i16(<16 x i16> %a0) {
 ; X64-SSE2-NEXT:    psrld $16, %xmm1
 ; X64-SSE2-NEXT:    pxor %xmm2, %xmm1
 ; X64-SSE2-NEXT:    pminsw %xmm0, %xmm1
-; X64-SSE2-NEXT:    pxor %xmm2, %xmm1
 ; X64-SSE2-NEXT:    movd %xmm1, %eax
+; X64-SSE2-NEXT:    xorl $32768, %eax ## imm = 0x8000
 ; X64-SSE2-NEXT:    ## kill: def $ax killed $ax killed $eax
 ; X64-SSE2-NEXT:    retq
 ;
@@ -1592,8 +1592,8 @@ define i16 @test_reduce_v32i16(<32 x i16> %a0) {
 ; X86-SSE2-NEXT:    psrld $16, %xmm1
 ; X86-SSE2-NEXT:    pxor %xmm4, %xmm1
 ; X86-SSE2-NEXT:    pminsw %xmm0, %xmm1
-; X86-SSE2-NEXT:    pxor %xmm4, %xmm1
 ; X86-SSE2-NEXT:    movd %xmm1, %eax
+; X86-SSE2-NEXT:    xorl $32768, %eax ## imm = 0x8000
 ; X86-SSE2-NEXT:    ## kill: def $ax killed $ax killed $eax
 ; X86-SSE2-NEXT:    retl
 ;
@@ -1656,8 +1656,8 @@ define i16 @test_reduce_v32i16(<32 x i16> %a0) {
 ; X64-SSE2-NEXT:    psrld $16, %xmm1
 ; X64-SSE2-NEXT:    pxor %xmm4, %xmm1
 ; X64-SSE2-NEXT:    pminsw %xmm0, %xmm1
-; X64-SSE2-NEXT:    pxor %xmm4, %xmm1
 ; X64-SSE2-NEXT:    movd %xmm1, %eax
+; X64-SSE2-NEXT:    xorl $32768, %eax ## imm = 0x8000
 ; X64-SSE2-NEXT:    ## kill: def $ax killed $ax killed $eax
 ; X64-SSE2-NEXT:    retq
 ;
diff --git a/test/CodeGen/X86/i16lshr8pat.ll b/test/CodeGen/X86/i16lshr8pat.ll
index 425680f9d05c32575587e5e3d7c03b8d0d90f8aa..68974b72a88c31f273205629a23237be606d5930 100644
--- a/test/CodeGen/X86/i16lshr8pat.ll
+++ b/test/CodeGen/X86/i16lshr8pat.ll
@@ -1,4 +1,4 @@
-; RUN: llc -stop-after expand-isel-pseudos <%s 2>&1 | FileCheck %s
+; RUN: llc -stop-after expand-isel-pseudos < %s 2>&1 | FileCheck %s
 
 target datalayout = "e-m:e-p:32:32-f64:32:64-f80:32-n8:16:32-S128"
 target triple = "i386-unknown-linux-gnu"
diff --git a/test/CodeGen/X86/inconsistent_landingpad.ll b/test/CodeGen/X86/inconsistent_landingpad.ll
index 495e999c4a955228deddc860a7a9035c137654a2..9979c5e50bad5904789af869b5e5e2dcc0b59c7a 100644
--- a/test/CodeGen/X86/inconsistent_landingpad.ll
+++ b/test/CodeGen/X86/inconsistent_landingpad.ll
@@ -1,4 +1,4 @@
-; RUN: not llvm-as -disable-output <%s 2>&1 | FileCheck %s
+; RUN: not llvm-as -disable-output < %s 2>&1 | FileCheck %s
 
 define void @test() personality i32 (...)* @dummy_personality {
 ; CHECK: The landingpad instruction should have a consistent result type inside a function
diff --git a/test/CodeGen/X86/inline-asm-default-clobbers.ll b/test/CodeGen/X86/inline-asm-default-clobbers.ll
new file mode 100644
index 0000000000000000000000000000000000000000..34a77ea5fecdbe532f15ab1d8daacc4c27d24095
--- /dev/null
+++ b/test/CodeGen/X86/inline-asm-default-clobbers.ll
@@ -0,0 +1,8 @@
+; RUN: llc < %s -mtriple=i686 -stop-after=expand-isel-pseudos | FileCheck %s
+
+; CHECK: INLINEASM &"", 1, 12, implicit-def early-clobber $df, 12, implicit-def early-clobber $fpsw, 12, implicit-def early-clobber $eflags
+define void @foo() {
+entry:
+  call void asm sideeffect "", "~{dirflag},~{fpsr},~{flags}"()
+  ret void
+}
diff --git a/test/CodeGen/X86/inline-asm-flag-output.ll b/test/CodeGen/X86/inline-asm-flag-output.ll
new file mode 100644
index 0000000000000000000000000000000000000000..67aa3b828f9a070a4320c7e4c32b9025f35c7c8b
--- /dev/null
+++ b/test/CodeGen/X86/inline-asm-flag-output.ll
@@ -0,0 +1,954 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686--   -no-integrated-as | FileCheck %s -check-prefix=X32
+; RUN: llc < %s -mtriple=x86_64-- -no-integrated-as | FileCheck %s -check-prefix=X64
+
+define i32 @test_cca(i64 %nr, i64* %addr) {
+; X32-LABEL: test_cca:
+; X32:       # %bb.0: # %entry
+; X32-NEXT:    pushl %esi
+; X32-NEXT:    .cfi_def_cfa_offset 8
+; X32-NEXT:    .cfi_offset %esi, -8
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X32-NEXT:    xorl %eax, %eax
+; X32-NEXT:    #APP
+; X32-NEXT:    cmp %ecx,(%esi)
+; X32-NEXT:    #NO_APP
+; X32-NEXT:    setbe %al
+; X32-NEXT:    popl %esi
+; X32-NEXT:    .cfi_def_cfa_offset 4
+; X32-NEXT:    retl
+;
+; X64-LABEL: test_cca:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    #APP
+; X64-NEXT:    cmp %rdi,(%rsi)
+; X64-NEXT:    #NO_APP
+; X64-NEXT:    setbe %al
+; X64-NEXT:    retq
+entry:
+  %cc = tail call i32 asm "cmp $2,$1", "={@cca},=*m,r,~{cc},~{dirflag},~{fpsr},~{flags}"(i64* %addr, i64 %nr) nounwind
+  %tobool = icmp eq i32 %cc, 0
+  %rv = zext i1 %tobool to i32
+  ret i32 %rv
+}
+
+
+define i32 @test_ccae(i64 %nr, i64* %addr) {
+; X32-LABEL: test_ccae:
+; X32:       # %bb.0: # %entry
+; X32-NEXT:    pushl %esi
+; X32-NEXT:    .cfi_def_cfa_offset 8
+; X32-NEXT:    .cfi_offset %esi, -8
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X32-NEXT:    xorl %eax, %eax
+; X32-NEXT:    #APP
+; X32-NEXT:    cmp %ecx,(%esi)
+; X32-NEXT:    #NO_APP
+; X32-NEXT:    setb %al
+; X32-NEXT:    popl %esi
+; X32-NEXT:    .cfi_def_cfa_offset 4
+; X32-NEXT:    retl
+;
+; X64-LABEL: test_ccae:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    #APP
+; X64-NEXT:    cmp %rdi,(%rsi)
+; X64-NEXT:    #NO_APP
+; X64-NEXT:    setb %al
+; X64-NEXT:    retq
+entry:
+  %cc = tail call i32 asm "cmp $2,$1", "={@ccae},=*m,r,~{cc},~{dirflag},~{fpsr},~{flags}"(i64* %addr, i64 %nr) nounwind
+  %tobool = icmp eq i32 %cc, 0
+  %rv = zext i1 %tobool to i32
+  ret i32 %rv
+}
+
+
+define i32 @test_ccb(i64 %nr, i64* %addr) {
+; X32-LABEL: test_ccb:
+; X32:       # %bb.0: # %entry
+; X32-NEXT:    pushl %esi
+; X32-NEXT:    .cfi_def_cfa_offset 8
+; X32-NEXT:    .cfi_offset %esi, -8
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X32-NEXT:    xorl %eax, %eax
+; X32-NEXT:    #APP
+; X32-NEXT:    cmp %ecx,(%esi)
+; X32-NEXT:    #NO_APP
+; X32-NEXT:    setae %al
+; X32-NEXT:    popl %esi
+; X32-NEXT:    .cfi_def_cfa_offset 4
+; X32-NEXT:    retl
+;
+; X64-LABEL: test_ccb:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    #APP
+; X64-NEXT:    cmp %rdi,(%rsi)
+; X64-NEXT:    #NO_APP
+; X64-NEXT:    setae %al
+; X64-NEXT:    retq
+entry:
+  %cc = tail call i32 asm "cmp $2,$1", "={@ccb},=*m,r,~{cc},~{dirflag},~{fpsr},~{flags}"(i64* %addr, i64 %nr) nounwind
+  %tobool = icmp eq i32 %cc, 0
+  %rv = zext i1 %tobool to i32
+  ret i32 %rv
+}
+
+
+define i32 @test_ccbe(i64 %nr, i64* %addr) {
+; X32-LABEL: test_ccbe:
+; X32:       # %bb.0: # %entry
+; X32-NEXT:    pushl %esi
+; X32-NEXT:    .cfi_def_cfa_offset 8
+; X32-NEXT:    .cfi_offset %esi, -8
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X32-NEXT:    xorl %eax, %eax
+; X32-NEXT:    #APP
+; X32-NEXT:    cmp %ecx,(%esi)
+; X32-NEXT:    #NO_APP
+; X32-NEXT:    seta %al
+; X32-NEXT:    popl %esi
+; X32-NEXT:    .cfi_def_cfa_offset 4
+; X32-NEXT:    retl
+;
+; X64-LABEL: test_ccbe:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    #APP
+; X64-NEXT:    cmp %rdi,(%rsi)
+; X64-NEXT:    #NO_APP
+; X64-NEXT:    seta %al
+; X64-NEXT:    retq
+entry:
+  %cc = tail call i32 asm "cmp $2,$1", "={@ccbe},=*m,r,~{cc},~{dirflag},~{fpsr},~{flags}"(i64* %addr, i64 %nr) nounwind
+  %tobool = icmp eq i32 %cc, 0
+  %rv = zext i1 %tobool to i32
+  ret i32 %rv
+}
+
+
+define i32 @test_ccc(i64 %nr, i64* %addr) {
+; X32-LABEL: test_ccc:
+; X32:       # %bb.0: # %entry
+; X32-NEXT:    pushl %esi
+; X32-NEXT:    .cfi_def_cfa_offset 8
+; X32-NEXT:    .cfi_offset %esi, -8
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X32-NEXT:    xorl %eax, %eax
+; X32-NEXT:    #APP
+; X32-NEXT:    cmp %ecx,(%esi)
+; X32-NEXT:    #NO_APP
+; X32-NEXT:    setae %al
+; X32-NEXT:    popl %esi
+; X32-NEXT:    .cfi_def_cfa_offset 4
+; X32-NEXT:    retl
+;
+; X64-LABEL: test_ccc:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    #APP
+; X64-NEXT:    cmp %rdi,(%rsi)
+; X64-NEXT:    #NO_APP
+; X64-NEXT:    setae %al
+; X64-NEXT:    retq
+entry:
+  %cc = tail call i32 asm "cmp $2,$1", "={@ccc},=*m,r,~{cc},~{dirflag},~{fpsr},~{flags}"(i64* %addr, i64 %nr) nounwind
+  %tobool = icmp eq i32 %cc, 0
+  %rv = zext i1 %tobool to i32
+  ret i32 %rv
+}
+
+
+define i32 @test_cce(i64 %nr, i64* %addr) {
+; X32-LABEL: test_cce:
+; X32:       # %bb.0: # %entry
+; X32-NEXT:    pushl %esi
+; X32-NEXT:    .cfi_def_cfa_offset 8
+; X32-NEXT:    .cfi_offset %esi, -8
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X32-NEXT:    xorl %eax, %eax
+; X32-NEXT:    #APP
+; X32-NEXT:    cmp %ecx,(%esi)
+; X32-NEXT:    #NO_APP
+; X32-NEXT:    setne %al
+; X32-NEXT:    popl %esi
+; X32-NEXT:    .cfi_def_cfa_offset 4
+; X32-NEXT:    retl
+;
+; X64-LABEL: test_cce:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    #APP
+; X64-NEXT:    cmp %rdi,(%rsi)
+; X64-NEXT:    #NO_APP
+; X64-NEXT:    setne %al
+; X64-NEXT:    retq
+entry:
+  %cc = tail call i32 asm "cmp $2,$1", "={@cce},=*m,r,~{cc},~{dirflag},~{fpsr},~{flags}"(i64* %addr, i64 %nr) nounwind
+  %tobool = icmp eq i32 %cc, 0
+  %rv = zext i1 %tobool to i32
+  ret i32 %rv
+}
+
+
+define i32 @test_ccz(i64 %nr, i64* %addr) {
+; X32-LABEL: test_ccz:
+; X32:       # %bb.0: # %entry
+; X32-NEXT:    pushl %esi
+; X32-NEXT:    .cfi_def_cfa_offset 8
+; X32-NEXT:    .cfi_offset %esi, -8
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X32-NEXT:    xorl %eax, %eax
+; X32-NEXT:    #APP
+; X32-NEXT:    cmp %ecx,(%esi)
+; X32-NEXT:    #NO_APP
+; X32-NEXT:    sete %al
+; X32-NEXT:    popl %esi
+; X32-NEXT:    .cfi_def_cfa_offset 4
+; X32-NEXT:    retl
+;
+; X64-LABEL: test_ccz:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    #APP
+; X64-NEXT:    cmp %rdi,(%rsi)
+; X64-NEXT:    #NO_APP
+; X64-NEXT:    sete %al
+; X64-NEXT:    retq
+entry:
+  %cc = tail call i32 asm "cmp $2,$1", "={@ccz},=*m,r,~{cc},~{dirflag},~{fpsr},~{flags}"(i64* %addr, i64 %nr) nounwind
+  %tobool = icmp eq i32 %cc, 0
+  %rv = zext i1 %tobool to i32
+  ret i32 %rv
+}
+
+
+define i32 @test_ccg(i64 %nr, i64* %addr) {
+; X32-LABEL: test_ccg:
+; X32:       # %bb.0: # %entry
+; X32-NEXT:    pushl %esi
+; X32-NEXT:    .cfi_def_cfa_offset 8
+; X32-NEXT:    .cfi_offset %esi, -8
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X32-NEXT:    xorl %eax, %eax
+; X32-NEXT:    #APP
+; X32-NEXT:    cmp %ecx,(%esi)
+; X32-NEXT:    #NO_APP
+; X32-NEXT:    setle %al
+; X32-NEXT:    popl %esi
+; X32-NEXT:    .cfi_def_cfa_offset 4
+; X32-NEXT:    retl
+;
+; X64-LABEL: test_ccg:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    #APP
+; X64-NEXT:    cmp %rdi,(%rsi)
+; X64-NEXT:    #NO_APP
+; X64-NEXT:    setle %al
+; X64-NEXT:    retq
+entry:
+  %cc = tail call i32 asm "cmp $2,$1", "={@ccg},=*m,r,~{cc},~{dirflag},~{fpsr},~{flags}"(i64* %addr, i64 %nr) nounwind
+  %tobool = icmp eq i32 %cc, 0
+  %rv = zext i1 %tobool to i32
+  ret i32 %rv
+}
+
+
+define i32 @test_ccge(i64 %nr, i64* %addr) {
+; X32-LABEL: test_ccge:
+; X32:       # %bb.0: # %entry
+; X32-NEXT:    pushl %esi
+; X32-NEXT:    .cfi_def_cfa_offset 8
+; X32-NEXT:    .cfi_offset %esi, -8
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X32-NEXT:    xorl %eax, %eax
+; X32-NEXT:    #APP
+; X32-NEXT:    cmp %ecx,(%esi)
+; X32-NEXT:    #NO_APP
+; X32-NEXT:    setl %al
+; X32-NEXT:    popl %esi
+; X32-NEXT:    .cfi_def_cfa_offset 4
+; X32-NEXT:    retl
+;
+; X64-LABEL: test_ccge:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    #APP
+; X64-NEXT:    cmp %rdi,(%rsi)
+; X64-NEXT:    #NO_APP
+; X64-NEXT:    setl %al
+; X64-NEXT:    retq
+entry:
+  %cc = tail call i32 asm "cmp $2,$1", "={@ccge},=*m,r,~{cc},~{dirflag},~{fpsr},~{flags}"(i64* %addr, i64 %nr) nounwind
+  %tobool = icmp eq i32 %cc, 0
+  %rv = zext i1 %tobool to i32
+  ret i32 %rv
+}
+
+
+define i32 @test_ccl(i64 %nr, i64* %addr) {
+; X32-LABEL: test_ccl:
+; X32:       # %bb.0: # %entry
+; X32-NEXT:    pushl %esi
+; X32-NEXT:    .cfi_def_cfa_offset 8
+; X32-NEXT:    .cfi_offset %esi, -8
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X32-NEXT:    xorl %eax, %eax
+; X32-NEXT:    #APP
+; X32-NEXT:    cmp %ecx,(%esi)
+; X32-NEXT:    #NO_APP
+; X32-NEXT:    setge %al
+; X32-NEXT:    popl %esi
+; X32-NEXT:    .cfi_def_cfa_offset 4
+; X32-NEXT:    retl
+;
+; X64-LABEL: test_ccl:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    #APP
+; X64-NEXT:    cmp %rdi,(%rsi)
+; X64-NEXT:    #NO_APP
+; X64-NEXT:    setge %al
+; X64-NEXT:    retq
+entry:
+  %cc = tail call i32 asm "cmp $2,$1", "={@ccl},=*m,r,~{cc},~{dirflag},~{fpsr},~{flags}"(i64* %addr, i64 %nr) nounwind
+  %tobool = icmp eq i32 %cc, 0
+  %rv = zext i1 %tobool to i32
+  ret i32 %rv
+}
+
+
+define i32 @test_ccle(i64 %nr, i64* %addr) {
+; X32-LABEL: test_ccle:
+; X32:       # %bb.0: # %entry
+; X32-NEXT:    pushl %esi
+; X32-NEXT:    .cfi_def_cfa_offset 8
+; X32-NEXT:    .cfi_offset %esi, -8
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X32-NEXT:    xorl %eax, %eax
+; X32-NEXT:    #APP
+; X32-NEXT:    cmp %ecx,(%esi)
+; X32-NEXT:    #NO_APP
+; X32-NEXT:    setg %al
+; X32-NEXT:    popl %esi
+; X32-NEXT:    .cfi_def_cfa_offset 4
+; X32-NEXT:    retl
+;
+; X64-LABEL: test_ccle:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    #APP
+; X64-NEXT:    cmp %rdi,(%rsi)
+; X64-NEXT:    #NO_APP
+; X64-NEXT:    setg %al
+; X64-NEXT:    retq
+entry:
+  %cc = tail call i32 asm "cmp $2,$1", "={@ccle},=*m,r,~{cc},~{dirflag},~{fpsr},~{flags}"(i64* %addr, i64 %nr) nounwind
+  %tobool = icmp eq i32 %cc, 0
+  %rv = zext i1 %tobool to i32
+  ret i32 %rv
+}
+
+
+define i32 @test_ccna(i64 %nr, i64* %addr) {
+; X32-LABEL: test_ccna:
+; X32:       # %bb.0: # %entry
+; X32-NEXT:    pushl %esi
+; X32-NEXT:    .cfi_def_cfa_offset 8
+; X32-NEXT:    .cfi_offset %esi, -8
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X32-NEXT:    xorl %eax, %eax
+; X32-NEXT:    #APP
+; X32-NEXT:    cmp %ecx,(%esi)
+; X32-NEXT:    #NO_APP
+; X32-NEXT:    seta %al
+; X32-NEXT:    popl %esi
+; X32-NEXT:    .cfi_def_cfa_offset 4
+; X32-NEXT:    retl
+;
+; X64-LABEL: test_ccna:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    #APP
+; X64-NEXT:    cmp %rdi,(%rsi)
+; X64-NEXT:    #NO_APP
+; X64-NEXT:    seta %al
+; X64-NEXT:    retq
+entry:
+  %cc = tail call i32 asm "cmp $2,$1", "={@ccna},=*m,r,~{cc},~{dirflag},~{fpsr},~{flags}"(i64* %addr, i64 %nr) nounwind
+  %tobool = icmp eq i32 %cc, 0
+  %rv = zext i1 %tobool to i32
+  ret i32 %rv
+}
+
+
+define i32 @test_ccnae(i64 %nr, i64* %addr) {
+; X32-LABEL: test_ccnae:
+; X32:       # %bb.0: # %entry
+; X32-NEXT:    pushl %esi
+; X32-NEXT:    .cfi_def_cfa_offset 8
+; X32-NEXT:    .cfi_offset %esi, -8
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X32-NEXT:    xorl %eax, %eax
+; X32-NEXT:    #APP
+; X32-NEXT:    cmp %ecx,(%esi)
+; X32-NEXT:    #NO_APP
+; X32-NEXT:    setae %al
+; X32-NEXT:    popl %esi
+; X32-NEXT:    .cfi_def_cfa_offset 4
+; X32-NEXT:    retl
+;
+; X64-LABEL: test_ccnae:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    #APP
+; X64-NEXT:    cmp %rdi,(%rsi)
+; X64-NEXT:    #NO_APP
+; X64-NEXT:    setae %al
+; X64-NEXT:    retq
+entry:
+  %cc = tail call i32 asm "cmp $2,$1", "={@ccnae},=*m,r,~{cc},~{dirflag},~{fpsr},~{flags}"(i64* %addr, i64 %nr) nounwind
+  %tobool = icmp eq i32 %cc, 0
+  %rv = zext i1 %tobool to i32
+  ret i32 %rv
+}
+
+
+define i32 @test_ccnb(i64 %nr, i64* %addr) {
+; X32-LABEL: test_ccnb:
+; X32:       # %bb.0: # %entry
+; X32-NEXT:    pushl %esi
+; X32-NEXT:    .cfi_def_cfa_offset 8
+; X32-NEXT:    .cfi_offset %esi, -8
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X32-NEXT:    xorl %eax, %eax
+; X32-NEXT:    #APP
+; X32-NEXT:    cmp %ecx,(%esi)
+; X32-NEXT:    #NO_APP
+; X32-NEXT:    setb %al
+; X32-NEXT:    popl %esi
+; X32-NEXT:    .cfi_def_cfa_offset 4
+; X32-NEXT:    retl
+;
+; X64-LABEL: test_ccnb:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    #APP
+; X64-NEXT:    cmp %rdi,(%rsi)
+; X64-NEXT:    #NO_APP
+; X64-NEXT:    setb %al
+; X64-NEXT:    retq
+entry:
+  %cc = tail call i32 asm "cmp $2,$1", "={@ccnb},=*m,r,~{cc},~{dirflag},~{fpsr},~{flags}"(i64* %addr, i64 %nr) nounwind
+  %tobool = icmp eq i32 %cc, 0
+  %rv = zext i1 %tobool to i32
+  ret i32 %rv
+}
+
+
+define i32 @test_ccnbe(i64 %nr, i64* %addr) {
+; X32-LABEL: test_ccnbe:
+; X32:       # %bb.0: # %entry
+; X32-NEXT:    pushl %esi
+; X32-NEXT:    .cfi_def_cfa_offset 8
+; X32-NEXT:    .cfi_offset %esi, -8
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X32-NEXT:    xorl %eax, %eax
+; X32-NEXT:    #APP
+; X32-NEXT:    cmp %ecx,(%esi)
+; X32-NEXT:    #NO_APP
+; X32-NEXT:    setbe %al
+; X32-NEXT:    popl %esi
+; X32-NEXT:    .cfi_def_cfa_offset 4
+; X32-NEXT:    retl
+;
+; X64-LABEL: test_ccnbe:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    #APP
+; X64-NEXT:    cmp %rdi,(%rsi)
+; X64-NEXT:    #NO_APP
+; X64-NEXT:    setbe %al
+; X64-NEXT:    retq
+entry:
+  %cc = tail call i32 asm "cmp $2,$1", "={@ccnbe},=*m,r,~{cc},~{dirflag},~{fpsr},~{flags}"(i64* %addr, i64 %nr) nounwind
+  %tobool = icmp eq i32 %cc, 0
+  %rv = zext i1 %tobool to i32
+  ret i32 %rv
+}
+
+
+define i32 @test_ccnc(i64 %nr, i64* %addr) {
+; X32-LABEL: test_ccnc:
+; X32:       # %bb.0: # %entry
+; X32-NEXT:    pushl %esi
+; X32-NEXT:    .cfi_def_cfa_offset 8
+; X32-NEXT:    .cfi_offset %esi, -8
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X32-NEXT:    xorl %eax, %eax
+; X32-NEXT:    #APP
+; X32-NEXT:    cmp %ecx,(%esi)
+; X32-NEXT:    #NO_APP
+; X32-NEXT:    setb %al
+; X32-NEXT:    popl %esi
+; X32-NEXT:    .cfi_def_cfa_offset 4
+; X32-NEXT:    retl
+;
+; X64-LABEL: test_ccnc:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    #APP
+; X64-NEXT:    cmp %rdi,(%rsi)
+; X64-NEXT:    #NO_APP
+; X64-NEXT:    setb %al
+; X64-NEXT:    retq
+entry:
+  %cc = tail call i32 asm "cmp $2,$1", "={@ccnc},=*m,r,~{cc},~{dirflag},~{fpsr},~{flags}"(i64* %addr, i64 %nr) nounwind
+  %tobool = icmp eq i32 %cc, 0
+  %rv = zext i1 %tobool to i32
+  ret i32 %rv
+}
+
+
+define i32 @test_ccne(i64 %nr, i64* %addr) {
+; X32-LABEL: test_ccne:
+; X32:       # %bb.0: # %entry
+; X32-NEXT:    pushl %esi
+; X32-NEXT:    .cfi_def_cfa_offset 8
+; X32-NEXT:    .cfi_offset %esi, -8
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X32-NEXT:    xorl %eax, %eax
+; X32-NEXT:    #APP
+; X32-NEXT:    cmp %ecx,(%esi)
+; X32-NEXT:    #NO_APP
+; X32-NEXT:    sete %al
+; X32-NEXT:    popl %esi
+; X32-NEXT:    .cfi_def_cfa_offset 4
+; X32-NEXT:    retl
+;
+; X64-LABEL: test_ccne:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    #APP
+; X64-NEXT:    cmp %rdi,(%rsi)
+; X64-NEXT:    #NO_APP
+; X64-NEXT:    sete %al
+; X64-NEXT:    retq
+entry:
+  %cc = tail call i32 asm "cmp $2,$1", "={@ccne},=*m,r,~{cc},~{dirflag},~{fpsr},~{flags}"(i64* %addr, i64 %nr) nounwind
+  %tobool = icmp eq i32 %cc, 0
+  %rv = zext i1 %tobool to i32
+  ret i32 %rv
+}
+
+
+define i32 @test_ccnz(i64 %nr, i64* %addr) {
+; X32-LABEL: test_ccnz:
+; X32:       # %bb.0: # %entry
+; X32-NEXT:    pushl %esi
+; X32-NEXT:    .cfi_def_cfa_offset 8
+; X32-NEXT:    .cfi_offset %esi, -8
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X32-NEXT:    xorl %eax, %eax
+; X32-NEXT:    #APP
+; X32-NEXT:    cmp %ecx,(%esi)
+; X32-NEXT:    #NO_APP
+; X32-NEXT:    sete %al
+; X32-NEXT:    popl %esi
+; X32-NEXT:    .cfi_def_cfa_offset 4
+; X32-NEXT:    retl
+;
+; X64-LABEL: test_ccnz:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    #APP
+; X64-NEXT:    cmp %rdi,(%rsi)
+; X64-NEXT:    #NO_APP
+; X64-NEXT:    sete %al
+; X64-NEXT:    retq
+entry:
+  %cc = tail call i32 asm "cmp $2,$1", "={@ccnz},=*m,r,~{cc},~{dirflag},~{fpsr},~{flags}"(i64* %addr, i64 %nr) nounwind
+  %tobool = icmp eq i32 %cc, 0
+  %rv = zext i1 %tobool to i32
+  ret i32 %rv
+}
+
+
+define i32 @test_ccng(i64 %nr, i64* %addr) {
+; X32-LABEL: test_ccng:
+; X32:       # %bb.0: # %entry
+; X32-NEXT:    pushl %esi
+; X32-NEXT:    .cfi_def_cfa_offset 8
+; X32-NEXT:    .cfi_offset %esi, -8
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X32-NEXT:    xorl %eax, %eax
+; X32-NEXT:    #APP
+; X32-NEXT:    cmp %ecx,(%esi)
+; X32-NEXT:    #NO_APP
+; X32-NEXT:    setg %al
+; X32-NEXT:    popl %esi
+; X32-NEXT:    .cfi_def_cfa_offset 4
+; X32-NEXT:    retl
+;
+; X64-LABEL: test_ccng:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    #APP
+; X64-NEXT:    cmp %rdi,(%rsi)
+; X64-NEXT:    #NO_APP
+; X64-NEXT:    setg %al
+; X64-NEXT:    retq
+entry:
+  %cc = tail call i32 asm "cmp $2,$1", "={@ccng},=*m,r,~{cc},~{dirflag},~{fpsr},~{flags}"(i64* %addr, i64 %nr) nounwind
+  %tobool = icmp eq i32 %cc, 0
+  %rv = zext i1 %tobool to i32
+  ret i32 %rv
+}
+
+
+define i32 @test_ccnge(i64 %nr, i64* %addr) {
+; X32-LABEL: test_ccnge:
+; X32:       # %bb.0: # %entry
+; X32-NEXT:    pushl %esi
+; X32-NEXT:    .cfi_def_cfa_offset 8
+; X32-NEXT:    .cfi_offset %esi, -8
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X32-NEXT:    xorl %eax, %eax
+; X32-NEXT:    #APP
+; X32-NEXT:    cmp %ecx,(%esi)
+; X32-NEXT:    #NO_APP
+; X32-NEXT:    setge %al
+; X32-NEXT:    popl %esi
+; X32-NEXT:    .cfi_def_cfa_offset 4
+; X32-NEXT:    retl
+;
+; X64-LABEL: test_ccnge:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    #APP
+; X64-NEXT:    cmp %rdi,(%rsi)
+; X64-NEXT:    #NO_APP
+; X64-NEXT:    setge %al
+; X64-NEXT:    retq
+entry:
+  %cc = tail call i32 asm "cmp $2,$1", "={@ccnge},=*m,r,~{cc},~{dirflag},~{fpsr},~{flags}"(i64* %addr, i64 %nr) nounwind
+  %tobool = icmp eq i32 %cc, 0
+  %rv = zext i1 %tobool to i32
+  ret i32 %rv
+}
+
+
+define i32 @test_ccnl(i64 %nr, i64* %addr) {
+; X32-LABEL: test_ccnl:
+; X32:       # %bb.0: # %entry
+; X32-NEXT:    pushl %esi
+; X32-NEXT:    .cfi_def_cfa_offset 8
+; X32-NEXT:    .cfi_offset %esi, -8
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X32-NEXT:    xorl %eax, %eax
+; X32-NEXT:    #APP
+; X32-NEXT:    cmp %ecx,(%esi)
+; X32-NEXT:    #NO_APP
+; X32-NEXT:    setl %al
+; X32-NEXT:    popl %esi
+; X32-NEXT:    .cfi_def_cfa_offset 4
+; X32-NEXT:    retl
+;
+; X64-LABEL: test_ccnl:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    #APP
+; X64-NEXT:    cmp %rdi,(%rsi)
+; X64-NEXT:    #NO_APP
+; X64-NEXT:    setl %al
+; X64-NEXT:    retq
+entry:
+  %cc = tail call i32 asm "cmp $2,$1", "={@ccnl},=*m,r,~{cc},~{dirflag},~{fpsr},~{flags}"(i64* %addr, i64 %nr) nounwind
+  %tobool = icmp eq i32 %cc, 0
+  %rv = zext i1 %tobool to i32
+  ret i32 %rv
+}
+
+
+define i32 @test_ccnle(i64 %nr, i64* %addr) {
+; X32-LABEL: test_ccnle:
+; X32:       # %bb.0: # %entry
+; X32-NEXT:    pushl %esi
+; X32-NEXT:    .cfi_def_cfa_offset 8
+; X32-NEXT:    .cfi_offset %esi, -8
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X32-NEXT:    xorl %eax, %eax
+; X32-NEXT:    #APP
+; X32-NEXT:    cmp %ecx,(%esi)
+; X32-NEXT:    #NO_APP
+; X32-NEXT:    setle %al
+; X32-NEXT:    popl %esi
+; X32-NEXT:    .cfi_def_cfa_offset 4
+; X32-NEXT:    retl
+;
+; X64-LABEL: test_ccnle:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    #APP
+; X64-NEXT:    cmp %rdi,(%rsi)
+; X64-NEXT:    #NO_APP
+; X64-NEXT:    setle %al
+; X64-NEXT:    retq
+entry:
+  %cc = tail call i32 asm "cmp $2,$1", "={@ccnle},=*m,r,~{cc},~{dirflag},~{fpsr},~{flags}"(i64* %addr, i64 %nr) nounwind
+  %tobool = icmp eq i32 %cc, 0
+  %rv = zext i1 %tobool to i32
+  ret i32 %rv
+}
+
+
+define i32 @test_ccno(i64 %nr, i64* %addr) {
+; X32-LABEL: test_ccno:
+; X32:       # %bb.0: # %entry
+; X32-NEXT:    pushl %esi
+; X32-NEXT:    .cfi_def_cfa_offset 8
+; X32-NEXT:    .cfi_offset %esi, -8
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X32-NEXT:    xorl %eax, %eax
+; X32-NEXT:    #APP
+; X32-NEXT:    cmp %ecx,(%esi)
+; X32-NEXT:    #NO_APP
+; X32-NEXT:    seto %al
+; X32-NEXT:    popl %esi
+; X32-NEXT:    .cfi_def_cfa_offset 4
+; X32-NEXT:    retl
+;
+; X64-LABEL: test_ccno:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    #APP
+; X64-NEXT:    cmp %rdi,(%rsi)
+; X64-NEXT:    #NO_APP
+; X64-NEXT:    seto %al
+; X64-NEXT:    retq
+entry:
+  %cc = tail call i32 asm "cmp $2,$1", "={@ccno},=*m,r,~{cc},~{dirflag},~{fpsr},~{flags}"(i64* %addr, i64 %nr) nounwind
+  %tobool = icmp eq i32 %cc, 0
+  %rv = zext i1 %tobool to i32
+  ret i32 %rv
+}
+
+
+define i32 @test_ccnp(i64 %nr, i64* %addr) {
+; X32-LABEL: test_ccnp:
+; X32:       # %bb.0: # %entry
+; X32-NEXT:    pushl %esi
+; X32-NEXT:    .cfi_def_cfa_offset 8
+; X32-NEXT:    .cfi_offset %esi, -8
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X32-NEXT:    xorl %eax, %eax
+; X32-NEXT:    #APP
+; X32-NEXT:    cmp %ecx,(%esi)
+; X32-NEXT:    #NO_APP
+; X32-NEXT:    setnp %al
+; X32-NEXT:    popl %esi
+; X32-NEXT:    .cfi_def_cfa_offset 4
+; X32-NEXT:    retl
+;
+; X64-LABEL: test_ccnp:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    #APP
+; X64-NEXT:    cmp %rdi,(%rsi)
+; X64-NEXT:    #NO_APP
+; X64-NEXT:    setnp %al
+; X64-NEXT:    retq
+entry:
+  %cc = tail call i32 asm "cmp $2,$1", "={@ccnp},=*m,r,~{cc},~{dirflag},~{fpsr},~{flags}"(i64* %addr, i64 %nr) nounwind
+  %tobool = icmp eq i32 %cc, 0
+  %rv = zext i1 %tobool to i32
+  ret i32 %rv
+}
+
+
+define i32 @test_ccns(i64 %nr, i64* %addr) {
+; X32-LABEL: test_ccns:
+; X32:       # %bb.0: # %entry
+; X32-NEXT:    pushl %esi
+; X32-NEXT:    .cfi_def_cfa_offset 8
+; X32-NEXT:    .cfi_offset %esi, -8
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X32-NEXT:    xorl %eax, %eax
+; X32-NEXT:    #APP
+; X32-NEXT:    cmp %ecx,(%esi)
+; X32-NEXT:    #NO_APP
+; X32-NEXT:    sets %al
+; X32-NEXT:    popl %esi
+; X32-NEXT:    .cfi_def_cfa_offset 4
+; X32-NEXT:    retl
+;
+; X64-LABEL: test_ccns:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    #APP
+; X64-NEXT:    cmp %rdi,(%rsi)
+; X64-NEXT:    #NO_APP
+; X64-NEXT:    sets %al
+; X64-NEXT:    retq
+entry:
+  %cc = tail call i32 asm "cmp $2,$1", "={@ccns},=*m,r,~{cc},~{dirflag},~{fpsr},~{flags}"(i64* %addr, i64 %nr) nounwind
+  %tobool = icmp eq i32 %cc, 0
+  %rv = zext i1 %tobool to i32
+  ret i32 %rv
+}
+
+
+define i32 @test_cco(i64 %nr, i64* %addr) {
+; X32-LABEL: test_cco:
+; X32:       # %bb.0: # %entry
+; X32-NEXT:    pushl %esi
+; X32-NEXT:    .cfi_def_cfa_offset 8
+; X32-NEXT:    .cfi_offset %esi, -8
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X32-NEXT:    xorl %eax, %eax
+; X32-NEXT:    #APP
+; X32-NEXT:    cmp %ecx,(%esi)
+; X32-NEXT:    #NO_APP
+; X32-NEXT:    setno %al
+; X32-NEXT:    popl %esi
+; X32-NEXT:    .cfi_def_cfa_offset 4
+; X32-NEXT:    retl
+;
+; X64-LABEL: test_cco:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    #APP
+; X64-NEXT:    cmp %rdi,(%rsi)
+; X64-NEXT:    #NO_APP
+; X64-NEXT:    setno %al
+; X64-NEXT:    retq
+entry:
+  %cc = tail call i32 asm "cmp $2,$1", "={@cco},=*m,r,~{cc},~{dirflag},~{fpsr},~{flags}"(i64* %addr, i64 %nr) nounwind
+  %tobool = icmp eq i32 %cc, 0
+  %rv = zext i1 %tobool to i32
+  ret i32 %rv
+}
+
+
+define i32 @test_ccp(i64 %nr, i64* %addr) {
+; X32-LABEL: test_ccp:
+; X32:       # %bb.0: # %entry
+; X32-NEXT:    pushl %esi
+; X32-NEXT:    .cfi_def_cfa_offset 8
+; X32-NEXT:    .cfi_offset %esi, -8
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X32-NEXT:    xorl %eax, %eax
+; X32-NEXT:    #APP
+; X32-NEXT:    cmp %ecx,(%esi)
+; X32-NEXT:    #NO_APP
+; X32-NEXT:    setnp %al
+; X32-NEXT:    popl %esi
+; X32-NEXT:    .cfi_def_cfa_offset 4
+; X32-NEXT:    retl
+;
+; X64-LABEL: test_ccp:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    #APP
+; X64-NEXT:    cmp %rdi,(%rsi)
+; X64-NEXT:    #NO_APP
+; X64-NEXT:    setnp %al
+; X64-NEXT:    retq
+entry:
+  %cc = tail call i32 asm "cmp $2,$1", "={@ccp},=*m,r,~{cc},~{dirflag},~{fpsr},~{flags}"(i64* %addr, i64 %nr) nounwind
+  %tobool = icmp eq i32 %cc, 0
+  %rv = zext i1 %tobool to i32
+  ret i32 %rv
+}
+
+
+define i32 @test_ccs(i64 %nr, i64* %addr) {
+; X32-LABEL: test_ccs:
+; X32:       # %bb.0: # %entry
+; X32-NEXT:    pushl %esi
+; X32-NEXT:    .cfi_def_cfa_offset 8
+; X32-NEXT:    .cfi_offset %esi, -8
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X32-NEXT:    xorl %eax, %eax
+; X32-NEXT:    #APP
+; X32-NEXT:    cmp %ecx,(%esi)
+; X32-NEXT:    #NO_APP
+; X32-NEXT:    setns %al
+; X32-NEXT:    popl %esi
+; X32-NEXT:    .cfi_def_cfa_offset 4
+; X32-NEXT:    retl
+;
+; X64-LABEL: test_ccs:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    #APP
+; X64-NEXT:    cmp %rdi,(%rsi)
+; X64-NEXT:    #NO_APP
+; X64-NEXT:    setns %al
+; X64-NEXT:    retq
+entry:
+  %cc = tail call i32 asm "cmp $2,$1", "={@ccs},=*m,r,~{cc},~{dirflag},~{fpsr},~{flags}"(i64* %addr, i64 %nr) nounwind
+  %tobool = icmp eq i32 %cc, 0
+  %rv = zext i1 %tobool to i32
+  ret i32 %rv
+}
diff --git a/test/CodeGen/X86/inline-asm-fpstack.ll b/test/CodeGen/X86/inline-asm-fpstack.ll
index 1c36d31c480b5837bf2e852a478efc41178dc41c..71c89df084fdca20a35b9b2e42cdf47bd40de8b5 100644
--- a/test/CodeGen/X86/inline-asm-fpstack.ll
+++ b/test/CodeGen/X86/inline-asm-fpstack.ll
@@ -75,24 +75,23 @@ define void @test6(double %A, double %B, double %C, double %D, double %E) nounwi
 ; CHECK-NEXT:    fldl {{[0-9]+}}(%esp)
 ; CHECK-NEXT:    fldl {{[0-9]+}}(%esp)
 ; CHECK-NEXT:    ## InlineAsm Start
-; CHECK-NEXT:    foo %st(0) %st(0)
+; CHECK-NEXT:    foo %st %st
 ; CHECK-NEXT:    ## InlineAsm End
 ; CHECK-NEXT:    fstp %st(0)
 ; CHECK-NEXT:    ## InlineAsm Start
-; CHECK-NEXT:    bar %st(1) %st(0)
+; CHECK-NEXT:    bar %st(1) %st
 ; CHECK-NEXT:    ## InlineAsm End
 ; CHECK-NEXT:    fstp %st(1)
 ; CHECK-NEXT:    fstp %st(0)
 ; CHECK-NEXT:    ## InlineAsm Start
-; CHECK-NEXT:    baz %st(1) %st(0)
+; CHECK-NEXT:    baz %st(1) %st
 ; CHECK-NEXT:    ## InlineAsm End
 ; CHECK-NEXT:    fstp %st(0)
 ; CHECK-NEXT:    ## InlineAsm Start
-; CHECK-NEXT:    baz %st(0)
+; CHECK-NEXT:    baz %st
 ; CHECK-NEXT:    ## InlineAsm End
 ; CHECK-NEXT:    fstp %st(0)
 ; CHECK-NEXT:    retl
-; CHECK-NEXT:    ## -- End function
 entry:
 ; Uses the same value twice, should have one fstp after the asm.
   tail call void asm sideeffect "foo $0 $1", "f,f,~{dirflag},~{fpsr},~{flags}"( double %A, double %A ) nounwind
@@ -117,10 +116,10 @@ define void @testPR4185() {
 ; CHECK-NEXT:    flds LCPI6_0
 ; CHECK-NEXT:    fld %st(0)
 ; CHECK-NEXT:    ## InlineAsm Start
-; CHECK-NEXT:    fistpl %st(0)
+; CHECK-NEXT:    fistpl %st
 ; CHECK-NEXT:    ## InlineAsm End
 ; CHECK-NEXT:    ## InlineAsm Start
-; CHECK-NEXT:    fistpl %st(0)
+; CHECK-NEXT:    fistpl %st
 ; CHECK-NEXT:    ## InlineAsm End
 ; CHECK-NEXT:    retl
 return:
@@ -138,10 +137,10 @@ define void @testPR4185b() {
 ; CHECK:       ## %bb.0: ## %return
 ; CHECK-NEXT:    flds LCPI7_0
 ; CHECK-NEXT:    ## InlineAsm Start
-; CHECK-NEXT:    fistl %st(0)
+; CHECK-NEXT:    fistl %st
 ; CHECK-NEXT:    ## InlineAsm End
 ; CHECK-NEXT:    ## InlineAsm Start
-; CHECK-NEXT:    fistpl %st(0)
+; CHECK-NEXT:    fistpl %st
 ; CHECK-NEXT:    ## InlineAsm End
 ; CHECK-NEXT:    retl
 return:
@@ -163,7 +162,7 @@ define void @testPR4459(x86_fp80 %a) {
 ; CHECK-NEXT:    fld %st(0)
 ; CHECK-NEXT:    fxch %st(1)
 ; CHECK-NEXT:    ## InlineAsm Start
-; CHECK-NEXT:    fistpl %st(0)
+; CHECK-NEXT:    fistpl %st
 ; CHECK-NEXT:    ## InlineAsm End
 ; CHECK-NEXT:    fstpt (%esp)
 ; CHECK-NEXT:    calll _test3
@@ -187,11 +186,11 @@ define void @testPR4484(x86_fp80 %a) {
 ; CHECK-NEXT:    subl $28, %esp
 ; CHECK-NEXT:    .cfi_def_cfa_offset 32
 ; CHECK-NEXT:    fldt {{[0-9]+}}(%esp)
-; CHECK-NEXT:    fstpt {{[0-9]+}}(%esp) ## 10-byte Folded Spill
+; CHECK-NEXT:    fstpt {{[-0-9]+}}(%e{{[sb]}}p) ## 10-byte Folded Spill
 ; CHECK-NEXT:    calll _test1
-; CHECK-NEXT:    fldt {{[0-9]+}}(%esp) ## 10-byte Folded Reload
+; CHECK-NEXT:    fldt {{[-0-9]+}}(%e{{[sb]}}p) ## 10-byte Folded Reload
 ; CHECK-NEXT:    ## InlineAsm Start
-; CHECK-NEXT:    fistpl %st(0)
+; CHECK-NEXT:    fistpl %st
 ; CHECK-NEXT:    ## InlineAsm End
 ; CHECK-NEXT:    fstpt (%esp)
 ; CHECK-NEXT:    calll _test3
@@ -211,18 +210,18 @@ define void @testPR4485(x86_fp80* %a) {
 ; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; CHECK-NEXT:    fldt (%eax)
 ; CHECK-NEXT:    flds LCPI10_0
-; CHECK-NEXT:    fmul %st(0), %st(1)
+; CHECK-NEXT:    fmul %st, %st(1)
 ; CHECK-NEXT:    flds LCPI10_1
-; CHECK-NEXT:    fmul %st(0), %st(2)
+; CHECK-NEXT:    fmul %st, %st(2)
 ; CHECK-NEXT:    fxch %st(2)
 ; CHECK-NEXT:    ## InlineAsm Start
-; CHECK-NEXT:    fistpl %st(0)
+; CHECK-NEXT:    fistpl %st
 ; CHECK-NEXT:    ## InlineAsm End
 ; CHECK-NEXT:    fldt (%eax)
-; CHECK-NEXT:    fmulp %st(1)
-; CHECK-NEXT:    fmulp %st(1)
+; CHECK-NEXT:    fmulp %st, %st(1)
+; CHECK-NEXT:    fmulp %st, %st(1)
 ; CHECK-NEXT:    ## InlineAsm Start
-; CHECK-NEXT:    fistpl %st(0)
+; CHECK-NEXT:    fistpl %st
 ; CHECK-NEXT:    ## InlineAsm End
 ; CHECK-NEXT:    retl
 entry:
@@ -256,7 +255,6 @@ define void @fist1(x86_fp80 %x, i32* %p) nounwind ssp {
 ; CHECK-NEXT:    ## InlineAsm End
 ; CHECK-NEXT:    fstp %st(0)
 ; CHECK-NEXT:    retl
-; CHECK-NEXT:    ## -- End function
 entry:
   tail call void asm sideeffect "fistl $1", "{st},*m,~{memory},~{dirflag},~{fpsr},~{flags}"(x86_fp80 %x, i32* %p) nounwind
   ret void
@@ -279,7 +277,6 @@ define x86_fp80 @fist2(x86_fp80 %x, i32* %p) nounwind ssp {
 ; CHECK-NEXT:    fistl (%eax)
 ; CHECK-NEXT:    ## InlineAsm End
 ; CHECK-NEXT:    retl
-; CHECK-NEXT:    ## -- End function
 entry:
   %0 = tail call x86_fp80 asm "fistl $2", "=&{st},0,*m,~{memory},~{dirflag},~{fpsr},~{flags}"(x86_fp80 %x, i32* %p) nounwind
   ret x86_fp80 %0
@@ -301,7 +298,6 @@ define void @fucomp1(x86_fp80 %x, x86_fp80 %y) nounwind ssp {
 ; CHECK-NEXT:    ## InlineAsm End
 ; CHECK-NEXT:    fstp %st(0)
 ; CHECK-NEXT:    retl
-; CHECK-NEXT:    ## -- End function
 entry:
   tail call void asm sideeffect "fucomp $1", "{st},f,~{st},~{dirflag},~{fpsr},~{flags}"(x86_fp80 %x, x86_fp80 %y) nounwind
   ret void
@@ -328,7 +324,6 @@ define void @fucomp2(x86_fp80 %x, x86_fp80 %y) nounwind ssp {
 ; CHECK-NEXT:    ## InlineAsm End
 ; CHECK-NEXT:    fstp %st(0)
 ; CHECK-NEXT:    retl
-; CHECK-NEXT:    ## -- End function
 entry:
   tail call void asm sideeffect "fucomp $1", "{st},{st(1)},~{st},~{dirflag},~{fpsr},~{flags}"(x86_fp80 %x, x86_fp80 %y) nounwind
   ret void
@@ -344,7 +339,6 @@ define void @fucomp3(x86_fp80 %x, x86_fp80 %y) nounwind ssp {
 ; CHECK-NEXT:    fucompp %st(1)
 ; CHECK-NEXT:    ## InlineAsm End
 ; CHECK-NEXT:    retl
-; CHECK-NEXT:    ## -- End function
 entry:
   tail call void asm sideeffect "fucompp $1", "{st},{st(1)},~{st},~{st(1)},~{dirflag},~{fpsr},~{flags}"(x86_fp80 %x, x86_fp80 %y) nounwind
   ret void
@@ -361,7 +355,6 @@ define float @sincos1(float %x) nounwind ssp {
 ; CHECK-NEXT:    ## InlineAsm End
 ; CHECK-NEXT:    fstp %st(1)
 ; CHECK-NEXT:    retl
-; CHECK-NEXT:    ## -- End function
 entry:
   %0 = tail call %complex asm "sincos", "={st},={st(1)},0,~{dirflag},~{fpsr},~{flags}"(float %x) nounwind
   %asmresult = extractvalue %complex %0, 0
@@ -378,7 +371,6 @@ define float @sincos2(float %x) nounwind ssp {
 ; CHECK-NEXT:    ## InlineAsm End
 ; CHECK-NEXT:    fstp %st(1)
 ; CHECK-NEXT:    retl
-; CHECK-NEXT:    ## -- End function
 entry:
   %0 = tail call %complex asm "sincos", "={st(1)},={st},1,~{dirflag},~{fpsr},~{flags}"(float %x) nounwind
   %asmresult = extractvalue %complex %0, 1
@@ -406,7 +398,6 @@ define float @sincos3(float %x) nounwind ssp {
 ; CHECK-NEXT:    fstp %st(1)
 ; CHECK-NEXT:    fstp %st(0)
 ; CHECK-NEXT:    retl
-; CHECK-NEXT:    ## -- End function
 entry:
   %0 = tail call %complex asm sideeffect "sincos", "={st(1)},={st},1,~{dirflag},~{fpsr},~{flags}"(float %x) nounwind
   %1 = tail call %complex asm sideeffect "sincos", "={st(1)},={st},1,~{dirflag},~{fpsr},~{flags}"(float %x) nounwind
@@ -422,12 +413,11 @@ define i32 @PR10602() nounwind ssp {
 ; CHECK-NEXT:    fld %st(0)
 ; CHECK-NEXT:    fxch %st(1)
 ; CHECK-NEXT:    ## InlineAsm Start
-; CHECK-NEXT:    fcomi %st(1), %st(0); pushf; pop %eax
+; CHECK-NEXT:    fcomi %st(1), %st; pushf; pop %eax
 ; CHECK-NEXT:    ## InlineAsm End
 ; CHECK-NEXT:    fstp %st(0)
 ; CHECK-NEXT:    fstp %st(0)
 ; CHECK-NEXT:    retl
-; CHECK-NEXT:    ## -- End function
 entry:
   %0 = tail call i32 asm "fcomi $2, $1; pushf; pop $0", "=r,{st},{st(1)},~{dirflag},~{fpsr},~{flags}"(double 2.000000e+00, double 2.000000e+00) nounwind
   ret i32 %0
@@ -505,9 +495,9 @@ define double @test_operand_rewrite() {
 ; CHECK-LABEL: test_operand_rewrite:
 ; CHECK:       ## %bb.0: ## %entry
 ; CHECK-NEXT:    ## InlineAsm Start
-; CHECK-NEXT:    foo %st(0), %st(1)
+; CHECK-NEXT:    foo %st, %st(1)
 ; CHECK-NEXT:    ## InlineAsm End
-; CHECK-NEXT:    fsubp %st(1)
+; CHECK-NEXT:    fsubp %st, %st(1)
 ; CHECK-NEXT:    retl
 entry:
   %0 = tail call { double, double } asm sideeffect "foo $0, $1", "={st},={st(1)},~{dirflag},~{fpsr},~{flags}"()
diff --git a/test/CodeGen/X86/inline-asm-ptr-cast.ll b/test/CodeGen/X86/inline-asm-ptr-cast.ll
index 21353468b1fcf8fee2a41552e6742c9ee9dd999c..2120ab9f188bcd0d8abc26e3d5b8b5049b0536cd 100644
--- a/test/CodeGen/X86/inline-asm-ptr-cast.ll
+++ b/test/CodeGen/X86/inline-asm-ptr-cast.ll
@@ -1,4 +1,4 @@
-; RUN: llc -mtriple=x86_64-unknown-linux-gnu <%s
+; RUN: llc -mtriple=x86_64-unknown-linux-gnu < %s
 ; ModuleID = 'bug.c'
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 target triple = "x86_64-unknown-linux-gnu"
diff --git a/test/CodeGen/X86/inlineasm-sched-bug.ll b/test/CodeGen/X86/inlineasm-sched-bug.ll
index 3eb5fb0eb46684a33d964dc7ed9b9f32c5fb796a..25bf5e07ce76b7b6dab434ce275bb73d9c1d06a2 100644
--- a/test/CodeGen/X86/inlineasm-sched-bug.ll
+++ b/test/CodeGen/X86/inlineasm-sched-bug.ll
@@ -1,5 +1,5 @@
 ; PR13504
-; RUN: llc -mtriple=i686-- -mcpu=atom <%s | FileCheck %s
+; RUN: llc -mtriple=i686-- -mcpu=atom < %s | FileCheck %s
 ; CHECK: bsfl
 ; CHECK-NOT: movl
 
diff --git a/test/CodeGen/X86/insert-into-constant-vector.ll b/test/CodeGen/X86/insert-into-constant-vector.ll
index 9d95f98a57b6f40a6ea34837e0a6f9511915256b..c34e9409dff99c95e7981cea472119f79d45a6f6 100644
--- a/test/CodeGen/X86/insert-into-constant-vector.ll
+++ b/test/CodeGen/X86/insert-into-constant-vector.ll
@@ -271,13 +271,11 @@ define <8 x i32> @elt7_v8i32(i32 %x) {
 ; X64SSE4-NEXT:    movaps {{.*#+}} xmm0 = [42,1,2,3]
 ; X64SSE4-NEXT:    retq
 ;
-; X32AVX1-LABEL: elt7_v8i32:
-; X32AVX1:       # %bb.0:
-; X32AVX1-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; X32AVX1-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,1,2,0]
-; X32AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
-; X32AVX1-NEXT:    vblendps {{.*#+}} ymm0 = mem[0,1,2,3,4,5,6],ymm0[7]
-; X32AVX1-NEXT:    retl
+; X32AVX-LABEL: elt7_v8i32:
+; X32AVX:       # %bb.0:
+; X32AVX-NEXT:    vbroadcastss {{[0-9]+}}(%esp), %ymm0
+; X32AVX-NEXT:    vblendps {{.*#+}} ymm0 = mem[0,1,2,3,4,5,6],ymm0[7]
+; X32AVX-NEXT:    retl
 ;
 ; X64AVX1-LABEL: elt7_v8i32:
 ; X64AVX1:       # %bb.0:
@@ -287,33 +285,17 @@ define <8 x i32> @elt7_v8i32(i32 %x) {
 ; X64AVX1-NEXT:    vblendps {{.*#+}} ymm0 = mem[0,1,2,3,4,5,6],ymm0[7]
 ; X64AVX1-NEXT:    retq
 ;
-; X32AVX2-LABEL: elt7_v8i32:
-; X32AVX2:       # %bb.0:
-; X32AVX2-NEXT:    vbroadcastss {{[0-9]+}}(%esp), %xmm0
-; X32AVX2-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
-; X32AVX2-NEXT:    vblendps {{.*#+}} ymm0 = mem[0,1,2,3,4,5,6],ymm0[7]
-; X32AVX2-NEXT:    retl
-;
 ; X64AVX2-LABEL: elt7_v8i32:
 ; X64AVX2:       # %bb.0:
 ; X64AVX2-NEXT:    vmovd %edi, %xmm0
-; X64AVX2-NEXT:    vpbroadcastd %xmm0, %xmm0
-; X64AVX2-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
+; X64AVX2-NEXT:    vpbroadcastd %xmm0, %ymm0
 ; X64AVX2-NEXT:    vpblendd {{.*#+}} ymm0 = mem[0,1,2,3,4,5,6],ymm0[7]
 ; X64AVX2-NEXT:    retq
 ;
-; X32AVX512F-LABEL: elt7_v8i32:
-; X32AVX512F:       # %bb.0:
-; X32AVX512F-NEXT:    vbroadcastss {{[0-9]+}}(%esp), %xmm0
-; X32AVX512F-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
-; X32AVX512F-NEXT:    vblendps {{.*#+}} ymm0 = mem[0,1,2,3,4,5,6],ymm0[7]
-; X32AVX512F-NEXT:    retl
-;
 ; X64AVX512F-LABEL: elt7_v8i32:
 ; X64AVX512F:       # %bb.0:
 ; X64AVX512F-NEXT:    vmovd %edi, %xmm0
-; X64AVX512F-NEXT:    vpbroadcastd %xmm0, %xmm0
-; X64AVX512F-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
+; X64AVX512F-NEXT:    vpbroadcastd %xmm0, %ymm0
 ; X64AVX512F-NEXT:    vpblendd {{.*#+}} ymm0 = mem[0,1,2,3,4,5,6],ymm0[7]
 ; X64AVX512F-NEXT:    retq
    %ins = insertelement <8 x i32> <i32 42, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>, i32 %x, i32 7
diff --git a/test/CodeGen/X86/insert-loaded-scalar.ll b/test/CodeGen/X86/insert-loaded-scalar.ll
index 81cb533f442edd3d2cb0f2f5277baac9a2278935..fe00e50137bbe13a1fa81c597e3e49c77f55d8e6 100644
--- a/test/CodeGen/X86/insert-loaded-scalar.ll
+++ b/test/CodeGen/X86/insert-loaded-scalar.ll
@@ -178,16 +178,10 @@ define <2 x i64> @load64_ins_eltc_v2i64(i64* %p) nounwind {
 ; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,1,0,1]
 ; SSE-NEXT:    retq
 ;
-; AVX1-LABEL: load64_ins_eltc_v2i64:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,1,0,1]
-; AVX1-NEXT:    retq
-;
-; AVX2-LABEL: load64_ins_eltc_v2i64:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpbroadcastq (%rdi), %xmm0
-; AVX2-NEXT:    retq
+; AVX-LABEL: load64_ins_eltc_v2i64:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovddup {{.*#+}} xmm0 = mem[0,0]
+; AVX-NEXT:    retq
   %x = load i64, i64* %p
   %ins = insertelement <2 x i64> undef, i64 %x, i32 1
   ret <2 x i64> %ins
diff --git a/test/CodeGen/X86/insert-prefetch-inline.ll b/test/CodeGen/X86/insert-prefetch-inline.ll
index 5f8373f9480c9624f0d0222b301bd8c1ada51ebe..62c02fa33291d2d592a624ea08d6a99af0d5e425 100644
--- a/test/CodeGen/X86/insert-prefetch-inline.ll
+++ b/test/CodeGen/X86/insert-prefetch-inline.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -prefetch-hints-file=%S/insert-prefetch-inline.afdo | FileCheck %s
+; RUN: llc < %s -x86-discriminate-memops -prefetch-hints-file=%S/insert-prefetch-inline.afdo | FileCheck %s
 ;
 ; Verify we can insert prefetch instructions in code belonging to inlined
 ; functions.
diff --git a/test/CodeGen/X86/insert-prefetch-invalid-instr.ll b/test/CodeGen/X86/insert-prefetch-invalid-instr.ll
index 004fb56a56eb8a289705cc1ff2911b3f22e0617d..d0c4ac378b63b285558d8c507b1448aa18a60c97 100644
--- a/test/CodeGen/X86/insert-prefetch-invalid-instr.ll
+++ b/test/CodeGen/X86/insert-prefetch-invalid-instr.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -prefetch-hints-file=%S/insert-prefetch-invalid-instr.afdo | FileCheck %s
+; RUN: llc < %s -x86-discriminate-memops -prefetch-hints-file=%S/insert-prefetch-invalid-instr.afdo | FileCheck %s
 ; ModuleID = 'prefetch.cc'
 source_filename = "prefetch.cc"
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
diff --git a/test/CodeGen/X86/insert-prefetch.ll b/test/CodeGen/X86/insert-prefetch.ll
index 9e77772df7746b96c42aefa3172da73d54fc986d..fe0fd9877f193ee047a37a68fa3a0ae5f5b8f960 100644
--- a/test/CodeGen/X86/insert-prefetch.ll
+++ b/test/CodeGen/X86/insert-prefetch.ll
@@ -1,5 +1,5 @@
-; RUN: llc < %s -prefetch-hints-file=%S/insert-prefetch.afdo | FileCheck %s
-; RUN: llc < %s -prefetch-hints-file=%S/insert-prefetch-other.afdo | FileCheck %s -check-prefix=OTHERS
+; RUN: llc < %s -x86-discriminate-memops -prefetch-hints-file=%S/insert-prefetch.afdo | FileCheck %s
+; RUN: llc < %s -x86-discriminate-memops -prefetch-hints-file=%S/insert-prefetch-other.afdo | FileCheck %s -check-prefix=OTHERS
 ;
 ; original source, compiled with -O3 -gmlt -fdebug-info-for-profiling:
 ; int sum(int* arr, int pos1, int pos2) {
diff --git a/test/CodeGen/X86/insertelement-shuffle.ll b/test/CodeGen/X86/insertelement-shuffle.ll
index 1756c2aa836686d0541e07d3c90b0c7888df197f..90934ed7945497cf4ed2a36984aaa17058262fb1 100644
--- a/test/CodeGen/X86/insertelement-shuffle.ll
+++ b/test/CodeGen/X86/insertelement-shuffle.ll
@@ -1,41 +1,25 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=i686-unknown-unknown   -mattr=avx2    | FileCheck %s --check-prefix=X32_AVX256
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx2    | FileCheck %s --check-prefix=X64_AVX256
-; RUN: llc < %s -mtriple=i686-unknown-unknown   -mattr=avx512f | FileCheck %s --check-prefix=X32_AVX512
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx512f | FileCheck %s --check-prefix=X64_AVX512
+; RUN: llc < %s -mtriple=i686-unknown-unknown   -mattr=avx2    | FileCheck %s --check-prefixes=X86,X86_AVX256
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx2    | FileCheck %s --check-prefixes=X64,X64_AVX256
+; RUN: llc < %s -mtriple=i686-unknown-unknown   -mattr=avx512f | FileCheck %s --check-prefixes=X86,X86_AVX512
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx512f | FileCheck %s --check-prefixes=X64,X64_AVX512
 
 define <8 x float> @insert_subvector_256(i16 %x0, i16 %x1, <8 x float> %v) nounwind {
-; X32_AVX256-LABEL: insert_subvector_256:
-; X32_AVX256:       # %bb.0:
-; X32_AVX256-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; X32_AVX256-NEXT:    vpinsrw $1, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; X32_AVX256-NEXT:    vpbroadcastd %xmm1, %xmm1
-; X32_AVX256-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2,3,4,5,6,7]
-; X32_AVX256-NEXT:    retl
+; X86-LABEL: insert_subvector_256:
+; X86:       # %bb.0:
+; X86-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; X86-NEXT:    vpinsrw $1, {{[0-9]+}}(%esp), %xmm1, %xmm1
+; X86-NEXT:    vpbroadcastd %xmm1, %xmm1
+; X86-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2,3,4,5,6,7]
+; X86-NEXT:    retl
 ;
-; X64_AVX256-LABEL: insert_subvector_256:
-; X64_AVX256:       # %bb.0:
-; X64_AVX256-NEXT:    vmovd %edi, %xmm1
-; X64_AVX256-NEXT:    vpinsrw $1, %esi, %xmm1, %xmm1
-; X64_AVX256-NEXT:    vpbroadcastd %xmm1, %xmm1
-; X64_AVX256-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2,3,4,5,6,7]
-; X64_AVX256-NEXT:    retq
-;
-; X32_AVX512-LABEL: insert_subvector_256:
-; X32_AVX512:       # %bb.0:
-; X32_AVX512-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; X32_AVX512-NEXT:    vpinsrw $1, {{[0-9]+}}(%esp), %xmm1, %xmm1
-; X32_AVX512-NEXT:    vpbroadcastd %xmm1, %xmm1
-; X32_AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2,3,4,5,6,7]
-; X32_AVX512-NEXT:    retl
-;
-; X64_AVX512-LABEL: insert_subvector_256:
-; X64_AVX512:       # %bb.0:
-; X64_AVX512-NEXT:    vmovd %edi, %xmm1
-; X64_AVX512-NEXT:    vpinsrw $1, %esi, %xmm1, %xmm1
-; X64_AVX512-NEXT:    vpbroadcastd %xmm1, %xmm1
-; X64_AVX512-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2,3,4,5,6,7]
-; X64_AVX512-NEXT:    retq
+; X64-LABEL: insert_subvector_256:
+; X64:       # %bb.0:
+; X64-NEXT:    vmovd %edi, %xmm1
+; X64-NEXT:    vpinsrw $1, %esi, %xmm1, %xmm1
+; X64-NEXT:    vpbroadcastd %xmm1, %xmm1
+; X64-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2,3,4,5,6,7]
+; X64-NEXT:    retq
   %ins1 = insertelement <2 x i16> undef, i16 %x0, i32 0
   %ins2 = insertelement <2 x i16> %ins1, i16 %x1, i32 1
   %bc = bitcast <2 x i16> %ins2 to float
@@ -44,21 +28,21 @@ define <8 x float> @insert_subvector_256(i16 %x0, i16 %x1, <8 x float> %v) nounw
 }
 
 define <8 x i64> @insert_subvector_512(i32 %x0, i32 %x1, <8 x i64> %v) nounwind {
-; X32_AVX256-LABEL: insert_subvector_512:
-; X32_AVX256:       # %bb.0:
-; X32_AVX256-NEXT:    pushl %ebp
-; X32_AVX256-NEXT:    movl %esp, %ebp
-; X32_AVX256-NEXT:    andl $-8, %esp
-; X32_AVX256-NEXT:    subl $8, %esp
-; X32_AVX256-NEXT:    vmovsd {{.*#+}} xmm2 = mem[0],zero
-; X32_AVX256-NEXT:    vmovlps %xmm2, (%esp)
-; X32_AVX256-NEXT:    vextracti128 $1, %ymm0, %xmm2
-; X32_AVX256-NEXT:    vpinsrd $0, (%esp), %xmm2, %xmm2
-; X32_AVX256-NEXT:    vpinsrd $1, {{[0-9]+}}(%esp), %xmm2, %xmm2
-; X32_AVX256-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm0
-; X32_AVX256-NEXT:    movl %ebp, %esp
-; X32_AVX256-NEXT:    popl %ebp
-; X32_AVX256-NEXT:    retl
+; X86_AVX256-LABEL: insert_subvector_512:
+; X86_AVX256:       # %bb.0:
+; X86_AVX256-NEXT:    pushl %ebp
+; X86_AVX256-NEXT:    movl %esp, %ebp
+; X86_AVX256-NEXT:    andl $-8, %esp
+; X86_AVX256-NEXT:    subl $8, %esp
+; X86_AVX256-NEXT:    vmovsd {{.*#+}} xmm2 = mem[0],zero
+; X86_AVX256-NEXT:    vmovlps %xmm2, (%esp)
+; X86_AVX256-NEXT:    vextracti128 $1, %ymm0, %xmm2
+; X86_AVX256-NEXT:    vpinsrd $0, (%esp), %xmm2, %xmm2
+; X86_AVX256-NEXT:    vpinsrd $1, {{[0-9]+}}(%esp), %xmm2, %xmm2
+; X86_AVX256-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm0
+; X86_AVX256-NEXT:    movl %ebp, %esp
+; X86_AVX256-NEXT:    popl %ebp
+; X86_AVX256-NEXT:    retl
 ;
 ; X64_AVX256-LABEL: insert_subvector_512:
 ; X64_AVX256:       # %bb.0:
@@ -70,12 +54,12 @@ define <8 x i64> @insert_subvector_512(i32 %x0, i32 %x1, <8 x i64> %v) nounwind
 ; X64_AVX256-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm0
 ; X64_AVX256-NEXT:    retq
 ;
-; X32_AVX512-LABEL: insert_subvector_512:
-; X32_AVX512:       # %bb.0:
-; X32_AVX512-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
-; X32_AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [0,0,1,0,8,0,3,0,4,0,5,0,6,0,7,0]
-; X32_AVX512-NEXT:    vpermt2q %zmm1, %zmm2, %zmm0
-; X32_AVX512-NEXT:    retl
+; X86_AVX512-LABEL: insert_subvector_512:
+; X86_AVX512:       # %bb.0:
+; X86_AVX512-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero
+; X86_AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [0,0,1,0,8,0,3,0,4,0,5,0,6,0,7,0]
+; X86_AVX512-NEXT:    vpermt2q %zmm1, %zmm2, %zmm0
+; X86_AVX512-NEXT:    retl
 ;
 ; X64_AVX512-LABEL: insert_subvector_512:
 ; X64_AVX512:       # %bb.0:
@@ -95,12 +79,12 @@ define <8 x i64> @insert_subvector_512(i32 %x0, i32 %x1, <8 x i64> %v) nounwind
 ; Special case: if we're inserting into an undef vector, we can optimize more.
 
 define <8 x i64> @insert_subvector_into_undef(i32 %x0, i32 %x1) nounwind {
-; X32_AVX256-LABEL: insert_subvector_into_undef:
-; X32_AVX256:       # %bb.0:
-; X32_AVX256-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
-; X32_AVX256-NEXT:    vbroadcastsd %xmm0, %ymm0
-; X32_AVX256-NEXT:    vmovaps %ymm0, %ymm1
-; X32_AVX256-NEXT:    retl
+; X86_AVX256-LABEL: insert_subvector_into_undef:
+; X86_AVX256:       # %bb.0:
+; X86_AVX256-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; X86_AVX256-NEXT:    vbroadcastsd %xmm0, %ymm0
+; X86_AVX256-NEXT:    vmovaps %ymm0, %ymm1
+; X86_AVX256-NEXT:    retl
 ;
 ; X64_AVX256-LABEL: insert_subvector_into_undef:
 ; X64_AVX256:       # %bb.0:
@@ -110,11 +94,11 @@ define <8 x i64> @insert_subvector_into_undef(i32 %x0, i32 %x1) nounwind {
 ; X64_AVX256-NEXT:    vmovdqa %ymm0, %ymm1
 ; X64_AVX256-NEXT:    retq
 ;
-; X32_AVX512-LABEL: insert_subvector_into_undef:
-; X32_AVX512:       # %bb.0:
-; X32_AVX512-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
-; X32_AVX512-NEXT:    vbroadcastsd %xmm0, %zmm0
-; X32_AVX512-NEXT:    retl
+; X86_AVX512-LABEL: insert_subvector_into_undef:
+; X86_AVX512:       # %bb.0:
+; X86_AVX512-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; X86_AVX512-NEXT:    vbroadcastsd %xmm0, %zmm0
+; X86_AVX512-NEXT:    retl
 ;
 ; X64_AVX512-LABEL: insert_subvector_into_undef:
 ; X64_AVX512:       # %bb.0:
diff --git a/test/CodeGen/X86/insertelement-var-index.ll b/test/CodeGen/X86/insertelement-var-index.ll
index 3288416da542701c42400bd917ab61f479c84858..cbb29202cca29e2291ce69f6aa320df9adfc53ac 100644
--- a/test/CodeGen/X86/insertelement-var-index.ll
+++ b/test/CodeGen/X86/insertelement-var-index.ll
@@ -203,16 +203,10 @@ define <2 x i64> @load_i64_v2i64(i64* %p, i32 %y) nounwind {
 ; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,1,0,1]
 ; SSE-NEXT:    retq
 ;
-; AVX1-LABEL: load_i64_v2i64:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,1,0,1]
-; AVX1-NEXT:    retq
-;
-; AVX2-LABEL: load_i64_v2i64:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpbroadcastq (%rdi), %xmm0
-; AVX2-NEXT:    retq
+; AVX-LABEL: load_i64_v2i64:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovddup {{.*#+}} xmm0 = mem[0,0]
+; AVX-NEXT:    retq
   %x = load i64, i64* %p
   %ins = insertelement <2 x i64> undef, i64 %x, i32 %y
   ret <2 x i64> %ins
diff --git a/test/CodeGen/X86/insertps-combine.ll b/test/CodeGen/X86/insertps-combine.ll
index b99605ad50be00354f9a087b9c841aaeab15cebb..6bef76ee9df8b6afa0eb055b2059a090e21f3847 100644
--- a/test/CodeGen/X86/insertps-combine.ll
+++ b/test/CodeGen/X86/insertps-combine.ll
@@ -1,7 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSE41
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX1
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefixes=CHECK,SSE,SSE41
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefixes=CHECK,AVX,AVX1
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefixes=CHECK,AVX,AVX2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f | FileCheck %s --check-prefixes=CHECK,AVX,AVX512F
 
 define <4 x float> @shuffle_v4f32_0z27(<4 x float> %x, <4 x float> %a) {
 ; SSE-LABEL: shuffle_v4f32_0z27:
@@ -297,4 +298,20 @@ define float @extract_lane_insertps_6123(<4 x float> %a0, <4 x float> *%p1) {
   ret float %ext
 }
 
+; PR40340
+define <4 x float> @commute_load_insertps(<4 x float>, <4 x float>* nocapture readonly) {
+; SSE-LABEL: commute_load_insertps:
+; SSE:       # %bb.0:
+; SSE-NEXT:    insertps {{.*#+}} xmm0 = zero,xmm0[1],zero,mem[0]
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: commute_load_insertps:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = zero,xmm0[1],zero,mem[0]
+; AVX-NEXT:    retq
+  %3 = load <4 x float>, <4 x float>* %1
+  %4 = tail call <4 x float> @llvm.x86.sse41.insertps(<4 x float> %3, <4 x float> %0, i8 85)
+  ret <4 x float> %4
+}
+
 declare <4 x float> @llvm.x86.sse41.insertps(<4 x float>, <4 x float>, i8) nounwind readnone
diff --git a/test/CodeGen/X86/ipra-reg-usage.ll b/test/CodeGen/X86/ipra-reg-usage.ll
index 3e57ef218442678b3e572cfcd3d7376fdc42ff3e..2a557f2902a21d506c01384b45d8504597498e63 100644
--- a/test/CodeGen/X86/ipra-reg-usage.ll
+++ b/test/CodeGen/X86/ipra-reg-usage.ll
@@ -3,7 +3,7 @@
 target triple = "x86_64-unknown-unknown"
 declare void @bar1()
 define preserve_allcc void @foo()#0 {
-; CHECK: foo Clobbered Registers: $cs $df $ds $eflags $eip $eiz $es $fpsw $fs $gs $hip $ip $rip $riz $ss $ssp $bnd0 $bnd1 $bnd2 $bnd3 $cr0 $cr1 $cr2 $cr3 $cr4 $cr5 $cr6 $cr7 $cr8 $cr9 $cr10 $cr11 $cr12 $cr13 $cr14 $cr15 $dr0 $dr1 $dr2 $dr3 $dr4 $dr5 $dr6 $dr7 $dr8 $dr9 $dr10 $dr11 $dr12 $dr13 $dr14 $dr15 $fp0 $fp1 $fp2 $fp3 $fp4 $fp5 $fp6 $fp7 $k0 $k1 $k2 $k3 $k4 $k5 $k6 $k7 $mm0 $mm1 $mm2 $mm3 $mm4 $mm5 $mm6 $mm7 $r11 $st0 $st1 $st2 $st3 $st4 $st5 $st6 $st7 $xmm16 $xmm17 $xmm18 $xmm19 $xmm20 $xmm21 $xmm22 $xmm23 $xmm24 $xmm25 $xmm26 $xmm27 $xmm28 $xmm29 $xmm30 $xmm31 $ymm0 $ymm1 $ymm2 $ymm3 $ymm4 $ymm5 $ymm6 $ymm7 $ymm8 $ymm9 $ymm10 $ymm11 $ymm12 $ymm13 $ymm14 $ymm15 $ymm16 $ymm17 $ymm18 $ymm19 $ymm20 $ymm21 $ymm22 $ymm23 $ymm24 $ymm25 $ymm26 $ymm27 $ymm28 $ymm29 $ymm30 $ymm31 $zmm0 $zmm1 $zmm2 $zmm3 $zmm4 $zmm5 $zmm6 $zmm7 $zmm8 $zmm9 $zmm10 $zmm11 $zmm12 $zmm13 $zmm14 $zmm15 $zmm16 $zmm17 $zmm18 $zmm19 $zmm20 $zmm21 $zmm22 $zmm23 $zmm24 $zmm25 $zmm26 $zmm27 $zmm28 $zmm29 $zmm30 $zmm31 $r11b $r11bh $r11d $r11w $r11wh
+; CHECK: foo Clobbered Registers: $cs $df $ds $eflags $eip $eiz $es $fpcw $fpsw $fs $gs $hip $ip $rip $riz $ss $ssp $bnd0 $bnd1 $bnd2 $bnd3 $cr0 $cr1 $cr2 $cr3 $cr4 $cr5 $cr6 $cr7 $cr8 $cr9 $cr10 $cr11 $cr12 $cr13 $cr14 $cr15 $dr0 $dr1 $dr2 $dr3 $dr4 $dr5 $dr6 $dr7 $dr8 $dr9 $dr10 $dr11 $dr12 $dr13 $dr14 $dr15 $fp0 $fp1 $fp2 $fp3 $fp4 $fp5 $fp6 $fp7 $k0 $k1 $k2 $k3 $k4 $k5 $k6 $k7 $mm0 $mm1 $mm2 $mm3 $mm4 $mm5 $mm6 $mm7 $r11 $st0 $st1 $st2 $st3 $st4 $st5 $st6 $st7 $xmm16 $xmm17 $xmm18 $xmm19 $xmm20 $xmm21 $xmm22 $xmm23 $xmm24 $xmm25 $xmm26 $xmm27 $xmm28 $xmm29 $xmm30 $xmm31 $ymm0 $ymm1 $ymm2 $ymm3 $ymm4 $ymm5 $ymm6 $ymm7 $ymm8 $ymm9 $ymm10 $ymm11 $ymm12 $ymm13 $ymm14 $ymm15 $ymm16 $ymm17 $ymm18 $ymm19 $ymm20 $ymm21 $ymm22 $ymm23 $ymm24 $ymm25 $ymm26 $ymm27 $ymm28 $ymm29 $ymm30 $ymm31 $zmm0 $zmm1 $zmm2 $zmm3 $zmm4 $zmm5 $zmm6 $zmm7 $zmm8 $zmm9 $zmm10 $zmm11 $zmm12 $zmm13 $zmm14 $zmm15 $zmm16 $zmm17 $zmm18 $zmm19 $zmm20 $zmm21 $zmm22 $zmm23 $zmm24 $zmm25 $zmm26 $zmm27 $zmm28 $zmm29 $zmm30 $zmm31 $r11b $r11bh $r11d $r11w $r11wh
   call void @bar1()
   call void @bar2()
   ret void
diff --git a/test/CodeGen/X86/is-constant.ll b/test/CodeGen/X86/is-constant.ll
index d02bbae2085fc3d83b8704985b98a641f899c7bb..b4c1f351a2a6b1e4a670c77dc45bf1d033ff175c 100644
--- a/test/CodeGen/X86/is-constant.ll
+++ b/test/CodeGen/X86/is-constant.ll
@@ -15,7 +15,7 @@ target triple = "x86_64-unknown-linux-gnu"
 
 declare i1 @llvm.is.constant.i32(i32 %a) nounwind readnone
 declare i1 @llvm.is.constant.i64(i64 %a) nounwind readnone
-declare i64 @llvm.objectsize.i64.p0i8(i8*, i1, i1) nounwind readnone
+declare i64 @llvm.objectsize.i64.p0i8(i8*, i1, i1, i1) nounwind readnone
 
 declare i32 @subfun_1()
 declare i32 @subfun_2()
@@ -44,7 +44,7 @@ define i1 @test_objectsize(i8* %obj) nounwind {
 ; CHECK-O2:       %bb.0:
 ; CHECK-O2:       movb $1, %al
 ; CHECK-O2-NEXT:  retq
-  %os = call i64 @llvm.objectsize.i64.p0i8(i8* %obj, i1 false, i1 false)
+  %os = call i64 @llvm.objectsize.i64.p0i8(i8* %obj, i1 false, i1 false, i1 false)
   %v = call i1 @llvm.is.constant.i64(i64 %os)
   ret i1 %v
 }
diff --git a/test/CodeGen/X86/known-bits-vector.ll b/test/CodeGen/X86/known-bits-vector.ll
index d3e4843f4abed947be28bd6ef1e27d9bf92f754d..d0eaed897f4265584c7eb78855e4ef1d3d986aa6 100644
--- a/test/CodeGen/X86/known-bits-vector.ll
+++ b/test/CodeGen/X86/known-bits-vector.ll
@@ -5,18 +5,14 @@
 define i32 @knownbits_mask_extract_sext(<8 x i16> %a0) nounwind {
 ; X32-LABEL: knownbits_mask_extract_sext:
 ; X32:       # %bb.0:
-; X32-NEXT:    movl $15, %eax
-; X32-NEXT:    vmovd %eax, %xmm1
-; X32-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; X32-NEXT:    vpextrw $0, %xmm0, %eax
+; X32-NEXT:    vmovd %xmm0, %eax
+; X32-NEXT:    andl $15, %eax
 ; X32-NEXT:    retl
 ;
 ; X64-LABEL: knownbits_mask_extract_sext:
 ; X64:       # %bb.0:
-; X64-NEXT:    movl $15, %eax
-; X64-NEXT:    vmovd %eax, %xmm1
-; X64-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; X64-NEXT:    vpextrw $0, %xmm0, %eax
+; X64-NEXT:    vmovd %xmm0, %eax
+; X64-NEXT:    andl $15, %eax
 ; X64-NEXT:    retq
   %1 = and <8 x i16> %a0, <i16 15, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>
   %2 = extractelement <8 x i16> %1, i32 0
@@ -29,8 +25,7 @@ define float @knownbits_mask_extract_uitofp(<2 x i64> %a0) nounwind {
 ; X32:       # %bb.0:
 ; X32-NEXT:    pushl %eax
 ; X32-NEXT:    vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
-; X32-NEXT:    vmovd %xmm0, %eax
-; X32-NEXT:    vcvtsi2ssl %eax, %xmm1, %xmm0
+; X32-NEXT:    vcvtdq2ps %xmm0, %xmm0
 ; X32-NEXT:    vmovss %xmm0, (%esp)
 ; X32-NEXT:    flds (%esp)
 ; X32-NEXT:    popl %eax
@@ -38,8 +33,8 @@ define float @knownbits_mask_extract_uitofp(<2 x i64> %a0) nounwind {
 ;
 ; X64-LABEL: knownbits_mask_extract_uitofp:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
 ; X64-NEXT:    vmovq %xmm0, %rax
+; X64-NEXT:    movzwl %ax, %eax
 ; X64-NEXT:    vcvtsi2ssl %eax, %xmm1, %xmm0
 ; X64-NEXT:    retq
   %1 = and <2 x i64> %a0, <i64 65535, i64 -1>
@@ -162,18 +157,12 @@ define <4 x float> @knownbits_mask_shuffle_uitofp(<4 x i32> %a0) nounwind {
 define <4 x float> @knownbits_mask_or_shuffle_uitofp(<4 x i32> %a0) nounwind {
 ; X32-LABEL: knownbits_mask_or_shuffle_uitofp:
 ; X32:       # %bb.0:
-; X32-NEXT:    vandps {{\.LCPI.*}}, %xmm0, %xmm0
-; X32-NEXT:    vorps {{\.LCPI.*}}, %xmm0, %xmm0
-; X32-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[2,2,3,3]
-; X32-NEXT:    vcvtdq2ps %xmm0, %xmm0
+; X32-NEXT:    vmovaps {{.*#+}} xmm0 = [6.5535E+4,6.5535E+4,6.5535E+4,6.5535E+4]
 ; X32-NEXT:    retl
 ;
 ; X64-LABEL: knownbits_mask_or_shuffle_uitofp:
 ; X64:       # %bb.0:
-; X64-NEXT:    vandps {{.*}}(%rip), %xmm0, %xmm0
-; X64-NEXT:    vorps {{.*}}(%rip), %xmm0, %xmm0
-; X64-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[2,2,3,3]
-; X64-NEXT:    vcvtdq2ps %xmm0, %xmm0
+; X64-NEXT:    vmovaps {{.*#+}} xmm0 = [6.5535E+4,6.5535E+4,6.5535E+4,6.5535E+4]
 ; X64-NEXT:    retq
   %1 = and <4 x i32> %a0, <i32 -1, i32 -1, i32 255, i32 4085>
   %2 = or <4 x i32> %1, <i32 65535, i32 65535, i32 65535, i32 65535>
diff --git a/test/CodeGen/X86/known-signbits-vector.ll b/test/CodeGen/X86/known-signbits-vector.ll
index 0e27c71362b8a76f5048a211b34d675ddb9ab445..c4e994e6c76db87a7cf61a54e95c0c4865ce74dd 100644
--- a/test/CodeGen/X86/known-signbits-vector.ll
+++ b/test/CodeGen/X86/known-signbits-vector.ll
@@ -74,8 +74,8 @@ define float @signbits_ashr_extract_sitofp_0(<2 x i64> %a0) nounwind {
 ;
 ; X64-LABEL: signbits_ashr_extract_sitofp_0:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpsrlq $32, %xmm0, %xmm0
 ; X64-NEXT:    vmovq %xmm0, %rax
+; X64-NEXT:    shrq $32, %rax
 ; X64-NEXT:    vcvtsi2ssl %eax, %xmm1, %xmm0
 ; X64-NEXT:    retq
   %1 = ashr <2 x i64> %a0, <i64 32, i64 32>
@@ -92,8 +92,7 @@ define float @signbits_ashr_extract_sitofp_1(<2 x i64> %a0) nounwind {
 ; X32-NEXT:    vmovdqa {{.*#+}} xmm1 = [0,32768,0,0,1,0,0,0]
 ; X32-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; X32-NEXT:    vpsubq %xmm1, %xmm0, %xmm0
-; X32-NEXT:    vmovd %xmm0, %eax
-; X32-NEXT:    vcvtsi2ssl %eax, %xmm2, %xmm0
+; X32-NEXT:    vcvtdq2ps %xmm0, %xmm0
 ; X32-NEXT:    vmovss %xmm0, (%esp)
 ; X32-NEXT:    flds (%esp)
 ; X32-NEXT:    popl %eax
@@ -101,12 +100,9 @@ define float @signbits_ashr_extract_sitofp_1(<2 x i64> %a0) nounwind {
 ;
 ; X64-LABEL: signbits_ashr_extract_sitofp_1:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpsrlq $32, %xmm0, %xmm0
-; X64-NEXT:    vmovdqa {{.*#+}} xmm1 = [2147483648,1]
-; X64-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; X64-NEXT:    vpsubq %xmm1, %xmm0, %xmm0
 ; X64-NEXT:    vmovq %xmm0, %rax
-; X64-NEXT:    vcvtsi2ssl %eax, %xmm2, %xmm0
+; X64-NEXT:    shrq $32, %rax
+; X64-NEXT:    vcvtsi2ssl %eax, %xmm1, %xmm0
 ; X64-NEXT:    retq
   %1 = ashr <2 x i64> %a0, <i64 32, i64 63>
   %2 = extractelement <2 x i64> %1, i32 0
@@ -123,8 +119,7 @@ define float @signbits_ashr_shl_extract_sitofp(<2 x i64> %a0) nounwind {
 ; X32-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; X32-NEXT:    vpsubq %xmm1, %xmm0, %xmm0
 ; X32-NEXT:    vpsllq $20, %xmm0, %xmm0
-; X32-NEXT:    vmovd %xmm0, %eax
-; X32-NEXT:    vcvtsi2ssl %eax, %xmm2, %xmm0
+; X32-NEXT:    vcvtdq2ps %xmm0, %xmm0
 ; X32-NEXT:    vmovss %xmm0, (%esp)
 ; X32-NEXT:    flds (%esp)
 ; X32-NEXT:    popl %eax
@@ -132,13 +127,10 @@ define float @signbits_ashr_shl_extract_sitofp(<2 x i64> %a0) nounwind {
 ;
 ; X64-LABEL: signbits_ashr_shl_extract_sitofp:
 ; X64:       # %bb.0:
-; X64-NEXT:    vpsrlq $61, %xmm0, %xmm0
-; X64-NEXT:    vmovdqa {{.*#+}} xmm1 = [4,8]
-; X64-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; X64-NEXT:    vpsubq %xmm1, %xmm0, %xmm0
-; X64-NEXT:    vpsllq $20, %xmm0, %xmm0
 ; X64-NEXT:    vmovq %xmm0, %rax
-; X64-NEXT:    vcvtsi2ssl %eax, %xmm2, %xmm0
+; X64-NEXT:    sarq $61, %rax
+; X64-NEXT:    shll $20, %eax
+; X64-NEXT:    vcvtsi2ssl %eax, %xmm1, %xmm0
 ; X64-NEXT:    retq
   %1 = ashr <2 x i64> %a0, <i64 61, i64 60>
   %2 = shl <2 x i64> %1, <i64 20, i64 16>
@@ -158,8 +150,7 @@ define float @signbits_ashr_insert_ashr_extract_sitofp(i64 %a0, i64 %a1) nounwin
 ; X32-NEXT:    vmovd %eax, %xmm0
 ; X32-NEXT:    vpinsrd $1, %ecx, %xmm0, %xmm0
 ; X32-NEXT:    vpsrlq $3, %xmm0, %xmm0
-; X32-NEXT:    vmovd %xmm0, %eax
-; X32-NEXT:    vcvtsi2ssl %eax, %xmm1, %xmm0
+; X32-NEXT:    vcvtdq2ps %xmm0, %xmm0
 ; X32-NEXT:    vmovss %xmm0, (%esp)
 ; X32-NEXT:    flds (%esp)
 ; X32-NEXT:    popl %eax
@@ -168,10 +159,8 @@ define float @signbits_ashr_insert_ashr_extract_sitofp(i64 %a0, i64 %a1) nounwin
 ; X64-LABEL: signbits_ashr_insert_ashr_extract_sitofp:
 ; X64:       # %bb.0:
 ; X64-NEXT:    sarq $30, %rdi
-; X64-NEXT:    vmovq %rdi, %xmm0
-; X64-NEXT:    vpsrlq $3, %xmm0, %xmm0
-; X64-NEXT:    vmovq %xmm0, %rax
-; X64-NEXT:    vcvtsi2ssl %eax, %xmm1, %xmm0
+; X64-NEXT:    shrq $3, %rdi
+; X64-NEXT:    vcvtsi2ssl %edi, %xmm0, %xmm0
 ; X64-NEXT:    retq
   %1 = ashr i64 %a0, 30
   %2 = insertelement <2 x i64> undef, i64 %1, i32 0
@@ -247,8 +236,7 @@ define float @signbits_ashr_sext_sextinreg_and_extract_sitofp(<2 x i64> %a0, <2
 ; X32-NEXT:    vpsubq %xmm1, %xmm0, %xmm0
 ; X32-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
 ; X32-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; X32-NEXT:    vmovd %xmm0, %eax
-; X32-NEXT:    vcvtsi2ssl %eax, %xmm2, %xmm0
+; X32-NEXT:    vcvtdq2ps %xmm0, %xmm0
 ; X32-NEXT:    vmovss %xmm0, (%esp)
 ; X32-NEXT:    flds (%esp)
 ; X32-NEXT:    popl %eax
@@ -291,8 +279,7 @@ define float @signbits_ashr_sextvecinreg_bitops_extract_sitofp(<2 x i64> %a0, <4
 ; X32-NEXT:    vpand %xmm1, %xmm0, %xmm2
 ; X32-NEXT:    vpor %xmm1, %xmm2, %xmm1
 ; X32-NEXT:    vpxor %xmm0, %xmm1, %xmm0
-; X32-NEXT:    vmovd %xmm0, %eax
-; X32-NEXT:    vcvtsi2ssl %eax, %xmm3, %xmm0
+; X32-NEXT:    vcvtdq2ps %xmm0, %xmm0
 ; X32-NEXT:    vmovss %xmm0, (%esp)
 ; X32-NEXT:    flds (%esp)
 ; X32-NEXT:    popl %eax
@@ -393,3 +380,51 @@ define <4 x float> @signbits_ashr_sext_select_shuffle_sitofp(<4 x i64> %a0, <4 x
   %6 = sitofp <4 x i64> %5 to <4 x float>
   ret <4 x float> %6
 }
+
+; Make sure we can preserve sign bit information into the second basic block
+; so we can avoid having to shift bit 0 into bit 7 for each element due to
+; v32i1->v32i8 promotion and the splitting of v32i8 into 2xv16i8. This requires
+; ComputeNumSignBits handling for insert_subvector.
+define void @cross_bb_signbits_insert_subvec(<32 x i8>* %ptr, <32 x i8> %x, <32 x i8> %z) {
+; X32-LABEL: cross_bb_signbits_insert_subvec:
+; X32:       # %bb.0:
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; X32-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; X32-NEXT:    vpcmpeqb %xmm3, %xmm2, %xmm2
+; X32-NEXT:    vpcmpeqb %xmm3, %xmm0, %xmm0
+; X32-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; X32-NEXT:    vandnps %ymm1, %ymm0, %ymm1
+; X32-NEXT:    vandps {{\.LCPI.*}}, %ymm0, %ymm0
+; X32-NEXT:    vorps %ymm1, %ymm0, %ymm0
+; X32-NEXT:    vmovaps %ymm0, (%eax)
+; X32-NEXT:    vzeroupper
+; X32-NEXT:    retl
+;
+; X64-LABEL: cross_bb_signbits_insert_subvec:
+; X64:       # %bb.0:
+; X64-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; X64-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; X64-NEXT:    vpcmpeqb %xmm3, %xmm2, %xmm2
+; X64-NEXT:    vpcmpeqb %xmm3, %xmm0, %xmm0
+; X64-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; X64-NEXT:    vandnps %ymm1, %ymm0, %ymm1
+; X64-NEXT:    vandps {{.*}}(%rip), %ymm0, %ymm0
+; X64-NEXT:    vorps %ymm1, %ymm0, %ymm0
+; X64-NEXT:    vmovaps %ymm0, (%rdi)
+; X64-NEXT:    vzeroupper
+; X64-NEXT:    retq
+  %a = icmp eq <32 x i8> %x, zeroinitializer
+  %b = icmp eq <32 x i8> %x, zeroinitializer
+  %c = and <32 x i1> %a, %b
+  br label %block
+
+block:
+  %d = select <32 x i1> %c, <32 x i8> <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1>, <32 x i8> %z
+  store <32 x i8> %d, <32 x i8>* %ptr, align 32
+  br label %exit
+
+exit:
+  ret void
+}
+
diff --git a/test/CodeGen/X86/late-remat-update-2.mir b/test/CodeGen/X86/late-remat-update-2.mir
new file mode 100644
index 0000000000000000000000000000000000000000..c6052df62e1711681808e554ad99209e6996abbc
--- /dev/null
+++ b/test/CodeGen/X86/late-remat-update-2.mir
@@ -0,0 +1,63 @@
+# RUN: llc -mtriple=x86_64-- -run-pass=simple-register-coalescing -run-pass=regallocbasic -run-pass=virtregrewriter -late-remat-update-threshold=0 %s -o - | FileCheck %s
+#
+# PR40061: %t2 = %t1 is rematerialized and %t1 is added into toBeUpdated set
+# to postpone its live interval update. After the rematerialization, the live
+# interval of %t1 is larger than necessary. Then %t1 is merged into %t3 and %t1
+# gets removed. After the merge, %t3 contains live interval larger than
+# necessary. Because %t3 is not in toBeUpdated set so its live interval is not
+# updated after register coalescing, and it will break some assumption in
+# regalloc. This test wants to check the live interval is up-to-date after
+# register coalescing.
+#
+# To prevent the test from taking effect only in assert enabled mode, we want
+# to achieve the test goal without dumping regalloc trace. We add strong hint
+# to allocate both %t1 and %t2 to $rax register. If the %t1's live interval is
+# not shrinked properly after register coalescing, %t1 and %t2 will not be
+# both allocated to $rax because of inference, and we utilize the fact to
+# achieve the test goal. But note that the assumption only holds when we use
+# regallocbasic instead of greedy because greedy can update the live interval
+# in the process of splitting.
+#
+# CHECK-LABEL: name: foo
+# CHECK: bb.0.entry:
+# CHECK: $rax = MOV64ri32 -11
+# CHECK: bb.1:
+# CHECK: $rax = MOV64ri32 -11
+# CHECK: $rax = ADD64ri8 killed renamable $rax, 5
+# CHECK: CMP64ri8 renamable $rax
+# CHECK: RET 0, $rax
+# CHECK: bb.2:
+# CHECK: $rax = ADD64ri8 killed renamable $rax, 10
+# CHECK: bb.3:
+# CHECK: RET 0, $rax
+---
+name:            foo
+body:             |
+  bb.0.entry:
+    successors: %bb.1(0x15555555), %bb.2(0x6aaaaaab)
+
+    %t1:gr64 = MOV64ri32 -11
+    CMP64ri8 %t1, 1, implicit-def $eflags
+    JE_1 %bb.2, implicit killed $eflags
+    JMP_1 %bb.1
+
+  bb.1:
+    successors: %bb.1(0x80000000)
+
+    %t2:gr64 = COPY %t1
+    %t2:gr64 = ADD64ri8 %t2, 5, implicit-def $eflags
+    $rax = COPY %t2
+    CMP64ri8 %t2, 1, implicit-def $eflags
+    JE_1 %bb.1, implicit killed $eflags
+    RET 0, $rax
+
+  bb.2:
+    successors: %bb.3(0x80000000)
+    %t3:gr64 = COPY %t1
+    %t3:gr64 = ADD64ri8 %t3, 10, implicit-def $eflags
+
+  bb.3:
+    $rax = COPY %t3
+    RET 0, $rax
+
+...
diff --git a/test/CodeGen/X86/lea32-schedule.ll b/test/CodeGen/X86/lea32-schedule.ll
deleted file mode 100644
index 1e8ebfb766b386175291ec59017b7c336eb7d8a7..0000000000000000000000000000000000000000
--- a/test/CodeGen/X86/lea32-schedule.ll
+++ /dev/null
@@ -1,898 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=x86-64      | FileCheck %s --check-prefix=CHECK --check-prefix=GENERIC
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=atom        | FileCheck %s --check-prefix=CHECK --check-prefix=ATOM
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=slm         | FileCheck %s --check-prefix=CHECK --check-prefix=SLM
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=sandybridge | FileCheck %s --check-prefix=CHECK --check-prefix=SANDY
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=ivybridge   | FileCheck %s --check-prefix=CHECK --check-prefix=SANDY
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=haswell     | FileCheck %s --check-prefix=CHECK --check-prefix=HASWELL
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=broadwell   | FileCheck %s --check-prefix=CHECK --check-prefix=BROADWELL
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=skylake     | FileCheck %s --check-prefix=CHECK --check-prefix=SKYLAKE
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=knl         | FileCheck %s --check-prefix=CHECK --check-prefix=HASWELL
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=bdver2      | FileCheck %s --check-prefix=CHECK --check-prefix=BDVER2
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=btver2      | FileCheck %s --check-prefix=CHECK --check-prefix=BTVER2
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=znver1      | FileCheck %s --check-prefix=CHECK --check-prefix=ZNVER1
-
-define i32 @test_lea_offset(i32) {
-; GENERIC-LABEL: test_lea_offset:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    # kill: def $edi killed $edi def $rdi
-; GENERIC-NEXT:    leal -24(%rdi), %eax # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_lea_offset:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    # kill: def $edi killed $edi def $rdi
-; ATOM-NEXT:    leal -24(%rdi), %eax # sched: [1:1.00]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_lea_offset:
-; SLM:       # %bb.0:
-; SLM-NEXT:    # kill: def $edi killed $edi def $rdi
-; SLM-NEXT:    leal -24(%rdi), %eax # sched: [1:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_lea_offset:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    # kill: def $edi killed $edi def $rdi
-; SANDY-NEXT:    leal -24(%rdi), %eax # sched: [1:0.50]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_lea_offset:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    # kill: def $edi killed $edi def $rdi
-; HASWELL-NEXT:    leal -24(%rdi), %eax # sched: [1:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_lea_offset:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    # kill: def $edi killed $edi def $rdi
-; BROADWELL-NEXT:    leal -24(%rdi), %eax # sched: [1:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_lea_offset:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    # kill: def $edi killed $edi def $rdi
-; SKYLAKE-NEXT:    leal -24(%rdi), %eax # sched: [1:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_lea_offset:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    # kill: def $edi killed $edi def $rdi
-; BDVER2-NEXT:    leal -24(%rdi), %eax # sched: [1:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_lea_offset:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    # kill: def $edi killed $edi def $rdi
-; BTVER2-NEXT:    leal -24(%rdi), %eax # sched: [1:0.50]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_lea_offset:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    # kill: def $edi killed $edi def $rdi
-; ZNVER1-NEXT:    leal -24(%rdi), %eax # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %2 = add nsw i32 %0, -24
-  ret i32 %2
-}
-
-define i32 @test_lea_offset_big(i32) {
-; GENERIC-LABEL: test_lea_offset_big:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    # kill: def $edi killed $edi def $rdi
-; GENERIC-NEXT:    leal 1024(%rdi), %eax # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_lea_offset_big:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    # kill: def $edi killed $edi def $rdi
-; ATOM-NEXT:    leal 1024(%rdi), %eax # sched: [1:1.00]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_lea_offset_big:
-; SLM:       # %bb.0:
-; SLM-NEXT:    # kill: def $edi killed $edi def $rdi
-; SLM-NEXT:    leal 1024(%rdi), %eax # sched: [1:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_lea_offset_big:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    # kill: def $edi killed $edi def $rdi
-; SANDY-NEXT:    leal 1024(%rdi), %eax # sched: [1:0.50]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_lea_offset_big:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    # kill: def $edi killed $edi def $rdi
-; HASWELL-NEXT:    leal 1024(%rdi), %eax # sched: [1:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_lea_offset_big:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    # kill: def $edi killed $edi def $rdi
-; BROADWELL-NEXT:    leal 1024(%rdi), %eax # sched: [1:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_lea_offset_big:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    # kill: def $edi killed $edi def $rdi
-; SKYLAKE-NEXT:    leal 1024(%rdi), %eax # sched: [1:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_lea_offset_big:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    # kill: def $edi killed $edi def $rdi
-; BDVER2-NEXT:    leal 1024(%rdi), %eax # sched: [1:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_lea_offset_big:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    # kill: def $edi killed $edi def $rdi
-; BTVER2-NEXT:    leal 1024(%rdi), %eax # sched: [1:0.50]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_lea_offset_big:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    # kill: def $edi killed $edi def $rdi
-; ZNVER1-NEXT:    leal 1024(%rdi), %eax # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %2 = add nsw i32 %0, 1024
-  ret i32 %2
-}
-
-; Function Attrs: norecurse nounwind readnone uwtable
-define i32 @test_lea_add(i32, i32) {
-; GENERIC-LABEL: test_lea_add:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    # kill: def $esi killed $esi def $rsi
-; GENERIC-NEXT:    # kill: def $edi killed $edi def $rdi
-; GENERIC-NEXT:    leal (%rdi,%rsi), %eax # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_lea_add:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    # kill: def $esi killed $esi def $rsi
-; ATOM-NEXT:    # kill: def $edi killed $edi def $rdi
-; ATOM-NEXT:    leal (%rdi,%rsi), %eax # sched: [1:1.00]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_lea_add:
-; SLM:       # %bb.0:
-; SLM-NEXT:    # kill: def $esi killed $esi def $rsi
-; SLM-NEXT:    # kill: def $edi killed $edi def $rdi
-; SLM-NEXT:    leal (%rdi,%rsi), %eax # sched: [1:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_lea_add:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    # kill: def $esi killed $esi def $rsi
-; SANDY-NEXT:    # kill: def $edi killed $edi def $rdi
-; SANDY-NEXT:    leal (%rdi,%rsi), %eax # sched: [1:0.50]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_lea_add:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    # kill: def $esi killed $esi def $rsi
-; HASWELL-NEXT:    # kill: def $edi killed $edi def $rdi
-; HASWELL-NEXT:    leal (%rdi,%rsi), %eax # sched: [1:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_lea_add:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    # kill: def $esi killed $esi def $rsi
-; BROADWELL-NEXT:    # kill: def $edi killed $edi def $rdi
-; BROADWELL-NEXT:    leal (%rdi,%rsi), %eax # sched: [1:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_lea_add:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    # kill: def $esi killed $esi def $rsi
-; SKYLAKE-NEXT:    # kill: def $edi killed $edi def $rdi
-; SKYLAKE-NEXT:    leal (%rdi,%rsi), %eax # sched: [1:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_lea_add:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    # kill: def $esi killed $esi def $rsi
-; BDVER2-NEXT:    # kill: def $edi killed $edi def $rdi
-; BDVER2-NEXT:    leal (%rdi,%rsi), %eax # sched: [1:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_lea_add:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    # kill: def $esi killed $esi def $rsi
-; BTVER2-NEXT:    # kill: def $edi killed $edi def $rdi
-; BTVER2-NEXT:    leal (%rdi,%rsi), %eax # sched: [1:0.50]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_lea_add:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    # kill: def $esi killed $esi def $rsi
-; ZNVER1-NEXT:    # kill: def $edi killed $edi def $rdi
-; ZNVER1-NEXT:    leal (%rdi,%rsi), %eax # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %3 = add nsw i32 %1, %0
-  ret i32 %3
-}
-
-define i32 @test_lea_add_offset(i32, i32) {
-; GENERIC-LABEL: test_lea_add_offset:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    # kill: def $esi killed $esi def $rsi
-; GENERIC-NEXT:    # kill: def $edi killed $edi def $rdi
-; GENERIC-NEXT:    leal (%rdi,%rsi), %eax # sched: [1:0.50]
-; GENERIC-NEXT:    addl $16, %eax # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_lea_add_offset:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    # kill: def $esi killed $esi def $rsi
-; ATOM-NEXT:    # kill: def $edi killed $edi def $rdi
-; ATOM-NEXT:    leal 16(%rdi,%rsi), %eax # sched: [1:1.00]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_lea_add_offset:
-; SLM:       # %bb.0:
-; SLM-NEXT:    # kill: def $esi killed $esi def $rsi
-; SLM-NEXT:    # kill: def $edi killed $edi def $rdi
-; SLM-NEXT:    leal 16(%rdi,%rsi), %eax # sched: [1:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_lea_add_offset:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    # kill: def $esi killed $esi def $rsi
-; SANDY-NEXT:    # kill: def $edi killed $edi def $rdi
-; SANDY-NEXT:    leal (%rdi,%rsi), %eax # sched: [1:0.50]
-; SANDY-NEXT:    addl $16, %eax # sched: [1:0.33]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_lea_add_offset:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    # kill: def $esi killed $esi def $rsi
-; HASWELL-NEXT:    # kill: def $edi killed $edi def $rdi
-; HASWELL-NEXT:    leal (%rdi,%rsi), %eax # sched: [1:0.50]
-; HASWELL-NEXT:    addl $16, %eax # sched: [1:0.25]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_lea_add_offset:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    # kill: def $esi killed $esi def $rsi
-; BROADWELL-NEXT:    # kill: def $edi killed $edi def $rdi
-; BROADWELL-NEXT:    leal (%rdi,%rsi), %eax # sched: [1:0.50]
-; BROADWELL-NEXT:    addl $16, %eax # sched: [1:0.25]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_lea_add_offset:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    # kill: def $esi killed $esi def $rsi
-; SKYLAKE-NEXT:    # kill: def $edi killed $edi def $rdi
-; SKYLAKE-NEXT:    leal (%rdi,%rsi), %eax # sched: [1:0.50]
-; SKYLAKE-NEXT:    addl $16, %eax # sched: [1:0.25]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_lea_add_offset:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    # kill: def $esi killed $esi def $rsi
-; BDVER2-NEXT:    # kill: def $edi killed $edi def $rdi
-; BDVER2-NEXT:    leal 16(%rdi,%rsi), %eax # sched: [1:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_lea_add_offset:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    # kill: def $esi killed $esi def $rsi
-; BTVER2-NEXT:    # kill: def $edi killed $edi def $rdi
-; BTVER2-NEXT:    leal 16(%rdi,%rsi), %eax # sched: [2:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_lea_add_offset:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    # kill: def $esi killed $esi def $rsi
-; ZNVER1-NEXT:    # kill: def $edi killed $edi def $rdi
-; ZNVER1-NEXT:    leal 16(%rdi,%rsi), %eax # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %3 = add i32 %0, 16
-  %4 = add i32 %3, %1
-  ret i32 %4
-}
-
-define i32 @test_lea_add_offset_big(i32, i32) {
-; GENERIC-LABEL: test_lea_add_offset_big:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    # kill: def $esi killed $esi def $rsi
-; GENERIC-NEXT:    # kill: def $edi killed $edi def $rdi
-; GENERIC-NEXT:    leal (%rdi,%rsi), %eax # sched: [1:0.50]
-; GENERIC-NEXT:    addl $-4096, %eax # imm = 0xF000
-; GENERIC-NEXT:    # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_lea_add_offset_big:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    # kill: def $esi killed $esi def $rsi
-; ATOM-NEXT:    # kill: def $edi killed $edi def $rdi
-; ATOM-NEXT:    leal -4096(%rdi,%rsi), %eax # sched: [1:1.00]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_lea_add_offset_big:
-; SLM:       # %bb.0:
-; SLM-NEXT:    # kill: def $esi killed $esi def $rsi
-; SLM-NEXT:    # kill: def $edi killed $edi def $rdi
-; SLM-NEXT:    leal -4096(%rdi,%rsi), %eax # sched: [1:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_lea_add_offset_big:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    # kill: def $esi killed $esi def $rsi
-; SANDY-NEXT:    # kill: def $edi killed $edi def $rdi
-; SANDY-NEXT:    leal (%rdi,%rsi), %eax # sched: [1:0.50]
-; SANDY-NEXT:    addl $-4096, %eax # imm = 0xF000
-; SANDY-NEXT:    # sched: [1:0.33]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_lea_add_offset_big:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    # kill: def $esi killed $esi def $rsi
-; HASWELL-NEXT:    # kill: def $edi killed $edi def $rdi
-; HASWELL-NEXT:    leal (%rdi,%rsi), %eax # sched: [1:0.50]
-; HASWELL-NEXT:    addl $-4096, %eax # imm = 0xF000
-; HASWELL-NEXT:    # sched: [1:0.25]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_lea_add_offset_big:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    # kill: def $esi killed $esi def $rsi
-; BROADWELL-NEXT:    # kill: def $edi killed $edi def $rdi
-; BROADWELL-NEXT:    leal (%rdi,%rsi), %eax # sched: [1:0.50]
-; BROADWELL-NEXT:    addl $-4096, %eax # imm = 0xF000
-; BROADWELL-NEXT:    # sched: [1:0.25]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_lea_add_offset_big:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    # kill: def $esi killed $esi def $rsi
-; SKYLAKE-NEXT:    # kill: def $edi killed $edi def $rdi
-; SKYLAKE-NEXT:    leal (%rdi,%rsi), %eax # sched: [1:0.50]
-; SKYLAKE-NEXT:    addl $-4096, %eax # imm = 0xF000
-; SKYLAKE-NEXT:    # sched: [1:0.25]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_lea_add_offset_big:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    # kill: def $esi killed $esi def $rsi
-; BDVER2-NEXT:    # kill: def $edi killed $edi def $rdi
-; BDVER2-NEXT:    leal -4096(%rdi,%rsi), %eax # sched: [1:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_lea_add_offset_big:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    # kill: def $esi killed $esi def $rsi
-; BTVER2-NEXT:    # kill: def $edi killed $edi def $rdi
-; BTVER2-NEXT:    leal -4096(%rdi,%rsi), %eax # sched: [2:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_lea_add_offset_big:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    # kill: def $esi killed $esi def $rsi
-; ZNVER1-NEXT:    # kill: def $edi killed $edi def $rdi
-; ZNVER1-NEXT:    leal -4096(%rdi,%rsi), %eax # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %3 = add i32 %0, -4096
-  %4 = add i32 %3, %1
-  ret i32 %4
-}
-
-define i32 @test_lea_mul(i32) {
-; GENERIC-LABEL: test_lea_mul:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    # kill: def $edi killed $edi def $rdi
-; GENERIC-NEXT:    leal (%rdi,%rdi,2), %eax # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_lea_mul:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    # kill: def $edi killed $edi def $rdi
-; ATOM-NEXT:    leal (%rdi,%rdi,2), %eax # sched: [1:1.00]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_lea_mul:
-; SLM:       # %bb.0:
-; SLM-NEXT:    # kill: def $edi killed $edi def $rdi
-; SLM-NEXT:    leal (%rdi,%rdi,2), %eax # sched: [1:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_lea_mul:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    # kill: def $edi killed $edi def $rdi
-; SANDY-NEXT:    leal (%rdi,%rdi,2), %eax # sched: [1:0.50]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_lea_mul:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    # kill: def $edi killed $edi def $rdi
-; HASWELL-NEXT:    leal (%rdi,%rdi,2), %eax # sched: [1:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_lea_mul:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    # kill: def $edi killed $edi def $rdi
-; BROADWELL-NEXT:    leal (%rdi,%rdi,2), %eax # sched: [1:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_lea_mul:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    # kill: def $edi killed $edi def $rdi
-; SKYLAKE-NEXT:    leal (%rdi,%rdi,2), %eax # sched: [1:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_lea_mul:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    # kill: def $edi killed $edi def $rdi
-; BDVER2-NEXT:    leal (%rdi,%rdi,2), %eax # sched: [1:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_lea_mul:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    # kill: def $edi killed $edi def $rdi
-; BTVER2-NEXT:    leal (%rdi,%rdi,2), %eax # sched: [2:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_lea_mul:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    # kill: def $edi killed $edi def $rdi
-; ZNVER1-NEXT:    leal (%rdi,%rdi,2), %eax # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %2 = mul nsw i32 %0, 3
-  ret i32 %2
-}
-
-define i32 @test_lea_mul_offset(i32) {
-; GENERIC-LABEL: test_lea_mul_offset:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    # kill: def $edi killed $edi def $rdi
-; GENERIC-NEXT:    leal (%rdi,%rdi,2), %eax # sched: [1:0.50]
-; GENERIC-NEXT:    addl $-32, %eax # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_lea_mul_offset:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    # kill: def $edi killed $edi def $rdi
-; ATOM-NEXT:    leal -32(%rdi,%rdi,2), %eax # sched: [1:1.00]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_lea_mul_offset:
-; SLM:       # %bb.0:
-; SLM-NEXT:    # kill: def $edi killed $edi def $rdi
-; SLM-NEXT:    leal -32(%rdi,%rdi,2), %eax # sched: [1:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_lea_mul_offset:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    # kill: def $edi killed $edi def $rdi
-; SANDY-NEXT:    leal (%rdi,%rdi,2), %eax # sched: [1:0.50]
-; SANDY-NEXT:    addl $-32, %eax # sched: [1:0.33]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_lea_mul_offset:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    # kill: def $edi killed $edi def $rdi
-; HASWELL-NEXT:    leal (%rdi,%rdi,2), %eax # sched: [1:0.50]
-; HASWELL-NEXT:    addl $-32, %eax # sched: [1:0.25]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_lea_mul_offset:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    # kill: def $edi killed $edi def $rdi
-; BROADWELL-NEXT:    leal (%rdi,%rdi,2), %eax # sched: [1:0.50]
-; BROADWELL-NEXT:    addl $-32, %eax # sched: [1:0.25]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_lea_mul_offset:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    # kill: def $edi killed $edi def $rdi
-; SKYLAKE-NEXT:    leal (%rdi,%rdi,2), %eax # sched: [1:0.50]
-; SKYLAKE-NEXT:    addl $-32, %eax # sched: [1:0.25]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_lea_mul_offset:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    # kill: def $edi killed $edi def $rdi
-; BDVER2-NEXT:    leal -32(%rdi,%rdi,2), %eax # sched: [1:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_lea_mul_offset:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    # kill: def $edi killed $edi def $rdi
-; BTVER2-NEXT:    leal -32(%rdi,%rdi,2), %eax # sched: [2:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_lea_mul_offset:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    # kill: def $edi killed $edi def $rdi
-; ZNVER1-NEXT:    leal -32(%rdi,%rdi,2), %eax # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %2 = mul nsw i32 %0, 3
-  %3 = add nsw i32 %2, -32
-  ret i32 %3
-}
-
-define i32 @test_lea_mul_offset_big(i32) {
-; GENERIC-LABEL: test_lea_mul_offset_big:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    # kill: def $edi killed $edi def $rdi
-; GENERIC-NEXT:    leal (%rdi,%rdi,8), %eax # sched: [1:0.50]
-; GENERIC-NEXT:    addl $10000, %eax # imm = 0x2710
-; GENERIC-NEXT:    # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_lea_mul_offset_big:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    # kill: def $edi killed $edi def $rdi
-; ATOM-NEXT:    leal 10000(%rdi,%rdi,8), %eax # sched: [1:1.00]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_lea_mul_offset_big:
-; SLM:       # %bb.0:
-; SLM-NEXT:    # kill: def $edi killed $edi def $rdi
-; SLM-NEXT:    leal 10000(%rdi,%rdi,8), %eax # sched: [1:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_lea_mul_offset_big:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    # kill: def $edi killed $edi def $rdi
-; SANDY-NEXT:    leal (%rdi,%rdi,8), %eax # sched: [1:0.50]
-; SANDY-NEXT:    addl $10000, %eax # imm = 0x2710
-; SANDY-NEXT:    # sched: [1:0.33]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_lea_mul_offset_big:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    # kill: def $edi killed $edi def $rdi
-; HASWELL-NEXT:    leal (%rdi,%rdi,8), %eax # sched: [1:0.50]
-; HASWELL-NEXT:    addl $10000, %eax # imm = 0x2710
-; HASWELL-NEXT:    # sched: [1:0.25]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_lea_mul_offset_big:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    # kill: def $edi killed $edi def $rdi
-; BROADWELL-NEXT:    leal (%rdi,%rdi,8), %eax # sched: [1:0.50]
-; BROADWELL-NEXT:    addl $10000, %eax # imm = 0x2710
-; BROADWELL-NEXT:    # sched: [1:0.25]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_lea_mul_offset_big:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    # kill: def $edi killed $edi def $rdi
-; SKYLAKE-NEXT:    leal (%rdi,%rdi,8), %eax # sched: [1:0.50]
-; SKYLAKE-NEXT:    addl $10000, %eax # imm = 0x2710
-; SKYLAKE-NEXT:    # sched: [1:0.25]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_lea_mul_offset_big:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    # kill: def $edi killed $edi def $rdi
-; BDVER2-NEXT:    leal 10000(%rdi,%rdi,8), %eax # sched: [1:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_lea_mul_offset_big:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    # kill: def $edi killed $edi def $rdi
-; BTVER2-NEXT:    leal 10000(%rdi,%rdi,8), %eax # sched: [2:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_lea_mul_offset_big:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    # kill: def $edi killed $edi def $rdi
-; ZNVER1-NEXT:    leal 10000(%rdi,%rdi,8), %eax # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %2 = mul nsw i32 %0, 9
-  %3 = add nsw i32 %2, 10000
-  ret i32 %3
-}
-
-define i32 @test_lea_add_scale(i32, i32) {
-; GENERIC-LABEL: test_lea_add_scale:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    # kill: def $esi killed $esi def $rsi
-; GENERIC-NEXT:    # kill: def $edi killed $edi def $rdi
-; GENERIC-NEXT:    leal (%rdi,%rsi,2), %eax # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_lea_add_scale:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    # kill: def $esi killed $esi def $rsi
-; ATOM-NEXT:    # kill: def $edi killed $edi def $rdi
-; ATOM-NEXT:    leal (%rdi,%rsi,2), %eax # sched: [1:1.00]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_lea_add_scale:
-; SLM:       # %bb.0:
-; SLM-NEXT:    # kill: def $esi killed $esi def $rsi
-; SLM-NEXT:    # kill: def $edi killed $edi def $rdi
-; SLM-NEXT:    leal (%rdi,%rsi,2), %eax # sched: [1:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_lea_add_scale:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    # kill: def $esi killed $esi def $rsi
-; SANDY-NEXT:    # kill: def $edi killed $edi def $rdi
-; SANDY-NEXT:    leal (%rdi,%rsi,2), %eax # sched: [1:0.50]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_lea_add_scale:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    # kill: def $esi killed $esi def $rsi
-; HASWELL-NEXT:    # kill: def $edi killed $edi def $rdi
-; HASWELL-NEXT:    leal (%rdi,%rsi,2), %eax # sched: [1:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_lea_add_scale:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    # kill: def $esi killed $esi def $rsi
-; BROADWELL-NEXT:    # kill: def $edi killed $edi def $rdi
-; BROADWELL-NEXT:    leal (%rdi,%rsi,2), %eax # sched: [1:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_lea_add_scale:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    # kill: def $esi killed $esi def $rsi
-; SKYLAKE-NEXT:    # kill: def $edi killed $edi def $rdi
-; SKYLAKE-NEXT:    leal (%rdi,%rsi,2), %eax # sched: [1:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_lea_add_scale:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    # kill: def $esi killed $esi def $rsi
-; BDVER2-NEXT:    # kill: def $edi killed $edi def $rdi
-; BDVER2-NEXT:    leal (%rdi,%rsi,2), %eax # sched: [1:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_lea_add_scale:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    # kill: def $esi killed $esi def $rsi
-; BTVER2-NEXT:    # kill: def $edi killed $edi def $rdi
-; BTVER2-NEXT:    leal (%rdi,%rsi,2), %eax # sched: [2:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_lea_add_scale:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    # kill: def $esi killed $esi def $rsi
-; ZNVER1-NEXT:    # kill: def $edi killed $edi def $rdi
-; ZNVER1-NEXT:    leal (%rdi,%rsi,2), %eax # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %3 = shl i32 %1, 1
-  %4 = add nsw i32 %3, %0
-  ret i32 %4
-}
-
-define i32 @test_lea_add_scale_offset(i32, i32) {
-; GENERIC-LABEL: test_lea_add_scale_offset:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    # kill: def $esi killed $esi def $rsi
-; GENERIC-NEXT:    # kill: def $edi killed $edi def $rdi
-; GENERIC-NEXT:    leal (%rdi,%rsi,4), %eax # sched: [1:0.50]
-; GENERIC-NEXT:    addl $96, %eax # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_lea_add_scale_offset:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    # kill: def $esi killed $esi def $rsi
-; ATOM-NEXT:    # kill: def $edi killed $edi def $rdi
-; ATOM-NEXT:    leal 96(%rdi,%rsi,4), %eax # sched: [1:1.00]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_lea_add_scale_offset:
-; SLM:       # %bb.0:
-; SLM-NEXT:    # kill: def $esi killed $esi def $rsi
-; SLM-NEXT:    # kill: def $edi killed $edi def $rdi
-; SLM-NEXT:    leal 96(%rdi,%rsi,4), %eax # sched: [1:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_lea_add_scale_offset:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    # kill: def $esi killed $esi def $rsi
-; SANDY-NEXT:    # kill: def $edi killed $edi def $rdi
-; SANDY-NEXT:    leal (%rdi,%rsi,4), %eax # sched: [1:0.50]
-; SANDY-NEXT:    addl $96, %eax # sched: [1:0.33]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_lea_add_scale_offset:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    # kill: def $esi killed $esi def $rsi
-; HASWELL-NEXT:    # kill: def $edi killed $edi def $rdi
-; HASWELL-NEXT:    leal (%rdi,%rsi,4), %eax # sched: [1:0.50]
-; HASWELL-NEXT:    addl $96, %eax # sched: [1:0.25]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_lea_add_scale_offset:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    # kill: def $esi killed $esi def $rsi
-; BROADWELL-NEXT:    # kill: def $edi killed $edi def $rdi
-; BROADWELL-NEXT:    leal (%rdi,%rsi,4), %eax # sched: [1:0.50]
-; BROADWELL-NEXT:    addl $96, %eax # sched: [1:0.25]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_lea_add_scale_offset:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    # kill: def $esi killed $esi def $rsi
-; SKYLAKE-NEXT:    # kill: def $edi killed $edi def $rdi
-; SKYLAKE-NEXT:    leal (%rdi,%rsi,4), %eax # sched: [1:0.50]
-; SKYLAKE-NEXT:    addl $96, %eax # sched: [1:0.25]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_lea_add_scale_offset:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    # kill: def $esi killed $esi def $rsi
-; BDVER2-NEXT:    # kill: def $edi killed $edi def $rdi
-; BDVER2-NEXT:    leal 96(%rdi,%rsi,4), %eax # sched: [1:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_lea_add_scale_offset:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    # kill: def $esi killed $esi def $rsi
-; BTVER2-NEXT:    # kill: def $edi killed $edi def $rdi
-; BTVER2-NEXT:    leal 96(%rdi,%rsi,4), %eax # sched: [2:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_lea_add_scale_offset:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    # kill: def $esi killed $esi def $rsi
-; ZNVER1-NEXT:    # kill: def $edi killed $edi def $rdi
-; ZNVER1-NEXT:    leal 96(%rdi,%rsi,4), %eax # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %3 = shl i32 %1, 2
-  %4 = add i32 %0, 96
-  %5 = add i32 %4, %3
-  ret i32 %5
-}
-
-define i32 @test_lea_add_scale_offset_big(i32, i32) {
-; GENERIC-LABEL: test_lea_add_scale_offset_big:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    # kill: def $esi killed $esi def $rsi
-; GENERIC-NEXT:    # kill: def $edi killed $edi def $rdi
-; GENERIC-NEXT:    leal (%rdi,%rsi,8), %eax # sched: [1:0.50]
-; GENERIC-NEXT:    addl $-1200, %eax # imm = 0xFB50
-; GENERIC-NEXT:    # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_lea_add_scale_offset_big:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    # kill: def $esi killed $esi def $rsi
-; ATOM-NEXT:    # kill: def $edi killed $edi def $rdi
-; ATOM-NEXT:    leal -1200(%rdi,%rsi,8), %eax # sched: [1:1.00]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_lea_add_scale_offset_big:
-; SLM:       # %bb.0:
-; SLM-NEXT:    # kill: def $esi killed $esi def $rsi
-; SLM-NEXT:    # kill: def $edi killed $edi def $rdi
-; SLM-NEXT:    leal -1200(%rdi,%rsi,8), %eax # sched: [1:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_lea_add_scale_offset_big:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    # kill: def $esi killed $esi def $rsi
-; SANDY-NEXT:    # kill: def $edi killed $edi def $rdi
-; SANDY-NEXT:    leal (%rdi,%rsi,8), %eax # sched: [1:0.50]
-; SANDY-NEXT:    addl $-1200, %eax # imm = 0xFB50
-; SANDY-NEXT:    # sched: [1:0.33]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_lea_add_scale_offset_big:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    # kill: def $esi killed $esi def $rsi
-; HASWELL-NEXT:    # kill: def $edi killed $edi def $rdi
-; HASWELL-NEXT:    leal (%rdi,%rsi,8), %eax # sched: [1:0.50]
-; HASWELL-NEXT:    addl $-1200, %eax # imm = 0xFB50
-; HASWELL-NEXT:    # sched: [1:0.25]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_lea_add_scale_offset_big:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    # kill: def $esi killed $esi def $rsi
-; BROADWELL-NEXT:    # kill: def $edi killed $edi def $rdi
-; BROADWELL-NEXT:    leal (%rdi,%rsi,8), %eax # sched: [1:0.50]
-; BROADWELL-NEXT:    addl $-1200, %eax # imm = 0xFB50
-; BROADWELL-NEXT:    # sched: [1:0.25]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_lea_add_scale_offset_big:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    # kill: def $esi killed $esi def $rsi
-; SKYLAKE-NEXT:    # kill: def $edi killed $edi def $rdi
-; SKYLAKE-NEXT:    leal (%rdi,%rsi,8), %eax # sched: [1:0.50]
-; SKYLAKE-NEXT:    addl $-1200, %eax # imm = 0xFB50
-; SKYLAKE-NEXT:    # sched: [1:0.25]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_lea_add_scale_offset_big:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    # kill: def $esi killed $esi def $rsi
-; BDVER2-NEXT:    # kill: def $edi killed $edi def $rdi
-; BDVER2-NEXT:    leal -1200(%rdi,%rsi,8), %eax # sched: [1:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_lea_add_scale_offset_big:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    # kill: def $esi killed $esi def $rsi
-; BTVER2-NEXT:    # kill: def $edi killed $edi def $rdi
-; BTVER2-NEXT:    leal -1200(%rdi,%rsi,8), %eax # sched: [2:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_lea_add_scale_offset_big:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    # kill: def $esi killed $esi def $rsi
-; ZNVER1-NEXT:    # kill: def $edi killed $edi def $rdi
-; ZNVER1-NEXT:    leal -1200(%rdi,%rsi,8), %eax # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %3 = shl i32 %1, 3
-  %4 = add i32 %0, -1200
-  %5 = add i32 %4, %3
-  ret i32 %5
-}
diff --git a/test/CodeGen/X86/lea64-schedule.ll b/test/CodeGen/X86/lea64-schedule.ll
deleted file mode 100644
index cac9d2b506267664d77edb0683e5c813a4965eba..0000000000000000000000000000000000000000
--- a/test/CodeGen/X86/lea64-schedule.ll
+++ /dev/null
@@ -1,728 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=x86-64      | FileCheck %s --check-prefix=CHECK --check-prefix=GENERIC
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=atom        | FileCheck %s --check-prefix=CHECK --check-prefix=ATOM
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=slm         | FileCheck %s --check-prefix=CHECK --check-prefix=SLM
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=sandybridge | FileCheck %s --check-prefix=CHECK --check-prefix=SANDY
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=ivybridge   | FileCheck %s --check-prefix=CHECK --check-prefix=SANDY
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=haswell     | FileCheck %s --check-prefix=CHECK --check-prefix=HASWELL
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=broadwell   | FileCheck %s --check-prefix=CHECK --check-prefix=BROADWELL
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=skylake     | FileCheck %s --check-prefix=CHECK --check-prefix=SKYLAKE
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=knl         | FileCheck %s --check-prefix=CHECK --check-prefix=HASWELL
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=bdver2      | FileCheck %s --check-prefix=CHECK --check-prefix=BDVER2
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=btver2      | FileCheck %s --check-prefix=CHECK --check-prefix=BTVER2
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=znver1      | FileCheck %s --check-prefix=CHECK --check-prefix=ZNVER1
-
-define i64 @test_lea_offset(i64) {
-; GENERIC-LABEL: test_lea_offset:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    leaq -24(%rdi), %rax # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_lea_offset:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    leaq -24(%rdi), %rax # sched: [1:1.00]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_lea_offset:
-; SLM:       # %bb.0:
-; SLM-NEXT:    leaq -24(%rdi), %rax # sched: [1:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_lea_offset:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    leaq -24(%rdi), %rax # sched: [1:0.50]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_lea_offset:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    leaq -24(%rdi), %rax # sched: [1:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_lea_offset:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    leaq -24(%rdi), %rax # sched: [1:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_lea_offset:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    leaq -24(%rdi), %rax # sched: [1:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_lea_offset:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    leaq -24(%rdi), %rax # sched: [1:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_lea_offset:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    leaq -24(%rdi), %rax # sched: [1:0.50]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_lea_offset:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    leaq -24(%rdi), %rax # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %2 = add nsw i64 %0, -24
-  ret i64 %2
-}
-
-define i64 @test_lea_offset_big(i64) {
-; GENERIC-LABEL: test_lea_offset_big:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    leaq 1024(%rdi), %rax # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_lea_offset_big:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    leaq 1024(%rdi), %rax # sched: [1:1.00]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_lea_offset_big:
-; SLM:       # %bb.0:
-; SLM-NEXT:    leaq 1024(%rdi), %rax # sched: [1:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_lea_offset_big:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    leaq 1024(%rdi), %rax # sched: [1:0.50]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_lea_offset_big:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    leaq 1024(%rdi), %rax # sched: [1:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_lea_offset_big:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    leaq 1024(%rdi), %rax # sched: [1:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_lea_offset_big:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    leaq 1024(%rdi), %rax # sched: [1:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_lea_offset_big:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    leaq 1024(%rdi), %rax # sched: [1:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_lea_offset_big:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    leaq 1024(%rdi), %rax # sched: [1:0.50]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_lea_offset_big:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    leaq 1024(%rdi), %rax # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %2 = add nsw i64 %0, 1024
-  ret i64 %2
-}
-
-; Function Attrs: norecurse nounwind readnone uwtable
-define i64 @test_lea_add(i64, i64) {
-; GENERIC-LABEL: test_lea_add:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    leaq (%rdi,%rsi), %rax # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_lea_add:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    leaq (%rdi,%rsi), %rax # sched: [1:1.00]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_lea_add:
-; SLM:       # %bb.0:
-; SLM-NEXT:    leaq (%rdi,%rsi), %rax # sched: [1:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_lea_add:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    leaq (%rdi,%rsi), %rax # sched: [1:0.50]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_lea_add:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    leaq (%rdi,%rsi), %rax # sched: [1:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_lea_add:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    leaq (%rdi,%rsi), %rax # sched: [1:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_lea_add:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    leaq (%rdi,%rsi), %rax # sched: [1:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_lea_add:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    leaq (%rdi,%rsi), %rax # sched: [1:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_lea_add:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    leaq (%rdi,%rsi), %rax # sched: [1:0.50]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_lea_add:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    leaq (%rdi,%rsi), %rax # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %3 = add nsw i64 %1, %0
-  ret i64 %3
-}
-
-define i64 @test_lea_add_offset(i64, i64) {
-; GENERIC-LABEL: test_lea_add_offset:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    leaq (%rdi,%rsi), %rax # sched: [1:0.50]
-; GENERIC-NEXT:    addq $16, %rax # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_lea_add_offset:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    leaq 16(%rdi,%rsi), %rax # sched: [1:1.00]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_lea_add_offset:
-; SLM:       # %bb.0:
-; SLM-NEXT:    leaq 16(%rdi,%rsi), %rax # sched: [1:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_lea_add_offset:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    leaq (%rdi,%rsi), %rax # sched: [1:0.50]
-; SANDY-NEXT:    addq $16, %rax # sched: [1:0.33]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_lea_add_offset:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    leaq (%rdi,%rsi), %rax # sched: [1:0.50]
-; HASWELL-NEXT:    addq $16, %rax # sched: [1:0.25]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_lea_add_offset:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    leaq (%rdi,%rsi), %rax # sched: [1:0.50]
-; BROADWELL-NEXT:    addq $16, %rax # sched: [1:0.25]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_lea_add_offset:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    leaq (%rdi,%rsi), %rax # sched: [1:0.50]
-; SKYLAKE-NEXT:    addq $16, %rax # sched: [1:0.25]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_lea_add_offset:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    leaq 16(%rdi,%rsi), %rax # sched: [1:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_lea_add_offset:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    leaq 16(%rdi,%rsi), %rax # sched: [2:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_lea_add_offset:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    leaq 16(%rdi,%rsi), %rax # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %3 = add i64 %0, 16
-  %4 = add i64 %3, %1
-  ret i64 %4
-}
-
-define i64 @test_lea_add_offset_big(i64, i64) {
-; GENERIC-LABEL: test_lea_add_offset_big:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    leaq (%rdi,%rsi), %rax # sched: [1:0.50]
-; GENERIC-NEXT:    addq $-4096, %rax # imm = 0xF000
-; GENERIC-NEXT:    # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_lea_add_offset_big:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    leaq -4096(%rdi,%rsi), %rax # sched: [1:1.00]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_lea_add_offset_big:
-; SLM:       # %bb.0:
-; SLM-NEXT:    leaq -4096(%rdi,%rsi), %rax # sched: [1:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_lea_add_offset_big:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    leaq (%rdi,%rsi), %rax # sched: [1:0.50]
-; SANDY-NEXT:    addq $-4096, %rax # imm = 0xF000
-; SANDY-NEXT:    # sched: [1:0.33]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_lea_add_offset_big:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    leaq (%rdi,%rsi), %rax # sched: [1:0.50]
-; HASWELL-NEXT:    addq $-4096, %rax # imm = 0xF000
-; HASWELL-NEXT:    # sched: [1:0.25]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_lea_add_offset_big:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    leaq (%rdi,%rsi), %rax # sched: [1:0.50]
-; BROADWELL-NEXT:    addq $-4096, %rax # imm = 0xF000
-; BROADWELL-NEXT:    # sched: [1:0.25]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_lea_add_offset_big:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    leaq (%rdi,%rsi), %rax # sched: [1:0.50]
-; SKYLAKE-NEXT:    addq $-4096, %rax # imm = 0xF000
-; SKYLAKE-NEXT:    # sched: [1:0.25]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_lea_add_offset_big:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    leaq -4096(%rdi,%rsi), %rax # sched: [1:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_lea_add_offset_big:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    leaq -4096(%rdi,%rsi), %rax # sched: [2:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_lea_add_offset_big:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    leaq -4096(%rdi,%rsi), %rax # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %3 = add i64 %0, -4096
-  %4 = add i64 %3, %1
-  ret i64 %4
-}
-
-define i64 @test_lea_mul(i64) {
-; GENERIC-LABEL: test_lea_mul:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    leaq (%rdi,%rdi,2), %rax # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_lea_mul:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    leaq (%rdi,%rdi,2), %rax # sched: [1:1.00]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_lea_mul:
-; SLM:       # %bb.0:
-; SLM-NEXT:    leaq (%rdi,%rdi,2), %rax # sched: [1:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_lea_mul:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    leaq (%rdi,%rdi,2), %rax # sched: [1:0.50]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_lea_mul:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    leaq (%rdi,%rdi,2), %rax # sched: [1:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_lea_mul:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    leaq (%rdi,%rdi,2), %rax # sched: [1:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_lea_mul:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    leaq (%rdi,%rdi,2), %rax # sched: [1:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_lea_mul:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    leaq (%rdi,%rdi,2), %rax # sched: [1:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_lea_mul:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    leaq (%rdi,%rdi,2), %rax # sched: [2:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_lea_mul:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    leaq (%rdi,%rdi,2), %rax # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %2 = mul nsw i64 %0, 3
-  ret i64 %2
-}
-
-define i64 @test_lea_mul_offset(i64) {
-; GENERIC-LABEL: test_lea_mul_offset:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    leaq (%rdi,%rdi,2), %rax # sched: [1:0.50]
-; GENERIC-NEXT:    addq $-32, %rax # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_lea_mul_offset:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    leaq -32(%rdi,%rdi,2), %rax # sched: [1:1.00]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_lea_mul_offset:
-; SLM:       # %bb.0:
-; SLM-NEXT:    leaq -32(%rdi,%rdi,2), %rax # sched: [1:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_lea_mul_offset:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    leaq (%rdi,%rdi,2), %rax # sched: [1:0.50]
-; SANDY-NEXT:    addq $-32, %rax # sched: [1:0.33]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_lea_mul_offset:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    leaq (%rdi,%rdi,2), %rax # sched: [1:0.50]
-; HASWELL-NEXT:    addq $-32, %rax # sched: [1:0.25]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_lea_mul_offset:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    leaq (%rdi,%rdi,2), %rax # sched: [1:0.50]
-; BROADWELL-NEXT:    addq $-32, %rax # sched: [1:0.25]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_lea_mul_offset:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    leaq (%rdi,%rdi,2), %rax # sched: [1:0.50]
-; SKYLAKE-NEXT:    addq $-32, %rax # sched: [1:0.25]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_lea_mul_offset:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    leaq -32(%rdi,%rdi,2), %rax # sched: [1:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_lea_mul_offset:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    leaq -32(%rdi,%rdi,2), %rax # sched: [2:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_lea_mul_offset:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    leaq -32(%rdi,%rdi,2), %rax # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %2 = mul nsw i64 %0, 3
-  %3 = add nsw i64 %2, -32
-  ret i64 %3
-}
-
-define i64 @test_lea_mul_offset_big(i64) {
-; GENERIC-LABEL: test_lea_mul_offset_big:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    leaq (%rdi,%rdi,8), %rax # sched: [1:0.50]
-; GENERIC-NEXT:    addq $10000, %rax # imm = 0x2710
-; GENERIC-NEXT:    # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_lea_mul_offset_big:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    leaq 10000(%rdi,%rdi,8), %rax # sched: [1:1.00]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_lea_mul_offset_big:
-; SLM:       # %bb.0:
-; SLM-NEXT:    leaq 10000(%rdi,%rdi,8), %rax # sched: [1:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_lea_mul_offset_big:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    leaq (%rdi,%rdi,8), %rax # sched: [1:0.50]
-; SANDY-NEXT:    addq $10000, %rax # imm = 0x2710
-; SANDY-NEXT:    # sched: [1:0.33]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_lea_mul_offset_big:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    leaq (%rdi,%rdi,8), %rax # sched: [1:0.50]
-; HASWELL-NEXT:    addq $10000, %rax # imm = 0x2710
-; HASWELL-NEXT:    # sched: [1:0.25]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_lea_mul_offset_big:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    leaq (%rdi,%rdi,8), %rax # sched: [1:0.50]
-; BROADWELL-NEXT:    addq $10000, %rax # imm = 0x2710
-; BROADWELL-NEXT:    # sched: [1:0.25]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_lea_mul_offset_big:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    leaq (%rdi,%rdi,8), %rax # sched: [1:0.50]
-; SKYLAKE-NEXT:    addq $10000, %rax # imm = 0x2710
-; SKYLAKE-NEXT:    # sched: [1:0.25]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_lea_mul_offset_big:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    leaq 10000(%rdi,%rdi,8), %rax # sched: [1:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_lea_mul_offset_big:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    leaq 10000(%rdi,%rdi,8), %rax # sched: [2:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_lea_mul_offset_big:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    leaq 10000(%rdi,%rdi,8), %rax # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %2 = mul nsw i64 %0, 9
-  %3 = add nsw i64 %2, 10000
-  ret i64 %3
-}
-
-define i64 @test_lea_add_scale(i64, i64) {
-; GENERIC-LABEL: test_lea_add_scale:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    leaq (%rdi,%rsi,2), %rax # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_lea_add_scale:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    leaq (%rdi,%rsi,2), %rax # sched: [1:1.00]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_lea_add_scale:
-; SLM:       # %bb.0:
-; SLM-NEXT:    leaq (%rdi,%rsi,2), %rax # sched: [1:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_lea_add_scale:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    leaq (%rdi,%rsi,2), %rax # sched: [1:0.50]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_lea_add_scale:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    leaq (%rdi,%rsi,2), %rax # sched: [1:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_lea_add_scale:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    leaq (%rdi,%rsi,2), %rax # sched: [1:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_lea_add_scale:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    leaq (%rdi,%rsi,2), %rax # sched: [1:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_lea_add_scale:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    leaq (%rdi,%rsi,2), %rax # sched: [1:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_lea_add_scale:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    leaq (%rdi,%rsi,2), %rax # sched: [2:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_lea_add_scale:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    leaq (%rdi,%rsi,2), %rax # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %3 = shl i64 %1, 1
-  %4 = add nsw i64 %3, %0
-  ret i64 %4
-}
-
-define i64 @test_lea_add_scale_offset(i64, i64) {
-; GENERIC-LABEL: test_lea_add_scale_offset:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    leaq (%rdi,%rsi,4), %rax # sched: [1:0.50]
-; GENERIC-NEXT:    addq $96, %rax # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_lea_add_scale_offset:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    leaq 96(%rdi,%rsi,4), %rax # sched: [1:1.00]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_lea_add_scale_offset:
-; SLM:       # %bb.0:
-; SLM-NEXT:    leaq 96(%rdi,%rsi,4), %rax # sched: [1:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_lea_add_scale_offset:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    leaq (%rdi,%rsi,4), %rax # sched: [1:0.50]
-; SANDY-NEXT:    addq $96, %rax # sched: [1:0.33]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_lea_add_scale_offset:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    leaq (%rdi,%rsi,4), %rax # sched: [1:0.50]
-; HASWELL-NEXT:    addq $96, %rax # sched: [1:0.25]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_lea_add_scale_offset:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    leaq (%rdi,%rsi,4), %rax # sched: [1:0.50]
-; BROADWELL-NEXT:    addq $96, %rax # sched: [1:0.25]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_lea_add_scale_offset:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    leaq (%rdi,%rsi,4), %rax # sched: [1:0.50]
-; SKYLAKE-NEXT:    addq $96, %rax # sched: [1:0.25]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_lea_add_scale_offset:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    leaq 96(%rdi,%rsi,4), %rax # sched: [1:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_lea_add_scale_offset:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    leaq 96(%rdi,%rsi,4), %rax # sched: [2:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_lea_add_scale_offset:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    leaq 96(%rdi,%rsi,4), %rax # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %3 = shl i64 %1, 2
-  %4 = add i64 %0, 96
-  %5 = add i64 %4, %3
-  ret i64 %5
-}
-
-define i64 @test_lea_add_scale_offset_big(i64, i64) {
-; GENERIC-LABEL: test_lea_add_scale_offset_big:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    leaq (%rdi,%rsi,8), %rax # sched: [1:0.50]
-; GENERIC-NEXT:    addq $-1200, %rax # imm = 0xFB50
-; GENERIC-NEXT:    # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_lea_add_scale_offset_big:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    leaq -1200(%rdi,%rsi,8), %rax # sched: [1:1.00]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_lea_add_scale_offset_big:
-; SLM:       # %bb.0:
-; SLM-NEXT:    leaq -1200(%rdi,%rsi,8), %rax # sched: [1:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_lea_add_scale_offset_big:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    leaq (%rdi,%rsi,8), %rax # sched: [1:0.50]
-; SANDY-NEXT:    addq $-1200, %rax # imm = 0xFB50
-; SANDY-NEXT:    # sched: [1:0.33]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_lea_add_scale_offset_big:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    leaq (%rdi,%rsi,8), %rax # sched: [1:0.50]
-; HASWELL-NEXT:    addq $-1200, %rax # imm = 0xFB50
-; HASWELL-NEXT:    # sched: [1:0.25]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_lea_add_scale_offset_big:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    leaq (%rdi,%rsi,8), %rax # sched: [1:0.50]
-; BROADWELL-NEXT:    addq $-1200, %rax # imm = 0xFB50
-; BROADWELL-NEXT:    # sched: [1:0.25]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_lea_add_scale_offset_big:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    leaq (%rdi,%rsi,8), %rax # sched: [1:0.50]
-; SKYLAKE-NEXT:    addq $-1200, %rax # imm = 0xFB50
-; SKYLAKE-NEXT:    # sched: [1:0.25]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_lea_add_scale_offset_big:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    leaq -1200(%rdi,%rsi,8), %rax # sched: [1:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_lea_add_scale_offset_big:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    leaq -1200(%rdi,%rsi,8), %rax # sched: [2:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_lea_add_scale_offset_big:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    leaq -1200(%rdi,%rsi,8), %rax # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %3 = shl i64 %1, 3
-  %4 = add i64 %0, -1200
-  %5 = add i64 %4, %3
-  ret i64 %5
-}
diff --git a/test/CodeGen/X86/load-combine.ll b/test/CodeGen/X86/load-combine.ll
index 3934bf5c1ce83866ab76ab7357f524d14769374b..8c69dba8c3b02496045073a2ee05a9211d3465e8 100644
--- a/test/CodeGen/X86/load-combine.ll
+++ b/test/CodeGen/X86/load-combine.ll
@@ -915,7 +915,7 @@ define i32 @load_i32_by_i8_base_offset_index(i8* %arg, i32 %i) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; CHECK-NEXT:    movl 12(%ecx,%eax), %eax
+; CHECK-NEXT:    movl 12(%eax,%ecx), %eax
 ; CHECK-NEXT:    retl
 ;
 ; CHECK64-LABEL: load_i32_by_i8_base_offset_index:
@@ -960,7 +960,7 @@ define i32 @load_i32_by_i8_base_offset_index_2(i8* %arg, i32 %i) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; CHECK-NEXT:    movl 13(%ecx,%eax), %eax
+; CHECK-NEXT:    movl 13(%eax,%ecx), %eax
 ; CHECK-NEXT:    retl
 ;
 ; CHECK64-LABEL: load_i32_by_i8_base_offset_index_2:
@@ -1016,7 +1016,7 @@ define i32 @load_i32_by_i8_zaext_loads(i8* %arg, i32 %arg1) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; CHECK-NEXT:    movl 12(%ecx,%eax), %eax
+; CHECK-NEXT:    movl 12(%eax,%ecx), %eax
 ; CHECK-NEXT:    retl
 ;
 ; CHECK64-LABEL: load_i32_by_i8_zaext_loads:
@@ -1072,7 +1072,7 @@ define i32 @load_i32_by_i8_zsext_loads(i8* %arg, i32 %arg1) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; CHECK-NEXT:    movl 12(%ecx,%eax), %eax
+; CHECK-NEXT:    movl 12(%eax,%ecx), %eax
 ; CHECK-NEXT:    retl
 ;
 ; CHECK64-LABEL: load_i32_by_i8_zsext_loads:
diff --git a/test/CodeGen/X86/lwp-schedule.ll b/test/CodeGen/X86/lwp-schedule.ll
deleted file mode 100644
index c10282cfb8e9700caecee65bb99c96ffae1c3254..0000000000000000000000000000000000000000
--- a/test/CodeGen/X86/lwp-schedule.ll
+++ /dev/null
@@ -1,299 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-unknown -print-schedule -mcpu=x86-64 -mattr=+lwp | FileCheck %s --check-prefix=GENERIC
-; RUN: llc < %s -mtriple=x86_64-unknown -print-schedule -mcpu=bdver1 | FileCheck %s --check-prefix=BDVER --check-prefix=BDVER12 --check-prefix=BDVER1
-; RUN: llc < %s -mtriple=x86_64-unknown -print-schedule -mcpu=bdver2 | FileCheck %s --check-prefix=BDVER --check-prefix=BDVER12 --check-prefix=BDVER2
-; RUN: llc < %s -mtriple=x86_64-unknown -print-schedule -mcpu=bdver3 | FileCheck %s --check-prefix=BDVER --check-prefix=BDVER3
-; RUN: llc < %s -mtriple=x86_64-unknown -print-schedule -mcpu=bdver4 | FileCheck %s --check-prefix=BDVER --check-prefix=BDVER4
-
-define void @test_llwpcb(i8 *%a0) nounwind {
-; GENERIC-LABEL: test_llwpcb:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    llwpcb %rdi # sched: [100:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER12-LABEL: test_llwpcb:
-; BDVER12:       # %bb.0:
-; BDVER12-NEXT:    llwpcb %rdi # sched: [100:0.50]
-; BDVER12-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER3-LABEL: test_llwpcb:
-; BDVER3:       # %bb.0:
-; BDVER3-NEXT:    llwpcb %rdi
-; BDVER3-NEXT:    retq
-;
-; BDVER4-LABEL: test_llwpcb:
-; BDVER4:       # %bb.0:
-; BDVER4-NEXT:    llwpcb %rdi
-; BDVER4-NEXT:    retq
-  tail call void @llvm.x86.llwpcb(i8 *%a0)
-  ret void
-}
-
-define i8* @test_slwpcb(i8 *%a0) nounwind {
-; GENERIC-LABEL: test_slwpcb:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    slwpcb %rax # sched: [100:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER12-LABEL: test_slwpcb:
-; BDVER12:       # %bb.0:
-; BDVER12-NEXT:    slwpcb %rax # sched: [100:0.50]
-; BDVER12-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER3-LABEL: test_slwpcb:
-; BDVER3:       # %bb.0:
-; BDVER3-NEXT:    slwpcb %rax
-; BDVER3-NEXT:    retq
-;
-; BDVER4-LABEL: test_slwpcb:
-; BDVER4:       # %bb.0:
-; BDVER4-NEXT:    slwpcb %rax
-; BDVER4-NEXT:    retq
-  %1 = tail call i8* @llvm.x86.slwpcb()
-  ret i8 *%1
-}
-
-define i8 @test_lwpins32_rri(i32 %a0, i32 %a1) nounwind {
-; GENERIC-LABEL: test_lwpins32_rri:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    addl %esi, %esi # sched: [1:0.33]
-; GENERIC-NEXT:    lwpins $-1985229329, %esi, %edi # imm = 0x89ABCDEF
-; GENERIC-NEXT:    # sched: [100:0.33]
-; GENERIC-NEXT:    setb %al # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER12-LABEL: test_lwpins32_rri:
-; BDVER12:       # %bb.0:
-; BDVER12-NEXT:    addl %esi, %esi # sched: [1:0.50]
-; BDVER12-NEXT:    lwpins $-1985229329, %esi, %edi # imm = 0x89ABCDEF
-; BDVER12-NEXT:    # sched: [100:0.50]
-; BDVER12-NEXT:    setb %al # sched: [1:0.50]
-; BDVER12-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER3-LABEL: test_lwpins32_rri:
-; BDVER3:       # %bb.0:
-; BDVER3-NEXT:    addl %esi, %esi
-; BDVER3-NEXT:    lwpins $-1985229329, %esi, %edi # imm = 0x89ABCDEF
-; BDVER3-NEXT:    setb %al
-; BDVER3-NEXT:    retq
-;
-; BDVER4-LABEL: test_lwpins32_rri:
-; BDVER4:       # %bb.0:
-; BDVER4-NEXT:    addl %esi, %esi
-; BDVER4-NEXT:    lwpins $-1985229329, %esi, %edi # imm = 0x89ABCDEF
-; BDVER4-NEXT:    setb %al
-; BDVER4-NEXT:    retq
-  %1 = add i32 %a1, %a1
-  %2 = tail call i8 @llvm.x86.lwpins32(i32 %a0, i32 %1, i32 2309737967)
-  ret i8 %2
-}
-
-define i8 @test_lwpins32_rmi(i32 %a0, i32 *%p1) nounwind {
-; GENERIC-LABEL: test_lwpins32_rmi:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    lwpins $1985229328, (%rsi), %edi # imm = 0x76543210
-; GENERIC-NEXT:    # sched: [100:0.33]
-; GENERIC-NEXT:    setb %al # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER12-LABEL: test_lwpins32_rmi:
-; BDVER12:       # %bb.0:
-; BDVER12-NEXT:    lwpins $1985229328, (%rsi), %edi # imm = 0x76543210
-; BDVER12-NEXT:    # sched: [100:0.50]
-; BDVER12-NEXT:    setb %al # sched: [1:0.50]
-; BDVER12-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER3-LABEL: test_lwpins32_rmi:
-; BDVER3:       # %bb.0:
-; BDVER3-NEXT:    lwpins $1985229328, (%rsi), %edi # imm = 0x76543210
-; BDVER3-NEXT:    setb %al
-; BDVER3-NEXT:    retq
-;
-; BDVER4-LABEL: test_lwpins32_rmi:
-; BDVER4:       # %bb.0:
-; BDVER4-NEXT:    lwpins $1985229328, (%rsi), %edi # imm = 0x76543210
-; BDVER4-NEXT:    setb %al
-; BDVER4-NEXT:    retq
-  %a1 = load i32, i32 *%p1
-  %1 = tail call i8 @llvm.x86.lwpins32(i32 %a0, i32 %a1, i32 1985229328)
-  ret i8 %1
-}
-
-define i8 @test_lwpins64_rri(i64 %a0, i32 %a1) nounwind {
-; GENERIC-LABEL: test_lwpins64_rri:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    lwpins $-1985229329, %esi, %rdi # imm = 0x89ABCDEF
-; GENERIC-NEXT:    # sched: [100:0.33]
-; GENERIC-NEXT:    setb %al # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER12-LABEL: test_lwpins64_rri:
-; BDVER12:       # %bb.0:
-; BDVER12-NEXT:    lwpins $-1985229329, %esi, %rdi # imm = 0x89ABCDEF
-; BDVER12-NEXT:    # sched: [100:0.50]
-; BDVER12-NEXT:    setb %al # sched: [1:0.50]
-; BDVER12-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER3-LABEL: test_lwpins64_rri:
-; BDVER3:       # %bb.0:
-; BDVER3-NEXT:    lwpins $-1985229329, %esi, %rdi # imm = 0x89ABCDEF
-; BDVER3-NEXT:    setb %al
-; BDVER3-NEXT:    retq
-;
-; BDVER4-LABEL: test_lwpins64_rri:
-; BDVER4:       # %bb.0:
-; BDVER4-NEXT:    lwpins $-1985229329, %esi, %rdi # imm = 0x89ABCDEF
-; BDVER4-NEXT:    setb %al
-; BDVER4-NEXT:    retq
-  %1 = tail call i8 @llvm.x86.lwpins64(i64 %a0, i32 %a1, i32 2309737967)
-  ret i8 %1
-}
-
-define i8 @test_lwpins64_rmi(i64 %a0, i32 *%p1) nounwind {
-; GENERIC-LABEL: test_lwpins64_rmi:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    lwpins $1985229328, (%rsi), %rdi # imm = 0x76543210
-; GENERIC-NEXT:    # sched: [100:0.33]
-; GENERIC-NEXT:    setb %al # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER12-LABEL: test_lwpins64_rmi:
-; BDVER12:       # %bb.0:
-; BDVER12-NEXT:    lwpins $1985229328, (%rsi), %rdi # imm = 0x76543210
-; BDVER12-NEXT:    # sched: [100:0.50]
-; BDVER12-NEXT:    setb %al # sched: [1:0.50]
-; BDVER12-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER3-LABEL: test_lwpins64_rmi:
-; BDVER3:       # %bb.0:
-; BDVER3-NEXT:    lwpins $1985229328, (%rsi), %rdi # imm = 0x76543210
-; BDVER3-NEXT:    setb %al
-; BDVER3-NEXT:    retq
-;
-; BDVER4-LABEL: test_lwpins64_rmi:
-; BDVER4:       # %bb.0:
-; BDVER4-NEXT:    lwpins $1985229328, (%rsi), %rdi # imm = 0x76543210
-; BDVER4-NEXT:    setb %al
-; BDVER4-NEXT:    retq
-  %a1 = load i32, i32 *%p1
-  %1 = tail call i8 @llvm.x86.lwpins64(i64 %a0, i32 %a1, i32 1985229328)
-  ret i8 %1
-}
-
-define void @test_lwpval32_rri(i32 %a0, i32 %a1) nounwind {
-; GENERIC-LABEL: test_lwpval32_rri:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    addl %esi, %esi # sched: [1:0.33]
-; GENERIC-NEXT:    lwpval $-19088744, %esi, %edi # imm = 0xFEDCBA98
-; GENERIC-NEXT:    # sched: [100:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER12-LABEL: test_lwpval32_rri:
-; BDVER12:       # %bb.0:
-; BDVER12-NEXT:    addl %esi, %esi # sched: [1:0.50]
-; BDVER12-NEXT:    lwpval $-19088744, %esi, %edi # imm = 0xFEDCBA98
-; BDVER12-NEXT:    # sched: [100:0.50]
-; BDVER12-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER3-LABEL: test_lwpval32_rri:
-; BDVER3:       # %bb.0:
-; BDVER3-NEXT:    addl %esi, %esi
-; BDVER3-NEXT:    lwpval $-19088744, %esi, %edi # imm = 0xFEDCBA98
-; BDVER3-NEXT:    retq
-;
-; BDVER4-LABEL: test_lwpval32_rri:
-; BDVER4:       # %bb.0:
-; BDVER4-NEXT:    addl %esi, %esi
-; BDVER4-NEXT:    lwpval $-19088744, %esi, %edi # imm = 0xFEDCBA98
-; BDVER4-NEXT:    retq
-  %1 = add i32 %a1, %a1
-  tail call void @llvm.x86.lwpval32(i32 %a0, i32 %1, i32 4275878552)
-  ret void
-}
-
-define void @test_lwpval32_rmi(i32 %a0, i32 *%p1) nounwind {
-; GENERIC-LABEL: test_lwpval32_rmi:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    lwpval $305419896, (%rsi), %edi # imm = 0x12345678
-; GENERIC-NEXT:    # sched: [100:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER12-LABEL: test_lwpval32_rmi:
-; BDVER12:       # %bb.0:
-; BDVER12-NEXT:    lwpval $305419896, (%rsi), %edi # imm = 0x12345678
-; BDVER12-NEXT:    # sched: [100:0.50]
-; BDVER12-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER3-LABEL: test_lwpval32_rmi:
-; BDVER3:       # %bb.0:
-; BDVER3-NEXT:    lwpval $305419896, (%rsi), %edi # imm = 0x12345678
-; BDVER3-NEXT:    retq
-;
-; BDVER4-LABEL: test_lwpval32_rmi:
-; BDVER4:       # %bb.0:
-; BDVER4-NEXT:    lwpval $305419896, (%rsi), %edi # imm = 0x12345678
-; BDVER4-NEXT:    retq
-  %a1 = load i32, i32 *%p1
-  tail call void @llvm.x86.lwpval32(i32 %a0, i32 %a1, i32 305419896)
-  ret void
-}
-
-define void @test_lwpval64_rri(i64 %a0, i32 %a1) nounwind {
-; GENERIC-LABEL: test_lwpval64_rri:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    lwpval $-19088744, %esi, %rdi # imm = 0xFEDCBA98
-; GENERIC-NEXT:    # sched: [100:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER12-LABEL: test_lwpval64_rri:
-; BDVER12:       # %bb.0:
-; BDVER12-NEXT:    lwpval $-19088744, %esi, %rdi # imm = 0xFEDCBA98
-; BDVER12-NEXT:    # sched: [100:0.50]
-; BDVER12-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER3-LABEL: test_lwpval64_rri:
-; BDVER3:       # %bb.0:
-; BDVER3-NEXT:    lwpval $-19088744, %esi, %rdi # imm = 0xFEDCBA98
-; BDVER3-NEXT:    retq
-;
-; BDVER4-LABEL: test_lwpval64_rri:
-; BDVER4:       # %bb.0:
-; BDVER4-NEXT:    lwpval $-19088744, %esi, %rdi # imm = 0xFEDCBA98
-; BDVER4-NEXT:    retq
-  tail call void @llvm.x86.lwpval64(i64 %a0, i32 %a1, i32 4275878552)
-  ret void
-}
-
-define void @test_lwpval64_rmi(i64 %a0, i32 *%p1) nounwind {
-; GENERIC-LABEL: test_lwpval64_rmi:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    lwpval $305419896, (%rsi), %rdi # imm = 0x12345678
-; GENERIC-NEXT:    # sched: [100:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER12-LABEL: test_lwpval64_rmi:
-; BDVER12:       # %bb.0:
-; BDVER12-NEXT:    lwpval $305419896, (%rsi), %rdi # imm = 0x12345678
-; BDVER12-NEXT:    # sched: [100:0.50]
-; BDVER12-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER3-LABEL: test_lwpval64_rmi:
-; BDVER3:       # %bb.0:
-; BDVER3-NEXT:    lwpval $305419896, (%rsi), %rdi # imm = 0x12345678
-; BDVER3-NEXT:    retq
-;
-; BDVER4-LABEL: test_lwpval64_rmi:
-; BDVER4:       # %bb.0:
-; BDVER4-NEXT:    lwpval $305419896, (%rsi), %rdi # imm = 0x12345678
-; BDVER4-NEXT:    retq
-  %a1 = load i32, i32 *%p1
-  tail call void @llvm.x86.lwpval64(i64 %a0, i32 %a1, i32 305419896)
-  ret void
-}
-
-declare void @llvm.x86.llwpcb(i8*) nounwind
-declare i8* @llvm.x86.slwpcb() nounwind
-declare i8 @llvm.x86.lwpins32(i32, i32, i32) nounwind
-declare i8 @llvm.x86.lwpins64(i64, i32, i32) nounwind
-declare void @llvm.x86.lwpval32(i32, i32, i32) nounwind
-declare void @llvm.x86.lwpval64(i64, i32, i32) nounwind
diff --git a/test/CodeGen/X86/lzcnt-schedule.ll b/test/CodeGen/X86/lzcnt-schedule.ll
deleted file mode 100644
index d8f9416b92ba5a98911a7c4f54b12b00596c2d95..0000000000000000000000000000000000000000
--- a/test/CodeGen/X86/lzcnt-schedule.ll
+++ /dev/null
@@ -1,187 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=x86-64 -mattr=+lzcnt | FileCheck %s --check-prefix=CHECK --check-prefix=GENERIC
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=haswell   | FileCheck %s --check-prefix=CHECK --check-prefix=HASWELL
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=broadwell | FileCheck %s --check-prefix=CHECK --check-prefix=BROADWELL
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=skylake   | FileCheck %s --check-prefix=CHECK --check-prefix=SKYLAKE
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=knl       | FileCheck %s --check-prefix=CHECK --check-prefix=HASWELL
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=bdver2    | FileCheck %s --check-prefix=CHECK --check-prefix=BDVER2
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=btver2    | FileCheck %s --check-prefix=CHECK --check-prefix=BTVER2
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=znver1    | FileCheck %s --check-prefix=CHECK --check-prefix=ZNVER1
-
-define i16 @test_ctlz_i16(i16 zeroext %a0, i16 *%a1) {
-; GENERIC-LABEL: test_ctlz_i16:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    lzcntw (%rsi), %cx # sched: [8:1.00]
-; GENERIC-NEXT:    lzcntw %di, %ax # sched: [3:1.00]
-; GENERIC-NEXT:    orl %ecx, %eax # sched: [1:0.33]
-; GENERIC-NEXT:    # kill: def $ax killed $ax killed $eax
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_ctlz_i16:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    lzcntw (%rsi), %cx # sched: [8:1.00]
-; HASWELL-NEXT:    lzcntw %di, %ax # sched: [3:1.00]
-; HASWELL-NEXT:    orl %ecx, %eax # sched: [1:0.25]
-; HASWELL-NEXT:    # kill: def $ax killed $ax killed $eax
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_ctlz_i16:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    lzcntw (%rsi), %cx # sched: [8:1.00]
-; BROADWELL-NEXT:    lzcntw %di, %ax # sched: [3:1.00]
-; BROADWELL-NEXT:    orl %ecx, %eax # sched: [1:0.25]
-; BROADWELL-NEXT:    # kill: def $ax killed $ax killed $eax
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_ctlz_i16:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    lzcntw (%rsi), %cx # sched: [8:1.00]
-; SKYLAKE-NEXT:    lzcntw %di, %ax # sched: [3:1.00]
-; SKYLAKE-NEXT:    orl %ecx, %eax # sched: [1:0.25]
-; SKYLAKE-NEXT:    # kill: def $ax killed $ax killed $eax
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_ctlz_i16:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    lzcntw (%rsi), %cx # sched: [6:0.50]
-; BDVER2-NEXT:    lzcntw %di, %ax # sched: [2:0.50]
-; BDVER2-NEXT:    orl %ecx, %eax # sched: [1:0.50]
-; BDVER2-NEXT:    # kill: def $ax killed $ax killed $eax
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_ctlz_i16:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    lzcntw (%rsi), %cx # sched: [4:1.00]
-; BTVER2-NEXT:    lzcntw %di, %ax # sched: [1:0.50]
-; BTVER2-NEXT:    orl %ecx, %eax # sched: [1:0.50]
-; BTVER2-NEXT:    # kill: def $ax killed $ax killed $eax
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_ctlz_i16:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    lzcntw (%rsi), %cx # sched: [6:0.50]
-; ZNVER1-NEXT:    lzcntw %di, %ax # sched: [2:0.25]
-; ZNVER1-NEXT:    orl %ecx, %eax # sched: [1:0.25]
-; ZNVER1-NEXT:    # kill: def $ax killed $ax killed $eax
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = load i16, i16 *%a1
-  %2 = tail call i16 @llvm.ctlz.i16( i16 %1, i1 false )
-  %3 = tail call i16 @llvm.ctlz.i16( i16 %a0, i1 false )
-  %4 = or i16 %2, %3
-  ret i16 %4
-}
-declare i16 @llvm.ctlz.i16(i16, i1)
-
-define i32 @test_ctlz_i32(i32 %a0, i32 *%a1) {
-; GENERIC-LABEL: test_ctlz_i32:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    lzcntl (%rsi), %ecx # sched: [8:1.00]
-; GENERIC-NEXT:    lzcntl %edi, %eax # sched: [3:1.00]
-; GENERIC-NEXT:    orl %ecx, %eax # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_ctlz_i32:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    lzcntl (%rsi), %ecx # sched: [8:1.00]
-; HASWELL-NEXT:    lzcntl %edi, %eax # sched: [3:1.00]
-; HASWELL-NEXT:    orl %ecx, %eax # sched: [1:0.25]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_ctlz_i32:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    lzcntl (%rsi), %ecx # sched: [8:1.00]
-; BROADWELL-NEXT:    lzcntl %edi, %eax # sched: [3:1.00]
-; BROADWELL-NEXT:    orl %ecx, %eax # sched: [1:0.25]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_ctlz_i32:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    lzcntl (%rsi), %ecx # sched: [8:1.00]
-; SKYLAKE-NEXT:    lzcntl %edi, %eax # sched: [3:1.00]
-; SKYLAKE-NEXT:    orl %ecx, %eax # sched: [1:0.25]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_ctlz_i32:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    lzcntl (%rsi), %ecx # sched: [6:0.50]
-; BDVER2-NEXT:    lzcntl %edi, %eax # sched: [2:0.50]
-; BDVER2-NEXT:    orl %ecx, %eax # sched: [1:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_ctlz_i32:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    lzcntl (%rsi), %ecx # sched: [4:1.00]
-; BTVER2-NEXT:    lzcntl %edi, %eax # sched: [1:0.50]
-; BTVER2-NEXT:    orl %ecx, %eax # sched: [1:0.50]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_ctlz_i32:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    lzcntl (%rsi), %ecx # sched: [6:0.50]
-; ZNVER1-NEXT:    lzcntl %edi, %eax # sched: [2:0.25]
-; ZNVER1-NEXT:    orl %ecx, %eax # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = load i32, i32 *%a1
-  %2 = tail call i32 @llvm.ctlz.i32( i32 %1, i1 false )
-  %3 = tail call i32 @llvm.ctlz.i32( i32 %a0, i1 false )
-  %4 = or i32 %2, %3
-  ret i32 %4
-}
-declare i32 @llvm.ctlz.i32(i32, i1)
-
-define i64 @test_ctlz_i64(i64 %a0, i64 *%a1) {
-; GENERIC-LABEL: test_ctlz_i64:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    lzcntq (%rsi), %rcx # sched: [8:1.00]
-; GENERIC-NEXT:    lzcntq %rdi, %rax # sched: [3:1.00]
-; GENERIC-NEXT:    orq %rcx, %rax # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_ctlz_i64:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    lzcntq (%rsi), %rcx # sched: [8:1.00]
-; HASWELL-NEXT:    lzcntq %rdi, %rax # sched: [3:1.00]
-; HASWELL-NEXT:    orq %rcx, %rax # sched: [1:0.25]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_ctlz_i64:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    lzcntq (%rsi), %rcx # sched: [8:1.00]
-; BROADWELL-NEXT:    lzcntq %rdi, %rax # sched: [3:1.00]
-; BROADWELL-NEXT:    orq %rcx, %rax # sched: [1:0.25]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_ctlz_i64:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    lzcntq (%rsi), %rcx # sched: [8:1.00]
-; SKYLAKE-NEXT:    lzcntq %rdi, %rax # sched: [3:1.00]
-; SKYLAKE-NEXT:    orq %rcx, %rax # sched: [1:0.25]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_ctlz_i64:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    lzcntq (%rsi), %rcx # sched: [6:0.50]
-; BDVER2-NEXT:    lzcntq %rdi, %rax # sched: [2:0.50]
-; BDVER2-NEXT:    orq %rcx, %rax # sched: [1:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_ctlz_i64:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    lzcntq (%rsi), %rcx # sched: [4:1.00]
-; BTVER2-NEXT:    lzcntq %rdi, %rax # sched: [1:0.50]
-; BTVER2-NEXT:    orq %rcx, %rax # sched: [1:0.50]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_ctlz_i64:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    lzcntq (%rsi), %rcx # sched: [6:0.50]
-; ZNVER1-NEXT:    lzcntq %rdi, %rax # sched: [2:0.25]
-; ZNVER1-NEXT:    orq %rcx, %rax # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = load i64, i64 *%a1
-  %2 = tail call i64 @llvm.ctlz.i64( i64 %1, i1 false )
-  %3 = tail call i64 @llvm.ctlz.i64( i64 %a0, i1 false )
-  %4 = or i64 %2, %3
-  ret i64 %4
-}
-declare i64 @llvm.ctlz.i64(i64, i1)
diff --git a/test/CodeGen/X86/madd.ll b/test/CodeGen/X86/madd.ll
index 78c6dbe4af0d00bdae85cab7662c99cce8015d58..517492b848deb3ae9bb05050f3f79234dc25f079 100644
--- a/test/CodeGen/X86/madd.ll
+++ b/test/CodeGen/X86/madd.ll
@@ -152,9 +152,9 @@ define i32 @_Z10test_shortPsS_i_256(i16* nocapture readonly, i16* nocapture read
 ; AVX256-NEXT:    jne .LBB1_1
 ; AVX256-NEXT:  # %bb.2: # %middle.block
 ; AVX256-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX256-NEXT:    vpaddd %ymm1, %ymm0, %ymm0
+; AVX256-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX256-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX256-NEXT:    vpaddd %ymm1, %ymm0, %ymm0
+; AVX256-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX256-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX256-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX256-NEXT:    vmovd %xmm0, %eax
@@ -277,9 +277,9 @@ define i32 @_Z10test_shortPsS_i_512(i16* nocapture readonly, i16* nocapture read
 ; AVX2-NEXT:  # %bb.2: # %middle.block
 ; AVX2-NEXT:    vpaddd %ymm0, %ymm1, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpaddd %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX2-NEXT:    vpaddd %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
@@ -304,9 +304,9 @@ define i32 @_Z10test_shortPsS_i_512(i16* nocapture readonly, i16* nocapture read
 ; AVX512-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
@@ -465,9 +465,9 @@ define i32 @_Z10test_shortPsS_i_1024(i16* nocapture readonly, i16* nocapture rea
 ; AVX2-NEXT:    vpaddd %ymm0, %ymm2, %ymm0
 ; AVX2-NEXT:    vpaddd %ymm0, %ymm1, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpaddd %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX2-NEXT:    vpaddd %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
@@ -497,9 +497,9 @@ define i32 @_Z10test_shortPsS_i_1024(i16* nocapture readonly, i16* nocapture rea
 ; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512F-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
 ; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512F-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
+; AVX512F-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX512F-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512F-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
+; AVX512F-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX512F-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512F-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX512F-NEXT:    vmovd %xmm0, %eax
@@ -526,9 +526,9 @@ define i32 @_Z10test_shortPsS_i_1024(i16* nocapture readonly, i16* nocapture rea
 ; AVX512BW-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512BW-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512BW-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512BW-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512BW-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX512BW-NEXT:    vmovd %xmm0, %eax
@@ -581,16 +581,14 @@ define i32 @_Z9test_charPcS_i_128(i8* nocapture readonly, i8* nocapture readonly
 ; SSE2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; SSE2-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3]
-; SSE2-NEXT:    psrad $24, %xmm1
 ; SSE2-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3]
-; SSE2-NEXT:    psrad $24, %xmm2
+; SSE2-NEXT:    psraw $8, %xmm1
+; SSE2-NEXT:    psraw $8, %xmm2
 ; SSE2-NEXT:    pmullw %xmm1, %xmm2
-; SSE2-NEXT:    pslld $16, %xmm2
-; SSE2-NEXT:    psrad $16, %xmm2
-; SSE2-NEXT:    paddd %xmm2, %xmm0
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
+; SSE2-NEXT:    psrad $16, %xmm1
+; SSE2-NEXT:    paddd %xmm1, %xmm0
 ; SSE2-NEXT:    addq $16, %rcx
 ; SSE2-NEXT:    cmpq %rcx, %rax
 ; SSE2-NEXT:    jne .LBB4_1
@@ -666,9 +664,9 @@ define i32 @_Z9test_charPcS_i_256(i8* nocapture readonly, i8* nocapture readonly
 ; SSE2-NEXT:    # =>This Inner Loop Header: Depth=1
 ; SSE2-NEXT:    movq {{.*#+}} xmm2 = mem[0],zero
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    psraw $8, %xmm2
 ; SSE2-NEXT:    movq {{.*#+}} xmm3 = mem[0],zero
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    psraw $8, %xmm2
 ; SSE2-NEXT:    psraw $8, %xmm3
 ; SSE2-NEXT:    pmaddwd %xmm2, %xmm3
 ; SSE2-NEXT:    paddd %xmm3, %xmm1
@@ -728,9 +726,9 @@ define i32 @_Z9test_charPcS_i_256(i8* nocapture readonly, i8* nocapture readonly
 ; AVX256-NEXT:    jne .LBB5_1
 ; AVX256-NEXT:  # %bb.2: # %middle.block
 ; AVX256-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX256-NEXT:    vpaddd %ymm1, %ymm0, %ymm0
+; AVX256-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX256-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX256-NEXT:    vpaddd %ymm1, %ymm0, %ymm0
+; AVX256-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX256-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX256-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX256-NEXT:    vmovd %xmm0, %eax
@@ -779,22 +777,20 @@ define i32 @_Z9test_charPcS_i_512(i8* nocapture readonly, i8* nocapture readonly
 ; SSE2-NEXT:    .p2align 4, 0x90
 ; SSE2-NEXT:  .LBB6_1: # %vector.body
 ; SSE2-NEXT:    # =>This Inner Loop Header: Depth=1
-; SSE2-NEXT:    movq {{.*#+}} xmm3 = mem[0],zero
+; SSE2-NEXT:    movdqu (%rdi,%rcx), %xmm3
+; SSE2-NEXT:    movdqu (%rsi,%rcx), %xmm4
+; SSE2-NEXT:    punpckhbw {{.*#+}} xmm5 = xmm5[8],xmm3[8],xmm5[9],xmm3[9],xmm5[10],xmm3[10],xmm5[11],xmm3[11],xmm5[12],xmm3[12],xmm5[13],xmm3[13],xmm5[14],xmm3[14],xmm5[15],xmm3[15]
+; SSE2-NEXT:    psraw $8, %xmm5
+; SSE2-NEXT:    punpckhbw {{.*#+}} xmm6 = xmm6[8],xmm4[8],xmm6[9],xmm4[9],xmm6[10],xmm4[10],xmm6[11],xmm4[11],xmm6[12],xmm4[12],xmm6[13],xmm4[13],xmm6[14],xmm4[14],xmm6[15],xmm4[15]
+; SSE2-NEXT:    psraw $8, %xmm6
+; SSE2-NEXT:    pmaddwd %xmm5, %xmm6
+; SSE2-NEXT:    paddd %xmm6, %xmm1
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
 ; SSE2-NEXT:    psraw $8, %xmm3
-; SSE2-NEXT:    movq {{.*#+}} xmm4 = mem[0],zero
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
 ; SSE2-NEXT:    psraw $8, %xmm4
-; SSE2-NEXT:    movq {{.*#+}} xmm5 = mem[0],zero
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    psraw $8, %xmm5
-; SSE2-NEXT:    pmaddwd %xmm3, %xmm5
-; SSE2-NEXT:    paddd %xmm5, %xmm2
-; SSE2-NEXT:    movq {{.*#+}} xmm3 = mem[0],zero
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    psraw $8, %xmm3
-; SSE2-NEXT:    pmaddwd %xmm4, %xmm3
-; SSE2-NEXT:    paddd %xmm3, %xmm1
+; SSE2-NEXT:    pmaddwd %xmm3, %xmm4
+; SSE2-NEXT:    paddd %xmm4, %xmm2
 ; SSE2-NEXT:    addq $16, %rcx
 ; SSE2-NEXT:    cmpq %rcx, %rax
 ; SSE2-NEXT:    jne .LBB6_1
@@ -864,9 +860,9 @@ define i32 @_Z9test_charPcS_i_512(i8* nocapture readonly, i8* nocapture readonly
 ; AVX2-NEXT:  # %bb.2: # %middle.block
 ; AVX2-NEXT:    vpaddd %ymm0, %ymm1, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpaddd %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX2-NEXT:    vpaddd %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
@@ -892,9 +888,9 @@ define i32 @_Z9test_charPcS_i_512(i8* nocapture readonly, i8* nocapture readonly
 ; AVX512-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
@@ -940,45 +936,41 @@ define i32 @_Z9test_charPcS_i_1024(i8* nocapture readonly, i8* nocapture readonl
 ; SSE2-NEXT:    movl %edx, %eax
 ; SSE2-NEXT:    pxor %xmm8, %xmm8
 ; SSE2-NEXT:    xorl %ecx, %ecx
-; SSE2-NEXT:    pxor %xmm9, %xmm9
+; SSE2-NEXT:    pxor %xmm2, %xmm2
 ; SSE2-NEXT:    pxor %xmm4, %xmm4
 ; SSE2-NEXT:    pxor %xmm1, %xmm1
 ; SSE2-NEXT:    pxor %xmm3, %xmm3
 ; SSE2-NEXT:    .p2align 4, 0x90
 ; SSE2-NEXT:  .LBB7_1: # %vector.body
 ; SSE2-NEXT:    # =>This Inner Loop Header: Depth=1
-; SSE2-NEXT:    movq {{.*#+}} xmm5 = mem[0],zero
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    movdqu (%rdi,%rcx), %xmm10
+; SSE2-NEXT:    movdqu 16(%rdi,%rcx), %xmm7
+; SSE2-NEXT:    movdqu (%rsi,%rcx), %xmm9
+; SSE2-NEXT:    movdqu 16(%rsi,%rcx), %xmm0
+; SSE2-NEXT:    punpckhbw {{.*#+}} xmm5 = xmm5[8],xmm7[8],xmm5[9],xmm7[9],xmm5[10],xmm7[10],xmm5[11],xmm7[11],xmm5[12],xmm7[12],xmm5[13],xmm7[13],xmm5[14],xmm7[14],xmm5[15],xmm7[15]
 ; SSE2-NEXT:    psraw $8, %xmm5
-; SSE2-NEXT:    movq {{.*#+}} xmm6 = mem[0],zero
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm6 = xmm6[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    punpckhbw {{.*#+}} xmm6 = xmm6[8],xmm0[8],xmm6[9],xmm0[9],xmm6[10],xmm0[10],xmm6[11],xmm0[11],xmm6[12],xmm0[12],xmm6[13],xmm0[13],xmm6[14],xmm0[14],xmm6[15],xmm0[15]
 ; SSE2-NEXT:    psraw $8, %xmm6
-; SSE2-NEXT:    movq {{.*#+}} xmm7 = mem[0],zero
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm7 = xmm7[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    psraw $8, %xmm7
-; SSE2-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
+; SSE2-NEXT:    pmaddwd %xmm5, %xmm6
+; SSE2-NEXT:    paddd %xmm6, %xmm3
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm7[0],xmm5[1],xmm7[1],xmm5[2],xmm7[2],xmm5[3],xmm7[3],xmm5[4],xmm7[4],xmm5[5],xmm7[5],xmm5[6],xmm7[6],xmm5[7],xmm7[7]
+; SSE2-NEXT:    psraw $8, %xmm5
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
 ; SSE2-NEXT:    psraw $8, %xmm0
-; SSE2-NEXT:    movq {{.*#+}} xmm2 = mem[0],zero
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    psraw $8, %xmm2
-; SSE2-NEXT:    pmaddwd %xmm5, %xmm2
-; SSE2-NEXT:    paddd %xmm2, %xmm9
-; SSE2-NEXT:    movq {{.*#+}} xmm2 = mem[0],zero
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    psraw $8, %xmm2
-; SSE2-NEXT:    pmaddwd %xmm6, %xmm2
-; SSE2-NEXT:    paddd %xmm2, %xmm4
-; SSE2-NEXT:    movq {{.*#+}} xmm2 = mem[0],zero
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    psraw $8, %xmm2
-; SSE2-NEXT:    pmaddwd %xmm7, %xmm2
-; SSE2-NEXT:    paddd %xmm2, %xmm1
-; SSE2-NEXT:    movq {{.*#+}} xmm2 = mem[0],zero
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    psraw $8, %xmm2
-; SSE2-NEXT:    pmaddwd %xmm0, %xmm2
-; SSE2-NEXT:    paddd %xmm2, %xmm3
+; SSE2-NEXT:    pmaddwd %xmm5, %xmm0
+; SSE2-NEXT:    paddd %xmm0, %xmm1
+; SSE2-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm10[8],xmm0[9],xmm10[9],xmm0[10],xmm10[10],xmm0[11],xmm10[11],xmm0[12],xmm10[12],xmm0[13],xmm10[13],xmm0[14],xmm10[14],xmm0[15],xmm10[15]
+; SSE2-NEXT:    psraw $8, %xmm0
+; SSE2-NEXT:    punpckhbw {{.*#+}} xmm5 = xmm5[8],xmm9[8],xmm5[9],xmm9[9],xmm5[10],xmm9[10],xmm5[11],xmm9[11],xmm5[12],xmm9[12],xmm5[13],xmm9[13],xmm5[14],xmm9[14],xmm5[15],xmm9[15]
+; SSE2-NEXT:    psraw $8, %xmm5
+; SSE2-NEXT:    pmaddwd %xmm0, %xmm5
+; SSE2-NEXT:    paddd %xmm5, %xmm4
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm10[0],xmm0[1],xmm10[1],xmm0[2],xmm10[2],xmm0[3],xmm10[3],xmm0[4],xmm10[4],xmm0[5],xmm10[5],xmm0[6],xmm10[6],xmm0[7],xmm10[7]
+; SSE2-NEXT:    psraw $8, %xmm0
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm9[0],xmm5[1],xmm9[1],xmm5[2],xmm9[2],xmm5[3],xmm9[3],xmm5[4],xmm9[4],xmm5[5],xmm9[5],xmm5[6],xmm9[6],xmm5[7],xmm9[7]
+; SSE2-NEXT:    psraw $8, %xmm5
+; SSE2-NEXT:    pmaddwd %xmm0, %xmm5
+; SSE2-NEXT:    paddd %xmm5, %xmm2
 ; SSE2-NEXT:    addq $32, %rcx
 ; SSE2-NEXT:    cmpq %rcx, %rax
 ; SSE2-NEXT:    jne .LBB7_1
@@ -986,10 +978,10 @@ define i32 @_Z9test_charPcS_i_1024(i8* nocapture readonly, i8* nocapture readonl
 ; SSE2-NEXT:    paddd %xmm8, %xmm4
 ; SSE2-NEXT:    paddd %xmm8, %xmm3
 ; SSE2-NEXT:    paddd %xmm4, %xmm3
-; SSE2-NEXT:    paddd %xmm8, %xmm9
+; SSE2-NEXT:    paddd %xmm8, %xmm2
 ; SSE2-NEXT:    paddd %xmm8, %xmm1
 ; SSE2-NEXT:    paddd %xmm3, %xmm1
-; SSE2-NEXT:    paddd %xmm9, %xmm1
+; SSE2-NEXT:    paddd %xmm2, %xmm1
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
 ; SSE2-NEXT:    paddd %xmm1, %xmm0
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
@@ -1075,9 +1067,9 @@ define i32 @_Z9test_charPcS_i_1024(i8* nocapture readonly, i8* nocapture readonl
 ; AVX2-NEXT:    vpaddd %ymm0, %ymm2, %ymm0
 ; AVX2-NEXT:    vpaddd %ymm0, %ymm1, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpaddd %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX2-NEXT:    vpaddd %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
@@ -1109,9 +1101,9 @@ define i32 @_Z9test_charPcS_i_1024(i8* nocapture readonly, i8* nocapture readonl
 ; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512F-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
 ; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512F-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
+; AVX512F-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX512F-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512F-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
+; AVX512F-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX512F-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512F-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX512F-NEXT:    vmovd %xmm0, %eax
@@ -1139,9 +1131,9 @@ define i32 @_Z9test_charPcS_i_1024(i8* nocapture readonly, i8* nocapture readonl
 ; AVX512BW-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512BW-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512BW-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512BW-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512BW-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX512BW-NEXT:    vmovd %xmm0, %eax
@@ -1343,9 +1335,9 @@ define i32 @test_unsigned_short_256(i16* nocapture readonly, i16* nocapture read
 ; AVX256-NEXT:    jne .LBB9_1
 ; AVX256-NEXT:  # %bb.2: # %middle.block
 ; AVX256-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX256-NEXT:    vpaddd %ymm1, %ymm0, %ymm0
+; AVX256-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX256-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX256-NEXT:    vpaddd %ymm1, %ymm0, %ymm0
+; AVX256-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX256-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX256-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX256-NEXT:    vmovd %xmm0, %eax
@@ -1498,9 +1490,9 @@ define i32 @test_unsigned_short_512(i16* nocapture readonly, i16* nocapture read
 ; AVX2-NEXT:  # %bb.2: # %middle.block
 ; AVX2-NEXT:    vpaddd %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpaddd %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX2-NEXT:    vpaddd %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
@@ -1526,9 +1518,9 @@ define i32 @test_unsigned_short_512(i16* nocapture readonly, i16* nocapture read
 ; AVX512-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
@@ -1751,9 +1743,9 @@ define i32 @test_unsigned_short_1024(i16* nocapture readonly, i16* nocapture rea
 ; AVX2-NEXT:    vpaddd %ymm3, %ymm1, %ymm1
 ; AVX2-NEXT:    vpaddd %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpaddd %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX2-NEXT:    vpaddd %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
@@ -1785,9 +1777,9 @@ define i32 @test_unsigned_short_1024(i16* nocapture readonly, i16* nocapture rea
 ; AVX512-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
@@ -2703,9 +2695,9 @@ define i32 @madd_double_reduction(<8 x i16>* %arg, <8 x i16>* %arg1, <8 x i16>*
 ; AVX256-NEXT:    vpmaddwd (%rcx), %xmm1, %xmm1
 ; AVX256-NEXT:    vpaddd %ymm0, %ymm1, %ymm0
 ; AVX256-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX256-NEXT:    vpaddd %ymm1, %ymm0, %ymm0
+; AVX256-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX256-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX256-NEXT:    vpaddd %ymm1, %ymm0, %ymm0
+; AVX256-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX256-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX256-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX256-NEXT:    vmovd %xmm0, %eax
diff --git a/test/CodeGen/X86/masked_gather_scatter.ll b/test/CodeGen/X86/masked_gather_scatter.ll
index 82120537f913d530c2f3ba832d63c8a81f467978..5e6d88ad6f52ecc6633aae0087664ac86f9cdacf 100644
--- a/test/CodeGen/X86/masked_gather_scatter.ll
+++ b/test/CodeGen/X86/masked_gather_scatter.ll
@@ -1743,7 +1743,7 @@ define <3 x i32> @test30(<3 x i32*> %base, <3 x i32> %ind, <3 x i1> %mask, <3 x
 ; SKX-NEXT:    vmovq %xmm0, %rax
 ; SKX-NEXT:    vpinsrd $0, (%rax), %xmm3, %xmm3
 ; SKX-NEXT:  .LBB31_2: # %else
-; SKX-NEXT:    kshiftrw $1, %k0, %k1
+; SKX-NEXT:    kshiftrb $1, %k0, %k1
 ; SKX-NEXT:    kmovw %k1, %eax
 ; SKX-NEXT:    testb $1, %al
 ; SKX-NEXT:    je .LBB31_4
@@ -1751,7 +1751,7 @@ define <3 x i32> @test30(<3 x i32*> %base, <3 x i32> %ind, <3 x i1> %mask, <3 x
 ; SKX-NEXT:    vpextrq $1, %xmm0, %rax
 ; SKX-NEXT:    vpinsrd $1, (%rax), %xmm3, %xmm3
 ; SKX-NEXT:  .LBB31_4: # %else2
-; SKX-NEXT:    kshiftrw $2, %k0, %k0
+; SKX-NEXT:    kshiftrb $2, %k0, %k0
 ; SKX-NEXT:    kmovw %k0, %eax
 ; SKX-NEXT:    testb $1, %al
 ; SKX-NEXT:    je .LBB31_6
@@ -1781,7 +1781,7 @@ define <3 x i32> @test30(<3 x i32*> %base, <3 x i32> %ind, <3 x i1> %mask, <3 x
 ; SKX_32-NEXT:    vmovd %xmm1, %eax
 ; SKX_32-NEXT:    vpinsrd $0, (%eax), %xmm0, %xmm0
 ; SKX_32-NEXT:  .LBB31_2: # %else
-; SKX_32-NEXT:    kshiftrw $1, %k0, %k1
+; SKX_32-NEXT:    kshiftrb $1, %k0, %k1
 ; SKX_32-NEXT:    kmovw %k1, %eax
 ; SKX_32-NEXT:    testb $1, %al
 ; SKX_32-NEXT:    je .LBB31_4
@@ -1789,7 +1789,7 @@ define <3 x i32> @test30(<3 x i32*> %base, <3 x i32> %ind, <3 x i1> %mask, <3 x
 ; SKX_32-NEXT:    vpextrd $1, %xmm1, %eax
 ; SKX_32-NEXT:    vpinsrd $1, (%eax), %xmm0, %xmm0
 ; SKX_32-NEXT:  .LBB31_4: # %else2
-; SKX_32-NEXT:    kshiftrw $2, %k0, %k0
+; SKX_32-NEXT:    kshiftrb $2, %k0, %k0
 ; SKX_32-NEXT:    kmovw %k0, %eax
 ; SKX_32-NEXT:    testb $1, %al
 ; SKX_32-NEXT:    je .LBB31_6
diff --git a/test/CodeGen/X86/masked_load.ll b/test/CodeGen/X86/masked_load.ll
index 5a456bd2d718ed7b205a21dab44deacb4d2f6c3a..8d28f45d98865d20f248e6b817269f87f4e92787 100644
--- a/test/CodeGen/X86/masked_load.ll
+++ b/test/CodeGen/X86/masked_load.ll
@@ -1388,7 +1388,7 @@ define <2 x i32> @load_v2i32_v2i32(<2 x i32> %trigger, <2 x i32>* %addr, <2 x i3
 ; AVX1-NEXT:    vmaskmovps (%rdi), %xmm0, %xmm2
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm1[0,2,2,3]
 ; AVX1-NEXT:    vblendvps %xmm0, %xmm2, %xmm1, %xmm0
-; AVX1-NEXT:    vpmovsxdq %xmm0, %xmm0
+; AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: load_v2i32_v2i32:
@@ -1400,7 +1400,7 @@ define <2 x i32> @load_v2i32_v2i32(<2 x i32> %trigger, <2 x i32>* %addr, <2 x i3
 ; AVX2-NEXT:    vpmaskmovd (%rdi), %xmm0, %xmm2
 ; AVX2-NEXT:    vpermilps {{.*#+}} xmm1 = xmm1[0,2,2,3]
 ; AVX2-NEXT:    vblendvps %xmm0, %xmm2, %xmm1, %xmm0
-; AVX2-NEXT:    vpmovsxdq %xmm0, %xmm0
+; AVX2-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
 ; AVX2-NEXT:    retq
 ;
 ; AVX512F-LABEL: load_v2i32_v2i32:
@@ -1412,7 +1412,7 @@ define <2 x i32> @load_v2i32_v2i32(<2 x i32> %trigger, <2 x i32>* %addr, <2 x i3
 ; AVX512F-NEXT:    kshiftlw $14, %k0, %k0
 ; AVX512F-NEXT:    kshiftrw $14, %k0, %k1
 ; AVX512F-NEXT:    vmovdqu32 (%rdi), %zmm0 {%k1}
-; AVX512F-NEXT:    vpmovsxdq %xmm0, %xmm0
+; AVX512F-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
@@ -1423,7 +1423,7 @@ define <2 x i32> @load_v2i32_v2i32(<2 x i32> %trigger, <2 x i32>* %addr, <2 x i3
 ; AVX512VLBW-NEXT:    vptestnmq %xmm0, %xmm0, %k1
 ; AVX512VLBW-NEXT:    vpshufd {{.*#+}} xmm0 = xmm1[0,2,2,3]
 ; AVX512VLBW-NEXT:    vmovdqu32 (%rdi), %xmm0 {%k1}
-; AVX512VLBW-NEXT:    vpmovsxdq %xmm0, %xmm0
+; AVX512VLBW-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
 ; AVX512VLBW-NEXT:    retq
   %mask = icmp eq <2 x i32> %trigger, zeroinitializer
   %res = call <2 x i32> @llvm.masked.load.v2i32.p0v2i32(<2 x i32>* %addr, i32 4, <2 x i1>%mask, <2 x i32>%dst)
diff --git a/test/CodeGen/X86/masked_store_trunc.ll b/test/CodeGen/X86/masked_store_trunc.ll
new file mode 100644
index 0000000000000000000000000000000000000000..bbbd941ffd7265cbc6076d5a7cd511eedc91c046
--- /dev/null
+++ b/test/CodeGen/X86/masked_store_trunc.ll
@@ -0,0 +1,1870 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx512f | FileCheck %s --check-prefixes=AVX512,AVX512F
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx512vl | FileCheck %s --check-prefixes=AVX512,AVX512VL
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx512bw | FileCheck %s --check-prefixes=AVX512,AVX512BW,AVX512BWNOVL
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx512vl,avx512bw | FileCheck %s --check-prefixes=AVX512,AVX512VL,AVX512BWVL
+
+define void @truncstore_v8i64_v8i32(<8 x i64> %x, <8 x i32>* %p, <8 x i32> %mask) {
+; AVX512F-LABEL: truncstore_v8i64_v8i32:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k1
+; AVX512F-NEXT:    vpmovqd %zmm0, (%rdi) {%k1}
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: truncstore_v8i64_v8i32:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vptestmd %ymm1, %ymm1, %k1
+; AVX512VL-NEXT:    vpmovqd %zmm0, (%rdi) {%k1}
+; AVX512VL-NEXT:    vzeroupper
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: truncstore_v8i64_v8i32:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512BW-NEXT:    vptestmd %zmm1, %zmm1, %k1
+; AVX512BW-NEXT:    vpmovqd %zmm0, (%rdi) {%k1}
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+  %a = icmp ne <8 x i32> %mask, zeroinitializer
+  %b = trunc <8 x i64> %x to <8 x i32>
+  call void @llvm.masked.store.v8i32.p0v8i32(<8 x i32> %b, <8 x i32>* %p, i32 1, <8 x i1> %a)
+  ret void
+}
+
+define void @truncstore_v8i64_v8i16(<8 x i64> %x, <8 x i16>* %p, <8 x i32> %mask) {
+; AVX512F-LABEL: truncstore_v8i64_v8i16:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    vpmovqw %zmm0, %xmm0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB1_2
+; AVX512F-NEXT:  # %bb.1: # %cond.store
+; AVX512F-NEXT:    vpextrw $0, %xmm0, (%rdi)
+; AVX512F-NEXT:  .LBB1_2: # %else
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $1, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB1_4
+; AVX512F-NEXT:  # %bb.3: # %cond.store1
+; AVX512F-NEXT:    vpextrw $1, %xmm0, 2(%rdi)
+; AVX512F-NEXT:  .LBB1_4: # %else2
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $2, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB1_6
+; AVX512F-NEXT:  # %bb.5: # %cond.store3
+; AVX512F-NEXT:    vpextrw $2, %xmm0, 4(%rdi)
+; AVX512F-NEXT:  .LBB1_6: # %else4
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $3, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB1_8
+; AVX512F-NEXT:  # %bb.7: # %cond.store5
+; AVX512F-NEXT:    vpextrw $3, %xmm0, 6(%rdi)
+; AVX512F-NEXT:  .LBB1_8: # %else6
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $4, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB1_10
+; AVX512F-NEXT:  # %bb.9: # %cond.store7
+; AVX512F-NEXT:    vpextrw $4, %xmm0, 8(%rdi)
+; AVX512F-NEXT:  .LBB1_10: # %else8
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $5, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB1_12
+; AVX512F-NEXT:  # %bb.11: # %cond.store9
+; AVX512F-NEXT:    vpextrw $5, %xmm0, 10(%rdi)
+; AVX512F-NEXT:  .LBB1_12: # %else10
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $6, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB1_14
+; AVX512F-NEXT:  # %bb.13: # %cond.store11
+; AVX512F-NEXT:    vpextrw $6, %xmm0, 12(%rdi)
+; AVX512F-NEXT:  .LBB1_14: # %else12
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $7, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB1_16
+; AVX512F-NEXT:  # %bb.15: # %cond.store13
+; AVX512F-NEXT:    vpextrw $7, %xmm0, 14(%rdi)
+; AVX512F-NEXT:  .LBB1_16: # %else14
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512BW-LABEL: truncstore_v8i64_v8i16:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512BW-NEXT:    vptestmd %zmm1, %zmm1, %k1
+; AVX512BW-NEXT:    vpmovqw %zmm0, (%rdi) {%k1}
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+;
+; AVX512BWVL-LABEL: truncstore_v8i64_v8i16:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vptestmd %ymm1, %ymm1, %k1
+; AVX512BWVL-NEXT:    vpmovqw %zmm0, (%rdi) {%k1}
+; AVX512BWVL-NEXT:    vzeroupper
+; AVX512BWVL-NEXT:    retq
+  %a = icmp ne <8 x i32> %mask, zeroinitializer
+  %b = trunc <8 x i64> %x to <8 x i16>
+  call void @llvm.masked.store.v8i16.p0v8i16(<8 x i16> %b, <8 x i16>* %p, i32 1, <8 x i1> %a)
+  ret void
+}
+
+define void @truncstore_v8i64_v8i8(<8 x i64> %x, <8 x i8>* %p, <8 x i32> %mask) {
+; AVX512F-LABEL: truncstore_v8i64_v8i8:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    vpmovqw %zmm0, %xmm0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB2_2
+; AVX512F-NEXT:  # %bb.1: # %cond.store
+; AVX512F-NEXT:    vpextrb $0, %xmm0, (%rdi)
+; AVX512F-NEXT:  .LBB2_2: # %else
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $1, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB2_4
+; AVX512F-NEXT:  # %bb.3: # %cond.store1
+; AVX512F-NEXT:    vpextrb $2, %xmm0, 1(%rdi)
+; AVX512F-NEXT:  .LBB2_4: # %else2
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $2, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB2_6
+; AVX512F-NEXT:  # %bb.5: # %cond.store3
+; AVX512F-NEXT:    vpextrb $4, %xmm0, 2(%rdi)
+; AVX512F-NEXT:  .LBB2_6: # %else4
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $3, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB2_8
+; AVX512F-NEXT:  # %bb.7: # %cond.store5
+; AVX512F-NEXT:    vpextrb $6, %xmm0, 3(%rdi)
+; AVX512F-NEXT:  .LBB2_8: # %else6
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $4, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB2_10
+; AVX512F-NEXT:  # %bb.9: # %cond.store7
+; AVX512F-NEXT:    vpextrb $8, %xmm0, 4(%rdi)
+; AVX512F-NEXT:  .LBB2_10: # %else8
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $5, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB2_12
+; AVX512F-NEXT:  # %bb.11: # %cond.store9
+; AVX512F-NEXT:    vpextrb $10, %xmm0, 5(%rdi)
+; AVX512F-NEXT:  .LBB2_12: # %else10
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $6, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB2_14
+; AVX512F-NEXT:  # %bb.13: # %cond.store11
+; AVX512F-NEXT:    vpextrb $12, %xmm0, 6(%rdi)
+; AVX512F-NEXT:  .LBB2_14: # %else12
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $7, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB2_16
+; AVX512F-NEXT:  # %bb.15: # %cond.store13
+; AVX512F-NEXT:    vpextrb $14, %xmm0, 7(%rdi)
+; AVX512F-NEXT:  .LBB2_16: # %else14
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512BW-LABEL: truncstore_v8i64_v8i8:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512BW-NEXT:    vptestmd %zmm1, %zmm1, %k1
+; AVX512BW-NEXT:    vpmovqb %zmm0, (%rdi) {%k1}
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+;
+; AVX512BWVL-LABEL: truncstore_v8i64_v8i8:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vptestmd %ymm1, %ymm1, %k1
+; AVX512BWVL-NEXT:    vpmovqb %zmm0, (%rdi) {%k1}
+; AVX512BWVL-NEXT:    vzeroupper
+; AVX512BWVL-NEXT:    retq
+  %a = icmp ne <8 x i32> %mask, zeroinitializer
+  %b = trunc <8 x i64> %x to <8 x i8>
+  call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> %b, <8 x i8>* %p, i32 1, <8 x i1> %a)
+  ret void
+}
+
+define void @truncstore_v4i64_v4i32(<4 x i64> %x, <4 x i32>* %p, <4 x i32> %mask) {
+; AVX512F-LABEL: truncstore_v4i64_v4i32:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512F-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftlw $12, %k0, %k0
+; AVX512F-NEXT:    kshiftrw $12, %k0, %k1
+; AVX512F-NEXT:    vpmovqd %zmm0, %ymm0
+; AVX512F-NEXT:    vmovdqu32 %zmm0, (%rdi) {%k1}
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: truncstore_v4i64_v4i32:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vptestmd %xmm1, %xmm1, %k1
+; AVX512VL-NEXT:    vpmovqd %ymm0, (%rdi) {%k1}
+; AVX512VL-NEXT:    vzeroupper
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: truncstore_v4i64_v4i32:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512BW-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512BW-NEXT:    kshiftlw $12, %k0, %k0
+; AVX512BW-NEXT:    kshiftrw $12, %k0, %k1
+; AVX512BW-NEXT:    vpmovqd %zmm0, %ymm0
+; AVX512BW-NEXT:    vmovdqu32 %zmm0, (%rdi) {%k1}
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+  %a = icmp ne <4 x i32> %mask, zeroinitializer
+  %b = trunc <4 x i64> %x to <4 x i32>
+  call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %b, <4 x i32>* %p, i32 1, <4 x i1> %a)
+  ret void
+}
+
+define void @truncstore_v4i64_v4i16(<4 x i64> %x, <4 x i16>* %p, <4 x i32> %mask) {
+; AVX512F-LABEL: truncstore_v4i64_v4i16:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512F-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    vpmovqd %zmm0, %ymm0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB4_2
+; AVX512F-NEXT:  # %bb.1: # %cond.store
+; AVX512F-NEXT:    vpextrw $0, %xmm0, (%rdi)
+; AVX512F-NEXT:  .LBB4_2: # %else
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $1, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB4_4
+; AVX512F-NEXT:  # %bb.3: # %cond.store1
+; AVX512F-NEXT:    vpextrw $2, %xmm0, 2(%rdi)
+; AVX512F-NEXT:  .LBB4_4: # %else2
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $2, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB4_6
+; AVX512F-NEXT:  # %bb.5: # %cond.store3
+; AVX512F-NEXT:    vpextrw $4, %xmm0, 4(%rdi)
+; AVX512F-NEXT:  .LBB4_6: # %else4
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $3, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB4_8
+; AVX512F-NEXT:  # %bb.7: # %cond.store5
+; AVX512F-NEXT:    vpextrw $6, %xmm0, 6(%rdi)
+; AVX512F-NEXT:  .LBB4_8: # %else6
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512BW-LABEL: truncstore_v4i64_v4i16:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512BW-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512BW-NEXT:    vpmovqd %zmm0, %ymm0
+; AVX512BW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15]
+; AVX512BW-NEXT:    kshiftld $28, %k0, %k0
+; AVX512BW-NEXT:    kshiftrd $28, %k0, %k1
+; AVX512BW-NEXT:    vmovdqu16 %zmm0, (%rdi) {%k1}
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+;
+; AVX512BWVL-LABEL: truncstore_v4i64_v4i16:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vptestmd %xmm1, %xmm1, %k1
+; AVX512BWVL-NEXT:    vpmovqw %ymm0, (%rdi) {%k1}
+; AVX512BWVL-NEXT:    vzeroupper
+; AVX512BWVL-NEXT:    retq
+  %a = icmp ne <4 x i32> %mask, zeroinitializer
+  %b = trunc <4 x i64> %x to <4 x i16>
+  call void @llvm.masked.store.v4i16.p0v4i16(<4 x i16> %b, <4 x i16>* %p, i32 1, <4 x i1> %a)
+  ret void
+}
+
+define void @truncstore_v4i64_v4i8(<4 x i64> %x, <4 x i8>* %p, <4 x i32> %mask) {
+; AVX512F-LABEL: truncstore_v4i64_v4i8:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512F-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    vpmovqd %zmm0, %ymm0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB5_2
+; AVX512F-NEXT:  # %bb.1: # %cond.store
+; AVX512F-NEXT:    vpextrb $0, %xmm0, (%rdi)
+; AVX512F-NEXT:  .LBB5_2: # %else
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $1, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB5_4
+; AVX512F-NEXT:  # %bb.3: # %cond.store1
+; AVX512F-NEXT:    vpextrb $4, %xmm0, 1(%rdi)
+; AVX512F-NEXT:  .LBB5_4: # %else2
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $2, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB5_6
+; AVX512F-NEXT:  # %bb.5: # %cond.store3
+; AVX512F-NEXT:    vpextrb $8, %xmm0, 2(%rdi)
+; AVX512F-NEXT:  .LBB5_6: # %else4
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $3, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB5_8
+; AVX512F-NEXT:  # %bb.7: # %cond.store5
+; AVX512F-NEXT:    vpextrb $12, %xmm0, 3(%rdi)
+; AVX512F-NEXT:  .LBB5_8: # %else6
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512BW-LABEL: truncstore_v4i64_v4i8:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512BW-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512BW-NEXT:    vpmovqd %zmm0, %ymm0
+; AVX512BW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512BW-NEXT:    kshiftlq $60, %k0, %k0
+; AVX512BW-NEXT:    kshiftrq $60, %k0, %k1
+; AVX512BW-NEXT:    vmovdqu8 %zmm0, (%rdi) {%k1}
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+;
+; AVX512BWVL-LABEL: truncstore_v4i64_v4i8:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vptestmd %xmm1, %xmm1, %k1
+; AVX512BWVL-NEXT:    vpmovqb %ymm0, (%rdi) {%k1}
+; AVX512BWVL-NEXT:    vzeroupper
+; AVX512BWVL-NEXT:    retq
+  %a = icmp ne <4 x i32> %mask, zeroinitializer
+  %b = trunc <4 x i64> %x to <4 x i8>
+  call void @llvm.masked.store.v4i8.p0v4i8(<4 x i8> %b, <4 x i8>* %p, i32 1, <4 x i1> %a)
+  ret void
+}
+
+define void @truncstore_v2i64_v2i32(<2 x i64> %x, <2 x i32>* %p, <2 x i64> %mask) {
+; AVX512F-LABEL: truncstore_v2i64_v2i32:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512F-NEXT:    vptestmq %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; AVX512F-NEXT:    kshiftlw $14, %k0, %k0
+; AVX512F-NEXT:    kshiftrw $14, %k0, %k1
+; AVX512F-NEXT:    vmovdqu32 %zmm0, (%rdi) {%k1}
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: truncstore_v2i64_v2i32:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vptestmq %xmm1, %xmm1, %k1
+; AVX512VL-NEXT:    vpmovqd %xmm0, (%rdi) {%k1}
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: truncstore_v2i64_v2i32:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512BW-NEXT:    vptestmq %zmm1, %zmm1, %k0
+; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; AVX512BW-NEXT:    kshiftlw $14, %k0, %k0
+; AVX512BW-NEXT:    kshiftrw $14, %k0, %k1
+; AVX512BW-NEXT:    vmovdqu32 %zmm0, (%rdi) {%k1}
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+  %a = icmp ne <2 x i64> %mask, zeroinitializer
+  %b = trunc <2 x i64> %x to <2 x i32>
+  call void @llvm.masked.store.v2i32.p0v2i32(<2 x i32> %b, <2 x i32>* %p, i32 1, <2 x i1> %a)
+  ret void
+}
+
+define void @truncstore_v2i64_v2i16(<2 x i64> %x, <2 x i16>* %p, <2 x i64> %mask) {
+; AVX512F-LABEL: truncstore_v2i64_v2i16:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512F-NEXT:    vptestmq %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB7_2
+; AVX512F-NEXT:  # %bb.1: # %cond.store
+; AVX512F-NEXT:    vpextrw $0, %xmm0, (%rdi)
+; AVX512F-NEXT:  .LBB7_2: # %else
+; AVX512F-NEXT:    vptestmq %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $1, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB7_4
+; AVX512F-NEXT:  # %bb.3: # %cond.store1
+; AVX512F-NEXT:    vpextrw $4, %xmm0, 2(%rdi)
+; AVX512F-NEXT:  .LBB7_4: # %else2
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512BW-LABEL: truncstore_v2i64_v2i16:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512BW-NEXT:    vptestmq %zmm1, %zmm1, %k0
+; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; AVX512BW-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; AVX512BW-NEXT:    kshiftld $30, %k0, %k0
+; AVX512BW-NEXT:    kshiftrd $30, %k0, %k1
+; AVX512BW-NEXT:    vmovdqu16 %zmm0, (%rdi) {%k1}
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+;
+; AVX512BWVL-LABEL: truncstore_v2i64_v2i16:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vptestmq %xmm1, %xmm1, %k1
+; AVX512BWVL-NEXT:    vpmovqw %xmm0, (%rdi) {%k1}
+; AVX512BWVL-NEXT:    retq
+  %a = icmp ne <2 x i64> %mask, zeroinitializer
+  %b = trunc <2 x i64> %x to <2 x i16>
+  call void @llvm.masked.store.v2i16.p0v2i16(<2 x i16> %b, <2 x i16>* %p, i32 1, <2 x i1> %a)
+  ret void
+}
+
+define void @truncstore_v2i64_v2i8(<2 x i64> %x, <2 x i8>* %p, <2 x i64> %mask) {
+; AVX512F-LABEL: truncstore_v2i64_v2i8:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512F-NEXT:    vptestmq %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB8_2
+; AVX512F-NEXT:  # %bb.1: # %cond.store
+; AVX512F-NEXT:    vpextrb $0, %xmm0, (%rdi)
+; AVX512F-NEXT:  .LBB8_2: # %else
+; AVX512F-NEXT:    vptestmq %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $1, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB8_4
+; AVX512F-NEXT:  # %bb.3: # %cond.store1
+; AVX512F-NEXT:    vpextrb $8, %xmm0, 1(%rdi)
+; AVX512F-NEXT:  .LBB8_4: # %else2
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512BW-LABEL: truncstore_v2i64_v2i8:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512BW-NEXT:    vptestmq %zmm1, %zmm1, %k0
+; AVX512BW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,8,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512BW-NEXT:    kshiftlq $62, %k0, %k0
+; AVX512BW-NEXT:    kshiftrq $62, %k0, %k1
+; AVX512BW-NEXT:    vmovdqu8 %zmm0, (%rdi) {%k1}
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+;
+; AVX512BWVL-LABEL: truncstore_v2i64_v2i8:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vptestmq %xmm1, %xmm1, %k1
+; AVX512BWVL-NEXT:    vpmovqb %xmm0, (%rdi) {%k1}
+; AVX512BWVL-NEXT:    retq
+  %a = icmp ne <2 x i64> %mask, zeroinitializer
+  %b = trunc <2 x i64> %x to <2 x i8>
+  call void @llvm.masked.store.v2i8.p0v2i8(<2 x i8> %b, <2 x i8>* %p, i32 1, <2 x i1> %a)
+  ret void
+}
+
+define void @truncstore_v16i32_v16i16(<16 x i32> %x, <16 x i16>* %p, <16 x i32> %mask) {
+; AVX512F-LABEL: truncstore_v16i32_v16i16:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    vpmovdw %zmm0, %ymm0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB9_2
+; AVX512F-NEXT:  # %bb.1: # %cond.store
+; AVX512F-NEXT:    vpextrw $0, %xmm0, (%rdi)
+; AVX512F-NEXT:  .LBB9_2: # %else
+; AVX512F-NEXT:    kshiftrw $1, %k0, %k1
+; AVX512F-NEXT:    kmovw %k1, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB9_4
+; AVX512F-NEXT:  # %bb.3: # %cond.store1
+; AVX512F-NEXT:    vpextrw $1, %xmm0, 2(%rdi)
+; AVX512F-NEXT:  .LBB9_4: # %else2
+; AVX512F-NEXT:    kshiftrw $2, %k0, %k1
+; AVX512F-NEXT:    kmovw %k1, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB9_6
+; AVX512F-NEXT:  # %bb.5: # %cond.store3
+; AVX512F-NEXT:    vpextrw $2, %xmm0, 4(%rdi)
+; AVX512F-NEXT:  .LBB9_6: # %else4
+; AVX512F-NEXT:    kshiftrw $3, %k0, %k1
+; AVX512F-NEXT:    kmovw %k1, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB9_8
+; AVX512F-NEXT:  # %bb.7: # %cond.store5
+; AVX512F-NEXT:    vpextrw $3, %xmm0, 6(%rdi)
+; AVX512F-NEXT:  .LBB9_8: # %else6
+; AVX512F-NEXT:    kshiftrw $4, %k0, %k1
+; AVX512F-NEXT:    kmovw %k1, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB9_10
+; AVX512F-NEXT:  # %bb.9: # %cond.store7
+; AVX512F-NEXT:    vpextrw $4, %xmm0, 8(%rdi)
+; AVX512F-NEXT:  .LBB9_10: # %else8
+; AVX512F-NEXT:    kshiftrw $5, %k0, %k1
+; AVX512F-NEXT:    kmovw %k1, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB9_12
+; AVX512F-NEXT:  # %bb.11: # %cond.store9
+; AVX512F-NEXT:    vpextrw $5, %xmm0, 10(%rdi)
+; AVX512F-NEXT:  .LBB9_12: # %else10
+; AVX512F-NEXT:    kshiftrw $6, %k0, %k1
+; AVX512F-NEXT:    kmovw %k1, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB9_14
+; AVX512F-NEXT:  # %bb.13: # %cond.store11
+; AVX512F-NEXT:    vpextrw $6, %xmm0, 12(%rdi)
+; AVX512F-NEXT:  .LBB9_14: # %else12
+; AVX512F-NEXT:    kshiftrw $7, %k0, %k1
+; AVX512F-NEXT:    kmovw %k1, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB9_16
+; AVX512F-NEXT:  # %bb.15: # %cond.store13
+; AVX512F-NEXT:    vpextrw $7, %xmm0, 14(%rdi)
+; AVX512F-NEXT:  .LBB9_16: # %else14
+; AVX512F-NEXT:    kshiftrw $8, %k0, %k1
+; AVX512F-NEXT:    kmovw %k1, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB9_18
+; AVX512F-NEXT:  # %bb.17: # %cond.store15
+; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX512F-NEXT:    vpextrw $0, %xmm1, 16(%rdi)
+; AVX512F-NEXT:  .LBB9_18: # %else16
+; AVX512F-NEXT:    kshiftrw $9, %k0, %k1
+; AVX512F-NEXT:    kmovw %k1, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB9_20
+; AVX512F-NEXT:  # %bb.19: # %cond.store17
+; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX512F-NEXT:    vpextrw $1, %xmm1, 18(%rdi)
+; AVX512F-NEXT:  .LBB9_20: # %else18
+; AVX512F-NEXT:    kshiftrw $10, %k0, %k1
+; AVX512F-NEXT:    kmovw %k1, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB9_22
+; AVX512F-NEXT:  # %bb.21: # %cond.store19
+; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX512F-NEXT:    vpextrw $2, %xmm1, 20(%rdi)
+; AVX512F-NEXT:  .LBB9_22: # %else20
+; AVX512F-NEXT:    kshiftrw $11, %k0, %k1
+; AVX512F-NEXT:    kmovw %k1, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB9_24
+; AVX512F-NEXT:  # %bb.23: # %cond.store21
+; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX512F-NEXT:    vpextrw $3, %xmm1, 22(%rdi)
+; AVX512F-NEXT:  .LBB9_24: # %else22
+; AVX512F-NEXT:    kshiftrw $12, %k0, %k1
+; AVX512F-NEXT:    kmovw %k1, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB9_26
+; AVX512F-NEXT:  # %bb.25: # %cond.store23
+; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX512F-NEXT:    vpextrw $4, %xmm1, 24(%rdi)
+; AVX512F-NEXT:  .LBB9_26: # %else24
+; AVX512F-NEXT:    kshiftrw $13, %k0, %k1
+; AVX512F-NEXT:    kmovw %k1, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB9_28
+; AVX512F-NEXT:  # %bb.27: # %cond.store25
+; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX512F-NEXT:    vpextrw $5, %xmm1, 26(%rdi)
+; AVX512F-NEXT:  .LBB9_28: # %else26
+; AVX512F-NEXT:    kshiftrw $14, %k0, %k1
+; AVX512F-NEXT:    kmovw %k1, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB9_30
+; AVX512F-NEXT:  # %bb.29: # %cond.store27
+; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX512F-NEXT:    vpextrw $6, %xmm1, 28(%rdi)
+; AVX512F-NEXT:  .LBB9_30: # %else28
+; AVX512F-NEXT:    kshiftrw $15, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB9_32
+; AVX512F-NEXT:  # %bb.31: # %cond.store29
+; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm0
+; AVX512F-NEXT:    vpextrw $7, %xmm0, 30(%rdi)
+; AVX512F-NEXT:  .LBB9_32: # %else30
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512BW-LABEL: truncstore_v16i32_v16i16:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vptestmd %zmm1, %zmm1, %k1
+; AVX512BW-NEXT:    vpmovdw %zmm0, (%rdi) {%k1}
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+;
+; AVX512BWVL-LABEL: truncstore_v16i32_v16i16:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vptestmd %zmm1, %zmm1, %k1
+; AVX512BWVL-NEXT:    vpmovdw %zmm0, (%rdi) {%k1}
+; AVX512BWVL-NEXT:    vzeroupper
+; AVX512BWVL-NEXT:    retq
+  %a = icmp ne <16 x i32> %mask, zeroinitializer
+  %b = trunc <16 x i32> %x to <16 x i16>
+  call void @llvm.masked.store.v16i16.p0v16i16(<16 x i16> %b, <16 x i16>* %p, i32 1, <16 x i1> %a)
+  ret void
+}
+
+define void @truncstore_v16i32_v16i8(<16 x i32> %x, <16 x i8>* %p, <16 x i32> %mask) {
+; AVX512F-LABEL: truncstore_v16i32_v16i8:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    vpmovdb %zmm0, %xmm0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB10_2
+; AVX512F-NEXT:  # %bb.1: # %cond.store
+; AVX512F-NEXT:    vpextrb $0, %xmm0, (%rdi)
+; AVX512F-NEXT:  .LBB10_2: # %else
+; AVX512F-NEXT:    kshiftrw $1, %k0, %k1
+; AVX512F-NEXT:    kmovw %k1, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB10_4
+; AVX512F-NEXT:  # %bb.3: # %cond.store1
+; AVX512F-NEXT:    vpextrb $1, %xmm0, 1(%rdi)
+; AVX512F-NEXT:  .LBB10_4: # %else2
+; AVX512F-NEXT:    kshiftrw $2, %k0, %k1
+; AVX512F-NEXT:    kmovw %k1, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB10_6
+; AVX512F-NEXT:  # %bb.5: # %cond.store3
+; AVX512F-NEXT:    vpextrb $2, %xmm0, 2(%rdi)
+; AVX512F-NEXT:  .LBB10_6: # %else4
+; AVX512F-NEXT:    kshiftrw $3, %k0, %k1
+; AVX512F-NEXT:    kmovw %k1, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB10_8
+; AVX512F-NEXT:  # %bb.7: # %cond.store5
+; AVX512F-NEXT:    vpextrb $3, %xmm0, 3(%rdi)
+; AVX512F-NEXT:  .LBB10_8: # %else6
+; AVX512F-NEXT:    kshiftrw $4, %k0, %k1
+; AVX512F-NEXT:    kmovw %k1, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB10_10
+; AVX512F-NEXT:  # %bb.9: # %cond.store7
+; AVX512F-NEXT:    vpextrb $4, %xmm0, 4(%rdi)
+; AVX512F-NEXT:  .LBB10_10: # %else8
+; AVX512F-NEXT:    kshiftrw $5, %k0, %k1
+; AVX512F-NEXT:    kmovw %k1, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB10_12
+; AVX512F-NEXT:  # %bb.11: # %cond.store9
+; AVX512F-NEXT:    vpextrb $5, %xmm0, 5(%rdi)
+; AVX512F-NEXT:  .LBB10_12: # %else10
+; AVX512F-NEXT:    kshiftrw $6, %k0, %k1
+; AVX512F-NEXT:    kmovw %k1, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB10_14
+; AVX512F-NEXT:  # %bb.13: # %cond.store11
+; AVX512F-NEXT:    vpextrb $6, %xmm0, 6(%rdi)
+; AVX512F-NEXT:  .LBB10_14: # %else12
+; AVX512F-NEXT:    kshiftrw $7, %k0, %k1
+; AVX512F-NEXT:    kmovw %k1, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB10_16
+; AVX512F-NEXT:  # %bb.15: # %cond.store13
+; AVX512F-NEXT:    vpextrb $7, %xmm0, 7(%rdi)
+; AVX512F-NEXT:  .LBB10_16: # %else14
+; AVX512F-NEXT:    kshiftrw $8, %k0, %k1
+; AVX512F-NEXT:    kmovw %k1, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB10_18
+; AVX512F-NEXT:  # %bb.17: # %cond.store15
+; AVX512F-NEXT:    vpextrb $8, %xmm0, 8(%rdi)
+; AVX512F-NEXT:  .LBB10_18: # %else16
+; AVX512F-NEXT:    kshiftrw $9, %k0, %k1
+; AVX512F-NEXT:    kmovw %k1, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB10_20
+; AVX512F-NEXT:  # %bb.19: # %cond.store17
+; AVX512F-NEXT:    vpextrb $9, %xmm0, 9(%rdi)
+; AVX512F-NEXT:  .LBB10_20: # %else18
+; AVX512F-NEXT:    kshiftrw $10, %k0, %k1
+; AVX512F-NEXT:    kmovw %k1, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB10_22
+; AVX512F-NEXT:  # %bb.21: # %cond.store19
+; AVX512F-NEXT:    vpextrb $10, %xmm0, 10(%rdi)
+; AVX512F-NEXT:  .LBB10_22: # %else20
+; AVX512F-NEXT:    kshiftrw $11, %k0, %k1
+; AVX512F-NEXT:    kmovw %k1, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB10_24
+; AVX512F-NEXT:  # %bb.23: # %cond.store21
+; AVX512F-NEXT:    vpextrb $11, %xmm0, 11(%rdi)
+; AVX512F-NEXT:  .LBB10_24: # %else22
+; AVX512F-NEXT:    kshiftrw $12, %k0, %k1
+; AVX512F-NEXT:    kmovw %k1, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB10_26
+; AVX512F-NEXT:  # %bb.25: # %cond.store23
+; AVX512F-NEXT:    vpextrb $12, %xmm0, 12(%rdi)
+; AVX512F-NEXT:  .LBB10_26: # %else24
+; AVX512F-NEXT:    kshiftrw $13, %k0, %k1
+; AVX512F-NEXT:    kmovw %k1, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB10_28
+; AVX512F-NEXT:  # %bb.27: # %cond.store25
+; AVX512F-NEXT:    vpextrb $13, %xmm0, 13(%rdi)
+; AVX512F-NEXT:  .LBB10_28: # %else26
+; AVX512F-NEXT:    kshiftrw $14, %k0, %k1
+; AVX512F-NEXT:    kmovw %k1, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB10_30
+; AVX512F-NEXT:  # %bb.29: # %cond.store27
+; AVX512F-NEXT:    vpextrb $14, %xmm0, 14(%rdi)
+; AVX512F-NEXT:  .LBB10_30: # %else28
+; AVX512F-NEXT:    kshiftrw $15, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB10_32
+; AVX512F-NEXT:  # %bb.31: # %cond.store29
+; AVX512F-NEXT:    vpextrb $15, %xmm0, 15(%rdi)
+; AVX512F-NEXT:  .LBB10_32: # %else30
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512BW-LABEL: truncstore_v16i32_v16i8:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vptestmd %zmm1, %zmm1, %k1
+; AVX512BW-NEXT:    vpmovdb %zmm0, (%rdi) {%k1}
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+;
+; AVX512BWVL-LABEL: truncstore_v16i32_v16i8:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vptestmd %zmm1, %zmm1, %k1
+; AVX512BWVL-NEXT:    vpmovdb %zmm0, (%rdi) {%k1}
+; AVX512BWVL-NEXT:    vzeroupper
+; AVX512BWVL-NEXT:    retq
+  %a = icmp ne <16 x i32> %mask, zeroinitializer
+  %b = trunc <16 x i32> %x to <16 x i8>
+  call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %b, <16 x i8>* %p, i32 1, <16 x i1> %a)
+  ret void
+}
+
+define void @truncstore_v8i32_v8i16(<8 x i32> %x, <8 x i16>* %p, <8 x i32> %mask) {
+; AVX512F-LABEL: truncstore_v8i32_v8i16:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512F-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    vpmovdw %zmm0, %ymm0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB11_2
+; AVX512F-NEXT:  # %bb.1: # %cond.store
+; AVX512F-NEXT:    vpextrw $0, %xmm0, (%rdi)
+; AVX512F-NEXT:  .LBB11_2: # %else
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $1, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB11_4
+; AVX512F-NEXT:  # %bb.3: # %cond.store1
+; AVX512F-NEXT:    vpextrw $1, %xmm0, 2(%rdi)
+; AVX512F-NEXT:  .LBB11_4: # %else2
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $2, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB11_6
+; AVX512F-NEXT:  # %bb.5: # %cond.store3
+; AVX512F-NEXT:    vpextrw $2, %xmm0, 4(%rdi)
+; AVX512F-NEXT:  .LBB11_6: # %else4
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $3, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB11_8
+; AVX512F-NEXT:  # %bb.7: # %cond.store5
+; AVX512F-NEXT:    vpextrw $3, %xmm0, 6(%rdi)
+; AVX512F-NEXT:  .LBB11_8: # %else6
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $4, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB11_10
+; AVX512F-NEXT:  # %bb.9: # %cond.store7
+; AVX512F-NEXT:    vpextrw $4, %xmm0, 8(%rdi)
+; AVX512F-NEXT:  .LBB11_10: # %else8
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $5, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB11_12
+; AVX512F-NEXT:  # %bb.11: # %cond.store9
+; AVX512F-NEXT:    vpextrw $5, %xmm0, 10(%rdi)
+; AVX512F-NEXT:  .LBB11_12: # %else10
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $6, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB11_14
+; AVX512F-NEXT:  # %bb.13: # %cond.store11
+; AVX512F-NEXT:    vpextrw $6, %xmm0, 12(%rdi)
+; AVX512F-NEXT:  .LBB11_14: # %else12
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $7, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB11_16
+; AVX512F-NEXT:  # %bb.15: # %cond.store13
+; AVX512F-NEXT:    vpextrw $7, %xmm0, 14(%rdi)
+; AVX512F-NEXT:  .LBB11_16: # %else14
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512BW-LABEL: truncstore_v8i32_v8i16:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512BW-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512BW-NEXT:    kshiftld $24, %k0, %k0
+; AVX512BW-NEXT:    kshiftrd $24, %k0, %k1
+; AVX512BW-NEXT:    vpmovdw %zmm0, %ymm0
+; AVX512BW-NEXT:    vmovdqu16 %zmm0, (%rdi) {%k1}
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+;
+; AVX512BWVL-LABEL: truncstore_v8i32_v8i16:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vptestmd %ymm1, %ymm1, %k1
+; AVX512BWVL-NEXT:    vpmovdw %ymm0, (%rdi) {%k1}
+; AVX512BWVL-NEXT:    vzeroupper
+; AVX512BWVL-NEXT:    retq
+  %a = icmp ne <8 x i32> %mask, zeroinitializer
+  %b = trunc <8 x i32> %x to <8 x i16>
+  call void @llvm.masked.store.v8i16.p0v8i16(<8 x i16> %b, <8 x i16>* %p, i32 1, <8 x i1> %a)
+  ret void
+}
+
+define void @truncstore_v8i32_v8i8(<8 x i32> %x, <8 x i8>* %p, <8 x i32> %mask) {
+; AVX512F-LABEL: truncstore_v8i32_v8i8:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512F-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    vpmovdw %zmm0, %ymm0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB12_2
+; AVX512F-NEXT:  # %bb.1: # %cond.store
+; AVX512F-NEXT:    vpextrb $0, %xmm0, (%rdi)
+; AVX512F-NEXT:  .LBB12_2: # %else
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $1, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB12_4
+; AVX512F-NEXT:  # %bb.3: # %cond.store1
+; AVX512F-NEXT:    vpextrb $2, %xmm0, 1(%rdi)
+; AVX512F-NEXT:  .LBB12_4: # %else2
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $2, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB12_6
+; AVX512F-NEXT:  # %bb.5: # %cond.store3
+; AVX512F-NEXT:    vpextrb $4, %xmm0, 2(%rdi)
+; AVX512F-NEXT:  .LBB12_6: # %else4
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $3, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB12_8
+; AVX512F-NEXT:  # %bb.7: # %cond.store5
+; AVX512F-NEXT:    vpextrb $6, %xmm0, 3(%rdi)
+; AVX512F-NEXT:  .LBB12_8: # %else6
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $4, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB12_10
+; AVX512F-NEXT:  # %bb.9: # %cond.store7
+; AVX512F-NEXT:    vpextrb $8, %xmm0, 4(%rdi)
+; AVX512F-NEXT:  .LBB12_10: # %else8
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $5, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB12_12
+; AVX512F-NEXT:  # %bb.11: # %cond.store9
+; AVX512F-NEXT:    vpextrb $10, %xmm0, 5(%rdi)
+; AVX512F-NEXT:  .LBB12_12: # %else10
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $6, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB12_14
+; AVX512F-NEXT:  # %bb.13: # %cond.store11
+; AVX512F-NEXT:    vpextrb $12, %xmm0, 6(%rdi)
+; AVX512F-NEXT:  .LBB12_14: # %else12
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $7, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB12_16
+; AVX512F-NEXT:  # %bb.15: # %cond.store13
+; AVX512F-NEXT:    vpextrb $14, %xmm0, 7(%rdi)
+; AVX512F-NEXT:  .LBB12_16: # %else14
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512BW-LABEL: truncstore_v8i32_v8i8:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512BW-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512BW-NEXT:    vpmovdw %zmm0, %ymm0
+; AVX512BW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u]
+; AVX512BW-NEXT:    kshiftlq $56, %k0, %k0
+; AVX512BW-NEXT:    kshiftrq $56, %k0, %k1
+; AVX512BW-NEXT:    vmovdqu8 %zmm0, (%rdi) {%k1}
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+;
+; AVX512BWVL-LABEL: truncstore_v8i32_v8i8:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vptestmd %ymm1, %ymm1, %k1
+; AVX512BWVL-NEXT:    vpmovdb %ymm0, (%rdi) {%k1}
+; AVX512BWVL-NEXT:    vzeroupper
+; AVX512BWVL-NEXT:    retq
+  %a = icmp ne <8 x i32> %mask, zeroinitializer
+  %b = trunc <8 x i32> %x to <8 x i8>
+  call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> %b, <8 x i8>* %p, i32 1, <8 x i1> %a)
+  ret void
+}
+
+define void @truncstore_v4i32_v4i16(<4 x i32> %x, <4 x i16>* %p, <4 x i32> %mask) {
+; AVX512F-LABEL: truncstore_v4i32_v4i16:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB13_2
+; AVX512F-NEXT:  # %bb.1: # %cond.store
+; AVX512F-NEXT:    vpextrw $0, %xmm0, (%rdi)
+; AVX512F-NEXT:  .LBB13_2: # %else
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $1, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB13_4
+; AVX512F-NEXT:  # %bb.3: # %cond.store1
+; AVX512F-NEXT:    vpextrw $2, %xmm0, 2(%rdi)
+; AVX512F-NEXT:  .LBB13_4: # %else2
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $2, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB13_6
+; AVX512F-NEXT:  # %bb.5: # %cond.store3
+; AVX512F-NEXT:    vpextrw $4, %xmm0, 4(%rdi)
+; AVX512F-NEXT:  .LBB13_6: # %else4
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $3, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB13_8
+; AVX512F-NEXT:  # %bb.7: # %cond.store5
+; AVX512F-NEXT:    vpextrw $6, %xmm0, 6(%rdi)
+; AVX512F-NEXT:  .LBB13_8: # %else6
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512BW-LABEL: truncstore_v4i32_v4i16:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512BW-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512BW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15]
+; AVX512BW-NEXT:    kshiftld $28, %k0, %k0
+; AVX512BW-NEXT:    kshiftrd $28, %k0, %k1
+; AVX512BW-NEXT:    vmovdqu16 %zmm0, (%rdi) {%k1}
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+;
+; AVX512BWVL-LABEL: truncstore_v4i32_v4i16:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vptestmd %xmm1, %xmm1, %k1
+; AVX512BWVL-NEXT:    vpmovdw %xmm0, (%rdi) {%k1}
+; AVX512BWVL-NEXT:    retq
+  %a = icmp ne <4 x i32> %mask, zeroinitializer
+  %b = trunc <4 x i32> %x to <4 x i16>
+  call void @llvm.masked.store.v4i16.p0v4i16(<4 x i16> %b, <4 x i16>* %p, i32 1, <4 x i1> %a)
+  ret void
+}
+
+define void @truncstore_v4i32_v4i8(<4 x i32> %x, <4 x i8>* %p, <4 x i32> %mask) {
+; AVX512F-LABEL: truncstore_v4i32_v4i8:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB14_2
+; AVX512F-NEXT:  # %bb.1: # %cond.store
+; AVX512F-NEXT:    vpextrb $0, %xmm0, (%rdi)
+; AVX512F-NEXT:  .LBB14_2: # %else
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $1, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB14_4
+; AVX512F-NEXT:  # %bb.3: # %cond.store1
+; AVX512F-NEXT:    vpextrb $4, %xmm0, 1(%rdi)
+; AVX512F-NEXT:  .LBB14_4: # %else2
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $2, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB14_6
+; AVX512F-NEXT:  # %bb.5: # %cond.store3
+; AVX512F-NEXT:    vpextrb $8, %xmm0, 2(%rdi)
+; AVX512F-NEXT:  .LBB14_6: # %else4
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $3, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB14_8
+; AVX512F-NEXT:  # %bb.7: # %cond.store5
+; AVX512F-NEXT:    vpextrb $12, %xmm0, 3(%rdi)
+; AVX512F-NEXT:  .LBB14_8: # %else6
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512BW-LABEL: truncstore_v4i32_v4i8:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512BW-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512BW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512BW-NEXT:    kshiftlq $60, %k0, %k0
+; AVX512BW-NEXT:    kshiftrq $60, %k0, %k1
+; AVX512BW-NEXT:    vmovdqu8 %zmm0, (%rdi) {%k1}
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+;
+; AVX512BWVL-LABEL: truncstore_v4i32_v4i8:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vptestmd %xmm1, %xmm1, %k1
+; AVX512BWVL-NEXT:    vpmovdb %xmm0, (%rdi) {%k1}
+; AVX512BWVL-NEXT:    retq
+  %a = icmp ne <4 x i32> %mask, zeroinitializer
+  %b = trunc <4 x i32> %x to <4 x i8>
+  call void @llvm.masked.store.v4i8.p0v4i8(<4 x i8> %b, <4 x i8>* %p, i32 1, <4 x i1> %a)
+  ret void
+}
+
+define void @truncstore_v32i16_v32i8(<32 x i16> %x, <32 x i8>* %p, <32 x i8> %mask) {
+; AVX512F-LABEL: truncstore_v32i16_v32i8:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; AVX512F-NEXT:    vpcmpeqb %ymm3, %ymm2, %ymm3
+; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm4
+; AVX512F-NEXT:    vpternlogq $15, %zmm3, %zmm3, %zmm4
+; AVX512F-NEXT:    vpmovsxbd %xmm4, %zmm4
+; AVX512F-NEXT:    vptestmd %zmm4, %zmm4, %k0
+; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
+; AVX512F-NEXT:    vpmovdb %zmm0, %xmm0
+; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero
+; AVX512F-NEXT:    vpmovdb %zmm1, %xmm1
+; AVX512F-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB15_2
+; AVX512F-NEXT:  # %bb.1: # %cond.store
+; AVX512F-NEXT:    vpextrb $0, %xmm0, (%rdi)
+; AVX512F-NEXT:  .LBB15_2: # %else
+; AVX512F-NEXT:    vpternlogq $15, %zmm3, %zmm3, %zmm3
+; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm1
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $1, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB15_4
+; AVX512F-NEXT:  # %bb.3: # %cond.store1
+; AVX512F-NEXT:    vpextrb $1, %xmm0, 1(%rdi)
+; AVX512F-NEXT:  .LBB15_4: # %else2
+; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512F-NEXT:    vpcmpeqb %ymm1, %ymm2, %ymm1
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm3
+; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm3
+; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
+; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
+; AVX512F-NEXT:    kshiftrw $2, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB15_6
+; AVX512F-NEXT:  # %bb.5: # %cond.store3
+; AVX512F-NEXT:    vpextrb $2, %xmm0, 2(%rdi)
+; AVX512F-NEXT:  .LBB15_6: # %else4
+; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm1
+; AVX512F-NEXT:    vpmovsxbd %xmm1, %zmm1
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $3, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB15_8
+; AVX512F-NEXT:  # %bb.7: # %cond.store5
+; AVX512F-NEXT:    vpextrb $3, %xmm0, 3(%rdi)
+; AVX512F-NEXT:  .LBB15_8: # %else6
+; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512F-NEXT:    vpcmpeqb %ymm1, %ymm2, %ymm1
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm3
+; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm3
+; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
+; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
+; AVX512F-NEXT:    kshiftrw $4, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB15_10
+; AVX512F-NEXT:  # %bb.9: # %cond.store7
+; AVX512F-NEXT:    vpextrb $4, %xmm0, 4(%rdi)
+; AVX512F-NEXT:  .LBB15_10: # %else8
+; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm1
+; AVX512F-NEXT:    vpmovsxbd %xmm1, %zmm1
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $5, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB15_12
+; AVX512F-NEXT:  # %bb.11: # %cond.store9
+; AVX512F-NEXT:    vpextrb $5, %xmm0, 5(%rdi)
+; AVX512F-NEXT:  .LBB15_12: # %else10
+; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512F-NEXT:    vpcmpeqb %ymm1, %ymm2, %ymm1
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm3
+; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm3
+; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
+; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
+; AVX512F-NEXT:    kshiftrw $6, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB15_14
+; AVX512F-NEXT:  # %bb.13: # %cond.store11
+; AVX512F-NEXT:    vpextrb $6, %xmm0, 6(%rdi)
+; AVX512F-NEXT:  .LBB15_14: # %else12
+; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm1
+; AVX512F-NEXT:    vpmovsxbd %xmm1, %zmm1
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $7, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB15_16
+; AVX512F-NEXT:  # %bb.15: # %cond.store13
+; AVX512F-NEXT:    vpextrb $7, %xmm0, 7(%rdi)
+; AVX512F-NEXT:  .LBB15_16: # %else14
+; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512F-NEXT:    vpcmpeqb %ymm1, %ymm2, %ymm1
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm3
+; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm3
+; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
+; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
+; AVX512F-NEXT:    kshiftrw $8, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB15_18
+; AVX512F-NEXT:  # %bb.17: # %cond.store15
+; AVX512F-NEXT:    vpextrb $8, %xmm0, 8(%rdi)
+; AVX512F-NEXT:  .LBB15_18: # %else16
+; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm1
+; AVX512F-NEXT:    vpmovsxbd %xmm1, %zmm1
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $9, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB15_20
+; AVX512F-NEXT:  # %bb.19: # %cond.store17
+; AVX512F-NEXT:    vpextrb $9, %xmm0, 9(%rdi)
+; AVX512F-NEXT:  .LBB15_20: # %else18
+; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512F-NEXT:    vpcmpeqb %ymm1, %ymm2, %ymm1
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm3
+; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm3
+; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
+; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
+; AVX512F-NEXT:    kshiftrw $10, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB15_22
+; AVX512F-NEXT:  # %bb.21: # %cond.store19
+; AVX512F-NEXT:    vpextrb $10, %xmm0, 10(%rdi)
+; AVX512F-NEXT:  .LBB15_22: # %else20
+; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm1
+; AVX512F-NEXT:    vpmovsxbd %xmm1, %zmm1
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $11, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB15_24
+; AVX512F-NEXT:  # %bb.23: # %cond.store21
+; AVX512F-NEXT:    vpextrb $11, %xmm0, 11(%rdi)
+; AVX512F-NEXT:  .LBB15_24: # %else22
+; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512F-NEXT:    vpcmpeqb %ymm1, %ymm2, %ymm1
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm3
+; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm3
+; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
+; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
+; AVX512F-NEXT:    kshiftrw $12, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB15_26
+; AVX512F-NEXT:  # %bb.25: # %cond.store23
+; AVX512F-NEXT:    vpextrb $12, %xmm0, 12(%rdi)
+; AVX512F-NEXT:  .LBB15_26: # %else24
+; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm1
+; AVX512F-NEXT:    vpmovsxbd %xmm1, %zmm1
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $13, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB15_28
+; AVX512F-NEXT:  # %bb.27: # %cond.store25
+; AVX512F-NEXT:    vpextrb $13, %xmm0, 13(%rdi)
+; AVX512F-NEXT:  .LBB15_28: # %else26
+; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512F-NEXT:    vpcmpeqb %ymm1, %ymm2, %ymm1
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm3
+; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm3
+; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
+; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
+; AVX512F-NEXT:    kshiftrw $14, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB15_30
+; AVX512F-NEXT:  # %bb.29: # %cond.store27
+; AVX512F-NEXT:    vpextrb $14, %xmm0, 14(%rdi)
+; AVX512F-NEXT:  .LBB15_30: # %else28
+; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm1
+; AVX512F-NEXT:    vpmovsxbd %xmm1, %zmm1
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $15, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB15_32
+; AVX512F-NEXT:  # %bb.31: # %cond.store29
+; AVX512F-NEXT:    vpextrb $15, %xmm0, 15(%rdi)
+; AVX512F-NEXT:  .LBB15_32: # %else30
+; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512F-NEXT:    vpcmpeqb %ymm1, %ymm2, %ymm1
+; AVX512F-NEXT:    vextracti128 $1, %ymm1, %xmm1
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm3
+; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm3
+; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
+; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB15_34
+; AVX512F-NEXT:  # %bb.33: # %cond.store31
+; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm3
+; AVX512F-NEXT:    vpextrb $0, %xmm3, 16(%rdi)
+; AVX512F-NEXT:  .LBB15_34: # %else32
+; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm1
+; AVX512F-NEXT:    vpmovsxbd %xmm1, %zmm1
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $1, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB15_36
+; AVX512F-NEXT:  # %bb.35: # %cond.store33
+; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX512F-NEXT:    vpextrb $1, %xmm1, 17(%rdi)
+; AVX512F-NEXT:  .LBB15_36: # %else34
+; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512F-NEXT:    vpcmpeqb %ymm1, %ymm2, %ymm1
+; AVX512F-NEXT:    vextracti128 $1, %ymm1, %xmm1
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm3
+; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm3
+; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
+; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
+; AVX512F-NEXT:    kshiftrw $2, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB15_38
+; AVX512F-NEXT:  # %bb.37: # %cond.store35
+; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm3
+; AVX512F-NEXT:    vpextrb $2, %xmm3, 18(%rdi)
+; AVX512F-NEXT:  .LBB15_38: # %else36
+; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm1
+; AVX512F-NEXT:    vpmovsxbd %xmm1, %zmm1
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $3, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB15_40
+; AVX512F-NEXT:  # %bb.39: # %cond.store37
+; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX512F-NEXT:    vpextrb $3, %xmm1, 19(%rdi)
+; AVX512F-NEXT:  .LBB15_40: # %else38
+; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512F-NEXT:    vpcmpeqb %ymm1, %ymm2, %ymm1
+; AVX512F-NEXT:    vextracti128 $1, %ymm1, %xmm1
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm3
+; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm3
+; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
+; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
+; AVX512F-NEXT:    kshiftrw $4, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB15_42
+; AVX512F-NEXT:  # %bb.41: # %cond.store39
+; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm3
+; AVX512F-NEXT:    vpextrb $4, %xmm3, 20(%rdi)
+; AVX512F-NEXT:  .LBB15_42: # %else40
+; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm1
+; AVX512F-NEXT:    vpmovsxbd %xmm1, %zmm1
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $5, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB15_44
+; AVX512F-NEXT:  # %bb.43: # %cond.store41
+; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX512F-NEXT:    vpextrb $5, %xmm1, 21(%rdi)
+; AVX512F-NEXT:  .LBB15_44: # %else42
+; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512F-NEXT:    vpcmpeqb %ymm1, %ymm2, %ymm1
+; AVX512F-NEXT:    vextracti128 $1, %ymm1, %xmm1
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm3
+; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm3
+; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
+; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
+; AVX512F-NEXT:    kshiftrw $6, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB15_46
+; AVX512F-NEXT:  # %bb.45: # %cond.store43
+; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm3
+; AVX512F-NEXT:    vpextrb $6, %xmm3, 22(%rdi)
+; AVX512F-NEXT:  .LBB15_46: # %else44
+; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm1
+; AVX512F-NEXT:    vpmovsxbd %xmm1, %zmm1
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $7, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB15_48
+; AVX512F-NEXT:  # %bb.47: # %cond.store45
+; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX512F-NEXT:    vpextrb $7, %xmm1, 23(%rdi)
+; AVX512F-NEXT:  .LBB15_48: # %else46
+; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512F-NEXT:    vpcmpeqb %ymm1, %ymm2, %ymm1
+; AVX512F-NEXT:    vextracti128 $1, %ymm1, %xmm1
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm3
+; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm3
+; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
+; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
+; AVX512F-NEXT:    kshiftrw $8, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB15_50
+; AVX512F-NEXT:  # %bb.49: # %cond.store47
+; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm3
+; AVX512F-NEXT:    vpextrb $8, %xmm3, 24(%rdi)
+; AVX512F-NEXT:  .LBB15_50: # %else48
+; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm1
+; AVX512F-NEXT:    vpmovsxbd %xmm1, %zmm1
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $9, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB15_52
+; AVX512F-NEXT:  # %bb.51: # %cond.store49
+; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX512F-NEXT:    vpextrb $9, %xmm1, 25(%rdi)
+; AVX512F-NEXT:  .LBB15_52: # %else50
+; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512F-NEXT:    vpcmpeqb %ymm1, %ymm2, %ymm1
+; AVX512F-NEXT:    vextracti128 $1, %ymm1, %xmm1
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm3
+; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm3
+; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
+; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
+; AVX512F-NEXT:    kshiftrw $10, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB15_54
+; AVX512F-NEXT:  # %bb.53: # %cond.store51
+; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm3
+; AVX512F-NEXT:    vpextrb $10, %xmm3, 26(%rdi)
+; AVX512F-NEXT:  .LBB15_54: # %else52
+; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm1
+; AVX512F-NEXT:    vpmovsxbd %xmm1, %zmm1
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $11, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB15_56
+; AVX512F-NEXT:  # %bb.55: # %cond.store53
+; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX512F-NEXT:    vpextrb $11, %xmm1, 27(%rdi)
+; AVX512F-NEXT:  .LBB15_56: # %else54
+; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512F-NEXT:    vpcmpeqb %ymm1, %ymm2, %ymm1
+; AVX512F-NEXT:    vextracti128 $1, %ymm1, %xmm1
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm3
+; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm3
+; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
+; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
+; AVX512F-NEXT:    kshiftrw $12, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB15_58
+; AVX512F-NEXT:  # %bb.57: # %cond.store55
+; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm3
+; AVX512F-NEXT:    vpextrb $12, %xmm3, 28(%rdi)
+; AVX512F-NEXT:  .LBB15_58: # %else56
+; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm1
+; AVX512F-NEXT:    vpmovsxbd %xmm1, %zmm1
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $13, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB15_60
+; AVX512F-NEXT:  # %bb.59: # %cond.store57
+; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX512F-NEXT:    vpextrb $13, %xmm1, 29(%rdi)
+; AVX512F-NEXT:  .LBB15_60: # %else58
+; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512F-NEXT:    vpcmpeqb %ymm1, %ymm2, %ymm1
+; AVX512F-NEXT:    vextracti128 $1, %ymm1, %xmm1
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm2
+; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm2
+; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
+; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
+; AVX512F-NEXT:    kshiftrw $14, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB15_62
+; AVX512F-NEXT:  # %bb.61: # %cond.store59
+; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm2
+; AVX512F-NEXT:    vpextrb $14, %xmm2, 30(%rdi)
+; AVX512F-NEXT:  .LBB15_62: # %else60
+; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm1
+; AVX512F-NEXT:    vpmovsxbd %xmm1, %zmm1
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $15, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB15_64
+; AVX512F-NEXT:  # %bb.63: # %cond.store61
+; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm0
+; AVX512F-NEXT:    vpextrb $15, %xmm0, 31(%rdi)
+; AVX512F-NEXT:  .LBB15_64: # %else62
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512BW-LABEL: truncstore_v32i16_v32i8:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512BW-NEXT:    vptestmb %zmm1, %zmm1, %k1
+; AVX512BW-NEXT:    vpmovwb %zmm0, (%rdi) {%k1}
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+;
+; AVX512BWVL-LABEL: truncstore_v32i16_v32i8:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vptestmb %ymm1, %ymm1, %k1
+; AVX512BWVL-NEXT:    vpmovwb %zmm0, (%rdi) {%k1}
+; AVX512BWVL-NEXT:    vzeroupper
+; AVX512BWVL-NEXT:    retq
+  %a = icmp ne <32 x i8> %mask, zeroinitializer
+  %b = trunc <32 x i16> %x to <32 x i8>
+  call void @llvm.masked.store.v32i8.p0v32i8(<32 x i8> %b, <32 x i8>* %p, i32 1, <32 x i1> %a)
+  ret void
+}
+
+define void @truncstore_v16i16_v16i8(<16 x i16> %x, <16 x i8>* %p, <16 x i8> %mask) {
+; AVX512F-LABEL: truncstore_v16i16_v16i8:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
+; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3
+; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm3
+; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
+; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
+; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
+; AVX512F-NEXT:    vpmovdb %zmm0, %xmm0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB16_2
+; AVX512F-NEXT:  # %bb.1: # %cond.store
+; AVX512F-NEXT:    vpextrb $0, %xmm0, (%rdi)
+; AVX512F-NEXT:  .LBB16_2: # %else
+; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm2
+; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
+; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
+; AVX512F-NEXT:    kshiftrw $1, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB16_4
+; AVX512F-NEXT:  # %bb.3: # %cond.store1
+; AVX512F-NEXT:    vpextrb $1, %xmm0, 1(%rdi)
+; AVX512F-NEXT:  .LBB16_4: # %else2
+; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
+; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3
+; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm3
+; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
+; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
+; AVX512F-NEXT:    kshiftrw $2, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB16_6
+; AVX512F-NEXT:  # %bb.5: # %cond.store3
+; AVX512F-NEXT:    vpextrb $2, %xmm0, 2(%rdi)
+; AVX512F-NEXT:  .LBB16_6: # %else4
+; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm2
+; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
+; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
+; AVX512F-NEXT:    kshiftrw $3, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB16_8
+; AVX512F-NEXT:  # %bb.7: # %cond.store5
+; AVX512F-NEXT:    vpextrb $3, %xmm0, 3(%rdi)
+; AVX512F-NEXT:  .LBB16_8: # %else6
+; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
+; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3
+; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm3
+; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
+; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
+; AVX512F-NEXT:    kshiftrw $4, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB16_10
+; AVX512F-NEXT:  # %bb.9: # %cond.store7
+; AVX512F-NEXT:    vpextrb $4, %xmm0, 4(%rdi)
+; AVX512F-NEXT:  .LBB16_10: # %else8
+; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm2
+; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
+; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
+; AVX512F-NEXT:    kshiftrw $5, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB16_12
+; AVX512F-NEXT:  # %bb.11: # %cond.store9
+; AVX512F-NEXT:    vpextrb $5, %xmm0, 5(%rdi)
+; AVX512F-NEXT:  .LBB16_12: # %else10
+; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
+; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3
+; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm3
+; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
+; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
+; AVX512F-NEXT:    kshiftrw $6, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB16_14
+; AVX512F-NEXT:  # %bb.13: # %cond.store11
+; AVX512F-NEXT:    vpextrb $6, %xmm0, 6(%rdi)
+; AVX512F-NEXT:  .LBB16_14: # %else12
+; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm2
+; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
+; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
+; AVX512F-NEXT:    kshiftrw $7, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB16_16
+; AVX512F-NEXT:  # %bb.15: # %cond.store13
+; AVX512F-NEXT:    vpextrb $7, %xmm0, 7(%rdi)
+; AVX512F-NEXT:  .LBB16_16: # %else14
+; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
+; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3
+; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm3
+; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
+; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
+; AVX512F-NEXT:    kshiftrw $8, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB16_18
+; AVX512F-NEXT:  # %bb.17: # %cond.store15
+; AVX512F-NEXT:    vpextrb $8, %xmm0, 8(%rdi)
+; AVX512F-NEXT:  .LBB16_18: # %else16
+; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm2
+; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
+; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
+; AVX512F-NEXT:    kshiftrw $9, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB16_20
+; AVX512F-NEXT:  # %bb.19: # %cond.store17
+; AVX512F-NEXT:    vpextrb $9, %xmm0, 9(%rdi)
+; AVX512F-NEXT:  .LBB16_20: # %else18
+; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
+; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3
+; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm3
+; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
+; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
+; AVX512F-NEXT:    kshiftrw $10, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB16_22
+; AVX512F-NEXT:  # %bb.21: # %cond.store19
+; AVX512F-NEXT:    vpextrb $10, %xmm0, 10(%rdi)
+; AVX512F-NEXT:  .LBB16_22: # %else20
+; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm2
+; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
+; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
+; AVX512F-NEXT:    kshiftrw $11, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB16_24
+; AVX512F-NEXT:  # %bb.23: # %cond.store21
+; AVX512F-NEXT:    vpextrb $11, %xmm0, 11(%rdi)
+; AVX512F-NEXT:  .LBB16_24: # %else22
+; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
+; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3
+; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm3
+; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
+; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
+; AVX512F-NEXT:    kshiftrw $12, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB16_26
+; AVX512F-NEXT:  # %bb.25: # %cond.store23
+; AVX512F-NEXT:    vpextrb $12, %xmm0, 12(%rdi)
+; AVX512F-NEXT:  .LBB16_26: # %else24
+; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm2
+; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
+; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
+; AVX512F-NEXT:    kshiftrw $13, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB16_28
+; AVX512F-NEXT:  # %bb.27: # %cond.store25
+; AVX512F-NEXT:    vpextrb $13, %xmm0, 13(%rdi)
+; AVX512F-NEXT:  .LBB16_28: # %else26
+; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm1
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm2
+; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm2
+; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
+; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
+; AVX512F-NEXT:    kshiftrw $14, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB16_30
+; AVX512F-NEXT:  # %bb.29: # %cond.store27
+; AVX512F-NEXT:    vpextrb $14, %xmm0, 14(%rdi)
+; AVX512F-NEXT:  .LBB16_30: # %else28
+; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm1
+; AVX512F-NEXT:    vpmovsxbd %xmm1, %zmm1
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $15, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB16_32
+; AVX512F-NEXT:  # %bb.31: # %cond.store29
+; AVX512F-NEXT:    vpextrb $15, %xmm0, 15(%rdi)
+; AVX512F-NEXT:  .LBB16_32: # %else30
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512BW-LABEL: truncstore_v16i16_v16i8:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512BW-NEXT:    vptestmb %zmm1, %zmm1, %k0
+; AVX512BW-NEXT:    kmovw %k0, %k1
+; AVX512BW-NEXT:    vpmovwb %zmm0, %ymm0
+; AVX512BW-NEXT:    vmovdqu8 %zmm0, (%rdi) {%k1}
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+;
+; AVX512BWVL-LABEL: truncstore_v16i16_v16i8:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vptestmb %xmm1, %xmm1, %k1
+; AVX512BWVL-NEXT:    vpmovwb %ymm0, (%rdi) {%k1}
+; AVX512BWVL-NEXT:    vzeroupper
+; AVX512BWVL-NEXT:    retq
+  %a = icmp ne <16 x i8> %mask, zeroinitializer
+  %b = trunc <16 x i16> %x to <16 x i8>
+  call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %b, <16 x i8>* %p, i32 1, <16 x i1> %a)
+  ret void
+}
+
+define void @truncstore_v8i16_v8i8(<8 x i16> %x, <8 x i8>* %p, <8 x i16> %mask) {
+; AVX512F-LABEL: truncstore_v8i16_v8i8:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX512F-NEXT:    vpcmpeqw %xmm2, %xmm1, %xmm2
+; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3
+; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm3
+; AVX512F-NEXT:    vpmovsxwq %xmm3, %zmm3
+; AVX512F-NEXT:    vptestmq %zmm3, %zmm3, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB17_2
+; AVX512F-NEXT:  # %bb.1: # %cond.store
+; AVX512F-NEXT:    vpextrb $0, %xmm0, (%rdi)
+; AVX512F-NEXT:  .LBB17_2: # %else
+; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm2
+; AVX512F-NEXT:    vpmovsxwq %xmm2, %zmm2
+; AVX512F-NEXT:    vptestmq %zmm2, %zmm2, %k0
+; AVX512F-NEXT:    kshiftrw $1, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB17_4
+; AVX512F-NEXT:  # %bb.3: # %cond.store1
+; AVX512F-NEXT:    vpextrb $2, %xmm0, 1(%rdi)
+; AVX512F-NEXT:  .LBB17_4: # %else2
+; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX512F-NEXT:    vpcmpeqw %xmm2, %xmm1, %xmm2
+; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3
+; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm3
+; AVX512F-NEXT:    vpmovsxwq %xmm3, %zmm3
+; AVX512F-NEXT:    vptestmq %zmm3, %zmm3, %k0
+; AVX512F-NEXT:    kshiftrw $2, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB17_6
+; AVX512F-NEXT:  # %bb.5: # %cond.store3
+; AVX512F-NEXT:    vpextrb $4, %xmm0, 2(%rdi)
+; AVX512F-NEXT:  .LBB17_6: # %else4
+; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm2
+; AVX512F-NEXT:    vpmovsxwq %xmm2, %zmm2
+; AVX512F-NEXT:    vptestmq %zmm2, %zmm2, %k0
+; AVX512F-NEXT:    kshiftrw $3, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB17_8
+; AVX512F-NEXT:  # %bb.7: # %cond.store5
+; AVX512F-NEXT:    vpextrb $6, %xmm0, 3(%rdi)
+; AVX512F-NEXT:  .LBB17_8: # %else6
+; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX512F-NEXT:    vpcmpeqw %xmm2, %xmm1, %xmm2
+; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3
+; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm3
+; AVX512F-NEXT:    vpmovsxwq %xmm3, %zmm3
+; AVX512F-NEXT:    vptestmq %zmm3, %zmm3, %k0
+; AVX512F-NEXT:    kshiftrw $4, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB17_10
+; AVX512F-NEXT:  # %bb.9: # %cond.store7
+; AVX512F-NEXT:    vpextrb $8, %xmm0, 4(%rdi)
+; AVX512F-NEXT:  .LBB17_10: # %else8
+; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm2
+; AVX512F-NEXT:    vpmovsxwq %xmm2, %zmm2
+; AVX512F-NEXT:    vptestmq %zmm2, %zmm2, %k0
+; AVX512F-NEXT:    kshiftrw $5, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB17_12
+; AVX512F-NEXT:  # %bb.11: # %cond.store9
+; AVX512F-NEXT:    vpextrb $10, %xmm0, 5(%rdi)
+; AVX512F-NEXT:  .LBB17_12: # %else10
+; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX512F-NEXT:    vpcmpeqw %xmm2, %xmm1, %xmm1
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm2
+; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm2
+; AVX512F-NEXT:    vpmovsxwq %xmm2, %zmm2
+; AVX512F-NEXT:    vptestmq %zmm2, %zmm2, %k0
+; AVX512F-NEXT:    kshiftrw $6, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB17_14
+; AVX512F-NEXT:  # %bb.13: # %cond.store11
+; AVX512F-NEXT:    vpextrb $12, %xmm0, 6(%rdi)
+; AVX512F-NEXT:  .LBB17_14: # %else12
+; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm1
+; AVX512F-NEXT:    vpmovsxwq %xmm1, %zmm1
+; AVX512F-NEXT:    vptestmq %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $7, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB17_16
+; AVX512F-NEXT:  # %bb.15: # %cond.store13
+; AVX512F-NEXT:    vpextrb $14, %xmm0, 7(%rdi)
+; AVX512F-NEXT:  .LBB17_16: # %else14
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512BW-LABEL: truncstore_v8i16_v8i8:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512BW-NEXT:    vptestmw %zmm1, %zmm1, %k0
+; AVX512BW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,2,4,6,8,10,12,14,u,u,u,u,u,u,u,u]
+; AVX512BW-NEXT:    kshiftlq $56, %k0, %k0
+; AVX512BW-NEXT:    kshiftrq $56, %k0, %k1
+; AVX512BW-NEXT:    vmovdqu8 %zmm0, (%rdi) {%k1}
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+;
+; AVX512BWVL-LABEL: truncstore_v8i16_v8i8:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vptestmw %xmm1, %xmm1, %k1
+; AVX512BWVL-NEXT:    vpmovwb %xmm0, (%rdi) {%k1}
+; AVX512BWVL-NEXT:    retq
+  %a = icmp ne <8 x i16> %mask, zeroinitializer
+  %b = trunc <8 x i16> %x to <8 x i8>
+  call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> %b, <8 x i8>* %p, i32 1, <8 x i1> %a)
+  ret void
+}
+
+declare void @llvm.masked.store.v8i32.p0v8i32(<8 x i32>, <8 x i32>*, i32, <8 x i1>)
+declare void @llvm.masked.store.v8i16.p0v8i16(<8 x i16>, <8 x i16>*, i32, <8 x i1>)
+declare void @llvm.masked.store.v8i8.p0v8i8(<8 x i8>, <8 x i8>*, i32, <8 x i1>)
+declare void @llvm.masked.store.v4i32.p0v4i32(<4 x i32>, <4 x i32>*, i32, <4 x i1>)
+declare void @llvm.masked.store.v4i16.p0v4i16(<4 x i16>, <4 x i16>*, i32, <4 x i1>)
+declare void @llvm.masked.store.v4i8.p0v4i8(<4 x i8>, <4 x i8>*, i32, <4 x i1>)
+declare void @llvm.masked.store.v2i32.p0v2i32(<2 x i32>, <2 x i32>*, i32, <2 x i1>)
+declare void @llvm.masked.store.v2i16.p0v2i16(<2 x i16>, <2 x i16>*, i32, <2 x i1>)
+declare void @llvm.masked.store.v2i8.p0v2i8(<2 x i8>, <2 x i8>*, i32, <2 x i1>)
+declare void @llvm.masked.store.v16i16.p0v16i16(<16 x i16>, <16 x i16>*, i32, <16 x i1>)
+declare void @llvm.masked.store.v16i8.p0v16i8(<16 x i8>, <16 x i8>*, i32, <16 x i1>)
+declare void @llvm.masked.store.v32i8.p0v32i8(<32 x i8>, <32 x i8>*, i32, <32 x i1>)
diff --git a/test/CodeGen/X86/masked_store_trunc_ssat.ll b/test/CodeGen/X86/masked_store_trunc_ssat.ll
new file mode 100644
index 0000000000000000000000000000000000000000..2d2563221b0fb29ce2f3b69de483d329bcb07acb
--- /dev/null
+++ b/test/CodeGen/X86/masked_store_trunc_ssat.ll
@@ -0,0 +1,2089 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx512f | FileCheck %s --check-prefixes=AVX512,AVX512F
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx512vl | FileCheck %s --check-prefixes=AVX512,AVX512VL
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx512bw | FileCheck %s --check-prefixes=AVX512,AVX512BW,AVX512BWNOVL
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx512vl,avx512bw | FileCheck %s --check-prefixes=AVX512,AVX512VL,AVX512BWVL
+
+define void @truncstore_v8i64_v8i32(<8 x i64> %x, <8 x i32>* %p, <8 x i32> %mask) {
+; AVX512F-LABEL: truncstore_v8i64_v8i32:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k1
+; AVX512F-NEXT:    vpminsq {{.*}}(%rip){1to8}, %zmm0, %zmm0
+; AVX512F-NEXT:    vpmaxsq {{.*}}(%rip){1to8}, %zmm0, %zmm0
+; AVX512F-NEXT:    vpmovqd %zmm0, (%rdi) {%k1}
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: truncstore_v8i64_v8i32:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vptestmd %ymm1, %ymm1, %k1
+; AVX512VL-NEXT:    vpminsq {{.*}}(%rip){1to8}, %zmm0, %zmm0
+; AVX512VL-NEXT:    vpmaxsq {{.*}}(%rip){1to8}, %zmm0, %zmm0
+; AVX512VL-NEXT:    vpmovqd %zmm0, (%rdi) {%k1}
+; AVX512VL-NEXT:    vzeroupper
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: truncstore_v8i64_v8i32:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512BW-NEXT:    vptestmd %zmm1, %zmm1, %k1
+; AVX512BW-NEXT:    vpminsq {{.*}}(%rip){1to8}, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpmaxsq {{.*}}(%rip){1to8}, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpmovqd %zmm0, (%rdi) {%k1}
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+  %a = icmp ne <8 x i32> %mask, zeroinitializer
+  %b = icmp slt <8 x i64> %x, <i64 2147483647, i64 2147483647, i64 2147483647, i64 2147483647, i64 2147483647, i64 2147483647, i64 2147483647, i64 2147483647>
+  %c = select <8 x i1> %b, <8 x i64> %x, <8 x i64> <i64 2147483647, i64 2147483647, i64 2147483647, i64 2147483647, i64 2147483647, i64 2147483647, i64 2147483647, i64 2147483647>
+  %d = icmp sgt <8 x i64> %c, <i64 -2147483648, i64 -2147483648, i64 -2147483648, i64 -2147483648, i64 -2147483648, i64 -2147483648, i64 -2147483648, i64 -2147483648>
+  %e = select <8 x i1> %d, <8 x i64> %c, <8 x i64> <i64 -2147483648, i64 -2147483648, i64 -2147483648, i64 -2147483648, i64 -2147483648, i64 -2147483648, i64 -2147483648, i64 -2147483648>
+  %f = trunc <8 x i64> %e to <8 x i32>
+  call void @llvm.masked.store.v8i32.p0v8i32(<8 x i32> %f, <8 x i32>* %p, i32 1, <8 x i1> %a)
+  ret void
+}
+
+define void @truncstore_v8i64_v8i16(<8 x i64> %x, <8 x i16>* %p, <8 x i32> %mask) {
+; AVX512F-LABEL: truncstore_v8i64_v8i16:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    vpmovsqw %zmm0, %xmm0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB1_2
+; AVX512F-NEXT:  # %bb.1: # %cond.store
+; AVX512F-NEXT:    vpextrw $0, %xmm0, (%rdi)
+; AVX512F-NEXT:  .LBB1_2: # %else
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $1, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB1_4
+; AVX512F-NEXT:  # %bb.3: # %cond.store1
+; AVX512F-NEXT:    vpextrw $1, %xmm0, 2(%rdi)
+; AVX512F-NEXT:  .LBB1_4: # %else2
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $2, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB1_6
+; AVX512F-NEXT:  # %bb.5: # %cond.store3
+; AVX512F-NEXT:    vpextrw $2, %xmm0, 4(%rdi)
+; AVX512F-NEXT:  .LBB1_6: # %else4
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $3, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB1_8
+; AVX512F-NEXT:  # %bb.7: # %cond.store5
+; AVX512F-NEXT:    vpextrw $3, %xmm0, 6(%rdi)
+; AVX512F-NEXT:  .LBB1_8: # %else6
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $4, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB1_10
+; AVX512F-NEXT:  # %bb.9: # %cond.store7
+; AVX512F-NEXT:    vpextrw $4, %xmm0, 8(%rdi)
+; AVX512F-NEXT:  .LBB1_10: # %else8
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $5, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB1_12
+; AVX512F-NEXT:  # %bb.11: # %cond.store9
+; AVX512F-NEXT:    vpextrw $5, %xmm0, 10(%rdi)
+; AVX512F-NEXT:  .LBB1_12: # %else10
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $6, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB1_14
+; AVX512F-NEXT:  # %bb.13: # %cond.store11
+; AVX512F-NEXT:    vpextrw $6, %xmm0, 12(%rdi)
+; AVX512F-NEXT:  .LBB1_14: # %else12
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $7, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB1_16
+; AVX512F-NEXT:  # %bb.15: # %cond.store13
+; AVX512F-NEXT:    vpextrw $7, %xmm0, 14(%rdi)
+; AVX512F-NEXT:  .LBB1_16: # %else14
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512BW-LABEL: truncstore_v8i64_v8i16:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512BW-NEXT:    vptestmd %zmm1, %zmm1, %k1
+; AVX512BW-NEXT:    vpminsq {{.*}}(%rip){1to8}, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpmaxsq {{.*}}(%rip){1to8}, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpmovqw %zmm0, (%rdi) {%k1}
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+;
+; AVX512BWVL-LABEL: truncstore_v8i64_v8i16:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vptestmd %ymm1, %ymm1, %k1
+; AVX512BWVL-NEXT:    vpminsq {{.*}}(%rip){1to8}, %zmm0, %zmm0
+; AVX512BWVL-NEXT:    vpmaxsq {{.*}}(%rip){1to8}, %zmm0, %zmm0
+; AVX512BWVL-NEXT:    vpmovqw %zmm0, (%rdi) {%k1}
+; AVX512BWVL-NEXT:    vzeroupper
+; AVX512BWVL-NEXT:    retq
+  %a = icmp ne <8 x i32> %mask, zeroinitializer
+  %b = icmp slt <8 x i64> %x, <i64 32767, i64 32767, i64 32767, i64 32767, i64 32767, i64 32767, i64 32767, i64 32767>
+  %c = select <8 x i1> %b, <8 x i64> %x, <8 x i64> <i64 32767, i64 32767, i64 32767, i64 32767, i64 32767, i64 32767, i64 32767, i64 32767>
+  %d = icmp sgt <8 x i64> %c, <i64 -32768, i64 -32768, i64 -32768, i64 -32768, i64 -32768, i64 -32768, i64 -32768, i64 -32768>
+  %e = select <8 x i1> %d, <8 x i64> %c, <8 x i64> <i64 -32768, i64 -32768, i64 -32768, i64 -32768, i64 -32768, i64 -32768, i64 -32768, i64 -32768>
+  %f = trunc <8 x i64> %e to <8 x i16>
+  call void @llvm.masked.store.v8i16.p0v8i16(<8 x i16> %f, <8 x i16>* %p, i32 1, <8 x i1> %a)
+  ret void
+}
+
+define void @truncstore_v8i64_v8i8(<8 x i64> %x, <8 x i8>* %p, <8 x i32> %mask) {
+; AVX512F-LABEL: truncstore_v8i64_v8i8:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    vpminsq {{.*}}(%rip){1to8}, %zmm0, %zmm0
+; AVX512F-NEXT:    vpmaxsq {{.*}}(%rip){1to8}, %zmm0, %zmm0
+; AVX512F-NEXT:    vpmovqw %zmm0, %xmm0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB2_2
+; AVX512F-NEXT:  # %bb.1: # %cond.store
+; AVX512F-NEXT:    vpextrb $0, %xmm0, (%rdi)
+; AVX512F-NEXT:  .LBB2_2: # %else
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $1, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB2_4
+; AVX512F-NEXT:  # %bb.3: # %cond.store1
+; AVX512F-NEXT:    vpextrb $2, %xmm0, 1(%rdi)
+; AVX512F-NEXT:  .LBB2_4: # %else2
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $2, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB2_6
+; AVX512F-NEXT:  # %bb.5: # %cond.store3
+; AVX512F-NEXT:    vpextrb $4, %xmm0, 2(%rdi)
+; AVX512F-NEXT:  .LBB2_6: # %else4
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $3, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB2_8
+; AVX512F-NEXT:  # %bb.7: # %cond.store5
+; AVX512F-NEXT:    vpextrb $6, %xmm0, 3(%rdi)
+; AVX512F-NEXT:  .LBB2_8: # %else6
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $4, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB2_10
+; AVX512F-NEXT:  # %bb.9: # %cond.store7
+; AVX512F-NEXT:    vpextrb $8, %xmm0, 4(%rdi)
+; AVX512F-NEXT:  .LBB2_10: # %else8
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $5, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB2_12
+; AVX512F-NEXT:  # %bb.11: # %cond.store9
+; AVX512F-NEXT:    vpextrb $10, %xmm0, 5(%rdi)
+; AVX512F-NEXT:  .LBB2_12: # %else10
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $6, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB2_14
+; AVX512F-NEXT:  # %bb.13: # %cond.store11
+; AVX512F-NEXT:    vpextrb $12, %xmm0, 6(%rdi)
+; AVX512F-NEXT:  .LBB2_14: # %else12
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $7, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB2_16
+; AVX512F-NEXT:  # %bb.15: # %cond.store13
+; AVX512F-NEXT:    vpextrb $14, %xmm0, 7(%rdi)
+; AVX512F-NEXT:  .LBB2_16: # %else14
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512BW-LABEL: truncstore_v8i64_v8i8:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512BW-NEXT:    vptestmd %zmm1, %zmm1, %k1
+; AVX512BW-NEXT:    vpminsq {{.*}}(%rip){1to8}, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpmaxsq {{.*}}(%rip){1to8}, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpmovqb %zmm0, (%rdi) {%k1}
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+;
+; AVX512BWVL-LABEL: truncstore_v8i64_v8i8:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vptestmd %ymm1, %ymm1, %k1
+; AVX512BWVL-NEXT:    vpminsq {{.*}}(%rip){1to8}, %zmm0, %zmm0
+; AVX512BWVL-NEXT:    vpmaxsq {{.*}}(%rip){1to8}, %zmm0, %zmm0
+; AVX512BWVL-NEXT:    vpmovqb %zmm0, (%rdi) {%k1}
+; AVX512BWVL-NEXT:    vzeroupper
+; AVX512BWVL-NEXT:    retq
+  %a = icmp ne <8 x i32> %mask, zeroinitializer
+  %b = icmp slt <8 x i64> %x, <i64 127, i64 127, i64 127, i64 127, i64 127, i64 127, i64 127, i64 127>
+  %c = select <8 x i1> %b, <8 x i64> %x, <8 x i64> <i64 127, i64 127, i64 127, i64 127, i64 127, i64 127, i64 127, i64 127>
+  %d = icmp sgt <8 x i64> %c, <i64 -128, i64 -128, i64 -128, i64 -128, i64 -128, i64 -128, i64 -128, i64 -128>
+  %e = select <8 x i1> %d, <8 x i64> %c, <8 x i64> <i64 -128, i64 -128, i64 -128, i64 -128, i64 -128, i64 -128, i64 -128, i64 -128>
+  %f = trunc <8 x i64> %e to <8 x i8>
+  call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> %f, <8 x i8>* %p, i32 1, <8 x i1> %a)
+  ret void
+}
+
+define void @truncstore_v4i64_v4i32(<4 x i64> %x, <4 x i32>* %p, <4 x i32> %mask) {
+; AVX512F-LABEL: truncstore_v4i64_v4i32:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512F-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftlw $12, %k0, %k0
+; AVX512F-NEXT:    kshiftrw $12, %k0, %k1
+; AVX512F-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [2147483647,2147483647,2147483647,2147483647]
+; AVX512F-NEXT:    vpminsq %zmm1, %zmm0, %zmm0
+; AVX512F-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [18446744071562067968,18446744071562067968,18446744071562067968,18446744071562067968]
+; AVX512F-NEXT:    vpmaxsq %zmm1, %zmm0, %zmm0
+; AVX512F-NEXT:    vpmovqd %zmm0, %ymm0
+; AVX512F-NEXT:    vmovdqu32 %zmm0, (%rdi) {%k1}
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: truncstore_v4i64_v4i32:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vptestmd %xmm1, %xmm1, %k1
+; AVX512VL-NEXT:    vpminsq {{.*}}(%rip){1to4}, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpmaxsq {{.*}}(%rip){1to4}, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpmovqd %ymm0, (%rdi) {%k1}
+; AVX512VL-NEXT:    vzeroupper
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: truncstore_v4i64_v4i32:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512BW-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512BW-NEXT:    kshiftlw $12, %k0, %k0
+; AVX512BW-NEXT:    kshiftrw $12, %k0, %k1
+; AVX512BW-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [2147483647,2147483647,2147483647,2147483647]
+; AVX512BW-NEXT:    vpminsq %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [18446744071562067968,18446744071562067968,18446744071562067968,18446744071562067968]
+; AVX512BW-NEXT:    vpmaxsq %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpmovqd %zmm0, %ymm0
+; AVX512BW-NEXT:    vmovdqu32 %zmm0, (%rdi) {%k1}
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+  %a = icmp ne <4 x i32> %mask, zeroinitializer
+  %b = icmp slt <4 x i64> %x, <i64 2147483647, i64 2147483647, i64 2147483647, i64 2147483647>
+  %c = select <4 x i1> %b, <4 x i64> %x, <4 x i64> <i64 2147483647, i64 2147483647, i64 2147483647, i64 2147483647>
+  %d = icmp sgt <4 x i64> %c, <i64 -2147483648, i64 -2147483648, i64 -2147483648, i64 -2147483648>
+  %e = select <4 x i1> %d, <4 x i64> %c, <4 x i64> <i64 -2147483648, i64 -2147483648, i64 -2147483648, i64 -2147483648>
+  %f = trunc <4 x i64> %e to <4 x i32>
+  call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %f, <4 x i32>* %p, i32 1, <4 x i1> %a)
+  ret void
+}
+
+define void @truncstore_v4i64_v4i16(<4 x i64> %x, <4 x i16>* %p, <4 x i32> %mask) {
+; AVX512F-LABEL: truncstore_v4i64_v4i16:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512F-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [32767,32767,32767,32767]
+; AVX512F-NEXT:    vpminsq %zmm2, %zmm0, %zmm0
+; AVX512F-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [18446744073709518848,18446744073709518848,18446744073709518848,18446744073709518848]
+; AVX512F-NEXT:    vpmaxsq %zmm2, %zmm0, %zmm0
+; AVX512F-NEXT:    vpmovqd %zmm0, %ymm0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB4_2
+; AVX512F-NEXT:  # %bb.1: # %cond.store
+; AVX512F-NEXT:    vpextrw $0, %xmm0, (%rdi)
+; AVX512F-NEXT:  .LBB4_2: # %else
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $1, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB4_4
+; AVX512F-NEXT:  # %bb.3: # %cond.store1
+; AVX512F-NEXT:    vpextrw $2, %xmm0, 2(%rdi)
+; AVX512F-NEXT:  .LBB4_4: # %else2
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $2, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB4_6
+; AVX512F-NEXT:  # %bb.5: # %cond.store3
+; AVX512F-NEXT:    vpextrw $4, %xmm0, 4(%rdi)
+; AVX512F-NEXT:  .LBB4_6: # %else4
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $3, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB4_8
+; AVX512F-NEXT:  # %bb.7: # %cond.store5
+; AVX512F-NEXT:    vpextrw $6, %xmm0, 6(%rdi)
+; AVX512F-NEXT:  .LBB4_8: # %else6
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512BW-LABEL: truncstore_v4i64_v4i16:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512BW-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512BW-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [32767,32767,32767,32767]
+; AVX512BW-NEXT:    vpminsq %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [18446744073709518848,18446744073709518848,18446744073709518848,18446744073709518848]
+; AVX512BW-NEXT:    vpmaxsq %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpmovqd %zmm0, %ymm0
+; AVX512BW-NEXT:    vpackssdw %xmm0, %xmm0, %xmm0
+; AVX512BW-NEXT:    kshiftld $28, %k0, %k0
+; AVX512BW-NEXT:    kshiftrd $28, %k0, %k1
+; AVX512BW-NEXT:    vmovdqu16 %zmm0, (%rdi) {%k1}
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+;
+; AVX512BWVL-LABEL: truncstore_v4i64_v4i16:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vptestmd %xmm1, %xmm1, %k1
+; AVX512BWVL-NEXT:    vpminsq {{.*}}(%rip){1to4}, %ymm0, %ymm0
+; AVX512BWVL-NEXT:    vpmaxsq {{.*}}(%rip){1to4}, %ymm0, %ymm0
+; AVX512BWVL-NEXT:    vpmovqw %ymm0, (%rdi) {%k1}
+; AVX512BWVL-NEXT:    vzeroupper
+; AVX512BWVL-NEXT:    retq
+  %a = icmp ne <4 x i32> %mask, zeroinitializer
+  %b = icmp slt <4 x i64> %x, <i64 32767, i64 32767, i64 32767, i64 32767>
+  %c = select <4 x i1> %b, <4 x i64> %x, <4 x i64> <i64 32767, i64 32767, i64 32767, i64 32767>
+  %d = icmp sgt <4 x i64> %c, <i64 -32768, i64 -32768, i64 -32768, i64 -32768>
+  %e = select <4 x i1> %d, <4 x i64> %c, <4 x i64> <i64 -32768, i64 -32768, i64 -32768, i64 -32768>
+  %f = trunc <4 x i64> %e to <4 x i16>
+  call void @llvm.masked.store.v4i16.p0v4i16(<4 x i16> %f, <4 x i16>* %p, i32 1, <4 x i1> %a)
+  ret void
+}
+
+define void @truncstore_v4i64_v4i8(<4 x i64> %x, <4 x i8>* %p, <4 x i32> %mask) {
+; AVX512F-LABEL: truncstore_v4i64_v4i8:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512F-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [127,127,127,127]
+; AVX512F-NEXT:    vpminsq %zmm2, %zmm0, %zmm0
+; AVX512F-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [18446744073709551488,18446744073709551488,18446744073709551488,18446744073709551488]
+; AVX512F-NEXT:    vpmaxsq %zmm2, %zmm0, %zmm0
+; AVX512F-NEXT:    vpmovqd %zmm0, %ymm0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB5_2
+; AVX512F-NEXT:  # %bb.1: # %cond.store
+; AVX512F-NEXT:    vpextrb $0, %xmm0, (%rdi)
+; AVX512F-NEXT:  .LBB5_2: # %else
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $1, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB5_4
+; AVX512F-NEXT:  # %bb.3: # %cond.store1
+; AVX512F-NEXT:    vpextrb $4, %xmm0, 1(%rdi)
+; AVX512F-NEXT:  .LBB5_4: # %else2
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $2, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB5_6
+; AVX512F-NEXT:  # %bb.5: # %cond.store3
+; AVX512F-NEXT:    vpextrb $8, %xmm0, 2(%rdi)
+; AVX512F-NEXT:  .LBB5_6: # %else4
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $3, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB5_8
+; AVX512F-NEXT:  # %bb.7: # %cond.store5
+; AVX512F-NEXT:    vpextrb $12, %xmm0, 3(%rdi)
+; AVX512F-NEXT:  .LBB5_8: # %else6
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512BW-LABEL: truncstore_v4i64_v4i8:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512BW-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512BW-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [127,127,127,127]
+; AVX512BW-NEXT:    vpminsq %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [18446744073709551488,18446744073709551488,18446744073709551488,18446744073709551488]
+; AVX512BW-NEXT:    vpmaxsq %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpmovqd %zmm0, %ymm0
+; AVX512BW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512BW-NEXT:    kshiftlq $60, %k0, %k0
+; AVX512BW-NEXT:    kshiftrq $60, %k0, %k1
+; AVX512BW-NEXT:    vmovdqu8 %zmm0, (%rdi) {%k1}
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+;
+; AVX512BWVL-LABEL: truncstore_v4i64_v4i8:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vptestmd %xmm1, %xmm1, %k1
+; AVX512BWVL-NEXT:    vpminsq {{.*}}(%rip){1to4}, %ymm0, %ymm0
+; AVX512BWVL-NEXT:    vpmaxsq {{.*}}(%rip){1to4}, %ymm0, %ymm0
+; AVX512BWVL-NEXT:    vpmovqb %ymm0, (%rdi) {%k1}
+; AVX512BWVL-NEXT:    vzeroupper
+; AVX512BWVL-NEXT:    retq
+  %a = icmp ne <4 x i32> %mask, zeroinitializer
+  %b = icmp slt <4 x i64> %x, <i64 127, i64 127, i64 127, i64 127>
+  %c = select <4 x i1> %b, <4 x i64> %x, <4 x i64> <i64 127, i64 127, i64 127, i64 127>
+  %d = icmp sgt <4 x i64> %c, <i64 -128, i64 -128, i64 -128, i64 -128>
+  %e = select <4 x i1> %d, <4 x i64> %c, <4 x i64> <i64 -128, i64 -128, i64 -128, i64 -128>
+  %f = trunc <4 x i64> %e to <4 x i8>
+  call void @llvm.masked.store.v4i8.p0v4i8(<4 x i8> %f, <4 x i8>* %p, i32 1, <4 x i1> %a)
+  ret void
+}
+
+define void @truncstore_v2i64_v2i32(<2 x i64> %x, <2 x i32>* %p, <2 x i64> %mask) {
+; AVX512F-LABEL: truncstore_v2i64_v2i32:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512F-NEXT:    vptestmq %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm1 = [2147483647,2147483647]
+; AVX512F-NEXT:    vpminsq %zmm1, %zmm0, %zmm0
+; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm1 = [18446744071562067968,18446744071562067968]
+; AVX512F-NEXT:    vpmaxsq %zmm1, %zmm0, %zmm0
+; AVX512F-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; AVX512F-NEXT:    kshiftlw $14, %k0, %k0
+; AVX512F-NEXT:    kshiftrw $14, %k0, %k1
+; AVX512F-NEXT:    vmovdqu32 %zmm0, (%rdi) {%k1}
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: truncstore_v2i64_v2i32:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vptestmq %xmm1, %xmm1, %k1
+; AVX512VL-NEXT:    vpminsq {{.*}}(%rip), %xmm0, %xmm0
+; AVX512VL-NEXT:    vpmaxsq {{.*}}(%rip), %xmm0, %xmm0
+; AVX512VL-NEXT:    vpmovqd %xmm0, (%rdi) {%k1}
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: truncstore_v2i64_v2i32:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512BW-NEXT:    vptestmq %zmm1, %zmm1, %k0
+; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm1 = [2147483647,2147483647]
+; AVX512BW-NEXT:    vpminsq %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm1 = [18446744071562067968,18446744071562067968]
+; AVX512BW-NEXT:    vpmaxsq %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; AVX512BW-NEXT:    kshiftlw $14, %k0, %k0
+; AVX512BW-NEXT:    kshiftrw $14, %k0, %k1
+; AVX512BW-NEXT:    vmovdqu32 %zmm0, (%rdi) {%k1}
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+  %a = icmp ne <2 x i64> %mask, zeroinitializer
+  %b = icmp slt <2 x i64> %x, <i64 2147483647, i64 2147483647>
+  %c = select <2 x i1> %b, <2 x i64> %x, <2 x i64> <i64 2147483647, i64 2147483647>
+  %d = icmp sgt <2 x i64> %c, <i64 -2147483648, i64 -2147483648>
+  %e = select <2 x i1> %d, <2 x i64> %c, <2 x i64> <i64 -2147483648, i64 -2147483648>
+  %f = trunc <2 x i64> %e to <2 x i32>
+  call void @llvm.masked.store.v2i32.p0v2i32(<2 x i32> %f, <2 x i32>* %p, i32 1, <2 x i1> %a)
+  ret void
+}
+
+define void @truncstore_v2i64_v2i16(<2 x i64> %x, <2 x i16>* %p, <2 x i64> %mask) {
+; AVX512F-LABEL: truncstore_v2i64_v2i16:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512F-NEXT:    vptestmq %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm2 = [32767,32767]
+; AVX512F-NEXT:    vpminsq %zmm2, %zmm0, %zmm0
+; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm2 = [18446744073709518848,18446744073709518848]
+; AVX512F-NEXT:    vpmaxsq %zmm2, %zmm0, %zmm0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB7_2
+; AVX512F-NEXT:  # %bb.1: # %cond.store
+; AVX512F-NEXT:    vpextrw $0, %xmm0, (%rdi)
+; AVX512F-NEXT:  .LBB7_2: # %else
+; AVX512F-NEXT:    vptestmq %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $1, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB7_4
+; AVX512F-NEXT:  # %bb.3: # %cond.store1
+; AVX512F-NEXT:    vpextrw $4, %xmm0, 2(%rdi)
+; AVX512F-NEXT:  .LBB7_4: # %else2
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512BW-LABEL: truncstore_v2i64_v2i16:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512BW-NEXT:    vptestmq %zmm1, %zmm1, %k0
+; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm1 = [32767,32767]
+; AVX512BW-NEXT:    vpminsq %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm1 = [18446744073709518848,18446744073709518848]
+; AVX512BW-NEXT:    vpmaxsq %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; AVX512BW-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; AVX512BW-NEXT:    kshiftld $30, %k0, %k0
+; AVX512BW-NEXT:    kshiftrd $30, %k0, %k1
+; AVX512BW-NEXT:    vmovdqu16 %zmm0, (%rdi) {%k1}
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+;
+; AVX512BWVL-LABEL: truncstore_v2i64_v2i16:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vptestmq %xmm1, %xmm1, %k1
+; AVX512BWVL-NEXT:    vpminsq {{.*}}(%rip), %xmm0, %xmm0
+; AVX512BWVL-NEXT:    vpmaxsq {{.*}}(%rip), %xmm0, %xmm0
+; AVX512BWVL-NEXT:    vpmovqw %xmm0, (%rdi) {%k1}
+; AVX512BWVL-NEXT:    retq
+  %a = icmp ne <2 x i64> %mask, zeroinitializer
+  %b = icmp slt <2 x i64> %x, <i64 32767, i64 32767>
+  %c = select <2 x i1> %b, <2 x i64> %x, <2 x i64> <i64 32767, i64 32767>
+  %d = icmp sgt <2 x i64> %c, <i64 -32768, i64 -32768>
+  %e = select <2 x i1> %d, <2 x i64> %c, <2 x i64> <i64 -32768, i64 -32768>
+  %f = trunc <2 x i64> %e to <2 x i16>
+  call void @llvm.masked.store.v2i16.p0v2i16(<2 x i16> %f, <2 x i16>* %p, i32 1, <2 x i1> %a)
+  ret void
+}
+
+define void @truncstore_v2i64_v2i8(<2 x i64> %x, <2 x i8>* %p, <2 x i64> %mask) {
+; AVX512F-LABEL: truncstore_v2i64_v2i8:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512F-NEXT:    vptestmq %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm2 = [127,127]
+; AVX512F-NEXT:    vpminsq %zmm2, %zmm0, %zmm0
+; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm2 = [18446744073709551488,18446744073709551488]
+; AVX512F-NEXT:    vpmaxsq %zmm2, %zmm0, %zmm0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB8_2
+; AVX512F-NEXT:  # %bb.1: # %cond.store
+; AVX512F-NEXT:    vpextrb $0, %xmm0, (%rdi)
+; AVX512F-NEXT:  .LBB8_2: # %else
+; AVX512F-NEXT:    vptestmq %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $1, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB8_4
+; AVX512F-NEXT:  # %bb.3: # %cond.store1
+; AVX512F-NEXT:    vpextrb $8, %xmm0, 1(%rdi)
+; AVX512F-NEXT:  .LBB8_4: # %else2
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512BW-LABEL: truncstore_v2i64_v2i8:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512BW-NEXT:    vptestmq %zmm1, %zmm1, %k0
+; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm1 = [127,127]
+; AVX512BW-NEXT:    vpminsq %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm1 = [18446744073709551488,18446744073709551488]
+; AVX512BW-NEXT:    vpmaxsq %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,8,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512BW-NEXT:    kshiftlq $62, %k0, %k0
+; AVX512BW-NEXT:    kshiftrq $62, %k0, %k1
+; AVX512BW-NEXT:    vmovdqu8 %zmm0, (%rdi) {%k1}
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+;
+; AVX512BWVL-LABEL: truncstore_v2i64_v2i8:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vptestmq %xmm1, %xmm1, %k1
+; AVX512BWVL-NEXT:    vpminsq {{.*}}(%rip), %xmm0, %xmm0
+; AVX512BWVL-NEXT:    vpmaxsq {{.*}}(%rip), %xmm0, %xmm0
+; AVX512BWVL-NEXT:    vpmovqb %xmm0, (%rdi) {%k1}
+; AVX512BWVL-NEXT:    retq
+  %a = icmp ne <2 x i64> %mask, zeroinitializer
+  %b = icmp slt <2 x i64> %x, <i64 127, i64 127>
+  %c = select <2 x i1> %b, <2 x i64> %x, <2 x i64> <i64 127, i64 127>
+  %d = icmp sgt <2 x i64> %c, <i64 -128, i64 -128>
+  %e = select <2 x i1> %d, <2 x i64> %c, <2 x i64> <i64 -128, i64 -128>
+  %f = trunc <2 x i64> %e to <2 x i8>
+  call void @llvm.masked.store.v2i8.p0v2i8(<2 x i8> %f, <2 x i8>* %p, i32 1, <2 x i1> %a)
+  ret void
+}
+
+define void @truncstore_v16i32_v16i16(<16 x i32> %x, <16 x i16>* %p, <16 x i32> %mask) {
+; AVX512F-LABEL: truncstore_v16i32_v16i16:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    vpmovsdw %zmm0, %ymm0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB9_2
+; AVX512F-NEXT:  # %bb.1: # %cond.store
+; AVX512F-NEXT:    vpextrw $0, %xmm0, (%rdi)
+; AVX512F-NEXT:  .LBB9_2: # %else
+; AVX512F-NEXT:    kshiftrw $1, %k0, %k1
+; AVX512F-NEXT:    kmovw %k1, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB9_4
+; AVX512F-NEXT:  # %bb.3: # %cond.store1
+; AVX512F-NEXT:    vpextrw $1, %xmm0, 2(%rdi)
+; AVX512F-NEXT:  .LBB9_4: # %else2
+; AVX512F-NEXT:    kshiftrw $2, %k0, %k1
+; AVX512F-NEXT:    kmovw %k1, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB9_6
+; AVX512F-NEXT:  # %bb.5: # %cond.store3
+; AVX512F-NEXT:    vpextrw $2, %xmm0, 4(%rdi)
+; AVX512F-NEXT:  .LBB9_6: # %else4
+; AVX512F-NEXT:    kshiftrw $3, %k0, %k1
+; AVX512F-NEXT:    kmovw %k1, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB9_8
+; AVX512F-NEXT:  # %bb.7: # %cond.store5
+; AVX512F-NEXT:    vpextrw $3, %xmm0, 6(%rdi)
+; AVX512F-NEXT:  .LBB9_8: # %else6
+; AVX512F-NEXT:    kshiftrw $4, %k0, %k1
+; AVX512F-NEXT:    kmovw %k1, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB9_10
+; AVX512F-NEXT:  # %bb.9: # %cond.store7
+; AVX512F-NEXT:    vpextrw $4, %xmm0, 8(%rdi)
+; AVX512F-NEXT:  .LBB9_10: # %else8
+; AVX512F-NEXT:    kshiftrw $5, %k0, %k1
+; AVX512F-NEXT:    kmovw %k1, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB9_12
+; AVX512F-NEXT:  # %bb.11: # %cond.store9
+; AVX512F-NEXT:    vpextrw $5, %xmm0, 10(%rdi)
+; AVX512F-NEXT:  .LBB9_12: # %else10
+; AVX512F-NEXT:    kshiftrw $6, %k0, %k1
+; AVX512F-NEXT:    kmovw %k1, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB9_14
+; AVX512F-NEXT:  # %bb.13: # %cond.store11
+; AVX512F-NEXT:    vpextrw $6, %xmm0, 12(%rdi)
+; AVX512F-NEXT:  .LBB9_14: # %else12
+; AVX512F-NEXT:    kshiftrw $7, %k0, %k1
+; AVX512F-NEXT:    kmovw %k1, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB9_16
+; AVX512F-NEXT:  # %bb.15: # %cond.store13
+; AVX512F-NEXT:    vpextrw $7, %xmm0, 14(%rdi)
+; AVX512F-NEXT:  .LBB9_16: # %else14
+; AVX512F-NEXT:    kshiftrw $8, %k0, %k1
+; AVX512F-NEXT:    kmovw %k1, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB9_18
+; AVX512F-NEXT:  # %bb.17: # %cond.store15
+; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX512F-NEXT:    vpextrw $0, %xmm1, 16(%rdi)
+; AVX512F-NEXT:  .LBB9_18: # %else16
+; AVX512F-NEXT:    kshiftrw $9, %k0, %k1
+; AVX512F-NEXT:    kmovw %k1, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB9_20
+; AVX512F-NEXT:  # %bb.19: # %cond.store17
+; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX512F-NEXT:    vpextrw $1, %xmm1, 18(%rdi)
+; AVX512F-NEXT:  .LBB9_20: # %else18
+; AVX512F-NEXT:    kshiftrw $10, %k0, %k1
+; AVX512F-NEXT:    kmovw %k1, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB9_22
+; AVX512F-NEXT:  # %bb.21: # %cond.store19
+; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX512F-NEXT:    vpextrw $2, %xmm1, 20(%rdi)
+; AVX512F-NEXT:  .LBB9_22: # %else20
+; AVX512F-NEXT:    kshiftrw $11, %k0, %k1
+; AVX512F-NEXT:    kmovw %k1, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB9_24
+; AVX512F-NEXT:  # %bb.23: # %cond.store21
+; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX512F-NEXT:    vpextrw $3, %xmm1, 22(%rdi)
+; AVX512F-NEXT:  .LBB9_24: # %else22
+; AVX512F-NEXT:    kshiftrw $12, %k0, %k1
+; AVX512F-NEXT:    kmovw %k1, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB9_26
+; AVX512F-NEXT:  # %bb.25: # %cond.store23
+; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX512F-NEXT:    vpextrw $4, %xmm1, 24(%rdi)
+; AVX512F-NEXT:  .LBB9_26: # %else24
+; AVX512F-NEXT:    kshiftrw $13, %k0, %k1
+; AVX512F-NEXT:    kmovw %k1, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB9_28
+; AVX512F-NEXT:  # %bb.27: # %cond.store25
+; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX512F-NEXT:    vpextrw $5, %xmm1, 26(%rdi)
+; AVX512F-NEXT:  .LBB9_28: # %else26
+; AVX512F-NEXT:    kshiftrw $14, %k0, %k1
+; AVX512F-NEXT:    kmovw %k1, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB9_30
+; AVX512F-NEXT:  # %bb.29: # %cond.store27
+; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX512F-NEXT:    vpextrw $6, %xmm1, 28(%rdi)
+; AVX512F-NEXT:  .LBB9_30: # %else28
+; AVX512F-NEXT:    kshiftrw $15, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB9_32
+; AVX512F-NEXT:  # %bb.31: # %cond.store29
+; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm0
+; AVX512F-NEXT:    vpextrw $7, %xmm0, 30(%rdi)
+; AVX512F-NEXT:  .LBB9_32: # %else30
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512BW-LABEL: truncstore_v16i32_v16i16:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vptestmd %zmm1, %zmm1, %k1
+; AVX512BW-NEXT:    vpminsd {{.*}}(%rip){1to16}, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpmaxsd {{.*}}(%rip){1to16}, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpmovdw %zmm0, (%rdi) {%k1}
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+;
+; AVX512BWVL-LABEL: truncstore_v16i32_v16i16:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vptestmd %zmm1, %zmm1, %k1
+; AVX512BWVL-NEXT:    vpminsd {{.*}}(%rip){1to16}, %zmm0, %zmm0
+; AVX512BWVL-NEXT:    vpmaxsd {{.*}}(%rip){1to16}, %zmm0, %zmm0
+; AVX512BWVL-NEXT:    vpmovdw %zmm0, (%rdi) {%k1}
+; AVX512BWVL-NEXT:    vzeroupper
+; AVX512BWVL-NEXT:    retq
+  %a = icmp ne <16 x i32> %mask, zeroinitializer
+  %b = icmp slt <16 x i32> %x, <i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767>
+  %c = select <16 x i1> %b, <16 x i32> %x, <16 x i32> <i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767>
+  %d = icmp sgt <16 x i32> %c, <i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768>
+  %e = select <16 x i1> %d, <16 x i32> %c, <16 x i32> <i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768>
+  %f = trunc <16 x i32> %e to <16 x i16>
+  call void @llvm.masked.store.v16i16.p0v16i16(<16 x i16> %f, <16 x i16>* %p, i32 1, <16 x i1> %a)
+  ret void
+}
+
+define void @truncstore_v16i32_v16i8(<16 x i32> %x, <16 x i8>* %p, <16 x i32> %mask) {
+; AVX512F-LABEL: truncstore_v16i32_v16i8:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    vpmovsdb %zmm0, %xmm0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB10_2
+; AVX512F-NEXT:  # %bb.1: # %cond.store
+; AVX512F-NEXT:    vpextrb $0, %xmm0, (%rdi)
+; AVX512F-NEXT:  .LBB10_2: # %else
+; AVX512F-NEXT:    kshiftrw $1, %k0, %k1
+; AVX512F-NEXT:    kmovw %k1, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB10_4
+; AVX512F-NEXT:  # %bb.3: # %cond.store1
+; AVX512F-NEXT:    vpextrb $1, %xmm0, 1(%rdi)
+; AVX512F-NEXT:  .LBB10_4: # %else2
+; AVX512F-NEXT:    kshiftrw $2, %k0, %k1
+; AVX512F-NEXT:    kmovw %k1, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB10_6
+; AVX512F-NEXT:  # %bb.5: # %cond.store3
+; AVX512F-NEXT:    vpextrb $2, %xmm0, 2(%rdi)
+; AVX512F-NEXT:  .LBB10_6: # %else4
+; AVX512F-NEXT:    kshiftrw $3, %k0, %k1
+; AVX512F-NEXT:    kmovw %k1, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB10_8
+; AVX512F-NEXT:  # %bb.7: # %cond.store5
+; AVX512F-NEXT:    vpextrb $3, %xmm0, 3(%rdi)
+; AVX512F-NEXT:  .LBB10_8: # %else6
+; AVX512F-NEXT:    kshiftrw $4, %k0, %k1
+; AVX512F-NEXT:    kmovw %k1, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB10_10
+; AVX512F-NEXT:  # %bb.9: # %cond.store7
+; AVX512F-NEXT:    vpextrb $4, %xmm0, 4(%rdi)
+; AVX512F-NEXT:  .LBB10_10: # %else8
+; AVX512F-NEXT:    kshiftrw $5, %k0, %k1
+; AVX512F-NEXT:    kmovw %k1, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB10_12
+; AVX512F-NEXT:  # %bb.11: # %cond.store9
+; AVX512F-NEXT:    vpextrb $5, %xmm0, 5(%rdi)
+; AVX512F-NEXT:  .LBB10_12: # %else10
+; AVX512F-NEXT:    kshiftrw $6, %k0, %k1
+; AVX512F-NEXT:    kmovw %k1, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB10_14
+; AVX512F-NEXT:  # %bb.13: # %cond.store11
+; AVX512F-NEXT:    vpextrb $6, %xmm0, 6(%rdi)
+; AVX512F-NEXT:  .LBB10_14: # %else12
+; AVX512F-NEXT:    kshiftrw $7, %k0, %k1
+; AVX512F-NEXT:    kmovw %k1, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB10_16
+; AVX512F-NEXT:  # %bb.15: # %cond.store13
+; AVX512F-NEXT:    vpextrb $7, %xmm0, 7(%rdi)
+; AVX512F-NEXT:  .LBB10_16: # %else14
+; AVX512F-NEXT:    kshiftrw $8, %k0, %k1
+; AVX512F-NEXT:    kmovw %k1, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB10_18
+; AVX512F-NEXT:  # %bb.17: # %cond.store15
+; AVX512F-NEXT:    vpextrb $8, %xmm0, 8(%rdi)
+; AVX512F-NEXT:  .LBB10_18: # %else16
+; AVX512F-NEXT:    kshiftrw $9, %k0, %k1
+; AVX512F-NEXT:    kmovw %k1, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB10_20
+; AVX512F-NEXT:  # %bb.19: # %cond.store17
+; AVX512F-NEXT:    vpextrb $9, %xmm0, 9(%rdi)
+; AVX512F-NEXT:  .LBB10_20: # %else18
+; AVX512F-NEXT:    kshiftrw $10, %k0, %k1
+; AVX512F-NEXT:    kmovw %k1, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB10_22
+; AVX512F-NEXT:  # %bb.21: # %cond.store19
+; AVX512F-NEXT:    vpextrb $10, %xmm0, 10(%rdi)
+; AVX512F-NEXT:  .LBB10_22: # %else20
+; AVX512F-NEXT:    kshiftrw $11, %k0, %k1
+; AVX512F-NEXT:    kmovw %k1, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB10_24
+; AVX512F-NEXT:  # %bb.23: # %cond.store21
+; AVX512F-NEXT:    vpextrb $11, %xmm0, 11(%rdi)
+; AVX512F-NEXT:  .LBB10_24: # %else22
+; AVX512F-NEXT:    kshiftrw $12, %k0, %k1
+; AVX512F-NEXT:    kmovw %k1, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB10_26
+; AVX512F-NEXT:  # %bb.25: # %cond.store23
+; AVX512F-NEXT:    vpextrb $12, %xmm0, 12(%rdi)
+; AVX512F-NEXT:  .LBB10_26: # %else24
+; AVX512F-NEXT:    kshiftrw $13, %k0, %k1
+; AVX512F-NEXT:    kmovw %k1, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB10_28
+; AVX512F-NEXT:  # %bb.27: # %cond.store25
+; AVX512F-NEXT:    vpextrb $13, %xmm0, 13(%rdi)
+; AVX512F-NEXT:  .LBB10_28: # %else26
+; AVX512F-NEXT:    kshiftrw $14, %k0, %k1
+; AVX512F-NEXT:    kmovw %k1, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB10_30
+; AVX512F-NEXT:  # %bb.29: # %cond.store27
+; AVX512F-NEXT:    vpextrb $14, %xmm0, 14(%rdi)
+; AVX512F-NEXT:  .LBB10_30: # %else28
+; AVX512F-NEXT:    kshiftrw $15, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB10_32
+; AVX512F-NEXT:  # %bb.31: # %cond.store29
+; AVX512F-NEXT:    vpextrb $15, %xmm0, 15(%rdi)
+; AVX512F-NEXT:  .LBB10_32: # %else30
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512BW-LABEL: truncstore_v16i32_v16i8:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vptestmd %zmm1, %zmm1, %k1
+; AVX512BW-NEXT:    vpminsd {{.*}}(%rip){1to16}, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpmaxsd {{.*}}(%rip){1to16}, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpmovdb %zmm0, (%rdi) {%k1}
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+;
+; AVX512BWVL-LABEL: truncstore_v16i32_v16i8:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vptestmd %zmm1, %zmm1, %k1
+; AVX512BWVL-NEXT:    vpminsd {{.*}}(%rip){1to16}, %zmm0, %zmm0
+; AVX512BWVL-NEXT:    vpmaxsd {{.*}}(%rip){1to16}, %zmm0, %zmm0
+; AVX512BWVL-NEXT:    vpmovdb %zmm0, (%rdi) {%k1}
+; AVX512BWVL-NEXT:    vzeroupper
+; AVX512BWVL-NEXT:    retq
+  %a = icmp ne <16 x i32> %mask, zeroinitializer
+  %b = icmp slt <16 x i32> %x, <i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127>
+  %c = select <16 x i1> %b, <16 x i32> %x, <16 x i32> <i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127>
+  %d = icmp sgt <16 x i32> %c, <i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128>
+  %e = select <16 x i1> %d, <16 x i32> %c, <16 x i32> <i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128>
+  %f = trunc <16 x i32> %e to <16 x i8>
+  call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %f, <16 x i8>* %p, i32 1, <16 x i1> %a)
+  ret void
+}
+
+define void @truncstore_v8i32_v8i16(<8 x i32> %x, <8 x i16>* %p, <8 x i32> %mask) {
+; AVX512F-LABEL: truncstore_v8i32_v8i16:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    vpbroadcastd {{.*#+}} ymm2 = [32767,32767,32767,32767,32767,32767,32767,32767]
+; AVX512F-NEXT:    vpminsd %ymm2, %ymm0, %ymm0
+; AVX512F-NEXT:    vpbroadcastd {{.*#+}} ymm2 = [4294934528,4294934528,4294934528,4294934528,4294934528,4294934528,4294934528,4294934528]
+; AVX512F-NEXT:    vpmaxsd %ymm2, %ymm0, %ymm0
+; AVX512F-NEXT:    vpmovdw %zmm0, %ymm0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB11_2
+; AVX512F-NEXT:  # %bb.1: # %cond.store
+; AVX512F-NEXT:    vpextrw $0, %xmm0, (%rdi)
+; AVX512F-NEXT:  .LBB11_2: # %else
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $1, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB11_4
+; AVX512F-NEXT:  # %bb.3: # %cond.store1
+; AVX512F-NEXT:    vpextrw $1, %xmm0, 2(%rdi)
+; AVX512F-NEXT:  .LBB11_4: # %else2
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $2, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB11_6
+; AVX512F-NEXT:  # %bb.5: # %cond.store3
+; AVX512F-NEXT:    vpextrw $2, %xmm0, 4(%rdi)
+; AVX512F-NEXT:  .LBB11_6: # %else4
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $3, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB11_8
+; AVX512F-NEXT:  # %bb.7: # %cond.store5
+; AVX512F-NEXT:    vpextrw $3, %xmm0, 6(%rdi)
+; AVX512F-NEXT:  .LBB11_8: # %else6
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $4, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB11_10
+; AVX512F-NEXT:  # %bb.9: # %cond.store7
+; AVX512F-NEXT:    vpextrw $4, %xmm0, 8(%rdi)
+; AVX512F-NEXT:  .LBB11_10: # %else8
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $5, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB11_12
+; AVX512F-NEXT:  # %bb.11: # %cond.store9
+; AVX512F-NEXT:    vpextrw $5, %xmm0, 10(%rdi)
+; AVX512F-NEXT:  .LBB11_12: # %else10
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $6, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB11_14
+; AVX512F-NEXT:  # %bb.13: # %cond.store11
+; AVX512F-NEXT:    vpextrw $6, %xmm0, 12(%rdi)
+; AVX512F-NEXT:  .LBB11_14: # %else12
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $7, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB11_16
+; AVX512F-NEXT:  # %bb.15: # %cond.store13
+; AVX512F-NEXT:    vpextrw $7, %xmm0, 14(%rdi)
+; AVX512F-NEXT:  .LBB11_16: # %else14
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512BW-LABEL: truncstore_v8i32_v8i16:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512BW-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512BW-NEXT:    kshiftld $24, %k0, %k0
+; AVX512BW-NEXT:    kshiftrd $24, %k0, %k1
+; AVX512BW-NEXT:    vpbroadcastd {{.*#+}} ymm1 = [32767,32767,32767,32767,32767,32767,32767,32767]
+; AVX512BW-NEXT:    vpminsd %ymm1, %ymm0, %ymm0
+; AVX512BW-NEXT:    vpbroadcastd {{.*#+}} ymm1 = [4294934528,4294934528,4294934528,4294934528,4294934528,4294934528,4294934528,4294934528]
+; AVX512BW-NEXT:    vpmaxsd %ymm1, %ymm0, %ymm0
+; AVX512BW-NEXT:    vpmovdw %zmm0, %ymm0
+; AVX512BW-NEXT:    vmovdqu16 %zmm0, (%rdi) {%k1}
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+;
+; AVX512BWVL-LABEL: truncstore_v8i32_v8i16:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vptestmd %ymm1, %ymm1, %k1
+; AVX512BWVL-NEXT:    vpminsd {{.*}}(%rip){1to8}, %ymm0, %ymm0
+; AVX512BWVL-NEXT:    vpmaxsd {{.*}}(%rip){1to8}, %ymm0, %ymm0
+; AVX512BWVL-NEXT:    vpmovdw %ymm0, (%rdi) {%k1}
+; AVX512BWVL-NEXT:    vzeroupper
+; AVX512BWVL-NEXT:    retq
+  %a = icmp ne <8 x i32> %mask, zeroinitializer
+  %b = icmp slt <8 x i32> %x, <i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767>
+  %c = select <8 x i1> %b, <8 x i32> %x, <8 x i32> <i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767>
+  %d = icmp sgt <8 x i32> %c, <i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768>
+  %e = select <8 x i1> %d, <8 x i32> %c, <8 x i32> <i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768, i32 -32768>
+  %f = trunc <8 x i32> %e to <8 x i16>
+  call void @llvm.masked.store.v8i16.p0v8i16(<8 x i16> %f, <8 x i16>* %p, i32 1, <8 x i1> %a)
+  ret void
+}
+
+define void @truncstore_v8i32_v8i8(<8 x i32> %x, <8 x i8>* %p, <8 x i32> %mask) {
+; AVX512F-LABEL: truncstore_v8i32_v8i8:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    vpbroadcastd {{.*#+}} ymm2 = [127,127,127,127,127,127,127,127]
+; AVX512F-NEXT:    vpminsd %ymm2, %ymm0, %ymm0
+; AVX512F-NEXT:    vpbroadcastd {{.*#+}} ymm2 = [4294967168,4294967168,4294967168,4294967168,4294967168,4294967168,4294967168,4294967168]
+; AVX512F-NEXT:    vpmaxsd %ymm2, %ymm0, %ymm0
+; AVX512F-NEXT:    vpmovdw %zmm0, %ymm0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB12_2
+; AVX512F-NEXT:  # %bb.1: # %cond.store
+; AVX512F-NEXT:    vpextrb $0, %xmm0, (%rdi)
+; AVX512F-NEXT:  .LBB12_2: # %else
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $1, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB12_4
+; AVX512F-NEXT:  # %bb.3: # %cond.store1
+; AVX512F-NEXT:    vpextrb $2, %xmm0, 1(%rdi)
+; AVX512F-NEXT:  .LBB12_4: # %else2
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $2, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB12_6
+; AVX512F-NEXT:  # %bb.5: # %cond.store3
+; AVX512F-NEXT:    vpextrb $4, %xmm0, 2(%rdi)
+; AVX512F-NEXT:  .LBB12_6: # %else4
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $3, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB12_8
+; AVX512F-NEXT:  # %bb.7: # %cond.store5
+; AVX512F-NEXT:    vpextrb $6, %xmm0, 3(%rdi)
+; AVX512F-NEXT:  .LBB12_8: # %else6
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $4, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB12_10
+; AVX512F-NEXT:  # %bb.9: # %cond.store7
+; AVX512F-NEXT:    vpextrb $8, %xmm0, 4(%rdi)
+; AVX512F-NEXT:  .LBB12_10: # %else8
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $5, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB12_12
+; AVX512F-NEXT:  # %bb.11: # %cond.store9
+; AVX512F-NEXT:    vpextrb $10, %xmm0, 5(%rdi)
+; AVX512F-NEXT:  .LBB12_12: # %else10
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $6, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB12_14
+; AVX512F-NEXT:  # %bb.13: # %cond.store11
+; AVX512F-NEXT:    vpextrb $12, %xmm0, 6(%rdi)
+; AVX512F-NEXT:  .LBB12_14: # %else12
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $7, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB12_16
+; AVX512F-NEXT:  # %bb.15: # %cond.store13
+; AVX512F-NEXT:    vpextrb $14, %xmm0, 7(%rdi)
+; AVX512F-NEXT:  .LBB12_16: # %else14
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512BW-LABEL: truncstore_v8i32_v8i8:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512BW-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512BW-NEXT:    vpbroadcastd {{.*#+}} ymm1 = [127,127,127,127,127,127,127,127]
+; AVX512BW-NEXT:    vpminsd %ymm1, %ymm0, %ymm0
+; AVX512BW-NEXT:    vpbroadcastd {{.*#+}} ymm1 = [4294967168,4294967168,4294967168,4294967168,4294967168,4294967168,4294967168,4294967168]
+; AVX512BW-NEXT:    vpmaxsd %ymm1, %ymm0, %ymm0
+; AVX512BW-NEXT:    vpmovdw %zmm0, %ymm0
+; AVX512BW-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
+; AVX512BW-NEXT:    kshiftlq $56, %k0, %k0
+; AVX512BW-NEXT:    kshiftrq $56, %k0, %k1
+; AVX512BW-NEXT:    vmovdqu8 %zmm0, (%rdi) {%k1}
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+;
+; AVX512BWVL-LABEL: truncstore_v8i32_v8i8:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vptestmd %ymm1, %ymm1, %k1
+; AVX512BWVL-NEXT:    vpminsd {{.*}}(%rip){1to8}, %ymm0, %ymm0
+; AVX512BWVL-NEXT:    vpmaxsd {{.*}}(%rip){1to8}, %ymm0, %ymm0
+; AVX512BWVL-NEXT:    vpmovdb %ymm0, (%rdi) {%k1}
+; AVX512BWVL-NEXT:    vzeroupper
+; AVX512BWVL-NEXT:    retq
+  %a = icmp ne <8 x i32> %mask, zeroinitializer
+  %b = icmp slt <8 x i32> %x, <i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127>
+  %c = select <8 x i1> %b, <8 x i32> %x, <8 x i32> <i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127, i32 127>
+  %d = icmp sgt <8 x i32> %c, <i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128>
+  %e = select <8 x i1> %d, <8 x i32> %c, <8 x i32> <i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128, i32 -128>
+  %f = trunc <8 x i32> %e to <8 x i8>
+  call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> %f, <8 x i8>* %p, i32 1, <8 x i1> %a)
+  ret void
+}
+
+define void @truncstore_v4i32_v4i16(<4 x i32> %x, <4 x i16>* %p, <4 x i32> %mask) {
+; AVX512F-LABEL: truncstore_v4i32_v4i16:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [32767,32767,32767,32767]
+; AVX512F-NEXT:    vpminsd %xmm2, %xmm0, %xmm0
+; AVX512F-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [4294934528,4294934528,4294934528,4294934528]
+; AVX512F-NEXT:    vpmaxsd %xmm2, %xmm0, %xmm0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB13_2
+; AVX512F-NEXT:  # %bb.1: # %cond.store
+; AVX512F-NEXT:    vpextrw $0, %xmm0, (%rdi)
+; AVX512F-NEXT:  .LBB13_2: # %else
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $1, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB13_4
+; AVX512F-NEXT:  # %bb.3: # %cond.store1
+; AVX512F-NEXT:    vpextrw $2, %xmm0, 2(%rdi)
+; AVX512F-NEXT:  .LBB13_4: # %else2
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $2, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB13_6
+; AVX512F-NEXT:  # %bb.5: # %cond.store3
+; AVX512F-NEXT:    vpextrw $4, %xmm0, 4(%rdi)
+; AVX512F-NEXT:  .LBB13_6: # %else4
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $3, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB13_8
+; AVX512F-NEXT:  # %bb.7: # %cond.store5
+; AVX512F-NEXT:    vpextrw $6, %xmm0, 6(%rdi)
+; AVX512F-NEXT:  .LBB13_8: # %else6
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512BW-LABEL: truncstore_v4i32_v4i16:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512BW-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512BW-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [32767,32767,32767,32767]
+; AVX512BW-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [4294934528,4294934528,4294934528,4294934528]
+; AVX512BW-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT:    vpackssdw %xmm0, %xmm0, %xmm0
+; AVX512BW-NEXT:    kshiftld $28, %k0, %k0
+; AVX512BW-NEXT:    kshiftrd $28, %k0, %k1
+; AVX512BW-NEXT:    vmovdqu16 %zmm0, (%rdi) {%k1}
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+;
+; AVX512BWVL-LABEL: truncstore_v4i32_v4i16:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vptestmd %xmm1, %xmm1, %k1
+; AVX512BWVL-NEXT:    vpminsd {{.*}}(%rip){1to4}, %xmm0, %xmm0
+; AVX512BWVL-NEXT:    vpmaxsd {{.*}}(%rip){1to4}, %xmm0, %xmm0
+; AVX512BWVL-NEXT:    vpmovdw %xmm0, (%rdi) {%k1}
+; AVX512BWVL-NEXT:    retq
+  %a = icmp ne <4 x i32> %mask, zeroinitializer
+  %b = icmp slt <4 x i32> %x, <i32 32767, i32 32767, i32 32767, i32 32767>
+  %c = select <4 x i1> %b, <4 x i32> %x, <4 x i32> <i32 32767, i32 32767, i32 32767, i32 32767>
+  %d = icmp sgt <4 x i32> %c, <i32 -32768, i32 -32768, i32 -32768, i32 -32768>
+  %e = select <4 x i1> %d, <4 x i32> %c, <4 x i32> <i32 -32768, i32 -32768, i32 -32768, i32 -32768>
+  %f = trunc <4 x i32> %e to <4 x i16>
+  call void @llvm.masked.store.v4i16.p0v4i16(<4 x i16> %f, <4 x i16>* %p, i32 1, <4 x i1> %a)
+  ret void
+}
+
+define void @truncstore_v4i32_v4i8(<4 x i32> %x, <4 x i8>* %p, <4 x i32> %mask) {
+; AVX512F-LABEL: truncstore_v4i32_v4i8:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [127,127,127,127]
+; AVX512F-NEXT:    vpminsd %xmm2, %xmm0, %xmm0
+; AVX512F-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [4294967168,4294967168,4294967168,4294967168]
+; AVX512F-NEXT:    vpmaxsd %xmm2, %xmm0, %xmm0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB14_2
+; AVX512F-NEXT:  # %bb.1: # %cond.store
+; AVX512F-NEXT:    vpextrb $0, %xmm0, (%rdi)
+; AVX512F-NEXT:  .LBB14_2: # %else
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $1, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB14_4
+; AVX512F-NEXT:  # %bb.3: # %cond.store1
+; AVX512F-NEXT:    vpextrb $4, %xmm0, 1(%rdi)
+; AVX512F-NEXT:  .LBB14_4: # %else2
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $2, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB14_6
+; AVX512F-NEXT:  # %bb.5: # %cond.store3
+; AVX512F-NEXT:    vpextrb $8, %xmm0, 2(%rdi)
+; AVX512F-NEXT:  .LBB14_6: # %else4
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $3, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB14_8
+; AVX512F-NEXT:  # %bb.7: # %cond.store5
+; AVX512F-NEXT:    vpextrb $12, %xmm0, 3(%rdi)
+; AVX512F-NEXT:  .LBB14_8: # %else6
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512BW-LABEL: truncstore_v4i32_v4i8:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512BW-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512BW-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [127,127,127,127]
+; AVX512BW-NEXT:    vpminsd %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [4294967168,4294967168,4294967168,4294967168]
+; AVX512BW-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512BW-NEXT:    kshiftlq $60, %k0, %k0
+; AVX512BW-NEXT:    kshiftrq $60, %k0, %k1
+; AVX512BW-NEXT:    vmovdqu8 %zmm0, (%rdi) {%k1}
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+;
+; AVX512BWVL-LABEL: truncstore_v4i32_v4i8:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vptestmd %xmm1, %xmm1, %k1
+; AVX512BWVL-NEXT:    vpminsd {{.*}}(%rip){1to4}, %xmm0, %xmm0
+; AVX512BWVL-NEXT:    vpmaxsd {{.*}}(%rip){1to4}, %xmm0, %xmm0
+; AVX512BWVL-NEXT:    vpmovdb %xmm0, (%rdi) {%k1}
+; AVX512BWVL-NEXT:    retq
+  %a = icmp ne <4 x i32> %mask, zeroinitializer
+  %b = icmp slt <4 x i32> %x, <i32 127, i32 127, i32 127, i32 127>
+  %c = select <4 x i1> %b, <4 x i32> %x, <4 x i32> <i32 127, i32 127, i32 127, i32 127>
+  %d = icmp sgt <4 x i32> %c, <i32 -128, i32 -128, i32 -128, i32 -128>
+  %e = select <4 x i1> %d, <4 x i32> %c, <4 x i32> <i32 -128, i32 -128, i32 -128, i32 -128>
+  %f = trunc <4 x i32> %e to <4 x i8>
+  call void @llvm.masked.store.v4i8.p0v4i8(<4 x i8> %f, <4 x i8>* %p, i32 1, <4 x i1> %a)
+  ret void
+}
+
+define void @truncstore_v32i16_v32i8(<32 x i16> %x, <32 x i8>* %p, <32 x i8> %mask) {
+; AVX512F-LABEL: truncstore_v32i16_v32i8:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; AVX512F-NEXT:    vpcmpeqb %ymm3, %ymm2, %ymm3
+; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm4
+; AVX512F-NEXT:    vpternlogq $15, %zmm3, %zmm3, %zmm4
+; AVX512F-NEXT:    vpmovsxbd %xmm4, %zmm4
+; AVX512F-NEXT:    vptestmd %zmm4, %zmm4, %k0
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm4 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
+; AVX512F-NEXT:    vpminsw %ymm4, %ymm0, %ymm0
+; AVX512F-NEXT:    vpminsw %ymm4, %ymm1, %ymm1
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm4 = [65408,65408,65408,65408,65408,65408,65408,65408,65408,65408,65408,65408,65408,65408,65408,65408]
+; AVX512F-NEXT:    vpmaxsw %ymm4, %ymm1, %ymm1
+; AVX512F-NEXT:    vpmaxsw %ymm4, %ymm0, %ymm0
+; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
+; AVX512F-NEXT:    vpmovdb %zmm0, %xmm0
+; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero
+; AVX512F-NEXT:    vpmovdb %zmm1, %xmm1
+; AVX512F-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB15_2
+; AVX512F-NEXT:  # %bb.1: # %cond.store
+; AVX512F-NEXT:    vpextrb $0, %xmm0, (%rdi)
+; AVX512F-NEXT:  .LBB15_2: # %else
+; AVX512F-NEXT:    vpternlogq $15, %zmm3, %zmm3, %zmm3
+; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm1
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $1, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB15_4
+; AVX512F-NEXT:  # %bb.3: # %cond.store1
+; AVX512F-NEXT:    vpextrb $1, %xmm0, 1(%rdi)
+; AVX512F-NEXT:  .LBB15_4: # %else2
+; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512F-NEXT:    vpcmpeqb %ymm1, %ymm2, %ymm1
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm3
+; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm3
+; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
+; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
+; AVX512F-NEXT:    kshiftrw $2, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB15_6
+; AVX512F-NEXT:  # %bb.5: # %cond.store3
+; AVX512F-NEXT:    vpextrb $2, %xmm0, 2(%rdi)
+; AVX512F-NEXT:  .LBB15_6: # %else4
+; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm1
+; AVX512F-NEXT:    vpmovsxbd %xmm1, %zmm1
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $3, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB15_8
+; AVX512F-NEXT:  # %bb.7: # %cond.store5
+; AVX512F-NEXT:    vpextrb $3, %xmm0, 3(%rdi)
+; AVX512F-NEXT:  .LBB15_8: # %else6
+; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512F-NEXT:    vpcmpeqb %ymm1, %ymm2, %ymm1
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm3
+; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm3
+; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
+; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
+; AVX512F-NEXT:    kshiftrw $4, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB15_10
+; AVX512F-NEXT:  # %bb.9: # %cond.store7
+; AVX512F-NEXT:    vpextrb $4, %xmm0, 4(%rdi)
+; AVX512F-NEXT:  .LBB15_10: # %else8
+; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm1
+; AVX512F-NEXT:    vpmovsxbd %xmm1, %zmm1
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $5, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB15_12
+; AVX512F-NEXT:  # %bb.11: # %cond.store9
+; AVX512F-NEXT:    vpextrb $5, %xmm0, 5(%rdi)
+; AVX512F-NEXT:  .LBB15_12: # %else10
+; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512F-NEXT:    vpcmpeqb %ymm1, %ymm2, %ymm1
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm3
+; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm3
+; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
+; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
+; AVX512F-NEXT:    kshiftrw $6, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB15_14
+; AVX512F-NEXT:  # %bb.13: # %cond.store11
+; AVX512F-NEXT:    vpextrb $6, %xmm0, 6(%rdi)
+; AVX512F-NEXT:  .LBB15_14: # %else12
+; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm1
+; AVX512F-NEXT:    vpmovsxbd %xmm1, %zmm1
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $7, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB15_16
+; AVX512F-NEXT:  # %bb.15: # %cond.store13
+; AVX512F-NEXT:    vpextrb $7, %xmm0, 7(%rdi)
+; AVX512F-NEXT:  .LBB15_16: # %else14
+; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512F-NEXT:    vpcmpeqb %ymm1, %ymm2, %ymm1
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm3
+; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm3
+; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
+; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
+; AVX512F-NEXT:    kshiftrw $8, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB15_18
+; AVX512F-NEXT:  # %bb.17: # %cond.store15
+; AVX512F-NEXT:    vpextrb $8, %xmm0, 8(%rdi)
+; AVX512F-NEXT:  .LBB15_18: # %else16
+; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm1
+; AVX512F-NEXT:    vpmovsxbd %xmm1, %zmm1
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $9, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB15_20
+; AVX512F-NEXT:  # %bb.19: # %cond.store17
+; AVX512F-NEXT:    vpextrb $9, %xmm0, 9(%rdi)
+; AVX512F-NEXT:  .LBB15_20: # %else18
+; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512F-NEXT:    vpcmpeqb %ymm1, %ymm2, %ymm1
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm3
+; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm3
+; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
+; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
+; AVX512F-NEXT:    kshiftrw $10, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB15_22
+; AVX512F-NEXT:  # %bb.21: # %cond.store19
+; AVX512F-NEXT:    vpextrb $10, %xmm0, 10(%rdi)
+; AVX512F-NEXT:  .LBB15_22: # %else20
+; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm1
+; AVX512F-NEXT:    vpmovsxbd %xmm1, %zmm1
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $11, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB15_24
+; AVX512F-NEXT:  # %bb.23: # %cond.store21
+; AVX512F-NEXT:    vpextrb $11, %xmm0, 11(%rdi)
+; AVX512F-NEXT:  .LBB15_24: # %else22
+; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512F-NEXT:    vpcmpeqb %ymm1, %ymm2, %ymm1
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm3
+; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm3
+; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
+; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
+; AVX512F-NEXT:    kshiftrw $12, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB15_26
+; AVX512F-NEXT:  # %bb.25: # %cond.store23
+; AVX512F-NEXT:    vpextrb $12, %xmm0, 12(%rdi)
+; AVX512F-NEXT:  .LBB15_26: # %else24
+; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm1
+; AVX512F-NEXT:    vpmovsxbd %xmm1, %zmm1
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $13, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB15_28
+; AVX512F-NEXT:  # %bb.27: # %cond.store25
+; AVX512F-NEXT:    vpextrb $13, %xmm0, 13(%rdi)
+; AVX512F-NEXT:  .LBB15_28: # %else26
+; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512F-NEXT:    vpcmpeqb %ymm1, %ymm2, %ymm1
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm3
+; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm3
+; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
+; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
+; AVX512F-NEXT:    kshiftrw $14, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB15_30
+; AVX512F-NEXT:  # %bb.29: # %cond.store27
+; AVX512F-NEXT:    vpextrb $14, %xmm0, 14(%rdi)
+; AVX512F-NEXT:  .LBB15_30: # %else28
+; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm1
+; AVX512F-NEXT:    vpmovsxbd %xmm1, %zmm1
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $15, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB15_32
+; AVX512F-NEXT:  # %bb.31: # %cond.store29
+; AVX512F-NEXT:    vpextrb $15, %xmm0, 15(%rdi)
+; AVX512F-NEXT:  .LBB15_32: # %else30
+; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512F-NEXT:    vpcmpeqb %ymm1, %ymm2, %ymm1
+; AVX512F-NEXT:    vextracti128 $1, %ymm1, %xmm1
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm3
+; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm3
+; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
+; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB15_34
+; AVX512F-NEXT:  # %bb.33: # %cond.store31
+; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm3
+; AVX512F-NEXT:    vpextrb $0, %xmm3, 16(%rdi)
+; AVX512F-NEXT:  .LBB15_34: # %else32
+; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm1
+; AVX512F-NEXT:    vpmovsxbd %xmm1, %zmm1
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $1, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB15_36
+; AVX512F-NEXT:  # %bb.35: # %cond.store33
+; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX512F-NEXT:    vpextrb $1, %xmm1, 17(%rdi)
+; AVX512F-NEXT:  .LBB15_36: # %else34
+; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512F-NEXT:    vpcmpeqb %ymm1, %ymm2, %ymm1
+; AVX512F-NEXT:    vextracti128 $1, %ymm1, %xmm1
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm3
+; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm3
+; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
+; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
+; AVX512F-NEXT:    kshiftrw $2, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB15_38
+; AVX512F-NEXT:  # %bb.37: # %cond.store35
+; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm3
+; AVX512F-NEXT:    vpextrb $2, %xmm3, 18(%rdi)
+; AVX512F-NEXT:  .LBB15_38: # %else36
+; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm1
+; AVX512F-NEXT:    vpmovsxbd %xmm1, %zmm1
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $3, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB15_40
+; AVX512F-NEXT:  # %bb.39: # %cond.store37
+; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX512F-NEXT:    vpextrb $3, %xmm1, 19(%rdi)
+; AVX512F-NEXT:  .LBB15_40: # %else38
+; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512F-NEXT:    vpcmpeqb %ymm1, %ymm2, %ymm1
+; AVX512F-NEXT:    vextracti128 $1, %ymm1, %xmm1
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm3
+; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm3
+; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
+; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
+; AVX512F-NEXT:    kshiftrw $4, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB15_42
+; AVX512F-NEXT:  # %bb.41: # %cond.store39
+; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm3
+; AVX512F-NEXT:    vpextrb $4, %xmm3, 20(%rdi)
+; AVX512F-NEXT:  .LBB15_42: # %else40
+; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm1
+; AVX512F-NEXT:    vpmovsxbd %xmm1, %zmm1
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $5, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB15_44
+; AVX512F-NEXT:  # %bb.43: # %cond.store41
+; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX512F-NEXT:    vpextrb $5, %xmm1, 21(%rdi)
+; AVX512F-NEXT:  .LBB15_44: # %else42
+; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512F-NEXT:    vpcmpeqb %ymm1, %ymm2, %ymm1
+; AVX512F-NEXT:    vextracti128 $1, %ymm1, %xmm1
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm3
+; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm3
+; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
+; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
+; AVX512F-NEXT:    kshiftrw $6, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB15_46
+; AVX512F-NEXT:  # %bb.45: # %cond.store43
+; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm3
+; AVX512F-NEXT:    vpextrb $6, %xmm3, 22(%rdi)
+; AVX512F-NEXT:  .LBB15_46: # %else44
+; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm1
+; AVX512F-NEXT:    vpmovsxbd %xmm1, %zmm1
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $7, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB15_48
+; AVX512F-NEXT:  # %bb.47: # %cond.store45
+; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX512F-NEXT:    vpextrb $7, %xmm1, 23(%rdi)
+; AVX512F-NEXT:  .LBB15_48: # %else46
+; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512F-NEXT:    vpcmpeqb %ymm1, %ymm2, %ymm1
+; AVX512F-NEXT:    vextracti128 $1, %ymm1, %xmm1
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm3
+; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm3
+; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
+; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
+; AVX512F-NEXT:    kshiftrw $8, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB15_50
+; AVX512F-NEXT:  # %bb.49: # %cond.store47
+; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm3
+; AVX512F-NEXT:    vpextrb $8, %xmm3, 24(%rdi)
+; AVX512F-NEXT:  .LBB15_50: # %else48
+; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm1
+; AVX512F-NEXT:    vpmovsxbd %xmm1, %zmm1
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $9, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB15_52
+; AVX512F-NEXT:  # %bb.51: # %cond.store49
+; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX512F-NEXT:    vpextrb $9, %xmm1, 25(%rdi)
+; AVX512F-NEXT:  .LBB15_52: # %else50
+; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512F-NEXT:    vpcmpeqb %ymm1, %ymm2, %ymm1
+; AVX512F-NEXT:    vextracti128 $1, %ymm1, %xmm1
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm3
+; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm3
+; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
+; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
+; AVX512F-NEXT:    kshiftrw $10, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB15_54
+; AVX512F-NEXT:  # %bb.53: # %cond.store51
+; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm3
+; AVX512F-NEXT:    vpextrb $10, %xmm3, 26(%rdi)
+; AVX512F-NEXT:  .LBB15_54: # %else52
+; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm1
+; AVX512F-NEXT:    vpmovsxbd %xmm1, %zmm1
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $11, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB15_56
+; AVX512F-NEXT:  # %bb.55: # %cond.store53
+; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX512F-NEXT:    vpextrb $11, %xmm1, 27(%rdi)
+; AVX512F-NEXT:  .LBB15_56: # %else54
+; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512F-NEXT:    vpcmpeqb %ymm1, %ymm2, %ymm1
+; AVX512F-NEXT:    vextracti128 $1, %ymm1, %xmm1
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm3
+; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm3
+; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
+; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
+; AVX512F-NEXT:    kshiftrw $12, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB15_58
+; AVX512F-NEXT:  # %bb.57: # %cond.store55
+; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm3
+; AVX512F-NEXT:    vpextrb $12, %xmm3, 28(%rdi)
+; AVX512F-NEXT:  .LBB15_58: # %else56
+; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm1
+; AVX512F-NEXT:    vpmovsxbd %xmm1, %zmm1
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $13, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB15_60
+; AVX512F-NEXT:  # %bb.59: # %cond.store57
+; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX512F-NEXT:    vpextrb $13, %xmm1, 29(%rdi)
+; AVX512F-NEXT:  .LBB15_60: # %else58
+; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512F-NEXT:    vpcmpeqb %ymm1, %ymm2, %ymm1
+; AVX512F-NEXT:    vextracti128 $1, %ymm1, %xmm1
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm2
+; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm2
+; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
+; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
+; AVX512F-NEXT:    kshiftrw $14, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB15_62
+; AVX512F-NEXT:  # %bb.61: # %cond.store59
+; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm2
+; AVX512F-NEXT:    vpextrb $14, %xmm2, 30(%rdi)
+; AVX512F-NEXT:  .LBB15_62: # %else60
+; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm1
+; AVX512F-NEXT:    vpmovsxbd %xmm1, %zmm1
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $15, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB15_64
+; AVX512F-NEXT:  # %bb.63: # %cond.store61
+; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm0
+; AVX512F-NEXT:    vpextrb $15, %xmm0, 31(%rdi)
+; AVX512F-NEXT:  .LBB15_64: # %else62
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512BW-LABEL: truncstore_v32i16_v32i8:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512BW-NEXT:    vptestmb %zmm1, %zmm1, %k1
+; AVX512BW-NEXT:    vpminsw {{.*}}(%rip), %zmm0, %zmm0
+; AVX512BW-NEXT:    vpmaxsw {{.*}}(%rip), %zmm0, %zmm0
+; AVX512BW-NEXT:    vpmovwb %zmm0, (%rdi) {%k1}
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+;
+; AVX512BWVL-LABEL: truncstore_v32i16_v32i8:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vptestmb %ymm1, %ymm1, %k1
+; AVX512BWVL-NEXT:    vpminsw {{.*}}(%rip), %zmm0, %zmm0
+; AVX512BWVL-NEXT:    vpmaxsw {{.*}}(%rip), %zmm0, %zmm0
+; AVX512BWVL-NEXT:    vpmovwb %zmm0, (%rdi) {%k1}
+; AVX512BWVL-NEXT:    vzeroupper
+; AVX512BWVL-NEXT:    retq
+  %a = icmp ne <32 x i8> %mask, zeroinitializer
+  %b = icmp slt <32 x i16> %x, <i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127>
+  %c = select <32 x i1> %b, <32 x i16> %x, <32 x i16> <i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127>
+  %d = icmp sgt <32 x i16> %c, <i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128>
+  %e = select <32 x i1> %d, <32 x i16> %c, <32 x i16> <i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128>
+  %f = trunc <32 x i16> %e to <32 x i8>
+  call void @llvm.masked.store.v32i8.p0v32i8(<32 x i8> %f, <32 x i8>* %p, i32 1, <32 x i1> %a)
+  ret void
+}
+
+define void @truncstore_v16i16_v16i8(<16 x i16> %x, <16 x i8>* %p, <16 x i8> %mask) {
+; AVX512F-LABEL: truncstore_v16i16_v16i8:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
+; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3
+; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm3
+; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
+; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
+; AVX512F-NEXT:    vpminsw {{.*}}(%rip), %ymm0, %ymm0
+; AVX512F-NEXT:    vpmaxsw {{.*}}(%rip), %ymm0, %ymm0
+; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
+; AVX512F-NEXT:    vpmovdb %zmm0, %xmm0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB16_2
+; AVX512F-NEXT:  # %bb.1: # %cond.store
+; AVX512F-NEXT:    vpextrb $0, %xmm0, (%rdi)
+; AVX512F-NEXT:  .LBB16_2: # %else
+; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm2
+; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
+; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
+; AVX512F-NEXT:    kshiftrw $1, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB16_4
+; AVX512F-NEXT:  # %bb.3: # %cond.store1
+; AVX512F-NEXT:    vpextrb $1, %xmm0, 1(%rdi)
+; AVX512F-NEXT:  .LBB16_4: # %else2
+; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
+; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3
+; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm3
+; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
+; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
+; AVX512F-NEXT:    kshiftrw $2, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB16_6
+; AVX512F-NEXT:  # %bb.5: # %cond.store3
+; AVX512F-NEXT:    vpextrb $2, %xmm0, 2(%rdi)
+; AVX512F-NEXT:  .LBB16_6: # %else4
+; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm2
+; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
+; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
+; AVX512F-NEXT:    kshiftrw $3, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB16_8
+; AVX512F-NEXT:  # %bb.7: # %cond.store5
+; AVX512F-NEXT:    vpextrb $3, %xmm0, 3(%rdi)
+; AVX512F-NEXT:  .LBB16_8: # %else6
+; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
+; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3
+; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm3
+; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
+; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
+; AVX512F-NEXT:    kshiftrw $4, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB16_10
+; AVX512F-NEXT:  # %bb.9: # %cond.store7
+; AVX512F-NEXT:    vpextrb $4, %xmm0, 4(%rdi)
+; AVX512F-NEXT:  .LBB16_10: # %else8
+; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm2
+; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
+; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
+; AVX512F-NEXT:    kshiftrw $5, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB16_12
+; AVX512F-NEXT:  # %bb.11: # %cond.store9
+; AVX512F-NEXT:    vpextrb $5, %xmm0, 5(%rdi)
+; AVX512F-NEXT:  .LBB16_12: # %else10
+; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
+; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3
+; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm3
+; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
+; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
+; AVX512F-NEXT:    kshiftrw $6, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB16_14
+; AVX512F-NEXT:  # %bb.13: # %cond.store11
+; AVX512F-NEXT:    vpextrb $6, %xmm0, 6(%rdi)
+; AVX512F-NEXT:  .LBB16_14: # %else12
+; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm2
+; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
+; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
+; AVX512F-NEXT:    kshiftrw $7, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB16_16
+; AVX512F-NEXT:  # %bb.15: # %cond.store13
+; AVX512F-NEXT:    vpextrb $7, %xmm0, 7(%rdi)
+; AVX512F-NEXT:  .LBB16_16: # %else14
+; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
+; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3
+; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm3
+; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
+; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
+; AVX512F-NEXT:    kshiftrw $8, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB16_18
+; AVX512F-NEXT:  # %bb.17: # %cond.store15
+; AVX512F-NEXT:    vpextrb $8, %xmm0, 8(%rdi)
+; AVX512F-NEXT:  .LBB16_18: # %else16
+; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm2
+; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
+; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
+; AVX512F-NEXT:    kshiftrw $9, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB16_20
+; AVX512F-NEXT:  # %bb.19: # %cond.store17
+; AVX512F-NEXT:    vpextrb $9, %xmm0, 9(%rdi)
+; AVX512F-NEXT:  .LBB16_20: # %else18
+; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
+; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3
+; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm3
+; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
+; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
+; AVX512F-NEXT:    kshiftrw $10, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB16_22
+; AVX512F-NEXT:  # %bb.21: # %cond.store19
+; AVX512F-NEXT:    vpextrb $10, %xmm0, 10(%rdi)
+; AVX512F-NEXT:  .LBB16_22: # %else20
+; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm2
+; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
+; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
+; AVX512F-NEXT:    kshiftrw $11, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB16_24
+; AVX512F-NEXT:  # %bb.23: # %cond.store21
+; AVX512F-NEXT:    vpextrb $11, %xmm0, 11(%rdi)
+; AVX512F-NEXT:  .LBB16_24: # %else22
+; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
+; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3
+; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm3
+; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
+; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
+; AVX512F-NEXT:    kshiftrw $12, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB16_26
+; AVX512F-NEXT:  # %bb.25: # %cond.store23
+; AVX512F-NEXT:    vpextrb $12, %xmm0, 12(%rdi)
+; AVX512F-NEXT:  .LBB16_26: # %else24
+; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm2
+; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
+; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
+; AVX512F-NEXT:    kshiftrw $13, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB16_28
+; AVX512F-NEXT:  # %bb.27: # %cond.store25
+; AVX512F-NEXT:    vpextrb $13, %xmm0, 13(%rdi)
+; AVX512F-NEXT:  .LBB16_28: # %else26
+; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm1
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm2
+; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm2
+; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
+; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
+; AVX512F-NEXT:    kshiftrw $14, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB16_30
+; AVX512F-NEXT:  # %bb.29: # %cond.store27
+; AVX512F-NEXT:    vpextrb $14, %xmm0, 14(%rdi)
+; AVX512F-NEXT:  .LBB16_30: # %else28
+; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm1
+; AVX512F-NEXT:    vpmovsxbd %xmm1, %zmm1
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $15, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB16_32
+; AVX512F-NEXT:  # %bb.31: # %cond.store29
+; AVX512F-NEXT:    vpextrb $15, %xmm0, 15(%rdi)
+; AVX512F-NEXT:  .LBB16_32: # %else30
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512BW-LABEL: truncstore_v16i16_v16i8:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512BW-NEXT:    vptestmb %zmm1, %zmm1, %k0
+; AVX512BW-NEXT:    kmovw %k0, %k1
+; AVX512BW-NEXT:    vpminsw {{.*}}(%rip), %ymm0, %ymm0
+; AVX512BW-NEXT:    vpmaxsw {{.*}}(%rip), %ymm0, %ymm0
+; AVX512BW-NEXT:    vpmovwb %zmm0, %ymm0
+; AVX512BW-NEXT:    vmovdqu8 %zmm0, (%rdi) {%k1}
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+;
+; AVX512BWVL-LABEL: truncstore_v16i16_v16i8:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vptestmb %xmm1, %xmm1, %k1
+; AVX512BWVL-NEXT:    vpminsw {{.*}}(%rip), %ymm0, %ymm0
+; AVX512BWVL-NEXT:    vpmaxsw {{.*}}(%rip), %ymm0, %ymm0
+; AVX512BWVL-NEXT:    vpmovwb %ymm0, (%rdi) {%k1}
+; AVX512BWVL-NEXT:    vzeroupper
+; AVX512BWVL-NEXT:    retq
+  %a = icmp ne <16 x i8> %mask, zeroinitializer
+  %b = icmp slt <16 x i16> %x, <i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127>
+  %c = select <16 x i1> %b, <16 x i16> %x, <16 x i16> <i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127>
+  %d = icmp sgt <16 x i16> %c, <i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128>
+  %e = select <16 x i1> %d, <16 x i16> %c, <16 x i16> <i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128>
+  %f = trunc <16 x i16> %e to <16 x i8>
+  call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %f, <16 x i8>* %p, i32 1, <16 x i1> %a)
+  ret void
+}
+
+define void @truncstore_v8i16_v8i8(<8 x i16> %x, <8 x i8>* %p, <8 x i16> %mask) {
+; AVX512F-LABEL: truncstore_v8i16_v8i8:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX512F-NEXT:    vpcmpeqw %xmm2, %xmm1, %xmm2
+; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3
+; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm3
+; AVX512F-NEXT:    vpmovsxwq %xmm3, %zmm3
+; AVX512F-NEXT:    vptestmq %zmm3, %zmm3, %k0
+; AVX512F-NEXT:    vpminsw {{.*}}(%rip), %xmm0, %xmm0
+; AVX512F-NEXT:    vpmaxsw {{.*}}(%rip), %xmm0, %xmm0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB17_2
+; AVX512F-NEXT:  # %bb.1: # %cond.store
+; AVX512F-NEXT:    vpextrb $0, %xmm0, (%rdi)
+; AVX512F-NEXT:  .LBB17_2: # %else
+; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm2
+; AVX512F-NEXT:    vpmovsxwq %xmm2, %zmm2
+; AVX512F-NEXT:    vptestmq %zmm2, %zmm2, %k0
+; AVX512F-NEXT:    kshiftrw $1, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB17_4
+; AVX512F-NEXT:  # %bb.3: # %cond.store1
+; AVX512F-NEXT:    vpextrb $2, %xmm0, 1(%rdi)
+; AVX512F-NEXT:  .LBB17_4: # %else2
+; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX512F-NEXT:    vpcmpeqw %xmm2, %xmm1, %xmm2
+; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3
+; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm3
+; AVX512F-NEXT:    vpmovsxwq %xmm3, %zmm3
+; AVX512F-NEXT:    vptestmq %zmm3, %zmm3, %k0
+; AVX512F-NEXT:    kshiftrw $2, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB17_6
+; AVX512F-NEXT:  # %bb.5: # %cond.store3
+; AVX512F-NEXT:    vpextrb $4, %xmm0, 2(%rdi)
+; AVX512F-NEXT:  .LBB17_6: # %else4
+; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm2
+; AVX512F-NEXT:    vpmovsxwq %xmm2, %zmm2
+; AVX512F-NEXT:    vptestmq %zmm2, %zmm2, %k0
+; AVX512F-NEXT:    kshiftrw $3, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB17_8
+; AVX512F-NEXT:  # %bb.7: # %cond.store5
+; AVX512F-NEXT:    vpextrb $6, %xmm0, 3(%rdi)
+; AVX512F-NEXT:  .LBB17_8: # %else6
+; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX512F-NEXT:    vpcmpeqw %xmm2, %xmm1, %xmm2
+; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3
+; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm3
+; AVX512F-NEXT:    vpmovsxwq %xmm3, %zmm3
+; AVX512F-NEXT:    vptestmq %zmm3, %zmm3, %k0
+; AVX512F-NEXT:    kshiftrw $4, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB17_10
+; AVX512F-NEXT:  # %bb.9: # %cond.store7
+; AVX512F-NEXT:    vpextrb $8, %xmm0, 4(%rdi)
+; AVX512F-NEXT:  .LBB17_10: # %else8
+; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm2
+; AVX512F-NEXT:    vpmovsxwq %xmm2, %zmm2
+; AVX512F-NEXT:    vptestmq %zmm2, %zmm2, %k0
+; AVX512F-NEXT:    kshiftrw $5, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB17_12
+; AVX512F-NEXT:  # %bb.11: # %cond.store9
+; AVX512F-NEXT:    vpextrb $10, %xmm0, 5(%rdi)
+; AVX512F-NEXT:  .LBB17_12: # %else10
+; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX512F-NEXT:    vpcmpeqw %xmm2, %xmm1, %xmm1
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm2
+; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm2
+; AVX512F-NEXT:    vpmovsxwq %xmm2, %zmm2
+; AVX512F-NEXT:    vptestmq %zmm2, %zmm2, %k0
+; AVX512F-NEXT:    kshiftrw $6, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB17_14
+; AVX512F-NEXT:  # %bb.13: # %cond.store11
+; AVX512F-NEXT:    vpextrb $12, %xmm0, 6(%rdi)
+; AVX512F-NEXT:  .LBB17_14: # %else12
+; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm1
+; AVX512F-NEXT:    vpmovsxwq %xmm1, %zmm1
+; AVX512F-NEXT:    vptestmq %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $7, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB17_16
+; AVX512F-NEXT:  # %bb.15: # %cond.store13
+; AVX512F-NEXT:    vpextrb $14, %xmm0, 7(%rdi)
+; AVX512F-NEXT:  .LBB17_16: # %else14
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512BW-LABEL: truncstore_v8i16_v8i8:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512BW-NEXT:    vptestmw %zmm1, %zmm1, %k0
+; AVX512BW-NEXT:    vpminsw {{.*}}(%rip), %xmm0, %xmm0
+; AVX512BW-NEXT:    vpmaxsw {{.*}}(%rip), %xmm0, %xmm0
+; AVX512BW-NEXT:    vpacksswb %xmm0, %xmm0, %xmm0
+; AVX512BW-NEXT:    kshiftlq $56, %k0, %k0
+; AVX512BW-NEXT:    kshiftrq $56, %k0, %k1
+; AVX512BW-NEXT:    vmovdqu8 %zmm0, (%rdi) {%k1}
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+;
+; AVX512BWVL-LABEL: truncstore_v8i16_v8i8:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vptestmw %xmm1, %xmm1, %k1
+; AVX512BWVL-NEXT:    vpminsw {{.*}}(%rip), %xmm0, %xmm0
+; AVX512BWVL-NEXT:    vpmaxsw {{.*}}(%rip), %xmm0, %xmm0
+; AVX512BWVL-NEXT:    vpmovwb %xmm0, (%rdi) {%k1}
+; AVX512BWVL-NEXT:    retq
+  %a = icmp ne <8 x i16> %mask, zeroinitializer
+  %b = icmp slt <8 x i16> %x, <i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127>
+  %c = select <8 x i1> %b, <8 x i16> %x, <8 x i16> <i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127>
+  %d = icmp sgt <8 x i16> %c, <i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128>
+  %e = select <8 x i1> %d, <8 x i16> %c, <8 x i16> <i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128>
+  %f = trunc <8 x i16> %e to <8 x i8>
+  call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> %f, <8 x i8>* %p, i32 1, <8 x i1> %a)
+  ret void
+}
+
+declare void @llvm.masked.store.v8i32.p0v8i32(<8 x i32>, <8 x i32>*, i32, <8 x i1>)
+declare void @llvm.masked.store.v8i16.p0v8i16(<8 x i16>, <8 x i16>*, i32, <8 x i1>)
+declare void @llvm.masked.store.v8i8.p0v8i8(<8 x i8>, <8 x i8>*, i32, <8 x i1>)
+declare void @llvm.masked.store.v4i32.p0v4i32(<4 x i32>, <4 x i32>*, i32, <4 x i1>)
+declare void @llvm.masked.store.v4i16.p0v4i16(<4 x i16>, <4 x i16>*, i32, <4 x i1>)
+declare void @llvm.masked.store.v4i8.p0v4i8(<4 x i8>, <4 x i8>*, i32, <4 x i1>)
+declare void @llvm.masked.store.v2i32.p0v2i32(<2 x i32>, <2 x i32>*, i32, <2 x i1>)
+declare void @llvm.masked.store.v2i16.p0v2i16(<2 x i16>, <2 x i16>*, i32, <2 x i1>)
+declare void @llvm.masked.store.v2i8.p0v2i8(<2 x i8>, <2 x i8>*, i32, <2 x i1>)
+declare void @llvm.masked.store.v16i16.p0v16i16(<16 x i16>, <16 x i16>*, i32, <16 x i1>)
+declare void @llvm.masked.store.v16i8.p0v16i8(<16 x i8>, <16 x i8>*, i32, <16 x i1>)
+declare void @llvm.masked.store.v32i8.p0v32i8(<32 x i8>, <32 x i8>*, i32, <32 x i1>)
diff --git a/test/CodeGen/X86/masked_store_trunc_usat.ll b/test/CodeGen/X86/masked_store_trunc_usat.ll
new file mode 100644
index 0000000000000000000000000000000000000000..8f070235ac67d29562c2d349881d7f177bc79585
--- /dev/null
+++ b/test/CodeGen/X86/masked_store_trunc_usat.ll
@@ -0,0 +1,1980 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx512f | FileCheck %s --check-prefixes=AVX512,AVX512F
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx512vl | FileCheck %s --check-prefixes=AVX512,AVX512VL
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx512bw | FileCheck %s --check-prefixes=AVX512,AVX512BW,AVX512BWNOVL
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=avx512vl,avx512bw | FileCheck %s --check-prefixes=AVX512,AVX512VL,AVX512BWVL
+
+define void @truncstore_v8i64_v8i32(<8 x i64> %x, <8 x i32>* %p, <8 x i32> %mask) {
+; AVX512F-LABEL: truncstore_v8i64_v8i32:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k1
+; AVX512F-NEXT:    vpminuq {{.*}}(%rip){1to8}, %zmm0, %zmm0
+; AVX512F-NEXT:    vpmovqd %zmm0, (%rdi) {%k1}
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: truncstore_v8i64_v8i32:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vptestmd %ymm1, %ymm1, %k1
+; AVX512VL-NEXT:    vpminuq {{.*}}(%rip){1to8}, %zmm0, %zmm0
+; AVX512VL-NEXT:    vpmovqd %zmm0, (%rdi) {%k1}
+; AVX512VL-NEXT:    vzeroupper
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: truncstore_v8i64_v8i32:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512BW-NEXT:    vptestmd %zmm1, %zmm1, %k1
+; AVX512BW-NEXT:    vpminuq {{.*}}(%rip){1to8}, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpmovqd %zmm0, (%rdi) {%k1}
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+  %a = icmp ne <8 x i32> %mask, zeroinitializer
+  %b = icmp ult <8 x i64> %x, <i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295>
+  %c = select <8 x i1> %b, <8 x i64> %x, <8 x i64> <i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295>
+  %d = trunc <8 x i64> %c to <8 x i32>
+  call void @llvm.masked.store.v8i32.p0v8i32(<8 x i32> %d, <8 x i32>* %p, i32 1, <8 x i1> %a)
+  ret void
+}
+
+define void @truncstore_v8i64_v8i16(<8 x i64> %x, <8 x i16>* %p, <8 x i32> %mask) {
+; AVX512F-LABEL: truncstore_v8i64_v8i16:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    vpmovusqw %zmm0, %xmm0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB1_2
+; AVX512F-NEXT:  # %bb.1: # %cond.store
+; AVX512F-NEXT:    vpextrw $0, %xmm0, (%rdi)
+; AVX512F-NEXT:  .LBB1_2: # %else
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $1, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB1_4
+; AVX512F-NEXT:  # %bb.3: # %cond.store1
+; AVX512F-NEXT:    vpextrw $1, %xmm0, 2(%rdi)
+; AVX512F-NEXT:  .LBB1_4: # %else2
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $2, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB1_6
+; AVX512F-NEXT:  # %bb.5: # %cond.store3
+; AVX512F-NEXT:    vpextrw $2, %xmm0, 4(%rdi)
+; AVX512F-NEXT:  .LBB1_6: # %else4
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $3, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB1_8
+; AVX512F-NEXT:  # %bb.7: # %cond.store5
+; AVX512F-NEXT:    vpextrw $3, %xmm0, 6(%rdi)
+; AVX512F-NEXT:  .LBB1_8: # %else6
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $4, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB1_10
+; AVX512F-NEXT:  # %bb.9: # %cond.store7
+; AVX512F-NEXT:    vpextrw $4, %xmm0, 8(%rdi)
+; AVX512F-NEXT:  .LBB1_10: # %else8
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $5, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB1_12
+; AVX512F-NEXT:  # %bb.11: # %cond.store9
+; AVX512F-NEXT:    vpextrw $5, %xmm0, 10(%rdi)
+; AVX512F-NEXT:  .LBB1_12: # %else10
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $6, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB1_14
+; AVX512F-NEXT:  # %bb.13: # %cond.store11
+; AVX512F-NEXT:    vpextrw $6, %xmm0, 12(%rdi)
+; AVX512F-NEXT:  .LBB1_14: # %else12
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $7, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB1_16
+; AVX512F-NEXT:  # %bb.15: # %cond.store13
+; AVX512F-NEXT:    vpextrw $7, %xmm0, 14(%rdi)
+; AVX512F-NEXT:  .LBB1_16: # %else14
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512BW-LABEL: truncstore_v8i64_v8i16:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512BW-NEXT:    vptestmd %zmm1, %zmm1, %k1
+; AVX512BW-NEXT:    vpminuq {{.*}}(%rip){1to8}, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpmovqw %zmm0, (%rdi) {%k1}
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+;
+; AVX512BWVL-LABEL: truncstore_v8i64_v8i16:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vptestmd %ymm1, %ymm1, %k1
+; AVX512BWVL-NEXT:    vpminuq {{.*}}(%rip){1to8}, %zmm0, %zmm0
+; AVX512BWVL-NEXT:    vpmovqw %zmm0, (%rdi) {%k1}
+; AVX512BWVL-NEXT:    vzeroupper
+; AVX512BWVL-NEXT:    retq
+  %a = icmp ne <8 x i32> %mask, zeroinitializer
+  %b = icmp ult <8 x i64> %x, <i64 65535, i64 65535, i64 65535, i64 65535, i64 65535, i64 65535, i64 65535, i64 65535>
+  %c = select <8 x i1> %b, <8 x i64> %x, <8 x i64> <i64 65535, i64 65535, i64 65535, i64 65535, i64 65535, i64 65535, i64 65535, i64 65535>
+  %d = trunc <8 x i64> %c to <8 x i16>
+  call void @llvm.masked.store.v8i16.p0v8i16(<8 x i16> %d, <8 x i16>* %p, i32 1, <8 x i1> %a)
+  ret void
+}
+
+define void @truncstore_v8i64_v8i8(<8 x i64> %x, <8 x i8>* %p, <8 x i32> %mask) {
+; AVX512F-LABEL: truncstore_v8i64_v8i8:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    vpminuq {{.*}}(%rip){1to8}, %zmm0, %zmm0
+; AVX512F-NEXT:    vpmovqw %zmm0, %xmm0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB2_2
+; AVX512F-NEXT:  # %bb.1: # %cond.store
+; AVX512F-NEXT:    vpextrb $0, %xmm0, (%rdi)
+; AVX512F-NEXT:  .LBB2_2: # %else
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $1, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB2_4
+; AVX512F-NEXT:  # %bb.3: # %cond.store1
+; AVX512F-NEXT:    vpextrb $2, %xmm0, 1(%rdi)
+; AVX512F-NEXT:  .LBB2_4: # %else2
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $2, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB2_6
+; AVX512F-NEXT:  # %bb.5: # %cond.store3
+; AVX512F-NEXT:    vpextrb $4, %xmm0, 2(%rdi)
+; AVX512F-NEXT:  .LBB2_6: # %else4
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $3, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB2_8
+; AVX512F-NEXT:  # %bb.7: # %cond.store5
+; AVX512F-NEXT:    vpextrb $6, %xmm0, 3(%rdi)
+; AVX512F-NEXT:  .LBB2_8: # %else6
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $4, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB2_10
+; AVX512F-NEXT:  # %bb.9: # %cond.store7
+; AVX512F-NEXT:    vpextrb $8, %xmm0, 4(%rdi)
+; AVX512F-NEXT:  .LBB2_10: # %else8
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $5, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB2_12
+; AVX512F-NEXT:  # %bb.11: # %cond.store9
+; AVX512F-NEXT:    vpextrb $10, %xmm0, 5(%rdi)
+; AVX512F-NEXT:  .LBB2_12: # %else10
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $6, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB2_14
+; AVX512F-NEXT:  # %bb.13: # %cond.store11
+; AVX512F-NEXT:    vpextrb $12, %xmm0, 6(%rdi)
+; AVX512F-NEXT:  .LBB2_14: # %else12
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $7, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB2_16
+; AVX512F-NEXT:  # %bb.15: # %cond.store13
+; AVX512F-NEXT:    vpextrb $14, %xmm0, 7(%rdi)
+; AVX512F-NEXT:  .LBB2_16: # %else14
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512BW-LABEL: truncstore_v8i64_v8i8:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512BW-NEXT:    vptestmd %zmm1, %zmm1, %k1
+; AVX512BW-NEXT:    vpminuq {{.*}}(%rip){1to8}, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpmovqb %zmm0, (%rdi) {%k1}
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+;
+; AVX512BWVL-LABEL: truncstore_v8i64_v8i8:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vptestmd %ymm1, %ymm1, %k1
+; AVX512BWVL-NEXT:    vpminuq {{.*}}(%rip){1to8}, %zmm0, %zmm0
+; AVX512BWVL-NEXT:    vpmovqb %zmm0, (%rdi) {%k1}
+; AVX512BWVL-NEXT:    vzeroupper
+; AVX512BWVL-NEXT:    retq
+  %a = icmp ne <8 x i32> %mask, zeroinitializer
+  %b = icmp ult <8 x i64> %x, <i64 255, i64 255, i64 255, i64 255, i64 255, i64 255, i64 255, i64 255>
+  %c = select <8 x i1> %b, <8 x i64> %x, <8 x i64> <i64 255, i64 255, i64 255, i64 255, i64 255, i64 255, i64 255, i64 255>
+  %d = trunc <8 x i64> %c to <8 x i8>
+  call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> %d, <8 x i8>* %p, i32 1, <8 x i1> %a)
+  ret void
+}
+
+define void @truncstore_v4i64_v4i32(<4 x i64> %x, <4 x i32>* %p, <4 x i32> %mask) {
+; AVX512F-LABEL: truncstore_v4i64_v4i32:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512F-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftlw $12, %k0, %k0
+; AVX512F-NEXT:    kshiftrw $12, %k0, %k1
+; AVX512F-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [4294967295,4294967295,4294967295,4294967295]
+; AVX512F-NEXT:    vpminuq %zmm1, %zmm0, %zmm0
+; AVX512F-NEXT:    vpmovqd %zmm0, %ymm0
+; AVX512F-NEXT:    vmovdqu32 %zmm0, (%rdi) {%k1}
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: truncstore_v4i64_v4i32:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vptestmd %xmm1, %xmm1, %k1
+; AVX512VL-NEXT:    vpminuq {{.*}}(%rip){1to4}, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpmovqd %ymm0, (%rdi) {%k1}
+; AVX512VL-NEXT:    vzeroupper
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: truncstore_v4i64_v4i32:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512BW-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512BW-NEXT:    kshiftlw $12, %k0, %k0
+; AVX512BW-NEXT:    kshiftrw $12, %k0, %k1
+; AVX512BW-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [4294967295,4294967295,4294967295,4294967295]
+; AVX512BW-NEXT:    vpminuq %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpmovqd %zmm0, %ymm0
+; AVX512BW-NEXT:    vmovdqu32 %zmm0, (%rdi) {%k1}
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+  %a = icmp ne <4 x i32> %mask, zeroinitializer
+  %b = icmp ult <4 x i64> %x, <i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295>
+  %c = select <4 x i1> %b, <4 x i64> %x, <4 x i64> <i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295>
+  %d = trunc <4 x i64> %c to <4 x i32>
+  call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %d, <4 x i32>* %p, i32 1, <4 x i1> %a)
+  ret void
+}
+
+define void @truncstore_v4i64_v4i16(<4 x i64> %x, <4 x i16>* %p, <4 x i32> %mask) {
+; AVX512F-LABEL: truncstore_v4i64_v4i16:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512F-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [65535,65535,65535,65535]
+; AVX512F-NEXT:    vpminuq %zmm2, %zmm0, %zmm0
+; AVX512F-NEXT:    vpmovqd %zmm0, %ymm0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB4_2
+; AVX512F-NEXT:  # %bb.1: # %cond.store
+; AVX512F-NEXT:    vpextrw $0, %xmm0, (%rdi)
+; AVX512F-NEXT:  .LBB4_2: # %else
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $1, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB4_4
+; AVX512F-NEXT:  # %bb.3: # %cond.store1
+; AVX512F-NEXT:    vpextrw $2, %xmm0, 2(%rdi)
+; AVX512F-NEXT:  .LBB4_4: # %else2
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $2, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB4_6
+; AVX512F-NEXT:  # %bb.5: # %cond.store3
+; AVX512F-NEXT:    vpextrw $4, %xmm0, 4(%rdi)
+; AVX512F-NEXT:  .LBB4_6: # %else4
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $3, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB4_8
+; AVX512F-NEXT:  # %bb.7: # %cond.store5
+; AVX512F-NEXT:    vpextrw $6, %xmm0, 6(%rdi)
+; AVX512F-NEXT:  .LBB4_8: # %else6
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512BW-LABEL: truncstore_v4i64_v4i16:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512BW-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512BW-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [65535,65535,65535,65535]
+; AVX512BW-NEXT:    vpminuq %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpmovqd %zmm0, %ymm0
+; AVX512BW-NEXT:    vpackusdw %xmm0, %xmm0, %xmm0
+; AVX512BW-NEXT:    kshiftld $28, %k0, %k0
+; AVX512BW-NEXT:    kshiftrd $28, %k0, %k1
+; AVX512BW-NEXT:    vmovdqu16 %zmm0, (%rdi) {%k1}
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+;
+; AVX512BWVL-LABEL: truncstore_v4i64_v4i16:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vptestmd %xmm1, %xmm1, %k1
+; AVX512BWVL-NEXT:    vpminuq {{.*}}(%rip){1to4}, %ymm0, %ymm0
+; AVX512BWVL-NEXT:    vpmovqw %ymm0, (%rdi) {%k1}
+; AVX512BWVL-NEXT:    vzeroupper
+; AVX512BWVL-NEXT:    retq
+  %a = icmp ne <4 x i32> %mask, zeroinitializer
+  %b = icmp ult <4 x i64> %x, <i64 65535, i64 65535, i64 65535, i64 65535>
+  %c = select <4 x i1> %b, <4 x i64> %x, <4 x i64> <i64 65535, i64 65535, i64 65535, i64 65535>
+  %d = trunc <4 x i64> %c to <4 x i16>
+  call void @llvm.masked.store.v4i16.p0v4i16(<4 x i16> %d, <4 x i16>* %p, i32 1, <4 x i1> %a)
+  ret void
+}
+
+define void @truncstore_v4i64_v4i8(<4 x i64> %x, <4 x i8>* %p, <4 x i32> %mask) {
+; AVX512F-LABEL: truncstore_v4i64_v4i8:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512F-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [255,255,255,255]
+; AVX512F-NEXT:    vpminuq %zmm2, %zmm0, %zmm0
+; AVX512F-NEXT:    vpmovqd %zmm0, %ymm0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB5_2
+; AVX512F-NEXT:  # %bb.1: # %cond.store
+; AVX512F-NEXT:    vpextrb $0, %xmm0, (%rdi)
+; AVX512F-NEXT:  .LBB5_2: # %else
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $1, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB5_4
+; AVX512F-NEXT:  # %bb.3: # %cond.store1
+; AVX512F-NEXT:    vpextrb $4, %xmm0, 1(%rdi)
+; AVX512F-NEXT:  .LBB5_4: # %else2
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $2, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB5_6
+; AVX512F-NEXT:  # %bb.5: # %cond.store3
+; AVX512F-NEXT:    vpextrb $8, %xmm0, 2(%rdi)
+; AVX512F-NEXT:  .LBB5_6: # %else4
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $3, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB5_8
+; AVX512F-NEXT:  # %bb.7: # %cond.store5
+; AVX512F-NEXT:    vpextrb $12, %xmm0, 3(%rdi)
+; AVX512F-NEXT:  .LBB5_8: # %else6
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512BW-LABEL: truncstore_v4i64_v4i8:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512BW-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512BW-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [255,255,255,255]
+; AVX512BW-NEXT:    vpminuq %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpmovqd %zmm0, %ymm0
+; AVX512BW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512BW-NEXT:    kshiftlq $60, %k0, %k0
+; AVX512BW-NEXT:    kshiftrq $60, %k0, %k1
+; AVX512BW-NEXT:    vmovdqu8 %zmm0, (%rdi) {%k1}
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+;
+; AVX512BWVL-LABEL: truncstore_v4i64_v4i8:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vptestmd %xmm1, %xmm1, %k1
+; AVX512BWVL-NEXT:    vpminuq {{.*}}(%rip){1to4}, %ymm0, %ymm0
+; AVX512BWVL-NEXT:    vpmovqb %ymm0, (%rdi) {%k1}
+; AVX512BWVL-NEXT:    vzeroupper
+; AVX512BWVL-NEXT:    retq
+  %a = icmp ne <4 x i32> %mask, zeroinitializer
+  %b = icmp ult <4 x i64> %x, <i64 255, i64 255, i64 255, i64 255>
+  %c = select <4 x i1> %b, <4 x i64> %x, <4 x i64> <i64 255, i64 255, i64 255, i64 255>
+  %d = trunc <4 x i64> %c to <4 x i8>
+  call void @llvm.masked.store.v4i8.p0v4i8(<4 x i8> %d, <4 x i8>* %p, i32 1, <4 x i1> %a)
+  ret void
+}
+
+define void @truncstore_v2i64_v2i32(<2 x i64> %x, <2 x i32>* %p, <2 x i64> %mask) {
+; AVX512F-LABEL: truncstore_v2i64_v2i32:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512F-NEXT:    vptestmq %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm1 = [4294967295,4294967295]
+; AVX512F-NEXT:    vpminuq %zmm1, %zmm0, %zmm0
+; AVX512F-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; AVX512F-NEXT:    kshiftlw $14, %k0, %k0
+; AVX512F-NEXT:    kshiftrw $14, %k0, %k1
+; AVX512F-NEXT:    vmovdqu32 %zmm0, (%rdi) {%k1}
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: truncstore_v2i64_v2i32:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vptestmq %xmm1, %xmm1, %k1
+; AVX512VL-NEXT:    vpminuq {{.*}}(%rip), %xmm0, %xmm0
+; AVX512VL-NEXT:    vpmovqd %xmm0, (%rdi) {%k1}
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: truncstore_v2i64_v2i32:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512BW-NEXT:    vptestmq %zmm1, %zmm1, %k0
+; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm1 = [4294967295,4294967295]
+; AVX512BW-NEXT:    vpminuq %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; AVX512BW-NEXT:    kshiftlw $14, %k0, %k0
+; AVX512BW-NEXT:    kshiftrw $14, %k0, %k1
+; AVX512BW-NEXT:    vmovdqu32 %zmm0, (%rdi) {%k1}
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+  %a = icmp ne <2 x i64> %mask, zeroinitializer
+  %b = icmp ult <2 x i64> %x, <i64 4294967295, i64 4294967295>
+  %c = select <2 x i1> %b, <2 x i64> %x, <2 x i64> <i64 4294967295, i64 4294967295>
+  %d = trunc <2 x i64> %c to <2 x i32>
+  call void @llvm.masked.store.v2i32.p0v2i32(<2 x i32> %d, <2 x i32>* %p, i32 1, <2 x i1> %a)
+  ret void
+}
+
+define void @truncstore_v2i64_v2i16(<2 x i64> %x, <2 x i16>* %p, <2 x i64> %mask) {
+; AVX512F-LABEL: truncstore_v2i64_v2i16:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512F-NEXT:    vptestmq %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm2 = [65535,65535]
+; AVX512F-NEXT:    vpminuq %zmm2, %zmm0, %zmm0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB7_2
+; AVX512F-NEXT:  # %bb.1: # %cond.store
+; AVX512F-NEXT:    vpextrw $0, %xmm0, (%rdi)
+; AVX512F-NEXT:  .LBB7_2: # %else
+; AVX512F-NEXT:    vptestmq %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $1, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB7_4
+; AVX512F-NEXT:  # %bb.3: # %cond.store1
+; AVX512F-NEXT:    vpextrw $4, %xmm0, 2(%rdi)
+; AVX512F-NEXT:  .LBB7_4: # %else2
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512BW-LABEL: truncstore_v2i64_v2i16:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512BW-NEXT:    vptestmq %zmm1, %zmm1, %k0
+; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm1 = [65535,65535]
+; AVX512BW-NEXT:    vpminuq %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; AVX512BW-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
+; AVX512BW-NEXT:    kshiftld $30, %k0, %k0
+; AVX512BW-NEXT:    kshiftrd $30, %k0, %k1
+; AVX512BW-NEXT:    vmovdqu16 %zmm0, (%rdi) {%k1}
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+;
+; AVX512BWVL-LABEL: truncstore_v2i64_v2i16:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vptestmq %xmm1, %xmm1, %k1
+; AVX512BWVL-NEXT:    vpminuq {{.*}}(%rip), %xmm0, %xmm0
+; AVX512BWVL-NEXT:    vpmovqw %xmm0, (%rdi) {%k1}
+; AVX512BWVL-NEXT:    retq
+  %a = icmp ne <2 x i64> %mask, zeroinitializer
+  %b = icmp ult <2 x i64> %x, <i64 65535, i64 65535>
+  %c = select <2 x i1> %b, <2 x i64> %x, <2 x i64> <i64 65535, i64 65535>
+  %d = trunc <2 x i64> %c to <2 x i16>
+  call void @llvm.masked.store.v2i16.p0v2i16(<2 x i16> %d, <2 x i16>* %p, i32 1, <2 x i1> %a)
+  ret void
+}
+
+define void @truncstore_v2i64_v2i8(<2 x i64> %x, <2 x i8>* %p, <2 x i64> %mask) {
+; AVX512F-LABEL: truncstore_v2i64_v2i8:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512F-NEXT:    vptestmq %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm2 = [255,255]
+; AVX512F-NEXT:    vpminuq %zmm2, %zmm0, %zmm0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB8_2
+; AVX512F-NEXT:  # %bb.1: # %cond.store
+; AVX512F-NEXT:    vpextrb $0, %xmm0, (%rdi)
+; AVX512F-NEXT:  .LBB8_2: # %else
+; AVX512F-NEXT:    vptestmq %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $1, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB8_4
+; AVX512F-NEXT:  # %bb.3: # %cond.store1
+; AVX512F-NEXT:    vpextrb $8, %xmm0, 1(%rdi)
+; AVX512F-NEXT:  .LBB8_4: # %else2
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512BW-LABEL: truncstore_v2i64_v2i8:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512BW-NEXT:    vptestmq %zmm1, %zmm1, %k0
+; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm1 = [255,255]
+; AVX512BW-NEXT:    vpminuq %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,8,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512BW-NEXT:    kshiftlq $62, %k0, %k0
+; AVX512BW-NEXT:    kshiftrq $62, %k0, %k1
+; AVX512BW-NEXT:    vmovdqu8 %zmm0, (%rdi) {%k1}
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+;
+; AVX512BWVL-LABEL: truncstore_v2i64_v2i8:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vptestmq %xmm1, %xmm1, %k1
+; AVX512BWVL-NEXT:    vpminuq {{.*}}(%rip), %xmm0, %xmm0
+; AVX512BWVL-NEXT:    vpmovqb %xmm0, (%rdi) {%k1}
+; AVX512BWVL-NEXT:    retq
+  %a = icmp ne <2 x i64> %mask, zeroinitializer
+  %b = icmp ult <2 x i64> %x, <i64 255, i64 255>
+  %c = select <2 x i1> %b, <2 x i64> %x, <2 x i64> <i64 255, i64 255>
+  %d = trunc <2 x i64> %c to <2 x i8>
+  call void @llvm.masked.store.v2i8.p0v2i8(<2 x i8> %d, <2 x i8>* %p, i32 1, <2 x i1> %a)
+  ret void
+}
+
+define void @truncstore_v16i32_v16i16(<16 x i32> %x, <16 x i16>* %p, <16 x i32> %mask) {
+; AVX512F-LABEL: truncstore_v16i32_v16i16:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    vpmovusdw %zmm0, %ymm0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB9_2
+; AVX512F-NEXT:  # %bb.1: # %cond.store
+; AVX512F-NEXT:    vpextrw $0, %xmm0, (%rdi)
+; AVX512F-NEXT:  .LBB9_2: # %else
+; AVX512F-NEXT:    kshiftrw $1, %k0, %k1
+; AVX512F-NEXT:    kmovw %k1, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB9_4
+; AVX512F-NEXT:  # %bb.3: # %cond.store1
+; AVX512F-NEXT:    vpextrw $1, %xmm0, 2(%rdi)
+; AVX512F-NEXT:  .LBB9_4: # %else2
+; AVX512F-NEXT:    kshiftrw $2, %k0, %k1
+; AVX512F-NEXT:    kmovw %k1, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB9_6
+; AVX512F-NEXT:  # %bb.5: # %cond.store3
+; AVX512F-NEXT:    vpextrw $2, %xmm0, 4(%rdi)
+; AVX512F-NEXT:  .LBB9_6: # %else4
+; AVX512F-NEXT:    kshiftrw $3, %k0, %k1
+; AVX512F-NEXT:    kmovw %k1, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB9_8
+; AVX512F-NEXT:  # %bb.7: # %cond.store5
+; AVX512F-NEXT:    vpextrw $3, %xmm0, 6(%rdi)
+; AVX512F-NEXT:  .LBB9_8: # %else6
+; AVX512F-NEXT:    kshiftrw $4, %k0, %k1
+; AVX512F-NEXT:    kmovw %k1, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB9_10
+; AVX512F-NEXT:  # %bb.9: # %cond.store7
+; AVX512F-NEXT:    vpextrw $4, %xmm0, 8(%rdi)
+; AVX512F-NEXT:  .LBB9_10: # %else8
+; AVX512F-NEXT:    kshiftrw $5, %k0, %k1
+; AVX512F-NEXT:    kmovw %k1, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB9_12
+; AVX512F-NEXT:  # %bb.11: # %cond.store9
+; AVX512F-NEXT:    vpextrw $5, %xmm0, 10(%rdi)
+; AVX512F-NEXT:  .LBB9_12: # %else10
+; AVX512F-NEXT:    kshiftrw $6, %k0, %k1
+; AVX512F-NEXT:    kmovw %k1, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB9_14
+; AVX512F-NEXT:  # %bb.13: # %cond.store11
+; AVX512F-NEXT:    vpextrw $6, %xmm0, 12(%rdi)
+; AVX512F-NEXT:  .LBB9_14: # %else12
+; AVX512F-NEXT:    kshiftrw $7, %k0, %k1
+; AVX512F-NEXT:    kmovw %k1, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB9_16
+; AVX512F-NEXT:  # %bb.15: # %cond.store13
+; AVX512F-NEXT:    vpextrw $7, %xmm0, 14(%rdi)
+; AVX512F-NEXT:  .LBB9_16: # %else14
+; AVX512F-NEXT:    kshiftrw $8, %k0, %k1
+; AVX512F-NEXT:    kmovw %k1, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB9_18
+; AVX512F-NEXT:  # %bb.17: # %cond.store15
+; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX512F-NEXT:    vpextrw $0, %xmm1, 16(%rdi)
+; AVX512F-NEXT:  .LBB9_18: # %else16
+; AVX512F-NEXT:    kshiftrw $9, %k0, %k1
+; AVX512F-NEXT:    kmovw %k1, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB9_20
+; AVX512F-NEXT:  # %bb.19: # %cond.store17
+; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX512F-NEXT:    vpextrw $1, %xmm1, 18(%rdi)
+; AVX512F-NEXT:  .LBB9_20: # %else18
+; AVX512F-NEXT:    kshiftrw $10, %k0, %k1
+; AVX512F-NEXT:    kmovw %k1, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB9_22
+; AVX512F-NEXT:  # %bb.21: # %cond.store19
+; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX512F-NEXT:    vpextrw $2, %xmm1, 20(%rdi)
+; AVX512F-NEXT:  .LBB9_22: # %else20
+; AVX512F-NEXT:    kshiftrw $11, %k0, %k1
+; AVX512F-NEXT:    kmovw %k1, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB9_24
+; AVX512F-NEXT:  # %bb.23: # %cond.store21
+; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX512F-NEXT:    vpextrw $3, %xmm1, 22(%rdi)
+; AVX512F-NEXT:  .LBB9_24: # %else22
+; AVX512F-NEXT:    kshiftrw $12, %k0, %k1
+; AVX512F-NEXT:    kmovw %k1, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB9_26
+; AVX512F-NEXT:  # %bb.25: # %cond.store23
+; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX512F-NEXT:    vpextrw $4, %xmm1, 24(%rdi)
+; AVX512F-NEXT:  .LBB9_26: # %else24
+; AVX512F-NEXT:    kshiftrw $13, %k0, %k1
+; AVX512F-NEXT:    kmovw %k1, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB9_28
+; AVX512F-NEXT:  # %bb.27: # %cond.store25
+; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX512F-NEXT:    vpextrw $5, %xmm1, 26(%rdi)
+; AVX512F-NEXT:  .LBB9_28: # %else26
+; AVX512F-NEXT:    kshiftrw $14, %k0, %k1
+; AVX512F-NEXT:    kmovw %k1, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB9_30
+; AVX512F-NEXT:  # %bb.29: # %cond.store27
+; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX512F-NEXT:    vpextrw $6, %xmm1, 28(%rdi)
+; AVX512F-NEXT:  .LBB9_30: # %else28
+; AVX512F-NEXT:    kshiftrw $15, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB9_32
+; AVX512F-NEXT:  # %bb.31: # %cond.store29
+; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm0
+; AVX512F-NEXT:    vpextrw $7, %xmm0, 30(%rdi)
+; AVX512F-NEXT:  .LBB9_32: # %else30
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512BW-LABEL: truncstore_v16i32_v16i16:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vptestmd %zmm1, %zmm1, %k1
+; AVX512BW-NEXT:    vpminud {{.*}}(%rip){1to16}, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpmovdw %zmm0, (%rdi) {%k1}
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+;
+; AVX512BWVL-LABEL: truncstore_v16i32_v16i16:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vptestmd %zmm1, %zmm1, %k1
+; AVX512BWVL-NEXT:    vpminud {{.*}}(%rip){1to16}, %zmm0, %zmm0
+; AVX512BWVL-NEXT:    vpmovdw %zmm0, (%rdi) {%k1}
+; AVX512BWVL-NEXT:    vzeroupper
+; AVX512BWVL-NEXT:    retq
+  %a = icmp ne <16 x i32> %mask, zeroinitializer
+  %b = icmp ult <16 x i32> %x, <i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535>
+  %c = select <16 x i1> %b, <16 x i32> %x, <16 x i32> <i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535>
+  %d = trunc <16 x i32> %c to <16 x i16>
+  call void @llvm.masked.store.v16i16.p0v16i16(<16 x i16> %d, <16 x i16>* %p, i32 1, <16 x i1> %a)
+  ret void
+}
+
+define void @truncstore_v16i32_v16i8(<16 x i32> %x, <16 x i8>* %p, <16 x i32> %mask) {
+; AVX512F-LABEL: truncstore_v16i32_v16i8:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    vpmovusdb %zmm0, %xmm0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB10_2
+; AVX512F-NEXT:  # %bb.1: # %cond.store
+; AVX512F-NEXT:    vpextrb $0, %xmm0, (%rdi)
+; AVX512F-NEXT:  .LBB10_2: # %else
+; AVX512F-NEXT:    kshiftrw $1, %k0, %k1
+; AVX512F-NEXT:    kmovw %k1, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB10_4
+; AVX512F-NEXT:  # %bb.3: # %cond.store1
+; AVX512F-NEXT:    vpextrb $1, %xmm0, 1(%rdi)
+; AVX512F-NEXT:  .LBB10_4: # %else2
+; AVX512F-NEXT:    kshiftrw $2, %k0, %k1
+; AVX512F-NEXT:    kmovw %k1, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB10_6
+; AVX512F-NEXT:  # %bb.5: # %cond.store3
+; AVX512F-NEXT:    vpextrb $2, %xmm0, 2(%rdi)
+; AVX512F-NEXT:  .LBB10_6: # %else4
+; AVX512F-NEXT:    kshiftrw $3, %k0, %k1
+; AVX512F-NEXT:    kmovw %k1, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB10_8
+; AVX512F-NEXT:  # %bb.7: # %cond.store5
+; AVX512F-NEXT:    vpextrb $3, %xmm0, 3(%rdi)
+; AVX512F-NEXT:  .LBB10_8: # %else6
+; AVX512F-NEXT:    kshiftrw $4, %k0, %k1
+; AVX512F-NEXT:    kmovw %k1, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB10_10
+; AVX512F-NEXT:  # %bb.9: # %cond.store7
+; AVX512F-NEXT:    vpextrb $4, %xmm0, 4(%rdi)
+; AVX512F-NEXT:  .LBB10_10: # %else8
+; AVX512F-NEXT:    kshiftrw $5, %k0, %k1
+; AVX512F-NEXT:    kmovw %k1, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB10_12
+; AVX512F-NEXT:  # %bb.11: # %cond.store9
+; AVX512F-NEXT:    vpextrb $5, %xmm0, 5(%rdi)
+; AVX512F-NEXT:  .LBB10_12: # %else10
+; AVX512F-NEXT:    kshiftrw $6, %k0, %k1
+; AVX512F-NEXT:    kmovw %k1, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB10_14
+; AVX512F-NEXT:  # %bb.13: # %cond.store11
+; AVX512F-NEXT:    vpextrb $6, %xmm0, 6(%rdi)
+; AVX512F-NEXT:  .LBB10_14: # %else12
+; AVX512F-NEXT:    kshiftrw $7, %k0, %k1
+; AVX512F-NEXT:    kmovw %k1, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB10_16
+; AVX512F-NEXT:  # %bb.15: # %cond.store13
+; AVX512F-NEXT:    vpextrb $7, %xmm0, 7(%rdi)
+; AVX512F-NEXT:  .LBB10_16: # %else14
+; AVX512F-NEXT:    kshiftrw $8, %k0, %k1
+; AVX512F-NEXT:    kmovw %k1, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB10_18
+; AVX512F-NEXT:  # %bb.17: # %cond.store15
+; AVX512F-NEXT:    vpextrb $8, %xmm0, 8(%rdi)
+; AVX512F-NEXT:  .LBB10_18: # %else16
+; AVX512F-NEXT:    kshiftrw $9, %k0, %k1
+; AVX512F-NEXT:    kmovw %k1, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB10_20
+; AVX512F-NEXT:  # %bb.19: # %cond.store17
+; AVX512F-NEXT:    vpextrb $9, %xmm0, 9(%rdi)
+; AVX512F-NEXT:  .LBB10_20: # %else18
+; AVX512F-NEXT:    kshiftrw $10, %k0, %k1
+; AVX512F-NEXT:    kmovw %k1, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB10_22
+; AVX512F-NEXT:  # %bb.21: # %cond.store19
+; AVX512F-NEXT:    vpextrb $10, %xmm0, 10(%rdi)
+; AVX512F-NEXT:  .LBB10_22: # %else20
+; AVX512F-NEXT:    kshiftrw $11, %k0, %k1
+; AVX512F-NEXT:    kmovw %k1, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB10_24
+; AVX512F-NEXT:  # %bb.23: # %cond.store21
+; AVX512F-NEXT:    vpextrb $11, %xmm0, 11(%rdi)
+; AVX512F-NEXT:  .LBB10_24: # %else22
+; AVX512F-NEXT:    kshiftrw $12, %k0, %k1
+; AVX512F-NEXT:    kmovw %k1, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB10_26
+; AVX512F-NEXT:  # %bb.25: # %cond.store23
+; AVX512F-NEXT:    vpextrb $12, %xmm0, 12(%rdi)
+; AVX512F-NEXT:  .LBB10_26: # %else24
+; AVX512F-NEXT:    kshiftrw $13, %k0, %k1
+; AVX512F-NEXT:    kmovw %k1, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB10_28
+; AVX512F-NEXT:  # %bb.27: # %cond.store25
+; AVX512F-NEXT:    vpextrb $13, %xmm0, 13(%rdi)
+; AVX512F-NEXT:  .LBB10_28: # %else26
+; AVX512F-NEXT:    kshiftrw $14, %k0, %k1
+; AVX512F-NEXT:    kmovw %k1, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB10_30
+; AVX512F-NEXT:  # %bb.29: # %cond.store27
+; AVX512F-NEXT:    vpextrb $14, %xmm0, 14(%rdi)
+; AVX512F-NEXT:  .LBB10_30: # %else28
+; AVX512F-NEXT:    kshiftrw $15, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB10_32
+; AVX512F-NEXT:  # %bb.31: # %cond.store29
+; AVX512F-NEXT:    vpextrb $15, %xmm0, 15(%rdi)
+; AVX512F-NEXT:  .LBB10_32: # %else30
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512BW-LABEL: truncstore_v16i32_v16i8:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vptestmd %zmm1, %zmm1, %k1
+; AVX512BW-NEXT:    vpminud {{.*}}(%rip){1to16}, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpmovdb %zmm0, (%rdi) {%k1}
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+;
+; AVX512BWVL-LABEL: truncstore_v16i32_v16i8:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vptestmd %zmm1, %zmm1, %k1
+; AVX512BWVL-NEXT:    vpminud {{.*}}(%rip){1to16}, %zmm0, %zmm0
+; AVX512BWVL-NEXT:    vpmovdb %zmm0, (%rdi) {%k1}
+; AVX512BWVL-NEXT:    vzeroupper
+; AVX512BWVL-NEXT:    retq
+  %a = icmp ne <16 x i32> %mask, zeroinitializer
+  %b = icmp ult <16 x i32> %x, <i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255>
+  %c = select <16 x i1> %b, <16 x i32> %x, <16 x i32> <i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255>
+  %d = trunc <16 x i32> %c to <16 x i8>
+  call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %d, <16 x i8>* %p, i32 1, <16 x i1> %a)
+  ret void
+}
+
+define void @truncstore_v8i32_v8i16(<8 x i32> %x, <8 x i16>* %p, <8 x i32> %mask) {
+; AVX512F-LABEL: truncstore_v8i32_v8i16:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    vpbroadcastd {{.*#+}} ymm2 = [65535,65535,65535,65535,65535,65535,65535,65535]
+; AVX512F-NEXT:    vpminud %ymm2, %ymm0, %ymm0
+; AVX512F-NEXT:    vpmovdw %zmm0, %ymm0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB11_2
+; AVX512F-NEXT:  # %bb.1: # %cond.store
+; AVX512F-NEXT:    vpextrw $0, %xmm0, (%rdi)
+; AVX512F-NEXT:  .LBB11_2: # %else
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $1, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB11_4
+; AVX512F-NEXT:  # %bb.3: # %cond.store1
+; AVX512F-NEXT:    vpextrw $1, %xmm0, 2(%rdi)
+; AVX512F-NEXT:  .LBB11_4: # %else2
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $2, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB11_6
+; AVX512F-NEXT:  # %bb.5: # %cond.store3
+; AVX512F-NEXT:    vpextrw $2, %xmm0, 4(%rdi)
+; AVX512F-NEXT:  .LBB11_6: # %else4
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $3, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB11_8
+; AVX512F-NEXT:  # %bb.7: # %cond.store5
+; AVX512F-NEXT:    vpextrw $3, %xmm0, 6(%rdi)
+; AVX512F-NEXT:  .LBB11_8: # %else6
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $4, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB11_10
+; AVX512F-NEXT:  # %bb.9: # %cond.store7
+; AVX512F-NEXT:    vpextrw $4, %xmm0, 8(%rdi)
+; AVX512F-NEXT:  .LBB11_10: # %else8
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $5, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB11_12
+; AVX512F-NEXT:  # %bb.11: # %cond.store9
+; AVX512F-NEXT:    vpextrw $5, %xmm0, 10(%rdi)
+; AVX512F-NEXT:  .LBB11_12: # %else10
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $6, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB11_14
+; AVX512F-NEXT:  # %bb.13: # %cond.store11
+; AVX512F-NEXT:    vpextrw $6, %xmm0, 12(%rdi)
+; AVX512F-NEXT:  .LBB11_14: # %else12
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $7, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB11_16
+; AVX512F-NEXT:  # %bb.15: # %cond.store13
+; AVX512F-NEXT:    vpextrw $7, %xmm0, 14(%rdi)
+; AVX512F-NEXT:  .LBB11_16: # %else14
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512BW-LABEL: truncstore_v8i32_v8i16:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512BW-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512BW-NEXT:    kshiftld $24, %k0, %k0
+; AVX512BW-NEXT:    kshiftrd $24, %k0, %k1
+; AVX512BW-NEXT:    vpbroadcastd {{.*#+}} ymm1 = [65535,65535,65535,65535,65535,65535,65535,65535]
+; AVX512BW-NEXT:    vpminud %ymm1, %ymm0, %ymm0
+; AVX512BW-NEXT:    vpmovdw %zmm0, %ymm0
+; AVX512BW-NEXT:    vmovdqu16 %zmm0, (%rdi) {%k1}
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+;
+; AVX512BWVL-LABEL: truncstore_v8i32_v8i16:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vptestmd %ymm1, %ymm1, %k1
+; AVX512BWVL-NEXT:    vpminud {{.*}}(%rip){1to8}, %ymm0, %ymm0
+; AVX512BWVL-NEXT:    vpmovdw %ymm0, (%rdi) {%k1}
+; AVX512BWVL-NEXT:    vzeroupper
+; AVX512BWVL-NEXT:    retq
+  %a = icmp ne <8 x i32> %mask, zeroinitializer
+  %b = icmp ult <8 x i32> %x, <i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535>
+  %c = select <8 x i1> %b, <8 x i32> %x, <8 x i32> <i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535>
+  %d = trunc <8 x i32> %c to <8 x i16>
+  call void @llvm.masked.store.v8i16.p0v8i16(<8 x i16> %d, <8 x i16>* %p, i32 1, <8 x i1> %a)
+  ret void
+}
+
+define void @truncstore_v8i32_v8i8(<8 x i32> %x, <8 x i8>* %p, <8 x i32> %mask) {
+; AVX512F-LABEL: truncstore_v8i32_v8i8:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    vpbroadcastd {{.*#+}} ymm2 = [255,255,255,255,255,255,255,255]
+; AVX512F-NEXT:    vpminud %ymm2, %ymm0, %ymm0
+; AVX512F-NEXT:    vpmovdw %zmm0, %ymm0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB12_2
+; AVX512F-NEXT:  # %bb.1: # %cond.store
+; AVX512F-NEXT:    vpextrb $0, %xmm0, (%rdi)
+; AVX512F-NEXT:  .LBB12_2: # %else
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $1, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB12_4
+; AVX512F-NEXT:  # %bb.3: # %cond.store1
+; AVX512F-NEXT:    vpextrb $2, %xmm0, 1(%rdi)
+; AVX512F-NEXT:  .LBB12_4: # %else2
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $2, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB12_6
+; AVX512F-NEXT:  # %bb.5: # %cond.store3
+; AVX512F-NEXT:    vpextrb $4, %xmm0, 2(%rdi)
+; AVX512F-NEXT:  .LBB12_6: # %else4
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $3, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB12_8
+; AVX512F-NEXT:  # %bb.7: # %cond.store5
+; AVX512F-NEXT:    vpextrb $6, %xmm0, 3(%rdi)
+; AVX512F-NEXT:  .LBB12_8: # %else6
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $4, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB12_10
+; AVX512F-NEXT:  # %bb.9: # %cond.store7
+; AVX512F-NEXT:    vpextrb $8, %xmm0, 4(%rdi)
+; AVX512F-NEXT:  .LBB12_10: # %else8
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $5, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB12_12
+; AVX512F-NEXT:  # %bb.11: # %cond.store9
+; AVX512F-NEXT:    vpextrb $10, %xmm0, 5(%rdi)
+; AVX512F-NEXT:  .LBB12_12: # %else10
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $6, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB12_14
+; AVX512F-NEXT:  # %bb.13: # %cond.store11
+; AVX512F-NEXT:    vpextrb $12, %xmm0, 6(%rdi)
+; AVX512F-NEXT:  .LBB12_14: # %else12
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $7, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB12_16
+; AVX512F-NEXT:  # %bb.15: # %cond.store13
+; AVX512F-NEXT:    vpextrb $14, %xmm0, 7(%rdi)
+; AVX512F-NEXT:  .LBB12_16: # %else14
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512BW-LABEL: truncstore_v8i32_v8i8:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512BW-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512BW-NEXT:    vpbroadcastd {{.*#+}} ymm1 = [255,255,255,255,255,255,255,255]
+; AVX512BW-NEXT:    vpminud %ymm1, %ymm0, %ymm0
+; AVX512BW-NEXT:    vpmovdw %zmm0, %ymm0
+; AVX512BW-NEXT:    vpackuswb %xmm0, %xmm0, %xmm0
+; AVX512BW-NEXT:    kshiftlq $56, %k0, %k0
+; AVX512BW-NEXT:    kshiftrq $56, %k0, %k1
+; AVX512BW-NEXT:    vmovdqu8 %zmm0, (%rdi) {%k1}
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+;
+; AVX512BWVL-LABEL: truncstore_v8i32_v8i8:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vptestmd %ymm1, %ymm1, %k1
+; AVX512BWVL-NEXT:    vpminud {{.*}}(%rip){1to8}, %ymm0, %ymm0
+; AVX512BWVL-NEXT:    vpmovdb %ymm0, (%rdi) {%k1}
+; AVX512BWVL-NEXT:    vzeroupper
+; AVX512BWVL-NEXT:    retq
+  %a = icmp ne <8 x i32> %mask, zeroinitializer
+  %b = icmp ult <8 x i32> %x, <i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255>
+  %c = select <8 x i1> %b, <8 x i32> %x, <8 x i32> <i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255>
+  %d = trunc <8 x i32> %c to <8 x i8>
+  call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> %d, <8 x i8>* %p, i32 1, <8 x i1> %a)
+  ret void
+}
+
+define void @truncstore_v4i32_v4i16(<4 x i32> %x, <4 x i16>* %p, <4 x i32> %mask) {
+; AVX512F-LABEL: truncstore_v4i32_v4i16:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [65535,65535,65535,65535]
+; AVX512F-NEXT:    vpminud %xmm2, %xmm0, %xmm0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB13_2
+; AVX512F-NEXT:  # %bb.1: # %cond.store
+; AVX512F-NEXT:    vpextrw $0, %xmm0, (%rdi)
+; AVX512F-NEXT:  .LBB13_2: # %else
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $1, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB13_4
+; AVX512F-NEXT:  # %bb.3: # %cond.store1
+; AVX512F-NEXT:    vpextrw $2, %xmm0, 2(%rdi)
+; AVX512F-NEXT:  .LBB13_4: # %else2
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $2, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB13_6
+; AVX512F-NEXT:  # %bb.5: # %cond.store3
+; AVX512F-NEXT:    vpextrw $4, %xmm0, 4(%rdi)
+; AVX512F-NEXT:  .LBB13_6: # %else4
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $3, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB13_8
+; AVX512F-NEXT:  # %bb.7: # %cond.store5
+; AVX512F-NEXT:    vpextrw $6, %xmm0, 6(%rdi)
+; AVX512F-NEXT:  .LBB13_8: # %else6
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512BW-LABEL: truncstore_v4i32_v4i16:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512BW-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512BW-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [65535,65535,65535,65535]
+; AVX512BW-NEXT:    vpminud %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT:    vpackusdw %xmm0, %xmm0, %xmm0
+; AVX512BW-NEXT:    kshiftld $28, %k0, %k0
+; AVX512BW-NEXT:    kshiftrd $28, %k0, %k1
+; AVX512BW-NEXT:    vmovdqu16 %zmm0, (%rdi) {%k1}
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+;
+; AVX512BWVL-LABEL: truncstore_v4i32_v4i16:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vptestmd %xmm1, %xmm1, %k1
+; AVX512BWVL-NEXT:    vpminud {{.*}}(%rip){1to4}, %xmm0, %xmm0
+; AVX512BWVL-NEXT:    vpmovdw %xmm0, (%rdi) {%k1}
+; AVX512BWVL-NEXT:    retq
+  %a = icmp ne <4 x i32> %mask, zeroinitializer
+  %b = icmp ult <4 x i32> %x, <i32 65535, i32 65535, i32 65535, i32 65535>
+  %c = select <4 x i1> %b, <4 x i32> %x, <4 x i32> <i32 65535, i32 65535, i32 65535, i32 65535>
+  %d = trunc <4 x i32> %c to <4 x i16>
+  call void @llvm.masked.store.v4i16.p0v4i16(<4 x i16> %d, <4 x i16>* %p, i32 1, <4 x i1> %a)
+  ret void
+}
+
+define void @truncstore_v4i32_v4i8(<4 x i32> %x, <4 x i8>* %p, <4 x i32> %mask) {
+; AVX512F-LABEL: truncstore_v4i32_v4i8:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [255,255,255,255]
+; AVX512F-NEXT:    vpminud %xmm2, %xmm0, %xmm0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB14_2
+; AVX512F-NEXT:  # %bb.1: # %cond.store
+; AVX512F-NEXT:    vpextrb $0, %xmm0, (%rdi)
+; AVX512F-NEXT:  .LBB14_2: # %else
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $1, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB14_4
+; AVX512F-NEXT:  # %bb.3: # %cond.store1
+; AVX512F-NEXT:    vpextrb $4, %xmm0, 1(%rdi)
+; AVX512F-NEXT:  .LBB14_4: # %else2
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $2, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB14_6
+; AVX512F-NEXT:  # %bb.5: # %cond.store3
+; AVX512F-NEXT:    vpextrb $8, %xmm0, 2(%rdi)
+; AVX512F-NEXT:  .LBB14_6: # %else4
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $3, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB14_8
+; AVX512F-NEXT:  # %bb.7: # %cond.store5
+; AVX512F-NEXT:    vpextrb $12, %xmm0, 3(%rdi)
+; AVX512F-NEXT:  .LBB14_8: # %else6
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512BW-LABEL: truncstore_v4i32_v4i8:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512BW-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512BW-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [255,255,255,255]
+; AVX512BW-NEXT:    vpminud %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,4,8,12,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512BW-NEXT:    kshiftlq $60, %k0, %k0
+; AVX512BW-NEXT:    kshiftrq $60, %k0, %k1
+; AVX512BW-NEXT:    vmovdqu8 %zmm0, (%rdi) {%k1}
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+;
+; AVX512BWVL-LABEL: truncstore_v4i32_v4i8:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vptestmd %xmm1, %xmm1, %k1
+; AVX512BWVL-NEXT:    vpminud {{.*}}(%rip){1to4}, %xmm0, %xmm0
+; AVX512BWVL-NEXT:    vpmovdb %xmm0, (%rdi) {%k1}
+; AVX512BWVL-NEXT:    retq
+  %a = icmp ne <4 x i32> %mask, zeroinitializer
+  %b = icmp ult <4 x i32> %x, <i32 255, i32 255, i32 255, i32 255>
+  %c = select <4 x i1> %b, <4 x i32> %x, <4 x i32> <i32 255, i32 255, i32 255, i32 255>
+  %d = trunc <4 x i32> %c to <4 x i8>
+  call void @llvm.masked.store.v4i8.p0v4i8(<4 x i8> %d, <4 x i8>* %p, i32 1, <4 x i1> %a)
+  ret void
+}
+
+define void @truncstore_v32i16_v32i8(<32 x i16> %x, <32 x i8>* %p, <32 x i8> %mask) {
+; AVX512F-LABEL: truncstore_v32i16_v32i8:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; AVX512F-NEXT:    vpcmpeqb %ymm3, %ymm2, %ymm3
+; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm4
+; AVX512F-NEXT:    vpternlogq $15, %zmm3, %zmm3, %zmm4
+; AVX512F-NEXT:    vpmovsxbd %xmm4, %zmm4
+; AVX512F-NEXT:    vptestmd %zmm4, %zmm4, %k0
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm4 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; AVX512F-NEXT:    vpminuw %ymm4, %ymm1, %ymm1
+; AVX512F-NEXT:    vpminuw %ymm4, %ymm0, %ymm0
+; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
+; AVX512F-NEXT:    vpmovdb %zmm0, %xmm0
+; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero
+; AVX512F-NEXT:    vpmovdb %zmm1, %xmm1
+; AVX512F-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB15_2
+; AVX512F-NEXT:  # %bb.1: # %cond.store
+; AVX512F-NEXT:    vpextrb $0, %xmm0, (%rdi)
+; AVX512F-NEXT:  .LBB15_2: # %else
+; AVX512F-NEXT:    vpternlogq $15, %zmm3, %zmm3, %zmm3
+; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm1
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $1, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB15_4
+; AVX512F-NEXT:  # %bb.3: # %cond.store1
+; AVX512F-NEXT:    vpextrb $1, %xmm0, 1(%rdi)
+; AVX512F-NEXT:  .LBB15_4: # %else2
+; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512F-NEXT:    vpcmpeqb %ymm1, %ymm2, %ymm1
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm3
+; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm3
+; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
+; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
+; AVX512F-NEXT:    kshiftrw $2, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB15_6
+; AVX512F-NEXT:  # %bb.5: # %cond.store3
+; AVX512F-NEXT:    vpextrb $2, %xmm0, 2(%rdi)
+; AVX512F-NEXT:  .LBB15_6: # %else4
+; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm1
+; AVX512F-NEXT:    vpmovsxbd %xmm1, %zmm1
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $3, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB15_8
+; AVX512F-NEXT:  # %bb.7: # %cond.store5
+; AVX512F-NEXT:    vpextrb $3, %xmm0, 3(%rdi)
+; AVX512F-NEXT:  .LBB15_8: # %else6
+; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512F-NEXT:    vpcmpeqb %ymm1, %ymm2, %ymm1
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm3
+; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm3
+; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
+; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
+; AVX512F-NEXT:    kshiftrw $4, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB15_10
+; AVX512F-NEXT:  # %bb.9: # %cond.store7
+; AVX512F-NEXT:    vpextrb $4, %xmm0, 4(%rdi)
+; AVX512F-NEXT:  .LBB15_10: # %else8
+; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm1
+; AVX512F-NEXT:    vpmovsxbd %xmm1, %zmm1
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $5, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB15_12
+; AVX512F-NEXT:  # %bb.11: # %cond.store9
+; AVX512F-NEXT:    vpextrb $5, %xmm0, 5(%rdi)
+; AVX512F-NEXT:  .LBB15_12: # %else10
+; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512F-NEXT:    vpcmpeqb %ymm1, %ymm2, %ymm1
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm3
+; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm3
+; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
+; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
+; AVX512F-NEXT:    kshiftrw $6, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB15_14
+; AVX512F-NEXT:  # %bb.13: # %cond.store11
+; AVX512F-NEXT:    vpextrb $6, %xmm0, 6(%rdi)
+; AVX512F-NEXT:  .LBB15_14: # %else12
+; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm1
+; AVX512F-NEXT:    vpmovsxbd %xmm1, %zmm1
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $7, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB15_16
+; AVX512F-NEXT:  # %bb.15: # %cond.store13
+; AVX512F-NEXT:    vpextrb $7, %xmm0, 7(%rdi)
+; AVX512F-NEXT:  .LBB15_16: # %else14
+; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512F-NEXT:    vpcmpeqb %ymm1, %ymm2, %ymm1
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm3
+; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm3
+; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
+; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
+; AVX512F-NEXT:    kshiftrw $8, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB15_18
+; AVX512F-NEXT:  # %bb.17: # %cond.store15
+; AVX512F-NEXT:    vpextrb $8, %xmm0, 8(%rdi)
+; AVX512F-NEXT:  .LBB15_18: # %else16
+; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm1
+; AVX512F-NEXT:    vpmovsxbd %xmm1, %zmm1
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $9, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB15_20
+; AVX512F-NEXT:  # %bb.19: # %cond.store17
+; AVX512F-NEXT:    vpextrb $9, %xmm0, 9(%rdi)
+; AVX512F-NEXT:  .LBB15_20: # %else18
+; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512F-NEXT:    vpcmpeqb %ymm1, %ymm2, %ymm1
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm3
+; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm3
+; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
+; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
+; AVX512F-NEXT:    kshiftrw $10, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB15_22
+; AVX512F-NEXT:  # %bb.21: # %cond.store19
+; AVX512F-NEXT:    vpextrb $10, %xmm0, 10(%rdi)
+; AVX512F-NEXT:  .LBB15_22: # %else20
+; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm1
+; AVX512F-NEXT:    vpmovsxbd %xmm1, %zmm1
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $11, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB15_24
+; AVX512F-NEXT:  # %bb.23: # %cond.store21
+; AVX512F-NEXT:    vpextrb $11, %xmm0, 11(%rdi)
+; AVX512F-NEXT:  .LBB15_24: # %else22
+; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512F-NEXT:    vpcmpeqb %ymm1, %ymm2, %ymm1
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm3
+; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm3
+; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
+; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
+; AVX512F-NEXT:    kshiftrw $12, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB15_26
+; AVX512F-NEXT:  # %bb.25: # %cond.store23
+; AVX512F-NEXT:    vpextrb $12, %xmm0, 12(%rdi)
+; AVX512F-NEXT:  .LBB15_26: # %else24
+; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm1
+; AVX512F-NEXT:    vpmovsxbd %xmm1, %zmm1
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $13, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB15_28
+; AVX512F-NEXT:  # %bb.27: # %cond.store25
+; AVX512F-NEXT:    vpextrb $13, %xmm0, 13(%rdi)
+; AVX512F-NEXT:  .LBB15_28: # %else26
+; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512F-NEXT:    vpcmpeqb %ymm1, %ymm2, %ymm1
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm3
+; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm3
+; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
+; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
+; AVX512F-NEXT:    kshiftrw $14, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB15_30
+; AVX512F-NEXT:  # %bb.29: # %cond.store27
+; AVX512F-NEXT:    vpextrb $14, %xmm0, 14(%rdi)
+; AVX512F-NEXT:  .LBB15_30: # %else28
+; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm1
+; AVX512F-NEXT:    vpmovsxbd %xmm1, %zmm1
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $15, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB15_32
+; AVX512F-NEXT:  # %bb.31: # %cond.store29
+; AVX512F-NEXT:    vpextrb $15, %xmm0, 15(%rdi)
+; AVX512F-NEXT:  .LBB15_32: # %else30
+; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512F-NEXT:    vpcmpeqb %ymm1, %ymm2, %ymm1
+; AVX512F-NEXT:    vextracti128 $1, %ymm1, %xmm1
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm3
+; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm3
+; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
+; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB15_34
+; AVX512F-NEXT:  # %bb.33: # %cond.store31
+; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm3
+; AVX512F-NEXT:    vpextrb $0, %xmm3, 16(%rdi)
+; AVX512F-NEXT:  .LBB15_34: # %else32
+; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm1
+; AVX512F-NEXT:    vpmovsxbd %xmm1, %zmm1
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $1, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB15_36
+; AVX512F-NEXT:  # %bb.35: # %cond.store33
+; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX512F-NEXT:    vpextrb $1, %xmm1, 17(%rdi)
+; AVX512F-NEXT:  .LBB15_36: # %else34
+; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512F-NEXT:    vpcmpeqb %ymm1, %ymm2, %ymm1
+; AVX512F-NEXT:    vextracti128 $1, %ymm1, %xmm1
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm3
+; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm3
+; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
+; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
+; AVX512F-NEXT:    kshiftrw $2, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB15_38
+; AVX512F-NEXT:  # %bb.37: # %cond.store35
+; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm3
+; AVX512F-NEXT:    vpextrb $2, %xmm3, 18(%rdi)
+; AVX512F-NEXT:  .LBB15_38: # %else36
+; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm1
+; AVX512F-NEXT:    vpmovsxbd %xmm1, %zmm1
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $3, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB15_40
+; AVX512F-NEXT:  # %bb.39: # %cond.store37
+; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX512F-NEXT:    vpextrb $3, %xmm1, 19(%rdi)
+; AVX512F-NEXT:  .LBB15_40: # %else38
+; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512F-NEXT:    vpcmpeqb %ymm1, %ymm2, %ymm1
+; AVX512F-NEXT:    vextracti128 $1, %ymm1, %xmm1
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm3
+; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm3
+; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
+; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
+; AVX512F-NEXT:    kshiftrw $4, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB15_42
+; AVX512F-NEXT:  # %bb.41: # %cond.store39
+; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm3
+; AVX512F-NEXT:    vpextrb $4, %xmm3, 20(%rdi)
+; AVX512F-NEXT:  .LBB15_42: # %else40
+; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm1
+; AVX512F-NEXT:    vpmovsxbd %xmm1, %zmm1
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $5, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB15_44
+; AVX512F-NEXT:  # %bb.43: # %cond.store41
+; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX512F-NEXT:    vpextrb $5, %xmm1, 21(%rdi)
+; AVX512F-NEXT:  .LBB15_44: # %else42
+; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512F-NEXT:    vpcmpeqb %ymm1, %ymm2, %ymm1
+; AVX512F-NEXT:    vextracti128 $1, %ymm1, %xmm1
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm3
+; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm3
+; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
+; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
+; AVX512F-NEXT:    kshiftrw $6, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB15_46
+; AVX512F-NEXT:  # %bb.45: # %cond.store43
+; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm3
+; AVX512F-NEXT:    vpextrb $6, %xmm3, 22(%rdi)
+; AVX512F-NEXT:  .LBB15_46: # %else44
+; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm1
+; AVX512F-NEXT:    vpmovsxbd %xmm1, %zmm1
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $7, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB15_48
+; AVX512F-NEXT:  # %bb.47: # %cond.store45
+; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX512F-NEXT:    vpextrb $7, %xmm1, 23(%rdi)
+; AVX512F-NEXT:  .LBB15_48: # %else46
+; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512F-NEXT:    vpcmpeqb %ymm1, %ymm2, %ymm1
+; AVX512F-NEXT:    vextracti128 $1, %ymm1, %xmm1
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm3
+; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm3
+; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
+; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
+; AVX512F-NEXT:    kshiftrw $8, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB15_50
+; AVX512F-NEXT:  # %bb.49: # %cond.store47
+; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm3
+; AVX512F-NEXT:    vpextrb $8, %xmm3, 24(%rdi)
+; AVX512F-NEXT:  .LBB15_50: # %else48
+; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm1
+; AVX512F-NEXT:    vpmovsxbd %xmm1, %zmm1
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $9, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB15_52
+; AVX512F-NEXT:  # %bb.51: # %cond.store49
+; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX512F-NEXT:    vpextrb $9, %xmm1, 25(%rdi)
+; AVX512F-NEXT:  .LBB15_52: # %else50
+; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512F-NEXT:    vpcmpeqb %ymm1, %ymm2, %ymm1
+; AVX512F-NEXT:    vextracti128 $1, %ymm1, %xmm1
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm3
+; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm3
+; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
+; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
+; AVX512F-NEXT:    kshiftrw $10, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB15_54
+; AVX512F-NEXT:  # %bb.53: # %cond.store51
+; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm3
+; AVX512F-NEXT:    vpextrb $10, %xmm3, 26(%rdi)
+; AVX512F-NEXT:  .LBB15_54: # %else52
+; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm1
+; AVX512F-NEXT:    vpmovsxbd %xmm1, %zmm1
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $11, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB15_56
+; AVX512F-NEXT:  # %bb.55: # %cond.store53
+; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX512F-NEXT:    vpextrb $11, %xmm1, 27(%rdi)
+; AVX512F-NEXT:  .LBB15_56: # %else54
+; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512F-NEXT:    vpcmpeqb %ymm1, %ymm2, %ymm1
+; AVX512F-NEXT:    vextracti128 $1, %ymm1, %xmm1
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm3
+; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm3
+; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
+; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
+; AVX512F-NEXT:    kshiftrw $12, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB15_58
+; AVX512F-NEXT:  # %bb.57: # %cond.store55
+; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm3
+; AVX512F-NEXT:    vpextrb $12, %xmm3, 28(%rdi)
+; AVX512F-NEXT:  .LBB15_58: # %else56
+; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm1
+; AVX512F-NEXT:    vpmovsxbd %xmm1, %zmm1
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $13, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB15_60
+; AVX512F-NEXT:  # %bb.59: # %cond.store57
+; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX512F-NEXT:    vpextrb $13, %xmm1, 29(%rdi)
+; AVX512F-NEXT:  .LBB15_60: # %else58
+; AVX512F-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX512F-NEXT:    vpcmpeqb %ymm1, %ymm2, %ymm1
+; AVX512F-NEXT:    vextracti128 $1, %ymm1, %xmm1
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm2
+; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm2
+; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
+; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
+; AVX512F-NEXT:    kshiftrw $14, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB15_62
+; AVX512F-NEXT:  # %bb.61: # %cond.store59
+; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm2
+; AVX512F-NEXT:    vpextrb $14, %xmm2, 30(%rdi)
+; AVX512F-NEXT:  .LBB15_62: # %else60
+; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm1
+; AVX512F-NEXT:    vpmovsxbd %xmm1, %zmm1
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $15, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB15_64
+; AVX512F-NEXT:  # %bb.63: # %cond.store61
+; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm0
+; AVX512F-NEXT:    vpextrb $15, %xmm0, 31(%rdi)
+; AVX512F-NEXT:  .LBB15_64: # %else62
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512BW-LABEL: truncstore_v32i16_v32i8:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512BW-NEXT:    vptestmb %zmm1, %zmm1, %k1
+; AVX512BW-NEXT:    vpminuw {{.*}}(%rip), %zmm0, %zmm0
+; AVX512BW-NEXT:    vpmovwb %zmm0, (%rdi) {%k1}
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+;
+; AVX512BWVL-LABEL: truncstore_v32i16_v32i8:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vptestmb %ymm1, %ymm1, %k1
+; AVX512BWVL-NEXT:    vpminuw {{.*}}(%rip), %zmm0, %zmm0
+; AVX512BWVL-NEXT:    vpmovwb %zmm0, (%rdi) {%k1}
+; AVX512BWVL-NEXT:    vzeroupper
+; AVX512BWVL-NEXT:    retq
+  %a = icmp ne <32 x i8> %mask, zeroinitializer
+  %b = icmp ult <32 x i16> %x, <i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255>
+  %c = select <32 x i1> %b, <32 x i16> %x, <32 x i16> <i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255>
+  %d = trunc <32 x i16> %c to <32 x i8>
+  call void @llvm.masked.store.v32i8.p0v32i8(<32 x i8> %d, <32 x i8>* %p, i32 1, <32 x i1> %a)
+  ret void
+}
+
+define void @truncstore_v16i16_v16i8(<16 x i16> %x, <16 x i8>* %p, <16 x i8> %mask) {
+; AVX512F-LABEL: truncstore_v16i16_v16i8:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
+; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3
+; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm3
+; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
+; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
+; AVX512F-NEXT:    vpminuw {{.*}}(%rip), %ymm0, %ymm0
+; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
+; AVX512F-NEXT:    vpmovdb %zmm0, %xmm0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB16_2
+; AVX512F-NEXT:  # %bb.1: # %cond.store
+; AVX512F-NEXT:    vpextrb $0, %xmm0, (%rdi)
+; AVX512F-NEXT:  .LBB16_2: # %else
+; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm2
+; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
+; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
+; AVX512F-NEXT:    kshiftrw $1, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB16_4
+; AVX512F-NEXT:  # %bb.3: # %cond.store1
+; AVX512F-NEXT:    vpextrb $1, %xmm0, 1(%rdi)
+; AVX512F-NEXT:  .LBB16_4: # %else2
+; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
+; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3
+; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm3
+; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
+; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
+; AVX512F-NEXT:    kshiftrw $2, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB16_6
+; AVX512F-NEXT:  # %bb.5: # %cond.store3
+; AVX512F-NEXT:    vpextrb $2, %xmm0, 2(%rdi)
+; AVX512F-NEXT:  .LBB16_6: # %else4
+; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm2
+; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
+; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
+; AVX512F-NEXT:    kshiftrw $3, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB16_8
+; AVX512F-NEXT:  # %bb.7: # %cond.store5
+; AVX512F-NEXT:    vpextrb $3, %xmm0, 3(%rdi)
+; AVX512F-NEXT:  .LBB16_8: # %else6
+; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
+; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3
+; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm3
+; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
+; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
+; AVX512F-NEXT:    kshiftrw $4, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB16_10
+; AVX512F-NEXT:  # %bb.9: # %cond.store7
+; AVX512F-NEXT:    vpextrb $4, %xmm0, 4(%rdi)
+; AVX512F-NEXT:  .LBB16_10: # %else8
+; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm2
+; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
+; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
+; AVX512F-NEXT:    kshiftrw $5, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB16_12
+; AVX512F-NEXT:  # %bb.11: # %cond.store9
+; AVX512F-NEXT:    vpextrb $5, %xmm0, 5(%rdi)
+; AVX512F-NEXT:  .LBB16_12: # %else10
+; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
+; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3
+; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm3
+; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
+; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
+; AVX512F-NEXT:    kshiftrw $6, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB16_14
+; AVX512F-NEXT:  # %bb.13: # %cond.store11
+; AVX512F-NEXT:    vpextrb $6, %xmm0, 6(%rdi)
+; AVX512F-NEXT:  .LBB16_14: # %else12
+; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm2
+; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
+; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
+; AVX512F-NEXT:    kshiftrw $7, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB16_16
+; AVX512F-NEXT:  # %bb.15: # %cond.store13
+; AVX512F-NEXT:    vpextrb $7, %xmm0, 7(%rdi)
+; AVX512F-NEXT:  .LBB16_16: # %else14
+; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
+; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3
+; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm3
+; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
+; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
+; AVX512F-NEXT:    kshiftrw $8, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB16_18
+; AVX512F-NEXT:  # %bb.17: # %cond.store15
+; AVX512F-NEXT:    vpextrb $8, %xmm0, 8(%rdi)
+; AVX512F-NEXT:  .LBB16_18: # %else16
+; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm2
+; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
+; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
+; AVX512F-NEXT:    kshiftrw $9, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB16_20
+; AVX512F-NEXT:  # %bb.19: # %cond.store17
+; AVX512F-NEXT:    vpextrb $9, %xmm0, 9(%rdi)
+; AVX512F-NEXT:  .LBB16_20: # %else18
+; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
+; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3
+; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm3
+; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
+; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
+; AVX512F-NEXT:    kshiftrw $10, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB16_22
+; AVX512F-NEXT:  # %bb.21: # %cond.store19
+; AVX512F-NEXT:    vpextrb $10, %xmm0, 10(%rdi)
+; AVX512F-NEXT:  .LBB16_22: # %else20
+; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm2
+; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
+; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
+; AVX512F-NEXT:    kshiftrw $11, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB16_24
+; AVX512F-NEXT:  # %bb.23: # %cond.store21
+; AVX512F-NEXT:    vpextrb $11, %xmm0, 11(%rdi)
+; AVX512F-NEXT:  .LBB16_24: # %else22
+; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm2
+; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3
+; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm3
+; AVX512F-NEXT:    vpmovsxbd %xmm3, %zmm3
+; AVX512F-NEXT:    vptestmd %zmm3, %zmm3, %k0
+; AVX512F-NEXT:    kshiftrw $12, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB16_26
+; AVX512F-NEXT:  # %bb.25: # %cond.store23
+; AVX512F-NEXT:    vpextrb $12, %xmm0, 12(%rdi)
+; AVX512F-NEXT:  .LBB16_26: # %else24
+; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm2
+; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
+; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
+; AVX512F-NEXT:    kshiftrw $13, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB16_28
+; AVX512F-NEXT:  # %bb.27: # %cond.store25
+; AVX512F-NEXT:    vpextrb $13, %xmm0, 13(%rdi)
+; AVX512F-NEXT:  .LBB16_28: # %else26
+; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX512F-NEXT:    vpcmpeqb %xmm2, %xmm1, %xmm1
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm2
+; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm2
+; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm2
+; AVX512F-NEXT:    vptestmd %zmm2, %zmm2, %k0
+; AVX512F-NEXT:    kshiftrw $14, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB16_30
+; AVX512F-NEXT:  # %bb.29: # %cond.store27
+; AVX512F-NEXT:    vpextrb $14, %xmm0, 14(%rdi)
+; AVX512F-NEXT:  .LBB16_30: # %else28
+; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm1
+; AVX512F-NEXT:    vpmovsxbd %xmm1, %zmm1
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $15, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB16_32
+; AVX512F-NEXT:  # %bb.31: # %cond.store29
+; AVX512F-NEXT:    vpextrb $15, %xmm0, 15(%rdi)
+; AVX512F-NEXT:  .LBB16_32: # %else30
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512BW-LABEL: truncstore_v16i16_v16i8:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512BW-NEXT:    vptestmb %zmm1, %zmm1, %k0
+; AVX512BW-NEXT:    kmovw %k0, %k1
+; AVX512BW-NEXT:    vpminuw {{.*}}(%rip), %ymm0, %ymm0
+; AVX512BW-NEXT:    vpmovwb %zmm0, %ymm0
+; AVX512BW-NEXT:    vmovdqu8 %zmm0, (%rdi) {%k1}
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+;
+; AVX512BWVL-LABEL: truncstore_v16i16_v16i8:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vptestmb %xmm1, %xmm1, %k1
+; AVX512BWVL-NEXT:    vpminuw {{.*}}(%rip), %ymm0, %ymm0
+; AVX512BWVL-NEXT:    vpmovwb %ymm0, (%rdi) {%k1}
+; AVX512BWVL-NEXT:    vzeroupper
+; AVX512BWVL-NEXT:    retq
+  %a = icmp ne <16 x i8> %mask, zeroinitializer
+  %b = icmp ult <16 x i16> %x, <i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255>
+  %c = select <16 x i1> %b, <16 x i16> %x, <16 x i16> <i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255>
+  %d = trunc <16 x i16> %c to <16 x i8>
+  call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %d, <16 x i8>* %p, i32 1, <16 x i1> %a)
+  ret void
+}
+
+define void @truncstore_v8i16_v8i8(<8 x i16> %x, <8 x i8>* %p, <8 x i16> %mask) {
+; AVX512F-LABEL: truncstore_v8i16_v8i8:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX512F-NEXT:    vpcmpeqw %xmm2, %xmm1, %xmm2
+; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3
+; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm3
+; AVX512F-NEXT:    vpmovsxwq %xmm3, %zmm3
+; AVX512F-NEXT:    vptestmq %zmm3, %zmm3, %k0
+; AVX512F-NEXT:    vpminuw {{.*}}(%rip), %xmm0, %xmm0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB17_2
+; AVX512F-NEXT:  # %bb.1: # %cond.store
+; AVX512F-NEXT:    vpextrb $0, %xmm0, (%rdi)
+; AVX512F-NEXT:  .LBB17_2: # %else
+; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm2
+; AVX512F-NEXT:    vpmovsxwq %xmm2, %zmm2
+; AVX512F-NEXT:    vptestmq %zmm2, %zmm2, %k0
+; AVX512F-NEXT:    kshiftrw $1, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB17_4
+; AVX512F-NEXT:  # %bb.3: # %cond.store1
+; AVX512F-NEXT:    vpextrb $2, %xmm0, 1(%rdi)
+; AVX512F-NEXT:  .LBB17_4: # %else2
+; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX512F-NEXT:    vpcmpeqw %xmm2, %xmm1, %xmm2
+; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3
+; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm3
+; AVX512F-NEXT:    vpmovsxwq %xmm3, %zmm3
+; AVX512F-NEXT:    vptestmq %zmm3, %zmm3, %k0
+; AVX512F-NEXT:    kshiftrw $2, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB17_6
+; AVX512F-NEXT:  # %bb.5: # %cond.store3
+; AVX512F-NEXT:    vpextrb $4, %xmm0, 2(%rdi)
+; AVX512F-NEXT:  .LBB17_6: # %else4
+; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm2
+; AVX512F-NEXT:    vpmovsxwq %xmm2, %zmm2
+; AVX512F-NEXT:    vptestmq %zmm2, %zmm2, %k0
+; AVX512F-NEXT:    kshiftrw $3, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB17_8
+; AVX512F-NEXT:  # %bb.7: # %cond.store5
+; AVX512F-NEXT:    vpextrb $6, %xmm0, 3(%rdi)
+; AVX512F-NEXT:  .LBB17_8: # %else6
+; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX512F-NEXT:    vpcmpeqw %xmm2, %xmm1, %xmm2
+; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3
+; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm3
+; AVX512F-NEXT:    vpmovsxwq %xmm3, %zmm3
+; AVX512F-NEXT:    vptestmq %zmm3, %zmm3, %k0
+; AVX512F-NEXT:    kshiftrw $4, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB17_10
+; AVX512F-NEXT:  # %bb.9: # %cond.store7
+; AVX512F-NEXT:    vpextrb $8, %xmm0, 4(%rdi)
+; AVX512F-NEXT:  .LBB17_10: # %else8
+; AVX512F-NEXT:    vpternlogq $15, %zmm2, %zmm2, %zmm2
+; AVX512F-NEXT:    vpmovsxwq %xmm2, %zmm2
+; AVX512F-NEXT:    vptestmq %zmm2, %zmm2, %k0
+; AVX512F-NEXT:    kshiftrw $5, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB17_12
+; AVX512F-NEXT:  # %bb.11: # %cond.store9
+; AVX512F-NEXT:    vpextrb $10, %xmm0, 5(%rdi)
+; AVX512F-NEXT:  .LBB17_12: # %else10
+; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX512F-NEXT:    vpcmpeqw %xmm2, %xmm1, %xmm1
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm2
+; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm2
+; AVX512F-NEXT:    vpmovsxwq %xmm2, %zmm2
+; AVX512F-NEXT:    vptestmq %zmm2, %zmm2, %k0
+; AVX512F-NEXT:    kshiftrw $6, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB17_14
+; AVX512F-NEXT:  # %bb.13: # %cond.store11
+; AVX512F-NEXT:    vpextrb $12, %xmm0, 6(%rdi)
+; AVX512F-NEXT:  .LBB17_14: # %else12
+; AVX512F-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm1
+; AVX512F-NEXT:    vpmovsxwq %xmm1, %zmm1
+; AVX512F-NEXT:    vptestmq %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    kshiftrw $7, %k0, %k0
+; AVX512F-NEXT:    kmovw %k0, %eax
+; AVX512F-NEXT:    testb $1, %al
+; AVX512F-NEXT:    je .LBB17_16
+; AVX512F-NEXT:  # %bb.15: # %cond.store13
+; AVX512F-NEXT:    vpextrb $14, %xmm0, 7(%rdi)
+; AVX512F-NEXT:  .LBB17_16: # %else14
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512BW-LABEL: truncstore_v8i16_v8i8:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512BW-NEXT:    vptestmw %zmm1, %zmm1, %k0
+; AVX512BW-NEXT:    vpminuw {{.*}}(%rip), %xmm0, %xmm0
+; AVX512BW-NEXT:    vpackuswb %xmm0, %xmm0, %xmm0
+; AVX512BW-NEXT:    kshiftlq $56, %k0, %k0
+; AVX512BW-NEXT:    kshiftrq $56, %k0, %k1
+; AVX512BW-NEXT:    vmovdqu8 %zmm0, (%rdi) {%k1}
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+;
+; AVX512BWVL-LABEL: truncstore_v8i16_v8i8:
+; AVX512BWVL:       # %bb.0:
+; AVX512BWVL-NEXT:    vptestmw %xmm1, %xmm1, %k1
+; AVX512BWVL-NEXT:    vpminuw {{.*}}(%rip), %xmm0, %xmm0
+; AVX512BWVL-NEXT:    vpmovwb %xmm0, (%rdi) {%k1}
+; AVX512BWVL-NEXT:    retq
+  %a = icmp ne <8 x i16> %mask, zeroinitializer
+  %b = icmp ult <8 x i16> %x, <i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255>
+  %c = select <8 x i1> %b, <8 x i16> %x, <8 x i16> <i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255>
+  %d = trunc <8 x i16> %c to <8 x i8>
+  call void @llvm.masked.store.v8i8.p0v8i8(<8 x i8> %d, <8 x i8>* %p, i32 1, <8 x i1> %a)
+  ret void
+}
+
+declare void @llvm.masked.store.v8i32.p0v8i32(<8 x i32>, <8 x i32>*, i32, <8 x i1>)
+declare void @llvm.masked.store.v8i16.p0v8i16(<8 x i16>, <8 x i16>*, i32, <8 x i1>)
+declare void @llvm.masked.store.v8i8.p0v8i8(<8 x i8>, <8 x i8>*, i32, <8 x i1>)
+declare void @llvm.masked.store.v4i32.p0v4i32(<4 x i32>, <4 x i32>*, i32, <4 x i1>)
+declare void @llvm.masked.store.v4i16.p0v4i16(<4 x i16>, <4 x i16>*, i32, <4 x i1>)
+declare void @llvm.masked.store.v4i8.p0v4i8(<4 x i8>, <4 x i8>*, i32, <4 x i1>)
+declare void @llvm.masked.store.v2i32.p0v2i32(<2 x i32>, <2 x i32>*, i32, <2 x i1>)
+declare void @llvm.masked.store.v2i16.p0v2i16(<2 x i16>, <2 x i16>*, i32, <2 x i1>)
+declare void @llvm.masked.store.v2i8.p0v2i8(<2 x i8>, <2 x i8>*, i32, <2 x i1>)
+declare void @llvm.masked.store.v16i16.p0v16i16(<16 x i16>, <16 x i16>*, i32, <16 x i1>)
+declare void @llvm.masked.store.v16i8.p0v16i8(<16 x i8>, <16 x i8>*, i32, <16 x i1>)
+declare void @llvm.masked.store.v32i8.p0v32i8(<32 x i8>, <32 x i8>*, i32, <32 x i1>)
diff --git a/test/CodeGen/X86/memcmp-optsize.ll b/test/CodeGen/X86/memcmp-optsize.ll
index 7683d1a4b311441043a7c754401c75575ffc7963..d2b390f0943a0cb93bb1052700d56d95053fe5cd 100644
--- a/test/CodeGen/X86/memcmp-optsize.ll
+++ b/test/CodeGen/X86/memcmp-optsize.ll
@@ -639,17 +639,33 @@ define i32 @length24(i8* %X, i8* %Y) nounwind optsize {
 }
 
 define i1 @length24_eq(i8* %x, i8* %y) nounwind optsize {
-; X86-LABEL: length24_eq:
-; X86:       # %bb.0:
-; X86-NEXT:    pushl $0
-; X86-NEXT:    pushl $24
-; X86-NEXT:    pushl {{[0-9]+}}(%esp)
-; X86-NEXT:    pushl {{[0-9]+}}(%esp)
-; X86-NEXT:    calll memcmp
-; X86-NEXT:    addl $16, %esp
-; X86-NEXT:    testl %eax, %eax
-; X86-NEXT:    sete %al
-; X86-NEXT:    retl
+; X86-NOSSE-LABEL: length24_eq:
+; X86-NOSSE:       # %bb.0:
+; X86-NOSSE-NEXT:    pushl $0
+; X86-NOSSE-NEXT:    pushl $24
+; X86-NOSSE-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    calll memcmp
+; X86-NOSSE-NEXT:    addl $16, %esp
+; X86-NOSSE-NEXT:    testl %eax, %eax
+; X86-NOSSE-NEXT:    sete %al
+; X86-NOSSE-NEXT:    retl
+;
+; X86-SSE2-LABEL: length24_eq:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-SSE2-NEXT:    movdqu (%ecx), %xmm0
+; X86-SSE2-NEXT:    movdqu 8(%ecx), %xmm1
+; X86-SSE2-NEXT:    movdqu (%eax), %xmm2
+; X86-SSE2-NEXT:    pcmpeqb %xmm0, %xmm2
+; X86-SSE2-NEXT:    movdqu 8(%eax), %xmm0
+; X86-SSE2-NEXT:    pcmpeqb %xmm1, %xmm0
+; X86-SSE2-NEXT:    pand %xmm2, %xmm0
+; X86-SSE2-NEXT:    pmovmskb %xmm0, %eax
+; X86-SSE2-NEXT:    cmpl $65535, %eax # imm = 0xFFFF
+; X86-SSE2-NEXT:    sete %al
+; X86-SSE2-NEXT:    retl
 ;
 ; X64-SSE2-LABEL: length24_eq:
 ; X64-SSE2:       # %bb.0:
@@ -683,17 +699,30 @@ define i1 @length24_eq(i8* %x, i8* %y) nounwind optsize {
 }
 
 define i1 @length24_eq_const(i8* %X) nounwind optsize {
-; X86-LABEL: length24_eq_const:
-; X86:       # %bb.0:
-; X86-NEXT:    pushl $0
-; X86-NEXT:    pushl $24
-; X86-NEXT:    pushl $.L.str
-; X86-NEXT:    pushl {{[0-9]+}}(%esp)
-; X86-NEXT:    calll memcmp
-; X86-NEXT:    addl $16, %esp
-; X86-NEXT:    testl %eax, %eax
-; X86-NEXT:    setne %al
-; X86-NEXT:    retl
+; X86-NOSSE-LABEL: length24_eq_const:
+; X86-NOSSE:       # %bb.0:
+; X86-NOSSE-NEXT:    pushl $0
+; X86-NOSSE-NEXT:    pushl $24
+; X86-NOSSE-NEXT:    pushl $.L.str
+; X86-NOSSE-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    calll memcmp
+; X86-NOSSE-NEXT:    addl $16, %esp
+; X86-NOSSE-NEXT:    testl %eax, %eax
+; X86-NOSSE-NEXT:    setne %al
+; X86-NOSSE-NEXT:    retl
+;
+; X86-SSE2-LABEL: length24_eq_const:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT:    movdqu (%eax), %xmm0
+; X86-SSE2-NEXT:    movdqu 8(%eax), %xmm1
+; X86-SSE2-NEXT:    pcmpeqb {{\.LCPI.*}}, %xmm1
+; X86-SSE2-NEXT:    pcmpeqb {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand %xmm1, %xmm0
+; X86-SSE2-NEXT:    pmovmskb %xmm0, %eax
+; X86-SSE2-NEXT:    cmpl $65535, %eax # imm = 0xFFFF
+; X86-SSE2-NEXT:    setne %al
+; X86-SSE2-NEXT:    retl
 ;
 ; X64-SSE2-LABEL: length24_eq_const:
 ; X64-SSE2:       # %bb.0:
diff --git a/test/CodeGen/X86/memcmp.ll b/test/CodeGen/X86/memcmp.ll
index 0bb46eeeac92bdb7c4fa13f7f03a604365d30447..371c16881ff30da3847dd180e57eb68805e99b7a 100644
--- a/test/CodeGen/X86/memcmp.ll
+++ b/test/CodeGen/X86/memcmp.ll
@@ -362,24 +362,24 @@ define i1 @length5_eq(i8* %X, i8* %Y) nounwind {
 define i1 @length7_eq(i8* %X, i8* %Y) nounwind {
 ; X86-LABEL: length7_eq:
 ; X86:       # %bb.0:
-; X86-NEXT:    pushl $0
-; X86-NEXT:    pushl $7
-; X86-NEXT:    pushl {{[0-9]+}}(%esp)
-; X86-NEXT:    pushl {{[0-9]+}}(%esp)
-; X86-NEXT:    calll memcmp
-; X86-NEXT:    addl $16, %esp
-; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl (%ecx), %edx
+; X86-NEXT:    movl 3(%ecx), %ecx
+; X86-NEXT:    xorl (%eax), %edx
+; X86-NEXT:    xorl 3(%eax), %ecx
+; X86-NEXT:    orl %edx, %ecx
 ; X86-NEXT:    setne %al
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: length7_eq:
 ; X64:       # %bb.0:
-; X64-NEXT:    pushq %rax
-; X64-NEXT:    movl $7, %edx
-; X64-NEXT:    callq memcmp
-; X64-NEXT:    testl %eax, %eax
+; X64-NEXT:    movl (%rdi), %eax
+; X64-NEXT:    movl 3(%rdi), %ecx
+; X64-NEXT:    xorl (%rsi), %eax
+; X64-NEXT:    xorl 3(%rsi), %ecx
+; X64-NEXT:    orl %eax, %ecx
 ; X64-NEXT:    setne %al
-; X64-NEXT:    popq %rcx
 ; X64-NEXT:    retq
   %m = tail call i32 @memcmp(i8* %X, i8* %Y, i64 7) nounwind
   %c = icmp ne i32 %m, 0
@@ -548,12 +548,12 @@ define i1 @length11_eq(i8* %X, i8* %Y) nounwind {
 ;
 ; X64-LABEL: length11_eq:
 ; X64:       # %bb.0:
-; X64-NEXT:    pushq %rax
-; X64-NEXT:    movl $11, %edx
-; X64-NEXT:    callq memcmp
-; X64-NEXT:    testl %eax, %eax
+; X64-NEXT:    movq (%rdi), %rax
+; X64-NEXT:    movq 3(%rdi), %rcx
+; X64-NEXT:    xorq (%rsi), %rax
+; X64-NEXT:    xorq 3(%rsi), %rcx
+; X64-NEXT:    orq %rax, %rcx
 ; X64-NEXT:    sete %al
-; X64-NEXT:    popq %rcx
 ; X64-NEXT:    retq
   %m = tail call i32 @memcmp(i8* %X, i8* %Y, i64 11) nounwind
   %c = icmp eq i32 %m, 0
@@ -640,12 +640,12 @@ define i1 @length13_eq(i8* %X, i8* %Y) nounwind {
 ;
 ; X64-LABEL: length13_eq:
 ; X64:       # %bb.0:
-; X64-NEXT:    pushq %rax
-; X64-NEXT:    movl $13, %edx
-; X64-NEXT:    callq memcmp
-; X64-NEXT:    testl %eax, %eax
+; X64-NEXT:    movq (%rdi), %rax
+; X64-NEXT:    movq 5(%rdi), %rcx
+; X64-NEXT:    xorq (%rsi), %rax
+; X64-NEXT:    xorq 5(%rsi), %rcx
+; X64-NEXT:    orq %rax, %rcx
 ; X64-NEXT:    sete %al
-; X64-NEXT:    popq %rcx
 ; X64-NEXT:    retq
   %m = tail call i32 @memcmp(i8* %X, i8* %Y, i64 13) nounwind
   %c = icmp eq i32 %m, 0
@@ -667,12 +667,12 @@ define i1 @length14_eq(i8* %X, i8* %Y) nounwind {
 ;
 ; X64-LABEL: length14_eq:
 ; X64:       # %bb.0:
-; X64-NEXT:    pushq %rax
-; X64-NEXT:    movl $14, %edx
-; X64-NEXT:    callq memcmp
-; X64-NEXT:    testl %eax, %eax
+; X64-NEXT:    movq (%rdi), %rax
+; X64-NEXT:    movq 6(%rdi), %rcx
+; X64-NEXT:    xorq (%rsi), %rax
+; X64-NEXT:    xorq 6(%rsi), %rcx
+; X64-NEXT:    orq %rax, %rcx
 ; X64-NEXT:    sete %al
-; X64-NEXT:    popq %rcx
 ; X64-NEXT:    retq
   %m = tail call i32 @memcmp(i8* %X, i8* %Y, i64 14) nounwind
   %c = icmp eq i32 %m, 0
@@ -694,12 +694,12 @@ define i1 @length15_eq(i8* %X, i8* %Y) nounwind {
 ;
 ; X64-LABEL: length15_eq:
 ; X64:       # %bb.0:
-; X64-NEXT:    pushq %rax
-; X64-NEXT:    movl $15, %edx
-; X64-NEXT:    callq memcmp
-; X64-NEXT:    testl %eax, %eax
+; X64-NEXT:    movq (%rdi), %rax
+; X64-NEXT:    movq 7(%rdi), %rcx
+; X64-NEXT:    xorq (%rsi), %rax
+; X64-NEXT:    xorq 7(%rsi), %rcx
+; X64-NEXT:    orq %rax, %rcx
 ; X64-NEXT:    sete %al
-; X64-NEXT:    popq %rcx
 ; X64-NEXT:    retq
   %m = tail call i32 @memcmp(i8* %X, i8* %Y, i64 15) nounwind
   %c = icmp eq i32 %m, 0
@@ -885,17 +885,45 @@ define i32 @length24(i8* %X, i8* %Y) nounwind {
 }
 
 define i1 @length24_eq(i8* %x, i8* %y) nounwind {
-; X86-LABEL: length24_eq:
-; X86:       # %bb.0:
-; X86-NEXT:    pushl $0
-; X86-NEXT:    pushl $24
-; X86-NEXT:    pushl {{[0-9]+}}(%esp)
-; X86-NEXT:    pushl {{[0-9]+}}(%esp)
-; X86-NEXT:    calll memcmp
-; X86-NEXT:    addl $16, %esp
-; X86-NEXT:    testl %eax, %eax
-; X86-NEXT:    sete %al
-; X86-NEXT:    retl
+; X86-NOSSE-LABEL: length24_eq:
+; X86-NOSSE:       # %bb.0:
+; X86-NOSSE-NEXT:    pushl $0
+; X86-NOSSE-NEXT:    pushl $24
+; X86-NOSSE-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    calll memcmp
+; X86-NOSSE-NEXT:    addl $16, %esp
+; X86-NOSSE-NEXT:    testl %eax, %eax
+; X86-NOSSE-NEXT:    sete %al
+; X86-NOSSE-NEXT:    retl
+;
+; X86-SSE1-LABEL: length24_eq:
+; X86-SSE1:       # %bb.0:
+; X86-SSE1-NEXT:    pushl $0
+; X86-SSE1-NEXT:    pushl $24
+; X86-SSE1-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-SSE1-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-SSE1-NEXT:    calll memcmp
+; X86-SSE1-NEXT:    addl $16, %esp
+; X86-SSE1-NEXT:    testl %eax, %eax
+; X86-SSE1-NEXT:    sete %al
+; X86-SSE1-NEXT:    retl
+;
+; X86-SSE2-LABEL: length24_eq:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-SSE2-NEXT:    movdqu (%ecx), %xmm0
+; X86-SSE2-NEXT:    movdqu 8(%ecx), %xmm1
+; X86-SSE2-NEXT:    movdqu (%eax), %xmm2
+; X86-SSE2-NEXT:    pcmpeqb %xmm0, %xmm2
+; X86-SSE2-NEXT:    movdqu 8(%eax), %xmm0
+; X86-SSE2-NEXT:    pcmpeqb %xmm1, %xmm0
+; X86-SSE2-NEXT:    pand %xmm2, %xmm0
+; X86-SSE2-NEXT:    pmovmskb %xmm0, %eax
+; X86-SSE2-NEXT:    cmpl $65535, %eax # imm = 0xFFFF
+; X86-SSE2-NEXT:    sete %al
+; X86-SSE2-NEXT:    retl
 ;
 ; X64-SSE2-LABEL: length24_eq:
 ; X64-SSE2:       # %bb.0:
@@ -929,17 +957,42 @@ define i1 @length24_eq(i8* %x, i8* %y) nounwind {
 }
 
 define i1 @length24_eq_const(i8* %X) nounwind {
-; X86-LABEL: length24_eq_const:
-; X86:       # %bb.0:
-; X86-NEXT:    pushl $0
-; X86-NEXT:    pushl $24
-; X86-NEXT:    pushl $.L.str
-; X86-NEXT:    pushl {{[0-9]+}}(%esp)
-; X86-NEXT:    calll memcmp
-; X86-NEXT:    addl $16, %esp
-; X86-NEXT:    testl %eax, %eax
-; X86-NEXT:    setne %al
-; X86-NEXT:    retl
+; X86-NOSSE-LABEL: length24_eq_const:
+; X86-NOSSE:       # %bb.0:
+; X86-NOSSE-NEXT:    pushl $0
+; X86-NOSSE-NEXT:    pushl $24
+; X86-NOSSE-NEXT:    pushl $.L.str
+; X86-NOSSE-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-NOSSE-NEXT:    calll memcmp
+; X86-NOSSE-NEXT:    addl $16, %esp
+; X86-NOSSE-NEXT:    testl %eax, %eax
+; X86-NOSSE-NEXT:    setne %al
+; X86-NOSSE-NEXT:    retl
+;
+; X86-SSE1-LABEL: length24_eq_const:
+; X86-SSE1:       # %bb.0:
+; X86-SSE1-NEXT:    pushl $0
+; X86-SSE1-NEXT:    pushl $24
+; X86-SSE1-NEXT:    pushl $.L.str
+; X86-SSE1-NEXT:    pushl {{[0-9]+}}(%esp)
+; X86-SSE1-NEXT:    calll memcmp
+; X86-SSE1-NEXT:    addl $16, %esp
+; X86-SSE1-NEXT:    testl %eax, %eax
+; X86-SSE1-NEXT:    setne %al
+; X86-SSE1-NEXT:    retl
+;
+; X86-SSE2-LABEL: length24_eq_const:
+; X86-SSE2:       # %bb.0:
+; X86-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT:    movdqu (%eax), %xmm0
+; X86-SSE2-NEXT:    movdqu 8(%eax), %xmm1
+; X86-SSE2-NEXT:    pcmpeqb {{\.LCPI.*}}, %xmm1
+; X86-SSE2-NEXT:    pcmpeqb {{\.LCPI.*}}, %xmm0
+; X86-SSE2-NEXT:    pand %xmm1, %xmm0
+; X86-SSE2-NEXT:    pmovmskb %xmm0, %eax
+; X86-SSE2-NEXT:    cmpl $65535, %eax # imm = 0xFFFF
+; X86-SSE2-NEXT:    setne %al
+; X86-SSE2-NEXT:    retl
 ;
 ; X64-SSE2-LABEL: length24_eq_const:
 ; X64-SSE2:       # %bb.0:
diff --git a/test/CodeGen/X86/min-legal-vector-width.ll b/test/CodeGen/X86/min-legal-vector-width.ll
index 70ce1809967da6a84cb9c056051a40d54d1b1539..1a28852f5aae1aad73259a80353a28107be783ea 100644
--- a/test/CodeGen/X86/min-legal-vector-width.ll
+++ b/test/CodeGen/X86/min-legal-vector-width.ll
@@ -187,9 +187,9 @@ define i32 @_Z9test_charPcS_i_256(i8* nocapture readonly, i8* nocapture readonly
 ; CHECK-NEXT:    vpaddd %ymm0, %ymm2, %ymm0
 ; CHECK-NEXT:    vpaddd %ymm0, %ymm1, %ymm0
 ; CHECK-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; CHECK-NEXT:    vpaddd %ymm1, %ymm0, %ymm0
+; CHECK-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; CHECK-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; CHECK-NEXT:    vpaddd %ymm1, %ymm0, %ymm0
+; CHECK-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; CHECK-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; CHECK-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; CHECK-NEXT:    vmovd %xmm0, %eax
@@ -253,9 +253,9 @@ define i32 @_Z9test_charPcS_i_512(i8* nocapture readonly, i8* nocapture readonly
 ; CHECK-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; CHECK-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
 ; CHECK-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; CHECK-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
+; CHECK-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; CHECK-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; CHECK-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
+; CHECK-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; CHECK-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; CHECK-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; CHECK-NEXT:    vmovd %xmm0, %eax
@@ -317,9 +317,9 @@ define i32 @sad_16i8_256() "min-legal-vector-width"="256" {
 ; CHECK-NEXT:  # %bb.2: # %middle.block
 ; CHECK-NEXT:    vpaddd %ymm0, %ymm1, %ymm0
 ; CHECK-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; CHECK-NEXT:    vpaddd %ymm1, %ymm0, %ymm0
+; CHECK-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; CHECK-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; CHECK-NEXT:    vpaddd %ymm1, %ymm0, %ymm0
+; CHECK-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; CHECK-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; CHECK-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; CHECK-NEXT:    vmovd %xmm0, %eax
@@ -379,9 +379,9 @@ define i32 @sad_16i8_512() "min-legal-vector-width"="512" {
 ; CHECK-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; CHECK-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
 ; CHECK-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; CHECK-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
+; CHECK-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; CHECK-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; CHECK-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
+; CHECK-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; CHECK-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; CHECK-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; CHECK-NEXT:    vmovd %xmm0, %eax
diff --git a/test/CodeGen/X86/mmx-schedule.ll b/test/CodeGen/X86/mmx-schedule.ll
deleted file mode 100644
index 51dc5e102ff1578aa59144b24d37e4623ba23cf3..0000000000000000000000000000000000000000
--- a/test/CodeGen/X86/mmx-schedule.ll
+++ /dev/null
@@ -1,7559 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=x86-64 -mattr=+ssse3 | FileCheck %s --check-prefix=CHECK --check-prefix=GENERIC
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=atom | FileCheck %s --check-prefix=CHECK --check-prefix=ATOM
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=slm | FileCheck %s --check-prefix=CHECK --check-prefix=SLM
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=sandybridge | FileCheck %s --check-prefix=CHECK --check-prefix=SANDY
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=ivybridge | FileCheck %s --check-prefix=CHECK --check-prefix=SANDY
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=haswell | FileCheck %s --check-prefix=CHECK --check-prefix=HASWELL
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=broadwell | FileCheck %s --check-prefix=CHECK --check-prefix=BROADWELL
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=skylake | FileCheck %s --check-prefix=CHECK --check-prefix=SKYLAKE
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=skx | FileCheck %s --check-prefix=CHECK --check-prefix=SKX
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=bdver2 | FileCheck %s --check-prefix=CHECK --check-prefix=BDVER2
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=btver2 | FileCheck %s --check-prefix=CHECK --check-prefix=BTVER2
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=znver1 | FileCheck %s --check-prefix=CHECK --check-prefix=ZNVER1
-
-define i64 @test_cvtpd2pi(<2 x double> %a0, <2 x double>* %a1) optsize {
-; GENERIC-LABEL: test_cvtpd2pi:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    cvtpd2pi (%rdi), %mm0 # sched: [10:1.00]
-; GENERIC-NEXT:    cvtpd2pi %xmm0, %mm1 # sched: [4:1.00]
-; GENERIC-NEXT:    por %mm1, %mm0 # sched: [1:0.33]
-; GENERIC-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_cvtpd2pi:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    cvtpd2pi (%rdi), %mm0 # sched: [8:4.00]
-; ATOM-NEXT:    cvtpd2pi %xmm0, %mm1 # sched: [7:3.50]
-; ATOM-NEXT:    por %mm1, %mm0 # sched: [1:0.50]
-; ATOM-NEXT:    movq %mm0, %rax # sched: [3:3.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_cvtpd2pi:
-; SLM:       # %bb.0:
-; SLM-NEXT:    cvtpd2pi (%rdi), %mm1 # sched: [7:1.00]
-; SLM-NEXT:    cvtpd2pi %xmm0, %mm0 # sched: [4:0.50]
-; SLM-NEXT:    por %mm0, %mm1 # sched: [1:0.50]
-; SLM-NEXT:    movq %mm1, %rax # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_cvtpd2pi:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    cvtpd2pi (%rdi), %mm0 # sched: [10:1.00]
-; SANDY-NEXT:    cvtpd2pi %xmm0, %mm1 # sched: [4:1.00]
-; SANDY-NEXT:    por %mm1, %mm0 # sched: [1:0.33]
-; SANDY-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_cvtpd2pi:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    cvtpd2pi (%rdi), %mm0 # sched: [10:1.00]
-; HASWELL-NEXT:    cvtpd2pi %xmm0, %mm1 # sched: [4:1.00]
-; HASWELL-NEXT:    por %mm1, %mm0 # sched: [1:0.33]
-; HASWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_cvtpd2pi:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    cvtpd2pi %xmm0, %mm0 # sched: [4:1.00]
-; BROADWELL-NEXT:    cvtpd2pi (%rdi), %mm1 # sched: [9:1.00]
-; BROADWELL-NEXT:    por %mm0, %mm1 # sched: [1:0.33]
-; BROADWELL-NEXT:    movq %mm1, %rax # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_cvtpd2pi:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    cvtpd2pi %xmm0, %mm0 # sched: [5:1.00]
-; SKYLAKE-NEXT:    cvtpd2pi (%rdi), %mm1 # sched: [11:1.00]
-; SKYLAKE-NEXT:    por %mm0, %mm1 # sched: [1:0.50]
-; SKYLAKE-NEXT:    movq %mm1, %rax # sched: [2:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_cvtpd2pi:
-; SKX:       # %bb.0:
-; SKX-NEXT:    cvtpd2pi %xmm0, %mm0 # sched: [5:1.00]
-; SKX-NEXT:    cvtpd2pi (%rdi), %mm1 # sched: [11:1.00]
-; SKX-NEXT:    por %mm0, %mm1 # sched: [1:0.50]
-; SKX-NEXT:    movq %mm1, %rax # sched: [2:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_cvtpd2pi:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    cvtpd2pi (%rdi), %mm1 # sched: [13:1.00]
-; BDVER2-NEXT:    cvtpd2pi %xmm0, %mm0 # sched: [6:1.00]
-; BDVER2-NEXT:    por %mm0, %mm1 # sched: [2:0.50]
-; BDVER2-NEXT:    movq %mm1, %rax # sched: [10:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_cvtpd2pi:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    cvtpd2pi (%rdi), %mm1 # sched: [8:1.00]
-; BTVER2-NEXT:    cvtpd2pi %xmm0, %mm0 # sched: [3:1.00]
-; BTVER2-NEXT:    por %mm0, %mm1 # sched: [1:0.50]
-; BTVER2-NEXT:    movq %mm1, %rax # sched: [4:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_cvtpd2pi:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    cvtpd2pi (%rdi), %mm1 # sched: [12:1.00]
-; ZNVER1-NEXT:    cvtpd2pi %xmm0, %mm0 # sched: [4:1.00]
-; ZNVER1-NEXT:    por %mm0, %mm1 # sched: [1:0.25]
-; ZNVER1-NEXT:    movq %mm1, %rax # sched: [2:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call x86_mmx @llvm.x86.sse.cvtpd2pi(<2 x double> %a0)
-  %2 = load <2 x double>, <2 x double> *%a1, align 16
-  %3 = call x86_mmx @llvm.x86.sse.cvtpd2pi(<2 x double> %2)
-  %4 = call x86_mmx @llvm.x86.mmx.por(x86_mmx %1, x86_mmx %3)
-  %5 = bitcast x86_mmx %4 to i64
-  ret i64 %5
-}
-declare x86_mmx @llvm.x86.sse.cvtpd2pi(<2 x double>) nounwind readnone
-
-define <2 x double> @test_cvtpi2pd(x86_mmx %a0, x86_mmx* %a1) optsize {
-; GENERIC-LABEL: test_cvtpi2pd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    cvtpi2pd %mm0, %xmm1 # sched: [4:1.00]
-; GENERIC-NEXT:    cvtpi2pd (%rdi), %xmm0 # sched: [10:1.00]
-; GENERIC-NEXT:    addpd %xmm1, %xmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_cvtpi2pd:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    cvtpi2pd (%rdi), %xmm0 # sched: [8:4.00]
-; ATOM-NEXT:    cvtpi2pd %mm0, %xmm1 # sched: [7:3.50]
-; ATOM-NEXT:    addpd %xmm1, %xmm0 # sched: [6:3.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_cvtpi2pd:
-; SLM:       # %bb.0:
-; SLM-NEXT:    cvtpi2pd (%rdi), %xmm0 # sched: [7:1.00]
-; SLM-NEXT:    cvtpi2pd %mm0, %xmm1 # sched: [4:0.50]
-; SLM-NEXT:    addpd %xmm1, %xmm0 # sched: [3:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_cvtpi2pd:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    cvtpi2pd %mm0, %xmm0 # sched: [4:1.00]
-; SANDY-NEXT:    cvtpi2pd (%rdi), %xmm1 # sched: [10:1.00]
-; SANDY-NEXT:    vaddpd %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_cvtpi2pd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    cvtpi2pd %mm0, %xmm0 # sched: [4:1.00]
-; HASWELL-NEXT:    cvtpi2pd (%rdi), %xmm1 # sched: [9:1.00]
-; HASWELL-NEXT:    vaddpd %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_cvtpi2pd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    cvtpi2pd (%rdi), %xmm0 # sched: [9:1.00]
-; BROADWELL-NEXT:    cvtpi2pd %mm0, %xmm1 # sched: [4:1.00]
-; BROADWELL-NEXT:    vaddpd %xmm0, %xmm1, %xmm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_cvtpi2pd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    cvtpi2pd %mm0, %xmm0 # sched: [5:1.00]
-; SKYLAKE-NEXT:    cvtpi2pd (%rdi), %xmm1 # sched: [10:1.00]
-; SKYLAKE-NEXT:    vaddpd %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_cvtpi2pd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    cvtpi2pd %mm0, %xmm0 # sched: [4:0.50]
-; SKX-NEXT:    cvtpi2pd (%rdi), %xmm1 # sched: [9:0.50]
-; SKX-NEXT:    vaddpd %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_cvtpi2pd:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    cvtpi2pd (%rdi), %xmm1 # sched: [13:1.00]
-; BDVER2-NEXT:    cvtpi2pd %mm0, %xmm0 # sched: [6:1.00]
-; BDVER2-NEXT:    vaddpd %xmm1, %xmm0, %xmm0 # sched: [5:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_cvtpi2pd:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    cvtpi2pd (%rdi), %xmm1 # sched: [8:1.00]
-; BTVER2-NEXT:    cvtpi2pd %mm0, %xmm0 # sched: [3:1.00]
-; BTVER2-NEXT:    vaddpd %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_cvtpi2pd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    cvtpi2pd (%rdi), %xmm1 # sched: [12:1.00]
-; ZNVER1-NEXT:    cvtpi2pd %mm0, %xmm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    vaddpd %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <2 x double> @llvm.x86.sse.cvtpi2pd(x86_mmx %a0)
-  %2 = load x86_mmx, x86_mmx *%a1, align 8
-  %3 = call <2 x double> @llvm.x86.sse.cvtpi2pd(x86_mmx %2)
-  %4 = fadd <2 x double> %1, %3
-  ret <2 x double> %4
-}
-declare <2 x double> @llvm.x86.sse.cvtpi2pd(x86_mmx) nounwind readnone
-
-define <4 x float> @test_cvtpi2ps(x86_mmx %a0, x86_mmx* %a1, <4 x float> %a2, <4 x float> %a3) optsize {
-; GENERIC-LABEL: test_cvtpi2ps:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    cvtpi2ps %mm0, %xmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    cvtpi2ps (%rdi), %xmm1 # sched: [9:1.00]
-; GENERIC-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_cvtpi2ps:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    cvtpi2ps %mm0, %xmm0 # sched: [5:5.00]
-; ATOM-NEXT:    cvtpi2ps (%rdi), %xmm1 # sched: [5:5.00]
-; ATOM-NEXT:    addps %xmm1, %xmm0 # sched: [5:5.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_cvtpi2ps:
-; SLM:       # %bb.0:
-; SLM-NEXT:    cvtpi2ps (%rdi), %xmm1 # sched: [7:1.00]
-; SLM-NEXT:    cvtpi2ps %mm0, %xmm0 # sched: [4:0.50]
-; SLM-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_cvtpi2ps:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    cvtpi2ps %mm0, %xmm0 # sched: [3:1.00]
-; SANDY-NEXT:    cvtpi2ps (%rdi), %xmm1 # sched: [9:1.00]
-; SANDY-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_cvtpi2ps:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    cvtpi2ps %mm0, %xmm0 # sched: [3:1.00]
-; HASWELL-NEXT:    cvtpi2ps (%rdi), %xmm1 # sched: [8:1.00]
-; HASWELL-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_cvtpi2ps:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    cvtpi2ps %mm0, %xmm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    cvtpi2ps (%rdi), %xmm1 # sched: [8:1.00]
-; BROADWELL-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_cvtpi2ps:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    cvtpi2ps %mm0, %xmm0 # sched: [6:2.00]
-; SKYLAKE-NEXT:    cvtpi2ps (%rdi), %xmm1 # sched: [9:1.00]
-; SKYLAKE-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_cvtpi2ps:
-; SKX:       # %bb.0:
-; SKX-NEXT:    cvtpi2ps %mm0, %xmm0 # sched: [6:2.00]
-; SKX-NEXT:    cvtpi2ps (%rdi), %xmm1 # sched: [9:1.00]
-; SKX-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_cvtpi2ps:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    cvtpi2ps (%rdi), %xmm1 # sched: [9:1.00]
-; BDVER2-NEXT:    cvtpi2ps %mm0, %xmm0 # sched: [4:1.00]
-; BDVER2-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [5:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_cvtpi2ps:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    cvtpi2ps (%rdi), %xmm1 # sched: [8:1.00]
-; BTVER2-NEXT:    cvtpi2ps %mm0, %xmm0 # sched: [3:1.00]
-; BTVER2-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_cvtpi2ps:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    cvtpi2ps (%rdi), %xmm1 # sched: [12:1.00]
-; ZNVER1-NEXT:    cvtpi2ps %mm0, %xmm0 # sched: [5:1.00]
-; ZNVER1-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <4 x float> @llvm.x86.sse.cvtpi2ps(<4 x float> %a2, x86_mmx %a0)
-  %2 = load x86_mmx, x86_mmx *%a1, align 8
-  %3 = call <4 x float> @llvm.x86.sse.cvtpi2ps(<4 x float> %a3, x86_mmx %2)
-  %4 = fadd <4 x float> %1, %3
-  ret <4 x float> %4
-}
-declare <4 x float> @llvm.x86.sse.cvtpi2ps(<4 x float>, x86_mmx) nounwind readnone
-
-define i64 @test_cvtps2pi(<4 x float> %a0, <4 x float>* %a1) optsize {
-; GENERIC-LABEL: test_cvtps2pi:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    cvtps2pi %xmm0, %mm0 # sched: [3:1.00]
-; GENERIC-NEXT:    cvtps2pi (%rdi), %mm1 # sched: [9:1.00]
-; GENERIC-NEXT:    por %mm0, %mm1 # sched: [1:0.33]
-; GENERIC-NEXT:    movq %mm1, %rax # sched: [2:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_cvtps2pi:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    cvtps2pi %xmm0, %mm0 # sched: [5:5.00]
-; ATOM-NEXT:    cvtps2pi (%rdi), %mm1 # sched: [5:5.00]
-; ATOM-NEXT:    por %mm0, %mm1 # sched: [1:0.50]
-; ATOM-NEXT:    movq %mm1, %rax # sched: [3:3.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_cvtps2pi:
-; SLM:       # %bb.0:
-; SLM-NEXT:    cvtps2pi (%rdi), %mm1 # sched: [7:1.00]
-; SLM-NEXT:    cvtps2pi %xmm0, %mm0 # sched: [4:0.50]
-; SLM-NEXT:    por %mm0, %mm1 # sched: [1:0.50]
-; SLM-NEXT:    movq %mm1, %rax # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_cvtps2pi:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    cvtps2pi %xmm0, %mm0 # sched: [3:1.00]
-; SANDY-NEXT:    cvtps2pi (%rdi), %mm1 # sched: [9:1.00]
-; SANDY-NEXT:    por %mm0, %mm1 # sched: [1:0.33]
-; SANDY-NEXT:    movq %mm1, %rax # sched: [2:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_cvtps2pi:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    cvtps2pi %xmm0, %mm0 # sched: [4:1.00]
-; HASWELL-NEXT:    cvtps2pi (%rdi), %mm1 # sched: [8:1.00]
-; HASWELL-NEXT:    por %mm0, %mm1 # sched: [1:0.33]
-; HASWELL-NEXT:    movq %mm1, %rax # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_cvtps2pi:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    cvtps2pi %xmm0, %mm0 # sched: [4:1.00]
-; BROADWELL-NEXT:    cvtps2pi (%rdi), %mm1 # sched: [8:1.00]
-; BROADWELL-NEXT:    por %mm0, %mm1 # sched: [1:0.33]
-; BROADWELL-NEXT:    movq %mm1, %rax # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_cvtps2pi:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    cvtps2pi %xmm0, %mm0 # sched: [5:1.00]
-; SKYLAKE-NEXT:    cvtps2pi (%rdi), %mm1 # sched: [9:0.50]
-; SKYLAKE-NEXT:    por %mm0, %mm1 # sched: [1:0.50]
-; SKYLAKE-NEXT:    movq %mm1, %rax # sched: [2:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_cvtps2pi:
-; SKX:       # %bb.0:
-; SKX-NEXT:    cvtps2pi %xmm0, %mm0 # sched: [5:1.00]
-; SKX-NEXT:    cvtps2pi (%rdi), %mm1 # sched: [9:0.50]
-; SKX-NEXT:    por %mm0, %mm1 # sched: [1:0.50]
-; SKX-NEXT:    movq %mm1, %rax # sched: [2:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_cvtps2pi:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    cvtps2pi (%rdi), %mm1 # sched: [9:1.00]
-; BDVER2-NEXT:    cvtps2pi %xmm0, %mm0 # sched: [4:1.00]
-; BDVER2-NEXT:    por %mm0, %mm1 # sched: [2:0.50]
-; BDVER2-NEXT:    movq %mm1, %rax # sched: [10:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_cvtps2pi:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    cvtps2pi (%rdi), %mm1 # sched: [8:1.00]
-; BTVER2-NEXT:    cvtps2pi %xmm0, %mm0 # sched: [3:1.00]
-; BTVER2-NEXT:    por %mm0, %mm1 # sched: [1:0.50]
-; BTVER2-NEXT:    movq %mm1, %rax # sched: [4:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_cvtps2pi:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    cvtps2pi (%rdi), %mm1 # sched: [12:1.00]
-; ZNVER1-NEXT:    cvtps2pi %xmm0, %mm0 # sched: [4:1.00]
-; ZNVER1-NEXT:    por %mm0, %mm1 # sched: [1:0.25]
-; ZNVER1-NEXT:    movq %mm1, %rax # sched: [2:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call x86_mmx @llvm.x86.sse.cvtps2pi(<4 x float> %a0)
-  %2 = load <4 x float>, <4 x float> *%a1, align 16
-  %3 = call x86_mmx @llvm.x86.sse.cvtps2pi(<4 x float> %2)
-  %4 = call x86_mmx @llvm.x86.mmx.por(x86_mmx %1, x86_mmx %3)
-  %5 = bitcast x86_mmx %4 to i64
-  ret i64 %5
-}
-declare x86_mmx @llvm.x86.sse.cvtps2pi(<4 x float>) nounwind readnone
-
-define i64 @test_cvttpd2pi(<2 x double> %a0, <2 x double>* %a1) optsize {
-; GENERIC-LABEL: test_cvttpd2pi:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    cvttpd2pi (%rdi), %mm0 # sched: [10:1.00]
-; GENERIC-NEXT:    cvttpd2pi %xmm0, %mm1 # sched: [4:1.00]
-; GENERIC-NEXT:    por %mm1, %mm0 # sched: [1:0.33]
-; GENERIC-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_cvttpd2pi:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    cvttpd2pi (%rdi), %mm0 # sched: [8:4.00]
-; ATOM-NEXT:    cvttpd2pi %xmm0, %mm1 # sched: [7:3.50]
-; ATOM-NEXT:    por %mm1, %mm0 # sched: [1:0.50]
-; ATOM-NEXT:    movq %mm0, %rax # sched: [3:3.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_cvttpd2pi:
-; SLM:       # %bb.0:
-; SLM-NEXT:    cvttpd2pi (%rdi), %mm1 # sched: [7:1.00]
-; SLM-NEXT:    cvttpd2pi %xmm0, %mm0 # sched: [4:0.50]
-; SLM-NEXT:    por %mm0, %mm1 # sched: [1:0.50]
-; SLM-NEXT:    movq %mm1, %rax # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_cvttpd2pi:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    cvttpd2pi (%rdi), %mm0 # sched: [10:1.00]
-; SANDY-NEXT:    cvttpd2pi %xmm0, %mm1 # sched: [4:1.00]
-; SANDY-NEXT:    por %mm1, %mm0 # sched: [1:0.33]
-; SANDY-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_cvttpd2pi:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    cvttpd2pi (%rdi), %mm0 # sched: [10:1.00]
-; HASWELL-NEXT:    cvttpd2pi %xmm0, %mm1 # sched: [4:1.00]
-; HASWELL-NEXT:    por %mm1, %mm0 # sched: [1:0.33]
-; HASWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_cvttpd2pi:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    cvttpd2pi %xmm0, %mm0 # sched: [4:1.00]
-; BROADWELL-NEXT:    cvttpd2pi (%rdi), %mm1 # sched: [9:1.00]
-; BROADWELL-NEXT:    por %mm0, %mm1 # sched: [1:0.33]
-; BROADWELL-NEXT:    movq %mm1, %rax # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_cvttpd2pi:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    cvttpd2pi %xmm0, %mm0 # sched: [5:1.00]
-; SKYLAKE-NEXT:    cvttpd2pi (%rdi), %mm1 # sched: [11:1.00]
-; SKYLAKE-NEXT:    por %mm0, %mm1 # sched: [1:0.50]
-; SKYLAKE-NEXT:    movq %mm1, %rax # sched: [2:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_cvttpd2pi:
-; SKX:       # %bb.0:
-; SKX-NEXT:    cvttpd2pi %xmm0, %mm0 # sched: [5:1.00]
-; SKX-NEXT:    cvttpd2pi (%rdi), %mm1 # sched: [11:1.00]
-; SKX-NEXT:    por %mm0, %mm1 # sched: [1:0.50]
-; SKX-NEXT:    movq %mm1, %rax # sched: [2:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_cvttpd2pi:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    cvttpd2pi (%rdi), %mm1 # sched: [13:1.00]
-; BDVER2-NEXT:    cvttpd2pi %xmm0, %mm0 # sched: [6:1.00]
-; BDVER2-NEXT:    por %mm0, %mm1 # sched: [2:0.50]
-; BDVER2-NEXT:    movq %mm1, %rax # sched: [10:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_cvttpd2pi:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    cvttpd2pi (%rdi), %mm1 # sched: [8:1.00]
-; BTVER2-NEXT:    cvttpd2pi %xmm0, %mm0 # sched: [3:1.00]
-; BTVER2-NEXT:    por %mm0, %mm1 # sched: [1:0.50]
-; BTVER2-NEXT:    movq %mm1, %rax # sched: [4:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_cvttpd2pi:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    cvttpd2pi (%rdi), %mm1 # sched: [12:1.00]
-; ZNVER1-NEXT:    cvttpd2pi %xmm0, %mm0 # sched: [4:1.00]
-; ZNVER1-NEXT:    por %mm0, %mm1 # sched: [1:0.25]
-; ZNVER1-NEXT:    movq %mm1, %rax # sched: [2:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call x86_mmx @llvm.x86.sse.cvttpd2pi(<2 x double> %a0)
-  %2 = load <2 x double>, <2 x double> *%a1, align 16
-  %3 = call x86_mmx @llvm.x86.sse.cvttpd2pi(<2 x double> %2)
-  %4 = call x86_mmx @llvm.x86.mmx.por(x86_mmx %1, x86_mmx %3)
-  %5 = bitcast x86_mmx %4 to i64
-  ret i64 %5
-}
-declare x86_mmx @llvm.x86.sse.cvttpd2pi(<2 x double>) nounwind readnone
-
-define i64 @test_cvttps2pi(<4 x float> %a0, <4 x float>* %a1) optsize {
-; GENERIC-LABEL: test_cvttps2pi:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    cvttps2pi %xmm0, %mm0 # sched: [3:1.00]
-; GENERIC-NEXT:    cvttps2pi (%rdi), %mm1 # sched: [9:1.00]
-; GENERIC-NEXT:    por %mm0, %mm1 # sched: [1:0.33]
-; GENERIC-NEXT:    movq %mm1, %rax # sched: [2:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_cvttps2pi:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    cvttps2pi %xmm0, %mm0 # sched: [5:5.00]
-; ATOM-NEXT:    cvttps2pi (%rdi), %mm1 # sched: [5:5.00]
-; ATOM-NEXT:    por %mm0, %mm1 # sched: [1:0.50]
-; ATOM-NEXT:    movq %mm1, %rax # sched: [3:3.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_cvttps2pi:
-; SLM:       # %bb.0:
-; SLM-NEXT:    cvttps2pi (%rdi), %mm1 # sched: [7:1.00]
-; SLM-NEXT:    cvttps2pi %xmm0, %mm0 # sched: [4:0.50]
-; SLM-NEXT:    por %mm0, %mm1 # sched: [1:0.50]
-; SLM-NEXT:    movq %mm1, %rax # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_cvttps2pi:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    cvttps2pi %xmm0, %mm0 # sched: [3:1.00]
-; SANDY-NEXT:    cvttps2pi (%rdi), %mm1 # sched: [9:1.00]
-; SANDY-NEXT:    por %mm0, %mm1 # sched: [1:0.33]
-; SANDY-NEXT:    movq %mm1, %rax # sched: [2:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_cvttps2pi:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    cvttps2pi %xmm0, %mm0 # sched: [4:1.00]
-; HASWELL-NEXT:    cvttps2pi (%rdi), %mm1 # sched: [8:1.00]
-; HASWELL-NEXT:    por %mm0, %mm1 # sched: [1:0.33]
-; HASWELL-NEXT:    movq %mm1, %rax # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_cvttps2pi:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    cvttps2pi %xmm0, %mm0 # sched: [4:1.00]
-; BROADWELL-NEXT:    cvttps2pi (%rdi), %mm1 # sched: [8:1.00]
-; BROADWELL-NEXT:    por %mm0, %mm1 # sched: [1:0.33]
-; BROADWELL-NEXT:    movq %mm1, %rax # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_cvttps2pi:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    cvttps2pi %xmm0, %mm0 # sched: [5:1.00]
-; SKYLAKE-NEXT:    cvttps2pi (%rdi), %mm1 # sched: [9:0.50]
-; SKYLAKE-NEXT:    por %mm0, %mm1 # sched: [1:0.50]
-; SKYLAKE-NEXT:    movq %mm1, %rax # sched: [2:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_cvttps2pi:
-; SKX:       # %bb.0:
-; SKX-NEXT:    cvttps2pi %xmm0, %mm0 # sched: [5:1.00]
-; SKX-NEXT:    cvttps2pi (%rdi), %mm1 # sched: [9:0.50]
-; SKX-NEXT:    por %mm0, %mm1 # sched: [1:0.50]
-; SKX-NEXT:    movq %mm1, %rax # sched: [2:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_cvttps2pi:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    cvttps2pi (%rdi), %mm1 # sched: [9:1.00]
-; BDVER2-NEXT:    cvttps2pi %xmm0, %mm0 # sched: [4:1.00]
-; BDVER2-NEXT:    por %mm0, %mm1 # sched: [2:0.50]
-; BDVER2-NEXT:    movq %mm1, %rax # sched: [10:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_cvttps2pi:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    cvttps2pi (%rdi), %mm1 # sched: [8:1.00]
-; BTVER2-NEXT:    cvttps2pi %xmm0, %mm0 # sched: [3:1.00]
-; BTVER2-NEXT:    por %mm0, %mm1 # sched: [1:0.50]
-; BTVER2-NEXT:    movq %mm1, %rax # sched: [4:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_cvttps2pi:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    cvttps2pi (%rdi), %mm1 # sched: [12:1.00]
-; ZNVER1-NEXT:    cvttps2pi %xmm0, %mm0 # sched: [4:1.00]
-; ZNVER1-NEXT:    por %mm0, %mm1 # sched: [1:0.25]
-; ZNVER1-NEXT:    movq %mm1, %rax # sched: [2:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call x86_mmx @llvm.x86.sse.cvttps2pi(<4 x float> %a0)
-  %2 = load <4 x float>, <4 x float> *%a1, align 16
-  %3 = call x86_mmx @llvm.x86.sse.cvttps2pi(<4 x float> %2)
-  %4 = call x86_mmx @llvm.x86.mmx.por(x86_mmx %1, x86_mmx %3)
-  %5 = bitcast x86_mmx %4 to i64
-  ret i64 %5
-}
-declare x86_mmx @llvm.x86.sse.cvttps2pi(<4 x float>) nounwind readnone
-
-define void @test_emms() optsize {
-; GENERIC-LABEL: test_emms:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    emms # sched: [31:10.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_emms:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    emms # sched: [5:2.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_emms:
-; SLM:       # %bb.0:
-; SLM-NEXT:    emms # sched: [10:5.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_emms:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    emms # sched: [31:10.33]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_emms:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    emms # sched: [31:10.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_emms:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    emms # sched: [31:10.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_emms:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    emms # sched: [10:4.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_emms:
-; SKX:       # %bb.0:
-; SKX-NEXT:    emms # sched: [10:4.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_emms:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    emms # sched: [2:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_emms:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    emms # sched: [2:0.50]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_emms:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    emms # sched: [2:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  call void @llvm.x86.mmx.emms()
-  ret void
-}
-declare void @llvm.x86.mmx.emms()
-
-define void @test_maskmovq(x86_mmx %a0, x86_mmx %a1, i8* %a2) optsize {
-; GENERIC-LABEL: test_maskmovq:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    maskmovq %mm1, %mm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_maskmovq:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    maskmovq %mm1, %mm0 # sched: [1:1.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_maskmovq:
-; SLM:       # %bb.0:
-; SLM-NEXT:    maskmovq %mm1, %mm0 # sched: [1:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_maskmovq:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    maskmovq %mm1, %mm0 # sched: [1:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_maskmovq:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    maskmovq %mm1, %mm0 # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_maskmovq:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    maskmovq %mm1, %mm0 # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_maskmovq:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    maskmovq %mm1, %mm0 # sched: [1:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_maskmovq:
-; SKX:       # %bb.0:
-; SKX-NEXT:    maskmovq %mm1, %mm0 # sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_maskmovq:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    maskmovq %mm1, %mm0 # sched: [2:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_maskmovq:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    maskmovq %mm1, %mm0 # sched: [1:0.50]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_maskmovq:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    maskmovq %mm1, %mm0 # sched: [100:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  call void @llvm.x86.mmx.maskmovq(x86_mmx %a0, x86_mmx %a1, i8* %a2)
-  ret void
-}
-declare void @llvm.x86.mmx.maskmovq(x86_mmx, x86_mmx, i8*) nounwind
-
-define i32 @test_movd(x86_mmx %a0, i32 %a1, i32 *%a2) {
-; GENERIC-LABEL: test_movd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    movd %edi, %mm1 # sched: [1:1.00]
-; GENERIC-NEXT:    movd (%rsi), %mm2 # sched: [5:0.50]
-; GENERIC-NEXT:    paddd %mm1, %mm2 # sched: [3:1.00]
-; GENERIC-NEXT:    paddd %mm2, %mm0 # sched: [3:1.00]
-; GENERIC-NEXT:    movd %mm2, %ecx # sched: [2:1.00]
-; GENERIC-NEXT:    movd %mm0, %eax # sched: [2:1.00]
-; GENERIC-NEXT:    movl %ecx, (%rsi) # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_movd:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    movd %edi, %mm1 # sched: [1:1.00]
-; ATOM-NEXT:    movd (%rsi), %mm2 # sched: [1:1.00]
-; ATOM-NEXT:    paddd %mm1, %mm2 # sched: [1:0.50]
-; ATOM-NEXT:    paddd %mm2, %mm0 # sched: [1:0.50]
-; ATOM-NEXT:    movd %mm2, %ecx # sched: [3:3.00]
-; ATOM-NEXT:    movd %mm0, %eax # sched: [3:3.00]
-; ATOM-NEXT:    movl %ecx, (%rsi) # sched: [1:1.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_movd:
-; SLM:       # %bb.0:
-; SLM-NEXT:    movd (%rsi), %mm2 # sched: [3:1.00]
-; SLM-NEXT:    movd %edi, %mm1 # sched: [1:0.50]
-; SLM-NEXT:    paddd %mm1, %mm2 # sched: [1:0.50]
-; SLM-NEXT:    paddd %mm2, %mm0 # sched: [1:0.50]
-; SLM-NEXT:    movd %mm2, %ecx # sched: [1:0.50]
-; SLM-NEXT:    movd %mm0, %eax # sched: [1:0.50]
-; SLM-NEXT:    movl %ecx, (%rsi) # sched: [1:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_movd:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    movd %edi, %mm1 # sched: [1:1.00]
-; SANDY-NEXT:    movd (%rsi), %mm2 # sched: [5:0.50]
-; SANDY-NEXT:    paddd %mm1, %mm2 # sched: [3:1.00]
-; SANDY-NEXT:    paddd %mm2, %mm0 # sched: [3:1.00]
-; SANDY-NEXT:    movd %mm2, %ecx # sched: [2:1.00]
-; SANDY-NEXT:    movd %mm0, %eax # sched: [2:1.00]
-; SANDY-NEXT:    movl %ecx, (%rsi) # sched: [1:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_movd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    movd %edi, %mm1 # sched: [1:1.00]
-; HASWELL-NEXT:    movd (%rsi), %mm2 # sched: [5:0.50]
-; HASWELL-NEXT:    paddd %mm1, %mm2 # sched: [1:0.50]
-; HASWELL-NEXT:    paddd %mm2, %mm0 # sched: [1:0.50]
-; HASWELL-NEXT:    movd %mm2, %ecx # sched: [1:1.00]
-; HASWELL-NEXT:    movd %mm0, %eax # sched: [1:1.00]
-; HASWELL-NEXT:    movl %ecx, (%rsi) # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_movd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    movd %edi, %mm1 # sched: [1:1.00]
-; BROADWELL-NEXT:    movd (%rsi), %mm2 # sched: [5:0.50]
-; BROADWELL-NEXT:    paddd %mm1, %mm2 # sched: [1:0.50]
-; BROADWELL-NEXT:    paddd %mm2, %mm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    movd %mm2, %ecx # sched: [1:1.00]
-; BROADWELL-NEXT:    movd %mm0, %eax # sched: [1:1.00]
-; BROADWELL-NEXT:    movl %ecx, (%rsi) # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_movd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    movd %edi, %mm1 # sched: [1:1.00]
-; SKYLAKE-NEXT:    movd (%rsi), %mm2 # sched: [5:0.50]
-; SKYLAKE-NEXT:    paddd %mm1, %mm2 # sched: [1:0.50]
-; SKYLAKE-NEXT:    paddd %mm2, %mm0 # sched: [1:0.50]
-; SKYLAKE-NEXT:    movd %mm2, %ecx # sched: [2:1.00]
-; SKYLAKE-NEXT:    movd %mm0, %eax # sched: [2:1.00]
-; SKYLAKE-NEXT:    movl %ecx, (%rsi) # sched: [1:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_movd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    movd %edi, %mm1 # sched: [1:1.00]
-; SKX-NEXT:    movd (%rsi), %mm2 # sched: [5:0.50]
-; SKX-NEXT:    paddd %mm1, %mm2 # sched: [1:0.50]
-; SKX-NEXT:    paddd %mm2, %mm0 # sched: [1:0.50]
-; SKX-NEXT:    movd %mm2, %ecx # sched: [2:1.00]
-; SKX-NEXT:    movd %mm0, %eax # sched: [2:1.00]
-; SKX-NEXT:    movl %ecx, (%rsi) # sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_movd:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    movd %edi, %mm1 # sched: [10:0.50]
-; BDVER2-NEXT:    movd (%rsi), %mm2 # sched: [5:0.50]
-; BDVER2-NEXT:    paddd %mm1, %mm2 # sched: [2:0.50]
-; BDVER2-NEXT:    paddd %mm2, %mm0 # sched: [2:0.50]
-; BDVER2-NEXT:    movd %mm2, %ecx # sched: [10:1.00]
-; BDVER2-NEXT:    movd %mm0, %eax # sched: [10:1.00]
-; BDVER2-NEXT:    movl %ecx, (%rsi) # sched: [1:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_movd:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    movd %edi, %mm1 # sched: [8:0.50]
-; BTVER2-NEXT:    movd (%rsi), %mm2 # sched: [5:1.00]
-; BTVER2-NEXT:    paddd %mm1, %mm2 # sched: [1:0.50]
-; BTVER2-NEXT:    paddd %mm2, %mm0 # sched: [1:0.50]
-; BTVER2-NEXT:    movd %mm2, %ecx # sched: [4:1.00]
-; BTVER2-NEXT:    movd %mm0, %eax # sched: [4:1.00]
-; BTVER2-NEXT:    movl %ecx, (%rsi) # sched: [1:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_movd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    movd (%rsi), %mm2 # sched: [8:0.50]
-; ZNVER1-NEXT:    movd %edi, %mm1 # sched: [3:1.00]
-; ZNVER1-NEXT:    paddd %mm1, %mm2 # sched: [1:0.25]
-; ZNVER1-NEXT:    paddd %mm2, %mm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    movd %mm2, %ecx # sched: [2:1.00]
-; ZNVER1-NEXT:    movd %mm0, %eax # sched: [2:1.00]
-; ZNVER1-NEXT:    movl %ecx, (%rsi) # sched: [1:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1  = insertelement <2 x i32> undef, i32 %a1, i32 0
-  %2  = bitcast <2 x i32> %1 to x86_mmx
-  %3  = load i32, i32 *%a2
-  %4  = insertelement <2 x i32> undef, i32 %3, i32 0
-  %5  = bitcast <2 x i32> %4 to x86_mmx
-  %6  = call x86_mmx @llvm.x86.mmx.padd.d(x86_mmx %2, x86_mmx %5)
-  %7  = call x86_mmx @llvm.x86.mmx.padd.d(x86_mmx %a0, x86_mmx %6)
-  %8  = bitcast x86_mmx %6 to <2 x i32>
-  %9  = bitcast x86_mmx %7 to <2 x i32>
-  %10 = extractelement <2 x i32> %8, i32 0
-  %11 = extractelement <2 x i32> %9, i32 0
-  store i32 %10, i32* %a2
-  ret i32 %11
-}
-
-define i64 @test_movdq2q(<2 x i64> %a0) optsize {
-; GENERIC-LABEL: test_movdq2q:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    movdq2q %xmm0, %mm0 # sched: [2:1.00]
-; GENERIC-NEXT:    paddd %mm0, %mm0 # sched: [3:1.00]
-; GENERIC-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_movdq2q:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    movdq2q %xmm0, %mm0 # sched: [1:0.50]
-; ATOM-NEXT:    paddd %mm0, %mm0 # sched: [1:0.50]
-; ATOM-NEXT:    movq %mm0, %rax # sched: [3:3.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_movdq2q:
-; SLM:       # %bb.0:
-; SLM-NEXT:    movdq2q %xmm0, %mm0 # sched: [1:0.50]
-; SLM-NEXT:    paddd %mm0, %mm0 # sched: [1:0.50]
-; SLM-NEXT:    movq %mm0, %rax # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_movdq2q:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    movdq2q %xmm0, %mm0 # sched: [2:1.00]
-; SANDY-NEXT:    paddd %mm0, %mm0 # sched: [3:1.00]
-; SANDY-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_movdq2q:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    movdq2q %xmm0, %mm0 # sched: [2:0.67]
-; HASWELL-NEXT:    paddd %mm0, %mm0 # sched: [1:0.50]
-; HASWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_movdq2q:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    movdq2q %xmm0, %mm0 # sched: [2:0.67]
-; BROADWELL-NEXT:    paddd %mm0, %mm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_movdq2q:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    movdq2q %xmm0, %mm0 # sched: [2:1.00]
-; SKYLAKE-NEXT:    paddd %mm0, %mm0 # sched: [1:0.50]
-; SKYLAKE-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_movdq2q:
-; SKX:       # %bb.0:
-; SKX-NEXT:    movdq2q %xmm0, %mm0 # sched: [2:1.00]
-; SKX-NEXT:    paddd %mm0, %mm0 # sched: [1:0.50]
-; SKX-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_movdq2q:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    movdq2q %xmm0, %mm0 # sched: [2:0.50]
-; BDVER2-NEXT:    paddd %mm0, %mm0 # sched: [2:0.50]
-; BDVER2-NEXT:    movq %mm0, %rax # sched: [10:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_movdq2q:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    movdq2q %xmm0, %mm0 # sched: [1:0.50]
-; BTVER2-NEXT:    paddd %mm0, %mm0 # sched: [1:0.50]
-; BTVER2-NEXT:    movq %mm0, %rax # sched: [4:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_movdq2q:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    movdq2q %xmm0, %mm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    paddd %mm0, %mm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = extractelement <2 x i64> %a0, i32 0
-  %2 = bitcast i64 %1 to x86_mmx
-  %3 = call x86_mmx @llvm.x86.mmx.padd.d(x86_mmx %2, x86_mmx %2)
-  %4 = bitcast x86_mmx %3 to i64
-  ret i64 %4
-}
-
-define void @test_movntq(x86_mmx* %a0, x86_mmx %a1) optsize {
-; GENERIC-LABEL: test_movntq:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    movntq %mm0, (%rdi) # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_movntq:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    movntq %mm0, (%rdi) # sched: [1:1.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_movntq:
-; SLM:       # %bb.0:
-; SLM-NEXT:    movntq %mm0, (%rdi) # sched: [1:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_movntq:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    movntq %mm0, (%rdi) # sched: [1:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_movntq:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    movntq %mm0, (%rdi) # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_movntq:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    movntq %mm0, (%rdi) # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_movntq:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    movntq %mm0, (%rdi) # sched: [1:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_movntq:
-; SKX:       # %bb.0:
-; SKX-NEXT:    movntq %mm0, (%rdi) # sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_movntq:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    movntq %mm0, (%rdi) # sched: [2:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_movntq:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    movntq %mm0, (%rdi) # sched: [2:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_movntq:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    movntq %mm0, (%rdi) # sched: [1:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  call void @llvm.x86.mmx.movnt.dq(x86_mmx* %a0, x86_mmx %a1)
-  ret void
-}
-declare void @llvm.x86.mmx.movnt.dq(x86_mmx*, x86_mmx) nounwind
-
-define void @test_movq(i64 *%a0) {
-; GENERIC-LABEL: test_movq:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    movq (%rdi), %mm0 # sched: [5:0.50]
-; GENERIC-NEXT:    paddd %mm0, %mm0 # sched: [3:1.00]
-; GENERIC-NEXT:    movq %mm0, (%rdi) # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_movq:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    movq (%rdi), %mm0 # sched: [1:1.00]
-; ATOM-NEXT:    paddd %mm0, %mm0 # sched: [1:0.50]
-; ATOM-NEXT:    movq %mm0, (%rdi) # sched: [1:1.00]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_movq:
-; SLM:       # %bb.0:
-; SLM-NEXT:    movq (%rdi), %mm0 # sched: [3:1.00]
-; SLM-NEXT:    paddd %mm0, %mm0 # sched: [1:0.50]
-; SLM-NEXT:    movq %mm0, (%rdi) # sched: [1:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_movq:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    movq (%rdi), %mm0 # sched: [5:0.50]
-; SANDY-NEXT:    paddd %mm0, %mm0 # sched: [3:1.00]
-; SANDY-NEXT:    movq %mm0, (%rdi) # sched: [1:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_movq:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    movq (%rdi), %mm0 # sched: [5:0.50]
-; HASWELL-NEXT:    paddd %mm0, %mm0 # sched: [1:0.50]
-; HASWELL-NEXT:    movq %mm0, (%rdi) # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_movq:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    movq (%rdi), %mm0 # sched: [5:0.50]
-; BROADWELL-NEXT:    paddd %mm0, %mm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    movq %mm0, (%rdi) # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_movq:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    movq (%rdi), %mm0 # sched: [5:0.50]
-; SKYLAKE-NEXT:    paddd %mm0, %mm0 # sched: [1:0.50]
-; SKYLAKE-NEXT:    movq %mm0, (%rdi) # sched: [1:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_movq:
-; SKX:       # %bb.0:
-; SKX-NEXT:    movq (%rdi), %mm0 # sched: [5:0.50]
-; SKX-NEXT:    paddd %mm0, %mm0 # sched: [1:0.50]
-; SKX-NEXT:    movq %mm0, (%rdi) # sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_movq:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    movq (%rdi), %mm0 # sched: [5:0.50]
-; BDVER2-NEXT:    paddd %mm0, %mm0 # sched: [2:0.50]
-; BDVER2-NEXT:    movq %mm0, (%rdi) # sched: [2:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_movq:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    movq (%rdi), %mm0 # sched: [5:1.00]
-; BTVER2-NEXT:    paddd %mm0, %mm0 # sched: [1:0.50]
-; BTVER2-NEXT:    movq %mm0, (%rdi) # sched: [2:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_movq:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    movq (%rdi), %mm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    paddd %mm0, %mm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    movq %mm0, (%rdi) # sched: [1:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = load i64, i64* %a0, align 8
-  %2 = bitcast i64 %1 to x86_mmx
-  %3 = call x86_mmx @llvm.x86.mmx.padd.d(x86_mmx %2, x86_mmx %2)
-  %4 = bitcast x86_mmx %3 to i64
-  store i64 %4, i64* %a0, align 8
-  ret void
-}
-
-define <2 x i64> @test_movq2dq(x86_mmx %a0) optsize {
-; GENERIC-LABEL: test_movq2dq:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    movq2dq %mm0, %xmm0 # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_movq2dq:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    movq2dq %mm0, %xmm0 # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_movq2dq:
-; SLM:       # %bb.0:
-; SLM-NEXT:    movq2dq %mm0, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_movq2dq:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    movq2dq %mm0, %xmm0 # sched: [1:0.33]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_movq2dq:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    movq2dq %mm0, %xmm0 # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_movq2dq:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    movq2dq %mm0, %xmm0 # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_movq2dq:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    movq2dq %mm0, %xmm0 # sched: [2:2.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_movq2dq:
-; SKX:       # %bb.0:
-; SKX-NEXT:    movq2dq %mm0, %xmm0 # sched: [2:2.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_movq2dq:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    movq2dq %mm0, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_movq2dq:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    movq2dq %mm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_movq2dq:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    movq2dq %mm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = bitcast x86_mmx %a0 to i64
-  %2 = insertelement <2 x i64> undef, i64 %1, i32 0
-  ret <2 x i64> %2
-}
-
-define i64 @test_pabsb(x86_mmx *%a0) optsize {
-; GENERIC-LABEL: test_pabsb:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    pabsb (%rdi), %mm0 # sched: [6:0.50]
-; GENERIC-NEXT:    pabsb %mm0, %mm0 # sched: [1:0.50]
-; GENERIC-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_pabsb:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    pabsb (%rdi), %mm0 # sched: [1:1.00]
-; ATOM-NEXT:    pabsb %mm0, %mm0 # sched: [1:0.50]
-; ATOM-NEXT:    movq %mm0, %rax # sched: [3:3.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_pabsb:
-; SLM:       # %bb.0:
-; SLM-NEXT:    pabsb (%rdi), %mm0 # sched: [4:1.00]
-; SLM-NEXT:    pabsb %mm0, %mm0 # sched: [1:0.50]
-; SLM-NEXT:    movq %mm0, %rax # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_pabsb:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    pabsb (%rdi), %mm0 # sched: [6:0.50]
-; SANDY-NEXT:    pabsb %mm0, %mm0 # sched: [1:0.50]
-; SANDY-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_pabsb:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    pabsb (%rdi), %mm0 # sched: [6:0.50]
-; HASWELL-NEXT:    pabsb %mm0, %mm0 # sched: [1:0.50]
-; HASWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pabsb:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    pabsb (%rdi), %mm0 # sched: [6:0.50]
-; BROADWELL-NEXT:    pabsb %mm0, %mm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pabsb:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    pabsb (%rdi), %mm0 # sched: [6:0.50]
-; SKYLAKE-NEXT:    pabsb %mm0, %mm0 # sched: [1:0.50]
-; SKYLAKE-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pabsb:
-; SKX:       # %bb.0:
-; SKX-NEXT:    pabsb (%rdi), %mm0 # sched: [6:0.50]
-; SKX-NEXT:    pabsb %mm0, %mm0 # sched: [1:0.50]
-; SKX-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_pabsb:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    pabsb (%rdi), %mm0 # sched: [7:0.50]
-; BDVER2-NEXT:    pabsb %mm0, %mm0 # sched: [2:0.50]
-; BDVER2-NEXT:    movq %mm0, %rax # sched: [10:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_pabsb:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    pabsb (%rdi), %mm0 # sched: [6:1.00]
-; BTVER2-NEXT:    pabsb %mm0, %mm0 # sched: [1:0.50]
-; BTVER2-NEXT:    movq %mm0, %rax # sched: [4:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_pabsb:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    pabsb (%rdi), %mm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    pabsb %mm0, %mm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = load x86_mmx, x86_mmx *%a0, align 8
-  %2 = call x86_mmx @llvm.x86.ssse3.pabs.b(x86_mmx %1)
-  %3 = call x86_mmx @llvm.x86.ssse3.pabs.b(x86_mmx %2)
-  %4 = bitcast x86_mmx %3 to i64
-  ret i64 %4
-}
-declare x86_mmx @llvm.x86.ssse3.pabs.b(x86_mmx) nounwind readnone
-
-define i64 @test_pabsd(x86_mmx *%a0) optsize {
-; GENERIC-LABEL: test_pabsd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    pabsd (%rdi), %mm0 # sched: [6:0.50]
-; GENERIC-NEXT:    pabsd %mm0, %mm0 # sched: [1:0.50]
-; GENERIC-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_pabsd:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    pabsd (%rdi), %mm0 # sched: [1:1.00]
-; ATOM-NEXT:    pabsd %mm0, %mm0 # sched: [1:0.50]
-; ATOM-NEXT:    movq %mm0, %rax # sched: [3:3.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_pabsd:
-; SLM:       # %bb.0:
-; SLM-NEXT:    pabsd (%rdi), %mm0 # sched: [4:1.00]
-; SLM-NEXT:    pabsd %mm0, %mm0 # sched: [1:0.50]
-; SLM-NEXT:    movq %mm0, %rax # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_pabsd:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    pabsd (%rdi), %mm0 # sched: [6:0.50]
-; SANDY-NEXT:    pabsd %mm0, %mm0 # sched: [1:0.50]
-; SANDY-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_pabsd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    pabsd (%rdi), %mm0 # sched: [6:0.50]
-; HASWELL-NEXT:    pabsd %mm0, %mm0 # sched: [1:0.50]
-; HASWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pabsd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    pabsd (%rdi), %mm0 # sched: [6:0.50]
-; BROADWELL-NEXT:    pabsd %mm0, %mm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pabsd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    pabsd (%rdi), %mm0 # sched: [6:0.50]
-; SKYLAKE-NEXT:    pabsd %mm0, %mm0 # sched: [1:0.50]
-; SKYLAKE-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pabsd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    pabsd (%rdi), %mm0 # sched: [6:0.50]
-; SKX-NEXT:    pabsd %mm0, %mm0 # sched: [1:0.50]
-; SKX-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_pabsd:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    pabsd (%rdi), %mm0 # sched: [7:0.50]
-; BDVER2-NEXT:    pabsd %mm0, %mm0 # sched: [2:0.50]
-; BDVER2-NEXT:    movq %mm0, %rax # sched: [10:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_pabsd:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    pabsd (%rdi), %mm0 # sched: [6:1.00]
-; BTVER2-NEXT:    pabsd %mm0, %mm0 # sched: [1:0.50]
-; BTVER2-NEXT:    movq %mm0, %rax # sched: [4:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_pabsd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    pabsd (%rdi), %mm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    pabsd %mm0, %mm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = load x86_mmx, x86_mmx *%a0, align 8
-  %2 = call x86_mmx @llvm.x86.ssse3.pabs.d(x86_mmx %1)
-  %3 = call x86_mmx @llvm.x86.ssse3.pabs.d(x86_mmx %2)
-  %4 = bitcast x86_mmx %3 to i64
-  ret i64 %4
-}
-declare x86_mmx @llvm.x86.ssse3.pabs.d(x86_mmx) nounwind readnone
-
-define i64 @test_pabsw(x86_mmx *%a0) optsize {
-; GENERIC-LABEL: test_pabsw:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    pabsw (%rdi), %mm0 # sched: [6:0.50]
-; GENERIC-NEXT:    pabsw %mm0, %mm0 # sched: [1:0.50]
-; GENERIC-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_pabsw:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    pabsw (%rdi), %mm0 # sched: [1:1.00]
-; ATOM-NEXT:    pabsw %mm0, %mm0 # sched: [1:0.50]
-; ATOM-NEXT:    movq %mm0, %rax # sched: [3:3.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_pabsw:
-; SLM:       # %bb.0:
-; SLM-NEXT:    pabsw (%rdi), %mm0 # sched: [4:1.00]
-; SLM-NEXT:    pabsw %mm0, %mm0 # sched: [1:0.50]
-; SLM-NEXT:    movq %mm0, %rax # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_pabsw:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    pabsw (%rdi), %mm0 # sched: [6:0.50]
-; SANDY-NEXT:    pabsw %mm0, %mm0 # sched: [1:0.50]
-; SANDY-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_pabsw:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    pabsw (%rdi), %mm0 # sched: [6:0.50]
-; HASWELL-NEXT:    pabsw %mm0, %mm0 # sched: [1:0.50]
-; HASWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pabsw:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    pabsw (%rdi), %mm0 # sched: [6:0.50]
-; BROADWELL-NEXT:    pabsw %mm0, %mm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pabsw:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    pabsw (%rdi), %mm0 # sched: [6:0.50]
-; SKYLAKE-NEXT:    pabsw %mm0, %mm0 # sched: [1:0.50]
-; SKYLAKE-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pabsw:
-; SKX:       # %bb.0:
-; SKX-NEXT:    pabsw (%rdi), %mm0 # sched: [6:0.50]
-; SKX-NEXT:    pabsw %mm0, %mm0 # sched: [1:0.50]
-; SKX-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_pabsw:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    pabsw (%rdi), %mm0 # sched: [7:0.50]
-; BDVER2-NEXT:    pabsw %mm0, %mm0 # sched: [2:0.50]
-; BDVER2-NEXT:    movq %mm0, %rax # sched: [10:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_pabsw:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    pabsw (%rdi), %mm0 # sched: [6:1.00]
-; BTVER2-NEXT:    pabsw %mm0, %mm0 # sched: [1:0.50]
-; BTVER2-NEXT:    movq %mm0, %rax # sched: [4:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_pabsw:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    pabsw (%rdi), %mm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    pabsw %mm0, %mm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = load x86_mmx, x86_mmx *%a0, align 8
-  %2 = call x86_mmx @llvm.x86.ssse3.pabs.w(x86_mmx %1)
-  %3 = call x86_mmx @llvm.x86.ssse3.pabs.w(x86_mmx %2)
-  %4 = bitcast x86_mmx %3 to i64
-  ret i64 %4
-}
-declare x86_mmx @llvm.x86.ssse3.pabs.w(x86_mmx) nounwind readnone
-
-define i64 @test_packssdw(x86_mmx %a0, x86_mmx %a1, x86_mmx* %a2) optsize {
-; GENERIC-LABEL: test_packssdw:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    packssdw %mm1, %mm0 # sched: [1:1.00]
-; GENERIC-NEXT:    packssdw (%rdi), %mm0 # sched: [6:1.00]
-; GENERIC-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_packssdw:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    packssdw %mm1, %mm0 # sched: [1:0.50]
-; ATOM-NEXT:    packssdw (%rdi), %mm0 # sched: [1:1.00]
-; ATOM-NEXT:    movq %mm0, %rax # sched: [3:3.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_packssdw:
-; SLM:       # %bb.0:
-; SLM-NEXT:    packssdw %mm1, %mm0 # sched: [1:1.00]
-; SLM-NEXT:    packssdw (%rdi), %mm0 # sched: [4:1.00]
-; SLM-NEXT:    movq %mm0, %rax # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_packssdw:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    packssdw %mm1, %mm0 # sched: [1:1.00]
-; SANDY-NEXT:    packssdw (%rdi), %mm0 # sched: [6:1.00]
-; SANDY-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_packssdw:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    packssdw %mm1, %mm0 # sched: [3:2.00]
-; HASWELL-NEXT:    packssdw (%rdi), %mm0 # sched: [7:2.00]
-; HASWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_packssdw:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    packssdw %mm1, %mm0 # sched: [3:2.00]
-; BROADWELL-NEXT:    packssdw (%rdi), %mm0 # sched: [7:2.00]
-; BROADWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_packssdw:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    packssdw %mm1, %mm0 # sched: [3:2.00]
-; SKYLAKE-NEXT:    packssdw (%rdi), %mm0 # sched: [7:2.00]
-; SKYLAKE-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_packssdw:
-; SKX:       # %bb.0:
-; SKX-NEXT:    packssdw %mm1, %mm0 # sched: [3:2.00]
-; SKX-NEXT:    packssdw (%rdi), %mm0 # sched: [7:2.00]
-; SKX-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_packssdw:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    packssdw %mm1, %mm0 # sched: [2:0.50]
-; BDVER2-NEXT:    packssdw (%rdi), %mm0 # sched: [7:0.50]
-; BDVER2-NEXT:    movq %mm0, %rax # sched: [10:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_packssdw:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    packssdw %mm1, %mm0 # sched: [1:0.50]
-; BTVER2-NEXT:    packssdw (%rdi), %mm0 # sched: [6:1.00]
-; BTVER2-NEXT:    movq %mm0, %rax # sched: [4:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_packssdw:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    packssdw %mm1, %mm0 # sched: [1:0.50]
-; ZNVER1-NEXT:    packssdw (%rdi), %mm0 # sched: [1:0.50]
-; ZNVER1-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call x86_mmx @llvm.x86.mmx.packssdw(x86_mmx %a0, x86_mmx %a1)
-  %2 = load x86_mmx, x86_mmx *%a2, align 8
-  %3 = call x86_mmx @llvm.x86.mmx.packssdw(x86_mmx %1, x86_mmx %2)
-  %4 = bitcast x86_mmx %3 to i64
-  ret i64 %4
-}
-declare x86_mmx @llvm.x86.mmx.packssdw(x86_mmx, x86_mmx) nounwind readnone
-
-define i64 @test_packsswb(x86_mmx %a0, x86_mmx %a1, x86_mmx* %a2) optsize {
-; GENERIC-LABEL: test_packsswb:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    packsswb %mm1, %mm0 # sched: [1:1.00]
-; GENERIC-NEXT:    packsswb (%rdi), %mm0 # sched: [6:1.00]
-; GENERIC-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_packsswb:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    packsswb %mm1, %mm0 # sched: [1:0.50]
-; ATOM-NEXT:    packsswb (%rdi), %mm0 # sched: [1:1.00]
-; ATOM-NEXT:    movq %mm0, %rax # sched: [3:3.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_packsswb:
-; SLM:       # %bb.0:
-; SLM-NEXT:    packsswb %mm1, %mm0 # sched: [1:1.00]
-; SLM-NEXT:    packsswb (%rdi), %mm0 # sched: [4:1.00]
-; SLM-NEXT:    movq %mm0, %rax # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_packsswb:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    packsswb %mm1, %mm0 # sched: [1:1.00]
-; SANDY-NEXT:    packsswb (%rdi), %mm0 # sched: [6:1.00]
-; SANDY-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_packsswb:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    packsswb %mm1, %mm0 # sched: [3:2.00]
-; HASWELL-NEXT:    packsswb (%rdi), %mm0 # sched: [7:2.00]
-; HASWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_packsswb:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    packsswb %mm1, %mm0 # sched: [3:2.00]
-; BROADWELL-NEXT:    packsswb (%rdi), %mm0 # sched: [7:2.00]
-; BROADWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_packsswb:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    packsswb %mm1, %mm0 # sched: [3:2.00]
-; SKYLAKE-NEXT:    packsswb (%rdi), %mm0 # sched: [7:2.00]
-; SKYLAKE-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_packsswb:
-; SKX:       # %bb.0:
-; SKX-NEXT:    packsswb %mm1, %mm0 # sched: [3:2.00]
-; SKX-NEXT:    packsswb (%rdi), %mm0 # sched: [7:2.00]
-; SKX-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_packsswb:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    packsswb %mm1, %mm0 # sched: [2:0.50]
-; BDVER2-NEXT:    packsswb (%rdi), %mm0 # sched: [7:0.50]
-; BDVER2-NEXT:    movq %mm0, %rax # sched: [10:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_packsswb:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    packsswb %mm1, %mm0 # sched: [1:0.50]
-; BTVER2-NEXT:    packsswb (%rdi), %mm0 # sched: [6:1.00]
-; BTVER2-NEXT:    movq %mm0, %rax # sched: [4:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_packsswb:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    packsswb %mm1, %mm0 # sched: [1:0.50]
-; ZNVER1-NEXT:    packsswb (%rdi), %mm0 # sched: [1:0.50]
-; ZNVER1-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call x86_mmx @llvm.x86.mmx.packsswb(x86_mmx %a0, x86_mmx %a1)
-  %2 = load x86_mmx, x86_mmx *%a2, align 8
-  %3 = call x86_mmx @llvm.x86.mmx.packsswb(x86_mmx %1, x86_mmx %2)
-  %4 = bitcast x86_mmx %3 to i64
-  ret i64 %4
-}
-declare x86_mmx @llvm.x86.mmx.packsswb(x86_mmx, x86_mmx) nounwind readnone
-
-define i64 @test_packuswb(x86_mmx %a0, x86_mmx %a1, x86_mmx* %a2) optsize {
-; GENERIC-LABEL: test_packuswb:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    packuswb %mm1, %mm0 # sched: [1:1.00]
-; GENERIC-NEXT:    packuswb (%rdi), %mm0 # sched: [6:1.00]
-; GENERIC-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_packuswb:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    packuswb %mm1, %mm0 # sched: [1:0.50]
-; ATOM-NEXT:    packuswb (%rdi), %mm0 # sched: [1:1.00]
-; ATOM-NEXT:    movq %mm0, %rax # sched: [3:3.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_packuswb:
-; SLM:       # %bb.0:
-; SLM-NEXT:    packuswb %mm1, %mm0 # sched: [1:1.00]
-; SLM-NEXT:    packuswb (%rdi), %mm0 # sched: [4:1.00]
-; SLM-NEXT:    movq %mm0, %rax # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_packuswb:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    packuswb %mm1, %mm0 # sched: [1:1.00]
-; SANDY-NEXT:    packuswb (%rdi), %mm0 # sched: [6:1.00]
-; SANDY-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_packuswb:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    packuswb %mm1, %mm0 # sched: [3:2.00]
-; HASWELL-NEXT:    packuswb (%rdi), %mm0 # sched: [7:2.00]
-; HASWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_packuswb:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    packuswb %mm1, %mm0 # sched: [3:2.00]
-; BROADWELL-NEXT:    packuswb (%rdi), %mm0 # sched: [7:2.00]
-; BROADWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_packuswb:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    packuswb %mm1, %mm0 # sched: [3:2.00]
-; SKYLAKE-NEXT:    packuswb (%rdi), %mm0 # sched: [7:2.00]
-; SKYLAKE-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_packuswb:
-; SKX:       # %bb.0:
-; SKX-NEXT:    packuswb %mm1, %mm0 # sched: [3:2.00]
-; SKX-NEXT:    packuswb (%rdi), %mm0 # sched: [7:2.00]
-; SKX-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_packuswb:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    packuswb %mm1, %mm0 # sched: [2:0.50]
-; BDVER2-NEXT:    packuswb (%rdi), %mm0 # sched: [7:0.50]
-; BDVER2-NEXT:    movq %mm0, %rax # sched: [10:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_packuswb:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    packuswb %mm1, %mm0 # sched: [1:0.50]
-; BTVER2-NEXT:    packuswb (%rdi), %mm0 # sched: [6:1.00]
-; BTVER2-NEXT:    movq %mm0, %rax # sched: [4:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_packuswb:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    packuswb %mm1, %mm0 # sched: [1:0.50]
-; ZNVER1-NEXT:    packuswb (%rdi), %mm0 # sched: [1:0.50]
-; ZNVER1-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call x86_mmx @llvm.x86.mmx.packuswb(x86_mmx %a0, x86_mmx %a1)
-  %2 = load x86_mmx, x86_mmx *%a2, align 8
-  %3 = call x86_mmx @llvm.x86.mmx.packuswb(x86_mmx %1, x86_mmx %2)
-  %4 = bitcast x86_mmx %3 to i64
-  ret i64 %4
-}
-declare x86_mmx @llvm.x86.mmx.packuswb(x86_mmx, x86_mmx) nounwind readnone
-
-define i64 @test_paddb(x86_mmx %a0, x86_mmx %a1, x86_mmx* %a2) optsize {
-; GENERIC-LABEL: test_paddb:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    paddb %mm1, %mm0 # sched: [3:1.00]
-; GENERIC-NEXT:    paddb (%rdi), %mm0 # sched: [8:1.00]
-; GENERIC-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_paddb:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    paddb %mm1, %mm0 # sched: [1:0.50]
-; ATOM-NEXT:    paddb (%rdi), %mm0 # sched: [1:1.00]
-; ATOM-NEXT:    movq %mm0, %rax # sched: [3:3.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_paddb:
-; SLM:       # %bb.0:
-; SLM-NEXT:    paddb %mm1, %mm0 # sched: [1:0.50]
-; SLM-NEXT:    paddb (%rdi), %mm0 # sched: [4:1.00]
-; SLM-NEXT:    movq %mm0, %rax # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_paddb:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    paddb %mm1, %mm0 # sched: [3:1.00]
-; SANDY-NEXT:    paddb (%rdi), %mm0 # sched: [8:1.00]
-; SANDY-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_paddb:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    paddb %mm1, %mm0 # sched: [1:0.50]
-; HASWELL-NEXT:    paddb (%rdi), %mm0 # sched: [6:0.50]
-; HASWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_paddb:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    paddb %mm1, %mm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    paddb (%rdi), %mm0 # sched: [6:0.50]
-; BROADWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_paddb:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    paddb %mm1, %mm0 # sched: [1:0.50]
-; SKYLAKE-NEXT:    paddb (%rdi), %mm0 # sched: [6:0.50]
-; SKYLAKE-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_paddb:
-; SKX:       # %bb.0:
-; SKX-NEXT:    paddb %mm1, %mm0 # sched: [1:0.50]
-; SKX-NEXT:    paddb (%rdi), %mm0 # sched: [6:0.50]
-; SKX-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_paddb:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    paddb %mm1, %mm0 # sched: [2:0.50]
-; BDVER2-NEXT:    paddb (%rdi), %mm0 # sched: [7:0.50]
-; BDVER2-NEXT:    movq %mm0, %rax # sched: [10:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_paddb:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    paddb %mm1, %mm0 # sched: [1:0.50]
-; BTVER2-NEXT:    paddb (%rdi), %mm0 # sched: [6:1.00]
-; BTVER2-NEXT:    movq %mm0, %rax # sched: [4:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_paddb:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    paddb %mm1, %mm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    paddb (%rdi), %mm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call x86_mmx @llvm.x86.mmx.padd.b(x86_mmx %a0, x86_mmx %a1)
-  %2 = load x86_mmx, x86_mmx *%a2, align 8
-  %3 = call x86_mmx @llvm.x86.mmx.padd.b(x86_mmx %1, x86_mmx %2)
-  %4 = bitcast x86_mmx %3 to i64
-  ret i64 %4
-}
-declare x86_mmx @llvm.x86.mmx.padd.b(x86_mmx, x86_mmx) nounwind readnone
-
-define i64 @test_paddd(x86_mmx %a0, x86_mmx %a1, x86_mmx* %a2) optsize {
-; GENERIC-LABEL: test_paddd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    paddd %mm1, %mm0 # sched: [3:1.00]
-; GENERIC-NEXT:    paddd (%rdi), %mm0 # sched: [8:1.00]
-; GENERIC-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_paddd:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    paddd %mm1, %mm0 # sched: [1:0.50]
-; ATOM-NEXT:    paddd (%rdi), %mm0 # sched: [1:1.00]
-; ATOM-NEXT:    movq %mm0, %rax # sched: [3:3.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_paddd:
-; SLM:       # %bb.0:
-; SLM-NEXT:    paddd %mm1, %mm0 # sched: [1:0.50]
-; SLM-NEXT:    paddd (%rdi), %mm0 # sched: [4:1.00]
-; SLM-NEXT:    movq %mm0, %rax # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_paddd:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    paddd %mm1, %mm0 # sched: [3:1.00]
-; SANDY-NEXT:    paddd (%rdi), %mm0 # sched: [8:1.00]
-; SANDY-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_paddd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    paddd %mm1, %mm0 # sched: [1:0.50]
-; HASWELL-NEXT:    paddd (%rdi), %mm0 # sched: [6:0.50]
-; HASWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_paddd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    paddd %mm1, %mm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    paddd (%rdi), %mm0 # sched: [6:0.50]
-; BROADWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_paddd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    paddd %mm1, %mm0 # sched: [1:0.50]
-; SKYLAKE-NEXT:    paddd (%rdi), %mm0 # sched: [6:0.50]
-; SKYLAKE-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_paddd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    paddd %mm1, %mm0 # sched: [1:0.50]
-; SKX-NEXT:    paddd (%rdi), %mm0 # sched: [6:0.50]
-; SKX-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_paddd:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    paddd %mm1, %mm0 # sched: [2:0.50]
-; BDVER2-NEXT:    paddd (%rdi), %mm0 # sched: [7:0.50]
-; BDVER2-NEXT:    movq %mm0, %rax # sched: [10:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_paddd:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    paddd %mm1, %mm0 # sched: [1:0.50]
-; BTVER2-NEXT:    paddd (%rdi), %mm0 # sched: [6:1.00]
-; BTVER2-NEXT:    movq %mm0, %rax # sched: [4:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_paddd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    paddd %mm1, %mm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    paddd (%rdi), %mm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call x86_mmx @llvm.x86.mmx.padd.d(x86_mmx %a0, x86_mmx %a1)
-  %2 = load x86_mmx, x86_mmx *%a2, align 8
-  %3 = call x86_mmx @llvm.x86.mmx.padd.d(x86_mmx %1, x86_mmx %2)
-  %4 = bitcast x86_mmx %3 to i64
-  ret i64 %4
-}
-declare x86_mmx @llvm.x86.mmx.padd.d(x86_mmx, x86_mmx) nounwind readnone
-
-define i64 @test_paddq(x86_mmx %a0, x86_mmx %a1, x86_mmx* %a2) optsize {
-; GENERIC-LABEL: test_paddq:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    paddq %mm1, %mm0 # sched: [1:0.50]
-; GENERIC-NEXT:    paddq (%rdi), %mm0 # sched: [7:0.50]
-; GENERIC-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_paddq:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    paddq %mm1, %mm0 # sched: [2:1.00]
-; ATOM-NEXT:    paddq (%rdi), %mm0 # sched: [3:1.50]
-; ATOM-NEXT:    movq %mm0, %rax # sched: [3:3.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_paddq:
-; SLM:       # %bb.0:
-; SLM-NEXT:    paddq %mm1, %mm0 # sched: [1:0.50]
-; SLM-NEXT:    paddq (%rdi), %mm0 # sched: [4:1.00]
-; SLM-NEXT:    movq %mm0, %rax # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_paddq:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    paddq %mm1, %mm0 # sched: [1:0.50]
-; SANDY-NEXT:    paddq (%rdi), %mm0 # sched: [7:0.50]
-; SANDY-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_paddq:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    paddq %mm1, %mm0 # sched: [1:0.50]
-; HASWELL-NEXT:    paddq (%rdi), %mm0 # sched: [6:0.50]
-; HASWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_paddq:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    paddq %mm1, %mm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    paddq (%rdi), %mm0 # sched: [6:0.50]
-; BROADWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_paddq:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    paddq %mm1, %mm0 # sched: [1:0.50]
-; SKYLAKE-NEXT:    paddq (%rdi), %mm0 # sched: [6:0.50]
-; SKYLAKE-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_paddq:
-; SKX:       # %bb.0:
-; SKX-NEXT:    paddq %mm1, %mm0 # sched: [1:0.50]
-; SKX-NEXT:    paddq (%rdi), %mm0 # sched: [6:0.50]
-; SKX-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_paddq:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    paddq %mm1, %mm0 # sched: [2:0.50]
-; BDVER2-NEXT:    paddq (%rdi), %mm0 # sched: [7:0.50]
-; BDVER2-NEXT:    movq %mm0, %rax # sched: [10:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_paddq:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    paddq %mm1, %mm0 # sched: [1:0.50]
-; BTVER2-NEXT:    paddq (%rdi), %mm0 # sched: [6:1.00]
-; BTVER2-NEXT:    movq %mm0, %rax # sched: [4:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_paddq:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    paddq %mm1, %mm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    paddq (%rdi), %mm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call x86_mmx @llvm.x86.mmx.padd.q(x86_mmx %a0, x86_mmx %a1)
-  %2 = load x86_mmx, x86_mmx *%a2, align 8
-  %3 = call x86_mmx @llvm.x86.mmx.padd.q(x86_mmx %1, x86_mmx %2)
-  %4 = bitcast x86_mmx %3 to i64
-  ret i64 %4
-}
-declare x86_mmx @llvm.x86.mmx.padd.q(x86_mmx, x86_mmx) nounwind readnone
-
-define i64 @test_paddsb(x86_mmx %a0, x86_mmx %a1, x86_mmx* %a2) optsize {
-; GENERIC-LABEL: test_paddsb:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    paddsb %mm1, %mm0 # sched: [3:1.00]
-; GENERIC-NEXT:    paddsb (%rdi), %mm0 # sched: [8:1.00]
-; GENERIC-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_paddsb:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    paddsb %mm1, %mm0 # sched: [1:0.50]
-; ATOM-NEXT:    paddsb (%rdi), %mm0 # sched: [1:1.00]
-; ATOM-NEXT:    movq %mm0, %rax # sched: [3:3.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_paddsb:
-; SLM:       # %bb.0:
-; SLM-NEXT:    paddsb %mm1, %mm0 # sched: [1:0.50]
-; SLM-NEXT:    paddsb (%rdi), %mm0 # sched: [4:1.00]
-; SLM-NEXT:    movq %mm0, %rax # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_paddsb:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    paddsb %mm1, %mm0 # sched: [3:1.00]
-; SANDY-NEXT:    paddsb (%rdi), %mm0 # sched: [8:1.00]
-; SANDY-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_paddsb:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    paddsb %mm1, %mm0 # sched: [1:0.50]
-; HASWELL-NEXT:    paddsb (%rdi), %mm0 # sched: [6:0.50]
-; HASWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_paddsb:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    paddsb %mm1, %mm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    paddsb (%rdi), %mm0 # sched: [6:0.50]
-; BROADWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_paddsb:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    paddsb %mm1, %mm0 # sched: [1:1.00]
-; SKYLAKE-NEXT:    paddsb (%rdi), %mm0 # sched: [6:1.00]
-; SKYLAKE-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_paddsb:
-; SKX:       # %bb.0:
-; SKX-NEXT:    paddsb %mm1, %mm0 # sched: [1:1.00]
-; SKX-NEXT:    paddsb (%rdi), %mm0 # sched: [6:1.00]
-; SKX-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_paddsb:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    paddsb %mm1, %mm0 # sched: [2:0.50]
-; BDVER2-NEXT:    paddsb (%rdi), %mm0 # sched: [7:0.50]
-; BDVER2-NEXT:    movq %mm0, %rax # sched: [10:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_paddsb:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    paddsb %mm1, %mm0 # sched: [1:0.50]
-; BTVER2-NEXT:    paddsb (%rdi), %mm0 # sched: [6:1.00]
-; BTVER2-NEXT:    movq %mm0, %rax # sched: [4:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_paddsb:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    paddsb %mm1, %mm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    paddsb (%rdi), %mm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call x86_mmx @llvm.x86.mmx.padds.b(x86_mmx %a0, x86_mmx %a1)
-  %2 = load x86_mmx, x86_mmx *%a2, align 8
-  %3 = call x86_mmx @llvm.x86.mmx.padds.b(x86_mmx %1, x86_mmx %2)
-  %4 = bitcast x86_mmx %3 to i64
-  ret i64 %4
-}
-declare x86_mmx @llvm.x86.mmx.padds.b(x86_mmx, x86_mmx) nounwind readnone
-
-define i64 @test_paddsw(x86_mmx %a0, x86_mmx %a1, x86_mmx* %a2) optsize {
-; GENERIC-LABEL: test_paddsw:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    paddsw %mm1, %mm0 # sched: [3:1.00]
-; GENERIC-NEXT:    paddsw (%rdi), %mm0 # sched: [8:1.00]
-; GENERIC-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_paddsw:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    paddsw %mm1, %mm0 # sched: [1:0.50]
-; ATOM-NEXT:    paddsw (%rdi), %mm0 # sched: [1:1.00]
-; ATOM-NEXT:    movq %mm0, %rax # sched: [3:3.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_paddsw:
-; SLM:       # %bb.0:
-; SLM-NEXT:    paddsw %mm1, %mm0 # sched: [1:0.50]
-; SLM-NEXT:    paddsw (%rdi), %mm0 # sched: [4:1.00]
-; SLM-NEXT:    movq %mm0, %rax # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_paddsw:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    paddsw %mm1, %mm0 # sched: [3:1.00]
-; SANDY-NEXT:    paddsw (%rdi), %mm0 # sched: [8:1.00]
-; SANDY-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_paddsw:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    paddsw %mm1, %mm0 # sched: [1:0.50]
-; HASWELL-NEXT:    paddsw (%rdi), %mm0 # sched: [6:0.50]
-; HASWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_paddsw:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    paddsw %mm1, %mm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    paddsw (%rdi), %mm0 # sched: [6:0.50]
-; BROADWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_paddsw:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    paddsw %mm1, %mm0 # sched: [1:1.00]
-; SKYLAKE-NEXT:    paddsw (%rdi), %mm0 # sched: [6:1.00]
-; SKYLAKE-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_paddsw:
-; SKX:       # %bb.0:
-; SKX-NEXT:    paddsw %mm1, %mm0 # sched: [1:1.00]
-; SKX-NEXT:    paddsw (%rdi), %mm0 # sched: [6:1.00]
-; SKX-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_paddsw:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    paddsw %mm1, %mm0 # sched: [2:0.50]
-; BDVER2-NEXT:    paddsw (%rdi), %mm0 # sched: [7:0.50]
-; BDVER2-NEXT:    movq %mm0, %rax # sched: [10:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_paddsw:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    paddsw %mm1, %mm0 # sched: [1:0.50]
-; BTVER2-NEXT:    paddsw (%rdi), %mm0 # sched: [6:1.00]
-; BTVER2-NEXT:    movq %mm0, %rax # sched: [4:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_paddsw:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    paddsw %mm1, %mm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    paddsw (%rdi), %mm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call x86_mmx @llvm.x86.mmx.padds.w(x86_mmx %a0, x86_mmx %a1)
-  %2 = load x86_mmx, x86_mmx *%a2, align 8
-  %3 = call x86_mmx @llvm.x86.mmx.padds.w(x86_mmx %1, x86_mmx %2)
-  %4 = bitcast x86_mmx %3 to i64
-  ret i64 %4
-}
-declare x86_mmx @llvm.x86.mmx.padds.w(x86_mmx, x86_mmx) nounwind readnone
-
-define i64 @test_paddusb(x86_mmx %a0, x86_mmx %a1, x86_mmx* %a2) optsize {
-; GENERIC-LABEL: test_paddusb:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    paddusb %mm1, %mm0 # sched: [3:1.00]
-; GENERIC-NEXT:    paddusb (%rdi), %mm0 # sched: [8:1.00]
-; GENERIC-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_paddusb:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    paddusb %mm1, %mm0 # sched: [1:0.50]
-; ATOM-NEXT:    paddusb (%rdi), %mm0 # sched: [1:1.00]
-; ATOM-NEXT:    movq %mm0, %rax # sched: [3:3.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_paddusb:
-; SLM:       # %bb.0:
-; SLM-NEXT:    paddusb %mm1, %mm0 # sched: [1:0.50]
-; SLM-NEXT:    paddusb (%rdi), %mm0 # sched: [4:1.00]
-; SLM-NEXT:    movq %mm0, %rax # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_paddusb:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    paddusb %mm1, %mm0 # sched: [3:1.00]
-; SANDY-NEXT:    paddusb (%rdi), %mm0 # sched: [8:1.00]
-; SANDY-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_paddusb:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    paddusb %mm1, %mm0 # sched: [1:0.50]
-; HASWELL-NEXT:    paddusb (%rdi), %mm0 # sched: [6:0.50]
-; HASWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_paddusb:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    paddusb %mm1, %mm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    paddusb (%rdi), %mm0 # sched: [6:0.50]
-; BROADWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_paddusb:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    paddusb %mm1, %mm0 # sched: [1:1.00]
-; SKYLAKE-NEXT:    paddusb (%rdi), %mm0 # sched: [6:1.00]
-; SKYLAKE-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_paddusb:
-; SKX:       # %bb.0:
-; SKX-NEXT:    paddusb %mm1, %mm0 # sched: [1:1.00]
-; SKX-NEXT:    paddusb (%rdi), %mm0 # sched: [6:1.00]
-; SKX-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_paddusb:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    paddusb %mm1, %mm0 # sched: [2:0.50]
-; BDVER2-NEXT:    paddusb (%rdi), %mm0 # sched: [7:0.50]
-; BDVER2-NEXT:    movq %mm0, %rax # sched: [10:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_paddusb:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    paddusb %mm1, %mm0 # sched: [1:0.50]
-; BTVER2-NEXT:    paddusb (%rdi), %mm0 # sched: [6:1.00]
-; BTVER2-NEXT:    movq %mm0, %rax # sched: [4:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_paddusb:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    paddusb %mm1, %mm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    paddusb (%rdi), %mm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call x86_mmx @llvm.x86.mmx.paddus.b(x86_mmx %a0, x86_mmx %a1)
-  %2 = load x86_mmx, x86_mmx *%a2, align 8
-  %3 = call x86_mmx @llvm.x86.mmx.paddus.b(x86_mmx %1, x86_mmx %2)
-  %4 = bitcast x86_mmx %3 to i64
-  ret i64 %4
-}
-declare x86_mmx @llvm.x86.mmx.paddus.b(x86_mmx, x86_mmx) nounwind readnone
-
-define i64 @test_paddusw(x86_mmx %a0, x86_mmx %a1, x86_mmx* %a2) optsize {
-; GENERIC-LABEL: test_paddusw:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    paddusw %mm1, %mm0 # sched: [3:1.00]
-; GENERIC-NEXT:    paddusw (%rdi), %mm0 # sched: [8:1.00]
-; GENERIC-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_paddusw:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    paddusw %mm1, %mm0 # sched: [1:0.50]
-; ATOM-NEXT:    paddusw (%rdi), %mm0 # sched: [1:1.00]
-; ATOM-NEXT:    movq %mm0, %rax # sched: [3:3.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_paddusw:
-; SLM:       # %bb.0:
-; SLM-NEXT:    paddusw %mm1, %mm0 # sched: [1:0.50]
-; SLM-NEXT:    paddusw (%rdi), %mm0 # sched: [4:1.00]
-; SLM-NEXT:    movq %mm0, %rax # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_paddusw:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    paddusw %mm1, %mm0 # sched: [3:1.00]
-; SANDY-NEXT:    paddusw (%rdi), %mm0 # sched: [8:1.00]
-; SANDY-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_paddusw:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    paddusw %mm1, %mm0 # sched: [1:0.50]
-; HASWELL-NEXT:    paddusw (%rdi), %mm0 # sched: [6:0.50]
-; HASWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_paddusw:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    paddusw %mm1, %mm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    paddusw (%rdi), %mm0 # sched: [6:0.50]
-; BROADWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_paddusw:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    paddusw %mm1, %mm0 # sched: [1:1.00]
-; SKYLAKE-NEXT:    paddusw (%rdi), %mm0 # sched: [6:1.00]
-; SKYLAKE-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_paddusw:
-; SKX:       # %bb.0:
-; SKX-NEXT:    paddusw %mm1, %mm0 # sched: [1:1.00]
-; SKX-NEXT:    paddusw (%rdi), %mm0 # sched: [6:1.00]
-; SKX-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_paddusw:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    paddusw %mm1, %mm0 # sched: [2:0.50]
-; BDVER2-NEXT:    paddusw (%rdi), %mm0 # sched: [7:0.50]
-; BDVER2-NEXT:    movq %mm0, %rax # sched: [10:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_paddusw:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    paddusw %mm1, %mm0 # sched: [1:0.50]
-; BTVER2-NEXT:    paddusw (%rdi), %mm0 # sched: [6:1.00]
-; BTVER2-NEXT:    movq %mm0, %rax # sched: [4:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_paddusw:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    paddusw %mm1, %mm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    paddusw (%rdi), %mm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call x86_mmx @llvm.x86.mmx.paddus.w(x86_mmx %a0, x86_mmx %a1)
-  %2 = load x86_mmx, x86_mmx *%a2, align 8
-  %3 = call x86_mmx @llvm.x86.mmx.paddus.w(x86_mmx %1, x86_mmx %2)
-  %4 = bitcast x86_mmx %3 to i64
-  ret i64 %4
-}
-declare x86_mmx @llvm.x86.mmx.paddus.w(x86_mmx, x86_mmx) nounwind readnone
-
-define i64 @test_paddw(x86_mmx %a0, x86_mmx %a1, x86_mmx* %a2) optsize {
-; GENERIC-LABEL: test_paddw:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    paddw %mm1, %mm0 # sched: [3:1.00]
-; GENERIC-NEXT:    paddw (%rdi), %mm0 # sched: [8:1.00]
-; GENERIC-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_paddw:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    paddw %mm1, %mm0 # sched: [1:0.50]
-; ATOM-NEXT:    paddw (%rdi), %mm0 # sched: [1:1.00]
-; ATOM-NEXT:    movq %mm0, %rax # sched: [3:3.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_paddw:
-; SLM:       # %bb.0:
-; SLM-NEXT:    paddw %mm1, %mm0 # sched: [1:0.50]
-; SLM-NEXT:    paddw (%rdi), %mm0 # sched: [4:1.00]
-; SLM-NEXT:    movq %mm0, %rax # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_paddw:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    paddw %mm1, %mm0 # sched: [3:1.00]
-; SANDY-NEXT:    paddw (%rdi), %mm0 # sched: [8:1.00]
-; SANDY-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_paddw:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    paddw %mm1, %mm0 # sched: [1:0.50]
-; HASWELL-NEXT:    paddw (%rdi), %mm0 # sched: [6:0.50]
-; HASWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_paddw:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    paddw %mm1, %mm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    paddw (%rdi), %mm0 # sched: [6:0.50]
-; BROADWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_paddw:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    paddw %mm1, %mm0 # sched: [1:0.50]
-; SKYLAKE-NEXT:    paddw (%rdi), %mm0 # sched: [6:0.50]
-; SKYLAKE-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_paddw:
-; SKX:       # %bb.0:
-; SKX-NEXT:    paddw %mm1, %mm0 # sched: [1:0.50]
-; SKX-NEXT:    paddw (%rdi), %mm0 # sched: [6:0.50]
-; SKX-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_paddw:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    paddw %mm1, %mm0 # sched: [2:0.50]
-; BDVER2-NEXT:    paddw (%rdi), %mm0 # sched: [7:0.50]
-; BDVER2-NEXT:    movq %mm0, %rax # sched: [10:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_paddw:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    paddw %mm1, %mm0 # sched: [1:0.50]
-; BTVER2-NEXT:    paddw (%rdi), %mm0 # sched: [6:1.00]
-; BTVER2-NEXT:    movq %mm0, %rax # sched: [4:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_paddw:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    paddw %mm1, %mm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    paddw (%rdi), %mm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call x86_mmx @llvm.x86.mmx.padd.w(x86_mmx %a0, x86_mmx %a1)
-  %2 = load x86_mmx, x86_mmx *%a2, align 8
-  %3 = call x86_mmx @llvm.x86.mmx.padd.w(x86_mmx %1, x86_mmx %2)
-  %4 = bitcast x86_mmx %3 to i64
-  ret i64 %4
-}
-declare x86_mmx @llvm.x86.mmx.padd.w(x86_mmx, x86_mmx) nounwind readnone
-
-define i64 @test_palignr(x86_mmx %a0, x86_mmx %a1, x86_mmx* %a2) optsize {
-; GENERIC-LABEL: test_palignr:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    palignr $1, %mm1, %mm0 # sched: [1:0.50]
-; GENERIC-NEXT:    palignr $1, (%rdi), %mm0 # sched: [6:0.50]
-; GENERIC-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_palignr:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    palignr $1, %mm1, %mm0 # sched: [1:1.00]
-; ATOM-NEXT:    palignr $1, (%rdi), %mm0 # sched: [1:1.00]
-; ATOM-NEXT:    movq %mm0, %rax # sched: [3:3.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_palignr:
-; SLM:       # %bb.0:
-; SLM-NEXT:    palignr $1, %mm1, %mm0 # sched: [1:1.00]
-; SLM-NEXT:    palignr $1, (%rdi), %mm0 # sched: [4:1.00]
-; SLM-NEXT:    movq %mm0, %rax # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_palignr:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    palignr $1, %mm1, %mm0 # sched: [1:0.50]
-; SANDY-NEXT:    palignr $1, (%rdi), %mm0 # sched: [6:0.50]
-; SANDY-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_palignr:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    palignr $1, %mm1, %mm0 # sched: [1:1.00]
-; HASWELL-NEXT:    palignr $1, (%rdi), %mm0 # sched: [6:1.00]
-; HASWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_palignr:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    palignr $1, %mm1, %mm0 # sched: [1:1.00]
-; BROADWELL-NEXT:    palignr $1, (%rdi), %mm0 # sched: [6:1.00]
-; BROADWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_palignr:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    palignr $1, %mm1, %mm0 # sched: [1:1.00]
-; SKYLAKE-NEXT:    palignr $1, (%rdi), %mm0 # sched: [6:1.00]
-; SKYLAKE-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_palignr:
-; SKX:       # %bb.0:
-; SKX-NEXT:    palignr $1, %mm1, %mm0 # sched: [1:1.00]
-; SKX-NEXT:    palignr $1, (%rdi), %mm0 # sched: [6:1.00]
-; SKX-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_palignr:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    palignr $1, %mm1, %mm0 # sched: [2:0.50]
-; BDVER2-NEXT:    palignr $1, (%rdi), %mm0 # sched: [7:0.50]
-; BDVER2-NEXT:    movq %mm0, %rax # sched: [10:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_palignr:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    palignr $1, %mm1, %mm0 # sched: [1:0.50]
-; BTVER2-NEXT:    palignr $1, (%rdi), %mm0 # sched: [6:1.00]
-; BTVER2-NEXT:    movq %mm0, %rax # sched: [4:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_palignr:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    palignr $1, %mm1, %mm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    palignr $1, (%rdi), %mm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call x86_mmx @llvm.x86.mmx.palignr.b(x86_mmx %a0, x86_mmx %a1, i8 1)
-  %2 = load x86_mmx, x86_mmx *%a2, align 8
-  %3 = call x86_mmx @llvm.x86.mmx.palignr.b(x86_mmx %1, x86_mmx %2, i8 1)
-  %4 = bitcast x86_mmx %3 to i64
-  ret i64 %4
-}
-declare x86_mmx @llvm.x86.mmx.palignr.b(x86_mmx, x86_mmx, i8) nounwind readnone
-
-define i64 @test_pand(x86_mmx %a0, x86_mmx %a1, x86_mmx* %a2) optsize {
-; GENERIC-LABEL: test_pand:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    pand %mm1, %mm0 # sched: [1:0.33]
-; GENERIC-NEXT:    pand (%rdi), %mm0 # sched: [6:0.50]
-; GENERIC-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_pand:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    pand %mm1, %mm0 # sched: [1:0.50]
-; ATOM-NEXT:    pand (%rdi), %mm0 # sched: [1:1.00]
-; ATOM-NEXT:    movq %mm0, %rax # sched: [3:3.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_pand:
-; SLM:       # %bb.0:
-; SLM-NEXT:    pand %mm1, %mm0 # sched: [1:0.50]
-; SLM-NEXT:    pand (%rdi), %mm0 # sched: [4:1.00]
-; SLM-NEXT:    movq %mm0, %rax # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_pand:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    pand %mm1, %mm0 # sched: [1:0.33]
-; SANDY-NEXT:    pand (%rdi), %mm0 # sched: [6:0.50]
-; SANDY-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_pand:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    pand %mm1, %mm0 # sched: [1:0.33]
-; HASWELL-NEXT:    pand (%rdi), %mm0 # sched: [6:0.50]
-; HASWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pand:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    pand %mm1, %mm0 # sched: [1:0.33]
-; BROADWELL-NEXT:    pand (%rdi), %mm0 # sched: [6:0.50]
-; BROADWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pand:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    pand %mm1, %mm0 # sched: [1:0.50]
-; SKYLAKE-NEXT:    pand (%rdi), %mm0 # sched: [6:0.50]
-; SKYLAKE-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pand:
-; SKX:       # %bb.0:
-; SKX-NEXT:    pand %mm1, %mm0 # sched: [1:0.50]
-; SKX-NEXT:    pand (%rdi), %mm0 # sched: [6:0.50]
-; SKX-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_pand:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    pand %mm1, %mm0 # sched: [2:0.50]
-; BDVER2-NEXT:    pand (%rdi), %mm0 # sched: [7:0.50]
-; BDVER2-NEXT:    movq %mm0, %rax # sched: [10:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_pand:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    pand %mm1, %mm0 # sched: [1:0.50]
-; BTVER2-NEXT:    pand (%rdi), %mm0 # sched: [6:1.00]
-; BTVER2-NEXT:    movq %mm0, %rax # sched: [4:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_pand:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    pand %mm1, %mm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    pand (%rdi), %mm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call x86_mmx @llvm.x86.mmx.pand(x86_mmx %a0, x86_mmx %a1)
-  %2 = load x86_mmx, x86_mmx *%a2, align 8
-  %3 = call x86_mmx @llvm.x86.mmx.pand(x86_mmx %1, x86_mmx %2)
-  %4 = bitcast x86_mmx %3 to i64
-  ret i64 %4
-}
-declare x86_mmx @llvm.x86.mmx.pand(x86_mmx, x86_mmx) nounwind readnone
-
-define i64 @test_pandn(x86_mmx %a0, x86_mmx %a1, x86_mmx* %a2) optsize {
-; GENERIC-LABEL: test_pandn:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    pandn %mm1, %mm0 # sched: [1:0.33]
-; GENERIC-NEXT:    pandn (%rdi), %mm0 # sched: [6:0.50]
-; GENERIC-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_pandn:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    pandn %mm1, %mm0 # sched: [1:0.50]
-; ATOM-NEXT:    pandn (%rdi), %mm0 # sched: [1:1.00]
-; ATOM-NEXT:    movq %mm0, %rax # sched: [3:3.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_pandn:
-; SLM:       # %bb.0:
-; SLM-NEXT:    pandn %mm1, %mm0 # sched: [1:0.50]
-; SLM-NEXT:    pandn (%rdi), %mm0 # sched: [4:1.00]
-; SLM-NEXT:    movq %mm0, %rax # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_pandn:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    pandn %mm1, %mm0 # sched: [1:0.33]
-; SANDY-NEXT:    pandn (%rdi), %mm0 # sched: [6:0.50]
-; SANDY-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_pandn:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    pandn %mm1, %mm0 # sched: [1:0.33]
-; HASWELL-NEXT:    pandn (%rdi), %mm0 # sched: [6:0.50]
-; HASWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pandn:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    pandn %mm1, %mm0 # sched: [1:0.33]
-; BROADWELL-NEXT:    pandn (%rdi), %mm0 # sched: [6:0.50]
-; BROADWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pandn:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    pandn %mm1, %mm0 # sched: [1:0.50]
-; SKYLAKE-NEXT:    pandn (%rdi), %mm0 # sched: [6:0.50]
-; SKYLAKE-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pandn:
-; SKX:       # %bb.0:
-; SKX-NEXT:    pandn %mm1, %mm0 # sched: [1:0.50]
-; SKX-NEXT:    pandn (%rdi), %mm0 # sched: [6:0.50]
-; SKX-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_pandn:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    pandn %mm1, %mm0 # sched: [2:0.50]
-; BDVER2-NEXT:    pandn (%rdi), %mm0 # sched: [7:0.50]
-; BDVER2-NEXT:    movq %mm0, %rax # sched: [10:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_pandn:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    pandn %mm1, %mm0 # sched: [1:0.50]
-; BTVER2-NEXT:    pandn (%rdi), %mm0 # sched: [6:1.00]
-; BTVER2-NEXT:    movq %mm0, %rax # sched: [4:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_pandn:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    pandn %mm1, %mm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    pandn (%rdi), %mm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call x86_mmx @llvm.x86.mmx.pandn(x86_mmx %a0, x86_mmx %a1)
-  %2 = load x86_mmx, x86_mmx *%a2, align 8
-  %3 = call x86_mmx @llvm.x86.mmx.pandn(x86_mmx %1, x86_mmx %2)
-  %4 = bitcast x86_mmx %3 to i64
-  ret i64 %4
-}
-declare x86_mmx @llvm.x86.mmx.pandn(x86_mmx, x86_mmx) nounwind readnone
-
-define i64 @test_pavgb(x86_mmx %a0, x86_mmx %a1, x86_mmx* %a2) optsize {
-; GENERIC-LABEL: test_pavgb:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    pavgb %mm1, %mm0 # sched: [3:1.00]
-; GENERIC-NEXT:    pavgb (%rdi), %mm0 # sched: [8:1.00]
-; GENERIC-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_pavgb:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    pavgb %mm1, %mm0 # sched: [1:0.50]
-; ATOM-NEXT:    pavgb (%rdi), %mm0 # sched: [1:1.00]
-; ATOM-NEXT:    movq %mm0, %rax # sched: [3:3.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_pavgb:
-; SLM:       # %bb.0:
-; SLM-NEXT:    pavgb %mm1, %mm0 # sched: [1:0.50]
-; SLM-NEXT:    pavgb (%rdi), %mm0 # sched: [4:1.00]
-; SLM-NEXT:    movq %mm0, %rax # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_pavgb:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    pavgb %mm1, %mm0 # sched: [3:1.00]
-; SANDY-NEXT:    pavgb (%rdi), %mm0 # sched: [8:1.00]
-; SANDY-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_pavgb:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    pavgb %mm1, %mm0 # sched: [1:0.50]
-; HASWELL-NEXT:    pavgb (%rdi), %mm0 # sched: [6:0.50]
-; HASWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pavgb:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    pavgb %mm1, %mm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    pavgb (%rdi), %mm0 # sched: [6:0.50]
-; BROADWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pavgb:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    pavgb %mm1, %mm0 # sched: [1:1.00]
-; SKYLAKE-NEXT:    pavgb (%rdi), %mm0 # sched: [6:1.00]
-; SKYLAKE-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pavgb:
-; SKX:       # %bb.0:
-; SKX-NEXT:    pavgb %mm1, %mm0 # sched: [1:1.00]
-; SKX-NEXT:    pavgb (%rdi), %mm0 # sched: [6:1.00]
-; SKX-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_pavgb:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    pavgb %mm1, %mm0 # sched: [2:0.50]
-; BDVER2-NEXT:    pavgb (%rdi), %mm0 # sched: [7:0.50]
-; BDVER2-NEXT:    movq %mm0, %rax # sched: [10:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_pavgb:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    pavgb %mm1, %mm0 # sched: [1:0.50]
-; BTVER2-NEXT:    pavgb (%rdi), %mm0 # sched: [6:1.00]
-; BTVER2-NEXT:    movq %mm0, %rax # sched: [4:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_pavgb:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    pavgb %mm1, %mm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    pavgb (%rdi), %mm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call x86_mmx @llvm.x86.mmx.pavg.b(x86_mmx %a0, x86_mmx %a1)
-  %2 = load x86_mmx, x86_mmx *%a2, align 8
-  %3 = call x86_mmx @llvm.x86.mmx.pavg.b(x86_mmx %1, x86_mmx %2)
-  %4 = bitcast x86_mmx %3 to i64
-  ret i64 %4
-}
-declare x86_mmx @llvm.x86.mmx.pavg.b(x86_mmx, x86_mmx) nounwind readnone
-
-define i64 @test_pavgw(x86_mmx %a0, x86_mmx %a1, x86_mmx* %a2) optsize {
-; GENERIC-LABEL: test_pavgw:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    pavgw %mm1, %mm0 # sched: [3:1.00]
-; GENERIC-NEXT:    pavgw (%rdi), %mm0 # sched: [8:1.00]
-; GENERIC-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_pavgw:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    pavgw %mm1, %mm0 # sched: [1:0.50]
-; ATOM-NEXT:    pavgw (%rdi), %mm0 # sched: [1:1.00]
-; ATOM-NEXT:    movq %mm0, %rax # sched: [3:3.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_pavgw:
-; SLM:       # %bb.0:
-; SLM-NEXT:    pavgw %mm1, %mm0 # sched: [1:0.50]
-; SLM-NEXT:    pavgw (%rdi), %mm0 # sched: [4:1.00]
-; SLM-NEXT:    movq %mm0, %rax # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_pavgw:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    pavgw %mm1, %mm0 # sched: [3:1.00]
-; SANDY-NEXT:    pavgw (%rdi), %mm0 # sched: [8:1.00]
-; SANDY-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_pavgw:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    pavgw %mm1, %mm0 # sched: [1:0.50]
-; HASWELL-NEXT:    pavgw (%rdi), %mm0 # sched: [6:0.50]
-; HASWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pavgw:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    pavgw %mm1, %mm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    pavgw (%rdi), %mm0 # sched: [6:0.50]
-; BROADWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pavgw:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    pavgw %mm1, %mm0 # sched: [1:1.00]
-; SKYLAKE-NEXT:    pavgw (%rdi), %mm0 # sched: [6:1.00]
-; SKYLAKE-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pavgw:
-; SKX:       # %bb.0:
-; SKX-NEXT:    pavgw %mm1, %mm0 # sched: [1:1.00]
-; SKX-NEXT:    pavgw (%rdi), %mm0 # sched: [6:1.00]
-; SKX-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_pavgw:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    pavgw %mm1, %mm0 # sched: [2:0.50]
-; BDVER2-NEXT:    pavgw (%rdi), %mm0 # sched: [7:0.50]
-; BDVER2-NEXT:    movq %mm0, %rax # sched: [10:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_pavgw:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    pavgw %mm1, %mm0 # sched: [1:0.50]
-; BTVER2-NEXT:    pavgw (%rdi), %mm0 # sched: [6:1.00]
-; BTVER2-NEXT:    movq %mm0, %rax # sched: [4:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_pavgw:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    pavgw %mm1, %mm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    pavgw (%rdi), %mm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call x86_mmx @llvm.x86.mmx.pavg.w(x86_mmx %a0, x86_mmx %a1)
-  %2 = load x86_mmx, x86_mmx *%a2, align 8
-  %3 = call x86_mmx @llvm.x86.mmx.pavg.w(x86_mmx %1, x86_mmx %2)
-  %4 = bitcast x86_mmx %3 to i64
-  ret i64 %4
-}
-declare x86_mmx @llvm.x86.mmx.pavg.w(x86_mmx, x86_mmx) nounwind readnone
-
-define i64 @test_pcmpeqb(x86_mmx %a0, x86_mmx %a1, x86_mmx* %a2) optsize {
-; GENERIC-LABEL: test_pcmpeqb:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    pcmpeqb %mm1, %mm0 # sched: [3:1.00]
-; GENERIC-NEXT:    pcmpeqb (%rdi), %mm0 # sched: [8:1.00]
-; GENERIC-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_pcmpeqb:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    pcmpeqb %mm1, %mm0 # sched: [1:0.50]
-; ATOM-NEXT:    pcmpeqb (%rdi), %mm0 # sched: [1:1.00]
-; ATOM-NEXT:    movq %mm0, %rax # sched: [3:3.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_pcmpeqb:
-; SLM:       # %bb.0:
-; SLM-NEXT:    pcmpeqb %mm1, %mm0 # sched: [1:0.50]
-; SLM-NEXT:    pcmpeqb (%rdi), %mm0 # sched: [4:1.00]
-; SLM-NEXT:    movq %mm0, %rax # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_pcmpeqb:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    pcmpeqb %mm1, %mm0 # sched: [3:1.00]
-; SANDY-NEXT:    pcmpeqb (%rdi), %mm0 # sched: [8:1.00]
-; SANDY-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_pcmpeqb:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    pcmpeqb %mm1, %mm0 # sched: [1:0.50]
-; HASWELL-NEXT:    pcmpeqb (%rdi), %mm0 # sched: [6:0.50]
-; HASWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pcmpeqb:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    pcmpeqb %mm1, %mm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    pcmpeqb (%rdi), %mm0 # sched: [6:0.50]
-; BROADWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pcmpeqb:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    pcmpeqb %mm1, %mm0 # sched: [1:1.00]
-; SKYLAKE-NEXT:    pcmpeqb (%rdi), %mm0 # sched: [6:1.00]
-; SKYLAKE-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pcmpeqb:
-; SKX:       # %bb.0:
-; SKX-NEXT:    pcmpeqb %mm1, %mm0 # sched: [1:1.00]
-; SKX-NEXT:    pcmpeqb (%rdi), %mm0 # sched: [6:1.00]
-; SKX-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_pcmpeqb:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    pcmpeqb %mm1, %mm0 # sched: [2:0.50]
-; BDVER2-NEXT:    pcmpeqb (%rdi), %mm0 # sched: [7:0.50]
-; BDVER2-NEXT:    movq %mm0, %rax # sched: [10:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_pcmpeqb:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    pcmpeqb %mm1, %mm0 # sched: [1:0.50]
-; BTVER2-NEXT:    pcmpeqb (%rdi), %mm0 # sched: [6:1.00]
-; BTVER2-NEXT:    movq %mm0, %rax # sched: [4:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_pcmpeqb:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    pcmpeqb %mm1, %mm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    pcmpeqb (%rdi), %mm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call x86_mmx @llvm.x86.mmx.pcmpeq.b(x86_mmx %a0, x86_mmx %a1)
-  %2 = load x86_mmx, x86_mmx *%a2, align 8
-  %3 = call x86_mmx @llvm.x86.mmx.pcmpeq.b(x86_mmx %1, x86_mmx %2)
-  %4 = bitcast x86_mmx %3 to i64
-  ret i64 %4
-}
-declare x86_mmx @llvm.x86.mmx.pcmpeq.b(x86_mmx, x86_mmx) nounwind readnone
-
-define i64 @test_pcmpeqd(x86_mmx %a0, x86_mmx %a1, x86_mmx* %a2) optsize {
-; GENERIC-LABEL: test_pcmpeqd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    pcmpeqd %mm1, %mm0 # sched: [3:1.00]
-; GENERIC-NEXT:    pcmpeqd (%rdi), %mm0 # sched: [8:1.00]
-; GENERIC-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_pcmpeqd:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    pcmpeqd %mm1, %mm0 # sched: [1:0.50]
-; ATOM-NEXT:    pcmpeqd (%rdi), %mm0 # sched: [1:1.00]
-; ATOM-NEXT:    movq %mm0, %rax # sched: [3:3.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_pcmpeqd:
-; SLM:       # %bb.0:
-; SLM-NEXT:    pcmpeqd %mm1, %mm0 # sched: [1:0.50]
-; SLM-NEXT:    pcmpeqd (%rdi), %mm0 # sched: [4:1.00]
-; SLM-NEXT:    movq %mm0, %rax # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_pcmpeqd:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    pcmpeqd %mm1, %mm0 # sched: [3:1.00]
-; SANDY-NEXT:    pcmpeqd (%rdi), %mm0 # sched: [8:1.00]
-; SANDY-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_pcmpeqd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    pcmpeqd %mm1, %mm0 # sched: [1:0.50]
-; HASWELL-NEXT:    pcmpeqd (%rdi), %mm0 # sched: [6:0.50]
-; HASWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pcmpeqd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    pcmpeqd %mm1, %mm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    pcmpeqd (%rdi), %mm0 # sched: [6:0.50]
-; BROADWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pcmpeqd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    pcmpeqd %mm1, %mm0 # sched: [1:1.00]
-; SKYLAKE-NEXT:    pcmpeqd (%rdi), %mm0 # sched: [6:1.00]
-; SKYLAKE-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pcmpeqd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    pcmpeqd %mm1, %mm0 # sched: [1:1.00]
-; SKX-NEXT:    pcmpeqd (%rdi), %mm0 # sched: [6:1.00]
-; SKX-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_pcmpeqd:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    pcmpeqd %mm1, %mm0 # sched: [2:0.50]
-; BDVER2-NEXT:    pcmpeqd (%rdi), %mm0 # sched: [7:0.50]
-; BDVER2-NEXT:    movq %mm0, %rax # sched: [10:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_pcmpeqd:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    pcmpeqd %mm1, %mm0 # sched: [1:0.50]
-; BTVER2-NEXT:    pcmpeqd (%rdi), %mm0 # sched: [6:1.00]
-; BTVER2-NEXT:    movq %mm0, %rax # sched: [4:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_pcmpeqd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    pcmpeqd %mm1, %mm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    pcmpeqd (%rdi), %mm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call x86_mmx @llvm.x86.mmx.pcmpeq.d(x86_mmx %a0, x86_mmx %a1)
-  %2 = load x86_mmx, x86_mmx *%a2, align 8
-  %3 = call x86_mmx @llvm.x86.mmx.pcmpeq.d(x86_mmx %1, x86_mmx %2)
-  %4 = bitcast x86_mmx %3 to i64
-  ret i64 %4
-}
-declare x86_mmx @llvm.x86.mmx.pcmpeq.d(x86_mmx, x86_mmx) nounwind readnone
-
-define i64 @test_pcmpeqw(x86_mmx %a0, x86_mmx %a1, x86_mmx* %a2) optsize {
-; GENERIC-LABEL: test_pcmpeqw:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    pcmpeqw %mm1, %mm0 # sched: [3:1.00]
-; GENERIC-NEXT:    pcmpeqw (%rdi), %mm0 # sched: [8:1.00]
-; GENERIC-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_pcmpeqw:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    pcmpeqw %mm1, %mm0 # sched: [1:0.50]
-; ATOM-NEXT:    pcmpeqw (%rdi), %mm0 # sched: [1:1.00]
-; ATOM-NEXT:    movq %mm0, %rax # sched: [3:3.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_pcmpeqw:
-; SLM:       # %bb.0:
-; SLM-NEXT:    pcmpeqw %mm1, %mm0 # sched: [1:0.50]
-; SLM-NEXT:    pcmpeqw (%rdi), %mm0 # sched: [4:1.00]
-; SLM-NEXT:    movq %mm0, %rax # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_pcmpeqw:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    pcmpeqw %mm1, %mm0 # sched: [3:1.00]
-; SANDY-NEXT:    pcmpeqw (%rdi), %mm0 # sched: [8:1.00]
-; SANDY-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_pcmpeqw:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    pcmpeqw %mm1, %mm0 # sched: [1:0.50]
-; HASWELL-NEXT:    pcmpeqw (%rdi), %mm0 # sched: [6:0.50]
-; HASWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pcmpeqw:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    pcmpeqw %mm1, %mm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    pcmpeqw (%rdi), %mm0 # sched: [6:0.50]
-; BROADWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pcmpeqw:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    pcmpeqw %mm1, %mm0 # sched: [1:1.00]
-; SKYLAKE-NEXT:    pcmpeqw (%rdi), %mm0 # sched: [6:1.00]
-; SKYLAKE-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pcmpeqw:
-; SKX:       # %bb.0:
-; SKX-NEXT:    pcmpeqw %mm1, %mm0 # sched: [1:1.00]
-; SKX-NEXT:    pcmpeqw (%rdi), %mm0 # sched: [6:1.00]
-; SKX-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_pcmpeqw:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    pcmpeqw %mm1, %mm0 # sched: [2:0.50]
-; BDVER2-NEXT:    pcmpeqw (%rdi), %mm0 # sched: [7:0.50]
-; BDVER2-NEXT:    movq %mm0, %rax # sched: [10:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_pcmpeqw:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    pcmpeqw %mm1, %mm0 # sched: [1:0.50]
-; BTVER2-NEXT:    pcmpeqw (%rdi), %mm0 # sched: [6:1.00]
-; BTVER2-NEXT:    movq %mm0, %rax # sched: [4:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_pcmpeqw:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    pcmpeqw %mm1, %mm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    pcmpeqw (%rdi), %mm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call x86_mmx @llvm.x86.mmx.pcmpeq.w(x86_mmx %a0, x86_mmx %a1)
-  %2 = load x86_mmx, x86_mmx *%a2, align 8
-  %3 = call x86_mmx @llvm.x86.mmx.pcmpeq.w(x86_mmx %1, x86_mmx %2)
-  %4 = bitcast x86_mmx %3 to i64
-  ret i64 %4
-}
-declare x86_mmx @llvm.x86.mmx.pcmpeq.w(x86_mmx, x86_mmx) nounwind readnone
-
-define i64 @test_pcmpgtb(x86_mmx %a0, x86_mmx %a1, x86_mmx* %a2) optsize {
-; GENERIC-LABEL: test_pcmpgtb:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    pcmpgtb %mm1, %mm0 # sched: [3:1.00]
-; GENERIC-NEXT:    pcmpgtb (%rdi), %mm0 # sched: [8:1.00]
-; GENERIC-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_pcmpgtb:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    pcmpgtb %mm1, %mm0 # sched: [1:0.50]
-; ATOM-NEXT:    pcmpgtb (%rdi), %mm0 # sched: [1:1.00]
-; ATOM-NEXT:    movq %mm0, %rax # sched: [3:3.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_pcmpgtb:
-; SLM:       # %bb.0:
-; SLM-NEXT:    pcmpgtb %mm1, %mm0 # sched: [1:0.50]
-; SLM-NEXT:    pcmpgtb (%rdi), %mm0 # sched: [4:1.00]
-; SLM-NEXT:    movq %mm0, %rax # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_pcmpgtb:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    pcmpgtb %mm1, %mm0 # sched: [3:1.00]
-; SANDY-NEXT:    pcmpgtb (%rdi), %mm0 # sched: [8:1.00]
-; SANDY-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_pcmpgtb:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    pcmpgtb %mm1, %mm0 # sched: [1:0.50]
-; HASWELL-NEXT:    pcmpgtb (%rdi), %mm0 # sched: [6:0.50]
-; HASWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pcmpgtb:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    pcmpgtb %mm1, %mm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    pcmpgtb (%rdi), %mm0 # sched: [6:0.50]
-; BROADWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pcmpgtb:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    pcmpgtb %mm1, %mm0 # sched: [1:1.00]
-; SKYLAKE-NEXT:    pcmpgtb (%rdi), %mm0 # sched: [6:1.00]
-; SKYLAKE-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pcmpgtb:
-; SKX:       # %bb.0:
-; SKX-NEXT:    pcmpgtb %mm1, %mm0 # sched: [1:1.00]
-; SKX-NEXT:    pcmpgtb (%rdi), %mm0 # sched: [6:1.00]
-; SKX-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_pcmpgtb:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    pcmpgtb %mm1, %mm0 # sched: [2:0.50]
-; BDVER2-NEXT:    pcmpgtb (%rdi), %mm0 # sched: [7:0.50]
-; BDVER2-NEXT:    movq %mm0, %rax # sched: [10:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_pcmpgtb:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    pcmpgtb %mm1, %mm0 # sched: [1:0.50]
-; BTVER2-NEXT:    pcmpgtb (%rdi), %mm0 # sched: [6:1.00]
-; BTVER2-NEXT:    movq %mm0, %rax # sched: [4:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_pcmpgtb:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    pcmpgtb %mm1, %mm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    pcmpgtb (%rdi), %mm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call x86_mmx @llvm.x86.mmx.pcmpgt.b(x86_mmx %a0, x86_mmx %a1)
-  %2 = load x86_mmx, x86_mmx *%a2, align 8
-  %3 = call x86_mmx @llvm.x86.mmx.pcmpgt.b(x86_mmx %1, x86_mmx %2)
-  %4 = bitcast x86_mmx %3 to i64
-  ret i64 %4
-}
-declare x86_mmx @llvm.x86.mmx.pcmpgt.b(x86_mmx, x86_mmx) nounwind readnone
-
-define i64 @test_pcmpgtd(x86_mmx %a0, x86_mmx %a1, x86_mmx* %a2) optsize {
-; GENERIC-LABEL: test_pcmpgtd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    pcmpgtd %mm1, %mm0 # sched: [3:1.00]
-; GENERIC-NEXT:    pcmpgtd (%rdi), %mm0 # sched: [8:1.00]
-; GENERIC-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_pcmpgtd:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    pcmpgtd %mm1, %mm0 # sched: [1:0.50]
-; ATOM-NEXT:    pcmpgtd (%rdi), %mm0 # sched: [1:1.00]
-; ATOM-NEXT:    movq %mm0, %rax # sched: [3:3.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_pcmpgtd:
-; SLM:       # %bb.0:
-; SLM-NEXT:    pcmpgtd %mm1, %mm0 # sched: [1:0.50]
-; SLM-NEXT:    pcmpgtd (%rdi), %mm0 # sched: [4:1.00]
-; SLM-NEXT:    movq %mm0, %rax # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_pcmpgtd:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    pcmpgtd %mm1, %mm0 # sched: [3:1.00]
-; SANDY-NEXT:    pcmpgtd (%rdi), %mm0 # sched: [8:1.00]
-; SANDY-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_pcmpgtd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    pcmpgtd %mm1, %mm0 # sched: [1:0.50]
-; HASWELL-NEXT:    pcmpgtd (%rdi), %mm0 # sched: [6:0.50]
-; HASWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pcmpgtd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    pcmpgtd %mm1, %mm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    pcmpgtd (%rdi), %mm0 # sched: [6:0.50]
-; BROADWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pcmpgtd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    pcmpgtd %mm1, %mm0 # sched: [1:1.00]
-; SKYLAKE-NEXT:    pcmpgtd (%rdi), %mm0 # sched: [6:1.00]
-; SKYLAKE-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pcmpgtd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    pcmpgtd %mm1, %mm0 # sched: [1:1.00]
-; SKX-NEXT:    pcmpgtd (%rdi), %mm0 # sched: [6:1.00]
-; SKX-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_pcmpgtd:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    pcmpgtd %mm1, %mm0 # sched: [2:0.50]
-; BDVER2-NEXT:    pcmpgtd (%rdi), %mm0 # sched: [7:0.50]
-; BDVER2-NEXT:    movq %mm0, %rax # sched: [10:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_pcmpgtd:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    pcmpgtd %mm1, %mm0 # sched: [1:0.50]
-; BTVER2-NEXT:    pcmpgtd (%rdi), %mm0 # sched: [6:1.00]
-; BTVER2-NEXT:    movq %mm0, %rax # sched: [4:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_pcmpgtd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    pcmpgtd %mm1, %mm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    pcmpgtd (%rdi), %mm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call x86_mmx @llvm.x86.mmx.pcmpgt.d(x86_mmx %a0, x86_mmx %a1)
-  %2 = load x86_mmx, x86_mmx *%a2, align 8
-  %3 = call x86_mmx @llvm.x86.mmx.pcmpgt.d(x86_mmx %1, x86_mmx %2)
-  %4 = bitcast x86_mmx %3 to i64
-  ret i64 %4
-}
-declare x86_mmx @llvm.x86.mmx.pcmpgt.d(x86_mmx, x86_mmx) nounwind readnone
-
-define i64 @test_pcmpgtw(x86_mmx %a0, x86_mmx %a1, x86_mmx* %a2) optsize {
-; GENERIC-LABEL: test_pcmpgtw:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    pcmpgtw %mm1, %mm0 # sched: [3:1.00]
-; GENERIC-NEXT:    pcmpgtw (%rdi), %mm0 # sched: [8:1.00]
-; GENERIC-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_pcmpgtw:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    pcmpgtw %mm1, %mm0 # sched: [1:0.50]
-; ATOM-NEXT:    pcmpgtw (%rdi), %mm0 # sched: [1:1.00]
-; ATOM-NEXT:    movq %mm0, %rax # sched: [3:3.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_pcmpgtw:
-; SLM:       # %bb.0:
-; SLM-NEXT:    pcmpgtw %mm1, %mm0 # sched: [1:0.50]
-; SLM-NEXT:    pcmpgtw (%rdi), %mm0 # sched: [4:1.00]
-; SLM-NEXT:    movq %mm0, %rax # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_pcmpgtw:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    pcmpgtw %mm1, %mm0 # sched: [3:1.00]
-; SANDY-NEXT:    pcmpgtw (%rdi), %mm0 # sched: [8:1.00]
-; SANDY-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_pcmpgtw:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    pcmpgtw %mm1, %mm0 # sched: [1:0.50]
-; HASWELL-NEXT:    pcmpgtw (%rdi), %mm0 # sched: [6:0.50]
-; HASWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pcmpgtw:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    pcmpgtw %mm1, %mm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    pcmpgtw (%rdi), %mm0 # sched: [6:0.50]
-; BROADWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pcmpgtw:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    pcmpgtw %mm1, %mm0 # sched: [1:1.00]
-; SKYLAKE-NEXT:    pcmpgtw (%rdi), %mm0 # sched: [6:1.00]
-; SKYLAKE-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pcmpgtw:
-; SKX:       # %bb.0:
-; SKX-NEXT:    pcmpgtw %mm1, %mm0 # sched: [1:1.00]
-; SKX-NEXT:    pcmpgtw (%rdi), %mm0 # sched: [6:1.00]
-; SKX-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_pcmpgtw:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    pcmpgtw %mm1, %mm0 # sched: [2:0.50]
-; BDVER2-NEXT:    pcmpgtw (%rdi), %mm0 # sched: [7:0.50]
-; BDVER2-NEXT:    movq %mm0, %rax # sched: [10:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_pcmpgtw:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    pcmpgtw %mm1, %mm0 # sched: [1:0.50]
-; BTVER2-NEXT:    pcmpgtw (%rdi), %mm0 # sched: [6:1.00]
-; BTVER2-NEXT:    movq %mm0, %rax # sched: [4:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_pcmpgtw:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    pcmpgtw %mm1, %mm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    pcmpgtw (%rdi), %mm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call x86_mmx @llvm.x86.mmx.pcmpgt.w(x86_mmx %a0, x86_mmx %a1)
-  %2 = load x86_mmx, x86_mmx *%a2, align 8
-  %3 = call x86_mmx @llvm.x86.mmx.pcmpgt.w(x86_mmx %1, x86_mmx %2)
-  %4 = bitcast x86_mmx %3 to i64
-  ret i64 %4
-}
-declare x86_mmx @llvm.x86.mmx.pcmpgt.w(x86_mmx, x86_mmx) nounwind readnone
-
-define i32 @test_pextrw(x86_mmx %a0) optsize {
-; GENERIC-LABEL: test_pextrw:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    pextrw $0, %mm0, %eax # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_pextrw:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    pextrw $0, %mm0, %eax # sched: [4:2.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_pextrw:
-; SLM:       # %bb.0:
-; SLM-NEXT:    pextrw $0, %mm0, %eax # sched: [1:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_pextrw:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    pextrw $0, %mm0, %eax # sched: [3:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_pextrw:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    pextrw $0, %mm0, %eax # sched: [2:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pextrw:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    pextrw $0, %mm0, %eax # sched: [2:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pextrw:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    pextrw $0, %mm0, %eax # sched: [3:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pextrw:
-; SKX:       # %bb.0:
-; SKX-NEXT:    pextrw $0, %mm0, %eax # sched: [3:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_pextrw:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    pextrw $0, %mm0, %eax # sched: [13:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_pextrw:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    pextrw $0, %mm0, %eax # sched: [3:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_pextrw:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    pextrw $0, %mm0, %eax # sched: [2:2.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call i32 @llvm.x86.mmx.pextr.w(x86_mmx %a0, i32 0)
-  ret i32 %1
-}
-declare i32 @llvm.x86.mmx.pextr.w(x86_mmx, i32) nounwind readnone
-
-define i64 @test_phaddd(x86_mmx %a0, x86_mmx %a1, x86_mmx* %a2) optsize {
-; GENERIC-LABEL: test_phaddd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    phaddd %mm1, %mm0 # sched: [3:1.50]
-; GENERIC-NEXT:    phaddd (%rdi), %mm0 # sched: [8:1.50]
-; GENERIC-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_phaddd:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    phaddd %mm1, %mm0 # sched: [3:1.50]
-; ATOM-NEXT:    phaddd (%rdi), %mm0 # sched: [4:2.00]
-; ATOM-NEXT:    movq %mm0, %rax # sched: [3:3.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_phaddd:
-; SLM:       # %bb.0:
-; SLM-NEXT:    phaddd %mm1, %mm0 # sched: [1:0.50]
-; SLM-NEXT:    phaddd (%rdi), %mm0 # sched: [4:1.00]
-; SLM-NEXT:    movq %mm0, %rax # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_phaddd:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    phaddd %mm1, %mm0 # sched: [3:1.50]
-; SANDY-NEXT:    phaddd (%rdi), %mm0 # sched: [8:1.50]
-; SANDY-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_phaddd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    phaddd %mm1, %mm0 # sched: [3:2.00]
-; HASWELL-NEXT:    phaddd (%rdi), %mm0 # sched: [8:2.00]
-; HASWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_phaddd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    phaddd %mm1, %mm0 # sched: [3:2.00]
-; BROADWELL-NEXT:    phaddd (%rdi), %mm0 # sched: [8:2.00]
-; BROADWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_phaddd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    phaddd %mm1, %mm0 # sched: [3:2.00]
-; SKYLAKE-NEXT:    phaddd (%rdi), %mm0 # sched: [8:2.00]
-; SKYLAKE-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_phaddd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    phaddd %mm1, %mm0 # sched: [3:2.00]
-; SKX-NEXT:    phaddd (%rdi), %mm0 # sched: [8:2.00]
-; SKX-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_phaddd:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    phaddd %mm1, %mm0 # sched: [5:0.50]
-; BDVER2-NEXT:    phaddd (%rdi), %mm0 # sched: [10:0.50]
-; BDVER2-NEXT:    movq %mm0, %rax # sched: [10:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_phaddd:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    phaddd %mm1, %mm0 # sched: [1:0.50]
-; BTVER2-NEXT:    phaddd (%rdi), %mm0 # sched: [6:1.00]
-; BTVER2-NEXT:    movq %mm0, %rax # sched: [4:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_phaddd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    phaddd %mm1, %mm0 # sched: [100:0.25]
-; ZNVER1-NEXT:    phaddd (%rdi), %mm0 # sched: [100:0.25]
-; ZNVER1-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call x86_mmx @llvm.x86.ssse3.phadd.d(x86_mmx %a0, x86_mmx %a1)
-  %2 = load x86_mmx, x86_mmx *%a2, align 8
-  %3 = call x86_mmx @llvm.x86.ssse3.phadd.d(x86_mmx %1, x86_mmx %2)
-  %4 = bitcast x86_mmx %3 to i64
-  ret i64 %4
-}
-declare x86_mmx @llvm.x86.ssse3.phadd.d(x86_mmx, x86_mmx) nounwind readnone
-
-define i64 @test_phaddsw(x86_mmx %a0, x86_mmx %a1, x86_mmx* %a2) optsize {
-; GENERIC-LABEL: test_phaddsw:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    phaddsw %mm1, %mm0 # sched: [3:1.50]
-; GENERIC-NEXT:    phaddsw (%rdi), %mm0 # sched: [8:1.50]
-; GENERIC-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_phaddsw:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    phaddsw %mm1, %mm0 # sched: [5:2.50]
-; ATOM-NEXT:    phaddsw (%rdi), %mm0 # sched: [6:3.00]
-; ATOM-NEXT:    movq %mm0, %rax # sched: [3:3.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_phaddsw:
-; SLM:       # %bb.0:
-; SLM-NEXT:    phaddsw %mm1, %mm0 # sched: [1:0.50]
-; SLM-NEXT:    phaddsw (%rdi), %mm0 # sched: [4:1.00]
-; SLM-NEXT:    movq %mm0, %rax # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_phaddsw:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    phaddsw %mm1, %mm0 # sched: [3:1.50]
-; SANDY-NEXT:    phaddsw (%rdi), %mm0 # sched: [8:1.50]
-; SANDY-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_phaddsw:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    phaddsw %mm1, %mm0 # sched: [3:2.00]
-; HASWELL-NEXT:    phaddsw (%rdi), %mm0 # sched: [8:2.00]
-; HASWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_phaddsw:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    phaddsw %mm1, %mm0 # sched: [3:2.00]
-; BROADWELL-NEXT:    phaddsw (%rdi), %mm0 # sched: [8:2.00]
-; BROADWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_phaddsw:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    phaddsw %mm1, %mm0 # sched: [3:2.00]
-; SKYLAKE-NEXT:    phaddsw (%rdi), %mm0 # sched: [8:2.00]
-; SKYLAKE-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_phaddsw:
-; SKX:       # %bb.0:
-; SKX-NEXT:    phaddsw %mm1, %mm0 # sched: [3:2.00]
-; SKX-NEXT:    phaddsw (%rdi), %mm0 # sched: [8:2.00]
-; SKX-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_phaddsw:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    phaddsw %mm1, %mm0 # sched: [5:0.50]
-; BDVER2-NEXT:    phaddsw (%rdi), %mm0 # sched: [10:0.50]
-; BDVER2-NEXT:    movq %mm0, %rax # sched: [10:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_phaddsw:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    phaddsw %mm1, %mm0 # sched: [1:0.50]
-; BTVER2-NEXT:    phaddsw (%rdi), %mm0 # sched: [6:1.00]
-; BTVER2-NEXT:    movq %mm0, %rax # sched: [4:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_phaddsw:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    phaddsw %mm1, %mm0 # sched: [100:0.25]
-; ZNVER1-NEXT:    phaddsw (%rdi), %mm0 # sched: [100:0.25]
-; ZNVER1-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call x86_mmx @llvm.x86.ssse3.phadd.sw(x86_mmx %a0, x86_mmx %a1)
-  %2 = load x86_mmx, x86_mmx *%a2, align 8
-  %3 = call x86_mmx @llvm.x86.ssse3.phadd.sw(x86_mmx %1, x86_mmx %2)
-  %4 = bitcast x86_mmx %3 to i64
-  ret i64 %4
-}
-declare x86_mmx @llvm.x86.ssse3.phadd.sw(x86_mmx, x86_mmx) nounwind readnone
-
-define i64 @test_phaddw(x86_mmx %a0, x86_mmx %a1, x86_mmx* %a2) optsize {
-; GENERIC-LABEL: test_phaddw:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    phaddw %mm1, %mm0 # sched: [3:1.50]
-; GENERIC-NEXT:    phaddw (%rdi), %mm0 # sched: [8:1.50]
-; GENERIC-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_phaddw:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    phaddw %mm1, %mm0 # sched: [5:2.50]
-; ATOM-NEXT:    phaddw (%rdi), %mm0 # sched: [6:3.00]
-; ATOM-NEXT:    movq %mm0, %rax # sched: [3:3.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_phaddw:
-; SLM:       # %bb.0:
-; SLM-NEXT:    phaddw %mm1, %mm0 # sched: [1:0.50]
-; SLM-NEXT:    phaddw (%rdi), %mm0 # sched: [4:1.00]
-; SLM-NEXT:    movq %mm0, %rax # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_phaddw:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    phaddw %mm1, %mm0 # sched: [3:1.50]
-; SANDY-NEXT:    phaddw (%rdi), %mm0 # sched: [8:1.50]
-; SANDY-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_phaddw:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    phaddw %mm1, %mm0 # sched: [3:2.00]
-; HASWELL-NEXT:    phaddw (%rdi), %mm0 # sched: [8:2.00]
-; HASWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_phaddw:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    phaddw %mm1, %mm0 # sched: [3:2.00]
-; BROADWELL-NEXT:    phaddw (%rdi), %mm0 # sched: [8:2.00]
-; BROADWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_phaddw:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    phaddw %mm1, %mm0 # sched: [3:2.00]
-; SKYLAKE-NEXT:    phaddw (%rdi), %mm0 # sched: [8:2.00]
-; SKYLAKE-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_phaddw:
-; SKX:       # %bb.0:
-; SKX-NEXT:    phaddw %mm1, %mm0 # sched: [3:2.00]
-; SKX-NEXT:    phaddw (%rdi), %mm0 # sched: [8:2.00]
-; SKX-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_phaddw:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    phaddw %mm1, %mm0 # sched: [5:0.50]
-; BDVER2-NEXT:    phaddw (%rdi), %mm0 # sched: [10:0.50]
-; BDVER2-NEXT:    movq %mm0, %rax # sched: [10:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_phaddw:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    phaddw %mm1, %mm0 # sched: [1:0.50]
-; BTVER2-NEXT:    phaddw (%rdi), %mm0 # sched: [6:1.00]
-; BTVER2-NEXT:    movq %mm0, %rax # sched: [4:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_phaddw:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    phaddw %mm1, %mm0 # sched: [100:0.25]
-; ZNVER1-NEXT:    phaddw (%rdi), %mm0 # sched: [100:0.25]
-; ZNVER1-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call x86_mmx @llvm.x86.ssse3.phadd.w(x86_mmx %a0, x86_mmx %a1)
-  %2 = load x86_mmx, x86_mmx *%a2, align 8
-  %3 = call x86_mmx @llvm.x86.ssse3.phadd.w(x86_mmx %1, x86_mmx %2)
-  %4 = bitcast x86_mmx %3 to i64
-  ret i64 %4
-}
-declare x86_mmx @llvm.x86.ssse3.phadd.w(x86_mmx, x86_mmx) nounwind readnone
-
-define i64 @test_phsubd(x86_mmx %a0, x86_mmx %a1, x86_mmx* %a2) optsize {
-; GENERIC-LABEL: test_phsubd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    phsubd %mm1, %mm0 # sched: [3:1.50]
-; GENERIC-NEXT:    phsubd (%rdi), %mm0 # sched: [8:1.50]
-; GENERIC-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_phsubd:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    phsubd %mm1, %mm0 # sched: [3:1.50]
-; ATOM-NEXT:    phsubd (%rdi), %mm0 # sched: [4:2.00]
-; ATOM-NEXT:    movq %mm0, %rax # sched: [3:3.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_phsubd:
-; SLM:       # %bb.0:
-; SLM-NEXT:    phsubd %mm1, %mm0 # sched: [1:0.50]
-; SLM-NEXT:    phsubd (%rdi), %mm0 # sched: [4:1.00]
-; SLM-NEXT:    movq %mm0, %rax # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_phsubd:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    phsubd %mm1, %mm0 # sched: [3:1.50]
-; SANDY-NEXT:    phsubd (%rdi), %mm0 # sched: [8:1.50]
-; SANDY-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_phsubd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    phsubd %mm1, %mm0 # sched: [3:2.00]
-; HASWELL-NEXT:    phsubd (%rdi), %mm0 # sched: [8:2.00]
-; HASWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_phsubd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    phsubd %mm1, %mm0 # sched: [3:2.00]
-; BROADWELL-NEXT:    phsubd (%rdi), %mm0 # sched: [8:2.00]
-; BROADWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_phsubd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    phsubd %mm1, %mm0 # sched: [3:2.00]
-; SKYLAKE-NEXT:    phsubd (%rdi), %mm0 # sched: [8:2.00]
-; SKYLAKE-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_phsubd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    phsubd %mm1, %mm0 # sched: [3:2.00]
-; SKX-NEXT:    phsubd (%rdi), %mm0 # sched: [8:2.00]
-; SKX-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_phsubd:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    phsubd %mm1, %mm0 # sched: [5:0.50]
-; BDVER2-NEXT:    phsubd (%rdi), %mm0 # sched: [10:0.50]
-; BDVER2-NEXT:    movq %mm0, %rax # sched: [10:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_phsubd:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    phsubd %mm1, %mm0 # sched: [1:0.50]
-; BTVER2-NEXT:    phsubd (%rdi), %mm0 # sched: [6:1.00]
-; BTVER2-NEXT:    movq %mm0, %rax # sched: [4:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_phsubd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    phsubd %mm1, %mm0 # sched: [100:0.25]
-; ZNVER1-NEXT:    phsubd (%rdi), %mm0 # sched: [100:0.25]
-; ZNVER1-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call x86_mmx @llvm.x86.ssse3.phsub.d(x86_mmx %a0, x86_mmx %a1)
-  %2 = load x86_mmx, x86_mmx *%a2, align 8
-  %3 = call x86_mmx @llvm.x86.ssse3.phsub.d(x86_mmx %1, x86_mmx %2)
-  %4 = bitcast x86_mmx %3 to i64
-  ret i64 %4
-}
-declare x86_mmx @llvm.x86.ssse3.phsub.d(x86_mmx, x86_mmx) nounwind readnone
-
-define i64 @test_phsubsw(x86_mmx %a0, x86_mmx %a1, x86_mmx* %a2) optsize {
-; GENERIC-LABEL: test_phsubsw:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    phsubsw %mm1, %mm0 # sched: [3:1.50]
-; GENERIC-NEXT:    phsubsw (%rdi), %mm0 # sched: [8:1.50]
-; GENERIC-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_phsubsw:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    phsubsw %mm1, %mm0 # sched: [5:2.50]
-; ATOM-NEXT:    phsubsw (%rdi), %mm0 # sched: [6:3.00]
-; ATOM-NEXT:    movq %mm0, %rax # sched: [3:3.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_phsubsw:
-; SLM:       # %bb.0:
-; SLM-NEXT:    phsubsw %mm1, %mm0 # sched: [1:0.50]
-; SLM-NEXT:    phsubsw (%rdi), %mm0 # sched: [4:1.00]
-; SLM-NEXT:    movq %mm0, %rax # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_phsubsw:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    phsubsw %mm1, %mm0 # sched: [3:1.50]
-; SANDY-NEXT:    phsubsw (%rdi), %mm0 # sched: [8:1.50]
-; SANDY-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_phsubsw:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    phsubsw %mm1, %mm0 # sched: [3:2.00]
-; HASWELL-NEXT:    phsubsw (%rdi), %mm0 # sched: [8:2.00]
-; HASWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_phsubsw:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    phsubsw %mm1, %mm0 # sched: [3:2.00]
-; BROADWELL-NEXT:    phsubsw (%rdi), %mm0 # sched: [8:2.00]
-; BROADWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_phsubsw:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    phsubsw %mm1, %mm0 # sched: [3:2.00]
-; SKYLAKE-NEXT:    phsubsw (%rdi), %mm0 # sched: [8:2.00]
-; SKYLAKE-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_phsubsw:
-; SKX:       # %bb.0:
-; SKX-NEXT:    phsubsw %mm1, %mm0 # sched: [3:2.00]
-; SKX-NEXT:    phsubsw (%rdi), %mm0 # sched: [8:2.00]
-; SKX-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_phsubsw:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    phsubsw %mm1, %mm0 # sched: [5:0.50]
-; BDVER2-NEXT:    phsubsw (%rdi), %mm0 # sched: [10:0.50]
-; BDVER2-NEXT:    movq %mm0, %rax # sched: [10:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_phsubsw:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    phsubsw %mm1, %mm0 # sched: [1:0.50]
-; BTVER2-NEXT:    phsubsw (%rdi), %mm0 # sched: [6:1.00]
-; BTVER2-NEXT:    movq %mm0, %rax # sched: [4:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_phsubsw:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    phsubsw %mm1, %mm0 # sched: [100:0.25]
-; ZNVER1-NEXT:    phsubsw (%rdi), %mm0 # sched: [100:0.25]
-; ZNVER1-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call x86_mmx @llvm.x86.ssse3.phsub.sw(x86_mmx %a0, x86_mmx %a1)
-  %2 = load x86_mmx, x86_mmx *%a2, align 8
-  %3 = call x86_mmx @llvm.x86.ssse3.phsub.sw(x86_mmx %1, x86_mmx %2)
-  %4 = bitcast x86_mmx %3 to i64
-  ret i64 %4
-}
-declare x86_mmx @llvm.x86.ssse3.phsub.sw(x86_mmx, x86_mmx) nounwind readnone
-
-define i64 @test_phsubw(x86_mmx %a0, x86_mmx %a1, x86_mmx* %a2) optsize {
-; GENERIC-LABEL: test_phsubw:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    phsubw %mm1, %mm0 # sched: [3:1.50]
-; GENERIC-NEXT:    phsubw (%rdi), %mm0 # sched: [8:1.50]
-; GENERIC-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_phsubw:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    phsubw %mm1, %mm0 # sched: [5:2.50]
-; ATOM-NEXT:    phsubw (%rdi), %mm0 # sched: [6:3.00]
-; ATOM-NEXT:    movq %mm0, %rax # sched: [3:3.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_phsubw:
-; SLM:       # %bb.0:
-; SLM-NEXT:    phsubw %mm1, %mm0 # sched: [1:0.50]
-; SLM-NEXT:    phsubw (%rdi), %mm0 # sched: [4:1.00]
-; SLM-NEXT:    movq %mm0, %rax # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_phsubw:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    phsubw %mm1, %mm0 # sched: [3:1.50]
-; SANDY-NEXT:    phsubw (%rdi), %mm0 # sched: [8:1.50]
-; SANDY-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_phsubw:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    phsubw %mm1, %mm0 # sched: [3:2.00]
-; HASWELL-NEXT:    phsubw (%rdi), %mm0 # sched: [8:2.00]
-; HASWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_phsubw:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    phsubw %mm1, %mm0 # sched: [3:2.00]
-; BROADWELL-NEXT:    phsubw (%rdi), %mm0 # sched: [8:2.00]
-; BROADWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_phsubw:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    phsubw %mm1, %mm0 # sched: [3:2.00]
-; SKYLAKE-NEXT:    phsubw (%rdi), %mm0 # sched: [8:2.00]
-; SKYLAKE-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_phsubw:
-; SKX:       # %bb.0:
-; SKX-NEXT:    phsubw %mm1, %mm0 # sched: [3:2.00]
-; SKX-NEXT:    phsubw (%rdi), %mm0 # sched: [8:2.00]
-; SKX-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_phsubw:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    phsubw %mm1, %mm0 # sched: [5:0.50]
-; BDVER2-NEXT:    phsubw (%rdi), %mm0 # sched: [10:0.50]
-; BDVER2-NEXT:    movq %mm0, %rax # sched: [10:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_phsubw:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    phsubw %mm1, %mm0 # sched: [1:0.50]
-; BTVER2-NEXT:    phsubw (%rdi), %mm0 # sched: [6:1.00]
-; BTVER2-NEXT:    movq %mm0, %rax # sched: [4:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_phsubw:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    phsubw %mm1, %mm0 # sched: [100:0.25]
-; ZNVER1-NEXT:    phsubw (%rdi), %mm0 # sched: [100:0.25]
-; ZNVER1-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call x86_mmx @llvm.x86.ssse3.phsub.w(x86_mmx %a0, x86_mmx %a1)
-  %2 = load x86_mmx, x86_mmx *%a2, align 8
-  %3 = call x86_mmx @llvm.x86.ssse3.phsub.w(x86_mmx %1, x86_mmx %2)
-  %4 = bitcast x86_mmx %3 to i64
-  ret i64 %4
-}
-declare x86_mmx @llvm.x86.ssse3.phsub.w(x86_mmx, x86_mmx) nounwind readnone
-
-define i64 @test_pinsrw(x86_mmx %a0, i32 %a1, i16* %a2) optsize {
-; GENERIC-LABEL: test_pinsrw:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    pinsrw $0, %edi, %mm0 # sched: [2:1.00]
-; GENERIC-NEXT:    movswl (%rsi), %eax # sched: [5:0.50]
-; GENERIC-NEXT:    pinsrw $1, %eax, %mm0 # sched: [2:1.00]
-; GENERIC-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_pinsrw:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    pinsrw $0, %edi, %mm0 # sched: [1:1.00]
-; ATOM-NEXT:    movswl (%rsi), %eax # sched: [1:1.00]
-; ATOM-NEXT:    pinsrw $1, %eax, %mm0 # sched: [1:1.00]
-; ATOM-NEXT:    movq %mm0, %rax # sched: [3:3.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_pinsrw:
-; SLM:       # %bb.0:
-; SLM-NEXT:    movswl (%rsi), %eax # sched: [4:1.00]
-; SLM-NEXT:    pinsrw $0, %edi, %mm0 # sched: [1:1.00]
-; SLM-NEXT:    pinsrw $1, %eax, %mm0 # sched: [1:1.00]
-; SLM-NEXT:    movq %mm0, %rax # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_pinsrw:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    pinsrw $0, %edi, %mm0 # sched: [2:1.00]
-; SANDY-NEXT:    movswl (%rsi), %eax # sched: [5:0.50]
-; SANDY-NEXT:    pinsrw $1, %eax, %mm0 # sched: [2:1.00]
-; SANDY-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_pinsrw:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    pinsrw $0, %edi, %mm0 # sched: [2:2.00]
-; HASWELL-NEXT:    movswl (%rsi), %eax # sched: [5:0.50]
-; HASWELL-NEXT:    pinsrw $1, %eax, %mm0 # sched: [2:2.00]
-; HASWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pinsrw:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    pinsrw $0, %edi, %mm0 # sched: [2:2.00]
-; BROADWELL-NEXT:    movswl (%rsi), %eax # sched: [5:0.50]
-; BROADWELL-NEXT:    pinsrw $1, %eax, %mm0 # sched: [2:2.00]
-; BROADWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pinsrw:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    pinsrw $0, %edi, %mm0 # sched: [2:2.00]
-; SKYLAKE-NEXT:    movswl (%rsi), %eax # sched: [5:0.50]
-; SKYLAKE-NEXT:    pinsrw $1, %eax, %mm0 # sched: [2:2.00]
-; SKYLAKE-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pinsrw:
-; SKX:       # %bb.0:
-; SKX-NEXT:    pinsrw $0, %edi, %mm0 # sched: [2:2.00]
-; SKX-NEXT:    movswl (%rsi), %eax # sched: [5:0.50]
-; SKX-NEXT:    pinsrw $1, %eax, %mm0 # sched: [2:2.00]
-; SKX-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_pinsrw:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    movswl (%rsi), %eax # sched: [5:0.50]
-; BDVER2-NEXT:    pinsrw $0, %edi, %mm0 # sched: [2:0.50]
-; BDVER2-NEXT:    pinsrw $1, %eax, %mm0 # sched: [2:0.50]
-; BDVER2-NEXT:    movq %mm0, %rax # sched: [10:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_pinsrw:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    pinsrw $0, %edi, %mm0 # sched: [7:0.50]
-; BTVER2-NEXT:    movswl (%rsi), %eax # sched: [4:1.00]
-; BTVER2-NEXT:    pinsrw $1, %eax, %mm0 # sched: [7:0.50]
-; BTVER2-NEXT:    movq %mm0, %rax # sched: [4:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_pinsrw:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    movswl (%rsi), %eax # sched: [8:0.50]
-; ZNVER1-NEXT:    pinsrw $0, %edi, %mm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    pinsrw $1, %eax, %mm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call x86_mmx @llvm.x86.mmx.pinsr.w(x86_mmx %a0, i32 %a1, i32 0)
-  %2 = load i16, i16 *%a2, align 2
-  %3 = sext i16 %2 to i32
-  %4 = call x86_mmx @llvm.x86.mmx.pinsr.w(x86_mmx %1, i32 %3, i32 1)
-  %5 = bitcast x86_mmx %4 to i64
-  ret i64 %5
-}
-declare x86_mmx @llvm.x86.mmx.pinsr.w(x86_mmx, i32, i32) nounwind readnone
-
-define i64 @test_pmaddwd(x86_mmx %a0, x86_mmx %a1, x86_mmx* %a2) optsize {
-; GENERIC-LABEL: test_pmaddwd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    pmaddwd %mm1, %mm0 # sched: [5:1.00]
-; GENERIC-NEXT:    pmaddwd (%rdi), %mm0 # sched: [10:1.00]
-; GENERIC-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_pmaddwd:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    pmaddwd %mm1, %mm0 # sched: [4:4.00]
-; ATOM-NEXT:    pmaddwd (%rdi), %mm0 # sched: [4:4.00]
-; ATOM-NEXT:    movq %mm0, %rax # sched: [3:3.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_pmaddwd:
-; SLM:       # %bb.0:
-; SLM-NEXT:    pmaddwd %mm1, %mm0 # sched: [4:1.00]
-; SLM-NEXT:    pmaddwd (%rdi), %mm0 # sched: [7:1.00]
-; SLM-NEXT:    movq %mm0, %rax # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_pmaddwd:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    pmaddwd %mm1, %mm0 # sched: [5:1.00]
-; SANDY-NEXT:    pmaddwd (%rdi), %mm0 # sched: [10:1.00]
-; SANDY-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_pmaddwd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    pmaddwd %mm1, %mm0 # sched: [5:1.00]
-; HASWELL-NEXT:    pmaddwd (%rdi), %mm0 # sched: [10:1.00]
-; HASWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pmaddwd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    pmaddwd %mm1, %mm0 # sched: [5:1.00]
-; BROADWELL-NEXT:    pmaddwd (%rdi), %mm0 # sched: [10:1.00]
-; BROADWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pmaddwd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    pmaddwd %mm1, %mm0 # sched: [4:1.00]
-; SKYLAKE-NEXT:    pmaddwd (%rdi), %mm0 # sched: [9:1.00]
-; SKYLAKE-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pmaddwd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    pmaddwd %mm1, %mm0 # sched: [4:1.00]
-; SKX-NEXT:    pmaddwd (%rdi), %mm0 # sched: [9:1.00]
-; SKX-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_pmaddwd:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    pmaddwd %mm1, %mm0 # sched: [4:1.00]
-; BDVER2-NEXT:    pmaddwd (%rdi), %mm0 # sched: [9:1.00]
-; BDVER2-NEXT:    movq %mm0, %rax # sched: [10:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_pmaddwd:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    pmaddwd %mm1, %mm0 # sched: [2:1.00]
-; BTVER2-NEXT:    pmaddwd (%rdi), %mm0 # sched: [7:1.00]
-; BTVER2-NEXT:    movq %mm0, %rax # sched: [4:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_pmaddwd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    pmaddwd %mm1, %mm0 # sched: [4:1.00]
-; ZNVER1-NEXT:    pmaddwd (%rdi), %mm0 # sched: [11:1.00]
-; ZNVER1-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call x86_mmx @llvm.x86.mmx.pmadd.wd(x86_mmx %a0, x86_mmx %a1)
-  %2 = load x86_mmx, x86_mmx *%a2, align 8
-  %3 = call x86_mmx @llvm.x86.mmx.pmadd.wd(x86_mmx %1, x86_mmx %2)
-  %4 = bitcast x86_mmx %3 to i64
-  ret i64 %4
-}
-declare x86_mmx @llvm.x86.mmx.pmadd.wd(x86_mmx, x86_mmx) nounwind readnone
-
-define i64 @test_pmaddubsw(x86_mmx %a0, x86_mmx %a1, x86_mmx* %a2) optsize {
-; GENERIC-LABEL: test_pmaddubsw:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    pmaddubsw %mm1, %mm0 # sched: [5:1.00]
-; GENERIC-NEXT:    pmaddubsw (%rdi), %mm0 # sched: [10:1.00]
-; GENERIC-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_pmaddubsw:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    pmaddubsw %mm1, %mm0 # sched: [4:4.00]
-; ATOM-NEXT:    pmaddubsw (%rdi), %mm0 # sched: [4:4.00]
-; ATOM-NEXT:    movq %mm0, %rax # sched: [3:3.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_pmaddubsw:
-; SLM:       # %bb.0:
-; SLM-NEXT:    pmaddubsw %mm1, %mm0 # sched: [4:1.00]
-; SLM-NEXT:    pmaddubsw (%rdi), %mm0 # sched: [7:1.00]
-; SLM-NEXT:    movq %mm0, %rax # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_pmaddubsw:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    pmaddubsw %mm1, %mm0 # sched: [5:1.00]
-; SANDY-NEXT:    pmaddubsw (%rdi), %mm0 # sched: [10:1.00]
-; SANDY-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_pmaddubsw:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    pmaddubsw %mm1, %mm0 # sched: [5:1.00]
-; HASWELL-NEXT:    pmaddubsw (%rdi), %mm0 # sched: [10:1.00]
-; HASWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pmaddubsw:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    pmaddubsw %mm1, %mm0 # sched: [5:1.00]
-; BROADWELL-NEXT:    pmaddubsw (%rdi), %mm0 # sched: [10:1.00]
-; BROADWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pmaddubsw:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    pmaddubsw %mm1, %mm0 # sched: [4:1.00]
-; SKYLAKE-NEXT:    pmaddubsw (%rdi), %mm0 # sched: [9:1.00]
-; SKYLAKE-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pmaddubsw:
-; SKX:       # %bb.0:
-; SKX-NEXT:    pmaddubsw %mm1, %mm0 # sched: [4:1.00]
-; SKX-NEXT:    pmaddubsw (%rdi), %mm0 # sched: [9:1.00]
-; SKX-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_pmaddubsw:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    pmaddubsw %mm1, %mm0 # sched: [4:1.00]
-; BDVER2-NEXT:    pmaddubsw (%rdi), %mm0 # sched: [9:1.00]
-; BDVER2-NEXT:    movq %mm0, %rax # sched: [10:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_pmaddubsw:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    pmaddubsw %mm1, %mm0 # sched: [2:1.00]
-; BTVER2-NEXT:    pmaddubsw (%rdi), %mm0 # sched: [7:1.00]
-; BTVER2-NEXT:    movq %mm0, %rax # sched: [4:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_pmaddubsw:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    pmaddubsw %mm1, %mm0 # sched: [4:1.00]
-; ZNVER1-NEXT:    pmaddubsw (%rdi), %mm0 # sched: [11:1.00]
-; ZNVER1-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call x86_mmx @llvm.x86.ssse3.pmadd.ub.sw(x86_mmx %a0, x86_mmx %a1)
-  %2 = load x86_mmx, x86_mmx *%a2, align 8
-  %3 = call x86_mmx @llvm.x86.ssse3.pmadd.ub.sw(x86_mmx %1, x86_mmx %2)
-  %4 = bitcast x86_mmx %3 to i64
-  ret i64 %4
-}
-declare x86_mmx @llvm.x86.ssse3.pmadd.ub.sw(x86_mmx, x86_mmx) nounwind readnone
-
-define i64 @test_pmaxsw(x86_mmx %a0, x86_mmx %a1, x86_mmx* %a2) optsize {
-; GENERIC-LABEL: test_pmaxsw:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    pmaxsw %mm1, %mm0 # sched: [3:1.00]
-; GENERIC-NEXT:    pmaxsw (%rdi), %mm0 # sched: [8:1.00]
-; GENERIC-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_pmaxsw:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    pmaxsw %mm1, %mm0 # sched: [1:0.50]
-; ATOM-NEXT:    pmaxsw (%rdi), %mm0 # sched: [1:1.00]
-; ATOM-NEXT:    movq %mm0, %rax # sched: [3:3.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_pmaxsw:
-; SLM:       # %bb.0:
-; SLM-NEXT:    pmaxsw %mm1, %mm0 # sched: [1:0.50]
-; SLM-NEXT:    pmaxsw (%rdi), %mm0 # sched: [4:1.00]
-; SLM-NEXT:    movq %mm0, %rax # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_pmaxsw:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    pmaxsw %mm1, %mm0 # sched: [3:1.00]
-; SANDY-NEXT:    pmaxsw (%rdi), %mm0 # sched: [8:1.00]
-; SANDY-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_pmaxsw:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    pmaxsw %mm1, %mm0 # sched: [1:0.50]
-; HASWELL-NEXT:    pmaxsw (%rdi), %mm0 # sched: [6:0.50]
-; HASWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pmaxsw:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    pmaxsw %mm1, %mm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    pmaxsw (%rdi), %mm0 # sched: [6:0.50]
-; BROADWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pmaxsw:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    pmaxsw %mm1, %mm0 # sched: [1:1.00]
-; SKYLAKE-NEXT:    pmaxsw (%rdi), %mm0 # sched: [6:1.00]
-; SKYLAKE-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pmaxsw:
-; SKX:       # %bb.0:
-; SKX-NEXT:    pmaxsw %mm1, %mm0 # sched: [1:1.00]
-; SKX-NEXT:    pmaxsw (%rdi), %mm0 # sched: [6:1.00]
-; SKX-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_pmaxsw:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    pmaxsw %mm1, %mm0 # sched: [2:0.50]
-; BDVER2-NEXT:    pmaxsw (%rdi), %mm0 # sched: [7:0.50]
-; BDVER2-NEXT:    movq %mm0, %rax # sched: [10:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_pmaxsw:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    pmaxsw %mm1, %mm0 # sched: [1:0.50]
-; BTVER2-NEXT:    pmaxsw (%rdi), %mm0 # sched: [6:1.00]
-; BTVER2-NEXT:    movq %mm0, %rax # sched: [4:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_pmaxsw:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    pmaxsw %mm1, %mm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    pmaxsw (%rdi), %mm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call x86_mmx @llvm.x86.mmx.pmaxs.w(x86_mmx %a0, x86_mmx %a1)
-  %2 = load x86_mmx, x86_mmx *%a2, align 8
-  %3 = call x86_mmx @llvm.x86.mmx.pmaxs.w(x86_mmx %1, x86_mmx %2)
-  %4 = bitcast x86_mmx %3 to i64
-  ret i64 %4
-}
-declare x86_mmx @llvm.x86.mmx.pmaxs.w(x86_mmx, x86_mmx) nounwind readnone
-
-define i64 @test_pmaxub(x86_mmx %a0, x86_mmx %a1, x86_mmx* %a2) optsize {
-; GENERIC-LABEL: test_pmaxub:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    pmaxub %mm1, %mm0 # sched: [3:1.00]
-; GENERIC-NEXT:    pmaxub (%rdi), %mm0 # sched: [8:1.00]
-; GENERIC-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_pmaxub:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    pmaxub %mm1, %mm0 # sched: [1:0.50]
-; ATOM-NEXT:    pmaxub (%rdi), %mm0 # sched: [1:1.00]
-; ATOM-NEXT:    movq %mm0, %rax # sched: [3:3.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_pmaxub:
-; SLM:       # %bb.0:
-; SLM-NEXT:    pmaxub %mm1, %mm0 # sched: [1:0.50]
-; SLM-NEXT:    pmaxub (%rdi), %mm0 # sched: [4:1.00]
-; SLM-NEXT:    movq %mm0, %rax # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_pmaxub:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    pmaxub %mm1, %mm0 # sched: [3:1.00]
-; SANDY-NEXT:    pmaxub (%rdi), %mm0 # sched: [8:1.00]
-; SANDY-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_pmaxub:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    pmaxub %mm1, %mm0 # sched: [1:0.50]
-; HASWELL-NEXT:    pmaxub (%rdi), %mm0 # sched: [6:0.50]
-; HASWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pmaxub:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    pmaxub %mm1, %mm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    pmaxub (%rdi), %mm0 # sched: [6:0.50]
-; BROADWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pmaxub:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    pmaxub %mm1, %mm0 # sched: [1:1.00]
-; SKYLAKE-NEXT:    pmaxub (%rdi), %mm0 # sched: [6:1.00]
-; SKYLAKE-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pmaxub:
-; SKX:       # %bb.0:
-; SKX-NEXT:    pmaxub %mm1, %mm0 # sched: [1:1.00]
-; SKX-NEXT:    pmaxub (%rdi), %mm0 # sched: [6:1.00]
-; SKX-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_pmaxub:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    pmaxub %mm1, %mm0 # sched: [2:0.50]
-; BDVER2-NEXT:    pmaxub (%rdi), %mm0 # sched: [7:0.50]
-; BDVER2-NEXT:    movq %mm0, %rax # sched: [10:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_pmaxub:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    pmaxub %mm1, %mm0 # sched: [1:0.50]
-; BTVER2-NEXT:    pmaxub (%rdi), %mm0 # sched: [6:1.00]
-; BTVER2-NEXT:    movq %mm0, %rax # sched: [4:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_pmaxub:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    pmaxub %mm1, %mm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    pmaxub (%rdi), %mm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call x86_mmx @llvm.x86.mmx.pmaxu.b(x86_mmx %a0, x86_mmx %a1)
-  %2 = load x86_mmx, x86_mmx *%a2, align 8
-  %3 = call x86_mmx @llvm.x86.mmx.pmaxu.b(x86_mmx %1, x86_mmx %2)
-  %4 = bitcast x86_mmx %3 to i64
-  ret i64 %4
-}
-declare x86_mmx @llvm.x86.mmx.pmaxu.b(x86_mmx, x86_mmx) nounwind readnone
-
-define i64 @test_pminsw(x86_mmx %a0, x86_mmx %a1, x86_mmx* %a2) optsize {
-; GENERIC-LABEL: test_pminsw:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    pminsw %mm1, %mm0 # sched: [3:1.00]
-; GENERIC-NEXT:    pminsw (%rdi), %mm0 # sched: [8:1.00]
-; GENERIC-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_pminsw:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    pminsw %mm1, %mm0 # sched: [1:0.50]
-; ATOM-NEXT:    pminsw (%rdi), %mm0 # sched: [1:1.00]
-; ATOM-NEXT:    movq %mm0, %rax # sched: [3:3.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_pminsw:
-; SLM:       # %bb.0:
-; SLM-NEXT:    pminsw %mm1, %mm0 # sched: [1:0.50]
-; SLM-NEXT:    pminsw (%rdi), %mm0 # sched: [4:1.00]
-; SLM-NEXT:    movq %mm0, %rax # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_pminsw:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    pminsw %mm1, %mm0 # sched: [3:1.00]
-; SANDY-NEXT:    pminsw (%rdi), %mm0 # sched: [8:1.00]
-; SANDY-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_pminsw:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    pminsw %mm1, %mm0 # sched: [1:0.50]
-; HASWELL-NEXT:    pminsw (%rdi), %mm0 # sched: [6:0.50]
-; HASWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pminsw:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    pminsw %mm1, %mm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    pminsw (%rdi), %mm0 # sched: [6:0.50]
-; BROADWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pminsw:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    pminsw %mm1, %mm0 # sched: [1:1.00]
-; SKYLAKE-NEXT:    pminsw (%rdi), %mm0 # sched: [6:1.00]
-; SKYLAKE-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pminsw:
-; SKX:       # %bb.0:
-; SKX-NEXT:    pminsw %mm1, %mm0 # sched: [1:1.00]
-; SKX-NEXT:    pminsw (%rdi), %mm0 # sched: [6:1.00]
-; SKX-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_pminsw:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    pminsw %mm1, %mm0 # sched: [2:0.50]
-; BDVER2-NEXT:    pminsw (%rdi), %mm0 # sched: [7:0.50]
-; BDVER2-NEXT:    movq %mm0, %rax # sched: [10:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_pminsw:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    pminsw %mm1, %mm0 # sched: [1:0.50]
-; BTVER2-NEXT:    pminsw (%rdi), %mm0 # sched: [6:1.00]
-; BTVER2-NEXT:    movq %mm0, %rax # sched: [4:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_pminsw:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    pminsw %mm1, %mm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    pminsw (%rdi), %mm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call x86_mmx @llvm.x86.mmx.pmins.w(x86_mmx %a0, x86_mmx %a1)
-  %2 = load x86_mmx, x86_mmx *%a2, align 8
-  %3 = call x86_mmx @llvm.x86.mmx.pmins.w(x86_mmx %1, x86_mmx %2)
-  %4 = bitcast x86_mmx %3 to i64
-  ret i64 %4
-}
-declare x86_mmx @llvm.x86.mmx.pmins.w(x86_mmx, x86_mmx) nounwind readnone
-
-define i64 @test_pminub(x86_mmx %a0, x86_mmx %a1, x86_mmx* %a2) optsize {
-; GENERIC-LABEL: test_pminub:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    pminub %mm1, %mm0 # sched: [3:1.00]
-; GENERIC-NEXT:    pminub (%rdi), %mm0 # sched: [8:1.00]
-; GENERIC-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_pminub:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    pminub %mm1, %mm0 # sched: [1:0.50]
-; ATOM-NEXT:    pminub (%rdi), %mm0 # sched: [1:1.00]
-; ATOM-NEXT:    movq %mm0, %rax # sched: [3:3.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_pminub:
-; SLM:       # %bb.0:
-; SLM-NEXT:    pminub %mm1, %mm0 # sched: [1:0.50]
-; SLM-NEXT:    pminub (%rdi), %mm0 # sched: [4:1.00]
-; SLM-NEXT:    movq %mm0, %rax # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_pminub:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    pminub %mm1, %mm0 # sched: [3:1.00]
-; SANDY-NEXT:    pminub (%rdi), %mm0 # sched: [8:1.00]
-; SANDY-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_pminub:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    pminub %mm1, %mm0 # sched: [1:0.50]
-; HASWELL-NEXT:    pminub (%rdi), %mm0 # sched: [6:0.50]
-; HASWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pminub:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    pminub %mm1, %mm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    pminub (%rdi), %mm0 # sched: [6:0.50]
-; BROADWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pminub:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    pminub %mm1, %mm0 # sched: [1:1.00]
-; SKYLAKE-NEXT:    pminub (%rdi), %mm0 # sched: [6:1.00]
-; SKYLAKE-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pminub:
-; SKX:       # %bb.0:
-; SKX-NEXT:    pminub %mm1, %mm0 # sched: [1:1.00]
-; SKX-NEXT:    pminub (%rdi), %mm0 # sched: [6:1.00]
-; SKX-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_pminub:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    pminub %mm1, %mm0 # sched: [2:0.50]
-; BDVER2-NEXT:    pminub (%rdi), %mm0 # sched: [7:0.50]
-; BDVER2-NEXT:    movq %mm0, %rax # sched: [10:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_pminub:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    pminub %mm1, %mm0 # sched: [1:0.50]
-; BTVER2-NEXT:    pminub (%rdi), %mm0 # sched: [6:1.00]
-; BTVER2-NEXT:    movq %mm0, %rax # sched: [4:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_pminub:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    pminub %mm1, %mm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    pminub (%rdi), %mm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call x86_mmx @llvm.x86.mmx.pminu.b(x86_mmx %a0, x86_mmx %a1)
-  %2 = load x86_mmx, x86_mmx *%a2, align 8
-  %3 = call x86_mmx @llvm.x86.mmx.pminu.b(x86_mmx %1, x86_mmx %2)
-  %4 = bitcast x86_mmx %3 to i64
-  ret i64 %4
-}
-declare x86_mmx @llvm.x86.mmx.pminu.b(x86_mmx, x86_mmx) nounwind readnone
-
-define i32 @test_pmovmskb(x86_mmx %a0) optsize {
-; GENERIC-LABEL: test_pmovmskb:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    pmovmskb %mm0, %eax # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_pmovmskb:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    pmovmskb %mm0, %eax # sched: [3:3.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_pmovmskb:
-; SLM:       # %bb.0:
-; SLM-NEXT:    pmovmskb %mm0, %eax # sched: [4:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_pmovmskb:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    pmovmskb %mm0, %eax # sched: [1:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_pmovmskb:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    pmovmskb %mm0, %eax # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pmovmskb:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    pmovmskb %mm0, %eax # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pmovmskb:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    pmovmskb %mm0, %eax # sched: [2:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pmovmskb:
-; SKX:       # %bb.0:
-; SKX-NEXT:    pmovmskb %mm0, %eax # sched: [2:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_pmovmskb:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    pmovmskb %mm0, %eax # sched: [10:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_pmovmskb:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    pmovmskb %mm0, %eax # sched: [3:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_pmovmskb:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    pmovmskb %mm0, %eax # sched: [1:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call i32 @llvm.x86.mmx.pmovmskb(x86_mmx %a0)
-  ret i32 %1
-}
-declare i32 @llvm.x86.mmx.pmovmskb(x86_mmx) nounwind readnone
-
-define i64 @test_pmulhrsw(x86_mmx %a0, x86_mmx %a1, x86_mmx* %a2) optsize {
-; GENERIC-LABEL: test_pmulhrsw:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    pmulhrsw %mm1, %mm0 # sched: [5:1.00]
-; GENERIC-NEXT:    pmulhrsw (%rdi), %mm0 # sched: [10:1.00]
-; GENERIC-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_pmulhrsw:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    pmulhrsw %mm1, %mm0 # sched: [4:4.00]
-; ATOM-NEXT:    pmulhrsw (%rdi), %mm0 # sched: [4:4.00]
-; ATOM-NEXT:    movq %mm0, %rax # sched: [3:3.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_pmulhrsw:
-; SLM:       # %bb.0:
-; SLM-NEXT:    pmulhrsw %mm1, %mm0 # sched: [4:1.00]
-; SLM-NEXT:    pmulhrsw (%rdi), %mm0 # sched: [7:1.00]
-; SLM-NEXT:    movq %mm0, %rax # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_pmulhrsw:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    pmulhrsw %mm1, %mm0 # sched: [5:1.00]
-; SANDY-NEXT:    pmulhrsw (%rdi), %mm0 # sched: [10:1.00]
-; SANDY-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_pmulhrsw:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    pmulhrsw %mm1, %mm0 # sched: [5:1.00]
-; HASWELL-NEXT:    pmulhrsw (%rdi), %mm0 # sched: [10:1.00]
-; HASWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pmulhrsw:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    pmulhrsw %mm1, %mm0 # sched: [5:1.00]
-; BROADWELL-NEXT:    pmulhrsw (%rdi), %mm0 # sched: [10:1.00]
-; BROADWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pmulhrsw:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    pmulhrsw %mm1, %mm0 # sched: [4:1.00]
-; SKYLAKE-NEXT:    pmulhrsw (%rdi), %mm0 # sched: [9:1.00]
-; SKYLAKE-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pmulhrsw:
-; SKX:       # %bb.0:
-; SKX-NEXT:    pmulhrsw %mm1, %mm0 # sched: [4:1.00]
-; SKX-NEXT:    pmulhrsw (%rdi), %mm0 # sched: [9:1.00]
-; SKX-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_pmulhrsw:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    pmulhrsw %mm1, %mm0 # sched: [4:1.00]
-; BDVER2-NEXT:    pmulhrsw (%rdi), %mm0 # sched: [9:1.00]
-; BDVER2-NEXT:    movq %mm0, %rax # sched: [10:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_pmulhrsw:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    pmulhrsw %mm1, %mm0 # sched: [2:1.00]
-; BTVER2-NEXT:    pmulhrsw (%rdi), %mm0 # sched: [7:1.00]
-; BTVER2-NEXT:    movq %mm0, %rax # sched: [4:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_pmulhrsw:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    pmulhrsw %mm1, %mm0 # sched: [4:1.00]
-; ZNVER1-NEXT:    pmulhrsw (%rdi), %mm0 # sched: [11:1.00]
-; ZNVER1-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call x86_mmx @llvm.x86.ssse3.pmul.hr.sw(x86_mmx %a0, x86_mmx %a1)
-  %2 = load x86_mmx, x86_mmx *%a2, align 8
-  %3 = call x86_mmx @llvm.x86.ssse3.pmul.hr.sw(x86_mmx %1, x86_mmx %2)
-  %4 = bitcast x86_mmx %3 to i64
-  ret i64 %4
-}
-declare x86_mmx @llvm.x86.ssse3.pmul.hr.sw(x86_mmx, x86_mmx) nounwind readnone
-
-define i64 @test_pmulhw(x86_mmx %a0, x86_mmx %a1, x86_mmx* %a2) optsize {
-; GENERIC-LABEL: test_pmulhw:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    pmulhw %mm1, %mm0 # sched: [5:1.00]
-; GENERIC-NEXT:    pmulhw (%rdi), %mm0 # sched: [10:1.00]
-; GENERIC-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_pmulhw:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    pmulhw %mm1, %mm0 # sched: [4:4.00]
-; ATOM-NEXT:    pmulhw (%rdi), %mm0 # sched: [4:4.00]
-; ATOM-NEXT:    movq %mm0, %rax # sched: [3:3.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_pmulhw:
-; SLM:       # %bb.0:
-; SLM-NEXT:    pmulhw %mm1, %mm0 # sched: [4:1.00]
-; SLM-NEXT:    pmulhw (%rdi), %mm0 # sched: [7:1.00]
-; SLM-NEXT:    movq %mm0, %rax # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_pmulhw:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    pmulhw %mm1, %mm0 # sched: [5:1.00]
-; SANDY-NEXT:    pmulhw (%rdi), %mm0 # sched: [10:1.00]
-; SANDY-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_pmulhw:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    pmulhw %mm1, %mm0 # sched: [5:1.00]
-; HASWELL-NEXT:    pmulhw (%rdi), %mm0 # sched: [10:1.00]
-; HASWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pmulhw:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    pmulhw %mm1, %mm0 # sched: [5:1.00]
-; BROADWELL-NEXT:    pmulhw (%rdi), %mm0 # sched: [10:1.00]
-; BROADWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pmulhw:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    pmulhw %mm1, %mm0 # sched: [4:1.00]
-; SKYLAKE-NEXT:    pmulhw (%rdi), %mm0 # sched: [9:1.00]
-; SKYLAKE-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pmulhw:
-; SKX:       # %bb.0:
-; SKX-NEXT:    pmulhw %mm1, %mm0 # sched: [4:1.00]
-; SKX-NEXT:    pmulhw (%rdi), %mm0 # sched: [9:1.00]
-; SKX-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_pmulhw:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    pmulhw %mm1, %mm0 # sched: [4:1.00]
-; BDVER2-NEXT:    pmulhw (%rdi), %mm0 # sched: [9:1.00]
-; BDVER2-NEXT:    movq %mm0, %rax # sched: [10:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_pmulhw:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    pmulhw %mm1, %mm0 # sched: [2:1.00]
-; BTVER2-NEXT:    pmulhw (%rdi), %mm0 # sched: [7:1.00]
-; BTVER2-NEXT:    movq %mm0, %rax # sched: [4:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_pmulhw:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    pmulhw %mm1, %mm0 # sched: [4:1.00]
-; ZNVER1-NEXT:    pmulhw (%rdi), %mm0 # sched: [11:1.00]
-; ZNVER1-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call x86_mmx @llvm.x86.mmx.pmulh.w(x86_mmx %a0, x86_mmx %a1)
-  %2 = load x86_mmx, x86_mmx *%a2, align 8
-  %3 = call x86_mmx @llvm.x86.mmx.pmulh.w(x86_mmx %1, x86_mmx %2)
-  %4 = bitcast x86_mmx %3 to i64
-  ret i64 %4
-}
-declare x86_mmx @llvm.x86.mmx.pmulh.w(x86_mmx, x86_mmx) nounwind readnone
-
-define i64 @test_pmulhuw(x86_mmx %a0, x86_mmx %a1, x86_mmx* %a2) optsize {
-; GENERIC-LABEL: test_pmulhuw:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    pmulhuw %mm1, %mm0 # sched: [5:1.00]
-; GENERIC-NEXT:    pmulhuw (%rdi), %mm0 # sched: [10:1.00]
-; GENERIC-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_pmulhuw:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    pmulhuw %mm1, %mm0 # sched: [4:4.00]
-; ATOM-NEXT:    pmulhuw (%rdi), %mm0 # sched: [4:4.00]
-; ATOM-NEXT:    movq %mm0, %rax # sched: [3:3.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_pmulhuw:
-; SLM:       # %bb.0:
-; SLM-NEXT:    pmulhuw %mm1, %mm0 # sched: [4:1.00]
-; SLM-NEXT:    pmulhuw (%rdi), %mm0 # sched: [7:1.00]
-; SLM-NEXT:    movq %mm0, %rax # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_pmulhuw:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    pmulhuw %mm1, %mm0 # sched: [5:1.00]
-; SANDY-NEXT:    pmulhuw (%rdi), %mm0 # sched: [10:1.00]
-; SANDY-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_pmulhuw:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    pmulhuw %mm1, %mm0 # sched: [5:1.00]
-; HASWELL-NEXT:    pmulhuw (%rdi), %mm0 # sched: [10:1.00]
-; HASWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pmulhuw:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    pmulhuw %mm1, %mm0 # sched: [5:1.00]
-; BROADWELL-NEXT:    pmulhuw (%rdi), %mm0 # sched: [10:1.00]
-; BROADWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pmulhuw:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    pmulhuw %mm1, %mm0 # sched: [4:1.00]
-; SKYLAKE-NEXT:    pmulhuw (%rdi), %mm0 # sched: [9:1.00]
-; SKYLAKE-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pmulhuw:
-; SKX:       # %bb.0:
-; SKX-NEXT:    pmulhuw %mm1, %mm0 # sched: [4:1.00]
-; SKX-NEXT:    pmulhuw (%rdi), %mm0 # sched: [9:1.00]
-; SKX-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_pmulhuw:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    pmulhuw %mm1, %mm0 # sched: [4:1.00]
-; BDVER2-NEXT:    pmulhuw (%rdi), %mm0 # sched: [9:1.00]
-; BDVER2-NEXT:    movq %mm0, %rax # sched: [10:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_pmulhuw:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    pmulhuw %mm1, %mm0 # sched: [2:1.00]
-; BTVER2-NEXT:    pmulhuw (%rdi), %mm0 # sched: [7:1.00]
-; BTVER2-NEXT:    movq %mm0, %rax # sched: [4:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_pmulhuw:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    pmulhuw %mm1, %mm0 # sched: [4:1.00]
-; ZNVER1-NEXT:    pmulhuw (%rdi), %mm0 # sched: [11:1.00]
-; ZNVER1-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call x86_mmx @llvm.x86.mmx.pmulhu.w(x86_mmx %a0, x86_mmx %a1)
-  %2 = load x86_mmx, x86_mmx *%a2, align 8
-  %3 = call x86_mmx @llvm.x86.mmx.pmulhu.w(x86_mmx %1, x86_mmx %2)
-  %4 = bitcast x86_mmx %3 to i64
-  ret i64 %4
-}
-declare x86_mmx @llvm.x86.mmx.pmulhu.w(x86_mmx, x86_mmx) nounwind readnone
-
-define i64 @test_pmullw(x86_mmx %a0, x86_mmx %a1, x86_mmx* %a2) optsize {
-; GENERIC-LABEL: test_pmullw:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    pmullw %mm1, %mm0 # sched: [5:1.00]
-; GENERIC-NEXT:    pmullw (%rdi), %mm0 # sched: [10:1.00]
-; GENERIC-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_pmullw:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    pmullw %mm1, %mm0 # sched: [4:4.00]
-; ATOM-NEXT:    pmullw (%rdi), %mm0 # sched: [4:4.00]
-; ATOM-NEXT:    movq %mm0, %rax # sched: [3:3.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_pmullw:
-; SLM:       # %bb.0:
-; SLM-NEXT:    pmullw %mm1, %mm0 # sched: [4:1.00]
-; SLM-NEXT:    pmullw (%rdi), %mm0 # sched: [7:1.00]
-; SLM-NEXT:    movq %mm0, %rax # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_pmullw:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    pmullw %mm1, %mm0 # sched: [5:1.00]
-; SANDY-NEXT:    pmullw (%rdi), %mm0 # sched: [10:1.00]
-; SANDY-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_pmullw:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    pmullw %mm1, %mm0 # sched: [5:1.00]
-; HASWELL-NEXT:    pmullw (%rdi), %mm0 # sched: [10:1.00]
-; HASWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pmullw:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    pmullw %mm1, %mm0 # sched: [5:1.00]
-; BROADWELL-NEXT:    pmullw (%rdi), %mm0 # sched: [10:1.00]
-; BROADWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pmullw:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    pmullw %mm1, %mm0 # sched: [4:1.00]
-; SKYLAKE-NEXT:    pmullw (%rdi), %mm0 # sched: [9:1.00]
-; SKYLAKE-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pmullw:
-; SKX:       # %bb.0:
-; SKX-NEXT:    pmullw %mm1, %mm0 # sched: [4:1.00]
-; SKX-NEXT:    pmullw (%rdi), %mm0 # sched: [9:1.00]
-; SKX-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_pmullw:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    pmullw %mm1, %mm0 # sched: [4:1.00]
-; BDVER2-NEXT:    pmullw (%rdi), %mm0 # sched: [9:1.00]
-; BDVER2-NEXT:    movq %mm0, %rax # sched: [10:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_pmullw:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    pmullw %mm1, %mm0 # sched: [2:1.00]
-; BTVER2-NEXT:    pmullw (%rdi), %mm0 # sched: [7:1.00]
-; BTVER2-NEXT:    movq %mm0, %rax # sched: [4:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_pmullw:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    pmullw %mm1, %mm0 # sched: [4:1.00]
-; ZNVER1-NEXT:    pmullw (%rdi), %mm0 # sched: [11:1.00]
-; ZNVER1-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call x86_mmx @llvm.x86.mmx.pmull.w(x86_mmx %a0, x86_mmx %a1)
-  %2 = load x86_mmx, x86_mmx *%a2, align 8
-  %3 = call x86_mmx @llvm.x86.mmx.pmull.w(x86_mmx %1, x86_mmx %2)
-  %4 = bitcast x86_mmx %3 to i64
-  ret i64 %4
-}
-declare x86_mmx @llvm.x86.mmx.pmull.w(x86_mmx, x86_mmx) nounwind readnone
-
-define i64 @test_pmuludq(x86_mmx %a0, x86_mmx %a1, x86_mmx* %a2) optsize {
-; GENERIC-LABEL: test_pmuludq:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    pmuludq %mm1, %mm0 # sched: [5:1.00]
-; GENERIC-NEXT:    pmuludq (%rdi), %mm0 # sched: [10:1.00]
-; GENERIC-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_pmuludq:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    pmuludq %mm1, %mm0 # sched: [4:4.00]
-; ATOM-NEXT:    pmuludq (%rdi), %mm0 # sched: [4:4.00]
-; ATOM-NEXT:    movq %mm0, %rax # sched: [3:3.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_pmuludq:
-; SLM:       # %bb.0:
-; SLM-NEXT:    pmuludq %mm1, %mm0 # sched: [4:1.00]
-; SLM-NEXT:    pmuludq (%rdi), %mm0 # sched: [7:1.00]
-; SLM-NEXT:    movq %mm0, %rax # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_pmuludq:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    pmuludq %mm1, %mm0 # sched: [5:1.00]
-; SANDY-NEXT:    pmuludq (%rdi), %mm0 # sched: [10:1.00]
-; SANDY-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_pmuludq:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    pmuludq %mm1, %mm0 # sched: [5:1.00]
-; HASWELL-NEXT:    pmuludq (%rdi), %mm0 # sched: [10:1.00]
-; HASWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pmuludq:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    pmuludq %mm1, %mm0 # sched: [5:1.00]
-; BROADWELL-NEXT:    pmuludq (%rdi), %mm0 # sched: [10:1.00]
-; BROADWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pmuludq:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    pmuludq %mm1, %mm0 # sched: [4:1.00]
-; SKYLAKE-NEXT:    pmuludq (%rdi), %mm0 # sched: [9:1.00]
-; SKYLAKE-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pmuludq:
-; SKX:       # %bb.0:
-; SKX-NEXT:    pmuludq %mm1, %mm0 # sched: [4:1.00]
-; SKX-NEXT:    pmuludq (%rdi), %mm0 # sched: [9:1.00]
-; SKX-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_pmuludq:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    pmuludq %mm1, %mm0 # sched: [4:1.00]
-; BDVER2-NEXT:    pmuludq (%rdi), %mm0 # sched: [9:1.00]
-; BDVER2-NEXT:    movq %mm0, %rax # sched: [10:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_pmuludq:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    pmuludq %mm1, %mm0 # sched: [2:1.00]
-; BTVER2-NEXT:    pmuludq (%rdi), %mm0 # sched: [7:1.00]
-; BTVER2-NEXT:    movq %mm0, %rax # sched: [4:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_pmuludq:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    pmuludq %mm1, %mm0 # sched: [4:1.00]
-; ZNVER1-NEXT:    pmuludq (%rdi), %mm0 # sched: [11:1.00]
-; ZNVER1-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call x86_mmx @llvm.x86.mmx.pmulu.dq(x86_mmx %a0, x86_mmx %a1)
-  %2 = load x86_mmx, x86_mmx *%a2, align 8
-  %3 = call x86_mmx @llvm.x86.mmx.pmulu.dq(x86_mmx %1, x86_mmx %2)
-  %4 = bitcast x86_mmx %3 to i64
-  ret i64 %4
-}
-declare x86_mmx @llvm.x86.mmx.pmulu.dq(x86_mmx, x86_mmx) nounwind readnone
-
-define i64 @test_por(x86_mmx %a0, x86_mmx %a1, x86_mmx* %a2) optsize {
-; GENERIC-LABEL: test_por:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    por %mm1, %mm0 # sched: [1:0.33]
-; GENERIC-NEXT:    por (%rdi), %mm0 # sched: [6:0.50]
-; GENERIC-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_por:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    por %mm1, %mm0 # sched: [1:0.50]
-; ATOM-NEXT:    por (%rdi), %mm0 # sched: [1:1.00]
-; ATOM-NEXT:    movq %mm0, %rax # sched: [3:3.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_por:
-; SLM:       # %bb.0:
-; SLM-NEXT:    por %mm1, %mm0 # sched: [1:0.50]
-; SLM-NEXT:    por (%rdi), %mm0 # sched: [4:1.00]
-; SLM-NEXT:    movq %mm0, %rax # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_por:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    por %mm1, %mm0 # sched: [1:0.33]
-; SANDY-NEXT:    por (%rdi), %mm0 # sched: [6:0.50]
-; SANDY-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_por:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    por %mm1, %mm0 # sched: [1:0.33]
-; HASWELL-NEXT:    por (%rdi), %mm0 # sched: [6:0.50]
-; HASWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_por:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    por %mm1, %mm0 # sched: [1:0.33]
-; BROADWELL-NEXT:    por (%rdi), %mm0 # sched: [6:0.50]
-; BROADWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_por:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    por %mm1, %mm0 # sched: [1:0.50]
-; SKYLAKE-NEXT:    por (%rdi), %mm0 # sched: [6:0.50]
-; SKYLAKE-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_por:
-; SKX:       # %bb.0:
-; SKX-NEXT:    por %mm1, %mm0 # sched: [1:0.50]
-; SKX-NEXT:    por (%rdi), %mm0 # sched: [6:0.50]
-; SKX-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_por:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    por %mm1, %mm0 # sched: [2:0.50]
-; BDVER2-NEXT:    por (%rdi), %mm0 # sched: [7:0.50]
-; BDVER2-NEXT:    movq %mm0, %rax # sched: [10:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_por:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    por %mm1, %mm0 # sched: [1:0.50]
-; BTVER2-NEXT:    por (%rdi), %mm0 # sched: [6:1.00]
-; BTVER2-NEXT:    movq %mm0, %rax # sched: [4:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_por:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    por %mm1, %mm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    por (%rdi), %mm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call x86_mmx @llvm.x86.mmx.por(x86_mmx %a0, x86_mmx %a1)
-  %2 = load x86_mmx, x86_mmx *%a2, align 8
-  %3 = call x86_mmx @llvm.x86.mmx.por(x86_mmx %1, x86_mmx %2)
-  %4 = bitcast x86_mmx %3 to i64
-  ret i64 %4
-}
-declare x86_mmx @llvm.x86.mmx.por(x86_mmx, x86_mmx) nounwind readnone
-
-define i64 @test_psadbw(x86_mmx %a0, x86_mmx %a1, x86_mmx* %a2) optsize {
-; GENERIC-LABEL: test_psadbw:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    psadbw %mm1, %mm0 # sched: [5:1.00]
-; GENERIC-NEXT:    psadbw (%rdi), %mm0 # sched: [10:1.00]
-; GENERIC-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_psadbw:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    psadbw %mm1, %mm0 # sched: [4:2.00]
-; ATOM-NEXT:    psadbw (%rdi), %mm0 # sched: [4:2.00]
-; ATOM-NEXT:    movq %mm0, %rax # sched: [3:3.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_psadbw:
-; SLM:       # %bb.0:
-; SLM-NEXT:    psadbw %mm1, %mm0 # sched: [4:1.00]
-; SLM-NEXT:    psadbw (%rdi), %mm0 # sched: [7:1.00]
-; SLM-NEXT:    movq %mm0, %rax # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_psadbw:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    psadbw %mm1, %mm0 # sched: [5:1.00]
-; SANDY-NEXT:    psadbw (%rdi), %mm0 # sched: [10:1.00]
-; SANDY-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_psadbw:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    psadbw %mm1, %mm0 # sched: [5:1.00]
-; HASWELL-NEXT:    psadbw (%rdi), %mm0 # sched: [10:1.00]
-; HASWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_psadbw:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    psadbw %mm1, %mm0 # sched: [5:1.00]
-; BROADWELL-NEXT:    psadbw (%rdi), %mm0 # sched: [10:1.00]
-; BROADWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_psadbw:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    psadbw %mm1, %mm0 # sched: [3:1.00]
-; SKYLAKE-NEXT:    psadbw (%rdi), %mm0 # sched: [8:1.00]
-; SKYLAKE-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_psadbw:
-; SKX:       # %bb.0:
-; SKX-NEXT:    psadbw %mm1, %mm0 # sched: [3:1.00]
-; SKX-NEXT:    psadbw (%rdi), %mm0 # sched: [8:1.00]
-; SKX-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_psadbw:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    psadbw %mm1, %mm0 # sched: [4:0.50]
-; BDVER2-NEXT:    psadbw (%rdi), %mm0 # sched: [9:0.50]
-; BDVER2-NEXT:    movq %mm0, %rax # sched: [10:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_psadbw:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    psadbw %mm1, %mm0 # sched: [2:0.50]
-; BTVER2-NEXT:    psadbw (%rdi), %mm0 # sched: [7:1.00]
-; BTVER2-NEXT:    movq %mm0, %rax # sched: [4:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_psadbw:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    psadbw %mm1, %mm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    psadbw (%rdi), %mm0 # sched: [10:1.00]
-; ZNVER1-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call x86_mmx @llvm.x86.mmx.psad.bw(x86_mmx %a0, x86_mmx %a1)
-  %2 = load x86_mmx, x86_mmx *%a2, align 8
-  %3 = call x86_mmx @llvm.x86.mmx.psad.bw(x86_mmx %1, x86_mmx %2)
-  %4 = bitcast x86_mmx %3 to i64
-  ret i64 %4
-}
-declare x86_mmx @llvm.x86.mmx.psad.bw(x86_mmx, x86_mmx) nounwind readnone
-
-define i64 @test_pshufb(x86_mmx %a0, x86_mmx %a1, x86_mmx *%a2) optsize {
-; GENERIC-LABEL: test_pshufb:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    pshufb %mm1, %mm0 # sched: [1:0.50]
-; GENERIC-NEXT:    pshufb (%rdi), %mm0 # sched: [6:0.50]
-; GENERIC-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_pshufb:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    pshufb %mm1, %mm0 # sched: [1:1.00]
-; ATOM-NEXT:    pshufb (%rdi), %mm0 # sched: [1:1.00]
-; ATOM-NEXT:    movq %mm0, %rax # sched: [3:3.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_pshufb:
-; SLM:       # %bb.0:
-; SLM-NEXT:    pshufb %mm1, %mm0 # sched: [1:1.00]
-; SLM-NEXT:    pshufb (%rdi), %mm0 # sched: [4:1.00]
-; SLM-NEXT:    movq %mm0, %rax # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_pshufb:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    pshufb %mm1, %mm0 # sched: [1:0.50]
-; SANDY-NEXT:    pshufb (%rdi), %mm0 # sched: [6:0.50]
-; SANDY-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_pshufb:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    pshufb %mm1, %mm0 # sched: [1:1.00]
-; HASWELL-NEXT:    pshufb (%rdi), %mm0 # sched: [6:1.00]
-; HASWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pshufb:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    pshufb %mm1, %mm0 # sched: [1:1.00]
-; BROADWELL-NEXT:    pshufb (%rdi), %mm0 # sched: [6:1.00]
-; BROADWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pshufb:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    pshufb %mm1, %mm0 # sched: [1:1.00]
-; SKYLAKE-NEXT:    pshufb (%rdi), %mm0 # sched: [6:1.00]
-; SKYLAKE-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pshufb:
-; SKX:       # %bb.0:
-; SKX-NEXT:    pshufb %mm1, %mm0 # sched: [1:1.00]
-; SKX-NEXT:    pshufb (%rdi), %mm0 # sched: [6:1.00]
-; SKX-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_pshufb:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    pshufb %mm1, %mm0 # sched: [3:2.00]
-; BDVER2-NEXT:    pshufb (%rdi), %mm0 # sched: [8:2.00]
-; BDVER2-NEXT:    movq %mm0, %rax # sched: [10:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_pshufb:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    pshufb %mm1, %mm0 # sched: [2:0.50]
-; BTVER2-NEXT:    pshufb (%rdi), %mm0 # sched: [7:1.00]
-; BTVER2-NEXT:    movq %mm0, %rax # sched: [4:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_pshufb:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    pshufb %mm1, %mm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    pshufb (%rdi), %mm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call x86_mmx @llvm.x86.ssse3.pshuf.b(x86_mmx %a0, x86_mmx %a1)
-  %2 = load x86_mmx, x86_mmx *%a2, align 8
-  %3 = call x86_mmx @llvm.x86.ssse3.pshuf.b(x86_mmx %1, x86_mmx %2)
-  %4 = bitcast x86_mmx %3 to i64
-  ret i64 %4
-}
-declare x86_mmx @llvm.x86.ssse3.pshuf.b(x86_mmx, x86_mmx) nounwind readnone
-
-define i64 @test_pshufw(x86_mmx *%a0) optsize {
-; GENERIC-LABEL: test_pshufw:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    pshufw $0, (%rdi), %mm0 # mm0 = mem[0,0,0,0] sched: [6:1.00]
-; GENERIC-NEXT:    pshufw $0, %mm0, %mm0 # mm0 = mm0[0,0,0,0] sched: [1:1.00]
-; GENERIC-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_pshufw:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    pshufw $0, (%rdi), %mm0 # mm0 = mem[0,0,0,0] sched: [1:1.00]
-; ATOM-NEXT:    pshufw $0, %mm0, %mm0 # mm0 = mm0[0,0,0,0] sched: [1:1.00]
-; ATOM-NEXT:    movq %mm0, %rax # sched: [3:3.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_pshufw:
-; SLM:       # %bb.0:
-; SLM-NEXT:    pshufw $0, (%rdi), %mm0 # mm0 = mem[0,0,0,0] sched: [4:1.00]
-; SLM-NEXT:    pshufw $0, %mm0, %mm0 # mm0 = mm0[0,0,0,0] sched: [1:1.00]
-; SLM-NEXT:    movq %mm0, %rax # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_pshufw:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    pshufw $0, (%rdi), %mm0 # mm0 = mem[0,0,0,0] sched: [6:1.00]
-; SANDY-NEXT:    pshufw $0, %mm0, %mm0 # mm0 = mm0[0,0,0,0] sched: [1:1.00]
-; SANDY-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_pshufw:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    pshufw $0, (%rdi), %mm0 # mm0 = mem[0,0,0,0] sched: [6:1.00]
-; HASWELL-NEXT:    pshufw $0, %mm0, %mm0 # mm0 = mm0[0,0,0,0] sched: [1:1.00]
-; HASWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pshufw:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    pshufw $0, (%rdi), %mm0 # mm0 = mem[0,0,0,0] sched: [6:1.00]
-; BROADWELL-NEXT:    pshufw $0, %mm0, %mm0 # mm0 = mm0[0,0,0,0] sched: [1:1.00]
-; BROADWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pshufw:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    pshufw $0, (%rdi), %mm0 # mm0 = mem[0,0,0,0] sched: [6:1.00]
-; SKYLAKE-NEXT:    pshufw $0, %mm0, %mm0 # mm0 = mm0[0,0,0,0] sched: [1:1.00]
-; SKYLAKE-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pshufw:
-; SKX:       # %bb.0:
-; SKX-NEXT:    pshufw $0, (%rdi), %mm0 # mm0 = mem[0,0,0,0] sched: [6:1.00]
-; SKX-NEXT:    pshufw $0, %mm0, %mm0 # mm0 = mm0[0,0,0,0] sched: [1:1.00]
-; SKX-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_pshufw:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    pshufw $0, (%rdi), %mm0 # mm0 = mem[0,0,0,0] sched: [7:0.50]
-; BDVER2-NEXT:    pshufw $0, %mm0, %mm0 # mm0 = mm0[0,0,0,0] sched: [2:0.50]
-; BDVER2-NEXT:    movq %mm0, %rax # sched: [10:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_pshufw:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    pshufw $0, (%rdi), %mm0 # mm0 = mem[0,0,0,0] sched: [6:1.00]
-; BTVER2-NEXT:    pshufw $0, %mm0, %mm0 # mm0 = mm0[0,0,0,0] sched: [1:0.50]
-; BTVER2-NEXT:    movq %mm0, %rax # sched: [4:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_pshufw:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    pshufw $0, (%rdi), %mm0 # mm0 = mem[0,0,0,0] sched: [8:0.50]
-; ZNVER1-NEXT:    pshufw $0, %mm0, %mm0 # mm0 = mm0[0,0,0,0] sched: [1:0.25]
-; ZNVER1-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = load x86_mmx, x86_mmx *%a0, align 8
-  %2 = call x86_mmx @llvm.x86.sse.pshuf.w(x86_mmx %1, i8 0)
-  %3 = call x86_mmx @llvm.x86.sse.pshuf.w(x86_mmx %2, i8 0)
-  %4 = bitcast x86_mmx %3 to i64
-  ret i64 %4
-}
-declare x86_mmx @llvm.x86.sse.pshuf.w(x86_mmx, i8) nounwind readnone
-
-define i64 @test_psignb(x86_mmx %a0, x86_mmx %a1, x86_mmx* %a2) optsize {
-; GENERIC-LABEL: test_psignb:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    psignb %mm1, %mm0 # sched: [1:0.50]
-; GENERIC-NEXT:    psignb (%rdi), %mm0 # sched: [6:0.50]
-; GENERIC-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_psignb:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    psignb %mm1, %mm0 # sched: [1:0.50]
-; ATOM-NEXT:    psignb (%rdi), %mm0 # sched: [1:1.00]
-; ATOM-NEXT:    movq %mm0, %rax # sched: [3:3.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_psignb:
-; SLM:       # %bb.0:
-; SLM-NEXT:    psignb %mm1, %mm0 # sched: [1:0.50]
-; SLM-NEXT:    psignb (%rdi), %mm0 # sched: [4:1.00]
-; SLM-NEXT:    movq %mm0, %rax # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_psignb:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    psignb %mm1, %mm0 # sched: [1:0.50]
-; SANDY-NEXT:    psignb (%rdi), %mm0 # sched: [6:0.50]
-; SANDY-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_psignb:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    psignb %mm1, %mm0 # sched: [1:0.50]
-; HASWELL-NEXT:    psignb (%rdi), %mm0 # sched: [6:0.50]
-; HASWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_psignb:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    psignb %mm1, %mm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    psignb (%rdi), %mm0 # sched: [6:0.50]
-; BROADWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_psignb:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    psignb %mm1, %mm0 # sched: [1:0.50]
-; SKYLAKE-NEXT:    psignb (%rdi), %mm0 # sched: [6:0.50]
-; SKYLAKE-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_psignb:
-; SKX:       # %bb.0:
-; SKX-NEXT:    psignb %mm1, %mm0 # sched: [1:0.50]
-; SKX-NEXT:    psignb (%rdi), %mm0 # sched: [6:0.50]
-; SKX-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_psignb:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    psignb %mm1, %mm0 # sched: [2:0.50]
-; BDVER2-NEXT:    psignb (%rdi), %mm0 # sched: [7:0.50]
-; BDVER2-NEXT:    movq %mm0, %rax # sched: [10:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_psignb:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    psignb %mm1, %mm0 # sched: [1:0.50]
-; BTVER2-NEXT:    psignb (%rdi), %mm0 # sched: [6:1.00]
-; BTVER2-NEXT:    movq %mm0, %rax # sched: [4:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_psignb:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    psignb %mm1, %mm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    psignb (%rdi), %mm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call x86_mmx @llvm.x86.ssse3.psign.b(x86_mmx %a0, x86_mmx %a1)
-  %2 = load x86_mmx, x86_mmx *%a2, align 8
-  %3 = call x86_mmx @llvm.x86.ssse3.psign.b(x86_mmx %1, x86_mmx %2)
-  %4 = bitcast x86_mmx %3 to i64
-  ret i64 %4
-}
-declare x86_mmx @llvm.x86.ssse3.psign.b(x86_mmx, x86_mmx) nounwind readnone
-
-define i64 @test_psignd(x86_mmx %a0, x86_mmx %a1, x86_mmx* %a2) optsize {
-; GENERIC-LABEL: test_psignd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    psignd %mm1, %mm0 # sched: [1:0.50]
-; GENERIC-NEXT:    psignd (%rdi), %mm0 # sched: [6:0.50]
-; GENERIC-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_psignd:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    psignd %mm1, %mm0 # sched: [1:0.50]
-; ATOM-NEXT:    psignd (%rdi), %mm0 # sched: [1:1.00]
-; ATOM-NEXT:    movq %mm0, %rax # sched: [3:3.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_psignd:
-; SLM:       # %bb.0:
-; SLM-NEXT:    psignd %mm1, %mm0 # sched: [1:0.50]
-; SLM-NEXT:    psignd (%rdi), %mm0 # sched: [4:1.00]
-; SLM-NEXT:    movq %mm0, %rax # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_psignd:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    psignd %mm1, %mm0 # sched: [1:0.50]
-; SANDY-NEXT:    psignd (%rdi), %mm0 # sched: [6:0.50]
-; SANDY-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_psignd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    psignd %mm1, %mm0 # sched: [1:0.50]
-; HASWELL-NEXT:    psignd (%rdi), %mm0 # sched: [6:0.50]
-; HASWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_psignd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    psignd %mm1, %mm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    psignd (%rdi), %mm0 # sched: [6:0.50]
-; BROADWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_psignd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    psignd %mm1, %mm0 # sched: [1:0.50]
-; SKYLAKE-NEXT:    psignd (%rdi), %mm0 # sched: [6:0.50]
-; SKYLAKE-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_psignd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    psignd %mm1, %mm0 # sched: [1:0.50]
-; SKX-NEXT:    psignd (%rdi), %mm0 # sched: [6:0.50]
-; SKX-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_psignd:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    psignd %mm1, %mm0 # sched: [2:0.50]
-; BDVER2-NEXT:    psignd (%rdi), %mm0 # sched: [7:0.50]
-; BDVER2-NEXT:    movq %mm0, %rax # sched: [10:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_psignd:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    psignd %mm1, %mm0 # sched: [1:0.50]
-; BTVER2-NEXT:    psignd (%rdi), %mm0 # sched: [6:1.00]
-; BTVER2-NEXT:    movq %mm0, %rax # sched: [4:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_psignd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    psignd %mm1, %mm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    psignd (%rdi), %mm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call x86_mmx @llvm.x86.ssse3.psign.d(x86_mmx %a0, x86_mmx %a1)
-  %2 = load x86_mmx, x86_mmx *%a2, align 8
-  %3 = call x86_mmx @llvm.x86.ssse3.psign.d(x86_mmx %1, x86_mmx %2)
-  %4 = bitcast x86_mmx %3 to i64
-  ret i64 %4
-}
-declare x86_mmx @llvm.x86.ssse3.psign.d(x86_mmx, x86_mmx) nounwind readnone
-
-define i64 @test_psignw(x86_mmx %a0, x86_mmx %a1, x86_mmx* %a2) optsize {
-; GENERIC-LABEL: test_psignw:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    psignw %mm1, %mm0 # sched: [1:0.50]
-; GENERIC-NEXT:    psignw (%rdi), %mm0 # sched: [6:0.50]
-; GENERIC-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_psignw:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    psignw %mm1, %mm0 # sched: [1:0.50]
-; ATOM-NEXT:    psignw (%rdi), %mm0 # sched: [1:1.00]
-; ATOM-NEXT:    movq %mm0, %rax # sched: [3:3.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_psignw:
-; SLM:       # %bb.0:
-; SLM-NEXT:    psignw %mm1, %mm0 # sched: [1:0.50]
-; SLM-NEXT:    psignw (%rdi), %mm0 # sched: [4:1.00]
-; SLM-NEXT:    movq %mm0, %rax # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_psignw:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    psignw %mm1, %mm0 # sched: [1:0.50]
-; SANDY-NEXT:    psignw (%rdi), %mm0 # sched: [6:0.50]
-; SANDY-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_psignw:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    psignw %mm1, %mm0 # sched: [1:0.50]
-; HASWELL-NEXT:    psignw (%rdi), %mm0 # sched: [6:0.50]
-; HASWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_psignw:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    psignw %mm1, %mm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    psignw (%rdi), %mm0 # sched: [6:0.50]
-; BROADWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_psignw:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    psignw %mm1, %mm0 # sched: [1:0.50]
-; SKYLAKE-NEXT:    psignw (%rdi), %mm0 # sched: [6:0.50]
-; SKYLAKE-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_psignw:
-; SKX:       # %bb.0:
-; SKX-NEXT:    psignw %mm1, %mm0 # sched: [1:0.50]
-; SKX-NEXT:    psignw (%rdi), %mm0 # sched: [6:0.50]
-; SKX-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_psignw:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    psignw %mm1, %mm0 # sched: [2:0.50]
-; BDVER2-NEXT:    psignw (%rdi), %mm0 # sched: [7:0.50]
-; BDVER2-NEXT:    movq %mm0, %rax # sched: [10:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_psignw:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    psignw %mm1, %mm0 # sched: [1:0.50]
-; BTVER2-NEXT:    psignw (%rdi), %mm0 # sched: [6:1.00]
-; BTVER2-NEXT:    movq %mm0, %rax # sched: [4:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_psignw:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    psignw %mm1, %mm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    psignw (%rdi), %mm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call x86_mmx @llvm.x86.ssse3.psign.w(x86_mmx %a0, x86_mmx %a1)
-  %2 = load x86_mmx, x86_mmx *%a2, align 8
-  %3 = call x86_mmx @llvm.x86.ssse3.psign.w(x86_mmx %1, x86_mmx %2)
-  %4 = bitcast x86_mmx %3 to i64
-  ret i64 %4
-}
-declare x86_mmx @llvm.x86.ssse3.psign.w(x86_mmx, x86_mmx) nounwind readnone
-
-define i64 @test_pslld(x86_mmx %a0, x86_mmx %a1, x86_mmx* %a2) optsize {
-; GENERIC-LABEL: test_pslld:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    pslld %mm1, %mm0 # sched: [1:1.00]
-; GENERIC-NEXT:    pslld (%rdi), %mm0 # sched: [6:1.00]
-; GENERIC-NEXT:    pslld $7, %mm0 # sched: [1:1.00]
-; GENERIC-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_pslld:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    pslld %mm1, %mm0 # sched: [2:1.00]
-; ATOM-NEXT:    pslld (%rdi), %mm0 # sched: [3:1.50]
-; ATOM-NEXT:    pslld $7, %mm0 # sched: [1:0.50]
-; ATOM-NEXT:    movq %mm0, %rax # sched: [3:3.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_pslld:
-; SLM:       # %bb.0:
-; SLM-NEXT:    pslld %mm1, %mm0 # sched: [1:1.00]
-; SLM-NEXT:    pslld (%rdi), %mm0 # sched: [4:1.00]
-; SLM-NEXT:    pslld $7, %mm0 # sched: [1:1.00]
-; SLM-NEXT:    movq %mm0, %rax # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_pslld:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    pslld %mm1, %mm0 # sched: [1:1.00]
-; SANDY-NEXT:    pslld (%rdi), %mm0 # sched: [6:1.00]
-; SANDY-NEXT:    pslld $7, %mm0 # sched: [1:1.00]
-; SANDY-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_pslld:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    pslld %mm1, %mm0 # sched: [1:1.00]
-; HASWELL-NEXT:    pslld (%rdi), %mm0 # sched: [6:1.00]
-; HASWELL-NEXT:    pslld $7, %mm0 # sched: [1:1.00]
-; HASWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pslld:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    pslld %mm1, %mm0 # sched: [1:1.00]
-; BROADWELL-NEXT:    pslld (%rdi), %mm0 # sched: [6:1.00]
-; BROADWELL-NEXT:    pslld $7, %mm0 # sched: [1:1.00]
-; BROADWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pslld:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    pslld %mm1, %mm0 # sched: [1:1.00]
-; SKYLAKE-NEXT:    pslld (%rdi), %mm0 # sched: [6:1.00]
-; SKYLAKE-NEXT:    pslld $7, %mm0 # sched: [1:1.00]
-; SKYLAKE-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pslld:
-; SKX:       # %bb.0:
-; SKX-NEXT:    pslld %mm1, %mm0 # sched: [1:1.00]
-; SKX-NEXT:    pslld (%rdi), %mm0 # sched: [6:1.00]
-; SKX-NEXT:    pslld $7, %mm0 # sched: [1:1.00]
-; SKX-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_pslld:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    pslld %mm1, %mm0 # sched: [3:0.50]
-; BDVER2-NEXT:    pslld (%rdi), %mm0 # sched: [8:0.50]
-; BDVER2-NEXT:    pslld $7, %mm0 # sched: [2:0.50]
-; BDVER2-NEXT:    movq %mm0, %rax # sched: [10:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_pslld:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    pslld %mm1, %mm0 # sched: [1:0.50]
-; BTVER2-NEXT:    pslld (%rdi), %mm0 # sched: [6:1.00]
-; BTVER2-NEXT:    pslld $7, %mm0 # sched: [1:0.50]
-; BTVER2-NEXT:    movq %mm0, %rax # sched: [4:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_pslld:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    pslld %mm1, %mm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    pslld (%rdi), %mm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    pslld $7, %mm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call x86_mmx @llvm.x86.mmx.psll.d(x86_mmx %a0, x86_mmx %a1)
-  %2 = load x86_mmx, x86_mmx *%a2, align 8
-  %3 = call x86_mmx @llvm.x86.mmx.psll.d(x86_mmx %1, x86_mmx %2)
-  %4 = call x86_mmx @llvm.x86.mmx.pslli.d(x86_mmx %3, i32 7)
-  %5 = bitcast x86_mmx %4 to i64
-  ret i64 %5
-}
-declare x86_mmx @llvm.x86.mmx.psll.d(x86_mmx, x86_mmx) nounwind readnone
-declare x86_mmx @llvm.x86.mmx.pslli.d(x86_mmx, i32) nounwind readnone
-
-define i64 @test_psllq(x86_mmx %a0, x86_mmx %a1, x86_mmx* %a2) optsize {
-; GENERIC-LABEL: test_psllq:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    psllq %mm1, %mm0 # sched: [1:1.00]
-; GENERIC-NEXT:    psllq (%rdi), %mm0 # sched: [6:1.00]
-; GENERIC-NEXT:    psllq $7, %mm0 # sched: [1:1.00]
-; GENERIC-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_psllq:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    psllq %mm1, %mm0 # sched: [2:1.00]
-; ATOM-NEXT:    psllq (%rdi), %mm0 # sched: [3:1.50]
-; ATOM-NEXT:    psllq $7, %mm0 # sched: [1:0.50]
-; ATOM-NEXT:    movq %mm0, %rax # sched: [3:3.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_psllq:
-; SLM:       # %bb.0:
-; SLM-NEXT:    psllq %mm1, %mm0 # sched: [1:1.00]
-; SLM-NEXT:    psllq (%rdi), %mm0 # sched: [4:1.00]
-; SLM-NEXT:    psllq $7, %mm0 # sched: [1:1.00]
-; SLM-NEXT:    movq %mm0, %rax # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_psllq:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    psllq %mm1, %mm0 # sched: [1:1.00]
-; SANDY-NEXT:    psllq (%rdi), %mm0 # sched: [6:1.00]
-; SANDY-NEXT:    psllq $7, %mm0 # sched: [1:1.00]
-; SANDY-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_psllq:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    psllq %mm1, %mm0 # sched: [1:1.00]
-; HASWELL-NEXT:    psllq (%rdi), %mm0 # sched: [6:1.00]
-; HASWELL-NEXT:    psllq $7, %mm0 # sched: [1:1.00]
-; HASWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_psllq:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    psllq %mm1, %mm0 # sched: [1:1.00]
-; BROADWELL-NEXT:    psllq (%rdi), %mm0 # sched: [6:1.00]
-; BROADWELL-NEXT:    psllq $7, %mm0 # sched: [1:1.00]
-; BROADWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_psllq:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    psllq %mm1, %mm0 # sched: [1:1.00]
-; SKYLAKE-NEXT:    psllq (%rdi), %mm0 # sched: [6:1.00]
-; SKYLAKE-NEXT:    psllq $7, %mm0 # sched: [1:1.00]
-; SKYLAKE-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_psllq:
-; SKX:       # %bb.0:
-; SKX-NEXT:    psllq %mm1, %mm0 # sched: [1:1.00]
-; SKX-NEXT:    psllq (%rdi), %mm0 # sched: [6:1.00]
-; SKX-NEXT:    psllq $7, %mm0 # sched: [1:1.00]
-; SKX-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_psllq:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    psllq %mm1, %mm0 # sched: [3:0.50]
-; BDVER2-NEXT:    psllq (%rdi), %mm0 # sched: [8:0.50]
-; BDVER2-NEXT:    psllq $7, %mm0 # sched: [2:0.50]
-; BDVER2-NEXT:    movq %mm0, %rax # sched: [10:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_psllq:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    psllq %mm1, %mm0 # sched: [1:0.50]
-; BTVER2-NEXT:    psllq (%rdi), %mm0 # sched: [6:1.00]
-; BTVER2-NEXT:    psllq $7, %mm0 # sched: [1:0.50]
-; BTVER2-NEXT:    movq %mm0, %rax # sched: [4:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_psllq:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    psllq %mm1, %mm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    psllq (%rdi), %mm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    psllq $7, %mm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call x86_mmx @llvm.x86.mmx.psll.q(x86_mmx %a0, x86_mmx %a1)
-  %2 = load x86_mmx, x86_mmx *%a2, align 8
-  %3 = call x86_mmx @llvm.x86.mmx.psll.q(x86_mmx %1, x86_mmx %2)
-  %4 = call x86_mmx @llvm.x86.mmx.pslli.q(x86_mmx %3, i32 7)
-  %5 = bitcast x86_mmx %4 to i64
-  ret i64 %5
-}
-declare x86_mmx @llvm.x86.mmx.psll.q(x86_mmx, x86_mmx) nounwind readnone
-declare x86_mmx @llvm.x86.mmx.pslli.q(x86_mmx, i32) nounwind readnone
-
-define i64 @test_psllw(x86_mmx %a0, x86_mmx %a1, x86_mmx* %a2) optsize {
-; GENERIC-LABEL: test_psllw:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    psllw %mm1, %mm0 # sched: [1:1.00]
-; GENERIC-NEXT:    psllw (%rdi), %mm0 # sched: [6:1.00]
-; GENERIC-NEXT:    psllw $7, %mm0 # sched: [1:1.00]
-; GENERIC-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_psllw:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    psllw %mm1, %mm0 # sched: [2:1.00]
-; ATOM-NEXT:    psllw (%rdi), %mm0 # sched: [3:1.50]
-; ATOM-NEXT:    psllw $7, %mm0 # sched: [1:0.50]
-; ATOM-NEXT:    movq %mm0, %rax # sched: [3:3.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_psllw:
-; SLM:       # %bb.0:
-; SLM-NEXT:    psllw %mm1, %mm0 # sched: [1:1.00]
-; SLM-NEXT:    psllw (%rdi), %mm0 # sched: [4:1.00]
-; SLM-NEXT:    psllw $7, %mm0 # sched: [1:1.00]
-; SLM-NEXT:    movq %mm0, %rax # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_psllw:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    psllw %mm1, %mm0 # sched: [1:1.00]
-; SANDY-NEXT:    psllw (%rdi), %mm0 # sched: [6:1.00]
-; SANDY-NEXT:    psllw $7, %mm0 # sched: [1:1.00]
-; SANDY-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_psllw:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    psllw %mm1, %mm0 # sched: [1:1.00]
-; HASWELL-NEXT:    psllw (%rdi), %mm0 # sched: [6:1.00]
-; HASWELL-NEXT:    psllw $7, %mm0 # sched: [1:1.00]
-; HASWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_psllw:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    psllw %mm1, %mm0 # sched: [1:1.00]
-; BROADWELL-NEXT:    psllw (%rdi), %mm0 # sched: [6:1.00]
-; BROADWELL-NEXT:    psllw $7, %mm0 # sched: [1:1.00]
-; BROADWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_psllw:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    psllw %mm1, %mm0 # sched: [1:1.00]
-; SKYLAKE-NEXT:    psllw (%rdi), %mm0 # sched: [6:1.00]
-; SKYLAKE-NEXT:    psllw $7, %mm0 # sched: [1:1.00]
-; SKYLAKE-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_psllw:
-; SKX:       # %bb.0:
-; SKX-NEXT:    psllw %mm1, %mm0 # sched: [1:1.00]
-; SKX-NEXT:    psllw (%rdi), %mm0 # sched: [6:1.00]
-; SKX-NEXT:    psllw $7, %mm0 # sched: [1:1.00]
-; SKX-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_psllw:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    psllw %mm1, %mm0 # sched: [3:0.50]
-; BDVER2-NEXT:    psllw (%rdi), %mm0 # sched: [8:0.50]
-; BDVER2-NEXT:    psllw $7, %mm0 # sched: [2:0.50]
-; BDVER2-NEXT:    movq %mm0, %rax # sched: [10:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_psllw:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    psllw %mm1, %mm0 # sched: [1:0.50]
-; BTVER2-NEXT:    psllw (%rdi), %mm0 # sched: [6:1.00]
-; BTVER2-NEXT:    psllw $7, %mm0 # sched: [1:0.50]
-; BTVER2-NEXT:    movq %mm0, %rax # sched: [4:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_psllw:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    psllw %mm1, %mm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    psllw (%rdi), %mm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    psllw $7, %mm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call x86_mmx @llvm.x86.mmx.psll.w(x86_mmx %a0, x86_mmx %a1)
-  %2 = load x86_mmx, x86_mmx *%a2, align 8
-  %3 = call x86_mmx @llvm.x86.mmx.psll.w(x86_mmx %1, x86_mmx %2)
-  %4 = call x86_mmx @llvm.x86.mmx.pslli.w(x86_mmx %3, i32 7)
-  %5 = bitcast x86_mmx %4 to i64
-  ret i64 %5
-}
-declare x86_mmx @llvm.x86.mmx.psll.w(x86_mmx, x86_mmx) nounwind readnone
-declare x86_mmx @llvm.x86.mmx.pslli.w(x86_mmx, i32) nounwind readnone
-
-define i64 @test_psrad(x86_mmx %a0, x86_mmx %a1, x86_mmx* %a2) optsize {
-; GENERIC-LABEL: test_psrad:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    psrad %mm1, %mm0 # sched: [1:1.00]
-; GENERIC-NEXT:    psrad (%rdi), %mm0 # sched: [6:1.00]
-; GENERIC-NEXT:    psrad $7, %mm0 # sched: [1:1.00]
-; GENERIC-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_psrad:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    psrad %mm1, %mm0 # sched: [2:1.00]
-; ATOM-NEXT:    psrad (%rdi), %mm0 # sched: [3:1.50]
-; ATOM-NEXT:    psrad $7, %mm0 # sched: [1:0.50]
-; ATOM-NEXT:    movq %mm0, %rax # sched: [3:3.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_psrad:
-; SLM:       # %bb.0:
-; SLM-NEXT:    psrad %mm1, %mm0 # sched: [1:1.00]
-; SLM-NEXT:    psrad (%rdi), %mm0 # sched: [4:1.00]
-; SLM-NEXT:    psrad $7, %mm0 # sched: [1:1.00]
-; SLM-NEXT:    movq %mm0, %rax # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_psrad:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    psrad %mm1, %mm0 # sched: [1:1.00]
-; SANDY-NEXT:    psrad (%rdi), %mm0 # sched: [6:1.00]
-; SANDY-NEXT:    psrad $7, %mm0 # sched: [1:1.00]
-; SANDY-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_psrad:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    psrad %mm1, %mm0 # sched: [1:1.00]
-; HASWELL-NEXT:    psrad (%rdi), %mm0 # sched: [6:1.00]
-; HASWELL-NEXT:    psrad $7, %mm0 # sched: [1:1.00]
-; HASWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_psrad:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    psrad %mm1, %mm0 # sched: [1:1.00]
-; BROADWELL-NEXT:    psrad (%rdi), %mm0 # sched: [6:1.00]
-; BROADWELL-NEXT:    psrad $7, %mm0 # sched: [1:1.00]
-; BROADWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_psrad:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    psrad %mm1, %mm0 # sched: [1:1.00]
-; SKYLAKE-NEXT:    psrad (%rdi), %mm0 # sched: [6:1.00]
-; SKYLAKE-NEXT:    psrad $7, %mm0 # sched: [1:1.00]
-; SKYLAKE-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_psrad:
-; SKX:       # %bb.0:
-; SKX-NEXT:    psrad %mm1, %mm0 # sched: [1:1.00]
-; SKX-NEXT:    psrad (%rdi), %mm0 # sched: [6:1.00]
-; SKX-NEXT:    psrad $7, %mm0 # sched: [1:1.00]
-; SKX-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_psrad:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    psrad %mm1, %mm0 # sched: [3:0.50]
-; BDVER2-NEXT:    psrad (%rdi), %mm0 # sched: [8:0.50]
-; BDVER2-NEXT:    psrad $7, %mm0 # sched: [2:0.50]
-; BDVER2-NEXT:    movq %mm0, %rax # sched: [10:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_psrad:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    psrad %mm1, %mm0 # sched: [1:0.50]
-; BTVER2-NEXT:    psrad (%rdi), %mm0 # sched: [6:1.00]
-; BTVER2-NEXT:    psrad $7, %mm0 # sched: [1:0.50]
-; BTVER2-NEXT:    movq %mm0, %rax # sched: [4:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_psrad:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    psrad %mm1, %mm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    psrad (%rdi), %mm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    psrad $7, %mm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call x86_mmx @llvm.x86.mmx.psra.d(x86_mmx %a0, x86_mmx %a1)
-  %2 = load x86_mmx, x86_mmx *%a2, align 8
-  %3 = call x86_mmx @llvm.x86.mmx.psra.d(x86_mmx %1, x86_mmx %2)
-  %4 = call x86_mmx @llvm.x86.mmx.psrai.d(x86_mmx %3, i32 7)
-  %5 = bitcast x86_mmx %4 to i64
-  ret i64 %5
-}
-declare x86_mmx @llvm.x86.mmx.psra.d(x86_mmx, x86_mmx) nounwind readnone
-declare x86_mmx @llvm.x86.mmx.psrai.d(x86_mmx, i32) nounwind readnone
-
-define i64 @test_psraw(x86_mmx %a0, x86_mmx %a1, x86_mmx* %a2) optsize {
-; GENERIC-LABEL: test_psraw:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    psraw %mm1, %mm0 # sched: [1:1.00]
-; GENERIC-NEXT:    psraw (%rdi), %mm0 # sched: [6:1.00]
-; GENERIC-NEXT:    psraw $7, %mm0 # sched: [1:1.00]
-; GENERIC-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_psraw:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    psraw %mm1, %mm0 # sched: [2:1.00]
-; ATOM-NEXT:    psraw (%rdi), %mm0 # sched: [3:1.50]
-; ATOM-NEXT:    psraw $7, %mm0 # sched: [1:0.50]
-; ATOM-NEXT:    movq %mm0, %rax # sched: [3:3.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_psraw:
-; SLM:       # %bb.0:
-; SLM-NEXT:    psraw %mm1, %mm0 # sched: [1:1.00]
-; SLM-NEXT:    psraw (%rdi), %mm0 # sched: [4:1.00]
-; SLM-NEXT:    psraw $7, %mm0 # sched: [1:1.00]
-; SLM-NEXT:    movq %mm0, %rax # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_psraw:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    psraw %mm1, %mm0 # sched: [1:1.00]
-; SANDY-NEXT:    psraw (%rdi), %mm0 # sched: [6:1.00]
-; SANDY-NEXT:    psraw $7, %mm0 # sched: [1:1.00]
-; SANDY-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_psraw:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    psraw %mm1, %mm0 # sched: [1:1.00]
-; HASWELL-NEXT:    psraw (%rdi), %mm0 # sched: [6:1.00]
-; HASWELL-NEXT:    psraw $7, %mm0 # sched: [1:1.00]
-; HASWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_psraw:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    psraw %mm1, %mm0 # sched: [1:1.00]
-; BROADWELL-NEXT:    psraw (%rdi), %mm0 # sched: [6:1.00]
-; BROADWELL-NEXT:    psraw $7, %mm0 # sched: [1:1.00]
-; BROADWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_psraw:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    psraw %mm1, %mm0 # sched: [1:1.00]
-; SKYLAKE-NEXT:    psraw (%rdi), %mm0 # sched: [6:1.00]
-; SKYLAKE-NEXT:    psraw $7, %mm0 # sched: [1:1.00]
-; SKYLAKE-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_psraw:
-; SKX:       # %bb.0:
-; SKX-NEXT:    psraw %mm1, %mm0 # sched: [1:1.00]
-; SKX-NEXT:    psraw (%rdi), %mm0 # sched: [6:1.00]
-; SKX-NEXT:    psraw $7, %mm0 # sched: [1:1.00]
-; SKX-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_psraw:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    psraw %mm1, %mm0 # sched: [3:0.50]
-; BDVER2-NEXT:    psraw (%rdi), %mm0 # sched: [8:0.50]
-; BDVER2-NEXT:    psraw $7, %mm0 # sched: [2:0.50]
-; BDVER2-NEXT:    movq %mm0, %rax # sched: [10:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_psraw:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    psraw %mm1, %mm0 # sched: [1:0.50]
-; BTVER2-NEXT:    psraw (%rdi), %mm0 # sched: [6:1.00]
-; BTVER2-NEXT:    psraw $7, %mm0 # sched: [1:0.50]
-; BTVER2-NEXT:    movq %mm0, %rax # sched: [4:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_psraw:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    psraw %mm1, %mm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    psraw (%rdi), %mm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    psraw $7, %mm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call x86_mmx @llvm.x86.mmx.psra.w(x86_mmx %a0, x86_mmx %a1)
-  %2 = load x86_mmx, x86_mmx *%a2, align 8
-  %3 = call x86_mmx @llvm.x86.mmx.psra.w(x86_mmx %1, x86_mmx %2)
-  %4 = call x86_mmx @llvm.x86.mmx.psrai.w(x86_mmx %3, i32 7)
-  %5 = bitcast x86_mmx %4 to i64
-  ret i64 %5
-}
-declare x86_mmx @llvm.x86.mmx.psra.w(x86_mmx, x86_mmx) nounwind readnone
-declare x86_mmx @llvm.x86.mmx.psrai.w(x86_mmx, i32) nounwind readnone
-
-define i64 @test_psrld(x86_mmx %a0, x86_mmx %a1, x86_mmx* %a2) optsize {
-; GENERIC-LABEL: test_psrld:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    psrld %mm1, %mm0 # sched: [1:1.00]
-; GENERIC-NEXT:    psrld (%rdi), %mm0 # sched: [6:1.00]
-; GENERIC-NEXT:    psrld $7, %mm0 # sched: [1:1.00]
-; GENERIC-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_psrld:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    psrld %mm1, %mm0 # sched: [2:1.00]
-; ATOM-NEXT:    psrld (%rdi), %mm0 # sched: [3:1.50]
-; ATOM-NEXT:    psrld $7, %mm0 # sched: [1:0.50]
-; ATOM-NEXT:    movq %mm0, %rax # sched: [3:3.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_psrld:
-; SLM:       # %bb.0:
-; SLM-NEXT:    psrld %mm1, %mm0 # sched: [1:1.00]
-; SLM-NEXT:    psrld (%rdi), %mm0 # sched: [4:1.00]
-; SLM-NEXT:    psrld $7, %mm0 # sched: [1:1.00]
-; SLM-NEXT:    movq %mm0, %rax # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_psrld:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    psrld %mm1, %mm0 # sched: [1:1.00]
-; SANDY-NEXT:    psrld (%rdi), %mm0 # sched: [6:1.00]
-; SANDY-NEXT:    psrld $7, %mm0 # sched: [1:1.00]
-; SANDY-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_psrld:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    psrld %mm1, %mm0 # sched: [1:1.00]
-; HASWELL-NEXT:    psrld (%rdi), %mm0 # sched: [6:1.00]
-; HASWELL-NEXT:    psrld $7, %mm0 # sched: [1:1.00]
-; HASWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_psrld:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    psrld %mm1, %mm0 # sched: [1:1.00]
-; BROADWELL-NEXT:    psrld (%rdi), %mm0 # sched: [6:1.00]
-; BROADWELL-NEXT:    psrld $7, %mm0 # sched: [1:1.00]
-; BROADWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_psrld:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    psrld %mm1, %mm0 # sched: [1:1.00]
-; SKYLAKE-NEXT:    psrld (%rdi), %mm0 # sched: [6:1.00]
-; SKYLAKE-NEXT:    psrld $7, %mm0 # sched: [1:1.00]
-; SKYLAKE-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_psrld:
-; SKX:       # %bb.0:
-; SKX-NEXT:    psrld %mm1, %mm0 # sched: [1:1.00]
-; SKX-NEXT:    psrld (%rdi), %mm0 # sched: [6:1.00]
-; SKX-NEXT:    psrld $7, %mm0 # sched: [1:1.00]
-; SKX-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_psrld:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    psrld %mm1, %mm0 # sched: [3:0.50]
-; BDVER2-NEXT:    psrld (%rdi), %mm0 # sched: [8:0.50]
-; BDVER2-NEXT:    psrld $7, %mm0 # sched: [2:0.50]
-; BDVER2-NEXT:    movq %mm0, %rax # sched: [10:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_psrld:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    psrld %mm1, %mm0 # sched: [1:0.50]
-; BTVER2-NEXT:    psrld (%rdi), %mm0 # sched: [6:1.00]
-; BTVER2-NEXT:    psrld $7, %mm0 # sched: [1:0.50]
-; BTVER2-NEXT:    movq %mm0, %rax # sched: [4:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_psrld:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    psrld %mm1, %mm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    psrld (%rdi), %mm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    psrld $7, %mm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call x86_mmx @llvm.x86.mmx.psrl.d(x86_mmx %a0, x86_mmx %a1)
-  %2 = load x86_mmx, x86_mmx *%a2, align 8
-  %3 = call x86_mmx @llvm.x86.mmx.psrl.d(x86_mmx %1, x86_mmx %2)
-  %4 = call x86_mmx @llvm.x86.mmx.psrli.d(x86_mmx %3, i32 7)
-  %5 = bitcast x86_mmx %4 to i64
-  ret i64 %5
-}
-declare x86_mmx @llvm.x86.mmx.psrl.d(x86_mmx, x86_mmx) nounwind readnone
-declare x86_mmx @llvm.x86.mmx.psrli.d(x86_mmx, i32) nounwind readnone
-
-define i64 @test_psrlq(x86_mmx %a0, x86_mmx %a1, x86_mmx* %a2) optsize {
-; GENERIC-LABEL: test_psrlq:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    psrlq %mm1, %mm0 # sched: [1:1.00]
-; GENERIC-NEXT:    psrlq (%rdi), %mm0 # sched: [6:1.00]
-; GENERIC-NEXT:    psrlq $7, %mm0 # sched: [1:1.00]
-; GENERIC-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_psrlq:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    psrlq %mm1, %mm0 # sched: [2:1.00]
-; ATOM-NEXT:    psrlq (%rdi), %mm0 # sched: [3:1.50]
-; ATOM-NEXT:    psrlq $7, %mm0 # sched: [1:0.50]
-; ATOM-NEXT:    movq %mm0, %rax # sched: [3:3.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_psrlq:
-; SLM:       # %bb.0:
-; SLM-NEXT:    psrlq %mm1, %mm0 # sched: [1:1.00]
-; SLM-NEXT:    psrlq (%rdi), %mm0 # sched: [4:1.00]
-; SLM-NEXT:    psrlq $7, %mm0 # sched: [1:1.00]
-; SLM-NEXT:    movq %mm0, %rax # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_psrlq:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    psrlq %mm1, %mm0 # sched: [1:1.00]
-; SANDY-NEXT:    psrlq (%rdi), %mm0 # sched: [6:1.00]
-; SANDY-NEXT:    psrlq $7, %mm0 # sched: [1:1.00]
-; SANDY-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_psrlq:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    psrlq %mm1, %mm0 # sched: [1:1.00]
-; HASWELL-NEXT:    psrlq (%rdi), %mm0 # sched: [6:1.00]
-; HASWELL-NEXT:    psrlq $7, %mm0 # sched: [1:1.00]
-; HASWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_psrlq:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    psrlq %mm1, %mm0 # sched: [1:1.00]
-; BROADWELL-NEXT:    psrlq (%rdi), %mm0 # sched: [6:1.00]
-; BROADWELL-NEXT:    psrlq $7, %mm0 # sched: [1:1.00]
-; BROADWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_psrlq:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    psrlq %mm1, %mm0 # sched: [1:1.00]
-; SKYLAKE-NEXT:    psrlq (%rdi), %mm0 # sched: [6:1.00]
-; SKYLAKE-NEXT:    psrlq $7, %mm0 # sched: [1:1.00]
-; SKYLAKE-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_psrlq:
-; SKX:       # %bb.0:
-; SKX-NEXT:    psrlq %mm1, %mm0 # sched: [1:1.00]
-; SKX-NEXT:    psrlq (%rdi), %mm0 # sched: [6:1.00]
-; SKX-NEXT:    psrlq $7, %mm0 # sched: [1:1.00]
-; SKX-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_psrlq:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    psrlq %mm1, %mm0 # sched: [3:0.50]
-; BDVER2-NEXT:    psrlq (%rdi), %mm0 # sched: [8:0.50]
-; BDVER2-NEXT:    psrlq $7, %mm0 # sched: [2:0.50]
-; BDVER2-NEXT:    movq %mm0, %rax # sched: [10:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_psrlq:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    psrlq %mm1, %mm0 # sched: [1:0.50]
-; BTVER2-NEXT:    psrlq (%rdi), %mm0 # sched: [6:1.00]
-; BTVER2-NEXT:    psrlq $7, %mm0 # sched: [1:0.50]
-; BTVER2-NEXT:    movq %mm0, %rax # sched: [4:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_psrlq:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    psrlq %mm1, %mm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    psrlq (%rdi), %mm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    psrlq $7, %mm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call x86_mmx @llvm.x86.mmx.psrl.q(x86_mmx %a0, x86_mmx %a1)
-  %2 = load x86_mmx, x86_mmx *%a2, align 8
-  %3 = call x86_mmx @llvm.x86.mmx.psrl.q(x86_mmx %1, x86_mmx %2)
-  %4 = call x86_mmx @llvm.x86.mmx.psrli.q(x86_mmx %3, i32 7)
-  %5 = bitcast x86_mmx %4 to i64
-  ret i64 %5
-}
-declare x86_mmx @llvm.x86.mmx.psrl.q(x86_mmx, x86_mmx) nounwind readnone
-declare x86_mmx @llvm.x86.mmx.psrli.q(x86_mmx, i32) nounwind readnone
-
-define i64 @test_psrlw(x86_mmx %a0, x86_mmx %a1, x86_mmx* %a2) optsize {
-; GENERIC-LABEL: test_psrlw:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    psrlw %mm1, %mm0 # sched: [1:1.00]
-; GENERIC-NEXT:    psrlw (%rdi), %mm0 # sched: [6:1.00]
-; GENERIC-NEXT:    psrlw $7, %mm0 # sched: [1:1.00]
-; GENERIC-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_psrlw:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    psrlw %mm1, %mm0 # sched: [2:1.00]
-; ATOM-NEXT:    psrlw (%rdi), %mm0 # sched: [3:1.50]
-; ATOM-NEXT:    psrlw $7, %mm0 # sched: [1:0.50]
-; ATOM-NEXT:    movq %mm0, %rax # sched: [3:3.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_psrlw:
-; SLM:       # %bb.0:
-; SLM-NEXT:    psrlw %mm1, %mm0 # sched: [1:1.00]
-; SLM-NEXT:    psrlw (%rdi), %mm0 # sched: [4:1.00]
-; SLM-NEXT:    psrlw $7, %mm0 # sched: [1:1.00]
-; SLM-NEXT:    movq %mm0, %rax # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_psrlw:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    psrlw %mm1, %mm0 # sched: [1:1.00]
-; SANDY-NEXT:    psrlw (%rdi), %mm0 # sched: [6:1.00]
-; SANDY-NEXT:    psrlw $7, %mm0 # sched: [1:1.00]
-; SANDY-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_psrlw:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    psrlw %mm1, %mm0 # sched: [1:1.00]
-; HASWELL-NEXT:    psrlw (%rdi), %mm0 # sched: [6:1.00]
-; HASWELL-NEXT:    psrlw $7, %mm0 # sched: [1:1.00]
-; HASWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_psrlw:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    psrlw %mm1, %mm0 # sched: [1:1.00]
-; BROADWELL-NEXT:    psrlw (%rdi), %mm0 # sched: [6:1.00]
-; BROADWELL-NEXT:    psrlw $7, %mm0 # sched: [1:1.00]
-; BROADWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_psrlw:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    psrlw %mm1, %mm0 # sched: [1:1.00]
-; SKYLAKE-NEXT:    psrlw (%rdi), %mm0 # sched: [6:1.00]
-; SKYLAKE-NEXT:    psrlw $7, %mm0 # sched: [1:1.00]
-; SKYLAKE-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_psrlw:
-; SKX:       # %bb.0:
-; SKX-NEXT:    psrlw %mm1, %mm0 # sched: [1:1.00]
-; SKX-NEXT:    psrlw (%rdi), %mm0 # sched: [6:1.00]
-; SKX-NEXT:    psrlw $7, %mm0 # sched: [1:1.00]
-; SKX-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_psrlw:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    psrlw %mm1, %mm0 # sched: [3:0.50]
-; BDVER2-NEXT:    psrlw (%rdi), %mm0 # sched: [8:0.50]
-; BDVER2-NEXT:    psrlw $7, %mm0 # sched: [2:0.50]
-; BDVER2-NEXT:    movq %mm0, %rax # sched: [10:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_psrlw:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    psrlw %mm1, %mm0 # sched: [1:0.50]
-; BTVER2-NEXT:    psrlw (%rdi), %mm0 # sched: [6:1.00]
-; BTVER2-NEXT:    psrlw $7, %mm0 # sched: [1:0.50]
-; BTVER2-NEXT:    movq %mm0, %rax # sched: [4:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_psrlw:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    psrlw %mm1, %mm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    psrlw (%rdi), %mm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    psrlw $7, %mm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call x86_mmx @llvm.x86.mmx.psrl.w(x86_mmx %a0, x86_mmx %a1)
-  %2 = load x86_mmx, x86_mmx *%a2, align 8
-  %3 = call x86_mmx @llvm.x86.mmx.psrl.w(x86_mmx %1, x86_mmx %2)
-  %4 = call x86_mmx @llvm.x86.mmx.psrli.w(x86_mmx %3, i32 7)
-  %5 = bitcast x86_mmx %4 to i64
-  ret i64 %5
-}
-declare x86_mmx @llvm.x86.mmx.psrl.w(x86_mmx, x86_mmx) nounwind readnone
-declare x86_mmx @llvm.x86.mmx.psrli.w(x86_mmx, i32) nounwind readnone
-
-define i64 @test_psubb(x86_mmx %a0, x86_mmx %a1, x86_mmx* %a2) optsize {
-; GENERIC-LABEL: test_psubb:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    psubb %mm1, %mm0 # sched: [3:1.00]
-; GENERIC-NEXT:    psubb (%rdi), %mm0 # sched: [8:1.00]
-; GENERIC-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_psubb:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    psubb %mm1, %mm0 # sched: [1:0.50]
-; ATOM-NEXT:    psubb (%rdi), %mm0 # sched: [1:1.00]
-; ATOM-NEXT:    movq %mm0, %rax # sched: [3:3.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_psubb:
-; SLM:       # %bb.0:
-; SLM-NEXT:    psubb %mm1, %mm0 # sched: [1:0.50]
-; SLM-NEXT:    psubb (%rdi), %mm0 # sched: [4:1.00]
-; SLM-NEXT:    movq %mm0, %rax # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_psubb:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    psubb %mm1, %mm0 # sched: [3:1.00]
-; SANDY-NEXT:    psubb (%rdi), %mm0 # sched: [8:1.00]
-; SANDY-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_psubb:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    psubb %mm1, %mm0 # sched: [1:0.50]
-; HASWELL-NEXT:    psubb (%rdi), %mm0 # sched: [6:0.50]
-; HASWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_psubb:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    psubb %mm1, %mm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    psubb (%rdi), %mm0 # sched: [6:0.50]
-; BROADWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_psubb:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    psubb %mm1, %mm0 # sched: [1:0.50]
-; SKYLAKE-NEXT:    psubb (%rdi), %mm0 # sched: [6:0.50]
-; SKYLAKE-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_psubb:
-; SKX:       # %bb.0:
-; SKX-NEXT:    psubb %mm1, %mm0 # sched: [1:0.50]
-; SKX-NEXT:    psubb (%rdi), %mm0 # sched: [6:0.50]
-; SKX-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_psubb:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    psubb %mm1, %mm0 # sched: [2:0.50]
-; BDVER2-NEXT:    psubb (%rdi), %mm0 # sched: [7:0.50]
-; BDVER2-NEXT:    movq %mm0, %rax # sched: [10:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_psubb:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    psubb %mm1, %mm0 # sched: [1:0.50]
-; BTVER2-NEXT:    psubb (%rdi), %mm0 # sched: [6:1.00]
-; BTVER2-NEXT:    movq %mm0, %rax # sched: [4:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_psubb:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    psubb %mm1, %mm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    psubb (%rdi), %mm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call x86_mmx @llvm.x86.mmx.psub.b(x86_mmx %a0, x86_mmx %a1)
-  %2 = load x86_mmx, x86_mmx *%a2, align 8
-  %3 = call x86_mmx @llvm.x86.mmx.psub.b(x86_mmx %1, x86_mmx %2)
-  %4 = bitcast x86_mmx %3 to i64
-  ret i64 %4
-}
-declare x86_mmx @llvm.x86.mmx.psub.b(x86_mmx, x86_mmx) nounwind readnone
-
-define i64 @test_psubd(x86_mmx %a0, x86_mmx %a1, x86_mmx* %a2) optsize {
-; GENERIC-LABEL: test_psubd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    psubd %mm1, %mm0 # sched: [3:1.00]
-; GENERIC-NEXT:    psubd (%rdi), %mm0 # sched: [8:1.00]
-; GENERIC-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_psubd:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    psubd %mm1, %mm0 # sched: [1:0.50]
-; ATOM-NEXT:    psubd (%rdi), %mm0 # sched: [1:1.00]
-; ATOM-NEXT:    movq %mm0, %rax # sched: [3:3.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_psubd:
-; SLM:       # %bb.0:
-; SLM-NEXT:    psubd %mm1, %mm0 # sched: [1:0.50]
-; SLM-NEXT:    psubd (%rdi), %mm0 # sched: [4:1.00]
-; SLM-NEXT:    movq %mm0, %rax # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_psubd:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    psubd %mm1, %mm0 # sched: [3:1.00]
-; SANDY-NEXT:    psubd (%rdi), %mm0 # sched: [8:1.00]
-; SANDY-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_psubd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    psubd %mm1, %mm0 # sched: [1:0.50]
-; HASWELL-NEXT:    psubd (%rdi), %mm0 # sched: [6:0.50]
-; HASWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_psubd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    psubd %mm1, %mm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    psubd (%rdi), %mm0 # sched: [6:0.50]
-; BROADWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_psubd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    psubd %mm1, %mm0 # sched: [1:0.50]
-; SKYLAKE-NEXT:    psubd (%rdi), %mm0 # sched: [6:0.50]
-; SKYLAKE-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_psubd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    psubd %mm1, %mm0 # sched: [1:0.50]
-; SKX-NEXT:    psubd (%rdi), %mm0 # sched: [6:0.50]
-; SKX-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_psubd:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    psubd %mm1, %mm0 # sched: [2:0.50]
-; BDVER2-NEXT:    psubd (%rdi), %mm0 # sched: [7:0.50]
-; BDVER2-NEXT:    movq %mm0, %rax # sched: [10:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_psubd:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    psubd %mm1, %mm0 # sched: [1:0.50]
-; BTVER2-NEXT:    psubd (%rdi), %mm0 # sched: [6:1.00]
-; BTVER2-NEXT:    movq %mm0, %rax # sched: [4:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_psubd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    psubd %mm1, %mm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    psubd (%rdi), %mm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call x86_mmx @llvm.x86.mmx.psub.d(x86_mmx %a0, x86_mmx %a1)
-  %2 = load x86_mmx, x86_mmx *%a2, align 8
-  %3 = call x86_mmx @llvm.x86.mmx.psub.d(x86_mmx %1, x86_mmx %2)
-  %4 = bitcast x86_mmx %3 to i64
-  ret i64 %4
-}
-declare x86_mmx @llvm.x86.mmx.psub.d(x86_mmx, x86_mmx) nounwind readnone
-
-define i64 @test_psubq(x86_mmx %a0, x86_mmx %a1, x86_mmx* %a2) optsize {
-; GENERIC-LABEL: test_psubq:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    psubq %mm1, %mm0 # sched: [3:1.00]
-; GENERIC-NEXT:    psubq (%rdi), %mm0 # sched: [8:1.00]
-; GENERIC-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_psubq:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    psubq %mm1, %mm0 # sched: [2:1.00]
-; ATOM-NEXT:    psubq (%rdi), %mm0 # sched: [3:1.50]
-; ATOM-NEXT:    movq %mm0, %rax # sched: [3:3.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_psubq:
-; SLM:       # %bb.0:
-; SLM-NEXT:    psubq %mm1, %mm0 # sched: [1:0.50]
-; SLM-NEXT:    psubq (%rdi), %mm0 # sched: [4:1.00]
-; SLM-NEXT:    movq %mm0, %rax # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_psubq:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    psubq %mm1, %mm0 # sched: [3:1.00]
-; SANDY-NEXT:    psubq (%rdi), %mm0 # sched: [8:1.00]
-; SANDY-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_psubq:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    psubq %mm1, %mm0 # sched: [1:0.50]
-; HASWELL-NEXT:    psubq (%rdi), %mm0 # sched: [6:0.50]
-; HASWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_psubq:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    psubq %mm1, %mm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    psubq (%rdi), %mm0 # sched: [6:0.50]
-; BROADWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_psubq:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    psubq %mm1, %mm0 # sched: [1:0.50]
-; SKYLAKE-NEXT:    psubq (%rdi), %mm0 # sched: [6:0.50]
-; SKYLAKE-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_psubq:
-; SKX:       # %bb.0:
-; SKX-NEXT:    psubq %mm1, %mm0 # sched: [1:0.50]
-; SKX-NEXT:    psubq (%rdi), %mm0 # sched: [6:0.50]
-; SKX-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_psubq:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    psubq %mm1, %mm0 # sched: [2:0.50]
-; BDVER2-NEXT:    psubq (%rdi), %mm0 # sched: [7:0.50]
-; BDVER2-NEXT:    movq %mm0, %rax # sched: [10:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_psubq:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    psubq %mm1, %mm0 # sched: [1:0.50]
-; BTVER2-NEXT:    psubq (%rdi), %mm0 # sched: [6:1.00]
-; BTVER2-NEXT:    movq %mm0, %rax # sched: [4:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_psubq:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    psubq %mm1, %mm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    psubq (%rdi), %mm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call x86_mmx @llvm.x86.mmx.psub.q(x86_mmx %a0, x86_mmx %a1)
-  %2 = load x86_mmx, x86_mmx *%a2, align 8
-  %3 = call x86_mmx @llvm.x86.mmx.psub.q(x86_mmx %1, x86_mmx %2)
-  %4 = bitcast x86_mmx %3 to i64
-  ret i64 %4
-}
-declare x86_mmx @llvm.x86.mmx.psub.q(x86_mmx, x86_mmx) nounwind readnone
-
-define i64 @test_psubsb(x86_mmx %a0, x86_mmx %a1, x86_mmx* %a2) optsize {
-; GENERIC-LABEL: test_psubsb:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    psubsb %mm1, %mm0 # sched: [3:1.00]
-; GENERIC-NEXT:    psubsb (%rdi), %mm0 # sched: [8:1.00]
-; GENERIC-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_psubsb:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    psubsb %mm1, %mm0 # sched: [1:0.50]
-; ATOM-NEXT:    psubsb (%rdi), %mm0 # sched: [1:1.00]
-; ATOM-NEXT:    movq %mm0, %rax # sched: [3:3.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_psubsb:
-; SLM:       # %bb.0:
-; SLM-NEXT:    psubsb %mm1, %mm0 # sched: [1:0.50]
-; SLM-NEXT:    psubsb (%rdi), %mm0 # sched: [4:1.00]
-; SLM-NEXT:    movq %mm0, %rax # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_psubsb:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    psubsb %mm1, %mm0 # sched: [3:1.00]
-; SANDY-NEXT:    psubsb (%rdi), %mm0 # sched: [8:1.00]
-; SANDY-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_psubsb:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    psubsb %mm1, %mm0 # sched: [1:0.50]
-; HASWELL-NEXT:    psubsb (%rdi), %mm0 # sched: [6:0.50]
-; HASWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_psubsb:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    psubsb %mm1, %mm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    psubsb (%rdi), %mm0 # sched: [6:0.50]
-; BROADWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_psubsb:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    psubsb %mm1, %mm0 # sched: [1:1.00]
-; SKYLAKE-NEXT:    psubsb (%rdi), %mm0 # sched: [6:1.00]
-; SKYLAKE-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_psubsb:
-; SKX:       # %bb.0:
-; SKX-NEXT:    psubsb %mm1, %mm0 # sched: [1:1.00]
-; SKX-NEXT:    psubsb (%rdi), %mm0 # sched: [6:1.00]
-; SKX-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_psubsb:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    psubsb %mm1, %mm0 # sched: [2:0.50]
-; BDVER2-NEXT:    psubsb (%rdi), %mm0 # sched: [7:0.50]
-; BDVER2-NEXT:    movq %mm0, %rax # sched: [10:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_psubsb:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    psubsb %mm1, %mm0 # sched: [1:0.50]
-; BTVER2-NEXT:    psubsb (%rdi), %mm0 # sched: [6:1.00]
-; BTVER2-NEXT:    movq %mm0, %rax # sched: [4:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_psubsb:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    psubsb %mm1, %mm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    psubsb (%rdi), %mm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call x86_mmx @llvm.x86.mmx.psubs.b(x86_mmx %a0, x86_mmx %a1)
-  %2 = load x86_mmx, x86_mmx *%a2, align 8
-  %3 = call x86_mmx @llvm.x86.mmx.psubs.b(x86_mmx %1, x86_mmx %2)
-  %4 = bitcast x86_mmx %3 to i64
-  ret i64 %4
-}
-declare x86_mmx @llvm.x86.mmx.psubs.b(x86_mmx, x86_mmx) nounwind readnone
-
-define i64 @test_psubsw(x86_mmx %a0, x86_mmx %a1, x86_mmx* %a2) optsize {
-; GENERIC-LABEL: test_psubsw:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    psubsw %mm1, %mm0 # sched: [3:1.00]
-; GENERIC-NEXT:    psubsw (%rdi), %mm0 # sched: [8:1.00]
-; GENERIC-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_psubsw:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    psubsw %mm1, %mm0 # sched: [1:0.50]
-; ATOM-NEXT:    psubsw (%rdi), %mm0 # sched: [1:1.00]
-; ATOM-NEXT:    movq %mm0, %rax # sched: [3:3.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_psubsw:
-; SLM:       # %bb.0:
-; SLM-NEXT:    psubsw %mm1, %mm0 # sched: [1:0.50]
-; SLM-NEXT:    psubsw (%rdi), %mm0 # sched: [4:1.00]
-; SLM-NEXT:    movq %mm0, %rax # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_psubsw:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    psubsw %mm1, %mm0 # sched: [3:1.00]
-; SANDY-NEXT:    psubsw (%rdi), %mm0 # sched: [8:1.00]
-; SANDY-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_psubsw:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    psubsw %mm1, %mm0 # sched: [1:0.50]
-; HASWELL-NEXT:    psubsw (%rdi), %mm0 # sched: [6:0.50]
-; HASWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_psubsw:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    psubsw %mm1, %mm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    psubsw (%rdi), %mm0 # sched: [6:0.50]
-; BROADWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_psubsw:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    psubsw %mm1, %mm0 # sched: [1:1.00]
-; SKYLAKE-NEXT:    psubsw (%rdi), %mm0 # sched: [6:1.00]
-; SKYLAKE-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_psubsw:
-; SKX:       # %bb.0:
-; SKX-NEXT:    psubsw %mm1, %mm0 # sched: [1:1.00]
-; SKX-NEXT:    psubsw (%rdi), %mm0 # sched: [6:1.00]
-; SKX-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_psubsw:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    psubsw %mm1, %mm0 # sched: [2:0.50]
-; BDVER2-NEXT:    psubsw (%rdi), %mm0 # sched: [7:0.50]
-; BDVER2-NEXT:    movq %mm0, %rax # sched: [10:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_psubsw:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    psubsw %mm1, %mm0 # sched: [1:0.50]
-; BTVER2-NEXT:    psubsw (%rdi), %mm0 # sched: [6:1.00]
-; BTVER2-NEXT:    movq %mm0, %rax # sched: [4:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_psubsw:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    psubsw %mm1, %mm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    psubsw (%rdi), %mm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call x86_mmx @llvm.x86.mmx.psubs.w(x86_mmx %a0, x86_mmx %a1)
-  %2 = load x86_mmx, x86_mmx *%a2, align 8
-  %3 = call x86_mmx @llvm.x86.mmx.psubs.w(x86_mmx %1, x86_mmx %2)
-  %4 = bitcast x86_mmx %3 to i64
-  ret i64 %4
-}
-declare x86_mmx @llvm.x86.mmx.psubs.w(x86_mmx, x86_mmx) nounwind readnone
-
-define i64 @test_psubusb(x86_mmx %a0, x86_mmx %a1, x86_mmx* %a2) optsize {
-; GENERIC-LABEL: test_psubusb:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    psubusb %mm1, %mm0 # sched: [3:1.00]
-; GENERIC-NEXT:    psubusb (%rdi), %mm0 # sched: [8:1.00]
-; GENERIC-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_psubusb:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    psubusb %mm1, %mm0 # sched: [1:0.50]
-; ATOM-NEXT:    psubusb (%rdi), %mm0 # sched: [1:1.00]
-; ATOM-NEXT:    movq %mm0, %rax # sched: [3:3.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_psubusb:
-; SLM:       # %bb.0:
-; SLM-NEXT:    psubusb %mm1, %mm0 # sched: [1:0.50]
-; SLM-NEXT:    psubusb (%rdi), %mm0 # sched: [4:1.00]
-; SLM-NEXT:    movq %mm0, %rax # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_psubusb:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    psubusb %mm1, %mm0 # sched: [3:1.00]
-; SANDY-NEXT:    psubusb (%rdi), %mm0 # sched: [8:1.00]
-; SANDY-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_psubusb:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    psubusb %mm1, %mm0 # sched: [1:0.50]
-; HASWELL-NEXT:    psubusb (%rdi), %mm0 # sched: [6:0.50]
-; HASWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_psubusb:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    psubusb %mm1, %mm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    psubusb (%rdi), %mm0 # sched: [6:0.50]
-; BROADWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_psubusb:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    psubusb %mm1, %mm0 # sched: [1:1.00]
-; SKYLAKE-NEXT:    psubusb (%rdi), %mm0 # sched: [6:1.00]
-; SKYLAKE-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_psubusb:
-; SKX:       # %bb.0:
-; SKX-NEXT:    psubusb %mm1, %mm0 # sched: [1:1.00]
-; SKX-NEXT:    psubusb (%rdi), %mm0 # sched: [6:1.00]
-; SKX-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_psubusb:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    psubusb %mm1, %mm0 # sched: [2:0.50]
-; BDVER2-NEXT:    psubusb (%rdi), %mm0 # sched: [7:0.50]
-; BDVER2-NEXT:    movq %mm0, %rax # sched: [10:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_psubusb:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    psubusb %mm1, %mm0 # sched: [1:0.50]
-; BTVER2-NEXT:    psubusb (%rdi), %mm0 # sched: [6:1.00]
-; BTVER2-NEXT:    movq %mm0, %rax # sched: [4:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_psubusb:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    psubusb %mm1, %mm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    psubusb (%rdi), %mm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call x86_mmx @llvm.x86.mmx.psubus.b(x86_mmx %a0, x86_mmx %a1)
-  %2 = load x86_mmx, x86_mmx *%a2, align 8
-  %3 = call x86_mmx @llvm.x86.mmx.psubus.b(x86_mmx %1, x86_mmx %2)
-  %4 = bitcast x86_mmx %3 to i64
-  ret i64 %4
-}
-declare x86_mmx @llvm.x86.mmx.psubus.b(x86_mmx, x86_mmx) nounwind readnone
-
-define i64 @test_psubusw(x86_mmx %a0, x86_mmx %a1, x86_mmx* %a2) optsize {
-; GENERIC-LABEL: test_psubusw:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    psubusw %mm1, %mm0 # sched: [3:1.00]
-; GENERIC-NEXT:    psubusw (%rdi), %mm0 # sched: [8:1.00]
-; GENERIC-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_psubusw:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    psubusw %mm1, %mm0 # sched: [1:0.50]
-; ATOM-NEXT:    psubusw (%rdi), %mm0 # sched: [1:1.00]
-; ATOM-NEXT:    movq %mm0, %rax # sched: [3:3.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_psubusw:
-; SLM:       # %bb.0:
-; SLM-NEXT:    psubusw %mm1, %mm0 # sched: [1:0.50]
-; SLM-NEXT:    psubusw (%rdi), %mm0 # sched: [4:1.00]
-; SLM-NEXT:    movq %mm0, %rax # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_psubusw:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    psubusw %mm1, %mm0 # sched: [3:1.00]
-; SANDY-NEXT:    psubusw (%rdi), %mm0 # sched: [8:1.00]
-; SANDY-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_psubusw:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    psubusw %mm1, %mm0 # sched: [1:0.50]
-; HASWELL-NEXT:    psubusw (%rdi), %mm0 # sched: [6:0.50]
-; HASWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_psubusw:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    psubusw %mm1, %mm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    psubusw (%rdi), %mm0 # sched: [6:0.50]
-; BROADWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_psubusw:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    psubusw %mm1, %mm0 # sched: [1:1.00]
-; SKYLAKE-NEXT:    psubusw (%rdi), %mm0 # sched: [6:1.00]
-; SKYLAKE-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_psubusw:
-; SKX:       # %bb.0:
-; SKX-NEXT:    psubusw %mm1, %mm0 # sched: [1:1.00]
-; SKX-NEXT:    psubusw (%rdi), %mm0 # sched: [6:1.00]
-; SKX-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_psubusw:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    psubusw %mm1, %mm0 # sched: [2:0.50]
-; BDVER2-NEXT:    psubusw (%rdi), %mm0 # sched: [7:0.50]
-; BDVER2-NEXT:    movq %mm0, %rax # sched: [10:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_psubusw:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    psubusw %mm1, %mm0 # sched: [1:0.50]
-; BTVER2-NEXT:    psubusw (%rdi), %mm0 # sched: [6:1.00]
-; BTVER2-NEXT:    movq %mm0, %rax # sched: [4:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_psubusw:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    psubusw %mm1, %mm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    psubusw (%rdi), %mm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call x86_mmx @llvm.x86.mmx.psubus.w(x86_mmx %a0, x86_mmx %a1)
-  %2 = load x86_mmx, x86_mmx *%a2, align 8
-  %3 = call x86_mmx @llvm.x86.mmx.psubus.w(x86_mmx %1, x86_mmx %2)
-  %4 = bitcast x86_mmx %3 to i64
-  ret i64 %4
-}
-declare x86_mmx @llvm.x86.mmx.psubus.w(x86_mmx, x86_mmx) nounwind readnone
-
-define i64 @test_psubw(x86_mmx %a0, x86_mmx %a1, x86_mmx* %a2) optsize {
-; GENERIC-LABEL: test_psubw:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    psubw %mm1, %mm0 # sched: [3:1.00]
-; GENERIC-NEXT:    psubw (%rdi), %mm0 # sched: [8:1.00]
-; GENERIC-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_psubw:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    psubw %mm1, %mm0 # sched: [1:0.50]
-; ATOM-NEXT:    psubw (%rdi), %mm0 # sched: [1:1.00]
-; ATOM-NEXT:    movq %mm0, %rax # sched: [3:3.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_psubw:
-; SLM:       # %bb.0:
-; SLM-NEXT:    psubw %mm1, %mm0 # sched: [1:0.50]
-; SLM-NEXT:    psubw (%rdi), %mm0 # sched: [4:1.00]
-; SLM-NEXT:    movq %mm0, %rax # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_psubw:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    psubw %mm1, %mm0 # sched: [3:1.00]
-; SANDY-NEXT:    psubw (%rdi), %mm0 # sched: [8:1.00]
-; SANDY-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_psubw:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    psubw %mm1, %mm0 # sched: [1:0.50]
-; HASWELL-NEXT:    psubw (%rdi), %mm0 # sched: [6:0.50]
-; HASWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_psubw:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    psubw %mm1, %mm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    psubw (%rdi), %mm0 # sched: [6:0.50]
-; BROADWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_psubw:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    psubw %mm1, %mm0 # sched: [1:0.50]
-; SKYLAKE-NEXT:    psubw (%rdi), %mm0 # sched: [6:0.50]
-; SKYLAKE-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_psubw:
-; SKX:       # %bb.0:
-; SKX-NEXT:    psubw %mm1, %mm0 # sched: [1:0.50]
-; SKX-NEXT:    psubw (%rdi), %mm0 # sched: [6:0.50]
-; SKX-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_psubw:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    psubw %mm1, %mm0 # sched: [2:0.50]
-; BDVER2-NEXT:    psubw (%rdi), %mm0 # sched: [7:0.50]
-; BDVER2-NEXT:    movq %mm0, %rax # sched: [10:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_psubw:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    psubw %mm1, %mm0 # sched: [1:0.50]
-; BTVER2-NEXT:    psubw (%rdi), %mm0 # sched: [6:1.00]
-; BTVER2-NEXT:    movq %mm0, %rax # sched: [4:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_psubw:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    psubw %mm1, %mm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    psubw (%rdi), %mm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call x86_mmx @llvm.x86.mmx.psub.w(x86_mmx %a0, x86_mmx %a1)
-  %2 = load x86_mmx, x86_mmx *%a2, align 8
-  %3 = call x86_mmx @llvm.x86.mmx.psub.w(x86_mmx %1, x86_mmx %2)
-  %4 = bitcast x86_mmx %3 to i64
-  ret i64 %4
-}
-declare x86_mmx @llvm.x86.mmx.psub.w(x86_mmx, x86_mmx) nounwind readnone
-
-define i64 @test_punpckhbw(x86_mmx %a0, x86_mmx %a1, x86_mmx* %a2) optsize {
-; GENERIC-LABEL: test_punpckhbw:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    punpckhbw %mm1, %mm0 # mm0 = mm0[4],mm1[4],mm0[5],mm1[5],mm0[6],mm1[6],mm0[7],mm1[7] sched: [1:1.00]
-; GENERIC-NEXT:    punpckhbw (%rdi), %mm0 # mm0 = mm0[4],mem[4],mm0[5],mem[5],mm0[6],mem[6],mm0[7],mem[7] sched: [6:1.00]
-; GENERIC-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_punpckhbw:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    punpckhbw %mm1, %mm0 # mm0 = mm0[4],mm1[4],mm0[5],mm1[5],mm0[6],mm1[6],mm0[7],mm1[7] sched: [1:0.50]
-; ATOM-NEXT:    punpckhbw (%rdi), %mm0 # mm0 = mm0[4],mem[4],mm0[5],mem[5],mm0[6],mem[6],mm0[7],mem[7] sched: [1:1.00]
-; ATOM-NEXT:    movq %mm0, %rax # sched: [3:3.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_punpckhbw:
-; SLM:       # %bb.0:
-; SLM-NEXT:    punpckhbw %mm1, %mm0 # mm0 = mm0[4],mm1[4],mm0[5],mm1[5],mm0[6],mm1[6],mm0[7],mm1[7] sched: [1:1.00]
-; SLM-NEXT:    punpckhbw (%rdi), %mm0 # mm0 = mm0[4],mem[4],mm0[5],mem[5],mm0[6],mem[6],mm0[7],mem[7] sched: [4:1.00]
-; SLM-NEXT:    movq %mm0, %rax # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_punpckhbw:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    punpckhbw %mm1, %mm0 # mm0 = mm0[4],mm1[4],mm0[5],mm1[5],mm0[6],mm1[6],mm0[7],mm1[7] sched: [1:1.00]
-; SANDY-NEXT:    punpckhbw (%rdi), %mm0 # mm0 = mm0[4],mem[4],mm0[5],mem[5],mm0[6],mem[6],mm0[7],mem[7] sched: [6:1.00]
-; SANDY-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_punpckhbw:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    punpckhbw %mm1, %mm0 # mm0 = mm0[4],mm1[4],mm0[5],mm1[5],mm0[6],mm1[6],mm0[7],mm1[7] sched: [1:1.00]
-; HASWELL-NEXT:    punpckhbw (%rdi), %mm0 # mm0 = mm0[4],mem[4],mm0[5],mem[5],mm0[6],mem[6],mm0[7],mem[7] sched: [6:1.00]
-; HASWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_punpckhbw:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    punpckhbw %mm1, %mm0 # mm0 = mm0[4],mm1[4],mm0[5],mm1[5],mm0[6],mm1[6],mm0[7],mm1[7] sched: [1:1.00]
-; BROADWELL-NEXT:    punpckhbw (%rdi), %mm0 # mm0 = mm0[4],mem[4],mm0[5],mem[5],mm0[6],mem[6],mm0[7],mem[7] sched: [6:1.00]
-; BROADWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_punpckhbw:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    punpckhbw %mm1, %mm0 # mm0 = mm0[4],mm1[4],mm0[5],mm1[5],mm0[6],mm1[6],mm0[7],mm1[7] sched: [1:1.00]
-; SKYLAKE-NEXT:    punpckhbw (%rdi), %mm0 # mm0 = mm0[4],mem[4],mm0[5],mem[5],mm0[6],mem[6],mm0[7],mem[7] sched: [6:1.00]
-; SKYLAKE-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_punpckhbw:
-; SKX:       # %bb.0:
-; SKX-NEXT:    punpckhbw %mm1, %mm0 # mm0 = mm0[4],mm1[4],mm0[5],mm1[5],mm0[6],mm1[6],mm0[7],mm1[7] sched: [1:1.00]
-; SKX-NEXT:    punpckhbw (%rdi), %mm0 # mm0 = mm0[4],mem[4],mm0[5],mem[5],mm0[6],mem[6],mm0[7],mem[7] sched: [6:1.00]
-; SKX-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_punpckhbw:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    punpckhbw %mm1, %mm0 # mm0 = mm0[4],mm1[4],mm0[5],mm1[5],mm0[6],mm1[6],mm0[7],mm1[7] sched: [2:0.50]
-; BDVER2-NEXT:    punpckhbw (%rdi), %mm0 # mm0 = mm0[4],mem[4],mm0[5],mem[5],mm0[6],mem[6],mm0[7],mem[7] sched: [7:0.50]
-; BDVER2-NEXT:    movq %mm0, %rax # sched: [10:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_punpckhbw:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    punpckhbw %mm1, %mm0 # mm0 = mm0[4],mm1[4],mm0[5],mm1[5],mm0[6],mm1[6],mm0[7],mm1[7] sched: [1:0.50]
-; BTVER2-NEXT:    punpckhbw (%rdi), %mm0 # mm0 = mm0[4],mem[4],mm0[5],mem[5],mm0[6],mem[6],mm0[7],mem[7] sched: [6:1.00]
-; BTVER2-NEXT:    movq %mm0, %rax # sched: [4:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_punpckhbw:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    punpckhbw %mm1, %mm0 # mm0 = mm0[4],mm1[4],mm0[5],mm1[5],mm0[6],mm1[6],mm0[7],mm1[7] sched: [1:0.25]
-; ZNVER1-NEXT:    punpckhbw (%rdi), %mm0 # mm0 = mm0[4],mem[4],mm0[5],mem[5],mm0[6],mem[6],mm0[7],mem[7] sched: [8:0.50]
-; ZNVER1-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call x86_mmx @llvm.x86.mmx.punpckhbw(x86_mmx %a0, x86_mmx %a1)
-  %2 = load x86_mmx, x86_mmx *%a2, align 8
-  %3 = call x86_mmx @llvm.x86.mmx.punpckhbw(x86_mmx %1, x86_mmx %2)
-  %4 = bitcast x86_mmx %3 to i64
-  ret i64 %4
-}
-declare x86_mmx @llvm.x86.mmx.punpckhbw(x86_mmx, x86_mmx) nounwind readnone
-
-define i64 @test_punpckhdq(x86_mmx %a0, x86_mmx %a1, x86_mmx* %a2) optsize {
-; GENERIC-LABEL: test_punpckhdq:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    punpckhdq %mm1, %mm0 # mm0 = mm0[1],mm1[1] sched: [1:1.00]
-; GENERIC-NEXT:    punpckhdq (%rdi), %mm0 # mm0 = mm0[1],mem[1] sched: [6:1.00]
-; GENERIC-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_punpckhdq:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    punpckhdq %mm1, %mm0 # mm0 = mm0[1],mm1[1] sched: [1:0.50]
-; ATOM-NEXT:    punpckhdq (%rdi), %mm0 # mm0 = mm0[1],mem[1] sched: [1:1.00]
-; ATOM-NEXT:    movq %mm0, %rax # sched: [3:3.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_punpckhdq:
-; SLM:       # %bb.0:
-; SLM-NEXT:    punpckhdq %mm1, %mm0 # mm0 = mm0[1],mm1[1] sched: [1:1.00]
-; SLM-NEXT:    punpckhdq (%rdi), %mm0 # mm0 = mm0[1],mem[1] sched: [4:1.00]
-; SLM-NEXT:    movq %mm0, %rax # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_punpckhdq:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    punpckhdq %mm1, %mm0 # mm0 = mm0[1],mm1[1] sched: [1:1.00]
-; SANDY-NEXT:    punpckhdq (%rdi), %mm0 # mm0 = mm0[1],mem[1] sched: [6:1.00]
-; SANDY-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_punpckhdq:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    punpckhdq %mm1, %mm0 # mm0 = mm0[1],mm1[1] sched: [1:1.00]
-; HASWELL-NEXT:    punpckhdq (%rdi), %mm0 # mm0 = mm0[1],mem[1] sched: [6:1.00]
-; HASWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_punpckhdq:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    punpckhdq %mm1, %mm0 # mm0 = mm0[1],mm1[1] sched: [1:1.00]
-; BROADWELL-NEXT:    punpckhdq (%rdi), %mm0 # mm0 = mm0[1],mem[1] sched: [6:1.00]
-; BROADWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_punpckhdq:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    punpckhdq %mm1, %mm0 # mm0 = mm0[1],mm1[1] sched: [1:1.00]
-; SKYLAKE-NEXT:    punpckhdq (%rdi), %mm0 # mm0 = mm0[1],mem[1] sched: [6:1.00]
-; SKYLAKE-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_punpckhdq:
-; SKX:       # %bb.0:
-; SKX-NEXT:    punpckhdq %mm1, %mm0 # mm0 = mm0[1],mm1[1] sched: [1:1.00]
-; SKX-NEXT:    punpckhdq (%rdi), %mm0 # mm0 = mm0[1],mem[1] sched: [6:1.00]
-; SKX-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_punpckhdq:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    punpckhdq %mm1, %mm0 # mm0 = mm0[1],mm1[1] sched: [2:0.50]
-; BDVER2-NEXT:    punpckhdq (%rdi), %mm0 # mm0 = mm0[1],mem[1] sched: [7:0.50]
-; BDVER2-NEXT:    movq %mm0, %rax # sched: [10:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_punpckhdq:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    punpckhdq %mm1, %mm0 # mm0 = mm0[1],mm1[1] sched: [1:0.50]
-; BTVER2-NEXT:    punpckhdq (%rdi), %mm0 # mm0 = mm0[1],mem[1] sched: [6:1.00]
-; BTVER2-NEXT:    movq %mm0, %rax # sched: [4:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_punpckhdq:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    punpckhdq %mm1, %mm0 # mm0 = mm0[1],mm1[1] sched: [1:0.25]
-; ZNVER1-NEXT:    punpckhdq (%rdi), %mm0 # mm0 = mm0[1],mem[1] sched: [8:0.50]
-; ZNVER1-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call x86_mmx @llvm.x86.mmx.punpckhdq(x86_mmx %a0, x86_mmx %a1)
-  %2 = load x86_mmx, x86_mmx *%a2, align 8
-  %3 = call x86_mmx @llvm.x86.mmx.punpckhdq(x86_mmx %1, x86_mmx %2)
-  %4 = bitcast x86_mmx %3 to i64
-  ret i64 %4
-}
-declare x86_mmx @llvm.x86.mmx.punpckhdq(x86_mmx, x86_mmx) nounwind readnone
-
-define i64 @test_punpckhwd(x86_mmx %a0, x86_mmx %a1, x86_mmx* %a2) optsize {
-; GENERIC-LABEL: test_punpckhwd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    punpckhwd %mm1, %mm0 # mm0 = mm0[2],mm1[2],mm0[3],mm1[3] sched: [1:1.00]
-; GENERIC-NEXT:    punpckhwd (%rdi), %mm0 # mm0 = mm0[2],mem[2],mm0[3],mem[3] sched: [6:1.00]
-; GENERIC-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_punpckhwd:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    punpckhwd %mm1, %mm0 # mm0 = mm0[2],mm1[2],mm0[3],mm1[3] sched: [1:0.50]
-; ATOM-NEXT:    punpckhwd (%rdi), %mm0 # mm0 = mm0[2],mem[2],mm0[3],mem[3] sched: [1:1.00]
-; ATOM-NEXT:    movq %mm0, %rax # sched: [3:3.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_punpckhwd:
-; SLM:       # %bb.0:
-; SLM-NEXT:    punpckhwd %mm1, %mm0 # mm0 = mm0[2],mm1[2],mm0[3],mm1[3] sched: [1:1.00]
-; SLM-NEXT:    punpckhwd (%rdi), %mm0 # mm0 = mm0[2],mem[2],mm0[3],mem[3] sched: [4:1.00]
-; SLM-NEXT:    movq %mm0, %rax # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_punpckhwd:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    punpckhwd %mm1, %mm0 # mm0 = mm0[2],mm1[2],mm0[3],mm1[3] sched: [1:1.00]
-; SANDY-NEXT:    punpckhwd (%rdi), %mm0 # mm0 = mm0[2],mem[2],mm0[3],mem[3] sched: [6:1.00]
-; SANDY-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_punpckhwd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    punpckhwd %mm1, %mm0 # mm0 = mm0[2],mm1[2],mm0[3],mm1[3] sched: [1:1.00]
-; HASWELL-NEXT:    punpckhwd (%rdi), %mm0 # mm0 = mm0[2],mem[2],mm0[3],mem[3] sched: [6:1.00]
-; HASWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_punpckhwd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    punpckhwd %mm1, %mm0 # mm0 = mm0[2],mm1[2],mm0[3],mm1[3] sched: [1:1.00]
-; BROADWELL-NEXT:    punpckhwd (%rdi), %mm0 # mm0 = mm0[2],mem[2],mm0[3],mem[3] sched: [6:1.00]
-; BROADWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_punpckhwd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    punpckhwd %mm1, %mm0 # mm0 = mm0[2],mm1[2],mm0[3],mm1[3] sched: [1:1.00]
-; SKYLAKE-NEXT:    punpckhwd (%rdi), %mm0 # mm0 = mm0[2],mem[2],mm0[3],mem[3] sched: [6:1.00]
-; SKYLAKE-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_punpckhwd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    punpckhwd %mm1, %mm0 # mm0 = mm0[2],mm1[2],mm0[3],mm1[3] sched: [1:1.00]
-; SKX-NEXT:    punpckhwd (%rdi), %mm0 # mm0 = mm0[2],mem[2],mm0[3],mem[3] sched: [6:1.00]
-; SKX-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_punpckhwd:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    punpckhwd %mm1, %mm0 # mm0 = mm0[2],mm1[2],mm0[3],mm1[3] sched: [2:0.50]
-; BDVER2-NEXT:    punpckhwd (%rdi), %mm0 # mm0 = mm0[2],mem[2],mm0[3],mem[3] sched: [7:0.50]
-; BDVER2-NEXT:    movq %mm0, %rax # sched: [10:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_punpckhwd:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    punpckhwd %mm1, %mm0 # mm0 = mm0[2],mm1[2],mm0[3],mm1[3] sched: [1:0.50]
-; BTVER2-NEXT:    punpckhwd (%rdi), %mm0 # mm0 = mm0[2],mem[2],mm0[3],mem[3] sched: [6:1.00]
-; BTVER2-NEXT:    movq %mm0, %rax # sched: [4:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_punpckhwd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    punpckhwd %mm1, %mm0 # mm0 = mm0[2],mm1[2],mm0[3],mm1[3] sched: [1:0.25]
-; ZNVER1-NEXT:    punpckhwd (%rdi), %mm0 # mm0 = mm0[2],mem[2],mm0[3],mem[3] sched: [8:0.50]
-; ZNVER1-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call x86_mmx @llvm.x86.mmx.punpckhwd(x86_mmx %a0, x86_mmx %a1)
-  %2 = load x86_mmx, x86_mmx *%a2, align 8
-  %3 = call x86_mmx @llvm.x86.mmx.punpckhwd(x86_mmx %1, x86_mmx %2)
-  %4 = bitcast x86_mmx %3 to i64
-  ret i64 %4
-}
-declare x86_mmx @llvm.x86.mmx.punpckhwd(x86_mmx, x86_mmx) nounwind readnone
-
-define i64 @test_punpcklbw(x86_mmx %a0, x86_mmx %a1, x86_mmx* %a2) optsize {
-; GENERIC-LABEL: test_punpcklbw:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    punpcklbw %mm1, %mm0 # mm0 = mm0[0],mm1[0],mm0[1],mm1[1],mm0[2],mm1[2],mm0[3],mm1[3] sched: [1:1.00]
-; GENERIC-NEXT:    punpcklbw (%rdi), %mm0 # mm0 = mm0[0],mem[0],mm0[1],mem[1],mm0[2],mem[2],mm0[3],mem[3] sched: [6:1.00]
-; GENERIC-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_punpcklbw:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    punpcklbw %mm1, %mm0 # mm0 = mm0[0],mm1[0],mm0[1],mm1[1],mm0[2],mm1[2],mm0[3],mm1[3] sched: [1:1.00]
-; ATOM-NEXT:    punpcklbw (%rdi), %mm0 # mm0 = mm0[0],mem[0],mm0[1],mem[1],mm0[2],mem[2],mm0[3],mem[3] sched: [1:1.00]
-; ATOM-NEXT:    movq %mm0, %rax # sched: [3:3.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_punpcklbw:
-; SLM:       # %bb.0:
-; SLM-NEXT:    punpcklbw %mm1, %mm0 # mm0 = mm0[0],mm1[0],mm0[1],mm1[1],mm0[2],mm1[2],mm0[3],mm1[3] sched: [1:1.00]
-; SLM-NEXT:    punpcklbw (%rdi), %mm0 # mm0 = mm0[0],mem[0],mm0[1],mem[1],mm0[2],mem[2],mm0[3],mem[3] sched: [4:1.00]
-; SLM-NEXT:    movq %mm0, %rax # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_punpcklbw:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    punpcklbw %mm1, %mm0 # mm0 = mm0[0],mm1[0],mm0[1],mm1[1],mm0[2],mm1[2],mm0[3],mm1[3] sched: [1:1.00]
-; SANDY-NEXT:    punpcklbw (%rdi), %mm0 # mm0 = mm0[0],mem[0],mm0[1],mem[1],mm0[2],mem[2],mm0[3],mem[3] sched: [6:1.00]
-; SANDY-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_punpcklbw:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    punpcklbw %mm1, %mm0 # mm0 = mm0[0],mm1[0],mm0[1],mm1[1],mm0[2],mm1[2],mm0[3],mm1[3] sched: [1:1.00]
-; HASWELL-NEXT:    punpcklbw (%rdi), %mm0 # mm0 = mm0[0],mem[0],mm0[1],mem[1],mm0[2],mem[2],mm0[3],mem[3] sched: [6:1.00]
-; HASWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_punpcklbw:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    punpcklbw %mm1, %mm0 # mm0 = mm0[0],mm1[0],mm0[1],mm1[1],mm0[2],mm1[2],mm0[3],mm1[3] sched: [1:1.00]
-; BROADWELL-NEXT:    punpcklbw (%rdi), %mm0 # mm0 = mm0[0],mem[0],mm0[1],mem[1],mm0[2],mem[2],mm0[3],mem[3] sched: [6:1.00]
-; BROADWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_punpcklbw:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    punpcklbw %mm1, %mm0 # mm0 = mm0[0],mm1[0],mm0[1],mm1[1],mm0[2],mm1[2],mm0[3],mm1[3] sched: [1:1.00]
-; SKYLAKE-NEXT:    punpcklbw (%rdi), %mm0 # mm0 = mm0[0],mem[0],mm0[1],mem[1],mm0[2],mem[2],mm0[3],mem[3] sched: [6:1.00]
-; SKYLAKE-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_punpcklbw:
-; SKX:       # %bb.0:
-; SKX-NEXT:    punpcklbw %mm1, %mm0 # mm0 = mm0[0],mm1[0],mm0[1],mm1[1],mm0[2],mm1[2],mm0[3],mm1[3] sched: [1:1.00]
-; SKX-NEXT:    punpcklbw (%rdi), %mm0 # mm0 = mm0[0],mem[0],mm0[1],mem[1],mm0[2],mem[2],mm0[3],mem[3] sched: [6:1.00]
-; SKX-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_punpcklbw:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    punpcklbw %mm1, %mm0 # mm0 = mm0[0],mm1[0],mm0[1],mm1[1],mm0[2],mm1[2],mm0[3],mm1[3] sched: [2:0.50]
-; BDVER2-NEXT:    punpcklbw (%rdi), %mm0 # mm0 = mm0[0],mem[0],mm0[1],mem[1],mm0[2],mem[2],mm0[3],mem[3] sched: [7:0.50]
-; BDVER2-NEXT:    movq %mm0, %rax # sched: [10:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_punpcklbw:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    punpcklbw %mm1, %mm0 # mm0 = mm0[0],mm1[0],mm0[1],mm1[1],mm0[2],mm1[2],mm0[3],mm1[3] sched: [1:0.50]
-; BTVER2-NEXT:    punpcklbw (%rdi), %mm0 # mm0 = mm0[0],mem[0],mm0[1],mem[1],mm0[2],mem[2],mm0[3],mem[3] sched: [6:1.00]
-; BTVER2-NEXT:    movq %mm0, %rax # sched: [4:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_punpcklbw:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    punpcklbw %mm1, %mm0 # mm0 = mm0[0],mm1[0],mm0[1],mm1[1],mm0[2],mm1[2],mm0[3],mm1[3] sched: [1:0.25]
-; ZNVER1-NEXT:    punpcklbw (%rdi), %mm0 # mm0 = mm0[0],mem[0],mm0[1],mem[1],mm0[2],mem[2],mm0[3],mem[3] sched: [8:0.50]
-; ZNVER1-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call x86_mmx @llvm.x86.mmx.punpcklbw(x86_mmx %a0, x86_mmx %a1)
-  %2 = load x86_mmx, x86_mmx *%a2, align 8
-  %3 = call x86_mmx @llvm.x86.mmx.punpcklbw(x86_mmx %1, x86_mmx %2)
-  %4 = bitcast x86_mmx %3 to i64
-  ret i64 %4
-}
-declare x86_mmx @llvm.x86.mmx.punpcklbw(x86_mmx, x86_mmx) nounwind readnone
-
-define i64 @test_punpckldq(x86_mmx %a0, x86_mmx %a1, x86_mmx* %a2) optsize {
-; GENERIC-LABEL: test_punpckldq:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    punpckldq %mm1, %mm0 # mm0 = mm0[0],mm1[0] sched: [1:1.00]
-; GENERIC-NEXT:    punpckldq (%rdi), %mm0 # mm0 = mm0[0],mem[0] sched: [6:1.00]
-; GENERIC-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_punpckldq:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    punpckldq %mm1, %mm0 # mm0 = mm0[0],mm1[0] sched: [1:1.00]
-; ATOM-NEXT:    punpckldq (%rdi), %mm0 # mm0 = mm0[0],mem[0] sched: [1:1.00]
-; ATOM-NEXT:    movq %mm0, %rax # sched: [3:3.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_punpckldq:
-; SLM:       # %bb.0:
-; SLM-NEXT:    punpckldq %mm1, %mm0 # mm0 = mm0[0],mm1[0] sched: [1:1.00]
-; SLM-NEXT:    punpckldq (%rdi), %mm0 # mm0 = mm0[0],mem[0] sched: [4:1.00]
-; SLM-NEXT:    movq %mm0, %rax # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_punpckldq:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    punpckldq %mm1, %mm0 # mm0 = mm0[0],mm1[0] sched: [1:1.00]
-; SANDY-NEXT:    punpckldq (%rdi), %mm0 # mm0 = mm0[0],mem[0] sched: [6:1.00]
-; SANDY-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_punpckldq:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    punpckldq %mm1, %mm0 # mm0 = mm0[0],mm1[0] sched: [1:1.00]
-; HASWELL-NEXT:    punpckldq (%rdi), %mm0 # mm0 = mm0[0],mem[0] sched: [6:1.00]
-; HASWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_punpckldq:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    punpckldq %mm1, %mm0 # mm0 = mm0[0],mm1[0] sched: [1:1.00]
-; BROADWELL-NEXT:    punpckldq (%rdi), %mm0 # mm0 = mm0[0],mem[0] sched: [6:1.00]
-; BROADWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_punpckldq:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    punpckldq %mm1, %mm0 # mm0 = mm0[0],mm1[0] sched: [1:1.00]
-; SKYLAKE-NEXT:    punpckldq (%rdi), %mm0 # mm0 = mm0[0],mem[0] sched: [6:1.00]
-; SKYLAKE-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_punpckldq:
-; SKX:       # %bb.0:
-; SKX-NEXT:    punpckldq %mm1, %mm0 # mm0 = mm0[0],mm1[0] sched: [1:1.00]
-; SKX-NEXT:    punpckldq (%rdi), %mm0 # mm0 = mm0[0],mem[0] sched: [6:1.00]
-; SKX-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_punpckldq:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    punpckldq %mm1, %mm0 # mm0 = mm0[0],mm1[0] sched: [2:0.50]
-; BDVER2-NEXT:    punpckldq (%rdi), %mm0 # mm0 = mm0[0],mem[0] sched: [7:0.50]
-; BDVER2-NEXT:    movq %mm0, %rax # sched: [10:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_punpckldq:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    punpckldq %mm1, %mm0 # mm0 = mm0[0],mm1[0] sched: [1:0.50]
-; BTVER2-NEXT:    punpckldq (%rdi), %mm0 # mm0 = mm0[0],mem[0] sched: [6:1.00]
-; BTVER2-NEXT:    movq %mm0, %rax # sched: [4:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_punpckldq:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    punpckldq %mm1, %mm0 # mm0 = mm0[0],mm1[0] sched: [1:0.25]
-; ZNVER1-NEXT:    punpckldq (%rdi), %mm0 # mm0 = mm0[0],mem[0] sched: [8:0.50]
-; ZNVER1-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call x86_mmx @llvm.x86.mmx.punpckldq(x86_mmx %a0, x86_mmx %a1)
-  %2 = load x86_mmx, x86_mmx *%a2, align 8
-  %3 = call x86_mmx @llvm.x86.mmx.punpckldq(x86_mmx %1, x86_mmx %2)
-  %4 = bitcast x86_mmx %3 to i64
-  ret i64 %4
-}
-declare x86_mmx @llvm.x86.mmx.punpckldq(x86_mmx, x86_mmx) nounwind readnone
-
-define i64 @test_punpcklwd(x86_mmx %a0, x86_mmx %a1, x86_mmx* %a2) optsize {
-; GENERIC-LABEL: test_punpcklwd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    punpcklwd %mm1, %mm0 # mm0 = mm0[0],mm1[0],mm0[1],mm1[1] sched: [1:1.00]
-; GENERIC-NEXT:    punpcklwd (%rdi), %mm0 # mm0 = mm0[0],mem[0],mm0[1],mem[1] sched: [6:1.00]
-; GENERIC-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_punpcklwd:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    punpcklwd %mm1, %mm0 # mm0 = mm0[0],mm1[0],mm0[1],mm1[1] sched: [1:1.00]
-; ATOM-NEXT:    punpcklwd (%rdi), %mm0 # mm0 = mm0[0],mem[0],mm0[1],mem[1] sched: [1:1.00]
-; ATOM-NEXT:    movq %mm0, %rax # sched: [3:3.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_punpcklwd:
-; SLM:       # %bb.0:
-; SLM-NEXT:    punpcklwd %mm1, %mm0 # mm0 = mm0[0],mm1[0],mm0[1],mm1[1] sched: [1:1.00]
-; SLM-NEXT:    punpcklwd (%rdi), %mm0 # mm0 = mm0[0],mem[0],mm0[1],mem[1] sched: [4:1.00]
-; SLM-NEXT:    movq %mm0, %rax # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_punpcklwd:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    punpcklwd %mm1, %mm0 # mm0 = mm0[0],mm1[0],mm0[1],mm1[1] sched: [1:1.00]
-; SANDY-NEXT:    punpcklwd (%rdi), %mm0 # mm0 = mm0[0],mem[0],mm0[1],mem[1] sched: [6:1.00]
-; SANDY-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_punpcklwd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    punpcklwd %mm1, %mm0 # mm0 = mm0[0],mm1[0],mm0[1],mm1[1] sched: [1:1.00]
-; HASWELL-NEXT:    punpcklwd (%rdi), %mm0 # mm0 = mm0[0],mem[0],mm0[1],mem[1] sched: [6:1.00]
-; HASWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_punpcklwd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    punpcklwd %mm1, %mm0 # mm0 = mm0[0],mm1[0],mm0[1],mm1[1] sched: [1:1.00]
-; BROADWELL-NEXT:    punpcklwd (%rdi), %mm0 # mm0 = mm0[0],mem[0],mm0[1],mem[1] sched: [6:1.00]
-; BROADWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_punpcklwd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    punpcklwd %mm1, %mm0 # mm0 = mm0[0],mm1[0],mm0[1],mm1[1] sched: [1:1.00]
-; SKYLAKE-NEXT:    punpcklwd (%rdi), %mm0 # mm0 = mm0[0],mem[0],mm0[1],mem[1] sched: [6:1.00]
-; SKYLAKE-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_punpcklwd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    punpcklwd %mm1, %mm0 # mm0 = mm0[0],mm1[0],mm0[1],mm1[1] sched: [1:1.00]
-; SKX-NEXT:    punpcklwd (%rdi), %mm0 # mm0 = mm0[0],mem[0],mm0[1],mem[1] sched: [6:1.00]
-; SKX-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_punpcklwd:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    punpcklwd %mm1, %mm0 # mm0 = mm0[0],mm1[0],mm0[1],mm1[1] sched: [2:0.50]
-; BDVER2-NEXT:    punpcklwd (%rdi), %mm0 # mm0 = mm0[0],mem[0],mm0[1],mem[1] sched: [7:0.50]
-; BDVER2-NEXT:    movq %mm0, %rax # sched: [10:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_punpcklwd:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    punpcklwd %mm1, %mm0 # mm0 = mm0[0],mm1[0],mm0[1],mm1[1] sched: [1:0.50]
-; BTVER2-NEXT:    punpcklwd (%rdi), %mm0 # mm0 = mm0[0],mem[0],mm0[1],mem[1] sched: [6:1.00]
-; BTVER2-NEXT:    movq %mm0, %rax # sched: [4:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_punpcklwd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    punpcklwd %mm1, %mm0 # mm0 = mm0[0],mm1[0],mm0[1],mm1[1] sched: [1:0.25]
-; ZNVER1-NEXT:    punpcklwd (%rdi), %mm0 # mm0 = mm0[0],mem[0],mm0[1],mem[1] sched: [8:0.50]
-; ZNVER1-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call x86_mmx @llvm.x86.mmx.punpcklwd(x86_mmx %a0, x86_mmx %a1)
-  %2 = load x86_mmx, x86_mmx *%a2, align 8
-  %3 = call x86_mmx @llvm.x86.mmx.punpcklwd(x86_mmx %1, x86_mmx %2)
-  %4 = bitcast x86_mmx %3 to i64
-  ret i64 %4
-}
-declare x86_mmx @llvm.x86.mmx.punpcklwd(x86_mmx, x86_mmx) nounwind readnone
-
-define i64 @test_pxor(x86_mmx %a0, x86_mmx %a1, x86_mmx* %a2) optsize {
-; GENERIC-LABEL: test_pxor:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    pxor %mm1, %mm0 # sched: [1:0.33]
-; GENERIC-NEXT:    pxor (%rdi), %mm0 # sched: [6:0.50]
-; GENERIC-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_pxor:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    pxor %mm1, %mm0 # sched: [1:0.50]
-; ATOM-NEXT:    pxor (%rdi), %mm0 # sched: [1:1.00]
-; ATOM-NEXT:    movq %mm0, %rax # sched: [3:3.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_pxor:
-; SLM:       # %bb.0:
-; SLM-NEXT:    pxor %mm1, %mm0 # sched: [1:0.50]
-; SLM-NEXT:    pxor (%rdi), %mm0 # sched: [4:1.00]
-; SLM-NEXT:    movq %mm0, %rax # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_pxor:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    pxor %mm1, %mm0 # sched: [1:0.33]
-; SANDY-NEXT:    pxor (%rdi), %mm0 # sched: [6:0.50]
-; SANDY-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_pxor:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    pxor %mm1, %mm0 # sched: [1:0.33]
-; HASWELL-NEXT:    pxor (%rdi), %mm0 # sched: [6:0.50]
-; HASWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pxor:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    pxor %mm1, %mm0 # sched: [1:0.33]
-; BROADWELL-NEXT:    pxor (%rdi), %mm0 # sched: [6:0.50]
-; BROADWELL-NEXT:    movq %mm0, %rax # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pxor:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    pxor %mm1, %mm0 # sched: [1:0.50]
-; SKYLAKE-NEXT:    pxor (%rdi), %mm0 # sched: [6:0.50]
-; SKYLAKE-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pxor:
-; SKX:       # %bb.0:
-; SKX-NEXT:    pxor %mm1, %mm0 # sched: [1:0.50]
-; SKX-NEXT:    pxor (%rdi), %mm0 # sched: [6:0.50]
-; SKX-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_pxor:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    pxor %mm1, %mm0 # sched: [2:0.50]
-; BDVER2-NEXT:    pxor (%rdi), %mm0 # sched: [7:0.50]
-; BDVER2-NEXT:    movq %mm0, %rax # sched: [10:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_pxor:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    pxor %mm1, %mm0 # sched: [1:0.50]
-; BTVER2-NEXT:    pxor (%rdi), %mm0 # sched: [6:1.00]
-; BTVER2-NEXT:    movq %mm0, %rax # sched: [4:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_pxor:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    pxor %mm1, %mm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    pxor (%rdi), %mm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    movq %mm0, %rax # sched: [2:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call x86_mmx @llvm.x86.mmx.pxor(x86_mmx %a0, x86_mmx %a1)
-  %2 = load x86_mmx, x86_mmx *%a2, align 8
-  %3 = call x86_mmx @llvm.x86.mmx.pxor(x86_mmx %1, x86_mmx %2)
-  %4 = bitcast x86_mmx %3 to i64
-  ret i64 %4
-}
-declare x86_mmx @llvm.x86.mmx.pxor(x86_mmx, x86_mmx) nounwind readnone
diff --git a/test/CodeGen/X86/movbe-schedule.ll b/test/CodeGen/X86/movbe-schedule.ll
deleted file mode 100644
index c0b65f366220fa3132de8e208dea60230c54e3b6..0000000000000000000000000000000000000000
--- a/test/CodeGen/X86/movbe-schedule.ll
+++ /dev/null
@@ -1,190 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=x86-64 -mattr=+movbe | FileCheck %s --check-prefix=CHECK --check-prefix=GENERIC
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=atom | FileCheck %s --check-prefix=CHECK --check-prefix=ATOM
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=slm | FileCheck %s --check-prefix=CHECK --check-prefix=SLM
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=haswell | FileCheck %s --check-prefix=CHECK --check-prefix=HASWELL
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=broadwell | FileCheck %s --check-prefix=CHECK --check-prefix=BROADWELL
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=skylake | FileCheck %s --check-prefix=CHECK --check-prefix=SKYLAKE
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=knl | FileCheck %s --check-prefix=CHECK --check-prefix=HASWELL
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=btver2 | FileCheck %s --check-prefix=CHECK --check-prefix=BTVER2
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=znver1 | FileCheck %s --check-prefix=CHECK --check-prefix=ZNVER1
-
-define i16 @test_movbe_i16(i16 *%a0, i16 %a1, i16 *%a2) {
-; GENERIC-LABEL: test_movbe_i16:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    movbew (%rdi), %ax # sched: [6:0.50]
-; GENERIC-NEXT:    movbew %si, (%rdx) # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_movbe_i16:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    movbew (%rdi), %ax # sched: [1:1.00]
-; ATOM-NEXT:    movbew %si, (%rdx) # sched: [1:1.00]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_movbe_i16:
-; SLM:       # %bb.0:
-; SLM-NEXT:    movbew (%rdi), %ax # sched: [4:1.00]
-; SLM-NEXT:    movbew %si, (%rdx) # sched: [1:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; HASWELL-LABEL: test_movbe_i16:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    movbew (%rdi), %ax # sched: [6:0.50]
-; HASWELL-NEXT:    movbew %si, (%rdx) # sched: [2:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_movbe_i16:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    movbew (%rdi), %ax # sched: [6:0.50]
-; BROADWELL-NEXT:    movbew %si, (%rdx) # sched: [2:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_movbe_i16:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    movbew (%rdi), %ax # sched: [6:0.50]
-; SKYLAKE-NEXT:    movbew %si, (%rdx) # sched: [2:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; BTVER2-LABEL: test_movbe_i16:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    movbew (%rdi), %ax # sched: [4:1.00]
-; BTVER2-NEXT:    movbew %si, (%rdx) # sched: [1:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_movbe_i16:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    movbew (%rdi), %ax # sched: [5:0.50]
-; ZNVER1-NEXT:    movbew %si, (%rdx) # sched: [5:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = load i16, i16 *%a0
-  %2 = tail call i16 @llvm.bswap.i16( i16 %1 )
-  %3 = tail call i16 @llvm.bswap.i16( i16 %a1 )
-  store i16 %3, i16* %a2, align 2
-  ret i16 %2
-}
-declare i16 @llvm.bswap.i16(i16)
-
-define i32 @test_movbe_i32(i32 *%a0, i32 %a1, i32 *%a2) {
-; GENERIC-LABEL: test_movbe_i32:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    movbel (%rdi), %eax # sched: [6:0.50]
-; GENERIC-NEXT:    movbel %esi, (%rdx) # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_movbe_i32:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    movbel (%rdi), %eax # sched: [1:1.00]
-; ATOM-NEXT:    movbel %esi, (%rdx) # sched: [1:1.00]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_movbe_i32:
-; SLM:       # %bb.0:
-; SLM-NEXT:    movbel (%rdi), %eax # sched: [4:1.00]
-; SLM-NEXT:    movbel %esi, (%rdx) # sched: [1:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; HASWELL-LABEL: test_movbe_i32:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    movbel (%rdi), %eax # sched: [6:0.50]
-; HASWELL-NEXT:    movbel %esi, (%rdx) # sched: [2:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_movbe_i32:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    movbel (%rdi), %eax # sched: [6:0.50]
-; BROADWELL-NEXT:    movbel %esi, (%rdx) # sched: [2:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_movbe_i32:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    movbel (%rdi), %eax # sched: [6:0.50]
-; SKYLAKE-NEXT:    movbel %esi, (%rdx) # sched: [2:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; BTVER2-LABEL: test_movbe_i32:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    movbel (%rdi), %eax # sched: [4:1.00]
-; BTVER2-NEXT:    movbel %esi, (%rdx) # sched: [1:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_movbe_i32:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    movbel (%rdi), %eax # sched: [5:0.50]
-; ZNVER1-NEXT:    movbel %esi, (%rdx) # sched: [5:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = load i32, i32 *%a0
-  %2 = tail call i32 @llvm.bswap.i32( i32 %1 )
-  %3 = tail call i32 @llvm.bswap.i32( i32 %a1 )
-  store i32 %3, i32* %a2, align 2
-  ret i32 %2
-}
-declare i32 @llvm.bswap.i32(i32)
-
-define i64 @test_movbe_i64(i64 *%a0, i64 %a1, i64 *%a2) {
-; GENERIC-LABEL: test_movbe_i64:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    movbeq (%rdi), %rax # sched: [6:0.50]
-; GENERIC-NEXT:    movbeq %rsi, (%rdx) # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_movbe_i64:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    movbeq (%rdi), %rax # sched: [1:1.00]
-; ATOM-NEXT:    movbeq %rsi, (%rdx) # sched: [1:1.00]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_movbe_i64:
-; SLM:       # %bb.0:
-; SLM-NEXT:    movbeq (%rdi), %rax # sched: [4:1.00]
-; SLM-NEXT:    movbeq %rsi, (%rdx) # sched: [1:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; HASWELL-LABEL: test_movbe_i64:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    movbeq (%rdi), %rax # sched: [6:0.50]
-; HASWELL-NEXT:    movbeq %rsi, (%rdx) # sched: [2:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_movbe_i64:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    movbeq (%rdi), %rax # sched: [6:0.50]
-; BROADWELL-NEXT:    movbeq %rsi, (%rdx) # sched: [2:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_movbe_i64:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    movbeq (%rdi), %rax # sched: [6:0.50]
-; SKYLAKE-NEXT:    movbeq %rsi, (%rdx) # sched: [2:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; BTVER2-LABEL: test_movbe_i64:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    movbeq (%rdi), %rax # sched: [4:1.00]
-; BTVER2-NEXT:    movbeq %rsi, (%rdx) # sched: [1:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_movbe_i64:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    movbeq (%rdi), %rax # sched: [5:0.50]
-; ZNVER1-NEXT:    movbeq %rsi, (%rdx) # sched: [5:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = load i64, i64 *%a0
-  %2 = tail call i64 @llvm.bswap.i64( i64 %1 )
-  %3 = tail call i64 @llvm.bswap.i64( i64 %a1 )
-  store i64 %3, i64* %a2, align 2
-  ret i64 %2
-}
-declare i64 @llvm.bswap.i64(i64)
diff --git a/test/CodeGen/X86/movddup-load-fold.ll b/test/CodeGen/X86/movddup-load-fold.ll
new file mode 100644
index 0000000000000000000000000000000000000000..f1af6e9654e0304a1b42f23799fe373e57d57f1f
--- /dev/null
+++ b/test/CodeGen/X86/movddup-load-fold.ll
@@ -0,0 +1,24 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-- -mattr=+sse4.1   | FileCheck %s --check-prefixes=SSE,SSE41
+; RUN: llc < %s -mtriple=i686-- -mattr=+avx      | FileCheck %s --check-prefixes=AVX,AVX1
+; RUN: llc < %s -mtriple=i686-- -mattr=+avx2     | FileCheck %s --check-prefixes=AVX,AVX2
+; RUN: llc < %s -mtriple=i686-- -mattr=+avx512vl | FileCheck %s --check-prefixes=AVX,AVX512VL
+
+; Test an isel pattern for a splatted VZLOAD.
+
+define <4 x float> @movddup_load_fold(float %x, float %y) {
+; SSE-LABEL: movddup_load_fold:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movddup {{.*#+}} xmm0 = mem[0,0]
+; SSE-NEXT:    retl
+;
+; AVX-LABEL: movddup_load_fold:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovddup {{.*#+}} xmm0 = mem[0,0]
+; AVX-NEXT:    retl
+  %i0 = insertelement <4 x float> zeroinitializer, float %x, i32 0
+  %i1 = insertelement <4 x float> %i0, float %y, i32 1
+  %dup = shufflevector <4 x float> %i1, <4 x float> undef, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
+  ret <4 x float> %dup
+}
+
diff --git a/test/CodeGen/X86/movmsk-cmp.ll b/test/CodeGen/X86/movmsk-cmp.ll
index 718ade024356687b2a7ac2c927ecea56f7be6073..3ff101761dbd1926e0c24c66ee672a92ca5634d5 100644
--- a/test/CodeGen/X86/movmsk-cmp.ll
+++ b/test/CodeGen/X86/movmsk-cmp.ll
@@ -22,13 +22,9 @@ define i1 @allones_v16i8_sign(<16 x i8> %arg) {
 ;
 ; KNL-LABEL: allones_v16i8_sign:
 ; KNL:       # %bb.0:
-; KNL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; KNL-NEXT:    vpcmpgtb %xmm0, %xmm1, %xmm0
-; KNL-NEXT:    vpmovsxbd %xmm0, %zmm0
-; KNL-NEXT:    vptestmd %zmm0, %zmm0, %k0
-; KNL-NEXT:    kortestw %k0, %k0
-; KNL-NEXT:    setb %al
-; KNL-NEXT:    vzeroupper
+; KNL-NEXT:    vpmovmskb %xmm0, %eax
+; KNL-NEXT:    cmpw $-1, %ax
+; KNL-NEXT:    sete %al
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: allones_v16i8_sign:
@@ -60,13 +56,9 @@ define i1 @allzeros_v16i8_sign(<16 x i8> %arg) {
 ;
 ; KNL-LABEL: allzeros_v16i8_sign:
 ; KNL:       # %bb.0:
-; KNL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; KNL-NEXT:    vpcmpgtb %xmm0, %xmm1, %xmm0
-; KNL-NEXT:    vpmovsxbd %xmm0, %zmm0
-; KNL-NEXT:    vptestmd %zmm0, %zmm0, %k0
-; KNL-NEXT:    kortestw %k0, %k0
+; KNL-NEXT:    vpmovmskb %xmm0, %eax
+; KNL-NEXT:    testw %ax, %ax
 ; KNL-NEXT:    sete %al
-; KNL-NEXT:    vzeroupper
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: allzeros_v16i8_sign:
@@ -94,11 +86,8 @@ define i1 @allones_v32i8_sign(<32 x i8> %arg) {
 ;
 ; AVX1-LABEL: allones_v32i8_sign:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpgtb %xmm0, %xmm1, %xmm2
-; AVX1-NEXT:    vpmovmskb %xmm2, %eax
+; AVX1-NEXT:    vpmovmskb %xmm0, %eax
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
-; AVX1-NEXT:    vpcmpgtb %xmm0, %xmm1, %xmm0
 ; AVX1-NEXT:    vpmovmskb %xmm0, %ecx
 ; AVX1-NEXT:    shll $16, %ecx
 ; AVX1-NEXT:    orl %eax, %ecx
@@ -117,18 +106,8 @@ define i1 @allones_v32i8_sign(<32 x i8> %arg) {
 ;
 ; KNL-LABEL: allones_v32i8_sign:
 ; KNL:       # %bb.0:
-; KNL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; KNL-NEXT:    vpcmpgtb %ymm0, %ymm1, %ymm0
-; KNL-NEXT:    vpmovsxbd %xmm0, %zmm1
-; KNL-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; KNL-NEXT:    kmovw %k0, %eax
-; KNL-NEXT:    vextracti128 $1, %ymm0, %xmm0
-; KNL-NEXT:    vpmovsxbd %xmm0, %zmm0
-; KNL-NEXT:    vptestmd %zmm0, %zmm0, %k0
-; KNL-NEXT:    kmovw %k0, %ecx
-; KNL-NEXT:    shll $16, %ecx
-; KNL-NEXT:    orl %eax, %ecx
-; KNL-NEXT:    cmpl $-1, %ecx
+; KNL-NEXT:    vpmovmskb %ymm0, %eax
+; KNL-NEXT:    cmpl $-1, %eax
 ; KNL-NEXT:    sete %al
 ; KNL-NEXT:    vzeroupper
 ; KNL-NEXT:    retq
@@ -158,11 +137,8 @@ define i1 @allzeros_v32i8_sign(<32 x i8> %arg) {
 ;
 ; AVX1-LABEL: allzeros_v32i8_sign:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpgtb %xmm0, %xmm1, %xmm2
-; AVX1-NEXT:    vpmovmskb %xmm2, %eax
+; AVX1-NEXT:    vpmovmskb %xmm0, %eax
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
-; AVX1-NEXT:    vpcmpgtb %xmm0, %xmm1, %xmm0
 ; AVX1-NEXT:    vpmovmskb %xmm0, %ecx
 ; AVX1-NEXT:    shll $16, %ecx
 ; AVX1-NEXT:    orl %eax, %ecx
@@ -180,17 +156,8 @@ define i1 @allzeros_v32i8_sign(<32 x i8> %arg) {
 ;
 ; KNL-LABEL: allzeros_v32i8_sign:
 ; KNL:       # %bb.0:
-; KNL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; KNL-NEXT:    vpcmpgtb %ymm0, %ymm1, %ymm0
-; KNL-NEXT:    vpmovsxbd %xmm0, %zmm1
-; KNL-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; KNL-NEXT:    kmovw %k0, %eax
-; KNL-NEXT:    vextracti128 $1, %ymm0, %xmm0
-; KNL-NEXT:    vpmovsxbd %xmm0, %zmm0
-; KNL-NEXT:    vptestmd %zmm0, %zmm0, %k0
-; KNL-NEXT:    kmovw %k0, %ecx
-; KNL-NEXT:    shll $16, %ecx
-; KNL-NEXT:    orl %eax, %ecx
+; KNL-NEXT:    vpmovmskb %ymm0, %eax
+; KNL-NEXT:    testl %eax, %eax
 ; KNL-NEXT:    sete %al
 ; KNL-NEXT:    vzeroupper
 ; KNL-NEXT:    retq
@@ -227,18 +194,13 @@ define i1 @allones_v64i8_sign(<64 x i8> %arg) {
 ;
 ; AVX1-LABEL: allones_v64i8_sign:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm3
-; AVX1-NEXT:    vpmovmskb %xmm3, %eax
+; AVX1-NEXT:    vpmovmskb %xmm0, %eax
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
-; AVX1-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm0
 ; AVX1-NEXT:    vpmovmskb %xmm0, %ecx
 ; AVX1-NEXT:    shll $16, %ecx
 ; AVX1-NEXT:    orl %eax, %ecx
-; AVX1-NEXT:    vpcmpgtb %xmm1, %xmm2, %xmm0
-; AVX1-NEXT:    vpmovmskb %xmm0, %eax
+; AVX1-NEXT:    vpmovmskb %xmm1, %eax
 ; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm0
-; AVX1-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm0
 ; AVX1-NEXT:    vpmovmskb %xmm0, %edx
 ; AVX1-NEXT:    shll $16, %edx
 ; AVX1-NEXT:    orl %eax, %edx
@@ -262,30 +224,11 @@ define i1 @allones_v64i8_sign(<64 x i8> %arg) {
 ;
 ; KNL-LABEL: allones_v64i8_sign:
 ; KNL:       # %bb.0:
-; KNL-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; KNL-NEXT:    vpcmpgtb %ymm0, %ymm2, %ymm0
-; KNL-NEXT:    vpmovsxbd %xmm0, %zmm3
-; KNL-NEXT:    vptestmd %zmm3, %zmm3, %k0
-; KNL-NEXT:    kmovw %k0, %eax
-; KNL-NEXT:    vextracti128 $1, %ymm0, %xmm0
-; KNL-NEXT:    vpmovsxbd %xmm0, %zmm0
-; KNL-NEXT:    vptestmd %zmm0, %zmm0, %k0
-; KNL-NEXT:    kmovw %k0, %ecx
-; KNL-NEXT:    shll $16, %ecx
-; KNL-NEXT:    orl %eax, %ecx
-; KNL-NEXT:    vpcmpgtb %ymm1, %ymm2, %ymm0
-; KNL-NEXT:    vpmovsxbd %xmm0, %zmm1
-; KNL-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; KNL-NEXT:    kmovw %k0, %eax
-; KNL-NEXT:    vextracti128 $1, %ymm0, %xmm0
-; KNL-NEXT:    vpmovsxbd %xmm0, %zmm0
-; KNL-NEXT:    vptestmd %zmm0, %zmm0, %k0
-; KNL-NEXT:    kmovw %k0, %edx
-; KNL-NEXT:    shll $16, %edx
-; KNL-NEXT:    orl %eax, %edx
-; KNL-NEXT:    shlq $32, %rdx
-; KNL-NEXT:    orq %rcx, %rdx
-; KNL-NEXT:    cmpq $-1, %rdx
+; KNL-NEXT:    vpmovmskb %ymm1, %eax
+; KNL-NEXT:    shlq $32, %rax
+; KNL-NEXT:    vpmovmskb %ymm0, %ecx
+; KNL-NEXT:    orq %rax, %rcx
+; KNL-NEXT:    cmpq $-1, %rcx
 ; KNL-NEXT:    sete %al
 ; KNL-NEXT:    vzeroupper
 ; KNL-NEXT:    retq
@@ -321,18 +264,13 @@ define i1 @allzeros_v64i8_sign(<64 x i8> %arg) {
 ;
 ; AVX1-LABEL: allzeros_v64i8_sign:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm3
-; AVX1-NEXT:    vpmovmskb %xmm3, %eax
+; AVX1-NEXT:    vpmovmskb %xmm0, %eax
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
-; AVX1-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm0
 ; AVX1-NEXT:    vpmovmskb %xmm0, %ecx
 ; AVX1-NEXT:    shll $16, %ecx
 ; AVX1-NEXT:    orl %eax, %ecx
-; AVX1-NEXT:    vpcmpgtb %xmm1, %xmm2, %xmm0
-; AVX1-NEXT:    vpmovmskb %xmm0, %eax
+; AVX1-NEXT:    vpmovmskb %xmm1, %eax
 ; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm0
-; AVX1-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm0
 ; AVX1-NEXT:    vpmovmskb %xmm0, %edx
 ; AVX1-NEXT:    shll $16, %edx
 ; AVX1-NEXT:    orl %eax, %edx
@@ -354,29 +292,10 @@ define i1 @allzeros_v64i8_sign(<64 x i8> %arg) {
 ;
 ; KNL-LABEL: allzeros_v64i8_sign:
 ; KNL:       # %bb.0:
-; KNL-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; KNL-NEXT:    vpcmpgtb %ymm0, %ymm2, %ymm0
-; KNL-NEXT:    vpmovsxbd %xmm0, %zmm3
-; KNL-NEXT:    vptestmd %zmm3, %zmm3, %k0
-; KNL-NEXT:    kmovw %k0, %eax
-; KNL-NEXT:    vextracti128 $1, %ymm0, %xmm0
-; KNL-NEXT:    vpmovsxbd %xmm0, %zmm0
-; KNL-NEXT:    vptestmd %zmm0, %zmm0, %k0
-; KNL-NEXT:    kmovw %k0, %ecx
-; KNL-NEXT:    shll $16, %ecx
-; KNL-NEXT:    orl %eax, %ecx
-; KNL-NEXT:    vpcmpgtb %ymm1, %ymm2, %ymm0
-; KNL-NEXT:    vpmovsxbd %xmm0, %zmm1
-; KNL-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; KNL-NEXT:    kmovw %k0, %eax
-; KNL-NEXT:    vextracti128 $1, %ymm0, %xmm0
-; KNL-NEXT:    vpmovsxbd %xmm0, %zmm0
-; KNL-NEXT:    vptestmd %zmm0, %zmm0, %k0
-; KNL-NEXT:    kmovw %k0, %edx
-; KNL-NEXT:    shll $16, %edx
-; KNL-NEXT:    orl %eax, %edx
-; KNL-NEXT:    shlq $32, %rdx
-; KNL-NEXT:    orq %rcx, %rdx
+; KNL-NEXT:    vpmovmskb %ymm1, %eax
+; KNL-NEXT:    shlq $32, %rax
+; KNL-NEXT:    vpmovmskb %ymm0, %ecx
+; KNL-NEXT:    orq %rax, %rcx
 ; KNL-NEXT:    sete %al
 ; KNL-NEXT:    vzeroupper
 ; KNL-NEXT:    retq
@@ -1438,14 +1357,10 @@ define i1 @allones_v16i8_and1(<16 x i8> %arg) {
 ;
 ; KNL-LABEL: allones_v16i8_and1:
 ; KNL:       # %bb.0:
-; KNL-NEXT:    vmovdqa {{.*#+}} xmm1 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
-; KNL-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; KNL-NEXT:    vpcmpeqb %xmm1, %xmm0, %xmm0
-; KNL-NEXT:    vpmovsxbd %xmm0, %zmm0
-; KNL-NEXT:    vptestmd %zmm0, %zmm0, %k0
-; KNL-NEXT:    kortestw %k0, %k0
-; KNL-NEXT:    setb %al
-; KNL-NEXT:    vzeroupper
+; KNL-NEXT:    vpsllw $7, %xmm0, %xmm0
+; KNL-NEXT:    vpmovmskb %xmm0, %eax
+; KNL-NEXT:    cmpw $-1, %ax
+; KNL-NEXT:    sete %al
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: allones_v16i8_and1:
@@ -1480,14 +1395,10 @@ define i1 @allzeros_v16i8_and1(<16 x i8> %arg) {
 ;
 ; KNL-LABEL: allzeros_v16i8_and1:
 ; KNL:       # %bb.0:
-; KNL-NEXT:    vmovdqa {{.*#+}} xmm1 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
-; KNL-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; KNL-NEXT:    vpcmpeqb %xmm1, %xmm0, %xmm0
-; KNL-NEXT:    vpmovsxbd %xmm0, %zmm0
-; KNL-NEXT:    vptestmd %zmm0, %zmm0, %k0
-; KNL-NEXT:    kortestw %k0, %k0
+; KNL-NEXT:    vpsllw $7, %xmm0, %xmm0
+; KNL-NEXT:    vpmovmskb %xmm0, %eax
+; KNL-NEXT:    testw %ax, %ax
 ; KNL-NEXT:    sete %al
-; KNL-NEXT:    vzeroupper
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: allzeros_v16i8_and1:
@@ -1546,19 +1457,9 @@ define i1 @allones_v32i8_and1(<32 x i8> %arg) {
 ;
 ; KNL-LABEL: allones_v32i8_and1:
 ; KNL:       # %bb.0:
-; KNL-NEXT:    vmovdqa {{.*#+}} ymm1 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
-; KNL-NEXT:    vpand %ymm1, %ymm0, %ymm0
-; KNL-NEXT:    vpcmpeqb %ymm1, %ymm0, %ymm0
-; KNL-NEXT:    vpmovsxbd %xmm0, %zmm1
-; KNL-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; KNL-NEXT:    kmovw %k0, %eax
-; KNL-NEXT:    vextracti128 $1, %ymm0, %xmm0
-; KNL-NEXT:    vpmovsxbd %xmm0, %zmm0
-; KNL-NEXT:    vptestmd %zmm0, %zmm0, %k0
-; KNL-NEXT:    kmovw %k0, %ecx
-; KNL-NEXT:    shll $16, %ecx
-; KNL-NEXT:    orl %eax, %ecx
-; KNL-NEXT:    cmpl $-1, %ecx
+; KNL-NEXT:    vpsllw $7, %ymm0, %ymm0
+; KNL-NEXT:    vpmovmskb %ymm0, %eax
+; KNL-NEXT:    cmpl $-1, %eax
 ; KNL-NEXT:    sete %al
 ; KNL-NEXT:    vzeroupper
 ; KNL-NEXT:    retq
@@ -1618,18 +1519,9 @@ define i1 @allzeros_v32i8_and1(<32 x i8> %arg) {
 ;
 ; KNL-LABEL: allzeros_v32i8_and1:
 ; KNL:       # %bb.0:
-; KNL-NEXT:    vmovdqa {{.*#+}} ymm1 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
-; KNL-NEXT:    vpand %ymm1, %ymm0, %ymm0
-; KNL-NEXT:    vpcmpeqb %ymm1, %ymm0, %ymm0
-; KNL-NEXT:    vpmovsxbd %xmm0, %zmm1
-; KNL-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; KNL-NEXT:    kmovw %k0, %eax
-; KNL-NEXT:    vextracti128 $1, %ymm0, %xmm0
-; KNL-NEXT:    vpmovsxbd %xmm0, %zmm0
-; KNL-NEXT:    vptestmd %zmm0, %zmm0, %k0
-; KNL-NEXT:    kmovw %k0, %ecx
-; KNL-NEXT:    shll $16, %ecx
-; KNL-NEXT:    orl %eax, %ecx
+; KNL-NEXT:    vpsllw $7, %ymm0, %ymm0
+; KNL-NEXT:    vpmovmskb %ymm0, %eax
+; KNL-NEXT:    testl %eax, %eax
 ; KNL-NEXT:    sete %al
 ; KNL-NEXT:    vzeroupper
 ; KNL-NEXT:    retq
@@ -1716,32 +1608,13 @@ define i1 @allones_v64i8_and1(<64 x i8> %arg) {
 ;
 ; KNL-LABEL: allones_v64i8_and1:
 ; KNL:       # %bb.0:
-; KNL-NEXT:    vmovdqa {{.*#+}} ymm2 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
-; KNL-NEXT:    vpand %ymm2, %ymm1, %ymm1
-; KNL-NEXT:    vpand %ymm2, %ymm0, %ymm0
-; KNL-NEXT:    vpcmpeqb %ymm2, %ymm0, %ymm0
-; KNL-NEXT:    vpmovsxbd %xmm0, %zmm3
-; KNL-NEXT:    vptestmd %zmm3, %zmm3, %k0
-; KNL-NEXT:    kmovw %k0, %eax
-; KNL-NEXT:    vextracti128 $1, %ymm0, %xmm0
-; KNL-NEXT:    vpmovsxbd %xmm0, %zmm0
-; KNL-NEXT:    vptestmd %zmm0, %zmm0, %k0
-; KNL-NEXT:    kmovw %k0, %ecx
-; KNL-NEXT:    shll $16, %ecx
-; KNL-NEXT:    orl %eax, %ecx
-; KNL-NEXT:    vpcmpeqb %ymm2, %ymm1, %ymm0
-; KNL-NEXT:    vpmovsxbd %xmm0, %zmm1
-; KNL-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; KNL-NEXT:    kmovw %k0, %eax
-; KNL-NEXT:    vextracti128 $1, %ymm0, %xmm0
-; KNL-NEXT:    vpmovsxbd %xmm0, %zmm0
-; KNL-NEXT:    vptestmd %zmm0, %zmm0, %k0
-; KNL-NEXT:    kmovw %k0, %edx
-; KNL-NEXT:    shll $16, %edx
-; KNL-NEXT:    orl %eax, %edx
-; KNL-NEXT:    shlq $32, %rdx
-; KNL-NEXT:    orq %rcx, %rdx
-; KNL-NEXT:    cmpq $-1, %rdx
+; KNL-NEXT:    vpsllw $7, %ymm0, %ymm0
+; KNL-NEXT:    vpsllw $7, %ymm1, %ymm1
+; KNL-NEXT:    vpmovmskb %ymm1, %eax
+; KNL-NEXT:    shlq $32, %rax
+; KNL-NEXT:    vpmovmskb %ymm0, %ecx
+; KNL-NEXT:    orq %rax, %rcx
+; KNL-NEXT:    cmpq $-1, %rcx
 ; KNL-NEXT:    sete %al
 ; KNL-NEXT:    vzeroupper
 ; KNL-NEXT:    retq
@@ -1825,31 +1698,12 @@ define i1 @allzeros_v64i8_and1(<64 x i8> %arg) {
 ;
 ; KNL-LABEL: allzeros_v64i8_and1:
 ; KNL:       # %bb.0:
-; KNL-NEXT:    vmovdqa {{.*#+}} ymm2 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
-; KNL-NEXT:    vpand %ymm2, %ymm1, %ymm1
-; KNL-NEXT:    vpand %ymm2, %ymm0, %ymm0
-; KNL-NEXT:    vpcmpeqb %ymm2, %ymm0, %ymm0
-; KNL-NEXT:    vpmovsxbd %xmm0, %zmm3
-; KNL-NEXT:    vptestmd %zmm3, %zmm3, %k0
-; KNL-NEXT:    kmovw %k0, %eax
-; KNL-NEXT:    vextracti128 $1, %ymm0, %xmm0
-; KNL-NEXT:    vpmovsxbd %xmm0, %zmm0
-; KNL-NEXT:    vptestmd %zmm0, %zmm0, %k0
-; KNL-NEXT:    kmovw %k0, %ecx
-; KNL-NEXT:    shll $16, %ecx
-; KNL-NEXT:    orl %eax, %ecx
-; KNL-NEXT:    vpcmpeqb %ymm2, %ymm1, %ymm0
-; KNL-NEXT:    vpmovsxbd %xmm0, %zmm1
-; KNL-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; KNL-NEXT:    kmovw %k0, %eax
-; KNL-NEXT:    vextracti128 $1, %ymm0, %xmm0
-; KNL-NEXT:    vpmovsxbd %xmm0, %zmm0
-; KNL-NEXT:    vptestmd %zmm0, %zmm0, %k0
-; KNL-NEXT:    kmovw %k0, %edx
-; KNL-NEXT:    shll $16, %edx
-; KNL-NEXT:    orl %eax, %edx
-; KNL-NEXT:    shlq $32, %rdx
-; KNL-NEXT:    orq %rcx, %rdx
+; KNL-NEXT:    vpsllw $7, %ymm0, %ymm0
+; KNL-NEXT:    vpsllw $7, %ymm1, %ymm1
+; KNL-NEXT:    vpmovmskb %ymm1, %eax
+; KNL-NEXT:    shlq $32, %rax
+; KNL-NEXT:    vpmovmskb %ymm0, %ecx
+; KNL-NEXT:    orq %rax, %rcx
 ; KNL-NEXT:    sete %al
 ; KNL-NEXT:    vzeroupper
 ; KNL-NEXT:    retq
@@ -3102,14 +2956,10 @@ define i1 @allones_v16i8_and4(<16 x i8> %arg) {
 ;
 ; KNL-LABEL: allones_v16i8_and4:
 ; KNL:       # %bb.0:
-; KNL-NEXT:    vmovdqa {{.*#+}} xmm1 = [4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4]
-; KNL-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; KNL-NEXT:    vpcmpeqb %xmm1, %xmm0, %xmm0
-; KNL-NEXT:    vpmovsxbd %xmm0, %zmm0
-; KNL-NEXT:    vptestmd %zmm0, %zmm0, %k0
-; KNL-NEXT:    kortestw %k0, %k0
-; KNL-NEXT:    setb %al
-; KNL-NEXT:    vzeroupper
+; KNL-NEXT:    vpsllw $5, %xmm0, %xmm0
+; KNL-NEXT:    vpmovmskb %xmm0, %eax
+; KNL-NEXT:    cmpw $-1, %ax
+; KNL-NEXT:    sete %al
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: allones_v16i8_and4:
@@ -3144,14 +2994,10 @@ define i1 @allzeros_v16i8_and4(<16 x i8> %arg) {
 ;
 ; KNL-LABEL: allzeros_v16i8_and4:
 ; KNL:       # %bb.0:
-; KNL-NEXT:    vmovdqa {{.*#+}} xmm1 = [4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4]
-; KNL-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; KNL-NEXT:    vpcmpeqb %xmm1, %xmm0, %xmm0
-; KNL-NEXT:    vpmovsxbd %xmm0, %zmm0
-; KNL-NEXT:    vptestmd %zmm0, %zmm0, %k0
-; KNL-NEXT:    kortestw %k0, %k0
+; KNL-NEXT:    vpsllw $5, %xmm0, %xmm0
+; KNL-NEXT:    vpmovmskb %xmm0, %eax
+; KNL-NEXT:    testw %ax, %ax
 ; KNL-NEXT:    sete %al
-; KNL-NEXT:    vzeroupper
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: allzeros_v16i8_and4:
@@ -3210,19 +3056,9 @@ define i1 @allones_v32i8_and4(<32 x i8> %arg) {
 ;
 ; KNL-LABEL: allones_v32i8_and4:
 ; KNL:       # %bb.0:
-; KNL-NEXT:    vmovdqa {{.*#+}} ymm1 = [4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4]
-; KNL-NEXT:    vpand %ymm1, %ymm0, %ymm0
-; KNL-NEXT:    vpcmpeqb %ymm1, %ymm0, %ymm0
-; KNL-NEXT:    vpmovsxbd %xmm0, %zmm1
-; KNL-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; KNL-NEXT:    kmovw %k0, %eax
-; KNL-NEXT:    vextracti128 $1, %ymm0, %xmm0
-; KNL-NEXT:    vpmovsxbd %xmm0, %zmm0
-; KNL-NEXT:    vptestmd %zmm0, %zmm0, %k0
-; KNL-NEXT:    kmovw %k0, %ecx
-; KNL-NEXT:    shll $16, %ecx
-; KNL-NEXT:    orl %eax, %ecx
-; KNL-NEXT:    cmpl $-1, %ecx
+; KNL-NEXT:    vpsllw $5, %ymm0, %ymm0
+; KNL-NEXT:    vpmovmskb %ymm0, %eax
+; KNL-NEXT:    cmpl $-1, %eax
 ; KNL-NEXT:    sete %al
 ; KNL-NEXT:    vzeroupper
 ; KNL-NEXT:    retq
@@ -3282,18 +3118,9 @@ define i1 @allzeros_v32i8_and4(<32 x i8> %arg) {
 ;
 ; KNL-LABEL: allzeros_v32i8_and4:
 ; KNL:       # %bb.0:
-; KNL-NEXT:    vmovdqa {{.*#+}} ymm1 = [4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4]
-; KNL-NEXT:    vpand %ymm1, %ymm0, %ymm0
-; KNL-NEXT:    vpcmpeqb %ymm1, %ymm0, %ymm0
-; KNL-NEXT:    vpmovsxbd %xmm0, %zmm1
-; KNL-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; KNL-NEXT:    kmovw %k0, %eax
-; KNL-NEXT:    vextracti128 $1, %ymm0, %xmm0
-; KNL-NEXT:    vpmovsxbd %xmm0, %zmm0
-; KNL-NEXT:    vptestmd %zmm0, %zmm0, %k0
-; KNL-NEXT:    kmovw %k0, %ecx
-; KNL-NEXT:    shll $16, %ecx
-; KNL-NEXT:    orl %eax, %ecx
+; KNL-NEXT:    vpsllw $5, %ymm0, %ymm0
+; KNL-NEXT:    vpmovmskb %ymm0, %eax
+; KNL-NEXT:    testl %eax, %eax
 ; KNL-NEXT:    sete %al
 ; KNL-NEXT:    vzeroupper
 ; KNL-NEXT:    retq
@@ -3380,32 +3207,13 @@ define i1 @allones_v64i8_and4(<64 x i8> %arg) {
 ;
 ; KNL-LABEL: allones_v64i8_and4:
 ; KNL:       # %bb.0:
-; KNL-NEXT:    vmovdqa {{.*#+}} ymm2 = [4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4]
-; KNL-NEXT:    vpand %ymm2, %ymm1, %ymm1
-; KNL-NEXT:    vpand %ymm2, %ymm0, %ymm0
-; KNL-NEXT:    vpcmpeqb %ymm2, %ymm0, %ymm0
-; KNL-NEXT:    vpmovsxbd %xmm0, %zmm3
-; KNL-NEXT:    vptestmd %zmm3, %zmm3, %k0
-; KNL-NEXT:    kmovw %k0, %eax
-; KNL-NEXT:    vextracti128 $1, %ymm0, %xmm0
-; KNL-NEXT:    vpmovsxbd %xmm0, %zmm0
-; KNL-NEXT:    vptestmd %zmm0, %zmm0, %k0
-; KNL-NEXT:    kmovw %k0, %ecx
-; KNL-NEXT:    shll $16, %ecx
-; KNL-NEXT:    orl %eax, %ecx
-; KNL-NEXT:    vpcmpeqb %ymm2, %ymm1, %ymm0
-; KNL-NEXT:    vpmovsxbd %xmm0, %zmm1
-; KNL-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; KNL-NEXT:    kmovw %k0, %eax
-; KNL-NEXT:    vextracti128 $1, %ymm0, %xmm0
-; KNL-NEXT:    vpmovsxbd %xmm0, %zmm0
-; KNL-NEXT:    vptestmd %zmm0, %zmm0, %k0
-; KNL-NEXT:    kmovw %k0, %edx
-; KNL-NEXT:    shll $16, %edx
-; KNL-NEXT:    orl %eax, %edx
-; KNL-NEXT:    shlq $32, %rdx
-; KNL-NEXT:    orq %rcx, %rdx
-; KNL-NEXT:    cmpq $-1, %rdx
+; KNL-NEXT:    vpsllw $5, %ymm0, %ymm0
+; KNL-NEXT:    vpsllw $5, %ymm1, %ymm1
+; KNL-NEXT:    vpmovmskb %ymm1, %eax
+; KNL-NEXT:    shlq $32, %rax
+; KNL-NEXT:    vpmovmskb %ymm0, %ecx
+; KNL-NEXT:    orq %rax, %rcx
+; KNL-NEXT:    cmpq $-1, %rcx
 ; KNL-NEXT:    sete %al
 ; KNL-NEXT:    vzeroupper
 ; KNL-NEXT:    retq
@@ -3489,31 +3297,12 @@ define i1 @allzeros_v64i8_and4(<64 x i8> %arg) {
 ;
 ; KNL-LABEL: allzeros_v64i8_and4:
 ; KNL:       # %bb.0:
-; KNL-NEXT:    vmovdqa {{.*#+}} ymm2 = [4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4]
-; KNL-NEXT:    vpand %ymm2, %ymm1, %ymm1
-; KNL-NEXT:    vpand %ymm2, %ymm0, %ymm0
-; KNL-NEXT:    vpcmpeqb %ymm2, %ymm0, %ymm0
-; KNL-NEXT:    vpmovsxbd %xmm0, %zmm3
-; KNL-NEXT:    vptestmd %zmm3, %zmm3, %k0
-; KNL-NEXT:    kmovw %k0, %eax
-; KNL-NEXT:    vextracti128 $1, %ymm0, %xmm0
-; KNL-NEXT:    vpmovsxbd %xmm0, %zmm0
-; KNL-NEXT:    vptestmd %zmm0, %zmm0, %k0
-; KNL-NEXT:    kmovw %k0, %ecx
-; KNL-NEXT:    shll $16, %ecx
-; KNL-NEXT:    orl %eax, %ecx
-; KNL-NEXT:    vpcmpeqb %ymm2, %ymm1, %ymm0
-; KNL-NEXT:    vpmovsxbd %xmm0, %zmm1
-; KNL-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; KNL-NEXT:    kmovw %k0, %eax
-; KNL-NEXT:    vextracti128 $1, %ymm0, %xmm0
-; KNL-NEXT:    vpmovsxbd %xmm0, %zmm0
-; KNL-NEXT:    vptestmd %zmm0, %zmm0, %k0
-; KNL-NEXT:    kmovw %k0, %edx
-; KNL-NEXT:    shll $16, %edx
-; KNL-NEXT:    orl %eax, %edx
-; KNL-NEXT:    shlq $32, %rdx
-; KNL-NEXT:    orq %rcx, %rdx
+; KNL-NEXT:    vpsllw $5, %ymm0, %ymm0
+; KNL-NEXT:    vpsllw $5, %ymm1, %ymm1
+; KNL-NEXT:    vpmovmskb %ymm1, %eax
+; KNL-NEXT:    shlq $32, %rax
+; KNL-NEXT:    vpmovmskb %ymm0, %ecx
+; KNL-NEXT:    orq %rax, %rcx
 ; KNL-NEXT:    sete %al
 ; KNL-NEXT:    vzeroupper
 ; KNL-NEXT:    retq
@@ -4926,12 +4715,7 @@ define i32 @movmskb(<16 x i8> %x) {
 ;
 ; KNL-LABEL: movmskb:
 ; KNL:       # %bb.0:
-; KNL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; KNL-NEXT:    vpcmpgtb %xmm0, %xmm1, %xmm0
-; KNL-NEXT:    vpmovsxbd %xmm0, %zmm0
-; KNL-NEXT:    vptestmd %zmm0, %zmm0, %k0
-; KNL-NEXT:    kmovw %k0, %eax
-; KNL-NEXT:    vzeroupper
+; KNL-NEXT:    vpmovmskb %xmm0, %eax
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: movmskb:
@@ -4956,11 +4740,8 @@ define i32 @movmskb256(<32 x i8> %x) {
 ;
 ; AVX1-LABEL: movmskb256:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpgtb %xmm0, %xmm1, %xmm2
-; AVX1-NEXT:    vpmovmskb %xmm2, %ecx
+; AVX1-NEXT:    vpmovmskb %xmm0, %ecx
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
-; AVX1-NEXT:    vpcmpgtb %xmm0, %xmm1, %xmm0
 ; AVX1-NEXT:    vpmovmskb %xmm0, %eax
 ; AVX1-NEXT:    shll $16, %eax
 ; AVX1-NEXT:    orl %ecx, %eax
@@ -4975,17 +4756,7 @@ define i32 @movmskb256(<32 x i8> %x) {
 ;
 ; KNL-LABEL: movmskb256:
 ; KNL:       # %bb.0:
-; KNL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; KNL-NEXT:    vpcmpgtb %ymm0, %ymm1, %ymm0
-; KNL-NEXT:    vpmovsxbd %xmm0, %zmm1
-; KNL-NEXT:    vptestmd %zmm1, %zmm1, %k0
-; KNL-NEXT:    kmovw %k0, %ecx
-; KNL-NEXT:    vextracti128 $1, %ymm0, %xmm0
-; KNL-NEXT:    vpmovsxbd %xmm0, %zmm0
-; KNL-NEXT:    vptestmd %zmm0, %zmm0, %k0
-; KNL-NEXT:    kmovw %k0, %eax
-; KNL-NEXT:    shll $16, %eax
-; KNL-NEXT:    orl %ecx, %eax
+; KNL-NEXT:    vpmovmskb %ymm0, %eax
 ; KNL-NEXT:    vzeroupper
 ; KNL-NEXT:    retq
 ;
diff --git a/test/CodeGen/X86/movpc32-check.ll b/test/CodeGen/X86/movpc32-check.ll
index 3a8c404f6cc06591f47d5bc47919031b2e88ad49..7545b52d6d02de6dc5db09ba79811182db0b9a20 100644
--- a/test/CodeGen/X86/movpc32-check.ll
+++ b/test/CodeGen/X86/movpc32-check.ll
@@ -19,7 +19,7 @@ attributes #1 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-
 !llvm.module.flags = !{!7, !8, !9}
 !llvm.ident = !{!10}
 
-!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 3.8.0 (http://llvm.org/git/clang.git 3490ab8630d5643f71f1f04e46984f05b27b8d67) (http://llvm.org/git/llvm.git d2643e2ff955ed234944fe3c6b4ffc1250085843)", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !2)
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 3.8.0", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !2)
 !1 = !DIFile(filename: "test.c", directory: "movpc-test")
 !2 = !{}
 !4 = distinct !DISubprogram(name: "test", scope: !1, file: !1, line: 2, type: !5, isLocal: false, isDefinition: true, scopeLine: 3, isOptimized: false, unit: !0, retainedNodes: !2)
@@ -28,7 +28,7 @@ attributes #1 = { "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-
 !7 = !{i32 2, !"Dwarf Version", i32 4}
 !8 = !{i32 2, !"Debug Info Version", i32 3}
 !9 = !{i32 1, !"PIC Level", i32 2}
-!10 = !{!"clang version 3.8.0 (http://llvm.org/git/clang.git 3490ab8630d5643f71f1f04e46984f05b27b8d67) (http://llvm.org/git/llvm.git d2643e2ff955ed234944fe3c6b4ffc1250085843)"}
+!10 = !{!"clang version 3.8.0"}
 !11 = !DILocation(line: 4, column: 3, scope: !4)
 !12 = !DILocation(line: 5, column: 1, scope: !4)
 
diff --git a/test/CodeGen/X86/mul-constant-i32.ll b/test/CodeGen/X86/mul-constant-i32.ll
index 0ad16878b7bcf8aabdc90add3a1ed797dfc00084..3b2abf8c1f351772ad6a74c5c4fcc35e3c4ada0b 100644
--- a/test/CodeGen/X86/mul-constant-i32.ll
+++ b/test/CodeGen/X86/mul-constant-i32.ll
@@ -1,12 +1,12 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=i686-unknown | FileCheck %s --check-prefix=X86
-; RUN: llc < %s -mtriple=x86_64-unknown -print-schedule=true -mcpu=haswell| FileCheck %s --check-prefix=X64-HSW
-; RUN: llc < %s -mtriple=x86_64-unknown -print-schedule=true -mcpu=btver2| FileCheck %s --check-prefix=X64-JAG
-; RUN: llc < %s -mtriple=i686-unknown -mul-constant-optimization=false | FileCheck %s --check-prefix=X86-NOOPT
-; RUN: llc < %s -mtriple=x86_64-unknown -mul-constant-optimization=false -print-schedule=true -mcpu=haswell| FileCheck %s --check-prefix=HSW-NOOPT
-; RUN: llc < %s -mtriple=x86_64-unknown -mul-constant-optimization=false -print-schedule=true -mcpu=btver2| FileCheck %s --check-prefix=JAG-NOOPT
-; RUN: llc < %s -mtriple=x86_64-unknown -print-schedule=true -mcpu=slm| FileCheck %s --check-prefix=X64-SLM
-; RUN: llc < %s -mtriple=x86_64-unknown -mul-constant-optimization=false -print-schedule=true -mcpu=slm| FileCheck %s --check-prefix=SLM-NOOPT
+; RUN: llc < %s -mtriple=i686-unknown                                                  | FileCheck %s --check-prefix=X86
+; RUN: llc < %s -mtriple=x86_64-unknown -mcpu=haswell                                  | FileCheck %s --check-prefixes=X64,X64-HSW
+; RUN: llc < %s -mtriple=x86_64-unknown -mcpu=btver2                                   | FileCheck %s --check-prefixes=X64,X64-JAG
+; RUN: llc < %s -mtriple=i686-unknown -mul-constant-optimization=false                 | FileCheck %s --check-prefix=X86-NOOPT
+; RUN: llc < %s -mtriple=x86_64-unknown -mul-constant-optimization=false -mcpu=haswell | FileCheck %s --check-prefixes=X64-NOOPT,HSW-NOOPT
+; RUN: llc < %s -mtriple=x86_64-unknown -mul-constant-optimization=false -mcpu=btver2  | FileCheck %s --check-prefixes=X64-NOOPT,JAG-NOOPT
+; RUN: llc < %s -mtriple=x86_64-unknown -mcpu=slm                                      | FileCheck %s --check-prefixes=X64,X64-SLM
+; RUN: llc < %s -mtriple=x86_64-unknown -mul-constant-optimization=false -mcpu=slm     | FileCheck %s --check-prefixes=X64-NOOPT,SLM-NOOPT
 
 define i32 @test_mul_by_1(i32 %x) {
 ; X86-LABEL: test_mul_by_1:
@@ -14,40 +14,20 @@ define i32 @test_mul_by_1(i32 %x) {
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    retl
 ;
-; X64-HSW-LABEL: test_mul_by_1:
-; X64-HSW:       # %bb.0:
-; X64-HSW-NEXT:    movl %edi, %eax # sched: [1:0.25]
-; X64-HSW-NEXT:    retq # sched: [7:1.00]
-;
-; X64-JAG-LABEL: test_mul_by_1:
-; X64-JAG:       # %bb.0:
-; X64-JAG-NEXT:    movl %edi, %eax # sched: [1:0.50]
-; X64-JAG-NEXT:    retq # sched: [4:1.00]
+; X64-LABEL: test_mul_by_1:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    retq
 ;
 ; X86-NOOPT-LABEL: test_mul_by_1:
 ; X86-NOOPT:       # %bb.0:
 ; X86-NOOPT-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-NOOPT-NEXT:    retl
 ;
-; HSW-NOOPT-LABEL: test_mul_by_1:
-; HSW-NOOPT:       # %bb.0:
-; HSW-NOOPT-NEXT:    movl %edi, %eax # sched: [1:0.25]
-; HSW-NOOPT-NEXT:    retq # sched: [7:1.00]
-;
-; JAG-NOOPT-LABEL: test_mul_by_1:
-; JAG-NOOPT:       # %bb.0:
-; JAG-NOOPT-NEXT:    movl %edi, %eax # sched: [1:0.50]
-; JAG-NOOPT-NEXT:    retq # sched: [4:1.00]
-;
-; X64-SLM-LABEL: test_mul_by_1:
-; X64-SLM:       # %bb.0:
-; X64-SLM-NEXT:    movl %edi, %eax # sched: [1:0.50]
-; X64-SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SLM-NOOPT-LABEL: test_mul_by_1:
-; SLM-NOOPT:       # %bb.0:
-; SLM-NOOPT-NEXT:    movl %edi, %eax # sched: [1:0.50]
-; SLM-NOOPT-NEXT:    retq # sched: [4:1.00]
+; X64-NOOPT-LABEL: test_mul_by_1:
+; X64-NOOPT:       # %bb.0:
+; X64-NOOPT-NEXT:    movl %edi, %eax
+; X64-NOOPT-NEXT:    retq
   %mul = mul nsw i32 %x, 1
   ret i32 %mul
 }
@@ -59,17 +39,11 @@ define i32 @test_mul_by_2(i32 %x) {
 ; X86-NEXT:    addl %eax, %eax
 ; X86-NEXT:    retl
 ;
-; X64-HSW-LABEL: test_mul_by_2:
-; X64-HSW:       # %bb.0:
-; X64-HSW-NEXT:    # kill: def $edi killed $edi def $rdi
-; X64-HSW-NEXT:    leal (%rdi,%rdi), %eax # sched: [1:0.50]
-; X64-HSW-NEXT:    retq # sched: [7:1.00]
-;
-; X64-JAG-LABEL: test_mul_by_2:
-; X64-JAG:       # %bb.0:
-; X64-JAG-NEXT:    # kill: def $edi killed $edi def $rdi
-; X64-JAG-NEXT:    leal (%rdi,%rdi), %eax # sched: [1:0.50]
-; X64-JAG-NEXT:    retq # sched: [4:1.00]
+; X64-LABEL: test_mul_by_2:
+; X64:       # %bb.0:
+; X64-NEXT:    # kill: def $edi killed $edi def $rdi
+; X64-NEXT:    leal (%rdi,%rdi), %eax
+; X64-NEXT:    retq
 ;
 ; X86-NOOPT-LABEL: test_mul_by_2:
 ; X86-NOOPT:       # %bb.0:
@@ -77,29 +51,11 @@ define i32 @test_mul_by_2(i32 %x) {
 ; X86-NOOPT-NEXT:    addl %eax, %eax
 ; X86-NOOPT-NEXT:    retl
 ;
-; HSW-NOOPT-LABEL: test_mul_by_2:
-; HSW-NOOPT:       # %bb.0:
-; HSW-NOOPT-NEXT:    # kill: def $edi killed $edi def $rdi
-; HSW-NOOPT-NEXT:    leal (%rdi,%rdi), %eax # sched: [1:0.50]
-; HSW-NOOPT-NEXT:    retq # sched: [7:1.00]
-;
-; JAG-NOOPT-LABEL: test_mul_by_2:
-; JAG-NOOPT:       # %bb.0:
-; JAG-NOOPT-NEXT:    # kill: def $edi killed $edi def $rdi
-; JAG-NOOPT-NEXT:    leal (%rdi,%rdi), %eax # sched: [1:0.50]
-; JAG-NOOPT-NEXT:    retq # sched: [4:1.00]
-;
-; X64-SLM-LABEL: test_mul_by_2:
-; X64-SLM:       # %bb.0:
-; X64-SLM-NEXT:    # kill: def $edi killed $edi def $rdi
-; X64-SLM-NEXT:    leal (%rdi,%rdi), %eax # sched: [1:1.00]
-; X64-SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SLM-NOOPT-LABEL: test_mul_by_2:
-; SLM-NOOPT:       # %bb.0:
-; SLM-NOOPT-NEXT:    # kill: def $edi killed $edi def $rdi
-; SLM-NOOPT-NEXT:    leal (%rdi,%rdi), %eax # sched: [1:1.00]
-; SLM-NOOPT-NEXT:    retq # sched: [4:1.00]
+; X64-NOOPT-LABEL: test_mul_by_2:
+; X64-NOOPT:       # %bb.0:
+; X64-NOOPT-NEXT:    # kill: def $edi killed $edi def $rdi
+; X64-NOOPT-NEXT:    leal (%rdi,%rdi), %eax
+; X64-NOOPT-NEXT:    retq
   %mul = mul nsw i32 %x, 2
   ret i32 %mul
 }
@@ -111,46 +67,22 @@ define i32 @test_mul_by_3(i32 %x) {
 ; X86-NEXT:    leal (%eax,%eax,2), %eax
 ; X86-NEXT:    retl
 ;
-; X64-HSW-LABEL: test_mul_by_3:
-; X64-HSW:       # %bb.0:
-; X64-HSW-NEXT:    # kill: def $edi killed $edi def $rdi
-; X64-HSW-NEXT:    leal (%rdi,%rdi,2), %eax # sched: [1:0.50]
-; X64-HSW-NEXT:    retq # sched: [7:1.00]
-;
-; X64-JAG-LABEL: test_mul_by_3:
-; X64-JAG:       # %bb.0:
-; X64-JAG-NEXT:    # kill: def $edi killed $edi def $rdi
-; X64-JAG-NEXT:    leal (%rdi,%rdi,2), %eax # sched: [2:1.00]
-; X64-JAG-NEXT:    retq # sched: [4:1.00]
+; X64-LABEL: test_mul_by_3:
+; X64:       # %bb.0:
+; X64-NEXT:    # kill: def $edi killed $edi def $rdi
+; X64-NEXT:    leal (%rdi,%rdi,2), %eax
+; X64-NEXT:    retq
 ;
 ; X86-NOOPT-LABEL: test_mul_by_3:
 ; X86-NOOPT:       # %bb.0:
 ; X86-NOOPT-NEXT:    imull $3, {{[0-9]+}}(%esp), %eax
 ; X86-NOOPT-NEXT:    retl
 ;
-; HSW-NOOPT-LABEL: test_mul_by_3:
-; HSW-NOOPT:       # %bb.0:
-; HSW-NOOPT-NEXT:    # kill: def $edi killed $edi def $rdi
-; HSW-NOOPT-NEXT:    leal (%rdi,%rdi,2), %eax # sched: [1:0.50]
-; HSW-NOOPT-NEXT:    retq # sched: [7:1.00]
-;
-; JAG-NOOPT-LABEL: test_mul_by_3:
-; JAG-NOOPT:       # %bb.0:
-; JAG-NOOPT-NEXT:    # kill: def $edi killed $edi def $rdi
-; JAG-NOOPT-NEXT:    leal (%rdi,%rdi,2), %eax # sched: [2:1.00]
-; JAG-NOOPT-NEXT:    retq # sched: [4:1.00]
-;
-; X64-SLM-LABEL: test_mul_by_3:
-; X64-SLM:       # %bb.0:
-; X64-SLM-NEXT:    # kill: def $edi killed $edi def $rdi
-; X64-SLM-NEXT:    leal (%rdi,%rdi,2), %eax # sched: [1:1.00]
-; X64-SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SLM-NOOPT-LABEL: test_mul_by_3:
-; SLM-NOOPT:       # %bb.0:
-; SLM-NOOPT-NEXT:    # kill: def $edi killed $edi def $rdi
-; SLM-NOOPT-NEXT:    leal (%rdi,%rdi,2), %eax # sched: [1:1.00]
-; SLM-NOOPT-NEXT:    retq # sched: [4:1.00]
+; X64-NOOPT-LABEL: test_mul_by_3:
+; X64-NOOPT:       # %bb.0:
+; X64-NOOPT-NEXT:    # kill: def $edi killed $edi def $rdi
+; X64-NOOPT-NEXT:    leal (%rdi,%rdi,2), %eax
+; X64-NOOPT-NEXT:    retq
   %mul = mul nsw i32 %x, 3
   ret i32 %mul
 }
@@ -162,17 +94,11 @@ define i32 @test_mul_by_4(i32 %x) {
 ; X86-NEXT:    shll $2, %eax
 ; X86-NEXT:    retl
 ;
-; X64-HSW-LABEL: test_mul_by_4:
-; X64-HSW:       # %bb.0:
-; X64-HSW-NEXT:    # kill: def $edi killed $edi def $rdi
-; X64-HSW-NEXT:    leal (,%rdi,4), %eax # sched: [1:0.50]
-; X64-HSW-NEXT:    retq # sched: [7:1.00]
-;
-; X64-JAG-LABEL: test_mul_by_4:
-; X64-JAG:       # %bb.0:
-; X64-JAG-NEXT:    # kill: def $edi killed $edi def $rdi
-; X64-JAG-NEXT:    leal (,%rdi,4), %eax # sched: [2:1.00]
-; X64-JAG-NEXT:    retq # sched: [4:1.00]
+; X64-LABEL: test_mul_by_4:
+; X64:       # %bb.0:
+; X64-NEXT:    # kill: def $edi killed $edi def $rdi
+; X64-NEXT:    leal (,%rdi,4), %eax
+; X64-NEXT:    retq
 ;
 ; X86-NOOPT-LABEL: test_mul_by_4:
 ; X86-NOOPT:       # %bb.0:
@@ -180,29 +106,11 @@ define i32 @test_mul_by_4(i32 %x) {
 ; X86-NOOPT-NEXT:    shll $2, %eax
 ; X86-NOOPT-NEXT:    retl
 ;
-; HSW-NOOPT-LABEL: test_mul_by_4:
-; HSW-NOOPT:       # %bb.0:
-; HSW-NOOPT-NEXT:    # kill: def $edi killed $edi def $rdi
-; HSW-NOOPT-NEXT:    leal (,%rdi,4), %eax # sched: [1:0.50]
-; HSW-NOOPT-NEXT:    retq # sched: [7:1.00]
-;
-; JAG-NOOPT-LABEL: test_mul_by_4:
-; JAG-NOOPT:       # %bb.0:
-; JAG-NOOPT-NEXT:    # kill: def $edi killed $edi def $rdi
-; JAG-NOOPT-NEXT:    leal (,%rdi,4), %eax # sched: [2:1.00]
-; JAG-NOOPT-NEXT:    retq # sched: [4:1.00]
-;
-; X64-SLM-LABEL: test_mul_by_4:
-; X64-SLM:       # %bb.0:
-; X64-SLM-NEXT:    # kill: def $edi killed $edi def $rdi
-; X64-SLM-NEXT:    leal (,%rdi,4), %eax # sched: [1:1.00]
-; X64-SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SLM-NOOPT-LABEL: test_mul_by_4:
-; SLM-NOOPT:       # %bb.0:
-; SLM-NOOPT-NEXT:    # kill: def $edi killed $edi def $rdi
-; SLM-NOOPT-NEXT:    leal (,%rdi,4), %eax # sched: [1:1.00]
-; SLM-NOOPT-NEXT:    retq # sched: [4:1.00]
+; X64-NOOPT-LABEL: test_mul_by_4:
+; X64-NOOPT:       # %bb.0:
+; X64-NOOPT-NEXT:    # kill: def $edi killed $edi def $rdi
+; X64-NOOPT-NEXT:    leal (,%rdi,4), %eax
+; X64-NOOPT-NEXT:    retq
   %mul = mul nsw i32 %x, 4
   ret i32 %mul
 }
@@ -214,46 +122,22 @@ define i32 @test_mul_by_5(i32 %x) {
 ; X86-NEXT:    leal (%eax,%eax,4), %eax
 ; X86-NEXT:    retl
 ;
-; X64-HSW-LABEL: test_mul_by_5:
-; X64-HSW:       # %bb.0:
-; X64-HSW-NEXT:    # kill: def $edi killed $edi def $rdi
-; X64-HSW-NEXT:    leal (%rdi,%rdi,4), %eax # sched: [1:0.50]
-; X64-HSW-NEXT:    retq # sched: [7:1.00]
-;
-; X64-JAG-LABEL: test_mul_by_5:
-; X64-JAG:       # %bb.0:
-; X64-JAG-NEXT:    # kill: def $edi killed $edi def $rdi
-; X64-JAG-NEXT:    leal (%rdi,%rdi,4), %eax # sched: [2:1.00]
-; X64-JAG-NEXT:    retq # sched: [4:1.00]
+; X64-LABEL: test_mul_by_5:
+; X64:       # %bb.0:
+; X64-NEXT:    # kill: def $edi killed $edi def $rdi
+; X64-NEXT:    leal (%rdi,%rdi,4), %eax
+; X64-NEXT:    retq
 ;
 ; X86-NOOPT-LABEL: test_mul_by_5:
 ; X86-NOOPT:       # %bb.0:
 ; X86-NOOPT-NEXT:    imull $5, {{[0-9]+}}(%esp), %eax
 ; X86-NOOPT-NEXT:    retl
 ;
-; HSW-NOOPT-LABEL: test_mul_by_5:
-; HSW-NOOPT:       # %bb.0:
-; HSW-NOOPT-NEXT:    # kill: def $edi killed $edi def $rdi
-; HSW-NOOPT-NEXT:    leal (%rdi,%rdi,4), %eax # sched: [1:0.50]
-; HSW-NOOPT-NEXT:    retq # sched: [7:1.00]
-;
-; JAG-NOOPT-LABEL: test_mul_by_5:
-; JAG-NOOPT:       # %bb.0:
-; JAG-NOOPT-NEXT:    # kill: def $edi killed $edi def $rdi
-; JAG-NOOPT-NEXT:    leal (%rdi,%rdi,4), %eax # sched: [2:1.00]
-; JAG-NOOPT-NEXT:    retq # sched: [4:1.00]
-;
-; X64-SLM-LABEL: test_mul_by_5:
-; X64-SLM:       # %bb.0:
-; X64-SLM-NEXT:    # kill: def $edi killed $edi def $rdi
-; X64-SLM-NEXT:    leal (%rdi,%rdi,4), %eax # sched: [1:1.00]
-; X64-SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SLM-NOOPT-LABEL: test_mul_by_5:
-; SLM-NOOPT:       # %bb.0:
-; SLM-NOOPT-NEXT:    # kill: def $edi killed $edi def $rdi
-; SLM-NOOPT-NEXT:    leal (%rdi,%rdi,4), %eax # sched: [1:1.00]
-; SLM-NOOPT-NEXT:    retq # sched: [4:1.00]
+; X64-NOOPT-LABEL: test_mul_by_5:
+; X64-NOOPT:       # %bb.0:
+; X64-NOOPT-NEXT:    # kill: def $edi killed $edi def $rdi
+; X64-NOOPT-NEXT:    leal (%rdi,%rdi,4), %eax
+; X64-NOOPT-NEXT:    retq
   %mul = mul nsw i32 %x, 5
   ret i32 %mul
 }
@@ -266,46 +150,22 @@ define i32 @test_mul_by_6(i32 %x) {
 ; X86-NEXT:    leal (%eax,%eax,2), %eax
 ; X86-NEXT:    retl
 ;
-; X64-HSW-LABEL: test_mul_by_6:
-; X64-HSW:       # %bb.0:
-; X64-HSW-NEXT:    # kill: def $edi killed $edi def $rdi
-; X64-HSW-NEXT:    addl %edi, %edi # sched: [1:0.25]
-; X64-HSW-NEXT:    leal (%rdi,%rdi,2), %eax # sched: [1:0.50]
-; X64-HSW-NEXT:    retq # sched: [7:1.00]
-;
-; X64-JAG-LABEL: test_mul_by_6:
-; X64-JAG:       # %bb.0:
-; X64-JAG-NEXT:    # kill: def $edi killed $edi def $rdi
-; X64-JAG-NEXT:    addl %edi, %edi # sched: [1:0.50]
-; X64-JAG-NEXT:    leal (%rdi,%rdi,2), %eax # sched: [2:1.00]
-; X64-JAG-NEXT:    retq # sched: [4:1.00]
+; X64-LABEL: test_mul_by_6:
+; X64:       # %bb.0:
+; X64-NEXT:    # kill: def $edi killed $edi def $rdi
+; X64-NEXT:    addl %edi, %edi
+; X64-NEXT:    leal (%rdi,%rdi,2), %eax
+; X64-NEXT:    retq
 ;
 ; X86-NOOPT-LABEL: test_mul_by_6:
 ; X86-NOOPT:       # %bb.0:
 ; X86-NOOPT-NEXT:    imull $6, {{[0-9]+}}(%esp), %eax
 ; X86-NOOPT-NEXT:    retl
 ;
-; HSW-NOOPT-LABEL: test_mul_by_6:
-; HSW-NOOPT:       # %bb.0:
-; HSW-NOOPT-NEXT:    imull $6, %edi, %eax # sched: [3:1.00]
-; HSW-NOOPT-NEXT:    retq # sched: [7:1.00]
-;
-; JAG-NOOPT-LABEL: test_mul_by_6:
-; JAG-NOOPT:       # %bb.0:
-; JAG-NOOPT-NEXT:    imull $6, %edi, %eax # sched: [3:1.00]
-; JAG-NOOPT-NEXT:    retq # sched: [4:1.00]
-;
-; X64-SLM-LABEL: test_mul_by_6:
-; X64-SLM:       # %bb.0:
-; X64-SLM-NEXT:    # kill: def $edi killed $edi def $rdi
-; X64-SLM-NEXT:    addl %edi, %edi # sched: [1:0.50]
-; X64-SLM-NEXT:    leal (%rdi,%rdi,2), %eax # sched: [1:1.00]
-; X64-SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SLM-NOOPT-LABEL: test_mul_by_6:
-; SLM-NOOPT:       # %bb.0:
-; SLM-NOOPT-NEXT:    imull $6, %edi, %eax # sched: [3:1.00]
-; SLM-NOOPT-NEXT:    retq # sched: [4:1.00]
+; X64-NOOPT-LABEL: test_mul_by_6:
+; X64-NOOPT:       # %bb.0:
+; X64-NOOPT-NEXT:    imull $6, %edi, %eax
+; X64-NOOPT-NEXT:    retq
   %mul = mul nsw i32 %x, 6
   ret i32 %mul
 }
@@ -318,46 +178,22 @@ define i32 @test_mul_by_7(i32 %x) {
 ; X86-NEXT:    subl %ecx, %eax
 ; X86-NEXT:    retl
 ;
-; X64-HSW-LABEL: test_mul_by_7:
-; X64-HSW:       # %bb.0:
-; X64-HSW-NEXT:    # kill: def $edi killed $edi def $rdi
-; X64-HSW-NEXT:    leal (,%rdi,8), %eax # sched: [1:0.50]
-; X64-HSW-NEXT:    subl %edi, %eax # sched: [1:0.25]
-; X64-HSW-NEXT:    retq # sched: [7:1.00]
-;
-; X64-JAG-LABEL: test_mul_by_7:
-; X64-JAG:       # %bb.0:
-; X64-JAG-NEXT:    # kill: def $edi killed $edi def $rdi
-; X64-JAG-NEXT:    leal (,%rdi,8), %eax # sched: [2:1.00]
-; X64-JAG-NEXT:    subl %edi, %eax # sched: [1:0.50]
-; X64-JAG-NEXT:    retq # sched: [4:1.00]
+; X64-LABEL: test_mul_by_7:
+; X64:       # %bb.0:
+; X64-NEXT:    # kill: def $edi killed $edi def $rdi
+; X64-NEXT:    leal (,%rdi,8), %eax
+; X64-NEXT:    subl %edi, %eax
+; X64-NEXT:    retq
 ;
 ; X86-NOOPT-LABEL: test_mul_by_7:
 ; X86-NOOPT:       # %bb.0:
 ; X86-NOOPT-NEXT:    imull $7, {{[0-9]+}}(%esp), %eax
 ; X86-NOOPT-NEXT:    retl
 ;
-; HSW-NOOPT-LABEL: test_mul_by_7:
-; HSW-NOOPT:       # %bb.0:
-; HSW-NOOPT-NEXT:    imull $7, %edi, %eax # sched: [3:1.00]
-; HSW-NOOPT-NEXT:    retq # sched: [7:1.00]
-;
-; JAG-NOOPT-LABEL: test_mul_by_7:
-; JAG-NOOPT:       # %bb.0:
-; JAG-NOOPT-NEXT:    imull $7, %edi, %eax # sched: [3:1.00]
-; JAG-NOOPT-NEXT:    retq # sched: [4:1.00]
-;
-; X64-SLM-LABEL: test_mul_by_7:
-; X64-SLM:       # %bb.0:
-; X64-SLM-NEXT:    # kill: def $edi killed $edi def $rdi
-; X64-SLM-NEXT:    leal (,%rdi,8), %eax # sched: [1:1.00]
-; X64-SLM-NEXT:    subl %edi, %eax # sched: [1:0.50]
-; X64-SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SLM-NOOPT-LABEL: test_mul_by_7:
-; SLM-NOOPT:       # %bb.0:
-; SLM-NOOPT-NEXT:    imull $7, %edi, %eax # sched: [3:1.00]
-; SLM-NOOPT-NEXT:    retq # sched: [4:1.00]
+; X64-NOOPT-LABEL: test_mul_by_7:
+; X64-NOOPT:       # %bb.0:
+; X64-NOOPT-NEXT:    imull $7, %edi, %eax
+; X64-NOOPT-NEXT:    retq
   %mul = mul nsw i32 %x, 7
   ret i32 %mul
 }
@@ -369,17 +205,11 @@ define i32 @test_mul_by_8(i32 %x) {
 ; X86-NEXT:    shll $3, %eax
 ; X86-NEXT:    retl
 ;
-; X64-HSW-LABEL: test_mul_by_8:
-; X64-HSW:       # %bb.0:
-; X64-HSW-NEXT:    # kill: def $edi killed $edi def $rdi
-; X64-HSW-NEXT:    leal (,%rdi,8), %eax # sched: [1:0.50]
-; X64-HSW-NEXT:    retq # sched: [7:1.00]
-;
-; X64-JAG-LABEL: test_mul_by_8:
-; X64-JAG:       # %bb.0:
-; X64-JAG-NEXT:    # kill: def $edi killed $edi def $rdi
-; X64-JAG-NEXT:    leal (,%rdi,8), %eax # sched: [2:1.00]
-; X64-JAG-NEXT:    retq # sched: [4:1.00]
+; X64-LABEL: test_mul_by_8:
+; X64:       # %bb.0:
+; X64-NEXT:    # kill: def $edi killed $edi def $rdi
+; X64-NEXT:    leal (,%rdi,8), %eax
+; X64-NEXT:    retq
 ;
 ; X86-NOOPT-LABEL: test_mul_by_8:
 ; X86-NOOPT:       # %bb.0:
@@ -387,29 +217,11 @@ define i32 @test_mul_by_8(i32 %x) {
 ; X86-NOOPT-NEXT:    shll $3, %eax
 ; X86-NOOPT-NEXT:    retl
 ;
-; HSW-NOOPT-LABEL: test_mul_by_8:
-; HSW-NOOPT:       # %bb.0:
-; HSW-NOOPT-NEXT:    # kill: def $edi killed $edi def $rdi
-; HSW-NOOPT-NEXT:    leal (,%rdi,8), %eax # sched: [1:0.50]
-; HSW-NOOPT-NEXT:    retq # sched: [7:1.00]
-;
-; JAG-NOOPT-LABEL: test_mul_by_8:
-; JAG-NOOPT:       # %bb.0:
-; JAG-NOOPT-NEXT:    # kill: def $edi killed $edi def $rdi
-; JAG-NOOPT-NEXT:    leal (,%rdi,8), %eax # sched: [2:1.00]
-; JAG-NOOPT-NEXT:    retq # sched: [4:1.00]
-;
-; X64-SLM-LABEL: test_mul_by_8:
-; X64-SLM:       # %bb.0:
-; X64-SLM-NEXT:    # kill: def $edi killed $edi def $rdi
-; X64-SLM-NEXT:    leal (,%rdi,8), %eax # sched: [1:1.00]
-; X64-SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SLM-NOOPT-LABEL: test_mul_by_8:
-; SLM-NOOPT:       # %bb.0:
-; SLM-NOOPT-NEXT:    # kill: def $edi killed $edi def $rdi
-; SLM-NOOPT-NEXT:    leal (,%rdi,8), %eax # sched: [1:1.00]
-; SLM-NOOPT-NEXT:    retq # sched: [4:1.00]
+; X64-NOOPT-LABEL: test_mul_by_8:
+; X64-NOOPT:       # %bb.0:
+; X64-NOOPT-NEXT:    # kill: def $edi killed $edi def $rdi
+; X64-NOOPT-NEXT:    leal (,%rdi,8), %eax
+; X64-NOOPT-NEXT:    retq
   %mul = mul nsw i32 %x, 8
   ret i32 %mul
 }
@@ -421,46 +233,22 @@ define i32 @test_mul_by_9(i32 %x) {
 ; X86-NEXT:    leal (%eax,%eax,8), %eax
 ; X86-NEXT:    retl
 ;
-; X64-HSW-LABEL: test_mul_by_9:
-; X64-HSW:       # %bb.0:
-; X64-HSW-NEXT:    # kill: def $edi killed $edi def $rdi
-; X64-HSW-NEXT:    leal (%rdi,%rdi,8), %eax # sched: [1:0.50]
-; X64-HSW-NEXT:    retq # sched: [7:1.00]
-;
-; X64-JAG-LABEL: test_mul_by_9:
-; X64-JAG:       # %bb.0:
-; X64-JAG-NEXT:    # kill: def $edi killed $edi def $rdi
-; X64-JAG-NEXT:    leal (%rdi,%rdi,8), %eax # sched: [2:1.00]
-; X64-JAG-NEXT:    retq # sched: [4:1.00]
+; X64-LABEL: test_mul_by_9:
+; X64:       # %bb.0:
+; X64-NEXT:    # kill: def $edi killed $edi def $rdi
+; X64-NEXT:    leal (%rdi,%rdi,8), %eax
+; X64-NEXT:    retq
 ;
 ; X86-NOOPT-LABEL: test_mul_by_9:
 ; X86-NOOPT:       # %bb.0:
 ; X86-NOOPT-NEXT:    imull $9, {{[0-9]+}}(%esp), %eax
 ; X86-NOOPT-NEXT:    retl
 ;
-; HSW-NOOPT-LABEL: test_mul_by_9:
-; HSW-NOOPT:       # %bb.0:
-; HSW-NOOPT-NEXT:    # kill: def $edi killed $edi def $rdi
-; HSW-NOOPT-NEXT:    leal (%rdi,%rdi,8), %eax # sched: [1:0.50]
-; HSW-NOOPT-NEXT:    retq # sched: [7:1.00]
-;
-; JAG-NOOPT-LABEL: test_mul_by_9:
-; JAG-NOOPT:       # %bb.0:
-; JAG-NOOPT-NEXT:    # kill: def $edi killed $edi def $rdi
-; JAG-NOOPT-NEXT:    leal (%rdi,%rdi,8), %eax # sched: [2:1.00]
-; JAG-NOOPT-NEXT:    retq # sched: [4:1.00]
-;
-; X64-SLM-LABEL: test_mul_by_9:
-; X64-SLM:       # %bb.0:
-; X64-SLM-NEXT:    # kill: def $edi killed $edi def $rdi
-; X64-SLM-NEXT:    leal (%rdi,%rdi,8), %eax # sched: [1:1.00]
-; X64-SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SLM-NOOPT-LABEL: test_mul_by_9:
-; SLM-NOOPT:       # %bb.0:
-; SLM-NOOPT-NEXT:    # kill: def $edi killed $edi def $rdi
-; SLM-NOOPT-NEXT:    leal (%rdi,%rdi,8), %eax # sched: [1:1.00]
-; SLM-NOOPT-NEXT:    retq # sched: [4:1.00]
+; X64-NOOPT-LABEL: test_mul_by_9:
+; X64-NOOPT:       # %bb.0:
+; X64-NOOPT-NEXT:    # kill: def $edi killed $edi def $rdi
+; X64-NOOPT-NEXT:    leal (%rdi,%rdi,8), %eax
+; X64-NOOPT-NEXT:    retq
   %mul = mul nsw i32 %x, 9
   ret i32 %mul
 }
@@ -473,46 +261,22 @@ define i32 @test_mul_by_10(i32 %x) {
 ; X86-NEXT:    leal (%eax,%eax,4), %eax
 ; X86-NEXT:    retl
 ;
-; X64-HSW-LABEL: test_mul_by_10:
-; X64-HSW:       # %bb.0:
-; X64-HSW-NEXT:    # kill: def $edi killed $edi def $rdi
-; X64-HSW-NEXT:    addl %edi, %edi # sched: [1:0.25]
-; X64-HSW-NEXT:    leal (%rdi,%rdi,4), %eax # sched: [1:0.50]
-; X64-HSW-NEXT:    retq # sched: [7:1.00]
-;
-; X64-JAG-LABEL: test_mul_by_10:
-; X64-JAG:       # %bb.0:
-; X64-JAG-NEXT:    # kill: def $edi killed $edi def $rdi
-; X64-JAG-NEXT:    addl %edi, %edi # sched: [1:0.50]
-; X64-JAG-NEXT:    leal (%rdi,%rdi,4), %eax # sched: [2:1.00]
-; X64-JAG-NEXT:    retq # sched: [4:1.00]
+; X64-LABEL: test_mul_by_10:
+; X64:       # %bb.0:
+; X64-NEXT:    # kill: def $edi killed $edi def $rdi
+; X64-NEXT:    addl %edi, %edi
+; X64-NEXT:    leal (%rdi,%rdi,4), %eax
+; X64-NEXT:    retq
 ;
 ; X86-NOOPT-LABEL: test_mul_by_10:
 ; X86-NOOPT:       # %bb.0:
 ; X86-NOOPT-NEXT:    imull $10, {{[0-9]+}}(%esp), %eax
 ; X86-NOOPT-NEXT:    retl
 ;
-; HSW-NOOPT-LABEL: test_mul_by_10:
-; HSW-NOOPT:       # %bb.0:
-; HSW-NOOPT-NEXT:    imull $10, %edi, %eax # sched: [3:1.00]
-; HSW-NOOPT-NEXT:    retq # sched: [7:1.00]
-;
-; JAG-NOOPT-LABEL: test_mul_by_10:
-; JAG-NOOPT:       # %bb.0:
-; JAG-NOOPT-NEXT:    imull $10, %edi, %eax # sched: [3:1.00]
-; JAG-NOOPT-NEXT:    retq # sched: [4:1.00]
-;
-; X64-SLM-LABEL: test_mul_by_10:
-; X64-SLM:       # %bb.0:
-; X64-SLM-NEXT:    # kill: def $edi killed $edi def $rdi
-; X64-SLM-NEXT:    addl %edi, %edi # sched: [1:0.50]
-; X64-SLM-NEXT:    leal (%rdi,%rdi,4), %eax # sched: [1:1.00]
-; X64-SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SLM-NOOPT-LABEL: test_mul_by_10:
-; SLM-NOOPT:       # %bb.0:
-; SLM-NOOPT-NEXT:    imull $10, %edi, %eax # sched: [3:1.00]
-; SLM-NOOPT-NEXT:    retq # sched: [4:1.00]
+; X64-NOOPT-LABEL: test_mul_by_10:
+; X64-NOOPT:       # %bb.0:
+; X64-NOOPT-NEXT:    imull $10, %edi, %eax
+; X64-NOOPT-NEXT:    retq
   %mul = mul nsw i32 %x, 10
   ret i32 %mul
 }
@@ -528,41 +292,31 @@ define i32 @test_mul_by_11(i32 %x) {
 ; X64-HSW-LABEL: test_mul_by_11:
 ; X64-HSW:       # %bb.0:
 ; X64-HSW-NEXT:    # kill: def $edi killed $edi def $rdi
-; X64-HSW-NEXT:    leal (%rdi,%rdi,4), %eax # sched: [1:0.50]
-; X64-HSW-NEXT:    leal (%rdi,%rax,2), %eax # sched: [1:0.50]
-; X64-HSW-NEXT:    retq # sched: [7:1.00]
+; X64-HSW-NEXT:    leal (%rdi,%rdi,4), %eax
+; X64-HSW-NEXT:    leal (%rdi,%rax,2), %eax
+; X64-HSW-NEXT:    retq
 ;
 ; X64-JAG-LABEL: test_mul_by_11:
 ; X64-JAG:       # %bb.0:
 ; X64-JAG-NEXT:    # kill: def $edi killed $edi def $rdi
-; X64-JAG-NEXT:    leal (%rdi,%rdi,4), %eax # sched: [2:1.00]
-; X64-JAG-NEXT:    leal (%rdi,%rax,2), %eax # sched: [2:1.00]
-; X64-JAG-NEXT:    retq # sched: [4:1.00]
+; X64-JAG-NEXT:    leal (%rdi,%rdi,4), %eax
+; X64-JAG-NEXT:    leal (%rdi,%rax,2), %eax
+; X64-JAG-NEXT:    retq
 ;
 ; X86-NOOPT-LABEL: test_mul_by_11:
 ; X86-NOOPT:       # %bb.0:
 ; X86-NOOPT-NEXT:    imull $11, {{[0-9]+}}(%esp), %eax
 ; X86-NOOPT-NEXT:    retl
 ;
-; HSW-NOOPT-LABEL: test_mul_by_11:
-; HSW-NOOPT:       # %bb.0:
-; HSW-NOOPT-NEXT:    imull $11, %edi, %eax # sched: [3:1.00]
-; HSW-NOOPT-NEXT:    retq # sched: [7:1.00]
-;
-; JAG-NOOPT-LABEL: test_mul_by_11:
-; JAG-NOOPT:       # %bb.0:
-; JAG-NOOPT-NEXT:    imull $11, %edi, %eax # sched: [3:1.00]
-; JAG-NOOPT-NEXT:    retq # sched: [4:1.00]
+; X64-NOOPT-LABEL: test_mul_by_11:
+; X64-NOOPT:       # %bb.0:
+; X64-NOOPT-NEXT:    imull $11, %edi, %eax
+; X64-NOOPT-NEXT:    retq
 ;
 ; X64-SLM-LABEL: test_mul_by_11:
 ; X64-SLM:       # %bb.0:
-; X64-SLM-NEXT:    imull $11, %edi, %eax # sched: [3:1.00]
-; X64-SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SLM-NOOPT-LABEL: test_mul_by_11:
-; SLM-NOOPT:       # %bb.0:
-; SLM-NOOPT-NEXT:    imull $11, %edi, %eax # sched: [3:1.00]
-; SLM-NOOPT-NEXT:    retq # sched: [4:1.00]
+; X64-SLM-NEXT:    imull $11, %edi, %eax
+; X64-SLM-NEXT:    retq
   %mul = mul nsw i32 %x, 11
   ret i32 %mul
 }
@@ -575,46 +329,22 @@ define i32 @test_mul_by_12(i32 %x) {
 ; X86-NEXT:    leal (%eax,%eax,2), %eax
 ; X86-NEXT:    retl
 ;
-; X64-HSW-LABEL: test_mul_by_12:
-; X64-HSW:       # %bb.0:
-; X64-HSW-NEXT:    # kill: def $edi killed $edi def $rdi
-; X64-HSW-NEXT:    shll $2, %edi # sched: [1:0.50]
-; X64-HSW-NEXT:    leal (%rdi,%rdi,2), %eax # sched: [1:0.50]
-; X64-HSW-NEXT:    retq # sched: [7:1.00]
-;
-; X64-JAG-LABEL: test_mul_by_12:
-; X64-JAG:       # %bb.0:
-; X64-JAG-NEXT:    # kill: def $edi killed $edi def $rdi
-; X64-JAG-NEXT:    shll $2, %edi # sched: [1:0.50]
-; X64-JAG-NEXT:    leal (%rdi,%rdi,2), %eax # sched: [2:1.00]
-; X64-JAG-NEXT:    retq # sched: [4:1.00]
+; X64-LABEL: test_mul_by_12:
+; X64:       # %bb.0:
+; X64-NEXT:    # kill: def $edi killed $edi def $rdi
+; X64-NEXT:    shll $2, %edi
+; X64-NEXT:    leal (%rdi,%rdi,2), %eax
+; X64-NEXT:    retq
 ;
 ; X86-NOOPT-LABEL: test_mul_by_12:
 ; X86-NOOPT:       # %bb.0:
 ; X86-NOOPT-NEXT:    imull $12, {{[0-9]+}}(%esp), %eax
 ; X86-NOOPT-NEXT:    retl
 ;
-; HSW-NOOPT-LABEL: test_mul_by_12:
-; HSW-NOOPT:       # %bb.0:
-; HSW-NOOPT-NEXT:    imull $12, %edi, %eax # sched: [3:1.00]
-; HSW-NOOPT-NEXT:    retq # sched: [7:1.00]
-;
-; JAG-NOOPT-LABEL: test_mul_by_12:
-; JAG-NOOPT:       # %bb.0:
-; JAG-NOOPT-NEXT:    imull $12, %edi, %eax # sched: [3:1.00]
-; JAG-NOOPT-NEXT:    retq # sched: [4:1.00]
-;
-; X64-SLM-LABEL: test_mul_by_12:
-; X64-SLM:       # %bb.0:
-; X64-SLM-NEXT:    # kill: def $edi killed $edi def $rdi
-; X64-SLM-NEXT:    shll $2, %edi # sched: [1:1.00]
-; X64-SLM-NEXT:    leal (%rdi,%rdi,2), %eax # sched: [1:1.00]
-; X64-SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SLM-NOOPT-LABEL: test_mul_by_12:
-; SLM-NOOPT:       # %bb.0:
-; SLM-NOOPT-NEXT:    imull $12, %edi, %eax # sched: [3:1.00]
-; SLM-NOOPT-NEXT:    retq # sched: [4:1.00]
+; X64-NOOPT-LABEL: test_mul_by_12:
+; X64-NOOPT:       # %bb.0:
+; X64-NOOPT-NEXT:    imull $12, %edi, %eax
+; X64-NOOPT-NEXT:    retq
   %mul = mul nsw i32 %x, 12
   ret i32 %mul
 }
@@ -630,41 +360,31 @@ define i32 @test_mul_by_13(i32 %x) {
 ; X64-HSW-LABEL: test_mul_by_13:
 ; X64-HSW:       # %bb.0:
 ; X64-HSW-NEXT:    # kill: def $edi killed $edi def $rdi
-; X64-HSW-NEXT:    leal (%rdi,%rdi,2), %eax # sched: [1:0.50]
-; X64-HSW-NEXT:    leal (%rdi,%rax,4), %eax # sched: [1:0.50]
-; X64-HSW-NEXT:    retq # sched: [7:1.00]
+; X64-HSW-NEXT:    leal (%rdi,%rdi,2), %eax
+; X64-HSW-NEXT:    leal (%rdi,%rax,4), %eax
+; X64-HSW-NEXT:    retq
 ;
 ; X64-JAG-LABEL: test_mul_by_13:
 ; X64-JAG:       # %bb.0:
 ; X64-JAG-NEXT:    # kill: def $edi killed $edi def $rdi
-; X64-JAG-NEXT:    leal (%rdi,%rdi,2), %eax # sched: [2:1.00]
-; X64-JAG-NEXT:    leal (%rdi,%rax,4), %eax # sched: [2:1.00]
-; X64-JAG-NEXT:    retq # sched: [4:1.00]
+; X64-JAG-NEXT:    leal (%rdi,%rdi,2), %eax
+; X64-JAG-NEXT:    leal (%rdi,%rax,4), %eax
+; X64-JAG-NEXT:    retq
 ;
 ; X86-NOOPT-LABEL: test_mul_by_13:
 ; X86-NOOPT:       # %bb.0:
 ; X86-NOOPT-NEXT:    imull $13, {{[0-9]+}}(%esp), %eax
 ; X86-NOOPT-NEXT:    retl
 ;
-; HSW-NOOPT-LABEL: test_mul_by_13:
-; HSW-NOOPT:       # %bb.0:
-; HSW-NOOPT-NEXT:    imull $13, %edi, %eax # sched: [3:1.00]
-; HSW-NOOPT-NEXT:    retq # sched: [7:1.00]
-;
-; JAG-NOOPT-LABEL: test_mul_by_13:
-; JAG-NOOPT:       # %bb.0:
-; JAG-NOOPT-NEXT:    imull $13, %edi, %eax # sched: [3:1.00]
-; JAG-NOOPT-NEXT:    retq # sched: [4:1.00]
+; X64-NOOPT-LABEL: test_mul_by_13:
+; X64-NOOPT:       # %bb.0:
+; X64-NOOPT-NEXT:    imull $13, %edi, %eax
+; X64-NOOPT-NEXT:    retq
 ;
 ; X64-SLM-LABEL: test_mul_by_13:
 ; X64-SLM:       # %bb.0:
-; X64-SLM-NEXT:    imull $13, %edi, %eax # sched: [3:1.00]
-; X64-SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SLM-NOOPT-LABEL: test_mul_by_13:
-; SLM-NOOPT:       # %bb.0:
-; SLM-NOOPT-NEXT:    imull $13, %edi, %eax # sched: [3:1.00]
-; SLM-NOOPT-NEXT:    retq # sched: [4:1.00]
+; X64-SLM-NEXT:    imull $13, %edi, %eax
+; X64-SLM-NEXT:    retq
   %mul = mul nsw i32 %x, 13
   ret i32 %mul
 }
@@ -679,49 +399,23 @@ define i32 @test_mul_by_14(i32 %x) {
 ; X86-NEXT:    subl %ecx, %eax
 ; X86-NEXT:    retl
 ;
-; X64-HSW-LABEL: test_mul_by_14:
-; X64-HSW:       # %bb.0:
-; X64-HSW-NEXT:    movl %edi, %eax # sched: [1:0.25]
-; X64-HSW-NEXT:    shll $4, %eax # sched: [1:0.50]
-; X64-HSW-NEXT:    subl %edi, %eax # sched: [1:0.25]
-; X64-HSW-NEXT:    subl %edi, %eax # sched: [1:0.25]
-; X64-HSW-NEXT:    retq # sched: [7:1.00]
-;
-; X64-JAG-LABEL: test_mul_by_14:
-; X64-JAG:       # %bb.0:
-; X64-JAG-NEXT:    movl %edi, %eax # sched: [1:0.50]
-; X64-JAG-NEXT:    shll $4, %eax # sched: [1:0.50]
-; X64-JAG-NEXT:    subl %edi, %eax # sched: [1:0.50]
-; X64-JAG-NEXT:    subl %edi, %eax # sched: [1:0.50]
-; X64-JAG-NEXT:    retq # sched: [4:1.00]
+; X64-LABEL: test_mul_by_14:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    shll $4, %eax
+; X64-NEXT:    subl %edi, %eax
+; X64-NEXT:    subl %edi, %eax
+; X64-NEXT:    retq
 ;
 ; X86-NOOPT-LABEL: test_mul_by_14:
 ; X86-NOOPT:       # %bb.0:
 ; X86-NOOPT-NEXT:    imull $14, {{[0-9]+}}(%esp), %eax
 ; X86-NOOPT-NEXT:    retl
 ;
-; HSW-NOOPT-LABEL: test_mul_by_14:
-; HSW-NOOPT:       # %bb.0:
-; HSW-NOOPT-NEXT:    imull $14, %edi, %eax # sched: [3:1.00]
-; HSW-NOOPT-NEXT:    retq # sched: [7:1.00]
-;
-; JAG-NOOPT-LABEL: test_mul_by_14:
-; JAG-NOOPT:       # %bb.0:
-; JAG-NOOPT-NEXT:    imull $14, %edi, %eax # sched: [3:1.00]
-; JAG-NOOPT-NEXT:    retq # sched: [4:1.00]
-;
-; X64-SLM-LABEL: test_mul_by_14:
-; X64-SLM:       # %bb.0:
-; X64-SLM-NEXT:    movl %edi, %eax # sched: [1:0.50]
-; X64-SLM-NEXT:    shll $4, %eax # sched: [1:1.00]
-; X64-SLM-NEXT:    subl %edi, %eax # sched: [1:0.50]
-; X64-SLM-NEXT:    subl %edi, %eax # sched: [1:0.50]
-; X64-SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SLM-NOOPT-LABEL: test_mul_by_14:
-; SLM-NOOPT:       # %bb.0:
-; SLM-NOOPT-NEXT:    imull $14, %edi, %eax # sched: [3:1.00]
-; SLM-NOOPT-NEXT:    retq # sched: [4:1.00]
+; X64-NOOPT-LABEL: test_mul_by_14:
+; X64-NOOPT:       # %bb.0:
+; X64-NOOPT-NEXT:    imull $14, %edi, %eax
+; X64-NOOPT-NEXT:    retq
   %mul = mul nsw i32 %x, 14
   ret i32 %mul
 }
@@ -734,46 +428,22 @@ define i32 @test_mul_by_15(i32 %x) {
 ; X86-NEXT:    leal (%eax,%eax,2), %eax
 ; X86-NEXT:    retl
 ;
-; X64-HSW-LABEL: test_mul_by_15:
-; X64-HSW:       # %bb.0:
-; X64-HSW-NEXT:    # kill: def $edi killed $edi def $rdi
-; X64-HSW-NEXT:    leal (%rdi,%rdi,4), %eax # sched: [1:0.50]
-; X64-HSW-NEXT:    leal (%rax,%rax,2), %eax # sched: [1:0.50]
-; X64-HSW-NEXT:    retq # sched: [7:1.00]
-;
-; X64-JAG-LABEL: test_mul_by_15:
-; X64-JAG:       # %bb.0:
-; X64-JAG-NEXT:    # kill: def $edi killed $edi def $rdi
-; X64-JAG-NEXT:    leal (%rdi,%rdi,4), %eax # sched: [2:1.00]
-; X64-JAG-NEXT:    leal (%rax,%rax,2), %eax # sched: [2:1.00]
-; X64-JAG-NEXT:    retq # sched: [4:1.00]
+; X64-LABEL: test_mul_by_15:
+; X64:       # %bb.0:
+; X64-NEXT:    # kill: def $edi killed $edi def $rdi
+; X64-NEXT:    leal (%rdi,%rdi,4), %eax
+; X64-NEXT:    leal (%rax,%rax,2), %eax
+; X64-NEXT:    retq
 ;
 ; X86-NOOPT-LABEL: test_mul_by_15:
 ; X86-NOOPT:       # %bb.0:
 ; X86-NOOPT-NEXT:    imull $15, {{[0-9]+}}(%esp), %eax
 ; X86-NOOPT-NEXT:    retl
 ;
-; HSW-NOOPT-LABEL: test_mul_by_15:
-; HSW-NOOPT:       # %bb.0:
-; HSW-NOOPT-NEXT:    imull $15, %edi, %eax # sched: [3:1.00]
-; HSW-NOOPT-NEXT:    retq # sched: [7:1.00]
-;
-; JAG-NOOPT-LABEL: test_mul_by_15:
-; JAG-NOOPT:       # %bb.0:
-; JAG-NOOPT-NEXT:    imull $15, %edi, %eax # sched: [3:1.00]
-; JAG-NOOPT-NEXT:    retq # sched: [4:1.00]
-;
-; X64-SLM-LABEL: test_mul_by_15:
-; X64-SLM:       # %bb.0:
-; X64-SLM-NEXT:    # kill: def $edi killed $edi def $rdi
-; X64-SLM-NEXT:    leal (%rdi,%rdi,4), %eax # sched: [1:1.00]
-; X64-SLM-NEXT:    leal (%rax,%rax,2), %eax # sched: [1:1.00]
-; X64-SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SLM-NOOPT-LABEL: test_mul_by_15:
-; SLM-NOOPT:       # %bb.0:
-; SLM-NOOPT-NEXT:    imull $15, %edi, %eax # sched: [3:1.00]
-; SLM-NOOPT-NEXT:    retq # sched: [4:1.00]
+; X64-NOOPT-LABEL: test_mul_by_15:
+; X64-NOOPT:       # %bb.0:
+; X64-NOOPT-NEXT:    imull $15, %edi, %eax
+; X64-NOOPT-NEXT:    retq
   %mul = mul nsw i32 %x, 15
   ret i32 %mul
 }
@@ -785,17 +455,11 @@ define i32 @test_mul_by_16(i32 %x) {
 ; X86-NEXT:    shll $4, %eax
 ; X86-NEXT:    retl
 ;
-; X64-HSW-LABEL: test_mul_by_16:
-; X64-HSW:       # %bb.0:
-; X64-HSW-NEXT:    movl %edi, %eax # sched: [1:0.25]
-; X64-HSW-NEXT:    shll $4, %eax # sched: [1:0.50]
-; X64-HSW-NEXT:    retq # sched: [7:1.00]
-;
-; X64-JAG-LABEL: test_mul_by_16:
-; X64-JAG:       # %bb.0:
-; X64-JAG-NEXT:    movl %edi, %eax # sched: [1:0.50]
-; X64-JAG-NEXT:    shll $4, %eax # sched: [1:0.50]
-; X64-JAG-NEXT:    retq # sched: [4:1.00]
+; X64-LABEL: test_mul_by_16:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    shll $4, %eax
+; X64-NEXT:    retq
 ;
 ; X86-NOOPT-LABEL: test_mul_by_16:
 ; X86-NOOPT:       # %bb.0:
@@ -803,29 +467,11 @@ define i32 @test_mul_by_16(i32 %x) {
 ; X86-NOOPT-NEXT:    shll $4, %eax
 ; X86-NOOPT-NEXT:    retl
 ;
-; HSW-NOOPT-LABEL: test_mul_by_16:
-; HSW-NOOPT:       # %bb.0:
-; HSW-NOOPT-NEXT:    movl %edi, %eax # sched: [1:0.25]
-; HSW-NOOPT-NEXT:    shll $4, %eax # sched: [1:0.50]
-; HSW-NOOPT-NEXT:    retq # sched: [7:1.00]
-;
-; JAG-NOOPT-LABEL: test_mul_by_16:
-; JAG-NOOPT:       # %bb.0:
-; JAG-NOOPT-NEXT:    movl %edi, %eax # sched: [1:0.50]
-; JAG-NOOPT-NEXT:    shll $4, %eax # sched: [1:0.50]
-; JAG-NOOPT-NEXT:    retq # sched: [4:1.00]
-;
-; X64-SLM-LABEL: test_mul_by_16:
-; X64-SLM:       # %bb.0:
-; X64-SLM-NEXT:    movl %edi, %eax # sched: [1:0.50]
-; X64-SLM-NEXT:    shll $4, %eax # sched: [1:1.00]
-; X64-SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SLM-NOOPT-LABEL: test_mul_by_16:
-; SLM-NOOPT:       # %bb.0:
-; SLM-NOOPT-NEXT:    movl %edi, %eax # sched: [1:0.50]
-; SLM-NOOPT-NEXT:    shll $4, %eax # sched: [1:1.00]
-; SLM-NOOPT-NEXT:    retq # sched: [4:1.00]
+; X64-NOOPT-LABEL: test_mul_by_16:
+; X64-NOOPT:       # %bb.0:
+; X64-NOOPT-NEXT:    movl %edi, %eax
+; X64-NOOPT-NEXT:    shll $4, %eax
+; X64-NOOPT-NEXT:    retq
   %mul = mul nsw i32 %x, 16
   ret i32 %mul
 }
@@ -839,49 +485,23 @@ define i32 @test_mul_by_17(i32 %x) {
 ; X86-NEXT:    addl %ecx, %eax
 ; X86-NEXT:    retl
 ;
-; X64-HSW-LABEL: test_mul_by_17:
-; X64-HSW:       # %bb.0:
-; X64-HSW-NEXT:    # kill: def $edi killed $edi def $rdi
-; X64-HSW-NEXT:    movl %edi, %eax # sched: [1:0.25]
-; X64-HSW-NEXT:    shll $4, %eax # sched: [1:0.50]
-; X64-HSW-NEXT:    leal (%rax,%rdi), %eax # sched: [1:0.50]
-; X64-HSW-NEXT:    retq # sched: [7:1.00]
-;
-; X64-JAG-LABEL: test_mul_by_17:
-; X64-JAG:       # %bb.0:
-; X64-JAG-NEXT:    # kill: def $edi killed $edi def $rdi
-; X64-JAG-NEXT:    movl %edi, %eax # sched: [1:0.50]
-; X64-JAG-NEXT:    shll $4, %eax # sched: [1:0.50]
-; X64-JAG-NEXT:    leal (%rax,%rdi), %eax # sched: [1:0.50]
-; X64-JAG-NEXT:    retq # sched: [4:1.00]
+; X64-LABEL: test_mul_by_17:
+; X64:       # %bb.0:
+; X64-NEXT:    # kill: def $edi killed $edi def $rdi
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    shll $4, %eax
+; X64-NEXT:    leal (%rax,%rdi), %eax
+; X64-NEXT:    retq
 ;
 ; X86-NOOPT-LABEL: test_mul_by_17:
 ; X86-NOOPT:       # %bb.0:
 ; X86-NOOPT-NEXT:    imull $17, {{[0-9]+}}(%esp), %eax
 ; X86-NOOPT-NEXT:    retl
 ;
-; HSW-NOOPT-LABEL: test_mul_by_17:
-; HSW-NOOPT:       # %bb.0:
-; HSW-NOOPT-NEXT:    imull $17, %edi, %eax # sched: [3:1.00]
-; HSW-NOOPT-NEXT:    retq # sched: [7:1.00]
-;
-; JAG-NOOPT-LABEL: test_mul_by_17:
-; JAG-NOOPT:       # %bb.0:
-; JAG-NOOPT-NEXT:    imull $17, %edi, %eax # sched: [3:1.00]
-; JAG-NOOPT-NEXT:    retq # sched: [4:1.00]
-;
-; X64-SLM-LABEL: test_mul_by_17:
-; X64-SLM:       # %bb.0:
-; X64-SLM-NEXT:    # kill: def $edi killed $edi def $rdi
-; X64-SLM-NEXT:    movl %edi, %eax # sched: [1:0.50]
-; X64-SLM-NEXT:    shll $4, %eax # sched: [1:1.00]
-; X64-SLM-NEXT:    leal (%rax,%rdi), %eax # sched: [1:1.00]
-; X64-SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SLM-NOOPT-LABEL: test_mul_by_17:
-; SLM-NOOPT:       # %bb.0:
-; SLM-NOOPT-NEXT:    imull $17, %edi, %eax # sched: [3:1.00]
-; SLM-NOOPT-NEXT:    retq # sched: [4:1.00]
+; X64-NOOPT-LABEL: test_mul_by_17:
+; X64-NOOPT:       # %bb.0:
+; X64-NOOPT-NEXT:    imull $17, %edi, %eax
+; X64-NOOPT-NEXT:    retq
   %mul = mul nsw i32 %x, 17
   ret i32 %mul
 }
@@ -894,46 +514,22 @@ define i32 @test_mul_by_18(i32 %x) {
 ; X86-NEXT:    leal (%eax,%eax,8), %eax
 ; X86-NEXT:    retl
 ;
-; X64-HSW-LABEL: test_mul_by_18:
-; X64-HSW:       # %bb.0:
-; X64-HSW-NEXT:    # kill: def $edi killed $edi def $rdi
-; X64-HSW-NEXT:    addl %edi, %edi # sched: [1:0.25]
-; X64-HSW-NEXT:    leal (%rdi,%rdi,8), %eax # sched: [1:0.50]
-; X64-HSW-NEXT:    retq # sched: [7:1.00]
-;
-; X64-JAG-LABEL: test_mul_by_18:
-; X64-JAG:       # %bb.0:
-; X64-JAG-NEXT:    # kill: def $edi killed $edi def $rdi
-; X64-JAG-NEXT:    addl %edi, %edi # sched: [1:0.50]
-; X64-JAG-NEXT:    leal (%rdi,%rdi,8), %eax # sched: [2:1.00]
-; X64-JAG-NEXT:    retq # sched: [4:1.00]
+; X64-LABEL: test_mul_by_18:
+; X64:       # %bb.0:
+; X64-NEXT:    # kill: def $edi killed $edi def $rdi
+; X64-NEXT:    addl %edi, %edi
+; X64-NEXT:    leal (%rdi,%rdi,8), %eax
+; X64-NEXT:    retq
 ;
 ; X86-NOOPT-LABEL: test_mul_by_18:
 ; X86-NOOPT:       # %bb.0:
 ; X86-NOOPT-NEXT:    imull $18, {{[0-9]+}}(%esp), %eax
 ; X86-NOOPT-NEXT:    retl
 ;
-; HSW-NOOPT-LABEL: test_mul_by_18:
-; HSW-NOOPT:       # %bb.0:
-; HSW-NOOPT-NEXT:    imull $18, %edi, %eax # sched: [3:1.00]
-; HSW-NOOPT-NEXT:    retq # sched: [7:1.00]
-;
-; JAG-NOOPT-LABEL: test_mul_by_18:
-; JAG-NOOPT:       # %bb.0:
-; JAG-NOOPT-NEXT:    imull $18, %edi, %eax # sched: [3:1.00]
-; JAG-NOOPT-NEXT:    retq # sched: [4:1.00]
-;
-; X64-SLM-LABEL: test_mul_by_18:
-; X64-SLM:       # %bb.0:
-; X64-SLM-NEXT:    # kill: def $edi killed $edi def $rdi
-; X64-SLM-NEXT:    addl %edi, %edi # sched: [1:0.50]
-; X64-SLM-NEXT:    leal (%rdi,%rdi,8), %eax # sched: [1:1.00]
-; X64-SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SLM-NOOPT-LABEL: test_mul_by_18:
-; SLM-NOOPT:       # %bb.0:
-; SLM-NOOPT-NEXT:    imull $18, %edi, %eax # sched: [3:1.00]
-; SLM-NOOPT-NEXT:    retq # sched: [4:1.00]
+; X64-NOOPT-LABEL: test_mul_by_18:
+; X64-NOOPT:       # %bb.0:
+; X64-NOOPT-NEXT:    imull $18, %edi, %eax
+; X64-NOOPT-NEXT:    retq
   %mul = mul nsw i32 %x, 18
   ret i32 %mul
 }
@@ -949,41 +545,31 @@ define i32 @test_mul_by_19(i32 %x) {
 ; X64-HSW-LABEL: test_mul_by_19:
 ; X64-HSW:       # %bb.0:
 ; X64-HSW-NEXT:    # kill: def $edi killed $edi def $rdi
-; X64-HSW-NEXT:    leal (%rdi,%rdi,8), %eax # sched: [1:0.50]
-; X64-HSW-NEXT:    leal (%rdi,%rax,2), %eax # sched: [1:0.50]
-; X64-HSW-NEXT:    retq # sched: [7:1.00]
+; X64-HSW-NEXT:    leal (%rdi,%rdi,8), %eax
+; X64-HSW-NEXT:    leal (%rdi,%rax,2), %eax
+; X64-HSW-NEXT:    retq
 ;
 ; X64-JAG-LABEL: test_mul_by_19:
 ; X64-JAG:       # %bb.0:
 ; X64-JAG-NEXT:    # kill: def $edi killed $edi def $rdi
-; X64-JAG-NEXT:    leal (%rdi,%rdi,8), %eax # sched: [2:1.00]
-; X64-JAG-NEXT:    leal (%rdi,%rax,2), %eax # sched: [2:1.00]
-; X64-JAG-NEXT:    retq # sched: [4:1.00]
+; X64-JAG-NEXT:    leal (%rdi,%rdi,8), %eax
+; X64-JAG-NEXT:    leal (%rdi,%rax,2), %eax
+; X64-JAG-NEXT:    retq
 ;
 ; X86-NOOPT-LABEL: test_mul_by_19:
 ; X86-NOOPT:       # %bb.0:
 ; X86-NOOPT-NEXT:    imull $19, {{[0-9]+}}(%esp), %eax
 ; X86-NOOPT-NEXT:    retl
 ;
-; HSW-NOOPT-LABEL: test_mul_by_19:
-; HSW-NOOPT:       # %bb.0:
-; HSW-NOOPT-NEXT:    imull $19, %edi, %eax # sched: [3:1.00]
-; HSW-NOOPT-NEXT:    retq # sched: [7:1.00]
-;
-; JAG-NOOPT-LABEL: test_mul_by_19:
-; JAG-NOOPT:       # %bb.0:
-; JAG-NOOPT-NEXT:    imull $19, %edi, %eax # sched: [3:1.00]
-; JAG-NOOPT-NEXT:    retq # sched: [4:1.00]
+; X64-NOOPT-LABEL: test_mul_by_19:
+; X64-NOOPT:       # %bb.0:
+; X64-NOOPT-NEXT:    imull $19, %edi, %eax
+; X64-NOOPT-NEXT:    retq
 ;
 ; X64-SLM-LABEL: test_mul_by_19:
 ; X64-SLM:       # %bb.0:
-; X64-SLM-NEXT:    imull $19, %edi, %eax # sched: [3:1.00]
-; X64-SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SLM-NOOPT-LABEL: test_mul_by_19:
-; SLM-NOOPT:       # %bb.0:
-; SLM-NOOPT-NEXT:    imull $19, %edi, %eax # sched: [3:1.00]
-; SLM-NOOPT-NEXT:    retq # sched: [4:1.00]
+; X64-SLM-NEXT:    imull $19, %edi, %eax
+; X64-SLM-NEXT:    retq
   %mul = mul nsw i32 %x, 19
   ret i32 %mul
 }
@@ -996,46 +582,22 @@ define i32 @test_mul_by_20(i32 %x) {
 ; X86-NEXT:    leal (%eax,%eax,4), %eax
 ; X86-NEXT:    retl
 ;
-; X64-HSW-LABEL: test_mul_by_20:
-; X64-HSW:       # %bb.0:
-; X64-HSW-NEXT:    # kill: def $edi killed $edi def $rdi
-; X64-HSW-NEXT:    shll $2, %edi # sched: [1:0.50]
-; X64-HSW-NEXT:    leal (%rdi,%rdi,4), %eax # sched: [1:0.50]
-; X64-HSW-NEXT:    retq # sched: [7:1.00]
-;
-; X64-JAG-LABEL: test_mul_by_20:
-; X64-JAG:       # %bb.0:
-; X64-JAG-NEXT:    # kill: def $edi killed $edi def $rdi
-; X64-JAG-NEXT:    shll $2, %edi # sched: [1:0.50]
-; X64-JAG-NEXT:    leal (%rdi,%rdi,4), %eax # sched: [2:1.00]
-; X64-JAG-NEXT:    retq # sched: [4:1.00]
+; X64-LABEL: test_mul_by_20:
+; X64:       # %bb.0:
+; X64-NEXT:    # kill: def $edi killed $edi def $rdi
+; X64-NEXT:    shll $2, %edi
+; X64-NEXT:    leal (%rdi,%rdi,4), %eax
+; X64-NEXT:    retq
 ;
 ; X86-NOOPT-LABEL: test_mul_by_20:
 ; X86-NOOPT:       # %bb.0:
 ; X86-NOOPT-NEXT:    imull $20, {{[0-9]+}}(%esp), %eax
 ; X86-NOOPT-NEXT:    retl
 ;
-; HSW-NOOPT-LABEL: test_mul_by_20:
-; HSW-NOOPT:       # %bb.0:
-; HSW-NOOPT-NEXT:    imull $20, %edi, %eax # sched: [3:1.00]
-; HSW-NOOPT-NEXT:    retq # sched: [7:1.00]
-;
-; JAG-NOOPT-LABEL: test_mul_by_20:
-; JAG-NOOPT:       # %bb.0:
-; JAG-NOOPT-NEXT:    imull $20, %edi, %eax # sched: [3:1.00]
-; JAG-NOOPT-NEXT:    retq # sched: [4:1.00]
-;
-; X64-SLM-LABEL: test_mul_by_20:
-; X64-SLM:       # %bb.0:
-; X64-SLM-NEXT:    # kill: def $edi killed $edi def $rdi
-; X64-SLM-NEXT:    shll $2, %edi # sched: [1:1.00]
-; X64-SLM-NEXT:    leal (%rdi,%rdi,4), %eax # sched: [1:1.00]
-; X64-SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SLM-NOOPT-LABEL: test_mul_by_20:
-; SLM-NOOPT:       # %bb.0:
-; SLM-NOOPT-NEXT:    imull $20, %edi, %eax # sched: [3:1.00]
-; SLM-NOOPT-NEXT:    retq # sched: [4:1.00]
+; X64-NOOPT-LABEL: test_mul_by_20:
+; X64-NOOPT:       # %bb.0:
+; X64-NOOPT-NEXT:    imull $20, %edi, %eax
+; X64-NOOPT-NEXT:    retq
   %mul = mul nsw i32 %x, 20
   ret i32 %mul
 }
@@ -1051,41 +613,31 @@ define i32 @test_mul_by_21(i32 %x) {
 ; X64-HSW-LABEL: test_mul_by_21:
 ; X64-HSW:       # %bb.0:
 ; X64-HSW-NEXT:    # kill: def $edi killed $edi def $rdi
-; X64-HSW-NEXT:    leal (%rdi,%rdi,4), %eax # sched: [1:0.50]
-; X64-HSW-NEXT:    leal (%rdi,%rax,4), %eax # sched: [1:0.50]
-; X64-HSW-NEXT:    retq # sched: [7:1.00]
+; X64-HSW-NEXT:    leal (%rdi,%rdi,4), %eax
+; X64-HSW-NEXT:    leal (%rdi,%rax,4), %eax
+; X64-HSW-NEXT:    retq
 ;
 ; X64-JAG-LABEL: test_mul_by_21:
 ; X64-JAG:       # %bb.0:
 ; X64-JAG-NEXT:    # kill: def $edi killed $edi def $rdi
-; X64-JAG-NEXT:    leal (%rdi,%rdi,4), %eax # sched: [2:1.00]
-; X64-JAG-NEXT:    leal (%rdi,%rax,4), %eax # sched: [2:1.00]
-; X64-JAG-NEXT:    retq # sched: [4:1.00]
+; X64-JAG-NEXT:    leal (%rdi,%rdi,4), %eax
+; X64-JAG-NEXT:    leal (%rdi,%rax,4), %eax
+; X64-JAG-NEXT:    retq
 ;
 ; X86-NOOPT-LABEL: test_mul_by_21:
 ; X86-NOOPT:       # %bb.0:
 ; X86-NOOPT-NEXT:    imull $21, {{[0-9]+}}(%esp), %eax
 ; X86-NOOPT-NEXT:    retl
 ;
-; HSW-NOOPT-LABEL: test_mul_by_21:
-; HSW-NOOPT:       # %bb.0:
-; HSW-NOOPT-NEXT:    imull $21, %edi, %eax # sched: [3:1.00]
-; HSW-NOOPT-NEXT:    retq # sched: [7:1.00]
-;
-; JAG-NOOPT-LABEL: test_mul_by_21:
-; JAG-NOOPT:       # %bb.0:
-; JAG-NOOPT-NEXT:    imull $21, %edi, %eax # sched: [3:1.00]
-; JAG-NOOPT-NEXT:    retq # sched: [4:1.00]
+; X64-NOOPT-LABEL: test_mul_by_21:
+; X64-NOOPT:       # %bb.0:
+; X64-NOOPT-NEXT:    imull $21, %edi, %eax
+; X64-NOOPT-NEXT:    retq
 ;
 ; X64-SLM-LABEL: test_mul_by_21:
 ; X64-SLM:       # %bb.0:
-; X64-SLM-NEXT:    imull $21, %edi, %eax # sched: [3:1.00]
-; X64-SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SLM-NOOPT-LABEL: test_mul_by_21:
-; SLM-NOOPT:       # %bb.0:
-; SLM-NOOPT-NEXT:    imull $21, %edi, %eax # sched: [3:1.00]
-; SLM-NOOPT-NEXT:    retq # sched: [4:1.00]
+; X64-SLM-NEXT:    imull $21, %edi, %eax
+; X64-SLM-NEXT:    retq
   %mul = mul nsw i32 %x, 21
   ret i32 %mul
 }
@@ -1102,43 +654,33 @@ define i32 @test_mul_by_22(i32 %x) {
 ; X64-HSW-LABEL: test_mul_by_22:
 ; X64-HSW:       # %bb.0:
 ; X64-HSW-NEXT:    # kill: def $edi killed $edi def $rdi
-; X64-HSW-NEXT:    leal (%rdi,%rdi,4), %eax # sched: [1:0.50]
-; X64-HSW-NEXT:    leal (%rdi,%rax,4), %eax # sched: [1:0.50]
-; X64-HSW-NEXT:    addl %edi, %eax # sched: [1:0.25]
-; X64-HSW-NEXT:    retq # sched: [7:1.00]
+; X64-HSW-NEXT:    leal (%rdi,%rdi,4), %eax
+; X64-HSW-NEXT:    leal (%rdi,%rax,4), %eax
+; X64-HSW-NEXT:    addl %edi, %eax
+; X64-HSW-NEXT:    retq
 ;
 ; X64-JAG-LABEL: test_mul_by_22:
 ; X64-JAG:       # %bb.0:
 ; X64-JAG-NEXT:    # kill: def $edi killed $edi def $rdi
-; X64-JAG-NEXT:    leal (%rdi,%rdi,4), %eax # sched: [2:1.00]
-; X64-JAG-NEXT:    leal (%rdi,%rax,4), %eax # sched: [2:1.00]
-; X64-JAG-NEXT:    addl %edi, %eax # sched: [1:0.50]
-; X64-JAG-NEXT:    retq # sched: [4:1.00]
+; X64-JAG-NEXT:    leal (%rdi,%rdi,4), %eax
+; X64-JAG-NEXT:    leal (%rdi,%rax,4), %eax
+; X64-JAG-NEXT:    addl %edi, %eax
+; X64-JAG-NEXT:    retq
 ;
 ; X86-NOOPT-LABEL: test_mul_by_22:
 ; X86-NOOPT:       # %bb.0:
 ; X86-NOOPT-NEXT:    imull $22, {{[0-9]+}}(%esp), %eax
 ; X86-NOOPT-NEXT:    retl
 ;
-; HSW-NOOPT-LABEL: test_mul_by_22:
-; HSW-NOOPT:       # %bb.0:
-; HSW-NOOPT-NEXT:    imull $22, %edi, %eax # sched: [3:1.00]
-; HSW-NOOPT-NEXT:    retq # sched: [7:1.00]
-;
-; JAG-NOOPT-LABEL: test_mul_by_22:
-; JAG-NOOPT:       # %bb.0:
-; JAG-NOOPT-NEXT:    imull $22, %edi, %eax # sched: [3:1.00]
-; JAG-NOOPT-NEXT:    retq # sched: [4:1.00]
+; X64-NOOPT-LABEL: test_mul_by_22:
+; X64-NOOPT:       # %bb.0:
+; X64-NOOPT-NEXT:    imull $22, %edi, %eax
+; X64-NOOPT-NEXT:    retq
 ;
 ; X64-SLM-LABEL: test_mul_by_22:
 ; X64-SLM:       # %bb.0:
-; X64-SLM-NEXT:    imull $22, %edi, %eax # sched: [3:1.00]
-; X64-SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SLM-NOOPT-LABEL: test_mul_by_22:
-; SLM-NOOPT:       # %bb.0:
-; SLM-NOOPT-NEXT:    imull $22, %edi, %eax # sched: [3:1.00]
-; SLM-NOOPT-NEXT:    retq # sched: [4:1.00]
+; X64-SLM-NEXT:    imull $22, %edi, %eax
+; X64-SLM-NEXT:    retq
   %mul = mul nsw i32 %x, 22
   ret i32 %mul
 }
@@ -1155,43 +697,33 @@ define i32 @test_mul_by_23(i32 %x) {
 ; X64-HSW-LABEL: test_mul_by_23:
 ; X64-HSW:       # %bb.0:
 ; X64-HSW-NEXT:    # kill: def $edi killed $edi def $rdi
-; X64-HSW-NEXT:    leal (%rdi,%rdi,2), %eax # sched: [1:0.50]
-; X64-HSW-NEXT:    shll $3, %eax # sched: [1:0.50]
-; X64-HSW-NEXT:    subl %edi, %eax # sched: [1:0.25]
-; X64-HSW-NEXT:    retq # sched: [7:1.00]
+; X64-HSW-NEXT:    leal (%rdi,%rdi,2), %eax
+; X64-HSW-NEXT:    shll $3, %eax
+; X64-HSW-NEXT:    subl %edi, %eax
+; X64-HSW-NEXT:    retq
 ;
 ; X64-JAG-LABEL: test_mul_by_23:
 ; X64-JAG:       # %bb.0:
 ; X64-JAG-NEXT:    # kill: def $edi killed $edi def $rdi
-; X64-JAG-NEXT:    leal (%rdi,%rdi,2), %eax # sched: [2:1.00]
-; X64-JAG-NEXT:    shll $3, %eax # sched: [1:0.50]
-; X64-JAG-NEXT:    subl %edi, %eax # sched: [1:0.50]
-; X64-JAG-NEXT:    retq # sched: [4:1.00]
+; X64-JAG-NEXT:    leal (%rdi,%rdi,2), %eax
+; X64-JAG-NEXT:    shll $3, %eax
+; X64-JAG-NEXT:    subl %edi, %eax
+; X64-JAG-NEXT:    retq
 ;
 ; X86-NOOPT-LABEL: test_mul_by_23:
 ; X86-NOOPT:       # %bb.0:
 ; X86-NOOPT-NEXT:    imull $23, {{[0-9]+}}(%esp), %eax
 ; X86-NOOPT-NEXT:    retl
 ;
-; HSW-NOOPT-LABEL: test_mul_by_23:
-; HSW-NOOPT:       # %bb.0:
-; HSW-NOOPT-NEXT:    imull $23, %edi, %eax # sched: [3:1.00]
-; HSW-NOOPT-NEXT:    retq # sched: [7:1.00]
-;
-; JAG-NOOPT-LABEL: test_mul_by_23:
-; JAG-NOOPT:       # %bb.0:
-; JAG-NOOPT-NEXT:    imull $23, %edi, %eax # sched: [3:1.00]
-; JAG-NOOPT-NEXT:    retq # sched: [4:1.00]
+; X64-NOOPT-LABEL: test_mul_by_23:
+; X64-NOOPT:       # %bb.0:
+; X64-NOOPT-NEXT:    imull $23, %edi, %eax
+; X64-NOOPT-NEXT:    retq
 ;
 ; X64-SLM-LABEL: test_mul_by_23:
 ; X64-SLM:       # %bb.0:
-; X64-SLM-NEXT:    imull $23, %edi, %eax # sched: [3:1.00]
-; X64-SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SLM-NOOPT-LABEL: test_mul_by_23:
-; SLM-NOOPT:       # %bb.0:
-; SLM-NOOPT-NEXT:    imull $23, %edi, %eax # sched: [3:1.00]
-; SLM-NOOPT-NEXT:    retq # sched: [4:1.00]
+; X64-SLM-NEXT:    imull $23, %edi, %eax
+; X64-SLM-NEXT:    retq
   %mul = mul nsw i32 %x, 23
   ret i32 %mul
 }
@@ -1204,46 +736,22 @@ define i32 @test_mul_by_24(i32 %x) {
 ; X86-NEXT:    leal (%eax,%eax,2), %eax
 ; X86-NEXT:    retl
 ;
-; X64-HSW-LABEL: test_mul_by_24:
-; X64-HSW:       # %bb.0:
-; X64-HSW-NEXT:    # kill: def $edi killed $edi def $rdi
-; X64-HSW-NEXT:    shll $3, %edi # sched: [1:0.50]
-; X64-HSW-NEXT:    leal (%rdi,%rdi,2), %eax # sched: [1:0.50]
-; X64-HSW-NEXT:    retq # sched: [7:1.00]
-;
-; X64-JAG-LABEL: test_mul_by_24:
-; X64-JAG:       # %bb.0:
-; X64-JAG-NEXT:    # kill: def $edi killed $edi def $rdi
-; X64-JAG-NEXT:    shll $3, %edi # sched: [1:0.50]
-; X64-JAG-NEXT:    leal (%rdi,%rdi,2), %eax # sched: [2:1.00]
-; X64-JAG-NEXT:    retq # sched: [4:1.00]
+; X64-LABEL: test_mul_by_24:
+; X64:       # %bb.0:
+; X64-NEXT:    # kill: def $edi killed $edi def $rdi
+; X64-NEXT:    shll $3, %edi
+; X64-NEXT:    leal (%rdi,%rdi,2), %eax
+; X64-NEXT:    retq
 ;
 ; X86-NOOPT-LABEL: test_mul_by_24:
 ; X86-NOOPT:       # %bb.0:
 ; X86-NOOPT-NEXT:    imull $24, {{[0-9]+}}(%esp), %eax
 ; X86-NOOPT-NEXT:    retl
 ;
-; HSW-NOOPT-LABEL: test_mul_by_24:
-; HSW-NOOPT:       # %bb.0:
-; HSW-NOOPT-NEXT:    imull $24, %edi, %eax # sched: [3:1.00]
-; HSW-NOOPT-NEXT:    retq # sched: [7:1.00]
-;
-; JAG-NOOPT-LABEL: test_mul_by_24:
-; JAG-NOOPT:       # %bb.0:
-; JAG-NOOPT-NEXT:    imull $24, %edi, %eax # sched: [3:1.00]
-; JAG-NOOPT-NEXT:    retq # sched: [4:1.00]
-;
-; X64-SLM-LABEL: test_mul_by_24:
-; X64-SLM:       # %bb.0:
-; X64-SLM-NEXT:    # kill: def $edi killed $edi def $rdi
-; X64-SLM-NEXT:    shll $3, %edi # sched: [1:1.00]
-; X64-SLM-NEXT:    leal (%rdi,%rdi,2), %eax # sched: [1:1.00]
-; X64-SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SLM-NOOPT-LABEL: test_mul_by_24:
-; SLM-NOOPT:       # %bb.0:
-; SLM-NOOPT-NEXT:    imull $24, %edi, %eax # sched: [3:1.00]
-; SLM-NOOPT-NEXT:    retq # sched: [4:1.00]
+; X64-NOOPT-LABEL: test_mul_by_24:
+; X64-NOOPT:       # %bb.0:
+; X64-NOOPT-NEXT:    imull $24, %edi, %eax
+; X64-NOOPT-NEXT:    retq
   %mul = mul nsw i32 %x, 24
   ret i32 %mul
 }
@@ -1256,46 +764,22 @@ define i32 @test_mul_by_25(i32 %x) {
 ; X86-NEXT:    leal (%eax,%eax,4), %eax
 ; X86-NEXT:    retl
 ;
-; X64-HSW-LABEL: test_mul_by_25:
-; X64-HSW:       # %bb.0:
-; X64-HSW-NEXT:    # kill: def $edi killed $edi def $rdi
-; X64-HSW-NEXT:    leal (%rdi,%rdi,4), %eax # sched: [1:0.50]
-; X64-HSW-NEXT:    leal (%rax,%rax,4), %eax # sched: [1:0.50]
-; X64-HSW-NEXT:    retq # sched: [7:1.00]
-;
-; X64-JAG-LABEL: test_mul_by_25:
-; X64-JAG:       # %bb.0:
-; X64-JAG-NEXT:    # kill: def $edi killed $edi def $rdi
-; X64-JAG-NEXT:    leal (%rdi,%rdi,4), %eax # sched: [2:1.00]
-; X64-JAG-NEXT:    leal (%rax,%rax,4), %eax # sched: [2:1.00]
-; X64-JAG-NEXT:    retq # sched: [4:1.00]
+; X64-LABEL: test_mul_by_25:
+; X64:       # %bb.0:
+; X64-NEXT:    # kill: def $edi killed $edi def $rdi
+; X64-NEXT:    leal (%rdi,%rdi,4), %eax
+; X64-NEXT:    leal (%rax,%rax,4), %eax
+; X64-NEXT:    retq
 ;
 ; X86-NOOPT-LABEL: test_mul_by_25:
 ; X86-NOOPT:       # %bb.0:
 ; X86-NOOPT-NEXT:    imull $25, {{[0-9]+}}(%esp), %eax
 ; X86-NOOPT-NEXT:    retl
 ;
-; HSW-NOOPT-LABEL: test_mul_by_25:
-; HSW-NOOPT:       # %bb.0:
-; HSW-NOOPT-NEXT:    imull $25, %edi, %eax # sched: [3:1.00]
-; HSW-NOOPT-NEXT:    retq # sched: [7:1.00]
-;
-; JAG-NOOPT-LABEL: test_mul_by_25:
-; JAG-NOOPT:       # %bb.0:
-; JAG-NOOPT-NEXT:    imull $25, %edi, %eax # sched: [3:1.00]
-; JAG-NOOPT-NEXT:    retq # sched: [4:1.00]
-;
-; X64-SLM-LABEL: test_mul_by_25:
-; X64-SLM:       # %bb.0:
-; X64-SLM-NEXT:    # kill: def $edi killed $edi def $rdi
-; X64-SLM-NEXT:    leal (%rdi,%rdi,4), %eax # sched: [1:1.00]
-; X64-SLM-NEXT:    leal (%rax,%rax,4), %eax # sched: [1:1.00]
-; X64-SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SLM-NOOPT-LABEL: test_mul_by_25:
-; SLM-NOOPT:       # %bb.0:
-; SLM-NOOPT-NEXT:    imull $25, %edi, %eax # sched: [3:1.00]
-; SLM-NOOPT-NEXT:    retq # sched: [4:1.00]
+; X64-NOOPT-LABEL: test_mul_by_25:
+; X64-NOOPT:       # %bb.0:
+; X64-NOOPT-NEXT:    imull $25, %edi, %eax
+; X64-NOOPT-NEXT:    retq
   %mul = mul nsw i32 %x, 25
   ret i32 %mul
 }
@@ -1312,43 +796,33 @@ define i32 @test_mul_by_26(i32 %x) {
 ; X64-HSW-LABEL: test_mul_by_26:
 ; X64-HSW:       # %bb.0:
 ; X64-HSW-NEXT:    # kill: def $edi killed $edi def $rdi
-; X64-HSW-NEXT:    leal (%rdi,%rdi,4), %eax # sched: [1:0.50]
-; X64-HSW-NEXT:    leal (%rax,%rax,4), %eax # sched: [1:0.50]
-; X64-HSW-NEXT:    addl %edi, %eax # sched: [1:0.25]
-; X64-HSW-NEXT:    retq # sched: [7:1.00]
+; X64-HSW-NEXT:    leal (%rdi,%rdi,4), %eax
+; X64-HSW-NEXT:    leal (%rax,%rax,4), %eax
+; X64-HSW-NEXT:    addl %edi, %eax
+; X64-HSW-NEXT:    retq
 ;
 ; X64-JAG-LABEL: test_mul_by_26:
 ; X64-JAG:       # %bb.0:
 ; X64-JAG-NEXT:    # kill: def $edi killed $edi def $rdi
-; X64-JAG-NEXT:    leal (%rdi,%rdi,4), %eax # sched: [2:1.00]
-; X64-JAG-NEXT:    leal (%rax,%rax,4), %eax # sched: [2:1.00]
-; X64-JAG-NEXT:    addl %edi, %eax # sched: [1:0.50]
-; X64-JAG-NEXT:    retq # sched: [4:1.00]
+; X64-JAG-NEXT:    leal (%rdi,%rdi,4), %eax
+; X64-JAG-NEXT:    leal (%rax,%rax,4), %eax
+; X64-JAG-NEXT:    addl %edi, %eax
+; X64-JAG-NEXT:    retq
 ;
 ; X86-NOOPT-LABEL: test_mul_by_26:
 ; X86-NOOPT:       # %bb.0:
 ; X86-NOOPT-NEXT:    imull $26, {{[0-9]+}}(%esp), %eax
 ; X86-NOOPT-NEXT:    retl
 ;
-; HSW-NOOPT-LABEL: test_mul_by_26:
-; HSW-NOOPT:       # %bb.0:
-; HSW-NOOPT-NEXT:    imull $26, %edi, %eax # sched: [3:1.00]
-; HSW-NOOPT-NEXT:    retq # sched: [7:1.00]
-;
-; JAG-NOOPT-LABEL: test_mul_by_26:
-; JAG-NOOPT:       # %bb.0:
-; JAG-NOOPT-NEXT:    imull $26, %edi, %eax # sched: [3:1.00]
-; JAG-NOOPT-NEXT:    retq # sched: [4:1.00]
+; X64-NOOPT-LABEL: test_mul_by_26:
+; X64-NOOPT:       # %bb.0:
+; X64-NOOPT-NEXT:    imull $26, %edi, %eax
+; X64-NOOPT-NEXT:    retq
 ;
 ; X64-SLM-LABEL: test_mul_by_26:
 ; X64-SLM:       # %bb.0:
-; X64-SLM-NEXT:    imull $26, %edi, %eax # sched: [3:1.00]
-; X64-SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SLM-NOOPT-LABEL: test_mul_by_26:
-; SLM-NOOPT:       # %bb.0:
-; SLM-NOOPT-NEXT:    imull $26, %edi, %eax # sched: [3:1.00]
-; SLM-NOOPT-NEXT:    retq # sched: [4:1.00]
+; X64-SLM-NEXT:    imull $26, %edi, %eax
+; X64-SLM-NEXT:    retq
   %mul = mul nsw i32 %x, 26
   ret i32 %mul
 }
@@ -1361,46 +835,22 @@ define i32 @test_mul_by_27(i32 %x) {
 ; X86-NEXT:    leal (%eax,%eax,2), %eax
 ; X86-NEXT:    retl
 ;
-; X64-HSW-LABEL: test_mul_by_27:
-; X64-HSW:       # %bb.0:
-; X64-HSW-NEXT:    # kill: def $edi killed $edi def $rdi
-; X64-HSW-NEXT:    leal (%rdi,%rdi,8), %eax # sched: [1:0.50]
-; X64-HSW-NEXT:    leal (%rax,%rax,2), %eax # sched: [1:0.50]
-; X64-HSW-NEXT:    retq # sched: [7:1.00]
-;
-; X64-JAG-LABEL: test_mul_by_27:
-; X64-JAG:       # %bb.0:
-; X64-JAG-NEXT:    # kill: def $edi killed $edi def $rdi
-; X64-JAG-NEXT:    leal (%rdi,%rdi,8), %eax # sched: [2:1.00]
-; X64-JAG-NEXT:    leal (%rax,%rax,2), %eax # sched: [2:1.00]
-; X64-JAG-NEXT:    retq # sched: [4:1.00]
+; X64-LABEL: test_mul_by_27:
+; X64:       # %bb.0:
+; X64-NEXT:    # kill: def $edi killed $edi def $rdi
+; X64-NEXT:    leal (%rdi,%rdi,8), %eax
+; X64-NEXT:    leal (%rax,%rax,2), %eax
+; X64-NEXT:    retq
 ;
 ; X86-NOOPT-LABEL: test_mul_by_27:
 ; X86-NOOPT:       # %bb.0:
 ; X86-NOOPT-NEXT:    imull $27, {{[0-9]+}}(%esp), %eax
 ; X86-NOOPT-NEXT:    retl
 ;
-; HSW-NOOPT-LABEL: test_mul_by_27:
-; HSW-NOOPT:       # %bb.0:
-; HSW-NOOPT-NEXT:    imull $27, %edi, %eax # sched: [3:1.00]
-; HSW-NOOPT-NEXT:    retq # sched: [7:1.00]
-;
-; JAG-NOOPT-LABEL: test_mul_by_27:
-; JAG-NOOPT:       # %bb.0:
-; JAG-NOOPT-NEXT:    imull $27, %edi, %eax # sched: [3:1.00]
-; JAG-NOOPT-NEXT:    retq # sched: [4:1.00]
-;
-; X64-SLM-LABEL: test_mul_by_27:
-; X64-SLM:       # %bb.0:
-; X64-SLM-NEXT:    # kill: def $edi killed $edi def $rdi
-; X64-SLM-NEXT:    leal (%rdi,%rdi,8), %eax # sched: [1:1.00]
-; X64-SLM-NEXT:    leal (%rax,%rax,2), %eax # sched: [1:1.00]
-; X64-SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SLM-NOOPT-LABEL: test_mul_by_27:
-; SLM-NOOPT:       # %bb.0:
-; SLM-NOOPT-NEXT:    imull $27, %edi, %eax # sched: [3:1.00]
-; SLM-NOOPT-NEXT:    retq # sched: [4:1.00]
+; X64-NOOPT-LABEL: test_mul_by_27:
+; X64-NOOPT:       # %bb.0:
+; X64-NOOPT-NEXT:    imull $27, %edi, %eax
+; X64-NOOPT-NEXT:    retq
   %mul = mul nsw i32 %x, 27
   ret i32 %mul
 }
@@ -1417,43 +867,33 @@ define i32 @test_mul_by_28(i32 %x) {
 ; X64-HSW-LABEL: test_mul_by_28:
 ; X64-HSW:       # %bb.0:
 ; X64-HSW-NEXT:    # kill: def $edi killed $edi def $rdi
-; X64-HSW-NEXT:    leal (%rdi,%rdi,8), %eax # sched: [1:0.50]
-; X64-HSW-NEXT:    leal (%rax,%rax,2), %eax # sched: [1:0.50]
-; X64-HSW-NEXT:    addl %edi, %eax # sched: [1:0.25]
-; X64-HSW-NEXT:    retq # sched: [7:1.00]
+; X64-HSW-NEXT:    leal (%rdi,%rdi,8), %eax
+; X64-HSW-NEXT:    leal (%rax,%rax,2), %eax
+; X64-HSW-NEXT:    addl %edi, %eax
+; X64-HSW-NEXT:    retq
 ;
 ; X64-JAG-LABEL: test_mul_by_28:
 ; X64-JAG:       # %bb.0:
 ; X64-JAG-NEXT:    # kill: def $edi killed $edi def $rdi
-; X64-JAG-NEXT:    leal (%rdi,%rdi,8), %eax # sched: [2:1.00]
-; X64-JAG-NEXT:    leal (%rax,%rax,2), %eax # sched: [2:1.00]
-; X64-JAG-NEXT:    addl %edi, %eax # sched: [1:0.50]
-; X64-JAG-NEXT:    retq # sched: [4:1.00]
+; X64-JAG-NEXT:    leal (%rdi,%rdi,8), %eax
+; X64-JAG-NEXT:    leal (%rax,%rax,2), %eax
+; X64-JAG-NEXT:    addl %edi, %eax
+; X64-JAG-NEXT:    retq
 ;
 ; X86-NOOPT-LABEL: test_mul_by_28:
 ; X86-NOOPT:       # %bb.0:
 ; X86-NOOPT-NEXT:    imull $28, {{[0-9]+}}(%esp), %eax
 ; X86-NOOPT-NEXT:    retl
 ;
-; HSW-NOOPT-LABEL: test_mul_by_28:
-; HSW-NOOPT:       # %bb.0:
-; HSW-NOOPT-NEXT:    imull $28, %edi, %eax # sched: [3:1.00]
-; HSW-NOOPT-NEXT:    retq # sched: [7:1.00]
-;
-; JAG-NOOPT-LABEL: test_mul_by_28:
-; JAG-NOOPT:       # %bb.0:
-; JAG-NOOPT-NEXT:    imull $28, %edi, %eax # sched: [3:1.00]
-; JAG-NOOPT-NEXT:    retq # sched: [4:1.00]
+; X64-NOOPT-LABEL: test_mul_by_28:
+; X64-NOOPT:       # %bb.0:
+; X64-NOOPT-NEXT:    imull $28, %edi, %eax
+; X64-NOOPT-NEXT:    retq
 ;
 ; X64-SLM-LABEL: test_mul_by_28:
 ; X64-SLM:       # %bb.0:
-; X64-SLM-NEXT:    imull $28, %edi, %eax # sched: [3:1.00]
-; X64-SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SLM-NOOPT-LABEL: test_mul_by_28:
-; SLM-NOOPT:       # %bb.0:
-; SLM-NOOPT-NEXT:    imull $28, %edi, %eax # sched: [3:1.00]
-; SLM-NOOPT-NEXT:    retq # sched: [4:1.00]
+; X64-SLM-NEXT:    imull $28, %edi, %eax
+; X64-SLM-NEXT:    retq
   %mul = mul nsw i32 %x, 28
   ret i32 %mul
 }
@@ -1471,45 +911,35 @@ define i32 @test_mul_by_29(i32 %x) {
 ; X64-HSW-LABEL: test_mul_by_29:
 ; X64-HSW:       # %bb.0:
 ; X64-HSW-NEXT:    # kill: def $edi killed $edi def $rdi
-; X64-HSW-NEXT:    leal (%rdi,%rdi,8), %eax # sched: [1:0.50]
-; X64-HSW-NEXT:    leal (%rax,%rax,2), %eax # sched: [1:0.50]
-; X64-HSW-NEXT:    addl %edi, %eax # sched: [1:0.25]
-; X64-HSW-NEXT:    addl %edi, %eax # sched: [1:0.25]
-; X64-HSW-NEXT:    retq # sched: [7:1.00]
+; X64-HSW-NEXT:    leal (%rdi,%rdi,8), %eax
+; X64-HSW-NEXT:    leal (%rax,%rax,2), %eax
+; X64-HSW-NEXT:    addl %edi, %eax
+; X64-HSW-NEXT:    addl %edi, %eax
+; X64-HSW-NEXT:    retq
 ;
 ; X64-JAG-LABEL: test_mul_by_29:
 ; X64-JAG:       # %bb.0:
 ; X64-JAG-NEXT:    # kill: def $edi killed $edi def $rdi
-; X64-JAG-NEXT:    leal (%rdi,%rdi,8), %eax # sched: [2:1.00]
-; X64-JAG-NEXT:    leal (%rax,%rax,2), %eax # sched: [2:1.00]
-; X64-JAG-NEXT:    addl %edi, %eax # sched: [1:0.50]
-; X64-JAG-NEXT:    addl %edi, %eax # sched: [1:0.50]
-; X64-JAG-NEXT:    retq # sched: [4:1.00]
+; X64-JAG-NEXT:    leal (%rdi,%rdi,8), %eax
+; X64-JAG-NEXT:    leal (%rax,%rax,2), %eax
+; X64-JAG-NEXT:    addl %edi, %eax
+; X64-JAG-NEXT:    addl %edi, %eax
+; X64-JAG-NEXT:    retq
 ;
 ; X86-NOOPT-LABEL: test_mul_by_29:
 ; X86-NOOPT:       # %bb.0:
 ; X86-NOOPT-NEXT:    imull $29, {{[0-9]+}}(%esp), %eax
 ; X86-NOOPT-NEXT:    retl
 ;
-; HSW-NOOPT-LABEL: test_mul_by_29:
-; HSW-NOOPT:       # %bb.0:
-; HSW-NOOPT-NEXT:    imull $29, %edi, %eax # sched: [3:1.00]
-; HSW-NOOPT-NEXT:    retq # sched: [7:1.00]
-;
-; JAG-NOOPT-LABEL: test_mul_by_29:
-; JAG-NOOPT:       # %bb.0:
-; JAG-NOOPT-NEXT:    imull $29, %edi, %eax # sched: [3:1.00]
-; JAG-NOOPT-NEXT:    retq # sched: [4:1.00]
+; X64-NOOPT-LABEL: test_mul_by_29:
+; X64-NOOPT:       # %bb.0:
+; X64-NOOPT-NEXT:    imull $29, %edi, %eax
+; X64-NOOPT-NEXT:    retq
 ;
 ; X64-SLM-LABEL: test_mul_by_29:
 ; X64-SLM:       # %bb.0:
-; X64-SLM-NEXT:    imull $29, %edi, %eax # sched: [3:1.00]
-; X64-SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SLM-NOOPT-LABEL: test_mul_by_29:
-; SLM-NOOPT:       # %bb.0:
-; SLM-NOOPT-NEXT:    imull $29, %edi, %eax # sched: [3:1.00]
-; SLM-NOOPT-NEXT:    retq # sched: [4:1.00]
+; X64-SLM-NEXT:    imull $29, %edi, %eax
+; X64-SLM-NEXT:    retq
   %mul = mul nsw i32 %x, 29
   ret i32 %mul
 }
@@ -1524,49 +954,23 @@ define i32 @test_mul_by_30(i32 %x) {
 ; X86-NEXT:    subl %ecx, %eax
 ; X86-NEXT:    retl
 ;
-; X64-HSW-LABEL: test_mul_by_30:
-; X64-HSW:       # %bb.0:
-; X64-HSW-NEXT:    movl %edi, %eax # sched: [1:0.25]
-; X64-HSW-NEXT:    shll $5, %eax # sched: [1:0.50]
-; X64-HSW-NEXT:    subl %edi, %eax # sched: [1:0.25]
-; X64-HSW-NEXT:    subl %edi, %eax # sched: [1:0.25]
-; X64-HSW-NEXT:    retq # sched: [7:1.00]
-;
-; X64-JAG-LABEL: test_mul_by_30:
-; X64-JAG:       # %bb.0:
-; X64-JAG-NEXT:    movl %edi, %eax # sched: [1:0.50]
-; X64-JAG-NEXT:    shll $5, %eax # sched: [1:0.50]
-; X64-JAG-NEXT:    subl %edi, %eax # sched: [1:0.50]
-; X64-JAG-NEXT:    subl %edi, %eax # sched: [1:0.50]
-; X64-JAG-NEXT:    retq # sched: [4:1.00]
+; X64-LABEL: test_mul_by_30:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    shll $5, %eax
+; X64-NEXT:    subl %edi, %eax
+; X64-NEXT:    subl %edi, %eax
+; X64-NEXT:    retq
 ;
 ; X86-NOOPT-LABEL: test_mul_by_30:
 ; X86-NOOPT:       # %bb.0:
 ; X86-NOOPT-NEXT:    imull $30, {{[0-9]+}}(%esp), %eax
 ; X86-NOOPT-NEXT:    retl
 ;
-; HSW-NOOPT-LABEL: test_mul_by_30:
-; HSW-NOOPT:       # %bb.0:
-; HSW-NOOPT-NEXT:    imull $30, %edi, %eax # sched: [3:1.00]
-; HSW-NOOPT-NEXT:    retq # sched: [7:1.00]
-;
-; JAG-NOOPT-LABEL: test_mul_by_30:
-; JAG-NOOPT:       # %bb.0:
-; JAG-NOOPT-NEXT:    imull $30, %edi, %eax # sched: [3:1.00]
-; JAG-NOOPT-NEXT:    retq # sched: [4:1.00]
-;
-; X64-SLM-LABEL: test_mul_by_30:
-; X64-SLM:       # %bb.0:
-; X64-SLM-NEXT:    movl %edi, %eax # sched: [1:0.50]
-; X64-SLM-NEXT:    shll $5, %eax # sched: [1:1.00]
-; X64-SLM-NEXT:    subl %edi, %eax # sched: [1:0.50]
-; X64-SLM-NEXT:    subl %edi, %eax # sched: [1:0.50]
-; X64-SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SLM-NOOPT-LABEL: test_mul_by_30:
-; SLM-NOOPT:       # %bb.0:
-; SLM-NOOPT-NEXT:    imull $30, %edi, %eax # sched: [3:1.00]
-; SLM-NOOPT-NEXT:    retq # sched: [4:1.00]
+; X64-NOOPT-LABEL: test_mul_by_30:
+; X64-NOOPT:       # %bb.0:
+; X64-NOOPT-NEXT:    imull $30, %edi, %eax
+; X64-NOOPT-NEXT:    retq
   %mul = mul nsw i32 %x, 30
   ret i32 %mul
 }
@@ -1580,46 +984,22 @@ define i32 @test_mul_by_31(i32 %x) {
 ; X86-NEXT:    subl %ecx, %eax
 ; X86-NEXT:    retl
 ;
-; X64-HSW-LABEL: test_mul_by_31:
-; X64-HSW:       # %bb.0:
-; X64-HSW-NEXT:    movl %edi, %eax # sched: [1:0.25]
-; X64-HSW-NEXT:    shll $5, %eax # sched: [1:0.50]
-; X64-HSW-NEXT:    subl %edi, %eax # sched: [1:0.25]
-; X64-HSW-NEXT:    retq # sched: [7:1.00]
-;
-; X64-JAG-LABEL: test_mul_by_31:
-; X64-JAG:       # %bb.0:
-; X64-JAG-NEXT:    movl %edi, %eax # sched: [1:0.50]
-; X64-JAG-NEXT:    shll $5, %eax # sched: [1:0.50]
-; X64-JAG-NEXT:    subl %edi, %eax # sched: [1:0.50]
-; X64-JAG-NEXT:    retq # sched: [4:1.00]
+; X64-LABEL: test_mul_by_31:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    shll $5, %eax
+; X64-NEXT:    subl %edi, %eax
+; X64-NEXT:    retq
 ;
 ; X86-NOOPT-LABEL: test_mul_by_31:
 ; X86-NOOPT:       # %bb.0:
 ; X86-NOOPT-NEXT:    imull $31, {{[0-9]+}}(%esp), %eax
 ; X86-NOOPT-NEXT:    retl
 ;
-; HSW-NOOPT-LABEL: test_mul_by_31:
-; HSW-NOOPT:       # %bb.0:
-; HSW-NOOPT-NEXT:    imull $31, %edi, %eax # sched: [3:1.00]
-; HSW-NOOPT-NEXT:    retq # sched: [7:1.00]
-;
-; JAG-NOOPT-LABEL: test_mul_by_31:
-; JAG-NOOPT:       # %bb.0:
-; JAG-NOOPT-NEXT:    imull $31, %edi, %eax # sched: [3:1.00]
-; JAG-NOOPT-NEXT:    retq # sched: [4:1.00]
-;
-; X64-SLM-LABEL: test_mul_by_31:
-; X64-SLM:       # %bb.0:
-; X64-SLM-NEXT:    movl %edi, %eax # sched: [1:0.50]
-; X64-SLM-NEXT:    shll $5, %eax # sched: [1:1.00]
-; X64-SLM-NEXT:    subl %edi, %eax # sched: [1:0.50]
-; X64-SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SLM-NOOPT-LABEL: test_mul_by_31:
-; SLM-NOOPT:       # %bb.0:
-; SLM-NOOPT-NEXT:    imull $31, %edi, %eax # sched: [3:1.00]
-; SLM-NOOPT-NEXT:    retq # sched: [4:1.00]
+; X64-NOOPT-LABEL: test_mul_by_31:
+; X64-NOOPT:       # %bb.0:
+; X64-NOOPT-NEXT:    imull $31, %edi, %eax
+; X64-NOOPT-NEXT:    retq
   %mul = mul nsw i32 %x, 31
   ret i32 %mul
 }
@@ -1631,17 +1011,11 @@ define i32 @test_mul_by_32(i32 %x) {
 ; X86-NEXT:    shll $5, %eax
 ; X86-NEXT:    retl
 ;
-; X64-HSW-LABEL: test_mul_by_32:
-; X64-HSW:       # %bb.0:
-; X64-HSW-NEXT:    movl %edi, %eax # sched: [1:0.25]
-; X64-HSW-NEXT:    shll $5, %eax # sched: [1:0.50]
-; X64-HSW-NEXT:    retq # sched: [7:1.00]
-;
-; X64-JAG-LABEL: test_mul_by_32:
-; X64-JAG:       # %bb.0:
-; X64-JAG-NEXT:    movl %edi, %eax # sched: [1:0.50]
-; X64-JAG-NEXT:    shll $5, %eax # sched: [1:0.50]
-; X64-JAG-NEXT:    retq # sched: [4:1.00]
+; X64-LABEL: test_mul_by_32:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    shll $5, %eax
+; X64-NEXT:    retq
 ;
 ; X86-NOOPT-LABEL: test_mul_by_32:
 ; X86-NOOPT:       # %bb.0:
@@ -1649,29 +1023,11 @@ define i32 @test_mul_by_32(i32 %x) {
 ; X86-NOOPT-NEXT:    shll $5, %eax
 ; X86-NOOPT-NEXT:    retl
 ;
-; HSW-NOOPT-LABEL: test_mul_by_32:
-; HSW-NOOPT:       # %bb.0:
-; HSW-NOOPT-NEXT:    movl %edi, %eax # sched: [1:0.25]
-; HSW-NOOPT-NEXT:    shll $5, %eax # sched: [1:0.50]
-; HSW-NOOPT-NEXT:    retq # sched: [7:1.00]
-;
-; JAG-NOOPT-LABEL: test_mul_by_32:
-; JAG-NOOPT:       # %bb.0:
-; JAG-NOOPT-NEXT:    movl %edi, %eax # sched: [1:0.50]
-; JAG-NOOPT-NEXT:    shll $5, %eax # sched: [1:0.50]
-; JAG-NOOPT-NEXT:    retq # sched: [4:1.00]
-;
-; X64-SLM-LABEL: test_mul_by_32:
-; X64-SLM:       # %bb.0:
-; X64-SLM-NEXT:    movl %edi, %eax # sched: [1:0.50]
-; X64-SLM-NEXT:    shll $5, %eax # sched: [1:1.00]
-; X64-SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SLM-NOOPT-LABEL: test_mul_by_32:
-; SLM-NOOPT:       # %bb.0:
-; SLM-NOOPT-NEXT:    movl %edi, %eax # sched: [1:0.50]
-; SLM-NOOPT-NEXT:    shll $5, %eax # sched: [1:1.00]
-; SLM-NOOPT-NEXT:    retq # sched: [4:1.00]
+; X64-NOOPT-LABEL: test_mul_by_32:
+; X64-NOOPT:       # %bb.0:
+; X64-NOOPT-NEXT:    movl %edi, %eax
+; X64-NOOPT-NEXT:    shll $5, %eax
+; X64-NOOPT-NEXT:    retq
   %mul = mul nsw i32 %x, 32
   ret i32 %mul
 }
@@ -1687,41 +1043,31 @@ define i32 @test_mul_by_37(i32 %x) {
 ; X64-HSW-LABEL: test_mul_by_37:
 ; X64-HSW:       # %bb.0:
 ; X64-HSW-NEXT:    # kill: def $edi killed $edi def $rdi
-; X64-HSW-NEXT:    leal (%rdi,%rdi,8), %eax # sched: [1:0.50]
-; X64-HSW-NEXT:    leal (%rdi,%rax,4), %eax # sched: [1:0.50]
-; X64-HSW-NEXT:    retq # sched: [7:1.00]
+; X64-HSW-NEXT:    leal (%rdi,%rdi,8), %eax
+; X64-HSW-NEXT:    leal (%rdi,%rax,4), %eax
+; X64-HSW-NEXT:    retq
 ;
 ; X64-JAG-LABEL: test_mul_by_37:
 ; X64-JAG:       # %bb.0:
 ; X64-JAG-NEXT:    # kill: def $edi killed $edi def $rdi
-; X64-JAG-NEXT:    leal (%rdi,%rdi,8), %eax # sched: [2:1.00]
-; X64-JAG-NEXT:    leal (%rdi,%rax,4), %eax # sched: [2:1.00]
-; X64-JAG-NEXT:    retq # sched: [4:1.00]
+; X64-JAG-NEXT:    leal (%rdi,%rdi,8), %eax
+; X64-JAG-NEXT:    leal (%rdi,%rax,4), %eax
+; X64-JAG-NEXT:    retq
 ;
 ; X86-NOOPT-LABEL: test_mul_by_37:
 ; X86-NOOPT:       # %bb.0:
 ; X86-NOOPT-NEXT:    imull $37, {{[0-9]+}}(%esp), %eax
 ; X86-NOOPT-NEXT:    retl
 ;
-; HSW-NOOPT-LABEL: test_mul_by_37:
-; HSW-NOOPT:       # %bb.0:
-; HSW-NOOPT-NEXT:    imull $37, %edi, %eax # sched: [3:1.00]
-; HSW-NOOPT-NEXT:    retq # sched: [7:1.00]
-;
-; JAG-NOOPT-LABEL: test_mul_by_37:
-; JAG-NOOPT:       # %bb.0:
-; JAG-NOOPT-NEXT:    imull $37, %edi, %eax # sched: [3:1.00]
-; JAG-NOOPT-NEXT:    retq # sched: [4:1.00]
+; X64-NOOPT-LABEL: test_mul_by_37:
+; X64-NOOPT:       # %bb.0:
+; X64-NOOPT-NEXT:    imull $37, %edi, %eax
+; X64-NOOPT-NEXT:    retq
 ;
 ; X64-SLM-LABEL: test_mul_by_37:
 ; X64-SLM:       # %bb.0:
-; X64-SLM-NEXT:    imull $37, %edi, %eax # sched: [3:1.00]
-; X64-SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SLM-NOOPT-LABEL: test_mul_by_37:
-; SLM-NOOPT:       # %bb.0:
-; SLM-NOOPT-NEXT:    imull $37, %edi, %eax # sched: [3:1.00]
-; SLM-NOOPT-NEXT:    retq # sched: [4:1.00]
+; X64-SLM-NEXT:    imull $37, %edi, %eax
+; X64-SLM-NEXT:    retq
   %mul = mul nsw i32 %x, 37
   ret i32 %mul
 }
@@ -1737,41 +1083,31 @@ define i32 @test_mul_by_41(i32 %x) {
 ; X64-HSW-LABEL: test_mul_by_41:
 ; X64-HSW:       # %bb.0:
 ; X64-HSW-NEXT:    # kill: def $edi killed $edi def $rdi
-; X64-HSW-NEXT:    leal (%rdi,%rdi,4), %eax # sched: [1:0.50]
-; X64-HSW-NEXT:    leal (%rdi,%rax,8), %eax # sched: [1:0.50]
-; X64-HSW-NEXT:    retq # sched: [7:1.00]
+; X64-HSW-NEXT:    leal (%rdi,%rdi,4), %eax
+; X64-HSW-NEXT:    leal (%rdi,%rax,8), %eax
+; X64-HSW-NEXT:    retq
 ;
 ; X64-JAG-LABEL: test_mul_by_41:
 ; X64-JAG:       # %bb.0:
 ; X64-JAG-NEXT:    # kill: def $edi killed $edi def $rdi
-; X64-JAG-NEXT:    leal (%rdi,%rdi,4), %eax # sched: [2:1.00]
-; X64-JAG-NEXT:    leal (%rdi,%rax,8), %eax # sched: [2:1.00]
-; X64-JAG-NEXT:    retq # sched: [4:1.00]
+; X64-JAG-NEXT:    leal (%rdi,%rdi,4), %eax
+; X64-JAG-NEXT:    leal (%rdi,%rax,8), %eax
+; X64-JAG-NEXT:    retq
 ;
 ; X86-NOOPT-LABEL: test_mul_by_41:
 ; X86-NOOPT:       # %bb.0:
 ; X86-NOOPT-NEXT:    imull $41, {{[0-9]+}}(%esp), %eax
 ; X86-NOOPT-NEXT:    retl
 ;
-; HSW-NOOPT-LABEL: test_mul_by_41:
-; HSW-NOOPT:       # %bb.0:
-; HSW-NOOPT-NEXT:    imull $41, %edi, %eax # sched: [3:1.00]
-; HSW-NOOPT-NEXT:    retq # sched: [7:1.00]
-;
-; JAG-NOOPT-LABEL: test_mul_by_41:
-; JAG-NOOPT:       # %bb.0:
-; JAG-NOOPT-NEXT:    imull $41, %edi, %eax # sched: [3:1.00]
-; JAG-NOOPT-NEXT:    retq # sched: [4:1.00]
+; X64-NOOPT-LABEL: test_mul_by_41:
+; X64-NOOPT:       # %bb.0:
+; X64-NOOPT-NEXT:    imull $41, %edi, %eax
+; X64-NOOPT-NEXT:    retq
 ;
 ; X64-SLM-LABEL: test_mul_by_41:
 ; X64-SLM:       # %bb.0:
-; X64-SLM-NEXT:    imull $41, %edi, %eax # sched: [3:1.00]
-; X64-SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SLM-NOOPT-LABEL: test_mul_by_41:
-; SLM-NOOPT:       # %bb.0:
-; SLM-NOOPT-NEXT:    imull $41, %edi, %eax # sched: [3:1.00]
-; SLM-NOOPT-NEXT:    retq # sched: [4:1.00]
+; X64-SLM-NEXT:    imull $41, %edi, %eax
+; X64-SLM-NEXT:    retq
   %mul = mul nsw i32 %x, 41
   ret i32 %mul
 }
@@ -1786,49 +1122,23 @@ define i32 @test_mul_by_62(i32 %x) {
 ; X86-NEXT:    subl %ecx, %eax
 ; X86-NEXT:    retl
 ;
-; X64-HSW-LABEL: test_mul_by_62:
-; X64-HSW:       # %bb.0:
-; X64-HSW-NEXT:    movl %edi, %eax # sched: [1:0.25]
-; X64-HSW-NEXT:    shll $6, %eax # sched: [1:0.50]
-; X64-HSW-NEXT:    subl %edi, %eax # sched: [1:0.25]
-; X64-HSW-NEXT:    subl %edi, %eax # sched: [1:0.25]
-; X64-HSW-NEXT:    retq # sched: [7:1.00]
-;
-; X64-JAG-LABEL: test_mul_by_62:
-; X64-JAG:       # %bb.0:
-; X64-JAG-NEXT:    movl %edi, %eax # sched: [1:0.50]
-; X64-JAG-NEXT:    shll $6, %eax # sched: [1:0.50]
-; X64-JAG-NEXT:    subl %edi, %eax # sched: [1:0.50]
-; X64-JAG-NEXT:    subl %edi, %eax # sched: [1:0.50]
-; X64-JAG-NEXT:    retq # sched: [4:1.00]
+; X64-LABEL: test_mul_by_62:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    shll $6, %eax
+; X64-NEXT:    subl %edi, %eax
+; X64-NEXT:    subl %edi, %eax
+; X64-NEXT:    retq
 ;
 ; X86-NOOPT-LABEL: test_mul_by_62:
 ; X86-NOOPT:       # %bb.0:
 ; X86-NOOPT-NEXT:    imull $62, {{[0-9]+}}(%esp), %eax
 ; X86-NOOPT-NEXT:    retl
 ;
-; HSW-NOOPT-LABEL: test_mul_by_62:
-; HSW-NOOPT:       # %bb.0:
-; HSW-NOOPT-NEXT:    imull $62, %edi, %eax # sched: [3:1.00]
-; HSW-NOOPT-NEXT:    retq # sched: [7:1.00]
-;
-; JAG-NOOPT-LABEL: test_mul_by_62:
-; JAG-NOOPT:       # %bb.0:
-; JAG-NOOPT-NEXT:    imull $62, %edi, %eax # sched: [3:1.00]
-; JAG-NOOPT-NEXT:    retq # sched: [4:1.00]
-;
-; X64-SLM-LABEL: test_mul_by_62:
-; X64-SLM:       # %bb.0:
-; X64-SLM-NEXT:    movl %edi, %eax # sched: [1:0.50]
-; X64-SLM-NEXT:    shll $6, %eax # sched: [1:1.00]
-; X64-SLM-NEXT:    subl %edi, %eax # sched: [1:0.50]
-; X64-SLM-NEXT:    subl %edi, %eax # sched: [1:0.50]
-; X64-SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SLM-NOOPT-LABEL: test_mul_by_62:
-; SLM-NOOPT:       # %bb.0:
-; SLM-NOOPT-NEXT:    imull $62, %edi, %eax # sched: [3:1.00]
-; SLM-NOOPT-NEXT:    retq # sched: [4:1.00]
+; X64-NOOPT-LABEL: test_mul_by_62:
+; X64-NOOPT:       # %bb.0:
+; X64-NOOPT-NEXT:    imull $62, %edi, %eax
+; X64-NOOPT-NEXT:    retq
   %mul = mul nsw i32 %x, 62
   ret i32 %mul
 }
@@ -1845,47 +1155,37 @@ define i32 @test_mul_by_66(i32 %x) {
 ; X64-HSW-LABEL: test_mul_by_66:
 ; X64-HSW:       # %bb.0:
 ; X64-HSW-NEXT:    # kill: def $edi killed $edi def $rdi
-; X64-HSW-NEXT:    movl %edi, %eax # sched: [1:0.25]
-; X64-HSW-NEXT:    shll $6, %eax # sched: [1:0.50]
-; X64-HSW-NEXT:    leal (%rax,%rdi,2), %eax # sched: [1:0.50]
-; X64-HSW-NEXT:    retq # sched: [7:1.00]
+; X64-HSW-NEXT:    movl %edi, %eax
+; X64-HSW-NEXT:    shll $6, %eax
+; X64-HSW-NEXT:    leal (%rax,%rdi,2), %eax
+; X64-HSW-NEXT:    retq
 ;
 ; X64-JAG-LABEL: test_mul_by_66:
 ; X64-JAG:       # %bb.0:
 ; X64-JAG-NEXT:    # kill: def $edi killed $edi def $rdi
-; X64-JAG-NEXT:    movl %edi, %eax # sched: [1:0.50]
-; X64-JAG-NEXT:    shll $6, %eax # sched: [1:0.50]
-; X64-JAG-NEXT:    leal (%rax,%rdi,2), %eax # sched: [2:1.00]
-; X64-JAG-NEXT:    retq # sched: [4:1.00]
+; X64-JAG-NEXT:    movl %edi, %eax
+; X64-JAG-NEXT:    shll $6, %eax
+; X64-JAG-NEXT:    leal (%rax,%rdi,2), %eax
+; X64-JAG-NEXT:    retq
 ;
 ; X86-NOOPT-LABEL: test_mul_by_66:
 ; X86-NOOPT:       # %bb.0:
 ; X86-NOOPT-NEXT:    imull $66, {{[0-9]+}}(%esp), %eax
 ; X86-NOOPT-NEXT:    retl
 ;
-; HSW-NOOPT-LABEL: test_mul_by_66:
-; HSW-NOOPT:       # %bb.0:
-; HSW-NOOPT-NEXT:    imull $66, %edi, %eax # sched: [3:1.00]
-; HSW-NOOPT-NEXT:    retq # sched: [7:1.00]
-;
-; JAG-NOOPT-LABEL: test_mul_by_66:
-; JAG-NOOPT:       # %bb.0:
-; JAG-NOOPT-NEXT:    imull $66, %edi, %eax # sched: [3:1.00]
-; JAG-NOOPT-NEXT:    retq # sched: [4:1.00]
+; X64-NOOPT-LABEL: test_mul_by_66:
+; X64-NOOPT:       # %bb.0:
+; X64-NOOPT-NEXT:    imull $66, %edi, %eax
+; X64-NOOPT-NEXT:    retq
 ;
 ; X64-SLM-LABEL: test_mul_by_66:
 ; X64-SLM:       # %bb.0:
 ; X64-SLM-NEXT:    # kill: def $edi killed $edi def $rdi
-; X64-SLM-NEXT:    movl %edi, %eax # sched: [1:0.50]
-; X64-SLM-NEXT:    shll $6, %eax # sched: [1:1.00]
-; X64-SLM-NEXT:    leal (%rax,%rdi), %eax # sched: [1:1.00]
-; X64-SLM-NEXT:    addl %edi, %eax # sched: [1:0.50]
-; X64-SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SLM-NOOPT-LABEL: test_mul_by_66:
-; SLM-NOOPT:       # %bb.0:
-; SLM-NOOPT-NEXT:    imull $66, %edi, %eax # sched: [3:1.00]
-; SLM-NOOPT-NEXT:    retq # sched: [4:1.00]
+; X64-SLM-NEXT:    movl %edi, %eax
+; X64-SLM-NEXT:    shll $6, %eax
+; X64-SLM-NEXT:    leal (%rax,%rdi), %eax
+; X64-SLM-NEXT:    addl %edi, %eax
+; X64-SLM-NEXT:    retq
   %mul = mul nsw i32 %x, 66
   ret i32 %mul
 }
@@ -1901,41 +1201,31 @@ define i32 @test_mul_by_73(i32 %x) {
 ; X64-HSW-LABEL: test_mul_by_73:
 ; X64-HSW:       # %bb.0:
 ; X64-HSW-NEXT:    # kill: def $edi killed $edi def $rdi
-; X64-HSW-NEXT:    leal (%rdi,%rdi,8), %eax # sched: [1:0.50]
-; X64-HSW-NEXT:    leal (%rdi,%rax,8), %eax # sched: [1:0.50]
-; X64-HSW-NEXT:    retq # sched: [7:1.00]
+; X64-HSW-NEXT:    leal (%rdi,%rdi,8), %eax
+; X64-HSW-NEXT:    leal (%rdi,%rax,8), %eax
+; X64-HSW-NEXT:    retq
 ;
 ; X64-JAG-LABEL: test_mul_by_73:
 ; X64-JAG:       # %bb.0:
 ; X64-JAG-NEXT:    # kill: def $edi killed $edi def $rdi
-; X64-JAG-NEXT:    leal (%rdi,%rdi,8), %eax # sched: [2:1.00]
-; X64-JAG-NEXT:    leal (%rdi,%rax,8), %eax # sched: [2:1.00]
-; X64-JAG-NEXT:    retq # sched: [4:1.00]
+; X64-JAG-NEXT:    leal (%rdi,%rdi,8), %eax
+; X64-JAG-NEXT:    leal (%rdi,%rax,8), %eax
+; X64-JAG-NEXT:    retq
 ;
 ; X86-NOOPT-LABEL: test_mul_by_73:
 ; X86-NOOPT:       # %bb.0:
 ; X86-NOOPT-NEXT:    imull $73, {{[0-9]+}}(%esp), %eax
 ; X86-NOOPT-NEXT:    retl
 ;
-; HSW-NOOPT-LABEL: test_mul_by_73:
-; HSW-NOOPT:       # %bb.0:
-; HSW-NOOPT-NEXT:    imull $73, %edi, %eax # sched: [3:1.00]
-; HSW-NOOPT-NEXT:    retq # sched: [7:1.00]
-;
-; JAG-NOOPT-LABEL: test_mul_by_73:
-; JAG-NOOPT:       # %bb.0:
-; JAG-NOOPT-NEXT:    imull $73, %edi, %eax # sched: [3:1.00]
-; JAG-NOOPT-NEXT:    retq # sched: [4:1.00]
+; X64-NOOPT-LABEL: test_mul_by_73:
+; X64-NOOPT:       # %bb.0:
+; X64-NOOPT-NEXT:    imull $73, %edi, %eax
+; X64-NOOPT-NEXT:    retq
 ;
 ; X64-SLM-LABEL: test_mul_by_73:
 ; X64-SLM:       # %bb.0:
-; X64-SLM-NEXT:    imull $73, %edi, %eax # sched: [3:1.00]
-; X64-SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SLM-NOOPT-LABEL: test_mul_by_73:
-; SLM-NOOPT:       # %bb.0:
-; SLM-NOOPT-NEXT:    imull $73, %edi, %eax # sched: [3:1.00]
-; SLM-NOOPT-NEXT:    retq # sched: [4:1.00]
+; X64-SLM-NEXT:    imull $73, %edi, %eax
+; X64-SLM-NEXT:    retq
   %mul = mul nsw i32 %x, 73
   ret i32 %mul
 }
@@ -1952,47 +1242,33 @@ define i32 @test_mul_by_520(i32 %x) {
 ; X64-HSW-LABEL: test_mul_by_520:
 ; X64-HSW:       # %bb.0:
 ; X64-HSW-NEXT:    # kill: def $edi killed $edi def $rdi
-; X64-HSW-NEXT:    movl %edi, %eax # sched: [1:0.25]
-; X64-HSW-NEXT:    shll $9, %eax # sched: [1:0.50]
-; X64-HSW-NEXT:    leal (%rax,%rdi,8), %eax # sched: [1:0.50]
-; X64-HSW-NEXT:    retq # sched: [7:1.00]
+; X64-HSW-NEXT:    movl %edi, %eax
+; X64-HSW-NEXT:    shll $9, %eax
+; X64-HSW-NEXT:    leal (%rax,%rdi,8), %eax
+; X64-HSW-NEXT:    retq
 ;
 ; X64-JAG-LABEL: test_mul_by_520:
 ; X64-JAG:       # %bb.0:
 ; X64-JAG-NEXT:    # kill: def $edi killed $edi def $rdi
-; X64-JAG-NEXT:    movl %edi, %eax # sched: [1:0.50]
-; X64-JAG-NEXT:    shll $9, %eax # sched: [1:0.50]
-; X64-JAG-NEXT:    leal (%rax,%rdi,8), %eax # sched: [2:1.00]
-; X64-JAG-NEXT:    retq # sched: [4:1.00]
+; X64-JAG-NEXT:    movl %edi, %eax
+; X64-JAG-NEXT:    shll $9, %eax
+; X64-JAG-NEXT:    leal (%rax,%rdi,8), %eax
+; X64-JAG-NEXT:    retq
 ;
 ; X86-NOOPT-LABEL: test_mul_by_520:
 ; X86-NOOPT:       # %bb.0:
 ; X86-NOOPT-NEXT:    imull $520, {{[0-9]+}}(%esp), %eax # imm = 0x208
 ; X86-NOOPT-NEXT:    retl
 ;
-; HSW-NOOPT-LABEL: test_mul_by_520:
-; HSW-NOOPT:       # %bb.0:
-; HSW-NOOPT-NEXT:    imull $520, %edi, %eax # imm = 0x208
-; HSW-NOOPT-NEXT:    # sched: [3:1.00]
-; HSW-NOOPT-NEXT:    retq # sched: [7:1.00]
-;
-; JAG-NOOPT-LABEL: test_mul_by_520:
-; JAG-NOOPT:       # %bb.0:
-; JAG-NOOPT-NEXT:    imull $520, %edi, %eax # imm = 0x208
-; JAG-NOOPT-NEXT:    # sched: [3:1.00]
-; JAG-NOOPT-NEXT:    retq # sched: [4:1.00]
+; X64-NOOPT-LABEL: test_mul_by_520:
+; X64-NOOPT:       # %bb.0:
+; X64-NOOPT-NEXT:    imull $520, %edi, %eax # imm = 0x208
+; X64-NOOPT-NEXT:    retq
 ;
 ; X64-SLM-LABEL: test_mul_by_520:
 ; X64-SLM:       # %bb.0:
 ; X64-SLM-NEXT:    imull $520, %edi, %eax # imm = 0x208
-; X64-SLM-NEXT:    # sched: [3:1.00]
-; X64-SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SLM-NOOPT-LABEL: test_mul_by_520:
-; SLM-NOOPT:       # %bb.0:
-; SLM-NOOPT-NEXT:    imull $520, %edi, %eax # imm = 0x208
-; SLM-NOOPT-NEXT:    # sched: [3:1.00]
-; SLM-NOOPT-NEXT:    retq # sched: [4:1.00]
+; X64-SLM-NEXT:    retq
   %mul = mul nsw i32 %x, 520
   ret i32 %mul
 }
@@ -2006,49 +1282,23 @@ define i32 @test_mul_by_neg10(i32 %x) {
 ; X86-NEXT:    negl %eax
 ; X86-NEXT:    retl
 ;
-; X64-HSW-LABEL: test_mul_by_neg10:
-; X64-HSW:       # %bb.0:
-; X64-HSW-NEXT:    # kill: def $edi killed $edi def $rdi
-; X64-HSW-NEXT:    addl %edi, %edi # sched: [1:0.25]
-; X64-HSW-NEXT:    leal (%rdi,%rdi,4), %eax # sched: [1:0.50]
-; X64-HSW-NEXT:    negl %eax # sched: [1:0.25]
-; X64-HSW-NEXT:    retq # sched: [7:1.00]
-;
-; X64-JAG-LABEL: test_mul_by_neg10:
-; X64-JAG:       # %bb.0:
-; X64-JAG-NEXT:    # kill: def $edi killed $edi def $rdi
-; X64-JAG-NEXT:    addl %edi, %edi # sched: [1:0.50]
-; X64-JAG-NEXT:    leal (%rdi,%rdi,4), %eax # sched: [2:1.00]
-; X64-JAG-NEXT:    negl %eax # sched: [1:0.50]
-; X64-JAG-NEXT:    retq # sched: [4:1.00]
+; X64-LABEL: test_mul_by_neg10:
+; X64:       # %bb.0:
+; X64-NEXT:    # kill: def $edi killed $edi def $rdi
+; X64-NEXT:    addl %edi, %edi
+; X64-NEXT:    leal (%rdi,%rdi,4), %eax
+; X64-NEXT:    negl %eax
+; X64-NEXT:    retq
 ;
 ; X86-NOOPT-LABEL: test_mul_by_neg10:
 ; X86-NOOPT:       # %bb.0:
 ; X86-NOOPT-NEXT:    imull $-10, {{[0-9]+}}(%esp), %eax
 ; X86-NOOPT-NEXT:    retl
 ;
-; HSW-NOOPT-LABEL: test_mul_by_neg10:
-; HSW-NOOPT:       # %bb.0:
-; HSW-NOOPT-NEXT:    imull $-10, %edi, %eax # sched: [3:1.00]
-; HSW-NOOPT-NEXT:    retq # sched: [7:1.00]
-;
-; JAG-NOOPT-LABEL: test_mul_by_neg10:
-; JAG-NOOPT:       # %bb.0:
-; JAG-NOOPT-NEXT:    imull $-10, %edi, %eax # sched: [3:1.00]
-; JAG-NOOPT-NEXT:    retq # sched: [4:1.00]
-;
-; X64-SLM-LABEL: test_mul_by_neg10:
-; X64-SLM:       # %bb.0:
-; X64-SLM-NEXT:    # kill: def $edi killed $edi def $rdi
-; X64-SLM-NEXT:    addl %edi, %edi # sched: [1:0.50]
-; X64-SLM-NEXT:    leal (%rdi,%rdi,4), %eax # sched: [1:1.00]
-; X64-SLM-NEXT:    negl %eax # sched: [1:0.50]
-; X64-SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SLM-NOOPT-LABEL: test_mul_by_neg10:
-; SLM-NOOPT:       # %bb.0:
-; SLM-NOOPT-NEXT:    imull $-10, %edi, %eax # sched: [3:1.00]
-; SLM-NOOPT-NEXT:    retq # sched: [4:1.00]
+; X64-NOOPT-LABEL: test_mul_by_neg10:
+; X64-NOOPT:       # %bb.0:
+; X64-NOOPT-NEXT:    imull $-10, %edi, %eax
+; X64-NOOPT-NEXT:    retq
   %mul = mul nsw i32 %x, -10
   ret i32 %mul
 }
@@ -2062,49 +1312,23 @@ define i32 @test_mul_by_neg36(i32 %x) {
 ; X86-NEXT:    negl %eax
 ; X86-NEXT:    retl
 ;
-; X64-HSW-LABEL: test_mul_by_neg36:
-; X64-HSW:       # %bb.0:
-; X64-HSW-NEXT:    # kill: def $edi killed $edi def $rdi
-; X64-HSW-NEXT:    shll $2, %edi # sched: [1:0.50]
-; X64-HSW-NEXT:    leal (%rdi,%rdi,8), %eax # sched: [1:0.50]
-; X64-HSW-NEXT:    negl %eax # sched: [1:0.25]
-; X64-HSW-NEXT:    retq # sched: [7:1.00]
-;
-; X64-JAG-LABEL: test_mul_by_neg36:
-; X64-JAG:       # %bb.0:
-; X64-JAG-NEXT:    # kill: def $edi killed $edi def $rdi
-; X64-JAG-NEXT:    shll $2, %edi # sched: [1:0.50]
-; X64-JAG-NEXT:    leal (%rdi,%rdi,8), %eax # sched: [2:1.00]
-; X64-JAG-NEXT:    negl %eax # sched: [1:0.50]
-; X64-JAG-NEXT:    retq # sched: [4:1.00]
+; X64-LABEL: test_mul_by_neg36:
+; X64:       # %bb.0:
+; X64-NEXT:    # kill: def $edi killed $edi def $rdi
+; X64-NEXT:    shll $2, %edi
+; X64-NEXT:    leal (%rdi,%rdi,8), %eax
+; X64-NEXT:    negl %eax
+; X64-NEXT:    retq
 ;
 ; X86-NOOPT-LABEL: test_mul_by_neg36:
 ; X86-NOOPT:       # %bb.0:
 ; X86-NOOPT-NEXT:    imull $-36, {{[0-9]+}}(%esp), %eax
 ; X86-NOOPT-NEXT:    retl
 ;
-; HSW-NOOPT-LABEL: test_mul_by_neg36:
-; HSW-NOOPT:       # %bb.0:
-; HSW-NOOPT-NEXT:    imull $-36, %edi, %eax # sched: [3:1.00]
-; HSW-NOOPT-NEXT:    retq # sched: [7:1.00]
-;
-; JAG-NOOPT-LABEL: test_mul_by_neg36:
-; JAG-NOOPT:       # %bb.0:
-; JAG-NOOPT-NEXT:    imull $-36, %edi, %eax # sched: [3:1.00]
-; JAG-NOOPT-NEXT:    retq # sched: [4:1.00]
-;
-; X64-SLM-LABEL: test_mul_by_neg36:
-; X64-SLM:       # %bb.0:
-; X64-SLM-NEXT:    # kill: def $edi killed $edi def $rdi
-; X64-SLM-NEXT:    shll $2, %edi # sched: [1:1.00]
-; X64-SLM-NEXT:    leal (%rdi,%rdi,8), %eax # sched: [1:1.00]
-; X64-SLM-NEXT:    negl %eax # sched: [1:0.50]
-; X64-SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SLM-NOOPT-LABEL: test_mul_by_neg36:
-; SLM-NOOPT:       # %bb.0:
-; SLM-NOOPT-NEXT:    imull $-36, %edi, %eax # sched: [3:1.00]
-; SLM-NOOPT-NEXT:    retq # sched: [4:1.00]
+; X64-NOOPT-LABEL: test_mul_by_neg36:
+; X64-NOOPT:       # %bb.0:
+; X64-NOOPT-NEXT:    imull $-36, %edi, %eax
+; X64-NOOPT-NEXT:    retq
   %mul = mul nsw i32 %x, -36
   ret i32 %mul
 }
@@ -2122,20 +1346,20 @@ define i32 @test_mul_spec(i32 %x) nounwind {
 ; X64-HSW-LABEL: test_mul_spec:
 ; X64-HSW:       # %bb.0:
 ; X64-HSW-NEXT:    # kill: def $edi killed $edi def $rdi
-; X64-HSW-NEXT:    leal (%rdi,%rdi,8), %ecx # sched: [1:0.50]
-; X64-HSW-NEXT:    addl $42, %ecx # sched: [1:0.25]
-; X64-HSW-NEXT:    leal (%rdi,%rdi,4), %eax # sched: [1:0.50]
-; X64-HSW-NEXT:    addl $2, %eax # sched: [1:0.25]
-; X64-HSW-NEXT:    imull %ecx, %eax # sched: [3:1.00]
-; X64-HSW-NEXT:    retq # sched: [7:1.00]
+; X64-HSW-NEXT:    leal (%rdi,%rdi,8), %ecx
+; X64-HSW-NEXT:    addl $42, %ecx
+; X64-HSW-NEXT:    leal (%rdi,%rdi,4), %eax
+; X64-HSW-NEXT:    addl $2, %eax
+; X64-HSW-NEXT:    imull %ecx, %eax
+; X64-HSW-NEXT:    retq
 ;
 ; X64-JAG-LABEL: test_mul_spec:
 ; X64-JAG:       # %bb.0:
 ; X64-JAG-NEXT:    # kill: def $edi killed $edi def $rdi
-; X64-JAG-NEXT:    leal 42(%rdi,%rdi,8), %ecx # sched: [2:1.00]
-; X64-JAG-NEXT:    leal 2(%rdi,%rdi,4), %eax # sched: [2:1.00]
-; X64-JAG-NEXT:    imull %ecx, %eax # sched: [3:1.00]
-; X64-JAG-NEXT:    retq # sched: [4:1.00]
+; X64-JAG-NEXT:    leal 42(%rdi,%rdi,8), %ecx
+; X64-JAG-NEXT:    leal 2(%rdi,%rdi,4), %eax
+; X64-JAG-NEXT:    imull %ecx, %eax
+; X64-JAG-NEXT:    retq
 ;
 ; X86-NOOPT-LABEL: test_mul_spec:
 ; X86-NOOPT:       # %bb.0:
@@ -2148,36 +1372,36 @@ define i32 @test_mul_spec(i32 %x) nounwind {
 ; HSW-NOOPT-LABEL: test_mul_spec:
 ; HSW-NOOPT:       # %bb.0:
 ; HSW-NOOPT-NEXT:    # kill: def $edi killed $edi def $rdi
-; HSW-NOOPT-NEXT:    leal (%rdi,%rdi,8), %ecx # sched: [1:0.50]
-; HSW-NOOPT-NEXT:    addl $42, %ecx # sched: [1:0.25]
-; HSW-NOOPT-NEXT:    leal (%rdi,%rdi,4), %eax # sched: [1:0.50]
-; HSW-NOOPT-NEXT:    addl $2, %eax # sched: [1:0.25]
-; HSW-NOOPT-NEXT:    imull %ecx, %eax # sched: [3:1.00]
-; HSW-NOOPT-NEXT:    retq # sched: [7:1.00]
+; HSW-NOOPT-NEXT:    leal (%rdi,%rdi,8), %ecx
+; HSW-NOOPT-NEXT:    addl $42, %ecx
+; HSW-NOOPT-NEXT:    leal (%rdi,%rdi,4), %eax
+; HSW-NOOPT-NEXT:    addl $2, %eax
+; HSW-NOOPT-NEXT:    imull %ecx, %eax
+; HSW-NOOPT-NEXT:    retq
 ;
 ; JAG-NOOPT-LABEL: test_mul_spec:
 ; JAG-NOOPT:       # %bb.0:
 ; JAG-NOOPT-NEXT:    # kill: def $edi killed $edi def $rdi
-; JAG-NOOPT-NEXT:    leal 42(%rdi,%rdi,8), %ecx # sched: [2:1.00]
-; JAG-NOOPT-NEXT:    leal 2(%rdi,%rdi,4), %eax # sched: [2:1.00]
-; JAG-NOOPT-NEXT:    imull %ecx, %eax # sched: [3:1.00]
-; JAG-NOOPT-NEXT:    retq # sched: [4:1.00]
+; JAG-NOOPT-NEXT:    leal 42(%rdi,%rdi,8), %ecx
+; JAG-NOOPT-NEXT:    leal 2(%rdi,%rdi,4), %eax
+; JAG-NOOPT-NEXT:    imull %ecx, %eax
+; JAG-NOOPT-NEXT:    retq
 ;
 ; X64-SLM-LABEL: test_mul_spec:
 ; X64-SLM:       # %bb.0:
 ; X64-SLM-NEXT:    # kill: def $edi killed $edi def $rdi
-; X64-SLM-NEXT:    leal 42(%rdi,%rdi,8), %ecx # sched: [1:1.00]
-; X64-SLM-NEXT:    leal 2(%rdi,%rdi,4), %eax # sched: [1:1.00]
-; X64-SLM-NEXT:    imull %ecx, %eax # sched: [3:1.00]
-; X64-SLM-NEXT:    retq # sched: [4:1.00]
+; X64-SLM-NEXT:    leal 42(%rdi,%rdi,8), %ecx
+; X64-SLM-NEXT:    leal 2(%rdi,%rdi,4), %eax
+; X64-SLM-NEXT:    imull %ecx, %eax
+; X64-SLM-NEXT:    retq
 ;
 ; SLM-NOOPT-LABEL: test_mul_spec:
 ; SLM-NOOPT:       # %bb.0:
 ; SLM-NOOPT-NEXT:    # kill: def $edi killed $edi def $rdi
-; SLM-NOOPT-NEXT:    leal 42(%rdi,%rdi,8), %ecx # sched: [1:1.00]
-; SLM-NOOPT-NEXT:    leal 2(%rdi,%rdi,4), %eax # sched: [1:1.00]
-; SLM-NOOPT-NEXT:    imull %ecx, %eax # sched: [3:1.00]
-; SLM-NOOPT-NEXT:    retq # sched: [4:1.00]
+; SLM-NOOPT-NEXT:    leal 42(%rdi,%rdi,8), %ecx
+; SLM-NOOPT-NEXT:    leal 2(%rdi,%rdi,4), %eax
+; SLM-NOOPT-NEXT:    imull %ecx, %eax
+; SLM-NOOPT-NEXT:    retq
   %mul = mul nsw i32 %x, 9
   %add = add nsw i32 %mul, 42
   %mul2 = mul nsw i32 %x, 5
@@ -2200,19 +1424,19 @@ define i32 @mul_neg_fold(i32 %a, i32 %b) {
 ;
 ; X64-HSW-LABEL: mul_neg_fold:
 ; X64-HSW:       # %bb.0:
-; X64-HSW-NEXT:    movl %esi, %eax # sched: [1:0.25]
+; X64-HSW-NEXT:    movl %esi, %eax
 ; X64-HSW-NEXT:    # kill: def $edi killed $edi def $rdi
-; X64-HSW-NEXT:    leal (%rdi,%rdi,8), %ecx # sched: [1:0.50]
-; X64-HSW-NEXT:    subl %ecx, %eax # sched: [1:0.25]
-; X64-HSW-NEXT:    retq # sched: [7:1.00]
+; X64-HSW-NEXT:    leal (%rdi,%rdi,8), %ecx
+; X64-HSW-NEXT:    subl %ecx, %eax
+; X64-HSW-NEXT:    retq
 ;
 ; X64-JAG-LABEL: mul_neg_fold:
 ; X64-JAG:       # %bb.0:
 ; X64-JAG-NEXT:    # kill: def $edi killed $edi def $rdi
-; X64-JAG-NEXT:    leal (%rdi,%rdi,8), %ecx # sched: [2:1.00]
-; X64-JAG-NEXT:    movl %esi, %eax # sched: [1:0.50]
-; X64-JAG-NEXT:    subl %ecx, %eax # sched: [1:0.50]
-; X64-JAG-NEXT:    retq # sched: [4:1.00]
+; X64-JAG-NEXT:    leal (%rdi,%rdi,8), %ecx
+; X64-JAG-NEXT:    movl %esi, %eax
+; X64-JAG-NEXT:    subl %ecx, %eax
+; X64-JAG-NEXT:    retq
 ;
 ; X86-NOOPT-LABEL: mul_neg_fold:
 ; X86-NOOPT:       # %bb.0:
@@ -2220,31 +1444,19 @@ define i32 @mul_neg_fold(i32 %a, i32 %b) {
 ; X86-NOOPT-NEXT:    addl {{[0-9]+}}(%esp), %eax
 ; X86-NOOPT-NEXT:    retl
 ;
-; HSW-NOOPT-LABEL: mul_neg_fold:
-; HSW-NOOPT:       # %bb.0:
-; HSW-NOOPT-NEXT:    imull $-9, %edi, %eax # sched: [3:1.00]
-; HSW-NOOPT-NEXT:    addl %esi, %eax # sched: [1:0.25]
-; HSW-NOOPT-NEXT:    retq # sched: [7:1.00]
-;
-; JAG-NOOPT-LABEL: mul_neg_fold:
-; JAG-NOOPT:       # %bb.0:
-; JAG-NOOPT-NEXT:    imull $-9, %edi, %eax # sched: [3:1.00]
-; JAG-NOOPT-NEXT:    addl %esi, %eax # sched: [1:0.50]
-; JAG-NOOPT-NEXT:    retq # sched: [4:1.00]
+; X64-NOOPT-LABEL: mul_neg_fold:
+; X64-NOOPT:       # %bb.0:
+; X64-NOOPT-NEXT:    imull $-9, %edi, %eax
+; X64-NOOPT-NEXT:    addl %esi, %eax
+; X64-NOOPT-NEXT:    retq
 ;
 ; X64-SLM-LABEL: mul_neg_fold:
 ; X64-SLM:       # %bb.0:
 ; X64-SLM-NEXT:    # kill: def $edi killed $edi def $rdi
-; X64-SLM-NEXT:    movl %esi, %eax # sched: [1:0.50]
-; X64-SLM-NEXT:    leal (%rdi,%rdi,8), %ecx # sched: [1:1.00]
-; X64-SLM-NEXT:    subl %ecx, %eax # sched: [1:0.50]
-; X64-SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SLM-NOOPT-LABEL: mul_neg_fold:
-; SLM-NOOPT:       # %bb.0:
-; SLM-NOOPT-NEXT:    imull $-9, %edi, %eax # sched: [3:1.00]
-; SLM-NOOPT-NEXT:    addl %esi, %eax # sched: [1:0.50]
-; SLM-NOOPT-NEXT:    retq # sched: [4:1.00]
+; X64-SLM-NEXT:    movl %esi, %eax
+; X64-SLM-NEXT:    leal (%rdi,%rdi,8), %ecx
+; X64-SLM-NEXT:    subl %ecx, %eax
+; X64-SLM-NEXT:    retq
   %c = mul i32 %a, -9
   %d = add i32 %b, %c
   ret i32 %d
diff --git a/test/CodeGen/X86/mul-constant-i64.ll b/test/CodeGen/X86/mul-constant-i64.ll
index 0e4680becaf0920825d253a339b3403501568de6..9ca2e695a273c52a631333b6062c7e50067c65f8 100644
--- a/test/CodeGen/X86/mul-constant-i64.ll
+++ b/test/CodeGen/X86/mul-constant-i64.ll
@@ -1,12 +1,12 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=i686-unknown | FileCheck %s --check-prefix=X86
-; RUN: llc < %s -mtriple=x86_64-unknown -print-schedule=true -mcpu=haswell| FileCheck %s --check-prefix=X64-HSW
-; RUN: llc < %s -mtriple=x86_64-unknown -print-schedule=true -mcpu=btver2| FileCheck %s --check-prefix=X64-JAG
-; RUN: llc < %s -mtriple=i686-unknown -mul-constant-optimization=false | FileCheck %s --check-prefix=X86-NOOPT
-; RUN: llc < %s -mtriple=x86_64-unknown -mul-constant-optimization=false -print-schedule=true -mcpu=haswell| FileCheck %s --check-prefix=HSW-NOOPT
-; RUN: llc < %s -mtriple=x86_64-unknown -mul-constant-optimization=false -print-schedule=true -mcpu=btver2| FileCheck %s --check-prefix=JAG-NOOPT
-; RUN: llc < %s -mtriple=x86_64-unknown -print-schedule=true -mcpu=slm| FileCheck %s --check-prefix=X64-SLM
-; RUN: llc < %s -mtriple=x86_64-unknown -mul-constant-optimization=false -print-schedule=true -mcpu=slm| FileCheck %s --check-prefix=SLM-NOOPT
+; RUN: llc < %s -mtriple=i686-unknown | FileCheck %s --check-prefixes=X86
+; RUN: llc < %s -mtriple=x86_64-unknown -mcpu=haswell                                  | FileCheck %s --check-prefixes=X64-OPT,X64-HSW
+; RUN: llc < %s -mtriple=x86_64-unknown -mcpu=btver2                                   | FileCheck %s --check-prefixes=X64-OPT,X64-JAG
+; RUN: llc < %s -mtriple=i686-unknown -mul-constant-optimization=false                 | FileCheck %s --check-prefix=X86-NOOPT
+; RUN: llc < %s -mtriple=x86_64-unknown -mul-constant-optimization=false -mcpu=haswell | FileCheck %s --check-prefixes=X64-NOOPT,HSW-NOOPT
+; RUN: llc < %s -mtriple=x86_64-unknown -mul-constant-optimization=false -mcpu=btver2  | FileCheck %s --check-prefixes=X64-NOOPT,JAG-NOOPT
+; RUN: llc < %s -mtriple=x86_64-unknown -mcpu=slm                                      | FileCheck %s --check-prefixes=X64-OPT,X64-SLM
+; RUN: llc < %s -mtriple=x86_64-unknown -mul-constant-optimization=false -mcpu=slm     | FileCheck %s --check-prefixes=X64-NOOPT,SLM-NOOPT
 
 define i64 @test_mul_by_1(i64 %x) nounwind {
 ; X86-LABEL: test_mul_by_1:
@@ -15,15 +15,10 @@ define i64 @test_mul_by_1(i64 %x) nounwind {
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; X86-NEXT:    retl
 ;
-; X64-HSW-LABEL: test_mul_by_1:
-; X64-HSW:       # %bb.0:
-; X64-HSW-NEXT:    movq %rdi, %rax # sched: [1:0.25]
-; X64-HSW-NEXT:    retq # sched: [7:1.00]
-;
-; X64-JAG-LABEL: test_mul_by_1:
-; X64-JAG:       # %bb.0:
-; X64-JAG-NEXT:    movq %rdi, %rax # sched: [1:0.50]
-; X64-JAG-NEXT:    retq # sched: [4:1.00]
+; X64-OPT-LABEL: test_mul_by_1:
+; X64-OPT:       # %bb.0:
+; X64-OPT-NEXT:    movq %rdi, %rax
+; X64-OPT-NEXT:    retq
 ;
 ; X86-NOOPT-LABEL: test_mul_by_1:
 ; X86-NOOPT:       # %bb.0:
@@ -31,25 +26,14 @@ define i64 @test_mul_by_1(i64 %x) nounwind {
 ; X86-NOOPT-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; X86-NOOPT-NEXT:    retl
 ;
-; HSW-NOOPT-LABEL: test_mul_by_1:
-; HSW-NOOPT:       # %bb.0:
-; HSW-NOOPT-NEXT:    movq %rdi, %rax # sched: [1:0.25]
-; HSW-NOOPT-NEXT:    retq # sched: [7:1.00]
-;
-; JAG-NOOPT-LABEL: test_mul_by_1:
-; JAG-NOOPT:       # %bb.0:
-; JAG-NOOPT-NEXT:    movq %rdi, %rax # sched: [1:0.50]
-; JAG-NOOPT-NEXT:    retq # sched: [4:1.00]
-;
-; X64-SLM-LABEL: test_mul_by_1:
-; X64-SLM:       # %bb.0:
-; X64-SLM-NEXT:    movq %rdi, %rax # sched: [1:0.50]
-; X64-SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SLM-NOOPT-LABEL: test_mul_by_1:
-; SLM-NOOPT:       # %bb.0:
-; SLM-NOOPT-NEXT:    movq %rdi, %rax # sched: [1:0.50]
-; SLM-NOOPT-NEXT:    retq # sched: [4:1.00]
+; X64-NOOPT-LABEL: test_mul_by_1:
+; X64-NOOPT:       # %bb.0:
+; X64-NOOPT-NEXT:    movq %rdi, %rax
+; X64-NOOPT-NEXT:    retq
+; X86-OPT-LABEL: test_mul_by_1:
+; X86-OPT:       # %bb.0:
+; X86-OPT-NEXT:    movq %rdi, %rax
+; X86-OPT-NEXT:    retq
   %mul = mul nsw i64 %x, 1
   ret i64 %mul
 }
@@ -63,15 +47,10 @@ define i64 @test_mul_by_2(i64 %x) {
 ; X86-NEXT:    addl %eax, %eax
 ; X86-NEXT:    retl
 ;
-; X64-HSW-LABEL: test_mul_by_2:
-; X64-HSW:       # %bb.0:
-; X64-HSW-NEXT:    leaq (%rdi,%rdi), %rax # sched: [1:0.50]
-; X64-HSW-NEXT:    retq # sched: [7:1.00]
-;
-; X64-JAG-LABEL: test_mul_by_2:
-; X64-JAG:       # %bb.0:
-; X64-JAG-NEXT:    leaq (%rdi,%rdi), %rax # sched: [1:0.50]
-; X64-JAG-NEXT:    retq # sched: [4:1.00]
+; X64-OPT-LABEL: test_mul_by_2:
+; X64-OPT:       # %bb.0:
+; X64-OPT-NEXT:    leaq (%rdi,%rdi), %rax
+; X64-OPT-NEXT:    retq
 ;
 ; X86-NOOPT-LABEL: test_mul_by_2:
 ; X86-NOOPT:       # %bb.0:
@@ -81,25 +60,14 @@ define i64 @test_mul_by_2(i64 %x) {
 ; X86-NOOPT-NEXT:    addl %eax, %eax
 ; X86-NOOPT-NEXT:    retl
 ;
-; HSW-NOOPT-LABEL: test_mul_by_2:
-; HSW-NOOPT:       # %bb.0:
-; HSW-NOOPT-NEXT:    leaq (%rdi,%rdi), %rax # sched: [1:0.50]
-; HSW-NOOPT-NEXT:    retq # sched: [7:1.00]
-;
-; JAG-NOOPT-LABEL: test_mul_by_2:
-; JAG-NOOPT:       # %bb.0:
-; JAG-NOOPT-NEXT:    leaq (%rdi,%rdi), %rax # sched: [1:0.50]
-; JAG-NOOPT-NEXT:    retq # sched: [4:1.00]
-;
-; X64-SLM-LABEL: test_mul_by_2:
-; X64-SLM:       # %bb.0:
-; X64-SLM-NEXT:    leaq (%rdi,%rdi), %rax # sched: [1:1.00]
-; X64-SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SLM-NOOPT-LABEL: test_mul_by_2:
-; SLM-NOOPT:       # %bb.0:
-; SLM-NOOPT-NEXT:    leaq (%rdi,%rdi), %rax # sched: [1:1.00]
-; SLM-NOOPT-NEXT:    retq # sched: [4:1.00]
+; X64-NOOPT-LABEL: test_mul_by_2:
+; X64-NOOPT:       # %bb.0:
+; X64-NOOPT-NEXT:    leaq (%rdi,%rdi), %rax
+; X64-NOOPT-NEXT:    retq
+; X86-OPT-LABEL: test_mul_by_2:
+; X86-OPT:       # %bb.0:
+; X86-OPT-NEXT:    leaq (%rdi,%rdi), %rax
+; X86-OPT-NEXT:    retq
   %mul = mul nsw i64 %x, 2
   ret i64 %mul
 }
@@ -114,15 +82,10 @@ define i64 @test_mul_by_3(i64 %x) {
 ; X86-NEXT:    addl %ecx, %edx
 ; X86-NEXT:    retl
 ;
-; X64-HSW-LABEL: test_mul_by_3:
-; X64-HSW:       # %bb.0:
-; X64-HSW-NEXT:    leaq (%rdi,%rdi,2), %rax # sched: [1:0.50]
-; X64-HSW-NEXT:    retq # sched: [7:1.00]
-;
-; X64-JAG-LABEL: test_mul_by_3:
-; X64-JAG:       # %bb.0:
-; X64-JAG-NEXT:    leaq (%rdi,%rdi,2), %rax # sched: [2:1.00]
-; X64-JAG-NEXT:    retq # sched: [4:1.00]
+; X64-OPT-LABEL: test_mul_by_3:
+; X64-OPT:       # %bb.0:
+; X64-OPT-NEXT:    leaq (%rdi,%rdi,2), %rax
+; X64-OPT-NEXT:    retq
 ;
 ; X86-NOOPT-LABEL: test_mul_by_3:
 ; X86-NOOPT:       # %bb.0:
@@ -132,25 +95,14 @@ define i64 @test_mul_by_3(i64 %x) {
 ; X86-NOOPT-NEXT:    addl %ecx, %edx
 ; X86-NOOPT-NEXT:    retl
 ;
-; HSW-NOOPT-LABEL: test_mul_by_3:
-; HSW-NOOPT:       # %bb.0:
-; HSW-NOOPT-NEXT:    leaq (%rdi,%rdi,2), %rax # sched: [1:0.50]
-; HSW-NOOPT-NEXT:    retq # sched: [7:1.00]
-;
-; JAG-NOOPT-LABEL: test_mul_by_3:
-; JAG-NOOPT:       # %bb.0:
-; JAG-NOOPT-NEXT:    leaq (%rdi,%rdi,2), %rax # sched: [2:1.00]
-; JAG-NOOPT-NEXT:    retq # sched: [4:1.00]
-;
-; X64-SLM-LABEL: test_mul_by_3:
-; X64-SLM:       # %bb.0:
-; X64-SLM-NEXT:    leaq (%rdi,%rdi,2), %rax # sched: [1:1.00]
-; X64-SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SLM-NOOPT-LABEL: test_mul_by_3:
-; SLM-NOOPT:       # %bb.0:
-; SLM-NOOPT-NEXT:    leaq (%rdi,%rdi,2), %rax # sched: [1:1.00]
-; SLM-NOOPT-NEXT:    retq # sched: [4:1.00]
+; X64-NOOPT-LABEL: test_mul_by_3:
+; X64-NOOPT:       # %bb.0:
+; X64-NOOPT-NEXT:    leaq (%rdi,%rdi,2), %rax
+; X64-NOOPT-NEXT:    retq
+; X86-OPT-LABEL: test_mul_by_3:
+; X86-OPT:       # %bb.0:
+; X86-OPT-NEXT:    leaq (%rdi,%rdi,2), %rax
+; X86-OPT-NEXT:    retq
   %mul = mul nsw i64 %x, 3
   ret i64 %mul
 }
@@ -164,15 +116,10 @@ define i64 @test_mul_by_4(i64 %x) {
 ; X86-NEXT:    shll $2, %eax
 ; X86-NEXT:    retl
 ;
-; X64-HSW-LABEL: test_mul_by_4:
-; X64-HSW:       # %bb.0:
-; X64-HSW-NEXT:    leaq (,%rdi,4), %rax # sched: [1:0.50]
-; X64-HSW-NEXT:    retq # sched: [7:1.00]
-;
-; X64-JAG-LABEL: test_mul_by_4:
-; X64-JAG:       # %bb.0:
-; X64-JAG-NEXT:    leaq (,%rdi,4), %rax # sched: [2:1.00]
-; X64-JAG-NEXT:    retq # sched: [4:1.00]
+; X64-OPT-LABEL: test_mul_by_4:
+; X64-OPT:       # %bb.0:
+; X64-OPT-NEXT:    leaq (,%rdi,4), %rax
+; X64-OPT-NEXT:    retq
 ;
 ; X86-NOOPT-LABEL: test_mul_by_4:
 ; X86-NOOPT:       # %bb.0:
@@ -182,25 +129,14 @@ define i64 @test_mul_by_4(i64 %x) {
 ; X86-NOOPT-NEXT:    shll $2, %eax
 ; X86-NOOPT-NEXT:    retl
 ;
-; HSW-NOOPT-LABEL: test_mul_by_4:
-; HSW-NOOPT:       # %bb.0:
-; HSW-NOOPT-NEXT:    leaq (,%rdi,4), %rax # sched: [1:0.50]
-; HSW-NOOPT-NEXT:    retq # sched: [7:1.00]
-;
-; JAG-NOOPT-LABEL: test_mul_by_4:
-; JAG-NOOPT:       # %bb.0:
-; JAG-NOOPT-NEXT:    leaq (,%rdi,4), %rax # sched: [2:1.00]
-; JAG-NOOPT-NEXT:    retq # sched: [4:1.00]
-;
-; X64-SLM-LABEL: test_mul_by_4:
-; X64-SLM:       # %bb.0:
-; X64-SLM-NEXT:    leaq (,%rdi,4), %rax # sched: [1:1.00]
-; X64-SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SLM-NOOPT-LABEL: test_mul_by_4:
-; SLM-NOOPT:       # %bb.0:
-; SLM-NOOPT-NEXT:    leaq (,%rdi,4), %rax # sched: [1:1.00]
-; SLM-NOOPT-NEXT:    retq # sched: [4:1.00]
+; X64-NOOPT-LABEL: test_mul_by_4:
+; X64-NOOPT:       # %bb.0:
+; X64-NOOPT-NEXT:    leaq (,%rdi,4), %rax
+; X64-NOOPT-NEXT:    retq
+; X86-OPT-LABEL: test_mul_by_4:
+; X86-OPT:       # %bb.0:
+; X86-OPT-NEXT:    leaq (,%rdi,4), %rax
+; X86-OPT-NEXT:    retq
   %mul = mul nsw i64 %x, 4
   ret i64 %mul
 }
@@ -215,15 +151,10 @@ define i64 @test_mul_by_5(i64 %x) {
 ; X86-NEXT:    addl %ecx, %edx
 ; X86-NEXT:    retl
 ;
-; X64-HSW-LABEL: test_mul_by_5:
-; X64-HSW:       # %bb.0:
-; X64-HSW-NEXT:    leaq (%rdi,%rdi,4), %rax # sched: [1:0.50]
-; X64-HSW-NEXT:    retq # sched: [7:1.00]
-;
-; X64-JAG-LABEL: test_mul_by_5:
-; X64-JAG:       # %bb.0:
-; X64-JAG-NEXT:    leaq (%rdi,%rdi,4), %rax # sched: [2:1.00]
-; X64-JAG-NEXT:    retq # sched: [4:1.00]
+; X64-OPT-LABEL: test_mul_by_5:
+; X64-OPT:       # %bb.0:
+; X64-OPT-NEXT:    leaq (%rdi,%rdi,4), %rax
+; X64-OPT-NEXT:    retq
 ;
 ; X86-NOOPT-LABEL: test_mul_by_5:
 ; X86-NOOPT:       # %bb.0:
@@ -233,25 +164,14 @@ define i64 @test_mul_by_5(i64 %x) {
 ; X86-NOOPT-NEXT:    addl %ecx, %edx
 ; X86-NOOPT-NEXT:    retl
 ;
-; HSW-NOOPT-LABEL: test_mul_by_5:
-; HSW-NOOPT:       # %bb.0:
-; HSW-NOOPT-NEXT:    leaq (%rdi,%rdi,4), %rax # sched: [1:0.50]
-; HSW-NOOPT-NEXT:    retq # sched: [7:1.00]
-;
-; JAG-NOOPT-LABEL: test_mul_by_5:
-; JAG-NOOPT:       # %bb.0:
-; JAG-NOOPT-NEXT:    leaq (%rdi,%rdi,4), %rax # sched: [2:1.00]
-; JAG-NOOPT-NEXT:    retq # sched: [4:1.00]
-;
-; X64-SLM-LABEL: test_mul_by_5:
-; X64-SLM:       # %bb.0:
-; X64-SLM-NEXT:    leaq (%rdi,%rdi,4), %rax # sched: [1:1.00]
-; X64-SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SLM-NOOPT-LABEL: test_mul_by_5:
-; SLM-NOOPT:       # %bb.0:
-; SLM-NOOPT-NEXT:    leaq (%rdi,%rdi,4), %rax # sched: [1:1.00]
-; SLM-NOOPT-NEXT:    retq # sched: [4:1.00]
+; X64-NOOPT-LABEL: test_mul_by_5:
+; X64-NOOPT:       # %bb.0:
+; X64-NOOPT-NEXT:    leaq (%rdi,%rdi,4), %rax
+; X64-NOOPT-NEXT:    retq
+; X86-OPT-LABEL: test_mul_by_5:
+; X86-OPT:       # %bb.0:
+; X86-OPT-NEXT:    leaq (%rdi,%rdi,4), %rax
+; X86-OPT-NEXT:    retq
   %mul = mul nsw i64 %x, 5
   ret i64 %mul
 }
@@ -266,17 +186,11 @@ define i64 @test_mul_by_6(i64 %x) {
 ; X86-NEXT:    leal (%edx,%ecx,2), %edx
 ; X86-NEXT:    retl
 ;
-; X64-HSW-LABEL: test_mul_by_6:
-; X64-HSW:       # %bb.0:
-; X64-HSW-NEXT:    addq %rdi, %rdi # sched: [1:0.25]
-; X64-HSW-NEXT:    leaq (%rdi,%rdi,2), %rax # sched: [1:0.50]
-; X64-HSW-NEXT:    retq # sched: [7:1.00]
-;
-; X64-JAG-LABEL: test_mul_by_6:
-; X64-JAG:       # %bb.0:
-; X64-JAG-NEXT:    addq %rdi, %rdi # sched: [1:0.50]
-; X64-JAG-NEXT:    leaq (%rdi,%rdi,2), %rax # sched: [2:1.00]
-; X64-JAG-NEXT:    retq # sched: [4:1.00]
+; X64-OPT-LABEL: test_mul_by_6:
+; X64-OPT:       # %bb.0:
+; X64-OPT-NEXT:    addq %rdi, %rdi
+; X64-OPT-NEXT:    leaq (%rdi,%rdi,2), %rax
+; X64-OPT-NEXT:    retq
 ;
 ; X86-NOOPT-LABEL: test_mul_by_6:
 ; X86-NOOPT:       # %bb.0:
@@ -286,26 +200,15 @@ define i64 @test_mul_by_6(i64 %x) {
 ; X86-NOOPT-NEXT:    addl %ecx, %edx
 ; X86-NOOPT-NEXT:    retl
 ;
-; HSW-NOOPT-LABEL: test_mul_by_6:
-; HSW-NOOPT:       # %bb.0:
-; HSW-NOOPT-NEXT:    imulq $6, %rdi, %rax # sched: [3:1.00]
-; HSW-NOOPT-NEXT:    retq # sched: [7:1.00]
-;
-; JAG-NOOPT-LABEL: test_mul_by_6:
-; JAG-NOOPT:       # %bb.0:
-; JAG-NOOPT-NEXT:    imulq $6, %rdi, %rax # sched: [6:4.00]
-; JAG-NOOPT-NEXT:    retq # sched: [4:1.00]
-;
-; X64-SLM-LABEL: test_mul_by_6:
-; X64-SLM:       # %bb.0:
-; X64-SLM-NEXT:    addq %rdi, %rdi # sched: [1:0.50]
-; X64-SLM-NEXT:    leaq (%rdi,%rdi,2), %rax # sched: [1:1.00]
-; X64-SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SLM-NOOPT-LABEL: test_mul_by_6:
-; SLM-NOOPT:       # %bb.0:
-; SLM-NOOPT-NEXT:    imulq $6, %rdi, %rax # sched: [3:1.00]
-; SLM-NOOPT-NEXT:    retq # sched: [4:1.00]
+; X64-NOOPT-LABEL: test_mul_by_6:
+; X64-NOOPT:       # %bb.0:
+; X64-NOOPT-NEXT:    imulq $6, %rdi, %rax
+; X64-NOOPT-NEXT:    retq
+; X86-OPT-LABEL: test_mul_by_6:
+; X86-OPT:       # %bb.0:
+; X86-OPT-NEXT:    addq %rdi, %rdi
+; X86-OPT-NEXT:    leaq (%rdi,%rdi,2), %rax
+; X86-OPT-NEXT:    retq
   %mul = mul nsw i64 %x, 6
   ret i64 %mul
 }
@@ -321,17 +224,11 @@ define i64 @test_mul_by_7(i64 %x) {
 ; X86-NEXT:    addl %ecx, %edx
 ; X86-NEXT:    retl
 ;
-; X64-HSW-LABEL: test_mul_by_7:
-; X64-HSW:       # %bb.0:
-; X64-HSW-NEXT:    leaq (,%rdi,8), %rax # sched: [1:0.50]
-; X64-HSW-NEXT:    subq %rdi, %rax # sched: [1:0.25]
-; X64-HSW-NEXT:    retq # sched: [7:1.00]
-;
-; X64-JAG-LABEL: test_mul_by_7:
-; X64-JAG:       # %bb.0:
-; X64-JAG-NEXT:    leaq (,%rdi,8), %rax # sched: [2:1.00]
-; X64-JAG-NEXT:    subq %rdi, %rax # sched: [1:0.50]
-; X64-JAG-NEXT:    retq # sched: [4:1.00]
+; X64-OPT-LABEL: test_mul_by_7:
+; X64-OPT:       # %bb.0:
+; X64-OPT-NEXT:    leaq (,%rdi,8), %rax
+; X64-OPT-NEXT:    subq %rdi, %rax
+; X64-OPT-NEXT:    retq
 ;
 ; X86-NOOPT-LABEL: test_mul_by_7:
 ; X86-NOOPT:       # %bb.0:
@@ -341,26 +238,15 @@ define i64 @test_mul_by_7(i64 %x) {
 ; X86-NOOPT-NEXT:    addl %ecx, %edx
 ; X86-NOOPT-NEXT:    retl
 ;
-; HSW-NOOPT-LABEL: test_mul_by_7:
-; HSW-NOOPT:       # %bb.0:
-; HSW-NOOPT-NEXT:    imulq $7, %rdi, %rax # sched: [3:1.00]
-; HSW-NOOPT-NEXT:    retq # sched: [7:1.00]
-;
-; JAG-NOOPT-LABEL: test_mul_by_7:
-; JAG-NOOPT:       # %bb.0:
-; JAG-NOOPT-NEXT:    imulq $7, %rdi, %rax # sched: [6:4.00]
-; JAG-NOOPT-NEXT:    retq # sched: [4:1.00]
-;
-; X64-SLM-LABEL: test_mul_by_7:
-; X64-SLM:       # %bb.0:
-; X64-SLM-NEXT:    leaq (,%rdi,8), %rax # sched: [1:1.00]
-; X64-SLM-NEXT:    subq %rdi, %rax # sched: [1:0.50]
-; X64-SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SLM-NOOPT-LABEL: test_mul_by_7:
-; SLM-NOOPT:       # %bb.0:
-; SLM-NOOPT-NEXT:    imulq $7, %rdi, %rax # sched: [3:1.00]
-; SLM-NOOPT-NEXT:    retq # sched: [4:1.00]
+; X64-NOOPT-LABEL: test_mul_by_7:
+; X64-NOOPT:       # %bb.0:
+; X64-NOOPT-NEXT:    imulq $7, %rdi, %rax
+; X64-NOOPT-NEXT:    retq
+; X86-OPT-LABEL: test_mul_by_7:
+; X86-OPT:       # %bb.0:
+; X86-OPT-NEXT:    leaq (,%rdi,8), %rax
+; X86-OPT-NEXT:    subq %rdi, %rax
+; X86-OPT-NEXT:    retq
   %mul = mul nsw i64 %x, 7
   ret i64 %mul
 }
@@ -374,15 +260,10 @@ define i64 @test_mul_by_8(i64 %x) {
 ; X86-NEXT:    shll $3, %eax
 ; X86-NEXT:    retl
 ;
-; X64-HSW-LABEL: test_mul_by_8:
-; X64-HSW:       # %bb.0:
-; X64-HSW-NEXT:    leaq (,%rdi,8), %rax # sched: [1:0.50]
-; X64-HSW-NEXT:    retq # sched: [7:1.00]
-;
-; X64-JAG-LABEL: test_mul_by_8:
-; X64-JAG:       # %bb.0:
-; X64-JAG-NEXT:    leaq (,%rdi,8), %rax # sched: [2:1.00]
-; X64-JAG-NEXT:    retq # sched: [4:1.00]
+; X64-OPT-LABEL: test_mul_by_8:
+; X64-OPT:       # %bb.0:
+; X64-OPT-NEXT:    leaq (,%rdi,8), %rax
+; X64-OPT-NEXT:    retq
 ;
 ; X86-NOOPT-LABEL: test_mul_by_8:
 ; X86-NOOPT:       # %bb.0:
@@ -392,25 +273,14 @@ define i64 @test_mul_by_8(i64 %x) {
 ; X86-NOOPT-NEXT:    shll $3, %eax
 ; X86-NOOPT-NEXT:    retl
 ;
-; HSW-NOOPT-LABEL: test_mul_by_8:
-; HSW-NOOPT:       # %bb.0:
-; HSW-NOOPT-NEXT:    leaq (,%rdi,8), %rax # sched: [1:0.50]
-; HSW-NOOPT-NEXT:    retq # sched: [7:1.00]
-;
-; JAG-NOOPT-LABEL: test_mul_by_8:
-; JAG-NOOPT:       # %bb.0:
-; JAG-NOOPT-NEXT:    leaq (,%rdi,8), %rax # sched: [2:1.00]
-; JAG-NOOPT-NEXT:    retq # sched: [4:1.00]
-;
-; X64-SLM-LABEL: test_mul_by_8:
-; X64-SLM:       # %bb.0:
-; X64-SLM-NEXT:    leaq (,%rdi,8), %rax # sched: [1:1.00]
-; X64-SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SLM-NOOPT-LABEL: test_mul_by_8:
-; SLM-NOOPT:       # %bb.0:
-; SLM-NOOPT-NEXT:    leaq (,%rdi,8), %rax # sched: [1:1.00]
-; SLM-NOOPT-NEXT:    retq # sched: [4:1.00]
+; X64-NOOPT-LABEL: test_mul_by_8:
+; X64-NOOPT:       # %bb.0:
+; X64-NOOPT-NEXT:    leaq (,%rdi,8), %rax
+; X64-NOOPT-NEXT:    retq
+; X86-OPT-LABEL: test_mul_by_8:
+; X86-OPT:       # %bb.0:
+; X86-OPT-NEXT:    leaq (,%rdi,8), %rax
+; X86-OPT-NEXT:    retq
   %mul = mul nsw i64 %x, 8
   ret i64 %mul
 }
@@ -425,15 +295,10 @@ define i64 @test_mul_by_9(i64 %x) {
 ; X86-NEXT:    addl %ecx, %edx
 ; X86-NEXT:    retl
 ;
-; X64-HSW-LABEL: test_mul_by_9:
-; X64-HSW:       # %bb.0:
-; X64-HSW-NEXT:    leaq (%rdi,%rdi,8), %rax # sched: [1:0.50]
-; X64-HSW-NEXT:    retq # sched: [7:1.00]
-;
-; X64-JAG-LABEL: test_mul_by_9:
-; X64-JAG:       # %bb.0:
-; X64-JAG-NEXT:    leaq (%rdi,%rdi,8), %rax # sched: [2:1.00]
-; X64-JAG-NEXT:    retq # sched: [4:1.00]
+; X64-OPT-LABEL: test_mul_by_9:
+; X64-OPT:       # %bb.0:
+; X64-OPT-NEXT:    leaq (%rdi,%rdi,8), %rax
+; X64-OPT-NEXT:    retq
 ;
 ; X86-NOOPT-LABEL: test_mul_by_9:
 ; X86-NOOPT:       # %bb.0:
@@ -443,25 +308,14 @@ define i64 @test_mul_by_9(i64 %x) {
 ; X86-NOOPT-NEXT:    addl %ecx, %edx
 ; X86-NOOPT-NEXT:    retl
 ;
-; HSW-NOOPT-LABEL: test_mul_by_9:
-; HSW-NOOPT:       # %bb.0:
-; HSW-NOOPT-NEXT:    leaq (%rdi,%rdi,8), %rax # sched: [1:0.50]
-; HSW-NOOPT-NEXT:    retq # sched: [7:1.00]
-;
-; JAG-NOOPT-LABEL: test_mul_by_9:
-; JAG-NOOPT:       # %bb.0:
-; JAG-NOOPT-NEXT:    leaq (%rdi,%rdi,8), %rax # sched: [2:1.00]
-; JAG-NOOPT-NEXT:    retq # sched: [4:1.00]
-;
-; X64-SLM-LABEL: test_mul_by_9:
-; X64-SLM:       # %bb.0:
-; X64-SLM-NEXT:    leaq (%rdi,%rdi,8), %rax # sched: [1:1.00]
-; X64-SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SLM-NOOPT-LABEL: test_mul_by_9:
-; SLM-NOOPT:       # %bb.0:
-; SLM-NOOPT-NEXT:    leaq (%rdi,%rdi,8), %rax # sched: [1:1.00]
-; SLM-NOOPT-NEXT:    retq # sched: [4:1.00]
+; X64-NOOPT-LABEL: test_mul_by_9:
+; X64-NOOPT:       # %bb.0:
+; X64-NOOPT-NEXT:    leaq (%rdi,%rdi,8), %rax
+; X64-NOOPT-NEXT:    retq
+; X86-OPT-LABEL: test_mul_by_9:
+; X86-OPT:       # %bb.0:
+; X86-OPT-NEXT:    leaq (%rdi,%rdi,8), %rax
+; X86-OPT-NEXT:    retq
   %mul = mul nsw i64 %x, 9
   ret i64 %mul
 }
@@ -476,17 +330,11 @@ define i64 @test_mul_by_10(i64 %x) {
 ; X86-NEXT:    leal (%edx,%ecx,2), %edx
 ; X86-NEXT:    retl
 ;
-; X64-HSW-LABEL: test_mul_by_10:
-; X64-HSW:       # %bb.0:
-; X64-HSW-NEXT:    addq %rdi, %rdi # sched: [1:0.25]
-; X64-HSW-NEXT:    leaq (%rdi,%rdi,4), %rax # sched: [1:0.50]
-; X64-HSW-NEXT:    retq # sched: [7:1.00]
-;
-; X64-JAG-LABEL: test_mul_by_10:
-; X64-JAG:       # %bb.0:
-; X64-JAG-NEXT:    addq %rdi, %rdi # sched: [1:0.50]
-; X64-JAG-NEXT:    leaq (%rdi,%rdi,4), %rax # sched: [2:1.00]
-; X64-JAG-NEXT:    retq # sched: [4:1.00]
+; X64-OPT-LABEL: test_mul_by_10:
+; X64-OPT:       # %bb.0:
+; X64-OPT-NEXT:    addq %rdi, %rdi
+; X64-OPT-NEXT:    leaq (%rdi,%rdi,4), %rax
+; X64-OPT-NEXT:    retq
 ;
 ; X86-NOOPT-LABEL: test_mul_by_10:
 ; X86-NOOPT:       # %bb.0:
@@ -496,26 +344,15 @@ define i64 @test_mul_by_10(i64 %x) {
 ; X86-NOOPT-NEXT:    addl %ecx, %edx
 ; X86-NOOPT-NEXT:    retl
 ;
-; HSW-NOOPT-LABEL: test_mul_by_10:
-; HSW-NOOPT:       # %bb.0:
-; HSW-NOOPT-NEXT:    imulq $10, %rdi, %rax # sched: [3:1.00]
-; HSW-NOOPT-NEXT:    retq # sched: [7:1.00]
-;
-; JAG-NOOPT-LABEL: test_mul_by_10:
-; JAG-NOOPT:       # %bb.0:
-; JAG-NOOPT-NEXT:    imulq $10, %rdi, %rax # sched: [6:4.00]
-; JAG-NOOPT-NEXT:    retq # sched: [4:1.00]
-;
-; X64-SLM-LABEL: test_mul_by_10:
-; X64-SLM:       # %bb.0:
-; X64-SLM-NEXT:    addq %rdi, %rdi # sched: [1:0.50]
-; X64-SLM-NEXT:    leaq (%rdi,%rdi,4), %rax # sched: [1:1.00]
-; X64-SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SLM-NOOPT-LABEL: test_mul_by_10:
-; SLM-NOOPT:       # %bb.0:
-; SLM-NOOPT-NEXT:    imulq $10, %rdi, %rax # sched: [3:1.00]
-; SLM-NOOPT-NEXT:    retq # sched: [4:1.00]
+; X64-NOOPT-LABEL: test_mul_by_10:
+; X64-NOOPT:       # %bb.0:
+; X64-NOOPT-NEXT:    imulq $10, %rdi, %rax
+; X64-NOOPT-NEXT:    retq
+; X86-OPT-LABEL: test_mul_by_10:
+; X86-OPT:       # %bb.0:
+; X86-OPT-NEXT:    addq %rdi, %rdi
+; X86-OPT-NEXT:    leaq (%rdi,%rdi,4), %rax
+; X86-OPT-NEXT:    retq
   %mul = mul nsw i64 %x, 10
   ret i64 %mul
 }
@@ -533,15 +370,15 @@ define i64 @test_mul_by_11(i64 %x) {
 ;
 ; X64-HSW-LABEL: test_mul_by_11:
 ; X64-HSW:       # %bb.0:
-; X64-HSW-NEXT:    leaq (%rdi,%rdi,4), %rax # sched: [1:0.50]
-; X64-HSW-NEXT:    leaq (%rdi,%rax,2), %rax # sched: [1:0.50]
-; X64-HSW-NEXT:    retq # sched: [7:1.00]
+; X64-HSW-NEXT:    leaq (%rdi,%rdi,4), %rax
+; X64-HSW-NEXT:    leaq (%rdi,%rax,2), %rax
+; X64-HSW-NEXT:    retq
 ;
 ; X64-JAG-LABEL: test_mul_by_11:
 ; X64-JAG:       # %bb.0:
-; X64-JAG-NEXT:    leaq (%rdi,%rdi,4), %rax # sched: [2:1.00]
-; X64-JAG-NEXT:    leaq (%rdi,%rax,2), %rax # sched: [2:1.00]
-; X64-JAG-NEXT:    retq # sched: [4:1.00]
+; X64-JAG-NEXT:    leaq (%rdi,%rdi,4), %rax
+; X64-JAG-NEXT:    leaq (%rdi,%rax,2), %rax
+; X64-JAG-NEXT:    retq
 ;
 ; X86-NOOPT-LABEL: test_mul_by_11:
 ; X86-NOOPT:       # %bb.0:
@@ -551,25 +388,20 @@ define i64 @test_mul_by_11(i64 %x) {
 ; X86-NOOPT-NEXT:    addl %ecx, %edx
 ; X86-NOOPT-NEXT:    retl
 ;
-; HSW-NOOPT-LABEL: test_mul_by_11:
-; HSW-NOOPT:       # %bb.0:
-; HSW-NOOPT-NEXT:    imulq $11, %rdi, %rax # sched: [3:1.00]
-; HSW-NOOPT-NEXT:    retq # sched: [7:1.00]
-;
-; JAG-NOOPT-LABEL: test_mul_by_11:
-; JAG-NOOPT:       # %bb.0:
-; JAG-NOOPT-NEXT:    imulq $11, %rdi, %rax # sched: [6:4.00]
-; JAG-NOOPT-NEXT:    retq # sched: [4:1.00]
+; X64-NOOPT-LABEL: test_mul_by_11:
+; X64-NOOPT:       # %bb.0:
+; X64-NOOPT-NEXT:    imulq $11, %rdi, %rax
+; X64-NOOPT-NEXT:    retq
 ;
 ; X64-SLM-LABEL: test_mul_by_11:
 ; X64-SLM:       # %bb.0:
-; X64-SLM-NEXT:    imulq $11, %rdi, %rax # sched: [3:1.00]
-; X64-SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SLM-NOOPT-LABEL: test_mul_by_11:
-; SLM-NOOPT:       # %bb.0:
-; SLM-NOOPT-NEXT:    imulq $11, %rdi, %rax # sched: [3:1.00]
-; SLM-NOOPT-NEXT:    retq # sched: [4:1.00]
+; X64-SLM-NEXT:    imulq $11, %rdi, %rax
+; X64-SLM-NEXT:    retq
+; X86-OPT-LABEL: test_mul_by_11:
+; X86-OPT:       # %bb.0:
+; X86-OPT-NEXT:    leaq (%rdi,%rdi,4), %rax
+; X86-OPT-NEXT:    leaq (%rdi,%rax,2), %rax
+; X86-OPT-NEXT:    retq
   %mul = mul nsw i64 %x, 11
   ret i64 %mul
 }
@@ -584,17 +416,11 @@ define i64 @test_mul_by_12(i64 %x) {
 ; X86-NEXT:    leal (%edx,%ecx,4), %edx
 ; X86-NEXT:    retl
 ;
-; X64-HSW-LABEL: test_mul_by_12:
-; X64-HSW:       # %bb.0:
-; X64-HSW-NEXT:    shlq $2, %rdi # sched: [1:0.50]
-; X64-HSW-NEXT:    leaq (%rdi,%rdi,2), %rax # sched: [1:0.50]
-; X64-HSW-NEXT:    retq # sched: [7:1.00]
-;
-; X64-JAG-LABEL: test_mul_by_12:
-; X64-JAG:       # %bb.0:
-; X64-JAG-NEXT:    shlq $2, %rdi # sched: [1:0.50]
-; X64-JAG-NEXT:    leaq (%rdi,%rdi,2), %rax # sched: [2:1.00]
-; X64-JAG-NEXT:    retq # sched: [4:1.00]
+; X64-OPT-LABEL: test_mul_by_12:
+; X64-OPT:       # %bb.0:
+; X64-OPT-NEXT:    shlq $2, %rdi
+; X64-OPT-NEXT:    leaq (%rdi,%rdi,2), %rax
+; X64-OPT-NEXT:    retq
 ;
 ; X86-NOOPT-LABEL: test_mul_by_12:
 ; X86-NOOPT:       # %bb.0:
@@ -604,26 +430,15 @@ define i64 @test_mul_by_12(i64 %x) {
 ; X86-NOOPT-NEXT:    addl %ecx, %edx
 ; X86-NOOPT-NEXT:    retl
 ;
-; HSW-NOOPT-LABEL: test_mul_by_12:
-; HSW-NOOPT:       # %bb.0:
-; HSW-NOOPT-NEXT:    imulq $12, %rdi, %rax # sched: [3:1.00]
-; HSW-NOOPT-NEXT:    retq # sched: [7:1.00]
-;
-; JAG-NOOPT-LABEL: test_mul_by_12:
-; JAG-NOOPT:       # %bb.0:
-; JAG-NOOPT-NEXT:    imulq $12, %rdi, %rax # sched: [6:4.00]
-; JAG-NOOPT-NEXT:    retq # sched: [4:1.00]
-;
-; X64-SLM-LABEL: test_mul_by_12:
-; X64-SLM:       # %bb.0:
-; X64-SLM-NEXT:    shlq $2, %rdi # sched: [1:1.00]
-; X64-SLM-NEXT:    leaq (%rdi,%rdi,2), %rax # sched: [1:1.00]
-; X64-SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SLM-NOOPT-LABEL: test_mul_by_12:
-; SLM-NOOPT:       # %bb.0:
-; SLM-NOOPT-NEXT:    imulq $12, %rdi, %rax # sched: [3:1.00]
-; SLM-NOOPT-NEXT:    retq # sched: [4:1.00]
+; X64-NOOPT-LABEL: test_mul_by_12:
+; X64-NOOPT:       # %bb.0:
+; X64-NOOPT-NEXT:    imulq $12, %rdi, %rax
+; X64-NOOPT-NEXT:    retq
+; X86-OPT-LABEL: test_mul_by_12:
+; X86-OPT:       # %bb.0:
+; X86-OPT-NEXT:    shlq $2, %rdi
+; X86-OPT-NEXT:    leaq (%rdi,%rdi,2), %rax
+; X86-OPT-NEXT:    retq
   %mul = mul nsw i64 %x, 12
   ret i64 %mul
 }
@@ -641,15 +456,15 @@ define i64 @test_mul_by_13(i64 %x) {
 ;
 ; X64-HSW-LABEL: test_mul_by_13:
 ; X64-HSW:       # %bb.0:
-; X64-HSW-NEXT:    leaq (%rdi,%rdi,2), %rax # sched: [1:0.50]
-; X64-HSW-NEXT:    leaq (%rdi,%rax,4), %rax # sched: [1:0.50]
-; X64-HSW-NEXT:    retq # sched: [7:1.00]
+; X64-HSW-NEXT:    leaq (%rdi,%rdi,2), %rax
+; X64-HSW-NEXT:    leaq (%rdi,%rax,4), %rax
+; X64-HSW-NEXT:    retq
 ;
 ; X64-JAG-LABEL: test_mul_by_13:
 ; X64-JAG:       # %bb.0:
-; X64-JAG-NEXT:    leaq (%rdi,%rdi,2), %rax # sched: [2:1.00]
-; X64-JAG-NEXT:    leaq (%rdi,%rax,4), %rax # sched: [2:1.00]
-; X64-JAG-NEXT:    retq # sched: [4:1.00]
+; X64-JAG-NEXT:    leaq (%rdi,%rdi,2), %rax
+; X64-JAG-NEXT:    leaq (%rdi,%rax,4), %rax
+; X64-JAG-NEXT:    retq
 ;
 ; X86-NOOPT-LABEL: test_mul_by_13:
 ; X86-NOOPT:       # %bb.0:
@@ -659,25 +474,20 @@ define i64 @test_mul_by_13(i64 %x) {
 ; X86-NOOPT-NEXT:    addl %ecx, %edx
 ; X86-NOOPT-NEXT:    retl
 ;
-; HSW-NOOPT-LABEL: test_mul_by_13:
-; HSW-NOOPT:       # %bb.0:
-; HSW-NOOPT-NEXT:    imulq $13, %rdi, %rax # sched: [3:1.00]
-; HSW-NOOPT-NEXT:    retq # sched: [7:1.00]
-;
-; JAG-NOOPT-LABEL: test_mul_by_13:
-; JAG-NOOPT:       # %bb.0:
-; JAG-NOOPT-NEXT:    imulq $13, %rdi, %rax # sched: [6:4.00]
-; JAG-NOOPT-NEXT:    retq # sched: [4:1.00]
+; X64-NOOPT-LABEL: test_mul_by_13:
+; X64-NOOPT:       # %bb.0:
+; X64-NOOPT-NEXT:    imulq $13, %rdi, %rax
+; X64-NOOPT-NEXT:    retq
 ;
 ; X64-SLM-LABEL: test_mul_by_13:
 ; X64-SLM:       # %bb.0:
-; X64-SLM-NEXT:    imulq $13, %rdi, %rax # sched: [3:1.00]
-; X64-SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SLM-NOOPT-LABEL: test_mul_by_13:
-; SLM-NOOPT:       # %bb.0:
-; SLM-NOOPT-NEXT:    imulq $13, %rdi, %rax # sched: [3:1.00]
-; SLM-NOOPT-NEXT:    retq # sched: [4:1.00]
+; X64-SLM-NEXT:    imulq $13, %rdi, %rax
+; X64-SLM-NEXT:    retq
+; X86-OPT-LABEL: test_mul_by_13:
+; X86-OPT:       # %bb.0:
+; X86-OPT-NEXT:    leaq (%rdi,%rdi,2), %rax
+; X86-OPT-NEXT:    leaq (%rdi,%rax,4), %rax
+; X86-OPT-NEXT:    retq
   %mul = mul nsw i64 %x, 13
   ret i64 %mul
 }
@@ -695,21 +505,13 @@ define i64 @test_mul_by_14(i64 %x) {
 ; X86-NEXT:    addl %ecx, %edx
 ; X86-NEXT:    retl
 ;
-; X64-HSW-LABEL: test_mul_by_14:
-; X64-HSW:       # %bb.0:
-; X64-HSW-NEXT:    movq %rdi, %rax # sched: [1:0.25]
-; X64-HSW-NEXT:    shlq $4, %rax # sched: [1:0.50]
-; X64-HSW-NEXT:    subq %rdi, %rax # sched: [1:0.25]
-; X64-HSW-NEXT:    subq %rdi, %rax # sched: [1:0.25]
-; X64-HSW-NEXT:    retq # sched: [7:1.00]
-;
-; X64-JAG-LABEL: test_mul_by_14:
-; X64-JAG:       # %bb.0:
-; X64-JAG-NEXT:    movq %rdi, %rax # sched: [1:0.50]
-; X64-JAG-NEXT:    shlq $4, %rax # sched: [1:0.50]
-; X64-JAG-NEXT:    subq %rdi, %rax # sched: [1:0.50]
-; X64-JAG-NEXT:    subq %rdi, %rax # sched: [1:0.50]
-; X64-JAG-NEXT:    retq # sched: [4:1.00]
+; X64-OPT-LABEL: test_mul_by_14:
+; X64-OPT:       # %bb.0:
+; X64-OPT-NEXT:    movq %rdi, %rax
+; X64-OPT-NEXT:    shlq $4, %rax
+; X64-OPT-NEXT:    subq %rdi, %rax
+; X64-OPT-NEXT:    subq %rdi, %rax
+; X64-OPT-NEXT:    retq
 ;
 ; X86-NOOPT-LABEL: test_mul_by_14:
 ; X86-NOOPT:       # %bb.0:
@@ -719,28 +521,17 @@ define i64 @test_mul_by_14(i64 %x) {
 ; X86-NOOPT-NEXT:    addl %ecx, %edx
 ; X86-NOOPT-NEXT:    retl
 ;
-; HSW-NOOPT-LABEL: test_mul_by_14:
-; HSW-NOOPT:       # %bb.0:
-; HSW-NOOPT-NEXT:    imulq $14, %rdi, %rax # sched: [3:1.00]
-; HSW-NOOPT-NEXT:    retq # sched: [7:1.00]
-;
-; JAG-NOOPT-LABEL: test_mul_by_14:
-; JAG-NOOPT:       # %bb.0:
-; JAG-NOOPT-NEXT:    imulq $14, %rdi, %rax # sched: [6:4.00]
-; JAG-NOOPT-NEXT:    retq # sched: [4:1.00]
-;
-; X64-SLM-LABEL: test_mul_by_14:
-; X64-SLM:       # %bb.0:
-; X64-SLM-NEXT:    movq %rdi, %rax # sched: [1:0.50]
-; X64-SLM-NEXT:    shlq $4, %rax # sched: [1:1.00]
-; X64-SLM-NEXT:    subq %rdi, %rax # sched: [1:0.50]
-; X64-SLM-NEXT:    subq %rdi, %rax # sched: [1:0.50]
-; X64-SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SLM-NOOPT-LABEL: test_mul_by_14:
-; SLM-NOOPT:       # %bb.0:
-; SLM-NOOPT-NEXT:    imulq $14, %rdi, %rax # sched: [3:1.00]
-; SLM-NOOPT-NEXT:    retq # sched: [4:1.00]
+; X64-NOOPT-LABEL: test_mul_by_14:
+; X64-NOOPT:       # %bb.0:
+; X64-NOOPT-NEXT:    imulq $14, %rdi, %rax
+; X64-NOOPT-NEXT:    retq
+; X86-OPT-LABEL: test_mul_by_14:
+; X86-OPT:       # %bb.0:
+; X86-OPT-NEXT:    movq %rdi, %rax
+; X86-OPT-NEXT:    shlq $4, %rax
+; X86-OPT-NEXT:    subq %rdi, %rax
+; X86-OPT-NEXT:    subq %rdi, %rax
+; X86-OPT-NEXT:    retq
   %mul = mul nsw i64 %x, 14
   ret i64 %mul
 }
@@ -756,17 +547,11 @@ define i64 @test_mul_by_15(i64 %x) {
 ; X86-NEXT:    addl %ecx, %edx
 ; X86-NEXT:    retl
 ;
-; X64-HSW-LABEL: test_mul_by_15:
-; X64-HSW:       # %bb.0:
-; X64-HSW-NEXT:    leaq (%rdi,%rdi,4), %rax # sched: [1:0.50]
-; X64-HSW-NEXT:    leaq (%rax,%rax,2), %rax # sched: [1:0.50]
-; X64-HSW-NEXT:    retq # sched: [7:1.00]
-;
-; X64-JAG-LABEL: test_mul_by_15:
-; X64-JAG:       # %bb.0:
-; X64-JAG-NEXT:    leaq (%rdi,%rdi,4), %rax # sched: [2:1.00]
-; X64-JAG-NEXT:    leaq (%rax,%rax,2), %rax # sched: [2:1.00]
-; X64-JAG-NEXT:    retq # sched: [4:1.00]
+; X64-OPT-LABEL: test_mul_by_15:
+; X64-OPT:       # %bb.0:
+; X64-OPT-NEXT:    leaq (%rdi,%rdi,4), %rax
+; X64-OPT-NEXT:    leaq (%rax,%rax,2), %rax
+; X64-OPT-NEXT:    retq
 ;
 ; X86-NOOPT-LABEL: test_mul_by_15:
 ; X86-NOOPT:       # %bb.0:
@@ -776,26 +561,15 @@ define i64 @test_mul_by_15(i64 %x) {
 ; X86-NOOPT-NEXT:    addl %ecx, %edx
 ; X86-NOOPT-NEXT:    retl
 ;
-; HSW-NOOPT-LABEL: test_mul_by_15:
-; HSW-NOOPT:       # %bb.0:
-; HSW-NOOPT-NEXT:    imulq $15, %rdi, %rax # sched: [3:1.00]
-; HSW-NOOPT-NEXT:    retq # sched: [7:1.00]
-;
-; JAG-NOOPT-LABEL: test_mul_by_15:
-; JAG-NOOPT:       # %bb.0:
-; JAG-NOOPT-NEXT:    imulq $15, %rdi, %rax # sched: [6:4.00]
-; JAG-NOOPT-NEXT:    retq # sched: [4:1.00]
-;
-; X64-SLM-LABEL: test_mul_by_15:
-; X64-SLM:       # %bb.0:
-; X64-SLM-NEXT:    leaq (%rdi,%rdi,4), %rax # sched: [1:1.00]
-; X64-SLM-NEXT:    leaq (%rax,%rax,2), %rax # sched: [1:1.00]
-; X64-SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SLM-NOOPT-LABEL: test_mul_by_15:
-; SLM-NOOPT:       # %bb.0:
-; SLM-NOOPT-NEXT:    imulq $15, %rdi, %rax # sched: [3:1.00]
-; SLM-NOOPT-NEXT:    retq # sched: [4:1.00]
+; X64-NOOPT-LABEL: test_mul_by_15:
+; X64-NOOPT:       # %bb.0:
+; X64-NOOPT-NEXT:    imulq $15, %rdi, %rax
+; X64-NOOPT-NEXT:    retq
+; X86-OPT-LABEL: test_mul_by_15:
+; X86-OPT:       # %bb.0:
+; X86-OPT-NEXT:    leaq (%rdi,%rdi,4), %rax
+; X86-OPT-NEXT:    leaq (%rax,%rax,2), %rax
+; X86-OPT-NEXT:    retq
   %mul = mul nsw i64 %x, 15
   ret i64 %mul
 }
@@ -809,17 +583,11 @@ define i64 @test_mul_by_16(i64 %x) {
 ; X86-NEXT:    shll $4, %eax
 ; X86-NEXT:    retl
 ;
-; X64-HSW-LABEL: test_mul_by_16:
-; X64-HSW:       # %bb.0:
-; X64-HSW-NEXT:    movq %rdi, %rax # sched: [1:0.25]
-; X64-HSW-NEXT:    shlq $4, %rax # sched: [1:0.50]
-; X64-HSW-NEXT:    retq # sched: [7:1.00]
-;
-; X64-JAG-LABEL: test_mul_by_16:
-; X64-JAG:       # %bb.0:
-; X64-JAG-NEXT:    movq %rdi, %rax # sched: [1:0.50]
-; X64-JAG-NEXT:    shlq $4, %rax # sched: [1:0.50]
-; X64-JAG-NEXT:    retq # sched: [4:1.00]
+; X64-OPT-LABEL: test_mul_by_16:
+; X64-OPT:       # %bb.0:
+; X64-OPT-NEXT:    movq %rdi, %rax
+; X64-OPT-NEXT:    shlq $4, %rax
+; X64-OPT-NEXT:    retq
 ;
 ; X86-NOOPT-LABEL: test_mul_by_16:
 ; X86-NOOPT:       # %bb.0:
@@ -829,29 +597,16 @@ define i64 @test_mul_by_16(i64 %x) {
 ; X86-NOOPT-NEXT:    shll $4, %eax
 ; X86-NOOPT-NEXT:    retl
 ;
-; HSW-NOOPT-LABEL: test_mul_by_16:
-; HSW-NOOPT:       # %bb.0:
-; HSW-NOOPT-NEXT:    movq %rdi, %rax # sched: [1:0.25]
-; HSW-NOOPT-NEXT:    shlq $4, %rax # sched: [1:0.50]
-; HSW-NOOPT-NEXT:    retq # sched: [7:1.00]
-;
-; JAG-NOOPT-LABEL: test_mul_by_16:
-; JAG-NOOPT:       # %bb.0:
-; JAG-NOOPT-NEXT:    movq %rdi, %rax # sched: [1:0.50]
-; JAG-NOOPT-NEXT:    shlq $4, %rax # sched: [1:0.50]
-; JAG-NOOPT-NEXT:    retq # sched: [4:1.00]
-;
-; X64-SLM-LABEL: test_mul_by_16:
-; X64-SLM:       # %bb.0:
-; X64-SLM-NEXT:    movq %rdi, %rax # sched: [1:0.50]
-; X64-SLM-NEXT:    shlq $4, %rax # sched: [1:1.00]
-; X64-SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SLM-NOOPT-LABEL: test_mul_by_16:
-; SLM-NOOPT:       # %bb.0:
-; SLM-NOOPT-NEXT:    movq %rdi, %rax # sched: [1:0.50]
-; SLM-NOOPT-NEXT:    shlq $4, %rax # sched: [1:1.00]
-; SLM-NOOPT-NEXT:    retq # sched: [4:1.00]
+; X64-NOOPT-LABEL: test_mul_by_16:
+; X64-NOOPT:       # %bb.0:
+; X64-NOOPT-NEXT:    movq %rdi, %rax
+; X64-NOOPT-NEXT:    shlq $4, %rax
+; X64-NOOPT-NEXT:    retq
+; X86-OPT-LABEL: test_mul_by_16:
+; X86-OPT:       # %bb.0:
+; X86-OPT-NEXT:    movq %rdi, %rax
+; X86-OPT-NEXT:    shlq $4, %rax
+; X86-OPT-NEXT:    retq
   %mul = mul nsw i64 %x, 16
   ret i64 %mul
 }
@@ -870,17 +625,17 @@ define i64 @test_mul_by_17(i64 %x) {
 ;
 ; X64-HSW-LABEL: test_mul_by_17:
 ; X64-HSW:       # %bb.0:
-; X64-HSW-NEXT:    movq %rdi, %rax # sched: [1:0.25]
-; X64-HSW-NEXT:    shlq $4, %rax # sched: [1:0.50]
-; X64-HSW-NEXT:    leaq (%rax,%rdi), %rax # sched: [1:0.50]
-; X64-HSW-NEXT:    retq # sched: [7:1.00]
+; X64-HSW-NEXT:    movq %rdi, %rax
+; X64-HSW-NEXT:    shlq $4, %rax
+; X64-HSW-NEXT:    leaq (%rax,%rdi), %rax
+; X64-HSW-NEXT:    retq
 ;
 ; X64-JAG-LABEL: test_mul_by_17:
 ; X64-JAG:       # %bb.0:
-; X64-JAG-NEXT:    movq %rdi, %rax # sched: [1:0.50]
-; X64-JAG-NEXT:    shlq $4, %rax # sched: [1:0.50]
-; X64-JAG-NEXT:    leaq (%rax,%rdi), %rax # sched: [1:0.50]
-; X64-JAG-NEXT:    retq # sched: [4:1.00]
+; X64-JAG-NEXT:    movq %rdi, %rax
+; X64-JAG-NEXT:    shlq $4, %rax
+; X64-JAG-NEXT:    leaq (%rax,%rdi), %rax
+; X64-JAG-NEXT:    retq
 ;
 ; X86-NOOPT-LABEL: test_mul_by_17:
 ; X86-NOOPT:       # %bb.0:
@@ -890,27 +645,23 @@ define i64 @test_mul_by_17(i64 %x) {
 ; X86-NOOPT-NEXT:    addl %ecx, %edx
 ; X86-NOOPT-NEXT:    retl
 ;
-; HSW-NOOPT-LABEL: test_mul_by_17:
-; HSW-NOOPT:       # %bb.0:
-; HSW-NOOPT-NEXT:    imulq $17, %rdi, %rax # sched: [3:1.00]
-; HSW-NOOPT-NEXT:    retq # sched: [7:1.00]
-;
-; JAG-NOOPT-LABEL: test_mul_by_17:
-; JAG-NOOPT:       # %bb.0:
-; JAG-NOOPT-NEXT:    imulq $17, %rdi, %rax # sched: [6:4.00]
-; JAG-NOOPT-NEXT:    retq # sched: [4:1.00]
+; X64-NOOPT-LABEL: test_mul_by_17:
+; X64-NOOPT:       # %bb.0:
+; X64-NOOPT-NEXT:    imulq $17, %rdi, %rax
+; X64-NOOPT-NEXT:    retq
 ;
 ; X64-SLM-LABEL: test_mul_by_17:
 ; X64-SLM:       # %bb.0:
-; X64-SLM-NEXT:    movq %rdi, %rax # sched: [1:0.50]
-; X64-SLM-NEXT:    shlq $4, %rax # sched: [1:1.00]
-; X64-SLM-NEXT:    addq %rdi, %rax # sched: [1:0.50]
-; X64-SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SLM-NOOPT-LABEL: test_mul_by_17:
-; SLM-NOOPT:       # %bb.0:
-; SLM-NOOPT-NEXT:    imulq $17, %rdi, %rax # sched: [3:1.00]
-; SLM-NOOPT-NEXT:    retq # sched: [4:1.00]
+; X64-SLM-NEXT:    movq %rdi, %rax
+; X64-SLM-NEXT:    shlq $4, %rax
+; X64-SLM-NEXT:    addq %rdi, %rax
+; X64-SLM-NEXT:    retq
+; X86-OPT-LABEL: test_mul_by_17:
+; X86-OPT:       # %bb.0:
+; X86-OPT-NEXT:    movq %rdi, %rax
+; X86-OPT-NEXT:    shlq $4, %rax
+; X86-OPT-NEXT:    leaq (%rax,%rdi), %rax
+; X86-OPT-NEXT:    retq
   %mul = mul nsw i64 %x, 17
   ret i64 %mul
 }
@@ -925,17 +676,11 @@ define i64 @test_mul_by_18(i64 %x) {
 ; X86-NEXT:    leal (%edx,%ecx,2), %edx
 ; X86-NEXT:    retl
 ;
-; X64-HSW-LABEL: test_mul_by_18:
-; X64-HSW:       # %bb.0:
-; X64-HSW-NEXT:    addq %rdi, %rdi # sched: [1:0.25]
-; X64-HSW-NEXT:    leaq (%rdi,%rdi,8), %rax # sched: [1:0.50]
-; X64-HSW-NEXT:    retq # sched: [7:1.00]
-;
-; X64-JAG-LABEL: test_mul_by_18:
-; X64-JAG:       # %bb.0:
-; X64-JAG-NEXT:    addq %rdi, %rdi # sched: [1:0.50]
-; X64-JAG-NEXT:    leaq (%rdi,%rdi,8), %rax # sched: [2:1.00]
-; X64-JAG-NEXT:    retq # sched: [4:1.00]
+; X64-OPT-LABEL: test_mul_by_18:
+; X64-OPT:       # %bb.0:
+; X64-OPT-NEXT:    addq %rdi, %rdi
+; X64-OPT-NEXT:    leaq (%rdi,%rdi,8), %rax
+; X64-OPT-NEXT:    retq
 ;
 ; X86-NOOPT-LABEL: test_mul_by_18:
 ; X86-NOOPT:       # %bb.0:
@@ -945,26 +690,15 @@ define i64 @test_mul_by_18(i64 %x) {
 ; X86-NOOPT-NEXT:    addl %ecx, %edx
 ; X86-NOOPT-NEXT:    retl
 ;
-; HSW-NOOPT-LABEL: test_mul_by_18:
-; HSW-NOOPT:       # %bb.0:
-; HSW-NOOPT-NEXT:    imulq $18, %rdi, %rax # sched: [3:1.00]
-; HSW-NOOPT-NEXT:    retq # sched: [7:1.00]
-;
-; JAG-NOOPT-LABEL: test_mul_by_18:
-; JAG-NOOPT:       # %bb.0:
-; JAG-NOOPT-NEXT:    imulq $18, %rdi, %rax # sched: [6:4.00]
-; JAG-NOOPT-NEXT:    retq # sched: [4:1.00]
-;
-; X64-SLM-LABEL: test_mul_by_18:
-; X64-SLM:       # %bb.0:
-; X64-SLM-NEXT:    addq %rdi, %rdi # sched: [1:0.50]
-; X64-SLM-NEXT:    leaq (%rdi,%rdi,8), %rax # sched: [1:1.00]
-; X64-SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SLM-NOOPT-LABEL: test_mul_by_18:
-; SLM-NOOPT:       # %bb.0:
-; SLM-NOOPT-NEXT:    imulq $18, %rdi, %rax # sched: [3:1.00]
-; SLM-NOOPT-NEXT:    retq # sched: [4:1.00]
+; X64-NOOPT-LABEL: test_mul_by_18:
+; X64-NOOPT:       # %bb.0:
+; X64-NOOPT-NEXT:    imulq $18, %rdi, %rax
+; X64-NOOPT-NEXT:    retq
+; X86-OPT-LABEL: test_mul_by_18:
+; X86-OPT:       # %bb.0:
+; X86-OPT-NEXT:    addq %rdi, %rdi
+; X86-OPT-NEXT:    leaq (%rdi,%rdi,8), %rax
+; X86-OPT-NEXT:    retq
   %mul = mul nsw i64 %x, 18
   ret i64 %mul
 }
@@ -982,15 +716,15 @@ define i64 @test_mul_by_19(i64 %x) {
 ;
 ; X64-HSW-LABEL: test_mul_by_19:
 ; X64-HSW:       # %bb.0:
-; X64-HSW-NEXT:    leaq (%rdi,%rdi,8), %rax # sched: [1:0.50]
-; X64-HSW-NEXT:    leaq (%rdi,%rax,2), %rax # sched: [1:0.50]
-; X64-HSW-NEXT:    retq # sched: [7:1.00]
+; X64-HSW-NEXT:    leaq (%rdi,%rdi,8), %rax
+; X64-HSW-NEXT:    leaq (%rdi,%rax,2), %rax
+; X64-HSW-NEXT:    retq
 ;
 ; X64-JAG-LABEL: test_mul_by_19:
 ; X64-JAG:       # %bb.0:
-; X64-JAG-NEXT:    leaq (%rdi,%rdi,8), %rax # sched: [2:1.00]
-; X64-JAG-NEXT:    leaq (%rdi,%rax,2), %rax # sched: [2:1.00]
-; X64-JAG-NEXT:    retq # sched: [4:1.00]
+; X64-JAG-NEXT:    leaq (%rdi,%rdi,8), %rax
+; X64-JAG-NEXT:    leaq (%rdi,%rax,2), %rax
+; X64-JAG-NEXT:    retq
 ;
 ; X86-NOOPT-LABEL: test_mul_by_19:
 ; X86-NOOPT:       # %bb.0:
@@ -1000,25 +734,20 @@ define i64 @test_mul_by_19(i64 %x) {
 ; X86-NOOPT-NEXT:    addl %ecx, %edx
 ; X86-NOOPT-NEXT:    retl
 ;
-; HSW-NOOPT-LABEL: test_mul_by_19:
-; HSW-NOOPT:       # %bb.0:
-; HSW-NOOPT-NEXT:    imulq $19, %rdi, %rax # sched: [3:1.00]
-; HSW-NOOPT-NEXT:    retq # sched: [7:1.00]
-;
-; JAG-NOOPT-LABEL: test_mul_by_19:
-; JAG-NOOPT:       # %bb.0:
-; JAG-NOOPT-NEXT:    imulq $19, %rdi, %rax # sched: [6:4.00]
-; JAG-NOOPT-NEXT:    retq # sched: [4:1.00]
+; X64-NOOPT-LABEL: test_mul_by_19:
+; X64-NOOPT:       # %bb.0:
+; X64-NOOPT-NEXT:    imulq $19, %rdi, %rax
+; X64-NOOPT-NEXT:    retq
 ;
 ; X64-SLM-LABEL: test_mul_by_19:
 ; X64-SLM:       # %bb.0:
-; X64-SLM-NEXT:    imulq $19, %rdi, %rax # sched: [3:1.00]
-; X64-SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SLM-NOOPT-LABEL: test_mul_by_19:
-; SLM-NOOPT:       # %bb.0:
-; SLM-NOOPT-NEXT:    imulq $19, %rdi, %rax # sched: [3:1.00]
-; SLM-NOOPT-NEXT:    retq # sched: [4:1.00]
+; X64-SLM-NEXT:    imulq $19, %rdi, %rax
+; X64-SLM-NEXT:    retq
+; X86-OPT-LABEL: test_mul_by_19:
+; X86-OPT:       # %bb.0:
+; X86-OPT-NEXT:    leaq (%rdi,%rdi,8), %rax
+; X86-OPT-NEXT:    leaq (%rdi,%rax,2), %rax
+; X86-OPT-NEXT:    retq
   %mul = mul nsw i64 %x, 19
   ret i64 %mul
 }
@@ -1033,17 +762,11 @@ define i64 @test_mul_by_20(i64 %x) {
 ; X86-NEXT:    leal (%edx,%ecx,4), %edx
 ; X86-NEXT:    retl
 ;
-; X64-HSW-LABEL: test_mul_by_20:
-; X64-HSW:       # %bb.0:
-; X64-HSW-NEXT:    shlq $2, %rdi # sched: [1:0.50]
-; X64-HSW-NEXT:    leaq (%rdi,%rdi,4), %rax # sched: [1:0.50]
-; X64-HSW-NEXT:    retq # sched: [7:1.00]
-;
-; X64-JAG-LABEL: test_mul_by_20:
-; X64-JAG:       # %bb.0:
-; X64-JAG-NEXT:    shlq $2, %rdi # sched: [1:0.50]
-; X64-JAG-NEXT:    leaq (%rdi,%rdi,4), %rax # sched: [2:1.00]
-; X64-JAG-NEXT:    retq # sched: [4:1.00]
+; X64-OPT-LABEL: test_mul_by_20:
+; X64-OPT:       # %bb.0:
+; X64-OPT-NEXT:    shlq $2, %rdi
+; X64-OPT-NEXT:    leaq (%rdi,%rdi,4), %rax
+; X64-OPT-NEXT:    retq
 ;
 ; X86-NOOPT-LABEL: test_mul_by_20:
 ; X86-NOOPT:       # %bb.0:
@@ -1053,26 +776,15 @@ define i64 @test_mul_by_20(i64 %x) {
 ; X86-NOOPT-NEXT:    addl %ecx, %edx
 ; X86-NOOPT-NEXT:    retl
 ;
-; HSW-NOOPT-LABEL: test_mul_by_20:
-; HSW-NOOPT:       # %bb.0:
-; HSW-NOOPT-NEXT:    imulq $20, %rdi, %rax # sched: [3:1.00]
-; HSW-NOOPT-NEXT:    retq # sched: [7:1.00]
-;
-; JAG-NOOPT-LABEL: test_mul_by_20:
-; JAG-NOOPT:       # %bb.0:
-; JAG-NOOPT-NEXT:    imulq $20, %rdi, %rax # sched: [6:4.00]
-; JAG-NOOPT-NEXT:    retq # sched: [4:1.00]
-;
-; X64-SLM-LABEL: test_mul_by_20:
-; X64-SLM:       # %bb.0:
-; X64-SLM-NEXT:    shlq $2, %rdi # sched: [1:1.00]
-; X64-SLM-NEXT:    leaq (%rdi,%rdi,4), %rax # sched: [1:1.00]
-; X64-SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SLM-NOOPT-LABEL: test_mul_by_20:
-; SLM-NOOPT:       # %bb.0:
-; SLM-NOOPT-NEXT:    imulq $20, %rdi, %rax # sched: [3:1.00]
-; SLM-NOOPT-NEXT:    retq # sched: [4:1.00]
+; X64-NOOPT-LABEL: test_mul_by_20:
+; X64-NOOPT:       # %bb.0:
+; X64-NOOPT-NEXT:    imulq $20, %rdi, %rax
+; X64-NOOPT-NEXT:    retq
+; X86-OPT-LABEL: test_mul_by_20:
+; X86-OPT:       # %bb.0:
+; X86-OPT-NEXT:    shlq $2, %rdi
+; X86-OPT-NEXT:    leaq (%rdi,%rdi,4), %rax
+; X86-OPT-NEXT:    retq
   %mul = mul nsw i64 %x, 20
   ret i64 %mul
 }
@@ -1090,15 +802,15 @@ define i64 @test_mul_by_21(i64 %x) {
 ;
 ; X64-HSW-LABEL: test_mul_by_21:
 ; X64-HSW:       # %bb.0:
-; X64-HSW-NEXT:    leaq (%rdi,%rdi,4), %rax # sched: [1:0.50]
-; X64-HSW-NEXT:    leaq (%rdi,%rax,4), %rax # sched: [1:0.50]
-; X64-HSW-NEXT:    retq # sched: [7:1.00]
+; X64-HSW-NEXT:    leaq (%rdi,%rdi,4), %rax
+; X64-HSW-NEXT:    leaq (%rdi,%rax,4), %rax
+; X64-HSW-NEXT:    retq
 ;
 ; X64-JAG-LABEL: test_mul_by_21:
 ; X64-JAG:       # %bb.0:
-; X64-JAG-NEXT:    leaq (%rdi,%rdi,4), %rax # sched: [2:1.00]
-; X64-JAG-NEXT:    leaq (%rdi,%rax,4), %rax # sched: [2:1.00]
-; X64-JAG-NEXT:    retq # sched: [4:1.00]
+; X64-JAG-NEXT:    leaq (%rdi,%rdi,4), %rax
+; X64-JAG-NEXT:    leaq (%rdi,%rax,4), %rax
+; X64-JAG-NEXT:    retq
 ;
 ; X86-NOOPT-LABEL: test_mul_by_21:
 ; X86-NOOPT:       # %bb.0:
@@ -1108,25 +820,20 @@ define i64 @test_mul_by_21(i64 %x) {
 ; X86-NOOPT-NEXT:    addl %ecx, %edx
 ; X86-NOOPT-NEXT:    retl
 ;
-; HSW-NOOPT-LABEL: test_mul_by_21:
-; HSW-NOOPT:       # %bb.0:
-; HSW-NOOPT-NEXT:    imulq $21, %rdi, %rax # sched: [3:1.00]
-; HSW-NOOPT-NEXT:    retq # sched: [7:1.00]
-;
-; JAG-NOOPT-LABEL: test_mul_by_21:
-; JAG-NOOPT:       # %bb.0:
-; JAG-NOOPT-NEXT:    imulq $21, %rdi, %rax # sched: [6:4.00]
-; JAG-NOOPT-NEXT:    retq # sched: [4:1.00]
+; X64-NOOPT-LABEL: test_mul_by_21:
+; X64-NOOPT:       # %bb.0:
+; X64-NOOPT-NEXT:    imulq $21, %rdi, %rax
+; X64-NOOPT-NEXT:    retq
 ;
 ; X64-SLM-LABEL: test_mul_by_21:
 ; X64-SLM:       # %bb.0:
-; X64-SLM-NEXT:    imulq $21, %rdi, %rax # sched: [3:1.00]
-; X64-SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SLM-NOOPT-LABEL: test_mul_by_21:
-; SLM-NOOPT:       # %bb.0:
-; SLM-NOOPT-NEXT:    imulq $21, %rdi, %rax # sched: [3:1.00]
-; SLM-NOOPT-NEXT:    retq # sched: [4:1.00]
+; X64-SLM-NEXT:    imulq $21, %rdi, %rax
+; X64-SLM-NEXT:    retq
+; X86-OPT-LABEL: test_mul_by_21:
+; X86-OPT:       # %bb.0:
+; X86-OPT-NEXT:    leaq (%rdi,%rdi,4), %rax
+; X86-OPT-NEXT:    leaq (%rdi,%rax,4), %rax
+; X86-OPT-NEXT:    retq
   %mul = mul nsw i64 %x, 21
   ret i64 %mul
 }
@@ -1145,17 +852,17 @@ define i64 @test_mul_by_22(i64 %x) {
 ;
 ; X64-HSW-LABEL: test_mul_by_22:
 ; X64-HSW:       # %bb.0:
-; X64-HSW-NEXT:    leaq (%rdi,%rdi,4), %rax # sched: [1:0.50]
-; X64-HSW-NEXT:    leaq (%rdi,%rax,4), %rax # sched: [1:0.50]
-; X64-HSW-NEXT:    addq %rdi, %rax # sched: [1:0.25]
-; X64-HSW-NEXT:    retq # sched: [7:1.00]
+; X64-HSW-NEXT:    leaq (%rdi,%rdi,4), %rax
+; X64-HSW-NEXT:    leaq (%rdi,%rax,4), %rax
+; X64-HSW-NEXT:    addq %rdi, %rax
+; X64-HSW-NEXT:    retq
 ;
 ; X64-JAG-LABEL: test_mul_by_22:
 ; X64-JAG:       # %bb.0:
-; X64-JAG-NEXT:    leaq (%rdi,%rdi,4), %rax # sched: [2:1.00]
-; X64-JAG-NEXT:    leaq (%rdi,%rax,4), %rax # sched: [2:1.00]
-; X64-JAG-NEXT:    addq %rdi, %rax # sched: [1:0.50]
-; X64-JAG-NEXT:    retq # sched: [4:1.00]
+; X64-JAG-NEXT:    leaq (%rdi,%rdi,4), %rax
+; X64-JAG-NEXT:    leaq (%rdi,%rax,4), %rax
+; X64-JAG-NEXT:    addq %rdi, %rax
+; X64-JAG-NEXT:    retq
 ;
 ; X86-NOOPT-LABEL: test_mul_by_22:
 ; X86-NOOPT:       # %bb.0:
@@ -1165,25 +872,21 @@ define i64 @test_mul_by_22(i64 %x) {
 ; X86-NOOPT-NEXT:    addl %ecx, %edx
 ; X86-NOOPT-NEXT:    retl
 ;
-; HSW-NOOPT-LABEL: test_mul_by_22:
-; HSW-NOOPT:       # %bb.0:
-; HSW-NOOPT-NEXT:    imulq $22, %rdi, %rax # sched: [3:1.00]
-; HSW-NOOPT-NEXT:    retq # sched: [7:1.00]
-;
-; JAG-NOOPT-LABEL: test_mul_by_22:
-; JAG-NOOPT:       # %bb.0:
-; JAG-NOOPT-NEXT:    imulq $22, %rdi, %rax # sched: [6:4.00]
-; JAG-NOOPT-NEXT:    retq # sched: [4:1.00]
+; X64-NOOPT-LABEL: test_mul_by_22:
+; X64-NOOPT:       # %bb.0:
+; X64-NOOPT-NEXT:    imulq $22, %rdi, %rax
+; X64-NOOPT-NEXT:    retq
 ;
 ; X64-SLM-LABEL: test_mul_by_22:
 ; X64-SLM:       # %bb.0:
-; X64-SLM-NEXT:    imulq $22, %rdi, %rax # sched: [3:1.00]
-; X64-SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SLM-NOOPT-LABEL: test_mul_by_22:
-; SLM-NOOPT:       # %bb.0:
-; SLM-NOOPT-NEXT:    imulq $22, %rdi, %rax # sched: [3:1.00]
-; SLM-NOOPT-NEXT:    retq # sched: [4:1.00]
+; X64-SLM-NEXT:    imulq $22, %rdi, %rax
+; X64-SLM-NEXT:    retq
+; X86-OPT-LABEL: test_mul_by_22:
+; X86-OPT:       # %bb.0:
+; X86-OPT-NEXT:    leaq (%rdi,%rdi,4), %rax
+; X86-OPT-NEXT:    leaq (%rdi,%rax,4), %rax
+; X86-OPT-NEXT:    addq %rdi, %rax
+; X86-OPT-NEXT:    retq
   %mul = mul nsw i64 %x, 22
   ret i64 %mul
 }
@@ -1202,17 +905,17 @@ define i64 @test_mul_by_23(i64 %x) {
 ;
 ; X64-HSW-LABEL: test_mul_by_23:
 ; X64-HSW:       # %bb.0:
-; X64-HSW-NEXT:    leaq (%rdi,%rdi,2), %rax # sched: [1:0.50]
-; X64-HSW-NEXT:    shlq $3, %rax # sched: [1:0.50]
-; X64-HSW-NEXT:    subq %rdi, %rax # sched: [1:0.25]
-; X64-HSW-NEXT:    retq # sched: [7:1.00]
+; X64-HSW-NEXT:    leaq (%rdi,%rdi,2), %rax
+; X64-HSW-NEXT:    shlq $3, %rax
+; X64-HSW-NEXT:    subq %rdi, %rax
+; X64-HSW-NEXT:    retq
 ;
 ; X64-JAG-LABEL: test_mul_by_23:
 ; X64-JAG:       # %bb.0:
-; X64-JAG-NEXT:    leaq (%rdi,%rdi,2), %rax # sched: [2:1.00]
-; X64-JAG-NEXT:    shlq $3, %rax # sched: [1:0.50]
-; X64-JAG-NEXT:    subq %rdi, %rax # sched: [1:0.50]
-; X64-JAG-NEXT:    retq # sched: [4:1.00]
+; X64-JAG-NEXT:    leaq (%rdi,%rdi,2), %rax
+; X64-JAG-NEXT:    shlq $3, %rax
+; X64-JAG-NEXT:    subq %rdi, %rax
+; X64-JAG-NEXT:    retq
 ;
 ; X86-NOOPT-LABEL: test_mul_by_23:
 ; X86-NOOPT:       # %bb.0:
@@ -1222,25 +925,21 @@ define i64 @test_mul_by_23(i64 %x) {
 ; X86-NOOPT-NEXT:    addl %ecx, %edx
 ; X86-NOOPT-NEXT:    retl
 ;
-; HSW-NOOPT-LABEL: test_mul_by_23:
-; HSW-NOOPT:       # %bb.0:
-; HSW-NOOPT-NEXT:    imulq $23, %rdi, %rax # sched: [3:1.00]
-; HSW-NOOPT-NEXT:    retq # sched: [7:1.00]
-;
-; JAG-NOOPT-LABEL: test_mul_by_23:
-; JAG-NOOPT:       # %bb.0:
-; JAG-NOOPT-NEXT:    imulq $23, %rdi, %rax # sched: [6:4.00]
-; JAG-NOOPT-NEXT:    retq # sched: [4:1.00]
+; X64-NOOPT-LABEL: test_mul_by_23:
+; X64-NOOPT:       # %bb.0:
+; X64-NOOPT-NEXT:    imulq $23, %rdi, %rax
+; X64-NOOPT-NEXT:    retq
 ;
 ; X64-SLM-LABEL: test_mul_by_23:
 ; X64-SLM:       # %bb.0:
-; X64-SLM-NEXT:    imulq $23, %rdi, %rax # sched: [3:1.00]
-; X64-SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SLM-NOOPT-LABEL: test_mul_by_23:
-; SLM-NOOPT:       # %bb.0:
-; SLM-NOOPT-NEXT:    imulq $23, %rdi, %rax # sched: [3:1.00]
-; SLM-NOOPT-NEXT:    retq # sched: [4:1.00]
+; X64-SLM-NEXT:    imulq $23, %rdi, %rax
+; X64-SLM-NEXT:    retq
+; X86-OPT-LABEL: test_mul_by_23:
+; X86-OPT:       # %bb.0:
+; X86-OPT-NEXT:    leaq (%rdi,%rdi,2), %rax
+; X86-OPT-NEXT:    shlq $3, %rax
+; X86-OPT-NEXT:    subq %rdi, %rax
+; X86-OPT-NEXT:    retq
   %mul = mul nsw i64 %x, 23
   ret i64 %mul
 }
@@ -1255,17 +954,11 @@ define i64 @test_mul_by_24(i64 %x) {
 ; X86-NEXT:    leal (%edx,%ecx,8), %edx
 ; X86-NEXT:    retl
 ;
-; X64-HSW-LABEL: test_mul_by_24:
-; X64-HSW:       # %bb.0:
-; X64-HSW-NEXT:    shlq $3, %rdi # sched: [1:0.50]
-; X64-HSW-NEXT:    leaq (%rdi,%rdi,2), %rax # sched: [1:0.50]
-; X64-HSW-NEXT:    retq # sched: [7:1.00]
-;
-; X64-JAG-LABEL: test_mul_by_24:
-; X64-JAG:       # %bb.0:
-; X64-JAG-NEXT:    shlq $3, %rdi # sched: [1:0.50]
-; X64-JAG-NEXT:    leaq (%rdi,%rdi,2), %rax # sched: [2:1.00]
-; X64-JAG-NEXT:    retq # sched: [4:1.00]
+; X64-OPT-LABEL: test_mul_by_24:
+; X64-OPT:       # %bb.0:
+; X64-OPT-NEXT:    shlq $3, %rdi
+; X64-OPT-NEXT:    leaq (%rdi,%rdi,2), %rax
+; X64-OPT-NEXT:    retq
 ;
 ; X86-NOOPT-LABEL: test_mul_by_24:
 ; X86-NOOPT:       # %bb.0:
@@ -1275,26 +968,15 @@ define i64 @test_mul_by_24(i64 %x) {
 ; X86-NOOPT-NEXT:    addl %ecx, %edx
 ; X86-NOOPT-NEXT:    retl
 ;
-; HSW-NOOPT-LABEL: test_mul_by_24:
-; HSW-NOOPT:       # %bb.0:
-; HSW-NOOPT-NEXT:    imulq $24, %rdi, %rax # sched: [3:1.00]
-; HSW-NOOPT-NEXT:    retq # sched: [7:1.00]
-;
-; JAG-NOOPT-LABEL: test_mul_by_24:
-; JAG-NOOPT:       # %bb.0:
-; JAG-NOOPT-NEXT:    imulq $24, %rdi, %rax # sched: [6:4.00]
-; JAG-NOOPT-NEXT:    retq # sched: [4:1.00]
-;
-; X64-SLM-LABEL: test_mul_by_24:
-; X64-SLM:       # %bb.0:
-; X64-SLM-NEXT:    shlq $3, %rdi # sched: [1:1.00]
-; X64-SLM-NEXT:    leaq (%rdi,%rdi,2), %rax # sched: [1:1.00]
-; X64-SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SLM-NOOPT-LABEL: test_mul_by_24:
-; SLM-NOOPT:       # %bb.0:
-; SLM-NOOPT-NEXT:    imulq $24, %rdi, %rax # sched: [3:1.00]
-; SLM-NOOPT-NEXT:    retq # sched: [4:1.00]
+; X64-NOOPT-LABEL: test_mul_by_24:
+; X64-NOOPT:       # %bb.0:
+; X64-NOOPT-NEXT:    imulq $24, %rdi, %rax
+; X64-NOOPT-NEXT:    retq
+; X86-OPT-LABEL: test_mul_by_24:
+; X86-OPT:       # %bb.0:
+; X86-OPT-NEXT:    shlq $3, %rdi
+; X86-OPT-NEXT:    leaq (%rdi,%rdi,2), %rax
+; X86-OPT-NEXT:    retq
   %mul = mul nsw i64 %x, 24
   ret i64 %mul
 }
@@ -1310,17 +992,11 @@ define i64 @test_mul_by_25(i64 %x) {
 ; X86-NEXT:    addl %ecx, %edx
 ; X86-NEXT:    retl
 ;
-; X64-HSW-LABEL: test_mul_by_25:
-; X64-HSW:       # %bb.0:
-; X64-HSW-NEXT:    leaq (%rdi,%rdi,4), %rax # sched: [1:0.50]
-; X64-HSW-NEXT:    leaq (%rax,%rax,4), %rax # sched: [1:0.50]
-; X64-HSW-NEXT:    retq # sched: [7:1.00]
-;
-; X64-JAG-LABEL: test_mul_by_25:
-; X64-JAG:       # %bb.0:
-; X64-JAG-NEXT:    leaq (%rdi,%rdi,4), %rax # sched: [2:1.00]
-; X64-JAG-NEXT:    leaq (%rax,%rax,4), %rax # sched: [2:1.00]
-; X64-JAG-NEXT:    retq # sched: [4:1.00]
+; X64-OPT-LABEL: test_mul_by_25:
+; X64-OPT:       # %bb.0:
+; X64-OPT-NEXT:    leaq (%rdi,%rdi,4), %rax
+; X64-OPT-NEXT:    leaq (%rax,%rax,4), %rax
+; X64-OPT-NEXT:    retq
 ;
 ; X86-NOOPT-LABEL: test_mul_by_25:
 ; X86-NOOPT:       # %bb.0:
@@ -1330,26 +1006,15 @@ define i64 @test_mul_by_25(i64 %x) {
 ; X86-NOOPT-NEXT:    addl %ecx, %edx
 ; X86-NOOPT-NEXT:    retl
 ;
-; HSW-NOOPT-LABEL: test_mul_by_25:
-; HSW-NOOPT:       # %bb.0:
-; HSW-NOOPT-NEXT:    imulq $25, %rdi, %rax # sched: [3:1.00]
-; HSW-NOOPT-NEXT:    retq # sched: [7:1.00]
-;
-; JAG-NOOPT-LABEL: test_mul_by_25:
-; JAG-NOOPT:       # %bb.0:
-; JAG-NOOPT-NEXT:    imulq $25, %rdi, %rax # sched: [6:4.00]
-; JAG-NOOPT-NEXT:    retq # sched: [4:1.00]
-;
-; X64-SLM-LABEL: test_mul_by_25:
-; X64-SLM:       # %bb.0:
-; X64-SLM-NEXT:    leaq (%rdi,%rdi,4), %rax # sched: [1:1.00]
-; X64-SLM-NEXT:    leaq (%rax,%rax,4), %rax # sched: [1:1.00]
-; X64-SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SLM-NOOPT-LABEL: test_mul_by_25:
-; SLM-NOOPT:       # %bb.0:
-; SLM-NOOPT-NEXT:    imulq $25, %rdi, %rax # sched: [3:1.00]
-; SLM-NOOPT-NEXT:    retq # sched: [4:1.00]
+; X64-NOOPT-LABEL: test_mul_by_25:
+; X64-NOOPT:       # %bb.0:
+; X64-NOOPT-NEXT:    imulq $25, %rdi, %rax
+; X64-NOOPT-NEXT:    retq
+; X86-OPT-LABEL: test_mul_by_25:
+; X86-OPT:       # %bb.0:
+; X86-OPT-NEXT:    leaq (%rdi,%rdi,4), %rax
+; X86-OPT-NEXT:    leaq (%rax,%rax,4), %rax
+; X86-OPT-NEXT:    retq
   %mul = mul nsw i64 %x, 25
   ret i64 %mul
 }
@@ -1368,17 +1033,17 @@ define i64 @test_mul_by_26(i64 %x) {
 ;
 ; X64-HSW-LABEL: test_mul_by_26:
 ; X64-HSW:       # %bb.0:
-; X64-HSW-NEXT:    leaq (%rdi,%rdi,4), %rax # sched: [1:0.50]
-; X64-HSW-NEXT:    leaq (%rax,%rax,4), %rax # sched: [1:0.50]
-; X64-HSW-NEXT:    addq %rdi, %rax # sched: [1:0.25]
-; X64-HSW-NEXT:    retq # sched: [7:1.00]
+; X64-HSW-NEXT:    leaq (%rdi,%rdi,4), %rax
+; X64-HSW-NEXT:    leaq (%rax,%rax,4), %rax
+; X64-HSW-NEXT:    addq %rdi, %rax
+; X64-HSW-NEXT:    retq
 ;
 ; X64-JAG-LABEL: test_mul_by_26:
 ; X64-JAG:       # %bb.0:
-; X64-JAG-NEXT:    leaq (%rdi,%rdi,4), %rax # sched: [2:1.00]
-; X64-JAG-NEXT:    leaq (%rax,%rax,4), %rax # sched: [2:1.00]
-; X64-JAG-NEXT:    addq %rdi, %rax # sched: [1:0.50]
-; X64-JAG-NEXT:    retq # sched: [4:1.00]
+; X64-JAG-NEXT:    leaq (%rdi,%rdi,4), %rax
+; X64-JAG-NEXT:    leaq (%rax,%rax,4), %rax
+; X64-JAG-NEXT:    addq %rdi, %rax
+; X64-JAG-NEXT:    retq
 ;
 ; X86-NOOPT-LABEL: test_mul_by_26:
 ; X86-NOOPT:       # %bb.0:
@@ -1388,25 +1053,21 @@ define i64 @test_mul_by_26(i64 %x) {
 ; X86-NOOPT-NEXT:    addl %ecx, %edx
 ; X86-NOOPT-NEXT:    retl
 ;
-; HSW-NOOPT-LABEL: test_mul_by_26:
-; HSW-NOOPT:       # %bb.0:
-; HSW-NOOPT-NEXT:    imulq $26, %rdi, %rax # sched: [3:1.00]
-; HSW-NOOPT-NEXT:    retq # sched: [7:1.00]
-;
-; JAG-NOOPT-LABEL: test_mul_by_26:
-; JAG-NOOPT:       # %bb.0:
-; JAG-NOOPT-NEXT:    imulq $26, %rdi, %rax # sched: [6:4.00]
-; JAG-NOOPT-NEXT:    retq # sched: [4:1.00]
+; X64-NOOPT-LABEL: test_mul_by_26:
+; X64-NOOPT:       # %bb.0:
+; X64-NOOPT-NEXT:    imulq $26, %rdi, %rax
+; X64-NOOPT-NEXT:    retq
 ;
 ; X64-SLM-LABEL: test_mul_by_26:
 ; X64-SLM:       # %bb.0:
-; X64-SLM-NEXT:    imulq $26, %rdi, %rax # sched: [3:1.00]
-; X64-SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SLM-NOOPT-LABEL: test_mul_by_26:
-; SLM-NOOPT:       # %bb.0:
-; SLM-NOOPT-NEXT:    imulq $26, %rdi, %rax # sched: [3:1.00]
-; SLM-NOOPT-NEXT:    retq # sched: [4:1.00]
+; X64-SLM-NEXT:    imulq $26, %rdi, %rax
+; X64-SLM-NEXT:    retq
+; X86-OPT-LABEL: test_mul_by_26:
+; X86-OPT:       # %bb.0:
+; X86-OPT-NEXT:    leaq (%rdi,%rdi,4), %rax
+; X86-OPT-NEXT:    leaq (%rax,%rax,4), %rax
+; X86-OPT-NEXT:    addq %rdi, %rax
+; X86-OPT-NEXT:    retq
   %mul = mul nsw i64 %x, 26
   ret i64 %mul
 }
@@ -1422,17 +1083,11 @@ define i64 @test_mul_by_27(i64 %x) {
 ; X86-NEXT:    addl %ecx, %edx
 ; X86-NEXT:    retl
 ;
-; X64-HSW-LABEL: test_mul_by_27:
-; X64-HSW:       # %bb.0:
-; X64-HSW-NEXT:    leaq (%rdi,%rdi,8), %rax # sched: [1:0.50]
-; X64-HSW-NEXT:    leaq (%rax,%rax,2), %rax # sched: [1:0.50]
-; X64-HSW-NEXT:    retq # sched: [7:1.00]
-;
-; X64-JAG-LABEL: test_mul_by_27:
-; X64-JAG:       # %bb.0:
-; X64-JAG-NEXT:    leaq (%rdi,%rdi,8), %rax # sched: [2:1.00]
-; X64-JAG-NEXT:    leaq (%rax,%rax,2), %rax # sched: [2:1.00]
-; X64-JAG-NEXT:    retq # sched: [4:1.00]
+; X64-OPT-LABEL: test_mul_by_27:
+; X64-OPT:       # %bb.0:
+; X64-OPT-NEXT:    leaq (%rdi,%rdi,8), %rax
+; X64-OPT-NEXT:    leaq (%rax,%rax,2), %rax
+; X64-OPT-NEXT:    retq
 ;
 ; X86-NOOPT-LABEL: test_mul_by_27:
 ; X86-NOOPT:       # %bb.0:
@@ -1442,26 +1097,15 @@ define i64 @test_mul_by_27(i64 %x) {
 ; X86-NOOPT-NEXT:    addl %ecx, %edx
 ; X86-NOOPT-NEXT:    retl
 ;
-; HSW-NOOPT-LABEL: test_mul_by_27:
-; HSW-NOOPT:       # %bb.0:
-; HSW-NOOPT-NEXT:    imulq $27, %rdi, %rax # sched: [3:1.00]
-; HSW-NOOPT-NEXT:    retq # sched: [7:1.00]
-;
-; JAG-NOOPT-LABEL: test_mul_by_27:
-; JAG-NOOPT:       # %bb.0:
-; JAG-NOOPT-NEXT:    imulq $27, %rdi, %rax # sched: [6:4.00]
-; JAG-NOOPT-NEXT:    retq # sched: [4:1.00]
-;
-; X64-SLM-LABEL: test_mul_by_27:
-; X64-SLM:       # %bb.0:
-; X64-SLM-NEXT:    leaq (%rdi,%rdi,8), %rax # sched: [1:1.00]
-; X64-SLM-NEXT:    leaq (%rax,%rax,2), %rax # sched: [1:1.00]
-; X64-SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SLM-NOOPT-LABEL: test_mul_by_27:
-; SLM-NOOPT:       # %bb.0:
-; SLM-NOOPT-NEXT:    imulq $27, %rdi, %rax # sched: [3:1.00]
-; SLM-NOOPT-NEXT:    retq # sched: [4:1.00]
+; X64-NOOPT-LABEL: test_mul_by_27:
+; X64-NOOPT:       # %bb.0:
+; X64-NOOPT-NEXT:    imulq $27, %rdi, %rax
+; X64-NOOPT-NEXT:    retq
+; X86-OPT-LABEL: test_mul_by_27:
+; X86-OPT:       # %bb.0:
+; X86-OPT-NEXT:    leaq (%rdi,%rdi,8), %rax
+; X86-OPT-NEXT:    leaq (%rax,%rax,2), %rax
+; X86-OPT-NEXT:    retq
   %mul = mul nsw i64 %x, 27
   ret i64 %mul
 }
@@ -1480,17 +1124,17 @@ define i64 @test_mul_by_28(i64 %x) {
 ;
 ; X64-HSW-LABEL: test_mul_by_28:
 ; X64-HSW:       # %bb.0:
-; X64-HSW-NEXT:    leaq (%rdi,%rdi,8), %rax # sched: [1:0.50]
-; X64-HSW-NEXT:    leaq (%rax,%rax,2), %rax # sched: [1:0.50]
-; X64-HSW-NEXT:    addq %rdi, %rax # sched: [1:0.25]
-; X64-HSW-NEXT:    retq # sched: [7:1.00]
+; X64-HSW-NEXT:    leaq (%rdi,%rdi,8), %rax
+; X64-HSW-NEXT:    leaq (%rax,%rax,2), %rax
+; X64-HSW-NEXT:    addq %rdi, %rax
+; X64-HSW-NEXT:    retq
 ;
 ; X64-JAG-LABEL: test_mul_by_28:
 ; X64-JAG:       # %bb.0:
-; X64-JAG-NEXT:    leaq (%rdi,%rdi,8), %rax # sched: [2:1.00]
-; X64-JAG-NEXT:    leaq (%rax,%rax,2), %rax # sched: [2:1.00]
-; X64-JAG-NEXT:    addq %rdi, %rax # sched: [1:0.50]
-; X64-JAG-NEXT:    retq # sched: [4:1.00]
+; X64-JAG-NEXT:    leaq (%rdi,%rdi,8), %rax
+; X64-JAG-NEXT:    leaq (%rax,%rax,2), %rax
+; X64-JAG-NEXT:    addq %rdi, %rax
+; X64-JAG-NEXT:    retq
 ;
 ; X86-NOOPT-LABEL: test_mul_by_28:
 ; X86-NOOPT:       # %bb.0:
@@ -1500,25 +1144,21 @@ define i64 @test_mul_by_28(i64 %x) {
 ; X86-NOOPT-NEXT:    addl %ecx, %edx
 ; X86-NOOPT-NEXT:    retl
 ;
-; HSW-NOOPT-LABEL: test_mul_by_28:
-; HSW-NOOPT:       # %bb.0:
-; HSW-NOOPT-NEXT:    imulq $28, %rdi, %rax # sched: [3:1.00]
-; HSW-NOOPT-NEXT:    retq # sched: [7:1.00]
-;
-; JAG-NOOPT-LABEL: test_mul_by_28:
-; JAG-NOOPT:       # %bb.0:
-; JAG-NOOPT-NEXT:    imulq $28, %rdi, %rax # sched: [6:4.00]
-; JAG-NOOPT-NEXT:    retq # sched: [4:1.00]
+; X64-NOOPT-LABEL: test_mul_by_28:
+; X64-NOOPT:       # %bb.0:
+; X64-NOOPT-NEXT:    imulq $28, %rdi, %rax
+; X64-NOOPT-NEXT:    retq
 ;
 ; X64-SLM-LABEL: test_mul_by_28:
 ; X64-SLM:       # %bb.0:
-; X64-SLM-NEXT:    imulq $28, %rdi, %rax # sched: [3:1.00]
-; X64-SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SLM-NOOPT-LABEL: test_mul_by_28:
-; SLM-NOOPT:       # %bb.0:
-; SLM-NOOPT-NEXT:    imulq $28, %rdi, %rax # sched: [3:1.00]
-; SLM-NOOPT-NEXT:    retq # sched: [4:1.00]
+; X64-SLM-NEXT:    imulq $28, %rdi, %rax
+; X64-SLM-NEXT:    retq
+; X86-OPT-LABEL: test_mul_by_28:
+; X86-OPT:       # %bb.0:
+; X86-OPT-NEXT:    leaq (%rdi,%rdi,8), %rax
+; X86-OPT-NEXT:    leaq (%rax,%rax,2), %rax
+; X86-OPT-NEXT:    addq %rdi, %rax
+; X86-OPT-NEXT:    retq
   %mul = mul nsw i64 %x, 28
   ret i64 %mul
 }
@@ -1538,19 +1178,19 @@ define i64 @test_mul_by_29(i64 %x) {
 ;
 ; X64-HSW-LABEL: test_mul_by_29:
 ; X64-HSW:       # %bb.0:
-; X64-HSW-NEXT:    leaq (%rdi,%rdi,8), %rax # sched: [1:0.50]
-; X64-HSW-NEXT:    leaq (%rax,%rax,2), %rax # sched: [1:0.50]
-; X64-HSW-NEXT:    addq %rdi, %rax # sched: [1:0.25]
-; X64-HSW-NEXT:    addq %rdi, %rax # sched: [1:0.25]
-; X64-HSW-NEXT:    retq # sched: [7:1.00]
+; X64-HSW-NEXT:    leaq (%rdi,%rdi,8), %rax
+; X64-HSW-NEXT:    leaq (%rax,%rax,2), %rax
+; X64-HSW-NEXT:    addq %rdi, %rax
+; X64-HSW-NEXT:    addq %rdi, %rax
+; X64-HSW-NEXT:    retq
 ;
 ; X64-JAG-LABEL: test_mul_by_29:
 ; X64-JAG:       # %bb.0:
-; X64-JAG-NEXT:    leaq (%rdi,%rdi,8), %rax # sched: [2:1.00]
-; X64-JAG-NEXT:    leaq (%rax,%rax,2), %rax # sched: [2:1.00]
-; X64-JAG-NEXT:    addq %rdi, %rax # sched: [1:0.50]
-; X64-JAG-NEXT:    addq %rdi, %rax # sched: [1:0.50]
-; X64-JAG-NEXT:    retq # sched: [4:1.00]
+; X64-JAG-NEXT:    leaq (%rdi,%rdi,8), %rax
+; X64-JAG-NEXT:    leaq (%rax,%rax,2), %rax
+; X64-JAG-NEXT:    addq %rdi, %rax
+; X64-JAG-NEXT:    addq %rdi, %rax
+; X64-JAG-NEXT:    retq
 ;
 ; X86-NOOPT-LABEL: test_mul_by_29:
 ; X86-NOOPT:       # %bb.0:
@@ -1560,25 +1200,22 @@ define i64 @test_mul_by_29(i64 %x) {
 ; X86-NOOPT-NEXT:    addl %ecx, %edx
 ; X86-NOOPT-NEXT:    retl
 ;
-; HSW-NOOPT-LABEL: test_mul_by_29:
-; HSW-NOOPT:       # %bb.0:
-; HSW-NOOPT-NEXT:    imulq $29, %rdi, %rax # sched: [3:1.00]
-; HSW-NOOPT-NEXT:    retq # sched: [7:1.00]
-;
-; JAG-NOOPT-LABEL: test_mul_by_29:
-; JAG-NOOPT:       # %bb.0:
-; JAG-NOOPT-NEXT:    imulq $29, %rdi, %rax # sched: [6:4.00]
-; JAG-NOOPT-NEXT:    retq # sched: [4:1.00]
+; X64-NOOPT-LABEL: test_mul_by_29:
+; X64-NOOPT:       # %bb.0:
+; X64-NOOPT-NEXT:    imulq $29, %rdi, %rax
+; X64-NOOPT-NEXT:    retq
 ;
 ; X64-SLM-LABEL: test_mul_by_29:
 ; X64-SLM:       # %bb.0:
-; X64-SLM-NEXT:    imulq $29, %rdi, %rax # sched: [3:1.00]
-; X64-SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SLM-NOOPT-LABEL: test_mul_by_29:
-; SLM-NOOPT:       # %bb.0:
-; SLM-NOOPT-NEXT:    imulq $29, %rdi, %rax # sched: [3:1.00]
-; SLM-NOOPT-NEXT:    retq # sched: [4:1.00]
+; X64-SLM-NEXT:    imulq $29, %rdi, %rax
+; X64-SLM-NEXT:    retq
+; X86-OPT-LABEL: test_mul_by_29:
+; X86-OPT:       # %bb.0:
+; X86-OPT-NEXT:    leaq (%rdi,%rdi,8), %rax
+; X86-OPT-NEXT:    leaq (%rax,%rax,2), %rax
+; X86-OPT-NEXT:    addq %rdi, %rax
+; X86-OPT-NEXT:    addq %rdi, %rax
+; X86-OPT-NEXT:    retq
   %mul = mul nsw i64 %x, 29
   ret i64 %mul
 }
@@ -1596,21 +1233,13 @@ define i64 @test_mul_by_30(i64 %x) {
 ; X86-NEXT:    addl %ecx, %edx
 ; X86-NEXT:    retl
 ;
-; X64-HSW-LABEL: test_mul_by_30:
-; X64-HSW:       # %bb.0:
-; X64-HSW-NEXT:    movq %rdi, %rax # sched: [1:0.25]
-; X64-HSW-NEXT:    shlq $5, %rax # sched: [1:0.50]
-; X64-HSW-NEXT:    subq %rdi, %rax # sched: [1:0.25]
-; X64-HSW-NEXT:    subq %rdi, %rax # sched: [1:0.25]
-; X64-HSW-NEXT:    retq # sched: [7:1.00]
-;
-; X64-JAG-LABEL: test_mul_by_30:
-; X64-JAG:       # %bb.0:
-; X64-JAG-NEXT:    movq %rdi, %rax # sched: [1:0.50]
-; X64-JAG-NEXT:    shlq $5, %rax # sched: [1:0.50]
-; X64-JAG-NEXT:    subq %rdi, %rax # sched: [1:0.50]
-; X64-JAG-NEXT:    subq %rdi, %rax # sched: [1:0.50]
-; X64-JAG-NEXT:    retq # sched: [4:1.00]
+; X64-OPT-LABEL: test_mul_by_30:
+; X64-OPT:       # %bb.0:
+; X64-OPT-NEXT:    movq %rdi, %rax
+; X64-OPT-NEXT:    shlq $5, %rax
+; X64-OPT-NEXT:    subq %rdi, %rax
+; X64-OPT-NEXT:    subq %rdi, %rax
+; X64-OPT-NEXT:    retq
 ;
 ; X86-NOOPT-LABEL: test_mul_by_30:
 ; X86-NOOPT:       # %bb.0:
@@ -1620,28 +1249,17 @@ define i64 @test_mul_by_30(i64 %x) {
 ; X86-NOOPT-NEXT:    addl %ecx, %edx
 ; X86-NOOPT-NEXT:    retl
 ;
-; HSW-NOOPT-LABEL: test_mul_by_30:
-; HSW-NOOPT:       # %bb.0:
-; HSW-NOOPT-NEXT:    imulq $30, %rdi, %rax # sched: [3:1.00]
-; HSW-NOOPT-NEXT:    retq # sched: [7:1.00]
-;
-; JAG-NOOPT-LABEL: test_mul_by_30:
-; JAG-NOOPT:       # %bb.0:
-; JAG-NOOPT-NEXT:    imulq $30, %rdi, %rax # sched: [6:4.00]
-; JAG-NOOPT-NEXT:    retq # sched: [4:1.00]
-;
-; X64-SLM-LABEL: test_mul_by_30:
-; X64-SLM:       # %bb.0:
-; X64-SLM-NEXT:    movq %rdi, %rax # sched: [1:0.50]
-; X64-SLM-NEXT:    shlq $5, %rax # sched: [1:1.00]
-; X64-SLM-NEXT:    subq %rdi, %rax # sched: [1:0.50]
-; X64-SLM-NEXT:    subq %rdi, %rax # sched: [1:0.50]
-; X64-SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SLM-NOOPT-LABEL: test_mul_by_30:
-; SLM-NOOPT:       # %bb.0:
-; SLM-NOOPT-NEXT:    imulq $30, %rdi, %rax # sched: [3:1.00]
-; SLM-NOOPT-NEXT:    retq # sched: [4:1.00]
+; X64-NOOPT-LABEL: test_mul_by_30:
+; X64-NOOPT:       # %bb.0:
+; X64-NOOPT-NEXT:    imulq $30, %rdi, %rax
+; X64-NOOPT-NEXT:    retq
+; X86-OPT-LABEL: test_mul_by_30:
+; X86-OPT:       # %bb.0:
+; X86-OPT-NEXT:    movq %rdi, %rax
+; X86-OPT-NEXT:    shlq $5, %rax
+; X86-OPT-NEXT:    subq %rdi, %rax
+; X86-OPT-NEXT:    subq %rdi, %rax
+; X86-OPT-NEXT:    retq
   %mul = mul nsw i64 %x, 30
   ret i64 %mul
 }
@@ -1658,19 +1276,12 @@ define i64 @test_mul_by_31(i64 %x) {
 ; X86-NEXT:    addl %ecx, %edx
 ; X86-NEXT:    retl
 ;
-; X64-HSW-LABEL: test_mul_by_31:
-; X64-HSW:       # %bb.0:
-; X64-HSW-NEXT:    movq %rdi, %rax # sched: [1:0.25]
-; X64-HSW-NEXT:    shlq $5, %rax # sched: [1:0.50]
-; X64-HSW-NEXT:    subq %rdi, %rax # sched: [1:0.25]
-; X64-HSW-NEXT:    retq # sched: [7:1.00]
-;
-; X64-JAG-LABEL: test_mul_by_31:
-; X64-JAG:       # %bb.0:
-; X64-JAG-NEXT:    movq %rdi, %rax # sched: [1:0.50]
-; X64-JAG-NEXT:    shlq $5, %rax # sched: [1:0.50]
-; X64-JAG-NEXT:    subq %rdi, %rax # sched: [1:0.50]
-; X64-JAG-NEXT:    retq # sched: [4:1.00]
+; X64-OPT-LABEL: test_mul_by_31:
+; X64-OPT:       # %bb.0:
+; X64-OPT-NEXT:    movq %rdi, %rax
+; X64-OPT-NEXT:    shlq $5, %rax
+; X64-OPT-NEXT:    subq %rdi, %rax
+; X64-OPT-NEXT:    retq
 ;
 ; X86-NOOPT-LABEL: test_mul_by_31:
 ; X86-NOOPT:       # %bb.0:
@@ -1680,27 +1291,16 @@ define i64 @test_mul_by_31(i64 %x) {
 ; X86-NOOPT-NEXT:    addl %ecx, %edx
 ; X86-NOOPT-NEXT:    retl
 ;
-; HSW-NOOPT-LABEL: test_mul_by_31:
-; HSW-NOOPT:       # %bb.0:
-; HSW-NOOPT-NEXT:    imulq $31, %rdi, %rax # sched: [3:1.00]
-; HSW-NOOPT-NEXT:    retq # sched: [7:1.00]
-;
-; JAG-NOOPT-LABEL: test_mul_by_31:
-; JAG-NOOPT:       # %bb.0:
-; JAG-NOOPT-NEXT:    imulq $31, %rdi, %rax # sched: [6:4.00]
-; JAG-NOOPT-NEXT:    retq # sched: [4:1.00]
-;
-; X64-SLM-LABEL: test_mul_by_31:
-; X64-SLM:       # %bb.0:
-; X64-SLM-NEXT:    movq %rdi, %rax # sched: [1:0.50]
-; X64-SLM-NEXT:    shlq $5, %rax # sched: [1:1.00]
-; X64-SLM-NEXT:    subq %rdi, %rax # sched: [1:0.50]
-; X64-SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SLM-NOOPT-LABEL: test_mul_by_31:
-; SLM-NOOPT:       # %bb.0:
-; SLM-NOOPT-NEXT:    imulq $31, %rdi, %rax # sched: [3:1.00]
-; SLM-NOOPT-NEXT:    retq # sched: [4:1.00]
+; X64-NOOPT-LABEL: test_mul_by_31:
+; X64-NOOPT:       # %bb.0:
+; X64-NOOPT-NEXT:    imulq $31, %rdi, %rax
+; X64-NOOPT-NEXT:    retq
+; X86-OPT-LABEL: test_mul_by_31:
+; X86-OPT:       # %bb.0:
+; X86-OPT-NEXT:    movq %rdi, %rax
+; X86-OPT-NEXT:    shlq $5, %rax
+; X86-OPT-NEXT:    subq %rdi, %rax
+; X86-OPT-NEXT:    retq
   %mul = mul nsw i64 %x, 31
   ret i64 %mul
 }
@@ -1714,17 +1314,11 @@ define i64 @test_mul_by_32(i64 %x) {
 ; X86-NEXT:    shll $5, %eax
 ; X86-NEXT:    retl
 ;
-; X64-HSW-LABEL: test_mul_by_32:
-; X64-HSW:       # %bb.0:
-; X64-HSW-NEXT:    movq %rdi, %rax # sched: [1:0.25]
-; X64-HSW-NEXT:    shlq $5, %rax # sched: [1:0.50]
-; X64-HSW-NEXT:    retq # sched: [7:1.00]
-;
-; X64-JAG-LABEL: test_mul_by_32:
-; X64-JAG:       # %bb.0:
-; X64-JAG-NEXT:    movq %rdi, %rax # sched: [1:0.50]
-; X64-JAG-NEXT:    shlq $5, %rax # sched: [1:0.50]
-; X64-JAG-NEXT:    retq # sched: [4:1.00]
+; X64-OPT-LABEL: test_mul_by_32:
+; X64-OPT:       # %bb.0:
+; X64-OPT-NEXT:    movq %rdi, %rax
+; X64-OPT-NEXT:    shlq $5, %rax
+; X64-OPT-NEXT:    retq
 ;
 ; X86-NOOPT-LABEL: test_mul_by_32:
 ; X86-NOOPT:       # %bb.0:
@@ -1734,29 +1328,16 @@ define i64 @test_mul_by_32(i64 %x) {
 ; X86-NOOPT-NEXT:    shll $5, %eax
 ; X86-NOOPT-NEXT:    retl
 ;
-; HSW-NOOPT-LABEL: test_mul_by_32:
-; HSW-NOOPT:       # %bb.0:
-; HSW-NOOPT-NEXT:    movq %rdi, %rax # sched: [1:0.25]
-; HSW-NOOPT-NEXT:    shlq $5, %rax # sched: [1:0.50]
-; HSW-NOOPT-NEXT:    retq # sched: [7:1.00]
-;
-; JAG-NOOPT-LABEL: test_mul_by_32:
-; JAG-NOOPT:       # %bb.0:
-; JAG-NOOPT-NEXT:    movq %rdi, %rax # sched: [1:0.50]
-; JAG-NOOPT-NEXT:    shlq $5, %rax # sched: [1:0.50]
-; JAG-NOOPT-NEXT:    retq # sched: [4:1.00]
-;
-; X64-SLM-LABEL: test_mul_by_32:
-; X64-SLM:       # %bb.0:
-; X64-SLM-NEXT:    movq %rdi, %rax # sched: [1:0.50]
-; X64-SLM-NEXT:    shlq $5, %rax # sched: [1:1.00]
-; X64-SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SLM-NOOPT-LABEL: test_mul_by_32:
-; SLM-NOOPT:       # %bb.0:
-; SLM-NOOPT-NEXT:    movq %rdi, %rax # sched: [1:0.50]
-; SLM-NOOPT-NEXT:    shlq $5, %rax # sched: [1:1.00]
-; SLM-NOOPT-NEXT:    retq # sched: [4:1.00]
+; X64-NOOPT-LABEL: test_mul_by_32:
+; X64-NOOPT:       # %bb.0:
+; X64-NOOPT-NEXT:    movq %rdi, %rax
+; X64-NOOPT-NEXT:    shlq $5, %rax
+; X64-NOOPT-NEXT:    retq
+; X86-OPT-LABEL: test_mul_by_32:
+; X86-OPT:       # %bb.0:
+; X86-OPT-NEXT:    movq %rdi, %rax
+; X86-OPT-NEXT:    shlq $5, %rax
+; X86-OPT-NEXT:    retq
   %mul = mul nsw i64 %x, 32
   ret i64 %mul
 }
@@ -1774,15 +1355,15 @@ define i64 @test_mul_by_37(i64 %x) {
 ;
 ; X64-HSW-LABEL: test_mul_by_37:
 ; X64-HSW:       # %bb.0:
-; X64-HSW-NEXT:    leaq (%rdi,%rdi,8), %rax # sched: [1:0.50]
-; X64-HSW-NEXT:    leaq (%rdi,%rax,4), %rax # sched: [1:0.50]
-; X64-HSW-NEXT:    retq # sched: [7:1.00]
+; X64-HSW-NEXT:    leaq (%rdi,%rdi,8), %rax
+; X64-HSW-NEXT:    leaq (%rdi,%rax,4), %rax
+; X64-HSW-NEXT:    retq
 ;
 ; X64-JAG-LABEL: test_mul_by_37:
 ; X64-JAG:       # %bb.0:
-; X64-JAG-NEXT:    leaq (%rdi,%rdi,8), %rax # sched: [2:1.00]
-; X64-JAG-NEXT:    leaq (%rdi,%rax,4), %rax # sched: [2:1.00]
-; X64-JAG-NEXT:    retq # sched: [4:1.00]
+; X64-JAG-NEXT:    leaq (%rdi,%rdi,8), %rax
+; X64-JAG-NEXT:    leaq (%rdi,%rax,4), %rax
+; X64-JAG-NEXT:    retq
 ;
 ; X86-NOOPT-LABEL: test_mul_by_37:
 ; X86-NOOPT:       # %bb.0:
@@ -1792,25 +1373,20 @@ define i64 @test_mul_by_37(i64 %x) {
 ; X86-NOOPT-NEXT:    addl %ecx, %edx
 ; X86-NOOPT-NEXT:    retl
 ;
-; HSW-NOOPT-LABEL: test_mul_by_37:
-; HSW-NOOPT:       # %bb.0:
-; HSW-NOOPT-NEXT:    imulq $37, %rdi, %rax # sched: [3:1.00]
-; HSW-NOOPT-NEXT:    retq # sched: [7:1.00]
-;
-; JAG-NOOPT-LABEL: test_mul_by_37:
-; JAG-NOOPT:       # %bb.0:
-; JAG-NOOPT-NEXT:    imulq $37, %rdi, %rax # sched: [6:4.00]
-; JAG-NOOPT-NEXT:    retq # sched: [4:1.00]
+; X64-NOOPT-LABEL: test_mul_by_37:
+; X64-NOOPT:       # %bb.0:
+; X64-NOOPT-NEXT:    imulq $37, %rdi, %rax
+; X64-NOOPT-NEXT:    retq
 ;
 ; X64-SLM-LABEL: test_mul_by_37:
 ; X64-SLM:       # %bb.0:
-; X64-SLM-NEXT:    imulq $37, %rdi, %rax # sched: [3:1.00]
-; X64-SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SLM-NOOPT-LABEL: test_mul_by_37:
-; SLM-NOOPT:       # %bb.0:
-; SLM-NOOPT-NEXT:    imulq $37, %rdi, %rax # sched: [3:1.00]
-; SLM-NOOPT-NEXT:    retq # sched: [4:1.00]
+; X64-SLM-NEXT:    imulq $37, %rdi, %rax
+; X64-SLM-NEXT:    retq
+; X86-OPT-LABEL: test_mul_by_37:
+; X86-OPT:       # %bb.0:
+; X86-OPT-NEXT:    leaq (%rdi,%rdi,8), %rax
+; X86-OPT-NEXT:    leaq (%rdi,%rax,4), %rax
+; X86-OPT-NEXT:    retq
   %mul = mul nsw i64 %x, 37
   ret i64 %mul
 }
@@ -1828,15 +1404,15 @@ define i64 @test_mul_by_41(i64 %x) {
 ;
 ; X64-HSW-LABEL: test_mul_by_41:
 ; X64-HSW:       # %bb.0:
-; X64-HSW-NEXT:    leaq (%rdi,%rdi,4), %rax # sched: [1:0.50]
-; X64-HSW-NEXT:    leaq (%rdi,%rax,8), %rax # sched: [1:0.50]
-; X64-HSW-NEXT:    retq # sched: [7:1.00]
+; X64-HSW-NEXT:    leaq (%rdi,%rdi,4), %rax
+; X64-HSW-NEXT:    leaq (%rdi,%rax,8), %rax
+; X64-HSW-NEXT:    retq
 ;
 ; X64-JAG-LABEL: test_mul_by_41:
 ; X64-JAG:       # %bb.0:
-; X64-JAG-NEXT:    leaq (%rdi,%rdi,4), %rax # sched: [2:1.00]
-; X64-JAG-NEXT:    leaq (%rdi,%rax,8), %rax # sched: [2:1.00]
-; X64-JAG-NEXT:    retq # sched: [4:1.00]
+; X64-JAG-NEXT:    leaq (%rdi,%rdi,4), %rax
+; X64-JAG-NEXT:    leaq (%rdi,%rax,8), %rax
+; X64-JAG-NEXT:    retq
 ;
 ; X86-NOOPT-LABEL: test_mul_by_41:
 ; X86-NOOPT:       # %bb.0:
@@ -1846,25 +1422,20 @@ define i64 @test_mul_by_41(i64 %x) {
 ; X86-NOOPT-NEXT:    addl %ecx, %edx
 ; X86-NOOPT-NEXT:    retl
 ;
-; HSW-NOOPT-LABEL: test_mul_by_41:
-; HSW-NOOPT:       # %bb.0:
-; HSW-NOOPT-NEXT:    imulq $41, %rdi, %rax # sched: [3:1.00]
-; HSW-NOOPT-NEXT:    retq # sched: [7:1.00]
-;
-; JAG-NOOPT-LABEL: test_mul_by_41:
-; JAG-NOOPT:       # %bb.0:
-; JAG-NOOPT-NEXT:    imulq $41, %rdi, %rax # sched: [6:4.00]
-; JAG-NOOPT-NEXT:    retq # sched: [4:1.00]
+; X64-NOOPT-LABEL: test_mul_by_41:
+; X64-NOOPT:       # %bb.0:
+; X64-NOOPT-NEXT:    imulq $41, %rdi, %rax
+; X64-NOOPT-NEXT:    retq
 ;
 ; X64-SLM-LABEL: test_mul_by_41:
 ; X64-SLM:       # %bb.0:
-; X64-SLM-NEXT:    imulq $41, %rdi, %rax # sched: [3:1.00]
-; X64-SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SLM-NOOPT-LABEL: test_mul_by_41:
-; SLM-NOOPT:       # %bb.0:
-; SLM-NOOPT-NEXT:    imulq $41, %rdi, %rax # sched: [3:1.00]
-; SLM-NOOPT-NEXT:    retq # sched: [4:1.00]
+; X64-SLM-NEXT:    imulq $41, %rdi, %rax
+; X64-SLM-NEXT:    retq
+; X86-OPT-LABEL: test_mul_by_41:
+; X86-OPT:       # %bb.0:
+; X86-OPT-NEXT:    leaq (%rdi,%rdi,4), %rax
+; X86-OPT-NEXT:    leaq (%rdi,%rax,8), %rax
+; X86-OPT-NEXT:    retq
   %mul = mul nsw i64 %x, 41
   ret i64 %mul
 }
@@ -1882,21 +1453,13 @@ define i64 @test_mul_by_62(i64 %x) {
 ; X86-NEXT:    addl %ecx, %edx
 ; X86-NEXT:    retl
 ;
-; X64-HSW-LABEL: test_mul_by_62:
-; X64-HSW:       # %bb.0:
-; X64-HSW-NEXT:    movq %rdi, %rax # sched: [1:0.25]
-; X64-HSW-NEXT:    shlq $6, %rax # sched: [1:0.50]
-; X64-HSW-NEXT:    subq %rdi, %rax # sched: [1:0.25]
-; X64-HSW-NEXT:    subq %rdi, %rax # sched: [1:0.25]
-; X64-HSW-NEXT:    retq # sched: [7:1.00]
-;
-; X64-JAG-LABEL: test_mul_by_62:
-; X64-JAG:       # %bb.0:
-; X64-JAG-NEXT:    movq %rdi, %rax # sched: [1:0.50]
-; X64-JAG-NEXT:    shlq $6, %rax # sched: [1:0.50]
-; X64-JAG-NEXT:    subq %rdi, %rax # sched: [1:0.50]
-; X64-JAG-NEXT:    subq %rdi, %rax # sched: [1:0.50]
-; X64-JAG-NEXT:    retq # sched: [4:1.00]
+; X64-OPT-LABEL: test_mul_by_62:
+; X64-OPT:       # %bb.0:
+; X64-OPT-NEXT:    movq %rdi, %rax
+; X64-OPT-NEXT:    shlq $6, %rax
+; X64-OPT-NEXT:    subq %rdi, %rax
+; X64-OPT-NEXT:    subq %rdi, %rax
+; X64-OPT-NEXT:    retq
 ;
 ; X86-NOOPT-LABEL: test_mul_by_62:
 ; X86-NOOPT:       # %bb.0:
@@ -1906,28 +1469,17 @@ define i64 @test_mul_by_62(i64 %x) {
 ; X86-NOOPT-NEXT:    addl %ecx, %edx
 ; X86-NOOPT-NEXT:    retl
 ;
-; HSW-NOOPT-LABEL: test_mul_by_62:
-; HSW-NOOPT:       # %bb.0:
-; HSW-NOOPT-NEXT:    imulq $62, %rdi, %rax # sched: [3:1.00]
-; HSW-NOOPT-NEXT:    retq # sched: [7:1.00]
-;
-; JAG-NOOPT-LABEL: test_mul_by_62:
-; JAG-NOOPT:       # %bb.0:
-; JAG-NOOPT-NEXT:    imulq $62, %rdi, %rax # sched: [6:4.00]
-; JAG-NOOPT-NEXT:    retq # sched: [4:1.00]
-;
-; X64-SLM-LABEL: test_mul_by_62:
-; X64-SLM:       # %bb.0:
-; X64-SLM-NEXT:    movq %rdi, %rax # sched: [1:0.50]
-; X64-SLM-NEXT:    shlq $6, %rax # sched: [1:1.00]
-; X64-SLM-NEXT:    subq %rdi, %rax # sched: [1:0.50]
-; X64-SLM-NEXT:    subq %rdi, %rax # sched: [1:0.50]
-; X64-SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SLM-NOOPT-LABEL: test_mul_by_62:
-; SLM-NOOPT:       # %bb.0:
-; SLM-NOOPT-NEXT:    imulq $62, %rdi, %rax # sched: [3:1.00]
-; SLM-NOOPT-NEXT:    retq # sched: [4:1.00]
+; X64-NOOPT-LABEL: test_mul_by_62:
+; X64-NOOPT:       # %bb.0:
+; X64-NOOPT-NEXT:    imulq $62, %rdi, %rax
+; X64-NOOPT-NEXT:    retq
+; X86-OPT-LABEL: test_mul_by_62:
+; X86-OPT:       # %bb.0:
+; X86-OPT-NEXT:    movq %rdi, %rax
+; X86-OPT-NEXT:    shlq $6, %rax
+; X86-OPT-NEXT:    subq %rdi, %rax
+; X86-OPT-NEXT:    subq %rdi, %rax
+; X86-OPT-NEXT:    retq
   %mul = mul nsw i64 %x, 62
   ret i64 %mul
 }
@@ -1946,17 +1498,17 @@ define i64 @test_mul_by_66(i64 %x) {
 ;
 ; X64-HSW-LABEL: test_mul_by_66:
 ; X64-HSW:       # %bb.0:
-; X64-HSW-NEXT:    movq %rdi, %rax # sched: [1:0.25]
-; X64-HSW-NEXT:    shlq $6, %rax # sched: [1:0.50]
-; X64-HSW-NEXT:    leaq (%rax,%rdi,2), %rax # sched: [1:0.50]
-; X64-HSW-NEXT:    retq # sched: [7:1.00]
+; X64-HSW-NEXT:    movq %rdi, %rax
+; X64-HSW-NEXT:    shlq $6, %rax
+; X64-HSW-NEXT:    leaq (%rax,%rdi,2), %rax
+; X64-HSW-NEXT:    retq
 ;
 ; X64-JAG-LABEL: test_mul_by_66:
 ; X64-JAG:       # %bb.0:
-; X64-JAG-NEXT:    movq %rdi, %rax # sched: [1:0.50]
-; X64-JAG-NEXT:    shlq $6, %rax # sched: [1:0.50]
-; X64-JAG-NEXT:    leaq (%rax,%rdi,2), %rax # sched: [2:1.00]
-; X64-JAG-NEXT:    retq # sched: [4:1.00]
+; X64-JAG-NEXT:    movq %rdi, %rax
+; X64-JAG-NEXT:    shlq $6, %rax
+; X64-JAG-NEXT:    leaq (%rax,%rdi,2), %rax
+; X64-JAG-NEXT:    retq
 ;
 ; X86-NOOPT-LABEL: test_mul_by_66:
 ; X86-NOOPT:       # %bb.0:
@@ -1966,28 +1518,24 @@ define i64 @test_mul_by_66(i64 %x) {
 ; X86-NOOPT-NEXT:    addl %ecx, %edx
 ; X86-NOOPT-NEXT:    retl
 ;
-; HSW-NOOPT-LABEL: test_mul_by_66:
-; HSW-NOOPT:       # %bb.0:
-; HSW-NOOPT-NEXT:    imulq $66, %rdi, %rax # sched: [3:1.00]
-; HSW-NOOPT-NEXT:    retq # sched: [7:1.00]
-;
-; JAG-NOOPT-LABEL: test_mul_by_66:
-; JAG-NOOPT:       # %bb.0:
-; JAG-NOOPT-NEXT:    imulq $66, %rdi, %rax # sched: [6:4.00]
-; JAG-NOOPT-NEXT:    retq # sched: [4:1.00]
+; X64-NOOPT-LABEL: test_mul_by_66:
+; X64-NOOPT:       # %bb.0:
+; X64-NOOPT-NEXT:    imulq $66, %rdi, %rax
+; X64-NOOPT-NEXT:    retq
 ;
 ; X64-SLM-LABEL: test_mul_by_66:
 ; X64-SLM:       # %bb.0:
-; X64-SLM-NEXT:    movq %rdi, %rax # sched: [1:0.50]
-; X64-SLM-NEXT:    shlq $6, %rax # sched: [1:1.00]
-; X64-SLM-NEXT:    addq %rdi, %rax # sched: [1:0.50]
-; X64-SLM-NEXT:    addq %rdi, %rax # sched: [1:0.50]
-; X64-SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SLM-NOOPT-LABEL: test_mul_by_66:
-; SLM-NOOPT:       # %bb.0:
-; SLM-NOOPT-NEXT:    imulq $66, %rdi, %rax # sched: [3:1.00]
-; SLM-NOOPT-NEXT:    retq # sched: [4:1.00]
+; X64-SLM-NEXT:    movq %rdi, %rax
+; X64-SLM-NEXT:    shlq $6, %rax
+; X64-SLM-NEXT:    addq %rdi, %rax
+; X64-SLM-NEXT:    addq %rdi, %rax
+; X64-SLM-NEXT:    retq
+; X86-OPT-LABEL: test_mul_by_66:
+; X86-OPT:       # %bb.0:
+; X86-OPT-NEXT:    movq %rdi, %rax
+; X86-OPT-NEXT:    shlq $6, %rax
+; X86-OPT-NEXT:    leaq (%rax,%rdi,2), %rax
+; X86-OPT-NEXT:    retq
   %mul = mul nsw i64 %x, 66
   ret i64 %mul
 }
@@ -2005,15 +1553,15 @@ define i64 @test_mul_by_73(i64 %x) {
 ;
 ; X64-HSW-LABEL: test_mul_by_73:
 ; X64-HSW:       # %bb.0:
-; X64-HSW-NEXT:    leaq (%rdi,%rdi,8), %rax # sched: [1:0.50]
-; X64-HSW-NEXT:    leaq (%rdi,%rax,8), %rax # sched: [1:0.50]
-; X64-HSW-NEXT:    retq # sched: [7:1.00]
+; X64-HSW-NEXT:    leaq (%rdi,%rdi,8), %rax
+; X64-HSW-NEXT:    leaq (%rdi,%rax,8), %rax
+; X64-HSW-NEXT:    retq
 ;
 ; X64-JAG-LABEL: test_mul_by_73:
 ; X64-JAG:       # %bb.0:
-; X64-JAG-NEXT:    leaq (%rdi,%rdi,8), %rax # sched: [2:1.00]
-; X64-JAG-NEXT:    leaq (%rdi,%rax,8), %rax # sched: [2:1.00]
-; X64-JAG-NEXT:    retq # sched: [4:1.00]
+; X64-JAG-NEXT:    leaq (%rdi,%rdi,8), %rax
+; X64-JAG-NEXT:    leaq (%rdi,%rax,8), %rax
+; X64-JAG-NEXT:    retq
 ;
 ; X86-NOOPT-LABEL: test_mul_by_73:
 ; X86-NOOPT:       # %bb.0:
@@ -2023,25 +1571,20 @@ define i64 @test_mul_by_73(i64 %x) {
 ; X86-NOOPT-NEXT:    addl %ecx, %edx
 ; X86-NOOPT-NEXT:    retl
 ;
-; HSW-NOOPT-LABEL: test_mul_by_73:
-; HSW-NOOPT:       # %bb.0:
-; HSW-NOOPT-NEXT:    imulq $73, %rdi, %rax # sched: [3:1.00]
-; HSW-NOOPT-NEXT:    retq # sched: [7:1.00]
-;
-; JAG-NOOPT-LABEL: test_mul_by_73:
-; JAG-NOOPT:       # %bb.0:
-; JAG-NOOPT-NEXT:    imulq $73, %rdi, %rax # sched: [6:4.00]
-; JAG-NOOPT-NEXT:    retq # sched: [4:1.00]
+; X64-NOOPT-LABEL: test_mul_by_73:
+; X64-NOOPT:       # %bb.0:
+; X64-NOOPT-NEXT:    imulq $73, %rdi, %rax
+; X64-NOOPT-NEXT:    retq
 ;
 ; X64-SLM-LABEL: test_mul_by_73:
 ; X64-SLM:       # %bb.0:
-; X64-SLM-NEXT:    imulq $73, %rdi, %rax # sched: [3:1.00]
-; X64-SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SLM-NOOPT-LABEL: test_mul_by_73:
-; SLM-NOOPT:       # %bb.0:
-; SLM-NOOPT-NEXT:    imulq $73, %rdi, %rax # sched: [3:1.00]
-; SLM-NOOPT-NEXT:    retq # sched: [4:1.00]
+; X64-SLM-NEXT:    imulq $73, %rdi, %rax
+; X64-SLM-NEXT:    retq
+; X86-OPT-LABEL: test_mul_by_73:
+; X86-OPT:       # %bb.0:
+; X86-OPT-NEXT:    leaq (%rdi,%rdi,8), %rax
+; X86-OPT-NEXT:    leaq (%rdi,%rax,8), %rax
+; X86-OPT-NEXT:    retq
   %mul = mul nsw i64 %x, 73
   ret i64 %mul
 }
@@ -2060,17 +1603,17 @@ define i64 @test_mul_by_520(i64 %x) {
 ;
 ; X64-HSW-LABEL: test_mul_by_520:
 ; X64-HSW:       # %bb.0:
-; X64-HSW-NEXT:    movq %rdi, %rax # sched: [1:0.25]
-; X64-HSW-NEXT:    shlq $9, %rax # sched: [1:0.50]
-; X64-HSW-NEXT:    leaq (%rax,%rdi,8), %rax # sched: [1:0.50]
-; X64-HSW-NEXT:    retq # sched: [7:1.00]
+; X64-HSW-NEXT:    movq %rdi, %rax
+; X64-HSW-NEXT:    shlq $9, %rax
+; X64-HSW-NEXT:    leaq (%rax,%rdi,8), %rax
+; X64-HSW-NEXT:    retq
 ;
 ; X64-JAG-LABEL: test_mul_by_520:
 ; X64-JAG:       # %bb.0:
-; X64-JAG-NEXT:    movq %rdi, %rax # sched: [1:0.50]
-; X64-JAG-NEXT:    shlq $9, %rax # sched: [1:0.50]
-; X64-JAG-NEXT:    leaq (%rax,%rdi,8), %rax # sched: [2:1.00]
-; X64-JAG-NEXT:    retq # sched: [4:1.00]
+; X64-JAG-NEXT:    movq %rdi, %rax
+; X64-JAG-NEXT:    shlq $9, %rax
+; X64-JAG-NEXT:    leaq (%rax,%rdi,8), %rax
+; X64-JAG-NEXT:    retq
 ;
 ; X86-NOOPT-LABEL: test_mul_by_520:
 ; X86-NOOPT:       # %bb.0:
@@ -2080,29 +1623,21 @@ define i64 @test_mul_by_520(i64 %x) {
 ; X86-NOOPT-NEXT:    addl %ecx, %edx
 ; X86-NOOPT-NEXT:    retl
 ;
-; HSW-NOOPT-LABEL: test_mul_by_520:
-; HSW-NOOPT:       # %bb.0:
-; HSW-NOOPT-NEXT:    imulq $520, %rdi, %rax # imm = 0x208
-; HSW-NOOPT-NEXT:    # sched: [3:1.00]
-; HSW-NOOPT-NEXT:    retq # sched: [7:1.00]
-;
-; JAG-NOOPT-LABEL: test_mul_by_520:
-; JAG-NOOPT:       # %bb.0:
-; JAG-NOOPT-NEXT:    imulq $520, %rdi, %rax # imm = 0x208
-; JAG-NOOPT-NEXT:    # sched: [6:4.00]
-; JAG-NOOPT-NEXT:    retq # sched: [4:1.00]
+; X64-NOOPT-LABEL: test_mul_by_520:
+; X64-NOOPT:       # %bb.0:
+; X64-NOOPT-NEXT:    imulq $520, %rdi, %rax # imm = 0x208
+; X64-NOOPT-NEXT:    retq
 ;
 ; X64-SLM-LABEL: test_mul_by_520:
 ; X64-SLM:       # %bb.0:
 ; X64-SLM-NEXT:    imulq $520, %rdi, %rax # imm = 0x208
-; X64-SLM-NEXT:    # sched: [3:1.00]
-; X64-SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SLM-NOOPT-LABEL: test_mul_by_520:
-; SLM-NOOPT:       # %bb.0:
-; SLM-NOOPT-NEXT:    imulq $520, %rdi, %rax # imm = 0x208
-; SLM-NOOPT-NEXT:    # sched: [3:1.00]
-; SLM-NOOPT-NEXT:    retq # sched: [4:1.00]
+; X64-SLM-NEXT:    retq
+; X86-OPT-LABEL: test_mul_by_520:
+; X86-OPT:       # %bb.0:
+; X86-OPT-NEXT:    movq %rdi, %rax
+; X86-OPT-NEXT:    shlq $9, %rax
+; X86-OPT-NEXT:    leaq (%rax,%rdi,8), %rax
+; X86-OPT-NEXT:    retq
   %mul = mul nsw i64 %x, 520
   ret i64 %mul
 }
@@ -2126,19 +1661,12 @@ define i64 @test_mul_by_neg10(i64 %x) {
 ; X86-NEXT:    .cfi_def_cfa_offset 4
 ; X86-NEXT:    retl
 ;
-; X64-HSW-LABEL: test_mul_by_neg10:
-; X64-HSW:       # %bb.0:
-; X64-HSW-NEXT:    addq %rdi, %rdi # sched: [1:0.25]
-; X64-HSW-NEXT:    leaq (%rdi,%rdi,4), %rax # sched: [1:0.50]
-; X64-HSW-NEXT:    negq %rax # sched: [1:0.25]
-; X64-HSW-NEXT:    retq # sched: [7:1.00]
-;
-; X64-JAG-LABEL: test_mul_by_neg10:
-; X64-JAG:       # %bb.0:
-; X64-JAG-NEXT:    addq %rdi, %rdi # sched: [1:0.50]
-; X64-JAG-NEXT:    leaq (%rdi,%rdi,4), %rax # sched: [2:1.00]
-; X64-JAG-NEXT:    negq %rax # sched: [1:0.50]
-; X64-JAG-NEXT:    retq # sched: [4:1.00]
+; X64-OPT-LABEL: test_mul_by_neg10:
+; X64-OPT:       # %bb.0:
+; X64-OPT-NEXT:    addq %rdi, %rdi
+; X64-OPT-NEXT:    leaq (%rdi,%rdi,4), %rax
+; X64-OPT-NEXT:    negq %rax
+; X64-OPT-NEXT:    retq
 ;
 ; X86-NOOPT-LABEL: test_mul_by_neg10:
 ; X86-NOOPT:       # %bb.0:
@@ -2151,27 +1679,16 @@ define i64 @test_mul_by_neg10(i64 %x) {
 ; X86-NOOPT-NEXT:    addl %ecx, %edx
 ; X86-NOOPT-NEXT:    retl
 ;
-; HSW-NOOPT-LABEL: test_mul_by_neg10:
-; HSW-NOOPT:       # %bb.0:
-; HSW-NOOPT-NEXT:    imulq $-10, %rdi, %rax # sched: [3:1.00]
-; HSW-NOOPT-NEXT:    retq # sched: [7:1.00]
-;
-; JAG-NOOPT-LABEL: test_mul_by_neg10:
-; JAG-NOOPT:       # %bb.0:
-; JAG-NOOPT-NEXT:    imulq $-10, %rdi, %rax # sched: [6:4.00]
-; JAG-NOOPT-NEXT:    retq # sched: [4:1.00]
-;
-; X64-SLM-LABEL: test_mul_by_neg10:
-; X64-SLM:       # %bb.0:
-; X64-SLM-NEXT:    addq %rdi, %rdi # sched: [1:0.50]
-; X64-SLM-NEXT:    leaq (%rdi,%rdi,4), %rax # sched: [1:1.00]
-; X64-SLM-NEXT:    negq %rax # sched: [1:0.50]
-; X64-SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SLM-NOOPT-LABEL: test_mul_by_neg10:
-; SLM-NOOPT:       # %bb.0:
-; SLM-NOOPT-NEXT:    imulq $-10, %rdi, %rax # sched: [3:1.00]
-; SLM-NOOPT-NEXT:    retq # sched: [4:1.00]
+; X64-NOOPT-LABEL: test_mul_by_neg10:
+; X64-NOOPT:       # %bb.0:
+; X64-NOOPT-NEXT:    imulq $-10, %rdi, %rax
+; X64-NOOPT-NEXT:    retq
+; X86-OPT-LABEL: test_mul_by_neg10:
+; X86-OPT:       # %bb.0:
+; X86-OPT-NEXT:    addq %rdi, %rdi
+; X86-OPT-NEXT:    leaq (%rdi,%rdi,4), %rax
+; X86-OPT-NEXT:    negq %rax
+; X86-OPT-NEXT:    retq
   %mul = mul nsw i64 %x, -10
   ret i64 %mul
 }
@@ -2195,19 +1712,12 @@ define i64 @test_mul_by_neg36(i64 %x) {
 ; X86-NEXT:    .cfi_def_cfa_offset 4
 ; X86-NEXT:    retl
 ;
-; X64-HSW-LABEL: test_mul_by_neg36:
-; X64-HSW:       # %bb.0:
-; X64-HSW-NEXT:    shlq $2, %rdi # sched: [1:0.50]
-; X64-HSW-NEXT:    leaq (%rdi,%rdi,8), %rax # sched: [1:0.50]
-; X64-HSW-NEXT:    negq %rax # sched: [1:0.25]
-; X64-HSW-NEXT:    retq # sched: [7:1.00]
-;
-; X64-JAG-LABEL: test_mul_by_neg36:
-; X64-JAG:       # %bb.0:
-; X64-JAG-NEXT:    shlq $2, %rdi # sched: [1:0.50]
-; X64-JAG-NEXT:    leaq (%rdi,%rdi,8), %rax # sched: [2:1.00]
-; X64-JAG-NEXT:    negq %rax # sched: [1:0.50]
-; X64-JAG-NEXT:    retq # sched: [4:1.00]
+; X64-OPT-LABEL: test_mul_by_neg36:
+; X64-OPT:       # %bb.0:
+; X64-OPT-NEXT:    shlq $2, %rdi
+; X64-OPT-NEXT:    leaq (%rdi,%rdi,8), %rax
+; X64-OPT-NEXT:    negq %rax
+; X64-OPT-NEXT:    retq
 ;
 ; X86-NOOPT-LABEL: test_mul_by_neg36:
 ; X86-NOOPT:       # %bb.0:
@@ -2220,27 +1730,16 @@ define i64 @test_mul_by_neg36(i64 %x) {
 ; X86-NOOPT-NEXT:    addl %ecx, %edx
 ; X86-NOOPT-NEXT:    retl
 ;
-; HSW-NOOPT-LABEL: test_mul_by_neg36:
-; HSW-NOOPT:       # %bb.0:
-; HSW-NOOPT-NEXT:    imulq $-36, %rdi, %rax # sched: [3:1.00]
-; HSW-NOOPT-NEXT:    retq # sched: [7:1.00]
-;
-; JAG-NOOPT-LABEL: test_mul_by_neg36:
-; JAG-NOOPT:       # %bb.0:
-; JAG-NOOPT-NEXT:    imulq $-36, %rdi, %rax # sched: [6:4.00]
-; JAG-NOOPT-NEXT:    retq # sched: [4:1.00]
-;
-; X64-SLM-LABEL: test_mul_by_neg36:
-; X64-SLM:       # %bb.0:
-; X64-SLM-NEXT:    shlq $2, %rdi # sched: [1:1.00]
-; X64-SLM-NEXT:    leaq (%rdi,%rdi,8), %rax # sched: [1:1.00]
-; X64-SLM-NEXT:    negq %rax # sched: [1:0.50]
-; X64-SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SLM-NOOPT-LABEL: test_mul_by_neg36:
-; SLM-NOOPT:       # %bb.0:
-; SLM-NOOPT-NEXT:    imulq $-36, %rdi, %rax # sched: [3:1.00]
-; SLM-NOOPT-NEXT:    retq # sched: [4:1.00]
+; X64-NOOPT-LABEL: test_mul_by_neg36:
+; X64-NOOPT:       # %bb.0:
+; X64-NOOPT-NEXT:    imulq $-36, %rdi, %rax
+; X64-NOOPT-NEXT:    retq
+; X86-OPT-LABEL: test_mul_by_neg36:
+; X86-OPT:       # %bb.0:
+; X86-OPT-NEXT:    shlq $2, %rdi
+; X86-OPT-NEXT:    leaq (%rdi,%rdi,8), %rax
+; X86-OPT-NEXT:    negq %rax
+; X86-OPT-NEXT:    retq
   %mul = mul nsw i64 %x, -36
   ret i64 %mul
 }
@@ -2281,19 +1780,19 @@ define i64 @test_mul_spec(i64 %x) nounwind {
 ;
 ; X64-HSW-LABEL: test_mul_spec:
 ; X64-HSW:       # %bb.0:
-; X64-HSW-NEXT:    leaq (%rdi,%rdi,8), %rcx # sched: [1:0.50]
-; X64-HSW-NEXT:    addq $42, %rcx # sched: [1:0.25]
-; X64-HSW-NEXT:    leaq (%rdi,%rdi,4), %rax # sched: [1:0.50]
-; X64-HSW-NEXT:    addq $2, %rax # sched: [1:0.25]
-; X64-HSW-NEXT:    imulq %rcx, %rax # sched: [3:1.00]
-; X64-HSW-NEXT:    retq # sched: [7:1.00]
+; X64-HSW-NEXT:    leaq (%rdi,%rdi,8), %rcx
+; X64-HSW-NEXT:    addq $42, %rcx
+; X64-HSW-NEXT:    leaq (%rdi,%rdi,4), %rax
+; X64-HSW-NEXT:    addq $2, %rax
+; X64-HSW-NEXT:    imulq %rcx, %rax
+; X64-HSW-NEXT:    retq
 ;
 ; X64-JAG-LABEL: test_mul_spec:
 ; X64-JAG:       # %bb.0:
-; X64-JAG-NEXT:    leaq 42(%rdi,%rdi,8), %rcx # sched: [2:1.00]
-; X64-JAG-NEXT:    leaq 2(%rdi,%rdi,4), %rax # sched: [2:1.00]
-; X64-JAG-NEXT:    imulq %rcx, %rax # sched: [6:4.00]
-; X64-JAG-NEXT:    retq # sched: [4:1.00]
+; X64-JAG-NEXT:    leaq 42(%rdi,%rdi,8), %rcx
+; X64-JAG-NEXT:    leaq 2(%rdi,%rdi,4), %rax
+; X64-JAG-NEXT:    imulq %rcx, %rax
+; X64-JAG-NEXT:    retq
 ;
 ; X86-NOOPT-LABEL: test_mul_spec:
 ; X86-NOOPT:       # %bb.0:
@@ -2329,33 +1828,39 @@ define i64 @test_mul_spec(i64 %x) nounwind {
 ;
 ; HSW-NOOPT-LABEL: test_mul_spec:
 ; HSW-NOOPT:       # %bb.0:
-; HSW-NOOPT-NEXT:    leaq (%rdi,%rdi,8), %rcx # sched: [1:0.50]
-; HSW-NOOPT-NEXT:    addq $42, %rcx # sched: [1:0.25]
-; HSW-NOOPT-NEXT:    leaq (%rdi,%rdi,4), %rax # sched: [1:0.50]
-; HSW-NOOPT-NEXT:    addq $2, %rax # sched: [1:0.25]
-; HSW-NOOPT-NEXT:    imulq %rcx, %rax # sched: [3:1.00]
-; HSW-NOOPT-NEXT:    retq # sched: [7:1.00]
+; HSW-NOOPT-NEXT:    leaq (%rdi,%rdi,8), %rcx
+; HSW-NOOPT-NEXT:    addq $42, %rcx
+; HSW-NOOPT-NEXT:    leaq (%rdi,%rdi,4), %rax
+; HSW-NOOPT-NEXT:    addq $2, %rax
+; HSW-NOOPT-NEXT:    imulq %rcx, %rax
+; HSW-NOOPT-NEXT:    retq
 ;
 ; JAG-NOOPT-LABEL: test_mul_spec:
 ; JAG-NOOPT:       # %bb.0:
-; JAG-NOOPT-NEXT:    leaq 42(%rdi,%rdi,8), %rcx # sched: [2:1.00]
-; JAG-NOOPT-NEXT:    leaq 2(%rdi,%rdi,4), %rax # sched: [2:1.00]
-; JAG-NOOPT-NEXT:    imulq %rcx, %rax # sched: [6:4.00]
-; JAG-NOOPT-NEXT:    retq # sched: [4:1.00]
+; JAG-NOOPT-NEXT:    leaq 42(%rdi,%rdi,8), %rcx
+; JAG-NOOPT-NEXT:    leaq 2(%rdi,%rdi,4), %rax
+; JAG-NOOPT-NEXT:    imulq %rcx, %rax
+; JAG-NOOPT-NEXT:    retq
 ;
 ; X64-SLM-LABEL: test_mul_spec:
 ; X64-SLM:       # %bb.0:
-; X64-SLM-NEXT:    leaq 42(%rdi,%rdi,8), %rcx # sched: [1:1.00]
-; X64-SLM-NEXT:    leaq 2(%rdi,%rdi,4), %rax # sched: [1:1.00]
-; X64-SLM-NEXT:    imulq %rcx, %rax # sched: [3:1.00]
-; X64-SLM-NEXT:    retq # sched: [4:1.00]
+; X64-SLM-NEXT:    leaq 42(%rdi,%rdi,8), %rcx
+; X64-SLM-NEXT:    leaq 2(%rdi,%rdi,4), %rax
+; X64-SLM-NEXT:    imulq %rcx, %rax
+; X64-SLM-NEXT:    retq
 ;
 ; SLM-NOOPT-LABEL: test_mul_spec:
 ; SLM-NOOPT:       # %bb.0:
-; SLM-NOOPT-NEXT:    leaq 42(%rdi,%rdi,8), %rcx # sched: [1:1.00]
-; SLM-NOOPT-NEXT:    leaq 2(%rdi,%rdi,4), %rax # sched: [1:1.00]
-; SLM-NOOPT-NEXT:    imulq %rcx, %rax # sched: [3:1.00]
-; SLM-NOOPT-NEXT:    retq # sched: [4:1.00]
+; SLM-NOOPT-NEXT:    leaq 42(%rdi,%rdi,8), %rcx
+; SLM-NOOPT-NEXT:    leaq 2(%rdi,%rdi,4), %rax
+; SLM-NOOPT-NEXT:    imulq %rcx, %rax
+; SLM-NOOPT-NEXT:    retq
+; X86-OPT-LABEL: test_mul_spec:
+; X86-OPT:       # %bb.0:
+; X86-OPT-NEXT:    leaq 42(%rdi,%rdi,8), %rcx
+; X86-OPT-NEXT:    leaq 2(%rdi,%rdi,4), %rax
+; X86-OPT-NEXT:    imulq %rcx, %rax
+; X86-OPT-NEXT:    retq
   %mul = mul nsw i64 %x, 9
   %add = add nsw i64 %mul, 42
   %mul2 = mul nsw i64 %x, 5
diff --git a/test/CodeGen/X86/mulvi32.ll b/test/CodeGen/X86/mulvi32.ll
index 6c6737a614b0dadffc9319451448c9e87496f175..fc185d18044733ddaca90b3687344f5bbceb1096 100644
--- a/test/CodeGen/X86/mulvi32.ll
+++ b/test/CodeGen/X86/mulvi32.ll
@@ -131,29 +131,24 @@ define <4 x i32> @_mul4xi32b(<4 x i32>, <4 x i32>) {
 define <4 x i64> @_mul4xi32toi64a(<4 x i32>, <4 x i32>) {
 ; SSE2-LABEL: _mul4xi32toi64a:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pxor %xmm3, %xmm3
-; SSE2-NEXT:    movdqa %xmm0, %xmm4
-; SSE2-NEXT:    punpckhdq {{.*#+}} xmm4 = xmm4[2],xmm3[2],xmm4[3],xmm3[3]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
-; SSE2-NEXT:    movdqa %xmm1, %xmm2
-; SSE2-NEXT:    punpckhdq {{.*#+}} xmm2 = xmm2[2],xmm3[2],xmm2[3],xmm3[3]
-; SSE2-NEXT:    pmuludq %xmm4, %xmm2
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
-; SSE2-NEXT:    pmuludq %xmm1, %xmm0
-; SSE2-NEXT:    movdqa %xmm2, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[0,1,1,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[0,1,1,3]
+; SSE2-NEXT:    pmuludq %xmm3, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[2,1,3,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,1,3,3]
+; SSE2-NEXT:    pmuludq %xmm3, %xmm1
+; SSE2-NEXT:    movdqa %xmm2, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE42-LABEL: _mul4xi32toi64a:
 ; SSE42:       # %bb.0:
-; SSE42-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
-; SSE42-NEXT:    pmovzxdq {{.*#+}} xmm3 = xmm2[0],zero,xmm2[1],zero
-; SSE42-NEXT:    pmovzxdq {{.*#+}} xmm4 = xmm0[0],zero,xmm0[1],zero
-; SSE42-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[2,3,0,1]
+; SSE42-NEXT:    pmovzxdq {{.*#+}} xmm3 = xmm1[0],zero,xmm1[1],zero
 ; SSE42-NEXT:    pmovzxdq {{.*#+}} xmm2 = xmm0[0],zero,xmm0[1],zero
 ; SSE42-NEXT:    pmuludq %xmm3, %xmm2
-; SSE42-NEXT:    pmovzxdq {{.*#+}} xmm0 = xmm1[0],zero,xmm1[1],zero
-; SSE42-NEXT:    pmuludq %xmm4, %xmm0
-; SSE42-NEXT:    movdqa %xmm2, %xmm1
+; SSE42-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[2,2,3,3]
+; SSE42-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,2,3,3]
+; SSE42-NEXT:    pmuludq %xmm3, %xmm1
+; SSE42-NEXT:    movdqa %xmm2, %xmm0
 ; SSE42-NEXT:    retq
 ;
 ; AVX1-LABEL: _mul4xi32toi64a:
diff --git a/test/CodeGen/X86/mwaitx-schedule.ll b/test/CodeGen/X86/mwaitx-schedule.ll
deleted file mode 100644
index ea135fdda808dba8a225213ca18ffbd82d5e047f..0000000000000000000000000000000000000000
--- a/test/CodeGen/X86/mwaitx-schedule.ll
+++ /dev/null
@@ -1,65 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=x86-64 -mattr=+mwaitx | FileCheck %s --check-prefix=GENERIC
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=bdver4 | FileCheck %s --check-prefix=BDVER4
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=znver1 | FileCheck %s --check-prefix=ZNVER1
-
-define void @foo(i8* %P, i32 %E, i32 %H) nounwind {
-; GENERIC-LABEL: foo:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    movl %esi, %ecx # sched: [1:0.33]
-; GENERIC-NEXT:    leaq (%rdi), %rax # sched: [1:0.50]
-; GENERIC-NEXT:    monitorx # sched: [100:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER4-LABEL: foo:
-; BDVER4:       # %bb.0:
-; BDVER4-NEXT:    movl %esi, %ecx
-; BDVER4-NEXT:    leaq (%rdi), %rax
-; BDVER4-NEXT:    monitorx
-; BDVER4-NEXT:    retq
-;
-; ZNVER1-LABEL: foo:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    movl %esi, %ecx # sched: [1:0.25]
-; ZNVER1-NEXT:    leaq (%rdi), %rax # sched: [1:0.25]
-; ZNVER1-NEXT:    monitorx # sched: [100:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  tail call void @llvm.x86.monitorx(i8* %P, i32 %E, i32 %H)
-  ret void
-}
-declare void @llvm.x86.monitorx(i8*, i32, i32) nounwind
-
-define void @bar(i32 %E, i32 %H, i32 %C) nounwind {
-; GENERIC-LABEL: bar:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    pushq %rbx # sched: [5:1.00]
-; GENERIC-NEXT:    movl %edx, %ebx # sched: [1:0.33]
-; GENERIC-NEXT:    movl %esi, %eax # sched: [1:0.33]
-; GENERIC-NEXT:    movl %edi, %ecx # sched: [1:0.33]
-; GENERIC-NEXT:    mwaitx # sched: [100:0.33]
-; GENERIC-NEXT:    popq %rbx # sched: [6:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER4-LABEL: bar:
-; BDVER4:       # %bb.0:
-; BDVER4-NEXT:    pushq %rbx
-; BDVER4-NEXT:    movl %edx, %ebx
-; BDVER4-NEXT:    movl %esi, %eax
-; BDVER4-NEXT:    movl %edi, %ecx
-; BDVER4-NEXT:    mwaitx
-; BDVER4-NEXT:    popq %rbx
-; BDVER4-NEXT:    retq
-;
-; ZNVER1-LABEL: bar:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    pushq %rbx # sched: [1:0.50]
-; ZNVER1-NEXT:    movl %edx, %ebx # sched: [1:0.25]
-; ZNVER1-NEXT:    movl %esi, %eax # sched: [1:0.25]
-; ZNVER1-NEXT:    movl %edi, %ecx # sched: [1:0.25]
-; ZNVER1-NEXT:    mwaitx # sched: [100:0.25]
-; ZNVER1-NEXT:    popq %rbx # sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  tail call void @llvm.x86.mwaitx(i32 %E, i32 %H, i32 %C)
-  ret void
-}
-declare void @llvm.x86.mwaitx(i32, i32, i32) nounwind
diff --git a/test/CodeGen/X86/note-cet-property.ll b/test/CodeGen/X86/note-cet-property.ll
index 2b7dbbeb38c301f1ea835e0d595be182d802afa9..f3cc32b65dc35be98092c98c9b2eec73861b8f2d 100644
--- a/test/CodeGen/X86/note-cet-property.ll
+++ b/test/CodeGen/X86/note-cet-property.ll
@@ -22,8 +22,8 @@
 ; X86_64-NEXT: .long    5
 ; X86_64-NEXT: .asciz   "GNU"
 ; X86_64-NEXT: .long    3221225474
-; X86_64-NEXT: .long    8
-; X86_64-NEXT: .quad    3
+; X86_64-NEXT: .long    4
+; X86_64-NEXT: .long    3
 ; X86_64-NEXT: .p2align 3
 
 !llvm.module.flags = !{!0, !1}
diff --git a/test/CodeGen/X86/oddshuffles.ll b/test/CodeGen/X86/oddshuffles.ll
index 0d501fe5233c03ca69bad093bc49408541ed82d6..0f0c82390517cb83d6617fbc57522889b32b2da4 100644
--- a/test/CodeGen/X86/oddshuffles.ll
+++ b/test/CodeGen/X86/oddshuffles.ll
@@ -1673,7 +1673,7 @@ define void @interleave_24i32_in(<24 x i32>* %p, <8 x i32>* %q1, <8 x i32>* %q2,
 ; XOP-LABEL: interleave_24i32_in:
 ; XOP:       # %bb.0:
 ; XOP-NEXT:    vmovupd (%rsi), %ymm0
-; XOP-NEXT:    vmovupd (%rcx), %ymm1
+; XOP-NEXT:    vmovups (%rcx), %ymm1
 ; XOP-NEXT:    vmovups 16(%rcx), %xmm2
 ; XOP-NEXT:    vmovups (%rdx), %xmm3
 ; XOP-NEXT:    vmovups 16(%rdx), %xmm4
@@ -1743,10 +1743,10 @@ define <2 x double> @wrongorder(<4 x double> %A, <8 x double>* %P) #0 {
 ;
 ; AVX2-LABEL: wrongorder:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vbroadcastsd %xmm0, %ymm1
-; AVX2-NEXT:    vmovapd %ymm1, 32(%rdi)
-; AVX2-NEXT:    vmovapd %ymm1, (%rdi)
-; AVX2-NEXT:    vmovddup {{.*#+}} xmm0 = xmm0[0,0]
+; AVX2-NEXT:    vbroadcastsd %xmm0, %ymm0
+; AVX2-NEXT:    vmovaps %ymm0, 32(%rdi)
+; AVX2-NEXT:    vmovaps %ymm0, (%rdi)
+; AVX2-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
diff --git a/test/CodeGen/X86/phaddsub-extract.ll b/test/CodeGen/X86/phaddsub-extract.ll
new file mode 100644
index 0000000000000000000000000000000000000000..2ec1f254c354c51a2499796484655a2f25e60628
--- /dev/null
+++ b/test/CodeGen/X86/phaddsub-extract.ll
@@ -0,0 +1,1122 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+ssse3              | FileCheck %s --check-prefixes=SSE3,SSE3-SLOW
+; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+ssse3,fast-hops    | FileCheck %s --check-prefixes=SSE3,SSE3-FAST
+; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx                | FileCheck %s --check-prefixes=AVX,AVX-SLOW,AVX1-SLOW
+; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx,fast-hops      | FileCheck %s --check-prefixes=AVX,AVX-FAST,AVX1-FAST
+; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx2               | FileCheck %s --check-prefixes=AVX,AVX-SLOW,AVX2-SLOW
+; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx2,fast-hops     | FileCheck %s --check-prefixes=AVX,AVX-FAST,AVX2-FAST
+; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx512vl           | FileCheck %s --check-prefixes=AVX,AVX-SLOW,AVX512-SLOW
+; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx512vl,fast-hops | FileCheck %s --check-prefixes=AVX,AVX-FAST,AVX512-FAST
+
+; 128-bit vectors, 16/32-bit, add/sub
+
+define i32 @extract_extract_v4i32_add_i32(<4 x i32> %x) {
+; SSE3-SLOW-LABEL: extract_extract_v4i32_add_i32:
+; SSE3-SLOW:       # %bb.0:
+; SSE3-SLOW-NEXT:    movd %xmm0, %ecx
+; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE3-SLOW-NEXT:    movd %xmm0, %eax
+; SSE3-SLOW-NEXT:    addl %ecx, %eax
+; SSE3-SLOW-NEXT:    retq
+;
+; SSE3-FAST-LABEL: extract_extract_v4i32_add_i32:
+; SSE3-FAST:       # %bb.0:
+; SSE3-FAST-NEXT:    phaddd %xmm0, %xmm0
+; SSE3-FAST-NEXT:    movd %xmm0, %eax
+; SSE3-FAST-NEXT:    retq
+;
+; AVX-SLOW-LABEL: extract_extract_v4i32_add_i32:
+; AVX-SLOW:       # %bb.0:
+; AVX-SLOW-NEXT:    vmovd %xmm0, %ecx
+; AVX-SLOW-NEXT:    vpextrd $1, %xmm0, %eax
+; AVX-SLOW-NEXT:    addl %ecx, %eax
+; AVX-SLOW-NEXT:    retq
+;
+; AVX-FAST-LABEL: extract_extract_v4i32_add_i32:
+; AVX-FAST:       # %bb.0:
+; AVX-FAST-NEXT:    vphaddd %xmm0, %xmm0, %xmm0
+; AVX-FAST-NEXT:    vmovd %xmm0, %eax
+; AVX-FAST-NEXT:    retq
+  %x0 = extractelement <4 x i32> %x, i32 0
+  %x1 = extractelement <4 x i32> %x, i32 1
+  %x01 = add i32 %x0, %x1
+  ret i32 %x01
+}
+
+define i32 @extract_extract_v4i32_add_i32_commute(<4 x i32> %x) {
+; SSE3-SLOW-LABEL: extract_extract_v4i32_add_i32_commute:
+; SSE3-SLOW:       # %bb.0:
+; SSE3-SLOW-NEXT:    movd %xmm0, %ecx
+; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE3-SLOW-NEXT:    movd %xmm0, %eax
+; SSE3-SLOW-NEXT:    addl %ecx, %eax
+; SSE3-SLOW-NEXT:    retq
+;
+; SSE3-FAST-LABEL: extract_extract_v4i32_add_i32_commute:
+; SSE3-FAST:       # %bb.0:
+; SSE3-FAST-NEXT:    phaddd %xmm0, %xmm0
+; SSE3-FAST-NEXT:    movd %xmm0, %eax
+; SSE3-FAST-NEXT:    retq
+;
+; AVX-SLOW-LABEL: extract_extract_v4i32_add_i32_commute:
+; AVX-SLOW:       # %bb.0:
+; AVX-SLOW-NEXT:    vmovd %xmm0, %ecx
+; AVX-SLOW-NEXT:    vpextrd $1, %xmm0, %eax
+; AVX-SLOW-NEXT:    addl %ecx, %eax
+; AVX-SLOW-NEXT:    retq
+;
+; AVX-FAST-LABEL: extract_extract_v4i32_add_i32_commute:
+; AVX-FAST:       # %bb.0:
+; AVX-FAST-NEXT:    vphaddd %xmm0, %xmm0, %xmm0
+; AVX-FAST-NEXT:    vmovd %xmm0, %eax
+; AVX-FAST-NEXT:    retq
+  %x0 = extractelement <4 x i32> %x, i32 0
+  %x1 = extractelement <4 x i32> %x, i32 1
+  %x01 = add i32 %x1, %x0
+  ret i32 %x01
+}
+
+define i16 @extract_extract_v8i16_add_i16(<8 x i16> %x) {
+; SSE3-SLOW-LABEL: extract_extract_v8i16_add_i16:
+; SSE3-SLOW:       # %bb.0:
+; SSE3-SLOW-NEXT:    movd %xmm0, %ecx
+; SSE3-SLOW-NEXT:    pextrw $1, %xmm0, %eax
+; SSE3-SLOW-NEXT:    addl %ecx, %eax
+; SSE3-SLOW-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE3-SLOW-NEXT:    retq
+;
+; SSE3-FAST-LABEL: extract_extract_v8i16_add_i16:
+; SSE3-FAST:       # %bb.0:
+; SSE3-FAST-NEXT:    phaddw %xmm0, %xmm0
+; SSE3-FAST-NEXT:    movd %xmm0, %eax
+; SSE3-FAST-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE3-FAST-NEXT:    retq
+;
+; AVX-SLOW-LABEL: extract_extract_v8i16_add_i16:
+; AVX-SLOW:       # %bb.0:
+; AVX-SLOW-NEXT:    vmovd %xmm0, %ecx
+; AVX-SLOW-NEXT:    vpextrw $1, %xmm0, %eax
+; AVX-SLOW-NEXT:    addl %ecx, %eax
+; AVX-SLOW-NEXT:    # kill: def $ax killed $ax killed $eax
+; AVX-SLOW-NEXT:    retq
+;
+; AVX-FAST-LABEL: extract_extract_v8i16_add_i16:
+; AVX-FAST:       # %bb.0:
+; AVX-FAST-NEXT:    vphaddw %xmm0, %xmm0, %xmm0
+; AVX-FAST-NEXT:    vmovd %xmm0, %eax
+; AVX-FAST-NEXT:    # kill: def $ax killed $ax killed $eax
+; AVX-FAST-NEXT:    retq
+  %x0 = extractelement <8 x i16> %x, i32 0
+  %x1 = extractelement <8 x i16> %x, i32 1
+  %x01 = add i16 %x0, %x1
+  ret i16 %x01
+}
+
+define i16 @extract_extract_v8i16_add_i16_commute(<8 x i16> %x) {
+; SSE3-SLOW-LABEL: extract_extract_v8i16_add_i16_commute:
+; SSE3-SLOW:       # %bb.0:
+; SSE3-SLOW-NEXT:    movd %xmm0, %ecx
+; SSE3-SLOW-NEXT:    pextrw $1, %xmm0, %eax
+; SSE3-SLOW-NEXT:    addl %ecx, %eax
+; SSE3-SLOW-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE3-SLOW-NEXT:    retq
+;
+; SSE3-FAST-LABEL: extract_extract_v8i16_add_i16_commute:
+; SSE3-FAST:       # %bb.0:
+; SSE3-FAST-NEXT:    phaddw %xmm0, %xmm0
+; SSE3-FAST-NEXT:    movd %xmm0, %eax
+; SSE3-FAST-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE3-FAST-NEXT:    retq
+;
+; AVX-SLOW-LABEL: extract_extract_v8i16_add_i16_commute:
+; AVX-SLOW:       # %bb.0:
+; AVX-SLOW-NEXT:    vmovd %xmm0, %ecx
+; AVX-SLOW-NEXT:    vpextrw $1, %xmm0, %eax
+; AVX-SLOW-NEXT:    addl %ecx, %eax
+; AVX-SLOW-NEXT:    # kill: def $ax killed $ax killed $eax
+; AVX-SLOW-NEXT:    retq
+;
+; AVX-FAST-LABEL: extract_extract_v8i16_add_i16_commute:
+; AVX-FAST:       # %bb.0:
+; AVX-FAST-NEXT:    vphaddw %xmm0, %xmm0, %xmm0
+; AVX-FAST-NEXT:    vmovd %xmm0, %eax
+; AVX-FAST-NEXT:    # kill: def $ax killed $ax killed $eax
+; AVX-FAST-NEXT:    retq
+  %x0 = extractelement <8 x i16> %x, i32 0
+  %x1 = extractelement <8 x i16> %x, i32 1
+  %x01 = add i16 %x1, %x0
+  ret i16 %x01
+}
+
+define i32 @extract_extract_v4i32_sub_i32(<4 x i32> %x) {
+; SSE3-SLOW-LABEL: extract_extract_v4i32_sub_i32:
+; SSE3-SLOW:       # %bb.0:
+; SSE3-SLOW-NEXT:    movd %xmm0, %eax
+; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE3-SLOW-NEXT:    movd %xmm0, %ecx
+; SSE3-SLOW-NEXT:    subl %ecx, %eax
+; SSE3-SLOW-NEXT:    retq
+;
+; SSE3-FAST-LABEL: extract_extract_v4i32_sub_i32:
+; SSE3-FAST:       # %bb.0:
+; SSE3-FAST-NEXT:    phsubd %xmm0, %xmm0
+; SSE3-FAST-NEXT:    movd %xmm0, %eax
+; SSE3-FAST-NEXT:    retq
+;
+; AVX-SLOW-LABEL: extract_extract_v4i32_sub_i32:
+; AVX-SLOW:       # %bb.0:
+; AVX-SLOW-NEXT:    vmovd %xmm0, %eax
+; AVX-SLOW-NEXT:    vpextrd $1, %xmm0, %ecx
+; AVX-SLOW-NEXT:    subl %ecx, %eax
+; AVX-SLOW-NEXT:    retq
+;
+; AVX-FAST-LABEL: extract_extract_v4i32_sub_i32:
+; AVX-FAST:       # %bb.0:
+; AVX-FAST-NEXT:    vphsubd %xmm0, %xmm0, %xmm0
+; AVX-FAST-NEXT:    vmovd %xmm0, %eax
+; AVX-FAST-NEXT:    retq
+  %x0 = extractelement <4 x i32> %x, i32 0
+  %x1 = extractelement <4 x i32> %x, i32 1
+  %x01 = sub i32 %x0, %x1
+  ret i32 %x01
+}
+
+define i32 @extract_extract_v4i32_sub_i32_commute(<4 x i32> %x) {
+; SSE3-LABEL: extract_extract_v4i32_sub_i32_commute:
+; SSE3:       # %bb.0:
+; SSE3-NEXT:    movd %xmm0, %ecx
+; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE3-NEXT:    movd %xmm0, %eax
+; SSE3-NEXT:    subl %ecx, %eax
+; SSE3-NEXT:    retq
+;
+; AVX-LABEL: extract_extract_v4i32_sub_i32_commute:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovd %xmm0, %ecx
+; AVX-NEXT:    vpextrd $1, %xmm0, %eax
+; AVX-NEXT:    subl %ecx, %eax
+; AVX-NEXT:    retq
+  %x0 = extractelement <4 x i32> %x, i32 0
+  %x1 = extractelement <4 x i32> %x, i32 1
+  %x01 = sub i32 %x1, %x0
+  ret i32 %x01
+}
+
+define i16 @extract_extract_v8i16_sub_i16(<8 x i16> %x) {
+; SSE3-SLOW-LABEL: extract_extract_v8i16_sub_i16:
+; SSE3-SLOW:       # %bb.0:
+; SSE3-SLOW-NEXT:    movd %xmm0, %eax
+; SSE3-SLOW-NEXT:    pextrw $1, %xmm0, %ecx
+; SSE3-SLOW-NEXT:    subl %ecx, %eax
+; SSE3-SLOW-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE3-SLOW-NEXT:    retq
+;
+; SSE3-FAST-LABEL: extract_extract_v8i16_sub_i16:
+; SSE3-FAST:       # %bb.0:
+; SSE3-FAST-NEXT:    phsubw %xmm0, %xmm0
+; SSE3-FAST-NEXT:    movd %xmm0, %eax
+; SSE3-FAST-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE3-FAST-NEXT:    retq
+;
+; AVX-SLOW-LABEL: extract_extract_v8i16_sub_i16:
+; AVX-SLOW:       # %bb.0:
+; AVX-SLOW-NEXT:    vmovd %xmm0, %eax
+; AVX-SLOW-NEXT:    vpextrw $1, %xmm0, %ecx
+; AVX-SLOW-NEXT:    subl %ecx, %eax
+; AVX-SLOW-NEXT:    # kill: def $ax killed $ax killed $eax
+; AVX-SLOW-NEXT:    retq
+;
+; AVX-FAST-LABEL: extract_extract_v8i16_sub_i16:
+; AVX-FAST:       # %bb.0:
+; AVX-FAST-NEXT:    vphsubw %xmm0, %xmm0, %xmm0
+; AVX-FAST-NEXT:    vmovd %xmm0, %eax
+; AVX-FAST-NEXT:    # kill: def $ax killed $ax killed $eax
+; AVX-FAST-NEXT:    retq
+  %x0 = extractelement <8 x i16> %x, i32 0
+  %x1 = extractelement <8 x i16> %x, i32 1
+  %x01 = sub i16 %x0, %x1
+  ret i16 %x01
+}
+
+define i16 @extract_extract_v8i16_sub_i16_commute(<8 x i16> %x) {
+; SSE3-LABEL: extract_extract_v8i16_sub_i16_commute:
+; SSE3:       # %bb.0:
+; SSE3-NEXT:    movd %xmm0, %ecx
+; SSE3-NEXT:    pextrw $1, %xmm0, %eax
+; SSE3-NEXT:    subl %ecx, %eax
+; SSE3-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE3-NEXT:    retq
+;
+; AVX-LABEL: extract_extract_v8i16_sub_i16_commute:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovd %xmm0, %ecx
+; AVX-NEXT:    vpextrw $1, %xmm0, %eax
+; AVX-NEXT:    subl %ecx, %eax
+; AVX-NEXT:    # kill: def $ax killed $ax killed $eax
+; AVX-NEXT:    retq
+  %x0 = extractelement <8 x i16> %x, i32 0
+  %x1 = extractelement <8 x i16> %x, i32 1
+  %x01 = sub i16 %x1, %x0
+  ret i16 %x01
+}
+
+; 256-bit vectors, i32/i16, add/sub
+
+define i32 @extract_extract_v8i32_add_i32(<8 x i32> %x) {
+; SSE3-SLOW-LABEL: extract_extract_v8i32_add_i32:
+; SSE3-SLOW:       # %bb.0:
+; SSE3-SLOW-NEXT:    movd %xmm0, %ecx
+; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE3-SLOW-NEXT:    movd %xmm0, %eax
+; SSE3-SLOW-NEXT:    addl %ecx, %eax
+; SSE3-SLOW-NEXT:    retq
+;
+; SSE3-FAST-LABEL: extract_extract_v8i32_add_i32:
+; SSE3-FAST:       # %bb.0:
+; SSE3-FAST-NEXT:    phaddd %xmm0, %xmm0
+; SSE3-FAST-NEXT:    movd %xmm0, %eax
+; SSE3-FAST-NEXT:    retq
+;
+; AVX-SLOW-LABEL: extract_extract_v8i32_add_i32:
+; AVX-SLOW:       # %bb.0:
+; AVX-SLOW-NEXT:    vmovd %xmm0, %ecx
+; AVX-SLOW-NEXT:    vpextrd $1, %xmm0, %eax
+; AVX-SLOW-NEXT:    addl %ecx, %eax
+; AVX-SLOW-NEXT:    vzeroupper
+; AVX-SLOW-NEXT:    retq
+;
+; AVX-FAST-LABEL: extract_extract_v8i32_add_i32:
+; AVX-FAST:       # %bb.0:
+; AVX-FAST-NEXT:    vphaddd %xmm0, %xmm0, %xmm0
+; AVX-FAST-NEXT:    vmovd %xmm0, %eax
+; AVX-FAST-NEXT:    vzeroupper
+; AVX-FAST-NEXT:    retq
+  %x0 = extractelement <8 x i32> %x, i32 0
+  %x1 = extractelement <8 x i32> %x, i32 1
+  %x01 = add i32 %x0, %x1
+  ret i32 %x01
+}
+
+define i32 @extract_extract_v8i32_add_i32_commute(<8 x i32> %x) {
+; SSE3-SLOW-LABEL: extract_extract_v8i32_add_i32_commute:
+; SSE3-SLOW:       # %bb.0:
+; SSE3-SLOW-NEXT:    movd %xmm0, %ecx
+; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE3-SLOW-NEXT:    movd %xmm0, %eax
+; SSE3-SLOW-NEXT:    addl %ecx, %eax
+; SSE3-SLOW-NEXT:    retq
+;
+; SSE3-FAST-LABEL: extract_extract_v8i32_add_i32_commute:
+; SSE3-FAST:       # %bb.0:
+; SSE3-FAST-NEXT:    phaddd %xmm0, %xmm0
+; SSE3-FAST-NEXT:    movd %xmm0, %eax
+; SSE3-FAST-NEXT:    retq
+;
+; AVX-SLOW-LABEL: extract_extract_v8i32_add_i32_commute:
+; AVX-SLOW:       # %bb.0:
+; AVX-SLOW-NEXT:    vmovd %xmm0, %ecx
+; AVX-SLOW-NEXT:    vpextrd $1, %xmm0, %eax
+; AVX-SLOW-NEXT:    addl %ecx, %eax
+; AVX-SLOW-NEXT:    vzeroupper
+; AVX-SLOW-NEXT:    retq
+;
+; AVX-FAST-LABEL: extract_extract_v8i32_add_i32_commute:
+; AVX-FAST:       # %bb.0:
+; AVX-FAST-NEXT:    vphaddd %xmm0, %xmm0, %xmm0
+; AVX-FAST-NEXT:    vmovd %xmm0, %eax
+; AVX-FAST-NEXT:    vzeroupper
+; AVX-FAST-NEXT:    retq
+  %x0 = extractelement <8 x i32> %x, i32 0
+  %x1 = extractelement <8 x i32> %x, i32 1
+  %x01 = add i32 %x1, %x0
+  ret i32 %x01
+}
+
+define i16 @extract_extract_v16i16_add_i16(<16 x i16> %x) {
+; SSE3-SLOW-LABEL: extract_extract_v16i16_add_i16:
+; SSE3-SLOW:       # %bb.0:
+; SSE3-SLOW-NEXT:    movd %xmm0, %ecx
+; SSE3-SLOW-NEXT:    pextrw $1, %xmm0, %eax
+; SSE3-SLOW-NEXT:    addl %ecx, %eax
+; SSE3-SLOW-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE3-SLOW-NEXT:    retq
+;
+; SSE3-FAST-LABEL: extract_extract_v16i16_add_i16:
+; SSE3-FAST:       # %bb.0:
+; SSE3-FAST-NEXT:    phaddw %xmm0, %xmm0
+; SSE3-FAST-NEXT:    movd %xmm0, %eax
+; SSE3-FAST-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE3-FAST-NEXT:    retq
+;
+; AVX-SLOW-LABEL: extract_extract_v16i16_add_i16:
+; AVX-SLOW:       # %bb.0:
+; AVX-SLOW-NEXT:    vmovd %xmm0, %ecx
+; AVX-SLOW-NEXT:    vpextrw $1, %xmm0, %eax
+; AVX-SLOW-NEXT:    addl %ecx, %eax
+; AVX-SLOW-NEXT:    # kill: def $ax killed $ax killed $eax
+; AVX-SLOW-NEXT:    vzeroupper
+; AVX-SLOW-NEXT:    retq
+;
+; AVX-FAST-LABEL: extract_extract_v16i16_add_i16:
+; AVX-FAST:       # %bb.0:
+; AVX-FAST-NEXT:    vphaddw %xmm0, %xmm0, %xmm0
+; AVX-FAST-NEXT:    vmovd %xmm0, %eax
+; AVX-FAST-NEXT:    # kill: def $ax killed $ax killed $eax
+; AVX-FAST-NEXT:    vzeroupper
+; AVX-FAST-NEXT:    retq
+  %x0 = extractelement <16 x i16> %x, i32 0
+  %x1 = extractelement <16 x i16> %x, i32 1
+  %x01 = add i16 %x0, %x1
+  ret i16 %x01
+}
+
+define i16 @extract_extract_v16i16_add_i16_commute(<16 x i16> %x) {
+; SSE3-SLOW-LABEL: extract_extract_v16i16_add_i16_commute:
+; SSE3-SLOW:       # %bb.0:
+; SSE3-SLOW-NEXT:    movd %xmm0, %ecx
+; SSE3-SLOW-NEXT:    pextrw $1, %xmm0, %eax
+; SSE3-SLOW-NEXT:    addl %ecx, %eax
+; SSE3-SLOW-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE3-SLOW-NEXT:    retq
+;
+; SSE3-FAST-LABEL: extract_extract_v16i16_add_i16_commute:
+; SSE3-FAST:       # %bb.0:
+; SSE3-FAST-NEXT:    phaddw %xmm0, %xmm0
+; SSE3-FAST-NEXT:    movd %xmm0, %eax
+; SSE3-FAST-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE3-FAST-NEXT:    retq
+;
+; AVX-SLOW-LABEL: extract_extract_v16i16_add_i16_commute:
+; AVX-SLOW:       # %bb.0:
+; AVX-SLOW-NEXT:    vmovd %xmm0, %ecx
+; AVX-SLOW-NEXT:    vpextrw $1, %xmm0, %eax
+; AVX-SLOW-NEXT:    addl %ecx, %eax
+; AVX-SLOW-NEXT:    # kill: def $ax killed $ax killed $eax
+; AVX-SLOW-NEXT:    vzeroupper
+; AVX-SLOW-NEXT:    retq
+;
+; AVX-FAST-LABEL: extract_extract_v16i16_add_i16_commute:
+; AVX-FAST:       # %bb.0:
+; AVX-FAST-NEXT:    vphaddw %xmm0, %xmm0, %xmm0
+; AVX-FAST-NEXT:    vmovd %xmm0, %eax
+; AVX-FAST-NEXT:    # kill: def $ax killed $ax killed $eax
+; AVX-FAST-NEXT:    vzeroupper
+; AVX-FAST-NEXT:    retq
+  %x0 = extractelement <16 x i16> %x, i32 0
+  %x1 = extractelement <16 x i16> %x, i32 1
+  %x01 = add i16 %x1, %x0
+  ret i16 %x01
+}
+
+define i32 @extract_extract_v8i32_sub_i32(<8 x i32> %x) {
+; SSE3-SLOW-LABEL: extract_extract_v8i32_sub_i32:
+; SSE3-SLOW:       # %bb.0:
+; SSE3-SLOW-NEXT:    movd %xmm0, %eax
+; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE3-SLOW-NEXT:    movd %xmm0, %ecx
+; SSE3-SLOW-NEXT:    subl %ecx, %eax
+; SSE3-SLOW-NEXT:    retq
+;
+; SSE3-FAST-LABEL: extract_extract_v8i32_sub_i32:
+; SSE3-FAST:       # %bb.0:
+; SSE3-FAST-NEXT:    phsubd %xmm0, %xmm0
+; SSE3-FAST-NEXT:    movd %xmm0, %eax
+; SSE3-FAST-NEXT:    retq
+;
+; AVX-SLOW-LABEL: extract_extract_v8i32_sub_i32:
+; AVX-SLOW:       # %bb.0:
+; AVX-SLOW-NEXT:    vmovd %xmm0, %eax
+; AVX-SLOW-NEXT:    vpextrd $1, %xmm0, %ecx
+; AVX-SLOW-NEXT:    subl %ecx, %eax
+; AVX-SLOW-NEXT:    vzeroupper
+; AVX-SLOW-NEXT:    retq
+;
+; AVX-FAST-LABEL: extract_extract_v8i32_sub_i32:
+; AVX-FAST:       # %bb.0:
+; AVX-FAST-NEXT:    vphsubd %xmm0, %xmm0, %xmm0
+; AVX-FAST-NEXT:    vmovd %xmm0, %eax
+; AVX-FAST-NEXT:    vzeroupper
+; AVX-FAST-NEXT:    retq
+  %x0 = extractelement <8 x i32> %x, i32 0
+  %x1 = extractelement <8 x i32> %x, i32 1
+  %x01 = sub i32 %x0, %x1
+  ret i32 %x01
+}
+
+; Negative test...or get hoppy and negate?
+
+define i32 @extract_extract_v8i32_sub_i32_commute(<8 x i32> %x) {
+; SSE3-LABEL: extract_extract_v8i32_sub_i32_commute:
+; SSE3:       # %bb.0:
+; SSE3-NEXT:    movd %xmm0, %ecx
+; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE3-NEXT:    movd %xmm0, %eax
+; SSE3-NEXT:    subl %ecx, %eax
+; SSE3-NEXT:    retq
+;
+; AVX-LABEL: extract_extract_v8i32_sub_i32_commute:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovd %xmm0, %ecx
+; AVX-NEXT:    vpextrd $1, %xmm0, %eax
+; AVX-NEXT:    subl %ecx, %eax
+; AVX-NEXT:    vzeroupper
+; AVX-NEXT:    retq
+  %x0 = extractelement <8 x i32> %x, i32 0
+  %x1 = extractelement <8 x i32> %x, i32 1
+  %x01 = sub i32 %x1, %x0
+  ret i32 %x01
+}
+
+define i16 @extract_extract_v16i16_sub_i16(<16 x i16> %x) {
+; SSE3-SLOW-LABEL: extract_extract_v16i16_sub_i16:
+; SSE3-SLOW:       # %bb.0:
+; SSE3-SLOW-NEXT:    movd %xmm0, %eax
+; SSE3-SLOW-NEXT:    pextrw $1, %xmm0, %ecx
+; SSE3-SLOW-NEXT:    subl %ecx, %eax
+; SSE3-SLOW-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE3-SLOW-NEXT:    retq
+;
+; SSE3-FAST-LABEL: extract_extract_v16i16_sub_i16:
+; SSE3-FAST:       # %bb.0:
+; SSE3-FAST-NEXT:    phsubw %xmm0, %xmm0
+; SSE3-FAST-NEXT:    movd %xmm0, %eax
+; SSE3-FAST-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE3-FAST-NEXT:    retq
+;
+; AVX-SLOW-LABEL: extract_extract_v16i16_sub_i16:
+; AVX-SLOW:       # %bb.0:
+; AVX-SLOW-NEXT:    vmovd %xmm0, %eax
+; AVX-SLOW-NEXT:    vpextrw $1, %xmm0, %ecx
+; AVX-SLOW-NEXT:    subl %ecx, %eax
+; AVX-SLOW-NEXT:    # kill: def $ax killed $ax killed $eax
+; AVX-SLOW-NEXT:    vzeroupper
+; AVX-SLOW-NEXT:    retq
+;
+; AVX-FAST-LABEL: extract_extract_v16i16_sub_i16:
+; AVX-FAST:       # %bb.0:
+; AVX-FAST-NEXT:    vphsubw %xmm0, %xmm0, %xmm0
+; AVX-FAST-NEXT:    vmovd %xmm0, %eax
+; AVX-FAST-NEXT:    # kill: def $ax killed $ax killed $eax
+; AVX-FAST-NEXT:    vzeroupper
+; AVX-FAST-NEXT:    retq
+  %x0 = extractelement <16 x i16> %x, i32 0
+  %x1 = extractelement <16 x i16> %x, i32 1
+  %x01 = sub i16 %x0, %x1
+  ret i16 %x01
+}
+
+; Negative test...or get hoppy and negate?
+
+define i16 @extract_extract_v16i16_sub_i16_commute(<16 x i16> %x) {
+; SSE3-LABEL: extract_extract_v16i16_sub_i16_commute:
+; SSE3:       # %bb.0:
+; SSE3-NEXT:    movd %xmm0, %ecx
+; SSE3-NEXT:    pextrw $1, %xmm0, %eax
+; SSE3-NEXT:    subl %ecx, %eax
+; SSE3-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE3-NEXT:    retq
+;
+; AVX-LABEL: extract_extract_v16i16_sub_i16_commute:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovd %xmm0, %ecx
+; AVX-NEXT:    vpextrw $1, %xmm0, %eax
+; AVX-NEXT:    subl %ecx, %eax
+; AVX-NEXT:    # kill: def $ax killed $ax killed $eax
+; AVX-NEXT:    vzeroupper
+; AVX-NEXT:    retq
+  %x0 = extractelement <16 x i16> %x, i32 0
+  %x1 = extractelement <16 x i16> %x, i32 1
+  %x01 = sub i16 %x1, %x0
+  ret i16 %x01
+}
+
+; 512-bit vectors, i32/i16, add/sub
+
+define i32 @extract_extract_v16i32_add_i32(<16 x i32> %x) {
+; SSE3-SLOW-LABEL: extract_extract_v16i32_add_i32:
+; SSE3-SLOW:       # %bb.0:
+; SSE3-SLOW-NEXT:    movd %xmm0, %ecx
+; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE3-SLOW-NEXT:    movd %xmm0, %eax
+; SSE3-SLOW-NEXT:    addl %ecx, %eax
+; SSE3-SLOW-NEXT:    retq
+;
+; SSE3-FAST-LABEL: extract_extract_v16i32_add_i32:
+; SSE3-FAST:       # %bb.0:
+; SSE3-FAST-NEXT:    phaddd %xmm0, %xmm0
+; SSE3-FAST-NEXT:    movd %xmm0, %eax
+; SSE3-FAST-NEXT:    retq
+;
+; AVX-SLOW-LABEL: extract_extract_v16i32_add_i32:
+; AVX-SLOW:       # %bb.0:
+; AVX-SLOW-NEXT:    vmovd %xmm0, %ecx
+; AVX-SLOW-NEXT:    vpextrd $1, %xmm0, %eax
+; AVX-SLOW-NEXT:    addl %ecx, %eax
+; AVX-SLOW-NEXT:    vzeroupper
+; AVX-SLOW-NEXT:    retq
+;
+; AVX-FAST-LABEL: extract_extract_v16i32_add_i32:
+; AVX-FAST:       # %bb.0:
+; AVX-FAST-NEXT:    vphaddd %xmm0, %xmm0, %xmm0
+; AVX-FAST-NEXT:    vmovd %xmm0, %eax
+; AVX-FAST-NEXT:    vzeroupper
+; AVX-FAST-NEXT:    retq
+  %x0 = extractelement <16 x i32> %x, i32 0
+  %x1 = extractelement <16 x i32> %x, i32 1
+  %x01 = add i32 %x0, %x1
+  ret i32 %x01
+}
+
+define i32 @extract_extract_v16i32_add_i32_commute(<16 x i32> %x) {
+; SSE3-SLOW-LABEL: extract_extract_v16i32_add_i32_commute:
+; SSE3-SLOW:       # %bb.0:
+; SSE3-SLOW-NEXT:    movd %xmm0, %ecx
+; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE3-SLOW-NEXT:    movd %xmm0, %eax
+; SSE3-SLOW-NEXT:    addl %ecx, %eax
+; SSE3-SLOW-NEXT:    retq
+;
+; SSE3-FAST-LABEL: extract_extract_v16i32_add_i32_commute:
+; SSE3-FAST:       # %bb.0:
+; SSE3-FAST-NEXT:    phaddd %xmm0, %xmm0
+; SSE3-FAST-NEXT:    movd %xmm0, %eax
+; SSE3-FAST-NEXT:    retq
+;
+; AVX-SLOW-LABEL: extract_extract_v16i32_add_i32_commute:
+; AVX-SLOW:       # %bb.0:
+; AVX-SLOW-NEXT:    vmovd %xmm0, %ecx
+; AVX-SLOW-NEXT:    vpextrd $1, %xmm0, %eax
+; AVX-SLOW-NEXT:    addl %ecx, %eax
+; AVX-SLOW-NEXT:    vzeroupper
+; AVX-SLOW-NEXT:    retq
+;
+; AVX-FAST-LABEL: extract_extract_v16i32_add_i32_commute:
+; AVX-FAST:       # %bb.0:
+; AVX-FAST-NEXT:    vphaddd %xmm0, %xmm0, %xmm0
+; AVX-FAST-NEXT:    vmovd %xmm0, %eax
+; AVX-FAST-NEXT:    vzeroupper
+; AVX-FAST-NEXT:    retq
+  %x0 = extractelement <16 x i32> %x, i32 0
+  %x1 = extractelement <16 x i32> %x, i32 1
+  %x01 = add i32 %x1, %x0
+  ret i32 %x01
+}
+
+define i16 @extract_extract_v32i16_add_i16(<32 x i16> %x) {
+; SSE3-SLOW-LABEL: extract_extract_v32i16_add_i16:
+; SSE3-SLOW:       # %bb.0:
+; SSE3-SLOW-NEXT:    movd %xmm0, %ecx
+; SSE3-SLOW-NEXT:    pextrw $1, %xmm0, %eax
+; SSE3-SLOW-NEXT:    addl %ecx, %eax
+; SSE3-SLOW-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE3-SLOW-NEXT:    retq
+;
+; SSE3-FAST-LABEL: extract_extract_v32i16_add_i16:
+; SSE3-FAST:       # %bb.0:
+; SSE3-FAST-NEXT:    phaddw %xmm0, %xmm0
+; SSE3-FAST-NEXT:    movd %xmm0, %eax
+; SSE3-FAST-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE3-FAST-NEXT:    retq
+;
+; AVX-SLOW-LABEL: extract_extract_v32i16_add_i16:
+; AVX-SLOW:       # %bb.0:
+; AVX-SLOW-NEXT:    vmovd %xmm0, %ecx
+; AVX-SLOW-NEXT:    vpextrw $1, %xmm0, %eax
+; AVX-SLOW-NEXT:    addl %ecx, %eax
+; AVX-SLOW-NEXT:    # kill: def $ax killed $ax killed $eax
+; AVX-SLOW-NEXT:    vzeroupper
+; AVX-SLOW-NEXT:    retq
+;
+; AVX-FAST-LABEL: extract_extract_v32i16_add_i16:
+; AVX-FAST:       # %bb.0:
+; AVX-FAST-NEXT:    vphaddw %xmm0, %xmm0, %xmm0
+; AVX-FAST-NEXT:    vmovd %xmm0, %eax
+; AVX-FAST-NEXT:    # kill: def $ax killed $ax killed $eax
+; AVX-FAST-NEXT:    vzeroupper
+; AVX-FAST-NEXT:    retq
+  %x0 = extractelement <32 x i16> %x, i32 0
+  %x1 = extractelement <32 x i16> %x, i32 1
+  %x01 = add i16 %x0, %x1
+  ret i16 %x01
+}
+
+define i16 @extract_extract_v32i16_add_i16_commute(<32 x i16> %x) {
+; SSE3-SLOW-LABEL: extract_extract_v32i16_add_i16_commute:
+; SSE3-SLOW:       # %bb.0:
+; SSE3-SLOW-NEXT:    movd %xmm0, %ecx
+; SSE3-SLOW-NEXT:    pextrw $1, %xmm0, %eax
+; SSE3-SLOW-NEXT:    addl %ecx, %eax
+; SSE3-SLOW-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE3-SLOW-NEXT:    retq
+;
+; SSE3-FAST-LABEL: extract_extract_v32i16_add_i16_commute:
+; SSE3-FAST:       # %bb.0:
+; SSE3-FAST-NEXT:    phaddw %xmm0, %xmm0
+; SSE3-FAST-NEXT:    movd %xmm0, %eax
+; SSE3-FAST-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE3-FAST-NEXT:    retq
+;
+; AVX-SLOW-LABEL: extract_extract_v32i16_add_i16_commute:
+; AVX-SLOW:       # %bb.0:
+; AVX-SLOW-NEXT:    vmovd %xmm0, %ecx
+; AVX-SLOW-NEXT:    vpextrw $1, %xmm0, %eax
+; AVX-SLOW-NEXT:    addl %ecx, %eax
+; AVX-SLOW-NEXT:    # kill: def $ax killed $ax killed $eax
+; AVX-SLOW-NEXT:    vzeroupper
+; AVX-SLOW-NEXT:    retq
+;
+; AVX-FAST-LABEL: extract_extract_v32i16_add_i16_commute:
+; AVX-FAST:       # %bb.0:
+; AVX-FAST-NEXT:    vphaddw %xmm0, %xmm0, %xmm0
+; AVX-FAST-NEXT:    vmovd %xmm0, %eax
+; AVX-FAST-NEXT:    # kill: def $ax killed $ax killed $eax
+; AVX-FAST-NEXT:    vzeroupper
+; AVX-FAST-NEXT:    retq
+  %x0 = extractelement <32 x i16> %x, i32 0
+  %x1 = extractelement <32 x i16> %x, i32 1
+  %x01 = add i16 %x1, %x0
+  ret i16 %x01
+}
+
+define i32 @extract_extract_v16i32_sub_i32(<16 x i32> %x) {
+; SSE3-SLOW-LABEL: extract_extract_v16i32_sub_i32:
+; SSE3-SLOW:       # %bb.0:
+; SSE3-SLOW-NEXT:    movd %xmm0, %eax
+; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE3-SLOW-NEXT:    movd %xmm0, %ecx
+; SSE3-SLOW-NEXT:    subl %ecx, %eax
+; SSE3-SLOW-NEXT:    retq
+;
+; SSE3-FAST-LABEL: extract_extract_v16i32_sub_i32:
+; SSE3-FAST:       # %bb.0:
+; SSE3-FAST-NEXT:    phsubd %xmm0, %xmm0
+; SSE3-FAST-NEXT:    movd %xmm0, %eax
+; SSE3-FAST-NEXT:    retq
+;
+; AVX-SLOW-LABEL: extract_extract_v16i32_sub_i32:
+; AVX-SLOW:       # %bb.0:
+; AVX-SLOW-NEXT:    vmovd %xmm0, %eax
+; AVX-SLOW-NEXT:    vpextrd $1, %xmm0, %ecx
+; AVX-SLOW-NEXT:    subl %ecx, %eax
+; AVX-SLOW-NEXT:    vzeroupper
+; AVX-SLOW-NEXT:    retq
+;
+; AVX-FAST-LABEL: extract_extract_v16i32_sub_i32:
+; AVX-FAST:       # %bb.0:
+; AVX-FAST-NEXT:    vphsubd %xmm0, %xmm0, %xmm0
+; AVX-FAST-NEXT:    vmovd %xmm0, %eax
+; AVX-FAST-NEXT:    vzeroupper
+; AVX-FAST-NEXT:    retq
+  %x0 = extractelement <16 x i32> %x, i32 0
+  %x1 = extractelement <16 x i32> %x, i32 1
+  %x01 = sub i32 %x0, %x1
+  ret i32 %x01
+}
+
+define i32 @extract_extract_v16i32_sub_i32_commute(<16 x i32> %x) {
+; SSE3-LABEL: extract_extract_v16i32_sub_i32_commute:
+; SSE3:       # %bb.0:
+; SSE3-NEXT:    movd %xmm0, %ecx
+; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE3-NEXT:    movd %xmm0, %eax
+; SSE3-NEXT:    subl %ecx, %eax
+; SSE3-NEXT:    retq
+;
+; AVX-LABEL: extract_extract_v16i32_sub_i32_commute:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovd %xmm0, %ecx
+; AVX-NEXT:    vpextrd $1, %xmm0, %eax
+; AVX-NEXT:    subl %ecx, %eax
+; AVX-NEXT:    vzeroupper
+; AVX-NEXT:    retq
+  %x0 = extractelement <16 x i32> %x, i32 0
+  %x1 = extractelement <16 x i32> %x, i32 1
+  %x01 = sub i32 %x1, %x0
+  ret i32 %x01
+}
+
+define i16 @extract_extract_v32i16_sub_i16(<32 x i16> %x) {
+; SSE3-SLOW-LABEL: extract_extract_v32i16_sub_i16:
+; SSE3-SLOW:       # %bb.0:
+; SSE3-SLOW-NEXT:    movd %xmm0, %eax
+; SSE3-SLOW-NEXT:    pextrw $1, %xmm0, %ecx
+; SSE3-SLOW-NEXT:    subl %ecx, %eax
+; SSE3-SLOW-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE3-SLOW-NEXT:    retq
+;
+; SSE3-FAST-LABEL: extract_extract_v32i16_sub_i16:
+; SSE3-FAST:       # %bb.0:
+; SSE3-FAST-NEXT:    phsubw %xmm0, %xmm0
+; SSE3-FAST-NEXT:    movd %xmm0, %eax
+; SSE3-FAST-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE3-FAST-NEXT:    retq
+;
+; AVX-SLOW-LABEL: extract_extract_v32i16_sub_i16:
+; AVX-SLOW:       # %bb.0:
+; AVX-SLOW-NEXT:    vmovd %xmm0, %eax
+; AVX-SLOW-NEXT:    vpextrw $1, %xmm0, %ecx
+; AVX-SLOW-NEXT:    subl %ecx, %eax
+; AVX-SLOW-NEXT:    # kill: def $ax killed $ax killed $eax
+; AVX-SLOW-NEXT:    vzeroupper
+; AVX-SLOW-NEXT:    retq
+;
+; AVX-FAST-LABEL: extract_extract_v32i16_sub_i16:
+; AVX-FAST:       # %bb.0:
+; AVX-FAST-NEXT:    vphsubw %xmm0, %xmm0, %xmm0
+; AVX-FAST-NEXT:    vmovd %xmm0, %eax
+; AVX-FAST-NEXT:    # kill: def $ax killed $ax killed $eax
+; AVX-FAST-NEXT:    vzeroupper
+; AVX-FAST-NEXT:    retq
+  %x0 = extractelement <32 x i16> %x, i32 0
+  %x1 = extractelement <32 x i16> %x, i32 1
+  %x01 = sub i16 %x0, %x1
+  ret i16 %x01
+}
+
+define i16 @extract_extract_v32i16_sub_i16_commute(<32 x i16> %x) {
+; SSE3-LABEL: extract_extract_v32i16_sub_i16_commute:
+; SSE3:       # %bb.0:
+; SSE3-NEXT:    movd %xmm0, %ecx
+; SSE3-NEXT:    pextrw $1, %xmm0, %eax
+; SSE3-NEXT:    subl %ecx, %eax
+; SSE3-NEXT:    # kill: def $ax killed $ax killed $eax
+; SSE3-NEXT:    retq
+;
+; AVX-LABEL: extract_extract_v32i16_sub_i16_commute:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovd %xmm0, %ecx
+; AVX-NEXT:    vpextrw $1, %xmm0, %eax
+; AVX-NEXT:    subl %ecx, %eax
+; AVX-NEXT:    # kill: def $ax killed $ax killed $eax
+; AVX-NEXT:    vzeroupper
+; AVX-NEXT:    retq
+  %x0 = extractelement <32 x i16> %x, i32 0
+  %x1 = extractelement <32 x i16> %x, i32 1
+  %x01 = sub i16 %x1, %x0
+  ret i16 %x01
+}
+
+; Check output when 1 or both extracts have extra uses.
+
+define i32 @extract_extract_v4i32_add_i32_uses1(<4 x i32> %x, i32* %p) {
+; SSE3-SLOW-LABEL: extract_extract_v4i32_add_i32_uses1:
+; SSE3-SLOW:       # %bb.0:
+; SSE3-SLOW-NEXT:    movd %xmm0, %ecx
+; SSE3-SLOW-NEXT:    movd %xmm0, (%rdi)
+; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE3-SLOW-NEXT:    movd %xmm0, %eax
+; SSE3-SLOW-NEXT:    addl %ecx, %eax
+; SSE3-SLOW-NEXT:    retq
+;
+; SSE3-FAST-LABEL: extract_extract_v4i32_add_i32_uses1:
+; SSE3-FAST:       # %bb.0:
+; SSE3-FAST-NEXT:    movd %xmm0, (%rdi)
+; SSE3-FAST-NEXT:    phaddd %xmm0, %xmm0
+; SSE3-FAST-NEXT:    movd %xmm0, %eax
+; SSE3-FAST-NEXT:    retq
+;
+; AVX-SLOW-LABEL: extract_extract_v4i32_add_i32_uses1:
+; AVX-SLOW:       # %bb.0:
+; AVX-SLOW-NEXT:    vmovd %xmm0, %ecx
+; AVX-SLOW-NEXT:    vmovd %xmm0, (%rdi)
+; AVX-SLOW-NEXT:    vpextrd $1, %xmm0, %eax
+; AVX-SLOW-NEXT:    addl %ecx, %eax
+; AVX-SLOW-NEXT:    retq
+;
+; AVX-FAST-LABEL: extract_extract_v4i32_add_i32_uses1:
+; AVX-FAST:       # %bb.0:
+; AVX-FAST-NEXT:    vmovd %xmm0, (%rdi)
+; AVX-FAST-NEXT:    vphaddd %xmm0, %xmm0, %xmm0
+; AVX-FAST-NEXT:    vmovd %xmm0, %eax
+; AVX-FAST-NEXT:    retq
+  %x0 = extractelement <4 x i32> %x, i32 0
+  store i32 %x0, i32* %p
+  %x1 = extractelement <4 x i32> %x, i32 1
+  %x01 = add i32 %x0, %x1
+  ret i32 %x01
+}
+
+define i32 @extract_extract_v4i32_add_i32_uses2(<4 x i32> %x, i32* %p) {
+; SSE3-SLOW-LABEL: extract_extract_v4i32_add_i32_uses2:
+; SSE3-SLOW:       # %bb.0:
+; SSE3-SLOW-NEXT:    movd %xmm0, %ecx
+; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE3-SLOW-NEXT:    movd %xmm0, %eax
+; SSE3-SLOW-NEXT:    addl %ecx, %eax
+; SSE3-SLOW-NEXT:    movd %xmm0, (%rdi)
+; SSE3-SLOW-NEXT:    retq
+;
+; SSE3-FAST-LABEL: extract_extract_v4i32_add_i32_uses2:
+; SSE3-FAST:       # %bb.0:
+; SSE3-FAST-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE3-FAST-NEXT:    movd %xmm1, (%rdi)
+; SSE3-FAST-NEXT:    phaddd %xmm0, %xmm0
+; SSE3-FAST-NEXT:    movd %xmm0, %eax
+; SSE3-FAST-NEXT:    retq
+;
+; AVX-SLOW-LABEL: extract_extract_v4i32_add_i32_uses2:
+; AVX-SLOW:       # %bb.0:
+; AVX-SLOW-NEXT:    vmovd %xmm0, %ecx
+; AVX-SLOW-NEXT:    vpextrd $1, %xmm0, %eax
+; AVX-SLOW-NEXT:    addl %ecx, %eax
+; AVX-SLOW-NEXT:    vpextrd $1, %xmm0, (%rdi)
+; AVX-SLOW-NEXT:    retq
+;
+; AVX-FAST-LABEL: extract_extract_v4i32_add_i32_uses2:
+; AVX-FAST:       # %bb.0:
+; AVX-FAST-NEXT:    vpextrd $1, %xmm0, (%rdi)
+; AVX-FAST-NEXT:    vphaddd %xmm0, %xmm0, %xmm0
+; AVX-FAST-NEXT:    vmovd %xmm0, %eax
+; AVX-FAST-NEXT:    retq
+  %x0 = extractelement <4 x i32> %x, i32 0
+  %x1 = extractelement <4 x i32> %x, i32 1
+  store i32 %x1, i32* %p
+  %x01 = add i32 %x0, %x1
+  ret i32 %x01
+}
+
+define i32 @extract_extract_v4i32_add_i32_uses3(<4 x i32> %x, i32* %p1, i32* %p2) {
+; SSE3-LABEL: extract_extract_v4i32_add_i32_uses3:
+; SSE3:       # %bb.0:
+; SSE3-NEXT:    movd %xmm0, %ecx
+; SSE3-NEXT:    movd %xmm0, (%rdi)
+; SSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE3-NEXT:    movd %xmm0, %eax
+; SSE3-NEXT:    addl %ecx, %eax
+; SSE3-NEXT:    movd %xmm0, (%rsi)
+; SSE3-NEXT:    retq
+;
+; AVX-LABEL: extract_extract_v4i32_add_i32_uses3:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovd %xmm0, %ecx
+; AVX-NEXT:    vmovd %xmm0, (%rdi)
+; AVX-NEXT:    vpextrd $1, %xmm0, %eax
+; AVX-NEXT:    addl %ecx, %eax
+; AVX-NEXT:    vpextrd $1, %xmm0, (%rsi)
+; AVX-NEXT:    retq
+  %x0 = extractelement <4 x i32> %x, i32 0
+  store i32 %x0, i32* %p1
+  %x1 = extractelement <4 x i32> %x, i32 1
+  store i32 %x1, i32* %p2
+  %x01 = add i32 %x0, %x1
+  ret i32 %x01
+}
+
+; PR33758: https://bugs.llvm.org/show_bug.cgi?id=33758
+
+define i32 @partial_reduction_add_v8i32(<8 x i32> %x) {
+; SSE3-SLOW-LABEL: partial_reduction_add_v8i32:
+; SSE3-SLOW:       # %bb.0:
+; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE3-SLOW-NEXT:    paddd %xmm0, %xmm1
+; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE3-SLOW-NEXT:    paddd %xmm1, %xmm0
+; SSE3-SLOW-NEXT:    movd %xmm0, %eax
+; SSE3-SLOW-NEXT:    retq
+;
+; SSE3-FAST-LABEL: partial_reduction_add_v8i32:
+; SSE3-FAST:       # %bb.0:
+; SSE3-FAST-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE3-FAST-NEXT:    paddd %xmm0, %xmm1
+; SSE3-FAST-NEXT:    phaddd %xmm1, %xmm1
+; SSE3-FAST-NEXT:    movd %xmm1, %eax
+; SSE3-FAST-NEXT:    retq
+;
+; AVX-SLOW-LABEL: partial_reduction_add_v8i32:
+; AVX-SLOW:       # %bb.0:
+; AVX-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX-SLOW-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
+; AVX-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX-SLOW-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
+; AVX-SLOW-NEXT:    vmovd %xmm0, %eax
+; AVX-SLOW-NEXT:    vzeroupper
+; AVX-SLOW-NEXT:    retq
+;
+; AVX-FAST-LABEL: partial_reduction_add_v8i32:
+; AVX-FAST:       # %bb.0:
+; AVX-FAST-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX-FAST-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
+; AVX-FAST-NEXT:    vphaddd %xmm0, %xmm0, %xmm0
+; AVX-FAST-NEXT:    vmovd %xmm0, %eax
+; AVX-FAST-NEXT:    vzeroupper
+; AVX-FAST-NEXT:    retq
+  %x23 = shufflevector <8 x i32> %x, <8 x i32> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %x0213 = add <8 x i32> %x, %x23
+  %x13 = shufflevector <8 x i32> %x0213, <8 x i32> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %x0123 = add <8 x i32> %x0213, %x13
+  %r = extractelement <8 x i32> %x0123, i32 0
+  ret i32 %r
+}
+
+define i32 @partial_reduction_add_v16i32(<16 x i32> %x) {
+; SSE3-SLOW-LABEL: partial_reduction_add_v16i32:
+; SSE3-SLOW:       # %bb.0:
+; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE3-SLOW-NEXT:    paddd %xmm0, %xmm1
+; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE3-SLOW-NEXT:    paddd %xmm1, %xmm0
+; SSE3-SLOW-NEXT:    movd %xmm0, %eax
+; SSE3-SLOW-NEXT:    retq
+;
+; SSE3-FAST-LABEL: partial_reduction_add_v16i32:
+; SSE3-FAST:       # %bb.0:
+; SSE3-FAST-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE3-FAST-NEXT:    paddd %xmm0, %xmm1
+; SSE3-FAST-NEXT:    phaddd %xmm1, %xmm1
+; SSE3-FAST-NEXT:    movd %xmm1, %eax
+; SSE3-FAST-NEXT:    retq
+;
+; AVX-SLOW-LABEL: partial_reduction_add_v16i32:
+; AVX-SLOW:       # %bb.0:
+; AVX-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX-SLOW-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
+; AVX-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX-SLOW-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
+; AVX-SLOW-NEXT:    vmovd %xmm0, %eax
+; AVX-SLOW-NEXT:    vzeroupper
+; AVX-SLOW-NEXT:    retq
+;
+; AVX1-FAST-LABEL: partial_reduction_add_v16i32:
+; AVX1-FAST:       # %bb.0:
+; AVX1-FAST-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-FAST-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
+; AVX1-FAST-NEXT:    vphaddd %xmm0, %xmm0, %xmm0
+; AVX1-FAST-NEXT:    vmovd %xmm0, %eax
+; AVX1-FAST-NEXT:    vzeroupper
+; AVX1-FAST-NEXT:    retq
+;
+; AVX2-FAST-LABEL: partial_reduction_add_v16i32:
+; AVX2-FAST:       # %bb.0:
+; AVX2-FAST-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX2-FAST-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
+; AVX2-FAST-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX2-FAST-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
+; AVX2-FAST-NEXT:    vmovd %xmm0, %eax
+; AVX2-FAST-NEXT:    vzeroupper
+; AVX2-FAST-NEXT:    retq
+;
+; AVX512-FAST-LABEL: partial_reduction_add_v16i32:
+; AVX512-FAST:       # %bb.0:
+; AVX512-FAST-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512-FAST-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
+; AVX512-FAST-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX512-FAST-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
+; AVX512-FAST-NEXT:    vmovd %xmm0, %eax
+; AVX512-FAST-NEXT:    vzeroupper
+; AVX512-FAST-NEXT:    retq
+  %x23 = shufflevector <16 x i32> %x, <16 x i32> undef, <16 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %x0213 = add <16 x i32> %x, %x23
+  %x13 = shufflevector <16 x i32> %x0213, <16 x i32> undef, <16 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %x0123 = add <16 x i32> %x0213, %x13
+  %r = extractelement <16 x i32> %x0123, i32 0
+  ret i32 %r
+}
+
+define i32 @partial_reduction_sub_v8i32(<8 x i32> %x) {
+; SSE3-SLOW-LABEL: partial_reduction_sub_v8i32:
+; SSE3-SLOW:       # %bb.0:
+; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE3-SLOW-NEXT:    psubd %xmm1, %xmm0
+; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE3-SLOW-NEXT:    psubd %xmm1, %xmm0
+; SSE3-SLOW-NEXT:    movd %xmm0, %eax
+; SSE3-SLOW-NEXT:    retq
+;
+; SSE3-FAST-LABEL: partial_reduction_sub_v8i32:
+; SSE3-FAST:       # %bb.0:
+; SSE3-FAST-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE3-FAST-NEXT:    psubd %xmm1, %xmm0
+; SSE3-FAST-NEXT:    phsubd %xmm0, %xmm0
+; SSE3-FAST-NEXT:    movd %xmm0, %eax
+; SSE3-FAST-NEXT:    retq
+;
+; AVX-SLOW-LABEL: partial_reduction_sub_v8i32:
+; AVX-SLOW:       # %bb.0:
+; AVX-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX-SLOW-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
+; AVX-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX-SLOW-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
+; AVX-SLOW-NEXT:    vmovd %xmm0, %eax
+; AVX-SLOW-NEXT:    vzeroupper
+; AVX-SLOW-NEXT:    retq
+;
+; AVX-FAST-LABEL: partial_reduction_sub_v8i32:
+; AVX-FAST:       # %bb.0:
+; AVX-FAST-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX-FAST-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
+; AVX-FAST-NEXT:    vphsubd %xmm0, %xmm0, %xmm0
+; AVX-FAST-NEXT:    vmovd %xmm0, %eax
+; AVX-FAST-NEXT:    vzeroupper
+; AVX-FAST-NEXT:    retq
+  %x23 = shufflevector <8 x i32> %x, <8 x i32> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %x0213 = sub <8 x i32> %x, %x23
+  %x13 = shufflevector <8 x i32> %x0213, <8 x i32> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %x0123 = sub <8 x i32> %x0213, %x13
+  %r = extractelement <8 x i32> %x0123, i32 0
+  ret i32 %r
+}
+
+define i32 @partial_reduction_sub_v16i32(<16 x i32> %x) {
+; SSE3-SLOW-LABEL: partial_reduction_sub_v16i32:
+; SSE3-SLOW:       # %bb.0:
+; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE3-SLOW-NEXT:    psubd %xmm1, %xmm0
+; SSE3-SLOW-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; SSE3-SLOW-NEXT:    psubd %xmm1, %xmm0
+; SSE3-SLOW-NEXT:    movd %xmm0, %eax
+; SSE3-SLOW-NEXT:    retq
+;
+; SSE3-FAST-LABEL: partial_reduction_sub_v16i32:
+; SSE3-FAST:       # %bb.0:
+; SSE3-FAST-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; SSE3-FAST-NEXT:    psubd %xmm1, %xmm0
+; SSE3-FAST-NEXT:    phsubd %xmm0, %xmm0
+; SSE3-FAST-NEXT:    movd %xmm0, %eax
+; SSE3-FAST-NEXT:    retq
+;
+; AVX-SLOW-LABEL: partial_reduction_sub_v16i32:
+; AVX-SLOW:       # %bb.0:
+; AVX-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX-SLOW-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
+; AVX-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX-SLOW-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
+; AVX-SLOW-NEXT:    vmovd %xmm0, %eax
+; AVX-SLOW-NEXT:    vzeroupper
+; AVX-SLOW-NEXT:    retq
+;
+; AVX1-FAST-LABEL: partial_reduction_sub_v16i32:
+; AVX1-FAST:       # %bb.0:
+; AVX1-FAST-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-FAST-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
+; AVX1-FAST-NEXT:    vphsubd %xmm0, %xmm0, %xmm0
+; AVX1-FAST-NEXT:    vmovd %xmm0, %eax
+; AVX1-FAST-NEXT:    vzeroupper
+; AVX1-FAST-NEXT:    retq
+;
+; AVX2-FAST-LABEL: partial_reduction_sub_v16i32:
+; AVX2-FAST:       # %bb.0:
+; AVX2-FAST-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX2-FAST-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
+; AVX2-FAST-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX2-FAST-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
+; AVX2-FAST-NEXT:    vmovd %xmm0, %eax
+; AVX2-FAST-NEXT:    vzeroupper
+; AVX2-FAST-NEXT:    retq
+;
+; AVX512-FAST-LABEL: partial_reduction_sub_v16i32:
+; AVX512-FAST:       # %bb.0:
+; AVX512-FAST-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512-FAST-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
+; AVX512-FAST-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX512-FAST-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
+; AVX512-FAST-NEXT:    vmovd %xmm0, %eax
+; AVX512-FAST-NEXT:    vzeroupper
+; AVX512-FAST-NEXT:    retq
+  %x23 = shufflevector <16 x i32> %x, <16 x i32> undef, <16 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %x0213 = sub <16 x i32> %x, %x23
+  %x13 = shufflevector <16 x i32> %x0213, <16 x i32> undef, <16 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %x0123 = sub <16 x i32> %x0213, %x13
+  %r = extractelement <16 x i32> %x0123, i32 0
+  ret i32 %r
+}
+
diff --git a/test/CodeGen/X86/phaddsub-undef.ll b/test/CodeGen/X86/phaddsub-undef.ll
new file mode 100644
index 0000000000000000000000000000000000000000..6fffbefa4274a27f76a672711488c3664102ef49
--- /dev/null
+++ b/test/CodeGen/X86/phaddsub-undef.ll
@@ -0,0 +1,291 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+ssse3              | FileCheck %s --check-prefixes=SSE,SSE-SLOW
+; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+ssse3,fast-hops    | FileCheck %s --check-prefixes=SSE,SSE-FAST
+; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx                | FileCheck %s --check-prefixes=AVX,AVX-SLOW,AVX1,AVX1-SLOW
+; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx,fast-hops      | FileCheck %s --check-prefixes=AVX,AVX-FAST,AVX1,AVX1-FAST
+; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx2               | FileCheck %s --check-prefixes=AVX,AVX-SLOW,AVX2,AVX2-SLOW
+; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx2,fast-hops     | FileCheck %s --check-prefixes=AVX,AVX-FAST,AVX2,AVX2-FAST
+; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx512vl           | FileCheck %s --check-prefixes=AVX,AVX-SLOW,AVX512,AVX512-SLOW
+; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx512vl,fast-hops | FileCheck %s --check-prefixes=AVX,AVX-FAST,AVX512,AVX512-FAST
+
+; Verify that we correctly fold horizontal binop even in the presence of UNDEFs.
+
+define <8 x i32> @test14_undef(<8 x i32> %a, <8 x i32> %b) {
+; SSE-LABEL: test14_undef:
+; SSE:       # %bb.0:
+; SSE-NEXT:    phaddd %xmm2, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX1-LABEL: test14_undef:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vphaddd %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: test14_undef:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vphaddd %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: test14_undef:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vphaddd %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    retq
+  %vecext = extractelement <8 x i32> %a, i32 0
+  %vecext1 = extractelement <8 x i32> %a, i32 1
+  %add = add i32 %vecext, %vecext1
+  %vecinit = insertelement <8 x i32> undef, i32 %add, i32 0
+  %vecext2 = extractelement <8 x i32> %b, i32 2
+  %vecext3 = extractelement <8 x i32> %b, i32 3
+  %add4 = add i32 %vecext2, %vecext3
+  %vecinit5 = insertelement <8 x i32> %vecinit, i32 %add4, i32 3
+  ret <8 x i32> %vecinit5
+}
+
+; integer horizontal adds instead of two scalar adds followed by vector inserts.
+define <8 x i32> @test15_undef(<8 x i32> %a, <8 x i32> %b) {
+; SSE-SLOW-LABEL: test15_undef:
+; SSE-SLOW:       # %bb.0:
+; SSE-SLOW-NEXT:    movd %xmm0, %eax
+; SSE-SLOW-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE-SLOW-NEXT:    movd %xmm0, %ecx
+; SSE-SLOW-NEXT:    addl %eax, %ecx
+; SSE-SLOW-NEXT:    movd %xmm3, %eax
+; SSE-SLOW-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,2,3]
+; SSE-SLOW-NEXT:    movd %xmm0, %edx
+; SSE-SLOW-NEXT:    addl %eax, %edx
+; SSE-SLOW-NEXT:    movd %ecx, %xmm0
+; SSE-SLOW-NEXT:    movd %edx, %xmm1
+; SSE-SLOW-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,1,0,1]
+; SSE-SLOW-NEXT:    retq
+;
+; SSE-FAST-LABEL: test15_undef:
+; SSE-FAST:       # %bb.0:
+; SSE-FAST-NEXT:    phaddd %xmm0, %xmm0
+; SSE-FAST-NEXT:    phaddd %xmm3, %xmm3
+; SSE-FAST-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[0,1,0,1]
+; SSE-FAST-NEXT:    retq
+;
+; AVX1-SLOW-LABEL: test15_undef:
+; AVX1-SLOW:       # %bb.0:
+; AVX1-SLOW-NEXT:    vmovd %xmm0, %eax
+; AVX1-SLOW-NEXT:    vpextrd $1, %xmm0, %ecx
+; AVX1-SLOW-NEXT:    addl %eax, %ecx
+; AVX1-SLOW-NEXT:    vextractf128 $1, %ymm1, %xmm0
+; AVX1-SLOW-NEXT:    vmovd %xmm0, %eax
+; AVX1-SLOW-NEXT:    vpextrd $1, %xmm0, %edx
+; AVX1-SLOW-NEXT:    addl %eax, %edx
+; AVX1-SLOW-NEXT:    vmovd %ecx, %xmm0
+; AVX1-SLOW-NEXT:    vmovd %edx, %xmm1
+; AVX1-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,1,0,1]
+; AVX1-SLOW-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-SLOW-NEXT:    retq
+;
+; AVX1-FAST-LABEL: test15_undef:
+; AVX1-FAST:       # %bb.0:
+; AVX1-FAST-NEXT:    vphaddd %xmm0, %xmm0, %xmm0
+; AVX1-FAST-NEXT:    vextractf128 $1, %ymm1, %xmm1
+; AVX1-FAST-NEXT:    vphaddd %xmm1, %xmm1, %xmm1
+; AVX1-FAST-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,1,0,1]
+; AVX1-FAST-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-FAST-NEXT:    retq
+;
+; AVX2-LABEL: test15_undef:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vphaddd %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: test15_undef:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vphaddd %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    retq
+  %vecext = extractelement <8 x i32> %a, i32 0
+  %vecext1 = extractelement <8 x i32> %a, i32 1
+  %add = add i32 %vecext, %vecext1
+  %vecinit = insertelement <8 x i32> undef, i32 %add, i32 0
+  %vecext2 = extractelement <8 x i32> %b, i32 4
+  %vecext3 = extractelement <8 x i32> %b, i32 5
+  %add4 = add i32 %vecext2, %vecext3
+  %vecinit5 = insertelement <8 x i32> %vecinit, i32 %add4, i32 6
+  ret <8 x i32> %vecinit5
+}
+
+define <8 x i32> @PR40243_alt(<8 x i32> %a, <8 x i32> %b) {
+; SSE-LABEL: PR40243_alt:
+; SSE:       # %bb.0:
+; SSE-NEXT:    phaddd %xmm3, %xmm1
+; SSE-NEXT:    retq
+;
+; AVX1-LABEL: PR40243_alt:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm1
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; AVX1-NEXT:    vphaddd %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: PR40243_alt:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vphaddd %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: PR40243_alt:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vphaddd %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    retq
+  %a4 = extractelement <8 x i32> %a, i32 4
+  %a5 = extractelement <8 x i32> %a, i32 5
+  %add4 = add i32 %a4, %a5
+  %b6 = extractelement <8 x i32> %b, i32 6
+  %b7 = extractelement <8 x i32> %b, i32 7
+  %add7 = add i32 %b6, %b7
+  %r4 = insertelement <8 x i32> undef, i32 %add4, i32 4
+  %r = insertelement <8 x i32> %r4, i32 %add7, i32 7
+  ret <8 x i32> %r
+}
+
+define <8 x i32> @test16_undef(<8 x i32> %a, <8 x i32> %b) {
+; SSE-LABEL: test16_undef:
+; SSE:       # %bb.0:
+; SSE-NEXT:    phaddd %xmm0, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX1-LABEL: test16_undef:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vphaddd %xmm0, %xmm0, %xmm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: test16_undef:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vphaddd %ymm0, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: test16_undef:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vphaddd %ymm0, %ymm0, %ymm0
+; AVX512-NEXT:    retq
+  %vecext = extractelement <8 x i32> %a, i32 0
+  %vecext1 = extractelement <8 x i32> %a, i32 1
+  %add = add i32 %vecext, %vecext1
+  %vecinit = insertelement <8 x i32> undef, i32 %add, i32 0
+  %vecext2 = extractelement <8 x i32> %a, i32 2
+  %vecext3 = extractelement <8 x i32> %a, i32 3
+  %add4 = add i32 %vecext2, %vecext3
+  %vecinit5 = insertelement <8 x i32> %vecinit, i32 %add4, i32 1
+  ret <8 x i32> %vecinit5
+}
+
+define <16 x i32> @test16_v16i32_undef(<16 x i32> %a, <16 x i32> %b) {
+; SSE-LABEL: test16_v16i32_undef:
+; SSE:       # %bb.0:
+; SSE-NEXT:    phaddd %xmm0, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX1-LABEL: test16_v16i32_undef:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vphaddd %xmm0, %xmm0, %xmm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: test16_v16i32_undef:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vphaddd %ymm0, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: test16_v16i32_undef:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vphaddd %ymm0, %ymm0, %ymm0
+; AVX512-NEXT:    retq
+  %vecext = extractelement <16 x i32> %a, i32 0
+  %vecext1 = extractelement <16 x i32> %a, i32 1
+  %add = add i32 %vecext, %vecext1
+  %vecinit = insertelement <16 x i32> undef, i32 %add, i32 0
+  %vecext2 = extractelement <16 x i32> %a, i32 2
+  %vecext3 = extractelement <16 x i32> %a, i32 3
+  %add4 = add i32 %vecext2, %vecext3
+  %vecinit5 = insertelement <16 x i32> %vecinit, i32 %add4, i32 1
+  ret <16 x i32> %vecinit5
+}
+
+define <8 x i32> @test17_undef(<8 x i32> %a, <8 x i32> %b) {
+; SSE-LABEL: test17_undef:
+; SSE:       # %bb.0:
+; SSE-NEXT:    phaddd %xmm1, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX1-LABEL: test17_undef:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX1-NEXT:    vphaddd %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: test17_undef:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT:    vphaddd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: test17_undef:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX512-NEXT:    vphaddd %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    retq
+  %vecext = extractelement <8 x i32> %a, i32 0
+  %vecext1 = extractelement <8 x i32> %a, i32 1
+  %add1 = add i32 %vecext, %vecext1
+  %vecinit1 = insertelement <8 x i32> undef, i32 %add1, i32 0
+  %vecext2 = extractelement <8 x i32> %a, i32 2
+  %vecext3 = extractelement <8 x i32> %a, i32 3
+  %add2 = add i32 %vecext2, %vecext3
+  %vecinit2 = insertelement <8 x i32> %vecinit1, i32 %add2, i32 1
+  %vecext4 = extractelement <8 x i32> %a, i32 4
+  %vecext5 = extractelement <8 x i32> %a, i32 5
+  %add3 = add i32 %vecext4, %vecext5
+  %vecinit3 = insertelement <8 x i32> %vecinit2, i32 %add3, i32 2
+  %vecext6 = extractelement <8 x i32> %a, i32 6
+  %vecext7 = extractelement <8 x i32> %a, i32 7
+  %add4 = add i32 %vecext6, %vecext7
+  %vecinit4 = insertelement <8 x i32> %vecinit3, i32 %add4, i32 3
+  ret <8 x i32> %vecinit4
+}
+
+define <16 x i32> @test17_v16i32_undef(<16 x i32> %a, <16 x i32> %b) {
+; SSE-LABEL: test17_v16i32_undef:
+; SSE:       # %bb.0:
+; SSE-NEXT:    phaddd %xmm1, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX1-LABEL: test17_v16i32_undef:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX1-NEXT:    vphaddd %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: test17_v16i32_undef:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT:    vphaddd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: test17_v16i32_undef:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX512-NEXT:    vphaddd %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    retq
+  %vecext = extractelement <16 x i32> %a, i32 0
+  %vecext1 = extractelement <16 x i32> %a, i32 1
+  %add1 = add i32 %vecext, %vecext1
+  %vecinit1 = insertelement <16 x i32> undef, i32 %add1, i32 0
+  %vecext2 = extractelement <16 x i32> %a, i32 2
+  %vecext3 = extractelement <16 x i32> %a, i32 3
+  %add2 = add i32 %vecext2, %vecext3
+  %vecinit2 = insertelement <16 x i32> %vecinit1, i32 %add2, i32 1
+  %vecext4 = extractelement <16 x i32> %a, i32 4
+  %vecext5 = extractelement <16 x i32> %a, i32 5
+  %add3 = add i32 %vecext4, %vecext5
+  %vecinit3 = insertelement <16 x i32> %vecinit2, i32 %add3, i32 2
+  %vecext6 = extractelement <16 x i32> %a, i32 6
+  %vecext7 = extractelement <16 x i32> %a, i32 7
+  %add4 = add i32 %vecext6, %vecext7
+  %vecinit4 = insertelement <16 x i32> %vecinit3, i32 %add4, i32 3
+  ret <16 x i32> %vecinit4
+}
+
diff --git a/test/CodeGen/X86/pic-load-remat.ll b/test/CodeGen/X86/pic-load-remat.ll
index 77297521cd0d864e0415aa2e61b6893852034063..61849c895a1cad23e2e0fd2d1bdd44b82309c443 100644
--- a/test/CodeGen/X86/pic-load-remat.ll
+++ b/test/CodeGen/X86/pic-load-remat.ll
@@ -5,30 +5,30 @@ entry:
 	br label %bb
 
 bb:		; preds = %bb, %entry
-	%tmp4403 = tail call <8 x i16> @llvm.x86.sse2.psubs.w( <8 x i16> zeroinitializer, <8 x i16> zeroinitializer ) nounwind readnone 		; <<8 x i16>> [#uses=2]
-	%tmp4443 = tail call <8 x i16> @llvm.x86.sse2.padds.w( <8 x i16> zeroinitializer, <8 x i16> zeroinitializer ) nounwind readnone 		; <<8 x i16>> [#uses=1]
+	%tmp4403 = tail call <8 x i16> @llvm.ssub.sat.v8i16( <8 x i16> zeroinitializer, <8 x i16> zeroinitializer ) nounwind readnone 		; <<8 x i16>> [#uses=2]
+	%tmp4443 = tail call <8 x i16> @llvm.sadd.sat.v8i16( <8 x i16> zeroinitializer, <8 x i16> zeroinitializer ) nounwind readnone 		; <<8 x i16>> [#uses=1]
 	%tmp4609 = tail call <8 x i16> @llvm.x86.sse2.psll.w( <8 x i16> zeroinitializer, <8 x i16> bitcast (<4 x i32> < i32 3, i32 5, i32 6, i32 9 > to <8 x i16>) )		; <<8 x i16>> [#uses=1]
 	%tmp4651 = add <8 x i16> %tmp4609, < i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1 >		; <<8 x i16>> [#uses=1]
 	%tmp4658 = tail call <8 x i16> @llvm.x86.sse2.psll.w( <8 x i16> %tmp4651, <8 x i16> bitcast (<4 x i32> < i32 4, i32 1, i32 2, i32 3 > to <8 x i16>) )		; <<8 x i16>> [#uses=1]
 	%tmp4669 = tail call <8 x i16> @llvm.x86.sse2.pavg.w( <8 x i16> < i16 -23170, i16 -23170, i16 -23170, i16 -23170, i16 -23170, i16 -23170, i16 -23170, i16 -23170 >, <8 x i16> %tmp4443 ) nounwind readnone 		; <<8 x i16>> [#uses=2]
-	%tmp4679 = tail call <8 x i16> @llvm.x86.sse2.padds.w( <8 x i16> %tmp4669, <8 x i16> %tmp4669 ) nounwind readnone 		; <<8 x i16>> [#uses=1]
+	%tmp4679 = tail call <8 x i16> @llvm.sadd.sat.v8i16( <8 x i16> %tmp4669, <8 x i16> %tmp4669 ) nounwind readnone 		; <<8 x i16>> [#uses=1]
 	%tmp4689 = add <8 x i16> %tmp4679, %tmp4658		; <<8 x i16>> [#uses=1]
-	%tmp4700 = tail call <8 x i16> @llvm.x86.sse2.padds.w( <8 x i16> %tmp4689, <8 x i16> zeroinitializer ) nounwind readnone 		; <<8 x i16>> [#uses=1]
+	%tmp4700 = tail call <8 x i16> @llvm.sadd.sat.v8i16( <8 x i16> %tmp4689, <8 x i16> zeroinitializer ) nounwind readnone 		; <<8 x i16>> [#uses=1]
 	%tmp4708 = bitcast <8 x i16> %tmp4700 to <2 x i64>		; <<2 x i64>> [#uses=1]
 	%tmp4772 = add <8 x i16> zeroinitializer, < i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1 >		; <<8 x i16>> [#uses=1]
 	%tmp4779 = tail call <8 x i16> @llvm.x86.sse2.psll.w( <8 x i16> %tmp4772, <8 x i16> bitcast (<4 x i32> < i32 3, i32 5, i32 undef, i32 7 > to <8 x i16>) )		; <<8 x i16>> [#uses=1]
 	%tmp4810 = add <8 x i16> zeroinitializer, %tmp4779		; <<8 x i16>> [#uses=1]
-	%tmp4821 = tail call <8 x i16> @llvm.x86.sse2.padds.w( <8 x i16> %tmp4810, <8 x i16> zeroinitializer ) nounwind readnone 		; <<8 x i16>> [#uses=1]
+	%tmp4821 = tail call <8 x i16> @llvm.sadd.sat.v8i16( <8 x i16> %tmp4810, <8 x i16> zeroinitializer ) nounwind readnone 		; <<8 x i16>> [#uses=1]
 	%tmp4829 = bitcast <8 x i16> %tmp4821 to <2 x i64>		; <<2 x i64>> [#uses=1]
 	%tmp4900 = tail call <8 x i16> @llvm.x86.sse2.psll.w( <8 x i16> zeroinitializer, <8 x i16> bitcast (<4 x i32> < i32 1, i32 1, i32 2, i32 2 > to <8 x i16>) )		; <<8 x i16>> [#uses=1]
 	%tmp4911 = tail call <8 x i16> @llvm.x86.sse2.pavg.w( <8 x i16> < i16 -23170, i16 -23170, i16 -23170, i16 -23170, i16 -23170, i16 -23170, i16 -23170, i16 -23170 >, <8 x i16> zeroinitializer ) nounwind readnone 		; <<8 x i16>> [#uses=2]
-	%tmp4921 = tail call <8 x i16> @llvm.x86.sse2.padds.w( <8 x i16> %tmp4911, <8 x i16> %tmp4911 ) nounwind readnone 		; <<8 x i16>> [#uses=1]
+	%tmp4921 = tail call <8 x i16> @llvm.sadd.sat.v8i16( <8 x i16> %tmp4911, <8 x i16> %tmp4911 ) nounwind readnone 		; <<8 x i16>> [#uses=1]
 	%tmp4931 = add <8 x i16> %tmp4921, %tmp4900		; <<8 x i16>> [#uses=1]
-	%tmp4942 = tail call <8 x i16> @llvm.x86.sse2.padds.w( <8 x i16> %tmp4931, <8 x i16> zeroinitializer ) nounwind readnone 		; <<8 x i16>> [#uses=1]
+	%tmp4942 = tail call <8 x i16> @llvm.sadd.sat.v8i16( <8 x i16> %tmp4931, <8 x i16> zeroinitializer ) nounwind readnone 		; <<8 x i16>> [#uses=1]
 	%tmp4950 = bitcast <8 x i16> %tmp4942 to <2 x i64>		; <<2 x i64>> [#uses=1]
-	%tmp4957 = tail call <8 x i16> @llvm.x86.sse2.padds.w( <8 x i16> %tmp4403, <8 x i16> zeroinitializer ) nounwind readnone 		; <<8 x i16>> [#uses=1]
+	%tmp4957 = tail call <8 x i16> @llvm.sadd.sat.v8i16( <8 x i16> %tmp4403, <8 x i16> zeroinitializer ) nounwind readnone 		; <<8 x i16>> [#uses=1]
 	%tmp4958 = bitcast <8 x i16> %tmp4957 to <2 x i64>		; <<2 x i64>> [#uses=1]
-	%tmp4967 = tail call <8 x i16> @llvm.x86.sse2.psubs.w( <8 x i16> %tmp4403, <8 x i16> zeroinitializer ) nounwind readnone 		; <<8 x i16>> [#uses=1]
+	%tmp4967 = tail call <8 x i16> @llvm.ssub.sat.v8i16( <8 x i16> %tmp4403, <8 x i16> zeroinitializer ) nounwind readnone 		; <<8 x i16>> [#uses=1]
 	%tmp4968 = bitcast <8 x i16> %tmp4967 to <2 x i64>		; <<2 x i64>> [#uses=1]
 	store <2 x i64> %tmp4829, <2 x i64>* null, align 16
 	store <2 x i64> %tmp4958, <2 x i64>* null, align 16
@@ -42,6 +42,6 @@ declare <8 x i16> @llvm.x86.sse2.psll.w(<8 x i16>, <8 x i16>) nounwind readnone
 
 declare <8 x i16> @llvm.x86.sse2.pavg.w(<8 x i16>, <8 x i16>) nounwind readnone 
 
-declare <8 x i16> @llvm.x86.sse2.padds.w(<8 x i16>, <8 x i16>) nounwind readnone 
+declare <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16>, <8 x i16>) nounwind readnone 
 
-declare <8 x i16> @llvm.x86.sse2.psubs.w(<8 x i16>, <8 x i16>) nounwind readnone 
+declare <8 x i16> @llvm.ssub.sat.v8i16(<8 x i16>, <8 x i16>) nounwind readnone 
diff --git a/test/CodeGen/X86/pmovsx-inreg.ll b/test/CodeGen/X86/pmovsx-inreg.ll
index ea9e296d619382c62409709d2e88543f148216f4..9ab6917966b384cc258e1f333675d85994b725d5 100644
--- a/test/CodeGen/X86/pmovsx-inreg.ll
+++ b/test/CodeGen/X86/pmovsx-inreg.ll
@@ -53,10 +53,8 @@ define void @test2(<4 x i8>* %in, <4 x i64>* %out) nounwind {
 ;
 ; AVX1-LABEL: test2:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpmovsxbd (%rdi), %xmm0
-; AVX1-NEXT:    vpmovsxdq %xmm0, %xmm1
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; AVX1-NEXT:    vpmovsxdq %xmm0, %xmm0
+; AVX1-NEXT:    vpmovsxbq 2(%rdi), %xmm0
+; AVX1-NEXT:    vpmovsxbq (%rdi), %xmm1
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
 ; AVX1-NEXT:    vmovdqu %ymm1, (%rax)
@@ -136,10 +134,8 @@ define void @test4(<8 x i8>* %in, <8 x i32>* %out) nounwind {
 ;
 ; AVX1-LABEL: test4:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpmovsxbw (%rdi), %xmm0
-; AVX1-NEXT:    vpmovsxwd %xmm0, %xmm1
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; AVX1-NEXT:    vpmovsxwd %xmm0, %xmm0
+; AVX1-NEXT:    vpmovsxbd 4(%rdi), %xmm0
+; AVX1-NEXT:    vpmovsxbd (%rdi), %xmm1
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
 ; AVX1-NEXT:    vmovdqu %ymm1, (%rax)
@@ -300,10 +296,8 @@ define void @test8(<4 x i16>* %in, <4 x i64>* %out) nounwind {
 ;
 ; AVX1-LABEL: test8:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpmovsxwd (%rdi), %xmm0
-; AVX1-NEXT:    vpmovsxdq %xmm0, %xmm1
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; AVX1-NEXT:    vpmovsxdq %xmm0, %xmm0
+; AVX1-NEXT:    vpmovsxwq 4(%rdi), %xmm0
+; AVX1-NEXT:    vpmovsxwq (%rdi), %xmm1
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
 ; AVX1-NEXT:    vmovdqu %ymm1, (%rax)
diff --git a/test/CodeGen/X86/pmul.ll b/test/CodeGen/X86/pmul.ll
index 2d09877155093ad339f6d4cb2b41a3dc266cb70e..011ca1240c93921ab574b60d2475356fe28dc5be 100644
--- a/test/CodeGen/X86/pmul.ll
+++ b/test/CodeGen/X86/pmul.ll
@@ -1251,76 +1251,70 @@ entry:
 define <8 x i64> @mul_v8i64_sext(<8 x i16> %val1, <8 x i32> %val2) {
 ; SSE2-LABEL: mul_v8i64_sext:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    movdqa %xmm1, %xmm4
-; SSE2-NEXT:    movdqa %xmm0, %xmm5
-; SSE2-NEXT:    punpckhwd {{.*#+}} xmm7 = xmm7[4],xmm0[4],xmm7[5],xmm0[5],xmm7[6],xmm0[6],xmm7[7],xmm0[7]
+; SSE2-NEXT:    movdqa %xmm1, %xmm12
+; SSE2-NEXT:    punpckhwd {{.*#+}} xmm10 = xmm10[4],xmm0[4],xmm10[5],xmm0[5],xmm10[6],xmm0[6],xmm10[7],xmm0[7]
+; SSE2-NEXT:    psrad $16, %xmm10
 ; SSE2-NEXT:    pxor %xmm8, %xmm8
+; SSE2-NEXT:    pxor %xmm13, %xmm13
+; SSE2-NEXT:    pcmpgtd %xmm10, %xmm13
+; SSE2-NEXT:    movdqa %xmm10, %xmm9
+; SSE2-NEXT:    punpckhdq {{.*#+}} xmm9 = xmm9[2],xmm13[2],xmm9[3],xmm13[3]
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm10 = xmm10[0],xmm13[0],xmm10[1],xmm13[1]
 ; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
-; SSE2-NEXT:    pxor %xmm3, %xmm3
-; SSE2-NEXT:    pcmpgtd %xmm0, %xmm3
 ; SSE2-NEXT:    psrad $16, %xmm0
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
+; SSE2-NEXT:    pxor %xmm15, %xmm15
+; SSE2-NEXT:    pcmpgtd %xmm0, %xmm15
+; SSE2-NEXT:    movdqa %xmm0, %xmm11
+; SSE2-NEXT:    punpckhdq {{.*#+}} xmm11 = xmm11[2],xmm15[2],xmm11[3],xmm15[3]
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm15[0],xmm0[1],xmm15[1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[2,3,0,1]
+; SSE2-NEXT:    pxor %xmm14, %xmm14
+; SSE2-NEXT:    pcmpgtd %xmm3, %xmm14
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm14[0],xmm3[1],xmm14[1]
+; SSE2-NEXT:    pxor %xmm5, %xmm5
+; SSE2-NEXT:    pcmpgtd %xmm2, %xmm5
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1]
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
 ; SSE2-NEXT:    pxor %xmm6, %xmm6
-; SSE2-NEXT:    pcmpgtd %xmm4, %xmm6
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm6[0],xmm4[1],xmm6[1]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm8[0],xmm3[1],xmm8[1]
-; SSE2-NEXT:    pmuludq %xmm4, %xmm3
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm6 = xmm6[0],xmm8[0],xmm6[1],xmm8[1]
-; SSE2-NEXT:    pmuludq %xmm0, %xmm6
-; SSE2-NEXT:    paddq %xmm3, %xmm6
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm3 = xmm5[0,2,2,3,4,5,6,7]
-; SSE2-NEXT:    pmuludq %xmm4, %xmm0
-; SSE2-NEXT:    pxor %xmm4, %xmm4
-; SSE2-NEXT:    pcmpgtd %xmm3, %xmm4
-; SSE2-NEXT:    psrad $16, %xmm3
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1]
-; SSE2-NEXT:    psllq $32, %xmm6
-; SSE2-NEXT:    paddq %xmm6, %xmm0
-; SSE2-NEXT:    pxor %xmm6, %xmm6
 ; SSE2-NEXT:    pcmpgtd %xmm1, %xmm6
 ; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm6[0],xmm1[1],xmm6[1]
+; SSE2-NEXT:    pxor %xmm7, %xmm7
+; SSE2-NEXT:    pcmpgtd %xmm12, %xmm7
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm12 = xmm12[0],xmm7[0],xmm12[1],xmm7[1]
+; SSE2-NEXT:    movdqa %xmm15, %xmm4
 ; SSE2-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm8[0],xmm4[1],xmm8[1]
-; SSE2-NEXT:    pmuludq %xmm1, %xmm4
+; SSE2-NEXT:    pmuludq %xmm12, %xmm4
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm7 = xmm7[0],xmm8[0],xmm7[1],xmm8[1]
+; SSE2-NEXT:    pmuludq %xmm0, %xmm7
+; SSE2-NEXT:    paddq %xmm4, %xmm7
+; SSE2-NEXT:    psllq $32, %xmm7
+; SSE2-NEXT:    pmuludq %xmm12, %xmm0
+; SSE2-NEXT:    paddq %xmm7, %xmm0
+; SSE2-NEXT:    punpckhdq {{.*#+}} xmm15 = xmm15[2],xmm8[2],xmm15[3],xmm8[3]
+; SSE2-NEXT:    pmuludq %xmm1, %xmm15
 ; SSE2-NEXT:    punpckldq {{.*#+}} xmm6 = xmm6[0],xmm8[0],xmm6[1],xmm8[1]
-; SSE2-NEXT:    pmuludq %xmm3, %xmm6
-; SSE2-NEXT:    paddq %xmm4, %xmm6
-; SSE2-NEXT:    pxor %xmm4, %xmm4
-; SSE2-NEXT:    pcmpgtd %xmm7, %xmm4
-; SSE2-NEXT:    psrad $16, %xmm7
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm7 = xmm7[0],xmm4[0],xmm7[1],xmm4[1]
-; SSE2-NEXT:    pmuludq %xmm3, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[2,3,0,1]
+; SSE2-NEXT:    pmuludq %xmm11, %xmm6
+; SSE2-NEXT:    paddq %xmm15, %xmm6
 ; SSE2-NEXT:    psllq $32, %xmm6
+; SSE2-NEXT:    pmuludq %xmm11, %xmm1
 ; SSE2-NEXT:    paddq %xmm6, %xmm1
-; SSE2-NEXT:    pxor %xmm6, %xmm6
-; SSE2-NEXT:    pcmpgtd %xmm2, %xmm6
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm6[0],xmm2[1],xmm6[1]
+; SSE2-NEXT:    movdqa %xmm13, %xmm4
 ; SSE2-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm8[0],xmm4[1],xmm8[1]
 ; SSE2-NEXT:    pmuludq %xmm2, %xmm4
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm6 = xmm6[0],xmm8[0],xmm6[1],xmm8[1]
-; SSE2-NEXT:    pmuludq %xmm7, %xmm6
-; SSE2-NEXT:    paddq %xmm4, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm5[2,3,0,1]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm4 = xmm4[0,2,2,3,4,5,6,7]
-; SSE2-NEXT:    pmuludq %xmm7, %xmm2
-; SSE2-NEXT:    pxor %xmm5, %xmm5
-; SSE2-NEXT:    pcmpgtd %xmm4, %xmm5
-; SSE2-NEXT:    psrad $16, %xmm4
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1]
-; SSE2-NEXT:    psllq $32, %xmm6
-; SSE2-NEXT:    paddq %xmm6, %xmm2
-; SSE2-NEXT:    pxor %xmm6, %xmm6
-; SSE2-NEXT:    pcmpgtd %xmm3, %xmm6
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm6[0],xmm3[1],xmm6[1]
 ; SSE2-NEXT:    punpckldq {{.*#+}} xmm5 = xmm5[0],xmm8[0],xmm5[1],xmm8[1]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm6 = xmm6[0],xmm8[0],xmm6[1],xmm8[1]
-; SSE2-NEXT:    pmuludq %xmm3, %xmm5
-; SSE2-NEXT:    pmuludq %xmm4, %xmm6
-; SSE2-NEXT:    paddq %xmm5, %xmm6
-; SSE2-NEXT:    pmuludq %xmm4, %xmm3
-; SSE2-NEXT:    psllq $32, %xmm6
-; SSE2-NEXT:    paddq %xmm6, %xmm3
+; SSE2-NEXT:    pmuludq %xmm10, %xmm5
+; SSE2-NEXT:    paddq %xmm4, %xmm5
+; SSE2-NEXT:    psllq $32, %xmm5
+; SSE2-NEXT:    pmuludq %xmm10, %xmm2
+; SSE2-NEXT:    paddq %xmm5, %xmm2
+; SSE2-NEXT:    punpckhdq {{.*#+}} xmm13 = xmm13[2],xmm8[2],xmm13[3],xmm8[3]
+; SSE2-NEXT:    pmuludq %xmm3, %xmm13
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm14 = xmm14[0],xmm8[0],xmm14[1],xmm8[1]
+; SSE2-NEXT:    pmuludq %xmm9, %xmm14
+; SSE2-NEXT:    paddq %xmm13, %xmm14
+; SSE2-NEXT:    psllq $32, %xmm14
+; SSE2-NEXT:    pmuludq %xmm9, %xmm3
+; SSE2-NEXT:    paddq %xmm14, %xmm3
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: mul_v8i64_sext:
@@ -1369,3 +1363,49 @@ define <8 x i64> @mul_v8i64_sext(<8 x i16> %val1, <8 x i32> %val2) {
   %3 = mul <8 x i64> %1, %2
   ret <8 x i64> %3
 }
+
+define <2 x i64> @pmuldq_square(<2 x i64> %x) {
+; SSE2-LABEL: pmuldq_square:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    psllq $32, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,3,2,3]
+; SSE2-NEXT:    psrad $31, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,3,2,3]
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
+; SSE2-NEXT:    psrlq $32, %xmm0
+; SSE2-NEXT:    pmuludq %xmm1, %xmm0
+; SSE2-NEXT:    paddq %xmm0, %xmm0
+; SSE2-NEXT:    psllq $32, %xmm0
+; SSE2-NEXT:    pmuludq %xmm1, %xmm1
+; SSE2-NEXT:    paddq %xmm1, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: pmuldq_square:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    pmuldq %xmm0, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: pmuldq_square:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpmuldq %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    retq
+  %1 = shl <2 x i64> %x, <i64 32, i64 32>
+  %2 = ashr exact <2 x i64> %1, <i64 32, i64 32>
+  %3 = mul nsw <2 x i64> %2, %2
+  ret <2 x i64> %3
+}
+
+define <2 x i64> @pmuludq_square(<2 x i64> %x) {
+; SSE-LABEL: pmuludq_square:
+; SSE:       # %bb.0:
+; SSE-NEXT:    pmuludq %xmm0, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: pmuludq_square:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpmuludq %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    retq
+  %1 = and <2 x i64> %x, <i64 4294967295, i64 4294967295>
+  %2 = mul nuw <2 x i64> %1, %1
+  ret <2 x i64> %2
+}
diff --git a/test/CodeGen/X86/popcnt-schedule.ll b/test/CodeGen/X86/popcnt-schedule.ll
deleted file mode 100644
index a039ba01a230a8de37d4a1ed84e87259509bfade..0000000000000000000000000000000000000000
--- a/test/CodeGen/X86/popcnt-schedule.ll
+++ /dev/null
@@ -1,235 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=x86-64 -mattr=+popcnt | FileCheck %s --check-prefix=CHECK --check-prefix=GENERIC
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=slm         | FileCheck %s --check-prefix=CHECK --check-prefix=SLM
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=goldmont    | FileCheck %s --check-prefix=CHECK --check-prefix=SLM
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=sandybridge | FileCheck %s --check-prefix=CHECK --check-prefix=SANDY
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=ivybridge   | FileCheck %s --check-prefix=CHECK --check-prefix=SANDY
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=haswell     | FileCheck %s --check-prefix=CHECK --check-prefix=HASWELL
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=broadwell   | FileCheck %s --check-prefix=CHECK --check-prefix=BROADWELL
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=skylake     | FileCheck %s --check-prefix=CHECK --check-prefix=SKYLAKE
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=knl         | FileCheck %s --check-prefix=CHECK --check-prefix=HASWELL
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=bdver2      | FileCheck %s --check-prefix=CHECK --check-prefix=BDVER2
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=btver2      | FileCheck %s --check-prefix=CHECK --check-prefix=BTVER2
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=znver1      | FileCheck %s --check-prefix=CHECK --check-prefix=ZNVER1
-
-define i16 @test_ctpop_i16(i16 zeroext %a0, i16 *%a1) {
-; GENERIC-LABEL: test_ctpop_i16:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    popcntw (%rsi), %cx # sched: [9:1.00]
-; GENERIC-NEXT:    popcntw %di, %ax # sched: [3:1.00]
-; GENERIC-NEXT:    orl %ecx, %eax # sched: [1:0.33]
-; GENERIC-NEXT:    # kill: def $ax killed $ax killed $eax
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SLM-LABEL: test_ctpop_i16:
-; SLM:       # %bb.0:
-; SLM-NEXT:    popcntw (%rsi), %cx # sched: [6:1.00]
-; SLM-NEXT:    popcntw %di, %ax # sched: [3:1.00]
-; SLM-NEXT:    orl %ecx, %eax # sched: [1:0.50]
-; SLM-NEXT:    # kill: def $ax killed $ax killed $eax
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_ctpop_i16:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    popcntw (%rsi), %cx # sched: [9:1.00]
-; SANDY-NEXT:    popcntw %di, %ax # sched: [3:1.00]
-; SANDY-NEXT:    orl %ecx, %eax # sched: [1:0.33]
-; SANDY-NEXT:    # kill: def $ax killed $ax killed $eax
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_ctpop_i16:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    popcntw (%rsi), %cx # sched: [8:1.00]
-; HASWELL-NEXT:    popcntw %di, %ax # sched: [3:1.00]
-; HASWELL-NEXT:    orl %ecx, %eax # sched: [1:0.25]
-; HASWELL-NEXT:    # kill: def $ax killed $ax killed $eax
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_ctpop_i16:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    popcntw (%rsi), %cx # sched: [8:1.00]
-; BROADWELL-NEXT:    popcntw %di, %ax # sched: [3:1.00]
-; BROADWELL-NEXT:    orl %ecx, %eax # sched: [1:0.25]
-; BROADWELL-NEXT:    # kill: def $ax killed $ax killed $eax
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_ctpop_i16:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    popcntw (%rsi), %cx # sched: [8:1.00]
-; SKYLAKE-NEXT:    popcntw %di, %ax # sched: [3:1.00]
-; SKYLAKE-NEXT:    orl %ecx, %eax # sched: [1:0.25]
-; SKYLAKE-NEXT:    # kill: def $ax killed $ax killed $eax
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_ctpop_i16:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    popcntw (%rsi), %cx # sched: [8:0.50]
-; BDVER2-NEXT:    popcntw %di, %ax # sched: [4:0.50]
-; BDVER2-NEXT:    orl %ecx, %eax # sched: [1:0.50]
-; BDVER2-NEXT:    # kill: def $ax killed $ax killed $eax
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_ctpop_i16:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    popcntw (%rsi), %cx # sched: [4:1.00]
-; BTVER2-NEXT:    popcntw %di, %ax # sched: [1:0.50]
-; BTVER2-NEXT:    orl %ecx, %eax # sched: [1:0.50]
-; BTVER2-NEXT:    # kill: def $ax killed $ax killed $eax
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_ctpop_i16:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    popcntw (%rsi), %cx # sched: [5:0.50]
-; ZNVER1-NEXT:    popcntw %di, %ax # sched: [1:0.25]
-; ZNVER1-NEXT:    orl %ecx, %eax # sched: [1:0.25]
-; ZNVER1-NEXT:    # kill: def $ax killed $ax killed $eax
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = load i16, i16 *%a1
-  %2 = tail call i16 @llvm.ctpop.i16( i16 %1 )
-  %3 = tail call i16 @llvm.ctpop.i16( i16 %a0 )
-  %4 = or i16 %2, %3
-  ret i16 %4
-}
-declare i16 @llvm.ctpop.i16(i16)
-
-define i32 @test_ctpop_i32(i32 %a0, i32 *%a1) {
-; GENERIC-LABEL: test_ctpop_i32:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    popcntl (%rsi), %ecx # sched: [9:1.00]
-; GENERIC-NEXT:    popcntl %edi, %eax # sched: [3:1.00]
-; GENERIC-NEXT:    orl %ecx, %eax # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SLM-LABEL: test_ctpop_i32:
-; SLM:       # %bb.0:
-; SLM-NEXT:    popcntl (%rsi), %ecx # sched: [6:1.00]
-; SLM-NEXT:    popcntl %edi, %eax # sched: [3:1.00]
-; SLM-NEXT:    orl %ecx, %eax # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_ctpop_i32:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    popcntl (%rsi), %ecx # sched: [9:1.00]
-; SANDY-NEXT:    popcntl %edi, %eax # sched: [3:1.00]
-; SANDY-NEXT:    orl %ecx, %eax # sched: [1:0.33]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_ctpop_i32:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    popcntl (%rsi), %ecx # sched: [8:1.00]
-; HASWELL-NEXT:    popcntl %edi, %eax # sched: [3:1.00]
-; HASWELL-NEXT:    orl %ecx, %eax # sched: [1:0.25]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_ctpop_i32:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    popcntl (%rsi), %ecx # sched: [8:1.00]
-; BROADWELL-NEXT:    popcntl %edi, %eax # sched: [3:1.00]
-; BROADWELL-NEXT:    orl %ecx, %eax # sched: [1:0.25]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_ctpop_i32:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    popcntl (%rsi), %ecx # sched: [8:1.00]
-; SKYLAKE-NEXT:    popcntl %edi, %eax # sched: [3:1.00]
-; SKYLAKE-NEXT:    orl %ecx, %eax # sched: [1:0.25]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_ctpop_i32:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    popcntl (%rsi), %ecx # sched: [8:0.50]
-; BDVER2-NEXT:    popcntl %edi, %eax # sched: [4:0.50]
-; BDVER2-NEXT:    orl %ecx, %eax # sched: [1:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_ctpop_i32:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    popcntl (%rsi), %ecx # sched: [4:1.00]
-; BTVER2-NEXT:    popcntl %edi, %eax # sched: [1:0.50]
-; BTVER2-NEXT:    orl %ecx, %eax # sched: [1:0.50]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_ctpop_i32:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    popcntl (%rsi), %ecx # sched: [5:0.50]
-; ZNVER1-NEXT:    popcntl %edi, %eax # sched: [1:0.25]
-; ZNVER1-NEXT:    orl %ecx, %eax # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = load i32, i32 *%a1
-  %2 = tail call i32 @llvm.ctpop.i32( i32 %1 )
-  %3 = tail call i32 @llvm.ctpop.i32( i32 %a0 )
-  %4 = or i32 %2, %3
-  ret i32 %4
-}
-declare i32 @llvm.ctpop.i32(i32)
-
-define i64 @test_ctpop_i64(i64 %a0, i64 *%a1) {
-; GENERIC-LABEL: test_ctpop_i64:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    popcntq (%rsi), %rcx # sched: [9:1.00]
-; GENERIC-NEXT:    popcntq %rdi, %rax # sched: [3:1.00]
-; GENERIC-NEXT:    orq %rcx, %rax # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SLM-LABEL: test_ctpop_i64:
-; SLM:       # %bb.0:
-; SLM-NEXT:    popcntq (%rsi), %rcx # sched: [6:1.00]
-; SLM-NEXT:    popcntq %rdi, %rax # sched: [3:1.00]
-; SLM-NEXT:    orq %rcx, %rax # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_ctpop_i64:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    popcntq (%rsi), %rcx # sched: [9:1.00]
-; SANDY-NEXT:    popcntq %rdi, %rax # sched: [3:1.00]
-; SANDY-NEXT:    orq %rcx, %rax # sched: [1:0.33]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_ctpop_i64:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    popcntq (%rsi), %rcx # sched: [8:1.00]
-; HASWELL-NEXT:    popcntq %rdi, %rax # sched: [3:1.00]
-; HASWELL-NEXT:    orq %rcx, %rax # sched: [1:0.25]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_ctpop_i64:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    popcntq (%rsi), %rcx # sched: [8:1.00]
-; BROADWELL-NEXT:    popcntq %rdi, %rax # sched: [3:1.00]
-; BROADWELL-NEXT:    orq %rcx, %rax # sched: [1:0.25]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_ctpop_i64:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    popcntq (%rsi), %rcx # sched: [8:1.00]
-; SKYLAKE-NEXT:    popcntq %rdi, %rax # sched: [3:1.00]
-; SKYLAKE-NEXT:    orq %rcx, %rax # sched: [1:0.25]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_ctpop_i64:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    popcntq (%rsi), %rcx # sched: [8:0.50]
-; BDVER2-NEXT:    popcntq %rdi, %rax # sched: [4:0.50]
-; BDVER2-NEXT:    orq %rcx, %rax # sched: [1:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_ctpop_i64:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    popcntq (%rsi), %rcx # sched: [4:1.00]
-; BTVER2-NEXT:    popcntq %rdi, %rax # sched: [1:0.50]
-; BTVER2-NEXT:    orq %rcx, %rax # sched: [1:0.50]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_ctpop_i64:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    popcntq (%rsi), %rcx # sched: [5:0.50]
-; ZNVER1-NEXT:    popcntq %rdi, %rax # sched: [1:0.25]
-; ZNVER1-NEXT:    orq %rcx, %rax # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = load i64, i64 *%a1
-  %2 = tail call i64 @llvm.ctpop.i64( i64 %1 )
-  %3 = tail call i64 @llvm.ctpop.i64( i64 %a0 )
-  %4 = or i64 %2, %3
-  ret i64 %4
-}
-declare i64 @llvm.ctpop.i64(i64)
diff --git a/test/CodeGen/X86/postra-licm.ll b/test/CodeGen/X86/postra-licm.ll
index 329184a88ff024b4a852a840091048529e981031..72018b10d707b31e980a89023f73cca0da5427bd 100644
--- a/test/CodeGen/X86/postra-licm.ll
+++ b/test/CodeGen/X86/postra-licm.ll
@@ -1,5 +1,5 @@
-; RUN: llc < %s -mtriple=i386-apple-darwin -relocation-model=pic -disable-fp-elim | FileCheck %s -check-prefix=X86-32
-; RUN: llc < %s -mtriple=x86_64-apple-darwin -relocation-model=pic -disable-fp-elim | FileCheck %s -check-prefix=X86-64
+; RUN: llc < %s -mtriple=i386-apple-darwin -relocation-model=pic -frame-pointer=all | FileCheck %s -check-prefix=X86-32
+; RUN: llc < %s -mtriple=x86_64-apple-darwin -relocation-model=pic -frame-pointer=all | FileCheck %s -check-prefix=X86-64
 
 ; MachineLICM should be able to hoist loop invariant reload out of the loop.
 ; Only linear scan needs this, -regalloc=greedy sinks the spill instead.
diff --git a/test/CodeGen/X86/pow.75.ll b/test/CodeGen/X86/pow.75.ll
new file mode 100644
index 0000000000000000000000000000000000000000..5f4e228945e227c171f131aeae5c035fdcb601f8
--- /dev/null
+++ b/test/CodeGen/X86/pow.75.ll
@@ -0,0 +1,48 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-- -debug 2>&1 | FileCheck %s
+; REQUIRES: asserts
+
+declare float @llvm.pow.f32(float, float)
+declare <4 x float> @llvm.pow.v4f32(<4 x float>, <4 x float>)
+declare double @llvm.pow.f64(double, double)
+declare <2 x double> @llvm.pow.v2f64(<2 x double>, <2 x double>)
+
+define float @pow_f32_three_fourth_fmf(float %x) nounwind {
+; CHECK: Combining: {{.*}}: f32 = fpow ninf nsz afn [[X:t[0-9]+]], ConstantFP:f32<7.500000e-01>
+; CHECK-NEXT: Creating new node: [[SQRT:t[0-9]+]]: f32 = fsqrt ninf nsz afn [[X]]
+; CHECK-NEXT: Creating new node: [[SQRTSQRT:t[0-9]+]]: f32 = fsqrt ninf nsz afn [[SQRT]]
+; CHECK-NEXT: Creating new node: [[R:t[0-9]+]]: f32 = fmul ninf nsz afn [[SQRT]], [[SQRTSQRT]]
+; CHECK-NEXT:  ... into: [[R]]: f32 = fmul ninf nsz afn [[SQRT]], [[SQRTSQRT]]
+  %r = call nsz ninf afn float @llvm.pow.f32(float %x, float 7.5e-01)
+  ret float %r
+}
+
+define double @pow_f64_three_fourth_fmf(double %x) nounwind {
+; CHECK: Combining: {{.*}}: f64 = fpow ninf nsz afn [[X:t[0-9]+]], ConstantFP:f64<7.500000e-01>
+; CHECK-NEXT: Creating new node: [[SQRT:t[0-9]+]]: f64 = fsqrt ninf nsz afn [[X]]
+; CHECK-NEXT: Creating new node: [[SQRTSQRT:t[0-9]+]]: f64 = fsqrt ninf nsz afn [[SQRT]]
+; CHECK-NEXT: Creating new node: [[R:t[0-9]+]]: f64 = fmul ninf nsz afn [[SQRT]], [[SQRTSQRT]]
+; CHECK-NEXT:  ... into: [[R]]: f64 = fmul ninf nsz afn [[SQRT]], [[SQRTSQRT]]
+  %r = call nsz ninf afn double @llvm.pow.f64(double %x, double 7.5e-01)
+  ret double %r
+}
+
+define <4 x float> @pow_v4f32_three_fourth_fmf(<4 x float> %x) nounwind {
+; CHECK: Combining: {{.*}}: v4f32 = fpow nnan ninf nsz arcp contract afn reassoc [[X:t[0-9]+]], {{.*}}
+; CHECK-NEXT: Creating new node: [[SQRT:t[0-9]+]]: v4f32 = fsqrt nnan ninf nsz arcp contract afn reassoc [[X]]
+; CHECK-NEXT: Creating new node: [[SQRTSQRT:t[0-9]+]]: v4f32 = fsqrt nnan ninf nsz arcp contract afn reassoc [[SQRT]]
+; CHECK-NEXT: Creating new node: [[R:t[0-9]+]]: v4f32 = fmul nnan ninf nsz arcp contract afn reassoc [[SQRT]], [[SQRTSQRT]]
+; CHECK-NEXT:  ... into: [[R]]: v4f32 = fmul nnan ninf nsz arcp contract afn reassoc [[SQRT]], [[SQRTSQRT]]
+  %r = call fast <4 x float> @llvm.pow.v4f32(<4 x float> %x, <4 x float> <float 7.5e-1, float 7.5e-1, float 7.5e-01, float 7.5e-01>)
+  ret <4 x float> %r
+}
+
+define <2 x double> @pow_v2f64_three_fourth_fmf(<2 x double> %x) nounwind {
+; CHECK: Combining: {{.*}}: v2f64 = fpow nnan ninf nsz arcp contract afn reassoc [[X:t[0-9]+]], {{.*}}
+; CHECK-NEXT: Creating new node: [[SQRT:t[0-9]+]]: v2f64 = fsqrt nnan ninf nsz arcp contract afn reassoc [[X]]
+; CHECK-NEXT: Creating new node: [[SQRTSQRT:t[0-9]+]]: v2f64 = fsqrt nnan ninf nsz arcp contract afn reassoc [[SQRT]]
+; CHECK-NEXT: Creating new node: [[R:t[0-9]+]]: v2f64 = fmul nnan ninf nsz arcp contract afn reassoc [[SQRT]], [[SQRTSQRT]]
+; CHECK-NEXT:  ... into: [[R]]: v2f64 = fmul nnan ninf nsz arcp contract afn reassoc [[SQRT]], [[SQRTSQRT]]
+  %r = call fast <2 x double> @llvm.pow.v2f64(<2 x double> %x, <2 x double> <double 7.5e-1, double 7.5e-1>)
+  ret <2 x double> %r
+}
diff --git a/test/CodeGen/X86/pr13577.ll b/test/CodeGen/X86/pr13577.ll
index e0e90f81bc42c32460cbbbb7d8105cf3d88442a1..3f9e2f953bae370aa707e3bff4499f9cc377bfb5 100644
--- a/test/CodeGen/X86/pr13577.ll
+++ b/test/CodeGen/X86/pr13577.ll
@@ -14,7 +14,7 @@ define x86_fp80 @foo(x86_fp80 %a) {
 ; CHECK-NEXT:    testb $-128, -{{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    flds {{.*}}(%rip)
 ; CHECK-NEXT:    flds {{.*}}(%rip)
-; CHECK-NEXT:    fcmovne %st(1), %st(0)
+; CHECK-NEXT:    fcmovne %st(1), %st
 ; CHECK-NEXT:    fstp %st(1)
 ; CHECK-NEXT:    retq
   %1 = tail call x86_fp80 @copysignl(x86_fp80 0xK7FFF8000000000000000, x86_fp80 %a) nounwind readnone
diff --git a/test/CodeGen/X86/pr1489.ll b/test/CodeGen/X86/pr1489.ll
index 13ced2a327459fd81078fbc7ec1f06fe2dd1fa34..36d30e8bef33ad9a912e51e7c6afce7a59672822 100644
--- a/test/CodeGen/X86/pr1489.ll
+++ b/test/CodeGen/X86/pr1489.ll
@@ -1,5 +1,5 @@
-; RUN: llc < %s -disable-fp-elim -O0 -mcpu=i486 | grep 1082126238 | count 3
-; RUN: llc < %s -disable-fp-elim -O0 -mcpu=i486 | grep -- -1236950581 | count 1
+; RUN: llc < %s -frame-pointer=all -O0 -mcpu=i486 | grep 1082126238 | count 3
+; RUN: llc < %s -frame-pointer=all -O0 -mcpu=i486 | grep -- -1236950581 | count 1
 ;; magic constants are 3.999f and half of 3.999
 ; ModuleID = '1489.c'
 target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64"
diff --git a/test/CodeGen/X86/pr30511.ll b/test/CodeGen/X86/pr30511.ll
index 69d0a94cc11caef8a35e2305e13f9ee0bb0d03bc..6764656e3337ceccb1d820683fee7cd9957de4f0 100644
--- a/test/CodeGen/X86/pr30511.ll
+++ b/test/CodeGen/X86/pr30511.ll
@@ -9,7 +9,7 @@ define i64 @PR30511(<2 x double> %a) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    addpd {{.*}}(%rip), %xmm0
 ; CHECK-NEXT:    cvtdq2pd %xmm0, %xmm0
-; CHECK-NEXT:    mulpd {{.*}}(%rip), %xmm0
+; CHECK-NEXT:    mulsd {{.*}}(%rip), %xmm0
 ; CHECK-NEXT:    movq %xmm0, %rax
 ; CHECK-NEXT:    retq
   %1 = fadd <2 x double> %a, <double 0x4338000000000000, double 0x4338000000000000>
diff --git a/test/CodeGen/X86/pr30562.ll b/test/CodeGen/X86/pr30562.ll
index a8e6480741949d7d5aec7e5c8ab047577b770abb..24cbf10ed53de47756bbd832076a23383eba528a 100644
--- a/test/CodeGen/X86/pr30562.ll
+++ b/test/CodeGen/X86/pr30562.ll
@@ -1,13 +1,33 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown | FileCheck %s
 
 
 define i32 @foo(i64* nocapture %perm, i32 %n) {
+; CHECK-LABEL: foo:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movl %esi, %eax
+; CHECK-NEXT:    movl $1, %ecx
+; CHECK-NEXT:    movq %rcx, %xmm0
+; CHECK-NEXT:    pslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1,2,3,4,5,6,7]
+; CHECK-NEXT:    movl %esi, %ecx
+; CHECK-NEXT:    andl $1, %ecx
+; CHECK-NEXT:    movdqa {{.*#+}} xmm1 = [2,3]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB0_1: # %body
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    movdqa %xmm0, -{{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movq -24(%rsp,%rcx,8), %rdx
+; CHECK-NEXT:    movdqu %xmm0, (%rdi,%rdx,8)
+; CHECK-NEXT:    testq %rdx, %rdx
+; CHECK-NEXT:    movdqa %xmm1, %xmm0
+; CHECK-NEXT:    jne .LBB0_1
+; CHECK-NEXT:  # %bb.2: # %exit
+; CHECK-NEXT:    # kill: def $eax killed $eax killed $rax
+; CHECK-NEXT:    retq
 entry:
   br label %body
 
 body:
-; CHECK-LABEL: foo:
-; CHECK: pslldq  $8, %xmm0
   %vec.ind = phi <2 x i64> [ <i64 0, i64 1>, %entry ], [ <i64 2, i64 3>, %body ]
   %l13 = extractelement <2 x i64> %vec.ind, i32 %n
   %l14 = getelementptr inbounds i64, i64* %perm, i64 %l13
diff --git a/test/CodeGen/X86/pr3154.ll b/test/CodeGen/X86/pr3154.ll
index 5da88403802e2c5a2d7e4d90d7e2617f20424fbc..fa788b2231f0409f2d227ea7c4fd47ce792b1998 100644
--- a/test/CodeGen/X86/pr3154.ll
+++ b/test/CodeGen/X86/pr3154.ll
@@ -1,5 +1,5 @@
 ; RUN: llc < %s -mtriple=i386-pc-linux-gnu -mattr=+sse2
-; RUN: llc < %s -mtriple=i386-pc-linux-gnu -mattr=+sse2 -relocation-model=pic -disable-fp-elim
+; RUN: llc < %s -mtriple=i386-pc-linux-gnu -mattr=+sse2 -relocation-model=pic -frame-pointer=all
 ; PR3154
 
 define void @ff_flac_compute_autocorr_sse2(i32* %data, i32 %len, i32 %lag, double* %autoc) nounwind {
diff --git a/test/CodeGen/X86/pr32329.ll b/test/CodeGen/X86/pr32329.ll
index 7ccd5597a3daef58998d70fb39ffaaefb9789ffa..21109469723dca578737b911e8ac63e65c4d5bc4 100644
--- a/test/CodeGen/X86/pr32329.ll
+++ b/test/CodeGen/X86/pr32329.ll
@@ -41,7 +41,7 @@ define void @foo() local_unnamed_addr {
 ; X86-NEXT:    movl %ebx, %edi
 ; X86-NEXT:    subl %esi, %edi
 ; X86-NEXT:    imull %edi, %ecx
-; X86-NEXT:    addl $-1437483407, %ecx # imm = 0xAA51BE71
+; X86-NEXT:    addb $113, %cl
 ; X86-NEXT:    movl $9, %esi
 ; X86-NEXT:    xorl %ebp, %ebp
 ; X86-NEXT:    shldl %cl, %esi, %ebp
@@ -80,7 +80,7 @@ define void @foo() local_unnamed_addr {
 ; X64-NEXT:    movl %edi, %esi
 ; X64-NEXT:    subl %r8d, %esi
 ; X64-NEXT:    imull %esi, %ecx
-; X64-NEXT:    addl $-1437483407, %ecx # imm = 0xAA51BE71
+; X64-NEXT:    addb $113, %cl
 ; X64-NEXT:    movl $9, %edx
 ; X64-NEXT:    # kill: def $cl killed $cl killed $ecx
 ; X64-NEXT:    shlq %cl, %rdx
diff --git a/test/CodeGen/X86/pr32345.ll b/test/CodeGen/X86/pr32345.ll
index 65fcf055f284cabe63a75e41a01de9eeceb290d4..cec3a6925c31cd87603d64fefc70b9d8849773a5 100644
--- a/test/CodeGen/X86/pr32345.ll
+++ b/test/CodeGen/X86/pr32345.ll
@@ -69,8 +69,8 @@ define void @foo() {
 ; 6860-NEXT:    xorl %ecx, %esi
 ; 6860-NEXT:    movw %si, %ax
 ; 6860-NEXT:    movzwl %ax, %esi
-; 6860-NEXT:    addl $-16610, %ecx # imm = 0xBF1E
 ; 6860-NEXT:    movb %cl, %bl
+; 6860-NEXT:    addb $30, %bl
 ; 6860-NEXT:    xorl %ecx, %ecx
 ; 6860-NEXT:    movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
 ; 6860-NEXT:    movb %bl, %cl
@@ -98,14 +98,13 @@ define void @foo() {
 ;
 ; X64-LABEL: foo:
 ; X64:       # %bb.0: # %bb
-; X64-NEXT:    movzwl {{.*}}(%rip), %eax
 ; X64-NEXT:    movzwl {{.*}}(%rip), %ecx
-; X64-NEXT:    movl %ecx, %edx
-; X64-NEXT:    xorl %edx, %edx
-; X64-NEXT:    xorl %eax, %edx
-; X64-NEXT:    movzwl %dx, %eax
+; X64-NEXT:    movzwl {{.*}}(%rip), %eax
+; X64-NEXT:    xorw %cx, %ax
+; X64-NEXT:    xorl %ecx, %eax
+; X64-NEXT:    movzwl %ax, %eax
 ; X64-NEXT:    movq %rax, -{{[0-9]+}}(%rsp)
-; X64-NEXT:    addl $-16610, %ecx # imm = 0xBF1E
+; X64-NEXT:    addb $30, %cl
 ; X64-NEXT:    # kill: def $cl killed $cl killed $ecx
 ; X64-NEXT:    shrq %cl, %rax
 ; X64-NEXT:    movb %al, (%rax)
@@ -120,15 +119,14 @@ define void @foo() {
 ; 686-NEXT:    .cfi_def_cfa_register %ebp
 ; 686-NEXT:    andl $-8, %esp
 ; 686-NEXT:    subl $8, %esp
-; 686-NEXT:    movzwl var_22, %eax
 ; 686-NEXT:    movzwl var_27, %ecx
-; 686-NEXT:    movl %ecx, %edx
-; 686-NEXT:    xorl %ecx, %edx
-; 686-NEXT:    xorl %eax, %edx
-; 686-NEXT:    movzwl %dx, %eax
+; 686-NEXT:    movzwl var_22, %eax
+; 686-NEXT:    xorw %cx, %ax
+; 686-NEXT:    xorl %ecx, %eax
+; 686-NEXT:    movzwl %ax, %eax
 ; 686-NEXT:    movl %eax, (%esp)
 ; 686-NEXT:    movl $0, {{[0-9]+}}(%esp)
-; 686-NEXT:    addl $-16610, %ecx # imm = 0xBF1E
+; 686-NEXT:    addb $30, %cl
 ; 686-NEXT:    xorl %edx, %edx
 ; 686-NEXT:    shrdl %cl, %edx, %eax
 ; 686-NEXT:    testb $32, %cl
diff --git a/test/CodeGen/X86/pr33290.ll b/test/CodeGen/X86/pr33290.ll
index b5d9754eba7623790ec174ccf7f69d66a774559f..44b7dca0ae2fb0f406091e26cfa2ee1194b4f343 100644
--- a/test/CodeGen/X86/pr33290.ll
+++ b/test/CodeGen/X86/pr33290.ll
@@ -14,8 +14,8 @@ define void @e() {
 ; X86-NEXT:  .LBB0_1: # %for.cond
 ; X86-NEXT:    # =>This Inner Loop Header: Depth=1
 ; X86-NEXT:    movzbl c, %ecx
-; X86-NEXT:    leal a+2(%ecx), %ecx
 ; X86-NEXT:    movb $0, c
+; X86-NEXT:    leal a+2(%ecx), %ecx
 ; X86-NEXT:    movl %ecx, (%eax)
 ; X86-NEXT:    jmp .LBB0_1
 ;
diff --git a/test/CodeGen/X86/pr33349.ll b/test/CodeGen/X86/pr33349.ll
index ec9f238192838b00e65532d94eb472baa90781f1..9aa28384f4e81d1efeb8243b2243133d04046daa 100644
--- a/test/CodeGen/X86/pr33349.ll
+++ b/test/CodeGen/X86/pr33349.ll
@@ -12,32 +12,32 @@ target triple = "x86_64-unknown-linux-gnu"
 ; KNL-NEXT:    vptestmd %zmm0, %zmm0, %k0
 ; KNL-NEXT:    kshiftrw $1, %k0, %k1
 ; KNL-NEXT:    kmovw %k1, %eax
+; KNL-NEXT:    kshiftrw $2, %k0, %k1
+; KNL-NEXT:    kshiftrw $1, %k1, %k2
+; KNL-NEXT:    kmovw %k1, %ecx
 ; KNL-NEXT:    testb $1, %al
 ; KNL-NEXT:    fld1
 ; KNL-NEXT:    fldz
 ; KNL-NEXT:    fld %st(0)
-; KNL-NEXT:    fcmovne %st(2), %st(0)
-; KNL-NEXT:    kshiftrw $2, %k0, %k1
-; KNL-NEXT:    kshiftrw $1, %k1, %k2
-; KNL-NEXT:    kmovw %k2, %eax
-; KNL-NEXT:    testb $1, %al
+; KNL-NEXT:    fcmovne %st(2), %st
+; KNL-NEXT:    testb $1, %cl
 ; KNL-NEXT:    fld %st(1)
-; KNL-NEXT:    fcmovne %st(3), %st(0)
-; KNL-NEXT:    kmovw %k0, %eax
+; KNL-NEXT:    fcmovne %st(3), %st
+; KNL-NEXT:    kmovw %k2, %eax
 ; KNL-NEXT:    testb $1, %al
 ; KNL-NEXT:    fld %st(2)
-; KNL-NEXT:    fcmovne %st(4), %st(0)
-; KNL-NEXT:    kmovw %k1, %eax
+; KNL-NEXT:    fcmovne %st(4), %st
+; KNL-NEXT:    kmovw %k0, %eax
 ; KNL-NEXT:    testb $1, %al
 ; KNL-NEXT:    fxch %st(3)
-; KNL-NEXT:    fcmovne %st(4), %st(0)
+; KNL-NEXT:    fcmovne %st(4), %st
 ; KNL-NEXT:    fstp %st(4)
 ; KNL-NEXT:    fxch %st(3)
-; KNL-NEXT:    fstpt 20(%rdi)
-; KNL-NEXT:    fxch %st(1)
 ; KNL-NEXT:    fstpt (%rdi)
 ; KNL-NEXT:    fxch %st(1)
 ; KNL-NEXT:    fstpt 30(%rdi)
+; KNL-NEXT:    fxch %st(1)
+; KNL-NEXT:    fstpt 20(%rdi)
 ; KNL-NEXT:    fstpt 10(%rdi)
 ; KNL-NEXT:    vzeroupper
 ; KNL-NEXT:    retq
@@ -46,35 +46,35 @@ target triple = "x86_64-unknown-linux-gnu"
 ; SKX:       # %bb.0: # %bb
 ; SKX-NEXT:    vpslld $31, %xmm0, %xmm0
 ; SKX-NEXT:    vpmovd2m %xmm0, %k0
+; SKX-NEXT:    kshiftrb $1, %k0, %k1
+; SKX-NEXT:    kmovd %k1, %eax
 ; SKX-NEXT:    kshiftrb $2, %k0, %k1
-; SKX-NEXT:    kshiftrw $1, %k1, %k2
-; SKX-NEXT:    kmovd %k2, %eax
+; SKX-NEXT:    kshiftrb $1, %k1, %k2
+; SKX-NEXT:    kmovd %k1, %ecx
 ; SKX-NEXT:    testb $1, %al
 ; SKX-NEXT:    fld1
 ; SKX-NEXT:    fldz
 ; SKX-NEXT:    fld %st(0)
-; SKX-NEXT:    fcmovne %st(2), %st(0)
-; SKX-NEXT:    kmovd %k1, %eax
-; SKX-NEXT:    testb $1, %al
+; SKX-NEXT:    fcmovne %st(2), %st
+; SKX-NEXT:    testb $1, %cl
 ; SKX-NEXT:    fld %st(1)
-; SKX-NEXT:    fcmovne %st(3), %st(0)
-; SKX-NEXT:    kshiftrw $1, %k0, %k1
-; SKX-NEXT:    kmovd %k1, %eax
+; SKX-NEXT:    fcmovne %st(3), %st
+; SKX-NEXT:    kmovd %k2, %eax
 ; SKX-NEXT:    testb $1, %al
 ; SKX-NEXT:    fld %st(2)
-; SKX-NEXT:    fcmovne %st(4), %st(0)
+; SKX-NEXT:    fcmovne %st(4), %st
 ; SKX-NEXT:    kmovd %k0, %eax
 ; SKX-NEXT:    testb $1, %al
 ; SKX-NEXT:    fxch %st(3)
-; SKX-NEXT:    fcmovne %st(4), %st(0)
+; SKX-NEXT:    fcmovne %st(4), %st
 ; SKX-NEXT:    fstp %st(4)
 ; SKX-NEXT:    fxch %st(3)
 ; SKX-NEXT:    fstpt (%rdi)
 ; SKX-NEXT:    fxch %st(1)
-; SKX-NEXT:    fstpt 10(%rdi)
+; SKX-NEXT:    fstpt 30(%rdi)
 ; SKX-NEXT:    fxch %st(1)
 ; SKX-NEXT:    fstpt 20(%rdi)
-; SKX-NEXT:    fstpt 30(%rdi)
+; SKX-NEXT:    fstpt 10(%rdi)
 ; SKX-NEXT:    retq
  bb:
    %tmp = select <4 x i1> %m, <4 x x86_fp80> <x86_fp80 0xK3FFF8000000000000000, x86_fp80 0xK3FFF8000000000000000, x86_fp80 0xK3FFF8000000000000000, x86_fp80             0xK3FFF8000000000000000>, <4 x x86_fp80> zeroinitializer
diff --git a/test/CodeGen/X86/pr34080.ll b/test/CodeGen/X86/pr34080.ll
index a709a4840e5303572373193cbc02519a6a73df3a..007b0827c67deec2b600656e81cf90e4db2fbe80 100644
--- a/test/CodeGen/X86/pr34080.ll
+++ b/test/CodeGen/X86/pr34080.ll
@@ -27,7 +27,7 @@ define void @_Z1fe(x86_fp80 %z) local_unnamed_addr #0 {
 ; SSE2-NEXT:    movsd %xmm0, -32(%rbp)
 ; SSE2-NEXT:    fsubl -32(%rbp)
 ; SSE2-NEXT:    flds {{.*}}(%rip)
-; SSE2-NEXT:    fmul %st(0), %st(1)
+; SSE2-NEXT:    fmul %st, %st(1)
 ; SSE2-NEXT:    fnstcw -2(%rbp)
 ; SSE2-NEXT:    movzwl -2(%rbp), %eax
 ; SSE2-NEXT:    movw $3199, -2(%rbp) ## imm = 0xC7F
@@ -41,7 +41,7 @@ define void @_Z1fe(x86_fp80 %z) local_unnamed_addr #0 {
 ; SSE2-NEXT:    movsd %xmm0, -56(%rbp)
 ; SSE2-NEXT:    movsd %xmm0, -24(%rbp)
 ; SSE2-NEXT:    fsubl -24(%rbp)
-; SSE2-NEXT:    fmulp %st(1)
+; SSE2-NEXT:    fmulp %st, %st(1)
 ; SSE2-NEXT:    fstpl -48(%rbp)
 ; SSE2-NEXT:    popq %rbp
 ; SSE2-NEXT:    retq
@@ -54,10 +54,10 @@ define void @_Z1fe(x86_fp80 %z) local_unnamed_addr #0 {
 ; SSE2-SCHEDULE-NEXT:    movq %rsp, %rbp
 ; SSE2-SCHEDULE-NEXT:    .cfi_def_cfa_register %rbp
 ; SSE2-SCHEDULE-NEXT:    fnstcw -4(%rbp)
+; SSE2-SCHEDULE-NEXT:    fldt 16(%rbp)
 ; SSE2-SCHEDULE-NEXT:    movzwl -4(%rbp), %eax
 ; SSE2-SCHEDULE-NEXT:    movw $3199, -4(%rbp) ## imm = 0xC7F
 ; SSE2-SCHEDULE-NEXT:    fldcw -4(%rbp)
-; SSE2-SCHEDULE-NEXT:    fldt 16(%rbp)
 ; SSE2-SCHEDULE-NEXT:    movw %ax, -4(%rbp)
 ; SSE2-SCHEDULE-NEXT:    fistl -8(%rbp)
 ; SSE2-SCHEDULE-NEXT:    fldcw -4(%rbp)
@@ -65,12 +65,12 @@ define void @_Z1fe(x86_fp80 %z) local_unnamed_addr #0 {
 ; SSE2-SCHEDULE-NEXT:    movsd %xmm0, -64(%rbp)
 ; SSE2-SCHEDULE-NEXT:    movsd %xmm0, -32(%rbp)
 ; SSE2-SCHEDULE-NEXT:    fsubl -32(%rbp)
-; SSE2-SCHEDULE-NEXT:    fnstcw -2(%rbp)
 ; SSE2-SCHEDULE-NEXT:    flds {{.*}}(%rip)
+; SSE2-SCHEDULE-NEXT:    fnstcw -2(%rbp)
+; SSE2-SCHEDULE-NEXT:    fmul %st, %st(1)
 ; SSE2-SCHEDULE-NEXT:    movzwl -2(%rbp), %eax
 ; SSE2-SCHEDULE-NEXT:    movw $3199, -2(%rbp) ## imm = 0xC7F
 ; SSE2-SCHEDULE-NEXT:    fldcw -2(%rbp)
-; SSE2-SCHEDULE-NEXT:    fmul %st(0), %st(1)
 ; SSE2-SCHEDULE-NEXT:    movw %ax, -2(%rbp)
 ; SSE2-SCHEDULE-NEXT:    fxch %st(1)
 ; SSE2-SCHEDULE-NEXT:    fistl -12(%rbp)
@@ -80,7 +80,7 @@ define void @_Z1fe(x86_fp80 %z) local_unnamed_addr #0 {
 ; SSE2-SCHEDULE-NEXT:    movsd %xmm0, -56(%rbp)
 ; SSE2-SCHEDULE-NEXT:    movsd %xmm0, -24(%rbp)
 ; SSE2-SCHEDULE-NEXT:    fsubl -24(%rbp)
-; SSE2-SCHEDULE-NEXT:    fmulp %st(1)
+; SSE2-SCHEDULE-NEXT:    fmulp %st, %st(1)
 ; SSE2-SCHEDULE-NEXT:    fstpl -48(%rbp)
 ; SSE2-SCHEDULE-NEXT:    popq %rbp
 ; SSE2-SCHEDULE-NEXT:    retq
@@ -100,7 +100,7 @@ define void @_Z1fe(x86_fp80 %z) local_unnamed_addr #0 {
 ; SSE3-NEXT:    movsd %xmm0, -24(%rbp)
 ; SSE3-NEXT:    fsubl -24(%rbp)
 ; SSE3-NEXT:    flds {{.*}}(%rip)
-; SSE3-NEXT:    fmul %st(0), %st(1)
+; SSE3-NEXT:    fmul %st, %st(1)
 ; SSE3-NEXT:    fld %st(1)
 ; SSE3-NEXT:    fisttpl -8(%rbp)
 ; SSE3-NEXT:    xorps %xmm0, %xmm0
@@ -109,7 +109,7 @@ define void @_Z1fe(x86_fp80 %z) local_unnamed_addr #0 {
 ; SSE3-NEXT:    movsd %xmm0, -16(%rbp)
 ; SSE3-NEXT:    fxch %st(1)
 ; SSE3-NEXT:    fsubl -16(%rbp)
-; SSE3-NEXT:    fmulp %st(1)
+; SSE3-NEXT:    fmulp %st, %st(1)
 ; SSE3-NEXT:    fstpl -32(%rbp)
 ; SSE3-NEXT:    popq %rbp
 ; SSE3-NEXT:    retq
@@ -129,7 +129,7 @@ define void @_Z1fe(x86_fp80 %z) local_unnamed_addr #0 {
 ; AVX-NEXT:    vmovsd %xmm0, -24(%rbp)
 ; AVX-NEXT:    fsubl -24(%rbp)
 ; AVX-NEXT:    flds {{.*}}(%rip)
-; AVX-NEXT:    fmul %st(0), %st(1)
+; AVX-NEXT:    fmul %st, %st(1)
 ; AVX-NEXT:    fld %st(1)
 ; AVX-NEXT:    fisttpl -8(%rbp)
 ; AVX-NEXT:    vcvtsi2sdl -8(%rbp), %xmm1, %xmm0
@@ -137,7 +137,7 @@ define void @_Z1fe(x86_fp80 %z) local_unnamed_addr #0 {
 ; AVX-NEXT:    vmovsd %xmm0, -16(%rbp)
 ; AVX-NEXT:    fxch %st(1)
 ; AVX-NEXT:    fsubl -16(%rbp)
-; AVX-NEXT:    fmulp %st(1)
+; AVX-NEXT:    fmulp %st, %st(1)
 ; AVX-NEXT:    fstpl -32(%rbp)
 ; AVX-NEXT:    popq %rbp
 ; AVX-NEXT:    retq
diff --git a/test/CodeGen/X86/pr34177.ll b/test/CodeGen/X86/pr34177.ll
index 3fe56277fb4b3ead0b3a324753f8162e57f16422..3d77ddb47152173ca70cb08d22e8b19a84b79224 100644
--- a/test/CodeGen/X86/pr34177.ll
+++ b/test/CodeGen/X86/pr34177.ll
@@ -1,11 +1,11 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mattr=+avx512f | FileCheck %s
-; RUN: llc < %s -mattr=+avx512f,+avx512vl,+avx512bw,+avx512dq | FileCheck %s
+; RUN: llc < %s -mattr=+avx512f | FileCheck %s --check-prefixes=CHECK,AVX512F
+; RUN: llc < %s -mattr=+avx512f,+avx512vl,+avx512bw,+avx512dq | FileCheck %s --check-prefixes=CHECK,AVX512VL
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
-define void @test(<4x i64> %a, <4 x x86_fp80> %b, <8 x x86_fp80>* %c) local_unnamed_addr {
+define void @test(<4 x i64> %a, <4 x x86_fp80> %b, <8 x x86_fp80>* %c) local_unnamed_addr {
 ; CHECK-LABEL: test:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vpextrq $1, %xmm0, %r8
@@ -20,17 +20,17 @@ define void @test(<4x i64> %a, <4 x x86_fp80> %b, <8 x x86_fp80>* %c) local_unna
 ; CHECK-NEXT:    fld1
 ; CHECK-NEXT:    fldz
 ; CHECK-NEXT:    fld %st(0)
-; CHECK-NEXT:    fcmove %st(2), %st(0)
+; CHECK-NEXT:    fcmove %st(2), %st
 ; CHECK-NEXT:    cmpq %rax, %rsi
 ; CHECK-NEXT:    fld %st(1)
-; CHECK-NEXT:    fcmove %st(3), %st(0)
+; CHECK-NEXT:    fcmove %st(3), %st
 ; CHECK-NEXT:    cmpq %rdx, %r9
 ; CHECK-NEXT:    fld %st(2)
-; CHECK-NEXT:    fcmove %st(4), %st(0)
+; CHECK-NEXT:    fcmove %st(4), %st
 ; CHECK-NEXT:    movl $1, %eax
 ; CHECK-NEXT:    cmpq %r8, %rax
 ; CHECK-NEXT:    fxch %st(3)
-; CHECK-NEXT:    fcmove %st(4), %st(0)
+; CHECK-NEXT:    fcmove %st(4), %st
 ; CHECK-NEXT:    fstp %st(4)
 ; CHECK-NEXT:    fldt {{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    fstpt 70(%rdi)
@@ -40,15 +40,15 @@ define void @test(<4x i64> %a, <4 x x86_fp80> %b, <8 x x86_fp80>* %c) local_unna
 ; CHECK-NEXT:    fstpt 30(%rdi)
 ; CHECK-NEXT:    fldt {{[0-9]+}}(%rsp)
 ; CHECK-NEXT:    fstpt 10(%rdi)
-; CHECK-NEXT:    fadd %st(0), %st(0)
+; CHECK-NEXT:    fadd %st, %st(0)
 ; CHECK-NEXT:    fstpt 60(%rdi)
 ; CHECK-NEXT:    fxch %st(1)
-; CHECK-NEXT:    fadd %st(0), %st(0)
+; CHECK-NEXT:    fadd %st, %st(0)
 ; CHECK-NEXT:    fstpt 40(%rdi)
 ; CHECK-NEXT:    fxch %st(1)
-; CHECK-NEXT:    fadd %st(0), %st(0)
+; CHECK-NEXT:    fadd %st, %st(0)
 ; CHECK-NEXT:    fstpt 20(%rdi)
-; CHECK-NEXT:    fadd %st(0), %st(0)
+; CHECK-NEXT:    fadd %st, %st(0)
 ; CHECK-NEXT:    fstpt (%rdi)
   %1 = icmp eq <4 x i64> <i64 0, i64 1, i64 2, i64 3>, %a
   %2 = select <4 x i1> %1, <4 x x86_fp80> <x86_fp80 0xK3FFF8000000000000000, x86_fp80 0xK3FFF8000000000000000, x86_fp80 0xK3FFF8000000000000000, x86_fp80 0xK3FFF8000000000000000>, <4 x x86_fp80> zeroinitializer
diff --git a/test/CodeGen/X86/pr34292.ll b/test/CodeGen/X86/pr34292.ll
new file mode 100644
index 0000000000000000000000000000000000000000..cfd5754e577a2f14b5bce98523280c028abcc854
--- /dev/null
+++ b/test/CodeGen/X86/pr34292.ll
@@ -0,0 +1,57 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown | FileCheck %s --check-prefixes=CHECK,X64
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+adx | FileCheck %s --check-prefixes=CHECK,ADX
+
+; PR34292
+@_ZL1c = external global i8
+define void @sum_unroll(i64* nocapture readonly, i64* nocapture) {
+; CHECK-LABEL: sum_unroll:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movb {{.*}}(%rip), %al
+; CHECK-NEXT:    movq (%rdi), %rcx
+; CHECK-NEXT:    addb $-1, %al
+; CHECK-NEXT:    adcq %rcx, (%rsi)
+; CHECK-NEXT:    movq 8(%rdi), %rax
+; CHECK-NEXT:    adcq %rax, 8(%rsi)
+; CHECK-NEXT:    movq 16(%rdi), %rax
+; CHECK-NEXT:    adcq %rax, 16(%rsi)
+; CHECK-NEXT:    movq 24(%rdi), %rax
+; CHECK-NEXT:    adcq %rax, 24(%rsi)
+; CHECK-NEXT:    setb {{.*}}(%rip)
+; CHECK-NEXT:    retq
+  %3 = load i8, i8* @_ZL1c, align 1
+  %4 = load i64, i64* %0, align 8
+  %5 = load i64, i64* %1, align 8
+  %6 = tail call { i8, i64 } @llvm.x86.addcarry.64(i8 %3, i64 %4, i64 %5)
+  %7 = extractvalue { i8, i64 } %6, 1
+  store i64 %7, i64* %1, align 8
+  %8 = extractvalue { i8, i64 } %6, 0
+  %9 = getelementptr inbounds i64, i64* %0, i64 1
+  %10 = load i64, i64* %9, align 8
+  %11 = getelementptr inbounds i64, i64* %1, i64 1
+  %12 = load i64, i64* %11, align 8
+  %13 = tail call { i8, i64 } @llvm.x86.addcarry.64(i8 %8, i64 %10, i64 %12)
+  %14 = extractvalue { i8, i64 } %13, 1
+  store i64 %14, i64* %11, align 8
+  %15 = extractvalue { i8, i64 } %13, 0
+  %16 = getelementptr inbounds i64, i64* %0, i64 2
+  %17 = load i64, i64* %16, align 8
+  %18 = getelementptr inbounds i64, i64* %1, i64 2
+  %19 = load i64, i64* %18, align 8
+  %20 = tail call { i8, i64 } @llvm.x86.addcarry.64(i8 %15, i64 %17, i64 %19)
+  %21 = extractvalue { i8, i64 } %20, 1
+  store i64 %21, i64* %18, align 8
+  %22 = extractvalue { i8, i64 } %20, 0
+  %23 = getelementptr inbounds i64, i64* %0, i64 3
+  %24 = load i64, i64* %23, align 8
+  %25 = getelementptr inbounds i64, i64* %1, i64 3
+  %26 = load i64, i64* %25, align 8
+  %27 = tail call { i8, i64 } @llvm.x86.addcarry.64(i8 %22, i64 %24, i64 %26)
+  %28 = extractvalue { i8, i64 } %27, 1
+  store i64 %28, i64* %25, align 8
+  %29 = extractvalue { i8, i64 } %27, 0
+  store i8 %29, i8* @_ZL1c, align 1
+  ret void
+}
+
+declare { i8, i64 } @llvm.x86.addcarry.64(i8, i64, i64)
diff --git a/test/CodeGen/X86/pr34381.ll b/test/CodeGen/X86/pr34381.ll
index 3053ddda5f89c48e40c6952c83fddc1d311230a8..831b1d23402dddf4058bc74f229c076d7569bac7 100644
--- a/test/CodeGen/X86/pr34381.ll
+++ b/test/CodeGen/X86/pr34381.ll
@@ -13,11 +13,9 @@ define void @_Z3foov() {
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    movsbl {{.*}}(%rip), %eax
 ; CHECK-NEXT:    negl %eax
-; CHECK-NEXT:    cmpl %eax, {{.*}}(%rip)
-; CHECK-NEXT:    setb %al
 ; CHECK-NEXT:    xorl %ecx, %ecx
-; CHECK-NEXT:    addb $-1, %al
-; CHECK-NEXT:    sete %cl
+; CHECK-NEXT:    cmpl %eax, {{.*}}(%rip)
+; CHECK-NEXT:    setb %cl
 ; CHECK-NEXT:    movl %ecx, {{.*}}(%rip)
 ; CHECK-NEXT:    movb {{.*}}(%rip), %al
 ; CHECK-NEXT:    movb %al, {{.*}}(%rip)
diff --git a/test/CodeGen/X86/pr34592.ll b/test/CodeGen/X86/pr34592.ll
index b010429d973e6aa7e70dc7ff477954ae5f32775d..110e109f2b4ddc0376186854c835cf2875e3bc4c 100644
--- a/test/CodeGen/X86/pr34592.ll
+++ b/test/CodeGen/X86/pr34592.ll
@@ -10,7 +10,7 @@ define <16 x i64> @pluto(<16 x i64> %arg, <16 x i64> %arg1, <16 x i64> %arg2, <1
 ; CHECK-NEXT:    movq %rsp, %rbp
 ; CHECK-NEXT:    .cfi_def_cfa_register %rbp
 ; CHECK-NEXT:    andq $-32, %rsp
-; CHECK-NEXT:    subq $352, %rsp # imm = 0x160
+; CHECK-NEXT:    subq $320, %rsp # imm = 0x140
 ; CHECK-NEXT:    vmovaps 240(%rbp), %ymm8
 ; CHECK-NEXT:    vmovaps 208(%rbp), %ymm9
 ; CHECK-NEXT:    vmovaps 176(%rbp), %ymm10
@@ -24,8 +24,6 @@ define <16 x i64> @pluto(<16 x i64> %arg, <16 x i64> %arg1, <16 x i64> %arg2, <1
 ; CHECK-NEXT:    vmovdqa %xmm6, %xmm9
 ; CHECK-NEXT:    # kill: def $ymm9 killed $xmm9
 ; CHECK-NEXT:    vmovaps %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; CHECK-NEXT:    vxorps %xmm0, %xmm0, %xmm0
-; CHECK-NEXT:    vmovaps %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; CHECK-NEXT:    # implicit-def: $ymm0
 ; CHECK-NEXT:    vinserti128 $1, %xmm6, %ymm0, %ymm0
 ; CHECK-NEXT:    vpalignr {{.*#+}} ymm11 = ymm2[8,9,10,11,12,13,14,15],ymm11[0,1,2,3,4,5,6,7],ymm2[24,25,26,27,28,29,30,31],ymm11[16,17,18,19,20,21,22,23]
@@ -34,9 +32,10 @@ define <16 x i64> @pluto(<16 x i64> %arg, <16 x i64> %arg1, <16 x i64> %arg2, <1
 ; CHECK-NEXT:    vmovaps %xmm2, %xmm6
 ; CHECK-NEXT:    # implicit-def: $ymm2
 ; CHECK-NEXT:    vinserti128 $1, %xmm6, %ymm2, %ymm2
-; CHECK-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %ymm11 # 32-byte Reload
-; CHECK-NEXT:    vpblendd {{.*#+}} ymm11 = ymm7[0,1],ymm11[2,3],ymm7[4,5],ymm11[6,7]
-; CHECK-NEXT:    vpermq {{.*#+}} ymm11 = ymm11[2,1,2,3]
+; CHECK-NEXT:    vextracti128 $1, %ymm7, %xmm6
+; CHECK-NEXT:    vmovq {{.*#+}} xmm6 = xmm6[0],zero
+; CHECK-NEXT:    # implicit-def: $ymm11
+; CHECK-NEXT:    vmovaps %xmm6, %xmm11
 ; CHECK-NEXT:    vpblendd {{.*#+}} ymm2 = ymm11[0,1,2,3],ymm2[4,5,6,7]
 ; CHECK-NEXT:    vmovaps %xmm7, %xmm6
 ; CHECK-NEXT:    vpslldq {{.*#+}} xmm6 = zero,zero,zero,zero,zero,zero,zero,zero,xmm6[0,1,2,3,4,5,6,7]
diff --git a/test/CodeGen/X86/pr35765.ll b/test/CodeGen/X86/pr35765.ll
index 6ff504d1d926ee2a5c6de643d9244b9509ce4e2a..1c6035fdc11084cbe7dfb2d6c12009623c7a8361 100644
--- a/test/CodeGen/X86/pr35765.ll
+++ b/test/CodeGen/X86/pr35765.ll
@@ -9,10 +9,9 @@
 define void @PR35765() {
 ; CHECK-LABEL: PR35765:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    movzwl {{.*}}(%rip), %ecx
-; CHECK-NEXT:    addl $-1398, %ecx # imm = 0xFA8A
+; CHECK-NEXT:    movb {{.*}}(%rip), %cl
+; CHECK-NEXT:    addb $-118, %cl
 ; CHECK-NEXT:    movl $4, %eax
-; CHECK-NEXT:    # kill: def $cl killed $cl killed $ecx
 ; CHECK-NEXT:    shll %cl, %eax
 ; CHECK-NEXT:    movzwl {{.*}}(%rip), %ecx
 ; CHECK-NEXT:    movzwl {{.*}}(%rip), %edx
diff --git a/test/CodeGen/X86/pr35982.ll b/test/CodeGen/X86/pr35982.ll
index 6f92f1adb09dd7375e45ddab2fbfec9c728e81ad..5f0ac0ef1c6176201a73be3994b2b2f44bb66ed3 100644
--- a/test/CodeGen/X86/pr35982.ll
+++ b/test/CodeGen/X86/pr35982.ll
@@ -3,49 +3,27 @@
 ; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+mmx,+3dnowa -post-RA-scheduler=true | FileCheck %s --check-prefixes=CHECK,POST
 
 define float @PR35982_emms(<1 x i64>) nounwind {
-; NOPOST-LABEL: PR35982_emms:
-; NOPOST:       # %bb.0:
-; NOPOST-NEXT:    pushl %ebp
-; NOPOST-NEXT:    movl %esp, %ebp
-; NOPOST-NEXT:    andl $-8, %esp
-; NOPOST-NEXT:    subl $16, %esp
-; NOPOST-NEXT:    movl 8(%ebp), %eax
-; NOPOST-NEXT:    movl 12(%ebp), %ecx
-; NOPOST-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; NOPOST-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; NOPOST-NEXT:    movq {{[0-9]+}}(%esp), %mm0
-; NOPOST-NEXT:    punpckhdq %mm0, %mm0 # mm0 = mm0[1,1]
-; NOPOST-NEXT:    movd %mm0, %ecx
-; NOPOST-NEXT:    emms
-; NOPOST-NEXT:    movl %eax, (%esp)
-; NOPOST-NEXT:    fildl (%esp)
-; NOPOST-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; NOPOST-NEXT:    fiaddl {{[0-9]+}}(%esp)
-; NOPOST-NEXT:    movl %ebp, %esp
-; NOPOST-NEXT:    popl %ebp
-; NOPOST-NEXT:    retl
-;
-; POST-LABEL: PR35982_emms:
-; POST:       # %bb.0:
-; POST-NEXT:    pushl %ebp
-; POST-NEXT:    movl %esp, %ebp
-; POST-NEXT:    andl $-8, %esp
-; POST-NEXT:    subl $16, %esp
-; POST-NEXT:    movl 8(%ebp), %eax
-; POST-NEXT:    movl 12(%ebp), %ecx
-; POST-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; POST-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; POST-NEXT:    movq {{[0-9]+}}(%esp), %mm0
-; POST-NEXT:    emms
-; POST-NEXT:    movl %eax, (%esp)
-; POST-NEXT:    fildl (%esp)
-; POST-NEXT:    punpckhdq %mm0, %mm0 # mm0 = mm0[1,1]
-; POST-NEXT:    movd %mm0, %ecx
-; POST-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; POST-NEXT:    fiaddl {{[0-9]+}}(%esp)
-; POST-NEXT:    movl %ebp, %esp
-; POST-NEXT:    popl %ebp
-; POST-NEXT:    retl
+; CHECK-LABEL: PR35982_emms:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushl %ebp
+; CHECK-NEXT:    movl %esp, %ebp
+; CHECK-NEXT:    andl $-8, %esp
+; CHECK-NEXT:    subl $16, %esp
+; CHECK-NEXT:    movl 8(%ebp), %eax
+; CHECK-NEXT:    movl 12(%ebp), %ecx
+; CHECK-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; CHECK-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; CHECK-NEXT:    movq {{[0-9]+}}(%esp), %mm0
+; CHECK-NEXT:    punpckhdq %mm0, %mm0 # mm0 = mm0[1,1]
+; CHECK-NEXT:    movd %mm0, %ecx
+; CHECK-NEXT:    emms
+; CHECK-NEXT:    movl %eax, (%esp)
+; CHECK-NEXT:    fildl (%esp)
+; CHECK-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; CHECK-NEXT:    fiaddl {{[0-9]+}}(%esp)
+; CHECK-NEXT:    movl %ebp, %esp
+; CHECK-NEXT:    popl %ebp
+; CHECK-NEXT:    retl
   %2 = bitcast <1 x i64> %0 to <2 x i32>
   %3 = extractelement <2 x i32> %2, i32 0
   %4 = extractelement <1 x i64> %0, i32 0
@@ -61,49 +39,27 @@ define float @PR35982_emms(<1 x i64>) nounwind {
 }
 
 define float @PR35982_femms(<1 x i64>) nounwind {
-; NOPOST-LABEL: PR35982_femms:
-; NOPOST:       # %bb.0:
-; NOPOST-NEXT:    pushl %ebp
-; NOPOST-NEXT:    movl %esp, %ebp
-; NOPOST-NEXT:    andl $-8, %esp
-; NOPOST-NEXT:    subl $16, %esp
-; NOPOST-NEXT:    movl 8(%ebp), %eax
-; NOPOST-NEXT:    movl 12(%ebp), %ecx
-; NOPOST-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; NOPOST-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; NOPOST-NEXT:    movq {{[0-9]+}}(%esp), %mm0
-; NOPOST-NEXT:    punpckhdq %mm0, %mm0 # mm0 = mm0[1,1]
-; NOPOST-NEXT:    movd %mm0, %ecx
-; NOPOST-NEXT:    femms
-; NOPOST-NEXT:    movl %eax, (%esp)
-; NOPOST-NEXT:    fildl (%esp)
-; NOPOST-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; NOPOST-NEXT:    fiaddl {{[0-9]+}}(%esp)
-; NOPOST-NEXT:    movl %ebp, %esp
-; NOPOST-NEXT:    popl %ebp
-; NOPOST-NEXT:    retl
-;
-; POST-LABEL: PR35982_femms:
-; POST:       # %bb.0:
-; POST-NEXT:    pushl %ebp
-; POST-NEXT:    movl %esp, %ebp
-; POST-NEXT:    andl $-8, %esp
-; POST-NEXT:    subl $16, %esp
-; POST-NEXT:    movl 8(%ebp), %eax
-; POST-NEXT:    movl 12(%ebp), %ecx
-; POST-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; POST-NEXT:    movl %eax, {{[0-9]+}}(%esp)
-; POST-NEXT:    movq {{[0-9]+}}(%esp), %mm0
-; POST-NEXT:    femms
-; POST-NEXT:    movl %eax, (%esp)
-; POST-NEXT:    fildl (%esp)
-; POST-NEXT:    punpckhdq %mm0, %mm0 # mm0 = mm0[1,1]
-; POST-NEXT:    movd %mm0, %ecx
-; POST-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
-; POST-NEXT:    fiaddl {{[0-9]+}}(%esp)
-; POST-NEXT:    movl %ebp, %esp
-; POST-NEXT:    popl %ebp
-; POST-NEXT:    retl
+; CHECK-LABEL: PR35982_femms:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushl %ebp
+; CHECK-NEXT:    movl %esp, %ebp
+; CHECK-NEXT:    andl $-8, %esp
+; CHECK-NEXT:    subl $16, %esp
+; CHECK-NEXT:    movl 8(%ebp), %eax
+; CHECK-NEXT:    movl 12(%ebp), %ecx
+; CHECK-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; CHECK-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; CHECK-NEXT:    movq {{[0-9]+}}(%esp), %mm0
+; CHECK-NEXT:    punpckhdq %mm0, %mm0 # mm0 = mm0[1,1]
+; CHECK-NEXT:    movd %mm0, %ecx
+; CHECK-NEXT:    femms
+; CHECK-NEXT:    movl %eax, (%esp)
+; CHECK-NEXT:    fildl (%esp)
+; CHECK-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; CHECK-NEXT:    fiaddl {{[0-9]+}}(%esp)
+; CHECK-NEXT:    movl %ebp, %esp
+; CHECK-NEXT:    popl %ebp
+; CHECK-NEXT:    retl
   %2 = bitcast <1 x i64> %0 to <2 x i32>
   %3 = extractelement <2 x i32> %2, i32 0
   %4 = extractelement <1 x i64> %0, i32 0
diff --git a/test/CodeGen/X86/pr37499.ll b/test/CodeGen/X86/pr37499.ll
index 3fd2ca6b1d84438a30b5390c1059d161e6109b51..2995017275c18d7f3d59b512ac657a74771b1bfd 100644
--- a/test/CodeGen/X86/pr37499.ll
+++ b/test/CodeGen/X86/pr37499.ll
@@ -4,7 +4,11 @@
 define <2 x i64> @undef_tval() {
 ; CHECK-LABEL: undef_tval:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vmovaps {{.*#+}} xmm0 = [1,1,1,1,1,1,1,1]
+; CHECK-NEXT:    vmovdqa {{.*#+}} xmm0 = [1,1,1,1,1,1,1,1]
+; CHECK-NEXT:    movb $1, %al
+; CHECK-NEXT:    kmovw %eax, %k1
+; CHECK-NEXT:    vpmovqw %zmm0, %xmm0 {%k1}
+; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retq
   %1 = tail call <8 x i16> @llvm.x86.avx512.mask.pmov.qw.512(<8 x i64> undef, <8 x i16> <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>, i8 1) #3
   %2 = bitcast <8 x i16> %1 to <2 x i64>
diff --git a/test/CodeGen/X86/pr38217.ll b/test/CodeGen/X86/pr38217.ll
new file mode 100644
index 0000000000000000000000000000000000000000..951d4645d5d0c3681c10111b1477fd05386d8c8e
--- /dev/null
+++ b/test/CodeGen/X86/pr38217.ll
@@ -0,0 +1,76 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown | FileCheck %s
+
+@_ZL11DIGIT_TABLE = constant [201 x i8] c"00010203040506070809101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899\00", align 16
+
+define void @_Z12d2s_bufferedmPc(i64, i8* nocapture) {
+; CHECK-LABEL: _Z12d2s_bufferedmPc:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    cmpq $10000, %rdi # imm = 0x2710
+; CHECK-NEXT:    jb .LBB0_3
+; CHECK-NEXT:  # %bb.1: # %.preheader
+; CHECK-NEXT:    movq %rdi, %r9
+; CHECK-NEXT:    xorl %r10d, %r10d
+; CHECK-NEXT:    movabsq $3777893186295716171, %r8 # imm = 0x346DC5D63886594B
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB0_2: # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    movq %r9, %rax
+; CHECK-NEXT:    mulq %r8
+; CHECK-NEXT:    shrq $11, %rdx
+; CHECK-NEXT:    imulq $10000, %rdx, %rax # imm = 0x2710
+; CHECK-NEXT:    movq %r9, %rdi
+; CHECK-NEXT:    subq %rax, %rdi
+; CHECK-NEXT:    imulq $1374389535, %rdi, %rax # imm = 0x51EB851F
+; CHECK-NEXT:    shrq $37, %rax
+; CHECK-NEXT:    imull $100, %eax, %ecx
+; CHECK-NEXT:    subl %ecx, %edi
+; CHECK-NEXT:    movl %r10d, %r11d
+; CHECK-NEXT:    movq %rsi, %rcx
+; CHECK-NEXT:    subq %r11, %rcx
+; CHECK-NEXT:    movzwl _ZL11DIGIT_TABLE(%rdi,%rdi), %edi
+; CHECK-NEXT:    movw %di, -1(%rcx)
+; CHECK-NEXT:    movzwl _ZL11DIGIT_TABLE(%rax,%rax), %eax
+; CHECK-NEXT:    movw %ax, -3(%rcx)
+; CHECK-NEXT:    addl $4, %r10d
+; CHECK-NEXT:    cmpq $99999999, %r9 # imm = 0x5F5E0FF
+; CHECK-NEXT:    movq %rdx, %r9
+; CHECK-NEXT:    ja .LBB0_2
+; CHECK-NEXT:  .LBB0_3:
+; CHECK-NEXT:    retq
+  %3 = icmp ugt i64 %0, 9999
+  br i1 %3, label %4, label %31
+
+; <label>:4:                                      ; preds = %2, %4
+  %5 = phi i64 [ %9, %4 ], [ %0, %2 ]
+  %6 = phi i32 [ %29, %4 ], [ 0, %2 ]
+  %7 = urem i64 %5, 10000
+  %8 = trunc i64 %7 to i32
+  %9 = udiv i64 %5, 10000
+  %10 = urem i32 %8, 100
+  %11 = shl nuw nsw i32 %10, 1
+  %12 = udiv i32 %8, 100
+  %13 = shl nuw nsw i32 %12, 1
+  %14 = zext i32 %6 to i64
+  %15 = sub nsw i64 0, %14
+  %16 = getelementptr inbounds i8, i8* %1, i64 %15
+  %17 = getelementptr inbounds i8, i8* %16, i64 -1
+  %18 = zext i32 %11 to i64
+  %19 = getelementptr inbounds [201 x i8], [201 x i8]* @_ZL11DIGIT_TABLE, i64 0, i64 %18
+  %20 = bitcast i8* %19 to i16*
+  %21 = bitcast i8* %17 to i16*
+  %22 = load i16, i16* %20, align 2
+  store i16 %22, i16* %21, align 1
+  %23 = getelementptr inbounds i8, i8* %16, i64 -3
+  %24 = zext i32 %13 to i64
+  %25 = getelementptr inbounds [201 x i8], [201 x i8]* @_ZL11DIGIT_TABLE, i64 0, i64 %24
+  %26 = bitcast i8* %25 to i16*
+  %27 = bitcast i8* %23 to i16*
+  %28 = load i16, i16* %26, align 2
+  store i16 %28, i16* %27, align 1
+  %29 = add i32 %6, 4
+  %30 = icmp ugt i64 %5, 99999999
+  br i1 %30, label %4, label %31
+
+; <label>:31:                                     ; preds = %4, %2
+  ret void
+}
diff --git a/test/CodeGen/X86/pr38743.ll b/test/CodeGen/X86/pr38743.ll
new file mode 100644
index 0000000000000000000000000000000000000000..ac5d48ef5f55c35cfb3a817e92e7ada6245f5bf4
--- /dev/null
+++ b/test/CodeGen/X86/pr38743.ll
@@ -0,0 +1,94 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown | FileCheck %s
+
+%0 = type { %1 }
+%1 = type { %2 }
+%2 = type { %3 }
+%3 = type { %4 }
+%4 = type { %5 }
+%5 = type { i64, i64, i8* }
+%6 = type { %7, [23 x i8] }
+%7 = type { i8 }
+
+@.str.16 = external dso_local unnamed_addr constant [16 x i8], align 1
+@.str.17 = external dso_local unnamed_addr constant [12 x i8], align 1
+@.str.18 = external dso_local unnamed_addr constant [15 x i8], align 1
+
+declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture writeonly, i8* nocapture readonly, i64, i1) #0
+
+define void @pr38743() #1 align 2 {
+; CHECK-LABEL: pr38743:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    cmpl $3, %eax
+; CHECK-NEXT:    je .LBB0_4
+; CHECK-NEXT:  # %bb.1: # %bb
+; CHECK-NEXT:    cmpl $1, %eax
+; CHECK-NEXT:    je .LBB0_2
+; CHECK-NEXT:  # %bb.3: # %bb5
+; CHECK-NEXT:    movzwl .str.17+{{.*}}(%rip), %eax
+; CHECK-NEXT:    movw %ax, -{{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movq {{.*}}(%rip), %rax
+; CHECK-NEXT:    jmp .LBB0_5
+; CHECK-NEXT:  .LBB0_4: # %bb8
+; CHECK-NEXT:    movq .str.18+{{.*}}(%rip), %rax
+; CHECK-NEXT:    movq %rax, -{{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movq {{.*}}(%rip), %rax
+; CHECK-NEXT:    jmp .LBB0_5
+; CHECK-NEXT:  .LBB0_2: # %bb2
+; CHECK-NEXT:    movq .str.16+{{.*}}(%rip), %rax
+; CHECK-NEXT:    movq %rax, -{{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movq {{.*}}(%rip), %rax
+; CHECK-NEXT:  .LBB0_5: # %bb12
+; CHECK-NEXT:    movq %rax, -{{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movq -{{[0-9]+}}(%rsp), %rax
+; CHECK-NEXT:    movq %rax, (%rax)
+; CHECK-NEXT:    movb -{{[0-9]+}}(%rsp), %al
+; CHECK-NEXT:    movq -{{[0-9]+}}(%rsp), %rcx
+; CHECK-NEXT:    movzwl -{{[0-9]+}}(%rsp), %edx
+; CHECK-NEXT:    movl -{{[0-9]+}}(%rsp), %esi
+; CHECK-NEXT:    movb -{{[0-9]+}}(%rsp), %dil
+; CHECK-NEXT:    movb %al, (%rax)
+; CHECK-NEXT:    movq %rcx, 1(%rax)
+; CHECK-NEXT:    movw %dx, 9(%rax)
+; CHECK-NEXT:    movl %esi, 11(%rax)
+; CHECK-NEXT:    movb %dil, 15(%rax)
+; CHECK-NEXT:    retq
+bb:
+  %tmp = alloca %0, align 16
+  %tmp1 = bitcast %0* %tmp to i8*
+  switch i32 undef, label %bb11 [
+    i32 1, label %bb2
+    i32 4, label %bb5
+    i32 2, label %bb5
+    i32 3, label %bb8
+  ]
+
+bb2:                                              ; preds = %bb
+  %tmp3 = bitcast %0* %tmp to %6*
+  %tmp4 = getelementptr inbounds %6, %6* %tmp3, i64 0, i32 1, i64 0
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* nonnull align 1 %tmp4, i8* align 1 getelementptr inbounds ([16 x i8], [16 x i8]* @.str.16, i64 0, i64 0), i64 15, i1 false)
+  br label %bb12
+
+bb5:                                              ; preds = %bb, %bb
+  %tmp6 = bitcast %0* %tmp to %6*
+  %tmp7 = getelementptr inbounds %6, %6* %tmp6, i64 0, i32 1, i64 0
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* nonnull align 1 %tmp7, i8* align 1 getelementptr inbounds ([12 x i8], [12 x i8]* @.str.17, i64 0, i64 0), i64 10, i1 false)
+  br label %bb12
+
+bb8:                                              ; preds = %bb
+  %tmp9 = bitcast %0* %tmp to %6*
+  %tmp10 = getelementptr inbounds %6, %6* %tmp9, i64 0, i32 1, i64 0
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* nonnull align 1 %tmp10, i8* align 1 getelementptr inbounds ([15 x i8], [15 x i8]* @.str.18, i64 0, i64 0), i64 14, i1 false)
+  br label %bb12
+
+bb11:                                             ; preds = %bb
+  unreachable
+
+bb12:                                             ; preds = %bb8, %bb5, %bb2
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* nonnull align 8 undef, i8* nonnull align 16 %tmp1, i64 24, i1 false) #2
+  ret void
+}
+
+attributes #0 = { argmemonly nounwind }
+attributes #1 = { "target-features"="+sse,+sse2,+sse3,+sse4.2" }
+attributes #2 = { nounwind }
diff --git a/test/CodeGen/X86/pr39098.ll b/test/CodeGen/X86/pr39098.ll
new file mode 100644
index 0000000000000000000000000000000000000000..fd586d7bb6945395b88c6cf964b92af7995845b9
--- /dev/null
+++ b/test/CodeGen/X86/pr39098.ll
@@ -0,0 +1,20 @@
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown | FileCheck %s
+
+define dso_local void @test_cancel2(i40* %p1, i40* %p2) {
+; CHECK:       # %entry
+; CHECK-NEXT:  movl    (%rdi), %eax
+; CHECK-NEXT:  shrl    %eax
+; CHECK-NEXT:  andl    $524287, %eax
+; CHECK-NEXT:  movl    %eax, (%rsi)
+; CHECK-NEXT:  movb    $0, 4(%rsi)
+; CHECK-NEXT:  retq
+entry:
+  %0 = load i40, i40* %p1, align 8
+  %shl414 = shl i40 %0, 19
+  %unsclear415 = and i40 %shl414, 549755813887
+  %shr416 = lshr i40 %unsclear415, 20
+  %unsclear417 = and i40 %shr416, 549755813887
+  store i40 %unsclear417, i40* %p2, align 8
+  ret void
+}
+
diff --git a/test/CodeGen/X86/pr39187-g.ll b/test/CodeGen/X86/pr39187-g.ll
index 46da64111224411654c93500a5f6e663138cf663..2c9dcc5dfff35576d7b6aa6043b42c826dd0a923 100644
--- a/test/CodeGen/X86/pr39187-g.ll
+++ b/test/CodeGen/X86/pr39187-g.ll
@@ -7,22 +7,8 @@
 ; location keep references to its basic block, causing the debug information
 ; to become ambiguous. It causes the debugger to display unreached lines.
 
-; Change the debug location associated with the hoisted instruction, to
-; the debug location from the insertion point in the 'if' block.
-
-; The insertion point is the previous non-debug instruction before the
-; terminator in the parent basic block of the hoisted instruction.
-
-; IR with '-g':
-;
-;  [...]
-;  %frombool = zext i1 %cmp to i8, !dbg !26
-;  call void @llvm.dbg.value(metadata i8 %frombool, metadata !15, metadata !DIExpression()), !dbg !26
-;  call void @llvm.dbg.value(metadata i32 0, metadata !17, metadata !DIExpression()), !dbg !27
-;  br i1 %cmp, label %if.then, label %if.else
-;  [...]
-;
-; Insertion point is: %frombool = zext i1 %cmp to i8, !dbg !26
+; Check that hoisted instructions get unknown-location line numbers -- there
+; is no correct line number for code that has been common'd in this way.
 
 ; IR generated with:
 ; clang -S -g -gno-column-info -O2 -emit-llvm pr39187.cpp -o pr39187-g.ll -mllvm -opt-bisect-limit=10
@@ -54,7 +40,8 @@
 ; CHECK:  %frombool = zext i1 %cmp to i8, !dbg !16
 ; CHECK:  call void @llvm.dbg.value(metadata i8 %frombool, metadata !13, metadata !DIExpression()), !dbg !16
 ; CHECK:  call void @llvm.dbg.value(metadata i32 0, metadata !15, metadata !DIExpression()), !dbg !17
-; CHECK:  %. = select i1 %cmp, i32 8, i32 4, !dbg !16
+; CHECK:  %. = select i1 %cmp, i32 8, i32 4, !dbg ![[MERGEDLOC:[0-9]+]]
+; CHECK:  ![[MERGEDLOC]] = !DILocation(line: 0, scope: !7)
 
 ; ModuleID = 'pr39187.cpp'
 source_filename = "pr39187.cpp"
@@ -77,11 +64,11 @@ entry:
 
 if.then:                                          ; preds = %entry
   call void @llvm.dbg.value(metadata i32 8, metadata !14, metadata !DIExpression()), !dbg !25
-  br label %if.end
+  br label %if.end, !dbg !25
 
 if.else:                                          ; preds = %entry
-  call void @llvm.dbg.value(metadata i32 4, metadata !14, metadata !DIExpression()), !dbg !25
-  br label %if.end
+  call void @llvm.dbg.value(metadata i32 4, metadata !14, metadata !DIExpression()), !dbg !27
+  br label %if.end, !dbg !27
 
 if.end:                                           ; preds = %if.else, %if.then
   %beards.0 = phi i32 [ 8, %if.then ], [ 4, %if.else ]
diff --git a/test/CodeGen/X86/pr40289-64bit.ll b/test/CodeGen/X86/pr40289-64bit.ll
new file mode 100644
index 0000000000000000000000000000000000000000..fd7e2d55bdcaff416112bdd855d82198b9eb841d
--- /dev/null
+++ b/test/CodeGen/X86/pr40289-64bit.ll
@@ -0,0 +1,10 @@
+; RUN: llc < %s -mtriple=x86_64-pc-windows-msvc | FileCheck %s
+
+define cc 92 < 9 x i64 > @clobber() {
+  %1 = alloca i64
+  %2 = load volatile i64, i64* %1
+  ret < 9 x i64 > undef
+  ; CHECK-LABEL: clobber:
+  ; CHECK-NOT: popq %rsp
+  ; CHECK: addq $8, %rsp
+}
diff --git a/test/CodeGen/X86/pr40289.ll b/test/CodeGen/X86/pr40289.ll
new file mode 100644
index 0000000000000000000000000000000000000000..abcb5fa41fa672c3f74e9a787a101e9158513f44
--- /dev/null
+++ b/test/CodeGen/X86/pr40289.ll
@@ -0,0 +1,10 @@
+; RUN: llc < %s -mtriple=i686-pc-windows-msvc | FileCheck %s
+
+define < 3 x i32 > @clobber() {
+  %1 = alloca i32
+  %2 = load volatile i32, i32* %1
+  ret < 3 x i32 > undef
+  ; CHECK-LABEL: clobber:
+  ; CHECK-NOT: popl %esp
+  ; CHECK: addl $4, %esp
+}
diff --git a/test/CodeGen/X86/pr40529.ll b/test/CodeGen/X86/pr40529.ll
new file mode 100644
index 0000000000000000000000000000000000000000..bbbf8e95019a43b4ed268da7a5dc37dbb2ac2d11
--- /dev/null
+++ b/test/CodeGen/X86/pr40529.ll
@@ -0,0 +1,43 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown-linux -mcpu=x86-64 | FileCheck %s
+
+define x86_fp80 @rem_pio2l_min(x86_fp80 %z) {
+; CHECK-LABEL: rem_pio2l_min:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fnstcw -{{[0-9]+}}(%rsp)
+; CHECK-NEXT:    fldt {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movzwl -{{[0-9]+}}(%rsp), %eax
+; CHECK-NEXT:    movw $3199, -{{[0-9]+}}(%rsp) # imm = 0xC7F
+; CHECK-NEXT:    fldcw -{{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movw %ax, -{{[0-9]+}}(%rsp)
+; CHECK-NEXT:    fistl -{{[0-9]+}}(%rsp)
+; CHECK-NEXT:    fldcw -{{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movl -{{[0-9]+}}(%rsp), %eax
+; CHECK-NEXT:    movl %eax, -{{[0-9]+}}(%rsp)
+; CHECK-NEXT:    fisubl -{{[0-9]+}}(%rsp)
+; CHECK-NEXT:    flds {{.*}}(%rip)
+; CHECK-NEXT:    fnstcw -{{[0-9]+}}(%rsp)
+; CHECK-NEXT:    fmul %st, %st(1)
+; CHECK-NEXT:    movzwl -{{[0-9]+}}(%rsp), %eax
+; CHECK-NEXT:    movw $3199, -{{[0-9]+}}(%rsp) # imm = 0xC7F
+; CHECK-NEXT:    fldcw -{{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movw %ax, -{{[0-9]+}}(%rsp)
+; CHECK-NEXT:    fxch %st(1)
+; CHECK-NEXT:    fistl -{{[0-9]+}}(%rsp)
+; CHECK-NEXT:    fldcw -{{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movl -{{[0-9]+}}(%rsp), %eax
+; CHECK-NEXT:    movl %eax, -{{[0-9]+}}(%rsp)
+; CHECK-NEXT:    fisubl -{{[0-9]+}}(%rsp)
+; CHECK-NEXT:    fmulp %st, %st(1)
+; CHECK-NEXT:    retq
+entry:
+  %conv = fptosi x86_fp80 %z to i32
+  %conv1 = sitofp i32 %conv to x86_fp80
+  %sub = fsub x86_fp80 %z, %conv1
+  %mul = fmul x86_fp80 %sub, 0xK40178000000000000000
+  %conv2 = fptosi x86_fp80 %mul to i32
+  %conv3 = sitofp i32 %conv2 to x86_fp80
+  %sub4 = fsub x86_fp80 %mul, %conv3
+  %mul5 = fmul x86_fp80 %sub4, 0xK40178000000000000000
+  ret x86_fp80 %mul5
+}
diff --git a/test/CodeGen/X86/pr40539.ll b/test/CodeGen/X86/pr40539.ll
new file mode 100644
index 0000000000000000000000000000000000000000..f2135cd2e73b2d47ba5270abd9f4ff8d8bebca0f
--- /dev/null
+++ b/test/CodeGen/X86/pr40539.ll
@@ -0,0 +1,70 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mcpu=pentium4 | FileCheck %s
+
+@f1 = global float 1.000000e+00, align 4
+
+define zeroext i1 @_Z9test_log2v() {
+; CHECK-LABEL: _Z9test_log2v:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushl %eax
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    flds f1
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    fld1
+; CHECK-NEXT:    fxch %st(1)
+; CHECK-NEXT:    fyl2x
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    fstps (%esp)
+; CHECK-NEXT:    xorps %xmm0, %xmm0
+; CHECK-NEXT:    cmpeqss (%esp), %xmm0
+; CHECK-NEXT:    movd %xmm0, %eax
+; CHECK-NEXT:    andl $1, %eax
+; CHECK-NEXT:    # kill: def $al killed $al killed $eax
+; CHECK-NEXT:    popl %ecx
+; CHECK-NEXT:    .cfi_def_cfa_offset 4
+; CHECK-NEXT:    retl
+entry:
+  %0 = load float, float* @f1, align 4
+  %1 = fpext float %0 to x86_fp80
+  %2 = tail call x86_fp80 asm "fld1; fxch; fyl2x", "={st},0,~{st(1)},~{dirflag},~{fpsr},~{flags}"(x86_fp80 %1)
+  %conv = fptrunc x86_fp80 %2 to float
+  %3 = fcmp oeq float %conv, 0.000000e+00
+  ret i1 %3
+}
+
+@fpi = external global float, align 4
+
+define zeroext i1 @_Z8test_cosv() {
+; CHECK-LABEL: _Z8test_cosv:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    subl $8, %esp
+; CHECK-NEXT:    .cfi_def_cfa_offset 12
+; CHECK-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; CHECK-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; CHECK-NEXT:    divss {{\.LCPI.*}}, %xmm0
+; CHECK-NEXT:    movss %xmm0, {{[0-9]+}}(%esp)
+; CHECK-NEXT:    flds {{[0-9]+}}(%esp)
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    fcos
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    fstps (%esp)
+; CHECK-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; CHECK-NEXT:    ucomiss %xmm0, %xmm1
+; CHECK-NEXT:    setae %cl
+; CHECK-NEXT:    ucomiss {{\.LCPI.*}}, %xmm0
+; CHECK-NEXT:    setae %al
+; CHECK-NEXT:    andb %cl, %al
+; CHECK-NEXT:    addl $8, %esp
+; CHECK-NEXT:    .cfi_def_cfa_offset 4
+; CHECK-NEXT:    retl
+entry:
+  %0 = load float, float* @fpi, align 4
+  %div = fdiv float %0, 6.000000e+00
+  %1 = fpext float %div to x86_fp80
+  %2 = tail call x86_fp80 asm "fcos", "={st},0,~{dirflag},~{fpsr},~{flags}"(x86_fp80 %1)
+  %conv = fptrunc x86_fp80 %2 to float
+  %cmp = fcmp ole float %conv, 0x3FEBD70A40000000
+  %cmp1 = fcmp oge float %conv, 0x3FEB851EC0000000
+  %or.cond = and i1 %cmp, %cmp1
+  ret i1 %or.cond
+}
diff --git a/test/CodeGen/X86/pr9743.ll b/test/CodeGen/X86/pr9743.ll
index ac3d457551086d94526f72eec9b28ce1f13081e8..976b74972574de4bcb5db90e46b29863264bb41d 100644
--- a/test/CodeGen/X86/pr9743.ll
+++ b/test/CodeGen/X86/pr9743.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -disable-fp-elim -asm-verbose=0 | FileCheck %s
+; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -frame-pointer=all -asm-verbose=0 | FileCheck %s
 
 define void @f() {
   ret void
diff --git a/test/CodeGen/X86/prefer-avx256-mask-extend.ll b/test/CodeGen/X86/prefer-avx256-mask-extend.ll
index b4f8e5b0b6c83ff10f1d5fd4018c7bf6bc9b938d..b4d452f2d3e8dad971e4f0000997b0eeea2dadae 100644
--- a/test/CodeGen/X86/prefer-avx256-mask-extend.ll
+++ b/test/CodeGen/X86/prefer-avx256-mask-extend.ll
@@ -48,11 +48,9 @@ define <16 x i8> @testv16i1_sext_v16i8(<8 x i32>* %p, <8 x i32>* %q) {
 ; AVX256-NEXT:    vpcmpeqd %ymm0, %ymm0, %ymm0
 ; AVX256-NEXT:    vmovdqa32 %ymm0, %ymm1 {%k2} {z}
 ; AVX256-NEXT:    vpmovdw %ymm1, %xmm1
-; AVX256-NEXT:    vpsrlw $8, %xmm1, %xmm1
 ; AVX256-NEXT:    vmovdqa32 %ymm0, %ymm0 {%k1} {z}
 ; AVX256-NEXT:    vpmovdw %ymm0, %xmm0
-; AVX256-NEXT:    vpsrlw $8, %xmm0, %xmm0
-; AVX256-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
+; AVX256-NEXT:    vpacksswb %xmm1, %xmm0, %xmm0
 ; AVX256-NEXT:    vzeroupper
 ; AVX256-NEXT:    retq
 ;
diff --git a/test/CodeGen/X86/prefer-avx256-mask-shuffle.ll b/test/CodeGen/X86/prefer-avx256-mask-shuffle.ll
index 92f6e27112660226e6a155cc55aa7d924bc63ddc..7f4480ceb6378d675cc5fd89572e4e68fc0cdf77 100644
--- a/test/CodeGen/X86/prefer-avx256-mask-shuffle.ll
+++ b/test/CodeGen/X86/prefer-avx256-mask-shuffle.ll
@@ -34,11 +34,9 @@ define <16 x i1> @shuf16i1_3_6_22_12_3_7_7_0_3_6_1_13_3_21_7_0(<8 x i32>* %a, <8
 ; AVX256VL-NEXT:    kshiftrw $8, %k0, %k2
 ; AVX256VL-NEXT:    vmovdqa32 %ymm0, %ymm1 {%k2} {z}
 ; AVX256VL-NEXT:    vpmovdw %ymm1, %xmm1
-; AVX256VL-NEXT:    vpsrlw $8, %xmm1, %xmm1
 ; AVX256VL-NEXT:    vmovdqa32 %ymm0, %ymm0 {%k1} {z}
 ; AVX256VL-NEXT:    vpmovdw %ymm0, %xmm0
-; AVX256VL-NEXT:    vpsrlw $8, %xmm0, %xmm0
-; AVX256VL-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
+; AVX256VL-NEXT:    vpacksswb %xmm1, %xmm0, %xmm0
 ; AVX256VL-NEXT:    vzeroupper
 ; AVX256VL-NEXT:    retq
 ;
@@ -169,11 +167,9 @@ define <32 x i1> @shuf32i1_3_6_22_12_3_7_7_0_3_6_1_13_3_21_7_0_3_6_22_12_3_7_7_0
 ; AVX256VL-NEXT:    kshiftrw $8, %k0, %k2
 ; AVX256VL-NEXT:    vmovdqa32 %ymm0, %ymm1 {%k2} {z}
 ; AVX256VL-NEXT:    vpmovdw %ymm1, %xmm1
-; AVX256VL-NEXT:    vpsrlw $8, %xmm1, %xmm1
 ; AVX256VL-NEXT:    vmovdqa32 %ymm0, %ymm0 {%k1} {z}
 ; AVX256VL-NEXT:    vpmovdw %ymm0, %xmm0
-; AVX256VL-NEXT:    vpsrlw $8, %xmm0, %xmm0
-; AVX256VL-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
+; AVX256VL-NEXT:    vpacksswb %xmm1, %xmm0, %xmm0
 ; AVX256VL-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
 ; AVX256VL-NEXT:    retq
 ;
diff --git a/test/CodeGen/X86/psubus.ll b/test/CodeGen/X86/psubus.ll
index 2fc70291b0a090479fb3e954786adddb92a324c0..39dbf4061061a46022e189c23ec35576e5ad2600 100644
--- a/test/CodeGen/X86/psubus.ll
+++ b/test/CodeGen/X86/psubus.ll
@@ -2725,3 +2725,47 @@ entry:
   %2 = add <32 x i16> %1, <i16 -1, i16 2200, i16 50, i16 114, i16 77, i16 70, i16 -123, i16 -9800, i16 -635, i16 -19567, i16 22, i16 -100, i16 -2534, i16 -34, i16 -55, i16 -70, i16 -1, i16 2200, i16 50, i16 114, i16 77, i16 70, i16 -123, i16 -9805, i16 -635, i16 -19567, i16 22, i16 -100, i16 -2534, i16 -346, i16 -55, i16 -70>
   ret <32 x i16> %2
 }
+
+; PR40083
+define i64 @test30(<8 x i16> %x) {
+; SSE-LABEL: test30:
+; SSE:       # %bb.0: # %entry
+; SSE-NEXT:    psubusw {{.*}}(%rip), %xmm0
+; SSE-NEXT:    movq %xmm0, %rax
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: test30:
+; AVX:       # %bb.0: # %entry
+; AVX-NEXT:    vpsubusw {{.*}}(%rip), %xmm0, %xmm0
+; AVX-NEXT:    vmovq %xmm0, %rax
+; AVX-NEXT:    retq
+entry:
+  %0 = icmp ugt <8 x i16> %x, <i16 1, i16 -2200, i16 -50, i16 -114, i16 undef, i16 undef, i16 undef, i16 undef>
+  %1 = select <8 x i1> %0, <8 x i16> %x, <8 x i16> <i16 1, i16 -2200, i16 -50, i16 -114, i16 undef, i16 undef, i16 undef, i16 undef>
+  %2 = add <8 x i16> %1, <i16 -1, i16 2200, i16 50, i16 114, i16 undef, i16 undef, i16 undef, i16 undef>
+  %3 = bitcast <8 x i16> %2 to <2 x i64>
+  %4 = extractelement <2 x i64> %3, i32 0
+  ret i64 %4
+}
+
+; PR40083
+define i64 @test31(<2 x i64> %x) {
+; SSE-LABEL: test31:
+; SSE:       # %bb.0:
+; SSE-NEXT:    psubusb {{.*}}(%rip), %xmm0
+; SSE-NEXT:    movq %xmm0, %rax
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: test31:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpsubusb {{.*}}(%rip), %xmm0, %xmm0
+; AVX-NEXT:    vmovq %xmm0, %rax
+; AVX-NEXT:    retq
+  %t0 = bitcast <2 x i64> %x to <16 x i8>
+  %cmp = icmp ugt <16 x i8> %t0, <i8 70, i8 70, i8 70, i8 70, i8 70, i8 70, i8 70, i8 70, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef>
+  %bop = add <16 x i8> %t0, <i8 -71, i8 -71, i8 -71, i8 -71, i8 -71, i8 -71, i8 -71, i8 -71, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef>
+  %sel = select <16 x i1> %cmp, <16 x i8> %bop, <16 x i8> zeroinitializer
+  %bc = bitcast <16 x i8> %sel to <2 x i64>
+  %ext = extractelement <2 x i64> %bc, i32 0
+  ret i64 %ext
+}
diff --git a/test/CodeGen/X86/rd-mod-wr-eflags.ll b/test/CodeGen/X86/rd-mod-wr-eflags.ll
index c49d5c91f61e50e38b54372c3a8586257ca8690b..410d736af7304a961868e9e878c63e8fbc1aae99 100644
--- a/test/CodeGen/X86/rd-mod-wr-eflags.ll
+++ b/test/CodeGen/X86/rd-mod-wr-eflags.ll
@@ -1,12 +1,18 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=x86_64-- | FileCheck %s
 
 %struct.obj = type { i64 }
 
-; CHECK: _Z7releaseP3obj
 define void @_Z7releaseP3obj(%struct.obj* nocapture %o) nounwind uwtable ssp {
+; CHECK-LABEL: _Z7releaseP3obj:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    decq (%rdi)
+; CHECK-NEXT:    je .LBB0_2
+; CHECK-NEXT:  # %bb.1: # %return
+; CHECK-NEXT:    retq
+; CHECK-NEXT:  .LBB0_2: # %if.end
+; CHECK-NEXT:    jmp free # TAILCALL
 entry:
-; CHECK: decq	(%{{rdi|rcx}})
-; CHECK-NEXT: je
   %refcnt = getelementptr inbounds %struct.obj, %struct.obj* %o, i64 0, i32 0
   %0 = load i64, i64* %refcnt, align 8
   %dec = add i64 %0, -1
@@ -28,11 +34,25 @@ return:                                           ; preds = %entry, %if.end
 @.str = private unnamed_addr constant [5 x i8] c"%ld\0A\00", align 1
 @b = common global i32 0, align 4
 
-; CHECK: test
 define i32 @test() nounwind uwtable ssp {
+; CHECK-LABEL: test:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    movq {{.*}}(%rip), %rsi
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    decq %rsi
+; CHECK-NEXT:    movq %rsi, {{.*}}(%rip)
+; CHECK-NEXT:    setne %al
+; CHECK-NEXT:    movl %eax, {{.*}}(%rip)
+; CHECK-NEXT:    movl $.L.str, %edi
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    callq printf
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    popq %rcx
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
 entry:
-; CHECK: decq
-; CHECK-NOT: decq
 %0 = load i64, i64* @c, align 8
 %dec.i = add nsw i64 %0, -1
 store i64 %dec.i, i64* @c, align 8
@@ -43,10 +63,26 @@ store i32 %lor.ext.i, i32* @a, align 4
 ret i32 0
 }
 
-; CHECK: test2
 define i32 @test2() nounwind uwtable ssp {
+; CHECK-LABEL: test2:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    movq {{.*}}(%rip), %rax
+; CHECK-NEXT:    leaq -1(%rax), %rsi
+; CHECK-NEXT:    movq %rsi, {{.*}}(%rip)
+; CHECK-NEXT:    xorl %ecx, %ecx
+; CHECK-NEXT:    testq %rax, %rax
+; CHECK-NEXT:    setne %cl
+; CHECK-NEXT:    movl %ecx, {{.*}}(%rip)
+; CHECK-NEXT:    movl $.L.str, %edi
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    callq printf
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    popq %rcx
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
 entry:
-; CHECK-NOT: decq ({{.*}})
 %0 = load i64, i64* @c, align 8
 %dec.i = add nsw i64 %0, -1
 store i64 %dec.i, i64* @c, align 8
@@ -65,14 +101,28 @@ declare void @free(i8* nocapture) nounwind
 
 declare void @other(%struct.obj2* ) nounwind;
 
-; CHECK: example_dec
 define void @example_dec(%struct.obj2* %o) nounwind uwtable ssp {
 ; 64 bit dec
+; CHECK-LABEL: example_dec:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    decq (%rdi)
+; CHECK-NEXT:    jne .LBB3_4
+; CHECK-NEXT:  # %bb.1: # %if.end
+; CHECK-NEXT:    decl 8(%rdi)
+; CHECK-NEXT:    jne .LBB3_4
+; CHECK-NEXT:  # %bb.2: # %if.end1
+; CHECK-NEXT:    decw 12(%rdi)
+; CHECK-NEXT:    jne .LBB3_4
+; CHECK-NEXT:  # %bb.3: # %if.end2
+; CHECK-NEXT:    decb 14(%rdi)
+; CHECK-NEXT:    je .LBB3_5
+; CHECK-NEXT:  .LBB3_4: # %return
+; CHECK-NEXT:    retq
+; CHECK-NEXT:  .LBB3_5: # %if.end4
+; CHECK-NEXT:    jmp other # TAILCALL
 entry:
   %s64 = getelementptr inbounds %struct.obj2, %struct.obj2* %o, i64 0, i32 0
-; CHECK-NOT: load 
   %0 = load i64, i64* %s64, align 8
-; CHECK: decq ({{.*}})
   %dec = add i64 %0, -1
   store i64 %dec, i64* %s64, align 8
   %tobool = icmp eq i64 %dec, 0
@@ -81,9 +131,7 @@ entry:
 ; 32 bit dec
 if.end:
   %s32 = getelementptr inbounds %struct.obj2, %struct.obj2* %o, i64 0, i32 1
-; CHECK-NOT: load 
   %1 = load i32, i32* %s32, align 4
-; CHECK: decl {{[0-9][0-9]*}}({{.*}})
   %dec1 = add i32 %1, -1
   store i32 %dec1, i32* %s32, align 4
   %tobool2 = icmp eq i32 %dec1, 0
@@ -92,9 +140,7 @@ if.end:
 ; 16 bit dec
 if.end1:
   %s16 = getelementptr inbounds %struct.obj2, %struct.obj2* %o, i64 0, i32 2
-; CHECK-NOT: load 
   %2 = load i16, i16* %s16, align 2
-; CHECK: decw {{[0-9][0-9]*}}({{.*}})
   %dec2 = add i16 %2, -1
   store i16 %dec2, i16* %s16, align 2
   %tobool3 = icmp eq i16 %dec2, 0
@@ -103,9 +149,7 @@ if.end1:
 ; 8 bit dec
 if.end2:
   %s8 = getelementptr inbounds %struct.obj2, %struct.obj2* %o, i64 0, i32 3
-; CHECK-NOT: load 
   %3 = load i8, i8* %s8
-; CHECK: decb {{[0-9][0-9]*}}({{.*}})
   %dec3 = add i8 %3, -1
   store i8 %dec3, i8* %s8
   %tobool4 = icmp eq i8 %dec3, 0
@@ -115,18 +159,32 @@ if.end4:
   tail call void @other(%struct.obj2* %o) nounwind
   br label %return
 
-return:                                           ; preds = %if.end4, %if.end, %entry                                                                                                                                                                               
+return:                                           ; preds = %if.end4, %if.end, %entry
   ret void
 }
 
-; CHECK: example_inc
 define void @example_inc(%struct.obj2* %o) nounwind uwtable ssp {
 ; 64 bit inc
+; CHECK-LABEL: example_inc:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    incq (%rdi)
+; CHECK-NEXT:    jne .LBB4_4
+; CHECK-NEXT:  # %bb.1: # %if.end
+; CHECK-NEXT:    incl 8(%rdi)
+; CHECK-NEXT:    jne .LBB4_4
+; CHECK-NEXT:  # %bb.2: # %if.end1
+; CHECK-NEXT:    incw 12(%rdi)
+; CHECK-NEXT:    jne .LBB4_4
+; CHECK-NEXT:  # %bb.3: # %if.end2
+; CHECK-NEXT:    incb 14(%rdi)
+; CHECK-NEXT:    jne .LBB4_4
+; CHECK-NEXT:  # %bb.5: # %if.end4
+; CHECK-NEXT:    jmp other # TAILCALL
+; CHECK-NEXT:  .LBB4_4: # %return
+; CHECK-NEXT:    retq
 entry:
   %s64 = getelementptr inbounds %struct.obj2, %struct.obj2* %o, i64 0, i32 0
-; CHECK-NOT: load 
   %0 = load i64, i64* %s64, align 8
-; CHECK: incq ({{.*}})
   %inc = add i64 %0, 1
   store i64 %inc, i64* %s64, align 8
   %tobool = icmp eq i64 %inc, 0
@@ -135,9 +193,7 @@ entry:
 ; 32 bit inc
 if.end:
   %s32 = getelementptr inbounds %struct.obj2, %struct.obj2* %o, i64 0, i32 1
-; CHECK-NOT: load 
   %1 = load i32, i32* %s32, align 4
-; CHECK: incl {{[0-9][0-9]*}}({{.*}})
   %inc1 = add i32 %1, 1
   store i32 %inc1, i32* %s32, align 4
   %tobool2 = icmp eq i32 %inc1, 0
@@ -146,9 +202,7 @@ if.end:
 ; 16 bit inc
 if.end1:
   %s16 = getelementptr inbounds %struct.obj2, %struct.obj2* %o, i64 0, i32 2
-; CHECK-NOT: load 
   %2 = load i16, i16* %s16, align 2
-; CHECK: incw {{[0-9][0-9]*}}({{.*}})
   %inc2 = add i16 %2, 1
   store i16 %inc2, i16* %s16, align 2
   %tobool3 = icmp eq i16 %inc2, 0
@@ -157,9 +211,7 @@ if.end1:
 ; 8 bit inc
 if.end2:
   %s8 = getelementptr inbounds %struct.obj2, %struct.obj2* %o, i64 0, i32 3
-; CHECK-NOT: load 
   %3 = load i8, i8* %s8
-; CHECK: incb {{[0-9][0-9]*}}({{.*}})
   %inc3 = add i8 %3, 1
   store i8 %inc3, i8* %s8
   %tobool4 = icmp eq i8 %inc3, 0
@@ -178,9 +230,16 @@ return:
 @foo = external global i64*, align 8
 
 define void @test3() nounwind ssp {
-entry:
 ; CHECK-LABEL: test3:
-; CHECK: decq 16(%rax)
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movq {{.*}}(%rip), %rax
+; CHECK-NEXT:    decq 16(%rax)
+; CHECK-NEXT:    je .LBB5_2
+; CHECK-NEXT:  # %bb.1: # %if.end
+; CHECK-NEXT:    retq
+; CHECK-NEXT:  .LBB5_2: # %if.then
+; CHECK-NEXT:    jmp baz # TAILCALL
+entry:
   %0 = load i64*, i64** @foo, align 8
   %arrayidx = getelementptr inbounds i64, i64* %0, i64 2
   %1 = load i64, i64* %arrayidx, align 8
@@ -208,6 +267,16 @@ declare void @baz()
 @z = external global i32, align 4
 
 define void @test4() nounwind uwtable ssp {
+; CHECK-LABEL: test4:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    decl {{.*}}(%rip)
+; CHECK-NEXT:    je .LBB6_2
+; CHECK-NEXT:  # %bb.1: # %entry
+; CHECK-NEXT:    movl {{.*}}(%rip), %eax
+; CHECK-NEXT:  .LBB6_2: # %entry
+; CHECK-NEXT:    movl %eax, {{.*}}(%rip)
+; CHECK-NEXT:    retq
 entry:
   %0 = load i32, i32* @x, align 4
   %1 = load i32, i32* @y, align 4
diff --git a/test/CodeGen/X86/rdpid-schedule.ll b/test/CodeGen/X86/rdpid-schedule.ll
deleted file mode 100644
index f1a60dc54a6d46f61c8beebd4ec0721f783c0edf..0000000000000000000000000000000000000000
--- a/test/CodeGen/X86/rdpid-schedule.ll
+++ /dev/null
@@ -1,21 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=x86-64 -mattr=+rdpid | FileCheck %s --check-prefix=CHECK --check-prefix=GENERIC
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=icelake-client | FileCheck %s --check-prefix=CHECK --check-prefix=ICELAKE
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=icelake-server | FileCheck %s --check-prefix=CHECK --check-prefix=ICELAKE
-
-define i32 @test_rdpid() {
-; GENERIC-LABEL: test_rdpid:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    rdpid %rax # sched: [100:0.33]
-; GENERIC-NEXT:    # kill: def $eax killed $eax killed $rax
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ICELAKE-LABEL: test_rdpid:
-; ICELAKE:       # %bb.0:
-; ICELAKE-NEXT:    rdpid %rax # sched: [100:0.25]
-; ICELAKE-NEXT:    # kill: def $eax killed $eax killed $rax
-; ICELAKE-NEXT:    retq # sched: [7:1.00]
-  %1 = tail call i32 @llvm.x86.rdpid()
-  ret i32 %1
-}
-declare i32 @llvm.x86.rdpid()
diff --git a/test/CodeGen/X86/rdrand-schedule.ll b/test/CodeGen/X86/rdrand-schedule.ll
deleted file mode 100644
index fa38b24bd82e63cc5a8023f5cd30e0e0024b5a1a..0000000000000000000000000000000000000000
--- a/test/CodeGen/X86/rdrand-schedule.ll
+++ /dev/null
@@ -1,148 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=x86-64 -mattr=+rdrnd | FileCheck %s --check-prefix=CHECK --check-prefix=GENERIC
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=goldmont | FileCheck %s --check-prefix=CHECK --check-prefix=GOLDMONT
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=ivybridge | FileCheck %s --check-prefix=CHECK --check-prefix=IVY
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=haswell | FileCheck %s --check-prefix=CHECK --check-prefix=HASWELL
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=broadwell | FileCheck %s --check-prefix=CHECK --check-prefix=BROADWELL
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=skylake | FileCheck %s --check-prefix=CHECK --check-prefix=SKYLAKE
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=skx | FileCheck %s --check-prefix=CHECK --check-prefix=SKX
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=znver1 | FileCheck %s --check-prefix=CHECK --check-prefix=ZNVER1
-
-declare {i16, i32} @llvm.x86.rdrand.16()
-declare {i32, i32} @llvm.x86.rdrand.32()
-declare {i64, i32} @llvm.x86.rdrand.64()
-
-define i16 @test_rdrand_16(i16* %random_val) {
-; GENERIC-LABEL: test_rdrand_16:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    rdrandw %ax # sched: [100:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; GOLDMONT-LABEL: test_rdrand_16:
-; GOLDMONT:       # %bb.0:
-; GOLDMONT-NEXT:    rdrandw %ax # sched: [100:1.00]
-; GOLDMONT-NEXT:    retq # sched: [4:1.00]
-;
-; IVY-LABEL: test_rdrand_16:
-; IVY:       # %bb.0:
-; IVY-NEXT:    rdrandw %ax # sched: [100:0.33]
-; IVY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_rdrand_16:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    rdrandw %ax # sched: [1:5.33]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_rdrand_16:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    rdrandw %ax # sched: [9:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_rdrand_16:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    rdrandw %ax # sched: [100:0.25]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_rdrand_16:
-; SKX:       # %bb.0:
-; SKX-NEXT:    rdrandw %ax # sched: [100:0.25]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_rdrand_16:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    rdrandw %ax # sched: [100:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %call = call {i16, i32} @llvm.x86.rdrand.16()
-  %randval = extractvalue {i16, i32} %call, 0
-  ret i16 %randval
-}
-
-define i32 @test_rdrand_32(i32* %random_val) {
-; GENERIC-LABEL: test_rdrand_32:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    rdrandl %eax # sched: [100:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; GOLDMONT-LABEL: test_rdrand_32:
-; GOLDMONT:       # %bb.0:
-; GOLDMONT-NEXT:    rdrandl %eax # sched: [100:1.00]
-; GOLDMONT-NEXT:    retq # sched: [4:1.00]
-;
-; IVY-LABEL: test_rdrand_32:
-; IVY:       # %bb.0:
-; IVY-NEXT:    rdrandl %eax # sched: [100:0.33]
-; IVY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_rdrand_32:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    rdrandl %eax # sched: [1:5.33]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_rdrand_32:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    rdrandl %eax # sched: [9:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_rdrand_32:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    rdrandl %eax # sched: [100:0.25]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_rdrand_32:
-; SKX:       # %bb.0:
-; SKX-NEXT:    rdrandl %eax # sched: [100:0.25]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_rdrand_32:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    rdrandl %eax # sched: [100:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %call = call {i32, i32} @llvm.x86.rdrand.32()
-  %randval = extractvalue {i32, i32} %call, 0
-  ret i32 %randval
-}
-
-define i64 @test_rdrand_64(i64* %random_val) {
-; GENERIC-LABEL: test_rdrand_64:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    rdrandq %rax # sched: [100:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; GOLDMONT-LABEL: test_rdrand_64:
-; GOLDMONT:       # %bb.0:
-; GOLDMONT-NEXT:    rdrandq %rax # sched: [100:1.00]
-; GOLDMONT-NEXT:    retq # sched: [4:1.00]
-;
-; IVY-LABEL: test_rdrand_64:
-; IVY:       # %bb.0:
-; IVY-NEXT:    rdrandq %rax # sched: [100:0.33]
-; IVY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_rdrand_64:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    rdrandq %rax # sched: [1:5.33]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_rdrand_64:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    rdrandq %rax # sched: [9:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_rdrand_64:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    rdrandq %rax # sched: [100:0.25]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_rdrand_64:
-; SKX:       # %bb.0:
-; SKX-NEXT:    rdrandq %rax # sched: [100:0.25]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_rdrand_64:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    rdrandq %rax # sched: [100:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %call = call {i64, i32} @llvm.x86.rdrand.64()
-  %randval = extractvalue {i64, i32} %call, 0
-  ret i64 %randval
-}
diff --git a/test/CodeGen/X86/rdseed-schedule.ll b/test/CodeGen/X86/rdseed-schedule.ll
deleted file mode 100644
index b3feeabe797afdcc5e65df4c7d23a0233985681c..0000000000000000000000000000000000000000
--- a/test/CodeGen/X86/rdseed-schedule.ll
+++ /dev/null
@@ -1,116 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=x86-64 -mattr=+rdseed | FileCheck %s --check-prefix=CHECK --check-prefix=GENERIC
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=goldmont | FileCheck %s --check-prefix=CHECK --check-prefix=GOLDMONT
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=broadwell | FileCheck %s --check-prefix=CHECK --check-prefix=BROADWELL
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=skylake | FileCheck %s --check-prefix=CHECK --check-prefix=SKYLAKE
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=skx | FileCheck %s --check-prefix=CHECK --check-prefix=SKX
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=znver1 | FileCheck %s --check-prefix=CHECK --check-prefix=ZNVER1
-
-declare {i16, i32} @llvm.x86.rdseed.16()
-declare {i32, i32} @llvm.x86.rdseed.32()
-declare {i64, i32} @llvm.x86.rdseed.64()
-
-define i16 @test_rdseed_16(i16* %random_val) {
-; GENERIC-LABEL: test_rdseed_16:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    rdseedw %ax # sched: [100:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; GOLDMONT-LABEL: test_rdseed_16:
-; GOLDMONT:       # %bb.0:
-; GOLDMONT-NEXT:    rdseedw %ax # sched: [100:1.00]
-; GOLDMONT-NEXT:    retq # sched: [4:1.00]
-;
-; BROADWELL-LABEL: test_rdseed_16:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    rdseedw %ax # sched: [100:0.25]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_rdseed_16:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    rdseedw %ax # sched: [100:0.25]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_rdseed_16:
-; SKX:       # %bb.0:
-; SKX-NEXT:    rdseedw %ax # sched: [100:0.25]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_rdseed_16:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    rdseedw %ax # sched: [100:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %call = call {i16, i32} @llvm.x86.rdseed.16()
-  %randval = extractvalue {i16, i32} %call, 0
-  ret i16 %randval
-}
-
-define i32 @test_rdseed_32(i16* %random_val) {
-; GENERIC-LABEL: test_rdseed_32:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    rdseedl %eax # sched: [100:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; GOLDMONT-LABEL: test_rdseed_32:
-; GOLDMONT:       # %bb.0:
-; GOLDMONT-NEXT:    rdseedl %eax # sched: [100:1.00]
-; GOLDMONT-NEXT:    retq # sched: [4:1.00]
-;
-; BROADWELL-LABEL: test_rdseed_32:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    rdseedl %eax # sched: [100:0.25]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_rdseed_32:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    rdseedl %eax # sched: [100:0.25]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_rdseed_32:
-; SKX:       # %bb.0:
-; SKX-NEXT:    rdseedl %eax # sched: [100:0.25]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_rdseed_32:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    rdseedl %eax # sched: [100:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %call = call {i32, i32} @llvm.x86.rdseed.32()
-  %randval = extractvalue {i32, i32} %call, 0
-  ret i32 %randval
-}
-
-define i64 @test_rdseed_64(i64* %random_val) {
-; GENERIC-LABEL: test_rdseed_64:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    rdseedq %rax # sched: [100:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; GOLDMONT-LABEL: test_rdseed_64:
-; GOLDMONT:       # %bb.0:
-; GOLDMONT-NEXT:    rdseedq %rax # sched: [100:1.00]
-; GOLDMONT-NEXT:    retq # sched: [4:1.00]
-;
-; BROADWELL-LABEL: test_rdseed_64:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    rdseedq %rax # sched: [100:0.25]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_rdseed_64:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    rdseedq %rax # sched: [100:0.25]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_rdseed_64:
-; SKX:       # %bb.0:
-; SKX-NEXT:    rdseedq %rax # sched: [100:0.25]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_rdseed_64:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    rdseedq %rax # sched: [100:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %call = call {i64, i32} @llvm.x86.rdseed.64()
-  %randval = extractvalue {i64, i32} %call, 0
-  ret i64 %randval
-}
diff --git a/test/CodeGen/X86/recip-fastmath.ll b/test/CodeGen/X86/recip-fastmath.ll
index a68940eb11a3b6414f1a145acce3432b0077d05a..c8908b54c507926a14a11fbce95d0ced02492d31 100644
--- a/test/CodeGen/X86/recip-fastmath.ll
+++ b/test/CodeGen/X86/recip-fastmath.ll
@@ -2,13 +2,13 @@
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2     | FileCheck %s --check-prefix=CHECK --check-prefix=SSE --check-prefix=SSE-RECIP
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx      | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX-RECIP
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx,+fma | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=FMA-RECIP
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=bdver2 -print-schedule      | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=BDVER2
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=btver2 -print-schedule      | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=BTVER2
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=sandybridge -print-schedule | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=SANDY
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=haswell -print-schedule     | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=HASWELL
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=bdver2 | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=BDVER2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=btver2 | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=BTVER2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=sandybridge | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=SANDY
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=haswell | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=HASWELL
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=haswell -mattr=-fma | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=HASWELL-NO-FMA
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=knl -print-schedule | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX512 --check-prefix=KNL
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=skx -print-schedule | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX512 --check-prefix=SKX
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=knl | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX512 --check-prefix=KNL
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=skx | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX512 --check-prefix=SKX
 
 ; If the target's divss/divps instructions are substantially
 ; slower than rcpss/rcpps with a Newton-Raphson refinement,
@@ -26,59 +26,11 @@ define float @f32_no_estimate(float %x) #0 {
 ; SSE-NEXT:    movaps %xmm1, %xmm0
 ; SSE-NEXT:    retq
 ;
-; AVX-RECIP-LABEL: f32_no_estimate:
-; AVX-RECIP:       # %bb.0:
-; AVX-RECIP-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; AVX-RECIP-NEXT:    vdivss %xmm0, %xmm1, %xmm0
-; AVX-RECIP-NEXT:    retq
-;
-; FMA-RECIP-LABEL: f32_no_estimate:
-; FMA-RECIP:       # %bb.0:
-; FMA-RECIP-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; FMA-RECIP-NEXT:    vdivss %xmm0, %xmm1, %xmm0
-; FMA-RECIP-NEXT:    retq
-;
-; BDVER2-LABEL: f32_no_estimate:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero sched: [5:0.50]
-; BDVER2-NEXT:    vdivss %xmm0, %xmm1, %xmm0 # sched: [9:9.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: f32_no_estimate:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero sched: [5:1.00]
-; BTVER2-NEXT:    vdivss %xmm0, %xmm1, %xmm0 # sched: [19:19.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: f32_no_estimate:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero sched: [6:0.50]
-; SANDY-NEXT:    vdivss %xmm0, %xmm1, %xmm0 # sched: [14:14.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: f32_no_estimate:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero sched: [5:0.50]
-; HASWELL-NEXT:    vdivss %xmm0, %xmm1, %xmm0 # sched: [13:7.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-NO-FMA-LABEL: f32_no_estimate:
-; HASWELL-NO-FMA:       # %bb.0:
-; HASWELL-NO-FMA-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; HASWELL-NO-FMA-NEXT:    vdivss %xmm0, %xmm1, %xmm0
-; HASWELL-NO-FMA-NEXT:    retq
-;
-; KNL-LABEL: f32_no_estimate:
-; KNL:       # %bb.0:
-; KNL-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero sched: [5:0.50]
-; KNL-NEXT:    vdivss %xmm0, %xmm1, %xmm0 # sched: [13:7.00]
-; KNL-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: f32_no_estimate:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero sched: [5:0.50]
-; SKX-NEXT:    vdivss %xmm0, %xmm1, %xmm0 # sched: [11:3.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
+; AVX-LABEL: f32_no_estimate:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; AVX-NEXT:    vdivss %xmm0, %xmm1, %xmm0
+; AVX-NEXT:    retq
   %div = fdiv fast float 1.0, %x
   ret float %div
 }
@@ -114,37 +66,37 @@ define float @f32_one_step(float %x) #1 {
 ;
 ; BDVER2-LABEL: f32_one_step:
 ; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vrcpss %xmm0, %xmm0, %xmm1 # sched: [5:1.00]
-; BDVER2-NEXT:    vfnmaddss {{.*}}(%rip), %xmm1, %xmm0, %xmm0 # sched: [10:0.50]
-; BDVER2-NEXT:    vfmaddss %xmm1, %xmm0, %xmm1, %xmm0 # sched: [5:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
+; BDVER2-NEXT:    vrcpss %xmm0, %xmm0, %xmm1
+; BDVER2-NEXT:    vfnmaddss {{.*}}(%rip), %xmm1, %xmm0, %xmm0
+; BDVER2-NEXT:    vfmaddss %xmm1, %xmm0, %xmm1, %xmm0
+; BDVER2-NEXT:    retq
 ;
 ; BTVER2-LABEL: f32_one_step:
 ; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vmovss {{.*#+}} xmm2 = mem[0],zero,zero,zero sched: [5:1.00]
-; BTVER2-NEXT:    vrcpss %xmm0, %xmm0, %xmm1 # sched: [2:1.00]
-; BTVER2-NEXT:    vmulss %xmm1, %xmm0, %xmm0 # sched: [2:1.00]
-; BTVER2-NEXT:    vsubss %xmm0, %xmm2, %xmm0 # sched: [3:1.00]
-; BTVER2-NEXT:    vmulss %xmm0, %xmm1, %xmm0 # sched: [2:1.00]
-; BTVER2-NEXT:    vaddss %xmm0, %xmm1, %xmm0 # sched: [3:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
+; BTVER2-NEXT:    vmovss {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; BTVER2-NEXT:    vrcpss %xmm0, %xmm0, %xmm1
+; BTVER2-NEXT:    vmulss %xmm1, %xmm0, %xmm0
+; BTVER2-NEXT:    vsubss %xmm0, %xmm2, %xmm0
+; BTVER2-NEXT:    vmulss %xmm0, %xmm1, %xmm0
+; BTVER2-NEXT:    vaddss %xmm0, %xmm1, %xmm0
+; BTVER2-NEXT:    retq
 ;
 ; SANDY-LABEL: f32_one_step:
 ; SANDY:       # %bb.0:
-; SANDY-NEXT:    vrcpss %xmm0, %xmm0, %xmm1 # sched: [5:1.00]
-; SANDY-NEXT:    vmulss %xmm1, %xmm0, %xmm0 # sched: [5:1.00]
-; SANDY-NEXT:    vmovss {{.*#+}} xmm2 = mem[0],zero,zero,zero sched: [6:0.50]
-; SANDY-NEXT:    vsubss %xmm0, %xmm2, %xmm0 # sched: [3:1.00]
-; SANDY-NEXT:    vmulss %xmm0, %xmm1, %xmm0 # sched: [5:1.00]
-; SANDY-NEXT:    vaddss %xmm0, %xmm1, %xmm0 # sched: [3:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
+; SANDY-NEXT:    vrcpss %xmm0, %xmm0, %xmm1
+; SANDY-NEXT:    vmulss %xmm1, %xmm0, %xmm0
+; SANDY-NEXT:    vmovss {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; SANDY-NEXT:    vsubss %xmm0, %xmm2, %xmm0
+; SANDY-NEXT:    vmulss %xmm0, %xmm1, %xmm0
+; SANDY-NEXT:    vaddss %xmm0, %xmm1, %xmm0
+; SANDY-NEXT:    retq
 ;
 ; HASWELL-LABEL: f32_one_step:
 ; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vrcpss %xmm0, %xmm0, %xmm1 # sched: [5:1.00]
-; HASWELL-NEXT:    vfnmadd213ss {{.*#+}} xmm0 = -(xmm1 * xmm0) + mem sched: [10:0.50]
-; HASWELL-NEXT:    vfmadd132ss {{.*#+}} xmm0 = (xmm0 * xmm1) + xmm1 sched: [5:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
+; HASWELL-NEXT:    vrcpss %xmm0, %xmm0, %xmm1
+; HASWELL-NEXT:    vfnmadd213ss {{.*#+}} xmm0 = -(xmm1 * xmm0) + mem
+; HASWELL-NEXT:    vfmadd132ss {{.*#+}} xmm0 = (xmm0 * xmm1) + xmm1
+; HASWELL-NEXT:    retq
 ;
 ; HASWELL-NO-FMA-LABEL: f32_one_step:
 ; HASWELL-NO-FMA:       # %bb.0:
@@ -156,19 +108,12 @@ define float @f32_one_step(float %x) #1 {
 ; HASWELL-NO-FMA-NEXT:    vaddss %xmm0, %xmm1, %xmm0
 ; HASWELL-NO-FMA-NEXT:    retq
 ;
-; KNL-LABEL: f32_one_step:
-; KNL:       # %bb.0:
-; KNL-NEXT:    vrcpss %xmm0, %xmm0, %xmm1 # sched: [5:1.00]
-; KNL-NEXT:    vfnmadd213ss {{.*#+}} xmm0 = -(xmm1 * xmm0) + mem sched: [10:0.50]
-; KNL-NEXT:    vfmadd132ss {{.*#+}} xmm0 = (xmm0 * xmm1) + xmm1 sched: [5:0.50]
-; KNL-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: f32_one_step:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vrcpss %xmm0, %xmm0, %xmm1 # sched: [4:1.00]
-; SKX-NEXT:    vfnmadd213ss {{.*#+}} xmm0 = -(xmm1 * xmm0) + mem sched: [9:0.50]
-; SKX-NEXT:    vfmadd132ss {{.*#+}} xmm0 = (xmm0 * xmm1) + xmm1 sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
+; AVX512-LABEL: f32_one_step:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vrcpss %xmm0, %xmm0, %xmm1
+; AVX512-NEXT:    vfnmadd213ss {{.*#+}} xmm0 = -(xmm1 * xmm0) + mem
+; AVX512-NEXT:    vfmadd132ss {{.*#+}} xmm0 = (xmm0 * xmm1) + xmm1
+; AVX512-NEXT:    retq
   %div = fdiv fast float 1.0, %x
   ret float %div
 }
@@ -218,52 +163,52 @@ define float @f32_two_step(float %x) #2 {
 ;
 ; BDVER2-LABEL: f32_two_step:
 ; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vrcpss %xmm0, %xmm0, %xmm1 # sched: [5:1.00]
-; BDVER2-NEXT:    vmovss {{.*#+}} xmm2 = mem[0],zero,zero,zero sched: [5:0.50]
-; BDVER2-NEXT:    vfnmaddss %xmm2, %xmm1, %xmm0, %xmm3 # sched: [5:0.50]
-; BDVER2-NEXT:    vfmaddss %xmm1, %xmm3, %xmm1, %xmm1 # sched: [5:0.50]
-; BDVER2-NEXT:    vfnmaddss %xmm2, %xmm1, %xmm0, %xmm0 # sched: [5:0.50]
-; BDVER2-NEXT:    vfmaddss %xmm1, %xmm0, %xmm1, %xmm0 # sched: [5:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
+; BDVER2-NEXT:    vrcpss %xmm0, %xmm0, %xmm1
+; BDVER2-NEXT:    vmovss {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; BDVER2-NEXT:    vfnmaddss %xmm2, %xmm1, %xmm0, %xmm3
+; BDVER2-NEXT:    vfmaddss %xmm1, %xmm3, %xmm1, %xmm1
+; BDVER2-NEXT:    vfnmaddss %xmm2, %xmm1, %xmm0, %xmm0
+; BDVER2-NEXT:    vfmaddss %xmm1, %xmm0, %xmm1, %xmm0
+; BDVER2-NEXT:    retq
 ;
 ; BTVER2-LABEL: f32_two_step:
 ; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vmovss {{.*#+}} xmm3 = mem[0],zero,zero,zero sched: [5:1.00]
-; BTVER2-NEXT:    vrcpss %xmm0, %xmm0, %xmm1 # sched: [2:1.00]
-; BTVER2-NEXT:    vmulss %xmm1, %xmm0, %xmm2 # sched: [2:1.00]
-; BTVER2-NEXT:    vsubss %xmm2, %xmm3, %xmm2 # sched: [3:1.00]
-; BTVER2-NEXT:    vmulss %xmm2, %xmm1, %xmm2 # sched: [2:1.00]
-; BTVER2-NEXT:    vaddss %xmm2, %xmm1, %xmm1 # sched: [3:1.00]
-; BTVER2-NEXT:    vmulss %xmm1, %xmm0, %xmm0 # sched: [2:1.00]
-; BTVER2-NEXT:    vsubss %xmm0, %xmm3, %xmm0 # sched: [3:1.00]
-; BTVER2-NEXT:    vmulss %xmm0, %xmm1, %xmm0 # sched: [2:1.00]
-; BTVER2-NEXT:    vaddss %xmm0, %xmm1, %xmm0 # sched: [3:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
+; BTVER2-NEXT:    vmovss {{.*#+}} xmm3 = mem[0],zero,zero,zero
+; BTVER2-NEXT:    vrcpss %xmm0, %xmm0, %xmm1
+; BTVER2-NEXT:    vmulss %xmm1, %xmm0, %xmm2
+; BTVER2-NEXT:    vsubss %xmm2, %xmm3, %xmm2
+; BTVER2-NEXT:    vmulss %xmm2, %xmm1, %xmm2
+; BTVER2-NEXT:    vaddss %xmm2, %xmm1, %xmm1
+; BTVER2-NEXT:    vmulss %xmm1, %xmm0, %xmm0
+; BTVER2-NEXT:    vsubss %xmm0, %xmm3, %xmm0
+; BTVER2-NEXT:    vmulss %xmm0, %xmm1, %xmm0
+; BTVER2-NEXT:    vaddss %xmm0, %xmm1, %xmm0
+; BTVER2-NEXT:    retq
 ;
 ; SANDY-LABEL: f32_two_step:
 ; SANDY:       # %bb.0:
-; SANDY-NEXT:    vrcpss %xmm0, %xmm0, %xmm1 # sched: [5:1.00]
-; SANDY-NEXT:    vmulss %xmm1, %xmm0, %xmm2 # sched: [5:1.00]
-; SANDY-NEXT:    vmovss {{.*#+}} xmm3 = mem[0],zero,zero,zero sched: [6:0.50]
-; SANDY-NEXT:    vsubss %xmm2, %xmm3, %xmm2 # sched: [3:1.00]
-; SANDY-NEXT:    vmulss %xmm2, %xmm1, %xmm2 # sched: [5:1.00]
-; SANDY-NEXT:    vaddss %xmm2, %xmm1, %xmm1 # sched: [3:1.00]
-; SANDY-NEXT:    vmulss %xmm1, %xmm0, %xmm0 # sched: [5:1.00]
-; SANDY-NEXT:    vsubss %xmm0, %xmm3, %xmm0 # sched: [3:1.00]
-; SANDY-NEXT:    vmulss %xmm0, %xmm1, %xmm0 # sched: [5:1.00]
-; SANDY-NEXT:    vaddss %xmm0, %xmm1, %xmm0 # sched: [3:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
+; SANDY-NEXT:    vrcpss %xmm0, %xmm0, %xmm1
+; SANDY-NEXT:    vmulss %xmm1, %xmm0, %xmm2
+; SANDY-NEXT:    vmovss {{.*#+}} xmm3 = mem[0],zero,zero,zero
+; SANDY-NEXT:    vsubss %xmm2, %xmm3, %xmm2
+; SANDY-NEXT:    vmulss %xmm2, %xmm1, %xmm2
+; SANDY-NEXT:    vaddss %xmm2, %xmm1, %xmm1
+; SANDY-NEXT:    vmulss %xmm1, %xmm0, %xmm0
+; SANDY-NEXT:    vsubss %xmm0, %xmm3, %xmm0
+; SANDY-NEXT:    vmulss %xmm0, %xmm1, %xmm0
+; SANDY-NEXT:    vaddss %xmm0, %xmm1, %xmm0
+; SANDY-NEXT:    retq
 ;
 ; HASWELL-LABEL: f32_two_step:
 ; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vrcpss %xmm0, %xmm0, %xmm1 # sched: [5:1.00]
-; HASWELL-NEXT:    vmovss {{.*#+}} xmm2 = mem[0],zero,zero,zero sched: [5:0.50]
-; HASWELL-NEXT:    vmovaps %xmm1, %xmm3 # sched: [1:1.00]
-; HASWELL-NEXT:    vfnmadd213ss {{.*#+}} xmm3 = -(xmm0 * xmm3) + xmm2 sched: [5:0.50]
-; HASWELL-NEXT:    vfmadd132ss {{.*#+}} xmm3 = (xmm3 * xmm1) + xmm1 sched: [5:0.50]
-; HASWELL-NEXT:    vfnmadd213ss {{.*#+}} xmm0 = -(xmm3 * xmm0) + xmm2 sched: [5:0.50]
-; HASWELL-NEXT:    vfmadd132ss {{.*#+}} xmm0 = (xmm0 * xmm3) + xmm3 sched: [5:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
+; HASWELL-NEXT:    vrcpss %xmm0, %xmm0, %xmm1
+; HASWELL-NEXT:    vmovss {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; HASWELL-NEXT:    vmovaps %xmm1, %xmm3
+; HASWELL-NEXT:    vfnmadd213ss {{.*#+}} xmm3 = -(xmm0 * xmm3) + xmm2
+; HASWELL-NEXT:    vfmadd132ss {{.*#+}} xmm3 = (xmm3 * xmm1) + xmm1
+; HASWELL-NEXT:    vfnmadd213ss {{.*#+}} xmm0 = -(xmm3 * xmm0) + xmm2
+; HASWELL-NEXT:    vfmadd132ss {{.*#+}} xmm0 = (xmm0 * xmm3) + xmm3
+; HASWELL-NEXT:    retq
 ;
 ; HASWELL-NO-FMA-LABEL: f32_two_step:
 ; HASWELL-NO-FMA:       # %bb.0:
@@ -279,27 +224,16 @@ define float @f32_two_step(float %x) #2 {
 ; HASWELL-NO-FMA-NEXT:    vaddss %xmm0, %xmm1, %xmm0
 ; HASWELL-NO-FMA-NEXT:    retq
 ;
-; KNL-LABEL: f32_two_step:
-; KNL:       # %bb.0:
-; KNL-NEXT:    vrcpss %xmm0, %xmm0, %xmm1 # sched: [5:1.00]
-; KNL-NEXT:    vmovss {{.*#+}} xmm2 = mem[0],zero,zero,zero sched: [5:0.50]
-; KNL-NEXT:    vmovaps %xmm1, %xmm3 # sched: [1:1.00]
-; KNL-NEXT:    vfnmadd213ss {{.*#+}} xmm3 = -(xmm0 * xmm3) + xmm2 sched: [5:0.50]
-; KNL-NEXT:    vfmadd132ss {{.*#+}} xmm3 = (xmm3 * xmm1) + xmm1 sched: [5:0.50]
-; KNL-NEXT:    vfnmadd213ss {{.*#+}} xmm0 = -(xmm3 * xmm0) + xmm2 sched: [5:0.50]
-; KNL-NEXT:    vfmadd132ss {{.*#+}} xmm0 = (xmm0 * xmm3) + xmm3 sched: [5:0.50]
-; KNL-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: f32_two_step:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vrcpss %xmm0, %xmm0, %xmm1 # sched: [4:1.00]
-; SKX-NEXT:    vmovss {{.*#+}} xmm2 = mem[0],zero,zero,zero sched: [5:0.50]
-; SKX-NEXT:    vmovaps %xmm1, %xmm3 # sched: [1:0.33]
-; SKX-NEXT:    vfnmadd213ss {{.*#+}} xmm3 = -(xmm0 * xmm3) + xmm2 sched: [4:0.50]
-; SKX-NEXT:    vfmadd132ss {{.*#+}} xmm3 = (xmm3 * xmm1) + xmm1 sched: [4:0.50]
-; SKX-NEXT:    vfnmadd213ss {{.*#+}} xmm0 = -(xmm3 * xmm0) + xmm2 sched: [4:0.50]
-; SKX-NEXT:    vfmadd132ss {{.*#+}} xmm0 = (xmm0 * xmm3) + xmm3 sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
+; AVX512-LABEL: f32_two_step:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vrcpss %xmm0, %xmm0, %xmm1
+; AVX512-NEXT:    vmovss {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; AVX512-NEXT:    vmovaps %xmm1, %xmm3
+; AVX512-NEXT:    vfnmadd213ss {{.*#+}} xmm3 = -(xmm0 * xmm3) + xmm2
+; AVX512-NEXT:    vfmadd132ss {{.*#+}} xmm3 = (xmm3 * xmm1) + xmm1
+; AVX512-NEXT:    vfnmadd213ss {{.*#+}} xmm0 = -(xmm3 * xmm0) + xmm2
+; AVX512-NEXT:    vfmadd132ss {{.*#+}} xmm0 = (xmm0 * xmm3) + xmm3
+; AVX512-NEXT:    retq
   %div = fdiv fast float 1.0, %x
   ret float %div
 }
@@ -326,27 +260,27 @@ define <4 x float> @v4f32_no_estimate(<4 x float> %x) #0 {
 ;
 ; BDVER2-LABEL: v4f32_no_estimate:
 ; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vmovaps {{.*#+}} xmm1 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0] sched: [5:0.50]
-; BDVER2-NEXT:    vdivps %xmm0, %xmm1, %xmm0 # sched: [9:9.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
+; BDVER2-NEXT:    vmovaps {{.*#+}} xmm1 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; BDVER2-NEXT:    vdivps %xmm0, %xmm1, %xmm0
+; BDVER2-NEXT:    retq
 ;
 ; BTVER2-LABEL: v4f32_no_estimate:
 ; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vmovaps {{.*#+}} xmm1 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0] sched: [5:1.00]
-; BTVER2-NEXT:    vdivps %xmm0, %xmm1, %xmm0 # sched: [19:19.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
+; BTVER2-NEXT:    vmovaps {{.*#+}} xmm1 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; BTVER2-NEXT:    vdivps %xmm0, %xmm1, %xmm0
+; BTVER2-NEXT:    retq
 ;
 ; SANDY-LABEL: v4f32_no_estimate:
 ; SANDY:       # %bb.0:
-; SANDY-NEXT:    vmovaps {{.*#+}} xmm1 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0] sched: [6:0.50]
-; SANDY-NEXT:    vdivps %xmm0, %xmm1, %xmm0 # sched: [14:14.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
+; SANDY-NEXT:    vmovaps {{.*#+}} xmm1 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; SANDY-NEXT:    vdivps %xmm0, %xmm1, %xmm0
+; SANDY-NEXT:    retq
 ;
 ; HASWELL-LABEL: v4f32_no_estimate:
 ; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vbroadcastss {{.*#+}} xmm1 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0] sched: [6:0.50]
-; HASWELL-NEXT:    vdivps %xmm0, %xmm1, %xmm0 # sched: [13:7.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
+; HASWELL-NEXT:    vbroadcastss {{.*#+}} xmm1 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; HASWELL-NEXT:    vdivps %xmm0, %xmm1, %xmm0
+; HASWELL-NEXT:    retq
 ;
 ; HASWELL-NO-FMA-LABEL: v4f32_no_estimate:
 ; HASWELL-NO-FMA:       # %bb.0:
@@ -354,17 +288,11 @@ define <4 x float> @v4f32_no_estimate(<4 x float> %x) #0 {
 ; HASWELL-NO-FMA-NEXT:    vdivps %xmm0, %xmm1, %xmm0
 ; HASWELL-NO-FMA-NEXT:    retq
 ;
-; KNL-LABEL: v4f32_no_estimate:
-; KNL:       # %bb.0:
-; KNL-NEXT:    vbroadcastss {{.*#+}} xmm1 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0] sched: [6:0.50]
-; KNL-NEXT:    vdivps %xmm0, %xmm1, %xmm0 # sched: [13:7.00]
-; KNL-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: v4f32_no_estimate:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vbroadcastss {{.*#+}} xmm1 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0] sched: [6:0.50]
-; SKX-NEXT:    vdivps %xmm0, %xmm1, %xmm0 # sched: [11:3.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
+; AVX512-LABEL: v4f32_no_estimate:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vbroadcastss {{.*#+}} xmm1 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; AVX512-NEXT:    vdivps %xmm0, %xmm1, %xmm0
+; AVX512-NEXT:    retq
   %div = fdiv fast <4 x float> <float 1.0, float 1.0, float 1.0, float 1.0>, %x
   ret <4 x float> %div
 }
@@ -400,38 +328,38 @@ define <4 x float> @v4f32_one_step(<4 x float> %x) #1 {
 ;
 ; BDVER2-LABEL: v4f32_one_step:
 ; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vrcpps %xmm0, %xmm1 # sched: [5:1.00]
-; BDVER2-NEXT:    vfnmaddps {{.*}}(%rip), %xmm1, %xmm0, %xmm0 # sched: [10:0.50]
-; BDVER2-NEXT:    vfmaddps %xmm1, %xmm0, %xmm1, %xmm0 # sched: [5:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
+; BDVER2-NEXT:    vrcpps %xmm0, %xmm1
+; BDVER2-NEXT:    vfnmaddps {{.*}}(%rip), %xmm1, %xmm0, %xmm0
+; BDVER2-NEXT:    vfmaddps %xmm1, %xmm0, %xmm1, %xmm0
+; BDVER2-NEXT:    retq
 ;
 ; BTVER2-LABEL: v4f32_one_step:
 ; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vmovaps {{.*#+}} xmm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0] sched: [5:1.00]
-; BTVER2-NEXT:    vrcpps %xmm0, %xmm1 # sched: [2:1.00]
-; BTVER2-NEXT:    vmulps %xmm1, %xmm0, %xmm0 # sched: [2:1.00]
-; BTVER2-NEXT:    vsubps %xmm0, %xmm2, %xmm0 # sched: [3:1.00]
-; BTVER2-NEXT:    vmulps %xmm0, %xmm1, %xmm0 # sched: [2:1.00]
-; BTVER2-NEXT:    vaddps %xmm0, %xmm1, %xmm0 # sched: [3:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
+; BTVER2-NEXT:    vmovaps {{.*#+}} xmm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; BTVER2-NEXT:    vrcpps %xmm0, %xmm1
+; BTVER2-NEXT:    vmulps %xmm1, %xmm0, %xmm0
+; BTVER2-NEXT:    vsubps %xmm0, %xmm2, %xmm0
+; BTVER2-NEXT:    vmulps %xmm0, %xmm1, %xmm0
+; BTVER2-NEXT:    vaddps %xmm0, %xmm1, %xmm0
+; BTVER2-NEXT:    retq
 ;
 ; SANDY-LABEL: v4f32_one_step:
 ; SANDY:       # %bb.0:
-; SANDY-NEXT:    vrcpps %xmm0, %xmm1 # sched: [5:1.00]
-; SANDY-NEXT:    vmulps %xmm1, %xmm0, %xmm0 # sched: [5:1.00]
-; SANDY-NEXT:    vmovaps {{.*#+}} xmm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0] sched: [6:0.50]
-; SANDY-NEXT:    vsubps %xmm0, %xmm2, %xmm0 # sched: [3:1.00]
-; SANDY-NEXT:    vmulps %xmm0, %xmm1, %xmm0 # sched: [5:1.00]
-; SANDY-NEXT:    vaddps %xmm0, %xmm1, %xmm0 # sched: [3:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
+; SANDY-NEXT:    vrcpps %xmm0, %xmm1
+; SANDY-NEXT:    vmulps %xmm1, %xmm0, %xmm0
+; SANDY-NEXT:    vmovaps {{.*#+}} xmm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; SANDY-NEXT:    vsubps %xmm0, %xmm2, %xmm0
+; SANDY-NEXT:    vmulps %xmm0, %xmm1, %xmm0
+; SANDY-NEXT:    vaddps %xmm0, %xmm1, %xmm0
+; SANDY-NEXT:    retq
 ;
 ; HASWELL-LABEL: v4f32_one_step:
 ; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vrcpps %xmm0, %xmm1 # sched: [5:1.00]
-; HASWELL-NEXT:    vbroadcastss {{.*#+}} xmm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0] sched: [6:0.50]
-; HASWELL-NEXT:    vfnmadd213ps {{.*#+}} xmm0 = -(xmm1 * xmm0) + xmm2 sched: [5:0.50]
-; HASWELL-NEXT:    vfmadd132ps {{.*#+}} xmm0 = (xmm0 * xmm1) + xmm1 sched: [5:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
+; HASWELL-NEXT:    vrcpps %xmm0, %xmm1
+; HASWELL-NEXT:    vbroadcastss {{.*#+}} xmm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; HASWELL-NEXT:    vfnmadd213ps {{.*#+}} xmm0 = -(xmm1 * xmm0) + xmm2
+; HASWELL-NEXT:    vfmadd132ps {{.*#+}} xmm0 = (xmm0 * xmm1) + xmm1
+; HASWELL-NEXT:    retq
 ;
 ; HASWELL-NO-FMA-LABEL: v4f32_one_step:
 ; HASWELL-NO-FMA:       # %bb.0:
@@ -445,18 +373,18 @@ define <4 x float> @v4f32_one_step(<4 x float> %x) #1 {
 ;
 ; KNL-LABEL: v4f32_one_step:
 ; KNL:       # %bb.0:
-; KNL-NEXT:    vrcpps %xmm0, %xmm1 # sched: [5:1.00]
-; KNL-NEXT:    vbroadcastss {{.*#+}} xmm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0] sched: [6:0.50]
-; KNL-NEXT:    vfnmadd213ps {{.*#+}} xmm0 = -(xmm1 * xmm0) + xmm2 sched: [5:0.50]
-; KNL-NEXT:    vfmadd132ps {{.*#+}} xmm0 = (xmm0 * xmm1) + xmm1 sched: [5:0.50]
-; KNL-NEXT:    retq # sched: [7:1.00]
+; KNL-NEXT:    vrcpps %xmm0, %xmm1
+; KNL-NEXT:    vbroadcastss {{.*#+}} xmm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; KNL-NEXT:    vfnmadd213ps {{.*#+}} xmm0 = -(xmm1 * xmm0) + xmm2
+; KNL-NEXT:    vfmadd132ps {{.*#+}} xmm0 = (xmm0 * xmm1) + xmm1
+; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: v4f32_one_step:
 ; SKX:       # %bb.0:
-; SKX-NEXT:    vrcpps %xmm0, %xmm1 # sched: [4:1.00]
-; SKX-NEXT:    vfnmadd213ps {{.*#+}} xmm0 = -(xmm1 * xmm0) + mem sched: [10:0.50]
-; SKX-NEXT:    vfmadd132ps {{.*#+}} xmm0 = (xmm0 * xmm1) + xmm1 sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
+; SKX-NEXT:    vrcpps %xmm0, %xmm1
+; SKX-NEXT:    vfnmadd213ps {{.*#+}} xmm0 = -(xmm1 * xmm0) + mem
+; SKX-NEXT:    vfmadd132ps {{.*#+}} xmm0 = (xmm0 * xmm1) + xmm1
+; SKX-NEXT:    retq
   %div = fdiv fast <4 x float> <float 1.0, float 1.0, float 1.0, float 1.0>, %x
   ret <4 x float> %div
 }
@@ -506,52 +434,52 @@ define <4 x float> @v4f32_two_step(<4 x float> %x) #2 {
 ;
 ; BDVER2-LABEL: v4f32_two_step:
 ; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vrcpps %xmm0, %xmm1 # sched: [5:1.00]
-; BDVER2-NEXT:    vmovaps {{.*#+}} xmm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0] sched: [5:0.50]
-; BDVER2-NEXT:    vfnmaddps %xmm2, %xmm1, %xmm0, %xmm3 # sched: [5:0.50]
-; BDVER2-NEXT:    vfmaddps %xmm1, %xmm3, %xmm1, %xmm1 # sched: [5:0.50]
-; BDVER2-NEXT:    vfnmaddps %xmm2, %xmm1, %xmm0, %xmm0 # sched: [5:0.50]
-; BDVER2-NEXT:    vfmaddps %xmm1, %xmm0, %xmm1, %xmm0 # sched: [5:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
+; BDVER2-NEXT:    vrcpps %xmm0, %xmm1
+; BDVER2-NEXT:    vmovaps {{.*#+}} xmm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; BDVER2-NEXT:    vfnmaddps %xmm2, %xmm1, %xmm0, %xmm3
+; BDVER2-NEXT:    vfmaddps %xmm1, %xmm3, %xmm1, %xmm1
+; BDVER2-NEXT:    vfnmaddps %xmm2, %xmm1, %xmm0, %xmm0
+; BDVER2-NEXT:    vfmaddps %xmm1, %xmm0, %xmm1, %xmm0
+; BDVER2-NEXT:    retq
 ;
 ; BTVER2-LABEL: v4f32_two_step:
 ; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vmovaps {{.*#+}} xmm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0] sched: [5:1.00]
-; BTVER2-NEXT:    vrcpps %xmm0, %xmm1 # sched: [2:1.00]
-; BTVER2-NEXT:    vmulps %xmm1, %xmm0, %xmm2 # sched: [2:1.00]
-; BTVER2-NEXT:    vsubps %xmm2, %xmm3, %xmm2 # sched: [3:1.00]
-; BTVER2-NEXT:    vmulps %xmm2, %xmm1, %xmm2 # sched: [2:1.00]
-; BTVER2-NEXT:    vaddps %xmm2, %xmm1, %xmm1 # sched: [3:1.00]
-; BTVER2-NEXT:    vmulps %xmm1, %xmm0, %xmm0 # sched: [2:1.00]
-; BTVER2-NEXT:    vsubps %xmm0, %xmm3, %xmm0 # sched: [3:1.00]
-; BTVER2-NEXT:    vmulps %xmm0, %xmm1, %xmm0 # sched: [2:1.00]
-; BTVER2-NEXT:    vaddps %xmm0, %xmm1, %xmm0 # sched: [3:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
+; BTVER2-NEXT:    vmovaps {{.*#+}} xmm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; BTVER2-NEXT:    vrcpps %xmm0, %xmm1
+; BTVER2-NEXT:    vmulps %xmm1, %xmm0, %xmm2
+; BTVER2-NEXT:    vsubps %xmm2, %xmm3, %xmm2
+; BTVER2-NEXT:    vmulps %xmm2, %xmm1, %xmm2
+; BTVER2-NEXT:    vaddps %xmm2, %xmm1, %xmm1
+; BTVER2-NEXT:    vmulps %xmm1, %xmm0, %xmm0
+; BTVER2-NEXT:    vsubps %xmm0, %xmm3, %xmm0
+; BTVER2-NEXT:    vmulps %xmm0, %xmm1, %xmm0
+; BTVER2-NEXT:    vaddps %xmm0, %xmm1, %xmm0
+; BTVER2-NEXT:    retq
 ;
 ; SANDY-LABEL: v4f32_two_step:
 ; SANDY:       # %bb.0:
-; SANDY-NEXT:    vrcpps %xmm0, %xmm1 # sched: [5:1.00]
-; SANDY-NEXT:    vmulps %xmm1, %xmm0, %xmm2 # sched: [5:1.00]
-; SANDY-NEXT:    vmovaps {{.*#+}} xmm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0] sched: [6:0.50]
-; SANDY-NEXT:    vsubps %xmm2, %xmm3, %xmm2 # sched: [3:1.00]
-; SANDY-NEXT:    vmulps %xmm2, %xmm1, %xmm2 # sched: [5:1.00]
-; SANDY-NEXT:    vaddps %xmm2, %xmm1, %xmm1 # sched: [3:1.00]
-; SANDY-NEXT:    vmulps %xmm1, %xmm0, %xmm0 # sched: [5:1.00]
-; SANDY-NEXT:    vsubps %xmm0, %xmm3, %xmm0 # sched: [3:1.00]
-; SANDY-NEXT:    vmulps %xmm0, %xmm1, %xmm0 # sched: [5:1.00]
-; SANDY-NEXT:    vaddps %xmm0, %xmm1, %xmm0 # sched: [3:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
+; SANDY-NEXT:    vrcpps %xmm0, %xmm1
+; SANDY-NEXT:    vmulps %xmm1, %xmm0, %xmm2
+; SANDY-NEXT:    vmovaps {{.*#+}} xmm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; SANDY-NEXT:    vsubps %xmm2, %xmm3, %xmm2
+; SANDY-NEXT:    vmulps %xmm2, %xmm1, %xmm2
+; SANDY-NEXT:    vaddps %xmm2, %xmm1, %xmm1
+; SANDY-NEXT:    vmulps %xmm1, %xmm0, %xmm0
+; SANDY-NEXT:    vsubps %xmm0, %xmm3, %xmm0
+; SANDY-NEXT:    vmulps %xmm0, %xmm1, %xmm0
+; SANDY-NEXT:    vaddps %xmm0, %xmm1, %xmm0
+; SANDY-NEXT:    retq
 ;
 ; HASWELL-LABEL: v4f32_two_step:
 ; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vrcpps %xmm0, %xmm1 # sched: [5:1.00]
-; HASWELL-NEXT:    vbroadcastss {{.*#+}} xmm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0] sched: [6:0.50]
-; HASWELL-NEXT:    vmovaps %xmm1, %xmm3 # sched: [1:1.00]
-; HASWELL-NEXT:    vfnmadd213ps {{.*#+}} xmm3 = -(xmm0 * xmm3) + xmm2 sched: [5:0.50]
-; HASWELL-NEXT:    vfmadd132ps {{.*#+}} xmm3 = (xmm3 * xmm1) + xmm1 sched: [5:0.50]
-; HASWELL-NEXT:    vfnmadd213ps {{.*#+}} xmm0 = -(xmm3 * xmm0) + xmm2 sched: [5:0.50]
-; HASWELL-NEXT:    vfmadd132ps {{.*#+}} xmm0 = (xmm0 * xmm3) + xmm3 sched: [5:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
+; HASWELL-NEXT:    vrcpps %xmm0, %xmm1
+; HASWELL-NEXT:    vbroadcastss {{.*#+}} xmm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; HASWELL-NEXT:    vmovaps %xmm1, %xmm3
+; HASWELL-NEXT:    vfnmadd213ps {{.*#+}} xmm3 = -(xmm0 * xmm3) + xmm2
+; HASWELL-NEXT:    vfmadd132ps {{.*#+}} xmm3 = (xmm3 * xmm1) + xmm1
+; HASWELL-NEXT:    vfnmadd213ps {{.*#+}} xmm0 = -(xmm3 * xmm0) + xmm2
+; HASWELL-NEXT:    vfmadd132ps {{.*#+}} xmm0 = (xmm0 * xmm3) + xmm3
+; HASWELL-NEXT:    retq
 ;
 ; HASWELL-NO-FMA-LABEL: v4f32_two_step:
 ; HASWELL-NO-FMA:       # %bb.0:
@@ -567,27 +495,16 @@ define <4 x float> @v4f32_two_step(<4 x float> %x) #2 {
 ; HASWELL-NO-FMA-NEXT:    vaddps %xmm0, %xmm1, %xmm0
 ; HASWELL-NO-FMA-NEXT:    retq
 ;
-; KNL-LABEL: v4f32_two_step:
-; KNL:       # %bb.0:
-; KNL-NEXT:    vrcpps %xmm0, %xmm1 # sched: [5:1.00]
-; KNL-NEXT:    vbroadcastss {{.*#+}} xmm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0] sched: [6:0.50]
-; KNL-NEXT:    vmovaps %xmm1, %xmm3 # sched: [1:1.00]
-; KNL-NEXT:    vfnmadd213ps {{.*#+}} xmm3 = -(xmm0 * xmm3) + xmm2 sched: [5:0.50]
-; KNL-NEXT:    vfmadd132ps {{.*#+}} xmm3 = (xmm3 * xmm1) + xmm1 sched: [5:0.50]
-; KNL-NEXT:    vfnmadd213ps {{.*#+}} xmm0 = -(xmm3 * xmm0) + xmm2 sched: [5:0.50]
-; KNL-NEXT:    vfmadd132ps {{.*#+}} xmm0 = (xmm0 * xmm3) + xmm3 sched: [5:0.50]
-; KNL-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: v4f32_two_step:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vrcpps %xmm0, %xmm1 # sched: [4:1.00]
-; SKX-NEXT:    vbroadcastss {{.*#+}} xmm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0] sched: [6:0.50]
-; SKX-NEXT:    vmovaps %xmm1, %xmm3 # sched: [1:0.33]
-; SKX-NEXT:    vfnmadd213ps {{.*#+}} xmm3 = -(xmm0 * xmm3) + xmm2 sched: [4:0.50]
-; SKX-NEXT:    vfmadd132ps {{.*#+}} xmm3 = (xmm3 * xmm1) + xmm1 sched: [4:0.50]
-; SKX-NEXT:    vfnmadd213ps {{.*#+}} xmm0 = -(xmm3 * xmm0) + xmm2 sched: [4:0.50]
-; SKX-NEXT:    vfmadd132ps {{.*#+}} xmm0 = (xmm0 * xmm3) + xmm3 sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
+; AVX512-LABEL: v4f32_two_step:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vrcpps %xmm0, %xmm1
+; AVX512-NEXT:    vbroadcastss {{.*#+}} xmm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; AVX512-NEXT:    vmovaps %xmm1, %xmm3
+; AVX512-NEXT:    vfnmadd213ps {{.*#+}} xmm3 = -(xmm0 * xmm3) + xmm2
+; AVX512-NEXT:    vfmadd132ps {{.*#+}} xmm3 = (xmm3 * xmm1) + xmm1
+; AVX512-NEXT:    vfnmadd213ps {{.*#+}} xmm0 = -(xmm3 * xmm0) + xmm2
+; AVX512-NEXT:    vfmadd132ps {{.*#+}} xmm0 = (xmm0 * xmm3) + xmm3
+; AVX512-NEXT:    retq
   %div = fdiv fast <4 x float> <float 1.0, float 1.0, float 1.0, float 1.0>, %x
   ret <4 x float> %div
 }
@@ -617,27 +534,27 @@ define <8 x float> @v8f32_no_estimate(<8 x float> %x) #0 {
 ;
 ; BDVER2-LABEL: v8f32_no_estimate:
 ; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vmovaps {{.*#+}} ymm1 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0] sched: [5:0.50]
-; BDVER2-NEXT:    vdivps %ymm0, %ymm1, %ymm0 # sched: [9:19.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
+; BDVER2-NEXT:    vmovaps {{.*#+}} ymm1 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; BDVER2-NEXT:    vdivps %ymm0, %ymm1, %ymm0
+; BDVER2-NEXT:    retq
 ;
 ; BTVER2-LABEL: v8f32_no_estimate:
 ; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vmovaps {{.*#+}} ymm1 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0] sched: [5:1.00]
-; BTVER2-NEXT:    vdivps %ymm0, %ymm1, %ymm0 # sched: [38:38.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
+; BTVER2-NEXT:    vmovaps {{.*#+}} ymm1 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; BTVER2-NEXT:    vdivps %ymm0, %ymm1, %ymm0
+; BTVER2-NEXT:    retq
 ;
 ; SANDY-LABEL: v8f32_no_estimate:
 ; SANDY:       # %bb.0:
-; SANDY-NEXT:    vmovaps {{.*#+}} ymm1 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0] sched: [7:0.50]
-; SANDY-NEXT:    vdivps %ymm0, %ymm1, %ymm0 # sched: [29:28.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
+; SANDY-NEXT:    vmovaps {{.*#+}} ymm1 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; SANDY-NEXT:    vdivps %ymm0, %ymm1, %ymm0
+; SANDY-NEXT:    retq
 ;
 ; HASWELL-LABEL: v8f32_no_estimate:
 ; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vbroadcastss {{.*#+}} ymm1 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0] sched: [7:0.50]
-; HASWELL-NEXT:    vdivps %ymm0, %ymm1, %ymm0 # sched: [21:14.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
+; HASWELL-NEXT:    vbroadcastss {{.*#+}} ymm1 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; HASWELL-NEXT:    vdivps %ymm0, %ymm1, %ymm0
+; HASWELL-NEXT:    retq
 ;
 ; HASWELL-NO-FMA-LABEL: v8f32_no_estimate:
 ; HASWELL-NO-FMA:       # %bb.0:
@@ -645,17 +562,11 @@ define <8 x float> @v8f32_no_estimate(<8 x float> %x) #0 {
 ; HASWELL-NO-FMA-NEXT:    vdivps %ymm0, %ymm1, %ymm0
 ; HASWELL-NO-FMA-NEXT:    retq
 ;
-; KNL-LABEL: v8f32_no_estimate:
-; KNL:       # %bb.0:
-; KNL-NEXT:    vbroadcastss {{.*#+}} ymm1 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0] sched: [7:0.50]
-; KNL-NEXT:    vdivps %ymm0, %ymm1, %ymm0 # sched: [21:14.00]
-; KNL-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: v8f32_no_estimate:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vbroadcastss {{.*#+}} ymm1 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0] sched: [7:0.50]
-; SKX-NEXT:    vdivps %ymm0, %ymm1, %ymm0 # sched: [11:5.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
+; AVX512-LABEL: v8f32_no_estimate:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vbroadcastss {{.*#+}} ymm1 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; AVX512-NEXT:    vdivps %ymm0, %ymm1, %ymm0
+; AVX512-NEXT:    retq
   %div = fdiv fast <8 x float> <float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0>, %x
   ret <8 x float> %div
 }
@@ -698,38 +609,38 @@ define <8 x float> @v8f32_one_step(<8 x float> %x) #1 {
 ;
 ; BDVER2-LABEL: v8f32_one_step:
 ; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vrcpps %ymm0, %ymm1 # sched: [5:2.00]
-; BDVER2-NEXT:    vfnmaddps {{.*}}(%rip), %ymm1, %ymm0, %ymm0 # sched: [10:1.00]
-; BDVER2-NEXT:    vfmaddps %ymm1, %ymm0, %ymm1, %ymm0 # sched: [5:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
+; BDVER2-NEXT:    vrcpps %ymm0, %ymm1
+; BDVER2-NEXT:    vfnmaddps {{.*}}(%rip), %ymm1, %ymm0, %ymm0
+; BDVER2-NEXT:    vfmaddps %ymm1, %ymm0, %ymm1, %ymm0
+; BDVER2-NEXT:    retq
 ;
 ; BTVER2-LABEL: v8f32_one_step:
 ; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vmovaps {{.*#+}} ymm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0] sched: [5:1.00]
-; BTVER2-NEXT:    vrcpps %ymm0, %ymm1 # sched: [2:2.00]
-; BTVER2-NEXT:    vmulps %ymm1, %ymm0, %ymm0 # sched: [2:2.00]
-; BTVER2-NEXT:    vsubps %ymm0, %ymm2, %ymm0 # sched: [3:2.00]
-; BTVER2-NEXT:    vmulps %ymm0, %ymm1, %ymm0 # sched: [2:2.00]
-; BTVER2-NEXT:    vaddps %ymm0, %ymm1, %ymm0 # sched: [3:2.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
+; BTVER2-NEXT:    vmovaps {{.*#+}} ymm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; BTVER2-NEXT:    vrcpps %ymm0, %ymm1
+; BTVER2-NEXT:    vmulps %ymm1, %ymm0, %ymm0
+; BTVER2-NEXT:    vsubps %ymm0, %ymm2, %ymm0
+; BTVER2-NEXT:    vmulps %ymm0, %ymm1, %ymm0
+; BTVER2-NEXT:    vaddps %ymm0, %ymm1, %ymm0
+; BTVER2-NEXT:    retq
 ;
 ; SANDY-LABEL: v8f32_one_step:
 ; SANDY:       # %bb.0:
-; SANDY-NEXT:    vrcpps %ymm0, %ymm1 # sched: [7:2.00]
-; SANDY-NEXT:    vmulps %ymm1, %ymm0, %ymm0 # sched: [5:1.00]
-; SANDY-NEXT:    vmovaps {{.*#+}} ymm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0] sched: [7:0.50]
-; SANDY-NEXT:    vsubps %ymm0, %ymm2, %ymm0 # sched: [3:1.00]
-; SANDY-NEXT:    vmulps %ymm0, %ymm1, %ymm0 # sched: [5:1.00]
-; SANDY-NEXT:    vaddps %ymm0, %ymm1, %ymm0 # sched: [3:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
+; SANDY-NEXT:    vrcpps %ymm0, %ymm1
+; SANDY-NEXT:    vmulps %ymm1, %ymm0, %ymm0
+; SANDY-NEXT:    vmovaps {{.*#+}} ymm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; SANDY-NEXT:    vsubps %ymm0, %ymm2, %ymm0
+; SANDY-NEXT:    vmulps %ymm0, %ymm1, %ymm0
+; SANDY-NEXT:    vaddps %ymm0, %ymm1, %ymm0
+; SANDY-NEXT:    retq
 ;
 ; HASWELL-LABEL: v8f32_one_step:
 ; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vrcpps %ymm0, %ymm1 # sched: [11:2.00]
-; HASWELL-NEXT:    vbroadcastss {{.*#+}} ymm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0] sched: [7:0.50]
-; HASWELL-NEXT:    vfnmadd213ps {{.*#+}} ymm0 = -(ymm1 * ymm0) + ymm2 sched: [5:0.50]
-; HASWELL-NEXT:    vfmadd132ps {{.*#+}} ymm0 = (ymm0 * ymm1) + ymm1 sched: [5:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
+; HASWELL-NEXT:    vrcpps %ymm0, %ymm1
+; HASWELL-NEXT:    vbroadcastss {{.*#+}} ymm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; HASWELL-NEXT:    vfnmadd213ps {{.*#+}} ymm0 = -(ymm1 * ymm0) + ymm2
+; HASWELL-NEXT:    vfmadd132ps {{.*#+}} ymm0 = (ymm0 * ymm1) + ymm1
+; HASWELL-NEXT:    retq
 ;
 ; HASWELL-NO-FMA-LABEL: v8f32_one_step:
 ; HASWELL-NO-FMA:       # %bb.0:
@@ -743,18 +654,18 @@ define <8 x float> @v8f32_one_step(<8 x float> %x) #1 {
 ;
 ; KNL-LABEL: v8f32_one_step:
 ; KNL:       # %bb.0:
-; KNL-NEXT:    vrcpps %ymm0, %ymm1 # sched: [11:2.00]
-; KNL-NEXT:    vbroadcastss {{.*#+}} ymm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0] sched: [7:0.50]
-; KNL-NEXT:    vfnmadd213ps {{.*#+}} ymm0 = -(ymm1 * ymm0) + ymm2 sched: [5:0.50]
-; KNL-NEXT:    vfmadd132ps {{.*#+}} ymm0 = (ymm0 * ymm1) + ymm1 sched: [5:0.50]
-; KNL-NEXT:    retq # sched: [7:1.00]
+; KNL-NEXT:    vrcpps %ymm0, %ymm1
+; KNL-NEXT:    vbroadcastss {{.*#+}} ymm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; KNL-NEXT:    vfnmadd213ps {{.*#+}} ymm0 = -(ymm1 * ymm0) + ymm2
+; KNL-NEXT:    vfmadd132ps {{.*#+}} ymm0 = (ymm0 * ymm1) + ymm1
+; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: v8f32_one_step:
 ; SKX:       # %bb.0:
-; SKX-NEXT:    vrcpps %ymm0, %ymm1 # sched: [4:1.00]
-; SKX-NEXT:    vfnmadd213ps {{.*#+}} ymm0 = -(ymm1 * ymm0) + mem sched: [11:0.50]
-; SKX-NEXT:    vfmadd132ps {{.*#+}} ymm0 = (ymm0 * ymm1) + ymm1 sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
+; SKX-NEXT:    vrcpps %ymm0, %ymm1
+; SKX-NEXT:    vfnmadd213ps {{.*#+}} ymm0 = -(ymm1 * ymm0) + mem
+; SKX-NEXT:    vfmadd132ps {{.*#+}} ymm0 = (ymm0 * ymm1) + ymm1
+; SKX-NEXT:    retq
   %div = fdiv fast <8 x float> <float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0>, %x
   ret <8 x float> %div
 }
@@ -817,52 +728,52 @@ define <8 x float> @v8f32_two_step(<8 x float> %x) #2 {
 ;
 ; BDVER2-LABEL: v8f32_two_step:
 ; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vrcpps %ymm0, %ymm1 # sched: [5:2.00]
-; BDVER2-NEXT:    vmovaps {{.*#+}} ymm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0] sched: [5:0.50]
-; BDVER2-NEXT:    vfnmaddps %ymm2, %ymm1, %ymm0, %ymm3 # sched: [5:0.50]
-; BDVER2-NEXT:    vfmaddps %ymm1, %ymm3, %ymm1, %ymm1 # sched: [5:0.50]
-; BDVER2-NEXT:    vfnmaddps %ymm2, %ymm1, %ymm0, %ymm0 # sched: [5:0.50]
-; BDVER2-NEXT:    vfmaddps %ymm1, %ymm0, %ymm1, %ymm0 # sched: [5:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
+; BDVER2-NEXT:    vrcpps %ymm0, %ymm1
+; BDVER2-NEXT:    vmovaps {{.*#+}} ymm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; BDVER2-NEXT:    vfnmaddps %ymm2, %ymm1, %ymm0, %ymm3
+; BDVER2-NEXT:    vfmaddps %ymm1, %ymm3, %ymm1, %ymm1
+; BDVER2-NEXT:    vfnmaddps %ymm2, %ymm1, %ymm0, %ymm0
+; BDVER2-NEXT:    vfmaddps %ymm1, %ymm0, %ymm1, %ymm0
+; BDVER2-NEXT:    retq
 ;
 ; BTVER2-LABEL: v8f32_two_step:
 ; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vmovaps {{.*#+}} ymm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0] sched: [5:1.00]
-; BTVER2-NEXT:    vrcpps %ymm0, %ymm1 # sched: [2:2.00]
-; BTVER2-NEXT:    vmulps %ymm1, %ymm0, %ymm2 # sched: [2:2.00]
-; BTVER2-NEXT:    vsubps %ymm2, %ymm3, %ymm2 # sched: [3:2.00]
-; BTVER2-NEXT:    vmulps %ymm2, %ymm1, %ymm2 # sched: [2:2.00]
-; BTVER2-NEXT:    vaddps %ymm2, %ymm1, %ymm1 # sched: [3:2.00]
-; BTVER2-NEXT:    vmulps %ymm1, %ymm0, %ymm0 # sched: [2:2.00]
-; BTVER2-NEXT:    vsubps %ymm0, %ymm3, %ymm0 # sched: [3:2.00]
-; BTVER2-NEXT:    vmulps %ymm0, %ymm1, %ymm0 # sched: [2:2.00]
-; BTVER2-NEXT:    vaddps %ymm0, %ymm1, %ymm0 # sched: [3:2.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
+; BTVER2-NEXT:    vmovaps {{.*#+}} ymm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; BTVER2-NEXT:    vrcpps %ymm0, %ymm1
+; BTVER2-NEXT:    vmulps %ymm1, %ymm0, %ymm2
+; BTVER2-NEXT:    vsubps %ymm2, %ymm3, %ymm2
+; BTVER2-NEXT:    vmulps %ymm2, %ymm1, %ymm2
+; BTVER2-NEXT:    vaddps %ymm2, %ymm1, %ymm1
+; BTVER2-NEXT:    vmulps %ymm1, %ymm0, %ymm0
+; BTVER2-NEXT:    vsubps %ymm0, %ymm3, %ymm0
+; BTVER2-NEXT:    vmulps %ymm0, %ymm1, %ymm0
+; BTVER2-NEXT:    vaddps %ymm0, %ymm1, %ymm0
+; BTVER2-NEXT:    retq
 ;
 ; SANDY-LABEL: v8f32_two_step:
 ; SANDY:       # %bb.0:
-; SANDY-NEXT:    vrcpps %ymm0, %ymm1 # sched: [7:2.00]
-; SANDY-NEXT:    vmulps %ymm1, %ymm0, %ymm2 # sched: [5:1.00]
-; SANDY-NEXT:    vmovaps {{.*#+}} ymm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0] sched: [7:0.50]
-; SANDY-NEXT:    vsubps %ymm2, %ymm3, %ymm2 # sched: [3:1.00]
-; SANDY-NEXT:    vmulps %ymm2, %ymm1, %ymm2 # sched: [5:1.00]
-; SANDY-NEXT:    vaddps %ymm2, %ymm1, %ymm1 # sched: [3:1.00]
-; SANDY-NEXT:    vmulps %ymm1, %ymm0, %ymm0 # sched: [5:1.00]
-; SANDY-NEXT:    vsubps %ymm0, %ymm3, %ymm0 # sched: [3:1.00]
-; SANDY-NEXT:    vmulps %ymm0, %ymm1, %ymm0 # sched: [5:1.00]
-; SANDY-NEXT:    vaddps %ymm0, %ymm1, %ymm0 # sched: [3:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
+; SANDY-NEXT:    vrcpps %ymm0, %ymm1
+; SANDY-NEXT:    vmulps %ymm1, %ymm0, %ymm2
+; SANDY-NEXT:    vmovaps {{.*#+}} ymm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; SANDY-NEXT:    vsubps %ymm2, %ymm3, %ymm2
+; SANDY-NEXT:    vmulps %ymm2, %ymm1, %ymm2
+; SANDY-NEXT:    vaddps %ymm2, %ymm1, %ymm1
+; SANDY-NEXT:    vmulps %ymm1, %ymm0, %ymm0
+; SANDY-NEXT:    vsubps %ymm0, %ymm3, %ymm0
+; SANDY-NEXT:    vmulps %ymm0, %ymm1, %ymm0
+; SANDY-NEXT:    vaddps %ymm0, %ymm1, %ymm0
+; SANDY-NEXT:    retq
 ;
 ; HASWELL-LABEL: v8f32_two_step:
 ; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vrcpps %ymm0, %ymm1 # sched: [11:2.00]
-; HASWELL-NEXT:    vbroadcastss {{.*#+}} ymm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0] sched: [7:0.50]
-; HASWELL-NEXT:    vmovaps %ymm1, %ymm3 # sched: [1:1.00]
-; HASWELL-NEXT:    vfnmadd213ps {{.*#+}} ymm3 = -(ymm0 * ymm3) + ymm2 sched: [5:0.50]
-; HASWELL-NEXT:    vfmadd132ps {{.*#+}} ymm3 = (ymm3 * ymm1) + ymm1 sched: [5:0.50]
-; HASWELL-NEXT:    vfnmadd213ps {{.*#+}} ymm0 = -(ymm3 * ymm0) + ymm2 sched: [5:0.50]
-; HASWELL-NEXT:    vfmadd132ps {{.*#+}} ymm0 = (ymm0 * ymm3) + ymm3 sched: [5:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
+; HASWELL-NEXT:    vrcpps %ymm0, %ymm1
+; HASWELL-NEXT:    vbroadcastss {{.*#+}} ymm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; HASWELL-NEXT:    vmovaps %ymm1, %ymm3
+; HASWELL-NEXT:    vfnmadd213ps {{.*#+}} ymm3 = -(ymm0 * ymm3) + ymm2
+; HASWELL-NEXT:    vfmadd132ps {{.*#+}} ymm3 = (ymm3 * ymm1) + ymm1
+; HASWELL-NEXT:    vfnmadd213ps {{.*#+}} ymm0 = -(ymm3 * ymm0) + ymm2
+; HASWELL-NEXT:    vfmadd132ps {{.*#+}} ymm0 = (ymm0 * ymm3) + ymm3
+; HASWELL-NEXT:    retq
 ;
 ; HASWELL-NO-FMA-LABEL: v8f32_two_step:
 ; HASWELL-NO-FMA:       # %bb.0:
@@ -878,27 +789,16 @@ define <8 x float> @v8f32_two_step(<8 x float> %x) #2 {
 ; HASWELL-NO-FMA-NEXT:    vaddps %ymm0, %ymm1, %ymm0
 ; HASWELL-NO-FMA-NEXT:    retq
 ;
-; KNL-LABEL: v8f32_two_step:
-; KNL:       # %bb.0:
-; KNL-NEXT:    vrcpps %ymm0, %ymm1 # sched: [11:2.00]
-; KNL-NEXT:    vbroadcastss {{.*#+}} ymm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0] sched: [7:0.50]
-; KNL-NEXT:    vmovaps %ymm1, %ymm3 # sched: [1:1.00]
-; KNL-NEXT:    vfnmadd213ps {{.*#+}} ymm3 = -(ymm0 * ymm3) + ymm2 sched: [5:0.50]
-; KNL-NEXT:    vfmadd132ps {{.*#+}} ymm3 = (ymm3 * ymm1) + ymm1 sched: [5:0.50]
-; KNL-NEXT:    vfnmadd213ps {{.*#+}} ymm0 = -(ymm3 * ymm0) + ymm2 sched: [5:0.50]
-; KNL-NEXT:    vfmadd132ps {{.*#+}} ymm0 = (ymm0 * ymm3) + ymm3 sched: [5:0.50]
-; KNL-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: v8f32_two_step:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vrcpps %ymm0, %ymm1 # sched: [4:1.00]
-; SKX-NEXT:    vbroadcastss {{.*#+}} ymm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0] sched: [7:0.50]
-; SKX-NEXT:    vmovaps %ymm1, %ymm3 # sched: [1:0.33]
-; SKX-NEXT:    vfnmadd213ps {{.*#+}} ymm3 = -(ymm0 * ymm3) + ymm2 sched: [4:0.50]
-; SKX-NEXT:    vfmadd132ps {{.*#+}} ymm3 = (ymm3 * ymm1) + ymm1 sched: [4:0.50]
-; SKX-NEXT:    vfnmadd213ps {{.*#+}} ymm0 = -(ymm3 * ymm0) + ymm2 sched: [4:0.50]
-; SKX-NEXT:    vfmadd132ps {{.*#+}} ymm0 = (ymm0 * ymm3) + ymm3 sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
+; AVX512-LABEL: v8f32_two_step:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vrcpps %ymm0, %ymm1
+; AVX512-NEXT:    vbroadcastss {{.*#+}} ymm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; AVX512-NEXT:    vmovaps %ymm1, %ymm3
+; AVX512-NEXT:    vfnmadd213ps {{.*#+}} ymm3 = -(ymm0 * ymm3) + ymm2
+; AVX512-NEXT:    vfmadd132ps {{.*#+}} ymm3 = (ymm3 * ymm1) + ymm1
+; AVX512-NEXT:    vfnmadd213ps {{.*#+}} ymm0 = -(ymm3 * ymm0) + ymm2
+; AVX512-NEXT:    vfmadd132ps {{.*#+}} ymm0 = (ymm0 * ymm3) + ymm3
+; AVX512-NEXT:    retq
   %div = fdiv fast <8 x float> <float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0>, %x
   ret <8 x float> %div
 }
@@ -936,31 +836,31 @@ define <16 x float> @v16f32_no_estimate(<16 x float> %x) #0 {
 ;
 ; BDVER2-LABEL: v16f32_no_estimate:
 ; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vmovaps {{.*#+}} ymm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0] sched: [5:0.50]
-; BDVER2-NEXT:    vdivps %ymm0, %ymm2, %ymm0 # sched: [9:19.00]
-; BDVER2-NEXT:    vdivps %ymm1, %ymm2, %ymm1 # sched: [9:19.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
+; BDVER2-NEXT:    vmovaps {{.*#+}} ymm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; BDVER2-NEXT:    vdivps %ymm0, %ymm2, %ymm0
+; BDVER2-NEXT:    vdivps %ymm1, %ymm2, %ymm1
+; BDVER2-NEXT:    retq
 ;
 ; BTVER2-LABEL: v16f32_no_estimate:
 ; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vmovaps {{.*#+}} ymm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0] sched: [5:1.00]
-; BTVER2-NEXT:    vdivps %ymm0, %ymm2, %ymm0 # sched: [38:38.00]
-; BTVER2-NEXT:    vdivps %ymm1, %ymm2, %ymm1 # sched: [38:38.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
+; BTVER2-NEXT:    vmovaps {{.*#+}} ymm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; BTVER2-NEXT:    vdivps %ymm0, %ymm2, %ymm0
+; BTVER2-NEXT:    vdivps %ymm1, %ymm2, %ymm1
+; BTVER2-NEXT:    retq
 ;
 ; SANDY-LABEL: v16f32_no_estimate:
 ; SANDY:       # %bb.0:
-; SANDY-NEXT:    vmovaps {{.*#+}} ymm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0] sched: [7:0.50]
-; SANDY-NEXT:    vdivps %ymm0, %ymm2, %ymm0 # sched: [29:28.00]
-; SANDY-NEXT:    vdivps %ymm1, %ymm2, %ymm1 # sched: [29:28.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
+; SANDY-NEXT:    vmovaps {{.*#+}} ymm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; SANDY-NEXT:    vdivps %ymm0, %ymm2, %ymm0
+; SANDY-NEXT:    vdivps %ymm1, %ymm2, %ymm1
+; SANDY-NEXT:    retq
 ;
 ; HASWELL-LABEL: v16f32_no_estimate:
 ; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vbroadcastss {{.*#+}} ymm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0] sched: [7:0.50]
-; HASWELL-NEXT:    vdivps %ymm0, %ymm2, %ymm0 # sched: [21:14.00]
-; HASWELL-NEXT:    vdivps %ymm1, %ymm2, %ymm1 # sched: [21:14.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
+; HASWELL-NEXT:    vbroadcastss {{.*#+}} ymm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; HASWELL-NEXT:    vdivps %ymm0, %ymm2, %ymm0
+; HASWELL-NEXT:    vdivps %ymm1, %ymm2, %ymm1
+; HASWELL-NEXT:    retq
 ;
 ; HASWELL-NO-FMA-LABEL: v16f32_no_estimate:
 ; HASWELL-NO-FMA:       # %bb.0:
@@ -969,17 +869,11 @@ define <16 x float> @v16f32_no_estimate(<16 x float> %x) #0 {
 ; HASWELL-NO-FMA-NEXT:    vdivps %ymm1, %ymm2, %ymm1
 ; HASWELL-NO-FMA-NEXT:    retq
 ;
-; KNL-LABEL: v16f32_no_estimate:
-; KNL:       # %bb.0:
-; KNL-NEXT:    vbroadcastss {{.*#+}} zmm1 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0] sched: [10:1.00]
-; KNL-NEXT:    vdivps %zmm0, %zmm1, %zmm0 # sched: [21:14.00]
-; KNL-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: v16f32_no_estimate:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vbroadcastss {{.*#+}} zmm1 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0] sched: [8:0.50]
-; SKX-NEXT:    vdivps %zmm0, %zmm1, %zmm0 # sched: [18:10.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
+; AVX512-LABEL: v16f32_no_estimate:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vbroadcastss {{.*#+}} zmm1 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; AVX512-NEXT:    vdivps %zmm0, %zmm1, %zmm0
+; AVX512-NEXT:    retq
   %div = fdiv fast <16 x float> <float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0>, %x
   ret <16 x float> %div
 }
@@ -1045,55 +939,55 @@ define <16 x float> @v16f32_one_step(<16 x float> %x) #1 {
 ;
 ; BDVER2-LABEL: v16f32_one_step:
 ; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vrcpps %ymm0, %ymm2 # sched: [5:2.00]
-; BDVER2-NEXT:    vmovaps {{.*#+}} ymm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0] sched: [5:0.50]
-; BDVER2-NEXT:    vrcpps %ymm1, %ymm4 # sched: [5:2.00]
-; BDVER2-NEXT:    vfnmaddps %ymm3, %ymm2, %ymm0, %ymm0 # sched: [5:0.50]
-; BDVER2-NEXT:    vfnmaddps %ymm3, %ymm4, %ymm1, %ymm1 # sched: [5:0.50]
-; BDVER2-NEXT:    vfmaddps %ymm2, %ymm0, %ymm2, %ymm0 # sched: [5:0.50]
-; BDVER2-NEXT:    vfmaddps %ymm4, %ymm1, %ymm4, %ymm1 # sched: [5:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
+; BDVER2-NEXT:    vrcpps %ymm0, %ymm2
+; BDVER2-NEXT:    vmovaps {{.*#+}} ymm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; BDVER2-NEXT:    vrcpps %ymm1, %ymm4
+; BDVER2-NEXT:    vfnmaddps %ymm3, %ymm2, %ymm0, %ymm0
+; BDVER2-NEXT:    vfnmaddps %ymm3, %ymm4, %ymm1, %ymm1
+; BDVER2-NEXT:    vfmaddps %ymm2, %ymm0, %ymm2, %ymm0
+; BDVER2-NEXT:    vfmaddps %ymm4, %ymm1, %ymm4, %ymm1
+; BDVER2-NEXT:    retq
 ;
 ; BTVER2-LABEL: v16f32_one_step:
 ; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vmovaps {{.*#+}} ymm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0] sched: [5:1.00]
-; BTVER2-NEXT:    vrcpps %ymm0, %ymm2 # sched: [2:2.00]
-; BTVER2-NEXT:    vrcpps %ymm1, %ymm4 # sched: [2:2.00]
-; BTVER2-NEXT:    vmulps %ymm2, %ymm0, %ymm0 # sched: [2:2.00]
-; BTVER2-NEXT:    vmulps %ymm4, %ymm1, %ymm1 # sched: [2:2.00]
-; BTVER2-NEXT:    vsubps %ymm0, %ymm3, %ymm0 # sched: [3:2.00]
-; BTVER2-NEXT:    vsubps %ymm1, %ymm3, %ymm1 # sched: [3:2.00]
-; BTVER2-NEXT:    vmulps %ymm0, %ymm2, %ymm0 # sched: [2:2.00]
-; BTVER2-NEXT:    vmulps %ymm1, %ymm4, %ymm1 # sched: [2:2.00]
-; BTVER2-NEXT:    vaddps %ymm0, %ymm2, %ymm0 # sched: [3:2.00]
-; BTVER2-NEXT:    vaddps %ymm1, %ymm4, %ymm1 # sched: [3:2.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
+; BTVER2-NEXT:    vmovaps {{.*#+}} ymm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; BTVER2-NEXT:    vrcpps %ymm0, %ymm2
+; BTVER2-NEXT:    vrcpps %ymm1, %ymm4
+; BTVER2-NEXT:    vmulps %ymm2, %ymm0, %ymm0
+; BTVER2-NEXT:    vmulps %ymm4, %ymm1, %ymm1
+; BTVER2-NEXT:    vsubps %ymm0, %ymm3, %ymm0
+; BTVER2-NEXT:    vsubps %ymm1, %ymm3, %ymm1
+; BTVER2-NEXT:    vmulps %ymm0, %ymm2, %ymm0
+; BTVER2-NEXT:    vmulps %ymm1, %ymm4, %ymm1
+; BTVER2-NEXT:    vaddps %ymm0, %ymm2, %ymm0
+; BTVER2-NEXT:    vaddps %ymm1, %ymm4, %ymm1
+; BTVER2-NEXT:    retq
 ;
 ; SANDY-LABEL: v16f32_one_step:
 ; SANDY:       # %bb.0:
-; SANDY-NEXT:    vrcpps %ymm0, %ymm2 # sched: [7:2.00]
-; SANDY-NEXT:    vmulps %ymm2, %ymm0, %ymm0 # sched: [5:1.00]
-; SANDY-NEXT:    vmovaps {{.*#+}} ymm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0] sched: [7:0.50]
-; SANDY-NEXT:    vsubps %ymm0, %ymm3, %ymm0 # sched: [3:1.00]
-; SANDY-NEXT:    vmulps %ymm0, %ymm2, %ymm0 # sched: [5:1.00]
-; SANDY-NEXT:    vaddps %ymm0, %ymm2, %ymm0 # sched: [3:1.00]
-; SANDY-NEXT:    vrcpps %ymm1, %ymm2 # sched: [7:2.00]
-; SANDY-NEXT:    vmulps %ymm2, %ymm1, %ymm1 # sched: [5:1.00]
-; SANDY-NEXT:    vsubps %ymm1, %ymm3, %ymm1 # sched: [3:1.00]
-; SANDY-NEXT:    vmulps %ymm1, %ymm2, %ymm1 # sched: [5:1.00]
-; SANDY-NEXT:    vaddps %ymm1, %ymm2, %ymm1 # sched: [3:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
+; SANDY-NEXT:    vrcpps %ymm0, %ymm2
+; SANDY-NEXT:    vmulps %ymm2, %ymm0, %ymm0
+; SANDY-NEXT:    vmovaps {{.*#+}} ymm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; SANDY-NEXT:    vsubps %ymm0, %ymm3, %ymm0
+; SANDY-NEXT:    vmulps %ymm0, %ymm2, %ymm0
+; SANDY-NEXT:    vaddps %ymm0, %ymm2, %ymm0
+; SANDY-NEXT:    vrcpps %ymm1, %ymm2
+; SANDY-NEXT:    vmulps %ymm2, %ymm1, %ymm1
+; SANDY-NEXT:    vsubps %ymm1, %ymm3, %ymm1
+; SANDY-NEXT:    vmulps %ymm1, %ymm2, %ymm1
+; SANDY-NEXT:    vaddps %ymm1, %ymm2, %ymm1
+; SANDY-NEXT:    retq
 ;
 ; HASWELL-LABEL: v16f32_one_step:
 ; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vrcpps %ymm0, %ymm2 # sched: [11:2.00]
-; HASWELL-NEXT:    vbroadcastss {{.*#+}} ymm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0] sched: [7:0.50]
-; HASWELL-NEXT:    vrcpps %ymm1, %ymm4 # sched: [11:2.00]
-; HASWELL-NEXT:    vfnmadd213ps {{.*#+}} ymm0 = -(ymm2 * ymm0) + ymm3 sched: [5:0.50]
-; HASWELL-NEXT:    vfmadd132ps {{.*#+}} ymm0 = (ymm0 * ymm2) + ymm2 sched: [5:0.50]
-; HASWELL-NEXT:    vfnmadd213ps {{.*#+}} ymm1 = -(ymm4 * ymm1) + ymm3 sched: [5:0.50]
-; HASWELL-NEXT:    vfmadd132ps {{.*#+}} ymm1 = (ymm1 * ymm4) + ymm4 sched: [5:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
+; HASWELL-NEXT:    vrcpps %ymm0, %ymm2
+; HASWELL-NEXT:    vbroadcastss {{.*#+}} ymm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; HASWELL-NEXT:    vrcpps %ymm1, %ymm4
+; HASWELL-NEXT:    vfnmadd213ps {{.*#+}} ymm0 = -(ymm2 * ymm0) + ymm3
+; HASWELL-NEXT:    vfmadd132ps {{.*#+}} ymm0 = (ymm0 * ymm2) + ymm2
+; HASWELL-NEXT:    vfnmadd213ps {{.*#+}} ymm1 = -(ymm4 * ymm1) + ymm3
+; HASWELL-NEXT:    vfmadd132ps {{.*#+}} ymm1 = (ymm1 * ymm4) + ymm4
+; HASWELL-NEXT:    retq
 ;
 ; HASWELL-NO-FMA-LABEL: v16f32_one_step:
 ; HASWELL-NO-FMA:       # %bb.0:
@@ -1110,19 +1004,12 @@ define <16 x float> @v16f32_one_step(<16 x float> %x) #1 {
 ; HASWELL-NO-FMA-NEXT:    vaddps %ymm1, %ymm2, %ymm1
 ; HASWELL-NO-FMA-NEXT:    retq
 ;
-; KNL-LABEL: v16f32_one_step:
-; KNL:       # %bb.0:
-; KNL-NEXT:    vrcp14ps %zmm0, %zmm1 # sched: [11:2.00]
-; KNL-NEXT:    vfnmadd213ps {{.*#+}} zmm0 = -(zmm1 * zmm0) + mem sched: [12:0.50]
-; KNL-NEXT:    vfmadd132ps {{.*#+}} zmm0 = (zmm0 * zmm1) + zmm1 sched: [5:0.50]
-; KNL-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: v16f32_one_step:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vrcp14ps %zmm0, %zmm1 # sched: [4:2.00]
-; SKX-NEXT:    vfnmadd213ps {{.*#+}} zmm0 = -(zmm1 * zmm0) + mem sched: [11:0.50]
-; SKX-NEXT:    vfmadd132ps {{.*#+}} zmm0 = (zmm0 * zmm1) + zmm1 sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
+; AVX512-LABEL: v16f32_one_step:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vrcp14ps %zmm0, %zmm1
+; AVX512-NEXT:    vfnmadd213ps {{.*#+}} zmm0 = -(zmm1 * zmm0) + mem
+; AVX512-NEXT:    vfmadd132ps {{.*#+}} zmm0 = (zmm0 * zmm1) + zmm1
+; AVX512-NEXT:    retq
   %div = fdiv fast <16 x float> <float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0>, %x
   ret <16 x float> %div
 }
@@ -1226,81 +1113,81 @@ define <16 x float> @v16f32_two_step(<16 x float> %x) #2 {
 ;
 ; BDVER2-LABEL: v16f32_two_step:
 ; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vrcpps %ymm0, %ymm2 # sched: [5:2.00]
-; BDVER2-NEXT:    vmovaps {{.*#+}} ymm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0] sched: [5:0.50]
-; BDVER2-NEXT:    vfnmaddps %ymm3, %ymm2, %ymm0, %ymm4 # sched: [5:0.50]
-; BDVER2-NEXT:    vfmaddps %ymm2, %ymm4, %ymm2, %ymm2 # sched: [5:0.50]
-; BDVER2-NEXT:    vfnmaddps %ymm3, %ymm2, %ymm0, %ymm0 # sched: [5:0.50]
-; BDVER2-NEXT:    vfmaddps %ymm2, %ymm0, %ymm2, %ymm0 # sched: [5:0.50]
-; BDVER2-NEXT:    vrcpps %ymm1, %ymm2 # sched: [5:2.00]
-; BDVER2-NEXT:    vfnmaddps %ymm3, %ymm2, %ymm1, %ymm4 # sched: [5:0.50]
-; BDVER2-NEXT:    vfmaddps %ymm2, %ymm4, %ymm2, %ymm2 # sched: [5:0.50]
-; BDVER2-NEXT:    vfnmaddps %ymm3, %ymm2, %ymm1, %ymm1 # sched: [5:0.50]
-; BDVER2-NEXT:    vfmaddps %ymm2, %ymm1, %ymm2, %ymm1 # sched: [5:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
+; BDVER2-NEXT:    vrcpps %ymm0, %ymm2
+; BDVER2-NEXT:    vmovaps {{.*#+}} ymm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; BDVER2-NEXT:    vfnmaddps %ymm3, %ymm2, %ymm0, %ymm4
+; BDVER2-NEXT:    vfmaddps %ymm2, %ymm4, %ymm2, %ymm2
+; BDVER2-NEXT:    vfnmaddps %ymm3, %ymm2, %ymm0, %ymm0
+; BDVER2-NEXT:    vfmaddps %ymm2, %ymm0, %ymm2, %ymm0
+; BDVER2-NEXT:    vrcpps %ymm1, %ymm2
+; BDVER2-NEXT:    vfnmaddps %ymm3, %ymm2, %ymm1, %ymm4
+; BDVER2-NEXT:    vfmaddps %ymm2, %ymm4, %ymm2, %ymm2
+; BDVER2-NEXT:    vfnmaddps %ymm3, %ymm2, %ymm1, %ymm1
+; BDVER2-NEXT:    vfmaddps %ymm2, %ymm1, %ymm2, %ymm1
+; BDVER2-NEXT:    retq
 ;
 ; BTVER2-LABEL: v16f32_two_step:
 ; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vmovaps {{.*#+}} ymm4 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0] sched: [5:1.00]
-; BTVER2-NEXT:    vrcpps %ymm0, %ymm2 # sched: [2:2.00]
-; BTVER2-NEXT:    vmulps %ymm2, %ymm0, %ymm3 # sched: [2:2.00]
-; BTVER2-NEXT:    vsubps %ymm3, %ymm4, %ymm3 # sched: [3:2.00]
-; BTVER2-NEXT:    vmulps %ymm3, %ymm2, %ymm3 # sched: [2:2.00]
-; BTVER2-NEXT:    vaddps %ymm3, %ymm2, %ymm2 # sched: [3:2.00]
-; BTVER2-NEXT:    vmulps %ymm2, %ymm0, %ymm0 # sched: [2:2.00]
-; BTVER2-NEXT:    vsubps %ymm0, %ymm4, %ymm0 # sched: [3:2.00]
-; BTVER2-NEXT:    vmulps %ymm0, %ymm2, %ymm0 # sched: [2:2.00]
-; BTVER2-NEXT:    vaddps %ymm0, %ymm2, %ymm0 # sched: [3:2.00]
-; BTVER2-NEXT:    vrcpps %ymm1, %ymm2 # sched: [2:2.00]
-; BTVER2-NEXT:    vmulps %ymm2, %ymm1, %ymm3 # sched: [2:2.00]
-; BTVER2-NEXT:    vsubps %ymm3, %ymm4, %ymm3 # sched: [3:2.00]
-; BTVER2-NEXT:    vmulps %ymm3, %ymm2, %ymm3 # sched: [2:2.00]
-; BTVER2-NEXT:    vaddps %ymm3, %ymm2, %ymm2 # sched: [3:2.00]
-; BTVER2-NEXT:    vmulps %ymm2, %ymm1, %ymm1 # sched: [2:2.00]
-; BTVER2-NEXT:    vsubps %ymm1, %ymm4, %ymm1 # sched: [3:2.00]
-; BTVER2-NEXT:    vmulps %ymm1, %ymm2, %ymm1 # sched: [2:2.00]
-; BTVER2-NEXT:    vaddps %ymm1, %ymm2, %ymm1 # sched: [3:2.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
+; BTVER2-NEXT:    vmovaps {{.*#+}} ymm4 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; BTVER2-NEXT:    vrcpps %ymm0, %ymm2
+; BTVER2-NEXT:    vmulps %ymm2, %ymm0, %ymm3
+; BTVER2-NEXT:    vsubps %ymm3, %ymm4, %ymm3
+; BTVER2-NEXT:    vmulps %ymm3, %ymm2, %ymm3
+; BTVER2-NEXT:    vaddps %ymm3, %ymm2, %ymm2
+; BTVER2-NEXT:    vmulps %ymm2, %ymm0, %ymm0
+; BTVER2-NEXT:    vsubps %ymm0, %ymm4, %ymm0
+; BTVER2-NEXT:    vmulps %ymm0, %ymm2, %ymm0
+; BTVER2-NEXT:    vaddps %ymm0, %ymm2, %ymm0
+; BTVER2-NEXT:    vrcpps %ymm1, %ymm2
+; BTVER2-NEXT:    vmulps %ymm2, %ymm1, %ymm3
+; BTVER2-NEXT:    vsubps %ymm3, %ymm4, %ymm3
+; BTVER2-NEXT:    vmulps %ymm3, %ymm2, %ymm3
+; BTVER2-NEXT:    vaddps %ymm3, %ymm2, %ymm2
+; BTVER2-NEXT:    vmulps %ymm2, %ymm1, %ymm1
+; BTVER2-NEXT:    vsubps %ymm1, %ymm4, %ymm1
+; BTVER2-NEXT:    vmulps %ymm1, %ymm2, %ymm1
+; BTVER2-NEXT:    vaddps %ymm1, %ymm2, %ymm1
+; BTVER2-NEXT:    retq
 ;
 ; SANDY-LABEL: v16f32_two_step:
 ; SANDY:       # %bb.0:
-; SANDY-NEXT:    vrcpps %ymm0, %ymm2 # sched: [7:2.00]
-; SANDY-NEXT:    vmulps %ymm2, %ymm0, %ymm3 # sched: [5:1.00]
-; SANDY-NEXT:    vmovaps {{.*#+}} ymm4 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0] sched: [7:0.50]
-; SANDY-NEXT:    vsubps %ymm3, %ymm4, %ymm3 # sched: [3:1.00]
-; SANDY-NEXT:    vmulps %ymm3, %ymm2, %ymm3 # sched: [5:1.00]
-; SANDY-NEXT:    vaddps %ymm3, %ymm2, %ymm2 # sched: [3:1.00]
-; SANDY-NEXT:    vmulps %ymm2, %ymm0, %ymm0 # sched: [5:1.00]
-; SANDY-NEXT:    vsubps %ymm0, %ymm4, %ymm0 # sched: [3:1.00]
-; SANDY-NEXT:    vmulps %ymm0, %ymm2, %ymm0 # sched: [5:1.00]
-; SANDY-NEXT:    vaddps %ymm0, %ymm2, %ymm0 # sched: [3:1.00]
-; SANDY-NEXT:    vrcpps %ymm1, %ymm2 # sched: [7:2.00]
-; SANDY-NEXT:    vmulps %ymm2, %ymm1, %ymm3 # sched: [5:1.00]
-; SANDY-NEXT:    vsubps %ymm3, %ymm4, %ymm3 # sched: [3:1.00]
-; SANDY-NEXT:    vmulps %ymm3, %ymm2, %ymm3 # sched: [5:1.00]
-; SANDY-NEXT:    vaddps %ymm3, %ymm2, %ymm2 # sched: [3:1.00]
-; SANDY-NEXT:    vmulps %ymm2, %ymm1, %ymm1 # sched: [5:1.00]
-; SANDY-NEXT:    vsubps %ymm1, %ymm4, %ymm1 # sched: [3:1.00]
-; SANDY-NEXT:    vmulps %ymm1, %ymm2, %ymm1 # sched: [5:1.00]
-; SANDY-NEXT:    vaddps %ymm1, %ymm2, %ymm1 # sched: [3:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
+; SANDY-NEXT:    vrcpps %ymm0, %ymm2
+; SANDY-NEXT:    vmulps %ymm2, %ymm0, %ymm3
+; SANDY-NEXT:    vmovaps {{.*#+}} ymm4 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; SANDY-NEXT:    vsubps %ymm3, %ymm4, %ymm3
+; SANDY-NEXT:    vmulps %ymm3, %ymm2, %ymm3
+; SANDY-NEXT:    vaddps %ymm3, %ymm2, %ymm2
+; SANDY-NEXT:    vmulps %ymm2, %ymm0, %ymm0
+; SANDY-NEXT:    vsubps %ymm0, %ymm4, %ymm0
+; SANDY-NEXT:    vmulps %ymm0, %ymm2, %ymm0
+; SANDY-NEXT:    vaddps %ymm0, %ymm2, %ymm0
+; SANDY-NEXT:    vrcpps %ymm1, %ymm2
+; SANDY-NEXT:    vmulps %ymm2, %ymm1, %ymm3
+; SANDY-NEXT:    vsubps %ymm3, %ymm4, %ymm3
+; SANDY-NEXT:    vmulps %ymm3, %ymm2, %ymm3
+; SANDY-NEXT:    vaddps %ymm3, %ymm2, %ymm2
+; SANDY-NEXT:    vmulps %ymm2, %ymm1, %ymm1
+; SANDY-NEXT:    vsubps %ymm1, %ymm4, %ymm1
+; SANDY-NEXT:    vmulps %ymm1, %ymm2, %ymm1
+; SANDY-NEXT:    vaddps %ymm1, %ymm2, %ymm1
+; SANDY-NEXT:    retq
 ;
 ; HASWELL-LABEL: v16f32_two_step:
 ; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vrcpps %ymm0, %ymm2 # sched: [11:2.00]
-; HASWELL-NEXT:    vbroadcastss {{.*#+}} ymm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0] sched: [7:0.50]
-; HASWELL-NEXT:    vmovaps %ymm2, %ymm4 # sched: [1:1.00]
-; HASWELL-NEXT:    vfnmadd213ps {{.*#+}} ymm4 = -(ymm0 * ymm4) + ymm3 sched: [5:0.50]
-; HASWELL-NEXT:    vfmadd132ps {{.*#+}} ymm4 = (ymm4 * ymm2) + ymm2 sched: [5:0.50]
-; HASWELL-NEXT:    vfnmadd213ps {{.*#+}} ymm0 = -(ymm4 * ymm0) + ymm3 sched: [5:0.50]
-; HASWELL-NEXT:    vfmadd132ps {{.*#+}} ymm0 = (ymm0 * ymm4) + ymm4 sched: [5:0.50]
-; HASWELL-NEXT:    vrcpps %ymm1, %ymm2 # sched: [11:2.00]
-; HASWELL-NEXT:    vmovaps %ymm2, %ymm4 # sched: [1:1.00]
-; HASWELL-NEXT:    vfnmadd213ps {{.*#+}} ymm4 = -(ymm1 * ymm4) + ymm3 sched: [5:0.50]
-; HASWELL-NEXT:    vfmadd132ps {{.*#+}} ymm4 = (ymm4 * ymm2) + ymm2 sched: [5:0.50]
-; HASWELL-NEXT:    vfnmadd213ps {{.*#+}} ymm1 = -(ymm4 * ymm1) + ymm3 sched: [5:0.50]
-; HASWELL-NEXT:    vfmadd132ps {{.*#+}} ymm1 = (ymm1 * ymm4) + ymm4 sched: [5:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
+; HASWELL-NEXT:    vrcpps %ymm0, %ymm2
+; HASWELL-NEXT:    vbroadcastss {{.*#+}} ymm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; HASWELL-NEXT:    vmovaps %ymm2, %ymm4
+; HASWELL-NEXT:    vfnmadd213ps {{.*#+}} ymm4 = -(ymm0 * ymm4) + ymm3
+; HASWELL-NEXT:    vfmadd132ps {{.*#+}} ymm4 = (ymm4 * ymm2) + ymm2
+; HASWELL-NEXT:    vfnmadd213ps {{.*#+}} ymm0 = -(ymm4 * ymm0) + ymm3
+; HASWELL-NEXT:    vfmadd132ps {{.*#+}} ymm0 = (ymm0 * ymm4) + ymm4
+; HASWELL-NEXT:    vrcpps %ymm1, %ymm2
+; HASWELL-NEXT:    vmovaps %ymm2, %ymm4
+; HASWELL-NEXT:    vfnmadd213ps {{.*#+}} ymm4 = -(ymm1 * ymm4) + ymm3
+; HASWELL-NEXT:    vfmadd132ps {{.*#+}} ymm4 = (ymm4 * ymm2) + ymm2
+; HASWELL-NEXT:    vfnmadd213ps {{.*#+}} ymm1 = -(ymm4 * ymm1) + ymm3
+; HASWELL-NEXT:    vfmadd132ps {{.*#+}} ymm1 = (ymm1 * ymm4) + ymm4
+; HASWELL-NEXT:    retq
 ;
 ; HASWELL-NO-FMA-LABEL: v16f32_two_step:
 ; HASWELL-NO-FMA:       # %bb.0:
@@ -1325,27 +1212,16 @@ define <16 x float> @v16f32_two_step(<16 x float> %x) #2 {
 ; HASWELL-NO-FMA-NEXT:    vaddps %ymm1, %ymm2, %ymm1
 ; HASWELL-NO-FMA-NEXT:    retq
 ;
-; KNL-LABEL: v16f32_two_step:
-; KNL:       # %bb.0:
-; KNL-NEXT:    vrcp14ps %zmm0, %zmm1 # sched: [11:2.00]
-; KNL-NEXT:    vbroadcastss {{.*#+}} zmm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0] sched: [10:1.00]
-; KNL-NEXT:    vmovaps %zmm1, %zmm3 # sched: [1:1.00]
-; KNL-NEXT:    vfnmadd213ps {{.*#+}} zmm3 = -(zmm0 * zmm3) + zmm2 sched: [5:0.50]
-; KNL-NEXT:    vfmadd132ps {{.*#+}} zmm3 = (zmm3 * zmm1) + zmm1 sched: [5:0.50]
-; KNL-NEXT:    vfnmadd213ps {{.*#+}} zmm0 = -(zmm3 * zmm0) + zmm2 sched: [5:0.50]
-; KNL-NEXT:    vfmadd132ps {{.*#+}} zmm0 = (zmm0 * zmm3) + zmm3 sched: [5:0.50]
-; KNL-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: v16f32_two_step:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vrcp14ps %zmm0, %zmm1 # sched: [4:2.00]
-; SKX-NEXT:    vbroadcastss {{.*#+}} zmm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0] sched: [8:0.50]
-; SKX-NEXT:    vmovaps %zmm1, %zmm3 # sched: [1:0.33]
-; SKX-NEXT:    vfnmadd213ps {{.*#+}} zmm3 = -(zmm0 * zmm3) + zmm2 sched: [4:0.50]
-; SKX-NEXT:    vfmadd132ps {{.*#+}} zmm3 = (zmm3 * zmm1) + zmm1 sched: [4:0.50]
-; SKX-NEXT:    vfnmadd213ps {{.*#+}} zmm0 = -(zmm3 * zmm0) + zmm2 sched: [4:0.50]
-; SKX-NEXT:    vfmadd132ps {{.*#+}} zmm0 = (zmm0 * zmm3) + zmm3 sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
+; AVX512-LABEL: v16f32_two_step:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vrcp14ps %zmm0, %zmm1
+; AVX512-NEXT:    vbroadcastss {{.*#+}} zmm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; AVX512-NEXT:    vmovaps %zmm1, %zmm3
+; AVX512-NEXT:    vfnmadd213ps {{.*#+}} zmm3 = -(zmm0 * zmm3) + zmm2
+; AVX512-NEXT:    vfmadd132ps {{.*#+}} zmm3 = (zmm3 * zmm1) + zmm1
+; AVX512-NEXT:    vfnmadd213ps {{.*#+}} zmm0 = -(zmm3 * zmm0) + zmm2
+; AVX512-NEXT:    vfmadd132ps {{.*#+}} zmm0 = (zmm0 * zmm3) + zmm3
+; AVX512-NEXT:    retq
   %div = fdiv fast <16 x float> <float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0>, %x
   ret <16 x float> %div
 }
diff --git a/test/CodeGen/X86/recip-fastmath2.ll b/test/CodeGen/X86/recip-fastmath2.ll
index dbe2689077e1a1cf95db5326fbfdde2e3a0a69a3..de511b411b3f59bde810f87ddf2fae62ac645d04 100644
--- a/test/CodeGen/X86/recip-fastmath2.ll
+++ b/test/CodeGen/X86/recip-fastmath2.ll
@@ -1,14 +1,14 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2 -print-schedule      | FileCheck %s --check-prefix=CHECK --check-prefix=SSE --check-prefix=SSE-RECIP
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx -print-schedule       | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX-RECIP
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx,+fma -print-schedule  | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=FMA-RECIP
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=bdver2 -print-schedule      | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=BDVER2
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=btver2 -print-schedule      | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=BTVER2
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=sandybridge -print-schedule | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=SANDY
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=haswell -print-schedule     | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=HASWELL
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=haswell -print-schedule -mattr=-fma | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=HASWELL-NO-FMA
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=knl -print-schedule         | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX512 --check-prefix=KNL
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=skx -print-schedule         | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX512 --check-prefix=SKX
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2      | FileCheck %s --check-prefix=CHECK --check-prefix=SSE --check-prefix=SSE-RECIP
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx       | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX-RECIP
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx,+fma  | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=FMA-RECIP
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=bdver2      | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=BDVER2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=btver2      | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=BTVER2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=sandybridge | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=SANDY
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=haswell     | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=HASWELL
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=haswell -mattr=-fma | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=HASWELL-NO-FMA
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=knl         | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX512 --check-prefix=KNL
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=skx         | FileCheck %s --check-prefix=CHECK --check-prefix=AVX --check-prefix=AVX512 --check-prefix=SKX
 
 ; It's the extra tests coverage for recip as discussed on D26855.
 
@@ -19,59 +19,11 @@ define float @f32_no_step_2(float %x) #3 {
 ; SSE-NEXT:    mulss {{.*}}(%rip), %xmm0
 ; SSE-NEXT:    retq
 ;
-; AVX-RECIP-LABEL: f32_no_step_2:
-; AVX-RECIP:       # %bb.0:
-; AVX-RECIP-NEXT:    vrcpss %xmm0, %xmm0, %xmm0
-; AVX-RECIP-NEXT:    vmulss {{.*}}(%rip), %xmm0, %xmm0
-; AVX-RECIP-NEXT:    retq
-;
-; FMA-RECIP-LABEL: f32_no_step_2:
-; FMA-RECIP:       # %bb.0:
-; FMA-RECIP-NEXT:    vrcpss %xmm0, %xmm0, %xmm0
-; FMA-RECIP-NEXT:    vmulss {{.*}}(%rip), %xmm0, %xmm0
-; FMA-RECIP-NEXT:    retq
-;
-; BDVER2-LABEL: f32_no_step_2:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vrcpss %xmm0, %xmm0, %xmm0 # sched: [5:1.00]
-; BDVER2-NEXT:    vmulss {{.*}}(%rip), %xmm0, %xmm0 # sched: [10:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: f32_no_step_2:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vrcpss %xmm0, %xmm0, %xmm0 # sched: [2:1.00]
-; BTVER2-NEXT:    vmulss {{.*}}(%rip), %xmm0, %xmm0 # sched: [7:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: f32_no_step_2:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vrcpss %xmm0, %xmm0, %xmm0 # sched: [5:1.00]
-; SANDY-NEXT:    vmulss {{.*}}(%rip), %xmm0, %xmm0 # sched: [11:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: f32_no_step_2:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vrcpss %xmm0, %xmm0, %xmm0 # sched: [5:1.00]
-; HASWELL-NEXT:    vmulss {{.*}}(%rip), %xmm0, %xmm0 # sched: [10:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-NO-FMA-LABEL: f32_no_step_2:
-; HASWELL-NO-FMA:       # %bb.0:
-; HASWELL-NO-FMA-NEXT:    vrcpss %xmm0, %xmm0, %xmm0 # sched: [5:1.00]
-; HASWELL-NO-FMA-NEXT:    vmulss {{.*}}(%rip), %xmm0, %xmm0 # sched: [10:0.50]
-; HASWELL-NO-FMA-NEXT:    retq # sched: [7:1.00]
-;
-; KNL-LABEL: f32_no_step_2:
-; KNL:       # %bb.0:
-; KNL-NEXT:    vrcpss %xmm0, %xmm0, %xmm0 # sched: [5:1.00]
-; KNL-NEXT:    vmulss {{.*}}(%rip), %xmm0, %xmm0 # sched: [10:0.50]
-; KNL-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: f32_no_step_2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vrcpss %xmm0, %xmm0, %xmm0 # sched: [4:1.00]
-; SKX-NEXT:    vmulss {{.*}}(%rip), %xmm0, %xmm0 # sched: [9:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
+; AVX-LABEL: f32_no_step_2:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vrcpss %xmm0, %xmm0, %xmm0
+; AVX-NEXT:    vmulss {{.*}}(%rip), %xmm0, %xmm0
+; AVX-NEXT:    retq
   %div = fdiv fast float 1234.0, %x
   ret float %div
 }
@@ -110,68 +62,60 @@ define float @f32_one_step_2(float %x) #1 {
 ;
 ; BDVER2-LABEL: f32_one_step_2:
 ; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vrcpss %xmm0, %xmm0, %xmm1 # sched: [5:1.00]
-; BDVER2-NEXT:    vfnmaddss {{.*}}(%rip), %xmm1, %xmm0, %xmm0 # sched: [10:0.50]
-; BDVER2-NEXT:    vfmaddss %xmm1, %xmm0, %xmm1, %xmm0 # sched: [5:0.50]
-; BDVER2-NEXT:    vmulss {{.*}}(%rip), %xmm0, %xmm0 # sched: [10:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
+; BDVER2-NEXT:    vrcpss %xmm0, %xmm0, %xmm1
+; BDVER2-NEXT:    vfnmaddss {{.*}}(%rip), %xmm1, %xmm0, %xmm0
+; BDVER2-NEXT:    vfmaddss %xmm1, %xmm0, %xmm1, %xmm0
+; BDVER2-NEXT:    vmulss {{.*}}(%rip), %xmm0, %xmm0
+; BDVER2-NEXT:    retq
 ;
 ; BTVER2-LABEL: f32_one_step_2:
 ; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vmovss {{.*#+}} xmm2 = mem[0],zero,zero,zero sched: [5:1.00]
-; BTVER2-NEXT:    vrcpss %xmm0, %xmm0, %xmm1 # sched: [2:1.00]
-; BTVER2-NEXT:    vmulss %xmm1, %xmm0, %xmm0 # sched: [2:1.00]
-; BTVER2-NEXT:    vsubss %xmm0, %xmm2, %xmm0 # sched: [3:1.00]
-; BTVER2-NEXT:    vmulss %xmm0, %xmm1, %xmm0 # sched: [2:1.00]
-; BTVER2-NEXT:    vaddss %xmm0, %xmm1, %xmm0 # sched: [3:1.00]
-; BTVER2-NEXT:    vmulss {{.*}}(%rip), %xmm0, %xmm0 # sched: [7:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
+; BTVER2-NEXT:    vmovss {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; BTVER2-NEXT:    vrcpss %xmm0, %xmm0, %xmm1
+; BTVER2-NEXT:    vmulss %xmm1, %xmm0, %xmm0
+; BTVER2-NEXT:    vsubss %xmm0, %xmm2, %xmm0
+; BTVER2-NEXT:    vmulss %xmm0, %xmm1, %xmm0
+; BTVER2-NEXT:    vaddss %xmm0, %xmm1, %xmm0
+; BTVER2-NEXT:    vmulss {{.*}}(%rip), %xmm0, %xmm0
+; BTVER2-NEXT:    retq
 ;
 ; SANDY-LABEL: f32_one_step_2:
 ; SANDY:       # %bb.0:
-; SANDY-NEXT:    vrcpss %xmm0, %xmm0, %xmm1 # sched: [5:1.00]
-; SANDY-NEXT:    vmulss %xmm1, %xmm0, %xmm0 # sched: [5:1.00]
-; SANDY-NEXT:    vmovss {{.*#+}} xmm2 = mem[0],zero,zero,zero sched: [6:0.50]
-; SANDY-NEXT:    vsubss %xmm0, %xmm2, %xmm0 # sched: [3:1.00]
-; SANDY-NEXT:    vmulss %xmm0, %xmm1, %xmm0 # sched: [5:1.00]
-; SANDY-NEXT:    vaddss %xmm0, %xmm1, %xmm0 # sched: [3:1.00]
-; SANDY-NEXT:    vmulss {{.*}}(%rip), %xmm0, %xmm0 # sched: [11:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
+; SANDY-NEXT:    vrcpss %xmm0, %xmm0, %xmm1
+; SANDY-NEXT:    vmulss %xmm1, %xmm0, %xmm0
+; SANDY-NEXT:    vmovss {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; SANDY-NEXT:    vsubss %xmm0, %xmm2, %xmm0
+; SANDY-NEXT:    vmulss %xmm0, %xmm1, %xmm0
+; SANDY-NEXT:    vaddss %xmm0, %xmm1, %xmm0
+; SANDY-NEXT:    vmulss {{.*}}(%rip), %xmm0, %xmm0
+; SANDY-NEXT:    retq
 ;
 ; HASWELL-LABEL: f32_one_step_2:
 ; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vrcpss %xmm0, %xmm0, %xmm1 # sched: [5:1.00]
-; HASWELL-NEXT:    vfnmadd213ss {{.*#+}} xmm0 = -(xmm1 * xmm0) + mem sched: [10:0.50]
-; HASWELL-NEXT:    vfmadd132ss {{.*#+}} xmm0 = (xmm0 * xmm1) + xmm1 sched: [5:0.50]
-; HASWELL-NEXT:    vmulss {{.*}}(%rip), %xmm0, %xmm0 # sched: [10:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
+; HASWELL-NEXT:    vrcpss %xmm0, %xmm0, %xmm1
+; HASWELL-NEXT:    vfnmadd213ss {{.*#+}} xmm0 = -(xmm1 * xmm0) + mem
+; HASWELL-NEXT:    vfmadd132ss {{.*#+}} xmm0 = (xmm0 * xmm1) + xmm1
+; HASWELL-NEXT:    vmulss {{.*}}(%rip), %xmm0, %xmm0
+; HASWELL-NEXT:    retq
 ;
 ; HASWELL-NO-FMA-LABEL: f32_one_step_2:
 ; HASWELL-NO-FMA:       # %bb.0:
-; HASWELL-NO-FMA-NEXT:    vrcpss %xmm0, %xmm0, %xmm1 # sched: [5:1.00]
-; HASWELL-NO-FMA-NEXT:    vmulss %xmm1, %xmm0, %xmm0 # sched: [5:0.50]
-; HASWELL-NO-FMA-NEXT:    vmovss {{.*#+}} xmm2 = mem[0],zero,zero,zero sched: [5:0.50]
-; HASWELL-NO-FMA-NEXT:    vsubss %xmm0, %xmm2, %xmm0 # sched: [3:1.00]
-; HASWELL-NO-FMA-NEXT:    vmulss %xmm0, %xmm1, %xmm0 # sched: [5:0.50]
-; HASWELL-NO-FMA-NEXT:    vaddss %xmm0, %xmm1, %xmm0 # sched: [3:1.00]
-; HASWELL-NO-FMA-NEXT:    vmulss {{.*}}(%rip), %xmm0, %xmm0 # sched: [10:0.50]
-; HASWELL-NO-FMA-NEXT:    retq # sched: [7:1.00]
-;
-; KNL-LABEL: f32_one_step_2:
-; KNL:       # %bb.0:
-; KNL-NEXT:    vrcpss %xmm0, %xmm0, %xmm1 # sched: [5:1.00]
-; KNL-NEXT:    vfnmadd213ss {{.*#+}} xmm0 = -(xmm1 * xmm0) + mem sched: [10:0.50]
-; KNL-NEXT:    vfmadd132ss {{.*#+}} xmm0 = (xmm0 * xmm1) + xmm1 sched: [5:0.50]
-; KNL-NEXT:    vmulss {{.*}}(%rip), %xmm0, %xmm0 # sched: [10:0.50]
-; KNL-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: f32_one_step_2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vrcpss %xmm0, %xmm0, %xmm1 # sched: [4:1.00]
-; SKX-NEXT:    vfnmadd213ss {{.*#+}} xmm0 = -(xmm1 * xmm0) + mem sched: [9:0.50]
-; SKX-NEXT:    vfmadd132ss {{.*#+}} xmm0 = (xmm0 * xmm1) + xmm1 sched: [4:0.50]
-; SKX-NEXT:    vmulss {{.*}}(%rip), %xmm0, %xmm0 # sched: [9:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
+; HASWELL-NO-FMA-NEXT:    vrcpss %xmm0, %xmm0, %xmm1
+; HASWELL-NO-FMA-NEXT:    vmulss %xmm1, %xmm0, %xmm0
+; HASWELL-NO-FMA-NEXT:    vmovss {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; HASWELL-NO-FMA-NEXT:    vsubss %xmm0, %xmm2, %xmm0
+; HASWELL-NO-FMA-NEXT:    vmulss %xmm0, %xmm1, %xmm0
+; HASWELL-NO-FMA-NEXT:    vaddss %xmm0, %xmm1, %xmm0
+; HASWELL-NO-FMA-NEXT:    vmulss {{.*}}(%rip), %xmm0, %xmm0
+; HASWELL-NO-FMA-NEXT:    retq
+;
+; AVX512-LABEL: f32_one_step_2:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vrcpss %xmm0, %xmm0, %xmm1
+; AVX512-NEXT:    vfnmadd213ss {{.*#+}} xmm0 = -(xmm1 * xmm0) + mem
+; AVX512-NEXT:    vfmadd132ss {{.*#+}} xmm0 = (xmm0 * xmm1) + xmm1
+; AVX512-NEXT:    vmulss {{.*}}(%rip), %xmm0, %xmm0
+; AVX512-NEXT:    retq
   %div = fdiv fast float 3456.0, %x
   ret float %div
 }
@@ -213,75 +157,66 @@ define float @f32_one_step_2_divs(float %x) #1 {
 ;
 ; BDVER2-LABEL: f32_one_step_2_divs:
 ; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vrcpss %xmm0, %xmm0, %xmm1 # sched: [5:1.00]
-; BDVER2-NEXT:    vfnmaddss {{.*}}(%rip), %xmm1, %xmm0, %xmm0 # sched: [10:0.50]
-; BDVER2-NEXT:    vfmaddss %xmm1, %xmm0, %xmm1, %xmm0 # sched: [5:0.50]
-; BDVER2-NEXT:    vmulss {{.*}}(%rip), %xmm0, %xmm1 # sched: [10:1.00]
-; BDVER2-NEXT:    vmulss %xmm0, %xmm1, %xmm0 # sched: [5:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
+; BDVER2-NEXT:    vrcpss %xmm0, %xmm0, %xmm1
+; BDVER2-NEXT:    vfnmaddss {{.*}}(%rip), %xmm1, %xmm0, %xmm0
+; BDVER2-NEXT:    vfmaddss %xmm1, %xmm0, %xmm1, %xmm0
+; BDVER2-NEXT:    vmulss {{.*}}(%rip), %xmm0, %xmm1
+; BDVER2-NEXT:    vmulss %xmm0, %xmm1, %xmm0
+; BDVER2-NEXT:    retq
 ;
 ; BTVER2-LABEL: f32_one_step_2_divs:
 ; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vmovss {{.*#+}} xmm2 = mem[0],zero,zero,zero sched: [5:1.00]
-; BTVER2-NEXT:    vrcpss %xmm0, %xmm0, %xmm1 # sched: [2:1.00]
-; BTVER2-NEXT:    vmulss %xmm1, %xmm0, %xmm0 # sched: [2:1.00]
-; BTVER2-NEXT:    vsubss %xmm0, %xmm2, %xmm0 # sched: [3:1.00]
-; BTVER2-NEXT:    vmulss %xmm0, %xmm1, %xmm0 # sched: [2:1.00]
-; BTVER2-NEXT:    vaddss %xmm0, %xmm1, %xmm0 # sched: [3:1.00]
-; BTVER2-NEXT:    vmulss {{.*}}(%rip), %xmm0, %xmm1 # sched: [7:1.00]
-; BTVER2-NEXT:    vmulss %xmm0, %xmm1, %xmm0 # sched: [2:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
+; BTVER2-NEXT:    vmovss {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; BTVER2-NEXT:    vrcpss %xmm0, %xmm0, %xmm1
+; BTVER2-NEXT:    vmulss %xmm1, %xmm0, %xmm0
+; BTVER2-NEXT:    vsubss %xmm0, %xmm2, %xmm0
+; BTVER2-NEXT:    vmulss %xmm0, %xmm1, %xmm0
+; BTVER2-NEXT:    vaddss %xmm0, %xmm1, %xmm0
+; BTVER2-NEXT:    vmulss {{.*}}(%rip), %xmm0, %xmm1
+; BTVER2-NEXT:    vmulss %xmm0, %xmm1, %xmm0
+; BTVER2-NEXT:    retq
 ;
 ; SANDY-LABEL: f32_one_step_2_divs:
 ; SANDY:       # %bb.0:
-; SANDY-NEXT:    vrcpss %xmm0, %xmm0, %xmm1 # sched: [5:1.00]
-; SANDY-NEXT:    vmulss %xmm1, %xmm0, %xmm0 # sched: [5:1.00]
-; SANDY-NEXT:    vmovss {{.*#+}} xmm2 = mem[0],zero,zero,zero sched: [6:0.50]
-; SANDY-NEXT:    vsubss %xmm0, %xmm2, %xmm0 # sched: [3:1.00]
-; SANDY-NEXT:    vmulss %xmm0, %xmm1, %xmm0 # sched: [5:1.00]
-; SANDY-NEXT:    vaddss %xmm0, %xmm1, %xmm0 # sched: [3:1.00]
-; SANDY-NEXT:    vmulss {{.*}}(%rip), %xmm0, %xmm1 # sched: [11:1.00]
-; SANDY-NEXT:    vmulss %xmm0, %xmm1, %xmm0 # sched: [5:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
+; SANDY-NEXT:    vrcpss %xmm0, %xmm0, %xmm1
+; SANDY-NEXT:    vmulss %xmm1, %xmm0, %xmm0
+; SANDY-NEXT:    vmovss {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; SANDY-NEXT:    vsubss %xmm0, %xmm2, %xmm0
+; SANDY-NEXT:    vmulss %xmm0, %xmm1, %xmm0
+; SANDY-NEXT:    vaddss %xmm0, %xmm1, %xmm0
+; SANDY-NEXT:    vmulss {{.*}}(%rip), %xmm0, %xmm1
+; SANDY-NEXT:    vmulss %xmm0, %xmm1, %xmm0
+; SANDY-NEXT:    retq
 ;
 ; HASWELL-LABEL: f32_one_step_2_divs:
 ; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vrcpss %xmm0, %xmm0, %xmm1 # sched: [5:1.00]
-; HASWELL-NEXT:    vfnmadd213ss {{.*#+}} xmm0 = -(xmm1 * xmm0) + mem sched: [10:0.50]
-; HASWELL-NEXT:    vfmadd132ss {{.*#+}} xmm0 = (xmm0 * xmm1) + xmm1 sched: [5:0.50]
-; HASWELL-NEXT:    vmulss {{.*}}(%rip), %xmm0, %xmm1 # sched: [10:0.50]
-; HASWELL-NEXT:    vmulss %xmm0, %xmm1, %xmm0 # sched: [5:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
+; HASWELL-NEXT:    vrcpss %xmm0, %xmm0, %xmm1
+; HASWELL-NEXT:    vfnmadd213ss {{.*#+}} xmm0 = -(xmm1 * xmm0) + mem
+; HASWELL-NEXT:    vfmadd132ss {{.*#+}} xmm0 = (xmm0 * xmm1) + xmm1
+; HASWELL-NEXT:    vmulss {{.*}}(%rip), %xmm0, %xmm1
+; HASWELL-NEXT:    vmulss %xmm0, %xmm1, %xmm0
+; HASWELL-NEXT:    retq
 ;
 ; HASWELL-NO-FMA-LABEL: f32_one_step_2_divs:
 ; HASWELL-NO-FMA:       # %bb.0:
-; HASWELL-NO-FMA-NEXT:    vrcpss %xmm0, %xmm0, %xmm1 # sched: [5:1.00]
-; HASWELL-NO-FMA-NEXT:    vmulss %xmm1, %xmm0, %xmm0 # sched: [5:0.50]
-; HASWELL-NO-FMA-NEXT:    vmovss {{.*#+}} xmm2 = mem[0],zero,zero,zero sched: [5:0.50]
-; HASWELL-NO-FMA-NEXT:    vsubss %xmm0, %xmm2, %xmm0 # sched: [3:1.00]
-; HASWELL-NO-FMA-NEXT:    vmulss %xmm0, %xmm1, %xmm0 # sched: [5:0.50]
-; HASWELL-NO-FMA-NEXT:    vaddss %xmm0, %xmm1, %xmm0 # sched: [3:1.00]
-; HASWELL-NO-FMA-NEXT:    vmulss {{.*}}(%rip), %xmm0, %xmm1 # sched: [10:0.50]
-; HASWELL-NO-FMA-NEXT:    vmulss %xmm0, %xmm1, %xmm0 # sched: [5:0.50]
-; HASWELL-NO-FMA-NEXT:    retq # sched: [7:1.00]
-;
-; KNL-LABEL: f32_one_step_2_divs:
-; KNL:       # %bb.0:
-; KNL-NEXT:    vrcpss %xmm0, %xmm0, %xmm1 # sched: [5:1.00]
-; KNL-NEXT:    vfnmadd213ss {{.*#+}} xmm0 = -(xmm1 * xmm0) + mem sched: [10:0.50]
-; KNL-NEXT:    vfmadd132ss {{.*#+}} xmm0 = (xmm0 * xmm1) + xmm1 sched: [5:0.50]
-; KNL-NEXT:    vmulss {{.*}}(%rip), %xmm0, %xmm1 # sched: [10:0.50]
-; KNL-NEXT:    vmulss %xmm0, %xmm1, %xmm0 # sched: [5:0.50]
-; KNL-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: f32_one_step_2_divs:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vrcpss %xmm0, %xmm0, %xmm1 # sched: [4:1.00]
-; SKX-NEXT:    vfnmadd213ss {{.*#+}} xmm0 = -(xmm1 * xmm0) + mem sched: [9:0.50]
-; SKX-NEXT:    vfmadd132ss {{.*#+}} xmm0 = (xmm0 * xmm1) + xmm1 sched: [4:0.50]
-; SKX-NEXT:    vmulss {{.*}}(%rip), %xmm0, %xmm1 # sched: [9:0.50]
-; SKX-NEXT:    vmulss %xmm0, %xmm1, %xmm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
+; HASWELL-NO-FMA-NEXT:    vrcpss %xmm0, %xmm0, %xmm1
+; HASWELL-NO-FMA-NEXT:    vmulss %xmm1, %xmm0, %xmm0
+; HASWELL-NO-FMA-NEXT:    vmovss {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; HASWELL-NO-FMA-NEXT:    vsubss %xmm0, %xmm2, %xmm0
+; HASWELL-NO-FMA-NEXT:    vmulss %xmm0, %xmm1, %xmm0
+; HASWELL-NO-FMA-NEXT:    vaddss %xmm0, %xmm1, %xmm0
+; HASWELL-NO-FMA-NEXT:    vmulss {{.*}}(%rip), %xmm0, %xmm1
+; HASWELL-NO-FMA-NEXT:    vmulss %xmm0, %xmm1, %xmm0
+; HASWELL-NO-FMA-NEXT:    retq
+;
+; AVX512-LABEL: f32_one_step_2_divs:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vrcpss %xmm0, %xmm0, %xmm1
+; AVX512-NEXT:    vfnmadd213ss {{.*#+}} xmm0 = -(xmm1 * xmm0) + mem
+; AVX512-NEXT:    vfmadd132ss {{.*#+}} xmm0 = (xmm0 * xmm1) + xmm1
+; AVX512-NEXT:    vmulss {{.*}}(%rip), %xmm0, %xmm1
+; AVX512-NEXT:    vmulss %xmm0, %xmm1, %xmm0
+; AVX512-NEXT:    retq
   %div = fdiv fast float 3456.0, %x
   %div2 = fdiv fast float %div, %x
   ret float %div2
@@ -335,95 +270,83 @@ define float @f32_two_step_2(float %x) #2 {
 ;
 ; BDVER2-LABEL: f32_two_step_2:
 ; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vrcpss %xmm0, %xmm0, %xmm1 # sched: [5:1.00]
-; BDVER2-NEXT:    vmovss {{.*#+}} xmm2 = mem[0],zero,zero,zero sched: [5:0.50]
-; BDVER2-NEXT:    vfnmaddss %xmm2, %xmm1, %xmm0, %xmm3 # sched: [5:0.50]
-; BDVER2-NEXT:    vfmaddss %xmm1, %xmm3, %xmm1, %xmm1 # sched: [5:0.50]
-; BDVER2-NEXT:    vfnmaddss %xmm2, %xmm1, %xmm0, %xmm0 # sched: [5:0.50]
-; BDVER2-NEXT:    vfmaddss %xmm1, %xmm0, %xmm1, %xmm0 # sched: [5:0.50]
-; BDVER2-NEXT:    vmulss {{.*}}(%rip), %xmm0, %xmm0 # sched: [10:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
+; BDVER2-NEXT:    vrcpss %xmm0, %xmm0, %xmm1
+; BDVER2-NEXT:    vmovss {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; BDVER2-NEXT:    vfnmaddss %xmm2, %xmm1, %xmm0, %xmm3
+; BDVER2-NEXT:    vfmaddss %xmm1, %xmm3, %xmm1, %xmm1
+; BDVER2-NEXT:    vfnmaddss %xmm2, %xmm1, %xmm0, %xmm0
+; BDVER2-NEXT:    vfmaddss %xmm1, %xmm0, %xmm1, %xmm0
+; BDVER2-NEXT:    vmulss {{.*}}(%rip), %xmm0, %xmm0
+; BDVER2-NEXT:    retq
 ;
 ; BTVER2-LABEL: f32_two_step_2:
 ; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vmovss {{.*#+}} xmm3 = mem[0],zero,zero,zero sched: [5:1.00]
-; BTVER2-NEXT:    vrcpss %xmm0, %xmm0, %xmm1 # sched: [2:1.00]
-; BTVER2-NEXT:    vmulss %xmm1, %xmm0, %xmm2 # sched: [2:1.00]
-; BTVER2-NEXT:    vsubss %xmm2, %xmm3, %xmm2 # sched: [3:1.00]
-; BTVER2-NEXT:    vmulss %xmm2, %xmm1, %xmm2 # sched: [2:1.00]
-; BTVER2-NEXT:    vaddss %xmm2, %xmm1, %xmm1 # sched: [3:1.00]
-; BTVER2-NEXT:    vmulss %xmm1, %xmm0, %xmm0 # sched: [2:1.00]
-; BTVER2-NEXT:    vsubss %xmm0, %xmm3, %xmm0 # sched: [3:1.00]
-; BTVER2-NEXT:    vmulss %xmm0, %xmm1, %xmm0 # sched: [2:1.00]
-; BTVER2-NEXT:    vaddss %xmm0, %xmm1, %xmm0 # sched: [3:1.00]
-; BTVER2-NEXT:    vmulss {{.*}}(%rip), %xmm0, %xmm0 # sched: [7:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
+; BTVER2-NEXT:    vmovss {{.*#+}} xmm3 = mem[0],zero,zero,zero
+; BTVER2-NEXT:    vrcpss %xmm0, %xmm0, %xmm1
+; BTVER2-NEXT:    vmulss %xmm1, %xmm0, %xmm2
+; BTVER2-NEXT:    vsubss %xmm2, %xmm3, %xmm2
+; BTVER2-NEXT:    vmulss %xmm2, %xmm1, %xmm2
+; BTVER2-NEXT:    vaddss %xmm2, %xmm1, %xmm1
+; BTVER2-NEXT:    vmulss %xmm1, %xmm0, %xmm0
+; BTVER2-NEXT:    vsubss %xmm0, %xmm3, %xmm0
+; BTVER2-NEXT:    vmulss %xmm0, %xmm1, %xmm0
+; BTVER2-NEXT:    vaddss %xmm0, %xmm1, %xmm0
+; BTVER2-NEXT:    vmulss {{.*}}(%rip), %xmm0, %xmm0
+; BTVER2-NEXT:    retq
 ;
 ; SANDY-LABEL: f32_two_step_2:
 ; SANDY:       # %bb.0:
-; SANDY-NEXT:    vrcpss %xmm0, %xmm0, %xmm1 # sched: [5:1.00]
-; SANDY-NEXT:    vmulss %xmm1, %xmm0, %xmm2 # sched: [5:1.00]
-; SANDY-NEXT:    vmovss {{.*#+}} xmm3 = mem[0],zero,zero,zero sched: [6:0.50]
-; SANDY-NEXT:    vsubss %xmm2, %xmm3, %xmm2 # sched: [3:1.00]
-; SANDY-NEXT:    vmulss %xmm2, %xmm1, %xmm2 # sched: [5:1.00]
-; SANDY-NEXT:    vaddss %xmm2, %xmm1, %xmm1 # sched: [3:1.00]
-; SANDY-NEXT:    vmulss %xmm1, %xmm0, %xmm0 # sched: [5:1.00]
-; SANDY-NEXT:    vsubss %xmm0, %xmm3, %xmm0 # sched: [3:1.00]
-; SANDY-NEXT:    vmulss %xmm0, %xmm1, %xmm0 # sched: [5:1.00]
-; SANDY-NEXT:    vaddss %xmm0, %xmm1, %xmm0 # sched: [3:1.00]
-; SANDY-NEXT:    vmulss {{.*}}(%rip), %xmm0, %xmm0 # sched: [11:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
+; SANDY-NEXT:    vrcpss %xmm0, %xmm0, %xmm1
+; SANDY-NEXT:    vmulss %xmm1, %xmm0, %xmm2
+; SANDY-NEXT:    vmovss {{.*#+}} xmm3 = mem[0],zero,zero,zero
+; SANDY-NEXT:    vsubss %xmm2, %xmm3, %xmm2
+; SANDY-NEXT:    vmulss %xmm2, %xmm1, %xmm2
+; SANDY-NEXT:    vaddss %xmm2, %xmm1, %xmm1
+; SANDY-NEXT:    vmulss %xmm1, %xmm0, %xmm0
+; SANDY-NEXT:    vsubss %xmm0, %xmm3, %xmm0
+; SANDY-NEXT:    vmulss %xmm0, %xmm1, %xmm0
+; SANDY-NEXT:    vaddss %xmm0, %xmm1, %xmm0
+; SANDY-NEXT:    vmulss {{.*}}(%rip), %xmm0, %xmm0
+; SANDY-NEXT:    retq
 ;
 ; HASWELL-LABEL: f32_two_step_2:
 ; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vrcpss %xmm0, %xmm0, %xmm1 # sched: [5:1.00]
-; HASWELL-NEXT:    vmovss {{.*#+}} xmm2 = mem[0],zero,zero,zero sched: [5:0.50]
-; HASWELL-NEXT:    vmovaps %xmm1, %xmm3 # sched: [1:1.00]
-; HASWELL-NEXT:    vfnmadd213ss {{.*#+}} xmm3 = -(xmm0 * xmm3) + xmm2 sched: [5:0.50]
-; HASWELL-NEXT:    vfmadd132ss {{.*#+}} xmm3 = (xmm3 * xmm1) + xmm1 sched: [5:0.50]
-; HASWELL-NEXT:    vfnmadd213ss {{.*#+}} xmm0 = -(xmm3 * xmm0) + xmm2 sched: [5:0.50]
-; HASWELL-NEXT:    vfmadd132ss {{.*#+}} xmm0 = (xmm0 * xmm3) + xmm3 sched: [5:0.50]
-; HASWELL-NEXT:    vmulss {{.*}}(%rip), %xmm0, %xmm0 # sched: [10:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
+; HASWELL-NEXT:    vrcpss %xmm0, %xmm0, %xmm1
+; HASWELL-NEXT:    vmovss {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; HASWELL-NEXT:    vmovaps %xmm1, %xmm3
+; HASWELL-NEXT:    vfnmadd213ss {{.*#+}} xmm3 = -(xmm0 * xmm3) + xmm2
+; HASWELL-NEXT:    vfmadd132ss {{.*#+}} xmm3 = (xmm3 * xmm1) + xmm1
+; HASWELL-NEXT:    vfnmadd213ss {{.*#+}} xmm0 = -(xmm3 * xmm0) + xmm2
+; HASWELL-NEXT:    vfmadd132ss {{.*#+}} xmm0 = (xmm0 * xmm3) + xmm3
+; HASWELL-NEXT:    vmulss {{.*}}(%rip), %xmm0, %xmm0
+; HASWELL-NEXT:    retq
 ;
 ; HASWELL-NO-FMA-LABEL: f32_two_step_2:
 ; HASWELL-NO-FMA:       # %bb.0:
-; HASWELL-NO-FMA-NEXT:    vrcpss %xmm0, %xmm0, %xmm1 # sched: [5:1.00]
-; HASWELL-NO-FMA-NEXT:    vmulss %xmm1, %xmm0, %xmm2 # sched: [5:0.50]
-; HASWELL-NO-FMA-NEXT:    vmovss {{.*#+}} xmm3 = mem[0],zero,zero,zero sched: [5:0.50]
-; HASWELL-NO-FMA-NEXT:    vsubss %xmm2, %xmm3, %xmm2 # sched: [3:1.00]
-; HASWELL-NO-FMA-NEXT:    vmulss %xmm2, %xmm1, %xmm2 # sched: [5:0.50]
-; HASWELL-NO-FMA-NEXT:    vaddss %xmm2, %xmm1, %xmm1 # sched: [3:1.00]
-; HASWELL-NO-FMA-NEXT:    vmulss %xmm1, %xmm0, %xmm0 # sched: [5:0.50]
-; HASWELL-NO-FMA-NEXT:    vsubss %xmm0, %xmm3, %xmm0 # sched: [3:1.00]
-; HASWELL-NO-FMA-NEXT:    vmulss %xmm0, %xmm1, %xmm0 # sched: [5:0.50]
-; HASWELL-NO-FMA-NEXT:    vaddss %xmm0, %xmm1, %xmm0 # sched: [3:1.00]
-; HASWELL-NO-FMA-NEXT:    vmulss {{.*}}(%rip), %xmm0, %xmm0 # sched: [10:0.50]
-; HASWELL-NO-FMA-NEXT:    retq # sched: [7:1.00]
-;
-; KNL-LABEL: f32_two_step_2:
-; KNL:       # %bb.0:
-; KNL-NEXT:    vrcpss %xmm0, %xmm0, %xmm1 # sched: [5:1.00]
-; KNL-NEXT:    vmovss {{.*#+}} xmm2 = mem[0],zero,zero,zero sched: [5:0.50]
-; KNL-NEXT:    vmovaps %xmm1, %xmm3 # sched: [1:1.00]
-; KNL-NEXT:    vfnmadd213ss {{.*#+}} xmm3 = -(xmm0 * xmm3) + xmm2 sched: [5:0.50]
-; KNL-NEXT:    vfmadd132ss {{.*#+}} xmm3 = (xmm3 * xmm1) + xmm1 sched: [5:0.50]
-; KNL-NEXT:    vfnmadd213ss {{.*#+}} xmm0 = -(xmm3 * xmm0) + xmm2 sched: [5:0.50]
-; KNL-NEXT:    vfmadd132ss {{.*#+}} xmm0 = (xmm0 * xmm3) + xmm3 sched: [5:0.50]
-; KNL-NEXT:    vmulss {{.*}}(%rip), %xmm0, %xmm0 # sched: [10:0.50]
-; KNL-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: f32_two_step_2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vrcpss %xmm0, %xmm0, %xmm1 # sched: [4:1.00]
-; SKX-NEXT:    vmovss {{.*#+}} xmm2 = mem[0],zero,zero,zero sched: [5:0.50]
-; SKX-NEXT:    vmovaps %xmm1, %xmm3 # sched: [1:0.33]
-; SKX-NEXT:    vfnmadd213ss {{.*#+}} xmm3 = -(xmm0 * xmm3) + xmm2 sched: [4:0.50]
-; SKX-NEXT:    vfmadd132ss {{.*#+}} xmm3 = (xmm3 * xmm1) + xmm1 sched: [4:0.50]
-; SKX-NEXT:    vfnmadd213ss {{.*#+}} xmm0 = -(xmm3 * xmm0) + xmm2 sched: [4:0.50]
-; SKX-NEXT:    vfmadd132ss {{.*#+}} xmm0 = (xmm0 * xmm3) + xmm3 sched: [4:0.50]
-; SKX-NEXT:    vmulss {{.*}}(%rip), %xmm0, %xmm0 # sched: [9:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
+; HASWELL-NO-FMA-NEXT:    vrcpss %xmm0, %xmm0, %xmm1
+; HASWELL-NO-FMA-NEXT:    vmulss %xmm1, %xmm0, %xmm2
+; HASWELL-NO-FMA-NEXT:    vmovss {{.*#+}} xmm3 = mem[0],zero,zero,zero
+; HASWELL-NO-FMA-NEXT:    vsubss %xmm2, %xmm3, %xmm2
+; HASWELL-NO-FMA-NEXT:    vmulss %xmm2, %xmm1, %xmm2
+; HASWELL-NO-FMA-NEXT:    vaddss %xmm2, %xmm1, %xmm1
+; HASWELL-NO-FMA-NEXT:    vmulss %xmm1, %xmm0, %xmm0
+; HASWELL-NO-FMA-NEXT:    vsubss %xmm0, %xmm3, %xmm0
+; HASWELL-NO-FMA-NEXT:    vmulss %xmm0, %xmm1, %xmm0
+; HASWELL-NO-FMA-NEXT:    vaddss %xmm0, %xmm1, %xmm0
+; HASWELL-NO-FMA-NEXT:    vmulss {{.*}}(%rip), %xmm0, %xmm0
+; HASWELL-NO-FMA-NEXT:    retq
+;
+; AVX512-LABEL: f32_two_step_2:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vrcpss %xmm0, %xmm0, %xmm1
+; AVX512-NEXT:    vmovss {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; AVX512-NEXT:    vmovaps %xmm1, %xmm3
+; AVX512-NEXT:    vfnmadd213ss {{.*#+}} xmm3 = -(xmm0 * xmm3) + xmm2
+; AVX512-NEXT:    vfmadd132ss {{.*#+}} xmm3 = (xmm3 * xmm1) + xmm1
+; AVX512-NEXT:    vfnmadd213ss {{.*#+}} xmm0 = -(xmm3 * xmm0) + xmm2
+; AVX512-NEXT:    vfmadd132ss {{.*#+}} xmm0 = (xmm0 * xmm3) + xmm3
+; AVX512-NEXT:    vmulss {{.*}}(%rip), %xmm0, %xmm0
+; AVX512-NEXT:    retq
   %div = fdiv fast float 6789.0, %x
   ret float %div
 }
@@ -462,70 +385,70 @@ define <4 x float> @v4f32_one_step2(<4 x float> %x) #1 {
 ;
 ; BDVER2-LABEL: v4f32_one_step2:
 ; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vrcpps %xmm0, %xmm1 # sched: [5:1.00]
-; BDVER2-NEXT:    vfnmaddps {{.*}}(%rip), %xmm1, %xmm0, %xmm0 # sched: [10:0.50]
-; BDVER2-NEXT:    vfmaddps %xmm1, %xmm0, %xmm1, %xmm0 # sched: [5:0.50]
-; BDVER2-NEXT:    vmulps {{.*}}(%rip), %xmm0, %xmm0 # sched: [10:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
+; BDVER2-NEXT:    vrcpps %xmm0, %xmm1
+; BDVER2-NEXT:    vfnmaddps {{.*}}(%rip), %xmm1, %xmm0, %xmm0
+; BDVER2-NEXT:    vfmaddps %xmm1, %xmm0, %xmm1, %xmm0
+; BDVER2-NEXT:    vmulps {{.*}}(%rip), %xmm0, %xmm0
+; BDVER2-NEXT:    retq
 ;
 ; BTVER2-LABEL: v4f32_one_step2:
 ; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vmovaps {{.*#+}} xmm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0] sched: [5:1.00]
-; BTVER2-NEXT:    vrcpps %xmm0, %xmm1 # sched: [2:1.00]
-; BTVER2-NEXT:    vmulps %xmm1, %xmm0, %xmm0 # sched: [2:1.00]
-; BTVER2-NEXT:    vsubps %xmm0, %xmm2, %xmm0 # sched: [3:1.00]
-; BTVER2-NEXT:    vmulps %xmm0, %xmm1, %xmm0 # sched: [2:1.00]
-; BTVER2-NEXT:    vaddps %xmm0, %xmm1, %xmm0 # sched: [3:1.00]
-; BTVER2-NEXT:    vmulps {{.*}}(%rip), %xmm0, %xmm0 # sched: [7:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
+; BTVER2-NEXT:    vmovaps {{.*#+}} xmm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; BTVER2-NEXT:    vrcpps %xmm0, %xmm1
+; BTVER2-NEXT:    vmulps %xmm1, %xmm0, %xmm0
+; BTVER2-NEXT:    vsubps %xmm0, %xmm2, %xmm0
+; BTVER2-NEXT:    vmulps %xmm0, %xmm1, %xmm0
+; BTVER2-NEXT:    vaddps %xmm0, %xmm1, %xmm0
+; BTVER2-NEXT:    vmulps {{.*}}(%rip), %xmm0, %xmm0
+; BTVER2-NEXT:    retq
 ;
 ; SANDY-LABEL: v4f32_one_step2:
 ; SANDY:       # %bb.0:
-; SANDY-NEXT:    vrcpps %xmm0, %xmm1 # sched: [5:1.00]
-; SANDY-NEXT:    vmulps %xmm1, %xmm0, %xmm0 # sched: [5:1.00]
-; SANDY-NEXT:    vmovaps {{.*#+}} xmm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0] sched: [6:0.50]
-; SANDY-NEXT:    vsubps %xmm0, %xmm2, %xmm0 # sched: [3:1.00]
-; SANDY-NEXT:    vmulps %xmm0, %xmm1, %xmm0 # sched: [5:1.00]
-; SANDY-NEXT:    vaddps %xmm0, %xmm1, %xmm0 # sched: [3:1.00]
-; SANDY-NEXT:    vmulps {{.*}}(%rip), %xmm0, %xmm0 # sched: [11:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
+; SANDY-NEXT:    vrcpps %xmm0, %xmm1
+; SANDY-NEXT:    vmulps %xmm1, %xmm0, %xmm0
+; SANDY-NEXT:    vmovaps {{.*#+}} xmm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; SANDY-NEXT:    vsubps %xmm0, %xmm2, %xmm0
+; SANDY-NEXT:    vmulps %xmm0, %xmm1, %xmm0
+; SANDY-NEXT:    vaddps %xmm0, %xmm1, %xmm0
+; SANDY-NEXT:    vmulps {{.*}}(%rip), %xmm0, %xmm0
+; SANDY-NEXT:    retq
 ;
 ; HASWELL-LABEL: v4f32_one_step2:
 ; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vrcpps %xmm0, %xmm1 # sched: [5:1.00]
-; HASWELL-NEXT:    vbroadcastss {{.*#+}} xmm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0] sched: [6:0.50]
-; HASWELL-NEXT:    vfnmadd213ps {{.*#+}} xmm0 = -(xmm1 * xmm0) + xmm2 sched: [5:0.50]
-; HASWELL-NEXT:    vfmadd132ps {{.*#+}} xmm0 = (xmm0 * xmm1) + xmm1 sched: [5:0.50]
-; HASWELL-NEXT:    vmulps {{.*}}(%rip), %xmm0, %xmm0 # sched: [11:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
+; HASWELL-NEXT:    vrcpps %xmm0, %xmm1
+; HASWELL-NEXT:    vbroadcastss {{.*#+}} xmm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; HASWELL-NEXT:    vfnmadd213ps {{.*#+}} xmm0 = -(xmm1 * xmm0) + xmm2
+; HASWELL-NEXT:    vfmadd132ps {{.*#+}} xmm0 = (xmm0 * xmm1) + xmm1
+; HASWELL-NEXT:    vmulps {{.*}}(%rip), %xmm0, %xmm0
+; HASWELL-NEXT:    retq
 ;
 ; HASWELL-NO-FMA-LABEL: v4f32_one_step2:
 ; HASWELL-NO-FMA:       # %bb.0:
-; HASWELL-NO-FMA-NEXT:    vrcpps %xmm0, %xmm1 # sched: [5:1.00]
-; HASWELL-NO-FMA-NEXT:    vmulps %xmm1, %xmm0, %xmm0 # sched: [5:0.50]
-; HASWELL-NO-FMA-NEXT:    vbroadcastss {{.*#+}} xmm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0] sched: [6:0.50]
-; HASWELL-NO-FMA-NEXT:    vsubps %xmm0, %xmm2, %xmm0 # sched: [3:1.00]
-; HASWELL-NO-FMA-NEXT:    vmulps %xmm0, %xmm1, %xmm0 # sched: [5:0.50]
-; HASWELL-NO-FMA-NEXT:    vaddps %xmm0, %xmm1, %xmm0 # sched: [3:1.00]
-; HASWELL-NO-FMA-NEXT:    vmulps {{.*}}(%rip), %xmm0, %xmm0 # sched: [11:0.50]
-; HASWELL-NO-FMA-NEXT:    retq # sched: [7:1.00]
+; HASWELL-NO-FMA-NEXT:    vrcpps %xmm0, %xmm1
+; HASWELL-NO-FMA-NEXT:    vmulps %xmm1, %xmm0, %xmm0
+; HASWELL-NO-FMA-NEXT:    vbroadcastss {{.*#+}} xmm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; HASWELL-NO-FMA-NEXT:    vsubps %xmm0, %xmm2, %xmm0
+; HASWELL-NO-FMA-NEXT:    vmulps %xmm0, %xmm1, %xmm0
+; HASWELL-NO-FMA-NEXT:    vaddps %xmm0, %xmm1, %xmm0
+; HASWELL-NO-FMA-NEXT:    vmulps {{.*}}(%rip), %xmm0, %xmm0
+; HASWELL-NO-FMA-NEXT:    retq
 ;
 ; KNL-LABEL: v4f32_one_step2:
 ; KNL:       # %bb.0:
-; KNL-NEXT:    vrcpps %xmm0, %xmm1 # sched: [5:1.00]
-; KNL-NEXT:    vbroadcastss {{.*#+}} xmm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0] sched: [6:0.50]
-; KNL-NEXT:    vfnmadd213ps {{.*#+}} xmm0 = -(xmm1 * xmm0) + xmm2 sched: [5:0.50]
-; KNL-NEXT:    vfmadd132ps {{.*#+}} xmm0 = (xmm0 * xmm1) + xmm1 sched: [5:0.50]
-; KNL-NEXT:    vmulps {{.*}}(%rip), %xmm0, %xmm0 # sched: [11:0.50]
-; KNL-NEXT:    retq # sched: [7:1.00]
+; KNL-NEXT:    vrcpps %xmm0, %xmm1
+; KNL-NEXT:    vbroadcastss {{.*#+}} xmm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; KNL-NEXT:    vfnmadd213ps {{.*#+}} xmm0 = -(xmm1 * xmm0) + xmm2
+; KNL-NEXT:    vfmadd132ps {{.*#+}} xmm0 = (xmm0 * xmm1) + xmm1
+; KNL-NEXT:    vmulps {{.*}}(%rip), %xmm0, %xmm0
+; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: v4f32_one_step2:
 ; SKX:       # %bb.0:
-; SKX-NEXT:    vrcpps %xmm0, %xmm1 # sched: [4:1.00]
-; SKX-NEXT:    vfnmadd213ps {{.*#+}} xmm0 = -(xmm1 * xmm0) + mem sched: [10:0.50]
-; SKX-NEXT:    vfmadd132ps {{.*#+}} xmm0 = (xmm0 * xmm1) + xmm1 sched: [4:0.50]
-; SKX-NEXT:    vmulps {{.*}}(%rip), %xmm0, %xmm0 # sched: [10:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
+; SKX-NEXT:    vrcpps %xmm0, %xmm1
+; SKX-NEXT:    vfnmadd213ps {{.*#+}} xmm0 = -(xmm1 * xmm0) + mem
+; SKX-NEXT:    vfmadd132ps {{.*#+}} xmm0 = (xmm0 * xmm1) + xmm1
+; SKX-NEXT:    vmulps {{.*}}(%rip), %xmm0, %xmm0
+; SKX-NEXT:    retq
   %div = fdiv fast <4 x float> <float 1.0, float 2.0, float 3.0, float 4.0>, %x
   ret <4 x float> %div
 }
@@ -567,77 +490,77 @@ define <4 x float> @v4f32_one_step_2_divs(<4 x float> %x) #1 {
 ;
 ; BDVER2-LABEL: v4f32_one_step_2_divs:
 ; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vrcpps %xmm0, %xmm1 # sched: [5:1.00]
-; BDVER2-NEXT:    vfnmaddps {{.*}}(%rip), %xmm1, %xmm0, %xmm0 # sched: [10:0.50]
-; BDVER2-NEXT:    vfmaddps %xmm1, %xmm0, %xmm1, %xmm0 # sched: [5:0.50]
-; BDVER2-NEXT:    vmulps {{.*}}(%rip), %xmm0, %xmm1 # sched: [10:1.00]
-; BDVER2-NEXT:    vmulps %xmm0, %xmm1, %xmm0 # sched: [5:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
+; BDVER2-NEXT:    vrcpps %xmm0, %xmm1
+; BDVER2-NEXT:    vfnmaddps {{.*}}(%rip), %xmm1, %xmm0, %xmm0
+; BDVER2-NEXT:    vfmaddps %xmm1, %xmm0, %xmm1, %xmm0
+; BDVER2-NEXT:    vmulps {{.*}}(%rip), %xmm0, %xmm1
+; BDVER2-NEXT:    vmulps %xmm0, %xmm1, %xmm0
+; BDVER2-NEXT:    retq
 ;
 ; BTVER2-LABEL: v4f32_one_step_2_divs:
 ; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vmovaps {{.*#+}} xmm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0] sched: [5:1.00]
-; BTVER2-NEXT:    vrcpps %xmm0, %xmm1 # sched: [2:1.00]
-; BTVER2-NEXT:    vmulps %xmm1, %xmm0, %xmm0 # sched: [2:1.00]
-; BTVER2-NEXT:    vsubps %xmm0, %xmm2, %xmm0 # sched: [3:1.00]
-; BTVER2-NEXT:    vmulps %xmm0, %xmm1, %xmm0 # sched: [2:1.00]
-; BTVER2-NEXT:    vaddps %xmm0, %xmm1, %xmm0 # sched: [3:1.00]
-; BTVER2-NEXT:    vmulps {{.*}}(%rip), %xmm0, %xmm1 # sched: [7:1.00]
-; BTVER2-NEXT:    vmulps %xmm0, %xmm1, %xmm0 # sched: [2:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
+; BTVER2-NEXT:    vmovaps {{.*#+}} xmm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; BTVER2-NEXT:    vrcpps %xmm0, %xmm1
+; BTVER2-NEXT:    vmulps %xmm1, %xmm0, %xmm0
+; BTVER2-NEXT:    vsubps %xmm0, %xmm2, %xmm0
+; BTVER2-NEXT:    vmulps %xmm0, %xmm1, %xmm0
+; BTVER2-NEXT:    vaddps %xmm0, %xmm1, %xmm0
+; BTVER2-NEXT:    vmulps {{.*}}(%rip), %xmm0, %xmm1
+; BTVER2-NEXT:    vmulps %xmm0, %xmm1, %xmm0
+; BTVER2-NEXT:    retq
 ;
 ; SANDY-LABEL: v4f32_one_step_2_divs:
 ; SANDY:       # %bb.0:
-; SANDY-NEXT:    vrcpps %xmm0, %xmm1 # sched: [5:1.00]
-; SANDY-NEXT:    vmulps %xmm1, %xmm0, %xmm0 # sched: [5:1.00]
-; SANDY-NEXT:    vmovaps {{.*#+}} xmm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0] sched: [6:0.50]
-; SANDY-NEXT:    vsubps %xmm0, %xmm2, %xmm0 # sched: [3:1.00]
-; SANDY-NEXT:    vmulps %xmm0, %xmm1, %xmm0 # sched: [5:1.00]
-; SANDY-NEXT:    vaddps %xmm0, %xmm1, %xmm0 # sched: [3:1.00]
-; SANDY-NEXT:    vmulps {{.*}}(%rip), %xmm0, %xmm1 # sched: [11:1.00]
-; SANDY-NEXT:    vmulps %xmm0, %xmm1, %xmm0 # sched: [5:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
+; SANDY-NEXT:    vrcpps %xmm0, %xmm1
+; SANDY-NEXT:    vmulps %xmm1, %xmm0, %xmm0
+; SANDY-NEXT:    vmovaps {{.*#+}} xmm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; SANDY-NEXT:    vsubps %xmm0, %xmm2, %xmm0
+; SANDY-NEXT:    vmulps %xmm0, %xmm1, %xmm0
+; SANDY-NEXT:    vaddps %xmm0, %xmm1, %xmm0
+; SANDY-NEXT:    vmulps {{.*}}(%rip), %xmm0, %xmm1
+; SANDY-NEXT:    vmulps %xmm0, %xmm1, %xmm0
+; SANDY-NEXT:    retq
 ;
 ; HASWELL-LABEL: v4f32_one_step_2_divs:
 ; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vrcpps %xmm0, %xmm1 # sched: [5:1.00]
-; HASWELL-NEXT:    vbroadcastss {{.*#+}} xmm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0] sched: [6:0.50]
-; HASWELL-NEXT:    vfnmadd213ps {{.*#+}} xmm0 = -(xmm1 * xmm0) + xmm2 sched: [5:0.50]
-; HASWELL-NEXT:    vfmadd132ps {{.*#+}} xmm0 = (xmm0 * xmm1) + xmm1 sched: [5:0.50]
-; HASWELL-NEXT:    vmulps {{.*}}(%rip), %xmm0, %xmm1 # sched: [11:0.50]
-; HASWELL-NEXT:    vmulps %xmm0, %xmm1, %xmm0 # sched: [5:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
+; HASWELL-NEXT:    vrcpps %xmm0, %xmm1
+; HASWELL-NEXT:    vbroadcastss {{.*#+}} xmm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; HASWELL-NEXT:    vfnmadd213ps {{.*#+}} xmm0 = -(xmm1 * xmm0) + xmm2
+; HASWELL-NEXT:    vfmadd132ps {{.*#+}} xmm0 = (xmm0 * xmm1) + xmm1
+; HASWELL-NEXT:    vmulps {{.*}}(%rip), %xmm0, %xmm1
+; HASWELL-NEXT:    vmulps %xmm0, %xmm1, %xmm0
+; HASWELL-NEXT:    retq
 ;
 ; HASWELL-NO-FMA-LABEL: v4f32_one_step_2_divs:
 ; HASWELL-NO-FMA:       # %bb.0:
-; HASWELL-NO-FMA-NEXT:    vrcpps %xmm0, %xmm1 # sched: [5:1.00]
-; HASWELL-NO-FMA-NEXT:    vmulps %xmm1, %xmm0, %xmm0 # sched: [5:0.50]
-; HASWELL-NO-FMA-NEXT:    vbroadcastss {{.*#+}} xmm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0] sched: [6:0.50]
-; HASWELL-NO-FMA-NEXT:    vsubps %xmm0, %xmm2, %xmm0 # sched: [3:1.00]
-; HASWELL-NO-FMA-NEXT:    vmulps %xmm0, %xmm1, %xmm0 # sched: [5:0.50]
-; HASWELL-NO-FMA-NEXT:    vaddps %xmm0, %xmm1, %xmm0 # sched: [3:1.00]
-; HASWELL-NO-FMA-NEXT:    vmulps {{.*}}(%rip), %xmm0, %xmm1 # sched: [11:0.50]
-; HASWELL-NO-FMA-NEXT:    vmulps %xmm0, %xmm1, %xmm0 # sched: [5:0.50]
-; HASWELL-NO-FMA-NEXT:    retq # sched: [7:1.00]
+; HASWELL-NO-FMA-NEXT:    vrcpps %xmm0, %xmm1
+; HASWELL-NO-FMA-NEXT:    vmulps %xmm1, %xmm0, %xmm0
+; HASWELL-NO-FMA-NEXT:    vbroadcastss {{.*#+}} xmm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; HASWELL-NO-FMA-NEXT:    vsubps %xmm0, %xmm2, %xmm0
+; HASWELL-NO-FMA-NEXT:    vmulps %xmm0, %xmm1, %xmm0
+; HASWELL-NO-FMA-NEXT:    vaddps %xmm0, %xmm1, %xmm0
+; HASWELL-NO-FMA-NEXT:    vmulps {{.*}}(%rip), %xmm0, %xmm1
+; HASWELL-NO-FMA-NEXT:    vmulps %xmm0, %xmm1, %xmm0
+; HASWELL-NO-FMA-NEXT:    retq
 ;
 ; KNL-LABEL: v4f32_one_step_2_divs:
 ; KNL:       # %bb.0:
-; KNL-NEXT:    vrcpps %xmm0, %xmm1 # sched: [5:1.00]
-; KNL-NEXT:    vbroadcastss {{.*#+}} xmm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0] sched: [6:0.50]
-; KNL-NEXT:    vfnmadd213ps {{.*#+}} xmm0 = -(xmm1 * xmm0) + xmm2 sched: [5:0.50]
-; KNL-NEXT:    vfmadd132ps {{.*#+}} xmm0 = (xmm0 * xmm1) + xmm1 sched: [5:0.50]
-; KNL-NEXT:    vmulps {{.*}}(%rip), %xmm0, %xmm1 # sched: [11:0.50]
-; KNL-NEXT:    vmulps %xmm0, %xmm1, %xmm0 # sched: [5:0.50]
-; KNL-NEXT:    retq # sched: [7:1.00]
+; KNL-NEXT:    vrcpps %xmm0, %xmm1
+; KNL-NEXT:    vbroadcastss {{.*#+}} xmm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; KNL-NEXT:    vfnmadd213ps {{.*#+}} xmm0 = -(xmm1 * xmm0) + xmm2
+; KNL-NEXT:    vfmadd132ps {{.*#+}} xmm0 = (xmm0 * xmm1) + xmm1
+; KNL-NEXT:    vmulps {{.*}}(%rip), %xmm0, %xmm1
+; KNL-NEXT:    vmulps %xmm0, %xmm1, %xmm0
+; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: v4f32_one_step_2_divs:
 ; SKX:       # %bb.0:
-; SKX-NEXT:    vrcpps %xmm0, %xmm1 # sched: [4:1.00]
-; SKX-NEXT:    vfnmadd213ps {{.*#+}} xmm0 = -(xmm1 * xmm0) + mem sched: [10:0.50]
-; SKX-NEXT:    vfmadd132ps {{.*#+}} xmm0 = (xmm0 * xmm1) + xmm1 sched: [4:0.50]
-; SKX-NEXT:    vmulps {{.*}}(%rip), %xmm0, %xmm1 # sched: [10:0.50]
-; SKX-NEXT:    vmulps %xmm0, %xmm1, %xmm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
+; SKX-NEXT:    vrcpps %xmm0, %xmm1
+; SKX-NEXT:    vfnmadd213ps {{.*#+}} xmm0 = -(xmm1 * xmm0) + mem
+; SKX-NEXT:    vfmadd132ps {{.*#+}} xmm0 = (xmm0 * xmm1) + xmm1
+; SKX-NEXT:    vmulps {{.*}}(%rip), %xmm0, %xmm1
+; SKX-NEXT:    vmulps %xmm0, %xmm1, %xmm0
+; SKX-NEXT:    retq
   %div = fdiv fast <4 x float> <float 1.0, float 2.0, float 3.0, float 4.0>, %x
   %div2 = fdiv fast <4 x float> %div, %x
   ret <4 x float> %div2
@@ -691,95 +614,83 @@ define <4 x float> @v4f32_two_step2(<4 x float> %x) #2 {
 ;
 ; BDVER2-LABEL: v4f32_two_step2:
 ; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vrcpps %xmm0, %xmm1 # sched: [5:1.00]
-; BDVER2-NEXT:    vmovaps {{.*#+}} xmm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0] sched: [5:0.50]
-; BDVER2-NEXT:    vfnmaddps %xmm2, %xmm1, %xmm0, %xmm3 # sched: [5:0.50]
-; BDVER2-NEXT:    vfmaddps %xmm1, %xmm3, %xmm1, %xmm1 # sched: [5:0.50]
-; BDVER2-NEXT:    vfnmaddps %xmm2, %xmm1, %xmm0, %xmm0 # sched: [5:0.50]
-; BDVER2-NEXT:    vfmaddps %xmm1, %xmm0, %xmm1, %xmm0 # sched: [5:0.50]
-; BDVER2-NEXT:    vmulps {{.*}}(%rip), %xmm0, %xmm0 # sched: [10:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
+; BDVER2-NEXT:    vrcpps %xmm0, %xmm1
+; BDVER2-NEXT:    vmovaps {{.*#+}} xmm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; BDVER2-NEXT:    vfnmaddps %xmm2, %xmm1, %xmm0, %xmm3
+; BDVER2-NEXT:    vfmaddps %xmm1, %xmm3, %xmm1, %xmm1
+; BDVER2-NEXT:    vfnmaddps %xmm2, %xmm1, %xmm0, %xmm0
+; BDVER2-NEXT:    vfmaddps %xmm1, %xmm0, %xmm1, %xmm0
+; BDVER2-NEXT:    vmulps {{.*}}(%rip), %xmm0, %xmm0
+; BDVER2-NEXT:    retq
 ;
 ; BTVER2-LABEL: v4f32_two_step2:
 ; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vmovaps {{.*#+}} xmm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0] sched: [5:1.00]
-; BTVER2-NEXT:    vrcpps %xmm0, %xmm1 # sched: [2:1.00]
-; BTVER2-NEXT:    vmulps %xmm1, %xmm0, %xmm2 # sched: [2:1.00]
-; BTVER2-NEXT:    vsubps %xmm2, %xmm3, %xmm2 # sched: [3:1.00]
-; BTVER2-NEXT:    vmulps %xmm2, %xmm1, %xmm2 # sched: [2:1.00]
-; BTVER2-NEXT:    vaddps %xmm2, %xmm1, %xmm1 # sched: [3:1.00]
-; BTVER2-NEXT:    vmulps %xmm1, %xmm0, %xmm0 # sched: [2:1.00]
-; BTVER2-NEXT:    vsubps %xmm0, %xmm3, %xmm0 # sched: [3:1.00]
-; BTVER2-NEXT:    vmulps %xmm0, %xmm1, %xmm0 # sched: [2:1.00]
-; BTVER2-NEXT:    vaddps %xmm0, %xmm1, %xmm0 # sched: [3:1.00]
-; BTVER2-NEXT:    vmulps {{.*}}(%rip), %xmm0, %xmm0 # sched: [7:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
+; BTVER2-NEXT:    vmovaps {{.*#+}} xmm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; BTVER2-NEXT:    vrcpps %xmm0, %xmm1
+; BTVER2-NEXT:    vmulps %xmm1, %xmm0, %xmm2
+; BTVER2-NEXT:    vsubps %xmm2, %xmm3, %xmm2
+; BTVER2-NEXT:    vmulps %xmm2, %xmm1, %xmm2
+; BTVER2-NEXT:    vaddps %xmm2, %xmm1, %xmm1
+; BTVER2-NEXT:    vmulps %xmm1, %xmm0, %xmm0
+; BTVER2-NEXT:    vsubps %xmm0, %xmm3, %xmm0
+; BTVER2-NEXT:    vmulps %xmm0, %xmm1, %xmm0
+; BTVER2-NEXT:    vaddps %xmm0, %xmm1, %xmm0
+; BTVER2-NEXT:    vmulps {{.*}}(%rip), %xmm0, %xmm0
+; BTVER2-NEXT:    retq
 ;
 ; SANDY-LABEL: v4f32_two_step2:
 ; SANDY:       # %bb.0:
-; SANDY-NEXT:    vrcpps %xmm0, %xmm1 # sched: [5:1.00]
-; SANDY-NEXT:    vmulps %xmm1, %xmm0, %xmm2 # sched: [5:1.00]
-; SANDY-NEXT:    vmovaps {{.*#+}} xmm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0] sched: [6:0.50]
-; SANDY-NEXT:    vsubps %xmm2, %xmm3, %xmm2 # sched: [3:1.00]
-; SANDY-NEXT:    vmulps %xmm2, %xmm1, %xmm2 # sched: [5:1.00]
-; SANDY-NEXT:    vaddps %xmm2, %xmm1, %xmm1 # sched: [3:1.00]
-; SANDY-NEXT:    vmulps %xmm1, %xmm0, %xmm0 # sched: [5:1.00]
-; SANDY-NEXT:    vsubps %xmm0, %xmm3, %xmm0 # sched: [3:1.00]
-; SANDY-NEXT:    vmulps %xmm0, %xmm1, %xmm0 # sched: [5:1.00]
-; SANDY-NEXT:    vaddps %xmm0, %xmm1, %xmm0 # sched: [3:1.00]
-; SANDY-NEXT:    vmulps {{.*}}(%rip), %xmm0, %xmm0 # sched: [11:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
+; SANDY-NEXT:    vrcpps %xmm0, %xmm1
+; SANDY-NEXT:    vmulps %xmm1, %xmm0, %xmm2
+; SANDY-NEXT:    vmovaps {{.*#+}} xmm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; SANDY-NEXT:    vsubps %xmm2, %xmm3, %xmm2
+; SANDY-NEXT:    vmulps %xmm2, %xmm1, %xmm2
+; SANDY-NEXT:    vaddps %xmm2, %xmm1, %xmm1
+; SANDY-NEXT:    vmulps %xmm1, %xmm0, %xmm0
+; SANDY-NEXT:    vsubps %xmm0, %xmm3, %xmm0
+; SANDY-NEXT:    vmulps %xmm0, %xmm1, %xmm0
+; SANDY-NEXT:    vaddps %xmm0, %xmm1, %xmm0
+; SANDY-NEXT:    vmulps {{.*}}(%rip), %xmm0, %xmm0
+; SANDY-NEXT:    retq
 ;
 ; HASWELL-LABEL: v4f32_two_step2:
 ; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vrcpps %xmm0, %xmm1 # sched: [5:1.00]
-; HASWELL-NEXT:    vbroadcastss {{.*#+}} xmm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0] sched: [6:0.50]
-; HASWELL-NEXT:    vmovaps %xmm1, %xmm3 # sched: [1:1.00]
-; HASWELL-NEXT:    vfnmadd213ps {{.*#+}} xmm3 = -(xmm0 * xmm3) + xmm2 sched: [5:0.50]
-; HASWELL-NEXT:    vfmadd132ps {{.*#+}} xmm3 = (xmm3 * xmm1) + xmm1 sched: [5:0.50]
-; HASWELL-NEXT:    vfnmadd213ps {{.*#+}} xmm0 = -(xmm3 * xmm0) + xmm2 sched: [5:0.50]
-; HASWELL-NEXT:    vfmadd132ps {{.*#+}} xmm0 = (xmm0 * xmm3) + xmm3 sched: [5:0.50]
-; HASWELL-NEXT:    vmulps {{.*}}(%rip), %xmm0, %xmm0 # sched: [11:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
+; HASWELL-NEXT:    vrcpps %xmm0, %xmm1
+; HASWELL-NEXT:    vbroadcastss {{.*#+}} xmm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; HASWELL-NEXT:    vmovaps %xmm1, %xmm3
+; HASWELL-NEXT:    vfnmadd213ps {{.*#+}} xmm3 = -(xmm0 * xmm3) + xmm2
+; HASWELL-NEXT:    vfmadd132ps {{.*#+}} xmm3 = (xmm3 * xmm1) + xmm1
+; HASWELL-NEXT:    vfnmadd213ps {{.*#+}} xmm0 = -(xmm3 * xmm0) + xmm2
+; HASWELL-NEXT:    vfmadd132ps {{.*#+}} xmm0 = (xmm0 * xmm3) + xmm3
+; HASWELL-NEXT:    vmulps {{.*}}(%rip), %xmm0, %xmm0
+; HASWELL-NEXT:    retq
 ;
 ; HASWELL-NO-FMA-LABEL: v4f32_two_step2:
 ; HASWELL-NO-FMA:       # %bb.0:
-; HASWELL-NO-FMA-NEXT:    vrcpps %xmm0, %xmm1 # sched: [5:1.00]
-; HASWELL-NO-FMA-NEXT:    vmulps %xmm1, %xmm0, %xmm2 # sched: [5:0.50]
-; HASWELL-NO-FMA-NEXT:    vbroadcastss {{.*#+}} xmm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0] sched: [6:0.50]
-; HASWELL-NO-FMA-NEXT:    vsubps %xmm2, %xmm3, %xmm2 # sched: [3:1.00]
-; HASWELL-NO-FMA-NEXT:    vmulps %xmm2, %xmm1, %xmm2 # sched: [5:0.50]
-; HASWELL-NO-FMA-NEXT:    vaddps %xmm2, %xmm1, %xmm1 # sched: [3:1.00]
-; HASWELL-NO-FMA-NEXT:    vmulps %xmm1, %xmm0, %xmm0 # sched: [5:0.50]
-; HASWELL-NO-FMA-NEXT:    vsubps %xmm0, %xmm3, %xmm0 # sched: [3:1.00]
-; HASWELL-NO-FMA-NEXT:    vmulps %xmm0, %xmm1, %xmm0 # sched: [5:0.50]
-; HASWELL-NO-FMA-NEXT:    vaddps %xmm0, %xmm1, %xmm0 # sched: [3:1.00]
-; HASWELL-NO-FMA-NEXT:    vmulps {{.*}}(%rip), %xmm0, %xmm0 # sched: [11:0.50]
-; HASWELL-NO-FMA-NEXT:    retq # sched: [7:1.00]
-;
-; KNL-LABEL: v4f32_two_step2:
-; KNL:       # %bb.0:
-; KNL-NEXT:    vrcpps %xmm0, %xmm1 # sched: [5:1.00]
-; KNL-NEXT:    vbroadcastss {{.*#+}} xmm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0] sched: [6:0.50]
-; KNL-NEXT:    vmovaps %xmm1, %xmm3 # sched: [1:1.00]
-; KNL-NEXT:    vfnmadd213ps {{.*#+}} xmm3 = -(xmm0 * xmm3) + xmm2 sched: [5:0.50]
-; KNL-NEXT:    vfmadd132ps {{.*#+}} xmm3 = (xmm3 * xmm1) + xmm1 sched: [5:0.50]
-; KNL-NEXT:    vfnmadd213ps {{.*#+}} xmm0 = -(xmm3 * xmm0) + xmm2 sched: [5:0.50]
-; KNL-NEXT:    vfmadd132ps {{.*#+}} xmm0 = (xmm0 * xmm3) + xmm3 sched: [5:0.50]
-; KNL-NEXT:    vmulps {{.*}}(%rip), %xmm0, %xmm0 # sched: [11:0.50]
-; KNL-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: v4f32_two_step2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vrcpps %xmm0, %xmm1 # sched: [4:1.00]
-; SKX-NEXT:    vbroadcastss {{.*#+}} xmm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0] sched: [6:0.50]
-; SKX-NEXT:    vmovaps %xmm1, %xmm3 # sched: [1:0.33]
-; SKX-NEXT:    vfnmadd213ps {{.*#+}} xmm3 = -(xmm0 * xmm3) + xmm2 sched: [4:0.50]
-; SKX-NEXT:    vfmadd132ps {{.*#+}} xmm3 = (xmm3 * xmm1) + xmm1 sched: [4:0.50]
-; SKX-NEXT:    vfnmadd213ps {{.*#+}} xmm0 = -(xmm3 * xmm0) + xmm2 sched: [4:0.50]
-; SKX-NEXT:    vfmadd132ps {{.*#+}} xmm0 = (xmm0 * xmm3) + xmm3 sched: [4:0.50]
-; SKX-NEXT:    vmulps {{.*}}(%rip), %xmm0, %xmm0 # sched: [10:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
+; HASWELL-NO-FMA-NEXT:    vrcpps %xmm0, %xmm1
+; HASWELL-NO-FMA-NEXT:    vmulps %xmm1, %xmm0, %xmm2
+; HASWELL-NO-FMA-NEXT:    vbroadcastss {{.*#+}} xmm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; HASWELL-NO-FMA-NEXT:    vsubps %xmm2, %xmm3, %xmm2
+; HASWELL-NO-FMA-NEXT:    vmulps %xmm2, %xmm1, %xmm2
+; HASWELL-NO-FMA-NEXT:    vaddps %xmm2, %xmm1, %xmm1
+; HASWELL-NO-FMA-NEXT:    vmulps %xmm1, %xmm0, %xmm0
+; HASWELL-NO-FMA-NEXT:    vsubps %xmm0, %xmm3, %xmm0
+; HASWELL-NO-FMA-NEXT:    vmulps %xmm0, %xmm1, %xmm0
+; HASWELL-NO-FMA-NEXT:    vaddps %xmm0, %xmm1, %xmm0
+; HASWELL-NO-FMA-NEXT:    vmulps {{.*}}(%rip), %xmm0, %xmm0
+; HASWELL-NO-FMA-NEXT:    retq
+;
+; AVX512-LABEL: v4f32_two_step2:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vrcpps %xmm0, %xmm1
+; AVX512-NEXT:    vbroadcastss {{.*#+}} xmm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; AVX512-NEXT:    vmovaps %xmm1, %xmm3
+; AVX512-NEXT:    vfnmadd213ps {{.*#+}} xmm3 = -(xmm0 * xmm3) + xmm2
+; AVX512-NEXT:    vfmadd132ps {{.*#+}} xmm3 = (xmm3 * xmm1) + xmm1
+; AVX512-NEXT:    vfnmadd213ps {{.*#+}} xmm0 = -(xmm3 * xmm0) + xmm2
+; AVX512-NEXT:    vfmadd132ps {{.*#+}} xmm0 = (xmm0 * xmm3) + xmm3
+; AVX512-NEXT:    vmulps {{.*}}(%rip), %xmm0, %xmm0
+; AVX512-NEXT:    retq
   %div = fdiv fast <4 x float> <float 1.0, float 2.0, float 3.0, float 4.0>, %x
   ret <4 x float> %div
 }
@@ -826,70 +737,70 @@ define <8 x float> @v8f32_one_step2(<8 x float> %x) #1 {
 ;
 ; BDVER2-LABEL: v8f32_one_step2:
 ; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vrcpps %ymm0, %ymm1 # sched: [5:2.00]
-; BDVER2-NEXT:    vfnmaddps {{.*}}(%rip), %ymm1, %ymm0, %ymm0 # sched: [10:1.00]
-; BDVER2-NEXT:    vfmaddps %ymm1, %ymm0, %ymm1, %ymm0 # sched: [5:0.50]
-; BDVER2-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm0 # sched: [10:2.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
+; BDVER2-NEXT:    vrcpps %ymm0, %ymm1
+; BDVER2-NEXT:    vfnmaddps {{.*}}(%rip), %ymm1, %ymm0, %ymm0
+; BDVER2-NEXT:    vfmaddps %ymm1, %ymm0, %ymm1, %ymm0
+; BDVER2-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm0
+; BDVER2-NEXT:    retq
 ;
 ; BTVER2-LABEL: v8f32_one_step2:
 ; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vmovaps {{.*#+}} ymm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0] sched: [5:1.00]
-; BTVER2-NEXT:    vrcpps %ymm0, %ymm1 # sched: [2:2.00]
-; BTVER2-NEXT:    vmulps %ymm1, %ymm0, %ymm0 # sched: [2:2.00]
-; BTVER2-NEXT:    vsubps %ymm0, %ymm2, %ymm0 # sched: [3:2.00]
-; BTVER2-NEXT:    vmulps %ymm0, %ymm1, %ymm0 # sched: [2:2.00]
-; BTVER2-NEXT:    vaddps %ymm0, %ymm1, %ymm0 # sched: [3:2.00]
-; BTVER2-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm0 # sched: [7:2.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
+; BTVER2-NEXT:    vmovaps {{.*#+}} ymm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; BTVER2-NEXT:    vrcpps %ymm0, %ymm1
+; BTVER2-NEXT:    vmulps %ymm1, %ymm0, %ymm0
+; BTVER2-NEXT:    vsubps %ymm0, %ymm2, %ymm0
+; BTVER2-NEXT:    vmulps %ymm0, %ymm1, %ymm0
+; BTVER2-NEXT:    vaddps %ymm0, %ymm1, %ymm0
+; BTVER2-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm0
+; BTVER2-NEXT:    retq
 ;
 ; SANDY-LABEL: v8f32_one_step2:
 ; SANDY:       # %bb.0:
-; SANDY-NEXT:    vrcpps %ymm0, %ymm1 # sched: [7:2.00]
-; SANDY-NEXT:    vmulps %ymm1, %ymm0, %ymm0 # sched: [5:1.00]
-; SANDY-NEXT:    vmovaps {{.*#+}} ymm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0] sched: [7:0.50]
-; SANDY-NEXT:    vsubps %ymm0, %ymm2, %ymm0 # sched: [3:1.00]
-; SANDY-NEXT:    vmulps %ymm0, %ymm1, %ymm0 # sched: [5:1.00]
-; SANDY-NEXT:    vaddps %ymm0, %ymm1, %ymm0 # sched: [3:1.00]
-; SANDY-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm0 # sched: [12:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
+; SANDY-NEXT:    vrcpps %ymm0, %ymm1
+; SANDY-NEXT:    vmulps %ymm1, %ymm0, %ymm0
+; SANDY-NEXT:    vmovaps {{.*#+}} ymm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; SANDY-NEXT:    vsubps %ymm0, %ymm2, %ymm0
+; SANDY-NEXT:    vmulps %ymm0, %ymm1, %ymm0
+; SANDY-NEXT:    vaddps %ymm0, %ymm1, %ymm0
+; SANDY-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm0
+; SANDY-NEXT:    retq
 ;
 ; HASWELL-LABEL: v8f32_one_step2:
 ; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vrcpps %ymm0, %ymm1 # sched: [11:2.00]
-; HASWELL-NEXT:    vbroadcastss {{.*#+}} ymm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0] sched: [7:0.50]
-; HASWELL-NEXT:    vfnmadd213ps {{.*#+}} ymm0 = -(ymm1 * ymm0) + ymm2 sched: [5:0.50]
-; HASWELL-NEXT:    vfmadd132ps {{.*#+}} ymm0 = (ymm0 * ymm1) + ymm1 sched: [5:0.50]
-; HASWELL-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm0 # sched: [12:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
+; HASWELL-NEXT:    vrcpps %ymm0, %ymm1
+; HASWELL-NEXT:    vbroadcastss {{.*#+}} ymm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; HASWELL-NEXT:    vfnmadd213ps {{.*#+}} ymm0 = -(ymm1 * ymm0) + ymm2
+; HASWELL-NEXT:    vfmadd132ps {{.*#+}} ymm0 = (ymm0 * ymm1) + ymm1
+; HASWELL-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm0
+; HASWELL-NEXT:    retq
 ;
 ; HASWELL-NO-FMA-LABEL: v8f32_one_step2:
 ; HASWELL-NO-FMA:       # %bb.0:
-; HASWELL-NO-FMA-NEXT:    vrcpps %ymm0, %ymm1 # sched: [11:2.00]
-; HASWELL-NO-FMA-NEXT:    vmulps %ymm1, %ymm0, %ymm0 # sched: [5:0.50]
-; HASWELL-NO-FMA-NEXT:    vbroadcastss {{.*#+}} ymm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0] sched: [7:0.50]
-; HASWELL-NO-FMA-NEXT:    vsubps %ymm0, %ymm2, %ymm0 # sched: [3:1.00]
-; HASWELL-NO-FMA-NEXT:    vmulps %ymm0, %ymm1, %ymm0 # sched: [5:0.50]
-; HASWELL-NO-FMA-NEXT:    vaddps %ymm0, %ymm1, %ymm0 # sched: [3:1.00]
-; HASWELL-NO-FMA-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm0 # sched: [12:0.50]
-; HASWELL-NO-FMA-NEXT:    retq # sched: [7:1.00]
+; HASWELL-NO-FMA-NEXT:    vrcpps %ymm0, %ymm1
+; HASWELL-NO-FMA-NEXT:    vmulps %ymm1, %ymm0, %ymm0
+; HASWELL-NO-FMA-NEXT:    vbroadcastss {{.*#+}} ymm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; HASWELL-NO-FMA-NEXT:    vsubps %ymm0, %ymm2, %ymm0
+; HASWELL-NO-FMA-NEXT:    vmulps %ymm0, %ymm1, %ymm0
+; HASWELL-NO-FMA-NEXT:    vaddps %ymm0, %ymm1, %ymm0
+; HASWELL-NO-FMA-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm0
+; HASWELL-NO-FMA-NEXT:    retq
 ;
 ; KNL-LABEL: v8f32_one_step2:
 ; KNL:       # %bb.0:
-; KNL-NEXT:    vrcpps %ymm0, %ymm1 # sched: [11:2.00]
-; KNL-NEXT:    vbroadcastss {{.*#+}} ymm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0] sched: [7:0.50]
-; KNL-NEXT:    vfnmadd213ps {{.*#+}} ymm0 = -(ymm1 * ymm0) + ymm2 sched: [5:0.50]
-; KNL-NEXT:    vfmadd132ps {{.*#+}} ymm0 = (ymm0 * ymm1) + ymm1 sched: [5:0.50]
-; KNL-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm0 # sched: [12:0.50]
-; KNL-NEXT:    retq # sched: [7:1.00]
+; KNL-NEXT:    vrcpps %ymm0, %ymm1
+; KNL-NEXT:    vbroadcastss {{.*#+}} ymm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; KNL-NEXT:    vfnmadd213ps {{.*#+}} ymm0 = -(ymm1 * ymm0) + ymm2
+; KNL-NEXT:    vfmadd132ps {{.*#+}} ymm0 = (ymm0 * ymm1) + ymm1
+; KNL-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm0
+; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: v8f32_one_step2:
 ; SKX:       # %bb.0:
-; SKX-NEXT:    vrcpps %ymm0, %ymm1 # sched: [4:1.00]
-; SKX-NEXT:    vfnmadd213ps {{.*#+}} ymm0 = -(ymm1 * ymm0) + mem sched: [11:0.50]
-; SKX-NEXT:    vfmadd132ps {{.*#+}} ymm0 = (ymm0 * ymm1) + ymm1 sched: [4:0.50]
-; SKX-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm0 # sched: [11:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
+; SKX-NEXT:    vrcpps %ymm0, %ymm1
+; SKX-NEXT:    vfnmadd213ps {{.*#+}} ymm0 = -(ymm1 * ymm0) + mem
+; SKX-NEXT:    vfmadd132ps {{.*#+}} ymm0 = (ymm0 * ymm1) + ymm1
+; SKX-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm0
+; SKX-NEXT:    retq
   %div = fdiv fast <8 x float> <float 1.0, float 2.0, float 3.0, float 4.0, float 5.0, float 6.0, float 7.0, float 8.0>, %x
   ret <8 x float> %div
 }
@@ -940,77 +851,77 @@ define <8 x float> @v8f32_one_step_2_divs(<8 x float> %x) #1 {
 ;
 ; BDVER2-LABEL: v8f32_one_step_2_divs:
 ; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vrcpps %ymm0, %ymm1 # sched: [5:2.00]
-; BDVER2-NEXT:    vfnmaddps {{.*}}(%rip), %ymm1, %ymm0, %ymm0 # sched: [10:1.00]
-; BDVER2-NEXT:    vfmaddps %ymm1, %ymm0, %ymm1, %ymm0 # sched: [5:0.50]
-; BDVER2-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm1 # sched: [10:2.00]
-; BDVER2-NEXT:    vmulps %ymm0, %ymm1, %ymm0 # sched: [5:2.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
+; BDVER2-NEXT:    vrcpps %ymm0, %ymm1
+; BDVER2-NEXT:    vfnmaddps {{.*}}(%rip), %ymm1, %ymm0, %ymm0
+; BDVER2-NEXT:    vfmaddps %ymm1, %ymm0, %ymm1, %ymm0
+; BDVER2-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm1
+; BDVER2-NEXT:    vmulps %ymm0, %ymm1, %ymm0
+; BDVER2-NEXT:    retq
 ;
 ; BTVER2-LABEL: v8f32_one_step_2_divs:
 ; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vmovaps {{.*#+}} ymm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0] sched: [5:1.00]
-; BTVER2-NEXT:    vrcpps %ymm0, %ymm1 # sched: [2:2.00]
-; BTVER2-NEXT:    vmulps %ymm1, %ymm0, %ymm0 # sched: [2:2.00]
-; BTVER2-NEXT:    vsubps %ymm0, %ymm2, %ymm0 # sched: [3:2.00]
-; BTVER2-NEXT:    vmulps %ymm0, %ymm1, %ymm0 # sched: [2:2.00]
-; BTVER2-NEXT:    vaddps %ymm0, %ymm1, %ymm0 # sched: [3:2.00]
-; BTVER2-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm1 # sched: [7:2.00]
-; BTVER2-NEXT:    vmulps %ymm0, %ymm1, %ymm0 # sched: [2:2.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
+; BTVER2-NEXT:    vmovaps {{.*#+}} ymm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; BTVER2-NEXT:    vrcpps %ymm0, %ymm1
+; BTVER2-NEXT:    vmulps %ymm1, %ymm0, %ymm0
+; BTVER2-NEXT:    vsubps %ymm0, %ymm2, %ymm0
+; BTVER2-NEXT:    vmulps %ymm0, %ymm1, %ymm0
+; BTVER2-NEXT:    vaddps %ymm0, %ymm1, %ymm0
+; BTVER2-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm1
+; BTVER2-NEXT:    vmulps %ymm0, %ymm1, %ymm0
+; BTVER2-NEXT:    retq
 ;
 ; SANDY-LABEL: v8f32_one_step_2_divs:
 ; SANDY:       # %bb.0:
-; SANDY-NEXT:    vrcpps %ymm0, %ymm1 # sched: [7:2.00]
-; SANDY-NEXT:    vmulps %ymm1, %ymm0, %ymm0 # sched: [5:1.00]
-; SANDY-NEXT:    vmovaps {{.*#+}} ymm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0] sched: [7:0.50]
-; SANDY-NEXT:    vsubps %ymm0, %ymm2, %ymm0 # sched: [3:1.00]
-; SANDY-NEXT:    vmulps %ymm0, %ymm1, %ymm0 # sched: [5:1.00]
-; SANDY-NEXT:    vaddps %ymm0, %ymm1, %ymm0 # sched: [3:1.00]
-; SANDY-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm1 # sched: [12:1.00]
-; SANDY-NEXT:    vmulps %ymm0, %ymm1, %ymm0 # sched: [5:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
+; SANDY-NEXT:    vrcpps %ymm0, %ymm1
+; SANDY-NEXT:    vmulps %ymm1, %ymm0, %ymm0
+; SANDY-NEXT:    vmovaps {{.*#+}} ymm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; SANDY-NEXT:    vsubps %ymm0, %ymm2, %ymm0
+; SANDY-NEXT:    vmulps %ymm0, %ymm1, %ymm0
+; SANDY-NEXT:    vaddps %ymm0, %ymm1, %ymm0
+; SANDY-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm1
+; SANDY-NEXT:    vmulps %ymm0, %ymm1, %ymm0
+; SANDY-NEXT:    retq
 ;
 ; HASWELL-LABEL: v8f32_one_step_2_divs:
 ; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vrcpps %ymm0, %ymm1 # sched: [11:2.00]
-; HASWELL-NEXT:    vbroadcastss {{.*#+}} ymm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0] sched: [7:0.50]
-; HASWELL-NEXT:    vfnmadd213ps {{.*#+}} ymm0 = -(ymm1 * ymm0) + ymm2 sched: [5:0.50]
-; HASWELL-NEXT:    vfmadd132ps {{.*#+}} ymm0 = (ymm0 * ymm1) + ymm1 sched: [5:0.50]
-; HASWELL-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm1 # sched: [12:0.50]
-; HASWELL-NEXT:    vmulps %ymm0, %ymm1, %ymm0 # sched: [5:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
+; HASWELL-NEXT:    vrcpps %ymm0, %ymm1
+; HASWELL-NEXT:    vbroadcastss {{.*#+}} ymm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; HASWELL-NEXT:    vfnmadd213ps {{.*#+}} ymm0 = -(ymm1 * ymm0) + ymm2
+; HASWELL-NEXT:    vfmadd132ps {{.*#+}} ymm0 = (ymm0 * ymm1) + ymm1
+; HASWELL-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm1
+; HASWELL-NEXT:    vmulps %ymm0, %ymm1, %ymm0
+; HASWELL-NEXT:    retq
 ;
 ; HASWELL-NO-FMA-LABEL: v8f32_one_step_2_divs:
 ; HASWELL-NO-FMA:       # %bb.0:
-; HASWELL-NO-FMA-NEXT:    vrcpps %ymm0, %ymm1 # sched: [11:2.00]
-; HASWELL-NO-FMA-NEXT:    vmulps %ymm1, %ymm0, %ymm0 # sched: [5:0.50]
-; HASWELL-NO-FMA-NEXT:    vbroadcastss {{.*#+}} ymm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0] sched: [7:0.50]
-; HASWELL-NO-FMA-NEXT:    vsubps %ymm0, %ymm2, %ymm0 # sched: [3:1.00]
-; HASWELL-NO-FMA-NEXT:    vmulps %ymm0, %ymm1, %ymm0 # sched: [5:0.50]
-; HASWELL-NO-FMA-NEXT:    vaddps %ymm0, %ymm1, %ymm0 # sched: [3:1.00]
-; HASWELL-NO-FMA-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm1 # sched: [12:0.50]
-; HASWELL-NO-FMA-NEXT:    vmulps %ymm0, %ymm1, %ymm0 # sched: [5:0.50]
-; HASWELL-NO-FMA-NEXT:    retq # sched: [7:1.00]
+; HASWELL-NO-FMA-NEXT:    vrcpps %ymm0, %ymm1
+; HASWELL-NO-FMA-NEXT:    vmulps %ymm1, %ymm0, %ymm0
+; HASWELL-NO-FMA-NEXT:    vbroadcastss {{.*#+}} ymm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; HASWELL-NO-FMA-NEXT:    vsubps %ymm0, %ymm2, %ymm0
+; HASWELL-NO-FMA-NEXT:    vmulps %ymm0, %ymm1, %ymm0
+; HASWELL-NO-FMA-NEXT:    vaddps %ymm0, %ymm1, %ymm0
+; HASWELL-NO-FMA-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm1
+; HASWELL-NO-FMA-NEXT:    vmulps %ymm0, %ymm1, %ymm0
+; HASWELL-NO-FMA-NEXT:    retq
 ;
 ; KNL-LABEL: v8f32_one_step_2_divs:
 ; KNL:       # %bb.0:
-; KNL-NEXT:    vrcpps %ymm0, %ymm1 # sched: [11:2.00]
-; KNL-NEXT:    vbroadcastss {{.*#+}} ymm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0] sched: [7:0.50]
-; KNL-NEXT:    vfnmadd213ps {{.*#+}} ymm0 = -(ymm1 * ymm0) + ymm2 sched: [5:0.50]
-; KNL-NEXT:    vfmadd132ps {{.*#+}} ymm0 = (ymm0 * ymm1) + ymm1 sched: [5:0.50]
-; KNL-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm1 # sched: [12:0.50]
-; KNL-NEXT:    vmulps %ymm0, %ymm1, %ymm0 # sched: [5:0.50]
-; KNL-NEXT:    retq # sched: [7:1.00]
+; KNL-NEXT:    vrcpps %ymm0, %ymm1
+; KNL-NEXT:    vbroadcastss {{.*#+}} ymm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; KNL-NEXT:    vfnmadd213ps {{.*#+}} ymm0 = -(ymm1 * ymm0) + ymm2
+; KNL-NEXT:    vfmadd132ps {{.*#+}} ymm0 = (ymm0 * ymm1) + ymm1
+; KNL-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm1
+; KNL-NEXT:    vmulps %ymm0, %ymm1, %ymm0
+; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: v8f32_one_step_2_divs:
 ; SKX:       # %bb.0:
-; SKX-NEXT:    vrcpps %ymm0, %ymm1 # sched: [4:1.00]
-; SKX-NEXT:    vfnmadd213ps {{.*#+}} ymm0 = -(ymm1 * ymm0) + mem sched: [11:0.50]
-; SKX-NEXT:    vfmadd132ps {{.*#+}} ymm0 = (ymm0 * ymm1) + ymm1 sched: [4:0.50]
-; SKX-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm1 # sched: [11:0.50]
-; SKX-NEXT:    vmulps %ymm0, %ymm1, %ymm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
+; SKX-NEXT:    vrcpps %ymm0, %ymm1
+; SKX-NEXT:    vfnmadd213ps {{.*#+}} ymm0 = -(ymm1 * ymm0) + mem
+; SKX-NEXT:    vfmadd132ps {{.*#+}} ymm0 = (ymm0 * ymm1) + ymm1
+; SKX-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm1
+; SKX-NEXT:    vmulps %ymm0, %ymm1, %ymm0
+; SKX-NEXT:    retq
   %div = fdiv fast <8 x float> <float 1.0, float 2.0, float 3.0, float 4.0, float 5.0, float 6.0, float 7.0, float 8.0>, %x
   %div2 = fdiv fast <8 x float> %div, %x
   ret <8 x float> %div2
@@ -1078,95 +989,83 @@ define <8 x float> @v8f32_two_step2(<8 x float> %x) #2 {
 ;
 ; BDVER2-LABEL: v8f32_two_step2:
 ; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vrcpps %ymm0, %ymm1 # sched: [5:2.00]
-; BDVER2-NEXT:    vmovaps {{.*#+}} ymm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0] sched: [5:0.50]
-; BDVER2-NEXT:    vfnmaddps %ymm2, %ymm1, %ymm0, %ymm3 # sched: [5:0.50]
-; BDVER2-NEXT:    vfmaddps %ymm1, %ymm3, %ymm1, %ymm1 # sched: [5:0.50]
-; BDVER2-NEXT:    vfnmaddps %ymm2, %ymm1, %ymm0, %ymm0 # sched: [5:0.50]
-; BDVER2-NEXT:    vfmaddps %ymm1, %ymm0, %ymm1, %ymm0 # sched: [5:0.50]
-; BDVER2-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm0 # sched: [10:2.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
+; BDVER2-NEXT:    vrcpps %ymm0, %ymm1
+; BDVER2-NEXT:    vmovaps {{.*#+}} ymm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; BDVER2-NEXT:    vfnmaddps %ymm2, %ymm1, %ymm0, %ymm3
+; BDVER2-NEXT:    vfmaddps %ymm1, %ymm3, %ymm1, %ymm1
+; BDVER2-NEXT:    vfnmaddps %ymm2, %ymm1, %ymm0, %ymm0
+; BDVER2-NEXT:    vfmaddps %ymm1, %ymm0, %ymm1, %ymm0
+; BDVER2-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm0
+; BDVER2-NEXT:    retq
 ;
 ; BTVER2-LABEL: v8f32_two_step2:
 ; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vmovaps {{.*#+}} ymm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0] sched: [5:1.00]
-; BTVER2-NEXT:    vrcpps %ymm0, %ymm1 # sched: [2:2.00]
-; BTVER2-NEXT:    vmulps %ymm1, %ymm0, %ymm2 # sched: [2:2.00]
-; BTVER2-NEXT:    vsubps %ymm2, %ymm3, %ymm2 # sched: [3:2.00]
-; BTVER2-NEXT:    vmulps %ymm2, %ymm1, %ymm2 # sched: [2:2.00]
-; BTVER2-NEXT:    vaddps %ymm2, %ymm1, %ymm1 # sched: [3:2.00]
-; BTVER2-NEXT:    vmulps %ymm1, %ymm0, %ymm0 # sched: [2:2.00]
-; BTVER2-NEXT:    vsubps %ymm0, %ymm3, %ymm0 # sched: [3:2.00]
-; BTVER2-NEXT:    vmulps %ymm0, %ymm1, %ymm0 # sched: [2:2.00]
-; BTVER2-NEXT:    vaddps %ymm0, %ymm1, %ymm0 # sched: [3:2.00]
-; BTVER2-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm0 # sched: [7:2.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
+; BTVER2-NEXT:    vmovaps {{.*#+}} ymm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; BTVER2-NEXT:    vrcpps %ymm0, %ymm1
+; BTVER2-NEXT:    vmulps %ymm1, %ymm0, %ymm2
+; BTVER2-NEXT:    vsubps %ymm2, %ymm3, %ymm2
+; BTVER2-NEXT:    vmulps %ymm2, %ymm1, %ymm2
+; BTVER2-NEXT:    vaddps %ymm2, %ymm1, %ymm1
+; BTVER2-NEXT:    vmulps %ymm1, %ymm0, %ymm0
+; BTVER2-NEXT:    vsubps %ymm0, %ymm3, %ymm0
+; BTVER2-NEXT:    vmulps %ymm0, %ymm1, %ymm0
+; BTVER2-NEXT:    vaddps %ymm0, %ymm1, %ymm0
+; BTVER2-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm0
+; BTVER2-NEXT:    retq
 ;
 ; SANDY-LABEL: v8f32_two_step2:
 ; SANDY:       # %bb.0:
-; SANDY-NEXT:    vrcpps %ymm0, %ymm1 # sched: [7:2.00]
-; SANDY-NEXT:    vmulps %ymm1, %ymm0, %ymm2 # sched: [5:1.00]
-; SANDY-NEXT:    vmovaps {{.*#+}} ymm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0] sched: [7:0.50]
-; SANDY-NEXT:    vsubps %ymm2, %ymm3, %ymm2 # sched: [3:1.00]
-; SANDY-NEXT:    vmulps %ymm2, %ymm1, %ymm2 # sched: [5:1.00]
-; SANDY-NEXT:    vaddps %ymm2, %ymm1, %ymm1 # sched: [3:1.00]
-; SANDY-NEXT:    vmulps %ymm1, %ymm0, %ymm0 # sched: [5:1.00]
-; SANDY-NEXT:    vsubps %ymm0, %ymm3, %ymm0 # sched: [3:1.00]
-; SANDY-NEXT:    vmulps %ymm0, %ymm1, %ymm0 # sched: [5:1.00]
-; SANDY-NEXT:    vaddps %ymm0, %ymm1, %ymm0 # sched: [3:1.00]
-; SANDY-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm0 # sched: [12:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
+; SANDY-NEXT:    vrcpps %ymm0, %ymm1
+; SANDY-NEXT:    vmulps %ymm1, %ymm0, %ymm2
+; SANDY-NEXT:    vmovaps {{.*#+}} ymm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; SANDY-NEXT:    vsubps %ymm2, %ymm3, %ymm2
+; SANDY-NEXT:    vmulps %ymm2, %ymm1, %ymm2
+; SANDY-NEXT:    vaddps %ymm2, %ymm1, %ymm1
+; SANDY-NEXT:    vmulps %ymm1, %ymm0, %ymm0
+; SANDY-NEXT:    vsubps %ymm0, %ymm3, %ymm0
+; SANDY-NEXT:    vmulps %ymm0, %ymm1, %ymm0
+; SANDY-NEXT:    vaddps %ymm0, %ymm1, %ymm0
+; SANDY-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm0
+; SANDY-NEXT:    retq
 ;
 ; HASWELL-LABEL: v8f32_two_step2:
 ; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vrcpps %ymm0, %ymm1 # sched: [11:2.00]
-; HASWELL-NEXT:    vbroadcastss {{.*#+}} ymm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0] sched: [7:0.50]
-; HASWELL-NEXT:    vmovaps %ymm1, %ymm3 # sched: [1:1.00]
-; HASWELL-NEXT:    vfnmadd213ps {{.*#+}} ymm3 = -(ymm0 * ymm3) + ymm2 sched: [5:0.50]
-; HASWELL-NEXT:    vfmadd132ps {{.*#+}} ymm3 = (ymm3 * ymm1) + ymm1 sched: [5:0.50]
-; HASWELL-NEXT:    vfnmadd213ps {{.*#+}} ymm0 = -(ymm3 * ymm0) + ymm2 sched: [5:0.50]
-; HASWELL-NEXT:    vfmadd132ps {{.*#+}} ymm0 = (ymm0 * ymm3) + ymm3 sched: [5:0.50]
-; HASWELL-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm0 # sched: [12:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
+; HASWELL-NEXT:    vrcpps %ymm0, %ymm1
+; HASWELL-NEXT:    vbroadcastss {{.*#+}} ymm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; HASWELL-NEXT:    vmovaps %ymm1, %ymm3
+; HASWELL-NEXT:    vfnmadd213ps {{.*#+}} ymm3 = -(ymm0 * ymm3) + ymm2
+; HASWELL-NEXT:    vfmadd132ps {{.*#+}} ymm3 = (ymm3 * ymm1) + ymm1
+; HASWELL-NEXT:    vfnmadd213ps {{.*#+}} ymm0 = -(ymm3 * ymm0) + ymm2
+; HASWELL-NEXT:    vfmadd132ps {{.*#+}} ymm0 = (ymm0 * ymm3) + ymm3
+; HASWELL-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm0
+; HASWELL-NEXT:    retq
 ;
 ; HASWELL-NO-FMA-LABEL: v8f32_two_step2:
 ; HASWELL-NO-FMA:       # %bb.0:
-; HASWELL-NO-FMA-NEXT:    vrcpps %ymm0, %ymm1 # sched: [11:2.00]
-; HASWELL-NO-FMA-NEXT:    vmulps %ymm1, %ymm0, %ymm2 # sched: [5:0.50]
-; HASWELL-NO-FMA-NEXT:    vbroadcastss {{.*#+}} ymm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0] sched: [7:0.50]
-; HASWELL-NO-FMA-NEXT:    vsubps %ymm2, %ymm3, %ymm2 # sched: [3:1.00]
-; HASWELL-NO-FMA-NEXT:    vmulps %ymm2, %ymm1, %ymm2 # sched: [5:0.50]
-; HASWELL-NO-FMA-NEXT:    vaddps %ymm2, %ymm1, %ymm1 # sched: [3:1.00]
-; HASWELL-NO-FMA-NEXT:    vmulps %ymm1, %ymm0, %ymm0 # sched: [5:0.50]
-; HASWELL-NO-FMA-NEXT:    vsubps %ymm0, %ymm3, %ymm0 # sched: [3:1.00]
-; HASWELL-NO-FMA-NEXT:    vmulps %ymm0, %ymm1, %ymm0 # sched: [5:0.50]
-; HASWELL-NO-FMA-NEXT:    vaddps %ymm0, %ymm1, %ymm0 # sched: [3:1.00]
-; HASWELL-NO-FMA-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm0 # sched: [12:0.50]
-; HASWELL-NO-FMA-NEXT:    retq # sched: [7:1.00]
-;
-; KNL-LABEL: v8f32_two_step2:
-; KNL:       # %bb.0:
-; KNL-NEXT:    vrcpps %ymm0, %ymm1 # sched: [11:2.00]
-; KNL-NEXT:    vbroadcastss {{.*#+}} ymm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0] sched: [7:0.50]
-; KNL-NEXT:    vmovaps %ymm1, %ymm3 # sched: [1:1.00]
-; KNL-NEXT:    vfnmadd213ps {{.*#+}} ymm3 = -(ymm0 * ymm3) + ymm2 sched: [5:0.50]
-; KNL-NEXT:    vfmadd132ps {{.*#+}} ymm3 = (ymm3 * ymm1) + ymm1 sched: [5:0.50]
-; KNL-NEXT:    vfnmadd213ps {{.*#+}} ymm0 = -(ymm3 * ymm0) + ymm2 sched: [5:0.50]
-; KNL-NEXT:    vfmadd132ps {{.*#+}} ymm0 = (ymm0 * ymm3) + ymm3 sched: [5:0.50]
-; KNL-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm0 # sched: [12:0.50]
-; KNL-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: v8f32_two_step2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vrcpps %ymm0, %ymm1 # sched: [4:1.00]
-; SKX-NEXT:    vbroadcastss {{.*#+}} ymm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0] sched: [7:0.50]
-; SKX-NEXT:    vmovaps %ymm1, %ymm3 # sched: [1:0.33]
-; SKX-NEXT:    vfnmadd213ps {{.*#+}} ymm3 = -(ymm0 * ymm3) + ymm2 sched: [4:0.50]
-; SKX-NEXT:    vfmadd132ps {{.*#+}} ymm3 = (ymm3 * ymm1) + ymm1 sched: [4:0.50]
-; SKX-NEXT:    vfnmadd213ps {{.*#+}} ymm0 = -(ymm3 * ymm0) + ymm2 sched: [4:0.50]
-; SKX-NEXT:    vfmadd132ps {{.*#+}} ymm0 = (ymm0 * ymm3) + ymm3 sched: [4:0.50]
-; SKX-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm0 # sched: [11:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
+; HASWELL-NO-FMA-NEXT:    vrcpps %ymm0, %ymm1
+; HASWELL-NO-FMA-NEXT:    vmulps %ymm1, %ymm0, %ymm2
+; HASWELL-NO-FMA-NEXT:    vbroadcastss {{.*#+}} ymm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; HASWELL-NO-FMA-NEXT:    vsubps %ymm2, %ymm3, %ymm2
+; HASWELL-NO-FMA-NEXT:    vmulps %ymm2, %ymm1, %ymm2
+; HASWELL-NO-FMA-NEXT:    vaddps %ymm2, %ymm1, %ymm1
+; HASWELL-NO-FMA-NEXT:    vmulps %ymm1, %ymm0, %ymm0
+; HASWELL-NO-FMA-NEXT:    vsubps %ymm0, %ymm3, %ymm0
+; HASWELL-NO-FMA-NEXT:    vmulps %ymm0, %ymm1, %ymm0
+; HASWELL-NO-FMA-NEXT:    vaddps %ymm0, %ymm1, %ymm0
+; HASWELL-NO-FMA-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm0
+; HASWELL-NO-FMA-NEXT:    retq
+;
+; AVX512-LABEL: v8f32_two_step2:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vrcpps %ymm0, %ymm1
+; AVX512-NEXT:    vbroadcastss {{.*#+}} ymm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; AVX512-NEXT:    vmovaps %ymm1, %ymm3
+; AVX512-NEXT:    vfnmadd213ps {{.*#+}} ymm3 = -(ymm0 * ymm3) + ymm2
+; AVX512-NEXT:    vfmadd132ps {{.*#+}} ymm3 = (ymm3 * ymm1) + ymm1
+; AVX512-NEXT:    vfnmadd213ps {{.*#+}} ymm0 = -(ymm3 * ymm0) + ymm2
+; AVX512-NEXT:    vfmadd132ps {{.*#+}} ymm0 = (ymm0 * ymm3) + ymm3
+; AVX512-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm0
+; AVX512-NEXT:    retq
   %div = fdiv fast <8 x float> <float 1.0, float 2.0, float 3.0, float 4.0, float 5.0, float 6.0, float 7.0, float 8.0>, %x
   ret <8 x float> %div
 }
@@ -1178,50 +1077,10 @@ define <8 x float> @v8f32_no_step(<8 x float> %x) #3 {
 ; SSE-NEXT:    rcpps %xmm1, %xmm1
 ; SSE-NEXT:    retq
 ;
-; AVX-RECIP-LABEL: v8f32_no_step:
-; AVX-RECIP:       # %bb.0:
-; AVX-RECIP-NEXT:    vrcpps %ymm0, %ymm0
-; AVX-RECIP-NEXT:    retq
-;
-; FMA-RECIP-LABEL: v8f32_no_step:
-; FMA-RECIP:       # %bb.0:
-; FMA-RECIP-NEXT:    vrcpps %ymm0, %ymm0
-; FMA-RECIP-NEXT:    retq
-;
-; BDVER2-LABEL: v8f32_no_step:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vrcpps %ymm0, %ymm0 # sched: [5:2.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: v8f32_no_step:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vrcpps %ymm0, %ymm0 # sched: [2:2.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: v8f32_no_step:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vrcpps %ymm0, %ymm0 # sched: [7:2.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: v8f32_no_step:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vrcpps %ymm0, %ymm0 # sched: [11:2.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-NO-FMA-LABEL: v8f32_no_step:
-; HASWELL-NO-FMA:       # %bb.0:
-; HASWELL-NO-FMA-NEXT:    vrcpps %ymm0, %ymm0 # sched: [11:2.00]
-; HASWELL-NO-FMA-NEXT:    retq # sched: [7:1.00]
-;
-; KNL-LABEL: v8f32_no_step:
-; KNL:       # %bb.0:
-; KNL-NEXT:    vrcpps %ymm0, %ymm0 # sched: [11:2.00]
-; KNL-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: v8f32_no_step:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vrcpps %ymm0, %ymm0 # sched: [4:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
+; AVX-LABEL: v8f32_no_step:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vrcpps %ymm0, %ymm0
+; AVX-NEXT:    retq
   %div = fdiv fast <8 x float> <float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0>, %x
   ret <8 x float> %div
 }
@@ -1235,59 +1094,11 @@ define <8 x float> @v8f32_no_step2(<8 x float> %x) #3 {
 ; SSE-NEXT:    mulps {{.*}}(%rip), %xmm1
 ; SSE-NEXT:    retq
 ;
-; AVX-RECIP-LABEL: v8f32_no_step2:
-; AVX-RECIP:       # %bb.0:
-; AVX-RECIP-NEXT:    vrcpps %ymm0, %ymm0
-; AVX-RECIP-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm0
-; AVX-RECIP-NEXT:    retq
-;
-; FMA-RECIP-LABEL: v8f32_no_step2:
-; FMA-RECIP:       # %bb.0:
-; FMA-RECIP-NEXT:    vrcpps %ymm0, %ymm0
-; FMA-RECIP-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm0
-; FMA-RECIP-NEXT:    retq
-;
-; BDVER2-LABEL: v8f32_no_step2:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vrcpps %ymm0, %ymm0 # sched: [5:2.00]
-; BDVER2-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm0 # sched: [10:2.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: v8f32_no_step2:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vrcpps %ymm0, %ymm0 # sched: [2:2.00]
-; BTVER2-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm0 # sched: [7:2.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: v8f32_no_step2:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vrcpps %ymm0, %ymm0 # sched: [7:2.00]
-; SANDY-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm0 # sched: [12:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: v8f32_no_step2:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vrcpps %ymm0, %ymm0 # sched: [11:2.00]
-; HASWELL-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm0 # sched: [12:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-NO-FMA-LABEL: v8f32_no_step2:
-; HASWELL-NO-FMA:       # %bb.0:
-; HASWELL-NO-FMA-NEXT:    vrcpps %ymm0, %ymm0 # sched: [11:2.00]
-; HASWELL-NO-FMA-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm0 # sched: [12:0.50]
-; HASWELL-NO-FMA-NEXT:    retq # sched: [7:1.00]
-;
-; KNL-LABEL: v8f32_no_step2:
-; KNL:       # %bb.0:
-; KNL-NEXT:    vrcpps %ymm0, %ymm0 # sched: [11:2.00]
-; KNL-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm0 # sched: [12:0.50]
-; KNL-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: v8f32_no_step2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vrcpps %ymm0, %ymm0 # sched: [4:1.00]
-; SKX-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm0 # sched: [11:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
+; AVX-LABEL: v8f32_no_step2:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vrcpps %ymm0, %ymm0
+; AVX-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm0
+; AVX-NEXT:    retq
   %div = fdiv fast <8 x float> <float 1.0, float 2.0, float 3.0, float 4.0, float 5.0, float 6.0, float 7.0, float 8.0>, %x
   ret <8 x float> %div
 }
@@ -1361,96 +1172,88 @@ define <16 x float> @v16f32_one_step2(<16 x float> %x) #1 {
 ;
 ; BDVER2-LABEL: v16f32_one_step2:
 ; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vrcpps %ymm1, %ymm2 # sched: [5:2.00]
-; BDVER2-NEXT:    vmovaps {{.*#+}} ymm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0] sched: [5:0.50]
-; BDVER2-NEXT:    vrcpps %ymm0, %ymm4 # sched: [5:2.00]
-; BDVER2-NEXT:    vfnmaddps %ymm3, %ymm2, %ymm1, %ymm1 # sched: [5:0.50]
-; BDVER2-NEXT:    vfnmaddps %ymm3, %ymm4, %ymm0, %ymm0 # sched: [5:0.50]
-; BDVER2-NEXT:    vfmaddps %ymm2, %ymm1, %ymm2, %ymm1 # sched: [5:0.50]
-; BDVER2-NEXT:    vfmaddps %ymm4, %ymm0, %ymm4, %ymm0 # sched: [5:0.50]
-; BDVER2-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm0 # sched: [10:2.00]
-; BDVER2-NEXT:    vmulps {{.*}}(%rip), %ymm1, %ymm1 # sched: [10:2.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
+; BDVER2-NEXT:    vrcpps %ymm1, %ymm2
+; BDVER2-NEXT:    vmovaps {{.*#+}} ymm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; BDVER2-NEXT:    vrcpps %ymm0, %ymm4
+; BDVER2-NEXT:    vfnmaddps %ymm3, %ymm2, %ymm1, %ymm1
+; BDVER2-NEXT:    vfnmaddps %ymm3, %ymm4, %ymm0, %ymm0
+; BDVER2-NEXT:    vfmaddps %ymm2, %ymm1, %ymm2, %ymm1
+; BDVER2-NEXT:    vfmaddps %ymm4, %ymm0, %ymm4, %ymm0
+; BDVER2-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm0
+; BDVER2-NEXT:    vmulps {{.*}}(%rip), %ymm1, %ymm1
+; BDVER2-NEXT:    retq
 ;
 ; BTVER2-LABEL: v16f32_one_step2:
 ; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vmovaps {{.*#+}} ymm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0] sched: [5:1.00]
-; BTVER2-NEXT:    vrcpps %ymm1, %ymm2 # sched: [2:2.00]
-; BTVER2-NEXT:    vrcpps %ymm0, %ymm4 # sched: [2:2.00]
-; BTVER2-NEXT:    vmulps %ymm2, %ymm1, %ymm1 # sched: [2:2.00]
-; BTVER2-NEXT:    vmulps %ymm4, %ymm0, %ymm0 # sched: [2:2.00]
-; BTVER2-NEXT:    vsubps %ymm1, %ymm3, %ymm1 # sched: [3:2.00]
-; BTVER2-NEXT:    vsubps %ymm0, %ymm3, %ymm0 # sched: [3:2.00]
-; BTVER2-NEXT:    vmulps %ymm1, %ymm2, %ymm1 # sched: [2:2.00]
-; BTVER2-NEXT:    vmulps %ymm0, %ymm4, %ymm0 # sched: [2:2.00]
-; BTVER2-NEXT:    vaddps %ymm1, %ymm2, %ymm1 # sched: [3:2.00]
-; BTVER2-NEXT:    vaddps %ymm0, %ymm4, %ymm0 # sched: [3:2.00]
-; BTVER2-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm0 # sched: [7:2.00]
-; BTVER2-NEXT:    vmulps {{.*}}(%rip), %ymm1, %ymm1 # sched: [7:2.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
+; BTVER2-NEXT:    vmovaps {{.*#+}} ymm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; BTVER2-NEXT:    vrcpps %ymm1, %ymm2
+; BTVER2-NEXT:    vrcpps %ymm0, %ymm4
+; BTVER2-NEXT:    vmulps %ymm2, %ymm1, %ymm1
+; BTVER2-NEXT:    vmulps %ymm4, %ymm0, %ymm0
+; BTVER2-NEXT:    vsubps %ymm1, %ymm3, %ymm1
+; BTVER2-NEXT:    vsubps %ymm0, %ymm3, %ymm0
+; BTVER2-NEXT:    vmulps %ymm1, %ymm2, %ymm1
+; BTVER2-NEXT:    vmulps %ymm0, %ymm4, %ymm0
+; BTVER2-NEXT:    vaddps %ymm1, %ymm2, %ymm1
+; BTVER2-NEXT:    vaddps %ymm0, %ymm4, %ymm0
+; BTVER2-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm0
+; BTVER2-NEXT:    vmulps {{.*}}(%rip), %ymm1, %ymm1
+; BTVER2-NEXT:    retq
 ;
 ; SANDY-LABEL: v16f32_one_step2:
 ; SANDY:       # %bb.0:
-; SANDY-NEXT:    vrcpps %ymm1, %ymm2 # sched: [7:2.00]
-; SANDY-NEXT:    vmulps %ymm2, %ymm1, %ymm1 # sched: [5:1.00]
-; SANDY-NEXT:    vmovaps {{.*#+}} ymm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0] sched: [7:0.50]
-; SANDY-NEXT:    vsubps %ymm1, %ymm3, %ymm1 # sched: [3:1.00]
-; SANDY-NEXT:    vmulps %ymm1, %ymm2, %ymm1 # sched: [5:1.00]
-; SANDY-NEXT:    vaddps %ymm1, %ymm2, %ymm1 # sched: [3:1.00]
-; SANDY-NEXT:    vrcpps %ymm0, %ymm2 # sched: [7:2.00]
-; SANDY-NEXT:    vmulps %ymm2, %ymm0, %ymm0 # sched: [5:1.00]
-; SANDY-NEXT:    vsubps %ymm0, %ymm3, %ymm0 # sched: [3:1.00]
-; SANDY-NEXT:    vmulps %ymm0, %ymm2, %ymm0 # sched: [5:1.00]
-; SANDY-NEXT:    vaddps %ymm0, %ymm2, %ymm0 # sched: [3:1.00]
-; SANDY-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm0 # sched: [12:1.00]
-; SANDY-NEXT:    vmulps {{.*}}(%rip), %ymm1, %ymm1 # sched: [12:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
+; SANDY-NEXT:    vrcpps %ymm1, %ymm2
+; SANDY-NEXT:    vmulps %ymm2, %ymm1, %ymm1
+; SANDY-NEXT:    vmovaps {{.*#+}} ymm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; SANDY-NEXT:    vsubps %ymm1, %ymm3, %ymm1
+; SANDY-NEXT:    vmulps %ymm1, %ymm2, %ymm1
+; SANDY-NEXT:    vaddps %ymm1, %ymm2, %ymm1
+; SANDY-NEXT:    vrcpps %ymm0, %ymm2
+; SANDY-NEXT:    vmulps %ymm2, %ymm0, %ymm0
+; SANDY-NEXT:    vsubps %ymm0, %ymm3, %ymm0
+; SANDY-NEXT:    vmulps %ymm0, %ymm2, %ymm0
+; SANDY-NEXT:    vaddps %ymm0, %ymm2, %ymm0
+; SANDY-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm0
+; SANDY-NEXT:    vmulps {{.*}}(%rip), %ymm1, %ymm1
+; SANDY-NEXT:    retq
 ;
 ; HASWELL-LABEL: v16f32_one_step2:
 ; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vrcpps %ymm1, %ymm2 # sched: [11:2.00]
-; HASWELL-NEXT:    vbroadcastss {{.*#+}} ymm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0] sched: [7:0.50]
-; HASWELL-NEXT:    vrcpps %ymm0, %ymm4 # sched: [11:2.00]
-; HASWELL-NEXT:    vfnmadd213ps {{.*#+}} ymm1 = -(ymm2 * ymm1) + ymm3 sched: [5:0.50]
-; HASWELL-NEXT:    vfmadd132ps {{.*#+}} ymm1 = (ymm1 * ymm2) + ymm2 sched: [5:0.50]
-; HASWELL-NEXT:    vfnmadd213ps {{.*#+}} ymm0 = -(ymm4 * ymm0) + ymm3 sched: [5:0.50]
-; HASWELL-NEXT:    vfmadd132ps {{.*#+}} ymm0 = (ymm0 * ymm4) + ymm4 sched: [5:0.50]
-; HASWELL-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm0 # sched: [12:0.50]
-; HASWELL-NEXT:    vmulps {{.*}}(%rip), %ymm1, %ymm1 # sched: [12:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
+; HASWELL-NEXT:    vrcpps %ymm1, %ymm2
+; HASWELL-NEXT:    vbroadcastss {{.*#+}} ymm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; HASWELL-NEXT:    vrcpps %ymm0, %ymm4
+; HASWELL-NEXT:    vfnmadd213ps {{.*#+}} ymm1 = -(ymm2 * ymm1) + ymm3
+; HASWELL-NEXT:    vfmadd132ps {{.*#+}} ymm1 = (ymm1 * ymm2) + ymm2
+; HASWELL-NEXT:    vfnmadd213ps {{.*#+}} ymm0 = -(ymm4 * ymm0) + ymm3
+; HASWELL-NEXT:    vfmadd132ps {{.*#+}} ymm0 = (ymm0 * ymm4) + ymm4
+; HASWELL-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm0
+; HASWELL-NEXT:    vmulps {{.*}}(%rip), %ymm1, %ymm1
+; HASWELL-NEXT:    retq
 ;
 ; HASWELL-NO-FMA-LABEL: v16f32_one_step2:
 ; HASWELL-NO-FMA:       # %bb.0:
-; HASWELL-NO-FMA-NEXT:    vrcpps %ymm1, %ymm2 # sched: [11:2.00]
-; HASWELL-NO-FMA-NEXT:    vmulps %ymm2, %ymm1, %ymm1 # sched: [5:0.50]
-; HASWELL-NO-FMA-NEXT:    vbroadcastss {{.*#+}} ymm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0] sched: [7:0.50]
-; HASWELL-NO-FMA-NEXT:    vsubps %ymm1, %ymm3, %ymm1 # sched: [3:1.00]
-; HASWELL-NO-FMA-NEXT:    vmulps %ymm1, %ymm2, %ymm1 # sched: [5:0.50]
-; HASWELL-NO-FMA-NEXT:    vaddps %ymm1, %ymm2, %ymm1 # sched: [3:1.00]
-; HASWELL-NO-FMA-NEXT:    vrcpps %ymm0, %ymm2 # sched: [11:2.00]
-; HASWELL-NO-FMA-NEXT:    vmulps %ymm2, %ymm0, %ymm0 # sched: [5:0.50]
-; HASWELL-NO-FMA-NEXT:    vsubps %ymm0, %ymm3, %ymm0 # sched: [3:1.00]
-; HASWELL-NO-FMA-NEXT:    vmulps %ymm0, %ymm2, %ymm0 # sched: [5:0.50]
-; HASWELL-NO-FMA-NEXT:    vaddps %ymm0, %ymm2, %ymm0 # sched: [3:1.00]
-; HASWELL-NO-FMA-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm0 # sched: [12:0.50]
-; HASWELL-NO-FMA-NEXT:    vmulps {{.*}}(%rip), %ymm1, %ymm1 # sched: [12:0.50]
-; HASWELL-NO-FMA-NEXT:    retq # sched: [7:1.00]
-;
-; KNL-LABEL: v16f32_one_step2:
-; KNL:       # %bb.0:
-; KNL-NEXT:    vrcp14ps %zmm0, %zmm1 # sched: [11:2.00]
-; KNL-NEXT:    vfnmadd213ps {{.*#+}} zmm0 = -(zmm1 * zmm0) + mem sched: [12:0.50]
-; KNL-NEXT:    vfmadd132ps {{.*#+}} zmm0 = (zmm0 * zmm1) + zmm1 sched: [5:0.50]
-; KNL-NEXT:    vmulps {{.*}}(%rip), %zmm0, %zmm0 # sched: [12:0.50]
-; KNL-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: v16f32_one_step2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vrcp14ps %zmm0, %zmm1 # sched: [4:2.00]
-; SKX-NEXT:    vfnmadd213ps {{.*#+}} zmm0 = -(zmm1 * zmm0) + mem sched: [11:0.50]
-; SKX-NEXT:    vfmadd132ps {{.*#+}} zmm0 = (zmm0 * zmm1) + zmm1 sched: [4:0.50]
-; SKX-NEXT:    vmulps {{.*}}(%rip), %zmm0, %zmm0 # sched: [11:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
+; HASWELL-NO-FMA-NEXT:    vrcpps %ymm1, %ymm2
+; HASWELL-NO-FMA-NEXT:    vmulps %ymm2, %ymm1, %ymm1
+; HASWELL-NO-FMA-NEXT:    vbroadcastss {{.*#+}} ymm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; HASWELL-NO-FMA-NEXT:    vsubps %ymm1, %ymm3, %ymm1
+; HASWELL-NO-FMA-NEXT:    vmulps %ymm1, %ymm2, %ymm1
+; HASWELL-NO-FMA-NEXT:    vaddps %ymm1, %ymm2, %ymm1
+; HASWELL-NO-FMA-NEXT:    vrcpps %ymm0, %ymm2
+; HASWELL-NO-FMA-NEXT:    vmulps %ymm2, %ymm0, %ymm0
+; HASWELL-NO-FMA-NEXT:    vsubps %ymm0, %ymm3, %ymm0
+; HASWELL-NO-FMA-NEXT:    vmulps %ymm0, %ymm2, %ymm0
+; HASWELL-NO-FMA-NEXT:    vaddps %ymm0, %ymm2, %ymm0
+; HASWELL-NO-FMA-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm0
+; HASWELL-NO-FMA-NEXT:    vmulps {{.*}}(%rip), %ymm1, %ymm1
+; HASWELL-NO-FMA-NEXT:    retq
+;
+; AVX512-LABEL: v16f32_one_step2:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vrcp14ps %zmm0, %zmm1
+; AVX512-NEXT:    vfnmadd213ps {{.*#+}} zmm0 = -(zmm1 * zmm0) + mem
+; AVX512-NEXT:    vfmadd132ps {{.*#+}} zmm0 = (zmm0 * zmm1) + zmm1
+; AVX512-NEXT:    vmulps {{.*}}(%rip), %zmm0, %zmm0
+; AVX512-NEXT:    retq
   %div = fdiv fast <16 x float> <float 1.0, float 2.0, float 3.0, float 4.0, float 5.0, float 6.0, float 7.0, float 8.0, float 9.0, float 10.0, float 11.0, float 12.0, float 13.0, float 14.0, float 15.0, float 16.0>, %x
   ret <16 x float> %div
 }
@@ -1532,108 +1335,99 @@ define <16 x float> @v16f32_one_step_2_divs(<16 x float> %x) #1 {
 ;
 ; BDVER2-LABEL: v16f32_one_step_2_divs:
 ; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vrcpps %ymm0, %ymm2 # sched: [5:2.00]
-; BDVER2-NEXT:    vmovaps {{.*#+}} ymm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0] sched: [5:0.50]
-; BDVER2-NEXT:    vfnmaddps %ymm3, %ymm2, %ymm0, %ymm0 # sched: [5:0.50]
-; BDVER2-NEXT:    vfmaddps %ymm2, %ymm0, %ymm2, %ymm0 # sched: [5:0.50]
-; BDVER2-NEXT:    vrcpps %ymm1, %ymm2 # sched: [5:2.00]
-; BDVER2-NEXT:    vfnmaddps %ymm3, %ymm2, %ymm1, %ymm1 # sched: [5:0.50]
-; BDVER2-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm3 # sched: [10:2.00]
-; BDVER2-NEXT:    vfmaddps %ymm2, %ymm1, %ymm2, %ymm1 # sched: [5:0.50]
-; BDVER2-NEXT:    vmulps {{.*}}(%rip), %ymm1, %ymm2 # sched: [10:2.00]
-; BDVER2-NEXT:    vmulps %ymm0, %ymm3, %ymm0 # sched: [5:2.00]
-; BDVER2-NEXT:    vmulps %ymm1, %ymm2, %ymm1 # sched: [5:2.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
+; BDVER2-NEXT:    vrcpps %ymm0, %ymm2
+; BDVER2-NEXT:    vmovaps {{.*#+}} ymm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; BDVER2-NEXT:    vfnmaddps %ymm3, %ymm2, %ymm0, %ymm0
+; BDVER2-NEXT:    vfmaddps %ymm2, %ymm0, %ymm2, %ymm0
+; BDVER2-NEXT:    vrcpps %ymm1, %ymm2
+; BDVER2-NEXT:    vfnmaddps %ymm3, %ymm2, %ymm1, %ymm1
+; BDVER2-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm3
+; BDVER2-NEXT:    vfmaddps %ymm2, %ymm1, %ymm2, %ymm1
+; BDVER2-NEXT:    vmulps {{.*}}(%rip), %ymm1, %ymm2
+; BDVER2-NEXT:    vmulps %ymm0, %ymm3, %ymm0
+; BDVER2-NEXT:    vmulps %ymm1, %ymm2, %ymm1
+; BDVER2-NEXT:    retq
 ;
 ; BTVER2-LABEL: v16f32_one_step_2_divs:
 ; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vmovaps {{.*#+}} ymm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0] sched: [5:1.00]
-; BTVER2-NEXT:    vrcpps %ymm0, %ymm2 # sched: [2:2.00]
-; BTVER2-NEXT:    vmulps %ymm2, %ymm0, %ymm0 # sched: [2:2.00]
-; BTVER2-NEXT:    vsubps %ymm0, %ymm3, %ymm0 # sched: [3:2.00]
-; BTVER2-NEXT:    vmulps %ymm0, %ymm2, %ymm0 # sched: [2:2.00]
-; BTVER2-NEXT:    vaddps %ymm0, %ymm2, %ymm0 # sched: [3:2.00]
-; BTVER2-NEXT:    vrcpps %ymm1, %ymm2 # sched: [2:2.00]
-; BTVER2-NEXT:    vmulps %ymm2, %ymm1, %ymm1 # sched: [2:2.00]
-; BTVER2-NEXT:    vsubps %ymm1, %ymm3, %ymm1 # sched: [3:2.00]
-; BTVER2-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm3 # sched: [7:2.00]
-; BTVER2-NEXT:    vmulps %ymm1, %ymm2, %ymm1 # sched: [2:2.00]
-; BTVER2-NEXT:    vaddps %ymm1, %ymm2, %ymm1 # sched: [3:2.00]
-; BTVER2-NEXT:    vmulps {{.*}}(%rip), %ymm1, %ymm2 # sched: [7:2.00]
-; BTVER2-NEXT:    vmulps %ymm0, %ymm3, %ymm0 # sched: [2:2.00]
-; BTVER2-NEXT:    vmulps %ymm1, %ymm2, %ymm1 # sched: [2:2.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
+; BTVER2-NEXT:    vmovaps {{.*#+}} ymm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; BTVER2-NEXT:    vrcpps %ymm0, %ymm2
+; BTVER2-NEXT:    vmulps %ymm2, %ymm0, %ymm0
+; BTVER2-NEXT:    vsubps %ymm0, %ymm3, %ymm0
+; BTVER2-NEXT:    vmulps %ymm0, %ymm2, %ymm0
+; BTVER2-NEXT:    vaddps %ymm0, %ymm2, %ymm0
+; BTVER2-NEXT:    vrcpps %ymm1, %ymm2
+; BTVER2-NEXT:    vmulps %ymm2, %ymm1, %ymm1
+; BTVER2-NEXT:    vsubps %ymm1, %ymm3, %ymm1
+; BTVER2-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm3
+; BTVER2-NEXT:    vmulps %ymm1, %ymm2, %ymm1
+; BTVER2-NEXT:    vaddps %ymm1, %ymm2, %ymm1
+; BTVER2-NEXT:    vmulps {{.*}}(%rip), %ymm1, %ymm2
+; BTVER2-NEXT:    vmulps %ymm0, %ymm3, %ymm0
+; BTVER2-NEXT:    vmulps %ymm1, %ymm2, %ymm1
+; BTVER2-NEXT:    retq
 ;
 ; SANDY-LABEL: v16f32_one_step_2_divs:
 ; SANDY:       # %bb.0:
-; SANDY-NEXT:    vrcpps %ymm0, %ymm2 # sched: [7:2.00]
-; SANDY-NEXT:    vmulps %ymm2, %ymm0, %ymm0 # sched: [5:1.00]
-; SANDY-NEXT:    vmovaps {{.*#+}} ymm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0] sched: [7:0.50]
-; SANDY-NEXT:    vsubps %ymm0, %ymm3, %ymm0 # sched: [3:1.00]
-; SANDY-NEXT:    vrcpps %ymm1, %ymm4 # sched: [7:2.00]
-; SANDY-NEXT:    vmulps %ymm0, %ymm2, %ymm0 # sched: [5:1.00]
-; SANDY-NEXT:    vaddps %ymm0, %ymm2, %ymm0 # sched: [3:1.00]
-; SANDY-NEXT:    vmulps %ymm4, %ymm1, %ymm1 # sched: [5:1.00]
-; SANDY-NEXT:    vsubps %ymm1, %ymm3, %ymm1 # sched: [3:1.00]
-; SANDY-NEXT:    vmulps %ymm1, %ymm4, %ymm1 # sched: [5:1.00]
-; SANDY-NEXT:    vaddps %ymm1, %ymm4, %ymm1 # sched: [3:1.00]
-; SANDY-NEXT:    vmulps {{.*}}(%rip), %ymm1, %ymm2 # sched: [12:1.00]
-; SANDY-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm3 # sched: [12:1.00]
-; SANDY-NEXT:    vmulps %ymm0, %ymm3, %ymm0 # sched: [5:1.00]
-; SANDY-NEXT:    vmulps %ymm1, %ymm2, %ymm1 # sched: [5:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
+; SANDY-NEXT:    vrcpps %ymm0, %ymm2
+; SANDY-NEXT:    vmulps %ymm2, %ymm0, %ymm0
+; SANDY-NEXT:    vmovaps {{.*#+}} ymm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; SANDY-NEXT:    vsubps %ymm0, %ymm3, %ymm0
+; SANDY-NEXT:    vrcpps %ymm1, %ymm4
+; SANDY-NEXT:    vmulps %ymm0, %ymm2, %ymm0
+; SANDY-NEXT:    vaddps %ymm0, %ymm2, %ymm0
+; SANDY-NEXT:    vmulps %ymm4, %ymm1, %ymm1
+; SANDY-NEXT:    vsubps %ymm1, %ymm3, %ymm1
+; SANDY-NEXT:    vmulps %ymm1, %ymm4, %ymm1
+; SANDY-NEXT:    vaddps %ymm1, %ymm4, %ymm1
+; SANDY-NEXT:    vmulps {{.*}}(%rip), %ymm1, %ymm2
+; SANDY-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm3
+; SANDY-NEXT:    vmulps %ymm0, %ymm3, %ymm0
+; SANDY-NEXT:    vmulps %ymm1, %ymm2, %ymm1
+; SANDY-NEXT:    retq
 ;
 ; HASWELL-LABEL: v16f32_one_step_2_divs:
 ; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vrcpps %ymm0, %ymm2 # sched: [11:2.00]
-; HASWELL-NEXT:    vbroadcastss {{.*#+}} ymm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0] sched: [7:0.50]
-; HASWELL-NEXT:    vfnmadd213ps {{.*#+}} ymm0 = -(ymm2 * ymm0) + ymm3 sched: [5:0.50]
-; HASWELL-NEXT:    vfmadd132ps {{.*#+}} ymm0 = (ymm0 * ymm2) + ymm2 sched: [5:0.50]
-; HASWELL-NEXT:    vrcpps %ymm1, %ymm2 # sched: [11:2.00]
-; HASWELL-NEXT:    vfnmadd213ps {{.*#+}} ymm1 = -(ymm2 * ymm1) + ymm3 sched: [5:0.50]
-; HASWELL-NEXT:    vfmadd132ps {{.*#+}} ymm1 = (ymm1 * ymm2) + ymm2 sched: [5:0.50]
-; HASWELL-NEXT:    vmulps {{.*}}(%rip), %ymm1, %ymm2 # sched: [12:0.50]
-; HASWELL-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm3 # sched: [12:0.50]
-; HASWELL-NEXT:    vmulps %ymm0, %ymm3, %ymm0 # sched: [5:0.50]
-; HASWELL-NEXT:    vmulps %ymm1, %ymm2, %ymm1 # sched: [5:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
+; HASWELL-NEXT:    vrcpps %ymm0, %ymm2
+; HASWELL-NEXT:    vbroadcastss {{.*#+}} ymm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; HASWELL-NEXT:    vfnmadd213ps {{.*#+}} ymm0 = -(ymm2 * ymm0) + ymm3
+; HASWELL-NEXT:    vfmadd132ps {{.*#+}} ymm0 = (ymm0 * ymm2) + ymm2
+; HASWELL-NEXT:    vrcpps %ymm1, %ymm2
+; HASWELL-NEXT:    vfnmadd213ps {{.*#+}} ymm1 = -(ymm2 * ymm1) + ymm3
+; HASWELL-NEXT:    vfmadd132ps {{.*#+}} ymm1 = (ymm1 * ymm2) + ymm2
+; HASWELL-NEXT:    vmulps {{.*}}(%rip), %ymm1, %ymm2
+; HASWELL-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm3
+; HASWELL-NEXT:    vmulps %ymm0, %ymm3, %ymm0
+; HASWELL-NEXT:    vmulps %ymm1, %ymm2, %ymm1
+; HASWELL-NEXT:    retq
 ;
 ; HASWELL-NO-FMA-LABEL: v16f32_one_step_2_divs:
 ; HASWELL-NO-FMA:       # %bb.0:
-; HASWELL-NO-FMA-NEXT:    vrcpps %ymm0, %ymm2 # sched: [11:2.00]
-; HASWELL-NO-FMA-NEXT:    vmulps %ymm2, %ymm0, %ymm0 # sched: [5:0.50]
-; HASWELL-NO-FMA-NEXT:    vbroadcastss {{.*#+}} ymm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0] sched: [7:0.50]
-; HASWELL-NO-FMA-NEXT:    vsubps %ymm0, %ymm3, %ymm0 # sched: [3:1.00]
-; HASWELL-NO-FMA-NEXT:    vrcpps %ymm1, %ymm4 # sched: [11:2.00]
-; HASWELL-NO-FMA-NEXT:    vmulps %ymm0, %ymm2, %ymm0 # sched: [5:0.50]
-; HASWELL-NO-FMA-NEXT:    vaddps %ymm0, %ymm2, %ymm0 # sched: [3:1.00]
-; HASWELL-NO-FMA-NEXT:    vmulps %ymm4, %ymm1, %ymm1 # sched: [5:0.50]
-; HASWELL-NO-FMA-NEXT:    vsubps %ymm1, %ymm3, %ymm1 # sched: [3:1.00]
-; HASWELL-NO-FMA-NEXT:    vmulps %ymm1, %ymm4, %ymm1 # sched: [5:0.50]
-; HASWELL-NO-FMA-NEXT:    vaddps %ymm1, %ymm4, %ymm1 # sched: [3:1.00]
-; HASWELL-NO-FMA-NEXT:    vmulps {{.*}}(%rip), %ymm1, %ymm2 # sched: [12:0.50]
-; HASWELL-NO-FMA-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm3 # sched: [12:0.50]
-; HASWELL-NO-FMA-NEXT:    vmulps %ymm0, %ymm3, %ymm0 # sched: [5:0.50]
-; HASWELL-NO-FMA-NEXT:    vmulps %ymm1, %ymm2, %ymm1 # sched: [5:0.50]
-; HASWELL-NO-FMA-NEXT:    retq # sched: [7:1.00]
-;
-; KNL-LABEL: v16f32_one_step_2_divs:
-; KNL:       # %bb.0:
-; KNL-NEXT:    vrcp14ps %zmm0, %zmm1 # sched: [11:2.00]
-; KNL-NEXT:    vfnmadd213ps {{.*#+}} zmm0 = -(zmm1 * zmm0) + mem sched: [12:0.50]
-; KNL-NEXT:    vfmadd132ps {{.*#+}} zmm0 = (zmm0 * zmm1) + zmm1 sched: [5:0.50]
-; KNL-NEXT:    vmulps {{.*}}(%rip), %zmm0, %zmm1 # sched: [12:0.50]
-; KNL-NEXT:    vmulps %zmm0, %zmm1, %zmm0 # sched: [5:0.50]
-; KNL-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: v16f32_one_step_2_divs:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vrcp14ps %zmm0, %zmm1 # sched: [4:2.00]
-; SKX-NEXT:    vfnmadd213ps {{.*#+}} zmm0 = -(zmm1 * zmm0) + mem sched: [11:0.50]
-; SKX-NEXT:    vfmadd132ps {{.*#+}} zmm0 = (zmm0 * zmm1) + zmm1 sched: [4:0.50]
-; SKX-NEXT:    vmulps {{.*}}(%rip), %zmm0, %zmm1 # sched: [11:0.50]
-; SKX-NEXT:    vmulps %zmm0, %zmm1, %zmm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
+; HASWELL-NO-FMA-NEXT:    vrcpps %ymm0, %ymm2
+; HASWELL-NO-FMA-NEXT:    vmulps %ymm2, %ymm0, %ymm0
+; HASWELL-NO-FMA-NEXT:    vbroadcastss {{.*#+}} ymm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; HASWELL-NO-FMA-NEXT:    vsubps %ymm0, %ymm3, %ymm0
+; HASWELL-NO-FMA-NEXT:    vrcpps %ymm1, %ymm4
+; HASWELL-NO-FMA-NEXT:    vmulps %ymm0, %ymm2, %ymm0
+; HASWELL-NO-FMA-NEXT:    vaddps %ymm0, %ymm2, %ymm0
+; HASWELL-NO-FMA-NEXT:    vmulps %ymm4, %ymm1, %ymm1
+; HASWELL-NO-FMA-NEXT:    vsubps %ymm1, %ymm3, %ymm1
+; HASWELL-NO-FMA-NEXT:    vmulps %ymm1, %ymm4, %ymm1
+; HASWELL-NO-FMA-NEXT:    vaddps %ymm1, %ymm4, %ymm1
+; HASWELL-NO-FMA-NEXT:    vmulps {{.*}}(%rip), %ymm1, %ymm2
+; HASWELL-NO-FMA-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm3
+; HASWELL-NO-FMA-NEXT:    vmulps %ymm0, %ymm3, %ymm0
+; HASWELL-NO-FMA-NEXT:    vmulps %ymm1, %ymm2, %ymm1
+; HASWELL-NO-FMA-NEXT:    retq
+;
+; AVX512-LABEL: v16f32_one_step_2_divs:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vrcp14ps %zmm0, %zmm1
+; AVX512-NEXT:    vfnmadd213ps {{.*#+}} zmm0 = -(zmm1 * zmm0) + mem
+; AVX512-NEXT:    vfmadd132ps {{.*#+}} zmm0 = (zmm0 * zmm1) + zmm1
+; AVX512-NEXT:    vmulps {{.*}}(%rip), %zmm0, %zmm1
+; AVX512-NEXT:    vmulps %zmm0, %zmm1, %zmm0
+; AVX512-NEXT:    retq
   %div = fdiv fast <16 x float> <float 1.0, float 2.0, float 3.0, float 4.0, float 5.0, float 6.0, float 7.0, float 8.0, float 9.0, float 10.0, float 11.0, float 12.0, float 13.0, float 14.0, float 15.0, float 16.0>, %x
   %div2 = fdiv fast <16 x float> %div, %x
   ret <16 x float> %div2
@@ -1745,138 +1539,126 @@ define <16 x float> @v16f32_two_step2(<16 x float> %x) #2 {
 ;
 ; BDVER2-LABEL: v16f32_two_step2:
 ; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vrcpps %ymm1, %ymm2 # sched: [5:2.00]
-; BDVER2-NEXT:    vmovaps {{.*#+}} ymm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0] sched: [5:0.50]
-; BDVER2-NEXT:    vfnmaddps %ymm3, %ymm2, %ymm1, %ymm4 # sched: [5:0.50]
-; BDVER2-NEXT:    vfmaddps %ymm2, %ymm4, %ymm2, %ymm2 # sched: [5:0.50]
-; BDVER2-NEXT:    vfnmaddps %ymm3, %ymm2, %ymm1, %ymm1 # sched: [5:0.50]
-; BDVER2-NEXT:    vfmaddps %ymm2, %ymm1, %ymm2, %ymm1 # sched: [5:0.50]
-; BDVER2-NEXT:    vrcpps %ymm0, %ymm2 # sched: [5:2.00]
-; BDVER2-NEXT:    vmulps {{.*}}(%rip), %ymm1, %ymm1 # sched: [10:2.00]
-; BDVER2-NEXT:    vfnmaddps %ymm3, %ymm2, %ymm0, %ymm4 # sched: [5:0.50]
-; BDVER2-NEXT:    vfmaddps %ymm2, %ymm4, %ymm2, %ymm2 # sched: [5:0.50]
-; BDVER2-NEXT:    vfnmaddps %ymm3, %ymm2, %ymm0, %ymm0 # sched: [5:0.50]
-; BDVER2-NEXT:    vfmaddps %ymm2, %ymm0, %ymm2, %ymm0 # sched: [5:0.50]
-; BDVER2-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm0 # sched: [10:2.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
+; BDVER2-NEXT:    vrcpps %ymm1, %ymm2
+; BDVER2-NEXT:    vmovaps {{.*#+}} ymm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; BDVER2-NEXT:    vfnmaddps %ymm3, %ymm2, %ymm1, %ymm4
+; BDVER2-NEXT:    vfmaddps %ymm2, %ymm4, %ymm2, %ymm2
+; BDVER2-NEXT:    vfnmaddps %ymm3, %ymm2, %ymm1, %ymm1
+; BDVER2-NEXT:    vfmaddps %ymm2, %ymm1, %ymm2, %ymm1
+; BDVER2-NEXT:    vrcpps %ymm0, %ymm2
+; BDVER2-NEXT:    vmulps {{.*}}(%rip), %ymm1, %ymm1
+; BDVER2-NEXT:    vfnmaddps %ymm3, %ymm2, %ymm0, %ymm4
+; BDVER2-NEXT:    vfmaddps %ymm2, %ymm4, %ymm2, %ymm2
+; BDVER2-NEXT:    vfnmaddps %ymm3, %ymm2, %ymm0, %ymm0
+; BDVER2-NEXT:    vfmaddps %ymm2, %ymm0, %ymm2, %ymm0
+; BDVER2-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm0
+; BDVER2-NEXT:    retq
 ;
 ; BTVER2-LABEL: v16f32_two_step2:
 ; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vmovaps {{.*#+}} ymm4 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0] sched: [5:1.00]
-; BTVER2-NEXT:    vrcpps %ymm1, %ymm2 # sched: [2:2.00]
-; BTVER2-NEXT:    vmulps %ymm2, %ymm1, %ymm3 # sched: [2:2.00]
-; BTVER2-NEXT:    vsubps %ymm3, %ymm4, %ymm3 # sched: [3:2.00]
-; BTVER2-NEXT:    vmulps %ymm3, %ymm2, %ymm3 # sched: [2:2.00]
-; BTVER2-NEXT:    vaddps %ymm3, %ymm2, %ymm2 # sched: [3:2.00]
-; BTVER2-NEXT:    vmulps %ymm2, %ymm1, %ymm1 # sched: [2:2.00]
-; BTVER2-NEXT:    vsubps %ymm1, %ymm4, %ymm1 # sched: [3:2.00]
-; BTVER2-NEXT:    vmulps %ymm1, %ymm2, %ymm1 # sched: [2:2.00]
-; BTVER2-NEXT:    vaddps %ymm1, %ymm2, %ymm1 # sched: [3:2.00]
-; BTVER2-NEXT:    vrcpps %ymm0, %ymm2 # sched: [2:2.00]
-; BTVER2-NEXT:    vmulps {{.*}}(%rip), %ymm1, %ymm1 # sched: [7:2.00]
-; BTVER2-NEXT:    vmulps %ymm2, %ymm0, %ymm3 # sched: [2:2.00]
-; BTVER2-NEXT:    vsubps %ymm3, %ymm4, %ymm3 # sched: [3:2.00]
-; BTVER2-NEXT:    vmulps %ymm3, %ymm2, %ymm3 # sched: [2:2.00]
-; BTVER2-NEXT:    vaddps %ymm3, %ymm2, %ymm2 # sched: [3:2.00]
-; BTVER2-NEXT:    vmulps %ymm2, %ymm0, %ymm0 # sched: [2:2.00]
-; BTVER2-NEXT:    vsubps %ymm0, %ymm4, %ymm0 # sched: [3:2.00]
-; BTVER2-NEXT:    vmulps %ymm0, %ymm2, %ymm0 # sched: [2:2.00]
-; BTVER2-NEXT:    vaddps %ymm0, %ymm2, %ymm0 # sched: [3:2.00]
-; BTVER2-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm0 # sched: [7:2.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
+; BTVER2-NEXT:    vmovaps {{.*#+}} ymm4 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; BTVER2-NEXT:    vrcpps %ymm1, %ymm2
+; BTVER2-NEXT:    vmulps %ymm2, %ymm1, %ymm3
+; BTVER2-NEXT:    vsubps %ymm3, %ymm4, %ymm3
+; BTVER2-NEXT:    vmulps %ymm3, %ymm2, %ymm3
+; BTVER2-NEXT:    vaddps %ymm3, %ymm2, %ymm2
+; BTVER2-NEXT:    vmulps %ymm2, %ymm1, %ymm1
+; BTVER2-NEXT:    vsubps %ymm1, %ymm4, %ymm1
+; BTVER2-NEXT:    vmulps %ymm1, %ymm2, %ymm1
+; BTVER2-NEXT:    vaddps %ymm1, %ymm2, %ymm1
+; BTVER2-NEXT:    vrcpps %ymm0, %ymm2
+; BTVER2-NEXT:    vmulps {{.*}}(%rip), %ymm1, %ymm1
+; BTVER2-NEXT:    vmulps %ymm2, %ymm0, %ymm3
+; BTVER2-NEXT:    vsubps %ymm3, %ymm4, %ymm3
+; BTVER2-NEXT:    vmulps %ymm3, %ymm2, %ymm3
+; BTVER2-NEXT:    vaddps %ymm3, %ymm2, %ymm2
+; BTVER2-NEXT:    vmulps %ymm2, %ymm0, %ymm0
+; BTVER2-NEXT:    vsubps %ymm0, %ymm4, %ymm0
+; BTVER2-NEXT:    vmulps %ymm0, %ymm2, %ymm0
+; BTVER2-NEXT:    vaddps %ymm0, %ymm2, %ymm0
+; BTVER2-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm0
+; BTVER2-NEXT:    retq
 ;
 ; SANDY-LABEL: v16f32_two_step2:
 ; SANDY:       # %bb.0:
-; SANDY-NEXT:    vrcpps %ymm1, %ymm2 # sched: [7:2.00]
-; SANDY-NEXT:    vmulps %ymm2, %ymm1, %ymm3 # sched: [5:1.00]
-; SANDY-NEXT:    vmovaps {{.*#+}} ymm4 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0] sched: [7:0.50]
-; SANDY-NEXT:    vsubps %ymm3, %ymm4, %ymm3 # sched: [3:1.00]
-; SANDY-NEXT:    vmulps %ymm3, %ymm2, %ymm3 # sched: [5:1.00]
-; SANDY-NEXT:    vaddps %ymm3, %ymm2, %ymm2 # sched: [3:1.00]
-; SANDY-NEXT:    vmulps %ymm2, %ymm1, %ymm1 # sched: [5:1.00]
-; SANDY-NEXT:    vsubps %ymm1, %ymm4, %ymm1 # sched: [3:1.00]
-; SANDY-NEXT:    vmulps %ymm1, %ymm2, %ymm1 # sched: [5:1.00]
-; SANDY-NEXT:    vaddps %ymm1, %ymm2, %ymm1 # sched: [3:1.00]
-; SANDY-NEXT:    vrcpps %ymm0, %ymm2 # sched: [7:2.00]
-; SANDY-NEXT:    vmulps %ymm2, %ymm0, %ymm3 # sched: [5:1.00]
-; SANDY-NEXT:    vsubps %ymm3, %ymm4, %ymm3 # sched: [3:1.00]
-; SANDY-NEXT:    vmulps %ymm3, %ymm2, %ymm3 # sched: [5:1.00]
-; SANDY-NEXT:    vaddps %ymm3, %ymm2, %ymm2 # sched: [3:1.00]
-; SANDY-NEXT:    vmulps %ymm2, %ymm0, %ymm0 # sched: [5:1.00]
-; SANDY-NEXT:    vsubps %ymm0, %ymm4, %ymm0 # sched: [3:1.00]
-; SANDY-NEXT:    vmulps %ymm0, %ymm2, %ymm0 # sched: [5:1.00]
-; SANDY-NEXT:    vaddps %ymm0, %ymm2, %ymm0 # sched: [3:1.00]
-; SANDY-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm0 # sched: [12:1.00]
-; SANDY-NEXT:    vmulps {{.*}}(%rip), %ymm1, %ymm1 # sched: [12:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
+; SANDY-NEXT:    vrcpps %ymm1, %ymm2
+; SANDY-NEXT:    vmulps %ymm2, %ymm1, %ymm3
+; SANDY-NEXT:    vmovaps {{.*#+}} ymm4 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; SANDY-NEXT:    vsubps %ymm3, %ymm4, %ymm3
+; SANDY-NEXT:    vmulps %ymm3, %ymm2, %ymm3
+; SANDY-NEXT:    vaddps %ymm3, %ymm2, %ymm2
+; SANDY-NEXT:    vmulps %ymm2, %ymm1, %ymm1
+; SANDY-NEXT:    vsubps %ymm1, %ymm4, %ymm1
+; SANDY-NEXT:    vmulps %ymm1, %ymm2, %ymm1
+; SANDY-NEXT:    vaddps %ymm1, %ymm2, %ymm1
+; SANDY-NEXT:    vrcpps %ymm0, %ymm2
+; SANDY-NEXT:    vmulps %ymm2, %ymm0, %ymm3
+; SANDY-NEXT:    vsubps %ymm3, %ymm4, %ymm3
+; SANDY-NEXT:    vmulps %ymm3, %ymm2, %ymm3
+; SANDY-NEXT:    vaddps %ymm3, %ymm2, %ymm2
+; SANDY-NEXT:    vmulps %ymm2, %ymm0, %ymm0
+; SANDY-NEXT:    vsubps %ymm0, %ymm4, %ymm0
+; SANDY-NEXT:    vmulps %ymm0, %ymm2, %ymm0
+; SANDY-NEXT:    vaddps %ymm0, %ymm2, %ymm0
+; SANDY-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm0
+; SANDY-NEXT:    vmulps {{.*}}(%rip), %ymm1, %ymm1
+; SANDY-NEXT:    retq
 ;
 ; HASWELL-LABEL: v16f32_two_step2:
 ; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vrcpps %ymm1, %ymm2 # sched: [11:2.00]
-; HASWELL-NEXT:    vbroadcastss {{.*#+}} ymm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0] sched: [7:0.50]
-; HASWELL-NEXT:    vmovaps %ymm2, %ymm4 # sched: [1:1.00]
-; HASWELL-NEXT:    vfnmadd213ps {{.*#+}} ymm4 = -(ymm1 * ymm4) + ymm3 sched: [5:0.50]
-; HASWELL-NEXT:    vfmadd132ps {{.*#+}} ymm4 = (ymm4 * ymm2) + ymm2 sched: [5:0.50]
-; HASWELL-NEXT:    vfnmadd213ps {{.*#+}} ymm1 = -(ymm4 * ymm1) + ymm3 sched: [5:0.50]
-; HASWELL-NEXT:    vfmadd132ps {{.*#+}} ymm1 = (ymm1 * ymm4) + ymm4 sched: [5:0.50]
-; HASWELL-NEXT:    vrcpps %ymm0, %ymm2 # sched: [11:2.00]
-; HASWELL-NEXT:    vmovaps %ymm2, %ymm4 # sched: [1:1.00]
-; HASWELL-NEXT:    vfnmadd213ps {{.*#+}} ymm4 = -(ymm0 * ymm4) + ymm3 sched: [5:0.50]
-; HASWELL-NEXT:    vfmadd132ps {{.*#+}} ymm4 = (ymm4 * ymm2) + ymm2 sched: [5:0.50]
-; HASWELL-NEXT:    vfnmadd213ps {{.*#+}} ymm0 = -(ymm4 * ymm0) + ymm3 sched: [5:0.50]
-; HASWELL-NEXT:    vfmadd132ps {{.*#+}} ymm0 = (ymm0 * ymm4) + ymm4 sched: [5:0.50]
-; HASWELL-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm0 # sched: [12:0.50]
-; HASWELL-NEXT:    vmulps {{.*}}(%rip), %ymm1, %ymm1 # sched: [12:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
+; HASWELL-NEXT:    vrcpps %ymm1, %ymm2
+; HASWELL-NEXT:    vbroadcastss {{.*#+}} ymm3 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; HASWELL-NEXT:    vmovaps %ymm2, %ymm4
+; HASWELL-NEXT:    vfnmadd213ps {{.*#+}} ymm4 = -(ymm1 * ymm4) + ymm3
+; HASWELL-NEXT:    vfmadd132ps {{.*#+}} ymm4 = (ymm4 * ymm2) + ymm2
+; HASWELL-NEXT:    vfnmadd213ps {{.*#+}} ymm1 = -(ymm4 * ymm1) + ymm3
+; HASWELL-NEXT:    vfmadd132ps {{.*#+}} ymm1 = (ymm1 * ymm4) + ymm4
+; HASWELL-NEXT:    vrcpps %ymm0, %ymm2
+; HASWELL-NEXT:    vmovaps %ymm2, %ymm4
+; HASWELL-NEXT:    vfnmadd213ps {{.*#+}} ymm4 = -(ymm0 * ymm4) + ymm3
+; HASWELL-NEXT:    vfmadd132ps {{.*#+}} ymm4 = (ymm4 * ymm2) + ymm2
+; HASWELL-NEXT:    vfnmadd213ps {{.*#+}} ymm0 = -(ymm4 * ymm0) + ymm3
+; HASWELL-NEXT:    vfmadd132ps {{.*#+}} ymm0 = (ymm0 * ymm4) + ymm4
+; HASWELL-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm0
+; HASWELL-NEXT:    vmulps {{.*}}(%rip), %ymm1, %ymm1
+; HASWELL-NEXT:    retq
 ;
 ; HASWELL-NO-FMA-LABEL: v16f32_two_step2:
 ; HASWELL-NO-FMA:       # %bb.0:
-; HASWELL-NO-FMA-NEXT:    vrcpps %ymm1, %ymm2 # sched: [11:2.00]
-; HASWELL-NO-FMA-NEXT:    vmulps %ymm2, %ymm1, %ymm3 # sched: [5:0.50]
-; HASWELL-NO-FMA-NEXT:    vbroadcastss {{.*#+}} ymm4 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0] sched: [7:0.50]
-; HASWELL-NO-FMA-NEXT:    vsubps %ymm3, %ymm4, %ymm3 # sched: [3:1.00]
-; HASWELL-NO-FMA-NEXT:    vmulps %ymm3, %ymm2, %ymm3 # sched: [5:0.50]
-; HASWELL-NO-FMA-NEXT:    vaddps %ymm3, %ymm2, %ymm2 # sched: [3:1.00]
-; HASWELL-NO-FMA-NEXT:    vmulps %ymm2, %ymm1, %ymm1 # sched: [5:0.50]
-; HASWELL-NO-FMA-NEXT:    vsubps %ymm1, %ymm4, %ymm1 # sched: [3:1.00]
-; HASWELL-NO-FMA-NEXT:    vmulps %ymm1, %ymm2, %ymm1 # sched: [5:0.50]
-; HASWELL-NO-FMA-NEXT:    vaddps %ymm1, %ymm2, %ymm1 # sched: [3:1.00]
-; HASWELL-NO-FMA-NEXT:    vrcpps %ymm0, %ymm2 # sched: [11:2.00]
-; HASWELL-NO-FMA-NEXT:    vmulps %ymm2, %ymm0, %ymm3 # sched: [5:0.50]
-; HASWELL-NO-FMA-NEXT:    vsubps %ymm3, %ymm4, %ymm3 # sched: [3:1.00]
-; HASWELL-NO-FMA-NEXT:    vmulps %ymm3, %ymm2, %ymm3 # sched: [5:0.50]
-; HASWELL-NO-FMA-NEXT:    vaddps %ymm3, %ymm2, %ymm2 # sched: [3:1.00]
-; HASWELL-NO-FMA-NEXT:    vmulps %ymm2, %ymm0, %ymm0 # sched: [5:0.50]
-; HASWELL-NO-FMA-NEXT:    vsubps %ymm0, %ymm4, %ymm0 # sched: [3:1.00]
-; HASWELL-NO-FMA-NEXT:    vmulps %ymm0, %ymm2, %ymm0 # sched: [5:0.50]
-; HASWELL-NO-FMA-NEXT:    vaddps %ymm0, %ymm2, %ymm0 # sched: [3:1.00]
-; HASWELL-NO-FMA-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm0 # sched: [12:0.50]
-; HASWELL-NO-FMA-NEXT:    vmulps {{.*}}(%rip), %ymm1, %ymm1 # sched: [12:0.50]
-; HASWELL-NO-FMA-NEXT:    retq # sched: [7:1.00]
-;
-; KNL-LABEL: v16f32_two_step2:
-; KNL:       # %bb.0:
-; KNL-NEXT:    vrcp14ps %zmm0, %zmm1 # sched: [11:2.00]
-; KNL-NEXT:    vbroadcastss {{.*#+}} zmm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0] sched: [10:1.00]
-; KNL-NEXT:    vmovaps %zmm1, %zmm3 # sched: [1:1.00]
-; KNL-NEXT:    vfnmadd213ps {{.*#+}} zmm3 = -(zmm0 * zmm3) + zmm2 sched: [5:0.50]
-; KNL-NEXT:    vfmadd132ps {{.*#+}} zmm3 = (zmm3 * zmm1) + zmm1 sched: [5:0.50]
-; KNL-NEXT:    vfnmadd213ps {{.*#+}} zmm0 = -(zmm3 * zmm0) + zmm2 sched: [5:0.50]
-; KNL-NEXT:    vfmadd132ps {{.*#+}} zmm0 = (zmm0 * zmm3) + zmm3 sched: [5:0.50]
-; KNL-NEXT:    vmulps {{.*}}(%rip), %zmm0, %zmm0 # sched: [12:0.50]
-; KNL-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: v16f32_two_step2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vrcp14ps %zmm0, %zmm1 # sched: [4:2.00]
-; SKX-NEXT:    vbroadcastss {{.*#+}} zmm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0] sched: [8:0.50]
-; SKX-NEXT:    vmovaps %zmm1, %zmm3 # sched: [1:0.33]
-; SKX-NEXT:    vfnmadd213ps {{.*#+}} zmm3 = -(zmm0 * zmm3) + zmm2 sched: [4:0.50]
-; SKX-NEXT:    vfmadd132ps {{.*#+}} zmm3 = (zmm3 * zmm1) + zmm1 sched: [4:0.50]
-; SKX-NEXT:    vfnmadd213ps {{.*#+}} zmm0 = -(zmm3 * zmm0) + zmm2 sched: [4:0.50]
-; SKX-NEXT:    vfmadd132ps {{.*#+}} zmm0 = (zmm0 * zmm3) + zmm3 sched: [4:0.50]
-; SKX-NEXT:    vmulps {{.*}}(%rip), %zmm0, %zmm0 # sched: [11:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
+; HASWELL-NO-FMA-NEXT:    vrcpps %ymm1, %ymm2
+; HASWELL-NO-FMA-NEXT:    vmulps %ymm2, %ymm1, %ymm3
+; HASWELL-NO-FMA-NEXT:    vbroadcastss {{.*#+}} ymm4 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; HASWELL-NO-FMA-NEXT:    vsubps %ymm3, %ymm4, %ymm3
+; HASWELL-NO-FMA-NEXT:    vmulps %ymm3, %ymm2, %ymm3
+; HASWELL-NO-FMA-NEXT:    vaddps %ymm3, %ymm2, %ymm2
+; HASWELL-NO-FMA-NEXT:    vmulps %ymm2, %ymm1, %ymm1
+; HASWELL-NO-FMA-NEXT:    vsubps %ymm1, %ymm4, %ymm1
+; HASWELL-NO-FMA-NEXT:    vmulps %ymm1, %ymm2, %ymm1
+; HASWELL-NO-FMA-NEXT:    vaddps %ymm1, %ymm2, %ymm1
+; HASWELL-NO-FMA-NEXT:    vrcpps %ymm0, %ymm2
+; HASWELL-NO-FMA-NEXT:    vmulps %ymm2, %ymm0, %ymm3
+; HASWELL-NO-FMA-NEXT:    vsubps %ymm3, %ymm4, %ymm3
+; HASWELL-NO-FMA-NEXT:    vmulps %ymm3, %ymm2, %ymm3
+; HASWELL-NO-FMA-NEXT:    vaddps %ymm3, %ymm2, %ymm2
+; HASWELL-NO-FMA-NEXT:    vmulps %ymm2, %ymm0, %ymm0
+; HASWELL-NO-FMA-NEXT:    vsubps %ymm0, %ymm4, %ymm0
+; HASWELL-NO-FMA-NEXT:    vmulps %ymm0, %ymm2, %ymm0
+; HASWELL-NO-FMA-NEXT:    vaddps %ymm0, %ymm2, %ymm0
+; HASWELL-NO-FMA-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm0
+; HASWELL-NO-FMA-NEXT:    vmulps {{.*}}(%rip), %ymm1, %ymm1
+; HASWELL-NO-FMA-NEXT:    retq
+;
+; AVX512-LABEL: v16f32_two_step2:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vrcp14ps %zmm0, %zmm1
+; AVX512-NEXT:    vbroadcastss {{.*#+}} zmm2 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0]
+; AVX512-NEXT:    vmovaps %zmm1, %zmm3
+; AVX512-NEXT:    vfnmadd213ps {{.*#+}} zmm3 = -(zmm0 * zmm3) + zmm2
+; AVX512-NEXT:    vfmadd132ps {{.*#+}} zmm3 = (zmm3 * zmm1) + zmm1
+; AVX512-NEXT:    vfnmadd213ps {{.*#+}} zmm0 = -(zmm3 * zmm0) + zmm2
+; AVX512-NEXT:    vfmadd132ps {{.*#+}} zmm0 = (zmm0 * zmm3) + zmm3
+; AVX512-NEXT:    vmulps {{.*}}(%rip), %zmm0, %zmm0
+; AVX512-NEXT:    retq
   %div = fdiv fast <16 x float> <float 1.0, float 2.0, float 3.0, float 4.0, float 5.0, float 6.0, float 7.0, float 8.0, float 9.0, float 10.0, float 11.0, float 12.0, float 13.0, float 14.0, float 15.0, float 16.0>, %x
   ret <16 x float> %div
 }
@@ -1904,43 +1686,38 @@ define <16 x float> @v16f32_no_step(<16 x float> %x) #3 {
 ;
 ; BDVER2-LABEL: v16f32_no_step:
 ; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vrcpps %ymm0, %ymm0 # sched: [5:2.00]
-; BDVER2-NEXT:    vrcpps %ymm1, %ymm1 # sched: [5:2.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
+; BDVER2-NEXT:    vrcpps %ymm0, %ymm0
+; BDVER2-NEXT:    vrcpps %ymm1, %ymm1
+; BDVER2-NEXT:    retq
 ;
 ; BTVER2-LABEL: v16f32_no_step:
 ; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vrcpps %ymm0, %ymm0 # sched: [2:2.00]
-; BTVER2-NEXT:    vrcpps %ymm1, %ymm1 # sched: [2:2.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
+; BTVER2-NEXT:    vrcpps %ymm0, %ymm0
+; BTVER2-NEXT:    vrcpps %ymm1, %ymm1
+; BTVER2-NEXT:    retq
 ;
 ; SANDY-LABEL: v16f32_no_step:
 ; SANDY:       # %bb.0:
-; SANDY-NEXT:    vrcpps %ymm0, %ymm0 # sched: [7:2.00]
-; SANDY-NEXT:    vrcpps %ymm1, %ymm1 # sched: [7:2.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
+; SANDY-NEXT:    vrcpps %ymm0, %ymm0
+; SANDY-NEXT:    vrcpps %ymm1, %ymm1
+; SANDY-NEXT:    retq
 ;
 ; HASWELL-LABEL: v16f32_no_step:
 ; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vrcpps %ymm0, %ymm0 # sched: [11:2.00]
-; HASWELL-NEXT:    vrcpps %ymm1, %ymm1 # sched: [11:2.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
+; HASWELL-NEXT:    vrcpps %ymm0, %ymm0
+; HASWELL-NEXT:    vrcpps %ymm1, %ymm1
+; HASWELL-NEXT:    retq
 ;
 ; HASWELL-NO-FMA-LABEL: v16f32_no_step:
 ; HASWELL-NO-FMA:       # %bb.0:
-; HASWELL-NO-FMA-NEXT:    vrcpps %ymm0, %ymm0 # sched: [11:2.00]
-; HASWELL-NO-FMA-NEXT:    vrcpps %ymm1, %ymm1 # sched: [11:2.00]
-; HASWELL-NO-FMA-NEXT:    retq # sched: [7:1.00]
-;
-; KNL-LABEL: v16f32_no_step:
-; KNL:       # %bb.0:
-; KNL-NEXT:    vrcp14ps %zmm0, %zmm0 # sched: [11:2.00]
-; KNL-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: v16f32_no_step:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vrcp14ps %zmm0, %zmm0 # sched: [4:2.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
+; HASWELL-NO-FMA-NEXT:    vrcpps %ymm0, %ymm0
+; HASWELL-NO-FMA-NEXT:    vrcpps %ymm1, %ymm1
+; HASWELL-NO-FMA-NEXT:    retq
+;
+; AVX512-LABEL: v16f32_no_step:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vrcp14ps %zmm0, %zmm0
+; AVX512-NEXT:    retq
   %div = fdiv fast <16 x float> <float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0, float 1.0>, %x
   ret <16 x float> %div
 }
@@ -1976,55 +1753,49 @@ define <16 x float> @v16f32_no_step2(<16 x float> %x) #3 {
 ;
 ; BDVER2-LABEL: v16f32_no_step2:
 ; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vrcpps %ymm1, %ymm1 # sched: [5:2.00]
-; BDVER2-NEXT:    vrcpps %ymm0, %ymm0 # sched: [5:2.00]
-; BDVER2-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm0 # sched: [10:2.00]
-; BDVER2-NEXT:    vmulps {{.*}}(%rip), %ymm1, %ymm1 # sched: [10:2.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
+; BDVER2-NEXT:    vrcpps %ymm1, %ymm1
+; BDVER2-NEXT:    vrcpps %ymm0, %ymm0
+; BDVER2-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm0
+; BDVER2-NEXT:    vmulps {{.*}}(%rip), %ymm1, %ymm1
+; BDVER2-NEXT:    retq
 ;
 ; BTVER2-LABEL: v16f32_no_step2:
 ; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vrcpps %ymm1, %ymm1 # sched: [2:2.00]
-; BTVER2-NEXT:    vrcpps %ymm0, %ymm0 # sched: [2:2.00]
-; BTVER2-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm0 # sched: [7:2.00]
-; BTVER2-NEXT:    vmulps {{.*}}(%rip), %ymm1, %ymm1 # sched: [7:2.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
+; BTVER2-NEXT:    vrcpps %ymm1, %ymm1
+; BTVER2-NEXT:    vrcpps %ymm0, %ymm0
+; BTVER2-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm0
+; BTVER2-NEXT:    vmulps {{.*}}(%rip), %ymm1, %ymm1
+; BTVER2-NEXT:    retq
 ;
 ; SANDY-LABEL: v16f32_no_step2:
 ; SANDY:       # %bb.0:
-; SANDY-NEXT:    vrcpps %ymm1, %ymm1 # sched: [7:2.00]
-; SANDY-NEXT:    vrcpps %ymm0, %ymm0 # sched: [7:2.00]
-; SANDY-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm0 # sched: [12:1.00]
-; SANDY-NEXT:    vmulps {{.*}}(%rip), %ymm1, %ymm1 # sched: [12:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
+; SANDY-NEXT:    vrcpps %ymm1, %ymm1
+; SANDY-NEXT:    vrcpps %ymm0, %ymm0
+; SANDY-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm0
+; SANDY-NEXT:    vmulps {{.*}}(%rip), %ymm1, %ymm1
+; SANDY-NEXT:    retq
 ;
 ; HASWELL-LABEL: v16f32_no_step2:
 ; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vrcpps %ymm1, %ymm1 # sched: [11:2.00]
-; HASWELL-NEXT:    vrcpps %ymm0, %ymm0 # sched: [11:2.00]
-; HASWELL-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm0 # sched: [12:0.50]
-; HASWELL-NEXT:    vmulps {{.*}}(%rip), %ymm1, %ymm1 # sched: [12:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
+; HASWELL-NEXT:    vrcpps %ymm1, %ymm1
+; HASWELL-NEXT:    vrcpps %ymm0, %ymm0
+; HASWELL-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm0
+; HASWELL-NEXT:    vmulps {{.*}}(%rip), %ymm1, %ymm1
+; HASWELL-NEXT:    retq
 ;
 ; HASWELL-NO-FMA-LABEL: v16f32_no_step2:
 ; HASWELL-NO-FMA:       # %bb.0:
-; HASWELL-NO-FMA-NEXT:    vrcpps %ymm1, %ymm1 # sched: [11:2.00]
-; HASWELL-NO-FMA-NEXT:    vrcpps %ymm0, %ymm0 # sched: [11:2.00]
-; HASWELL-NO-FMA-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm0 # sched: [12:0.50]
-; HASWELL-NO-FMA-NEXT:    vmulps {{.*}}(%rip), %ymm1, %ymm1 # sched: [12:0.50]
-; HASWELL-NO-FMA-NEXT:    retq # sched: [7:1.00]
-;
-; KNL-LABEL: v16f32_no_step2:
-; KNL:       # %bb.0:
-; KNL-NEXT:    vrcp14ps %zmm0, %zmm0 # sched: [11:2.00]
-; KNL-NEXT:    vmulps {{.*}}(%rip), %zmm0, %zmm0 # sched: [12:0.50]
-; KNL-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: v16f32_no_step2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vrcp14ps %zmm0, %zmm0 # sched: [4:2.00]
-; SKX-NEXT:    vmulps {{.*}}(%rip), %zmm0, %zmm0 # sched: [11:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
+; HASWELL-NO-FMA-NEXT:    vrcpps %ymm1, %ymm1
+; HASWELL-NO-FMA-NEXT:    vrcpps %ymm0, %ymm0
+; HASWELL-NO-FMA-NEXT:    vmulps {{.*}}(%rip), %ymm0, %ymm0
+; HASWELL-NO-FMA-NEXT:    vmulps {{.*}}(%rip), %ymm1, %ymm1
+; HASWELL-NO-FMA-NEXT:    retq
+;
+; AVX512-LABEL: v16f32_no_step2:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vrcp14ps %zmm0, %zmm0
+; AVX512-NEXT:    vmulps {{.*}}(%rip), %zmm0, %zmm0
+; AVX512-NEXT:    retq
   %div = fdiv fast <16 x float> <float 1.0, float 2.0, float 3.0, float 4.0, float 5.0, float 6.0, float 7.0, float 8.0, float 9.0, float 10.0, float 11.0, float 12.0, float 13.0, float 14.0, float 15.0, float 16.0>, %x
   ret <16 x float> %div
 }
diff --git a/test/CodeGen/X86/remat-fold-load.ll b/test/CodeGen/X86/remat-fold-load.ll
index e640974bdd29d7f03457de6c319de96b2fc2cc27..bd2895128873986eb4107593d4ec7bf5b4842db6 100644
--- a/test/CodeGen/X86/remat-fold-load.ll
+++ b/test/CodeGen/X86/remat-fold-load.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -disable-fp-elim -verify-coalescing
+; RUN: llc < %s -frame-pointer=all -verify-coalescing
 ; PR13414
 ;
 ; During coalescing, remat triggers DCE which deletes the penultimate use of a
diff --git a/test/CodeGen/X86/ret-addr.ll b/test/CodeGen/X86/ret-addr.ll
index cf164cc567a1261d2de0d9c0fc335882b31041c1..d90f639ba4701e45bc1fcbcf10a31e8f0b722d03 100644
--- a/test/CodeGen/X86/ret-addr.ll
+++ b/test/CodeGen/X86/ret-addr.ll
@@ -1,5 +1,5 @@
-; RUN: llc < %s -disable-fp-elim -mtriple=i686-- | not grep xor
-; RUN: llc < %s -disable-fp-elim -mtriple=x86_64-- | not grep xor
+; RUN: llc < %s -frame-pointer=all -mtriple=i686-- | not grep xor
+; RUN: llc < %s -frame-pointer=all -mtriple=x86_64-- | not grep xor
 
 define i8* @h() nounwind readnone optsize {
 entry:
diff --git a/test/CodeGen/X86/reverse_branches.ll b/test/CodeGen/X86/reverse_branches.ll
index 83a172b8de17e8d52c76681253469dbd7111e6a0..a0ca990415c4cf3ee90a24d737d8a816d12f0d38 100644
--- a/test/CodeGen/X86/reverse_branches.ll
+++ b/test/CodeGen/X86/reverse_branches.ll
@@ -1,4 +1,5 @@
-; RUN: llc -mtriple=x86_64-apple-darwin < %s | FileCheck %s
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-apple-darwin | FileCheck %s
 
 @.str2 = private unnamed_addr constant [7 x i8] c"memchr\00", align 1
 @.str3 = private unnamed_addr constant [11 x i8] c"bsd_memchr\00", align 1
@@ -8,12 +9,142 @@
 ; inner loop.
 define i32 @test_branches_order() uwtable ssp {
 ; CHECK-LABEL: test_branches_order:
-; CHECK: [[L0:LBB0_[0-9]+]]: ## %do.body.i
-; CHECK: je
-; CHECK: %do.cond.i
-; CHECK: jne [[L0]]
-; CHECK: jmp
-; CHECK: %exit
+; CHECK:       ## %bb.0: ## %entry
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    pushq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    pushq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 56
+; CHECK-NEXT:    subq $1001016, %rsp ## imm = 0xF4638
+; CHECK-NEXT:    .cfi_def_cfa_offset 1001072
+; CHECK-NEXT:    .cfi_offset %rbx, -56
+; CHECK-NEXT:    .cfi_offset %r12, -48
+; CHECK-NEXT:    .cfi_offset %r13, -40
+; CHECK-NEXT:    .cfi_offset %r14, -32
+; CHECK-NEXT:    .cfi_offset %r15, -24
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    movq ___stack_chk_guard@{{.*}}(%rip), %rax
+; CHECK-NEXT:    movq (%rax), %rax
+; CHECK-NEXT:    movq %rax, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    xorl %r12d, %r12d
+; CHECK-NEXT:    leaq -{{[0-9]+}}(%rsp), %r14
+; CHECK-NEXT:    movq %rsp, %r15
+; CHECK-NEXT:    cmpl $999, %r12d ## imm = 0x3E7
+; CHECK-NEXT:    jle LBB0_2
+; CHECK-NEXT:    jmp LBB0_7
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  LBB0_6: ## %for.inc9
+; CHECK-NEXT:    ## in Loop: Header=BB0_2 Depth=1
+; CHECK-NEXT:    incl %r12d
+; CHECK-NEXT:    cmpl $999, %r12d ## imm = 0x3E7
+; CHECK-NEXT:    jg LBB0_7
+; CHECK-NEXT:  LBB0_2: ## %for.cond1.preheader
+; CHECK-NEXT:    ## =>This Loop Header: Depth=1
+; CHECK-NEXT:    ## Child Loop BB0_3 Depth 2
+; CHECK-NEXT:    movl $-1, %r13d
+; CHECK-NEXT:    movq %r15, %rbx
+; CHECK-NEXT:    movq %r14, %rbp
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  LBB0_3: ## %for.cond1
+; CHECK-NEXT:    ## Parent Loop BB0_2 Depth=1
+; CHECK-NEXT:    ## => This Inner Loop Header: Depth=2
+; CHECK-NEXT:    incl %r13d
+; CHECK-NEXT:    cmpl $999, %r13d ## imm = 0x3E7
+; CHECK-NEXT:    jg LBB0_6
+; CHECK-NEXT:  ## %bb.4: ## %for.body3
+; CHECK-NEXT:    ## in Loop: Header=BB0_3 Depth=2
+; CHECK-NEXT:    addq $1002, %rbp ## imm = 0x3EA
+; CHECK-NEXT:    movq %rbx, %rdi
+; CHECK-NEXT:    leaq 1001(%rbx), %rbx
+; CHECK-NEXT:    movl $1000, %edx ## imm = 0x3E8
+; CHECK-NEXT:    movl $120, %esi
+; CHECK-NEXT:    callq _memchr
+; CHECK-NEXT:    cmpq %rax, %rbp
+; CHECK-NEXT:    je LBB0_3
+; CHECK-NEXT:    jmp LBB0_5
+; CHECK-NEXT:  LBB0_7: ## %for.end11
+; CHECK-NEXT:    leaq {{.*}}(%rip), %rdi
+; CHECK-NEXT:    callq _puts
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    movq %rsp, %rcx
+; CHECK-NEXT:    cmpl $999, %eax ## imm = 0x3E7
+; CHECK-NEXT:    jle LBB0_9
+; CHECK-NEXT:    jmp LBB0_16
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  LBB0_15: ## %for.inc38
+; CHECK-NEXT:    ## in Loop: Header=BB0_9 Depth=1
+; CHECK-NEXT:    incl %eax
+; CHECK-NEXT:    cmpl $999, %eax ## imm = 0x3E7
+; CHECK-NEXT:    jg LBB0_16
+; CHECK-NEXT:  LBB0_9: ## %for.cond18.preheader
+; CHECK-NEXT:    ## =>This Loop Header: Depth=1
+; CHECK-NEXT:    ## Child Loop BB0_10 Depth 2
+; CHECK-NEXT:    ## Child Loop BB0_12 Depth 3
+; CHECK-NEXT:    movq %rcx, %rdx
+; CHECK-NEXT:    xorl %esi, %esi
+; CHECK-NEXT:    xorl %edi, %edi
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  LBB0_10: ## %for.cond18
+; CHECK-NEXT:    ## Parent Loop BB0_9 Depth=1
+; CHECK-NEXT:    ## => This Loop Header: Depth=2
+; CHECK-NEXT:    ## Child Loop BB0_12 Depth 3
+; CHECK-NEXT:    cmpl $999, %edi ## imm = 0x3E7
+; CHECK-NEXT:    jg LBB0_15
+; CHECK-NEXT:  ## %bb.11: ## %for.body20
+; CHECK-NEXT:    ## in Loop: Header=BB0_10 Depth=2
+; CHECK-NEXT:    movq $-1000, %rbp ## imm = 0xFC18
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  LBB0_12: ## %do.body.i
+; CHECK-NEXT:    ## Parent Loop BB0_9 Depth=1
+; CHECK-NEXT:    ## Parent Loop BB0_10 Depth=2
+; CHECK-NEXT:    ## => This Inner Loop Header: Depth=3
+; CHECK-NEXT:    cmpb $120, 1000(%rdx,%rbp)
+; CHECK-NEXT:    je LBB0_14
+; CHECK-NEXT:  ## %bb.13: ## %do.cond.i
+; CHECK-NEXT:    ## in Loop: Header=BB0_12 Depth=3
+; CHECK-NEXT:    incq %rbp
+; CHECK-NEXT:    jne LBB0_12
+; CHECK-NEXT:    jmp LBB0_5
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  LBB0_14: ## %exit
+; CHECK-NEXT:    ## in Loop: Header=BB0_10 Depth=2
+; CHECK-NEXT:    addq %rsi, %rbp
+; CHECK-NEXT:    incq %rdi
+; CHECK-NEXT:    decq %rsi
+; CHECK-NEXT:    addq $1001, %rdx ## imm = 0x3E9
+; CHECK-NEXT:    cmpq $-1000, %rbp ## imm = 0xFC18
+; CHECK-NEXT:    je LBB0_10
+; CHECK-NEXT:  LBB0_5: ## %if.then
+; CHECK-NEXT:    leaq {{.*}}(%rip), %rdi
+; CHECK-NEXT:    callq _puts
+; CHECK-NEXT:    movl $1, %edi
+; CHECK-NEXT:    callq _exit
+; CHECK-NEXT:  LBB0_16: ## %for.end40
+; CHECK-NEXT:    leaq {{.*}}(%rip), %rdi
+; CHECK-NEXT:    callq _puts
+; CHECK-NEXT:    movq ___stack_chk_guard@{{.*}}(%rip), %rax
+; CHECK-NEXT:    movq (%rax), %rax
+; CHECK-NEXT:    cmpq {{[0-9]+}}(%rsp), %rax
+; CHECK-NEXT:    jne LBB0_18
+; CHECK-NEXT:  ## %bb.17: ## %for.end40
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    addq $1001016, %rsp ## imm = 0xF4638
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    popq %r12
+; CHECK-NEXT:    popq %r13
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    retq
+; CHECK-NEXT:  LBB0_18: ## %for.end40
+; CHECK-NEXT:    callq ___stack_chk_fail
 entry:
   %strs = alloca [1000 x [1001 x i8]], align 16
   br label %for.cond
diff --git a/test/CodeGen/X86/rot16.ll b/test/CodeGen/X86/rot16.ll
index 69ea7013cc78ef90d0fa6ffdc8bd3aa25b760d2a..b0221c48532a51493a1e0005d6fba0f402992131 100644
--- a/test/CodeGen/X86/rot16.ll
+++ b/test/CodeGen/X86/rot16.ll
@@ -28,9 +28,10 @@ define i16 @foo(i16 %x, i16 %y, i16 %z) nounwind {
 define i16 @bar(i16 %x, i16 %y, i16 %z) nounwind {
 ; X32-LABEL: bar:
 ; X32:       # %bb.0:
-; X32-NEXT:    movb {{[0-9]+}}(%esp), %cl
 ; X32-NEXT:    movzwl {{[0-9]+}}(%esp), %edx
 ; X32-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X32-NEXT:    movb {{[0-9]+}}(%esp), %cl
+; X32-NEXT:    andb $15, %cl
 ; X32-NEXT:    shldw %cl, %dx, %ax
 ; X32-NEXT:    retl
 ;
@@ -38,6 +39,7 @@ define i16 @bar(i16 %x, i16 %y, i16 %z) nounwind {
 ; X64:       # %bb.0:
 ; X64-NEXT:    movl %edx, %ecx
 ; X64-NEXT:    movl %esi, %eax
+; X64-NEXT:    andb $15, %cl
 ; X64-NEXT:    # kill: def $cl killed $cl killed $ecx
 ; X64-NEXT:    shldw %cl, %di, %ax
 ; X64-NEXT:    # kill: def $ax killed $ax killed $eax
@@ -75,9 +77,10 @@ define i16 @un(i16 %x, i16 %y, i16 %z) nounwind {
 define i16 @bu(i16 %x, i16 %y, i16 %z) nounwind {
 ; X32-LABEL: bu:
 ; X32:       # %bb.0:
-; X32-NEXT:    movb {{[0-9]+}}(%esp), %cl
 ; X32-NEXT:    movzwl {{[0-9]+}}(%esp), %edx
 ; X32-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X32-NEXT:    movb {{[0-9]+}}(%esp), %cl
+; X32-NEXT:    andb $15, %cl
 ; X32-NEXT:    shrdw %cl, %dx, %ax
 ; X32-NEXT:    retl
 ;
@@ -85,6 +88,7 @@ define i16 @bu(i16 %x, i16 %y, i16 %z) nounwind {
 ; X64:       # %bb.0:
 ; X64-NEXT:    movl %edx, %ecx
 ; X64-NEXT:    movl %esi, %eax
+; X64-NEXT:    andb $15, %cl
 ; X64-NEXT:    # kill: def $cl killed $cl killed $ecx
 ; X64-NEXT:    shrdw %cl, %di, %ax
 ; X64-NEXT:    # kill: def $ax killed $ax killed $eax
diff --git a/test/CodeGen/X86/rtm-schedule.ll b/test/CodeGen/X86/rtm-schedule.ll
deleted file mode 100644
index 1a724f5799e4aa0180312990135de61489da2234..0000000000000000000000000000000000000000
--- a/test/CodeGen/X86/rtm-schedule.ll
+++ /dev/null
@@ -1,62 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mattr=+rtm -mcpu=x86-64 -mattr=+rtm | FileCheck %s --check-prefix=CHECK --check-prefix=GENERIC
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mattr=+rtm -mcpu=skylake | FileCheck %s --check-prefix=CHECK --check-prefix=SKYLAKE --check-prefix=SKL
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mattr=+rtm -mcpu=skx | FileCheck %s --check-prefix=CHECK --check-prefix=SKYLAKE --check-prefix=SKX
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mattr=+rtm -mcpu=cannonlake | FileCheck %s --check-prefix=CHECK --check-prefix=SKYLAKE --check-prefix=CNL
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mattr=+rtm -mcpu=icelake-client | FileCheck %s --check-prefix=CHECK --check-prefix=SKYLAKE --check-prefix=ICL
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mattr=+rtm -mcpu=icelake-server | FileCheck %s --check-prefix=CHECK --check-prefix=SKYLAKE --check-prefix=ICL
-
-define i32 @test_xbegin() nounwind uwtable {
-; GENERIC-LABEL: test_xbegin:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    xbegin .LBB0_2 # sched: [100:0.33]
-; GENERIC-NEXT:  # %bb.1:
-; GENERIC-NEXT:    movl $-1, %eax # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-; GENERIC-NEXT:  .LBB0_2:
-; GENERIC-NEXT:    # XABORT DEF # sched: [100:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKYLAKE-LABEL: test_xbegin:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    xbegin .LBB0_2 # sched: [100:0.25]
-; SKYLAKE-NEXT:  # %bb.1:
-; SKYLAKE-NEXT:    movl $-1, %eax # sched: [1:0.25]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-; SKYLAKE-NEXT:  .LBB0_2:
-; SKYLAKE-NEXT:    # XABORT DEF # sched: [100:0.25]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-  %1 = tail call i32 @llvm.x86.xbegin() nounwind
-  ret i32 %1
-}
-declare i32 @llvm.x86.xbegin() nounwind
-
-define void @test_xend() nounwind uwtable {
-; GENERIC-LABEL: test_xend:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    xend # sched: [100:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKYLAKE-LABEL: test_xend:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    xend # sched: [100:0.25]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-  tail call void @llvm.x86.xend() nounwind
-  ret void
-}
-declare void @llvm.x86.xend() nounwind
-
-define void @test_xabort() nounwind uwtable {
-; GENERIC-LABEL: test_xabort:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    xabort $2 # sched: [100:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SKYLAKE-LABEL: test_xabort:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    xabort $2 # sched: [100:0.25]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-  tail call void @llvm.x86.xabort(i8 2)
-  ret void
-}
-declare void @llvm.x86.xabort(i8) nounwind
diff --git a/test/CodeGen/X86/sad.ll b/test/CodeGen/X86/sad.ll
index f64364340658cf0a3ca19fb4b54964f9c90748a2..2a8e62b05aad342cb76d7a5a01603a35716289c9 100644
--- a/test/CodeGen/X86/sad.ll
+++ b/test/CodeGen/X86/sad.ll
@@ -78,9 +78,9 @@ define i32 @sad_16i8() nounwind {
 ; AVX2-NEXT:  # %bb.2: # %middle.block
 ; AVX2-NEXT:    vpaddd %ymm0, %ymm1, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpaddd %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX2-NEXT:    vpaddd %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
@@ -103,9 +103,9 @@ define i32 @sad_16i8() nounwind {
 ; AVX512-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
@@ -339,9 +339,9 @@ define i32 @sad_32i8() nounwind {
 ; AVX2-NEXT:    vpaddd %ymm0, %ymm0, %ymm0
 ; AVX2-NEXT:    vpaddd %ymm0, %ymm1, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpaddd %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX2-NEXT:    vpaddd %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
@@ -366,9 +366,9 @@ define i32 @sad_32i8() nounwind {
 ; AVX512-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
@@ -933,9 +933,9 @@ define i32 @sad_avx64i8() nounwind {
 ; AVX2-NEXT:    vpaddd %ymm2, %ymm1, %ymm1
 ; AVX2-NEXT:    vpaddd %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpaddd %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX2-NEXT:    vpaddd %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
@@ -981,9 +981,9 @@ define i32 @sad_avx64i8() nounwind {
 ; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512F-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
 ; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512F-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
+; AVX512F-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX512F-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512F-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
+; AVX512F-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX512F-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512F-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX512F-NEXT:    vmovd %xmm0, %eax
@@ -1010,9 +1010,9 @@ define i32 @sad_avx64i8() nounwind {
 ; AVX512BW-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512BW-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512BW-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512BW-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512BW-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX512BW-NEXT:    vmovd %xmm0, %eax
@@ -1355,7 +1355,7 @@ define i32 @sad_nonloop_32i8(<32 x i8>* nocapture readonly %p, i64, <32 x i8>* n
 ; AVX2-NEXT:    vmovdqu (%rdi), %ymm0
 ; AVX2-NEXT:    vpsadbw (%rdx), %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpaddq %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; AVX2-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
@@ -1367,7 +1367,7 @@ define i32 @sad_nonloop_32i8(<32 x i8>* nocapture readonly %p, i64, <32 x i8>* n
 ; AVX512-NEXT:    vmovdqu (%rdi), %ymm0
 ; AVX512-NEXT:    vpsadbw (%rdx), %ymm0, %ymm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vpaddq %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; AVX512-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
@@ -1445,9 +1445,9 @@ define i32 @sad_unroll_nonzero_initial(<16 x i8>* %arg, <16 x i8>* %arg1, <16 x
 ; AVX2-NEXT:    vpsadbw (%rcx), %xmm1, %xmm1
 ; AVX2-NEXT:    vpaddd %ymm0, %ymm1, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpaddd %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX2-NEXT:    vpaddd %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
@@ -1467,9 +1467,9 @@ define i32 @sad_unroll_nonzero_initial(<16 x i8>* %arg, <16 x i8>* %arg1, <16 x
 ; AVX512-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
@@ -1547,9 +1547,9 @@ define i32 @sad_double_reduction(<16 x i8>* %arg, <16 x i8>* %arg1, <16 x i8>* %
 ; AVX2-NEXT:    vpsadbw (%rcx), %xmm1, %xmm1
 ; AVX2-NEXT:    vpaddd %ymm0, %ymm1, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpaddd %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX2-NEXT:    vpaddd %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
@@ -1566,9 +1566,9 @@ define i32 @sad_double_reduction(<16 x i8>* %arg, <16 x i8>* %arg1, <16 x i8>* %
 ; AVX512-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
diff --git a/test/CodeGen/X86/sadd_sat.ll b/test/CodeGen/X86/sadd_sat.ll
index 39788e86cc7b17165fed06f39cefb618c70ca39c..b5524e1db8b1cecda2602ccade4c90a987cc5096 100644
--- a/test/CodeGen/X86/sadd_sat.ll
+++ b/test/CodeGen/X86/sadd_sat.ll
@@ -1,267 +1,213 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mcpu=generic -mtriple=x86_64-linux | FileCheck %s
-; RUN: llc < %s -mcpu=generic -mtriple=i686 -mattr=cmov | FileCheck %s --check-prefix=CHECK32
+; RUN: llc < %s -mtriple=i686 -mattr=cmov | FileCheck %s --check-prefixes=CHECK,X86
+; RUN: llc < %s -mtriple=x86_64-linux | FileCheck %s --check-prefixes=CHECK,X64
 
 declare  i4  @llvm.sadd.sat.i4   (i4,  i4)
 declare  i32 @llvm.sadd.sat.i32  (i32, i32)
 declare  i64 @llvm.sadd.sat.i64  (i64, i64)
 declare  <4 x i32> @llvm.sadd.sat.v4i32(<4 x i32>, <4 x i32>)
 
-define i32 @func(i32 %x, i32 %y) {
-; CHECK-LABEL: func:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    xorl %eax, %eax
-; CHECK-NEXT:    movl %edi, %ecx
-; CHECK-NEXT:    addl %esi, %ecx
-; CHECK-NEXT:    setns %al
-; CHECK-NEXT:    addl $2147483647, %eax # imm = 0x7FFFFFFF
-; CHECK-NEXT:    addl %esi, %edi
-; CHECK-NEXT:    cmovnol %edi, %eax
-; CHECK-NEXT:    retq
+define i32 @func(i32 %x, i32 %y) nounwind {
+; X86-LABEL: func:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    xorl %ecx, %ecx
+; X86-NEXT:    movl %eax, %esi
+; X86-NEXT:    addl %edx, %esi
+; X86-NEXT:    setns %cl
+; X86-NEXT:    addl $2147483647, %ecx # imm = 0x7FFFFFFF
+; X86-NEXT:    addl %edx, %eax
+; X86-NEXT:    cmovol %ecx, %eax
+; X86-NEXT:    popl %esi
+; X86-NEXT:    retl
 ;
-; CHECK32-LABEL: func:
-; CHECK32:       # %bb.0:
-; CHECK32-NEXT:    pushl %esi
-; CHECK32-NEXT:    .cfi_def_cfa_offset 8
-; CHECK32-NEXT:    .cfi_offset %esi, -8
-; CHECK32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; CHECK32-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; CHECK32-NEXT:    xorl %ecx, %ecx
-; CHECK32-NEXT:    movl %eax, %esi
-; CHECK32-NEXT:    addl %edx, %esi
-; CHECK32-NEXT:    setns %cl
-; CHECK32-NEXT:    addl $2147483647, %ecx # imm = 0x7FFFFFFF
-; CHECK32-NEXT:    addl %edx, %eax
-; CHECK32-NEXT:    cmovol %ecx, %eax
-; CHECK32-NEXT:    popl %esi
-; CHECK32-NEXT:    .cfi_def_cfa_offset 4
-; CHECK32-NEXT:    retl
+; X64-LABEL: func:
+; X64:       # %bb.0:
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    movl %edi, %ecx
+; X64-NEXT:    addl %esi, %ecx
+; X64-NEXT:    setns %al
+; X64-NEXT:    addl $2147483647, %eax # imm = 0x7FFFFFFF
+; X64-NEXT:    addl %esi, %edi
+; X64-NEXT:    cmovnol %edi, %eax
+; X64-NEXT:    retq
   %tmp = call i32 @llvm.sadd.sat.i32(i32 %x, i32 %y);
   ret i32 %tmp;
 }
 
-define i64 @func2(i64 %x, i64 %y) {
-; CHECK-LABEL: func2:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    xorl %ecx, %ecx
-; CHECK-NEXT:    movq %rdi, %rax
-; CHECK-NEXT:    addq %rsi, %rax
-; CHECK-NEXT:    setns %cl
-; CHECK-NEXT:    movabsq $9223372036854775807, %rax # imm = 0x7FFFFFFFFFFFFFFF
-; CHECK-NEXT:    addq %rcx, %rax
-; CHECK-NEXT:    addq %rsi, %rdi
-; CHECK-NEXT:    cmovnoq %rdi, %rax
-; CHECK-NEXT:    retq
+define i64 @func2(i64 %x, i64 %y) nounwind {
+; X86-LABEL: func2:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %ebp
+; X86-NEXT:    pushl %ebx
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ebx
+; X86-NEXT:    addl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    movl %ebx, %ebp
+; X86-NEXT:    adcl %esi, %ebp
+; X86-NEXT:    movl %ebp, %eax
+; X86-NEXT:    sarl $31, %eax
+; X86-NEXT:    xorl %ecx, %ecx
+; X86-NEXT:    testl %ebp, %ebp
+; X86-NEXT:    setns %cl
+; X86-NEXT:    movl %ecx, %edx
+; X86-NEXT:    addl $2147483647, %edx # imm = 0x7FFFFFFF
+; X86-NEXT:    testl %ebx, %ebx
+; X86-NEXT:    setns %bl
+; X86-NEXT:    cmpb %cl, %bl
+; X86-NEXT:    setne %cl
+; X86-NEXT:    testl %esi, %esi
+; X86-NEXT:    setns %ch
+; X86-NEXT:    cmpb %ch, %bl
+; X86-NEXT:    sete %ch
+; X86-NEXT:    testb %cl, %ch
+; X86-NEXT:    cmovel %ebp, %edx
+; X86-NEXT:    cmovel %edi, %eax
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    popl %ebx
+; X86-NEXT:    popl %ebp
+; X86-NEXT:    retl
 ;
-; CHECK32-LABEL: func2:
-; CHECK32:       # %bb.0:
-; CHECK32-NEXT:    pushl %ebp
-; CHECK32-NEXT:    .cfi_def_cfa_offset 8
-; CHECK32-NEXT:    pushl %ebx
-; CHECK32-NEXT:    .cfi_def_cfa_offset 12
-; CHECK32-NEXT:    pushl %edi
-; CHECK32-NEXT:    .cfi_def_cfa_offset 16
-; CHECK32-NEXT:    pushl %esi
-; CHECK32-NEXT:    .cfi_def_cfa_offset 20
-; CHECK32-NEXT:    .cfi_offset %esi, -20
-; CHECK32-NEXT:    .cfi_offset %edi, -16
-; CHECK32-NEXT:    .cfi_offset %ebx, -12
-; CHECK32-NEXT:    .cfi_offset %ebp, -8
-; CHECK32-NEXT:    movl {{[0-9]+}}(%esp), %esi
-; CHECK32-NEXT:    movl {{[0-9]+}}(%esp), %edi
-; CHECK32-NEXT:    movl {{[0-9]+}}(%esp), %ebx
-; CHECK32-NEXT:    addl {{[0-9]+}}(%esp), %edi
-; CHECK32-NEXT:    movl %ebx, %ebp
-; CHECK32-NEXT:    adcl %esi, %ebp
-; CHECK32-NEXT:    movl %ebp, %eax
-; CHECK32-NEXT:    sarl $31, %eax
-; CHECK32-NEXT:    xorl %ecx, %ecx
-; CHECK32-NEXT:    testl %ebp, %ebp
-; CHECK32-NEXT:    setns %cl
-; CHECK32-NEXT:    movl %ecx, %edx
-; CHECK32-NEXT:    addl $2147483647, %edx # imm = 0x7FFFFFFF
-; CHECK32-NEXT:    testl %ebx, %ebx
-; CHECK32-NEXT:    setns %bl
-; CHECK32-NEXT:    cmpb %cl, %bl
-; CHECK32-NEXT:    setne %cl
-; CHECK32-NEXT:    testl %esi, %esi
-; CHECK32-NEXT:    setns %ch
-; CHECK32-NEXT:    cmpb %ch, %bl
-; CHECK32-NEXT:    sete %ch
-; CHECK32-NEXT:    testb %cl, %ch
-; CHECK32-NEXT:    cmovel %ebp, %edx
-; CHECK32-NEXT:    cmovel %edi, %eax
-; CHECK32-NEXT:    popl %esi
-; CHECK32-NEXT:    .cfi_def_cfa_offset 16
-; CHECK32-NEXT:    popl %edi
-; CHECK32-NEXT:    .cfi_def_cfa_offset 12
-; CHECK32-NEXT:    popl %ebx
-; CHECK32-NEXT:    .cfi_def_cfa_offset 8
-; CHECK32-NEXT:    popl %ebp
-; CHECK32-NEXT:    .cfi_def_cfa_offset 4
-; CHECK32-NEXT:    retl
+; X64-LABEL: func2:
+; X64:       # %bb.0:
+; X64-NEXT:    xorl %ecx, %ecx
+; X64-NEXT:    movq %rdi, %rax
+; X64-NEXT:    addq %rsi, %rax
+; X64-NEXT:    setns %cl
+; X64-NEXT:    movabsq $9223372036854775807, %rax # imm = 0x7FFFFFFFFFFFFFFF
+; X64-NEXT:    addq %rcx, %rax
+; X64-NEXT:    addq %rsi, %rdi
+; X64-NEXT:    cmovnoq %rdi, %rax
+; X64-NEXT:    retq
   %tmp = call i64 @llvm.sadd.sat.i64(i64 %x, i64 %y);
   ret i64 %tmp;
 }
 
-define i4 @func3(i4 %x, i4 %y) {
-; CHECK-LABEL: func3:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    movl %edi, %eax
-; CHECK-NEXT:    shlb $4, %sil
-; CHECK-NEXT:    shlb $4, %al
-; CHECK-NEXT:    movl %eax, %ecx
-; CHECK-NEXT:    addb %sil, %cl
-; CHECK-NEXT:    setns %cl
-; CHECK-NEXT:    addb %sil, %al
-; CHECK-NEXT:    jno .LBB2_2
-; CHECK-NEXT:  # %bb.1:
-; CHECK-NEXT:    addb $127, %cl
-; CHECK-NEXT:    movl %ecx, %eax
-; CHECK-NEXT:  .LBB2_2:
-; CHECK-NEXT:    sarb $4, %al
-; CHECK-NEXT:    # kill: def $al killed $al killed $eax
-; CHECK-NEXT:    retq
+define i4 @func3(i4 %x, i4 %y) nounwind {
+; X86-LABEL: func3:
+; X86:       # %bb.0:
+; X86-NEXT:    movb {{[0-9]+}}(%esp), %al
+; X86-NEXT:    movb {{[0-9]+}}(%esp), %dl
+; X86-NEXT:    shlb $4, %dl
+; X86-NEXT:    shlb $4, %al
+; X86-NEXT:    movl %eax, %ecx
+; X86-NEXT:    addb %dl, %cl
+; X86-NEXT:    setns %cl
+; X86-NEXT:    addb %dl, %al
+; X86-NEXT:    jno .LBB2_2
+; X86-NEXT:  # %bb.1:
+; X86-NEXT:    addb $127, %cl
+; X86-NEXT:    movl %ecx, %eax
+; X86-NEXT:  .LBB2_2:
+; X86-NEXT:    sarb $4, %al
+; X86-NEXT:    retl
 ;
-; CHECK32-LABEL: func3:
-; CHECK32:       # %bb.0:
-; CHECK32-NEXT:    movb {{[0-9]+}}(%esp), %al
-; CHECK32-NEXT:    movb {{[0-9]+}}(%esp), %dl
-; CHECK32-NEXT:    shlb $4, %dl
-; CHECK32-NEXT:    shlb $4, %al
-; CHECK32-NEXT:    movl %eax, %ecx
-; CHECK32-NEXT:    addb %dl, %cl
-; CHECK32-NEXT:    setns %cl
-; CHECK32-NEXT:    addb %dl, %al
-; CHECK32-NEXT:    jno .LBB2_2
-; CHECK32-NEXT:  # %bb.1:
-; CHECK32-NEXT:    addb $127, %cl
-; CHECK32-NEXT:    movl %ecx, %eax
-; CHECK32-NEXT:  .LBB2_2:
-; CHECK32-NEXT:    sarb $4, %al
-; CHECK32-NEXT:    retl
+; X64-LABEL: func3:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    shlb $4, %sil
+; X64-NEXT:    shlb $4, %al
+; X64-NEXT:    movl %eax, %ecx
+; X64-NEXT:    addb %sil, %cl
+; X64-NEXT:    setns %cl
+; X64-NEXT:    addb %sil, %al
+; X64-NEXT:    jno .LBB2_2
+; X64-NEXT:  # %bb.1:
+; X64-NEXT:    addb $127, %cl
+; X64-NEXT:    movl %ecx, %eax
+; X64-NEXT:  .LBB2_2:
+; X64-NEXT:    sarb $4, %al
+; X64-NEXT:    # kill: def $al killed $al killed $eax
+; X64-NEXT:    retq
   %tmp = call i4 @llvm.sadd.sat.i4(i4 %x, i4 %y);
   ret i4 %tmp;
 }
 
-define <4 x i32> @vec(<4 x i32> %x, <4 x i32> %y) {
-; CHECK-LABEL: vec:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,2,3]
-; CHECK-NEXT:    movd %xmm2, %ecx
-; CHECK-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,2,3]
-; CHECK-NEXT:    movd %xmm2, %r8d
-; CHECK-NEXT:    xorl %edx, %edx
-; CHECK-NEXT:    movl %r8d, %esi
-; CHECK-NEXT:    addl %ecx, %esi
-; CHECK-NEXT:    setns %dl
-; CHECK-NEXT:    addl $2147483647, %edx # imm = 0x7FFFFFFF
-; CHECK-NEXT:    addl %ecx, %r8d
-; CHECK-NEXT:    cmovol %edx, %r8d
-; CHECK-NEXT:    movd %xmm1, %edx
-; CHECK-NEXT:    movd %xmm0, %ecx
-; CHECK-NEXT:    xorl %esi, %esi
-; CHECK-NEXT:    movl %ecx, %edi
-; CHECK-NEXT:    addl %edx, %edi
-; CHECK-NEXT:    setns %sil
-; CHECK-NEXT:    addl $2147483647, %esi # imm = 0x7FFFFFFF
-; CHECK-NEXT:    addl %edx, %ecx
-; CHECK-NEXT:    cmovol %esi, %ecx
-; CHECK-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,0,1]
-; CHECK-NEXT:    movd %xmm2, %edx
-; CHECK-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
-; CHECK-NEXT:    movd %xmm2, %eax
-; CHECK-NEXT:    xorl %edi, %edi
-; CHECK-NEXT:    movl %eax, %esi
-; CHECK-NEXT:    addl %edx, %esi
-; CHECK-NEXT:    setns %dil
-; CHECK-NEXT:    addl $2147483647, %edi # imm = 0x7FFFFFFF
-; CHECK-NEXT:    addl %edx, %eax
-; CHECK-NEXT:    cmovol %edi, %eax
-; CHECK-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[3,1,2,3]
-; CHECK-NEXT:    movd %xmm1, %r9d
-; CHECK-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
-; CHECK-NEXT:    movd %xmm0, %edx
-; CHECK-NEXT:    xorl %edi, %edi
-; CHECK-NEXT:    movl %edx, %esi
-; CHECK-NEXT:    addl %r9d, %esi
-; CHECK-NEXT:    setns %dil
-; CHECK-NEXT:    addl $2147483647, %edi # imm = 0x7FFFFFFF
-; CHECK-NEXT:    addl %r9d, %edx
-; CHECK-NEXT:    cmovol %edi, %edx
-; CHECK-NEXT:    movd %edx, %xmm0
-; CHECK-NEXT:    movd %eax, %xmm1
-; CHECK-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
-; CHECK-NEXT:    movd %ecx, %xmm0
-; CHECK-NEXT:    movd %r8d, %xmm2
-; CHECK-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
-; CHECK-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; CHECK-NEXT:    retq
+define <4 x i32> @vec(<4 x i32> %x, <4 x i32> %y) nounwind {
+; X86-LABEL: vec:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %ebp
+; X86-NEXT:    pushl %ebx
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    xorl %eax, %eax
+; X86-NEXT:    movl %ecx, %esi
+; X86-NEXT:    addl %edx, %esi
+; X86-NEXT:    setns %al
+; X86-NEXT:    addl $2147483647, %eax # imm = 0x7FFFFFFF
+; X86-NEXT:    addl %edx, %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    cmovol %eax, %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    xorl %eax, %eax
+; X86-NEXT:    movl %edx, %edi
+; X86-NEXT:    addl %esi, %edi
+; X86-NEXT:    setns %al
+; X86-NEXT:    addl $2147483647, %eax # imm = 0x7FFFFFFF
+; X86-NEXT:    addl %esi, %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    cmovol %eax, %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    xorl %eax, %eax
+; X86-NEXT:    movl %esi, %ebx
+; X86-NEXT:    addl %edi, %ebx
+; X86-NEXT:    setns %al
+; X86-NEXT:    addl $2147483647, %eax # imm = 0x7FFFFFFF
+; X86-NEXT:    addl %edi, %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    cmovol %eax, %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    xorl %ebx, %ebx
+; X86-NEXT:    movl %edi, %ebp
+; X86-NEXT:    addl %eax, %ebp
+; X86-NEXT:    setns %bl
+; X86-NEXT:    addl $2147483647, %ebx # imm = 0x7FFFFFFF
+; X86-NEXT:    addl %eax, %edi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmovol %ebx, %edi
+; X86-NEXT:    movl %ecx, 12(%eax)
+; X86-NEXT:    movl %edx, 8(%eax)
+; X86-NEXT:    movl %esi, 4(%eax)
+; X86-NEXT:    movl %edi, (%eax)
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    popl %ebx
+; X86-NEXT:    popl %ebp
+; X86-NEXT:    retl $4
 ;
-; CHECK32-LABEL: vec:
-; CHECK32:       # %bb.0:
-; CHECK32-NEXT:    pushl %ebp
-; CHECK32-NEXT:    .cfi_def_cfa_offset 8
-; CHECK32-NEXT:    pushl %ebx
-; CHECK32-NEXT:    .cfi_def_cfa_offset 12
-; CHECK32-NEXT:    pushl %edi
-; CHECK32-NEXT:    .cfi_def_cfa_offset 16
-; CHECK32-NEXT:    pushl %esi
-; CHECK32-NEXT:    .cfi_def_cfa_offset 20
-; CHECK32-NEXT:    .cfi_offset %esi, -20
-; CHECK32-NEXT:    .cfi_offset %edi, -16
-; CHECK32-NEXT:    .cfi_offset %ebx, -12
-; CHECK32-NEXT:    .cfi_offset %ebp, -8
-; CHECK32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; CHECK32-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; CHECK32-NEXT:    xorl %eax, %eax
-; CHECK32-NEXT:    movl %ecx, %esi
-; CHECK32-NEXT:    addl %edx, %esi
-; CHECK32-NEXT:    setns %al
-; CHECK32-NEXT:    addl $2147483647, %eax # imm = 0x7FFFFFFF
-; CHECK32-NEXT:    addl %edx, %ecx
-; CHECK32-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; CHECK32-NEXT:    cmovol %eax, %ecx
-; CHECK32-NEXT:    movl {{[0-9]+}}(%esp), %esi
-; CHECK32-NEXT:    xorl %eax, %eax
-; CHECK32-NEXT:    movl %edx, %edi
-; CHECK32-NEXT:    addl %esi, %edi
-; CHECK32-NEXT:    setns %al
-; CHECK32-NEXT:    addl $2147483647, %eax # imm = 0x7FFFFFFF
-; CHECK32-NEXT:    addl %esi, %edx
-; CHECK32-NEXT:    movl {{[0-9]+}}(%esp), %esi
-; CHECK32-NEXT:    cmovol %eax, %edx
-; CHECK32-NEXT:    movl {{[0-9]+}}(%esp), %edi
-; CHECK32-NEXT:    xorl %eax, %eax
-; CHECK32-NEXT:    movl %esi, %ebx
-; CHECK32-NEXT:    addl %edi, %ebx
-; CHECK32-NEXT:    setns %al
-; CHECK32-NEXT:    addl $2147483647, %eax # imm = 0x7FFFFFFF
-; CHECK32-NEXT:    addl %edi, %esi
-; CHECK32-NEXT:    movl {{[0-9]+}}(%esp), %edi
-; CHECK32-NEXT:    cmovol %eax, %esi
-; CHECK32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; CHECK32-NEXT:    xorl %ebx, %ebx
-; CHECK32-NEXT:    movl %edi, %ebp
-; CHECK32-NEXT:    addl %eax, %ebp
-; CHECK32-NEXT:    setns %bl
-; CHECK32-NEXT:    addl $2147483647, %ebx # imm = 0x7FFFFFFF
-; CHECK32-NEXT:    addl %eax, %edi
-; CHECK32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; CHECK32-NEXT:    cmovol %ebx, %edi
-; CHECK32-NEXT:    movl %ecx, 12(%eax)
-; CHECK32-NEXT:    movl %edx, 8(%eax)
-; CHECK32-NEXT:    movl %esi, 4(%eax)
-; CHECK32-NEXT:    movl %edi, (%eax)
-; CHECK32-NEXT:    popl %esi
-; CHECK32-NEXT:    .cfi_def_cfa_offset 16
-; CHECK32-NEXT:    popl %edi
-; CHECK32-NEXT:    .cfi_def_cfa_offset 12
-; CHECK32-NEXT:    popl %ebx
-; CHECK32-NEXT:    .cfi_def_cfa_offset 8
-; CHECK32-NEXT:    popl %ebp
-; CHECK32-NEXT:    .cfi_def_cfa_offset 4
-; CHECK32-NEXT:    retl $4
+; X64-LABEL: vec:
+; X64:       # %bb.0:
+; X64-NEXT:    pxor %xmm3, %xmm3
+; X64-NEXT:    pxor %xmm4, %xmm4
+; X64-NEXT:    pcmpgtd %xmm1, %xmm4
+; X64-NEXT:    pcmpeqd %xmm2, %xmm2
+; X64-NEXT:    pxor %xmm2, %xmm4
+; X64-NEXT:    pxor %xmm5, %xmm5
+; X64-NEXT:    pcmpgtd %xmm0, %xmm5
+; X64-NEXT:    pxor %xmm2, %xmm5
+; X64-NEXT:    pcmpeqd %xmm5, %xmm4
+; X64-NEXT:    paddd %xmm1, %xmm0
+; X64-NEXT:    pcmpgtd %xmm0, %xmm3
+; X64-NEXT:    pxor %xmm3, %xmm2
+; X64-NEXT:    pcmpeqd %xmm5, %xmm2
+; X64-NEXT:    pandn %xmm4, %xmm2
+; X64-NEXT:    movdqa %xmm3, %xmm1
+; X64-NEXT:    pandn {{.*}}(%rip), %xmm1
+; X64-NEXT:    psrld $1, %xmm3
+; X64-NEXT:    por %xmm1, %xmm3
+; X64-NEXT:    pand %xmm2, %xmm3
+; X64-NEXT:    pandn %xmm0, %xmm2
+; X64-NEXT:    por %xmm3, %xmm2
+; X64-NEXT:    movdqa %xmm2, %xmm0
+; X64-NEXT:    retq
   %tmp = call <4 x i32> @llvm.sadd.sat.v4i32(<4 x i32> %x, <4 x i32> %y);
   ret <4 x i32> %tmp;
 }
diff --git a/test/CodeGen/X86/sadd_sat_vec.ll b/test/CodeGen/X86/sadd_sat_vec.ll
index 421dc6c29f52046591799bfc4d9e3b5abc45efe6..320cc076e5f1bca36df388909170439fdaa886df 100644
--- a/test/CodeGen/X86/sadd_sat_vec.ll
+++ b/test/CodeGen/X86/sadd_sat_vec.ll
@@ -26,8 +26,14 @@ declare <32 x i16> @llvm.sadd.sat.v32i16(<32 x i16>, <32 x i16>)
 declare <16 x i1> @llvm.sadd.sat.v16i1(<16 x i1>, <16 x i1>)
 declare <16 x i4> @llvm.sadd.sat.v16i4(<16 x i4>, <16 x i4>)
 
-declare <4 x i32> @llvm.sadd.sat.v4i32(<4 x i32>, <4 x i32>)
 declare <2 x i32> @llvm.sadd.sat.v2i32(<2 x i32>, <2 x i32>)
+declare <4 x i32> @llvm.sadd.sat.v4i32(<4 x i32>, <4 x i32>)
+declare <8 x i32> @llvm.sadd.sat.v8i32(<8 x i32>, <8 x i32>)
+declare <16 x i32> @llvm.sadd.sat.v16i32(<16 x i32>, <16 x i32>)
+declare <2 x i64> @llvm.sadd.sat.v2i64(<2 x i64>, <2 x i64>)
+declare <4 x i64> @llvm.sadd.sat.v4i64(<4 x i64>, <4 x i64>)
+declare <8 x i64> @llvm.sadd.sat.v8i64(<8 x i64>, <8 x i64>)
+
 declare <4 x i24> @llvm.sadd.sat.v4i24(<4 x i24>, <4 x i24>)
 declare <2 x i128> @llvm.sadd.sat.v2i128(<2 x i128>, <2 x i128>)
 
@@ -651,330 +657,226 @@ define <16 x i1> @v16i1(<16 x i1> %x, <16 x i1> %y) nounwind {
 
 ; Expanded
 
-define <4 x i32> @v4i32(<4 x i32> %x, <4 x i32> %y) nounwind {
-; SSE2-LABEL: v4i32:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,2,3]
-; SSE2-NEXT:    movd %xmm2, %ecx
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,2,3]
-; SSE2-NEXT:    movd %xmm2, %r8d
-; SSE2-NEXT:    xorl %edx, %edx
-; SSE2-NEXT:    movl %r8d, %esi
-; SSE2-NEXT:    addl %ecx, %esi
-; SSE2-NEXT:    setns %dl
-; SSE2-NEXT:    addl $2147483647, %edx # imm = 0x7FFFFFFF
-; SSE2-NEXT:    addl %ecx, %r8d
-; SSE2-NEXT:    cmovol %edx, %r8d
-; SSE2-NEXT:    movd %xmm1, %edx
-; SSE2-NEXT:    movd %xmm0, %ecx
-; SSE2-NEXT:    xorl %esi, %esi
-; SSE2-NEXT:    movl %ecx, %edi
-; SSE2-NEXT:    addl %edx, %edi
-; SSE2-NEXT:    setns %sil
-; SSE2-NEXT:    addl $2147483647, %esi # imm = 0x7FFFFFFF
-; SSE2-NEXT:    addl %edx, %ecx
-; SSE2-NEXT:    cmovol %esi, %ecx
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,0,1]
-; SSE2-NEXT:    movd %xmm2, %edx
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
-; SSE2-NEXT:    movd %xmm2, %eax
-; SSE2-NEXT:    xorl %edi, %edi
-; SSE2-NEXT:    movl %eax, %esi
-; SSE2-NEXT:    addl %edx, %esi
-; SSE2-NEXT:    setns %dil
-; SSE2-NEXT:    addl $2147483647, %edi # imm = 0x7FFFFFFF
-; SSE2-NEXT:    addl %edx, %eax
-; SSE2-NEXT:    cmovol %edi, %eax
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[3,1,2,3]
-; SSE2-NEXT:    movd %xmm1, %r9d
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
-; SSE2-NEXT:    movd %xmm0, %edx
-; SSE2-NEXT:    xorl %edi, %edi
-; SSE2-NEXT:    movl %edx, %esi
-; SSE2-NEXT:    addl %r9d, %esi
-; SSE2-NEXT:    setns %dil
-; SSE2-NEXT:    addl $2147483647, %edi # imm = 0x7FFFFFFF
-; SSE2-NEXT:    addl %r9d, %edx
-; SSE2-NEXT:    cmovol %edi, %edx
-; SSE2-NEXT:    movd %edx, %xmm0
-; SSE2-NEXT:    movd %eax, %xmm1
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
-; SSE2-NEXT:    movd %ecx, %xmm0
-; SSE2-NEXT:    movd %r8d, %xmm2
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
-; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; SSE2-NEXT:    retq
-;
-; SSSE3-LABEL: v4i32:
-; SSSE3:       # %bb.0:
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,2,3]
-; SSSE3-NEXT:    movd %xmm2, %ecx
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,2,3]
-; SSSE3-NEXT:    movd %xmm2, %r8d
-; SSSE3-NEXT:    xorl %edx, %edx
-; SSSE3-NEXT:    movl %r8d, %esi
-; SSSE3-NEXT:    addl %ecx, %esi
-; SSSE3-NEXT:    setns %dl
-; SSSE3-NEXT:    addl $2147483647, %edx # imm = 0x7FFFFFFF
-; SSSE3-NEXT:    addl %ecx, %r8d
-; SSSE3-NEXT:    cmovol %edx, %r8d
-; SSSE3-NEXT:    movd %xmm1, %edx
-; SSSE3-NEXT:    movd %xmm0, %ecx
-; SSSE3-NEXT:    xorl %esi, %esi
-; SSSE3-NEXT:    movl %ecx, %edi
-; SSSE3-NEXT:    addl %edx, %edi
-; SSSE3-NEXT:    setns %sil
-; SSSE3-NEXT:    addl $2147483647, %esi # imm = 0x7FFFFFFF
-; SSSE3-NEXT:    addl %edx, %ecx
-; SSSE3-NEXT:    cmovol %esi, %ecx
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,0,1]
-; SSSE3-NEXT:    movd %xmm2, %edx
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
-; SSSE3-NEXT:    movd %xmm2, %eax
-; SSSE3-NEXT:    xorl %edi, %edi
-; SSSE3-NEXT:    movl %eax, %esi
-; SSSE3-NEXT:    addl %edx, %esi
-; SSSE3-NEXT:    setns %dil
-; SSSE3-NEXT:    addl $2147483647, %edi # imm = 0x7FFFFFFF
-; SSSE3-NEXT:    addl %edx, %eax
-; SSSE3-NEXT:    cmovol %edi, %eax
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[3,1,2,3]
-; SSSE3-NEXT:    movd %xmm1, %r9d
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
-; SSSE3-NEXT:    movd %xmm0, %edx
-; SSSE3-NEXT:    xorl %edi, %edi
-; SSSE3-NEXT:    movl %edx, %esi
-; SSSE3-NEXT:    addl %r9d, %esi
-; SSSE3-NEXT:    setns %dil
-; SSSE3-NEXT:    addl $2147483647, %edi # imm = 0x7FFFFFFF
-; SSSE3-NEXT:    addl %r9d, %edx
-; SSSE3-NEXT:    cmovol %edi, %edx
-; SSSE3-NEXT:    movd %edx, %xmm0
-; SSSE3-NEXT:    movd %eax, %xmm1
-; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
-; SSSE3-NEXT:    movd %ecx, %xmm0
-; SSSE3-NEXT:    movd %r8d, %xmm2
-; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
-; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; SSSE3-NEXT:    retq
-;
-; SSE41-LABEL: v4i32:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    pextrd $3, %xmm1, %ecx
-; SSE41-NEXT:    pextrd $3, %xmm0, %r8d
-; SSE41-NEXT:    xorl %edx, %edx
-; SSE41-NEXT:    movl %r8d, %esi
-; SSE41-NEXT:    addl %ecx, %esi
-; SSE41-NEXT:    setns %dl
-; SSE41-NEXT:    addl $2147483647, %edx # imm = 0x7FFFFFFF
-; SSE41-NEXT:    addl %ecx, %r8d
-; SSE41-NEXT:    cmovol %edx, %r8d
-; SSE41-NEXT:    pextrd $2, %xmm1, %edx
-; SSE41-NEXT:    pextrd $2, %xmm0, %ecx
-; SSE41-NEXT:    xorl %esi, %esi
-; SSE41-NEXT:    movl %ecx, %edi
-; SSE41-NEXT:    addl %edx, %edi
-; SSE41-NEXT:    setns %sil
-; SSE41-NEXT:    addl $2147483647, %esi # imm = 0x7FFFFFFF
-; SSE41-NEXT:    addl %edx, %ecx
-; SSE41-NEXT:    cmovol %esi, %ecx
-; SSE41-NEXT:    movd %xmm1, %edx
-; SSE41-NEXT:    movd %xmm0, %eax
-; SSE41-NEXT:    xorl %edi, %edi
-; SSE41-NEXT:    movl %eax, %esi
-; SSE41-NEXT:    addl %edx, %esi
-; SSE41-NEXT:    setns %dil
-; SSE41-NEXT:    addl $2147483647, %edi # imm = 0x7FFFFFFF
-; SSE41-NEXT:    addl %edx, %eax
-; SSE41-NEXT:    cmovol %edi, %eax
-; SSE41-NEXT:    pextrd $1, %xmm1, %r9d
-; SSE41-NEXT:    pextrd $1, %xmm0, %edx
-; SSE41-NEXT:    xorl %edi, %edi
-; SSE41-NEXT:    movl %edx, %esi
-; SSE41-NEXT:    addl %r9d, %esi
-; SSE41-NEXT:    setns %dil
-; SSE41-NEXT:    addl $2147483647, %edi # imm = 0x7FFFFFFF
-; SSE41-NEXT:    addl %r9d, %edx
-; SSE41-NEXT:    cmovol %edi, %edx
-; SSE41-NEXT:    movd %eax, %xmm0
-; SSE41-NEXT:    pinsrd $1, %edx, %xmm0
-; SSE41-NEXT:    pinsrd $2, %ecx, %xmm0
-; SSE41-NEXT:    pinsrd $3, %r8d, %xmm0
-; SSE41-NEXT:    retq
-;
-; AVX-LABEL: v4i32:
-; AVX:       # %bb.0:
-; AVX-NEXT:    vpextrd $3, %xmm1, %ecx
-; AVX-NEXT:    vpextrd $3, %xmm0, %r9d
-; AVX-NEXT:    xorl %edx, %edx
-; AVX-NEXT:    movl %r9d, %esi
-; AVX-NEXT:    addl %ecx, %esi
-; AVX-NEXT:    setns %dl
-; AVX-NEXT:    addl $2147483647, %edx # imm = 0x7FFFFFFF
-; AVX-NEXT:    addl %ecx, %r9d
-; AVX-NEXT:    cmovol %edx, %r9d
-; AVX-NEXT:    vpextrd $2, %xmm1, %edx
-; AVX-NEXT:    vpextrd $2, %xmm0, %ecx
-; AVX-NEXT:    xorl %esi, %esi
-; AVX-NEXT:    movl %ecx, %edi
-; AVX-NEXT:    addl %edx, %edi
-; AVX-NEXT:    setns %sil
-; AVX-NEXT:    addl $2147483647, %esi # imm = 0x7FFFFFFF
-; AVX-NEXT:    addl %edx, %ecx
-; AVX-NEXT:    cmovol %esi, %ecx
-; AVX-NEXT:    vmovd %xmm1, %r8d
-; AVX-NEXT:    vmovd %xmm0, %edx
-; AVX-NEXT:    xorl %edi, %edi
-; AVX-NEXT:    movl %edx, %esi
-; AVX-NEXT:    addl %r8d, %esi
-; AVX-NEXT:    setns %dil
-; AVX-NEXT:    addl $2147483647, %edi # imm = 0x7FFFFFFF
-; AVX-NEXT:    addl %r8d, %edx
-; AVX-NEXT:    cmovol %edi, %edx
-; AVX-NEXT:    vpextrd $1, %xmm1, %r8d
-; AVX-NEXT:    vpextrd $1, %xmm0, %eax
-; AVX-NEXT:    xorl %esi, %esi
-; AVX-NEXT:    movl %eax, %edi
-; AVX-NEXT:    addl %r8d, %edi
-; AVX-NEXT:    setns %sil
-; AVX-NEXT:    addl $2147483647, %esi # imm = 0x7FFFFFFF
-; AVX-NEXT:    addl %r8d, %eax
-; AVX-NEXT:    cmovol %esi, %eax
-; AVX-NEXT:    vmovd %edx, %xmm0
-; AVX-NEXT:    vpinsrd $1, %eax, %xmm0, %xmm0
-; AVX-NEXT:    vpinsrd $2, %ecx, %xmm0, %xmm0
-; AVX-NEXT:    vpinsrd $3, %r9d, %xmm0, %xmm0
-; AVX-NEXT:    retq
-  %z = call <4 x i32> @llvm.sadd.sat.v4i32(<4 x i32> %x, <4 x i32> %y)
-  ret <4 x i32> %z
-}
-
 define <2 x i32> @v2i32(<2 x i32> %x, <2 x i32> %y) nounwind {
 ; SSE2-LABEL: v2i32:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    psllq $32, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,0,1]
-; SSE2-NEXT:    movq %xmm2, %rax
+; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648]
 ; SSE2-NEXT:    psllq $32, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
-; SSE2-NEXT:    movq %xmm2, %rcx
-; SSE2-NEXT:    xorl %edx, %edx
-; SSE2-NEXT:    movq %rcx, %rsi
-; SSE2-NEXT:    addq %rax, %rsi
-; SSE2-NEXT:    setns %dl
-; SSE2-NEXT:    movabsq $9223372036854775807, %r8 # imm = 0x7FFFFFFFFFFFFFFF
-; SSE2-NEXT:    addq %r8, %rdx
-; SSE2-NEXT:    addq %rax, %rcx
-; SSE2-NEXT:    cmovoq %rdx, %rcx
-; SSE2-NEXT:    movq %xmm1, %rax
-; SSE2-NEXT:    movq %xmm0, %rsi
-; SSE2-NEXT:    xorl %edi, %edi
-; SSE2-NEXT:    movq %rsi, %rdx
-; SSE2-NEXT:    addq %rax, %rdx
-; SSE2-NEXT:    setns %dil
-; SSE2-NEXT:    addq %r8, %rdi
-; SSE2-NEXT:    addq %rax, %rsi
-; SSE2-NEXT:    cmovoq %rdi, %rsi
-; SSE2-NEXT:    movq %rsi, %xmm1
-; SSE2-NEXT:    movq %rcx, %xmm0
-; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,3,2,3]
-; SSE2-NEXT:    psrad $31, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,3,2,3]
+; SSE2-NEXT:    movdqa %xmm0, %xmm3
+; SSE2-NEXT:    paddq %xmm1, %xmm0
+; SSE2-NEXT:    pxor %xmm2, %xmm1
+; SSE2-NEXT:    movdqa %xmm2, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm1, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm2, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSE2-NEXT:    pand %xmm5, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSE2-NEXT:    por %xmm1, %xmm4
+; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
+; SSE2-NEXT:    pxor %xmm1, %xmm4
+; SSE2-NEXT:    pxor %xmm2, %xmm3
+; SSE2-NEXT:    movdqa %xmm2, %xmm5
+; SSE2-NEXT:    pcmpgtd %xmm3, %xmm5
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm2, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
+; SSE2-NEXT:    pand %xmm6, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
+; SSE2-NEXT:    por %xmm3, %xmm5
+; SSE2-NEXT:    pxor %xmm1, %xmm5
+; SSE2-NEXT:    pcmpeqd %xmm5, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[1,0,3,2]
+; SSE2-NEXT:    pand %xmm4, %xmm3
+; SSE2-NEXT:    movdqa %xmm0, %xmm4
+; SSE2-NEXT:    pxor %xmm2, %xmm4
+; SSE2-NEXT:    movdqa %xmm2, %xmm6
+; SSE2-NEXT:    pcmpgtd %xmm4, %xmm6
+; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm2, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSE2-NEXT:    pand %xmm7, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
+; SSE2-NEXT:    por %xmm4, %xmm6
+; SSE2-NEXT:    pxor %xmm1, %xmm6
+; SSE2-NEXT:    pcmpeqd %xmm5, %xmm6
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm6[1,0,3,2]
+; SSE2-NEXT:    pand %xmm6, %xmm1
+; SSE2-NEXT:    pandn %xmm3, %xmm1
+; SSE2-NEXT:    movdqa %xmm1, %xmm3
+; SSE2-NEXT:    pandn %xmm0, %xmm3
+; SSE2-NEXT:    por %xmm2, %xmm0
+; SSE2-NEXT:    movdqa %xmm2, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm0, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm2, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE2-NEXT:    pand %xmm5, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm4[1,1,3,3]
+; SSE2-NEXT:    por %xmm0, %xmm2
+; SSE2-NEXT:    movdqa %xmm2, %xmm0
+; SSE2-NEXT:    pandn {{.*}}(%rip), %xmm0
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm2
+; SSE2-NEXT:    por %xmm0, %xmm2
+; SSE2-NEXT:    pand %xmm1, %xmm2
+; SSE2-NEXT:    por %xmm3, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,3,2,3]
+; SSE2-NEXT:    psrad $31, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,3,2,3]
 ; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
 ; SSE2-NEXT:    retq
 ;
 ; SSSE3-LABEL: v2i32:
 ; SSSE3:       # %bb.0:
 ; SSSE3-NEXT:    psllq $32, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,0,1]
-; SSSE3-NEXT:    movq %xmm2, %rax
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648]
 ; SSSE3-NEXT:    psllq $32, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
-; SSSE3-NEXT:    movq %xmm2, %rcx
-; SSSE3-NEXT:    xorl %edx, %edx
-; SSSE3-NEXT:    movq %rcx, %rsi
-; SSSE3-NEXT:    addq %rax, %rsi
-; SSSE3-NEXT:    setns %dl
-; SSSE3-NEXT:    movabsq $9223372036854775807, %r8 # imm = 0x7FFFFFFFFFFFFFFF
-; SSSE3-NEXT:    addq %r8, %rdx
-; SSSE3-NEXT:    addq %rax, %rcx
-; SSSE3-NEXT:    cmovoq %rdx, %rcx
-; SSSE3-NEXT:    movq %xmm1, %rax
-; SSSE3-NEXT:    movq %xmm0, %rsi
-; SSSE3-NEXT:    xorl %edi, %edi
-; SSSE3-NEXT:    movq %rsi, %rdx
-; SSSE3-NEXT:    addq %rax, %rdx
-; SSSE3-NEXT:    setns %dil
-; SSSE3-NEXT:    addq %r8, %rdi
-; SSSE3-NEXT:    addq %rax, %rsi
-; SSSE3-NEXT:    cmovoq %rdi, %rsi
-; SSSE3-NEXT:    movq %rsi, %xmm1
-; SSSE3-NEXT:    movq %rcx, %xmm0
-; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,3,2,3]
-; SSSE3-NEXT:    psrad $31, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,3,2,3]
+; SSSE3-NEXT:    movdqa %xmm0, %xmm3
+; SSSE3-NEXT:    paddq %xmm1, %xmm0
+; SSSE3-NEXT:    pxor %xmm2, %xmm1
+; SSSE3-NEXT:    movdqa %xmm2, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm5, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    por %xmm1, %xmm4
+; SSSE3-NEXT:    pcmpeqd %xmm1, %xmm1
+; SSSE3-NEXT:    pxor %xmm1, %xmm4
+; SSSE3-NEXT:    pxor %xmm2, %xmm3
+; SSSE3-NEXT:    movdqa %xmm2, %xmm5
+; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm5
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm3
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm6, %xmm3
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
+; SSSE3-NEXT:    por %xmm3, %xmm5
+; SSSE3-NEXT:    pxor %xmm1, %xmm5
+; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[1,0,3,2]
+; SSSE3-NEXT:    pand %xmm4, %xmm3
+; SSSE3-NEXT:    movdqa %xmm0, %xmm4
+; SSSE3-NEXT:    pxor %xmm2, %xmm4
+; SSSE3-NEXT:    movdqa %xmm2, %xmm6
+; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm6
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm7, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
+; SSSE3-NEXT:    por %xmm4, %xmm6
+; SSSE3-NEXT:    pxor %xmm1, %xmm6
+; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm6
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm6[1,0,3,2]
+; SSSE3-NEXT:    pand %xmm6, %xmm1
+; SSSE3-NEXT:    pandn %xmm3, %xmm1
+; SSSE3-NEXT:    movdqa %xmm1, %xmm3
+; SSSE3-NEXT:    pandn %xmm0, %xmm3
+; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    movdqa %xmm2, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm5, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    por %xmm0, %xmm2
+; SSSE3-NEXT:    movdqa %xmm2, %xmm0
+; SSSE3-NEXT:    pandn {{.*}}(%rip), %xmm0
+; SSSE3-NEXT:    pand {{.*}}(%rip), %xmm2
+; SSSE3-NEXT:    por %xmm0, %xmm2
+; SSSE3-NEXT:    pand %xmm1, %xmm2
+; SSSE3-NEXT:    por %xmm3, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,3,2,3]
+; SSSE3-NEXT:    psrad $31, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,3,2,3]
 ; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: v2i32:
 ; SSE41:       # %bb.0:
+; SSE41-NEXT:    movdqa %xmm0, %xmm2
 ; SSE41-NEXT:    psllq $32, %xmm1
-; SSE41-NEXT:    movq %xmm1, %rax
-; SSE41-NEXT:    psllq $32, %xmm0
-; SSE41-NEXT:    movq %xmm0, %rcx
-; SSE41-NEXT:    xorl %edx, %edx
-; SSE41-NEXT:    movq %rcx, %rsi
-; SSE41-NEXT:    addq %rax, %rsi
-; SSE41-NEXT:    setns %dl
-; SSE41-NEXT:    movabsq $9223372036854775807, %r8 # imm = 0x7FFFFFFFFFFFFFFF
-; SSE41-NEXT:    addq %r8, %rdx
-; SSE41-NEXT:    addq %rax, %rcx
-; SSE41-NEXT:    cmovoq %rdx, %rcx
-; SSE41-NEXT:    pextrq $1, %xmm1, %rax
-; SSE41-NEXT:    pextrq $1, %xmm0, %rsi
-; SSE41-NEXT:    xorl %edi, %edi
-; SSE41-NEXT:    movq %rsi, %rdx
-; SSE41-NEXT:    addq %rax, %rdx
-; SSE41-NEXT:    setns %dil
-; SSE41-NEXT:    addq %r8, %rdi
-; SSE41-NEXT:    addq %rax, %rsi
-; SSE41-NEXT:    cmovoq %rdi, %rsi
-; SSE41-NEXT:    movq %rsi, %xmm1
-; SSE41-NEXT:    movq %rcx, %xmm0
-; SSE41-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
-; SSE41-NEXT:    psrad $31, %xmm0
-; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3],xmm1[4,5],xmm0[6,7]
+; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483648,2147483648]
+; SSE41-NEXT:    psllq $32, %xmm2
+; SSE41-NEXT:    movdqa %xmm2, %xmm3
+; SSE41-NEXT:    paddq %xmm1, %xmm2
+; SSE41-NEXT:    pxor %xmm0, %xmm1
+; SSE41-NEXT:    movdqa %xmm0, %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm1, %xmm4
+; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSE41-NEXT:    pand %xmm5, %xmm1
+; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSE41-NEXT:    por %xmm1, %xmm4
+; SSE41-NEXT:    pcmpeqd %xmm8, %xmm8
+; SSE41-NEXT:    pxor %xmm8, %xmm4
+; SSE41-NEXT:    pxor %xmm0, %xmm3
+; SSE41-NEXT:    movdqa %xmm0, %xmm1
+; SSE41-NEXT:    pcmpgtd %xmm3, %xmm1
+; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm1[0,0,2,2]
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm3
+; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
+; SSE41-NEXT:    pand %xmm6, %xmm3
+; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm1[1,1,3,3]
+; SSE41-NEXT:    por %xmm3, %xmm6
+; SSE41-NEXT:    pxor %xmm8, %xmm6
+; SSE41-NEXT:    pcmpeqq %xmm6, %xmm4
+; SSE41-NEXT:    movdqa %xmm2, %xmm1
+; SSE41-NEXT:    pxor %xmm0, %xmm1
+; SSE41-NEXT:    movdqa %xmm0, %xmm3
+; SSE41-NEXT:    pcmpgtd %xmm1, %xmm3
+; SSE41-NEXT:    pshufd {{.*#+}} xmm7 = xmm3[0,0,2,2]
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
+; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm1[1,1,3,3]
+; SSE41-NEXT:    pand %xmm7, %xmm5
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,3,3]
+; SSE41-NEXT:    por %xmm5, %xmm1
+; SSE41-NEXT:    pxor %xmm8, %xmm1
+; SSE41-NEXT:    pcmpeqq %xmm6, %xmm1
+; SSE41-NEXT:    pandn %xmm4, %xmm1
+; SSE41-NEXT:    movdqa %xmm2, %xmm3
+; SSE41-NEXT:    por %xmm0, %xmm3
+; SSE41-NEXT:    movdqa %xmm0, %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm3, %xmm4
+; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm3
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
+; SSE41-NEXT:    pand %xmm5, %xmm0
+; SSE41-NEXT:    por %xmm4, %xmm0
+; SSE41-NEXT:    movapd {{.*#+}} xmm3 = [9223372036854775808,9223372036854775808]
+; SSE41-NEXT:    blendvpd %xmm0, {{.*}}(%rip), %xmm3
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm3, %xmm2
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,3,3]
+; SSE41-NEXT:    psrad $31, %xmm2
+; SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm0[0,1],xmm2[2,3],xmm0[4,5],xmm2[6,7]
+; SSE41-NEXT:    movdqa %xmm2, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: v2i32:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vpsllq $32, %xmm1, %xmm1
-; AVX1-NEXT:    vmovq %xmm1, %rax
+; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm2, %xmm3
+; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
+; AVX1-NEXT:    vpxor %xmm4, %xmm3, %xmm3
 ; AVX1-NEXT:    vpsllq $32, %xmm0, %xmm0
-; AVX1-NEXT:    vmovq %xmm0, %rcx
-; AVX1-NEXT:    xorl %edx, %edx
-; AVX1-NEXT:    movq %rcx, %rsi
-; AVX1-NEXT:    addq %rax, %rsi
-; AVX1-NEXT:    setns %dl
-; AVX1-NEXT:    movabsq $9223372036854775807, %r8 # imm = 0x7FFFFFFFFFFFFFFF
-; AVX1-NEXT:    addq %r8, %rdx
-; AVX1-NEXT:    addq %rax, %rcx
-; AVX1-NEXT:    cmovoq %rdx, %rcx
-; AVX1-NEXT:    vpextrq $1, %xmm1, %rax
-; AVX1-NEXT:    vpextrq $1, %xmm0, %rsi
-; AVX1-NEXT:    xorl %edi, %edi
-; AVX1-NEXT:    movq %rsi, %rdx
-; AVX1-NEXT:    addq %rax, %rdx
-; AVX1-NEXT:    setns %dil
-; AVX1-NEXT:    addq %r8, %rdi
-; AVX1-NEXT:    addq %rax, %rsi
-; AVX1-NEXT:    cmovoq %rdi, %rsi
-; AVX1-NEXT:    vmovq %rsi, %xmm0
-; AVX1-NEXT:    vmovq %rcx, %xmm1
-; AVX1-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; AVX1-NEXT:    vpcmpgtq %xmm0, %xmm2, %xmm5
+; AVX1-NEXT:    vpxor %xmm4, %xmm5, %xmm5
+; AVX1-NEXT:    vpcmpeqq %xmm3, %xmm5, %xmm3
+; AVX1-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpcmpgtq %xmm0, %xmm2, %xmm1
+; AVX1-NEXT:    vpxor %xmm4, %xmm1, %xmm1
+; AVX1-NEXT:    vpcmpeqq %xmm1, %xmm5, %xmm1
+; AVX1-NEXT:    vpandn %xmm3, %xmm1, %xmm1
+; AVX1-NEXT:    vmovapd {{.*#+}} xmm2 = [9223372036854775808,9223372036854775808]
+; AVX1-NEXT:    vblendvpd %xmm0, {{.*}}(%rip), %xmm2, %xmm2
+; AVX1-NEXT:    vblendvpd %xmm1, %xmm2, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrad $31, %xmm0, %xmm1
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
 ; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3],xmm0[4,5],xmm1[6,7]
@@ -983,29 +885,22 @@ define <2 x i32> @v2i32(<2 x i32> %x, <2 x i32> %y) nounwind {
 ; AVX2-LABEL: v2i32:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpsllq $32, %xmm1, %xmm1
-; AVX2-NEXT:    vmovq %xmm1, %rax
+; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX2-NEXT:    vpcmpgtq %xmm1, %xmm2, %xmm3
+; AVX2-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
+; AVX2-NEXT:    vpxor %xmm4, %xmm3, %xmm3
 ; AVX2-NEXT:    vpsllq $32, %xmm0, %xmm0
-; AVX2-NEXT:    vmovq %xmm0, %rcx
-; AVX2-NEXT:    xorl %edx, %edx
-; AVX2-NEXT:    movq %rcx, %rsi
-; AVX2-NEXT:    addq %rax, %rsi
-; AVX2-NEXT:    setns %dl
-; AVX2-NEXT:    movabsq $9223372036854775807, %r8 # imm = 0x7FFFFFFFFFFFFFFF
-; AVX2-NEXT:    addq %r8, %rdx
-; AVX2-NEXT:    addq %rax, %rcx
-; AVX2-NEXT:    cmovoq %rdx, %rcx
-; AVX2-NEXT:    vpextrq $1, %xmm1, %rax
-; AVX2-NEXT:    vpextrq $1, %xmm0, %rsi
-; AVX2-NEXT:    xorl %edi, %edi
-; AVX2-NEXT:    movq %rsi, %rdx
-; AVX2-NEXT:    addq %rax, %rdx
-; AVX2-NEXT:    setns %dil
-; AVX2-NEXT:    addq %r8, %rdi
-; AVX2-NEXT:    addq %rax, %rsi
-; AVX2-NEXT:    cmovoq %rdi, %rsi
-; AVX2-NEXT:    vmovq %rsi, %xmm0
-; AVX2-NEXT:    vmovq %rcx, %xmm1
-; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; AVX2-NEXT:    vpcmpgtq %xmm0, %xmm2, %xmm5
+; AVX2-NEXT:    vpxor %xmm4, %xmm5, %xmm5
+; AVX2-NEXT:    vpcmpeqq %xmm3, %xmm5, %xmm3
+; AVX2-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpcmpgtq %xmm0, %xmm2, %xmm1
+; AVX2-NEXT:    vpxor %xmm4, %xmm1, %xmm1
+; AVX2-NEXT:    vpcmpeqq %xmm1, %xmm5, %xmm1
+; AVX2-NEXT:    vpandn %xmm3, %xmm1, %xmm1
+; AVX2-NEXT:    vmovapd {{.*#+}} xmm2 = [9223372036854775808,9223372036854775808]
+; AVX2-NEXT:    vblendvpd %xmm0, {{.*}}(%rip), %xmm2, %xmm2
+; AVX2-NEXT:    vblendvpd %xmm1, %xmm2, %xmm0, %xmm0
 ; AVX2-NEXT:    vpsrad $31, %xmm0, %xmm1
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
 ; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3]
@@ -1014,243 +909,2042 @@ define <2 x i32> @v2i32(<2 x i32> %x, <2 x i32> %y) nounwind {
 ; AVX512-LABEL: v2i32:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpsllq $32, %xmm1, %xmm1
-; AVX512-NEXT:    vmovq %xmm1, %rax
+; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX512-NEXT:    vpcmpnltq %xmm2, %xmm1, %k0
 ; AVX512-NEXT:    vpsllq $32, %xmm0, %xmm0
-; AVX512-NEXT:    vmovq %xmm0, %rcx
-; AVX512-NEXT:    xorl %edx, %edx
-; AVX512-NEXT:    movq %rcx, %rsi
-; AVX512-NEXT:    addq %rax, %rsi
-; AVX512-NEXT:    setns %dl
-; AVX512-NEXT:    movabsq $9223372036854775807, %r8 # imm = 0x7FFFFFFFFFFFFFFF
-; AVX512-NEXT:    addq %r8, %rdx
-; AVX512-NEXT:    addq %rax, %rcx
-; AVX512-NEXT:    cmovoq %rdx, %rcx
-; AVX512-NEXT:    vpextrq $1, %xmm1, %rax
-; AVX512-NEXT:    vpextrq $1, %xmm0, %rsi
-; AVX512-NEXT:    xorl %edi, %edi
-; AVX512-NEXT:    movq %rsi, %rdx
-; AVX512-NEXT:    addq %rax, %rdx
-; AVX512-NEXT:    setns %dil
-; AVX512-NEXT:    addq %r8, %rdi
-; AVX512-NEXT:    addq %rax, %rsi
-; AVX512-NEXT:    cmovoq %rdi, %rsi
-; AVX512-NEXT:    vmovq %rsi, %xmm0
-; AVX512-NEXT:    vmovq %rcx, %xmm1
-; AVX512-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; AVX512-NEXT:    vpcmpnltq %xmm2, %xmm0, %k1
+; AVX512-NEXT:    kxorw %k0, %k1, %k0
+; AVX512-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vpcmpnltq %xmm2, %xmm0, %k2
+; AVX512-NEXT:    kxorw %k2, %k1, %k1
+; AVX512-NEXT:    kandnw %k1, %k0, %k1
+; AVX512-NEXT:    vpcmpgtq %xmm0, %xmm2, %k2
+; AVX512-NEXT:    vmovdqa {{.*#+}} xmm1 = [9223372036854775808,9223372036854775808]
+; AVX512-NEXT:    vmovdqa64 {{.*}}(%rip), %xmm1 {%k2}
+; AVX512-NEXT:    vmovdqa64 %xmm1, %xmm0 {%k1}
 ; AVX512-NEXT:    vpsraq $32, %xmm0, %xmm0
 ; AVX512-NEXT:    retq
   %z = call <2 x i32> @llvm.sadd.sat.v2i32(<2 x i32> %x, <2 x i32> %y)
   ret <2 x i32> %z
 }
 
-define <4 x i24> @v4i24(<4 x i24> %x, <4 x i24> %y) nounwind {
-; SSE2-LABEL: v4i24:
+define <4 x i32> @v4i32(<4 x i32> %x, <4 x i32> %y) nounwind {
+; SSE2-LABEL: v4i32:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pslld $8, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,2,3]
-; SSE2-NEXT:    movd %xmm2, %ecx
-; SSE2-NEXT:    pslld $8, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,2,3]
-; SSE2-NEXT:    movd %xmm2, %r8d
-; SSE2-NEXT:    xorl %edx, %edx
-; SSE2-NEXT:    movl %r8d, %esi
-; SSE2-NEXT:    addl %ecx, %esi
-; SSE2-NEXT:    setns %dl
-; SSE2-NEXT:    addl $2147483647, %edx # imm = 0x7FFFFFFF
-; SSE2-NEXT:    addl %ecx, %r8d
-; SSE2-NEXT:    cmovol %edx, %r8d
-; SSE2-NEXT:    movd %xmm1, %edx
-; SSE2-NEXT:    movd %xmm0, %ecx
-; SSE2-NEXT:    xorl %esi, %esi
-; SSE2-NEXT:    movl %ecx, %edi
-; SSE2-NEXT:    addl %edx, %edi
-; SSE2-NEXT:    setns %sil
-; SSE2-NEXT:    addl $2147483647, %esi # imm = 0x7FFFFFFF
-; SSE2-NEXT:    addl %edx, %ecx
-; SSE2-NEXT:    cmovol %esi, %ecx
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,0,1]
-; SSE2-NEXT:    movd %xmm2, %edx
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
-; SSE2-NEXT:    movd %xmm2, %eax
-; SSE2-NEXT:    xorl %edi, %edi
-; SSE2-NEXT:    movl %eax, %esi
-; SSE2-NEXT:    addl %edx, %esi
-; SSE2-NEXT:    setns %dil
-; SSE2-NEXT:    addl $2147483647, %edi # imm = 0x7FFFFFFF
-; SSE2-NEXT:    addl %edx, %eax
-; SSE2-NEXT:    cmovol %edi, %eax
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[3,1,2,3]
-; SSE2-NEXT:    movd %xmm1, %r9d
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
-; SSE2-NEXT:    movd %xmm0, %edx
-; SSE2-NEXT:    xorl %edi, %edi
-; SSE2-NEXT:    movl %edx, %esi
-; SSE2-NEXT:    addl %r9d, %esi
-; SSE2-NEXT:    setns %dil
-; SSE2-NEXT:    addl $2147483647, %edi # imm = 0x7FFFFFFF
-; SSE2-NEXT:    addl %r9d, %edx
-; SSE2-NEXT:    cmovol %edi, %edx
-; SSE2-NEXT:    movd %edx, %xmm0
-; SSE2-NEXT:    movd %eax, %xmm1
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
-; SSE2-NEXT:    movd %ecx, %xmm0
-; SSE2-NEXT:    movd %r8d, %xmm2
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
-; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; SSE2-NEXT:    psrad $8, %xmm0
+; SSE2-NEXT:    pxor %xmm3, %xmm3
+; SSE2-NEXT:    pxor %xmm4, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm1, %xmm4
+; SSE2-NEXT:    pcmpeqd %xmm2, %xmm2
+; SSE2-NEXT:    pxor %xmm2, %xmm4
+; SSE2-NEXT:    pxor %xmm5, %xmm5
+; SSE2-NEXT:    pcmpgtd %xmm0, %xmm5
+; SSE2-NEXT:    pxor %xmm2, %xmm5
+; SSE2-NEXT:    pcmpeqd %xmm5, %xmm4
+; SSE2-NEXT:    paddd %xmm1, %xmm0
+; SSE2-NEXT:    pcmpgtd %xmm0, %xmm3
+; SSE2-NEXT:    pxor %xmm3, %xmm2
+; SSE2-NEXT:    pcmpeqd %xmm5, %xmm2
+; SSE2-NEXT:    pandn %xmm4, %xmm2
+; SSE2-NEXT:    movdqa %xmm3, %xmm1
+; SSE2-NEXT:    pandn {{.*}}(%rip), %xmm1
+; SSE2-NEXT:    psrld $1, %xmm3
+; SSE2-NEXT:    por %xmm1, %xmm3
+; SSE2-NEXT:    pand %xmm2, %xmm3
+; SSE2-NEXT:    pandn %xmm0, %xmm2
+; SSE2-NEXT:    por %xmm3, %xmm2
+; SSE2-NEXT:    movdqa %xmm2, %xmm0
 ; SSE2-NEXT:    retq
 ;
-; SSSE3-LABEL: v4i24:
+; SSSE3-LABEL: v4i32:
 ; SSSE3:       # %bb.0:
-; SSSE3-NEXT:    pslld $8, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,2,3]
-; SSSE3-NEXT:    movd %xmm2, %ecx
-; SSSE3-NEXT:    pslld $8, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,2,3]
-; SSSE3-NEXT:    movd %xmm2, %r8d
-; SSSE3-NEXT:    xorl %edx, %edx
-; SSSE3-NEXT:    movl %r8d, %esi
-; SSSE3-NEXT:    addl %ecx, %esi
-; SSSE3-NEXT:    setns %dl
-; SSSE3-NEXT:    addl $2147483647, %edx # imm = 0x7FFFFFFF
-; SSSE3-NEXT:    addl %ecx, %r8d
-; SSSE3-NEXT:    cmovol %edx, %r8d
-; SSSE3-NEXT:    movd %xmm1, %edx
-; SSSE3-NEXT:    movd %xmm0, %ecx
-; SSSE3-NEXT:    xorl %esi, %esi
-; SSSE3-NEXT:    movl %ecx, %edi
-; SSSE3-NEXT:    addl %edx, %edi
-; SSSE3-NEXT:    setns %sil
-; SSSE3-NEXT:    addl $2147483647, %esi # imm = 0x7FFFFFFF
-; SSSE3-NEXT:    addl %edx, %ecx
-; SSSE3-NEXT:    cmovol %esi, %ecx
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,0,1]
-; SSSE3-NEXT:    movd %xmm2, %edx
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
-; SSSE3-NEXT:    movd %xmm2, %eax
-; SSSE3-NEXT:    xorl %edi, %edi
-; SSSE3-NEXT:    movl %eax, %esi
-; SSSE3-NEXT:    addl %edx, %esi
-; SSSE3-NEXT:    setns %dil
-; SSSE3-NEXT:    addl $2147483647, %edi # imm = 0x7FFFFFFF
-; SSSE3-NEXT:    addl %edx, %eax
-; SSSE3-NEXT:    cmovol %edi, %eax
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[3,1,2,3]
-; SSSE3-NEXT:    movd %xmm1, %r9d
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
-; SSSE3-NEXT:    movd %xmm0, %edx
-; SSSE3-NEXT:    xorl %edi, %edi
-; SSSE3-NEXT:    movl %edx, %esi
-; SSSE3-NEXT:    addl %r9d, %esi
-; SSSE3-NEXT:    setns %dil
-; SSSE3-NEXT:    addl $2147483647, %edi # imm = 0x7FFFFFFF
-; SSSE3-NEXT:    addl %r9d, %edx
-; SSSE3-NEXT:    cmovol %edi, %edx
-; SSSE3-NEXT:    movd %edx, %xmm0
-; SSSE3-NEXT:    movd %eax, %xmm1
-; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
-; SSSE3-NEXT:    movd %ecx, %xmm0
-; SSSE3-NEXT:    movd %r8d, %xmm2
-; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
-; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; SSSE3-NEXT:    psrad $8, %xmm0
+; SSSE3-NEXT:    pxor %xmm3, %xmm3
+; SSSE3-NEXT:    pxor %xmm4, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm4
+; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm2
+; SSSE3-NEXT:    pxor %xmm2, %xmm4
+; SSSE3-NEXT:    pxor %xmm5, %xmm5
+; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm5
+; SSSE3-NEXT:    pxor %xmm2, %xmm5
+; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm4
+; SSSE3-NEXT:    paddd %xmm1, %xmm0
+; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm3
+; SSSE3-NEXT:    pxor %xmm3, %xmm2
+; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm2
+; SSSE3-NEXT:    pandn %xmm4, %xmm2
+; SSSE3-NEXT:    movdqa %xmm3, %xmm1
+; SSSE3-NEXT:    pandn {{.*}}(%rip), %xmm1
+; SSSE3-NEXT:    psrld $1, %xmm3
+; SSSE3-NEXT:    por %xmm1, %xmm3
+; SSSE3-NEXT:    pand %xmm2, %xmm3
+; SSSE3-NEXT:    pandn %xmm0, %xmm2
+; SSSE3-NEXT:    por %xmm3, %xmm2
+; SSSE3-NEXT:    movdqa %xmm2, %xmm0
 ; SSSE3-NEXT:    retq
 ;
-; SSE41-LABEL: v4i24:
+; SSE41-LABEL: v4i32:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    pslld $8, %xmm1
-; SSE41-NEXT:    pextrd $3, %xmm1, %ecx
-; SSE41-NEXT:    pslld $8, %xmm0
-; SSE41-NEXT:    pextrd $3, %xmm0, %r8d
-; SSE41-NEXT:    xorl %edx, %edx
-; SSE41-NEXT:    movl %r8d, %esi
-; SSE41-NEXT:    addl %ecx, %esi
-; SSE41-NEXT:    setns %dl
-; SSE41-NEXT:    addl $2147483647, %edx # imm = 0x7FFFFFFF
-; SSE41-NEXT:    addl %ecx, %r8d
-; SSE41-NEXT:    cmovol %edx, %r8d
-; SSE41-NEXT:    pextrd $2, %xmm1, %edx
-; SSE41-NEXT:    pextrd $2, %xmm0, %ecx
-; SSE41-NEXT:    xorl %esi, %esi
-; SSE41-NEXT:    movl %ecx, %edi
-; SSE41-NEXT:    addl %edx, %edi
-; SSE41-NEXT:    setns %sil
-; SSE41-NEXT:    addl $2147483647, %esi # imm = 0x7FFFFFFF
-; SSE41-NEXT:    addl %edx, %ecx
-; SSE41-NEXT:    cmovol %esi, %ecx
-; SSE41-NEXT:    movd %xmm1, %edx
-; SSE41-NEXT:    movd %xmm0, %eax
-; SSE41-NEXT:    xorl %edi, %edi
-; SSE41-NEXT:    movl %eax, %esi
-; SSE41-NEXT:    addl %edx, %esi
-; SSE41-NEXT:    setns %dil
-; SSE41-NEXT:    addl $2147483647, %edi # imm = 0x7FFFFFFF
-; SSE41-NEXT:    addl %edx, %eax
-; SSE41-NEXT:    cmovol %edi, %eax
-; SSE41-NEXT:    pextrd $1, %xmm1, %r9d
-; SSE41-NEXT:    pextrd $1, %xmm0, %edx
-; SSE41-NEXT:    xorl %edi, %edi
-; SSE41-NEXT:    movl %edx, %esi
-; SSE41-NEXT:    addl %r9d, %esi
-; SSE41-NEXT:    setns %dil
-; SSE41-NEXT:    addl $2147483647, %edi # imm = 0x7FFFFFFF
-; SSE41-NEXT:    addl %r9d, %edx
-; SSE41-NEXT:    cmovol %edi, %edx
-; SSE41-NEXT:    movd %eax, %xmm0
-; SSE41-NEXT:    pinsrd $1, %edx, %xmm0
-; SSE41-NEXT:    pinsrd $2, %ecx, %xmm0
-; SSE41-NEXT:    pinsrd $3, %r8d, %xmm0
-; SSE41-NEXT:    psrad $8, %xmm0
+; SSE41-NEXT:    movdqa %xmm0, %xmm2
+; SSE41-NEXT:    pxor %xmm3, %xmm3
+; SSE41-NEXT:    pxor %xmm0, %xmm0
+; SSE41-NEXT:    pcmpgtd %xmm1, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm4, %xmm4
+; SSE41-NEXT:    pxor %xmm4, %xmm0
+; SSE41-NEXT:    pxor %xmm5, %xmm5
+; SSE41-NEXT:    pcmpgtd %xmm2, %xmm5
+; SSE41-NEXT:    pxor %xmm4, %xmm5
+; SSE41-NEXT:    pcmpeqd %xmm5, %xmm0
+; SSE41-NEXT:    paddd %xmm1, %xmm2
+; SSE41-NEXT:    pcmpgtd %xmm2, %xmm3
+; SSE41-NEXT:    pxor %xmm4, %xmm3
+; SSE41-NEXT:    pcmpeqd %xmm5, %xmm3
+; SSE41-NEXT:    pandn %xmm0, %xmm3
+; SSE41-NEXT:    movaps {{.*#+}} xmm1 = [2147483648,2147483648,2147483648,2147483648]
+; SSE41-NEXT:    movdqa %xmm2, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, {{.*}}(%rip), %xmm1
+; SSE41-NEXT:    movdqa %xmm3, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm1, %xmm2
+; SSE41-NEXT:    movaps %xmm2, %xmm0
 ; SSE41-NEXT:    retq
 ;
-; AVX-LABEL: v4i24:
-; AVX:       # %bb.0:
-; AVX-NEXT:    vpslld $8, %xmm1, %xmm1
-; AVX-NEXT:    vpextrd $3, %xmm1, %ecx
-; AVX-NEXT:    vpslld $8, %xmm0, %xmm0
-; AVX-NEXT:    vpextrd $3, %xmm0, %r9d
-; AVX-NEXT:    xorl %edx, %edx
-; AVX-NEXT:    movl %r9d, %esi
-; AVX-NEXT:    addl %ecx, %esi
-; AVX-NEXT:    setns %dl
-; AVX-NEXT:    addl $2147483647, %edx # imm = 0x7FFFFFFF
-; AVX-NEXT:    addl %ecx, %r9d
-; AVX-NEXT:    cmovol %edx, %r9d
-; AVX-NEXT:    vpextrd $2, %xmm1, %edx
-; AVX-NEXT:    vpextrd $2, %xmm0, %ecx
-; AVX-NEXT:    xorl %esi, %esi
-; AVX-NEXT:    movl %ecx, %edi
-; AVX-NEXT:    addl %edx, %edi
-; AVX-NEXT:    setns %sil
-; AVX-NEXT:    addl $2147483647, %esi # imm = 0x7FFFFFFF
-; AVX-NEXT:    addl %edx, %ecx
-; AVX-NEXT:    cmovol %esi, %ecx
-; AVX-NEXT:    vmovd %xmm1, %r8d
-; AVX-NEXT:    vmovd %xmm0, %edx
-; AVX-NEXT:    xorl %edi, %edi
-; AVX-NEXT:    movl %edx, %esi
-; AVX-NEXT:    addl %r8d, %esi
-; AVX-NEXT:    setns %dil
-; AVX-NEXT:    addl $2147483647, %edi # imm = 0x7FFFFFFF
-; AVX-NEXT:    addl %r8d, %edx
-; AVX-NEXT:    cmovol %edi, %edx
-; AVX-NEXT:    vpextrd $1, %xmm1, %r8d
-; AVX-NEXT:    vpextrd $1, %xmm0, %eax
-; AVX-NEXT:    xorl %esi, %esi
-; AVX-NEXT:    movl %eax, %edi
-; AVX-NEXT:    addl %r8d, %edi
-; AVX-NEXT:    setns %sil
-; AVX-NEXT:    addl $2147483647, %esi # imm = 0x7FFFFFFF
-; AVX-NEXT:    addl %r8d, %eax
-; AVX-NEXT:    cmovol %esi, %eax
-; AVX-NEXT:    vmovd %edx, %xmm0
-; AVX-NEXT:    vpinsrd $1, %eax, %xmm0, %xmm0
-; AVX-NEXT:    vpinsrd $2, %ecx, %xmm0, %xmm0
-; AVX-NEXT:    vpinsrd $3, %r9d, %xmm0, %xmm0
-; AVX-NEXT:    vpsrad $8, %xmm0, %xmm0
-; AVX-NEXT:    retq
-  %z = call <4 x i24> @llvm.sadd.sat.v4i24(<4 x i24> %x, <4 x i24> %y)
-  ret <4 x i24> %z
+; AVX1-LABEL: v4i32:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm2, %xmm3
+; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
+; AVX1-NEXT:    vpxor %xmm4, %xmm3, %xmm3
+; AVX1-NEXT:    vpcmpgtd %xmm0, %xmm2, %xmm5
+; AVX1-NEXT:    vpxor %xmm4, %xmm5, %xmm5
+; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm5, %xmm3
+; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpcmpgtd %xmm0, %xmm2, %xmm1
+; AVX1-NEXT:    vpxor %xmm4, %xmm1, %xmm1
+; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm5, %xmm1
+; AVX1-NEXT:    vpandn %xmm3, %xmm1, %xmm1
+; AVX1-NEXT:    vmovaps {{.*#+}} xmm2 = [2147483648,2147483648,2147483648,2147483648]
+; AVX1-NEXT:    vblendvps %xmm0, {{.*}}(%rip), %xmm2, %xmm2
+; AVX1-NEXT:    vblendvps %xmm1, %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: v4i32:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX2-NEXT:    vpcmpgtd %xmm1, %xmm2, %xmm3
+; AVX2-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
+; AVX2-NEXT:    vpxor %xmm4, %xmm3, %xmm3
+; AVX2-NEXT:    vpcmpgtd %xmm0, %xmm2, %xmm5
+; AVX2-NEXT:    vpxor %xmm4, %xmm5, %xmm5
+; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm5, %xmm3
+; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpcmpgtd %xmm0, %xmm2, %xmm1
+; AVX2-NEXT:    vpxor %xmm4, %xmm1, %xmm1
+; AVX2-NEXT:    vpcmpeqd %xmm1, %xmm5, %xmm1
+; AVX2-NEXT:    vpandn %xmm3, %xmm1, %xmm1
+; AVX2-NEXT:    vbroadcastss {{.*#+}} xmm2 = [2147483647,2147483647,2147483647,2147483647]
+; AVX2-NEXT:    vbroadcastss {{.*#+}} xmm3 = [2147483648,2147483648,2147483648,2147483648]
+; AVX2-NEXT:    vblendvps %xmm0, %xmm2, %xmm3, %xmm2
+; AVX2-NEXT:    vblendvps %xmm1, %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: v4i32:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX512-NEXT:    vpcmpnltd %xmm2, %xmm1, %k0
+; AVX512-NEXT:    vpcmpnltd %xmm2, %xmm0, %k1
+; AVX512-NEXT:    kxorw %k0, %k1, %k0
+; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vpcmpnltd %xmm2, %xmm0, %k2
+; AVX512-NEXT:    kxorw %k2, %k1, %k1
+; AVX512-NEXT:    kandnw %k1, %k0, %k1
+; AVX512-NEXT:    vpcmpgtd %xmm0, %xmm2, %k2
+; AVX512-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [2147483648,2147483648,2147483648,2147483648]
+; AVX512-NEXT:    vpbroadcastd {{.*}}(%rip), %xmm1 {%k2}
+; AVX512-NEXT:    vmovdqa32 %xmm1, %xmm0 {%k1}
+; AVX512-NEXT:    retq
+  %z = call <4 x i32> @llvm.sadd.sat.v4i32(<4 x i32> %x, <4 x i32> %y)
+  ret <4 x i32> %z
+}
+
+define <8 x i32> @v8i32(<8 x i32> %x, <8 x i32> %y) nounwind {
+; SSE2-LABEL: v8i32:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa %xmm0, %xmm8
+; SSE2-NEXT:    pxor %xmm6, %xmm6
+; SSE2-NEXT:    pxor %xmm7, %xmm7
+; SSE2-NEXT:    pcmpgtd %xmm2, %xmm7
+; SSE2-NEXT:    pcmpeqd %xmm5, %xmm5
+; SSE2-NEXT:    pxor %xmm5, %xmm7
+; SSE2-NEXT:    pxor %xmm4, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm0, %xmm4
+; SSE2-NEXT:    pxor %xmm5, %xmm4
+; SSE2-NEXT:    pcmpeqd %xmm4, %xmm7
+; SSE2-NEXT:    paddd %xmm2, %xmm8
+; SSE2-NEXT:    pxor %xmm2, %xmm2
+; SSE2-NEXT:    pcmpgtd %xmm8, %xmm2
+; SSE2-NEXT:    movdqa %xmm2, %xmm0
+; SSE2-NEXT:    pxor %xmm5, %xmm0
+; SSE2-NEXT:    pcmpeqd %xmm4, %xmm0
+; SSE2-NEXT:    pandn %xmm7, %xmm0
+; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [2147483648,2147483648,2147483648,2147483648]
+; SSE2-NEXT:    movdqa %xmm2, %xmm7
+; SSE2-NEXT:    pandn %xmm4, %xmm7
+; SSE2-NEXT:    psrld $1, %xmm2
+; SSE2-NEXT:    por %xmm7, %xmm2
+; SSE2-NEXT:    pand %xmm0, %xmm2
+; SSE2-NEXT:    pandn %xmm8, %xmm0
+; SSE2-NEXT:    por %xmm2, %xmm0
+; SSE2-NEXT:    pxor %xmm2, %xmm2
+; SSE2-NEXT:    pcmpgtd %xmm3, %xmm2
+; SSE2-NEXT:    pxor %xmm5, %xmm2
+; SSE2-NEXT:    pxor %xmm7, %xmm7
+; SSE2-NEXT:    pcmpgtd %xmm1, %xmm7
+; SSE2-NEXT:    pxor %xmm5, %xmm7
+; SSE2-NEXT:    pcmpeqd %xmm7, %xmm2
+; SSE2-NEXT:    paddd %xmm3, %xmm1
+; SSE2-NEXT:    pcmpgtd %xmm1, %xmm6
+; SSE2-NEXT:    pxor %xmm6, %xmm5
+; SSE2-NEXT:    pcmpeqd %xmm7, %xmm5
+; SSE2-NEXT:    pandn %xmm2, %xmm5
+; SSE2-NEXT:    movdqa %xmm6, %xmm2
+; SSE2-NEXT:    pandn %xmm4, %xmm2
+; SSE2-NEXT:    psrld $1, %xmm6
+; SSE2-NEXT:    por %xmm2, %xmm6
+; SSE2-NEXT:    pand %xmm5, %xmm6
+; SSE2-NEXT:    pandn %xmm1, %xmm5
+; SSE2-NEXT:    por %xmm6, %xmm5
+; SSE2-NEXT:    movdqa %xmm5, %xmm1
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: v8i32:
+; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    movdqa %xmm0, %xmm8
+; SSSE3-NEXT:    pxor %xmm6, %xmm6
+; SSSE3-NEXT:    pxor %xmm7, %xmm7
+; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm7
+; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm5
+; SSSE3-NEXT:    pxor %xmm5, %xmm7
+; SSSE3-NEXT:    pxor %xmm4, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm4
+; SSSE3-NEXT:    pxor %xmm5, %xmm4
+; SSSE3-NEXT:    pcmpeqd %xmm4, %xmm7
+; SSSE3-NEXT:    paddd %xmm2, %xmm8
+; SSSE3-NEXT:    pxor %xmm2, %xmm2
+; SSSE3-NEXT:    pcmpgtd %xmm8, %xmm2
+; SSSE3-NEXT:    movdqa %xmm2, %xmm0
+; SSSE3-NEXT:    pxor %xmm5, %xmm0
+; SSSE3-NEXT:    pcmpeqd %xmm4, %xmm0
+; SSSE3-NEXT:    pandn %xmm7, %xmm0
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm4 = [2147483648,2147483648,2147483648,2147483648]
+; SSSE3-NEXT:    movdqa %xmm2, %xmm7
+; SSSE3-NEXT:    pandn %xmm4, %xmm7
+; SSSE3-NEXT:    psrld $1, %xmm2
+; SSSE3-NEXT:    por %xmm7, %xmm2
+; SSSE3-NEXT:    pand %xmm0, %xmm2
+; SSSE3-NEXT:    pandn %xmm8, %xmm0
+; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    pxor %xmm2, %xmm2
+; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm2
+; SSSE3-NEXT:    pxor %xmm5, %xmm2
+; SSSE3-NEXT:    pxor %xmm7, %xmm7
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm7
+; SSSE3-NEXT:    pxor %xmm5, %xmm7
+; SSSE3-NEXT:    pcmpeqd %xmm7, %xmm2
+; SSSE3-NEXT:    paddd %xmm3, %xmm1
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm6
+; SSSE3-NEXT:    pxor %xmm6, %xmm5
+; SSSE3-NEXT:    pcmpeqd %xmm7, %xmm5
+; SSSE3-NEXT:    pandn %xmm2, %xmm5
+; SSSE3-NEXT:    movdqa %xmm6, %xmm2
+; SSSE3-NEXT:    pandn %xmm4, %xmm2
+; SSSE3-NEXT:    psrld $1, %xmm6
+; SSSE3-NEXT:    por %xmm2, %xmm6
+; SSSE3-NEXT:    pand %xmm5, %xmm6
+; SSSE3-NEXT:    pandn %xmm1, %xmm5
+; SSSE3-NEXT:    por %xmm6, %xmm5
+; SSSE3-NEXT:    movdqa %xmm5, %xmm1
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: v8i32:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movdqa %xmm0, %xmm5
+; SSE41-NEXT:    pxor %xmm4, %xmm4
+; SSE41-NEXT:    pxor %xmm0, %xmm0
+; SSE41-NEXT:    pcmpgtd %xmm2, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm9, %xmm9
+; SSE41-NEXT:    pxor %xmm9, %xmm0
+; SSE41-NEXT:    pxor %xmm7, %xmm7
+; SSE41-NEXT:    pcmpgtd %xmm5, %xmm7
+; SSE41-NEXT:    pxor %xmm9, %xmm7
+; SSE41-NEXT:    pcmpeqd %xmm7, %xmm0
+; SSE41-NEXT:    paddd %xmm2, %xmm5
+; SSE41-NEXT:    pxor %xmm2, %xmm2
+; SSE41-NEXT:    pcmpgtd %xmm5, %xmm2
+; SSE41-NEXT:    pxor %xmm9, %xmm2
+; SSE41-NEXT:    pcmpeqd %xmm7, %xmm2
+; SSE41-NEXT:    pandn %xmm0, %xmm2
+; SSE41-NEXT:    movaps {{.*#+}} xmm8 = [2147483647,2147483647,2147483647,2147483647]
+; SSE41-NEXT:    movaps {{.*#+}} xmm6 = [2147483648,2147483648,2147483648,2147483648]
+; SSE41-NEXT:    movaps %xmm6, %xmm7
+; SSE41-NEXT:    movdqa %xmm5, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm8, %xmm7
+; SSE41-NEXT:    movdqa %xmm2, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm7, %xmm5
+; SSE41-NEXT:    pxor %xmm0, %xmm0
+; SSE41-NEXT:    pcmpgtd %xmm3, %xmm0
+; SSE41-NEXT:    pxor %xmm9, %xmm0
+; SSE41-NEXT:    pxor %xmm2, %xmm2
+; SSE41-NEXT:    pcmpgtd %xmm1, %xmm2
+; SSE41-NEXT:    pxor %xmm9, %xmm2
+; SSE41-NEXT:    pcmpeqd %xmm2, %xmm0
+; SSE41-NEXT:    paddd %xmm3, %xmm1
+; SSE41-NEXT:    pcmpgtd %xmm1, %xmm4
+; SSE41-NEXT:    pxor %xmm9, %xmm4
+; SSE41-NEXT:    pcmpeqd %xmm2, %xmm4
+; SSE41-NEXT:    pandn %xmm0, %xmm4
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm8, %xmm6
+; SSE41-NEXT:    movdqa %xmm4, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm6, %xmm1
+; SSE41-NEXT:    movaps %xmm5, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: v8i32:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; AVX1-NEXT:    vpcmpgtd %xmm2, %xmm3, %xmm4
+; AVX1-NEXT:    vpcmpeqd %xmm5, %xmm5, %xmm5
+; AVX1-NEXT:    vpxor %xmm5, %xmm4, %xmm4
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm6
+; AVX1-NEXT:    vpcmpgtd %xmm6, %xmm3, %xmm7
+; AVX1-NEXT:    vpxor %xmm5, %xmm7, %xmm7
+; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm7, %xmm8
+; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm3, %xmm4
+; AVX1-NEXT:    vpxor %xmm5, %xmm4, %xmm9
+; AVX1-NEXT:    vpcmpgtd %xmm0, %xmm3, %xmm4
+; AVX1-NEXT:    vpxor %xmm5, %xmm4, %xmm4
+; AVX1-NEXT:    vpcmpeqd %xmm9, %xmm4, %xmm9
+; AVX1-NEXT:    vinsertf128 $1, %xmm8, %ymm9, %ymm8
+; AVX1-NEXT:    vpaddd %xmm2, %xmm6, %xmm9
+; AVX1-NEXT:    vpcmpgtd %xmm9, %xmm3, %xmm6
+; AVX1-NEXT:    vpxor %xmm5, %xmm6, %xmm2
+; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm7, %xmm2
+; AVX1-NEXT:    vpxor %xmm5, %xmm2, %xmm2
+; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpcmpgtd %xmm0, %xmm3, %xmm1
+; AVX1-NEXT:    vpxor %xmm5, %xmm1, %xmm1
+; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm4, %xmm1
+; AVX1-NEXT:    vpxor %xmm5, %xmm1, %xmm1
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
+; AVX1-NEXT:    vandps %ymm1, %ymm8, %ymm1
+; AVX1-NEXT:    vinsertf128 $1, %xmm6, %ymm0, %ymm2
+; AVX1-NEXT:    vmovaps {{.*#+}} ymm3 = [2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648]
+; AVX1-NEXT:    vblendvps %ymm2, {{.*}}(%rip), %ymm3, %ymm2
+; AVX1-NEXT:    vinsertf128 $1, %xmm9, %ymm0, %ymm0
+; AVX1-NEXT:    vblendvps %ymm1, %ymm2, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: v8i32:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX2-NEXT:    vpcmpgtd %ymm1, %ymm2, %ymm3
+; AVX2-NEXT:    vpcmpeqd %ymm4, %ymm4, %ymm4
+; AVX2-NEXT:    vpxor %ymm4, %ymm3, %ymm3
+; AVX2-NEXT:    vpcmpgtd %ymm0, %ymm2, %ymm5
+; AVX2-NEXT:    vpxor %ymm4, %ymm5, %ymm5
+; AVX2-NEXT:    vpcmpeqd %ymm3, %ymm5, %ymm3
+; AVX2-NEXT:    vpaddd %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpcmpgtd %ymm0, %ymm2, %ymm1
+; AVX2-NEXT:    vpxor %ymm4, %ymm1, %ymm1
+; AVX2-NEXT:    vpcmpeqd %ymm1, %ymm5, %ymm1
+; AVX2-NEXT:    vpandn %ymm3, %ymm1, %ymm1
+; AVX2-NEXT:    vbroadcastss {{.*#+}} ymm2 = [2147483647,2147483647,2147483647,2147483647,2147483647,2147483647,2147483647,2147483647]
+; AVX2-NEXT:    vbroadcastss {{.*#+}} ymm3 = [2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648]
+; AVX2-NEXT:    vblendvps %ymm0, %ymm2, %ymm3, %ymm2
+; AVX2-NEXT:    vblendvps %ymm1, %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: v8i32:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX512-NEXT:    vpcmpnltd %ymm2, %ymm1, %k0
+; AVX512-NEXT:    vpcmpnltd %ymm2, %ymm0, %k1
+; AVX512-NEXT:    kxorw %k0, %k1, %k0
+; AVX512-NEXT:    vpaddd %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpcmpnltd %ymm2, %ymm0, %k2
+; AVX512-NEXT:    kxorw %k2, %k1, %k1
+; AVX512-NEXT:    kandnw %k1, %k0, %k1
+; AVX512-NEXT:    vpcmpgtd %ymm0, %ymm2, %k2
+; AVX512-NEXT:    vpbroadcastd {{.*#+}} ymm1 = [2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648]
+; AVX512-NEXT:    vpbroadcastd {{.*}}(%rip), %ymm1 {%k2}
+; AVX512-NEXT:    vmovdqa32 %ymm1, %ymm0 {%k1}
+; AVX512-NEXT:    retq
+  %z = call <8 x i32> @llvm.sadd.sat.v8i32(<8 x i32> %x, <8 x i32> %y)
+  ret <8 x i32> %z
+}
+
+define <16 x i32> @v16i32(<16 x i32> %x, <16 x i32> %y) nounwind {
+; SSE2-LABEL: v16i32:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa %xmm1, %xmm8
+; SSE2-NEXT:    movdqa %xmm0, %xmm11
+; SSE2-NEXT:    pxor %xmm10, %xmm10
+; SSE2-NEXT:    pxor %xmm12, %xmm12
+; SSE2-NEXT:    pcmpgtd %xmm4, %xmm12
+; SSE2-NEXT:    pcmpeqd %xmm9, %xmm9
+; SSE2-NEXT:    pxor %xmm9, %xmm12
+; SSE2-NEXT:    pxor %xmm1, %xmm1
+; SSE2-NEXT:    pcmpgtd %xmm0, %xmm1
+; SSE2-NEXT:    pxor %xmm9, %xmm1
+; SSE2-NEXT:    pcmpeqd %xmm1, %xmm12
+; SSE2-NEXT:    paddd %xmm4, %xmm11
+; SSE2-NEXT:    pxor %xmm4, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm11, %xmm4
+; SSE2-NEXT:    movdqa %xmm4, %xmm0
+; SSE2-NEXT:    pxor %xmm9, %xmm0
+; SSE2-NEXT:    pcmpeqd %xmm1, %xmm0
+; SSE2-NEXT:    pandn %xmm12, %xmm0
+; SSE2-NEXT:    movdqa {{.*#+}} xmm12 = [2147483648,2147483648,2147483648,2147483648]
+; SSE2-NEXT:    movdqa %xmm4, %xmm1
+; SSE2-NEXT:    pandn %xmm12, %xmm1
+; SSE2-NEXT:    psrld $1, %xmm4
+; SSE2-NEXT:    por %xmm1, %xmm4
+; SSE2-NEXT:    pand %xmm0, %xmm4
+; SSE2-NEXT:    pandn %xmm11, %xmm0
+; SSE2-NEXT:    por %xmm4, %xmm0
+; SSE2-NEXT:    pxor %xmm11, %xmm11
+; SSE2-NEXT:    pcmpgtd %xmm5, %xmm11
+; SSE2-NEXT:    pxor %xmm9, %xmm11
+; SSE2-NEXT:    pxor %xmm4, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm8, %xmm4
+; SSE2-NEXT:    pxor %xmm9, %xmm4
+; SSE2-NEXT:    pcmpeqd %xmm4, %xmm11
+; SSE2-NEXT:    paddd %xmm5, %xmm8
+; SSE2-NEXT:    pxor %xmm5, %xmm5
+; SSE2-NEXT:    pcmpgtd %xmm8, %xmm5
+; SSE2-NEXT:    movdqa %xmm5, %xmm1
+; SSE2-NEXT:    pxor %xmm9, %xmm1
+; SSE2-NEXT:    pcmpeqd %xmm4, %xmm1
+; SSE2-NEXT:    pandn %xmm11, %xmm1
+; SSE2-NEXT:    movdqa %xmm5, %xmm4
+; SSE2-NEXT:    pandn %xmm12, %xmm4
+; SSE2-NEXT:    psrld $1, %xmm5
+; SSE2-NEXT:    por %xmm4, %xmm5
+; SSE2-NEXT:    pand %xmm1, %xmm5
+; SSE2-NEXT:    pandn %xmm8, %xmm1
+; SSE2-NEXT:    por %xmm5, %xmm1
+; SSE2-NEXT:    pxor %xmm4, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm6, %xmm4
+; SSE2-NEXT:    pxor %xmm9, %xmm4
+; SSE2-NEXT:    pxor %xmm5, %xmm5
+; SSE2-NEXT:    pcmpgtd %xmm2, %xmm5
+; SSE2-NEXT:    pxor %xmm9, %xmm5
+; SSE2-NEXT:    pcmpeqd %xmm5, %xmm4
+; SSE2-NEXT:    paddd %xmm6, %xmm2
+; SSE2-NEXT:    pxor %xmm6, %xmm6
+; SSE2-NEXT:    pcmpgtd %xmm2, %xmm6
+; SSE2-NEXT:    movdqa %xmm6, %xmm8
+; SSE2-NEXT:    pxor %xmm9, %xmm8
+; SSE2-NEXT:    pcmpeqd %xmm5, %xmm8
+; SSE2-NEXT:    pandn %xmm4, %xmm8
+; SSE2-NEXT:    movdqa %xmm6, %xmm4
+; SSE2-NEXT:    pandn %xmm12, %xmm4
+; SSE2-NEXT:    psrld $1, %xmm6
+; SSE2-NEXT:    por %xmm4, %xmm6
+; SSE2-NEXT:    pand %xmm8, %xmm6
+; SSE2-NEXT:    pandn %xmm2, %xmm8
+; SSE2-NEXT:    por %xmm6, %xmm8
+; SSE2-NEXT:    pxor %xmm2, %xmm2
+; SSE2-NEXT:    pcmpgtd %xmm7, %xmm2
+; SSE2-NEXT:    pxor %xmm9, %xmm2
+; SSE2-NEXT:    pxor %xmm4, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm3, %xmm4
+; SSE2-NEXT:    pxor %xmm9, %xmm4
+; SSE2-NEXT:    pcmpeqd %xmm4, %xmm2
+; SSE2-NEXT:    paddd %xmm7, %xmm3
+; SSE2-NEXT:    pcmpgtd %xmm3, %xmm10
+; SSE2-NEXT:    pxor %xmm10, %xmm9
+; SSE2-NEXT:    pcmpeqd %xmm4, %xmm9
+; SSE2-NEXT:    pandn %xmm2, %xmm9
+; SSE2-NEXT:    movdqa %xmm10, %xmm2
+; SSE2-NEXT:    pandn %xmm12, %xmm2
+; SSE2-NEXT:    psrld $1, %xmm10
+; SSE2-NEXT:    por %xmm2, %xmm10
+; SSE2-NEXT:    pand %xmm9, %xmm10
+; SSE2-NEXT:    pandn %xmm3, %xmm9
+; SSE2-NEXT:    por %xmm10, %xmm9
+; SSE2-NEXT:    movdqa %xmm8, %xmm2
+; SSE2-NEXT:    movdqa %xmm9, %xmm3
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: v16i32:
+; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    movdqa %xmm1, %xmm8
+; SSSE3-NEXT:    movdqa %xmm0, %xmm11
+; SSSE3-NEXT:    pxor %xmm10, %xmm10
+; SSSE3-NEXT:    pxor %xmm12, %xmm12
+; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm12
+; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm9
+; SSSE3-NEXT:    pxor %xmm9, %xmm12
+; SSSE3-NEXT:    pxor %xmm1, %xmm1
+; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm1
+; SSSE3-NEXT:    pxor %xmm9, %xmm1
+; SSSE3-NEXT:    pcmpeqd %xmm1, %xmm12
+; SSSE3-NEXT:    paddd %xmm4, %xmm11
+; SSSE3-NEXT:    pxor %xmm4, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm11, %xmm4
+; SSSE3-NEXT:    movdqa %xmm4, %xmm0
+; SSSE3-NEXT:    pxor %xmm9, %xmm0
+; SSSE3-NEXT:    pcmpeqd %xmm1, %xmm0
+; SSSE3-NEXT:    pandn %xmm12, %xmm0
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm12 = [2147483648,2147483648,2147483648,2147483648]
+; SSSE3-NEXT:    movdqa %xmm4, %xmm1
+; SSSE3-NEXT:    pandn %xmm12, %xmm1
+; SSSE3-NEXT:    psrld $1, %xmm4
+; SSSE3-NEXT:    por %xmm1, %xmm4
+; SSSE3-NEXT:    pand %xmm0, %xmm4
+; SSSE3-NEXT:    pandn %xmm11, %xmm0
+; SSSE3-NEXT:    por %xmm4, %xmm0
+; SSSE3-NEXT:    pxor %xmm11, %xmm11
+; SSSE3-NEXT:    pcmpgtd %xmm5, %xmm11
+; SSSE3-NEXT:    pxor %xmm9, %xmm11
+; SSSE3-NEXT:    pxor %xmm4, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm8, %xmm4
+; SSSE3-NEXT:    pxor %xmm9, %xmm4
+; SSSE3-NEXT:    pcmpeqd %xmm4, %xmm11
+; SSSE3-NEXT:    paddd %xmm5, %xmm8
+; SSSE3-NEXT:    pxor %xmm5, %xmm5
+; SSSE3-NEXT:    pcmpgtd %xmm8, %xmm5
+; SSSE3-NEXT:    movdqa %xmm5, %xmm1
+; SSSE3-NEXT:    pxor %xmm9, %xmm1
+; SSSE3-NEXT:    pcmpeqd %xmm4, %xmm1
+; SSSE3-NEXT:    pandn %xmm11, %xmm1
+; SSSE3-NEXT:    movdqa %xmm5, %xmm4
+; SSSE3-NEXT:    pandn %xmm12, %xmm4
+; SSSE3-NEXT:    psrld $1, %xmm5
+; SSSE3-NEXT:    por %xmm4, %xmm5
+; SSSE3-NEXT:    pand %xmm1, %xmm5
+; SSSE3-NEXT:    pandn %xmm8, %xmm1
+; SSSE3-NEXT:    por %xmm5, %xmm1
+; SSSE3-NEXT:    pxor %xmm4, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm6, %xmm4
+; SSSE3-NEXT:    pxor %xmm9, %xmm4
+; SSSE3-NEXT:    pxor %xmm5, %xmm5
+; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm5
+; SSSE3-NEXT:    pxor %xmm9, %xmm5
+; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm4
+; SSSE3-NEXT:    paddd %xmm6, %xmm2
+; SSSE3-NEXT:    pxor %xmm6, %xmm6
+; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm6
+; SSSE3-NEXT:    movdqa %xmm6, %xmm8
+; SSSE3-NEXT:    pxor %xmm9, %xmm8
+; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm8
+; SSSE3-NEXT:    pandn %xmm4, %xmm8
+; SSSE3-NEXT:    movdqa %xmm6, %xmm4
+; SSSE3-NEXT:    pandn %xmm12, %xmm4
+; SSSE3-NEXT:    psrld $1, %xmm6
+; SSSE3-NEXT:    por %xmm4, %xmm6
+; SSSE3-NEXT:    pand %xmm8, %xmm6
+; SSSE3-NEXT:    pandn %xmm2, %xmm8
+; SSSE3-NEXT:    por %xmm6, %xmm8
+; SSSE3-NEXT:    pxor %xmm2, %xmm2
+; SSSE3-NEXT:    pcmpgtd %xmm7, %xmm2
+; SSSE3-NEXT:    pxor %xmm9, %xmm2
+; SSSE3-NEXT:    pxor %xmm4, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm4
+; SSSE3-NEXT:    pxor %xmm9, %xmm4
+; SSSE3-NEXT:    pcmpeqd %xmm4, %xmm2
+; SSSE3-NEXT:    paddd %xmm7, %xmm3
+; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm10
+; SSSE3-NEXT:    pxor %xmm10, %xmm9
+; SSSE3-NEXT:    pcmpeqd %xmm4, %xmm9
+; SSSE3-NEXT:    pandn %xmm2, %xmm9
+; SSSE3-NEXT:    movdqa %xmm10, %xmm2
+; SSSE3-NEXT:    pandn %xmm12, %xmm2
+; SSSE3-NEXT:    psrld $1, %xmm10
+; SSSE3-NEXT:    por %xmm2, %xmm10
+; SSSE3-NEXT:    pand %xmm9, %xmm10
+; SSSE3-NEXT:    pandn %xmm3, %xmm9
+; SSSE3-NEXT:    por %xmm10, %xmm9
+; SSSE3-NEXT:    movdqa %xmm8, %xmm2
+; SSSE3-NEXT:    movdqa %xmm9, %xmm3
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: v16i32:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movdqa %xmm0, %xmm8
+; SSE41-NEXT:    pxor %xmm9, %xmm9
+; SSE41-NEXT:    pxor %xmm11, %xmm11
+; SSE41-NEXT:    pcmpgtd %xmm4, %xmm11
+; SSE41-NEXT:    pcmpeqd %xmm10, %xmm10
+; SSE41-NEXT:    pxor %xmm10, %xmm11
+; SSE41-NEXT:    pxor %xmm0, %xmm0
+; SSE41-NEXT:    pcmpgtd %xmm8, %xmm0
+; SSE41-NEXT:    pxor %xmm10, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm11
+; SSE41-NEXT:    paddd %xmm4, %xmm8
+; SSE41-NEXT:    pxor %xmm4, %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm8, %xmm4
+; SSE41-NEXT:    pxor %xmm10, %xmm4
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm4
+; SSE41-NEXT:    pandn %xmm11, %xmm4
+; SSE41-NEXT:    movaps {{.*#+}} xmm12 = [2147483647,2147483647,2147483647,2147483647]
+; SSE41-NEXT:    movaps {{.*#+}} xmm11 = [2147483648,2147483648,2147483648,2147483648]
+; SSE41-NEXT:    movaps %xmm11, %xmm13
+; SSE41-NEXT:    movdqa %xmm8, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm12, %xmm13
+; SSE41-NEXT:    movdqa %xmm4, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm13, %xmm8
+; SSE41-NEXT:    xorps %xmm13, %xmm13
+; SSE41-NEXT:    pcmpgtd %xmm5, %xmm13
+; SSE41-NEXT:    pxor %xmm10, %xmm13
+; SSE41-NEXT:    pxor %xmm0, %xmm0
+; SSE41-NEXT:    pcmpgtd %xmm1, %xmm0
+; SSE41-NEXT:    pxor %xmm10, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm13
+; SSE41-NEXT:    paddd %xmm5, %xmm1
+; SSE41-NEXT:    pxor %xmm4, %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm1, %xmm4
+; SSE41-NEXT:    pxor %xmm10, %xmm4
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm4
+; SSE41-NEXT:    pandn %xmm13, %xmm4
+; SSE41-NEXT:    movaps %xmm11, %xmm5
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm12, %xmm5
+; SSE41-NEXT:    movdqa %xmm4, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm5, %xmm1
+; SSE41-NEXT:    pxor %xmm0, %xmm0
+; SSE41-NEXT:    pcmpgtd %xmm6, %xmm0
+; SSE41-NEXT:    pxor %xmm10, %xmm0
+; SSE41-NEXT:    xorps %xmm5, %xmm5
+; SSE41-NEXT:    pcmpgtd %xmm2, %xmm5
+; SSE41-NEXT:    pxor %xmm10, %xmm5
+; SSE41-NEXT:    pcmpeqd %xmm5, %xmm0
+; SSE41-NEXT:    paddd %xmm6, %xmm2
+; SSE41-NEXT:    pxor %xmm4, %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm2, %xmm4
+; SSE41-NEXT:    pxor %xmm10, %xmm4
+; SSE41-NEXT:    pcmpeqd %xmm5, %xmm4
+; SSE41-NEXT:    pandn %xmm0, %xmm4
+; SSE41-NEXT:    movaps %xmm11, %xmm5
+; SSE41-NEXT:    movdqa %xmm2, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm12, %xmm5
+; SSE41-NEXT:    movdqa %xmm4, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm5, %xmm2
+; SSE41-NEXT:    pxor %xmm0, %xmm0
+; SSE41-NEXT:    pcmpgtd %xmm7, %xmm0
+; SSE41-NEXT:    pxor %xmm10, %xmm0
+; SSE41-NEXT:    pxor %xmm4, %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm3, %xmm4
+; SSE41-NEXT:    pxor %xmm10, %xmm4
+; SSE41-NEXT:    pcmpeqd %xmm4, %xmm0
+; SSE41-NEXT:    paddd %xmm7, %xmm3
+; SSE41-NEXT:    pcmpgtd %xmm3, %xmm9
+; SSE41-NEXT:    pxor %xmm10, %xmm9
+; SSE41-NEXT:    pcmpeqd %xmm4, %xmm9
+; SSE41-NEXT:    pandn %xmm0, %xmm9
+; SSE41-NEXT:    movdqa %xmm3, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm12, %xmm11
+; SSE41-NEXT:    movdqa %xmm9, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm11, %xmm3
+; SSE41-NEXT:    movaps %xmm8, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: v16i32:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm9
+; AVX1-NEXT:    vpxor %xmm12, %xmm12, %xmm12
+; AVX1-NEXT:    vpcmpgtd %xmm9, %xmm12, %xmm7
+; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
+; AVX1-NEXT:    vpxor %xmm4, %xmm7, %xmm8
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm7
+; AVX1-NEXT:    vpcmpgtd %xmm7, %xmm12, %xmm6
+; AVX1-NEXT:    vpxor %xmm4, %xmm6, %xmm10
+; AVX1-NEXT:    vpcmpeqd %xmm8, %xmm10, %xmm8
+; AVX1-NEXT:    vpcmpgtd %xmm2, %xmm12, %xmm6
+; AVX1-NEXT:    vpxor %xmm4, %xmm6, %xmm11
+; AVX1-NEXT:    vpcmpgtd %xmm0, %xmm12, %xmm6
+; AVX1-NEXT:    vpxor %xmm4, %xmm6, %xmm6
+; AVX1-NEXT:    vpcmpeqd %xmm11, %xmm6, %xmm11
+; AVX1-NEXT:    vinsertf128 $1, %xmm8, %ymm11, %ymm8
+; AVX1-NEXT:    vpaddd %xmm9, %xmm7, %xmm9
+; AVX1-NEXT:    vpcmpgtd %xmm9, %xmm12, %xmm7
+; AVX1-NEXT:    vpxor %xmm4, %xmm7, %xmm5
+; AVX1-NEXT:    vpcmpeqd %xmm5, %xmm10, %xmm5
+; AVX1-NEXT:    vpxor %xmm4, %xmm5, %xmm5
+; AVX1-NEXT:    vpaddd %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpcmpgtd %xmm0, %xmm12, %xmm2
+; AVX1-NEXT:    vpxor %xmm4, %xmm2, %xmm2
+; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm6, %xmm2
+; AVX1-NEXT:    vpxor %xmm4, %xmm2, %xmm2
+; AVX1-NEXT:    vinsertf128 $1, %xmm5, %ymm2, %ymm2
+; AVX1-NEXT:    vandps %ymm2, %ymm8, %ymm5
+; AVX1-NEXT:    vinsertf128 $1, %xmm7, %ymm0, %ymm7
+; AVX1-NEXT:    vmovaps {{.*#+}} ymm8 = [2147483647,2147483647,2147483647,2147483647,2147483647,2147483647,2147483647,2147483647]
+; AVX1-NEXT:    vmovaps {{.*#+}} ymm10 = [2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648]
+; AVX1-NEXT:    vblendvps %ymm7, %ymm8, %ymm10, %ymm7
+; AVX1-NEXT:    vinsertf128 $1, %xmm9, %ymm0, %ymm0
+; AVX1-NEXT:    vblendvps %ymm5, %ymm7, %ymm0, %ymm0
+; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm5
+; AVX1-NEXT:    vpcmpgtd %xmm5, %xmm12, %xmm7
+; AVX1-NEXT:    vpxor %xmm4, %xmm7, %xmm7
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; AVX1-NEXT:    vpcmpgtd %xmm2, %xmm12, %xmm6
+; AVX1-NEXT:    vpxor %xmm4, %xmm6, %xmm6
+; AVX1-NEXT:    vpcmpeqd %xmm7, %xmm6, %xmm9
+; AVX1-NEXT:    vpcmpgtd %xmm3, %xmm12, %xmm7
+; AVX1-NEXT:    vpxor %xmm4, %xmm7, %xmm11
+; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm12, %xmm7
+; AVX1-NEXT:    vpxor %xmm4, %xmm7, %xmm7
+; AVX1-NEXT:    vpcmpeqd %xmm11, %xmm7, %xmm11
+; AVX1-NEXT:    vinsertf128 $1, %xmm9, %ymm11, %ymm9
+; AVX1-NEXT:    vpaddd %xmm5, %xmm2, %xmm11
+; AVX1-NEXT:    vpcmpgtd %xmm11, %xmm12, %xmm5
+; AVX1-NEXT:    vpxor %xmm4, %xmm5, %xmm2
+; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm6, %xmm2
+; AVX1-NEXT:    vpxor %xmm4, %xmm2, %xmm2
+; AVX1-NEXT:    vpaddd %xmm3, %xmm1, %xmm1
+; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm12, %xmm3
+; AVX1-NEXT:    vpxor %xmm4, %xmm3, %xmm3
+; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm7, %xmm3
+; AVX1-NEXT:    vpxor %xmm4, %xmm3, %xmm3
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm3, %ymm2
+; AVX1-NEXT:    vandps %ymm2, %ymm9, %ymm2
+; AVX1-NEXT:    vinsertf128 $1, %xmm5, %ymm1, %ymm3
+; AVX1-NEXT:    vblendvps %ymm3, %ymm8, %ymm10, %ymm3
+; AVX1-NEXT:    vinsertf128 $1, %xmm11, %ymm1, %ymm1
+; AVX1-NEXT:    vblendvps %ymm2, %ymm3, %ymm1, %ymm1
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: v16i32:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpxor %xmm4, %xmm4, %xmm4
+; AVX2-NEXT:    vpcmpgtd %ymm2, %ymm4, %ymm5
+; AVX2-NEXT:    vpcmpeqd %ymm6, %ymm6, %ymm6
+; AVX2-NEXT:    vpxor %ymm6, %ymm5, %ymm5
+; AVX2-NEXT:    vpcmpgtd %ymm0, %ymm4, %ymm7
+; AVX2-NEXT:    vpxor %ymm6, %ymm7, %ymm7
+; AVX2-NEXT:    vpcmpeqd %ymm5, %ymm7, %ymm5
+; AVX2-NEXT:    vpaddd %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vpcmpgtd %ymm0, %ymm4, %ymm2
+; AVX2-NEXT:    vpxor %ymm6, %ymm2, %ymm2
+; AVX2-NEXT:    vpcmpeqd %ymm2, %ymm7, %ymm2
+; AVX2-NEXT:    vpandn %ymm5, %ymm2, %ymm2
+; AVX2-NEXT:    vbroadcastss {{.*#+}} ymm5 = [2147483647,2147483647,2147483647,2147483647,2147483647,2147483647,2147483647,2147483647]
+; AVX2-NEXT:    vbroadcastss {{.*#+}} ymm7 = [2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648]
+; AVX2-NEXT:    vblendvps %ymm0, %ymm5, %ymm7, %ymm8
+; AVX2-NEXT:    vblendvps %ymm2, %ymm8, %ymm0, %ymm0
+; AVX2-NEXT:    vpcmpgtd %ymm3, %ymm4, %ymm2
+; AVX2-NEXT:    vpxor %ymm6, %ymm2, %ymm2
+; AVX2-NEXT:    vpcmpgtd %ymm1, %ymm4, %ymm8
+; AVX2-NEXT:    vpxor %ymm6, %ymm8, %ymm8
+; AVX2-NEXT:    vpcmpeqd %ymm2, %ymm8, %ymm2
+; AVX2-NEXT:    vpaddd %ymm3, %ymm1, %ymm1
+; AVX2-NEXT:    vpcmpgtd %ymm1, %ymm4, %ymm3
+; AVX2-NEXT:    vpxor %ymm6, %ymm3, %ymm3
+; AVX2-NEXT:    vpcmpeqd %ymm3, %ymm8, %ymm3
+; AVX2-NEXT:    vpandn %ymm2, %ymm3, %ymm2
+; AVX2-NEXT:    vblendvps %ymm1, %ymm5, %ymm7, %ymm3
+; AVX2-NEXT:    vblendvps %ymm2, %ymm3, %ymm1, %ymm1
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: v16i32:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX512-NEXT:    vpcmpnltd %zmm2, %zmm1, %k0
+; AVX512-NEXT:    vpcmpnltd %zmm2, %zmm0, %k1
+; AVX512-NEXT:    kxorw %k0, %k1, %k0
+; AVX512-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpcmpnltd %zmm2, %zmm0, %k2
+; AVX512-NEXT:    kxorw %k2, %k1, %k1
+; AVX512-NEXT:    kandnw %k1, %k0, %k1
+; AVX512-NEXT:    vpcmpgtd %zmm0, %zmm2, %k2
+; AVX512-NEXT:    vpbroadcastd {{.*#+}} zmm1 = [2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648]
+; AVX512-NEXT:    vpbroadcastd {{.*}}(%rip), %zmm1 {%k2}
+; AVX512-NEXT:    vmovdqa32 %zmm1, %zmm0 {%k1}
+; AVX512-NEXT:    retq
+  %z = call <16 x i32> @llvm.sadd.sat.v16i32(<16 x i32> %x, <16 x i32> %y)
+  ret <16 x i32> %z
+}
+
+define <2 x i64> @v2i64(<2 x i64> %x, <2 x i64> %y) nounwind {
+; SSE2-LABEL: v2i64:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648]
+; SSE2-NEXT:    movdqa %xmm0, %xmm3
+; SSE2-NEXT:    paddq %xmm1, %xmm0
+; SSE2-NEXT:    pxor %xmm2, %xmm1
+; SSE2-NEXT:    movdqa %xmm2, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm1, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm2, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSE2-NEXT:    pand %xmm5, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSE2-NEXT:    por %xmm1, %xmm4
+; SSE2-NEXT:    pcmpeqd %xmm5, %xmm5
+; SSE2-NEXT:    pxor %xmm5, %xmm4
+; SSE2-NEXT:    pxor %xmm2, %xmm3
+; SSE2-NEXT:    movdqa %xmm2, %xmm1
+; SSE2-NEXT:    pcmpgtd %xmm3, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm1[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm2, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
+; SSE2-NEXT:    pand %xmm6, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSE2-NEXT:    por %xmm3, %xmm1
+; SSE2-NEXT:    pxor %xmm5, %xmm1
+; SSE2-NEXT:    pcmpeqd %xmm1, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[1,0,3,2]
+; SSE2-NEXT:    pand %xmm4, %xmm3
+; SSE2-NEXT:    movdqa %xmm0, %xmm4
+; SSE2-NEXT:    pxor %xmm2, %xmm4
+; SSE2-NEXT:    movdqa %xmm2, %xmm6
+; SSE2-NEXT:    pcmpgtd %xmm4, %xmm6
+; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm2, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm4[1,1,3,3]
+; SSE2-NEXT:    pand %xmm7, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm6[1,1,3,3]
+; SSE2-NEXT:    por %xmm2, %xmm4
+; SSE2-NEXT:    pxor %xmm4, %xmm5
+; SSE2-NEXT:    pcmpeqd %xmm1, %xmm5
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm5[1,0,3,2]
+; SSE2-NEXT:    pand %xmm5, %xmm1
+; SSE2-NEXT:    pandn %xmm3, %xmm1
+; SSE2-NEXT:    movdqa %xmm4, %xmm2
+; SSE2-NEXT:    pandn {{.*}}(%rip), %xmm2
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm4
+; SSE2-NEXT:    por %xmm2, %xmm4
+; SSE2-NEXT:    pand %xmm1, %xmm4
+; SSE2-NEXT:    pandn %xmm0, %xmm1
+; SSE2-NEXT:    por %xmm4, %xmm1
+; SSE2-NEXT:    movdqa %xmm1, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: v2i64:
+; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648]
+; SSSE3-NEXT:    movdqa %xmm0, %xmm3
+; SSSE3-NEXT:    paddq %xmm1, %xmm0
+; SSSE3-NEXT:    pxor %xmm2, %xmm1
+; SSSE3-NEXT:    movdqa %xmm2, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm5, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    por %xmm1, %xmm4
+; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm5
+; SSSE3-NEXT:    pxor %xmm5, %xmm4
+; SSSE3-NEXT:    pxor %xmm2, %xmm3
+; SSSE3-NEXT:    movdqa %xmm2, %xmm1
+; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm1[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm3
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm6, %xmm3
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSSE3-NEXT:    por %xmm3, %xmm1
+; SSSE3-NEXT:    pxor %xmm5, %xmm1
+; SSSE3-NEXT:    pcmpeqd %xmm1, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[1,0,3,2]
+; SSSE3-NEXT:    pand %xmm4, %xmm3
+; SSSE3-NEXT:    movdqa %xmm0, %xmm4
+; SSSE3-NEXT:    pxor %xmm2, %xmm4
+; SSSE3-NEXT:    movdqa %xmm2, %xmm6
+; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm6
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm7, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm6[1,1,3,3]
+; SSSE3-NEXT:    por %xmm2, %xmm4
+; SSSE3-NEXT:    pxor %xmm4, %xmm5
+; SSSE3-NEXT:    pcmpeqd %xmm1, %xmm5
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm5[1,0,3,2]
+; SSSE3-NEXT:    pand %xmm5, %xmm1
+; SSSE3-NEXT:    pandn %xmm3, %xmm1
+; SSSE3-NEXT:    movdqa %xmm4, %xmm2
+; SSSE3-NEXT:    pandn {{.*}}(%rip), %xmm2
+; SSSE3-NEXT:    pand {{.*}}(%rip), %xmm4
+; SSSE3-NEXT:    por %xmm2, %xmm4
+; SSSE3-NEXT:    pand %xmm1, %xmm4
+; SSSE3-NEXT:    pandn %xmm0, %xmm1
+; SSSE3-NEXT:    por %xmm4, %xmm1
+; SSSE3-NEXT:    movdqa %xmm1, %xmm0
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: v2i64:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movdqa %xmm0, %xmm2
+; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483648,2147483648]
+; SSE41-NEXT:    movdqa %xmm2, %xmm3
+; SSE41-NEXT:    paddq %xmm1, %xmm2
+; SSE41-NEXT:    pxor %xmm0, %xmm1
+; SSE41-NEXT:    movdqa %xmm0, %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm1, %xmm4
+; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSE41-NEXT:    pand %xmm5, %xmm1
+; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSE41-NEXT:    por %xmm1, %xmm4
+; SSE41-NEXT:    pcmpeqd %xmm1, %xmm1
+; SSE41-NEXT:    pxor %xmm1, %xmm4
+; SSE41-NEXT:    pxor %xmm0, %xmm3
+; SSE41-NEXT:    movdqa %xmm0, %xmm5
+; SSE41-NEXT:    pcmpgtd %xmm3, %xmm5
+; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm3
+; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
+; SSE41-NEXT:    pand %xmm6, %xmm3
+; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
+; SSE41-NEXT:    por %xmm3, %xmm5
+; SSE41-NEXT:    pxor %xmm1, %xmm5
+; SSE41-NEXT:    pcmpeqq %xmm5, %xmm4
+; SSE41-NEXT:    movdqa %xmm2, %xmm3
+; SSE41-NEXT:    pxor %xmm0, %xmm3
+; SSE41-NEXT:    movdqa %xmm0, %xmm6
+; SSE41-NEXT:    pcmpgtd %xmm3, %xmm6
+; SSE41-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm3
+; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
+; SSE41-NEXT:    pand %xmm7, %xmm3
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[1,1,3,3]
+; SSE41-NEXT:    por %xmm3, %xmm0
+; SSE41-NEXT:    pxor %xmm0, %xmm1
+; SSE41-NEXT:    pcmpeqq %xmm5, %xmm1
+; SSE41-NEXT:    pandn %xmm4, %xmm1
+; SSE41-NEXT:    movapd {{.*#+}} xmm3 = [9223372036854775808,9223372036854775808]
+; SSE41-NEXT:    blendvpd %xmm0, {{.*}}(%rip), %xmm3
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm3, %xmm2
+; SSE41-NEXT:    movapd %xmm2, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: v2i64:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm2, %xmm3
+; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
+; AVX1-NEXT:    vpxor %xmm4, %xmm3, %xmm3
+; AVX1-NEXT:    vpcmpgtq %xmm0, %xmm2, %xmm5
+; AVX1-NEXT:    vpxor %xmm4, %xmm5, %xmm5
+; AVX1-NEXT:    vpcmpeqq %xmm3, %xmm5, %xmm3
+; AVX1-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpcmpgtq %xmm0, %xmm2, %xmm1
+; AVX1-NEXT:    vpxor %xmm4, %xmm1, %xmm1
+; AVX1-NEXT:    vpcmpeqq %xmm1, %xmm5, %xmm1
+; AVX1-NEXT:    vpandn %xmm3, %xmm1, %xmm1
+; AVX1-NEXT:    vmovapd {{.*#+}} xmm2 = [9223372036854775808,9223372036854775808]
+; AVX1-NEXT:    vblendvpd %xmm0, {{.*}}(%rip), %xmm2, %xmm2
+; AVX1-NEXT:    vblendvpd %xmm1, %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: v2i64:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX2-NEXT:    vpcmpgtq %xmm1, %xmm2, %xmm3
+; AVX2-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
+; AVX2-NEXT:    vpxor %xmm4, %xmm3, %xmm3
+; AVX2-NEXT:    vpcmpgtq %xmm0, %xmm2, %xmm5
+; AVX2-NEXT:    vpxor %xmm4, %xmm5, %xmm5
+; AVX2-NEXT:    vpcmpeqq %xmm3, %xmm5, %xmm3
+; AVX2-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpcmpgtq %xmm0, %xmm2, %xmm1
+; AVX2-NEXT:    vpxor %xmm4, %xmm1, %xmm1
+; AVX2-NEXT:    vpcmpeqq %xmm1, %xmm5, %xmm1
+; AVX2-NEXT:    vpandn %xmm3, %xmm1, %xmm1
+; AVX2-NEXT:    vmovapd {{.*#+}} xmm2 = [9223372036854775808,9223372036854775808]
+; AVX2-NEXT:    vblendvpd %xmm0, {{.*}}(%rip), %xmm2, %xmm2
+; AVX2-NEXT:    vblendvpd %xmm1, %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: v2i64:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX512-NEXT:    vpcmpnltq %xmm2, %xmm1, %k0
+; AVX512-NEXT:    vpcmpnltq %xmm2, %xmm0, %k1
+; AVX512-NEXT:    kxorw %k0, %k1, %k0
+; AVX512-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vpcmpnltq %xmm2, %xmm0, %k2
+; AVX512-NEXT:    kxorw %k2, %k1, %k1
+; AVX512-NEXT:    kandnw %k1, %k0, %k1
+; AVX512-NEXT:    vpcmpgtq %xmm0, %xmm2, %k2
+; AVX512-NEXT:    vmovdqa {{.*#+}} xmm1 = [9223372036854775808,9223372036854775808]
+; AVX512-NEXT:    vmovdqa64 {{.*}}(%rip), %xmm1 {%k2}
+; AVX512-NEXT:    vmovdqa64 %xmm1, %xmm0 {%k1}
+; AVX512-NEXT:    retq
+  %z = call <2 x i64> @llvm.sadd.sat.v2i64(<2 x i64> %x, <2 x i64> %y)
+  ret <2 x i64> %z
+}
+
+define <4 x i64> @v4i64(<4 x i64> %x, <4 x i64> %y) nounwind {
+; SSE2-LABEL: v4i64:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa %xmm0, %xmm9
+; SSE2-NEXT:    movdqa {{.*#+}} xmm10 = [2147483648,2147483648]
+; SSE2-NEXT:    paddq %xmm2, %xmm9
+; SSE2-NEXT:    pxor %xmm10, %xmm2
+; SSE2-NEXT:    movdqa %xmm10, %xmm6
+; SSE2-NEXT:    pcmpgtd %xmm2, %xmm6
+; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm10, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; SSE2-NEXT:    pand %xmm7, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[1,1,3,3]
+; SSE2-NEXT:    por %xmm2, %xmm7
+; SSE2-NEXT:    pcmpeqd %xmm6, %xmm6
+; SSE2-NEXT:    pxor %xmm6, %xmm7
+; SSE2-NEXT:    pxor %xmm10, %xmm0
+; SSE2-NEXT:    movdqa %xmm10, %xmm2
+; SSE2-NEXT:    pcmpgtd %xmm0, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm10, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE2-NEXT:    pand %xmm4, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; SSE2-NEXT:    por %xmm0, %xmm2
+; SSE2-NEXT:    pxor %xmm6, %xmm2
+; SSE2-NEXT:    pcmpeqd %xmm2, %xmm7
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm7[1,0,3,2]
+; SSE2-NEXT:    pand %xmm7, %xmm4
+; SSE2-NEXT:    movdqa %xmm9, %xmm0
+; SSE2-NEXT:    pxor %xmm10, %xmm0
+; SSE2-NEXT:    movdqa %xmm10, %xmm7
+; SSE2-NEXT:    pcmpgtd %xmm0, %xmm7
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm7[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm10, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE2-NEXT:    pand %xmm5, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm7[1,1,3,3]
+; SSE2-NEXT:    por %xmm0, %xmm5
+; SSE2-NEXT:    movdqa %xmm5, %xmm7
+; SSE2-NEXT:    pxor %xmm6, %xmm7
+; SSE2-NEXT:    pcmpeqd %xmm2, %xmm7
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm7[1,0,3,2]
+; SSE2-NEXT:    pand %xmm7, %xmm0
+; SSE2-NEXT:    pandn %xmm4, %xmm0
+; SSE2-NEXT:    movdqa {{.*#+}} xmm11 = [9223372036854775808,9223372036854775808]
+; SSE2-NEXT:    movdqa %xmm5, %xmm2
+; SSE2-NEXT:    pandn %xmm11, %xmm2
+; SSE2-NEXT:    movdqa {{.*#+}} xmm8 = [9223372036854775807,9223372036854775807]
+; SSE2-NEXT:    pand %xmm8, %xmm5
+; SSE2-NEXT:    por %xmm2, %xmm5
+; SSE2-NEXT:    pand %xmm0, %xmm5
+; SSE2-NEXT:    pandn %xmm9, %xmm0
+; SSE2-NEXT:    por %xmm5, %xmm0
+; SSE2-NEXT:    movdqa %xmm1, %xmm2
+; SSE2-NEXT:    paddq %xmm3, %xmm1
+; SSE2-NEXT:    pxor %xmm10, %xmm3
+; SSE2-NEXT:    movdqa %xmm10, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm3, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm10, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
+; SSE2-NEXT:    pand %xmm5, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSE2-NEXT:    por %xmm3, %xmm4
+; SSE2-NEXT:    pxor %xmm6, %xmm4
+; SSE2-NEXT:    pxor %xmm10, %xmm2
+; SSE2-NEXT:    movdqa %xmm10, %xmm3
+; SSE2-NEXT:    pcmpgtd %xmm2, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm3[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm10, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; SSE2-NEXT:    pand %xmm5, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
+; SSE2-NEXT:    por %xmm2, %xmm3
+; SSE2-NEXT:    pxor %xmm6, %xmm3
+; SSE2-NEXT:    pcmpeqd %xmm3, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[1,0,3,2]
+; SSE2-NEXT:    pand %xmm4, %xmm5
+; SSE2-NEXT:    movdqa %xmm1, %xmm2
+; SSE2-NEXT:    pxor %xmm10, %xmm2
+; SSE2-NEXT:    movdqa %xmm10, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm2, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm4[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm10, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; SSE2-NEXT:    pand %xmm7, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSE2-NEXT:    por %xmm2, %xmm4
+; SSE2-NEXT:    pxor %xmm4, %xmm6
+; SSE2-NEXT:    pcmpeqd %xmm3, %xmm6
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm6[1,0,3,2]
+; SSE2-NEXT:    pand %xmm6, %xmm2
+; SSE2-NEXT:    pandn %xmm5, %xmm2
+; SSE2-NEXT:    movdqa %xmm4, %xmm3
+; SSE2-NEXT:    pandn %xmm11, %xmm3
+; SSE2-NEXT:    pand %xmm8, %xmm4
+; SSE2-NEXT:    por %xmm3, %xmm4
+; SSE2-NEXT:    pand %xmm2, %xmm4
+; SSE2-NEXT:    pandn %xmm1, %xmm2
+; SSE2-NEXT:    por %xmm4, %xmm2
+; SSE2-NEXT:    movdqa %xmm2, %xmm1
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: v4i64:
+; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    movdqa %xmm0, %xmm9
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm10 = [2147483648,2147483648]
+; SSSE3-NEXT:    paddq %xmm2, %xmm9
+; SSSE3-NEXT:    pxor %xmm10, %xmm2
+; SSSE3-NEXT:    movdqa %xmm10, %xmm6
+; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm6
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm10, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm7, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[1,1,3,3]
+; SSSE3-NEXT:    por %xmm2, %xmm7
+; SSSE3-NEXT:    pcmpeqd %xmm6, %xmm6
+; SSSE3-NEXT:    pxor %xmm6, %xmm7
+; SSSE3-NEXT:    pxor %xmm10, %xmm0
+; SSSE3-NEXT:    movdqa %xmm10, %xmm2
+; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm10, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm4, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; SSSE3-NEXT:    por %xmm0, %xmm2
+; SSSE3-NEXT:    pxor %xmm6, %xmm2
+; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm7
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm7[1,0,3,2]
+; SSSE3-NEXT:    pand %xmm7, %xmm4
+; SSSE3-NEXT:    movdqa %xmm9, %xmm0
+; SSSE3-NEXT:    pxor %xmm10, %xmm0
+; SSSE3-NEXT:    movdqa %xmm10, %xmm7
+; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm7
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm7[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm10, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm5, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm7[1,1,3,3]
+; SSSE3-NEXT:    por %xmm0, %xmm5
+; SSSE3-NEXT:    movdqa %xmm5, %xmm7
+; SSSE3-NEXT:    pxor %xmm6, %xmm7
+; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm7
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm7[1,0,3,2]
+; SSSE3-NEXT:    pand %xmm7, %xmm0
+; SSSE3-NEXT:    pandn %xmm4, %xmm0
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm11 = [9223372036854775808,9223372036854775808]
+; SSSE3-NEXT:    movdqa %xmm5, %xmm2
+; SSSE3-NEXT:    pandn %xmm11, %xmm2
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm8 = [9223372036854775807,9223372036854775807]
+; SSSE3-NEXT:    pand %xmm8, %xmm5
+; SSSE3-NEXT:    por %xmm2, %xmm5
+; SSSE3-NEXT:    pand %xmm0, %xmm5
+; SSSE3-NEXT:    pandn %xmm9, %xmm0
+; SSSE3-NEXT:    por %xmm5, %xmm0
+; SSSE3-NEXT:    movdqa %xmm1, %xmm2
+; SSSE3-NEXT:    paddq %xmm3, %xmm1
+; SSSE3-NEXT:    pxor %xmm10, %xmm3
+; SSSE3-NEXT:    movdqa %xmm10, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm10, %xmm3
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm5, %xmm3
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    por %xmm3, %xmm4
+; SSSE3-NEXT:    pxor %xmm6, %xmm4
+; SSSE3-NEXT:    pxor %xmm10, %xmm2
+; SSSE3-NEXT:    movdqa %xmm10, %xmm3
+; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm3
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm3[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm10, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm5, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
+; SSSE3-NEXT:    por %xmm2, %xmm3
+; SSSE3-NEXT:    pxor %xmm6, %xmm3
+; SSSE3-NEXT:    pcmpeqd %xmm3, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[1,0,3,2]
+; SSSE3-NEXT:    pand %xmm4, %xmm5
+; SSSE3-NEXT:    movdqa %xmm1, %xmm2
+; SSSE3-NEXT:    pxor %xmm10, %xmm2
+; SSSE3-NEXT:    movdqa %xmm10, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm4[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm10, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm7, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    por %xmm2, %xmm4
+; SSSE3-NEXT:    pxor %xmm4, %xmm6
+; SSSE3-NEXT:    pcmpeqd %xmm3, %xmm6
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm6[1,0,3,2]
+; SSSE3-NEXT:    pand %xmm6, %xmm2
+; SSSE3-NEXT:    pandn %xmm5, %xmm2
+; SSSE3-NEXT:    movdqa %xmm4, %xmm3
+; SSSE3-NEXT:    pandn %xmm11, %xmm3
+; SSSE3-NEXT:    pand %xmm8, %xmm4
+; SSSE3-NEXT:    por %xmm3, %xmm4
+; SSSE3-NEXT:    pand %xmm2, %xmm4
+; SSSE3-NEXT:    pandn %xmm1, %xmm2
+; SSSE3-NEXT:    por %xmm4, %xmm2
+; SSSE3-NEXT:    movdqa %xmm2, %xmm1
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: v4i64:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movdqa %xmm0, %xmm9
+; SSE41-NEXT:    movdqa {{.*#+}} xmm10 = [2147483648,2147483648]
+; SSE41-NEXT:    paddq %xmm2, %xmm9
+; SSE41-NEXT:    pxor %xmm10, %xmm2
+; SSE41-NEXT:    movdqa %xmm10, %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm2, %xmm4
+; SSE41-NEXT:    pshufd {{.*#+}} xmm7 = xmm4[0,0,2,2]
+; SSE41-NEXT:    pcmpeqd %xmm10, %xmm2
+; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; SSE41-NEXT:    pand %xmm7, %xmm2
+; SSE41-NEXT:    pshufd {{.*#+}} xmm7 = xmm4[1,1,3,3]
+; SSE41-NEXT:    por %xmm2, %xmm7
+; SSE41-NEXT:    pcmpeqd %xmm4, %xmm4
+; SSE41-NEXT:    pxor %xmm4, %xmm7
+; SSE41-NEXT:    pxor %xmm10, %xmm0
+; SSE41-NEXT:    movdqa %xmm10, %xmm2
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm2
+; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm2[0,0,2,2]
+; SSE41-NEXT:    pcmpeqd %xmm10, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE41-NEXT:    pand %xmm5, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm2[1,1,3,3]
+; SSE41-NEXT:    por %xmm0, %xmm5
+; SSE41-NEXT:    pxor %xmm4, %xmm5
+; SSE41-NEXT:    pcmpeqq %xmm5, %xmm7
+; SSE41-NEXT:    movdqa %xmm9, %xmm0
+; SSE41-NEXT:    pxor %xmm10, %xmm0
+; SSE41-NEXT:    movdqa %xmm10, %xmm2
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm2
+; SSE41-NEXT:    pshufd {{.*#+}} xmm8 = xmm2[0,0,2,2]
+; SSE41-NEXT:    pcmpeqd %xmm10, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm0[1,1,3,3]
+; SSE41-NEXT:    pand %xmm8, %xmm6
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,3,3]
+; SSE41-NEXT:    por %xmm6, %xmm0
+; SSE41-NEXT:    movdqa %xmm0, %xmm2
+; SSE41-NEXT:    pxor %xmm4, %xmm2
+; SSE41-NEXT:    pcmpeqq %xmm5, %xmm2
+; SSE41-NEXT:    pandn %xmm7, %xmm2
+; SSE41-NEXT:    movapd {{.*#+}} xmm8 = [9223372036854775807,9223372036854775807]
+; SSE41-NEXT:    movapd {{.*#+}} xmm11 = [9223372036854775808,9223372036854775808]
+; SSE41-NEXT:    movapd %xmm11, %xmm5
+; SSE41-NEXT:    blendvpd %xmm0, %xmm8, %xmm5
+; SSE41-NEXT:    movdqa %xmm2, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm5, %xmm9
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    paddq %xmm3, %xmm1
+; SSE41-NEXT:    pxor %xmm10, %xmm3
+; SSE41-NEXT:    movdqa %xmm10, %xmm2
+; SSE41-NEXT:    pcmpgtd %xmm3, %xmm2
+; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm2[0,0,2,2]
+; SSE41-NEXT:    pcmpeqd %xmm10, %xmm3
+; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
+; SSE41-NEXT:    pand %xmm5, %xmm3
+; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; SSE41-NEXT:    por %xmm3, %xmm2
+; SSE41-NEXT:    pxor %xmm4, %xmm2
+; SSE41-NEXT:    pxor %xmm10, %xmm0
+; SSE41-NEXT:    movdqa %xmm10, %xmm3
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm3
+; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm3[0,0,2,2]
+; SSE41-NEXT:    pcmpeqd %xmm10, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE41-NEXT:    pand %xmm5, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
+; SSE41-NEXT:    por %xmm0, %xmm3
+; SSE41-NEXT:    pxor %xmm4, %xmm3
+; SSE41-NEXT:    pcmpeqq %xmm3, %xmm2
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    pxor %xmm10, %xmm0
+; SSE41-NEXT:    movdqa %xmm10, %xmm5
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm5
+; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
+; SSE41-NEXT:    pcmpeqd %xmm10, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm7 = xmm0[1,1,3,3]
+; SSE41-NEXT:    pand %xmm6, %xmm7
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm5[1,1,3,3]
+; SSE41-NEXT:    por %xmm7, %xmm0
+; SSE41-NEXT:    pxor %xmm0, %xmm4
+; SSE41-NEXT:    pcmpeqq %xmm3, %xmm4
+; SSE41-NEXT:    pandn %xmm2, %xmm4
+; SSE41-NEXT:    blendvpd %xmm0, %xmm8, %xmm11
+; SSE41-NEXT:    movdqa %xmm4, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm11, %xmm1
+; SSE41-NEXT:    movapd %xmm9, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: v4i64:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; AVX1-NEXT:    vpcmpgtq %xmm2, %xmm3, %xmm4
+; AVX1-NEXT:    vpcmpeqd %xmm5, %xmm5, %xmm5
+; AVX1-NEXT:    vpxor %xmm5, %xmm4, %xmm4
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm6
+; AVX1-NEXT:    vpcmpgtq %xmm6, %xmm3, %xmm7
+; AVX1-NEXT:    vpxor %xmm5, %xmm7, %xmm7
+; AVX1-NEXT:    vpcmpeqq %xmm4, %xmm7, %xmm8
+; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm3, %xmm4
+; AVX1-NEXT:    vpxor %xmm5, %xmm4, %xmm9
+; AVX1-NEXT:    vpcmpgtq %xmm0, %xmm3, %xmm4
+; AVX1-NEXT:    vpxor %xmm5, %xmm4, %xmm4
+; AVX1-NEXT:    vpcmpeqq %xmm9, %xmm4, %xmm9
+; AVX1-NEXT:    vinsertf128 $1, %xmm8, %ymm9, %ymm8
+; AVX1-NEXT:    vpaddq %xmm2, %xmm6, %xmm9
+; AVX1-NEXT:    vpcmpgtq %xmm9, %xmm3, %xmm6
+; AVX1-NEXT:    vpxor %xmm5, %xmm6, %xmm2
+; AVX1-NEXT:    vpcmpeqq %xmm2, %xmm7, %xmm2
+; AVX1-NEXT:    vpxor %xmm5, %xmm2, %xmm2
+; AVX1-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpcmpgtq %xmm0, %xmm3, %xmm1
+; AVX1-NEXT:    vpxor %xmm5, %xmm1, %xmm1
+; AVX1-NEXT:    vpcmpeqq %xmm1, %xmm4, %xmm1
+; AVX1-NEXT:    vpxor %xmm5, %xmm1, %xmm1
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
+; AVX1-NEXT:    vandpd %ymm1, %ymm8, %ymm1
+; AVX1-NEXT:    vinsertf128 $1, %xmm6, %ymm0, %ymm2
+; AVX1-NEXT:    vmovapd {{.*#+}} ymm3 = [9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808]
+; AVX1-NEXT:    vblendvpd %ymm2, {{.*}}(%rip), %ymm3, %ymm2
+; AVX1-NEXT:    vinsertf128 $1, %xmm9, %ymm0, %ymm0
+; AVX1-NEXT:    vblendvpd %ymm1, %ymm2, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: v4i64:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX2-NEXT:    vpcmpgtq %ymm1, %ymm2, %ymm3
+; AVX2-NEXT:    vpcmpeqd %ymm4, %ymm4, %ymm4
+; AVX2-NEXT:    vpxor %ymm4, %ymm3, %ymm3
+; AVX2-NEXT:    vpcmpgtq %ymm0, %ymm2, %ymm5
+; AVX2-NEXT:    vpxor %ymm4, %ymm5, %ymm5
+; AVX2-NEXT:    vpcmpeqq %ymm3, %ymm5, %ymm3
+; AVX2-NEXT:    vpaddq %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpcmpgtq %ymm0, %ymm2, %ymm1
+; AVX2-NEXT:    vpxor %ymm4, %ymm1, %ymm1
+; AVX2-NEXT:    vpcmpeqq %ymm1, %ymm5, %ymm1
+; AVX2-NEXT:    vpandn %ymm3, %ymm1, %ymm1
+; AVX2-NEXT:    vbroadcastsd {{.*#+}} ymm2 = [9223372036854775807,9223372036854775807,9223372036854775807,9223372036854775807]
+; AVX2-NEXT:    vbroadcastsd {{.*#+}} ymm3 = [9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808]
+; AVX2-NEXT:    vblendvpd %ymm0, %ymm2, %ymm3, %ymm2
+; AVX2-NEXT:    vblendvpd %ymm1, %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: v4i64:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX512-NEXT:    vpcmpnltq %ymm2, %ymm1, %k0
+; AVX512-NEXT:    vpcmpnltq %ymm2, %ymm0, %k1
+; AVX512-NEXT:    kxorw %k0, %k1, %k0
+; AVX512-NEXT:    vpaddq %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpcmpnltq %ymm2, %ymm0, %k2
+; AVX512-NEXT:    kxorw %k2, %k1, %k1
+; AVX512-NEXT:    kandnw %k1, %k0, %k1
+; AVX512-NEXT:    vpcmpgtq %ymm0, %ymm2, %k2
+; AVX512-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808]
+; AVX512-NEXT:    vpbroadcastq {{.*}}(%rip), %ymm1 {%k2}
+; AVX512-NEXT:    vmovdqa64 %ymm1, %ymm0 {%k1}
+; AVX512-NEXT:    retq
+  %z = call <4 x i64> @llvm.sadd.sat.v4i64(<4 x i64> %x, <4 x i64> %y)
+  ret <4 x i64> %z
+}
+
+define <8 x i64> @v8i64(<8 x i64> %x, <8 x i64> %y) nounwind {
+; SSE2-LABEL: v8i64:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa %xmm1, %xmm8
+; SSE2-NEXT:    movdqa %xmm0, %xmm13
+; SSE2-NEXT:    movdqa {{.*#+}} xmm9 = [2147483648,2147483648]
+; SSE2-NEXT:    paddq %xmm4, %xmm13
+; SSE2-NEXT:    pxor %xmm9, %xmm4
+; SSE2-NEXT:    movdqa %xmm9, %xmm1
+; SSE2-NEXT:    pcmpgtd %xmm4, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm10 = xmm1[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm9, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSE2-NEXT:    pand %xmm10, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSE2-NEXT:    por %xmm4, %xmm1
+; SSE2-NEXT:    pcmpeqd %xmm10, %xmm10
+; SSE2-NEXT:    pxor %xmm10, %xmm1
+; SSE2-NEXT:    pxor %xmm9, %xmm0
+; SSE2-NEXT:    movdqa %xmm9, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm0, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm11 = xmm4[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm9, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE2-NEXT:    pand %xmm11, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm14 = xmm4[1,1,3,3]
+; SSE2-NEXT:    por %xmm0, %xmm14
+; SSE2-NEXT:    pxor %xmm10, %xmm14
+; SSE2-NEXT:    pcmpeqd %xmm14, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm11 = xmm1[1,0,3,2]
+; SSE2-NEXT:    pand %xmm1, %xmm11
+; SSE2-NEXT:    movdqa %xmm13, %xmm0
+; SSE2-NEXT:    pxor %xmm9, %xmm0
+; SSE2-NEXT:    movdqa %xmm9, %xmm1
+; SSE2-NEXT:    pcmpgtd %xmm0, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm12 = xmm1[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm9, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE2-NEXT:    pand %xmm12, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSE2-NEXT:    por %xmm0, %xmm1
+; SSE2-NEXT:    movdqa %xmm1, %xmm4
+; SSE2-NEXT:    pxor %xmm10, %xmm4
+; SSE2-NEXT:    pcmpeqd %xmm14, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[1,0,3,2]
+; SSE2-NEXT:    pand %xmm4, %xmm0
+; SSE2-NEXT:    pandn %xmm11, %xmm0
+; SSE2-NEXT:    movdqa {{.*#+}} xmm11 = [9223372036854775808,9223372036854775808]
+; SSE2-NEXT:    movdqa %xmm1, %xmm4
+; SSE2-NEXT:    pandn %xmm11, %xmm4
+; SSE2-NEXT:    movdqa {{.*#+}} xmm12 = [9223372036854775807,9223372036854775807]
+; SSE2-NEXT:    pand %xmm12, %xmm1
+; SSE2-NEXT:    por %xmm4, %xmm1
+; SSE2-NEXT:    pand %xmm0, %xmm1
+; SSE2-NEXT:    pandn %xmm13, %xmm0
+; SSE2-NEXT:    por %xmm1, %xmm0
+; SSE2-NEXT:    movdqa %xmm8, %xmm1
+; SSE2-NEXT:    paddq %xmm5, %xmm8
+; SSE2-NEXT:    pxor %xmm9, %xmm5
+; SSE2-NEXT:    movdqa %xmm9, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm5, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm13 = xmm4[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm9, %xmm5
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
+; SSE2-NEXT:    pand %xmm13, %xmm5
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSE2-NEXT:    por %xmm5, %xmm4
+; SSE2-NEXT:    pxor %xmm10, %xmm4
+; SSE2-NEXT:    pxor %xmm9, %xmm1
+; SSE2-NEXT:    movdqa %xmm9, %xmm5
+; SSE2-NEXT:    pcmpgtd %xmm1, %xmm5
+; SSE2-NEXT:    pshufd {{.*#+}} xmm13 = xmm5[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm9, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSE2-NEXT:    pand %xmm13, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm15 = xmm5[1,1,3,3]
+; SSE2-NEXT:    por %xmm1, %xmm15
+; SSE2-NEXT:    pxor %xmm10, %xmm15
+; SSE2-NEXT:    pcmpeqd %xmm15, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm13 = xmm4[1,0,3,2]
+; SSE2-NEXT:    pand %xmm4, %xmm13
+; SSE2-NEXT:    movdqa %xmm8, %xmm1
+; SSE2-NEXT:    pxor %xmm9, %xmm1
+; SSE2-NEXT:    movdqa %xmm9, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm1, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm14 = xmm4[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm9, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSE2-NEXT:    pand %xmm14, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSE2-NEXT:    por %xmm1, %xmm4
+; SSE2-NEXT:    movdqa %xmm4, %xmm5
+; SSE2-NEXT:    pxor %xmm10, %xmm5
+; SSE2-NEXT:    pcmpeqd %xmm15, %xmm5
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm5[1,0,3,2]
+; SSE2-NEXT:    pand %xmm5, %xmm1
+; SSE2-NEXT:    pandn %xmm13, %xmm1
+; SSE2-NEXT:    movdqa %xmm4, %xmm5
+; SSE2-NEXT:    pandn %xmm11, %xmm5
+; SSE2-NEXT:    pand %xmm12, %xmm4
+; SSE2-NEXT:    por %xmm5, %xmm4
+; SSE2-NEXT:    pand %xmm1, %xmm4
+; SSE2-NEXT:    pandn %xmm8, %xmm1
+; SSE2-NEXT:    por %xmm4, %xmm1
+; SSE2-NEXT:    movdqa %xmm2, %xmm4
+; SSE2-NEXT:    paddq %xmm6, %xmm2
+; SSE2-NEXT:    pxor %xmm9, %xmm6
+; SSE2-NEXT:    movdqa %xmm9, %xmm5
+; SSE2-NEXT:    pcmpgtd %xmm6, %xmm5
+; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm5[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm9, %xmm6
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
+; SSE2-NEXT:    pand %xmm8, %xmm6
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
+; SSE2-NEXT:    por %xmm6, %xmm5
+; SSE2-NEXT:    pxor %xmm10, %xmm5
+; SSE2-NEXT:    pxor %xmm9, %xmm4
+; SSE2-NEXT:    movdqa %xmm9, %xmm6
+; SSE2-NEXT:    pcmpgtd %xmm4, %xmm6
+; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm6[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm9, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSE2-NEXT:    pand %xmm8, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
+; SSE2-NEXT:    por %xmm4, %xmm6
+; SSE2-NEXT:    pxor %xmm10, %xmm6
+; SSE2-NEXT:    pcmpeqd %xmm6, %xmm5
+; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm5[1,0,3,2]
+; SSE2-NEXT:    pand %xmm5, %xmm8
+; SSE2-NEXT:    movdqa %xmm2, %xmm5
+; SSE2-NEXT:    pxor %xmm9, %xmm5
+; SSE2-NEXT:    movdqa %xmm9, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm5, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm13 = xmm4[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm9, %xmm5
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
+; SSE2-NEXT:    pand %xmm13, %xmm5
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSE2-NEXT:    por %xmm5, %xmm4
+; SSE2-NEXT:    movdqa %xmm4, %xmm5
+; SSE2-NEXT:    pxor %xmm10, %xmm5
+; SSE2-NEXT:    pcmpeqd %xmm6, %xmm5
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[1,0,3,2]
+; SSE2-NEXT:    pand %xmm5, %xmm6
+; SSE2-NEXT:    pandn %xmm8, %xmm6
+; SSE2-NEXT:    movdqa %xmm4, %xmm5
+; SSE2-NEXT:    pandn %xmm11, %xmm5
+; SSE2-NEXT:    pand %xmm12, %xmm4
+; SSE2-NEXT:    por %xmm5, %xmm4
+; SSE2-NEXT:    pand %xmm6, %xmm4
+; SSE2-NEXT:    pandn %xmm2, %xmm6
+; SSE2-NEXT:    por %xmm4, %xmm6
+; SSE2-NEXT:    movdqa %xmm3, %xmm2
+; SSE2-NEXT:    paddq %xmm7, %xmm3
+; SSE2-NEXT:    pxor %xmm9, %xmm7
+; SSE2-NEXT:    movdqa %xmm9, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm7, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm9, %xmm7
+; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
+; SSE2-NEXT:    pand %xmm5, %xmm7
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSE2-NEXT:    por %xmm7, %xmm4
+; SSE2-NEXT:    pxor %xmm10, %xmm4
+; SSE2-NEXT:    pxor %xmm9, %xmm2
+; SSE2-NEXT:    movdqa %xmm9, %xmm5
+; SSE2-NEXT:    pcmpgtd %xmm2, %xmm5
+; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm5[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm9, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; SSE2-NEXT:    pand %xmm7, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
+; SSE2-NEXT:    por %xmm2, %xmm5
+; SSE2-NEXT:    pxor %xmm10, %xmm5
+; SSE2-NEXT:    pcmpeqd %xmm5, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm4[1,0,3,2]
+; SSE2-NEXT:    pand %xmm4, %xmm2
+; SSE2-NEXT:    movdqa %xmm3, %xmm4
+; SSE2-NEXT:    pxor %xmm9, %xmm4
+; SSE2-NEXT:    movdqa %xmm9, %xmm7
+; SSE2-NEXT:    pcmpgtd %xmm4, %xmm7
+; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm7[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm9, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSE2-NEXT:    pand %xmm8, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
+; SSE2-NEXT:    por %xmm4, %xmm7
+; SSE2-NEXT:    pxor %xmm7, %xmm10
+; SSE2-NEXT:    pcmpeqd %xmm5, %xmm10
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm10[1,0,3,2]
+; SSE2-NEXT:    pand %xmm10, %xmm5
+; SSE2-NEXT:    pandn %xmm2, %xmm5
+; SSE2-NEXT:    movdqa %xmm7, %xmm2
+; SSE2-NEXT:    pandn %xmm11, %xmm2
+; SSE2-NEXT:    pand %xmm12, %xmm7
+; SSE2-NEXT:    por %xmm2, %xmm7
+; SSE2-NEXT:    pand %xmm5, %xmm7
+; SSE2-NEXT:    pandn %xmm3, %xmm5
+; SSE2-NEXT:    por %xmm7, %xmm5
+; SSE2-NEXT:    movdqa %xmm6, %xmm2
+; SSE2-NEXT:    movdqa %xmm5, %xmm3
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: v8i64:
+; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    movdqa %xmm1, %xmm8
+; SSSE3-NEXT:    movdqa %xmm0, %xmm13
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm9 = [2147483648,2147483648]
+; SSSE3-NEXT:    paddq %xmm4, %xmm13
+; SSSE3-NEXT:    pxor %xmm9, %xmm4
+; SSSE3-NEXT:    movdqa %xmm9, %xmm1
+; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm10 = xmm1[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm10, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSSE3-NEXT:    por %xmm4, %xmm1
+; SSSE3-NEXT:    pcmpeqd %xmm10, %xmm10
+; SSSE3-NEXT:    pxor %xmm10, %xmm1
+; SSSE3-NEXT:    pxor %xmm9, %xmm0
+; SSSE3-NEXT:    movdqa %xmm9, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm11 = xmm4[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm11, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm14 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    por %xmm0, %xmm14
+; SSSE3-NEXT:    pxor %xmm10, %xmm14
+; SSSE3-NEXT:    pcmpeqd %xmm14, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm11 = xmm1[1,0,3,2]
+; SSSE3-NEXT:    pand %xmm1, %xmm11
+; SSSE3-NEXT:    movdqa %xmm13, %xmm0
+; SSSE3-NEXT:    pxor %xmm9, %xmm0
+; SSSE3-NEXT:    movdqa %xmm9, %xmm1
+; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm12 = xmm1[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm12, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSSE3-NEXT:    por %xmm0, %xmm1
+; SSSE3-NEXT:    movdqa %xmm1, %xmm4
+; SSSE3-NEXT:    pxor %xmm10, %xmm4
+; SSSE3-NEXT:    pcmpeqd %xmm14, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[1,0,3,2]
+; SSSE3-NEXT:    pand %xmm4, %xmm0
+; SSSE3-NEXT:    pandn %xmm11, %xmm0
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm11 = [9223372036854775808,9223372036854775808]
+; SSSE3-NEXT:    movdqa %xmm1, %xmm4
+; SSSE3-NEXT:    pandn %xmm11, %xmm4
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm12 = [9223372036854775807,9223372036854775807]
+; SSSE3-NEXT:    pand %xmm12, %xmm1
+; SSSE3-NEXT:    por %xmm4, %xmm1
+; SSSE3-NEXT:    pand %xmm0, %xmm1
+; SSSE3-NEXT:    pandn %xmm13, %xmm0
+; SSSE3-NEXT:    por %xmm1, %xmm0
+; SSSE3-NEXT:    movdqa %xmm8, %xmm1
+; SSSE3-NEXT:    paddq %xmm5, %xmm8
+; SSSE3-NEXT:    pxor %xmm9, %xmm5
+; SSSE3-NEXT:    movdqa %xmm9, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm5, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm13 = xmm4[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm5
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm13, %xmm5
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    por %xmm5, %xmm4
+; SSSE3-NEXT:    pxor %xmm10, %xmm4
+; SSSE3-NEXT:    pxor %xmm9, %xmm1
+; SSSE3-NEXT:    movdqa %xmm9, %xmm5
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm5
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm13 = xmm5[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm13, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm15 = xmm5[1,1,3,3]
+; SSSE3-NEXT:    por %xmm1, %xmm15
+; SSSE3-NEXT:    pxor %xmm10, %xmm15
+; SSSE3-NEXT:    pcmpeqd %xmm15, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm13 = xmm4[1,0,3,2]
+; SSSE3-NEXT:    pand %xmm4, %xmm13
+; SSSE3-NEXT:    movdqa %xmm8, %xmm1
+; SSSE3-NEXT:    pxor %xmm9, %xmm1
+; SSSE3-NEXT:    movdqa %xmm9, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm14 = xmm4[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm14, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    por %xmm1, %xmm4
+; SSSE3-NEXT:    movdqa %xmm4, %xmm5
+; SSSE3-NEXT:    pxor %xmm10, %xmm5
+; SSSE3-NEXT:    pcmpeqd %xmm15, %xmm5
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm5[1,0,3,2]
+; SSSE3-NEXT:    pand %xmm5, %xmm1
+; SSSE3-NEXT:    pandn %xmm13, %xmm1
+; SSSE3-NEXT:    movdqa %xmm4, %xmm5
+; SSSE3-NEXT:    pandn %xmm11, %xmm5
+; SSSE3-NEXT:    pand %xmm12, %xmm4
+; SSSE3-NEXT:    por %xmm5, %xmm4
+; SSSE3-NEXT:    pand %xmm1, %xmm4
+; SSSE3-NEXT:    pandn %xmm8, %xmm1
+; SSSE3-NEXT:    por %xmm4, %xmm1
+; SSSE3-NEXT:    movdqa %xmm2, %xmm4
+; SSSE3-NEXT:    paddq %xmm6, %xmm2
+; SSSE3-NEXT:    pxor %xmm9, %xmm6
+; SSSE3-NEXT:    movdqa %xmm9, %xmm5
+; SSSE3-NEXT:    pcmpgtd %xmm6, %xmm5
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm8 = xmm5[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm6
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm8, %xmm6
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
+; SSSE3-NEXT:    por %xmm6, %xmm5
+; SSSE3-NEXT:    pxor %xmm10, %xmm5
+; SSSE3-NEXT:    pxor %xmm9, %xmm4
+; SSSE3-NEXT:    movdqa %xmm9, %xmm6
+; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm6
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm8 = xmm6[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm8, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
+; SSSE3-NEXT:    por %xmm4, %xmm6
+; SSSE3-NEXT:    pxor %xmm10, %xmm6
+; SSSE3-NEXT:    pcmpeqd %xmm6, %xmm5
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm8 = xmm5[1,0,3,2]
+; SSSE3-NEXT:    pand %xmm5, %xmm8
+; SSSE3-NEXT:    movdqa %xmm2, %xmm5
+; SSSE3-NEXT:    pxor %xmm9, %xmm5
+; SSSE3-NEXT:    movdqa %xmm9, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm5, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm13 = xmm4[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm5
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm13, %xmm5
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    por %xmm5, %xmm4
+; SSSE3-NEXT:    movdqa %xmm4, %xmm5
+; SSSE3-NEXT:    pxor %xmm10, %xmm5
+; SSSE3-NEXT:    pcmpeqd %xmm6, %xmm5
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[1,0,3,2]
+; SSSE3-NEXT:    pand %xmm5, %xmm6
+; SSSE3-NEXT:    pandn %xmm8, %xmm6
+; SSSE3-NEXT:    movdqa %xmm4, %xmm5
+; SSSE3-NEXT:    pandn %xmm11, %xmm5
+; SSSE3-NEXT:    pand %xmm12, %xmm4
+; SSSE3-NEXT:    por %xmm5, %xmm4
+; SSSE3-NEXT:    pand %xmm6, %xmm4
+; SSSE3-NEXT:    pandn %xmm2, %xmm6
+; SSSE3-NEXT:    por %xmm4, %xmm6
+; SSSE3-NEXT:    movdqa %xmm3, %xmm2
+; SSSE3-NEXT:    paddq %xmm7, %xmm3
+; SSSE3-NEXT:    pxor %xmm9, %xmm7
+; SSSE3-NEXT:    movdqa %xmm9, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm7, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm7
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm5, %xmm7
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    por %xmm7, %xmm4
+; SSSE3-NEXT:    pxor %xmm10, %xmm4
+; SSSE3-NEXT:    pxor %xmm9, %xmm2
+; SSSE3-NEXT:    movdqa %xmm9, %xmm5
+; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm5
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm5[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm7, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
+; SSSE3-NEXT:    por %xmm2, %xmm5
+; SSSE3-NEXT:    pxor %xmm10, %xmm5
+; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm4[1,0,3,2]
+; SSSE3-NEXT:    pand %xmm4, %xmm2
+; SSSE3-NEXT:    movdqa %xmm3, %xmm4
+; SSSE3-NEXT:    pxor %xmm9, %xmm4
+; SSSE3-NEXT:    movdqa %xmm9, %xmm7
+; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm7
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm8 = xmm7[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm8, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
+; SSSE3-NEXT:    por %xmm4, %xmm7
+; SSSE3-NEXT:    pxor %xmm7, %xmm10
+; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm10
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm10[1,0,3,2]
+; SSSE3-NEXT:    pand %xmm10, %xmm5
+; SSSE3-NEXT:    pandn %xmm2, %xmm5
+; SSSE3-NEXT:    movdqa %xmm7, %xmm2
+; SSSE3-NEXT:    pandn %xmm11, %xmm2
+; SSSE3-NEXT:    pand %xmm12, %xmm7
+; SSSE3-NEXT:    por %xmm2, %xmm7
+; SSSE3-NEXT:    pand %xmm5, %xmm7
+; SSSE3-NEXT:    pandn %xmm3, %xmm5
+; SSSE3-NEXT:    por %xmm7, %xmm5
+; SSSE3-NEXT:    movdqa %xmm6, %xmm2
+; SSSE3-NEXT:    movdqa %xmm5, %xmm3
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: v8i64:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movdqa %xmm0, %xmm8
+; SSE41-NEXT:    movdqa {{.*#+}} xmm10 = [2147483648,2147483648]
+; SSE41-NEXT:    movdqa %xmm0, %xmm11
+; SSE41-NEXT:    paddq %xmm4, %xmm8
+; SSE41-NEXT:    pxor %xmm10, %xmm4
+; SSE41-NEXT:    movdqa %xmm10, %xmm0
+; SSE41-NEXT:    pcmpgtd %xmm4, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm9 = xmm0[0,0,2,2]
+; SSE41-NEXT:    pcmpeqd %xmm10, %xmm4
+; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSE41-NEXT:    pand %xmm9, %xmm4
+; SSE41-NEXT:    pshufd {{.*#+}} xmm12 = xmm0[1,1,3,3]
+; SSE41-NEXT:    por %xmm4, %xmm12
+; SSE41-NEXT:    pcmpeqd %xmm9, %xmm9
+; SSE41-NEXT:    pxor %xmm9, %xmm12
+; SSE41-NEXT:    pxor %xmm10, %xmm11
+; SSE41-NEXT:    movdqa %xmm10, %xmm0
+; SSE41-NEXT:    pcmpgtd %xmm11, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm13 = xmm0[0,0,2,2]
+; SSE41-NEXT:    pcmpeqd %xmm10, %xmm11
+; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm11[1,1,3,3]
+; SSE41-NEXT:    pand %xmm13, %xmm4
+; SSE41-NEXT:    pshufd {{.*#+}} xmm11 = xmm0[1,1,3,3]
+; SSE41-NEXT:    por %xmm4, %xmm11
+; SSE41-NEXT:    pxor %xmm9, %xmm11
+; SSE41-NEXT:    pcmpeqq %xmm11, %xmm12
+; SSE41-NEXT:    movdqa %xmm8, %xmm0
+; SSE41-NEXT:    pxor %xmm10, %xmm0
+; SSE41-NEXT:    movdqa %xmm10, %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm4
+; SSE41-NEXT:    pshufd {{.*#+}} xmm13 = xmm4[0,0,2,2]
+; SSE41-NEXT:    pcmpeqd %xmm10, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm14 = xmm0[1,1,3,3]
+; SSE41-NEXT:    pand %xmm13, %xmm14
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[1,1,3,3]
+; SSE41-NEXT:    por %xmm14, %xmm0
+; SSE41-NEXT:    movdqa %xmm0, %xmm4
+; SSE41-NEXT:    pxor %xmm9, %xmm4
+; SSE41-NEXT:    pcmpeqq %xmm11, %xmm4
+; SSE41-NEXT:    pandn %xmm12, %xmm4
+; SSE41-NEXT:    movapd {{.*#+}} xmm12 = [9223372036854775807,9223372036854775807]
+; SSE41-NEXT:    movapd {{.*#+}} xmm11 = [9223372036854775808,9223372036854775808]
+; SSE41-NEXT:    movapd %xmm11, %xmm13
+; SSE41-NEXT:    blendvpd %xmm0, %xmm12, %xmm13
+; SSE41-NEXT:    movdqa %xmm4, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm13, %xmm8
+; SSE41-NEXT:    movdqa %xmm1, %xmm14
+; SSE41-NEXT:    paddq %xmm5, %xmm1
+; SSE41-NEXT:    pxor %xmm10, %xmm5
+; SSE41-NEXT:    movdqa %xmm10, %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm5, %xmm4
+; SSE41-NEXT:    pshufd {{.*#+}} xmm13 = xmm4[0,0,2,2]
+; SSE41-NEXT:    pcmpeqd %xmm10, %xmm5
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm5[1,1,3,3]
+; SSE41-NEXT:    pand %xmm13, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm15 = xmm4[1,1,3,3]
+; SSE41-NEXT:    por %xmm0, %xmm15
+; SSE41-NEXT:    pxor %xmm9, %xmm15
+; SSE41-NEXT:    pxor %xmm10, %xmm14
+; SSE41-NEXT:    movdqa %xmm10, %xmm0
+; SSE41-NEXT:    pcmpgtd %xmm14, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm13 = xmm0[0,0,2,2]
+; SSE41-NEXT:    pcmpeqd %xmm10, %xmm14
+; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm14[1,1,3,3]
+; SSE41-NEXT:    pand %xmm13, %xmm4
+; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm0[1,1,3,3]
+; SSE41-NEXT:    por %xmm4, %xmm5
+; SSE41-NEXT:    pxor %xmm9, %xmm5
+; SSE41-NEXT:    pcmpeqq %xmm5, %xmm15
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    pxor %xmm10, %xmm0
+; SSE41-NEXT:    movdqa %xmm10, %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm4
+; SSE41-NEXT:    pshufd {{.*#+}} xmm13 = xmm4[0,0,2,2]
+; SSE41-NEXT:    pcmpeqd %xmm10, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm14 = xmm0[1,1,3,3]
+; SSE41-NEXT:    pand %xmm13, %xmm14
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[1,1,3,3]
+; SSE41-NEXT:    por %xmm14, %xmm0
+; SSE41-NEXT:    movdqa %xmm0, %xmm4
+; SSE41-NEXT:    pxor %xmm9, %xmm4
+; SSE41-NEXT:    pcmpeqq %xmm5, %xmm4
+; SSE41-NEXT:    pandn %xmm15, %xmm4
+; SSE41-NEXT:    movapd %xmm11, %xmm5
+; SSE41-NEXT:    blendvpd %xmm0, %xmm12, %xmm5
+; SSE41-NEXT:    movdqa %xmm4, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm5, %xmm1
+; SSE41-NEXT:    movdqa %xmm2, %xmm0
+; SSE41-NEXT:    paddq %xmm6, %xmm2
+; SSE41-NEXT:    pxor %xmm10, %xmm6
+; SSE41-NEXT:    movdqa %xmm10, %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm6, %xmm4
+; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSE41-NEXT:    pcmpeqd %xmm10, %xmm6
+; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
+; SSE41-NEXT:    pand %xmm5, %xmm6
+; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[1,1,3,3]
+; SSE41-NEXT:    por %xmm6, %xmm5
+; SSE41-NEXT:    pxor %xmm9, %xmm5
+; SSE41-NEXT:    pxor %xmm10, %xmm0
+; SSE41-NEXT:    movdqa %xmm10, %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm4
+; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
+; SSE41-NEXT:    pcmpeqd %xmm10, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE41-NEXT:    pand %xmm6, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[1,1,3,3]
+; SSE41-NEXT:    por %xmm0, %xmm6
+; SSE41-NEXT:    pxor %xmm9, %xmm6
+; SSE41-NEXT:    pcmpeqq %xmm6, %xmm5
+; SSE41-NEXT:    movdqa %xmm2, %xmm0
+; SSE41-NEXT:    pxor %xmm10, %xmm0
+; SSE41-NEXT:    movdqa %xmm10, %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm4
+; SSE41-NEXT:    pshufd {{.*#+}} xmm13 = xmm4[0,0,2,2]
+; SSE41-NEXT:    pcmpeqd %xmm10, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm14 = xmm0[1,1,3,3]
+; SSE41-NEXT:    pand %xmm13, %xmm14
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[1,1,3,3]
+; SSE41-NEXT:    por %xmm14, %xmm0
+; SSE41-NEXT:    movdqa %xmm0, %xmm4
+; SSE41-NEXT:    pxor %xmm9, %xmm4
+; SSE41-NEXT:    pcmpeqq %xmm6, %xmm4
+; SSE41-NEXT:    pandn %xmm5, %xmm4
+; SSE41-NEXT:    movapd %xmm11, %xmm5
+; SSE41-NEXT:    blendvpd %xmm0, %xmm12, %xmm5
+; SSE41-NEXT:    movdqa %xmm4, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm5, %xmm2
+; SSE41-NEXT:    movdqa %xmm3, %xmm0
+; SSE41-NEXT:    paddq %xmm7, %xmm3
+; SSE41-NEXT:    pxor %xmm10, %xmm7
+; SSE41-NEXT:    movdqa %xmm10, %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm7, %xmm4
+; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSE41-NEXT:    pcmpeqd %xmm10, %xmm7
+; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm7[1,1,3,3]
+; SSE41-NEXT:    pand %xmm5, %xmm6
+; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSE41-NEXT:    por %xmm6, %xmm4
+; SSE41-NEXT:    pxor %xmm9, %xmm4
+; SSE41-NEXT:    pxor %xmm10, %xmm0
+; SSE41-NEXT:    movdqa %xmm10, %xmm5
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm5
+; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
+; SSE41-NEXT:    pcmpeqd %xmm10, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE41-NEXT:    pand %xmm6, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
+; SSE41-NEXT:    por %xmm0, %xmm5
+; SSE41-NEXT:    pxor %xmm9, %xmm5
+; SSE41-NEXT:    pcmpeqq %xmm5, %xmm4
+; SSE41-NEXT:    movdqa %xmm3, %xmm0
+; SSE41-NEXT:    pxor %xmm10, %xmm0
+; SSE41-NEXT:    movdqa %xmm10, %xmm6
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm6
+; SSE41-NEXT:    pshufd {{.*#+}} xmm13 = xmm6[0,0,2,2]
+; SSE41-NEXT:    pcmpeqd %xmm10, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm7 = xmm0[1,1,3,3]
+; SSE41-NEXT:    pand %xmm13, %xmm7
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[1,1,3,3]
+; SSE41-NEXT:    por %xmm7, %xmm0
+; SSE41-NEXT:    pxor %xmm0, %xmm9
+; SSE41-NEXT:    pcmpeqq %xmm5, %xmm9
+; SSE41-NEXT:    pandn %xmm4, %xmm9
+; SSE41-NEXT:    blendvpd %xmm0, %xmm12, %xmm11
+; SSE41-NEXT:    movdqa %xmm9, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm11, %xmm3
+; SSE41-NEXT:    movapd %xmm8, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: v8i64:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm9
+; AVX1-NEXT:    vpxor %xmm12, %xmm12, %xmm12
+; AVX1-NEXT:    vpcmpgtq %xmm9, %xmm12, %xmm7
+; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
+; AVX1-NEXT:    vpxor %xmm4, %xmm7, %xmm8
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm7
+; AVX1-NEXT:    vpcmpgtq %xmm7, %xmm12, %xmm6
+; AVX1-NEXT:    vpxor %xmm4, %xmm6, %xmm10
+; AVX1-NEXT:    vpcmpeqq %xmm8, %xmm10, %xmm8
+; AVX1-NEXT:    vpcmpgtq %xmm2, %xmm12, %xmm6
+; AVX1-NEXT:    vpxor %xmm4, %xmm6, %xmm11
+; AVX1-NEXT:    vpcmpgtq %xmm0, %xmm12, %xmm6
+; AVX1-NEXT:    vpxor %xmm4, %xmm6, %xmm6
+; AVX1-NEXT:    vpcmpeqq %xmm11, %xmm6, %xmm11
+; AVX1-NEXT:    vinsertf128 $1, %xmm8, %ymm11, %ymm8
+; AVX1-NEXT:    vpaddq %xmm9, %xmm7, %xmm9
+; AVX1-NEXT:    vpcmpgtq %xmm9, %xmm12, %xmm7
+; AVX1-NEXT:    vpxor %xmm4, %xmm7, %xmm5
+; AVX1-NEXT:    vpcmpeqq %xmm5, %xmm10, %xmm5
+; AVX1-NEXT:    vpxor %xmm4, %xmm5, %xmm5
+; AVX1-NEXT:    vpaddq %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpcmpgtq %xmm0, %xmm12, %xmm2
+; AVX1-NEXT:    vpxor %xmm4, %xmm2, %xmm2
+; AVX1-NEXT:    vpcmpeqq %xmm2, %xmm6, %xmm2
+; AVX1-NEXT:    vpxor %xmm4, %xmm2, %xmm2
+; AVX1-NEXT:    vinsertf128 $1, %xmm5, %ymm2, %ymm2
+; AVX1-NEXT:    vandpd %ymm2, %ymm8, %ymm5
+; AVX1-NEXT:    vinsertf128 $1, %xmm7, %ymm0, %ymm7
+; AVX1-NEXT:    vmovapd {{.*#+}} ymm8 = [9223372036854775807,9223372036854775807,9223372036854775807,9223372036854775807]
+; AVX1-NEXT:    vmovapd {{.*#+}} ymm10 = [9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808]
+; AVX1-NEXT:    vblendvpd %ymm7, %ymm8, %ymm10, %ymm7
+; AVX1-NEXT:    vinsertf128 $1, %xmm9, %ymm0, %ymm0
+; AVX1-NEXT:    vblendvpd %ymm5, %ymm7, %ymm0, %ymm0
+; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm5
+; AVX1-NEXT:    vpcmpgtq %xmm5, %xmm12, %xmm7
+; AVX1-NEXT:    vpxor %xmm4, %xmm7, %xmm7
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; AVX1-NEXT:    vpcmpgtq %xmm2, %xmm12, %xmm6
+; AVX1-NEXT:    vpxor %xmm4, %xmm6, %xmm6
+; AVX1-NEXT:    vpcmpeqq %xmm7, %xmm6, %xmm9
+; AVX1-NEXT:    vpcmpgtq %xmm3, %xmm12, %xmm7
+; AVX1-NEXT:    vpxor %xmm4, %xmm7, %xmm11
+; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm12, %xmm7
+; AVX1-NEXT:    vpxor %xmm4, %xmm7, %xmm7
+; AVX1-NEXT:    vpcmpeqq %xmm11, %xmm7, %xmm11
+; AVX1-NEXT:    vinsertf128 $1, %xmm9, %ymm11, %ymm9
+; AVX1-NEXT:    vpaddq %xmm5, %xmm2, %xmm11
+; AVX1-NEXT:    vpcmpgtq %xmm11, %xmm12, %xmm5
+; AVX1-NEXT:    vpxor %xmm4, %xmm5, %xmm2
+; AVX1-NEXT:    vpcmpeqq %xmm2, %xmm6, %xmm2
+; AVX1-NEXT:    vpxor %xmm4, %xmm2, %xmm2
+; AVX1-NEXT:    vpaddq %xmm3, %xmm1, %xmm1
+; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm12, %xmm3
+; AVX1-NEXT:    vpxor %xmm4, %xmm3, %xmm3
+; AVX1-NEXT:    vpcmpeqq %xmm3, %xmm7, %xmm3
+; AVX1-NEXT:    vpxor %xmm4, %xmm3, %xmm3
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm3, %ymm2
+; AVX1-NEXT:    vandpd %ymm2, %ymm9, %ymm2
+; AVX1-NEXT:    vinsertf128 $1, %xmm5, %ymm1, %ymm3
+; AVX1-NEXT:    vblendvpd %ymm3, %ymm8, %ymm10, %ymm3
+; AVX1-NEXT:    vinsertf128 $1, %xmm11, %ymm1, %ymm1
+; AVX1-NEXT:    vblendvpd %ymm2, %ymm3, %ymm1, %ymm1
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: v8i64:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpxor %xmm4, %xmm4, %xmm4
+; AVX2-NEXT:    vpcmpgtq %ymm2, %ymm4, %ymm5
+; AVX2-NEXT:    vpcmpeqd %ymm6, %ymm6, %ymm6
+; AVX2-NEXT:    vpxor %ymm6, %ymm5, %ymm5
+; AVX2-NEXT:    vpcmpgtq %ymm0, %ymm4, %ymm7
+; AVX2-NEXT:    vpxor %ymm6, %ymm7, %ymm7
+; AVX2-NEXT:    vpcmpeqq %ymm5, %ymm7, %ymm5
+; AVX2-NEXT:    vpaddq %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vpcmpgtq %ymm0, %ymm4, %ymm2
+; AVX2-NEXT:    vpxor %ymm6, %ymm2, %ymm2
+; AVX2-NEXT:    vpcmpeqq %ymm2, %ymm7, %ymm2
+; AVX2-NEXT:    vpandn %ymm5, %ymm2, %ymm2
+; AVX2-NEXT:    vbroadcastsd {{.*#+}} ymm5 = [9223372036854775807,9223372036854775807,9223372036854775807,9223372036854775807]
+; AVX2-NEXT:    vbroadcastsd {{.*#+}} ymm7 = [9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808]
+; AVX2-NEXT:    vblendvpd %ymm0, %ymm5, %ymm7, %ymm8
+; AVX2-NEXT:    vblendvpd %ymm2, %ymm8, %ymm0, %ymm0
+; AVX2-NEXT:    vpcmpgtq %ymm3, %ymm4, %ymm2
+; AVX2-NEXT:    vpxor %ymm6, %ymm2, %ymm2
+; AVX2-NEXT:    vpcmpgtq %ymm1, %ymm4, %ymm8
+; AVX2-NEXT:    vpxor %ymm6, %ymm8, %ymm8
+; AVX2-NEXT:    vpcmpeqq %ymm2, %ymm8, %ymm2
+; AVX2-NEXT:    vpaddq %ymm3, %ymm1, %ymm1
+; AVX2-NEXT:    vpcmpgtq %ymm1, %ymm4, %ymm3
+; AVX2-NEXT:    vpxor %ymm6, %ymm3, %ymm3
+; AVX2-NEXT:    vpcmpeqq %ymm3, %ymm8, %ymm3
+; AVX2-NEXT:    vpandn %ymm2, %ymm3, %ymm2
+; AVX2-NEXT:    vblendvpd %ymm1, %ymm5, %ymm7, %ymm3
+; AVX2-NEXT:    vblendvpd %ymm2, %ymm3, %ymm1, %ymm1
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: v8i64:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX512-NEXT:    vpcmpnltq %zmm2, %zmm1, %k0
+; AVX512-NEXT:    vpcmpnltq %zmm2, %zmm0, %k1
+; AVX512-NEXT:    kxorw %k0, %k1, %k0
+; AVX512-NEXT:    vpaddq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpcmpnltq %zmm2, %zmm0, %k2
+; AVX512-NEXT:    kxorw %k2, %k1, %k1
+; AVX512-NEXT:    kandnw %k1, %k0, %k1
+; AVX512-NEXT:    vpcmpgtq %zmm0, %zmm2, %k2
+; AVX512-NEXT:    vpbroadcastq {{.*#+}} zmm1 = [9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808]
+; AVX512-NEXT:    vpbroadcastq {{.*}}(%rip), %zmm1 {%k2}
+; AVX512-NEXT:    vmovdqa64 %zmm1, %zmm0 {%k1}
+; AVX512-NEXT:    retq
+  %z = call <8 x i64> @llvm.sadd.sat.v8i64(<8 x i64> %x, <8 x i64> %y)
+  ret <8 x i64> %z
 }
 
 define <2 x i128> @v2i128(<2 x i128> %x, <2 x i128> %y) nounwind {
diff --git a/test/CodeGen/X86/sat-add.ll b/test/CodeGen/X86/sat-add.ll
index e09c2416d4896c4b55fe418adacc75dcab724f48..78e4d5a4920edd7b19b579f323500c636f0b0d14 100644
--- a/test/CodeGen/X86/sat-add.ll
+++ b/test/CodeGen/X86/sat-add.ll
@@ -537,6 +537,31 @@ define <4 x i32> @unsigned_sat_constant_v4i32_using_cmp_notval(<4 x i32> %x) {
   ret <4 x i32> %r
 }
 
+define <4 x i32> @unsigned_sat_constant_v4i32_using_cmp_notval_nonsplat(<4 x i32> %x) {
+; SSE2-LABEL: unsigned_sat_constant_v4i32_using_cmp_notval_nonsplat:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [43,44,45,46]
+; SSE2-NEXT:    paddd %xmm0, %xmm1
+; SSE2-NEXT:    pxor {{.*}}(%rip), %xmm0
+; SSE2-NEXT:    pcmpgtd {{.*}}(%rip), %xmm0
+; SSE2-NEXT:    por %xmm1, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: unsigned_sat_constant_v4i32_using_cmp_notval_nonsplat:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [43,44,45,46]
+; SSE41-NEXT:    paddd %xmm0, %xmm1
+; SSE41-NEXT:    movdqa {{.*#+}} xmm2 = [4294967253,4294967252,4294967251,4294967250]
+; SSE41-NEXT:    pmaxud %xmm0, %xmm2
+; SSE41-NEXT:    pcmpeqd %xmm2, %xmm0
+; SSE41-NEXT:    por %xmm1, %xmm0
+; SSE41-NEXT:    retq
+  %a = add <4 x i32> %x, <i32 43, i32 44, i32 45, i32 46>
+  %c = icmp ugt <4 x i32> %x, <i32 -44, i32 -45, i32 -46, i32 -47>
+  %r = select <4 x i1> %c, <4 x i32> <i32 -1, i32 -1, i32 -1, i32 -1>, <4 x i32> %a
+  ret <4 x i32> %r
+}
+
 define <2 x i64> @unsigned_sat_constant_v2i64_using_min(<2 x i64> %x) {
 ; SSE2-LABEL: unsigned_sat_constant_v2i64_using_min:
 ; SSE2:       # %bb.0:
diff --git a/test/CodeGen/X86/scalar-fp-to-i64.ll b/test/CodeGen/X86/scalar-fp-to-i64.ll
index 7ed61f8fdc7d24b73da97f075e5605ad13b588f8..d7e63ba046cc1f7a27f5f52912ab506b34bfe991 100644
--- a/test/CodeGen/X86/scalar-fp-to-i64.ll
+++ b/test/CodeGen/X86/scalar-fp-to-i64.ll
@@ -275,7 +275,7 @@ define i64 @f_to_u64(float %a) nounwind {
 ; X87_WIN-NEXT:    flds 8(%ebp)
 ; X87_WIN-NEXT:    flds __real@5f000000
 ; X87_WIN-NEXT:    fld %st(1)
-; X87_WIN-NEXT:    fsub %st(1)
+; X87_WIN-NEXT:    fsub %st(1), %st
 ; X87_WIN-NEXT:    fxch %st(1)
 ; X87_WIN-NEXT:    fucomp %st(2)
 ; X87_WIN-NEXT:    fnstsw %ax
@@ -309,7 +309,7 @@ define i64 @f_to_u64(float %a) nounwind {
 ; X87_LIN-NEXT:    flds {{[0-9]+}}(%esp)
 ; X87_LIN-NEXT:    flds {{\.LCPI.*}}
 ; X87_LIN-NEXT:    fld %st(1)
-; X87_LIN-NEXT:    fsub %st(1)
+; X87_LIN-NEXT:    fsub %st(1), %st
 ; X87_LIN-NEXT:    fxch %st(1)
 ; X87_LIN-NEXT:    fucomp %st(2)
 ; X87_LIN-NEXT:    fnstsw %ax
@@ -763,7 +763,7 @@ define i64 @d_to_u64(double %a) nounwind {
 ; X87_WIN-NEXT:    fldl 8(%ebp)
 ; X87_WIN-NEXT:    flds __real@5f000000
 ; X87_WIN-NEXT:    fld %st(1)
-; X87_WIN-NEXT:    fsub %st(1)
+; X87_WIN-NEXT:    fsub %st(1), %st
 ; X87_WIN-NEXT:    fxch %st(1)
 ; X87_WIN-NEXT:    fucomp %st(2)
 ; X87_WIN-NEXT:    fnstsw %ax
@@ -797,7 +797,7 @@ define i64 @d_to_u64(double %a) nounwind {
 ; X87_LIN-NEXT:    fldl {{[0-9]+}}(%esp)
 ; X87_LIN-NEXT:    flds {{\.LCPI.*}}
 ; X87_LIN-NEXT:    fld %st(1)
-; X87_LIN-NEXT:    fsub %st(1)
+; X87_LIN-NEXT:    fsub %st(1), %st
 ; X87_LIN-NEXT:    fxch %st(1)
 ; X87_LIN-NEXT:    fucomp %st(2)
 ; X87_LIN-NEXT:    fnstsw %ax
@@ -1024,11 +1024,11 @@ define i64 @x_to_u64(x86_fp80 %a) nounwind {
 ; AVX512_32_WIN-NEXT:    fldt 8(%ebp)
 ; AVX512_32_WIN-NEXT:    flds __real@5f000000
 ; AVX512_32_WIN-NEXT:    fld %st(1)
-; AVX512_32_WIN-NEXT:    fsub %st(1)
+; AVX512_32_WIN-NEXT:    fsub %st(1), %st
 ; AVX512_32_WIN-NEXT:    xorl %edx, %edx
 ; AVX512_32_WIN-NEXT:    fxch %st(1)
-; AVX512_32_WIN-NEXT:    fucompi %st(2)
-; AVX512_32_WIN-NEXT:    fcmovnbe %st(1), %st(0)
+; AVX512_32_WIN-NEXT:    fucompi %st(2), %st
+; AVX512_32_WIN-NEXT:    fcmovnbe %st(1), %st
 ; AVX512_32_WIN-NEXT:    fstp %st(1)
 ; AVX512_32_WIN-NEXT:    fisttpll (%esp)
 ; AVX512_32_WIN-NEXT:    setbe %dl
@@ -1045,11 +1045,11 @@ define i64 @x_to_u64(x86_fp80 %a) nounwind {
 ; AVX512_32_LIN-NEXT:    fldt {{[0-9]+}}(%esp)
 ; AVX512_32_LIN-NEXT:    flds {{\.LCPI.*}}
 ; AVX512_32_LIN-NEXT:    fld %st(1)
-; AVX512_32_LIN-NEXT:    fsub %st(1)
+; AVX512_32_LIN-NEXT:    fsub %st(1), %st
 ; AVX512_32_LIN-NEXT:    xorl %edx, %edx
 ; AVX512_32_LIN-NEXT:    fxch %st(1)
-; AVX512_32_LIN-NEXT:    fucompi %st(2)
-; AVX512_32_LIN-NEXT:    fcmovnbe %st(1), %st(0)
+; AVX512_32_LIN-NEXT:    fucompi %st(2), %st
+; AVX512_32_LIN-NEXT:    fcmovnbe %st(1), %st
 ; AVX512_32_LIN-NEXT:    fstp %st(1)
 ; AVX512_32_LIN-NEXT:    fisttpll (%esp)
 ; AVX512_32_LIN-NEXT:    setbe %dl
@@ -1065,19 +1065,16 @@ define i64 @x_to_u64(x86_fp80 %a) nounwind {
 ; AVX512_64_WIN-NEXT:    fldt (%rcx)
 ; AVX512_64_WIN-NEXT:    flds __real@{{.*}}(%rip)
 ; AVX512_64_WIN-NEXT:    fld %st(1)
-; AVX512_64_WIN-NEXT:    fsub %st(1)
-; AVX512_64_WIN-NEXT:    xorl %ecx, %ecx
+; AVX512_64_WIN-NEXT:    fsub %st(1), %st
+; AVX512_64_WIN-NEXT:    xorl %eax, %eax
 ; AVX512_64_WIN-NEXT:    fxch %st(1)
-; AVX512_64_WIN-NEXT:    fucompi %st(2)
-; AVX512_64_WIN-NEXT:    fcmovnbe %st(1), %st(0)
+; AVX512_64_WIN-NEXT:    fucompi %st(2), %st
+; AVX512_64_WIN-NEXT:    fcmovnbe %st(1), %st
 ; AVX512_64_WIN-NEXT:    fstp %st(1)
 ; AVX512_64_WIN-NEXT:    fisttpll (%rsp)
-; AVX512_64_WIN-NEXT:    setbe %cl
-; AVX512_64_WIN-NEXT:    shll $31, %ecx
-; AVX512_64_WIN-NEXT:    xorl {{[0-9]+}}(%rsp), %ecx
-; AVX512_64_WIN-NEXT:    shlq $32, %rcx
-; AVX512_64_WIN-NEXT:    movl (%rsp), %eax
-; AVX512_64_WIN-NEXT:    orq %rcx, %rax
+; AVX512_64_WIN-NEXT:    setbe %al
+; AVX512_64_WIN-NEXT:    shlq $63, %rax
+; AVX512_64_WIN-NEXT:    xorq (%rsp), %rax
 ; AVX512_64_WIN-NEXT:    popq %rcx
 ; AVX512_64_WIN-NEXT:    retq
 ;
@@ -1086,19 +1083,16 @@ define i64 @x_to_u64(x86_fp80 %a) nounwind {
 ; AVX512_64_LIN-NEXT:    fldt {{[0-9]+}}(%rsp)
 ; AVX512_64_LIN-NEXT:    flds {{.*}}(%rip)
 ; AVX512_64_LIN-NEXT:    fld %st(1)
-; AVX512_64_LIN-NEXT:    fsub %st(1)
-; AVX512_64_LIN-NEXT:    xorl %ecx, %ecx
+; AVX512_64_LIN-NEXT:    fsub %st(1), %st
+; AVX512_64_LIN-NEXT:    xorl %eax, %eax
 ; AVX512_64_LIN-NEXT:    fxch %st(1)
-; AVX512_64_LIN-NEXT:    fucompi %st(2)
-; AVX512_64_LIN-NEXT:    fcmovnbe %st(1), %st(0)
+; AVX512_64_LIN-NEXT:    fucompi %st(2), %st
+; AVX512_64_LIN-NEXT:    fcmovnbe %st(1), %st
 ; AVX512_64_LIN-NEXT:    fstp %st(1)
 ; AVX512_64_LIN-NEXT:    fisttpll -{{[0-9]+}}(%rsp)
-; AVX512_64_LIN-NEXT:    setbe %cl
-; AVX512_64_LIN-NEXT:    shll $31, %ecx
-; AVX512_64_LIN-NEXT:    xorl -{{[0-9]+}}(%rsp), %ecx
-; AVX512_64_LIN-NEXT:    shlq $32, %rcx
-; AVX512_64_LIN-NEXT:    movl -{{[0-9]+}}(%rsp), %eax
-; AVX512_64_LIN-NEXT:    orq %rcx, %rax
+; AVX512_64_LIN-NEXT:    setbe %al
+; AVX512_64_LIN-NEXT:    shlq $63, %rax
+; AVX512_64_LIN-NEXT:    xorq -{{[0-9]+}}(%rsp), %rax
 ; AVX512_64_LIN-NEXT:    retq
 ;
 ; SSE3_32_WIN-LABEL: x_to_u64:
@@ -1110,11 +1104,11 @@ define i64 @x_to_u64(x86_fp80 %a) nounwind {
 ; SSE3_32_WIN-NEXT:    fldt 8(%ebp)
 ; SSE3_32_WIN-NEXT:    flds __real@5f000000
 ; SSE3_32_WIN-NEXT:    fld %st(1)
-; SSE3_32_WIN-NEXT:    fsub %st(1)
+; SSE3_32_WIN-NEXT:    fsub %st(1), %st
 ; SSE3_32_WIN-NEXT:    xorl %edx, %edx
 ; SSE3_32_WIN-NEXT:    fxch %st(1)
-; SSE3_32_WIN-NEXT:    fucompi %st(2)
-; SSE3_32_WIN-NEXT:    fcmovnbe %st(1), %st(0)
+; SSE3_32_WIN-NEXT:    fucompi %st(2), %st
+; SSE3_32_WIN-NEXT:    fcmovnbe %st(1), %st
 ; SSE3_32_WIN-NEXT:    fstp %st(1)
 ; SSE3_32_WIN-NEXT:    fisttpll (%esp)
 ; SSE3_32_WIN-NEXT:    setbe %dl
@@ -1131,11 +1125,11 @@ define i64 @x_to_u64(x86_fp80 %a) nounwind {
 ; SSE3_32_LIN-NEXT:    fldt {{[0-9]+}}(%esp)
 ; SSE3_32_LIN-NEXT:    flds {{\.LCPI.*}}
 ; SSE3_32_LIN-NEXT:    fld %st(1)
-; SSE3_32_LIN-NEXT:    fsub %st(1)
+; SSE3_32_LIN-NEXT:    fsub %st(1), %st
 ; SSE3_32_LIN-NEXT:    xorl %edx, %edx
 ; SSE3_32_LIN-NEXT:    fxch %st(1)
-; SSE3_32_LIN-NEXT:    fucompi %st(2)
-; SSE3_32_LIN-NEXT:    fcmovnbe %st(1), %st(0)
+; SSE3_32_LIN-NEXT:    fucompi %st(2), %st
+; SSE3_32_LIN-NEXT:    fcmovnbe %st(1), %st
 ; SSE3_32_LIN-NEXT:    fstp %st(1)
 ; SSE3_32_LIN-NEXT:    fisttpll (%esp)
 ; SSE3_32_LIN-NEXT:    setbe %dl
@@ -1151,11 +1145,11 @@ define i64 @x_to_u64(x86_fp80 %a) nounwind {
 ; SSE3_64_WIN-NEXT:    fldt (%rcx)
 ; SSE3_64_WIN-NEXT:    flds __real@{{.*}}(%rip)
 ; SSE3_64_WIN-NEXT:    fld %st(1)
-; SSE3_64_WIN-NEXT:    fsub %st(1)
+; SSE3_64_WIN-NEXT:    fsub %st(1), %st
 ; SSE3_64_WIN-NEXT:    xorl %eax, %eax
 ; SSE3_64_WIN-NEXT:    fxch %st(1)
-; SSE3_64_WIN-NEXT:    fucompi %st(2)
-; SSE3_64_WIN-NEXT:    fcmovnbe %st(1), %st(0)
+; SSE3_64_WIN-NEXT:    fucompi %st(2), %st
+; SSE3_64_WIN-NEXT:    fcmovnbe %st(1), %st
 ; SSE3_64_WIN-NEXT:    fstp %st(1)
 ; SSE3_64_WIN-NEXT:    fisttpll (%rsp)
 ; SSE3_64_WIN-NEXT:    setbe %al
@@ -1169,11 +1163,11 @@ define i64 @x_to_u64(x86_fp80 %a) nounwind {
 ; SSE3_64_LIN-NEXT:    fldt {{[0-9]+}}(%rsp)
 ; SSE3_64_LIN-NEXT:    flds {{.*}}(%rip)
 ; SSE3_64_LIN-NEXT:    fld %st(1)
-; SSE3_64_LIN-NEXT:    fsub %st(1)
+; SSE3_64_LIN-NEXT:    fsub %st(1), %st
 ; SSE3_64_LIN-NEXT:    xorl %eax, %eax
 ; SSE3_64_LIN-NEXT:    fxch %st(1)
-; SSE3_64_LIN-NEXT:    fucompi %st(2)
-; SSE3_64_LIN-NEXT:    fcmovnbe %st(1), %st(0)
+; SSE3_64_LIN-NEXT:    fucompi %st(2), %st
+; SSE3_64_LIN-NEXT:    fcmovnbe %st(1), %st
 ; SSE3_64_LIN-NEXT:    fstp %st(1)
 ; SSE3_64_LIN-NEXT:    fisttpll -{{[0-9]+}}(%rsp)
 ; SSE3_64_LIN-NEXT:    setbe %al
@@ -1190,11 +1184,11 @@ define i64 @x_to_u64(x86_fp80 %a) nounwind {
 ; SSE2_32_WIN-NEXT:    fldt 8(%ebp)
 ; SSE2_32_WIN-NEXT:    flds __real@5f000000
 ; SSE2_32_WIN-NEXT:    fld %st(1)
-; SSE2_32_WIN-NEXT:    fsub %st(1)
+; SSE2_32_WIN-NEXT:    fsub %st(1), %st
 ; SSE2_32_WIN-NEXT:    xorl %edx, %edx
 ; SSE2_32_WIN-NEXT:    fxch %st(1)
-; SSE2_32_WIN-NEXT:    fucompi %st(2)
-; SSE2_32_WIN-NEXT:    fcmovnbe %st(1), %st(0)
+; SSE2_32_WIN-NEXT:    fucompi %st(2), %st
+; SSE2_32_WIN-NEXT:    fcmovnbe %st(1), %st
 ; SSE2_32_WIN-NEXT:    fstp %st(1)
 ; SSE2_32_WIN-NEXT:    fnstcw {{[0-9]+}}(%esp)
 ; SSE2_32_WIN-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
@@ -1217,11 +1211,11 @@ define i64 @x_to_u64(x86_fp80 %a) nounwind {
 ; SSE2_32_LIN-NEXT:    fldt {{[0-9]+}}(%esp)
 ; SSE2_32_LIN-NEXT:    flds {{\.LCPI.*}}
 ; SSE2_32_LIN-NEXT:    fld %st(1)
-; SSE2_32_LIN-NEXT:    fsub %st(1)
+; SSE2_32_LIN-NEXT:    fsub %st(1), %st
 ; SSE2_32_LIN-NEXT:    xorl %edx, %edx
 ; SSE2_32_LIN-NEXT:    fxch %st(1)
-; SSE2_32_LIN-NEXT:    fucompi %st(2)
-; SSE2_32_LIN-NEXT:    fcmovnbe %st(1), %st(0)
+; SSE2_32_LIN-NEXT:    fucompi %st(2), %st
+; SSE2_32_LIN-NEXT:    fcmovnbe %st(1), %st
 ; SSE2_32_LIN-NEXT:    fstp %st(1)
 ; SSE2_32_LIN-NEXT:    fnstcw {{[0-9]+}}(%esp)
 ; SSE2_32_LIN-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
@@ -1243,11 +1237,11 @@ define i64 @x_to_u64(x86_fp80 %a) nounwind {
 ; SSE2_64_WIN-NEXT:    fldt (%rcx)
 ; SSE2_64_WIN-NEXT:    flds __real@{{.*}}(%rip)
 ; SSE2_64_WIN-NEXT:    fld %st(1)
-; SSE2_64_WIN-NEXT:    fsub %st(1)
+; SSE2_64_WIN-NEXT:    fsub %st(1), %st
 ; SSE2_64_WIN-NEXT:    xorl %eax, %eax
 ; SSE2_64_WIN-NEXT:    fxch %st(1)
-; SSE2_64_WIN-NEXT:    fucompi %st(2)
-; SSE2_64_WIN-NEXT:    fcmovnbe %st(1), %st(0)
+; SSE2_64_WIN-NEXT:    fucompi %st(2), %st
+; SSE2_64_WIN-NEXT:    fcmovnbe %st(1), %st
 ; SSE2_64_WIN-NEXT:    fstp %st(1)
 ; SSE2_64_WIN-NEXT:    fnstcw {{[0-9]+}}(%rsp)
 ; SSE2_64_WIN-NEXT:    movzwl {{[0-9]+}}(%rsp), %ecx
@@ -1267,11 +1261,11 @@ define i64 @x_to_u64(x86_fp80 %a) nounwind {
 ; SSE2_64_LIN-NEXT:    fldt {{[0-9]+}}(%rsp)
 ; SSE2_64_LIN-NEXT:    flds {{.*}}(%rip)
 ; SSE2_64_LIN-NEXT:    fld %st(1)
-; SSE2_64_LIN-NEXT:    fsub %st(1)
+; SSE2_64_LIN-NEXT:    fsub %st(1), %st
 ; SSE2_64_LIN-NEXT:    xorl %eax, %eax
 ; SSE2_64_LIN-NEXT:    fxch %st(1)
-; SSE2_64_LIN-NEXT:    fucompi %st(2)
-; SSE2_64_LIN-NEXT:    fcmovnbe %st(1), %st(0)
+; SSE2_64_LIN-NEXT:    fucompi %st(2), %st
+; SSE2_64_LIN-NEXT:    fcmovnbe %st(1), %st
 ; SSE2_64_LIN-NEXT:    fstp %st(1)
 ; SSE2_64_LIN-NEXT:    fnstcw -{{[0-9]+}}(%rsp)
 ; SSE2_64_LIN-NEXT:    movzwl -{{[0-9]+}}(%rsp), %ecx
@@ -1294,7 +1288,7 @@ define i64 @x_to_u64(x86_fp80 %a) nounwind {
 ; X87_WIN-NEXT:    fldt 8(%ebp)
 ; X87_WIN-NEXT:    flds __real@5f000000
 ; X87_WIN-NEXT:    fld %st(1)
-; X87_WIN-NEXT:    fsub %st(1)
+; X87_WIN-NEXT:    fsub %st(1), %st
 ; X87_WIN-NEXT:    fxch %st(1)
 ; X87_WIN-NEXT:    fucomp %st(2)
 ; X87_WIN-NEXT:    fnstsw %ax
@@ -1328,7 +1322,7 @@ define i64 @x_to_u64(x86_fp80 %a) nounwind {
 ; X87_LIN-NEXT:    fldt {{[0-9]+}}(%esp)
 ; X87_LIN-NEXT:    flds {{\.LCPI.*}}
 ; X87_LIN-NEXT:    fld %st(1)
-; X87_LIN-NEXT:    fsub %st(1)
+; X87_LIN-NEXT:    fsub %st(1), %st
 ; X87_LIN-NEXT:    fxch %st(1)
 ; X87_LIN-NEXT:    fucomp %st(2)
 ; X87_LIN-NEXT:    fnstsw %ax
diff --git a/test/CodeGen/X86/scalarize-fp.ll b/test/CodeGen/X86/scalarize-fp.ll
index d9606b8735254ac3aff05e2555fcf888e0f3d1dd..5cf526dbff5388c9e0e48c8090c08d094ef027a1 100644
--- a/test/CodeGen/X86/scalarize-fp.ll
+++ b/test/CodeGen/X86/scalarize-fp.ll
@@ -198,9 +198,8 @@ define <4 x double> @fadd_op1_constant_v4f64(double %x) nounwind {
 ;
 ; AVX-LABEL: fadd_op1_constant_v4f64:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
 ; AVX-NEXT:    vmovsd {{.*#+}} xmm1 = mem[0],zero
-; AVX-NEXT:    vaddpd %ymm1, %ymm0, %ymm0
+; AVX-NEXT:    vaddpd %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    retq
   %v = insertelement <4 x double> undef, double %x, i32 0
   %b = fadd <4 x double> %v, <double 42.0, double undef, double undef, double undef>
@@ -219,7 +218,7 @@ define <4 x double> @load_fadd_op1_constant_v4f64(double* %p) nounwind {
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
 ; AVX-NEXT:    vmovsd {{.*#+}} xmm1 = mem[0],zero
-; AVX-NEXT:    vaddpd %ymm1, %ymm0, %ymm0
+; AVX-NEXT:    vaddpd %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    retq
   %x = load double, double* %p
   %v = insertelement <4 x double> undef, double %x, i32 0
@@ -237,9 +236,8 @@ define <4 x double> @fsub_op0_constant_v4f64(double %x) nounwind {
 ;
 ; AVX-LABEL: fsub_op0_constant_v4f64:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
 ; AVX-NEXT:    vmovsd {{.*#+}} xmm1 = mem[0],zero
-; AVX-NEXT:    vsubpd %ymm0, %ymm1, %ymm0
+; AVX-NEXT:    vsubpd %xmm0, %xmm1, %xmm0
 ; AVX-NEXT:    retq
   %v = insertelement <4 x double> undef, double %x, i32 0
   %b = fsub <4 x double> <double 42.0, double undef, double undef, double undef>, %v
@@ -258,7 +256,7 @@ define <4 x double> @load_fsub_op0_constant_v4f64(double* %p) nounwind {
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
 ; AVX-NEXT:    vmovsd {{.*#+}} xmm1 = mem[0],zero
-; AVX-NEXT:    vsubpd %ymm0, %ymm1, %ymm0
+; AVX-NEXT:    vsubpd %xmm0, %xmm1, %xmm0
 ; AVX-NEXT:    retq
   %x = load double, double* %p
   %v = insertelement <4 x double> undef, double %x, i32 0
@@ -275,9 +273,8 @@ define <4 x double> @fmul_op1_constant_v4f64(double %x) nounwind {
 ;
 ; AVX-LABEL: fmul_op1_constant_v4f64:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
 ; AVX-NEXT:    vmovsd {{.*#+}} xmm1 = mem[0],zero
-; AVX-NEXT:    vmulpd %ymm1, %ymm0, %ymm0
+; AVX-NEXT:    vmulpd %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    retq
   %v = insertelement <4 x double> undef, double %x, i32 0
   %b = fmul <4 x double> %v, <double 42.0, double undef, double undef, double undef>
@@ -296,7 +293,7 @@ define <4 x double> @load_fmul_op1_constant_v4f64(double* %p) nounwind {
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
 ; AVX-NEXT:    vmovsd {{.*#+}} xmm1 = mem[0],zero
-; AVX-NEXT:    vmulpd %ymm1, %ymm0, %ymm0
+; AVX-NEXT:    vmulpd %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    retq
   %x = load double, double* %p
   %v = insertelement <4 x double> undef, double %x, i32 0
@@ -313,9 +310,8 @@ define <4 x double> @fdiv_op1_constant_v4f64(double %x) nounwind {
 ;
 ; AVX-LABEL: fdiv_op1_constant_v4f64:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
 ; AVX-NEXT:    vmovsd {{.*#+}} xmm1 = mem[0],zero
-; AVX-NEXT:    vdivpd %ymm1, %ymm0, %ymm0
+; AVX-NEXT:    vdivpd %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    retq
   %v = insertelement <4 x double> undef, double %x, i32 0
   %b = fdiv <4 x double> %v, <double 42.0, double undef, double undef, double undef>
@@ -334,7 +330,7 @@ define <4 x double> @load_fdiv_op1_constant_v4f64(double* %p) nounwind {
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
 ; AVX-NEXT:    vmovsd {{.*#+}} xmm1 = mem[0],zero
-; AVX-NEXT:    vdivpd %ymm1, %ymm0, %ymm0
+; AVX-NEXT:    vdivpd %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    retq
   %x = load double, double* %p
   %v = insertelement <4 x double> undef, double %x, i32 0
@@ -352,9 +348,8 @@ define <4 x double> @fdiv_op0_constant_v4f64(double %x) nounwind {
 ;
 ; AVX-LABEL: fdiv_op0_constant_v4f64:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
 ; AVX-NEXT:    vmovsd {{.*#+}} xmm1 = mem[0],zero
-; AVX-NEXT:    vdivpd %ymm0, %ymm1, %ymm0
+; AVX-NEXT:    vdivpd %xmm0, %xmm1, %xmm0
 ; AVX-NEXT:    retq
   %v = insertelement <4 x double> undef, double %x, i32 0
   %b = fdiv <4 x double> <double 42.0, double undef, double undef, double undef>, %v
@@ -373,7 +368,7 @@ define <4 x double> @load_fdiv_op0_constant_v4f64(double* %p) nounwind {
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
 ; AVX-NEXT:    vmovsd {{.*#+}} xmm1 = mem[0],zero
-; AVX-NEXT:    vdivpd %ymm0, %ymm1, %ymm0
+; AVX-NEXT:    vdivpd %xmm0, %xmm1, %xmm0
 ; AVX-NEXT:    retq
   %x = load double, double* %p
   %v = insertelement <4 x double> undef, double %x, i32 0
diff --git a/test/CodeGen/X86/schedule-x86-64-shld.ll b/test/CodeGen/X86/schedule-x86-64-shld.ll
deleted file mode 100644
index b0faf4cc5fa68f853fec6c7246b757cf10d76021..0000000000000000000000000000000000000000
--- a/test/CodeGen/X86/schedule-x86-64-shld.ll
+++ /dev/null
@@ -1,471 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=x86-64 | FileCheck %s --check-prefix=CHECK --check-prefix=GENERIC
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=bdver1 | FileCheck %s --check-prefix=CHECK --check-prefix=BDVER12 --check-prefix=BDVER1
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=bdver2 | FileCheck %s --check-prefix=CHECK --check-prefix=BDVER12 --check-prefix=BDVER2
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=btver2 | FileCheck %s --check-prefix=CHECK --check-prefix=BTVER2
-
-
-; uint64_t lshift10(uint64_t a, uint64_t b)
-; {
-;     return (a << 10) | (b >> 54);
-; }
-
-define i64 @lshift10_optsize(i64 %a, i64 %b) nounwind readnone optsize {
-; GENERIC-LABEL: lshift10_optsize:
-; GENERIC:       # %bb.0: # %entry
-; GENERIC-NEXT:    movq %rdi, %rax # sched: [1:0.33]
-; GENERIC-NEXT:    shldq $10, %rsi, %rax # sched: [2:0.67]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER12-LABEL: lshift10_optsize:
-; BDVER12:       # %bb.0: # %entry
-; BDVER12-NEXT:    movq %rdi, %rax # sched: [1:0.50]
-; BDVER12-NEXT:    shldq $10, %rsi, %rax # sched: [4:3.00]
-; BDVER12-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: lshift10_optsize:
-; BTVER2:       # %bb.0: # %entry
-; BTVER2-NEXT:    movq %rdi, %rax # sched: [1:0.50]
-; BTVER2-NEXT:    shldq $10, %rsi, %rax # sched: [3:3.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-entry:
-  %shl = shl i64 %a, 10
-  %shr = lshr i64 %b, 54
-  %or = or i64 %shr, %shl
-  ret i64 %or
-}
-
-define i64 @lshift10(i64 %a, i64 %b) nounwind readnone {
-; GENERIC-LABEL: lshift10:
-; GENERIC:       # %bb.0: # %entry
-; GENERIC-NEXT:    movq %rdi, %rax # sched: [1:0.33]
-; GENERIC-NEXT:    shldq $10, %rsi, %rax # sched: [2:0.67]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER12-LABEL: lshift10:
-; BDVER12:       # %bb.0: # %entry
-; BDVER12-NEXT:    shlq $10, %rdi # sched: [1:0.50]
-; BDVER12-NEXT:    shrq $54, %rsi # sched: [1:0.50]
-; BDVER12-NEXT:    leaq (%rsi,%rdi), %rax # sched: [1:0.50]
-; BDVER12-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: lshift10:
-; BTVER2:       # %bb.0: # %entry
-; BTVER2-NEXT:    shlq $10, %rdi # sched: [1:0.50]
-; BTVER2-NEXT:    shrq $54, %rsi # sched: [1:0.50]
-; BTVER2-NEXT:    leaq (%rsi,%rdi), %rax # sched: [1:0.50]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-entry:
-  %shl = shl i64 %a, 10
-  %shr = lshr i64 %b, 54
-  %or = or i64 %shr, %shl
-  ret i64 %or
-}
-
-; uint64_t rshift10(uint64_t a, uint64_t b)
-; {
-;     return (a >> 62) | (b << 2);
-; }
-
-; Should be done via shld
-define i64 @rshift10_optsize(i64 %a, i64 %b) nounwind readnone optsize {
-; GENERIC-LABEL: rshift10_optsize:
-; GENERIC:       # %bb.0: # %entry
-; GENERIC-NEXT:    movq %rdi, %rax # sched: [1:0.33]
-; GENERIC-NEXT:    shrdq $62, %rsi, %rax # sched: [2:0.67]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER12-LABEL: rshift10_optsize:
-; BDVER12:       # %bb.0: # %entry
-; BDVER12-NEXT:    movq %rdi, %rax # sched: [1:0.50]
-; BDVER12-NEXT:    shrdq $62, %rsi, %rax # sched: [4:3.00]
-; BDVER12-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: rshift10_optsize:
-; BTVER2:       # %bb.0: # %entry
-; BTVER2-NEXT:    movq %rdi, %rax # sched: [1:0.50]
-; BTVER2-NEXT:    shrdq $62, %rsi, %rax # sched: [3:3.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-entry:
-  %shl = lshr i64 %a, 62
-  %shr = shl i64 %b, 2
-  %or = or i64 %shr, %shl
-  ret i64 %or
-}
-
-; Should be done via lea (x,y,4),z
-define i64 @rshift10(i64 %a, i64 %b) nounwind readnone {
-; GENERIC-LABEL: rshift10:
-; GENERIC:       # %bb.0: # %entry
-; GENERIC-NEXT:    movq %rdi, %rax # sched: [1:0.33]
-; GENERIC-NEXT:    shrdq $62, %rsi, %rax # sched: [2:0.67]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER12-LABEL: rshift10:
-; BDVER12:       # %bb.0: # %entry
-; BDVER12-NEXT:    shrq $62, %rdi # sched: [1:0.50]
-; BDVER12-NEXT:    leaq (%rdi,%rsi,4), %rax # sched: [1:0.50]
-; BDVER12-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: rshift10:
-; BTVER2:       # %bb.0: # %entry
-; BTVER2-NEXT:    shrq $62, %rdi # sched: [1:0.50]
-; BTVER2-NEXT:    leaq (%rdi,%rsi,4), %rax # sched: [2:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-entry:
-  %shl = lshr i64 %a, 62
-  %shr = shl i64 %b, 2
-  %or = or i64 %shr, %shl
-  ret i64 %or
-}
-
-;uint64_t lshift(uint64_t a, uint64_t b, uint64_t c)
-;{
-;    return (a << c) | (b >> (64-c));
-;}
-
-define i64 @lshift_cl_optsize(i64 %a, i64 %b, i64 %c) nounwind readnone optsize {
-; GENERIC-LABEL: lshift_cl_optsize:
-; GENERIC:       # %bb.0: # %entry
-; GENERIC-NEXT:    movq %rdx, %rcx # sched: [1:0.33]
-; GENERIC-NEXT:    movq %rdi, %rax # sched: [1:0.33]
-; GENERIC-NEXT:    # kill: def $cl killed $cl killed $rcx
-; GENERIC-NEXT:    shldq %cl, %rsi, %rax # sched: [4:1.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER12-LABEL: lshift_cl_optsize:
-; BDVER12:       # %bb.0: # %entry
-; BDVER12-NEXT:    movq %rdi, %rax # sched: [1:0.50]
-; BDVER12-NEXT:    movq %rdx, %rcx # sched: [1:0.50]
-; BDVER12-NEXT:    # kill: def $cl killed $cl killed $rcx
-; BDVER12-NEXT:    shldq %cl, %rsi, %rax # sched: [4:4.00]
-; BDVER12-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: lshift_cl_optsize:
-; BTVER2:       # %bb.0: # %entry
-; BTVER2-NEXT:    movq %rdi, %rax # sched: [1:0.50]
-; BTVER2-NEXT:    movq %rdx, %rcx # sched: [1:0.50]
-; BTVER2-NEXT:    # kill: def $cl killed $cl killed $rcx
-; BTVER2-NEXT:    shldq %cl, %rsi, %rax # sched: [4:4.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-entry:
-  %shl = shl i64 %a, %c
-  %sub = sub nsw i64 64, %c
-  %shr = lshr i64 %b, %sub
-  %or = or i64 %shr, %shl
-  ret i64 %or
-}
-
-define i64 @lshift_cl(i64 %a, i64 %b, i64 %c) nounwind readnone {
-; GENERIC-LABEL: lshift_cl:
-; GENERIC:       # %bb.0: # %entry
-; GENERIC-NEXT:    movq %rdx, %rcx # sched: [1:0.33]
-; GENERIC-NEXT:    movq %rdi, %rax # sched: [1:0.33]
-; GENERIC-NEXT:    # kill: def $cl killed $cl killed $rcx
-; GENERIC-NEXT:    shldq %cl, %rsi, %rax # sched: [4:1.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER12-LABEL: lshift_cl:
-; BDVER12:       # %bb.0: # %entry
-; BDVER12-NEXT:    movq %rdx, %rcx # sched: [1:0.50]
-; BDVER12-NEXT:    movq %rsi, %rax # sched: [1:0.50]
-; BDVER12-NEXT:    shlq %cl, %rdi # sched: [1:0.50]
-; BDVER12-NEXT:    negb %cl # sched: [1:0.50]
-; BDVER12-NEXT:    # kill: def $cl killed $cl killed $rcx
-; BDVER12-NEXT:    shrq %cl, %rax # sched: [1:0.50]
-; BDVER12-NEXT:    orq %rdi, %rax # sched: [1:0.50]
-; BDVER12-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: lshift_cl:
-; BTVER2:       # %bb.0: # %entry
-; BTVER2-NEXT:    movq %rdx, %rcx # sched: [1:0.50]
-; BTVER2-NEXT:    movq %rsi, %rax # sched: [1:0.50]
-; BTVER2-NEXT:    shlq %cl, %rdi # sched: [1:0.50]
-; BTVER2-NEXT:    negb %cl # sched: [1:0.50]
-; BTVER2-NEXT:    # kill: def $cl killed $cl killed $rcx
-; BTVER2-NEXT:    shrq %cl, %rax # sched: [1:0.50]
-; BTVER2-NEXT:    orq %rdi, %rax # sched: [1:0.50]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-entry:
-  %shl = shl i64 %a, %c
-  %sub = sub nsw i64 64, %c
-  %shr = lshr i64 %b, %sub
-  %or = or i64 %shr, %shl
-  ret i64 %or
-}
-
-
-;uint64_t rshift(uint64_t a, uint64_t b, int c)
-;{
-;    return (a >> c) | (b << (64-c));
-;}
-
-define i64 @rshift_cl_optsize(i64 %a, i64 %b, i64 %c) nounwind readnone optsize {
-; GENERIC-LABEL: rshift_cl_optsize:
-; GENERIC:       # %bb.0: # %entry
-; GENERIC-NEXT:    movq %rdx, %rcx # sched: [1:0.33]
-; GENERIC-NEXT:    movq %rdi, %rax # sched: [1:0.33]
-; GENERIC-NEXT:    # kill: def $cl killed $cl killed $rcx
-; GENERIC-NEXT:    shrdq %cl, %rsi, %rax # sched: [4:1.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER12-LABEL: rshift_cl_optsize:
-; BDVER12:       # %bb.0: # %entry
-; BDVER12-NEXT:    movq %rdi, %rax # sched: [1:0.50]
-; BDVER12-NEXT:    movq %rdx, %rcx # sched: [1:0.50]
-; BDVER12-NEXT:    # kill: def $cl killed $cl killed $rcx
-; BDVER12-NEXT:    shrdq %cl, %rsi, %rax # sched: [4:4.00]
-; BDVER12-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: rshift_cl_optsize:
-; BTVER2:       # %bb.0: # %entry
-; BTVER2-NEXT:    movq %rdi, %rax # sched: [1:0.50]
-; BTVER2-NEXT:    movq %rdx, %rcx # sched: [1:0.50]
-; BTVER2-NEXT:    # kill: def $cl killed $cl killed $rcx
-; BTVER2-NEXT:    shrdq %cl, %rsi, %rax # sched: [4:4.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-entry:
-  %shr = lshr i64 %a, %c
-  %sub = sub nsw i64 64, %c
-  %shl = shl i64 %b, %sub
-  %or = or i64 %shr, %shl
-  ret i64 %or
-}
-
-define i64 @rshift_cl(i64 %a, i64 %b, i64 %c) nounwind readnone {
-; GENERIC-LABEL: rshift_cl:
-; GENERIC:       # %bb.0: # %entry
-; GENERIC-NEXT:    movq %rdx, %rcx # sched: [1:0.33]
-; GENERIC-NEXT:    movq %rdi, %rax # sched: [1:0.33]
-; GENERIC-NEXT:    # kill: def $cl killed $cl killed $rcx
-; GENERIC-NEXT:    shrdq %cl, %rsi, %rax # sched: [4:1.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER12-LABEL: rshift_cl:
-; BDVER12:       # %bb.0: # %entry
-; BDVER12-NEXT:    movq %rdx, %rcx # sched: [1:0.50]
-; BDVER12-NEXT:    movq %rsi, %rax # sched: [1:0.50]
-; BDVER12-NEXT:    shrq %cl, %rdi # sched: [1:0.50]
-; BDVER12-NEXT:    negb %cl # sched: [1:0.50]
-; BDVER12-NEXT:    # kill: def $cl killed $cl killed $rcx
-; BDVER12-NEXT:    shlq %cl, %rax # sched: [1:0.50]
-; BDVER12-NEXT:    orq %rdi, %rax # sched: [1:0.50]
-; BDVER12-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: rshift_cl:
-; BTVER2:       # %bb.0: # %entry
-; BTVER2-NEXT:    movq %rdx, %rcx # sched: [1:0.50]
-; BTVER2-NEXT:    movq %rsi, %rax # sched: [1:0.50]
-; BTVER2-NEXT:    shrq %cl, %rdi # sched: [1:0.50]
-; BTVER2-NEXT:    negb %cl # sched: [1:0.50]
-; BTVER2-NEXT:    # kill: def $cl killed $cl killed $rcx
-; BTVER2-NEXT:    shlq %cl, %rax # sched: [1:0.50]
-; BTVER2-NEXT:    orq %rdi, %rax # sched: [1:0.50]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-entry:
-  %shr = lshr i64 %a, %c
-  %sub = sub nsw i64 64, %c
-  %shl = shl i64 %b, %sub
-  %or = or i64 %shr, %shl
-  ret i64 %or
-}
-
-; extern uint64_t x;
-;void lshift(uint64_t a, uint64_t b, uint_64_t c)
-;{
-;    x = (x << c) | (a >> (64-c));
-;}
-@x = global i64 0, align 4
-
-define void @lshift_mem_cl_optsize(i64 %a, i64 %c) nounwind readnone optsize {
-; GENERIC-LABEL: lshift_mem_cl_optsize:
-; GENERIC:       # %bb.0: # %entry
-; GENERIC-NEXT:    movq %rsi, %rcx # sched: [1:0.33]
-; GENERIC-NEXT:    # kill: def $cl killed $cl killed $rcx
-; GENERIC-NEXT:    shldq %cl, %rdi, {{.*}}(%rip) # sched: [10:1.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER12-LABEL: lshift_mem_cl_optsize:
-; BDVER12:       # %bb.0: # %entry
-; BDVER12-NEXT:    movq %rsi, %rcx # sched: [1:0.50]
-; BDVER12-NEXT:    # kill: def $cl killed $cl killed $rcx
-; BDVER12-NEXT:    shldq %cl, %rdi, {{.*}}(%rip) # sched: [4:11.00]
-; BDVER12-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: lshift_mem_cl_optsize:
-; BTVER2:       # %bb.0: # %entry
-; BTVER2-NEXT:    movq %rsi, %rcx # sched: [1:0.50]
-; BTVER2-NEXT:    # kill: def $cl killed $cl killed $rcx
-; BTVER2-NEXT:    shldq %cl, %rdi, {{.*}}(%rip) # sched: [9:11.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-entry:
-  %b = load i64, i64* @x
-  %shl = shl i64 %b, %c
-  %sub = sub nsw i64 64, %c
-  %shr = lshr i64 %a, %sub
-  %or = or i64 %shl, %shr
-  store i64 %or, i64* @x
-  ret void
-}
-
-define void @lshift_mem_cl(i64 %a, i64 %c) nounwind readnone {
-; GENERIC-LABEL: lshift_mem_cl:
-; GENERIC:       # %bb.0: # %entry
-; GENERIC-NEXT:    movq %rsi, %rcx # sched: [1:0.33]
-; GENERIC-NEXT:    # kill: def $cl killed $cl killed $rcx
-; GENERIC-NEXT:    shldq %cl, %rdi, {{.*}}(%rip) # sched: [10:1.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER12-LABEL: lshift_mem_cl:
-; BDVER12:       # %bb.0: # %entry
-; BDVER12-NEXT:    movq {{.*}}(%rip), %rax # sched: [5:0.50]
-; BDVER12-NEXT:    movq %rsi, %rcx # sched: [1:0.50]
-; BDVER12-NEXT:    shlq %cl, %rax # sched: [1:0.50]
-; BDVER12-NEXT:    negb %cl # sched: [1:0.50]
-; BDVER12-NEXT:    # kill: def $cl killed $cl killed $rcx
-; BDVER12-NEXT:    shrq %cl, %rdi # sched: [1:0.50]
-; BDVER12-NEXT:    orq %rax, %rdi # sched: [1:0.50]
-; BDVER12-NEXT:    movq %rdi, {{.*}}(%rip) # sched: [1:1.00]
-; BDVER12-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: lshift_mem_cl:
-; BTVER2:       # %bb.0: # %entry
-; BTVER2-NEXT:    movq {{.*}}(%rip), %rax # sched: [5:1.00]
-; BTVER2-NEXT:    movq %rsi, %rcx # sched: [1:0.50]
-; BTVER2-NEXT:    shlq %cl, %rax # sched: [1:0.50]
-; BTVER2-NEXT:    negb %cl # sched: [1:0.50]
-; BTVER2-NEXT:    # kill: def $cl killed $cl killed $rcx
-; BTVER2-NEXT:    shrq %cl, %rdi # sched: [1:0.50]
-; BTVER2-NEXT:    orq %rax, %rdi # sched: [1:0.50]
-; BTVER2-NEXT:    movq %rdi, {{.*}}(%rip) # sched: [1:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-entry:
-  %b = load i64, i64* @x
-  %shl = shl i64 %b, %c
-  %sub = sub nsw i64 64, %c
-  %shr = lshr i64 %a, %sub
-  %or = or i64 %shl, %shr
-  store i64 %or, i64* @x
-  ret void
-}
-
-define void @lshift_mem(i64 %a) nounwind readnone {
-; GENERIC-LABEL: lshift_mem:
-; GENERIC:       # %bb.0: # %entry
-; GENERIC-NEXT:    shldq $10, %rdi, {{.*}}(%rip) # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER12-LABEL: lshift_mem:
-; BDVER12:       # %bb.0: # %entry
-; BDVER12-NEXT:    movq {{.*}}(%rip), %rax # sched: [5:0.50]
-; BDVER12-NEXT:    shrq $54, %rdi # sched: [1:0.50]
-; BDVER12-NEXT:    shlq $10, %rax # sched: [1:0.50]
-; BDVER12-NEXT:    orq %rax, %rdi # sched: [1:0.50]
-; BDVER12-NEXT:    movq %rdi, {{.*}}(%rip) # sched: [1:1.00]
-; BDVER12-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: lshift_mem:
-; BTVER2:       # %bb.0: # %entry
-; BTVER2-NEXT:    movq {{.*}}(%rip), %rax # sched: [5:1.00]
-; BTVER2-NEXT:    shrq $54, %rdi # sched: [1:0.50]
-; BTVER2-NEXT:    shlq $10, %rax # sched: [1:0.50]
-; BTVER2-NEXT:    orq %rax, %rdi # sched: [1:0.50]
-; BTVER2-NEXT:    movq %rdi, {{.*}}(%rip) # sched: [1:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-entry:
-  %b = load i64, i64* @x
-  %shl = shl i64 %b, 10
-  %shr = lshr i64 %a, 54
-  %or = or i64 %shr, %shl
-  store i64 %or, i64* @x
-  ret void
-}
-
-define void @lshift_mem_optsize(i64 %a) nounwind readnone optsize {
-; GENERIC-LABEL: lshift_mem_optsize:
-; GENERIC:       # %bb.0: # %entry
-; GENERIC-NEXT:    shldq $10, %rdi, {{.*}}(%rip) # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER12-LABEL: lshift_mem_optsize:
-; BDVER12:       # %bb.0: # %entry
-; BDVER12-NEXT:    shldq $10, %rdi, {{.*}}(%rip) # sched: [4:11.00]
-; BDVER12-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: lshift_mem_optsize:
-; BTVER2:       # %bb.0: # %entry
-; BTVER2-NEXT:    shldq $10, %rdi, {{.*}}(%rip) # sched: [9:11.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-entry:
-  %b = load i64, i64* @x
-  %shl = shl i64 %b, 10
-  %shr = lshr i64 %a, 54
-  %or = or i64 %shr, %shl
-  store i64 %or, i64* @x
-  ret void
-}
-
-define void @lshift_mem_b(i64 %b) nounwind readnone {
-; GENERIC-LABEL: lshift_mem_b:
-; GENERIC:       # %bb.0: # %entry
-; GENERIC-NEXT:    movq {{.*}}(%rip), %rax # sched: [5:0.50]
-; GENERIC-NEXT:    shrdq $54, %rdi, %rax # sched: [2:0.67]
-; GENERIC-NEXT:    movq %rax, {{.*}}(%rip) # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER12-LABEL: lshift_mem_b:
-; BDVER12:       # %bb.0: # %entry
-; BDVER12-NEXT:    movq {{.*}}(%rip), %rax # sched: [5:0.50]
-; BDVER12-NEXT:    shlq $10, %rdi # sched: [1:0.50]
-; BDVER12-NEXT:    shrq $54, %rax # sched: [1:0.50]
-; BDVER12-NEXT:    orq %rdi, %rax # sched: [1:0.50]
-; BDVER12-NEXT:    movq %rax, {{.*}}(%rip) # sched: [1:1.00]
-; BDVER12-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: lshift_mem_b:
-; BTVER2:       # %bb.0: # %entry
-; BTVER2-NEXT:    movq {{.*}}(%rip), %rax # sched: [5:1.00]
-; BTVER2-NEXT:    shlq $10, %rdi # sched: [1:0.50]
-; BTVER2-NEXT:    shrq $54, %rax # sched: [1:0.50]
-; BTVER2-NEXT:    orq %rdi, %rax # sched: [1:0.50]
-; BTVER2-NEXT:    movq %rax, {{.*}}(%rip) # sched: [1:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-entry:
-  %a = load i64, i64* @x
-  %shl = shl i64 %b, 10
-  %shr = lshr i64 %a, 54
-  %or = or i64 %shr, %shl
-  store i64 %or, i64* @x
-  ret void
-}
-
-define void @lshift_mem_b_optsize(i64 %b) nounwind readnone optsize {
-; GENERIC-LABEL: lshift_mem_b_optsize:
-; GENERIC:       # %bb.0: # %entry
-; GENERIC-NEXT:    movq {{.*}}(%rip), %rax # sched: [5:0.50]
-; GENERIC-NEXT:    shrdq $54, %rdi, %rax # sched: [2:0.67]
-; GENERIC-NEXT:    movq %rax, {{.*}}(%rip) # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER12-LABEL: lshift_mem_b_optsize:
-; BDVER12:       # %bb.0: # %entry
-; BDVER12-NEXT:    movq {{.*}}(%rip), %rax # sched: [5:0.50]
-; BDVER12-NEXT:    shrdq $54, %rdi, %rax # sched: [4:3.00]
-; BDVER12-NEXT:    movq %rax, {{.*}}(%rip) # sched: [1:1.00]
-; BDVER12-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: lshift_mem_b_optsize:
-; BTVER2:       # %bb.0: # %entry
-; BTVER2-NEXT:    movq {{.*}}(%rip), %rax # sched: [5:1.00]
-; BTVER2-NEXT:    shrdq $54, %rdi, %rax # sched: [3:3.00]
-; BTVER2-NEXT:    movq %rax, {{.*}}(%rip) # sched: [1:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-entry:
-  %a = load i64, i64* @x
-  %shl = shl i64 %b, 10
-  %shr = lshr i64 %a, 54
-  %or = or i64 %shr, %shl
-  store i64 %or, i64* @x
-  ret void
-}
-
diff --git a/test/CodeGen/X86/schedule-x86_32.ll b/test/CodeGen/X86/schedule-x86_32.ll
deleted file mode 100644
index 917f1a7bfa3228adbfff00a95adfe0b0670e2a53..0000000000000000000000000000000000000000
--- a/test/CodeGen/X86/schedule-x86_32.ll
+++ /dev/null
@@ -1,2601 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=i686-unknown-unknown -print-schedule -mcpu=i686 | FileCheck %s --check-prefix=CHECK --check-prefix=GENERIC
-; RUN: llc < %s -mtriple=i686-unknown-unknown -print-schedule -mcpu=atom | FileCheck %s --check-prefix=CHECK --check-prefix=ATOM
-; RUN: llc < %s -mtriple=i686-unknown-unknown -print-schedule -mcpu=slm | FileCheck %s --check-prefix=CHECK --check-prefix=SLM
-; RUN: llc < %s -mtriple=i686-unknown-unknown -print-schedule -mcpu=sandybridge | FileCheck %s --check-prefix=CHECK --check-prefix=SANDY
-; RUN: llc < %s -mtriple=i686-unknown-unknown -print-schedule -mcpu=ivybridge | FileCheck %s --check-prefix=CHECK --check-prefix=SANDY
-; RUN: llc < %s -mtriple=i686-unknown-unknown -print-schedule -mcpu=haswell | FileCheck %s --check-prefix=CHECK --check-prefix=HASWELL
-; RUN: llc < %s -mtriple=i686-unknown-unknown -print-schedule -mcpu=broadwell | FileCheck %s --check-prefix=CHECK --check-prefix=BROADWELL
-; RUN: llc < %s -mtriple=i686-unknown-unknown -print-schedule -mcpu=skylake | FileCheck %s --check-prefix=CHECK --check-prefix=SKYLAKE
-; RUN: llc < %s -mtriple=i686-unknown-unknown -print-schedule -mcpu=skx | FileCheck %s --check-prefix=CHECK --check-prefix=SKX
-; RUN: llc < %s -mtriple=i686-unknown-unknown -print-schedule -mcpu=bdver2 | FileCheck %s --check-prefix=CHECK --check-prefix=BDVER2
-; RUN: llc < %s -mtriple=i686-unknown-unknown -print-schedule -mcpu=btver2 | FileCheck %s --check-prefix=CHECK --check-prefix=BTVER2
-; RUN: llc < %s -mtriple=i686-unknown-unknown -print-schedule -mcpu=znver1 | FileCheck %s --check-prefix=CHECK --check-prefix=ZNVER1
-
-define i8 @test_aaa(i8 %a0) optsize {
-; GENERIC-LABEL: test_aaa:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    movb {{[0-9]+}}(%esp), %al
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    aaa
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retl
-;
-; ATOM-LABEL: test_aaa:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    movb {{[0-9]+}}(%esp), %al # sched: [1:1.00]
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    aaa # sched: [13:6.50]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retl # sched: [79:39.50]
-;
-; SLM-LABEL: test_aaa:
-; SLM:       # %bb.0:
-; SLM-NEXT:    movb {{[0-9]+}}(%esp), %al # sched: [3:1.00]
-; SLM-NEXT:    #APP
-; SLM-NEXT:    aaa # sched: [100:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retl # sched: [4:1.00]
-;
-; SANDY-LABEL: test_aaa:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    movb {{[0-9]+}}(%esp), %al # sched: [5:0.50]
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    aaa # sched: [100:0.33]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retl # sched: [6:1.00]
-;
-; HASWELL-LABEL: test_aaa:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    movb {{[0-9]+}}(%esp), %al # sched: [5:0.50]
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    aaa # sched: [100:0.25]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retl # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_aaa:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    movb {{[0-9]+}}(%esp), %al # sched: [5:0.50]
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    aaa # sched: [100:0.25]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retl # sched: [6:0.50]
-;
-; SKYLAKE-LABEL: test_aaa:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    movb {{[0-9]+}}(%esp), %al # sched: [5:0.50]
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    aaa # sched: [100:0.25]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retl # sched: [6:0.50]
-;
-; SKX-LABEL: test_aaa:
-; SKX:       # %bb.0:
-; SKX-NEXT:    movb {{[0-9]+}}(%esp), %al # sched: [5:0.50]
-; SKX-NEXT:    #APP
-; SKX-NEXT:    aaa # sched: [100:0.25]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retl # sched: [6:0.50]
-;
-; BDVER2-LABEL: test_aaa:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    movb {{[0-9]+}}(%esp), %al # sched: [5:0.50]
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    aaa # sched: [100:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retl # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_aaa:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    movb {{[0-9]+}}(%esp), %al # sched: [5:1.00]
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    aaa # sched: [100:0.50]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retl # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_aaa:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    movb {{[0-9]+}}(%esp), %al # sched: [8:0.50]
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    aaa # sched: [100:0.25]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retl # sched: [1:0.50]
-  %1 = tail call i8 asm "aaa", "=r,r"(i8 %a0) nounwind
-  ret i8 %1
-}
-
-define void @test_aad(i16 %a0) optsize {
-; GENERIC-LABEL: test_aad:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    aad
-; GENERIC-NEXT:    aad $16
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retl
-;
-; ATOM-LABEL: test_aad:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    movzwl {{[0-9]+}}(%esp), %eax # sched: [1:1.00]
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    aad # sched: [7:3.50]
-; ATOM-NEXT:    aad $16 # sched: [7:3.50]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retl # sched: [79:39.50]
-;
-; SLM-LABEL: test_aad:
-; SLM:       # %bb.0:
-; SLM-NEXT:    movzwl {{[0-9]+}}(%esp), %eax # sched: [4:1.00]
-; SLM-NEXT:    #APP
-; SLM-NEXT:    aad # sched: [100:1.00]
-; SLM-NEXT:    aad $16 # sched: [100:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retl # sched: [4:1.00]
-;
-; SANDY-LABEL: test_aad:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    movzwl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    aad # sched: [100:0.33]
-; SANDY-NEXT:    aad $16 # sched: [100:0.33]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retl # sched: [6:1.00]
-;
-; HASWELL-LABEL: test_aad:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    movzwl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    aad # sched: [100:0.25]
-; HASWELL-NEXT:    aad $16 # sched: [100:0.25]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retl # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_aad:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    movzwl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    aad # sched: [100:0.25]
-; BROADWELL-NEXT:    aad $16 # sched: [100:0.25]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retl # sched: [6:0.50]
-;
-; SKYLAKE-LABEL: test_aad:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    movzwl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    aad # sched: [100:0.25]
-; SKYLAKE-NEXT:    aad $16 # sched: [100:0.25]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retl # sched: [6:0.50]
-;
-; SKX-LABEL: test_aad:
-; SKX:       # %bb.0:
-; SKX-NEXT:    movzwl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; SKX-NEXT:    #APP
-; SKX-NEXT:    aad # sched: [100:0.25]
-; SKX-NEXT:    aad $16 # sched: [100:0.25]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retl # sched: [6:0.50]
-;
-; BDVER2-LABEL: test_aad:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    movzwl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    aad # sched: [100:0.50]
-; BDVER2-NEXT:    aad $16 # sched: [100:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retl # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_aad:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    movzwl {{[0-9]+}}(%esp), %eax # sched: [4:1.00]
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    aad # sched: [100:0.50]
-; BTVER2-NEXT:    aad $16 # sched: [100:0.50]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retl # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_aad:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    movzwl {{[0-9]+}}(%esp), %eax # sched: [8:0.50]
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    aad # sched: [100:0.25]
-; ZNVER1-NEXT:    aad $16 # sched: [100:0.25]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retl # sched: [1:0.50]
-  tail call void asm "aad \0A\09 aad $1", "r,i"(i16 %a0, i16 16) nounwind
-  ret void
-}
-
-define void @test_aam(i8 %a0) optsize {
-; GENERIC-LABEL: test_aam:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    movb {{[0-9]+}}(%esp), %al
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    aam
-; GENERIC-NEXT:    aam $16
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retl
-;
-; ATOM-LABEL: test_aam:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    movb {{[0-9]+}}(%esp), %al # sched: [1:1.00]
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    aam # sched: [21:10.50]
-; ATOM-NEXT:    aam $16 # sched: [21:10.50]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retl # sched: [79:39.50]
-;
-; SLM-LABEL: test_aam:
-; SLM:       # %bb.0:
-; SLM-NEXT:    movb {{[0-9]+}}(%esp), %al # sched: [3:1.00]
-; SLM-NEXT:    #APP
-; SLM-NEXT:    aam # sched: [100:1.00]
-; SLM-NEXT:    aam $16 # sched: [100:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retl # sched: [4:1.00]
-;
-; SANDY-LABEL: test_aam:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    movb {{[0-9]+}}(%esp), %al # sched: [5:0.50]
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    aam # sched: [100:0.33]
-; SANDY-NEXT:    aam $16 # sched: [100:0.33]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retl # sched: [6:1.00]
-;
-; HASWELL-LABEL: test_aam:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    movb {{[0-9]+}}(%esp), %al # sched: [5:0.50]
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    aam # sched: [100:0.25]
-; HASWELL-NEXT:    aam $16 # sched: [100:0.25]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retl # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_aam:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    movb {{[0-9]+}}(%esp), %al # sched: [5:0.50]
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    aam # sched: [100:0.25]
-; BROADWELL-NEXT:    aam $16 # sched: [100:0.25]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retl # sched: [6:0.50]
-;
-; SKYLAKE-LABEL: test_aam:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    movb {{[0-9]+}}(%esp), %al # sched: [5:0.50]
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    aam # sched: [100:0.25]
-; SKYLAKE-NEXT:    aam $16 # sched: [100:0.25]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retl # sched: [6:0.50]
-;
-; SKX-LABEL: test_aam:
-; SKX:       # %bb.0:
-; SKX-NEXT:    movb {{[0-9]+}}(%esp), %al # sched: [5:0.50]
-; SKX-NEXT:    #APP
-; SKX-NEXT:    aam # sched: [100:0.25]
-; SKX-NEXT:    aam $16 # sched: [100:0.25]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retl # sched: [6:0.50]
-;
-; BDVER2-LABEL: test_aam:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    movb {{[0-9]+}}(%esp), %al # sched: [5:0.50]
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    aam # sched: [100:0.50]
-; BDVER2-NEXT:    aam $16 # sched: [100:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retl # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_aam:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    movb {{[0-9]+}}(%esp), %al # sched: [5:1.00]
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    aam # sched: [100:0.50]
-; BTVER2-NEXT:    aam $16 # sched: [100:0.50]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retl # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_aam:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    movb {{[0-9]+}}(%esp), %al # sched: [8:0.50]
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    aam # sched: [100:0.25]
-; ZNVER1-NEXT:    aam $16 # sched: [100:0.25]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retl # sched: [1:0.50]
-  tail call void asm "aam \0A\09 aam $1", "r,i"(i8 %a0, i8 16) nounwind
-  ret void
-}
-
-define i8 @test_aas(i8 %a0) optsize {
-; GENERIC-LABEL: test_aas:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    movb {{[0-9]+}}(%esp), %al
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    aas
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retl
-;
-; ATOM-LABEL: test_aas:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    movb {{[0-9]+}}(%esp), %al # sched: [1:1.00]
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    aas # sched: [13:6.50]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retl # sched: [79:39.50]
-;
-; SLM-LABEL: test_aas:
-; SLM:       # %bb.0:
-; SLM-NEXT:    movb {{[0-9]+}}(%esp), %al # sched: [3:1.00]
-; SLM-NEXT:    #APP
-; SLM-NEXT:    aas # sched: [100:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retl # sched: [4:1.00]
-;
-; SANDY-LABEL: test_aas:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    movb {{[0-9]+}}(%esp), %al # sched: [5:0.50]
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    aas # sched: [100:0.33]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retl # sched: [6:1.00]
-;
-; HASWELL-LABEL: test_aas:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    movb {{[0-9]+}}(%esp), %al # sched: [5:0.50]
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    aas # sched: [100:0.25]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retl # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_aas:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    movb {{[0-9]+}}(%esp), %al # sched: [5:0.50]
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    aas # sched: [100:0.25]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retl # sched: [6:0.50]
-;
-; SKYLAKE-LABEL: test_aas:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    movb {{[0-9]+}}(%esp), %al # sched: [5:0.50]
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    aas # sched: [100:0.25]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retl # sched: [6:0.50]
-;
-; SKX-LABEL: test_aas:
-; SKX:       # %bb.0:
-; SKX-NEXT:    movb {{[0-9]+}}(%esp), %al # sched: [5:0.50]
-; SKX-NEXT:    #APP
-; SKX-NEXT:    aas # sched: [100:0.25]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retl # sched: [6:0.50]
-;
-; BDVER2-LABEL: test_aas:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    movb {{[0-9]+}}(%esp), %al # sched: [5:0.50]
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    aas # sched: [100:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retl # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_aas:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    movb {{[0-9]+}}(%esp), %al # sched: [5:1.00]
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    aas # sched: [100:0.50]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retl # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_aas:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    movb {{[0-9]+}}(%esp), %al # sched: [8:0.50]
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    aas # sched: [100:0.25]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retl # sched: [1:0.50]
-  %1 = tail call i8 asm "aas", "=r,r"(i8 %a0) nounwind
-  ret i8 %1
-}
-
-define void @test_arpl(i16 %a0, i16 *%a1) optsize {
-; GENERIC-LABEL: test_arpl:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
-; GENERIC-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    arpl %ax, (%ecx)
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retl
-;
-; ATOM-LABEL: test_arpl:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    movzwl {{[0-9]+}}(%esp), %eax # sched: [1:1.00]
-; ATOM-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [1:1.00]
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    arpl %ax, (%ecx) # sched: [23:11.50]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retl # sched: [79:39.50]
-;
-; SLM-LABEL: test_arpl:
-; SLM:       # %bb.0:
-; SLM-NEXT:    movzwl {{[0-9]+}}(%esp), %eax # sched: [4:1.00]
-; SLM-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [3:1.00]
-; SLM-NEXT:    #APP
-; SLM-NEXT:    arpl %ax, (%ecx) # sched: [100:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retl # sched: [4:1.00]
-;
-; SANDY-LABEL: test_arpl:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    movzwl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; SANDY-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    arpl %ax, (%ecx) # sched: [100:0.33]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retl # sched: [6:1.00]
-;
-; HASWELL-LABEL: test_arpl:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    movzwl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; HASWELL-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    arpl %ax, (%ecx) # sched: [100:0.25]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retl # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_arpl:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    movzwl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; BROADWELL-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    arpl %ax, (%ecx) # sched: [100:0.25]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retl # sched: [6:0.50]
-;
-; SKYLAKE-LABEL: test_arpl:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    movzwl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; SKYLAKE-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    arpl %ax, (%ecx) # sched: [100:0.25]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retl # sched: [6:0.50]
-;
-; SKX-LABEL: test_arpl:
-; SKX:       # %bb.0:
-; SKX-NEXT:    movzwl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; SKX-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; SKX-NEXT:    #APP
-; SKX-NEXT:    arpl %ax, (%ecx) # sched: [100:0.25]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retl # sched: [6:0.50]
-;
-; BDVER2-LABEL: test_arpl:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    movzwl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; BDVER2-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    arpl %ax, (%ecx) # sched: [100:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retl # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_arpl:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    movzwl {{[0-9]+}}(%esp), %eax # sched: [4:1.00]
-; BTVER2-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:1.00]
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    arpl %ax, (%ecx) # sched: [100:0.50]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retl # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_arpl:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    movzwl {{[0-9]+}}(%esp), %eax # sched: [8:0.50]
-; ZNVER1-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [8:0.50]
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    arpl %ax, (%ecx) # sched: [100:0.25]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retl # sched: [1:0.50]
-  call void asm sideeffect "arpl $0, $1", "r,*m"(i16 %a0, i16 *%a1)
-  ret void
-}
-
-define void @test_bound(i16 %a0, i16 *%a1, i32 %a2, i32 *%a3) optsize {
-; GENERIC-LABEL: test_bound:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    pushl %esi
-; GENERIC-NEXT:    .cfi_def_cfa_offset 8
-; GENERIC-NEXT:    .cfi_offset %esi, -8
-; GENERIC-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
-; GENERIC-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; GENERIC-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; GENERIC-NEXT:    movl {{[0-9]+}}(%esp), %esi
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    bound %ax, (%esi)
-; GENERIC-NEXT:    bound %ecx, (%edx)
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    popl %esi
-; GENERIC-NEXT:    .cfi_def_cfa_offset 4
-; GENERIC-NEXT:    retl
-;
-; ATOM-LABEL: test_bound:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    pushl %esi # sched: [1:1.00]
-; ATOM-NEXT:    .cfi_def_cfa_offset 8
-; ATOM-NEXT:    .cfi_offset %esi, -8
-; ATOM-NEXT:    movzwl {{[0-9]+}}(%esp), %eax # sched: [1:1.00]
-; ATOM-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [1:1.00]
-; ATOM-NEXT:    movl {{[0-9]+}}(%esp), %edx # sched: [1:1.00]
-; ATOM-NEXT:    movl {{[0-9]+}}(%esp), %esi # sched: [1:1.00]
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    bound %ax, (%esi) # sched: [11:5.50]
-; ATOM-NEXT:    bound %ecx, (%edx) # sched: [11:5.50]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    popl %esi # sched: [1:1.00]
-; ATOM-NEXT:    .cfi_def_cfa_offset 4
-; ATOM-NEXT:    retl # sched: [79:39.50]
-;
-; SLM-LABEL: test_bound:
-; SLM:       # %bb.0:
-; SLM-NEXT:    pushl %esi # sched: [1:1.00]
-; SLM-NEXT:    .cfi_def_cfa_offset 8
-; SLM-NEXT:    .cfi_offset %esi, -8
-; SLM-NEXT:    movzwl {{[0-9]+}}(%esp), %eax # sched: [4:1.00]
-; SLM-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [3:1.00]
-; SLM-NEXT:    movl {{[0-9]+}}(%esp), %edx # sched: [3:1.00]
-; SLM-NEXT:    movl {{[0-9]+}}(%esp), %esi # sched: [3:1.00]
-; SLM-NEXT:    #APP
-; SLM-NEXT:    bound %ax, (%esi) # sched: [100:1.00]
-; SLM-NEXT:    bound %ecx, (%edx) # sched: [100:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    popl %esi # sched: [3:1.00]
-; SLM-NEXT:    .cfi_def_cfa_offset 4
-; SLM-NEXT:    retl # sched: [4:1.00]
-;
-; SANDY-LABEL: test_bound:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    pushl %esi # sched: [5:1.00]
-; SANDY-NEXT:    .cfi_def_cfa_offset 8
-; SANDY-NEXT:    .cfi_offset %esi, -8
-; SANDY-NEXT:    movzwl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; SANDY-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; SANDY-NEXT:    movl {{[0-9]+}}(%esp), %edx # sched: [5:0.50]
-; SANDY-NEXT:    movl {{[0-9]+}}(%esp), %esi # sched: [5:0.50]
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    bound %ax, (%esi) # sched: [100:0.33]
-; SANDY-NEXT:    bound %ecx, (%edx) # sched: [100:0.33]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    popl %esi # sched: [6:0.50]
-; SANDY-NEXT:    .cfi_def_cfa_offset 4
-; SANDY-NEXT:    retl # sched: [6:1.00]
-;
-; HASWELL-LABEL: test_bound:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    pushl %esi # sched: [2:1.00]
-; HASWELL-NEXT:    .cfi_def_cfa_offset 8
-; HASWELL-NEXT:    .cfi_offset %esi, -8
-; HASWELL-NEXT:    movzwl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; HASWELL-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; HASWELL-NEXT:    movl {{[0-9]+}}(%esp), %edx # sched: [5:0.50]
-; HASWELL-NEXT:    movl {{[0-9]+}}(%esp), %esi # sched: [5:0.50]
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    bound %ax, (%esi) # sched: [1:3.75]
-; HASWELL-NEXT:    bound %ecx, (%edx) # sched: [1:3.75]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    popl %esi # sched: [6:0.50]
-; HASWELL-NEXT:    .cfi_def_cfa_offset 4
-; HASWELL-NEXT:    retl # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_bound:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    pushl %esi # sched: [2:1.00]
-; BROADWELL-NEXT:    .cfi_def_cfa_offset 8
-; BROADWELL-NEXT:    .cfi_offset %esi, -8
-; BROADWELL-NEXT:    movzwl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; BROADWELL-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; BROADWELL-NEXT:    movl {{[0-9]+}}(%esp), %edx # sched: [5:0.50]
-; BROADWELL-NEXT:    movl {{[0-9]+}}(%esp), %esi # sched: [5:0.50]
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    bound %ax, (%esi) # sched: [100:0.25]
-; BROADWELL-NEXT:    bound %ecx, (%edx) # sched: [100:0.25]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    popl %esi # sched: [6:0.50]
-; BROADWELL-NEXT:    .cfi_def_cfa_offset 4
-; BROADWELL-NEXT:    retl # sched: [6:0.50]
-;
-; SKYLAKE-LABEL: test_bound:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    pushl %esi # sched: [2:1.00]
-; SKYLAKE-NEXT:    .cfi_def_cfa_offset 8
-; SKYLAKE-NEXT:    .cfi_offset %esi, -8
-; SKYLAKE-NEXT:    movzwl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; SKYLAKE-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; SKYLAKE-NEXT:    movl {{[0-9]+}}(%esp), %edx # sched: [5:0.50]
-; SKYLAKE-NEXT:    movl {{[0-9]+}}(%esp), %esi # sched: [5:0.50]
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    bound %ax, (%esi) # sched: [100:0.25]
-; SKYLAKE-NEXT:    bound %ecx, (%edx) # sched: [100:0.25]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    popl %esi # sched: [6:0.50]
-; SKYLAKE-NEXT:    .cfi_def_cfa_offset 4
-; SKYLAKE-NEXT:    retl # sched: [6:0.50]
-;
-; SKX-LABEL: test_bound:
-; SKX:       # %bb.0:
-; SKX-NEXT:    pushl %esi # sched: [2:1.00]
-; SKX-NEXT:    .cfi_def_cfa_offset 8
-; SKX-NEXT:    .cfi_offset %esi, -8
-; SKX-NEXT:    movzwl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; SKX-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; SKX-NEXT:    movl {{[0-9]+}}(%esp), %edx # sched: [5:0.50]
-; SKX-NEXT:    movl {{[0-9]+}}(%esp), %esi # sched: [5:0.50]
-; SKX-NEXT:    #APP
-; SKX-NEXT:    bound %ax, (%esi) # sched: [100:0.25]
-; SKX-NEXT:    bound %ecx, (%edx) # sched: [100:0.25]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    popl %esi # sched: [6:0.50]
-; SKX-NEXT:    .cfi_def_cfa_offset 4
-; SKX-NEXT:    retl # sched: [6:0.50]
-;
-; BDVER2-LABEL: test_bound:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    pushl %esi # sched: [1:1.00]
-; BDVER2-NEXT:    .cfi_def_cfa_offset 8
-; BDVER2-NEXT:    .cfi_offset %esi, -8
-; BDVER2-NEXT:    movzwl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; BDVER2-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; BDVER2-NEXT:    movl {{[0-9]+}}(%esp), %edx # sched: [5:0.50]
-; BDVER2-NEXT:    movl {{[0-9]+}}(%esp), %esi # sched: [5:0.50]
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    bound %ax, (%esi) # sched: [100:0.50]
-; BDVER2-NEXT:    bound %ecx, (%edx) # sched: [100:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    popl %esi # sched: [5:0.50]
-; BDVER2-NEXT:    .cfi_def_cfa_offset 4
-; BDVER2-NEXT:    retl # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_bound:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    pushl %esi # sched: [1:1.00]
-; BTVER2-NEXT:    .cfi_def_cfa_offset 8
-; BTVER2-NEXT:    .cfi_offset %esi, -8
-; BTVER2-NEXT:    movzwl {{[0-9]+}}(%esp), %eax # sched: [4:1.00]
-; BTVER2-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:1.00]
-; BTVER2-NEXT:    movl {{[0-9]+}}(%esp), %edx # sched: [5:1.00]
-; BTVER2-NEXT:    movl {{[0-9]+}}(%esp), %esi # sched: [5:1.00]
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    bound %ax, (%esi) # sched: [100:0.50]
-; BTVER2-NEXT:    bound %ecx, (%edx) # sched: [100:0.50]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    popl %esi # sched: [5:1.00]
-; BTVER2-NEXT:    .cfi_def_cfa_offset 4
-; BTVER2-NEXT:    retl # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_bound:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    pushl %esi # sched: [1:0.50]
-; ZNVER1-NEXT:    .cfi_def_cfa_offset 8
-; ZNVER1-NEXT:    .cfi_offset %esi, -8
-; ZNVER1-NEXT:    movzwl {{[0-9]+}}(%esp), %eax # sched: [8:0.50]
-; ZNVER1-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [8:0.50]
-; ZNVER1-NEXT:    movl {{[0-9]+}}(%esp), %edx # sched: [8:0.50]
-; ZNVER1-NEXT:    movl {{[0-9]+}}(%esp), %esi # sched: [8:0.50]
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    bound %ax, (%esi) # sched: [100:0.25]
-; ZNVER1-NEXT:    bound %ecx, (%edx) # sched: [100:0.25]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    popl %esi # sched: [8:0.50]
-; ZNVER1-NEXT:    .cfi_def_cfa_offset 4
-; ZNVER1-NEXT:    retl # sched: [1:0.50]
-  call void asm sideeffect "bound $0, $1 \0A\09 bound $2, $3", "r,*m,r,*m"(i16 %a0, i16 *%a1, i32 %a2, i32 *%a3)
-  ret void
-}
-
-; TODO - test_call
-
-define i8 @test_daa(i8 %a0) optsize {
-; GENERIC-LABEL: test_daa:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    movb {{[0-9]+}}(%esp), %al
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    daa
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retl
-;
-; ATOM-LABEL: test_daa:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    movb {{[0-9]+}}(%esp), %al # sched: [1:1.00]
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    daa # sched: [18:9.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retl # sched: [79:39.50]
-;
-; SLM-LABEL: test_daa:
-; SLM:       # %bb.0:
-; SLM-NEXT:    movb {{[0-9]+}}(%esp), %al # sched: [3:1.00]
-; SLM-NEXT:    #APP
-; SLM-NEXT:    daa # sched: [100:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retl # sched: [4:1.00]
-;
-; SANDY-LABEL: test_daa:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    movb {{[0-9]+}}(%esp), %al # sched: [5:0.50]
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    daa # sched: [100:0.33]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retl # sched: [6:1.00]
-;
-; HASWELL-LABEL: test_daa:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    movb {{[0-9]+}}(%esp), %al # sched: [5:0.50]
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    daa # sched: [100:0.25]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retl # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_daa:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    movb {{[0-9]+}}(%esp), %al # sched: [5:0.50]
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    daa # sched: [100:0.25]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retl # sched: [6:0.50]
-;
-; SKYLAKE-LABEL: test_daa:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    movb {{[0-9]+}}(%esp), %al # sched: [5:0.50]
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    daa # sched: [100:0.25]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retl # sched: [6:0.50]
-;
-; SKX-LABEL: test_daa:
-; SKX:       # %bb.0:
-; SKX-NEXT:    movb {{[0-9]+}}(%esp), %al # sched: [5:0.50]
-; SKX-NEXT:    #APP
-; SKX-NEXT:    daa # sched: [100:0.25]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retl # sched: [6:0.50]
-;
-; BDVER2-LABEL: test_daa:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    movb {{[0-9]+}}(%esp), %al # sched: [5:0.50]
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    daa # sched: [100:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retl # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_daa:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    movb {{[0-9]+}}(%esp), %al # sched: [5:1.00]
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    daa # sched: [100:0.50]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retl # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_daa:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    movb {{[0-9]+}}(%esp), %al # sched: [8:0.50]
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    daa # sched: [100:0.25]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retl # sched: [1:0.50]
-  %1 = tail call i8 asm "daa", "=r,r"(i8 %a0) nounwind
-  ret i8 %1
-}
-
-define i8 @test_das(i8 %a0) optsize {
-; GENERIC-LABEL: test_das:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    movb {{[0-9]+}}(%esp), %al
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    das
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retl
-;
-; ATOM-LABEL: test_das:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    movb {{[0-9]+}}(%esp), %al # sched: [1:1.00]
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    das # sched: [20:10.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retl # sched: [79:39.50]
-;
-; SLM-LABEL: test_das:
-; SLM:       # %bb.0:
-; SLM-NEXT:    movb {{[0-9]+}}(%esp), %al # sched: [3:1.00]
-; SLM-NEXT:    #APP
-; SLM-NEXT:    das # sched: [100:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retl # sched: [4:1.00]
-;
-; SANDY-LABEL: test_das:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    movb {{[0-9]+}}(%esp), %al # sched: [5:0.50]
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    das # sched: [100:0.33]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retl # sched: [6:1.00]
-;
-; HASWELL-LABEL: test_das:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    movb {{[0-9]+}}(%esp), %al # sched: [5:0.50]
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    das # sched: [100:0.25]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retl # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_das:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    movb {{[0-9]+}}(%esp), %al # sched: [5:0.50]
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    das # sched: [100:0.25]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retl # sched: [6:0.50]
-;
-; SKYLAKE-LABEL: test_das:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    movb {{[0-9]+}}(%esp), %al # sched: [5:0.50]
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    das # sched: [100:0.25]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retl # sched: [6:0.50]
-;
-; SKX-LABEL: test_das:
-; SKX:       # %bb.0:
-; SKX-NEXT:    movb {{[0-9]+}}(%esp), %al # sched: [5:0.50]
-; SKX-NEXT:    #APP
-; SKX-NEXT:    das # sched: [100:0.25]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retl # sched: [6:0.50]
-;
-; BDVER2-LABEL: test_das:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    movb {{[0-9]+}}(%esp), %al # sched: [5:0.50]
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    das # sched: [100:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retl # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_das:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    movb {{[0-9]+}}(%esp), %al # sched: [5:1.00]
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    das # sched: [100:0.50]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retl # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_das:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    movb {{[0-9]+}}(%esp), %al # sched: [8:0.50]
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    das # sched: [100:0.25]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retl # sched: [1:0.50]
-  %1 = tail call i8 asm "das", "=r,r"(i8 %a0) nounwind
-  ret i8 %1
-}
-
-define void @test_dec16(i16 %a0, i16* %a1) optsize {
-; GENERIC-LABEL: test_dec16:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
-; GENERIC-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    decw %ax
-; GENERIC-NEXT:    decw (%ecx)
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retl
-;
-; ATOM-LABEL: test_dec16:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    movzwl {{[0-9]+}}(%esp), %eax # sched: [1:1.00]
-; ATOM-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [1:1.00]
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    decw %ax # sched: [1:0.50]
-; ATOM-NEXT:    decw (%ecx) # sched: [1:1.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retl # sched: [79:39.50]
-;
-; SLM-LABEL: test_dec16:
-; SLM:       # %bb.0:
-; SLM-NEXT:    movzwl {{[0-9]+}}(%esp), %eax # sched: [4:1.00]
-; SLM-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [3:1.00]
-; SLM-NEXT:    #APP
-; SLM-NEXT:    decw %ax # sched: [1:0.50]
-; SLM-NEXT:    decw (%ecx) # sched: [5:2.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retl # sched: [4:1.00]
-;
-; SANDY-LABEL: test_dec16:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    movzwl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; SANDY-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    decw %ax # sched: [1:0.33]
-; SANDY-NEXT:    decw (%ecx) # sched: [7:1.00]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retl # sched: [6:1.00]
-;
-; HASWELL-LABEL: test_dec16:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    movzwl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; HASWELL-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    decw %ax # sched: [1:0.25]
-; HASWELL-NEXT:    decw (%ecx) # sched: [7:1.00]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retl # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_dec16:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    movzwl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; BROADWELL-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    decw %ax # sched: [1:0.25]
-; BROADWELL-NEXT:    decw (%ecx) # sched: [7:1.00]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retl # sched: [6:0.50]
-;
-; SKYLAKE-LABEL: test_dec16:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    movzwl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; SKYLAKE-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    decw %ax # sched: [1:0.25]
-; SKYLAKE-NEXT:    decw (%ecx) # sched: [7:1.00]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retl # sched: [6:0.50]
-;
-; SKX-LABEL: test_dec16:
-; SKX:       # %bb.0:
-; SKX-NEXT:    movzwl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; SKX-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; SKX-NEXT:    #APP
-; SKX-NEXT:    decw %ax # sched: [1:0.25]
-; SKX-NEXT:    decw (%ecx) # sched: [7:1.00]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retl # sched: [6:0.50]
-;
-; BDVER2-LABEL: test_dec16:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    movzwl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; BDVER2-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    decw %ax # sched: [1:0.50]
-; BDVER2-NEXT:    decw (%ecx) # sched: [6:1.00]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retl # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_dec16:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    movzwl {{[0-9]+}}(%esp), %eax # sched: [4:1.00]
-; BTVER2-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:1.00]
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    decw %ax # sched: [1:0.50]
-; BTVER2-NEXT:    decw (%ecx) # sched: [5:1.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retl # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_dec16:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    movzwl {{[0-9]+}}(%esp), %eax # sched: [8:0.50]
-; ZNVER1-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [8:0.50]
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    decw %ax # sched: [1:0.25]
-; ZNVER1-NEXT:    decw (%ecx) # sched: [5:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retl # sched: [1:0.50]
-  tail call void asm "decw $0 \0A\09 decw $1", "r,*m"(i16 %a0, i16* %a1) nounwind
-  ret void
-}
-define void @test_dec32(i32 %a0, i32* %a1) optsize {
-; GENERIC-LABEL: test_dec32:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; GENERIC-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    decl %eax
-; GENERIC-NEXT:    decl (%ecx)
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retl
-;
-; ATOM-LABEL: test_dec32:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [1:1.00]
-; ATOM-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [1:1.00]
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    decl %eax # sched: [1:0.50]
-; ATOM-NEXT:    decl (%ecx) # sched: [1:1.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retl # sched: [79:39.50]
-;
-; SLM-LABEL: test_dec32:
-; SLM:       # %bb.0:
-; SLM-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [3:1.00]
-; SLM-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [3:1.00]
-; SLM-NEXT:    #APP
-; SLM-NEXT:    decl %eax # sched: [1:0.50]
-; SLM-NEXT:    decl (%ecx) # sched: [5:2.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retl # sched: [4:1.00]
-;
-; SANDY-LABEL: test_dec32:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; SANDY-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    decl %eax # sched: [1:0.33]
-; SANDY-NEXT:    decl (%ecx) # sched: [7:1.00]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retl # sched: [6:1.00]
-;
-; HASWELL-LABEL: test_dec32:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; HASWELL-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    decl %eax # sched: [1:0.25]
-; HASWELL-NEXT:    decl (%ecx) # sched: [7:1.00]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retl # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_dec32:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; BROADWELL-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    decl %eax # sched: [1:0.25]
-; BROADWELL-NEXT:    decl (%ecx) # sched: [7:1.00]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retl # sched: [6:0.50]
-;
-; SKYLAKE-LABEL: test_dec32:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; SKYLAKE-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    decl %eax # sched: [1:0.25]
-; SKYLAKE-NEXT:    decl (%ecx) # sched: [7:1.00]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retl # sched: [6:0.50]
-;
-; SKX-LABEL: test_dec32:
-; SKX:       # %bb.0:
-; SKX-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; SKX-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; SKX-NEXT:    #APP
-; SKX-NEXT:    decl %eax # sched: [1:0.25]
-; SKX-NEXT:    decl (%ecx) # sched: [7:1.00]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retl # sched: [6:0.50]
-;
-; BDVER2-LABEL: test_dec32:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; BDVER2-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    decl %eax # sched: [1:0.50]
-; BDVER2-NEXT:    decl (%ecx) # sched: [6:1.00]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retl # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_dec32:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:1.00]
-; BTVER2-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:1.00]
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    decl %eax # sched: [1:0.50]
-; BTVER2-NEXT:    decl (%ecx) # sched: [5:1.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retl # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_dec32:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [8:0.50]
-; ZNVER1-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [8:0.50]
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    decl %eax # sched: [1:0.25]
-; ZNVER1-NEXT:    decl (%ecx) # sched: [5:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retl # sched: [1:0.50]
-  tail call void asm "decl $0 \0A\09 decl $1", "r,*m"(i32 %a0, i32* %a1) nounwind
-  ret void
-}
-
-define void @test_inc16(i16 %a0, i16* %a1) optsize {
-; GENERIC-LABEL: test_inc16:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
-; GENERIC-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    incw %ax
-; GENERIC-NEXT:    incw (%ecx)
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retl
-;
-; ATOM-LABEL: test_inc16:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    movzwl {{[0-9]+}}(%esp), %eax # sched: [1:1.00]
-; ATOM-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [1:1.00]
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    incw %ax # sched: [1:0.50]
-; ATOM-NEXT:    incw (%ecx) # sched: [1:1.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retl # sched: [79:39.50]
-;
-; SLM-LABEL: test_inc16:
-; SLM:       # %bb.0:
-; SLM-NEXT:    movzwl {{[0-9]+}}(%esp), %eax # sched: [4:1.00]
-; SLM-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [3:1.00]
-; SLM-NEXT:    #APP
-; SLM-NEXT:    incw %ax # sched: [1:0.50]
-; SLM-NEXT:    incw (%ecx) # sched: [5:2.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retl # sched: [4:1.00]
-;
-; SANDY-LABEL: test_inc16:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    movzwl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; SANDY-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    incw %ax # sched: [1:0.33]
-; SANDY-NEXT:    incw (%ecx) # sched: [7:1.00]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retl # sched: [6:1.00]
-;
-; HASWELL-LABEL: test_inc16:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    movzwl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; HASWELL-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    incw %ax # sched: [1:0.25]
-; HASWELL-NEXT:    incw (%ecx) # sched: [7:1.00]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retl # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_inc16:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    movzwl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; BROADWELL-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    incw %ax # sched: [1:0.25]
-; BROADWELL-NEXT:    incw (%ecx) # sched: [7:1.00]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retl # sched: [6:0.50]
-;
-; SKYLAKE-LABEL: test_inc16:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    movzwl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; SKYLAKE-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    incw %ax # sched: [1:0.25]
-; SKYLAKE-NEXT:    incw (%ecx) # sched: [7:1.00]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retl # sched: [6:0.50]
-;
-; SKX-LABEL: test_inc16:
-; SKX:       # %bb.0:
-; SKX-NEXT:    movzwl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; SKX-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; SKX-NEXT:    #APP
-; SKX-NEXT:    incw %ax # sched: [1:0.25]
-; SKX-NEXT:    incw (%ecx) # sched: [7:1.00]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retl # sched: [6:0.50]
-;
-; BDVER2-LABEL: test_inc16:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    movzwl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; BDVER2-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    incw %ax # sched: [1:0.50]
-; BDVER2-NEXT:    incw (%ecx) # sched: [6:1.00]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retl # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_inc16:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    movzwl {{[0-9]+}}(%esp), %eax # sched: [4:1.00]
-; BTVER2-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:1.00]
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    incw %ax # sched: [1:0.50]
-; BTVER2-NEXT:    incw (%ecx) # sched: [5:1.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retl # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_inc16:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    movzwl {{[0-9]+}}(%esp), %eax # sched: [8:0.50]
-; ZNVER1-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [8:0.50]
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    incw %ax # sched: [1:0.25]
-; ZNVER1-NEXT:    incw (%ecx) # sched: [5:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retl # sched: [1:0.50]
-  tail call void asm "incw $0 \0A\09 incw $1", "r,*m"(i16 %a0, i16* %a1) nounwind
-  ret void
-}
-define void @test_inc32(i32 %a0, i32* %a1) optsize {
-; GENERIC-LABEL: test_inc32:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; GENERIC-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    incl %eax
-; GENERIC-NEXT:    incl (%ecx)
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retl
-;
-; ATOM-LABEL: test_inc32:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [1:1.00]
-; ATOM-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [1:1.00]
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    incl %eax # sched: [1:0.50]
-; ATOM-NEXT:    incl (%ecx) # sched: [1:1.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retl # sched: [79:39.50]
-;
-; SLM-LABEL: test_inc32:
-; SLM:       # %bb.0:
-; SLM-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [3:1.00]
-; SLM-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [3:1.00]
-; SLM-NEXT:    #APP
-; SLM-NEXT:    incl %eax # sched: [1:0.50]
-; SLM-NEXT:    incl (%ecx) # sched: [5:2.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retl # sched: [4:1.00]
-;
-; SANDY-LABEL: test_inc32:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; SANDY-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    incl %eax # sched: [1:0.33]
-; SANDY-NEXT:    incl (%ecx) # sched: [7:1.00]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retl # sched: [6:1.00]
-;
-; HASWELL-LABEL: test_inc32:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; HASWELL-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    incl %eax # sched: [1:0.25]
-; HASWELL-NEXT:    incl (%ecx) # sched: [7:1.00]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retl # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_inc32:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; BROADWELL-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    incl %eax # sched: [1:0.25]
-; BROADWELL-NEXT:    incl (%ecx) # sched: [7:1.00]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retl # sched: [6:0.50]
-;
-; SKYLAKE-LABEL: test_inc32:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; SKYLAKE-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    incl %eax # sched: [1:0.25]
-; SKYLAKE-NEXT:    incl (%ecx) # sched: [7:1.00]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retl # sched: [6:0.50]
-;
-; SKX-LABEL: test_inc32:
-; SKX:       # %bb.0:
-; SKX-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; SKX-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; SKX-NEXT:    #APP
-; SKX-NEXT:    incl %eax # sched: [1:0.25]
-; SKX-NEXT:    incl (%ecx) # sched: [7:1.00]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retl # sched: [6:0.50]
-;
-; BDVER2-LABEL: test_inc32:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; BDVER2-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    incl %eax # sched: [1:0.50]
-; BDVER2-NEXT:    incl (%ecx) # sched: [6:1.00]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retl # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_inc32:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:1.00]
-; BTVER2-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:1.00]
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    incl %eax # sched: [1:0.50]
-; BTVER2-NEXT:    incl (%ecx) # sched: [5:1.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retl # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_inc32:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [8:0.50]
-; ZNVER1-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [8:0.50]
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    incl %eax # sched: [1:0.25]
-; ZNVER1-NEXT:    incl (%ecx) # sched: [5:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retl # sched: [1:0.50]
-  tail call void asm "incl $0 \0A\09 incl $1", "r,*m"(i32 %a0, i32* %a1) nounwind
-  ret void
-}
-
-define void @test_into() optsize {
-; GENERIC-LABEL: test_into:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    into
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retl
-;
-; ATOM-LABEL: test_into:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    into # sched: [6:3.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retl # sched: [79:39.50]
-;
-; SLM-LABEL: test_into:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    into # sched: [100:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retl # sched: [4:1.00]
-;
-; SANDY-LABEL: test_into:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    into # sched: [100:0.33]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retl # sched: [6:1.00]
-;
-; HASWELL-LABEL: test_into:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    into # sched: [1:1.00]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retl # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_into:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    into # sched: [100:0.25]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retl # sched: [6:0.50]
-;
-; SKYLAKE-LABEL: test_into:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    into # sched: [100:0.25]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retl # sched: [6:0.50]
-;
-; SKX-LABEL: test_into:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    into # sched: [100:0.25]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retl # sched: [6:0.50]
-;
-; BDVER2-LABEL: test_into:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    into # sched: [100:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retl # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_into:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    into # sched: [100:0.50]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retl # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_into:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    into # sched: [100:0.25]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retl # sched: [1:0.50]
-  call void asm sideeffect "into", ""()
-  ret void
-}
-
-; TODO - test_jmp
-
-define void @test_jcxz_jecxz() optsize {
-; GENERIC-LABEL: test_jcxz_jecxz:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:  JXTGT:
-; GENERIC-NEXT:    jcxz JXTGT
-; GENERIC-NEXT:    jecxz JXTGT
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retl
-;
-; ATOM-LABEL: test_jcxz_jecxz:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:  JXTGT:
-; ATOM-NEXT:    jcxz JXTGT # sched: [4:2.00]
-; ATOM-NEXT:    jecxz JXTGT # sched: [4:2.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retl # sched: [79:39.50]
-;
-; SLM-LABEL: test_jcxz_jecxz:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:  JXTGT:
-; SLM-NEXT:    jcxz JXTGT # sched: [1:1.00]
-; SLM-NEXT:    jecxz JXTGT # sched: [1:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retl # sched: [4:1.00]
-;
-; SANDY-LABEL: test_jcxz_jecxz:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:  JXTGT:
-; SANDY-NEXT:    jcxz JXTGT # sched: [2:1.00]
-; SANDY-NEXT:    jecxz JXTGT # sched: [2:1.00]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retl # sched: [6:1.00]
-;
-; HASWELL-LABEL: test_jcxz_jecxz:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:  JXTGT:
-; HASWELL-NEXT:    jcxz JXTGT # sched: [2:0.50]
-; HASWELL-NEXT:    jecxz JXTGT # sched: [2:0.50]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retl # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_jcxz_jecxz:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:  JXTGT:
-; BROADWELL-NEXT:    jcxz JXTGT # sched: [2:0.50]
-; BROADWELL-NEXT:    jecxz JXTGT # sched: [2:0.50]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retl # sched: [6:0.50]
-;
-; SKYLAKE-LABEL: test_jcxz_jecxz:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:  JXTGT:
-; SKYLAKE-NEXT:    jcxz JXTGT # sched: [2:0.50]
-; SKYLAKE-NEXT:    jecxz JXTGT # sched: [2:0.50]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retl # sched: [6:0.50]
-;
-; SKX-LABEL: test_jcxz_jecxz:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:  JXTGT:
-; SKX-NEXT:    jcxz JXTGT # sched: [2:0.50]
-; SKX-NEXT:    jecxz JXTGT # sched: [2:0.50]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retl # sched: [6:0.50]
-;
-; BDVER2-LABEL: test_jcxz_jecxz:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:  JXTGT:
-; BDVER2-NEXT:    jcxz JXTGT # sched: [1:1.00]
-; BDVER2-NEXT:    jecxz JXTGT # sched: [1:1.00]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retl # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_jcxz_jecxz:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:  JXTGT:
-; BTVER2-NEXT:    jcxz JXTGT # sched: [1:0.50]
-; BTVER2-NEXT:    jecxz JXTGT # sched: [1:0.50]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retl # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_jcxz_jecxz:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:  JXTGT:
-; ZNVER1-NEXT:    jcxz JXTGT # sched: [1:0.50]
-; ZNVER1-NEXT:    jecxz JXTGT # sched: [1:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retl # sched: [1:0.50]
-  call void asm sideeffect "JXTGT: \0A\09 jcxz JXTGT \0A\09 jecxz JXTGT", ""()
-  ret void
-}
-
-; TODO - test_lds
-
-define void @test_leave() optsize {
-; GENERIC-LABEL: test_leave:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    leave
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retl
-;
-; ATOM-LABEL: test_leave:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    leave # sched: [2:1.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retl # sched: [79:39.50]
-;
-; SLM-LABEL: test_leave:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    leave # sched: [1:0.50]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retl # sched: [4:1.00]
-;
-; SANDY-LABEL: test_leave:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    leave # sched: [7:0.67]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retl # sched: [6:1.00]
-;
-; HASWELL-LABEL: test_leave:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    leave # sched: [7:0.50]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retl # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_leave:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    leave # sched: [7:0.50]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retl # sched: [6:0.50]
-;
-; SKYLAKE-LABEL: test_leave:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    leave # sched: [7:0.50]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retl # sched: [6:0.50]
-;
-; SKX-LABEL: test_leave:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    leave # sched: [7:0.50]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retl # sched: [6:0.50]
-;
-; BDVER2-LABEL: test_leave:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    leave # sched: [1:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retl # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_leave:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    leave # sched: [1:0.50]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retl # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_leave:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    leave # sched: [8:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retl # sched: [1:0.50]
-  tail call void asm "leave", ""() nounwind
-  ret void
-}
-
-; TODO - test_les
-
-define void @test_pop_push() optsize {
-; GENERIC-LABEL: test_pop_push:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    popl %ds
-; GENERIC-NEXT:    popl %es
-; GENERIC-NEXT:    popl %ss
-; GENERIC-NEXT:    popl %fs
-; GENERIC-NEXT:    popl %gs
-; GENERIC-NEXT:    pushl %cs
-; GENERIC-NEXT:    pushl %ds
-; GENERIC-NEXT:    pushl %es
-; GENERIC-NEXT:    pushl %ss
-; GENERIC-NEXT:    pushl %fs
-; GENERIC-NEXT:    pushl %gs
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retl
-;
-; ATOM-LABEL: test_pop_push:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    popl %ds # sched: [29:14.50]
-; ATOM-NEXT:    popl %es # sched: [29:14.50]
-; ATOM-NEXT:    popl %ss # sched: [48:24.00]
-; ATOM-NEXT:    popl %fs # sched: [29:14.50]
-; ATOM-NEXT:    popl %gs # sched: [29:14.50]
-; ATOM-NEXT:    pushl %cs # sched: [2:1.00]
-; ATOM-NEXT:    pushl %ds # sched: [2:1.00]
-; ATOM-NEXT:    pushl %es # sched: [2:1.00]
-; ATOM-NEXT:    pushl %ss # sched: [2:1.00]
-; ATOM-NEXT:    pushl %fs # sched: [2:1.00]
-; ATOM-NEXT:    pushl %gs # sched: [2:1.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retl # sched: [79:39.50]
-;
-; SLM-LABEL: test_pop_push:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    popl %ds # sched: [100:1.00]
-; SLM-NEXT:    popl %es # sched: [100:1.00]
-; SLM-NEXT:    popl %ss # sched: [100:1.00]
-; SLM-NEXT:    popl %fs # sched: [100:1.00]
-; SLM-NEXT:    popl %gs # sched: [100:1.00]
-; SLM-NEXT:    pushl %cs # sched: [100:1.00]
-; SLM-NEXT:    pushl %ds # sched: [100:1.00]
-; SLM-NEXT:    pushl %es # sched: [100:1.00]
-; SLM-NEXT:    pushl %ss # sched: [100:1.00]
-; SLM-NEXT:    pushl %fs # sched: [100:1.00]
-; SLM-NEXT:    pushl %gs # sched: [100:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retl # sched: [4:1.00]
-;
-; SANDY-LABEL: test_pop_push:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    popl %ds # sched: [100:0.33]
-; SANDY-NEXT:    popl %es # sched: [100:0.33]
-; SANDY-NEXT:    popl %ss # sched: [100:0.33]
-; SANDY-NEXT:    popl %fs # sched: [100:0.33]
-; SANDY-NEXT:    popl %gs # sched: [100:0.33]
-; SANDY-NEXT:    pushl %cs # sched: [100:0.33]
-; SANDY-NEXT:    pushl %ds # sched: [100:0.33]
-; SANDY-NEXT:    pushl %es # sched: [100:0.33]
-; SANDY-NEXT:    pushl %ss # sched: [100:0.33]
-; SANDY-NEXT:    pushl %fs # sched: [100:0.33]
-; SANDY-NEXT:    pushl %gs # sched: [100:0.33]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retl # sched: [6:1.00]
-;
-; HASWELL-LABEL: test_pop_push:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    popl %ds # sched: [100:0.25]
-; HASWELL-NEXT:    popl %es # sched: [100:0.25]
-; HASWELL-NEXT:    popl %ss # sched: [100:0.25]
-; HASWELL-NEXT:    popl %fs # sched: [100:0.25]
-; HASWELL-NEXT:    popl %gs # sched: [100:0.25]
-; HASWELL-NEXT:    pushl %cs # sched: [100:0.25]
-; HASWELL-NEXT:    pushl %ds # sched: [100:0.25]
-; HASWELL-NEXT:    pushl %es # sched: [100:0.25]
-; HASWELL-NEXT:    pushl %ss # sched: [100:0.25]
-; HASWELL-NEXT:    pushl %fs # sched: [100:0.25]
-; HASWELL-NEXT:    pushl %gs # sched: [100:0.25]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retl # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pop_push:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    popl %ds # sched: [100:0.25]
-; BROADWELL-NEXT:    popl %es # sched: [100:0.25]
-; BROADWELL-NEXT:    popl %ss # sched: [100:0.25]
-; BROADWELL-NEXT:    popl %fs # sched: [100:0.25]
-; BROADWELL-NEXT:    popl %gs # sched: [100:0.25]
-; BROADWELL-NEXT:    pushl %cs # sched: [100:0.25]
-; BROADWELL-NEXT:    pushl %ds # sched: [100:0.25]
-; BROADWELL-NEXT:    pushl %es # sched: [100:0.25]
-; BROADWELL-NEXT:    pushl %ss # sched: [100:0.25]
-; BROADWELL-NEXT:    pushl %fs # sched: [100:0.25]
-; BROADWELL-NEXT:    pushl %gs # sched: [100:0.25]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retl # sched: [6:0.50]
-;
-; SKYLAKE-LABEL: test_pop_push:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    popl %ds # sched: [100:0.25]
-; SKYLAKE-NEXT:    popl %es # sched: [100:0.25]
-; SKYLAKE-NEXT:    popl %ss # sched: [100:0.25]
-; SKYLAKE-NEXT:    popl %fs # sched: [100:0.25]
-; SKYLAKE-NEXT:    popl %gs # sched: [100:0.25]
-; SKYLAKE-NEXT:    pushl %cs # sched: [100:0.25]
-; SKYLAKE-NEXT:    pushl %ds # sched: [100:0.25]
-; SKYLAKE-NEXT:    pushl %es # sched: [100:0.25]
-; SKYLAKE-NEXT:    pushl %ss # sched: [100:0.25]
-; SKYLAKE-NEXT:    pushl %fs # sched: [100:0.25]
-; SKYLAKE-NEXT:    pushl %gs # sched: [100:0.25]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retl # sched: [6:0.50]
-;
-; SKX-LABEL: test_pop_push:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    popl %ds # sched: [100:0.25]
-; SKX-NEXT:    popl %es # sched: [100:0.25]
-; SKX-NEXT:    popl %ss # sched: [100:0.25]
-; SKX-NEXT:    popl %fs # sched: [100:0.25]
-; SKX-NEXT:    popl %gs # sched: [100:0.25]
-; SKX-NEXT:    pushl %cs # sched: [100:0.25]
-; SKX-NEXT:    pushl %ds # sched: [100:0.25]
-; SKX-NEXT:    pushl %es # sched: [100:0.25]
-; SKX-NEXT:    pushl %ss # sched: [100:0.25]
-; SKX-NEXT:    pushl %fs # sched: [100:0.25]
-; SKX-NEXT:    pushl %gs # sched: [100:0.25]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retl # sched: [6:0.50]
-;
-; BDVER2-LABEL: test_pop_push:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    popl %ds # sched: [100:0.50]
-; BDVER2-NEXT:    popl %es # sched: [100:0.50]
-; BDVER2-NEXT:    popl %ss # sched: [100:0.50]
-; BDVER2-NEXT:    popl %fs # sched: [100:0.50]
-; BDVER2-NEXT:    popl %gs # sched: [100:0.50]
-; BDVER2-NEXT:    pushl %cs # sched: [100:0.50]
-; BDVER2-NEXT:    pushl %ds # sched: [100:0.50]
-; BDVER2-NEXT:    pushl %es # sched: [100:0.50]
-; BDVER2-NEXT:    pushl %ss # sched: [100:0.50]
-; BDVER2-NEXT:    pushl %fs # sched: [100:0.50]
-; BDVER2-NEXT:    pushl %gs # sched: [100:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retl # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_pop_push:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    popl %ds # sched: [100:0.50]
-; BTVER2-NEXT:    popl %es # sched: [100:0.50]
-; BTVER2-NEXT:    popl %ss # sched: [100:0.50]
-; BTVER2-NEXT:    popl %fs # sched: [100:0.50]
-; BTVER2-NEXT:    popl %gs # sched: [100:0.50]
-; BTVER2-NEXT:    pushl %cs # sched: [100:0.50]
-; BTVER2-NEXT:    pushl %ds # sched: [100:0.50]
-; BTVER2-NEXT:    pushl %es # sched: [100:0.50]
-; BTVER2-NEXT:    pushl %ss # sched: [100:0.50]
-; BTVER2-NEXT:    pushl %fs # sched: [100:0.50]
-; BTVER2-NEXT:    pushl %gs # sched: [100:0.50]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retl # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_pop_push:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    popl %ds # sched: [100:0.25]
-; ZNVER1-NEXT:    popl %es # sched: [100:0.25]
-; ZNVER1-NEXT:    popl %ss # sched: [100:0.25]
-; ZNVER1-NEXT:    popl %fs # sched: [100:0.25]
-; ZNVER1-NEXT:    popl %gs # sched: [100:0.25]
-; ZNVER1-NEXT:    pushl %cs # sched: [100:0.25]
-; ZNVER1-NEXT:    pushl %ds # sched: [100:0.25]
-; ZNVER1-NEXT:    pushl %es # sched: [100:0.25]
-; ZNVER1-NEXT:    pushl %ss # sched: [100:0.25]
-; ZNVER1-NEXT:    pushl %fs # sched: [100:0.25]
-; ZNVER1-NEXT:    pushl %gs # sched: [100:0.25]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retl # sched: [1:0.50]
-  call void asm sideeffect "pop %DS \0A\09 pop %ES \0A\09 pop %SS \0A\09 pop %FS \0A\09 pop %GS \0A\09 push %CS \0A\09 push %DS \0A\09 push %ES \0A\09 push %SS \0A\09 push %FS \0A\09 push %GS", ""()
-  ret void
-}
-define i16 @test_pop_push_16(i16 %a0, i16 *%a1) optsize {
-; GENERIC-LABEL: test_pop_push_16:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
-; GENERIC-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    popw %ax
-; GENERIC-NEXT:    popw (%ecx)
-; GENERIC-NEXT:    pushw %ax
-; GENERIC-NEXT:    pushw (%ecx)
-; GENERIC-NEXT:    pushw $4095 # imm = 0xFFF
-; GENERIC-NEXT:    pushw $7
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retl
-;
-; ATOM-LABEL: test_pop_push_16:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    movzwl {{[0-9]+}}(%esp), %eax # sched: [1:1.00]
-; ATOM-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [1:1.00]
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    popw %ax # sched: [2:1.00]
-; ATOM-NEXT:    popw (%ecx) # sched: [3:1.50]
-; ATOM-NEXT:    pushw %ax # sched: [1:1.00]
-; ATOM-NEXT:    pushw (%ecx) # sched: [2:1.00]
-; ATOM-NEXT:    pushw $4095 # imm = 0xFFF
-; ATOM-NEXT:    # sched: [1:1.00]
-; ATOM-NEXT:    pushw $7 # sched: [1:1.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retl # sched: [79:39.50]
-;
-; SLM-LABEL: test_pop_push_16:
-; SLM:       # %bb.0:
-; SLM-NEXT:    movzwl {{[0-9]+}}(%esp), %eax # sched: [4:1.00]
-; SLM-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [3:1.00]
-; SLM-NEXT:    #APP
-; SLM-NEXT:    popw %ax # sched: [3:1.00]
-; SLM-NEXT:    popw (%ecx) # sched: [4:2.00]
-; SLM-NEXT:    pushw %ax # sched: [1:1.00]
-; SLM-NEXT:    pushw (%ecx) # sched: [4:2.00]
-; SLM-NEXT:    pushw $4095 # imm = 0xFFF
-; SLM-NEXT:    # sched: [1:1.00]
-; SLM-NEXT:    pushw $7 # sched: [1:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retl # sched: [4:1.00]
-;
-; SANDY-LABEL: test_pop_push_16:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    movzwl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; SANDY-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    popw %ax # sched: [6:0.50]
-; SANDY-NEXT:    popw (%ecx) # sched: [6:0.50]
-; SANDY-NEXT:    pushw %ax # sched: [5:1.00]
-; SANDY-NEXT:    pushw (%ecx) # sched: [5:1.00]
-; SANDY-NEXT:    pushw $4095 # imm = 0xFFF
-; SANDY-NEXT:    # sched: [1:1.00]
-; SANDY-NEXT:    pushw $7 # sched: [1:1.00]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retl # sched: [6:1.00]
-;
-; HASWELL-LABEL: test_pop_push_16:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    movzwl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; HASWELL-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    popw %ax # sched: [6:0.50]
-; HASWELL-NEXT:    popw (%ecx) # sched: [7:1.00]
-; HASWELL-NEXT:    pushw %ax # sched: [2:1.00]
-; HASWELL-NEXT:    pushw (%ecx) # sched: [7:1.00]
-; HASWELL-NEXT:    pushw $4095 # imm = 0xFFF
-; HASWELL-NEXT:    # sched: [1:1.00]
-; HASWELL-NEXT:    pushw $7 # sched: [1:1.00]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retl # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pop_push_16:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    movzwl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; BROADWELL-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    popw %ax # sched: [6:0.50]
-; BROADWELL-NEXT:    popw (%ecx) # sched: [6:1.00]
-; BROADWELL-NEXT:    pushw %ax # sched: [2:1.00]
-; BROADWELL-NEXT:    pushw (%ecx) # sched: [6:1.00]
-; BROADWELL-NEXT:    pushw $4095 # imm = 0xFFF
-; BROADWELL-NEXT:    # sched: [1:1.00]
-; BROADWELL-NEXT:    pushw $7 # sched: [1:1.00]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retl # sched: [6:0.50]
-;
-; SKYLAKE-LABEL: test_pop_push_16:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    movzwl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; SKYLAKE-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    popw %ax # sched: [6:0.50]
-; SKYLAKE-NEXT:    popw (%ecx) # sched: [6:1.00]
-; SKYLAKE-NEXT:    pushw %ax # sched: [2:1.00]
-; SKYLAKE-NEXT:    pushw (%ecx) # sched: [6:1.00]
-; SKYLAKE-NEXT:    pushw $4095 # imm = 0xFFF
-; SKYLAKE-NEXT:    # sched: [1:1.00]
-; SKYLAKE-NEXT:    pushw $7 # sched: [1:1.00]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retl # sched: [6:0.50]
-;
-; SKX-LABEL: test_pop_push_16:
-; SKX:       # %bb.0:
-; SKX-NEXT:    movzwl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; SKX-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; SKX-NEXT:    #APP
-; SKX-NEXT:    popw %ax # sched: [6:0.50]
-; SKX-NEXT:    popw (%ecx) # sched: [6:1.00]
-; SKX-NEXT:    pushw %ax # sched: [2:1.00]
-; SKX-NEXT:    pushw (%ecx) # sched: [6:1.00]
-; SKX-NEXT:    pushw $4095 # imm = 0xFFF
-; SKX-NEXT:    # sched: [1:1.00]
-; SKX-NEXT:    pushw $7 # sched: [1:1.00]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retl # sched: [6:0.50]
-;
-; BDVER2-LABEL: test_pop_push_16:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    movzwl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; BDVER2-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    popw %ax # sched: [5:0.50]
-; BDVER2-NEXT:    popw (%ecx) # sched: [6:1.00]
-; BDVER2-NEXT:    pushw %ax # sched: [1:1.00]
-; BDVER2-NEXT:    pushw (%ecx) # sched: [6:1.00]
-; BDVER2-NEXT:    pushw $4095 # imm = 0xFFF
-; BDVER2-NEXT:    # sched: [1:1.00]
-; BDVER2-NEXT:    pushw $7 # sched: [1:1.00]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retl # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_pop_push_16:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    movzwl {{[0-9]+}}(%esp), %eax # sched: [4:1.00]
-; BTVER2-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:1.00]
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    popw %ax # sched: [5:1.00]
-; BTVER2-NEXT:    popw (%ecx) # sched: [6:1.00]
-; BTVER2-NEXT:    pushw %ax # sched: [1:1.00]
-; BTVER2-NEXT:    pushw (%ecx) # sched: [6:1.00]
-; BTVER2-NEXT:    pushw $4095 # imm = 0xFFF
-; BTVER2-NEXT:    # sched: [1:1.00]
-; BTVER2-NEXT:    pushw $7 # sched: [1:1.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retl # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_pop_push_16:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    movzwl {{[0-9]+}}(%esp), %eax # sched: [8:0.50]
-; ZNVER1-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [8:0.50]
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    popw %ax # sched: [8:0.50]
-; ZNVER1-NEXT:    popw (%ecx) # sched: [5:0.50]
-; ZNVER1-NEXT:    pushw %ax # sched: [1:0.50]
-; ZNVER1-NEXT:    pushw (%ecx) # sched: [4:0.50]
-; ZNVER1-NEXT:    pushw $4095 # imm = 0xFFF
-; ZNVER1-NEXT:    # sched: [1:0.50]
-; ZNVER1-NEXT:    pushw $7 # sched: [1:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retl # sched: [1:0.50]
-  %1 = call i16 asm sideeffect "popw $0 \0A\09 popw $2 \0A\09 pushw $1 \0A\09 pushw $2 \0A\09 pushw $3 \0A\09 pushw $4", "=r,r,*m,i,i"(i16 %a0, i16 *%a1, i16 4095, i8 7)
-  ret i16 %1
-}
-define i32 @test_pop_push_32(i32 %a0, i32 *%a1) optsize {
-; GENERIC-LABEL: test_pop_push_32:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; GENERIC-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    popl %eax
-; GENERIC-NEXT:    popl (%ecx)
-; GENERIC-NEXT:    pushl %eax
-; GENERIC-NEXT:    pushl (%ecx)
-; GENERIC-NEXT:    pushl $4095 # imm = 0xFFF
-; GENERIC-NEXT:    pushl $7
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retl
-;
-; ATOM-LABEL: test_pop_push_32:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [1:1.00]
-; ATOM-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [1:1.00]
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    popl %eax # sched: [1:1.00]
-; ATOM-NEXT:    popl (%ecx) # sched: [3:1.50]
-; ATOM-NEXT:    pushl %eax # sched: [1:1.00]
-; ATOM-NEXT:    pushl (%ecx) # sched: [2:1.00]
-; ATOM-NEXT:    pushl $4095 # imm = 0xFFF
-; ATOM-NEXT:    # sched: [1:1.00]
-; ATOM-NEXT:    pushl $7 # sched: [1:1.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retl # sched: [79:39.50]
-;
-; SLM-LABEL: test_pop_push_32:
-; SLM:       # %bb.0:
-; SLM-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [3:1.00]
-; SLM-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [3:1.00]
-; SLM-NEXT:    #APP
-; SLM-NEXT:    popl %eax # sched: [3:1.00]
-; SLM-NEXT:    popl (%ecx) # sched: [4:2.00]
-; SLM-NEXT:    pushl %eax # sched: [1:1.00]
-; SLM-NEXT:    pushl (%ecx) # sched: [4:2.00]
-; SLM-NEXT:    pushl $4095 # imm = 0xFFF
-; SLM-NEXT:    # sched: [1:1.00]
-; SLM-NEXT:    pushl $7 # sched: [1:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retl # sched: [4:1.00]
-;
-; SANDY-LABEL: test_pop_push_32:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; SANDY-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    popl %eax # sched: [6:0.50]
-; SANDY-NEXT:    popl (%ecx) # sched: [6:0.50]
-; SANDY-NEXT:    pushl %eax # sched: [5:1.00]
-; SANDY-NEXT:    pushl (%ecx) # sched: [5:1.00]
-; SANDY-NEXT:    pushl $4095 # imm = 0xFFF
-; SANDY-NEXT:    # sched: [1:1.00]
-; SANDY-NEXT:    pushl $7 # sched: [1:1.00]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retl # sched: [6:1.00]
-;
-; HASWELL-LABEL: test_pop_push_32:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; HASWELL-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    popl %eax # sched: [6:0.50]
-; HASWELL-NEXT:    popl (%ecx) # sched: [7:1.00]
-; HASWELL-NEXT:    pushl %eax # sched: [2:1.00]
-; HASWELL-NEXT:    pushl (%ecx) # sched: [7:1.00]
-; HASWELL-NEXT:    pushl $4095 # imm = 0xFFF
-; HASWELL-NEXT:    # sched: [1:1.00]
-; HASWELL-NEXT:    pushl $7 # sched: [1:1.00]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retl # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pop_push_32:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; BROADWELL-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    popl %eax # sched: [6:0.50]
-; BROADWELL-NEXT:    popl (%ecx) # sched: [6:1.00]
-; BROADWELL-NEXT:    pushl %eax # sched: [2:1.00]
-; BROADWELL-NEXT:    pushl (%ecx) # sched: [6:1.00]
-; BROADWELL-NEXT:    pushl $4095 # imm = 0xFFF
-; BROADWELL-NEXT:    # sched: [1:1.00]
-; BROADWELL-NEXT:    pushl $7 # sched: [1:1.00]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retl # sched: [6:0.50]
-;
-; SKYLAKE-LABEL: test_pop_push_32:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; SKYLAKE-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    popl %eax # sched: [6:0.50]
-; SKYLAKE-NEXT:    popl (%ecx) # sched: [6:1.00]
-; SKYLAKE-NEXT:    pushl %eax # sched: [2:1.00]
-; SKYLAKE-NEXT:    pushl (%ecx) # sched: [6:1.00]
-; SKYLAKE-NEXT:    pushl $4095 # imm = 0xFFF
-; SKYLAKE-NEXT:    # sched: [1:1.00]
-; SKYLAKE-NEXT:    pushl $7 # sched: [1:1.00]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retl # sched: [6:0.50]
-;
-; SKX-LABEL: test_pop_push_32:
-; SKX:       # %bb.0:
-; SKX-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; SKX-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; SKX-NEXT:    #APP
-; SKX-NEXT:    popl %eax # sched: [6:0.50]
-; SKX-NEXT:    popl (%ecx) # sched: [6:1.00]
-; SKX-NEXT:    pushl %eax # sched: [2:1.00]
-; SKX-NEXT:    pushl (%ecx) # sched: [6:1.00]
-; SKX-NEXT:    pushl $4095 # imm = 0xFFF
-; SKX-NEXT:    # sched: [1:1.00]
-; SKX-NEXT:    pushl $7 # sched: [1:1.00]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retl # sched: [6:0.50]
-;
-; BDVER2-LABEL: test_pop_push_32:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; BDVER2-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    popl %eax # sched: [5:0.50]
-; BDVER2-NEXT:    popl (%ecx) # sched: [6:1.00]
-; BDVER2-NEXT:    pushl %eax # sched: [1:1.00]
-; BDVER2-NEXT:    pushl (%ecx) # sched: [6:1.00]
-; BDVER2-NEXT:    pushl $4095 # imm = 0xFFF
-; BDVER2-NEXT:    # sched: [1:1.00]
-; BDVER2-NEXT:    pushl $7 # sched: [1:1.00]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retl # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_pop_push_32:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:1.00]
-; BTVER2-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:1.00]
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    popl %eax # sched: [5:1.00]
-; BTVER2-NEXT:    popl (%ecx) # sched: [6:1.00]
-; BTVER2-NEXT:    pushl %eax # sched: [1:1.00]
-; BTVER2-NEXT:    pushl (%ecx) # sched: [6:1.00]
-; BTVER2-NEXT:    pushl $4095 # imm = 0xFFF
-; BTVER2-NEXT:    # sched: [1:1.00]
-; BTVER2-NEXT:    pushl $7 # sched: [1:1.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retl # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_pop_push_32:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [8:0.50]
-; ZNVER1-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [8:0.50]
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    popl %eax # sched: [8:0.50]
-; ZNVER1-NEXT:    popl (%ecx) # sched: [9:1.00]
-; ZNVER1-NEXT:    pushl %eax # sched: [1:0.50]
-; ZNVER1-NEXT:    pushl (%ecx) # sched: [4:0.50]
-; ZNVER1-NEXT:    pushl $4095 # imm = 0xFFF
-; ZNVER1-NEXT:    # sched: [1:0.50]
-; ZNVER1-NEXT:    pushl $7 # sched: [1:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retl # sched: [1:0.50]
-  %1 = call i32 asm sideeffect "popl $0 \0A\09 popl $2 \0A\09 pushl $1 \0A\09 pushl $2 \0A\09 pushl $3 \0A\09 pushl $4", "=r,r,*m,i,i"(i32 %a0, i32 *%a1, i32 4095, i8 7)
-  ret i32 %1
-}
-
-define void @test_popa_popf_pusha_pushf() optsize {
-; GENERIC-LABEL: test_popa_popf_pusha_pushf:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    popal
-; GENERIC-NEXT:    popfl
-; GENERIC-NEXT:    pushal
-; GENERIC-NEXT:    pushfl
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retl
-;
-; ATOM-LABEL: test_popa_popf_pusha_pushf:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    popal # sched: [9:4.50]
-; ATOM-NEXT:    popfl # sched: [26:13.00]
-; ATOM-NEXT:    pushal # sched: [8:4.00]
-; ATOM-NEXT:    pushfl # sched: [9:4.50]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retl # sched: [79:39.50]
-;
-; SLM-LABEL: test_popa_popf_pusha_pushf:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    popal # sched: [3:1.00]
-; SLM-NEXT:    popfl # sched: [3:1.00]
-; SLM-NEXT:    pushal # sched: [1:1.00]
-; SLM-NEXT:    pushfl # sched: [1:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retl # sched: [4:1.00]
-;
-; SANDY-LABEL: test_popa_popf_pusha_pushf:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    popal # sched: [5:0.50]
-; SANDY-NEXT:    popfl # sched: [5:0.50]
-; SANDY-NEXT:    pushal # sched: [1:1.00]
-; SANDY-NEXT:    pushfl # sched: [1:1.00]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retl # sched: [6:1.00]
-;
-; HASWELL-LABEL: test_popa_popf_pusha_pushf:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    popal # sched: [1:4.50]
-; HASWELL-NEXT:    popfl # sched: [5:0.50]
-; HASWELL-NEXT:    pushal # sched: [1:4.75]
-; HASWELL-NEXT:    pushfl # sched: [1:1.00]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retl # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_popa_popf_pusha_pushf:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    popal # sched: [5:0.50]
-; BROADWELL-NEXT:    popfl # sched: [5:0.50]
-; BROADWELL-NEXT:    pushal # sched: [1:1.00]
-; BROADWELL-NEXT:    pushfl # sched: [1:1.00]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retl # sched: [6:0.50]
-;
-; SKYLAKE-LABEL: test_popa_popf_pusha_pushf:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    popal # sched: [5:0.50]
-; SKYLAKE-NEXT:    popfl # sched: [5:0.50]
-; SKYLAKE-NEXT:    pushal # sched: [1:1.00]
-; SKYLAKE-NEXT:    pushfl # sched: [1:1.00]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retl # sched: [6:0.50]
-;
-; SKX-LABEL: test_popa_popf_pusha_pushf:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    popal # sched: [5:0.50]
-; SKX-NEXT:    popfl # sched: [5:0.50]
-; SKX-NEXT:    pushal # sched: [1:1.00]
-; SKX-NEXT:    pushfl # sched: [1:1.00]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retl # sched: [6:0.50]
-;
-; BDVER2-LABEL: test_popa_popf_pusha_pushf:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    popal # sched: [5:0.50]
-; BDVER2-NEXT:    popfl # sched: [5:0.50]
-; BDVER2-NEXT:    pushal # sched: [1:1.00]
-; BDVER2-NEXT:    pushfl # sched: [1:1.00]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retl # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_popa_popf_pusha_pushf:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    popal # sched: [5:1.00]
-; BTVER2-NEXT:    popfl # sched: [5:1.00]
-; BTVER2-NEXT:    pushal # sched: [1:1.00]
-; BTVER2-NEXT:    pushfl # sched: [1:1.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retl # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_popa_popf_pusha_pushf:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    popal # sched: [100:0.25]
-; ZNVER1-NEXT:    popfl # sched: [100:0.25]
-; ZNVER1-NEXT:    pushal # sched: [8:0.50]
-; ZNVER1-NEXT:    pushfl # sched: [100:0.25]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retl # sched: [1:0.50]
-  call void asm sideeffect "popa \0A\09 popf \0A\09 pusha \0A\09 pushf", ""()
-  ret void
-}
-
-define void @test_ret() optsize {
-; GENERIC-LABEL: test_ret:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    retl
-; GENERIC-NEXT:    retl $4095 # imm = 0xFFF
-; GENERIC-NEXT:    lretl
-; GENERIC-NEXT:    lretl $4095 # imm = 0xFFF
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retl
-;
-; ATOM-LABEL: test_ret:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    retl # sched: [79:39.50]
-; ATOM-NEXT:    retl $4095 # imm = 0xFFF
-; ATOM-NEXT:    # sched: [1:1.00]
-; ATOM-NEXT:    lretl # sched: [79:39.50]
-; ATOM-NEXT:    lretl $4095 # imm = 0xFFF
-; ATOM-NEXT:    # sched: [79:39.50]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retl # sched: [79:39.50]
-;
-; SLM-LABEL: test_ret:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    retl # sched: [4:1.00]
-; SLM-NEXT:    retl $4095 # imm = 0xFFF
-; SLM-NEXT:    # sched: [4:1.00]
-; SLM-NEXT:    lretl # sched: [4:1.00]
-; SLM-NEXT:    lretl $4095 # imm = 0xFFF
-; SLM-NEXT:    # sched: [4:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retl # sched: [4:1.00]
-;
-; SANDY-LABEL: test_ret:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    retl # sched: [6:1.00]
-; SANDY-NEXT:    retl $4095 # imm = 0xFFF
-; SANDY-NEXT:    # sched: [6:1.00]
-; SANDY-NEXT:    lretl # sched: [6:1.00]
-; SANDY-NEXT:    lretl $4095 # imm = 0xFFF
-; SANDY-NEXT:    # sched: [6:1.00]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retl # sched: [6:1.00]
-;
-; HASWELL-LABEL: test_ret:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    retl # sched: [7:1.00]
-; HASWELL-NEXT:    retl $4095 # imm = 0xFFF
-; HASWELL-NEXT:    # sched: [1:2.00]
-; HASWELL-NEXT:    lretl # sched: [6:0.50]
-; HASWELL-NEXT:    lretl $4095 # imm = 0xFFF
-; HASWELL-NEXT:    # sched: [1:2.00]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retl # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_ret:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    retl # sched: [6:0.50]
-; BROADWELL-NEXT:    retl $4095 # imm = 0xFFF
-; BROADWELL-NEXT:    # sched: [6:0.50]
-; BROADWELL-NEXT:    lretl # sched: [6:0.50]
-; BROADWELL-NEXT:    lretl $4095 # imm = 0xFFF
-; BROADWELL-NEXT:    # sched: [6:0.50]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retl # sched: [6:0.50]
-;
-; SKYLAKE-LABEL: test_ret:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    retl # sched: [6:0.50]
-; SKYLAKE-NEXT:    retl $4095 # imm = 0xFFF
-; SKYLAKE-NEXT:    # sched: [6:0.50]
-; SKYLAKE-NEXT:    lretl # sched: [6:0.50]
-; SKYLAKE-NEXT:    lretl $4095 # imm = 0xFFF
-; SKYLAKE-NEXT:    # sched: [6:0.50]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retl # sched: [6:0.50]
-;
-; SKX-LABEL: test_ret:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    retl # sched: [6:0.50]
-; SKX-NEXT:    retl $4095 # imm = 0xFFF
-; SKX-NEXT:    # sched: [6:0.50]
-; SKX-NEXT:    lretl # sched: [6:0.50]
-; SKX-NEXT:    lretl $4095 # imm = 0xFFF
-; SKX-NEXT:    # sched: [6:0.50]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retl # sched: [6:0.50]
-;
-; BDVER2-LABEL: test_ret:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    retl # sched: [5:1.00]
-; BDVER2-NEXT:    retl $4095 # imm = 0xFFF
-; BDVER2-NEXT:    # sched: [5:1.00]
-; BDVER2-NEXT:    lretl # sched: [5:1.00]
-; BDVER2-NEXT:    lretl $4095 # imm = 0xFFF
-; BDVER2-NEXT:    # sched: [5:1.00]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retl # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_ret:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    retl # sched: [4:1.00]
-; BTVER2-NEXT:    retl $4095 # imm = 0xFFF
-; BTVER2-NEXT:    # sched: [4:1.00]
-; BTVER2-NEXT:    lretl # sched: [4:1.00]
-; BTVER2-NEXT:    lretl $4095 # imm = 0xFFF
-; BTVER2-NEXT:    # sched: [4:1.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retl # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_ret:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    retl # sched: [1:0.50]
-; ZNVER1-NEXT:    retl $4095 # imm = 0xFFF
-; ZNVER1-NEXT:    # sched: [5:0.50]
-; ZNVER1-NEXT:    lretl # sched: [1:0.50]
-; ZNVER1-NEXT:    lretl $4095 # imm = 0xFFF
-; ZNVER1-NEXT:    # sched: [5:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retl # sched: [1:0.50]
-  call void asm sideeffect "ret \0A\09 ret $0 \0A\09 lret \0A\09 lret $0", "i"(i16 4095)
-  ret void
-}
-
-define i8 @test_salc() optsize {
-; GENERIC-LABEL: test_salc:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    salc
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retl
-;
-; ATOM-LABEL: test_salc:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    salc # sched: [1:0.50]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retl # sched: [79:39.50]
-;
-; SLM-LABEL: test_salc:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    salc # sched: [1:0.50]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retl # sched: [4:1.00]
-;
-; SANDY-LABEL: test_salc:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    salc # sched: [1:0.33]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retl # sched: [6:1.00]
-;
-; HASWELL-LABEL: test_salc:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    salc # sched: [1:0.25]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retl # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_salc:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    salc # sched: [1:0.25]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retl # sched: [6:0.50]
-;
-; SKYLAKE-LABEL: test_salc:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    salc # sched: [1:0.25]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retl # sched: [6:0.50]
-;
-; SKX-LABEL: test_salc:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    salc # sched: [1:0.25]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retl # sched: [6:0.50]
-;
-; BDVER2-LABEL: test_salc:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    salc # sched: [1:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retl # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_salc:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    salc # sched: [1:0.50]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retl # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_salc:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    salc # sched: [1:0.25]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retl # sched: [1:0.50]
-  %1 = tail call i8 asm "salc", "=r"() nounwind
-  ret i8 %1
-}
-
-; TODO - test_sgdt
-; TODO - test_sidt
-
-define void @test_xchg_32(i32 %a0, i32 %a1, i32 *%a2) optsize {
-; GENERIC-LABEL: test_xchg_32:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; GENERIC-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; GENERIC-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    xchgl %eax, %eax
-; GENERIC-NEXT:    xchgl %ecx, %eax
-; GENERIC-NEXT:    xchgl %eax, (%edx)
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retl
-;
-; ATOM-LABEL: test_xchg_32:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [1:1.00]
-; ATOM-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [1:1.00]
-; ATOM-NEXT:    movl {{[0-9]+}}(%esp), %edx # sched: [1:1.00]
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    xchgl %eax, %eax # sched: [2:1.00]
-; ATOM-NEXT:    xchgl %ecx, %eax # sched: [2:1.00]
-; ATOM-NEXT:    xchgl %eax, (%edx) # sched: [3:1.50]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retl # sched: [79:39.50]
-;
-; SLM-LABEL: test_xchg_32:
-; SLM:       # %bb.0:
-; SLM-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [3:1.00]
-; SLM-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [3:1.00]
-; SLM-NEXT:    movl {{[0-9]+}}(%esp), %edx # sched: [3:1.00]
-; SLM-NEXT:    #APP
-; SLM-NEXT:    xchgl %eax, %eax # sched: [1:0.50]
-; SLM-NEXT:    xchgl %ecx, %eax # sched: [1:0.50]
-; SLM-NEXT:    xchgl %eax, (%edx) # sched: [4:2.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retl # sched: [4:1.00]
-;
-; SANDY-LABEL: test_xchg_32:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; SANDY-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; SANDY-NEXT:    movl {{[0-9]+}}(%esp), %edx # sched: [5:0.50]
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    xchgl %eax, %eax # sched: [2:1.00]
-; SANDY-NEXT:    xchgl %ecx, %eax # sched: [2:1.00]
-; SANDY-NEXT:    xchgl %eax, (%edx) # sched: [6:1.00]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retl # sched: [6:1.00]
-;
-; HASWELL-LABEL: test_xchg_32:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; HASWELL-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; HASWELL-NEXT:    movl {{[0-9]+}}(%esp), %edx # sched: [5:0.50]
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    xchgl %eax, %eax # sched: [2:0.75]
-; HASWELL-NEXT:    xchgl %ecx, %eax # sched: [2:0.75]
-; HASWELL-NEXT:    xchgl %eax, (%edx) # sched: [9:1.00]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retl # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_xchg_32:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; BROADWELL-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; BROADWELL-NEXT:    movl {{[0-9]+}}(%esp), %edx # sched: [5:0.50]
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    xchgl %eax, %eax # sched: [2:0.75]
-; BROADWELL-NEXT:    xchgl %ecx, %eax # sched: [2:0.75]
-; BROADWELL-NEXT:    xchgl %eax, (%edx) # sched: [8:1.00]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retl # sched: [6:0.50]
-;
-; SKYLAKE-LABEL: test_xchg_32:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; SKYLAKE-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; SKYLAKE-NEXT:    movl {{[0-9]+}}(%esp), %edx # sched: [5:0.50]
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    xchgl %eax, %eax # sched: [2:0.75]
-; SKYLAKE-NEXT:    xchgl %ecx, %eax # sched: [2:0.75]
-; SKYLAKE-NEXT:    xchgl %eax, (%edx) # sched: [10:1.25]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retl # sched: [6:0.50]
-;
-; SKX-LABEL: test_xchg_32:
-; SKX:       # %bb.0:
-; SKX-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; SKX-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; SKX-NEXT:    movl {{[0-9]+}}(%esp), %edx # sched: [5:0.50]
-; SKX-NEXT:    #APP
-; SKX-NEXT:    xchgl %eax, %eax # sched: [2:0.75]
-; SKX-NEXT:    xchgl %ecx, %eax # sched: [2:0.75]
-; SKX-NEXT:    xchgl %eax, (%edx) # sched: [10:1.25]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retl # sched: [6:0.50]
-;
-; BDVER2-LABEL: test_xchg_32:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; BDVER2-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; BDVER2-NEXT:    movl {{[0-9]+}}(%esp), %edx # sched: [5:0.50]
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    xchgl %eax, %eax # sched: [1:1.00]
-; BDVER2-NEXT:    xchgl %ecx, %eax # sched: [1:1.00]
-; BDVER2-NEXT:    xchgl %eax, (%edx) # sched: [5:1.00]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retl # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_xchg_32:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:1.00]
-; BTVER2-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:1.00]
-; BTVER2-NEXT:    movl {{[0-9]+}}(%esp), %edx # sched: [5:1.00]
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    xchgl %eax, %eax # sched: [1:0.50]
-; BTVER2-NEXT:    xchgl %ecx, %eax # sched: [1:0.50]
-; BTVER2-NEXT:    xchgl %eax, (%edx) # sched: [4:1.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retl # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_xchg_32:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [8:0.50]
-; ZNVER1-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [8:0.50]
-; ZNVER1-NEXT:    movl {{[0-9]+}}(%esp), %edx # sched: [8:0.50]
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    xchgl %eax, %eax # sched: [1:0.50]
-; ZNVER1-NEXT:    xchgl %ecx, %eax # sched: [1:0.50]
-; ZNVER1-NEXT:    xchgl %eax, (%edx) # sched: [5:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retl # sched: [1:0.50]
-  tail call void asm "xchg %EAX, $0 \0A\09 xchg $1, $0 \0A\09 xchg $2, $0", "r,r,*m"(i32 %a0, i32 %a1, i32 *%a2) nounwind
-  ret void
-}
diff --git a/test/CodeGen/X86/schedule-x86_64.ll b/test/CodeGen/X86/schedule-x86_64.ll
deleted file mode 100644
index c2772cc23183c80d9a76570fb4124f80e9bd11e7..0000000000000000000000000000000000000000
--- a/test/CodeGen/X86/schedule-x86_64.ll
+++ /dev/null
@@ -1,18893 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=x86-64 | FileCheck %s --check-prefix=CHECK --check-prefix=GENERIC
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=atom | FileCheck %s --check-prefix=CHECK --check-prefix=ATOM
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=slm | FileCheck %s --check-prefix=CHECK --check-prefix=SLM
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=sandybridge | FileCheck %s --check-prefix=CHECK --check-prefix=SANDY
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=ivybridge | FileCheck %s --check-prefix=CHECK --check-prefix=SANDY
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=haswell | FileCheck %s --check-prefix=CHECK --check-prefix=HASWELL
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=broadwell | FileCheck %s --check-prefix=CHECK --check-prefix=BROADWELL
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=skylake | FileCheck %s --check-prefix=CHECK --check-prefix=SKYLAKE
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=skx | FileCheck %s --check-prefix=CHECK --check-prefix=SKX
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=bdver2 | FileCheck %s --check-prefix=CHECK --check-prefix=BDVER2
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=btver2 | FileCheck %s --check-prefix=CHECK --check-prefix=BTVER2
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=znver1 | FileCheck %s --check-prefix=CHECK --check-prefix=ZNVER1
-
-define void @test_adc_8(i8 %a0, i8* %a1, i8 %a2) optsize {
-; GENERIC-LABEL: test_adc_8:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    adcb $7, %al # sched: [2:0.67]
-; GENERIC-NEXT:    adcb $7, %dil # sched: [2:0.67]
-; GENERIC-NEXT:    adcb $7, (%rsi) # sched: [9:1.00]
-; GENERIC-NEXT:    adcb %dl, %dil # sched: [2:0.67]
-; GENERIC-NEXT:    adcb %dil, (%rsi) # sched: [9:1.00]
-; GENERIC-NEXT:    adcb (%rsi), %dil # sched: [7:0.67]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_adc_8:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    adcb $7, %al # sched: [1:0.50]
-; ATOM-NEXT:    adcb $7, %dil # sched: [1:0.50]
-; ATOM-NEXT:    adcb $7, (%rsi) # sched: [1:1.00]
-; ATOM-NEXT:    adcb %dl, %dil # sched: [1:0.50]
-; ATOM-NEXT:    adcb %dil, (%rsi) # sched: [1:1.00]
-; ATOM-NEXT:    adcb (%rsi), %dil # sched: [1:1.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_adc_8:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    adcb $7, %al # sched: [1:0.50]
-; SLM-NEXT:    adcb $7, %dil # sched: [1:0.50]
-; SLM-NEXT:    adcb $7, (%rsi) # sched: [5:2.00]
-; SLM-NEXT:    adcb %dl, %dil # sched: [1:0.50]
-; SLM-NEXT:    adcb %dil, (%rsi) # sched: [5:2.00]
-; SLM-NEXT:    adcb (%rsi), %dil # sched: [4:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_adc_8:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    adcb $7, %al # sched: [2:0.67]
-; SANDY-NEXT:    adcb $7, %dil # sched: [2:0.67]
-; SANDY-NEXT:    adcb $7, (%rsi) # sched: [9:1.00]
-; SANDY-NEXT:    adcb %dl, %dil # sched: [2:0.67]
-; SANDY-NEXT:    adcb %dil, (%rsi) # sched: [9:1.00]
-; SANDY-NEXT:    adcb (%rsi), %dil # sched: [7:0.67]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_adc_8:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    adcb $7, %al # sched: [2:0.50]
-; HASWELL-NEXT:    adcb $7, %dil # sched: [2:0.50]
-; HASWELL-NEXT:    adcb $7, (%rsi) # sched: [9:1.00]
-; HASWELL-NEXT:    adcb %dl, %dil # sched: [2:0.50]
-; HASWELL-NEXT:    adcb %dil, (%rsi) # sched: [9:1.00]
-; HASWELL-NEXT:    adcb (%rsi), %dil # sched: [7:0.50]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_adc_8:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    adcb $7, %al # sched: [2:0.50]
-; BROADWELL-NEXT:    adcb $7, %dil # sched: [2:0.50]
-; BROADWELL-NEXT:    adcb $7, (%rsi) # sched: [8:1.00]
-; BROADWELL-NEXT:    adcb %dl, %dil # sched: [1:0.50]
-; BROADWELL-NEXT:    adcb %dil, (%rsi) # sched: [8:1.00]
-; BROADWELL-NEXT:    adcb (%rsi), %dil # sched: [6:0.50]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_adc_8:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    adcb $7, %al # sched: [2:0.50]
-; SKYLAKE-NEXT:    adcb $7, %dil # sched: [2:0.50]
-; SKYLAKE-NEXT:    adcb $7, (%rsi) # sched: [8:1.00]
-; SKYLAKE-NEXT:    adcb %dl, %dil # sched: [1:0.50]
-; SKYLAKE-NEXT:    adcb %dil, (%rsi) # sched: [8:1.00]
-; SKYLAKE-NEXT:    adcb (%rsi), %dil # sched: [6:0.50]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_adc_8:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    adcb $7, %al # sched: [2:0.50]
-; SKX-NEXT:    adcb $7, %dil # sched: [2:0.50]
-; SKX-NEXT:    adcb $7, (%rsi) # sched: [8:1.00]
-; SKX-NEXT:    adcb %dl, %dil # sched: [1:0.50]
-; SKX-NEXT:    adcb %dil, (%rsi) # sched: [8:1.00]
-; SKX-NEXT:    adcb (%rsi), %dil # sched: [6:0.50]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_adc_8:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    adcb $7, %al # sched: [1:1.00]
-; BDVER2-NEXT:    adcb $7, %dil # sched: [1:1.00]
-; BDVER2-NEXT:    adcb $7, (%rsi) # sched: [6:1.00]
-; BDVER2-NEXT:    adcb %dl, %dil # sched: [1:1.00]
-; BDVER2-NEXT:    adcb %dil, (%rsi) # sched: [6:1.00]
-; BDVER2-NEXT:    adcb (%rsi), %dil # sched: [5:1.00]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_adc_8:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    adcb $7, %al # sched: [1:1.00]
-; BTVER2-NEXT:    adcb $7, %dil # sched: [1:1.00]
-; BTVER2-NEXT:    adcb $7, (%rsi) # sched: [5:1.00]
-; BTVER2-NEXT:    adcb %dl, %dil # sched: [1:1.00]
-; BTVER2-NEXT:    adcb %dil, (%rsi) # sched: [5:1.00]
-; BTVER2-NEXT:    adcb (%rsi), %dil # sched: [4:1.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_adc_8:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    adcb $7, %al # sched: [1:0.25]
-; ZNVER1-NEXT:    adcb $7, %dil # sched: [1:0.25]
-; ZNVER1-NEXT:    adcb $7, (%rsi) # sched: [5:0.50]
-; ZNVER1-NEXT:    adcb %dl, %dil # sched: [1:0.25]
-; ZNVER1-NEXT:    adcb %dil, (%rsi) # sched: [5:0.50]
-; ZNVER1-NEXT:    adcb (%rsi), %dil # sched: [5:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  tail call void asm "adcb $3, %AL \0A\09 adcb $3, $0 \0A\09 adcb $3, $2 \0A\09 adcb $1, $0 \0A\09 adcb $0, $2 \0A\09 adcb $2, $0", "r,r,*m,i"(i8 %a0, i8 %a2, i8* %a1, i8 7) nounwind
-  ret void
-}
-define void @test_adc_16(i16 %a0, i16* %a1, i16 %a2) optsize {
-; GENERIC-LABEL: test_adc_16:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    adcw $511, %ax # imm = 0x1FF
-; GENERIC-NEXT:    # sched: [2:0.67]
-; GENERIC-NEXT:    adcw $511, %di # imm = 0x1FF
-; GENERIC-NEXT:    # sched: [2:0.67]
-; GENERIC-NEXT:    adcw $511, (%rsi) # imm = 0x1FF
-; GENERIC-NEXT:    # sched: [9:1.00]
-; GENERIC-NEXT:    adcw $7, %di # sched: [2:0.67]
-; GENERIC-NEXT:    adcw $7, (%rsi) # sched: [9:1.00]
-; GENERIC-NEXT:    adcw %dx, %di # sched: [2:0.67]
-; GENERIC-NEXT:    adcw %di, (%rsi) # sched: [9:1.00]
-; GENERIC-NEXT:    adcw (%rsi), %di # sched: [7:0.67]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_adc_16:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    adcw $511, %ax # imm = 0x1FF
-; ATOM-NEXT:    # sched: [1:0.50]
-; ATOM-NEXT:    adcw $511, %di # imm = 0x1FF
-; ATOM-NEXT:    # sched: [1:0.50]
-; ATOM-NEXT:    adcw $511, (%rsi) # imm = 0x1FF
-; ATOM-NEXT:    # sched: [1:1.00]
-; ATOM-NEXT:    adcw $7, %di # sched: [1:0.50]
-; ATOM-NEXT:    adcw $7, (%rsi) # sched: [1:1.00]
-; ATOM-NEXT:    adcw %dx, %di # sched: [1:0.50]
-; ATOM-NEXT:    adcw %di, (%rsi) # sched: [1:1.00]
-; ATOM-NEXT:    adcw (%rsi), %di # sched: [1:1.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_adc_16:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    adcw $511, %ax # imm = 0x1FF
-; SLM-NEXT:    # sched: [1:0.50]
-; SLM-NEXT:    adcw $511, %di # imm = 0x1FF
-; SLM-NEXT:    # sched: [1:0.50]
-; SLM-NEXT:    adcw $511, (%rsi) # imm = 0x1FF
-; SLM-NEXT:    # sched: [5:2.00]
-; SLM-NEXT:    adcw $7, %di # sched: [1:0.50]
-; SLM-NEXT:    adcw $7, (%rsi) # sched: [5:2.00]
-; SLM-NEXT:    adcw %dx, %di # sched: [1:0.50]
-; SLM-NEXT:    adcw %di, (%rsi) # sched: [5:2.00]
-; SLM-NEXT:    adcw (%rsi), %di # sched: [4:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_adc_16:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    adcw $511, %ax # imm = 0x1FF
-; SANDY-NEXT:    # sched: [2:0.67]
-; SANDY-NEXT:    adcw $511, %di # imm = 0x1FF
-; SANDY-NEXT:    # sched: [2:0.67]
-; SANDY-NEXT:    adcw $511, (%rsi) # imm = 0x1FF
-; SANDY-NEXT:    # sched: [9:1.00]
-; SANDY-NEXT:    adcw $7, %di # sched: [2:0.67]
-; SANDY-NEXT:    adcw $7, (%rsi) # sched: [9:1.00]
-; SANDY-NEXT:    adcw %dx, %di # sched: [2:0.67]
-; SANDY-NEXT:    adcw %di, (%rsi) # sched: [9:1.00]
-; SANDY-NEXT:    adcw (%rsi), %di # sched: [7:0.67]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_adc_16:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    adcw $511, %ax # imm = 0x1FF
-; HASWELL-NEXT:    # sched: [2:0.50]
-; HASWELL-NEXT:    adcw $511, %di # imm = 0x1FF
-; HASWELL-NEXT:    # sched: [2:0.50]
-; HASWELL-NEXT:    adcw $511, (%rsi) # imm = 0x1FF
-; HASWELL-NEXT:    # sched: [9:1.00]
-; HASWELL-NEXT:    adcw $7, %di # sched: [2:0.50]
-; HASWELL-NEXT:    adcw $7, (%rsi) # sched: [9:1.00]
-; HASWELL-NEXT:    adcw %dx, %di # sched: [2:0.50]
-; HASWELL-NEXT:    adcw %di, (%rsi) # sched: [9:1.00]
-; HASWELL-NEXT:    adcw (%rsi), %di # sched: [7:0.50]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_adc_16:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    adcw $511, %ax # imm = 0x1FF
-; BROADWELL-NEXT:    # sched: [1:0.50]
-; BROADWELL-NEXT:    adcw $511, %di # imm = 0x1FF
-; BROADWELL-NEXT:    # sched: [1:0.50]
-; BROADWELL-NEXT:    adcw $511, (%rsi) # imm = 0x1FF
-; BROADWELL-NEXT:    # sched: [8:1.00]
-; BROADWELL-NEXT:    adcw $7, %di # sched: [1:0.50]
-; BROADWELL-NEXT:    adcw $7, (%rsi) # sched: [8:1.00]
-; BROADWELL-NEXT:    adcw %dx, %di # sched: [1:0.50]
-; BROADWELL-NEXT:    adcw %di, (%rsi) # sched: [8:1.00]
-; BROADWELL-NEXT:    adcw (%rsi), %di # sched: [6:0.50]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_adc_16:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    adcw $511, %ax # imm = 0x1FF
-; SKYLAKE-NEXT:    # sched: [1:0.50]
-; SKYLAKE-NEXT:    adcw $511, %di # imm = 0x1FF
-; SKYLAKE-NEXT:    # sched: [1:0.50]
-; SKYLAKE-NEXT:    adcw $511, (%rsi) # imm = 0x1FF
-; SKYLAKE-NEXT:    # sched: [8:1.00]
-; SKYLAKE-NEXT:    adcw $7, %di # sched: [1:0.50]
-; SKYLAKE-NEXT:    adcw $7, (%rsi) # sched: [8:1.00]
-; SKYLAKE-NEXT:    adcw %dx, %di # sched: [1:0.50]
-; SKYLAKE-NEXT:    adcw %di, (%rsi) # sched: [8:1.00]
-; SKYLAKE-NEXT:    adcw (%rsi), %di # sched: [6:0.50]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_adc_16:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    adcw $511, %ax # imm = 0x1FF
-; SKX-NEXT:    # sched: [1:0.50]
-; SKX-NEXT:    adcw $511, %di # imm = 0x1FF
-; SKX-NEXT:    # sched: [1:0.50]
-; SKX-NEXT:    adcw $511, (%rsi) # imm = 0x1FF
-; SKX-NEXT:    # sched: [8:1.00]
-; SKX-NEXT:    adcw $7, %di # sched: [1:0.50]
-; SKX-NEXT:    adcw $7, (%rsi) # sched: [8:1.00]
-; SKX-NEXT:    adcw %dx, %di # sched: [1:0.50]
-; SKX-NEXT:    adcw %di, (%rsi) # sched: [8:1.00]
-; SKX-NEXT:    adcw (%rsi), %di # sched: [6:0.50]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_adc_16:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    adcw $511, %ax # imm = 0x1FF
-; BDVER2-NEXT:    # sched: [1:1.00]
-; BDVER2-NEXT:    adcw $511, %di # imm = 0x1FF
-; BDVER2-NEXT:    # sched: [1:1.00]
-; BDVER2-NEXT:    adcw $511, (%rsi) # imm = 0x1FF
-; BDVER2-NEXT:    # sched: [6:1.00]
-; BDVER2-NEXT:    adcw $7, %di # sched: [1:1.00]
-; BDVER2-NEXT:    adcw $7, (%rsi) # sched: [6:1.00]
-; BDVER2-NEXT:    adcw %dx, %di # sched: [1:1.00]
-; BDVER2-NEXT:    adcw %di, (%rsi) # sched: [6:1.00]
-; BDVER2-NEXT:    adcw (%rsi), %di # sched: [5:1.00]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_adc_16:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    adcw $511, %ax # imm = 0x1FF
-; BTVER2-NEXT:    # sched: [1:1.00]
-; BTVER2-NEXT:    adcw $511, %di # imm = 0x1FF
-; BTVER2-NEXT:    # sched: [1:1.00]
-; BTVER2-NEXT:    adcw $511, (%rsi) # imm = 0x1FF
-; BTVER2-NEXT:    # sched: [5:1.00]
-; BTVER2-NEXT:    adcw $7, %di # sched: [1:1.00]
-; BTVER2-NEXT:    adcw $7, (%rsi) # sched: [5:1.00]
-; BTVER2-NEXT:    adcw %dx, %di # sched: [1:1.00]
-; BTVER2-NEXT:    adcw %di, (%rsi) # sched: [5:1.00]
-; BTVER2-NEXT:    adcw (%rsi), %di # sched: [4:1.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_adc_16:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    adcw $511, %ax # imm = 0x1FF
-; ZNVER1-NEXT:    # sched: [1:0.25]
-; ZNVER1-NEXT:    adcw $511, %di # imm = 0x1FF
-; ZNVER1-NEXT:    # sched: [1:0.25]
-; ZNVER1-NEXT:    adcw $511, (%rsi) # imm = 0x1FF
-; ZNVER1-NEXT:    # sched: [5:0.50]
-; ZNVER1-NEXT:    adcw $7, %di # sched: [1:0.25]
-; ZNVER1-NEXT:    adcw $7, (%rsi) # sched: [5:0.50]
-; ZNVER1-NEXT:    adcw %dx, %di # sched: [1:0.25]
-; ZNVER1-NEXT:    adcw %di, (%rsi) # sched: [5:0.50]
-; ZNVER1-NEXT:    adcw (%rsi), %di # sched: [5:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  tail call void asm "adcw $3, %AX \0A\09 adcw $3, $0 \0A\09 adcw $3, $2 \0A\09 adcw $4, $0 \0A\09 adcw $4, $2 \0A\09 adcw $1, $0 \0A\09 adcw $0, $2 \0A\09 adcw $2, $0", "r,r,*m,i,i"(i16 %a0, i16 %a2, i16* %a1, i16 511, i8 7) nounwind
-  ret void
-}
-define void @test_adc_32(i32 %a0, i32* %a1, i32 %a2) optsize {
-; GENERIC-LABEL: test_adc_32:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    adcl $665536, %eax # imm = 0xA27C0
-; GENERIC-NEXT:    # sched: [2:0.67]
-; GENERIC-NEXT:    adcl $665536, %edi # imm = 0xA27C0
-; GENERIC-NEXT:    # sched: [2:0.67]
-; GENERIC-NEXT:    adcl $665536, (%rsi) # imm = 0xA27C0
-; GENERIC-NEXT:    # sched: [9:1.00]
-; GENERIC-NEXT:    adcl $7, %edi # sched: [2:0.67]
-; GENERIC-NEXT:    adcl $7, (%rsi) # sched: [9:1.00]
-; GENERIC-NEXT:    adcl %edx, %edi # sched: [2:0.67]
-; GENERIC-NEXT:    adcl %edi, (%rsi) # sched: [9:1.00]
-; GENERIC-NEXT:    adcl (%rsi), %edi # sched: [7:0.67]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_adc_32:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    adcl $665536, %eax # imm = 0xA27C0
-; ATOM-NEXT:    # sched: [1:0.50]
-; ATOM-NEXT:    adcl $665536, %edi # imm = 0xA27C0
-; ATOM-NEXT:    # sched: [1:0.50]
-; ATOM-NEXT:    adcl $665536, (%rsi) # imm = 0xA27C0
-; ATOM-NEXT:    # sched: [1:1.00]
-; ATOM-NEXT:    adcl $7, %edi # sched: [1:0.50]
-; ATOM-NEXT:    adcl $7, (%rsi) # sched: [1:1.00]
-; ATOM-NEXT:    adcl %edx, %edi # sched: [1:0.50]
-; ATOM-NEXT:    adcl %edi, (%rsi) # sched: [1:1.00]
-; ATOM-NEXT:    adcl (%rsi), %edi # sched: [1:1.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_adc_32:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    adcl $665536, %eax # imm = 0xA27C0
-; SLM-NEXT:    # sched: [1:0.50]
-; SLM-NEXT:    adcl $665536, %edi # imm = 0xA27C0
-; SLM-NEXT:    # sched: [1:0.50]
-; SLM-NEXT:    adcl $665536, (%rsi) # imm = 0xA27C0
-; SLM-NEXT:    # sched: [5:2.00]
-; SLM-NEXT:    adcl $7, %edi # sched: [1:0.50]
-; SLM-NEXT:    adcl $7, (%rsi) # sched: [5:2.00]
-; SLM-NEXT:    adcl %edx, %edi # sched: [1:0.50]
-; SLM-NEXT:    adcl %edi, (%rsi) # sched: [5:2.00]
-; SLM-NEXT:    adcl (%rsi), %edi # sched: [4:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_adc_32:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    adcl $665536, %eax # imm = 0xA27C0
-; SANDY-NEXT:    # sched: [2:0.67]
-; SANDY-NEXT:    adcl $665536, %edi # imm = 0xA27C0
-; SANDY-NEXT:    # sched: [2:0.67]
-; SANDY-NEXT:    adcl $665536, (%rsi) # imm = 0xA27C0
-; SANDY-NEXT:    # sched: [9:1.00]
-; SANDY-NEXT:    adcl $7, %edi # sched: [2:0.67]
-; SANDY-NEXT:    adcl $7, (%rsi) # sched: [9:1.00]
-; SANDY-NEXT:    adcl %edx, %edi # sched: [2:0.67]
-; SANDY-NEXT:    adcl %edi, (%rsi) # sched: [9:1.00]
-; SANDY-NEXT:    adcl (%rsi), %edi # sched: [7:0.67]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_adc_32:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    adcl $665536, %eax # imm = 0xA27C0
-; HASWELL-NEXT:    # sched: [2:0.50]
-; HASWELL-NEXT:    adcl $665536, %edi # imm = 0xA27C0
-; HASWELL-NEXT:    # sched: [2:0.50]
-; HASWELL-NEXT:    adcl $665536, (%rsi) # imm = 0xA27C0
-; HASWELL-NEXT:    # sched: [9:1.00]
-; HASWELL-NEXT:    adcl $7, %edi # sched: [2:0.50]
-; HASWELL-NEXT:    adcl $7, (%rsi) # sched: [9:1.00]
-; HASWELL-NEXT:    adcl %edx, %edi # sched: [2:0.50]
-; HASWELL-NEXT:    adcl %edi, (%rsi) # sched: [9:1.00]
-; HASWELL-NEXT:    adcl (%rsi), %edi # sched: [7:0.50]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_adc_32:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    adcl $665536, %eax # imm = 0xA27C0
-; BROADWELL-NEXT:    # sched: [1:0.50]
-; BROADWELL-NEXT:    adcl $665536, %edi # imm = 0xA27C0
-; BROADWELL-NEXT:    # sched: [1:0.50]
-; BROADWELL-NEXT:    adcl $665536, (%rsi) # imm = 0xA27C0
-; BROADWELL-NEXT:    # sched: [8:1.00]
-; BROADWELL-NEXT:    adcl $7, %edi # sched: [1:0.50]
-; BROADWELL-NEXT:    adcl $7, (%rsi) # sched: [8:1.00]
-; BROADWELL-NEXT:    adcl %edx, %edi # sched: [1:0.50]
-; BROADWELL-NEXT:    adcl %edi, (%rsi) # sched: [8:1.00]
-; BROADWELL-NEXT:    adcl (%rsi), %edi # sched: [6:0.50]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_adc_32:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    adcl $665536, %eax # imm = 0xA27C0
-; SKYLAKE-NEXT:    # sched: [1:0.50]
-; SKYLAKE-NEXT:    adcl $665536, %edi # imm = 0xA27C0
-; SKYLAKE-NEXT:    # sched: [1:0.50]
-; SKYLAKE-NEXT:    adcl $665536, (%rsi) # imm = 0xA27C0
-; SKYLAKE-NEXT:    # sched: [8:1.00]
-; SKYLAKE-NEXT:    adcl $7, %edi # sched: [1:0.50]
-; SKYLAKE-NEXT:    adcl $7, (%rsi) # sched: [8:1.00]
-; SKYLAKE-NEXT:    adcl %edx, %edi # sched: [1:0.50]
-; SKYLAKE-NEXT:    adcl %edi, (%rsi) # sched: [8:1.00]
-; SKYLAKE-NEXT:    adcl (%rsi), %edi # sched: [6:0.50]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_adc_32:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    adcl $665536, %eax # imm = 0xA27C0
-; SKX-NEXT:    # sched: [1:0.50]
-; SKX-NEXT:    adcl $665536, %edi # imm = 0xA27C0
-; SKX-NEXT:    # sched: [1:0.50]
-; SKX-NEXT:    adcl $665536, (%rsi) # imm = 0xA27C0
-; SKX-NEXT:    # sched: [8:1.00]
-; SKX-NEXT:    adcl $7, %edi # sched: [1:0.50]
-; SKX-NEXT:    adcl $7, (%rsi) # sched: [8:1.00]
-; SKX-NEXT:    adcl %edx, %edi # sched: [1:0.50]
-; SKX-NEXT:    adcl %edi, (%rsi) # sched: [8:1.00]
-; SKX-NEXT:    adcl (%rsi), %edi # sched: [6:0.50]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_adc_32:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    adcl $665536, %eax # imm = 0xA27C0
-; BDVER2-NEXT:    # sched: [1:1.00]
-; BDVER2-NEXT:    adcl $665536, %edi # imm = 0xA27C0
-; BDVER2-NEXT:    # sched: [1:1.00]
-; BDVER2-NEXT:    adcl $665536, (%rsi) # imm = 0xA27C0
-; BDVER2-NEXT:    # sched: [6:1.00]
-; BDVER2-NEXT:    adcl $7, %edi # sched: [1:1.00]
-; BDVER2-NEXT:    adcl $7, (%rsi) # sched: [6:1.00]
-; BDVER2-NEXT:    adcl %edx, %edi # sched: [1:1.00]
-; BDVER2-NEXT:    adcl %edi, (%rsi) # sched: [6:1.00]
-; BDVER2-NEXT:    adcl (%rsi), %edi # sched: [5:1.00]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_adc_32:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    adcl $665536, %eax # imm = 0xA27C0
-; BTVER2-NEXT:    # sched: [1:1.00]
-; BTVER2-NEXT:    adcl $665536, %edi # imm = 0xA27C0
-; BTVER2-NEXT:    # sched: [1:1.00]
-; BTVER2-NEXT:    adcl $665536, (%rsi) # imm = 0xA27C0
-; BTVER2-NEXT:    # sched: [5:1.00]
-; BTVER2-NEXT:    adcl $7, %edi # sched: [1:1.00]
-; BTVER2-NEXT:    adcl $7, (%rsi) # sched: [5:1.00]
-; BTVER2-NEXT:    adcl %edx, %edi # sched: [1:1.00]
-; BTVER2-NEXT:    adcl %edi, (%rsi) # sched: [5:1.00]
-; BTVER2-NEXT:    adcl (%rsi), %edi # sched: [4:1.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_adc_32:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    adcl $665536, %eax # imm = 0xA27C0
-; ZNVER1-NEXT:    # sched: [1:0.25]
-; ZNVER1-NEXT:    adcl $665536, %edi # imm = 0xA27C0
-; ZNVER1-NEXT:    # sched: [1:0.25]
-; ZNVER1-NEXT:    adcl $665536, (%rsi) # imm = 0xA27C0
-; ZNVER1-NEXT:    # sched: [5:0.50]
-; ZNVER1-NEXT:    adcl $7, %edi # sched: [1:0.25]
-; ZNVER1-NEXT:    adcl $7, (%rsi) # sched: [5:0.50]
-; ZNVER1-NEXT:    adcl %edx, %edi # sched: [1:0.25]
-; ZNVER1-NEXT:    adcl %edi, (%rsi) # sched: [5:0.50]
-; ZNVER1-NEXT:    adcl (%rsi), %edi # sched: [5:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  tail call void asm "adcl $3, %EAX \0A\09 adcl $3, $0 \0A\09 adcl $3, $2 \0A\09 adcl $4, $0 \0A\09 adcl $4, $2 \0A\09 adcl $1, $0 \0A\09 adcl $0, $2 \0A\09 adcl $2, $0", "r,r,*m,i,i"(i32 %a0, i32 %a2, i32* %a1, i32 665536, i8 7) nounwind
-  ret void
-}
-define void @test_adc_64(i64 %a0, i64* %a1, i64 %a2) optsize {
-; GENERIC-LABEL: test_adc_64:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    adcq $665536, %rax # imm = 0xA27C0
-; GENERIC-NEXT:    # sched: [2:0.67]
-; GENERIC-NEXT:    adcq $665536, %rdi # imm = 0xA27C0
-; GENERIC-NEXT:    # sched: [2:0.67]
-; GENERIC-NEXT:    adcq $665536, (%rsi) # imm = 0xA27C0
-; GENERIC-NEXT:    # sched: [9:1.00]
-; GENERIC-NEXT:    adcq $7, %rdi # sched: [2:0.67]
-; GENERIC-NEXT:    adcq $7, (%rsi) # sched: [9:1.00]
-; GENERIC-NEXT:    adcq %rdx, %rdi # sched: [2:0.67]
-; GENERIC-NEXT:    adcq %rdi, (%rsi) # sched: [9:1.00]
-; GENERIC-NEXT:    adcq (%rsi), %rdi # sched: [7:0.67]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_adc_64:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    adcq $665536, %rax # imm = 0xA27C0
-; ATOM-NEXT:    # sched: [1:0.50]
-; ATOM-NEXT:    adcq $665536, %rdi # imm = 0xA27C0
-; ATOM-NEXT:    # sched: [1:0.50]
-; ATOM-NEXT:    adcq $665536, (%rsi) # imm = 0xA27C0
-; ATOM-NEXT:    # sched: [1:1.00]
-; ATOM-NEXT:    adcq $7, %rdi # sched: [1:0.50]
-; ATOM-NEXT:    adcq $7, (%rsi) # sched: [1:1.00]
-; ATOM-NEXT:    adcq %rdx, %rdi # sched: [1:0.50]
-; ATOM-NEXT:    adcq %rdi, (%rsi) # sched: [1:1.00]
-; ATOM-NEXT:    adcq (%rsi), %rdi # sched: [1:1.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_adc_64:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    adcq $665536, %rax # imm = 0xA27C0
-; SLM-NEXT:    # sched: [1:0.50]
-; SLM-NEXT:    adcq $665536, %rdi # imm = 0xA27C0
-; SLM-NEXT:    # sched: [1:0.50]
-; SLM-NEXT:    adcq $665536, (%rsi) # imm = 0xA27C0
-; SLM-NEXT:    # sched: [5:2.00]
-; SLM-NEXT:    adcq $7, %rdi # sched: [1:0.50]
-; SLM-NEXT:    adcq $7, (%rsi) # sched: [5:2.00]
-; SLM-NEXT:    adcq %rdx, %rdi # sched: [1:0.50]
-; SLM-NEXT:    adcq %rdi, (%rsi) # sched: [5:2.00]
-; SLM-NEXT:    adcq (%rsi), %rdi # sched: [4:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_adc_64:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    adcq $665536, %rax # imm = 0xA27C0
-; SANDY-NEXT:    # sched: [2:0.67]
-; SANDY-NEXT:    adcq $665536, %rdi # imm = 0xA27C0
-; SANDY-NEXT:    # sched: [2:0.67]
-; SANDY-NEXT:    adcq $665536, (%rsi) # imm = 0xA27C0
-; SANDY-NEXT:    # sched: [9:1.00]
-; SANDY-NEXT:    adcq $7, %rdi # sched: [2:0.67]
-; SANDY-NEXT:    adcq $7, (%rsi) # sched: [9:1.00]
-; SANDY-NEXT:    adcq %rdx, %rdi # sched: [2:0.67]
-; SANDY-NEXT:    adcq %rdi, (%rsi) # sched: [9:1.00]
-; SANDY-NEXT:    adcq (%rsi), %rdi # sched: [7:0.67]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_adc_64:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    adcq $665536, %rax # imm = 0xA27C0
-; HASWELL-NEXT:    # sched: [2:0.50]
-; HASWELL-NEXT:    adcq $665536, %rdi # imm = 0xA27C0
-; HASWELL-NEXT:    # sched: [2:0.50]
-; HASWELL-NEXT:    adcq $665536, (%rsi) # imm = 0xA27C0
-; HASWELL-NEXT:    # sched: [9:1.00]
-; HASWELL-NEXT:    adcq $7, %rdi # sched: [2:0.50]
-; HASWELL-NEXT:    adcq $7, (%rsi) # sched: [9:1.00]
-; HASWELL-NEXT:    adcq %rdx, %rdi # sched: [2:0.50]
-; HASWELL-NEXT:    adcq %rdi, (%rsi) # sched: [9:1.00]
-; HASWELL-NEXT:    adcq (%rsi), %rdi # sched: [7:0.50]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_adc_64:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    adcq $665536, %rax # imm = 0xA27C0
-; BROADWELL-NEXT:    # sched: [1:0.50]
-; BROADWELL-NEXT:    adcq $665536, %rdi # imm = 0xA27C0
-; BROADWELL-NEXT:    # sched: [1:0.50]
-; BROADWELL-NEXT:    adcq $665536, (%rsi) # imm = 0xA27C0
-; BROADWELL-NEXT:    # sched: [8:1.00]
-; BROADWELL-NEXT:    adcq $7, %rdi # sched: [1:0.50]
-; BROADWELL-NEXT:    adcq $7, (%rsi) # sched: [8:1.00]
-; BROADWELL-NEXT:    adcq %rdx, %rdi # sched: [1:0.50]
-; BROADWELL-NEXT:    adcq %rdi, (%rsi) # sched: [8:1.00]
-; BROADWELL-NEXT:    adcq (%rsi), %rdi # sched: [6:0.50]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_adc_64:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    adcq $665536, %rax # imm = 0xA27C0
-; SKYLAKE-NEXT:    # sched: [1:0.50]
-; SKYLAKE-NEXT:    adcq $665536, %rdi # imm = 0xA27C0
-; SKYLAKE-NEXT:    # sched: [1:0.50]
-; SKYLAKE-NEXT:    adcq $665536, (%rsi) # imm = 0xA27C0
-; SKYLAKE-NEXT:    # sched: [8:1.00]
-; SKYLAKE-NEXT:    adcq $7, %rdi # sched: [1:0.50]
-; SKYLAKE-NEXT:    adcq $7, (%rsi) # sched: [8:1.00]
-; SKYLAKE-NEXT:    adcq %rdx, %rdi # sched: [1:0.50]
-; SKYLAKE-NEXT:    adcq %rdi, (%rsi) # sched: [8:1.00]
-; SKYLAKE-NEXT:    adcq (%rsi), %rdi # sched: [6:0.50]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_adc_64:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    adcq $665536, %rax # imm = 0xA27C0
-; SKX-NEXT:    # sched: [1:0.50]
-; SKX-NEXT:    adcq $665536, %rdi # imm = 0xA27C0
-; SKX-NEXT:    # sched: [1:0.50]
-; SKX-NEXT:    adcq $665536, (%rsi) # imm = 0xA27C0
-; SKX-NEXT:    # sched: [8:1.00]
-; SKX-NEXT:    adcq $7, %rdi # sched: [1:0.50]
-; SKX-NEXT:    adcq $7, (%rsi) # sched: [8:1.00]
-; SKX-NEXT:    adcq %rdx, %rdi # sched: [1:0.50]
-; SKX-NEXT:    adcq %rdi, (%rsi) # sched: [8:1.00]
-; SKX-NEXT:    adcq (%rsi), %rdi # sched: [6:0.50]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_adc_64:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    adcq $665536, %rax # imm = 0xA27C0
-; BDVER2-NEXT:    # sched: [1:1.00]
-; BDVER2-NEXT:    adcq $665536, %rdi # imm = 0xA27C0
-; BDVER2-NEXT:    # sched: [1:1.00]
-; BDVER2-NEXT:    adcq $665536, (%rsi) # imm = 0xA27C0
-; BDVER2-NEXT:    # sched: [6:1.00]
-; BDVER2-NEXT:    adcq $7, %rdi # sched: [1:1.00]
-; BDVER2-NEXT:    adcq $7, (%rsi) # sched: [6:1.00]
-; BDVER2-NEXT:    adcq %rdx, %rdi # sched: [1:1.00]
-; BDVER2-NEXT:    adcq %rdi, (%rsi) # sched: [6:1.00]
-; BDVER2-NEXT:    adcq (%rsi), %rdi # sched: [5:1.00]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_adc_64:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    adcq $665536, %rax # imm = 0xA27C0
-; BTVER2-NEXT:    # sched: [1:1.00]
-; BTVER2-NEXT:    adcq $665536, %rdi # imm = 0xA27C0
-; BTVER2-NEXT:    # sched: [1:1.00]
-; BTVER2-NEXT:    adcq $665536, (%rsi) # imm = 0xA27C0
-; BTVER2-NEXT:    # sched: [5:1.00]
-; BTVER2-NEXT:    adcq $7, %rdi # sched: [1:1.00]
-; BTVER2-NEXT:    adcq $7, (%rsi) # sched: [5:1.00]
-; BTVER2-NEXT:    adcq %rdx, %rdi # sched: [1:1.00]
-; BTVER2-NEXT:    adcq %rdi, (%rsi) # sched: [5:1.00]
-; BTVER2-NEXT:    adcq (%rsi), %rdi # sched: [4:1.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_adc_64:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    adcq $665536, %rax # imm = 0xA27C0
-; ZNVER1-NEXT:    # sched: [1:0.25]
-; ZNVER1-NEXT:    adcq $665536, %rdi # imm = 0xA27C0
-; ZNVER1-NEXT:    # sched: [1:0.25]
-; ZNVER1-NEXT:    adcq $665536, (%rsi) # imm = 0xA27C0
-; ZNVER1-NEXT:    # sched: [5:0.50]
-; ZNVER1-NEXT:    adcq $7, %rdi # sched: [1:0.25]
-; ZNVER1-NEXT:    adcq $7, (%rsi) # sched: [5:0.50]
-; ZNVER1-NEXT:    adcq %rdx, %rdi # sched: [1:0.25]
-; ZNVER1-NEXT:    adcq %rdi, (%rsi) # sched: [5:0.50]
-; ZNVER1-NEXT:    adcq (%rsi), %rdi # sched: [5:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  tail call void asm "adcq $3, %RAX \0A\09 adcq $3, $0 \0A\09 adcq $3, $2 \0A\09 adcq $4, $0 \0A\09 adcq $4, $2 \0A\09 adcq $1, $0 \0A\09 adcq $0, $2 \0A\09 adcq $2, $0", "r,r,*m,i,i"(i64 %a0, i64 %a2, i64* %a1, i32 665536, i8 7) nounwind
-  ret void
-}
-
-define void @test_add_8(i8 %a0, i8* %a1, i8 %a2) optsize {
-; GENERIC-LABEL: test_add_8:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    addb $7, %al # sched: [1:0.33]
-; GENERIC-NEXT:    addb $7, %dil # sched: [1:0.33]
-; GENERIC-NEXT:    addb $7, (%rsi) # sched: [7:1.00]
-; GENERIC-NEXT:    addb %dl, %dil # sched: [1:0.33]
-; GENERIC-NEXT:    addb %dil, (%rsi) # sched: [7:1.00]
-; GENERIC-NEXT:    addb (%rsi), %dil # sched: [6:0.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_add_8:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    addb $7, %al # sched: [1:0.50]
-; ATOM-NEXT:    addb $7, %dil # sched: [1:0.50]
-; ATOM-NEXT:    addb $7, (%rsi) # sched: [1:1.00]
-; ATOM-NEXT:    addb %dl, %dil # sched: [1:0.50]
-; ATOM-NEXT:    addb %dil, (%rsi) # sched: [1:1.00]
-; ATOM-NEXT:    addb (%rsi), %dil # sched: [1:1.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_add_8:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    addb $7, %al # sched: [1:0.50]
-; SLM-NEXT:    addb $7, %dil # sched: [1:0.50]
-; SLM-NEXT:    addb $7, (%rsi) # sched: [5:2.00]
-; SLM-NEXT:    addb %dl, %dil # sched: [1:0.50]
-; SLM-NEXT:    addb %dil, (%rsi) # sched: [5:2.00]
-; SLM-NEXT:    addb (%rsi), %dil # sched: [4:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_add_8:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    addb $7, %al # sched: [1:0.33]
-; SANDY-NEXT:    addb $7, %dil # sched: [1:0.33]
-; SANDY-NEXT:    addb $7, (%rsi) # sched: [7:1.00]
-; SANDY-NEXT:    addb %dl, %dil # sched: [1:0.33]
-; SANDY-NEXT:    addb %dil, (%rsi) # sched: [7:1.00]
-; SANDY-NEXT:    addb (%rsi), %dil # sched: [6:0.50]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_add_8:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    addb $7, %al # sched: [1:0.25]
-; HASWELL-NEXT:    addb $7, %dil # sched: [1:0.25]
-; HASWELL-NEXT:    addb $7, (%rsi) # sched: [7:1.00]
-; HASWELL-NEXT:    addb %dl, %dil # sched: [1:0.25]
-; HASWELL-NEXT:    addb %dil, (%rsi) # sched: [7:1.00]
-; HASWELL-NEXT:    addb (%rsi), %dil # sched: [6:0.50]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_add_8:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    addb $7, %al # sched: [1:0.25]
-; BROADWELL-NEXT:    addb $7, %dil # sched: [1:0.25]
-; BROADWELL-NEXT:    addb $7, (%rsi) # sched: [7:1.00]
-; BROADWELL-NEXT:    addb %dl, %dil # sched: [1:0.25]
-; BROADWELL-NEXT:    addb %dil, (%rsi) # sched: [7:1.00]
-; BROADWELL-NEXT:    addb (%rsi), %dil # sched: [6:0.50]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_add_8:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    addb $7, %al # sched: [1:0.25]
-; SKYLAKE-NEXT:    addb $7, %dil # sched: [1:0.25]
-; SKYLAKE-NEXT:    addb $7, (%rsi) # sched: [7:1.00]
-; SKYLAKE-NEXT:    addb %dl, %dil # sched: [1:0.25]
-; SKYLAKE-NEXT:    addb %dil, (%rsi) # sched: [7:1.00]
-; SKYLAKE-NEXT:    addb (%rsi), %dil # sched: [6:0.50]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_add_8:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    addb $7, %al # sched: [1:0.25]
-; SKX-NEXT:    addb $7, %dil # sched: [1:0.25]
-; SKX-NEXT:    addb $7, (%rsi) # sched: [7:1.00]
-; SKX-NEXT:    addb %dl, %dil # sched: [1:0.25]
-; SKX-NEXT:    addb %dil, (%rsi) # sched: [7:1.00]
-; SKX-NEXT:    addb (%rsi), %dil # sched: [6:0.50]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_add_8:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    addb $7, %al # sched: [1:0.50]
-; BDVER2-NEXT:    addb $7, %dil # sched: [1:0.50]
-; BDVER2-NEXT:    addb $7, (%rsi) # sched: [6:1.00]
-; BDVER2-NEXT:    addb %dl, %dil # sched: [1:0.50]
-; BDVER2-NEXT:    addb %dil, (%rsi) # sched: [6:1.00]
-; BDVER2-NEXT:    addb (%rsi), %dil # sched: [5:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_add_8:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    addb $7, %al # sched: [1:0.50]
-; BTVER2-NEXT:    addb $7, %dil # sched: [1:0.50]
-; BTVER2-NEXT:    addb $7, (%rsi) # sched: [5:1.00]
-; BTVER2-NEXT:    addb %dl, %dil # sched: [1:0.50]
-; BTVER2-NEXT:    addb %dil, (%rsi) # sched: [5:1.00]
-; BTVER2-NEXT:    addb (%rsi), %dil # sched: [4:1.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_add_8:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    addb $7, %al # sched: [1:0.25]
-; ZNVER1-NEXT:    addb $7, %dil # sched: [1:0.25]
-; ZNVER1-NEXT:    addb $7, (%rsi) # sched: [5:0.50]
-; ZNVER1-NEXT:    addb %dl, %dil # sched: [1:0.25]
-; ZNVER1-NEXT:    addb %dil, (%rsi) # sched: [5:0.50]
-; ZNVER1-NEXT:    addb (%rsi), %dil # sched: [5:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  tail call void asm "addb $3, %AL \0A\09 addb $3, $0 \0A\09 addb $3, $2 \0A\09 addb $1, $0 \0A\09 addb $0, $2 \0A\09 addb $2, $0", "r,r,*m,i"(i8 %a0, i8 %a2, i8* %a1, i8 7) nounwind
-  ret void
-}
-define void @test_add_16(i16 %a0, i16* %a1, i16 %a2) optsize {
-; GENERIC-LABEL: test_add_16:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    addw $511, %ax # imm = 0x1FF
-; GENERIC-NEXT:    # sched: [1:0.33]
-; GENERIC-NEXT:    addw $511, %di # imm = 0x1FF
-; GENERIC-NEXT:    # sched: [1:0.33]
-; GENERIC-NEXT:    addw $511, (%rsi) # imm = 0x1FF
-; GENERIC-NEXT:    # sched: [7:1.00]
-; GENERIC-NEXT:    addw $7, %di # sched: [1:0.33]
-; GENERIC-NEXT:    addw $7, (%rsi) # sched: [7:1.00]
-; GENERIC-NEXT:    addw %dx, %di # sched: [1:0.33]
-; GENERIC-NEXT:    addw %di, (%rsi) # sched: [7:1.00]
-; GENERIC-NEXT:    addw (%rsi), %di # sched: [6:0.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_add_16:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    addw $511, %ax # imm = 0x1FF
-; ATOM-NEXT:    # sched: [1:0.50]
-; ATOM-NEXT:    addw $511, %di # imm = 0x1FF
-; ATOM-NEXT:    # sched: [1:0.50]
-; ATOM-NEXT:    addw $511, (%rsi) # imm = 0x1FF
-; ATOM-NEXT:    # sched: [1:1.00]
-; ATOM-NEXT:    addw $7, %di # sched: [1:0.50]
-; ATOM-NEXT:    addw $7, (%rsi) # sched: [1:1.00]
-; ATOM-NEXT:    addw %dx, %di # sched: [1:0.50]
-; ATOM-NEXT:    addw %di, (%rsi) # sched: [1:1.00]
-; ATOM-NEXT:    addw (%rsi), %di # sched: [1:1.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_add_16:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    addw $511, %ax # imm = 0x1FF
-; SLM-NEXT:    # sched: [1:0.50]
-; SLM-NEXT:    addw $511, %di # imm = 0x1FF
-; SLM-NEXT:    # sched: [1:0.50]
-; SLM-NEXT:    addw $511, (%rsi) # imm = 0x1FF
-; SLM-NEXT:    # sched: [5:2.00]
-; SLM-NEXT:    addw $7, %di # sched: [1:0.50]
-; SLM-NEXT:    addw $7, (%rsi) # sched: [5:2.00]
-; SLM-NEXT:    addw %dx, %di # sched: [1:0.50]
-; SLM-NEXT:    addw %di, (%rsi) # sched: [5:2.00]
-; SLM-NEXT:    addw (%rsi), %di # sched: [4:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_add_16:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    addw $511, %ax # imm = 0x1FF
-; SANDY-NEXT:    # sched: [1:0.33]
-; SANDY-NEXT:    addw $511, %di # imm = 0x1FF
-; SANDY-NEXT:    # sched: [1:0.33]
-; SANDY-NEXT:    addw $511, (%rsi) # imm = 0x1FF
-; SANDY-NEXT:    # sched: [7:1.00]
-; SANDY-NEXT:    addw $7, %di # sched: [1:0.33]
-; SANDY-NEXT:    addw $7, (%rsi) # sched: [7:1.00]
-; SANDY-NEXT:    addw %dx, %di # sched: [1:0.33]
-; SANDY-NEXT:    addw %di, (%rsi) # sched: [7:1.00]
-; SANDY-NEXT:    addw (%rsi), %di # sched: [6:0.50]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_add_16:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    addw $511, %ax # imm = 0x1FF
-; HASWELL-NEXT:    # sched: [1:0.25]
-; HASWELL-NEXT:    addw $511, %di # imm = 0x1FF
-; HASWELL-NEXT:    # sched: [1:0.25]
-; HASWELL-NEXT:    addw $511, (%rsi) # imm = 0x1FF
-; HASWELL-NEXT:    # sched: [7:1.00]
-; HASWELL-NEXT:    addw $7, %di # sched: [1:0.25]
-; HASWELL-NEXT:    addw $7, (%rsi) # sched: [7:1.00]
-; HASWELL-NEXT:    addw %dx, %di # sched: [1:0.25]
-; HASWELL-NEXT:    addw %di, (%rsi) # sched: [7:1.00]
-; HASWELL-NEXT:    addw (%rsi), %di # sched: [6:0.50]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_add_16:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    addw $511, %ax # imm = 0x1FF
-; BROADWELL-NEXT:    # sched: [1:0.25]
-; BROADWELL-NEXT:    addw $511, %di # imm = 0x1FF
-; BROADWELL-NEXT:    # sched: [1:0.25]
-; BROADWELL-NEXT:    addw $511, (%rsi) # imm = 0x1FF
-; BROADWELL-NEXT:    # sched: [7:1.00]
-; BROADWELL-NEXT:    addw $7, %di # sched: [1:0.25]
-; BROADWELL-NEXT:    addw $7, (%rsi) # sched: [7:1.00]
-; BROADWELL-NEXT:    addw %dx, %di # sched: [1:0.25]
-; BROADWELL-NEXT:    addw %di, (%rsi) # sched: [7:1.00]
-; BROADWELL-NEXT:    addw (%rsi), %di # sched: [6:0.50]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_add_16:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    addw $511, %ax # imm = 0x1FF
-; SKYLAKE-NEXT:    # sched: [1:0.25]
-; SKYLAKE-NEXT:    addw $511, %di # imm = 0x1FF
-; SKYLAKE-NEXT:    # sched: [1:0.25]
-; SKYLAKE-NEXT:    addw $511, (%rsi) # imm = 0x1FF
-; SKYLAKE-NEXT:    # sched: [7:1.00]
-; SKYLAKE-NEXT:    addw $7, %di # sched: [1:0.25]
-; SKYLAKE-NEXT:    addw $7, (%rsi) # sched: [7:1.00]
-; SKYLAKE-NEXT:    addw %dx, %di # sched: [1:0.25]
-; SKYLAKE-NEXT:    addw %di, (%rsi) # sched: [7:1.00]
-; SKYLAKE-NEXT:    addw (%rsi), %di # sched: [6:0.50]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_add_16:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    addw $511, %ax # imm = 0x1FF
-; SKX-NEXT:    # sched: [1:0.25]
-; SKX-NEXT:    addw $511, %di # imm = 0x1FF
-; SKX-NEXT:    # sched: [1:0.25]
-; SKX-NEXT:    addw $511, (%rsi) # imm = 0x1FF
-; SKX-NEXT:    # sched: [7:1.00]
-; SKX-NEXT:    addw $7, %di # sched: [1:0.25]
-; SKX-NEXT:    addw $7, (%rsi) # sched: [7:1.00]
-; SKX-NEXT:    addw %dx, %di # sched: [1:0.25]
-; SKX-NEXT:    addw %di, (%rsi) # sched: [7:1.00]
-; SKX-NEXT:    addw (%rsi), %di # sched: [6:0.50]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_add_16:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    addw $511, %ax # imm = 0x1FF
-; BDVER2-NEXT:    # sched: [1:0.50]
-; BDVER2-NEXT:    addw $511, %di # imm = 0x1FF
-; BDVER2-NEXT:    # sched: [1:0.50]
-; BDVER2-NEXT:    addw $511, (%rsi) # imm = 0x1FF
-; BDVER2-NEXT:    # sched: [6:1.00]
-; BDVER2-NEXT:    addw $7, %di # sched: [1:0.50]
-; BDVER2-NEXT:    addw $7, (%rsi) # sched: [6:1.00]
-; BDVER2-NEXT:    addw %dx, %di # sched: [1:0.50]
-; BDVER2-NEXT:    addw %di, (%rsi) # sched: [6:1.00]
-; BDVER2-NEXT:    addw (%rsi), %di # sched: [5:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_add_16:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    addw $511, %ax # imm = 0x1FF
-; BTVER2-NEXT:    # sched: [1:0.50]
-; BTVER2-NEXT:    addw $511, %di # imm = 0x1FF
-; BTVER2-NEXT:    # sched: [1:0.50]
-; BTVER2-NEXT:    addw $511, (%rsi) # imm = 0x1FF
-; BTVER2-NEXT:    # sched: [5:1.00]
-; BTVER2-NEXT:    addw $7, %di # sched: [1:0.50]
-; BTVER2-NEXT:    addw $7, (%rsi) # sched: [5:1.00]
-; BTVER2-NEXT:    addw %dx, %di # sched: [1:0.50]
-; BTVER2-NEXT:    addw %di, (%rsi) # sched: [5:1.00]
-; BTVER2-NEXT:    addw (%rsi), %di # sched: [4:1.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_add_16:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    addw $511, %ax # imm = 0x1FF
-; ZNVER1-NEXT:    # sched: [1:0.25]
-; ZNVER1-NEXT:    addw $511, %di # imm = 0x1FF
-; ZNVER1-NEXT:    # sched: [1:0.25]
-; ZNVER1-NEXT:    addw $511, (%rsi) # imm = 0x1FF
-; ZNVER1-NEXT:    # sched: [5:0.50]
-; ZNVER1-NEXT:    addw $7, %di # sched: [1:0.25]
-; ZNVER1-NEXT:    addw $7, (%rsi) # sched: [5:0.50]
-; ZNVER1-NEXT:    addw %dx, %di # sched: [1:0.25]
-; ZNVER1-NEXT:    addw %di, (%rsi) # sched: [5:0.50]
-; ZNVER1-NEXT:    addw (%rsi), %di # sched: [5:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  tail call void asm "addw $3, %AX \0A\09 addw $3, $0 \0A\09 addw $3, $2 \0A\09 addw $4, $0 \0A\09 addw $4, $2 \0A\09 addw $1, $0 \0A\09 addw $0, $2 \0A\09 addw $2, $0", "r,r,*m,i,i"(i16 %a0, i16 %a2, i16* %a1, i16 511, i8 7) nounwind
-  ret void
-}
-define void @test_add_32(i32 %a0, i32* %a1, i32 %a2) optsize {
-; GENERIC-LABEL: test_add_32:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    addl $665536, %eax # imm = 0xA27C0
-; GENERIC-NEXT:    # sched: [1:0.33]
-; GENERIC-NEXT:    addl $665536, %edi # imm = 0xA27C0
-; GENERIC-NEXT:    # sched: [1:0.33]
-; GENERIC-NEXT:    addl $665536, (%rsi) # imm = 0xA27C0
-; GENERIC-NEXT:    # sched: [7:1.00]
-; GENERIC-NEXT:    addl $7, %edi # sched: [1:0.33]
-; GENERIC-NEXT:    addl $7, (%rsi) # sched: [7:1.00]
-; GENERIC-NEXT:    addl %edx, %edi # sched: [1:0.33]
-; GENERIC-NEXT:    addl %edi, (%rsi) # sched: [7:1.00]
-; GENERIC-NEXT:    addl (%rsi), %edi # sched: [6:0.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_add_32:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    addl $665536, %eax # imm = 0xA27C0
-; ATOM-NEXT:    # sched: [1:0.50]
-; ATOM-NEXT:    addl $665536, %edi # imm = 0xA27C0
-; ATOM-NEXT:    # sched: [1:0.50]
-; ATOM-NEXT:    addl $665536, (%rsi) # imm = 0xA27C0
-; ATOM-NEXT:    # sched: [1:1.00]
-; ATOM-NEXT:    addl $7, %edi # sched: [1:0.50]
-; ATOM-NEXT:    addl $7, (%rsi) # sched: [1:1.00]
-; ATOM-NEXT:    addl %edx, %edi # sched: [1:0.50]
-; ATOM-NEXT:    addl %edi, (%rsi) # sched: [1:1.00]
-; ATOM-NEXT:    addl (%rsi), %edi # sched: [1:1.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_add_32:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    addl $665536, %eax # imm = 0xA27C0
-; SLM-NEXT:    # sched: [1:0.50]
-; SLM-NEXT:    addl $665536, %edi # imm = 0xA27C0
-; SLM-NEXT:    # sched: [1:0.50]
-; SLM-NEXT:    addl $665536, (%rsi) # imm = 0xA27C0
-; SLM-NEXT:    # sched: [5:2.00]
-; SLM-NEXT:    addl $7, %edi # sched: [1:0.50]
-; SLM-NEXT:    addl $7, (%rsi) # sched: [5:2.00]
-; SLM-NEXT:    addl %edx, %edi # sched: [1:0.50]
-; SLM-NEXT:    addl %edi, (%rsi) # sched: [5:2.00]
-; SLM-NEXT:    addl (%rsi), %edi # sched: [4:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_add_32:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    addl $665536, %eax # imm = 0xA27C0
-; SANDY-NEXT:    # sched: [1:0.33]
-; SANDY-NEXT:    addl $665536, %edi # imm = 0xA27C0
-; SANDY-NEXT:    # sched: [1:0.33]
-; SANDY-NEXT:    addl $665536, (%rsi) # imm = 0xA27C0
-; SANDY-NEXT:    # sched: [7:1.00]
-; SANDY-NEXT:    addl $7, %edi # sched: [1:0.33]
-; SANDY-NEXT:    addl $7, (%rsi) # sched: [7:1.00]
-; SANDY-NEXT:    addl %edx, %edi # sched: [1:0.33]
-; SANDY-NEXT:    addl %edi, (%rsi) # sched: [7:1.00]
-; SANDY-NEXT:    addl (%rsi), %edi # sched: [6:0.50]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_add_32:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    addl $665536, %eax # imm = 0xA27C0
-; HASWELL-NEXT:    # sched: [1:0.25]
-; HASWELL-NEXT:    addl $665536, %edi # imm = 0xA27C0
-; HASWELL-NEXT:    # sched: [1:0.25]
-; HASWELL-NEXT:    addl $665536, (%rsi) # imm = 0xA27C0
-; HASWELL-NEXT:    # sched: [7:1.00]
-; HASWELL-NEXT:    addl $7, %edi # sched: [1:0.25]
-; HASWELL-NEXT:    addl $7, (%rsi) # sched: [7:1.00]
-; HASWELL-NEXT:    addl %edx, %edi # sched: [1:0.25]
-; HASWELL-NEXT:    addl %edi, (%rsi) # sched: [7:1.00]
-; HASWELL-NEXT:    addl (%rsi), %edi # sched: [6:0.50]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_add_32:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    addl $665536, %eax # imm = 0xA27C0
-; BROADWELL-NEXT:    # sched: [1:0.25]
-; BROADWELL-NEXT:    addl $665536, %edi # imm = 0xA27C0
-; BROADWELL-NEXT:    # sched: [1:0.25]
-; BROADWELL-NEXT:    addl $665536, (%rsi) # imm = 0xA27C0
-; BROADWELL-NEXT:    # sched: [7:1.00]
-; BROADWELL-NEXT:    addl $7, %edi # sched: [1:0.25]
-; BROADWELL-NEXT:    addl $7, (%rsi) # sched: [7:1.00]
-; BROADWELL-NEXT:    addl %edx, %edi # sched: [1:0.25]
-; BROADWELL-NEXT:    addl %edi, (%rsi) # sched: [7:1.00]
-; BROADWELL-NEXT:    addl (%rsi), %edi # sched: [6:0.50]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_add_32:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    addl $665536, %eax # imm = 0xA27C0
-; SKYLAKE-NEXT:    # sched: [1:0.25]
-; SKYLAKE-NEXT:    addl $665536, %edi # imm = 0xA27C0
-; SKYLAKE-NEXT:    # sched: [1:0.25]
-; SKYLAKE-NEXT:    addl $665536, (%rsi) # imm = 0xA27C0
-; SKYLAKE-NEXT:    # sched: [7:1.00]
-; SKYLAKE-NEXT:    addl $7, %edi # sched: [1:0.25]
-; SKYLAKE-NEXT:    addl $7, (%rsi) # sched: [7:1.00]
-; SKYLAKE-NEXT:    addl %edx, %edi # sched: [1:0.25]
-; SKYLAKE-NEXT:    addl %edi, (%rsi) # sched: [7:1.00]
-; SKYLAKE-NEXT:    addl (%rsi), %edi # sched: [6:0.50]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_add_32:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    addl $665536, %eax # imm = 0xA27C0
-; SKX-NEXT:    # sched: [1:0.25]
-; SKX-NEXT:    addl $665536, %edi # imm = 0xA27C0
-; SKX-NEXT:    # sched: [1:0.25]
-; SKX-NEXT:    addl $665536, (%rsi) # imm = 0xA27C0
-; SKX-NEXT:    # sched: [7:1.00]
-; SKX-NEXT:    addl $7, %edi # sched: [1:0.25]
-; SKX-NEXT:    addl $7, (%rsi) # sched: [7:1.00]
-; SKX-NEXT:    addl %edx, %edi # sched: [1:0.25]
-; SKX-NEXT:    addl %edi, (%rsi) # sched: [7:1.00]
-; SKX-NEXT:    addl (%rsi), %edi # sched: [6:0.50]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_add_32:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    addl $665536, %eax # imm = 0xA27C0
-; BDVER2-NEXT:    # sched: [1:0.50]
-; BDVER2-NEXT:    addl $665536, %edi # imm = 0xA27C0
-; BDVER2-NEXT:    # sched: [1:0.50]
-; BDVER2-NEXT:    addl $665536, (%rsi) # imm = 0xA27C0
-; BDVER2-NEXT:    # sched: [6:1.00]
-; BDVER2-NEXT:    addl $7, %edi # sched: [1:0.50]
-; BDVER2-NEXT:    addl $7, (%rsi) # sched: [6:1.00]
-; BDVER2-NEXT:    addl %edx, %edi # sched: [1:0.50]
-; BDVER2-NEXT:    addl %edi, (%rsi) # sched: [6:1.00]
-; BDVER2-NEXT:    addl (%rsi), %edi # sched: [5:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_add_32:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    addl $665536, %eax # imm = 0xA27C0
-; BTVER2-NEXT:    # sched: [1:0.50]
-; BTVER2-NEXT:    addl $665536, %edi # imm = 0xA27C0
-; BTVER2-NEXT:    # sched: [1:0.50]
-; BTVER2-NEXT:    addl $665536, (%rsi) # imm = 0xA27C0
-; BTVER2-NEXT:    # sched: [5:1.00]
-; BTVER2-NEXT:    addl $7, %edi # sched: [1:0.50]
-; BTVER2-NEXT:    addl $7, (%rsi) # sched: [5:1.00]
-; BTVER2-NEXT:    addl %edx, %edi # sched: [1:0.50]
-; BTVER2-NEXT:    addl %edi, (%rsi) # sched: [5:1.00]
-; BTVER2-NEXT:    addl (%rsi), %edi # sched: [4:1.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_add_32:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    addl $665536, %eax # imm = 0xA27C0
-; ZNVER1-NEXT:    # sched: [1:0.25]
-; ZNVER1-NEXT:    addl $665536, %edi # imm = 0xA27C0
-; ZNVER1-NEXT:    # sched: [1:0.25]
-; ZNVER1-NEXT:    addl $665536, (%rsi) # imm = 0xA27C0
-; ZNVER1-NEXT:    # sched: [5:0.50]
-; ZNVER1-NEXT:    addl $7, %edi # sched: [1:0.25]
-; ZNVER1-NEXT:    addl $7, (%rsi) # sched: [5:0.50]
-; ZNVER1-NEXT:    addl %edx, %edi # sched: [1:0.25]
-; ZNVER1-NEXT:    addl %edi, (%rsi) # sched: [5:0.50]
-; ZNVER1-NEXT:    addl (%rsi), %edi # sched: [5:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  tail call void asm "addl $3, %EAX \0A\09 addl $3, $0 \0A\09 addl $3, $2 \0A\09 addl $4, $0 \0A\09 addl $4, $2 \0A\09 addl $1, $0 \0A\09 addl $0, $2 \0A\09 addl $2, $0", "r,r,*m,i,i"(i32 %a0, i32 %a2, i32* %a1, i32 665536, i8 7) nounwind
-  ret void
-}
-define void @test_add_64(i64 %a0, i64* %a1, i64 %a2) optsize {
-; GENERIC-LABEL: test_add_64:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    addq $665536, %rax # imm = 0xA27C0
-; GENERIC-NEXT:    # sched: [1:0.33]
-; GENERIC-NEXT:    addq $665536, %rdi # imm = 0xA27C0
-; GENERIC-NEXT:    # sched: [1:0.33]
-; GENERIC-NEXT:    addq $665536, (%rsi) # imm = 0xA27C0
-; GENERIC-NEXT:    # sched: [7:1.00]
-; GENERIC-NEXT:    addq $7, %rdi # sched: [1:0.33]
-; GENERIC-NEXT:    addq $7, (%rsi) # sched: [7:1.00]
-; GENERIC-NEXT:    addq %rdx, %rdi # sched: [1:0.33]
-; GENERIC-NEXT:    addq %rdi, (%rsi) # sched: [7:1.00]
-; GENERIC-NEXT:    addq (%rsi), %rdi # sched: [6:0.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_add_64:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    addq $665536, %rax # imm = 0xA27C0
-; ATOM-NEXT:    # sched: [1:0.50]
-; ATOM-NEXT:    addq $665536, %rdi # imm = 0xA27C0
-; ATOM-NEXT:    # sched: [1:0.50]
-; ATOM-NEXT:    addq $665536, (%rsi) # imm = 0xA27C0
-; ATOM-NEXT:    # sched: [1:1.00]
-; ATOM-NEXT:    addq $7, %rdi # sched: [1:0.50]
-; ATOM-NEXT:    addq $7, (%rsi) # sched: [1:1.00]
-; ATOM-NEXT:    addq %rdx, %rdi # sched: [1:0.50]
-; ATOM-NEXT:    addq %rdi, (%rsi) # sched: [1:1.00]
-; ATOM-NEXT:    addq (%rsi), %rdi # sched: [1:1.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_add_64:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    addq $665536, %rax # imm = 0xA27C0
-; SLM-NEXT:    # sched: [1:0.50]
-; SLM-NEXT:    addq $665536, %rdi # imm = 0xA27C0
-; SLM-NEXT:    # sched: [1:0.50]
-; SLM-NEXT:    addq $665536, (%rsi) # imm = 0xA27C0
-; SLM-NEXT:    # sched: [5:2.00]
-; SLM-NEXT:    addq $7, %rdi # sched: [1:0.50]
-; SLM-NEXT:    addq $7, (%rsi) # sched: [5:2.00]
-; SLM-NEXT:    addq %rdx, %rdi # sched: [1:0.50]
-; SLM-NEXT:    addq %rdi, (%rsi) # sched: [5:2.00]
-; SLM-NEXT:    addq (%rsi), %rdi # sched: [4:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_add_64:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    addq $665536, %rax # imm = 0xA27C0
-; SANDY-NEXT:    # sched: [1:0.33]
-; SANDY-NEXT:    addq $665536, %rdi # imm = 0xA27C0
-; SANDY-NEXT:    # sched: [1:0.33]
-; SANDY-NEXT:    addq $665536, (%rsi) # imm = 0xA27C0
-; SANDY-NEXT:    # sched: [7:1.00]
-; SANDY-NEXT:    addq $7, %rdi # sched: [1:0.33]
-; SANDY-NEXT:    addq $7, (%rsi) # sched: [7:1.00]
-; SANDY-NEXT:    addq %rdx, %rdi # sched: [1:0.33]
-; SANDY-NEXT:    addq %rdi, (%rsi) # sched: [7:1.00]
-; SANDY-NEXT:    addq (%rsi), %rdi # sched: [6:0.50]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_add_64:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    addq $665536, %rax # imm = 0xA27C0
-; HASWELL-NEXT:    # sched: [1:0.25]
-; HASWELL-NEXT:    addq $665536, %rdi # imm = 0xA27C0
-; HASWELL-NEXT:    # sched: [1:0.25]
-; HASWELL-NEXT:    addq $665536, (%rsi) # imm = 0xA27C0
-; HASWELL-NEXT:    # sched: [7:1.00]
-; HASWELL-NEXT:    addq $7, %rdi # sched: [1:0.25]
-; HASWELL-NEXT:    addq $7, (%rsi) # sched: [7:1.00]
-; HASWELL-NEXT:    addq %rdx, %rdi # sched: [1:0.25]
-; HASWELL-NEXT:    addq %rdi, (%rsi) # sched: [7:1.00]
-; HASWELL-NEXT:    addq (%rsi), %rdi # sched: [6:0.50]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_add_64:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    addq $665536, %rax # imm = 0xA27C0
-; BROADWELL-NEXT:    # sched: [1:0.25]
-; BROADWELL-NEXT:    addq $665536, %rdi # imm = 0xA27C0
-; BROADWELL-NEXT:    # sched: [1:0.25]
-; BROADWELL-NEXT:    addq $665536, (%rsi) # imm = 0xA27C0
-; BROADWELL-NEXT:    # sched: [7:1.00]
-; BROADWELL-NEXT:    addq $7, %rdi # sched: [1:0.25]
-; BROADWELL-NEXT:    addq $7, (%rsi) # sched: [7:1.00]
-; BROADWELL-NEXT:    addq %rdx, %rdi # sched: [1:0.25]
-; BROADWELL-NEXT:    addq %rdi, (%rsi) # sched: [7:1.00]
-; BROADWELL-NEXT:    addq (%rsi), %rdi # sched: [6:0.50]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_add_64:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    addq $665536, %rax # imm = 0xA27C0
-; SKYLAKE-NEXT:    # sched: [1:0.25]
-; SKYLAKE-NEXT:    addq $665536, %rdi # imm = 0xA27C0
-; SKYLAKE-NEXT:    # sched: [1:0.25]
-; SKYLAKE-NEXT:    addq $665536, (%rsi) # imm = 0xA27C0
-; SKYLAKE-NEXT:    # sched: [7:1.00]
-; SKYLAKE-NEXT:    addq $7, %rdi # sched: [1:0.25]
-; SKYLAKE-NEXT:    addq $7, (%rsi) # sched: [7:1.00]
-; SKYLAKE-NEXT:    addq %rdx, %rdi # sched: [1:0.25]
-; SKYLAKE-NEXT:    addq %rdi, (%rsi) # sched: [7:1.00]
-; SKYLAKE-NEXT:    addq (%rsi), %rdi # sched: [6:0.50]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_add_64:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    addq $665536, %rax # imm = 0xA27C0
-; SKX-NEXT:    # sched: [1:0.25]
-; SKX-NEXT:    addq $665536, %rdi # imm = 0xA27C0
-; SKX-NEXT:    # sched: [1:0.25]
-; SKX-NEXT:    addq $665536, (%rsi) # imm = 0xA27C0
-; SKX-NEXT:    # sched: [7:1.00]
-; SKX-NEXT:    addq $7, %rdi # sched: [1:0.25]
-; SKX-NEXT:    addq $7, (%rsi) # sched: [7:1.00]
-; SKX-NEXT:    addq %rdx, %rdi # sched: [1:0.25]
-; SKX-NEXT:    addq %rdi, (%rsi) # sched: [7:1.00]
-; SKX-NEXT:    addq (%rsi), %rdi # sched: [6:0.50]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_add_64:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    addq $665536, %rax # imm = 0xA27C0
-; BDVER2-NEXT:    # sched: [1:0.50]
-; BDVER2-NEXT:    addq $665536, %rdi # imm = 0xA27C0
-; BDVER2-NEXT:    # sched: [1:0.50]
-; BDVER2-NEXT:    addq $665536, (%rsi) # imm = 0xA27C0
-; BDVER2-NEXT:    # sched: [6:1.00]
-; BDVER2-NEXT:    addq $7, %rdi # sched: [1:0.50]
-; BDVER2-NEXT:    addq $7, (%rsi) # sched: [6:1.00]
-; BDVER2-NEXT:    addq %rdx, %rdi # sched: [1:0.50]
-; BDVER2-NEXT:    addq %rdi, (%rsi) # sched: [6:1.00]
-; BDVER2-NEXT:    addq (%rsi), %rdi # sched: [5:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_add_64:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    addq $665536, %rax # imm = 0xA27C0
-; BTVER2-NEXT:    # sched: [1:0.50]
-; BTVER2-NEXT:    addq $665536, %rdi # imm = 0xA27C0
-; BTVER2-NEXT:    # sched: [1:0.50]
-; BTVER2-NEXT:    addq $665536, (%rsi) # imm = 0xA27C0
-; BTVER2-NEXT:    # sched: [5:1.00]
-; BTVER2-NEXT:    addq $7, %rdi # sched: [1:0.50]
-; BTVER2-NEXT:    addq $7, (%rsi) # sched: [5:1.00]
-; BTVER2-NEXT:    addq %rdx, %rdi # sched: [1:0.50]
-; BTVER2-NEXT:    addq %rdi, (%rsi) # sched: [5:1.00]
-; BTVER2-NEXT:    addq (%rsi), %rdi # sched: [4:1.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_add_64:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    addq $665536, %rax # imm = 0xA27C0
-; ZNVER1-NEXT:    # sched: [1:0.25]
-; ZNVER1-NEXT:    addq $665536, %rdi # imm = 0xA27C0
-; ZNVER1-NEXT:    # sched: [1:0.25]
-; ZNVER1-NEXT:    addq $665536, (%rsi) # imm = 0xA27C0
-; ZNVER1-NEXT:    # sched: [5:0.50]
-; ZNVER1-NEXT:    addq $7, %rdi # sched: [1:0.25]
-; ZNVER1-NEXT:    addq $7, (%rsi) # sched: [5:0.50]
-; ZNVER1-NEXT:    addq %rdx, %rdi # sched: [1:0.25]
-; ZNVER1-NEXT:    addq %rdi, (%rsi) # sched: [5:0.50]
-; ZNVER1-NEXT:    addq (%rsi), %rdi # sched: [5:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  tail call void asm "addq $3, %RAX \0A\09 addq $3, $0 \0A\09 addq $3, $2 \0A\09 addq $4, $0 \0A\09 addq $4, $2 \0A\09 addq $1, $0 \0A\09 addq $0, $2 \0A\09 addq $2, $0", "r,r,*m,i,i"(i64 %a0, i64 %a2, i64* %a1, i32 665536, i8 7) nounwind
-  ret void
-}
-
-define void @test_and_8(i8 %a0, i8* %a1, i8 %a2) optsize {
-; GENERIC-LABEL: test_and_8:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    andb $7, %al # sched: [1:0.33]
-; GENERIC-NEXT:    andb $7, %dil # sched: [1:0.33]
-; GENERIC-NEXT:    andb $7, (%rsi) # sched: [7:1.00]
-; GENERIC-NEXT:    andb %dl, %dil # sched: [1:0.33]
-; GENERIC-NEXT:    andb %dil, (%rsi) # sched: [7:1.00]
-; GENERIC-NEXT:    andb (%rsi), %dil # sched: [6:0.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_and_8:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    andb $7, %al # sched: [1:0.50]
-; ATOM-NEXT:    andb $7, %dil # sched: [1:0.50]
-; ATOM-NEXT:    andb $7, (%rsi) # sched: [1:1.00]
-; ATOM-NEXT:    andb %dl, %dil # sched: [1:0.50]
-; ATOM-NEXT:    andb %dil, (%rsi) # sched: [1:1.00]
-; ATOM-NEXT:    andb (%rsi), %dil # sched: [1:1.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_and_8:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    andb $7, %al # sched: [1:0.50]
-; SLM-NEXT:    andb $7, %dil # sched: [1:0.50]
-; SLM-NEXT:    andb $7, (%rsi) # sched: [5:2.00]
-; SLM-NEXT:    andb %dl, %dil # sched: [1:0.50]
-; SLM-NEXT:    andb %dil, (%rsi) # sched: [5:2.00]
-; SLM-NEXT:    andb (%rsi), %dil # sched: [4:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_and_8:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    andb $7, %al # sched: [1:0.33]
-; SANDY-NEXT:    andb $7, %dil # sched: [1:0.33]
-; SANDY-NEXT:    andb $7, (%rsi) # sched: [7:1.00]
-; SANDY-NEXT:    andb %dl, %dil # sched: [1:0.33]
-; SANDY-NEXT:    andb %dil, (%rsi) # sched: [7:1.00]
-; SANDY-NEXT:    andb (%rsi), %dil # sched: [6:0.50]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_and_8:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    andb $7, %al # sched: [1:0.25]
-; HASWELL-NEXT:    andb $7, %dil # sched: [1:0.25]
-; HASWELL-NEXT:    andb $7, (%rsi) # sched: [7:1.00]
-; HASWELL-NEXT:    andb %dl, %dil # sched: [1:0.25]
-; HASWELL-NEXT:    andb %dil, (%rsi) # sched: [7:1.00]
-; HASWELL-NEXT:    andb (%rsi), %dil # sched: [6:0.50]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_and_8:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    andb $7, %al # sched: [1:0.25]
-; BROADWELL-NEXT:    andb $7, %dil # sched: [1:0.25]
-; BROADWELL-NEXT:    andb $7, (%rsi) # sched: [7:1.00]
-; BROADWELL-NEXT:    andb %dl, %dil # sched: [1:0.25]
-; BROADWELL-NEXT:    andb %dil, (%rsi) # sched: [7:1.00]
-; BROADWELL-NEXT:    andb (%rsi), %dil # sched: [6:0.50]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_and_8:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    andb $7, %al # sched: [1:0.25]
-; SKYLAKE-NEXT:    andb $7, %dil # sched: [1:0.25]
-; SKYLAKE-NEXT:    andb $7, (%rsi) # sched: [7:1.00]
-; SKYLAKE-NEXT:    andb %dl, %dil # sched: [1:0.25]
-; SKYLAKE-NEXT:    andb %dil, (%rsi) # sched: [7:1.00]
-; SKYLAKE-NEXT:    andb (%rsi), %dil # sched: [6:0.50]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_and_8:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    andb $7, %al # sched: [1:0.25]
-; SKX-NEXT:    andb $7, %dil # sched: [1:0.25]
-; SKX-NEXT:    andb $7, (%rsi) # sched: [7:1.00]
-; SKX-NEXT:    andb %dl, %dil # sched: [1:0.25]
-; SKX-NEXT:    andb %dil, (%rsi) # sched: [7:1.00]
-; SKX-NEXT:    andb (%rsi), %dil # sched: [6:0.50]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_and_8:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    andb $7, %al # sched: [1:0.50]
-; BDVER2-NEXT:    andb $7, %dil # sched: [1:0.50]
-; BDVER2-NEXT:    andb $7, (%rsi) # sched: [6:1.00]
-; BDVER2-NEXT:    andb %dl, %dil # sched: [1:0.50]
-; BDVER2-NEXT:    andb %dil, (%rsi) # sched: [6:1.00]
-; BDVER2-NEXT:    andb (%rsi), %dil # sched: [5:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_and_8:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    andb $7, %al # sched: [1:0.50]
-; BTVER2-NEXT:    andb $7, %dil # sched: [1:0.50]
-; BTVER2-NEXT:    andb $7, (%rsi) # sched: [5:1.00]
-; BTVER2-NEXT:    andb %dl, %dil # sched: [1:0.50]
-; BTVER2-NEXT:    andb %dil, (%rsi) # sched: [5:1.00]
-; BTVER2-NEXT:    andb (%rsi), %dil # sched: [4:1.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_and_8:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    andb $7, %al # sched: [1:0.25]
-; ZNVER1-NEXT:    andb $7, %dil # sched: [1:0.25]
-; ZNVER1-NEXT:    andb $7, (%rsi) # sched: [5:0.50]
-; ZNVER1-NEXT:    andb %dl, %dil # sched: [1:0.25]
-; ZNVER1-NEXT:    andb %dil, (%rsi) # sched: [5:0.50]
-; ZNVER1-NEXT:    andb (%rsi), %dil # sched: [5:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  tail call void asm "andb $3, %AL \0A\09 andb $3, $0 \0A\09 andb $3, $2 \0A\09 andb $1, $0 \0A\09 andb $0, $2 \0A\09 andb $2, $0", "r,r,*m,i"(i8 %a0, i8 %a2, i8* %a1, i8 7) nounwind
-  ret void
-}
-define void @test_and_16(i16 %a0, i16* %a1, i16 %a2) optsize {
-; GENERIC-LABEL: test_and_16:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    andw $511, %ax # imm = 0x1FF
-; GENERIC-NEXT:    # sched: [1:0.33]
-; GENERIC-NEXT:    andw $511, %di # imm = 0x1FF
-; GENERIC-NEXT:    # sched: [1:0.33]
-; GENERIC-NEXT:    andw $511, (%rsi) # imm = 0x1FF
-; GENERIC-NEXT:    # sched: [7:1.00]
-; GENERIC-NEXT:    andw $7, %di # sched: [1:0.33]
-; GENERIC-NEXT:    andw $7, (%rsi) # sched: [7:1.00]
-; GENERIC-NEXT:    andw %dx, %di # sched: [1:0.33]
-; GENERIC-NEXT:    andw %di, (%rsi) # sched: [7:1.00]
-; GENERIC-NEXT:    andw (%rsi), %di # sched: [6:0.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_and_16:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    andw $511, %ax # imm = 0x1FF
-; ATOM-NEXT:    # sched: [1:0.50]
-; ATOM-NEXT:    andw $511, %di # imm = 0x1FF
-; ATOM-NEXT:    # sched: [1:0.50]
-; ATOM-NEXT:    andw $511, (%rsi) # imm = 0x1FF
-; ATOM-NEXT:    # sched: [1:1.00]
-; ATOM-NEXT:    andw $7, %di # sched: [1:0.50]
-; ATOM-NEXT:    andw $7, (%rsi) # sched: [1:1.00]
-; ATOM-NEXT:    andw %dx, %di # sched: [1:0.50]
-; ATOM-NEXT:    andw %di, (%rsi) # sched: [1:1.00]
-; ATOM-NEXT:    andw (%rsi), %di # sched: [1:1.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_and_16:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    andw $511, %ax # imm = 0x1FF
-; SLM-NEXT:    # sched: [1:0.50]
-; SLM-NEXT:    andw $511, %di # imm = 0x1FF
-; SLM-NEXT:    # sched: [1:0.50]
-; SLM-NEXT:    andw $511, (%rsi) # imm = 0x1FF
-; SLM-NEXT:    # sched: [5:2.00]
-; SLM-NEXT:    andw $7, %di # sched: [1:0.50]
-; SLM-NEXT:    andw $7, (%rsi) # sched: [5:2.00]
-; SLM-NEXT:    andw %dx, %di # sched: [1:0.50]
-; SLM-NEXT:    andw %di, (%rsi) # sched: [5:2.00]
-; SLM-NEXT:    andw (%rsi), %di # sched: [4:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_and_16:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    andw $511, %ax # imm = 0x1FF
-; SANDY-NEXT:    # sched: [1:0.33]
-; SANDY-NEXT:    andw $511, %di # imm = 0x1FF
-; SANDY-NEXT:    # sched: [1:0.33]
-; SANDY-NEXT:    andw $511, (%rsi) # imm = 0x1FF
-; SANDY-NEXT:    # sched: [7:1.00]
-; SANDY-NEXT:    andw $7, %di # sched: [1:0.33]
-; SANDY-NEXT:    andw $7, (%rsi) # sched: [7:1.00]
-; SANDY-NEXT:    andw %dx, %di # sched: [1:0.33]
-; SANDY-NEXT:    andw %di, (%rsi) # sched: [7:1.00]
-; SANDY-NEXT:    andw (%rsi), %di # sched: [6:0.50]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_and_16:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    andw $511, %ax # imm = 0x1FF
-; HASWELL-NEXT:    # sched: [1:0.25]
-; HASWELL-NEXT:    andw $511, %di # imm = 0x1FF
-; HASWELL-NEXT:    # sched: [1:0.25]
-; HASWELL-NEXT:    andw $511, (%rsi) # imm = 0x1FF
-; HASWELL-NEXT:    # sched: [7:1.00]
-; HASWELL-NEXT:    andw $7, %di # sched: [1:0.25]
-; HASWELL-NEXT:    andw $7, (%rsi) # sched: [7:1.00]
-; HASWELL-NEXT:    andw %dx, %di # sched: [1:0.25]
-; HASWELL-NEXT:    andw %di, (%rsi) # sched: [7:1.00]
-; HASWELL-NEXT:    andw (%rsi), %di # sched: [6:0.50]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_and_16:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    andw $511, %ax # imm = 0x1FF
-; BROADWELL-NEXT:    # sched: [1:0.25]
-; BROADWELL-NEXT:    andw $511, %di # imm = 0x1FF
-; BROADWELL-NEXT:    # sched: [1:0.25]
-; BROADWELL-NEXT:    andw $511, (%rsi) # imm = 0x1FF
-; BROADWELL-NEXT:    # sched: [7:1.00]
-; BROADWELL-NEXT:    andw $7, %di # sched: [1:0.25]
-; BROADWELL-NEXT:    andw $7, (%rsi) # sched: [7:1.00]
-; BROADWELL-NEXT:    andw %dx, %di # sched: [1:0.25]
-; BROADWELL-NEXT:    andw %di, (%rsi) # sched: [7:1.00]
-; BROADWELL-NEXT:    andw (%rsi), %di # sched: [6:0.50]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_and_16:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    andw $511, %ax # imm = 0x1FF
-; SKYLAKE-NEXT:    # sched: [1:0.25]
-; SKYLAKE-NEXT:    andw $511, %di # imm = 0x1FF
-; SKYLAKE-NEXT:    # sched: [1:0.25]
-; SKYLAKE-NEXT:    andw $511, (%rsi) # imm = 0x1FF
-; SKYLAKE-NEXT:    # sched: [7:1.00]
-; SKYLAKE-NEXT:    andw $7, %di # sched: [1:0.25]
-; SKYLAKE-NEXT:    andw $7, (%rsi) # sched: [7:1.00]
-; SKYLAKE-NEXT:    andw %dx, %di # sched: [1:0.25]
-; SKYLAKE-NEXT:    andw %di, (%rsi) # sched: [7:1.00]
-; SKYLAKE-NEXT:    andw (%rsi), %di # sched: [6:0.50]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_and_16:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    andw $511, %ax # imm = 0x1FF
-; SKX-NEXT:    # sched: [1:0.25]
-; SKX-NEXT:    andw $511, %di # imm = 0x1FF
-; SKX-NEXT:    # sched: [1:0.25]
-; SKX-NEXT:    andw $511, (%rsi) # imm = 0x1FF
-; SKX-NEXT:    # sched: [7:1.00]
-; SKX-NEXT:    andw $7, %di # sched: [1:0.25]
-; SKX-NEXT:    andw $7, (%rsi) # sched: [7:1.00]
-; SKX-NEXT:    andw %dx, %di # sched: [1:0.25]
-; SKX-NEXT:    andw %di, (%rsi) # sched: [7:1.00]
-; SKX-NEXT:    andw (%rsi), %di # sched: [6:0.50]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_and_16:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    andw $511, %ax # imm = 0x1FF
-; BDVER2-NEXT:    # sched: [1:0.50]
-; BDVER2-NEXT:    andw $511, %di # imm = 0x1FF
-; BDVER2-NEXT:    # sched: [1:0.50]
-; BDVER2-NEXT:    andw $511, (%rsi) # imm = 0x1FF
-; BDVER2-NEXT:    # sched: [6:1.00]
-; BDVER2-NEXT:    andw $7, %di # sched: [1:0.50]
-; BDVER2-NEXT:    andw $7, (%rsi) # sched: [6:1.00]
-; BDVER2-NEXT:    andw %dx, %di # sched: [1:0.50]
-; BDVER2-NEXT:    andw %di, (%rsi) # sched: [6:1.00]
-; BDVER2-NEXT:    andw (%rsi), %di # sched: [5:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_and_16:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    andw $511, %ax # imm = 0x1FF
-; BTVER2-NEXT:    # sched: [1:0.50]
-; BTVER2-NEXT:    andw $511, %di # imm = 0x1FF
-; BTVER2-NEXT:    # sched: [1:0.50]
-; BTVER2-NEXT:    andw $511, (%rsi) # imm = 0x1FF
-; BTVER2-NEXT:    # sched: [5:1.00]
-; BTVER2-NEXT:    andw $7, %di # sched: [1:0.50]
-; BTVER2-NEXT:    andw $7, (%rsi) # sched: [5:1.00]
-; BTVER2-NEXT:    andw %dx, %di # sched: [1:0.50]
-; BTVER2-NEXT:    andw %di, (%rsi) # sched: [5:1.00]
-; BTVER2-NEXT:    andw (%rsi), %di # sched: [4:1.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_and_16:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    andw $511, %ax # imm = 0x1FF
-; ZNVER1-NEXT:    # sched: [1:0.25]
-; ZNVER1-NEXT:    andw $511, %di # imm = 0x1FF
-; ZNVER1-NEXT:    # sched: [1:0.25]
-; ZNVER1-NEXT:    andw $511, (%rsi) # imm = 0x1FF
-; ZNVER1-NEXT:    # sched: [5:0.50]
-; ZNVER1-NEXT:    andw $7, %di # sched: [1:0.25]
-; ZNVER1-NEXT:    andw $7, (%rsi) # sched: [5:0.50]
-; ZNVER1-NEXT:    andw %dx, %di # sched: [1:0.25]
-; ZNVER1-NEXT:    andw %di, (%rsi) # sched: [5:0.50]
-; ZNVER1-NEXT:    andw (%rsi), %di # sched: [5:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  tail call void asm "andw $3, %AX \0A\09 andw $3, $0 \0A\09 andw $3, $2 \0A\09 andw $4, $0 \0A\09 andw $4, $2 \0A\09 andw $1, $0 \0A\09 andw $0, $2 \0A\09 andw $2, $0", "r,r,*m,i,i"(i16 %a0, i16 %a2, i16* %a1, i16 511, i8 7) nounwind
-  ret void
-}
-define void @test_and_32(i32 %a0, i32* %a1, i32 %a2) optsize {
-; GENERIC-LABEL: test_and_32:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    andl $665536, %eax # imm = 0xA27C0
-; GENERIC-NEXT:    # sched: [1:0.33]
-; GENERIC-NEXT:    andl $665536, %edi # imm = 0xA27C0
-; GENERIC-NEXT:    # sched: [1:0.33]
-; GENERIC-NEXT:    andl $665536, (%rsi) # imm = 0xA27C0
-; GENERIC-NEXT:    # sched: [7:1.00]
-; GENERIC-NEXT:    andl $7, %edi # sched: [1:0.33]
-; GENERIC-NEXT:    andl $7, (%rsi) # sched: [7:1.00]
-; GENERIC-NEXT:    andl %edx, %edi # sched: [1:0.33]
-; GENERIC-NEXT:    andl %edi, (%rsi) # sched: [7:1.00]
-; GENERIC-NEXT:    andl (%rsi), %edi # sched: [6:0.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_and_32:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    andl $665536, %eax # imm = 0xA27C0
-; ATOM-NEXT:    # sched: [1:0.50]
-; ATOM-NEXT:    andl $665536, %edi # imm = 0xA27C0
-; ATOM-NEXT:    # sched: [1:0.50]
-; ATOM-NEXT:    andl $665536, (%rsi) # imm = 0xA27C0
-; ATOM-NEXT:    # sched: [1:1.00]
-; ATOM-NEXT:    andl $7, %edi # sched: [1:0.50]
-; ATOM-NEXT:    andl $7, (%rsi) # sched: [1:1.00]
-; ATOM-NEXT:    andl %edx, %edi # sched: [1:0.50]
-; ATOM-NEXT:    andl %edi, (%rsi) # sched: [1:1.00]
-; ATOM-NEXT:    andl (%rsi), %edi # sched: [1:1.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_and_32:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    andl $665536, %eax # imm = 0xA27C0
-; SLM-NEXT:    # sched: [1:0.50]
-; SLM-NEXT:    andl $665536, %edi # imm = 0xA27C0
-; SLM-NEXT:    # sched: [1:0.50]
-; SLM-NEXT:    andl $665536, (%rsi) # imm = 0xA27C0
-; SLM-NEXT:    # sched: [5:2.00]
-; SLM-NEXT:    andl $7, %edi # sched: [1:0.50]
-; SLM-NEXT:    andl $7, (%rsi) # sched: [5:2.00]
-; SLM-NEXT:    andl %edx, %edi # sched: [1:0.50]
-; SLM-NEXT:    andl %edi, (%rsi) # sched: [5:2.00]
-; SLM-NEXT:    andl (%rsi), %edi # sched: [4:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_and_32:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    andl $665536, %eax # imm = 0xA27C0
-; SANDY-NEXT:    # sched: [1:0.33]
-; SANDY-NEXT:    andl $665536, %edi # imm = 0xA27C0
-; SANDY-NEXT:    # sched: [1:0.33]
-; SANDY-NEXT:    andl $665536, (%rsi) # imm = 0xA27C0
-; SANDY-NEXT:    # sched: [7:1.00]
-; SANDY-NEXT:    andl $7, %edi # sched: [1:0.33]
-; SANDY-NEXT:    andl $7, (%rsi) # sched: [7:1.00]
-; SANDY-NEXT:    andl %edx, %edi # sched: [1:0.33]
-; SANDY-NEXT:    andl %edi, (%rsi) # sched: [7:1.00]
-; SANDY-NEXT:    andl (%rsi), %edi # sched: [6:0.50]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_and_32:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    andl $665536, %eax # imm = 0xA27C0
-; HASWELL-NEXT:    # sched: [1:0.25]
-; HASWELL-NEXT:    andl $665536, %edi # imm = 0xA27C0
-; HASWELL-NEXT:    # sched: [1:0.25]
-; HASWELL-NEXT:    andl $665536, (%rsi) # imm = 0xA27C0
-; HASWELL-NEXT:    # sched: [7:1.00]
-; HASWELL-NEXT:    andl $7, %edi # sched: [1:0.25]
-; HASWELL-NEXT:    andl $7, (%rsi) # sched: [7:1.00]
-; HASWELL-NEXT:    andl %edx, %edi # sched: [1:0.25]
-; HASWELL-NEXT:    andl %edi, (%rsi) # sched: [7:1.00]
-; HASWELL-NEXT:    andl (%rsi), %edi # sched: [6:0.50]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_and_32:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    andl $665536, %eax # imm = 0xA27C0
-; BROADWELL-NEXT:    # sched: [1:0.25]
-; BROADWELL-NEXT:    andl $665536, %edi # imm = 0xA27C0
-; BROADWELL-NEXT:    # sched: [1:0.25]
-; BROADWELL-NEXT:    andl $665536, (%rsi) # imm = 0xA27C0
-; BROADWELL-NEXT:    # sched: [7:1.00]
-; BROADWELL-NEXT:    andl $7, %edi # sched: [1:0.25]
-; BROADWELL-NEXT:    andl $7, (%rsi) # sched: [7:1.00]
-; BROADWELL-NEXT:    andl %edx, %edi # sched: [1:0.25]
-; BROADWELL-NEXT:    andl %edi, (%rsi) # sched: [7:1.00]
-; BROADWELL-NEXT:    andl (%rsi), %edi # sched: [6:0.50]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_and_32:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    andl $665536, %eax # imm = 0xA27C0
-; SKYLAKE-NEXT:    # sched: [1:0.25]
-; SKYLAKE-NEXT:    andl $665536, %edi # imm = 0xA27C0
-; SKYLAKE-NEXT:    # sched: [1:0.25]
-; SKYLAKE-NEXT:    andl $665536, (%rsi) # imm = 0xA27C0
-; SKYLAKE-NEXT:    # sched: [7:1.00]
-; SKYLAKE-NEXT:    andl $7, %edi # sched: [1:0.25]
-; SKYLAKE-NEXT:    andl $7, (%rsi) # sched: [7:1.00]
-; SKYLAKE-NEXT:    andl %edx, %edi # sched: [1:0.25]
-; SKYLAKE-NEXT:    andl %edi, (%rsi) # sched: [7:1.00]
-; SKYLAKE-NEXT:    andl (%rsi), %edi # sched: [6:0.50]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_and_32:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    andl $665536, %eax # imm = 0xA27C0
-; SKX-NEXT:    # sched: [1:0.25]
-; SKX-NEXT:    andl $665536, %edi # imm = 0xA27C0
-; SKX-NEXT:    # sched: [1:0.25]
-; SKX-NEXT:    andl $665536, (%rsi) # imm = 0xA27C0
-; SKX-NEXT:    # sched: [7:1.00]
-; SKX-NEXT:    andl $7, %edi # sched: [1:0.25]
-; SKX-NEXT:    andl $7, (%rsi) # sched: [7:1.00]
-; SKX-NEXT:    andl %edx, %edi # sched: [1:0.25]
-; SKX-NEXT:    andl %edi, (%rsi) # sched: [7:1.00]
-; SKX-NEXT:    andl (%rsi), %edi # sched: [6:0.50]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_and_32:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    andl $665536, %eax # imm = 0xA27C0
-; BDVER2-NEXT:    # sched: [1:0.50]
-; BDVER2-NEXT:    andl $665536, %edi # imm = 0xA27C0
-; BDVER2-NEXT:    # sched: [1:0.50]
-; BDVER2-NEXT:    andl $665536, (%rsi) # imm = 0xA27C0
-; BDVER2-NEXT:    # sched: [6:1.00]
-; BDVER2-NEXT:    andl $7, %edi # sched: [1:0.50]
-; BDVER2-NEXT:    andl $7, (%rsi) # sched: [6:1.00]
-; BDVER2-NEXT:    andl %edx, %edi # sched: [1:0.50]
-; BDVER2-NEXT:    andl %edi, (%rsi) # sched: [6:1.00]
-; BDVER2-NEXT:    andl (%rsi), %edi # sched: [5:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_and_32:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    andl $665536, %eax # imm = 0xA27C0
-; BTVER2-NEXT:    # sched: [1:0.50]
-; BTVER2-NEXT:    andl $665536, %edi # imm = 0xA27C0
-; BTVER2-NEXT:    # sched: [1:0.50]
-; BTVER2-NEXT:    andl $665536, (%rsi) # imm = 0xA27C0
-; BTVER2-NEXT:    # sched: [5:1.00]
-; BTVER2-NEXT:    andl $7, %edi # sched: [1:0.50]
-; BTVER2-NEXT:    andl $7, (%rsi) # sched: [5:1.00]
-; BTVER2-NEXT:    andl %edx, %edi # sched: [1:0.50]
-; BTVER2-NEXT:    andl %edi, (%rsi) # sched: [5:1.00]
-; BTVER2-NEXT:    andl (%rsi), %edi # sched: [4:1.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_and_32:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    andl $665536, %eax # imm = 0xA27C0
-; ZNVER1-NEXT:    # sched: [1:0.25]
-; ZNVER1-NEXT:    andl $665536, %edi # imm = 0xA27C0
-; ZNVER1-NEXT:    # sched: [1:0.25]
-; ZNVER1-NEXT:    andl $665536, (%rsi) # imm = 0xA27C0
-; ZNVER1-NEXT:    # sched: [5:0.50]
-; ZNVER1-NEXT:    andl $7, %edi # sched: [1:0.25]
-; ZNVER1-NEXT:    andl $7, (%rsi) # sched: [5:0.50]
-; ZNVER1-NEXT:    andl %edx, %edi # sched: [1:0.25]
-; ZNVER1-NEXT:    andl %edi, (%rsi) # sched: [5:0.50]
-; ZNVER1-NEXT:    andl (%rsi), %edi # sched: [5:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  tail call void asm "andl $3, %EAX \0A\09 andl $3, $0 \0A\09 andl $3, $2 \0A\09 andl $4, $0 \0A\09 andl $4, $2 \0A\09 andl $1, $0 \0A\09 andl $0, $2 \0A\09 andl $2, $0", "r,r,*m,i,i"(i32 %a0, i32 %a2, i32* %a1, i32 665536, i8 7) nounwind
-  ret void
-}
-define void @test_and_64(i64 %a0, i64* %a1, i64 %a2) optsize {
-; GENERIC-LABEL: test_and_64:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    andq $665536, %rax # imm = 0xA27C0
-; GENERIC-NEXT:    # sched: [1:0.33]
-; GENERIC-NEXT:    andq $665536, %rdi # imm = 0xA27C0
-; GENERIC-NEXT:    # sched: [1:0.33]
-; GENERIC-NEXT:    andq $665536, (%rsi) # imm = 0xA27C0
-; GENERIC-NEXT:    # sched: [7:1.00]
-; GENERIC-NEXT:    andq $7, %rdi # sched: [1:0.33]
-; GENERIC-NEXT:    andq $7, (%rsi) # sched: [7:1.00]
-; GENERIC-NEXT:    andq %rdx, %rdi # sched: [1:0.33]
-; GENERIC-NEXT:    andq %rdi, (%rsi) # sched: [7:1.00]
-; GENERIC-NEXT:    andq (%rsi), %rdi # sched: [6:0.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_and_64:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    andq $665536, %rax # imm = 0xA27C0
-; ATOM-NEXT:    # sched: [1:0.50]
-; ATOM-NEXT:    andq $665536, %rdi # imm = 0xA27C0
-; ATOM-NEXT:    # sched: [1:0.50]
-; ATOM-NEXT:    andq $665536, (%rsi) # imm = 0xA27C0
-; ATOM-NEXT:    # sched: [1:1.00]
-; ATOM-NEXT:    andq $7, %rdi # sched: [1:0.50]
-; ATOM-NEXT:    andq $7, (%rsi) # sched: [1:1.00]
-; ATOM-NEXT:    andq %rdx, %rdi # sched: [1:0.50]
-; ATOM-NEXT:    andq %rdi, (%rsi) # sched: [1:1.00]
-; ATOM-NEXT:    andq (%rsi), %rdi # sched: [1:1.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_and_64:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    andq $665536, %rax # imm = 0xA27C0
-; SLM-NEXT:    # sched: [1:0.50]
-; SLM-NEXT:    andq $665536, %rdi # imm = 0xA27C0
-; SLM-NEXT:    # sched: [1:0.50]
-; SLM-NEXT:    andq $665536, (%rsi) # imm = 0xA27C0
-; SLM-NEXT:    # sched: [5:2.00]
-; SLM-NEXT:    andq $7, %rdi # sched: [1:0.50]
-; SLM-NEXT:    andq $7, (%rsi) # sched: [5:2.00]
-; SLM-NEXT:    andq %rdx, %rdi # sched: [1:0.50]
-; SLM-NEXT:    andq %rdi, (%rsi) # sched: [5:2.00]
-; SLM-NEXT:    andq (%rsi), %rdi # sched: [4:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_and_64:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    andq $665536, %rax # imm = 0xA27C0
-; SANDY-NEXT:    # sched: [1:0.33]
-; SANDY-NEXT:    andq $665536, %rdi # imm = 0xA27C0
-; SANDY-NEXT:    # sched: [1:0.33]
-; SANDY-NEXT:    andq $665536, (%rsi) # imm = 0xA27C0
-; SANDY-NEXT:    # sched: [7:1.00]
-; SANDY-NEXT:    andq $7, %rdi # sched: [1:0.33]
-; SANDY-NEXT:    andq $7, (%rsi) # sched: [7:1.00]
-; SANDY-NEXT:    andq %rdx, %rdi # sched: [1:0.33]
-; SANDY-NEXT:    andq %rdi, (%rsi) # sched: [7:1.00]
-; SANDY-NEXT:    andq (%rsi), %rdi # sched: [6:0.50]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_and_64:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    andq $665536, %rax # imm = 0xA27C0
-; HASWELL-NEXT:    # sched: [1:0.25]
-; HASWELL-NEXT:    andq $665536, %rdi # imm = 0xA27C0
-; HASWELL-NEXT:    # sched: [1:0.25]
-; HASWELL-NEXT:    andq $665536, (%rsi) # imm = 0xA27C0
-; HASWELL-NEXT:    # sched: [7:1.00]
-; HASWELL-NEXT:    andq $7, %rdi # sched: [1:0.25]
-; HASWELL-NEXT:    andq $7, (%rsi) # sched: [7:1.00]
-; HASWELL-NEXT:    andq %rdx, %rdi # sched: [1:0.25]
-; HASWELL-NEXT:    andq %rdi, (%rsi) # sched: [7:1.00]
-; HASWELL-NEXT:    andq (%rsi), %rdi # sched: [6:0.50]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_and_64:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    andq $665536, %rax # imm = 0xA27C0
-; BROADWELL-NEXT:    # sched: [1:0.25]
-; BROADWELL-NEXT:    andq $665536, %rdi # imm = 0xA27C0
-; BROADWELL-NEXT:    # sched: [1:0.25]
-; BROADWELL-NEXT:    andq $665536, (%rsi) # imm = 0xA27C0
-; BROADWELL-NEXT:    # sched: [7:1.00]
-; BROADWELL-NEXT:    andq $7, %rdi # sched: [1:0.25]
-; BROADWELL-NEXT:    andq $7, (%rsi) # sched: [7:1.00]
-; BROADWELL-NEXT:    andq %rdx, %rdi # sched: [1:0.25]
-; BROADWELL-NEXT:    andq %rdi, (%rsi) # sched: [7:1.00]
-; BROADWELL-NEXT:    andq (%rsi), %rdi # sched: [6:0.50]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_and_64:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    andq $665536, %rax # imm = 0xA27C0
-; SKYLAKE-NEXT:    # sched: [1:0.25]
-; SKYLAKE-NEXT:    andq $665536, %rdi # imm = 0xA27C0
-; SKYLAKE-NEXT:    # sched: [1:0.25]
-; SKYLAKE-NEXT:    andq $665536, (%rsi) # imm = 0xA27C0
-; SKYLAKE-NEXT:    # sched: [7:1.00]
-; SKYLAKE-NEXT:    andq $7, %rdi # sched: [1:0.25]
-; SKYLAKE-NEXT:    andq $7, (%rsi) # sched: [7:1.00]
-; SKYLAKE-NEXT:    andq %rdx, %rdi # sched: [1:0.25]
-; SKYLAKE-NEXT:    andq %rdi, (%rsi) # sched: [7:1.00]
-; SKYLAKE-NEXT:    andq (%rsi), %rdi # sched: [6:0.50]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_and_64:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    andq $665536, %rax # imm = 0xA27C0
-; SKX-NEXT:    # sched: [1:0.25]
-; SKX-NEXT:    andq $665536, %rdi # imm = 0xA27C0
-; SKX-NEXT:    # sched: [1:0.25]
-; SKX-NEXT:    andq $665536, (%rsi) # imm = 0xA27C0
-; SKX-NEXT:    # sched: [7:1.00]
-; SKX-NEXT:    andq $7, %rdi # sched: [1:0.25]
-; SKX-NEXT:    andq $7, (%rsi) # sched: [7:1.00]
-; SKX-NEXT:    andq %rdx, %rdi # sched: [1:0.25]
-; SKX-NEXT:    andq %rdi, (%rsi) # sched: [7:1.00]
-; SKX-NEXT:    andq (%rsi), %rdi # sched: [6:0.50]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_and_64:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    andq $665536, %rax # imm = 0xA27C0
-; BDVER2-NEXT:    # sched: [1:0.50]
-; BDVER2-NEXT:    andq $665536, %rdi # imm = 0xA27C0
-; BDVER2-NEXT:    # sched: [1:0.50]
-; BDVER2-NEXT:    andq $665536, (%rsi) # imm = 0xA27C0
-; BDVER2-NEXT:    # sched: [6:1.00]
-; BDVER2-NEXT:    andq $7, %rdi # sched: [1:0.50]
-; BDVER2-NEXT:    andq $7, (%rsi) # sched: [6:1.00]
-; BDVER2-NEXT:    andq %rdx, %rdi # sched: [1:0.50]
-; BDVER2-NEXT:    andq %rdi, (%rsi) # sched: [6:1.00]
-; BDVER2-NEXT:    andq (%rsi), %rdi # sched: [5:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_and_64:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    andq $665536, %rax # imm = 0xA27C0
-; BTVER2-NEXT:    # sched: [1:0.50]
-; BTVER2-NEXT:    andq $665536, %rdi # imm = 0xA27C0
-; BTVER2-NEXT:    # sched: [1:0.50]
-; BTVER2-NEXT:    andq $665536, (%rsi) # imm = 0xA27C0
-; BTVER2-NEXT:    # sched: [5:1.00]
-; BTVER2-NEXT:    andq $7, %rdi # sched: [1:0.50]
-; BTVER2-NEXT:    andq $7, (%rsi) # sched: [5:1.00]
-; BTVER2-NEXT:    andq %rdx, %rdi # sched: [1:0.50]
-; BTVER2-NEXT:    andq %rdi, (%rsi) # sched: [5:1.00]
-; BTVER2-NEXT:    andq (%rsi), %rdi # sched: [4:1.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_and_64:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    andq $665536, %rax # imm = 0xA27C0
-; ZNVER1-NEXT:    # sched: [1:0.25]
-; ZNVER1-NEXT:    andq $665536, %rdi # imm = 0xA27C0
-; ZNVER1-NEXT:    # sched: [1:0.25]
-; ZNVER1-NEXT:    andq $665536, (%rsi) # imm = 0xA27C0
-; ZNVER1-NEXT:    # sched: [5:0.50]
-; ZNVER1-NEXT:    andq $7, %rdi # sched: [1:0.25]
-; ZNVER1-NEXT:    andq $7, (%rsi) # sched: [5:0.50]
-; ZNVER1-NEXT:    andq %rdx, %rdi # sched: [1:0.25]
-; ZNVER1-NEXT:    andq %rdi, (%rsi) # sched: [5:0.50]
-; ZNVER1-NEXT:    andq (%rsi), %rdi # sched: [5:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  tail call void asm "andq $3, %RAX \0A\09 andq $3, $0 \0A\09 andq $3, $2 \0A\09 andq $4, $0 \0A\09 andq $4, $2 \0A\09 andq $1, $0 \0A\09 andq $0, $2 \0A\09 andq $2, $0", "r,r,*m,i,i"(i64 %a0, i64 %a2, i64* %a1, i32 665536, i8 7) nounwind
-  ret void
-}
-
-define i16 @test_bsf16(i16 %a0, i16* %a1) optsize {
-; GENERIC-LABEL: test_bsf16:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    bsfw %di, %ax # sched: [3:1.00]
-; GENERIC-NEXT:    bsfw (%rsi), %cx # sched: [8:1.00]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    orl %ecx, %eax # sched: [1:0.33]
-; GENERIC-NEXT:    # kill: def $ax killed $ax killed $eax
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_bsf16:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    bsfw %di, %ax # sched: [16:8.00]
-; ATOM-NEXT:    bsfw (%rsi), %cx # sched: [16:8.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    orl %ecx, %eax # sched: [1:0.50]
-; ATOM-NEXT:    # kill: def $ax killed $ax killed $eax
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_bsf16:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    bsfw %di, %ax # sched: [10:10.00]
-; SLM-NEXT:    bsfw (%rsi), %cx # sched: [13:10.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    orl %ecx, %eax # sched: [1:0.50]
-; SLM-NEXT:    # kill: def $ax killed $ax killed $eax
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_bsf16:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    bsfw %di, %ax # sched: [3:1.00]
-; SANDY-NEXT:    bsfw (%rsi), %cx # sched: [8:1.00]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    orl %ecx, %eax # sched: [1:0.33]
-; SANDY-NEXT:    # kill: def $ax killed $ax killed $eax
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_bsf16:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    bsfw %di, %ax # sched: [3:1.00]
-; HASWELL-NEXT:    bsfw (%rsi), %cx # sched: [8:1.00]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    orl %ecx, %eax # sched: [1:0.25]
-; HASWELL-NEXT:    # kill: def $ax killed $ax killed $eax
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_bsf16:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    bsfw %di, %ax # sched: [3:1.00]
-; BROADWELL-NEXT:    bsfw (%rsi), %cx # sched: [8:1.00]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    orl %ecx, %eax # sched: [1:0.25]
-; BROADWELL-NEXT:    # kill: def $ax killed $ax killed $eax
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_bsf16:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    bsfw %di, %ax # sched: [3:1.00]
-; SKYLAKE-NEXT:    bsfw (%rsi), %cx # sched: [8:1.00]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    orl %ecx, %eax # sched: [1:0.25]
-; SKYLAKE-NEXT:    # kill: def $ax killed $ax killed $eax
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_bsf16:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    bsfw %di, %ax # sched: [3:1.00]
-; SKX-NEXT:    bsfw (%rsi), %cx # sched: [8:1.00]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    orl %ecx, %eax # sched: [1:0.25]
-; SKX-NEXT:    # kill: def $ax killed $ax killed $eax
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_bsf16:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    bsfw %di, %ax # sched: [3:2.00]
-; BDVER2-NEXT:    bsfw (%rsi), %cx # sched: [7:2.00]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    orl %ecx, %eax # sched: [1:0.50]
-; BDVER2-NEXT:    # kill: def $ax killed $ax killed $eax
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_bsf16:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    bsfw %di, %ax # sched: [4:4.00]
-; BTVER2-NEXT:    bsfw (%rsi), %cx # sched: [7:4.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    orl %ecx, %eax # sched: [1:0.50]
-; BTVER2-NEXT:    # kill: def $ax killed $ax killed $eax
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_bsf16:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    bsfw %di, %ax # sched: [3:0.25]
-; ZNVER1-NEXT:    bsfw (%rsi), %cx # sched: [7:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    orl %ecx, %eax # sched: [1:0.25]
-; ZNVER1-NEXT:    # kill: def $ax killed $ax killed $eax
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call { i16, i16 } asm sideeffect "bsf $2, $0 \0A\09 bsf $3, $1", "=r,=r,r,*m,~{dirflag},~{fpsr},~{flags}"(i16 %a0, i16* %a1)
-  %2 = extractvalue { i16, i16 } %1, 0
-  %3 = extractvalue { i16, i16 } %1, 1
-  %4 = or i16 %2, %3
-  ret i16 %4
-}
-define i32 @test_bsf32(i32 %a0, i32* %a1) optsize {
-; GENERIC-LABEL: test_bsf32:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    bsfl %edi, %eax # sched: [3:1.00]
-; GENERIC-NEXT:    bsfl (%rsi), %ecx # sched: [8:1.00]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    orl %ecx, %eax # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_bsf32:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    bsfl %edi, %eax # sched: [16:8.00]
-; ATOM-NEXT:    bsfl (%rsi), %ecx # sched: [16:8.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    orl %ecx, %eax # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_bsf32:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    bsfl %edi, %eax # sched: [10:10.00]
-; SLM-NEXT:    bsfl (%rsi), %ecx # sched: [13:10.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    orl %ecx, %eax # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_bsf32:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    bsfl %edi, %eax # sched: [3:1.00]
-; SANDY-NEXT:    bsfl (%rsi), %ecx # sched: [8:1.00]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    orl %ecx, %eax # sched: [1:0.33]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_bsf32:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    bsfl %edi, %eax # sched: [3:1.00]
-; HASWELL-NEXT:    bsfl (%rsi), %ecx # sched: [8:1.00]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    orl %ecx, %eax # sched: [1:0.25]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_bsf32:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    bsfl %edi, %eax # sched: [3:1.00]
-; BROADWELL-NEXT:    bsfl (%rsi), %ecx # sched: [8:1.00]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    orl %ecx, %eax # sched: [1:0.25]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_bsf32:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    bsfl %edi, %eax # sched: [3:1.00]
-; SKYLAKE-NEXT:    bsfl (%rsi), %ecx # sched: [8:1.00]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    orl %ecx, %eax # sched: [1:0.25]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_bsf32:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    bsfl %edi, %eax # sched: [3:1.00]
-; SKX-NEXT:    bsfl (%rsi), %ecx # sched: [8:1.00]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    orl %ecx, %eax # sched: [1:0.25]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_bsf32:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    bsfl %edi, %eax # sched: [3:2.00]
-; BDVER2-NEXT:    bsfl (%rsi), %ecx # sched: [7:2.00]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    orl %ecx, %eax # sched: [1:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_bsf32:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    bsfl %edi, %eax # sched: [4:4.00]
-; BTVER2-NEXT:    bsfl (%rsi), %ecx # sched: [7:4.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    orl %ecx, %eax # sched: [1:0.50]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_bsf32:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    bsfl %edi, %eax # sched: [3:0.25]
-; ZNVER1-NEXT:    bsfl (%rsi), %ecx # sched: [7:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    orl %ecx, %eax # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call { i32, i32 } asm sideeffect "bsf $2, $0 \0A\09 bsf $3, $1", "=r,=r,r,*m,~{dirflag},~{fpsr},~{flags}"(i32 %a0, i32* %a1)
-  %2 = extractvalue { i32, i32 } %1, 0
-  %3 = extractvalue { i32, i32 } %1, 1
-  %4 = or i32 %2, %3
-  ret i32 %4
-}
-define i64 @test_bsf64(i64 %a0, i64* %a1) optsize {
-; GENERIC-LABEL: test_bsf64:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    bsfq %rdi, %rax # sched: [3:1.00]
-; GENERIC-NEXT:    bsfq (%rsi), %rcx # sched: [8:1.00]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    orq %rcx, %rax # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_bsf64:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    bsfq %rdi, %rax # sched: [16:8.00]
-; ATOM-NEXT:    bsfq (%rsi), %rcx # sched: [16:8.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    orq %rcx, %rax # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_bsf64:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    bsfq %rdi, %rax # sched: [10:10.00]
-; SLM-NEXT:    bsfq (%rsi), %rcx # sched: [13:10.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    orq %rcx, %rax # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_bsf64:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    bsfq %rdi, %rax # sched: [3:1.00]
-; SANDY-NEXT:    bsfq (%rsi), %rcx # sched: [8:1.00]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    orq %rcx, %rax # sched: [1:0.33]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_bsf64:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    bsfq %rdi, %rax # sched: [3:1.00]
-; HASWELL-NEXT:    bsfq (%rsi), %rcx # sched: [8:1.00]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    orq %rcx, %rax # sched: [1:0.25]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_bsf64:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    bsfq %rdi, %rax # sched: [3:1.00]
-; BROADWELL-NEXT:    bsfq (%rsi), %rcx # sched: [8:1.00]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    orq %rcx, %rax # sched: [1:0.25]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_bsf64:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    bsfq %rdi, %rax # sched: [3:1.00]
-; SKYLAKE-NEXT:    bsfq (%rsi), %rcx # sched: [8:1.00]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    orq %rcx, %rax # sched: [1:0.25]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_bsf64:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    bsfq %rdi, %rax # sched: [3:1.00]
-; SKX-NEXT:    bsfq (%rsi), %rcx # sched: [8:1.00]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    orq %rcx, %rax # sched: [1:0.25]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_bsf64:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    bsfq %rdi, %rax # sched: [3:2.00]
-; BDVER2-NEXT:    bsfq (%rsi), %rcx # sched: [7:2.00]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    orq %rcx, %rax # sched: [1:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_bsf64:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    bsfq %rdi, %rax # sched: [4:4.00]
-; BTVER2-NEXT:    bsfq (%rsi), %rcx # sched: [7:4.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    orq %rcx, %rax # sched: [1:0.50]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_bsf64:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    bsfq %rdi, %rax # sched: [3:0.25]
-; ZNVER1-NEXT:    bsfq (%rsi), %rcx # sched: [7:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    orq %rcx, %rax # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call { i64, i64 } asm sideeffect "bsf $2, $0 \0A\09 bsf $3, $1", "=r,=r,r,*m,~{dirflag},~{fpsr},~{flags}"(i64 %a0, i64* %a1)
-  %2 = extractvalue { i64, i64 } %1, 0
-  %3 = extractvalue { i64, i64 } %1, 1
-  %4 = or i64 %2, %3
-  ret i64 %4
-}
-
-define i16 @test_bsr16(i16 %a0, i16* %a1) optsize {
-; GENERIC-LABEL: test_bsr16:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    bsrw %di, %ax # sched: [3:1.00]
-; GENERIC-NEXT:    bsrw (%rsi), %cx # sched: [8:1.00]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    orl %ecx, %eax # sched: [1:0.33]
-; GENERIC-NEXT:    # kill: def $ax killed $ax killed $eax
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_bsr16:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    bsrw %di, %ax # sched: [16:8.00]
-; ATOM-NEXT:    bsrw (%rsi), %cx # sched: [16:8.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    orl %ecx, %eax # sched: [1:0.50]
-; ATOM-NEXT:    # kill: def $ax killed $ax killed $eax
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_bsr16:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    bsrw %di, %ax # sched: [10:10.00]
-; SLM-NEXT:    bsrw (%rsi), %cx # sched: [13:10.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    orl %ecx, %eax # sched: [1:0.50]
-; SLM-NEXT:    # kill: def $ax killed $ax killed $eax
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_bsr16:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    bsrw %di, %ax # sched: [3:1.00]
-; SANDY-NEXT:    bsrw (%rsi), %cx # sched: [8:1.00]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    orl %ecx, %eax # sched: [1:0.33]
-; SANDY-NEXT:    # kill: def $ax killed $ax killed $eax
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_bsr16:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    bsrw %di, %ax # sched: [3:1.00]
-; HASWELL-NEXT:    bsrw (%rsi), %cx # sched: [8:1.00]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    orl %ecx, %eax # sched: [1:0.25]
-; HASWELL-NEXT:    # kill: def $ax killed $ax killed $eax
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_bsr16:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    bsrw %di, %ax # sched: [3:1.00]
-; BROADWELL-NEXT:    bsrw (%rsi), %cx # sched: [8:1.00]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    orl %ecx, %eax # sched: [1:0.25]
-; BROADWELL-NEXT:    # kill: def $ax killed $ax killed $eax
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_bsr16:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    bsrw %di, %ax # sched: [3:1.00]
-; SKYLAKE-NEXT:    bsrw (%rsi), %cx # sched: [8:1.00]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    orl %ecx, %eax # sched: [1:0.25]
-; SKYLAKE-NEXT:    # kill: def $ax killed $ax killed $eax
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_bsr16:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    bsrw %di, %ax # sched: [3:1.00]
-; SKX-NEXT:    bsrw (%rsi), %cx # sched: [8:1.00]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    orl %ecx, %eax # sched: [1:0.25]
-; SKX-NEXT:    # kill: def $ax killed $ax killed $eax
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_bsr16:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    bsrw %di, %ax # sched: [4:2.00]
-; BDVER2-NEXT:    bsrw (%rsi), %cx # sched: [8:2.00]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    orl %ecx, %eax # sched: [1:0.50]
-; BDVER2-NEXT:    # kill: def $ax killed $ax killed $eax
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_bsr16:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    bsrw %di, %ax # sched: [5:4.00]
-; BTVER2-NEXT:    bsrw (%rsi), %cx # sched: [8:4.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    orl %ecx, %eax # sched: [1:0.50]
-; BTVER2-NEXT:    # kill: def $ax killed $ax killed $eax
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_bsr16:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    bsrw %di, %ax # sched: [3:0.25]
-; ZNVER1-NEXT:    bsrw (%rsi), %cx # sched: [7:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    orl %ecx, %eax # sched: [1:0.25]
-; ZNVER1-NEXT:    # kill: def $ax killed $ax killed $eax
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call { i16, i16 } asm sideeffect "bsr $2, $0 \0A\09 bsr $3, $1", "=r,=r,r,*m,~{dirflag},~{fpsr},~{flags}"(i16 %a0, i16* %a1)
-  %2 = extractvalue { i16, i16 } %1, 0
-  %3 = extractvalue { i16, i16 } %1, 1
-  %4 = or i16 %2, %3
-  ret i16 %4
-}
-define i32 @test_bsr32(i32 %a0, i32* %a1) optsize {
-; GENERIC-LABEL: test_bsr32:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    bsrl %edi, %eax # sched: [3:1.00]
-; GENERIC-NEXT:    bsrl (%rsi), %ecx # sched: [8:1.00]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    orl %ecx, %eax # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_bsr32:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    bsrl %edi, %eax # sched: [16:8.00]
-; ATOM-NEXT:    bsrl (%rsi), %ecx # sched: [16:8.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    orl %ecx, %eax # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_bsr32:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    bsrl %edi, %eax # sched: [10:10.00]
-; SLM-NEXT:    bsrl (%rsi), %ecx # sched: [13:10.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    orl %ecx, %eax # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_bsr32:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    bsrl %edi, %eax # sched: [3:1.00]
-; SANDY-NEXT:    bsrl (%rsi), %ecx # sched: [8:1.00]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    orl %ecx, %eax # sched: [1:0.33]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_bsr32:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    bsrl %edi, %eax # sched: [3:1.00]
-; HASWELL-NEXT:    bsrl (%rsi), %ecx # sched: [8:1.00]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    orl %ecx, %eax # sched: [1:0.25]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_bsr32:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    bsrl %edi, %eax # sched: [3:1.00]
-; BROADWELL-NEXT:    bsrl (%rsi), %ecx # sched: [8:1.00]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    orl %ecx, %eax # sched: [1:0.25]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_bsr32:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    bsrl %edi, %eax # sched: [3:1.00]
-; SKYLAKE-NEXT:    bsrl (%rsi), %ecx # sched: [8:1.00]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    orl %ecx, %eax # sched: [1:0.25]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_bsr32:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    bsrl %edi, %eax # sched: [3:1.00]
-; SKX-NEXT:    bsrl (%rsi), %ecx # sched: [8:1.00]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    orl %ecx, %eax # sched: [1:0.25]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_bsr32:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    bsrl %edi, %eax # sched: [4:2.00]
-; BDVER2-NEXT:    bsrl (%rsi), %ecx # sched: [8:2.00]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    orl %ecx, %eax # sched: [1:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_bsr32:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    bsrl %edi, %eax # sched: [5:4.00]
-; BTVER2-NEXT:    bsrl (%rsi), %ecx # sched: [8:4.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    orl %ecx, %eax # sched: [1:0.50]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_bsr32:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    bsrl %edi, %eax # sched: [3:0.25]
-; ZNVER1-NEXT:    bsrl (%rsi), %ecx # sched: [7:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    orl %ecx, %eax # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call { i32, i32 } asm sideeffect "bsr $2, $0 \0A\09 bsr $3, $1", "=r,=r,r,*m,~{dirflag},~{fpsr},~{flags}"(i32 %a0, i32* %a1)
-  %2 = extractvalue { i32, i32 } %1, 0
-  %3 = extractvalue { i32, i32 } %1, 1
-  %4 = or i32 %2, %3
-  ret i32 %4
-}
-define i64 @test_bsr64(i64 %a0, i64* %a1) optsize {
-; GENERIC-LABEL: test_bsr64:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    bsrq %rdi, %rax # sched: [3:1.00]
-; GENERIC-NEXT:    bsrq (%rsi), %rcx # sched: [8:1.00]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    orq %rcx, %rax # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_bsr64:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    bsrq %rdi, %rax # sched: [16:8.00]
-; ATOM-NEXT:    bsrq (%rsi), %rcx # sched: [16:8.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    orq %rcx, %rax # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_bsr64:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    bsrq %rdi, %rax # sched: [10:10.00]
-; SLM-NEXT:    bsrq (%rsi), %rcx # sched: [13:10.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    orq %rcx, %rax # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_bsr64:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    bsrq %rdi, %rax # sched: [3:1.00]
-; SANDY-NEXT:    bsrq (%rsi), %rcx # sched: [8:1.00]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    orq %rcx, %rax # sched: [1:0.33]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_bsr64:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    bsrq %rdi, %rax # sched: [3:1.00]
-; HASWELL-NEXT:    bsrq (%rsi), %rcx # sched: [8:1.00]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    orq %rcx, %rax # sched: [1:0.25]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_bsr64:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    bsrq %rdi, %rax # sched: [3:1.00]
-; BROADWELL-NEXT:    bsrq (%rsi), %rcx # sched: [8:1.00]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    orq %rcx, %rax # sched: [1:0.25]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_bsr64:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    bsrq %rdi, %rax # sched: [3:1.00]
-; SKYLAKE-NEXT:    bsrq (%rsi), %rcx # sched: [8:1.00]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    orq %rcx, %rax # sched: [1:0.25]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_bsr64:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    bsrq %rdi, %rax # sched: [3:1.00]
-; SKX-NEXT:    bsrq (%rsi), %rcx # sched: [8:1.00]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    orq %rcx, %rax # sched: [1:0.25]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_bsr64:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    bsrq %rdi, %rax # sched: [4:2.00]
-; BDVER2-NEXT:    bsrq (%rsi), %rcx # sched: [8:2.00]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    orq %rcx, %rax # sched: [1:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_bsr64:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    bsrq %rdi, %rax # sched: [5:4.00]
-; BTVER2-NEXT:    bsrq (%rsi), %rcx # sched: [8:4.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    orq %rcx, %rax # sched: [1:0.50]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_bsr64:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    bsrq %rdi, %rax # sched: [3:0.25]
-; ZNVER1-NEXT:    bsrq (%rsi), %rcx # sched: [7:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    orq %rcx, %rax # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call { i64, i64 } asm sideeffect "bsr $2, $0 \0A\09 bsr $3, $1", "=r,=r,r,*m,~{dirflag},~{fpsr},~{flags}"(i64 %a0, i64* %a1)
-  %2 = extractvalue { i64, i64 } %1, 0
-  %3 = extractvalue { i64, i64 } %1, 1
-  %4 = or i64 %2, %3
-  ret i64 %4
-}
-
-define i32 @test_bswap32(i32 %a0) optsize {
-; GENERIC-LABEL: test_bswap32:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    movl %edi, %eax # sched: [1:0.33]
-; GENERIC-NEXT:    bswapl %eax # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_bswap32:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    movl %edi, %eax # sched: [1:0.50]
-; ATOM-NEXT:    bswapl %eax # sched: [1:1.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_bswap32:
-; SLM:       # %bb.0:
-; SLM-NEXT:    movl %edi, %eax # sched: [1:0.50]
-; SLM-NEXT:    bswapl %eax # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_bswap32:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    movl %edi, %eax # sched: [1:0.33]
-; SANDY-NEXT:    bswapl %eax # sched: [1:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_bswap32:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    movl %edi, %eax # sched: [1:0.25]
-; HASWELL-NEXT:    bswapl %eax # sched: [1:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_bswap32:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    movl %edi, %eax # sched: [1:0.25]
-; BROADWELL-NEXT:    bswapl %eax # sched: [1:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_bswap32:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    movl %edi, %eax # sched: [1:0.25]
-; SKYLAKE-NEXT:    bswapl %eax # sched: [1:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_bswap32:
-; SKX:       # %bb.0:
-; SKX-NEXT:    movl %edi, %eax # sched: [1:0.25]
-; SKX-NEXT:    bswapl %eax # sched: [1:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_bswap32:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    movl %edi, %eax # sched: [1:0.50]
-; BDVER2-NEXT:    bswapl %eax # sched: [1:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_bswap32:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    movl %edi, %eax # sched: [1:0.50]
-; BTVER2-NEXT:    bswapl %eax # sched: [1:0.50]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_bswap32:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    movl %edi, %eax # sched: [1:0.25]
-; ZNVER1-NEXT:    bswapl %eax # sched: [1:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = tail call i32 asm "bswap $0", "=r,0"(i32 %a0) nounwind
-  ret i32 %1
-}
-define i64 @test_bswap64(i64 %a0) optsize {
-; GENERIC-LABEL: test_bswap64:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    movq %rdi, %rax # sched: [1:0.33]
-; GENERIC-NEXT:    bswapq %rax # sched: [2:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_bswap64:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    movq %rdi, %rax # sched: [1:0.50]
-; ATOM-NEXT:    bswapq %rax # sched: [1:1.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_bswap64:
-; SLM:       # %bb.0:
-; SLM-NEXT:    movq %rdi, %rax # sched: [1:0.50]
-; SLM-NEXT:    bswapq %rax # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_bswap64:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    movq %rdi, %rax # sched: [1:0.33]
-; SANDY-NEXT:    bswapq %rax # sched: [2:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_bswap64:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    movq %rdi, %rax # sched: [1:0.25]
-; HASWELL-NEXT:    bswapq %rax # sched: [2:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_bswap64:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    movq %rdi, %rax # sched: [1:0.25]
-; BROADWELL-NEXT:    bswapq %rax # sched: [2:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_bswap64:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    movq %rdi, %rax # sched: [1:0.25]
-; SKYLAKE-NEXT:    bswapq %rax # sched: [2:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_bswap64:
-; SKX:       # %bb.0:
-; SKX-NEXT:    movq %rdi, %rax # sched: [1:0.25]
-; SKX-NEXT:    bswapq %rax # sched: [2:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_bswap64:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    movq %rdi, %rax # sched: [1:0.50]
-; BDVER2-NEXT:    bswapq %rax # sched: [1:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_bswap64:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    movq %rdi, %rax # sched: [1:0.50]
-; BTVER2-NEXT:    bswapq %rax # sched: [1:0.50]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_bswap64:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    movq %rdi, %rax # sched: [1:0.25]
-; ZNVER1-NEXT:    bswapq %rax # sched: [1:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = tail call i64 asm "bswap $0", "=r,0"(i64 %a0) nounwind
-  ret i64 %1
-}
-
-define void @test_bt_btc_btr_bts_16(i16 %a0, i16 %a1, i16 *%a2) optsize {
-; GENERIC-LABEL: test_bt_btc_btr_bts_16:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    btw %si, %di # sched: [1:0.50]
-; GENERIC-NEXT:    btcw %si, %di # sched: [1:0.50]
-; GENERIC-NEXT:    btrw %si, %di # sched: [1:0.50]
-; GENERIC-NEXT:    btsw %si, %di # sched: [1:0.50]
-; GENERIC-NEXT:    btw %si, (%rdx) # sched: [9:1.00]
-; GENERIC-NEXT:    btcw %si, (%rdx) # sched: [9:1.00]
-; GENERIC-NEXT:    btrw %si, (%rdx) # sched: [9:1.00]
-; GENERIC-NEXT:    btsw %si, (%rdx) # sched: [9:1.00]
-; GENERIC-NEXT:    btw $7, %di # sched: [1:0.50]
-; GENERIC-NEXT:    btcw $7, %di # sched: [1:0.50]
-; GENERIC-NEXT:    btrw $7, %di # sched: [1:0.50]
-; GENERIC-NEXT:    btsw $7, %di # sched: [1:0.50]
-; GENERIC-NEXT:    btw $7, (%rdx) # sched: [6:0.50]
-; GENERIC-NEXT:    btcw $7, (%rdx) # sched: [7:1.00]
-; GENERIC-NEXT:    btrw $7, (%rdx) # sched: [7:1.00]
-; GENERIC-NEXT:    btsw $7, (%rdx) # sched: [7:1.00]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_bt_btc_btr_bts_16:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    btw %si, %di # sched: [1:1.00]
-; ATOM-NEXT:    btcw %si, %di # sched: [1:1.00]
-; ATOM-NEXT:    btrw %si, %di # sched: [1:1.00]
-; ATOM-NEXT:    btsw %si, %di # sched: [1:1.00]
-; ATOM-NEXT:    btw %si, (%rdx) # sched: [9:4.50]
-; ATOM-NEXT:    btcw %si, (%rdx) # sched: [11:5.50]
-; ATOM-NEXT:    btrw %si, (%rdx) # sched: [11:5.50]
-; ATOM-NEXT:    btsw %si, (%rdx) # sched: [11:5.50]
-; ATOM-NEXT:    btw $7, %di # sched: [1:1.00]
-; ATOM-NEXT:    btcw $7, %di # sched: [1:1.00]
-; ATOM-NEXT:    btrw $7, %di # sched: [1:1.00]
-; ATOM-NEXT:    btsw $7, %di # sched: [1:1.00]
-; ATOM-NEXT:    btw $7, (%rdx) # sched: [1:1.00]
-; ATOM-NEXT:    btcw $7, (%rdx) # sched: [2:1.00]
-; ATOM-NEXT:    btrw $7, (%rdx) # sched: [2:1.00]
-; ATOM-NEXT:    btsw $7, (%rdx) # sched: [2:1.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_bt_btc_btr_bts_16:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    btw %si, %di # sched: [1:0.50]
-; SLM-NEXT:    btcw %si, %di # sched: [1:0.50]
-; SLM-NEXT:    btrw %si, %di # sched: [1:0.50]
-; SLM-NEXT:    btsw %si, %di # sched: [1:0.50]
-; SLM-NEXT:    btw %si, (%rdx) # sched: [4:1.00]
-; SLM-NEXT:    btcw %si, (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    btrw %si, (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    btsw %si, (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    btw $7, %di # sched: [1:0.50]
-; SLM-NEXT:    btcw $7, %di # sched: [1:0.50]
-; SLM-NEXT:    btrw $7, %di # sched: [1:0.50]
-; SLM-NEXT:    btsw $7, %di # sched: [1:0.50]
-; SLM-NEXT:    btw $7, (%rdx) # sched: [4:1.00]
-; SLM-NEXT:    btcw $7, (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    btrw $7, (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    btsw $7, (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_bt_btc_btr_bts_16:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    btw %si, %di # sched: [1:0.50]
-; SANDY-NEXT:    btcw %si, %di # sched: [1:0.50]
-; SANDY-NEXT:    btrw %si, %di # sched: [1:0.50]
-; SANDY-NEXT:    btsw %si, %di # sched: [1:0.50]
-; SANDY-NEXT:    btw %si, (%rdx) # sched: [9:1.00]
-; SANDY-NEXT:    btcw %si, (%rdx) # sched: [9:1.00]
-; SANDY-NEXT:    btrw %si, (%rdx) # sched: [9:1.00]
-; SANDY-NEXT:    btsw %si, (%rdx) # sched: [9:1.00]
-; SANDY-NEXT:    btw $7, %di # sched: [1:0.50]
-; SANDY-NEXT:    btcw $7, %di # sched: [1:0.50]
-; SANDY-NEXT:    btrw $7, %di # sched: [1:0.50]
-; SANDY-NEXT:    btsw $7, %di # sched: [1:0.50]
-; SANDY-NEXT:    btw $7, (%rdx) # sched: [6:0.50]
-; SANDY-NEXT:    btcw $7, (%rdx) # sched: [7:1.00]
-; SANDY-NEXT:    btrw $7, (%rdx) # sched: [7:1.00]
-; SANDY-NEXT:    btsw $7, (%rdx) # sched: [7:1.00]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_bt_btc_btr_bts_16:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    btw %si, %di # sched: [1:0.50]
-; HASWELL-NEXT:    btcw %si, %di # sched: [1:0.50]
-; HASWELL-NEXT:    btrw %si, %di # sched: [1:0.50]
-; HASWELL-NEXT:    btsw %si, %di # sched: [1:0.50]
-; HASWELL-NEXT:    btw %si, (%rdx) # sched: [1:2.50]
-; HASWELL-NEXT:    btcw %si, (%rdx) # sched: [1:2.75]
-; HASWELL-NEXT:    btrw %si, (%rdx) # sched: [1:2.75]
-; HASWELL-NEXT:    btsw %si, (%rdx) # sched: [1:2.75]
-; HASWELL-NEXT:    btw $7, %di # sched: [1:0.50]
-; HASWELL-NEXT:    btcw $7, %di # sched: [1:0.50]
-; HASWELL-NEXT:    btrw $7, %di # sched: [1:0.50]
-; HASWELL-NEXT:    btsw $7, %di # sched: [1:0.50]
-; HASWELL-NEXT:    btw $7, (%rdx) # sched: [6:0.50]
-; HASWELL-NEXT:    btcw $7, (%rdx) # sched: [7:1.00]
-; HASWELL-NEXT:    btrw $7, (%rdx) # sched: [7:1.00]
-; HASWELL-NEXT:    btsw $7, (%rdx) # sched: [7:1.00]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_bt_btc_btr_bts_16:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    btw %si, %di # sched: [1:0.50]
-; BROADWELL-NEXT:    btcw %si, %di # sched: [1:0.50]
-; BROADWELL-NEXT:    btrw %si, %di # sched: [1:0.50]
-; BROADWELL-NEXT:    btsw %si, %di # sched: [1:0.50]
-; BROADWELL-NEXT:    btw %si, (%rdx) # sched: [6:0.50]
-; BROADWELL-NEXT:    btcw %si, (%rdx) # sched: [6:1.00]
-; BROADWELL-NEXT:    btrw %si, (%rdx) # sched: [6:1.00]
-; BROADWELL-NEXT:    btsw %si, (%rdx) # sched: [6:1.00]
-; BROADWELL-NEXT:    btw $7, %di # sched: [1:0.50]
-; BROADWELL-NEXT:    btcw $7, %di # sched: [1:0.50]
-; BROADWELL-NEXT:    btrw $7, %di # sched: [1:0.50]
-; BROADWELL-NEXT:    btsw $7, %di # sched: [1:0.50]
-; BROADWELL-NEXT:    btw $7, (%rdx) # sched: [6:0.50]
-; BROADWELL-NEXT:    btcw $7, (%rdx) # sched: [6:1.00]
-; BROADWELL-NEXT:    btrw $7, (%rdx) # sched: [6:1.00]
-; BROADWELL-NEXT:    btsw $7, (%rdx) # sched: [6:1.00]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_bt_btc_btr_bts_16:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    btw %si, %di # sched: [1:0.50]
-; SKYLAKE-NEXT:    btcw %si, %di # sched: [1:0.50]
-; SKYLAKE-NEXT:    btrw %si, %di # sched: [1:0.50]
-; SKYLAKE-NEXT:    btsw %si, %di # sched: [1:0.50]
-; SKYLAKE-NEXT:    btw %si, (%rdx) # sched: [6:0.50]
-; SKYLAKE-NEXT:    btcw %si, (%rdx) # sched: [6:1.00]
-; SKYLAKE-NEXT:    btrw %si, (%rdx) # sched: [6:1.00]
-; SKYLAKE-NEXT:    btsw %si, (%rdx) # sched: [6:1.00]
-; SKYLAKE-NEXT:    btw $7, %di # sched: [1:0.50]
-; SKYLAKE-NEXT:    btcw $7, %di # sched: [1:0.50]
-; SKYLAKE-NEXT:    btrw $7, %di # sched: [1:0.50]
-; SKYLAKE-NEXT:    btsw $7, %di # sched: [1:0.50]
-; SKYLAKE-NEXT:    btw $7, (%rdx) # sched: [6:0.50]
-; SKYLAKE-NEXT:    btcw $7, (%rdx) # sched: [6:1.00]
-; SKYLAKE-NEXT:    btrw $7, (%rdx) # sched: [6:1.00]
-; SKYLAKE-NEXT:    btsw $7, (%rdx) # sched: [6:1.00]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_bt_btc_btr_bts_16:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    btw %si, %di # sched: [1:0.50]
-; SKX-NEXT:    btcw %si, %di # sched: [1:0.50]
-; SKX-NEXT:    btrw %si, %di # sched: [1:0.50]
-; SKX-NEXT:    btsw %si, %di # sched: [1:0.50]
-; SKX-NEXT:    btw %si, (%rdx) # sched: [6:0.50]
-; SKX-NEXT:    btcw %si, (%rdx) # sched: [6:1.00]
-; SKX-NEXT:    btrw %si, (%rdx) # sched: [6:1.00]
-; SKX-NEXT:    btsw %si, (%rdx) # sched: [6:1.00]
-; SKX-NEXT:    btw $7, %di # sched: [1:0.50]
-; SKX-NEXT:    btcw $7, %di # sched: [1:0.50]
-; SKX-NEXT:    btrw $7, %di # sched: [1:0.50]
-; SKX-NEXT:    btsw $7, %di # sched: [1:0.50]
-; SKX-NEXT:    btw $7, (%rdx) # sched: [6:0.50]
-; SKX-NEXT:    btcw $7, (%rdx) # sched: [6:1.00]
-; SKX-NEXT:    btrw $7, (%rdx) # sched: [6:1.00]
-; SKX-NEXT:    btsw $7, (%rdx) # sched: [6:1.00]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_bt_btc_btr_bts_16:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    btw %si, %di # sched: [1:0.50]
-; BDVER2-NEXT:    btcw %si, %di # sched: [2:0.50]
-; BDVER2-NEXT:    btrw %si, %di # sched: [2:0.50]
-; BDVER2-NEXT:    btsw %si, %di # sched: [2:0.50]
-; BDVER2-NEXT:    btw %si, (%rdx) # sched: [5:0.50]
-; BDVER2-NEXT:    btcw %si, (%rdx) # sched: [7:1.00]
-; BDVER2-NEXT:    btrw %si, (%rdx) # sched: [7:1.00]
-; BDVER2-NEXT:    btsw %si, (%rdx) # sched: [7:1.00]
-; BDVER2-NEXT:    btw $7, %di # sched: [1:0.50]
-; BDVER2-NEXT:    btcw $7, %di # sched: [2:0.50]
-; BDVER2-NEXT:    btrw $7, %di # sched: [2:0.50]
-; BDVER2-NEXT:    btsw $7, %di # sched: [2:0.50]
-; BDVER2-NEXT:    btw $7, (%rdx) # sched: [5:0.50]
-; BDVER2-NEXT:    btcw $7, (%rdx) # sched: [7:1.00]
-; BDVER2-NEXT:    btrw $7, (%rdx) # sched: [7:1.00]
-; BDVER2-NEXT:    btsw $7, (%rdx) # sched: [7:1.00]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_bt_btc_btr_bts_16:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    btw %si, %di # sched: [1:0.50]
-; BTVER2-NEXT:    btcw %si, %di # sched: [1:0.50]
-; BTVER2-NEXT:    btrw %si, %di # sched: [1:0.50]
-; BTVER2-NEXT:    btsw %si, %di # sched: [1:0.50]
-; BTVER2-NEXT:    btw %si, (%rdx) # sched: [4:1.00]
-; BTVER2-NEXT:    btcw %si, (%rdx) # sched: [5:1.00]
-; BTVER2-NEXT:    btrw %si, (%rdx) # sched: [5:1.00]
-; BTVER2-NEXT:    btsw %si, (%rdx) # sched: [5:1.00]
-; BTVER2-NEXT:    btw $7, %di # sched: [1:0.50]
-; BTVER2-NEXT:    btcw $7, %di # sched: [1:0.50]
-; BTVER2-NEXT:    btrw $7, %di # sched: [1:0.50]
-; BTVER2-NEXT:    btsw $7, %di # sched: [1:0.50]
-; BTVER2-NEXT:    btw $7, (%rdx) # sched: [4:1.00]
-; BTVER2-NEXT:    btcw $7, (%rdx) # sched: [5:1.00]
-; BTVER2-NEXT:    btrw $7, (%rdx) # sched: [5:1.00]
-; BTVER2-NEXT:    btsw $7, (%rdx) # sched: [5:1.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_bt_btc_btr_bts_16:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    btw %si, %di # sched: [1:0.25]
-; ZNVER1-NEXT:    btcw %si, %di # sched: [2:0.25]
-; ZNVER1-NEXT:    btrw %si, %di # sched: [2:0.25]
-; ZNVER1-NEXT:    btsw %si, %di # sched: [2:0.25]
-; ZNVER1-NEXT:    btw %si, (%rdx) # sched: [5:0.50]
-; ZNVER1-NEXT:    btcw %si, (%rdx) # sched: [6:0.50]
-; ZNVER1-NEXT:    btrw %si, (%rdx) # sched: [6:0.50]
-; ZNVER1-NEXT:    btsw %si, (%rdx) # sched: [6:0.50]
-; ZNVER1-NEXT:    btw $7, %di # sched: [1:0.25]
-; ZNVER1-NEXT:    btcw $7, %di # sched: [2:0.25]
-; ZNVER1-NEXT:    btrw $7, %di # sched: [2:0.25]
-; ZNVER1-NEXT:    btsw $7, %di # sched: [2:0.25]
-; ZNVER1-NEXT:    btw $7, (%rdx) # sched: [5:0.50]
-; ZNVER1-NEXT:    btcw $7, (%rdx) # sched: [6:0.50]
-; ZNVER1-NEXT:    btrw $7, (%rdx) # sched: [6:0.50]
-; ZNVER1-NEXT:    btsw $7, (%rdx) # sched: [6:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  call void asm sideeffect "btw $1, $0 \0A\09  btcw $1, $0 \0A\09  btrw $1, $0 \0A\09 btsw $1, $0 \0A\09 btw $1, $2 \0A\09 btcw $1, $2 \0A\09 btrw $1, $2 \0A\09 btsw $1, $2 \0A\09 btw $3, $0 \0A\09 btcw $3, $0 \0A\09 btrw $3, $0 \0A\09 btsw $3, $0 \0A\09 btw $3, $2 \0A\09 btcw $3, $2 \0A\09 btrw $3, $2 \0A\09 btsw $3, $2", "r,r,*m,i"(i16 %a0, i16 %a1, i16 *%a2, i8 7)
-  ret void
-}
-define void @test_bt_btc_btr_bts_32(i32 %a0, i32 %a1, i32 *%a2) optsize {
-; GENERIC-LABEL: test_bt_btc_btr_bts_32:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    btl %esi, %edi # sched: [1:0.50]
-; GENERIC-NEXT:    btcl %esi, %edi # sched: [1:0.50]
-; GENERIC-NEXT:    btrl %esi, %edi # sched: [1:0.50]
-; GENERIC-NEXT:    btsl %esi, %edi # sched: [1:0.50]
-; GENERIC-NEXT:    btl %esi, (%rdx) # sched: [9:1.00]
-; GENERIC-NEXT:    btcl %esi, (%rdx) # sched: [9:1.00]
-; GENERIC-NEXT:    btrl %esi, (%rdx) # sched: [9:1.00]
-; GENERIC-NEXT:    btsl %esi, (%rdx) # sched: [9:1.00]
-; GENERIC-NEXT:    btl $7, %edi # sched: [1:0.50]
-; GENERIC-NEXT:    btcl $7, %edi # sched: [1:0.50]
-; GENERIC-NEXT:    btrl $7, %edi # sched: [1:0.50]
-; GENERIC-NEXT:    btsl $7, %edi # sched: [1:0.50]
-; GENERIC-NEXT:    btl $7, (%rdx) # sched: [6:0.50]
-; GENERIC-NEXT:    btcl $7, (%rdx) # sched: [7:1.00]
-; GENERIC-NEXT:    btrl $7, (%rdx) # sched: [7:1.00]
-; GENERIC-NEXT:    btsl $7, (%rdx) # sched: [7:1.00]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_bt_btc_btr_bts_32:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    btl %esi, %edi # sched: [1:1.00]
-; ATOM-NEXT:    btcl %esi, %edi # sched: [1:1.00]
-; ATOM-NEXT:    btrl %esi, %edi # sched: [1:1.00]
-; ATOM-NEXT:    btsl %esi, %edi # sched: [1:1.00]
-; ATOM-NEXT:    btl %esi, (%rdx) # sched: [9:4.50]
-; ATOM-NEXT:    btcl %esi, (%rdx) # sched: [11:5.50]
-; ATOM-NEXT:    btrl %esi, (%rdx) # sched: [11:5.50]
-; ATOM-NEXT:    btsl %esi, (%rdx) # sched: [11:5.50]
-; ATOM-NEXT:    btl $7, %edi # sched: [1:1.00]
-; ATOM-NEXT:    btcl $7, %edi # sched: [1:1.00]
-; ATOM-NEXT:    btrl $7, %edi # sched: [1:1.00]
-; ATOM-NEXT:    btsl $7, %edi # sched: [1:1.00]
-; ATOM-NEXT:    btl $7, (%rdx) # sched: [1:1.00]
-; ATOM-NEXT:    btcl $7, (%rdx) # sched: [2:1.00]
-; ATOM-NEXT:    btrl $7, (%rdx) # sched: [2:1.00]
-; ATOM-NEXT:    btsl $7, (%rdx) # sched: [2:1.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_bt_btc_btr_bts_32:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    btl %esi, %edi # sched: [1:0.50]
-; SLM-NEXT:    btcl %esi, %edi # sched: [1:0.50]
-; SLM-NEXT:    btrl %esi, %edi # sched: [1:0.50]
-; SLM-NEXT:    btsl %esi, %edi # sched: [1:0.50]
-; SLM-NEXT:    btl %esi, (%rdx) # sched: [4:1.00]
-; SLM-NEXT:    btcl %esi, (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    btrl %esi, (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    btsl %esi, (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    btl $7, %edi # sched: [1:0.50]
-; SLM-NEXT:    btcl $7, %edi # sched: [1:0.50]
-; SLM-NEXT:    btrl $7, %edi # sched: [1:0.50]
-; SLM-NEXT:    btsl $7, %edi # sched: [1:0.50]
-; SLM-NEXT:    btl $7, (%rdx) # sched: [4:1.00]
-; SLM-NEXT:    btcl $7, (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    btrl $7, (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    btsl $7, (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_bt_btc_btr_bts_32:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    btl %esi, %edi # sched: [1:0.50]
-; SANDY-NEXT:    btcl %esi, %edi # sched: [1:0.50]
-; SANDY-NEXT:    btrl %esi, %edi # sched: [1:0.50]
-; SANDY-NEXT:    btsl %esi, %edi # sched: [1:0.50]
-; SANDY-NEXT:    btl %esi, (%rdx) # sched: [9:1.00]
-; SANDY-NEXT:    btcl %esi, (%rdx) # sched: [9:1.00]
-; SANDY-NEXT:    btrl %esi, (%rdx) # sched: [9:1.00]
-; SANDY-NEXT:    btsl %esi, (%rdx) # sched: [9:1.00]
-; SANDY-NEXT:    btl $7, %edi # sched: [1:0.50]
-; SANDY-NEXT:    btcl $7, %edi # sched: [1:0.50]
-; SANDY-NEXT:    btrl $7, %edi # sched: [1:0.50]
-; SANDY-NEXT:    btsl $7, %edi # sched: [1:0.50]
-; SANDY-NEXT:    btl $7, (%rdx) # sched: [6:0.50]
-; SANDY-NEXT:    btcl $7, (%rdx) # sched: [7:1.00]
-; SANDY-NEXT:    btrl $7, (%rdx) # sched: [7:1.00]
-; SANDY-NEXT:    btsl $7, (%rdx) # sched: [7:1.00]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_bt_btc_btr_bts_32:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    btl %esi, %edi # sched: [1:0.50]
-; HASWELL-NEXT:    btcl %esi, %edi # sched: [1:0.50]
-; HASWELL-NEXT:    btrl %esi, %edi # sched: [1:0.50]
-; HASWELL-NEXT:    btsl %esi, %edi # sched: [1:0.50]
-; HASWELL-NEXT:    btl %esi, (%rdx) # sched: [1:2.50]
-; HASWELL-NEXT:    btcl %esi, (%rdx) # sched: [1:2.75]
-; HASWELL-NEXT:    btrl %esi, (%rdx) # sched: [1:2.75]
-; HASWELL-NEXT:    btsl %esi, (%rdx) # sched: [1:2.75]
-; HASWELL-NEXT:    btl $7, %edi # sched: [1:0.50]
-; HASWELL-NEXT:    btcl $7, %edi # sched: [1:0.50]
-; HASWELL-NEXT:    btrl $7, %edi # sched: [1:0.50]
-; HASWELL-NEXT:    btsl $7, %edi # sched: [1:0.50]
-; HASWELL-NEXT:    btl $7, (%rdx) # sched: [6:0.50]
-; HASWELL-NEXT:    btcl $7, (%rdx) # sched: [7:1.00]
-; HASWELL-NEXT:    btrl $7, (%rdx) # sched: [7:1.00]
-; HASWELL-NEXT:    btsl $7, (%rdx) # sched: [7:1.00]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_bt_btc_btr_bts_32:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    btl %esi, %edi # sched: [1:0.50]
-; BROADWELL-NEXT:    btcl %esi, %edi # sched: [1:0.50]
-; BROADWELL-NEXT:    btrl %esi, %edi # sched: [1:0.50]
-; BROADWELL-NEXT:    btsl %esi, %edi # sched: [1:0.50]
-; BROADWELL-NEXT:    btl %esi, (%rdx) # sched: [6:0.50]
-; BROADWELL-NEXT:    btcl %esi, (%rdx) # sched: [6:1.00]
-; BROADWELL-NEXT:    btrl %esi, (%rdx) # sched: [6:1.00]
-; BROADWELL-NEXT:    btsl %esi, (%rdx) # sched: [6:1.00]
-; BROADWELL-NEXT:    btl $7, %edi # sched: [1:0.50]
-; BROADWELL-NEXT:    btcl $7, %edi # sched: [1:0.50]
-; BROADWELL-NEXT:    btrl $7, %edi # sched: [1:0.50]
-; BROADWELL-NEXT:    btsl $7, %edi # sched: [1:0.50]
-; BROADWELL-NEXT:    btl $7, (%rdx) # sched: [6:0.50]
-; BROADWELL-NEXT:    btcl $7, (%rdx) # sched: [6:1.00]
-; BROADWELL-NEXT:    btrl $7, (%rdx) # sched: [6:1.00]
-; BROADWELL-NEXT:    btsl $7, (%rdx) # sched: [6:1.00]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_bt_btc_btr_bts_32:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    btl %esi, %edi # sched: [1:0.50]
-; SKYLAKE-NEXT:    btcl %esi, %edi # sched: [1:0.50]
-; SKYLAKE-NEXT:    btrl %esi, %edi # sched: [1:0.50]
-; SKYLAKE-NEXT:    btsl %esi, %edi # sched: [1:0.50]
-; SKYLAKE-NEXT:    btl %esi, (%rdx) # sched: [6:0.50]
-; SKYLAKE-NEXT:    btcl %esi, (%rdx) # sched: [6:1.00]
-; SKYLAKE-NEXT:    btrl %esi, (%rdx) # sched: [6:1.00]
-; SKYLAKE-NEXT:    btsl %esi, (%rdx) # sched: [6:1.00]
-; SKYLAKE-NEXT:    btl $7, %edi # sched: [1:0.50]
-; SKYLAKE-NEXT:    btcl $7, %edi # sched: [1:0.50]
-; SKYLAKE-NEXT:    btrl $7, %edi # sched: [1:0.50]
-; SKYLAKE-NEXT:    btsl $7, %edi # sched: [1:0.50]
-; SKYLAKE-NEXT:    btl $7, (%rdx) # sched: [6:0.50]
-; SKYLAKE-NEXT:    btcl $7, (%rdx) # sched: [6:1.00]
-; SKYLAKE-NEXT:    btrl $7, (%rdx) # sched: [6:1.00]
-; SKYLAKE-NEXT:    btsl $7, (%rdx) # sched: [6:1.00]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_bt_btc_btr_bts_32:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    btl %esi, %edi # sched: [1:0.50]
-; SKX-NEXT:    btcl %esi, %edi # sched: [1:0.50]
-; SKX-NEXT:    btrl %esi, %edi # sched: [1:0.50]
-; SKX-NEXT:    btsl %esi, %edi # sched: [1:0.50]
-; SKX-NEXT:    btl %esi, (%rdx) # sched: [6:0.50]
-; SKX-NEXT:    btcl %esi, (%rdx) # sched: [6:1.00]
-; SKX-NEXT:    btrl %esi, (%rdx) # sched: [6:1.00]
-; SKX-NEXT:    btsl %esi, (%rdx) # sched: [6:1.00]
-; SKX-NEXT:    btl $7, %edi # sched: [1:0.50]
-; SKX-NEXT:    btcl $7, %edi # sched: [1:0.50]
-; SKX-NEXT:    btrl $7, %edi # sched: [1:0.50]
-; SKX-NEXT:    btsl $7, %edi # sched: [1:0.50]
-; SKX-NEXT:    btl $7, (%rdx) # sched: [6:0.50]
-; SKX-NEXT:    btcl $7, (%rdx) # sched: [6:1.00]
-; SKX-NEXT:    btrl $7, (%rdx) # sched: [6:1.00]
-; SKX-NEXT:    btsl $7, (%rdx) # sched: [6:1.00]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_bt_btc_btr_bts_32:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    btl %esi, %edi # sched: [1:0.50]
-; BDVER2-NEXT:    btcl %esi, %edi # sched: [2:0.50]
-; BDVER2-NEXT:    btrl %esi, %edi # sched: [2:0.50]
-; BDVER2-NEXT:    btsl %esi, %edi # sched: [2:0.50]
-; BDVER2-NEXT:    btl %esi, (%rdx) # sched: [5:0.50]
-; BDVER2-NEXT:    btcl %esi, (%rdx) # sched: [7:1.00]
-; BDVER2-NEXT:    btrl %esi, (%rdx) # sched: [7:1.00]
-; BDVER2-NEXT:    btsl %esi, (%rdx) # sched: [7:1.00]
-; BDVER2-NEXT:    btl $7, %edi # sched: [1:0.50]
-; BDVER2-NEXT:    btcl $7, %edi # sched: [2:0.50]
-; BDVER2-NEXT:    btrl $7, %edi # sched: [2:0.50]
-; BDVER2-NEXT:    btsl $7, %edi # sched: [2:0.50]
-; BDVER2-NEXT:    btl $7, (%rdx) # sched: [5:0.50]
-; BDVER2-NEXT:    btcl $7, (%rdx) # sched: [7:1.00]
-; BDVER2-NEXT:    btrl $7, (%rdx) # sched: [7:1.00]
-; BDVER2-NEXT:    btsl $7, (%rdx) # sched: [7:1.00]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_bt_btc_btr_bts_32:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    btl %esi, %edi # sched: [1:0.50]
-; BTVER2-NEXT:    btcl %esi, %edi # sched: [1:0.50]
-; BTVER2-NEXT:    btrl %esi, %edi # sched: [1:0.50]
-; BTVER2-NEXT:    btsl %esi, %edi # sched: [1:0.50]
-; BTVER2-NEXT:    btl %esi, (%rdx) # sched: [4:1.00]
-; BTVER2-NEXT:    btcl %esi, (%rdx) # sched: [5:1.00]
-; BTVER2-NEXT:    btrl %esi, (%rdx) # sched: [5:1.00]
-; BTVER2-NEXT:    btsl %esi, (%rdx) # sched: [5:1.00]
-; BTVER2-NEXT:    btl $7, %edi # sched: [1:0.50]
-; BTVER2-NEXT:    btcl $7, %edi # sched: [1:0.50]
-; BTVER2-NEXT:    btrl $7, %edi # sched: [1:0.50]
-; BTVER2-NEXT:    btsl $7, %edi # sched: [1:0.50]
-; BTVER2-NEXT:    btl $7, (%rdx) # sched: [4:1.00]
-; BTVER2-NEXT:    btcl $7, (%rdx) # sched: [5:1.00]
-; BTVER2-NEXT:    btrl $7, (%rdx) # sched: [5:1.00]
-; BTVER2-NEXT:    btsl $7, (%rdx) # sched: [5:1.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_bt_btc_btr_bts_32:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    btl %esi, %edi # sched: [1:0.25]
-; ZNVER1-NEXT:    btcl %esi, %edi # sched: [2:0.25]
-; ZNVER1-NEXT:    btrl %esi, %edi # sched: [2:0.25]
-; ZNVER1-NEXT:    btsl %esi, %edi # sched: [2:0.25]
-; ZNVER1-NEXT:    btl %esi, (%rdx) # sched: [5:0.50]
-; ZNVER1-NEXT:    btcl %esi, (%rdx) # sched: [6:0.50]
-; ZNVER1-NEXT:    btrl %esi, (%rdx) # sched: [6:0.50]
-; ZNVER1-NEXT:    btsl %esi, (%rdx) # sched: [6:0.50]
-; ZNVER1-NEXT:    btl $7, %edi # sched: [1:0.25]
-; ZNVER1-NEXT:    btcl $7, %edi # sched: [2:0.25]
-; ZNVER1-NEXT:    btrl $7, %edi # sched: [2:0.25]
-; ZNVER1-NEXT:    btsl $7, %edi # sched: [2:0.25]
-; ZNVER1-NEXT:    btl $7, (%rdx) # sched: [5:0.50]
-; ZNVER1-NEXT:    btcl $7, (%rdx) # sched: [6:0.50]
-; ZNVER1-NEXT:    btrl $7, (%rdx) # sched: [6:0.50]
-; ZNVER1-NEXT:    btsl $7, (%rdx) # sched: [6:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  call void asm sideeffect "btl $1, $0 \0A\09  btcl $1, $0 \0A\09  btrl $1, $0 \0A\09 btsl $1, $0 \0A\09 btl $1, $2 \0A\09 btcl $1, $2 \0A\09 btrl $1, $2 \0A\09 btsl $1, $2 \0A\09 btl $3, $0 \0A\09 btcl $3, $0 \0A\09 btrl $3, $0 \0A\09 btsl $3, $0 \0A\09 btl $3, $2 \0A\09 btcl $3, $2 \0A\09 btrl $3, $2 \0A\09 btsl $3, $2", "r,r,*m,i"(i32 %a0, i32 %a1, i32 *%a2, i8 7)
-  ret void
-}
-define void @test_bt_btc_btr_bts_64(i64 %a0, i64 %a1, i64 *%a2) optsize {
-; GENERIC-LABEL: test_bt_btc_btr_bts_64:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    btq %rsi, %rdi # sched: [1:0.50]
-; GENERIC-NEXT:    btcq %rsi, %rdi # sched: [1:0.50]
-; GENERIC-NEXT:    btrq %rsi, %rdi # sched: [1:0.50]
-; GENERIC-NEXT:    btsq %rsi, %rdi # sched: [1:0.50]
-; GENERIC-NEXT:    btq %rsi, (%rdx) # sched: [9:1.00]
-; GENERIC-NEXT:    btcq %rsi, (%rdx) # sched: [9:1.00]
-; GENERIC-NEXT:    btrq %rsi, (%rdx) # sched: [9:1.00]
-; GENERIC-NEXT:    btsq %rsi, (%rdx) # sched: [9:1.00]
-; GENERIC-NEXT:    btq $7, %rdi # sched: [1:0.50]
-; GENERIC-NEXT:    btcq $7, %rdi # sched: [1:0.50]
-; GENERIC-NEXT:    btrq $7, %rdi # sched: [1:0.50]
-; GENERIC-NEXT:    btsq $7, %rdi # sched: [1:0.50]
-; GENERIC-NEXT:    btq $7, (%rdx) # sched: [6:0.50]
-; GENERIC-NEXT:    btcq $7, (%rdx) # sched: [7:1.00]
-; GENERIC-NEXT:    btrq $7, (%rdx) # sched: [7:1.00]
-; GENERIC-NEXT:    btsq $7, (%rdx) # sched: [7:1.00]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_bt_btc_btr_bts_64:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    btq %rsi, %rdi # sched: [1:1.00]
-; ATOM-NEXT:    btcq %rsi, %rdi # sched: [1:1.00]
-; ATOM-NEXT:    btrq %rsi, %rdi # sched: [1:1.00]
-; ATOM-NEXT:    btsq %rsi, %rdi # sched: [1:1.00]
-; ATOM-NEXT:    btq %rsi, (%rdx) # sched: [9:4.50]
-; ATOM-NEXT:    btcq %rsi, (%rdx) # sched: [11:5.50]
-; ATOM-NEXT:    btrq %rsi, (%rdx) # sched: [11:5.50]
-; ATOM-NEXT:    btsq %rsi, (%rdx) # sched: [11:5.50]
-; ATOM-NEXT:    btq $7, %rdi # sched: [1:1.00]
-; ATOM-NEXT:    btcq $7, %rdi # sched: [1:1.00]
-; ATOM-NEXT:    btrq $7, %rdi # sched: [1:1.00]
-; ATOM-NEXT:    btsq $7, %rdi # sched: [1:1.00]
-; ATOM-NEXT:    btq $7, (%rdx) # sched: [1:1.00]
-; ATOM-NEXT:    btcq $7, (%rdx) # sched: [2:1.00]
-; ATOM-NEXT:    btrq $7, (%rdx) # sched: [2:1.00]
-; ATOM-NEXT:    btsq $7, (%rdx) # sched: [2:1.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_bt_btc_btr_bts_64:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    btq %rsi, %rdi # sched: [1:0.50]
-; SLM-NEXT:    btcq %rsi, %rdi # sched: [1:0.50]
-; SLM-NEXT:    btrq %rsi, %rdi # sched: [1:0.50]
-; SLM-NEXT:    btsq %rsi, %rdi # sched: [1:0.50]
-; SLM-NEXT:    btq %rsi, (%rdx) # sched: [4:1.00]
-; SLM-NEXT:    btcq %rsi, (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    btrq %rsi, (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    btsq %rsi, (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    btq $7, %rdi # sched: [1:0.50]
-; SLM-NEXT:    btcq $7, %rdi # sched: [1:0.50]
-; SLM-NEXT:    btrq $7, %rdi # sched: [1:0.50]
-; SLM-NEXT:    btsq $7, %rdi # sched: [1:0.50]
-; SLM-NEXT:    btq $7, (%rdx) # sched: [4:1.00]
-; SLM-NEXT:    btcq $7, (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    btrq $7, (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    btsq $7, (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_bt_btc_btr_bts_64:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    btq %rsi, %rdi # sched: [1:0.50]
-; SANDY-NEXT:    btcq %rsi, %rdi # sched: [1:0.50]
-; SANDY-NEXT:    btrq %rsi, %rdi # sched: [1:0.50]
-; SANDY-NEXT:    btsq %rsi, %rdi # sched: [1:0.50]
-; SANDY-NEXT:    btq %rsi, (%rdx) # sched: [9:1.00]
-; SANDY-NEXT:    btcq %rsi, (%rdx) # sched: [9:1.00]
-; SANDY-NEXT:    btrq %rsi, (%rdx) # sched: [9:1.00]
-; SANDY-NEXT:    btsq %rsi, (%rdx) # sched: [9:1.00]
-; SANDY-NEXT:    btq $7, %rdi # sched: [1:0.50]
-; SANDY-NEXT:    btcq $7, %rdi # sched: [1:0.50]
-; SANDY-NEXT:    btrq $7, %rdi # sched: [1:0.50]
-; SANDY-NEXT:    btsq $7, %rdi # sched: [1:0.50]
-; SANDY-NEXT:    btq $7, (%rdx) # sched: [6:0.50]
-; SANDY-NEXT:    btcq $7, (%rdx) # sched: [7:1.00]
-; SANDY-NEXT:    btrq $7, (%rdx) # sched: [7:1.00]
-; SANDY-NEXT:    btsq $7, (%rdx) # sched: [7:1.00]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_bt_btc_btr_bts_64:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    btq %rsi, %rdi # sched: [1:0.50]
-; HASWELL-NEXT:    btcq %rsi, %rdi # sched: [1:0.50]
-; HASWELL-NEXT:    btrq %rsi, %rdi # sched: [1:0.50]
-; HASWELL-NEXT:    btsq %rsi, %rdi # sched: [1:0.50]
-; HASWELL-NEXT:    btq %rsi, (%rdx) # sched: [1:2.50]
-; HASWELL-NEXT:    btcq %rsi, (%rdx) # sched: [1:2.75]
-; HASWELL-NEXT:    btrq %rsi, (%rdx) # sched: [1:2.75]
-; HASWELL-NEXT:    btsq %rsi, (%rdx) # sched: [1:2.75]
-; HASWELL-NEXT:    btq $7, %rdi # sched: [1:0.50]
-; HASWELL-NEXT:    btcq $7, %rdi # sched: [1:0.50]
-; HASWELL-NEXT:    btrq $7, %rdi # sched: [1:0.50]
-; HASWELL-NEXT:    btsq $7, %rdi # sched: [1:0.50]
-; HASWELL-NEXT:    btq $7, (%rdx) # sched: [6:0.50]
-; HASWELL-NEXT:    btcq $7, (%rdx) # sched: [7:1.00]
-; HASWELL-NEXT:    btrq $7, (%rdx) # sched: [7:1.00]
-; HASWELL-NEXT:    btsq $7, (%rdx) # sched: [7:1.00]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_bt_btc_btr_bts_64:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    btq %rsi, %rdi # sched: [1:0.50]
-; BROADWELL-NEXT:    btcq %rsi, %rdi # sched: [1:0.50]
-; BROADWELL-NEXT:    btrq %rsi, %rdi # sched: [1:0.50]
-; BROADWELL-NEXT:    btsq %rsi, %rdi # sched: [1:0.50]
-; BROADWELL-NEXT:    btq %rsi, (%rdx) # sched: [6:0.50]
-; BROADWELL-NEXT:    btcq %rsi, (%rdx) # sched: [6:1.00]
-; BROADWELL-NEXT:    btrq %rsi, (%rdx) # sched: [6:1.00]
-; BROADWELL-NEXT:    btsq %rsi, (%rdx) # sched: [6:1.00]
-; BROADWELL-NEXT:    btq $7, %rdi # sched: [1:0.50]
-; BROADWELL-NEXT:    btcq $7, %rdi # sched: [1:0.50]
-; BROADWELL-NEXT:    btrq $7, %rdi # sched: [1:0.50]
-; BROADWELL-NEXT:    btsq $7, %rdi # sched: [1:0.50]
-; BROADWELL-NEXT:    btq $7, (%rdx) # sched: [6:0.50]
-; BROADWELL-NEXT:    btcq $7, (%rdx) # sched: [6:1.00]
-; BROADWELL-NEXT:    btrq $7, (%rdx) # sched: [6:1.00]
-; BROADWELL-NEXT:    btsq $7, (%rdx) # sched: [6:1.00]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_bt_btc_btr_bts_64:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    btq %rsi, %rdi # sched: [1:0.50]
-; SKYLAKE-NEXT:    btcq %rsi, %rdi # sched: [1:0.50]
-; SKYLAKE-NEXT:    btrq %rsi, %rdi # sched: [1:0.50]
-; SKYLAKE-NEXT:    btsq %rsi, %rdi # sched: [1:0.50]
-; SKYLAKE-NEXT:    btq %rsi, (%rdx) # sched: [6:0.50]
-; SKYLAKE-NEXT:    btcq %rsi, (%rdx) # sched: [6:1.00]
-; SKYLAKE-NEXT:    btrq %rsi, (%rdx) # sched: [6:1.00]
-; SKYLAKE-NEXT:    btsq %rsi, (%rdx) # sched: [6:1.00]
-; SKYLAKE-NEXT:    btq $7, %rdi # sched: [1:0.50]
-; SKYLAKE-NEXT:    btcq $7, %rdi # sched: [1:0.50]
-; SKYLAKE-NEXT:    btrq $7, %rdi # sched: [1:0.50]
-; SKYLAKE-NEXT:    btsq $7, %rdi # sched: [1:0.50]
-; SKYLAKE-NEXT:    btq $7, (%rdx) # sched: [6:0.50]
-; SKYLAKE-NEXT:    btcq $7, (%rdx) # sched: [6:1.00]
-; SKYLAKE-NEXT:    btrq $7, (%rdx) # sched: [6:1.00]
-; SKYLAKE-NEXT:    btsq $7, (%rdx) # sched: [6:1.00]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_bt_btc_btr_bts_64:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    btq %rsi, %rdi # sched: [1:0.50]
-; SKX-NEXT:    btcq %rsi, %rdi # sched: [1:0.50]
-; SKX-NEXT:    btrq %rsi, %rdi # sched: [1:0.50]
-; SKX-NEXT:    btsq %rsi, %rdi # sched: [1:0.50]
-; SKX-NEXT:    btq %rsi, (%rdx) # sched: [6:0.50]
-; SKX-NEXT:    btcq %rsi, (%rdx) # sched: [6:1.00]
-; SKX-NEXT:    btrq %rsi, (%rdx) # sched: [6:1.00]
-; SKX-NEXT:    btsq %rsi, (%rdx) # sched: [6:1.00]
-; SKX-NEXT:    btq $7, %rdi # sched: [1:0.50]
-; SKX-NEXT:    btcq $7, %rdi # sched: [1:0.50]
-; SKX-NEXT:    btrq $7, %rdi # sched: [1:0.50]
-; SKX-NEXT:    btsq $7, %rdi # sched: [1:0.50]
-; SKX-NEXT:    btq $7, (%rdx) # sched: [6:0.50]
-; SKX-NEXT:    btcq $7, (%rdx) # sched: [6:1.00]
-; SKX-NEXT:    btrq $7, (%rdx) # sched: [6:1.00]
-; SKX-NEXT:    btsq $7, (%rdx) # sched: [6:1.00]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_bt_btc_btr_bts_64:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    btq %rsi, %rdi # sched: [1:0.50]
-; BDVER2-NEXT:    btcq %rsi, %rdi # sched: [2:0.50]
-; BDVER2-NEXT:    btrq %rsi, %rdi # sched: [2:0.50]
-; BDVER2-NEXT:    btsq %rsi, %rdi # sched: [2:0.50]
-; BDVER2-NEXT:    btq %rsi, (%rdx) # sched: [5:0.50]
-; BDVER2-NEXT:    btcq %rsi, (%rdx) # sched: [7:1.00]
-; BDVER2-NEXT:    btrq %rsi, (%rdx) # sched: [7:1.00]
-; BDVER2-NEXT:    btsq %rsi, (%rdx) # sched: [7:1.00]
-; BDVER2-NEXT:    btq $7, %rdi # sched: [1:0.50]
-; BDVER2-NEXT:    btcq $7, %rdi # sched: [2:0.50]
-; BDVER2-NEXT:    btrq $7, %rdi # sched: [2:0.50]
-; BDVER2-NEXT:    btsq $7, %rdi # sched: [2:0.50]
-; BDVER2-NEXT:    btq $7, (%rdx) # sched: [5:0.50]
-; BDVER2-NEXT:    btcq $7, (%rdx) # sched: [7:1.00]
-; BDVER2-NEXT:    btrq $7, (%rdx) # sched: [7:1.00]
-; BDVER2-NEXT:    btsq $7, (%rdx) # sched: [7:1.00]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_bt_btc_btr_bts_64:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    btq %rsi, %rdi # sched: [1:0.50]
-; BTVER2-NEXT:    btcq %rsi, %rdi # sched: [1:0.50]
-; BTVER2-NEXT:    btrq %rsi, %rdi # sched: [1:0.50]
-; BTVER2-NEXT:    btsq %rsi, %rdi # sched: [1:0.50]
-; BTVER2-NEXT:    btq %rsi, (%rdx) # sched: [4:1.00]
-; BTVER2-NEXT:    btcq %rsi, (%rdx) # sched: [5:1.00]
-; BTVER2-NEXT:    btrq %rsi, (%rdx) # sched: [5:1.00]
-; BTVER2-NEXT:    btsq %rsi, (%rdx) # sched: [5:1.00]
-; BTVER2-NEXT:    btq $7, %rdi # sched: [1:0.50]
-; BTVER2-NEXT:    btcq $7, %rdi # sched: [1:0.50]
-; BTVER2-NEXT:    btrq $7, %rdi # sched: [1:0.50]
-; BTVER2-NEXT:    btsq $7, %rdi # sched: [1:0.50]
-; BTVER2-NEXT:    btq $7, (%rdx) # sched: [4:1.00]
-; BTVER2-NEXT:    btcq $7, (%rdx) # sched: [5:1.00]
-; BTVER2-NEXT:    btrq $7, (%rdx) # sched: [5:1.00]
-; BTVER2-NEXT:    btsq $7, (%rdx) # sched: [5:1.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_bt_btc_btr_bts_64:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    btq %rsi, %rdi # sched: [1:0.25]
-; ZNVER1-NEXT:    btcq %rsi, %rdi # sched: [2:0.25]
-; ZNVER1-NEXT:    btrq %rsi, %rdi # sched: [2:0.25]
-; ZNVER1-NEXT:    btsq %rsi, %rdi # sched: [2:0.25]
-; ZNVER1-NEXT:    btq %rsi, (%rdx) # sched: [5:0.50]
-; ZNVER1-NEXT:    btcq %rsi, (%rdx) # sched: [6:0.50]
-; ZNVER1-NEXT:    btrq %rsi, (%rdx) # sched: [6:0.50]
-; ZNVER1-NEXT:    btsq %rsi, (%rdx) # sched: [6:0.50]
-; ZNVER1-NEXT:    btq $7, %rdi # sched: [1:0.25]
-; ZNVER1-NEXT:    btcq $7, %rdi # sched: [2:0.25]
-; ZNVER1-NEXT:    btrq $7, %rdi # sched: [2:0.25]
-; ZNVER1-NEXT:    btsq $7, %rdi # sched: [2:0.25]
-; ZNVER1-NEXT:    btq $7, (%rdx) # sched: [5:0.50]
-; ZNVER1-NEXT:    btcq $7, (%rdx) # sched: [6:0.50]
-; ZNVER1-NEXT:    btrq $7, (%rdx) # sched: [6:0.50]
-; ZNVER1-NEXT:    btsq $7, (%rdx) # sched: [6:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  call void asm sideeffect "btq $1, $0 \0A\09  btcq $1, $0 \0A\09  btrq $1, $0 \0A\09 btsq $1, $0 \0A\09 btq $1, $2 \0A\09 btcq $1, $2 \0A\09 btrq $1, $2 \0A\09 btsq $1, $2 \0A\09 btq $3, $0 \0A\09 btcq $3, $0 \0A\09 btrq $3, $0 \0A\09 btsq $3, $0 \0A\09 btq $3, $2 \0A\09 btcq $3, $2 \0A\09 btrq $3, $2 \0A\09 btsq $3, $2", "r,r,*m,i"(i64 %a0, i64 %a1, i64 *%a2, i8 7)
-  ret void
-}
-
-; TODO - test_call
-
-define void @test_cbw_cdq_cdqe_cqo_cwd_cwde() optsize {
-; GENERIC-LABEL: test_cbw_cdq_cdqe_cqo_cwd_cwde:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    cbtw # sched: [1:0.33]
-; GENERIC-NEXT:    cltd # sched: [1:0.50]
-; GENERIC-NEXT:    cltq # sched: [1:0.33]
-; GENERIC-NEXT:    cqto # sched: [1:0.50]
-; GENERIC-NEXT:    cwtd # sched: [2:1.00]
-; GENERIC-NEXT:    cwtl # sched: [1:0.33]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_cbw_cdq_cdqe_cqo_cwd_cwde:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    cbtw # sched: [4:2.00]
-; ATOM-NEXT:    cltd # sched: [4:2.00]
-; ATOM-NEXT:    cltq # sched: [4:2.00]
-; ATOM-NEXT:    cqto # sched: [4:2.00]
-; ATOM-NEXT:    cwtd # sched: [4:2.00]
-; ATOM-NEXT:    cwtl # sched: [4:2.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_cbw_cdq_cdqe_cqo_cwd_cwde:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    cbtw # sched: [1:0.50]
-; SLM-NEXT:    cltd # sched: [1:0.50]
-; SLM-NEXT:    cltq # sched: [1:0.50]
-; SLM-NEXT:    cqto # sched: [1:0.50]
-; SLM-NEXT:    cwtd # sched: [1:0.50]
-; SLM-NEXT:    cwtl # sched: [1:0.50]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_cbw_cdq_cdqe_cqo_cwd_cwde:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    cbtw # sched: [1:0.33]
-; SANDY-NEXT:    cltd # sched: [1:0.50]
-; SANDY-NEXT:    cltq # sched: [1:0.33]
-; SANDY-NEXT:    cqto # sched: [1:0.50]
-; SANDY-NEXT:    cwtd # sched: [2:1.00]
-; SANDY-NEXT:    cwtl # sched: [1:0.33]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_cbw_cdq_cdqe_cqo_cwd_cwde:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    cbtw # sched: [1:0.25]
-; HASWELL-NEXT:    cltd # sched: [1:0.50]
-; HASWELL-NEXT:    cltq # sched: [1:0.25]
-; HASWELL-NEXT:    cqto # sched: [1:0.50]
-; HASWELL-NEXT:    cwtd # sched: [2:0.50]
-; HASWELL-NEXT:    cwtl # sched: [1:0.25]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_cbw_cdq_cdqe_cqo_cwd_cwde:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    cbtw # sched: [1:0.25]
-; BROADWELL-NEXT:    cltd # sched: [1:0.50]
-; BROADWELL-NEXT:    cltq # sched: [1:0.25]
-; BROADWELL-NEXT:    cqto # sched: [1:0.50]
-; BROADWELL-NEXT:    cwtd # sched: [2:0.50]
-; BROADWELL-NEXT:    cwtl # sched: [1:0.25]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_cbw_cdq_cdqe_cqo_cwd_cwde:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    cbtw # sched: [1:0.25]
-; SKYLAKE-NEXT:    cltd # sched: [1:0.50]
-; SKYLAKE-NEXT:    cltq # sched: [1:0.25]
-; SKYLAKE-NEXT:    cqto # sched: [1:0.50]
-; SKYLAKE-NEXT:    cwtd # sched: [2:0.50]
-; SKYLAKE-NEXT:    cwtl # sched: [1:0.25]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_cbw_cdq_cdqe_cqo_cwd_cwde:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    cbtw # sched: [1:0.25]
-; SKX-NEXT:    cltd # sched: [1:0.50]
-; SKX-NEXT:    cltq # sched: [1:0.25]
-; SKX-NEXT:    cqto # sched: [1:0.50]
-; SKX-NEXT:    cwtd # sched: [2:0.50]
-; SKX-NEXT:    cwtl # sched: [1:0.25]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_cbw_cdq_cdqe_cqo_cwd_cwde:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    cbtw # sched: [1:0.50]
-; BDVER2-NEXT:    cltd # sched: [1:0.50]
-; BDVER2-NEXT:    cltq # sched: [1:0.50]
-; BDVER2-NEXT:    cqto # sched: [1:0.50]
-; BDVER2-NEXT:    cwtd # sched: [1:0.50]
-; BDVER2-NEXT:    cwtl # sched: [1:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_cbw_cdq_cdqe_cqo_cwd_cwde:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    cbtw # sched: [1:0.50]
-; BTVER2-NEXT:    cltd # sched: [1:0.50]
-; BTVER2-NEXT:    cltq # sched: [1:0.50]
-; BTVER2-NEXT:    cqto # sched: [1:0.50]
-; BTVER2-NEXT:    cwtd # sched: [1:0.50]
-; BTVER2-NEXT:    cwtl # sched: [1:0.50]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_cbw_cdq_cdqe_cqo_cwd_cwde:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    cbtw # sched: [1:0.25]
-; ZNVER1-NEXT:    cltd # sched: [1:0.25]
-; ZNVER1-NEXT:    cltq # sched: [1:0.25]
-; ZNVER1-NEXT:    cqto # sched: [1:0.25]
-; ZNVER1-NEXT:    cwtd # sched: [1:0.25]
-; ZNVER1-NEXT:    cwtl # sched: [1:0.25]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  tail call void asm "cbw \0A\09 cdq \0A\09 cdqe \0A\09 cqo \0A\09 cwd \0A\09 cwde", ""() nounwind
-  ret void
-}
-
-define void @test_clc_cld_cmc() optsize {
-; GENERIC-LABEL: test_clc_cld_cmc:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    clc # sched: [1:0.25]
-; GENERIC-NEXT:    cld # sched: [1:0.33]
-; GENERIC-NEXT:    cmc # sched: [1:0.33]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_clc_cld_cmc:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    clc # sched: [1:0.50]
-; ATOM-NEXT:    cld # sched: [3:1.50]
-; ATOM-NEXT:    cmc # sched: [1:0.50]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_clc_cld_cmc:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    clc # sched: [1:0.50]
-; SLM-NEXT:    cld # sched: [1:0.50]
-; SLM-NEXT:    cmc # sched: [1:0.50]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_clc_cld_cmc:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    clc # sched: [1:0.25]
-; SANDY-NEXT:    cld # sched: [1:0.33]
-; SANDY-NEXT:    cmc # sched: [1:0.33]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_clc_cld_cmc:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    clc # sched: [1:0.25]
-; HASWELL-NEXT:    cld # sched: [3:1.00]
-; HASWELL-NEXT:    cmc # sched: [1:0.25]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_clc_cld_cmc:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    clc # sched: [1:0.25]
-; BROADWELL-NEXT:    cld # sched: [3:1.00]
-; BROADWELL-NEXT:    cmc # sched: [1:0.25]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_clc_cld_cmc:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    clc # sched: [1:0.17]
-; SKYLAKE-NEXT:    cld # sched: [3:1.00]
-; SKYLAKE-NEXT:    cmc # sched: [1:0.25]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_clc_cld_cmc:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    clc # sched: [1:0.17]
-; SKX-NEXT:    cld # sched: [3:1.00]
-; SKX-NEXT:    cmc # sched: [1:0.25]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_clc_cld_cmc:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    clc # sched: [1:0.50]
-; BDVER2-NEXT:    cld # sched: [1:0.50]
-; BDVER2-NEXT:    cmc # sched: [1:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_clc_cld_cmc:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    clc # sched: [1:0.50]
-; BTVER2-NEXT:    cld # sched: [1:0.50]
-; BTVER2-NEXT:    cmc # sched: [1:0.50]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_clc_cld_cmc:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    clc # sched: [1:0.25]
-; ZNVER1-NEXT:    cld # sched: [1:0.25]
-; ZNVER1-NEXT:    cmc # sched: [1:0.25]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  tail call void asm "clc \0A\09 cld \0A\09 cmc", ""() nounwind
-  ret void
-}
-
-define void @test_cmp_8(i8 %a0, i8* %a1) optsize {
-; GENERIC-LABEL: test_cmp_8:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    cmpb $7, %al # sched: [1:0.33]
-; GENERIC-NEXT:    cmpb $7, %dil # sched: [1:0.33]
-; GENERIC-NEXT:    cmpb $7, (%rsi) # sched: [6:0.50]
-; GENERIC-NEXT:    cmpb %dil, %dil # sched: [1:0.33]
-; GENERIC-NEXT:    cmpb %dil, (%rsi) # sched: [6:0.50]
-; GENERIC-NEXT:    cmpb (%rsi), %dil # sched: [6:0.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_cmp_8:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    cmpb $7, %al # sched: [1:0.50]
-; ATOM-NEXT:    cmpb $7, %dil # sched: [1:0.50]
-; ATOM-NEXT:    cmpb $7, (%rsi) # sched: [1:1.00]
-; ATOM-NEXT:    cmpb %dil, %dil # sched: [1:0.50]
-; ATOM-NEXT:    cmpb %dil, (%rsi) # sched: [1:1.00]
-; ATOM-NEXT:    cmpb (%rsi), %dil # sched: [1:1.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_cmp_8:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    cmpb $7, %al # sched: [1:0.50]
-; SLM-NEXT:    cmpb $7, %dil # sched: [1:0.50]
-; SLM-NEXT:    cmpb $7, (%rsi) # sched: [4:1.00]
-; SLM-NEXT:    cmpb %dil, %dil # sched: [1:0.50]
-; SLM-NEXT:    cmpb %dil, (%rsi) # sched: [4:1.00]
-; SLM-NEXT:    cmpb (%rsi), %dil # sched: [4:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_cmp_8:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    cmpb $7, %al # sched: [1:0.33]
-; SANDY-NEXT:    cmpb $7, %dil # sched: [1:0.33]
-; SANDY-NEXT:    cmpb $7, (%rsi) # sched: [6:0.50]
-; SANDY-NEXT:    cmpb %dil, %dil # sched: [1:0.33]
-; SANDY-NEXT:    cmpb %dil, (%rsi) # sched: [6:0.50]
-; SANDY-NEXT:    cmpb (%rsi), %dil # sched: [6:0.50]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_cmp_8:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    cmpb $7, %al # sched: [1:0.25]
-; HASWELL-NEXT:    cmpb $7, %dil # sched: [1:0.25]
-; HASWELL-NEXT:    cmpb $7, (%rsi) # sched: [6:0.50]
-; HASWELL-NEXT:    cmpb %dil, %dil # sched: [1:0.25]
-; HASWELL-NEXT:    cmpb %dil, (%rsi) # sched: [6:0.50]
-; HASWELL-NEXT:    cmpb (%rsi), %dil # sched: [6:0.50]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_cmp_8:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    cmpb $7, %al # sched: [1:0.25]
-; BROADWELL-NEXT:    cmpb $7, %dil # sched: [1:0.25]
-; BROADWELL-NEXT:    cmpb $7, (%rsi) # sched: [6:0.50]
-; BROADWELL-NEXT:    cmpb %dil, %dil # sched: [1:0.25]
-; BROADWELL-NEXT:    cmpb %dil, (%rsi) # sched: [6:0.50]
-; BROADWELL-NEXT:    cmpb (%rsi), %dil # sched: [6:0.50]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_cmp_8:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    cmpb $7, %al # sched: [1:0.25]
-; SKYLAKE-NEXT:    cmpb $7, %dil # sched: [1:0.25]
-; SKYLAKE-NEXT:    cmpb $7, (%rsi) # sched: [6:0.50]
-; SKYLAKE-NEXT:    cmpb %dil, %dil # sched: [1:0.25]
-; SKYLAKE-NEXT:    cmpb %dil, (%rsi) # sched: [6:0.50]
-; SKYLAKE-NEXT:    cmpb (%rsi), %dil # sched: [6:0.50]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_cmp_8:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    cmpb $7, %al # sched: [1:0.25]
-; SKX-NEXT:    cmpb $7, %dil # sched: [1:0.25]
-; SKX-NEXT:    cmpb $7, (%rsi) # sched: [6:0.50]
-; SKX-NEXT:    cmpb %dil, %dil # sched: [1:0.25]
-; SKX-NEXT:    cmpb %dil, (%rsi) # sched: [6:0.50]
-; SKX-NEXT:    cmpb (%rsi), %dil # sched: [6:0.50]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_cmp_8:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    cmpb $7, %al # sched: [1:0.50]
-; BDVER2-NEXT:    cmpb $7, %dil # sched: [1:0.50]
-; BDVER2-NEXT:    cmpb $7, (%rsi) # sched: [5:0.50]
-; BDVER2-NEXT:    cmpb %dil, %dil # sched: [1:0.50]
-; BDVER2-NEXT:    cmpb %dil, (%rsi) # sched: [5:0.50]
-; BDVER2-NEXT:    cmpb (%rsi), %dil # sched: [5:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_cmp_8:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    cmpb $7, %al # sched: [1:0.50]
-; BTVER2-NEXT:    cmpb $7, %dil # sched: [1:0.50]
-; BTVER2-NEXT:    cmpb $7, (%rsi) # sched: [4:1.00]
-; BTVER2-NEXT:    cmpb %dil, %dil # sched: [1:0.50]
-; BTVER2-NEXT:    cmpb %dil, (%rsi) # sched: [4:1.00]
-; BTVER2-NEXT:    cmpb (%rsi), %dil # sched: [4:1.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_cmp_8:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    cmpb $7, %al # sched: [1:0.25]
-; ZNVER1-NEXT:    cmpb $7, %dil # sched: [1:0.25]
-; ZNVER1-NEXT:    cmpb $7, (%rsi) # sched: [5:0.50]
-; ZNVER1-NEXT:    cmpb %dil, %dil # sched: [1:0.25]
-; ZNVER1-NEXT:    cmpb %dil, (%rsi) # sched: [5:0.50]
-; ZNVER1-NEXT:    cmpb (%rsi), %dil # sched: [5:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  tail call void asm "cmpb $2, %AL \0A\09 cmpb $2, $0 \0A\09 cmpb $2, $1 \0A\09 cmpb $0, $0 \0A\09 cmpb $0, $1 \0A\09 cmpb $1, $0", "r,*m,i"(i8 %a0, i8* %a1, i8 7) nounwind
-  ret void
-}
-define void @test_cmp_16(i16 %a0, i16* %a1) optsize {
-; GENERIC-LABEL: test_cmp_16:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    cmpw $511, %ax # imm = 0x1FF
-; GENERIC-NEXT:    # sched: [1:0.33]
-; GENERIC-NEXT:    cmpw $511, %di # imm = 0x1FF
-; GENERIC-NEXT:    # sched: [1:0.33]
-; GENERIC-NEXT:    cmpw $511, (%rsi) # imm = 0x1FF
-; GENERIC-NEXT:    # sched: [6:0.50]
-; GENERIC-NEXT:    cmpw $7, %di # sched: [1:0.33]
-; GENERIC-NEXT:    cmpw $7, (%rsi) # sched: [6:0.50]
-; GENERIC-NEXT:    cmpw %di, %di # sched: [1:0.33]
-; GENERIC-NEXT:    cmpw %di, (%rsi) # sched: [6:0.50]
-; GENERIC-NEXT:    cmpw (%rsi), %di # sched: [6:0.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_cmp_16:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    cmpw $511, %ax # imm = 0x1FF
-; ATOM-NEXT:    # sched: [1:0.50]
-; ATOM-NEXT:    cmpw $511, %di # imm = 0x1FF
-; ATOM-NEXT:    # sched: [1:0.50]
-; ATOM-NEXT:    cmpw $511, (%rsi) # imm = 0x1FF
-; ATOM-NEXT:    # sched: [1:1.00]
-; ATOM-NEXT:    cmpw $7, %di # sched: [1:0.50]
-; ATOM-NEXT:    cmpw $7, (%rsi) # sched: [1:1.00]
-; ATOM-NEXT:    cmpw %di, %di # sched: [1:0.50]
-; ATOM-NEXT:    cmpw %di, (%rsi) # sched: [1:1.00]
-; ATOM-NEXT:    cmpw (%rsi), %di # sched: [1:1.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_cmp_16:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    cmpw $511, %ax # imm = 0x1FF
-; SLM-NEXT:    # sched: [1:0.50]
-; SLM-NEXT:    cmpw $511, %di # imm = 0x1FF
-; SLM-NEXT:    # sched: [1:0.50]
-; SLM-NEXT:    cmpw $511, (%rsi) # imm = 0x1FF
-; SLM-NEXT:    # sched: [4:1.00]
-; SLM-NEXT:    cmpw $7, %di # sched: [1:0.50]
-; SLM-NEXT:    cmpw $7, (%rsi) # sched: [4:1.00]
-; SLM-NEXT:    cmpw %di, %di # sched: [1:0.50]
-; SLM-NEXT:    cmpw %di, (%rsi) # sched: [4:1.00]
-; SLM-NEXT:    cmpw (%rsi), %di # sched: [4:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_cmp_16:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    cmpw $511, %ax # imm = 0x1FF
-; SANDY-NEXT:    # sched: [1:0.33]
-; SANDY-NEXT:    cmpw $511, %di # imm = 0x1FF
-; SANDY-NEXT:    # sched: [1:0.33]
-; SANDY-NEXT:    cmpw $511, (%rsi) # imm = 0x1FF
-; SANDY-NEXT:    # sched: [6:0.50]
-; SANDY-NEXT:    cmpw $7, %di # sched: [1:0.33]
-; SANDY-NEXT:    cmpw $7, (%rsi) # sched: [6:0.50]
-; SANDY-NEXT:    cmpw %di, %di # sched: [1:0.33]
-; SANDY-NEXT:    cmpw %di, (%rsi) # sched: [6:0.50]
-; SANDY-NEXT:    cmpw (%rsi), %di # sched: [6:0.50]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_cmp_16:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    cmpw $511, %ax # imm = 0x1FF
-; HASWELL-NEXT:    # sched: [1:0.25]
-; HASWELL-NEXT:    cmpw $511, %di # imm = 0x1FF
-; HASWELL-NEXT:    # sched: [1:0.25]
-; HASWELL-NEXT:    cmpw $511, (%rsi) # imm = 0x1FF
-; HASWELL-NEXT:    # sched: [6:0.50]
-; HASWELL-NEXT:    cmpw $7, %di # sched: [1:0.25]
-; HASWELL-NEXT:    cmpw $7, (%rsi) # sched: [6:0.50]
-; HASWELL-NEXT:    cmpw %di, %di # sched: [1:0.25]
-; HASWELL-NEXT:    cmpw %di, (%rsi) # sched: [6:0.50]
-; HASWELL-NEXT:    cmpw (%rsi), %di # sched: [6:0.50]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_cmp_16:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    cmpw $511, %ax # imm = 0x1FF
-; BROADWELL-NEXT:    # sched: [1:0.25]
-; BROADWELL-NEXT:    cmpw $511, %di # imm = 0x1FF
-; BROADWELL-NEXT:    # sched: [1:0.25]
-; BROADWELL-NEXT:    cmpw $511, (%rsi) # imm = 0x1FF
-; BROADWELL-NEXT:    # sched: [6:0.50]
-; BROADWELL-NEXT:    cmpw $7, %di # sched: [1:0.25]
-; BROADWELL-NEXT:    cmpw $7, (%rsi) # sched: [6:0.50]
-; BROADWELL-NEXT:    cmpw %di, %di # sched: [1:0.25]
-; BROADWELL-NEXT:    cmpw %di, (%rsi) # sched: [6:0.50]
-; BROADWELL-NEXT:    cmpw (%rsi), %di # sched: [6:0.50]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_cmp_16:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    cmpw $511, %ax # imm = 0x1FF
-; SKYLAKE-NEXT:    # sched: [1:0.25]
-; SKYLAKE-NEXT:    cmpw $511, %di # imm = 0x1FF
-; SKYLAKE-NEXT:    # sched: [1:0.25]
-; SKYLAKE-NEXT:    cmpw $511, (%rsi) # imm = 0x1FF
-; SKYLAKE-NEXT:    # sched: [6:0.50]
-; SKYLAKE-NEXT:    cmpw $7, %di # sched: [1:0.25]
-; SKYLAKE-NEXT:    cmpw $7, (%rsi) # sched: [6:0.50]
-; SKYLAKE-NEXT:    cmpw %di, %di # sched: [1:0.25]
-; SKYLAKE-NEXT:    cmpw %di, (%rsi) # sched: [6:0.50]
-; SKYLAKE-NEXT:    cmpw (%rsi), %di # sched: [6:0.50]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_cmp_16:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    cmpw $511, %ax # imm = 0x1FF
-; SKX-NEXT:    # sched: [1:0.25]
-; SKX-NEXT:    cmpw $511, %di # imm = 0x1FF
-; SKX-NEXT:    # sched: [1:0.25]
-; SKX-NEXT:    cmpw $511, (%rsi) # imm = 0x1FF
-; SKX-NEXT:    # sched: [6:0.50]
-; SKX-NEXT:    cmpw $7, %di # sched: [1:0.25]
-; SKX-NEXT:    cmpw $7, (%rsi) # sched: [6:0.50]
-; SKX-NEXT:    cmpw %di, %di # sched: [1:0.25]
-; SKX-NEXT:    cmpw %di, (%rsi) # sched: [6:0.50]
-; SKX-NEXT:    cmpw (%rsi), %di # sched: [6:0.50]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_cmp_16:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    cmpw $511, %ax # imm = 0x1FF
-; BDVER2-NEXT:    # sched: [1:0.50]
-; BDVER2-NEXT:    cmpw $511, %di # imm = 0x1FF
-; BDVER2-NEXT:    # sched: [1:0.50]
-; BDVER2-NEXT:    cmpw $511, (%rsi) # imm = 0x1FF
-; BDVER2-NEXT:    # sched: [5:0.50]
-; BDVER2-NEXT:    cmpw $7, %di # sched: [1:0.50]
-; BDVER2-NEXT:    cmpw $7, (%rsi) # sched: [5:0.50]
-; BDVER2-NEXT:    cmpw %di, %di # sched: [1:0.50]
-; BDVER2-NEXT:    cmpw %di, (%rsi) # sched: [5:0.50]
-; BDVER2-NEXT:    cmpw (%rsi), %di # sched: [5:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_cmp_16:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    cmpw $511, %ax # imm = 0x1FF
-; BTVER2-NEXT:    # sched: [1:0.50]
-; BTVER2-NEXT:    cmpw $511, %di # imm = 0x1FF
-; BTVER2-NEXT:    # sched: [1:0.50]
-; BTVER2-NEXT:    cmpw $511, (%rsi) # imm = 0x1FF
-; BTVER2-NEXT:    # sched: [4:1.00]
-; BTVER2-NEXT:    cmpw $7, %di # sched: [1:0.50]
-; BTVER2-NEXT:    cmpw $7, (%rsi) # sched: [4:1.00]
-; BTVER2-NEXT:    cmpw %di, %di # sched: [1:0.50]
-; BTVER2-NEXT:    cmpw %di, (%rsi) # sched: [4:1.00]
-; BTVER2-NEXT:    cmpw (%rsi), %di # sched: [4:1.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_cmp_16:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    cmpw $511, %ax # imm = 0x1FF
-; ZNVER1-NEXT:    # sched: [1:0.25]
-; ZNVER1-NEXT:    cmpw $511, %di # imm = 0x1FF
-; ZNVER1-NEXT:    # sched: [1:0.25]
-; ZNVER1-NEXT:    cmpw $511, (%rsi) # imm = 0x1FF
-; ZNVER1-NEXT:    # sched: [5:0.50]
-; ZNVER1-NEXT:    cmpw $7, %di # sched: [1:0.25]
-; ZNVER1-NEXT:    cmpw $7, (%rsi) # sched: [5:0.50]
-; ZNVER1-NEXT:    cmpw %di, %di # sched: [1:0.25]
-; ZNVER1-NEXT:    cmpw %di, (%rsi) # sched: [5:0.50]
-; ZNVER1-NEXT:    cmpw (%rsi), %di # sched: [5:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  tail call void asm "cmpw $2, %AX \0A\09 cmpw $2, $0 \0A\09 cmpw $2, $1 \0A\09 cmpw $3, $0 \0A\09 cmpw $3, $1 \0A\09 cmpw $0, $0 \0A\09 cmpw $0, $1 \0A\09 cmpw $1, $0", "r,*m,i,i"(i16 %a0, i16* %a1, i16 511, i8 7) nounwind
-  ret void
-}
-define void @test_cmp_32(i32 %a0, i32* %a1) optsize {
-; GENERIC-LABEL: test_cmp_32:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    cmpl $665536, %eax # imm = 0xA27C0
-; GENERIC-NEXT:    # sched: [1:0.33]
-; GENERIC-NEXT:    cmpl $665536, %edi # imm = 0xA27C0
-; GENERIC-NEXT:    # sched: [1:0.33]
-; GENERIC-NEXT:    cmpl $665536, (%rsi) # imm = 0xA27C0
-; GENERIC-NEXT:    # sched: [6:0.50]
-; GENERIC-NEXT:    cmpl $7, %edi # sched: [1:0.33]
-; GENERIC-NEXT:    cmpl $7, (%rsi) # sched: [6:0.50]
-; GENERIC-NEXT:    cmpl %edi, %edi # sched: [1:0.33]
-; GENERIC-NEXT:    cmpl %edi, (%rsi) # sched: [6:0.50]
-; GENERIC-NEXT:    cmpl (%rsi), %edi # sched: [6:0.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_cmp_32:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    cmpl $665536, %eax # imm = 0xA27C0
-; ATOM-NEXT:    # sched: [1:0.50]
-; ATOM-NEXT:    cmpl $665536, %edi # imm = 0xA27C0
-; ATOM-NEXT:    # sched: [1:0.50]
-; ATOM-NEXT:    cmpl $665536, (%rsi) # imm = 0xA27C0
-; ATOM-NEXT:    # sched: [1:1.00]
-; ATOM-NEXT:    cmpl $7, %edi # sched: [1:0.50]
-; ATOM-NEXT:    cmpl $7, (%rsi) # sched: [1:1.00]
-; ATOM-NEXT:    cmpl %edi, %edi # sched: [1:0.50]
-; ATOM-NEXT:    cmpl %edi, (%rsi) # sched: [1:1.00]
-; ATOM-NEXT:    cmpl (%rsi), %edi # sched: [1:1.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_cmp_32:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    cmpl $665536, %eax # imm = 0xA27C0
-; SLM-NEXT:    # sched: [1:0.50]
-; SLM-NEXT:    cmpl $665536, %edi # imm = 0xA27C0
-; SLM-NEXT:    # sched: [1:0.50]
-; SLM-NEXT:    cmpl $665536, (%rsi) # imm = 0xA27C0
-; SLM-NEXT:    # sched: [4:1.00]
-; SLM-NEXT:    cmpl $7, %edi # sched: [1:0.50]
-; SLM-NEXT:    cmpl $7, (%rsi) # sched: [4:1.00]
-; SLM-NEXT:    cmpl %edi, %edi # sched: [1:0.50]
-; SLM-NEXT:    cmpl %edi, (%rsi) # sched: [4:1.00]
-; SLM-NEXT:    cmpl (%rsi), %edi # sched: [4:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_cmp_32:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    cmpl $665536, %eax # imm = 0xA27C0
-; SANDY-NEXT:    # sched: [1:0.33]
-; SANDY-NEXT:    cmpl $665536, %edi # imm = 0xA27C0
-; SANDY-NEXT:    # sched: [1:0.33]
-; SANDY-NEXT:    cmpl $665536, (%rsi) # imm = 0xA27C0
-; SANDY-NEXT:    # sched: [6:0.50]
-; SANDY-NEXT:    cmpl $7, %edi # sched: [1:0.33]
-; SANDY-NEXT:    cmpl $7, (%rsi) # sched: [6:0.50]
-; SANDY-NEXT:    cmpl %edi, %edi # sched: [1:0.33]
-; SANDY-NEXT:    cmpl %edi, (%rsi) # sched: [6:0.50]
-; SANDY-NEXT:    cmpl (%rsi), %edi # sched: [6:0.50]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_cmp_32:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    cmpl $665536, %eax # imm = 0xA27C0
-; HASWELL-NEXT:    # sched: [1:0.25]
-; HASWELL-NEXT:    cmpl $665536, %edi # imm = 0xA27C0
-; HASWELL-NEXT:    # sched: [1:0.25]
-; HASWELL-NEXT:    cmpl $665536, (%rsi) # imm = 0xA27C0
-; HASWELL-NEXT:    # sched: [6:0.50]
-; HASWELL-NEXT:    cmpl $7, %edi # sched: [1:0.25]
-; HASWELL-NEXT:    cmpl $7, (%rsi) # sched: [6:0.50]
-; HASWELL-NEXT:    cmpl %edi, %edi # sched: [1:0.25]
-; HASWELL-NEXT:    cmpl %edi, (%rsi) # sched: [6:0.50]
-; HASWELL-NEXT:    cmpl (%rsi), %edi # sched: [6:0.50]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_cmp_32:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    cmpl $665536, %eax # imm = 0xA27C0
-; BROADWELL-NEXT:    # sched: [1:0.25]
-; BROADWELL-NEXT:    cmpl $665536, %edi # imm = 0xA27C0
-; BROADWELL-NEXT:    # sched: [1:0.25]
-; BROADWELL-NEXT:    cmpl $665536, (%rsi) # imm = 0xA27C0
-; BROADWELL-NEXT:    # sched: [6:0.50]
-; BROADWELL-NEXT:    cmpl $7, %edi # sched: [1:0.25]
-; BROADWELL-NEXT:    cmpl $7, (%rsi) # sched: [6:0.50]
-; BROADWELL-NEXT:    cmpl %edi, %edi # sched: [1:0.25]
-; BROADWELL-NEXT:    cmpl %edi, (%rsi) # sched: [6:0.50]
-; BROADWELL-NEXT:    cmpl (%rsi), %edi # sched: [6:0.50]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_cmp_32:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    cmpl $665536, %eax # imm = 0xA27C0
-; SKYLAKE-NEXT:    # sched: [1:0.25]
-; SKYLAKE-NEXT:    cmpl $665536, %edi # imm = 0xA27C0
-; SKYLAKE-NEXT:    # sched: [1:0.25]
-; SKYLAKE-NEXT:    cmpl $665536, (%rsi) # imm = 0xA27C0
-; SKYLAKE-NEXT:    # sched: [6:0.50]
-; SKYLAKE-NEXT:    cmpl $7, %edi # sched: [1:0.25]
-; SKYLAKE-NEXT:    cmpl $7, (%rsi) # sched: [6:0.50]
-; SKYLAKE-NEXT:    cmpl %edi, %edi # sched: [1:0.25]
-; SKYLAKE-NEXT:    cmpl %edi, (%rsi) # sched: [6:0.50]
-; SKYLAKE-NEXT:    cmpl (%rsi), %edi # sched: [6:0.50]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_cmp_32:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    cmpl $665536, %eax # imm = 0xA27C0
-; SKX-NEXT:    # sched: [1:0.25]
-; SKX-NEXT:    cmpl $665536, %edi # imm = 0xA27C0
-; SKX-NEXT:    # sched: [1:0.25]
-; SKX-NEXT:    cmpl $665536, (%rsi) # imm = 0xA27C0
-; SKX-NEXT:    # sched: [6:0.50]
-; SKX-NEXT:    cmpl $7, %edi # sched: [1:0.25]
-; SKX-NEXT:    cmpl $7, (%rsi) # sched: [6:0.50]
-; SKX-NEXT:    cmpl %edi, %edi # sched: [1:0.25]
-; SKX-NEXT:    cmpl %edi, (%rsi) # sched: [6:0.50]
-; SKX-NEXT:    cmpl (%rsi), %edi # sched: [6:0.50]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_cmp_32:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    cmpl $665536, %eax # imm = 0xA27C0
-; BDVER2-NEXT:    # sched: [1:0.50]
-; BDVER2-NEXT:    cmpl $665536, %edi # imm = 0xA27C0
-; BDVER2-NEXT:    # sched: [1:0.50]
-; BDVER2-NEXT:    cmpl $665536, (%rsi) # imm = 0xA27C0
-; BDVER2-NEXT:    # sched: [5:0.50]
-; BDVER2-NEXT:    cmpl $7, %edi # sched: [1:0.50]
-; BDVER2-NEXT:    cmpl $7, (%rsi) # sched: [5:0.50]
-; BDVER2-NEXT:    cmpl %edi, %edi # sched: [1:0.50]
-; BDVER2-NEXT:    cmpl %edi, (%rsi) # sched: [5:0.50]
-; BDVER2-NEXT:    cmpl (%rsi), %edi # sched: [5:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_cmp_32:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    cmpl $665536, %eax # imm = 0xA27C0
-; BTVER2-NEXT:    # sched: [1:0.50]
-; BTVER2-NEXT:    cmpl $665536, %edi # imm = 0xA27C0
-; BTVER2-NEXT:    # sched: [1:0.50]
-; BTVER2-NEXT:    cmpl $665536, (%rsi) # imm = 0xA27C0
-; BTVER2-NEXT:    # sched: [4:1.00]
-; BTVER2-NEXT:    cmpl $7, %edi # sched: [1:0.50]
-; BTVER2-NEXT:    cmpl $7, (%rsi) # sched: [4:1.00]
-; BTVER2-NEXT:    cmpl %edi, %edi # sched: [1:0.50]
-; BTVER2-NEXT:    cmpl %edi, (%rsi) # sched: [4:1.00]
-; BTVER2-NEXT:    cmpl (%rsi), %edi # sched: [4:1.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_cmp_32:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    cmpl $665536, %eax # imm = 0xA27C0
-; ZNVER1-NEXT:    # sched: [1:0.25]
-; ZNVER1-NEXT:    cmpl $665536, %edi # imm = 0xA27C0
-; ZNVER1-NEXT:    # sched: [1:0.25]
-; ZNVER1-NEXT:    cmpl $665536, (%rsi) # imm = 0xA27C0
-; ZNVER1-NEXT:    # sched: [5:0.50]
-; ZNVER1-NEXT:    cmpl $7, %edi # sched: [1:0.25]
-; ZNVER1-NEXT:    cmpl $7, (%rsi) # sched: [5:0.50]
-; ZNVER1-NEXT:    cmpl %edi, %edi # sched: [1:0.25]
-; ZNVER1-NEXT:    cmpl %edi, (%rsi) # sched: [5:0.50]
-; ZNVER1-NEXT:    cmpl (%rsi), %edi # sched: [5:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  tail call void asm "cmpl $2, %EAX \0A\09 cmpl $2, $0 \0A\09 cmpl $2, $1 \0A\09 cmpl $3, $0 \0A\09 cmpl $3, $1 \0A\09 cmpl $0, $0 \0A\09 cmpl $0, $1 \0A\09 cmpl $1, $0", "r,*m,i,i"(i32 %a0, i32* %a1, i32 665536, i8 7) nounwind
-  ret void
-}
-define void @test_cmp_64(i64 %a0, i64* %a1) optsize {
-; GENERIC-LABEL: test_cmp_64:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    cmpq $665536, %rax # imm = 0xA27C0
-; GENERIC-NEXT:    # sched: [1:0.33]
-; GENERIC-NEXT:    cmpq $665536, %rdi # imm = 0xA27C0
-; GENERIC-NEXT:    # sched: [1:0.33]
-; GENERIC-NEXT:    cmpq $665536, (%rsi) # imm = 0xA27C0
-; GENERIC-NEXT:    # sched: [6:0.50]
-; GENERIC-NEXT:    cmpq $7, %rdi # sched: [1:0.33]
-; GENERIC-NEXT:    cmpq $7, (%rsi) # sched: [6:0.50]
-; GENERIC-NEXT:    cmpq %rdi, %rdi # sched: [1:0.33]
-; GENERIC-NEXT:    cmpq %rdi, (%rsi) # sched: [6:0.50]
-; GENERIC-NEXT:    cmpq (%rsi), %rdi # sched: [6:0.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_cmp_64:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    cmpq $665536, %rax # imm = 0xA27C0
-; ATOM-NEXT:    # sched: [1:0.50]
-; ATOM-NEXT:    cmpq $665536, %rdi # imm = 0xA27C0
-; ATOM-NEXT:    # sched: [1:0.50]
-; ATOM-NEXT:    cmpq $665536, (%rsi) # imm = 0xA27C0
-; ATOM-NEXT:    # sched: [1:1.00]
-; ATOM-NEXT:    cmpq $7, %rdi # sched: [1:0.50]
-; ATOM-NEXT:    cmpq $7, (%rsi) # sched: [1:1.00]
-; ATOM-NEXT:    cmpq %rdi, %rdi # sched: [1:0.50]
-; ATOM-NEXT:    cmpq %rdi, (%rsi) # sched: [1:1.00]
-; ATOM-NEXT:    cmpq (%rsi), %rdi # sched: [1:1.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_cmp_64:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    cmpq $665536, %rax # imm = 0xA27C0
-; SLM-NEXT:    # sched: [1:0.50]
-; SLM-NEXT:    cmpq $665536, %rdi # imm = 0xA27C0
-; SLM-NEXT:    # sched: [1:0.50]
-; SLM-NEXT:    cmpq $665536, (%rsi) # imm = 0xA27C0
-; SLM-NEXT:    # sched: [4:1.00]
-; SLM-NEXT:    cmpq $7, %rdi # sched: [1:0.50]
-; SLM-NEXT:    cmpq $7, (%rsi) # sched: [4:1.00]
-; SLM-NEXT:    cmpq %rdi, %rdi # sched: [1:0.50]
-; SLM-NEXT:    cmpq %rdi, (%rsi) # sched: [4:1.00]
-; SLM-NEXT:    cmpq (%rsi), %rdi # sched: [4:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_cmp_64:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    cmpq $665536, %rax # imm = 0xA27C0
-; SANDY-NEXT:    # sched: [1:0.33]
-; SANDY-NEXT:    cmpq $665536, %rdi # imm = 0xA27C0
-; SANDY-NEXT:    # sched: [1:0.33]
-; SANDY-NEXT:    cmpq $665536, (%rsi) # imm = 0xA27C0
-; SANDY-NEXT:    # sched: [6:0.50]
-; SANDY-NEXT:    cmpq $7, %rdi # sched: [1:0.33]
-; SANDY-NEXT:    cmpq $7, (%rsi) # sched: [6:0.50]
-; SANDY-NEXT:    cmpq %rdi, %rdi # sched: [1:0.33]
-; SANDY-NEXT:    cmpq %rdi, (%rsi) # sched: [6:0.50]
-; SANDY-NEXT:    cmpq (%rsi), %rdi # sched: [6:0.50]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_cmp_64:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    cmpq $665536, %rax # imm = 0xA27C0
-; HASWELL-NEXT:    # sched: [1:0.25]
-; HASWELL-NEXT:    cmpq $665536, %rdi # imm = 0xA27C0
-; HASWELL-NEXT:    # sched: [1:0.25]
-; HASWELL-NEXT:    cmpq $665536, (%rsi) # imm = 0xA27C0
-; HASWELL-NEXT:    # sched: [6:0.50]
-; HASWELL-NEXT:    cmpq $7, %rdi # sched: [1:0.25]
-; HASWELL-NEXT:    cmpq $7, (%rsi) # sched: [6:0.50]
-; HASWELL-NEXT:    cmpq %rdi, %rdi # sched: [1:0.25]
-; HASWELL-NEXT:    cmpq %rdi, (%rsi) # sched: [6:0.50]
-; HASWELL-NEXT:    cmpq (%rsi), %rdi # sched: [6:0.50]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_cmp_64:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    cmpq $665536, %rax # imm = 0xA27C0
-; BROADWELL-NEXT:    # sched: [1:0.25]
-; BROADWELL-NEXT:    cmpq $665536, %rdi # imm = 0xA27C0
-; BROADWELL-NEXT:    # sched: [1:0.25]
-; BROADWELL-NEXT:    cmpq $665536, (%rsi) # imm = 0xA27C0
-; BROADWELL-NEXT:    # sched: [6:0.50]
-; BROADWELL-NEXT:    cmpq $7, %rdi # sched: [1:0.25]
-; BROADWELL-NEXT:    cmpq $7, (%rsi) # sched: [6:0.50]
-; BROADWELL-NEXT:    cmpq %rdi, %rdi # sched: [1:0.25]
-; BROADWELL-NEXT:    cmpq %rdi, (%rsi) # sched: [6:0.50]
-; BROADWELL-NEXT:    cmpq (%rsi), %rdi # sched: [6:0.50]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_cmp_64:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    cmpq $665536, %rax # imm = 0xA27C0
-; SKYLAKE-NEXT:    # sched: [1:0.25]
-; SKYLAKE-NEXT:    cmpq $665536, %rdi # imm = 0xA27C0
-; SKYLAKE-NEXT:    # sched: [1:0.25]
-; SKYLAKE-NEXT:    cmpq $665536, (%rsi) # imm = 0xA27C0
-; SKYLAKE-NEXT:    # sched: [6:0.50]
-; SKYLAKE-NEXT:    cmpq $7, %rdi # sched: [1:0.25]
-; SKYLAKE-NEXT:    cmpq $7, (%rsi) # sched: [6:0.50]
-; SKYLAKE-NEXT:    cmpq %rdi, %rdi # sched: [1:0.25]
-; SKYLAKE-NEXT:    cmpq %rdi, (%rsi) # sched: [6:0.50]
-; SKYLAKE-NEXT:    cmpq (%rsi), %rdi # sched: [6:0.50]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_cmp_64:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    cmpq $665536, %rax # imm = 0xA27C0
-; SKX-NEXT:    # sched: [1:0.25]
-; SKX-NEXT:    cmpq $665536, %rdi # imm = 0xA27C0
-; SKX-NEXT:    # sched: [1:0.25]
-; SKX-NEXT:    cmpq $665536, (%rsi) # imm = 0xA27C0
-; SKX-NEXT:    # sched: [6:0.50]
-; SKX-NEXT:    cmpq $7, %rdi # sched: [1:0.25]
-; SKX-NEXT:    cmpq $7, (%rsi) # sched: [6:0.50]
-; SKX-NEXT:    cmpq %rdi, %rdi # sched: [1:0.25]
-; SKX-NEXT:    cmpq %rdi, (%rsi) # sched: [6:0.50]
-; SKX-NEXT:    cmpq (%rsi), %rdi # sched: [6:0.50]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_cmp_64:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    cmpq $665536, %rax # imm = 0xA27C0
-; BDVER2-NEXT:    # sched: [1:0.50]
-; BDVER2-NEXT:    cmpq $665536, %rdi # imm = 0xA27C0
-; BDVER2-NEXT:    # sched: [1:0.50]
-; BDVER2-NEXT:    cmpq $665536, (%rsi) # imm = 0xA27C0
-; BDVER2-NEXT:    # sched: [5:0.50]
-; BDVER2-NEXT:    cmpq $7, %rdi # sched: [1:0.50]
-; BDVER2-NEXT:    cmpq $7, (%rsi) # sched: [5:0.50]
-; BDVER2-NEXT:    cmpq %rdi, %rdi # sched: [1:0.50]
-; BDVER2-NEXT:    cmpq %rdi, (%rsi) # sched: [5:0.50]
-; BDVER2-NEXT:    cmpq (%rsi), %rdi # sched: [5:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_cmp_64:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    cmpq $665536, %rax # imm = 0xA27C0
-; BTVER2-NEXT:    # sched: [1:0.50]
-; BTVER2-NEXT:    cmpq $665536, %rdi # imm = 0xA27C0
-; BTVER2-NEXT:    # sched: [1:0.50]
-; BTVER2-NEXT:    cmpq $665536, (%rsi) # imm = 0xA27C0
-; BTVER2-NEXT:    # sched: [4:1.00]
-; BTVER2-NEXT:    cmpq $7, %rdi # sched: [1:0.50]
-; BTVER2-NEXT:    cmpq $7, (%rsi) # sched: [4:1.00]
-; BTVER2-NEXT:    cmpq %rdi, %rdi # sched: [1:0.50]
-; BTVER2-NEXT:    cmpq %rdi, (%rsi) # sched: [4:1.00]
-; BTVER2-NEXT:    cmpq (%rsi), %rdi # sched: [4:1.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_cmp_64:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    cmpq $665536, %rax # imm = 0xA27C0
-; ZNVER1-NEXT:    # sched: [1:0.25]
-; ZNVER1-NEXT:    cmpq $665536, %rdi # imm = 0xA27C0
-; ZNVER1-NEXT:    # sched: [1:0.25]
-; ZNVER1-NEXT:    cmpq $665536, (%rsi) # imm = 0xA27C0
-; ZNVER1-NEXT:    # sched: [5:0.50]
-; ZNVER1-NEXT:    cmpq $7, %rdi # sched: [1:0.25]
-; ZNVER1-NEXT:    cmpq $7, (%rsi) # sched: [5:0.50]
-; ZNVER1-NEXT:    cmpq %rdi, %rdi # sched: [1:0.25]
-; ZNVER1-NEXT:    cmpq %rdi, (%rsi) # sched: [5:0.50]
-; ZNVER1-NEXT:    cmpq (%rsi), %rdi # sched: [5:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  tail call void asm "cmpq $2, %RAX \0A\09 cmpq $2, $0 \0A\09 cmpq $2, $1 \0A\09 cmpq $3, $0 \0A\09 cmpq $3, $1 \0A\09 cmpq $0, $0 \0A\09 cmpq $0, $1 \0A\09 cmpq $1, $0", "r,*m,i,i"(i64 %a0, i64* %a1, i32 665536, i8 7) nounwind
-  ret void
-}
-
-define void @test_cmps() optsize {
-; GENERIC-LABEL: test_cmps:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    cmpsb %es:(%rdi), (%rsi) # sched: [8:1.00]
-; GENERIC-NEXT:    cmpsw %es:(%rdi), (%rsi) # sched: [8:1.00]
-; GENERIC-NEXT:    cmpsl %es:(%rdi), (%rsi) # sched: [8:1.00]
-; GENERIC-NEXT:    cmpsq %es:(%rdi), (%rsi) # sched: [8:1.00]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_cmps:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    cmpsb %es:(%rdi), (%rsi) # sched: [3:1.50]
-; ATOM-NEXT:    cmpsw %es:(%rdi), (%rsi) # sched: [3:1.50]
-; ATOM-NEXT:    cmpsl %es:(%rdi), (%rsi) # sched: [3:1.50]
-; ATOM-NEXT:    cmpsq %es:(%rdi), (%rsi) # sched: [3:1.50]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_cmps:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    cmpsb %es:(%rdi), (%rsi) # sched: [100:1.00]
-; SLM-NEXT:    cmpsw %es:(%rdi), (%rsi) # sched: [100:1.00]
-; SLM-NEXT:    cmpsl %es:(%rdi), (%rsi) # sched: [100:1.00]
-; SLM-NEXT:    cmpsq %es:(%rdi), (%rsi) # sched: [100:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_cmps:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    cmpsb %es:(%rdi), (%rsi) # sched: [8:1.00]
-; SANDY-NEXT:    cmpsw %es:(%rdi), (%rsi) # sched: [8:1.00]
-; SANDY-NEXT:    cmpsl %es:(%rdi), (%rsi) # sched: [8:1.00]
-; SANDY-NEXT:    cmpsq %es:(%rdi), (%rsi) # sched: [8:1.00]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_cmps:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    cmpsb %es:(%rdi), (%rsi) # sched: [4:1.00]
-; HASWELL-NEXT:    cmpsw %es:(%rdi), (%rsi) # sched: [4:1.00]
-; HASWELL-NEXT:    cmpsl %es:(%rdi), (%rsi) # sched: [4:1.00]
-; HASWELL-NEXT:    cmpsq %es:(%rdi), (%rsi) # sched: [4:1.00]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_cmps:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    cmpsb %es:(%rdi), (%rsi) # sched: [100:0.25]
-; BROADWELL-NEXT:    cmpsw %es:(%rdi), (%rsi) # sched: [100:0.25]
-; BROADWELL-NEXT:    cmpsl %es:(%rdi), (%rsi) # sched: [100:0.25]
-; BROADWELL-NEXT:    cmpsq %es:(%rdi), (%rsi) # sched: [100:0.25]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_cmps:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    cmpsb %es:(%rdi), (%rsi) # sched: [100:0.25]
-; SKYLAKE-NEXT:    cmpsw %es:(%rdi), (%rsi) # sched: [100:0.25]
-; SKYLAKE-NEXT:    cmpsl %es:(%rdi), (%rsi) # sched: [100:0.25]
-; SKYLAKE-NEXT:    cmpsq %es:(%rdi), (%rsi) # sched: [100:0.25]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_cmps:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    cmpsb %es:(%rdi), (%rsi) # sched: [100:0.25]
-; SKX-NEXT:    cmpsw %es:(%rdi), (%rsi) # sched: [100:0.25]
-; SKX-NEXT:    cmpsl %es:(%rdi), (%rsi) # sched: [100:0.25]
-; SKX-NEXT:    cmpsq %es:(%rdi), (%rsi) # sched: [100:0.25]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_cmps:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    cmpsb %es:(%rdi), (%rsi) # sched: [100:0.50]
-; BDVER2-NEXT:    cmpsw %es:(%rdi), (%rsi) # sched: [100:0.50]
-; BDVER2-NEXT:    cmpsl %es:(%rdi), (%rsi) # sched: [100:0.50]
-; BDVER2-NEXT:    cmpsq %es:(%rdi), (%rsi) # sched: [100:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_cmps:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    cmpsb %es:(%rdi), (%rsi) # sched: [100:0.50]
-; BTVER2-NEXT:    cmpsw %es:(%rdi), (%rsi) # sched: [100:0.50]
-; BTVER2-NEXT:    cmpsl %es:(%rdi), (%rsi) # sched: [100:0.50]
-; BTVER2-NEXT:    cmpsq %es:(%rdi), (%rsi) # sched: [100:0.50]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_cmps:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    cmpsb %es:(%rdi), (%rsi) # sched: [100:0.25]
-; ZNVER1-NEXT:    cmpsw %es:(%rdi), (%rsi) # sched: [100:0.25]
-; ZNVER1-NEXT:    cmpsl %es:(%rdi), (%rsi) # sched: [100:0.25]
-; ZNVER1-NEXT:    cmpsq %es:(%rdi), (%rsi) # sched: [100:0.25]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  call void asm sideeffect "cmpsb \0A\09 cmpsw \0A\09 cmpsl \0A\09 cmpsq", ""()
-  ret void
-}
-
-define void @test_cmpxchg_8(i8 %a0, i8 %a1, i8 *%a2) optsize {
-; GENERIC-LABEL: test_cmpxchg_8:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    cmpxchgb %dil, %sil # sched: [5:1.33]
-; GENERIC-NEXT:    cmpxchgb %dil, (%rdx) # sched: [8:2.00]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_cmpxchg_8:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    cmpxchgb %dil, %sil # sched: [9:4.50]
-; ATOM-NEXT:    cmpxchgb %dil, (%rdx) # sched: [6:3.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_cmpxchg_8:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    cmpxchgb %dil, %sil # sched: [1:0.50]
-; SLM-NEXT:    cmpxchgb %dil, (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_cmpxchg_8:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    cmpxchgb %dil, %sil # sched: [5:1.33]
-; SANDY-NEXT:    cmpxchgb %dil, (%rdx) # sched: [8:2.00]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_cmpxchg_8:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    cmpxchgb %dil, %sil # sched: [5:1.25]
-; HASWELL-NEXT:    cmpxchgb %dil, (%rdx) # sched: [9:1.00]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_cmpxchg_8:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    cmpxchgb %dil, %sil # sched: [5:1.25]
-; BROADWELL-NEXT:    cmpxchgb %dil, (%rdx) # sched: [8:1.00]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_cmpxchg_8:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    cmpxchgb %dil, %sil # sched: [5:1.25]
-; SKYLAKE-NEXT:    cmpxchgb %dil, (%rdx) # sched: [8:1.00]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_cmpxchg_8:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    cmpxchgb %dil, %sil # sched: [5:1.25]
-; SKX-NEXT:    cmpxchgb %dil, (%rdx) # sched: [8:1.00]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_cmpxchg_8:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    cmpxchgb %dil, %sil # sched: [3:1.00]
-; BDVER2-NEXT:    cmpxchgb %dil, (%rdx) # sched: [3:1.00]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_cmpxchg_8:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    cmpxchgb %dil, %sil # sched: [1:0.50]
-; BTVER2-NEXT:    cmpxchgb %dil, (%rdx) # sched: [4:1.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_cmpxchg_8:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    cmpxchgb %dil, %sil # sched: [1:0.25]
-; ZNVER1-NEXT:    cmpxchgb %dil, (%rdx) # sched: [8:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  tail call void asm "cmpxchgb $0, $1 \0a\09 cmpxchgb $0, $2", "r,r,*m"(i8 %a0, i8 %a1, i8 *%a2) nounwind
-  ret void
-}
-define void @test_cmpxchg_16(i16 %a0, i16 %a1, i16 *%a2) optsize {
-; GENERIC-LABEL: test_cmpxchg_16:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    cmpxchgw %di, %si # sched: [5:1.33]
-; GENERIC-NEXT:    cmpxchgw %di, (%rdx) # sched: [8:2.00]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_cmpxchg_16:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    cmpxchgw %di, %si # sched: [15:7.50]
-; ATOM-NEXT:    cmpxchgw %di, (%rdx) # sched: [14:7.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_cmpxchg_16:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    cmpxchgw %di, %si # sched: [1:0.50]
-; SLM-NEXT:    cmpxchgw %di, (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_cmpxchg_16:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    cmpxchgw %di, %si # sched: [5:1.33]
-; SANDY-NEXT:    cmpxchgw %di, (%rdx) # sched: [8:2.00]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_cmpxchg_16:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    cmpxchgw %di, %si # sched: [5:1.25]
-; HASWELL-NEXT:    cmpxchgw %di, (%rdx) # sched: [9:1.00]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_cmpxchg_16:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    cmpxchgw %di, %si # sched: [5:1.25]
-; BROADWELL-NEXT:    cmpxchgw %di, (%rdx) # sched: [8:1.00]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_cmpxchg_16:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    cmpxchgw %di, %si # sched: [5:1.25]
-; SKYLAKE-NEXT:    cmpxchgw %di, (%rdx) # sched: [8:1.00]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_cmpxchg_16:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    cmpxchgw %di, %si # sched: [5:1.25]
-; SKX-NEXT:    cmpxchgw %di, (%rdx) # sched: [8:1.00]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_cmpxchg_16:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    cmpxchgw %di, %si # sched: [3:1.00]
-; BDVER2-NEXT:    cmpxchgw %di, (%rdx) # sched: [3:1.00]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_cmpxchg_16:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    cmpxchgw %di, %si # sched: [1:0.50]
-; BTVER2-NEXT:    cmpxchgw %di, (%rdx) # sched: [4:1.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_cmpxchg_16:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    cmpxchgw %di, %si # sched: [1:0.25]
-; ZNVER1-NEXT:    cmpxchgw %di, (%rdx) # sched: [8:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  tail call void asm "cmpxchgw $0, $1 \0a\09 cmpxchgw $0, $2", "r,r,*m"(i16 %a0, i16 %a1, i16 *%a2) nounwind
-  ret void
-}
-define void @test_cmpxchg_32(i32 %a0, i32 %a1, i32 *%a2) optsize {
-; GENERIC-LABEL: test_cmpxchg_32:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    cmpxchgl %edi, %esi # sched: [5:1.33]
-; GENERIC-NEXT:    cmpxchgl %edi, (%rdx) # sched: [8:2.00]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_cmpxchg_32:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    cmpxchgl %edi, %esi # sched: [15:7.50]
-; ATOM-NEXT:    cmpxchgl %edi, (%rdx) # sched: [14:7.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_cmpxchg_32:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    cmpxchgl %edi, %esi # sched: [1:0.50]
-; SLM-NEXT:    cmpxchgl %edi, (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_cmpxchg_32:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    cmpxchgl %edi, %esi # sched: [5:1.33]
-; SANDY-NEXT:    cmpxchgl %edi, (%rdx) # sched: [8:2.00]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_cmpxchg_32:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    cmpxchgl %edi, %esi # sched: [5:1.25]
-; HASWELL-NEXT:    cmpxchgl %edi, (%rdx) # sched: [9:1.00]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_cmpxchg_32:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    cmpxchgl %edi, %esi # sched: [5:1.25]
-; BROADWELL-NEXT:    cmpxchgl %edi, (%rdx) # sched: [8:1.00]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_cmpxchg_32:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    cmpxchgl %edi, %esi # sched: [5:1.25]
-; SKYLAKE-NEXT:    cmpxchgl %edi, (%rdx) # sched: [8:1.00]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_cmpxchg_32:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    cmpxchgl %edi, %esi # sched: [5:1.25]
-; SKX-NEXT:    cmpxchgl %edi, (%rdx) # sched: [8:1.00]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_cmpxchg_32:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    cmpxchgl %edi, %esi # sched: [3:1.00]
-; BDVER2-NEXT:    cmpxchgl %edi, (%rdx) # sched: [3:1.00]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_cmpxchg_32:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    cmpxchgl %edi, %esi # sched: [1:0.50]
-; BTVER2-NEXT:    cmpxchgl %edi, (%rdx) # sched: [4:1.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_cmpxchg_32:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    cmpxchgl %edi, %esi # sched: [1:0.25]
-; ZNVER1-NEXT:    cmpxchgl %edi, (%rdx) # sched: [8:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  tail call void asm "cmpxchgl $0, $1 \0a\09 cmpxchgl $0, $2", "r,r,*m"(i32 %a0, i32 %a1, i32 *%a2) nounwind
-  ret void
-}
-define void @test_cmpxchg_64(i64 %a0, i64 %a1, i64 *%a2) optsize {
-; GENERIC-LABEL: test_cmpxchg_64:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    cmpxchgq %rdi, %rsi # sched: [5:1.33]
-; GENERIC-NEXT:    cmpxchgq %rdi, (%rdx) # sched: [8:2.00]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_cmpxchg_64:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    cmpxchgq %rdi, %rsi # sched: [15:7.50]
-; ATOM-NEXT:    cmpxchgq %rdi, (%rdx) # sched: [14:7.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_cmpxchg_64:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    cmpxchgq %rdi, %rsi # sched: [1:0.50]
-; SLM-NEXT:    cmpxchgq %rdi, (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_cmpxchg_64:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    cmpxchgq %rdi, %rsi # sched: [5:1.33]
-; SANDY-NEXT:    cmpxchgq %rdi, (%rdx) # sched: [8:2.00]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_cmpxchg_64:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    cmpxchgq %rdi, %rsi # sched: [5:1.25]
-; HASWELL-NEXT:    cmpxchgq %rdi, (%rdx) # sched: [9:1.00]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_cmpxchg_64:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    cmpxchgq %rdi, %rsi # sched: [5:1.25]
-; BROADWELL-NEXT:    cmpxchgq %rdi, (%rdx) # sched: [8:1.00]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_cmpxchg_64:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    cmpxchgq %rdi, %rsi # sched: [5:1.25]
-; SKYLAKE-NEXT:    cmpxchgq %rdi, (%rdx) # sched: [8:1.00]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_cmpxchg_64:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    cmpxchgq %rdi, %rsi # sched: [5:1.25]
-; SKX-NEXT:    cmpxchgq %rdi, (%rdx) # sched: [8:1.00]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_cmpxchg_64:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    cmpxchgq %rdi, %rsi # sched: [3:1.00]
-; BDVER2-NEXT:    cmpxchgq %rdi, (%rdx) # sched: [3:1.00]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_cmpxchg_64:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    cmpxchgq %rdi, %rsi # sched: [1:0.50]
-; BTVER2-NEXT:    cmpxchgq %rdi, (%rdx) # sched: [4:1.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_cmpxchg_64:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    cmpxchgq %rdi, %rsi # sched: [1:0.25]
-; ZNVER1-NEXT:    cmpxchgq %rdi, (%rdx) # sched: [8:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  tail call void asm "cmpxchgq $0, $1 \0a\09 cmpxchgq $0, $2", "r,r,*m"(i64 %a0, i64 %a1, i64 *%a2) nounwind
-  ret void
-}
-define void @test_cmpxchg8b_cmpxchg16b(i8 *%a0) optsize {
-; GENERIC-LABEL: test_cmpxchg8b_cmpxchg16b:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    cmpxchg8b (%rdi) # sched: [6:1.00]
-; GENERIC-NEXT:    cmpxchg16b (%rdi) # sched: [6:1.00]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_cmpxchg8b_cmpxchg16b:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    cmpxchg8b (%rdi) # sched: [18:9.00]
-; ATOM-NEXT:    cmpxchg16b (%rdi) # sched: [22:11.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_cmpxchg8b_cmpxchg16b:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    cmpxchg8b (%rdi) # sched: [4:2.00]
-; SLM-NEXT:    cmpxchg16b (%rdi) # sched: [4:2.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_cmpxchg8b_cmpxchg16b:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    cmpxchg8b (%rdi) # sched: [6:1.00]
-; SANDY-NEXT:    cmpxchg16b (%rdi) # sched: [6:1.00]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_cmpxchg8b_cmpxchg16b:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    cmpxchg8b (%rdi) # sched: [17:2.75]
-; HASWELL-NEXT:    cmpxchg16b (%rdi) # sched: [22:4.00]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_cmpxchg8b_cmpxchg16b:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    cmpxchg8b (%rdi) # sched: [16:2.75]
-; BROADWELL-NEXT:    cmpxchg16b (%rdi) # sched: [21:4.00]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_cmpxchg8b_cmpxchg16b:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    cmpxchg8b (%rdi) # sched: [16:2.75]
-; SKYLAKE-NEXT:    cmpxchg16b (%rdi) # sched: [23:4.00]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_cmpxchg8b_cmpxchg16b:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    cmpxchg8b (%rdi) # sched: [16:2.75]
-; SKX-NEXT:    cmpxchg16b (%rdi) # sched: [23:4.00]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_cmpxchg8b_cmpxchg16b:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    cmpxchg8b (%rdi) # sched: [3:1.00]
-; BDVER2-NEXT:    cmpxchg16b (%rdi) # sched: [3:1.00]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_cmpxchg8b_cmpxchg16b:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    cmpxchg8b (%rdi) # sched: [4:1.00]
-; BTVER2-NEXT:    cmpxchg16b (%rdi) # sched: [4:1.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_cmpxchg8b_cmpxchg16b:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    cmpxchg8b (%rdi) # sched: [1:0.50]
-; ZNVER1-NEXT:    cmpxchg16b (%rdi) # sched: [100:0.25]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  tail call void asm "cmpxchg8b $0 \0a\09 cmpxchg16b $0", "*m"(i8 *%a0) nounwind
-  ret void
-}
-
-define void @test_cpuid() optsize {
-; GENERIC-LABEL: test_cpuid:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    cpuid # sched: [100:0.33]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_cpuid:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    cpuid # sched: [121:60.50]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_cpuid:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    cpuid # sched: [100:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_cpuid:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    cpuid # sched: [100:0.33]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_cpuid:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    cpuid # sched: [18:2.00]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_cpuid:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    cpuid # sched: [18:2.00]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_cpuid:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    cpuid # sched: [18:2.00]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_cpuid:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    cpuid # sched: [18:2.00]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_cpuid:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    cpuid # sched: [100:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_cpuid:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    cpuid # sched: [100:0.50]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_cpuid:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    cpuid # sched: [100:0.25]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  tail call void asm "cpuid", ""() nounwind
-  ret void
-}
-
-define void @test_dec8(i8 %a0, i8* %a1) optsize {
-; GENERIC-LABEL: test_dec8:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    decb %dil # sched: [1:0.33]
-; GENERIC-NEXT:    decb (%rsi) # sched: [7:1.00]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_dec8:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    decb %dil # sched: [1:0.50]
-; ATOM-NEXT:    decb (%rsi) # sched: [1:1.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_dec8:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    decb %dil # sched: [1:0.50]
-; SLM-NEXT:    decb (%rsi) # sched: [5:2.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_dec8:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    decb %dil # sched: [1:0.33]
-; SANDY-NEXT:    decb (%rsi) # sched: [7:1.00]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_dec8:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    decb %dil # sched: [1:0.25]
-; HASWELL-NEXT:    decb (%rsi) # sched: [7:1.00]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_dec8:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    decb %dil # sched: [1:0.25]
-; BROADWELL-NEXT:    decb (%rsi) # sched: [7:1.00]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_dec8:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    decb %dil # sched: [1:0.25]
-; SKYLAKE-NEXT:    decb (%rsi) # sched: [7:1.00]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_dec8:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    decb %dil # sched: [1:0.25]
-; SKX-NEXT:    decb (%rsi) # sched: [7:1.00]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_dec8:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    decb %dil # sched: [1:0.50]
-; BDVER2-NEXT:    decb (%rsi) # sched: [6:1.00]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_dec8:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    decb %dil # sched: [1:0.50]
-; BTVER2-NEXT:    decb (%rsi) # sched: [5:1.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_dec8:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    decb %dil # sched: [1:0.25]
-; ZNVER1-NEXT:    decb (%rsi) # sched: [5:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  tail call void asm "decb $0 \0A\09 decb $1", "r,*m"(i8 %a0, i8* %a1) nounwind
-  ret void
-}
-define void @test_dec16(i16 %a0, i16* %a1) optsize {
-; GENERIC-LABEL: test_dec16:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    decw %di # sched: [1:0.33]
-; GENERIC-NEXT:    decw (%rsi) # sched: [7:1.00]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_dec16:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    decw %di # sched: [1:0.50]
-; ATOM-NEXT:    decw (%rsi) # sched: [1:1.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_dec16:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    decw %di # sched: [1:0.50]
-; SLM-NEXT:    decw (%rsi) # sched: [5:2.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_dec16:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    decw %di # sched: [1:0.33]
-; SANDY-NEXT:    decw (%rsi) # sched: [7:1.00]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_dec16:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    decw %di # sched: [1:0.25]
-; HASWELL-NEXT:    decw (%rsi) # sched: [7:1.00]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_dec16:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    decw %di # sched: [1:0.25]
-; BROADWELL-NEXT:    decw (%rsi) # sched: [7:1.00]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_dec16:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    decw %di # sched: [1:0.25]
-; SKYLAKE-NEXT:    decw (%rsi) # sched: [7:1.00]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_dec16:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    decw %di # sched: [1:0.25]
-; SKX-NEXT:    decw (%rsi) # sched: [7:1.00]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_dec16:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    decw %di # sched: [1:0.50]
-; BDVER2-NEXT:    decw (%rsi) # sched: [6:1.00]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_dec16:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    decw %di # sched: [1:0.50]
-; BTVER2-NEXT:    decw (%rsi) # sched: [5:1.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_dec16:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    decw %di # sched: [1:0.25]
-; ZNVER1-NEXT:    decw (%rsi) # sched: [5:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  tail call void asm "decw $0 \0A\09 decw $1", "r,*m"(i16 %a0, i16* %a1) nounwind
-  ret void
-}
-define void @test_dec32(i32 %a0, i32* %a1) optsize {
-; GENERIC-LABEL: test_dec32:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    decl %edi # sched: [1:0.33]
-; GENERIC-NEXT:    decl (%rsi) # sched: [7:1.00]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_dec32:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    decl %edi # sched: [1:0.50]
-; ATOM-NEXT:    decl (%rsi) # sched: [1:1.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_dec32:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    decl %edi # sched: [1:0.50]
-; SLM-NEXT:    decl (%rsi) # sched: [5:2.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_dec32:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    decl %edi # sched: [1:0.33]
-; SANDY-NEXT:    decl (%rsi) # sched: [7:1.00]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_dec32:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    decl %edi # sched: [1:0.25]
-; HASWELL-NEXT:    decl (%rsi) # sched: [7:1.00]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_dec32:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    decl %edi # sched: [1:0.25]
-; BROADWELL-NEXT:    decl (%rsi) # sched: [7:1.00]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_dec32:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    decl %edi # sched: [1:0.25]
-; SKYLAKE-NEXT:    decl (%rsi) # sched: [7:1.00]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_dec32:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    decl %edi # sched: [1:0.25]
-; SKX-NEXT:    decl (%rsi) # sched: [7:1.00]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_dec32:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    decl %edi # sched: [1:0.50]
-; BDVER2-NEXT:    decl (%rsi) # sched: [6:1.00]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_dec32:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    decl %edi # sched: [1:0.50]
-; BTVER2-NEXT:    decl (%rsi) # sched: [5:1.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_dec32:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    decl %edi # sched: [1:0.25]
-; ZNVER1-NEXT:    decl (%rsi) # sched: [5:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  tail call void asm "decl $0 \0A\09 decl $1", "r,*m"(i32 %a0, i32* %a1) nounwind
-  ret void
-}
-define void @test_dec64(i64 %a0, i64* %a1) optsize {
-; GENERIC-LABEL: test_dec64:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    decq %rdi # sched: [1:0.33]
-; GENERIC-NEXT:    decq (%rsi) # sched: [7:1.00]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_dec64:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    decq %rdi # sched: [1:0.50]
-; ATOM-NEXT:    decq (%rsi) # sched: [1:1.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_dec64:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    decq %rdi # sched: [1:0.50]
-; SLM-NEXT:    decq (%rsi) # sched: [5:2.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_dec64:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    decq %rdi # sched: [1:0.33]
-; SANDY-NEXT:    decq (%rsi) # sched: [7:1.00]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_dec64:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    decq %rdi # sched: [1:0.25]
-; HASWELL-NEXT:    decq (%rsi) # sched: [7:1.00]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_dec64:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    decq %rdi # sched: [1:0.25]
-; BROADWELL-NEXT:    decq (%rsi) # sched: [7:1.00]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_dec64:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    decq %rdi # sched: [1:0.25]
-; SKYLAKE-NEXT:    decq (%rsi) # sched: [7:1.00]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_dec64:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    decq %rdi # sched: [1:0.25]
-; SKX-NEXT:    decq (%rsi) # sched: [7:1.00]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_dec64:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    decq %rdi # sched: [1:0.50]
-; BDVER2-NEXT:    decq (%rsi) # sched: [6:1.00]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_dec64:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    decq %rdi # sched: [1:0.50]
-; BTVER2-NEXT:    decq (%rsi) # sched: [5:1.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_dec64:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    decq %rdi # sched: [1:0.25]
-; ZNVER1-NEXT:    decq (%rsi) # sched: [5:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  tail call void asm "decq $0 \0A\09 decq $1", "r,*m"(i64 %a0, i64* %a1) nounwind
-  ret void
-}
-
-define void @test_div(i8 %a0, i16 %a1, i32 %a2, i64 %a3, i8 *%p0, i16 *%p1, i32 *%p2, i64 *%p3) optsize {
-; GENERIC-LABEL: test_div:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    movq {{[0-9]+}}(%rsp), %r10 # sched: [5:0.50]
-; GENERIC-NEXT:    movq {{[0-9]+}}(%rsp), %rax # sched: [5:0.50]
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    divb %dil # sched: [25:10.00]
-; GENERIC-NEXT:    divb (%r8) # sched: [30:10.00]
-; GENERIC-NEXT:    divw %si # sched: [25:10.00]
-; GENERIC-NEXT:    divw (%r9) # sched: [30:10.00]
-; GENERIC-NEXT:    divl %edx # sched: [25:10.00]
-; GENERIC-NEXT:    divl (%rax) # sched: [30:10.00]
-; GENERIC-NEXT:    divq %rcx # sched: [25:10.00]
-; GENERIC-NEXT:    divq (%r10) # sched: [30:10.00]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_div:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    movq {{[0-9]+}}(%rsp), %r10 # sched: [1:1.00]
-; ATOM-NEXT:    movq {{[0-9]+}}(%rsp), %rax # sched: [1:1.00]
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    divb %dil # sched: [50:25.00]
-; ATOM-NEXT:    divb (%r8) # sched: [68:34.00]
-; ATOM-NEXT:    divw %si # sched: [50:25.00]
-; ATOM-NEXT:    divw (%r9) # sched: [50:25.00]
-; ATOM-NEXT:    divl %edx # sched: [50:25.00]
-; ATOM-NEXT:    divl (%rax) # sched: [50:25.00]
-; ATOM-NEXT:    divq %rcx # sched: [130:65.00]
-; ATOM-NEXT:    divq (%r10) # sched: [130:65.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_div:
-; SLM:       # %bb.0:
-; SLM-NEXT:    movq {{[0-9]+}}(%rsp), %r10 # sched: [3:1.00]
-; SLM-NEXT:    movq {{[0-9]+}}(%rsp), %rax # sched: [3:1.00]
-; SLM-NEXT:    #APP
-; SLM-NEXT:    divb %dil # sched: [25:25.00]
-; SLM-NEXT:    divb (%r8) # sched: [29:25.00]
-; SLM-NEXT:    divw %si # sched: [25:25.00]
-; SLM-NEXT:    divw (%r9) # sched: [29:25.00]
-; SLM-NEXT:    divl %edx # sched: [25:25.00]
-; SLM-NEXT:    divl (%rax) # sched: [29:25.00]
-; SLM-NEXT:    divq %rcx # sched: [25:25.00]
-; SLM-NEXT:    divq (%r10) # sched: [29:25.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_div:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    movq {{[0-9]+}}(%rsp), %r10 # sched: [5:0.50]
-; SANDY-NEXT:    movq {{[0-9]+}}(%rsp), %rax # sched: [5:0.50]
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    divb %dil # sched: [25:10.00]
-; SANDY-NEXT:    divb (%r8) # sched: [30:10.00]
-; SANDY-NEXT:    divw %si # sched: [25:10.00]
-; SANDY-NEXT:    divw (%r9) # sched: [30:10.00]
-; SANDY-NEXT:    divl %edx # sched: [25:10.00]
-; SANDY-NEXT:    divl (%rax) # sched: [30:10.00]
-; SANDY-NEXT:    divq %rcx # sched: [25:10.00]
-; SANDY-NEXT:    divq (%r10) # sched: [30:10.00]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_div:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    movq {{[0-9]+}}(%rsp), %r10 # sched: [5:0.50]
-; HASWELL-NEXT:    movq {{[0-9]+}}(%rsp), %rax # sched: [5:0.50]
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    divb %dil # sched: [22:1.00]
-; HASWELL-NEXT:    divb (%r8) # sched: [29:10.00]
-; HASWELL-NEXT:    divw %si # sched: [98:8.00]
-; HASWELL-NEXT:    divw (%r9) # sched: [29:10.00]
-; HASWELL-NEXT:    divl %edx # sched: [98:8.00]
-; HASWELL-NEXT:    divl (%rax) # sched: [29:10.00]
-; HASWELL-NEXT:    divq %rcx # sched: [98:8.00]
-; HASWELL-NEXT:    divq (%r10) # sched: [29:10.00]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_div:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    movq {{[0-9]+}}(%rsp), %r10 # sched: [5:0.50]
-; BROADWELL-NEXT:    movq {{[0-9]+}}(%rsp), %rax # sched: [5:0.50]
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    divb %dil # sched: [25:10.00]
-; BROADWELL-NEXT:    divb (%r8) # sched: [34:2.00]
-; BROADWELL-NEXT:    divw %si # sched: [80:8.00]
-; BROADWELL-NEXT:    divw (%r9) # sched: [34:2.00]
-; BROADWELL-NEXT:    divl %edx # sched: [80:8.00]
-; BROADWELL-NEXT:    divl (%rax) # sched: [34:2.00]
-; BROADWELL-NEXT:    divq %rcx # sched: [80:8.00]
-; BROADWELL-NEXT:    divq (%r10) # sched: [34:2.00]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_div:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    movq {{[0-9]+}}(%rsp), %r10 # sched: [5:0.50]
-; SKYLAKE-NEXT:    movq {{[0-9]+}}(%rsp), %rax # sched: [5:0.50]
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    divb %dil # sched: [25:10.00]
-; SKYLAKE-NEXT:    divb (%r8) # sched: [29:10.00]
-; SKYLAKE-NEXT:    divw %si # sched: [76:8.00]
-; SKYLAKE-NEXT:    divw (%r9) # sched: [29:10.00]
-; SKYLAKE-NEXT:    divl %edx # sched: [76:8.00]
-; SKYLAKE-NEXT:    divl (%rax) # sched: [29:10.00]
-; SKYLAKE-NEXT:    divq %rcx # sched: [76:8.00]
-; SKYLAKE-NEXT:    divq (%r10) # sched: [29:10.00]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_div:
-; SKX:       # %bb.0:
-; SKX-NEXT:    movq {{[0-9]+}}(%rsp), %r10 # sched: [5:0.50]
-; SKX-NEXT:    movq {{[0-9]+}}(%rsp), %rax # sched: [5:0.50]
-; SKX-NEXT:    #APP
-; SKX-NEXT:    divb %dil # sched: [25:10.00]
-; SKX-NEXT:    divb (%r8) # sched: [29:10.00]
-; SKX-NEXT:    divw %si # sched: [76:8.00]
-; SKX-NEXT:    divw (%r9) # sched: [29:10.00]
-; SKX-NEXT:    divl %edx # sched: [76:8.00]
-; SKX-NEXT:    divl (%rax) # sched: [29:10.00]
-; SKX-NEXT:    divq %rcx # sched: [76:8.00]
-; SKX-NEXT:    divq (%r10) # sched: [29:10.00]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_div:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    movq {{[0-9]+}}(%rsp), %r10 # sched: [5:0.50]
-; BDVER2-NEXT:    movq {{[0-9]+}}(%rsp), %rax # sched: [5:0.50]
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    divb %dil # sched: [12:12.00]
-; BDVER2-NEXT:    divb (%r8) # sched: [16:12.00]
-; BDVER2-NEXT:    divw %si # sched: [15:15.00]
-; BDVER2-NEXT:    divw (%r9) # sched: [19:15.00]
-; BDVER2-NEXT:    divl %edx # sched: [14:14.00]
-; BDVER2-NEXT:    divl (%rax) # sched: [18:14.00]
-; BDVER2-NEXT:    divq %rcx # sched: [14:14.00]
-; BDVER2-NEXT:    divq (%r10) # sched: [18:14.00]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_div:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    movq {{[0-9]+}}(%rsp), %r10 # sched: [5:1.00]
-; BTVER2-NEXT:    movq {{[0-9]+}}(%rsp), %rax # sched: [5:1.00]
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    divb %dil # sched: [12:12.00]
-; BTVER2-NEXT:    divb (%r8) # sched: [15:12.00]
-; BTVER2-NEXT:    divw %si # sched: [17:17.00]
-; BTVER2-NEXT:    divw (%r9) # sched: [20:17.00]
-; BTVER2-NEXT:    divl %edx # sched: [25:25.00]
-; BTVER2-NEXT:    divl (%rax) # sched: [28:25.00]
-; BTVER2-NEXT:    divq %rcx # sched: [41:41.00]
-; BTVER2-NEXT:    divq (%r10) # sched: [44:41.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_div:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    movq {{[0-9]+}}(%rsp), %r10 # sched: [8:0.50]
-; ZNVER1-NEXT:    movq {{[0-9]+}}(%rsp), %rax # sched: [8:0.50]
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    divb %dil # sched: [15:15.00]
-; ZNVER1-NEXT:    divb (%r8) # sched: [19:15.00]
-; ZNVER1-NEXT:    divw %si # sched: [17:17.00]
-; ZNVER1-NEXT:    divw (%r9) # sched: [21:17.00]
-; ZNVER1-NEXT:    divl %edx # sched: [25:25.00]
-; ZNVER1-NEXT:    divl (%rax) # sched: [29:25.00]
-; ZNVER1-NEXT:    divq %rcx # sched: [41:41.00]
-; ZNVER1-NEXT:    divq (%r10) # sched: [45:41.00]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  tail call void asm "divb $0 \0A\09 divb $4 \0A\09 divw $1 \0A\09 divw $5 \0A\09 divl $2 \0A\09 divl $6 \0A\09 divq $3 \0A\09 divq $7", "r,r,r,r,*m,*m,*m,*m"(i8 %a0, i16 %a1, i32 %a2, i64 %a3, i8 *%p0, i16 *%p1, i32 *%p2, i64 *%p3) nounwind
-  ret void
-}
-
-define void @test_enter() optsize {
-; GENERIC-LABEL: test_enter:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    enter $7, $4095 # imm = 0xFFF
-; GENERIC-NEXT:    # sched: [100:0.33]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_enter:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    enter $7, $4095 # imm = 0xFFF
-; ATOM-NEXT:    # sched: [32:16.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_enter:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    enter $7, $4095 # imm = 0xFFF
-; SLM-NEXT:    # sched: [100:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_enter:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    enter $7, $4095 # imm = 0xFFF
-; SANDY-NEXT:    # sched: [100:0.33]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_enter:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    enter $7, $4095 # imm = 0xFFF
-; HASWELL-NEXT:    # sched: [100:0.25]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_enter:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    enter $7, $4095 # imm = 0xFFF
-; BROADWELL-NEXT:    # sched: [100:0.25]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_enter:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    enter $7, $4095 # imm = 0xFFF
-; SKYLAKE-NEXT:    # sched: [100:0.25]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_enter:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    enter $7, $4095 # imm = 0xFFF
-; SKX-NEXT:    # sched: [100:0.25]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_enter:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    enter $7, $4095 # imm = 0xFFF
-; BDVER2-NEXT:    # sched: [100:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_enter:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    enter $7, $4095 # imm = 0xFFF
-; BTVER2-NEXT:    # sched: [100:0.50]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_enter:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    enter $7, $4095 # imm = 0xFFF
-; ZNVER1-NEXT:    # sched: [100:0.25]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  tail call void asm "enter $0, $1", "i,i"(i8 7, i16 4095) nounwind
-  ret void
-}
-
-define void @test_idiv(i8 %a0, i16 %a1, i32 %a2, i64 %a3, i8 *%p0, i16 *%p1, i32 *%p2, i64 *%p3) optsize {
-; GENERIC-LABEL: test_idiv:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    movq {{[0-9]+}}(%rsp), %r10 # sched: [5:0.50]
-; GENERIC-NEXT:    movq {{[0-9]+}}(%rsp), %rax # sched: [5:0.50]
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    idivb %dil # sched: [25:10.00]
-; GENERIC-NEXT:    idivb (%r8) # sched: [30:10.00]
-; GENERIC-NEXT:    idivw %si # sched: [25:10.00]
-; GENERIC-NEXT:    idivw (%r9) # sched: [30:10.00]
-; GENERIC-NEXT:    idivl %edx # sched: [25:10.00]
-; GENERIC-NEXT:    idivl (%rax) # sched: [30:10.00]
-; GENERIC-NEXT:    idivq %rcx # sched: [25:10.00]
-; GENERIC-NEXT:    idivq (%r10) # sched: [30:10.00]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_idiv:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    movq {{[0-9]+}}(%rsp), %r10 # sched: [1:1.00]
-; ATOM-NEXT:    movq {{[0-9]+}}(%rsp), %rax # sched: [1:1.00]
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    idivb %dil # sched: [62:31.00]
-; ATOM-NEXT:    idivb (%r8) # sched: [62:31.00]
-; ATOM-NEXT:    idivw %si # sched: [62:31.00]
-; ATOM-NEXT:    idivw (%r9) # sched: [62:31.00]
-; ATOM-NEXT:    idivl %edx # sched: [62:31.00]
-; ATOM-NEXT:    idivl (%rax) # sched: [62:31.00]
-; ATOM-NEXT:    idivq %rcx # sched: [130:65.00]
-; ATOM-NEXT:    idivq (%r10) # sched: [130:65.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_idiv:
-; SLM:       # %bb.0:
-; SLM-NEXT:    movq {{[0-9]+}}(%rsp), %r10 # sched: [3:1.00]
-; SLM-NEXT:    movq {{[0-9]+}}(%rsp), %rax # sched: [3:1.00]
-; SLM-NEXT:    #APP
-; SLM-NEXT:    idivb %dil # sched: [25:25.00]
-; SLM-NEXT:    idivb (%r8) # sched: [29:25.00]
-; SLM-NEXT:    idivw %si # sched: [25:25.00]
-; SLM-NEXT:    idivw (%r9) # sched: [29:25.00]
-; SLM-NEXT:    idivl %edx # sched: [25:25.00]
-; SLM-NEXT:    idivl (%rax) # sched: [29:25.00]
-; SLM-NEXT:    idivq %rcx # sched: [25:25.00]
-; SLM-NEXT:    idivq (%r10) # sched: [29:25.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_idiv:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    movq {{[0-9]+}}(%rsp), %r10 # sched: [5:0.50]
-; SANDY-NEXT:    movq {{[0-9]+}}(%rsp), %rax # sched: [5:0.50]
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    idivb %dil # sched: [25:10.00]
-; SANDY-NEXT:    idivb (%r8) # sched: [30:10.00]
-; SANDY-NEXT:    idivw %si # sched: [25:10.00]
-; SANDY-NEXT:    idivw (%r9) # sched: [30:10.00]
-; SANDY-NEXT:    idivl %edx # sched: [25:10.00]
-; SANDY-NEXT:    idivl (%rax) # sched: [30:10.00]
-; SANDY-NEXT:    idivq %rcx # sched: [25:10.00]
-; SANDY-NEXT:    idivq (%r10) # sched: [30:10.00]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_idiv:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    movq {{[0-9]+}}(%rsp), %r10 # sched: [5:0.50]
-; HASWELL-NEXT:    movq {{[0-9]+}}(%rsp), %rax # sched: [5:0.50]
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    idivb %dil # sched: [23:1.00]
-; HASWELL-NEXT:    idivb (%r8) # sched: [29:10.00]
-; HASWELL-NEXT:    idivw %si # sched: [112:16.50]
-; HASWELL-NEXT:    idivw (%r9) # sched: [29:10.00]
-; HASWELL-NEXT:    idivl %edx # sched: [112:16.50]
-; HASWELL-NEXT:    idivl (%rax) # sched: [29:10.00]
-; HASWELL-NEXT:    idivq %rcx # sched: [112:16.50]
-; HASWELL-NEXT:    idivq (%r10) # sched: [29:10.00]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_idiv:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    movq {{[0-9]+}}(%rsp), %r10 # sched: [5:0.50]
-; BROADWELL-NEXT:    movq {{[0-9]+}}(%rsp), %rax # sched: [5:0.50]
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    idivb %dil # sched: [25:10.00]
-; BROADWELL-NEXT:    idivb (%r8) # sched: [35:2.00]
-; BROADWELL-NEXT:    idivw %si # sched: [25:10.00]
-; BROADWELL-NEXT:    idivw (%r9) # sched: [35:2.00]
-; BROADWELL-NEXT:    idivl %edx # sched: [25:10.00]
-; BROADWELL-NEXT:    idivl (%rax) # sched: [35:2.00]
-; BROADWELL-NEXT:    idivq %rcx # sched: [25:10.00]
-; BROADWELL-NEXT:    idivq (%r10) # sched: [35:2.00]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_idiv:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    movq {{[0-9]+}}(%rsp), %r10 # sched: [5:0.50]
-; SKYLAKE-NEXT:    movq {{[0-9]+}}(%rsp), %rax # sched: [5:0.50]
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    idivb %dil # sched: [25:10.00]
-; SKYLAKE-NEXT:    idivb (%r8) # sched: [28:4.00]
-; SKYLAKE-NEXT:    idivw %si # sched: [102:16.50]
-; SKYLAKE-NEXT:    idivw (%r9) # sched: [28:4.00]
-; SKYLAKE-NEXT:    idivl %edx # sched: [102:16.50]
-; SKYLAKE-NEXT:    idivl (%rax) # sched: [28:4.00]
-; SKYLAKE-NEXT:    idivq %rcx # sched: [102:16.50]
-; SKYLAKE-NEXT:    idivq (%r10) # sched: [28:4.00]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_idiv:
-; SKX:       # %bb.0:
-; SKX-NEXT:    movq {{[0-9]+}}(%rsp), %r10 # sched: [5:0.50]
-; SKX-NEXT:    movq {{[0-9]+}}(%rsp), %rax # sched: [5:0.50]
-; SKX-NEXT:    #APP
-; SKX-NEXT:    idivb %dil # sched: [25:10.00]
-; SKX-NEXT:    idivb (%r8) # sched: [28:4.00]
-; SKX-NEXT:    idivw %si # sched: [102:16.50]
-; SKX-NEXT:    idivw (%r9) # sched: [28:4.00]
-; SKX-NEXT:    idivl %edx # sched: [102:16.50]
-; SKX-NEXT:    idivl (%rax) # sched: [28:4.00]
-; SKX-NEXT:    idivq %rcx # sched: [102:16.50]
-; SKX-NEXT:    idivq (%r10) # sched: [28:4.00]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_idiv:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    movq {{[0-9]+}}(%rsp), %r10 # sched: [5:0.50]
-; BDVER2-NEXT:    movq {{[0-9]+}}(%rsp), %rax # sched: [5:0.50]
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    idivb %dil # sched: [12:12.00]
-; BDVER2-NEXT:    idivb (%r8) # sched: [16:12.00]
-; BDVER2-NEXT:    idivw %si # sched: [15:17.00]
-; BDVER2-NEXT:    idivw (%r9) # sched: [19:17.00]
-; BDVER2-NEXT:    idivl %edx # sched: [14:25.00]
-; BDVER2-NEXT:    idivl (%rax) # sched: [18:25.00]
-; BDVER2-NEXT:    idivq %rcx # sched: [14:14.00]
-; BDVER2-NEXT:    idivq (%r10) # sched: [18:14.00]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_idiv:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    movq {{[0-9]+}}(%rsp), %r10 # sched: [5:1.00]
-; BTVER2-NEXT:    movq {{[0-9]+}}(%rsp), %rax # sched: [5:1.00]
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    idivb %dil # sched: [12:12.00]
-; BTVER2-NEXT:    idivb (%r8) # sched: [15:12.00]
-; BTVER2-NEXT:    idivw %si # sched: [17:17.00]
-; BTVER2-NEXT:    idivw (%r9) # sched: [20:17.00]
-; BTVER2-NEXT:    idivl %edx # sched: [25:25.00]
-; BTVER2-NEXT:    idivl (%rax) # sched: [28:25.00]
-; BTVER2-NEXT:    idivq %rcx # sched: [41:41.00]
-; BTVER2-NEXT:    idivq (%r10) # sched: [44:41.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_idiv:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    movq {{[0-9]+}}(%rsp), %r10 # sched: [8:0.50]
-; ZNVER1-NEXT:    movq {{[0-9]+}}(%rsp), %rax # sched: [8:0.50]
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    idivb %dil # sched: [15:15.00]
-; ZNVER1-NEXT:    idivb (%r8) # sched: [19:15.00]
-; ZNVER1-NEXT:    idivw %si # sched: [17:17.00]
-; ZNVER1-NEXT:    idivw (%r9) # sched: [21:17.00]
-; ZNVER1-NEXT:    idivl %edx # sched: [25:25.00]
-; ZNVER1-NEXT:    idivl (%rax) # sched: [29:25.00]
-; ZNVER1-NEXT:    idivq %rcx # sched: [41:41.00]
-; ZNVER1-NEXT:    idivq (%r10) # sched: [45:41.00]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  tail call void asm "idivb $0 \0A\09 idivb $4 \0A\09 idivw $1 \0A\09 idivw $5 \0A\09 idivl $2 \0A\09 idivl $6 \0A\09 idivq $3 \0A\09 idivq $7", "r,r,r,r,*m,*m,*m,*m"(i8 %a0, i16 %a1, i32 %a2, i64 %a3, i8 *%p0, i16 *%p1, i32 *%p2, i64 *%p3) nounwind
-  ret void
-}
-
-define void @test_imul_8(i8 %a0, i8* %a1) optsize {
-; GENERIC-LABEL: test_imul_8:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    imulb %dil # sched: [3:1.00]
-; GENERIC-NEXT:    imulb (%rsi) # sched: [8:1.00]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_imul_8:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    imulb %dil # sched: [7:3.50]
-; ATOM-NEXT:    imulb (%rsi) # sched: [7:3.50]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_imul_8:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    imulb %dil # sched: [3:1.00]
-; SLM-NEXT:    imulb (%rsi) # sched: [6:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_imul_8:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    imulb %dil # sched: [3:1.00]
-; SANDY-NEXT:    imulb (%rsi) # sched: [8:1.00]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_imul_8:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    imulb %dil # sched: [3:1.00]
-; HASWELL-NEXT:    imulb (%rsi) # sched: [8:1.00]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_imul_8:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    imulb %dil # sched: [3:1.00]
-; BROADWELL-NEXT:    imulb (%rsi) # sched: [8:1.00]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_imul_8:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    imulb %dil # sched: [3:1.00]
-; SKYLAKE-NEXT:    imulb (%rsi) # sched: [8:1.00]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_imul_8:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    imulb %dil # sched: [3:1.00]
-; SKX-NEXT:    imulb (%rsi) # sched: [8:1.00]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_imul_8:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    imulb %dil # sched: [4:1.00]
-; BDVER2-NEXT:    imulb (%rsi) # sched: [8:1.00]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_imul_8:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    imulb %dil # sched: [3:1.00]
-; BTVER2-NEXT:    imulb (%rsi) # sched: [6:1.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_imul_8:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    imulb %dil # sched: [4:1.00]
-; ZNVER1-NEXT:    imulb (%rsi) # sched: [8:1.00]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  tail call void asm "imulb $0 \0A\09 imulb $1", "r,*m"(i8 %a0, i8* %a1) nounwind
-  ret void
-}
-define void @test_imul_16(i16 %a0, i16* %a1, i16 %a2) optsize {
-; GENERIC-LABEL: test_imul_16:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    imulw %di # sched: [4:1.33]
-; GENERIC-NEXT:    imulw (%rsi) # sched: [9:1.33]
-; GENERIC-NEXT:    imulw %dx, %di # sched: [3:1.00]
-; GENERIC-NEXT:    imulw (%rsi), %di # sched: [8:1.00]
-; GENERIC-NEXT:    imulw $511, %di, %di # imm = 0x1FF
-; GENERIC-NEXT:    # sched: [4:1.00]
-; GENERIC-NEXT:    imulw $511, (%rsi), %di # imm = 0x1FF
-; GENERIC-NEXT:    # sched: [8:1.00]
-; GENERIC-NEXT:    imulw $7, %di, %di # sched: [4:1.00]
-; GENERIC-NEXT:    imulw $7, (%rsi), %di # sched: [8:1.00]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_imul_16:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    imulw %di # sched: [7:3.50]
-; ATOM-NEXT:    imulw (%rsi) # sched: [8:4.00]
-; ATOM-NEXT:    imulw %dx, %di # sched: [6:3.00]
-; ATOM-NEXT:    imulw (%rsi), %di # sched: [7:3.50]
-; ATOM-NEXT:    imulw $511, %di, %di # imm = 0x1FF
-; ATOM-NEXT:    # sched: [6:3.00]
-; ATOM-NEXT:    imulw $511, (%rsi), %di # imm = 0x1FF
-; ATOM-NEXT:    # sched: [7:3.50]
-; ATOM-NEXT:    imulw $7, %di, %di # sched: [6:3.00]
-; ATOM-NEXT:    imulw $7, (%rsi), %di # sched: [7:3.50]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_imul_16:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    imulw %di # sched: [3:1.00]
-; SLM-NEXT:    imulw (%rsi) # sched: [6:1.00]
-; SLM-NEXT:    imulw %dx, %di # sched: [3:1.00]
-; SLM-NEXT:    imulw (%rsi), %di # sched: [6:1.00]
-; SLM-NEXT:    imulw $511, %di, %di # imm = 0x1FF
-; SLM-NEXT:    # sched: [3:1.00]
-; SLM-NEXT:    imulw $511, (%rsi), %di # imm = 0x1FF
-; SLM-NEXT:    # sched: [6:1.00]
-; SLM-NEXT:    imulw $7, %di, %di # sched: [3:1.00]
-; SLM-NEXT:    imulw $7, (%rsi), %di # sched: [6:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_imul_16:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    imulw %di # sched: [4:1.33]
-; SANDY-NEXT:    imulw (%rsi) # sched: [9:1.33]
-; SANDY-NEXT:    imulw %dx, %di # sched: [3:1.00]
-; SANDY-NEXT:    imulw (%rsi), %di # sched: [8:1.00]
-; SANDY-NEXT:    imulw $511, %di, %di # imm = 0x1FF
-; SANDY-NEXT:    # sched: [4:1.00]
-; SANDY-NEXT:    imulw $511, (%rsi), %di # imm = 0x1FF
-; SANDY-NEXT:    # sched: [8:1.00]
-; SANDY-NEXT:    imulw $7, %di, %di # sched: [4:1.00]
-; SANDY-NEXT:    imulw $7, (%rsi), %di # sched: [8:1.00]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_imul_16:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    imulw %di # sched: [4:1.00]
-; HASWELL-NEXT:    imulw (%rsi) # sched: [9:1.00]
-; HASWELL-NEXT:    imulw %dx, %di # sched: [3:1.00]
-; HASWELL-NEXT:    imulw (%rsi), %di # sched: [8:1.00]
-; HASWELL-NEXT:    imulw $511, %di, %di # imm = 0x1FF
-; HASWELL-NEXT:    # sched: [4:1.00]
-; HASWELL-NEXT:    imulw $511, (%rsi), %di # imm = 0x1FF
-; HASWELL-NEXT:    # sched: [8:1.00]
-; HASWELL-NEXT:    imulw $7, %di, %di # sched: [4:1.00]
-; HASWELL-NEXT:    imulw $7, (%rsi), %di # sched: [8:1.00]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_imul_16:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    imulw %di # sched: [4:1.00]
-; BROADWELL-NEXT:    imulw (%rsi) # sched: [9:1.00]
-; BROADWELL-NEXT:    imulw %dx, %di # sched: [3:1.00]
-; BROADWELL-NEXT:    imulw (%rsi), %di # sched: [8:1.00]
-; BROADWELL-NEXT:    imulw $511, %di, %di # imm = 0x1FF
-; BROADWELL-NEXT:    # sched: [4:1.00]
-; BROADWELL-NEXT:    imulw $511, (%rsi), %di # imm = 0x1FF
-; BROADWELL-NEXT:    # sched: [8:1.00]
-; BROADWELL-NEXT:    imulw $7, %di, %di # sched: [4:1.00]
-; BROADWELL-NEXT:    imulw $7, (%rsi), %di # sched: [8:1.00]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_imul_16:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    imulw %di # sched: [4:1.00]
-; SKYLAKE-NEXT:    imulw (%rsi) # sched: [9:1.00]
-; SKYLAKE-NEXT:    imulw %dx, %di # sched: [3:1.00]
-; SKYLAKE-NEXT:    imulw (%rsi), %di # sched: [8:1.00]
-; SKYLAKE-NEXT:    imulw $511, %di, %di # imm = 0x1FF
-; SKYLAKE-NEXT:    # sched: [4:1.00]
-; SKYLAKE-NEXT:    imulw $511, (%rsi), %di # imm = 0x1FF
-; SKYLAKE-NEXT:    # sched: [8:1.00]
-; SKYLAKE-NEXT:    imulw $7, %di, %di # sched: [4:1.00]
-; SKYLAKE-NEXT:    imulw $7, (%rsi), %di # sched: [8:1.00]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_imul_16:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    imulw %di # sched: [4:1.00]
-; SKX-NEXT:    imulw (%rsi) # sched: [9:1.00]
-; SKX-NEXT:    imulw %dx, %di # sched: [3:1.00]
-; SKX-NEXT:    imulw (%rsi), %di # sched: [8:1.00]
-; SKX-NEXT:    imulw $511, %di, %di # imm = 0x1FF
-; SKX-NEXT:    # sched: [4:1.00]
-; SKX-NEXT:    imulw $511, (%rsi), %di # imm = 0x1FF
-; SKX-NEXT:    # sched: [8:1.00]
-; SKX-NEXT:    imulw $7, %di, %di # sched: [4:1.00]
-; SKX-NEXT:    imulw $7, (%rsi), %di # sched: [8:1.00]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_imul_16:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    imulw %di # sched: [4:1.00]
-; BDVER2-NEXT:    imulw (%rsi) # sched: [8:1.00]
-; BDVER2-NEXT:    imulw %dx, %di # sched: [4:1.00]
-; BDVER2-NEXT:    imulw (%rsi), %di # sched: [8:1.00]
-; BDVER2-NEXT:    imulw $511, %di, %di # imm = 0x1FF
-; BDVER2-NEXT:    # sched: [5:1.00]
-; BDVER2-NEXT:    imulw $511, (%rsi), %di # imm = 0x1FF
-; BDVER2-NEXT:    # sched: [9:1.00]
-; BDVER2-NEXT:    imulw $7, %di, %di # sched: [5:1.00]
-; BDVER2-NEXT:    imulw $7, (%rsi), %di # sched: [9:1.00]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_imul_16:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    imulw %di # sched: [3:1.00]
-; BTVER2-NEXT:    imulw (%rsi) # sched: [6:1.00]
-; BTVER2-NEXT:    imulw %dx, %di # sched: [3:1.00]
-; BTVER2-NEXT:    imulw (%rsi), %di # sched: [6:1.00]
-; BTVER2-NEXT:    imulw $511, %di, %di # imm = 0x1FF
-; BTVER2-NEXT:    # sched: [3:1.00]
-; BTVER2-NEXT:    imulw $511, (%rsi), %di # imm = 0x1FF
-; BTVER2-NEXT:    # sched: [6:1.00]
-; BTVER2-NEXT:    imulw $7, %di, %di # sched: [3:1.00]
-; BTVER2-NEXT:    imulw $7, (%rsi), %di # sched: [6:1.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_imul_16:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    imulw %di # sched: [3:1.00]
-; ZNVER1-NEXT:    imulw (%rsi) # sched: [8:1.00]
-; ZNVER1-NEXT:    imulw %dx, %di # sched: [3:1.00]
-; ZNVER1-NEXT:    imulw (%rsi), %di # sched: [3:1.00]
-; ZNVER1-NEXT:    imulw $511, %di, %di # imm = 0x1FF
-; ZNVER1-NEXT:    # sched: [3:1.00]
-; ZNVER1-NEXT:    imulw $511, (%rsi), %di # imm = 0x1FF
-; ZNVER1-NEXT:    # sched: [3:1.00]
-; ZNVER1-NEXT:    imulw $7, %di, %di # sched: [3:1.00]
-; ZNVER1-NEXT:    imulw $7, (%rsi), %di # sched: [3:1.00]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  tail call void asm "imulw $0 \0A\09 imulw $2 \0A\09 imulw $1, $0 \0A\09 imulw $2, $0 \0A\09 imulw $3, $0, $0 \0A\09 imulw $3, $2, $0 \0A\09 imulw $4, $0, $0 \0A\09 imulw $4, $2, $0", "r,r,*m,i,i"(i16 %a0, i16 %a2, i16* %a1, i16 511, i8 7) nounwind
-  ret void
-}
-define void @test_imul_32(i32 %a0, i32* %a1, i32 %a2) optsize {
-; GENERIC-LABEL: test_imul_32:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    imull %edi # sched: [4:1.00]
-; GENERIC-NEXT:    imull (%rsi) # sched: [9:1.00]
-; GENERIC-NEXT:    imull %edx, %edi # sched: [3:1.00]
-; GENERIC-NEXT:    imull (%rsi), %edi # sched: [8:1.00]
-; GENERIC-NEXT:    imull $665536, %edi, %edi # imm = 0xA27C0
-; GENERIC-NEXT:    # sched: [3:1.00]
-; GENERIC-NEXT:    imull $665536, (%rsi), %edi # imm = 0xA27C0
-; GENERIC-NEXT:    # sched: [8:1.00]
-; GENERIC-NEXT:    imull $7, %edi, %edi # sched: [3:1.00]
-; GENERIC-NEXT:    imull $7, (%rsi), %edi # sched: [8:1.00]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_imul_32:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    imull %edi # sched: [6:3.00]
-; ATOM-NEXT:    imull (%rsi) # sched: [7:3.50]
-; ATOM-NEXT:    imull %edx, %edi # sched: [5:5.00]
-; ATOM-NEXT:    imull (%rsi), %edi # sched: [5:5.00]
-; ATOM-NEXT:    imull $665536, %edi, %edi # imm = 0xA27C0
-; ATOM-NEXT:    # sched: [5:5.00]
-; ATOM-NEXT:    imull $665536, (%rsi), %edi # imm = 0xA27C0
-; ATOM-NEXT:    # sched: [5:5.00]
-; ATOM-NEXT:    imull $7, %edi, %edi # sched: [5:5.00]
-; ATOM-NEXT:    imull $7, (%rsi), %edi # sched: [5:5.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_imul_32:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    imull %edi # sched: [3:1.00]
-; SLM-NEXT:    imull (%rsi) # sched: [6:1.00]
-; SLM-NEXT:    imull %edx, %edi # sched: [3:1.00]
-; SLM-NEXT:    imull (%rsi), %edi # sched: [6:1.00]
-; SLM-NEXT:    imull $665536, %edi, %edi # imm = 0xA27C0
-; SLM-NEXT:    # sched: [3:1.00]
-; SLM-NEXT:    imull $665536, (%rsi), %edi # imm = 0xA27C0
-; SLM-NEXT:    # sched: [6:1.00]
-; SLM-NEXT:    imull $7, %edi, %edi # sched: [3:1.00]
-; SLM-NEXT:    imull $7, (%rsi), %edi # sched: [6:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_imul_32:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    imull %edi # sched: [4:1.00]
-; SANDY-NEXT:    imull (%rsi) # sched: [9:1.00]
-; SANDY-NEXT:    imull %edx, %edi # sched: [3:1.00]
-; SANDY-NEXT:    imull (%rsi), %edi # sched: [8:1.00]
-; SANDY-NEXT:    imull $665536, %edi, %edi # imm = 0xA27C0
-; SANDY-NEXT:    # sched: [3:1.00]
-; SANDY-NEXT:    imull $665536, (%rsi), %edi # imm = 0xA27C0
-; SANDY-NEXT:    # sched: [8:1.00]
-; SANDY-NEXT:    imull $7, %edi, %edi # sched: [3:1.00]
-; SANDY-NEXT:    imull $7, (%rsi), %edi # sched: [8:1.00]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_imul_32:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    imull %edi # sched: [4:1.00]
-; HASWELL-NEXT:    imull (%rsi) # sched: [9:1.00]
-; HASWELL-NEXT:    imull %edx, %edi # sched: [3:1.00]
-; HASWELL-NEXT:    imull (%rsi), %edi # sched: [8:1.00]
-; HASWELL-NEXT:    imull $665536, %edi, %edi # imm = 0xA27C0
-; HASWELL-NEXT:    # sched: [3:1.00]
-; HASWELL-NEXT:    imull $665536, (%rsi), %edi # imm = 0xA27C0
-; HASWELL-NEXT:    # sched: [8:1.00]
-; HASWELL-NEXT:    imull $7, %edi, %edi # sched: [3:1.00]
-; HASWELL-NEXT:    imull $7, (%rsi), %edi # sched: [8:1.00]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_imul_32:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    imull %edi # sched: [4:1.00]
-; BROADWELL-NEXT:    imull (%rsi) # sched: [9:1.00]
-; BROADWELL-NEXT:    imull %edx, %edi # sched: [3:1.00]
-; BROADWELL-NEXT:    imull (%rsi), %edi # sched: [8:1.00]
-; BROADWELL-NEXT:    imull $665536, %edi, %edi # imm = 0xA27C0
-; BROADWELL-NEXT:    # sched: [3:1.00]
-; BROADWELL-NEXT:    imull $665536, (%rsi), %edi # imm = 0xA27C0
-; BROADWELL-NEXT:    # sched: [8:1.00]
-; BROADWELL-NEXT:    imull $7, %edi, %edi # sched: [3:1.00]
-; BROADWELL-NEXT:    imull $7, (%rsi), %edi # sched: [8:1.00]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_imul_32:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    imull %edi # sched: [4:1.00]
-; SKYLAKE-NEXT:    imull (%rsi) # sched: [9:1.00]
-; SKYLAKE-NEXT:    imull %edx, %edi # sched: [3:1.00]
-; SKYLAKE-NEXT:    imull (%rsi), %edi # sched: [8:1.00]
-; SKYLAKE-NEXT:    imull $665536, %edi, %edi # imm = 0xA27C0
-; SKYLAKE-NEXT:    # sched: [3:1.00]
-; SKYLAKE-NEXT:    imull $665536, (%rsi), %edi # imm = 0xA27C0
-; SKYLAKE-NEXT:    # sched: [8:1.00]
-; SKYLAKE-NEXT:    imull $7, %edi, %edi # sched: [3:1.00]
-; SKYLAKE-NEXT:    imull $7, (%rsi), %edi # sched: [8:1.00]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_imul_32:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    imull %edi # sched: [4:1.00]
-; SKX-NEXT:    imull (%rsi) # sched: [9:1.00]
-; SKX-NEXT:    imull %edx, %edi # sched: [3:1.00]
-; SKX-NEXT:    imull (%rsi), %edi # sched: [8:1.00]
-; SKX-NEXT:    imull $665536, %edi, %edi # imm = 0xA27C0
-; SKX-NEXT:    # sched: [3:1.00]
-; SKX-NEXT:    imull $665536, (%rsi), %edi # imm = 0xA27C0
-; SKX-NEXT:    # sched: [8:1.00]
-; SKX-NEXT:    imull $7, %edi, %edi # sched: [3:1.00]
-; SKX-NEXT:    imull $7, (%rsi), %edi # sched: [8:1.00]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_imul_32:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    imull %edi # sched: [4:1.00]
-; BDVER2-NEXT:    imull (%rsi) # sched: [8:1.00]
-; BDVER2-NEXT:    imull %edx, %edi # sched: [4:1.00]
-; BDVER2-NEXT:    imull (%rsi), %edi # sched: [8:1.00]
-; BDVER2-NEXT:    imull $665536, %edi, %edi # imm = 0xA27C0
-; BDVER2-NEXT:    # sched: [4:1.00]
-; BDVER2-NEXT:    imull $665536, (%rsi), %edi # imm = 0xA27C0
-; BDVER2-NEXT:    # sched: [8:1.00]
-; BDVER2-NEXT:    imull $7, %edi, %edi # sched: [4:1.00]
-; BDVER2-NEXT:    imull $7, (%rsi), %edi # sched: [8:1.00]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_imul_32:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    imull %edi # sched: [3:1.00]
-; BTVER2-NEXT:    imull (%rsi) # sched: [6:1.00]
-; BTVER2-NEXT:    imull %edx, %edi # sched: [3:1.00]
-; BTVER2-NEXT:    imull (%rsi), %edi # sched: [6:1.00]
-; BTVER2-NEXT:    imull $665536, %edi, %edi # imm = 0xA27C0
-; BTVER2-NEXT:    # sched: [3:1.00]
-; BTVER2-NEXT:    imull $665536, (%rsi), %edi # imm = 0xA27C0
-; BTVER2-NEXT:    # sched: [6:1.00]
-; BTVER2-NEXT:    imull $7, %edi, %edi # sched: [3:1.00]
-; BTVER2-NEXT:    imull $7, (%rsi), %edi # sched: [6:1.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_imul_32:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    imull %edi # sched: [3:1.00]
-; ZNVER1-NEXT:    imull (%rsi) # sched: [8:1.00]
-; ZNVER1-NEXT:    imull %edx, %edi # sched: [3:1.00]
-; ZNVER1-NEXT:    imull (%rsi), %edi # sched: [3:1.00]
-; ZNVER1-NEXT:    imull $665536, %edi, %edi # imm = 0xA27C0
-; ZNVER1-NEXT:    # sched: [3:1.00]
-; ZNVER1-NEXT:    imull $665536, (%rsi), %edi # imm = 0xA27C0
-; ZNVER1-NEXT:    # sched: [3:1.00]
-; ZNVER1-NEXT:    imull $7, %edi, %edi # sched: [3:1.00]
-; ZNVER1-NEXT:    imull $7, (%rsi), %edi # sched: [3:1.00]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  tail call void asm "imull $0 \0A\09 imull $2 \0A\09 imull $1, $0 \0A\09 imull $2, $0 \0A\09 imull $3, $0, $0 \0A\09 imull $3, $2, $0 \0A\09 imull $4, $0, $0 \0A\09 imull $4, $2, $0", "r,r,*m,i,i"(i32 %a0, i32 %a2, i32* %a1, i32 665536, i8 7) nounwind
-  ret void
-}
-define void @test_imul_64(i64 %a0, i64* %a1, i64 %a2) optsize {
-; GENERIC-LABEL: test_imul_64:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    imulq %rdi # sched: [4:1.00]
-; GENERIC-NEXT:    imulq (%rsi) # sched: [9:1.00]
-; GENERIC-NEXT:    imulq %rdx, %rdi # sched: [3:1.00]
-; GENERIC-NEXT:    imulq (%rsi), %rdi # sched: [8:1.00]
-; GENERIC-NEXT:    imulq $665536, %rdi, %rdi # imm = 0xA27C0
-; GENERIC-NEXT:    # sched: [3:1.00]
-; GENERIC-NEXT:    imulq $665536, (%rsi), %rdi # imm = 0xA27C0
-; GENERIC-NEXT:    # sched: [8:1.00]
-; GENERIC-NEXT:    imulq $7, %rdi, %rdi # sched: [3:1.00]
-; GENERIC-NEXT:    imulq $7, (%rsi), %rdi # sched: [8:1.00]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_imul_64:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    imulq %rdi # sched: [12:6.00]
-; ATOM-NEXT:    imulq (%rsi) # sched: [12:6.00]
-; ATOM-NEXT:    imulq %rdx, %rdi # sched: [12:6.00]
-; ATOM-NEXT:    imulq (%rsi), %rdi # sched: [12:6.00]
-; ATOM-NEXT:    imulq $665536, %rdi, %rdi # imm = 0xA27C0
-; ATOM-NEXT:    # sched: [14:7.00]
-; ATOM-NEXT:    imulq $665536, (%rsi), %rdi # imm = 0xA27C0
-; ATOM-NEXT:    # sched: [14:7.00]
-; ATOM-NEXT:    imulq $7, %rdi, %rdi # sched: [14:7.00]
-; ATOM-NEXT:    imulq $7, (%rsi), %rdi # sched: [14:7.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_imul_64:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    imulq %rdi # sched: [3:1.00]
-; SLM-NEXT:    imulq (%rsi) # sched: [6:1.00]
-; SLM-NEXT:    imulq %rdx, %rdi # sched: [3:1.00]
-; SLM-NEXT:    imulq (%rsi), %rdi # sched: [6:1.00]
-; SLM-NEXT:    imulq $665536, %rdi, %rdi # imm = 0xA27C0
-; SLM-NEXT:    # sched: [3:1.00]
-; SLM-NEXT:    imulq $665536, (%rsi), %rdi # imm = 0xA27C0
-; SLM-NEXT:    # sched: [6:1.00]
-; SLM-NEXT:    imulq $7, %rdi, %rdi # sched: [3:1.00]
-; SLM-NEXT:    imulq $7, (%rsi), %rdi # sched: [6:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_imul_64:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    imulq %rdi # sched: [4:1.00]
-; SANDY-NEXT:    imulq (%rsi) # sched: [9:1.00]
-; SANDY-NEXT:    imulq %rdx, %rdi # sched: [3:1.00]
-; SANDY-NEXT:    imulq (%rsi), %rdi # sched: [8:1.00]
-; SANDY-NEXT:    imulq $665536, %rdi, %rdi # imm = 0xA27C0
-; SANDY-NEXT:    # sched: [3:1.00]
-; SANDY-NEXT:    imulq $665536, (%rsi), %rdi # imm = 0xA27C0
-; SANDY-NEXT:    # sched: [8:1.00]
-; SANDY-NEXT:    imulq $7, %rdi, %rdi # sched: [3:1.00]
-; SANDY-NEXT:    imulq $7, (%rsi), %rdi # sched: [8:1.00]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_imul_64:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    imulq %rdi # sched: [4:1.00]
-; HASWELL-NEXT:    imulq (%rsi) # sched: [9:1.00]
-; HASWELL-NEXT:    imulq %rdx, %rdi # sched: [3:1.00]
-; HASWELL-NEXT:    imulq (%rsi), %rdi # sched: [8:1.00]
-; HASWELL-NEXT:    imulq $665536, %rdi, %rdi # imm = 0xA27C0
-; HASWELL-NEXT:    # sched: [3:1.00]
-; HASWELL-NEXT:    imulq $665536, (%rsi), %rdi # imm = 0xA27C0
-; HASWELL-NEXT:    # sched: [8:1.00]
-; HASWELL-NEXT:    imulq $7, %rdi, %rdi # sched: [3:1.00]
-; HASWELL-NEXT:    imulq $7, (%rsi), %rdi # sched: [8:1.00]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_imul_64:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    imulq %rdi # sched: [4:1.00]
-; BROADWELL-NEXT:    imulq (%rsi) # sched: [9:1.00]
-; BROADWELL-NEXT:    imulq %rdx, %rdi # sched: [3:1.00]
-; BROADWELL-NEXT:    imulq (%rsi), %rdi # sched: [8:1.00]
-; BROADWELL-NEXT:    imulq $665536, %rdi, %rdi # imm = 0xA27C0
-; BROADWELL-NEXT:    # sched: [3:1.00]
-; BROADWELL-NEXT:    imulq $665536, (%rsi), %rdi # imm = 0xA27C0
-; BROADWELL-NEXT:    # sched: [8:1.00]
-; BROADWELL-NEXT:    imulq $7, %rdi, %rdi # sched: [3:1.00]
-; BROADWELL-NEXT:    imulq $7, (%rsi), %rdi # sched: [8:1.00]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_imul_64:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    imulq %rdi # sched: [4:1.00]
-; SKYLAKE-NEXT:    imulq (%rsi) # sched: [9:1.00]
-; SKYLAKE-NEXT:    imulq %rdx, %rdi # sched: [3:1.00]
-; SKYLAKE-NEXT:    imulq (%rsi), %rdi # sched: [8:1.00]
-; SKYLAKE-NEXT:    imulq $665536, %rdi, %rdi # imm = 0xA27C0
-; SKYLAKE-NEXT:    # sched: [3:1.00]
-; SKYLAKE-NEXT:    imulq $665536, (%rsi), %rdi # imm = 0xA27C0
-; SKYLAKE-NEXT:    # sched: [8:1.00]
-; SKYLAKE-NEXT:    imulq $7, %rdi, %rdi # sched: [3:1.00]
-; SKYLAKE-NEXT:    imulq $7, (%rsi), %rdi # sched: [8:1.00]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_imul_64:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    imulq %rdi # sched: [4:1.00]
-; SKX-NEXT:    imulq (%rsi) # sched: [9:1.00]
-; SKX-NEXT:    imulq %rdx, %rdi # sched: [3:1.00]
-; SKX-NEXT:    imulq (%rsi), %rdi # sched: [8:1.00]
-; SKX-NEXT:    imulq $665536, %rdi, %rdi # imm = 0xA27C0
-; SKX-NEXT:    # sched: [3:1.00]
-; SKX-NEXT:    imulq $665536, (%rsi), %rdi # imm = 0xA27C0
-; SKX-NEXT:    # sched: [8:1.00]
-; SKX-NEXT:    imulq $7, %rdi, %rdi # sched: [3:1.00]
-; SKX-NEXT:    imulq $7, (%rsi), %rdi # sched: [8:1.00]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_imul_64:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    imulq %rdi # sched: [6:4.00]
-; BDVER2-NEXT:    imulq (%rsi) # sched: [10:4.00]
-; BDVER2-NEXT:    imulq %rdx, %rdi # sched: [6:4.00]
-; BDVER2-NEXT:    imulq (%rsi), %rdi # sched: [10:4.00]
-; BDVER2-NEXT:    imulq $665536, %rdi, %rdi # imm = 0xA27C0
-; BDVER2-NEXT:    # sched: [6:4.00]
-; BDVER2-NEXT:    imulq $665536, (%rsi), %rdi # imm = 0xA27C0
-; BDVER2-NEXT:    # sched: [10:4.00]
-; BDVER2-NEXT:    imulq $7, %rdi, %rdi # sched: [6:4.00]
-; BDVER2-NEXT:    imulq $7, (%rsi), %rdi # sched: [10:4.00]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_imul_64:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    imulq %rdi # sched: [6:4.00]
-; BTVER2-NEXT:    imulq (%rsi) # sched: [9:4.00]
-; BTVER2-NEXT:    imulq %rdx, %rdi # sched: [6:4.00]
-; BTVER2-NEXT:    imulq (%rsi), %rdi # sched: [9:4.00]
-; BTVER2-NEXT:    imulq $665536, %rdi, %rdi # imm = 0xA27C0
-; BTVER2-NEXT:    # sched: [6:4.00]
-; BTVER2-NEXT:    imulq $665536, (%rsi), %rdi # imm = 0xA27C0
-; BTVER2-NEXT:    # sched: [9:4.00]
-; BTVER2-NEXT:    imulq $7, %rdi, %rdi # sched: [6:4.00]
-; BTVER2-NEXT:    imulq $7, (%rsi), %rdi # sched: [9:4.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_imul_64:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    imulq %rdi # sched: [4:1.00]
-; ZNVER1-NEXT:    imulq (%rsi) # sched: [9:1.00]
-; ZNVER1-NEXT:    imulq %rdx, %rdi # sched: [4:1.00]
-; ZNVER1-NEXT:    imulq (%rsi), %rdi # sched: [4:1.00]
-; ZNVER1-NEXT:    imulq $665536, %rdi, %rdi # imm = 0xA27C0
-; ZNVER1-NEXT:    # sched: [4:1.00]
-; ZNVER1-NEXT:    imulq $665536, (%rsi), %rdi # imm = 0xA27C0
-; ZNVER1-NEXT:    # sched: [4:1.00]
-; ZNVER1-NEXT:    imulq $7, %rdi, %rdi # sched: [4:1.00]
-; ZNVER1-NEXT:    imulq $7, (%rsi), %rdi # sched: [4:1.00]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  tail call void asm "imulq $0 \0A\09 imulq $2 \0A\09 imulq $1, $0 \0A\09 imulq $2, $0 \0A\09 imulq $3, $0, $0 \0A\09 imulq $3, $2, $0 \0A\09 imulq $4, $0, $0 \0A\09 imulq $4, $2, $0", "r,r,*m,i,i"(i64 %a0, i64 %a2, i64* %a1, i32 665536, i8 7) nounwind
-  ret void
-}
-
-define void @test_in() optsize {
-; GENERIC-LABEL: test_in:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    inb $7, %al # sched: [100:0.33]
-; GENERIC-NEXT:    inw $7, %ax # sched: [100:0.33]
-; GENERIC-NEXT:    inl $7, %eax # sched: [100:0.33]
-; GENERIC-NEXT:    inb %dx, %al # sched: [100:0.33]
-; GENERIC-NEXT:    inw %dx, %ax # sched: [100:0.33]
-; GENERIC-NEXT:    inl %dx, %eax # sched: [100:0.33]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_in:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    inb $7, %al # sched: [92:46.00]
-; ATOM-NEXT:    inw $7, %ax # sched: [92:46.00]
-; ATOM-NEXT:    inl $7, %eax # sched: [92:46.00]
-; ATOM-NEXT:    inb %dx, %al # sched: [94:47.00]
-; ATOM-NEXT:    inw %dx, %ax # sched: [94:47.00]
-; ATOM-NEXT:    inl %dx, %eax # sched: [94:47.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_in:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    inb $7, %al # sched: [100:1.00]
-; SLM-NEXT:    inw $7, %ax # sched: [100:1.00]
-; SLM-NEXT:    inl $7, %eax # sched: [100:1.00]
-; SLM-NEXT:    inb %dx, %al # sched: [100:1.00]
-; SLM-NEXT:    inw %dx, %ax # sched: [100:1.00]
-; SLM-NEXT:    inl %dx, %eax # sched: [100:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_in:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    inb $7, %al # sched: [100:0.33]
-; SANDY-NEXT:    inw $7, %ax # sched: [100:0.33]
-; SANDY-NEXT:    inl $7, %eax # sched: [100:0.33]
-; SANDY-NEXT:    inb %dx, %al # sched: [100:0.33]
-; SANDY-NEXT:    inw %dx, %ax # sched: [100:0.33]
-; SANDY-NEXT:    inl %dx, %eax # sched: [100:0.33]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_in:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    inb $7, %al # sched: [35:5.00]
-; HASWELL-NEXT:    inw $7, %ax # sched: [35:5.00]
-; HASWELL-NEXT:    inl $7, %eax # sched: [35:5.00]
-; HASWELL-NEXT:    inb %dx, %al # sched: [35:5.00]
-; HASWELL-NEXT:    inw %dx, %ax # sched: [35:5.00]
-; HASWELL-NEXT:    inl %dx, %eax # sched: [35:5.00]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_in:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    inb $7, %al # sched: [34:5.00]
-; BROADWELL-NEXT:    inw $7, %ax # sched: [34:5.00]
-; BROADWELL-NEXT:    inl $7, %eax # sched: [34:5.00]
-; BROADWELL-NEXT:    inb %dx, %al # sched: [34:5.00]
-; BROADWELL-NEXT:    inw %dx, %ax # sched: [34:5.00]
-; BROADWELL-NEXT:    inl %dx, %eax # sched: [34:5.00]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_in:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    inb $7, %al # sched: [35:5.00]
-; SKYLAKE-NEXT:    inw $7, %ax # sched: [35:5.00]
-; SKYLAKE-NEXT:    inl $7, %eax # sched: [35:5.00]
-; SKYLAKE-NEXT:    inb %dx, %al # sched: [35:5.00]
-; SKYLAKE-NEXT:    inw %dx, %ax # sched: [35:5.00]
-; SKYLAKE-NEXT:    inl %dx, %eax # sched: [35:5.00]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_in:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    inb $7, %al # sched: [35:5.00]
-; SKX-NEXT:    inw $7, %ax # sched: [35:5.00]
-; SKX-NEXT:    inl $7, %eax # sched: [35:5.00]
-; SKX-NEXT:    inb %dx, %al # sched: [35:5.00]
-; SKX-NEXT:    inw %dx, %ax # sched: [35:5.00]
-; SKX-NEXT:    inl %dx, %eax # sched: [35:5.00]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_in:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    inb $7, %al # sched: [100:0.50]
-; BDVER2-NEXT:    inw $7, %ax # sched: [100:0.50]
-; BDVER2-NEXT:    inl $7, %eax # sched: [100:0.50]
-; BDVER2-NEXT:    inb %dx, %al # sched: [100:0.50]
-; BDVER2-NEXT:    inw %dx, %ax # sched: [100:0.50]
-; BDVER2-NEXT:    inl %dx, %eax # sched: [100:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_in:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    inb $7, %al # sched: [100:0.50]
-; BTVER2-NEXT:    inw $7, %ax # sched: [100:0.50]
-; BTVER2-NEXT:    inl $7, %eax # sched: [100:0.50]
-; BTVER2-NEXT:    inb %dx, %al # sched: [100:0.50]
-; BTVER2-NEXT:    inw %dx, %ax # sched: [100:0.50]
-; BTVER2-NEXT:    inl %dx, %eax # sched: [100:0.50]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_in:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    inb $7, %al # sched: [100:0.25]
-; ZNVER1-NEXT:    inw $7, %ax # sched: [100:0.25]
-; ZNVER1-NEXT:    inl $7, %eax # sched: [100:0.25]
-; ZNVER1-NEXT:    inb %dx, %al # sched: [100:0.25]
-; ZNVER1-NEXT:    inw %dx, %ax # sched: [100:0.25]
-; ZNVER1-NEXT:    inl %dx, %eax # sched: [100:0.25]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  tail call void asm "inb $0, %AL \0A\09 inw $0, %AX \0A\09 inl $0, %EAX \0A\09 inb %DX, %AL \0A\09 inw %DX, %AX \0A\09 inl %DX, %EAX", "i"(i8 7) nounwind
-  ret void
-}
-
-define void @test_inc8(i8 %a0, i8* %a1) optsize {
-; GENERIC-LABEL: test_inc8:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    incb %dil # sched: [1:0.33]
-; GENERIC-NEXT:    incb (%rsi) # sched: [7:1.00]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_inc8:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    incb %dil # sched: [1:0.50]
-; ATOM-NEXT:    incb (%rsi) # sched: [1:1.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_inc8:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    incb %dil # sched: [1:0.50]
-; SLM-NEXT:    incb (%rsi) # sched: [5:2.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_inc8:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    incb %dil # sched: [1:0.33]
-; SANDY-NEXT:    incb (%rsi) # sched: [7:1.00]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_inc8:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    incb %dil # sched: [1:0.25]
-; HASWELL-NEXT:    incb (%rsi) # sched: [7:1.00]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_inc8:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    incb %dil # sched: [1:0.25]
-; BROADWELL-NEXT:    incb (%rsi) # sched: [7:1.00]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_inc8:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    incb %dil # sched: [1:0.25]
-; SKYLAKE-NEXT:    incb (%rsi) # sched: [7:1.00]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_inc8:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    incb %dil # sched: [1:0.25]
-; SKX-NEXT:    incb (%rsi) # sched: [7:1.00]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_inc8:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    incb %dil # sched: [1:0.50]
-; BDVER2-NEXT:    incb (%rsi) # sched: [6:1.00]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_inc8:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    incb %dil # sched: [1:0.50]
-; BTVER2-NEXT:    incb (%rsi) # sched: [5:1.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_inc8:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    incb %dil # sched: [1:0.25]
-; ZNVER1-NEXT:    incb (%rsi) # sched: [5:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  tail call void asm "incb $0 \0A\09 incb $1", "r,*m"(i8 %a0, i8* %a1) nounwind
-  ret void
-}
-define void @test_inc16(i16 %a0, i16* %a1) optsize {
-; GENERIC-LABEL: test_inc16:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    incw %di # sched: [1:0.33]
-; GENERIC-NEXT:    incw (%rsi) # sched: [7:1.00]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_inc16:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    incw %di # sched: [1:0.50]
-; ATOM-NEXT:    incw (%rsi) # sched: [1:1.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_inc16:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    incw %di # sched: [1:0.50]
-; SLM-NEXT:    incw (%rsi) # sched: [5:2.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_inc16:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    incw %di # sched: [1:0.33]
-; SANDY-NEXT:    incw (%rsi) # sched: [7:1.00]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_inc16:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    incw %di # sched: [1:0.25]
-; HASWELL-NEXT:    incw (%rsi) # sched: [7:1.00]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_inc16:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    incw %di # sched: [1:0.25]
-; BROADWELL-NEXT:    incw (%rsi) # sched: [7:1.00]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_inc16:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    incw %di # sched: [1:0.25]
-; SKYLAKE-NEXT:    incw (%rsi) # sched: [7:1.00]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_inc16:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    incw %di # sched: [1:0.25]
-; SKX-NEXT:    incw (%rsi) # sched: [7:1.00]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_inc16:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    incw %di # sched: [1:0.50]
-; BDVER2-NEXT:    incw (%rsi) # sched: [6:1.00]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_inc16:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    incw %di # sched: [1:0.50]
-; BTVER2-NEXT:    incw (%rsi) # sched: [5:1.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_inc16:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    incw %di # sched: [1:0.25]
-; ZNVER1-NEXT:    incw (%rsi) # sched: [5:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  tail call void asm "incw $0 \0A\09 incw $1", "r,*m"(i16 %a0, i16* %a1) nounwind
-  ret void
-}
-define void @test_inc32(i32 %a0, i32* %a1) optsize {
-; GENERIC-LABEL: test_inc32:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    incl %edi # sched: [1:0.33]
-; GENERIC-NEXT:    incl (%rsi) # sched: [7:1.00]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_inc32:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    incl %edi # sched: [1:0.50]
-; ATOM-NEXT:    incl (%rsi) # sched: [1:1.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_inc32:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    incl %edi # sched: [1:0.50]
-; SLM-NEXT:    incl (%rsi) # sched: [5:2.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_inc32:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    incl %edi # sched: [1:0.33]
-; SANDY-NEXT:    incl (%rsi) # sched: [7:1.00]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_inc32:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    incl %edi # sched: [1:0.25]
-; HASWELL-NEXT:    incl (%rsi) # sched: [7:1.00]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_inc32:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    incl %edi # sched: [1:0.25]
-; BROADWELL-NEXT:    incl (%rsi) # sched: [7:1.00]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_inc32:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    incl %edi # sched: [1:0.25]
-; SKYLAKE-NEXT:    incl (%rsi) # sched: [7:1.00]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_inc32:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    incl %edi # sched: [1:0.25]
-; SKX-NEXT:    incl (%rsi) # sched: [7:1.00]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_inc32:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    incl %edi # sched: [1:0.50]
-; BDVER2-NEXT:    incl (%rsi) # sched: [6:1.00]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_inc32:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    incl %edi # sched: [1:0.50]
-; BTVER2-NEXT:    incl (%rsi) # sched: [5:1.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_inc32:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    incl %edi # sched: [1:0.25]
-; ZNVER1-NEXT:    incl (%rsi) # sched: [5:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  tail call void asm "incl $0 \0A\09 incl $1", "r,*m"(i32 %a0, i32* %a1) nounwind
-  ret void
-}
-define void @test_inc64(i64 %a0, i64* %a1) optsize {
-; GENERIC-LABEL: test_inc64:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    incq %rdi # sched: [1:0.33]
-; GENERIC-NEXT:    incq (%rsi) # sched: [7:1.00]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_inc64:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    incq %rdi # sched: [1:0.50]
-; ATOM-NEXT:    incq (%rsi) # sched: [1:1.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_inc64:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    incq %rdi # sched: [1:0.50]
-; SLM-NEXT:    incq (%rsi) # sched: [5:2.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_inc64:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    incq %rdi # sched: [1:0.33]
-; SANDY-NEXT:    incq (%rsi) # sched: [7:1.00]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_inc64:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    incq %rdi # sched: [1:0.25]
-; HASWELL-NEXT:    incq (%rsi) # sched: [7:1.00]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_inc64:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    incq %rdi # sched: [1:0.25]
-; BROADWELL-NEXT:    incq (%rsi) # sched: [7:1.00]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_inc64:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    incq %rdi # sched: [1:0.25]
-; SKYLAKE-NEXT:    incq (%rsi) # sched: [7:1.00]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_inc64:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    incq %rdi # sched: [1:0.25]
-; SKX-NEXT:    incq (%rsi) # sched: [7:1.00]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_inc64:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    incq %rdi # sched: [1:0.50]
-; BDVER2-NEXT:    incq (%rsi) # sched: [6:1.00]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_inc64:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    incq %rdi # sched: [1:0.50]
-; BTVER2-NEXT:    incq (%rsi) # sched: [5:1.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_inc64:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    incq %rdi # sched: [1:0.25]
-; ZNVER1-NEXT:    incq (%rsi) # sched: [5:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  tail call void asm "incq $0 \0A\09 incq $1", "r,*m"(i64 %a0, i64* %a1) nounwind
-  ret void
-}
-
-define void @test_ins() optsize {
-; GENERIC-LABEL: test_ins:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    insb %dx, %es:(%rdi) # sched: [100:0.33]
-; GENERIC-NEXT:    insw %dx, %es:(%rdi) # sched: [100:0.33]
-; GENERIC-NEXT:    insl %dx, %es:(%rdi) # sched: [100:0.33]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_ins:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    insb %dx, %es:(%rdi) # sched: [59:29.50]
-; ATOM-NEXT:    insw %dx, %es:(%rdi) # sched: [59:29.50]
-; ATOM-NEXT:    insl %dx, %es:(%rdi) # sched: [59:29.50]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_ins:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    insb %dx, %es:(%rdi) # sched: [100:1.00]
-; SLM-NEXT:    insw %dx, %es:(%rdi) # sched: [100:1.00]
-; SLM-NEXT:    insl %dx, %es:(%rdi) # sched: [100:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_ins:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    insb %dx, %es:(%rdi) # sched: [100:0.33]
-; SANDY-NEXT:    insw %dx, %es:(%rdi) # sched: [100:0.33]
-; SANDY-NEXT:    insl %dx, %es:(%rdi) # sched: [100:0.33]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_ins:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    insb %dx, %es:(%rdi) # sched: [21:1.25]
-; HASWELL-NEXT:    insw %dx, %es:(%rdi) # sched: [21:1.25]
-; HASWELL-NEXT:    insl %dx, %es:(%rdi) # sched: [21:1.25]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_ins:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    insb %dx, %es:(%rdi) # sched: [20:1.25]
-; BROADWELL-NEXT:    insw %dx, %es:(%rdi) # sched: [20:1.25]
-; BROADWELL-NEXT:    insl %dx, %es:(%rdi) # sched: [20:1.25]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_ins:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    insb %dx, %es:(%rdi) # sched: [20:1.25]
-; SKYLAKE-NEXT:    insw %dx, %es:(%rdi) # sched: [20:1.25]
-; SKYLAKE-NEXT:    insl %dx, %es:(%rdi) # sched: [20:1.25]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_ins:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    insb %dx, %es:(%rdi) # sched: [20:1.25]
-; SKX-NEXT:    insw %dx, %es:(%rdi) # sched: [20:1.25]
-; SKX-NEXT:    insl %dx, %es:(%rdi) # sched: [20:1.25]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_ins:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    insb %dx, %es:(%rdi) # sched: [100:0.50]
-; BDVER2-NEXT:    insw %dx, %es:(%rdi) # sched: [100:0.50]
-; BDVER2-NEXT:    insl %dx, %es:(%rdi) # sched: [100:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_ins:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    insb %dx, %es:(%rdi) # sched: [100:0.50]
-; BTVER2-NEXT:    insw %dx, %es:(%rdi) # sched: [100:0.50]
-; BTVER2-NEXT:    insl %dx, %es:(%rdi) # sched: [100:0.50]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_ins:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    insb %dx, %es:(%rdi) # sched: [100:0.25]
-; ZNVER1-NEXT:    insw %dx, %es:(%rdi) # sched: [100:0.25]
-; ZNVER1-NEXT:    insl %dx, %es:(%rdi) # sched: [100:0.25]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  call void asm sideeffect "insb \0A\09 insw \0A\09 insl", ""()
-  ret void
-}
-
-define void @test_int() optsize {
-; GENERIC-LABEL: test_int:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    int $7 # sched: [100:0.33]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_int:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    int $7 # sched: [127:63.50]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_int:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    int $7 # sched: [100:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_int:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    int $7 # sched: [100:0.33]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_int:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    int $7 # sched: [100:0.25]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_int:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    int $7 # sched: [100:0.25]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_int:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    int $7 # sched: [100:0.25]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_int:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    int $7 # sched: [100:0.25]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_int:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    int $7 # sched: [100:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_int:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    int $7 # sched: [100:0.50]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_int:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    int $7 # sched: [100:0.25]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  call void asm sideeffect "int $0", "i"(i8 7)
-  ret void
-}
-
-define void @test_invlpg_invlpga(i8 *%a0) optsize {
-; GENERIC-LABEL: test_invlpg_invlpga:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    invlpg (%rdi) # sched: [100:0.33]
-; GENERIC-NEXT:    invlpga %rax, %ecx # sched: [100:0.33]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_invlpg_invlpga:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    invlpg (%rdi) # sched: [71:35.50]
-; ATOM-NEXT:    invlpga %rax, %ecx # sched: [71:35.50]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_invlpg_invlpga:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    invlpg (%rdi) # sched: [100:1.00]
-; SLM-NEXT:    invlpga %rax, %ecx # sched: [100:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_invlpg_invlpga:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    invlpg (%rdi) # sched: [100:0.33]
-; SANDY-NEXT:    invlpga %rax, %ecx # sched: [100:0.33]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_invlpg_invlpga:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    invlpg (%rdi) # sched: [100:0.25]
-; HASWELL-NEXT:    invlpga %rax, %ecx # sched: [100:0.25]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_invlpg_invlpga:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    invlpg (%rdi) # sched: [100:0.25]
-; BROADWELL-NEXT:    invlpga %rax, %ecx # sched: [100:0.25]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_invlpg_invlpga:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    invlpg (%rdi) # sched: [100:0.25]
-; SKYLAKE-NEXT:    invlpga %rax, %ecx # sched: [100:0.25]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_invlpg_invlpga:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    invlpg (%rdi) # sched: [100:0.25]
-; SKX-NEXT:    invlpga %rax, %ecx # sched: [100:0.25]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_invlpg_invlpga:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    invlpg (%rdi) # sched: [100:0.50]
-; BDVER2-NEXT:    invlpga %rax, %ecx # sched: [100:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_invlpg_invlpga:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    invlpg (%rdi) # sched: [100:0.50]
-; BTVER2-NEXT:    invlpga %rax, %ecx # sched: [100:0.50]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_invlpg_invlpga:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    invlpg (%rdi) # sched: [100:0.25]
-; ZNVER1-NEXT:    invlpga %rax, %ecx # sched: [100:0.25]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  tail call void asm sideeffect "invlpg $0 \0A\09 invlpga %rax, %ecx", "*m"(i8 *%a0) nounwind
-  ret void
-}
-
-define void @test_jcc() optsize {
-; GENERIC-LABEL: test_jcc:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:  JCCTGT:
-; GENERIC-NEXT:    jo JCCTGT # sched: [1:1.00]
-; GENERIC-NEXT:    jno JCCTGT # sched: [1:1.00]
-; GENERIC-NEXT:    jb JCCTGT # sched: [1:1.00]
-; GENERIC-NEXT:    jb JCCTGT # sched: [1:1.00]
-; GENERIC-NEXT:    jb JCCTGT # sched: [1:1.00]
-; GENERIC-NEXT:    jae JCCTGT # sched: [1:1.00]
-; GENERIC-NEXT:    jae JCCTGT # sched: [1:1.00]
-; GENERIC-NEXT:    jae JCCTGT # sched: [1:1.00]
-; GENERIC-NEXT:    je JCCTGT # sched: [1:1.00]
-; GENERIC-NEXT:    je JCCTGT # sched: [1:1.00]
-; GENERIC-NEXT:    jne JCCTGT # sched: [1:1.00]
-; GENERIC-NEXT:    jne JCCTGT # sched: [1:1.00]
-; GENERIC-NEXT:    jbe JCCTGT # sched: [1:1.00]
-; GENERIC-NEXT:    jbe JCCTGT # sched: [1:1.00]
-; GENERIC-NEXT:    ja JCCTGT # sched: [1:1.00]
-; GENERIC-NEXT:    ja JCCTGT # sched: [1:1.00]
-; GENERIC-NEXT:    js JCCTGT # sched: [1:1.00]
-; GENERIC-NEXT:    jns JCCTGT # sched: [1:1.00]
-; GENERIC-NEXT:    jp JCCTGT # sched: [1:1.00]
-; GENERIC-NEXT:    jp JCCTGT # sched: [1:1.00]
-; GENERIC-NEXT:    jnp JCCTGT # sched: [1:1.00]
-; GENERIC-NEXT:    jnp JCCTGT # sched: [1:1.00]
-; GENERIC-NEXT:    jl JCCTGT # sched: [1:1.00]
-; GENERIC-NEXT:    jl JCCTGT # sched: [1:1.00]
-; GENERIC-NEXT:    jge JCCTGT # sched: [1:1.00]
-; GENERIC-NEXT:    jge JCCTGT # sched: [1:1.00]
-; GENERIC-NEXT:    jle JCCTGT # sched: [1:1.00]
-; GENERIC-NEXT:    jle JCCTGT # sched: [1:1.00]
-; GENERIC-NEXT:    jg JCCTGT # sched: [1:1.00]
-; GENERIC-NEXT:    jg JCCTGT # sched: [1:1.00]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_jcc:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:  JCCTGT:
-; ATOM-NEXT:    jo JCCTGT # sched: [1:1.00]
-; ATOM-NEXT:    jno JCCTGT # sched: [1:1.00]
-; ATOM-NEXT:    jb JCCTGT # sched: [1:1.00]
-; ATOM-NEXT:    jb JCCTGT # sched: [1:1.00]
-; ATOM-NEXT:    jb JCCTGT # sched: [1:1.00]
-; ATOM-NEXT:    jae JCCTGT # sched: [1:1.00]
-; ATOM-NEXT:    jae JCCTGT # sched: [1:1.00]
-; ATOM-NEXT:    jae JCCTGT # sched: [1:1.00]
-; ATOM-NEXT:    je JCCTGT # sched: [1:1.00]
-; ATOM-NEXT:    je JCCTGT # sched: [1:1.00]
-; ATOM-NEXT:    jne JCCTGT # sched: [1:1.00]
-; ATOM-NEXT:    jne JCCTGT # sched: [1:1.00]
-; ATOM-NEXT:    jbe JCCTGT # sched: [1:1.00]
-; ATOM-NEXT:    jbe JCCTGT # sched: [1:1.00]
-; ATOM-NEXT:    ja JCCTGT # sched: [1:1.00]
-; ATOM-NEXT:    ja JCCTGT # sched: [1:1.00]
-; ATOM-NEXT:    js JCCTGT # sched: [1:1.00]
-; ATOM-NEXT:    jns JCCTGT # sched: [1:1.00]
-; ATOM-NEXT:    jp JCCTGT # sched: [1:1.00]
-; ATOM-NEXT:    jp JCCTGT # sched: [1:1.00]
-; ATOM-NEXT:    jnp JCCTGT # sched: [1:1.00]
-; ATOM-NEXT:    jnp JCCTGT # sched: [1:1.00]
-; ATOM-NEXT:    jl JCCTGT # sched: [1:1.00]
-; ATOM-NEXT:    jl JCCTGT # sched: [1:1.00]
-; ATOM-NEXT:    jge JCCTGT # sched: [1:1.00]
-; ATOM-NEXT:    jge JCCTGT # sched: [1:1.00]
-; ATOM-NEXT:    jle JCCTGT # sched: [1:1.00]
-; ATOM-NEXT:    jle JCCTGT # sched: [1:1.00]
-; ATOM-NEXT:    jg JCCTGT # sched: [1:1.00]
-; ATOM-NEXT:    jg JCCTGT # sched: [1:1.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_jcc:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:  JCCTGT:
-; SLM-NEXT:    jo JCCTGT # sched: [1:1.00]
-; SLM-NEXT:    jno JCCTGT # sched: [1:1.00]
-; SLM-NEXT:    jb JCCTGT # sched: [1:1.00]
-; SLM-NEXT:    jb JCCTGT # sched: [1:1.00]
-; SLM-NEXT:    jb JCCTGT # sched: [1:1.00]
-; SLM-NEXT:    jae JCCTGT # sched: [1:1.00]
-; SLM-NEXT:    jae JCCTGT # sched: [1:1.00]
-; SLM-NEXT:    jae JCCTGT # sched: [1:1.00]
-; SLM-NEXT:    je JCCTGT # sched: [1:1.00]
-; SLM-NEXT:    je JCCTGT # sched: [1:1.00]
-; SLM-NEXT:    jne JCCTGT # sched: [1:1.00]
-; SLM-NEXT:    jne JCCTGT # sched: [1:1.00]
-; SLM-NEXT:    jbe JCCTGT # sched: [1:1.00]
-; SLM-NEXT:    jbe JCCTGT # sched: [1:1.00]
-; SLM-NEXT:    ja JCCTGT # sched: [1:1.00]
-; SLM-NEXT:    ja JCCTGT # sched: [1:1.00]
-; SLM-NEXT:    js JCCTGT # sched: [1:1.00]
-; SLM-NEXT:    jns JCCTGT # sched: [1:1.00]
-; SLM-NEXT:    jp JCCTGT # sched: [1:1.00]
-; SLM-NEXT:    jp JCCTGT # sched: [1:1.00]
-; SLM-NEXT:    jnp JCCTGT # sched: [1:1.00]
-; SLM-NEXT:    jnp JCCTGT # sched: [1:1.00]
-; SLM-NEXT:    jl JCCTGT # sched: [1:1.00]
-; SLM-NEXT:    jl JCCTGT # sched: [1:1.00]
-; SLM-NEXT:    jge JCCTGT # sched: [1:1.00]
-; SLM-NEXT:    jge JCCTGT # sched: [1:1.00]
-; SLM-NEXT:    jle JCCTGT # sched: [1:1.00]
-; SLM-NEXT:    jle JCCTGT # sched: [1:1.00]
-; SLM-NEXT:    jg JCCTGT # sched: [1:1.00]
-; SLM-NEXT:    jg JCCTGT # sched: [1:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_jcc:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:  JCCTGT:
-; SANDY-NEXT:    jo JCCTGT # sched: [1:1.00]
-; SANDY-NEXT:    jno JCCTGT # sched: [1:1.00]
-; SANDY-NEXT:    jb JCCTGT # sched: [1:1.00]
-; SANDY-NEXT:    jb JCCTGT # sched: [1:1.00]
-; SANDY-NEXT:    jb JCCTGT # sched: [1:1.00]
-; SANDY-NEXT:    jae JCCTGT # sched: [1:1.00]
-; SANDY-NEXT:    jae JCCTGT # sched: [1:1.00]
-; SANDY-NEXT:    jae JCCTGT # sched: [1:1.00]
-; SANDY-NEXT:    je JCCTGT # sched: [1:1.00]
-; SANDY-NEXT:    je JCCTGT # sched: [1:1.00]
-; SANDY-NEXT:    jne JCCTGT # sched: [1:1.00]
-; SANDY-NEXT:    jne JCCTGT # sched: [1:1.00]
-; SANDY-NEXT:    jbe JCCTGT # sched: [1:1.00]
-; SANDY-NEXT:    jbe JCCTGT # sched: [1:1.00]
-; SANDY-NEXT:    ja JCCTGT # sched: [1:1.00]
-; SANDY-NEXT:    ja JCCTGT # sched: [1:1.00]
-; SANDY-NEXT:    js JCCTGT # sched: [1:1.00]
-; SANDY-NEXT:    jns JCCTGT # sched: [1:1.00]
-; SANDY-NEXT:    jp JCCTGT # sched: [1:1.00]
-; SANDY-NEXT:    jp JCCTGT # sched: [1:1.00]
-; SANDY-NEXT:    jnp JCCTGT # sched: [1:1.00]
-; SANDY-NEXT:    jnp JCCTGT # sched: [1:1.00]
-; SANDY-NEXT:    jl JCCTGT # sched: [1:1.00]
-; SANDY-NEXT:    jl JCCTGT # sched: [1:1.00]
-; SANDY-NEXT:    jge JCCTGT # sched: [1:1.00]
-; SANDY-NEXT:    jge JCCTGT # sched: [1:1.00]
-; SANDY-NEXT:    jle JCCTGT # sched: [1:1.00]
-; SANDY-NEXT:    jle JCCTGT # sched: [1:1.00]
-; SANDY-NEXT:    jg JCCTGT # sched: [1:1.00]
-; SANDY-NEXT:    jg JCCTGT # sched: [1:1.00]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_jcc:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:  JCCTGT:
-; HASWELL-NEXT:    jo JCCTGT # sched: [1:0.50]
-; HASWELL-NEXT:    jno JCCTGT # sched: [1:0.50]
-; HASWELL-NEXT:    jb JCCTGT # sched: [1:0.50]
-; HASWELL-NEXT:    jb JCCTGT # sched: [1:0.50]
-; HASWELL-NEXT:    jb JCCTGT # sched: [1:0.50]
-; HASWELL-NEXT:    jae JCCTGT # sched: [1:0.50]
-; HASWELL-NEXT:    jae JCCTGT # sched: [1:0.50]
-; HASWELL-NEXT:    jae JCCTGT # sched: [1:0.50]
-; HASWELL-NEXT:    je JCCTGT # sched: [1:0.50]
-; HASWELL-NEXT:    je JCCTGT # sched: [1:0.50]
-; HASWELL-NEXT:    jne JCCTGT # sched: [1:0.50]
-; HASWELL-NEXT:    jne JCCTGT # sched: [1:0.50]
-; HASWELL-NEXT:    jbe JCCTGT # sched: [1:0.50]
-; HASWELL-NEXT:    jbe JCCTGT # sched: [1:0.50]
-; HASWELL-NEXT:    ja JCCTGT # sched: [1:0.50]
-; HASWELL-NEXT:    ja JCCTGT # sched: [1:0.50]
-; HASWELL-NEXT:    js JCCTGT # sched: [1:0.50]
-; HASWELL-NEXT:    jns JCCTGT # sched: [1:0.50]
-; HASWELL-NEXT:    jp JCCTGT # sched: [1:0.50]
-; HASWELL-NEXT:    jp JCCTGT # sched: [1:0.50]
-; HASWELL-NEXT:    jnp JCCTGT # sched: [1:0.50]
-; HASWELL-NEXT:    jnp JCCTGT # sched: [1:0.50]
-; HASWELL-NEXT:    jl JCCTGT # sched: [1:0.50]
-; HASWELL-NEXT:    jl JCCTGT # sched: [1:0.50]
-; HASWELL-NEXT:    jge JCCTGT # sched: [1:0.50]
-; HASWELL-NEXT:    jge JCCTGT # sched: [1:0.50]
-; HASWELL-NEXT:    jle JCCTGT # sched: [1:0.50]
-; HASWELL-NEXT:    jle JCCTGT # sched: [1:0.50]
-; HASWELL-NEXT:    jg JCCTGT # sched: [1:0.50]
-; HASWELL-NEXT:    jg JCCTGT # sched: [1:0.50]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_jcc:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:  JCCTGT:
-; BROADWELL-NEXT:    jo JCCTGT # sched: [1:0.50]
-; BROADWELL-NEXT:    jno JCCTGT # sched: [1:0.50]
-; BROADWELL-NEXT:    jb JCCTGT # sched: [1:0.50]
-; BROADWELL-NEXT:    jb JCCTGT # sched: [1:0.50]
-; BROADWELL-NEXT:    jb JCCTGT # sched: [1:0.50]
-; BROADWELL-NEXT:    jae JCCTGT # sched: [1:0.50]
-; BROADWELL-NEXT:    jae JCCTGT # sched: [1:0.50]
-; BROADWELL-NEXT:    jae JCCTGT # sched: [1:0.50]
-; BROADWELL-NEXT:    je JCCTGT # sched: [1:0.50]
-; BROADWELL-NEXT:    je JCCTGT # sched: [1:0.50]
-; BROADWELL-NEXT:    jne JCCTGT # sched: [1:0.50]
-; BROADWELL-NEXT:    jne JCCTGT # sched: [1:0.50]
-; BROADWELL-NEXT:    jbe JCCTGT # sched: [1:0.50]
-; BROADWELL-NEXT:    jbe JCCTGT # sched: [1:0.50]
-; BROADWELL-NEXT:    ja JCCTGT # sched: [1:0.50]
-; BROADWELL-NEXT:    ja JCCTGT # sched: [1:0.50]
-; BROADWELL-NEXT:    js JCCTGT # sched: [1:0.50]
-; BROADWELL-NEXT:    jns JCCTGT # sched: [1:0.50]
-; BROADWELL-NEXT:    jp JCCTGT # sched: [1:0.50]
-; BROADWELL-NEXT:    jp JCCTGT # sched: [1:0.50]
-; BROADWELL-NEXT:    jnp JCCTGT # sched: [1:0.50]
-; BROADWELL-NEXT:    jnp JCCTGT # sched: [1:0.50]
-; BROADWELL-NEXT:    jl JCCTGT # sched: [1:0.50]
-; BROADWELL-NEXT:    jl JCCTGT # sched: [1:0.50]
-; BROADWELL-NEXT:    jge JCCTGT # sched: [1:0.50]
-; BROADWELL-NEXT:    jge JCCTGT # sched: [1:0.50]
-; BROADWELL-NEXT:    jle JCCTGT # sched: [1:0.50]
-; BROADWELL-NEXT:    jle JCCTGT # sched: [1:0.50]
-; BROADWELL-NEXT:    jg JCCTGT # sched: [1:0.50]
-; BROADWELL-NEXT:    jg JCCTGT # sched: [1:0.50]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_jcc:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:  JCCTGT:
-; SKYLAKE-NEXT:    jo JCCTGT # sched: [1:0.50]
-; SKYLAKE-NEXT:    jno JCCTGT # sched: [1:0.50]
-; SKYLAKE-NEXT:    jb JCCTGT # sched: [1:0.50]
-; SKYLAKE-NEXT:    jb JCCTGT # sched: [1:0.50]
-; SKYLAKE-NEXT:    jb JCCTGT # sched: [1:0.50]
-; SKYLAKE-NEXT:    jae JCCTGT # sched: [1:0.50]
-; SKYLAKE-NEXT:    jae JCCTGT # sched: [1:0.50]
-; SKYLAKE-NEXT:    jae JCCTGT # sched: [1:0.50]
-; SKYLAKE-NEXT:    je JCCTGT # sched: [1:0.50]
-; SKYLAKE-NEXT:    je JCCTGT # sched: [1:0.50]
-; SKYLAKE-NEXT:    jne JCCTGT # sched: [1:0.50]
-; SKYLAKE-NEXT:    jne JCCTGT # sched: [1:0.50]
-; SKYLAKE-NEXT:    jbe JCCTGT # sched: [1:0.50]
-; SKYLAKE-NEXT:    jbe JCCTGT # sched: [1:0.50]
-; SKYLAKE-NEXT:    ja JCCTGT # sched: [1:0.50]
-; SKYLAKE-NEXT:    ja JCCTGT # sched: [1:0.50]
-; SKYLAKE-NEXT:    js JCCTGT # sched: [1:0.50]
-; SKYLAKE-NEXT:    jns JCCTGT # sched: [1:0.50]
-; SKYLAKE-NEXT:    jp JCCTGT # sched: [1:0.50]
-; SKYLAKE-NEXT:    jp JCCTGT # sched: [1:0.50]
-; SKYLAKE-NEXT:    jnp JCCTGT # sched: [1:0.50]
-; SKYLAKE-NEXT:    jnp JCCTGT # sched: [1:0.50]
-; SKYLAKE-NEXT:    jl JCCTGT # sched: [1:0.50]
-; SKYLAKE-NEXT:    jl JCCTGT # sched: [1:0.50]
-; SKYLAKE-NEXT:    jge JCCTGT # sched: [1:0.50]
-; SKYLAKE-NEXT:    jge JCCTGT # sched: [1:0.50]
-; SKYLAKE-NEXT:    jle JCCTGT # sched: [1:0.50]
-; SKYLAKE-NEXT:    jle JCCTGT # sched: [1:0.50]
-; SKYLAKE-NEXT:    jg JCCTGT # sched: [1:0.50]
-; SKYLAKE-NEXT:    jg JCCTGT # sched: [1:0.50]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_jcc:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:  JCCTGT:
-; SKX-NEXT:    jo JCCTGT # sched: [1:0.50]
-; SKX-NEXT:    jno JCCTGT # sched: [1:0.50]
-; SKX-NEXT:    jb JCCTGT # sched: [1:0.50]
-; SKX-NEXT:    jb JCCTGT # sched: [1:0.50]
-; SKX-NEXT:    jb JCCTGT # sched: [1:0.50]
-; SKX-NEXT:    jae JCCTGT # sched: [1:0.50]
-; SKX-NEXT:    jae JCCTGT # sched: [1:0.50]
-; SKX-NEXT:    jae JCCTGT # sched: [1:0.50]
-; SKX-NEXT:    je JCCTGT # sched: [1:0.50]
-; SKX-NEXT:    je JCCTGT # sched: [1:0.50]
-; SKX-NEXT:    jne JCCTGT # sched: [1:0.50]
-; SKX-NEXT:    jne JCCTGT # sched: [1:0.50]
-; SKX-NEXT:    jbe JCCTGT # sched: [1:0.50]
-; SKX-NEXT:    jbe JCCTGT # sched: [1:0.50]
-; SKX-NEXT:    ja JCCTGT # sched: [1:0.50]
-; SKX-NEXT:    ja JCCTGT # sched: [1:0.50]
-; SKX-NEXT:    js JCCTGT # sched: [1:0.50]
-; SKX-NEXT:    jns JCCTGT # sched: [1:0.50]
-; SKX-NEXT:    jp JCCTGT # sched: [1:0.50]
-; SKX-NEXT:    jp JCCTGT # sched: [1:0.50]
-; SKX-NEXT:    jnp JCCTGT # sched: [1:0.50]
-; SKX-NEXT:    jnp JCCTGT # sched: [1:0.50]
-; SKX-NEXT:    jl JCCTGT # sched: [1:0.50]
-; SKX-NEXT:    jl JCCTGT # sched: [1:0.50]
-; SKX-NEXT:    jge JCCTGT # sched: [1:0.50]
-; SKX-NEXT:    jge JCCTGT # sched: [1:0.50]
-; SKX-NEXT:    jle JCCTGT # sched: [1:0.50]
-; SKX-NEXT:    jle JCCTGT # sched: [1:0.50]
-; SKX-NEXT:    jg JCCTGT # sched: [1:0.50]
-; SKX-NEXT:    jg JCCTGT # sched: [1:0.50]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_jcc:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:  JCCTGT:
-; BDVER2-NEXT:    jo JCCTGT # sched: [1:1.00]
-; BDVER2-NEXT:    jno JCCTGT # sched: [1:1.00]
-; BDVER2-NEXT:    jb JCCTGT # sched: [1:1.00]
-; BDVER2-NEXT:    jb JCCTGT # sched: [1:1.00]
-; BDVER2-NEXT:    jb JCCTGT # sched: [1:1.00]
-; BDVER2-NEXT:    jae JCCTGT # sched: [1:1.00]
-; BDVER2-NEXT:    jae JCCTGT # sched: [1:1.00]
-; BDVER2-NEXT:    jae JCCTGT # sched: [1:1.00]
-; BDVER2-NEXT:    je JCCTGT # sched: [1:1.00]
-; BDVER2-NEXT:    je JCCTGT # sched: [1:1.00]
-; BDVER2-NEXT:    jne JCCTGT # sched: [1:1.00]
-; BDVER2-NEXT:    jne JCCTGT # sched: [1:1.00]
-; BDVER2-NEXT:    jbe JCCTGT # sched: [1:1.00]
-; BDVER2-NEXT:    jbe JCCTGT # sched: [1:1.00]
-; BDVER2-NEXT:    ja JCCTGT # sched: [1:1.00]
-; BDVER2-NEXT:    ja JCCTGT # sched: [1:1.00]
-; BDVER2-NEXT:    js JCCTGT # sched: [1:1.00]
-; BDVER2-NEXT:    jns JCCTGT # sched: [1:1.00]
-; BDVER2-NEXT:    jp JCCTGT # sched: [1:1.00]
-; BDVER2-NEXT:    jp JCCTGT # sched: [1:1.00]
-; BDVER2-NEXT:    jnp JCCTGT # sched: [1:1.00]
-; BDVER2-NEXT:    jnp JCCTGT # sched: [1:1.00]
-; BDVER2-NEXT:    jl JCCTGT # sched: [1:1.00]
-; BDVER2-NEXT:    jl JCCTGT # sched: [1:1.00]
-; BDVER2-NEXT:    jge JCCTGT # sched: [1:1.00]
-; BDVER2-NEXT:    jge JCCTGT # sched: [1:1.00]
-; BDVER2-NEXT:    jle JCCTGT # sched: [1:1.00]
-; BDVER2-NEXT:    jle JCCTGT # sched: [1:1.00]
-; BDVER2-NEXT:    jg JCCTGT # sched: [1:1.00]
-; BDVER2-NEXT:    jg JCCTGT # sched: [1:1.00]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_jcc:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:  JCCTGT:
-; BTVER2-NEXT:    jo JCCTGT # sched: [1:0.50]
-; BTVER2-NEXT:    jno JCCTGT # sched: [1:0.50]
-; BTVER2-NEXT:    jb JCCTGT # sched: [1:0.50]
-; BTVER2-NEXT:    jb JCCTGT # sched: [1:0.50]
-; BTVER2-NEXT:    jb JCCTGT # sched: [1:0.50]
-; BTVER2-NEXT:    jae JCCTGT # sched: [1:0.50]
-; BTVER2-NEXT:    jae JCCTGT # sched: [1:0.50]
-; BTVER2-NEXT:    jae JCCTGT # sched: [1:0.50]
-; BTVER2-NEXT:    je JCCTGT # sched: [1:0.50]
-; BTVER2-NEXT:    je JCCTGT # sched: [1:0.50]
-; BTVER2-NEXT:    jne JCCTGT # sched: [1:0.50]
-; BTVER2-NEXT:    jne JCCTGT # sched: [1:0.50]
-; BTVER2-NEXT:    jbe JCCTGT # sched: [1:0.50]
-; BTVER2-NEXT:    jbe JCCTGT # sched: [1:0.50]
-; BTVER2-NEXT:    ja JCCTGT # sched: [1:0.50]
-; BTVER2-NEXT:    ja JCCTGT # sched: [1:0.50]
-; BTVER2-NEXT:    js JCCTGT # sched: [1:0.50]
-; BTVER2-NEXT:    jns JCCTGT # sched: [1:0.50]
-; BTVER2-NEXT:    jp JCCTGT # sched: [1:0.50]
-; BTVER2-NEXT:    jp JCCTGT # sched: [1:0.50]
-; BTVER2-NEXT:    jnp JCCTGT # sched: [1:0.50]
-; BTVER2-NEXT:    jnp JCCTGT # sched: [1:0.50]
-; BTVER2-NEXT:    jl JCCTGT # sched: [1:0.50]
-; BTVER2-NEXT:    jl JCCTGT # sched: [1:0.50]
-; BTVER2-NEXT:    jge JCCTGT # sched: [1:0.50]
-; BTVER2-NEXT:    jge JCCTGT # sched: [1:0.50]
-; BTVER2-NEXT:    jle JCCTGT # sched: [1:0.50]
-; BTVER2-NEXT:    jle JCCTGT # sched: [1:0.50]
-; BTVER2-NEXT:    jg JCCTGT # sched: [1:0.50]
-; BTVER2-NEXT:    jg JCCTGT # sched: [1:0.50]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_jcc:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:  JCCTGT:
-; ZNVER1-NEXT:    jo JCCTGT # sched: [1:0.25]
-; ZNVER1-NEXT:    jno JCCTGT # sched: [1:0.25]
-; ZNVER1-NEXT:    jb JCCTGT # sched: [1:0.25]
-; ZNVER1-NEXT:    jb JCCTGT # sched: [1:0.25]
-; ZNVER1-NEXT:    jb JCCTGT # sched: [1:0.25]
-; ZNVER1-NEXT:    jae JCCTGT # sched: [1:0.25]
-; ZNVER1-NEXT:    jae JCCTGT # sched: [1:0.25]
-; ZNVER1-NEXT:    jae JCCTGT # sched: [1:0.25]
-; ZNVER1-NEXT:    je JCCTGT # sched: [1:0.25]
-; ZNVER1-NEXT:    je JCCTGT # sched: [1:0.25]
-; ZNVER1-NEXT:    jne JCCTGT # sched: [1:0.25]
-; ZNVER1-NEXT:    jne JCCTGT # sched: [1:0.25]
-; ZNVER1-NEXT:    jbe JCCTGT # sched: [1:0.25]
-; ZNVER1-NEXT:    jbe JCCTGT # sched: [1:0.25]
-; ZNVER1-NEXT:    ja JCCTGT # sched: [1:0.25]
-; ZNVER1-NEXT:    ja JCCTGT # sched: [1:0.25]
-; ZNVER1-NEXT:    js JCCTGT # sched: [1:0.25]
-; ZNVER1-NEXT:    jns JCCTGT # sched: [1:0.25]
-; ZNVER1-NEXT:    jp JCCTGT # sched: [1:0.25]
-; ZNVER1-NEXT:    jp JCCTGT # sched: [1:0.25]
-; ZNVER1-NEXT:    jnp JCCTGT # sched: [1:0.25]
-; ZNVER1-NEXT:    jnp JCCTGT # sched: [1:0.25]
-; ZNVER1-NEXT:    jl JCCTGT # sched: [1:0.25]
-; ZNVER1-NEXT:    jl JCCTGT # sched: [1:0.25]
-; ZNVER1-NEXT:    jge JCCTGT # sched: [1:0.25]
-; ZNVER1-NEXT:    jge JCCTGT # sched: [1:0.25]
-; ZNVER1-NEXT:    jle JCCTGT # sched: [1:0.25]
-; ZNVER1-NEXT:    jle JCCTGT # sched: [1:0.25]
-; ZNVER1-NEXT:    jg JCCTGT # sched: [1:0.25]
-; ZNVER1-NEXT:    jg JCCTGT # sched: [1:0.25]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  call void asm sideeffect "JCCTGT: \0A\09 jo JCCTGT \0A\09 jno JCCTGT \0A\09 jb JCCTGT \0A\09 jc JCCTGT \0A\09 jnae JCCTGT \0A\09 jnb JCCTGT \0A\09 jnc JCCTGT \0A\09 jae JCCTGT \0A\09 jz JCCTGT \0A\09 je JCCTGT \0A\09 jnz JCCTGT \0A\09 jne JCCTGT \0A\09 jbe JCCTGT \0A\09 jna JCCTGT \0A\09 jnbe JCCTGT \0A\09 ja JCCTGT \0A\09 js JCCTGT \0A\09 jns JCCTGT \0A\09 jp JCCTGT \0A\09 jpe JCCTGT \0A\09 jnp JCCTGT \0A\09 jpo JCCTGT \0A\09 jl JCCTGT \0A\09 jnge JCCTGT \0A\09 jnl JCCTGT \0A\09 jge JCCTGT \0A\09 jle JCCTGT \0A\09 jng JCCTGT \0A\09 jnle JCCTGT \0A\09 jg JCCTGT", ""()
-  ret void
-}
-
-define void @test_jecxz_jrcxz() optsize {
-; GENERIC-LABEL: test_jecxz_jrcxz:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:  JXTGT:
-; GENERIC-NEXT:    jecxz JXTGT # sched: [2:1.00]
-; GENERIC-NEXT:    jrcxz JXTGT # sched: [2:1.00]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_jecxz_jrcxz:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:  JXTGT:
-; ATOM-NEXT:    jecxz JXTGT # sched: [4:2.00]
-; ATOM-NEXT:    jrcxz JXTGT # sched: [4:2.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_jecxz_jrcxz:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:  JXTGT:
-; SLM-NEXT:    jecxz JXTGT # sched: [1:1.00]
-; SLM-NEXT:    jrcxz JXTGT # sched: [1:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_jecxz_jrcxz:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:  JXTGT:
-; SANDY-NEXT:    jecxz JXTGT # sched: [2:1.00]
-; SANDY-NEXT:    jrcxz JXTGT # sched: [2:1.00]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_jecxz_jrcxz:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:  JXTGT:
-; HASWELL-NEXT:    jecxz JXTGT # sched: [2:0.50]
-; HASWELL-NEXT:    jrcxz JXTGT # sched: [2:0.50]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_jecxz_jrcxz:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:  JXTGT:
-; BROADWELL-NEXT:    jecxz JXTGT # sched: [2:0.50]
-; BROADWELL-NEXT:    jrcxz JXTGT # sched: [2:0.50]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_jecxz_jrcxz:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:  JXTGT:
-; SKYLAKE-NEXT:    jecxz JXTGT # sched: [2:0.50]
-; SKYLAKE-NEXT:    jrcxz JXTGT # sched: [2:0.50]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_jecxz_jrcxz:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:  JXTGT:
-; SKX-NEXT:    jecxz JXTGT # sched: [2:0.50]
-; SKX-NEXT:    jrcxz JXTGT # sched: [2:0.50]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_jecxz_jrcxz:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:  JXTGT:
-; BDVER2-NEXT:    jecxz JXTGT # sched: [1:1.00]
-; BDVER2-NEXT:    jrcxz JXTGT # sched: [1:1.00]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_jecxz_jrcxz:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:  JXTGT:
-; BTVER2-NEXT:    jecxz JXTGT # sched: [1:0.50]
-; BTVER2-NEXT:    jrcxz JXTGT # sched: [1:0.50]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_jecxz_jrcxz:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:  JXTGT:
-; ZNVER1-NEXT:    jecxz JXTGT # sched: [1:0.50]
-; ZNVER1-NEXT:    jrcxz JXTGT # sched: [1:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  call void asm sideeffect "JXTGT: \0A\09 jecxz JXTGT \0A\09 jrcxz JXTGT", ""()
-  ret void
-}
-
-; TODO - test_jmp
-
-define void @test_lahf_sahf() optsize {
-; GENERIC-LABEL: test_lahf_sahf:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    lahf # sched: [1:0.50]
-; GENERIC-NEXT:    sahf # sched: [1:0.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_lahf_sahf:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    lahf # sched: [2:1.00]
-; ATOM-NEXT:    sahf # sched: [2:1.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_lahf_sahf:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    lahf # sched: [1:0.50]
-; SLM-NEXT:    sahf # sched: [1:0.50]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_lahf_sahf:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    lahf # sched: [1:0.50]
-; SANDY-NEXT:    sahf # sched: [1:0.50]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_lahf_sahf:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    lahf # sched: [1:0.50]
-; HASWELL-NEXT:    sahf # sched: [1:0.50]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_lahf_sahf:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    lahf # sched: [1:0.50]
-; BROADWELL-NEXT:    sahf # sched: [1:0.50]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_lahf_sahf:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    lahf # sched: [1:0.50]
-; SKYLAKE-NEXT:    sahf # sched: [1:0.50]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_lahf_sahf:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    lahf # sched: [1:0.50]
-; SKX-NEXT:    sahf # sched: [1:0.50]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_lahf_sahf:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    lahf # sched: [2:0.50]
-; BDVER2-NEXT:    sahf # sched: [2:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_lahf_sahf:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    lahf # sched: [1:0.50]
-; BTVER2-NEXT:    sahf # sched: [1:0.50]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_lahf_sahf:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    lahf # sched: [100:0.25]
-; ZNVER1-NEXT:    sahf # sched: [2:0.25]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  tail call void asm "lahf \0A\09 sahf", ""() nounwind
-  ret void
-}
-
-; TODO - test_lds
-; TODO - test_les
-; TODO - test_lfs
-; TODO - test_lgs
-; TODO - test_lss
-
-; TODO - test_lea
-
-define void @test_leave() optsize {
-; GENERIC-LABEL: test_leave:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    leave # sched: [7:0.67]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_leave:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    leave # sched: [2:1.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_leave:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    leave # sched: [1:0.50]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_leave:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    leave # sched: [7:0.67]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_leave:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    leave # sched: [7:0.50]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_leave:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    leave # sched: [7:0.50]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_leave:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    leave # sched: [7:0.50]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_leave:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    leave # sched: [7:0.50]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_leave:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    leave # sched: [1:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_leave:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    leave # sched: [1:0.50]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_leave:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    leave # sched: [8:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  tail call void asm "leave", ""() nounwind
-  ret void
-}
-
-define void @test_lods() optsize {
-; GENERIC-LABEL: test_lods:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    lodsb (%rsi), %al # sched: [7:0.67]
-; GENERIC-NEXT:    lodsw (%rsi), %ax # sched: [7:0.67]
-; GENERIC-NEXT:    lodsl (%rsi), %eax # sched: [6:0.50]
-; GENERIC-NEXT:    lodsq (%rsi), %rax # sched: [6:0.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_lods:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    lodsb (%rsi), %al # sched: [2:1.00]
-; ATOM-NEXT:    lodsw (%rsi), %ax # sched: [2:1.00]
-; ATOM-NEXT:    lodsl (%rsi), %eax # sched: [2:1.00]
-; ATOM-NEXT:    lodsq (%rsi), %rax # sched: [2:1.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_lods:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    lodsb (%rsi), %al # sched: [100:1.00]
-; SLM-NEXT:    lodsw (%rsi), %ax # sched: [100:1.00]
-; SLM-NEXT:    lodsl (%rsi), %eax # sched: [100:1.00]
-; SLM-NEXT:    lodsq (%rsi), %rax # sched: [100:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_lods:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    lodsb (%rsi), %al # sched: [7:0.67]
-; SANDY-NEXT:    lodsw (%rsi), %ax # sched: [7:0.67]
-; SANDY-NEXT:    lodsl (%rsi), %eax # sched: [6:0.50]
-; SANDY-NEXT:    lodsq (%rsi), %rax # sched: [6:0.50]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_lods:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    lodsb (%rsi), %al # sched: [1:0.50]
-; HASWELL-NEXT:    lodsw (%rsi), %ax # sched: [1:0.50]
-; HASWELL-NEXT:    lodsl (%rsi), %eax # sched: [1:0.50]
-; HASWELL-NEXT:    lodsq (%rsi), %rax # sched: [1:0.50]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_lods:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    lodsb (%rsi), %al # sched: [100:0.25]
-; BROADWELL-NEXT:    lodsw (%rsi), %ax # sched: [100:0.25]
-; BROADWELL-NEXT:    lodsl (%rsi), %eax # sched: [100:0.25]
-; BROADWELL-NEXT:    lodsq (%rsi), %rax # sched: [100:0.25]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_lods:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    lodsb (%rsi), %al # sched: [100:0.25]
-; SKYLAKE-NEXT:    lodsw (%rsi), %ax # sched: [100:0.25]
-; SKYLAKE-NEXT:    lodsl (%rsi), %eax # sched: [100:0.25]
-; SKYLAKE-NEXT:    lodsq (%rsi), %rax # sched: [100:0.25]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_lods:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    lodsb (%rsi), %al # sched: [100:0.25]
-; SKX-NEXT:    lodsw (%rsi), %ax # sched: [100:0.25]
-; SKX-NEXT:    lodsl (%rsi), %eax # sched: [100:0.25]
-; SKX-NEXT:    lodsq (%rsi), %rax # sched: [100:0.25]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_lods:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    lodsb (%rsi), %al # sched: [100:0.50]
-; BDVER2-NEXT:    lodsw (%rsi), %ax # sched: [100:0.50]
-; BDVER2-NEXT:    lodsl (%rsi), %eax # sched: [100:0.50]
-; BDVER2-NEXT:    lodsq (%rsi), %rax # sched: [100:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_lods:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    lodsb (%rsi), %al # sched: [100:0.50]
-; BTVER2-NEXT:    lodsw (%rsi), %ax # sched: [100:0.50]
-; BTVER2-NEXT:    lodsl (%rsi), %eax # sched: [100:0.50]
-; BTVER2-NEXT:    lodsq (%rsi), %rax # sched: [100:0.50]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_lods:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    lodsb (%rsi), %al # sched: [100:0.25]
-; ZNVER1-NEXT:    lodsw (%rsi), %ax # sched: [100:0.25]
-; ZNVER1-NEXT:    lodsl (%rsi), %eax # sched: [100:0.25]
-; ZNVER1-NEXT:    lodsq (%rsi), %rax # sched: [100:0.25]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  call void asm sideeffect "lodsb \0A\09 lodsw \0A\09 lodsl \0A\09 lodsq", ""()
-  ret void
-}
-
-define void @test_loop() optsize {
-; GENERIC-LABEL: test_loop:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:  LTGT:
-; GENERIC-NEXT:    loop LTGT # sched: [1:1.00]
-; GENERIC-NEXT:    loope LTGT # sched: [1:1.00]
-; GENERIC-NEXT:    loopne LTGT # sched: [1:1.00]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_loop:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:  LTGT:
-; ATOM-NEXT:    loop LTGT # sched: [18:9.00]
-; ATOM-NEXT:    loope LTGT # sched: [8:4.00]
-; ATOM-NEXT:    loopne LTGT # sched: [17:8.50]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_loop:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:  LTGT:
-; SLM-NEXT:    loop LTGT # sched: [1:1.00]
-; SLM-NEXT:    loope LTGT # sched: [1:1.00]
-; SLM-NEXT:    loopne LTGT # sched: [1:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_loop:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:  LTGT:
-; SANDY-NEXT:    loop LTGT # sched: [1:1.00]
-; SANDY-NEXT:    loope LTGT # sched: [1:1.00]
-; SANDY-NEXT:    loopne LTGT # sched: [1:1.00]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_loop:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:  LTGT:
-; HASWELL-NEXT:    loop LTGT # sched: [7:2.00]
-; HASWELL-NEXT:    loope LTGT # sched: [11:2.75]
-; HASWELL-NEXT:    loopne LTGT # sched: [11:2.75]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_loop:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:  LTGT:
-; BROADWELL-NEXT:    loop LTGT # sched: [7:2.00]
-; BROADWELL-NEXT:    loope LTGT # sched: [11:2.75]
-; BROADWELL-NEXT:    loopne LTGT # sched: [11:2.75]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_loop:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:  LTGT:
-; SKYLAKE-NEXT:    loop LTGT # sched: [7:2.00]
-; SKYLAKE-NEXT:    loope LTGT # sched: [11:2.75]
-; SKYLAKE-NEXT:    loopne LTGT # sched: [11:2.75]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_loop:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:  LTGT:
-; SKX-NEXT:    loop LTGT # sched: [7:2.00]
-; SKX-NEXT:    loope LTGT # sched: [11:2.75]
-; SKX-NEXT:    loopne LTGT # sched: [11:2.75]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_loop:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:  LTGT:
-; BDVER2-NEXT:    loop LTGT # sched: [1:1.00]
-; BDVER2-NEXT:    loope LTGT # sched: [1:1.00]
-; BDVER2-NEXT:    loopne LTGT # sched: [1:1.00]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_loop:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:  LTGT:
-; BTVER2-NEXT:    loop LTGT # sched: [1:0.50]
-; BTVER2-NEXT:    loope LTGT # sched: [1:0.50]
-; BTVER2-NEXT:    loopne LTGT # sched: [1:0.50]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_loop:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:  LTGT:
-; ZNVER1-NEXT:    loop LTGT # sched: [1:0.50]
-; ZNVER1-NEXT:    loope LTGT # sched: [1:0.50]
-; ZNVER1-NEXT:    loopne LTGT # sched: [1:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  call void asm sideeffect "LTGT: \0A\09 loop LTGT \0A\09 loope LTGT \0A\09 loopne LTGT", ""()
-  ret void
-}
-
-; TODO - test_mov
-
-define void @test_movnti(i32 %a0, i32 *%a1, i64 %a2, i64 *%a3) optsize {
-; GENERIC-LABEL: test_movnti:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    movntil %edi, (%rsi) # sched: [1:1.00]
-; GENERIC-NEXT:    movntiq %rdx, (%rcx) # sched: [1:1.00]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_movnti:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    movntil %edi, (%rsi) # sched: [1:1.00]
-; ATOM-NEXT:    movntiq %rdx, (%rcx) # sched: [1:1.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_movnti:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    movntil %edi, (%rsi) # sched: [1:1.00]
-; SLM-NEXT:    movntiq %rdx, (%rcx) # sched: [1:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_movnti:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    movntil %edi, (%rsi) # sched: [1:1.00]
-; SANDY-NEXT:    movntiq %rdx, (%rcx) # sched: [1:1.00]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_movnti:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    movntil %edi, (%rsi) # sched: [1:1.00]
-; HASWELL-NEXT:    movntiq %rdx, (%rcx) # sched: [1:1.00]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_movnti:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    movntil %edi, (%rsi) # sched: [1:1.00]
-; BROADWELL-NEXT:    movntiq %rdx, (%rcx) # sched: [1:1.00]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_movnti:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    movntil %edi, (%rsi) # sched: [1:1.00]
-; SKYLAKE-NEXT:    movntiq %rdx, (%rcx) # sched: [1:1.00]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_movnti:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    movntil %edi, (%rsi) # sched: [1:1.00]
-; SKX-NEXT:    movntiq %rdx, (%rcx) # sched: [1:1.00]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_movnti:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    movntil %edi, (%rsi) # sched: [1:1.00]
-; BDVER2-NEXT:    movntiq %rdx, (%rcx) # sched: [1:1.00]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_movnti:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    movntil %edi, (%rsi) # sched: [1:1.00]
-; BTVER2-NEXT:    movntiq %rdx, (%rcx) # sched: [1:1.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_movnti:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    movntil %edi, (%rsi) # sched: [1:0.50]
-; ZNVER1-NEXT:    movntiq %rdx, (%rcx) # sched: [1:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  call void asm sideeffect "movnti $0, $1 \0A\09 movnti $2, $3", "r,*m,r,*m"(i32 %a0, i32 *%a1, i64 %a2, i64 *%a3)
-  ret void
-}
-
-define void @test_movs() optsize {
-; GENERIC-LABEL: test_movs:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    movsb (%rsi), %es:(%rdi) # sched: [8:1.00]
-; GENERIC-NEXT:    movsw (%rsi), %es:(%rdi) # sched: [8:1.00]
-; GENERIC-NEXT:    movsl (%rsi), %es:(%rdi) # sched: [8:1.00]
-; GENERIC-NEXT:    movsq (%rsi), %es:(%rdi) # sched: [8:1.00]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_movs:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    movsb (%rsi), %es:(%rdi) # sched: [3:1.50]
-; ATOM-NEXT:    movsw (%rsi), %es:(%rdi) # sched: [3:1.50]
-; ATOM-NEXT:    movsl (%rsi), %es:(%rdi) # sched: [3:1.50]
-; ATOM-NEXT:    movsq (%rsi), %es:(%rdi) # sched: [3:1.50]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_movs:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    movsb (%rsi), %es:(%rdi) # sched: [100:1.00]
-; SLM-NEXT:    movsw (%rsi), %es:(%rdi) # sched: [100:1.00]
-; SLM-NEXT:    movsl (%rsi), %es:(%rdi) # sched: [100:1.00]
-; SLM-NEXT:    movsq (%rsi), %es:(%rdi) # sched: [100:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_movs:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    movsb (%rsi), %es:(%rdi) # sched: [8:1.00]
-; SANDY-NEXT:    movsw (%rsi), %es:(%rdi) # sched: [8:1.00]
-; SANDY-NEXT:    movsl (%rsi), %es:(%rdi) # sched: [8:1.00]
-; SANDY-NEXT:    movsq (%rsi), %es:(%rdi) # sched: [8:1.00]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_movs:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    movsb (%rsi), %es:(%rdi) # sched: [4:1.00]
-; HASWELL-NEXT:    movsw (%rsi), %es:(%rdi) # sched: [4:1.00]
-; HASWELL-NEXT:    movsl (%rsi), %es:(%rdi) # sched: [4:1.00]
-; HASWELL-NEXT:    movsq (%rsi), %es:(%rdi) # sched: [4:1.00]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_movs:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    movsb (%rsi), %es:(%rdi) # sched: [100:0.25]
-; BROADWELL-NEXT:    movsw (%rsi), %es:(%rdi) # sched: [100:0.25]
-; BROADWELL-NEXT:    movsl (%rsi), %es:(%rdi) # sched: [100:0.25]
-; BROADWELL-NEXT:    movsq (%rsi), %es:(%rdi) # sched: [100:0.25]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_movs:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    movsb (%rsi), %es:(%rdi) # sched: [100:0.25]
-; SKYLAKE-NEXT:    movsw (%rsi), %es:(%rdi) # sched: [100:0.25]
-; SKYLAKE-NEXT:    movsl (%rsi), %es:(%rdi) # sched: [100:0.25]
-; SKYLAKE-NEXT:    movsq (%rsi), %es:(%rdi) # sched: [100:0.25]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_movs:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    movsb (%rsi), %es:(%rdi) # sched: [100:0.25]
-; SKX-NEXT:    movsw (%rsi), %es:(%rdi) # sched: [100:0.25]
-; SKX-NEXT:    movsl (%rsi), %es:(%rdi) # sched: [100:0.25]
-; SKX-NEXT:    movsq (%rsi), %es:(%rdi) # sched: [100:0.25]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_movs:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    movsb (%rsi), %es:(%rdi) # sched: [100:0.50]
-; BDVER2-NEXT:    movsw (%rsi), %es:(%rdi) # sched: [100:0.50]
-; BDVER2-NEXT:    movsl (%rsi), %es:(%rdi) # sched: [100:0.50]
-; BDVER2-NEXT:    movsq (%rsi), %es:(%rdi) # sched: [100:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_movs:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    movsb (%rsi), %es:(%rdi) # sched: [100:0.50]
-; BTVER2-NEXT:    movsw (%rsi), %es:(%rdi) # sched: [100:0.50]
-; BTVER2-NEXT:    movsl (%rsi), %es:(%rdi) # sched: [100:0.50]
-; BTVER2-NEXT:    movsq (%rsi), %es:(%rdi) # sched: [100:0.50]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_movs:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    movsb (%rsi), %es:(%rdi) # sched: [100:0.25]
-; ZNVER1-NEXT:    movsw (%rsi), %es:(%rdi) # sched: [100:0.25]
-; ZNVER1-NEXT:    movsl (%rsi), %es:(%rdi) # sched: [100:0.25]
-; ZNVER1-NEXT:    movsq (%rsi), %es:(%rdi) # sched: [100:0.25]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  call void asm sideeffect "movsb \0A\09 movsw \0A\09 movsl \0A\09 movsq", ""()
-  ret void
-}
-
-; TODO - test_movsx
-; TODO - test_movzx
-
-define i64 @test_movslq(i32 %a0, i32 *%a1) optsize {
-; GENERIC-LABEL: test_movslq:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    movslq %edi, %rax # sched: [1:0.33]
-; GENERIC-NEXT:    movslq (%rsi), %rcx # sched: [5:0.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    orq %rcx, %rax # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_movslq:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    movslq %edi, %rax # sched: [1:1.00]
-; ATOM-NEXT:    movslq (%rsi), %rcx # sched: [1:1.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    orq %rcx, %rax # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_movslq:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    movslq %edi, %rax # sched: [1:0.50]
-; SLM-NEXT:    movslq (%rsi), %rcx # sched: [4:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    orq %rcx, %rax # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_movslq:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    movslq %edi, %rax # sched: [1:0.33]
-; SANDY-NEXT:    movslq (%rsi), %rcx # sched: [5:0.50]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    orq %rcx, %rax # sched: [1:0.33]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_movslq:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    movslq %edi, %rax # sched: [1:0.25]
-; HASWELL-NEXT:    movslq (%rsi), %rcx # sched: [5:0.50]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    orq %rcx, %rax # sched: [1:0.25]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_movslq:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    movslq %edi, %rax # sched: [1:0.25]
-; BROADWELL-NEXT:    movslq (%rsi), %rcx # sched: [5:0.50]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    orq %rcx, %rax # sched: [1:0.25]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_movslq:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    movslq %edi, %rax # sched: [1:0.25]
-; SKYLAKE-NEXT:    movslq (%rsi), %rcx # sched: [5:0.50]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    orq %rcx, %rax # sched: [1:0.25]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_movslq:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    movslq %edi, %rax # sched: [1:0.25]
-; SKX-NEXT:    movslq (%rsi), %rcx # sched: [5:0.50]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    orq %rcx, %rax # sched: [1:0.25]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_movslq:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    movslq %edi, %rax # sched: [1:0.50]
-; BDVER2-NEXT:    movslq (%rsi), %rcx # sched: [5:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    orq %rcx, %rax # sched: [1:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_movslq:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    movslq %edi, %rax # sched: [1:0.50]
-; BTVER2-NEXT:    movslq (%rsi), %rcx # sched: [4:1.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    orq %rcx, %rax # sched: [1:0.50]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_movslq:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    movslq %edi, %rax # sched: [1:0.25]
-; ZNVER1-NEXT:    movslq (%rsi), %rcx # sched: [5:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    orq %rcx, %rax # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call { i64, i64 } asm sideeffect "movslq $2, $0 \0A\09 movslq $3, $1", "=r,=r,r,*m"(i32 %a0, i32 *%a1)
-  %2 = extractvalue { i64, i64 } %1, 0
-  %3 = extractvalue { i64, i64 } %1, 1
-  %4 = or i64 %2, %3
-  ret i64 %4
-}
-
-define void @test_mul(i8 %a0, i16 %a1, i32 %a2, i64 %a3, i8 *%p0, i16 *%p1, i32 *%p2, i64 *%p3) optsize {
-; GENERIC-LABEL: test_mul:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    movq {{[0-9]+}}(%rsp), %r10 # sched: [5:0.50]
-; GENERIC-NEXT:    movq {{[0-9]+}}(%rsp), %rax # sched: [5:0.50]
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    mulb %dil # sched: [3:1.00]
-; GENERIC-NEXT:    mulb (%r8) # sched: [8:1.00]
-; GENERIC-NEXT:    mulw %si # sched: [4:1.33]
-; GENERIC-NEXT:    mulw (%r9) # sched: [9:1.33]
-; GENERIC-NEXT:    mull %edx # sched: [4:1.00]
-; GENERIC-NEXT:    mull (%rax) # sched: [9:1.00]
-; GENERIC-NEXT:    mulq %rcx # sched: [4:1.00]
-; GENERIC-NEXT:    mulq (%r10) # sched: [9:1.00]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_mul:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    movq {{[0-9]+}}(%rsp), %r10 # sched: [1:1.00]
-; ATOM-NEXT:    movq {{[0-9]+}}(%rsp), %rax # sched: [1:1.00]
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    mulb %dil # sched: [7:3.50]
-; ATOM-NEXT:    mulb (%r8) # sched: [7:3.50]
-; ATOM-NEXT:    mulw %si # sched: [7:3.50]
-; ATOM-NEXT:    mulw (%r9) # sched: [8:4.00]
-; ATOM-NEXT:    mull %edx # sched: [6:3.00]
-; ATOM-NEXT:    mull (%rax) # sched: [7:3.50]
-; ATOM-NEXT:    mulq %rcx # sched: [12:6.00]
-; ATOM-NEXT:    mulq (%r10) # sched: [12:6.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_mul:
-; SLM:       # %bb.0:
-; SLM-NEXT:    movq {{[0-9]+}}(%rsp), %r10 # sched: [3:1.00]
-; SLM-NEXT:    movq {{[0-9]+}}(%rsp), %rax # sched: [3:1.00]
-; SLM-NEXT:    #APP
-; SLM-NEXT:    mulb %dil # sched: [3:1.00]
-; SLM-NEXT:    mulb (%r8) # sched: [6:1.00]
-; SLM-NEXT:    mulw %si # sched: [3:1.00]
-; SLM-NEXT:    mulw (%r9) # sched: [6:1.00]
-; SLM-NEXT:    mull %edx # sched: [3:1.00]
-; SLM-NEXT:    mull (%rax) # sched: [6:1.00]
-; SLM-NEXT:    mulq %rcx # sched: [3:1.00]
-; SLM-NEXT:    mulq (%r10) # sched: [6:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_mul:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    movq {{[0-9]+}}(%rsp), %r10 # sched: [5:0.50]
-; SANDY-NEXT:    movq {{[0-9]+}}(%rsp), %rax # sched: [5:0.50]
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    mulb %dil # sched: [3:1.00]
-; SANDY-NEXT:    mulb (%r8) # sched: [8:1.00]
-; SANDY-NEXT:    mulw %si # sched: [4:1.33]
-; SANDY-NEXT:    mulw (%r9) # sched: [9:1.33]
-; SANDY-NEXT:    mull %edx # sched: [4:1.00]
-; SANDY-NEXT:    mull (%rax) # sched: [9:1.00]
-; SANDY-NEXT:    mulq %rcx # sched: [4:1.00]
-; SANDY-NEXT:    mulq (%r10) # sched: [9:1.00]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_mul:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    movq {{[0-9]+}}(%rsp), %r10 # sched: [5:0.50]
-; HASWELL-NEXT:    movq {{[0-9]+}}(%rsp), %rax # sched: [5:0.50]
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    mulb %dil # sched: [3:1.00]
-; HASWELL-NEXT:    mulb (%r8) # sched: [8:1.00]
-; HASWELL-NEXT:    mulw %si # sched: [4:1.00]
-; HASWELL-NEXT:    mulw (%r9) # sched: [9:1.00]
-; HASWELL-NEXT:    mull %edx # sched: [4:1.00]
-; HASWELL-NEXT:    mull (%rax) # sched: [9:1.00]
-; HASWELL-NEXT:    mulq %rcx # sched: [4:1.00]
-; HASWELL-NEXT:    mulq (%r10) # sched: [9:1.00]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_mul:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    movq {{[0-9]+}}(%rsp), %r10 # sched: [5:0.50]
-; BROADWELL-NEXT:    movq {{[0-9]+}}(%rsp), %rax # sched: [5:0.50]
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    mulb %dil # sched: [3:1.00]
-; BROADWELL-NEXT:    mulb (%r8) # sched: [8:1.00]
-; BROADWELL-NEXT:    mulw %si # sched: [4:1.00]
-; BROADWELL-NEXT:    mulw (%r9) # sched: [9:1.00]
-; BROADWELL-NEXT:    mull %edx # sched: [4:1.00]
-; BROADWELL-NEXT:    mull (%rax) # sched: [9:1.00]
-; BROADWELL-NEXT:    mulq %rcx # sched: [4:1.00]
-; BROADWELL-NEXT:    mulq (%r10) # sched: [9:1.00]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_mul:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    movq {{[0-9]+}}(%rsp), %r10 # sched: [5:0.50]
-; SKYLAKE-NEXT:    movq {{[0-9]+}}(%rsp), %rax # sched: [5:0.50]
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    mulb %dil # sched: [3:1.00]
-; SKYLAKE-NEXT:    mulb (%r8) # sched: [8:1.00]
-; SKYLAKE-NEXT:    mulw %si # sched: [4:1.00]
-; SKYLAKE-NEXT:    mulw (%r9) # sched: [9:1.00]
-; SKYLAKE-NEXT:    mull %edx # sched: [4:1.00]
-; SKYLAKE-NEXT:    mull (%rax) # sched: [9:1.00]
-; SKYLAKE-NEXT:    mulq %rcx # sched: [4:1.00]
-; SKYLAKE-NEXT:    mulq (%r10) # sched: [9:1.00]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_mul:
-; SKX:       # %bb.0:
-; SKX-NEXT:    movq {{[0-9]+}}(%rsp), %r10 # sched: [5:0.50]
-; SKX-NEXT:    movq {{[0-9]+}}(%rsp), %rax # sched: [5:0.50]
-; SKX-NEXT:    #APP
-; SKX-NEXT:    mulb %dil # sched: [3:1.00]
-; SKX-NEXT:    mulb (%r8) # sched: [8:1.00]
-; SKX-NEXT:    mulw %si # sched: [4:1.00]
-; SKX-NEXT:    mulw (%r9) # sched: [9:1.00]
-; SKX-NEXT:    mull %edx # sched: [4:1.00]
-; SKX-NEXT:    mull (%rax) # sched: [9:1.00]
-; SKX-NEXT:    mulq %rcx # sched: [4:1.00]
-; SKX-NEXT:    mulq (%r10) # sched: [9:1.00]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_mul:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    movq {{[0-9]+}}(%rsp), %r10 # sched: [5:0.50]
-; BDVER2-NEXT:    movq {{[0-9]+}}(%rsp), %rax # sched: [5:0.50]
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    mulb %dil # sched: [4:1.00]
-; BDVER2-NEXT:    mulb (%r8) # sched: [8:1.00]
-; BDVER2-NEXT:    mulw %si # sched: [4:1.00]
-; BDVER2-NEXT:    mulw (%r9) # sched: [8:1.00]
-; BDVER2-NEXT:    mull %edx # sched: [4:1.00]
-; BDVER2-NEXT:    mull (%rax) # sched: [8:1.00]
-; BDVER2-NEXT:    mulq %rcx # sched: [6:4.00]
-; BDVER2-NEXT:    mulq (%r10) # sched: [10:4.00]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_mul:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    movq {{[0-9]+}}(%rsp), %r10 # sched: [5:1.00]
-; BTVER2-NEXT:    movq {{[0-9]+}}(%rsp), %rax # sched: [5:1.00]
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    mulb %dil # sched: [3:1.00]
-; BTVER2-NEXT:    mulb (%r8) # sched: [6:1.00]
-; BTVER2-NEXT:    mulw %si # sched: [3:1.00]
-; BTVER2-NEXT:    mulw (%r9) # sched: [6:1.00]
-; BTVER2-NEXT:    mull %edx # sched: [3:1.00]
-; BTVER2-NEXT:    mull (%rax) # sched: [6:1.00]
-; BTVER2-NEXT:    mulq %rcx # sched: [6:4.00]
-; BTVER2-NEXT:    mulq (%r10) # sched: [9:4.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_mul:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    movq {{[0-9]+}}(%rsp), %r10 # sched: [8:0.50]
-; ZNVER1-NEXT:    movq {{[0-9]+}}(%rsp), %rax # sched: [8:0.50]
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    mulb %dil # sched: [4:1.00]
-; ZNVER1-NEXT:    mulb (%r8) # sched: [8:1.00]
-; ZNVER1-NEXT:    mulw %si # sched: [3:1.00]
-; ZNVER1-NEXT:    mulw (%r9) # sched: [8:1.00]
-; ZNVER1-NEXT:    mull %edx # sched: [3:1.00]
-; ZNVER1-NEXT:    mull (%rax) # sched: [8:1.00]
-; ZNVER1-NEXT:    mulq %rcx # sched: [4:1.00]
-; ZNVER1-NEXT:    mulq (%r10) # sched: [9:1.00]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  tail call void asm "mulb $0 \0A\09 mulb $4 \0A\09 mulw $1 \0A\09 mulw $5 \0A\09 mull $2 \0A\09 mull $6 \0A\09 mulq $3 \0A\09 mulq $7", "r,r,r,r,*m,*m,*m,*m"(i8 %a0, i16 %a1, i32 %a2, i64 %a3, i8 *%p0, i16 *%p1, i32 *%p2, i64 *%p3) nounwind
-  ret void
-}
-
-define void @test_neg(i8 %a0, i16 %a1, i32 %a2, i64 %a3, i8 *%p0, i16 *%p1, i32 *%p2, i64 *%p3) optsize {
-; GENERIC-LABEL: test_neg:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    movq {{[0-9]+}}(%rsp), %r10 # sched: [5:0.50]
-; GENERIC-NEXT:    movq {{[0-9]+}}(%rsp), %rax # sched: [5:0.50]
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    negb %dil # sched: [1:0.33]
-; GENERIC-NEXT:    negb (%r8) # sched: [7:1.00]
-; GENERIC-NEXT:    negw %si # sched: [1:0.33]
-; GENERIC-NEXT:    negw (%r9) # sched: [7:1.00]
-; GENERIC-NEXT:    negl %edx # sched: [1:0.33]
-; GENERIC-NEXT:    negl (%rax) # sched: [7:1.00]
-; GENERIC-NEXT:    negq %rcx # sched: [1:0.33]
-; GENERIC-NEXT:    negq (%r10) # sched: [7:1.00]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_neg:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    movq {{[0-9]+}}(%rsp), %r10 # sched: [1:1.00]
-; ATOM-NEXT:    movq {{[0-9]+}}(%rsp), %rax # sched: [1:1.00]
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    negb %dil # sched: [1:0.50]
-; ATOM-NEXT:    negb (%r8) # sched: [1:1.00]
-; ATOM-NEXT:    negw %si # sched: [1:0.50]
-; ATOM-NEXT:    negw (%r9) # sched: [1:1.00]
-; ATOM-NEXT:    negl %edx # sched: [1:0.50]
-; ATOM-NEXT:    negl (%rax) # sched: [1:1.00]
-; ATOM-NEXT:    negq %rcx # sched: [1:0.50]
-; ATOM-NEXT:    negq (%r10) # sched: [1:1.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_neg:
-; SLM:       # %bb.0:
-; SLM-NEXT:    movq {{[0-9]+}}(%rsp), %r10 # sched: [3:1.00]
-; SLM-NEXT:    movq {{[0-9]+}}(%rsp), %rax # sched: [3:1.00]
-; SLM-NEXT:    #APP
-; SLM-NEXT:    negb %dil # sched: [1:0.50]
-; SLM-NEXT:    negb (%r8) # sched: [5:2.00]
-; SLM-NEXT:    negw %si # sched: [1:0.50]
-; SLM-NEXT:    negw (%r9) # sched: [5:2.00]
-; SLM-NEXT:    negl %edx # sched: [1:0.50]
-; SLM-NEXT:    negl (%rax) # sched: [5:2.00]
-; SLM-NEXT:    negq %rcx # sched: [1:0.50]
-; SLM-NEXT:    negq (%r10) # sched: [5:2.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_neg:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    movq {{[0-9]+}}(%rsp), %r10 # sched: [5:0.50]
-; SANDY-NEXT:    movq {{[0-9]+}}(%rsp), %rax # sched: [5:0.50]
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    negb %dil # sched: [1:0.33]
-; SANDY-NEXT:    negb (%r8) # sched: [7:1.00]
-; SANDY-NEXT:    negw %si # sched: [1:0.33]
-; SANDY-NEXT:    negw (%r9) # sched: [7:1.00]
-; SANDY-NEXT:    negl %edx # sched: [1:0.33]
-; SANDY-NEXT:    negl (%rax) # sched: [7:1.00]
-; SANDY-NEXT:    negq %rcx # sched: [1:0.33]
-; SANDY-NEXT:    negq (%r10) # sched: [7:1.00]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_neg:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    movq {{[0-9]+}}(%rsp), %r10 # sched: [5:0.50]
-; HASWELL-NEXT:    movq {{[0-9]+}}(%rsp), %rax # sched: [5:0.50]
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    negb %dil # sched: [1:0.25]
-; HASWELL-NEXT:    negb (%r8) # sched: [7:1.00]
-; HASWELL-NEXT:    negw %si # sched: [1:0.25]
-; HASWELL-NEXT:    negw (%r9) # sched: [7:1.00]
-; HASWELL-NEXT:    negl %edx # sched: [1:0.25]
-; HASWELL-NEXT:    negl (%rax) # sched: [7:1.00]
-; HASWELL-NEXT:    negq %rcx # sched: [1:0.25]
-; HASWELL-NEXT:    negq (%r10) # sched: [7:1.00]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_neg:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    movq {{[0-9]+}}(%rsp), %r10 # sched: [5:0.50]
-; BROADWELL-NEXT:    movq {{[0-9]+}}(%rsp), %rax # sched: [5:0.50]
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    negb %dil # sched: [1:0.25]
-; BROADWELL-NEXT:    negb (%r8) # sched: [7:1.00]
-; BROADWELL-NEXT:    negw %si # sched: [1:0.25]
-; BROADWELL-NEXT:    negw (%r9) # sched: [7:1.00]
-; BROADWELL-NEXT:    negl %edx # sched: [1:0.25]
-; BROADWELL-NEXT:    negl (%rax) # sched: [7:1.00]
-; BROADWELL-NEXT:    negq %rcx # sched: [1:0.25]
-; BROADWELL-NEXT:    negq (%r10) # sched: [7:1.00]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_neg:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    movq {{[0-9]+}}(%rsp), %r10 # sched: [5:0.50]
-; SKYLAKE-NEXT:    movq {{[0-9]+}}(%rsp), %rax # sched: [5:0.50]
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    negb %dil # sched: [1:0.25]
-; SKYLAKE-NEXT:    negb (%r8) # sched: [7:1.00]
-; SKYLAKE-NEXT:    negw %si # sched: [1:0.25]
-; SKYLAKE-NEXT:    negw (%r9) # sched: [7:1.00]
-; SKYLAKE-NEXT:    negl %edx # sched: [1:0.25]
-; SKYLAKE-NEXT:    negl (%rax) # sched: [7:1.00]
-; SKYLAKE-NEXT:    negq %rcx # sched: [1:0.25]
-; SKYLAKE-NEXT:    negq (%r10) # sched: [7:1.00]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_neg:
-; SKX:       # %bb.0:
-; SKX-NEXT:    movq {{[0-9]+}}(%rsp), %r10 # sched: [5:0.50]
-; SKX-NEXT:    movq {{[0-9]+}}(%rsp), %rax # sched: [5:0.50]
-; SKX-NEXT:    #APP
-; SKX-NEXT:    negb %dil # sched: [1:0.25]
-; SKX-NEXT:    negb (%r8) # sched: [7:1.00]
-; SKX-NEXT:    negw %si # sched: [1:0.25]
-; SKX-NEXT:    negw (%r9) # sched: [7:1.00]
-; SKX-NEXT:    negl %edx # sched: [1:0.25]
-; SKX-NEXT:    negl (%rax) # sched: [7:1.00]
-; SKX-NEXT:    negq %rcx # sched: [1:0.25]
-; SKX-NEXT:    negq (%r10) # sched: [7:1.00]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_neg:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    movq {{[0-9]+}}(%rsp), %r10 # sched: [5:0.50]
-; BDVER2-NEXT:    movq {{[0-9]+}}(%rsp), %rax # sched: [5:0.50]
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    negb %dil # sched: [1:0.50]
-; BDVER2-NEXT:    negb (%r8) # sched: [6:1.00]
-; BDVER2-NEXT:    negw %si # sched: [1:0.50]
-; BDVER2-NEXT:    negw (%r9) # sched: [6:1.00]
-; BDVER2-NEXT:    negl %edx # sched: [1:0.50]
-; BDVER2-NEXT:    negl (%rax) # sched: [6:1.00]
-; BDVER2-NEXT:    negq %rcx # sched: [1:0.50]
-; BDVER2-NEXT:    negq (%r10) # sched: [6:1.00]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_neg:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    movq {{[0-9]+}}(%rsp), %r10 # sched: [5:1.00]
-; BTVER2-NEXT:    movq {{[0-9]+}}(%rsp), %rax # sched: [5:1.00]
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    negb %dil # sched: [1:0.50]
-; BTVER2-NEXT:    negb (%r8) # sched: [5:1.00]
-; BTVER2-NEXT:    negw %si # sched: [1:0.50]
-; BTVER2-NEXT:    negw (%r9) # sched: [5:1.00]
-; BTVER2-NEXT:    negl %edx # sched: [1:0.50]
-; BTVER2-NEXT:    negl (%rax) # sched: [5:1.00]
-; BTVER2-NEXT:    negq %rcx # sched: [1:0.50]
-; BTVER2-NEXT:    negq (%r10) # sched: [5:1.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_neg:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    movq {{[0-9]+}}(%rsp), %r10 # sched: [8:0.50]
-; ZNVER1-NEXT:    movq {{[0-9]+}}(%rsp), %rax # sched: [8:0.50]
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    negb %dil # sched: [1:0.25]
-; ZNVER1-NEXT:    negb (%r8) # sched: [5:0.50]
-; ZNVER1-NEXT:    negw %si # sched: [1:0.25]
-; ZNVER1-NEXT:    negw (%r9) # sched: [5:0.50]
-; ZNVER1-NEXT:    negl %edx # sched: [1:0.25]
-; ZNVER1-NEXT:    negl (%rax) # sched: [5:0.50]
-; ZNVER1-NEXT:    negq %rcx # sched: [1:0.25]
-; ZNVER1-NEXT:    negq (%r10) # sched: [5:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  tail call void asm "negb $0 \0A\09 negb $4 \0A\09 negw $1 \0A\09 negw $5 \0A\09 negl $2 \0A\09 negl $6 \0A\09 negq $3 \0A\09 negq $7", "r,r,r,r,*m,*m,*m,*m"(i8 %a0, i16 %a1, i32 %a2, i64 %a3, i8 *%p0, i16 *%p1, i32 *%p2, i64 *%p3) nounwind
-  ret void
-}
-
-define void @test_nop(i16 %a0, i32 %a1, i64 %a2, i16 *%p0, i32 *%p1, i64 *%p2) optsize {
-; GENERIC-LABEL: test_nop:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    nop # sched: [1:0.25]
-; GENERIC-NEXT:    nopw %di # sched: [1:0.25]
-; GENERIC-NEXT:    nopw (%rcx) # sched: [1:0.25]
-; GENERIC-NEXT:    nopl %esi # sched: [1:0.25]
-; GENERIC-NEXT:    nopl (%r8) # sched: [1:0.25]
-; GENERIC-NEXT:    nopq %rdx # sched: [1:0.25]
-; GENERIC-NEXT:    nopq (%r9) # sched: [1:0.25]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_nop:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nopw %di # sched: [1:0.50]
-; ATOM-NEXT:    nopw (%rcx) # sched: [1:0.50]
-; ATOM-NEXT:    nopl %esi # sched: [1:0.50]
-; ATOM-NEXT:    nopl (%r8) # sched: [1:0.50]
-; ATOM-NEXT:    nopq %rdx # sched: [1:0.50]
-; ATOM-NEXT:    nopq (%r9) # sched: [1:0.50]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_nop:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    nop # sched: [1:0.50]
-; SLM-NEXT:    nopw %di # sched: [1:0.50]
-; SLM-NEXT:    nopw (%rcx) # sched: [1:0.50]
-; SLM-NEXT:    nopl %esi # sched: [1:0.50]
-; SLM-NEXT:    nopl (%r8) # sched: [1:0.50]
-; SLM-NEXT:    nopq %rdx # sched: [1:0.50]
-; SLM-NEXT:    nopq (%r9) # sched: [1:0.50]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_nop:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    nop # sched: [1:0.25]
-; SANDY-NEXT:    nopw %di # sched: [1:0.25]
-; SANDY-NEXT:    nopw (%rcx) # sched: [1:0.25]
-; SANDY-NEXT:    nopl %esi # sched: [1:0.25]
-; SANDY-NEXT:    nopl (%r8) # sched: [1:0.25]
-; SANDY-NEXT:    nopq %rdx # sched: [1:0.25]
-; SANDY-NEXT:    nopq (%r9) # sched: [1:0.25]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_nop:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    nop # sched: [1:0.25]
-; HASWELL-NEXT:    nopw %di # sched: [1:0.25]
-; HASWELL-NEXT:    nopw (%rcx) # sched: [1:0.25]
-; HASWELL-NEXT:    nopl %esi # sched: [1:0.25]
-; HASWELL-NEXT:    nopl (%r8) # sched: [1:0.25]
-; HASWELL-NEXT:    nopq %rdx # sched: [1:0.25]
-; HASWELL-NEXT:    nopq (%r9) # sched: [1:0.25]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_nop:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    nop # sched: [1:0.25]
-; BROADWELL-NEXT:    nopw %di # sched: [1:0.25]
-; BROADWELL-NEXT:    nopw (%rcx) # sched: [1:0.25]
-; BROADWELL-NEXT:    nopl %esi # sched: [1:0.25]
-; BROADWELL-NEXT:    nopl (%r8) # sched: [1:0.25]
-; BROADWELL-NEXT:    nopq %rdx # sched: [1:0.25]
-; BROADWELL-NEXT:    nopq (%r9) # sched: [1:0.25]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_nop:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    nop # sched: [1:0.17]
-; SKYLAKE-NEXT:    nopw %di # sched: [1:0.17]
-; SKYLAKE-NEXT:    nopw (%rcx) # sched: [1:0.17]
-; SKYLAKE-NEXT:    nopl %esi # sched: [1:0.17]
-; SKYLAKE-NEXT:    nopl (%r8) # sched: [1:0.17]
-; SKYLAKE-NEXT:    nopq %rdx # sched: [1:0.17]
-; SKYLAKE-NEXT:    nopq (%r9) # sched: [1:0.17]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_nop:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    nop # sched: [1:0.17]
-; SKX-NEXT:    nopw %di # sched: [1:0.17]
-; SKX-NEXT:    nopw (%rcx) # sched: [1:0.17]
-; SKX-NEXT:    nopl %esi # sched: [1:0.17]
-; SKX-NEXT:    nopl (%r8) # sched: [1:0.17]
-; SKX-NEXT:    nopq %rdx # sched: [1:0.17]
-; SKX-NEXT:    nopq (%r9) # sched: [1:0.17]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_nop:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    nop # sched: [1:0.50]
-; BDVER2-NEXT:    nopw %di # sched: [1:0.50]
-; BDVER2-NEXT:    nopw (%rcx) # sched: [1:0.50]
-; BDVER2-NEXT:    nopl %esi # sched: [1:0.50]
-; BDVER2-NEXT:    nopl (%r8) # sched: [1:0.50]
-; BDVER2-NEXT:    nopq %rdx # sched: [1:0.50]
-; BDVER2-NEXT:    nopq (%r9) # sched: [1:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_nop:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    nop # sched: [1:0.50]
-; BTVER2-NEXT:    nopw %di # sched: [1:0.50]
-; BTVER2-NEXT:    nopw (%rcx) # sched: [1:0.50]
-; BTVER2-NEXT:    nopl %esi # sched: [1:0.50]
-; BTVER2-NEXT:    nopl (%r8) # sched: [1:0.50]
-; BTVER2-NEXT:    nopq %rdx # sched: [1:0.50]
-; BTVER2-NEXT:    nopq (%r9) # sched: [1:0.50]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_nop:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    nop # sched: [1:0.25]
-; ZNVER1-NEXT:    nopw %di # sched: [1:0.25]
-; ZNVER1-NEXT:    nopw (%rcx) # sched: [1:0.25]
-; ZNVER1-NEXT:    nopl %esi # sched: [1:0.25]
-; ZNVER1-NEXT:    nopl (%r8) # sched: [1:0.25]
-; ZNVER1-NEXT:    nopq %rdx # sched: [1:0.25]
-; ZNVER1-NEXT:    nopq (%r9) # sched: [1:0.25]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  tail call void asm "nop \0A\09 nopw $0 \0A\09 nopw $3 \0A\09 nopl $1 \0A\09 nopl $4 \0A\09 nopq $2 \0A\09 nopq $5", "r,r,r,*m,*m,*m"(i16 %a0, i32 %a1, i64 %a2, i16 *%p0, i32 *%p1, i64 *%p2) nounwind
-  ret void
-}
-
-define void @test_not(i8 %a0, i16 %a1, i32 %a2, i64 %a3, i8 *%p0, i16 *%p1, i32 *%p2, i64 *%p3) optsize {
-; GENERIC-LABEL: test_not:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    movq {{[0-9]+}}(%rsp), %r10 # sched: [5:0.50]
-; GENERIC-NEXT:    movq {{[0-9]+}}(%rsp), %rax # sched: [5:0.50]
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    notb %dil # sched: [1:0.33]
-; GENERIC-NEXT:    notb (%r8) # sched: [7:1.00]
-; GENERIC-NEXT:    notw %si # sched: [1:0.33]
-; GENERIC-NEXT:    notw (%r9) # sched: [7:1.00]
-; GENERIC-NEXT:    notl %edx # sched: [1:0.33]
-; GENERIC-NEXT:    notl (%rax) # sched: [7:1.00]
-; GENERIC-NEXT:    notq %rcx # sched: [1:0.33]
-; GENERIC-NEXT:    notq (%r10) # sched: [7:1.00]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_not:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    movq {{[0-9]+}}(%rsp), %r10 # sched: [1:1.00]
-; ATOM-NEXT:    movq {{[0-9]+}}(%rsp), %rax # sched: [1:1.00]
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    notb %dil # sched: [1:0.50]
-; ATOM-NEXT:    notb (%r8) # sched: [1:1.00]
-; ATOM-NEXT:    notw %si # sched: [1:0.50]
-; ATOM-NEXT:    notw (%r9) # sched: [1:1.00]
-; ATOM-NEXT:    notl %edx # sched: [1:0.50]
-; ATOM-NEXT:    notl (%rax) # sched: [1:1.00]
-; ATOM-NEXT:    notq %rcx # sched: [1:0.50]
-; ATOM-NEXT:    notq (%r10) # sched: [1:1.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_not:
-; SLM:       # %bb.0:
-; SLM-NEXT:    movq {{[0-9]+}}(%rsp), %r10 # sched: [3:1.00]
-; SLM-NEXT:    movq {{[0-9]+}}(%rsp), %rax # sched: [3:1.00]
-; SLM-NEXT:    #APP
-; SLM-NEXT:    notb %dil # sched: [1:0.50]
-; SLM-NEXT:    notb (%r8) # sched: [5:2.00]
-; SLM-NEXT:    notw %si # sched: [1:0.50]
-; SLM-NEXT:    notw (%r9) # sched: [5:2.00]
-; SLM-NEXT:    notl %edx # sched: [1:0.50]
-; SLM-NEXT:    notl (%rax) # sched: [5:2.00]
-; SLM-NEXT:    notq %rcx # sched: [1:0.50]
-; SLM-NEXT:    notq (%r10) # sched: [5:2.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_not:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    movq {{[0-9]+}}(%rsp), %r10 # sched: [5:0.50]
-; SANDY-NEXT:    movq {{[0-9]+}}(%rsp), %rax # sched: [5:0.50]
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    notb %dil # sched: [1:0.33]
-; SANDY-NEXT:    notb (%r8) # sched: [7:1.00]
-; SANDY-NEXT:    notw %si # sched: [1:0.33]
-; SANDY-NEXT:    notw (%r9) # sched: [7:1.00]
-; SANDY-NEXT:    notl %edx # sched: [1:0.33]
-; SANDY-NEXT:    notl (%rax) # sched: [7:1.00]
-; SANDY-NEXT:    notq %rcx # sched: [1:0.33]
-; SANDY-NEXT:    notq (%r10) # sched: [7:1.00]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_not:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    movq {{[0-9]+}}(%rsp), %r10 # sched: [5:0.50]
-; HASWELL-NEXT:    movq {{[0-9]+}}(%rsp), %rax # sched: [5:0.50]
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    notb %dil # sched: [1:0.25]
-; HASWELL-NEXT:    notb (%r8) # sched: [7:1.00]
-; HASWELL-NEXT:    notw %si # sched: [1:0.25]
-; HASWELL-NEXT:    notw (%r9) # sched: [7:1.00]
-; HASWELL-NEXT:    notl %edx # sched: [1:0.25]
-; HASWELL-NEXT:    notl (%rax) # sched: [7:1.00]
-; HASWELL-NEXT:    notq %rcx # sched: [1:0.25]
-; HASWELL-NEXT:    notq (%r10) # sched: [7:1.00]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_not:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    movq {{[0-9]+}}(%rsp), %r10 # sched: [5:0.50]
-; BROADWELL-NEXT:    movq {{[0-9]+}}(%rsp), %rax # sched: [5:0.50]
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    notb %dil # sched: [1:0.25]
-; BROADWELL-NEXT:    notb (%r8) # sched: [7:1.00]
-; BROADWELL-NEXT:    notw %si # sched: [1:0.25]
-; BROADWELL-NEXT:    notw (%r9) # sched: [7:1.00]
-; BROADWELL-NEXT:    notl %edx # sched: [1:0.25]
-; BROADWELL-NEXT:    notl (%rax) # sched: [7:1.00]
-; BROADWELL-NEXT:    notq %rcx # sched: [1:0.25]
-; BROADWELL-NEXT:    notq (%r10) # sched: [7:1.00]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_not:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    movq {{[0-9]+}}(%rsp), %r10 # sched: [5:0.50]
-; SKYLAKE-NEXT:    movq {{[0-9]+}}(%rsp), %rax # sched: [5:0.50]
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    notb %dil # sched: [1:0.25]
-; SKYLAKE-NEXT:    notb (%r8) # sched: [7:1.00]
-; SKYLAKE-NEXT:    notw %si # sched: [1:0.25]
-; SKYLAKE-NEXT:    notw (%r9) # sched: [7:1.00]
-; SKYLAKE-NEXT:    notl %edx # sched: [1:0.25]
-; SKYLAKE-NEXT:    notl (%rax) # sched: [7:1.00]
-; SKYLAKE-NEXT:    notq %rcx # sched: [1:0.25]
-; SKYLAKE-NEXT:    notq (%r10) # sched: [7:1.00]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_not:
-; SKX:       # %bb.0:
-; SKX-NEXT:    movq {{[0-9]+}}(%rsp), %r10 # sched: [5:0.50]
-; SKX-NEXT:    movq {{[0-9]+}}(%rsp), %rax # sched: [5:0.50]
-; SKX-NEXT:    #APP
-; SKX-NEXT:    notb %dil # sched: [1:0.25]
-; SKX-NEXT:    notb (%r8) # sched: [7:1.00]
-; SKX-NEXT:    notw %si # sched: [1:0.25]
-; SKX-NEXT:    notw (%r9) # sched: [7:1.00]
-; SKX-NEXT:    notl %edx # sched: [1:0.25]
-; SKX-NEXT:    notl (%rax) # sched: [7:1.00]
-; SKX-NEXT:    notq %rcx # sched: [1:0.25]
-; SKX-NEXT:    notq (%r10) # sched: [7:1.00]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_not:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    movq {{[0-9]+}}(%rsp), %r10 # sched: [5:0.50]
-; BDVER2-NEXT:    movq {{[0-9]+}}(%rsp), %rax # sched: [5:0.50]
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    notb %dil # sched: [1:0.50]
-; BDVER2-NEXT:    notb (%r8) # sched: [6:1.00]
-; BDVER2-NEXT:    notw %si # sched: [1:0.50]
-; BDVER2-NEXT:    notw (%r9) # sched: [6:1.00]
-; BDVER2-NEXT:    notl %edx # sched: [1:0.50]
-; BDVER2-NEXT:    notl (%rax) # sched: [6:1.00]
-; BDVER2-NEXT:    notq %rcx # sched: [1:0.50]
-; BDVER2-NEXT:    notq (%r10) # sched: [6:1.00]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_not:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    movq {{[0-9]+}}(%rsp), %r10 # sched: [5:1.00]
-; BTVER2-NEXT:    movq {{[0-9]+}}(%rsp), %rax # sched: [5:1.00]
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    notb %dil # sched: [1:0.50]
-; BTVER2-NEXT:    notb (%r8) # sched: [5:1.00]
-; BTVER2-NEXT:    notw %si # sched: [1:0.50]
-; BTVER2-NEXT:    notw (%r9) # sched: [5:1.00]
-; BTVER2-NEXT:    notl %edx # sched: [1:0.50]
-; BTVER2-NEXT:    notl (%rax) # sched: [5:1.00]
-; BTVER2-NEXT:    notq %rcx # sched: [1:0.50]
-; BTVER2-NEXT:    notq (%r10) # sched: [5:1.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_not:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    movq {{[0-9]+}}(%rsp), %r10 # sched: [8:0.50]
-; ZNVER1-NEXT:    movq {{[0-9]+}}(%rsp), %rax # sched: [8:0.50]
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    notb %dil # sched: [1:0.25]
-; ZNVER1-NEXT:    notb (%r8) # sched: [5:0.50]
-; ZNVER1-NEXT:    notw %si # sched: [1:0.25]
-; ZNVER1-NEXT:    notw (%r9) # sched: [5:0.50]
-; ZNVER1-NEXT:    notl %edx # sched: [1:0.25]
-; ZNVER1-NEXT:    notl (%rax) # sched: [5:0.50]
-; ZNVER1-NEXT:    notq %rcx # sched: [1:0.25]
-; ZNVER1-NEXT:    notq (%r10) # sched: [5:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  tail call void asm "notb $0 \0A\09 notb $4 \0A\09 notw $1 \0A\09 notw $5 \0A\09 notl $2 \0A\09 notl $6 \0A\09 notq $3 \0A\09 notq $7", "r,r,r,r,*m,*m,*m,*m"(i8 %a0, i16 %a1, i32 %a2, i64 %a3, i8 *%p0, i16 *%p1, i32 *%p2, i64 *%p3) nounwind
-  ret void
-}
-
-define void @test_or_8(i8 %a0, i8* %a1, i8 %a2) optsize {
-; GENERIC-LABEL: test_or_8:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    orb $7, %al # sched: [1:0.33]
-; GENERIC-NEXT:    orb $7, %dil # sched: [1:0.33]
-; GENERIC-NEXT:    orb $7, (%rsi) # sched: [7:1.00]
-; GENERIC-NEXT:    orb %dl, %dil # sched: [1:0.33]
-; GENERIC-NEXT:    orb %dil, (%rsi) # sched: [7:1.00]
-; GENERIC-NEXT:    orb (%rsi), %dil # sched: [6:0.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_or_8:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    orb $7, %al # sched: [1:0.50]
-; ATOM-NEXT:    orb $7, %dil # sched: [1:0.50]
-; ATOM-NEXT:    orb $7, (%rsi) # sched: [1:1.00]
-; ATOM-NEXT:    orb %dl, %dil # sched: [1:0.50]
-; ATOM-NEXT:    orb %dil, (%rsi) # sched: [1:1.00]
-; ATOM-NEXT:    orb (%rsi), %dil # sched: [1:1.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_or_8:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    orb $7, %al # sched: [1:0.50]
-; SLM-NEXT:    orb $7, %dil # sched: [1:0.50]
-; SLM-NEXT:    orb $7, (%rsi) # sched: [5:2.00]
-; SLM-NEXT:    orb %dl, %dil # sched: [1:0.50]
-; SLM-NEXT:    orb %dil, (%rsi) # sched: [5:2.00]
-; SLM-NEXT:    orb (%rsi), %dil # sched: [4:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_or_8:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    orb $7, %al # sched: [1:0.33]
-; SANDY-NEXT:    orb $7, %dil # sched: [1:0.33]
-; SANDY-NEXT:    orb $7, (%rsi) # sched: [7:1.00]
-; SANDY-NEXT:    orb %dl, %dil # sched: [1:0.33]
-; SANDY-NEXT:    orb %dil, (%rsi) # sched: [7:1.00]
-; SANDY-NEXT:    orb (%rsi), %dil # sched: [6:0.50]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_or_8:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    orb $7, %al # sched: [1:0.25]
-; HASWELL-NEXT:    orb $7, %dil # sched: [1:0.25]
-; HASWELL-NEXT:    orb $7, (%rsi) # sched: [7:1.00]
-; HASWELL-NEXT:    orb %dl, %dil # sched: [1:0.25]
-; HASWELL-NEXT:    orb %dil, (%rsi) # sched: [7:1.00]
-; HASWELL-NEXT:    orb (%rsi), %dil # sched: [6:0.50]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_or_8:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    orb $7, %al # sched: [1:0.25]
-; BROADWELL-NEXT:    orb $7, %dil # sched: [1:0.25]
-; BROADWELL-NEXT:    orb $7, (%rsi) # sched: [7:1.00]
-; BROADWELL-NEXT:    orb %dl, %dil # sched: [1:0.25]
-; BROADWELL-NEXT:    orb %dil, (%rsi) # sched: [7:1.00]
-; BROADWELL-NEXT:    orb (%rsi), %dil # sched: [6:0.50]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_or_8:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    orb $7, %al # sched: [1:0.25]
-; SKYLAKE-NEXT:    orb $7, %dil # sched: [1:0.25]
-; SKYLAKE-NEXT:    orb $7, (%rsi) # sched: [7:1.00]
-; SKYLAKE-NEXT:    orb %dl, %dil # sched: [1:0.25]
-; SKYLAKE-NEXT:    orb %dil, (%rsi) # sched: [7:1.00]
-; SKYLAKE-NEXT:    orb (%rsi), %dil # sched: [6:0.50]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_or_8:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    orb $7, %al # sched: [1:0.25]
-; SKX-NEXT:    orb $7, %dil # sched: [1:0.25]
-; SKX-NEXT:    orb $7, (%rsi) # sched: [7:1.00]
-; SKX-NEXT:    orb %dl, %dil # sched: [1:0.25]
-; SKX-NEXT:    orb %dil, (%rsi) # sched: [7:1.00]
-; SKX-NEXT:    orb (%rsi), %dil # sched: [6:0.50]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_or_8:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    orb $7, %al # sched: [1:0.50]
-; BDVER2-NEXT:    orb $7, %dil # sched: [1:0.50]
-; BDVER2-NEXT:    orb $7, (%rsi) # sched: [6:1.00]
-; BDVER2-NEXT:    orb %dl, %dil # sched: [1:0.50]
-; BDVER2-NEXT:    orb %dil, (%rsi) # sched: [6:1.00]
-; BDVER2-NEXT:    orb (%rsi), %dil # sched: [5:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_or_8:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    orb $7, %al # sched: [1:0.50]
-; BTVER2-NEXT:    orb $7, %dil # sched: [1:0.50]
-; BTVER2-NEXT:    orb $7, (%rsi) # sched: [5:1.00]
-; BTVER2-NEXT:    orb %dl, %dil # sched: [1:0.50]
-; BTVER2-NEXT:    orb %dil, (%rsi) # sched: [5:1.00]
-; BTVER2-NEXT:    orb (%rsi), %dil # sched: [4:1.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_or_8:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    orb $7, %al # sched: [1:0.25]
-; ZNVER1-NEXT:    orb $7, %dil # sched: [1:0.25]
-; ZNVER1-NEXT:    orb $7, (%rsi) # sched: [5:0.50]
-; ZNVER1-NEXT:    orb %dl, %dil # sched: [1:0.25]
-; ZNVER1-NEXT:    orb %dil, (%rsi) # sched: [5:0.50]
-; ZNVER1-NEXT:    orb (%rsi), %dil # sched: [5:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  tail call void asm "orb $3, %AL \0A\09 orb $3, $0 \0A\09 orb $3, $2 \0A\09 orb $1, $0 \0A\09 orb $0, $2 \0A\09 orb $2, $0", "r,r,*m,i"(i8 %a0, i8 %a2, i8* %a1, i8 7) nounwind
-  ret void
-}
-define void @test_or_16(i16 %a0, i16* %a1, i16 %a2) optsize {
-; GENERIC-LABEL: test_or_16:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    orw $511, %ax # imm = 0x1FF
-; GENERIC-NEXT:    # sched: [1:0.33]
-; GENERIC-NEXT:    orw $511, %di # imm = 0x1FF
-; GENERIC-NEXT:    # sched: [1:0.33]
-; GENERIC-NEXT:    orw $511, (%rsi) # imm = 0x1FF
-; GENERIC-NEXT:    # sched: [7:1.00]
-; GENERIC-NEXT:    orw $7, %di # sched: [1:0.33]
-; GENERIC-NEXT:    orw $7, (%rsi) # sched: [7:1.00]
-; GENERIC-NEXT:    orw %dx, %di # sched: [1:0.33]
-; GENERIC-NEXT:    orw %di, (%rsi) # sched: [7:1.00]
-; GENERIC-NEXT:    orw (%rsi), %di # sched: [6:0.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_or_16:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    orw $511, %ax # imm = 0x1FF
-; ATOM-NEXT:    # sched: [1:0.50]
-; ATOM-NEXT:    orw $511, %di # imm = 0x1FF
-; ATOM-NEXT:    # sched: [1:0.50]
-; ATOM-NEXT:    orw $511, (%rsi) # imm = 0x1FF
-; ATOM-NEXT:    # sched: [1:1.00]
-; ATOM-NEXT:    orw $7, %di # sched: [1:0.50]
-; ATOM-NEXT:    orw $7, (%rsi) # sched: [1:1.00]
-; ATOM-NEXT:    orw %dx, %di # sched: [1:0.50]
-; ATOM-NEXT:    orw %di, (%rsi) # sched: [1:1.00]
-; ATOM-NEXT:    orw (%rsi), %di # sched: [1:1.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_or_16:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    orw $511, %ax # imm = 0x1FF
-; SLM-NEXT:    # sched: [1:0.50]
-; SLM-NEXT:    orw $511, %di # imm = 0x1FF
-; SLM-NEXT:    # sched: [1:0.50]
-; SLM-NEXT:    orw $511, (%rsi) # imm = 0x1FF
-; SLM-NEXT:    # sched: [5:2.00]
-; SLM-NEXT:    orw $7, %di # sched: [1:0.50]
-; SLM-NEXT:    orw $7, (%rsi) # sched: [5:2.00]
-; SLM-NEXT:    orw %dx, %di # sched: [1:0.50]
-; SLM-NEXT:    orw %di, (%rsi) # sched: [5:2.00]
-; SLM-NEXT:    orw (%rsi), %di # sched: [4:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_or_16:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    orw $511, %ax # imm = 0x1FF
-; SANDY-NEXT:    # sched: [1:0.33]
-; SANDY-NEXT:    orw $511, %di # imm = 0x1FF
-; SANDY-NEXT:    # sched: [1:0.33]
-; SANDY-NEXT:    orw $511, (%rsi) # imm = 0x1FF
-; SANDY-NEXT:    # sched: [7:1.00]
-; SANDY-NEXT:    orw $7, %di # sched: [1:0.33]
-; SANDY-NEXT:    orw $7, (%rsi) # sched: [7:1.00]
-; SANDY-NEXT:    orw %dx, %di # sched: [1:0.33]
-; SANDY-NEXT:    orw %di, (%rsi) # sched: [7:1.00]
-; SANDY-NEXT:    orw (%rsi), %di # sched: [6:0.50]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_or_16:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    orw $511, %ax # imm = 0x1FF
-; HASWELL-NEXT:    # sched: [1:0.25]
-; HASWELL-NEXT:    orw $511, %di # imm = 0x1FF
-; HASWELL-NEXT:    # sched: [1:0.25]
-; HASWELL-NEXT:    orw $511, (%rsi) # imm = 0x1FF
-; HASWELL-NEXT:    # sched: [7:1.00]
-; HASWELL-NEXT:    orw $7, %di # sched: [1:0.25]
-; HASWELL-NEXT:    orw $7, (%rsi) # sched: [7:1.00]
-; HASWELL-NEXT:    orw %dx, %di # sched: [1:0.25]
-; HASWELL-NEXT:    orw %di, (%rsi) # sched: [7:1.00]
-; HASWELL-NEXT:    orw (%rsi), %di # sched: [6:0.50]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_or_16:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    orw $511, %ax # imm = 0x1FF
-; BROADWELL-NEXT:    # sched: [1:0.25]
-; BROADWELL-NEXT:    orw $511, %di # imm = 0x1FF
-; BROADWELL-NEXT:    # sched: [1:0.25]
-; BROADWELL-NEXT:    orw $511, (%rsi) # imm = 0x1FF
-; BROADWELL-NEXT:    # sched: [7:1.00]
-; BROADWELL-NEXT:    orw $7, %di # sched: [1:0.25]
-; BROADWELL-NEXT:    orw $7, (%rsi) # sched: [7:1.00]
-; BROADWELL-NEXT:    orw %dx, %di # sched: [1:0.25]
-; BROADWELL-NEXT:    orw %di, (%rsi) # sched: [7:1.00]
-; BROADWELL-NEXT:    orw (%rsi), %di # sched: [6:0.50]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_or_16:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    orw $511, %ax # imm = 0x1FF
-; SKYLAKE-NEXT:    # sched: [1:0.25]
-; SKYLAKE-NEXT:    orw $511, %di # imm = 0x1FF
-; SKYLAKE-NEXT:    # sched: [1:0.25]
-; SKYLAKE-NEXT:    orw $511, (%rsi) # imm = 0x1FF
-; SKYLAKE-NEXT:    # sched: [7:1.00]
-; SKYLAKE-NEXT:    orw $7, %di # sched: [1:0.25]
-; SKYLAKE-NEXT:    orw $7, (%rsi) # sched: [7:1.00]
-; SKYLAKE-NEXT:    orw %dx, %di # sched: [1:0.25]
-; SKYLAKE-NEXT:    orw %di, (%rsi) # sched: [7:1.00]
-; SKYLAKE-NEXT:    orw (%rsi), %di # sched: [6:0.50]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_or_16:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    orw $511, %ax # imm = 0x1FF
-; SKX-NEXT:    # sched: [1:0.25]
-; SKX-NEXT:    orw $511, %di # imm = 0x1FF
-; SKX-NEXT:    # sched: [1:0.25]
-; SKX-NEXT:    orw $511, (%rsi) # imm = 0x1FF
-; SKX-NEXT:    # sched: [7:1.00]
-; SKX-NEXT:    orw $7, %di # sched: [1:0.25]
-; SKX-NEXT:    orw $7, (%rsi) # sched: [7:1.00]
-; SKX-NEXT:    orw %dx, %di # sched: [1:0.25]
-; SKX-NEXT:    orw %di, (%rsi) # sched: [7:1.00]
-; SKX-NEXT:    orw (%rsi), %di # sched: [6:0.50]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_or_16:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    orw $511, %ax # imm = 0x1FF
-; BDVER2-NEXT:    # sched: [1:0.50]
-; BDVER2-NEXT:    orw $511, %di # imm = 0x1FF
-; BDVER2-NEXT:    # sched: [1:0.50]
-; BDVER2-NEXT:    orw $511, (%rsi) # imm = 0x1FF
-; BDVER2-NEXT:    # sched: [6:1.00]
-; BDVER2-NEXT:    orw $7, %di # sched: [1:0.50]
-; BDVER2-NEXT:    orw $7, (%rsi) # sched: [6:1.00]
-; BDVER2-NEXT:    orw %dx, %di # sched: [1:0.50]
-; BDVER2-NEXT:    orw %di, (%rsi) # sched: [6:1.00]
-; BDVER2-NEXT:    orw (%rsi), %di # sched: [5:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_or_16:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    orw $511, %ax # imm = 0x1FF
-; BTVER2-NEXT:    # sched: [1:0.50]
-; BTVER2-NEXT:    orw $511, %di # imm = 0x1FF
-; BTVER2-NEXT:    # sched: [1:0.50]
-; BTVER2-NEXT:    orw $511, (%rsi) # imm = 0x1FF
-; BTVER2-NEXT:    # sched: [5:1.00]
-; BTVER2-NEXT:    orw $7, %di # sched: [1:0.50]
-; BTVER2-NEXT:    orw $7, (%rsi) # sched: [5:1.00]
-; BTVER2-NEXT:    orw %dx, %di # sched: [1:0.50]
-; BTVER2-NEXT:    orw %di, (%rsi) # sched: [5:1.00]
-; BTVER2-NEXT:    orw (%rsi), %di # sched: [4:1.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_or_16:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    orw $511, %ax # imm = 0x1FF
-; ZNVER1-NEXT:    # sched: [1:0.25]
-; ZNVER1-NEXT:    orw $511, %di # imm = 0x1FF
-; ZNVER1-NEXT:    # sched: [1:0.25]
-; ZNVER1-NEXT:    orw $511, (%rsi) # imm = 0x1FF
-; ZNVER1-NEXT:    # sched: [5:0.50]
-; ZNVER1-NEXT:    orw $7, %di # sched: [1:0.25]
-; ZNVER1-NEXT:    orw $7, (%rsi) # sched: [5:0.50]
-; ZNVER1-NEXT:    orw %dx, %di # sched: [1:0.25]
-; ZNVER1-NEXT:    orw %di, (%rsi) # sched: [5:0.50]
-; ZNVER1-NEXT:    orw (%rsi), %di # sched: [5:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  tail call void asm "orw $3, %AX \0A\09 orw $3, $0 \0A\09 orw $3, $2 \0A\09 orw $4, $0 \0A\09 orw $4, $2 \0A\09 orw $1, $0 \0A\09 orw $0, $2 \0A\09 orw $2, $0", "r,r,*m,i,i"(i16 %a0, i16 %a2, i16* %a1, i16 511, i8 7) nounwind
-  ret void
-}
-define void @test_or_32(i32 %a0, i32* %a1, i32 %a2) optsize {
-; GENERIC-LABEL: test_or_32:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    orl $665536, %eax # imm = 0xA27C0
-; GENERIC-NEXT:    # sched: [1:0.33]
-; GENERIC-NEXT:    orl $665536, %edi # imm = 0xA27C0
-; GENERIC-NEXT:    # sched: [1:0.33]
-; GENERIC-NEXT:    orl $665536, (%rsi) # imm = 0xA27C0
-; GENERIC-NEXT:    # sched: [7:1.00]
-; GENERIC-NEXT:    orl $7, %edi # sched: [1:0.33]
-; GENERIC-NEXT:    orl $7, (%rsi) # sched: [7:1.00]
-; GENERIC-NEXT:    orl %edx, %edi # sched: [1:0.33]
-; GENERIC-NEXT:    orl %edi, (%rsi) # sched: [7:1.00]
-; GENERIC-NEXT:    orl (%rsi), %edi # sched: [6:0.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_or_32:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    orl $665536, %eax # imm = 0xA27C0
-; ATOM-NEXT:    # sched: [1:0.50]
-; ATOM-NEXT:    orl $665536, %edi # imm = 0xA27C0
-; ATOM-NEXT:    # sched: [1:0.50]
-; ATOM-NEXT:    orl $665536, (%rsi) # imm = 0xA27C0
-; ATOM-NEXT:    # sched: [1:1.00]
-; ATOM-NEXT:    orl $7, %edi # sched: [1:0.50]
-; ATOM-NEXT:    orl $7, (%rsi) # sched: [1:1.00]
-; ATOM-NEXT:    orl %edx, %edi # sched: [1:0.50]
-; ATOM-NEXT:    orl %edi, (%rsi) # sched: [1:1.00]
-; ATOM-NEXT:    orl (%rsi), %edi # sched: [1:1.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_or_32:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    orl $665536, %eax # imm = 0xA27C0
-; SLM-NEXT:    # sched: [1:0.50]
-; SLM-NEXT:    orl $665536, %edi # imm = 0xA27C0
-; SLM-NEXT:    # sched: [1:0.50]
-; SLM-NEXT:    orl $665536, (%rsi) # imm = 0xA27C0
-; SLM-NEXT:    # sched: [5:2.00]
-; SLM-NEXT:    orl $7, %edi # sched: [1:0.50]
-; SLM-NEXT:    orl $7, (%rsi) # sched: [5:2.00]
-; SLM-NEXT:    orl %edx, %edi # sched: [1:0.50]
-; SLM-NEXT:    orl %edi, (%rsi) # sched: [5:2.00]
-; SLM-NEXT:    orl (%rsi), %edi # sched: [4:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_or_32:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    orl $665536, %eax # imm = 0xA27C0
-; SANDY-NEXT:    # sched: [1:0.33]
-; SANDY-NEXT:    orl $665536, %edi # imm = 0xA27C0
-; SANDY-NEXT:    # sched: [1:0.33]
-; SANDY-NEXT:    orl $665536, (%rsi) # imm = 0xA27C0
-; SANDY-NEXT:    # sched: [7:1.00]
-; SANDY-NEXT:    orl $7, %edi # sched: [1:0.33]
-; SANDY-NEXT:    orl $7, (%rsi) # sched: [7:1.00]
-; SANDY-NEXT:    orl %edx, %edi # sched: [1:0.33]
-; SANDY-NEXT:    orl %edi, (%rsi) # sched: [7:1.00]
-; SANDY-NEXT:    orl (%rsi), %edi # sched: [6:0.50]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_or_32:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    orl $665536, %eax # imm = 0xA27C0
-; HASWELL-NEXT:    # sched: [1:0.25]
-; HASWELL-NEXT:    orl $665536, %edi # imm = 0xA27C0
-; HASWELL-NEXT:    # sched: [1:0.25]
-; HASWELL-NEXT:    orl $665536, (%rsi) # imm = 0xA27C0
-; HASWELL-NEXT:    # sched: [7:1.00]
-; HASWELL-NEXT:    orl $7, %edi # sched: [1:0.25]
-; HASWELL-NEXT:    orl $7, (%rsi) # sched: [7:1.00]
-; HASWELL-NEXT:    orl %edx, %edi # sched: [1:0.25]
-; HASWELL-NEXT:    orl %edi, (%rsi) # sched: [7:1.00]
-; HASWELL-NEXT:    orl (%rsi), %edi # sched: [6:0.50]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_or_32:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    orl $665536, %eax # imm = 0xA27C0
-; BROADWELL-NEXT:    # sched: [1:0.25]
-; BROADWELL-NEXT:    orl $665536, %edi # imm = 0xA27C0
-; BROADWELL-NEXT:    # sched: [1:0.25]
-; BROADWELL-NEXT:    orl $665536, (%rsi) # imm = 0xA27C0
-; BROADWELL-NEXT:    # sched: [7:1.00]
-; BROADWELL-NEXT:    orl $7, %edi # sched: [1:0.25]
-; BROADWELL-NEXT:    orl $7, (%rsi) # sched: [7:1.00]
-; BROADWELL-NEXT:    orl %edx, %edi # sched: [1:0.25]
-; BROADWELL-NEXT:    orl %edi, (%rsi) # sched: [7:1.00]
-; BROADWELL-NEXT:    orl (%rsi), %edi # sched: [6:0.50]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_or_32:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    orl $665536, %eax # imm = 0xA27C0
-; SKYLAKE-NEXT:    # sched: [1:0.25]
-; SKYLAKE-NEXT:    orl $665536, %edi # imm = 0xA27C0
-; SKYLAKE-NEXT:    # sched: [1:0.25]
-; SKYLAKE-NEXT:    orl $665536, (%rsi) # imm = 0xA27C0
-; SKYLAKE-NEXT:    # sched: [7:1.00]
-; SKYLAKE-NEXT:    orl $7, %edi # sched: [1:0.25]
-; SKYLAKE-NEXT:    orl $7, (%rsi) # sched: [7:1.00]
-; SKYLAKE-NEXT:    orl %edx, %edi # sched: [1:0.25]
-; SKYLAKE-NEXT:    orl %edi, (%rsi) # sched: [7:1.00]
-; SKYLAKE-NEXT:    orl (%rsi), %edi # sched: [6:0.50]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_or_32:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    orl $665536, %eax # imm = 0xA27C0
-; SKX-NEXT:    # sched: [1:0.25]
-; SKX-NEXT:    orl $665536, %edi # imm = 0xA27C0
-; SKX-NEXT:    # sched: [1:0.25]
-; SKX-NEXT:    orl $665536, (%rsi) # imm = 0xA27C0
-; SKX-NEXT:    # sched: [7:1.00]
-; SKX-NEXT:    orl $7, %edi # sched: [1:0.25]
-; SKX-NEXT:    orl $7, (%rsi) # sched: [7:1.00]
-; SKX-NEXT:    orl %edx, %edi # sched: [1:0.25]
-; SKX-NEXT:    orl %edi, (%rsi) # sched: [7:1.00]
-; SKX-NEXT:    orl (%rsi), %edi # sched: [6:0.50]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_or_32:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    orl $665536, %eax # imm = 0xA27C0
-; BDVER2-NEXT:    # sched: [1:0.50]
-; BDVER2-NEXT:    orl $665536, %edi # imm = 0xA27C0
-; BDVER2-NEXT:    # sched: [1:0.50]
-; BDVER2-NEXT:    orl $665536, (%rsi) # imm = 0xA27C0
-; BDVER2-NEXT:    # sched: [6:1.00]
-; BDVER2-NEXT:    orl $7, %edi # sched: [1:0.50]
-; BDVER2-NEXT:    orl $7, (%rsi) # sched: [6:1.00]
-; BDVER2-NEXT:    orl %edx, %edi # sched: [1:0.50]
-; BDVER2-NEXT:    orl %edi, (%rsi) # sched: [6:1.00]
-; BDVER2-NEXT:    orl (%rsi), %edi # sched: [5:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_or_32:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    orl $665536, %eax # imm = 0xA27C0
-; BTVER2-NEXT:    # sched: [1:0.50]
-; BTVER2-NEXT:    orl $665536, %edi # imm = 0xA27C0
-; BTVER2-NEXT:    # sched: [1:0.50]
-; BTVER2-NEXT:    orl $665536, (%rsi) # imm = 0xA27C0
-; BTVER2-NEXT:    # sched: [5:1.00]
-; BTVER2-NEXT:    orl $7, %edi # sched: [1:0.50]
-; BTVER2-NEXT:    orl $7, (%rsi) # sched: [5:1.00]
-; BTVER2-NEXT:    orl %edx, %edi # sched: [1:0.50]
-; BTVER2-NEXT:    orl %edi, (%rsi) # sched: [5:1.00]
-; BTVER2-NEXT:    orl (%rsi), %edi # sched: [4:1.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_or_32:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    orl $665536, %eax # imm = 0xA27C0
-; ZNVER1-NEXT:    # sched: [1:0.25]
-; ZNVER1-NEXT:    orl $665536, %edi # imm = 0xA27C0
-; ZNVER1-NEXT:    # sched: [1:0.25]
-; ZNVER1-NEXT:    orl $665536, (%rsi) # imm = 0xA27C0
-; ZNVER1-NEXT:    # sched: [5:0.50]
-; ZNVER1-NEXT:    orl $7, %edi # sched: [1:0.25]
-; ZNVER1-NEXT:    orl $7, (%rsi) # sched: [5:0.50]
-; ZNVER1-NEXT:    orl %edx, %edi # sched: [1:0.25]
-; ZNVER1-NEXT:    orl %edi, (%rsi) # sched: [5:0.50]
-; ZNVER1-NEXT:    orl (%rsi), %edi # sched: [5:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  tail call void asm "orl $3, %EAX \0A\09 orl $3, $0 \0A\09 orl $3, $2 \0A\09 orl $4, $0 \0A\09 orl $4, $2 \0A\09 orl $1, $0 \0A\09 orl $0, $2 \0A\09 orl $2, $0", "r,r,*m,i,i"(i32 %a0, i32 %a2, i32* %a1, i32 665536, i8 7) nounwind
-  ret void
-}
-define void @test_or_64(i64 %a0, i64* %a1, i64 %a2) optsize {
-; GENERIC-LABEL: test_or_64:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    orq $665536, %rax # imm = 0xA27C0
-; GENERIC-NEXT:    # sched: [1:0.33]
-; GENERIC-NEXT:    orq $665536, %rdi # imm = 0xA27C0
-; GENERIC-NEXT:    # sched: [1:0.33]
-; GENERIC-NEXT:    orq $665536, (%rsi) # imm = 0xA27C0
-; GENERIC-NEXT:    # sched: [7:1.00]
-; GENERIC-NEXT:    orq $7, %rdi # sched: [1:0.33]
-; GENERIC-NEXT:    orq $7, (%rsi) # sched: [7:1.00]
-; GENERIC-NEXT:    orq %rdx, %rdi # sched: [1:0.33]
-; GENERIC-NEXT:    orq %rdi, (%rsi) # sched: [7:1.00]
-; GENERIC-NEXT:    orq (%rsi), %rdi # sched: [6:0.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_or_64:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    orq $665536, %rax # imm = 0xA27C0
-; ATOM-NEXT:    # sched: [1:0.50]
-; ATOM-NEXT:    orq $665536, %rdi # imm = 0xA27C0
-; ATOM-NEXT:    # sched: [1:0.50]
-; ATOM-NEXT:    orq $665536, (%rsi) # imm = 0xA27C0
-; ATOM-NEXT:    # sched: [1:1.00]
-; ATOM-NEXT:    orq $7, %rdi # sched: [1:0.50]
-; ATOM-NEXT:    orq $7, (%rsi) # sched: [1:1.00]
-; ATOM-NEXT:    orq %rdx, %rdi # sched: [1:0.50]
-; ATOM-NEXT:    orq %rdi, (%rsi) # sched: [1:1.00]
-; ATOM-NEXT:    orq (%rsi), %rdi # sched: [1:1.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_or_64:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    orq $665536, %rax # imm = 0xA27C0
-; SLM-NEXT:    # sched: [1:0.50]
-; SLM-NEXT:    orq $665536, %rdi # imm = 0xA27C0
-; SLM-NEXT:    # sched: [1:0.50]
-; SLM-NEXT:    orq $665536, (%rsi) # imm = 0xA27C0
-; SLM-NEXT:    # sched: [5:2.00]
-; SLM-NEXT:    orq $7, %rdi # sched: [1:0.50]
-; SLM-NEXT:    orq $7, (%rsi) # sched: [5:2.00]
-; SLM-NEXT:    orq %rdx, %rdi # sched: [1:0.50]
-; SLM-NEXT:    orq %rdi, (%rsi) # sched: [5:2.00]
-; SLM-NEXT:    orq (%rsi), %rdi # sched: [4:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_or_64:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    orq $665536, %rax # imm = 0xA27C0
-; SANDY-NEXT:    # sched: [1:0.33]
-; SANDY-NEXT:    orq $665536, %rdi # imm = 0xA27C0
-; SANDY-NEXT:    # sched: [1:0.33]
-; SANDY-NEXT:    orq $665536, (%rsi) # imm = 0xA27C0
-; SANDY-NEXT:    # sched: [7:1.00]
-; SANDY-NEXT:    orq $7, %rdi # sched: [1:0.33]
-; SANDY-NEXT:    orq $7, (%rsi) # sched: [7:1.00]
-; SANDY-NEXT:    orq %rdx, %rdi # sched: [1:0.33]
-; SANDY-NEXT:    orq %rdi, (%rsi) # sched: [7:1.00]
-; SANDY-NEXT:    orq (%rsi), %rdi # sched: [6:0.50]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_or_64:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    orq $665536, %rax # imm = 0xA27C0
-; HASWELL-NEXT:    # sched: [1:0.25]
-; HASWELL-NEXT:    orq $665536, %rdi # imm = 0xA27C0
-; HASWELL-NEXT:    # sched: [1:0.25]
-; HASWELL-NEXT:    orq $665536, (%rsi) # imm = 0xA27C0
-; HASWELL-NEXT:    # sched: [7:1.00]
-; HASWELL-NEXT:    orq $7, %rdi # sched: [1:0.25]
-; HASWELL-NEXT:    orq $7, (%rsi) # sched: [7:1.00]
-; HASWELL-NEXT:    orq %rdx, %rdi # sched: [1:0.25]
-; HASWELL-NEXT:    orq %rdi, (%rsi) # sched: [7:1.00]
-; HASWELL-NEXT:    orq (%rsi), %rdi # sched: [6:0.50]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_or_64:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    orq $665536, %rax # imm = 0xA27C0
-; BROADWELL-NEXT:    # sched: [1:0.25]
-; BROADWELL-NEXT:    orq $665536, %rdi # imm = 0xA27C0
-; BROADWELL-NEXT:    # sched: [1:0.25]
-; BROADWELL-NEXT:    orq $665536, (%rsi) # imm = 0xA27C0
-; BROADWELL-NEXT:    # sched: [7:1.00]
-; BROADWELL-NEXT:    orq $7, %rdi # sched: [1:0.25]
-; BROADWELL-NEXT:    orq $7, (%rsi) # sched: [7:1.00]
-; BROADWELL-NEXT:    orq %rdx, %rdi # sched: [1:0.25]
-; BROADWELL-NEXT:    orq %rdi, (%rsi) # sched: [7:1.00]
-; BROADWELL-NEXT:    orq (%rsi), %rdi # sched: [6:0.50]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_or_64:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    orq $665536, %rax # imm = 0xA27C0
-; SKYLAKE-NEXT:    # sched: [1:0.25]
-; SKYLAKE-NEXT:    orq $665536, %rdi # imm = 0xA27C0
-; SKYLAKE-NEXT:    # sched: [1:0.25]
-; SKYLAKE-NEXT:    orq $665536, (%rsi) # imm = 0xA27C0
-; SKYLAKE-NEXT:    # sched: [7:1.00]
-; SKYLAKE-NEXT:    orq $7, %rdi # sched: [1:0.25]
-; SKYLAKE-NEXT:    orq $7, (%rsi) # sched: [7:1.00]
-; SKYLAKE-NEXT:    orq %rdx, %rdi # sched: [1:0.25]
-; SKYLAKE-NEXT:    orq %rdi, (%rsi) # sched: [7:1.00]
-; SKYLAKE-NEXT:    orq (%rsi), %rdi # sched: [6:0.50]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_or_64:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    orq $665536, %rax # imm = 0xA27C0
-; SKX-NEXT:    # sched: [1:0.25]
-; SKX-NEXT:    orq $665536, %rdi # imm = 0xA27C0
-; SKX-NEXT:    # sched: [1:0.25]
-; SKX-NEXT:    orq $665536, (%rsi) # imm = 0xA27C0
-; SKX-NEXT:    # sched: [7:1.00]
-; SKX-NEXT:    orq $7, %rdi # sched: [1:0.25]
-; SKX-NEXT:    orq $7, (%rsi) # sched: [7:1.00]
-; SKX-NEXT:    orq %rdx, %rdi # sched: [1:0.25]
-; SKX-NEXT:    orq %rdi, (%rsi) # sched: [7:1.00]
-; SKX-NEXT:    orq (%rsi), %rdi # sched: [6:0.50]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_or_64:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    orq $665536, %rax # imm = 0xA27C0
-; BDVER2-NEXT:    # sched: [1:0.50]
-; BDVER2-NEXT:    orq $665536, %rdi # imm = 0xA27C0
-; BDVER2-NEXT:    # sched: [1:0.50]
-; BDVER2-NEXT:    orq $665536, (%rsi) # imm = 0xA27C0
-; BDVER2-NEXT:    # sched: [6:1.00]
-; BDVER2-NEXT:    orq $7, %rdi # sched: [1:0.50]
-; BDVER2-NEXT:    orq $7, (%rsi) # sched: [6:1.00]
-; BDVER2-NEXT:    orq %rdx, %rdi # sched: [1:0.50]
-; BDVER2-NEXT:    orq %rdi, (%rsi) # sched: [6:1.00]
-; BDVER2-NEXT:    orq (%rsi), %rdi # sched: [5:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_or_64:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    orq $665536, %rax # imm = 0xA27C0
-; BTVER2-NEXT:    # sched: [1:0.50]
-; BTVER2-NEXT:    orq $665536, %rdi # imm = 0xA27C0
-; BTVER2-NEXT:    # sched: [1:0.50]
-; BTVER2-NEXT:    orq $665536, (%rsi) # imm = 0xA27C0
-; BTVER2-NEXT:    # sched: [5:1.00]
-; BTVER2-NEXT:    orq $7, %rdi # sched: [1:0.50]
-; BTVER2-NEXT:    orq $7, (%rsi) # sched: [5:1.00]
-; BTVER2-NEXT:    orq %rdx, %rdi # sched: [1:0.50]
-; BTVER2-NEXT:    orq %rdi, (%rsi) # sched: [5:1.00]
-; BTVER2-NEXT:    orq (%rsi), %rdi # sched: [4:1.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_or_64:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    orq $665536, %rax # imm = 0xA27C0
-; ZNVER1-NEXT:    # sched: [1:0.25]
-; ZNVER1-NEXT:    orq $665536, %rdi # imm = 0xA27C0
-; ZNVER1-NEXT:    # sched: [1:0.25]
-; ZNVER1-NEXT:    orq $665536, (%rsi) # imm = 0xA27C0
-; ZNVER1-NEXT:    # sched: [5:0.50]
-; ZNVER1-NEXT:    orq $7, %rdi # sched: [1:0.25]
-; ZNVER1-NEXT:    orq $7, (%rsi) # sched: [5:0.50]
-; ZNVER1-NEXT:    orq %rdx, %rdi # sched: [1:0.25]
-; ZNVER1-NEXT:    orq %rdi, (%rsi) # sched: [5:0.50]
-; ZNVER1-NEXT:    orq (%rsi), %rdi # sched: [5:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  tail call void asm "orq $3, %RAX \0A\09 orq $3, $0 \0A\09 orq $3, $2 \0A\09 orq $4, $0 \0A\09 orq $4, $2 \0A\09 orq $1, $0 \0A\09 orq $0, $2 \0A\09 orq $2, $0", "r,r,*m,i,i"(i64 %a0, i64 %a2, i64* %a1, i32 665536, i8 7) nounwind
-  ret void
-}
-
-define void @test_out() optsize {
-; GENERIC-LABEL: test_out:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    outb %al, $7 # sched: [100:0.33]
-; GENERIC-NEXT:    outw %ax, $7 # sched: [100:0.33]
-; GENERIC-NEXT:    outl %eax, $7 # sched: [100:0.33]
-; GENERIC-NEXT:    outb %al, %dx # sched: [100:0.33]
-; GENERIC-NEXT:    outw %ax, %dx # sched: [100:0.33]
-; GENERIC-NEXT:    outl %eax, %dx # sched: [100:0.33]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_out:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    outb %al, $7 # sched: [72:36.00]
-; ATOM-NEXT:    outw %ax, $7 # sched: [72:36.00]
-; ATOM-NEXT:    outl %eax, $7 # sched: [72:36.00]
-; ATOM-NEXT:    outb %al, %dx # sched: [68:34.00]
-; ATOM-NEXT:    outw %ax, %dx # sched: [68:34.00]
-; ATOM-NEXT:    outl %eax, %dx # sched: [68:34.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_out:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    outb %al, $7 # sched: [100:1.00]
-; SLM-NEXT:    outw %ax, $7 # sched: [100:1.00]
-; SLM-NEXT:    outl %eax, $7 # sched: [100:1.00]
-; SLM-NEXT:    outb %al, %dx # sched: [100:1.00]
-; SLM-NEXT:    outw %ax, %dx # sched: [100:1.00]
-; SLM-NEXT:    outl %eax, %dx # sched: [100:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_out:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    outb %al, $7 # sched: [100:0.33]
-; SANDY-NEXT:    outw %ax, $7 # sched: [100:0.33]
-; SANDY-NEXT:    outl %eax, $7 # sched: [100:0.33]
-; SANDY-NEXT:    outb %al, %dx # sched: [100:0.33]
-; SANDY-NEXT:    outw %ax, %dx # sched: [100:0.33]
-; SANDY-NEXT:    outl %eax, %dx # sched: [100:0.33]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_out:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    outb %al, $7 # sched: [36:5.00]
-; HASWELL-NEXT:    outw %ax, $7 # sched: [36:5.00]
-; HASWELL-NEXT:    outl %eax, $7 # sched: [36:5.00]
-; HASWELL-NEXT:    outb %al, %dx # sched: [36:5.00]
-; HASWELL-NEXT:    outw %ax, %dx # sched: [36:5.00]
-; HASWELL-NEXT:    outl %eax, %dx # sched: [36:5.00]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_out:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    outb %al, $7 # sched: [35:5.00]
-; BROADWELL-NEXT:    outw %ax, $7 # sched: [35:5.00]
-; BROADWELL-NEXT:    outl %eax, $7 # sched: [35:5.00]
-; BROADWELL-NEXT:    outb %al, %dx # sched: [35:5.00]
-; BROADWELL-NEXT:    outw %ax, %dx # sched: [35:5.00]
-; BROADWELL-NEXT:    outl %eax, %dx # sched: [35:5.00]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_out:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    outb %al, $7 # sched: [35:5.00]
-; SKYLAKE-NEXT:    outw %ax, $7 # sched: [35:5.00]
-; SKYLAKE-NEXT:    outl %eax, $7 # sched: [35:5.00]
-; SKYLAKE-NEXT:    outb %al, %dx # sched: [35:5.00]
-; SKYLAKE-NEXT:    outw %ax, %dx # sched: [35:5.00]
-; SKYLAKE-NEXT:    outl %eax, %dx # sched: [35:5.00]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_out:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    outb %al, $7 # sched: [35:5.00]
-; SKX-NEXT:    outw %ax, $7 # sched: [35:5.00]
-; SKX-NEXT:    outl %eax, $7 # sched: [35:5.00]
-; SKX-NEXT:    outb %al, %dx # sched: [35:5.00]
-; SKX-NEXT:    outw %ax, %dx # sched: [35:5.00]
-; SKX-NEXT:    outl %eax, %dx # sched: [35:5.00]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_out:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    outb %al, $7 # sched: [100:0.50]
-; BDVER2-NEXT:    outw %ax, $7 # sched: [100:0.50]
-; BDVER2-NEXT:    outl %eax, $7 # sched: [100:0.50]
-; BDVER2-NEXT:    outb %al, %dx # sched: [100:0.50]
-; BDVER2-NEXT:    outw %ax, %dx # sched: [100:0.50]
-; BDVER2-NEXT:    outl %eax, %dx # sched: [100:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_out:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    outb %al, $7 # sched: [100:0.50]
-; BTVER2-NEXT:    outw %ax, $7 # sched: [100:0.50]
-; BTVER2-NEXT:    outl %eax, $7 # sched: [100:0.50]
-; BTVER2-NEXT:    outb %al, %dx # sched: [100:0.50]
-; BTVER2-NEXT:    outw %ax, %dx # sched: [100:0.50]
-; BTVER2-NEXT:    outl %eax, %dx # sched: [100:0.50]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_out:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    outb %al, $7 # sched: [100:0.25]
-; ZNVER1-NEXT:    outw %ax, $7 # sched: [100:0.25]
-; ZNVER1-NEXT:    outl %eax, $7 # sched: [100:0.25]
-; ZNVER1-NEXT:    outb %al, %dx # sched: [100:0.25]
-; ZNVER1-NEXT:    outw %ax, %dx # sched: [100:0.25]
-; ZNVER1-NEXT:    outl %eax, %dx # sched: [100:0.25]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  tail call void asm "outb %AL, $0 \0A\09 outw %AX, $0 \0A\09 outl %EAX, $0 \0A\09 outb %AL, %DX \0A\09 outw %AX, %DX \0A\09 outl %EAX, %DX", "i"(i8 7) nounwind
-  ret void
-}
-
-define void @test_outs() optsize {
-; GENERIC-LABEL: test_outs:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    outsb (%rsi), %dx # sched: [100:0.33]
-; GENERIC-NEXT:    outsw (%rsi), %dx # sched: [100:0.33]
-; GENERIC-NEXT:    outsl (%rsi), %dx # sched: [100:0.33]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_outs:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    outsb (%rsi), %dx # sched: [74:37.00]
-; ATOM-NEXT:    outsw (%rsi), %dx # sched: [74:37.00]
-; ATOM-NEXT:    outsl (%rsi), %dx # sched: [74:37.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_outs:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    outsb (%rsi), %dx # sched: [100:1.00]
-; SLM-NEXT:    outsw (%rsi), %dx # sched: [100:1.00]
-; SLM-NEXT:    outsl (%rsi), %dx # sched: [100:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_outs:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    outsb (%rsi), %dx # sched: [100:0.33]
-; SANDY-NEXT:    outsw (%rsi), %dx # sched: [100:0.33]
-; SANDY-NEXT:    outsl (%rsi), %dx # sched: [100:0.33]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_outs:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    outsb (%rsi), %dx # sched: [100:0.25]
-; HASWELL-NEXT:    outsw (%rsi), %dx # sched: [100:0.25]
-; HASWELL-NEXT:    outsl (%rsi), %dx # sched: [100:0.25]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_outs:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    outsb (%rsi), %dx # sched: [100:0.25]
-; BROADWELL-NEXT:    outsw (%rsi), %dx # sched: [100:0.25]
-; BROADWELL-NEXT:    outsl (%rsi), %dx # sched: [100:0.25]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_outs:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    outsb (%rsi), %dx # sched: [100:0.25]
-; SKYLAKE-NEXT:    outsw (%rsi), %dx # sched: [100:0.25]
-; SKYLAKE-NEXT:    outsl (%rsi), %dx # sched: [100:0.25]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_outs:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    outsb (%rsi), %dx # sched: [100:0.25]
-; SKX-NEXT:    outsw (%rsi), %dx # sched: [100:0.25]
-; SKX-NEXT:    outsl (%rsi), %dx # sched: [100:0.25]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_outs:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    outsb (%rsi), %dx # sched: [100:0.50]
-; BDVER2-NEXT:    outsw (%rsi), %dx # sched: [100:0.50]
-; BDVER2-NEXT:    outsl (%rsi), %dx # sched: [100:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_outs:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    outsb (%rsi), %dx # sched: [100:0.50]
-; BTVER2-NEXT:    outsw (%rsi), %dx # sched: [100:0.50]
-; BTVER2-NEXT:    outsl (%rsi), %dx # sched: [100:0.50]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_outs:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    outsb (%rsi), %dx # sched: [100:0.25]
-; ZNVER1-NEXT:    outsw (%rsi), %dx # sched: [100:0.25]
-; ZNVER1-NEXT:    outsl (%rsi), %dx # sched: [100:0.25]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  call void asm sideeffect "outsb \0A\09 outsw \0A\09 outsl", ""()
-  ret void
-}
-
-define void @test_pause() optsize {
-; GENERIC-LABEL: test_pause:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    pause # sched: [4:1.33]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_pause:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    pause # sched: [17:8.50]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_pause:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    pause # sched: [1:0.50]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_pause:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    pause # sched: [4:1.33]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_pause:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    pause # sched: [5:1.25]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pause:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    pause # sched: [5:1.25]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pause:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    pause # sched: [4:1.00]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pause:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    pause # sched: [140:1.00]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_pause:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    pause # sched: [1:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_pause:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    pause # sched: [1:0.50]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_pause:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    pause # sched: [100:0.25]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  call void asm sideeffect "pause", ""()
-  ret void
-}
-
-define void @test_pop_push() optsize {
-; GENERIC-LABEL: test_pop_push:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    popq %fs # sched: [100:0.33]
-; GENERIC-NEXT:    popq %gs # sched: [100:0.33]
-; GENERIC-NEXT:    pushq %fs # sched: [3:1.00]
-; GENERIC-NEXT:    pushq %gs # sched: [5:1.00]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_pop_push:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    popq %fs # sched: [29:14.50]
-; ATOM-NEXT:    popq %gs # sched: [29:14.50]
-; ATOM-NEXT:    pushq %fs # sched: [2:1.00]
-; ATOM-NEXT:    pushq %gs # sched: [2:1.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_pop_push:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    popq %fs # sched: [100:1.00]
-; SLM-NEXT:    popq %gs # sched: [100:1.00]
-; SLM-NEXT:    pushq %fs # sched: [100:1.00]
-; SLM-NEXT:    pushq %gs # sched: [100:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_pop_push:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    popq %fs # sched: [100:0.33]
-; SANDY-NEXT:    popq %gs # sched: [100:0.33]
-; SANDY-NEXT:    pushq %fs # sched: [3:1.00]
-; SANDY-NEXT:    pushq %gs # sched: [5:1.00]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_pop_push:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    popq %fs # sched: [100:0.25]
-; HASWELL-NEXT:    popq %gs # sched: [100:0.25]
-; HASWELL-NEXT:    pushq %fs # sched: [100:0.25]
-; HASWELL-NEXT:    pushq %gs # sched: [100:0.25]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pop_push:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    popq %fs # sched: [100:0.25]
-; BROADWELL-NEXT:    popq %gs # sched: [100:0.25]
-; BROADWELL-NEXT:    pushq %fs # sched: [100:0.25]
-; BROADWELL-NEXT:    pushq %gs # sched: [100:0.25]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pop_push:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    popq %fs # sched: [100:0.25]
-; SKYLAKE-NEXT:    popq %gs # sched: [100:0.25]
-; SKYLAKE-NEXT:    pushq %fs # sched: [100:0.25]
-; SKYLAKE-NEXT:    pushq %gs # sched: [100:0.25]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pop_push:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    popq %fs # sched: [100:0.25]
-; SKX-NEXT:    popq %gs # sched: [100:0.25]
-; SKX-NEXT:    pushq %fs # sched: [100:0.25]
-; SKX-NEXT:    pushq %gs # sched: [100:0.25]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_pop_push:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    popq %fs # sched: [100:0.50]
-; BDVER2-NEXT:    popq %gs # sched: [100:0.50]
-; BDVER2-NEXT:    pushq %fs # sched: [100:0.50]
-; BDVER2-NEXT:    pushq %gs # sched: [100:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_pop_push:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    popq %fs # sched: [100:0.50]
-; BTVER2-NEXT:    popq %gs # sched: [100:0.50]
-; BTVER2-NEXT:    pushq %fs # sched: [100:0.50]
-; BTVER2-NEXT:    pushq %gs # sched: [100:0.50]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_pop_push:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    popq %fs # sched: [100:0.25]
-; ZNVER1-NEXT:    popq %gs # sched: [100:0.25]
-; ZNVER1-NEXT:    pushq %fs # sched: [100:0.25]
-; ZNVER1-NEXT:    pushq %gs # sched: [100:0.25]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  call void asm sideeffect "pop %FS \0A\09 pop %GS \0A\09 push %FS \0A\09 push %GS", ""()
-  ret void
-}
-define i16 @test_pop_push_16(i16 %a0, i16 *%a1) optsize {
-; GENERIC-LABEL: test_pop_push_16:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    popw %ax # sched: [6:0.50]
-; GENERIC-NEXT:    popw (%rsi) # sched: [6:0.50]
-; GENERIC-NEXT:    pushw %di # sched: [5:1.00]
-; GENERIC-NEXT:    pushw (%rsi) # sched: [5:1.00]
-; GENERIC-NEXT:    pushw $4095 # imm = 0xFFF
-; GENERIC-NEXT:    # sched: [1:1.00]
-; GENERIC-NEXT:    pushw $7 # sched: [1:1.00]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_pop_push_16:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    popw %ax # sched: [2:1.00]
-; ATOM-NEXT:    popw (%rsi) # sched: [3:1.50]
-; ATOM-NEXT:    pushw %di # sched: [1:1.00]
-; ATOM-NEXT:    pushw (%rsi) # sched: [2:1.00]
-; ATOM-NEXT:    pushw $4095 # imm = 0xFFF
-; ATOM-NEXT:    # sched: [1:1.00]
-; ATOM-NEXT:    pushw $7 # sched: [1:1.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_pop_push_16:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    popw %ax # sched: [3:1.00]
-; SLM-NEXT:    popw (%rsi) # sched: [4:2.00]
-; SLM-NEXT:    pushw %di # sched: [1:1.00]
-; SLM-NEXT:    pushw (%rsi) # sched: [4:2.00]
-; SLM-NEXT:    pushw $4095 # imm = 0xFFF
-; SLM-NEXT:    # sched: [1:1.00]
-; SLM-NEXT:    pushw $7 # sched: [1:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_pop_push_16:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    popw %ax # sched: [6:0.50]
-; SANDY-NEXT:    popw (%rsi) # sched: [6:0.50]
-; SANDY-NEXT:    pushw %di # sched: [5:1.00]
-; SANDY-NEXT:    pushw (%rsi) # sched: [5:1.00]
-; SANDY-NEXT:    pushw $4095 # imm = 0xFFF
-; SANDY-NEXT:    # sched: [1:1.00]
-; SANDY-NEXT:    pushw $7 # sched: [1:1.00]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_pop_push_16:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    popw %ax # sched: [6:0.50]
-; HASWELL-NEXT:    popw (%rsi) # sched: [7:1.00]
-; HASWELL-NEXT:    pushw %di # sched: [2:1.00]
-; HASWELL-NEXT:    pushw (%rsi) # sched: [7:1.00]
-; HASWELL-NEXT:    pushw $4095 # imm = 0xFFF
-; HASWELL-NEXT:    # sched: [1:1.00]
-; HASWELL-NEXT:    pushw $7 # sched: [1:1.00]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pop_push_16:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    popw %ax # sched: [6:0.50]
-; BROADWELL-NEXT:    popw (%rsi) # sched: [6:1.00]
-; BROADWELL-NEXT:    pushw %di # sched: [2:1.00]
-; BROADWELL-NEXT:    pushw (%rsi) # sched: [6:1.00]
-; BROADWELL-NEXT:    pushw $4095 # imm = 0xFFF
-; BROADWELL-NEXT:    # sched: [1:1.00]
-; BROADWELL-NEXT:    pushw $7 # sched: [1:1.00]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pop_push_16:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    popw %ax # sched: [6:0.50]
-; SKYLAKE-NEXT:    popw (%rsi) # sched: [6:1.00]
-; SKYLAKE-NEXT:    pushw %di # sched: [2:1.00]
-; SKYLAKE-NEXT:    pushw (%rsi) # sched: [6:1.00]
-; SKYLAKE-NEXT:    pushw $4095 # imm = 0xFFF
-; SKYLAKE-NEXT:    # sched: [1:1.00]
-; SKYLAKE-NEXT:    pushw $7 # sched: [1:1.00]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pop_push_16:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    popw %ax # sched: [6:0.50]
-; SKX-NEXT:    popw (%rsi) # sched: [6:1.00]
-; SKX-NEXT:    pushw %di # sched: [2:1.00]
-; SKX-NEXT:    pushw (%rsi) # sched: [6:1.00]
-; SKX-NEXT:    pushw $4095 # imm = 0xFFF
-; SKX-NEXT:    # sched: [1:1.00]
-; SKX-NEXT:    pushw $7 # sched: [1:1.00]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_pop_push_16:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    popw %ax # sched: [5:0.50]
-; BDVER2-NEXT:    popw (%rsi) # sched: [6:1.00]
-; BDVER2-NEXT:    pushw %di # sched: [1:1.00]
-; BDVER2-NEXT:    pushw (%rsi) # sched: [6:1.00]
-; BDVER2-NEXT:    pushw $4095 # imm = 0xFFF
-; BDVER2-NEXT:    # sched: [1:1.00]
-; BDVER2-NEXT:    pushw $7 # sched: [1:1.00]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_pop_push_16:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    popw %ax # sched: [5:1.00]
-; BTVER2-NEXT:    popw (%rsi) # sched: [6:1.00]
-; BTVER2-NEXT:    pushw %di # sched: [1:1.00]
-; BTVER2-NEXT:    pushw (%rsi) # sched: [6:1.00]
-; BTVER2-NEXT:    pushw $4095 # imm = 0xFFF
-; BTVER2-NEXT:    # sched: [1:1.00]
-; BTVER2-NEXT:    pushw $7 # sched: [1:1.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_pop_push_16:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    popw %ax # sched: [8:0.50]
-; ZNVER1-NEXT:    popw (%rsi) # sched: [5:0.50]
-; ZNVER1-NEXT:    pushw %di # sched: [1:0.50]
-; ZNVER1-NEXT:    pushw (%rsi) # sched: [4:0.50]
-; ZNVER1-NEXT:    pushw $4095 # imm = 0xFFF
-; ZNVER1-NEXT:    # sched: [1:0.50]
-; ZNVER1-NEXT:    pushw $7 # sched: [1:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call i16 asm sideeffect "popw $0 \0A\09 popw $2 \0A\09 pushw $1 \0A\09 pushw $2 \0A\09 pushw $3 \0A\09 pushw $4", "=r,r,*m,i,i"(i16 %a0, i16 *%a1, i16 4095, i8 7)
-  ret i16 %1
-}
-define i64 @test_pop_push_64(i64 %a0, i64 *%a1) optsize {
-; GENERIC-LABEL: test_pop_push_64:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    popq %rax # sched: [6:0.50]
-; GENERIC-NEXT:    popq (%rsi) # sched: [6:0.50]
-; GENERIC-NEXT:    pushq %rdi # sched: [5:1.00]
-; GENERIC-NEXT:    pushq (%rsi) # sched: [5:1.00]
-; GENERIC-NEXT:    pushq $4095 # imm = 0xFFF
-; GENERIC-NEXT:    # sched: [1:1.00]
-; GENERIC-NEXT:    pushq $7 # sched: [5:1.00]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_pop_push_64:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    popq %rax # sched: [1:1.00]
-; ATOM-NEXT:    popq (%rsi) # sched: [3:1.50]
-; ATOM-NEXT:    pushq %rdi # sched: [1:1.00]
-; ATOM-NEXT:    pushq (%rsi) # sched: [2:1.00]
-; ATOM-NEXT:    pushq $4095 # imm = 0xFFF
-; ATOM-NEXT:    # sched: [1:1.00]
-; ATOM-NEXT:    pushq $7 # sched: [1:1.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_pop_push_64:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    popq %rax # sched: [3:1.00]
-; SLM-NEXT:    popq (%rsi) # sched: [4:2.00]
-; SLM-NEXT:    pushq %rdi # sched: [1:1.00]
-; SLM-NEXT:    pushq (%rsi) # sched: [4:2.00]
-; SLM-NEXT:    pushq $4095 # imm = 0xFFF
-; SLM-NEXT:    # sched: [1:1.00]
-; SLM-NEXT:    pushq $7 # sched: [1:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_pop_push_64:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    popq %rax # sched: [6:0.50]
-; SANDY-NEXT:    popq (%rsi) # sched: [6:0.50]
-; SANDY-NEXT:    pushq %rdi # sched: [5:1.00]
-; SANDY-NEXT:    pushq (%rsi) # sched: [5:1.00]
-; SANDY-NEXT:    pushq $4095 # imm = 0xFFF
-; SANDY-NEXT:    # sched: [1:1.00]
-; SANDY-NEXT:    pushq $7 # sched: [5:1.00]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_pop_push_64:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    popq %rax # sched: [6:0.50]
-; HASWELL-NEXT:    popq (%rsi) # sched: [7:1.00]
-; HASWELL-NEXT:    pushq %rdi # sched: [2:1.00]
-; HASWELL-NEXT:    pushq (%rsi) # sched: [7:1.00]
-; HASWELL-NEXT:    pushq $4095 # imm = 0xFFF
-; HASWELL-NEXT:    # sched: [1:1.00]
-; HASWELL-NEXT:    pushq $7 # sched: [2:1.00]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pop_push_64:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    popq %rax # sched: [6:0.50]
-; BROADWELL-NEXT:    popq (%rsi) # sched: [6:1.00]
-; BROADWELL-NEXT:    pushq %rdi # sched: [2:1.00]
-; BROADWELL-NEXT:    pushq (%rsi) # sched: [6:1.00]
-; BROADWELL-NEXT:    pushq $4095 # imm = 0xFFF
-; BROADWELL-NEXT:    # sched: [1:1.00]
-; BROADWELL-NEXT:    pushq $7 # sched: [2:1.00]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pop_push_64:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    popq %rax # sched: [6:0.50]
-; SKYLAKE-NEXT:    popq (%rsi) # sched: [6:1.00]
-; SKYLAKE-NEXT:    pushq %rdi # sched: [2:1.00]
-; SKYLAKE-NEXT:    pushq (%rsi) # sched: [6:1.00]
-; SKYLAKE-NEXT:    pushq $4095 # imm = 0xFFF
-; SKYLAKE-NEXT:    # sched: [1:1.00]
-; SKYLAKE-NEXT:    pushq $7 # sched: [2:1.00]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pop_push_64:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    popq %rax # sched: [6:0.50]
-; SKX-NEXT:    popq (%rsi) # sched: [6:1.00]
-; SKX-NEXT:    pushq %rdi # sched: [2:1.00]
-; SKX-NEXT:    pushq (%rsi) # sched: [6:1.00]
-; SKX-NEXT:    pushq $4095 # imm = 0xFFF
-; SKX-NEXT:    # sched: [1:1.00]
-; SKX-NEXT:    pushq $7 # sched: [2:1.00]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_pop_push_64:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    popq %rax # sched: [5:0.50]
-; BDVER2-NEXT:    popq (%rsi) # sched: [6:1.00]
-; BDVER2-NEXT:    pushq %rdi # sched: [1:1.00]
-; BDVER2-NEXT:    pushq (%rsi) # sched: [6:1.00]
-; BDVER2-NEXT:    pushq $4095 # imm = 0xFFF
-; BDVER2-NEXT:    # sched: [1:1.00]
-; BDVER2-NEXT:    pushq $7 # sched: [1:1.00]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_pop_push_64:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    popq %rax # sched: [5:1.00]
-; BTVER2-NEXT:    popq (%rsi) # sched: [6:1.00]
-; BTVER2-NEXT:    pushq %rdi # sched: [1:1.00]
-; BTVER2-NEXT:    pushq (%rsi) # sched: [6:1.00]
-; BTVER2-NEXT:    pushq $4095 # imm = 0xFFF
-; BTVER2-NEXT:    # sched: [1:1.00]
-; BTVER2-NEXT:    pushq $7 # sched: [1:1.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_pop_push_64:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    popq %rax # sched: [8:0.50]
-; ZNVER1-NEXT:    popq (%rsi) # sched: [9:1.00]
-; ZNVER1-NEXT:    pushq %rdi # sched: [1:0.50]
-; ZNVER1-NEXT:    pushq (%rsi) # sched: [9:1.00]
-; ZNVER1-NEXT:    pushq $4095 # imm = 0xFFF
-; ZNVER1-NEXT:    # sched: [1:0.50]
-; ZNVER1-NEXT:    pushq $7 # sched: [1:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call i64 asm sideeffect "popq $0 \0A\09 popq $2 \0A\09 pushq $1 \0A\09 pushq $2 \0A\09 pushq $3 \0A\09 pushq $4", "=r,r,*m,i,i"(i64 %a0, i64 *%a1, i64 4095, i8 7)
-  ret i64 %1
-}
-
-define void @test_popf_pushf() optsize {
-; GENERIC-LABEL: test_popf_pushf:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    popfq # sched: [5:0.50]
-; GENERIC-NEXT:    pushfq # sched: [5:1.00]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_popf_pushf:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    popfq # sched: [26:13.00]
-; ATOM-NEXT:    pushfq # sched: [9:4.50]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_popf_pushf:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    popfq # sched: [3:1.00]
-; SLM-NEXT:    pushfq # sched: [1:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_popf_pushf:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    popfq # sched: [5:0.50]
-; SANDY-NEXT:    pushfq # sched: [5:1.00]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_popf_pushf:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    popfq # sched: [5:0.50]
-; HASWELL-NEXT:    pushfq # sched: [5:1.00]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_popf_pushf:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    popfq # sched: [22:4.25]
-; BROADWELL-NEXT:    pushfq # sched: [5:1.00]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_popf_pushf:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    popfq # sched: [5:0.50]
-; SKYLAKE-NEXT:    pushfq # sched: [5:1.00]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_popf_pushf:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    popfq # sched: [5:0.50]
-; SKX-NEXT:    pushfq # sched: [5:1.00]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_popf_pushf:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    popfq # sched: [5:0.50]
-; BDVER2-NEXT:    pushfq # sched: [1:1.00]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_popf_pushf:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    popfq # sched: [5:1.00]
-; BTVER2-NEXT:    pushfq # sched: [1:1.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_popf_pushf:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    popfq # sched: [8:0.50]
-; ZNVER1-NEXT:    pushfq # sched: [1:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  call void asm sideeffect "popf \0A\09 pushf", ""()
-  ret void
-}
-
-define void @test_rcl_rcr_8(i8 %a0, i8 %a1, i8 *%a2) optsize {
-; GENERIC-LABEL: test_rcl_rcr_8:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    rclb %dil # sched: [2:1.50]
-; GENERIC-NEXT:    rcrb %dil # sched: [2:1.50]
-; GENERIC-NEXT:    rclb (%rdx) # sched: [11:3.50]
-; GENERIC-NEXT:    rcrb (%rdx) # sched: [11:3.50]
-; GENERIC-NEXT:    rclb $7, %dil # sched: [5:4.00]
-; GENERIC-NEXT:    rcrb $7, %dil # sched: [5:4.00]
-; GENERIC-NEXT:    rclb $7, (%rdx) # sched: [11:3.50]
-; GENERIC-NEXT:    rcrb $7, (%rdx) # sched: [11:3.50]
-; GENERIC-NEXT:    rclb %cl, %dil # sched: [5:4.00]
-; GENERIC-NEXT:    rcrb %cl, %dil # sched: [5:4.00]
-; GENERIC-NEXT:    rclb %cl, (%rdx) # sched: [11:3.50]
-; GENERIC-NEXT:    rcrb %cl, (%rdx) # sched: [11:3.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_rcl_rcr_8:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    rclb %dil # sched: [1:1.00]
-; ATOM-NEXT:    rcrb %dil # sched: [1:1.00]
-; ATOM-NEXT:    rclb (%rdx) # sched: [1:1.00]
-; ATOM-NEXT:    rcrb (%rdx) # sched: [1:1.00]
-; ATOM-NEXT:    rclb $7, %dil # sched: [1:1.00]
-; ATOM-NEXT:    rcrb $7, %dil # sched: [1:1.00]
-; ATOM-NEXT:    rclb $7, (%rdx) # sched: [1:1.00]
-; ATOM-NEXT:    rcrb $7, (%rdx) # sched: [1:1.00]
-; ATOM-NEXT:    rclb %cl, %dil # sched: [1:1.00]
-; ATOM-NEXT:    rcrb %cl, %dil # sched: [1:1.00]
-; ATOM-NEXT:    rclb %cl, (%rdx) # sched: [1:1.00]
-; ATOM-NEXT:    rcrb %cl, (%rdx) # sched: [1:1.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_rcl_rcr_8:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    rclb %dil # sched: [1:1.00]
-; SLM-NEXT:    rcrb %dil # sched: [1:1.00]
-; SLM-NEXT:    rclb (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    rcrb (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    rclb $7, %dil # sched: [1:1.00]
-; SLM-NEXT:    rcrb $7, %dil # sched: [1:1.00]
-; SLM-NEXT:    rclb $7, (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    rcrb $7, (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    rclb %cl, %dil # sched: [1:1.00]
-; SLM-NEXT:    rcrb %cl, %dil # sched: [1:1.00]
-; SLM-NEXT:    rclb %cl, (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    rcrb %cl, (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_rcl_rcr_8:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    rclb %dil # sched: [2:1.50]
-; SANDY-NEXT:    rcrb %dil # sched: [2:1.50]
-; SANDY-NEXT:    rclb (%rdx) # sched: [11:3.50]
-; SANDY-NEXT:    rcrb (%rdx) # sched: [11:3.50]
-; SANDY-NEXT:    rclb $7, %dil # sched: [5:4.00]
-; SANDY-NEXT:    rcrb $7, %dil # sched: [5:4.00]
-; SANDY-NEXT:    rclb $7, (%rdx) # sched: [11:3.50]
-; SANDY-NEXT:    rcrb $7, (%rdx) # sched: [11:3.50]
-; SANDY-NEXT:    rclb %cl, %dil # sched: [5:4.00]
-; SANDY-NEXT:    rcrb %cl, %dil # sched: [5:4.00]
-; SANDY-NEXT:    rclb %cl, (%rdx) # sched: [11:3.50]
-; SANDY-NEXT:    rcrb %cl, (%rdx) # sched: [11:3.50]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_rcl_rcr_8:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    rclb %dil # sched: [3:0.75]
-; HASWELL-NEXT:    rcrb %dil # sched: [3:0.75]
-; HASWELL-NEXT:    rclb (%rdx) # sched: [9:0.75]
-; HASWELL-NEXT:    rcrb (%rdx) # sched: [9:0.75]
-; HASWELL-NEXT:    rclb $7, %dil # sched: [3:0.75]
-; HASWELL-NEXT:    rcrb $7, %dil # sched: [3:0.75]
-; HASWELL-NEXT:    rclb $7, (%rdx) # sched: [9:0.75]
-; HASWELL-NEXT:    rcrb $7, (%rdx) # sched: [9:0.75]
-; HASWELL-NEXT:    rclb %cl, %dil # sched: [11:2.25]
-; HASWELL-NEXT:    rcrb %cl, %dil # sched: [14:2.50]
-; HASWELL-NEXT:    rclb %cl, (%rdx) # sched: [16:2.00]
-; HASWELL-NEXT:    rcrb %cl, (%rdx) # sched: [19:2.25]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_rcl_rcr_8:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    rclb %dil # sched: [3:0.75]
-; BROADWELL-NEXT:    rcrb %dil # sched: [3:0.75]
-; BROADWELL-NEXT:    rclb (%rdx) # sched: [8:0.75]
-; BROADWELL-NEXT:    rcrb (%rdx) # sched: [8:0.75]
-; BROADWELL-NEXT:    rclb $7, %dil # sched: [3:0.75]
-; BROADWELL-NEXT:    rcrb $7, %dil # sched: [3:0.75]
-; BROADWELL-NEXT:    rclb $7, (%rdx) # sched: [8:0.75]
-; BROADWELL-NEXT:    rcrb $7, (%rdx) # sched: [8:0.75]
-; BROADWELL-NEXT:    rclb %cl, %dil # sched: [11:2.25]
-; BROADWELL-NEXT:    rcrb %cl, %dil # sched: [14:2.50]
-; BROADWELL-NEXT:    rclb %cl, (%rdx) # sched: [15:2.00]
-; BROADWELL-NEXT:    rcrb %cl, (%rdx) # sched: [18:2.25]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_rcl_rcr_8:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    rclb %dil # sched: [3:0.75]
-; SKYLAKE-NEXT:    rcrb %dil # sched: [3:0.75]
-; SKYLAKE-NEXT:    rclb (%rdx) # sched: [8:0.75]
-; SKYLAKE-NEXT:    rcrb (%rdx) # sched: [8:0.75]
-; SKYLAKE-NEXT:    rclb $7, %dil # sched: [3:0.75]
-; SKYLAKE-NEXT:    rcrb $7, %dil # sched: [3:0.75]
-; SKYLAKE-NEXT:    rclb $7, (%rdx) # sched: [8:0.75]
-; SKYLAKE-NEXT:    rcrb $7, (%rdx) # sched: [8:0.75]
-; SKYLAKE-NEXT:    rclb %cl, %dil # sched: [11:2.50]
-; SKYLAKE-NEXT:    rcrb %cl, %dil # sched: [14:2.50]
-; SKYLAKE-NEXT:    rclb %cl, (%rdx) # sched: [15:2.50]
-; SKYLAKE-NEXT:    rcrb %cl, (%rdx) # sched: [18:2.25]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_rcl_rcr_8:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    rclb %dil # sched: [3:0.75]
-; SKX-NEXT:    rcrb %dil # sched: [3:0.75]
-; SKX-NEXT:    rclb (%rdx) # sched: [8:0.75]
-; SKX-NEXT:    rcrb (%rdx) # sched: [8:0.75]
-; SKX-NEXT:    rclb $7, %dil # sched: [3:0.75]
-; SKX-NEXT:    rcrb $7, %dil # sched: [3:0.75]
-; SKX-NEXT:    rclb $7, (%rdx) # sched: [8:0.75]
-; SKX-NEXT:    rcrb $7, (%rdx) # sched: [8:0.75]
-; SKX-NEXT:    rclb %cl, %dil # sched: [11:2.50]
-; SKX-NEXT:    rcrb %cl, %dil # sched: [14:2.50]
-; SKX-NEXT:    rclb %cl, (%rdx) # sched: [15:2.50]
-; SKX-NEXT:    rcrb %cl, (%rdx) # sched: [18:2.25]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_rcl_rcr_8:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    rclb %dil # sched: [1:0.50]
-; BDVER2-NEXT:    rcrb %dil # sched: [1:0.50]
-; BDVER2-NEXT:    rclb (%rdx) # sched: [5:1.00]
-; BDVER2-NEXT:    rcrb (%rdx) # sched: [5:1.00]
-; BDVER2-NEXT:    rclb $7, %dil # sched: [13:0.50]
-; BDVER2-NEXT:    rcrb $7, %dil # sched: [12:0.50]
-; BDVER2-NEXT:    rclb $7, (%rdx) # sched: [5:1.00]
-; BDVER2-NEXT:    rcrb $7, (%rdx) # sched: [5:1.00]
-; BDVER2-NEXT:    rclb %cl, %dil # sched: [12:0.50]
-; BDVER2-NEXT:    rcrb %cl, %dil # sched: [11:0.50]
-; BDVER2-NEXT:    rclb %cl, (%rdx) # sched: [5:1.00]
-; BDVER2-NEXT:    rcrb %cl, (%rdx) # sched: [5:1.00]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_rcl_rcr_8:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    rclb %dil # sched: [1:0.50]
-; BTVER2-NEXT:    rcrb %dil # sched: [1:0.50]
-; BTVER2-NEXT:    rclb (%rdx) # sched: [4:1.00]
-; BTVER2-NEXT:    rcrb (%rdx) # sched: [4:1.00]
-; BTVER2-NEXT:    rclb $7, %dil # sched: [1:0.50]
-; BTVER2-NEXT:    rcrb $7, %dil # sched: [1:0.50]
-; BTVER2-NEXT:    rclb $7, (%rdx) # sched: [4:1.00]
-; BTVER2-NEXT:    rcrb $7, (%rdx) # sched: [4:1.00]
-; BTVER2-NEXT:    rclb %cl, %dil # sched: [1:0.50]
-; BTVER2-NEXT:    rcrb %cl, %dil # sched: [1:0.50]
-; BTVER2-NEXT:    rclb %cl, (%rdx) # sched: [4:1.00]
-; BTVER2-NEXT:    rcrb %cl, (%rdx) # sched: [4:1.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_rcl_rcr_8:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    rclb %dil # sched: [1:0.25]
-; ZNVER1-NEXT:    rcrb %dil # sched: [1:0.25]
-; ZNVER1-NEXT:    rclb (%rdx) # sched: [100:0.25]
-; ZNVER1-NEXT:    rcrb (%rdx) # sched: [100:0.25]
-; ZNVER1-NEXT:    rclb $7, %dil # sched: [1:0.25]
-; ZNVER1-NEXT:    rcrb $7, %dil # sched: [1:0.25]
-; ZNVER1-NEXT:    rclb $7, (%rdx) # sched: [100:0.25]
-; ZNVER1-NEXT:    rcrb $7, (%rdx) # sched: [100:0.25]
-; ZNVER1-NEXT:    rclb %cl, %dil # sched: [1:0.25]
-; ZNVER1-NEXT:    rcrb %cl, %dil # sched: [1:0.25]
-; ZNVER1-NEXT:    rclb %cl, (%rdx) # sched: [100:0.25]
-; ZNVER1-NEXT:    rcrb %cl, (%rdx) # sched: [100:0.25]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  call void asm sideeffect "rclb $0 \0A\09 rcrb $0 \0A\09 rclb $2 \0A\09 rcrb $2 \0A\09 rclb $3, $0 \0A\09 rcrb $3, $0 \0A\09 rclb $3, $2 \0A\09 rcrb $3, $2 \0A\09 rclb %CL, $0 \0A\09 rcrb %CL, $0 \0A\09 rclb %CL, $2 \0A\09 rcrb %CL, $2", "r,r,*m,i"(i8 %a0, i8 %a1, i8 *%a2, i8 7)
-  ret void
-}
-define void @test_rcl_rcr_16(i16 %a0, i16 %a1, i16 *%a2) optsize {
-; GENERIC-LABEL: test_rcl_rcr_16:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    rclw %di # sched: [2:1.50]
-; GENERIC-NEXT:    rcrw %di # sched: [2:1.50]
-; GENERIC-NEXT:    rclw (%rdx) # sched: [11:3.50]
-; GENERIC-NEXT:    rcrw (%rdx) # sched: [11:3.50]
-; GENERIC-NEXT:    rclw $7, %di # sched: [5:4.00]
-; GENERIC-NEXT:    rcrw $7, %di # sched: [5:4.00]
-; GENERIC-NEXT:    rclw $7, (%rdx) # sched: [11:3.50]
-; GENERIC-NEXT:    rcrw $7, (%rdx) # sched: [11:3.50]
-; GENERIC-NEXT:    rclw %cl, %di # sched: [5:4.00]
-; GENERIC-NEXT:    rcrw %cl, %di # sched: [5:4.00]
-; GENERIC-NEXT:    rclw %cl, (%rdx) # sched: [11:3.50]
-; GENERIC-NEXT:    rcrw %cl, (%rdx) # sched: [11:3.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_rcl_rcr_16:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    rclw %di # sched: [1:1.00]
-; ATOM-NEXT:    rcrw %di # sched: [1:1.00]
-; ATOM-NEXT:    rclw (%rdx) # sched: [1:1.00]
-; ATOM-NEXT:    rcrw (%rdx) # sched: [1:1.00]
-; ATOM-NEXT:    rclw $7, %di # sched: [1:1.00]
-; ATOM-NEXT:    rcrw $7, %di # sched: [1:1.00]
-; ATOM-NEXT:    rclw $7, (%rdx) # sched: [1:1.00]
-; ATOM-NEXT:    rcrw $7, (%rdx) # sched: [1:1.00]
-; ATOM-NEXT:    rclw %cl, %di # sched: [1:1.00]
-; ATOM-NEXT:    rcrw %cl, %di # sched: [1:1.00]
-; ATOM-NEXT:    rclw %cl, (%rdx) # sched: [1:1.00]
-; ATOM-NEXT:    rcrw %cl, (%rdx) # sched: [1:1.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_rcl_rcr_16:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    rclw %di # sched: [1:1.00]
-; SLM-NEXT:    rcrw %di # sched: [1:1.00]
-; SLM-NEXT:    rclw (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    rcrw (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    rclw $7, %di # sched: [1:1.00]
-; SLM-NEXT:    rcrw $7, %di # sched: [1:1.00]
-; SLM-NEXT:    rclw $7, (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    rcrw $7, (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    rclw %cl, %di # sched: [1:1.00]
-; SLM-NEXT:    rcrw %cl, %di # sched: [1:1.00]
-; SLM-NEXT:    rclw %cl, (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    rcrw %cl, (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_rcl_rcr_16:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    rclw %di # sched: [2:1.50]
-; SANDY-NEXT:    rcrw %di # sched: [2:1.50]
-; SANDY-NEXT:    rclw (%rdx) # sched: [11:3.50]
-; SANDY-NEXT:    rcrw (%rdx) # sched: [11:3.50]
-; SANDY-NEXT:    rclw $7, %di # sched: [5:4.00]
-; SANDY-NEXT:    rcrw $7, %di # sched: [5:4.00]
-; SANDY-NEXT:    rclw $7, (%rdx) # sched: [11:3.50]
-; SANDY-NEXT:    rcrw $7, (%rdx) # sched: [11:3.50]
-; SANDY-NEXT:    rclw %cl, %di # sched: [5:4.00]
-; SANDY-NEXT:    rcrw %cl, %di # sched: [5:4.00]
-; SANDY-NEXT:    rclw %cl, (%rdx) # sched: [11:3.50]
-; SANDY-NEXT:    rcrw %cl, (%rdx) # sched: [11:3.50]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_rcl_rcr_16:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    rclw %di # sched: [3:0.75]
-; HASWELL-NEXT:    rcrw %di # sched: [3:0.75]
-; HASWELL-NEXT:    rclw (%rdx) # sched: [9:0.75]
-; HASWELL-NEXT:    rcrw (%rdx) # sched: [9:0.75]
-; HASWELL-NEXT:    rclw $7, %di # sched: [3:0.75]
-; HASWELL-NEXT:    rcrw $7, %di # sched: [3:0.75]
-; HASWELL-NEXT:    rclw $7, (%rdx) # sched: [9:0.75]
-; HASWELL-NEXT:    rcrw $7, (%rdx) # sched: [9:0.75]
-; HASWELL-NEXT:    rclw %cl, %di # sched: [11:2.00]
-; HASWELL-NEXT:    rcrw %cl, %di # sched: [11:2.00]
-; HASWELL-NEXT:    rclw %cl, (%rdx) # sched: [16:2.00]
-; HASWELL-NEXT:    rcrw %cl, (%rdx) # sched: [19:2.25]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_rcl_rcr_16:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    rclw %di # sched: [3:0.75]
-; BROADWELL-NEXT:    rcrw %di # sched: [3:0.75]
-; BROADWELL-NEXT:    rclw (%rdx) # sched: [8:0.75]
-; BROADWELL-NEXT:    rcrw (%rdx) # sched: [8:0.75]
-; BROADWELL-NEXT:    rclw $7, %di # sched: [3:0.75]
-; BROADWELL-NEXT:    rcrw $7, %di # sched: [3:0.75]
-; BROADWELL-NEXT:    rclw $7, (%rdx) # sched: [8:0.75]
-; BROADWELL-NEXT:    rcrw $7, (%rdx) # sched: [8:0.75]
-; BROADWELL-NEXT:    rclw %cl, %di # sched: [11:2.00]
-; BROADWELL-NEXT:    rcrw %cl, %di # sched: [11:2.00]
-; BROADWELL-NEXT:    rclw %cl, (%rdx) # sched: [15:2.00]
-; BROADWELL-NEXT:    rcrw %cl, (%rdx) # sched: [18:2.25]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_rcl_rcr_16:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    rclw %di # sched: [3:0.75]
-; SKYLAKE-NEXT:    rcrw %di # sched: [3:0.75]
-; SKYLAKE-NEXT:    rclw (%rdx) # sched: [8:0.75]
-; SKYLAKE-NEXT:    rcrw (%rdx) # sched: [8:0.75]
-; SKYLAKE-NEXT:    rclw $7, %di # sched: [3:0.75]
-; SKYLAKE-NEXT:    rcrw $7, %di # sched: [3:0.75]
-; SKYLAKE-NEXT:    rclw $7, (%rdx) # sched: [8:0.75]
-; SKYLAKE-NEXT:    rcrw $7, (%rdx) # sched: [8:0.75]
-; SKYLAKE-NEXT:    rclw %cl, %di # sched: [11:2.00]
-; SKYLAKE-NEXT:    rcrw %cl, %di # sched: [11:2.00]
-; SKYLAKE-NEXT:    rclw %cl, (%rdx) # sched: [15:2.50]
-; SKYLAKE-NEXT:    rcrw %cl, (%rdx) # sched: [18:2.25]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_rcl_rcr_16:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    rclw %di # sched: [3:0.75]
-; SKX-NEXT:    rcrw %di # sched: [3:0.75]
-; SKX-NEXT:    rclw (%rdx) # sched: [8:0.75]
-; SKX-NEXT:    rcrw (%rdx) # sched: [8:0.75]
-; SKX-NEXT:    rclw $7, %di # sched: [3:0.75]
-; SKX-NEXT:    rcrw $7, %di # sched: [3:0.75]
-; SKX-NEXT:    rclw $7, (%rdx) # sched: [8:0.75]
-; SKX-NEXT:    rcrw $7, (%rdx) # sched: [8:0.75]
-; SKX-NEXT:    rclw %cl, %di # sched: [11:2.00]
-; SKX-NEXT:    rcrw %cl, %di # sched: [11:2.00]
-; SKX-NEXT:    rclw %cl, (%rdx) # sched: [15:2.50]
-; SKX-NEXT:    rcrw %cl, (%rdx) # sched: [18:2.25]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_rcl_rcr_16:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    rclw %di # sched: [1:0.50]
-; BDVER2-NEXT:    rcrw %di # sched: [1:0.50]
-; BDVER2-NEXT:    rclw (%rdx) # sched: [5:1.00]
-; BDVER2-NEXT:    rcrw (%rdx) # sched: [5:1.00]
-; BDVER2-NEXT:    rclw $7, %di # sched: [11:0.50]
-; BDVER2-NEXT:    rcrw $7, %di # sched: [10:0.50]
-; BDVER2-NEXT:    rclw $7, (%rdx) # sched: [5:1.00]
-; BDVER2-NEXT:    rcrw $7, (%rdx) # sched: [5:1.00]
-; BDVER2-NEXT:    rclw %cl, %di # sched: [10:0.50]
-; BDVER2-NEXT:    rcrw %cl, %di # sched: [9:0.50]
-; BDVER2-NEXT:    rclw %cl, (%rdx) # sched: [5:1.00]
-; BDVER2-NEXT:    rcrw %cl, (%rdx) # sched: [5:1.00]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_rcl_rcr_16:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    rclw %di # sched: [1:0.50]
-; BTVER2-NEXT:    rcrw %di # sched: [1:0.50]
-; BTVER2-NEXT:    rclw (%rdx) # sched: [4:1.00]
-; BTVER2-NEXT:    rcrw (%rdx) # sched: [4:1.00]
-; BTVER2-NEXT:    rclw $7, %di # sched: [1:0.50]
-; BTVER2-NEXT:    rcrw $7, %di # sched: [1:0.50]
-; BTVER2-NEXT:    rclw $7, (%rdx) # sched: [4:1.00]
-; BTVER2-NEXT:    rcrw $7, (%rdx) # sched: [4:1.00]
-; BTVER2-NEXT:    rclw %cl, %di # sched: [1:0.50]
-; BTVER2-NEXT:    rcrw %cl, %di # sched: [1:0.50]
-; BTVER2-NEXT:    rclw %cl, (%rdx) # sched: [4:1.00]
-; BTVER2-NEXT:    rcrw %cl, (%rdx) # sched: [4:1.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_rcl_rcr_16:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    rclw %di # sched: [1:0.25]
-; ZNVER1-NEXT:    rcrw %di # sched: [1:0.25]
-; ZNVER1-NEXT:    rclw (%rdx) # sched: [100:0.25]
-; ZNVER1-NEXT:    rcrw (%rdx) # sched: [100:0.25]
-; ZNVER1-NEXT:    rclw $7, %di # sched: [1:0.25]
-; ZNVER1-NEXT:    rcrw $7, %di # sched: [1:0.25]
-; ZNVER1-NEXT:    rclw $7, (%rdx) # sched: [100:0.25]
-; ZNVER1-NEXT:    rcrw $7, (%rdx) # sched: [100:0.25]
-; ZNVER1-NEXT:    rclw %cl, %di # sched: [1:0.25]
-; ZNVER1-NEXT:    rcrw %cl, %di # sched: [1:0.25]
-; ZNVER1-NEXT:    rclw %cl, (%rdx) # sched: [100:0.25]
-; ZNVER1-NEXT:    rcrw %cl, (%rdx) # sched: [100:0.25]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  call void asm sideeffect "rclw $0 \0A\09 rcrw $0 \0A\09 rclw $2 \0A\09 rcrw $2 \0A\09 rclw $3, $0 \0A\09 rcrw $3, $0 \0A\09 rclw $3, $2 \0A\09 rcrw $3, $2 \0A\09 rclw %CL, $0 \0A\09 rcrw %CL, $0 \0A\09 rclw %CL, $2 \0A\09 rcrw %CL, $2", "r,r,*m,i"(i16 %a0, i16 %a1, i16 *%a2, i8 7)
-  ret void
-}
-define void @test_rcl_rcr_32(i32 %a0, i32 %a1, i32 *%a2) optsize {
-; GENERIC-LABEL: test_rcl_rcr_32:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    rcll %edi # sched: [2:1.50]
-; GENERIC-NEXT:    rcrl %edi # sched: [2:1.50]
-; GENERIC-NEXT:    rcll (%rdx) # sched: [11:3.50]
-; GENERIC-NEXT:    rcrl (%rdx) # sched: [11:3.50]
-; GENERIC-NEXT:    rcll $7, %edi # sched: [5:4.00]
-; GENERIC-NEXT:    rcrl $7, %edi # sched: [5:4.00]
-; GENERIC-NEXT:    rcll $7, (%rdx) # sched: [11:3.50]
-; GENERIC-NEXT:    rcrl $7, (%rdx) # sched: [11:3.50]
-; GENERIC-NEXT:    rcll %cl, %edi # sched: [5:4.00]
-; GENERIC-NEXT:    rcrl %cl, %edi # sched: [5:4.00]
-; GENERIC-NEXT:    rcll %cl, (%rdx) # sched: [11:3.50]
-; GENERIC-NEXT:    rcrl %cl, (%rdx) # sched: [11:3.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_rcl_rcr_32:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    rcll %edi # sched: [1:1.00]
-; ATOM-NEXT:    rcrl %edi # sched: [1:1.00]
-; ATOM-NEXT:    rcll (%rdx) # sched: [1:1.00]
-; ATOM-NEXT:    rcrl (%rdx) # sched: [1:1.00]
-; ATOM-NEXT:    rcll $7, %edi # sched: [1:1.00]
-; ATOM-NEXT:    rcrl $7, %edi # sched: [1:1.00]
-; ATOM-NEXT:    rcll $7, (%rdx) # sched: [1:1.00]
-; ATOM-NEXT:    rcrl $7, (%rdx) # sched: [1:1.00]
-; ATOM-NEXT:    rcll %cl, %edi # sched: [1:1.00]
-; ATOM-NEXT:    rcrl %cl, %edi # sched: [1:1.00]
-; ATOM-NEXT:    rcll %cl, (%rdx) # sched: [1:1.00]
-; ATOM-NEXT:    rcrl %cl, (%rdx) # sched: [1:1.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_rcl_rcr_32:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    rcll %edi # sched: [1:1.00]
-; SLM-NEXT:    rcrl %edi # sched: [1:1.00]
-; SLM-NEXT:    rcll (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    rcrl (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    rcll $7, %edi # sched: [1:1.00]
-; SLM-NEXT:    rcrl $7, %edi # sched: [1:1.00]
-; SLM-NEXT:    rcll $7, (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    rcrl $7, (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    rcll %cl, %edi # sched: [1:1.00]
-; SLM-NEXT:    rcrl %cl, %edi # sched: [1:1.00]
-; SLM-NEXT:    rcll %cl, (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    rcrl %cl, (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_rcl_rcr_32:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    rcll %edi # sched: [2:1.50]
-; SANDY-NEXT:    rcrl %edi # sched: [2:1.50]
-; SANDY-NEXT:    rcll (%rdx) # sched: [11:3.50]
-; SANDY-NEXT:    rcrl (%rdx) # sched: [11:3.50]
-; SANDY-NEXT:    rcll $7, %edi # sched: [5:4.00]
-; SANDY-NEXT:    rcrl $7, %edi # sched: [5:4.00]
-; SANDY-NEXT:    rcll $7, (%rdx) # sched: [11:3.50]
-; SANDY-NEXT:    rcrl $7, (%rdx) # sched: [11:3.50]
-; SANDY-NEXT:    rcll %cl, %edi # sched: [5:4.00]
-; SANDY-NEXT:    rcrl %cl, %edi # sched: [5:4.00]
-; SANDY-NEXT:    rcll %cl, (%rdx) # sched: [11:3.50]
-; SANDY-NEXT:    rcrl %cl, (%rdx) # sched: [11:3.50]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_rcl_rcr_32:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    rcll %edi # sched: [3:0.75]
-; HASWELL-NEXT:    rcrl %edi # sched: [3:0.75]
-; HASWELL-NEXT:    rcll (%rdx) # sched: [9:0.75]
-; HASWELL-NEXT:    rcrl (%rdx) # sched: [9:0.75]
-; HASWELL-NEXT:    rcll $7, %edi # sched: [3:0.75]
-; HASWELL-NEXT:    rcrl $7, %edi # sched: [3:0.75]
-; HASWELL-NEXT:    rcll $7, (%rdx) # sched: [9:0.75]
-; HASWELL-NEXT:    rcrl $7, (%rdx) # sched: [9:0.75]
-; HASWELL-NEXT:    rcll %cl, %edi # sched: [11:2.00]
-; HASWELL-NEXT:    rcrl %cl, %edi # sched: [11:2.00]
-; HASWELL-NEXT:    rcll %cl, (%rdx) # sched: [16:2.00]
-; HASWELL-NEXT:    rcrl %cl, (%rdx) # sched: [19:2.25]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_rcl_rcr_32:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    rcll %edi # sched: [3:0.75]
-; BROADWELL-NEXT:    rcrl %edi # sched: [3:0.75]
-; BROADWELL-NEXT:    rcll (%rdx) # sched: [8:0.75]
-; BROADWELL-NEXT:    rcrl (%rdx) # sched: [8:0.75]
-; BROADWELL-NEXT:    rcll $7, %edi # sched: [3:0.75]
-; BROADWELL-NEXT:    rcrl $7, %edi # sched: [3:0.75]
-; BROADWELL-NEXT:    rcll $7, (%rdx) # sched: [8:0.75]
-; BROADWELL-NEXT:    rcrl $7, (%rdx) # sched: [8:0.75]
-; BROADWELL-NEXT:    rcll %cl, %edi # sched: [11:2.00]
-; BROADWELL-NEXT:    rcrl %cl, %edi # sched: [11:2.00]
-; BROADWELL-NEXT:    rcll %cl, (%rdx) # sched: [15:2.00]
-; BROADWELL-NEXT:    rcrl %cl, (%rdx) # sched: [18:2.25]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_rcl_rcr_32:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    rcll %edi # sched: [3:0.75]
-; SKYLAKE-NEXT:    rcrl %edi # sched: [3:0.75]
-; SKYLAKE-NEXT:    rcll (%rdx) # sched: [8:0.75]
-; SKYLAKE-NEXT:    rcrl (%rdx) # sched: [8:0.75]
-; SKYLAKE-NEXT:    rcll $7, %edi # sched: [3:0.75]
-; SKYLAKE-NEXT:    rcrl $7, %edi # sched: [3:0.75]
-; SKYLAKE-NEXT:    rcll $7, (%rdx) # sched: [8:0.75]
-; SKYLAKE-NEXT:    rcrl $7, (%rdx) # sched: [8:0.75]
-; SKYLAKE-NEXT:    rcll %cl, %edi # sched: [11:2.00]
-; SKYLAKE-NEXT:    rcrl %cl, %edi # sched: [11:2.00]
-; SKYLAKE-NEXT:    rcll %cl, (%rdx) # sched: [15:2.50]
-; SKYLAKE-NEXT:    rcrl %cl, (%rdx) # sched: [18:2.25]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_rcl_rcr_32:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    rcll %edi # sched: [3:0.75]
-; SKX-NEXT:    rcrl %edi # sched: [3:0.75]
-; SKX-NEXT:    rcll (%rdx) # sched: [8:0.75]
-; SKX-NEXT:    rcrl (%rdx) # sched: [8:0.75]
-; SKX-NEXT:    rcll $7, %edi # sched: [3:0.75]
-; SKX-NEXT:    rcrl $7, %edi # sched: [3:0.75]
-; SKX-NEXT:    rcll $7, (%rdx) # sched: [8:0.75]
-; SKX-NEXT:    rcrl $7, (%rdx) # sched: [8:0.75]
-; SKX-NEXT:    rcll %cl, %edi # sched: [11:2.00]
-; SKX-NEXT:    rcrl %cl, %edi # sched: [11:2.00]
-; SKX-NEXT:    rcll %cl, (%rdx) # sched: [15:2.50]
-; SKX-NEXT:    rcrl %cl, (%rdx) # sched: [18:2.25]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_rcl_rcr_32:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    rcll %edi # sched: [1:0.50]
-; BDVER2-NEXT:    rcrl %edi # sched: [1:0.50]
-; BDVER2-NEXT:    rcll (%rdx) # sched: [5:1.00]
-; BDVER2-NEXT:    rcrl (%rdx) # sched: [5:1.00]
-; BDVER2-NEXT:    rcll $7, %edi # sched: [8:0.50]
-; BDVER2-NEXT:    rcrl $7, %edi # sched: [7:0.50]
-; BDVER2-NEXT:    rcll $7, (%rdx) # sched: [5:1.00]
-; BDVER2-NEXT:    rcrl $7, (%rdx) # sched: [5:1.00]
-; BDVER2-NEXT:    rcll %cl, %edi # sched: [7:0.50]
-; BDVER2-NEXT:    rcrl %cl, %edi # sched: [7:0.50]
-; BDVER2-NEXT:    rcll %cl, (%rdx) # sched: [5:1.00]
-; BDVER2-NEXT:    rcrl %cl, (%rdx) # sched: [5:1.00]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_rcl_rcr_32:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    rcll %edi # sched: [1:0.50]
-; BTVER2-NEXT:    rcrl %edi # sched: [1:0.50]
-; BTVER2-NEXT:    rcll (%rdx) # sched: [4:1.00]
-; BTVER2-NEXT:    rcrl (%rdx) # sched: [4:1.00]
-; BTVER2-NEXT:    rcll $7, %edi # sched: [1:0.50]
-; BTVER2-NEXT:    rcrl $7, %edi # sched: [1:0.50]
-; BTVER2-NEXT:    rcll $7, (%rdx) # sched: [4:1.00]
-; BTVER2-NEXT:    rcrl $7, (%rdx) # sched: [4:1.00]
-; BTVER2-NEXT:    rcll %cl, %edi # sched: [1:0.50]
-; BTVER2-NEXT:    rcrl %cl, %edi # sched: [1:0.50]
-; BTVER2-NEXT:    rcll %cl, (%rdx) # sched: [4:1.00]
-; BTVER2-NEXT:    rcrl %cl, (%rdx) # sched: [4:1.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_rcl_rcr_32:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    rcll %edi # sched: [1:0.25]
-; ZNVER1-NEXT:    rcrl %edi # sched: [1:0.25]
-; ZNVER1-NEXT:    rcll (%rdx) # sched: [100:0.25]
-; ZNVER1-NEXT:    rcrl (%rdx) # sched: [100:0.25]
-; ZNVER1-NEXT:    rcll $7, %edi # sched: [1:0.25]
-; ZNVER1-NEXT:    rcrl $7, %edi # sched: [1:0.25]
-; ZNVER1-NEXT:    rcll $7, (%rdx) # sched: [100:0.25]
-; ZNVER1-NEXT:    rcrl $7, (%rdx) # sched: [100:0.25]
-; ZNVER1-NEXT:    rcll %cl, %edi # sched: [1:0.25]
-; ZNVER1-NEXT:    rcrl %cl, %edi # sched: [1:0.25]
-; ZNVER1-NEXT:    rcll %cl, (%rdx) # sched: [100:0.25]
-; ZNVER1-NEXT:    rcrl %cl, (%rdx) # sched: [100:0.25]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  call void asm sideeffect "rcll $0 \0A\09 rcrl $0 \0A\09 rcll $2 \0A\09 rcrl $2 \0A\09 rcll $3, $0 \0A\09 rcrl $3, $0 \0A\09 rcll $3, $2 \0A\09 rcrl $3, $2 \0A\09 rcll %CL, $0 \0A\09 rcrl %CL, $0 \0A\09 rcll %CL, $2 \0A\09 rcrl %CL, $2", "r,r,*m,i"(i32 %a0, i32 %a1, i32 *%a2, i8 7)
-  ret void
-}
-define void @test_rcl_rcr_64(i64 %a0, i64 %a1, i64 *%a2) optsize {
-; GENERIC-LABEL: test_rcl_rcr_64:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    rclq %rdi # sched: [2:1.50]
-; GENERIC-NEXT:    rcrq %rdi # sched: [2:1.50]
-; GENERIC-NEXT:    rclq (%rdx) # sched: [11:3.50]
-; GENERIC-NEXT:    rcrq (%rdx) # sched: [11:3.50]
-; GENERIC-NEXT:    rclq $7, %rdi # sched: [5:4.00]
-; GENERIC-NEXT:    rcrq $7, %rdi # sched: [5:4.00]
-; GENERIC-NEXT:    rclq $7, (%rdx) # sched: [11:3.50]
-; GENERIC-NEXT:    rcrq $7, (%rdx) # sched: [11:3.50]
-; GENERIC-NEXT:    rclq %cl, %rdi # sched: [5:4.00]
-; GENERIC-NEXT:    rcrq %cl, %rdi # sched: [5:4.00]
-; GENERIC-NEXT:    rclq %cl, (%rdx) # sched: [11:3.50]
-; GENERIC-NEXT:    rcrq %cl, (%rdx) # sched: [11:3.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_rcl_rcr_64:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    rclq %rdi # sched: [1:1.00]
-; ATOM-NEXT:    rcrq %rdi # sched: [1:1.00]
-; ATOM-NEXT:    rclq (%rdx) # sched: [1:1.00]
-; ATOM-NEXT:    rcrq (%rdx) # sched: [1:1.00]
-; ATOM-NEXT:    rclq $7, %rdi # sched: [1:1.00]
-; ATOM-NEXT:    rcrq $7, %rdi # sched: [1:1.00]
-; ATOM-NEXT:    rclq $7, (%rdx) # sched: [1:1.00]
-; ATOM-NEXT:    rcrq $7, (%rdx) # sched: [1:1.00]
-; ATOM-NEXT:    rclq %cl, %rdi # sched: [1:1.00]
-; ATOM-NEXT:    rcrq %cl, %rdi # sched: [1:1.00]
-; ATOM-NEXT:    rclq %cl, (%rdx) # sched: [1:1.00]
-; ATOM-NEXT:    rcrq %cl, (%rdx) # sched: [1:1.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_rcl_rcr_64:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    rclq %rdi # sched: [1:1.00]
-; SLM-NEXT:    rcrq %rdi # sched: [1:1.00]
-; SLM-NEXT:    rclq (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    rcrq (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    rclq $7, %rdi # sched: [1:1.00]
-; SLM-NEXT:    rcrq $7, %rdi # sched: [1:1.00]
-; SLM-NEXT:    rclq $7, (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    rcrq $7, (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    rclq %cl, %rdi # sched: [1:1.00]
-; SLM-NEXT:    rcrq %cl, %rdi # sched: [1:1.00]
-; SLM-NEXT:    rclq %cl, (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    rcrq %cl, (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_rcl_rcr_64:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    rclq %rdi # sched: [2:1.50]
-; SANDY-NEXT:    rcrq %rdi # sched: [2:1.50]
-; SANDY-NEXT:    rclq (%rdx) # sched: [11:3.50]
-; SANDY-NEXT:    rcrq (%rdx) # sched: [11:3.50]
-; SANDY-NEXT:    rclq $7, %rdi # sched: [5:4.00]
-; SANDY-NEXT:    rcrq $7, %rdi # sched: [5:4.00]
-; SANDY-NEXT:    rclq $7, (%rdx) # sched: [11:3.50]
-; SANDY-NEXT:    rcrq $7, (%rdx) # sched: [11:3.50]
-; SANDY-NEXT:    rclq %cl, %rdi # sched: [5:4.00]
-; SANDY-NEXT:    rcrq %cl, %rdi # sched: [5:4.00]
-; SANDY-NEXT:    rclq %cl, (%rdx) # sched: [11:3.50]
-; SANDY-NEXT:    rcrq %cl, (%rdx) # sched: [11:3.50]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_rcl_rcr_64:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    rclq %rdi # sched: [3:0.75]
-; HASWELL-NEXT:    rcrq %rdi # sched: [3:0.75]
-; HASWELL-NEXT:    rclq (%rdx) # sched: [9:0.75]
-; HASWELL-NEXT:    rcrq (%rdx) # sched: [9:0.75]
-; HASWELL-NEXT:    rclq $7, %rdi # sched: [3:0.75]
-; HASWELL-NEXT:    rcrq $7, %rdi # sched: [3:0.75]
-; HASWELL-NEXT:    rclq $7, (%rdx) # sched: [9:0.75]
-; HASWELL-NEXT:    rcrq $7, (%rdx) # sched: [9:0.75]
-; HASWELL-NEXT:    rclq %cl, %rdi # sched: [11:2.00]
-; HASWELL-NEXT:    rcrq %cl, %rdi # sched: [11:2.00]
-; HASWELL-NEXT:    rclq %cl, (%rdx) # sched: [16:2.00]
-; HASWELL-NEXT:    rcrq %cl, (%rdx) # sched: [19:2.25]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_rcl_rcr_64:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    rclq %rdi # sched: [3:0.75]
-; BROADWELL-NEXT:    rcrq %rdi # sched: [3:0.75]
-; BROADWELL-NEXT:    rclq (%rdx) # sched: [8:0.75]
-; BROADWELL-NEXT:    rcrq (%rdx) # sched: [8:0.75]
-; BROADWELL-NEXT:    rclq $7, %rdi # sched: [3:0.75]
-; BROADWELL-NEXT:    rcrq $7, %rdi # sched: [3:0.75]
-; BROADWELL-NEXT:    rclq $7, (%rdx) # sched: [8:0.75]
-; BROADWELL-NEXT:    rcrq $7, (%rdx) # sched: [8:0.75]
-; BROADWELL-NEXT:    rclq %cl, %rdi # sched: [11:2.00]
-; BROADWELL-NEXT:    rcrq %cl, %rdi # sched: [11:2.00]
-; BROADWELL-NEXT:    rclq %cl, (%rdx) # sched: [15:2.00]
-; BROADWELL-NEXT:    rcrq %cl, (%rdx) # sched: [18:2.25]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_rcl_rcr_64:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    rclq %rdi # sched: [3:0.75]
-; SKYLAKE-NEXT:    rcrq %rdi # sched: [3:0.75]
-; SKYLAKE-NEXT:    rclq (%rdx) # sched: [8:0.75]
-; SKYLAKE-NEXT:    rcrq (%rdx) # sched: [8:0.75]
-; SKYLAKE-NEXT:    rclq $7, %rdi # sched: [3:0.75]
-; SKYLAKE-NEXT:    rcrq $7, %rdi # sched: [3:0.75]
-; SKYLAKE-NEXT:    rclq $7, (%rdx) # sched: [8:0.75]
-; SKYLAKE-NEXT:    rcrq $7, (%rdx) # sched: [8:0.75]
-; SKYLAKE-NEXT:    rclq %cl, %rdi # sched: [11:2.00]
-; SKYLAKE-NEXT:    rcrq %cl, %rdi # sched: [11:2.00]
-; SKYLAKE-NEXT:    rclq %cl, (%rdx) # sched: [15:2.50]
-; SKYLAKE-NEXT:    rcrq %cl, (%rdx) # sched: [18:2.25]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_rcl_rcr_64:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    rclq %rdi # sched: [3:0.75]
-; SKX-NEXT:    rcrq %rdi # sched: [3:0.75]
-; SKX-NEXT:    rclq (%rdx) # sched: [8:0.75]
-; SKX-NEXT:    rcrq (%rdx) # sched: [8:0.75]
-; SKX-NEXT:    rclq $7, %rdi # sched: [3:0.75]
-; SKX-NEXT:    rcrq $7, %rdi # sched: [3:0.75]
-; SKX-NEXT:    rclq $7, (%rdx) # sched: [8:0.75]
-; SKX-NEXT:    rcrq $7, (%rdx) # sched: [8:0.75]
-; SKX-NEXT:    rclq %cl, %rdi # sched: [11:2.00]
-; SKX-NEXT:    rcrq %cl, %rdi # sched: [11:2.00]
-; SKX-NEXT:    rclq %cl, (%rdx) # sched: [15:2.50]
-; SKX-NEXT:    rcrq %cl, (%rdx) # sched: [18:2.25]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_rcl_rcr_64:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    rclq %rdi # sched: [1:0.50]
-; BDVER2-NEXT:    rcrq %rdi # sched: [1:0.50]
-; BDVER2-NEXT:    rclq (%rdx) # sched: [5:1.00]
-; BDVER2-NEXT:    rcrq (%rdx) # sched: [5:1.00]
-; BDVER2-NEXT:    rclq $7, %rdi # sched: [8:0.50]
-; BDVER2-NEXT:    rcrq $7, %rdi # sched: [7:0.50]
-; BDVER2-NEXT:    rclq $7, (%rdx) # sched: [5:1.00]
-; BDVER2-NEXT:    rcrq $7, (%rdx) # sched: [5:1.00]
-; BDVER2-NEXT:    rclq %cl, %rdi # sched: [7:0.50]
-; BDVER2-NEXT:    rcrq %cl, %rdi # sched: [7:0.50]
-; BDVER2-NEXT:    rclq %cl, (%rdx) # sched: [5:1.00]
-; BDVER2-NEXT:    rcrq %cl, (%rdx) # sched: [5:1.00]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_rcl_rcr_64:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    rclq %rdi # sched: [1:0.50]
-; BTVER2-NEXT:    rcrq %rdi # sched: [1:0.50]
-; BTVER2-NEXT:    rclq (%rdx) # sched: [4:1.00]
-; BTVER2-NEXT:    rcrq (%rdx) # sched: [4:1.00]
-; BTVER2-NEXT:    rclq $7, %rdi # sched: [1:0.50]
-; BTVER2-NEXT:    rcrq $7, %rdi # sched: [1:0.50]
-; BTVER2-NEXT:    rclq $7, (%rdx) # sched: [4:1.00]
-; BTVER2-NEXT:    rcrq $7, (%rdx) # sched: [4:1.00]
-; BTVER2-NEXT:    rclq %cl, %rdi # sched: [1:0.50]
-; BTVER2-NEXT:    rcrq %cl, %rdi # sched: [1:0.50]
-; BTVER2-NEXT:    rclq %cl, (%rdx) # sched: [4:1.00]
-; BTVER2-NEXT:    rcrq %cl, (%rdx) # sched: [4:1.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_rcl_rcr_64:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    rclq %rdi # sched: [1:0.25]
-; ZNVER1-NEXT:    rcrq %rdi # sched: [1:0.25]
-; ZNVER1-NEXT:    rclq (%rdx) # sched: [100:0.25]
-; ZNVER1-NEXT:    rcrq (%rdx) # sched: [100:0.25]
-; ZNVER1-NEXT:    rclq $7, %rdi # sched: [1:0.25]
-; ZNVER1-NEXT:    rcrq $7, %rdi # sched: [1:0.25]
-; ZNVER1-NEXT:    rclq $7, (%rdx) # sched: [100:0.25]
-; ZNVER1-NEXT:    rcrq $7, (%rdx) # sched: [100:0.25]
-; ZNVER1-NEXT:    rclq %cl, %rdi # sched: [1:0.25]
-; ZNVER1-NEXT:    rcrq %cl, %rdi # sched: [1:0.25]
-; ZNVER1-NEXT:    rclq %cl, (%rdx) # sched: [100:0.25]
-; ZNVER1-NEXT:    rcrq %cl, (%rdx) # sched: [100:0.25]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  call void asm sideeffect "rclq $0 \0A\09 rcrq $0 \0A\09 rclq $2 \0A\09 rcrq $2 \0A\09 rclq $3, $0 \0A\09 rcrq $3, $0 \0A\09 rclq $3, $2 \0A\09 rcrq $3, $2 \0A\09 rclq %CL, $0 \0A\09 rcrq %CL, $0 \0A\09 rclq %CL, $2 \0A\09 rcrq %CL, $2", "r,r,*m,i"(i64 %a0, i64 %a1, i64 *%a2, i8 7)
-  ret void
-}
-
-define void @test_rdmsr_wrmsr() optsize {
-; GENERIC-LABEL: test_rdmsr_wrmsr:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    rdmsr # sched: [100:0.33]
-; GENERIC-NEXT:    wrmsr # sched: [100:0.33]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_rdmsr_wrmsr:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    rdmsr # sched: [78:39.00]
-; ATOM-NEXT:    wrmsr # sched: [202:101.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_rdmsr_wrmsr:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    rdmsr # sched: [100:1.00]
-; SLM-NEXT:    wrmsr # sched: [100:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_rdmsr_wrmsr:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    rdmsr # sched: [100:0.33]
-; SANDY-NEXT:    wrmsr # sched: [100:0.33]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_rdmsr_wrmsr:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    rdmsr # sched: [100:0.25]
-; HASWELL-NEXT:    wrmsr # sched: [100:0.25]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_rdmsr_wrmsr:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    rdmsr # sched: [100:0.25]
-; BROADWELL-NEXT:    wrmsr # sched: [100:0.25]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_rdmsr_wrmsr:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    rdmsr # sched: [100:0.25]
-; SKYLAKE-NEXT:    wrmsr # sched: [100:0.25]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_rdmsr_wrmsr:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    rdmsr # sched: [100:0.25]
-; SKX-NEXT:    wrmsr # sched: [100:0.25]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_rdmsr_wrmsr:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    rdmsr # sched: [100:0.50]
-; BDVER2-NEXT:    wrmsr # sched: [100:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_rdmsr_wrmsr:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    rdmsr # sched: [100:0.50]
-; BTVER2-NEXT:    wrmsr # sched: [100:0.50]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_rdmsr_wrmsr:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    rdmsr # sched: [100:0.25]
-; ZNVER1-NEXT:    wrmsr # sched: [100:0.25]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  call void asm sideeffect "rdmsr \0A\09 wrmsr", ""()
-  ret void
-}
-
-define void @test_rdpmc() optsize {
-; GENERIC-LABEL: test_rdpmc:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    rdpmc # sched: [100:0.33]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_rdpmc:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    rdpmc # sched: [46:23.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_rdpmc:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    rdpmc # sched: [100:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_rdpmc:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    rdpmc # sched: [100:0.33]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_rdpmc:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    rdpmc # sched: [1:8.50]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_rdpmc:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    rdpmc # sched: [100:0.25]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_rdpmc:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    rdpmc # sched: [100:0.25]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_rdpmc:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    rdpmc # sched: [100:0.25]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_rdpmc:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    rdpmc # sched: [100:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_rdpmc:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    rdpmc # sched: [100:0.50]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_rdpmc:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    rdpmc # sched: [100:0.25]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  call void asm sideeffect "rdpmc", ""()
-  ret void
-}
-
-define void @test_rdtsc_rdtscp() optsize {
-; GENERIC-LABEL: test_rdtsc_rdtscp:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    rdtsc # sched: [100:0.33]
-; GENERIC-NEXT:    rdtscp # sched: [100:0.33]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_rdtsc_rdtscp:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    rdtsc # sched: [30:15.00]
-; ATOM-NEXT:    rdtscp # sched: [30:15.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_rdtsc_rdtscp:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    rdtsc # sched: [100:1.00]
-; SLM-NEXT:    rdtscp # sched: [100:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_rdtsc_rdtscp:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    rdtsc # sched: [100:0.33]
-; SANDY-NEXT:    rdtscp # sched: [100:0.33]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_rdtsc_rdtscp:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    rdtsc # sched: [18:2.00]
-; HASWELL-NEXT:    rdtscp # sched: [42:5.50]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_rdtsc_rdtscp:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    rdtsc # sched: [18:2.00]
-; BROADWELL-NEXT:    rdtscp # sched: [42:5.50]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_rdtsc_rdtscp:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    rdtsc # sched: [18:2.00]
-; SKYLAKE-NEXT:    rdtscp # sched: [42:5.50]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_rdtsc_rdtscp:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    rdtsc # sched: [18:2.00]
-; SKX-NEXT:    rdtscp # sched: [42:5.50]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_rdtsc_rdtscp:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    rdtsc # sched: [100:0.50]
-; BDVER2-NEXT:    rdtscp # sched: [100:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_rdtsc_rdtscp:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    rdtsc # sched: [100:0.50]
-; BTVER2-NEXT:    rdtscp # sched: [100:0.50]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_rdtsc_rdtscp:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    rdtsc # sched: [100:0.25]
-; ZNVER1-NEXT:    rdtscp # sched: [100:0.25]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  call void asm sideeffect "rdtsc \0A\09 rdtscp", ""()
-  ret void
-}
-
-define void @test_ret() optsize {
-; GENERIC-LABEL: test_ret:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-; GENERIC-NEXT:    retq $4095 # imm = 0xFFF
-; GENERIC-NEXT:    # sched: [6:1.00]
-; GENERIC-NEXT:    lretl # sched: [6:1.00]
-; GENERIC-NEXT:    lretl $4095 # imm = 0xFFF
-; GENERIC-NEXT:    # sched: [6:1.00]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_ret:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    retq # sched: [79:39.50]
-; ATOM-NEXT:    retq $4095 # imm = 0xFFF
-; ATOM-NEXT:    # sched: [1:1.00]
-; ATOM-NEXT:    lretl # sched: [79:39.50]
-; ATOM-NEXT:    lretl $4095 # imm = 0xFFF
-; ATOM-NEXT:    # sched: [79:39.50]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_ret:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-; SLM-NEXT:    retq $4095 # imm = 0xFFF
-; SLM-NEXT:    # sched: [4:1.00]
-; SLM-NEXT:    lretl # sched: [4:1.00]
-; SLM-NEXT:    lretl $4095 # imm = 0xFFF
-; SLM-NEXT:    # sched: [4:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_ret:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-; SANDY-NEXT:    retq $4095 # imm = 0xFFF
-; SANDY-NEXT:    # sched: [6:1.00]
-; SANDY-NEXT:    lretl # sched: [6:1.00]
-; SANDY-NEXT:    lretl $4095 # imm = 0xFFF
-; SANDY-NEXT:    # sched: [6:1.00]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_ret:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-; HASWELL-NEXT:    retq $4095 # imm = 0xFFF
-; HASWELL-NEXT:    # sched: [1:2.00]
-; HASWELL-NEXT:    lretl # sched: [6:0.50]
-; HASWELL-NEXT:    lretl $4095 # imm = 0xFFF
-; HASWELL-NEXT:    # sched: [1:2.00]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_ret:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-; BROADWELL-NEXT:    retq $4095 # imm = 0xFFF
-; BROADWELL-NEXT:    # sched: [6:0.50]
-; BROADWELL-NEXT:    lretl # sched: [6:0.50]
-; BROADWELL-NEXT:    lretl $4095 # imm = 0xFFF
-; BROADWELL-NEXT:    # sched: [6:0.50]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_ret:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-; SKYLAKE-NEXT:    retq $4095 # imm = 0xFFF
-; SKYLAKE-NEXT:    # sched: [6:0.50]
-; SKYLAKE-NEXT:    lretl # sched: [6:0.50]
-; SKYLAKE-NEXT:    lretl $4095 # imm = 0xFFF
-; SKYLAKE-NEXT:    # sched: [6:0.50]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_ret:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-; SKX-NEXT:    retq $4095 # imm = 0xFFF
-; SKX-NEXT:    # sched: [6:0.50]
-; SKX-NEXT:    lretl # sched: [6:0.50]
-; SKX-NEXT:    lretl $4095 # imm = 0xFFF
-; SKX-NEXT:    # sched: [6:0.50]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_ret:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-; BDVER2-NEXT:    retq $4095 # imm = 0xFFF
-; BDVER2-NEXT:    # sched: [5:1.00]
-; BDVER2-NEXT:    lretl # sched: [5:1.00]
-; BDVER2-NEXT:    lretl $4095 # imm = 0xFFF
-; BDVER2-NEXT:    # sched: [5:1.00]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_ret:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-; BTVER2-NEXT:    retq $4095 # imm = 0xFFF
-; BTVER2-NEXT:    # sched: [4:1.00]
-; BTVER2-NEXT:    lretl # sched: [4:1.00]
-; BTVER2-NEXT:    lretl $4095 # imm = 0xFFF
-; BTVER2-NEXT:    # sched: [4:1.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_ret:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-; ZNVER1-NEXT:    retq $4095 # imm = 0xFFF
-; ZNVER1-NEXT:    # sched: [5:0.50]
-; ZNVER1-NEXT:    lretl # sched: [1:0.50]
-; ZNVER1-NEXT:    lretl $4095 # imm = 0xFFF
-; ZNVER1-NEXT:    # sched: [5:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  call void asm sideeffect "ret \0A\09 ret $0 \0A\09 lret \0A\09 lret $0", "i"(i16 4095)
-  ret void
-}
-
-define void @test_rol_ror_8(i8 %a0, i8 %a1, i8 *%a2) optsize {
-; GENERIC-LABEL: test_rol_ror_8:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    rolb %dil # sched: [2:1.00]
-; GENERIC-NEXT:    rorb %dil # sched: [2:1.00]
-; GENERIC-NEXT:    rolb (%rdx) # sched: [8:1.00]
-; GENERIC-NEXT:    rorb (%rdx) # sched: [8:1.00]
-; GENERIC-NEXT:    rolb $7, %dil # sched: [2:1.00]
-; GENERIC-NEXT:    rorb $7, %dil # sched: [2:1.00]
-; GENERIC-NEXT:    rolb $7, (%rdx) # sched: [8:1.00]
-; GENERIC-NEXT:    rorb $7, (%rdx) # sched: [8:1.00]
-; GENERIC-NEXT:    rolb %cl, %dil # sched: [3:1.50]
-; GENERIC-NEXT:    rorb %cl, %dil # sched: [3:1.50]
-; GENERIC-NEXT:    rolb %cl, (%rdx) # sched: [9:1.50]
-; GENERIC-NEXT:    rorb %cl, (%rdx) # sched: [9:1.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_rol_ror_8:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    rolb %dil # sched: [1:1.00]
-; ATOM-NEXT:    rorb %dil # sched: [1:1.00]
-; ATOM-NEXT:    rolb (%rdx) # sched: [1:1.00]
-; ATOM-NEXT:    rorb (%rdx) # sched: [1:1.00]
-; ATOM-NEXT:    rolb $7, %dil # sched: [1:1.00]
-; ATOM-NEXT:    rorb $7, %dil # sched: [1:1.00]
-; ATOM-NEXT:    rolb $7, (%rdx) # sched: [1:1.00]
-; ATOM-NEXT:    rorb $7, (%rdx) # sched: [1:1.00]
-; ATOM-NEXT:    rolb %cl, %dil # sched: [1:1.00]
-; ATOM-NEXT:    rorb %cl, %dil # sched: [1:1.00]
-; ATOM-NEXT:    rolb %cl, (%rdx) # sched: [1:1.00]
-; ATOM-NEXT:    rorb %cl, (%rdx) # sched: [1:1.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_rol_ror_8:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    rolb %dil # sched: [1:1.00]
-; SLM-NEXT:    rorb %dil # sched: [1:1.00]
-; SLM-NEXT:    rolb (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    rorb (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    rolb $7, %dil # sched: [1:1.00]
-; SLM-NEXT:    rorb $7, %dil # sched: [1:1.00]
-; SLM-NEXT:    rolb $7, (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    rorb $7, (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    rolb %cl, %dil # sched: [1:1.00]
-; SLM-NEXT:    rorb %cl, %dil # sched: [1:1.00]
-; SLM-NEXT:    rolb %cl, (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    rorb %cl, (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_rol_ror_8:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    rolb %dil # sched: [2:1.00]
-; SANDY-NEXT:    rorb %dil # sched: [2:1.00]
-; SANDY-NEXT:    rolb (%rdx) # sched: [8:1.00]
-; SANDY-NEXT:    rorb (%rdx) # sched: [8:1.00]
-; SANDY-NEXT:    rolb $7, %dil # sched: [2:1.00]
-; SANDY-NEXT:    rorb $7, %dil # sched: [2:1.00]
-; SANDY-NEXT:    rolb $7, (%rdx) # sched: [8:1.00]
-; SANDY-NEXT:    rorb $7, (%rdx) # sched: [8:1.00]
-; SANDY-NEXT:    rolb %cl, %dil # sched: [3:1.50]
-; SANDY-NEXT:    rorb %cl, %dil # sched: [3:1.50]
-; SANDY-NEXT:    rolb %cl, (%rdx) # sched: [9:1.50]
-; SANDY-NEXT:    rorb %cl, (%rdx) # sched: [9:1.50]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_rol_ror_8:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    rolb %dil # sched: [2:1.00]
-; HASWELL-NEXT:    rorb %dil # sched: [2:1.00]
-; HASWELL-NEXT:    rolb (%rdx) # sched: [8:1.00]
-; HASWELL-NEXT:    rorb (%rdx) # sched: [8:1.00]
-; HASWELL-NEXT:    rolb $7, %dil # sched: [2:1.00]
-; HASWELL-NEXT:    rorb $7, %dil # sched: [2:1.00]
-; HASWELL-NEXT:    rolb $7, (%rdx) # sched: [8:1.00]
-; HASWELL-NEXT:    rorb $7, (%rdx) # sched: [8:1.00]
-; HASWELL-NEXT:    rolb %cl, %dil # sched: [3:1.00]
-; HASWELL-NEXT:    rorb %cl, %dil # sched: [3:1.00]
-; HASWELL-NEXT:    rolb %cl, (%rdx) # sched: [9:1.00]
-; HASWELL-NEXT:    rorb %cl, (%rdx) # sched: [9:1.00]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_rol_ror_8:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    rolb %dil # sched: [2:1.00]
-; BROADWELL-NEXT:    rorb %dil # sched: [2:1.00]
-; BROADWELL-NEXT:    rolb (%rdx) # sched: [7:1.00]
-; BROADWELL-NEXT:    rorb (%rdx) # sched: [7:1.00]
-; BROADWELL-NEXT:    rolb $7, %dil # sched: [2:1.00]
-; BROADWELL-NEXT:    rorb $7, %dil # sched: [2:1.00]
-; BROADWELL-NEXT:    rolb $7, (%rdx) # sched: [7:1.00]
-; BROADWELL-NEXT:    rorb $7, (%rdx) # sched: [7:1.00]
-; BROADWELL-NEXT:    rolb %cl, %dil # sched: [3:1.00]
-; BROADWELL-NEXT:    rorb %cl, %dil # sched: [3:1.00]
-; BROADWELL-NEXT:    rolb %cl, (%rdx) # sched: [8:1.00]
-; BROADWELL-NEXT:    rorb %cl, (%rdx) # sched: [8:1.00]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_rol_ror_8:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    rolb %dil # sched: [2:1.00]
-; SKYLAKE-NEXT:    rorb %dil # sched: [2:1.00]
-; SKYLAKE-NEXT:    rolb (%rdx) # sched: [7:1.00]
-; SKYLAKE-NEXT:    rorb (%rdx) # sched: [7:1.00]
-; SKYLAKE-NEXT:    rolb $7, %dil # sched: [2:1.00]
-; SKYLAKE-NEXT:    rorb $7, %dil # sched: [2:1.00]
-; SKYLAKE-NEXT:    rolb $7, (%rdx) # sched: [7:1.00]
-; SKYLAKE-NEXT:    rorb $7, (%rdx) # sched: [7:1.00]
-; SKYLAKE-NEXT:    rolb %cl, %dil # sched: [3:1.50]
-; SKYLAKE-NEXT:    rorb %cl, %dil # sched: [3:1.50]
-; SKYLAKE-NEXT:    rolb %cl, (%rdx) # sched: [8:1.50]
-; SKYLAKE-NEXT:    rorb %cl, (%rdx) # sched: [8:1.50]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_rol_ror_8:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    rolb %dil # sched: [2:1.00]
-; SKX-NEXT:    rorb %dil # sched: [2:1.00]
-; SKX-NEXT:    rolb (%rdx) # sched: [7:1.00]
-; SKX-NEXT:    rorb (%rdx) # sched: [7:1.00]
-; SKX-NEXT:    rolb $7, %dil # sched: [2:1.00]
-; SKX-NEXT:    rorb $7, %dil # sched: [2:1.00]
-; SKX-NEXT:    rolb $7, (%rdx) # sched: [7:1.00]
-; SKX-NEXT:    rorb $7, (%rdx) # sched: [7:1.00]
-; SKX-NEXT:    rolb %cl, %dil # sched: [3:1.50]
-; SKX-NEXT:    rorb %cl, %dil # sched: [3:1.50]
-; SKX-NEXT:    rolb %cl, (%rdx) # sched: [8:1.50]
-; SKX-NEXT:    rorb %cl, (%rdx) # sched: [8:1.50]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_rol_ror_8:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    rolb %dil # sched: [1:0.50]
-; BDVER2-NEXT:    rorb %dil # sched: [1:0.50]
-; BDVER2-NEXT:    rolb (%rdx) # sched: [5:1.00]
-; BDVER2-NEXT:    rorb (%rdx) # sched: [5:1.00]
-; BDVER2-NEXT:    rolb $7, %dil # sched: [1:0.50]
-; BDVER2-NEXT:    rorb $7, %dil # sched: [1:0.50]
-; BDVER2-NEXT:    rolb $7, (%rdx) # sched: [5:1.00]
-; BDVER2-NEXT:    rorb $7, (%rdx) # sched: [5:1.00]
-; BDVER2-NEXT:    rolb %cl, %dil # sched: [1:0.50]
-; BDVER2-NEXT:    rorb %cl, %dil # sched: [1:0.50]
-; BDVER2-NEXT:    rolb %cl, (%rdx) # sched: [5:1.00]
-; BDVER2-NEXT:    rorb %cl, (%rdx) # sched: [5:1.00]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_rol_ror_8:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    rolb %dil # sched: [1:0.50]
-; BTVER2-NEXT:    rorb %dil # sched: [1:0.50]
-; BTVER2-NEXT:    rolb (%rdx) # sched: [4:1.00]
-; BTVER2-NEXT:    rorb (%rdx) # sched: [4:1.00]
-; BTVER2-NEXT:    rolb $7, %dil # sched: [1:0.50]
-; BTVER2-NEXT:    rorb $7, %dil # sched: [1:0.50]
-; BTVER2-NEXT:    rolb $7, (%rdx) # sched: [4:1.00]
-; BTVER2-NEXT:    rorb $7, (%rdx) # sched: [4:1.00]
-; BTVER2-NEXT:    rolb %cl, %dil # sched: [1:0.50]
-; BTVER2-NEXT:    rorb %cl, %dil # sched: [1:0.50]
-; BTVER2-NEXT:    rolb %cl, (%rdx) # sched: [4:1.00]
-; BTVER2-NEXT:    rorb %cl, (%rdx) # sched: [4:1.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_rol_ror_8:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    rolb %dil # sched: [1:0.25]
-; ZNVER1-NEXT:    rorb %dil # sched: [1:0.25]
-; ZNVER1-NEXT:    rolb (%rdx) # sched: [5:1.00]
-; ZNVER1-NEXT:    rorb (%rdx) # sched: [5:1.00]
-; ZNVER1-NEXT:    rolb $7, %dil # sched: [1:0.25]
-; ZNVER1-NEXT:    rorb $7, %dil # sched: [1:0.25]
-; ZNVER1-NEXT:    rolb $7, (%rdx) # sched: [5:1.00]
-; ZNVER1-NEXT:    rorb $7, (%rdx) # sched: [5:1.00]
-; ZNVER1-NEXT:    rolb %cl, %dil # sched: [1:0.25]
-; ZNVER1-NEXT:    rorb %cl, %dil # sched: [1:0.25]
-; ZNVER1-NEXT:    rolb %cl, (%rdx) # sched: [5:1.00]
-; ZNVER1-NEXT:    rorb %cl, (%rdx) # sched: [5:1.00]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  call void asm sideeffect "rolb $0 \0A\09 rorb $0 \0A\09 rolb $2 \0A\09 rorb $2 \0A\09 rolb $3, $0 \0A\09 rorb $3, $0 \0A\09 rolb $3, $2 \0A\09 rorb $3, $2 \0A\09 rolb %CL, $0 \0A\09 rorb %CL, $0 \0A\09 rolb %CL, $2 \0A\09 rorb %CL, $2", "r,r,*m,i"(i8 %a0, i8 %a1, i8 *%a2, i8 7)
-  ret void
-}
-define void @test_rol_ror_16(i16 %a0, i16 %a1, i16 *%a2) optsize {
-; GENERIC-LABEL: test_rol_ror_16:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    rolw %di # sched: [2:1.00]
-; GENERIC-NEXT:    rorw %di # sched: [2:1.00]
-; GENERIC-NEXT:    rolw (%rdx) # sched: [8:1.00]
-; GENERIC-NEXT:    rorw (%rdx) # sched: [8:1.00]
-; GENERIC-NEXT:    rolw $7, %di # sched: [2:1.00]
-; GENERIC-NEXT:    rorw $7, %di # sched: [2:1.00]
-; GENERIC-NEXT:    rolw $7, (%rdx) # sched: [8:1.00]
-; GENERIC-NEXT:    rorw $7, (%rdx) # sched: [8:1.00]
-; GENERIC-NEXT:    rolw %cl, %di # sched: [3:1.50]
-; GENERIC-NEXT:    rorw %cl, %di # sched: [3:1.50]
-; GENERIC-NEXT:    rolw %cl, (%rdx) # sched: [9:1.50]
-; GENERIC-NEXT:    rorw %cl, (%rdx) # sched: [9:1.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_rol_ror_16:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    rolw %di # sched: [1:1.00]
-; ATOM-NEXT:    rorw %di # sched: [1:1.00]
-; ATOM-NEXT:    rolw (%rdx) # sched: [1:1.00]
-; ATOM-NEXT:    rorw (%rdx) # sched: [1:1.00]
-; ATOM-NEXT:    rolw $7, %di # sched: [1:1.00]
-; ATOM-NEXT:    rorw $7, %di # sched: [1:1.00]
-; ATOM-NEXT:    rolw $7, (%rdx) # sched: [1:1.00]
-; ATOM-NEXT:    rorw $7, (%rdx) # sched: [1:1.00]
-; ATOM-NEXT:    rolw %cl, %di # sched: [1:1.00]
-; ATOM-NEXT:    rorw %cl, %di # sched: [1:1.00]
-; ATOM-NEXT:    rolw %cl, (%rdx) # sched: [1:1.00]
-; ATOM-NEXT:    rorw %cl, (%rdx) # sched: [1:1.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_rol_ror_16:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    rolw %di # sched: [1:1.00]
-; SLM-NEXT:    rorw %di # sched: [1:1.00]
-; SLM-NEXT:    rolw (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    rorw (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    rolw $7, %di # sched: [1:1.00]
-; SLM-NEXT:    rorw $7, %di # sched: [1:1.00]
-; SLM-NEXT:    rolw $7, (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    rorw $7, (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    rolw %cl, %di # sched: [1:1.00]
-; SLM-NEXT:    rorw %cl, %di # sched: [1:1.00]
-; SLM-NEXT:    rolw %cl, (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    rorw %cl, (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_rol_ror_16:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    rolw %di # sched: [2:1.00]
-; SANDY-NEXT:    rorw %di # sched: [2:1.00]
-; SANDY-NEXT:    rolw (%rdx) # sched: [8:1.00]
-; SANDY-NEXT:    rorw (%rdx) # sched: [8:1.00]
-; SANDY-NEXT:    rolw $7, %di # sched: [2:1.00]
-; SANDY-NEXT:    rorw $7, %di # sched: [2:1.00]
-; SANDY-NEXT:    rolw $7, (%rdx) # sched: [8:1.00]
-; SANDY-NEXT:    rorw $7, (%rdx) # sched: [8:1.00]
-; SANDY-NEXT:    rolw %cl, %di # sched: [3:1.50]
-; SANDY-NEXT:    rorw %cl, %di # sched: [3:1.50]
-; SANDY-NEXT:    rolw %cl, (%rdx) # sched: [9:1.50]
-; SANDY-NEXT:    rorw %cl, (%rdx) # sched: [9:1.50]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_rol_ror_16:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    rolw %di # sched: [2:1.00]
-; HASWELL-NEXT:    rorw %di # sched: [2:1.00]
-; HASWELL-NEXT:    rolw (%rdx) # sched: [8:1.00]
-; HASWELL-NEXT:    rorw (%rdx) # sched: [8:1.00]
-; HASWELL-NEXT:    rolw $7, %di # sched: [2:1.00]
-; HASWELL-NEXT:    rorw $7, %di # sched: [2:1.00]
-; HASWELL-NEXT:    rolw $7, (%rdx) # sched: [8:1.00]
-; HASWELL-NEXT:    rorw $7, (%rdx) # sched: [8:1.00]
-; HASWELL-NEXT:    rolw %cl, %di # sched: [3:1.00]
-; HASWELL-NEXT:    rorw %cl, %di # sched: [3:1.00]
-; HASWELL-NEXT:    rolw %cl, (%rdx) # sched: [9:1.00]
-; HASWELL-NEXT:    rorw %cl, (%rdx) # sched: [9:1.00]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_rol_ror_16:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    rolw %di # sched: [2:1.00]
-; BROADWELL-NEXT:    rorw %di # sched: [2:1.00]
-; BROADWELL-NEXT:    rolw (%rdx) # sched: [7:1.00]
-; BROADWELL-NEXT:    rorw (%rdx) # sched: [7:1.00]
-; BROADWELL-NEXT:    rolw $7, %di # sched: [2:1.00]
-; BROADWELL-NEXT:    rorw $7, %di # sched: [2:1.00]
-; BROADWELL-NEXT:    rolw $7, (%rdx) # sched: [7:1.00]
-; BROADWELL-NEXT:    rorw $7, (%rdx) # sched: [7:1.00]
-; BROADWELL-NEXT:    rolw %cl, %di # sched: [3:1.00]
-; BROADWELL-NEXT:    rorw %cl, %di # sched: [3:1.00]
-; BROADWELL-NEXT:    rolw %cl, (%rdx) # sched: [8:1.00]
-; BROADWELL-NEXT:    rorw %cl, (%rdx) # sched: [8:1.00]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_rol_ror_16:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    rolw %di # sched: [2:1.00]
-; SKYLAKE-NEXT:    rorw %di # sched: [2:1.00]
-; SKYLAKE-NEXT:    rolw (%rdx) # sched: [7:1.00]
-; SKYLAKE-NEXT:    rorw (%rdx) # sched: [7:1.00]
-; SKYLAKE-NEXT:    rolw $7, %di # sched: [2:1.00]
-; SKYLAKE-NEXT:    rorw $7, %di # sched: [2:1.00]
-; SKYLAKE-NEXT:    rolw $7, (%rdx) # sched: [7:1.00]
-; SKYLAKE-NEXT:    rorw $7, (%rdx) # sched: [7:1.00]
-; SKYLAKE-NEXT:    rolw %cl, %di # sched: [3:1.50]
-; SKYLAKE-NEXT:    rorw %cl, %di # sched: [3:1.50]
-; SKYLAKE-NEXT:    rolw %cl, (%rdx) # sched: [8:1.50]
-; SKYLAKE-NEXT:    rorw %cl, (%rdx) # sched: [8:1.50]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_rol_ror_16:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    rolw %di # sched: [2:1.00]
-; SKX-NEXT:    rorw %di # sched: [2:1.00]
-; SKX-NEXT:    rolw (%rdx) # sched: [7:1.00]
-; SKX-NEXT:    rorw (%rdx) # sched: [7:1.00]
-; SKX-NEXT:    rolw $7, %di # sched: [2:1.00]
-; SKX-NEXT:    rorw $7, %di # sched: [2:1.00]
-; SKX-NEXT:    rolw $7, (%rdx) # sched: [7:1.00]
-; SKX-NEXT:    rorw $7, (%rdx) # sched: [7:1.00]
-; SKX-NEXT:    rolw %cl, %di # sched: [3:1.50]
-; SKX-NEXT:    rorw %cl, %di # sched: [3:1.50]
-; SKX-NEXT:    rolw %cl, (%rdx) # sched: [8:1.50]
-; SKX-NEXT:    rorw %cl, (%rdx) # sched: [8:1.50]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_rol_ror_16:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    rolw %di # sched: [1:0.50]
-; BDVER2-NEXT:    rorw %di # sched: [1:0.50]
-; BDVER2-NEXT:    rolw (%rdx) # sched: [5:1.00]
-; BDVER2-NEXT:    rorw (%rdx) # sched: [5:1.00]
-; BDVER2-NEXT:    rolw $7, %di # sched: [1:0.50]
-; BDVER2-NEXT:    rorw $7, %di # sched: [1:0.50]
-; BDVER2-NEXT:    rolw $7, (%rdx) # sched: [5:1.00]
-; BDVER2-NEXT:    rorw $7, (%rdx) # sched: [5:1.00]
-; BDVER2-NEXT:    rolw %cl, %di # sched: [1:0.50]
-; BDVER2-NEXT:    rorw %cl, %di # sched: [1:0.50]
-; BDVER2-NEXT:    rolw %cl, (%rdx) # sched: [5:1.00]
-; BDVER2-NEXT:    rorw %cl, (%rdx) # sched: [5:1.00]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_rol_ror_16:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    rolw %di # sched: [1:0.50]
-; BTVER2-NEXT:    rorw %di # sched: [1:0.50]
-; BTVER2-NEXT:    rolw (%rdx) # sched: [4:1.00]
-; BTVER2-NEXT:    rorw (%rdx) # sched: [4:1.00]
-; BTVER2-NEXT:    rolw $7, %di # sched: [1:0.50]
-; BTVER2-NEXT:    rorw $7, %di # sched: [1:0.50]
-; BTVER2-NEXT:    rolw $7, (%rdx) # sched: [4:1.00]
-; BTVER2-NEXT:    rorw $7, (%rdx) # sched: [4:1.00]
-; BTVER2-NEXT:    rolw %cl, %di # sched: [1:0.50]
-; BTVER2-NEXT:    rorw %cl, %di # sched: [1:0.50]
-; BTVER2-NEXT:    rolw %cl, (%rdx) # sched: [4:1.00]
-; BTVER2-NEXT:    rorw %cl, (%rdx) # sched: [4:1.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_rol_ror_16:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    rolw %di # sched: [1:0.25]
-; ZNVER1-NEXT:    rorw %di # sched: [1:0.25]
-; ZNVER1-NEXT:    rolw (%rdx) # sched: [5:1.00]
-; ZNVER1-NEXT:    rorw (%rdx) # sched: [5:1.00]
-; ZNVER1-NEXT:    rolw $7, %di # sched: [1:0.25]
-; ZNVER1-NEXT:    rorw $7, %di # sched: [1:0.25]
-; ZNVER1-NEXT:    rolw $7, (%rdx) # sched: [5:1.00]
-; ZNVER1-NEXT:    rorw $7, (%rdx) # sched: [5:1.00]
-; ZNVER1-NEXT:    rolw %cl, %di # sched: [1:0.25]
-; ZNVER1-NEXT:    rorw %cl, %di # sched: [1:0.25]
-; ZNVER1-NEXT:    rolw %cl, (%rdx) # sched: [5:1.00]
-; ZNVER1-NEXT:    rorw %cl, (%rdx) # sched: [5:1.00]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  call void asm sideeffect "rolw $0 \0A\09 rorw $0 \0A\09 rolw $2 \0A\09 rorw $2 \0A\09 rolw $3, $0 \0A\09 rorw $3, $0 \0A\09 rolw $3, $2 \0A\09 rorw $3, $2 \0A\09 rolw %CL, $0 \0A\09 rorw %CL, $0 \0A\09 rolw %CL, $2 \0A\09 rorw %CL, $2", "r,r,*m,i"(i16 %a0, i16 %a1, i16 *%a2, i8 7)
-  ret void
-}
-define void @test_rol_ror_32(i32 %a0, i32 %a1, i32 *%a2) optsize {
-; GENERIC-LABEL: test_rol_ror_32:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    roll %edi # sched: [2:1.00]
-; GENERIC-NEXT:    rorl %edi # sched: [2:1.00]
-; GENERIC-NEXT:    roll (%rdx) # sched: [8:1.00]
-; GENERIC-NEXT:    rorl (%rdx) # sched: [8:1.00]
-; GENERIC-NEXT:    roll $7, %edi # sched: [2:1.00]
-; GENERIC-NEXT:    rorl $7, %edi # sched: [2:1.00]
-; GENERIC-NEXT:    roll $7, (%rdx) # sched: [8:1.00]
-; GENERIC-NEXT:    rorl $7, (%rdx) # sched: [8:1.00]
-; GENERIC-NEXT:    roll %cl, %edi # sched: [3:1.50]
-; GENERIC-NEXT:    rorl %cl, %edi # sched: [3:1.50]
-; GENERIC-NEXT:    roll %cl, (%rdx) # sched: [9:1.50]
-; GENERIC-NEXT:    rorl %cl, (%rdx) # sched: [9:1.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_rol_ror_32:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    roll %edi # sched: [1:1.00]
-; ATOM-NEXT:    rorl %edi # sched: [1:1.00]
-; ATOM-NEXT:    roll (%rdx) # sched: [1:1.00]
-; ATOM-NEXT:    rorl (%rdx) # sched: [1:1.00]
-; ATOM-NEXT:    roll $7, %edi # sched: [1:1.00]
-; ATOM-NEXT:    rorl $7, %edi # sched: [1:1.00]
-; ATOM-NEXT:    roll $7, (%rdx) # sched: [1:1.00]
-; ATOM-NEXT:    rorl $7, (%rdx) # sched: [1:1.00]
-; ATOM-NEXT:    roll %cl, %edi # sched: [1:1.00]
-; ATOM-NEXT:    rorl %cl, %edi # sched: [1:1.00]
-; ATOM-NEXT:    roll %cl, (%rdx) # sched: [1:1.00]
-; ATOM-NEXT:    rorl %cl, (%rdx) # sched: [1:1.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_rol_ror_32:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    roll %edi # sched: [1:1.00]
-; SLM-NEXT:    rorl %edi # sched: [1:1.00]
-; SLM-NEXT:    roll (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    rorl (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    roll $7, %edi # sched: [1:1.00]
-; SLM-NEXT:    rorl $7, %edi # sched: [1:1.00]
-; SLM-NEXT:    roll $7, (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    rorl $7, (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    roll %cl, %edi # sched: [1:1.00]
-; SLM-NEXT:    rorl %cl, %edi # sched: [1:1.00]
-; SLM-NEXT:    roll %cl, (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    rorl %cl, (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_rol_ror_32:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    roll %edi # sched: [2:1.00]
-; SANDY-NEXT:    rorl %edi # sched: [2:1.00]
-; SANDY-NEXT:    roll (%rdx) # sched: [8:1.00]
-; SANDY-NEXT:    rorl (%rdx) # sched: [8:1.00]
-; SANDY-NEXT:    roll $7, %edi # sched: [2:1.00]
-; SANDY-NEXT:    rorl $7, %edi # sched: [2:1.00]
-; SANDY-NEXT:    roll $7, (%rdx) # sched: [8:1.00]
-; SANDY-NEXT:    rorl $7, (%rdx) # sched: [8:1.00]
-; SANDY-NEXT:    roll %cl, %edi # sched: [3:1.50]
-; SANDY-NEXT:    rorl %cl, %edi # sched: [3:1.50]
-; SANDY-NEXT:    roll %cl, (%rdx) # sched: [9:1.50]
-; SANDY-NEXT:    rorl %cl, (%rdx) # sched: [9:1.50]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_rol_ror_32:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    roll %edi # sched: [2:1.00]
-; HASWELL-NEXT:    rorl %edi # sched: [2:1.00]
-; HASWELL-NEXT:    roll (%rdx) # sched: [8:1.00]
-; HASWELL-NEXT:    rorl (%rdx) # sched: [8:1.00]
-; HASWELL-NEXT:    roll $7, %edi # sched: [2:1.00]
-; HASWELL-NEXT:    rorl $7, %edi # sched: [2:1.00]
-; HASWELL-NEXT:    roll $7, (%rdx) # sched: [8:1.00]
-; HASWELL-NEXT:    rorl $7, (%rdx) # sched: [8:1.00]
-; HASWELL-NEXT:    roll %cl, %edi # sched: [3:1.00]
-; HASWELL-NEXT:    rorl %cl, %edi # sched: [3:1.00]
-; HASWELL-NEXT:    roll %cl, (%rdx) # sched: [9:1.00]
-; HASWELL-NEXT:    rorl %cl, (%rdx) # sched: [9:1.00]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_rol_ror_32:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    roll %edi # sched: [2:1.00]
-; BROADWELL-NEXT:    rorl %edi # sched: [2:1.00]
-; BROADWELL-NEXT:    roll (%rdx) # sched: [7:1.00]
-; BROADWELL-NEXT:    rorl (%rdx) # sched: [7:1.00]
-; BROADWELL-NEXT:    roll $7, %edi # sched: [2:1.00]
-; BROADWELL-NEXT:    rorl $7, %edi # sched: [2:1.00]
-; BROADWELL-NEXT:    roll $7, (%rdx) # sched: [7:1.00]
-; BROADWELL-NEXT:    rorl $7, (%rdx) # sched: [7:1.00]
-; BROADWELL-NEXT:    roll %cl, %edi # sched: [3:1.00]
-; BROADWELL-NEXT:    rorl %cl, %edi # sched: [3:1.00]
-; BROADWELL-NEXT:    roll %cl, (%rdx) # sched: [8:1.00]
-; BROADWELL-NEXT:    rorl %cl, (%rdx) # sched: [8:1.00]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_rol_ror_32:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    roll %edi # sched: [2:1.00]
-; SKYLAKE-NEXT:    rorl %edi # sched: [2:1.00]
-; SKYLAKE-NEXT:    roll (%rdx) # sched: [7:1.00]
-; SKYLAKE-NEXT:    rorl (%rdx) # sched: [7:1.00]
-; SKYLAKE-NEXT:    roll $7, %edi # sched: [2:1.00]
-; SKYLAKE-NEXT:    rorl $7, %edi # sched: [2:1.00]
-; SKYLAKE-NEXT:    roll $7, (%rdx) # sched: [7:1.00]
-; SKYLAKE-NEXT:    rorl $7, (%rdx) # sched: [7:1.00]
-; SKYLAKE-NEXT:    roll %cl, %edi # sched: [3:1.50]
-; SKYLAKE-NEXT:    rorl %cl, %edi # sched: [3:1.50]
-; SKYLAKE-NEXT:    roll %cl, (%rdx) # sched: [8:1.50]
-; SKYLAKE-NEXT:    rorl %cl, (%rdx) # sched: [8:1.50]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_rol_ror_32:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    roll %edi # sched: [2:1.00]
-; SKX-NEXT:    rorl %edi # sched: [2:1.00]
-; SKX-NEXT:    roll (%rdx) # sched: [7:1.00]
-; SKX-NEXT:    rorl (%rdx) # sched: [7:1.00]
-; SKX-NEXT:    roll $7, %edi # sched: [2:1.00]
-; SKX-NEXT:    rorl $7, %edi # sched: [2:1.00]
-; SKX-NEXT:    roll $7, (%rdx) # sched: [7:1.00]
-; SKX-NEXT:    rorl $7, (%rdx) # sched: [7:1.00]
-; SKX-NEXT:    roll %cl, %edi # sched: [3:1.50]
-; SKX-NEXT:    rorl %cl, %edi # sched: [3:1.50]
-; SKX-NEXT:    roll %cl, (%rdx) # sched: [8:1.50]
-; SKX-NEXT:    rorl %cl, (%rdx) # sched: [8:1.50]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_rol_ror_32:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    roll %edi # sched: [1:0.50]
-; BDVER2-NEXT:    rorl %edi # sched: [1:0.50]
-; BDVER2-NEXT:    roll (%rdx) # sched: [5:1.00]
-; BDVER2-NEXT:    rorl (%rdx) # sched: [5:1.00]
-; BDVER2-NEXT:    roll $7, %edi # sched: [1:0.50]
-; BDVER2-NEXT:    rorl $7, %edi # sched: [1:0.50]
-; BDVER2-NEXT:    roll $7, (%rdx) # sched: [5:1.00]
-; BDVER2-NEXT:    rorl $7, (%rdx) # sched: [5:1.00]
-; BDVER2-NEXT:    roll %cl, %edi # sched: [1:0.50]
-; BDVER2-NEXT:    rorl %cl, %edi # sched: [1:0.50]
-; BDVER2-NEXT:    roll %cl, (%rdx) # sched: [5:1.00]
-; BDVER2-NEXT:    rorl %cl, (%rdx) # sched: [5:1.00]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_rol_ror_32:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    roll %edi # sched: [1:0.50]
-; BTVER2-NEXT:    rorl %edi # sched: [1:0.50]
-; BTVER2-NEXT:    roll (%rdx) # sched: [4:1.00]
-; BTVER2-NEXT:    rorl (%rdx) # sched: [4:1.00]
-; BTVER2-NEXT:    roll $7, %edi # sched: [1:0.50]
-; BTVER2-NEXT:    rorl $7, %edi # sched: [1:0.50]
-; BTVER2-NEXT:    roll $7, (%rdx) # sched: [4:1.00]
-; BTVER2-NEXT:    rorl $7, (%rdx) # sched: [4:1.00]
-; BTVER2-NEXT:    roll %cl, %edi # sched: [1:0.50]
-; BTVER2-NEXT:    rorl %cl, %edi # sched: [1:0.50]
-; BTVER2-NEXT:    roll %cl, (%rdx) # sched: [4:1.00]
-; BTVER2-NEXT:    rorl %cl, (%rdx) # sched: [4:1.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_rol_ror_32:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    roll %edi # sched: [1:0.25]
-; ZNVER1-NEXT:    rorl %edi # sched: [1:0.25]
-; ZNVER1-NEXT:    roll (%rdx) # sched: [5:1.00]
-; ZNVER1-NEXT:    rorl (%rdx) # sched: [5:1.00]
-; ZNVER1-NEXT:    roll $7, %edi # sched: [1:0.25]
-; ZNVER1-NEXT:    rorl $7, %edi # sched: [1:0.25]
-; ZNVER1-NEXT:    roll $7, (%rdx) # sched: [5:1.00]
-; ZNVER1-NEXT:    rorl $7, (%rdx) # sched: [5:1.00]
-; ZNVER1-NEXT:    roll %cl, %edi # sched: [1:0.25]
-; ZNVER1-NEXT:    rorl %cl, %edi # sched: [1:0.25]
-; ZNVER1-NEXT:    roll %cl, (%rdx) # sched: [5:1.00]
-; ZNVER1-NEXT:    rorl %cl, (%rdx) # sched: [5:1.00]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  call void asm sideeffect "roll $0 \0A\09 rorl $0 \0A\09 roll $2 \0A\09 rorl $2 \0A\09 roll $3, $0 \0A\09 rorl $3, $0 \0A\09 roll $3, $2 \0A\09 rorl $3, $2 \0A\09 roll %CL, $0 \0A\09 rorl %CL, $0 \0A\09 roll %CL, $2 \0A\09 rorl %CL, $2", "r,r,*m,i"(i32 %a0, i32 %a1, i32 *%a2, i8 7)
-  ret void
-}
-define void @test_rol_ror_64(i64 %a0, i64 %a1, i64 *%a2) optsize {
-; GENERIC-LABEL: test_rol_ror_64:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    rolq %rdi # sched: [2:1.00]
-; GENERIC-NEXT:    rorq %rdi # sched: [2:1.00]
-; GENERIC-NEXT:    rolq (%rdx) # sched: [8:1.00]
-; GENERIC-NEXT:    rorq (%rdx) # sched: [8:1.00]
-; GENERIC-NEXT:    rolq $7, %rdi # sched: [2:1.00]
-; GENERIC-NEXT:    rorq $7, %rdi # sched: [2:1.00]
-; GENERIC-NEXT:    rolq $7, (%rdx) # sched: [8:1.00]
-; GENERIC-NEXT:    rorq $7, (%rdx) # sched: [8:1.00]
-; GENERIC-NEXT:    rolq %cl, %rdi # sched: [3:1.50]
-; GENERIC-NEXT:    rorq %cl, %rdi # sched: [3:1.50]
-; GENERIC-NEXT:    rolq %cl, (%rdx) # sched: [9:1.50]
-; GENERIC-NEXT:    rorq %cl, (%rdx) # sched: [9:1.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_rol_ror_64:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    rolq %rdi # sched: [1:1.00]
-; ATOM-NEXT:    rorq %rdi # sched: [1:1.00]
-; ATOM-NEXT:    rolq (%rdx) # sched: [1:1.00]
-; ATOM-NEXT:    rorq (%rdx) # sched: [1:1.00]
-; ATOM-NEXT:    rolq $7, %rdi # sched: [1:1.00]
-; ATOM-NEXT:    rorq $7, %rdi # sched: [1:1.00]
-; ATOM-NEXT:    rolq $7, (%rdx) # sched: [1:1.00]
-; ATOM-NEXT:    rorq $7, (%rdx) # sched: [1:1.00]
-; ATOM-NEXT:    rolq %cl, %rdi # sched: [1:1.00]
-; ATOM-NEXT:    rorq %cl, %rdi # sched: [1:1.00]
-; ATOM-NEXT:    rolq %cl, (%rdx) # sched: [1:1.00]
-; ATOM-NEXT:    rorq %cl, (%rdx) # sched: [1:1.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_rol_ror_64:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    rolq %rdi # sched: [1:1.00]
-; SLM-NEXT:    rorq %rdi # sched: [1:1.00]
-; SLM-NEXT:    rolq (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    rorq (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    rolq $7, %rdi # sched: [1:1.00]
-; SLM-NEXT:    rorq $7, %rdi # sched: [1:1.00]
-; SLM-NEXT:    rolq $7, (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    rorq $7, (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    rolq %cl, %rdi # sched: [1:1.00]
-; SLM-NEXT:    rorq %cl, %rdi # sched: [1:1.00]
-; SLM-NEXT:    rolq %cl, (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    rorq %cl, (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_rol_ror_64:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    rolq %rdi # sched: [2:1.00]
-; SANDY-NEXT:    rorq %rdi # sched: [2:1.00]
-; SANDY-NEXT:    rolq (%rdx) # sched: [8:1.00]
-; SANDY-NEXT:    rorq (%rdx) # sched: [8:1.00]
-; SANDY-NEXT:    rolq $7, %rdi # sched: [2:1.00]
-; SANDY-NEXT:    rorq $7, %rdi # sched: [2:1.00]
-; SANDY-NEXT:    rolq $7, (%rdx) # sched: [8:1.00]
-; SANDY-NEXT:    rorq $7, (%rdx) # sched: [8:1.00]
-; SANDY-NEXT:    rolq %cl, %rdi # sched: [3:1.50]
-; SANDY-NEXT:    rorq %cl, %rdi # sched: [3:1.50]
-; SANDY-NEXT:    rolq %cl, (%rdx) # sched: [9:1.50]
-; SANDY-NEXT:    rorq %cl, (%rdx) # sched: [9:1.50]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_rol_ror_64:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    rolq %rdi # sched: [2:1.00]
-; HASWELL-NEXT:    rorq %rdi # sched: [2:1.00]
-; HASWELL-NEXT:    rolq (%rdx) # sched: [8:1.00]
-; HASWELL-NEXT:    rorq (%rdx) # sched: [8:1.00]
-; HASWELL-NEXT:    rolq $7, %rdi # sched: [2:1.00]
-; HASWELL-NEXT:    rorq $7, %rdi # sched: [2:1.00]
-; HASWELL-NEXT:    rolq $7, (%rdx) # sched: [8:1.00]
-; HASWELL-NEXT:    rorq $7, (%rdx) # sched: [8:1.00]
-; HASWELL-NEXT:    rolq %cl, %rdi # sched: [3:1.00]
-; HASWELL-NEXT:    rorq %cl, %rdi # sched: [3:1.00]
-; HASWELL-NEXT:    rolq %cl, (%rdx) # sched: [9:1.00]
-; HASWELL-NEXT:    rorq %cl, (%rdx) # sched: [9:1.00]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_rol_ror_64:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    rolq %rdi # sched: [2:1.00]
-; BROADWELL-NEXT:    rorq %rdi # sched: [2:1.00]
-; BROADWELL-NEXT:    rolq (%rdx) # sched: [7:1.00]
-; BROADWELL-NEXT:    rorq (%rdx) # sched: [7:1.00]
-; BROADWELL-NEXT:    rolq $7, %rdi # sched: [2:1.00]
-; BROADWELL-NEXT:    rorq $7, %rdi # sched: [2:1.00]
-; BROADWELL-NEXT:    rolq $7, (%rdx) # sched: [7:1.00]
-; BROADWELL-NEXT:    rorq $7, (%rdx) # sched: [7:1.00]
-; BROADWELL-NEXT:    rolq %cl, %rdi # sched: [3:1.00]
-; BROADWELL-NEXT:    rorq %cl, %rdi # sched: [3:1.00]
-; BROADWELL-NEXT:    rolq %cl, (%rdx) # sched: [8:1.00]
-; BROADWELL-NEXT:    rorq %cl, (%rdx) # sched: [8:1.00]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_rol_ror_64:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    rolq %rdi # sched: [2:1.00]
-; SKYLAKE-NEXT:    rorq %rdi # sched: [2:1.00]
-; SKYLAKE-NEXT:    rolq (%rdx) # sched: [7:1.00]
-; SKYLAKE-NEXT:    rorq (%rdx) # sched: [7:1.00]
-; SKYLAKE-NEXT:    rolq $7, %rdi # sched: [2:1.00]
-; SKYLAKE-NEXT:    rorq $7, %rdi # sched: [2:1.00]
-; SKYLAKE-NEXT:    rolq $7, (%rdx) # sched: [7:1.00]
-; SKYLAKE-NEXT:    rorq $7, (%rdx) # sched: [7:1.00]
-; SKYLAKE-NEXT:    rolq %cl, %rdi # sched: [3:1.50]
-; SKYLAKE-NEXT:    rorq %cl, %rdi # sched: [3:1.50]
-; SKYLAKE-NEXT:    rolq %cl, (%rdx) # sched: [8:1.50]
-; SKYLAKE-NEXT:    rorq %cl, (%rdx) # sched: [8:1.50]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_rol_ror_64:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    rolq %rdi # sched: [2:1.00]
-; SKX-NEXT:    rorq %rdi # sched: [2:1.00]
-; SKX-NEXT:    rolq (%rdx) # sched: [7:1.00]
-; SKX-NEXT:    rorq (%rdx) # sched: [7:1.00]
-; SKX-NEXT:    rolq $7, %rdi # sched: [2:1.00]
-; SKX-NEXT:    rorq $7, %rdi # sched: [2:1.00]
-; SKX-NEXT:    rolq $7, (%rdx) # sched: [7:1.00]
-; SKX-NEXT:    rorq $7, (%rdx) # sched: [7:1.00]
-; SKX-NEXT:    rolq %cl, %rdi # sched: [3:1.50]
-; SKX-NEXT:    rorq %cl, %rdi # sched: [3:1.50]
-; SKX-NEXT:    rolq %cl, (%rdx) # sched: [8:1.50]
-; SKX-NEXT:    rorq %cl, (%rdx) # sched: [8:1.50]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_rol_ror_64:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    rolq %rdi # sched: [1:0.50]
-; BDVER2-NEXT:    rorq %rdi # sched: [1:0.50]
-; BDVER2-NEXT:    rolq (%rdx) # sched: [5:1.00]
-; BDVER2-NEXT:    rorq (%rdx) # sched: [5:1.00]
-; BDVER2-NEXT:    rolq $7, %rdi # sched: [1:0.50]
-; BDVER2-NEXT:    rorq $7, %rdi # sched: [1:0.50]
-; BDVER2-NEXT:    rolq $7, (%rdx) # sched: [5:1.00]
-; BDVER2-NEXT:    rorq $7, (%rdx) # sched: [5:1.00]
-; BDVER2-NEXT:    rolq %cl, %rdi # sched: [1:0.50]
-; BDVER2-NEXT:    rorq %cl, %rdi # sched: [1:0.50]
-; BDVER2-NEXT:    rolq %cl, (%rdx) # sched: [5:1.00]
-; BDVER2-NEXT:    rorq %cl, (%rdx) # sched: [5:1.00]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_rol_ror_64:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    rolq %rdi # sched: [1:0.50]
-; BTVER2-NEXT:    rorq %rdi # sched: [1:0.50]
-; BTVER2-NEXT:    rolq (%rdx) # sched: [4:1.00]
-; BTVER2-NEXT:    rorq (%rdx) # sched: [4:1.00]
-; BTVER2-NEXT:    rolq $7, %rdi # sched: [1:0.50]
-; BTVER2-NEXT:    rorq $7, %rdi # sched: [1:0.50]
-; BTVER2-NEXT:    rolq $7, (%rdx) # sched: [4:1.00]
-; BTVER2-NEXT:    rorq $7, (%rdx) # sched: [4:1.00]
-; BTVER2-NEXT:    rolq %cl, %rdi # sched: [1:0.50]
-; BTVER2-NEXT:    rorq %cl, %rdi # sched: [1:0.50]
-; BTVER2-NEXT:    rolq %cl, (%rdx) # sched: [4:1.00]
-; BTVER2-NEXT:    rorq %cl, (%rdx) # sched: [4:1.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_rol_ror_64:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    rolq %rdi # sched: [1:0.25]
-; ZNVER1-NEXT:    rorq %rdi # sched: [1:0.25]
-; ZNVER1-NEXT:    rolq (%rdx) # sched: [5:1.00]
-; ZNVER1-NEXT:    rorq (%rdx) # sched: [5:1.00]
-; ZNVER1-NEXT:    rolq $7, %rdi # sched: [1:0.25]
-; ZNVER1-NEXT:    rorq $7, %rdi # sched: [1:0.25]
-; ZNVER1-NEXT:    rolq $7, (%rdx) # sched: [5:1.00]
-; ZNVER1-NEXT:    rorq $7, (%rdx) # sched: [5:1.00]
-; ZNVER1-NEXT:    rolq %cl, %rdi # sched: [1:0.25]
-; ZNVER1-NEXT:    rorq %cl, %rdi # sched: [1:0.25]
-; ZNVER1-NEXT:    rolq %cl, (%rdx) # sched: [5:1.00]
-; ZNVER1-NEXT:    rorq %cl, (%rdx) # sched: [5:1.00]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  call void asm sideeffect "rolq $0 \0A\09 rorq $0 \0A\09 rolq $2 \0A\09 rorq $2 \0A\09 rolq $3, $0 \0A\09 rorq $3, $0 \0A\09 rolq $3, $2 \0A\09 rorq $3, $2 \0A\09 rolq %CL, $0 \0A\09 rorq %CL, $0 \0A\09 rolq %CL, $2 \0A\09 rorq %CL, $2", "r,r,*m,i"(i64 %a0, i64 %a1, i64 *%a2, i8 7)
-  ret void
-}
-
-define void @test_sar_shl_shr_8(i8 %a0, i8 %a1, i8 *%a2) optsize {
-; GENERIC-LABEL: test_sar_shl_shr_8:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    sarb %dil # sched: [1:0.50]
-; GENERIC-NEXT:    shlb %dil # sched: [1:0.50]
-; GENERIC-NEXT:    shrb %dil # sched: [1:0.50]
-; GENERIC-NEXT:    sarb (%rdx) # sched: [7:1.00]
-; GENERIC-NEXT:    shlb (%rdx) # sched: [7:1.00]
-; GENERIC-NEXT:    shrb (%rdx) # sched: [7:1.00]
-; GENERIC-NEXT:    sarb $7, %dil # sched: [1:0.50]
-; GENERIC-NEXT:    shlb $7, %dil # sched: [1:0.50]
-; GENERIC-NEXT:    shrb $7, %dil # sched: [1:0.50]
-; GENERIC-NEXT:    sarb $7, (%rdx) # sched: [7:1.00]
-; GENERIC-NEXT:    shlb $7, (%rdx) # sched: [7:1.00]
-; GENERIC-NEXT:    shrb $7, (%rdx) # sched: [7:1.00]
-; GENERIC-NEXT:    sarb %cl, %dil # sched: [3:1.50]
-; GENERIC-NEXT:    shlb %cl, %dil # sched: [3:1.50]
-; GENERIC-NEXT:    shrb %cl, %dil # sched: [3:1.50]
-; GENERIC-NEXT:    sarb %cl, (%rdx) # sched: [9:1.50]
-; GENERIC-NEXT:    shlb %cl, (%rdx) # sched: [9:1.50]
-; GENERIC-NEXT:    shrb %cl, (%rdx) # sched: [9:1.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_sar_shl_shr_8:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    sarb %dil # sched: [1:1.00]
-; ATOM-NEXT:    shlb %dil # sched: [1:1.00]
-; ATOM-NEXT:    shrb %dil # sched: [1:1.00]
-; ATOM-NEXT:    sarb (%rdx) # sched: [1:1.00]
-; ATOM-NEXT:    shlb (%rdx) # sched: [1:1.00]
-; ATOM-NEXT:    shrb (%rdx) # sched: [1:1.00]
-; ATOM-NEXT:    sarb $7, %dil # sched: [1:1.00]
-; ATOM-NEXT:    shlb $7, %dil # sched: [1:1.00]
-; ATOM-NEXT:    shrb $7, %dil # sched: [1:1.00]
-; ATOM-NEXT:    sarb $7, (%rdx) # sched: [1:1.00]
-; ATOM-NEXT:    shlb $7, (%rdx) # sched: [1:1.00]
-; ATOM-NEXT:    shrb $7, (%rdx) # sched: [1:1.00]
-; ATOM-NEXT:    sarb %cl, %dil # sched: [1:1.00]
-; ATOM-NEXT:    shlb %cl, %dil # sched: [1:1.00]
-; ATOM-NEXT:    shrb %cl, %dil # sched: [1:1.00]
-; ATOM-NEXT:    sarb %cl, (%rdx) # sched: [1:1.00]
-; ATOM-NEXT:    shlb %cl, (%rdx) # sched: [1:1.00]
-; ATOM-NEXT:    shrb %cl, (%rdx) # sched: [1:1.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_sar_shl_shr_8:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    sarb %dil # sched: [1:1.00]
-; SLM-NEXT:    shlb %dil # sched: [1:1.00]
-; SLM-NEXT:    shrb %dil # sched: [1:1.00]
-; SLM-NEXT:    sarb (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    shlb (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    shrb (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    sarb $7, %dil # sched: [1:1.00]
-; SLM-NEXT:    shlb $7, %dil # sched: [1:1.00]
-; SLM-NEXT:    shrb $7, %dil # sched: [1:1.00]
-; SLM-NEXT:    sarb $7, (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    shlb $7, (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    shrb $7, (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    sarb %cl, %dil # sched: [1:1.00]
-; SLM-NEXT:    shlb %cl, %dil # sched: [1:1.00]
-; SLM-NEXT:    shrb %cl, %dil # sched: [1:1.00]
-; SLM-NEXT:    sarb %cl, (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    shlb %cl, (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    shrb %cl, (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_sar_shl_shr_8:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    sarb %dil # sched: [1:0.50]
-; SANDY-NEXT:    shlb %dil # sched: [1:0.50]
-; SANDY-NEXT:    shrb %dil # sched: [1:0.50]
-; SANDY-NEXT:    sarb (%rdx) # sched: [7:1.00]
-; SANDY-NEXT:    shlb (%rdx) # sched: [7:1.00]
-; SANDY-NEXT:    shrb (%rdx) # sched: [7:1.00]
-; SANDY-NEXT:    sarb $7, %dil # sched: [1:0.50]
-; SANDY-NEXT:    shlb $7, %dil # sched: [1:0.50]
-; SANDY-NEXT:    shrb $7, %dil # sched: [1:0.50]
-; SANDY-NEXT:    sarb $7, (%rdx) # sched: [7:1.00]
-; SANDY-NEXT:    shlb $7, (%rdx) # sched: [7:1.00]
-; SANDY-NEXT:    shrb $7, (%rdx) # sched: [7:1.00]
-; SANDY-NEXT:    sarb %cl, %dil # sched: [3:1.50]
-; SANDY-NEXT:    shlb %cl, %dil # sched: [3:1.50]
-; SANDY-NEXT:    shrb %cl, %dil # sched: [3:1.50]
-; SANDY-NEXT:    sarb %cl, (%rdx) # sched: [9:1.50]
-; SANDY-NEXT:    shlb %cl, (%rdx) # sched: [9:1.50]
-; SANDY-NEXT:    shrb %cl, (%rdx) # sched: [9:1.50]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_sar_shl_shr_8:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    sarb %dil # sched: [1:0.50]
-; HASWELL-NEXT:    shlb %dil # sched: [1:0.50]
-; HASWELL-NEXT:    shrb %dil # sched: [1:0.50]
-; HASWELL-NEXT:    sarb (%rdx) # sched: [7:1.00]
-; HASWELL-NEXT:    shlb (%rdx) # sched: [7:1.00]
-; HASWELL-NEXT:    shrb (%rdx) # sched: [7:1.00]
-; HASWELL-NEXT:    sarb $7, %dil # sched: [1:0.50]
-; HASWELL-NEXT:    shlb $7, %dil # sched: [1:0.50]
-; HASWELL-NEXT:    shrb $7, %dil # sched: [1:0.50]
-; HASWELL-NEXT:    sarb $7, (%rdx) # sched: [7:1.00]
-; HASWELL-NEXT:    shlb $7, (%rdx) # sched: [7:1.00]
-; HASWELL-NEXT:    shrb $7, (%rdx) # sched: [7:1.00]
-; HASWELL-NEXT:    sarb %cl, %dil # sched: [3:1.00]
-; HASWELL-NEXT:    shlb %cl, %dil # sched: [3:1.00]
-; HASWELL-NEXT:    shrb %cl, %dil # sched: [3:1.00]
-; HASWELL-NEXT:    sarb %cl, (%rdx) # sched: [9:1.00]
-; HASWELL-NEXT:    shlb %cl, (%rdx) # sched: [9:1.00]
-; HASWELL-NEXT:    shrb %cl, (%rdx) # sched: [9:1.00]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_sar_shl_shr_8:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    sarb %dil # sched: [1:0.50]
-; BROADWELL-NEXT:    shlb %dil # sched: [1:0.50]
-; BROADWELL-NEXT:    shrb %dil # sched: [1:0.50]
-; BROADWELL-NEXT:    sarb (%rdx) # sched: [6:1.00]
-; BROADWELL-NEXT:    shlb (%rdx) # sched: [6:1.00]
-; BROADWELL-NEXT:    shrb (%rdx) # sched: [6:1.00]
-; BROADWELL-NEXT:    sarb $7, %dil # sched: [1:0.50]
-; BROADWELL-NEXT:    shlb $7, %dil # sched: [1:0.50]
-; BROADWELL-NEXT:    shrb $7, %dil # sched: [1:0.50]
-; BROADWELL-NEXT:    sarb $7, (%rdx) # sched: [6:1.00]
-; BROADWELL-NEXT:    shlb $7, (%rdx) # sched: [6:1.00]
-; BROADWELL-NEXT:    shrb $7, (%rdx) # sched: [6:1.00]
-; BROADWELL-NEXT:    sarb %cl, %dil # sched: [3:1.00]
-; BROADWELL-NEXT:    shlb %cl, %dil # sched: [3:1.00]
-; BROADWELL-NEXT:    shrb %cl, %dil # sched: [3:1.00]
-; BROADWELL-NEXT:    sarb %cl, (%rdx) # sched: [8:1.00]
-; BROADWELL-NEXT:    shlb %cl, (%rdx) # sched: [8:1.00]
-; BROADWELL-NEXT:    shrb %cl, (%rdx) # sched: [8:1.00]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_sar_shl_shr_8:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    sarb %dil # sched: [1:0.50]
-; SKYLAKE-NEXT:    shlb %dil # sched: [1:0.50]
-; SKYLAKE-NEXT:    shrb %dil # sched: [1:0.50]
-; SKYLAKE-NEXT:    sarb (%rdx) # sched: [6:1.00]
-; SKYLAKE-NEXT:    shlb (%rdx) # sched: [6:1.00]
-; SKYLAKE-NEXT:    shrb (%rdx) # sched: [6:1.00]
-; SKYLAKE-NEXT:    sarb $7, %dil # sched: [1:0.50]
-; SKYLAKE-NEXT:    shlb $7, %dil # sched: [1:0.50]
-; SKYLAKE-NEXT:    shrb $7, %dil # sched: [1:0.50]
-; SKYLAKE-NEXT:    sarb $7, (%rdx) # sched: [6:1.00]
-; SKYLAKE-NEXT:    shlb $7, (%rdx) # sched: [6:1.00]
-; SKYLAKE-NEXT:    shrb $7, (%rdx) # sched: [6:1.00]
-; SKYLAKE-NEXT:    sarb %cl, %dil # sched: [3:1.50]
-; SKYLAKE-NEXT:    shlb %cl, %dil # sched: [3:1.50]
-; SKYLAKE-NEXT:    shrb %cl, %dil # sched: [3:1.50]
-; SKYLAKE-NEXT:    sarb %cl, (%rdx) # sched: [8:1.50]
-; SKYLAKE-NEXT:    shlb %cl, (%rdx) # sched: [8:1.50]
-; SKYLAKE-NEXT:    shrb %cl, (%rdx) # sched: [8:1.50]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_sar_shl_shr_8:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    sarb %dil # sched: [1:0.50]
-; SKX-NEXT:    shlb %dil # sched: [1:0.50]
-; SKX-NEXT:    shrb %dil # sched: [1:0.50]
-; SKX-NEXT:    sarb (%rdx) # sched: [6:1.00]
-; SKX-NEXT:    shlb (%rdx) # sched: [6:1.00]
-; SKX-NEXT:    shrb (%rdx) # sched: [6:1.00]
-; SKX-NEXT:    sarb $7, %dil # sched: [1:0.50]
-; SKX-NEXT:    shlb $7, %dil # sched: [1:0.50]
-; SKX-NEXT:    shrb $7, %dil # sched: [1:0.50]
-; SKX-NEXT:    sarb $7, (%rdx) # sched: [6:1.00]
-; SKX-NEXT:    shlb $7, (%rdx) # sched: [6:1.00]
-; SKX-NEXT:    shrb $7, (%rdx) # sched: [6:1.00]
-; SKX-NEXT:    sarb %cl, %dil # sched: [3:1.50]
-; SKX-NEXT:    shlb %cl, %dil # sched: [3:1.50]
-; SKX-NEXT:    shrb %cl, %dil # sched: [3:1.50]
-; SKX-NEXT:    sarb %cl, (%rdx) # sched: [8:1.50]
-; SKX-NEXT:    shlb %cl, (%rdx) # sched: [8:1.50]
-; SKX-NEXT:    shrb %cl, (%rdx) # sched: [8:1.50]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_sar_shl_shr_8:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    sarb %dil # sched: [1:0.50]
-; BDVER2-NEXT:    shlb %dil # sched: [1:0.50]
-; BDVER2-NEXT:    shrb %dil # sched: [1:0.50]
-; BDVER2-NEXT:    sarb (%rdx) # sched: [5:1.00]
-; BDVER2-NEXT:    shlb (%rdx) # sched: [5:1.00]
-; BDVER2-NEXT:    shrb (%rdx) # sched: [5:1.00]
-; BDVER2-NEXT:    sarb $7, %dil # sched: [1:0.50]
-; BDVER2-NEXT:    shlb $7, %dil # sched: [1:0.50]
-; BDVER2-NEXT:    shrb $7, %dil # sched: [1:0.50]
-; BDVER2-NEXT:    sarb $7, (%rdx) # sched: [5:1.00]
-; BDVER2-NEXT:    shlb $7, (%rdx) # sched: [5:1.00]
-; BDVER2-NEXT:    shrb $7, (%rdx) # sched: [5:1.00]
-; BDVER2-NEXT:    sarb %cl, %dil # sched: [1:0.50]
-; BDVER2-NEXT:    shlb %cl, %dil # sched: [1:0.50]
-; BDVER2-NEXT:    shrb %cl, %dil # sched: [1:0.50]
-; BDVER2-NEXT:    sarb %cl, (%rdx) # sched: [5:1.00]
-; BDVER2-NEXT:    shlb %cl, (%rdx) # sched: [5:1.00]
-; BDVER2-NEXT:    shrb %cl, (%rdx) # sched: [5:1.00]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_sar_shl_shr_8:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    sarb %dil # sched: [1:0.50]
-; BTVER2-NEXT:    shlb %dil # sched: [1:0.50]
-; BTVER2-NEXT:    shrb %dil # sched: [1:0.50]
-; BTVER2-NEXT:    sarb (%rdx) # sched: [4:1.00]
-; BTVER2-NEXT:    shlb (%rdx) # sched: [4:1.00]
-; BTVER2-NEXT:    shrb (%rdx) # sched: [4:1.00]
-; BTVER2-NEXT:    sarb $7, %dil # sched: [1:0.50]
-; BTVER2-NEXT:    shlb $7, %dil # sched: [1:0.50]
-; BTVER2-NEXT:    shrb $7, %dil # sched: [1:0.50]
-; BTVER2-NEXT:    sarb $7, (%rdx) # sched: [4:1.00]
-; BTVER2-NEXT:    shlb $7, (%rdx) # sched: [4:1.00]
-; BTVER2-NEXT:    shrb $7, (%rdx) # sched: [4:1.00]
-; BTVER2-NEXT:    sarb %cl, %dil # sched: [1:0.50]
-; BTVER2-NEXT:    shlb %cl, %dil # sched: [1:0.50]
-; BTVER2-NEXT:    shrb %cl, %dil # sched: [1:0.50]
-; BTVER2-NEXT:    sarb %cl, (%rdx) # sched: [4:1.00]
-; BTVER2-NEXT:    shlb %cl, (%rdx) # sched: [4:1.00]
-; BTVER2-NEXT:    shrb %cl, (%rdx) # sched: [4:1.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_sar_shl_shr_8:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    sarb %dil # sched: [1:0.25]
-; ZNVER1-NEXT:    shlb %dil # sched: [1:0.25]
-; ZNVER1-NEXT:    shrb %dil # sched: [1:0.25]
-; ZNVER1-NEXT:    sarb (%rdx) # sched: [5:0.50]
-; ZNVER1-NEXT:    shlb (%rdx) # sched: [5:0.50]
-; ZNVER1-NEXT:    shrb (%rdx) # sched: [5:0.50]
-; ZNVER1-NEXT:    sarb $7, %dil # sched: [1:0.25]
-; ZNVER1-NEXT:    shlb $7, %dil # sched: [1:0.25]
-; ZNVER1-NEXT:    shrb $7, %dil # sched: [1:0.25]
-; ZNVER1-NEXT:    sarb $7, (%rdx) # sched: [5:0.50]
-; ZNVER1-NEXT:    shlb $7, (%rdx) # sched: [5:0.50]
-; ZNVER1-NEXT:    shrb $7, (%rdx) # sched: [5:0.50]
-; ZNVER1-NEXT:    sarb %cl, %dil # sched: [1:0.25]
-; ZNVER1-NEXT:    shlb %cl, %dil # sched: [1:0.25]
-; ZNVER1-NEXT:    shrb %cl, %dil # sched: [1:0.25]
-; ZNVER1-NEXT:    sarb %cl, (%rdx) # sched: [5:1.00]
-; ZNVER1-NEXT:    shlb %cl, (%rdx) # sched: [5:1.00]
-; ZNVER1-NEXT:    shrb %cl, (%rdx) # sched: [5:1.00]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  call void asm sideeffect "sarb $0 \0A\09 shlb $0 \0A\09 shrb $0 \0A\09 sarb $2 \0A\09 shlb $2 \0A\09 shrb $2 \0A\09 sarb $3, $0 \0A\09 shlb $3, $0 \0A\09 shrb $3, $0 \0A\09 sarb $3, $2 \0A\09 shlb $3, $2 \0A\09 shrb $3, $2 \0A\09 sarb %CL, $0 \0A\09 shlb %CL, $0 \0A\09 shrb %CL, $0 \0A\09 sarb %CL, $2  \0A\09 shlb %CL, $2 \0A\09 shrb %CL, $2", "r,r,*m,i"(i8 %a0, i8 %a1, i8 *%a2, i8 7)
-  ret void
-}
-define void @test_sar_shl_shr_16(i16 %a0, i16 %a1, i16 *%a2) optsize {
-; GENERIC-LABEL: test_sar_shl_shr_16:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    sarw %di # sched: [1:0.50]
-; GENERIC-NEXT:    shlw %di # sched: [1:0.50]
-; GENERIC-NEXT:    shrw %di # sched: [1:0.50]
-; GENERIC-NEXT:    sarw (%rdx) # sched: [7:1.00]
-; GENERIC-NEXT:    shlw (%rdx) # sched: [7:1.00]
-; GENERIC-NEXT:    shrw (%rdx) # sched: [7:1.00]
-; GENERIC-NEXT:    sarw $7, %di # sched: [1:0.50]
-; GENERIC-NEXT:    shlw $7, %di # sched: [1:0.50]
-; GENERIC-NEXT:    shrw $7, %di # sched: [1:0.50]
-; GENERIC-NEXT:    sarw $7, (%rdx) # sched: [7:1.00]
-; GENERIC-NEXT:    shlw $7, (%rdx) # sched: [7:1.00]
-; GENERIC-NEXT:    shrw $7, (%rdx) # sched: [7:1.00]
-; GENERIC-NEXT:    sarw %cl, %di # sched: [3:1.50]
-; GENERIC-NEXT:    shlw %cl, %di # sched: [3:1.50]
-; GENERIC-NEXT:    shrw %cl, %di # sched: [3:1.50]
-; GENERIC-NEXT:    sarw %cl, (%rdx) # sched: [9:1.50]
-; GENERIC-NEXT:    shlw %cl, (%rdx) # sched: [9:1.50]
-; GENERIC-NEXT:    shrw %cl, (%rdx) # sched: [9:1.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_sar_shl_shr_16:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    sarw %di # sched: [1:1.00]
-; ATOM-NEXT:    shlw %di # sched: [1:1.00]
-; ATOM-NEXT:    shrw %di # sched: [1:1.00]
-; ATOM-NEXT:    sarw (%rdx) # sched: [1:1.00]
-; ATOM-NEXT:    shlw (%rdx) # sched: [1:1.00]
-; ATOM-NEXT:    shrw (%rdx) # sched: [1:1.00]
-; ATOM-NEXT:    sarw $7, %di # sched: [1:1.00]
-; ATOM-NEXT:    shlw $7, %di # sched: [1:1.00]
-; ATOM-NEXT:    shrw $7, %di # sched: [1:1.00]
-; ATOM-NEXT:    sarw $7, (%rdx) # sched: [1:1.00]
-; ATOM-NEXT:    shlw $7, (%rdx) # sched: [1:1.00]
-; ATOM-NEXT:    shrw $7, (%rdx) # sched: [1:1.00]
-; ATOM-NEXT:    sarw %cl, %di # sched: [1:1.00]
-; ATOM-NEXT:    shlw %cl, %di # sched: [1:1.00]
-; ATOM-NEXT:    shrw %cl, %di # sched: [1:1.00]
-; ATOM-NEXT:    sarw %cl, (%rdx) # sched: [1:1.00]
-; ATOM-NEXT:    shlw %cl, (%rdx) # sched: [1:1.00]
-; ATOM-NEXT:    shrw %cl, (%rdx) # sched: [1:1.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_sar_shl_shr_16:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    sarw %di # sched: [1:1.00]
-; SLM-NEXT:    shlw %di # sched: [1:1.00]
-; SLM-NEXT:    shrw %di # sched: [1:1.00]
-; SLM-NEXT:    sarw (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    shlw (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    shrw (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    sarw $7, %di # sched: [1:1.00]
-; SLM-NEXT:    shlw $7, %di # sched: [1:1.00]
-; SLM-NEXT:    shrw $7, %di # sched: [1:1.00]
-; SLM-NEXT:    sarw $7, (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    shlw $7, (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    shrw $7, (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    sarw %cl, %di # sched: [1:1.00]
-; SLM-NEXT:    shlw %cl, %di # sched: [1:1.00]
-; SLM-NEXT:    shrw %cl, %di # sched: [1:1.00]
-; SLM-NEXT:    sarw %cl, (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    shlw %cl, (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    shrw %cl, (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_sar_shl_shr_16:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    sarw %di # sched: [1:0.50]
-; SANDY-NEXT:    shlw %di # sched: [1:0.50]
-; SANDY-NEXT:    shrw %di # sched: [1:0.50]
-; SANDY-NEXT:    sarw (%rdx) # sched: [7:1.00]
-; SANDY-NEXT:    shlw (%rdx) # sched: [7:1.00]
-; SANDY-NEXT:    shrw (%rdx) # sched: [7:1.00]
-; SANDY-NEXT:    sarw $7, %di # sched: [1:0.50]
-; SANDY-NEXT:    shlw $7, %di # sched: [1:0.50]
-; SANDY-NEXT:    shrw $7, %di # sched: [1:0.50]
-; SANDY-NEXT:    sarw $7, (%rdx) # sched: [7:1.00]
-; SANDY-NEXT:    shlw $7, (%rdx) # sched: [7:1.00]
-; SANDY-NEXT:    shrw $7, (%rdx) # sched: [7:1.00]
-; SANDY-NEXT:    sarw %cl, %di # sched: [3:1.50]
-; SANDY-NEXT:    shlw %cl, %di # sched: [3:1.50]
-; SANDY-NEXT:    shrw %cl, %di # sched: [3:1.50]
-; SANDY-NEXT:    sarw %cl, (%rdx) # sched: [9:1.50]
-; SANDY-NEXT:    shlw %cl, (%rdx) # sched: [9:1.50]
-; SANDY-NEXT:    shrw %cl, (%rdx) # sched: [9:1.50]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_sar_shl_shr_16:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    sarw %di # sched: [1:0.50]
-; HASWELL-NEXT:    shlw %di # sched: [1:0.50]
-; HASWELL-NEXT:    shrw %di # sched: [1:0.50]
-; HASWELL-NEXT:    sarw (%rdx) # sched: [7:1.00]
-; HASWELL-NEXT:    shlw (%rdx) # sched: [7:1.00]
-; HASWELL-NEXT:    shrw (%rdx) # sched: [7:1.00]
-; HASWELL-NEXT:    sarw $7, %di # sched: [1:0.50]
-; HASWELL-NEXT:    shlw $7, %di # sched: [1:0.50]
-; HASWELL-NEXT:    shrw $7, %di # sched: [1:0.50]
-; HASWELL-NEXT:    sarw $7, (%rdx) # sched: [7:1.00]
-; HASWELL-NEXT:    shlw $7, (%rdx) # sched: [7:1.00]
-; HASWELL-NEXT:    shrw $7, (%rdx) # sched: [7:1.00]
-; HASWELL-NEXT:    sarw %cl, %di # sched: [3:1.00]
-; HASWELL-NEXT:    shlw %cl, %di # sched: [3:1.00]
-; HASWELL-NEXT:    shrw %cl, %di # sched: [3:1.00]
-; HASWELL-NEXT:    sarw %cl, (%rdx) # sched: [9:1.00]
-; HASWELL-NEXT:    shlw %cl, (%rdx) # sched: [9:1.00]
-; HASWELL-NEXT:    shrw %cl, (%rdx) # sched: [9:1.00]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_sar_shl_shr_16:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    sarw %di # sched: [1:0.50]
-; BROADWELL-NEXT:    shlw %di # sched: [1:0.50]
-; BROADWELL-NEXT:    shrw %di # sched: [1:0.50]
-; BROADWELL-NEXT:    sarw (%rdx) # sched: [6:1.00]
-; BROADWELL-NEXT:    shlw (%rdx) # sched: [6:1.00]
-; BROADWELL-NEXT:    shrw (%rdx) # sched: [6:1.00]
-; BROADWELL-NEXT:    sarw $7, %di # sched: [1:0.50]
-; BROADWELL-NEXT:    shlw $7, %di # sched: [1:0.50]
-; BROADWELL-NEXT:    shrw $7, %di # sched: [1:0.50]
-; BROADWELL-NEXT:    sarw $7, (%rdx) # sched: [6:1.00]
-; BROADWELL-NEXT:    shlw $7, (%rdx) # sched: [6:1.00]
-; BROADWELL-NEXT:    shrw $7, (%rdx) # sched: [6:1.00]
-; BROADWELL-NEXT:    sarw %cl, %di # sched: [3:1.00]
-; BROADWELL-NEXT:    shlw %cl, %di # sched: [3:1.00]
-; BROADWELL-NEXT:    shrw %cl, %di # sched: [3:1.00]
-; BROADWELL-NEXT:    sarw %cl, (%rdx) # sched: [8:1.00]
-; BROADWELL-NEXT:    shlw %cl, (%rdx) # sched: [8:1.00]
-; BROADWELL-NEXT:    shrw %cl, (%rdx) # sched: [8:1.00]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_sar_shl_shr_16:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    sarw %di # sched: [1:0.50]
-; SKYLAKE-NEXT:    shlw %di # sched: [1:0.50]
-; SKYLAKE-NEXT:    shrw %di # sched: [1:0.50]
-; SKYLAKE-NEXT:    sarw (%rdx) # sched: [6:1.00]
-; SKYLAKE-NEXT:    shlw (%rdx) # sched: [6:1.00]
-; SKYLAKE-NEXT:    shrw (%rdx) # sched: [6:1.00]
-; SKYLAKE-NEXT:    sarw $7, %di # sched: [1:0.50]
-; SKYLAKE-NEXT:    shlw $7, %di # sched: [1:0.50]
-; SKYLAKE-NEXT:    shrw $7, %di # sched: [1:0.50]
-; SKYLAKE-NEXT:    sarw $7, (%rdx) # sched: [6:1.00]
-; SKYLAKE-NEXT:    shlw $7, (%rdx) # sched: [6:1.00]
-; SKYLAKE-NEXT:    shrw $7, (%rdx) # sched: [6:1.00]
-; SKYLAKE-NEXT:    sarw %cl, %di # sched: [3:1.50]
-; SKYLAKE-NEXT:    shlw %cl, %di # sched: [3:1.50]
-; SKYLAKE-NEXT:    shrw %cl, %di # sched: [3:1.50]
-; SKYLAKE-NEXT:    sarw %cl, (%rdx) # sched: [8:1.50]
-; SKYLAKE-NEXT:    shlw %cl, (%rdx) # sched: [8:1.50]
-; SKYLAKE-NEXT:    shrw %cl, (%rdx) # sched: [8:1.50]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_sar_shl_shr_16:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    sarw %di # sched: [1:0.50]
-; SKX-NEXT:    shlw %di # sched: [1:0.50]
-; SKX-NEXT:    shrw %di # sched: [1:0.50]
-; SKX-NEXT:    sarw (%rdx) # sched: [6:1.00]
-; SKX-NEXT:    shlw (%rdx) # sched: [6:1.00]
-; SKX-NEXT:    shrw (%rdx) # sched: [6:1.00]
-; SKX-NEXT:    sarw $7, %di # sched: [1:0.50]
-; SKX-NEXT:    shlw $7, %di # sched: [1:0.50]
-; SKX-NEXT:    shrw $7, %di # sched: [1:0.50]
-; SKX-NEXT:    sarw $7, (%rdx) # sched: [6:1.00]
-; SKX-NEXT:    shlw $7, (%rdx) # sched: [6:1.00]
-; SKX-NEXT:    shrw $7, (%rdx) # sched: [6:1.00]
-; SKX-NEXT:    sarw %cl, %di # sched: [3:1.50]
-; SKX-NEXT:    shlw %cl, %di # sched: [3:1.50]
-; SKX-NEXT:    shrw %cl, %di # sched: [3:1.50]
-; SKX-NEXT:    sarw %cl, (%rdx) # sched: [8:1.50]
-; SKX-NEXT:    shlw %cl, (%rdx) # sched: [8:1.50]
-; SKX-NEXT:    shrw %cl, (%rdx) # sched: [8:1.50]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_sar_shl_shr_16:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    sarw %di # sched: [1:0.50]
-; BDVER2-NEXT:    shlw %di # sched: [1:0.50]
-; BDVER2-NEXT:    shrw %di # sched: [1:0.50]
-; BDVER2-NEXT:    sarw (%rdx) # sched: [5:1.00]
-; BDVER2-NEXT:    shlw (%rdx) # sched: [5:1.00]
-; BDVER2-NEXT:    shrw (%rdx) # sched: [5:1.00]
-; BDVER2-NEXT:    sarw $7, %di # sched: [1:0.50]
-; BDVER2-NEXT:    shlw $7, %di # sched: [1:0.50]
-; BDVER2-NEXT:    shrw $7, %di # sched: [1:0.50]
-; BDVER2-NEXT:    sarw $7, (%rdx) # sched: [5:1.00]
-; BDVER2-NEXT:    shlw $7, (%rdx) # sched: [5:1.00]
-; BDVER2-NEXT:    shrw $7, (%rdx) # sched: [5:1.00]
-; BDVER2-NEXT:    sarw %cl, %di # sched: [1:0.50]
-; BDVER2-NEXT:    shlw %cl, %di # sched: [1:0.50]
-; BDVER2-NEXT:    shrw %cl, %di # sched: [1:0.50]
-; BDVER2-NEXT:    sarw %cl, (%rdx) # sched: [5:1.00]
-; BDVER2-NEXT:    shlw %cl, (%rdx) # sched: [5:1.00]
-; BDVER2-NEXT:    shrw %cl, (%rdx) # sched: [5:1.00]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_sar_shl_shr_16:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    sarw %di # sched: [1:0.50]
-; BTVER2-NEXT:    shlw %di # sched: [1:0.50]
-; BTVER2-NEXT:    shrw %di # sched: [1:0.50]
-; BTVER2-NEXT:    sarw (%rdx) # sched: [4:1.00]
-; BTVER2-NEXT:    shlw (%rdx) # sched: [4:1.00]
-; BTVER2-NEXT:    shrw (%rdx) # sched: [4:1.00]
-; BTVER2-NEXT:    sarw $7, %di # sched: [1:0.50]
-; BTVER2-NEXT:    shlw $7, %di # sched: [1:0.50]
-; BTVER2-NEXT:    shrw $7, %di # sched: [1:0.50]
-; BTVER2-NEXT:    sarw $7, (%rdx) # sched: [4:1.00]
-; BTVER2-NEXT:    shlw $7, (%rdx) # sched: [4:1.00]
-; BTVER2-NEXT:    shrw $7, (%rdx) # sched: [4:1.00]
-; BTVER2-NEXT:    sarw %cl, %di # sched: [1:0.50]
-; BTVER2-NEXT:    shlw %cl, %di # sched: [1:0.50]
-; BTVER2-NEXT:    shrw %cl, %di # sched: [1:0.50]
-; BTVER2-NEXT:    sarw %cl, (%rdx) # sched: [4:1.00]
-; BTVER2-NEXT:    shlw %cl, (%rdx) # sched: [4:1.00]
-; BTVER2-NEXT:    shrw %cl, (%rdx) # sched: [4:1.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_sar_shl_shr_16:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    sarw %di # sched: [1:0.25]
-; ZNVER1-NEXT:    shlw %di # sched: [1:0.25]
-; ZNVER1-NEXT:    shrw %di # sched: [1:0.25]
-; ZNVER1-NEXT:    sarw (%rdx) # sched: [5:0.50]
-; ZNVER1-NEXT:    shlw (%rdx) # sched: [5:0.50]
-; ZNVER1-NEXT:    shrw (%rdx) # sched: [5:0.50]
-; ZNVER1-NEXT:    sarw $7, %di # sched: [1:0.25]
-; ZNVER1-NEXT:    shlw $7, %di # sched: [1:0.25]
-; ZNVER1-NEXT:    shrw $7, %di # sched: [1:0.25]
-; ZNVER1-NEXT:    sarw $7, (%rdx) # sched: [5:0.50]
-; ZNVER1-NEXT:    shlw $7, (%rdx) # sched: [5:0.50]
-; ZNVER1-NEXT:    shrw $7, (%rdx) # sched: [5:0.50]
-; ZNVER1-NEXT:    sarw %cl, %di # sched: [1:0.25]
-; ZNVER1-NEXT:    shlw %cl, %di # sched: [1:0.25]
-; ZNVER1-NEXT:    shrw %cl, %di # sched: [1:0.25]
-; ZNVER1-NEXT:    sarw %cl, (%rdx) # sched: [5:1.00]
-; ZNVER1-NEXT:    shlw %cl, (%rdx) # sched: [5:1.00]
-; ZNVER1-NEXT:    shrw %cl, (%rdx) # sched: [5:1.00]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  call void asm sideeffect "sarw $0 \0A\09 shlw $0 \0A\09 shrw $0 \0A\09 sarw $2 \0A\09 shlw $2 \0A\09 shrw $2 \0A\09 sarw $3, $0 \0A\09 shlw $3, $0 \0A\09 shrw $3, $0 \0A\09 sarw $3, $2 \0A\09 shlw $3, $2 \0A\09 shrw $3, $2 \0A\09 sarw %CL, $0 \0A\09 shlw %CL, $0 \0A\09 shrw %CL, $0 \0A\09 sarw %CL, $2  \0A\09 shlw %CL, $2 \0A\09 shrw %CL, $2", "r,r,*m,i"(i16 %a0, i16 %a1, i16 *%a2, i8 7)
-  ret void
-}
-define void @test_sar_shl_shr_32(i32 %a0, i32 %a1, i32 *%a2) optsize {
-; GENERIC-LABEL: test_sar_shl_shr_32:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    sarl %edi # sched: [1:0.50]
-; GENERIC-NEXT:    shll %edi # sched: [1:0.50]
-; GENERIC-NEXT:    shrl %edi # sched: [1:0.50]
-; GENERIC-NEXT:    sarl (%rdx) # sched: [7:1.00]
-; GENERIC-NEXT:    shll (%rdx) # sched: [7:1.00]
-; GENERIC-NEXT:    shrl (%rdx) # sched: [7:1.00]
-; GENERIC-NEXT:    sarl $7, %edi # sched: [1:0.50]
-; GENERIC-NEXT:    shll $7, %edi # sched: [1:0.50]
-; GENERIC-NEXT:    shrl $7, %edi # sched: [1:0.50]
-; GENERIC-NEXT:    sarl $7, (%rdx) # sched: [7:1.00]
-; GENERIC-NEXT:    shll $7, (%rdx) # sched: [7:1.00]
-; GENERIC-NEXT:    shrl $7, (%rdx) # sched: [7:1.00]
-; GENERIC-NEXT:    sarl %cl, %edi # sched: [3:1.50]
-; GENERIC-NEXT:    shll %cl, %edi # sched: [3:1.50]
-; GENERIC-NEXT:    shrl %cl, %edi # sched: [3:1.50]
-; GENERIC-NEXT:    sarl %cl, (%rdx) # sched: [9:1.50]
-; GENERIC-NEXT:    shll %cl, (%rdx) # sched: [9:1.50]
-; GENERIC-NEXT:    shrl %cl, (%rdx) # sched: [9:1.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_sar_shl_shr_32:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    sarl %edi # sched: [1:1.00]
-; ATOM-NEXT:    shll %edi # sched: [1:1.00]
-; ATOM-NEXT:    shrl %edi # sched: [1:1.00]
-; ATOM-NEXT:    sarl (%rdx) # sched: [1:1.00]
-; ATOM-NEXT:    shll (%rdx) # sched: [1:1.00]
-; ATOM-NEXT:    shrl (%rdx) # sched: [1:1.00]
-; ATOM-NEXT:    sarl $7, %edi # sched: [1:1.00]
-; ATOM-NEXT:    shll $7, %edi # sched: [1:1.00]
-; ATOM-NEXT:    shrl $7, %edi # sched: [1:1.00]
-; ATOM-NEXT:    sarl $7, (%rdx) # sched: [1:1.00]
-; ATOM-NEXT:    shll $7, (%rdx) # sched: [1:1.00]
-; ATOM-NEXT:    shrl $7, (%rdx) # sched: [1:1.00]
-; ATOM-NEXT:    sarl %cl, %edi # sched: [1:1.00]
-; ATOM-NEXT:    shll %cl, %edi # sched: [1:1.00]
-; ATOM-NEXT:    shrl %cl, %edi # sched: [1:1.00]
-; ATOM-NEXT:    sarl %cl, (%rdx) # sched: [1:1.00]
-; ATOM-NEXT:    shll %cl, (%rdx) # sched: [1:1.00]
-; ATOM-NEXT:    shrl %cl, (%rdx) # sched: [1:1.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_sar_shl_shr_32:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    sarl %edi # sched: [1:1.00]
-; SLM-NEXT:    shll %edi # sched: [1:1.00]
-; SLM-NEXT:    shrl %edi # sched: [1:1.00]
-; SLM-NEXT:    sarl (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    shll (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    shrl (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    sarl $7, %edi # sched: [1:1.00]
-; SLM-NEXT:    shll $7, %edi # sched: [1:1.00]
-; SLM-NEXT:    shrl $7, %edi # sched: [1:1.00]
-; SLM-NEXT:    sarl $7, (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    shll $7, (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    shrl $7, (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    sarl %cl, %edi # sched: [1:1.00]
-; SLM-NEXT:    shll %cl, %edi # sched: [1:1.00]
-; SLM-NEXT:    shrl %cl, %edi # sched: [1:1.00]
-; SLM-NEXT:    sarl %cl, (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    shll %cl, (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    shrl %cl, (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_sar_shl_shr_32:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    sarl %edi # sched: [1:0.50]
-; SANDY-NEXT:    shll %edi # sched: [1:0.50]
-; SANDY-NEXT:    shrl %edi # sched: [1:0.50]
-; SANDY-NEXT:    sarl (%rdx) # sched: [7:1.00]
-; SANDY-NEXT:    shll (%rdx) # sched: [7:1.00]
-; SANDY-NEXT:    shrl (%rdx) # sched: [7:1.00]
-; SANDY-NEXT:    sarl $7, %edi # sched: [1:0.50]
-; SANDY-NEXT:    shll $7, %edi # sched: [1:0.50]
-; SANDY-NEXT:    shrl $7, %edi # sched: [1:0.50]
-; SANDY-NEXT:    sarl $7, (%rdx) # sched: [7:1.00]
-; SANDY-NEXT:    shll $7, (%rdx) # sched: [7:1.00]
-; SANDY-NEXT:    shrl $7, (%rdx) # sched: [7:1.00]
-; SANDY-NEXT:    sarl %cl, %edi # sched: [3:1.50]
-; SANDY-NEXT:    shll %cl, %edi # sched: [3:1.50]
-; SANDY-NEXT:    shrl %cl, %edi # sched: [3:1.50]
-; SANDY-NEXT:    sarl %cl, (%rdx) # sched: [9:1.50]
-; SANDY-NEXT:    shll %cl, (%rdx) # sched: [9:1.50]
-; SANDY-NEXT:    shrl %cl, (%rdx) # sched: [9:1.50]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_sar_shl_shr_32:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    sarl %edi # sched: [1:0.50]
-; HASWELL-NEXT:    shll %edi # sched: [1:0.50]
-; HASWELL-NEXT:    shrl %edi # sched: [1:0.50]
-; HASWELL-NEXT:    sarl (%rdx) # sched: [7:1.00]
-; HASWELL-NEXT:    shll (%rdx) # sched: [7:1.00]
-; HASWELL-NEXT:    shrl (%rdx) # sched: [7:1.00]
-; HASWELL-NEXT:    sarl $7, %edi # sched: [1:0.50]
-; HASWELL-NEXT:    shll $7, %edi # sched: [1:0.50]
-; HASWELL-NEXT:    shrl $7, %edi # sched: [1:0.50]
-; HASWELL-NEXT:    sarl $7, (%rdx) # sched: [7:1.00]
-; HASWELL-NEXT:    shll $7, (%rdx) # sched: [7:1.00]
-; HASWELL-NEXT:    shrl $7, (%rdx) # sched: [7:1.00]
-; HASWELL-NEXT:    sarl %cl, %edi # sched: [3:1.00]
-; HASWELL-NEXT:    shll %cl, %edi # sched: [3:1.00]
-; HASWELL-NEXT:    shrl %cl, %edi # sched: [3:1.00]
-; HASWELL-NEXT:    sarl %cl, (%rdx) # sched: [9:1.00]
-; HASWELL-NEXT:    shll %cl, (%rdx) # sched: [9:1.00]
-; HASWELL-NEXT:    shrl %cl, (%rdx) # sched: [9:1.00]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_sar_shl_shr_32:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    sarl %edi # sched: [1:0.50]
-; BROADWELL-NEXT:    shll %edi # sched: [1:0.50]
-; BROADWELL-NEXT:    shrl %edi # sched: [1:0.50]
-; BROADWELL-NEXT:    sarl (%rdx) # sched: [6:1.00]
-; BROADWELL-NEXT:    shll (%rdx) # sched: [6:1.00]
-; BROADWELL-NEXT:    shrl (%rdx) # sched: [6:1.00]
-; BROADWELL-NEXT:    sarl $7, %edi # sched: [1:0.50]
-; BROADWELL-NEXT:    shll $7, %edi # sched: [1:0.50]
-; BROADWELL-NEXT:    shrl $7, %edi # sched: [1:0.50]
-; BROADWELL-NEXT:    sarl $7, (%rdx) # sched: [6:1.00]
-; BROADWELL-NEXT:    shll $7, (%rdx) # sched: [6:1.00]
-; BROADWELL-NEXT:    shrl $7, (%rdx) # sched: [6:1.00]
-; BROADWELL-NEXT:    sarl %cl, %edi # sched: [3:1.00]
-; BROADWELL-NEXT:    shll %cl, %edi # sched: [3:1.00]
-; BROADWELL-NEXT:    shrl %cl, %edi # sched: [3:1.00]
-; BROADWELL-NEXT:    sarl %cl, (%rdx) # sched: [8:1.00]
-; BROADWELL-NEXT:    shll %cl, (%rdx) # sched: [8:1.00]
-; BROADWELL-NEXT:    shrl %cl, (%rdx) # sched: [8:1.00]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_sar_shl_shr_32:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    sarl %edi # sched: [1:0.50]
-; SKYLAKE-NEXT:    shll %edi # sched: [1:0.50]
-; SKYLAKE-NEXT:    shrl %edi # sched: [1:0.50]
-; SKYLAKE-NEXT:    sarl (%rdx) # sched: [6:1.00]
-; SKYLAKE-NEXT:    shll (%rdx) # sched: [6:1.00]
-; SKYLAKE-NEXT:    shrl (%rdx) # sched: [6:1.00]
-; SKYLAKE-NEXT:    sarl $7, %edi # sched: [1:0.50]
-; SKYLAKE-NEXT:    shll $7, %edi # sched: [1:0.50]
-; SKYLAKE-NEXT:    shrl $7, %edi # sched: [1:0.50]
-; SKYLAKE-NEXT:    sarl $7, (%rdx) # sched: [6:1.00]
-; SKYLAKE-NEXT:    shll $7, (%rdx) # sched: [6:1.00]
-; SKYLAKE-NEXT:    shrl $7, (%rdx) # sched: [6:1.00]
-; SKYLAKE-NEXT:    sarl %cl, %edi # sched: [3:1.50]
-; SKYLAKE-NEXT:    shll %cl, %edi # sched: [3:1.50]
-; SKYLAKE-NEXT:    shrl %cl, %edi # sched: [3:1.50]
-; SKYLAKE-NEXT:    sarl %cl, (%rdx) # sched: [8:1.50]
-; SKYLAKE-NEXT:    shll %cl, (%rdx) # sched: [8:1.50]
-; SKYLAKE-NEXT:    shrl %cl, (%rdx) # sched: [8:1.50]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_sar_shl_shr_32:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    sarl %edi # sched: [1:0.50]
-; SKX-NEXT:    shll %edi # sched: [1:0.50]
-; SKX-NEXT:    shrl %edi # sched: [1:0.50]
-; SKX-NEXT:    sarl (%rdx) # sched: [6:1.00]
-; SKX-NEXT:    shll (%rdx) # sched: [6:1.00]
-; SKX-NEXT:    shrl (%rdx) # sched: [6:1.00]
-; SKX-NEXT:    sarl $7, %edi # sched: [1:0.50]
-; SKX-NEXT:    shll $7, %edi # sched: [1:0.50]
-; SKX-NEXT:    shrl $7, %edi # sched: [1:0.50]
-; SKX-NEXT:    sarl $7, (%rdx) # sched: [6:1.00]
-; SKX-NEXT:    shll $7, (%rdx) # sched: [6:1.00]
-; SKX-NEXT:    shrl $7, (%rdx) # sched: [6:1.00]
-; SKX-NEXT:    sarl %cl, %edi # sched: [3:1.50]
-; SKX-NEXT:    shll %cl, %edi # sched: [3:1.50]
-; SKX-NEXT:    shrl %cl, %edi # sched: [3:1.50]
-; SKX-NEXT:    sarl %cl, (%rdx) # sched: [8:1.50]
-; SKX-NEXT:    shll %cl, (%rdx) # sched: [8:1.50]
-; SKX-NEXT:    shrl %cl, (%rdx) # sched: [8:1.50]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_sar_shl_shr_32:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    sarl %edi # sched: [1:0.50]
-; BDVER2-NEXT:    shll %edi # sched: [1:0.50]
-; BDVER2-NEXT:    shrl %edi # sched: [1:0.50]
-; BDVER2-NEXT:    sarl (%rdx) # sched: [5:1.00]
-; BDVER2-NEXT:    shll (%rdx) # sched: [5:1.00]
-; BDVER2-NEXT:    shrl (%rdx) # sched: [5:1.00]
-; BDVER2-NEXT:    sarl $7, %edi # sched: [1:0.50]
-; BDVER2-NEXT:    shll $7, %edi # sched: [1:0.50]
-; BDVER2-NEXT:    shrl $7, %edi # sched: [1:0.50]
-; BDVER2-NEXT:    sarl $7, (%rdx) # sched: [5:1.00]
-; BDVER2-NEXT:    shll $7, (%rdx) # sched: [5:1.00]
-; BDVER2-NEXT:    shrl $7, (%rdx) # sched: [5:1.00]
-; BDVER2-NEXT:    sarl %cl, %edi # sched: [1:0.50]
-; BDVER2-NEXT:    shll %cl, %edi # sched: [1:0.50]
-; BDVER2-NEXT:    shrl %cl, %edi # sched: [1:0.50]
-; BDVER2-NEXT:    sarl %cl, (%rdx) # sched: [5:1.00]
-; BDVER2-NEXT:    shll %cl, (%rdx) # sched: [5:1.00]
-; BDVER2-NEXT:    shrl %cl, (%rdx) # sched: [5:1.00]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_sar_shl_shr_32:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    sarl %edi # sched: [1:0.50]
-; BTVER2-NEXT:    shll %edi # sched: [1:0.50]
-; BTVER2-NEXT:    shrl %edi # sched: [1:0.50]
-; BTVER2-NEXT:    sarl (%rdx) # sched: [4:1.00]
-; BTVER2-NEXT:    shll (%rdx) # sched: [4:1.00]
-; BTVER2-NEXT:    shrl (%rdx) # sched: [4:1.00]
-; BTVER2-NEXT:    sarl $7, %edi # sched: [1:0.50]
-; BTVER2-NEXT:    shll $7, %edi # sched: [1:0.50]
-; BTVER2-NEXT:    shrl $7, %edi # sched: [1:0.50]
-; BTVER2-NEXT:    sarl $7, (%rdx) # sched: [4:1.00]
-; BTVER2-NEXT:    shll $7, (%rdx) # sched: [4:1.00]
-; BTVER2-NEXT:    shrl $7, (%rdx) # sched: [4:1.00]
-; BTVER2-NEXT:    sarl %cl, %edi # sched: [1:0.50]
-; BTVER2-NEXT:    shll %cl, %edi # sched: [1:0.50]
-; BTVER2-NEXT:    shrl %cl, %edi # sched: [1:0.50]
-; BTVER2-NEXT:    sarl %cl, (%rdx) # sched: [4:1.00]
-; BTVER2-NEXT:    shll %cl, (%rdx) # sched: [4:1.00]
-; BTVER2-NEXT:    shrl %cl, (%rdx) # sched: [4:1.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_sar_shl_shr_32:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    sarl %edi # sched: [1:0.25]
-; ZNVER1-NEXT:    shll %edi # sched: [1:0.25]
-; ZNVER1-NEXT:    shrl %edi # sched: [1:0.25]
-; ZNVER1-NEXT:    sarl (%rdx) # sched: [5:0.50]
-; ZNVER1-NEXT:    shll (%rdx) # sched: [5:0.50]
-; ZNVER1-NEXT:    shrl (%rdx) # sched: [5:0.50]
-; ZNVER1-NEXT:    sarl $7, %edi # sched: [1:0.25]
-; ZNVER1-NEXT:    shll $7, %edi # sched: [1:0.25]
-; ZNVER1-NEXT:    shrl $7, %edi # sched: [1:0.25]
-; ZNVER1-NEXT:    sarl $7, (%rdx) # sched: [5:0.50]
-; ZNVER1-NEXT:    shll $7, (%rdx) # sched: [5:0.50]
-; ZNVER1-NEXT:    shrl $7, (%rdx) # sched: [5:0.50]
-; ZNVER1-NEXT:    sarl %cl, %edi # sched: [1:0.25]
-; ZNVER1-NEXT:    shll %cl, %edi # sched: [1:0.25]
-; ZNVER1-NEXT:    shrl %cl, %edi # sched: [1:0.25]
-; ZNVER1-NEXT:    sarl %cl, (%rdx) # sched: [5:1.00]
-; ZNVER1-NEXT:    shll %cl, (%rdx) # sched: [5:1.00]
-; ZNVER1-NEXT:    shrl %cl, (%rdx) # sched: [5:1.00]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  call void asm sideeffect "sarl $0 \0A\09 shll $0 \0A\09 shrl $0 \0A\09 sarl $2 \0A\09 shll $2 \0A\09 shrl $2 \0A\09 sarl $3, $0 \0A\09 shll $3, $0 \0A\09 shrl $3, $0 \0A\09 sarl $3, $2 \0A\09 shll $3, $2 \0A\09 shrl $3, $2 \0A\09 sarl %CL, $0 \0A\09 shll %CL, $0 \0A\09 shrl %CL, $0 \0A\09 sarl %CL, $2  \0A\09 shll %CL, $2 \0A\09 shrl %CL, $2", "r,r,*m,i"(i32 %a0, i32 %a1, i32 *%a2, i8 7)
-  ret void
-}
-define void @test_sar_shl_shr_64(i64 %a0, i64 %a1, i64 *%a2) optsize {
-; GENERIC-LABEL: test_sar_shl_shr_64:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    sarq %rdi # sched: [1:0.50]
-; GENERIC-NEXT:    shlq %rdi # sched: [1:0.50]
-; GENERIC-NEXT:    shrq %rdi # sched: [1:0.50]
-; GENERIC-NEXT:    sarq (%rdx) # sched: [7:1.00]
-; GENERIC-NEXT:    shlq (%rdx) # sched: [7:1.00]
-; GENERIC-NEXT:    shrq (%rdx) # sched: [7:1.00]
-; GENERIC-NEXT:    sarq $7, %rdi # sched: [1:0.50]
-; GENERIC-NEXT:    shlq $7, %rdi # sched: [1:0.50]
-; GENERIC-NEXT:    shrq $7, %rdi # sched: [1:0.50]
-; GENERIC-NEXT:    sarq $7, (%rdx) # sched: [7:1.00]
-; GENERIC-NEXT:    shlq $7, (%rdx) # sched: [7:1.00]
-; GENERIC-NEXT:    shrq $7, (%rdx) # sched: [7:1.00]
-; GENERIC-NEXT:    sarq %cl, %rdi # sched: [3:1.50]
-; GENERIC-NEXT:    shlq %cl, %rdi # sched: [3:1.50]
-; GENERIC-NEXT:    shrq %cl, %rdi # sched: [3:1.50]
-; GENERIC-NEXT:    sarq %cl, (%rdx) # sched: [9:1.50]
-; GENERIC-NEXT:    shlq %cl, (%rdx) # sched: [9:1.50]
-; GENERIC-NEXT:    shrq %cl, (%rdx) # sched: [9:1.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_sar_shl_shr_64:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    sarq %rdi # sched: [1:1.00]
-; ATOM-NEXT:    shlq %rdi # sched: [1:1.00]
-; ATOM-NEXT:    shrq %rdi # sched: [1:1.00]
-; ATOM-NEXT:    sarq (%rdx) # sched: [1:1.00]
-; ATOM-NEXT:    shlq (%rdx) # sched: [1:1.00]
-; ATOM-NEXT:    shrq (%rdx) # sched: [1:1.00]
-; ATOM-NEXT:    sarq $7, %rdi # sched: [1:1.00]
-; ATOM-NEXT:    shlq $7, %rdi # sched: [1:1.00]
-; ATOM-NEXT:    shrq $7, %rdi # sched: [1:1.00]
-; ATOM-NEXT:    sarq $7, (%rdx) # sched: [1:1.00]
-; ATOM-NEXT:    shlq $7, (%rdx) # sched: [1:1.00]
-; ATOM-NEXT:    shrq $7, (%rdx) # sched: [1:1.00]
-; ATOM-NEXT:    sarq %cl, %rdi # sched: [1:1.00]
-; ATOM-NEXT:    shlq %cl, %rdi # sched: [1:1.00]
-; ATOM-NEXT:    shrq %cl, %rdi # sched: [1:1.00]
-; ATOM-NEXT:    sarq %cl, (%rdx) # sched: [1:1.00]
-; ATOM-NEXT:    shlq %cl, (%rdx) # sched: [1:1.00]
-; ATOM-NEXT:    shrq %cl, (%rdx) # sched: [1:1.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_sar_shl_shr_64:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    sarq %rdi # sched: [1:1.00]
-; SLM-NEXT:    shlq %rdi # sched: [1:1.00]
-; SLM-NEXT:    shrq %rdi # sched: [1:1.00]
-; SLM-NEXT:    sarq (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    shlq (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    shrq (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    sarq $7, %rdi # sched: [1:1.00]
-; SLM-NEXT:    shlq $7, %rdi # sched: [1:1.00]
-; SLM-NEXT:    shrq $7, %rdi # sched: [1:1.00]
-; SLM-NEXT:    sarq $7, (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    shlq $7, (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    shrq $7, (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    sarq %cl, %rdi # sched: [1:1.00]
-; SLM-NEXT:    shlq %cl, %rdi # sched: [1:1.00]
-; SLM-NEXT:    shrq %cl, %rdi # sched: [1:1.00]
-; SLM-NEXT:    sarq %cl, (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    shlq %cl, (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    shrq %cl, (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_sar_shl_shr_64:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    sarq %rdi # sched: [1:0.50]
-; SANDY-NEXT:    shlq %rdi # sched: [1:0.50]
-; SANDY-NEXT:    shrq %rdi # sched: [1:0.50]
-; SANDY-NEXT:    sarq (%rdx) # sched: [7:1.00]
-; SANDY-NEXT:    shlq (%rdx) # sched: [7:1.00]
-; SANDY-NEXT:    shrq (%rdx) # sched: [7:1.00]
-; SANDY-NEXT:    sarq $7, %rdi # sched: [1:0.50]
-; SANDY-NEXT:    shlq $7, %rdi # sched: [1:0.50]
-; SANDY-NEXT:    shrq $7, %rdi # sched: [1:0.50]
-; SANDY-NEXT:    sarq $7, (%rdx) # sched: [7:1.00]
-; SANDY-NEXT:    shlq $7, (%rdx) # sched: [7:1.00]
-; SANDY-NEXT:    shrq $7, (%rdx) # sched: [7:1.00]
-; SANDY-NEXT:    sarq %cl, %rdi # sched: [3:1.50]
-; SANDY-NEXT:    shlq %cl, %rdi # sched: [3:1.50]
-; SANDY-NEXT:    shrq %cl, %rdi # sched: [3:1.50]
-; SANDY-NEXT:    sarq %cl, (%rdx) # sched: [9:1.50]
-; SANDY-NEXT:    shlq %cl, (%rdx) # sched: [9:1.50]
-; SANDY-NEXT:    shrq %cl, (%rdx) # sched: [9:1.50]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_sar_shl_shr_64:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    sarq %rdi # sched: [1:0.50]
-; HASWELL-NEXT:    shlq %rdi # sched: [1:0.50]
-; HASWELL-NEXT:    shrq %rdi # sched: [1:0.50]
-; HASWELL-NEXT:    sarq (%rdx) # sched: [7:1.00]
-; HASWELL-NEXT:    shlq (%rdx) # sched: [7:1.00]
-; HASWELL-NEXT:    shrq (%rdx) # sched: [7:1.00]
-; HASWELL-NEXT:    sarq $7, %rdi # sched: [1:0.50]
-; HASWELL-NEXT:    shlq $7, %rdi # sched: [1:0.50]
-; HASWELL-NEXT:    shrq $7, %rdi # sched: [1:0.50]
-; HASWELL-NEXT:    sarq $7, (%rdx) # sched: [7:1.00]
-; HASWELL-NEXT:    shlq $7, (%rdx) # sched: [7:1.00]
-; HASWELL-NEXT:    shrq $7, (%rdx) # sched: [7:1.00]
-; HASWELL-NEXT:    sarq %cl, %rdi # sched: [3:1.00]
-; HASWELL-NEXT:    shlq %cl, %rdi # sched: [3:1.00]
-; HASWELL-NEXT:    shrq %cl, %rdi # sched: [3:1.00]
-; HASWELL-NEXT:    sarq %cl, (%rdx) # sched: [9:1.00]
-; HASWELL-NEXT:    shlq %cl, (%rdx) # sched: [9:1.00]
-; HASWELL-NEXT:    shrq %cl, (%rdx) # sched: [9:1.00]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_sar_shl_shr_64:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    sarq %rdi # sched: [1:0.50]
-; BROADWELL-NEXT:    shlq %rdi # sched: [1:0.50]
-; BROADWELL-NEXT:    shrq %rdi # sched: [1:0.50]
-; BROADWELL-NEXT:    sarq (%rdx) # sched: [6:1.00]
-; BROADWELL-NEXT:    shlq (%rdx) # sched: [6:1.00]
-; BROADWELL-NEXT:    shrq (%rdx) # sched: [6:1.00]
-; BROADWELL-NEXT:    sarq $7, %rdi # sched: [1:0.50]
-; BROADWELL-NEXT:    shlq $7, %rdi # sched: [1:0.50]
-; BROADWELL-NEXT:    shrq $7, %rdi # sched: [1:0.50]
-; BROADWELL-NEXT:    sarq $7, (%rdx) # sched: [6:1.00]
-; BROADWELL-NEXT:    shlq $7, (%rdx) # sched: [6:1.00]
-; BROADWELL-NEXT:    shrq $7, (%rdx) # sched: [6:1.00]
-; BROADWELL-NEXT:    sarq %cl, %rdi # sched: [3:1.00]
-; BROADWELL-NEXT:    shlq %cl, %rdi # sched: [3:1.00]
-; BROADWELL-NEXT:    shrq %cl, %rdi # sched: [3:1.00]
-; BROADWELL-NEXT:    sarq %cl, (%rdx) # sched: [8:1.00]
-; BROADWELL-NEXT:    shlq %cl, (%rdx) # sched: [8:1.00]
-; BROADWELL-NEXT:    shrq %cl, (%rdx) # sched: [8:1.00]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_sar_shl_shr_64:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    sarq %rdi # sched: [1:0.50]
-; SKYLAKE-NEXT:    shlq %rdi # sched: [1:0.50]
-; SKYLAKE-NEXT:    shrq %rdi # sched: [1:0.50]
-; SKYLAKE-NEXT:    sarq (%rdx) # sched: [6:1.00]
-; SKYLAKE-NEXT:    shlq (%rdx) # sched: [6:1.00]
-; SKYLAKE-NEXT:    shrq (%rdx) # sched: [6:1.00]
-; SKYLAKE-NEXT:    sarq $7, %rdi # sched: [1:0.50]
-; SKYLAKE-NEXT:    shlq $7, %rdi # sched: [1:0.50]
-; SKYLAKE-NEXT:    shrq $7, %rdi # sched: [1:0.50]
-; SKYLAKE-NEXT:    sarq $7, (%rdx) # sched: [6:1.00]
-; SKYLAKE-NEXT:    shlq $7, (%rdx) # sched: [6:1.00]
-; SKYLAKE-NEXT:    shrq $7, (%rdx) # sched: [6:1.00]
-; SKYLAKE-NEXT:    sarq %cl, %rdi # sched: [3:1.50]
-; SKYLAKE-NEXT:    shlq %cl, %rdi # sched: [3:1.50]
-; SKYLAKE-NEXT:    shrq %cl, %rdi # sched: [3:1.50]
-; SKYLAKE-NEXT:    sarq %cl, (%rdx) # sched: [8:1.50]
-; SKYLAKE-NEXT:    shlq %cl, (%rdx) # sched: [8:1.50]
-; SKYLAKE-NEXT:    shrq %cl, (%rdx) # sched: [8:1.50]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_sar_shl_shr_64:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    sarq %rdi # sched: [1:0.50]
-; SKX-NEXT:    shlq %rdi # sched: [1:0.50]
-; SKX-NEXT:    shrq %rdi # sched: [1:0.50]
-; SKX-NEXT:    sarq (%rdx) # sched: [6:1.00]
-; SKX-NEXT:    shlq (%rdx) # sched: [6:1.00]
-; SKX-NEXT:    shrq (%rdx) # sched: [6:1.00]
-; SKX-NEXT:    sarq $7, %rdi # sched: [1:0.50]
-; SKX-NEXT:    shlq $7, %rdi # sched: [1:0.50]
-; SKX-NEXT:    shrq $7, %rdi # sched: [1:0.50]
-; SKX-NEXT:    sarq $7, (%rdx) # sched: [6:1.00]
-; SKX-NEXT:    shlq $7, (%rdx) # sched: [6:1.00]
-; SKX-NEXT:    shrq $7, (%rdx) # sched: [6:1.00]
-; SKX-NEXT:    sarq %cl, %rdi # sched: [3:1.50]
-; SKX-NEXT:    shlq %cl, %rdi # sched: [3:1.50]
-; SKX-NEXT:    shrq %cl, %rdi # sched: [3:1.50]
-; SKX-NEXT:    sarq %cl, (%rdx) # sched: [8:1.50]
-; SKX-NEXT:    shlq %cl, (%rdx) # sched: [8:1.50]
-; SKX-NEXT:    shrq %cl, (%rdx) # sched: [8:1.50]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_sar_shl_shr_64:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    sarq %rdi # sched: [1:0.50]
-; BDVER2-NEXT:    shlq %rdi # sched: [1:0.50]
-; BDVER2-NEXT:    shrq %rdi # sched: [1:0.50]
-; BDVER2-NEXT:    sarq (%rdx) # sched: [5:1.00]
-; BDVER2-NEXT:    shlq (%rdx) # sched: [5:1.00]
-; BDVER2-NEXT:    shrq (%rdx) # sched: [5:1.00]
-; BDVER2-NEXT:    sarq $7, %rdi # sched: [1:0.50]
-; BDVER2-NEXT:    shlq $7, %rdi # sched: [1:0.50]
-; BDVER2-NEXT:    shrq $7, %rdi # sched: [1:0.50]
-; BDVER2-NEXT:    sarq $7, (%rdx) # sched: [5:1.00]
-; BDVER2-NEXT:    shlq $7, (%rdx) # sched: [5:1.00]
-; BDVER2-NEXT:    shrq $7, (%rdx) # sched: [5:1.00]
-; BDVER2-NEXT:    sarq %cl, %rdi # sched: [1:0.50]
-; BDVER2-NEXT:    shlq %cl, %rdi # sched: [1:0.50]
-; BDVER2-NEXT:    shrq %cl, %rdi # sched: [1:0.50]
-; BDVER2-NEXT:    sarq %cl, (%rdx) # sched: [5:1.00]
-; BDVER2-NEXT:    shlq %cl, (%rdx) # sched: [5:1.00]
-; BDVER2-NEXT:    shrq %cl, (%rdx) # sched: [5:1.00]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_sar_shl_shr_64:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    sarq %rdi # sched: [1:0.50]
-; BTVER2-NEXT:    shlq %rdi # sched: [1:0.50]
-; BTVER2-NEXT:    shrq %rdi # sched: [1:0.50]
-; BTVER2-NEXT:    sarq (%rdx) # sched: [4:1.00]
-; BTVER2-NEXT:    shlq (%rdx) # sched: [4:1.00]
-; BTVER2-NEXT:    shrq (%rdx) # sched: [4:1.00]
-; BTVER2-NEXT:    sarq $7, %rdi # sched: [1:0.50]
-; BTVER2-NEXT:    shlq $7, %rdi # sched: [1:0.50]
-; BTVER2-NEXT:    shrq $7, %rdi # sched: [1:0.50]
-; BTVER2-NEXT:    sarq $7, (%rdx) # sched: [4:1.00]
-; BTVER2-NEXT:    shlq $7, (%rdx) # sched: [4:1.00]
-; BTVER2-NEXT:    shrq $7, (%rdx) # sched: [4:1.00]
-; BTVER2-NEXT:    sarq %cl, %rdi # sched: [1:0.50]
-; BTVER2-NEXT:    shlq %cl, %rdi # sched: [1:0.50]
-; BTVER2-NEXT:    shrq %cl, %rdi # sched: [1:0.50]
-; BTVER2-NEXT:    sarq %cl, (%rdx) # sched: [4:1.00]
-; BTVER2-NEXT:    shlq %cl, (%rdx) # sched: [4:1.00]
-; BTVER2-NEXT:    shrq %cl, (%rdx) # sched: [4:1.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_sar_shl_shr_64:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    sarq %rdi # sched: [1:0.25]
-; ZNVER1-NEXT:    shlq %rdi # sched: [1:0.25]
-; ZNVER1-NEXT:    shrq %rdi # sched: [1:0.25]
-; ZNVER1-NEXT:    sarq (%rdx) # sched: [5:0.50]
-; ZNVER1-NEXT:    shlq (%rdx) # sched: [5:0.50]
-; ZNVER1-NEXT:    shrq (%rdx) # sched: [5:0.50]
-; ZNVER1-NEXT:    sarq $7, %rdi # sched: [1:0.25]
-; ZNVER1-NEXT:    shlq $7, %rdi # sched: [1:0.25]
-; ZNVER1-NEXT:    shrq $7, %rdi # sched: [1:0.25]
-; ZNVER1-NEXT:    sarq $7, (%rdx) # sched: [5:0.50]
-; ZNVER1-NEXT:    shlq $7, (%rdx) # sched: [5:0.50]
-; ZNVER1-NEXT:    shrq $7, (%rdx) # sched: [5:0.50]
-; ZNVER1-NEXT:    sarq %cl, %rdi # sched: [1:0.25]
-; ZNVER1-NEXT:    shlq %cl, %rdi # sched: [1:0.25]
-; ZNVER1-NEXT:    shrq %cl, %rdi # sched: [1:0.25]
-; ZNVER1-NEXT:    sarq %cl, (%rdx) # sched: [5:1.00]
-; ZNVER1-NEXT:    shlq %cl, (%rdx) # sched: [5:1.00]
-; ZNVER1-NEXT:    shrq %cl, (%rdx) # sched: [5:1.00]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  call void asm sideeffect "sarq $0 \0A\09 shlq $0 \0A\09 shrq $0 \0A\09 sarq $2 \0A\09 shlq $2 \0A\09 shrq $2 \0A\09 sarq $3, $0 \0A\09 shlq $3, $0 \0A\09 shrq $3, $0 \0A\09 sarq $3, $2 \0A\09 shlq $3, $2 \0A\09 shrq $3, $2 \0A\09 sarq %CL, $0 \0A\09 shlq %CL, $0 \0A\09 shrq %CL, $0 \0A\09 sarq %CL, $2  \0A\09 shlq %CL, $2 \0A\09 shrq %CL, $2", "r,r,*m,i"(i64 %a0, i64 %a1, i64 *%a2, i8 7)
-  ret void
-}
-
-define void @test_sbb_8(i8 %a0, i8* %a1, i8 %a2) optsize {
-; GENERIC-LABEL: test_sbb_8:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    sbbb $7, %al # sched: [2:0.67]
-; GENERIC-NEXT:    sbbb $7, %dil # sched: [2:0.67]
-; GENERIC-NEXT:    sbbb $7, (%rsi) # sched: [9:1.00]
-; GENERIC-NEXT:    sbbb %dl, %dil # sched: [2:0.67]
-; GENERIC-NEXT:    sbbb %dil, (%rsi) # sched: [9:1.00]
-; GENERIC-NEXT:    sbbb (%rsi), %dil # sched: [7:0.67]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_sbb_8:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    sbbb $7, %al # sched: [1:0.50]
-; ATOM-NEXT:    sbbb $7, %dil # sched: [1:0.50]
-; ATOM-NEXT:    sbbb $7, (%rsi) # sched: [1:1.00]
-; ATOM-NEXT:    sbbb %dl, %dil # sched: [1:0.50]
-; ATOM-NEXT:    sbbb %dil, (%rsi) # sched: [1:1.00]
-; ATOM-NEXT:    sbbb (%rsi), %dil # sched: [1:1.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_sbb_8:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    sbbb $7, %al # sched: [1:0.50]
-; SLM-NEXT:    sbbb $7, %dil # sched: [1:0.50]
-; SLM-NEXT:    sbbb $7, (%rsi) # sched: [5:2.00]
-; SLM-NEXT:    sbbb %dl, %dil # sched: [1:0.50]
-; SLM-NEXT:    sbbb %dil, (%rsi) # sched: [5:2.00]
-; SLM-NEXT:    sbbb (%rsi), %dil # sched: [4:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_sbb_8:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    sbbb $7, %al # sched: [2:0.67]
-; SANDY-NEXT:    sbbb $7, %dil # sched: [2:0.67]
-; SANDY-NEXT:    sbbb $7, (%rsi) # sched: [9:1.00]
-; SANDY-NEXT:    sbbb %dl, %dil # sched: [2:0.67]
-; SANDY-NEXT:    sbbb %dil, (%rsi) # sched: [9:1.00]
-; SANDY-NEXT:    sbbb (%rsi), %dil # sched: [7:0.67]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_sbb_8:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    sbbb $7, %al # sched: [2:0.50]
-; HASWELL-NEXT:    sbbb $7, %dil # sched: [2:0.50]
-; HASWELL-NEXT:    sbbb $7, (%rsi) # sched: [9:1.00]
-; HASWELL-NEXT:    sbbb %dl, %dil # sched: [2:0.50]
-; HASWELL-NEXT:    sbbb %dil, (%rsi) # sched: [9:1.00]
-; HASWELL-NEXT:    sbbb (%rsi), %dil # sched: [7:0.50]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_sbb_8:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    sbbb $7, %al # sched: [2:0.50]
-; BROADWELL-NEXT:    sbbb $7, %dil # sched: [2:0.50]
-; BROADWELL-NEXT:    sbbb $7, (%rsi) # sched: [8:1.00]
-; BROADWELL-NEXT:    sbbb %dl, %dil # sched: [1:0.50]
-; BROADWELL-NEXT:    sbbb %dil, (%rsi) # sched: [8:1.00]
-; BROADWELL-NEXT:    sbbb (%rsi), %dil # sched: [6:0.50]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_sbb_8:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    sbbb $7, %al # sched: [2:0.50]
-; SKYLAKE-NEXT:    sbbb $7, %dil # sched: [2:0.50]
-; SKYLAKE-NEXT:    sbbb $7, (%rsi) # sched: [8:1.00]
-; SKYLAKE-NEXT:    sbbb %dl, %dil # sched: [1:0.50]
-; SKYLAKE-NEXT:    sbbb %dil, (%rsi) # sched: [8:1.00]
-; SKYLAKE-NEXT:    sbbb (%rsi), %dil # sched: [6:0.50]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_sbb_8:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    sbbb $7, %al # sched: [2:0.50]
-; SKX-NEXT:    sbbb $7, %dil # sched: [2:0.50]
-; SKX-NEXT:    sbbb $7, (%rsi) # sched: [8:1.00]
-; SKX-NEXT:    sbbb %dl, %dil # sched: [1:0.50]
-; SKX-NEXT:    sbbb %dil, (%rsi) # sched: [8:1.00]
-; SKX-NEXT:    sbbb (%rsi), %dil # sched: [6:0.50]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_sbb_8:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    sbbb $7, %al # sched: [1:1.00]
-; BDVER2-NEXT:    sbbb $7, %dil # sched: [1:1.00]
-; BDVER2-NEXT:    sbbb $7, (%rsi) # sched: [6:1.00]
-; BDVER2-NEXT:    sbbb %dl, %dil # sched: [1:1.00]
-; BDVER2-NEXT:    sbbb %dil, (%rsi) # sched: [6:1.00]
-; BDVER2-NEXT:    sbbb (%rsi), %dil # sched: [5:1.00]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_sbb_8:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    sbbb $7, %al # sched: [1:1.00]
-; BTVER2-NEXT:    sbbb $7, %dil # sched: [1:1.00]
-; BTVER2-NEXT:    sbbb $7, (%rsi) # sched: [5:1.00]
-; BTVER2-NEXT:    sbbb %dl, %dil # sched: [1:1.00]
-; BTVER2-NEXT:    sbbb %dil, (%rsi) # sched: [5:1.00]
-; BTVER2-NEXT:    sbbb (%rsi), %dil # sched: [4:1.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_sbb_8:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    sbbb $7, %al # sched: [1:0.25]
-; ZNVER1-NEXT:    sbbb $7, %dil # sched: [1:0.25]
-; ZNVER1-NEXT:    sbbb $7, (%rsi) # sched: [5:0.50]
-; ZNVER1-NEXT:    sbbb %dl, %dil # sched: [1:0.25]
-; ZNVER1-NEXT:    sbbb %dil, (%rsi) # sched: [5:0.50]
-; ZNVER1-NEXT:    sbbb (%rsi), %dil # sched: [5:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  tail call void asm "sbbb $3, %AL \0A\09 sbbb $3, $0 \0A\09 sbbb $3, $2 \0A\09 sbbb $1, $0 \0A\09 sbbb $0, $2 \0A\09 sbbb $2, $0", "r,r,*m,i"(i8 %a0, i8 %a2, i8* %a1, i8 7) nounwind
-  ret void
-}
-define void @test_sbb_16(i16 %a0, i16* %a1, i16 %a2) optsize {
-; GENERIC-LABEL: test_sbb_16:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    sbbw $511, %ax # imm = 0x1FF
-; GENERIC-NEXT:    # sched: [2:0.67]
-; GENERIC-NEXT:    sbbw $511, %di # imm = 0x1FF
-; GENERIC-NEXT:    # sched: [2:0.67]
-; GENERIC-NEXT:    sbbw $511, (%rsi) # imm = 0x1FF
-; GENERIC-NEXT:    # sched: [9:1.00]
-; GENERIC-NEXT:    sbbw $7, %di # sched: [2:0.67]
-; GENERIC-NEXT:    sbbw $7, (%rsi) # sched: [9:1.00]
-; GENERIC-NEXT:    sbbw %dx, %di # sched: [2:0.67]
-; GENERIC-NEXT:    sbbw %di, (%rsi) # sched: [9:1.00]
-; GENERIC-NEXT:    sbbw (%rsi), %di # sched: [7:0.67]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_sbb_16:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    sbbw $511, %ax # imm = 0x1FF
-; ATOM-NEXT:    # sched: [1:0.50]
-; ATOM-NEXT:    sbbw $511, %di # imm = 0x1FF
-; ATOM-NEXT:    # sched: [1:0.50]
-; ATOM-NEXT:    sbbw $511, (%rsi) # imm = 0x1FF
-; ATOM-NEXT:    # sched: [1:1.00]
-; ATOM-NEXT:    sbbw $7, %di # sched: [1:0.50]
-; ATOM-NEXT:    sbbw $7, (%rsi) # sched: [1:1.00]
-; ATOM-NEXT:    sbbw %dx, %di # sched: [1:0.50]
-; ATOM-NEXT:    sbbw %di, (%rsi) # sched: [1:1.00]
-; ATOM-NEXT:    sbbw (%rsi), %di # sched: [1:1.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_sbb_16:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    sbbw $511, %ax # imm = 0x1FF
-; SLM-NEXT:    # sched: [1:0.50]
-; SLM-NEXT:    sbbw $511, %di # imm = 0x1FF
-; SLM-NEXT:    # sched: [1:0.50]
-; SLM-NEXT:    sbbw $511, (%rsi) # imm = 0x1FF
-; SLM-NEXT:    # sched: [5:2.00]
-; SLM-NEXT:    sbbw $7, %di # sched: [1:0.50]
-; SLM-NEXT:    sbbw $7, (%rsi) # sched: [5:2.00]
-; SLM-NEXT:    sbbw %dx, %di # sched: [1:0.50]
-; SLM-NEXT:    sbbw %di, (%rsi) # sched: [5:2.00]
-; SLM-NEXT:    sbbw (%rsi), %di # sched: [4:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_sbb_16:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    sbbw $511, %ax # imm = 0x1FF
-; SANDY-NEXT:    # sched: [2:0.67]
-; SANDY-NEXT:    sbbw $511, %di # imm = 0x1FF
-; SANDY-NEXT:    # sched: [2:0.67]
-; SANDY-NEXT:    sbbw $511, (%rsi) # imm = 0x1FF
-; SANDY-NEXT:    # sched: [9:1.00]
-; SANDY-NEXT:    sbbw $7, %di # sched: [2:0.67]
-; SANDY-NEXT:    sbbw $7, (%rsi) # sched: [9:1.00]
-; SANDY-NEXT:    sbbw %dx, %di # sched: [2:0.67]
-; SANDY-NEXT:    sbbw %di, (%rsi) # sched: [9:1.00]
-; SANDY-NEXT:    sbbw (%rsi), %di # sched: [7:0.67]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_sbb_16:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    sbbw $511, %ax # imm = 0x1FF
-; HASWELL-NEXT:    # sched: [2:0.50]
-; HASWELL-NEXT:    sbbw $511, %di # imm = 0x1FF
-; HASWELL-NEXT:    # sched: [2:0.50]
-; HASWELL-NEXT:    sbbw $511, (%rsi) # imm = 0x1FF
-; HASWELL-NEXT:    # sched: [9:1.00]
-; HASWELL-NEXT:    sbbw $7, %di # sched: [2:0.50]
-; HASWELL-NEXT:    sbbw $7, (%rsi) # sched: [9:1.00]
-; HASWELL-NEXT:    sbbw %dx, %di # sched: [2:0.50]
-; HASWELL-NEXT:    sbbw %di, (%rsi) # sched: [9:1.00]
-; HASWELL-NEXT:    sbbw (%rsi), %di # sched: [7:0.50]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_sbb_16:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    sbbw $511, %ax # imm = 0x1FF
-; BROADWELL-NEXT:    # sched: [1:0.50]
-; BROADWELL-NEXT:    sbbw $511, %di # imm = 0x1FF
-; BROADWELL-NEXT:    # sched: [1:0.50]
-; BROADWELL-NEXT:    sbbw $511, (%rsi) # imm = 0x1FF
-; BROADWELL-NEXT:    # sched: [8:1.00]
-; BROADWELL-NEXT:    sbbw $7, %di # sched: [1:0.50]
-; BROADWELL-NEXT:    sbbw $7, (%rsi) # sched: [8:1.00]
-; BROADWELL-NEXT:    sbbw %dx, %di # sched: [1:0.50]
-; BROADWELL-NEXT:    sbbw %di, (%rsi) # sched: [8:1.00]
-; BROADWELL-NEXT:    sbbw (%rsi), %di # sched: [6:0.50]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_sbb_16:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    sbbw $511, %ax # imm = 0x1FF
-; SKYLAKE-NEXT:    # sched: [1:0.50]
-; SKYLAKE-NEXT:    sbbw $511, %di # imm = 0x1FF
-; SKYLAKE-NEXT:    # sched: [1:0.50]
-; SKYLAKE-NEXT:    sbbw $511, (%rsi) # imm = 0x1FF
-; SKYLAKE-NEXT:    # sched: [8:1.00]
-; SKYLAKE-NEXT:    sbbw $7, %di # sched: [1:0.50]
-; SKYLAKE-NEXT:    sbbw $7, (%rsi) # sched: [8:1.00]
-; SKYLAKE-NEXT:    sbbw %dx, %di # sched: [1:0.50]
-; SKYLAKE-NEXT:    sbbw %di, (%rsi) # sched: [8:1.00]
-; SKYLAKE-NEXT:    sbbw (%rsi), %di # sched: [6:0.50]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_sbb_16:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    sbbw $511, %ax # imm = 0x1FF
-; SKX-NEXT:    # sched: [1:0.50]
-; SKX-NEXT:    sbbw $511, %di # imm = 0x1FF
-; SKX-NEXT:    # sched: [1:0.50]
-; SKX-NEXT:    sbbw $511, (%rsi) # imm = 0x1FF
-; SKX-NEXT:    # sched: [8:1.00]
-; SKX-NEXT:    sbbw $7, %di # sched: [1:0.50]
-; SKX-NEXT:    sbbw $7, (%rsi) # sched: [8:1.00]
-; SKX-NEXT:    sbbw %dx, %di # sched: [1:0.50]
-; SKX-NEXT:    sbbw %di, (%rsi) # sched: [8:1.00]
-; SKX-NEXT:    sbbw (%rsi), %di # sched: [6:0.50]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_sbb_16:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    sbbw $511, %ax # imm = 0x1FF
-; BDVER2-NEXT:    # sched: [1:1.00]
-; BDVER2-NEXT:    sbbw $511, %di # imm = 0x1FF
-; BDVER2-NEXT:    # sched: [1:1.00]
-; BDVER2-NEXT:    sbbw $511, (%rsi) # imm = 0x1FF
-; BDVER2-NEXT:    # sched: [6:1.00]
-; BDVER2-NEXT:    sbbw $7, %di # sched: [1:1.00]
-; BDVER2-NEXT:    sbbw $7, (%rsi) # sched: [6:1.00]
-; BDVER2-NEXT:    sbbw %dx, %di # sched: [1:1.00]
-; BDVER2-NEXT:    sbbw %di, (%rsi) # sched: [6:1.00]
-; BDVER2-NEXT:    sbbw (%rsi), %di # sched: [5:1.00]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_sbb_16:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    sbbw $511, %ax # imm = 0x1FF
-; BTVER2-NEXT:    # sched: [1:1.00]
-; BTVER2-NEXT:    sbbw $511, %di # imm = 0x1FF
-; BTVER2-NEXT:    # sched: [1:1.00]
-; BTVER2-NEXT:    sbbw $511, (%rsi) # imm = 0x1FF
-; BTVER2-NEXT:    # sched: [5:1.00]
-; BTVER2-NEXT:    sbbw $7, %di # sched: [1:1.00]
-; BTVER2-NEXT:    sbbw $7, (%rsi) # sched: [5:1.00]
-; BTVER2-NEXT:    sbbw %dx, %di # sched: [1:1.00]
-; BTVER2-NEXT:    sbbw %di, (%rsi) # sched: [5:1.00]
-; BTVER2-NEXT:    sbbw (%rsi), %di # sched: [4:1.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_sbb_16:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    sbbw $511, %ax # imm = 0x1FF
-; ZNVER1-NEXT:    # sched: [1:0.25]
-; ZNVER1-NEXT:    sbbw $511, %di # imm = 0x1FF
-; ZNVER1-NEXT:    # sched: [1:0.25]
-; ZNVER1-NEXT:    sbbw $511, (%rsi) # imm = 0x1FF
-; ZNVER1-NEXT:    # sched: [5:0.50]
-; ZNVER1-NEXT:    sbbw $7, %di # sched: [1:0.25]
-; ZNVER1-NEXT:    sbbw $7, (%rsi) # sched: [5:0.50]
-; ZNVER1-NEXT:    sbbw %dx, %di # sched: [1:0.25]
-; ZNVER1-NEXT:    sbbw %di, (%rsi) # sched: [5:0.50]
-; ZNVER1-NEXT:    sbbw (%rsi), %di # sched: [5:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  tail call void asm "sbbw $3, %AX \0A\09 sbbw $3, $0 \0A\09 sbbw $3, $2 \0A\09 sbbw $4, $0 \0A\09 sbbw $4, $2 \0A\09 sbbw $1, $0 \0A\09 sbbw $0, $2 \0A\09 sbbw $2, $0", "r,r,*m,i,i"(i16 %a0, i16 %a2, i16* %a1, i16 511, i8 7) nounwind
-  ret void
-}
-define void @test_sbb_32(i32 %a0, i32* %a1, i32 %a2) optsize {
-; GENERIC-LABEL: test_sbb_32:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    sbbl $665536, %eax # imm = 0xA27C0
-; GENERIC-NEXT:    # sched: [2:0.67]
-; GENERIC-NEXT:    sbbl $665536, %edi # imm = 0xA27C0
-; GENERIC-NEXT:    # sched: [2:0.67]
-; GENERIC-NEXT:    sbbl $665536, (%rsi) # imm = 0xA27C0
-; GENERIC-NEXT:    # sched: [9:1.00]
-; GENERIC-NEXT:    sbbl $7, %edi # sched: [2:0.67]
-; GENERIC-NEXT:    sbbl $7, (%rsi) # sched: [9:1.00]
-; GENERIC-NEXT:    sbbl %edx, %edi # sched: [2:0.67]
-; GENERIC-NEXT:    sbbl %edi, (%rsi) # sched: [9:1.00]
-; GENERIC-NEXT:    sbbl (%rsi), %edi # sched: [7:0.67]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_sbb_32:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    sbbl $665536, %eax # imm = 0xA27C0
-; ATOM-NEXT:    # sched: [1:0.50]
-; ATOM-NEXT:    sbbl $665536, %edi # imm = 0xA27C0
-; ATOM-NEXT:    # sched: [1:0.50]
-; ATOM-NEXT:    sbbl $665536, (%rsi) # imm = 0xA27C0
-; ATOM-NEXT:    # sched: [1:1.00]
-; ATOM-NEXT:    sbbl $7, %edi # sched: [1:0.50]
-; ATOM-NEXT:    sbbl $7, (%rsi) # sched: [1:1.00]
-; ATOM-NEXT:    sbbl %edx, %edi # sched: [1:0.50]
-; ATOM-NEXT:    sbbl %edi, (%rsi) # sched: [1:1.00]
-; ATOM-NEXT:    sbbl (%rsi), %edi # sched: [1:1.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_sbb_32:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    sbbl $665536, %eax # imm = 0xA27C0
-; SLM-NEXT:    # sched: [1:0.50]
-; SLM-NEXT:    sbbl $665536, %edi # imm = 0xA27C0
-; SLM-NEXT:    # sched: [1:0.50]
-; SLM-NEXT:    sbbl $665536, (%rsi) # imm = 0xA27C0
-; SLM-NEXT:    # sched: [5:2.00]
-; SLM-NEXT:    sbbl $7, %edi # sched: [1:0.50]
-; SLM-NEXT:    sbbl $7, (%rsi) # sched: [5:2.00]
-; SLM-NEXT:    sbbl %edx, %edi # sched: [1:0.50]
-; SLM-NEXT:    sbbl %edi, (%rsi) # sched: [5:2.00]
-; SLM-NEXT:    sbbl (%rsi), %edi # sched: [4:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_sbb_32:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    sbbl $665536, %eax # imm = 0xA27C0
-; SANDY-NEXT:    # sched: [2:0.67]
-; SANDY-NEXT:    sbbl $665536, %edi # imm = 0xA27C0
-; SANDY-NEXT:    # sched: [2:0.67]
-; SANDY-NEXT:    sbbl $665536, (%rsi) # imm = 0xA27C0
-; SANDY-NEXT:    # sched: [9:1.00]
-; SANDY-NEXT:    sbbl $7, %edi # sched: [2:0.67]
-; SANDY-NEXT:    sbbl $7, (%rsi) # sched: [9:1.00]
-; SANDY-NEXT:    sbbl %edx, %edi # sched: [2:0.67]
-; SANDY-NEXT:    sbbl %edi, (%rsi) # sched: [9:1.00]
-; SANDY-NEXT:    sbbl (%rsi), %edi # sched: [7:0.67]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_sbb_32:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    sbbl $665536, %eax # imm = 0xA27C0
-; HASWELL-NEXT:    # sched: [2:0.50]
-; HASWELL-NEXT:    sbbl $665536, %edi # imm = 0xA27C0
-; HASWELL-NEXT:    # sched: [2:0.50]
-; HASWELL-NEXT:    sbbl $665536, (%rsi) # imm = 0xA27C0
-; HASWELL-NEXT:    # sched: [9:1.00]
-; HASWELL-NEXT:    sbbl $7, %edi # sched: [2:0.50]
-; HASWELL-NEXT:    sbbl $7, (%rsi) # sched: [9:1.00]
-; HASWELL-NEXT:    sbbl %edx, %edi # sched: [2:0.50]
-; HASWELL-NEXT:    sbbl %edi, (%rsi) # sched: [9:1.00]
-; HASWELL-NEXT:    sbbl (%rsi), %edi # sched: [7:0.50]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_sbb_32:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    sbbl $665536, %eax # imm = 0xA27C0
-; BROADWELL-NEXT:    # sched: [1:0.50]
-; BROADWELL-NEXT:    sbbl $665536, %edi # imm = 0xA27C0
-; BROADWELL-NEXT:    # sched: [1:0.50]
-; BROADWELL-NEXT:    sbbl $665536, (%rsi) # imm = 0xA27C0
-; BROADWELL-NEXT:    # sched: [8:1.00]
-; BROADWELL-NEXT:    sbbl $7, %edi # sched: [1:0.50]
-; BROADWELL-NEXT:    sbbl $7, (%rsi) # sched: [8:1.00]
-; BROADWELL-NEXT:    sbbl %edx, %edi # sched: [1:0.50]
-; BROADWELL-NEXT:    sbbl %edi, (%rsi) # sched: [8:1.00]
-; BROADWELL-NEXT:    sbbl (%rsi), %edi # sched: [6:0.50]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_sbb_32:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    sbbl $665536, %eax # imm = 0xA27C0
-; SKYLAKE-NEXT:    # sched: [1:0.50]
-; SKYLAKE-NEXT:    sbbl $665536, %edi # imm = 0xA27C0
-; SKYLAKE-NEXT:    # sched: [1:0.50]
-; SKYLAKE-NEXT:    sbbl $665536, (%rsi) # imm = 0xA27C0
-; SKYLAKE-NEXT:    # sched: [8:1.00]
-; SKYLAKE-NEXT:    sbbl $7, %edi # sched: [1:0.50]
-; SKYLAKE-NEXT:    sbbl $7, (%rsi) # sched: [8:1.00]
-; SKYLAKE-NEXT:    sbbl %edx, %edi # sched: [1:0.50]
-; SKYLAKE-NEXT:    sbbl %edi, (%rsi) # sched: [8:1.00]
-; SKYLAKE-NEXT:    sbbl (%rsi), %edi # sched: [6:0.50]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_sbb_32:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    sbbl $665536, %eax # imm = 0xA27C0
-; SKX-NEXT:    # sched: [1:0.50]
-; SKX-NEXT:    sbbl $665536, %edi # imm = 0xA27C0
-; SKX-NEXT:    # sched: [1:0.50]
-; SKX-NEXT:    sbbl $665536, (%rsi) # imm = 0xA27C0
-; SKX-NEXT:    # sched: [8:1.00]
-; SKX-NEXT:    sbbl $7, %edi # sched: [1:0.50]
-; SKX-NEXT:    sbbl $7, (%rsi) # sched: [8:1.00]
-; SKX-NEXT:    sbbl %edx, %edi # sched: [1:0.50]
-; SKX-NEXT:    sbbl %edi, (%rsi) # sched: [8:1.00]
-; SKX-NEXT:    sbbl (%rsi), %edi # sched: [6:0.50]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_sbb_32:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    sbbl $665536, %eax # imm = 0xA27C0
-; BDVER2-NEXT:    # sched: [1:1.00]
-; BDVER2-NEXT:    sbbl $665536, %edi # imm = 0xA27C0
-; BDVER2-NEXT:    # sched: [1:1.00]
-; BDVER2-NEXT:    sbbl $665536, (%rsi) # imm = 0xA27C0
-; BDVER2-NEXT:    # sched: [6:1.00]
-; BDVER2-NEXT:    sbbl $7, %edi # sched: [1:1.00]
-; BDVER2-NEXT:    sbbl $7, (%rsi) # sched: [6:1.00]
-; BDVER2-NEXT:    sbbl %edx, %edi # sched: [1:1.00]
-; BDVER2-NEXT:    sbbl %edi, (%rsi) # sched: [6:1.00]
-; BDVER2-NEXT:    sbbl (%rsi), %edi # sched: [5:1.00]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_sbb_32:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    sbbl $665536, %eax # imm = 0xA27C0
-; BTVER2-NEXT:    # sched: [1:1.00]
-; BTVER2-NEXT:    sbbl $665536, %edi # imm = 0xA27C0
-; BTVER2-NEXT:    # sched: [1:1.00]
-; BTVER2-NEXT:    sbbl $665536, (%rsi) # imm = 0xA27C0
-; BTVER2-NEXT:    # sched: [5:1.00]
-; BTVER2-NEXT:    sbbl $7, %edi # sched: [1:1.00]
-; BTVER2-NEXT:    sbbl $7, (%rsi) # sched: [5:1.00]
-; BTVER2-NEXT:    sbbl %edx, %edi # sched: [1:1.00]
-; BTVER2-NEXT:    sbbl %edi, (%rsi) # sched: [5:1.00]
-; BTVER2-NEXT:    sbbl (%rsi), %edi # sched: [4:1.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_sbb_32:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    sbbl $665536, %eax # imm = 0xA27C0
-; ZNVER1-NEXT:    # sched: [1:0.25]
-; ZNVER1-NEXT:    sbbl $665536, %edi # imm = 0xA27C0
-; ZNVER1-NEXT:    # sched: [1:0.25]
-; ZNVER1-NEXT:    sbbl $665536, (%rsi) # imm = 0xA27C0
-; ZNVER1-NEXT:    # sched: [5:0.50]
-; ZNVER1-NEXT:    sbbl $7, %edi # sched: [1:0.25]
-; ZNVER1-NEXT:    sbbl $7, (%rsi) # sched: [5:0.50]
-; ZNVER1-NEXT:    sbbl %edx, %edi # sched: [1:0.25]
-; ZNVER1-NEXT:    sbbl %edi, (%rsi) # sched: [5:0.50]
-; ZNVER1-NEXT:    sbbl (%rsi), %edi # sched: [5:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  tail call void asm "sbbl $3, %EAX \0A\09 sbbl $3, $0 \0A\09 sbbl $3, $2 \0A\09 sbbl $4, $0 \0A\09 sbbl $4, $2 \0A\09 sbbl $1, $0 \0A\09 sbbl $0, $2 \0A\09 sbbl $2, $0", "r,r,*m,i,i"(i32 %a0, i32 %a2, i32* %a1, i32 665536, i8 7) nounwind
-  ret void
-}
-define void @test_sbb_64(i64 %a0, i64* %a1, i64 %a2) optsize {
-; GENERIC-LABEL: test_sbb_64:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    sbbq $665536, %rax # imm = 0xA27C0
-; GENERIC-NEXT:    # sched: [2:0.67]
-; GENERIC-NEXT:    sbbq $665536, %rdi # imm = 0xA27C0
-; GENERIC-NEXT:    # sched: [2:0.67]
-; GENERIC-NEXT:    sbbq $665536, (%rsi) # imm = 0xA27C0
-; GENERIC-NEXT:    # sched: [9:1.00]
-; GENERIC-NEXT:    sbbq $7, %rdi # sched: [2:0.67]
-; GENERIC-NEXT:    sbbq $7, (%rsi) # sched: [9:1.00]
-; GENERIC-NEXT:    sbbq %rdx, %rdi # sched: [2:0.67]
-; GENERIC-NEXT:    sbbq %rdi, (%rsi) # sched: [9:1.00]
-; GENERIC-NEXT:    sbbq (%rsi), %rdi # sched: [7:0.67]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_sbb_64:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    sbbq $665536, %rax # imm = 0xA27C0
-; ATOM-NEXT:    # sched: [1:0.50]
-; ATOM-NEXT:    sbbq $665536, %rdi # imm = 0xA27C0
-; ATOM-NEXT:    # sched: [1:0.50]
-; ATOM-NEXT:    sbbq $665536, (%rsi) # imm = 0xA27C0
-; ATOM-NEXT:    # sched: [1:1.00]
-; ATOM-NEXT:    sbbq $7, %rdi # sched: [1:0.50]
-; ATOM-NEXT:    sbbq $7, (%rsi) # sched: [1:1.00]
-; ATOM-NEXT:    sbbq %rdx, %rdi # sched: [1:0.50]
-; ATOM-NEXT:    sbbq %rdi, (%rsi) # sched: [1:1.00]
-; ATOM-NEXT:    sbbq (%rsi), %rdi # sched: [1:1.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_sbb_64:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    sbbq $665536, %rax # imm = 0xA27C0
-; SLM-NEXT:    # sched: [1:0.50]
-; SLM-NEXT:    sbbq $665536, %rdi # imm = 0xA27C0
-; SLM-NEXT:    # sched: [1:0.50]
-; SLM-NEXT:    sbbq $665536, (%rsi) # imm = 0xA27C0
-; SLM-NEXT:    # sched: [5:2.00]
-; SLM-NEXT:    sbbq $7, %rdi # sched: [1:0.50]
-; SLM-NEXT:    sbbq $7, (%rsi) # sched: [5:2.00]
-; SLM-NEXT:    sbbq %rdx, %rdi # sched: [1:0.50]
-; SLM-NEXT:    sbbq %rdi, (%rsi) # sched: [5:2.00]
-; SLM-NEXT:    sbbq (%rsi), %rdi # sched: [4:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_sbb_64:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    sbbq $665536, %rax # imm = 0xA27C0
-; SANDY-NEXT:    # sched: [2:0.67]
-; SANDY-NEXT:    sbbq $665536, %rdi # imm = 0xA27C0
-; SANDY-NEXT:    # sched: [2:0.67]
-; SANDY-NEXT:    sbbq $665536, (%rsi) # imm = 0xA27C0
-; SANDY-NEXT:    # sched: [9:1.00]
-; SANDY-NEXT:    sbbq $7, %rdi # sched: [2:0.67]
-; SANDY-NEXT:    sbbq $7, (%rsi) # sched: [9:1.00]
-; SANDY-NEXT:    sbbq %rdx, %rdi # sched: [2:0.67]
-; SANDY-NEXT:    sbbq %rdi, (%rsi) # sched: [9:1.00]
-; SANDY-NEXT:    sbbq (%rsi), %rdi # sched: [7:0.67]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_sbb_64:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    sbbq $665536, %rax # imm = 0xA27C0
-; HASWELL-NEXT:    # sched: [2:0.50]
-; HASWELL-NEXT:    sbbq $665536, %rdi # imm = 0xA27C0
-; HASWELL-NEXT:    # sched: [2:0.50]
-; HASWELL-NEXT:    sbbq $665536, (%rsi) # imm = 0xA27C0
-; HASWELL-NEXT:    # sched: [9:1.00]
-; HASWELL-NEXT:    sbbq $7, %rdi # sched: [2:0.50]
-; HASWELL-NEXT:    sbbq $7, (%rsi) # sched: [9:1.00]
-; HASWELL-NEXT:    sbbq %rdx, %rdi # sched: [2:0.50]
-; HASWELL-NEXT:    sbbq %rdi, (%rsi) # sched: [9:1.00]
-; HASWELL-NEXT:    sbbq (%rsi), %rdi # sched: [7:0.50]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_sbb_64:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    sbbq $665536, %rax # imm = 0xA27C0
-; BROADWELL-NEXT:    # sched: [1:0.50]
-; BROADWELL-NEXT:    sbbq $665536, %rdi # imm = 0xA27C0
-; BROADWELL-NEXT:    # sched: [1:0.50]
-; BROADWELL-NEXT:    sbbq $665536, (%rsi) # imm = 0xA27C0
-; BROADWELL-NEXT:    # sched: [8:1.00]
-; BROADWELL-NEXT:    sbbq $7, %rdi # sched: [1:0.50]
-; BROADWELL-NEXT:    sbbq $7, (%rsi) # sched: [8:1.00]
-; BROADWELL-NEXT:    sbbq %rdx, %rdi # sched: [1:0.50]
-; BROADWELL-NEXT:    sbbq %rdi, (%rsi) # sched: [8:1.00]
-; BROADWELL-NEXT:    sbbq (%rsi), %rdi # sched: [6:0.50]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_sbb_64:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    sbbq $665536, %rax # imm = 0xA27C0
-; SKYLAKE-NEXT:    # sched: [1:0.50]
-; SKYLAKE-NEXT:    sbbq $665536, %rdi # imm = 0xA27C0
-; SKYLAKE-NEXT:    # sched: [1:0.50]
-; SKYLAKE-NEXT:    sbbq $665536, (%rsi) # imm = 0xA27C0
-; SKYLAKE-NEXT:    # sched: [8:1.00]
-; SKYLAKE-NEXT:    sbbq $7, %rdi # sched: [1:0.50]
-; SKYLAKE-NEXT:    sbbq $7, (%rsi) # sched: [8:1.00]
-; SKYLAKE-NEXT:    sbbq %rdx, %rdi # sched: [1:0.50]
-; SKYLAKE-NEXT:    sbbq %rdi, (%rsi) # sched: [8:1.00]
-; SKYLAKE-NEXT:    sbbq (%rsi), %rdi # sched: [6:0.50]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_sbb_64:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    sbbq $665536, %rax # imm = 0xA27C0
-; SKX-NEXT:    # sched: [1:0.50]
-; SKX-NEXT:    sbbq $665536, %rdi # imm = 0xA27C0
-; SKX-NEXT:    # sched: [1:0.50]
-; SKX-NEXT:    sbbq $665536, (%rsi) # imm = 0xA27C0
-; SKX-NEXT:    # sched: [8:1.00]
-; SKX-NEXT:    sbbq $7, %rdi # sched: [1:0.50]
-; SKX-NEXT:    sbbq $7, (%rsi) # sched: [8:1.00]
-; SKX-NEXT:    sbbq %rdx, %rdi # sched: [1:0.50]
-; SKX-NEXT:    sbbq %rdi, (%rsi) # sched: [8:1.00]
-; SKX-NEXT:    sbbq (%rsi), %rdi # sched: [6:0.50]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_sbb_64:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    sbbq $665536, %rax # imm = 0xA27C0
-; BDVER2-NEXT:    # sched: [1:1.00]
-; BDVER2-NEXT:    sbbq $665536, %rdi # imm = 0xA27C0
-; BDVER2-NEXT:    # sched: [1:1.00]
-; BDVER2-NEXT:    sbbq $665536, (%rsi) # imm = 0xA27C0
-; BDVER2-NEXT:    # sched: [6:1.00]
-; BDVER2-NEXT:    sbbq $7, %rdi # sched: [1:1.00]
-; BDVER2-NEXT:    sbbq $7, (%rsi) # sched: [6:1.00]
-; BDVER2-NEXT:    sbbq %rdx, %rdi # sched: [1:1.00]
-; BDVER2-NEXT:    sbbq %rdi, (%rsi) # sched: [6:1.00]
-; BDVER2-NEXT:    sbbq (%rsi), %rdi # sched: [5:1.00]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_sbb_64:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    sbbq $665536, %rax # imm = 0xA27C0
-; BTVER2-NEXT:    # sched: [1:1.00]
-; BTVER2-NEXT:    sbbq $665536, %rdi # imm = 0xA27C0
-; BTVER2-NEXT:    # sched: [1:1.00]
-; BTVER2-NEXT:    sbbq $665536, (%rsi) # imm = 0xA27C0
-; BTVER2-NEXT:    # sched: [5:1.00]
-; BTVER2-NEXT:    sbbq $7, %rdi # sched: [1:1.00]
-; BTVER2-NEXT:    sbbq $7, (%rsi) # sched: [5:1.00]
-; BTVER2-NEXT:    sbbq %rdx, %rdi # sched: [1:1.00]
-; BTVER2-NEXT:    sbbq %rdi, (%rsi) # sched: [5:1.00]
-; BTVER2-NEXT:    sbbq (%rsi), %rdi # sched: [4:1.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_sbb_64:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    sbbq $665536, %rax # imm = 0xA27C0
-; ZNVER1-NEXT:    # sched: [1:0.25]
-; ZNVER1-NEXT:    sbbq $665536, %rdi # imm = 0xA27C0
-; ZNVER1-NEXT:    # sched: [1:0.25]
-; ZNVER1-NEXT:    sbbq $665536, (%rsi) # imm = 0xA27C0
-; ZNVER1-NEXT:    # sched: [5:0.50]
-; ZNVER1-NEXT:    sbbq $7, %rdi # sched: [1:0.25]
-; ZNVER1-NEXT:    sbbq $7, (%rsi) # sched: [5:0.50]
-; ZNVER1-NEXT:    sbbq %rdx, %rdi # sched: [1:0.25]
-; ZNVER1-NEXT:    sbbq %rdi, (%rsi) # sched: [5:0.50]
-; ZNVER1-NEXT:    sbbq (%rsi), %rdi # sched: [5:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  tail call void asm "sbbq $3, %RAX \0A\09 sbbq $3, $0 \0A\09 sbbq $3, $2 \0A\09 sbbq $4, $0 \0A\09 sbbq $4, $2 \0A\09 sbbq $1, $0 \0A\09 sbbq $0, $2 \0A\09 sbbq $2, $0", "r,r,*m,i,i"(i64 %a0, i64 %a2, i64* %a1, i32 665536, i8 7) nounwind
-  ret void
-}
-
-define void @test_scas() optsize {
-; GENERIC-LABEL: test_scas:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    scasb %es:(%rdi), %al # sched: [2:0.67]
-; GENERIC-NEXT:    scasw %es:(%rdi), %ax # sched: [2:0.67]
-; GENERIC-NEXT:    scasl %es:(%rdi), %eax # sched: [2:0.67]
-; GENERIC-NEXT:    scasq %es:(%rdi), %rax # sched: [2:0.67]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_scas:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    scasb %es:(%rdi), %al # sched: [2:1.00]
-; ATOM-NEXT:    scasw %es:(%rdi), %ax # sched: [2:1.00]
-; ATOM-NEXT:    scasl %es:(%rdi), %eax # sched: [2:1.00]
-; ATOM-NEXT:    scasq %es:(%rdi), %rax # sched: [2:1.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_scas:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    scasb %es:(%rdi), %al # sched: [100:1.00]
-; SLM-NEXT:    scasw %es:(%rdi), %ax # sched: [100:1.00]
-; SLM-NEXT:    scasl %es:(%rdi), %eax # sched: [100:1.00]
-; SLM-NEXT:    scasq %es:(%rdi), %rax # sched: [100:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_scas:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    scasb %es:(%rdi), %al # sched: [2:0.67]
-; SANDY-NEXT:    scasw %es:(%rdi), %ax # sched: [2:0.67]
-; SANDY-NEXT:    scasl %es:(%rdi), %eax # sched: [2:0.67]
-; SANDY-NEXT:    scasq %es:(%rdi), %rax # sched: [2:0.67]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_scas:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    scasb %es:(%rdi), %al # sched: [7:0.50]
-; HASWELL-NEXT:    scasw %es:(%rdi), %ax # sched: [7:0.50]
-; HASWELL-NEXT:    scasl %es:(%rdi), %eax # sched: [7:0.50]
-; HASWELL-NEXT:    scasq %es:(%rdi), %rax # sched: [7:0.50]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_scas:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    scasb %es:(%rdi), %al # sched: [7:0.50]
-; BROADWELL-NEXT:    scasw %es:(%rdi), %ax # sched: [7:0.50]
-; BROADWELL-NEXT:    scasl %es:(%rdi), %eax # sched: [7:0.50]
-; BROADWELL-NEXT:    scasq %es:(%rdi), %rax # sched: [7:0.50]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_scas:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    scasb %es:(%rdi), %al # sched: [7:0.50]
-; SKYLAKE-NEXT:    scasw %es:(%rdi), %ax # sched: [7:0.50]
-; SKYLAKE-NEXT:    scasl %es:(%rdi), %eax # sched: [7:0.50]
-; SKYLAKE-NEXT:    scasq %es:(%rdi), %rax # sched: [7:0.50]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_scas:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    scasb %es:(%rdi), %al # sched: [7:0.50]
-; SKX-NEXT:    scasw %es:(%rdi), %ax # sched: [7:0.50]
-; SKX-NEXT:    scasl %es:(%rdi), %eax # sched: [7:0.50]
-; SKX-NEXT:    scasq %es:(%rdi), %rax # sched: [7:0.50]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_scas:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    scasb %es:(%rdi), %al # sched: [100:0.50]
-; BDVER2-NEXT:    scasw %es:(%rdi), %ax # sched: [100:0.50]
-; BDVER2-NEXT:    scasl %es:(%rdi), %eax # sched: [100:0.50]
-; BDVER2-NEXT:    scasq %es:(%rdi), %rax # sched: [100:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_scas:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    scasb %es:(%rdi), %al # sched: [100:0.50]
-; BTVER2-NEXT:    scasw %es:(%rdi), %ax # sched: [100:0.50]
-; BTVER2-NEXT:    scasl %es:(%rdi), %eax # sched: [100:0.50]
-; BTVER2-NEXT:    scasq %es:(%rdi), %rax # sched: [100:0.50]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_scas:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    scasb %es:(%rdi), %al # sched: [100:0.25]
-; ZNVER1-NEXT:    scasw %es:(%rdi), %ax # sched: [100:0.25]
-; ZNVER1-NEXT:    scasl %es:(%rdi), %eax # sched: [100:0.25]
-; ZNVER1-NEXT:    scasq %es:(%rdi), %rax # sched: [100:0.25]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  call void asm sideeffect "scasb \0A\09 scasw \0A\09 scasl \0A\09 scasq", ""()
-  ret void
-}
-
-define void @test_setcc(i8 %a0, i8 *%a1) optsize {
-; GENERIC-LABEL: test_setcc:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    seto %dil # sched: [1:0.50]
-; GENERIC-NEXT:    setno %dil # sched: [1:0.50]
-; GENERIC-NEXT:    setb %dil # sched: [1:0.50]
-; GENERIC-NEXT:    setae %dil # sched: [1:0.50]
-; GENERIC-NEXT:    sete %dil # sched: [1:0.50]
-; GENERIC-NEXT:    setne %dil # sched: [1:0.50]
-; GENERIC-NEXT:    setbe %dil # sched: [2:1.00]
-; GENERIC-NEXT:    seta %dil # sched: [2:1.00]
-; GENERIC-NEXT:    sets %dil # sched: [1:0.50]
-; GENERIC-NEXT:    setns %dil # sched: [1:0.50]
-; GENERIC-NEXT:    setp %dil # sched: [1:0.50]
-; GENERIC-NEXT:    setnp %dil # sched: [1:0.50]
-; GENERIC-NEXT:    setl %dil # sched: [1:0.50]
-; GENERIC-NEXT:    setge %dil # sched: [1:0.50]
-; GENERIC-NEXT:    setle %dil # sched: [1:0.50]
-; GENERIC-NEXT:    setg %dil # sched: [1:0.50]
-; GENERIC-NEXT:    seto (%rsi) # sched: [2:1.00]
-; GENERIC-NEXT:    setno (%rsi) # sched: [2:1.00]
-; GENERIC-NEXT:    setb (%rsi) # sched: [2:1.00]
-; GENERIC-NEXT:    setae (%rsi) # sched: [2:1.00]
-; GENERIC-NEXT:    sete (%rsi) # sched: [2:1.00]
-; GENERIC-NEXT:    setne (%rsi) # sched: [2:1.00]
-; GENERIC-NEXT:    setbe (%rsi) # sched: [3:1.00]
-; GENERIC-NEXT:    seta (%rsi) # sched: [3:1.00]
-; GENERIC-NEXT:    sets (%rsi) # sched: [2:1.00]
-; GENERIC-NEXT:    setns (%rsi) # sched: [2:1.00]
-; GENERIC-NEXT:    setp (%rsi) # sched: [2:1.00]
-; GENERIC-NEXT:    setnp (%rsi) # sched: [2:1.00]
-; GENERIC-NEXT:    setl (%rsi) # sched: [2:1.00]
-; GENERIC-NEXT:    setge (%rsi) # sched: [2:1.00]
-; GENERIC-NEXT:    setle (%rsi) # sched: [2:1.00]
-; GENERIC-NEXT:    setg (%rsi) # sched: [2:1.00]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_setcc:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    seto %dil # sched: [1:0.50]
-; ATOM-NEXT:    setno %dil # sched: [1:0.50]
-; ATOM-NEXT:    setb %dil # sched: [1:0.50]
-; ATOM-NEXT:    setae %dil # sched: [1:0.50]
-; ATOM-NEXT:    sete %dil # sched: [1:0.50]
-; ATOM-NEXT:    setne %dil # sched: [1:0.50]
-; ATOM-NEXT:    setbe %dil # sched: [1:0.50]
-; ATOM-NEXT:    seta %dil # sched: [1:0.50]
-; ATOM-NEXT:    sets %dil # sched: [1:0.50]
-; ATOM-NEXT:    setns %dil # sched: [1:0.50]
-; ATOM-NEXT:    setp %dil # sched: [1:0.50]
-; ATOM-NEXT:    setnp %dil # sched: [1:0.50]
-; ATOM-NEXT:    setl %dil # sched: [1:0.50]
-; ATOM-NEXT:    setge %dil # sched: [1:0.50]
-; ATOM-NEXT:    setle %dil # sched: [1:0.50]
-; ATOM-NEXT:    setg %dil # sched: [1:0.50]
-; ATOM-NEXT:    seto (%rsi) # sched: [2:1.00]
-; ATOM-NEXT:    setno (%rsi) # sched: [2:1.00]
-; ATOM-NEXT:    setb (%rsi) # sched: [2:1.00]
-; ATOM-NEXT:    setae (%rsi) # sched: [2:1.00]
-; ATOM-NEXT:    sete (%rsi) # sched: [2:1.00]
-; ATOM-NEXT:    setne (%rsi) # sched: [2:1.00]
-; ATOM-NEXT:    setbe (%rsi) # sched: [2:1.00]
-; ATOM-NEXT:    seta (%rsi) # sched: [2:1.00]
-; ATOM-NEXT:    sets (%rsi) # sched: [2:1.00]
-; ATOM-NEXT:    setns (%rsi) # sched: [2:1.00]
-; ATOM-NEXT:    setp (%rsi) # sched: [2:1.00]
-; ATOM-NEXT:    setnp (%rsi) # sched: [2:1.00]
-; ATOM-NEXT:    setl (%rsi) # sched: [2:1.00]
-; ATOM-NEXT:    setge (%rsi) # sched: [2:1.00]
-; ATOM-NEXT:    setle (%rsi) # sched: [2:1.00]
-; ATOM-NEXT:    setg (%rsi) # sched: [2:1.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_setcc:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    seto %dil # sched: [1:0.50]
-; SLM-NEXT:    setno %dil # sched: [1:0.50]
-; SLM-NEXT:    setb %dil # sched: [1:0.50]
-; SLM-NEXT:    setae %dil # sched: [1:0.50]
-; SLM-NEXT:    sete %dil # sched: [1:0.50]
-; SLM-NEXT:    setne %dil # sched: [1:0.50]
-; SLM-NEXT:    setbe %dil # sched: [1:0.50]
-; SLM-NEXT:    seta %dil # sched: [1:0.50]
-; SLM-NEXT:    sets %dil # sched: [1:0.50]
-; SLM-NEXT:    setns %dil # sched: [1:0.50]
-; SLM-NEXT:    setp %dil # sched: [1:0.50]
-; SLM-NEXT:    setnp %dil # sched: [1:0.50]
-; SLM-NEXT:    setl %dil # sched: [1:0.50]
-; SLM-NEXT:    setge %dil # sched: [1:0.50]
-; SLM-NEXT:    setle %dil # sched: [1:0.50]
-; SLM-NEXT:    setg %dil # sched: [1:0.50]
-; SLM-NEXT:    seto (%rsi) # sched: [1:1.00]
-; SLM-NEXT:    setno (%rsi) # sched: [1:1.00]
-; SLM-NEXT:    setb (%rsi) # sched: [1:1.00]
-; SLM-NEXT:    setae (%rsi) # sched: [1:1.00]
-; SLM-NEXT:    sete (%rsi) # sched: [1:1.00]
-; SLM-NEXT:    setne (%rsi) # sched: [1:1.00]
-; SLM-NEXT:    setbe (%rsi) # sched: [1:1.00]
-; SLM-NEXT:    seta (%rsi) # sched: [1:1.00]
-; SLM-NEXT:    sets (%rsi) # sched: [1:1.00]
-; SLM-NEXT:    setns (%rsi) # sched: [1:1.00]
-; SLM-NEXT:    setp (%rsi) # sched: [1:1.00]
-; SLM-NEXT:    setnp (%rsi) # sched: [1:1.00]
-; SLM-NEXT:    setl (%rsi) # sched: [1:1.00]
-; SLM-NEXT:    setge (%rsi) # sched: [1:1.00]
-; SLM-NEXT:    setle (%rsi) # sched: [1:1.00]
-; SLM-NEXT:    setg (%rsi) # sched: [1:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_setcc:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    seto %dil # sched: [1:0.50]
-; SANDY-NEXT:    setno %dil # sched: [1:0.50]
-; SANDY-NEXT:    setb %dil # sched: [1:0.50]
-; SANDY-NEXT:    setae %dil # sched: [1:0.50]
-; SANDY-NEXT:    sete %dil # sched: [1:0.50]
-; SANDY-NEXT:    setne %dil # sched: [1:0.50]
-; SANDY-NEXT:    setbe %dil # sched: [2:1.00]
-; SANDY-NEXT:    seta %dil # sched: [2:1.00]
-; SANDY-NEXT:    sets %dil # sched: [1:0.50]
-; SANDY-NEXT:    setns %dil # sched: [1:0.50]
-; SANDY-NEXT:    setp %dil # sched: [1:0.50]
-; SANDY-NEXT:    setnp %dil # sched: [1:0.50]
-; SANDY-NEXT:    setl %dil # sched: [1:0.50]
-; SANDY-NEXT:    setge %dil # sched: [1:0.50]
-; SANDY-NEXT:    setle %dil # sched: [1:0.50]
-; SANDY-NEXT:    setg %dil # sched: [1:0.50]
-; SANDY-NEXT:    seto (%rsi) # sched: [2:1.00]
-; SANDY-NEXT:    setno (%rsi) # sched: [2:1.00]
-; SANDY-NEXT:    setb (%rsi) # sched: [2:1.00]
-; SANDY-NEXT:    setae (%rsi) # sched: [2:1.00]
-; SANDY-NEXT:    sete (%rsi) # sched: [2:1.00]
-; SANDY-NEXT:    setne (%rsi) # sched: [2:1.00]
-; SANDY-NEXT:    setbe (%rsi) # sched: [3:1.00]
-; SANDY-NEXT:    seta (%rsi) # sched: [3:1.00]
-; SANDY-NEXT:    sets (%rsi) # sched: [2:1.00]
-; SANDY-NEXT:    setns (%rsi) # sched: [2:1.00]
-; SANDY-NEXT:    setp (%rsi) # sched: [2:1.00]
-; SANDY-NEXT:    setnp (%rsi) # sched: [2:1.00]
-; SANDY-NEXT:    setl (%rsi) # sched: [2:1.00]
-; SANDY-NEXT:    setge (%rsi) # sched: [2:1.00]
-; SANDY-NEXT:    setle (%rsi) # sched: [2:1.00]
-; SANDY-NEXT:    setg (%rsi) # sched: [2:1.00]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_setcc:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    seto %dil # sched: [1:0.50]
-; HASWELL-NEXT:    setno %dil # sched: [1:0.50]
-; HASWELL-NEXT:    setb %dil # sched: [1:0.50]
-; HASWELL-NEXT:    setae %dil # sched: [1:0.50]
-; HASWELL-NEXT:    sete %dil # sched: [1:0.50]
-; HASWELL-NEXT:    setne %dil # sched: [1:0.50]
-; HASWELL-NEXT:    setbe %dil # sched: [2:0.50]
-; HASWELL-NEXT:    seta %dil # sched: [2:0.50]
-; HASWELL-NEXT:    sets %dil # sched: [1:0.50]
-; HASWELL-NEXT:    setns %dil # sched: [1:0.50]
-; HASWELL-NEXT:    setp %dil # sched: [1:0.50]
-; HASWELL-NEXT:    setnp %dil # sched: [1:0.50]
-; HASWELL-NEXT:    setl %dil # sched: [1:0.50]
-; HASWELL-NEXT:    setge %dil # sched: [1:0.50]
-; HASWELL-NEXT:    setle %dil # sched: [1:0.50]
-; HASWELL-NEXT:    setg %dil # sched: [1:0.50]
-; HASWELL-NEXT:    seto (%rsi) # sched: [2:1.00]
-; HASWELL-NEXT:    setno (%rsi) # sched: [2:1.00]
-; HASWELL-NEXT:    setb (%rsi) # sched: [2:1.00]
-; HASWELL-NEXT:    setae (%rsi) # sched: [2:1.00]
-; HASWELL-NEXT:    sete (%rsi) # sched: [2:1.00]
-; HASWELL-NEXT:    setne (%rsi) # sched: [2:1.00]
-; HASWELL-NEXT:    setbe (%rsi) # sched: [3:1.00]
-; HASWELL-NEXT:    seta (%rsi) # sched: [3:1.00]
-; HASWELL-NEXT:    sets (%rsi) # sched: [2:1.00]
-; HASWELL-NEXT:    setns (%rsi) # sched: [2:1.00]
-; HASWELL-NEXT:    setp (%rsi) # sched: [2:1.00]
-; HASWELL-NEXT:    setnp (%rsi) # sched: [2:1.00]
-; HASWELL-NEXT:    setl (%rsi) # sched: [2:1.00]
-; HASWELL-NEXT:    setge (%rsi) # sched: [2:1.00]
-; HASWELL-NEXT:    setle (%rsi) # sched: [2:1.00]
-; HASWELL-NEXT:    setg (%rsi) # sched: [2:1.00]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_setcc:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    seto %dil # sched: [1:0.50]
-; BROADWELL-NEXT:    setno %dil # sched: [1:0.50]
-; BROADWELL-NEXT:    setb %dil # sched: [1:0.50]
-; BROADWELL-NEXT:    setae %dil # sched: [1:0.50]
-; BROADWELL-NEXT:    sete %dil # sched: [1:0.50]
-; BROADWELL-NEXT:    setne %dil # sched: [1:0.50]
-; BROADWELL-NEXT:    setbe %dil # sched: [2:0.50]
-; BROADWELL-NEXT:    seta %dil # sched: [2:0.50]
-; BROADWELL-NEXT:    sets %dil # sched: [1:0.50]
-; BROADWELL-NEXT:    setns %dil # sched: [1:0.50]
-; BROADWELL-NEXT:    setp %dil # sched: [1:0.50]
-; BROADWELL-NEXT:    setnp %dil # sched: [1:0.50]
-; BROADWELL-NEXT:    setl %dil # sched: [1:0.50]
-; BROADWELL-NEXT:    setge %dil # sched: [1:0.50]
-; BROADWELL-NEXT:    setle %dil # sched: [1:0.50]
-; BROADWELL-NEXT:    setg %dil # sched: [1:0.50]
-; BROADWELL-NEXT:    seto (%rsi) # sched: [2:1.00]
-; BROADWELL-NEXT:    setno (%rsi) # sched: [2:1.00]
-; BROADWELL-NEXT:    setb (%rsi) # sched: [2:1.00]
-; BROADWELL-NEXT:    setae (%rsi) # sched: [2:1.00]
-; BROADWELL-NEXT:    sete (%rsi) # sched: [2:1.00]
-; BROADWELL-NEXT:    setne (%rsi) # sched: [2:1.00]
-; BROADWELL-NEXT:    setbe (%rsi) # sched: [3:1.00]
-; BROADWELL-NEXT:    seta (%rsi) # sched: [3:1.00]
-; BROADWELL-NEXT:    sets (%rsi) # sched: [2:1.00]
-; BROADWELL-NEXT:    setns (%rsi) # sched: [2:1.00]
-; BROADWELL-NEXT:    setp (%rsi) # sched: [2:1.00]
-; BROADWELL-NEXT:    setnp (%rsi) # sched: [2:1.00]
-; BROADWELL-NEXT:    setl (%rsi) # sched: [2:1.00]
-; BROADWELL-NEXT:    setge (%rsi) # sched: [2:1.00]
-; BROADWELL-NEXT:    setle (%rsi) # sched: [2:1.00]
-; BROADWELL-NEXT:    setg (%rsi) # sched: [2:1.00]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_setcc:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    seto %dil # sched: [1:0.50]
-; SKYLAKE-NEXT:    setno %dil # sched: [1:0.50]
-; SKYLAKE-NEXT:    setb %dil # sched: [1:0.50]
-; SKYLAKE-NEXT:    setae %dil # sched: [1:0.50]
-; SKYLAKE-NEXT:    sete %dil # sched: [1:0.50]
-; SKYLAKE-NEXT:    setne %dil # sched: [1:0.50]
-; SKYLAKE-NEXT:    setbe %dil # sched: [2:1.00]
-; SKYLAKE-NEXT:    seta %dil # sched: [2:1.00]
-; SKYLAKE-NEXT:    sets %dil # sched: [1:0.50]
-; SKYLAKE-NEXT:    setns %dil # sched: [1:0.50]
-; SKYLAKE-NEXT:    setp %dil # sched: [1:0.50]
-; SKYLAKE-NEXT:    setnp %dil # sched: [1:0.50]
-; SKYLAKE-NEXT:    setl %dil # sched: [1:0.50]
-; SKYLAKE-NEXT:    setge %dil # sched: [1:0.50]
-; SKYLAKE-NEXT:    setle %dil # sched: [1:0.50]
-; SKYLAKE-NEXT:    setg %dil # sched: [1:0.50]
-; SKYLAKE-NEXT:    seto (%rsi) # sched: [2:1.00]
-; SKYLAKE-NEXT:    setno (%rsi) # sched: [2:1.00]
-; SKYLAKE-NEXT:    setb (%rsi) # sched: [2:1.00]
-; SKYLAKE-NEXT:    setae (%rsi) # sched: [2:1.00]
-; SKYLAKE-NEXT:    sete (%rsi) # sched: [2:1.00]
-; SKYLAKE-NEXT:    setne (%rsi) # sched: [2:1.00]
-; SKYLAKE-NEXT:    setbe (%rsi) # sched: [3:1.00]
-; SKYLAKE-NEXT:    seta (%rsi) # sched: [3:1.00]
-; SKYLAKE-NEXT:    sets (%rsi) # sched: [2:1.00]
-; SKYLAKE-NEXT:    setns (%rsi) # sched: [2:1.00]
-; SKYLAKE-NEXT:    setp (%rsi) # sched: [2:1.00]
-; SKYLAKE-NEXT:    setnp (%rsi) # sched: [2:1.00]
-; SKYLAKE-NEXT:    setl (%rsi) # sched: [2:1.00]
-; SKYLAKE-NEXT:    setge (%rsi) # sched: [2:1.00]
-; SKYLAKE-NEXT:    setle (%rsi) # sched: [2:1.00]
-; SKYLAKE-NEXT:    setg (%rsi) # sched: [2:1.00]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_setcc:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    seto %dil # sched: [1:0.50]
-; SKX-NEXT:    setno %dil # sched: [1:0.50]
-; SKX-NEXT:    setb %dil # sched: [1:0.50]
-; SKX-NEXT:    setae %dil # sched: [1:0.50]
-; SKX-NEXT:    sete %dil # sched: [1:0.50]
-; SKX-NEXT:    setne %dil # sched: [1:0.50]
-; SKX-NEXT:    setbe %dil # sched: [2:1.00]
-; SKX-NEXT:    seta %dil # sched: [2:1.00]
-; SKX-NEXT:    sets %dil # sched: [1:0.50]
-; SKX-NEXT:    setns %dil # sched: [1:0.50]
-; SKX-NEXT:    setp %dil # sched: [1:0.50]
-; SKX-NEXT:    setnp %dil # sched: [1:0.50]
-; SKX-NEXT:    setl %dil # sched: [1:0.50]
-; SKX-NEXT:    setge %dil # sched: [1:0.50]
-; SKX-NEXT:    setle %dil # sched: [1:0.50]
-; SKX-NEXT:    setg %dil # sched: [1:0.50]
-; SKX-NEXT:    seto (%rsi) # sched: [2:1.00]
-; SKX-NEXT:    setno (%rsi) # sched: [2:1.00]
-; SKX-NEXT:    setb (%rsi) # sched: [2:1.00]
-; SKX-NEXT:    setae (%rsi) # sched: [2:1.00]
-; SKX-NEXT:    sete (%rsi) # sched: [2:1.00]
-; SKX-NEXT:    setne (%rsi) # sched: [2:1.00]
-; SKX-NEXT:    setbe (%rsi) # sched: [3:1.00]
-; SKX-NEXT:    seta (%rsi) # sched: [3:1.00]
-; SKX-NEXT:    sets (%rsi) # sched: [2:1.00]
-; SKX-NEXT:    setns (%rsi) # sched: [2:1.00]
-; SKX-NEXT:    setp (%rsi) # sched: [2:1.00]
-; SKX-NEXT:    setnp (%rsi) # sched: [2:1.00]
-; SKX-NEXT:    setl (%rsi) # sched: [2:1.00]
-; SKX-NEXT:    setge (%rsi) # sched: [2:1.00]
-; SKX-NEXT:    setle (%rsi) # sched: [2:1.00]
-; SKX-NEXT:    setg (%rsi) # sched: [2:1.00]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_setcc:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    seto %dil # sched: [1:0.50]
-; BDVER2-NEXT:    setno %dil # sched: [1:0.50]
-; BDVER2-NEXT:    setb %dil # sched: [1:0.50]
-; BDVER2-NEXT:    setae %dil # sched: [1:0.50]
-; BDVER2-NEXT:    sete %dil # sched: [1:0.50]
-; BDVER2-NEXT:    setne %dil # sched: [1:0.50]
-; BDVER2-NEXT:    setbe %dil # sched: [1:0.50]
-; BDVER2-NEXT:    seta %dil # sched: [1:0.50]
-; BDVER2-NEXT:    sets %dil # sched: [1:0.50]
-; BDVER2-NEXT:    setns %dil # sched: [1:0.50]
-; BDVER2-NEXT:    setp %dil # sched: [1:0.50]
-; BDVER2-NEXT:    setnp %dil # sched: [1:0.50]
-; BDVER2-NEXT:    setl %dil # sched: [1:0.50]
-; BDVER2-NEXT:    setge %dil # sched: [1:0.50]
-; BDVER2-NEXT:    setle %dil # sched: [1:0.50]
-; BDVER2-NEXT:    setg %dil # sched: [1:0.50]
-; BDVER2-NEXT:    seto (%rsi) # sched: [1:1.00]
-; BDVER2-NEXT:    setno (%rsi) # sched: [1:1.00]
-; BDVER2-NEXT:    setb (%rsi) # sched: [1:1.00]
-; BDVER2-NEXT:    setae (%rsi) # sched: [1:1.00]
-; BDVER2-NEXT:    sete (%rsi) # sched: [1:1.00]
-; BDVER2-NEXT:    setne (%rsi) # sched: [1:1.00]
-; BDVER2-NEXT:    setbe (%rsi) # sched: [1:1.00]
-; BDVER2-NEXT:    seta (%rsi) # sched: [1:1.00]
-; BDVER2-NEXT:    sets (%rsi) # sched: [1:1.00]
-; BDVER2-NEXT:    setns (%rsi) # sched: [1:1.00]
-; BDVER2-NEXT:    setp (%rsi) # sched: [1:1.00]
-; BDVER2-NEXT:    setnp (%rsi) # sched: [1:1.00]
-; BDVER2-NEXT:    setl (%rsi) # sched: [1:1.00]
-; BDVER2-NEXT:    setge (%rsi) # sched: [1:1.00]
-; BDVER2-NEXT:    setle (%rsi) # sched: [1:1.00]
-; BDVER2-NEXT:    setg (%rsi) # sched: [1:1.00]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_setcc:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    seto %dil # sched: [1:0.50]
-; BTVER2-NEXT:    setno %dil # sched: [1:0.50]
-; BTVER2-NEXT:    setb %dil # sched: [1:0.50]
-; BTVER2-NEXT:    setae %dil # sched: [1:0.50]
-; BTVER2-NEXT:    sete %dil # sched: [1:0.50]
-; BTVER2-NEXT:    setne %dil # sched: [1:0.50]
-; BTVER2-NEXT:    setbe %dil # sched: [1:0.50]
-; BTVER2-NEXT:    seta %dil # sched: [1:0.50]
-; BTVER2-NEXT:    sets %dil # sched: [1:0.50]
-; BTVER2-NEXT:    setns %dil # sched: [1:0.50]
-; BTVER2-NEXT:    setp %dil # sched: [1:0.50]
-; BTVER2-NEXT:    setnp %dil # sched: [1:0.50]
-; BTVER2-NEXT:    setl %dil # sched: [1:0.50]
-; BTVER2-NEXT:    setge %dil # sched: [1:0.50]
-; BTVER2-NEXT:    setle %dil # sched: [1:0.50]
-; BTVER2-NEXT:    setg %dil # sched: [1:0.50]
-; BTVER2-NEXT:    seto (%rsi) # sched: [1:1.00]
-; BTVER2-NEXT:    setno (%rsi) # sched: [1:1.00]
-; BTVER2-NEXT:    setb (%rsi) # sched: [1:1.00]
-; BTVER2-NEXT:    setae (%rsi) # sched: [1:1.00]
-; BTVER2-NEXT:    sete (%rsi) # sched: [1:1.00]
-; BTVER2-NEXT:    setne (%rsi) # sched: [1:1.00]
-; BTVER2-NEXT:    setbe (%rsi) # sched: [1:1.00]
-; BTVER2-NEXT:    seta (%rsi) # sched: [1:1.00]
-; BTVER2-NEXT:    sets (%rsi) # sched: [1:1.00]
-; BTVER2-NEXT:    setns (%rsi) # sched: [1:1.00]
-; BTVER2-NEXT:    setp (%rsi) # sched: [1:1.00]
-; BTVER2-NEXT:    setnp (%rsi) # sched: [1:1.00]
-; BTVER2-NEXT:    setl (%rsi) # sched: [1:1.00]
-; BTVER2-NEXT:    setge (%rsi) # sched: [1:1.00]
-; BTVER2-NEXT:    setle (%rsi) # sched: [1:1.00]
-; BTVER2-NEXT:    setg (%rsi) # sched: [1:1.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_setcc:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    seto %dil # sched: [1:0.25]
-; ZNVER1-NEXT:    setno %dil # sched: [1:0.25]
-; ZNVER1-NEXT:    setb %dil # sched: [1:0.25]
-; ZNVER1-NEXT:    setae %dil # sched: [1:0.25]
-; ZNVER1-NEXT:    sete %dil # sched: [1:0.25]
-; ZNVER1-NEXT:    setne %dil # sched: [1:0.25]
-; ZNVER1-NEXT:    setbe %dil # sched: [1:0.25]
-; ZNVER1-NEXT:    seta %dil # sched: [1:0.25]
-; ZNVER1-NEXT:    sets %dil # sched: [1:0.25]
-; ZNVER1-NEXT:    setns %dil # sched: [1:0.25]
-; ZNVER1-NEXT:    setp %dil # sched: [1:0.25]
-; ZNVER1-NEXT:    setnp %dil # sched: [1:0.25]
-; ZNVER1-NEXT:    setl %dil # sched: [1:0.25]
-; ZNVER1-NEXT:    setge %dil # sched: [1:0.25]
-; ZNVER1-NEXT:    setle %dil # sched: [1:0.25]
-; ZNVER1-NEXT:    setg %dil # sched: [1:0.25]
-; ZNVER1-NEXT:    seto (%rsi) # sched: [1:0.50]
-; ZNVER1-NEXT:    setno (%rsi) # sched: [1:0.50]
-; ZNVER1-NEXT:    setb (%rsi) # sched: [1:0.50]
-; ZNVER1-NEXT:    setae (%rsi) # sched: [1:0.50]
-; ZNVER1-NEXT:    sete (%rsi) # sched: [1:0.50]
-; ZNVER1-NEXT:    setne (%rsi) # sched: [1:0.50]
-; ZNVER1-NEXT:    setbe (%rsi) # sched: [1:0.50]
-; ZNVER1-NEXT:    seta (%rsi) # sched: [1:0.50]
-; ZNVER1-NEXT:    sets (%rsi) # sched: [1:0.50]
-; ZNVER1-NEXT:    setns (%rsi) # sched: [1:0.50]
-; ZNVER1-NEXT:    setp (%rsi) # sched: [1:0.50]
-; ZNVER1-NEXT:    setnp (%rsi) # sched: [1:0.50]
-; ZNVER1-NEXT:    setl (%rsi) # sched: [1:0.50]
-; ZNVER1-NEXT:    setge (%rsi) # sched: [1:0.50]
-; ZNVER1-NEXT:    setle (%rsi) # sched: [1:0.50]
-; ZNVER1-NEXT:    setg (%rsi) # sched: [1:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  call void asm sideeffect "seto $0 \0A\09 setno $0 \0A\09 setb $0 \0A\09 setnb $0 \0A\09 setz $0 \0A\09 setnz $0 \0A\09 setbe $0 \0A\09 setnbe $0 \0A\09 sets $0 \0A\09 setns $0 \0A\09 setp $0 \0A\09 setnp $0 \0A\09 setl $0 \0A\09 setnl $0 \0A\09 setle $0 \0A\09 setnle $0 \0A\09 seto $1 \0A\09 setno $1 \0A\09 setb $1 \0A\09 setnb $1 \0A\09 setz $1 \0A\09 setnz $1 \0A\09 setbe $1 \0A\09 setnbe $1 \0A\09 sets $1 \0A\09 setns $1 \0A\09 setp $1 \0A\09 setnp $1 \0A\09 setl $1 \0A\09 setnl $1 \0A\09 setle $1 \0A\09 setnle $1", "r,*m"(i8 %a0, i8 *%a1)
-  ret void
-}
-
-; TODO - test_sgdt
-
-define void @test_shld_shrd_16(i16 %a0, i16 %a1, i16 *%a2) optsize {
-; GENERIC-LABEL: test_shld_shrd_16:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    shldw %cl, %si, %di # sched: [4:1.50]
-; GENERIC-NEXT:    shrdw %cl, %si, %di # sched: [4:1.50]
-; GENERIC-NEXT:    shldw %cl, %si, (%rdx) # sched: [10:1.50]
-; GENERIC-NEXT:    shrdw %cl, %si, (%rdx) # sched: [10:1.50]
-; GENERIC-NEXT:    shldw $7, %si, %di # sched: [2:0.67]
-; GENERIC-NEXT:    shrdw $7, %si, %di # sched: [2:0.67]
-; GENERIC-NEXT:    shldw $7, %si, (%rdx) # sched: [8:1.00]
-; GENERIC-NEXT:    shrdw $7, %si, (%rdx) # sched: [8:1.00]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_shld_shrd_16:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    shldw %cl, %si, %di # sched: [6:3.00]
-; ATOM-NEXT:    shrdw %cl, %si, %di # sched: [6:3.00]
-; ATOM-NEXT:    shldw %cl, %si, (%rdx) # sched: [6:3.00]
-; ATOM-NEXT:    shrdw %cl, %si, (%rdx) # sched: [6:3.00]
-; ATOM-NEXT:    shldw $7, %si, %di # sched: [6:3.00]
-; ATOM-NEXT:    shrdw $7, %si, %di # sched: [6:3.00]
-; ATOM-NEXT:    shldw $7, %si, (%rdx) # sched: [6:3.00]
-; ATOM-NEXT:    shrdw $7, %si, (%rdx) # sched: [6:3.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_shld_shrd_16:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    shldw %cl, %si, %di # sched: [1:1.00]
-; SLM-NEXT:    shrdw %cl, %si, %di # sched: [1:1.00]
-; SLM-NEXT:    shldw %cl, %si, (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    shrdw %cl, %si, (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    shldw $7, %si, %di # sched: [1:1.00]
-; SLM-NEXT:    shrdw $7, %si, %di # sched: [1:1.00]
-; SLM-NEXT:    shldw $7, %si, (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    shrdw $7, %si, (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_shld_shrd_16:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    shldw %cl, %si, %di # sched: [4:1.50]
-; SANDY-NEXT:    shrdw %cl, %si, %di # sched: [4:1.50]
-; SANDY-NEXT:    shldw %cl, %si, (%rdx) # sched: [10:1.50]
-; SANDY-NEXT:    shrdw %cl, %si, (%rdx) # sched: [10:1.50]
-; SANDY-NEXT:    shldw $7, %si, %di # sched: [2:0.67]
-; SANDY-NEXT:    shrdw $7, %si, %di # sched: [2:0.67]
-; SANDY-NEXT:    shldw $7, %si, (%rdx) # sched: [8:1.00]
-; SANDY-NEXT:    shrdw $7, %si, (%rdx) # sched: [8:1.00]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_shld_shrd_16:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    shldw %cl, %si, %di # sched: [6:1.00]
-; HASWELL-NEXT:    shrdw %cl, %si, %di # sched: [6:1.00]
-; HASWELL-NEXT:    shldw %cl, %si, (%rdx) # sched: [12:1.00]
-; HASWELL-NEXT:    shrdw %cl, %si, (%rdx) # sched: [12:1.00]
-; HASWELL-NEXT:    shldw $7, %si, %di # sched: [3:1.00]
-; HASWELL-NEXT:    shrdw $7, %si, %di # sched: [3:1.00]
-; HASWELL-NEXT:    shldw $7, %si, (%rdx) # sched: [10:1.00]
-; HASWELL-NEXT:    shrdw $7, %si, (%rdx) # sched: [10:1.00]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_shld_shrd_16:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    shldw %cl, %si, %di # sched: [6:1.00]
-; BROADWELL-NEXT:    shrdw %cl, %si, %di # sched: [6:1.00]
-; BROADWELL-NEXT:    shldw %cl, %si, (%rdx) # sched: [11:1.00]
-; BROADWELL-NEXT:    shrdw %cl, %si, (%rdx) # sched: [11:1.00]
-; BROADWELL-NEXT:    shldw $7, %si, %di # sched: [3:1.00]
-; BROADWELL-NEXT:    shrdw $7, %si, %di # sched: [3:1.00]
-; BROADWELL-NEXT:    shldw $7, %si, (%rdx) # sched: [9:1.00]
-; BROADWELL-NEXT:    shrdw $7, %si, (%rdx) # sched: [9:1.00]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_shld_shrd_16:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    shldw %cl, %si, %di # sched: [6:1.00]
-; SKYLAKE-NEXT:    shrdw %cl, %si, %di # sched: [6:1.00]
-; SKYLAKE-NEXT:    shldw %cl, %si, (%rdx) # sched: [11:1.00]
-; SKYLAKE-NEXT:    shrdw %cl, %si, (%rdx) # sched: [11:1.00]
-; SKYLAKE-NEXT:    shldw $7, %si, %di # sched: [3:1.00]
-; SKYLAKE-NEXT:    shrdw $7, %si, %di # sched: [3:1.00]
-; SKYLAKE-NEXT:    shldw $7, %si, (%rdx) # sched: [9:1.00]
-; SKYLAKE-NEXT:    shrdw $7, %si, (%rdx) # sched: [9:1.00]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_shld_shrd_16:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    shldw %cl, %si, %di # sched: [6:1.00]
-; SKX-NEXT:    shrdw %cl, %si, %di # sched: [6:1.00]
-; SKX-NEXT:    shldw %cl, %si, (%rdx) # sched: [11:1.00]
-; SKX-NEXT:    shrdw %cl, %si, (%rdx) # sched: [11:1.00]
-; SKX-NEXT:    shldw $7, %si, %di # sched: [3:1.00]
-; SKX-NEXT:    shrdw $7, %si, %di # sched: [3:1.00]
-; SKX-NEXT:    shldw $7, %si, (%rdx) # sched: [9:1.00]
-; SKX-NEXT:    shrdw $7, %si, (%rdx) # sched: [9:1.00]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_shld_shrd_16:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    shldw %cl, %si, %di # sched: [4:4.00]
-; BDVER2-NEXT:    shrdw %cl, %si, %di # sched: [4:4.00]
-; BDVER2-NEXT:    shldw %cl, %si, (%rdx) # sched: [4:11.00]
-; BDVER2-NEXT:    shrdw %cl, %si, (%rdx) # sched: [4:11.00]
-; BDVER2-NEXT:    shldw $7, %si, %di # sched: [4:3.00]
-; BDVER2-NEXT:    shrdw $7, %si, %di # sched: [3:3.00]
-; BDVER2-NEXT:    shldw $7, %si, (%rdx) # sched: [4:11.00]
-; BDVER2-NEXT:    shrdw $7, %si, (%rdx) # sched: [4:11.00]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_shld_shrd_16:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    shldw %cl, %si, %di # sched: [4:4.00]
-; BTVER2-NEXT:    shrdw %cl, %si, %di # sched: [4:4.00]
-; BTVER2-NEXT:    shldw %cl, %si, (%rdx) # sched: [9:11.00]
-; BTVER2-NEXT:    shrdw %cl, %si, (%rdx) # sched: [9:11.00]
-; BTVER2-NEXT:    shldw $7, %si, %di # sched: [3:3.00]
-; BTVER2-NEXT:    shrdw $7, %si, %di # sched: [3:3.00]
-; BTVER2-NEXT:    shldw $7, %si, (%rdx) # sched: [9:11.00]
-; BTVER2-NEXT:    shrdw $7, %si, (%rdx) # sched: [9:11.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_shld_shrd_16:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    shldw %cl, %si, %di # sched: [100:0.25]
-; ZNVER1-NEXT:    shrdw %cl, %si, %di # sched: [100:0.25]
-; ZNVER1-NEXT:    shldw %cl, %si, (%rdx) # sched: [100:0.25]
-; ZNVER1-NEXT:    shrdw %cl, %si, (%rdx) # sched: [100:0.25]
-; ZNVER1-NEXT:    shldw $7, %si, %di # sched: [1:0.25]
-; ZNVER1-NEXT:    shrdw $7, %si, %di # sched: [1:0.25]
-; ZNVER1-NEXT:    shldw $7, %si, (%rdx) # sched: [5:0.50]
-; ZNVER1-NEXT:    shrdw $7, %si, (%rdx) # sched: [5:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  call void asm sideeffect "shld $1, $0 \0A\09 shrd $1, $0 \0A\09 shld $1, $2 \0A\09 shrd $1, $2 \0A\09 shld $3, $1, $0 \0A\09 shrd $3, $1, $0 \0A\09 shld $3, $1, $2 \0A\09 shrd $3, $1, $2", "r,r,*m,i"(i16 %a0, i16 %a1, i16 *%a2, i8 7)
-  ret void
-}
-define void @test_shld_shrd_32(i32 %a0, i32 %a1, i32 *%a2) optsize {
-; GENERIC-LABEL: test_shld_shrd_32:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    shldl %cl, %esi, %edi # sched: [4:1.50]
-; GENERIC-NEXT:    shrdl %cl, %esi, %edi # sched: [4:1.50]
-; GENERIC-NEXT:    shldl %cl, %esi, (%rdx) # sched: [10:1.50]
-; GENERIC-NEXT:    shrdl %cl, %esi, (%rdx) # sched: [10:1.50]
-; GENERIC-NEXT:    shldl $7, %esi, %edi # sched: [2:0.67]
-; GENERIC-NEXT:    shrdl $7, %esi, %edi # sched: [2:0.67]
-; GENERIC-NEXT:    shldl $7, %esi, (%rdx) # sched: [8:1.00]
-; GENERIC-NEXT:    shrdl $7, %esi, (%rdx) # sched: [8:1.00]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_shld_shrd_32:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    shldl %cl, %esi, %edi # sched: [2:1.00]
-; ATOM-NEXT:    shrdl %cl, %esi, %edi # sched: [2:1.00]
-; ATOM-NEXT:    shldl %cl, %esi, (%rdx) # sched: [4:2.00]
-; ATOM-NEXT:    shrdl %cl, %esi, (%rdx) # sched: [4:2.00]
-; ATOM-NEXT:    shldl $7, %esi, %edi # sched: [2:1.00]
-; ATOM-NEXT:    shrdl $7, %esi, %edi # sched: [2:1.00]
-; ATOM-NEXT:    shldl $7, %esi, (%rdx) # sched: [4:2.00]
-; ATOM-NEXT:    shrdl $7, %esi, (%rdx) # sched: [4:2.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_shld_shrd_32:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    shldl %cl, %esi, %edi # sched: [1:1.00]
-; SLM-NEXT:    shrdl %cl, %esi, %edi # sched: [1:1.00]
-; SLM-NEXT:    shldl %cl, %esi, (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    shrdl %cl, %esi, (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    shldl $7, %esi, %edi # sched: [1:1.00]
-; SLM-NEXT:    shrdl $7, %esi, %edi # sched: [1:1.00]
-; SLM-NEXT:    shldl $7, %esi, (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    shrdl $7, %esi, (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_shld_shrd_32:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    shldl %cl, %esi, %edi # sched: [4:1.50]
-; SANDY-NEXT:    shrdl %cl, %esi, %edi # sched: [4:1.50]
-; SANDY-NEXT:    shldl %cl, %esi, (%rdx) # sched: [10:1.50]
-; SANDY-NEXT:    shrdl %cl, %esi, (%rdx) # sched: [10:1.50]
-; SANDY-NEXT:    shldl $7, %esi, %edi # sched: [2:0.67]
-; SANDY-NEXT:    shrdl $7, %esi, %edi # sched: [2:0.67]
-; SANDY-NEXT:    shldl $7, %esi, (%rdx) # sched: [8:1.00]
-; SANDY-NEXT:    shrdl $7, %esi, (%rdx) # sched: [8:1.00]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_shld_shrd_32:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    shldl %cl, %esi, %edi # sched: [6:1.00]
-; HASWELL-NEXT:    shrdl %cl, %esi, %edi # sched: [6:1.00]
-; HASWELL-NEXT:    shldl %cl, %esi, (%rdx) # sched: [12:1.00]
-; HASWELL-NEXT:    shrdl %cl, %esi, (%rdx) # sched: [12:1.00]
-; HASWELL-NEXT:    shldl $7, %esi, %edi # sched: [3:1.00]
-; HASWELL-NEXT:    shrdl $7, %esi, %edi # sched: [3:1.00]
-; HASWELL-NEXT:    shldl $7, %esi, (%rdx) # sched: [10:1.00]
-; HASWELL-NEXT:    shrdl $7, %esi, (%rdx) # sched: [10:1.00]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_shld_shrd_32:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    shldl %cl, %esi, %edi # sched: [6:1.00]
-; BROADWELL-NEXT:    shrdl %cl, %esi, %edi # sched: [6:1.00]
-; BROADWELL-NEXT:    shldl %cl, %esi, (%rdx) # sched: [11:1.00]
-; BROADWELL-NEXT:    shrdl %cl, %esi, (%rdx) # sched: [11:1.00]
-; BROADWELL-NEXT:    shldl $7, %esi, %edi # sched: [3:1.00]
-; BROADWELL-NEXT:    shrdl $7, %esi, %edi # sched: [3:1.00]
-; BROADWELL-NEXT:    shldl $7, %esi, (%rdx) # sched: [9:1.00]
-; BROADWELL-NEXT:    shrdl $7, %esi, (%rdx) # sched: [9:1.00]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_shld_shrd_32:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    shldl %cl, %esi, %edi # sched: [6:1.00]
-; SKYLAKE-NEXT:    shrdl %cl, %esi, %edi # sched: [6:1.00]
-; SKYLAKE-NEXT:    shldl %cl, %esi, (%rdx) # sched: [11:1.00]
-; SKYLAKE-NEXT:    shrdl %cl, %esi, (%rdx) # sched: [11:1.00]
-; SKYLAKE-NEXT:    shldl $7, %esi, %edi # sched: [3:1.00]
-; SKYLAKE-NEXT:    shrdl $7, %esi, %edi # sched: [3:1.00]
-; SKYLAKE-NEXT:    shldl $7, %esi, (%rdx) # sched: [9:1.00]
-; SKYLAKE-NEXT:    shrdl $7, %esi, (%rdx) # sched: [9:1.00]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_shld_shrd_32:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    shldl %cl, %esi, %edi # sched: [6:1.00]
-; SKX-NEXT:    shrdl %cl, %esi, %edi # sched: [6:1.00]
-; SKX-NEXT:    shldl %cl, %esi, (%rdx) # sched: [11:1.00]
-; SKX-NEXT:    shrdl %cl, %esi, (%rdx) # sched: [11:1.00]
-; SKX-NEXT:    shldl $7, %esi, %edi # sched: [3:1.00]
-; SKX-NEXT:    shrdl $7, %esi, %edi # sched: [3:1.00]
-; SKX-NEXT:    shldl $7, %esi, (%rdx) # sched: [9:1.00]
-; SKX-NEXT:    shrdl $7, %esi, (%rdx) # sched: [9:1.00]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_shld_shrd_32:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    shldl %cl, %esi, %edi # sched: [4:4.00]
-; BDVER2-NEXT:    shrdl %cl, %esi, %edi # sched: [4:4.00]
-; BDVER2-NEXT:    shldl %cl, %esi, (%rdx) # sched: [4:11.00]
-; BDVER2-NEXT:    shrdl %cl, %esi, (%rdx) # sched: [4:11.00]
-; BDVER2-NEXT:    shldl $7, %esi, %edi # sched: [3:3.00]
-; BDVER2-NEXT:    shrdl $7, %esi, %edi # sched: [4:3.00]
-; BDVER2-NEXT:    shldl $7, %esi, (%rdx) # sched: [4:11.00]
-; BDVER2-NEXT:    shrdl $7, %esi, (%rdx) # sched: [4:11.00]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_shld_shrd_32:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    shldl %cl, %esi, %edi # sched: [4:4.00]
-; BTVER2-NEXT:    shrdl %cl, %esi, %edi # sched: [4:4.00]
-; BTVER2-NEXT:    shldl %cl, %esi, (%rdx) # sched: [9:11.00]
-; BTVER2-NEXT:    shrdl %cl, %esi, (%rdx) # sched: [9:11.00]
-; BTVER2-NEXT:    shldl $7, %esi, %edi # sched: [3:3.00]
-; BTVER2-NEXT:    shrdl $7, %esi, %edi # sched: [3:3.00]
-; BTVER2-NEXT:    shldl $7, %esi, (%rdx) # sched: [9:11.00]
-; BTVER2-NEXT:    shrdl $7, %esi, (%rdx) # sched: [9:11.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_shld_shrd_32:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    shldl %cl, %esi, %edi # sched: [100:0.25]
-; ZNVER1-NEXT:    shrdl %cl, %esi, %edi # sched: [100:0.25]
-; ZNVER1-NEXT:    shldl %cl, %esi, (%rdx) # sched: [100:0.25]
-; ZNVER1-NEXT:    shrdl %cl, %esi, (%rdx) # sched: [100:0.25]
-; ZNVER1-NEXT:    shldl $7, %esi, %edi # sched: [1:0.25]
-; ZNVER1-NEXT:    shrdl $7, %esi, %edi # sched: [1:0.25]
-; ZNVER1-NEXT:    shldl $7, %esi, (%rdx) # sched: [5:0.50]
-; ZNVER1-NEXT:    shrdl $7, %esi, (%rdx) # sched: [5:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  call void asm sideeffect "shld $1, $0 \0A\09 shrd $1, $0 \0A\09 shld $1, $2 \0A\09 shrd $1, $2 \0A\09 shld $3, $1, $0 \0A\09 shrd $3, $1, $0 \0A\09 shld $3, $1, $2 \0A\09 shrd $3, $1, $2", "r,r,*m,i"(i32 %a0, i32 %a1, i32 *%a2, i8 7)
-  ret void
-}
-define void @test_shld_shrd_64(i64 %a0, i64 %a1, i64 *%a2) optsize {
-; GENERIC-LABEL: test_shld_shrd_64:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    shldq %cl, %rsi, %rdi # sched: [4:1.50]
-; GENERIC-NEXT:    shrdq %cl, %rsi, %rdi # sched: [4:1.50]
-; GENERIC-NEXT:    shldq %cl, %rsi, (%rdx) # sched: [10:1.50]
-; GENERIC-NEXT:    shrdq %cl, %rsi, (%rdx) # sched: [10:1.50]
-; GENERIC-NEXT:    shldq $7, %rsi, %rdi # sched: [2:0.67]
-; GENERIC-NEXT:    shrdq $7, %rsi, %rdi # sched: [2:0.67]
-; GENERIC-NEXT:    shldq $7, %rsi, (%rdx) # sched: [8:1.00]
-; GENERIC-NEXT:    shrdq $7, %rsi, (%rdx) # sched: [8:1.00]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_shld_shrd_64:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    shldq %cl, %rsi, %rdi # sched: [8:4.00]
-; ATOM-NEXT:    shrdq %cl, %rsi, %rdi # sched: [8:4.00]
-; ATOM-NEXT:    shldq %cl, %rsi, (%rdx) # sched: [9:4.50]
-; ATOM-NEXT:    shrdq %cl, %rsi, (%rdx) # sched: [9:4.50]
-; ATOM-NEXT:    shldq $7, %rsi, %rdi # sched: [9:4.50]
-; ATOM-NEXT:    shrdq $7, %rsi, %rdi # sched: [9:4.50]
-; ATOM-NEXT:    shldq $7, %rsi, (%rdx) # sched: [9:4.50]
-; ATOM-NEXT:    shrdq $7, %rsi, (%rdx) # sched: [9:4.50]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_shld_shrd_64:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    shldq %cl, %rsi, %rdi # sched: [1:1.00]
-; SLM-NEXT:    shrdq %cl, %rsi, %rdi # sched: [1:1.00]
-; SLM-NEXT:    shldq %cl, %rsi, (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    shrdq %cl, %rsi, (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    shldq $7, %rsi, %rdi # sched: [1:1.00]
-; SLM-NEXT:    shrdq $7, %rsi, %rdi # sched: [1:1.00]
-; SLM-NEXT:    shldq $7, %rsi, (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    shrdq $7, %rsi, (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_shld_shrd_64:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    shldq %cl, %rsi, %rdi # sched: [4:1.50]
-; SANDY-NEXT:    shrdq %cl, %rsi, %rdi # sched: [4:1.50]
-; SANDY-NEXT:    shldq %cl, %rsi, (%rdx) # sched: [10:1.50]
-; SANDY-NEXT:    shrdq %cl, %rsi, (%rdx) # sched: [10:1.50]
-; SANDY-NEXT:    shldq $7, %rsi, %rdi # sched: [2:0.67]
-; SANDY-NEXT:    shrdq $7, %rsi, %rdi # sched: [2:0.67]
-; SANDY-NEXT:    shldq $7, %rsi, (%rdx) # sched: [8:1.00]
-; SANDY-NEXT:    shrdq $7, %rsi, (%rdx) # sched: [8:1.00]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_shld_shrd_64:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    shldq %cl, %rsi, %rdi # sched: [6:1.00]
-; HASWELL-NEXT:    shrdq %cl, %rsi, %rdi # sched: [6:1.00]
-; HASWELL-NEXT:    shldq %cl, %rsi, (%rdx) # sched: [12:1.00]
-; HASWELL-NEXT:    shrdq %cl, %rsi, (%rdx) # sched: [12:1.00]
-; HASWELL-NEXT:    shldq $7, %rsi, %rdi # sched: [3:1.00]
-; HASWELL-NEXT:    shrdq $7, %rsi, %rdi # sched: [3:1.00]
-; HASWELL-NEXT:    shldq $7, %rsi, (%rdx) # sched: [10:1.00]
-; HASWELL-NEXT:    shrdq $7, %rsi, (%rdx) # sched: [10:1.00]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_shld_shrd_64:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    shldq %cl, %rsi, %rdi # sched: [6:1.00]
-; BROADWELL-NEXT:    shrdq %cl, %rsi, %rdi # sched: [6:1.00]
-; BROADWELL-NEXT:    shldq %cl, %rsi, (%rdx) # sched: [11:1.00]
-; BROADWELL-NEXT:    shrdq %cl, %rsi, (%rdx) # sched: [11:1.00]
-; BROADWELL-NEXT:    shldq $7, %rsi, %rdi # sched: [3:1.00]
-; BROADWELL-NEXT:    shrdq $7, %rsi, %rdi # sched: [3:1.00]
-; BROADWELL-NEXT:    shldq $7, %rsi, (%rdx) # sched: [9:1.00]
-; BROADWELL-NEXT:    shrdq $7, %rsi, (%rdx) # sched: [9:1.00]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_shld_shrd_64:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    shldq %cl, %rsi, %rdi # sched: [6:1.00]
-; SKYLAKE-NEXT:    shrdq %cl, %rsi, %rdi # sched: [6:1.00]
-; SKYLAKE-NEXT:    shldq %cl, %rsi, (%rdx) # sched: [11:1.00]
-; SKYLAKE-NEXT:    shrdq %cl, %rsi, (%rdx) # sched: [11:1.00]
-; SKYLAKE-NEXT:    shldq $7, %rsi, %rdi # sched: [3:1.00]
-; SKYLAKE-NEXT:    shrdq $7, %rsi, %rdi # sched: [3:1.00]
-; SKYLAKE-NEXT:    shldq $7, %rsi, (%rdx) # sched: [9:1.00]
-; SKYLAKE-NEXT:    shrdq $7, %rsi, (%rdx) # sched: [9:1.00]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_shld_shrd_64:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    shldq %cl, %rsi, %rdi # sched: [6:1.00]
-; SKX-NEXT:    shrdq %cl, %rsi, %rdi # sched: [6:1.00]
-; SKX-NEXT:    shldq %cl, %rsi, (%rdx) # sched: [11:1.00]
-; SKX-NEXT:    shrdq %cl, %rsi, (%rdx) # sched: [11:1.00]
-; SKX-NEXT:    shldq $7, %rsi, %rdi # sched: [3:1.00]
-; SKX-NEXT:    shrdq $7, %rsi, %rdi # sched: [3:1.00]
-; SKX-NEXT:    shldq $7, %rsi, (%rdx) # sched: [9:1.00]
-; SKX-NEXT:    shrdq $7, %rsi, (%rdx) # sched: [9:1.00]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_shld_shrd_64:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    shldq %cl, %rsi, %rdi # sched: [4:4.00]
-; BDVER2-NEXT:    shrdq %cl, %rsi, %rdi # sched: [4:4.00]
-; BDVER2-NEXT:    shldq %cl, %rsi, (%rdx) # sched: [4:11.00]
-; BDVER2-NEXT:    shrdq %cl, %rsi, (%rdx) # sched: [4:11.00]
-; BDVER2-NEXT:    shldq $7, %rsi, %rdi # sched: [4:3.00]
-; BDVER2-NEXT:    shrdq $7, %rsi, %rdi # sched: [4:3.00]
-; BDVER2-NEXT:    shldq $7, %rsi, (%rdx) # sched: [4:11.00]
-; BDVER2-NEXT:    shrdq $7, %rsi, (%rdx) # sched: [4:11.00]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_shld_shrd_64:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    shldq %cl, %rsi, %rdi # sched: [4:4.00]
-; BTVER2-NEXT:    shrdq %cl, %rsi, %rdi # sched: [4:4.00]
-; BTVER2-NEXT:    shldq %cl, %rsi, (%rdx) # sched: [9:11.00]
-; BTVER2-NEXT:    shrdq %cl, %rsi, (%rdx) # sched: [9:11.00]
-; BTVER2-NEXT:    shldq $7, %rsi, %rdi # sched: [3:3.00]
-; BTVER2-NEXT:    shrdq $7, %rsi, %rdi # sched: [3:3.00]
-; BTVER2-NEXT:    shldq $7, %rsi, (%rdx) # sched: [9:11.00]
-; BTVER2-NEXT:    shrdq $7, %rsi, (%rdx) # sched: [9:11.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_shld_shrd_64:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    shldq %cl, %rsi, %rdi # sched: [100:0.25]
-; ZNVER1-NEXT:    shrdq %cl, %rsi, %rdi # sched: [100:0.25]
-; ZNVER1-NEXT:    shldq %cl, %rsi, (%rdx) # sched: [100:0.25]
-; ZNVER1-NEXT:    shrdq %cl, %rsi, (%rdx) # sched: [100:0.25]
-; ZNVER1-NEXT:    shldq $7, %rsi, %rdi # sched: [1:0.25]
-; ZNVER1-NEXT:    shrdq $7, %rsi, %rdi # sched: [1:0.25]
-; ZNVER1-NEXT:    shldq $7, %rsi, (%rdx) # sched: [5:0.50]
-; ZNVER1-NEXT:    shrdq $7, %rsi, (%rdx) # sched: [5:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  call void asm sideeffect "shld $1, $0 \0A\09 shrd $1, $0 \0A\09 shld $1, $2 \0A\09 shrd $1, $2 \0A\09 shld $3, $1, $0 \0A\09 shrd $3, $1, $0 \0A\09 shld $3, $1, $2 \0A\09 shrd $3, $1, $2", "r,r,*m,i"(i64 %a0, i64 %a1, i64 *%a2, i8 7)
-  ret void
-}
-
-; TODO - test_sidt
-; TODO - test_sldt
-; TODO - test_smsw
-
-define void @test_stc_std() optsize {
-; GENERIC-LABEL: test_stc_std:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    stc # sched: [1:0.33]
-; GENERIC-NEXT:    std # sched: [1:0.33]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_stc_std:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    stc # sched: [1:0.50]
-; ATOM-NEXT:    std # sched: [21:10.50]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_stc_std:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    stc # sched: [1:0.50]
-; SLM-NEXT:    std # sched: [1:0.50]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_stc_std:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    stc # sched: [1:0.33]
-; SANDY-NEXT:    std # sched: [1:0.33]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_stc_std:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    stc # sched: [1:0.25]
-; HASWELL-NEXT:    std # sched: [6:1.50]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_stc_std:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    stc # sched: [1:0.25]
-; BROADWELL-NEXT:    std # sched: [6:1.50]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_stc_std:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    stc # sched: [1:0.25]
-; SKYLAKE-NEXT:    std # sched: [6:1.50]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_stc_std:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    stc # sched: [1:0.25]
-; SKX-NEXT:    std # sched: [6:1.50]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_stc_std:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    stc # sched: [1:0.50]
-; BDVER2-NEXT:    std # sched: [1:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_stc_std:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    stc # sched: [1:0.50]
-; BTVER2-NEXT:    std # sched: [1:0.50]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_stc_std:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    stc # sched: [1:0.25]
-; ZNVER1-NEXT:    std # sched: [1:0.25]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  call void asm sideeffect "stc \0A\09 std", ""()
-  ret void
-}
-
-; TODO - test_sti
-; TODO - test_stgi
-
-define void @test_stos() optsize {
-; GENERIC-LABEL: test_stos:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    stosb %al, %es:(%rdi) # sched: [5:1.00]
-; GENERIC-NEXT:    stosw %ax, %es:(%rdi) # sched: [5:1.00]
-; GENERIC-NEXT:    stosl %eax, %es:(%rdi) # sched: [5:1.00]
-; GENERIC-NEXT:    stosq %rax, %es:(%rdi) # sched: [5:1.00]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_stos:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    stosb %al, %es:(%rdi) # sched: [1:0.50]
-; ATOM-NEXT:    stosw %ax, %es:(%rdi) # sched: [1:0.50]
-; ATOM-NEXT:    stosl %eax, %es:(%rdi) # sched: [1:0.50]
-; ATOM-NEXT:    stosq %rax, %es:(%rdi) # sched: [1:0.50]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_stos:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    stosb %al, %es:(%rdi) # sched: [100:1.00]
-; SLM-NEXT:    stosw %ax, %es:(%rdi) # sched: [100:1.00]
-; SLM-NEXT:    stosl %eax, %es:(%rdi) # sched: [100:1.00]
-; SLM-NEXT:    stosq %rax, %es:(%rdi) # sched: [100:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_stos:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    stosb %al, %es:(%rdi) # sched: [5:1.00]
-; SANDY-NEXT:    stosw %ax, %es:(%rdi) # sched: [5:1.00]
-; SANDY-NEXT:    stosl %eax, %es:(%rdi) # sched: [5:1.00]
-; SANDY-NEXT:    stosq %rax, %es:(%rdi) # sched: [5:1.00]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_stos:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    stosb %al, %es:(%rdi) # sched: [2:1.00]
-; HASWELL-NEXT:    stosw %ax, %es:(%rdi) # sched: [2:1.00]
-; HASWELL-NEXT:    stosl %eax, %es:(%rdi) # sched: [2:1.00]
-; HASWELL-NEXT:    stosq %rax, %es:(%rdi) # sched: [2:1.00]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_stos:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    stosb %al, %es:(%rdi) # sched: [2:1.00]
-; BROADWELL-NEXT:    stosw %ax, %es:(%rdi) # sched: [2:1.00]
-; BROADWELL-NEXT:    stosl %eax, %es:(%rdi) # sched: [2:1.00]
-; BROADWELL-NEXT:    stosq %rax, %es:(%rdi) # sched: [2:1.00]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_stos:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    stosb %al, %es:(%rdi) # sched: [2:1.00]
-; SKYLAKE-NEXT:    stosw %ax, %es:(%rdi) # sched: [2:1.00]
-; SKYLAKE-NEXT:    stosl %eax, %es:(%rdi) # sched: [2:1.00]
-; SKYLAKE-NEXT:    stosq %rax, %es:(%rdi) # sched: [2:1.00]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_stos:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    stosb %al, %es:(%rdi) # sched: [2:1.00]
-; SKX-NEXT:    stosw %ax, %es:(%rdi) # sched: [2:1.00]
-; SKX-NEXT:    stosl %eax, %es:(%rdi) # sched: [2:1.00]
-; SKX-NEXT:    stosq %rax, %es:(%rdi) # sched: [2:1.00]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_stos:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    stosb %al, %es:(%rdi) # sched: [100:0.50]
-; BDVER2-NEXT:    stosw %ax, %es:(%rdi) # sched: [100:0.50]
-; BDVER2-NEXT:    stosl %eax, %es:(%rdi) # sched: [100:0.50]
-; BDVER2-NEXT:    stosq %rax, %es:(%rdi) # sched: [100:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_stos:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    stosb %al, %es:(%rdi) # sched: [100:0.50]
-; BTVER2-NEXT:    stosw %ax, %es:(%rdi) # sched: [100:0.50]
-; BTVER2-NEXT:    stosl %eax, %es:(%rdi) # sched: [100:0.50]
-; BTVER2-NEXT:    stosq %rax, %es:(%rdi) # sched: [100:0.50]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_stos:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    stosb %al, %es:(%rdi) # sched: [100:0.25]
-; ZNVER1-NEXT:    stosw %ax, %es:(%rdi) # sched: [100:0.25]
-; ZNVER1-NEXT:    stosl %eax, %es:(%rdi) # sched: [100:0.25]
-; ZNVER1-NEXT:    stosq %rax, %es:(%rdi) # sched: [100:0.25]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  call void asm sideeffect "stosb \0A\09 stosw \0A\09 stosl \0A\09 stosq", ""()
-  ret void
-}
-
-; TODO - test_str
-
-define void @test_sub_8(i8 %a0, i8* %a1, i8 %a2) optsize {
-; GENERIC-LABEL: test_sub_8:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    subb $7, %al # sched: [1:0.33]
-; GENERIC-NEXT:    subb $7, %dil # sched: [1:0.33]
-; GENERIC-NEXT:    subb $7, (%rsi) # sched: [7:1.00]
-; GENERIC-NEXT:    subb %dl, %dil # sched: [1:0.33]
-; GENERIC-NEXT:    subb %dil, (%rsi) # sched: [7:1.00]
-; GENERIC-NEXT:    subb (%rsi), %dil # sched: [6:0.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_sub_8:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    subb $7, %al # sched: [1:0.50]
-; ATOM-NEXT:    subb $7, %dil # sched: [1:0.50]
-; ATOM-NEXT:    subb $7, (%rsi) # sched: [1:1.00]
-; ATOM-NEXT:    subb %dl, %dil # sched: [1:0.50]
-; ATOM-NEXT:    subb %dil, (%rsi) # sched: [1:1.00]
-; ATOM-NEXT:    subb (%rsi), %dil # sched: [1:1.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_sub_8:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    subb $7, %al # sched: [1:0.50]
-; SLM-NEXT:    subb $7, %dil # sched: [1:0.50]
-; SLM-NEXT:    subb $7, (%rsi) # sched: [5:2.00]
-; SLM-NEXT:    subb %dl, %dil # sched: [1:0.50]
-; SLM-NEXT:    subb %dil, (%rsi) # sched: [5:2.00]
-; SLM-NEXT:    subb (%rsi), %dil # sched: [4:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_sub_8:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    subb $7, %al # sched: [1:0.33]
-; SANDY-NEXT:    subb $7, %dil # sched: [1:0.33]
-; SANDY-NEXT:    subb $7, (%rsi) # sched: [7:1.00]
-; SANDY-NEXT:    subb %dl, %dil # sched: [1:0.33]
-; SANDY-NEXT:    subb %dil, (%rsi) # sched: [7:1.00]
-; SANDY-NEXT:    subb (%rsi), %dil # sched: [6:0.50]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_sub_8:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    subb $7, %al # sched: [1:0.25]
-; HASWELL-NEXT:    subb $7, %dil # sched: [1:0.25]
-; HASWELL-NEXT:    subb $7, (%rsi) # sched: [7:1.00]
-; HASWELL-NEXT:    subb %dl, %dil # sched: [1:0.25]
-; HASWELL-NEXT:    subb %dil, (%rsi) # sched: [7:1.00]
-; HASWELL-NEXT:    subb (%rsi), %dil # sched: [6:0.50]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_sub_8:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    subb $7, %al # sched: [1:0.25]
-; BROADWELL-NEXT:    subb $7, %dil # sched: [1:0.25]
-; BROADWELL-NEXT:    subb $7, (%rsi) # sched: [7:1.00]
-; BROADWELL-NEXT:    subb %dl, %dil # sched: [1:0.25]
-; BROADWELL-NEXT:    subb %dil, (%rsi) # sched: [7:1.00]
-; BROADWELL-NEXT:    subb (%rsi), %dil # sched: [6:0.50]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_sub_8:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    subb $7, %al # sched: [1:0.25]
-; SKYLAKE-NEXT:    subb $7, %dil # sched: [1:0.25]
-; SKYLAKE-NEXT:    subb $7, (%rsi) # sched: [7:1.00]
-; SKYLAKE-NEXT:    subb %dl, %dil # sched: [1:0.25]
-; SKYLAKE-NEXT:    subb %dil, (%rsi) # sched: [7:1.00]
-; SKYLAKE-NEXT:    subb (%rsi), %dil # sched: [6:0.50]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_sub_8:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    subb $7, %al # sched: [1:0.25]
-; SKX-NEXT:    subb $7, %dil # sched: [1:0.25]
-; SKX-NEXT:    subb $7, (%rsi) # sched: [7:1.00]
-; SKX-NEXT:    subb %dl, %dil # sched: [1:0.25]
-; SKX-NEXT:    subb %dil, (%rsi) # sched: [7:1.00]
-; SKX-NEXT:    subb (%rsi), %dil # sched: [6:0.50]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_sub_8:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    subb $7, %al # sched: [1:0.50]
-; BDVER2-NEXT:    subb $7, %dil # sched: [1:0.50]
-; BDVER2-NEXT:    subb $7, (%rsi) # sched: [6:1.00]
-; BDVER2-NEXT:    subb %dl, %dil # sched: [1:0.50]
-; BDVER2-NEXT:    subb %dil, (%rsi) # sched: [6:1.00]
-; BDVER2-NEXT:    subb (%rsi), %dil # sched: [5:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_sub_8:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    subb $7, %al # sched: [1:0.50]
-; BTVER2-NEXT:    subb $7, %dil # sched: [1:0.50]
-; BTVER2-NEXT:    subb $7, (%rsi) # sched: [5:1.00]
-; BTVER2-NEXT:    subb %dl, %dil # sched: [1:0.50]
-; BTVER2-NEXT:    subb %dil, (%rsi) # sched: [5:1.00]
-; BTVER2-NEXT:    subb (%rsi), %dil # sched: [4:1.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_sub_8:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    subb $7, %al # sched: [1:0.25]
-; ZNVER1-NEXT:    subb $7, %dil # sched: [1:0.25]
-; ZNVER1-NEXT:    subb $7, (%rsi) # sched: [5:0.50]
-; ZNVER1-NEXT:    subb %dl, %dil # sched: [1:0.25]
-; ZNVER1-NEXT:    subb %dil, (%rsi) # sched: [5:0.50]
-; ZNVER1-NEXT:    subb (%rsi), %dil # sched: [5:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  tail call void asm "subb $3, %AL \0A\09 subb $3, $0 \0A\09 subb $3, $2 \0A\09 subb $1, $0 \0A\09 subb $0, $2 \0A\09 subb $2, $0", "r,r,*m,i"(i8 %a0, i8 %a2, i8* %a1, i8 7) nounwind
-  ret void
-}
-define void @test_sub_16(i16 %a0, i16* %a1, i16 %a2) optsize {
-; GENERIC-LABEL: test_sub_16:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    subw $511, %ax # imm = 0x1FF
-; GENERIC-NEXT:    # sched: [1:0.33]
-; GENERIC-NEXT:    subw $511, %di # imm = 0x1FF
-; GENERIC-NEXT:    # sched: [1:0.33]
-; GENERIC-NEXT:    subw $511, (%rsi) # imm = 0x1FF
-; GENERIC-NEXT:    # sched: [7:1.00]
-; GENERIC-NEXT:    subw $7, %di # sched: [1:0.33]
-; GENERIC-NEXT:    subw $7, (%rsi) # sched: [7:1.00]
-; GENERIC-NEXT:    subw %dx, %di # sched: [1:0.33]
-; GENERIC-NEXT:    subw %di, (%rsi) # sched: [7:1.00]
-; GENERIC-NEXT:    subw (%rsi), %di # sched: [6:0.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_sub_16:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    subw $511, %ax # imm = 0x1FF
-; ATOM-NEXT:    # sched: [1:0.50]
-; ATOM-NEXT:    subw $511, %di # imm = 0x1FF
-; ATOM-NEXT:    # sched: [1:0.50]
-; ATOM-NEXT:    subw $511, (%rsi) # imm = 0x1FF
-; ATOM-NEXT:    # sched: [1:1.00]
-; ATOM-NEXT:    subw $7, %di # sched: [1:0.50]
-; ATOM-NEXT:    subw $7, (%rsi) # sched: [1:1.00]
-; ATOM-NEXT:    subw %dx, %di # sched: [1:0.50]
-; ATOM-NEXT:    subw %di, (%rsi) # sched: [1:1.00]
-; ATOM-NEXT:    subw (%rsi), %di # sched: [1:1.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_sub_16:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    subw $511, %ax # imm = 0x1FF
-; SLM-NEXT:    # sched: [1:0.50]
-; SLM-NEXT:    subw $511, %di # imm = 0x1FF
-; SLM-NEXT:    # sched: [1:0.50]
-; SLM-NEXT:    subw $511, (%rsi) # imm = 0x1FF
-; SLM-NEXT:    # sched: [5:2.00]
-; SLM-NEXT:    subw $7, %di # sched: [1:0.50]
-; SLM-NEXT:    subw $7, (%rsi) # sched: [5:2.00]
-; SLM-NEXT:    subw %dx, %di # sched: [1:0.50]
-; SLM-NEXT:    subw %di, (%rsi) # sched: [5:2.00]
-; SLM-NEXT:    subw (%rsi), %di # sched: [4:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_sub_16:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    subw $511, %ax # imm = 0x1FF
-; SANDY-NEXT:    # sched: [1:0.33]
-; SANDY-NEXT:    subw $511, %di # imm = 0x1FF
-; SANDY-NEXT:    # sched: [1:0.33]
-; SANDY-NEXT:    subw $511, (%rsi) # imm = 0x1FF
-; SANDY-NEXT:    # sched: [7:1.00]
-; SANDY-NEXT:    subw $7, %di # sched: [1:0.33]
-; SANDY-NEXT:    subw $7, (%rsi) # sched: [7:1.00]
-; SANDY-NEXT:    subw %dx, %di # sched: [1:0.33]
-; SANDY-NEXT:    subw %di, (%rsi) # sched: [7:1.00]
-; SANDY-NEXT:    subw (%rsi), %di # sched: [6:0.50]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_sub_16:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    subw $511, %ax # imm = 0x1FF
-; HASWELL-NEXT:    # sched: [1:0.25]
-; HASWELL-NEXT:    subw $511, %di # imm = 0x1FF
-; HASWELL-NEXT:    # sched: [1:0.25]
-; HASWELL-NEXT:    subw $511, (%rsi) # imm = 0x1FF
-; HASWELL-NEXT:    # sched: [7:1.00]
-; HASWELL-NEXT:    subw $7, %di # sched: [1:0.25]
-; HASWELL-NEXT:    subw $7, (%rsi) # sched: [7:1.00]
-; HASWELL-NEXT:    subw %dx, %di # sched: [1:0.25]
-; HASWELL-NEXT:    subw %di, (%rsi) # sched: [7:1.00]
-; HASWELL-NEXT:    subw (%rsi), %di # sched: [6:0.50]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_sub_16:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    subw $511, %ax # imm = 0x1FF
-; BROADWELL-NEXT:    # sched: [1:0.25]
-; BROADWELL-NEXT:    subw $511, %di # imm = 0x1FF
-; BROADWELL-NEXT:    # sched: [1:0.25]
-; BROADWELL-NEXT:    subw $511, (%rsi) # imm = 0x1FF
-; BROADWELL-NEXT:    # sched: [7:1.00]
-; BROADWELL-NEXT:    subw $7, %di # sched: [1:0.25]
-; BROADWELL-NEXT:    subw $7, (%rsi) # sched: [7:1.00]
-; BROADWELL-NEXT:    subw %dx, %di # sched: [1:0.25]
-; BROADWELL-NEXT:    subw %di, (%rsi) # sched: [7:1.00]
-; BROADWELL-NEXT:    subw (%rsi), %di # sched: [6:0.50]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_sub_16:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    subw $511, %ax # imm = 0x1FF
-; SKYLAKE-NEXT:    # sched: [1:0.25]
-; SKYLAKE-NEXT:    subw $511, %di # imm = 0x1FF
-; SKYLAKE-NEXT:    # sched: [1:0.25]
-; SKYLAKE-NEXT:    subw $511, (%rsi) # imm = 0x1FF
-; SKYLAKE-NEXT:    # sched: [7:1.00]
-; SKYLAKE-NEXT:    subw $7, %di # sched: [1:0.25]
-; SKYLAKE-NEXT:    subw $7, (%rsi) # sched: [7:1.00]
-; SKYLAKE-NEXT:    subw %dx, %di # sched: [1:0.25]
-; SKYLAKE-NEXT:    subw %di, (%rsi) # sched: [7:1.00]
-; SKYLAKE-NEXT:    subw (%rsi), %di # sched: [6:0.50]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_sub_16:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    subw $511, %ax # imm = 0x1FF
-; SKX-NEXT:    # sched: [1:0.25]
-; SKX-NEXT:    subw $511, %di # imm = 0x1FF
-; SKX-NEXT:    # sched: [1:0.25]
-; SKX-NEXT:    subw $511, (%rsi) # imm = 0x1FF
-; SKX-NEXT:    # sched: [7:1.00]
-; SKX-NEXT:    subw $7, %di # sched: [1:0.25]
-; SKX-NEXT:    subw $7, (%rsi) # sched: [7:1.00]
-; SKX-NEXT:    subw %dx, %di # sched: [1:0.25]
-; SKX-NEXT:    subw %di, (%rsi) # sched: [7:1.00]
-; SKX-NEXT:    subw (%rsi), %di # sched: [6:0.50]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_sub_16:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    subw $511, %ax # imm = 0x1FF
-; BDVER2-NEXT:    # sched: [1:0.50]
-; BDVER2-NEXT:    subw $511, %di # imm = 0x1FF
-; BDVER2-NEXT:    # sched: [1:0.50]
-; BDVER2-NEXT:    subw $511, (%rsi) # imm = 0x1FF
-; BDVER2-NEXT:    # sched: [6:1.00]
-; BDVER2-NEXT:    subw $7, %di # sched: [1:0.50]
-; BDVER2-NEXT:    subw $7, (%rsi) # sched: [6:1.00]
-; BDVER2-NEXT:    subw %dx, %di # sched: [1:0.50]
-; BDVER2-NEXT:    subw %di, (%rsi) # sched: [6:1.00]
-; BDVER2-NEXT:    subw (%rsi), %di # sched: [5:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_sub_16:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    subw $511, %ax # imm = 0x1FF
-; BTVER2-NEXT:    # sched: [1:0.50]
-; BTVER2-NEXT:    subw $511, %di # imm = 0x1FF
-; BTVER2-NEXT:    # sched: [1:0.50]
-; BTVER2-NEXT:    subw $511, (%rsi) # imm = 0x1FF
-; BTVER2-NEXT:    # sched: [5:1.00]
-; BTVER2-NEXT:    subw $7, %di # sched: [1:0.50]
-; BTVER2-NEXT:    subw $7, (%rsi) # sched: [5:1.00]
-; BTVER2-NEXT:    subw %dx, %di # sched: [1:0.50]
-; BTVER2-NEXT:    subw %di, (%rsi) # sched: [5:1.00]
-; BTVER2-NEXT:    subw (%rsi), %di # sched: [4:1.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_sub_16:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    subw $511, %ax # imm = 0x1FF
-; ZNVER1-NEXT:    # sched: [1:0.25]
-; ZNVER1-NEXT:    subw $511, %di # imm = 0x1FF
-; ZNVER1-NEXT:    # sched: [1:0.25]
-; ZNVER1-NEXT:    subw $511, (%rsi) # imm = 0x1FF
-; ZNVER1-NEXT:    # sched: [5:0.50]
-; ZNVER1-NEXT:    subw $7, %di # sched: [1:0.25]
-; ZNVER1-NEXT:    subw $7, (%rsi) # sched: [5:0.50]
-; ZNVER1-NEXT:    subw %dx, %di # sched: [1:0.25]
-; ZNVER1-NEXT:    subw %di, (%rsi) # sched: [5:0.50]
-; ZNVER1-NEXT:    subw (%rsi), %di # sched: [5:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  tail call void asm "subw $3, %AX \0A\09 subw $3, $0 \0A\09 subw $3, $2 \0A\09 subw $4, $0 \0A\09 subw $4, $2 \0A\09 subw $1, $0 \0A\09 subw $0, $2 \0A\09 subw $2, $0", "r,r,*m,i,i"(i16 %a0, i16 %a2, i16* %a1, i16 511, i8 7) nounwind
-  ret void
-}
-define void @test_sub_32(i32 %a0, i32* %a1, i32 %a2) optsize {
-; GENERIC-LABEL: test_sub_32:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    subl $665536, %eax # imm = 0xA27C0
-; GENERIC-NEXT:    # sched: [1:0.33]
-; GENERIC-NEXT:    subl $665536, %edi # imm = 0xA27C0
-; GENERIC-NEXT:    # sched: [1:0.33]
-; GENERIC-NEXT:    subl $665536, (%rsi) # imm = 0xA27C0
-; GENERIC-NEXT:    # sched: [7:1.00]
-; GENERIC-NEXT:    subl $7, %edi # sched: [1:0.33]
-; GENERIC-NEXT:    subl $7, (%rsi) # sched: [7:1.00]
-; GENERIC-NEXT:    subl %edx, %edi # sched: [1:0.33]
-; GENERIC-NEXT:    subl %edi, (%rsi) # sched: [7:1.00]
-; GENERIC-NEXT:    subl (%rsi), %edi # sched: [6:0.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_sub_32:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    subl $665536, %eax # imm = 0xA27C0
-; ATOM-NEXT:    # sched: [1:0.50]
-; ATOM-NEXT:    subl $665536, %edi # imm = 0xA27C0
-; ATOM-NEXT:    # sched: [1:0.50]
-; ATOM-NEXT:    subl $665536, (%rsi) # imm = 0xA27C0
-; ATOM-NEXT:    # sched: [1:1.00]
-; ATOM-NEXT:    subl $7, %edi # sched: [1:0.50]
-; ATOM-NEXT:    subl $7, (%rsi) # sched: [1:1.00]
-; ATOM-NEXT:    subl %edx, %edi # sched: [1:0.50]
-; ATOM-NEXT:    subl %edi, (%rsi) # sched: [1:1.00]
-; ATOM-NEXT:    subl (%rsi), %edi # sched: [1:1.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_sub_32:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    subl $665536, %eax # imm = 0xA27C0
-; SLM-NEXT:    # sched: [1:0.50]
-; SLM-NEXT:    subl $665536, %edi # imm = 0xA27C0
-; SLM-NEXT:    # sched: [1:0.50]
-; SLM-NEXT:    subl $665536, (%rsi) # imm = 0xA27C0
-; SLM-NEXT:    # sched: [5:2.00]
-; SLM-NEXT:    subl $7, %edi # sched: [1:0.50]
-; SLM-NEXT:    subl $7, (%rsi) # sched: [5:2.00]
-; SLM-NEXT:    subl %edx, %edi # sched: [1:0.50]
-; SLM-NEXT:    subl %edi, (%rsi) # sched: [5:2.00]
-; SLM-NEXT:    subl (%rsi), %edi # sched: [4:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_sub_32:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    subl $665536, %eax # imm = 0xA27C0
-; SANDY-NEXT:    # sched: [1:0.33]
-; SANDY-NEXT:    subl $665536, %edi # imm = 0xA27C0
-; SANDY-NEXT:    # sched: [1:0.33]
-; SANDY-NEXT:    subl $665536, (%rsi) # imm = 0xA27C0
-; SANDY-NEXT:    # sched: [7:1.00]
-; SANDY-NEXT:    subl $7, %edi # sched: [1:0.33]
-; SANDY-NEXT:    subl $7, (%rsi) # sched: [7:1.00]
-; SANDY-NEXT:    subl %edx, %edi # sched: [1:0.33]
-; SANDY-NEXT:    subl %edi, (%rsi) # sched: [7:1.00]
-; SANDY-NEXT:    subl (%rsi), %edi # sched: [6:0.50]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_sub_32:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    subl $665536, %eax # imm = 0xA27C0
-; HASWELL-NEXT:    # sched: [1:0.25]
-; HASWELL-NEXT:    subl $665536, %edi # imm = 0xA27C0
-; HASWELL-NEXT:    # sched: [1:0.25]
-; HASWELL-NEXT:    subl $665536, (%rsi) # imm = 0xA27C0
-; HASWELL-NEXT:    # sched: [7:1.00]
-; HASWELL-NEXT:    subl $7, %edi # sched: [1:0.25]
-; HASWELL-NEXT:    subl $7, (%rsi) # sched: [7:1.00]
-; HASWELL-NEXT:    subl %edx, %edi # sched: [1:0.25]
-; HASWELL-NEXT:    subl %edi, (%rsi) # sched: [7:1.00]
-; HASWELL-NEXT:    subl (%rsi), %edi # sched: [6:0.50]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_sub_32:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    subl $665536, %eax # imm = 0xA27C0
-; BROADWELL-NEXT:    # sched: [1:0.25]
-; BROADWELL-NEXT:    subl $665536, %edi # imm = 0xA27C0
-; BROADWELL-NEXT:    # sched: [1:0.25]
-; BROADWELL-NEXT:    subl $665536, (%rsi) # imm = 0xA27C0
-; BROADWELL-NEXT:    # sched: [7:1.00]
-; BROADWELL-NEXT:    subl $7, %edi # sched: [1:0.25]
-; BROADWELL-NEXT:    subl $7, (%rsi) # sched: [7:1.00]
-; BROADWELL-NEXT:    subl %edx, %edi # sched: [1:0.25]
-; BROADWELL-NEXT:    subl %edi, (%rsi) # sched: [7:1.00]
-; BROADWELL-NEXT:    subl (%rsi), %edi # sched: [6:0.50]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_sub_32:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    subl $665536, %eax # imm = 0xA27C0
-; SKYLAKE-NEXT:    # sched: [1:0.25]
-; SKYLAKE-NEXT:    subl $665536, %edi # imm = 0xA27C0
-; SKYLAKE-NEXT:    # sched: [1:0.25]
-; SKYLAKE-NEXT:    subl $665536, (%rsi) # imm = 0xA27C0
-; SKYLAKE-NEXT:    # sched: [7:1.00]
-; SKYLAKE-NEXT:    subl $7, %edi # sched: [1:0.25]
-; SKYLAKE-NEXT:    subl $7, (%rsi) # sched: [7:1.00]
-; SKYLAKE-NEXT:    subl %edx, %edi # sched: [1:0.25]
-; SKYLAKE-NEXT:    subl %edi, (%rsi) # sched: [7:1.00]
-; SKYLAKE-NEXT:    subl (%rsi), %edi # sched: [6:0.50]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_sub_32:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    subl $665536, %eax # imm = 0xA27C0
-; SKX-NEXT:    # sched: [1:0.25]
-; SKX-NEXT:    subl $665536, %edi # imm = 0xA27C0
-; SKX-NEXT:    # sched: [1:0.25]
-; SKX-NEXT:    subl $665536, (%rsi) # imm = 0xA27C0
-; SKX-NEXT:    # sched: [7:1.00]
-; SKX-NEXT:    subl $7, %edi # sched: [1:0.25]
-; SKX-NEXT:    subl $7, (%rsi) # sched: [7:1.00]
-; SKX-NEXT:    subl %edx, %edi # sched: [1:0.25]
-; SKX-NEXT:    subl %edi, (%rsi) # sched: [7:1.00]
-; SKX-NEXT:    subl (%rsi), %edi # sched: [6:0.50]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_sub_32:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    subl $665536, %eax # imm = 0xA27C0
-; BDVER2-NEXT:    # sched: [1:0.50]
-; BDVER2-NEXT:    subl $665536, %edi # imm = 0xA27C0
-; BDVER2-NEXT:    # sched: [1:0.50]
-; BDVER2-NEXT:    subl $665536, (%rsi) # imm = 0xA27C0
-; BDVER2-NEXT:    # sched: [6:1.00]
-; BDVER2-NEXT:    subl $7, %edi # sched: [1:0.50]
-; BDVER2-NEXT:    subl $7, (%rsi) # sched: [6:1.00]
-; BDVER2-NEXT:    subl %edx, %edi # sched: [1:0.50]
-; BDVER2-NEXT:    subl %edi, (%rsi) # sched: [6:1.00]
-; BDVER2-NEXT:    subl (%rsi), %edi # sched: [5:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_sub_32:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    subl $665536, %eax # imm = 0xA27C0
-; BTVER2-NEXT:    # sched: [1:0.50]
-; BTVER2-NEXT:    subl $665536, %edi # imm = 0xA27C0
-; BTVER2-NEXT:    # sched: [1:0.50]
-; BTVER2-NEXT:    subl $665536, (%rsi) # imm = 0xA27C0
-; BTVER2-NEXT:    # sched: [5:1.00]
-; BTVER2-NEXT:    subl $7, %edi # sched: [1:0.50]
-; BTVER2-NEXT:    subl $7, (%rsi) # sched: [5:1.00]
-; BTVER2-NEXT:    subl %edx, %edi # sched: [1:0.50]
-; BTVER2-NEXT:    subl %edi, (%rsi) # sched: [5:1.00]
-; BTVER2-NEXT:    subl (%rsi), %edi # sched: [4:1.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_sub_32:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    subl $665536, %eax # imm = 0xA27C0
-; ZNVER1-NEXT:    # sched: [1:0.25]
-; ZNVER1-NEXT:    subl $665536, %edi # imm = 0xA27C0
-; ZNVER1-NEXT:    # sched: [1:0.25]
-; ZNVER1-NEXT:    subl $665536, (%rsi) # imm = 0xA27C0
-; ZNVER1-NEXT:    # sched: [5:0.50]
-; ZNVER1-NEXT:    subl $7, %edi # sched: [1:0.25]
-; ZNVER1-NEXT:    subl $7, (%rsi) # sched: [5:0.50]
-; ZNVER1-NEXT:    subl %edx, %edi # sched: [1:0.25]
-; ZNVER1-NEXT:    subl %edi, (%rsi) # sched: [5:0.50]
-; ZNVER1-NEXT:    subl (%rsi), %edi # sched: [5:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  tail call void asm "subl $3, %EAX \0A\09 subl $3, $0 \0A\09 subl $3, $2 \0A\09 subl $4, $0 \0A\09 subl $4, $2 \0A\09 subl $1, $0 \0A\09 subl $0, $2 \0A\09 subl $2, $0", "r,r,*m,i,i"(i32 %a0, i32 %a2, i32* %a1, i32 665536, i8 7) nounwind
-  ret void
-}
-define void @test_sub_64(i64 %a0, i64* %a1, i64 %a2) optsize {
-; GENERIC-LABEL: test_sub_64:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    subq $665536, %rax # imm = 0xA27C0
-; GENERIC-NEXT:    # sched: [1:0.33]
-; GENERIC-NEXT:    subq $665536, %rdi # imm = 0xA27C0
-; GENERIC-NEXT:    # sched: [1:0.33]
-; GENERIC-NEXT:    subq $665536, (%rsi) # imm = 0xA27C0
-; GENERIC-NEXT:    # sched: [7:1.00]
-; GENERIC-NEXT:    subq $7, %rdi # sched: [1:0.33]
-; GENERIC-NEXT:    subq $7, (%rsi) # sched: [7:1.00]
-; GENERIC-NEXT:    subq %rdx, %rdi # sched: [1:0.33]
-; GENERIC-NEXT:    subq %rdi, (%rsi) # sched: [7:1.00]
-; GENERIC-NEXT:    subq (%rsi), %rdi # sched: [6:0.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_sub_64:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    subq $665536, %rax # imm = 0xA27C0
-; ATOM-NEXT:    # sched: [1:0.50]
-; ATOM-NEXT:    subq $665536, %rdi # imm = 0xA27C0
-; ATOM-NEXT:    # sched: [1:0.50]
-; ATOM-NEXT:    subq $665536, (%rsi) # imm = 0xA27C0
-; ATOM-NEXT:    # sched: [1:1.00]
-; ATOM-NEXT:    subq $7, %rdi # sched: [1:0.50]
-; ATOM-NEXT:    subq $7, (%rsi) # sched: [1:1.00]
-; ATOM-NEXT:    subq %rdx, %rdi # sched: [1:0.50]
-; ATOM-NEXT:    subq %rdi, (%rsi) # sched: [1:1.00]
-; ATOM-NEXT:    subq (%rsi), %rdi # sched: [1:1.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_sub_64:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    subq $665536, %rax # imm = 0xA27C0
-; SLM-NEXT:    # sched: [1:0.50]
-; SLM-NEXT:    subq $665536, %rdi # imm = 0xA27C0
-; SLM-NEXT:    # sched: [1:0.50]
-; SLM-NEXT:    subq $665536, (%rsi) # imm = 0xA27C0
-; SLM-NEXT:    # sched: [5:2.00]
-; SLM-NEXT:    subq $7, %rdi # sched: [1:0.50]
-; SLM-NEXT:    subq $7, (%rsi) # sched: [5:2.00]
-; SLM-NEXT:    subq %rdx, %rdi # sched: [1:0.50]
-; SLM-NEXT:    subq %rdi, (%rsi) # sched: [5:2.00]
-; SLM-NEXT:    subq (%rsi), %rdi # sched: [4:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_sub_64:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    subq $665536, %rax # imm = 0xA27C0
-; SANDY-NEXT:    # sched: [1:0.33]
-; SANDY-NEXT:    subq $665536, %rdi # imm = 0xA27C0
-; SANDY-NEXT:    # sched: [1:0.33]
-; SANDY-NEXT:    subq $665536, (%rsi) # imm = 0xA27C0
-; SANDY-NEXT:    # sched: [7:1.00]
-; SANDY-NEXT:    subq $7, %rdi # sched: [1:0.33]
-; SANDY-NEXT:    subq $7, (%rsi) # sched: [7:1.00]
-; SANDY-NEXT:    subq %rdx, %rdi # sched: [1:0.33]
-; SANDY-NEXT:    subq %rdi, (%rsi) # sched: [7:1.00]
-; SANDY-NEXT:    subq (%rsi), %rdi # sched: [6:0.50]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_sub_64:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    subq $665536, %rax # imm = 0xA27C0
-; HASWELL-NEXT:    # sched: [1:0.25]
-; HASWELL-NEXT:    subq $665536, %rdi # imm = 0xA27C0
-; HASWELL-NEXT:    # sched: [1:0.25]
-; HASWELL-NEXT:    subq $665536, (%rsi) # imm = 0xA27C0
-; HASWELL-NEXT:    # sched: [7:1.00]
-; HASWELL-NEXT:    subq $7, %rdi # sched: [1:0.25]
-; HASWELL-NEXT:    subq $7, (%rsi) # sched: [7:1.00]
-; HASWELL-NEXT:    subq %rdx, %rdi # sched: [1:0.25]
-; HASWELL-NEXT:    subq %rdi, (%rsi) # sched: [7:1.00]
-; HASWELL-NEXT:    subq (%rsi), %rdi # sched: [6:0.50]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_sub_64:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    subq $665536, %rax # imm = 0xA27C0
-; BROADWELL-NEXT:    # sched: [1:0.25]
-; BROADWELL-NEXT:    subq $665536, %rdi # imm = 0xA27C0
-; BROADWELL-NEXT:    # sched: [1:0.25]
-; BROADWELL-NEXT:    subq $665536, (%rsi) # imm = 0xA27C0
-; BROADWELL-NEXT:    # sched: [7:1.00]
-; BROADWELL-NEXT:    subq $7, %rdi # sched: [1:0.25]
-; BROADWELL-NEXT:    subq $7, (%rsi) # sched: [7:1.00]
-; BROADWELL-NEXT:    subq %rdx, %rdi # sched: [1:0.25]
-; BROADWELL-NEXT:    subq %rdi, (%rsi) # sched: [7:1.00]
-; BROADWELL-NEXT:    subq (%rsi), %rdi # sched: [6:0.50]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_sub_64:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    subq $665536, %rax # imm = 0xA27C0
-; SKYLAKE-NEXT:    # sched: [1:0.25]
-; SKYLAKE-NEXT:    subq $665536, %rdi # imm = 0xA27C0
-; SKYLAKE-NEXT:    # sched: [1:0.25]
-; SKYLAKE-NEXT:    subq $665536, (%rsi) # imm = 0xA27C0
-; SKYLAKE-NEXT:    # sched: [7:1.00]
-; SKYLAKE-NEXT:    subq $7, %rdi # sched: [1:0.25]
-; SKYLAKE-NEXT:    subq $7, (%rsi) # sched: [7:1.00]
-; SKYLAKE-NEXT:    subq %rdx, %rdi # sched: [1:0.25]
-; SKYLAKE-NEXT:    subq %rdi, (%rsi) # sched: [7:1.00]
-; SKYLAKE-NEXT:    subq (%rsi), %rdi # sched: [6:0.50]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_sub_64:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    subq $665536, %rax # imm = 0xA27C0
-; SKX-NEXT:    # sched: [1:0.25]
-; SKX-NEXT:    subq $665536, %rdi # imm = 0xA27C0
-; SKX-NEXT:    # sched: [1:0.25]
-; SKX-NEXT:    subq $665536, (%rsi) # imm = 0xA27C0
-; SKX-NEXT:    # sched: [7:1.00]
-; SKX-NEXT:    subq $7, %rdi # sched: [1:0.25]
-; SKX-NEXT:    subq $7, (%rsi) # sched: [7:1.00]
-; SKX-NEXT:    subq %rdx, %rdi # sched: [1:0.25]
-; SKX-NEXT:    subq %rdi, (%rsi) # sched: [7:1.00]
-; SKX-NEXT:    subq (%rsi), %rdi # sched: [6:0.50]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_sub_64:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    subq $665536, %rax # imm = 0xA27C0
-; BDVER2-NEXT:    # sched: [1:0.50]
-; BDVER2-NEXT:    subq $665536, %rdi # imm = 0xA27C0
-; BDVER2-NEXT:    # sched: [1:0.50]
-; BDVER2-NEXT:    subq $665536, (%rsi) # imm = 0xA27C0
-; BDVER2-NEXT:    # sched: [6:1.00]
-; BDVER2-NEXT:    subq $7, %rdi # sched: [1:0.50]
-; BDVER2-NEXT:    subq $7, (%rsi) # sched: [6:1.00]
-; BDVER2-NEXT:    subq %rdx, %rdi # sched: [1:0.50]
-; BDVER2-NEXT:    subq %rdi, (%rsi) # sched: [6:1.00]
-; BDVER2-NEXT:    subq (%rsi), %rdi # sched: [5:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_sub_64:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    subq $665536, %rax # imm = 0xA27C0
-; BTVER2-NEXT:    # sched: [1:0.50]
-; BTVER2-NEXT:    subq $665536, %rdi # imm = 0xA27C0
-; BTVER2-NEXT:    # sched: [1:0.50]
-; BTVER2-NEXT:    subq $665536, (%rsi) # imm = 0xA27C0
-; BTVER2-NEXT:    # sched: [5:1.00]
-; BTVER2-NEXT:    subq $7, %rdi # sched: [1:0.50]
-; BTVER2-NEXT:    subq $7, (%rsi) # sched: [5:1.00]
-; BTVER2-NEXT:    subq %rdx, %rdi # sched: [1:0.50]
-; BTVER2-NEXT:    subq %rdi, (%rsi) # sched: [5:1.00]
-; BTVER2-NEXT:    subq (%rsi), %rdi # sched: [4:1.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_sub_64:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    subq $665536, %rax # imm = 0xA27C0
-; ZNVER1-NEXT:    # sched: [1:0.25]
-; ZNVER1-NEXT:    subq $665536, %rdi # imm = 0xA27C0
-; ZNVER1-NEXT:    # sched: [1:0.25]
-; ZNVER1-NEXT:    subq $665536, (%rsi) # imm = 0xA27C0
-; ZNVER1-NEXT:    # sched: [5:0.50]
-; ZNVER1-NEXT:    subq $7, %rdi # sched: [1:0.25]
-; ZNVER1-NEXT:    subq $7, (%rsi) # sched: [5:0.50]
-; ZNVER1-NEXT:    subq %rdx, %rdi # sched: [1:0.25]
-; ZNVER1-NEXT:    subq %rdi, (%rsi) # sched: [5:0.50]
-; ZNVER1-NEXT:    subq (%rsi), %rdi # sched: [5:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  tail call void asm "subq $3, %RAX \0A\09 subq $3, $0 \0A\09 subq $3, $2 \0A\09 subq $4, $0 \0A\09 subq $4, $2 \0A\09 subq $1, $0 \0A\09 subq $0, $2 \0A\09 subq $2, $0", "r,r,*m,i,i"(i64 %a0, i64 %a2, i64* %a1, i32 665536, i8 7) nounwind
-  ret void
-}
-
-; TODO - test_swapgs
-; TODO - test_syscall
-; TODO - test_sysenter
-; TODO - test_sysexit
-; TODO - test_sysret
-
-define void @test_test_8(i8 %a0, i8* %a1) optsize {
-; GENERIC-LABEL: test_test_8:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    testb $7, %al # sched: [1:0.33]
-; GENERIC-NEXT:    testb $7, %dil # sched: [1:0.33]
-; GENERIC-NEXT:    testb $7, (%rsi) # sched: [6:0.50]
-; GENERIC-NEXT:    testb %dil, %dil # sched: [1:0.33]
-; GENERIC-NEXT:    testb %dil, (%rsi) # sched: [6:0.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_test_8:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    testb $7, %al # sched: [1:0.50]
-; ATOM-NEXT:    testb $7, %dil # sched: [1:0.50]
-; ATOM-NEXT:    testb $7, (%rsi) # sched: [1:1.00]
-; ATOM-NEXT:    testb %dil, %dil # sched: [1:0.50]
-; ATOM-NEXT:    testb %dil, (%rsi) # sched: [1:1.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_test_8:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    testb $7, %al # sched: [1:0.50]
-; SLM-NEXT:    testb $7, %dil # sched: [1:0.50]
-; SLM-NEXT:    testb $7, (%rsi) # sched: [4:1.00]
-; SLM-NEXT:    testb %dil, %dil # sched: [1:0.50]
-; SLM-NEXT:    testb %dil, (%rsi) # sched: [4:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_test_8:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    testb $7, %al # sched: [1:0.33]
-; SANDY-NEXT:    testb $7, %dil # sched: [1:0.33]
-; SANDY-NEXT:    testb $7, (%rsi) # sched: [6:0.50]
-; SANDY-NEXT:    testb %dil, %dil # sched: [1:0.33]
-; SANDY-NEXT:    testb %dil, (%rsi) # sched: [6:0.50]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_test_8:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    testb $7, %al # sched: [1:0.25]
-; HASWELL-NEXT:    testb $7, %dil # sched: [1:0.25]
-; HASWELL-NEXT:    testb $7, (%rsi) # sched: [6:0.50]
-; HASWELL-NEXT:    testb %dil, %dil # sched: [1:0.25]
-; HASWELL-NEXT:    testb %dil, (%rsi) # sched: [6:0.50]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_test_8:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    testb $7, %al # sched: [1:0.25]
-; BROADWELL-NEXT:    testb $7, %dil # sched: [1:0.25]
-; BROADWELL-NEXT:    testb $7, (%rsi) # sched: [6:0.50]
-; BROADWELL-NEXT:    testb %dil, %dil # sched: [1:0.25]
-; BROADWELL-NEXT:    testb %dil, (%rsi) # sched: [6:0.50]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_test_8:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    testb $7, %al # sched: [1:0.25]
-; SKYLAKE-NEXT:    testb $7, %dil # sched: [1:0.25]
-; SKYLAKE-NEXT:    testb $7, (%rsi) # sched: [6:0.50]
-; SKYLAKE-NEXT:    testb %dil, %dil # sched: [1:0.25]
-; SKYLAKE-NEXT:    testb %dil, (%rsi) # sched: [6:0.50]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_test_8:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    testb $7, %al # sched: [1:0.25]
-; SKX-NEXT:    testb $7, %dil # sched: [1:0.25]
-; SKX-NEXT:    testb $7, (%rsi) # sched: [6:0.50]
-; SKX-NEXT:    testb %dil, %dil # sched: [1:0.25]
-; SKX-NEXT:    testb %dil, (%rsi) # sched: [6:0.50]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_test_8:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    testb $7, %al # sched: [1:0.50]
-; BDVER2-NEXT:    testb $7, %dil # sched: [1:0.50]
-; BDVER2-NEXT:    testb $7, (%rsi) # sched: [5:0.50]
-; BDVER2-NEXT:    testb %dil, %dil # sched: [1:0.50]
-; BDVER2-NEXT:    testb %dil, (%rsi) # sched: [5:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_test_8:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    testb $7, %al # sched: [1:0.50]
-; BTVER2-NEXT:    testb $7, %dil # sched: [1:0.50]
-; BTVER2-NEXT:    testb $7, (%rsi) # sched: [4:1.00]
-; BTVER2-NEXT:    testb %dil, %dil # sched: [1:0.50]
-; BTVER2-NEXT:    testb %dil, (%rsi) # sched: [4:1.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_test_8:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    testb $7, %al # sched: [1:0.25]
-; ZNVER1-NEXT:    testb $7, %dil # sched: [1:0.25]
-; ZNVER1-NEXT:    testb $7, (%rsi) # sched: [5:0.50]
-; ZNVER1-NEXT:    testb %dil, %dil # sched: [1:0.25]
-; ZNVER1-NEXT:    testb %dil, (%rsi) # sched: [5:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  tail call void asm "testb $2, %AL \0A\09 testb $2, $0 \0A\09 testb $2, $1 \0A\09 testb $0, $0 \0A\09 testb $0, $1", "r,*m,i"(i8 %a0, i8* %a1, i8 7) nounwind
-  ret void
-}
-define void @test_test_16(i16 %a0, i16* %a1) optsize {
-; GENERIC-LABEL: test_test_16:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    testw $511, %ax # imm = 0x1FF
-; GENERIC-NEXT:    # sched: [1:0.33]
-; GENERIC-NEXT:    testw $511, %di # imm = 0x1FF
-; GENERIC-NEXT:    # sched: [1:0.33]
-; GENERIC-NEXT:    testw $511, (%rsi) # imm = 0x1FF
-; GENERIC-NEXT:    # sched: [6:0.50]
-; GENERIC-NEXT:    testw %di, %di # sched: [1:0.33]
-; GENERIC-NEXT:    testw %di, (%rsi) # sched: [6:0.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_test_16:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    testw $511, %ax # imm = 0x1FF
-; ATOM-NEXT:    # sched: [1:0.50]
-; ATOM-NEXT:    testw $511, %di # imm = 0x1FF
-; ATOM-NEXT:    # sched: [1:0.50]
-; ATOM-NEXT:    testw $511, (%rsi) # imm = 0x1FF
-; ATOM-NEXT:    # sched: [1:1.00]
-; ATOM-NEXT:    testw %di, %di # sched: [1:0.50]
-; ATOM-NEXT:    testw %di, (%rsi) # sched: [1:1.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_test_16:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    testw $511, %ax # imm = 0x1FF
-; SLM-NEXT:    # sched: [1:0.50]
-; SLM-NEXT:    testw $511, %di # imm = 0x1FF
-; SLM-NEXT:    # sched: [1:0.50]
-; SLM-NEXT:    testw $511, (%rsi) # imm = 0x1FF
-; SLM-NEXT:    # sched: [4:1.00]
-; SLM-NEXT:    testw %di, %di # sched: [1:0.50]
-; SLM-NEXT:    testw %di, (%rsi) # sched: [4:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_test_16:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    testw $511, %ax # imm = 0x1FF
-; SANDY-NEXT:    # sched: [1:0.33]
-; SANDY-NEXT:    testw $511, %di # imm = 0x1FF
-; SANDY-NEXT:    # sched: [1:0.33]
-; SANDY-NEXT:    testw $511, (%rsi) # imm = 0x1FF
-; SANDY-NEXT:    # sched: [6:0.50]
-; SANDY-NEXT:    testw %di, %di # sched: [1:0.33]
-; SANDY-NEXT:    testw %di, (%rsi) # sched: [6:0.50]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_test_16:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    testw $511, %ax # imm = 0x1FF
-; HASWELL-NEXT:    # sched: [1:0.25]
-; HASWELL-NEXT:    testw $511, %di # imm = 0x1FF
-; HASWELL-NEXT:    # sched: [1:0.25]
-; HASWELL-NEXT:    testw $511, (%rsi) # imm = 0x1FF
-; HASWELL-NEXT:    # sched: [6:0.50]
-; HASWELL-NEXT:    testw %di, %di # sched: [1:0.25]
-; HASWELL-NEXT:    testw %di, (%rsi) # sched: [6:0.50]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_test_16:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    testw $511, %ax # imm = 0x1FF
-; BROADWELL-NEXT:    # sched: [1:0.25]
-; BROADWELL-NEXT:    testw $511, %di # imm = 0x1FF
-; BROADWELL-NEXT:    # sched: [1:0.25]
-; BROADWELL-NEXT:    testw $511, (%rsi) # imm = 0x1FF
-; BROADWELL-NEXT:    # sched: [6:0.50]
-; BROADWELL-NEXT:    testw %di, %di # sched: [1:0.25]
-; BROADWELL-NEXT:    testw %di, (%rsi) # sched: [6:0.50]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_test_16:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    testw $511, %ax # imm = 0x1FF
-; SKYLAKE-NEXT:    # sched: [1:0.25]
-; SKYLAKE-NEXT:    testw $511, %di # imm = 0x1FF
-; SKYLAKE-NEXT:    # sched: [1:0.25]
-; SKYLAKE-NEXT:    testw $511, (%rsi) # imm = 0x1FF
-; SKYLAKE-NEXT:    # sched: [6:0.50]
-; SKYLAKE-NEXT:    testw %di, %di # sched: [1:0.25]
-; SKYLAKE-NEXT:    testw %di, (%rsi) # sched: [6:0.50]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_test_16:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    testw $511, %ax # imm = 0x1FF
-; SKX-NEXT:    # sched: [1:0.25]
-; SKX-NEXT:    testw $511, %di # imm = 0x1FF
-; SKX-NEXT:    # sched: [1:0.25]
-; SKX-NEXT:    testw $511, (%rsi) # imm = 0x1FF
-; SKX-NEXT:    # sched: [6:0.50]
-; SKX-NEXT:    testw %di, %di # sched: [1:0.25]
-; SKX-NEXT:    testw %di, (%rsi) # sched: [6:0.50]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_test_16:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    testw $511, %ax # imm = 0x1FF
-; BDVER2-NEXT:    # sched: [1:0.50]
-; BDVER2-NEXT:    testw $511, %di # imm = 0x1FF
-; BDVER2-NEXT:    # sched: [1:0.50]
-; BDVER2-NEXT:    testw $511, (%rsi) # imm = 0x1FF
-; BDVER2-NEXT:    # sched: [5:0.50]
-; BDVER2-NEXT:    testw %di, %di # sched: [1:0.50]
-; BDVER2-NEXT:    testw %di, (%rsi) # sched: [5:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_test_16:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    testw $511, %ax # imm = 0x1FF
-; BTVER2-NEXT:    # sched: [1:0.50]
-; BTVER2-NEXT:    testw $511, %di # imm = 0x1FF
-; BTVER2-NEXT:    # sched: [1:0.50]
-; BTVER2-NEXT:    testw $511, (%rsi) # imm = 0x1FF
-; BTVER2-NEXT:    # sched: [4:1.00]
-; BTVER2-NEXT:    testw %di, %di # sched: [1:0.50]
-; BTVER2-NEXT:    testw %di, (%rsi) # sched: [4:1.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_test_16:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    testw $511, %ax # imm = 0x1FF
-; ZNVER1-NEXT:    # sched: [1:0.25]
-; ZNVER1-NEXT:    testw $511, %di # imm = 0x1FF
-; ZNVER1-NEXT:    # sched: [1:0.25]
-; ZNVER1-NEXT:    testw $511, (%rsi) # imm = 0x1FF
-; ZNVER1-NEXT:    # sched: [5:0.50]
-; ZNVER1-NEXT:    testw %di, %di # sched: [1:0.25]
-; ZNVER1-NEXT:    testw %di, (%rsi) # sched: [5:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  tail call void asm "testw $2, %AX \0A\09 testw $2, $0 \0A\09 testw $2, $1 \0A\09 testw $0, $0 \0A\09 testw $0, $1", "r,*m,i"(i16 %a0, i16* %a1, i16 511) nounwind
-  ret void
-}
-define void @test_test_32(i32 %a0, i32* %a1) optsize {
-; GENERIC-LABEL: test_test_32:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    testl $665536, %eax # imm = 0xA27C0
-; GENERIC-NEXT:    # sched: [1:0.33]
-; GENERIC-NEXT:    testl $665536, %edi # imm = 0xA27C0
-; GENERIC-NEXT:    # sched: [1:0.33]
-; GENERIC-NEXT:    testl $665536, (%rsi) # imm = 0xA27C0
-; GENERIC-NEXT:    # sched: [6:0.50]
-; GENERIC-NEXT:    testl %edi, %edi # sched: [1:0.33]
-; GENERIC-NEXT:    testl %edi, (%rsi) # sched: [6:0.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_test_32:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    testl $665536, %eax # imm = 0xA27C0
-; ATOM-NEXT:    # sched: [1:0.50]
-; ATOM-NEXT:    testl $665536, %edi # imm = 0xA27C0
-; ATOM-NEXT:    # sched: [1:0.50]
-; ATOM-NEXT:    testl $665536, (%rsi) # imm = 0xA27C0
-; ATOM-NEXT:    # sched: [1:1.00]
-; ATOM-NEXT:    testl %edi, %edi # sched: [1:0.50]
-; ATOM-NEXT:    testl %edi, (%rsi) # sched: [1:1.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_test_32:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    testl $665536, %eax # imm = 0xA27C0
-; SLM-NEXT:    # sched: [1:0.50]
-; SLM-NEXT:    testl $665536, %edi # imm = 0xA27C0
-; SLM-NEXT:    # sched: [1:0.50]
-; SLM-NEXT:    testl $665536, (%rsi) # imm = 0xA27C0
-; SLM-NEXT:    # sched: [4:1.00]
-; SLM-NEXT:    testl %edi, %edi # sched: [1:0.50]
-; SLM-NEXT:    testl %edi, (%rsi) # sched: [4:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_test_32:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    testl $665536, %eax # imm = 0xA27C0
-; SANDY-NEXT:    # sched: [1:0.33]
-; SANDY-NEXT:    testl $665536, %edi # imm = 0xA27C0
-; SANDY-NEXT:    # sched: [1:0.33]
-; SANDY-NEXT:    testl $665536, (%rsi) # imm = 0xA27C0
-; SANDY-NEXT:    # sched: [6:0.50]
-; SANDY-NEXT:    testl %edi, %edi # sched: [1:0.33]
-; SANDY-NEXT:    testl %edi, (%rsi) # sched: [6:0.50]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_test_32:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    testl $665536, %eax # imm = 0xA27C0
-; HASWELL-NEXT:    # sched: [1:0.25]
-; HASWELL-NEXT:    testl $665536, %edi # imm = 0xA27C0
-; HASWELL-NEXT:    # sched: [1:0.25]
-; HASWELL-NEXT:    testl $665536, (%rsi) # imm = 0xA27C0
-; HASWELL-NEXT:    # sched: [6:0.50]
-; HASWELL-NEXT:    testl %edi, %edi # sched: [1:0.25]
-; HASWELL-NEXT:    testl %edi, (%rsi) # sched: [6:0.50]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_test_32:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    testl $665536, %eax # imm = 0xA27C0
-; BROADWELL-NEXT:    # sched: [1:0.25]
-; BROADWELL-NEXT:    testl $665536, %edi # imm = 0xA27C0
-; BROADWELL-NEXT:    # sched: [1:0.25]
-; BROADWELL-NEXT:    testl $665536, (%rsi) # imm = 0xA27C0
-; BROADWELL-NEXT:    # sched: [6:0.50]
-; BROADWELL-NEXT:    testl %edi, %edi # sched: [1:0.25]
-; BROADWELL-NEXT:    testl %edi, (%rsi) # sched: [6:0.50]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_test_32:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    testl $665536, %eax # imm = 0xA27C0
-; SKYLAKE-NEXT:    # sched: [1:0.25]
-; SKYLAKE-NEXT:    testl $665536, %edi # imm = 0xA27C0
-; SKYLAKE-NEXT:    # sched: [1:0.25]
-; SKYLAKE-NEXT:    testl $665536, (%rsi) # imm = 0xA27C0
-; SKYLAKE-NEXT:    # sched: [6:0.50]
-; SKYLAKE-NEXT:    testl %edi, %edi # sched: [1:0.25]
-; SKYLAKE-NEXT:    testl %edi, (%rsi) # sched: [6:0.50]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_test_32:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    testl $665536, %eax # imm = 0xA27C0
-; SKX-NEXT:    # sched: [1:0.25]
-; SKX-NEXT:    testl $665536, %edi # imm = 0xA27C0
-; SKX-NEXT:    # sched: [1:0.25]
-; SKX-NEXT:    testl $665536, (%rsi) # imm = 0xA27C0
-; SKX-NEXT:    # sched: [6:0.50]
-; SKX-NEXT:    testl %edi, %edi # sched: [1:0.25]
-; SKX-NEXT:    testl %edi, (%rsi) # sched: [6:0.50]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_test_32:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    testl $665536, %eax # imm = 0xA27C0
-; BDVER2-NEXT:    # sched: [1:0.50]
-; BDVER2-NEXT:    testl $665536, %edi # imm = 0xA27C0
-; BDVER2-NEXT:    # sched: [1:0.50]
-; BDVER2-NEXT:    testl $665536, (%rsi) # imm = 0xA27C0
-; BDVER2-NEXT:    # sched: [5:0.50]
-; BDVER2-NEXT:    testl %edi, %edi # sched: [1:0.50]
-; BDVER2-NEXT:    testl %edi, (%rsi) # sched: [5:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_test_32:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    testl $665536, %eax # imm = 0xA27C0
-; BTVER2-NEXT:    # sched: [1:0.50]
-; BTVER2-NEXT:    testl $665536, %edi # imm = 0xA27C0
-; BTVER2-NEXT:    # sched: [1:0.50]
-; BTVER2-NEXT:    testl $665536, (%rsi) # imm = 0xA27C0
-; BTVER2-NEXT:    # sched: [4:1.00]
-; BTVER2-NEXT:    testl %edi, %edi # sched: [1:0.50]
-; BTVER2-NEXT:    testl %edi, (%rsi) # sched: [4:1.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_test_32:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    testl $665536, %eax # imm = 0xA27C0
-; ZNVER1-NEXT:    # sched: [1:0.25]
-; ZNVER1-NEXT:    testl $665536, %edi # imm = 0xA27C0
-; ZNVER1-NEXT:    # sched: [1:0.25]
-; ZNVER1-NEXT:    testl $665536, (%rsi) # imm = 0xA27C0
-; ZNVER1-NEXT:    # sched: [5:0.50]
-; ZNVER1-NEXT:    testl %edi, %edi # sched: [1:0.25]
-; ZNVER1-NEXT:    testl %edi, (%rsi) # sched: [5:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  tail call void asm "testl $2, %EAX \0A\09 testl $2, $0 \0A\09 testl $2, $1 \0A\09 testl $0, $0 \0A\09 testl $0, $1", "r,*m,i"(i32 %a0, i32* %a1, i32 665536) nounwind
-  ret void
-}
-define void @test_test_64(i64 %a0, i64* %a1) optsize {
-; GENERIC-LABEL: test_test_64:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    testq $665536, %rax # imm = 0xA27C0
-; GENERIC-NEXT:    # sched: [1:0.33]
-; GENERIC-NEXT:    testq $665536, %rdi # imm = 0xA27C0
-; GENERIC-NEXT:    # sched: [1:0.33]
-; GENERIC-NEXT:    testq $665536, (%rsi) # imm = 0xA27C0
-; GENERIC-NEXT:    # sched: [6:0.50]
-; GENERIC-NEXT:    testq %rdi, %rdi # sched: [1:0.33]
-; GENERIC-NEXT:    testq %rdi, (%rsi) # sched: [6:0.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_test_64:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    testq $665536, %rax # imm = 0xA27C0
-; ATOM-NEXT:    # sched: [1:0.50]
-; ATOM-NEXT:    testq $665536, %rdi # imm = 0xA27C0
-; ATOM-NEXT:    # sched: [1:0.50]
-; ATOM-NEXT:    testq $665536, (%rsi) # imm = 0xA27C0
-; ATOM-NEXT:    # sched: [1:1.00]
-; ATOM-NEXT:    testq %rdi, %rdi # sched: [1:0.50]
-; ATOM-NEXT:    testq %rdi, (%rsi) # sched: [1:1.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_test_64:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    testq $665536, %rax # imm = 0xA27C0
-; SLM-NEXT:    # sched: [1:0.50]
-; SLM-NEXT:    testq $665536, %rdi # imm = 0xA27C0
-; SLM-NEXT:    # sched: [1:0.50]
-; SLM-NEXT:    testq $665536, (%rsi) # imm = 0xA27C0
-; SLM-NEXT:    # sched: [4:1.00]
-; SLM-NEXT:    testq %rdi, %rdi # sched: [1:0.50]
-; SLM-NEXT:    testq %rdi, (%rsi) # sched: [4:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_test_64:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    testq $665536, %rax # imm = 0xA27C0
-; SANDY-NEXT:    # sched: [1:0.33]
-; SANDY-NEXT:    testq $665536, %rdi # imm = 0xA27C0
-; SANDY-NEXT:    # sched: [1:0.33]
-; SANDY-NEXT:    testq $665536, (%rsi) # imm = 0xA27C0
-; SANDY-NEXT:    # sched: [6:0.50]
-; SANDY-NEXT:    testq %rdi, %rdi # sched: [1:0.33]
-; SANDY-NEXT:    testq %rdi, (%rsi) # sched: [6:0.50]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_test_64:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    testq $665536, %rax # imm = 0xA27C0
-; HASWELL-NEXT:    # sched: [1:0.25]
-; HASWELL-NEXT:    testq $665536, %rdi # imm = 0xA27C0
-; HASWELL-NEXT:    # sched: [1:0.25]
-; HASWELL-NEXT:    testq $665536, (%rsi) # imm = 0xA27C0
-; HASWELL-NEXT:    # sched: [6:0.50]
-; HASWELL-NEXT:    testq %rdi, %rdi # sched: [1:0.25]
-; HASWELL-NEXT:    testq %rdi, (%rsi) # sched: [6:0.50]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_test_64:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    testq $665536, %rax # imm = 0xA27C0
-; BROADWELL-NEXT:    # sched: [1:0.25]
-; BROADWELL-NEXT:    testq $665536, %rdi # imm = 0xA27C0
-; BROADWELL-NEXT:    # sched: [1:0.25]
-; BROADWELL-NEXT:    testq $665536, (%rsi) # imm = 0xA27C0
-; BROADWELL-NEXT:    # sched: [6:0.50]
-; BROADWELL-NEXT:    testq %rdi, %rdi # sched: [1:0.25]
-; BROADWELL-NEXT:    testq %rdi, (%rsi) # sched: [6:0.50]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_test_64:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    testq $665536, %rax # imm = 0xA27C0
-; SKYLAKE-NEXT:    # sched: [1:0.25]
-; SKYLAKE-NEXT:    testq $665536, %rdi # imm = 0xA27C0
-; SKYLAKE-NEXT:    # sched: [1:0.25]
-; SKYLAKE-NEXT:    testq $665536, (%rsi) # imm = 0xA27C0
-; SKYLAKE-NEXT:    # sched: [6:0.50]
-; SKYLAKE-NEXT:    testq %rdi, %rdi # sched: [1:0.25]
-; SKYLAKE-NEXT:    testq %rdi, (%rsi) # sched: [6:0.50]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_test_64:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    testq $665536, %rax # imm = 0xA27C0
-; SKX-NEXT:    # sched: [1:0.25]
-; SKX-NEXT:    testq $665536, %rdi # imm = 0xA27C0
-; SKX-NEXT:    # sched: [1:0.25]
-; SKX-NEXT:    testq $665536, (%rsi) # imm = 0xA27C0
-; SKX-NEXT:    # sched: [6:0.50]
-; SKX-NEXT:    testq %rdi, %rdi # sched: [1:0.25]
-; SKX-NEXT:    testq %rdi, (%rsi) # sched: [6:0.50]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_test_64:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    testq $665536, %rax # imm = 0xA27C0
-; BDVER2-NEXT:    # sched: [1:0.50]
-; BDVER2-NEXT:    testq $665536, %rdi # imm = 0xA27C0
-; BDVER2-NEXT:    # sched: [1:0.50]
-; BDVER2-NEXT:    testq $665536, (%rsi) # imm = 0xA27C0
-; BDVER2-NEXT:    # sched: [5:0.50]
-; BDVER2-NEXT:    testq %rdi, %rdi # sched: [1:0.50]
-; BDVER2-NEXT:    testq %rdi, (%rsi) # sched: [5:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_test_64:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    testq $665536, %rax # imm = 0xA27C0
-; BTVER2-NEXT:    # sched: [1:0.50]
-; BTVER2-NEXT:    testq $665536, %rdi # imm = 0xA27C0
-; BTVER2-NEXT:    # sched: [1:0.50]
-; BTVER2-NEXT:    testq $665536, (%rsi) # imm = 0xA27C0
-; BTVER2-NEXT:    # sched: [4:1.00]
-; BTVER2-NEXT:    testq %rdi, %rdi # sched: [1:0.50]
-; BTVER2-NEXT:    testq %rdi, (%rsi) # sched: [4:1.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_test_64:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    testq $665536, %rax # imm = 0xA27C0
-; ZNVER1-NEXT:    # sched: [1:0.25]
-; ZNVER1-NEXT:    testq $665536, %rdi # imm = 0xA27C0
-; ZNVER1-NEXT:    # sched: [1:0.25]
-; ZNVER1-NEXT:    testq $665536, (%rsi) # imm = 0xA27C0
-; ZNVER1-NEXT:    # sched: [5:0.50]
-; ZNVER1-NEXT:    testq %rdi, %rdi # sched: [1:0.25]
-; ZNVER1-NEXT:    testq %rdi, (%rsi) # sched: [5:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  tail call void asm "testq $2, %RAX \0A\09 testq $2, $0 \0A\09 testq $2, $1 \0A\09 testq $0, $0 \0A\09 testq $0, $1", "r,*m,i"(i64 %a0, i64* %a1, i32 665536) nounwind
-  ret void
-}
-
-; TODO: ud0, ud1
-define void @test_ud2() optsize {
-; GENERIC-LABEL: test_ud2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    ud2 # sched: [100:0.33]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_ud2:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    ud2 # sched: [100:0.50]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_ud2:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    ud2 # sched: [100:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_ud2:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    ud2 # sched: [100:0.33]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_ud2:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    ud2 # sched: [100:0.25]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_ud2:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    ud2 # sched: [100:0.25]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_ud2:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    ud2 # sched: [100:0.25]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_ud2:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    ud2 # sched: [100:0.25]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_ud2:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    ud2 # sched: [100:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_ud2:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    ud2 # sched: [100:0.50]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_ud2:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    ud2 # sched: [100:0.25]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  call void asm sideeffect "ud2", ""()
-  ret void
-}
-
-; TODO - test_verr
-; TODO - test_verw
-; TODO - test_vmload
-; TODO - test_vmmcall
-; TODO - test_vmrun
-; TODO - test_vmsave
-; TODO - test_wbinvd
-
-define void @test_xadd_8(i8 %a0, i8 %a1, i8 *%a2) optsize {
-; GENERIC-LABEL: test_xadd_8:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    xaddb %dil, %sil # sched: [2:1.00]
-; GENERIC-NEXT:    xaddb %dil, (%rdx) # sched: [8:1.00]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_xadd_8:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    xaddb %dil, %sil # sched: [2:1.00]
-; ATOM-NEXT:    xaddb %dil, (%rdx) # sched: [3:1.50]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_xadd_8:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    xaddb %dil, %sil # sched: [1:0.50]
-; SLM-NEXT:    xaddb %dil, (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_xadd_8:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    xaddb %dil, %sil # sched: [2:1.00]
-; SANDY-NEXT:    xaddb %dil, (%rdx) # sched: [8:1.00]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_xadd_8:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    xaddb %dil, %sil # sched: [2:0.75]
-; HASWELL-NEXT:    xaddb %dil, (%rdx) # sched: [8:1.00]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_xadd_8:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    xaddb %dil, %sil # sched: [2:0.75]
-; BROADWELL-NEXT:    xaddb %dil, (%rdx) # sched: [7:1.00]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_xadd_8:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    xaddb %dil, %sil # sched: [2:0.75]
-; SKYLAKE-NEXT:    xaddb %dil, (%rdx) # sched: [7:1.00]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_xadd_8:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    xaddb %dil, %sil # sched: [2:0.75]
-; SKX-NEXT:    xaddb %dil, (%rdx) # sched: [7:1.00]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_xadd_8:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    xaddb %dil, %sil # sched: [2:1.00]
-; BDVER2-NEXT:    xaddb %dil, (%rdx) # sched: [6:1.00]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_xadd_8:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    xaddb %dil, %sil # sched: [1:0.50]
-; BTVER2-NEXT:    xaddb %dil, (%rdx) # sched: [4:1.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_xadd_8:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    xaddb %dil, %sil # sched: [1:0.25]
-; ZNVER1-NEXT:    xaddb %dil, (%rdx) # sched: [100:0.25]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  tail call void asm "xaddb $0, $1 \0A\09 xaddb $0, $2", "r,r,*m"(i8 %a0, i8 %a1, i8 *%a2) nounwind
-  ret void
-}
-define void @test_xadd_16(i16 %a0, i16 %a1, i16 *%a2) optsize {
-; GENERIC-LABEL: test_xadd_16:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    xaddw %di, %si # sched: [2:1.00]
-; GENERIC-NEXT:    xaddw %di, (%rdx) # sched: [8:1.00]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_xadd_16:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    xaddw %di, %si # sched: [2:1.00]
-; ATOM-NEXT:    xaddw %di, (%rdx) # sched: [3:1.50]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_xadd_16:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    xaddw %di, %si # sched: [1:0.50]
-; SLM-NEXT:    xaddw %di, (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_xadd_16:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    xaddw %di, %si # sched: [2:1.00]
-; SANDY-NEXT:    xaddw %di, (%rdx) # sched: [8:1.00]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_xadd_16:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    xaddw %di, %si # sched: [2:0.75]
-; HASWELL-NEXT:    xaddw %di, (%rdx) # sched: [8:1.00]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_xadd_16:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    xaddw %di, %si # sched: [2:0.75]
-; BROADWELL-NEXT:    xaddw %di, (%rdx) # sched: [7:1.00]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_xadd_16:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    xaddw %di, %si # sched: [2:0.75]
-; SKYLAKE-NEXT:    xaddw %di, (%rdx) # sched: [7:1.00]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_xadd_16:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    xaddw %di, %si # sched: [2:0.75]
-; SKX-NEXT:    xaddw %di, (%rdx) # sched: [7:1.00]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_xadd_16:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    xaddw %di, %si # sched: [2:1.00]
-; BDVER2-NEXT:    xaddw %di, (%rdx) # sched: [6:1.00]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_xadd_16:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    xaddw %di, %si # sched: [1:0.50]
-; BTVER2-NEXT:    xaddw %di, (%rdx) # sched: [4:1.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_xadd_16:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    xaddw %di, %si # sched: [1:0.25]
-; ZNVER1-NEXT:    xaddw %di, (%rdx) # sched: [100:0.25]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  tail call void asm "xaddw $0, $1 \0A\09 xaddw $0, $2", "r,r,*m"(i16 %a0, i16 %a1, i16 *%a2) nounwind
-  ret void
-}
-define void @test_xadd_32(i32 %a0, i32 %a1, i32 *%a2) optsize {
-; GENERIC-LABEL: test_xadd_32:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    xaddl %edi, %esi # sched: [2:1.00]
-; GENERIC-NEXT:    xaddl %edi, (%rdx) # sched: [8:1.00]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_xadd_32:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    xaddl %edi, %esi # sched: [2:1.00]
-; ATOM-NEXT:    xaddl %edi, (%rdx) # sched: [3:1.50]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_xadd_32:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    xaddl %edi, %esi # sched: [1:0.50]
-; SLM-NEXT:    xaddl %edi, (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_xadd_32:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    xaddl %edi, %esi # sched: [2:1.00]
-; SANDY-NEXT:    xaddl %edi, (%rdx) # sched: [8:1.00]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_xadd_32:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    xaddl %edi, %esi # sched: [2:0.75]
-; HASWELL-NEXT:    xaddl %edi, (%rdx) # sched: [8:1.00]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_xadd_32:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    xaddl %edi, %esi # sched: [2:0.75]
-; BROADWELL-NEXT:    xaddl %edi, (%rdx) # sched: [7:1.00]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_xadd_32:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    xaddl %edi, %esi # sched: [2:0.75]
-; SKYLAKE-NEXT:    xaddl %edi, (%rdx) # sched: [7:1.00]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_xadd_32:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    xaddl %edi, %esi # sched: [2:0.75]
-; SKX-NEXT:    xaddl %edi, (%rdx) # sched: [7:1.00]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_xadd_32:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    xaddl %edi, %esi # sched: [2:1.00]
-; BDVER2-NEXT:    xaddl %edi, (%rdx) # sched: [6:1.00]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_xadd_32:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    xaddl %edi, %esi # sched: [1:0.50]
-; BTVER2-NEXT:    xaddl %edi, (%rdx) # sched: [4:1.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_xadd_32:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    xaddl %edi, %esi # sched: [1:0.25]
-; ZNVER1-NEXT:    xaddl %edi, (%rdx) # sched: [100:0.25]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  tail call void asm "xaddl $0, $1 \0A\09 xaddl $0, $2", "r,r,*m"(i32 %a0, i32 %a1, i32 *%a2) nounwind
-  ret void
-}
-define void @test_xadd_64(i64 %a0, i64 %a1, i64 *%a2) optsize {
-; GENERIC-LABEL: test_xadd_64:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    xaddq %rdi, %rsi # sched: [2:1.00]
-; GENERIC-NEXT:    xaddq %rdi, (%rdx) # sched: [8:1.00]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_xadd_64:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    xaddq %rdi, %rsi # sched: [2:1.00]
-; ATOM-NEXT:    xaddq %rdi, (%rdx) # sched: [3:1.50]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_xadd_64:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    xaddq %rdi, %rsi # sched: [1:0.50]
-; SLM-NEXT:    xaddq %rdi, (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_xadd_64:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    xaddq %rdi, %rsi # sched: [2:1.00]
-; SANDY-NEXT:    xaddq %rdi, (%rdx) # sched: [8:1.00]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_xadd_64:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    xaddq %rdi, %rsi # sched: [2:0.75]
-; HASWELL-NEXT:    xaddq %rdi, (%rdx) # sched: [8:1.00]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_xadd_64:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    xaddq %rdi, %rsi # sched: [2:0.75]
-; BROADWELL-NEXT:    xaddq %rdi, (%rdx) # sched: [7:1.00]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_xadd_64:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    xaddq %rdi, %rsi # sched: [2:0.75]
-; SKYLAKE-NEXT:    xaddq %rdi, (%rdx) # sched: [7:1.00]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_xadd_64:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    xaddq %rdi, %rsi # sched: [2:0.75]
-; SKX-NEXT:    xaddq %rdi, (%rdx) # sched: [7:1.00]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_xadd_64:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    xaddq %rdi, %rsi # sched: [2:1.00]
-; BDVER2-NEXT:    xaddq %rdi, (%rdx) # sched: [6:1.00]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_xadd_64:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    xaddq %rdi, %rsi # sched: [1:0.50]
-; BTVER2-NEXT:    xaddq %rdi, (%rdx) # sched: [4:1.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_xadd_64:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    xaddq %rdi, %rsi # sched: [1:0.25]
-; ZNVER1-NEXT:    xaddq %rdi, (%rdx) # sched: [100:0.25]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  tail call void asm "xaddq $0, $1 \0A\09 xaddq $0, $2", "r,r,*m"(i64 %a0, i64 %a1, i64 *%a2) nounwind
-  ret void
-}
-
-define void @test_xchg_8(i8 %a0, i8 %a1, i8 *%a2) optsize {
-; GENERIC-LABEL: test_xchg_8:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    xchgb %sil, %dil # sched: [2:1.00]
-; GENERIC-NEXT:    xchgb %dil, (%rdx) # sched: [6:1.00]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_xchg_8:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    xchgb %sil, %dil # sched: [2:1.00]
-; ATOM-NEXT:    xchgb %dil, (%rdx) # sched: [3:1.50]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_xchg_8:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    xchgb %sil, %dil # sched: [1:0.50]
-; SLM-NEXT:    xchgb %dil, (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_xchg_8:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    xchgb %sil, %dil # sched: [2:1.00]
-; SANDY-NEXT:    xchgb %dil, (%rdx) # sched: [6:1.00]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_xchg_8:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    xchgb %sil, %dil # sched: [2:0.75]
-; HASWELL-NEXT:    xchgb %dil, (%rdx) # sched: [9:1.00]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_xchg_8:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    xchgb %sil, %dil # sched: [2:0.75]
-; BROADWELL-NEXT:    xchgb %dil, (%rdx) # sched: [8:1.00]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_xchg_8:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    xchgb %sil, %dil # sched: [2:0.75]
-; SKYLAKE-NEXT:    xchgb %dil, (%rdx) # sched: [10:1.25]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_xchg_8:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    xchgb %sil, %dil # sched: [2:0.75]
-; SKX-NEXT:    xchgb %dil, (%rdx) # sched: [10:1.25]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_xchg_8:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    xchgb %sil, %dil # sched: [1:1.00]
-; BDVER2-NEXT:    xchgb %dil, (%rdx) # sched: [5:1.00]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_xchg_8:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    xchgb %sil, %dil # sched: [1:0.50]
-; BTVER2-NEXT:    xchgb %dil, (%rdx) # sched: [4:1.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_xchg_8:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    xchgb %sil, %dil # sched: [1:0.50]
-; ZNVER1-NEXT:    xchgb %dil, (%rdx) # sched: [5:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  tail call void asm "xchg $1, $0 \0A\09 xchg $2, $0", "r,r,*m"(i8 %a0, i8 %a1, i8 *%a2) nounwind
-  ret void
-}
-define void @test_xchg_16(i16 %a0, i16 %a1, i16 *%a2) optsize {
-; GENERIC-LABEL: test_xchg_16:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    xchgw %di, %ax # sched: [2:1.00]
-; GENERIC-NEXT:    xchgw %si, %di # sched: [2:1.00]
-; GENERIC-NEXT:    xchgw %di, (%rdx) # sched: [6:1.00]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_xchg_16:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    xchgw %di, %ax # sched: [2:1.00]
-; ATOM-NEXT:    xchgw %si, %di # sched: [2:1.00]
-; ATOM-NEXT:    xchgw %di, (%rdx) # sched: [3:1.50]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_xchg_16:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    xchgw %di, %ax # sched: [1:0.50]
-; SLM-NEXT:    xchgw %si, %di # sched: [1:0.50]
-; SLM-NEXT:    xchgw %di, (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_xchg_16:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    xchgw %di, %ax # sched: [2:1.00]
-; SANDY-NEXT:    xchgw %si, %di # sched: [2:1.00]
-; SANDY-NEXT:    xchgw %di, (%rdx) # sched: [6:1.00]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_xchg_16:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    xchgw %di, %ax # sched: [2:0.75]
-; HASWELL-NEXT:    xchgw %si, %di # sched: [2:0.75]
-; HASWELL-NEXT:    xchgw %di, (%rdx) # sched: [9:1.00]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_xchg_16:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    xchgw %di, %ax # sched: [2:0.75]
-; BROADWELL-NEXT:    xchgw %si, %di # sched: [2:0.75]
-; BROADWELL-NEXT:    xchgw %di, (%rdx) # sched: [8:1.00]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_xchg_16:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    xchgw %di, %ax # sched: [2:0.75]
-; SKYLAKE-NEXT:    xchgw %si, %di # sched: [2:0.75]
-; SKYLAKE-NEXT:    xchgw %di, (%rdx) # sched: [10:1.25]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_xchg_16:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    xchgw %di, %ax # sched: [2:0.75]
-; SKX-NEXT:    xchgw %si, %di # sched: [2:0.75]
-; SKX-NEXT:    xchgw %di, (%rdx) # sched: [10:1.25]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_xchg_16:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    xchgw %di, %ax # sched: [1:1.00]
-; BDVER2-NEXT:    xchgw %si, %di # sched: [2:1.00]
-; BDVER2-NEXT:    xchgw %di, (%rdx) # sched: [5:1.00]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_xchg_16:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    xchgw %di, %ax # sched: [1:0.50]
-; BTVER2-NEXT:    xchgw %si, %di # sched: [1:0.50]
-; BTVER2-NEXT:    xchgw %di, (%rdx) # sched: [4:1.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_xchg_16:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    xchgw %di, %ax # sched: [1:0.50]
-; ZNVER1-NEXT:    xchgw %si, %di # sched: [1:0.50]
-; ZNVER1-NEXT:    xchgw %di, (%rdx) # sched: [5:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  tail call void asm "xchg %AX, $0 \0A\09 xchg $1, $0 \0A\09 xchg $2, $0", "r,r,*m"(i16 %a0, i16 %a1, i16 *%a2) nounwind
-  ret void
-}
-define void @test_xchg_32(i32 %a0, i32 %a1, i32 *%a2) optsize {
-; GENERIC-LABEL: test_xchg_32:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    xchgl %edi, %eax # sched: [2:1.00]
-; GENERIC-NEXT:    xchgl %esi, %edi # sched: [2:1.00]
-; GENERIC-NEXT:    xchgl %edi, (%rdx) # sched: [6:1.00]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_xchg_32:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    xchgl %edi, %eax # sched: [2:1.00]
-; ATOM-NEXT:    xchgl %esi, %edi # sched: [2:1.00]
-; ATOM-NEXT:    xchgl %edi, (%rdx) # sched: [3:1.50]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_xchg_32:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    xchgl %edi, %eax # sched: [1:0.50]
-; SLM-NEXT:    xchgl %esi, %edi # sched: [1:0.50]
-; SLM-NEXT:    xchgl %edi, (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_xchg_32:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    xchgl %edi, %eax # sched: [2:1.00]
-; SANDY-NEXT:    xchgl %esi, %edi # sched: [2:1.00]
-; SANDY-NEXT:    xchgl %edi, (%rdx) # sched: [6:1.00]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_xchg_32:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    xchgl %edi, %eax # sched: [2:0.75]
-; HASWELL-NEXT:    xchgl %esi, %edi # sched: [2:0.75]
-; HASWELL-NEXT:    xchgl %edi, (%rdx) # sched: [9:1.00]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_xchg_32:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    xchgl %edi, %eax # sched: [2:0.75]
-; BROADWELL-NEXT:    xchgl %esi, %edi # sched: [2:0.75]
-; BROADWELL-NEXT:    xchgl %edi, (%rdx) # sched: [8:1.00]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_xchg_32:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    xchgl %edi, %eax # sched: [2:0.75]
-; SKYLAKE-NEXT:    xchgl %esi, %edi # sched: [2:0.75]
-; SKYLAKE-NEXT:    xchgl %edi, (%rdx) # sched: [10:1.25]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_xchg_32:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    xchgl %edi, %eax # sched: [2:0.75]
-; SKX-NEXT:    xchgl %esi, %edi # sched: [2:0.75]
-; SKX-NEXT:    xchgl %edi, (%rdx) # sched: [10:1.25]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_xchg_32:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    xchgl %edi, %eax # sched: [1:1.00]
-; BDVER2-NEXT:    xchgl %esi, %edi # sched: [1:1.00]
-; BDVER2-NEXT:    xchgl %edi, (%rdx) # sched: [5:1.00]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_xchg_32:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    xchgl %edi, %eax # sched: [1:0.50]
-; BTVER2-NEXT:    xchgl %esi, %edi # sched: [1:0.50]
-; BTVER2-NEXT:    xchgl %edi, (%rdx) # sched: [4:1.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_xchg_32:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    xchgl %edi, %eax # sched: [1:0.50]
-; ZNVER1-NEXT:    xchgl %esi, %edi # sched: [1:0.50]
-; ZNVER1-NEXT:    xchgl %edi, (%rdx) # sched: [5:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  tail call void asm "xchg %EAX, $0 \0A\09 xchg $1, $0 \0A\09 xchg $2, $0", "r,r,*m"(i32 %a0, i32 %a1, i32 *%a2) nounwind
-  ret void
-}
-define void @test_xchg_64(i64 %a0, i64 %a1, i64 *%a2) optsize {
-; GENERIC-LABEL: test_xchg_64:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    xchgq %rdi, %rax # sched: [2:1.00]
-; GENERIC-NEXT:    xchgq %rsi, %rdi # sched: [2:1.00]
-; GENERIC-NEXT:    xchgq %rdi, (%rdx) # sched: [6:1.00]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_xchg_64:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    xchgq %rdi, %rax # sched: [2:1.00]
-; ATOM-NEXT:    xchgq %rsi, %rdi # sched: [2:1.00]
-; ATOM-NEXT:    xchgq %rdi, (%rdx) # sched: [3:1.50]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_xchg_64:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    xchgq %rdi, %rax # sched: [1:0.50]
-; SLM-NEXT:    xchgq %rsi, %rdi # sched: [1:0.50]
-; SLM-NEXT:    xchgq %rdi, (%rdx) # sched: [4:2.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_xchg_64:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    xchgq %rdi, %rax # sched: [2:1.00]
-; SANDY-NEXT:    xchgq %rsi, %rdi # sched: [2:1.00]
-; SANDY-NEXT:    xchgq %rdi, (%rdx) # sched: [6:1.00]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_xchg_64:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    xchgq %rdi, %rax # sched: [2:0.75]
-; HASWELL-NEXT:    xchgq %rsi, %rdi # sched: [2:0.75]
-; HASWELL-NEXT:    xchgq %rdi, (%rdx) # sched: [9:1.00]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_xchg_64:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    xchgq %rdi, %rax # sched: [2:0.75]
-; BROADWELL-NEXT:    xchgq %rsi, %rdi # sched: [2:0.75]
-; BROADWELL-NEXT:    xchgq %rdi, (%rdx) # sched: [8:1.00]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_xchg_64:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    xchgq %rdi, %rax # sched: [2:0.75]
-; SKYLAKE-NEXT:    xchgq %rsi, %rdi # sched: [2:0.75]
-; SKYLAKE-NEXT:    xchgq %rdi, (%rdx) # sched: [10:1.25]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_xchg_64:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    xchgq %rdi, %rax # sched: [2:0.75]
-; SKX-NEXT:    xchgq %rsi, %rdi # sched: [2:0.75]
-; SKX-NEXT:    xchgq %rdi, (%rdx) # sched: [10:1.25]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_xchg_64:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    xchgq %rdi, %rax # sched: [1:1.00]
-; BDVER2-NEXT:    xchgq %rsi, %rdi # sched: [1:1.00]
-; BDVER2-NEXT:    xchgq %rdi, (%rdx) # sched: [5:1.00]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_xchg_64:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    xchgq %rdi, %rax # sched: [1:0.50]
-; BTVER2-NEXT:    xchgq %rsi, %rdi # sched: [1:0.50]
-; BTVER2-NEXT:    xchgq %rdi, (%rdx) # sched: [4:1.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_xchg_64:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    xchgq %rdi, %rax # sched: [1:0.50]
-; ZNVER1-NEXT:    xchgq %rsi, %rdi # sched: [1:0.50]
-; ZNVER1-NEXT:    xchgq %rdi, (%rdx) # sched: [5:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  tail call void asm "xchg %RAX, $0 \0A\09 xchg $1, $0 \0A\09 xchg $2, $0", "r,r,*m"(i64 %a0, i64 %a1, i64 *%a2) nounwind
-  ret void
-}
-
-define void @test_xlat() optsize {
-; GENERIC-LABEL: test_xlat:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    xlatb # sched: [5:0.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_xlat:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    xlatb # sched: [6:3.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_xlat:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    xlatb # sched: [3:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_xlat:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    xlatb # sched: [5:0.50]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_xlat:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    xlatb # sched: [7:0.75]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_xlat:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    xlatb # sched: [5:0.50]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_xlat:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    xlatb # sched: [5:0.50]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_xlat:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    xlatb # sched: [5:0.50]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_xlat:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    xlatb # sched: [6:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_xlat:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    xlatb # sched: [5:1.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_xlat:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    xlatb # sched: [100:0.25]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  tail call void asm "xlat", ""() nounwind
-  ret void
-}
-
-define void @test_xor_8(i8 %a0, i8* %a1, i8 %a2) optsize {
-; GENERIC-LABEL: test_xor_8:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    xorb $7, %al # sched: [1:0.33]
-; GENERIC-NEXT:    xorb $7, %dil # sched: [1:0.33]
-; GENERIC-NEXT:    xorb $7, (%rsi) # sched: [7:1.00]
-; GENERIC-NEXT:    xorb %dl, %dil # sched: [1:0.33]
-; GENERIC-NEXT:    xorb %dil, (%rsi) # sched: [7:1.00]
-; GENERIC-NEXT:    xorb (%rsi), %dil # sched: [6:0.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_xor_8:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    xorb $7, %al # sched: [1:0.50]
-; ATOM-NEXT:    xorb $7, %dil # sched: [1:0.50]
-; ATOM-NEXT:    xorb $7, (%rsi) # sched: [1:1.00]
-; ATOM-NEXT:    xorb %dl, %dil # sched: [1:0.50]
-; ATOM-NEXT:    xorb %dil, (%rsi) # sched: [1:1.00]
-; ATOM-NEXT:    xorb (%rsi), %dil # sched: [1:1.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_xor_8:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    xorb $7, %al # sched: [1:0.50]
-; SLM-NEXT:    xorb $7, %dil # sched: [1:0.50]
-; SLM-NEXT:    xorb $7, (%rsi) # sched: [5:2.00]
-; SLM-NEXT:    xorb %dl, %dil # sched: [1:0.50]
-; SLM-NEXT:    xorb %dil, (%rsi) # sched: [5:2.00]
-; SLM-NEXT:    xorb (%rsi), %dil # sched: [4:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_xor_8:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    xorb $7, %al # sched: [1:0.33]
-; SANDY-NEXT:    xorb $7, %dil # sched: [1:0.33]
-; SANDY-NEXT:    xorb $7, (%rsi) # sched: [7:1.00]
-; SANDY-NEXT:    xorb %dl, %dil # sched: [1:0.33]
-; SANDY-NEXT:    xorb %dil, (%rsi) # sched: [7:1.00]
-; SANDY-NEXT:    xorb (%rsi), %dil # sched: [6:0.50]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_xor_8:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    xorb $7, %al # sched: [1:0.25]
-; HASWELL-NEXT:    xorb $7, %dil # sched: [1:0.25]
-; HASWELL-NEXT:    xorb $7, (%rsi) # sched: [7:1.00]
-; HASWELL-NEXT:    xorb %dl, %dil # sched: [1:0.25]
-; HASWELL-NEXT:    xorb %dil, (%rsi) # sched: [7:1.00]
-; HASWELL-NEXT:    xorb (%rsi), %dil # sched: [6:0.50]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_xor_8:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    xorb $7, %al # sched: [1:0.25]
-; BROADWELL-NEXT:    xorb $7, %dil # sched: [1:0.25]
-; BROADWELL-NEXT:    xorb $7, (%rsi) # sched: [7:1.00]
-; BROADWELL-NEXT:    xorb %dl, %dil # sched: [1:0.25]
-; BROADWELL-NEXT:    xorb %dil, (%rsi) # sched: [7:1.00]
-; BROADWELL-NEXT:    xorb (%rsi), %dil # sched: [6:0.50]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_xor_8:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    xorb $7, %al # sched: [1:0.25]
-; SKYLAKE-NEXT:    xorb $7, %dil # sched: [1:0.25]
-; SKYLAKE-NEXT:    xorb $7, (%rsi) # sched: [7:1.00]
-; SKYLAKE-NEXT:    xorb %dl, %dil # sched: [1:0.25]
-; SKYLAKE-NEXT:    xorb %dil, (%rsi) # sched: [7:1.00]
-; SKYLAKE-NEXT:    xorb (%rsi), %dil # sched: [6:0.50]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_xor_8:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    xorb $7, %al # sched: [1:0.25]
-; SKX-NEXT:    xorb $7, %dil # sched: [1:0.25]
-; SKX-NEXT:    xorb $7, (%rsi) # sched: [7:1.00]
-; SKX-NEXT:    xorb %dl, %dil # sched: [1:0.25]
-; SKX-NEXT:    xorb %dil, (%rsi) # sched: [7:1.00]
-; SKX-NEXT:    xorb (%rsi), %dil # sched: [6:0.50]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_xor_8:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    xorb $7, %al # sched: [1:0.50]
-; BDVER2-NEXT:    xorb $7, %dil # sched: [1:0.50]
-; BDVER2-NEXT:    xorb $7, (%rsi) # sched: [6:1.00]
-; BDVER2-NEXT:    xorb %dl, %dil # sched: [1:0.50]
-; BDVER2-NEXT:    xorb %dil, (%rsi) # sched: [6:1.00]
-; BDVER2-NEXT:    xorb (%rsi), %dil # sched: [5:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_xor_8:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    xorb $7, %al # sched: [1:0.50]
-; BTVER2-NEXT:    xorb $7, %dil # sched: [1:0.50]
-; BTVER2-NEXT:    xorb $7, (%rsi) # sched: [5:1.00]
-; BTVER2-NEXT:    xorb %dl, %dil # sched: [1:0.50]
-; BTVER2-NEXT:    xorb %dil, (%rsi) # sched: [5:1.00]
-; BTVER2-NEXT:    xorb (%rsi), %dil # sched: [4:1.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_xor_8:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    xorb $7, %al # sched: [1:0.25]
-; ZNVER1-NEXT:    xorb $7, %dil # sched: [1:0.25]
-; ZNVER1-NEXT:    xorb $7, (%rsi) # sched: [5:0.50]
-; ZNVER1-NEXT:    xorb %dl, %dil # sched: [1:0.25]
-; ZNVER1-NEXT:    xorb %dil, (%rsi) # sched: [5:0.50]
-; ZNVER1-NEXT:    xorb (%rsi), %dil # sched: [5:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  tail call void asm "xorb $3, %AL \0A\09 xorb $3, $0 \0A\09 xorb $3, $2 \0A\09 xorb $1, $0 \0A\09 xorb $0, $2 \0A\09 xorb $2, $0", "r,r,*m,i"(i8 %a0, i8 %a2, i8* %a1, i8 7) nounwind
-  ret void
-}
-define void @test_xor_16(i16 %a0, i16* %a1, i16 %a2) optsize {
-; GENERIC-LABEL: test_xor_16:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    xorw $511, %ax # imm = 0x1FF
-; GENERIC-NEXT:    # sched: [1:0.33]
-; GENERIC-NEXT:    xorw $511, %di # imm = 0x1FF
-; GENERIC-NEXT:    # sched: [1:0.33]
-; GENERIC-NEXT:    xorw $511, (%rsi) # imm = 0x1FF
-; GENERIC-NEXT:    # sched: [7:1.00]
-; GENERIC-NEXT:    xorw $7, %di # sched: [1:0.33]
-; GENERIC-NEXT:    xorw $7, (%rsi) # sched: [7:1.00]
-; GENERIC-NEXT:    xorw %dx, %di # sched: [1:0.33]
-; GENERIC-NEXT:    xorw %di, (%rsi) # sched: [7:1.00]
-; GENERIC-NEXT:    xorw (%rsi), %di # sched: [6:0.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_xor_16:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    xorw $511, %ax # imm = 0x1FF
-; ATOM-NEXT:    # sched: [1:0.50]
-; ATOM-NEXT:    xorw $511, %di # imm = 0x1FF
-; ATOM-NEXT:    # sched: [1:0.50]
-; ATOM-NEXT:    xorw $511, (%rsi) # imm = 0x1FF
-; ATOM-NEXT:    # sched: [1:1.00]
-; ATOM-NEXT:    xorw $7, %di # sched: [1:0.50]
-; ATOM-NEXT:    xorw $7, (%rsi) # sched: [1:1.00]
-; ATOM-NEXT:    xorw %dx, %di # sched: [1:0.50]
-; ATOM-NEXT:    xorw %di, (%rsi) # sched: [1:1.00]
-; ATOM-NEXT:    xorw (%rsi), %di # sched: [1:1.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_xor_16:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    xorw $511, %ax # imm = 0x1FF
-; SLM-NEXT:    # sched: [1:0.50]
-; SLM-NEXT:    xorw $511, %di # imm = 0x1FF
-; SLM-NEXT:    # sched: [1:0.50]
-; SLM-NEXT:    xorw $511, (%rsi) # imm = 0x1FF
-; SLM-NEXT:    # sched: [5:2.00]
-; SLM-NEXT:    xorw $7, %di # sched: [1:0.50]
-; SLM-NEXT:    xorw $7, (%rsi) # sched: [5:2.00]
-; SLM-NEXT:    xorw %dx, %di # sched: [1:0.50]
-; SLM-NEXT:    xorw %di, (%rsi) # sched: [5:2.00]
-; SLM-NEXT:    xorw (%rsi), %di # sched: [4:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_xor_16:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    xorw $511, %ax # imm = 0x1FF
-; SANDY-NEXT:    # sched: [1:0.33]
-; SANDY-NEXT:    xorw $511, %di # imm = 0x1FF
-; SANDY-NEXT:    # sched: [1:0.33]
-; SANDY-NEXT:    xorw $511, (%rsi) # imm = 0x1FF
-; SANDY-NEXT:    # sched: [7:1.00]
-; SANDY-NEXT:    xorw $7, %di # sched: [1:0.33]
-; SANDY-NEXT:    xorw $7, (%rsi) # sched: [7:1.00]
-; SANDY-NEXT:    xorw %dx, %di # sched: [1:0.33]
-; SANDY-NEXT:    xorw %di, (%rsi) # sched: [7:1.00]
-; SANDY-NEXT:    xorw (%rsi), %di # sched: [6:0.50]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_xor_16:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    xorw $511, %ax # imm = 0x1FF
-; HASWELL-NEXT:    # sched: [1:0.25]
-; HASWELL-NEXT:    xorw $511, %di # imm = 0x1FF
-; HASWELL-NEXT:    # sched: [1:0.25]
-; HASWELL-NEXT:    xorw $511, (%rsi) # imm = 0x1FF
-; HASWELL-NEXT:    # sched: [7:1.00]
-; HASWELL-NEXT:    xorw $7, %di # sched: [1:0.25]
-; HASWELL-NEXT:    xorw $7, (%rsi) # sched: [7:1.00]
-; HASWELL-NEXT:    xorw %dx, %di # sched: [1:0.25]
-; HASWELL-NEXT:    xorw %di, (%rsi) # sched: [7:1.00]
-; HASWELL-NEXT:    xorw (%rsi), %di # sched: [6:0.50]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_xor_16:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    xorw $511, %ax # imm = 0x1FF
-; BROADWELL-NEXT:    # sched: [1:0.25]
-; BROADWELL-NEXT:    xorw $511, %di # imm = 0x1FF
-; BROADWELL-NEXT:    # sched: [1:0.25]
-; BROADWELL-NEXT:    xorw $511, (%rsi) # imm = 0x1FF
-; BROADWELL-NEXT:    # sched: [7:1.00]
-; BROADWELL-NEXT:    xorw $7, %di # sched: [1:0.25]
-; BROADWELL-NEXT:    xorw $7, (%rsi) # sched: [7:1.00]
-; BROADWELL-NEXT:    xorw %dx, %di # sched: [1:0.25]
-; BROADWELL-NEXT:    xorw %di, (%rsi) # sched: [7:1.00]
-; BROADWELL-NEXT:    xorw (%rsi), %di # sched: [6:0.50]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_xor_16:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    xorw $511, %ax # imm = 0x1FF
-; SKYLAKE-NEXT:    # sched: [1:0.25]
-; SKYLAKE-NEXT:    xorw $511, %di # imm = 0x1FF
-; SKYLAKE-NEXT:    # sched: [1:0.25]
-; SKYLAKE-NEXT:    xorw $511, (%rsi) # imm = 0x1FF
-; SKYLAKE-NEXT:    # sched: [7:1.00]
-; SKYLAKE-NEXT:    xorw $7, %di # sched: [1:0.25]
-; SKYLAKE-NEXT:    xorw $7, (%rsi) # sched: [7:1.00]
-; SKYLAKE-NEXT:    xorw %dx, %di # sched: [1:0.25]
-; SKYLAKE-NEXT:    xorw %di, (%rsi) # sched: [7:1.00]
-; SKYLAKE-NEXT:    xorw (%rsi), %di # sched: [6:0.50]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_xor_16:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    xorw $511, %ax # imm = 0x1FF
-; SKX-NEXT:    # sched: [1:0.25]
-; SKX-NEXT:    xorw $511, %di # imm = 0x1FF
-; SKX-NEXT:    # sched: [1:0.25]
-; SKX-NEXT:    xorw $511, (%rsi) # imm = 0x1FF
-; SKX-NEXT:    # sched: [7:1.00]
-; SKX-NEXT:    xorw $7, %di # sched: [1:0.25]
-; SKX-NEXT:    xorw $7, (%rsi) # sched: [7:1.00]
-; SKX-NEXT:    xorw %dx, %di # sched: [1:0.25]
-; SKX-NEXT:    xorw %di, (%rsi) # sched: [7:1.00]
-; SKX-NEXT:    xorw (%rsi), %di # sched: [6:0.50]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_xor_16:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    xorw $511, %ax # imm = 0x1FF
-; BDVER2-NEXT:    # sched: [1:0.50]
-; BDVER2-NEXT:    xorw $511, %di # imm = 0x1FF
-; BDVER2-NEXT:    # sched: [1:0.50]
-; BDVER2-NEXT:    xorw $511, (%rsi) # imm = 0x1FF
-; BDVER2-NEXT:    # sched: [6:1.00]
-; BDVER2-NEXT:    xorw $7, %di # sched: [1:0.50]
-; BDVER2-NEXT:    xorw $7, (%rsi) # sched: [6:1.00]
-; BDVER2-NEXT:    xorw %dx, %di # sched: [1:0.50]
-; BDVER2-NEXT:    xorw %di, (%rsi) # sched: [6:1.00]
-; BDVER2-NEXT:    xorw (%rsi), %di # sched: [5:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_xor_16:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    xorw $511, %ax # imm = 0x1FF
-; BTVER2-NEXT:    # sched: [1:0.50]
-; BTVER2-NEXT:    xorw $511, %di # imm = 0x1FF
-; BTVER2-NEXT:    # sched: [1:0.50]
-; BTVER2-NEXT:    xorw $511, (%rsi) # imm = 0x1FF
-; BTVER2-NEXT:    # sched: [5:1.00]
-; BTVER2-NEXT:    xorw $7, %di # sched: [1:0.50]
-; BTVER2-NEXT:    xorw $7, (%rsi) # sched: [5:1.00]
-; BTVER2-NEXT:    xorw %dx, %di # sched: [1:0.50]
-; BTVER2-NEXT:    xorw %di, (%rsi) # sched: [5:1.00]
-; BTVER2-NEXT:    xorw (%rsi), %di # sched: [4:1.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_xor_16:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    xorw $511, %ax # imm = 0x1FF
-; ZNVER1-NEXT:    # sched: [1:0.25]
-; ZNVER1-NEXT:    xorw $511, %di # imm = 0x1FF
-; ZNVER1-NEXT:    # sched: [1:0.25]
-; ZNVER1-NEXT:    xorw $511, (%rsi) # imm = 0x1FF
-; ZNVER1-NEXT:    # sched: [5:0.50]
-; ZNVER1-NEXT:    xorw $7, %di # sched: [1:0.25]
-; ZNVER1-NEXT:    xorw $7, (%rsi) # sched: [5:0.50]
-; ZNVER1-NEXT:    xorw %dx, %di # sched: [1:0.25]
-; ZNVER1-NEXT:    xorw %di, (%rsi) # sched: [5:0.50]
-; ZNVER1-NEXT:    xorw (%rsi), %di # sched: [5:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  tail call void asm "xorw $3, %AX \0A\09 xorw $3, $0 \0A\09 xorw $3, $2 \0A\09 xorw $4, $0 \0A\09 xorw $4, $2 \0A\09 xorw $1, $0 \0A\09 xorw $0, $2 \0A\09 xorw $2, $0", "r,r,*m,i,i"(i16 %a0, i16 %a2, i16* %a1, i16 511, i8 7) nounwind
-  ret void
-}
-define void @test_xor_32(i32 %a0, i32* %a1, i32 %a2) optsize {
-; GENERIC-LABEL: test_xor_32:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    xorl $665536, %eax # imm = 0xA27C0
-; GENERIC-NEXT:    # sched: [1:0.33]
-; GENERIC-NEXT:    xorl $665536, %edi # imm = 0xA27C0
-; GENERIC-NEXT:    # sched: [1:0.33]
-; GENERIC-NEXT:    xorl $665536, (%rsi) # imm = 0xA27C0
-; GENERIC-NEXT:    # sched: [7:1.00]
-; GENERIC-NEXT:    xorl $7, %edi # sched: [1:0.33]
-; GENERIC-NEXT:    xorl $7, (%rsi) # sched: [7:1.00]
-; GENERIC-NEXT:    xorl %edx, %edi # sched: [1:0.33]
-; GENERIC-NEXT:    xorl %edi, (%rsi) # sched: [7:1.00]
-; GENERIC-NEXT:    xorl (%rsi), %edi # sched: [6:0.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_xor_32:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    xorl $665536, %eax # imm = 0xA27C0
-; ATOM-NEXT:    # sched: [1:0.50]
-; ATOM-NEXT:    xorl $665536, %edi # imm = 0xA27C0
-; ATOM-NEXT:    # sched: [1:0.50]
-; ATOM-NEXT:    xorl $665536, (%rsi) # imm = 0xA27C0
-; ATOM-NEXT:    # sched: [1:1.00]
-; ATOM-NEXT:    xorl $7, %edi # sched: [1:0.50]
-; ATOM-NEXT:    xorl $7, (%rsi) # sched: [1:1.00]
-; ATOM-NEXT:    xorl %edx, %edi # sched: [1:0.50]
-; ATOM-NEXT:    xorl %edi, (%rsi) # sched: [1:1.00]
-; ATOM-NEXT:    xorl (%rsi), %edi # sched: [1:1.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_xor_32:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    xorl $665536, %eax # imm = 0xA27C0
-; SLM-NEXT:    # sched: [1:0.50]
-; SLM-NEXT:    xorl $665536, %edi # imm = 0xA27C0
-; SLM-NEXT:    # sched: [1:0.50]
-; SLM-NEXT:    xorl $665536, (%rsi) # imm = 0xA27C0
-; SLM-NEXT:    # sched: [5:2.00]
-; SLM-NEXT:    xorl $7, %edi # sched: [1:0.50]
-; SLM-NEXT:    xorl $7, (%rsi) # sched: [5:2.00]
-; SLM-NEXT:    xorl %edx, %edi # sched: [1:0.50]
-; SLM-NEXT:    xorl %edi, (%rsi) # sched: [5:2.00]
-; SLM-NEXT:    xorl (%rsi), %edi # sched: [4:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_xor_32:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    xorl $665536, %eax # imm = 0xA27C0
-; SANDY-NEXT:    # sched: [1:0.33]
-; SANDY-NEXT:    xorl $665536, %edi # imm = 0xA27C0
-; SANDY-NEXT:    # sched: [1:0.33]
-; SANDY-NEXT:    xorl $665536, (%rsi) # imm = 0xA27C0
-; SANDY-NEXT:    # sched: [7:1.00]
-; SANDY-NEXT:    xorl $7, %edi # sched: [1:0.33]
-; SANDY-NEXT:    xorl $7, (%rsi) # sched: [7:1.00]
-; SANDY-NEXT:    xorl %edx, %edi # sched: [1:0.33]
-; SANDY-NEXT:    xorl %edi, (%rsi) # sched: [7:1.00]
-; SANDY-NEXT:    xorl (%rsi), %edi # sched: [6:0.50]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_xor_32:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    xorl $665536, %eax # imm = 0xA27C0
-; HASWELL-NEXT:    # sched: [1:0.25]
-; HASWELL-NEXT:    xorl $665536, %edi # imm = 0xA27C0
-; HASWELL-NEXT:    # sched: [1:0.25]
-; HASWELL-NEXT:    xorl $665536, (%rsi) # imm = 0xA27C0
-; HASWELL-NEXT:    # sched: [7:1.00]
-; HASWELL-NEXT:    xorl $7, %edi # sched: [1:0.25]
-; HASWELL-NEXT:    xorl $7, (%rsi) # sched: [7:1.00]
-; HASWELL-NEXT:    xorl %edx, %edi # sched: [1:0.25]
-; HASWELL-NEXT:    xorl %edi, (%rsi) # sched: [7:1.00]
-; HASWELL-NEXT:    xorl (%rsi), %edi # sched: [6:0.50]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_xor_32:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    xorl $665536, %eax # imm = 0xA27C0
-; BROADWELL-NEXT:    # sched: [1:0.25]
-; BROADWELL-NEXT:    xorl $665536, %edi # imm = 0xA27C0
-; BROADWELL-NEXT:    # sched: [1:0.25]
-; BROADWELL-NEXT:    xorl $665536, (%rsi) # imm = 0xA27C0
-; BROADWELL-NEXT:    # sched: [7:1.00]
-; BROADWELL-NEXT:    xorl $7, %edi # sched: [1:0.25]
-; BROADWELL-NEXT:    xorl $7, (%rsi) # sched: [7:1.00]
-; BROADWELL-NEXT:    xorl %edx, %edi # sched: [1:0.25]
-; BROADWELL-NEXT:    xorl %edi, (%rsi) # sched: [7:1.00]
-; BROADWELL-NEXT:    xorl (%rsi), %edi # sched: [6:0.50]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_xor_32:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    xorl $665536, %eax # imm = 0xA27C0
-; SKYLAKE-NEXT:    # sched: [1:0.25]
-; SKYLAKE-NEXT:    xorl $665536, %edi # imm = 0xA27C0
-; SKYLAKE-NEXT:    # sched: [1:0.25]
-; SKYLAKE-NEXT:    xorl $665536, (%rsi) # imm = 0xA27C0
-; SKYLAKE-NEXT:    # sched: [7:1.00]
-; SKYLAKE-NEXT:    xorl $7, %edi # sched: [1:0.25]
-; SKYLAKE-NEXT:    xorl $7, (%rsi) # sched: [7:1.00]
-; SKYLAKE-NEXT:    xorl %edx, %edi # sched: [1:0.25]
-; SKYLAKE-NEXT:    xorl %edi, (%rsi) # sched: [7:1.00]
-; SKYLAKE-NEXT:    xorl (%rsi), %edi # sched: [6:0.50]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_xor_32:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    xorl $665536, %eax # imm = 0xA27C0
-; SKX-NEXT:    # sched: [1:0.25]
-; SKX-NEXT:    xorl $665536, %edi # imm = 0xA27C0
-; SKX-NEXT:    # sched: [1:0.25]
-; SKX-NEXT:    xorl $665536, (%rsi) # imm = 0xA27C0
-; SKX-NEXT:    # sched: [7:1.00]
-; SKX-NEXT:    xorl $7, %edi # sched: [1:0.25]
-; SKX-NEXT:    xorl $7, (%rsi) # sched: [7:1.00]
-; SKX-NEXT:    xorl %edx, %edi # sched: [1:0.25]
-; SKX-NEXT:    xorl %edi, (%rsi) # sched: [7:1.00]
-; SKX-NEXT:    xorl (%rsi), %edi # sched: [6:0.50]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_xor_32:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    xorl $665536, %eax # imm = 0xA27C0
-; BDVER2-NEXT:    # sched: [1:0.50]
-; BDVER2-NEXT:    xorl $665536, %edi # imm = 0xA27C0
-; BDVER2-NEXT:    # sched: [1:0.50]
-; BDVER2-NEXT:    xorl $665536, (%rsi) # imm = 0xA27C0
-; BDVER2-NEXT:    # sched: [6:1.00]
-; BDVER2-NEXT:    xorl $7, %edi # sched: [1:0.50]
-; BDVER2-NEXT:    xorl $7, (%rsi) # sched: [6:1.00]
-; BDVER2-NEXT:    xorl %edx, %edi # sched: [1:0.50]
-; BDVER2-NEXT:    xorl %edi, (%rsi) # sched: [6:1.00]
-; BDVER2-NEXT:    xorl (%rsi), %edi # sched: [5:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_xor_32:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    xorl $665536, %eax # imm = 0xA27C0
-; BTVER2-NEXT:    # sched: [1:0.50]
-; BTVER2-NEXT:    xorl $665536, %edi # imm = 0xA27C0
-; BTVER2-NEXT:    # sched: [1:0.50]
-; BTVER2-NEXT:    xorl $665536, (%rsi) # imm = 0xA27C0
-; BTVER2-NEXT:    # sched: [5:1.00]
-; BTVER2-NEXT:    xorl $7, %edi # sched: [1:0.50]
-; BTVER2-NEXT:    xorl $7, (%rsi) # sched: [5:1.00]
-; BTVER2-NEXT:    xorl %edx, %edi # sched: [1:0.50]
-; BTVER2-NEXT:    xorl %edi, (%rsi) # sched: [5:1.00]
-; BTVER2-NEXT:    xorl (%rsi), %edi # sched: [4:1.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_xor_32:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    xorl $665536, %eax # imm = 0xA27C0
-; ZNVER1-NEXT:    # sched: [1:0.25]
-; ZNVER1-NEXT:    xorl $665536, %edi # imm = 0xA27C0
-; ZNVER1-NEXT:    # sched: [1:0.25]
-; ZNVER1-NEXT:    xorl $665536, (%rsi) # imm = 0xA27C0
-; ZNVER1-NEXT:    # sched: [5:0.50]
-; ZNVER1-NEXT:    xorl $7, %edi # sched: [1:0.25]
-; ZNVER1-NEXT:    xorl $7, (%rsi) # sched: [5:0.50]
-; ZNVER1-NEXT:    xorl %edx, %edi # sched: [1:0.25]
-; ZNVER1-NEXT:    xorl %edi, (%rsi) # sched: [5:0.50]
-; ZNVER1-NEXT:    xorl (%rsi), %edi # sched: [5:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  tail call void asm "xorl $3, %EAX \0A\09 xorl $3, $0 \0A\09 xorl $3, $2 \0A\09 xorl $4, $0 \0A\09 xorl $4, $2 \0A\09 xorl $1, $0 \0A\09 xorl $0, $2 \0A\09 xorl $2, $0", "r,r,*m,i,i"(i32 %a0, i32 %a2, i32* %a1, i32 665536, i8 7) nounwind
-  ret void
-}
-define void @test_xor_64(i64 %a0, i64* %a1, i64 %a2) optsize {
-; GENERIC-LABEL: test_xor_64:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    xorq $665536, %rax # imm = 0xA27C0
-; GENERIC-NEXT:    # sched: [1:0.33]
-; GENERIC-NEXT:    xorq $665536, %rdi # imm = 0xA27C0
-; GENERIC-NEXT:    # sched: [1:0.33]
-; GENERIC-NEXT:    xorq $665536, (%rsi) # imm = 0xA27C0
-; GENERIC-NEXT:    # sched: [7:1.00]
-; GENERIC-NEXT:    xorq $7, %rdi # sched: [1:0.33]
-; GENERIC-NEXT:    xorq $7, (%rsi) # sched: [7:1.00]
-; GENERIC-NEXT:    xorq %rdx, %rdi # sched: [1:0.33]
-; GENERIC-NEXT:    xorq %rdi, (%rsi) # sched: [7:1.00]
-; GENERIC-NEXT:    xorq (%rsi), %rdi # sched: [6:0.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_xor_64:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    xorq $665536, %rax # imm = 0xA27C0
-; ATOM-NEXT:    # sched: [1:0.50]
-; ATOM-NEXT:    xorq $665536, %rdi # imm = 0xA27C0
-; ATOM-NEXT:    # sched: [1:0.50]
-; ATOM-NEXT:    xorq $665536, (%rsi) # imm = 0xA27C0
-; ATOM-NEXT:    # sched: [1:1.00]
-; ATOM-NEXT:    xorq $7, %rdi # sched: [1:0.50]
-; ATOM-NEXT:    xorq $7, (%rsi) # sched: [1:1.00]
-; ATOM-NEXT:    xorq %rdx, %rdi # sched: [1:0.50]
-; ATOM-NEXT:    xorq %rdi, (%rsi) # sched: [1:1.00]
-; ATOM-NEXT:    xorq (%rsi), %rdi # sched: [1:1.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_xor_64:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    xorq $665536, %rax # imm = 0xA27C0
-; SLM-NEXT:    # sched: [1:0.50]
-; SLM-NEXT:    xorq $665536, %rdi # imm = 0xA27C0
-; SLM-NEXT:    # sched: [1:0.50]
-; SLM-NEXT:    xorq $665536, (%rsi) # imm = 0xA27C0
-; SLM-NEXT:    # sched: [5:2.00]
-; SLM-NEXT:    xorq $7, %rdi # sched: [1:0.50]
-; SLM-NEXT:    xorq $7, (%rsi) # sched: [5:2.00]
-; SLM-NEXT:    xorq %rdx, %rdi # sched: [1:0.50]
-; SLM-NEXT:    xorq %rdi, (%rsi) # sched: [5:2.00]
-; SLM-NEXT:    xorq (%rsi), %rdi # sched: [4:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-LABEL: test_xor_64:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    xorq $665536, %rax # imm = 0xA27C0
-; SANDY-NEXT:    # sched: [1:0.33]
-; SANDY-NEXT:    xorq $665536, %rdi # imm = 0xA27C0
-; SANDY-NEXT:    # sched: [1:0.33]
-; SANDY-NEXT:    xorq $665536, (%rsi) # imm = 0xA27C0
-; SANDY-NEXT:    # sched: [7:1.00]
-; SANDY-NEXT:    xorq $7, %rdi # sched: [1:0.33]
-; SANDY-NEXT:    xorq $7, (%rsi) # sched: [7:1.00]
-; SANDY-NEXT:    xorq %rdx, %rdi # sched: [1:0.33]
-; SANDY-NEXT:    xorq %rdi, (%rsi) # sched: [7:1.00]
-; SANDY-NEXT:    xorq (%rsi), %rdi # sched: [6:0.50]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-LABEL: test_xor_64:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    xorq $665536, %rax # imm = 0xA27C0
-; HASWELL-NEXT:    # sched: [1:0.25]
-; HASWELL-NEXT:    xorq $665536, %rdi # imm = 0xA27C0
-; HASWELL-NEXT:    # sched: [1:0.25]
-; HASWELL-NEXT:    xorq $665536, (%rsi) # imm = 0xA27C0
-; HASWELL-NEXT:    # sched: [7:1.00]
-; HASWELL-NEXT:    xorq $7, %rdi # sched: [1:0.25]
-; HASWELL-NEXT:    xorq $7, (%rsi) # sched: [7:1.00]
-; HASWELL-NEXT:    xorq %rdx, %rdi # sched: [1:0.25]
-; HASWELL-NEXT:    xorq %rdi, (%rsi) # sched: [7:1.00]
-; HASWELL-NEXT:    xorq (%rsi), %rdi # sched: [6:0.50]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_xor_64:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    xorq $665536, %rax # imm = 0xA27C0
-; BROADWELL-NEXT:    # sched: [1:0.25]
-; BROADWELL-NEXT:    xorq $665536, %rdi # imm = 0xA27C0
-; BROADWELL-NEXT:    # sched: [1:0.25]
-; BROADWELL-NEXT:    xorq $665536, (%rsi) # imm = 0xA27C0
-; BROADWELL-NEXT:    # sched: [7:1.00]
-; BROADWELL-NEXT:    xorq $7, %rdi # sched: [1:0.25]
-; BROADWELL-NEXT:    xorq $7, (%rsi) # sched: [7:1.00]
-; BROADWELL-NEXT:    xorq %rdx, %rdi # sched: [1:0.25]
-; BROADWELL-NEXT:    xorq %rdi, (%rsi) # sched: [7:1.00]
-; BROADWELL-NEXT:    xorq (%rsi), %rdi # sched: [6:0.50]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_xor_64:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    xorq $665536, %rax # imm = 0xA27C0
-; SKYLAKE-NEXT:    # sched: [1:0.25]
-; SKYLAKE-NEXT:    xorq $665536, %rdi # imm = 0xA27C0
-; SKYLAKE-NEXT:    # sched: [1:0.25]
-; SKYLAKE-NEXT:    xorq $665536, (%rsi) # imm = 0xA27C0
-; SKYLAKE-NEXT:    # sched: [7:1.00]
-; SKYLAKE-NEXT:    xorq $7, %rdi # sched: [1:0.25]
-; SKYLAKE-NEXT:    xorq $7, (%rsi) # sched: [7:1.00]
-; SKYLAKE-NEXT:    xorq %rdx, %rdi # sched: [1:0.25]
-; SKYLAKE-NEXT:    xorq %rdi, (%rsi) # sched: [7:1.00]
-; SKYLAKE-NEXT:    xorq (%rsi), %rdi # sched: [6:0.50]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_xor_64:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    xorq $665536, %rax # imm = 0xA27C0
-; SKX-NEXT:    # sched: [1:0.25]
-; SKX-NEXT:    xorq $665536, %rdi # imm = 0xA27C0
-; SKX-NEXT:    # sched: [1:0.25]
-; SKX-NEXT:    xorq $665536, (%rsi) # imm = 0xA27C0
-; SKX-NEXT:    # sched: [7:1.00]
-; SKX-NEXT:    xorq $7, %rdi # sched: [1:0.25]
-; SKX-NEXT:    xorq $7, (%rsi) # sched: [7:1.00]
-; SKX-NEXT:    xorq %rdx, %rdi # sched: [1:0.25]
-; SKX-NEXT:    xorq %rdi, (%rsi) # sched: [7:1.00]
-; SKX-NEXT:    xorq (%rsi), %rdi # sched: [6:0.50]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-LABEL: test_xor_64:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    xorq $665536, %rax # imm = 0xA27C0
-; BDVER2-NEXT:    # sched: [1:0.50]
-; BDVER2-NEXT:    xorq $665536, %rdi # imm = 0xA27C0
-; BDVER2-NEXT:    # sched: [1:0.50]
-; BDVER2-NEXT:    xorq $665536, (%rsi) # imm = 0xA27C0
-; BDVER2-NEXT:    # sched: [6:1.00]
-; BDVER2-NEXT:    xorq $7, %rdi # sched: [1:0.50]
-; BDVER2-NEXT:    xorq $7, (%rsi) # sched: [6:1.00]
-; BDVER2-NEXT:    xorq %rdx, %rdi # sched: [1:0.50]
-; BDVER2-NEXT:    xorq %rdi, (%rsi) # sched: [6:1.00]
-; BDVER2-NEXT:    xorq (%rsi), %rdi # sched: [5:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_xor_64:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    xorq $665536, %rax # imm = 0xA27C0
-; BTVER2-NEXT:    # sched: [1:0.50]
-; BTVER2-NEXT:    xorq $665536, %rdi # imm = 0xA27C0
-; BTVER2-NEXT:    # sched: [1:0.50]
-; BTVER2-NEXT:    xorq $665536, (%rsi) # imm = 0xA27C0
-; BTVER2-NEXT:    # sched: [5:1.00]
-; BTVER2-NEXT:    xorq $7, %rdi # sched: [1:0.50]
-; BTVER2-NEXT:    xorq $7, (%rsi) # sched: [5:1.00]
-; BTVER2-NEXT:    xorq %rdx, %rdi # sched: [1:0.50]
-; BTVER2-NEXT:    xorq %rdi, (%rsi) # sched: [5:1.00]
-; BTVER2-NEXT:    xorq (%rsi), %rdi # sched: [4:1.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_xor_64:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    xorq $665536, %rax # imm = 0xA27C0
-; ZNVER1-NEXT:    # sched: [1:0.25]
-; ZNVER1-NEXT:    xorq $665536, %rdi # imm = 0xA27C0
-; ZNVER1-NEXT:    # sched: [1:0.25]
-; ZNVER1-NEXT:    xorq $665536, (%rsi) # imm = 0xA27C0
-; ZNVER1-NEXT:    # sched: [5:0.50]
-; ZNVER1-NEXT:    xorq $7, %rdi # sched: [1:0.25]
-; ZNVER1-NEXT:    xorq $7, (%rsi) # sched: [5:0.50]
-; ZNVER1-NEXT:    xorq %rdx, %rdi # sched: [1:0.25]
-; ZNVER1-NEXT:    xorq %rdi, (%rsi) # sched: [5:0.50]
-; ZNVER1-NEXT:    xorq (%rsi), %rdi # sched: [5:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  tail call void asm "xorq $3, %RAX \0A\09 xorq $3, $0 \0A\09 xorq $3, $2 \0A\09 xorq $4, $0 \0A\09 xorq $4, $2 \0A\09 xorq $1, $0 \0A\09 xorq $0, $2 \0A\09 xorq $2, $0", "r,r,*m,i,i"(i64 %a0, i64 %a2, i64* %a1, i32 665536, i8 7) nounwind
-  ret void
-}
diff --git a/test/CodeGen/X86/scheduler-backtracking.ll b/test/CodeGen/X86/scheduler-backtracking.ll
index cb8571bf61a18225d22a738425c83bba42c69eca..95eb8e7c98293ed9d84f8e8867d6d17c038fa11d 100644
--- a/test/CodeGen/X86/scheduler-backtracking.ll
+++ b/test/CodeGen/X86/scheduler-backtracking.ll
@@ -16,8 +16,8 @@ define i256 @test1(i256 %a) nounwind {
 ; ILP-NEXT:    pushq %rbx
 ; ILP-NEXT:    movq %rdi, %rax
 ; ILP-NEXT:    xorl %r8d, %r8d
-; ILP-NEXT:    incl %esi
 ; ILP-NEXT:    addb %sil, %sil
+; ILP-NEXT:    addb $2, %sil
 ; ILP-NEXT:    orb $1, %sil
 ; ILP-NEXT:    movl $1, %r10d
 ; ILP-NEXT:    xorl %r14d, %r14d
@@ -25,35 +25,35 @@ define i256 @test1(i256 %a) nounwind {
 ; ILP-NEXT:    shldq %cl, %r10, %r14
 ; ILP-NEXT:    movl $1, %edx
 ; ILP-NEXT:    shlq %cl, %rdx
-; ILP-NEXT:    leal -128(%rsi), %r9d
 ; ILP-NEXT:    movb $-128, %r11b
-; ILP-NEXT:    xorl %ebx, %ebx
+; ILP-NEXT:    subb %sil, %r11b
+; ILP-NEXT:    leal -128(%rsi), %r9d
+; ILP-NEXT:    xorl %edi, %edi
 ; ILP-NEXT:    movl %r9d, %ecx
-; ILP-NEXT:    shldq %cl, %r10, %rbx
+; ILP-NEXT:    shldq %cl, %r10, %rdi
+; ILP-NEXT:    movl $1, %ebx
+; ILP-NEXT:    shlq %cl, %rbx
+; ILP-NEXT:    movl %r11d, %ecx
+; ILP-NEXT:    shrdq %cl, %r8, %r10
 ; ILP-NEXT:    testb $64, %sil
 ; ILP-NEXT:    cmovneq %rdx, %r14
 ; ILP-NEXT:    cmovneq %r8, %rdx
-; ILP-NEXT:    movl $1, %edi
-; ILP-NEXT:    shlq %cl, %rdi
-; ILP-NEXT:    subb %sil, %r11b
-; ILP-NEXT:    movl %r11d, %ecx
-; ILP-NEXT:    shrdq %cl, %r8, %r10
 ; ILP-NEXT:    testb $64, %r11b
 ; ILP-NEXT:    cmovneq %r8, %r10
 ; ILP-NEXT:    testb $64, %r9b
-; ILP-NEXT:    cmovneq %rdi, %rbx
-; ILP-NEXT:    cmovneq %r8, %rdi
+; ILP-NEXT:    cmovneq %rbx, %rdi
+; ILP-NEXT:    cmovneq %r8, %rbx
 ; ILP-NEXT:    testb %sil, %sil
 ; ILP-NEXT:    cmovsq %r8, %r14
 ; ILP-NEXT:    cmovsq %r8, %rdx
 ; ILP-NEXT:    movq %r14, 8(%rax)
 ; ILP-NEXT:    movq %rdx, (%rax)
-; ILP-NEXT:    cmovnsq %r8, %rbx
-; ILP-NEXT:    cmoveq %r8, %rbx
-; ILP-NEXT:    movq %rbx, 24(%rax)
-; ILP-NEXT:    cmovnsq %r10, %rdi
+; ILP-NEXT:    cmovnsq %r8, %rdi
 ; ILP-NEXT:    cmoveq %r8, %rdi
-; ILP-NEXT:    movq %rdi, 16(%rax)
+; ILP-NEXT:    movq %rdi, 24(%rax)
+; ILP-NEXT:    cmovnsq %r10, %rbx
+; ILP-NEXT:    cmoveq %r8, %rbx
+; ILP-NEXT:    movq %rbx, 16(%rax)
 ; ILP-NEXT:    popq %rbx
 ; ILP-NEXT:    popq %r14
 ; ILP-NEXT:    retq
@@ -61,8 +61,8 @@ define i256 @test1(i256 %a) nounwind {
 ; HYBRID-LABEL: test1:
 ; HYBRID:       # %bb.0:
 ; HYBRID-NEXT:    movq %rdi, %rax
-; HYBRID-NEXT:    incl %esi
 ; HYBRID-NEXT:    addb %sil, %sil
+; HYBRID-NEXT:    addb $2, %sil
 ; HYBRID-NEXT:    orb $1, %sil
 ; HYBRID-NEXT:    movb $-128, %cl
 ; HYBRID-NEXT:    subb %sil, %cl
@@ -104,8 +104,8 @@ define i256 @test1(i256 %a) nounwind {
 ; BURR-LABEL: test1:
 ; BURR:       # %bb.0:
 ; BURR-NEXT:    movq %rdi, %rax
-; BURR-NEXT:    incl %esi
 ; BURR-NEXT:    addb %sil, %sil
+; BURR-NEXT:    addb $2, %sil
 ; BURR-NEXT:    orb $1, %sil
 ; BURR-NEXT:    movb $-128, %cl
 ; BURR-NEXT:    subb %sil, %cl
@@ -148,8 +148,8 @@ define i256 @test1(i256 %a) nounwind {
 ; SRC:       # %bb.0:
 ; SRC-NEXT:    pushq %rbx
 ; SRC-NEXT:    movq %rdi, %rax
-; SRC-NEXT:    incl %esi
 ; SRC-NEXT:    addb %sil, %sil
+; SRC-NEXT:    addb $2, %sil
 ; SRC-NEXT:    orb $1, %sil
 ; SRC-NEXT:    movb $-128, %cl
 ; SRC-NEXT:    subb %sil, %cl
@@ -195,8 +195,8 @@ define i256 @test1(i256 %a) nounwind {
 ; LIN-NEXT:    movq %rdi, %rax
 ; LIN-NEXT:    xorl %r9d, %r9d
 ; LIN-NEXT:    movl $1, %r8d
-; LIN-NEXT:    incl %esi
 ; LIN-NEXT:    addb %sil, %sil
+; LIN-NEXT:    addb $2, %sil
 ; LIN-NEXT:    orb $1, %sil
 ; LIN-NEXT:    movl $1, %edx
 ; LIN-NEXT:    movl %esi, %ecx
@@ -691,8 +691,8 @@ define i64 @test4(i64 %a, i64 %b) nounwind {
 ; ILP:       # %bb.0:
 ; ILP-NEXT:    xorl %ecx, %ecx
 ; ILP-NEXT:    xorl %edx, %edx
-; ILP-NEXT:    addq $1, %rsi
-; ILP-NEXT:    setb %dl
+; ILP-NEXT:    incq %rsi
+; ILP-NEXT:    sete %dl
 ; ILP-NEXT:    movl $2, %eax
 ; ILP-NEXT:    cmpq %rdi, %rsi
 ; ILP-NEXT:    sbbq $0, %rdx
@@ -708,8 +708,8 @@ define i64 @test4(i64 %a, i64 %b) nounwind {
 ; HYBRID:       # %bb.0:
 ; HYBRID-NEXT:    xorl %eax, %eax
 ; HYBRID-NEXT:    xorl %ecx, %ecx
-; HYBRID-NEXT:    addq $1, %rsi
-; HYBRID-NEXT:    setb %cl
+; HYBRID-NEXT:    incq %rsi
+; HYBRID-NEXT:    sete %cl
 ; HYBRID-NEXT:    cmpq %rdi, %rsi
 ; HYBRID-NEXT:    sbbq $0, %rcx
 ; HYBRID-NEXT:    movl $0, %ecx
@@ -725,8 +725,8 @@ define i64 @test4(i64 %a, i64 %b) nounwind {
 ; BURR:       # %bb.0:
 ; BURR-NEXT:    xorl %eax, %eax
 ; BURR-NEXT:    xorl %ecx, %ecx
-; BURR-NEXT:    addq $1, %rsi
-; BURR-NEXT:    setb %cl
+; BURR-NEXT:    incq %rsi
+; BURR-NEXT:    sete %cl
 ; BURR-NEXT:    cmpq %rdi, %rsi
 ; BURR-NEXT:    sbbq $0, %rcx
 ; BURR-NEXT:    movl $0, %ecx
@@ -741,8 +741,8 @@ define i64 @test4(i64 %a, i64 %b) nounwind {
 ; SRC-LABEL: test4:
 ; SRC:       # %bb.0:
 ; SRC-NEXT:    xorl %eax, %eax
-; SRC-NEXT:    addq $1, %rsi
-; SRC-NEXT:    setb %al
+; SRC-NEXT:    incq %rsi
+; SRC-NEXT:    sete %al
 ; SRC-NEXT:    xorl %ecx, %ecx
 ; SRC-NEXT:    cmpq %rdi, %rsi
 ; SRC-NEXT:    sbbq $0, %rax
@@ -760,8 +760,8 @@ define i64 @test4(i64 %a, i64 %b) nounwind {
 ; LIN-NEXT:    movl $2, %eax
 ; LIN-NEXT:    xorl %ecx, %ecx
 ; LIN-NEXT:    xorl %edx, %edx
-; LIN-NEXT:    addq $1, %rsi
-; LIN-NEXT:    setb %dl
+; LIN-NEXT:    incq %rsi
+; LIN-NEXT:    sete %dl
 ; LIN-NEXT:    cmpq %rdi, %rsi
 ; LIN-NEXT:    sbbq $0, %rdx
 ; LIN-NEXT:    movl $0, %edx
diff --git a/test/CodeGen/X86/seh-catch-all-win32.ll b/test/CodeGen/X86/seh-catch-all-win32.ll
index 315790a2f169a99e2cd855371b146995cca04b53..d90c9e48bcd3cd31610d01b4eb0485b6d1ba65ae 100644
--- a/test/CodeGen/X86/seh-catch-all-win32.ll
+++ b/test/CodeGen/X86/seh-catch-all-win32.ll
@@ -12,7 +12,7 @@ declare i32 @llvm.eh.typeid.for(i8*)
 declare i8* @llvm.frameaddress(i32)
 declare i8* @llvm.localrecover(i8*, i8*, i32)
 declare void @llvm.localescape(...)
-declare i8* @llvm.x86.seh.recoverfp(i8*, i8*)
+declare i8* @llvm.eh.recoverfp(i8*, i8*)
 
 define i32 @main() personality i8* bitcast (i32 (...)* @_except_handler3 to i8*) {
 entry:
@@ -37,7 +37,7 @@ __try.cont:                                       ; preds = %entry, %__except
 define internal i32 @"filt$main"() {
 entry:
   %ebp = tail call i8* @llvm.frameaddress(i32 1)
-  %parentfp = tail call i8* @llvm.x86.seh.recoverfp(i8* bitcast (i32 ()* @main to i8*), i8* %ebp)
+  %parentfp = tail call i8* @llvm.eh.recoverfp(i8* bitcast (i32 ()* @main to i8*), i8* %ebp)
   %code.i8 = tail call i8* @llvm.localrecover(i8* bitcast (i32 ()* @main to i8*), i8* %parentfp, i32 0)
   %__exceptioncode = bitcast i8* %code.i8 to i32*
   %info.addr = getelementptr inbounds i8, i8* %ebp, i32 -20
diff --git a/test/CodeGen/X86/seh-filter-no-personality.ll b/test/CodeGen/X86/seh-filter-no-personality.ll
index 87bc9c93f4004f4e54b8a06638113989b26c4b61..638969a53b29cfa8cc778064d08eb98ab5ab6caf 100644
--- a/test/CodeGen/X86/seh-filter-no-personality.ll
+++ b/test/CodeGen/X86/seh-filter-no-personality.ll
@@ -1,10 +1,10 @@
 ; RUN: llc -mtriple=i686-windows-msvc < %s | FileCheck %s
 
-; Mostly make sure that llvm.x86.seh.recoverfp doesn't crash if the parent
+; Mostly make sure that llvm.eh.recoverfp doesn't crash if the parent
 ; function lacks a personality.
 
 declare i8* @llvm.frameaddress(i32)
-declare i8* @llvm.x86.seh.recoverfp(i8*, i8*)
+declare i8* @llvm.eh.recoverfp(i8*, i8*)
 
 define i32 @main() {
 entry:
@@ -14,7 +14,7 @@ entry:
 define internal i32 @"filt$main"() {
 entry:
   %ebp = tail call i8* @llvm.frameaddress(i32 1)
-  %parentfp = tail call i8* @llvm.x86.seh.recoverfp(i8* bitcast (i32 ()* @main to i8*), i8* %ebp)
+  %parentfp = tail call i8* @llvm.eh.recoverfp(i8* bitcast (i32 ()* @main to i8*), i8* %ebp)
   %info.addr = getelementptr inbounds i8, i8* %ebp, i32 -20
   %0 = bitcast i8* %info.addr to i32***
   %1 = load i32**, i32*** %0, align 4
diff --git a/test/CodeGen/X86/seh-localaddress.ll b/test/CodeGen/X86/seh-localaddress.ll
new file mode 100644
index 0000000000000000000000000000000000000000..1bdcdcd1cda1ada3e595a93ff805569ceb8af50b
--- /dev/null
+++ b/test/CodeGen/X86/seh-localaddress.ll
@@ -0,0 +1,78 @@
+; RUN: llc -mtriple x86_64-pc-windows-msvc -o - %s | FileCheck %s
+
+; struct S { int x; };
+; void foo() {
+;   struct S __declspec(align(32)) o;
+;   __try { o.x; }
+;   __finally { o.x; }
+; }
+; void bar() {
+;   struct S o;
+;   __try { o.x; }
+;   __finally { o.x; }
+; }
+
+%struct.S = type { i32 }
+
+define dso_local void @"?foo@@YAXXZ"() #0 {
+entry:
+; CHECK-LABEL: foo
+; CHECK: movq  %rsp, %rdx
+; CHECK-NOT: movq  %rbp, %rdx
+
+  %o = alloca %struct.S, align 32
+  call void (...) @llvm.localescape(%struct.S* %o)
+  %x = getelementptr inbounds %struct.S, %struct.S* %o, i32 0, i32 0
+  %0 = call i8* @llvm.localaddress()
+  call void @"?fin$0@0@foo@@"(i8 0, i8* %0)
+  ret void
+}
+
+; void bar(void)
+; {
+;   int x;
+;   void (*fn)(int);
+;
+;   __try {
+;     x = 1;
+;     fn(x);
+;   } __finally {
+;     x = 2;
+;   }
+; }
+
+define dso_local void @"?bar@@YAXXZ"() personality i8* bitcast (i32 (...)* @__C_specific_handler to i8*) {
+entry:
+; CHECK-LABEL: bar
+; CHECK: movq  %rbp, %rdx
+; CHECK-NOT: movq  %rsp, %rdx
+  %x = alloca i32, align 4
+  %fn = alloca void (i32)*, align 8
+  call void (...) @llvm.localescape(i32* %x)
+  store i32 1, i32* %x, align 4
+  %0 = load void (i32)*, void (i32)** %fn, align 8
+  %1 = load i32, i32* %x, align 4
+  invoke void %0(i32 %1)
+  to label %invoke.cont unwind label %ehcleanup
+  invoke.cont:                                      ; preds = %entry
+  %2 = call i8* @llvm.localaddress()
+  call void @"?fin$0@0@bar@@"(i8 0, i8* %2)
+  ret void
+  ehcleanup:                                        ; preds = %entry
+  %3 = cleanuppad within none []
+  %4 = call i8* @llvm.localaddress()
+  call void @"?fin$0@0@bar@@"(i8 1, i8* %4) [ "funclet"(token %3) ]
+  cleanupret from %3 unwind to caller
+}
+
+declare void @"?fin$0@0@foo@@"(i8 %abnormal_termination, i8* %frame_pointer)
+
+declare void @"?fin$0@0@bar@@"(i8 %abnormal_termination, i8* %frame_pointer)
+
+declare i8* @llvm.localrecover(i8*, i8*, i32)
+
+declare i8* @llvm.localaddress()
+
+declare void @llvm.localescape(...)
+
+declare dso_local i32 @__C_specific_handler(...)
diff --git a/test/CodeGen/X86/seh-no-invokes.ll b/test/CodeGen/X86/seh-no-invokes.ll
index 4e64aa23da82eb9d544651a89312a790ed12fbae..f32ab5c1be56bd084cdea3be09101f88a0fa9337 100644
--- a/test/CodeGen/X86/seh-no-invokes.ll
+++ b/test/CodeGen/X86/seh-no-invokes.ll
@@ -38,7 +38,7 @@ __try.cont:
 define internal i32 @"\01?filt$0@0@f@@"() #1 {
 entry:
   %0 = tail call i8* @llvm.frameaddress(i32 1)
-  %1 = tail call i8* @llvm.x86.seh.recoverfp(i8* bitcast (void ()* @f to i8*), i8* %0)
+  %1 = tail call i8* @llvm.eh.recoverfp(i8* bitcast (void ()* @f to i8*), i8* %0)
   %2 = tail call i8* @llvm.localrecover(i8* bitcast (void ()* @f to i8*), i8* %1, i32 0)
   %__exception_code = bitcast i8* %2 to i32*
   %3 = getelementptr inbounds i8, i8* %0, i32 -20
@@ -55,7 +55,7 @@ entry:
 declare i8* @llvm.frameaddress(i32) #2
 
 ; Function Attrs: nounwind readnone
-declare i8* @llvm.x86.seh.recoverfp(i8*, i8*) #2
+declare i8* @llvm.eh.recoverfp(i8*, i8*) #2
 
 ; Function Attrs: nounwind readnone
 declare i8* @llvm.localrecover(i8*, i8*, i32) #2
diff --git a/test/CodeGen/X86/seh-stack-realign.ll b/test/CodeGen/X86/seh-stack-realign.ll
index 75a005cc4267132b580df3057d53a59a5e97fd8e..0301ae86e2fbcc43e63e858a815a1059f9615d9c 100644
--- a/test/CodeGen/X86/seh-stack-realign.ll
+++ b/test/CodeGen/X86/seh-stack-realign.ll
@@ -12,7 +12,7 @@ declare i32 @llvm.eh.typeid.for(i8*)
 declare i8* @llvm.frameaddress(i32)
 declare i8* @llvm.localrecover(i8*, i8*, i32)
 declare void @llvm.localescape(...)
-declare i8* @llvm.x86.seh.recoverfp(i8*, i8*)
+declare i8* @llvm.eh.recoverfp(i8*, i8*)
 
 define i32 @main() personality i8* bitcast (i32 (...)* @_except_handler3 to i8*) {
 entry:
@@ -38,7 +38,7 @@ __try.cont:                                       ; preds = %entry, %__except
 define internal i32 @"filt$main"() {
 entry:
   %ebp = tail call i8* @llvm.frameaddress(i32 1)
-  %parentfp = tail call i8* @llvm.x86.seh.recoverfp(i8* bitcast (i32 ()* @main to i8*), i8* %ebp)
+  %parentfp = tail call i8* @llvm.eh.recoverfp(i8* bitcast (i32 ()* @main to i8*), i8* %ebp)
   %code.i8 = tail call i8* @llvm.localrecover(i8* bitcast (i32 ()* @main to i8*), i8* %parentfp, i32 0)
   %__exceptioncode = bitcast i8* %code.i8 to i32*
   %info.addr = getelementptr inbounds i8, i8* %ebp, i32 -20
diff --git a/test/CodeGen/X86/select.ll b/test/CodeGen/X86/select.ll
index 947c95137206de701d5c01474b269cde991adb1a..9429e1854d44c3ce02514bff4998d1feba014df2 100644
--- a/test/CodeGen/X86/select.ll
+++ b/test/CodeGen/X86/select.ll
@@ -293,25 +293,25 @@ define void @test6(i32 %C, <4 x float>* %A, <4 x float>* %B) nounwind {
 ; ATHLON-NEXT:    flds 4(%ecx)
 ; ATHLON-NEXT:    flds (%ecx)
 ; ATHLON-NEXT:    flds (%eax)
-; ATHLON-NEXT:    fmul %st(0), %st(0)
+; ATHLON-NEXT:    fmul %st, %st(0)
 ; ATHLON-NEXT:    cmpl $0, {{[0-9]+}}(%esp)
 ; ATHLON-NEXT:    fxch %st(1)
-; ATHLON-NEXT:    fcmove %st(1), %st(0)
+; ATHLON-NEXT:    fcmove %st(1), %st
 ; ATHLON-NEXT:    fstp %st(1)
 ; ATHLON-NEXT:    flds 4(%eax)
-; ATHLON-NEXT:    fmul %st(0), %st(0)
+; ATHLON-NEXT:    fmul %st, %st(0)
 ; ATHLON-NEXT:    fxch %st(2)
-; ATHLON-NEXT:    fcmove %st(2), %st(0)
+; ATHLON-NEXT:    fcmove %st(2), %st
 ; ATHLON-NEXT:    fstp %st(2)
 ; ATHLON-NEXT:    flds 8(%eax)
-; ATHLON-NEXT:    fmul %st(0), %st(0)
+; ATHLON-NEXT:    fmul %st, %st(0)
 ; ATHLON-NEXT:    fxch %st(3)
-; ATHLON-NEXT:    fcmove %st(3), %st(0)
+; ATHLON-NEXT:    fcmove %st(3), %st
 ; ATHLON-NEXT:    fstp %st(3)
 ; ATHLON-NEXT:    flds 12(%eax)
-; ATHLON-NEXT:    fmul %st(0), %st(0)
+; ATHLON-NEXT:    fmul %st, %st(0)
 ; ATHLON-NEXT:    fxch %st(4)
-; ATHLON-NEXT:    fcmove %st(4), %st(0)
+; ATHLON-NEXT:    fcmove %st(4), %st
 ; ATHLON-NEXT:    fstp %st(4)
 ; ATHLON-NEXT:    fxch %st(3)
 ; ATHLON-NEXT:    fstps 12(%ecx)
@@ -332,13 +332,13 @@ define void @test6(i32 %C, <4 x float>* %A, <4 x float>* %B) nounwind {
 ; MCU-NEXT:    flds 4(%ecx)
 ; MCU-NEXT:    flds 8(%ecx)
 ; MCU-NEXT:    flds 12(%ecx)
-; MCU-NEXT:    fmul %st(0), %st(0)
+; MCU-NEXT:    fmul %st, %st(0)
 ; MCU-NEXT:    fxch %st(1)
-; MCU-NEXT:    fmul %st(0), %st(0)
+; MCU-NEXT:    fmul %st, %st(0)
 ; MCU-NEXT:    fxch %st(2)
-; MCU-NEXT:    fmul %st(0), %st(0)
+; MCU-NEXT:    fmul %st, %st(0)
 ; MCU-NEXT:    fxch %st(3)
-; MCU-NEXT:    fmul %st(0), %st(0)
+; MCU-NEXT:    fmul %st, %st(0)
 ; MCU-NEXT:    testl %eax, %eax
 ; MCU-NEXT:    flds (%edx)
 ; MCU-NEXT:    je .LBB5_2
diff --git a/test/CodeGen/X86/select_const.ll b/test/CodeGen/X86/select_const.ll
index 0f252c8261f465f4348bf5b736d628a149c5d5a6..8ab603d396f7987a81d77dfba35e88563fa6c1b5 100644
--- a/test/CodeGen/X86/select_const.ll
+++ b/test/CodeGen/X86/select_const.ll
@@ -193,9 +193,8 @@ define i32 @select_C_Cplus1_zeroext(i1 zeroext %cond) {
 define i32 @select_C_Cplus1_signext(i1 signext %cond) {
 ; CHECK-LABEL: select_C_Cplus1_signext:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    andl $1, %edi
-; CHECK-NEXT:    movl $42, %eax
-; CHECK-NEXT:    subl %edi, %eax
+; CHECK-NEXT:    # kill: def $edi killed $edi def $rdi
+; CHECK-NEXT:    leal 42(%rdi), %eax
 ; CHECK-NEXT:    retq
   %sel = select i1 %cond, i32 41, i32 42
   ret i32 %sel
diff --git a/test/CodeGen/X86/setcc-combine.ll b/test/CodeGen/X86/setcc-combine.ll
index 56cff4ab6f2fd37621f48c4686ec2dbd3204ee97..65e8a6d6ed256458b9d1173d0d41eab243893f73 100644
--- a/test/CodeGen/X86/setcc-combine.ll
+++ b/test/CodeGen/X86/setcc-combine.ll
@@ -1,15 +1,22 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=x86_64-unknown-unknown -mcpu=generic < %s | FileCheck %s
+; RUN: llc -mtriple=x86_64-unknown-unknown -mattr=+sse2 < %s | FileCheck %s --check-prefixes=CHECK,SSE2
+; RUN: llc -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 < %s | FileCheck %s --check-prefixes=CHECK,SSE41
 
 define i32 @test_eq_1(<4 x i32> %A, <4 x i32> %B) {
-; CHECK-LABEL: test_eq_1:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    pcmpgtd %xmm0, %xmm1
-; CHECK-NEXT:    pcmpeqd %xmm0, %xmm0
-; CHECK-NEXT:    pxor %xmm1, %xmm0
-; CHECK-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
-; CHECK-NEXT:    movd %xmm0, %eax
-; CHECK-NEXT:    retq
+; SSE2-LABEL: test_eq_1:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    pcmpgtd %xmm0, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE2-NEXT:    movd %xmm0, %eax
+; SSE2-NEXT:    notl %eax
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: test_eq_1:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm1
+; SSE41-NEXT:    pextrd $1, %xmm1, %eax
+; SSE41-NEXT:    notl %eax
+; SSE41-NEXT:    retq
   %cmp = icmp slt <4 x i32> %A, %B
   %sext = sext <4 x i1> %cmp to <4 x i32>
   %cmp1 = icmp eq <4 x i32> %sext, zeroinitializer
@@ -19,12 +26,18 @@ define i32 @test_eq_1(<4 x i32> %A, <4 x i32> %B) {
 }
 
 define i32 @test_ne_1(<4 x i32> %A, <4 x i32> %B) {
-; CHECK-LABEL: test_ne_1:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    pcmpgtd %xmm0, %xmm1
-; CHECK-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
-; CHECK-NEXT:    movd %xmm0, %eax
-; CHECK-NEXT:    retq
+; SSE2-LABEL: test_ne_1:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    pcmpgtd %xmm0, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE2-NEXT:    movd %xmm0, %eax
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: test_ne_1:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm1
+; SSE41-NEXT:    pextrd $1, %xmm1, %eax
+; SSE41-NEXT:    retq
   %cmp = icmp slt <4 x i32> %A, %B
   %sext = sext <4 x i1> %cmp to <4 x i32>
   %cmp1 = icmp ne <4 x i32> %sext, zeroinitializer
@@ -47,14 +60,20 @@ define i32 @test_le_1(<4 x i32> %A, <4 x i32> %B) {
 }
 
 define i32 @test_ge_1(<4 x i32> %A, <4 x i32> %B) {
-; CHECK-LABEL: test_ge_1:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    pcmpgtd %xmm0, %xmm1
-; CHECK-NEXT:    pcmpeqd %xmm0, %xmm0
-; CHECK-NEXT:    pxor %xmm1, %xmm0
-; CHECK-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
-; CHECK-NEXT:    movd %xmm0, %eax
-; CHECK-NEXT:    retq
+; SSE2-LABEL: test_ge_1:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    pcmpgtd %xmm0, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE2-NEXT:    movd %xmm0, %eax
+; SSE2-NEXT:    notl %eax
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: test_ge_1:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm1
+; SSE41-NEXT:    pextrd $1, %xmm1, %eax
+; SSE41-NEXT:    notl %eax
+; SSE41-NEXT:    retq
   %cmp = icmp slt <4 x i32> %A, %B
   %sext = sext <4 x i1> %cmp to <4 x i32>
   %cmp1 = icmp sge <4 x i32> %sext, zeroinitializer
@@ -64,12 +83,18 @@ define i32 @test_ge_1(<4 x i32> %A, <4 x i32> %B) {
 }
 
 define i32 @test_lt_1(<4 x i32> %A, <4 x i32> %B) {
-; CHECK-LABEL: test_lt_1:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    pcmpgtd %xmm0, %xmm1
-; CHECK-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
-; CHECK-NEXT:    movd %xmm0, %eax
-; CHECK-NEXT:    retq
+; SSE2-LABEL: test_lt_1:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    pcmpgtd %xmm0, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
+; SSE2-NEXT:    movd %xmm0, %eax
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: test_lt_1:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm1
+; SSE41-NEXT:    pextrd $1, %xmm1, %eax
+; SSE41-NEXT:    retq
   %cmp = icmp slt <4 x i32> %A, %B
   %sext = sext <4 x i1> %cmp to <4 x i32>
   %cmp1 = icmp slt <4 x i32> %sext, zeroinitializer
@@ -92,14 +117,20 @@ define i32 @test_gt_1(<4 x i32> %A, <4 x i32> %B) {
 }
 
 define i32 @test_eq_2(<4 x i32> %A, <4 x i32> %B) {
-; CHECK-LABEL: test_eq_2:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    pcmpgtd %xmm1, %xmm0
-; CHECK-NEXT:    pcmpeqd %xmm1, %xmm1
-; CHECK-NEXT:    pxor %xmm0, %xmm1
-; CHECK-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
-; CHECK-NEXT:    movd %xmm0, %eax
-; CHECK-NEXT:    retq
+; SSE2-LABEL: test_eq_2:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    pcmpgtd %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE2-NEXT:    movd %xmm0, %eax
+; SSE2-NEXT:    notl %eax
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: test_eq_2:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    pcmpgtd %xmm1, %xmm0
+; SSE41-NEXT:    pextrd $1, %xmm0, %eax
+; SSE41-NEXT:    notl %eax
+; SSE41-NEXT:    retq
   %cmp = icmp slt <4 x i32> %B, %A
   %sext = sext <4 x i1> %cmp to <4 x i32>
   %cmp1 = icmp eq <4 x i32> %sext, zeroinitializer
@@ -109,12 +140,18 @@ define i32 @test_eq_2(<4 x i32> %A, <4 x i32> %B) {
 }
 
 define i32 @test_ne_2(<4 x i32> %A, <4 x i32> %B) {
-; CHECK-LABEL: test_ne_2:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    pcmpgtd %xmm1, %xmm0
-; CHECK-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
-; CHECK-NEXT:    movd %xmm0, %eax
-; CHECK-NEXT:    retq
+; SSE2-LABEL: test_ne_2:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    pcmpgtd %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE2-NEXT:    movd %xmm0, %eax
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: test_ne_2:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    pcmpgtd %xmm1, %xmm0
+; SSE41-NEXT:    pextrd $1, %xmm0, %eax
+; SSE41-NEXT:    retq
   %cmp = icmp slt <4 x i32> %B, %A
   %sext = sext <4 x i1> %cmp to <4 x i32>
   %cmp1 = icmp ne <4 x i32> %sext, zeroinitializer
@@ -124,14 +161,20 @@ define i32 @test_ne_2(<4 x i32> %A, <4 x i32> %B) {
 }
 
 define i32 @test_le_2(<4 x i32> %A, <4 x i32> %B) {
-; CHECK-LABEL: test_le_2:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    pcmpgtd %xmm1, %xmm0
-; CHECK-NEXT:    pcmpeqd %xmm1, %xmm1
-; CHECK-NEXT:    pxor %xmm0, %xmm1
-; CHECK-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,2,3]
-; CHECK-NEXT:    movd %xmm0, %eax
-; CHECK-NEXT:    retq
+; SSE2-LABEL: test_le_2:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    pcmpgtd %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE2-NEXT:    movd %xmm0, %eax
+; SSE2-NEXT:    notl %eax
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: test_le_2:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    pcmpgtd %xmm1, %xmm0
+; SSE41-NEXT:    pextrd $1, %xmm0, %eax
+; SSE41-NEXT:    notl %eax
+; SSE41-NEXT:    retq
   %cmp = icmp slt <4 x i32> %B, %A
   %sext = sext <4 x i1> %cmp to <4 x i32>
   %cmp1 = icmp sle <4 x i32> zeroinitializer, %sext
@@ -154,12 +197,18 @@ define i32 @test_ge_2(<4 x i32> %A, <4 x i32> %B) {
 }
 
 define i32 @test_lt_2(<4 x i32> %A, <4 x i32> %B) {
-; CHECK-LABEL: test_lt_2:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    pcmpgtd %xmm1, %xmm0
-; CHECK-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
-; CHECK-NEXT:    movd %xmm0, %eax
-; CHECK-NEXT:    retq
+; SSE2-LABEL: test_lt_2:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    pcmpgtd %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE2-NEXT:    movd %xmm0, %eax
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: test_lt_2:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    pcmpgtd %xmm1, %xmm0
+; SSE41-NEXT:    pextrd $1, %xmm0, %eax
+; SSE41-NEXT:    retq
   %cmp = icmp slt <4 x i32> %B, %A
   %sext = sext <4 x i1> %cmp to <4 x i32>
   %cmp1 = icmp slt <4 x i32> zeroinitializer, %sext
@@ -169,12 +218,18 @@ define i32 @test_lt_2(<4 x i32> %A, <4 x i32> %B) {
 }
 
 define i32 @test_gt_2(<4 x i32> %A, <4 x i32> %B) {
-; CHECK-LABEL: test_gt_2:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    pcmpgtd %xmm1, %xmm0
-; CHECK-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
-; CHECK-NEXT:    movd %xmm0, %eax
-; CHECK-NEXT:    retq
+; SSE2-LABEL: test_gt_2:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    pcmpgtd %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; SSE2-NEXT:    movd %xmm0, %eax
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: test_gt_2:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    pcmpgtd %xmm1, %xmm0
+; SSE41-NEXT:    pextrd $1, %xmm0, %eax
+; SSE41-NEXT:    retq
   %cmp = icmp slt <4 x i32> %B, %A
   %sext = sext <4 x i1> %cmp to <4 x i32>
   %cmp1 = icmp sgt <4 x i32> zeroinitializer, %sext
@@ -207,3 +262,101 @@ define void @test_i1_uge(i1 *%A2) {
   ret void
 }
 
+; This should not get folded to 0.
+
+define i64 @PR40657(i8 %var2, i8 %var9) {
+; CHECK-LABEL: PR40657:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    notb %sil
+; CHECK-NEXT:    addb %dil, %sil
+; CHECK-NEXT:    movzbl %sil, %eax
+; CHECK-NEXT:    andl $1, %eax
+; CHECK-NEXT:    retq
+  %var6 = trunc i8 %var9 to i1
+  %var7 = trunc i8 175 to i1
+  %var3 = sub nsw i1 %var6, %var7
+  %var4 = icmp eq i64 1114591064, 1114591064
+  %var1 = udiv i1 %var3, %var4
+  %var0 = trunc i8 %var2 to i1
+  %res = sub nsw nuw i1 %var0, %var1
+  %res.cast = zext i1 %res to i64
+  ret i64 %res.cast
+}
+
+; This should not get folded to 0.
+
+define i64 @PR40657_commute(i8 %var7, i8 %var8, i8 %var9) {
+; CHECK-LABEL: PR40657_commute:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    subb %dil, %sil
+; CHECK-NEXT:    subb %sil, %dl
+; CHECK-NEXT:    subb %dl, %sil
+; CHECK-NEXT:    xorb %dl, %sil
+; CHECK-NEXT:    subb %sil, %dl
+; CHECK-NEXT:    movzbl %dl, %eax
+; CHECK-NEXT:    andl $1, %eax
+; CHECK-NEXT:    retq
+  %var4 = trunc i8 %var9 to i1
+  %var5 = trunc i8 %var8 to i1
+  %var6 = trunc i8 %var7 to i1
+  %var3 = sub nsw nuw i1 %var5, %var6
+  %var0 = sub nuw i1 %var4, %var3
+  %var2 = sub i1 %var3, %var0
+  %var1 = icmp ne i1 %var0, %var2
+  %res = sub nsw nuw i1 %var0, %var1
+  %res.cast = zext i1 %res to i64
+  ret i64 %res.cast
+}
+
+define i64 @sub_to_shift_to_add(i32 %x, i32 %y, i64 %s1, i64 %s2) {
+; CHECK-LABEL: sub_to_shift_to_add:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %rdx, %rax
+; CHECK-NEXT:    addl %esi, %esi
+; CHECK-NEXT:    cmpl %esi, %edi
+; CHECK-NEXT:    cmovneq %rcx, %rax
+; CHECK-NEXT:    retq
+  %sub = sub i32 %x, %y
+  %cmp = icmp eq i32 %sub, %y
+  %r = select i1 %cmp, i64 %s1, i64 %s2
+  ret i64 %r
+}
+
+define <4 x float> @sub_to_shift_to_add_vec(<4 x i32> %x, <4 x i32> %y, <4 x float> %s1, <4 x float> %s2) {
+; SSE2-LABEL: sub_to_shift_to_add_vec:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    paddd %xmm1, %xmm1
+; SSE2-NEXT:    pcmpeqd %xmm0, %xmm1
+; SSE2-NEXT:    pand %xmm1, %xmm2
+; SSE2-NEXT:    pandn %xmm3, %xmm1
+; SSE2-NEXT:    por %xmm2, %xmm1
+; SSE2-NEXT:    movdqa %xmm1, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: sub_to_shift_to_add_vec:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    paddd %xmm1, %xmm1
+; SSE41-NEXT:    pcmpeqd %xmm1, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm2, %xmm3
+; SSE41-NEXT:    movaps %xmm3, %xmm0
+; SSE41-NEXT:    retq
+  %sub = sub <4 x i32> %x, %y
+  %cmp = icmp eq <4 x i32> %sub, %y
+  %r = select <4 x i1> %cmp, <4 x float> %s1, <4 x float> %s2
+  ret <4 x float> %r
+}
+
+define i64 @sub_constant_to_shift_to_add(i32 %x, i64 %s1, i64 %s2) {
+; CHECK-LABEL: sub_constant_to_shift_to_add:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %rsi, %rax
+; CHECK-NEXT:    addl %edi, %edi
+; CHECK-NEXT:    cmpl $42, %edi
+; CHECK-NEXT:    cmovneq %rdx, %rax
+; CHECK-NEXT:    retq
+  %sub = sub i32 42, %x
+  %cmp = icmp eq i32 %sub, %x
+  %r = select i1 %cmp, i64 %s1, i64 %s2
+  ret i64 %r
+}
+
diff --git a/test/CodeGen/X86/setcc-logic.ll b/test/CodeGen/X86/setcc-logic.ll
index 231b9f4d0860e21c2820503c9c5a6408a41774d8..46db9271d31b07bbd9a7fe4a332c860bc0a8f56f 100644
--- a/test/CodeGen/X86/setcc-logic.ll
+++ b/test/CodeGen/X86/setcc-logic.ll
@@ -482,3 +482,99 @@ define <4 x i1> @and_eq_vec(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c, <4 x i32>
   ret <4 x i1> %and
 }
 
+define i1 @or_icmps_const_1bit_diff(i8 %x) {
+; CHECK-LABEL: or_icmps_const_1bit_diff:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addb $-43, %dil
+; CHECK-NEXT:    testb $-3, %dil
+; CHECK-NEXT:    sete %al
+; CHECK-NEXT:    retq
+  %a = icmp eq i8 %x, 43
+  %b = icmp eq i8 %x, 45
+  %r = or i1 %a, %b
+  ret i1 %r
+}
+
+define i1 @and_icmps_const_1bit_diff(i32 %x) {
+; CHECK-LABEL: and_icmps_const_1bit_diff:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    addl $-44, %edi
+; CHECK-NEXT:    testl $-17, %edi
+; CHECK-NEXT:    setne %al
+; CHECK-NEXT:    retq
+  %a = icmp ne i32 %x, 44
+  %b = icmp ne i32 %x, 60
+  %r = and i1 %a, %b
+  ret i1 %r
+}
+
+; Negative test - extra use prevents optimization
+
+define i1 @or_icmps_const_1bit_diff_extra_use(i8 %x, i8* %p) {
+; CHECK-LABEL: or_icmps_const_1bit_diff_extra_use:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    cmpb $45, %dil
+; CHECK-NEXT:    sete %cl
+; CHECK-NEXT:    cmpb $43, %dil
+; CHECK-NEXT:    sete %al
+; CHECK-NEXT:    sete (%rsi)
+; CHECK-NEXT:    orb %cl, %al
+; CHECK-NEXT:    retq
+  %a = icmp eq i8 %x, 43
+  %b = icmp eq i8 %x, 45
+  %r = or i1 %a, %b
+  %z = zext i1 %a to i8
+  store i8 %z, i8* %p
+  ret i1 %r
+}
+
+; Negative test - constant diff is >1 bit
+
+define i1 @and_icmps_const_not1bit_diff(i32 %x) {
+; CHECK-LABEL: and_icmps_const_not1bit_diff:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    cmpl $44, %edi
+; CHECK-NEXT:    setne %cl
+; CHECK-NEXT:    cmpl $92, %edi
+; CHECK-NEXT:    setne %al
+; CHECK-NEXT:    andb %cl, %al
+; CHECK-NEXT:    retq
+  %a = icmp ne i32 %x, 44
+  %b = icmp ne i32 %x, 92
+  %r = and i1 %a, %b
+  ret i1 %r
+}
+
+; Negative test - wrong comparison
+
+define i1 @and_icmps_const_1bit_diff_wrong_pred(i32 %x) {
+; CHECK-LABEL: and_icmps_const_1bit_diff_wrong_pred:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    cmpl $43, %edi
+; CHECK-NEXT:    sete %cl
+; CHECK-NEXT:    cmpl $45, %edi
+; CHECK-NEXT:    setl %al
+; CHECK-NEXT:    orb %cl, %al
+; CHECK-NEXT:    retq
+  %a = icmp eq i32 %x, 43
+  %b = icmp slt i32 %x, 45
+  %r = or i1 %a, %b
+  ret i1 %r
+}
+
+; Negative test - no common operand
+
+define i1 @and_icmps_const_1bit_diff_common_op(i32 %x, i32 %y) {
+; CHECK-LABEL: and_icmps_const_1bit_diff_common_op:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    cmpl $43, %edi
+; CHECK-NEXT:    sete %cl
+; CHECK-NEXT:    cmpl $45, %esi
+; CHECK-NEXT:    sete %al
+; CHECK-NEXT:    orb %cl, %al
+; CHECK-NEXT:    retq
+  %a = icmp eq i32 %x, 43
+  %b = icmp eq i32 %y, 45
+  %r = or i1 %a, %b
+  ret i1 %r
+}
diff --git a/test/CodeGen/X86/sha-schedule.ll b/test/CodeGen/X86/sha-schedule.ll
deleted file mode 100644
index 7069bd3a1192e04b7703244dd8e9bdad1a0870a3..0000000000000000000000000000000000000000
--- a/test/CodeGen/X86/sha-schedule.ll
+++ /dev/null
@@ -1,242 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=x86-64 -mattr=+sha | FileCheck %s --check-prefix=CHECK --check-prefix=GENERIC
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=goldmont | FileCheck %s --check-prefix=CHECK --check-prefix=GOLDMONT
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=cannonlake | FileCheck %s --check-prefix=CHECK --check-prefix=CANNONLAKE
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=znver1 | FileCheck %s --check-prefix=CHECK --check-prefix=ZNVER1
-
-;
-; SHA1
-;
-
-define <4 x i32> @test_sha1msg1(<4 x i32> %a0, <4 x i32> %a1, <4 x i32> *%a2) {
-; GENERIC-LABEL: test_sha1msg1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    sha1msg1 %xmm1, %xmm0 # sched: [5:1.00]
-; GENERIC-NEXT:    sha1msg1 (%rdi), %xmm0 # sched: [11:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; GOLDMONT-LABEL: test_sha1msg1:
-; GOLDMONT:       # %bb.0:
-; GOLDMONT-NEXT:    sha1msg1 %xmm1, %xmm0 # sched: [4:1.00]
-; GOLDMONT-NEXT:    sha1msg1 (%rdi), %xmm0 # sched: [7:1.00]
-; GOLDMONT-NEXT:    retq # sched: [4:1.00]
-;
-; CANNONLAKE-LABEL: test_sha1msg1:
-; CANNONLAKE:       # %bb.0:
-; CANNONLAKE-NEXT:    sha1msg1 %xmm1, %xmm0 # sched: [4:0.50]
-; CANNONLAKE-NEXT:    sha1msg1 (%rdi), %xmm0 # sched: [10:0.50]
-; CANNONLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_sha1msg1:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    sha1msg1 %xmm1, %xmm0 # sched: [2:1.00]
-; ZNVER1-NEXT:    sha1msg1 (%rdi), %xmm0 # sched: [9:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = load <4 x i32>, <4 x i32>* %a2
-  %2 = tail call <4 x i32> @llvm.x86.sha1msg1(<4 x i32> %a0, <4 x i32> %a1)
-  %3 = tail call <4 x i32> @llvm.x86.sha1msg1(<4 x i32> %2, <4 x i32> %1)
-  ret <4 x i32> %3
-}
-declare <4 x i32> @llvm.x86.sha1msg1(<4 x i32>, <4 x i32>)
-
-define <4 x i32> @test_sha1msg2(<4 x i32> %a0, <4 x i32> %a1, <4 x i32> *%a2) {
-; GENERIC-LABEL: test_sha1msg2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    sha1msg2 %xmm1, %xmm0 # sched: [5:1.00]
-; GENERIC-NEXT:    sha1msg2 (%rdi), %xmm0 # sched: [11:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; GOLDMONT-LABEL: test_sha1msg2:
-; GOLDMONT:       # %bb.0:
-; GOLDMONT-NEXT:    sha1msg2 %xmm1, %xmm0 # sched: [4:1.00]
-; GOLDMONT-NEXT:    sha1msg2 (%rdi), %xmm0 # sched: [7:1.00]
-; GOLDMONT-NEXT:    retq # sched: [4:1.00]
-;
-; CANNONLAKE-LABEL: test_sha1msg2:
-; CANNONLAKE:       # %bb.0:
-; CANNONLAKE-NEXT:    sha1msg2 %xmm1, %xmm0 # sched: [4:0.50]
-; CANNONLAKE-NEXT:    sha1msg2 (%rdi), %xmm0 # sched: [10:0.50]
-; CANNONLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_sha1msg2:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    sha1msg2 %xmm1, %xmm0 # sched: [1:0.50]
-; ZNVER1-NEXT:    sha1msg2 (%rdi), %xmm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = load <4 x i32>, <4 x i32>* %a2
-  %2 = tail call <4 x i32> @llvm.x86.sha1msg2(<4 x i32> %a0, <4 x i32> %a1)
-  %3 = tail call <4 x i32> @llvm.x86.sha1msg2(<4 x i32> %2, <4 x i32> %1)
-  ret <4 x i32> %3
-}
-declare <4 x i32> @llvm.x86.sha1msg2(<4 x i32>, <4 x i32>)
-
-define <4 x i32> @test_sha1nexte(<4 x i32> %a0, <4 x i32> %a1, <4 x i32> *%a2) {
-; GENERIC-LABEL: test_sha1nexte:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    sha1nexte %xmm1, %xmm0 # sched: [5:1.00]
-; GENERIC-NEXT:    sha1nexte (%rdi), %xmm0 # sched: [11:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; GOLDMONT-LABEL: test_sha1nexte:
-; GOLDMONT:       # %bb.0:
-; GOLDMONT-NEXT:    sha1nexte %xmm1, %xmm0 # sched: [4:1.00]
-; GOLDMONT-NEXT:    sha1nexte (%rdi), %xmm0 # sched: [7:1.00]
-; GOLDMONT-NEXT:    retq # sched: [4:1.00]
-;
-; CANNONLAKE-LABEL: test_sha1nexte:
-; CANNONLAKE:       # %bb.0:
-; CANNONLAKE-NEXT:    sha1nexte %xmm1, %xmm0 # sched: [4:0.50]
-; CANNONLAKE-NEXT:    sha1nexte (%rdi), %xmm0 # sched: [10:0.50]
-; CANNONLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_sha1nexte:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    sha1nexte %xmm1, %xmm0 # sched: [1:1.00]
-; ZNVER1-NEXT:    sha1nexte (%rdi), %xmm0 # sched: [8:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = load <4 x i32>, <4 x i32>* %a2
-  %2 = tail call <4 x i32> @llvm.x86.sha1nexte(<4 x i32> %a0, <4 x i32> %a1)
-  %3 = tail call <4 x i32> @llvm.x86.sha1nexte(<4 x i32> %2, <4 x i32> %1)
-  ret <4 x i32> %3
-}
-declare <4 x i32> @llvm.x86.sha1nexte(<4 x i32>, <4 x i32>)
-
-define <4 x i32> @test_sha1rnds4(<4 x i32> %a0, <4 x i32> %a1, <4 x i32> *%a2) {
-; GENERIC-LABEL: test_sha1rnds4:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    sha1rnds4 $3, %xmm1, %xmm0 # sched: [5:1.00]
-; GENERIC-NEXT:    sha1rnds4 $3, (%rdi), %xmm0 # sched: [11:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; GOLDMONT-LABEL: test_sha1rnds4:
-; GOLDMONT:       # %bb.0:
-; GOLDMONT-NEXT:    sha1rnds4 $3, %xmm1, %xmm0 # sched: [4:1.00]
-; GOLDMONT-NEXT:    sha1rnds4 $3, (%rdi), %xmm0 # sched: [7:1.00]
-; GOLDMONT-NEXT:    retq # sched: [4:1.00]
-;
-; CANNONLAKE-LABEL: test_sha1rnds4:
-; CANNONLAKE:       # %bb.0:
-; CANNONLAKE-NEXT:    sha1rnds4 $3, %xmm1, %xmm0 # sched: [4:0.50]
-; CANNONLAKE-NEXT:    sha1rnds4 $3, (%rdi), %xmm0 # sched: [10:0.50]
-; CANNONLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_sha1rnds4:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    sha1rnds4 $3, %xmm1, %xmm0 # sched: [6:1.00]
-; ZNVER1-NEXT:    sha1rnds4 $3, (%rdi), %xmm0 # sched: [13:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = load <4 x i32>, <4 x i32>* %a2
-  %2 = tail call <4 x i32> @llvm.x86.sha1rnds4(<4 x i32> %a0, <4 x i32> %a1, i8 3)
-  %3 = tail call <4 x i32> @llvm.x86.sha1rnds4(<4 x i32> %2, <4 x i32> %1, i8 3)
-  ret <4 x i32> %3
-}
-declare <4 x i32> @llvm.x86.sha1rnds4(<4 x i32>, <4 x i32>, i8)
-
-;
-; SHA256
-;
-
-define <4 x i32> @test_sha256msg1(<4 x i32> %a0, <4 x i32> %a1, <4 x i32> *%a2) {
-; GENERIC-LABEL: test_sha256msg1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    sha256msg1 %xmm1, %xmm0 # sched: [5:1.00]
-; GENERIC-NEXT:    sha256msg1 (%rdi), %xmm0 # sched: [11:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; GOLDMONT-LABEL: test_sha256msg1:
-; GOLDMONT:       # %bb.0:
-; GOLDMONT-NEXT:    sha256msg1 %xmm1, %xmm0 # sched: [4:1.00]
-; GOLDMONT-NEXT:    sha256msg1 (%rdi), %xmm0 # sched: [7:1.00]
-; GOLDMONT-NEXT:    retq # sched: [4:1.00]
-;
-; CANNONLAKE-LABEL: test_sha256msg1:
-; CANNONLAKE:       # %bb.0:
-; CANNONLAKE-NEXT:    sha256msg1 %xmm1, %xmm0 # sched: [4:0.50]
-; CANNONLAKE-NEXT:    sha256msg1 (%rdi), %xmm0 # sched: [10:0.50]
-; CANNONLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_sha256msg1:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    sha256msg1 %xmm1, %xmm0 # sched: [2:1.00]
-; ZNVER1-NEXT:    sha256msg1 (%rdi), %xmm0 # sched: [9:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = load <4 x i32>, <4 x i32>* %a2
-  %2 = tail call <4 x i32> @llvm.x86.sha256msg1(<4 x i32> %a0, <4 x i32> %a1)
-  %3 = tail call <4 x i32> @llvm.x86.sha256msg1(<4 x i32> %2, <4 x i32> %1)
-  ret <4 x i32> %3
-}
-declare <4 x i32> @llvm.x86.sha256msg1(<4 x i32>, <4 x i32>)
-
-define <4 x i32> @test_sha256msg2(<4 x i32> %a0, <4 x i32> %a1, <4 x i32> *%a2) {
-; GENERIC-LABEL: test_sha256msg2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    sha256msg2 %xmm1, %xmm0 # sched: [5:1.00]
-; GENERIC-NEXT:    sha256msg2 (%rdi), %xmm0 # sched: [11:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; GOLDMONT-LABEL: test_sha256msg2:
-; GOLDMONT:       # %bb.0:
-; GOLDMONT-NEXT:    sha256msg2 %xmm1, %xmm0 # sched: [4:1.00]
-; GOLDMONT-NEXT:    sha256msg2 (%rdi), %xmm0 # sched: [7:1.00]
-; GOLDMONT-NEXT:    retq # sched: [4:1.00]
-;
-; CANNONLAKE-LABEL: test_sha256msg2:
-; CANNONLAKE:       # %bb.0:
-; CANNONLAKE-NEXT:    sha256msg2 %xmm1, %xmm0 # sched: [4:0.50]
-; CANNONLAKE-NEXT:    sha256msg2 (%rdi), %xmm0 # sched: [10:0.50]
-; CANNONLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_sha256msg2:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    sha256msg2 %xmm1, %xmm0 # sched: [100:0.25]
-; ZNVER1-NEXT:    sha256msg2 (%rdi), %xmm0 # sched: [100:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = load <4 x i32>, <4 x i32>* %a2
-  %2 = tail call <4 x i32> @llvm.x86.sha256msg2(<4 x i32> %a0, <4 x i32> %a1)
-  %3 = tail call <4 x i32> @llvm.x86.sha256msg2(<4 x i32> %2, <4 x i32> %1)
-  ret <4 x i32> %3
-}
-declare <4 x i32> @llvm.x86.sha256msg2(<4 x i32>, <4 x i32>)
-
-define <4 x i32> @test_sha256rnds2(<4 x i32> %a0, <4 x i32> %a1, <4 x i32> %a2, <4 x i32> *%a3) {
-; GENERIC-LABEL: test_sha256rnds2:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    movaps %xmm0, %xmm3 # sched: [1:1.00]
-; GENERIC-NEXT:    movaps %xmm2, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    sha256rnds2 %xmm0, %xmm1, %xmm3 # sched: [5:1.00]
-; GENERIC-NEXT:    sha256rnds2 %xmm0, (%rdi), %xmm3 # sched: [11:1.00]
-; GENERIC-NEXT:    movaps %xmm3, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; GOLDMONT-LABEL: test_sha256rnds2:
-; GOLDMONT:       # %bb.0:
-; GOLDMONT-NEXT:    movaps %xmm0, %xmm3 # sched: [1:0.50]
-; GOLDMONT-NEXT:    movaps %xmm2, %xmm0 # sched: [1:0.50]
-; GOLDMONT-NEXT:    sha256rnds2 %xmm0, %xmm1, %xmm3 # sched: [4:1.00]
-; GOLDMONT-NEXT:    sha256rnds2 %xmm0, (%rdi), %xmm3 # sched: [7:1.00]
-; GOLDMONT-NEXT:    movaps %xmm3, %xmm0 # sched: [1:0.50]
-; GOLDMONT-NEXT:    retq # sched: [4:1.00]
-;
-; CANNONLAKE-LABEL: test_sha256rnds2:
-; CANNONLAKE:       # %bb.0:
-; CANNONLAKE-NEXT:    vmovaps %xmm0, %xmm3 # sched: [1:0.33]
-; CANNONLAKE-NEXT:    vmovaps %xmm2, %xmm0 # sched: [1:0.33]
-; CANNONLAKE-NEXT:    sha256rnds2 %xmm0, %xmm1, %xmm3 # sched: [4:0.50]
-; CANNONLAKE-NEXT:    sha256rnds2 %xmm0, (%rdi), %xmm3 # sched: [10:0.50]
-; CANNONLAKE-NEXT:    vmovaps %xmm3, %xmm0 # sched: [1:0.33]
-; CANNONLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; ZNVER1-LABEL: test_sha256rnds2:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vmovaps %xmm0, %xmm3 # sched: [1:0.25]
-; ZNVER1-NEXT:    vmovaps %xmm2, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    sha256rnds2 %xmm0, %xmm1, %xmm3 # sched: [4:1.00]
-; ZNVER1-NEXT:    sha256rnds2 %xmm0, (%rdi), %xmm3 # sched: [11:1.00]
-; ZNVER1-NEXT:    vmovaps %xmm3, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = load <4 x i32>, <4 x i32>* %a3
-  %2 = tail call <4 x i32> @llvm.x86.sha256rnds2(<4 x i32> %a0, <4 x i32> %a1, <4 x i32> %a2)
-  %3 = tail call <4 x i32> @llvm.x86.sha256rnds2(<4 x i32> %2, <4 x i32> %1, <4 x i32> %a2)
-  ret <4 x i32> %3
-}
-declare <4 x i32> @llvm.x86.sha256rnds2(<4 x i32>, <4 x i32>, <4 x i32>)
diff --git a/test/CodeGen/X86/shift-and-x86_64.ll b/test/CodeGen/X86/shift-and-x86_64.ll
new file mode 100644
index 0000000000000000000000000000000000000000..9dd7974d0ff4d6869f103da03c49787476f8ebe8
--- /dev/null
+++ b/test/CodeGen/X86/shift-and-x86_64.ll
@@ -0,0 +1,52 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown | FileCheck %s
+
+define { i64, i64 } @PR36721_u8(i64, i64, i8 zeroext) nounwind {
+; CHECK-LABEL: PR36721_u8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl %edx, %ecx
+; CHECK-NEXT:    movq %rsi, %rdx
+; CHECK-NEXT:    movq %rdi, %rax
+; CHECK-NEXT:    shldq %cl, %rdi, %rdx
+; CHECK-NEXT:    # kill: def $cl killed $cl killed $ecx
+; CHECK-NEXT:    shlq %cl, %rax
+; CHECK-NEXT:    retq
+  %4 = zext i64 %1 to i128
+  %5 = shl nuw i128 %4, 64
+  %6 = zext i64 %0 to i128
+  %7 = or i128 %5, %6
+  %8 = and i8 %2, 63
+  %9 = zext i8 %8 to i128
+  %10 = shl i128 %7, %9
+  %11 = trunc i128 %10 to i64
+  %12 = lshr i128 %10, 64
+  %13 = trunc i128 %12 to i64
+  %14 = insertvalue { i64, i64 } undef, i64 %11, 0
+  %15 = insertvalue { i64, i64 } %14, i64 %13, 1
+  ret { i64, i64 } %15
+}
+
+define { i64, i64 } @PR36721_u32(i64, i64, i32) nounwind {
+; CHECK-LABEL: PR36721_u32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl %edx, %ecx
+; CHECK-NEXT:    movq %rsi, %rdx
+; CHECK-NEXT:    movq %rdi, %rax
+; CHECK-NEXT:    shldq %cl, %rdi, %rdx
+; CHECK-NEXT:    # kill: def $cl killed $cl killed $ecx
+; CHECK-NEXT:    shlq %cl, %rax
+; CHECK-NEXT:    retq
+  %4 = zext i64 %1 to i128
+  %5 = shl nuw i128 %4, 64
+  %6 = zext i64 %0 to i128
+  %7 = or i128 %5, %6
+  %8 = and i32 %2, 63
+  %9 = zext i32 %8 to i128
+  %10 = shl i128 %7, %9
+  %11 = trunc i128 %10 to i64
+  %12 = lshr i128 %10, 64
+  %13 = trunc i128 %12 to i64
+  %14 = insertvalue { i64, i64 } undef, i64 %11, 0
+  %15 = insertvalue { i64, i64 } %14, i64 %13, 1
+  ret { i64, i64 } %15
+}
diff --git a/test/CodeGen/X86/shift-double.ll b/test/CodeGen/X86/shift-double.ll
index b1b5f1cf291d32e85965d2089481809f0685fe10..5e8efa7b9f06f0a34d439035d930c0efc7d8e9ee 100644
--- a/test/CodeGen/X86/shift-double.ll
+++ b/test/CodeGen/X86/shift-double.ll
@@ -128,9 +128,10 @@ define i32 @test4(i32 %A, i32 %B, i8 %C) nounwind {
 define i16 @test5(i16 %A, i16 %B, i8 %C) nounwind {
 ; X86-LABEL: test5:
 ; X86:       # %bb.0:
-; X86-NEXT:    movb {{[0-9]+}}(%esp), %cl
 ; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %edx
 ; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movb {{[0-9]+}}(%esp), %cl
+; X86-NEXT:    andb $15, %cl
 ; X86-NEXT:    shldw %cl, %dx, %ax
 ; X86-NEXT:    retl
 ;
@@ -138,6 +139,7 @@ define i16 @test5(i16 %A, i16 %B, i8 %C) nounwind {
 ; X64:       # %bb.0:
 ; X64-NEXT:    movl %edx, %ecx
 ; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    andb $15, %cl
 ; X64-NEXT:    # kill: def $cl killed $cl killed $ecx
 ; X64-NEXT:    shldw %cl, %si, %ax
 ; X64-NEXT:    # kill: def $ax killed $ax killed $eax
@@ -181,9 +183,10 @@ define i32 @test6(i32 %A, i32 %B, i8 %C) nounwind {
 define i16 @test7(i16 %A, i16 %B, i8 %C) nounwind {
 ; X86-LABEL: test7:
 ; X86:       # %bb.0:
-; X86-NEXT:    movb {{[0-9]+}}(%esp), %cl
 ; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %edx
 ; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movb {{[0-9]+}}(%esp), %cl
+; X86-NEXT:    andb $15, %cl
 ; X86-NEXT:    shrdw %cl, %dx, %ax
 ; X86-NEXT:    retl
 ;
@@ -191,6 +194,7 @@ define i16 @test7(i16 %A, i16 %B, i8 %C) nounwind {
 ; X64:       # %bb.0:
 ; X64-NEXT:    movl %edx, %ecx
 ; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    andb $15, %cl
 ; X64-NEXT:    # kill: def $cl killed $cl killed $ecx
 ; X64-NEXT:    shrdw %cl, %si, %ax
 ; X64-NEXT:    # kill: def $ax killed $ax killed $eax
@@ -454,3 +458,55 @@ define i32 @test17(i32 %hi, i32 %lo, i32 %bits) nounwind {
   %sh = or i32 %sh_lo, %sh_hi
   ret i32 %sh
 }
+
+; PR34641 - Masked Shift Counts
+
+define i32 @shld_safe_i32(i32, i32, i32) {
+; X86-LABEL: shld_safe_i32:
+; X86:       # %bb.0:
+; X86-NEXT:    movb {{[0-9]+}}(%esp), %cl
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    shldl %cl, %edx, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: shld_safe_i32:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %edx, %ecx
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X64-NEXT:    shldl %cl, %esi, %eax
+; X64-NEXT:    retq
+  %4 = and i32 %2, 31
+  %5 = shl i32 %0, %4
+  %6 = sub i32 0, %2
+  %7 = and i32 %6, 31
+  %8 = lshr i32 %1, %7
+  %9 = or i32 %5, %8
+  ret i32 %9
+}
+
+define i32 @shrd_safe_i32(i32, i32, i32) {
+; X86-LABEL: shrd_safe_i32:
+; X86:       # %bb.0:
+; X86-NEXT:    movb {{[0-9]+}}(%esp), %cl
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    shrdl %cl, %edx, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: shrd_safe_i32:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %edx, %ecx
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X64-NEXT:    shrdl %cl, %esi, %eax
+; X64-NEXT:    retq
+  %4 = and i32 %2, 31
+  %5 = lshr i32 %0, %4
+  %6 = sub i32 0, %2
+  %7 = and i32 %6, 31
+  %8 = shl i32 %1, %7
+  %9 = or i32 %5, %8
+  ret i32 %9
+}
diff --git a/test/CodeGen/X86/shift_minsize.ll b/test/CodeGen/X86/shift_minsize.ll
new file mode 100644
index 0000000000000000000000000000000000000000..5ba46544645fa40d9628ee016719d2f664502da9
--- /dev/null
+++ b/test/CodeGen/X86/shift_minsize.ll
@@ -0,0 +1,134 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown | FileCheck %s
+
+define i64 @f0(i64 %val, i64 %amt) minsize optsize {
+; CHECK-LABEL: f0:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %rsi, %rcx
+; CHECK-NEXT:    movq %rdi, %rax
+; CHECK-NEXT:    # kill: def $cl killed $cl killed $rcx
+; CHECK-NEXT:    shlq %cl, %rax
+; CHECK-NEXT:    retq
+  %res = shl i64 %val, %amt
+  ret i64 %res
+}
+
+define i32 @f1(i64 %x, i64 %y) minsize optsize {
+; CHECK-LABEL: f1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %rsi, %rcx
+; CHECK-NEXT:    movq %rdi, %rax
+; CHECK-NEXT:    # kill: def $cl killed $cl killed $rcx
+; CHECK-NEXT:    shlq %cl, %rax
+; CHECK-NEXT:    # kill: def $eax killed $eax killed $rax
+; CHECK-NEXT:    retq
+	%a = shl i64 %x, %y
+	%b = trunc i64 %a to i32
+	ret i32 %b
+}
+
+define i32 @f2(i64 %x, i64 %y) minsize optsize {
+; CHECK-LABEL: f2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %rsi, %rcx
+; CHECK-NEXT:    movq %rdi, %rax
+; CHECK-NEXT:    # kill: def $cl killed $cl killed $rcx
+; CHECK-NEXT:    sarq %cl, %rax
+; CHECK-NEXT:    # kill: def $eax killed $eax killed $rax
+; CHECK-NEXT:    retq
+	%a = ashr i64 %x, %y
+	%b = trunc i64 %a to i32
+	ret i32 %b
+}
+
+define i32 @f3(i64 %x, i64 %y) minsize optsize {
+; CHECK-LABEL: f3:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %rsi, %rcx
+; CHECK-NEXT:    movq %rdi, %rax
+; CHECK-NEXT:    # kill: def $cl killed $cl killed $rcx
+; CHECK-NEXT:    shrq %cl, %rax
+; CHECK-NEXT:    # kill: def $eax killed $eax killed $rax
+; CHECK-NEXT:    retq
+	%a = lshr i64 %x, %y
+	%b = trunc i64 %a to i32
+	ret i32 %b
+}
+
+define dso_local { i64, i64 } @shl128(i64 %x.coerce0, i64 %x.coerce1, i8 signext %y) minsize optsize {
+; CHECK-LABEL: shl128:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    movzbl %dl, %edx
+; CHECK-NEXT:    callq __ashlti3
+; CHECK-NEXT:    popq %rcx
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
+entry:
+  %x.sroa.2.0.insert.ext = zext i64 %x.coerce1 to i128
+  %x.sroa.2.0.insert.shift = shl nuw i128 %x.sroa.2.0.insert.ext, 64
+  %x.sroa.0.0.insert.ext = zext i64 %x.coerce0 to i128
+  %x.sroa.0.0.insert.insert = or i128 %x.sroa.2.0.insert.shift, %x.sroa.0.0.insert.ext
+  %conv = sext i8 %y to i32
+  %sh_prom = zext i32 %conv to i128
+  %shl = shl i128 %x.sroa.0.0.insert.insert, %sh_prom
+  %retval.sroa.0.0.extract.trunc = trunc i128 %shl to i64
+  %retval.sroa.2.0.extract.shift = lshr i128 %shl, 64
+  %retval.sroa.2.0.extract.trunc = trunc i128 %retval.sroa.2.0.extract.shift to i64
+  %.fca.0.insert = insertvalue { i64, i64 } undef, i64 %retval.sroa.0.0.extract.trunc, 0
+  %.fca.1.insert = insertvalue { i64, i64 } %.fca.0.insert, i64 %retval.sroa.2.0.extract.trunc, 1
+  ret { i64, i64 } %.fca.1.insert
+}
+
+define dso_local { i64, i64 } @ashr128(i64 %x.coerce0, i64 %x.coerce1, i8 signext %y) minsize optsize {
+; CHECK-LABEL: ashr128:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    callq __ashrti3
+; CHECK-NEXT:    popq %rcx
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
+entry:
+  %x.sroa.2.0.insert.ext = zext i64 %x.coerce1 to i128
+  %x.sroa.2.0.insert.shift = shl nuw i128 %x.sroa.2.0.insert.ext, 64
+  %x.sroa.0.0.insert.ext = zext i64 %x.coerce0 to i128
+  %x.sroa.0.0.insert.insert = or i128 %x.sroa.2.0.insert.shift, %x.sroa.0.0.insert.ext
+  %conv = sext i8 %y to i32
+  %sh_prom = zext i32 %conv to i128
+  %shr = ashr i128 %x.sroa.0.0.insert.insert, %sh_prom
+  %retval.sroa.0.0.extract.trunc = trunc i128 %shr to i64
+  %retval.sroa.2.0.extract.shift = lshr i128 %shr, 64
+  %retval.sroa.2.0.extract.trunc = trunc i128 %retval.sroa.2.0.extract.shift to i64
+  %.fca.0.insert = insertvalue { i64, i64 } undef, i64 %retval.sroa.0.0.extract.trunc, 0
+  %.fca.1.insert = insertvalue { i64, i64 } %.fca.0.insert, i64 %retval.sroa.2.0.extract.trunc, 1
+  ret { i64, i64 } %.fca.1.insert
+}
+
+define dso_local { i64, i64 } @lshr128(i64 %x.coerce0, i64 %x.coerce1, i8 signext %y) minsize optsize {
+; CHECK-LABEL: lshr128:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    movzbl %dl, %edx
+; CHECK-NEXT:    callq __lshrti3
+; CHECK-NEXT:    popq %rcx
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
+entry:
+  %x.sroa.2.0.insert.ext = zext i64 %x.coerce1 to i128
+  %x.sroa.2.0.insert.shift = shl nuw i128 %x.sroa.2.0.insert.ext, 64
+  %x.sroa.0.0.insert.ext = zext i64 %x.coerce0 to i128
+  %x.sroa.0.0.insert.insert = or i128 %x.sroa.2.0.insert.shift, %x.sroa.0.0.insert.ext
+  %conv = sext i8 %y to i32
+  %sh_prom = zext i32 %conv to i128
+  %shr = lshr i128 %x.sroa.0.0.insert.insert, %sh_prom
+  %retval.sroa.0.0.extract.trunc = trunc i128 %shr to i64
+  %retval.sroa.2.0.extract.shift = lshr i128 %shr, 64
+  %retval.sroa.2.0.extract.trunc = trunc i128 %retval.sroa.2.0.extract.shift to i64
+  %.fca.0.insert = insertvalue { i64, i64 } undef, i64 %retval.sroa.0.0.extract.trunc, 0
+  %.fca.1.insert = insertvalue { i64, i64 } %.fca.0.insert, i64 %retval.sroa.2.0.extract.trunc, 1
+  ret { i64, i64 } %.fca.1.insert
+}
+
diff --git a/test/CodeGen/X86/shrink_vmul-widen.ll b/test/CodeGen/X86/shrink_vmul-widen.ll
index 7a24743d5620b0602158d1f1c89d5a7f69f91ca5..d0fad23394bd48047c0e4d90065bcffeec560d62 100644
--- a/test/CodeGen/X86/shrink_vmul-widen.ll
+++ b/test/CodeGen/X86/shrink_vmul-widen.ll
@@ -2096,7 +2096,7 @@ define void @PR34947(<9 x i16>* %p0, <9 x i32>* %p1) nounwind {
 ; X86-SSE-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-SSE-NEXT:    movdqa (%eax), %xmm5
 ; X86-SSE-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; X86-SSE-NEXT:    movdqa (%ecx), %xmm3
+; X86-SSE-NEXT:    movdqa (%ecx), %xmm2
 ; X86-SSE-NEXT:    movdqa 16(%ecx), %xmm6
 ; X86-SSE-NEXT:    pxor %xmm0, %xmm0
 ; X86-SSE-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
@@ -2110,10 +2110,10 @@ define void @PR34947(<9 x i16>* %p0, <9 x i32>* %p1) nounwind {
 ; X86-SSE-NEXT:    xorl %edx, %edx
 ; X86-SSE-NEXT:    divl %esi
 ; X86-SSE-NEXT:    movd %edx, %xmm0
-; X86-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm5[2,3,0,1]
-; X86-SSE-NEXT:    movd %xmm2, %eax
-; X86-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm6[2,3,0,1]
-; X86-SSE-NEXT:    movd %xmm2, %esi
+; X86-SSE-NEXT:    pshufd {{.*#+}} xmm3 = xmm5[2,3,0,1]
+; X86-SSE-NEXT:    movd %xmm3, %eax
+; X86-SSE-NEXT:    pshufd {{.*#+}} xmm3 = xmm6[2,3,0,1]
+; X86-SSE-NEXT:    movd %xmm3, %esi
 ; X86-SSE-NEXT:    xorl %edx, %edx
 ; X86-SSE-NEXT:    divl %esi
 ; X86-SSE-NEXT:    movd %edx, %xmm7
@@ -2122,7 +2122,7 @@ define void @PR34947(<9 x i16>* %p0, <9 x i32>* %p1) nounwind {
 ; X86-SSE-NEXT:    movd %xmm6, %esi
 ; X86-SSE-NEXT:    xorl %edx, %edx
 ; X86-SSE-NEXT:    divl %esi
-; X86-SSE-NEXT:    movd %edx, %xmm2
+; X86-SSE-NEXT:    movd %edx, %xmm3
 ; X86-SSE-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,2,3]
 ; X86-SSE-NEXT:    movd %xmm5, %eax
 ; X86-SSE-NEXT:    pshufd {{.*#+}} xmm5 = xmm6[1,1,2,3]
@@ -2130,60 +2130,57 @@ define void @PR34947(<9 x i16>* %p0, <9 x i32>* %p1) nounwind {
 ; X86-SSE-NEXT:    xorl %edx, %edx
 ; X86-SSE-NEXT:    divl %esi
 ; X86-SSE-NEXT:    movd %edx, %xmm5
-; X86-SSE-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1]
-; X86-SSE-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm7[0]
+; X86-SSE-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1]
+; X86-SSE-NEXT:    punpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm7[0]
 ; X86-SSE-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[3,1,2,3]
 ; X86-SSE-NEXT:    movd %xmm6, %eax
-; X86-SSE-NEXT:    pshufd {{.*#+}} xmm6 = xmm3[3,1,2,3]
+; X86-SSE-NEXT:    pshufd {{.*#+}} xmm6 = xmm2[3,1,2,3]
 ; X86-SSE-NEXT:    movd %xmm6, %esi
 ; X86-SSE-NEXT:    xorl %edx, %edx
 ; X86-SSE-NEXT:    divl %esi
 ; X86-SSE-NEXT:    movd %edx, %xmm6
 ; X86-SSE-NEXT:    pshufd {{.*#+}} xmm7 = xmm4[2,3,0,1]
 ; X86-SSE-NEXT:    movd %xmm7, %eax
-; X86-SSE-NEXT:    pshufd {{.*#+}} xmm7 = xmm3[2,3,0,1]
+; X86-SSE-NEXT:    pshufd {{.*#+}} xmm7 = xmm2[2,3,0,1]
 ; X86-SSE-NEXT:    movd %xmm7, %esi
 ; X86-SSE-NEXT:    xorl %edx, %edx
 ; X86-SSE-NEXT:    divl %esi
 ; X86-SSE-NEXT:    movd %edx, %xmm7
 ; X86-SSE-NEXT:    punpckldq {{.*#+}} xmm7 = xmm7[0],xmm6[0],xmm7[1],xmm6[1]
 ; X86-SSE-NEXT:    movd %xmm4, %eax
-; X86-SSE-NEXT:    movd %xmm3, %esi
+; X86-SSE-NEXT:    movd %xmm2, %esi
 ; X86-SSE-NEXT:    xorl %edx, %edx
 ; X86-SSE-NEXT:    divl %esi
 ; X86-SSE-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,2,3]
 ; X86-SSE-NEXT:    movd %xmm4, %eax
 ; X86-SSE-NEXT:    movd %edx, %xmm4
-; X86-SSE-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,2,3]
-; X86-SSE-NEXT:    movd %xmm3, %esi
+; X86-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,2,3]
+; X86-SSE-NEXT:    movd %xmm2, %esi
 ; X86-SSE-NEXT:    xorl %edx, %edx
 ; X86-SSE-NEXT:    divl %esi
-; X86-SSE-NEXT:    movd %edx, %xmm3
-; X86-SSE-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
+; X86-SSE-NEXT:    movd %edx, %xmm2
+; X86-SSE-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1]
 ; X86-SSE-NEXT:    punpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm7[0]
 ; X86-SSE-NEXT:    movd %xmm1, %eax
-; X86-SSE-NEXT:    shufps {{.*#+}} xmm3 = xmm3[0,0],xmm6[0,0]
+; X86-SSE-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,0],xmm6[0,0]
 ; X86-SSE-NEXT:    movdqa {{.*#+}} xmm1 = [8199,8199,8199,8199]
 ; X86-SSE-NEXT:    pmuludq %xmm1, %xmm4
 ; X86-SSE-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[0,2,2,3]
-; X86-SSE-NEXT:    pmuludq %xmm1, %xmm3
-; X86-SSE-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3]
-; X86-SSE-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
+; X86-SSE-NEXT:    pmuludq %xmm1, %xmm2
+; X86-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
+; X86-SSE-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1]
 ; X86-SSE-NEXT:    shufps {{.*#+}} xmm5 = xmm5[0,0],xmm0[0,0]
+; X86-SSE-NEXT:    pmuludq %xmm1, %xmm3
+; X86-SSE-NEXT:    pmuludq %xmm1, %xmm5
+; X86-SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[0,2,2,3]
+; X86-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm5[0,2,2,3]
+; X86-SSE-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
 ; X86-SSE-NEXT:    xorl %edx, %edx
 ; X86-SSE-NEXT:    divl 32(%ecx)
-; X86-SSE-NEXT:    pmuludq %xmm1, %xmm2
-; X86-SSE-NEXT:    pmuludq %xmm1, %xmm5
-; X86-SSE-NEXT:    movd %edx, %xmm0
-; X86-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[0,2,2,3]
-; X86-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm5[0,2,2,3]
-; X86-SSE-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
-; X86-SSE-NEXT:    movl $8199, %eax # imm = 0x2007
-; X86-SSE-NEXT:    movd %eax, %xmm2
-; X86-SSE-NEXT:    pmuludq %xmm0, %xmm2
-; X86-SSE-NEXT:    movd %xmm2, (%eax)
-; X86-SSE-NEXT:    movdqa %xmm1, (%eax)
+; X86-SSE-NEXT:    movdqa %xmm0, (%eax)
 ; X86-SSE-NEXT:    movdqa %xmm4, (%eax)
+; X86-SSE-NEXT:    imull $8199, %edx, %eax # imm = 0x2007
+; X86-SSE-NEXT:    movl %eax, (%eax)
 ; X86-SSE-NEXT:    popl %esi
 ; X86-SSE-NEXT:    retl
 ;
@@ -2252,14 +2249,13 @@ define void @PR34947(<9 x i16>* %p0, <9 x i32>* %p1) nounwind {
 ; X86-AVX1-NEXT:    vpinsrd $1, (%esp), %xmm1, %xmm1 # 4-byte Folded Reload
 ; X86-AVX1-NEXT:    vpinsrd $2, {{[-0-9]+}}(%e{{[sb]}}p), %xmm1, %xmm1 # 4-byte Folded Reload
 ; X86-AVX1-NEXT:    vpinsrd $3, {{[-0-9]+}}(%e{{[sb]}}p), %xmm1, %xmm1 # 4-byte Folded Reload
-; X86-AVX1-NEXT:    vmovd {{[-0-9]+}}(%e{{[sb]}}p), %xmm2 # 4-byte Folded Reload
-; X86-AVX1-NEXT:    # xmm2 = mem[0],zero,zero,zero
-; X86-AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [8199,8199,8199,8199]
-; X86-AVX1-NEXT:    vpmulld %xmm3, %xmm0, %xmm0
-; X86-AVX1-NEXT:    vpmulld %xmm3, %xmm1, %xmm1
+; X86-AVX1-NEXT:    imull $8199, {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; X86-AVX1-NEXT:    # imm = 0x2007
+; X86-AVX1-NEXT:    movl %eax, (%eax)
+; X86-AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [8199,8199,8199,8199]
+; X86-AVX1-NEXT:    vpmulld %xmm2, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vpmulld %xmm2, %xmm1, %xmm1
 ; X86-AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
-; X86-AVX1-NEXT:    vpmulld %xmm3, %xmm2, %xmm1
-; X86-AVX1-NEXT:    vmovd %xmm1, (%eax)
 ; X86-AVX1-NEXT:    vmovaps %ymm0, (%eax)
 ; X86-AVX1-NEXT:    addl $16, %esp
 ; X86-AVX1-NEXT:    popl %esi
@@ -2326,12 +2322,11 @@ define void @PR34947(<9 x i16>* %p0, <9 x i32>* %p1) nounwind {
 ; X86-AVX2-NEXT:    vmovd %xmm0, %eax
 ; X86-AVX2-NEXT:    xorl %edx, %edx
 ; X86-AVX2-NEXT:    divl 32(%esi)
-; X86-AVX2-NEXT:    vmovd %edx, %xmm0
-; X86-AVX2-NEXT:    vpbroadcastd {{.*#+}} ymm2 = [8199,8199,8199,8199,8199,8199,8199,8199]
-; X86-AVX2-NEXT:    vpmulld %ymm2, %ymm1, %ymm1
-; X86-AVX2-NEXT:    vpmulld %xmm2, %xmm0, %xmm0
-; X86-AVX2-NEXT:    vmovd %xmm0, (%eax)
-; X86-AVX2-NEXT:    vmovdqa %ymm1, (%eax)
+; X86-AVX2-NEXT:    vpbroadcastd {{.*#+}} ymm0 = [8199,8199,8199,8199,8199,8199,8199,8199]
+; X86-AVX2-NEXT:    vpmulld %ymm0, %ymm1, %ymm0
+; X86-AVX2-NEXT:    imull $8199, %edx, %eax # imm = 0x2007
+; X86-AVX2-NEXT:    movl %eax, (%eax)
+; X86-AVX2-NEXT:    vmovdqa %ymm0, (%eax)
 ; X86-AVX2-NEXT:    popl %esi
 ; X86-AVX2-NEXT:    popl %edi
 ; X86-AVX2-NEXT:    vzeroupper
@@ -2345,8 +2340,8 @@ define void @PR34947(<9 x i16>* %p0, <9 x i32>* %p1) nounwind {
 ; X64-SSE-NEXT:    movdqa 16(%rsi), %xmm6
 ; X64-SSE-NEXT:    pxor %xmm0, %xmm0
 ; X64-SSE-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
-; X64-SSE-NEXT:    movdqa %xmm5, %xmm4
-; X64-SSE-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3]
+; X64-SSE-NEXT:    movdqa %xmm5, %xmm3
+; X64-SSE-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3]
 ; X64-SSE-NEXT:    punpckhwd {{.*#+}} xmm5 = xmm5[4],xmm0[4],xmm5[5],xmm0[5],xmm5[6],xmm0[6],xmm5[7],xmm0[7]
 ; X64-SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm5[3,1,2,3]
 ; X64-SSE-NEXT:    movd %xmm0, %eax
@@ -2355,10 +2350,10 @@ define void @PR34947(<9 x i16>* %p0, <9 x i32>* %p1) nounwind {
 ; X64-SSE-NEXT:    xorl %edx, %edx
 ; X64-SSE-NEXT:    divl %ecx
 ; X64-SSE-NEXT:    movd %edx, %xmm8
-; X64-SSE-NEXT:    pshufd {{.*#+}} xmm3 = xmm5[2,3,0,1]
-; X64-SSE-NEXT:    movd %xmm3, %eax
-; X64-SSE-NEXT:    pshufd {{.*#+}} xmm3 = xmm6[2,3,0,1]
-; X64-SSE-NEXT:    movd %xmm3, %ecx
+; X64-SSE-NEXT:    pshufd {{.*#+}} xmm4 = xmm5[2,3,0,1]
+; X64-SSE-NEXT:    movd %xmm4, %eax
+; X64-SSE-NEXT:    pshufd {{.*#+}} xmm4 = xmm6[2,3,0,1]
+; X64-SSE-NEXT:    movd %xmm4, %ecx
 ; X64-SSE-NEXT:    xorl %edx, %edx
 ; X64-SSE-NEXT:    divl %ecx
 ; X64-SSE-NEXT:    movd %edx, %xmm7
@@ -2367,7 +2362,7 @@ define void @PR34947(<9 x i16>* %p0, <9 x i32>* %p1) nounwind {
 ; X64-SSE-NEXT:    movd %xmm6, %ecx
 ; X64-SSE-NEXT:    xorl %edx, %edx
 ; X64-SSE-NEXT:    divl %ecx
-; X64-SSE-NEXT:    movd %edx, %xmm3
+; X64-SSE-NEXT:    movd %edx, %xmm4
 ; X64-SSE-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,2,3]
 ; X64-SSE-NEXT:    movd %xmm5, %eax
 ; X64-SSE-NEXT:    pshufd {{.*#+}} xmm5 = xmm6[1,1,2,3]
@@ -2375,16 +2370,16 @@ define void @PR34947(<9 x i16>* %p0, <9 x i32>* %p1) nounwind {
 ; X64-SSE-NEXT:    xorl %edx, %edx
 ; X64-SSE-NEXT:    divl %ecx
 ; X64-SSE-NEXT:    movd %edx, %xmm5
-; X64-SSE-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1]
-; X64-SSE-NEXT:    punpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm7[0]
-; X64-SSE-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[3,1,2,3]
+; X64-SSE-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1]
+; X64-SSE-NEXT:    punpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm7[0]
+; X64-SSE-NEXT:    pshufd {{.*#+}} xmm6 = xmm3[3,1,2,3]
 ; X64-SSE-NEXT:    movd %xmm6, %eax
 ; X64-SSE-NEXT:    pshufd {{.*#+}} xmm6 = xmm2[3,1,2,3]
 ; X64-SSE-NEXT:    movd %xmm6, %ecx
 ; X64-SSE-NEXT:    xorl %edx, %edx
 ; X64-SSE-NEXT:    divl %ecx
 ; X64-SSE-NEXT:    movd %edx, %xmm6
-; X64-SSE-NEXT:    pshufd {{.*#+}} xmm7 = xmm4[2,3,0,1]
+; X64-SSE-NEXT:    pshufd {{.*#+}} xmm7 = xmm3[2,3,0,1]
 ; X64-SSE-NEXT:    movd %xmm7, %eax
 ; X64-SSE-NEXT:    pshufd {{.*#+}} xmm7 = xmm2[2,3,0,1]
 ; X64-SSE-NEXT:    movd %xmm7, %ecx
@@ -2392,13 +2387,13 @@ define void @PR34947(<9 x i16>* %p0, <9 x i32>* %p1) nounwind {
 ; X64-SSE-NEXT:    divl %ecx
 ; X64-SSE-NEXT:    movd %edx, %xmm7
 ; X64-SSE-NEXT:    punpckldq {{.*#+}} xmm7 = xmm7[0],xmm6[0],xmm7[1],xmm6[1]
-; X64-SSE-NEXT:    movd %xmm4, %eax
+; X64-SSE-NEXT:    movd %xmm3, %eax
 ; X64-SSE-NEXT:    movd %xmm2, %ecx
 ; X64-SSE-NEXT:    xorl %edx, %edx
 ; X64-SSE-NEXT:    divl %ecx
 ; X64-SSE-NEXT:    movd %edx, %xmm0
-; X64-SSE-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,2,3]
-; X64-SSE-NEXT:    movd %xmm4, %eax
+; X64-SSE-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,2,3]
+; X64-SSE-NEXT:    movd %xmm3, %eax
 ; X64-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,2,3]
 ; X64-SSE-NEXT:    movd %xmm2, %ecx
 ; X64-SSE-NEXT:    xorl %edx, %edx
@@ -2409,24 +2404,21 @@ define void @PR34947(<9 x i16>* %p0, <9 x i32>* %p1) nounwind {
 ; X64-SSE-NEXT:    movd %xmm1, %eax
 ; X64-SSE-NEXT:    xorl %edx, %edx
 ; X64-SSE-NEXT:    divl 32(%rsi)
-; X64-SSE-NEXT:    movd %edx, %xmm1
-; X64-SSE-NEXT:    movdqa {{.*#+}} xmm4 = [8199,8199,8199,8199]
-; X64-SSE-NEXT:    pmuludq %xmm4, %xmm0
+; X64-SSE-NEXT:    movdqa {{.*#+}} xmm1 = [8199,8199,8199,8199]
+; X64-SSE-NEXT:    pmuludq %xmm1, %xmm0
 ; X64-SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
 ; X64-SSE-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,0],xmm6[0,0]
-; X64-SSE-NEXT:    pmuludq %xmm4, %xmm2
+; X64-SSE-NEXT:    pmuludq %xmm1, %xmm2
 ; X64-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
 ; X64-SSE-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
-; X64-SSE-NEXT:    pmuludq %xmm4, %xmm3
-; X64-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,2,2,3]
+; X64-SSE-NEXT:    pmuludq %xmm1, %xmm4
+; X64-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm4[0,2,2,3]
 ; X64-SSE-NEXT:    shufps {{.*#+}} xmm5 = xmm5[0,0],xmm8[0,0]
-; X64-SSE-NEXT:    pmuludq %xmm4, %xmm5
-; X64-SSE-NEXT:    pshufd {{.*#+}} xmm3 = xmm5[0,2,2,3]
-; X64-SSE-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
-; X64-SSE-NEXT:    movl $8199, %eax # imm = 0x2007
-; X64-SSE-NEXT:    movd %eax, %xmm3
-; X64-SSE-NEXT:    pmuludq %xmm1, %xmm3
-; X64-SSE-NEXT:    movd %xmm3, (%rax)
+; X64-SSE-NEXT:    pmuludq %xmm1, %xmm5
+; X64-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm5[0,2,2,3]
+; X64-SSE-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; X64-SSE-NEXT:    imull $8199, %edx, %eax # imm = 0x2007
+; X64-SSE-NEXT:    movl %eax, (%rax)
 ; X64-SSE-NEXT:    movdqa %xmm2, (%rax)
 ; X64-SSE-NEXT:    movdqa %xmm0, (%rax)
 ; X64-SSE-NEXT:    retq
@@ -2493,11 +2485,10 @@ define void @PR34947(<9 x i16>* %p0, <9 x i32>* %p1) nounwind {
 ; X64-AVX1-NEXT:    vpinsrd $1, %r11d, %xmm2, %xmm2
 ; X64-AVX1-NEXT:    vpinsrd $2, %r10d, %xmm2, %xmm2
 ; X64-AVX1-NEXT:    vpinsrd $3, %r9d, %xmm2, %xmm2
-; X64-AVX1-NEXT:    vpmulld %xmm1, %xmm2, %xmm2
-; X64-AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm2, %ymm0
-; X64-AVX1-NEXT:    vmovd %r8d, %xmm2
 ; X64-AVX1-NEXT:    vpmulld %xmm1, %xmm2, %xmm1
-; X64-AVX1-NEXT:    vmovd %xmm1, (%rax)
+; X64-AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; X64-AVX1-NEXT:    imull $8199, %r8d, %eax # imm = 0x2007
+; X64-AVX1-NEXT:    movl %eax, (%rax)
 ; X64-AVX1-NEXT:    vmovaps %ymm0, (%rax)
 ; X64-AVX1-NEXT:    popq %rbx
 ; X64-AVX1-NEXT:    popq %rbp
@@ -2557,12 +2548,11 @@ define void @PR34947(<9 x i16>* %p0, <9 x i32>* %p1) nounwind {
 ; X64-AVX2-NEXT:    vmovd %xmm0, %eax
 ; X64-AVX2-NEXT:    xorl %edx, %edx
 ; X64-AVX2-NEXT:    divl 32(%rsi)
-; X64-AVX2-NEXT:    vmovd %edx, %xmm0
-; X64-AVX2-NEXT:    vpbroadcastd {{.*#+}} ymm2 = [8199,8199,8199,8199,8199,8199,8199,8199]
-; X64-AVX2-NEXT:    vpmulld %ymm2, %ymm1, %ymm1
-; X64-AVX2-NEXT:    vpmulld %xmm2, %xmm0, %xmm0
-; X64-AVX2-NEXT:    vmovd %xmm0, (%rax)
-; X64-AVX2-NEXT:    vmovdqa %ymm1, (%rax)
+; X64-AVX2-NEXT:    vpbroadcastd {{.*#+}} ymm0 = [8199,8199,8199,8199,8199,8199,8199,8199]
+; X64-AVX2-NEXT:    vpmulld %ymm0, %ymm1, %ymm0
+; X64-AVX2-NEXT:    imull $8199, %edx, %eax # imm = 0x2007
+; X64-AVX2-NEXT:    movl %eax, (%rax)
+; X64-AVX2-NEXT:    vmovdqa %ymm0, (%rax)
 ; X64-AVX2-NEXT:    vzeroupper
 ; X64-AVX2-NEXT:    retq
   %a0 = load <9 x i16>, <9 x i16>* %p0, align 64
diff --git a/test/CodeGen/X86/shrink_vmul.ll b/test/CodeGen/X86/shrink_vmul.ll
index 97290730d90f7d89613a348b942a38e2a1cdb159..8a8a396775ba1ac03b08fd45f48c76fff50f408d 100644
--- a/test/CodeGen/X86/shrink_vmul.ll
+++ b/test/CodeGen/X86/shrink_vmul.ll
@@ -44,7 +44,7 @@ define void @mul_2xi8(i8* nocapture readonly %a, i8* nocapture readonly %b, i64
 ; X86-AVX-NEXT:    movl c, %esi
 ; X86-AVX-NEXT:    vpmovzxbq {{.*#+}} xmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero
 ; X86-AVX-NEXT:    vpmovzxbq {{.*#+}} xmm1 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero
-; X86-AVX-NEXT:    vpmaddwd %xmm0, %xmm1, %xmm0
+; X86-AVX-NEXT:    vpmuludq %xmm0, %xmm1, %xmm0
 ; X86-AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
 ; X86-AVX-NEXT:    vmovq %xmm0, (%esi,%ecx,4)
 ; X86-AVX-NEXT:    popl %esi
@@ -70,7 +70,7 @@ define void @mul_2xi8(i8* nocapture readonly %a, i8* nocapture readonly %b, i64
 ; X64-AVX-NEXT:    movq {{.*}}(%rip), %rax
 ; X64-AVX-NEXT:    vpmovzxbq {{.*#+}} xmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero
 ; X64-AVX-NEXT:    vpmovzxbq {{.*#+}} xmm1 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero
-; X64-AVX-NEXT:    vpmaddwd %xmm0, %xmm1, %xmm0
+; X64-AVX-NEXT:    vpmuludq %xmm0, %xmm1, %xmm0
 ; X64-AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
 ; X64-AVX-NEXT:    vmovq %xmm0, (%rax,%rdx,4)
 ; X64-AVX-NEXT:    retq
@@ -916,7 +916,7 @@ define void @mul_2xi8_sext(i8* nocapture readonly %a, i8* nocapture readonly %b,
 ; X86-AVX-NEXT:    movl c, %esi
 ; X86-AVX-NEXT:    vpmovsxbq (%edx,%ecx), %xmm0
 ; X86-AVX-NEXT:    vpmovsxbq (%eax,%ecx), %xmm1
-; X86-AVX-NEXT:    vpmulld %xmm0, %xmm1, %xmm0
+; X86-AVX-NEXT:    vpmuludq %xmm0, %xmm1, %xmm0
 ; X86-AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
 ; X86-AVX-NEXT:    vmovq %xmm0, (%esi,%ecx,4)
 ; X86-AVX-NEXT:    popl %esi
@@ -944,7 +944,7 @@ define void @mul_2xi8_sext(i8* nocapture readonly %a, i8* nocapture readonly %b,
 ; X64-AVX-NEXT:    movq {{.*}}(%rip), %rax
 ; X64-AVX-NEXT:    vpmovsxbq (%rdi,%rdx), %xmm0
 ; X64-AVX-NEXT:    vpmovsxbq (%rsi,%rdx), %xmm1
-; X64-AVX-NEXT:    vpmulld %xmm0, %xmm1, %xmm0
+; X64-AVX-NEXT:    vpmuludq %xmm0, %xmm1, %xmm0
 ; X64-AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
 ; X64-AVX-NEXT:    vmovq %xmm0, (%rax,%rdx,4)
 ; X64-AVX-NEXT:    retq
@@ -1004,7 +1004,7 @@ define void @mul_2xi8_sext_zext(i8* nocapture readonly %a, i8* nocapture readonl
 ; X86-AVX-NEXT:    movl c, %esi
 ; X86-AVX-NEXT:    vpmovsxbq (%edx,%ecx), %xmm0
 ; X86-AVX-NEXT:    vpmovzxbq {{.*#+}} xmm1 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero
-; X86-AVX-NEXT:    vpmulld %xmm0, %xmm1, %xmm0
+; X86-AVX-NEXT:    vpmuludq %xmm0, %xmm1, %xmm0
 ; X86-AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
 ; X86-AVX-NEXT:    vmovq %xmm0, (%esi,%ecx,4)
 ; X86-AVX-NEXT:    popl %esi
@@ -1033,7 +1033,7 @@ define void @mul_2xi8_sext_zext(i8* nocapture readonly %a, i8* nocapture readonl
 ; X64-AVX-NEXT:    movq {{.*}}(%rip), %rax
 ; X64-AVX-NEXT:    vpmovsxbq (%rdi,%rdx), %xmm0
 ; X64-AVX-NEXT:    vpmovzxbq {{.*#+}} xmm1 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero
-; X64-AVX-NEXT:    vpmulld %xmm0, %xmm1, %xmm0
+; X64-AVX-NEXT:    vpmuludq %xmm0, %xmm1, %xmm0
 ; X64-AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
 ; X64-AVX-NEXT:    vmovq %xmm0, (%rax,%rdx,4)
 ; X64-AVX-NEXT:    retq
@@ -1087,7 +1087,7 @@ define void @mul_2xi16_sext(i8* nocapture readonly %a, i8* nocapture readonly %b
 ; X86-AVX-NEXT:    movl c, %esi
 ; X86-AVX-NEXT:    vpmovsxwq (%edx,%ecx), %xmm0
 ; X86-AVX-NEXT:    vpmovsxwq (%eax,%ecx), %xmm1
-; X86-AVX-NEXT:    vpmulld %xmm0, %xmm1, %xmm0
+; X86-AVX-NEXT:    vpmuludq %xmm0, %xmm1, %xmm0
 ; X86-AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
 ; X86-AVX-NEXT:    vmovq %xmm0, (%esi,%ecx,4)
 ; X86-AVX-NEXT:    popl %esi
@@ -1110,7 +1110,7 @@ define void @mul_2xi16_sext(i8* nocapture readonly %a, i8* nocapture readonly %b
 ; X64-AVX-NEXT:    movq {{.*}}(%rip), %rax
 ; X64-AVX-NEXT:    vpmovsxwq (%rdi,%rdx), %xmm0
 ; X64-AVX-NEXT:    vpmovsxwq (%rsi,%rdx), %xmm1
-; X64-AVX-NEXT:    vpmulld %xmm0, %xmm1, %xmm0
+; X64-AVX-NEXT:    vpmuludq %xmm0, %xmm1, %xmm0
 ; X64-AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
 ; X64-AVX-NEXT:    vmovq %xmm0, (%rax,%rdx,4)
 ; X64-AVX-NEXT:    retq
@@ -1169,9 +1169,8 @@ define void @mul_2xi16_sext_zext(i8* nocapture readonly %a, i8* nocapture readon
 ; X86-AVX-NEXT:    vpmovsxwq (%edx,%ecx), %xmm0
 ; X86-AVX-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
 ; X86-AVX-NEXT:    vpmovzxwd {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
-; X86-AVX-NEXT:    vpmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero
-; X86-AVX-NEXT:    vpmulld %xmm0, %xmm1, %xmm0
 ; X86-AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; X86-AVX-NEXT:    vpmulld %xmm0, %xmm1, %xmm0
 ; X86-AVX-NEXT:    vmovq %xmm0, (%esi,%ecx,4)
 ; X86-AVX-NEXT:    popl %esi
 ; X86-AVX-NEXT:    retl
@@ -1198,9 +1197,8 @@ define void @mul_2xi16_sext_zext(i8* nocapture readonly %a, i8* nocapture readon
 ; X64-AVX-NEXT:    vpmovsxwq (%rdi,%rdx), %xmm0
 ; X64-AVX-NEXT:    vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
 ; X64-AVX-NEXT:    vpmovzxwd {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
-; X64-AVX-NEXT:    vpmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero
-; X64-AVX-NEXT:    vpmulld %xmm0, %xmm1, %xmm0
 ; X64-AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; X64-AVX-NEXT:    vpmulld %xmm0, %xmm1, %xmm0
 ; X64-AVX-NEXT:    vmovq %xmm0, (%rax,%rdx,4)
 ; X64-AVX-NEXT:    retq
 entry:
@@ -1406,7 +1404,7 @@ define void @mul_2xi8_varconst1(i8* nocapture readonly %a, i64 %index) {
 ; X86-AVX-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-AVX-NEXT:    movl c, %edx
 ; X86-AVX-NEXT:    vpmovzxbq {{.*#+}} xmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero
-; X86-AVX-NEXT:    vpmulld {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX-NEXT:    vpmuludq {{\.LCPI.*}}, %xmm0, %xmm0
 ; X86-AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
 ; X86-AVX-NEXT:    vmovq %xmm0, (%edx,%eax,4)
 ; X86-AVX-NEXT:    retl
@@ -1430,7 +1428,7 @@ define void @mul_2xi8_varconst1(i8* nocapture readonly %a, i64 %index) {
 ; X64-AVX-NEXT:    movl $255, %ecx
 ; X64-AVX-NEXT:    vmovq %rcx, %xmm1
 ; X64-AVX-NEXT:    vpslldq {{.*#+}} xmm1 = zero,zero,zero,zero,zero,zero,zero,zero,xmm1[0,1,2,3,4,5,6,7]
-; X64-AVX-NEXT:    vpmaddwd %xmm1, %xmm0, %xmm0
+; X64-AVX-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0
 ; X64-AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
 ; X64-AVX-NEXT:    vmovq %xmm0, (%rax,%rsi,4)
 ; X64-AVX-NEXT:    retq
@@ -1474,7 +1472,7 @@ define void @mul_2xi8_varconst2(i8* nocapture readonly %a, i64 %index) {
 ; X86-AVX-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-AVX-NEXT:    movl c, %edx
 ; X86-AVX-NEXT:    vpmovsxbq (%ecx,%eax), %xmm0
-; X86-AVX-NEXT:    vpmulld {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX-NEXT:    vpmuludq {{\.LCPI.*}}, %xmm0, %xmm0
 ; X86-AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
 ; X86-AVX-NEXT:    vmovq %xmm0, (%edx,%eax,4)
 ; X86-AVX-NEXT:    retl
@@ -1496,7 +1494,7 @@ define void @mul_2xi8_varconst2(i8* nocapture readonly %a, i64 %index) {
 ; X64-AVX:       # %bb.0: # %entry
 ; X64-AVX-NEXT:    movq {{.*}}(%rip), %rax
 ; X64-AVX-NEXT:    vpmovsxbq (%rdi,%rsi), %xmm0
-; X64-AVX-NEXT:    vpmulld {{.*}}(%rip), %xmm0, %xmm0
+; X64-AVX-NEXT:    vpmuludq {{.*}}(%rip), %xmm0, %xmm0
 ; X64-AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
 ; X64-AVX-NEXT:    vmovq %xmm0, (%rax,%rsi,4)
 ; X64-AVX-NEXT:    retq
@@ -1542,7 +1540,7 @@ define void @mul_2xi8_varconst3(i8* nocapture readonly %a, i64 %index) {
 ; X86-AVX-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-AVX-NEXT:    movl c, %edx
 ; X86-AVX-NEXT:    vpmovzxbq {{.*#+}} xmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero
-; X86-AVX-NEXT:    vpmulld {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX-NEXT:    vpmuludq {{\.LCPI.*}}, %xmm0, %xmm0
 ; X86-AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
 ; X86-AVX-NEXT:    vmovq %xmm0, (%edx,%eax,4)
 ; X86-AVX-NEXT:    retl
@@ -1569,7 +1567,7 @@ define void @mul_2xi8_varconst3(i8* nocapture readonly %a, i64 %index) {
 ; X64-AVX-NEXT:    movl $256, %ecx # imm = 0x100
 ; X64-AVX-NEXT:    vmovq %rcx, %xmm1
 ; X64-AVX-NEXT:    vpslldq {{.*#+}} xmm1 = zero,zero,zero,zero,zero,zero,zero,zero,xmm1[0,1,2,3,4,5,6,7]
-; X64-AVX-NEXT:    vpmaddwd %xmm1, %xmm0, %xmm0
+; X64-AVX-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0
 ; X64-AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
 ; X64-AVX-NEXT:    vmovq %xmm0, (%rax,%rsi,4)
 ; X64-AVX-NEXT:    retq
@@ -1615,7 +1613,7 @@ define void @mul_2xi8_varconst4(i8* nocapture readonly %a, i64 %index) {
 ; X86-AVX-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-AVX-NEXT:    movl c, %edx
 ; X86-AVX-NEXT:    vpmovzxbq {{.*#+}} xmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero
-; X86-AVX-NEXT:    vpmulld {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX-NEXT:    vpmuludq {{\.LCPI.*}}, %xmm0, %xmm0
 ; X86-AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
 ; X86-AVX-NEXT:    vmovq %xmm0, (%edx,%eax,4)
 ; X86-AVX-NEXT:    retl
@@ -1639,7 +1637,7 @@ define void @mul_2xi8_varconst4(i8* nocapture readonly %a, i64 %index) {
 ; X64-AVX:       # %bb.0: # %entry
 ; X64-AVX-NEXT:    movq {{.*}}(%rip), %rax
 ; X64-AVX-NEXT:    vpmovzxbq {{.*#+}} xmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero
-; X64-AVX-NEXT:    vpmulld {{.*}}(%rip), %xmm0, %xmm0
+; X64-AVX-NEXT:    vpmuludq {{.*}}(%rip), %xmm0, %xmm0
 ; X64-AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
 ; X64-AVX-NEXT:    vmovq %xmm0, (%rax,%rsi,4)
 ; X64-AVX-NEXT:    retq
@@ -1685,7 +1683,7 @@ define void @mul_2xi8_varconst5(i8* nocapture readonly %a, i64 %index) {
 ; X86-AVX-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-AVX-NEXT:    movl c, %edx
 ; X86-AVX-NEXT:    vpmovsxbq (%ecx,%eax), %xmm0
-; X86-AVX-NEXT:    vpmulld {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX-NEXT:    vpmuludq {{\.LCPI.*}}, %xmm0, %xmm0
 ; X86-AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
 ; X86-AVX-NEXT:    vmovq %xmm0, (%edx,%eax,4)
 ; X86-AVX-NEXT:    retl
@@ -1709,7 +1707,7 @@ define void @mul_2xi8_varconst5(i8* nocapture readonly %a, i64 %index) {
 ; X64-AVX:       # %bb.0: # %entry
 ; X64-AVX-NEXT:    movq {{.*}}(%rip), %rax
 ; X64-AVX-NEXT:    vpmovsxbq (%rdi,%rsi), %xmm0
-; X64-AVX-NEXT:    vpmulld {{.*}}(%rip), %xmm0, %xmm0
+; X64-AVX-NEXT:    vpmuludq {{.*}}(%rip), %xmm0, %xmm0
 ; X64-AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
 ; X64-AVX-NEXT:    vmovq %xmm0, (%rax,%rsi,4)
 ; X64-AVX-NEXT:    retq
@@ -1755,7 +1753,7 @@ define void @mul_2xi8_varconst6(i8* nocapture readonly %a, i64 %index) {
 ; X86-AVX-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-AVX-NEXT:    movl c, %edx
 ; X86-AVX-NEXT:    vpmovsxbq (%ecx,%eax), %xmm0
-; X86-AVX-NEXT:    vpmulld {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX-NEXT:    vpmuludq {{\.LCPI.*}}, %xmm0, %xmm0
 ; X86-AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
 ; X86-AVX-NEXT:    vmovq %xmm0, (%edx,%eax,4)
 ; X86-AVX-NEXT:    retl
@@ -1779,7 +1777,7 @@ define void @mul_2xi8_varconst6(i8* nocapture readonly %a, i64 %index) {
 ; X64-AVX:       # %bb.0: # %entry
 ; X64-AVX-NEXT:    movq {{.*}}(%rip), %rax
 ; X64-AVX-NEXT:    vpmovsxbq (%rdi,%rsi), %xmm0
-; X64-AVX-NEXT:    vpmulld {{.*}}(%rip), %xmm0, %xmm0
+; X64-AVX-NEXT:    vpmuludq {{.*}}(%rip), %xmm0, %xmm0
 ; X64-AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
 ; X64-AVX-NEXT:    vmovq %xmm0, (%rax,%rsi,4)
 ; X64-AVX-NEXT:    retq
@@ -1823,9 +1821,7 @@ define void @mul_2xi16_varconst1(i8* nocapture readonly %a, i64 %index) {
 ; X86-AVX-NEXT:    movl c, %edx
 ; X86-AVX-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; X86-AVX-NEXT:    vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
-; X86-AVX-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
 ; X86-AVX-NEXT:    vpmulld {{\.LCPI.*}}, %xmm0, %xmm0
-; X86-AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
 ; X86-AVX-NEXT:    vmovq %xmm0, (%edx,%eax,4)
 ; X86-AVX-NEXT:    retl
 ;
@@ -1846,12 +1842,11 @@ define void @mul_2xi16_varconst1(i8* nocapture readonly %a, i64 %index) {
 ; X64-AVX-NEXT:    movq {{.*}}(%rip), %rax
 ; X64-AVX-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; X64-AVX-NEXT:    vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
-; X64-AVX-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
 ; X64-AVX-NEXT:    movl $65535, %ecx # imm = 0xFFFF
 ; X64-AVX-NEXT:    vmovq %rcx, %xmm1
 ; X64-AVX-NEXT:    vpslldq {{.*#+}} xmm1 = zero,zero,zero,zero,zero,zero,zero,zero,xmm1[0,1,2,3,4,5,6,7]
+; X64-AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
 ; X64-AVX-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
-; X64-AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
 ; X64-AVX-NEXT:    vmovq %xmm0, (%rax,%rsi,4)
 ; X64-AVX-NEXT:    retq
 entry:
@@ -1893,7 +1888,7 @@ define void @mul_2xi16_varconst2(i8* nocapture readonly %a, i64 %index) {
 ; X86-AVX-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-AVX-NEXT:    movl c, %edx
 ; X86-AVX-NEXT:    vpmovsxwq (%ecx,%eax), %xmm0
-; X86-AVX-NEXT:    vpmulld {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX-NEXT:    vpmuludq {{\.LCPI.*}}, %xmm0, %xmm0
 ; X86-AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
 ; X86-AVX-NEXT:    vmovq %xmm0, (%edx,%eax,4)
 ; X86-AVX-NEXT:    retl
@@ -1914,7 +1909,7 @@ define void @mul_2xi16_varconst2(i8* nocapture readonly %a, i64 %index) {
 ; X64-AVX:       # %bb.0: # %entry
 ; X64-AVX-NEXT:    movq {{.*}}(%rip), %rax
 ; X64-AVX-NEXT:    vpmovsxwq (%rdi,%rsi), %xmm0
-; X64-AVX-NEXT:    vpmulld {{.*}}(%rip), %xmm0, %xmm0
+; X64-AVX-NEXT:    vpmuludq {{.*}}(%rip), %xmm0, %xmm0
 ; X64-AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
 ; X64-AVX-NEXT:    vmovq %xmm0, (%rax,%rsi,4)
 ; X64-AVX-NEXT:    retq
@@ -1958,9 +1953,7 @@ define void @mul_2xi16_varconst3(i8* nocapture readonly %a, i64 %index) {
 ; X86-AVX-NEXT:    movl c, %edx
 ; X86-AVX-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; X86-AVX-NEXT:    vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
-; X86-AVX-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
 ; X86-AVX-NEXT:    vpmulld {{\.LCPI.*}}, %xmm0, %xmm0
-; X86-AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
 ; X86-AVX-NEXT:    vmovq %xmm0, (%edx,%eax,4)
 ; X86-AVX-NEXT:    retl
 ;
@@ -1984,12 +1977,11 @@ define void @mul_2xi16_varconst3(i8* nocapture readonly %a, i64 %index) {
 ; X64-AVX-NEXT:    movq {{.*}}(%rip), %rax
 ; X64-AVX-NEXT:    vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; X64-AVX-NEXT:    vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
-; X64-AVX-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
 ; X64-AVX-NEXT:    movl $65536, %ecx # imm = 0x10000
 ; X64-AVX-NEXT:    vmovq %rcx, %xmm1
 ; X64-AVX-NEXT:    vpslldq {{.*#+}} xmm1 = zero,zero,zero,zero,zero,zero,zero,zero,xmm1[0,1,2,3,4,5,6,7]
+; X64-AVX-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
 ; X64-AVX-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
-; X64-AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
 ; X64-AVX-NEXT:    vmovq %xmm0, (%rax,%rsi,4)
 ; X64-AVX-NEXT:    retq
 entry:
@@ -2031,7 +2023,7 @@ define void @mul_2xi16_varconst4(i8* nocapture readonly %a, i64 %index) {
 ; X86-AVX-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-AVX-NEXT:    movl c, %edx
 ; X86-AVX-NEXT:    vpmovsxwq (%ecx,%eax), %xmm0
-; X86-AVX-NEXT:    vpmulld {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-AVX-NEXT:    vpmuludq {{\.LCPI.*}}, %xmm0, %xmm0
 ; X86-AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
 ; X86-AVX-NEXT:    vmovq %xmm0, (%edx,%eax,4)
 ; X86-AVX-NEXT:    retl
@@ -2058,7 +2050,7 @@ define void @mul_2xi16_varconst4(i8* nocapture readonly %a, i64 %index) {
 ; X64-AVX-NEXT:    movl $32768, %ecx # imm = 0x8000
 ; X64-AVX-NEXT:    vmovq %rcx, %xmm1
 ; X64-AVX-NEXT:    vpslldq {{.*#+}} xmm1 = zero,zero,zero,zero,zero,zero,zero,zero,xmm1[0,1,2,3,4,5,6,7]
-; X64-AVX-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
+; X64-AVX-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0
 ; X64-AVX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
 ; X64-AVX-NEXT:    vmovq %xmm0, (%rax,%rsi,4)
 ; X64-AVX-NEXT:    retq
@@ -2087,7 +2079,7 @@ define void @PR34947(<9 x i16>* %p0, <9 x i32>* %p1) nounwind {
 ; X86-SSE-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-SSE-NEXT:    movdqa (%eax), %xmm5
 ; X86-SSE-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; X86-SSE-NEXT:    movdqa (%ecx), %xmm3
+; X86-SSE-NEXT:    movdqa (%ecx), %xmm2
 ; X86-SSE-NEXT:    movdqa 16(%ecx), %xmm6
 ; X86-SSE-NEXT:    pxor %xmm0, %xmm0
 ; X86-SSE-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
@@ -2101,10 +2093,10 @@ define void @PR34947(<9 x i16>* %p0, <9 x i32>* %p1) nounwind {
 ; X86-SSE-NEXT:    xorl %edx, %edx
 ; X86-SSE-NEXT:    divl %esi
 ; X86-SSE-NEXT:    movd %edx, %xmm0
-; X86-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm5[2,3,0,1]
-; X86-SSE-NEXT:    movd %xmm2, %eax
-; X86-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm6[2,3,0,1]
-; X86-SSE-NEXT:    movd %xmm2, %esi
+; X86-SSE-NEXT:    pshufd {{.*#+}} xmm3 = xmm5[2,3,0,1]
+; X86-SSE-NEXT:    movd %xmm3, %eax
+; X86-SSE-NEXT:    pshufd {{.*#+}} xmm3 = xmm6[2,3,0,1]
+; X86-SSE-NEXT:    movd %xmm3, %esi
 ; X86-SSE-NEXT:    xorl %edx, %edx
 ; X86-SSE-NEXT:    divl %esi
 ; X86-SSE-NEXT:    movd %edx, %xmm7
@@ -2113,7 +2105,7 @@ define void @PR34947(<9 x i16>* %p0, <9 x i32>* %p1) nounwind {
 ; X86-SSE-NEXT:    movd %xmm6, %esi
 ; X86-SSE-NEXT:    xorl %edx, %edx
 ; X86-SSE-NEXT:    divl %esi
-; X86-SSE-NEXT:    movd %edx, %xmm2
+; X86-SSE-NEXT:    movd %edx, %xmm3
 ; X86-SSE-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,2,3]
 ; X86-SSE-NEXT:    movd %xmm5, %eax
 ; X86-SSE-NEXT:    pshufd {{.*#+}} xmm5 = xmm6[1,1,2,3]
@@ -2121,60 +2113,57 @@ define void @PR34947(<9 x i16>* %p0, <9 x i32>* %p1) nounwind {
 ; X86-SSE-NEXT:    xorl %edx, %edx
 ; X86-SSE-NEXT:    divl %esi
 ; X86-SSE-NEXT:    movd %edx, %xmm5
-; X86-SSE-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1]
-; X86-SSE-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm7[0]
+; X86-SSE-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1]
+; X86-SSE-NEXT:    punpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm7[0]
 ; X86-SSE-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[3,1,2,3]
 ; X86-SSE-NEXT:    movd %xmm6, %eax
-; X86-SSE-NEXT:    pshufd {{.*#+}} xmm6 = xmm3[3,1,2,3]
+; X86-SSE-NEXT:    pshufd {{.*#+}} xmm6 = xmm2[3,1,2,3]
 ; X86-SSE-NEXT:    movd %xmm6, %esi
 ; X86-SSE-NEXT:    xorl %edx, %edx
 ; X86-SSE-NEXT:    divl %esi
 ; X86-SSE-NEXT:    movd %edx, %xmm6
 ; X86-SSE-NEXT:    pshufd {{.*#+}} xmm7 = xmm4[2,3,0,1]
 ; X86-SSE-NEXT:    movd %xmm7, %eax
-; X86-SSE-NEXT:    pshufd {{.*#+}} xmm7 = xmm3[2,3,0,1]
+; X86-SSE-NEXT:    pshufd {{.*#+}} xmm7 = xmm2[2,3,0,1]
 ; X86-SSE-NEXT:    movd %xmm7, %esi
 ; X86-SSE-NEXT:    xorl %edx, %edx
 ; X86-SSE-NEXT:    divl %esi
 ; X86-SSE-NEXT:    movd %edx, %xmm7
 ; X86-SSE-NEXT:    punpckldq {{.*#+}} xmm7 = xmm7[0],xmm6[0],xmm7[1],xmm6[1]
 ; X86-SSE-NEXT:    movd %xmm4, %eax
-; X86-SSE-NEXT:    movd %xmm3, %esi
+; X86-SSE-NEXT:    movd %xmm2, %esi
 ; X86-SSE-NEXT:    xorl %edx, %edx
 ; X86-SSE-NEXT:    divl %esi
 ; X86-SSE-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,2,3]
 ; X86-SSE-NEXT:    movd %xmm4, %eax
 ; X86-SSE-NEXT:    movd %edx, %xmm4
-; X86-SSE-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,2,3]
-; X86-SSE-NEXT:    movd %xmm3, %esi
+; X86-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,2,3]
+; X86-SSE-NEXT:    movd %xmm2, %esi
 ; X86-SSE-NEXT:    xorl %edx, %edx
 ; X86-SSE-NEXT:    divl %esi
-; X86-SSE-NEXT:    movd %edx, %xmm3
-; X86-SSE-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
+; X86-SSE-NEXT:    movd %edx, %xmm2
+; X86-SSE-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1]
 ; X86-SSE-NEXT:    punpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm7[0]
 ; X86-SSE-NEXT:    movd %xmm1, %eax
-; X86-SSE-NEXT:    shufps {{.*#+}} xmm3 = xmm3[0,0],xmm6[0,0]
+; X86-SSE-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,0],xmm6[0,0]
 ; X86-SSE-NEXT:    movdqa {{.*#+}} xmm1 = [8199,8199,8199,8199]
 ; X86-SSE-NEXT:    pmuludq %xmm1, %xmm4
 ; X86-SSE-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[0,2,2,3]
-; X86-SSE-NEXT:    pmuludq %xmm1, %xmm3
-; X86-SSE-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3]
-; X86-SSE-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
+; X86-SSE-NEXT:    pmuludq %xmm1, %xmm2
+; X86-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
+; X86-SSE-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1]
 ; X86-SSE-NEXT:    shufps {{.*#+}} xmm5 = xmm5[0,0],xmm0[0,0]
+; X86-SSE-NEXT:    pmuludq %xmm1, %xmm3
+; X86-SSE-NEXT:    pmuludq %xmm1, %xmm5
+; X86-SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[0,2,2,3]
+; X86-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm5[0,2,2,3]
+; X86-SSE-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
 ; X86-SSE-NEXT:    xorl %edx, %edx
 ; X86-SSE-NEXT:    divl 32(%ecx)
-; X86-SSE-NEXT:    pmuludq %xmm1, %xmm2
-; X86-SSE-NEXT:    pmuludq %xmm1, %xmm5
-; X86-SSE-NEXT:    movd %edx, %xmm0
-; X86-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[0,2,2,3]
-; X86-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm5[0,2,2,3]
-; X86-SSE-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
-; X86-SSE-NEXT:    movl $8199, %eax # imm = 0x2007
-; X86-SSE-NEXT:    movd %eax, %xmm2
-; X86-SSE-NEXT:    pmuludq %xmm0, %xmm2
-; X86-SSE-NEXT:    movd %xmm2, (%eax)
-; X86-SSE-NEXT:    movdqa %xmm1, (%eax)
+; X86-SSE-NEXT:    movdqa %xmm0, (%eax)
 ; X86-SSE-NEXT:    movdqa %xmm4, (%eax)
+; X86-SSE-NEXT:    imull $8199, %edx, %eax # imm = 0x2007
+; X86-SSE-NEXT:    movl %eax, (%eax)
 ; X86-SSE-NEXT:    popl %esi
 ; X86-SSE-NEXT:    retl
 ;
@@ -2243,14 +2232,13 @@ define void @PR34947(<9 x i16>* %p0, <9 x i32>* %p1) nounwind {
 ; X86-AVX1-NEXT:    vpinsrd $1, (%esp), %xmm1, %xmm1 # 4-byte Folded Reload
 ; X86-AVX1-NEXT:    vpinsrd $2, {{[-0-9]+}}(%e{{[sb]}}p), %xmm1, %xmm1 # 4-byte Folded Reload
 ; X86-AVX1-NEXT:    vpinsrd $3, {{[-0-9]+}}(%e{{[sb]}}p), %xmm1, %xmm1 # 4-byte Folded Reload
-; X86-AVX1-NEXT:    vmovd {{[-0-9]+}}(%e{{[sb]}}p), %xmm2 # 4-byte Folded Reload
-; X86-AVX1-NEXT:    # xmm2 = mem[0],zero,zero,zero
-; X86-AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [8199,8199,8199,8199]
-; X86-AVX1-NEXT:    vpmulld %xmm3, %xmm0, %xmm0
-; X86-AVX1-NEXT:    vpmulld %xmm3, %xmm1, %xmm1
+; X86-AVX1-NEXT:    imull $8199, {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; X86-AVX1-NEXT:    # imm = 0x2007
+; X86-AVX1-NEXT:    movl %eax, (%eax)
+; X86-AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [8199,8199,8199,8199]
+; X86-AVX1-NEXT:    vpmulld %xmm2, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vpmulld %xmm2, %xmm1, %xmm1
 ; X86-AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
-; X86-AVX1-NEXT:    vpmulld %xmm3, %xmm2, %xmm1
-; X86-AVX1-NEXT:    vmovd %xmm1, (%eax)
 ; X86-AVX1-NEXT:    vmovaps %ymm0, (%eax)
 ; X86-AVX1-NEXT:    addl $16, %esp
 ; X86-AVX1-NEXT:    popl %esi
@@ -2317,12 +2305,11 @@ define void @PR34947(<9 x i16>* %p0, <9 x i32>* %p1) nounwind {
 ; X86-AVX2-NEXT:    vmovd %xmm0, %eax
 ; X86-AVX2-NEXT:    xorl %edx, %edx
 ; X86-AVX2-NEXT:    divl 32(%esi)
-; X86-AVX2-NEXT:    vmovd %edx, %xmm0
-; X86-AVX2-NEXT:    vpbroadcastd {{.*#+}} ymm2 = [8199,8199,8199,8199,8199,8199,8199,8199]
-; X86-AVX2-NEXT:    vpmulld %ymm2, %ymm1, %ymm1
-; X86-AVX2-NEXT:    vpmulld %xmm2, %xmm0, %xmm0
-; X86-AVX2-NEXT:    vmovd %xmm0, (%eax)
-; X86-AVX2-NEXT:    vmovdqa %ymm1, (%eax)
+; X86-AVX2-NEXT:    vpbroadcastd {{.*#+}} ymm0 = [8199,8199,8199,8199,8199,8199,8199,8199]
+; X86-AVX2-NEXT:    vpmulld %ymm0, %ymm1, %ymm0
+; X86-AVX2-NEXT:    imull $8199, %edx, %eax # imm = 0x2007
+; X86-AVX2-NEXT:    movl %eax, (%eax)
+; X86-AVX2-NEXT:    vmovdqa %ymm0, (%eax)
 ; X86-AVX2-NEXT:    popl %esi
 ; X86-AVX2-NEXT:    popl %edi
 ; X86-AVX2-NEXT:    vzeroupper
@@ -2336,8 +2323,8 @@ define void @PR34947(<9 x i16>* %p0, <9 x i32>* %p1) nounwind {
 ; X64-SSE-NEXT:    movdqa 16(%rsi), %xmm6
 ; X64-SSE-NEXT:    pxor %xmm0, %xmm0
 ; X64-SSE-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
-; X64-SSE-NEXT:    movdqa %xmm5, %xmm4
-; X64-SSE-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3]
+; X64-SSE-NEXT:    movdqa %xmm5, %xmm3
+; X64-SSE-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3]
 ; X64-SSE-NEXT:    punpckhwd {{.*#+}} xmm5 = xmm5[4],xmm0[4],xmm5[5],xmm0[5],xmm5[6],xmm0[6],xmm5[7],xmm0[7]
 ; X64-SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm5[3,1,2,3]
 ; X64-SSE-NEXT:    movd %xmm0, %eax
@@ -2346,10 +2333,10 @@ define void @PR34947(<9 x i16>* %p0, <9 x i32>* %p1) nounwind {
 ; X64-SSE-NEXT:    xorl %edx, %edx
 ; X64-SSE-NEXT:    divl %ecx
 ; X64-SSE-NEXT:    movd %edx, %xmm8
-; X64-SSE-NEXT:    pshufd {{.*#+}} xmm3 = xmm5[2,3,0,1]
-; X64-SSE-NEXT:    movd %xmm3, %eax
-; X64-SSE-NEXT:    pshufd {{.*#+}} xmm3 = xmm6[2,3,0,1]
-; X64-SSE-NEXT:    movd %xmm3, %ecx
+; X64-SSE-NEXT:    pshufd {{.*#+}} xmm4 = xmm5[2,3,0,1]
+; X64-SSE-NEXT:    movd %xmm4, %eax
+; X64-SSE-NEXT:    pshufd {{.*#+}} xmm4 = xmm6[2,3,0,1]
+; X64-SSE-NEXT:    movd %xmm4, %ecx
 ; X64-SSE-NEXT:    xorl %edx, %edx
 ; X64-SSE-NEXT:    divl %ecx
 ; X64-SSE-NEXT:    movd %edx, %xmm7
@@ -2358,7 +2345,7 @@ define void @PR34947(<9 x i16>* %p0, <9 x i32>* %p1) nounwind {
 ; X64-SSE-NEXT:    movd %xmm6, %ecx
 ; X64-SSE-NEXT:    xorl %edx, %edx
 ; X64-SSE-NEXT:    divl %ecx
-; X64-SSE-NEXT:    movd %edx, %xmm3
+; X64-SSE-NEXT:    movd %edx, %xmm4
 ; X64-SSE-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,2,3]
 ; X64-SSE-NEXT:    movd %xmm5, %eax
 ; X64-SSE-NEXT:    pshufd {{.*#+}} xmm5 = xmm6[1,1,2,3]
@@ -2366,16 +2353,16 @@ define void @PR34947(<9 x i16>* %p0, <9 x i32>* %p1) nounwind {
 ; X64-SSE-NEXT:    xorl %edx, %edx
 ; X64-SSE-NEXT:    divl %ecx
 ; X64-SSE-NEXT:    movd %edx, %xmm5
-; X64-SSE-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1]
-; X64-SSE-NEXT:    punpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm7[0]
-; X64-SSE-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[3,1,2,3]
+; X64-SSE-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1]
+; X64-SSE-NEXT:    punpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm7[0]
+; X64-SSE-NEXT:    pshufd {{.*#+}} xmm6 = xmm3[3,1,2,3]
 ; X64-SSE-NEXT:    movd %xmm6, %eax
 ; X64-SSE-NEXT:    pshufd {{.*#+}} xmm6 = xmm2[3,1,2,3]
 ; X64-SSE-NEXT:    movd %xmm6, %ecx
 ; X64-SSE-NEXT:    xorl %edx, %edx
 ; X64-SSE-NEXT:    divl %ecx
 ; X64-SSE-NEXT:    movd %edx, %xmm6
-; X64-SSE-NEXT:    pshufd {{.*#+}} xmm7 = xmm4[2,3,0,1]
+; X64-SSE-NEXT:    pshufd {{.*#+}} xmm7 = xmm3[2,3,0,1]
 ; X64-SSE-NEXT:    movd %xmm7, %eax
 ; X64-SSE-NEXT:    pshufd {{.*#+}} xmm7 = xmm2[2,3,0,1]
 ; X64-SSE-NEXT:    movd %xmm7, %ecx
@@ -2383,13 +2370,13 @@ define void @PR34947(<9 x i16>* %p0, <9 x i32>* %p1) nounwind {
 ; X64-SSE-NEXT:    divl %ecx
 ; X64-SSE-NEXT:    movd %edx, %xmm7
 ; X64-SSE-NEXT:    punpckldq {{.*#+}} xmm7 = xmm7[0],xmm6[0],xmm7[1],xmm6[1]
-; X64-SSE-NEXT:    movd %xmm4, %eax
+; X64-SSE-NEXT:    movd %xmm3, %eax
 ; X64-SSE-NEXT:    movd %xmm2, %ecx
 ; X64-SSE-NEXT:    xorl %edx, %edx
 ; X64-SSE-NEXT:    divl %ecx
 ; X64-SSE-NEXT:    movd %edx, %xmm0
-; X64-SSE-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,2,3]
-; X64-SSE-NEXT:    movd %xmm4, %eax
+; X64-SSE-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,2,3]
+; X64-SSE-NEXT:    movd %xmm3, %eax
 ; X64-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,2,3]
 ; X64-SSE-NEXT:    movd %xmm2, %ecx
 ; X64-SSE-NEXT:    xorl %edx, %edx
@@ -2400,24 +2387,21 @@ define void @PR34947(<9 x i16>* %p0, <9 x i32>* %p1) nounwind {
 ; X64-SSE-NEXT:    movd %xmm1, %eax
 ; X64-SSE-NEXT:    xorl %edx, %edx
 ; X64-SSE-NEXT:    divl 32(%rsi)
-; X64-SSE-NEXT:    movd %edx, %xmm1
-; X64-SSE-NEXT:    movdqa {{.*#+}} xmm4 = [8199,8199,8199,8199]
-; X64-SSE-NEXT:    pmuludq %xmm4, %xmm0
+; X64-SSE-NEXT:    movdqa {{.*#+}} xmm1 = [8199,8199,8199,8199]
+; X64-SSE-NEXT:    pmuludq %xmm1, %xmm0
 ; X64-SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
 ; X64-SSE-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,0],xmm6[0,0]
-; X64-SSE-NEXT:    pmuludq %xmm4, %xmm2
+; X64-SSE-NEXT:    pmuludq %xmm1, %xmm2
 ; X64-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
 ; X64-SSE-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
-; X64-SSE-NEXT:    pmuludq %xmm4, %xmm3
-; X64-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,2,2,3]
+; X64-SSE-NEXT:    pmuludq %xmm1, %xmm4
+; X64-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm4[0,2,2,3]
 ; X64-SSE-NEXT:    shufps {{.*#+}} xmm5 = xmm5[0,0],xmm8[0,0]
-; X64-SSE-NEXT:    pmuludq %xmm4, %xmm5
-; X64-SSE-NEXT:    pshufd {{.*#+}} xmm3 = xmm5[0,2,2,3]
-; X64-SSE-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
-; X64-SSE-NEXT:    movl $8199, %eax # imm = 0x2007
-; X64-SSE-NEXT:    movd %eax, %xmm3
-; X64-SSE-NEXT:    pmuludq %xmm1, %xmm3
-; X64-SSE-NEXT:    movd %xmm3, (%rax)
+; X64-SSE-NEXT:    pmuludq %xmm1, %xmm5
+; X64-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm5[0,2,2,3]
+; X64-SSE-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; X64-SSE-NEXT:    imull $8199, %edx, %eax # imm = 0x2007
+; X64-SSE-NEXT:    movl %eax, (%rax)
 ; X64-SSE-NEXT:    movdqa %xmm2, (%rax)
 ; X64-SSE-NEXT:    movdqa %xmm0, (%rax)
 ; X64-SSE-NEXT:    retq
@@ -2484,11 +2468,10 @@ define void @PR34947(<9 x i16>* %p0, <9 x i32>* %p1) nounwind {
 ; X64-AVX1-NEXT:    vpinsrd $1, %r11d, %xmm2, %xmm2
 ; X64-AVX1-NEXT:    vpinsrd $2, %r10d, %xmm2, %xmm2
 ; X64-AVX1-NEXT:    vpinsrd $3, %r9d, %xmm2, %xmm2
-; X64-AVX1-NEXT:    vpmulld %xmm1, %xmm2, %xmm2
-; X64-AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm2, %ymm0
-; X64-AVX1-NEXT:    vmovd %r8d, %xmm2
 ; X64-AVX1-NEXT:    vpmulld %xmm1, %xmm2, %xmm1
-; X64-AVX1-NEXT:    vmovd %xmm1, (%rax)
+; X64-AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; X64-AVX1-NEXT:    imull $8199, %r8d, %eax # imm = 0x2007
+; X64-AVX1-NEXT:    movl %eax, (%rax)
 ; X64-AVX1-NEXT:    vmovaps %ymm0, (%rax)
 ; X64-AVX1-NEXT:    popq %rbx
 ; X64-AVX1-NEXT:    popq %rbp
@@ -2548,12 +2531,11 @@ define void @PR34947(<9 x i16>* %p0, <9 x i32>* %p1) nounwind {
 ; X64-AVX2-NEXT:    vmovd %xmm0, %eax
 ; X64-AVX2-NEXT:    xorl %edx, %edx
 ; X64-AVX2-NEXT:    divl 32(%rsi)
-; X64-AVX2-NEXT:    vmovd %edx, %xmm0
-; X64-AVX2-NEXT:    vpbroadcastd {{.*#+}} ymm2 = [8199,8199,8199,8199,8199,8199,8199,8199]
-; X64-AVX2-NEXT:    vpmulld %ymm2, %ymm1, %ymm1
-; X64-AVX2-NEXT:    vpmulld %xmm2, %xmm0, %xmm0
-; X64-AVX2-NEXT:    vmovd %xmm0, (%rax)
-; X64-AVX2-NEXT:    vmovdqa %ymm1, (%rax)
+; X64-AVX2-NEXT:    vpbroadcastd {{.*#+}} ymm0 = [8199,8199,8199,8199,8199,8199,8199,8199]
+; X64-AVX2-NEXT:    vpmulld %ymm0, %ymm1, %ymm0
+; X64-AVX2-NEXT:    imull $8199, %edx, %eax # imm = 0x2007
+; X64-AVX2-NEXT:    movl %eax, (%rax)
+; X64-AVX2-NEXT:    vmovdqa %ymm0, (%rax)
 ; X64-AVX2-NEXT:    vzeroupper
 ; X64-AVX2-NEXT:    retq
   %a0 = load <9 x i16>, <9 x i16>* %p0, align 64
diff --git a/test/CodeGen/X86/sibcall-2.ll b/test/CodeGen/X86/sibcall-2.ll
index 6ed7b5a1505b0465e42ea9c90aaabdba2857dbcd..d2b78aae90b4108995203d8c68210516b4817b74 100644
--- a/test/CodeGen/X86/sibcall-2.ll
+++ b/test/CodeGen/X86/sibcall-2.ll
@@ -1,5 +1,5 @@
-; RUN: llc -verify-machineinstrs < %s -mtriple=i386-apple-darwin   -disable-fp-elim | FileCheck %s -check-prefix=32
-; RUN: llc -verify-machineinstrs < %s -mtriple=x86_64-apple-darwin -disable-fp-elim | FileCheck %s -check-prefix=64
+; RUN: llc -verify-machineinstrs < %s -mtriple=i386-apple-darwin   -frame-pointer=all | FileCheck %s -check-prefix=32
+; RUN: llc -verify-machineinstrs < %s -mtriple=x86_64-apple-darwin -frame-pointer=all | FileCheck %s -check-prefix=64
 
 ; Tail call should not use ebp / rbp after it's popped. Use esp / rsp.
 
diff --git a/test/CodeGen/X86/sincos-opt.ll b/test/CodeGen/X86/sincos-opt.ll
index b4330ea58ea5a3d832e1508b4c4397d2743ec25f..b64450863427ba8c37e49ff85932deffe8088fb3 100644
--- a/test/CodeGen/X86/sincos-opt.ll
+++ b/test/CodeGen/X86/sincos-opt.ll
@@ -115,13 +115,13 @@ entry:
 ; GNU_SINCOS: callq sincosl
 ; GNU_SINCOS: fldt 16(%rsp)
 ; GNU_SINCOS: fldt 32(%rsp)
-; GNU_SINCOS: faddp %st(1)
+; GNU_SINCOS: faddp %st, %st(1)
 
 ; GNU_SINCOS_FASTMATH-LABEL: test3:
 ; GNU_SINCOS_FASTMATH: callq sincosl
 ; GNU_SINCOS_FASTMATH: fldt 16(%{{[re]}}sp)
 ; GNU_SINCOS_FASTMATH: fldt 32(%{{[re]}}sp)
-; GNU_SINCOS_FASTMATH: faddp %st(1)
+; GNU_SINCOS_FASTMATH: faddp %st, %st(1)
   %call = tail call x86_fp80 @sinl(x86_fp80 %x) readnone
   %call1 = tail call x86_fp80 @cosl(x86_fp80 %x) readnone
   %add = fadd x86_fp80 %call, %call1
diff --git a/test/CodeGen/X86/slow-incdec.ll b/test/CodeGen/X86/slow-incdec.ll
index 5e466f99a38b7c20903214bbe813bae2aa7eb9cd..83bf250d44c365e7e67102ea20138aee6ad21116 100644
--- a/test/CodeGen/X86/slow-incdec.ll
+++ b/test/CodeGen/X86/slow-incdec.ll
@@ -53,3 +53,102 @@ define i32 @dec_size(i32 %x) optsize {
   %r = add i32 %x, -1
   ret i32 %r
 }
+
+declare {i32, i1} @llvm.uadd.with.overflow.i32(i32, i32)
+declare void @other(i32* ) nounwind;
+
+define void @cond_ae_to_cond_ne(i32* %p) nounwind {
+; INCDEC-LABEL: cond_ae_to_cond_ne:
+; INCDEC:       # %bb.0: # %entry
+; INCDEC-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; INCDEC-NEXT:    incl (%eax)
+; INCDEC-NEXT:    jne .LBB4_1
+; INCDEC-NEXT:  # %bb.2: # %if.end4
+; INCDEC-NEXT:    jmp other # TAILCALL
+; INCDEC-NEXT:  .LBB4_1: # %return
+; INCDEC-NEXT:    retl
+;
+; ADD-LABEL: cond_ae_to_cond_ne:
+; ADD:       # %bb.0: # %entry
+; ADD-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; ADD-NEXT:    addl $1, (%eax)
+; ADD-NEXT:    jne .LBB4_1
+; ADD-NEXT:  # %bb.2: # %if.end4
+; ADD-NEXT:    jmp other # TAILCALL
+; ADD-NEXT:  .LBB4_1: # %return
+; ADD-NEXT:    retl
+entry:
+  %t0 = load i32, i32* %p, align 8
+  %add_ov = call {i32, i1} @llvm.uadd.with.overflow.i32(i32 %t0, i32 1)
+  %inc = extractvalue { i32, i1 } %add_ov, 0
+  store i32 %inc, i32* %p, align 8
+  %ov = extractvalue { i32, i1 } %add_ov, 1
+  br i1 %ov, label %if.end4, label %return
+
+if.end4:
+  tail call void @other(i32* %p) nounwind
+  br label %return
+
+return:
+  ret void
+}
+
+@a = common global i8 0, align 1
+@d = common global i8 0, align 1
+
+declare void @external_a()
+declare void @external_b()
+declare {i8, i1} @llvm.uadd.with.overflow.i8(i8, i8)
+
+define void @test_tail_call(i32* %ptr) nounwind {
+; INCDEC-LABEL: test_tail_call:
+; INCDEC:       # %bb.0: # %entry
+; INCDEC-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; INCDEC-NEXT:    incl (%eax)
+; INCDEC-NEXT:    setne %al
+; INCDEC-NEXT:    incb a
+; INCDEC-NEXT:    sete d
+; INCDEC-NEXT:    testb %al, %al
+; INCDEC-NEXT:    jne .LBB5_2
+; INCDEC-NEXT:  # %bb.1: # %then
+; INCDEC-NEXT:    jmp external_a # TAILCALL
+; INCDEC-NEXT:  .LBB5_2: # %else
+; INCDEC-NEXT:    jmp external_b # TAILCALL
+;
+; ADD-LABEL: test_tail_call:
+; ADD:       # %bb.0: # %entry
+; ADD-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; ADD-NEXT:    addl $1, (%eax)
+; ADD-NEXT:    setne %al
+; ADD-NEXT:    addb $1, a
+; ADD-NEXT:    sete d
+; ADD-NEXT:    testb %al, %al
+; ADD-NEXT:    jne .LBB5_2
+; ADD-NEXT:  # %bb.1: # %then
+; ADD-NEXT:    jmp external_a # TAILCALL
+; ADD-NEXT:  .LBB5_2: # %else
+; ADD-NEXT:    jmp external_b # TAILCALL
+entry:
+  %val = load i32, i32* %ptr
+  %add_ov = call {i32, i1} @llvm.uadd.with.overflow.i32(i32 %val, i32 1)
+  %inc = extractvalue { i32, i1 } %add_ov, 0
+  store i32 %inc, i32* %ptr
+  %cmp = extractvalue { i32, i1 } %add_ov, 1
+  %aval = load volatile i8, i8* @a
+  %add_ov2 = call {i8, i1} @llvm.uadd.with.overflow.i8(i8 %aval, i8 1)
+  %inc2 = extractvalue { i8, i1 } %add_ov2, 0
+  store volatile i8 %inc2, i8* @a
+  %cmp2 = extractvalue { i8, i1 } %add_ov2, 1
+  %conv5 = zext i1 %cmp2 to i8
+  store i8 %conv5, i8* @d
+  br i1 %cmp, label %then, label %else
+
+then:
+  tail call void @external_a()
+  ret void
+
+else:
+  tail call void @external_b()
+  ret void
+}
+
diff --git a/test/CodeGen/X86/smul_fix.ll b/test/CodeGen/X86/smul_fix.ll
index 2e69a2666f9ab9e85ff2fc7dfcc8e54f0004c7e4..30db0d3b4d6c8f127cdd9044f871eca26aef8484 100644
--- a/test/CodeGen/X86/smul_fix.ll
+++ b/test/CodeGen/X86/smul_fix.ll
@@ -135,52 +135,27 @@ define i4 @func3(i4 %x, i4 %y) nounwind {
 define <4 x i32> @vec(<4 x i32> %x, <4 x i32> %y) nounwind {
 ; X64-LABEL: vec:
 ; X64:       # %bb.0:
-; X64-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[3,1,2,3]
-; X64-NEXT:    movd %xmm2, %eax
-; X64-NEXT:    cltq
-; X64-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[3,1,2,3]
-; X64-NEXT:    movd %xmm2, %ecx
-; X64-NEXT:    movslq %ecx, %rcx
-; X64-NEXT:    imulq %rax, %rcx
-; X64-NEXT:    movq %rcx, %rax
-; X64-NEXT:    shrq $32, %rax
-; X64-NEXT:    shldl $30, %ecx, %eax
-; X64-NEXT:    movd %eax, %xmm2
-; X64-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[2,3,0,1]
-; X64-NEXT:    movd %xmm3, %eax
-; X64-NEXT:    cltq
-; X64-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[2,3,0,1]
-; X64-NEXT:    movd %xmm3, %ecx
-; X64-NEXT:    movslq %ecx, %rcx
-; X64-NEXT:    imulq %rax, %rcx
-; X64-NEXT:    movq %rcx, %rax
-; X64-NEXT:    shrq $32, %rax
-; X64-NEXT:    shldl $30, %ecx, %eax
-; X64-NEXT:    movd %eax, %xmm3
-; X64-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
-; X64-NEXT:    movd %xmm1, %eax
-; X64-NEXT:    cltq
-; X64-NEXT:    movd %xmm0, %ecx
-; X64-NEXT:    movslq %ecx, %rcx
-; X64-NEXT:    imulq %rax, %rcx
-; X64-NEXT:    movq %rcx, %rax
-; X64-NEXT:    shrq $32, %rax
-; X64-NEXT:    shldl $30, %ecx, %eax
-; X64-NEXT:    movd %eax, %xmm2
-; X64-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,2,3]
-; X64-NEXT:    movd %xmm1, %eax
-; X64-NEXT:    cltq
-; X64-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
-; X64-NEXT:    movd %xmm0, %ecx
-; X64-NEXT:    movslq %ecx, %rcx
-; X64-NEXT:    imulq %rax, %rcx
-; X64-NEXT:    movq %rcx, %rax
-; X64-NEXT:    shrq $32, %rax
-; X64-NEXT:    shldl $30, %ecx, %eax
-; X64-NEXT:    movd %eax, %xmm0
-; X64-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
-; X64-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; X64-NEXT:    movdqa %xmm2, %xmm0
+; X64-NEXT:    pxor %xmm2, %xmm2
+; X64-NEXT:    pxor %xmm3, %xmm3
+; X64-NEXT:    pcmpgtd %xmm1, %xmm3
+; X64-NEXT:    pand %xmm0, %xmm3
+; X64-NEXT:    pcmpgtd %xmm0, %xmm2
+; X64-NEXT:    pand %xmm1, %xmm2
+; X64-NEXT:    paddd %xmm3, %xmm2
+; X64-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; X64-NEXT:    pmuludq %xmm1, %xmm0
+; X64-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[1,3,2,3]
+; X64-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; X64-NEXT:    pmuludq %xmm3, %xmm1
+; X64-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[1,3,2,3]
+; X64-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
+; X64-NEXT:    psubd %xmm2, %xmm4
+; X64-NEXT:    pslld $30, %xmm4
+; X64-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; X64-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; X64-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; X64-NEXT:    psrld $2, %xmm0
+; X64-NEXT:    por %xmm4, %xmm0
 ; X64-NEXT:    retq
 ;
 ; X86-LABEL: vec:
@@ -295,32 +270,13 @@ define i4 @func6(i4 %x, i4 %y) nounwind {
 define <4 x i32> @vec2(<4 x i32> %x, <4 x i32> %y) nounwind {
 ; X64-LABEL: vec2:
 ; X64:       # %bb.0:
-; X64-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[3,1,2,3]
-; X64-NEXT:    movd %xmm2, %eax
-; X64-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[3,1,2,3]
-; X64-NEXT:    movd %xmm2, %ecx
-; X64-NEXT:    imull %eax, %ecx
-; X64-NEXT:    movd %ecx, %xmm2
-; X64-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[2,3,0,1]
-; X64-NEXT:    movd %xmm3, %eax
-; X64-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[2,3,0,1]
-; X64-NEXT:    movd %xmm3, %ecx
-; X64-NEXT:    imull %eax, %ecx
-; X64-NEXT:    movd %ecx, %xmm3
-; X64-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
-; X64-NEXT:    movd %xmm1, %eax
-; X64-NEXT:    movd %xmm0, %ecx
-; X64-NEXT:    imull %eax, %ecx
-; X64-NEXT:    movd %ecx, %xmm2
-; X64-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,2,3]
-; X64-NEXT:    movd %xmm1, %eax
-; X64-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
-; X64-NEXT:    movd %xmm0, %ecx
-; X64-NEXT:    imull %eax, %ecx
-; X64-NEXT:    movd %ecx, %xmm0
-; X64-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
-; X64-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; X64-NEXT:    movdqa %xmm2, %xmm0
+; X64-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
+; X64-NEXT:    pmuludq %xmm1, %xmm0
+; X64-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; X64-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; X64-NEXT:    pmuludq %xmm2, %xmm1
+; X64-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; X64-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
 ; X64-NEXT:    retq
 ;
 ; X86-LABEL: vec2:
diff --git a/test/CodeGen/X86/sse-schedule.ll b/test/CodeGen/X86/sse-schedule.ll
deleted file mode 100644
index 40cf517de4f87a7c683e4aa0d0a3742f5e5b2e62..0000000000000000000000000000000000000000
--- a/test/CodeGen/X86/sse-schedule.ll
+++ /dev/null
@@ -1,6975 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=x86-64 | FileCheck %s --check-prefixes=CHECK,GENERIC
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=atom -mattr=-sse3 | FileCheck %s --check-prefixes=CHECK,ATOM
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=slm -mattr=-sse3 | FileCheck %s --check-prefixes=CHECK,SLM
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=sandybridge -mattr=-sse3 | FileCheck %s --check-prefixes=CHECK,SANDY-SSE
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=sandybridge -mattr=-avx2 | FileCheck %s --check-prefixes=CHECK,SANDY
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=ivybridge -mattr=-sse3 | FileCheck %s --check-prefixes=CHECK,SANDY-SSE
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=ivybridge -mattr=-avx2 | FileCheck %s --check-prefixes=CHECK,SANDY
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=haswell -mattr=-sse3 | FileCheck %s --check-prefixes=CHECK,HASWELL-SSE
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=haswell -mattr=-avx2 | FileCheck %s --check-prefixes=CHECK,HASWELL
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=broadwell -mattr=-sse3 | FileCheck %s --check-prefixes=CHECK,BROADWELL-SSE
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=broadwell -mattr=-avx2 | FileCheck %s --check-prefixes=CHECK,BROADWELL
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=skylake -mattr=-sse3 | FileCheck %s --check-prefixes=CHECK,SKYLAKE-SSE
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=skylake -mattr=-avx2 | FileCheck %s --check-prefixes=CHECK,SKYLAKE
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=skx -mattr=-sse3 | FileCheck %s --check-prefixes=CHECK,SKX-SSE
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=skx -mattr=-avx2 | FileCheck %s --check-prefixes=CHECK,SKX
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=bdver2 -mattr=-sse3 | FileCheck %s --check-prefixes=CHECK,BDVER2-SSE
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=bdver2 -mattr=-avx2 | FileCheck %s --check-prefixes=CHECK,BDVER2
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=btver2 -mattr=-sse3 | FileCheck %s --check-prefixes=CHECK,BTVER2-SSE
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=btver2 -mattr=-avx2 | FileCheck %s --check-prefixes=CHECK,BTVER2
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=znver1 -mattr=-sse3 | FileCheck %s --check-prefixes=CHECK,ZNVER1-SSE
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=znver1 -mattr=-avx2 | FileCheck %s --check-prefixes=CHECK,ZNVER1
-
-; FIXME: we should really use -mattr=-sse2 here but some of the comparison tests don't work without access to legal <4 x i32> types.
-
-define <4 x float> @test_addps(<4 x float> %a0, <4 x float> %a1, <4 x float> *%a2) {
-; GENERIC-LABEL: test_addps:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    addps (%rdi), %xmm0 # sched: [9:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_addps:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    addps %xmm1, %xmm0 # sched: [5:5.00]
-; ATOM-NEXT:    addps (%rdi), %xmm0 # sched: [5:5.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_addps:
-; SLM:       # %bb.0:
-; SLM-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; SLM-NEXT:    addps (%rdi), %xmm0 # sched: [6:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_addps:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; SANDY-SSE-NEXT:    addps (%rdi), %xmm0 # sched: [9:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_addps:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; SANDY-NEXT:    vaddps (%rdi), %xmm0, %xmm0 # sched: [9:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_addps:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; HASWELL-SSE-NEXT:    addps (%rdi), %xmm0 # sched: [9:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_addps:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; HASWELL-NEXT:    vaddps (%rdi), %xmm0, %xmm0 # sched: [9:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_addps:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; BROADWELL-SSE-NEXT:    addps (%rdi), %xmm0 # sched: [8:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_addps:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    vaddps (%rdi), %xmm0, %xmm0 # sched: [8:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_addps:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [4:0.50]
-; SKYLAKE-SSE-NEXT:    addps (%rdi), %xmm0 # sched: [10:0.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_addps:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    vaddps (%rdi), %xmm0, %xmm0 # sched: [10:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_addps:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [4:0.50]
-; SKX-SSE-NEXT:    addps (%rdi), %xmm0 # sched: [10:0.50]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_addps:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKX-NEXT:    vaddps (%rdi), %xmm0, %xmm0 # sched: [10:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_addps:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [5:1.00]
-; BDVER2-SSE-NEXT:    addps (%rdi), %xmm0 # sched: [10:1.00]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_addps:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [5:1.00]
-; BDVER2-NEXT:    vaddps (%rdi), %xmm0, %xmm0 # sched: [10:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_addps:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; BTVER2-SSE-NEXT:    addps (%rdi), %xmm0 # sched: [8:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_addps:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; BTVER2-NEXT:    vaddps (%rdi), %xmm0, %xmm0 # sched: [8:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_addps:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; ZNVER1-SSE-NEXT:    addps (%rdi), %xmm0 # sched: [10:1.00]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_addps:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    vaddps (%rdi), %xmm0, %xmm0 # sched: [10:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = fadd <4 x float> %a0, %a1
-  %2 = load <4 x float>, <4 x float> *%a2, align 16
-  %3 = fadd <4 x float> %1, %2
-  ret <4 x float> %3
-}
-
-define float @test_addss(float %a0, float %a1, float *%a2) {
-; GENERIC-LABEL: test_addss:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    addss %xmm1, %xmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    addss (%rdi), %xmm0 # sched: [9:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_addss:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    addss %xmm1, %xmm0 # sched: [5:5.00]
-; ATOM-NEXT:    addss (%rdi), %xmm0 # sched: [5:5.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_addss:
-; SLM:       # %bb.0:
-; SLM-NEXT:    addss %xmm1, %xmm0 # sched: [3:1.00]
-; SLM-NEXT:    addss (%rdi), %xmm0 # sched: [6:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_addss:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    addss %xmm1, %xmm0 # sched: [3:1.00]
-; SANDY-SSE-NEXT:    addss (%rdi), %xmm0 # sched: [9:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_addss:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vaddss %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; SANDY-NEXT:    vaddss (%rdi), %xmm0, %xmm0 # sched: [9:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_addss:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    addss %xmm1, %xmm0 # sched: [3:1.00]
-; HASWELL-SSE-NEXT:    addss (%rdi), %xmm0 # sched: [8:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_addss:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vaddss %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; HASWELL-NEXT:    vaddss (%rdi), %xmm0, %xmm0 # sched: [8:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_addss:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    addss %xmm1, %xmm0 # sched: [3:1.00]
-; BROADWELL-SSE-NEXT:    addss (%rdi), %xmm0 # sched: [8:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_addss:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vaddss %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    vaddss (%rdi), %xmm0, %xmm0 # sched: [8:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_addss:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    addss %xmm1, %xmm0 # sched: [4:0.50]
-; SKYLAKE-SSE-NEXT:    addss (%rdi), %xmm0 # sched: [9:0.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_addss:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vaddss %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    vaddss (%rdi), %xmm0, %xmm0 # sched: [9:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_addss:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    addss %xmm1, %xmm0 # sched: [4:0.50]
-; SKX-SSE-NEXT:    addss (%rdi), %xmm0 # sched: [9:0.50]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_addss:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vaddss %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKX-NEXT:    vaddss (%rdi), %xmm0, %xmm0 # sched: [9:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_addss:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    addss %xmm1, %xmm0 # sched: [5:1.00]
-; BDVER2-SSE-NEXT:    addss (%rdi), %xmm0 # sched: [10:1.00]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_addss:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vaddss %xmm1, %xmm0, %xmm0 # sched: [5:1.00]
-; BDVER2-NEXT:    vaddss (%rdi), %xmm0, %xmm0 # sched: [10:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_addss:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    addss %xmm1, %xmm0 # sched: [3:1.00]
-; BTVER2-SSE-NEXT:    addss (%rdi), %xmm0 # sched: [8:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_addss:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vaddss %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; BTVER2-NEXT:    vaddss (%rdi), %xmm0, %xmm0 # sched: [8:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_addss:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    addss %xmm1, %xmm0 # sched: [3:1.00]
-; ZNVER1-SSE-NEXT:    addss (%rdi), %xmm0 # sched: [10:1.00]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_addss:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vaddss %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    vaddss (%rdi), %xmm0, %xmm0 # sched: [10:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = fadd float %a0, %a1
-  %2 = load float, float *%a2, align 4
-  %3 = fadd float %1, %2
-  ret float %3
-}
-
-define <4 x float> @test_andps(<4 x float> %a0, <4 x float> %a1, <4 x float> *%a2) {
-; GENERIC-LABEL: test_andps:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    andps %xmm1, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    andps (%rdi), %xmm0 # sched: [7:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_andps:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    andps %xmm1, %xmm0 # sched: [1:0.50]
-; ATOM-NEXT:    andps (%rdi), %xmm0 # sched: [1:1.00]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_andps:
-; SLM:       # %bb.0:
-; SLM-NEXT:    andps %xmm1, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    andps (%rdi), %xmm0 # sched: [4:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_andps:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    andps %xmm1, %xmm0 # sched: [1:1.00]
-; SANDY-SSE-NEXT:    andps (%rdi), %xmm0 # sched: [7:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_andps:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vandps %xmm1, %xmm0, %xmm0 # sched: [1:1.00]
-; SANDY-NEXT:    vandps (%rdi), %xmm0, %xmm0 # sched: [7:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_andps:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    andps %xmm1, %xmm0 # sched: [1:1.00]
-; HASWELL-SSE-NEXT:    andps (%rdi), %xmm0 # sched: [7:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_andps:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vandps %xmm1, %xmm0, %xmm0 # sched: [1:1.00]
-; HASWELL-NEXT:    vandps (%rdi), %xmm0, %xmm0 # sched: [7:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_andps:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    andps %xmm1, %xmm0 # sched: [1:1.00]
-; BROADWELL-SSE-NEXT:    andps (%rdi), %xmm0 # sched: [6:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_andps:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vandps %xmm1, %xmm0, %xmm0 # sched: [1:1.00]
-; BROADWELL-NEXT:    vandps (%rdi), %xmm0, %xmm0 # sched: [6:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_andps:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    andps %xmm1, %xmm0 # sched: [1:0.33]
-; SKYLAKE-SSE-NEXT:    andps (%rdi), %xmm0 # sched: [7:0.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_andps:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vandps %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    vandps (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_andps:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    andps %xmm1, %xmm0 # sched: [1:0.33]
-; SKX-SSE-NEXT:    andps (%rdi), %xmm0 # sched: [7:0.50]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_andps:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vandps %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    vandps (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_andps:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    andps %xmm1, %xmm0 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    andps (%rdi), %xmm0 # sched: [7:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_andps:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vandps %xmm1, %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    vandps (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_andps:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    andps %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    andps (%rdi), %xmm0 # sched: [6:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_andps:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vandps %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    vandps (%rdi), %xmm0, %xmm0 # sched: [6:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_andps:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    andps %xmm1, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    andps (%rdi), %xmm0 # sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_andps:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vandps %xmm1, %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vandps (%rdi), %xmm0, %xmm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = bitcast <4 x float> %a0 to <4 x i32>
-  %2 = bitcast <4 x float> %a1 to <4 x i32>
-  %3 = and <4 x i32> %1, %2
-  %4 = load <4 x float>, <4 x float> *%a2, align 16
-  %5 = bitcast <4 x float> %4 to <4 x i32>
-  %6 = and <4 x i32> %3, %5
-  %7 = bitcast <4 x i32> %6 to <4 x float>
-  ret <4 x float> %7
-}
-
-define <4 x float> @test_andnotps(<4 x float> %a0, <4 x float> %a1, <4 x float> *%a2) {
-; GENERIC-LABEL: test_andnotps:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    andnps %xmm1, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    andnps (%rdi), %xmm0 # sched: [7:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_andnotps:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    andnps %xmm1, %xmm0 # sched: [1:0.50]
-; ATOM-NEXT:    andnps (%rdi), %xmm0 # sched: [1:1.00]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_andnotps:
-; SLM:       # %bb.0:
-; SLM-NEXT:    andnps %xmm1, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    andnps (%rdi), %xmm0 # sched: [4:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_andnotps:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    andnps %xmm1, %xmm0 # sched: [1:1.00]
-; SANDY-SSE-NEXT:    andnps (%rdi), %xmm0 # sched: [7:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_andnotps:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vandnps %xmm1, %xmm0, %xmm0 # sched: [1:1.00]
-; SANDY-NEXT:    vandnps (%rdi), %xmm0, %xmm0 # sched: [7:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_andnotps:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    andnps %xmm1, %xmm0 # sched: [1:1.00]
-; HASWELL-SSE-NEXT:    andnps (%rdi), %xmm0 # sched: [7:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_andnotps:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vandnps %xmm1, %xmm0, %xmm0 # sched: [1:1.00]
-; HASWELL-NEXT:    vandnps (%rdi), %xmm0, %xmm0 # sched: [7:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_andnotps:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    andnps %xmm1, %xmm0 # sched: [1:1.00]
-; BROADWELL-SSE-NEXT:    andnps (%rdi), %xmm0 # sched: [6:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_andnotps:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vandnps %xmm1, %xmm0, %xmm0 # sched: [1:1.00]
-; BROADWELL-NEXT:    vandnps (%rdi), %xmm0, %xmm0 # sched: [6:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_andnotps:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    andnps %xmm1, %xmm0 # sched: [1:0.33]
-; SKYLAKE-SSE-NEXT:    andnps (%rdi), %xmm0 # sched: [7:0.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_andnotps:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vandnps %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    vandnps (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_andnotps:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    andnps %xmm1, %xmm0 # sched: [1:0.33]
-; SKX-SSE-NEXT:    andnps (%rdi), %xmm0 # sched: [7:0.50]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_andnotps:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vandnps %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    vandnps (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_andnotps:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    andnps %xmm1, %xmm0 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    andnps (%rdi), %xmm0 # sched: [7:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_andnotps:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vandnps %xmm1, %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    vandnps (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_andnotps:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    andnps %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    andnps (%rdi), %xmm0 # sched: [6:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_andnotps:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vandnps %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    vandnps (%rdi), %xmm0, %xmm0 # sched: [6:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_andnotps:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    andnps %xmm1, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    andnps (%rdi), %xmm0 # sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_andnotps:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vandnps %xmm1, %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vandnps (%rdi), %xmm0, %xmm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = bitcast <4 x float> %a0 to <4 x i32>
-  %2 = bitcast <4 x float> %a1 to <4 x i32>
-  %3 = xor <4 x i32> %1, <i32 -1, i32 -1, i32 -1, i32 -1>
-  %4 = and <4 x i32> %3, %2
-  %5 = load <4 x float>, <4 x float> *%a2, align 16
-  %6 = bitcast <4 x float> %5 to <4 x i32>
-  %7 = xor <4 x i32> %4, <i32 -1, i32 -1, i32 -1, i32 -1>
-  %8 = and <4 x i32> %6, %7
-  %9 = bitcast <4 x i32> %8 to <4 x float>
-  ret <4 x float> %9
-}
-
-define <4 x float> @test_cmpps(<4 x float> %a0, <4 x float> %a1, <4 x float> *%a2) {
-; GENERIC-LABEL: test_cmpps:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    cmpeqps %xmm0, %xmm1 # sched: [3:1.00]
-; GENERIC-NEXT:    cmpeqps (%rdi), %xmm0 # sched: [9:1.00]
-; GENERIC-NEXT:    orps %xmm1, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_cmpps:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    cmpeqps %xmm0, %xmm1 # sched: [5:5.00]
-; ATOM-NEXT:    cmpeqps (%rdi), %xmm0 # sched: [5:5.00]
-; ATOM-NEXT:    orps %xmm1, %xmm0 # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_cmpps:
-; SLM:       # %bb.0:
-; SLM-NEXT:    cmpeqps %xmm0, %xmm1 # sched: [3:1.00]
-; SLM-NEXT:    cmpeqps (%rdi), %xmm0 # sched: [6:1.00]
-; SLM-NEXT:    orps %xmm1, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_cmpps:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    cmpeqps %xmm0, %xmm1 # sched: [3:1.00]
-; SANDY-SSE-NEXT:    cmpeqps (%rdi), %xmm0 # sched: [9:1.00]
-; SANDY-SSE-NEXT:    orps %xmm1, %xmm0 # sched: [1:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_cmpps:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vcmpeqps %xmm1, %xmm0, %xmm1 # sched: [3:1.00]
-; SANDY-NEXT:    vcmpeqps (%rdi), %xmm0, %xmm0 # sched: [9:1.00]
-; SANDY-NEXT:    vorps %xmm0, %xmm1, %xmm0 # sched: [1:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_cmpps:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    cmpeqps %xmm0, %xmm1 # sched: [3:1.00]
-; HASWELL-SSE-NEXT:    cmpeqps (%rdi), %xmm0 # sched: [9:1.00]
-; HASWELL-SSE-NEXT:    orps %xmm1, %xmm0 # sched: [1:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_cmpps:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vcmpeqps %xmm1, %xmm0, %xmm1 # sched: [3:1.00]
-; HASWELL-NEXT:    vcmpeqps (%rdi), %xmm0, %xmm0 # sched: [9:1.00]
-; HASWELL-NEXT:    vorps %xmm0, %xmm1, %xmm0 # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_cmpps:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    cmpeqps %xmm0, %xmm1 # sched: [3:1.00]
-; BROADWELL-SSE-NEXT:    cmpeqps (%rdi), %xmm0 # sched: [8:1.00]
-; BROADWELL-SSE-NEXT:    orps %xmm1, %xmm0 # sched: [1:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_cmpps:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vcmpeqps %xmm1, %xmm0, %xmm1 # sched: [3:1.00]
-; BROADWELL-NEXT:    vcmpeqps (%rdi), %xmm0, %xmm0 # sched: [8:1.00]
-; BROADWELL-NEXT:    vorps %xmm0, %xmm1, %xmm0 # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_cmpps:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    cmpeqps %xmm0, %xmm1 # sched: [4:0.50]
-; SKYLAKE-SSE-NEXT:    cmpeqps (%rdi), %xmm0 # sched: [10:0.50]
-; SKYLAKE-SSE-NEXT:    orps %xmm1, %xmm0 # sched: [1:0.33]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_cmpps:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vcmpeqps %xmm1, %xmm0, %xmm1 # sched: [4:0.50]
-; SKYLAKE-NEXT:    vcmpeqps (%rdi), %xmm0, %xmm0 # sched: [10:0.50]
-; SKYLAKE-NEXT:    vorps %xmm0, %xmm1, %xmm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_cmpps:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    cmpeqps %xmm0, %xmm1 # sched: [4:0.50]
-; SKX-SSE-NEXT:    cmpeqps (%rdi), %xmm0 # sched: [10:0.50]
-; SKX-SSE-NEXT:    orps %xmm1, %xmm0 # sched: [1:0.33]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_cmpps:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vcmpeqps %xmm1, %xmm0, %xmm1 # sched: [4:0.50]
-; SKX-NEXT:    vcmpeqps (%rdi), %xmm0, %xmm0 # sched: [10:0.50]
-; SKX-NEXT:    vorps %xmm0, %xmm1, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_cmpps:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    cmpeqps %xmm0, %xmm1 # sched: [2:1.00]
-; BDVER2-SSE-NEXT:    cmpeqps (%rdi), %xmm0 # sched: [7:1.00]
-; BDVER2-SSE-NEXT:    orps %xmm1, %xmm0 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_cmpps:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vcmpeqps %xmm1, %xmm0, %xmm1 # sched: [2:1.00]
-; BDVER2-NEXT:    vcmpeqps (%rdi), %xmm0, %xmm0 # sched: [7:1.00]
-; BDVER2-NEXT:    vorps %xmm0, %xmm1, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_cmpps:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    cmpeqps %xmm0, %xmm1 # sched: [2:1.00]
-; BTVER2-SSE-NEXT:    cmpeqps (%rdi), %xmm0 # sched: [7:1.00]
-; BTVER2-SSE-NEXT:    orps %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_cmpps:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vcmpeqps %xmm1, %xmm0, %xmm1 # sched: [2:1.00]
-; BTVER2-NEXT:    vcmpeqps (%rdi), %xmm0, %xmm0 # sched: [7:1.00]
-; BTVER2-NEXT:    vorps %xmm0, %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_cmpps:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    cmpeqps %xmm0, %xmm1 # sched: [3:1.00]
-; ZNVER1-SSE-NEXT:    cmpeqps (%rdi), %xmm0 # sched: [10:1.00]
-; ZNVER1-SSE-NEXT:    orps %xmm1, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_cmpps:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vcmpeqps %xmm1, %xmm0, %xmm1 # sched: [3:1.00]
-; ZNVER1-NEXT:    vcmpeqps (%rdi), %xmm0, %xmm0 # sched: [10:1.00]
-; ZNVER1-NEXT:    vorps %xmm0, %xmm1, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = fcmp oeq <4 x float> %a0, %a1
-  %2 = load <4 x float>, <4 x float> *%a2, align 16
-  %3 = fcmp oeq <4 x float> %a0, %2
-  %4 = sext <4 x i1> %1 to <4 x i32>
-  %5 = sext <4 x i1> %3 to <4 x i32>
-  %6 = or <4 x i32> %4, %5
-  %7 = bitcast <4 x i32> %6 to <4 x float>
-  ret <4 x float> %7
-}
-
-define float @test_cmpss(float %a0, float %a1, float *%a2) {
-; GENERIC-LABEL: test_cmpss:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    cmpeqss %xmm1, %xmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    cmpeqss (%rdi), %xmm0 # sched: [9:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_cmpss:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    cmpeqss %xmm1, %xmm0 # sched: [5:5.00]
-; ATOM-NEXT:    cmpeqss (%rdi), %xmm0 # sched: [5:5.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_cmpss:
-; SLM:       # %bb.0:
-; SLM-NEXT:    cmpeqss %xmm1, %xmm0 # sched: [3:1.00]
-; SLM-NEXT:    cmpeqss (%rdi), %xmm0 # sched: [6:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_cmpss:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    cmpeqss %xmm1, %xmm0 # sched: [3:1.00]
-; SANDY-SSE-NEXT:    cmpeqss (%rdi), %xmm0 # sched: [9:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_cmpss:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vcmpeqss %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; SANDY-NEXT:    vcmpeqss (%rdi), %xmm0, %xmm0 # sched: [9:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_cmpss:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    cmpeqss %xmm1, %xmm0 # sched: [3:1.00]
-; HASWELL-SSE-NEXT:    cmpeqss (%rdi), %xmm0 # sched: [8:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_cmpss:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vcmpeqss %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; HASWELL-NEXT:    vcmpeqss (%rdi), %xmm0, %xmm0 # sched: [8:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_cmpss:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    cmpeqss %xmm1, %xmm0 # sched: [3:1.00]
-; BROADWELL-SSE-NEXT:    cmpeqss (%rdi), %xmm0 # sched: [8:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_cmpss:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vcmpeqss %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    vcmpeqss (%rdi), %xmm0, %xmm0 # sched: [8:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_cmpss:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    cmpeqss %xmm1, %xmm0 # sched: [4:0.50]
-; SKYLAKE-SSE-NEXT:    cmpeqss (%rdi), %xmm0 # sched: [9:0.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_cmpss:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vcmpeqss %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    vcmpeqss (%rdi), %xmm0, %xmm0 # sched: [9:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_cmpss:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    cmpeqss %xmm1, %xmm0 # sched: [4:0.50]
-; SKX-SSE-NEXT:    cmpeqss (%rdi), %xmm0 # sched: [9:0.50]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_cmpss:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vcmpeqss %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKX-NEXT:    vcmpeqss (%rdi), %xmm0, %xmm0 # sched: [9:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_cmpss:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    cmpeqss %xmm1, %xmm0 # sched: [2:1.00]
-; BDVER2-SSE-NEXT:    cmpeqss (%rdi), %xmm0 # sched: [7:1.00]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_cmpss:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vcmpeqss %xmm1, %xmm0, %xmm0 # sched: [2:1.00]
-; BDVER2-NEXT:    vcmpeqss (%rdi), %xmm0, %xmm0 # sched: [7:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_cmpss:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    cmpeqss %xmm1, %xmm0 # sched: [2:1.00]
-; BTVER2-SSE-NEXT:    cmpeqss (%rdi), %xmm0 # sched: [7:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_cmpss:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vcmpeqss %xmm1, %xmm0, %xmm0 # sched: [2:1.00]
-; BTVER2-NEXT:    vcmpeqss (%rdi), %xmm0, %xmm0 # sched: [7:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_cmpss:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    cmpeqss %xmm1, %xmm0 # sched: [3:1.00]
-; ZNVER1-SSE-NEXT:    cmpeqss (%rdi), %xmm0 # sched: [10:1.00]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_cmpss:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vcmpeqss %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    vcmpeqss (%rdi), %xmm0, %xmm0 # sched: [10:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = insertelement <4 x float> undef, float %a0, i32 0
-  %2 = insertelement <4 x float> undef, float %a1, i32 0
-  %3 = call <4 x float> @llvm.x86.sse.cmp.ss(<4 x float> %1, <4 x float> %2, i8 0)
-  %4 = load float, float *%a2, align 4
-  %5 = insertelement <4 x float> undef, float %4, i32 0
-  %6 = call <4 x float> @llvm.x86.sse.cmp.ss(<4 x float> %3, <4 x float> %5, i8 0)
-  %7 = extractelement <4 x float> %6, i32 0
-  ret float %7
-}
-declare <4 x float> @llvm.x86.sse.cmp.ss(<4 x float>, <4 x float>, i8) nounwind readnone
-
-define i32 @test_comiss(<4 x float> %a0, <4 x float> %a1, <4 x float> *%a2) {
-; GENERIC-LABEL: test_comiss:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    comiss %xmm1, %xmm0 # sched: [2:1.00]
-; GENERIC-NEXT:    setnp %al # sched: [1:0.50]
-; GENERIC-NEXT:    sete %cl # sched: [1:0.50]
-; GENERIC-NEXT:    andb %al, %cl # sched: [1:0.33]
-; GENERIC-NEXT:    comiss (%rdi), %xmm0 # sched: [8:1.00]
-; GENERIC-NEXT:    setnp %al # sched: [1:0.50]
-; GENERIC-NEXT:    sete %dl # sched: [1:0.50]
-; GENERIC-NEXT:    andb %al, %dl # sched: [1:0.33]
-; GENERIC-NEXT:    orb %cl, %dl # sched: [1:0.33]
-; GENERIC-NEXT:    movzbl %dl, %eax # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_comiss:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    comiss %xmm1, %xmm0 # sched: [9:4.50]
-; ATOM-NEXT:    setnp %al # sched: [1:0.50]
-; ATOM-NEXT:    sete %cl # sched: [1:0.50]
-; ATOM-NEXT:    andb %al, %cl # sched: [1:0.50]
-; ATOM-NEXT:    comiss (%rdi), %xmm0 # sched: [10:5.00]
-; ATOM-NEXT:    setnp %al # sched: [1:0.50]
-; ATOM-NEXT:    sete %dl # sched: [1:0.50]
-; ATOM-NEXT:    andb %al, %dl # sched: [1:0.50]
-; ATOM-NEXT:    orb %cl, %dl # sched: [1:0.50]
-; ATOM-NEXT:    movzbl %dl, %eax # sched: [1:1.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_comiss:
-; SLM:       # %bb.0:
-; SLM-NEXT:    comiss %xmm1, %xmm0 # sched: [3:1.00]
-; SLM-NEXT:    setnp %al # sched: [1:0.50]
-; SLM-NEXT:    sete %cl # sched: [1:0.50]
-; SLM-NEXT:    andb %al, %cl # sched: [1:0.50]
-; SLM-NEXT:    comiss (%rdi), %xmm0 # sched: [6:1.00]
-; SLM-NEXT:    setnp %al # sched: [1:0.50]
-; SLM-NEXT:    sete %dl # sched: [1:0.50]
-; SLM-NEXT:    andb %al, %dl # sched: [1:0.50]
-; SLM-NEXT:    orb %cl, %dl # sched: [1:0.50]
-; SLM-NEXT:    movzbl %dl, %eax # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_comiss:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    comiss %xmm1, %xmm0 # sched: [2:1.00]
-; SANDY-SSE-NEXT:    setnp %al # sched: [1:0.50]
-; SANDY-SSE-NEXT:    sete %cl # sched: [1:0.50]
-; SANDY-SSE-NEXT:    andb %al, %cl # sched: [1:0.33]
-; SANDY-SSE-NEXT:    comiss (%rdi), %xmm0 # sched: [8:1.00]
-; SANDY-SSE-NEXT:    setnp %al # sched: [1:0.50]
-; SANDY-SSE-NEXT:    sete %dl # sched: [1:0.50]
-; SANDY-SSE-NEXT:    andb %al, %dl # sched: [1:0.33]
-; SANDY-SSE-NEXT:    orb %cl, %dl # sched: [1:0.33]
-; SANDY-SSE-NEXT:    movzbl %dl, %eax # sched: [1:0.33]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_comiss:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vcomiss %xmm1, %xmm0 # sched: [2:1.00]
-; SANDY-NEXT:    setnp %al # sched: [1:0.50]
-; SANDY-NEXT:    sete %cl # sched: [1:0.50]
-; SANDY-NEXT:    andb %al, %cl # sched: [1:0.33]
-; SANDY-NEXT:    vcomiss (%rdi), %xmm0 # sched: [8:1.00]
-; SANDY-NEXT:    setnp %al # sched: [1:0.50]
-; SANDY-NEXT:    sete %dl # sched: [1:0.50]
-; SANDY-NEXT:    andb %al, %dl # sched: [1:0.33]
-; SANDY-NEXT:    orb %cl, %dl # sched: [1:0.33]
-; SANDY-NEXT:    movzbl %dl, %eax # sched: [1:0.33]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_comiss:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    comiss %xmm1, %xmm0 # sched: [3:1.00]
-; HASWELL-SSE-NEXT:    setnp %al # sched: [1:0.50]
-; HASWELL-SSE-NEXT:    sete %cl # sched: [1:0.50]
-; HASWELL-SSE-NEXT:    andb %al, %cl # sched: [1:0.25]
-; HASWELL-SSE-NEXT:    comiss (%rdi), %xmm0 # sched: [8:1.00]
-; HASWELL-SSE-NEXT:    setnp %al # sched: [1:0.50]
-; HASWELL-SSE-NEXT:    sete %dl # sched: [1:0.50]
-; HASWELL-SSE-NEXT:    andb %al, %dl # sched: [1:0.25]
-; HASWELL-SSE-NEXT:    orb %cl, %dl # sched: [1:0.25]
-; HASWELL-SSE-NEXT:    movzbl %dl, %eax # sched: [1:0.25]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_comiss:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vcomiss %xmm1, %xmm0 # sched: [3:1.00]
-; HASWELL-NEXT:    setnp %al # sched: [1:0.50]
-; HASWELL-NEXT:    sete %cl # sched: [1:0.50]
-; HASWELL-NEXT:    andb %al, %cl # sched: [1:0.25]
-; HASWELL-NEXT:    vcomiss (%rdi), %xmm0 # sched: [8:1.00]
-; HASWELL-NEXT:    setnp %al # sched: [1:0.50]
-; HASWELL-NEXT:    sete %dl # sched: [1:0.50]
-; HASWELL-NEXT:    andb %al, %dl # sched: [1:0.25]
-; HASWELL-NEXT:    orb %cl, %dl # sched: [1:0.25]
-; HASWELL-NEXT:    movzbl %dl, %eax # sched: [1:0.25]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_comiss:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    comiss %xmm1, %xmm0 # sched: [3:1.00]
-; BROADWELL-SSE-NEXT:    setnp %al # sched: [1:0.50]
-; BROADWELL-SSE-NEXT:    sete %cl # sched: [1:0.50]
-; BROADWELL-SSE-NEXT:    andb %al, %cl # sched: [1:0.25]
-; BROADWELL-SSE-NEXT:    comiss (%rdi), %xmm0 # sched: [8:1.00]
-; BROADWELL-SSE-NEXT:    setnp %al # sched: [1:0.50]
-; BROADWELL-SSE-NEXT:    sete %dl # sched: [1:0.50]
-; BROADWELL-SSE-NEXT:    andb %al, %dl # sched: [1:0.25]
-; BROADWELL-SSE-NEXT:    orb %cl, %dl # sched: [1:0.25]
-; BROADWELL-SSE-NEXT:    movzbl %dl, %eax # sched: [1:0.25]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_comiss:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vcomiss %xmm1, %xmm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    setnp %al # sched: [1:0.50]
-; BROADWELL-NEXT:    sete %cl # sched: [1:0.50]
-; BROADWELL-NEXT:    andb %al, %cl # sched: [1:0.25]
-; BROADWELL-NEXT:    vcomiss (%rdi), %xmm0 # sched: [8:1.00]
-; BROADWELL-NEXT:    setnp %al # sched: [1:0.50]
-; BROADWELL-NEXT:    sete %dl # sched: [1:0.50]
-; BROADWELL-NEXT:    andb %al, %dl # sched: [1:0.25]
-; BROADWELL-NEXT:    orb %cl, %dl # sched: [1:0.25]
-; BROADWELL-NEXT:    movzbl %dl, %eax # sched: [1:0.25]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_comiss:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    comiss %xmm1, %xmm0 # sched: [2:1.00]
-; SKYLAKE-SSE-NEXT:    setnp %al # sched: [1:0.50]
-; SKYLAKE-SSE-NEXT:    sete %cl # sched: [1:0.50]
-; SKYLAKE-SSE-NEXT:    andb %al, %cl # sched: [1:0.25]
-; SKYLAKE-SSE-NEXT:    comiss (%rdi), %xmm0 # sched: [7:1.00]
-; SKYLAKE-SSE-NEXT:    setnp %al # sched: [1:0.50]
-; SKYLAKE-SSE-NEXT:    sete %dl # sched: [1:0.50]
-; SKYLAKE-SSE-NEXT:    andb %al, %dl # sched: [1:0.25]
-; SKYLAKE-SSE-NEXT:    orb %cl, %dl # sched: [1:0.25]
-; SKYLAKE-SSE-NEXT:    movzbl %dl, %eax # sched: [1:0.25]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_comiss:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vcomiss %xmm1, %xmm0 # sched: [2:1.00]
-; SKYLAKE-NEXT:    setnp %al # sched: [1:0.50]
-; SKYLAKE-NEXT:    sete %cl # sched: [1:0.50]
-; SKYLAKE-NEXT:    andb %al, %cl # sched: [1:0.25]
-; SKYLAKE-NEXT:    vcomiss (%rdi), %xmm0 # sched: [7:1.00]
-; SKYLAKE-NEXT:    setnp %al # sched: [1:0.50]
-; SKYLAKE-NEXT:    sete %dl # sched: [1:0.50]
-; SKYLAKE-NEXT:    andb %al, %dl # sched: [1:0.25]
-; SKYLAKE-NEXT:    orb %cl, %dl # sched: [1:0.25]
-; SKYLAKE-NEXT:    movzbl %dl, %eax # sched: [1:0.25]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_comiss:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    comiss %xmm1, %xmm0 # sched: [2:1.00]
-; SKX-SSE-NEXT:    setnp %al # sched: [1:0.50]
-; SKX-SSE-NEXT:    sete %cl # sched: [1:0.50]
-; SKX-SSE-NEXT:    andb %al, %cl # sched: [1:0.25]
-; SKX-SSE-NEXT:    comiss (%rdi), %xmm0 # sched: [7:1.00]
-; SKX-SSE-NEXT:    setnp %al # sched: [1:0.50]
-; SKX-SSE-NEXT:    sete %dl # sched: [1:0.50]
-; SKX-SSE-NEXT:    andb %al, %dl # sched: [1:0.25]
-; SKX-SSE-NEXT:    orb %cl, %dl # sched: [1:0.25]
-; SKX-SSE-NEXT:    movzbl %dl, %eax # sched: [1:0.25]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_comiss:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vcomiss %xmm1, %xmm0 # sched: [2:1.00]
-; SKX-NEXT:    setnp %al # sched: [1:0.50]
-; SKX-NEXT:    sete %cl # sched: [1:0.50]
-; SKX-NEXT:    andb %al, %cl # sched: [1:0.25]
-; SKX-NEXT:    vcomiss (%rdi), %xmm0 # sched: [7:1.00]
-; SKX-NEXT:    setnp %al # sched: [1:0.50]
-; SKX-NEXT:    sete %dl # sched: [1:0.50]
-; SKX-NEXT:    andb %al, %dl # sched: [1:0.25]
-; SKX-NEXT:    orb %cl, %dl # sched: [1:0.25]
-; SKX-NEXT:    movzbl %dl, %eax # sched: [1:0.25]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_comiss:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    comiss %xmm1, %xmm0 # sched: [1:1.00]
-; BDVER2-SSE-NEXT:    setnp %al # sched: [1:0.50]
-; BDVER2-SSE-NEXT:    sete %cl # sched: [1:0.50]
-; BDVER2-SSE-NEXT:    andb %al, %cl # sched: [1:0.50]
-; BDVER2-SSE-NEXT:    comiss (%rdi), %xmm0 # sched: [6:1.00]
-; BDVER2-SSE-NEXT:    setnp %al # sched: [1:0.50]
-; BDVER2-SSE-NEXT:    sete %dl # sched: [1:0.50]
-; BDVER2-SSE-NEXT:    andb %al, %dl # sched: [1:0.50]
-; BDVER2-SSE-NEXT:    orb %cl, %dl # sched: [1:0.50]
-; BDVER2-SSE-NEXT:    movzbl %dl, %eax # sched: [1:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_comiss:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vcomiss %xmm1, %xmm0 # sched: [1:1.00]
-; BDVER2-NEXT:    setnp %al # sched: [1:0.50]
-; BDVER2-NEXT:    sete %cl # sched: [1:0.50]
-; BDVER2-NEXT:    andb %al, %cl # sched: [1:0.50]
-; BDVER2-NEXT:    vcomiss (%rdi), %xmm0 # sched: [6:1.00]
-; BDVER2-NEXT:    setnp %al # sched: [1:0.50]
-; BDVER2-NEXT:    sete %dl # sched: [1:0.50]
-; BDVER2-NEXT:    andb %al, %dl # sched: [1:0.50]
-; BDVER2-NEXT:    orb %cl, %dl # sched: [1:0.50]
-; BDVER2-NEXT:    movzbl %dl, %eax # sched: [1:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_comiss:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    comiss %xmm1, %xmm0 # sched: [3:1.00]
-; BTVER2-SSE-NEXT:    setnp %al # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    sete %cl # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    andb %al, %cl # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    comiss (%rdi), %xmm0 # sched: [8:1.00]
-; BTVER2-SSE-NEXT:    setnp %al # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    sete %dl # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    andb %al, %dl # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    orb %cl, %dl # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    movzbl %dl, %eax # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_comiss:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vcomiss %xmm1, %xmm0 # sched: [3:1.00]
-; BTVER2-NEXT:    setnp %al # sched: [1:0.50]
-; BTVER2-NEXT:    sete %cl # sched: [1:0.50]
-; BTVER2-NEXT:    andb %al, %cl # sched: [1:0.50]
-; BTVER2-NEXT:    vcomiss (%rdi), %xmm0 # sched: [8:1.00]
-; BTVER2-NEXT:    setnp %al # sched: [1:0.50]
-; BTVER2-NEXT:    sete %dl # sched: [1:0.50]
-; BTVER2-NEXT:    andb %al, %dl # sched: [1:0.50]
-; BTVER2-NEXT:    orb %cl, %dl # sched: [1:0.50]
-; BTVER2-NEXT:    movzbl %dl, %eax # sched: [1:0.50]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_comiss:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    comiss %xmm1, %xmm0 # sched: [3:1.00]
-; ZNVER1-SSE-NEXT:    setnp %al # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    sete %cl # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    andb %al, %cl # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    comiss (%rdi), %xmm0 # sched: [10:1.00]
-; ZNVER1-SSE-NEXT:    setnp %al # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    sete %dl # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    andb %al, %dl # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    orb %cl, %dl # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    movzbl %dl, %eax # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_comiss:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vcomiss %xmm1, %xmm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    setnp %al # sched: [1:0.25]
-; ZNVER1-NEXT:    sete %cl # sched: [1:0.25]
-; ZNVER1-NEXT:    andb %al, %cl # sched: [1:0.25]
-; ZNVER1-NEXT:    vcomiss (%rdi), %xmm0 # sched: [10:1.00]
-; ZNVER1-NEXT:    setnp %al # sched: [1:0.25]
-; ZNVER1-NEXT:    sete %dl # sched: [1:0.25]
-; ZNVER1-NEXT:    andb %al, %dl # sched: [1:0.25]
-; ZNVER1-NEXT:    orb %cl, %dl # sched: [1:0.25]
-; ZNVER1-NEXT:    movzbl %dl, %eax # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call i32 @llvm.x86.sse.comieq.ss(<4 x float> %a0, <4 x float> %a1)
-  %2 = load <4 x float>, <4 x float> *%a2, align 4
-  %3 = call i32 @llvm.x86.sse.comieq.ss(<4 x float> %a0, <4 x float> %2)
-  %4 = or i32 %1, %3
-  ret i32 %4
-}
-declare i32 @llvm.x86.sse.comieq.ss(<4 x float>, <4 x float>) nounwind readnone
-
-define float @test_cvtsi2ss(i32 %a0, i32 *%a1) {
-; GENERIC-LABEL: test_cvtsi2ss:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    cvtsi2ssl %edi, %xmm1 # sched: [5:2.00]
-; GENERIC-NEXT:    cvtsi2ssl (%rsi), %xmm0 # sched: [10:1.00]
-; GENERIC-NEXT:    addss %xmm1, %xmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_cvtsi2ss:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    cvtsi2ssl (%rsi), %xmm0 # sched: [7:3.50]
-; ATOM-NEXT:    cvtsi2ssl %edi, %xmm1 # sched: [6:3.00]
-; ATOM-NEXT:    addss %xmm1, %xmm0 # sched: [5:5.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_cvtsi2ss:
-; SLM:       # %bb.0:
-; SLM-NEXT:    cvtsi2ssl (%rsi), %xmm0 # sched: [7:1.00]
-; SLM-NEXT:    cvtsi2ssl %edi, %xmm1 # sched: [4:0.50]
-; SLM-NEXT:    addss %xmm1, %xmm0 # sched: [3:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_cvtsi2ss:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    cvtsi2ssl %edi, %xmm1 # sched: [5:2.00]
-; SANDY-SSE-NEXT:    cvtsi2ssl (%rsi), %xmm0 # sched: [10:1.00]
-; SANDY-SSE-NEXT:    addss %xmm1, %xmm0 # sched: [3:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_cvtsi2ss:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vcvtsi2ssl %edi, %xmm0, %xmm0 # sched: [5:2.00]
-; SANDY-NEXT:    vcvtsi2ssl (%rsi), %xmm1, %xmm1 # sched: [10:1.00]
-; SANDY-NEXT:    vaddss %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_cvtsi2ss:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    cvtsi2ssl %edi, %xmm1 # sched: [4:1.00]
-; HASWELL-SSE-NEXT:    cvtsi2ssl (%rsi), %xmm0 # sched: [9:1.00]
-; HASWELL-SSE-NEXT:    addss %xmm1, %xmm0 # sched: [3:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_cvtsi2ss:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vcvtsi2ssl %edi, %xmm0, %xmm0 # sched: [4:1.00]
-; HASWELL-NEXT:    vcvtsi2ssl (%rsi), %xmm1, %xmm1 # sched: [9:1.00]
-; HASWELL-NEXT:    vaddss %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_cvtsi2ss:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    cvtsi2ssl %edi, %xmm1 # sched: [4:1.00]
-; BROADWELL-SSE-NEXT:    cvtsi2ssl (%rsi), %xmm0 # sched: [9:1.00]
-; BROADWELL-SSE-NEXT:    addss %xmm1, %xmm0 # sched: [3:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_cvtsi2ss:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vcvtsi2ssl %edi, %xmm0, %xmm0 # sched: [4:1.00]
-; BROADWELL-NEXT:    vcvtsi2ssl (%rsi), %xmm1, %xmm1 # sched: [9:1.00]
-; BROADWELL-NEXT:    vaddss %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_cvtsi2ss:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    cvtsi2ssl %edi, %xmm1 # sched: [5:1.00]
-; SKYLAKE-SSE-NEXT:    cvtsi2ssl (%rsi), %xmm0 # sched: [9:1.00]
-; SKYLAKE-SSE-NEXT:    addss %xmm1, %xmm0 # sched: [4:0.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_cvtsi2ss:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vcvtsi2ssl %edi, %xmm0, %xmm0 # sched: [5:1.00]
-; SKYLAKE-NEXT:    vcvtsi2ssl (%rsi), %xmm1, %xmm1 # sched: [9:1.00]
-; SKYLAKE-NEXT:    vaddss %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_cvtsi2ss:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    cvtsi2ssl %edi, %xmm1 # sched: [5:1.00]
-; SKX-SSE-NEXT:    cvtsi2ssl (%rsi), %xmm0 # sched: [9:1.00]
-; SKX-SSE-NEXT:    addss %xmm1, %xmm0 # sched: [4:0.50]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_cvtsi2ss:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vcvtsi2ssl %edi, %xmm0, %xmm0 # sched: [5:1.00]
-; SKX-NEXT:    vcvtsi2ssl (%rsi), %xmm1, %xmm1 # sched: [9:1.00]
-; SKX-NEXT:    vaddss %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_cvtsi2ss:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    cvtsi2ssl (%rsi), %xmm0 # sched: [9:1.00]
-; BDVER2-SSE-NEXT:    cvtsi2ssl %edi, %xmm1 # sched: [4:1.00]
-; BDVER2-SSE-NEXT:    addss %xmm1, %xmm0 # sched: [5:1.00]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_cvtsi2ss:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vcvtsi2ssl %edi, %xmm0, %xmm0 # sched: [4:1.00]
-; BDVER2-NEXT:    vcvtsi2ssl (%rsi), %xmm1, %xmm1 # sched: [9:1.00]
-; BDVER2-NEXT:    vaddss %xmm1, %xmm0, %xmm0 # sched: [5:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_cvtsi2ss:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    cvtsi2ssl (%rsi), %xmm0 # sched: [14:1.00]
-; BTVER2-SSE-NEXT:    cvtsi2ssl %edi, %xmm1 # sched: [9:1.00]
-; BTVER2-SSE-NEXT:    addss %xmm1, %xmm0 # sched: [3:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_cvtsi2ss:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vcvtsi2ssl %edi, %xmm0, %xmm0 # sched: [9:1.00]
-; BTVER2-NEXT:    vcvtsi2ssl (%rsi), %xmm1, %xmm1 # sched: [14:1.00]
-; BTVER2-NEXT:    vaddss %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_cvtsi2ss:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    cvtsi2ssl (%rsi), %xmm0 # sched: [12:1.00]
-; ZNVER1-SSE-NEXT:    cvtsi2ssl %edi, %xmm1 # sched: [5:1.00]
-; ZNVER1-SSE-NEXT:    addss %xmm1, %xmm0 # sched: [3:1.00]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_cvtsi2ss:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vcvtsi2ssl %edi, %xmm0, %xmm0 # sched: [5:1.00]
-; ZNVER1-NEXT:    vcvtsi2ssl (%rsi), %xmm1, %xmm1 # sched: [12:1.00]
-; ZNVER1-NEXT:    vaddss %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = sitofp i32 %a0 to float
-  %2 = load i32, i32 *%a1, align 4
-  %3 = sitofp i32 %2 to float
-  %4 = fadd float %1, %3
-  ret float %4
-}
-
-define float @test_cvtsi2ssq(i64 %a0, i64 *%a1) {
-; GENERIC-LABEL: test_cvtsi2ssq:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    cvtsi2ssq %rdi, %xmm1 # sched: [5:2.00]
-; GENERIC-NEXT:    cvtsi2ssq (%rsi), %xmm0 # sched: [10:1.00]
-; GENERIC-NEXT:    addss %xmm1, %xmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_cvtsi2ssq:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    cvtsi2ssq (%rsi), %xmm0 # sched: [7:3.50]
-; ATOM-NEXT:    cvtsi2ssq %rdi, %xmm1 # sched: [6:3.00]
-; ATOM-NEXT:    addss %xmm1, %xmm0 # sched: [5:5.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_cvtsi2ssq:
-; SLM:       # %bb.0:
-; SLM-NEXT:    cvtsi2ssq (%rsi), %xmm0 # sched: [7:1.00]
-; SLM-NEXT:    cvtsi2ssq %rdi, %xmm1 # sched: [4:0.50]
-; SLM-NEXT:    addss %xmm1, %xmm0 # sched: [3:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_cvtsi2ssq:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    cvtsi2ssq %rdi, %xmm1 # sched: [5:2.00]
-; SANDY-SSE-NEXT:    cvtsi2ssq (%rsi), %xmm0 # sched: [10:1.00]
-; SANDY-SSE-NEXT:    addss %xmm1, %xmm0 # sched: [3:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_cvtsi2ssq:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vcvtsi2ssq %rdi, %xmm0, %xmm0 # sched: [5:2.00]
-; SANDY-NEXT:    vcvtsi2ssq (%rsi), %xmm1, %xmm1 # sched: [10:1.00]
-; SANDY-NEXT:    vaddss %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_cvtsi2ssq:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    cvtsi2ssq %rdi, %xmm1 # sched: [5:2.00]
-; HASWELL-SSE-NEXT:    cvtsi2ssq (%rsi), %xmm0 # sched: [9:1.00]
-; HASWELL-SSE-NEXT:    addss %xmm1, %xmm0 # sched: [3:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_cvtsi2ssq:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vcvtsi2ssq %rdi, %xmm0, %xmm0 # sched: [5:2.00]
-; HASWELL-NEXT:    vcvtsi2ssq (%rsi), %xmm1, %xmm1 # sched: [9:1.00]
-; HASWELL-NEXT:    vaddss %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_cvtsi2ssq:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    cvtsi2ssq %rdi, %xmm1 # sched: [5:2.00]
-; BROADWELL-SSE-NEXT:    cvtsi2ssq (%rsi), %xmm0 # sched: [9:1.00]
-; BROADWELL-SSE-NEXT:    addss %xmm1, %xmm0 # sched: [3:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_cvtsi2ssq:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vcvtsi2ssq %rdi, %xmm0, %xmm0 # sched: [5:2.00]
-; BROADWELL-NEXT:    vcvtsi2ssq (%rsi), %xmm1, %xmm1 # sched: [9:1.00]
-; BROADWELL-NEXT:    vaddss %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_cvtsi2ssq:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    cvtsi2ssq %rdi, %xmm1 # sched: [6:2.00]
-; SKYLAKE-SSE-NEXT:    cvtsi2ssq (%rsi), %xmm0 # sched: [9:1.00]
-; SKYLAKE-SSE-NEXT:    addss %xmm1, %xmm0 # sched: [4:0.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_cvtsi2ssq:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vcvtsi2ssq %rdi, %xmm0, %xmm0 # sched: [6:2.00]
-; SKYLAKE-NEXT:    vcvtsi2ssq (%rsi), %xmm1, %xmm1 # sched: [9:1.00]
-; SKYLAKE-NEXT:    vaddss %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_cvtsi2ssq:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    cvtsi2ssq %rdi, %xmm1 # sched: [6:2.00]
-; SKX-SSE-NEXT:    cvtsi2ssq (%rsi), %xmm0 # sched: [9:1.00]
-; SKX-SSE-NEXT:    addss %xmm1, %xmm0 # sched: [4:0.50]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_cvtsi2ssq:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vcvtsi2ssq %rdi, %xmm0, %xmm0 # sched: [6:2.00]
-; SKX-NEXT:    vcvtsi2ssq (%rsi), %xmm1, %xmm1 # sched: [9:1.00]
-; SKX-NEXT:    vaddss %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_cvtsi2ssq:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    cvtsi2ssq %rdi, %xmm1 # sched: [13:1.00]
-; BDVER2-SSE-NEXT:    cvtsi2ssq (%rsi), %xmm0 # sched: [9:1.00]
-; BDVER2-SSE-NEXT:    addss %xmm1, %xmm0 # sched: [5:1.00]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_cvtsi2ssq:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vcvtsi2ssq %rdi, %xmm0, %xmm0 # sched: [4:1.00]
-; BDVER2-NEXT:    vcvtsi2ssq (%rsi), %xmm1, %xmm1 # sched: [9:1.00]
-; BDVER2-NEXT:    vaddss %xmm1, %xmm0, %xmm0 # sched: [5:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_cvtsi2ssq:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    cvtsi2ssq (%rsi), %xmm0 # sched: [14:1.00]
-; BTVER2-SSE-NEXT:    cvtsi2ssq %rdi, %xmm1 # sched: [9:1.00]
-; BTVER2-SSE-NEXT:    addss %xmm1, %xmm0 # sched: [3:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_cvtsi2ssq:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vcvtsi2ssq %rdi, %xmm0, %xmm0 # sched: [9:1.00]
-; BTVER2-NEXT:    vcvtsi2ssq (%rsi), %xmm1, %xmm1 # sched: [14:1.00]
-; BTVER2-NEXT:    vaddss %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_cvtsi2ssq:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    cvtsi2ssq (%rsi), %xmm0 # sched: [12:1.00]
-; ZNVER1-SSE-NEXT:    cvtsi2ssq %rdi, %xmm1 # sched: [5:1.00]
-; ZNVER1-SSE-NEXT:    addss %xmm1, %xmm0 # sched: [3:1.00]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_cvtsi2ssq:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vcvtsi2ssq %rdi, %xmm0, %xmm0 # sched: [5:1.00]
-; ZNVER1-NEXT:    vcvtsi2ssq (%rsi), %xmm1, %xmm1 # sched: [12:1.00]
-; ZNVER1-NEXT:    vaddss %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = sitofp i64 %a0 to float
-  %2 = load i64, i64 *%a1, align 8
-  %3 = sitofp i64 %2 to float
-  %4 = fadd float %1, %3
-  ret float %4
-}
-
-define i32 @test_cvtss2si(float %a0, float *%a1) {
-; GENERIC-LABEL: test_cvtss2si:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    cvtss2si %xmm0, %ecx # sched: [5:1.00]
-; GENERIC-NEXT:    cvtss2si (%rdi), %eax # sched: [9:1.00]
-; GENERIC-NEXT:    addl %ecx, %eax # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_cvtss2si:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    cvtss2si (%rdi), %eax # sched: [9:4.50]
-; ATOM-NEXT:    cvtss2si %xmm0, %ecx # sched: [8:4.00]
-; ATOM-NEXT:    addl %ecx, %eax # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_cvtss2si:
-; SLM:       # %bb.0:
-; SLM-NEXT:    cvtss2si (%rdi), %eax # sched: [7:1.00]
-; SLM-NEXT:    cvtss2si %xmm0, %ecx # sched: [4:0.50]
-; SLM-NEXT:    addl %ecx, %eax # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_cvtss2si:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    cvtss2si %xmm0, %ecx # sched: [5:1.00]
-; SANDY-SSE-NEXT:    cvtss2si (%rdi), %eax # sched: [9:1.00]
-; SANDY-SSE-NEXT:    addl %ecx, %eax # sched: [1:0.33]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_cvtss2si:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vcvtss2si %xmm0, %ecx # sched: [5:1.00]
-; SANDY-NEXT:    vcvtss2si (%rdi), %eax # sched: [10:1.00]
-; SANDY-NEXT:    addl %ecx, %eax # sched: [1:0.33]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_cvtss2si:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    cvtss2si %xmm0, %ecx # sched: [4:1.00]
-; HASWELL-SSE-NEXT:    cvtss2si (%rdi), %eax # sched: [9:1.00]
-; HASWELL-SSE-NEXT:    addl %ecx, %eax # sched: [1:0.25]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_cvtss2si:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vcvtss2si %xmm0, %ecx # sched: [4:1.00]
-; HASWELL-NEXT:    vcvtss2si (%rdi), %eax # sched: [9:1.00]
-; HASWELL-NEXT:    addl %ecx, %eax # sched: [1:0.25]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_cvtss2si:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    cvtss2si (%rdi), %eax # sched: [9:1.00]
-; BROADWELL-SSE-NEXT:    cvtss2si %xmm0, %ecx # sched: [4:1.00]
-; BROADWELL-SSE-NEXT:    addl %ecx, %eax # sched: [1:0.25]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_cvtss2si:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vcvtss2si (%rdi), %eax # sched: [9:1.00]
-; BROADWELL-NEXT:    vcvtss2si %xmm0, %ecx # sched: [4:1.00]
-; BROADWELL-NEXT:    addl %ecx, %eax # sched: [1:0.25]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_cvtss2si:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    cvtss2si %xmm0, %ecx # sched: [6:1.00]
-; SKYLAKE-SSE-NEXT:    cvtss2si (%rdi), %eax # sched: [11:1.00]
-; SKYLAKE-SSE-NEXT:    addl %ecx, %eax # sched: [1:0.25]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_cvtss2si:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vcvtss2si %xmm0, %ecx # sched: [6:1.00]
-; SKYLAKE-NEXT:    vcvtss2si (%rdi), %eax # sched: [11:1.00]
-; SKYLAKE-NEXT:    addl %ecx, %eax # sched: [1:0.25]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_cvtss2si:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    cvtss2si %xmm0, %ecx # sched: [6:1.00]
-; SKX-SSE-NEXT:    cvtss2si (%rdi), %eax # sched: [11:1.00]
-; SKX-SSE-NEXT:    addl %ecx, %eax # sched: [1:0.25]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_cvtss2si:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vcvtss2si %xmm0, %ecx # sched: [6:1.00]
-; SKX-NEXT:    vcvtss2si (%rdi), %eax # sched: [11:1.00]
-; SKX-NEXT:    addl %ecx, %eax # sched: [1:0.25]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_cvtss2si:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    cvtss2si (%rdi), %eax # sched: [18:1.00]
-; BDVER2-SSE-NEXT:    cvtss2si %xmm0, %ecx # sched: [13:1.00]
-; BDVER2-SSE-NEXT:    addl %ecx, %eax # sched: [1:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_cvtss2si:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vcvtss2si (%rdi), %eax # sched: [18:1.00]
-; BDVER2-NEXT:    vcvtss2si %xmm0, %ecx # sched: [13:1.00]
-; BDVER2-NEXT:    addl %ecx, %eax # sched: [1:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_cvtss2si:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    cvtss2si (%rdi), %eax # sched: [12:1.00]
-; BTVER2-SSE-NEXT:    cvtss2si %xmm0, %ecx # sched: [7:1.00]
-; BTVER2-SSE-NEXT:    addl %ecx, %eax # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_cvtss2si:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vcvtss2si (%rdi), %eax # sched: [12:1.00]
-; BTVER2-NEXT:    vcvtss2si %xmm0, %ecx # sched: [7:1.00]
-; BTVER2-NEXT:    addl %ecx, %eax # sched: [1:0.50]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_cvtss2si:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    cvtss2si (%rdi), %eax # sched: [12:1.00]
-; ZNVER1-SSE-NEXT:    cvtss2si %xmm0, %ecx # sched: [5:1.00]
-; ZNVER1-SSE-NEXT:    addl %ecx, %eax # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_cvtss2si:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vcvtss2si (%rdi), %eax # sched: [12:1.00]
-; ZNVER1-NEXT:    vcvtss2si %xmm0, %ecx # sched: [5:1.00]
-; ZNVER1-NEXT:    addl %ecx, %eax # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = insertelement <4 x float> undef, float %a0, i32 0
-  %2 = call i32 @llvm.x86.sse.cvtss2si(<4 x float> %1)
-  %3 = load float, float *%a1, align 4
-  %4 = insertelement <4 x float> undef, float %3, i32 0
-  %5 = call i32 @llvm.x86.sse.cvtss2si(<4 x float> %4)
-  %6 = add i32 %2, %5
-  ret i32 %6
-}
-declare i32 @llvm.x86.sse.cvtss2si(<4 x float>) nounwind readnone
-
-define i64 @test_cvtss2siq(float %a0, float *%a1) {
-; GENERIC-LABEL: test_cvtss2siq:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    cvtss2si %xmm0, %rcx # sched: [5:1.00]
-; GENERIC-NEXT:    cvtss2si (%rdi), %rax # sched: [9:1.00]
-; GENERIC-NEXT:    addq %rcx, %rax # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_cvtss2siq:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    cvtss2si (%rdi), %rax # sched: [10:5.00]
-; ATOM-NEXT:    cvtss2si %xmm0, %rcx # sched: [9:4.50]
-; ATOM-NEXT:    addq %rcx, %rax # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_cvtss2siq:
-; SLM:       # %bb.0:
-; SLM-NEXT:    cvtss2si (%rdi), %rax # sched: [7:1.00]
-; SLM-NEXT:    cvtss2si %xmm0, %rcx # sched: [4:0.50]
-; SLM-NEXT:    addq %rcx, %rax # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_cvtss2siq:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    cvtss2si %xmm0, %rcx # sched: [5:1.00]
-; SANDY-SSE-NEXT:    cvtss2si (%rdi), %rax # sched: [9:1.00]
-; SANDY-SSE-NEXT:    addq %rcx, %rax # sched: [1:0.33]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_cvtss2siq:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vcvtss2si %xmm0, %rcx # sched: [5:1.00]
-; SANDY-NEXT:    vcvtss2si (%rdi), %rax # sched: [10:1.00]
-; SANDY-NEXT:    addq %rcx, %rax # sched: [1:0.33]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_cvtss2siq:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    cvtss2si %xmm0, %rcx # sched: [4:1.00]
-; HASWELL-SSE-NEXT:    cvtss2si (%rdi), %rax # sched: [9:1.00]
-; HASWELL-SSE-NEXT:    addq %rcx, %rax # sched: [1:0.25]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_cvtss2siq:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vcvtss2si %xmm0, %rcx # sched: [4:1.00]
-; HASWELL-NEXT:    vcvtss2si (%rdi), %rax # sched: [9:1.00]
-; HASWELL-NEXT:    addq %rcx, %rax # sched: [1:0.25]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_cvtss2siq:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    cvtss2si (%rdi), %rax # sched: [9:1.00]
-; BROADWELL-SSE-NEXT:    cvtss2si %xmm0, %rcx # sched: [4:1.00]
-; BROADWELL-SSE-NEXT:    addq %rcx, %rax # sched: [1:0.25]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_cvtss2siq:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vcvtss2si (%rdi), %rax # sched: [9:1.00]
-; BROADWELL-NEXT:    vcvtss2si %xmm0, %rcx # sched: [4:1.00]
-; BROADWELL-NEXT:    addq %rcx, %rax # sched: [1:0.25]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_cvtss2siq:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    cvtss2si %xmm0, %rcx # sched: [6:1.00]
-; SKYLAKE-SSE-NEXT:    cvtss2si (%rdi), %rax # sched: [11:1.00]
-; SKYLAKE-SSE-NEXT:    addq %rcx, %rax # sched: [1:0.25]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_cvtss2siq:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vcvtss2si %xmm0, %rcx # sched: [6:1.00]
-; SKYLAKE-NEXT:    vcvtss2si (%rdi), %rax # sched: [11:1.00]
-; SKYLAKE-NEXT:    addq %rcx, %rax # sched: [1:0.25]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_cvtss2siq:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    cvtss2si %xmm0, %rcx # sched: [7:1.00]
-; SKX-SSE-NEXT:    cvtss2si (%rdi), %rax # sched: [11:1.00]
-; SKX-SSE-NEXT:    addq %rcx, %rax # sched: [1:0.25]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_cvtss2siq:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vcvtss2si %xmm0, %rcx # sched: [7:1.00]
-; SKX-NEXT:    vcvtss2si (%rdi), %rax # sched: [11:1.00]
-; SKX-NEXT:    addq %rcx, %rax # sched: [1:0.25]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_cvtss2siq:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    cvtss2si (%rdi), %rax # sched: [18:1.00]
-; BDVER2-SSE-NEXT:    cvtss2si %xmm0, %rcx # sched: [13:1.00]
-; BDVER2-SSE-NEXT:    addq %rcx, %rax # sched: [1:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_cvtss2siq:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vcvtss2si (%rdi), %rax # sched: [18:1.00]
-; BDVER2-NEXT:    vcvtss2si %xmm0, %rcx # sched: [13:1.00]
-; BDVER2-NEXT:    addq %rcx, %rax # sched: [1:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_cvtss2siq:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    cvtss2si (%rdi), %rax # sched: [12:1.00]
-; BTVER2-SSE-NEXT:    cvtss2si %xmm0, %rcx # sched: [7:1.00]
-; BTVER2-SSE-NEXT:    addq %rcx, %rax # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_cvtss2siq:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vcvtss2si (%rdi), %rax # sched: [12:1.00]
-; BTVER2-NEXT:    vcvtss2si %xmm0, %rcx # sched: [7:1.00]
-; BTVER2-NEXT:    addq %rcx, %rax # sched: [1:0.50]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_cvtss2siq:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    cvtss2si (%rdi), %rax # sched: [12:1.00]
-; ZNVER1-SSE-NEXT:    cvtss2si %xmm0, %rcx # sched: [5:1.00]
-; ZNVER1-SSE-NEXT:    addq %rcx, %rax # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_cvtss2siq:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vcvtss2si (%rdi), %rax # sched: [12:1.00]
-; ZNVER1-NEXT:    vcvtss2si %xmm0, %rcx # sched: [5:1.00]
-; ZNVER1-NEXT:    addq %rcx, %rax # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = insertelement <4 x float> undef, float %a0, i32 0
-  %2 = call i64 @llvm.x86.sse.cvtss2si64(<4 x float> %1)
-  %3 = load float, float *%a1, align 4
-  %4 = insertelement <4 x float> undef, float %3, i32 0
-  %5 = call i64 @llvm.x86.sse.cvtss2si64(<4 x float> %4)
-  %6 = add i64 %2, %5
-  ret i64 %6
-}
-declare i64 @llvm.x86.sse.cvtss2si64(<4 x float>) nounwind readnone
-
-define i32 @test_cvttss2si(float %a0, float *%a1) {
-; GENERIC-LABEL: test_cvttss2si:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    cvttss2si %xmm0, %ecx # sched: [5:1.00]
-; GENERIC-NEXT:    cvttss2si (%rdi), %eax # sched: [9:1.00]
-; GENERIC-NEXT:    addl %ecx, %eax # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_cvttss2si:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    cvttss2si (%rdi), %eax # sched: [9:4.50]
-; ATOM-NEXT:    cvttss2si %xmm0, %ecx # sched: [8:4.00]
-; ATOM-NEXT:    addl %ecx, %eax # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_cvttss2si:
-; SLM:       # %bb.0:
-; SLM-NEXT:    cvttss2si (%rdi), %eax # sched: [7:1.00]
-; SLM-NEXT:    cvttss2si %xmm0, %ecx # sched: [4:0.50]
-; SLM-NEXT:    addl %ecx, %eax # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_cvttss2si:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    cvttss2si %xmm0, %ecx # sched: [5:1.00]
-; SANDY-SSE-NEXT:    cvttss2si (%rdi), %eax # sched: [9:1.00]
-; SANDY-SSE-NEXT:    addl %ecx, %eax # sched: [1:0.33]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_cvttss2si:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vcvttss2si %xmm0, %ecx # sched: [5:1.00]
-; SANDY-NEXT:    vcvttss2si (%rdi), %eax # sched: [10:1.00]
-; SANDY-NEXT:    addl %ecx, %eax # sched: [1:0.33]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_cvttss2si:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    cvttss2si %xmm0, %ecx # sched: [4:1.00]
-; HASWELL-SSE-NEXT:    cvttss2si (%rdi), %eax # sched: [9:1.00]
-; HASWELL-SSE-NEXT:    addl %ecx, %eax # sched: [1:0.25]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_cvttss2si:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vcvttss2si %xmm0, %ecx # sched: [4:1.00]
-; HASWELL-NEXT:    vcvttss2si (%rdi), %eax # sched: [9:1.00]
-; HASWELL-NEXT:    addl %ecx, %eax # sched: [1:0.25]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_cvttss2si:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    cvttss2si (%rdi), %eax # sched: [9:1.00]
-; BROADWELL-SSE-NEXT:    cvttss2si %xmm0, %ecx # sched: [4:1.00]
-; BROADWELL-SSE-NEXT:    addl %ecx, %eax # sched: [1:0.25]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_cvttss2si:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vcvttss2si (%rdi), %eax # sched: [9:1.00]
-; BROADWELL-NEXT:    vcvttss2si %xmm0, %ecx # sched: [4:1.00]
-; BROADWELL-NEXT:    addl %ecx, %eax # sched: [1:0.25]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_cvttss2si:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    cvttss2si %xmm0, %ecx # sched: [7:1.00]
-; SKYLAKE-SSE-NEXT:    cvttss2si (%rdi), %eax # sched: [11:1.00]
-; SKYLAKE-SSE-NEXT:    addl %ecx, %eax # sched: [1:0.25]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_cvttss2si:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vcvttss2si %xmm0, %ecx # sched: [7:1.00]
-; SKYLAKE-NEXT:    vcvttss2si (%rdi), %eax # sched: [11:1.00]
-; SKYLAKE-NEXT:    addl %ecx, %eax # sched: [1:0.25]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_cvttss2si:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    cvttss2si %xmm0, %ecx # sched: [6:1.00]
-; SKX-SSE-NEXT:    cvttss2si (%rdi), %eax # sched: [11:1.00]
-; SKX-SSE-NEXT:    addl %ecx, %eax # sched: [1:0.25]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_cvttss2si:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vcvttss2si %xmm0, %ecx # sched: [6:1.00]
-; SKX-NEXT:    vcvttss2si (%rdi), %eax # sched: [11:1.00]
-; SKX-NEXT:    addl %ecx, %eax # sched: [1:0.25]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_cvttss2si:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    cvttss2si (%rdi), %eax # sched: [18:1.00]
-; BDVER2-SSE-NEXT:    cvttss2si %xmm0, %ecx # sched: [13:1.00]
-; BDVER2-SSE-NEXT:    addl %ecx, %eax # sched: [1:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_cvttss2si:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vcvttss2si (%rdi), %eax # sched: [18:1.00]
-; BDVER2-NEXT:    vcvttss2si %xmm0, %ecx # sched: [13:1.00]
-; BDVER2-NEXT:    addl %ecx, %eax # sched: [1:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_cvttss2si:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    cvttss2si (%rdi), %eax # sched: [12:1.00]
-; BTVER2-SSE-NEXT:    cvttss2si %xmm0, %ecx # sched: [7:1.00]
-; BTVER2-SSE-NEXT:    addl %ecx, %eax # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_cvttss2si:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vcvttss2si (%rdi), %eax # sched: [12:1.00]
-; BTVER2-NEXT:    vcvttss2si %xmm0, %ecx # sched: [7:1.00]
-; BTVER2-NEXT:    addl %ecx, %eax # sched: [1:0.50]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_cvttss2si:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    cvttss2si (%rdi), %eax # sched: [12:1.00]
-; ZNVER1-SSE-NEXT:    cvttss2si %xmm0, %ecx # sched: [5:1.00]
-; ZNVER1-SSE-NEXT:    addl %ecx, %eax # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_cvttss2si:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vcvttss2si (%rdi), %eax # sched: [12:1.00]
-; ZNVER1-NEXT:    vcvttss2si %xmm0, %ecx # sched: [5:1.00]
-; ZNVER1-NEXT:    addl %ecx, %eax # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = fptosi float %a0 to i32
-  %2 = load float, float *%a1, align 4
-  %3 = fptosi float %2 to i32
-  %4 = add i32 %1, %3
-  ret i32 %4
-}
-
-define i64 @test_cvttss2siq(float %a0, float *%a1) {
-; GENERIC-LABEL: test_cvttss2siq:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    cvttss2si %xmm0, %rcx # sched: [5:1.00]
-; GENERIC-NEXT:    cvttss2si (%rdi), %rax # sched: [9:1.00]
-; GENERIC-NEXT:    addq %rcx, %rax # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_cvttss2siq:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    cvttss2si (%rdi), %rax # sched: [10:5.00]
-; ATOM-NEXT:    cvttss2si %xmm0, %rcx # sched: [9:4.50]
-; ATOM-NEXT:    addq %rcx, %rax # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_cvttss2siq:
-; SLM:       # %bb.0:
-; SLM-NEXT:    cvttss2si (%rdi), %rax # sched: [7:1.00]
-; SLM-NEXT:    cvttss2si %xmm0, %rcx # sched: [4:0.50]
-; SLM-NEXT:    addq %rcx, %rax # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_cvttss2siq:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    cvttss2si %xmm0, %rcx # sched: [5:1.00]
-; SANDY-SSE-NEXT:    cvttss2si (%rdi), %rax # sched: [9:1.00]
-; SANDY-SSE-NEXT:    addq %rcx, %rax # sched: [1:0.33]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_cvttss2siq:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vcvttss2si %xmm0, %rcx # sched: [5:1.00]
-; SANDY-NEXT:    vcvttss2si (%rdi), %rax # sched: [10:1.00]
-; SANDY-NEXT:    addq %rcx, %rax # sched: [1:0.33]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_cvttss2siq:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    cvttss2si (%rdi), %rax # sched: [10:1.00]
-; HASWELL-SSE-NEXT:    cvttss2si %xmm0, %rcx # sched: [4:1.00]
-; HASWELL-SSE-NEXT:    addq %rcx, %rax # sched: [1:0.25]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_cvttss2siq:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vcvttss2si %xmm0, %rcx # sched: [4:1.00]
-; HASWELL-NEXT:    vcvttss2si (%rdi), %rax # sched: [9:1.00]
-; HASWELL-NEXT:    addq %rcx, %rax # sched: [1:0.25]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_cvttss2siq:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    cvttss2si (%rdi), %rax # sched: [10:1.00]
-; BROADWELL-SSE-NEXT:    cvttss2si %xmm0, %rcx # sched: [4:1.00]
-; BROADWELL-SSE-NEXT:    addq %rcx, %rax # sched: [1:0.25]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_cvttss2siq:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vcvttss2si (%rdi), %rax # sched: [9:1.00]
-; BROADWELL-NEXT:    vcvttss2si %xmm0, %rcx # sched: [4:1.00]
-; BROADWELL-NEXT:    addq %rcx, %rax # sched: [1:0.25]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_cvttss2siq:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    cvttss2si %xmm0, %rcx # sched: [7:1.00]
-; SKYLAKE-SSE-NEXT:    cvttss2si (%rdi), %rax # sched: [12:1.00]
-; SKYLAKE-SSE-NEXT:    addq %rcx, %rax # sched: [1:0.25]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_cvttss2siq:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vcvttss2si %xmm0, %rcx # sched: [7:1.00]
-; SKYLAKE-NEXT:    vcvttss2si (%rdi), %rax # sched: [11:1.00]
-; SKYLAKE-NEXT:    addq %rcx, %rax # sched: [1:0.25]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_cvttss2siq:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    cvttss2si %xmm0, %rcx # sched: [7:1.00]
-; SKX-SSE-NEXT:    cvttss2si (%rdi), %rax # sched: [12:1.00]
-; SKX-SSE-NEXT:    addq %rcx, %rax # sched: [1:0.25]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_cvttss2siq:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vcvttss2si %xmm0, %rcx # sched: [7:1.00]
-; SKX-NEXT:    vcvttss2si (%rdi), %rax # sched: [11:1.00]
-; SKX-NEXT:    addq %rcx, %rax # sched: [1:0.25]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_cvttss2siq:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    cvttss2si (%rdi), %rax # sched: [18:1.00]
-; BDVER2-SSE-NEXT:    cvttss2si %xmm0, %rcx # sched: [13:1.00]
-; BDVER2-SSE-NEXT:    addq %rcx, %rax # sched: [1:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_cvttss2siq:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vcvttss2si (%rdi), %rax # sched: [18:1.00]
-; BDVER2-NEXT:    vcvttss2si %xmm0, %rcx # sched: [13:1.00]
-; BDVER2-NEXT:    addq %rcx, %rax # sched: [1:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_cvttss2siq:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    cvttss2si (%rdi), %rax # sched: [12:1.00]
-; BTVER2-SSE-NEXT:    cvttss2si %xmm0, %rcx # sched: [7:1.00]
-; BTVER2-SSE-NEXT:    addq %rcx, %rax # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_cvttss2siq:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vcvttss2si (%rdi), %rax # sched: [12:1.00]
-; BTVER2-NEXT:    vcvttss2si %xmm0, %rcx # sched: [7:1.00]
-; BTVER2-NEXT:    addq %rcx, %rax # sched: [1:0.50]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_cvttss2siq:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    cvttss2si (%rdi), %rax # sched: [12:1.00]
-; ZNVER1-SSE-NEXT:    cvttss2si %xmm0, %rcx # sched: [5:1.00]
-; ZNVER1-SSE-NEXT:    addq %rcx, %rax # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_cvttss2siq:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vcvttss2si (%rdi), %rax # sched: [12:1.00]
-; ZNVER1-NEXT:    vcvttss2si %xmm0, %rcx # sched: [5:1.00]
-; ZNVER1-NEXT:    addq %rcx, %rax # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = fptosi float %a0 to i64
-  %2 = load float, float *%a1, align 4
-  %3 = fptosi float %2 to i64
-  %4 = add i64 %1, %3
-  ret i64 %4
-}
-
-define <4 x float> @test_divps(<4 x float> %a0, <4 x float> %a1, <4 x float> *%a2) {
-; GENERIC-LABEL: test_divps:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    divps %xmm1, %xmm0 # sched: [14:14.00]
-; GENERIC-NEXT:    divps (%rdi), %xmm0 # sched: [20:14.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_divps:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    divps %xmm1, %xmm0 # sched: [70:35.00]
-; ATOM-NEXT:    divps (%rdi), %xmm0 # sched: [70:35.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_divps:
-; SLM:       # %bb.0:
-; SLM-NEXT:    divps %xmm1, %xmm0 # sched: [39:39.00]
-; SLM-NEXT:    divps (%rdi), %xmm0 # sched: [42:39.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_divps:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    divps %xmm1, %xmm0 # sched: [14:14.00]
-; SANDY-SSE-NEXT:    divps (%rdi), %xmm0 # sched: [20:14.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_divps:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vdivps %xmm1, %xmm0, %xmm0 # sched: [14:14.00]
-; SANDY-NEXT:    vdivps (%rdi), %xmm0, %xmm0 # sched: [20:14.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_divps:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    divps %xmm1, %xmm0 # sched: [13:7.00]
-; HASWELL-SSE-NEXT:    divps (%rdi), %xmm0 # sched: [19:7.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_divps:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vdivps %xmm1, %xmm0, %xmm0 # sched: [13:7.00]
-; HASWELL-NEXT:    vdivps (%rdi), %xmm0, %xmm0 # sched: [19:7.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_divps:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    divps %xmm1, %xmm0 # sched: [11:5.00]
-; BROADWELL-SSE-NEXT:    divps (%rdi), %xmm0 # sched: [16:5.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_divps:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vdivps %xmm1, %xmm0, %xmm0 # sched: [11:5.00]
-; BROADWELL-NEXT:    vdivps (%rdi), %xmm0, %xmm0 # sched: [16:5.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_divps:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    divps %xmm1, %xmm0 # sched: [11:3.00]
-; SKYLAKE-SSE-NEXT:    divps (%rdi), %xmm0 # sched: [17:5.00]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_divps:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vdivps %xmm1, %xmm0, %xmm0 # sched: [11:3.00]
-; SKYLAKE-NEXT:    vdivps (%rdi), %xmm0, %xmm0 # sched: [17:5.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_divps:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    divps %xmm1, %xmm0 # sched: [11:3.00]
-; SKX-SSE-NEXT:    divps (%rdi), %xmm0 # sched: [17:5.00]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_divps:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vdivps %xmm1, %xmm0, %xmm0 # sched: [11:3.00]
-; SKX-NEXT:    vdivps (%rdi), %xmm0, %xmm0 # sched: [17:5.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_divps:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    divps %xmm1, %xmm0 # sched: [9:9.50]
-; BDVER2-SSE-NEXT:    divps (%rdi), %xmm0 # sched: [14:9.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_divps:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vdivps %xmm1, %xmm0, %xmm0 # sched: [9:9.50]
-; BDVER2-NEXT:    vdivps (%rdi), %xmm0, %xmm0 # sched: [14:9.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_divps:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    divps %xmm1, %xmm0 # sched: [19:19.00]
-; BTVER2-SSE-NEXT:    divps (%rdi), %xmm0 # sched: [24:19.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_divps:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vdivps %xmm1, %xmm0, %xmm0 # sched: [19:19.00]
-; BTVER2-NEXT:    vdivps (%rdi), %xmm0, %xmm0 # sched: [24:19.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_divps:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    divps %xmm1, %xmm0 # sched: [15:1.00]
-; ZNVER1-SSE-NEXT:    divps (%rdi), %xmm0 # sched: [22:1.00]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_divps:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vdivps %xmm1, %xmm0, %xmm0 # sched: [15:1.00]
-; ZNVER1-NEXT:    vdivps (%rdi), %xmm0, %xmm0 # sched: [22:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = fdiv <4 x float> %a0, %a1
-  %2 = load <4 x float>, <4 x float> *%a2, align 16
-  %3 = fdiv <4 x float> %1, %2
-  ret <4 x float> %3
-}
-
-define float @test_divss(float %a0, float %a1, float *%a2) {
-; GENERIC-LABEL: test_divss:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    divss %xmm1, %xmm0 # sched: [14:14.00]
-; GENERIC-NEXT:    divss (%rdi), %xmm0 # sched: [20:14.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_divss:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    divss %xmm1, %xmm0 # sched: [34:17.00]
-; ATOM-NEXT:    divss (%rdi), %xmm0 # sched: [34:17.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_divss:
-; SLM:       # %bb.0:
-; SLM-NEXT:    divss %xmm1, %xmm0 # sched: [19:17.00]
-; SLM-NEXT:    divss (%rdi), %xmm0 # sched: [22:17.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_divss:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    divss %xmm1, %xmm0 # sched: [14:14.00]
-; SANDY-SSE-NEXT:    divss (%rdi), %xmm0 # sched: [20:14.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_divss:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vdivss %xmm1, %xmm0, %xmm0 # sched: [14:14.00]
-; SANDY-NEXT:    vdivss (%rdi), %xmm0, %xmm0 # sched: [20:14.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_divss:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    divss %xmm1, %xmm0 # sched: [13:7.00]
-; HASWELL-SSE-NEXT:    divss (%rdi), %xmm0 # sched: [18:7.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_divss:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vdivss %xmm1, %xmm0, %xmm0 # sched: [13:7.00]
-; HASWELL-NEXT:    vdivss (%rdi), %xmm0, %xmm0 # sched: [18:7.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_divss:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    divss %xmm1, %xmm0 # sched: [11:3.00]
-; BROADWELL-SSE-NEXT:    divss (%rdi), %xmm0 # sched: [16:5.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_divss:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vdivss %xmm1, %xmm0, %xmm0 # sched: [11:3.00]
-; BROADWELL-NEXT:    vdivss (%rdi), %xmm0, %xmm0 # sched: [16:5.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_divss:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    divss %xmm1, %xmm0 # sched: [11:3.00]
-; SKYLAKE-SSE-NEXT:    divss (%rdi), %xmm0 # sched: [16:3.00]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_divss:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vdivss %xmm1, %xmm0, %xmm0 # sched: [11:3.00]
-; SKYLAKE-NEXT:    vdivss (%rdi), %xmm0, %xmm0 # sched: [16:3.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_divss:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    divss %xmm1, %xmm0 # sched: [11:3.00]
-; SKX-SSE-NEXT:    divss (%rdi), %xmm0 # sched: [16:3.00]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_divss:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vdivss %xmm1, %xmm0, %xmm0 # sched: [11:3.00]
-; SKX-NEXT:    vdivss (%rdi), %xmm0, %xmm0 # sched: [16:3.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_divss:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    divss %xmm1, %xmm0 # sched: [9:9.50]
-; BDVER2-SSE-NEXT:    divss (%rdi), %xmm0 # sched: [14:9.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_divss:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vdivss %xmm1, %xmm0, %xmm0 # sched: [9:9.50]
-; BDVER2-NEXT:    vdivss (%rdi), %xmm0, %xmm0 # sched: [14:9.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_divss:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    divss %xmm1, %xmm0 # sched: [19:19.00]
-; BTVER2-SSE-NEXT:    divss (%rdi), %xmm0 # sched: [24:19.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_divss:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vdivss %xmm1, %xmm0, %xmm0 # sched: [19:19.00]
-; BTVER2-NEXT:    vdivss (%rdi), %xmm0, %xmm0 # sched: [24:19.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_divss:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    divss %xmm1, %xmm0 # sched: [15:1.00]
-; ZNVER1-SSE-NEXT:    divss (%rdi), %xmm0 # sched: [22:1.00]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_divss:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vdivss %xmm1, %xmm0, %xmm0 # sched: [15:1.00]
-; ZNVER1-NEXT:    vdivss (%rdi), %xmm0, %xmm0 # sched: [22:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = fdiv float %a0, %a1
-  %2 = load float, float *%a2, align 4
-  %3 = fdiv float %1, %2
-  ret float %3
-}
-
-define void @test_ldmxcsr(i32 %a0) {
-; GENERIC-LABEL: test_ldmxcsr:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    movl %edi, -{{[0-9]+}}(%rsp) # sched: [1:1.00]
-; GENERIC-NEXT:    ldmxcsr -{{[0-9]+}}(%rsp) # sched: [5:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_ldmxcsr:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    movl %edi, -{{[0-9]+}}(%rsp) # sched: [1:1.00]
-; ATOM-NEXT:    ldmxcsr -{{[0-9]+}}(%rsp) # sched: [5:2.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_ldmxcsr:
-; SLM:       # %bb.0:
-; SLM-NEXT:    movl %edi, -{{[0-9]+}}(%rsp) # sched: [1:1.00]
-; SLM-NEXT:    ldmxcsr -{{[0-9]+}}(%rsp) # sched: [3:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_ldmxcsr:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    movl %edi, -{{[0-9]+}}(%rsp) # sched: [1:1.00]
-; SANDY-SSE-NEXT:    ldmxcsr -{{[0-9]+}}(%rsp) # sched: [5:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_ldmxcsr:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    movl %edi, -{{[0-9]+}}(%rsp) # sched: [1:1.00]
-; SANDY-NEXT:    vldmxcsr -{{[0-9]+}}(%rsp) # sched: [5:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_ldmxcsr:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    movl %edi, -{{[0-9]+}}(%rsp) # sched: [1:1.00]
-; HASWELL-SSE-NEXT:    ldmxcsr -{{[0-9]+}}(%rsp) # sched: [7:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_ldmxcsr:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    movl %edi, -{{[0-9]+}}(%rsp) # sched: [1:1.00]
-; HASWELL-NEXT:    vldmxcsr -{{[0-9]+}}(%rsp) # sched: [7:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_ldmxcsr:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    movl %edi, -{{[0-9]+}}(%rsp) # sched: [1:1.00]
-; BROADWELL-SSE-NEXT:    ldmxcsr -{{[0-9]+}}(%rsp) # sched: [7:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_ldmxcsr:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    movl %edi, -{{[0-9]+}}(%rsp) # sched: [1:1.00]
-; BROADWELL-NEXT:    vldmxcsr -{{[0-9]+}}(%rsp) # sched: [7:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_ldmxcsr:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    movl %edi, -{{[0-9]+}}(%rsp) # sched: [1:1.00]
-; SKYLAKE-SSE-NEXT:    ldmxcsr -{{[0-9]+}}(%rsp) # sched: [7:1.00]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_ldmxcsr:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    movl %edi, -{{[0-9]+}}(%rsp) # sched: [1:1.00]
-; SKYLAKE-NEXT:    vldmxcsr -{{[0-9]+}}(%rsp) # sched: [7:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_ldmxcsr:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    movl %edi, -{{[0-9]+}}(%rsp) # sched: [1:1.00]
-; SKX-SSE-NEXT:    ldmxcsr -{{[0-9]+}}(%rsp) # sched: [7:1.00]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_ldmxcsr:
-; SKX:       # %bb.0:
-; SKX-NEXT:    movl %edi, -{{[0-9]+}}(%rsp) # sched: [1:1.00]
-; SKX-NEXT:    vldmxcsr -{{[0-9]+}}(%rsp) # sched: [7:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_ldmxcsr:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    movl %edi, -{{[0-9]+}}(%rsp) # sched: [1:1.00]
-; BDVER2-SSE-NEXT:    ldmxcsr -{{[0-9]+}}(%rsp) # sched: [5:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_ldmxcsr:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    movl %edi, -{{[0-9]+}}(%rsp) # sched: [1:1.00]
-; BDVER2-NEXT:    vldmxcsr -{{[0-9]+}}(%rsp) # sched: [5:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_ldmxcsr:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    movl %edi, -{{[0-9]+}}(%rsp) # sched: [1:1.00]
-; BTVER2-SSE-NEXT:    ldmxcsr -{{[0-9]+}}(%rsp) # sched: [5:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_ldmxcsr:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    movl %edi, -{{[0-9]+}}(%rsp) # sched: [1:1.00]
-; BTVER2-NEXT:    vldmxcsr -{{[0-9]+}}(%rsp) # sched: [5:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_ldmxcsr:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    movl %edi, -{{[0-9]+}}(%rsp) # sched: [1:0.50]
-; ZNVER1-SSE-NEXT:    ldmxcsr -{{[0-9]+}}(%rsp) # sched: [100:0.25]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_ldmxcsr:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    movl %edi, -{{[0-9]+}}(%rsp) # sched: [1:0.50]
-; ZNVER1-NEXT:    vldmxcsr -{{[0-9]+}}(%rsp) # sched: [100:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = alloca i32, align 4
-  %2 = bitcast i32* %1 to i8*
-  store i32 %a0, i32* %1
-  call void @llvm.x86.sse.ldmxcsr(i8* %2)
-  ret void
-}
-declare void @llvm.x86.sse.ldmxcsr(i8*) nounwind readnone
-
-define <4 x float> @test_maxps(<4 x float> %a0, <4 x float> %a1, <4 x float> *%a2) {
-; GENERIC-LABEL: test_maxps:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    maxps %xmm1, %xmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    maxps (%rdi), %xmm0 # sched: [9:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_maxps:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    maxps %xmm1, %xmm0 # sched: [5:5.00]
-; ATOM-NEXT:    maxps (%rdi), %xmm0 # sched: [5:5.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_maxps:
-; SLM:       # %bb.0:
-; SLM-NEXT:    maxps %xmm1, %xmm0 # sched: [3:1.00]
-; SLM-NEXT:    maxps (%rdi), %xmm0 # sched: [6:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_maxps:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    maxps %xmm1, %xmm0 # sched: [3:1.00]
-; SANDY-SSE-NEXT:    maxps (%rdi), %xmm0 # sched: [9:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_maxps:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vmaxps %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; SANDY-NEXT:    vmaxps (%rdi), %xmm0, %xmm0 # sched: [9:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_maxps:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    maxps %xmm1, %xmm0 # sched: [3:1.00]
-; HASWELL-SSE-NEXT:    maxps (%rdi), %xmm0 # sched: [9:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_maxps:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vmaxps %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; HASWELL-NEXT:    vmaxps (%rdi), %xmm0, %xmm0 # sched: [9:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_maxps:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    maxps %xmm1, %xmm0 # sched: [3:1.00]
-; BROADWELL-SSE-NEXT:    maxps (%rdi), %xmm0 # sched: [8:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_maxps:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vmaxps %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    vmaxps (%rdi), %xmm0, %xmm0 # sched: [8:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_maxps:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    maxps %xmm1, %xmm0 # sched: [4:0.50]
-; SKYLAKE-SSE-NEXT:    maxps (%rdi), %xmm0 # sched: [10:0.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_maxps:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vmaxps %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    vmaxps (%rdi), %xmm0, %xmm0 # sched: [10:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_maxps:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    maxps %xmm1, %xmm0 # sched: [4:0.50]
-; SKX-SSE-NEXT:    maxps (%rdi), %xmm0 # sched: [10:0.50]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_maxps:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmaxps %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKX-NEXT:    vmaxps (%rdi), %xmm0, %xmm0 # sched: [10:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_maxps:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    maxps %xmm1, %xmm0 # sched: [2:1.00]
-; BDVER2-SSE-NEXT:    maxps (%rdi), %xmm0 # sched: [7:1.00]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_maxps:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vmaxps %xmm1, %xmm0, %xmm0 # sched: [2:1.00]
-; BDVER2-NEXT:    vmaxps (%rdi), %xmm0, %xmm0 # sched: [7:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_maxps:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    maxps %xmm1, %xmm0 # sched: [2:1.00]
-; BTVER2-SSE-NEXT:    maxps (%rdi), %xmm0 # sched: [7:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_maxps:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vmaxps %xmm1, %xmm0, %xmm0 # sched: [2:1.00]
-; BTVER2-NEXT:    vmaxps (%rdi), %xmm0, %xmm0 # sched: [7:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_maxps:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    maxps %xmm1, %xmm0 # sched: [3:1.00]
-; ZNVER1-SSE-NEXT:    maxps (%rdi), %xmm0 # sched: [10:1.00]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_maxps:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vmaxps %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    vmaxps (%rdi), %xmm0, %xmm0 # sched: [10:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <4 x float> @llvm.x86.sse.max.ps(<4 x float> %a0, <4 x float> %a1)
-  %2 = load <4 x float>, <4 x float> *%a2, align 16
-  %3 = call <4 x float> @llvm.x86.sse.max.ps(<4 x float> %1, <4 x float> %2)
-  ret <4 x float> %3
-}
-declare <4 x float> @llvm.x86.sse.max.ps(<4 x float>, <4 x float>) nounwind readnone
-
-define <4 x float> @test_maxss(<4 x float> %a0, <4 x float> %a1, <4 x float> *%a2) {
-; GENERIC-LABEL: test_maxss:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    maxss %xmm1, %xmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    maxss (%rdi), %xmm0 # sched: [9:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_maxss:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    maxss %xmm1, %xmm0 # sched: [5:5.00]
-; ATOM-NEXT:    maxss (%rdi), %xmm0 # sched: [5:5.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_maxss:
-; SLM:       # %bb.0:
-; SLM-NEXT:    maxss %xmm1, %xmm0 # sched: [3:1.00]
-; SLM-NEXT:    maxss (%rdi), %xmm0 # sched: [6:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_maxss:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    maxss %xmm1, %xmm0 # sched: [3:1.00]
-; SANDY-SSE-NEXT:    maxss (%rdi), %xmm0 # sched: [9:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_maxss:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vmaxss %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; SANDY-NEXT:    vmaxss (%rdi), %xmm0, %xmm0 # sched: [9:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_maxss:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    maxss %xmm1, %xmm0 # sched: [3:1.00]
-; HASWELL-SSE-NEXT:    maxss (%rdi), %xmm0 # sched: [8:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_maxss:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vmaxss %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; HASWELL-NEXT:    vmaxss (%rdi), %xmm0, %xmm0 # sched: [8:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_maxss:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    maxss %xmm1, %xmm0 # sched: [3:1.00]
-; BROADWELL-SSE-NEXT:    maxss (%rdi), %xmm0 # sched: [8:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_maxss:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vmaxss %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    vmaxss (%rdi), %xmm0, %xmm0 # sched: [8:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_maxss:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    maxss %xmm1, %xmm0 # sched: [4:0.50]
-; SKYLAKE-SSE-NEXT:    maxss (%rdi), %xmm0 # sched: [9:0.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_maxss:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vmaxss %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    vmaxss (%rdi), %xmm0, %xmm0 # sched: [9:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_maxss:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    maxss %xmm1, %xmm0 # sched: [4:0.50]
-; SKX-SSE-NEXT:    maxss (%rdi), %xmm0 # sched: [9:0.50]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_maxss:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmaxss %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKX-NEXT:    vmaxss (%rdi), %xmm0, %xmm0 # sched: [9:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_maxss:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    maxss %xmm1, %xmm0 # sched: [2:1.00]
-; BDVER2-SSE-NEXT:    maxss (%rdi), %xmm0 # sched: [7:1.00]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_maxss:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vmaxss %xmm1, %xmm0, %xmm0 # sched: [2:1.00]
-; BDVER2-NEXT:    vmaxss (%rdi), %xmm0, %xmm0 # sched: [7:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_maxss:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    maxss %xmm1, %xmm0 # sched: [2:1.00]
-; BTVER2-SSE-NEXT:    maxss (%rdi), %xmm0 # sched: [7:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_maxss:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vmaxss %xmm1, %xmm0, %xmm0 # sched: [2:1.00]
-; BTVER2-NEXT:    vmaxss (%rdi), %xmm0, %xmm0 # sched: [7:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_maxss:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    maxss %xmm1, %xmm0 # sched: [3:1.00]
-; ZNVER1-SSE-NEXT:    maxss (%rdi), %xmm0 # sched: [10:1.00]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_maxss:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vmaxss %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    vmaxss (%rdi), %xmm0, %xmm0 # sched: [10:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <4 x float> @llvm.x86.sse.max.ss(<4 x float> %a0, <4 x float> %a1)
-  %2 = load <4 x float>, <4 x float> *%a2, align 16
-  %3 = call <4 x float> @llvm.x86.sse.max.ss(<4 x float> %1, <4 x float> %2)
-  ret <4 x float> %3
-}
-declare <4 x float> @llvm.x86.sse.max.ss(<4 x float>, <4 x float>) nounwind readnone
-
-define <4 x float> @test_minps(<4 x float> %a0, <4 x float> %a1, <4 x float> *%a2) {
-; GENERIC-LABEL: test_minps:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    minps %xmm1, %xmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    minps (%rdi), %xmm0 # sched: [9:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_minps:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    minps %xmm1, %xmm0 # sched: [5:5.00]
-; ATOM-NEXT:    minps (%rdi), %xmm0 # sched: [5:5.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_minps:
-; SLM:       # %bb.0:
-; SLM-NEXT:    minps %xmm1, %xmm0 # sched: [3:1.00]
-; SLM-NEXT:    minps (%rdi), %xmm0 # sched: [6:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_minps:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    minps %xmm1, %xmm0 # sched: [3:1.00]
-; SANDY-SSE-NEXT:    minps (%rdi), %xmm0 # sched: [9:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_minps:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vminps %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; SANDY-NEXT:    vminps (%rdi), %xmm0, %xmm0 # sched: [9:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_minps:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    minps %xmm1, %xmm0 # sched: [3:1.00]
-; HASWELL-SSE-NEXT:    minps (%rdi), %xmm0 # sched: [9:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_minps:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vminps %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; HASWELL-NEXT:    vminps (%rdi), %xmm0, %xmm0 # sched: [9:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_minps:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    minps %xmm1, %xmm0 # sched: [3:1.00]
-; BROADWELL-SSE-NEXT:    minps (%rdi), %xmm0 # sched: [8:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_minps:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vminps %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    vminps (%rdi), %xmm0, %xmm0 # sched: [8:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_minps:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    minps %xmm1, %xmm0 # sched: [4:0.50]
-; SKYLAKE-SSE-NEXT:    minps (%rdi), %xmm0 # sched: [10:0.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_minps:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vminps %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    vminps (%rdi), %xmm0, %xmm0 # sched: [10:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_minps:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    minps %xmm1, %xmm0 # sched: [4:0.50]
-; SKX-SSE-NEXT:    minps (%rdi), %xmm0 # sched: [10:0.50]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_minps:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vminps %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKX-NEXT:    vminps (%rdi), %xmm0, %xmm0 # sched: [10:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_minps:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    minps %xmm1, %xmm0 # sched: [2:1.00]
-; BDVER2-SSE-NEXT:    minps (%rdi), %xmm0 # sched: [7:1.00]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_minps:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vminps %xmm1, %xmm0, %xmm0 # sched: [2:1.00]
-; BDVER2-NEXT:    vminps (%rdi), %xmm0, %xmm0 # sched: [7:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_minps:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    minps %xmm1, %xmm0 # sched: [2:1.00]
-; BTVER2-SSE-NEXT:    minps (%rdi), %xmm0 # sched: [7:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_minps:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vminps %xmm1, %xmm0, %xmm0 # sched: [2:1.00]
-; BTVER2-NEXT:    vminps (%rdi), %xmm0, %xmm0 # sched: [7:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_minps:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    minps %xmm1, %xmm0 # sched: [3:1.00]
-; ZNVER1-SSE-NEXT:    minps (%rdi), %xmm0 # sched: [10:1.00]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_minps:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vminps %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    vminps (%rdi), %xmm0, %xmm0 # sched: [10:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <4 x float> @llvm.x86.sse.min.ps(<4 x float> %a0, <4 x float> %a1)
-  %2 = load <4 x float>, <4 x float> *%a2, align 16
-  %3 = call <4 x float> @llvm.x86.sse.min.ps(<4 x float> %1, <4 x float> %2)
-  ret <4 x float> %3
-}
-declare <4 x float> @llvm.x86.sse.min.ps(<4 x float>, <4 x float>) nounwind readnone
-
-define <4 x float> @test_minss(<4 x float> %a0, <4 x float> %a1, <4 x float> *%a2) {
-; GENERIC-LABEL: test_minss:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    minss %xmm1, %xmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    minss (%rdi), %xmm0 # sched: [9:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_minss:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    minss %xmm1, %xmm0 # sched: [5:5.00]
-; ATOM-NEXT:    minss (%rdi), %xmm0 # sched: [5:5.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_minss:
-; SLM:       # %bb.0:
-; SLM-NEXT:    minss %xmm1, %xmm0 # sched: [3:1.00]
-; SLM-NEXT:    minss (%rdi), %xmm0 # sched: [6:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_minss:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    minss %xmm1, %xmm0 # sched: [3:1.00]
-; SANDY-SSE-NEXT:    minss (%rdi), %xmm0 # sched: [9:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_minss:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vminss %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; SANDY-NEXT:    vminss (%rdi), %xmm0, %xmm0 # sched: [9:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_minss:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    minss %xmm1, %xmm0 # sched: [3:1.00]
-; HASWELL-SSE-NEXT:    minss (%rdi), %xmm0 # sched: [8:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_minss:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vminss %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; HASWELL-NEXT:    vminss (%rdi), %xmm0, %xmm0 # sched: [8:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_minss:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    minss %xmm1, %xmm0 # sched: [3:1.00]
-; BROADWELL-SSE-NEXT:    minss (%rdi), %xmm0 # sched: [8:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_minss:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vminss %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    vminss (%rdi), %xmm0, %xmm0 # sched: [8:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_minss:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    minss %xmm1, %xmm0 # sched: [4:0.50]
-; SKYLAKE-SSE-NEXT:    minss (%rdi), %xmm0 # sched: [9:0.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_minss:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vminss %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    vminss (%rdi), %xmm0, %xmm0 # sched: [9:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_minss:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    minss %xmm1, %xmm0 # sched: [4:0.50]
-; SKX-SSE-NEXT:    minss (%rdi), %xmm0 # sched: [9:0.50]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_minss:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vminss %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKX-NEXT:    vminss (%rdi), %xmm0, %xmm0 # sched: [9:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_minss:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    minss %xmm1, %xmm0 # sched: [2:1.00]
-; BDVER2-SSE-NEXT:    minss (%rdi), %xmm0 # sched: [7:1.00]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_minss:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vminss %xmm1, %xmm0, %xmm0 # sched: [2:1.00]
-; BDVER2-NEXT:    vminss (%rdi), %xmm0, %xmm0 # sched: [7:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_minss:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    minss %xmm1, %xmm0 # sched: [2:1.00]
-; BTVER2-SSE-NEXT:    minss (%rdi), %xmm0 # sched: [7:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_minss:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vminss %xmm1, %xmm0, %xmm0 # sched: [2:1.00]
-; BTVER2-NEXT:    vminss (%rdi), %xmm0, %xmm0 # sched: [7:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_minss:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    minss %xmm1, %xmm0 # sched: [3:1.00]
-; ZNVER1-SSE-NEXT:    minss (%rdi), %xmm0 # sched: [10:1.00]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_minss:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vminss %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    vminss (%rdi), %xmm0, %xmm0 # sched: [10:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <4 x float> @llvm.x86.sse.min.ss(<4 x float> %a0, <4 x float> %a1)
-  %2 = load <4 x float>, <4 x float> *%a2, align 16
-  %3 = call <4 x float> @llvm.x86.sse.min.ss(<4 x float> %1, <4 x float> %2)
-  ret <4 x float> %3
-}
-declare <4 x float> @llvm.x86.sse.min.ss(<4 x float>, <4 x float>) nounwind readnone
-
-define void @test_movaps(<4 x float> *%a0, <4 x float> *%a1) {
-; GENERIC-LABEL: test_movaps:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    movaps (%rdi), %xmm0 # sched: [6:0.50]
-; GENERIC-NEXT:    addps %xmm0, %xmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    movaps %xmm0, (%rsi) # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_movaps:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    movaps (%rdi), %xmm0 # sched: [1:1.00]
-; ATOM-NEXT:    addps %xmm0, %xmm0 # sched: [5:5.00]
-; ATOM-NEXT:    movaps %xmm0, (%rsi) # sched: [1:1.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_movaps:
-; SLM:       # %bb.0:
-; SLM-NEXT:    movaps (%rdi), %xmm0 # sched: [3:1.00]
-; SLM-NEXT:    addps %xmm0, %xmm0 # sched: [3:1.00]
-; SLM-NEXT:    movaps %xmm0, (%rsi) # sched: [1:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_movaps:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    movaps (%rdi), %xmm0 # sched: [6:0.50]
-; SANDY-SSE-NEXT:    addps %xmm0, %xmm0 # sched: [3:1.00]
-; SANDY-SSE-NEXT:    movaps %xmm0, (%rsi) # sched: [1:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_movaps:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vmovaps (%rdi), %xmm0 # sched: [6:0.50]
-; SANDY-NEXT:    vaddps %xmm0, %xmm0, %xmm0 # sched: [3:1.00]
-; SANDY-NEXT:    vmovaps %xmm0, (%rsi) # sched: [1:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_movaps:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    movaps (%rdi), %xmm0 # sched: [6:0.50]
-; HASWELL-SSE-NEXT:    addps %xmm0, %xmm0 # sched: [3:1.00]
-; HASWELL-SSE-NEXT:    movaps %xmm0, (%rsi) # sched: [1:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_movaps:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vmovaps (%rdi), %xmm0 # sched: [6:0.50]
-; HASWELL-NEXT:    vaddps %xmm0, %xmm0, %xmm0 # sched: [3:1.00]
-; HASWELL-NEXT:    vmovaps %xmm0, (%rsi) # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_movaps:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    movaps (%rdi), %xmm0 # sched: [5:0.50]
-; BROADWELL-SSE-NEXT:    addps %xmm0, %xmm0 # sched: [3:1.00]
-; BROADWELL-SSE-NEXT:    movaps %xmm0, (%rsi) # sched: [1:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_movaps:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vmovaps (%rdi), %xmm0 # sched: [5:0.50]
-; BROADWELL-NEXT:    vaddps %xmm0, %xmm0, %xmm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    vmovaps %xmm0, (%rsi) # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_movaps:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    movaps (%rdi), %xmm0 # sched: [6:0.50]
-; SKYLAKE-SSE-NEXT:    addps %xmm0, %xmm0 # sched: [4:0.50]
-; SKYLAKE-SSE-NEXT:    movaps %xmm0, (%rsi) # sched: [1:1.00]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_movaps:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vmovaps (%rdi), %xmm0 # sched: [6:0.50]
-; SKYLAKE-NEXT:    vaddps %xmm0, %xmm0, %xmm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    vmovaps %xmm0, (%rsi) # sched: [1:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_movaps:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    movaps (%rdi), %xmm0 # sched: [6:0.50]
-; SKX-SSE-NEXT:    addps %xmm0, %xmm0 # sched: [4:0.50]
-; SKX-SSE-NEXT:    movaps %xmm0, (%rsi) # sched: [1:1.00]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_movaps:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovaps (%rdi), %xmm0 # sched: [6:0.50]
-; SKX-NEXT:    vaddps %xmm0, %xmm0, %xmm0 # sched: [4:0.50]
-; SKX-NEXT:    vmovaps %xmm0, (%rsi) # sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_movaps:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    movaps (%rdi), %xmm0 # sched: [5:0.50]
-; BDVER2-SSE-NEXT:    addps %xmm0, %xmm0 # sched: [5:1.00]
-; BDVER2-SSE-NEXT:    movaps %xmm0, (%rsi) # sched: [1:1.00]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_movaps:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vmovaps (%rdi), %xmm0 # sched: [5:0.50]
-; BDVER2-NEXT:    vaddps %xmm0, %xmm0, %xmm0 # sched: [5:1.00]
-; BDVER2-NEXT:    vmovaps %xmm0, (%rsi) # sched: [1:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_movaps:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    movaps (%rdi), %xmm0 # sched: [5:1.00]
-; BTVER2-SSE-NEXT:    addps %xmm0, %xmm0 # sched: [3:1.00]
-; BTVER2-SSE-NEXT:    movaps %xmm0, (%rsi) # sched: [1:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_movaps:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vmovaps (%rdi), %xmm0 # sched: [5:1.00]
-; BTVER2-NEXT:    vaddps %xmm0, %xmm0, %xmm0 # sched: [3:1.00]
-; BTVER2-NEXT:    vmovaps %xmm0, (%rsi) # sched: [1:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_movaps:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    movaps (%rdi), %xmm0 # sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    addps %xmm0, %xmm0 # sched: [3:1.00]
-; ZNVER1-SSE-NEXT:    movaps %xmm0, (%rsi) # sched: [1:0.50]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_movaps:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vmovaps (%rdi), %xmm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    vaddps %xmm0, %xmm0, %xmm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    vmovaps %xmm0, (%rsi) # sched: [1:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = load <4 x float>, <4 x float> *%a0, align 16
-  %2 = fadd <4 x float> %1, %1
-  store <4 x float> %2, <4 x float> *%a1, align 16
-  ret void
-}
-
-; TODO (v)movhlps
-
-define <4 x float> @test_movhlps(<4 x float> %a0, <4 x float> %a1) {
-; GENERIC-LABEL: test_movhlps:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    movhlps {{.*#+}} xmm0 = xmm1[1],xmm0[1] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_movhlps:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    movhlps {{.*#+}} xmm0 = xmm1[1],xmm0[1] sched: [1:1.00]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_movhlps:
-; SLM:       # %bb.0:
-; SLM-NEXT:    movhlps {{.*#+}} xmm0 = xmm1[1],xmm0[1] sched: [1:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_movhlps:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    movhlps {{.*#+}} xmm0 = xmm1[1],xmm0[1] sched: [1:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_movhlps:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vunpckhpd {{.*#+}} xmm0 = xmm1[1],xmm0[1] sched: [1:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_movhlps:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    movhlps {{.*#+}} xmm0 = xmm1[1],xmm0[1] sched: [1:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_movhlps:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vunpckhpd {{.*#+}} xmm0 = xmm1[1],xmm0[1] sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_movhlps:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    movhlps {{.*#+}} xmm0 = xmm1[1],xmm0[1] sched: [1:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_movhlps:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vunpckhpd {{.*#+}} xmm0 = xmm1[1],xmm0[1] sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_movhlps:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    movhlps {{.*#+}} xmm0 = xmm1[1],xmm0[1] sched: [1:1.00]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_movhlps:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vunpckhpd {{.*#+}} xmm0 = xmm1[1],xmm0[1] sched: [1:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_movhlps:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    movhlps {{.*#+}} xmm0 = xmm1[1],xmm0[1] sched: [1:1.00]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_movhlps:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vunpckhpd {{.*#+}} xmm0 = xmm1[1],xmm0[1] sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_movhlps:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    movhlps {{.*#+}} xmm0 = xmm1[1],xmm0[1] sched: [2:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_movhlps:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vunpckhpd {{.*#+}} xmm0 = xmm1[1],xmm0[1] sched: [2:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_movhlps:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    movhlps {{.*#+}} xmm0 = xmm1[1],xmm0[1] sched: [1:0.50]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_movhlps:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vunpckhpd {{.*#+}} xmm0 = xmm1[1],xmm0[1] sched: [1:0.50]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_movhlps:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    movhlps {{.*#+}} xmm0 = xmm1[1],xmm0[1] sched: [1:0.50]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_movhlps:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vunpckhpd {{.*#+}} xmm0 = xmm1[1],xmm0[1] sched: [1:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = shufflevector <4 x float> %a0, <4 x float> %a1, <4 x i32> <i32 6, i32 7, i32 2, i32 3>
-  ret <4 x float> %1
-}
-
-; TODO (v)movhps
-
-define <4 x float> @test_movhps(<4 x float> %a0, <4 x float> %a1, x86_mmx *%a2) {
-; GENERIC-LABEL: test_movhps:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    movhpd {{.*#+}} xmm1 = xmm1[0],mem[0] sched: [7:1.00]
-; GENERIC-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    movhps %xmm0, (%rdi) # sched: [1:1.00]
-; GENERIC-NEXT:    movaps %xmm1, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_movhps:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    movaps %xmm0, %xmm2 # sched: [1:0.50]
-; ATOM-NEXT:    movhpd {{.*#+}} xmm1 = xmm1[0],mem[0] sched: [1:1.00]
-; ATOM-NEXT:    addps %xmm1, %xmm2 # sched: [5:5.00]
-; ATOM-NEXT:    movaps %xmm1, %xmm0 # sched: [1:0.50]
-; ATOM-NEXT:    movhps %xmm2, (%rdi) # sched: [1:1.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_movhps:
-; SLM:       # %bb.0:
-; SLM-NEXT:    movhpd {{.*#+}} xmm1 = xmm1[0],mem[0] sched: [4:1.00]
-; SLM-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; SLM-NEXT:    movhps %xmm0, (%rdi) # sched: [1:1.00]
-; SLM-NEXT:    movaps %xmm1, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_movhps:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    movhpd {{.*#+}} xmm1 = xmm1[0],mem[0] sched: [7:1.00]
-; SANDY-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; SANDY-SSE-NEXT:    movhps %xmm0, (%rdi) # sched: [1:1.00]
-; SANDY-SSE-NEXT:    movaps %xmm1, %xmm0 # sched: [1:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_movhps:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vmovhpd {{.*#+}} xmm1 = xmm1[0],mem[0] sched: [7:1.00]
-; SANDY-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; SANDY-NEXT:    vmovhpd %xmm0, (%rdi) # sched: [1:1.00]
-; SANDY-NEXT:    vmovaps %xmm1, %xmm0 # sched: [1:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_movhps:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    movhpd {{.*#+}} xmm1 = xmm1[0],mem[0] sched: [6:1.00]
-; HASWELL-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; HASWELL-SSE-NEXT:    movhps %xmm0, (%rdi) # sched: [1:1.00]
-; HASWELL-SSE-NEXT:    movaps %xmm1, %xmm0 # sched: [1:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_movhps:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vmovhpd {{.*#+}} xmm1 = xmm1[0],mem[0] sched: [6:1.00]
-; HASWELL-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; HASWELL-NEXT:    vmovhpd %xmm0, (%rdi) # sched: [1:1.00]
-; HASWELL-NEXT:    vmovaps %xmm1, %xmm0 # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_movhps:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    movhpd {{.*#+}} xmm1 = xmm1[0],mem[0] sched: [6:1.00]
-; BROADWELL-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; BROADWELL-SSE-NEXT:    movhps %xmm0, (%rdi) # sched: [1:1.00]
-; BROADWELL-SSE-NEXT:    movaps %xmm1, %xmm0 # sched: [1:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_movhps:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vmovhpd {{.*#+}} xmm1 = xmm1[0],mem[0] sched: [6:1.00]
-; BROADWELL-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    vmovhpd %xmm0, (%rdi) # sched: [1:1.00]
-; BROADWELL-NEXT:    vmovaps %xmm1, %xmm0 # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_movhps:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    movhpd {{.*#+}} xmm1 = xmm1[0],mem[0] sched: [6:1.00]
-; SKYLAKE-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [4:0.50]
-; SKYLAKE-SSE-NEXT:    movhps %xmm0, (%rdi) # sched: [1:1.00]
-; SKYLAKE-SSE-NEXT:    movaps %xmm1, %xmm0 # sched: [1:0.33]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_movhps:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vmovhpd {{.*#+}} xmm1 = xmm1[0],mem[0] sched: [6:1.00]
-; SKYLAKE-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    vmovhpd %xmm0, (%rdi) # sched: [1:1.00]
-; SKYLAKE-NEXT:    vmovaps %xmm1, %xmm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_movhps:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    movhpd {{.*#+}} xmm1 = xmm1[0],mem[0] sched: [6:1.00]
-; SKX-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [4:0.50]
-; SKX-SSE-NEXT:    movhps %xmm0, (%rdi) # sched: [1:1.00]
-; SKX-SSE-NEXT:    movaps %xmm1, %xmm0 # sched: [1:0.33]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_movhps:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovhpd {{.*#+}} xmm1 = xmm1[0],mem[0] sched: [6:1.00]
-; SKX-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKX-NEXT:    vmovhpd %xmm0, (%rdi) # sched: [1:1.00]
-; SKX-NEXT:    vmovaps %xmm1, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_movhps:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    movhpd {{.*#+}} xmm1 = xmm1[0],mem[0] sched: [7:0.50]
-; BDVER2-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [5:1.00]
-; BDVER2-SSE-NEXT:    movhps %xmm0, (%rdi) # sched: [2:1.00]
-; BDVER2-SSE-NEXT:    movaps %xmm1, %xmm0 # sched: [1:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_movhps:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vmovhpd {{.*#+}} xmm1 = xmm1[0],mem[0] sched: [7:0.50]
-; BDVER2-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [5:1.00]
-; BDVER2-NEXT:    vmovhpd %xmm0, (%rdi) # sched: [2:1.00]
-; BDVER2-NEXT:    vmovaps %xmm1, %xmm0 # sched: [1:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_movhps:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    movhpd {{.*#+}} xmm1 = xmm1[0],mem[0] sched: [6:1.00]
-; BTVER2-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; BTVER2-SSE-NEXT:    movhps %xmm0, (%rdi) # sched: [2:1.00]
-; BTVER2-SSE-NEXT:    movaps %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_movhps:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vmovhpd {{.*#+}} xmm1 = xmm1[0],mem[0] sched: [6:1.00]
-; BTVER2-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; BTVER2-NEXT:    vmovhpd %xmm0, (%rdi) # sched: [2:1.00]
-; BTVER2-NEXT:    vmovaps %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_movhps:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    movhpd {{.*#+}} xmm1 = xmm1[0],mem[0] sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; ZNVER1-SSE-NEXT:    movhps %xmm0, (%rdi) # sched: [1:0.50]
-; ZNVER1-SSE-NEXT:    movaps %xmm1, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_movhps:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vmovhpd {{.*#+}} xmm1 = xmm1[0],mem[0] sched: [8:0.50]
-; ZNVER1-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    vmovhpd %xmm0, (%rdi) # sched: [1:0.50]
-; ZNVER1-NEXT:    vmovaps %xmm1, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = bitcast x86_mmx* %a2 to <2 x float>*
-  %2 = load <2 x float>, <2 x float> *%1, align 8
-  %3 = shufflevector <2 x float> %2, <2 x float> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-  %4 = shufflevector <4 x float> %a1, <4 x float> %3, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
-  %5 = fadd <4 x float> %a0, %4
-  %6 = shufflevector <4 x float> %5, <4 x float> undef, <2 x i32> <i32 2, i32 3>
-  store <2 x float> %6, <2 x float>* %1
-  ret <4 x float> %4
-}
-
-; TODO (v)movlhps
-
-define <4 x float> @test_movlhps(<4 x float> %a0, <4 x float> %a1) {
-; GENERIC-LABEL: test_movlhps:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    movlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0] sched: [1:1.00]
-; GENERIC-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_movlhps:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    movlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0] sched: [1:1.00]
-; ATOM-NEXT:    addps %xmm1, %xmm0 # sched: [5:5.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_movlhps:
-; SLM:       # %bb.0:
-; SLM-NEXT:    movlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0] sched: [1:1.00]
-; SLM-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_movlhps:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    movlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0] sched: [1:1.00]
-; SANDY-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_movlhps:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0] sched: [1:1.00]
-; SANDY-NEXT:    vaddps %xmm0, %xmm1, %xmm0 # sched: [3:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_movlhps:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    movlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0] sched: [1:1.00]
-; HASWELL-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_movlhps:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0] sched: [1:1.00]
-; HASWELL-NEXT:    vaddps %xmm0, %xmm1, %xmm0 # sched: [3:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_movlhps:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    movlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0] sched: [1:1.00]
-; BROADWELL-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_movlhps:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0] sched: [1:1.00]
-; BROADWELL-NEXT:    vaddps %xmm0, %xmm1, %xmm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_movlhps:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    movlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0] sched: [1:1.00]
-; SKYLAKE-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [4:0.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_movlhps:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0] sched: [1:1.00]
-; SKYLAKE-NEXT:    vaddps %xmm0, %xmm1, %xmm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_movlhps:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    movlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0] sched: [1:1.00]
-; SKX-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [4:0.50]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_movlhps:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0] sched: [1:1.00]
-; SKX-NEXT:    vaddps %xmm0, %xmm1, %xmm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_movlhps:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    movlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0] sched: [2:0.50]
-; BDVER2-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [5:1.00]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_movlhps:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0] sched: [2:0.50]
-; BDVER2-NEXT:    vaddps %xmm0, %xmm1, %xmm0 # sched: [5:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_movlhps:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    movlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0] sched: [1:0.50]
-; BTVER2-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_movlhps:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0] sched: [1:0.50]
-; BTVER2-NEXT:    vaddps %xmm0, %xmm1, %xmm0 # sched: [3:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_movlhps:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    movlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0] sched: [1:0.50]
-; ZNVER1-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_movlhps:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0] sched: [1:0.50]
-; ZNVER1-NEXT:    vaddps %xmm0, %xmm1, %xmm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = shufflevector <4 x float> %a0, <4 x float> %a1, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
-  %2 = fadd <4 x float> %a1, %1
-  ret <4 x float> %2
-}
-
-define <4 x float> @test_movlps(<4 x float> %a0, <4 x float> %a1, x86_mmx *%a2) {
-; GENERIC-LABEL: test_movlps:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    movlpd {{.*#+}} xmm1 = mem[0],xmm1[1] sched: [7:1.00]
-; GENERIC-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    movlps %xmm0, (%rdi) # sched: [1:1.00]
-; GENERIC-NEXT:    movaps %xmm1, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_movlps:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    movaps %xmm0, %xmm2 # sched: [1:0.50]
-; ATOM-NEXT:    movlpd {{.*#+}} xmm1 = mem[0],xmm1[1] sched: [1:1.00]
-; ATOM-NEXT:    addps %xmm1, %xmm2 # sched: [5:5.00]
-; ATOM-NEXT:    movaps %xmm1, %xmm0 # sched: [1:0.50]
-; ATOM-NEXT:    movlps %xmm2, (%rdi) # sched: [1:1.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_movlps:
-; SLM:       # %bb.0:
-; SLM-NEXT:    movlpd {{.*#+}} xmm1 = mem[0],xmm1[1] sched: [4:1.00]
-; SLM-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; SLM-NEXT:    movlps %xmm0, (%rdi) # sched: [1:1.00]
-; SLM-NEXT:    movaps %xmm1, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_movlps:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    movlpd {{.*#+}} xmm1 = mem[0],xmm1[1] sched: [7:1.00]
-; SANDY-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; SANDY-SSE-NEXT:    movlps %xmm0, (%rdi) # sched: [1:1.00]
-; SANDY-SSE-NEXT:    movaps %xmm1, %xmm0 # sched: [1:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_movlps:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vmovlpd {{.*#+}} xmm1 = mem[0],xmm1[1] sched: [7:1.00]
-; SANDY-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; SANDY-NEXT:    vmovlps %xmm0, (%rdi) # sched: [1:1.00]
-; SANDY-NEXT:    vmovaps %xmm1, %xmm0 # sched: [1:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_movlps:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    movlpd {{.*#+}} xmm1 = mem[0],xmm1[1] sched: [6:1.00]
-; HASWELL-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; HASWELL-SSE-NEXT:    movlps %xmm0, (%rdi) # sched: [1:1.00]
-; HASWELL-SSE-NEXT:    movaps %xmm1, %xmm0 # sched: [1:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_movlps:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vmovlpd {{.*#+}} xmm1 = mem[0],xmm1[1] sched: [6:1.00]
-; HASWELL-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; HASWELL-NEXT:    vmovlps %xmm0, (%rdi) # sched: [1:1.00]
-; HASWELL-NEXT:    vmovaps %xmm1, %xmm0 # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_movlps:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    movlpd {{.*#+}} xmm1 = mem[0],xmm1[1] sched: [6:1.00]
-; BROADWELL-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; BROADWELL-SSE-NEXT:    movlps %xmm0, (%rdi) # sched: [1:1.00]
-; BROADWELL-SSE-NEXT:    movaps %xmm1, %xmm0 # sched: [1:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_movlps:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vmovlpd {{.*#+}} xmm1 = mem[0],xmm1[1] sched: [6:1.00]
-; BROADWELL-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    vmovlps %xmm0, (%rdi) # sched: [1:1.00]
-; BROADWELL-NEXT:    vmovaps %xmm1, %xmm0 # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_movlps:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    movlpd {{.*#+}} xmm1 = mem[0],xmm1[1] sched: [6:1.00]
-; SKYLAKE-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [4:0.50]
-; SKYLAKE-SSE-NEXT:    movlps %xmm0, (%rdi) # sched: [1:1.00]
-; SKYLAKE-SSE-NEXT:    movaps %xmm1, %xmm0 # sched: [1:0.33]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_movlps:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vmovlpd {{.*#+}} xmm1 = mem[0],xmm1[1] sched: [6:1.00]
-; SKYLAKE-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    vmovlps %xmm0, (%rdi) # sched: [1:1.00]
-; SKYLAKE-NEXT:    vmovaps %xmm1, %xmm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_movlps:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    movlpd {{.*#+}} xmm1 = mem[0],xmm1[1] sched: [6:1.00]
-; SKX-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [4:0.50]
-; SKX-SSE-NEXT:    movlps %xmm0, (%rdi) # sched: [1:1.00]
-; SKX-SSE-NEXT:    movaps %xmm1, %xmm0 # sched: [1:0.33]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_movlps:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovlpd {{.*#+}} xmm1 = mem[0],xmm1[1] sched: [6:1.00]
-; SKX-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKX-NEXT:    vmovlps %xmm0, (%rdi) # sched: [1:1.00]
-; SKX-NEXT:    vmovaps %xmm1, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_movlps:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    movlpd {{.*#+}} xmm1 = mem[0],xmm1[1] sched: [7:0.50]
-; BDVER2-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [5:1.00]
-; BDVER2-SSE-NEXT:    movlps %xmm0, (%rdi) # sched: [2:1.00]
-; BDVER2-SSE-NEXT:    movaps %xmm1, %xmm0 # sched: [1:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_movlps:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vmovlpd {{.*#+}} xmm1 = mem[0],xmm1[1] sched: [7:0.50]
-; BDVER2-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [5:1.00]
-; BDVER2-NEXT:    vmovlps %xmm0, (%rdi) # sched: [2:1.00]
-; BDVER2-NEXT:    vmovaps %xmm1, %xmm0 # sched: [1:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_movlps:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    movlpd {{.*#+}} xmm1 = mem[0],xmm1[1] sched: [6:1.00]
-; BTVER2-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; BTVER2-SSE-NEXT:    movlps %xmm0, (%rdi) # sched: [2:1.00]
-; BTVER2-SSE-NEXT:    movaps %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_movlps:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vmovlpd {{.*#+}} xmm1 = mem[0],xmm1[1] sched: [6:1.00]
-; BTVER2-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; BTVER2-NEXT:    vmovlps %xmm0, (%rdi) # sched: [2:1.00]
-; BTVER2-NEXT:    vmovaps %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_movlps:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    movlpd {{.*#+}} xmm1 = mem[0],xmm1[1] sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; ZNVER1-SSE-NEXT:    movlps %xmm0, (%rdi) # sched: [1:0.50]
-; ZNVER1-SSE-NEXT:    movaps %xmm1, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_movlps:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vmovlpd {{.*#+}} xmm1 = mem[0],xmm1[1] sched: [8:0.50]
-; ZNVER1-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    vmovlps %xmm0, (%rdi) # sched: [1:0.50]
-; ZNVER1-NEXT:    vmovaps %xmm1, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = bitcast x86_mmx* %a2 to <2 x float>*
-  %2 = load <2 x float>, <2 x float> *%1, align 8
-  %3 = shufflevector <2 x float> %2, <2 x float> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-  %4 = shufflevector <4 x float> %a1, <4 x float> %3, <4 x i32> <i32 4, i32 5, i32 2, i32 3>
-  %5 = fadd <4 x float> %a0, %4
-  %6 = shufflevector <4 x float> %5, <4 x float> undef, <2 x i32> <i32 0, i32 1>
-  store <2 x float> %6, <2 x float>* %1
-  ret <4 x float> %4
-}
-
-define i32 @test_movmskps(<4 x float> %a0) {
-; GENERIC-LABEL: test_movmskps:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    movmskps %xmm0, %eax # sched: [2:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_movmskps:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    movmskps %xmm0, %eax # sched: [3:3.00]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_movmskps:
-; SLM:       # %bb.0:
-; SLM-NEXT:    movmskps %xmm0, %eax # sched: [4:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_movmskps:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    movmskps %xmm0, %eax # sched: [2:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_movmskps:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vmovmskps %xmm0, %eax # sched: [2:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_movmskps:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    movmskps %xmm0, %eax # sched: [3:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_movmskps:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vmovmskps %xmm0, %eax # sched: [3:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_movmskps:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    movmskps %xmm0, %eax # sched: [3:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_movmskps:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vmovmskps %xmm0, %eax # sched: [3:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_movmskps:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    movmskps %xmm0, %eax # sched: [2:1.00]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_movmskps:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vmovmskps %xmm0, %eax # sched: [2:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_movmskps:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    movmskps %xmm0, %eax # sched: [2:1.00]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_movmskps:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovmskps %xmm0, %eax # sched: [2:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_movmskps:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    movmskps %xmm0, %eax # sched: [10:1.00]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_movmskps:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vmovmskps %xmm0, %eax # sched: [10:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_movmskps:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    movmskps %xmm0, %eax # sched: [3:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_movmskps:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vmovmskps %xmm0, %eax # sched: [3:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_movmskps:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    movmskps %xmm0, %eax # sched: [1:1.00]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_movmskps:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vmovmskps %xmm0, %eax # sched: [1:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call i32 @llvm.x86.sse.movmsk.ps(<4 x float> %a0)
-  ret i32 %1
-}
-declare i32 @llvm.x86.sse.movmsk.ps(<4 x float>) nounwind readnone
-
-define void @test_movntps(<4 x float> %a0, <4 x float> *%a1) {
-; GENERIC-LABEL: test_movntps:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    movntps %xmm0, (%rdi) # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_movntps:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    movntps %xmm0, (%rdi) # sched: [1:1.00]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_movntps:
-; SLM:       # %bb.0:
-; SLM-NEXT:    movntps %xmm0, (%rdi) # sched: [1:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_movntps:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    movntps %xmm0, (%rdi) # sched: [1:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_movntps:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vmovntps %xmm0, (%rdi) # sched: [1:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_movntps:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    movntps %xmm0, (%rdi) # sched: [1:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_movntps:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vmovntps %xmm0, (%rdi) # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_movntps:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    movntps %xmm0, (%rdi) # sched: [1:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_movntps:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vmovntps %xmm0, (%rdi) # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_movntps:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    movntps %xmm0, (%rdi) # sched: [1:1.00]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_movntps:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vmovntps %xmm0, (%rdi) # sched: [1:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_movntps:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    movntps %xmm0, (%rdi) # sched: [1:1.00]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_movntps:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovntps %xmm0, (%rdi) # sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_movntps:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    movntps %xmm0, (%rdi) # sched: [3:1.00]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_movntps:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vmovntps %xmm0, (%rdi) # sched: [3:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_movntps:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    movntps %xmm0, (%rdi) # sched: [3:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_movntps:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vmovntps %xmm0, (%rdi) # sched: [3:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_movntps:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    movntps %xmm0, (%rdi) # sched: [1:0.50]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_movntps:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vmovntps %xmm0, (%rdi) # sched: [1:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  store <4 x float> %a0, <4 x float> *%a1, align 16, !nontemporal !0
-  ret void
-}
-
-define void @test_movss_mem(float* %a0, float* %a1) {
-; GENERIC-LABEL: test_movss_mem:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero sched: [6:0.50]
-; GENERIC-NEXT:    addss %xmm0, %xmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    movss %xmm0, (%rsi) # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_movss_mem:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero sched: [1:1.00]
-; ATOM-NEXT:    addss %xmm0, %xmm0 # sched: [5:5.00]
-; ATOM-NEXT:    movss %xmm0, (%rsi) # sched: [1:1.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_movss_mem:
-; SLM:       # %bb.0:
-; SLM-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero sched: [3:1.00]
-; SLM-NEXT:    addss %xmm0, %xmm0 # sched: [3:1.00]
-; SLM-NEXT:    movss %xmm0, (%rsi) # sched: [1:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_movss_mem:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero sched: [6:0.50]
-; SANDY-SSE-NEXT:    addss %xmm0, %xmm0 # sched: [3:1.00]
-; SANDY-SSE-NEXT:    movss %xmm0, (%rsi) # sched: [1:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_movss_mem:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero sched: [6:0.50]
-; SANDY-NEXT:    vaddss %xmm0, %xmm0, %xmm0 # sched: [3:1.00]
-; SANDY-NEXT:    vmovss %xmm0, (%rsi) # sched: [1:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_movss_mem:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero sched: [5:0.50]
-; HASWELL-SSE-NEXT:    addss %xmm0, %xmm0 # sched: [3:1.00]
-; HASWELL-SSE-NEXT:    movss %xmm0, (%rsi) # sched: [1:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_movss_mem:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero sched: [5:0.50]
-; HASWELL-NEXT:    vaddss %xmm0, %xmm0, %xmm0 # sched: [3:1.00]
-; HASWELL-NEXT:    vmovss %xmm0, (%rsi) # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_movss_mem:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero sched: [5:0.50]
-; BROADWELL-SSE-NEXT:    addss %xmm0, %xmm0 # sched: [3:1.00]
-; BROADWELL-SSE-NEXT:    movss %xmm0, (%rsi) # sched: [1:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_movss_mem:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero sched: [5:0.50]
-; BROADWELL-NEXT:    vaddss %xmm0, %xmm0, %xmm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    vmovss %xmm0, (%rsi) # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_movss_mem:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero sched: [5:0.50]
-; SKYLAKE-SSE-NEXT:    addss %xmm0, %xmm0 # sched: [4:0.50]
-; SKYLAKE-SSE-NEXT:    movss %xmm0, (%rsi) # sched: [1:1.00]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_movss_mem:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero sched: [5:0.50]
-; SKYLAKE-NEXT:    vaddss %xmm0, %xmm0, %xmm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    vmovss %xmm0, (%rsi) # sched: [1:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_movss_mem:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero sched: [5:0.50]
-; SKX-SSE-NEXT:    addss %xmm0, %xmm0 # sched: [4:0.50]
-; SKX-SSE-NEXT:    movss %xmm0, (%rsi) # sched: [1:1.00]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_movss_mem:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero sched: [5:0.50]
-; SKX-NEXT:    vaddss %xmm0, %xmm0, %xmm0 # sched: [4:0.50]
-; SKX-NEXT:    vmovss %xmm0, (%rsi) # sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_movss_mem:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero sched: [5:0.50]
-; BDVER2-SSE-NEXT:    addss %xmm0, %xmm0 # sched: [5:1.00]
-; BDVER2-SSE-NEXT:    movss %xmm0, (%rsi) # sched: [2:1.00]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_movss_mem:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero sched: [5:0.50]
-; BDVER2-NEXT:    vaddss %xmm0, %xmm0, %xmm0 # sched: [5:1.00]
-; BDVER2-NEXT:    vmovss %xmm0, (%rsi) # sched: [2:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_movss_mem:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero sched: [5:1.00]
-; BTVER2-SSE-NEXT:    addss %xmm0, %xmm0 # sched: [3:1.00]
-; BTVER2-SSE-NEXT:    movss %xmm0, (%rsi) # sched: [2:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_movss_mem:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero sched: [5:1.00]
-; BTVER2-NEXT:    vaddss %xmm0, %xmm0, %xmm0 # sched: [3:1.00]
-; BTVER2-NEXT:    vmovss %xmm0, (%rsi) # sched: [2:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_movss_mem:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    addss %xmm0, %xmm0 # sched: [3:1.00]
-; ZNVER1-SSE-NEXT:    movss %xmm0, (%rsi) # sched: [1:0.50]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_movss_mem:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero sched: [8:0.50]
-; ZNVER1-NEXT:    vaddss %xmm0, %xmm0, %xmm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    vmovss %xmm0, (%rsi) # sched: [1:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = load float, float* %a0, align 1
-  %2 = fadd float %1, %1
-  store float %2, float *%a1, align 1
-  ret void
-}
-
-define <4 x float> @test_movss_reg(<4 x float> %a0, <4 x float> %a1) {
-; GENERIC-LABEL: test_movss_reg:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    movss {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3] sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_movss_reg:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    movss {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3] sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_movss_reg:
-; SLM:       # %bb.0:
-; SLM-NEXT:    movss {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3] sched: [1:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_movss_reg:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    movss {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3] sched: [1:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_movss_reg:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vblendps {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3] sched: [1:0.50]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_movss_reg:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    movss {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3] sched: [1:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_movss_reg:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vblendps {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3] sched: [1:0.33]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_movss_reg:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    movss {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3] sched: [1:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_movss_reg:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vblendps {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3] sched: [1:0.33]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_movss_reg:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    movss {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3] sched: [1:1.00]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_movss_reg:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vblendps {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3] sched: [1:0.33]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_movss_reg:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    movss {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3] sched: [1:1.00]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_movss_reg:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vblendps {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3] sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_movss_reg:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    movss {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3] sched: [2:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_movss_reg:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vblendps {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3] sched: [2:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_movss_reg:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    movss {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3] sched: [1:0.50]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_movss_reg:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vblendps {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3] sched: [1:0.50]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_movss_reg:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    movss {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3] sched: [1:0.50]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_movss_reg:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vblendps {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3] sched: [1:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = shufflevector <4 x float> %a0, <4 x float> %a1, <4 x i32> <i32 4, i32 1, i32 2, i32 3>
-  ret <4 x float> %1
-}
-
-define void @test_movups(<4 x float> *%a0, <4 x float> *%a1) {
-; GENERIC-LABEL: test_movups:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    movups (%rdi), %xmm0 # sched: [6:0.50]
-; GENERIC-NEXT:    addps %xmm0, %xmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    movups %xmm0, (%rsi) # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_movups:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    movups (%rdi), %xmm0 # sched: [3:1.50]
-; ATOM-NEXT:    addps %xmm0, %xmm0 # sched: [5:5.00]
-; ATOM-NEXT:    movups %xmm0, (%rsi) # sched: [2:1.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_movups:
-; SLM:       # %bb.0:
-; SLM-NEXT:    movups (%rdi), %xmm0 # sched: [3:1.00]
-; SLM-NEXT:    addps %xmm0, %xmm0 # sched: [3:1.00]
-; SLM-NEXT:    movups %xmm0, (%rsi) # sched: [1:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_movups:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    movups (%rdi), %xmm0 # sched: [6:0.50]
-; SANDY-SSE-NEXT:    addps %xmm0, %xmm0 # sched: [3:1.00]
-; SANDY-SSE-NEXT:    movups %xmm0, (%rsi) # sched: [1:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_movups:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vmovups (%rdi), %xmm0 # sched: [6:0.50]
-; SANDY-NEXT:    vaddps %xmm0, %xmm0, %xmm0 # sched: [3:1.00]
-; SANDY-NEXT:    vmovups %xmm0, (%rsi) # sched: [1:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_movups:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    movups (%rdi), %xmm0 # sched: [6:0.50]
-; HASWELL-SSE-NEXT:    addps %xmm0, %xmm0 # sched: [3:1.00]
-; HASWELL-SSE-NEXT:    movups %xmm0, (%rsi) # sched: [1:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_movups:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vmovups (%rdi), %xmm0 # sched: [6:0.50]
-; HASWELL-NEXT:    vaddps %xmm0, %xmm0, %xmm0 # sched: [3:1.00]
-; HASWELL-NEXT:    vmovups %xmm0, (%rsi) # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_movups:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    movups (%rdi), %xmm0 # sched: [5:0.50]
-; BROADWELL-SSE-NEXT:    addps %xmm0, %xmm0 # sched: [3:1.00]
-; BROADWELL-SSE-NEXT:    movups %xmm0, (%rsi) # sched: [1:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_movups:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vmovups (%rdi), %xmm0 # sched: [5:0.50]
-; BROADWELL-NEXT:    vaddps %xmm0, %xmm0, %xmm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    vmovups %xmm0, (%rsi) # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_movups:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    movups (%rdi), %xmm0 # sched: [6:0.50]
-; SKYLAKE-SSE-NEXT:    addps %xmm0, %xmm0 # sched: [4:0.50]
-; SKYLAKE-SSE-NEXT:    movups %xmm0, (%rsi) # sched: [1:1.00]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_movups:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vmovups (%rdi), %xmm0 # sched: [6:0.50]
-; SKYLAKE-NEXT:    vaddps %xmm0, %xmm0, %xmm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    vmovups %xmm0, (%rsi) # sched: [1:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_movups:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    movups (%rdi), %xmm0 # sched: [6:0.50]
-; SKX-SSE-NEXT:    addps %xmm0, %xmm0 # sched: [4:0.50]
-; SKX-SSE-NEXT:    movups %xmm0, (%rsi) # sched: [1:1.00]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_movups:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovups (%rdi), %xmm0 # sched: [6:0.50]
-; SKX-NEXT:    vaddps %xmm0, %xmm0, %xmm0 # sched: [4:0.50]
-; SKX-NEXT:    vmovups %xmm0, (%rsi) # sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_movups:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    movups (%rdi), %xmm0 # sched: [5:0.50]
-; BDVER2-SSE-NEXT:    addps %xmm0, %xmm0 # sched: [5:1.00]
-; BDVER2-SSE-NEXT:    movups %xmm0, (%rsi) # sched: [1:1.00]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_movups:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vmovups (%rdi), %xmm0 # sched: [5:0.50]
-; BDVER2-NEXT:    vaddps %xmm0, %xmm0, %xmm0 # sched: [5:1.00]
-; BDVER2-NEXT:    vmovups %xmm0, (%rsi) # sched: [1:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_movups:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    movups (%rdi), %xmm0 # sched: [5:1.00]
-; BTVER2-SSE-NEXT:    addps %xmm0, %xmm0 # sched: [3:1.00]
-; BTVER2-SSE-NEXT:    movups %xmm0, (%rsi) # sched: [1:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_movups:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vmovups (%rdi), %xmm0 # sched: [5:1.00]
-; BTVER2-NEXT:    vaddps %xmm0, %xmm0, %xmm0 # sched: [3:1.00]
-; BTVER2-NEXT:    vmovups %xmm0, (%rsi) # sched: [1:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_movups:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    movups (%rdi), %xmm0 # sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    addps %xmm0, %xmm0 # sched: [3:1.00]
-; ZNVER1-SSE-NEXT:    movups %xmm0, (%rsi) # sched: [1:0.50]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_movups:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vmovups (%rdi), %xmm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    vaddps %xmm0, %xmm0, %xmm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    vmovups %xmm0, (%rsi) # sched: [1:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = load <4 x float>, <4 x float> *%a0, align 1
-  %2 = fadd <4 x float> %1, %1
-  store <4 x float> %2, <4 x float> *%a1, align 1
-  ret void
-}
-
-define <4 x float> @test_mulps(<4 x float> %a0, <4 x float> %a1, <4 x float> *%a2) {
-; GENERIC-LABEL: test_mulps:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    mulps %xmm1, %xmm0 # sched: [5:1.00]
-; GENERIC-NEXT:    mulps (%rdi), %xmm0 # sched: [11:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_mulps:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    mulps %xmm1, %xmm0 # sched: [5:5.00]
-; ATOM-NEXT:    mulps (%rdi), %xmm0 # sched: [5:5.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_mulps:
-; SLM:       # %bb.0:
-; SLM-NEXT:    mulps %xmm1, %xmm0 # sched: [5:2.00]
-; SLM-NEXT:    mulps (%rdi), %xmm0 # sched: [8:2.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_mulps:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    mulps %xmm1, %xmm0 # sched: [5:1.00]
-; SANDY-SSE-NEXT:    mulps (%rdi), %xmm0 # sched: [11:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_mulps:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vmulps %xmm1, %xmm0, %xmm0 # sched: [5:1.00]
-; SANDY-NEXT:    vmulps (%rdi), %xmm0, %xmm0 # sched: [11:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_mulps:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    mulps %xmm1, %xmm0 # sched: [5:0.50]
-; HASWELL-SSE-NEXT:    mulps (%rdi), %xmm0 # sched: [11:0.50]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_mulps:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vmulps %xmm1, %xmm0, %xmm0 # sched: [5:0.50]
-; HASWELL-NEXT:    vmulps (%rdi), %xmm0, %xmm0 # sched: [11:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_mulps:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    mulps %xmm1, %xmm0 # sched: [3:0.50]
-; BROADWELL-SSE-NEXT:    mulps (%rdi), %xmm0 # sched: [8:0.50]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_mulps:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vmulps %xmm1, %xmm0, %xmm0 # sched: [3:0.50]
-; BROADWELL-NEXT:    vmulps (%rdi), %xmm0, %xmm0 # sched: [8:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_mulps:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    mulps %xmm1, %xmm0 # sched: [4:0.50]
-; SKYLAKE-SSE-NEXT:    mulps (%rdi), %xmm0 # sched: [10:0.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_mulps:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vmulps %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    vmulps (%rdi), %xmm0, %xmm0 # sched: [10:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_mulps:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    mulps %xmm1, %xmm0 # sched: [4:0.50]
-; SKX-SSE-NEXT:    mulps (%rdi), %xmm0 # sched: [10:0.50]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_mulps:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmulps %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKX-NEXT:    vmulps (%rdi), %xmm0, %xmm0 # sched: [10:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_mulps:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    mulps %xmm1, %xmm0 # sched: [5:1.00]
-; BDVER2-SSE-NEXT:    mulps (%rdi), %xmm0 # sched: [10:1.00]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_mulps:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vmulps %xmm1, %xmm0, %xmm0 # sched: [5:1.00]
-; BDVER2-NEXT:    vmulps (%rdi), %xmm0, %xmm0 # sched: [10:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_mulps:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    mulps %xmm1, %xmm0 # sched: [2:1.00]
-; BTVER2-SSE-NEXT:    mulps (%rdi), %xmm0 # sched: [7:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_mulps:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vmulps %xmm1, %xmm0, %xmm0 # sched: [2:1.00]
-; BTVER2-NEXT:    vmulps (%rdi), %xmm0, %xmm0 # sched: [7:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_mulps:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    mulps %xmm1, %xmm0 # sched: [3:0.50]
-; ZNVER1-SSE-NEXT:    mulps (%rdi), %xmm0 # sched: [10:0.50]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_mulps:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vmulps %xmm1, %xmm0, %xmm0 # sched: [3:0.50]
-; ZNVER1-NEXT:    vmulps (%rdi), %xmm0, %xmm0 # sched: [10:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = fmul <4 x float> %a0, %a1
-  %2 = load <4 x float>, <4 x float> *%a2, align 16
-  %3 = fmul <4 x float> %1, %2
-  ret <4 x float> %3
-}
-
-define float @test_mulss(float %a0, float %a1, float *%a2) {
-; GENERIC-LABEL: test_mulss:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    mulss %xmm1, %xmm0 # sched: [5:1.00]
-; GENERIC-NEXT:    mulss (%rdi), %xmm0 # sched: [11:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_mulss:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    mulss %xmm1, %xmm0 # sched: [4:4.00]
-; ATOM-NEXT:    mulss (%rdi), %xmm0 # sched: [4:4.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_mulss:
-; SLM:       # %bb.0:
-; SLM-NEXT:    mulss %xmm1, %xmm0 # sched: [5:2.00]
-; SLM-NEXT:    mulss (%rdi), %xmm0 # sched: [8:2.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_mulss:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    mulss %xmm1, %xmm0 # sched: [5:1.00]
-; SANDY-SSE-NEXT:    mulss (%rdi), %xmm0 # sched: [11:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_mulss:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vmulss %xmm1, %xmm0, %xmm0 # sched: [5:1.00]
-; SANDY-NEXT:    vmulss (%rdi), %xmm0, %xmm0 # sched: [11:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_mulss:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    mulss %xmm1, %xmm0 # sched: [5:0.50]
-; HASWELL-SSE-NEXT:    mulss (%rdi), %xmm0 # sched: [10:0.50]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_mulss:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vmulss %xmm1, %xmm0, %xmm0 # sched: [5:0.50]
-; HASWELL-NEXT:    vmulss (%rdi), %xmm0, %xmm0 # sched: [10:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_mulss:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    mulss %xmm1, %xmm0 # sched: [3:0.50]
-; BROADWELL-SSE-NEXT:    mulss (%rdi), %xmm0 # sched: [8:0.50]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_mulss:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vmulss %xmm1, %xmm0, %xmm0 # sched: [3:0.50]
-; BROADWELL-NEXT:    vmulss (%rdi), %xmm0, %xmm0 # sched: [8:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_mulss:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    mulss %xmm1, %xmm0 # sched: [4:0.50]
-; SKYLAKE-SSE-NEXT:    mulss (%rdi), %xmm0 # sched: [9:0.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_mulss:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vmulss %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    vmulss (%rdi), %xmm0, %xmm0 # sched: [9:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_mulss:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    mulss %xmm1, %xmm0 # sched: [4:0.50]
-; SKX-SSE-NEXT:    mulss (%rdi), %xmm0 # sched: [9:0.50]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_mulss:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmulss %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKX-NEXT:    vmulss (%rdi), %xmm0, %xmm0 # sched: [9:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_mulss:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    mulss %xmm1, %xmm0 # sched: [5:1.00]
-; BDVER2-SSE-NEXT:    mulss (%rdi), %xmm0 # sched: [10:1.00]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_mulss:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vmulss %xmm1, %xmm0, %xmm0 # sched: [5:1.00]
-; BDVER2-NEXT:    vmulss (%rdi), %xmm0, %xmm0 # sched: [10:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_mulss:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    mulss %xmm1, %xmm0 # sched: [2:1.00]
-; BTVER2-SSE-NEXT:    mulss (%rdi), %xmm0 # sched: [7:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_mulss:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vmulss %xmm1, %xmm0, %xmm0 # sched: [2:1.00]
-; BTVER2-NEXT:    vmulss (%rdi), %xmm0, %xmm0 # sched: [7:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_mulss:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    mulss %xmm1, %xmm0 # sched: [3:0.50]
-; ZNVER1-SSE-NEXT:    mulss (%rdi), %xmm0 # sched: [10:0.50]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_mulss:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vmulss %xmm1, %xmm0, %xmm0 # sched: [3:0.50]
-; ZNVER1-NEXT:    vmulss (%rdi), %xmm0, %xmm0 # sched: [10:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = fmul float %a0, %a1
-  %2 = load float, float *%a2, align 4
-  %3 = fmul float %1, %2
-  ret float %3
-}
-
-define <4 x float> @test_orps(<4 x float> %a0, <4 x float> %a1, <4 x float> *%a2) {
-; GENERIC-LABEL: test_orps:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    orps %xmm1, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    orps (%rdi), %xmm0 # sched: [7:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_orps:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    orps %xmm1, %xmm0 # sched: [1:0.50]
-; ATOM-NEXT:    orps (%rdi), %xmm0 # sched: [1:1.00]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_orps:
-; SLM:       # %bb.0:
-; SLM-NEXT:    orps %xmm1, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    orps (%rdi), %xmm0 # sched: [4:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_orps:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    orps %xmm1, %xmm0 # sched: [1:1.00]
-; SANDY-SSE-NEXT:    orps (%rdi), %xmm0 # sched: [7:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_orps:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vorps %xmm1, %xmm0, %xmm0 # sched: [1:1.00]
-; SANDY-NEXT:    vorps (%rdi), %xmm0, %xmm0 # sched: [7:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_orps:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    orps %xmm1, %xmm0 # sched: [1:1.00]
-; HASWELL-SSE-NEXT:    orps (%rdi), %xmm0 # sched: [7:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_orps:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vorps %xmm1, %xmm0, %xmm0 # sched: [1:1.00]
-; HASWELL-NEXT:    vorps (%rdi), %xmm0, %xmm0 # sched: [7:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_orps:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    orps %xmm1, %xmm0 # sched: [1:1.00]
-; BROADWELL-SSE-NEXT:    orps (%rdi), %xmm0 # sched: [6:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_orps:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vorps %xmm1, %xmm0, %xmm0 # sched: [1:1.00]
-; BROADWELL-NEXT:    vorps (%rdi), %xmm0, %xmm0 # sched: [6:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_orps:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    orps %xmm1, %xmm0 # sched: [1:0.33]
-; SKYLAKE-SSE-NEXT:    orps (%rdi), %xmm0 # sched: [7:0.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_orps:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vorps %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    vorps (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_orps:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    orps %xmm1, %xmm0 # sched: [1:0.33]
-; SKX-SSE-NEXT:    orps (%rdi), %xmm0 # sched: [7:0.50]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_orps:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vorps %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    vorps (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_orps:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    orps %xmm1, %xmm0 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    orps (%rdi), %xmm0 # sched: [7:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_orps:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vorps %xmm1, %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    vorps (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_orps:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    orps %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    orps (%rdi), %xmm0 # sched: [6:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_orps:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vorps %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    vorps (%rdi), %xmm0, %xmm0 # sched: [6:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_orps:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    orps %xmm1, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    orps (%rdi), %xmm0 # sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_orps:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vorps %xmm1, %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vorps (%rdi), %xmm0, %xmm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = bitcast <4 x float> %a0 to <4 x i32>
-  %2 = bitcast <4 x float> %a1 to <4 x i32>
-  %3 = or <4 x i32> %1, %2
-  %4 = load <4 x float>, <4 x float> *%a2, align 16
-  %5 = bitcast <4 x float> %4 to <4 x i32>
-  %6 = or <4 x i32> %3, %5
-  %7 = bitcast <4 x i32> %6 to <4 x float>
-  ret <4 x float> %7
-}
-
-define void @test_prefetch(i8* %a0) optsize {
-; GENERIC-LABEL: test_prefetch:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    prefetchnta (%rdi) # sched: [5:0.50]
-; GENERIC-NEXT:    prefetcht0 (%rdi) # sched: [5:0.50]
-; GENERIC-NEXT:    prefetcht1 (%rdi) # sched: [5:0.50]
-; GENERIC-NEXT:    prefetcht2 (%rdi) # sched: [5:0.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_prefetch:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    prefetchnta (%rdi) # sched: [1:1.00]
-; ATOM-NEXT:    prefetcht0 (%rdi) # sched: [1:1.00]
-; ATOM-NEXT:    prefetcht1 (%rdi) # sched: [1:1.00]
-; ATOM-NEXT:    prefetcht2 (%rdi) # sched: [1:1.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_prefetch:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    prefetchnta (%rdi) # sched: [3:1.00]
-; SLM-NEXT:    prefetcht0 (%rdi) # sched: [3:1.00]
-; SLM-NEXT:    prefetcht1 (%rdi) # sched: [3:1.00]
-; SLM-NEXT:    prefetcht2 (%rdi) # sched: [3:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_prefetch:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    #APP
-; SANDY-SSE-NEXT:    prefetchnta (%rdi) # sched: [5:0.50]
-; SANDY-SSE-NEXT:    prefetcht0 (%rdi) # sched: [5:0.50]
-; SANDY-SSE-NEXT:    prefetcht1 (%rdi) # sched: [5:0.50]
-; SANDY-SSE-NEXT:    prefetcht2 (%rdi) # sched: [5:0.50]
-; SANDY-SSE-NEXT:    #NO_APP
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_prefetch:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    prefetchnta (%rdi) # sched: [5:0.50]
-; SANDY-NEXT:    prefetcht0 (%rdi) # sched: [5:0.50]
-; SANDY-NEXT:    prefetcht1 (%rdi) # sched: [5:0.50]
-; SANDY-NEXT:    prefetcht2 (%rdi) # sched: [5:0.50]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_prefetch:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    #APP
-; HASWELL-SSE-NEXT:    prefetchnta (%rdi) # sched: [5:0.50]
-; HASWELL-SSE-NEXT:    prefetcht0 (%rdi) # sched: [5:0.50]
-; HASWELL-SSE-NEXT:    prefetcht1 (%rdi) # sched: [5:0.50]
-; HASWELL-SSE-NEXT:    prefetcht2 (%rdi) # sched: [5:0.50]
-; HASWELL-SSE-NEXT:    #NO_APP
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_prefetch:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    prefetchnta (%rdi) # sched: [5:0.50]
-; HASWELL-NEXT:    prefetcht0 (%rdi) # sched: [5:0.50]
-; HASWELL-NEXT:    prefetcht1 (%rdi) # sched: [5:0.50]
-; HASWELL-NEXT:    prefetcht2 (%rdi) # sched: [5:0.50]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_prefetch:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    #APP
-; BROADWELL-SSE-NEXT:    prefetchnta (%rdi) # sched: [5:0.50]
-; BROADWELL-SSE-NEXT:    prefetcht0 (%rdi) # sched: [5:0.50]
-; BROADWELL-SSE-NEXT:    prefetcht1 (%rdi) # sched: [5:0.50]
-; BROADWELL-SSE-NEXT:    prefetcht2 (%rdi) # sched: [5:0.50]
-; BROADWELL-SSE-NEXT:    #NO_APP
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_prefetch:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    prefetchnta (%rdi) # sched: [5:0.50]
-; BROADWELL-NEXT:    prefetcht0 (%rdi) # sched: [5:0.50]
-; BROADWELL-NEXT:    prefetcht1 (%rdi) # sched: [5:0.50]
-; BROADWELL-NEXT:    prefetcht2 (%rdi) # sched: [5:0.50]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_prefetch:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    #APP
-; SKYLAKE-SSE-NEXT:    prefetchnta (%rdi) # sched: [5:0.50]
-; SKYLAKE-SSE-NEXT:    prefetcht0 (%rdi) # sched: [5:0.50]
-; SKYLAKE-SSE-NEXT:    prefetcht1 (%rdi) # sched: [5:0.50]
-; SKYLAKE-SSE-NEXT:    prefetcht2 (%rdi) # sched: [5:0.50]
-; SKYLAKE-SSE-NEXT:    #NO_APP
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_prefetch:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    prefetchnta (%rdi) # sched: [5:0.50]
-; SKYLAKE-NEXT:    prefetcht0 (%rdi) # sched: [5:0.50]
-; SKYLAKE-NEXT:    prefetcht1 (%rdi) # sched: [5:0.50]
-; SKYLAKE-NEXT:    prefetcht2 (%rdi) # sched: [5:0.50]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_prefetch:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    #APP
-; SKX-SSE-NEXT:    prefetchnta (%rdi) # sched: [5:0.50]
-; SKX-SSE-NEXT:    prefetcht0 (%rdi) # sched: [5:0.50]
-; SKX-SSE-NEXT:    prefetcht1 (%rdi) # sched: [5:0.50]
-; SKX-SSE-NEXT:    prefetcht2 (%rdi) # sched: [5:0.50]
-; SKX-SSE-NEXT:    #NO_APP
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_prefetch:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    prefetchnta (%rdi) # sched: [5:0.50]
-; SKX-NEXT:    prefetcht0 (%rdi) # sched: [5:0.50]
-; SKX-NEXT:    prefetcht1 (%rdi) # sched: [5:0.50]
-; SKX-NEXT:    prefetcht2 (%rdi) # sched: [5:0.50]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_prefetch:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    #APP
-; BDVER2-SSE-NEXT:    prefetchnta (%rdi) # sched: [5:0.50]
-; BDVER2-SSE-NEXT:    prefetcht0 (%rdi) # sched: [5:0.50]
-; BDVER2-SSE-NEXT:    prefetcht1 (%rdi) # sched: [5:0.50]
-; BDVER2-SSE-NEXT:    prefetcht2 (%rdi) # sched: [5:0.50]
-; BDVER2-SSE-NEXT:    #NO_APP
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_prefetch:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    prefetchnta (%rdi) # sched: [5:0.50]
-; BDVER2-NEXT:    prefetcht0 (%rdi) # sched: [5:0.50]
-; BDVER2-NEXT:    prefetcht1 (%rdi) # sched: [5:0.50]
-; BDVER2-NEXT:    prefetcht2 (%rdi) # sched: [5:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_prefetch:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    #APP
-; BTVER2-SSE-NEXT:    prefetchnta (%rdi) # sched: [5:1.00]
-; BTVER2-SSE-NEXT:    prefetcht0 (%rdi) # sched: [5:1.00]
-; BTVER2-SSE-NEXT:    prefetcht1 (%rdi) # sched: [5:1.00]
-; BTVER2-SSE-NEXT:    prefetcht2 (%rdi) # sched: [5:1.00]
-; BTVER2-SSE-NEXT:    #NO_APP
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_prefetch:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    prefetchnta (%rdi) # sched: [5:1.00]
-; BTVER2-NEXT:    prefetcht0 (%rdi) # sched: [5:1.00]
-; BTVER2-NEXT:    prefetcht1 (%rdi) # sched: [5:1.00]
-; BTVER2-NEXT:    prefetcht2 (%rdi) # sched: [5:1.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_prefetch:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    #APP
-; ZNVER1-SSE-NEXT:    prefetchnta (%rdi) # sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    prefetcht0 (%rdi) # sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    prefetcht1 (%rdi) # sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    prefetcht2 (%rdi) # sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    #NO_APP
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_prefetch:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    prefetchnta (%rdi) # sched: [8:0.50]
-; ZNVER1-NEXT:    prefetcht0 (%rdi) # sched: [8:0.50]
-; ZNVER1-NEXT:    prefetcht1 (%rdi) # sched: [8:0.50]
-; ZNVER1-NEXT:    prefetcht2 (%rdi) # sched: [8:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  call void asm sideeffect "prefetchnta $0 \0A\09 prefetcht0 $0 \0A\09 prefetcht1 $0 \0A\09 prefetcht2 $0", "*m"(i8 *%a0)
-  ret void
-}
-
-define <4 x float> @test_rcpps(<4 x float> %a0, <4 x float> *%a1) {
-; GENERIC-LABEL: test_rcpps:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    rcpps %xmm0, %xmm1 # sched: [5:1.00]
-; GENERIC-NEXT:    rcpps (%rdi), %xmm0 # sched: [11:1.00]
-; GENERIC-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_rcpps:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    rcpps (%rdi), %xmm1 # sched: [10:5.00]
-; ATOM-NEXT:    rcpps %xmm0, %xmm0 # sched: [9:4.50]
-; ATOM-NEXT:    addps %xmm0, %xmm1 # sched: [5:5.00]
-; ATOM-NEXT:    movaps %xmm1, %xmm0 # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_rcpps:
-; SLM:       # %bb.0:
-; SLM-NEXT:    rcpps (%rdi), %xmm1 # sched: [8:1.00]
-; SLM-NEXT:    rcpps %xmm0, %xmm0 # sched: [5:1.00]
-; SLM-NEXT:    addps %xmm0, %xmm1 # sched: [3:1.00]
-; SLM-NEXT:    movaps %xmm1, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_rcpps:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    rcpps %xmm0, %xmm1 # sched: [5:1.00]
-; SANDY-SSE-NEXT:    rcpps (%rdi), %xmm0 # sched: [11:1.00]
-; SANDY-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_rcpps:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vrcpps %xmm0, %xmm0 # sched: [5:1.00]
-; SANDY-NEXT:    vrcpps (%rdi), %xmm1 # sched: [11:1.00]
-; SANDY-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_rcpps:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    rcpps %xmm0, %xmm1 # sched: [5:1.00]
-; HASWELL-SSE-NEXT:    rcpps (%rdi), %xmm0 # sched: [11:1.00]
-; HASWELL-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_rcpps:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vrcpps %xmm0, %xmm0 # sched: [5:1.00]
-; HASWELL-NEXT:    vrcpps (%rdi), %xmm1 # sched: [11:1.00]
-; HASWELL-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_rcpps:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    rcpps %xmm0, %xmm1 # sched: [5:1.00]
-; BROADWELL-SSE-NEXT:    rcpps (%rdi), %xmm0 # sched: [10:1.00]
-; BROADWELL-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_rcpps:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vrcpps %xmm0, %xmm0 # sched: [5:1.00]
-; BROADWELL-NEXT:    vrcpps (%rdi), %xmm1 # sched: [10:1.00]
-; BROADWELL-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_rcpps:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    rcpps %xmm0, %xmm1 # sched: [4:1.00]
-; SKYLAKE-SSE-NEXT:    rcpps (%rdi), %xmm0 # sched: [10:1.00]
-; SKYLAKE-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [4:0.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_rcpps:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vrcpps %xmm0, %xmm0 # sched: [4:1.00]
-; SKYLAKE-NEXT:    vrcpps (%rdi), %xmm1 # sched: [10:1.00]
-; SKYLAKE-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_rcpps:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    rcpps %xmm0, %xmm1 # sched: [4:1.00]
-; SKX-SSE-NEXT:    rcpps (%rdi), %xmm0 # sched: [10:1.00]
-; SKX-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [4:0.50]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_rcpps:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vrcpps %xmm0, %xmm0 # sched: [4:1.00]
-; SKX-NEXT:    vrcpps (%rdi), %xmm1 # sched: [10:1.00]
-; SKX-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_rcpps:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    rcpps %xmm0, %xmm1 # sched: [5:1.00]
-; BDVER2-SSE-NEXT:    rcpps (%rdi), %xmm0 # sched: [10:1.00]
-; BDVER2-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [5:1.00]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_rcpps:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vrcpps (%rdi), %xmm1 # sched: [10:1.00]
-; BDVER2-NEXT:    vrcpps %xmm0, %xmm0 # sched: [5:1.00]
-; BDVER2-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [5:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_rcpps:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    rcpps %xmm0, %xmm1 # sched: [2:1.00]
-; BTVER2-SSE-NEXT:    rcpps (%rdi), %xmm0 # sched: [7:1.00]
-; BTVER2-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_rcpps:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vrcpps (%rdi), %xmm1 # sched: [7:1.00]
-; BTVER2-NEXT:    vrcpps %xmm0, %xmm0 # sched: [2:1.00]
-; BTVER2-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_rcpps:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    rcpps %xmm0, %xmm1 # sched: [5:0.50]
-; ZNVER1-SSE-NEXT:    rcpps (%rdi), %xmm0 # sched: [12:0.50]
-; ZNVER1-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_rcpps:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vrcpps (%rdi), %xmm1 # sched: [12:0.50]
-; ZNVER1-NEXT:    vrcpps %xmm0, %xmm0 # sched: [5:0.50]
-; ZNVER1-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <4 x float> @llvm.x86.sse.rcp.ps(<4 x float> %a0)
-  %2 = load <4 x float>, <4 x float> *%a1, align 16
-  %3 = call <4 x float> @llvm.x86.sse.rcp.ps(<4 x float> %2)
-  %4 = fadd <4 x float> %1, %3
-  ret <4 x float> %4
-}
-declare <4 x float> @llvm.x86.sse.rcp.ps(<4 x float>) nounwind readnone
-
-; TODO - rcpss_m
-
-define <4 x float> @test_rcpss(float %a0, float *%a1) {
-; GENERIC-LABEL: test_rcpss:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    rcpss %xmm0, %xmm0 # sched: [5:1.00]
-; GENERIC-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero sched: [6:0.50]
-; GENERIC-NEXT:    rcpss %xmm1, %xmm1 # sched: [5:1.00]
-; GENERIC-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_rcpss:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero sched: [1:1.00]
-; ATOM-NEXT:    rcpss %xmm0, %xmm0 # sched: [4:4.00]
-; ATOM-NEXT:    rcpss %xmm1, %xmm1 # sched: [4:4.00]
-; ATOM-NEXT:    addps %xmm1, %xmm0 # sched: [5:5.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_rcpss:
-; SLM:       # %bb.0:
-; SLM-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero sched: [3:1.00]
-; SLM-NEXT:    rcpss %xmm0, %xmm0 # sched: [5:1.00]
-; SLM-NEXT:    rcpss %xmm1, %xmm1 # sched: [5:1.00]
-; SLM-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_rcpss:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    rcpss %xmm0, %xmm0 # sched: [5:1.00]
-; SANDY-SSE-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero sched: [6:0.50]
-; SANDY-SSE-NEXT:    rcpss %xmm1, %xmm1 # sched: [5:1.00]
-; SANDY-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_rcpss:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vrcpss %xmm0, %xmm0, %xmm0 # sched: [5:1.00]
-; SANDY-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero sched: [6:0.50]
-; SANDY-NEXT:    vrcpss %xmm1, %xmm1, %xmm1 # sched: [5:1.00]
-; SANDY-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_rcpss:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    rcpss %xmm0, %xmm0 # sched: [5:1.00]
-; HASWELL-SSE-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero sched: [5:0.50]
-; HASWELL-SSE-NEXT:    rcpss %xmm1, %xmm1 # sched: [5:1.00]
-; HASWELL-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_rcpss:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vrcpss %xmm0, %xmm0, %xmm0 # sched: [5:1.00]
-; HASWELL-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero sched: [5:0.50]
-; HASWELL-NEXT:    vrcpss %xmm1, %xmm1, %xmm1 # sched: [5:1.00]
-; HASWELL-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_rcpss:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    rcpss %xmm0, %xmm0 # sched: [5:1.00]
-; BROADWELL-SSE-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero sched: [5:0.50]
-; BROADWELL-SSE-NEXT:    rcpss %xmm1, %xmm1 # sched: [5:1.00]
-; BROADWELL-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_rcpss:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vrcpss %xmm0, %xmm0, %xmm0 # sched: [5:1.00]
-; BROADWELL-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero sched: [5:0.50]
-; BROADWELL-NEXT:    vrcpss %xmm1, %xmm1, %xmm1 # sched: [5:1.00]
-; BROADWELL-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_rcpss:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    rcpss %xmm0, %xmm0 # sched: [4:1.00]
-; SKYLAKE-SSE-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero sched: [5:0.50]
-; SKYLAKE-SSE-NEXT:    rcpss %xmm1, %xmm1 # sched: [4:1.00]
-; SKYLAKE-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [4:0.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_rcpss:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vrcpss %xmm0, %xmm0, %xmm0 # sched: [4:1.00]
-; SKYLAKE-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero sched: [5:0.50]
-; SKYLAKE-NEXT:    vrcpss %xmm1, %xmm1, %xmm1 # sched: [4:1.00]
-; SKYLAKE-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_rcpss:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    rcpss %xmm0, %xmm0 # sched: [4:1.00]
-; SKX-SSE-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero sched: [5:0.50]
-; SKX-SSE-NEXT:    rcpss %xmm1, %xmm1 # sched: [4:1.00]
-; SKX-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [4:0.50]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_rcpss:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vrcpss %xmm0, %xmm0, %xmm0 # sched: [4:1.00]
-; SKX-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero sched: [5:0.50]
-; SKX-NEXT:    vrcpss %xmm1, %xmm1, %xmm1 # sched: [4:1.00]
-; SKX-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_rcpss:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero sched: [5:0.50]
-; BDVER2-SSE-NEXT:    rcpss %xmm0, %xmm0 # sched: [5:1.00]
-; BDVER2-SSE-NEXT:    rcpss %xmm1, %xmm1 # sched: [5:1.00]
-; BDVER2-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [5:1.00]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_rcpss:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero sched: [5:0.50]
-; BDVER2-NEXT:    vrcpss %xmm0, %xmm0, %xmm0 # sched: [5:1.00]
-; BDVER2-NEXT:    vrcpss %xmm1, %xmm1, %xmm1 # sched: [5:1.00]
-; BDVER2-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [5:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_rcpss:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero sched: [5:1.00]
-; BTVER2-SSE-NEXT:    rcpss %xmm0, %xmm0 # sched: [2:1.00]
-; BTVER2-SSE-NEXT:    rcpss %xmm1, %xmm1 # sched: [2:1.00]
-; BTVER2-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_rcpss:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero sched: [5:1.00]
-; BTVER2-NEXT:    vrcpss %xmm0, %xmm0, %xmm0 # sched: [2:1.00]
-; BTVER2-NEXT:    vrcpss %xmm1, %xmm1, %xmm1 # sched: [2:1.00]
-; BTVER2-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_rcpss:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    rcpss %xmm0, %xmm0 # sched: [5:0.50]
-; ZNVER1-SSE-NEXT:    rcpss %xmm1, %xmm1 # sched: [5:0.50]
-; ZNVER1-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_rcpss:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero sched: [8:0.50]
-; ZNVER1-NEXT:    vrcpss %xmm0, %xmm0, %xmm0 # sched: [5:0.50]
-; ZNVER1-NEXT:    vrcpss %xmm1, %xmm1, %xmm1 # sched: [5:0.50]
-; ZNVER1-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = insertelement <4 x float> undef, float %a0, i32 0
-  %2 = call <4 x float> @llvm.x86.sse.rcp.ss(<4 x float> %1)
-  %3 = load float, float *%a1, align 4
-  %4 = insertelement <4 x float> undef, float %3, i32 0
-  %5 = call <4 x float> @llvm.x86.sse.rcp.ss(<4 x float> %4)
-  %6 = fadd <4 x float> %2, %5
-  ret <4 x float> %6
-}
-declare <4 x float> @llvm.x86.sse.rcp.ss(<4 x float>) nounwind readnone
-
-define <4 x float> @test_rsqrtps(<4 x float> %a0, <4 x float> *%a1) {
-; GENERIC-LABEL: test_rsqrtps:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    rsqrtps %xmm0, %xmm1 # sched: [5:1.00]
-; GENERIC-NEXT:    rsqrtps (%rdi), %xmm0 # sched: [11:1.00]
-; GENERIC-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_rsqrtps:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    rsqrtps (%rdi), %xmm1 # sched: [10:5.00]
-; ATOM-NEXT:    rsqrtps %xmm0, %xmm0 # sched: [9:4.50]
-; ATOM-NEXT:    addps %xmm0, %xmm1 # sched: [5:5.00]
-; ATOM-NEXT:    movaps %xmm1, %xmm0 # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_rsqrtps:
-; SLM:       # %bb.0:
-; SLM-NEXT:    rsqrtps (%rdi), %xmm1 # sched: [8:1.00]
-; SLM-NEXT:    rsqrtps %xmm0, %xmm0 # sched: [5:1.00]
-; SLM-NEXT:    addps %xmm0, %xmm1 # sched: [3:1.00]
-; SLM-NEXT:    movaps %xmm1, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_rsqrtps:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    rsqrtps %xmm0, %xmm1 # sched: [5:1.00]
-; SANDY-SSE-NEXT:    rsqrtps (%rdi), %xmm0 # sched: [11:1.00]
-; SANDY-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_rsqrtps:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vrsqrtps %xmm0, %xmm0 # sched: [5:1.00]
-; SANDY-NEXT:    vrsqrtps (%rdi), %xmm1 # sched: [11:1.00]
-; SANDY-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_rsqrtps:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    rsqrtps %xmm0, %xmm1 # sched: [5:1.00]
-; HASWELL-SSE-NEXT:    rsqrtps (%rdi), %xmm0 # sched: [11:1.00]
-; HASWELL-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_rsqrtps:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vrsqrtps %xmm0, %xmm0 # sched: [5:1.00]
-; HASWELL-NEXT:    vrsqrtps (%rdi), %xmm1 # sched: [11:1.00]
-; HASWELL-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_rsqrtps:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    rsqrtps %xmm0, %xmm1 # sched: [5:1.00]
-; BROADWELL-SSE-NEXT:    rsqrtps (%rdi), %xmm0 # sched: [10:1.00]
-; BROADWELL-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_rsqrtps:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vrsqrtps %xmm0, %xmm0 # sched: [5:1.00]
-; BROADWELL-NEXT:    vrsqrtps (%rdi), %xmm1 # sched: [10:1.00]
-; BROADWELL-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_rsqrtps:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    rsqrtps %xmm0, %xmm1 # sched: [4:1.00]
-; SKYLAKE-SSE-NEXT:    rsqrtps (%rdi), %xmm0 # sched: [10:1.00]
-; SKYLAKE-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [4:0.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_rsqrtps:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vrsqrtps %xmm0, %xmm0 # sched: [4:1.00]
-; SKYLAKE-NEXT:    vrsqrtps (%rdi), %xmm1 # sched: [10:1.00]
-; SKYLAKE-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_rsqrtps:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    rsqrtps %xmm0, %xmm1 # sched: [4:1.00]
-; SKX-SSE-NEXT:    rsqrtps (%rdi), %xmm0 # sched: [10:1.00]
-; SKX-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [4:0.50]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_rsqrtps:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vrsqrtps %xmm0, %xmm0 # sched: [4:1.00]
-; SKX-NEXT:    vrsqrtps (%rdi), %xmm1 # sched: [10:1.00]
-; SKX-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_rsqrtps:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    rsqrtps %xmm0, %xmm1 # sched: [5:1.00]
-; BDVER2-SSE-NEXT:    rsqrtps (%rdi), %xmm0 # sched: [10:1.00]
-; BDVER2-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [5:1.00]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_rsqrtps:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vrsqrtps (%rdi), %xmm1 # sched: [10:1.00]
-; BDVER2-NEXT:    vrsqrtps %xmm0, %xmm0 # sched: [5:1.00]
-; BDVER2-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [5:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_rsqrtps:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    rsqrtps %xmm0, %xmm1 # sched: [2:1.00]
-; BTVER2-SSE-NEXT:    rsqrtps (%rdi), %xmm0 # sched: [7:1.00]
-; BTVER2-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_rsqrtps:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vrsqrtps (%rdi), %xmm1 # sched: [7:1.00]
-; BTVER2-NEXT:    vrsqrtps %xmm0, %xmm0 # sched: [2:1.00]
-; BTVER2-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_rsqrtps:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    rsqrtps %xmm0, %xmm1 # sched: [5:0.50]
-; ZNVER1-SSE-NEXT:    rsqrtps (%rdi), %xmm0 # sched: [12:0.50]
-; ZNVER1-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_rsqrtps:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vrsqrtps (%rdi), %xmm1 # sched: [12:0.50]
-; ZNVER1-NEXT:    vrsqrtps %xmm0, %xmm0 # sched: [5:0.50]
-; ZNVER1-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <4 x float> @llvm.x86.sse.rsqrt.ps(<4 x float> %a0)
-  %2 = load <4 x float>, <4 x float> *%a1, align 16
-  %3 = call <4 x float> @llvm.x86.sse.rsqrt.ps(<4 x float> %2)
-  %4 = fadd <4 x float> %1, %3
-  ret <4 x float> %4
-}
-declare <4 x float> @llvm.x86.sse.rsqrt.ps(<4 x float>) nounwind readnone
-
-; TODO - rsqrtss_m
-
-define <4 x float> @test_rsqrtss(float %a0, float *%a1) {
-; GENERIC-LABEL: test_rsqrtss:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    rsqrtss %xmm0, %xmm0 # sched: [5:1.00]
-; GENERIC-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero sched: [6:0.50]
-; GENERIC-NEXT:    rsqrtss %xmm1, %xmm1 # sched: [5:1.00]
-; GENERIC-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_rsqrtss:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero sched: [1:1.00]
-; ATOM-NEXT:    rsqrtss %xmm0, %xmm0 # sched: [4:4.00]
-; ATOM-NEXT:    rsqrtss %xmm1, %xmm1 # sched: [4:4.00]
-; ATOM-NEXT:    addps %xmm1, %xmm0 # sched: [5:5.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_rsqrtss:
-; SLM:       # %bb.0:
-; SLM-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero sched: [3:1.00]
-; SLM-NEXT:    rsqrtss %xmm0, %xmm0 # sched: [5:1.00]
-; SLM-NEXT:    rsqrtss %xmm1, %xmm1 # sched: [5:1.00]
-; SLM-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_rsqrtss:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    rsqrtss %xmm0, %xmm0 # sched: [5:1.00]
-; SANDY-SSE-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero sched: [6:0.50]
-; SANDY-SSE-NEXT:    rsqrtss %xmm1, %xmm1 # sched: [5:1.00]
-; SANDY-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_rsqrtss:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vrsqrtss %xmm0, %xmm0, %xmm0 # sched: [5:1.00]
-; SANDY-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero sched: [6:0.50]
-; SANDY-NEXT:    vrsqrtss %xmm1, %xmm1, %xmm1 # sched: [5:1.00]
-; SANDY-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_rsqrtss:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    rsqrtss %xmm0, %xmm0 # sched: [5:1.00]
-; HASWELL-SSE-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero sched: [5:0.50]
-; HASWELL-SSE-NEXT:    rsqrtss %xmm1, %xmm1 # sched: [5:1.00]
-; HASWELL-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_rsqrtss:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vrsqrtss %xmm0, %xmm0, %xmm0 # sched: [5:1.00]
-; HASWELL-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero sched: [5:0.50]
-; HASWELL-NEXT:    vrsqrtss %xmm1, %xmm1, %xmm1 # sched: [5:1.00]
-; HASWELL-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_rsqrtss:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    rsqrtss %xmm0, %xmm0 # sched: [5:1.00]
-; BROADWELL-SSE-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero sched: [5:0.50]
-; BROADWELL-SSE-NEXT:    rsqrtss %xmm1, %xmm1 # sched: [5:1.00]
-; BROADWELL-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_rsqrtss:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vrsqrtss %xmm0, %xmm0, %xmm0 # sched: [5:1.00]
-; BROADWELL-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero sched: [5:0.50]
-; BROADWELL-NEXT:    vrsqrtss %xmm1, %xmm1, %xmm1 # sched: [5:1.00]
-; BROADWELL-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_rsqrtss:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    rsqrtss %xmm0, %xmm0 # sched: [4:1.00]
-; SKYLAKE-SSE-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero sched: [5:0.50]
-; SKYLAKE-SSE-NEXT:    rsqrtss %xmm1, %xmm1 # sched: [4:1.00]
-; SKYLAKE-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [4:0.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_rsqrtss:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vrsqrtss %xmm0, %xmm0, %xmm0 # sched: [4:1.00]
-; SKYLAKE-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero sched: [5:0.50]
-; SKYLAKE-NEXT:    vrsqrtss %xmm1, %xmm1, %xmm1 # sched: [4:1.00]
-; SKYLAKE-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_rsqrtss:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    rsqrtss %xmm0, %xmm0 # sched: [4:1.00]
-; SKX-SSE-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero sched: [5:0.50]
-; SKX-SSE-NEXT:    rsqrtss %xmm1, %xmm1 # sched: [4:1.00]
-; SKX-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [4:0.50]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_rsqrtss:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vrsqrtss %xmm0, %xmm0, %xmm0 # sched: [4:1.00]
-; SKX-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero sched: [5:0.50]
-; SKX-NEXT:    vrsqrtss %xmm1, %xmm1, %xmm1 # sched: [4:1.00]
-; SKX-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_rsqrtss:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero sched: [5:0.50]
-; BDVER2-SSE-NEXT:    rsqrtss %xmm0, %xmm0 # sched: [5:1.00]
-; BDVER2-SSE-NEXT:    rsqrtss %xmm1, %xmm1 # sched: [5:1.00]
-; BDVER2-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [5:1.00]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_rsqrtss:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero sched: [5:0.50]
-; BDVER2-NEXT:    vrsqrtss %xmm0, %xmm0, %xmm0 # sched: [5:1.00]
-; BDVER2-NEXT:    vrsqrtss %xmm1, %xmm1, %xmm1 # sched: [5:1.00]
-; BDVER2-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [5:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_rsqrtss:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero sched: [5:1.00]
-; BTVER2-SSE-NEXT:    rsqrtss %xmm0, %xmm0 # sched: [2:1.00]
-; BTVER2-SSE-NEXT:    rsqrtss %xmm1, %xmm1 # sched: [2:1.00]
-; BTVER2-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_rsqrtss:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero sched: [5:1.00]
-; BTVER2-NEXT:    vrsqrtss %xmm0, %xmm0, %xmm0 # sched: [2:1.00]
-; BTVER2-NEXT:    vrsqrtss %xmm1, %xmm1, %xmm1 # sched: [2:1.00]
-; BTVER2-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_rsqrtss:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    rsqrtss %xmm0, %xmm0 # sched: [5:0.50]
-; ZNVER1-SSE-NEXT:    rsqrtss %xmm1, %xmm1 # sched: [5:0.50]
-; ZNVER1-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_rsqrtss:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero sched: [8:0.50]
-; ZNVER1-NEXT:    vrsqrtss %xmm0, %xmm0, %xmm0 # sched: [5:0.50]
-; ZNVER1-NEXT:    vrsqrtss %xmm1, %xmm1, %xmm1 # sched: [5:0.50]
-; ZNVER1-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = insertelement <4 x float> undef, float %a0, i32 0
-  %2 = call <4 x float> @llvm.x86.sse.rsqrt.ss(<4 x float> %1)
-  %3 = load float, float *%a1, align 4
-  %4 = insertelement <4 x float> undef, float %3, i32 0
-  %5 = call <4 x float> @llvm.x86.sse.rsqrt.ss(<4 x float> %4)
-  %6 = fadd <4 x float> %2, %5
-  ret <4 x float> %6
-}
-declare <4 x float> @llvm.x86.sse.rsqrt.ss(<4 x float>) nounwind readnone
-
-define void @test_sfence() {
-; GENERIC-LABEL: test_sfence:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    sfence # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_sfence:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    sfence # sched: [1:1.00]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_sfence:
-; SLM:       # %bb.0:
-; SLM-NEXT:    sfence # sched: [1:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_sfence:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    sfence # sched: [1:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_sfence:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    sfence # sched: [1:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_sfence:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    sfence # sched: [2:0.33]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_sfence:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    sfence # sched: [2:0.33]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_sfence:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    sfence # sched: [2:0.33]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_sfence:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    sfence # sched: [2:0.33]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_sfence:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    sfence # sched: [2:0.33]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_sfence:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    sfence # sched: [2:0.33]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_sfence:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    sfence # sched: [2:0.33]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_sfence:
-; SKX:       # %bb.0:
-; SKX-NEXT:    sfence # sched: [2:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_sfence:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    sfence # sched: [1:1.00]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_sfence:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    sfence # sched: [1:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_sfence:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    sfence # sched: [1:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_sfence:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    sfence # sched: [1:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_sfence:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    sfence # sched: [1:0.50]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_sfence:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    sfence # sched: [1:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  call void @llvm.x86.sse.sfence()
-  ret void
-}
-declare void @llvm.x86.sse.sfence() nounwind readnone
-
-define <4 x float> @test_shufps(<4 x float> %a0, <4 x float> %a1, <4 x float> *%a2) nounwind {
-; GENERIC-LABEL: test_shufps:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,0],xmm1[0,0] sched: [1:1.00]
-; GENERIC-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,3],mem[0,0] sched: [7:1.00]
-; GENERIC-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_shufps:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,0],xmm1[0,0] sched: [1:1.00]
-; ATOM-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,3],mem[0,0] sched: [1:1.00]
-; ATOM-NEXT:    addps %xmm1, %xmm0 # sched: [5:5.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_shufps:
-; SLM:       # %bb.0:
-; SLM-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,0],xmm1[0,0] sched: [1:1.00]
-; SLM-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,3],mem[0,0] sched: [4:1.00]
-; SLM-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_shufps:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,0],xmm1[0,0] sched: [1:1.00]
-; SANDY-SSE-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,3],mem[0,0] sched: [7:1.00]
-; SANDY-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_shufps:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,0],xmm1[0,0] sched: [1:1.00]
-; SANDY-NEXT:    vshufps {{.*#+}} xmm1 = xmm1[0,3],mem[0,0] sched: [7:1.00]
-; SANDY-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_shufps:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,0],xmm1[0,0] sched: [1:1.00]
-; HASWELL-SSE-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,3],mem[0,0] sched: [7:1.00]
-; HASWELL-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_shufps:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,0],xmm1[0,0] sched: [1:1.00]
-; HASWELL-NEXT:    vshufps {{.*#+}} xmm1 = xmm1[0,3],mem[0,0] sched: [7:1.00]
-; HASWELL-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_shufps:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,0],xmm1[0,0] sched: [1:1.00]
-; BROADWELL-SSE-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,3],mem[0,0] sched: [6:1.00]
-; BROADWELL-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_shufps:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,0],xmm1[0,0] sched: [1:1.00]
-; BROADWELL-NEXT:    vshufps {{.*#+}} xmm1 = xmm1[0,3],mem[0,0] sched: [6:1.00]
-; BROADWELL-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_shufps:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,0],xmm1[0,0] sched: [1:1.00]
-; SKYLAKE-SSE-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,3],mem[0,0] sched: [7:1.00]
-; SKYLAKE-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [4:0.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_shufps:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,0],xmm1[0,0] sched: [1:1.00]
-; SKYLAKE-NEXT:    vshufps {{.*#+}} xmm1 = xmm1[0,3],mem[0,0] sched: [7:1.00]
-; SKYLAKE-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_shufps:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,0],xmm1[0,0] sched: [1:1.00]
-; SKX-SSE-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,3],mem[0,0] sched: [7:1.00]
-; SKX-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [4:0.50]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_shufps:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,0],xmm1[0,0] sched: [1:1.00]
-; SKX-NEXT:    vshufps {{.*#+}} xmm1 = xmm1[0,3],mem[0,0] sched: [7:1.00]
-; SKX-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_shufps:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,0],xmm1[0,0] sched: [2:0.50]
-; BDVER2-SSE-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,3],mem[0,0] sched: [7:0.50]
-; BDVER2-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [5:1.00]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_shufps:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,0],xmm1[0,0] sched: [2:0.50]
-; BDVER2-NEXT:    vshufps {{.*#+}} xmm1 = xmm1[0,3],mem[0,0] sched: [7:0.50]
-; BDVER2-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [5:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_shufps:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,0],xmm1[0,0] sched: [1:0.50]
-; BTVER2-SSE-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,3],mem[0,0] sched: [6:1.00]
-; BTVER2-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_shufps:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,0],xmm1[0,0] sched: [1:0.50]
-; BTVER2-NEXT:    vshufps {{.*#+}} xmm1 = xmm1[0,3],mem[0,0] sched: [6:1.00]
-; BTVER2-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_shufps:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,0],xmm1[0,0] sched: [1:0.50]
-; ZNVER1-SSE-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,3],mem[0,0] sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_shufps:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[0,0],xmm1[0,0] sched: [1:0.50]
-; ZNVER1-NEXT:    vshufps {{.*#+}} xmm1 = xmm1[0,3],mem[0,0] sched: [8:0.50]
-; ZNVER1-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = shufflevector <4 x float> %a0, <4 x float> %a1, <4 x i32> <i32 0, i32 0, i32 4, i32 4>
-  %2 = load <4 x float>, <4 x float> *%a2, align 16
-  %3 = shufflevector <4 x float> %a1, <4 x float> %2, <4 x i32> <i32 0, i32 3, i32 4, i32 4>
-  %4 = fadd <4 x float> %1, %3
-  ret <4 x float> %4
-}
-
-define <4 x float> @test_sqrtps(<4 x float> %a0, <4 x float> *%a1) {
-; GENERIC-LABEL: test_sqrtps:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    sqrtps %xmm0, %xmm1 # sched: [14:14.00]
-; GENERIC-NEXT:    sqrtps (%rdi), %xmm0 # sched: [20:14.00]
-; GENERIC-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_sqrtps:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    sqrtps %xmm0, %xmm1 # sched: [70:35.00]
-; ATOM-NEXT:    sqrtps (%rdi), %xmm0 # sched: [70:35.00]
-; ATOM-NEXT:    addps %xmm1, %xmm0 # sched: [5:5.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_sqrtps:
-; SLM:       # %bb.0:
-; SLM-NEXT:    sqrtps (%rdi), %xmm1 # sched: [44:40.00]
-; SLM-NEXT:    sqrtps %xmm0, %xmm0 # sched: [41:40.00]
-; SLM-NEXT:    addps %xmm0, %xmm1 # sched: [3:1.00]
-; SLM-NEXT:    movaps %xmm1, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_sqrtps:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    sqrtps %xmm0, %xmm1 # sched: [14:14.00]
-; SANDY-SSE-NEXT:    sqrtps (%rdi), %xmm0 # sched: [20:14.00]
-; SANDY-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_sqrtps:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vsqrtps %xmm0, %xmm0 # sched: [14:14.00]
-; SANDY-NEXT:    vsqrtps (%rdi), %xmm1 # sched: [20:14.00]
-; SANDY-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_sqrtps:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    sqrtps %xmm0, %xmm1 # sched: [11:7.00]
-; HASWELL-SSE-NEXT:    sqrtps (%rdi), %xmm0 # sched: [17:7.00]
-; HASWELL-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_sqrtps:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vsqrtps %xmm0, %xmm0 # sched: [11:7.00]
-; HASWELL-NEXT:    vsqrtps (%rdi), %xmm1 # sched: [17:7.00]
-; HASWELL-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_sqrtps:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    sqrtps %xmm0, %xmm1 # sched: [11:7.00]
-; BROADWELL-SSE-NEXT:    sqrtps (%rdi), %xmm0 # sched: [16:7.00]
-; BROADWELL-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_sqrtps:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vsqrtps %xmm0, %xmm0 # sched: [11:7.00]
-; BROADWELL-NEXT:    vsqrtps (%rdi), %xmm1 # sched: [16:7.00]
-; BROADWELL-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_sqrtps:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    sqrtps %xmm0, %xmm1 # sched: [12:3.00]
-; SKYLAKE-SSE-NEXT:    sqrtps (%rdi), %xmm0 # sched: [18:3.00]
-; SKYLAKE-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [4:0.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_sqrtps:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vsqrtps %xmm0, %xmm0 # sched: [12:3.00]
-; SKYLAKE-NEXT:    vsqrtps (%rdi), %xmm1 # sched: [18:3.00]
-; SKYLAKE-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_sqrtps:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    sqrtps %xmm0, %xmm1 # sched: [12:3.00]
-; SKX-SSE-NEXT:    sqrtps (%rdi), %xmm0 # sched: [18:3.00]
-; SKX-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [4:0.50]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_sqrtps:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vsqrtps %xmm0, %xmm0 # sched: [12:3.00]
-; SKX-NEXT:    vsqrtps (%rdi), %xmm1 # sched: [18:3.00]
-; SKX-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_sqrtps:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    sqrtps %xmm0, %xmm1 # sched: [9:10.50]
-; BDVER2-SSE-NEXT:    sqrtps (%rdi), %xmm0 # sched: [14:10.50]
-; BDVER2-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [5:1.00]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_sqrtps:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vsqrtps (%rdi), %xmm1 # sched: [14:10.50]
-; BDVER2-NEXT:    vsqrtps %xmm0, %xmm0 # sched: [9:10.50]
-; BDVER2-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [5:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_sqrtps:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    sqrtps %xmm0, %xmm1 # sched: [21:21.00]
-; BTVER2-SSE-NEXT:    sqrtps (%rdi), %xmm0 # sched: [26:21.00]
-; BTVER2-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_sqrtps:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vsqrtps (%rdi), %xmm1 # sched: [26:21.00]
-; BTVER2-NEXT:    vsqrtps %xmm0, %xmm0 # sched: [21:21.00]
-; BTVER2-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_sqrtps:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    sqrtps %xmm0, %xmm1 # sched: [20:20.00]
-; ZNVER1-SSE-NEXT:    sqrtps (%rdi), %xmm0 # sched: [27:20.00]
-; ZNVER1-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_sqrtps:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vsqrtps (%rdi), %xmm1 # sched: [27:20.00]
-; ZNVER1-NEXT:    vsqrtps %xmm0, %xmm0 # sched: [20:20.00]
-; ZNVER1-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <4 x float> @llvm.x86.sse.sqrt.ps(<4 x float> %a0)
-  %2 = load <4 x float>, <4 x float> *%a1, align 16
-  %3 = call <4 x float> @llvm.x86.sse.sqrt.ps(<4 x float> %2)
-  %4 = fadd <4 x float> %1, %3
-  ret <4 x float> %4
-}
-declare <4 x float> @llvm.x86.sse.sqrt.ps(<4 x float>) nounwind readnone
-
-; TODO - sqrtss_m
-
-define <4 x float> @test_sqrtss(<4 x float> %a0, <4 x float> *%a1) {
-; GENERIC-LABEL: test_sqrtss:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    sqrtss %xmm0, %xmm0 # sched: [14:14.00]
-; GENERIC-NEXT:    movaps (%rdi), %xmm1 # sched: [6:0.50]
-; GENERIC-NEXT:    sqrtss %xmm1, %xmm1 # sched: [14:14.00]
-; GENERIC-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_sqrtss:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    movaps (%rdi), %xmm1 # sched: [1:1.00]
-; ATOM-NEXT:    sqrtss %xmm0, %xmm0 # sched: [34:17.00]
-; ATOM-NEXT:    sqrtss %xmm1, %xmm1 # sched: [34:17.00]
-; ATOM-NEXT:    addps %xmm1, %xmm0 # sched: [5:5.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_sqrtss:
-; SLM:       # %bb.0:
-; SLM-NEXT:    movaps (%rdi), %xmm1 # sched: [3:1.00]
-; SLM-NEXT:    sqrtss %xmm0, %xmm0 # sched: [20:20.00]
-; SLM-NEXT:    sqrtss %xmm1, %xmm1 # sched: [20:20.00]
-; SLM-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_sqrtss:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    sqrtss %xmm0, %xmm0 # sched: [14:14.00]
-; SANDY-SSE-NEXT:    movaps (%rdi), %xmm1 # sched: [6:0.50]
-; SANDY-SSE-NEXT:    sqrtss %xmm1, %xmm1 # sched: [14:14.00]
-; SANDY-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_sqrtss:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vsqrtss %xmm0, %xmm0, %xmm0 # sched: [14:14.00]
-; SANDY-NEXT:    vmovaps (%rdi), %xmm1 # sched: [6:0.50]
-; SANDY-NEXT:    vsqrtss %xmm1, %xmm1, %xmm1 # sched: [14:14.00]
-; SANDY-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_sqrtss:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    sqrtss %xmm0, %xmm0 # sched: [11:7.00]
-; HASWELL-SSE-NEXT:    movaps (%rdi), %xmm1 # sched: [6:0.50]
-; HASWELL-SSE-NEXT:    sqrtss %xmm1, %xmm1 # sched: [11:7.00]
-; HASWELL-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_sqrtss:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vsqrtss %xmm0, %xmm0, %xmm0 # sched: [11:7.00]
-; HASWELL-NEXT:    vmovaps (%rdi), %xmm1 # sched: [6:0.50]
-; HASWELL-NEXT:    vsqrtss %xmm1, %xmm1, %xmm1 # sched: [11:7.00]
-; HASWELL-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_sqrtss:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    sqrtss %xmm0, %xmm0 # sched: [11:4.00]
-; BROADWELL-SSE-NEXT:    movaps (%rdi), %xmm1 # sched: [5:0.50]
-; BROADWELL-SSE-NEXT:    sqrtss %xmm1, %xmm1 # sched: [11:4.00]
-; BROADWELL-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_sqrtss:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vsqrtss %xmm0, %xmm0, %xmm0 # sched: [11:4.00]
-; BROADWELL-NEXT:    vmovaps (%rdi), %xmm1 # sched: [5:0.50]
-; BROADWELL-NEXT:    vsqrtss %xmm1, %xmm1, %xmm1 # sched: [11:4.00]
-; BROADWELL-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_sqrtss:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    sqrtss %xmm0, %xmm0 # sched: [12:3.00]
-; SKYLAKE-SSE-NEXT:    movaps (%rdi), %xmm1 # sched: [6:0.50]
-; SKYLAKE-SSE-NEXT:    sqrtss %xmm1, %xmm1 # sched: [12:3.00]
-; SKYLAKE-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [4:0.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_sqrtss:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vsqrtss %xmm0, %xmm0, %xmm0 # sched: [12:3.00]
-; SKYLAKE-NEXT:    vmovaps (%rdi), %xmm1 # sched: [6:0.50]
-; SKYLAKE-NEXT:    vsqrtss %xmm1, %xmm1, %xmm1 # sched: [12:3.00]
-; SKYLAKE-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_sqrtss:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    sqrtss %xmm0, %xmm0 # sched: [12:3.00]
-; SKX-SSE-NEXT:    movaps (%rdi), %xmm1 # sched: [6:0.50]
-; SKX-SSE-NEXT:    sqrtss %xmm1, %xmm1 # sched: [12:3.00]
-; SKX-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [4:0.50]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_sqrtss:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vsqrtss %xmm0, %xmm0, %xmm0 # sched: [12:3.00]
-; SKX-NEXT:    vmovaps (%rdi), %xmm1 # sched: [6:0.50]
-; SKX-NEXT:    vsqrtss %xmm1, %xmm1, %xmm1 # sched: [12:3.00]
-; SKX-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_sqrtss:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    movaps (%rdi), %xmm1 # sched: [5:0.50]
-; BDVER2-SSE-NEXT:    sqrtss %xmm0, %xmm0 # sched: [9:10.50]
-; BDVER2-SSE-NEXT:    sqrtss %xmm1, %xmm1 # sched: [9:10.50]
-; BDVER2-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [5:1.00]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_sqrtss:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vmovaps (%rdi), %xmm1 # sched: [5:0.50]
-; BDVER2-NEXT:    vsqrtss %xmm0, %xmm0, %xmm0 # sched: [9:10.50]
-; BDVER2-NEXT:    vsqrtss %xmm1, %xmm1, %xmm1 # sched: [9:10.50]
-; BDVER2-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [5:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_sqrtss:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    movaps (%rdi), %xmm1 # sched: [5:1.00]
-; BTVER2-SSE-NEXT:    sqrtss %xmm0, %xmm0 # sched: [21:21.00]
-; BTVER2-SSE-NEXT:    sqrtss %xmm1, %xmm1 # sched: [21:21.00]
-; BTVER2-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_sqrtss:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vmovaps (%rdi), %xmm1 # sched: [5:1.00]
-; BTVER2-NEXT:    vsqrtss %xmm0, %xmm0, %xmm0 # sched: [21:21.00]
-; BTVER2-NEXT:    vsqrtss %xmm1, %xmm1, %xmm1 # sched: [21:21.00]
-; BTVER2-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_sqrtss:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    movaps (%rdi), %xmm1 # sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    sqrtss %xmm0, %xmm0 # sched: [20:20.00]
-; ZNVER1-SSE-NEXT:    sqrtss %xmm1, %xmm1 # sched: [20:20.00]
-; ZNVER1-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_sqrtss:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vmovaps (%rdi), %xmm1 # sched: [8:0.50]
-; ZNVER1-NEXT:    vsqrtss %xmm0, %xmm0, %xmm0 # sched: [20:20.00]
-; ZNVER1-NEXT:    vsqrtss %xmm1, %xmm1, %xmm1 # sched: [20:20.00]
-; ZNVER1-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <4 x float> @llvm.x86.sse.sqrt.ss(<4 x float> %a0)
-  %2 = load <4 x float>, <4 x float> *%a1, align 16
-  %3 = call <4 x float> @llvm.x86.sse.sqrt.ss(<4 x float> %2)
-  %4 = fadd <4 x float> %1, %3
-  ret <4 x float> %4
-}
-declare <4 x float> @llvm.x86.sse.sqrt.ss(<4 x float>) nounwind readnone
-
-define i32 @test_stmxcsr() {
-; GENERIC-LABEL: test_stmxcsr:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    stmxcsr -{{[0-9]+}}(%rsp) # sched: [5:1.00]
-; GENERIC-NEXT:    movl -{{[0-9]+}}(%rsp), %eax # sched: [5:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_stmxcsr:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    stmxcsr -{{[0-9]+}}(%rsp) # sched: [15:7.50]
-; ATOM-NEXT:    movl -{{[0-9]+}}(%rsp), %eax # sched: [1:1.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_stmxcsr:
-; SLM:       # %bb.0:
-; SLM-NEXT:    stmxcsr -{{[0-9]+}}(%rsp) # sched: [1:1.00]
-; SLM-NEXT:    movl -{{[0-9]+}}(%rsp), %eax # sched: [3:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_stmxcsr:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    stmxcsr -{{[0-9]+}}(%rsp) # sched: [5:1.00]
-; SANDY-SSE-NEXT:    movl -{{[0-9]+}}(%rsp), %eax # sched: [5:0.50]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_stmxcsr:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vstmxcsr -{{[0-9]+}}(%rsp) # sched: [5:1.00]
-; SANDY-NEXT:    movl -{{[0-9]+}}(%rsp), %eax # sched: [5:0.50]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_stmxcsr:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    stmxcsr -{{[0-9]+}}(%rsp) # sched: [2:1.00]
-; HASWELL-SSE-NEXT:    movl -{{[0-9]+}}(%rsp), %eax # sched: [5:0.50]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_stmxcsr:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vstmxcsr -{{[0-9]+}}(%rsp) # sched: [2:1.00]
-; HASWELL-NEXT:    movl -{{[0-9]+}}(%rsp), %eax # sched: [5:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_stmxcsr:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    stmxcsr -{{[0-9]+}}(%rsp) # sched: [2:1.00]
-; BROADWELL-SSE-NEXT:    movl -{{[0-9]+}}(%rsp), %eax # sched: [5:0.50]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_stmxcsr:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vstmxcsr -{{[0-9]+}}(%rsp) # sched: [2:1.00]
-; BROADWELL-NEXT:    movl -{{[0-9]+}}(%rsp), %eax # sched: [5:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_stmxcsr:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    stmxcsr -{{[0-9]+}}(%rsp) # sched: [2:1.00]
-; SKYLAKE-SSE-NEXT:    movl -{{[0-9]+}}(%rsp), %eax # sched: [5:0.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_stmxcsr:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vstmxcsr -{{[0-9]+}}(%rsp) # sched: [2:1.00]
-; SKYLAKE-NEXT:    movl -{{[0-9]+}}(%rsp), %eax # sched: [5:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_stmxcsr:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    stmxcsr -{{[0-9]+}}(%rsp) # sched: [2:1.00]
-; SKX-SSE-NEXT:    movl -{{[0-9]+}}(%rsp), %eax # sched: [5:0.50]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_stmxcsr:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vstmxcsr -{{[0-9]+}}(%rsp) # sched: [2:1.00]
-; SKX-NEXT:    movl -{{[0-9]+}}(%rsp), %eax # sched: [5:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_stmxcsr:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    stmxcsr -{{[0-9]+}}(%rsp) # sched: [1:1.00]
-; BDVER2-SSE-NEXT:    movl -{{[0-9]+}}(%rsp), %eax # sched: [5:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_stmxcsr:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vstmxcsr -{{[0-9]+}}(%rsp) # sched: [1:1.00]
-; BDVER2-NEXT:    movl -{{[0-9]+}}(%rsp), %eax # sched: [5:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_stmxcsr:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    stmxcsr -{{[0-9]+}}(%rsp) # sched: [1:1.00]
-; BTVER2-SSE-NEXT:    movl -{{[0-9]+}}(%rsp), %eax # sched: [5:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_stmxcsr:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vstmxcsr -{{[0-9]+}}(%rsp) # sched: [1:1.00]
-; BTVER2-NEXT:    movl -{{[0-9]+}}(%rsp), %eax # sched: [5:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_stmxcsr:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    stmxcsr -{{[0-9]+}}(%rsp) # sched: [100:0.25]
-; ZNVER1-SSE-NEXT:    movl -{{[0-9]+}}(%rsp), %eax # sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_stmxcsr:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vstmxcsr -{{[0-9]+}}(%rsp) # sched: [100:0.25]
-; ZNVER1-NEXT:    movl -{{[0-9]+}}(%rsp), %eax # sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = alloca i32, align 4
-  %2 = bitcast i32* %1 to i8*
-  call void @llvm.x86.sse.stmxcsr(i8* %2)
-  %3 = load i32, i32* %1, align 4
-  ret i32 %3
-}
-declare void @llvm.x86.sse.stmxcsr(i8*) nounwind readnone
-
-define <4 x float> @test_subps(<4 x float> %a0, <4 x float> %a1, <4 x float> *%a2) {
-; GENERIC-LABEL: test_subps:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    subps %xmm1, %xmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    subps (%rdi), %xmm0 # sched: [9:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_subps:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    subps %xmm1, %xmm0 # sched: [5:5.00]
-; ATOM-NEXT:    subps (%rdi), %xmm0 # sched: [5:5.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_subps:
-; SLM:       # %bb.0:
-; SLM-NEXT:    subps %xmm1, %xmm0 # sched: [3:1.00]
-; SLM-NEXT:    subps (%rdi), %xmm0 # sched: [6:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_subps:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    subps %xmm1, %xmm0 # sched: [3:1.00]
-; SANDY-SSE-NEXT:    subps (%rdi), %xmm0 # sched: [9:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_subps:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vsubps %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; SANDY-NEXT:    vsubps (%rdi), %xmm0, %xmm0 # sched: [9:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_subps:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    subps %xmm1, %xmm0 # sched: [3:1.00]
-; HASWELL-SSE-NEXT:    subps (%rdi), %xmm0 # sched: [9:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_subps:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vsubps %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; HASWELL-NEXT:    vsubps (%rdi), %xmm0, %xmm0 # sched: [9:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_subps:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    subps %xmm1, %xmm0 # sched: [3:1.00]
-; BROADWELL-SSE-NEXT:    subps (%rdi), %xmm0 # sched: [8:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_subps:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vsubps %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    vsubps (%rdi), %xmm0, %xmm0 # sched: [8:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_subps:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    subps %xmm1, %xmm0 # sched: [4:0.50]
-; SKYLAKE-SSE-NEXT:    subps (%rdi), %xmm0 # sched: [10:0.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_subps:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vsubps %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    vsubps (%rdi), %xmm0, %xmm0 # sched: [10:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_subps:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    subps %xmm1, %xmm0 # sched: [4:0.50]
-; SKX-SSE-NEXT:    subps (%rdi), %xmm0 # sched: [10:0.50]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_subps:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vsubps %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKX-NEXT:    vsubps (%rdi), %xmm0, %xmm0 # sched: [10:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_subps:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    subps %xmm1, %xmm0 # sched: [5:1.00]
-; BDVER2-SSE-NEXT:    subps (%rdi), %xmm0 # sched: [10:1.00]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_subps:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vsubps %xmm1, %xmm0, %xmm0 # sched: [5:1.00]
-; BDVER2-NEXT:    vsubps (%rdi), %xmm0, %xmm0 # sched: [10:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_subps:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    subps %xmm1, %xmm0 # sched: [3:1.00]
-; BTVER2-SSE-NEXT:    subps (%rdi), %xmm0 # sched: [8:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_subps:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vsubps %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; BTVER2-NEXT:    vsubps (%rdi), %xmm0, %xmm0 # sched: [8:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_subps:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    subps %xmm1, %xmm0 # sched: [3:1.00]
-; ZNVER1-SSE-NEXT:    subps (%rdi), %xmm0 # sched: [10:1.00]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_subps:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vsubps %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    vsubps (%rdi), %xmm0, %xmm0 # sched: [10:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = fsub <4 x float> %a0, %a1
-  %2 = load <4 x float>, <4 x float> *%a2, align 16
-  %3 = fsub <4 x float> %1, %2
-  ret <4 x float> %3
-}
-
-define float @test_subss(float %a0, float %a1, float *%a2) {
-; GENERIC-LABEL: test_subss:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    subss %xmm1, %xmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    subss (%rdi), %xmm0 # sched: [9:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_subss:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    subss %xmm1, %xmm0 # sched: [5:5.00]
-; ATOM-NEXT:    subss (%rdi), %xmm0 # sched: [5:5.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_subss:
-; SLM:       # %bb.0:
-; SLM-NEXT:    subss %xmm1, %xmm0 # sched: [3:1.00]
-; SLM-NEXT:    subss (%rdi), %xmm0 # sched: [6:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_subss:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    subss %xmm1, %xmm0 # sched: [3:1.00]
-; SANDY-SSE-NEXT:    subss (%rdi), %xmm0 # sched: [9:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_subss:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vsubss %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; SANDY-NEXT:    vsubss (%rdi), %xmm0, %xmm0 # sched: [9:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_subss:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    subss %xmm1, %xmm0 # sched: [3:1.00]
-; HASWELL-SSE-NEXT:    subss (%rdi), %xmm0 # sched: [8:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_subss:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vsubss %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; HASWELL-NEXT:    vsubss (%rdi), %xmm0, %xmm0 # sched: [8:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_subss:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    subss %xmm1, %xmm0 # sched: [3:1.00]
-; BROADWELL-SSE-NEXT:    subss (%rdi), %xmm0 # sched: [8:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_subss:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vsubss %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    vsubss (%rdi), %xmm0, %xmm0 # sched: [8:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_subss:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    subss %xmm1, %xmm0 # sched: [4:0.50]
-; SKYLAKE-SSE-NEXT:    subss (%rdi), %xmm0 # sched: [9:0.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_subss:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vsubss %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    vsubss (%rdi), %xmm0, %xmm0 # sched: [9:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_subss:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    subss %xmm1, %xmm0 # sched: [4:0.50]
-; SKX-SSE-NEXT:    subss (%rdi), %xmm0 # sched: [9:0.50]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_subss:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vsubss %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKX-NEXT:    vsubss (%rdi), %xmm0, %xmm0 # sched: [9:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_subss:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    subss %xmm1, %xmm0 # sched: [5:1.00]
-; BDVER2-SSE-NEXT:    subss (%rdi), %xmm0 # sched: [10:1.00]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_subss:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vsubss %xmm1, %xmm0, %xmm0 # sched: [5:1.00]
-; BDVER2-NEXT:    vsubss (%rdi), %xmm0, %xmm0 # sched: [10:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_subss:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    subss %xmm1, %xmm0 # sched: [3:1.00]
-; BTVER2-SSE-NEXT:    subss (%rdi), %xmm0 # sched: [8:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_subss:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vsubss %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; BTVER2-NEXT:    vsubss (%rdi), %xmm0, %xmm0 # sched: [8:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_subss:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    subss %xmm1, %xmm0 # sched: [3:1.00]
-; ZNVER1-SSE-NEXT:    subss (%rdi), %xmm0 # sched: [10:1.00]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_subss:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vsubss %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    vsubss (%rdi), %xmm0, %xmm0 # sched: [10:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = fsub float %a0, %a1
-  %2 = load float, float *%a2, align 4
-  %3 = fsub float %1, %2
-  ret float %3
-}
-
-define i32 @test_ucomiss(<4 x float> %a0, <4 x float> %a1, <4 x float> *%a2) {
-; GENERIC-LABEL: test_ucomiss:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    ucomiss %xmm1, %xmm0 # sched: [2:1.00]
-; GENERIC-NEXT:    setnp %al # sched: [1:0.50]
-; GENERIC-NEXT:    sete %cl # sched: [1:0.50]
-; GENERIC-NEXT:    andb %al, %cl # sched: [1:0.33]
-; GENERIC-NEXT:    ucomiss (%rdi), %xmm0 # sched: [8:1.00]
-; GENERIC-NEXT:    setnp %al # sched: [1:0.50]
-; GENERIC-NEXT:    sete %dl # sched: [1:0.50]
-; GENERIC-NEXT:    andb %al, %dl # sched: [1:0.33]
-; GENERIC-NEXT:    orb %cl, %dl # sched: [1:0.33]
-; GENERIC-NEXT:    movzbl %dl, %eax # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_ucomiss:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    ucomiss %xmm1, %xmm0 # sched: [9:4.50]
-; ATOM-NEXT:    setnp %al # sched: [1:0.50]
-; ATOM-NEXT:    sete %cl # sched: [1:0.50]
-; ATOM-NEXT:    andb %al, %cl # sched: [1:0.50]
-; ATOM-NEXT:    ucomiss (%rdi), %xmm0 # sched: [10:5.00]
-; ATOM-NEXT:    setnp %al # sched: [1:0.50]
-; ATOM-NEXT:    sete %dl # sched: [1:0.50]
-; ATOM-NEXT:    andb %al, %dl # sched: [1:0.50]
-; ATOM-NEXT:    orb %cl, %dl # sched: [1:0.50]
-; ATOM-NEXT:    movzbl %dl, %eax # sched: [1:1.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_ucomiss:
-; SLM:       # %bb.0:
-; SLM-NEXT:    ucomiss %xmm1, %xmm0 # sched: [3:1.00]
-; SLM-NEXT:    setnp %al # sched: [1:0.50]
-; SLM-NEXT:    sete %cl # sched: [1:0.50]
-; SLM-NEXT:    andb %al, %cl # sched: [1:0.50]
-; SLM-NEXT:    ucomiss (%rdi), %xmm0 # sched: [6:1.00]
-; SLM-NEXT:    setnp %al # sched: [1:0.50]
-; SLM-NEXT:    sete %dl # sched: [1:0.50]
-; SLM-NEXT:    andb %al, %dl # sched: [1:0.50]
-; SLM-NEXT:    orb %cl, %dl # sched: [1:0.50]
-; SLM-NEXT:    movzbl %dl, %eax # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_ucomiss:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    ucomiss %xmm1, %xmm0 # sched: [2:1.00]
-; SANDY-SSE-NEXT:    setnp %al # sched: [1:0.50]
-; SANDY-SSE-NEXT:    sete %cl # sched: [1:0.50]
-; SANDY-SSE-NEXT:    andb %al, %cl # sched: [1:0.33]
-; SANDY-SSE-NEXT:    ucomiss (%rdi), %xmm0 # sched: [8:1.00]
-; SANDY-SSE-NEXT:    setnp %al # sched: [1:0.50]
-; SANDY-SSE-NEXT:    sete %dl # sched: [1:0.50]
-; SANDY-SSE-NEXT:    andb %al, %dl # sched: [1:0.33]
-; SANDY-SSE-NEXT:    orb %cl, %dl # sched: [1:0.33]
-; SANDY-SSE-NEXT:    movzbl %dl, %eax # sched: [1:0.33]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_ucomiss:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vucomiss %xmm1, %xmm0 # sched: [2:1.00]
-; SANDY-NEXT:    setnp %al # sched: [1:0.50]
-; SANDY-NEXT:    sete %cl # sched: [1:0.50]
-; SANDY-NEXT:    andb %al, %cl # sched: [1:0.33]
-; SANDY-NEXT:    vucomiss (%rdi), %xmm0 # sched: [8:1.00]
-; SANDY-NEXT:    setnp %al # sched: [1:0.50]
-; SANDY-NEXT:    sete %dl # sched: [1:0.50]
-; SANDY-NEXT:    andb %al, %dl # sched: [1:0.33]
-; SANDY-NEXT:    orb %cl, %dl # sched: [1:0.33]
-; SANDY-NEXT:    movzbl %dl, %eax # sched: [1:0.33]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_ucomiss:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    ucomiss %xmm1, %xmm0 # sched: [3:1.00]
-; HASWELL-SSE-NEXT:    setnp %al # sched: [1:0.50]
-; HASWELL-SSE-NEXT:    sete %cl # sched: [1:0.50]
-; HASWELL-SSE-NEXT:    andb %al, %cl # sched: [1:0.25]
-; HASWELL-SSE-NEXT:    ucomiss (%rdi), %xmm0 # sched: [8:1.00]
-; HASWELL-SSE-NEXT:    setnp %al # sched: [1:0.50]
-; HASWELL-SSE-NEXT:    sete %dl # sched: [1:0.50]
-; HASWELL-SSE-NEXT:    andb %al, %dl # sched: [1:0.25]
-; HASWELL-SSE-NEXT:    orb %cl, %dl # sched: [1:0.25]
-; HASWELL-SSE-NEXT:    movzbl %dl, %eax # sched: [1:0.25]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_ucomiss:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vucomiss %xmm1, %xmm0 # sched: [3:1.00]
-; HASWELL-NEXT:    setnp %al # sched: [1:0.50]
-; HASWELL-NEXT:    sete %cl # sched: [1:0.50]
-; HASWELL-NEXT:    andb %al, %cl # sched: [1:0.25]
-; HASWELL-NEXT:    vucomiss (%rdi), %xmm0 # sched: [8:1.00]
-; HASWELL-NEXT:    setnp %al # sched: [1:0.50]
-; HASWELL-NEXT:    sete %dl # sched: [1:0.50]
-; HASWELL-NEXT:    andb %al, %dl # sched: [1:0.25]
-; HASWELL-NEXT:    orb %cl, %dl # sched: [1:0.25]
-; HASWELL-NEXT:    movzbl %dl, %eax # sched: [1:0.25]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_ucomiss:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    ucomiss %xmm1, %xmm0 # sched: [3:1.00]
-; BROADWELL-SSE-NEXT:    setnp %al # sched: [1:0.50]
-; BROADWELL-SSE-NEXT:    sete %cl # sched: [1:0.50]
-; BROADWELL-SSE-NEXT:    andb %al, %cl # sched: [1:0.25]
-; BROADWELL-SSE-NEXT:    ucomiss (%rdi), %xmm0 # sched: [8:1.00]
-; BROADWELL-SSE-NEXT:    setnp %al # sched: [1:0.50]
-; BROADWELL-SSE-NEXT:    sete %dl # sched: [1:0.50]
-; BROADWELL-SSE-NEXT:    andb %al, %dl # sched: [1:0.25]
-; BROADWELL-SSE-NEXT:    orb %cl, %dl # sched: [1:0.25]
-; BROADWELL-SSE-NEXT:    movzbl %dl, %eax # sched: [1:0.25]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_ucomiss:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vucomiss %xmm1, %xmm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    setnp %al # sched: [1:0.50]
-; BROADWELL-NEXT:    sete %cl # sched: [1:0.50]
-; BROADWELL-NEXT:    andb %al, %cl # sched: [1:0.25]
-; BROADWELL-NEXT:    vucomiss (%rdi), %xmm0 # sched: [8:1.00]
-; BROADWELL-NEXT:    setnp %al # sched: [1:0.50]
-; BROADWELL-NEXT:    sete %dl # sched: [1:0.50]
-; BROADWELL-NEXT:    andb %al, %dl # sched: [1:0.25]
-; BROADWELL-NEXT:    orb %cl, %dl # sched: [1:0.25]
-; BROADWELL-NEXT:    movzbl %dl, %eax # sched: [1:0.25]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_ucomiss:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    ucomiss %xmm1, %xmm0 # sched: [2:1.00]
-; SKYLAKE-SSE-NEXT:    setnp %al # sched: [1:0.50]
-; SKYLAKE-SSE-NEXT:    sete %cl # sched: [1:0.50]
-; SKYLAKE-SSE-NEXT:    andb %al, %cl # sched: [1:0.25]
-; SKYLAKE-SSE-NEXT:    ucomiss (%rdi), %xmm0 # sched: [7:1.00]
-; SKYLAKE-SSE-NEXT:    setnp %al # sched: [1:0.50]
-; SKYLAKE-SSE-NEXT:    sete %dl # sched: [1:0.50]
-; SKYLAKE-SSE-NEXT:    andb %al, %dl # sched: [1:0.25]
-; SKYLAKE-SSE-NEXT:    orb %cl, %dl # sched: [1:0.25]
-; SKYLAKE-SSE-NEXT:    movzbl %dl, %eax # sched: [1:0.25]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_ucomiss:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vucomiss %xmm1, %xmm0 # sched: [2:1.00]
-; SKYLAKE-NEXT:    setnp %al # sched: [1:0.50]
-; SKYLAKE-NEXT:    sete %cl # sched: [1:0.50]
-; SKYLAKE-NEXT:    andb %al, %cl # sched: [1:0.25]
-; SKYLAKE-NEXT:    vucomiss (%rdi), %xmm0 # sched: [7:1.00]
-; SKYLAKE-NEXT:    setnp %al # sched: [1:0.50]
-; SKYLAKE-NEXT:    sete %dl # sched: [1:0.50]
-; SKYLAKE-NEXT:    andb %al, %dl # sched: [1:0.25]
-; SKYLAKE-NEXT:    orb %cl, %dl # sched: [1:0.25]
-; SKYLAKE-NEXT:    movzbl %dl, %eax # sched: [1:0.25]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_ucomiss:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    ucomiss %xmm1, %xmm0 # sched: [2:1.00]
-; SKX-SSE-NEXT:    setnp %al # sched: [1:0.50]
-; SKX-SSE-NEXT:    sete %cl # sched: [1:0.50]
-; SKX-SSE-NEXT:    andb %al, %cl # sched: [1:0.25]
-; SKX-SSE-NEXT:    ucomiss (%rdi), %xmm0 # sched: [7:1.00]
-; SKX-SSE-NEXT:    setnp %al # sched: [1:0.50]
-; SKX-SSE-NEXT:    sete %dl # sched: [1:0.50]
-; SKX-SSE-NEXT:    andb %al, %dl # sched: [1:0.25]
-; SKX-SSE-NEXT:    orb %cl, %dl # sched: [1:0.25]
-; SKX-SSE-NEXT:    movzbl %dl, %eax # sched: [1:0.25]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_ucomiss:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vucomiss %xmm1, %xmm0 # sched: [2:1.00]
-; SKX-NEXT:    setnp %al # sched: [1:0.50]
-; SKX-NEXT:    sete %cl # sched: [1:0.50]
-; SKX-NEXT:    andb %al, %cl # sched: [1:0.25]
-; SKX-NEXT:    vucomiss (%rdi), %xmm0 # sched: [7:1.00]
-; SKX-NEXT:    setnp %al # sched: [1:0.50]
-; SKX-NEXT:    sete %dl # sched: [1:0.50]
-; SKX-NEXT:    andb %al, %dl # sched: [1:0.25]
-; SKX-NEXT:    orb %cl, %dl # sched: [1:0.25]
-; SKX-NEXT:    movzbl %dl, %eax # sched: [1:0.25]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_ucomiss:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    ucomiss %xmm1, %xmm0 # sched: [1:1.00]
-; BDVER2-SSE-NEXT:    setnp %al # sched: [1:0.50]
-; BDVER2-SSE-NEXT:    sete %cl # sched: [1:0.50]
-; BDVER2-SSE-NEXT:    andb %al, %cl # sched: [1:0.50]
-; BDVER2-SSE-NEXT:    ucomiss (%rdi), %xmm0 # sched: [6:1.00]
-; BDVER2-SSE-NEXT:    setnp %al # sched: [1:0.50]
-; BDVER2-SSE-NEXT:    sete %dl # sched: [1:0.50]
-; BDVER2-SSE-NEXT:    andb %al, %dl # sched: [1:0.50]
-; BDVER2-SSE-NEXT:    orb %cl, %dl # sched: [1:0.50]
-; BDVER2-SSE-NEXT:    movzbl %dl, %eax # sched: [1:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_ucomiss:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vucomiss %xmm1, %xmm0 # sched: [1:1.00]
-; BDVER2-NEXT:    setnp %al # sched: [1:0.50]
-; BDVER2-NEXT:    sete %cl # sched: [1:0.50]
-; BDVER2-NEXT:    andb %al, %cl # sched: [1:0.50]
-; BDVER2-NEXT:    vucomiss (%rdi), %xmm0 # sched: [6:1.00]
-; BDVER2-NEXT:    setnp %al # sched: [1:0.50]
-; BDVER2-NEXT:    sete %dl # sched: [1:0.50]
-; BDVER2-NEXT:    andb %al, %dl # sched: [1:0.50]
-; BDVER2-NEXT:    orb %cl, %dl # sched: [1:0.50]
-; BDVER2-NEXT:    movzbl %dl, %eax # sched: [1:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_ucomiss:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    ucomiss %xmm1, %xmm0 # sched: [3:1.00]
-; BTVER2-SSE-NEXT:    setnp %al # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    sete %cl # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    andb %al, %cl # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    ucomiss (%rdi), %xmm0 # sched: [8:1.00]
-; BTVER2-SSE-NEXT:    setnp %al # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    sete %dl # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    andb %al, %dl # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    orb %cl, %dl # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    movzbl %dl, %eax # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_ucomiss:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vucomiss %xmm1, %xmm0 # sched: [3:1.00]
-; BTVER2-NEXT:    setnp %al # sched: [1:0.50]
-; BTVER2-NEXT:    sete %cl # sched: [1:0.50]
-; BTVER2-NEXT:    andb %al, %cl # sched: [1:0.50]
-; BTVER2-NEXT:    vucomiss (%rdi), %xmm0 # sched: [8:1.00]
-; BTVER2-NEXT:    setnp %al # sched: [1:0.50]
-; BTVER2-NEXT:    sete %dl # sched: [1:0.50]
-; BTVER2-NEXT:    andb %al, %dl # sched: [1:0.50]
-; BTVER2-NEXT:    orb %cl, %dl # sched: [1:0.50]
-; BTVER2-NEXT:    movzbl %dl, %eax # sched: [1:0.50]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_ucomiss:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    ucomiss %xmm1, %xmm0 # sched: [3:1.00]
-; ZNVER1-SSE-NEXT:    setnp %al # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    sete %cl # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    andb %al, %cl # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    ucomiss (%rdi), %xmm0 # sched: [10:1.00]
-; ZNVER1-SSE-NEXT:    setnp %al # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    sete %dl # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    andb %al, %dl # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    orb %cl, %dl # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    movzbl %dl, %eax # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_ucomiss:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vucomiss %xmm1, %xmm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    setnp %al # sched: [1:0.25]
-; ZNVER1-NEXT:    sete %cl # sched: [1:0.25]
-; ZNVER1-NEXT:    andb %al, %cl # sched: [1:0.25]
-; ZNVER1-NEXT:    vucomiss (%rdi), %xmm0 # sched: [10:1.00]
-; ZNVER1-NEXT:    setnp %al # sched: [1:0.25]
-; ZNVER1-NEXT:    sete %dl # sched: [1:0.25]
-; ZNVER1-NEXT:    andb %al, %dl # sched: [1:0.25]
-; ZNVER1-NEXT:    orb %cl, %dl # sched: [1:0.25]
-; ZNVER1-NEXT:    movzbl %dl, %eax # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call i32 @llvm.x86.sse.ucomieq.ss(<4 x float> %a0, <4 x float> %a1)
-  %2 = load <4 x float>, <4 x float> *%a2, align 4
-  %3 = call i32 @llvm.x86.sse.ucomieq.ss(<4 x float> %a0, <4 x float> %2)
-  %4 = or i32 %1, %3
-  ret i32 %4
-}
-declare i32 @llvm.x86.sse.ucomieq.ss(<4 x float>, <4 x float>) nounwind readnone
-
-define <4 x float> @test_unpckhps(<4 x float> %a0, <4 x float> %a1, <4 x float> *%a2) {
-; GENERIC-LABEL: test_unpckhps:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    unpckhps {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3] sched: [1:1.00]
-; GENERIC-NEXT:    unpckhps {{.*#+}} xmm1 = xmm1[2],mem[2],xmm1[3],mem[3] sched: [7:1.00]
-; GENERIC-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_unpckhps:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    unpckhps {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3] sched: [1:1.00]
-; ATOM-NEXT:    unpckhps {{.*#+}} xmm1 = xmm1[2],mem[2],xmm1[3],mem[3] sched: [1:1.00]
-; ATOM-NEXT:    addps %xmm1, %xmm0 # sched: [5:5.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_unpckhps:
-; SLM:       # %bb.0:
-; SLM-NEXT:    unpckhps {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3] sched: [1:1.00]
-; SLM-NEXT:    unpckhps {{.*#+}} xmm1 = xmm1[2],mem[2],xmm1[3],mem[3] sched: [4:1.00]
-; SLM-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_unpckhps:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    unpckhps {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3] sched: [1:1.00]
-; SANDY-SSE-NEXT:    unpckhps {{.*#+}} xmm1 = xmm1[2],mem[2],xmm1[3],mem[3] sched: [7:1.00]
-; SANDY-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_unpckhps:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vunpckhps {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3] sched: [1:1.00]
-; SANDY-NEXT:    vunpckhps {{.*#+}} xmm1 = xmm1[2],mem[2],xmm1[3],mem[3] sched: [7:1.00]
-; SANDY-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_unpckhps:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    unpckhps {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3] sched: [1:1.00]
-; HASWELL-SSE-NEXT:    unpckhps {{.*#+}} xmm1 = xmm1[2],mem[2],xmm1[3],mem[3] sched: [7:1.00]
-; HASWELL-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_unpckhps:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vunpckhps {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3] sched: [1:1.00]
-; HASWELL-NEXT:    vunpckhps {{.*#+}} xmm1 = xmm1[2],mem[2],xmm1[3],mem[3] sched: [7:1.00]
-; HASWELL-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_unpckhps:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    unpckhps {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3] sched: [1:1.00]
-; BROADWELL-SSE-NEXT:    unpckhps {{.*#+}} xmm1 = xmm1[2],mem[2],xmm1[3],mem[3] sched: [6:1.00]
-; BROADWELL-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_unpckhps:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vunpckhps {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3] sched: [1:1.00]
-; BROADWELL-NEXT:    vunpckhps {{.*#+}} xmm1 = xmm1[2],mem[2],xmm1[3],mem[3] sched: [6:1.00]
-; BROADWELL-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_unpckhps:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    unpckhps {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3] sched: [1:1.00]
-; SKYLAKE-SSE-NEXT:    unpckhps {{.*#+}} xmm1 = xmm1[2],mem[2],xmm1[3],mem[3] sched: [7:1.00]
-; SKYLAKE-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [4:0.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_unpckhps:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vunpckhps {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3] sched: [1:1.00]
-; SKYLAKE-NEXT:    vunpckhps {{.*#+}} xmm1 = xmm1[2],mem[2],xmm1[3],mem[3] sched: [7:1.00]
-; SKYLAKE-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_unpckhps:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    unpckhps {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3] sched: [1:1.00]
-; SKX-SSE-NEXT:    unpckhps {{.*#+}} xmm1 = xmm1[2],mem[2],xmm1[3],mem[3] sched: [7:1.00]
-; SKX-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [4:0.50]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_unpckhps:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vunpckhps {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3] sched: [1:1.00]
-; SKX-NEXT:    vunpckhps {{.*#+}} xmm1 = xmm1[2],mem[2],xmm1[3],mem[3] sched: [7:1.00]
-; SKX-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_unpckhps:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    unpckhps {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3] sched: [2:0.50]
-; BDVER2-SSE-NEXT:    unpckhps {{.*#+}} xmm1 = xmm1[2],mem[2],xmm1[3],mem[3] sched: [7:0.50]
-; BDVER2-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [5:1.00]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_unpckhps:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vunpckhps {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3] sched: [2:0.50]
-; BDVER2-NEXT:    vunpckhps {{.*#+}} xmm1 = xmm1[2],mem[2],xmm1[3],mem[3] sched: [7:0.50]
-; BDVER2-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [5:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_unpckhps:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    unpckhps {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3] sched: [1:0.50]
-; BTVER2-SSE-NEXT:    unpckhps {{.*#+}} xmm1 = xmm1[2],mem[2],xmm1[3],mem[3] sched: [6:1.00]
-; BTVER2-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_unpckhps:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vunpckhps {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3] sched: [1:0.50]
-; BTVER2-NEXT:    vunpckhps {{.*#+}} xmm1 = xmm1[2],mem[2],xmm1[3],mem[3] sched: [6:1.00]
-; BTVER2-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_unpckhps:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    unpckhps {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3] sched: [1:0.50]
-; ZNVER1-SSE-NEXT:    unpckhps {{.*#+}} xmm1 = xmm1[2],mem[2],xmm1[3],mem[3] sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_unpckhps:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vunpckhps {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3] sched: [1:0.50]
-; ZNVER1-NEXT:    vunpckhps {{.*#+}} xmm1 = xmm1[2],mem[2],xmm1[3],mem[3] sched: [8:0.50]
-; ZNVER1-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = shufflevector <4 x float> %a0, <4 x float> %a1, <4 x i32> <i32 2, i32 6, i32 3, i32 7>
-  %2 = load <4 x float>, <4 x float> *%a2, align 16
-  %3 = shufflevector <4 x float> %a1, <4 x float> %2, <4 x i32> <i32 2, i32 6, i32 3, i32 7>
-  %4 = fadd <4 x float> %1, %3
-  ret <4 x float> %4
-}
-
-define <4 x float> @test_unpcklps(<4 x float> %a0, <4 x float> %a1, <4 x float> *%a2) {
-; GENERIC-LABEL: test_unpcklps:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    unpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] sched: [1:1.00]
-; GENERIC-NEXT:    unpcklps {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[1],mem[1] sched: [7:1.00]
-; GENERIC-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_unpcklps:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    unpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] sched: [1:1.00]
-; ATOM-NEXT:    unpcklps {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[1],mem[1] sched: [1:1.00]
-; ATOM-NEXT:    addps %xmm1, %xmm0 # sched: [5:5.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_unpcklps:
-; SLM:       # %bb.0:
-; SLM-NEXT:    unpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] sched: [1:1.00]
-; SLM-NEXT:    unpcklps {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[1],mem[1] sched: [4:1.00]
-; SLM-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_unpcklps:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    unpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] sched: [1:1.00]
-; SANDY-SSE-NEXT:    unpcklps {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[1],mem[1] sched: [7:1.00]
-; SANDY-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_unpcklps:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vunpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] sched: [1:1.00]
-; SANDY-NEXT:    vunpcklps {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[1],mem[1] sched: [7:1.00]
-; SANDY-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_unpcklps:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    unpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] sched: [1:1.00]
-; HASWELL-SSE-NEXT:    unpcklps {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[1],mem[1] sched: [7:1.00]
-; HASWELL-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_unpcklps:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vunpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] sched: [1:1.00]
-; HASWELL-NEXT:    vunpcklps {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[1],mem[1] sched: [7:1.00]
-; HASWELL-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_unpcklps:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    unpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] sched: [1:1.00]
-; BROADWELL-SSE-NEXT:    unpcklps {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[1],mem[1] sched: [6:1.00]
-; BROADWELL-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_unpcklps:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vunpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] sched: [1:1.00]
-; BROADWELL-NEXT:    vunpcklps {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[1],mem[1] sched: [6:1.00]
-; BROADWELL-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_unpcklps:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    unpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] sched: [1:1.00]
-; SKYLAKE-SSE-NEXT:    unpcklps {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[1],mem[1] sched: [7:1.00]
-; SKYLAKE-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [4:0.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_unpcklps:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vunpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] sched: [1:1.00]
-; SKYLAKE-NEXT:    vunpcklps {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[1],mem[1] sched: [7:1.00]
-; SKYLAKE-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_unpcklps:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    unpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] sched: [1:1.00]
-; SKX-SSE-NEXT:    unpcklps {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[1],mem[1] sched: [7:1.00]
-; SKX-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [4:0.50]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_unpcklps:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vunpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] sched: [1:1.00]
-; SKX-NEXT:    vunpcklps {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[1],mem[1] sched: [7:1.00]
-; SKX-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_unpcklps:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    unpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] sched: [2:0.50]
-; BDVER2-SSE-NEXT:    unpcklps {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[1],mem[1] sched: [7:0.50]
-; BDVER2-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [5:1.00]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_unpcklps:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vunpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] sched: [2:0.50]
-; BDVER2-NEXT:    vunpcklps {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[1],mem[1] sched: [7:0.50]
-; BDVER2-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [5:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_unpcklps:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    unpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] sched: [1:0.50]
-; BTVER2-SSE-NEXT:    unpcklps {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[1],mem[1] sched: [6:1.00]
-; BTVER2-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_unpcklps:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vunpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] sched: [1:0.50]
-; BTVER2-NEXT:    vunpcklps {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[1],mem[1] sched: [6:1.00]
-; BTVER2-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_unpcklps:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    unpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] sched: [1:0.50]
-; ZNVER1-SSE-NEXT:    unpcklps {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[1],mem[1] sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_unpcklps:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vunpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] sched: [1:0.50]
-; ZNVER1-NEXT:    vunpcklps {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[1],mem[1] sched: [8:0.50]
-; ZNVER1-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = shufflevector <4 x float> %a0, <4 x float> %a1, <4 x i32> <i32 0, i32 4, i32 1, i32 5>
-  %2 = load <4 x float>, <4 x float> *%a2, align 16
-  %3 = shufflevector <4 x float> %a1, <4 x float> %2, <4 x i32> <i32 0, i32 4, i32 1, i32 5>
-  %4 = fadd <4 x float> %1, %3
-  ret <4 x float> %4
-}
-
-define <4 x float> @test_xorps(<4 x float> %a0, <4 x float> %a1, <4 x float> *%a2) {
-; GENERIC-LABEL: test_xorps:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    xorps %xmm1, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    xorps (%rdi), %xmm0 # sched: [7:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_xorps:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    xorps %xmm1, %xmm0 # sched: [1:0.50]
-; ATOM-NEXT:    xorps (%rdi), %xmm0 # sched: [1:1.00]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_xorps:
-; SLM:       # %bb.0:
-; SLM-NEXT:    xorps %xmm1, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    xorps (%rdi), %xmm0 # sched: [4:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_xorps:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    xorps %xmm1, %xmm0 # sched: [1:1.00]
-; SANDY-SSE-NEXT:    xorps (%rdi), %xmm0 # sched: [7:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_xorps:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vxorps %xmm1, %xmm0, %xmm0 # sched: [1:1.00]
-; SANDY-NEXT:    vxorps (%rdi), %xmm0, %xmm0 # sched: [7:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_xorps:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    xorps %xmm1, %xmm0 # sched: [1:1.00]
-; HASWELL-SSE-NEXT:    xorps (%rdi), %xmm0 # sched: [7:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_xorps:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vxorps %xmm1, %xmm0, %xmm0 # sched: [1:1.00]
-; HASWELL-NEXT:    vxorps (%rdi), %xmm0, %xmm0 # sched: [7:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_xorps:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    xorps %xmm1, %xmm0 # sched: [1:1.00]
-; BROADWELL-SSE-NEXT:    xorps (%rdi), %xmm0 # sched: [6:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_xorps:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vxorps %xmm1, %xmm0, %xmm0 # sched: [1:1.00]
-; BROADWELL-NEXT:    vxorps (%rdi), %xmm0, %xmm0 # sched: [6:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_xorps:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    xorps %xmm1, %xmm0 # sched: [1:0.33]
-; SKYLAKE-SSE-NEXT:    xorps (%rdi), %xmm0 # sched: [7:0.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_xorps:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vxorps %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    vxorps (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_xorps:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    xorps %xmm1, %xmm0 # sched: [1:0.33]
-; SKX-SSE-NEXT:    xorps (%rdi), %xmm0 # sched: [7:0.50]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_xorps:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vxorps %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    vxorps (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_xorps:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    xorps %xmm1, %xmm0 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    xorps (%rdi), %xmm0 # sched: [7:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_xorps:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vxorps %xmm1, %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    vxorps (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_xorps:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    xorps %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    xorps (%rdi), %xmm0 # sched: [6:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_xorps:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vxorps %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    vxorps (%rdi), %xmm0, %xmm0 # sched: [6:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_xorps:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    xorps %xmm1, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    xorps (%rdi), %xmm0 # sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_xorps:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vxorps %xmm1, %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vxorps (%rdi), %xmm0, %xmm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = bitcast <4 x float> %a0 to <4 x i32>
-  %2 = bitcast <4 x float> %a1 to <4 x i32>
-  %3 = xor <4 x i32> %1, %2
-  %4 = load <4 x float>, <4 x float> *%a2, align 16
-  %5 = bitcast <4 x float> %4 to <4 x i32>
-  %6 = xor <4 x i32> %3, %5
-  %7 = bitcast <4 x i32> %6 to <4 x float>
-  ret <4 x float> %7
-}
-
-; 'WriteZero' and 'WriteNop' class instructions.
-
-define <4 x float> @test_fnop() nounwind {
-; GENERIC-LABEL: test_fnop:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    nop # sched: [1:0.25]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    xorps %xmm0, %xmm0 # sched: [0:0.25]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_fnop:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    xorps %xmm0, %xmm0 # sched: [1:0.50]
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_fnop:
-; SLM:       # %bb.0:
-; SLM-NEXT:    xorps %xmm0, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    #APP
-; SLM-NEXT:    nop # sched: [1:0.50]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_fnop:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    #APP
-; SANDY-SSE-NEXT:    nop # sched: [1:0.25]
-; SANDY-SSE-NEXT:    #NO_APP
-; SANDY-SSE-NEXT:    xorps %xmm0, %xmm0 # sched: [0:0.25]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_fnop:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    nop # sched: [1:0.25]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    vxorps %xmm0, %xmm0, %xmm0 # sched: [0:0.25]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_fnop:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    #APP
-; HASWELL-SSE-NEXT:    nop # sched: [1:0.25]
-; HASWELL-SSE-NEXT:    #NO_APP
-; HASWELL-SSE-NEXT:    xorps %xmm0, %xmm0 # sched: [1:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_fnop:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    nop # sched: [1:0.25]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    vxorps %xmm0, %xmm0, %xmm0 # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_fnop:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    #APP
-; BROADWELL-SSE-NEXT:    nop # sched: [1:0.25]
-; BROADWELL-SSE-NEXT:    #NO_APP
-; BROADWELL-SSE-NEXT:    xorps %xmm0, %xmm0 # sched: [1:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_fnop:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    nop # sched: [1:0.25]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    vxorps %xmm0, %xmm0, %xmm0 # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_fnop:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    #APP
-; SKYLAKE-SSE-NEXT:    nop # sched: [1:0.17]
-; SKYLAKE-SSE-NEXT:    #NO_APP
-; SKYLAKE-SSE-NEXT:    xorps %xmm0, %xmm0 # sched: [1:0.33]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_fnop:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    nop # sched: [1:0.17]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    vxorps %xmm0, %xmm0, %xmm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_fnop:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    #APP
-; SKX-SSE-NEXT:    nop # sched: [1:0.17]
-; SKX-SSE-NEXT:    #NO_APP
-; SKX-SSE-NEXT:    xorps %xmm0, %xmm0 # sched: [1:0.33]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_fnop:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    nop # sched: [1:0.17]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    vxorps %xmm0, %xmm0, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_fnop:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    xorps %xmm0, %xmm0 # sched: [0:0.25]
-; BDVER2-SSE-NEXT:    #APP
-; BDVER2-SSE-NEXT:    nop # sched: [1:0.50]
-; BDVER2-SSE-NEXT:    #NO_APP
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_fnop:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vxorps %xmm0, %xmm0, %xmm0 # sched: [0:0.25]
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    nop # sched: [1:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_fnop:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    xorps %xmm0, %xmm0 # sched: [0:0.50]
-; BTVER2-SSE-NEXT:    #APP
-; BTVER2-SSE-NEXT:    nop # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    #NO_APP
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_fnop:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vxorps %xmm0, %xmm0, %xmm0 # sched: [0:0.50]
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    nop # sched: [1:0.50]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_fnop:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    xorps %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    #APP
-; ZNVER1-SSE-NEXT:    nop # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    #NO_APP
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_fnop:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vxorps %xmm0, %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    nop # sched: [1:0.25]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  tail call void asm sideeffect "nop", ""() nounwind
-  ret <4 x float> zeroinitializer
-}
-
-!0 = !{i32 1}
diff --git a/test/CodeGen/X86/sse2-intrinsics-fast-isel.ll b/test/CodeGen/X86/sse2-intrinsics-fast-isel.ll
index 23d0d66acfbca50966e73886bb737ea21a3a488e..477ce131df5c247680d87335512fc8d406be5cb9 100644
--- a/test/CodeGen/X86/sse2-intrinsics-fast-isel.ll
+++ b/test/CodeGen/X86/sse2-intrinsics-fast-isel.ll
@@ -151,11 +151,11 @@ define <2 x i64> @test_mm_adds_epi8(<2 x i64> %a0, <2 x i64> %a1) nounwind {
 ; AVX512-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
   %arg0 = bitcast <2 x i64> %a0 to <16 x i8>
   %arg1 = bitcast <2 x i64> %a1 to <16 x i8>
-  %res = call <16 x i8> @llvm.x86.sse2.padds.b(<16 x i8> %arg0, <16 x i8> %arg1)
+  %res = call <16 x i8> @llvm.sadd.sat.v16i8(<16 x i8> %arg0, <16 x i8> %arg1)
   %bc = bitcast <16 x i8> %res to <2 x i64>
   ret <2 x i64> %bc
 }
-declare <16 x i8> @llvm.x86.sse2.padds.b(<16 x i8>, <16 x i8>) nounwind readnone
+declare <16 x i8> @llvm.sadd.sat.v16i8(<16 x i8>, <16 x i8>) nounwind readnone
 
 define <2 x i64> @test_mm_adds_epi16(<2 x i64> %a0, <2 x i64> %a1) nounwind {
 ; SSE-LABEL: test_mm_adds_epi16:
@@ -174,11 +174,11 @@ define <2 x i64> @test_mm_adds_epi16(<2 x i64> %a0, <2 x i64> %a1) nounwind {
 ; AVX512-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
   %arg0 = bitcast <2 x i64> %a0 to <8 x i16>
   %arg1 = bitcast <2 x i64> %a1 to <8 x i16>
-  %res = call <8 x i16> @llvm.x86.sse2.padds.w(<8 x i16> %arg0, <8 x i16> %arg1)
+  %res = call <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16> %arg0, <8 x i16> %arg1)
   %bc = bitcast <8 x i16> %res to <2 x i64>
   ret <2 x i64> %bc
 }
-declare <8 x i16> @llvm.x86.sse2.padds.w(<8 x i16>, <8 x i16>) nounwind readnone
+declare <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16>, <8 x i16>) nounwind readnone
 
 define <2 x i64> @test_mm_adds_epu8(<2 x i64> %a0, <2 x i64> %a1) nounwind {
 ; SSE-LABEL: test_mm_adds_epu8:
@@ -197,12 +197,11 @@ define <2 x i64> @test_mm_adds_epu8(<2 x i64> %a0, <2 x i64> %a1) nounwind {
 ; AVX512-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
   %arg0 = bitcast <2 x i64> %a0 to <16 x i8>
   %arg1 = bitcast <2 x i64> %a1 to <16 x i8>
-  %1 = add <16 x i8> %arg0, %arg1
-  %2 = icmp ugt <16 x i8> %arg0, %1
-  %3 = select <16 x i1> %2, <16 x i8> <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>, <16 x i8> %1
-  %bc = bitcast <16 x i8> %3 to <2 x i64>
+  %res = call <16 x i8> @llvm.uadd.sat.v16i8(<16 x i8> %arg0, <16 x i8> %arg1)
+  %bc = bitcast <16 x i8> %res to <2 x i64>
   ret <2 x i64> %bc
 }
+declare <16 x i8> @llvm.uadd.sat.v16i8(<16 x i8>, <16 x i8>)
 
 define <2 x i64> @test_mm_adds_epu16(<2 x i64> %a0, <2 x i64> %a1) nounwind {
 ; SSE-LABEL: test_mm_adds_epu16:
@@ -221,12 +220,11 @@ define <2 x i64> @test_mm_adds_epu16(<2 x i64> %a0, <2 x i64> %a1) nounwind {
 ; AVX512-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
   %arg0 = bitcast <2 x i64> %a0 to <8 x i16>
   %arg1 = bitcast <2 x i64> %a1 to <8 x i16>
-  %1 = add <8 x i16> %arg0, %arg1
-  %2 = icmp ugt <8 x i16> %arg0, %1
-  %3 = select <8 x i1> %2, <8 x i16> <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>, <8 x i16> %1
-  %bc = bitcast <8 x i16> %3 to <2 x i64>
+  %res = call <8 x i16> @llvm.uadd.sat.v8i16(<8 x i16> %arg0, <8 x i16> %arg1)
+  %bc = bitcast <8 x i16> %res to <2 x i64>
   ret <2 x i64> %bc
 }
+declare <8 x i16> @llvm.uadd.sat.v8i16(<8 x i16>, <8 x i16>)
 
 define <2 x double> @test_mm_and_pd(<2 x double> %a0, <2 x double> %a1) nounwind {
 ; SSE-LABEL: test_mm_and_pd:
@@ -2760,23 +2758,13 @@ define i32 @test_mm_movemask_pd(<2 x double> %a0) nounwind {
 declare i32 @llvm.x86.sse2.movmsk.pd(<2 x double>) nounwind readnone
 
 define <2 x i64> @test_mm_mul_epu32(<2 x i64> %a0, <2 x i64> %a1) nounwind {
-; X86-SSE-LABEL: test_mm_mul_epu32:
-; X86-SSE:       # %bb.0:
-; X86-SSE-NEXT:    movdqa {{.*#+}} xmm2 = [4294967295,0,4294967295,0]
-; X86-SSE-NEXT:    # encoding: [0x66,0x0f,0x6f,0x15,A,A,A,A]
-; X86-SSE-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}, kind: FK_Data_4
-; X86-SSE-NEXT:    pand %xmm2, %xmm0 # encoding: [0x66,0x0f,0xdb,0xc2]
-; X86-SSE-NEXT:    pand %xmm2, %xmm1 # encoding: [0x66,0x0f,0xdb,0xca]
-; X86-SSE-NEXT:    pmuludq %xmm1, %xmm0 # encoding: [0x66,0x0f,0xf4,0xc1]
-; X86-SSE-NEXT:    retl # encoding: [0xc3]
+; SSE-LABEL: test_mm_mul_epu32:
+; SSE:       # %bb.0:
+; SSE-NEXT:    pmuludq %xmm1, %xmm0 # encoding: [0x66,0x0f,0xf4,0xc1]
+; SSE-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
 ;
 ; AVX1-LABEL: test_mm_mul_epu32:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2 # encoding: [0xc5,0xe9,0xef,0xd2]
-; AVX1-NEXT:    vpblendw $204, %xmm2, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x0e,0xc2,0xcc]
-; AVX1-NEXT:    # xmm0 = xmm0[0,1],xmm2[2,3],xmm0[4,5],xmm2[6,7]
-; AVX1-NEXT:    vpblendw $204, %xmm2, %xmm1, %xmm1 # encoding: [0xc4,0xe3,0x71,0x0e,0xca,0xcc]
-; AVX1-NEXT:    # xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
 ; AVX1-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xf4,0xc1]
 ; AVX1-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
 ;
@@ -2789,16 +2777,6 @@ define <2 x i64> @test_mm_mul_epu32(<2 x i64> %a0, <2 x i64> %a1) nounwind {
 ; AVX512-NEXT:    # xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3]
 ; AVX512-NEXT:    vpmullq %xmm1, %xmm0, %xmm0 # encoding: [0x62,0xf2,0xfd,0x08,0x40,0xc1]
 ; AVX512-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
-;
-; X64-SSE-LABEL: test_mm_mul_epu32:
-; X64-SSE:       # %bb.0:
-; X64-SSE-NEXT:    movdqa {{.*#+}} xmm2 = [4294967295,0,4294967295,0]
-; X64-SSE-NEXT:    # encoding: [0x66,0x0f,0x6f,0x15,A,A,A,A]
-; X64-SSE-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
-; X64-SSE-NEXT:    pand %xmm2, %xmm0 # encoding: [0x66,0x0f,0xdb,0xc2]
-; X64-SSE-NEXT:    pand %xmm2, %xmm1 # encoding: [0x66,0x0f,0xdb,0xca]
-; X64-SSE-NEXT:    pmuludq %xmm1, %xmm0 # encoding: [0x66,0x0f,0xf4,0xc1]
-; X64-SSE-NEXT:    retq # encoding: [0xc3]
   %A = and <2 x i64> %a0, <i64 4294967295, i64 4294967295>
   %B = and <2 x i64> %a1, <i64 4294967295, i64 4294967295>
   %res = mul nuw <2 x i64> %A, %B
@@ -3981,12 +3959,11 @@ define <2 x i64> @test_mm_set1_epi64x(i64 %a0) nounwind {
 ;
 ; X86-AVX1-LABEL: test_mm_set1_epi64x:
 ; X86-AVX1:       # %bb.0:
-; X86-AVX1-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x08]
-; X86-AVX1-NEXT:    movl {{[0-9]+}}(%esp), %ecx # encoding: [0x8b,0x4c,0x24,0x04]
-; X86-AVX1-NEXT:    vmovd %ecx, %xmm0 # encoding: [0xc5,0xf9,0x6e,0xc1]
-; X86-AVX1-NEXT:    vpinsrd $1, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x22,0xc0,0x01]
-; X86-AVX1-NEXT:    vpinsrd $2, %ecx, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x22,0xc1,0x02]
-; X86-AVX1-NEXT:    vpinsrd $3, %eax, %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x22,0xc0,0x03]
+; X86-AVX1-NEXT:    vmovd {{[0-9]+}}(%esp), %xmm0 # encoding: [0xc5,0xf9,0x6e,0x44,0x24,0x04]
+; X86-AVX1-NEXT:    # xmm0 = mem[0],zero,zero,zero
+; X86-AVX1-NEXT:    vpinsrd $1, {{[0-9]+}}(%esp), %xmm0, %xmm0 # encoding: [0xc4,0xe3,0x79,0x22,0x44,0x24,0x08,0x01]
+; X86-AVX1-NEXT:    vpshufd $68, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0x70,0xc0,0x44]
+; X86-AVX1-NEXT:    # xmm0 = xmm0[0,1,0,1]
 ; X86-AVX1-NEXT:    retl # encoding: [0xc3]
 ;
 ; X86-AVX512-LABEL: test_mm_set1_epi64x:
@@ -5386,7 +5363,7 @@ define void @test_mm_store_pd1(double *%a0, <2 x double> %a1) {
 ; X86-AVX1-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
 ; X86-AVX1-NEXT:    vmovddup %xmm0, %xmm0 # encoding: [0xc5,0xfb,0x12,0xc0]
 ; X86-AVX1-NEXT:    # xmm0 = xmm0[0,0]
-; X86-AVX1-NEXT:    vmovapd %xmm0, (%eax) # encoding: [0xc5,0xf9,0x29,0x00]
+; X86-AVX1-NEXT:    vmovaps %xmm0, (%eax) # encoding: [0xc5,0xf8,0x29,0x00]
 ; X86-AVX1-NEXT:    retl # encoding: [0xc3]
 ;
 ; X86-AVX512-LABEL: test_mm_store_pd1:
@@ -5394,7 +5371,7 @@ define void @test_mm_store_pd1(double *%a0, <2 x double> %a1) {
 ; X86-AVX512-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
 ; X86-AVX512-NEXT:    vmovddup %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xfb,0x12,0xc0]
 ; X86-AVX512-NEXT:    # xmm0 = xmm0[0,0]
-; X86-AVX512-NEXT:    vmovapd %xmm0, (%eax) # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x29,0x00]
+; X86-AVX512-NEXT:    vmovaps %xmm0, (%eax) # EVEX TO VEX Compression encoding: [0xc5,0xf8,0x29,0x00]
 ; X86-AVX512-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-SSE-LABEL: test_mm_store_pd1:
@@ -5408,14 +5385,14 @@ define void @test_mm_store_pd1(double *%a0, <2 x double> %a1) {
 ; X64-AVX1:       # %bb.0:
 ; X64-AVX1-NEXT:    vmovddup %xmm0, %xmm0 # encoding: [0xc5,0xfb,0x12,0xc0]
 ; X64-AVX1-NEXT:    # xmm0 = xmm0[0,0]
-; X64-AVX1-NEXT:    vmovapd %xmm0, (%rdi) # encoding: [0xc5,0xf9,0x29,0x07]
+; X64-AVX1-NEXT:    vmovaps %xmm0, (%rdi) # encoding: [0xc5,0xf8,0x29,0x07]
 ; X64-AVX1-NEXT:    retq # encoding: [0xc3]
 ;
 ; X64-AVX512-LABEL: test_mm_store_pd1:
 ; X64-AVX512:       # %bb.0:
 ; X64-AVX512-NEXT:    vmovddup %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xfb,0x12,0xc0]
 ; X64-AVX512-NEXT:    # xmm0 = xmm0[0,0]
-; X64-AVX512-NEXT:    vmovapd %xmm0, (%rdi) # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x29,0x07]
+; X64-AVX512-NEXT:    vmovaps %xmm0, (%rdi) # EVEX TO VEX Compression encoding: [0xc5,0xf8,0x29,0x07]
 ; X64-AVX512-NEXT:    retq # encoding: [0xc3]
   %arg0 = bitcast double * %a0 to <2 x double>*
   %shuf = shufflevector <2 x double> %a1, <2 x double> undef, <2 x i32> zeroinitializer
@@ -5512,7 +5489,7 @@ define void @test_mm_store1_pd(double *%a0, <2 x double> %a1) {
 ; X86-AVX1-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
 ; X86-AVX1-NEXT:    vmovddup %xmm0, %xmm0 # encoding: [0xc5,0xfb,0x12,0xc0]
 ; X86-AVX1-NEXT:    # xmm0 = xmm0[0,0]
-; X86-AVX1-NEXT:    vmovapd %xmm0, (%eax) # encoding: [0xc5,0xf9,0x29,0x00]
+; X86-AVX1-NEXT:    vmovaps %xmm0, (%eax) # encoding: [0xc5,0xf8,0x29,0x00]
 ; X86-AVX1-NEXT:    retl # encoding: [0xc3]
 ;
 ; X86-AVX512-LABEL: test_mm_store1_pd:
@@ -5520,7 +5497,7 @@ define void @test_mm_store1_pd(double *%a0, <2 x double> %a1) {
 ; X86-AVX512-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
 ; X86-AVX512-NEXT:    vmovddup %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xfb,0x12,0xc0]
 ; X86-AVX512-NEXT:    # xmm0 = xmm0[0,0]
-; X86-AVX512-NEXT:    vmovapd %xmm0, (%eax) # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x29,0x00]
+; X86-AVX512-NEXT:    vmovaps %xmm0, (%eax) # EVEX TO VEX Compression encoding: [0xc5,0xf8,0x29,0x00]
 ; X86-AVX512-NEXT:    retl # encoding: [0xc3]
 ;
 ; X64-SSE-LABEL: test_mm_store1_pd:
@@ -5534,14 +5511,14 @@ define void @test_mm_store1_pd(double *%a0, <2 x double> %a1) {
 ; X64-AVX1:       # %bb.0:
 ; X64-AVX1-NEXT:    vmovddup %xmm0, %xmm0 # encoding: [0xc5,0xfb,0x12,0xc0]
 ; X64-AVX1-NEXT:    # xmm0 = xmm0[0,0]
-; X64-AVX1-NEXT:    vmovapd %xmm0, (%rdi) # encoding: [0xc5,0xf9,0x29,0x07]
+; X64-AVX1-NEXT:    vmovaps %xmm0, (%rdi) # encoding: [0xc5,0xf8,0x29,0x07]
 ; X64-AVX1-NEXT:    retq # encoding: [0xc3]
 ;
 ; X64-AVX512-LABEL: test_mm_store1_pd:
 ; X64-AVX512:       # %bb.0:
 ; X64-AVX512-NEXT:    vmovddup %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xfb,0x12,0xc0]
 ; X64-AVX512-NEXT:    # xmm0 = xmm0[0,0]
-; X64-AVX512-NEXT:    vmovapd %xmm0, (%rdi) # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x29,0x07]
+; X64-AVX512-NEXT:    vmovaps %xmm0, (%rdi) # EVEX TO VEX Compression encoding: [0xc5,0xf8,0x29,0x07]
 ; X64-AVX512-NEXT:    retq # encoding: [0xc3]
   %arg0 = bitcast double * %a0 to <2 x double>*
   %shuf = shufflevector <2 x double> %a1, <2 x double> undef, <2 x i32> zeroinitializer
@@ -6176,11 +6153,11 @@ define <2 x i64> @test_mm_subs_epi8(<2 x i64> %a0, <2 x i64> %a1) nounwind {
 ; AVX512-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
   %arg0 = bitcast <2 x i64> %a0 to <16 x i8>
   %arg1 = bitcast <2 x i64> %a1 to <16 x i8>
-  %res = call <16 x i8> @llvm.x86.sse2.psubs.b(<16 x i8> %arg0, <16 x i8> %arg1)
+  %res = call <16 x i8> @llvm.ssub.sat.v16i8(<16 x i8> %arg0, <16 x i8> %arg1)
   %bc = bitcast <16 x i8> %res to <2 x i64>
   ret <2 x i64> %bc
 }
-declare <16 x i8> @llvm.x86.sse2.psubs.b(<16 x i8>, <16 x i8>) nounwind readnone
+declare <16 x i8> @llvm.ssub.sat.v16i8(<16 x i8>, <16 x i8>) nounwind readnone
 
 define <2 x i64> @test_mm_subs_epi16(<2 x i64> %a0, <2 x i64> %a1) nounwind {
 ; SSE-LABEL: test_mm_subs_epi16:
@@ -6199,85 +6176,57 @@ define <2 x i64> @test_mm_subs_epi16(<2 x i64> %a0, <2 x i64> %a1) nounwind {
 ; AVX512-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
   %arg0 = bitcast <2 x i64> %a0 to <8 x i16>
   %arg1 = bitcast <2 x i64> %a1 to <8 x i16>
-  %res = call <8 x i16> @llvm.x86.sse2.psubs.w(<8 x i16> %arg0, <8 x i16> %arg1)
+  %res = call <8 x i16> @llvm.ssub.sat.v8i16(<8 x i16> %arg0, <8 x i16> %arg1)
   %bc = bitcast <8 x i16> %res to <2 x i64>
   ret <2 x i64> %bc
 }
-declare <8 x i16> @llvm.x86.sse2.psubs.w(<8 x i16>, <8 x i16>) nounwind readnone
+declare <8 x i16> @llvm.ssub.sat.v8i16(<8 x i16>, <8 x i16>) nounwind readnone
 
 define <2 x i64> @test_mm_subs_epu8(<2 x i64> %a0, <2 x i64> %a1) nounwind {
 ; SSE-LABEL: test_mm_subs_epu8:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    pmaxub %xmm1, %xmm0 # encoding: [0x66,0x0f,0xde,0xc1]
-; SSE-NEXT:    psubb %xmm1, %xmm0 # encoding: [0x66,0x0f,0xf8,0xc1]
+; SSE-NEXT:    psubusb %xmm1, %xmm0 # encoding: [0x66,0x0f,0xd8,0xc1]
 ; SSE-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
 ;
 ; AVX1-LABEL: test_mm_subs_epu8:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpmaxub %xmm1, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xde,0xc1]
-; AVX1-NEXT:    vpsubb %xmm1, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xf8,0xc1]
+; AVX1-NEXT:    vpsubusb %xmm1, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xd8,0xc1]
 ; AVX1-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
 ;
 ; AVX512-LABEL: test_mm_subs_epu8:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpmaxub %xmm1, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xde,0xc1]
-; AVX512-NEXT:    vpsubb %xmm1, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xf8,0xc1]
+; AVX512-NEXT:    vpsubusb %xmm1, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xd8,0xc1]
 ; AVX512-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
   %arg0 = bitcast <2 x i64> %a0 to <16 x i8>
   %arg1 = bitcast <2 x i64> %a1 to <16 x i8>
-  %cmp = icmp ugt <16 x i8> %arg0, %arg1
-  %sel = select <16 x i1> %cmp, <16 x i8> %arg0, <16 x i8> %arg1
-  %sub = sub <16 x i8> %sel, %arg1
-  %bc = bitcast <16 x i8> %sub to <2 x i64>
+  %res = call <16 x i8> @llvm.usub.sat.v16i8(<16 x i8> %arg0, <16 x i8> %arg1)
+  %bc = bitcast <16 x i8> %res to <2 x i64>
   ret <2 x i64> %bc
 }
+declare <16 x i8> @llvm.usub.sat.v16i8(<16 x i8>, <16 x i8>)
 
 define <2 x i64> @test_mm_subs_epu16(<2 x i64> %a0, <2 x i64> %a1) nounwind {
-; X86-SSE-LABEL: test_mm_subs_epu16:
-; X86-SSE:       # %bb.0:
-; X86-SSE-NEXT:    movdqa {{.*#+}} xmm2 = [32768,32768,32768,32768,32768,32768,32768,32768]
-; X86-SSE-NEXT:    # encoding: [0x66,0x0f,0x6f,0x15,A,A,A,A]
-; X86-SSE-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}, kind: FK_Data_4
-; X86-SSE-NEXT:    movdqa %xmm1, %xmm3 # encoding: [0x66,0x0f,0x6f,0xd9]
-; X86-SSE-NEXT:    pxor %xmm2, %xmm3 # encoding: [0x66,0x0f,0xef,0xda]
-; X86-SSE-NEXT:    pxor %xmm2, %xmm0 # encoding: [0x66,0x0f,0xef,0xc2]
-; X86-SSE-NEXT:    pmaxsw %xmm3, %xmm0 # encoding: [0x66,0x0f,0xee,0xc3]
-; X86-SSE-NEXT:    pxor %xmm2, %xmm0 # encoding: [0x66,0x0f,0xef,0xc2]
-; X86-SSE-NEXT:    psubw %xmm1, %xmm0 # encoding: [0x66,0x0f,0xf9,0xc1]
-; X86-SSE-NEXT:    retl # encoding: [0xc3]
+; SSE-LABEL: test_mm_subs_epu16:
+; SSE:       # %bb.0:
+; SSE-NEXT:    psubusw %xmm1, %xmm0 # encoding: [0x66,0x0f,0xd9,0xc1]
+; SSE-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
 ;
 ; AVX1-LABEL: test_mm_subs_epu16:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpmaxuw %xmm1, %xmm0, %xmm0 # encoding: [0xc4,0xe2,0x79,0x3e,0xc1]
-; AVX1-NEXT:    vpsubw %xmm1, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xf9,0xc1]
+; AVX1-NEXT:    vpsubusw %xmm1, %xmm0, %xmm0 # encoding: [0xc5,0xf9,0xd9,0xc1]
 ; AVX1-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
 ;
 ; AVX512-LABEL: test_mm_subs_epu16:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpmaxuw %xmm1, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x79,0x3e,0xc1]
-; AVX512-NEXT:    vpsubw %xmm1, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xf9,0xc1]
+; AVX512-NEXT:    vpsubusw %xmm1, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xd9,0xc1]
 ; AVX512-NEXT:    ret{{[l|q]}} # encoding: [0xc3]
-;
-; X64-SSE-LABEL: test_mm_subs_epu16:
-; X64-SSE:       # %bb.0:
-; X64-SSE-NEXT:    movdqa {{.*#+}} xmm2 = [32768,32768,32768,32768,32768,32768,32768,32768]
-; X64-SSE-NEXT:    # encoding: [0x66,0x0f,0x6f,0x15,A,A,A,A]
-; X64-SSE-NEXT:    # fixup A - offset: 4, value: {{\.LCPI.*}}-4, kind: reloc_riprel_4byte
-; X64-SSE-NEXT:    movdqa %xmm1, %xmm3 # encoding: [0x66,0x0f,0x6f,0xd9]
-; X64-SSE-NEXT:    pxor %xmm2, %xmm3 # encoding: [0x66,0x0f,0xef,0xda]
-; X64-SSE-NEXT:    pxor %xmm2, %xmm0 # encoding: [0x66,0x0f,0xef,0xc2]
-; X64-SSE-NEXT:    pmaxsw %xmm3, %xmm0 # encoding: [0x66,0x0f,0xee,0xc3]
-; X64-SSE-NEXT:    pxor %xmm2, %xmm0 # encoding: [0x66,0x0f,0xef,0xc2]
-; X64-SSE-NEXT:    psubw %xmm1, %xmm0 # encoding: [0x66,0x0f,0xf9,0xc1]
-; X64-SSE-NEXT:    retq # encoding: [0xc3]
   %arg0 = bitcast <2 x i64> %a0 to <8 x i16>
   %arg1 = bitcast <2 x i64> %a1 to <8 x i16>
-  %cmp = icmp ugt <8 x i16> %arg0, %arg1
-  %sel = select <8 x i1> %cmp, <8 x i16> %arg0, <8 x i16> %arg1
-  %sub = sub <8 x i16> %sel, %arg1
-  %bc = bitcast <8 x i16> %sub to <2 x i64>
+  %res = call <8 x i16> @llvm.usub.sat.v8i16(<8 x i16> %arg0, <8 x i16> %arg1)
+  %bc = bitcast <8 x i16> %res to <2 x i64>
   ret <2 x i64> %bc
 }
+declare <8 x i16> @llvm.usub.sat.v8i16(<8 x i16>, <8 x i16>)
 
 define i32 @test_mm_ucomieq_sd(<2 x double> %a0, <2 x double> %a1) nounwind {
 ; SSE-LABEL: test_mm_ucomieq_sd:
diff --git a/test/CodeGen/X86/sse2-intrinsics-x86-upgrade.ll b/test/CodeGen/X86/sse2-intrinsics-x86-upgrade.ll
index 7216f1927977ab87310448ceba5969d3b46bd281..7e4703f6957de7a512dfbc38ed2316e57967d01c 100644
--- a/test/CodeGen/X86/sse2-intrinsics-x86-upgrade.ll
+++ b/test/CodeGen/X86/sse2-intrinsics-x86-upgrade.ll
@@ -884,6 +884,48 @@ define <4 x float> @test_x86_sse2_cvtdq2ps(<4 x i32> %a0) {
 declare <4 x float> @llvm.x86.sse2.cvtdq2ps(<4 x i32>) nounwind readnone
 
 
+define <16 x i8> @test_x86_sse2_padds_b(<16 x i8> %a0, <16 x i8> %a1) {
+; SSE-LABEL: test_x86_sse2_padds_b:
+; SSE:       ## %bb.0:
+; SSE-NEXT:    paddsb %xmm1, %xmm0 ## encoding: [0x66,0x0f,0xec,0xc1]
+; SSE-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+;
+; AVX1-LABEL: test_x86_sse2_padds_b:
+; AVX1:       ## %bb.0:
+; AVX1-NEXT:    vpaddsb %xmm1, %xmm0, %xmm0 ## encoding: [0xc5,0xf9,0xec,0xc1]
+; AVX1-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+;
+; AVX512-LABEL: test_x86_sse2_padds_b:
+; AVX512:       ## %bb.0:
+; AVX512-NEXT:    vpaddsb %xmm1, %xmm0, %xmm0 ## EVEX TO VEX Compression encoding: [0xc5,0xf9,0xec,0xc1]
+; AVX512-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+  %res = call <16 x i8> @llvm.sadd.sat.v16i8(<16 x i8> %a0, <16 x i8> %a1) ; <<16 x i8>> [#uses=1]
+  ret <16 x i8> %res
+}
+declare <16 x i8> @llvm.sadd.sat.v16i8(<16 x i8>, <16 x i8>) nounwind readnone
+
+
+define <8 x i16> @test_x86_sse2_padds_w(<8 x i16> %a0, <8 x i16> %a1) {
+; SSE-LABEL: test_x86_sse2_padds_w:
+; SSE:       ## %bb.0:
+; SSE-NEXT:    paddsw %xmm1, %xmm0 ## encoding: [0x66,0x0f,0xed,0xc1]
+; SSE-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+;
+; AVX1-LABEL: test_x86_sse2_padds_w:
+; AVX1:       ## %bb.0:
+; AVX1-NEXT:    vpaddsw %xmm1, %xmm0, %xmm0 ## encoding: [0xc5,0xf9,0xed,0xc1]
+; AVX1-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+;
+; AVX512-LABEL: test_x86_sse2_padds_w:
+; AVX512:       ## %bb.0:
+; AVX512-NEXT:    vpaddsw %xmm1, %xmm0, %xmm0 ## EVEX TO VEX Compression encoding: [0xc5,0xf9,0xed,0xc1]
+; AVX512-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+  %res = call <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16> %a0, <8 x i16> %a1) ; <<8 x i16>> [#uses=1]
+  ret <8 x i16> %res
+}
+declare <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16>, <8 x i16>) nounwind readnone
+
+
 define <16 x i8> @test_x86_sse2_paddus_b(<16 x i8> %a0, <16 x i8> %a1) {
 ; SSE-LABEL: test_x86_sse2_paddus_b:
 ; SSE:       ## %bb.0:
@@ -926,6 +968,48 @@ define <8 x i16> @test_x86_sse2_paddus_w(<8 x i16> %a0, <8 x i16> %a1) {
 declare <8 x i16> @llvm.x86.sse2.paddus.w(<8 x i16>, <8 x i16>) nounwind readnone
 
 
+define <16 x i8> @test_x86_sse2_psubs_b(<16 x i8> %a0, <16 x i8> %a1) {
+; SSE-LABEL: test_x86_sse2_psubs_b:
+; SSE:       ## %bb.0:
+; SSE-NEXT:    psubsb %xmm1, %xmm0 ## encoding: [0x66,0x0f,0xe8,0xc1]
+; SSE-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+;
+; AVX1-LABEL: test_x86_sse2_psubs_b:
+; AVX1:       ## %bb.0:
+; AVX1-NEXT:    vpsubsb %xmm1, %xmm0, %xmm0 ## encoding: [0xc5,0xf9,0xe8,0xc1]
+; AVX1-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+;
+; AVX512-LABEL: test_x86_sse2_psubs_b:
+; AVX512:       ## %bb.0:
+; AVX512-NEXT:    vpsubsb %xmm1, %xmm0, %xmm0 ## EVEX TO VEX Compression encoding: [0xc5,0xf9,0xe8,0xc1]
+; AVX512-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+  %res = call <16 x i8> @llvm.ssub.sat.v16i8(<16 x i8> %a0, <16 x i8> %a1) ; <<16 x i8>> [#uses=1]
+  ret <16 x i8> %res
+}
+declare <16 x i8> @llvm.ssub.sat.v16i8(<16 x i8>, <16 x i8>) nounwind readnone
+
+
+define <8 x i16> @test_x86_sse2_psubs_w(<8 x i16> %a0, <8 x i16> %a1) {
+; SSE-LABEL: test_x86_sse2_psubs_w:
+; SSE:       ## %bb.0:
+; SSE-NEXT:    psubsw %xmm1, %xmm0 ## encoding: [0x66,0x0f,0xe9,0xc1]
+; SSE-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+;
+; AVX1-LABEL: test_x86_sse2_psubs_w:
+; AVX1:       ## %bb.0:
+; AVX1-NEXT:    vpsubsw %xmm1, %xmm0, %xmm0 ## encoding: [0xc5,0xf9,0xe9,0xc1]
+; AVX1-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+;
+; AVX512-LABEL: test_x86_sse2_psubs_w:
+; AVX512:       ## %bb.0:
+; AVX512-NEXT:    vpsubsw %xmm1, %xmm0, %xmm0 ## EVEX TO VEX Compression encoding: [0xc5,0xf9,0xe9,0xc1]
+; AVX512-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
+  %res = call <8 x i16> @llvm.ssub.sat.v8i16(<8 x i16> %a0, <8 x i16> %a1) ; <<8 x i16>> [#uses=1]
+  ret <8 x i16> %res
+}
+declare <8 x i16> @llvm.ssub.sat.v8i16(<8 x i16>, <8 x i16>) nounwind readnone
+
+
 define <16 x i8> @test_x86_sse2_psubus_b(<16 x i8> %a0, <16 x i8> %a1) {
 ; SSE-LABEL: test_x86_sse2_psubus_b:
 ; SSE:       ## %bb.0:
diff --git a/test/CodeGen/X86/sse2-intrinsics-x86.ll b/test/CodeGen/X86/sse2-intrinsics-x86.ll
index 8dedce5fc8b478dfbf5623aba68870d7126425af..f8a9074bc0032c9a5c35521afa9fae9cf6f778d1 100644
--- a/test/CodeGen/X86/sse2-intrinsics-x86.ll
+++ b/test/CodeGen/X86/sse2-intrinsics-x86.ll
@@ -919,48 +919,6 @@ define <16 x i8> @test_x86_sse2_packuswb_128_fold() {
 }
 
 
-define <16 x i8> @test_x86_sse2_padds_b(<16 x i8> %a0, <16 x i8> %a1) {
-; SSE-LABEL: test_x86_sse2_padds_b:
-; SSE:       ## %bb.0:
-; SSE-NEXT:    paddsb %xmm1, %xmm0 ## encoding: [0x66,0x0f,0xec,0xc1]
-; SSE-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
-;
-; AVX1-LABEL: test_x86_sse2_padds_b:
-; AVX1:       ## %bb.0:
-; AVX1-NEXT:    vpaddsb %xmm1, %xmm0, %xmm0 ## encoding: [0xc5,0xf9,0xec,0xc1]
-; AVX1-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
-;
-; AVX512-LABEL: test_x86_sse2_padds_b:
-; AVX512:       ## %bb.0:
-; AVX512-NEXT:    vpaddsb %xmm1, %xmm0, %xmm0 ## EVEX TO VEX Compression encoding: [0xc5,0xf9,0xec,0xc1]
-; AVX512-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
-  %res = call <16 x i8> @llvm.x86.sse2.padds.b(<16 x i8> %a0, <16 x i8> %a1) ; <<16 x i8>> [#uses=1]
-  ret <16 x i8> %res
-}
-declare <16 x i8> @llvm.x86.sse2.padds.b(<16 x i8>, <16 x i8>) nounwind readnone
-
-
-define <8 x i16> @test_x86_sse2_padds_w(<8 x i16> %a0, <8 x i16> %a1) {
-; SSE-LABEL: test_x86_sse2_padds_w:
-; SSE:       ## %bb.0:
-; SSE-NEXT:    paddsw %xmm1, %xmm0 ## encoding: [0x66,0x0f,0xed,0xc1]
-; SSE-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
-;
-; AVX1-LABEL: test_x86_sse2_padds_w:
-; AVX1:       ## %bb.0:
-; AVX1-NEXT:    vpaddsw %xmm1, %xmm0, %xmm0 ## encoding: [0xc5,0xf9,0xed,0xc1]
-; AVX1-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
-;
-; AVX512-LABEL: test_x86_sse2_padds_w:
-; AVX512:       ## %bb.0:
-; AVX512-NEXT:    vpaddsw %xmm1, %xmm0, %xmm0 ## EVEX TO VEX Compression encoding: [0xc5,0xf9,0xed,0xc1]
-; AVX512-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
-  %res = call <8 x i16> @llvm.x86.sse2.padds.w(<8 x i16> %a0, <8 x i16> %a1) ; <<8 x i16>> [#uses=1]
-  ret <8 x i16> %res
-}
-declare <8 x i16> @llvm.x86.sse2.padds.w(<8 x i16>, <8 x i16>) nounwind readnone
-
-
 define <4 x i32> @test_x86_sse2_pmadd_wd(<8 x i16> %a0, <8 x i16> %a1) {
 ; SSE-LABEL: test_x86_sse2_pmadd_wd:
 ; SSE:       ## %bb.0:
@@ -1520,48 +1478,6 @@ define <8 x i16> @test_x86_sse2_psrli_w(<8 x i16> %a0) {
 declare <8 x i16> @llvm.x86.sse2.psrli.w(<8 x i16>, i32) nounwind readnone
 
 
-define <16 x i8> @test_x86_sse2_psubs_b(<16 x i8> %a0, <16 x i8> %a1) {
-; SSE-LABEL: test_x86_sse2_psubs_b:
-; SSE:       ## %bb.0:
-; SSE-NEXT:    psubsb %xmm1, %xmm0 ## encoding: [0x66,0x0f,0xe8,0xc1]
-; SSE-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
-;
-; AVX1-LABEL: test_x86_sse2_psubs_b:
-; AVX1:       ## %bb.0:
-; AVX1-NEXT:    vpsubsb %xmm1, %xmm0, %xmm0 ## encoding: [0xc5,0xf9,0xe8,0xc1]
-; AVX1-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
-;
-; AVX512-LABEL: test_x86_sse2_psubs_b:
-; AVX512:       ## %bb.0:
-; AVX512-NEXT:    vpsubsb %xmm1, %xmm0, %xmm0 ## EVEX TO VEX Compression encoding: [0xc5,0xf9,0xe8,0xc1]
-; AVX512-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
-  %res = call <16 x i8> @llvm.x86.sse2.psubs.b(<16 x i8> %a0, <16 x i8> %a1) ; <<16 x i8>> [#uses=1]
-  ret <16 x i8> %res
-}
-declare <16 x i8> @llvm.x86.sse2.psubs.b(<16 x i8>, <16 x i8>) nounwind readnone
-
-
-define <8 x i16> @test_x86_sse2_psubs_w(<8 x i16> %a0, <8 x i16> %a1) {
-; SSE-LABEL: test_x86_sse2_psubs_w:
-; SSE:       ## %bb.0:
-; SSE-NEXT:    psubsw %xmm1, %xmm0 ## encoding: [0x66,0x0f,0xe9,0xc1]
-; SSE-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
-;
-; AVX1-LABEL: test_x86_sse2_psubs_w:
-; AVX1:       ## %bb.0:
-; AVX1-NEXT:    vpsubsw %xmm1, %xmm0, %xmm0 ## encoding: [0xc5,0xf9,0xe9,0xc1]
-; AVX1-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
-;
-; AVX512-LABEL: test_x86_sse2_psubs_w:
-; AVX512:       ## %bb.0:
-; AVX512-NEXT:    vpsubsw %xmm1, %xmm0, %xmm0 ## EVEX TO VEX Compression encoding: [0xc5,0xf9,0xe9,0xc1]
-; AVX512-NEXT:    ret{{[l|q]}} ## encoding: [0xc3]
-  %res = call <8 x i16> @llvm.x86.sse2.psubs.w(<8 x i16> %a0, <8 x i16> %a1) ; <<8 x i16>> [#uses=1]
-  ret <8 x i16> %res
-}
-declare <8 x i16> @llvm.x86.sse2.psubs.w(<8 x i16>, <8 x i16>) nounwind readnone
-
-
 define i32 @test_x86_sse2_ucomieq_sd(<2 x double> %a0, <2 x double> %a1) {
 ; SSE-LABEL: test_x86_sse2_ucomieq_sd:
 ; SSE:       ## %bb.0:
diff --git a/test/CodeGen/X86/sse2-schedule.ll b/test/CodeGen/X86/sse2-schedule.ll
deleted file mode 100644
index 660ba8ebb791d0f511cef4f68c31ec2cfb5d7437..0000000000000000000000000000000000000000
--- a/test/CodeGen/X86/sse2-schedule.ll
+++ /dev/null
@@ -1,16972 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=x86-64 | FileCheck %s --check-prefixes=CHECK,GENERIC
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=atom -mattr=-sse3 | FileCheck %s --check-prefixes=CHECK,ATOM
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=slm -mattr=-sse3 | FileCheck %s --check-prefixes=CHECK,SLM
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=sandybridge -mattr=-sse3 | FileCheck %s --check-prefixes=CHECK,SANDY-SSE
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=sandybridge -mattr=-avx2 | FileCheck %s --check-prefixes=CHECK,SANDY
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=ivybridge -mattr=-sse3 | FileCheck %s --check-prefixes=CHECK,SANDY-SSE
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=ivybridge -mattr=-avx2 | FileCheck %s --check-prefixes=CHECK,SANDY
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=haswell -mattr=-sse3 | FileCheck %s --check-prefixes=CHECK,HASWELL-SSE
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=haswell -mattr=-avx2 | FileCheck %s --check-prefixes=CHECK,HASWELL
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=broadwell -mattr=-sse3 | FileCheck %s --check-prefixes=CHECK,BROADWELL-SSE
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=broadwell -mattr=-avx2 | FileCheck %s --check-prefixes=CHECK,BROADWELL
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=skylake -mattr=-sse3 | FileCheck %s --check-prefixes=CHECK,SKYLAKE-SSE
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=skylake -mattr=-avx2 | FileCheck %s --check-prefixes=CHECK,SKYLAKE
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=skx -mattr=-sse3 | FileCheck %s --check-prefixes=CHECK,SKX-SSE
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=skx -mattr=-avx2 | FileCheck %s --check-prefixes=CHECK,SKX
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=bdver2 -mattr=-sse3 | FileCheck %s --check-prefixes=CHECK,BDVER2-SSE
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=bdver2 -mattr=-avx2,-xop | FileCheck %s --check-prefixes=CHECK,BDVER2
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=btver2 -mattr=-sse3 | FileCheck %s --check-prefixes=CHECK,BTVER2-SSE
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=btver2 -mattr=-avx2 | FileCheck %s --check-prefixes=CHECK,BTVER2
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=znver1 -mattr=-sse3 | FileCheck %s --check-prefixes=CHECK,ZNVER1-SSE
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=znver1 -mattr=-avx2 | FileCheck %s --check-prefixes=CHECK,ZNVER1
-
-define <2 x double> @test_addpd(<2 x double> %a0, <2 x double> %a1, <2 x double> *%a2) {
-; GENERIC-LABEL: test_addpd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    addpd %xmm1, %xmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    addpd (%rdi), %xmm0 # sched: [9:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_addpd:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    addpd %xmm1, %xmm0 # sched: [6:3.00]
-; ATOM-NEXT:    addpd (%rdi), %xmm0 # sched: [7:3.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_addpd:
-; SLM:       # %bb.0:
-; SLM-NEXT:    addpd %xmm1, %xmm0 # sched: [3:1.00]
-; SLM-NEXT:    addpd (%rdi), %xmm0 # sched: [6:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_addpd:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    addpd %xmm1, %xmm0 # sched: [3:1.00]
-; SANDY-SSE-NEXT:    addpd (%rdi), %xmm0 # sched: [9:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_addpd:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vaddpd %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; SANDY-NEXT:    vaddpd (%rdi), %xmm0, %xmm0 # sched: [9:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_addpd:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    addpd %xmm1, %xmm0 # sched: [3:1.00]
-; HASWELL-SSE-NEXT:    addpd (%rdi), %xmm0 # sched: [9:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_addpd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vaddpd %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; HASWELL-NEXT:    vaddpd (%rdi), %xmm0, %xmm0 # sched: [9:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_addpd:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    addpd %xmm1, %xmm0 # sched: [3:1.00]
-; BROADWELL-SSE-NEXT:    addpd (%rdi), %xmm0 # sched: [8:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_addpd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vaddpd %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    vaddpd (%rdi), %xmm0, %xmm0 # sched: [8:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_addpd:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    addpd %xmm1, %xmm0 # sched: [4:0.50]
-; SKYLAKE-SSE-NEXT:    addpd (%rdi), %xmm0 # sched: [10:0.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_addpd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vaddpd %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    vaddpd (%rdi), %xmm0, %xmm0 # sched: [10:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_addpd:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    addpd %xmm1, %xmm0 # sched: [4:0.50]
-; SKX-SSE-NEXT:    addpd (%rdi), %xmm0 # sched: [10:0.50]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_addpd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vaddpd %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKX-NEXT:    vaddpd (%rdi), %xmm0, %xmm0 # sched: [10:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_addpd:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    addpd %xmm1, %xmm0 # sched: [5:1.00]
-; BDVER2-SSE-NEXT:    addpd (%rdi), %xmm0 # sched: [10:1.00]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_addpd:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vaddpd %xmm1, %xmm0, %xmm0 # sched: [5:1.00]
-; BDVER2-NEXT:    vaddpd (%rdi), %xmm0, %xmm0 # sched: [10:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_addpd:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    addpd %xmm1, %xmm0 # sched: [3:1.00]
-; BTVER2-SSE-NEXT:    addpd (%rdi), %xmm0 # sched: [8:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_addpd:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vaddpd %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; BTVER2-NEXT:    vaddpd (%rdi), %xmm0, %xmm0 # sched: [8:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_addpd:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    addpd %xmm1, %xmm0 # sched: [3:1.00]
-; ZNVER1-SSE-NEXT:    addpd (%rdi), %xmm0 # sched: [10:1.00]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_addpd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vaddpd %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    vaddpd (%rdi), %xmm0, %xmm0 # sched: [10:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = fadd <2 x double> %a0, %a1
-  %2 = load <2 x double>, <2 x double> *%a2, align 16
-  %3 = fadd <2 x double> %1, %2
-  ret <2 x double> %3
-}
-
-define double @test_addsd(double %a0, double %a1, double *%a2) {
-; GENERIC-LABEL: test_addsd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    addsd %xmm1, %xmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    addsd (%rdi), %xmm0 # sched: [9:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_addsd:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    addsd %xmm1, %xmm0 # sched: [5:5.00]
-; ATOM-NEXT:    addsd (%rdi), %xmm0 # sched: [5:5.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_addsd:
-; SLM:       # %bb.0:
-; SLM-NEXT:    addsd %xmm1, %xmm0 # sched: [3:1.00]
-; SLM-NEXT:    addsd (%rdi), %xmm0 # sched: [6:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_addsd:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    addsd %xmm1, %xmm0 # sched: [3:1.00]
-; SANDY-SSE-NEXT:    addsd (%rdi), %xmm0 # sched: [9:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_addsd:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vaddsd %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; SANDY-NEXT:    vaddsd (%rdi), %xmm0, %xmm0 # sched: [9:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_addsd:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    addsd %xmm1, %xmm0 # sched: [3:1.00]
-; HASWELL-SSE-NEXT:    addsd (%rdi), %xmm0 # sched: [8:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_addsd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vaddsd %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; HASWELL-NEXT:    vaddsd (%rdi), %xmm0, %xmm0 # sched: [8:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_addsd:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    addsd %xmm1, %xmm0 # sched: [3:1.00]
-; BROADWELL-SSE-NEXT:    addsd (%rdi), %xmm0 # sched: [8:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_addsd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vaddsd %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    vaddsd (%rdi), %xmm0, %xmm0 # sched: [8:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_addsd:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    addsd %xmm1, %xmm0 # sched: [4:0.50]
-; SKYLAKE-SSE-NEXT:    addsd (%rdi), %xmm0 # sched: [9:0.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_addsd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vaddsd %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    vaddsd (%rdi), %xmm0, %xmm0 # sched: [9:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_addsd:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    addsd %xmm1, %xmm0 # sched: [4:0.50]
-; SKX-SSE-NEXT:    addsd (%rdi), %xmm0 # sched: [9:0.50]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_addsd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vaddsd %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKX-NEXT:    vaddsd (%rdi), %xmm0, %xmm0 # sched: [9:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_addsd:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    addsd %xmm1, %xmm0 # sched: [5:1.00]
-; BDVER2-SSE-NEXT:    addsd (%rdi), %xmm0 # sched: [10:1.00]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_addsd:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vaddsd %xmm1, %xmm0, %xmm0 # sched: [5:1.00]
-; BDVER2-NEXT:    vaddsd (%rdi), %xmm0, %xmm0 # sched: [10:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_addsd:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    addsd %xmm1, %xmm0 # sched: [3:1.00]
-; BTVER2-SSE-NEXT:    addsd (%rdi), %xmm0 # sched: [8:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_addsd:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vaddsd %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; BTVER2-NEXT:    vaddsd (%rdi), %xmm0, %xmm0 # sched: [8:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_addsd:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    addsd %xmm1, %xmm0 # sched: [3:1.00]
-; ZNVER1-SSE-NEXT:    addsd (%rdi), %xmm0 # sched: [10:1.00]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_addsd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vaddsd %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    vaddsd (%rdi), %xmm0, %xmm0 # sched: [10:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = fadd double %a0, %a1
-  %2 = load double, double *%a2, align 8
-  %3 = fadd double %1, %2
-  ret double %3
-}
-
-define <2 x double> @test_andpd(<2 x double> %a0, <2 x double> %a1, <2 x double> *%a2) {
-; GENERIC-LABEL: test_andpd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    andpd %xmm1, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    andpd (%rdi), %xmm0 # sched: [7:1.00]
-; GENERIC-NEXT:    addpd %xmm1, %xmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_andpd:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    andpd %xmm1, %xmm0 # sched: [1:0.50]
-; ATOM-NEXT:    andpd (%rdi), %xmm0 # sched: [1:1.00]
-; ATOM-NEXT:    addpd %xmm1, %xmm0 # sched: [6:3.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_andpd:
-; SLM:       # %bb.0:
-; SLM-NEXT:    andpd %xmm1, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    andpd (%rdi), %xmm0 # sched: [4:1.00]
-; SLM-NEXT:    addpd %xmm1, %xmm0 # sched: [3:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_andpd:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    andpd %xmm1, %xmm0 # sched: [1:1.00]
-; SANDY-SSE-NEXT:    andpd (%rdi), %xmm0 # sched: [7:1.00]
-; SANDY-SSE-NEXT:    addpd %xmm1, %xmm0 # sched: [3:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_andpd:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vandpd %xmm1, %xmm0, %xmm0 # sched: [1:1.00]
-; SANDY-NEXT:    vandpd (%rdi), %xmm0, %xmm0 # sched: [7:1.00]
-; SANDY-NEXT:    vaddpd %xmm0, %xmm1, %xmm0 # sched: [3:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_andpd:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    andpd %xmm1, %xmm0 # sched: [1:1.00]
-; HASWELL-SSE-NEXT:    andpd (%rdi), %xmm0 # sched: [7:1.00]
-; HASWELL-SSE-NEXT:    addpd %xmm1, %xmm0 # sched: [3:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_andpd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vandpd %xmm1, %xmm0, %xmm0 # sched: [1:1.00]
-; HASWELL-NEXT:    vandpd (%rdi), %xmm0, %xmm0 # sched: [7:1.00]
-; HASWELL-NEXT:    vaddpd %xmm0, %xmm1, %xmm0 # sched: [3:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_andpd:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    andpd %xmm1, %xmm0 # sched: [1:1.00]
-; BROADWELL-SSE-NEXT:    andpd (%rdi), %xmm0 # sched: [6:1.00]
-; BROADWELL-SSE-NEXT:    addpd %xmm1, %xmm0 # sched: [3:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_andpd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vandpd %xmm1, %xmm0, %xmm0 # sched: [1:1.00]
-; BROADWELL-NEXT:    vandpd (%rdi), %xmm0, %xmm0 # sched: [6:1.00]
-; BROADWELL-NEXT:    vaddpd %xmm0, %xmm1, %xmm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_andpd:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    andpd %xmm1, %xmm0 # sched: [1:0.33]
-; SKYLAKE-SSE-NEXT:    andpd (%rdi), %xmm0 # sched: [7:0.50]
-; SKYLAKE-SSE-NEXT:    addpd %xmm1, %xmm0 # sched: [4:0.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_andpd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vandpd %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    vandpd (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKYLAKE-NEXT:    vaddpd %xmm0, %xmm1, %xmm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_andpd:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    andpd %xmm1, %xmm0 # sched: [1:0.33]
-; SKX-SSE-NEXT:    andpd (%rdi), %xmm0 # sched: [7:0.50]
-; SKX-SSE-NEXT:    addpd %xmm1, %xmm0 # sched: [4:0.50]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_andpd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vandpd %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    vandpd (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKX-NEXT:    vaddpd %xmm0, %xmm1, %xmm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_andpd:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    andpd %xmm1, %xmm0 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    andpd (%rdi), %xmm0 # sched: [7:0.50]
-; BDVER2-SSE-NEXT:    addpd %xmm1, %xmm0 # sched: [5:1.00]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_andpd:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vandpd %xmm1, %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    vandpd (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; BDVER2-NEXT:    vaddpd %xmm0, %xmm1, %xmm0 # sched: [5:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_andpd:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    andpd %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    andpd (%rdi), %xmm0 # sched: [6:1.00]
-; BTVER2-SSE-NEXT:    addpd %xmm1, %xmm0 # sched: [3:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_andpd:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vandpd %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    vandpd (%rdi), %xmm0, %xmm0 # sched: [6:1.00]
-; BTVER2-NEXT:    vaddpd %xmm0, %xmm1, %xmm0 # sched: [3:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_andpd:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    andpd %xmm1, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    andpd (%rdi), %xmm0 # sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    addpd %xmm1, %xmm0 # sched: [3:1.00]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_andpd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vandpd %xmm1, %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vandpd (%rdi), %xmm0, %xmm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    vaddpd %xmm0, %xmm1, %xmm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = bitcast <2 x double> %a0 to <4 x i32>
-  %2 = bitcast <2 x double> %a1 to <4 x i32>
-  %3 = and <4 x i32> %1, %2
-  %4 = load <2 x double>, <2 x double> *%a2, align 16
-  %5 = bitcast <2 x double> %4 to <4 x i32>
-  %6 = and <4 x i32> %3, %5
-  %7 = bitcast <4 x i32> %6 to <2 x double>
-  %8 = fadd <2 x double> %a1, %7
-  ret <2 x double> %8
-}
-
-define <2 x double> @test_andnotpd(<2 x double> %a0, <2 x double> %a1, <2 x double> *%a2) {
-; GENERIC-LABEL: test_andnotpd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    andnpd %xmm1, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    andnpd (%rdi), %xmm0 # sched: [7:1.00]
-; GENERIC-NEXT:    addpd %xmm1, %xmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_andnotpd:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    andnpd %xmm1, %xmm0 # sched: [1:0.50]
-; ATOM-NEXT:    andnpd (%rdi), %xmm0 # sched: [1:1.00]
-; ATOM-NEXT:    addpd %xmm1, %xmm0 # sched: [6:3.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_andnotpd:
-; SLM:       # %bb.0:
-; SLM-NEXT:    andnpd %xmm1, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    andnpd (%rdi), %xmm0 # sched: [4:1.00]
-; SLM-NEXT:    addpd %xmm1, %xmm0 # sched: [3:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_andnotpd:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    andnpd %xmm1, %xmm0 # sched: [1:1.00]
-; SANDY-SSE-NEXT:    andnpd (%rdi), %xmm0 # sched: [7:1.00]
-; SANDY-SSE-NEXT:    addpd %xmm1, %xmm0 # sched: [3:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_andnotpd:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vandnpd %xmm1, %xmm0, %xmm0 # sched: [1:1.00]
-; SANDY-NEXT:    vandnpd (%rdi), %xmm0, %xmm0 # sched: [7:1.00]
-; SANDY-NEXT:    vaddpd %xmm0, %xmm1, %xmm0 # sched: [3:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_andnotpd:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    andnpd %xmm1, %xmm0 # sched: [1:1.00]
-; HASWELL-SSE-NEXT:    andnpd (%rdi), %xmm0 # sched: [7:1.00]
-; HASWELL-SSE-NEXT:    addpd %xmm1, %xmm0 # sched: [3:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_andnotpd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vandnpd %xmm1, %xmm0, %xmm0 # sched: [1:1.00]
-; HASWELL-NEXT:    vandnpd (%rdi), %xmm0, %xmm0 # sched: [7:1.00]
-; HASWELL-NEXT:    vaddpd %xmm0, %xmm1, %xmm0 # sched: [3:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_andnotpd:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    andnpd %xmm1, %xmm0 # sched: [1:1.00]
-; BROADWELL-SSE-NEXT:    andnpd (%rdi), %xmm0 # sched: [6:1.00]
-; BROADWELL-SSE-NEXT:    addpd %xmm1, %xmm0 # sched: [3:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_andnotpd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vandnpd %xmm1, %xmm0, %xmm0 # sched: [1:1.00]
-; BROADWELL-NEXT:    vandnpd (%rdi), %xmm0, %xmm0 # sched: [6:1.00]
-; BROADWELL-NEXT:    vaddpd %xmm0, %xmm1, %xmm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_andnotpd:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    andnpd %xmm1, %xmm0 # sched: [1:0.33]
-; SKYLAKE-SSE-NEXT:    andnpd (%rdi), %xmm0 # sched: [7:0.50]
-; SKYLAKE-SSE-NEXT:    addpd %xmm1, %xmm0 # sched: [4:0.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_andnotpd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vandnpd %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    vandnpd (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKYLAKE-NEXT:    vaddpd %xmm0, %xmm1, %xmm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_andnotpd:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    andnpd %xmm1, %xmm0 # sched: [1:0.33]
-; SKX-SSE-NEXT:    andnpd (%rdi), %xmm0 # sched: [7:0.50]
-; SKX-SSE-NEXT:    addpd %xmm1, %xmm0 # sched: [4:0.50]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_andnotpd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vandnpd %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    vandnpd (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKX-NEXT:    vaddpd %xmm0, %xmm1, %xmm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_andnotpd:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    andnpd %xmm1, %xmm0 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    andnpd (%rdi), %xmm0 # sched: [7:0.50]
-; BDVER2-SSE-NEXT:    addpd %xmm1, %xmm0 # sched: [5:1.00]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_andnotpd:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vandnpd %xmm1, %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    vandnpd (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; BDVER2-NEXT:    vaddpd %xmm0, %xmm1, %xmm0 # sched: [5:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_andnotpd:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    andnpd %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    andnpd (%rdi), %xmm0 # sched: [6:1.00]
-; BTVER2-SSE-NEXT:    addpd %xmm1, %xmm0 # sched: [3:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_andnotpd:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vandnpd %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    vandnpd (%rdi), %xmm0, %xmm0 # sched: [6:1.00]
-; BTVER2-NEXT:    vaddpd %xmm0, %xmm1, %xmm0 # sched: [3:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_andnotpd:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    andnpd %xmm1, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    andnpd (%rdi), %xmm0 # sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    addpd %xmm1, %xmm0 # sched: [3:1.00]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_andnotpd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vandnpd %xmm1, %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vandnpd (%rdi), %xmm0, %xmm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    vaddpd %xmm0, %xmm1, %xmm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = bitcast <2 x double> %a0 to <4 x i32>
-  %2 = bitcast <2 x double> %a1 to <4 x i32>
-  %3 = xor <4 x i32> %1, <i32 -1, i32 -1, i32 -1, i32 -1>
-  %4 = and <4 x i32> %3, %2
-  %5 = load <2 x double>, <2 x double> *%a2, align 16
-  %6 = bitcast <2 x double> %5 to <4 x i32>
-  %7 = xor <4 x i32> %4, <i32 -1, i32 -1, i32 -1, i32 -1>
-  %8 = and <4 x i32> %6, %7
-  %9 = bitcast <4 x i32> %8 to <2 x double>
-  %10 = fadd <2 x double> %a1, %9
-  ret <2 x double> %10
-}
-
-define void @test_clflush(i8* %p){
-; GENERIC-LABEL: test_clflush:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    clflush (%rdi) # sched: [5:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_clflush:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    clflush (%rdi) # sched: [1:1.00]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_clflush:
-; SLM:       # %bb.0:
-; SLM-NEXT:    clflush (%rdi) # sched: [3:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_clflush:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    clflush (%rdi) # sched: [5:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_clflush:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    clflush (%rdi) # sched: [5:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_clflush:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    clflush (%rdi) # sched: [2:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_clflush:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    clflush (%rdi) # sched: [2:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_clflush:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    clflush (%rdi) # sched: [2:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_clflush:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    clflush (%rdi) # sched: [2:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_clflush:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    clflush (%rdi) # sched: [2:1.00]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_clflush:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    clflush (%rdi) # sched: [2:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_clflush:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    clflush (%rdi) # sched: [2:1.00]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_clflush:
-; SKX:       # %bb.0:
-; SKX-NEXT:    clflush (%rdi) # sched: [2:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_clflush:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    clflush (%rdi) # sched: [5:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_clflush:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    clflush (%rdi) # sched: [5:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_clflush:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    clflush (%rdi) # sched: [5:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_clflush:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    clflush (%rdi) # sched: [5:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_clflush:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    clflush (%rdi) # sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_clflush:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    clflush (%rdi) # sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  tail call void @llvm.x86.sse2.clflush(i8* %p)
-  ret void
-}
-declare void @llvm.x86.sse2.clflush(i8*) nounwind
-
-define <2 x double> @test_cmppd(<2 x double> %a0, <2 x double> %a1, <2 x double> *%a2) {
-; GENERIC-LABEL: test_cmppd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    cmpeqpd %xmm0, %xmm1 # sched: [3:1.00]
-; GENERIC-NEXT:    cmpeqpd (%rdi), %xmm0 # sched: [9:1.00]
-; GENERIC-NEXT:    orpd %xmm1, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_cmppd:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    cmpeqpd %xmm0, %xmm1 # sched: [6:3.00]
-; ATOM-NEXT:    cmpeqpd (%rdi), %xmm0 # sched: [7:3.50]
-; ATOM-NEXT:    orpd %xmm1, %xmm0 # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_cmppd:
-; SLM:       # %bb.0:
-; SLM-NEXT:    cmpeqpd %xmm0, %xmm1 # sched: [3:1.00]
-; SLM-NEXT:    cmpeqpd (%rdi), %xmm0 # sched: [6:1.00]
-; SLM-NEXT:    orpd %xmm1, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_cmppd:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    cmpeqpd %xmm0, %xmm1 # sched: [3:1.00]
-; SANDY-SSE-NEXT:    cmpeqpd (%rdi), %xmm0 # sched: [9:1.00]
-; SANDY-SSE-NEXT:    orpd %xmm1, %xmm0 # sched: [1:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_cmppd:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vcmpeqpd %xmm1, %xmm0, %xmm1 # sched: [3:1.00]
-; SANDY-NEXT:    vcmpeqpd (%rdi), %xmm0, %xmm0 # sched: [9:1.00]
-; SANDY-NEXT:    vorpd %xmm0, %xmm1, %xmm0 # sched: [1:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_cmppd:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    cmpeqpd %xmm0, %xmm1 # sched: [3:1.00]
-; HASWELL-SSE-NEXT:    cmpeqpd (%rdi), %xmm0 # sched: [9:1.00]
-; HASWELL-SSE-NEXT:    orpd %xmm1, %xmm0 # sched: [1:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_cmppd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vcmpeqpd %xmm1, %xmm0, %xmm1 # sched: [3:1.00]
-; HASWELL-NEXT:    vcmpeqpd (%rdi), %xmm0, %xmm0 # sched: [9:1.00]
-; HASWELL-NEXT:    vorpd %xmm0, %xmm1, %xmm0 # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_cmppd:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    cmpeqpd %xmm0, %xmm1 # sched: [3:1.00]
-; BROADWELL-SSE-NEXT:    cmpeqpd (%rdi), %xmm0 # sched: [8:1.00]
-; BROADWELL-SSE-NEXT:    orpd %xmm1, %xmm0 # sched: [1:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_cmppd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vcmpeqpd %xmm1, %xmm0, %xmm1 # sched: [3:1.00]
-; BROADWELL-NEXT:    vcmpeqpd (%rdi), %xmm0, %xmm0 # sched: [8:1.00]
-; BROADWELL-NEXT:    vorpd %xmm0, %xmm1, %xmm0 # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_cmppd:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    cmpeqpd %xmm0, %xmm1 # sched: [4:0.50]
-; SKYLAKE-SSE-NEXT:    cmpeqpd (%rdi), %xmm0 # sched: [10:0.50]
-; SKYLAKE-SSE-NEXT:    orpd %xmm1, %xmm0 # sched: [1:0.33]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_cmppd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vcmpeqpd %xmm1, %xmm0, %xmm1 # sched: [4:0.50]
-; SKYLAKE-NEXT:    vcmpeqpd (%rdi), %xmm0, %xmm0 # sched: [10:0.50]
-; SKYLAKE-NEXT:    vorpd %xmm0, %xmm1, %xmm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_cmppd:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    cmpeqpd %xmm0, %xmm1 # sched: [4:0.50]
-; SKX-SSE-NEXT:    cmpeqpd (%rdi), %xmm0 # sched: [10:0.50]
-; SKX-SSE-NEXT:    orpd %xmm1, %xmm0 # sched: [1:0.33]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_cmppd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vcmpeqpd %xmm1, %xmm0, %xmm1 # sched: [4:0.50]
-; SKX-NEXT:    vcmpeqpd (%rdi), %xmm0, %xmm0 # sched: [10:0.50]
-; SKX-NEXT:    vorpd %xmm0, %xmm1, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_cmppd:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    cmpeqpd %xmm0, %xmm1 # sched: [2:1.00]
-; BDVER2-SSE-NEXT:    cmpeqpd (%rdi), %xmm0 # sched: [7:1.00]
-; BDVER2-SSE-NEXT:    orpd %xmm1, %xmm0 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_cmppd:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vcmpeqpd %xmm1, %xmm0, %xmm1 # sched: [2:1.00]
-; BDVER2-NEXT:    vcmpeqpd (%rdi), %xmm0, %xmm0 # sched: [7:1.00]
-; BDVER2-NEXT:    vorpd %xmm0, %xmm1, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_cmppd:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    cmpeqpd %xmm0, %xmm1 # sched: [2:1.00]
-; BTVER2-SSE-NEXT:    cmpeqpd (%rdi), %xmm0 # sched: [7:1.00]
-; BTVER2-SSE-NEXT:    orpd %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_cmppd:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vcmpeqpd %xmm1, %xmm0, %xmm1 # sched: [2:1.00]
-; BTVER2-NEXT:    vcmpeqpd (%rdi), %xmm0, %xmm0 # sched: [7:1.00]
-; BTVER2-NEXT:    vorpd %xmm0, %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_cmppd:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    cmpeqpd %xmm0, %xmm1 # sched: [3:1.00]
-; ZNVER1-SSE-NEXT:    cmpeqpd (%rdi), %xmm0 # sched: [10:1.00]
-; ZNVER1-SSE-NEXT:    orpd %xmm1, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_cmppd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vcmpeqpd %xmm1, %xmm0, %xmm1 # sched: [3:1.00]
-; ZNVER1-NEXT:    vcmpeqpd (%rdi), %xmm0, %xmm0 # sched: [10:1.00]
-; ZNVER1-NEXT:    vorpd %xmm0, %xmm1, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = fcmp oeq <2 x double> %a0, %a1
-  %2 = load <2 x double>, <2 x double> *%a2, align 16
-  %3 = fcmp oeq <2 x double> %a0, %2
-  %4 = or <2 x i1> %1, %3
-  %5 = sext <2 x i1> %4 to <2 x i64>
-  %6 = bitcast <2 x i64> %5 to <2 x double>
-  ret <2 x double> %6
-}
-
-define double @test_cmpsd(double %a0, double %a1, double *%a2) {
-; GENERIC-LABEL: test_cmpsd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    cmpeqsd %xmm1, %xmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    cmpeqsd (%rdi), %xmm0 # sched: [9:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_cmpsd:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    cmpeqsd %xmm1, %xmm0 # sched: [5:5.00]
-; ATOM-NEXT:    cmpeqsd (%rdi), %xmm0 # sched: [5:5.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_cmpsd:
-; SLM:       # %bb.0:
-; SLM-NEXT:    cmpeqsd %xmm1, %xmm0 # sched: [3:1.00]
-; SLM-NEXT:    cmpeqsd (%rdi), %xmm0 # sched: [6:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_cmpsd:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    cmpeqsd %xmm1, %xmm0 # sched: [3:1.00]
-; SANDY-SSE-NEXT:    cmpeqsd (%rdi), %xmm0 # sched: [9:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_cmpsd:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vcmpeqsd %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; SANDY-NEXT:    vcmpeqsd (%rdi), %xmm0, %xmm0 # sched: [9:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_cmpsd:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    cmpeqsd %xmm1, %xmm0 # sched: [3:1.00]
-; HASWELL-SSE-NEXT:    cmpeqsd (%rdi), %xmm0 # sched: [8:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_cmpsd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vcmpeqsd %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; HASWELL-NEXT:    vcmpeqsd (%rdi), %xmm0, %xmm0 # sched: [8:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_cmpsd:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    cmpeqsd %xmm1, %xmm0 # sched: [3:1.00]
-; BROADWELL-SSE-NEXT:    cmpeqsd (%rdi), %xmm0 # sched: [8:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_cmpsd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vcmpeqsd %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    vcmpeqsd (%rdi), %xmm0, %xmm0 # sched: [8:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_cmpsd:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    cmpeqsd %xmm1, %xmm0 # sched: [4:0.50]
-; SKYLAKE-SSE-NEXT:    cmpeqsd (%rdi), %xmm0 # sched: [9:0.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_cmpsd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vcmpeqsd %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    vcmpeqsd (%rdi), %xmm0, %xmm0 # sched: [9:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_cmpsd:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    cmpeqsd %xmm1, %xmm0 # sched: [4:0.50]
-; SKX-SSE-NEXT:    cmpeqsd (%rdi), %xmm0 # sched: [9:0.50]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_cmpsd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vcmpeqsd %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKX-NEXT:    vcmpeqsd (%rdi), %xmm0, %xmm0 # sched: [9:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_cmpsd:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    cmpeqsd %xmm1, %xmm0 # sched: [2:1.00]
-; BDVER2-SSE-NEXT:    cmpeqsd (%rdi), %xmm0 # sched: [7:1.00]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_cmpsd:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vcmpeqsd %xmm1, %xmm0, %xmm0 # sched: [2:1.00]
-; BDVER2-NEXT:    vcmpeqsd (%rdi), %xmm0, %xmm0 # sched: [7:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_cmpsd:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    cmpeqsd %xmm1, %xmm0 # sched: [2:1.00]
-; BTVER2-SSE-NEXT:    cmpeqsd (%rdi), %xmm0 # sched: [7:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_cmpsd:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vcmpeqsd %xmm1, %xmm0, %xmm0 # sched: [2:1.00]
-; BTVER2-NEXT:    vcmpeqsd (%rdi), %xmm0, %xmm0 # sched: [7:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_cmpsd:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    cmpeqsd %xmm1, %xmm0 # sched: [3:1.00]
-; ZNVER1-SSE-NEXT:    cmpeqsd (%rdi), %xmm0 # sched: [10:1.00]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_cmpsd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vcmpeqsd %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    vcmpeqsd (%rdi), %xmm0, %xmm0 # sched: [10:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = insertelement <2 x double> undef, double %a0, i32 0
-  %2 = insertelement <2 x double> undef, double %a1, i32 0
-  %3 = call <2 x double> @llvm.x86.sse2.cmp.sd(<2 x double> %1, <2 x double> %2, i8 0)
-  %4 = load double, double *%a2, align 8
-  %5 = insertelement <2 x double> undef, double %4, i32 0
-  %6 = call <2 x double> @llvm.x86.sse2.cmp.sd(<2 x double> %3, <2 x double> %5, i8 0)
-  %7 = extractelement <2 x double> %6, i32 0
-  ret double %7
-}
-declare <2 x double> @llvm.x86.sse2.cmp.sd(<2 x double>, <2 x double>, i8) nounwind readnone
-
-define i32 @test_comisd(<2 x double> %a0, <2 x double> %a1, <2 x double> *%a2) {
-; GENERIC-LABEL: test_comisd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    comisd %xmm1, %xmm0 # sched: [2:1.00]
-; GENERIC-NEXT:    setnp %al # sched: [1:0.50]
-; GENERIC-NEXT:    sete %cl # sched: [1:0.50]
-; GENERIC-NEXT:    andb %al, %cl # sched: [1:0.33]
-; GENERIC-NEXT:    comisd (%rdi), %xmm0 # sched: [8:1.00]
-; GENERIC-NEXT:    setnp %al # sched: [1:0.50]
-; GENERIC-NEXT:    sete %dl # sched: [1:0.50]
-; GENERIC-NEXT:    andb %al, %dl # sched: [1:0.33]
-; GENERIC-NEXT:    orb %cl, %dl # sched: [1:0.33]
-; GENERIC-NEXT:    movzbl %dl, %eax # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_comisd:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    comisd %xmm1, %xmm0 # sched: [9:4.50]
-; ATOM-NEXT:    setnp %al # sched: [1:0.50]
-; ATOM-NEXT:    sete %cl # sched: [1:0.50]
-; ATOM-NEXT:    andb %al, %cl # sched: [1:0.50]
-; ATOM-NEXT:    comisd (%rdi), %xmm0 # sched: [10:5.00]
-; ATOM-NEXT:    setnp %al # sched: [1:0.50]
-; ATOM-NEXT:    sete %dl # sched: [1:0.50]
-; ATOM-NEXT:    andb %al, %dl # sched: [1:0.50]
-; ATOM-NEXT:    orb %cl, %dl # sched: [1:0.50]
-; ATOM-NEXT:    movzbl %dl, %eax # sched: [1:1.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_comisd:
-; SLM:       # %bb.0:
-; SLM-NEXT:    comisd %xmm1, %xmm0 # sched: [3:1.00]
-; SLM-NEXT:    setnp %al # sched: [1:0.50]
-; SLM-NEXT:    sete %cl # sched: [1:0.50]
-; SLM-NEXT:    andb %al, %cl # sched: [1:0.50]
-; SLM-NEXT:    comisd (%rdi), %xmm0 # sched: [6:1.00]
-; SLM-NEXT:    setnp %al # sched: [1:0.50]
-; SLM-NEXT:    sete %dl # sched: [1:0.50]
-; SLM-NEXT:    andb %al, %dl # sched: [1:0.50]
-; SLM-NEXT:    orb %cl, %dl # sched: [1:0.50]
-; SLM-NEXT:    movzbl %dl, %eax # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_comisd:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    comisd %xmm1, %xmm0 # sched: [2:1.00]
-; SANDY-SSE-NEXT:    setnp %al # sched: [1:0.50]
-; SANDY-SSE-NEXT:    sete %cl # sched: [1:0.50]
-; SANDY-SSE-NEXT:    andb %al, %cl # sched: [1:0.33]
-; SANDY-SSE-NEXT:    comisd (%rdi), %xmm0 # sched: [8:1.00]
-; SANDY-SSE-NEXT:    setnp %al # sched: [1:0.50]
-; SANDY-SSE-NEXT:    sete %dl # sched: [1:0.50]
-; SANDY-SSE-NEXT:    andb %al, %dl # sched: [1:0.33]
-; SANDY-SSE-NEXT:    orb %cl, %dl # sched: [1:0.33]
-; SANDY-SSE-NEXT:    movzbl %dl, %eax # sched: [1:0.33]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_comisd:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vcomisd %xmm1, %xmm0 # sched: [2:1.00]
-; SANDY-NEXT:    setnp %al # sched: [1:0.50]
-; SANDY-NEXT:    sete %cl # sched: [1:0.50]
-; SANDY-NEXT:    andb %al, %cl # sched: [1:0.33]
-; SANDY-NEXT:    vcomisd (%rdi), %xmm0 # sched: [8:1.00]
-; SANDY-NEXT:    setnp %al # sched: [1:0.50]
-; SANDY-NEXT:    sete %dl # sched: [1:0.50]
-; SANDY-NEXT:    andb %al, %dl # sched: [1:0.33]
-; SANDY-NEXT:    orb %cl, %dl # sched: [1:0.33]
-; SANDY-NEXT:    movzbl %dl, %eax # sched: [1:0.33]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_comisd:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    comisd %xmm1, %xmm0 # sched: [3:1.00]
-; HASWELL-SSE-NEXT:    setnp %al # sched: [1:0.50]
-; HASWELL-SSE-NEXT:    sete %cl # sched: [1:0.50]
-; HASWELL-SSE-NEXT:    andb %al, %cl # sched: [1:0.25]
-; HASWELL-SSE-NEXT:    comisd (%rdi), %xmm0 # sched: [8:1.00]
-; HASWELL-SSE-NEXT:    setnp %al # sched: [1:0.50]
-; HASWELL-SSE-NEXT:    sete %dl # sched: [1:0.50]
-; HASWELL-SSE-NEXT:    andb %al, %dl # sched: [1:0.25]
-; HASWELL-SSE-NEXT:    orb %cl, %dl # sched: [1:0.25]
-; HASWELL-SSE-NEXT:    movzbl %dl, %eax # sched: [1:0.25]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_comisd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vcomisd %xmm1, %xmm0 # sched: [3:1.00]
-; HASWELL-NEXT:    setnp %al # sched: [1:0.50]
-; HASWELL-NEXT:    sete %cl # sched: [1:0.50]
-; HASWELL-NEXT:    andb %al, %cl # sched: [1:0.25]
-; HASWELL-NEXT:    vcomisd (%rdi), %xmm0 # sched: [8:1.00]
-; HASWELL-NEXT:    setnp %al # sched: [1:0.50]
-; HASWELL-NEXT:    sete %dl # sched: [1:0.50]
-; HASWELL-NEXT:    andb %al, %dl # sched: [1:0.25]
-; HASWELL-NEXT:    orb %cl, %dl # sched: [1:0.25]
-; HASWELL-NEXT:    movzbl %dl, %eax # sched: [1:0.25]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_comisd:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    comisd %xmm1, %xmm0 # sched: [3:1.00]
-; BROADWELL-SSE-NEXT:    setnp %al # sched: [1:0.50]
-; BROADWELL-SSE-NEXT:    sete %cl # sched: [1:0.50]
-; BROADWELL-SSE-NEXT:    andb %al, %cl # sched: [1:0.25]
-; BROADWELL-SSE-NEXT:    comisd (%rdi), %xmm0 # sched: [8:1.00]
-; BROADWELL-SSE-NEXT:    setnp %al # sched: [1:0.50]
-; BROADWELL-SSE-NEXT:    sete %dl # sched: [1:0.50]
-; BROADWELL-SSE-NEXT:    andb %al, %dl # sched: [1:0.25]
-; BROADWELL-SSE-NEXT:    orb %cl, %dl # sched: [1:0.25]
-; BROADWELL-SSE-NEXT:    movzbl %dl, %eax # sched: [1:0.25]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_comisd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vcomisd %xmm1, %xmm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    setnp %al # sched: [1:0.50]
-; BROADWELL-NEXT:    sete %cl # sched: [1:0.50]
-; BROADWELL-NEXT:    andb %al, %cl # sched: [1:0.25]
-; BROADWELL-NEXT:    vcomisd (%rdi), %xmm0 # sched: [8:1.00]
-; BROADWELL-NEXT:    setnp %al # sched: [1:0.50]
-; BROADWELL-NEXT:    sete %dl # sched: [1:0.50]
-; BROADWELL-NEXT:    andb %al, %dl # sched: [1:0.25]
-; BROADWELL-NEXT:    orb %cl, %dl # sched: [1:0.25]
-; BROADWELL-NEXT:    movzbl %dl, %eax # sched: [1:0.25]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_comisd:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    comisd %xmm1, %xmm0 # sched: [2:1.00]
-; SKYLAKE-SSE-NEXT:    setnp %al # sched: [1:0.50]
-; SKYLAKE-SSE-NEXT:    sete %cl # sched: [1:0.50]
-; SKYLAKE-SSE-NEXT:    andb %al, %cl # sched: [1:0.25]
-; SKYLAKE-SSE-NEXT:    comisd (%rdi), %xmm0 # sched: [7:1.00]
-; SKYLAKE-SSE-NEXT:    setnp %al # sched: [1:0.50]
-; SKYLAKE-SSE-NEXT:    sete %dl # sched: [1:0.50]
-; SKYLAKE-SSE-NEXT:    andb %al, %dl # sched: [1:0.25]
-; SKYLAKE-SSE-NEXT:    orb %cl, %dl # sched: [1:0.25]
-; SKYLAKE-SSE-NEXT:    movzbl %dl, %eax # sched: [1:0.25]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_comisd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vcomisd %xmm1, %xmm0 # sched: [2:1.00]
-; SKYLAKE-NEXT:    setnp %al # sched: [1:0.50]
-; SKYLAKE-NEXT:    sete %cl # sched: [1:0.50]
-; SKYLAKE-NEXT:    andb %al, %cl # sched: [1:0.25]
-; SKYLAKE-NEXT:    vcomisd (%rdi), %xmm0 # sched: [7:1.00]
-; SKYLAKE-NEXT:    setnp %al # sched: [1:0.50]
-; SKYLAKE-NEXT:    sete %dl # sched: [1:0.50]
-; SKYLAKE-NEXT:    andb %al, %dl # sched: [1:0.25]
-; SKYLAKE-NEXT:    orb %cl, %dl # sched: [1:0.25]
-; SKYLAKE-NEXT:    movzbl %dl, %eax # sched: [1:0.25]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_comisd:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    comisd %xmm1, %xmm0 # sched: [2:1.00]
-; SKX-SSE-NEXT:    setnp %al # sched: [1:0.50]
-; SKX-SSE-NEXT:    sete %cl # sched: [1:0.50]
-; SKX-SSE-NEXT:    andb %al, %cl # sched: [1:0.25]
-; SKX-SSE-NEXT:    comisd (%rdi), %xmm0 # sched: [7:1.00]
-; SKX-SSE-NEXT:    setnp %al # sched: [1:0.50]
-; SKX-SSE-NEXT:    sete %dl # sched: [1:0.50]
-; SKX-SSE-NEXT:    andb %al, %dl # sched: [1:0.25]
-; SKX-SSE-NEXT:    orb %cl, %dl # sched: [1:0.25]
-; SKX-SSE-NEXT:    movzbl %dl, %eax # sched: [1:0.25]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_comisd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vcomisd %xmm1, %xmm0 # sched: [2:1.00]
-; SKX-NEXT:    setnp %al # sched: [1:0.50]
-; SKX-NEXT:    sete %cl # sched: [1:0.50]
-; SKX-NEXT:    andb %al, %cl # sched: [1:0.25]
-; SKX-NEXT:    vcomisd (%rdi), %xmm0 # sched: [7:1.00]
-; SKX-NEXT:    setnp %al # sched: [1:0.50]
-; SKX-NEXT:    sete %dl # sched: [1:0.50]
-; SKX-NEXT:    andb %al, %dl # sched: [1:0.25]
-; SKX-NEXT:    orb %cl, %dl # sched: [1:0.25]
-; SKX-NEXT:    movzbl %dl, %eax # sched: [1:0.25]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_comisd:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    comisd %xmm1, %xmm0 # sched: [1:1.00]
-; BDVER2-SSE-NEXT:    setnp %al # sched: [1:0.50]
-; BDVER2-SSE-NEXT:    sete %cl # sched: [1:0.50]
-; BDVER2-SSE-NEXT:    andb %al, %cl # sched: [1:0.50]
-; BDVER2-SSE-NEXT:    comisd (%rdi), %xmm0 # sched: [6:1.00]
-; BDVER2-SSE-NEXT:    setnp %al # sched: [1:0.50]
-; BDVER2-SSE-NEXT:    sete %dl # sched: [1:0.50]
-; BDVER2-SSE-NEXT:    andb %al, %dl # sched: [1:0.50]
-; BDVER2-SSE-NEXT:    orb %cl, %dl # sched: [1:0.50]
-; BDVER2-SSE-NEXT:    movzbl %dl, %eax # sched: [1:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_comisd:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vcomisd %xmm1, %xmm0 # sched: [1:1.00]
-; BDVER2-NEXT:    setnp %al # sched: [1:0.50]
-; BDVER2-NEXT:    sete %cl # sched: [1:0.50]
-; BDVER2-NEXT:    andb %al, %cl # sched: [1:0.50]
-; BDVER2-NEXT:    vcomisd (%rdi), %xmm0 # sched: [6:1.00]
-; BDVER2-NEXT:    setnp %al # sched: [1:0.50]
-; BDVER2-NEXT:    sete %dl # sched: [1:0.50]
-; BDVER2-NEXT:    andb %al, %dl # sched: [1:0.50]
-; BDVER2-NEXT:    orb %cl, %dl # sched: [1:0.50]
-; BDVER2-NEXT:    movzbl %dl, %eax # sched: [1:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_comisd:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    comisd %xmm1, %xmm0 # sched: [3:1.00]
-; BTVER2-SSE-NEXT:    setnp %al # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    sete %cl # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    andb %al, %cl # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    comisd (%rdi), %xmm0 # sched: [8:1.00]
-; BTVER2-SSE-NEXT:    setnp %al # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    sete %dl # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    andb %al, %dl # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    orb %cl, %dl # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    movzbl %dl, %eax # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_comisd:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vcomisd %xmm1, %xmm0 # sched: [3:1.00]
-; BTVER2-NEXT:    setnp %al # sched: [1:0.50]
-; BTVER2-NEXT:    sete %cl # sched: [1:0.50]
-; BTVER2-NEXT:    andb %al, %cl # sched: [1:0.50]
-; BTVER2-NEXT:    vcomisd (%rdi), %xmm0 # sched: [8:1.00]
-; BTVER2-NEXT:    setnp %al # sched: [1:0.50]
-; BTVER2-NEXT:    sete %dl # sched: [1:0.50]
-; BTVER2-NEXT:    andb %al, %dl # sched: [1:0.50]
-; BTVER2-NEXT:    orb %cl, %dl # sched: [1:0.50]
-; BTVER2-NEXT:    movzbl %dl, %eax # sched: [1:0.50]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_comisd:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    comisd %xmm1, %xmm0 # sched: [3:1.00]
-; ZNVER1-SSE-NEXT:    setnp %al # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    sete %cl # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    andb %al, %cl # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    comisd (%rdi), %xmm0 # sched: [10:1.00]
-; ZNVER1-SSE-NEXT:    setnp %al # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    sete %dl # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    andb %al, %dl # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    orb %cl, %dl # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    movzbl %dl, %eax # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_comisd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vcomisd %xmm1, %xmm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    setnp %al # sched: [1:0.25]
-; ZNVER1-NEXT:    sete %cl # sched: [1:0.25]
-; ZNVER1-NEXT:    andb %al, %cl # sched: [1:0.25]
-; ZNVER1-NEXT:    vcomisd (%rdi), %xmm0 # sched: [10:1.00]
-; ZNVER1-NEXT:    setnp %al # sched: [1:0.25]
-; ZNVER1-NEXT:    sete %dl # sched: [1:0.25]
-; ZNVER1-NEXT:    andb %al, %dl # sched: [1:0.25]
-; ZNVER1-NEXT:    orb %cl, %dl # sched: [1:0.25]
-; ZNVER1-NEXT:    movzbl %dl, %eax # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call i32 @llvm.x86.sse2.comieq.sd(<2 x double> %a0, <2 x double> %a1)
-  %2 = load <2 x double>, <2 x double> *%a2, align 8
-  %3 = call i32 @llvm.x86.sse2.comieq.sd(<2 x double> %a0, <2 x double> %2)
-  %4 = or i32 %1, %3
-  ret i32 %4
-}
-declare i32 @llvm.x86.sse2.comieq.sd(<2 x double>, <2 x double>) nounwind readnone
-
-define <2 x double> @test_cvtdq2pd(<4 x i32> %a0, <4 x i32> *%a1) {
-; GENERIC-LABEL: test_cvtdq2pd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    cvtdq2pd %xmm0, %xmm1 # sched: [4:1.00]
-; GENERIC-NEXT:    cvtdq2pd (%rdi), %xmm0 # sched: [10:1.00]
-; GENERIC-NEXT:    addpd %xmm1, %xmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_cvtdq2pd:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    cvtdq2pd (%rdi), %xmm1 # sched: [8:4.00]
-; ATOM-NEXT:    cvtdq2pd %xmm0, %xmm0 # sched: [7:3.50]
-; ATOM-NEXT:    addpd %xmm0, %xmm1 # sched: [6:3.00]
-; ATOM-NEXT:    movapd %xmm1, %xmm0 # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_cvtdq2pd:
-; SLM:       # %bb.0:
-; SLM-NEXT:    cvtdq2pd %xmm0, %xmm1 # sched: [4:0.50]
-; SLM-NEXT:    cvtdq2pd (%rdi), %xmm0 # sched: [7:1.00]
-; SLM-NEXT:    addpd %xmm1, %xmm0 # sched: [3:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_cvtdq2pd:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    cvtdq2pd %xmm0, %xmm1 # sched: [4:1.00]
-; SANDY-SSE-NEXT:    cvtdq2pd (%rdi), %xmm0 # sched: [10:1.00]
-; SANDY-SSE-NEXT:    addpd %xmm1, %xmm0 # sched: [3:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_cvtdq2pd:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vcvtdq2pd %xmm0, %xmm0 # sched: [4:1.00]
-; SANDY-NEXT:    vcvtdq2pd (%rdi), %xmm1 # sched: [10:1.00]
-; SANDY-NEXT:    vaddpd %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_cvtdq2pd:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    cvtdq2pd %xmm0, %xmm1 # sched: [4:1.00]
-; HASWELL-SSE-NEXT:    cvtdq2pd (%rdi), %xmm0 # sched: [10:1.00]
-; HASWELL-SSE-NEXT:    addpd %xmm1, %xmm0 # sched: [3:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_cvtdq2pd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vcvtdq2pd %xmm0, %xmm0 # sched: [4:1.00]
-; HASWELL-NEXT:    vcvtdq2pd (%rdi), %xmm1 # sched: [10:1.00]
-; HASWELL-NEXT:    vaddpd %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_cvtdq2pd:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    cvtdq2pd (%rdi), %xmm1 # sched: [9:1.00]
-; BROADWELL-SSE-NEXT:    cvtdq2pd %xmm0, %xmm0 # sched: [4:1.00]
-; BROADWELL-SSE-NEXT:    addpd %xmm0, %xmm1 # sched: [3:1.00]
-; BROADWELL-SSE-NEXT:    movapd %xmm1, %xmm0 # sched: [1:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_cvtdq2pd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vcvtdq2pd (%rdi), %xmm1 # sched: [9:1.00]
-; BROADWELL-NEXT:    vcvtdq2pd %xmm0, %xmm0 # sched: [4:1.00]
-; BROADWELL-NEXT:    vaddpd %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_cvtdq2pd:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    cvtdq2pd %xmm0, %xmm1 # sched: [5:1.00]
-; SKYLAKE-SSE-NEXT:    cvtdq2pd (%rdi), %xmm0 # sched: [11:1.00]
-; SKYLAKE-SSE-NEXT:    addpd %xmm1, %xmm0 # sched: [4:0.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_cvtdq2pd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vcvtdq2pd %xmm0, %xmm0 # sched: [5:1.00]
-; SKYLAKE-NEXT:    vcvtdq2pd (%rdi), %xmm1 # sched: [11:1.00]
-; SKYLAKE-NEXT:    vaddpd %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_cvtdq2pd:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    cvtdq2pd %xmm0, %xmm1 # sched: [4:0.50]
-; SKX-SSE-NEXT:    cvtdq2pd (%rdi), %xmm0 # sched: [11:1.00]
-; SKX-SSE-NEXT:    addpd %xmm1, %xmm0 # sched: [4:0.50]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_cvtdq2pd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vcvtdq2pd %xmm0, %xmm0 # sched: [4:0.50]
-; SKX-NEXT:    vcvtdq2pd (%rdi), %xmm1 # sched: [11:1.00]
-; SKX-NEXT:    vaddpd %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_cvtdq2pd:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    cvtdq2pd %xmm0, %xmm1 # sched: [8:1.00]
-; BDVER2-SSE-NEXT:    cvtdq2pd (%rdi), %xmm0 # sched: [13:1.00]
-; BDVER2-SSE-NEXT:    addpd %xmm1, %xmm0 # sched: [5:1.00]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_cvtdq2pd:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vcvtdq2pd (%rdi), %xmm1 # sched: [13:1.00]
-; BDVER2-NEXT:    vcvtdq2pd %xmm0, %xmm0 # sched: [8:1.00]
-; BDVER2-NEXT:    vaddpd %xmm1, %xmm0, %xmm0 # sched: [5:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_cvtdq2pd:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    cvtdq2pd %xmm0, %xmm1 # sched: [3:1.00]
-; BTVER2-SSE-NEXT:    cvtdq2pd (%rdi), %xmm0 # sched: [8:1.00]
-; BTVER2-SSE-NEXT:    addpd %xmm1, %xmm0 # sched: [3:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_cvtdq2pd:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vcvtdq2pd (%rdi), %xmm1 # sched: [8:1.00]
-; BTVER2-NEXT:    vcvtdq2pd %xmm0, %xmm0 # sched: [3:1.00]
-; BTVER2-NEXT:    vaddpd %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_cvtdq2pd:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    cvtdq2pd %xmm0, %xmm1 # sched: [5:1.00]
-; ZNVER1-SSE-NEXT:    cvtdq2pd (%rdi), %xmm0 # sched: [12:1.00]
-; ZNVER1-SSE-NEXT:    addpd %xmm1, %xmm0 # sched: [3:1.00]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_cvtdq2pd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vcvtdq2pd (%rdi), %xmm1 # sched: [12:1.00]
-; ZNVER1-NEXT:    vcvtdq2pd %xmm0, %xmm0 # sched: [5:1.00]
-; ZNVER1-NEXT:    vaddpd %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = shufflevector <4 x i32> %a0, <4 x i32> undef, <2 x i32> <i32 0, i32 1>
-  %2 = sitofp <2 x i32> %1 to <2 x double>
-  %3 = load <4 x i32>, <4 x i32>*%a1, align 16
-  %4 = shufflevector <4 x i32> %3, <4 x i32> undef, <2 x i32> <i32 0, i32 1>
-  %5 = sitofp <2 x i32> %4 to <2 x double>
-  %6 = fadd <2 x double> %2, %5
-  ret <2 x double> %6
-}
-
-define <4 x float> @test_cvtdq2ps(<4 x i32> %a0, <4 x i32> *%a1) {
-; GENERIC-LABEL: test_cvtdq2ps:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    cvtdq2ps %xmm0, %xmm1 # sched: [3:1.00]
-; GENERIC-NEXT:    cvtdq2ps (%rdi), %xmm0 # sched: [9:1.00]
-; GENERIC-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_cvtdq2ps:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    cvtdq2ps (%rdi), %xmm1 # sched: [7:3.50]
-; ATOM-NEXT:    cvtdq2ps %xmm0, %xmm0 # sched: [6:3.00]
-; ATOM-NEXT:    addps %xmm0, %xmm1 # sched: [5:5.00]
-; ATOM-NEXT:    movaps %xmm1, %xmm0 # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_cvtdq2ps:
-; SLM:       # %bb.0:
-; SLM-NEXT:    cvtdq2ps %xmm0, %xmm1 # sched: [4:0.50]
-; SLM-NEXT:    cvtdq2ps (%rdi), %xmm0 # sched: [7:1.00]
-; SLM-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_cvtdq2ps:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    cvtdq2ps %xmm0, %xmm1 # sched: [3:1.00]
-; SANDY-SSE-NEXT:    cvtdq2ps (%rdi), %xmm0 # sched: [9:1.00]
-; SANDY-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_cvtdq2ps:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vcvtdq2ps %xmm0, %xmm0 # sched: [3:1.00]
-; SANDY-NEXT:    vcvtdq2ps (%rdi), %xmm1 # sched: [9:1.00]
-; SANDY-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_cvtdq2ps:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    cvtdq2ps %xmm0, %xmm1 # sched: [3:1.00]
-; HASWELL-SSE-NEXT:    cvtdq2ps (%rdi), %xmm0 # sched: [9:1.00]
-; HASWELL-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_cvtdq2ps:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vcvtdq2ps %xmm0, %xmm0 # sched: [3:1.00]
-; HASWELL-NEXT:    vcvtdq2ps (%rdi), %xmm1 # sched: [9:1.00]
-; HASWELL-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_cvtdq2ps:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    cvtdq2ps %xmm0, %xmm1 # sched: [3:1.00]
-; BROADWELL-SSE-NEXT:    cvtdq2ps (%rdi), %xmm0 # sched: [8:1.00]
-; BROADWELL-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_cvtdq2ps:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vcvtdq2ps %xmm0, %xmm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    vcvtdq2ps (%rdi), %xmm1 # sched: [8:1.00]
-; BROADWELL-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_cvtdq2ps:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    cvtdq2ps %xmm0, %xmm1 # sched: [4:0.50]
-; SKYLAKE-SSE-NEXT:    cvtdq2ps (%rdi), %xmm0 # sched: [10:0.50]
-; SKYLAKE-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [4:0.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_cvtdq2ps:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vcvtdq2ps %xmm0, %xmm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    vcvtdq2ps (%rdi), %xmm1 # sched: [10:0.50]
-; SKYLAKE-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_cvtdq2ps:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    cvtdq2ps %xmm0, %xmm1 # sched: [4:0.50]
-; SKX-SSE-NEXT:    cvtdq2ps (%rdi), %xmm0 # sched: [10:0.50]
-; SKX-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [4:0.50]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_cvtdq2ps:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vcvtdq2ps %xmm0, %xmm0 # sched: [4:0.50]
-; SKX-NEXT:    vcvtdq2ps (%rdi), %xmm1 # sched: [10:0.50]
-; SKX-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_cvtdq2ps:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    cvtdq2ps %xmm0, %xmm1 # sched: [4:1.00]
-; BDVER2-SSE-NEXT:    cvtdq2ps (%rdi), %xmm0 # sched: [9:1.00]
-; BDVER2-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [5:1.00]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_cvtdq2ps:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vcvtdq2ps (%rdi), %xmm1 # sched: [9:1.00]
-; BDVER2-NEXT:    vcvtdq2ps %xmm0, %xmm0 # sched: [4:1.00]
-; BDVER2-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [5:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_cvtdq2ps:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    cvtdq2ps %xmm0, %xmm1 # sched: [3:1.00]
-; BTVER2-SSE-NEXT:    cvtdq2ps (%rdi), %xmm0 # sched: [8:1.00]
-; BTVER2-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_cvtdq2ps:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vcvtdq2ps (%rdi), %xmm1 # sched: [8:1.00]
-; BTVER2-NEXT:    vcvtdq2ps %xmm0, %xmm0 # sched: [3:1.00]
-; BTVER2-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_cvtdq2ps:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    cvtdq2ps %xmm0, %xmm1 # sched: [5:1.00]
-; ZNVER1-SSE-NEXT:    cvtdq2ps (%rdi), %xmm0 # sched: [12:1.00]
-; ZNVER1-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_cvtdq2ps:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vcvtdq2ps (%rdi), %xmm1 # sched: [12:1.00]
-; ZNVER1-NEXT:    vcvtdq2ps %xmm0, %xmm0 # sched: [5:1.00]
-; ZNVER1-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = sitofp <4 x i32> %a0 to <4 x float>
-  %2 = load <4 x i32>, <4 x i32>*%a1, align 16
-  %3 = sitofp <4 x i32> %2 to <4 x float>
-  %4 = fadd <4 x float> %1, %3
-  ret <4 x float> %4
-}
-
-define <4 x i32> @test_cvtpd2dq(<2 x double> %a0, <2 x double> *%a1) {
-; GENERIC-LABEL: test_cvtpd2dq:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    cvtpd2dq %xmm0, %xmm1 # sched: [4:1.00]
-; GENERIC-NEXT:    cvtpd2dq (%rdi), %xmm0 # sched: [10:1.00]
-; GENERIC-NEXT:    paddd %xmm1, %xmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_cvtpd2dq:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    cvtpd2dq (%rdi), %xmm1 # sched: [8:4.00]
-; ATOM-NEXT:    cvtpd2dq %xmm0, %xmm0 # sched: [7:3.50]
-; ATOM-NEXT:    paddd %xmm0, %xmm1 # sched: [1:0.50]
-; ATOM-NEXT:    movdqa %xmm1, %xmm0 # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_cvtpd2dq:
-; SLM:       # %bb.0:
-; SLM-NEXT:    cvtpd2dq %xmm0, %xmm1 # sched: [4:0.50]
-; SLM-NEXT:    cvtpd2dq (%rdi), %xmm0 # sched: [7:1.00]
-; SLM-NEXT:    paddd %xmm1, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_cvtpd2dq:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    cvtpd2dq %xmm0, %xmm1 # sched: [4:1.00]
-; SANDY-SSE-NEXT:    cvtpd2dq (%rdi), %xmm0 # sched: [10:1.00]
-; SANDY-SSE-NEXT:    paddd %xmm1, %xmm0 # sched: [1:0.50]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_cvtpd2dq:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vcvtpd2dq %xmm0, %xmm0 # sched: [4:1.00]
-; SANDY-NEXT:    vcvtpd2dqx (%rdi), %xmm1 # sched: [10:1.00]
-; SANDY-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_cvtpd2dq:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    cvtpd2dq %xmm0, %xmm1 # sched: [4:1.00]
-; HASWELL-SSE-NEXT:    cvtpd2dq (%rdi), %xmm0 # sched: [10:1.00]
-; HASWELL-SSE-NEXT:    paddd %xmm1, %xmm0 # sched: [1:0.50]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_cvtpd2dq:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vcvtpd2dq %xmm0, %xmm0 # sched: [4:1.00]
-; HASWELL-NEXT:    vcvtpd2dqx (%rdi), %xmm1 # sched: [8:1.00]
-; HASWELL-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_cvtpd2dq:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    cvtpd2dq (%rdi), %xmm1 # sched: [9:1.00]
-; BROADWELL-SSE-NEXT:    cvtpd2dq %xmm0, %xmm0 # sched: [4:1.00]
-; BROADWELL-SSE-NEXT:    paddd %xmm0, %xmm1 # sched: [1:0.50]
-; BROADWELL-SSE-NEXT:    movdqa %xmm1, %xmm0 # sched: [1:0.33]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_cvtpd2dq:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vcvtpd2dq %xmm0, %xmm0 # sched: [4:1.00]
-; BROADWELL-NEXT:    vcvtpd2dqx (%rdi), %xmm1 # sched: [8:1.00]
-; BROADWELL-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_cvtpd2dq:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    cvtpd2dq %xmm0, %xmm1 # sched: [5:1.00]
-; SKYLAKE-SSE-NEXT:    cvtpd2dq (%rdi), %xmm0 # sched: [11:1.00]
-; SKYLAKE-SSE-NEXT:    paddd %xmm1, %xmm0 # sched: [1:0.33]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_cvtpd2dq:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vcvtpd2dq %xmm0, %xmm0 # sched: [5:1.00]
-; SKYLAKE-NEXT:    vcvtpd2dqx (%rdi), %xmm1 # sched: [8:1.00]
-; SKYLAKE-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_cvtpd2dq:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    cvtpd2dq %xmm0, %xmm1 # sched: [5:1.00]
-; SKX-SSE-NEXT:    cvtpd2dq (%rdi), %xmm0 # sched: [11:1.00]
-; SKX-SSE-NEXT:    paddd %xmm1, %xmm0 # sched: [1:0.33]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_cvtpd2dq:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vcvtpd2dq %xmm0, %xmm0 # sched: [5:1.00]
-; SKX-NEXT:    vcvtpd2dqx (%rdi), %xmm1 # sched: [8:0.50]
-; SKX-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_cvtpd2dq:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    cvtpd2dq %xmm0, %xmm1 # sched: [8:1.00]
-; BDVER2-SSE-NEXT:    cvtpd2dq (%rdi), %xmm0 # sched: [13:1.00]
-; BDVER2-SSE-NEXT:    paddd %xmm1, %xmm0 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_cvtpd2dq:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vcvtpd2dqx (%rdi), %xmm1 # sched: [13:1.00]
-; BDVER2-NEXT:    vcvtpd2dq %xmm0, %xmm0 # sched: [8:1.00]
-; BDVER2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_cvtpd2dq:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    cvtpd2dq %xmm0, %xmm1 # sched: [3:1.00]
-; BTVER2-SSE-NEXT:    cvtpd2dq (%rdi), %xmm0 # sched: [8:1.00]
-; BTVER2-SSE-NEXT:    paddd %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_cvtpd2dq:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vcvtpd2dqx (%rdi), %xmm1 # sched: [8:1.00]
-; BTVER2-NEXT:    vcvtpd2dq %xmm0, %xmm0 # sched: [3:1.00]
-; BTVER2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_cvtpd2dq:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    cvtpd2dq %xmm0, %xmm1 # sched: [5:1.00]
-; ZNVER1-SSE-NEXT:    cvtpd2dq (%rdi), %xmm0 # sched: [12:1.00]
-; ZNVER1-SSE-NEXT:    paddd %xmm1, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_cvtpd2dq:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vcvtpd2dqx (%rdi), %xmm1 # sched: [12:1.00]
-; ZNVER1-NEXT:    vcvtpd2dq %xmm0, %xmm0 # sched: [5:1.00]
-; ZNVER1-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <4 x i32> @llvm.x86.sse2.cvtpd2dq(<2 x double> %a0)
-  %2 = load <2 x double>, <2 x double> *%a1, align 16
-  %3 = call <4 x i32> @llvm.x86.sse2.cvtpd2dq(<2 x double> %2)
-  %4 = add <4 x i32> %1, %3
-  ret <4 x i32> %4
-}
-declare <4 x i32> @llvm.x86.sse2.cvtpd2dq(<2 x double>) nounwind readnone
-
-define <4 x float> @test_cvtpd2ps(<2 x double> %a0, <2 x double> *%a1) {
-; GENERIC-LABEL: test_cvtpd2ps:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    cvtpd2ps %xmm0, %xmm1 # sched: [4:1.00]
-; GENERIC-NEXT:    cvtpd2ps (%rdi), %xmm0 # sched: [10:1.00]
-; GENERIC-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_cvtpd2ps:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    cvtpd2ps (%rdi), %xmm1 # sched: [8:4.00]
-; ATOM-NEXT:    cvtpd2ps %xmm0, %xmm0 # sched: [7:3.50]
-; ATOM-NEXT:    addps %xmm0, %xmm1 # sched: [5:5.00]
-; ATOM-NEXT:    movaps %xmm1, %xmm0 # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_cvtpd2ps:
-; SLM:       # %bb.0:
-; SLM-NEXT:    cvtpd2ps %xmm0, %xmm1 # sched: [4:0.50]
-; SLM-NEXT:    cvtpd2ps (%rdi), %xmm0 # sched: [7:1.00]
-; SLM-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_cvtpd2ps:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    cvtpd2ps %xmm0, %xmm1 # sched: [4:1.00]
-; SANDY-SSE-NEXT:    cvtpd2ps (%rdi), %xmm0 # sched: [10:1.00]
-; SANDY-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_cvtpd2ps:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vcvtpd2ps %xmm0, %xmm0 # sched: [4:1.00]
-; SANDY-NEXT:    vcvtpd2psx (%rdi), %xmm1 # sched: [10:1.00]
-; SANDY-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_cvtpd2ps:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    cvtpd2ps %xmm0, %xmm1 # sched: [4:1.00]
-; HASWELL-SSE-NEXT:    cvtpd2ps (%rdi), %xmm0 # sched: [10:1.00]
-; HASWELL-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_cvtpd2ps:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vcvtpd2ps %xmm0, %xmm0 # sched: [4:1.00]
-; HASWELL-NEXT:    vcvtpd2psx (%rdi), %xmm1 # sched: [8:1.00]
-; HASWELL-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_cvtpd2ps:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    cvtpd2ps (%rdi), %xmm1 # sched: [9:1.00]
-; BROADWELL-SSE-NEXT:    cvtpd2ps %xmm0, %xmm0 # sched: [4:1.00]
-; BROADWELL-SSE-NEXT:    addps %xmm0, %xmm1 # sched: [3:1.00]
-; BROADWELL-SSE-NEXT:    movaps %xmm1, %xmm0 # sched: [1:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_cvtpd2ps:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vcvtpd2ps %xmm0, %xmm0 # sched: [4:1.00]
-; BROADWELL-NEXT:    vcvtpd2psx (%rdi), %xmm1 # sched: [8:1.00]
-; BROADWELL-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_cvtpd2ps:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    cvtpd2ps %xmm0, %xmm1 # sched: [5:1.00]
-; SKYLAKE-SSE-NEXT:    cvtpd2ps (%rdi), %xmm0 # sched: [11:1.00]
-; SKYLAKE-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [4:0.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_cvtpd2ps:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vcvtpd2ps %xmm0, %xmm0 # sched: [5:1.00]
-; SKYLAKE-NEXT:    vcvtpd2psx (%rdi), %xmm1 # sched: [8:1.00]
-; SKYLAKE-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_cvtpd2ps:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    cvtpd2ps %xmm0, %xmm1 # sched: [5:1.00]
-; SKX-SSE-NEXT:    cvtpd2ps (%rdi), %xmm0 # sched: [11:1.00]
-; SKX-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [4:0.50]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_cvtpd2ps:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vcvtpd2ps %xmm0, %xmm0 # sched: [5:1.00]
-; SKX-NEXT:    vcvtpd2psx (%rdi), %xmm1 # sched: [8:1.00]
-; SKX-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_cvtpd2ps:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    cvtpd2ps %xmm0, %xmm1 # sched: [8:1.00]
-; BDVER2-SSE-NEXT:    cvtpd2ps (%rdi), %xmm0 # sched: [13:1.00]
-; BDVER2-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [5:1.00]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_cvtpd2ps:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vcvtpd2psx (%rdi), %xmm1 # sched: [13:1.00]
-; BDVER2-NEXT:    vcvtpd2ps %xmm0, %xmm0 # sched: [8:1.00]
-; BDVER2-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [5:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_cvtpd2ps:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    cvtpd2ps %xmm0, %xmm1 # sched: [3:1.00]
-; BTVER2-SSE-NEXT:    cvtpd2ps (%rdi), %xmm0 # sched: [8:1.00]
-; BTVER2-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_cvtpd2ps:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vcvtpd2psx (%rdi), %xmm1 # sched: [8:1.00]
-; BTVER2-NEXT:    vcvtpd2ps %xmm0, %xmm0 # sched: [3:1.00]
-; BTVER2-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_cvtpd2ps:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    cvtpd2ps %xmm0, %xmm1 # sched: [4:1.00]
-; ZNVER1-SSE-NEXT:    cvtpd2ps (%rdi), %xmm0 # sched: [11:1.00]
-; ZNVER1-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_cvtpd2ps:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vcvtpd2psx (%rdi), %xmm1 # sched: [11:1.00]
-; ZNVER1-NEXT:    vcvtpd2ps %xmm0, %xmm0 # sched: [4:1.00]
-; ZNVER1-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <4 x float> @llvm.x86.sse2.cvtpd2ps(<2 x double> %a0)
-  %2 = load <2 x double>, <2 x double> *%a1, align 16
-  %3 = call <4 x float> @llvm.x86.sse2.cvtpd2ps(<2 x double> %2)
-  %4 = fadd <4 x float> %1, %3
-  ret <4 x float> %4
-}
-declare <4 x float> @llvm.x86.sse2.cvtpd2ps(<2 x double>) nounwind readnone
-
-define <4 x i32> @test_cvtps2dq(<4 x float> %a0, <4 x float> *%a1) {
-; GENERIC-LABEL: test_cvtps2dq:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    cvtps2dq %xmm0, %xmm1 # sched: [3:1.00]
-; GENERIC-NEXT:    cvtps2dq (%rdi), %xmm0 # sched: [9:1.00]
-; GENERIC-NEXT:    paddd %xmm1, %xmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_cvtps2dq:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    cvtps2dq (%rdi), %xmm1 # sched: [7:3.50]
-; ATOM-NEXT:    cvtps2dq %xmm0, %xmm0 # sched: [6:3.00]
-; ATOM-NEXT:    paddd %xmm0, %xmm1 # sched: [1:0.50]
-; ATOM-NEXT:    movdqa %xmm1, %xmm0 # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_cvtps2dq:
-; SLM:       # %bb.0:
-; SLM-NEXT:    cvtps2dq %xmm0, %xmm1 # sched: [4:0.50]
-; SLM-NEXT:    cvtps2dq (%rdi), %xmm0 # sched: [7:1.00]
-; SLM-NEXT:    paddd %xmm1, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_cvtps2dq:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    cvtps2dq %xmm0, %xmm1 # sched: [3:1.00]
-; SANDY-SSE-NEXT:    cvtps2dq (%rdi), %xmm0 # sched: [9:1.00]
-; SANDY-SSE-NEXT:    paddd %xmm1, %xmm0 # sched: [1:0.50]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_cvtps2dq:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vcvtps2dq %xmm0, %xmm0 # sched: [3:1.00]
-; SANDY-NEXT:    vcvtps2dq (%rdi), %xmm1 # sched: [9:1.00]
-; SANDY-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_cvtps2dq:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    cvtps2dq %xmm0, %xmm1 # sched: [3:1.00]
-; HASWELL-SSE-NEXT:    cvtps2dq (%rdi), %xmm0 # sched: [9:1.00]
-; HASWELL-SSE-NEXT:    paddd %xmm1, %xmm0 # sched: [1:0.50]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_cvtps2dq:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vcvtps2dq %xmm0, %xmm0 # sched: [3:1.00]
-; HASWELL-NEXT:    vcvtps2dq (%rdi), %xmm1 # sched: [9:1.00]
-; HASWELL-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_cvtps2dq:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    cvtps2dq %xmm0, %xmm1 # sched: [3:1.00]
-; BROADWELL-SSE-NEXT:    cvtps2dq (%rdi), %xmm0 # sched: [8:1.00]
-; BROADWELL-SSE-NEXT:    paddd %xmm1, %xmm0 # sched: [1:0.50]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_cvtps2dq:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vcvtps2dq %xmm0, %xmm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    vcvtps2dq (%rdi), %xmm1 # sched: [8:1.00]
-; BROADWELL-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_cvtps2dq:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    cvtps2dq %xmm0, %xmm1 # sched: [4:0.50]
-; SKYLAKE-SSE-NEXT:    cvtps2dq (%rdi), %xmm0 # sched: [10:0.50]
-; SKYLAKE-SSE-NEXT:    paddd %xmm1, %xmm0 # sched: [1:0.33]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_cvtps2dq:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vcvtps2dq %xmm0, %xmm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    vcvtps2dq (%rdi), %xmm1 # sched: [10:0.50]
-; SKYLAKE-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_cvtps2dq:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    cvtps2dq %xmm0, %xmm1 # sched: [4:0.50]
-; SKX-SSE-NEXT:    cvtps2dq (%rdi), %xmm0 # sched: [10:0.50]
-; SKX-SSE-NEXT:    paddd %xmm1, %xmm0 # sched: [1:0.33]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_cvtps2dq:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vcvtps2dq %xmm0, %xmm0 # sched: [4:0.50]
-; SKX-NEXT:    vcvtps2dq (%rdi), %xmm1 # sched: [10:0.50]
-; SKX-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_cvtps2dq:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    cvtps2dq %xmm0, %xmm1 # sched: [4:1.00]
-; BDVER2-SSE-NEXT:    cvtps2dq (%rdi), %xmm0 # sched: [9:1.00]
-; BDVER2-SSE-NEXT:    paddd %xmm1, %xmm0 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_cvtps2dq:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vcvtps2dq (%rdi), %xmm1 # sched: [9:1.00]
-; BDVER2-NEXT:    vcvtps2dq %xmm0, %xmm0 # sched: [4:1.00]
-; BDVER2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_cvtps2dq:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    cvtps2dq %xmm0, %xmm1 # sched: [3:1.00]
-; BTVER2-SSE-NEXT:    cvtps2dq (%rdi), %xmm0 # sched: [8:1.00]
-; BTVER2-SSE-NEXT:    paddd %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_cvtps2dq:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vcvtps2dq (%rdi), %xmm1 # sched: [8:1.00]
-; BTVER2-NEXT:    vcvtps2dq %xmm0, %xmm0 # sched: [3:1.00]
-; BTVER2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_cvtps2dq:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    cvtps2dq %xmm0, %xmm1 # sched: [5:1.00]
-; ZNVER1-SSE-NEXT:    cvtps2dq (%rdi), %xmm0 # sched: [12:1.00]
-; ZNVER1-SSE-NEXT:    paddd %xmm1, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_cvtps2dq:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vcvtps2dq (%rdi), %xmm1 # sched: [12:1.00]
-; ZNVER1-NEXT:    vcvtps2dq %xmm0, %xmm0 # sched: [5:1.00]
-; ZNVER1-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <4 x i32> @llvm.x86.sse2.cvtps2dq(<4 x float> %a0)
-  %2 = load <4 x float>, <4 x float> *%a1, align 16
-  %3 = call <4 x i32> @llvm.x86.sse2.cvtps2dq(<4 x float> %2)
-  %4 = add <4 x i32> %1, %3
-  ret <4 x i32> %4
-}
-declare <4 x i32> @llvm.x86.sse2.cvtps2dq(<4 x float>) nounwind readnone
-
-define <2 x double> @test_cvtps2pd(<4 x float> %a0, <4 x float> *%a1) {
-; GENERIC-LABEL: test_cvtps2pd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    cvtps2pd %xmm0, %xmm1 # sched: [2:1.00]
-; GENERIC-NEXT:    cvtps2pd (%rdi), %xmm0 # sched: [7:1.00]
-; GENERIC-NEXT:    addpd %xmm1, %xmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_cvtps2pd:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    cvtps2pd (%rdi), %xmm1 # sched: [8:4.00]
-; ATOM-NEXT:    cvtps2pd %xmm0, %xmm0 # sched: [7:3.50]
-; ATOM-NEXT:    addpd %xmm0, %xmm1 # sched: [6:3.00]
-; ATOM-NEXT:    movapd %xmm1, %xmm0 # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_cvtps2pd:
-; SLM:       # %bb.0:
-; SLM-NEXT:    cvtps2pd %xmm0, %xmm1 # sched: [4:0.50]
-; SLM-NEXT:    cvtps2pd (%rdi), %xmm0 # sched: [7:1.00]
-; SLM-NEXT:    addpd %xmm1, %xmm0 # sched: [3:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_cvtps2pd:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    cvtps2pd %xmm0, %xmm1 # sched: [2:1.00]
-; SANDY-SSE-NEXT:    cvtps2pd (%rdi), %xmm0 # sched: [7:1.00]
-; SANDY-SSE-NEXT:    addpd %xmm1, %xmm0 # sched: [3:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_cvtps2pd:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vcvtps2pd %xmm0, %xmm0 # sched: [2:1.00]
-; SANDY-NEXT:    vcvtps2pd (%rdi), %xmm1 # sched: [7:1.00]
-; SANDY-NEXT:    vaddpd %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_cvtps2pd:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    cvtps2pd %xmm0, %xmm1 # sched: [2:1.00]
-; HASWELL-SSE-NEXT:    cvtps2pd (%rdi), %xmm0 # sched: [6:1.00]
-; HASWELL-SSE-NEXT:    addpd %xmm1, %xmm0 # sched: [3:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_cvtps2pd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vcvtps2pd %xmm0, %xmm0 # sched: [2:1.00]
-; HASWELL-NEXT:    vcvtps2pd (%rdi), %xmm1 # sched: [6:1.00]
-; HASWELL-NEXT:    vaddpd %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_cvtps2pd:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    cvtps2pd %xmm0, %xmm1 # sched: [2:1.00]
-; BROADWELL-SSE-NEXT:    cvtps2pd (%rdi), %xmm0 # sched: [6:1.00]
-; BROADWELL-SSE-NEXT:    addpd %xmm1, %xmm0 # sched: [3:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_cvtps2pd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vcvtps2pd %xmm0, %xmm0 # sched: [2:1.00]
-; BROADWELL-NEXT:    vcvtps2pd (%rdi), %xmm1 # sched: [6:1.00]
-; BROADWELL-NEXT:    vaddpd %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_cvtps2pd:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    cvtps2pd %xmm0, %xmm1 # sched: [5:1.00]
-; SKYLAKE-SSE-NEXT:    cvtps2pd (%rdi), %xmm0 # sched: [9:0.50]
-; SKYLAKE-SSE-NEXT:    addpd %xmm1, %xmm0 # sched: [4:0.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_cvtps2pd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vcvtps2pd %xmm0, %xmm0 # sched: [5:1.00]
-; SKYLAKE-NEXT:    vcvtps2pd (%rdi), %xmm1 # sched: [9:0.50]
-; SKYLAKE-NEXT:    vaddpd %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_cvtps2pd:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    cvtps2pd %xmm0, %xmm1 # sched: [5:1.00]
-; SKX-SSE-NEXT:    cvtps2pd (%rdi), %xmm0 # sched: [9:0.50]
-; SKX-SSE-NEXT:    addpd %xmm1, %xmm0 # sched: [4:0.50]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_cvtps2pd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vcvtps2pd %xmm0, %xmm0 # sched: [5:1.00]
-; SKX-NEXT:    vcvtps2pd (%rdi), %xmm1 # sched: [9:0.50]
-; SKX-NEXT:    vaddpd %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_cvtps2pd:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    cvtps2pd %xmm0, %xmm1 # sched: [8:1.00]
-; BDVER2-SSE-NEXT:    cvtps2pd (%rdi), %xmm0 # sched: [13:1.00]
-; BDVER2-SSE-NEXT:    addpd %xmm1, %xmm0 # sched: [5:1.00]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_cvtps2pd:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vcvtps2pd (%rdi), %xmm1 # sched: [13:1.00]
-; BDVER2-NEXT:    vcvtps2pd %xmm0, %xmm0 # sched: [8:1.00]
-; BDVER2-NEXT:    vaddpd %xmm1, %xmm0, %xmm0 # sched: [5:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_cvtps2pd:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    cvtps2pd %xmm0, %xmm1 # sched: [2:1.00]
-; BTVER2-SSE-NEXT:    cvtps2pd (%rdi), %xmm0 # sched: [7:1.00]
-; BTVER2-SSE-NEXT:    addpd %xmm1, %xmm0 # sched: [3:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_cvtps2pd:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vcvtps2pd (%rdi), %xmm1 # sched: [7:1.00]
-; BTVER2-NEXT:    vcvtps2pd %xmm0, %xmm0 # sched: [2:1.00]
-; BTVER2-NEXT:    vaddpd %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_cvtps2pd:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    cvtps2pd %xmm0, %xmm1 # sched: [3:1.00]
-; ZNVER1-SSE-NEXT:    cvtps2pd (%rdi), %xmm0 # sched: [10:1.00]
-; ZNVER1-SSE-NEXT:    addpd %xmm1, %xmm0 # sched: [3:1.00]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_cvtps2pd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vcvtps2pd (%rdi), %xmm1 # sched: [10:1.00]
-; ZNVER1-NEXT:    vcvtps2pd %xmm0, %xmm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    vaddpd %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = shufflevector <4 x float> %a0, <4 x float> undef, <2 x i32> <i32 0, i32 1>
-  %2 = fpext <2 x float> %1 to <2 x double>
-  %3 = load <4 x float>, <4 x float> *%a1, align 16
-  %4 = shufflevector <4 x float> %3, <4 x float> undef, <2 x i32> <i32 0, i32 1>
-  %5 = fpext <2 x float> %4 to <2 x double>
-  %6 = fadd <2 x double> %2, %5
-  ret <2 x double> %6
-}
-
-define i32 @test_cvtsd2si(double %a0, double *%a1) {
-; GENERIC-LABEL: test_cvtsd2si:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    cvtsd2si %xmm0, %ecx # sched: [5:1.00]
-; GENERIC-NEXT:    cvtsd2si (%rdi), %eax # sched: [9:1.00]
-; GENERIC-NEXT:    addl %ecx, %eax # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_cvtsd2si:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    cvtsd2si (%rdi), %eax # sched: [9:4.50]
-; ATOM-NEXT:    cvtsd2si %xmm0, %ecx # sched: [8:4.00]
-; ATOM-NEXT:    addl %ecx, %eax # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_cvtsd2si:
-; SLM:       # %bb.0:
-; SLM-NEXT:    cvtsd2si (%rdi), %eax # sched: [7:1.00]
-; SLM-NEXT:    cvtsd2si %xmm0, %ecx # sched: [4:0.50]
-; SLM-NEXT:    addl %ecx, %eax # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_cvtsd2si:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    cvtsd2si %xmm0, %ecx # sched: [5:1.00]
-; SANDY-SSE-NEXT:    cvtsd2si (%rdi), %eax # sched: [9:1.00]
-; SANDY-SSE-NEXT:    addl %ecx, %eax # sched: [1:0.33]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_cvtsd2si:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vcvtsd2si %xmm0, %ecx # sched: [5:1.00]
-; SANDY-NEXT:    vcvtsd2si (%rdi), %eax # sched: [10:1.00]
-; SANDY-NEXT:    addl %ecx, %eax # sched: [1:0.33]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_cvtsd2si:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    cvtsd2si %xmm0, %ecx # sched: [4:1.00]
-; HASWELL-SSE-NEXT:    cvtsd2si (%rdi), %eax # sched: [9:1.00]
-; HASWELL-SSE-NEXT:    addl %ecx, %eax # sched: [1:0.25]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_cvtsd2si:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vcvtsd2si %xmm0, %ecx # sched: [4:1.00]
-; HASWELL-NEXT:    vcvtsd2si (%rdi), %eax # sched: [9:1.00]
-; HASWELL-NEXT:    addl %ecx, %eax # sched: [1:0.25]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_cvtsd2si:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    cvtsd2si (%rdi), %eax # sched: [9:1.00]
-; BROADWELL-SSE-NEXT:    cvtsd2si %xmm0, %ecx # sched: [4:1.00]
-; BROADWELL-SSE-NEXT:    addl %ecx, %eax # sched: [1:0.25]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_cvtsd2si:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vcvtsd2si (%rdi), %eax # sched: [9:1.00]
-; BROADWELL-NEXT:    vcvtsd2si %xmm0, %ecx # sched: [4:1.00]
-; BROADWELL-NEXT:    addl %ecx, %eax # sched: [1:0.25]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_cvtsd2si:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    cvtsd2si %xmm0, %ecx # sched: [6:1.00]
-; SKYLAKE-SSE-NEXT:    cvtsd2si (%rdi), %eax # sched: [11:1.00]
-; SKYLAKE-SSE-NEXT:    addl %ecx, %eax # sched: [1:0.25]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_cvtsd2si:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vcvtsd2si %xmm0, %ecx # sched: [6:1.00]
-; SKYLAKE-NEXT:    vcvtsd2si (%rdi), %eax # sched: [11:1.00]
-; SKYLAKE-NEXT:    addl %ecx, %eax # sched: [1:0.25]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_cvtsd2si:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    cvtsd2si %xmm0, %ecx # sched: [6:1.00]
-; SKX-SSE-NEXT:    cvtsd2si (%rdi), %eax # sched: [11:1.00]
-; SKX-SSE-NEXT:    addl %ecx, %eax # sched: [1:0.25]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_cvtsd2si:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vcvtsd2si %xmm0, %ecx # sched: [6:1.00]
-; SKX-NEXT:    vcvtsd2si (%rdi), %eax # sched: [11:1.00]
-; SKX-NEXT:    addl %ecx, %eax # sched: [1:0.25]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_cvtsd2si:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    cvtsd2si (%rdi), %eax # sched: [18:1.00]
-; BDVER2-SSE-NEXT:    cvtsd2si %xmm0, %ecx # sched: [13:1.00]
-; BDVER2-SSE-NEXT:    addl %ecx, %eax # sched: [1:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_cvtsd2si:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vcvtsd2si (%rdi), %eax # sched: [18:1.00]
-; BDVER2-NEXT:    vcvtsd2si %xmm0, %ecx # sched: [13:1.00]
-; BDVER2-NEXT:    addl %ecx, %eax # sched: [1:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_cvtsd2si:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    cvtsd2si (%rdi), %eax # sched: [12:1.00]
-; BTVER2-SSE-NEXT:    cvtsd2si %xmm0, %ecx # sched: [7:1.00]
-; BTVER2-SSE-NEXT:    addl %ecx, %eax # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_cvtsd2si:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vcvtsd2si (%rdi), %eax # sched: [12:1.00]
-; BTVER2-NEXT:    vcvtsd2si %xmm0, %ecx # sched: [7:1.00]
-; BTVER2-NEXT:    addl %ecx, %eax # sched: [1:0.50]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_cvtsd2si:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    cvtsd2si (%rdi), %eax # sched: [12:1.00]
-; ZNVER1-SSE-NEXT:    cvtsd2si %xmm0, %ecx # sched: [5:1.00]
-; ZNVER1-SSE-NEXT:    addl %ecx, %eax # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_cvtsd2si:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vcvtsd2si (%rdi), %eax # sched: [12:1.00]
-; ZNVER1-NEXT:    vcvtsd2si %xmm0, %ecx # sched: [5:1.00]
-; ZNVER1-NEXT:    addl %ecx, %eax # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = insertelement <2 x double> undef, double %a0, i32 0
-  %2 = call i32 @llvm.x86.sse2.cvtsd2si(<2 x double> %1)
-  %3 = load double, double *%a1, align 8
-  %4 = insertelement <2 x double> undef, double %3, i32 0
-  %5 = call i32 @llvm.x86.sse2.cvtsd2si(<2 x double> %4)
-  %6 = add i32 %2, %5
-  ret i32 %6
-}
-declare i32 @llvm.x86.sse2.cvtsd2si(<2 x double>) nounwind readnone
-
-define i64 @test_cvtsd2siq(double %a0, double *%a1) {
-; GENERIC-LABEL: test_cvtsd2siq:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    cvtsd2si %xmm0, %rcx # sched: [5:1.00]
-; GENERIC-NEXT:    cvtsd2si (%rdi), %rax # sched: [9:1.00]
-; GENERIC-NEXT:    addq %rcx, %rax # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_cvtsd2siq:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    cvtsd2si (%rdi), %rax # sched: [9:4.50]
-; ATOM-NEXT:    cvtsd2si %xmm0, %rcx # sched: [8:4.00]
-; ATOM-NEXT:    addq %rcx, %rax # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_cvtsd2siq:
-; SLM:       # %bb.0:
-; SLM-NEXT:    cvtsd2si (%rdi), %rax # sched: [7:1.00]
-; SLM-NEXT:    cvtsd2si %xmm0, %rcx # sched: [4:0.50]
-; SLM-NEXT:    addq %rcx, %rax # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_cvtsd2siq:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    cvtsd2si %xmm0, %rcx # sched: [5:1.00]
-; SANDY-SSE-NEXT:    cvtsd2si (%rdi), %rax # sched: [9:1.00]
-; SANDY-SSE-NEXT:    addq %rcx, %rax # sched: [1:0.33]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_cvtsd2siq:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vcvtsd2si %xmm0, %rcx # sched: [5:1.00]
-; SANDY-NEXT:    vcvtsd2si (%rdi), %rax # sched: [10:1.00]
-; SANDY-NEXT:    addq %rcx, %rax # sched: [1:0.33]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_cvtsd2siq:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    cvtsd2si %xmm0, %rcx # sched: [4:1.00]
-; HASWELL-SSE-NEXT:    cvtsd2si (%rdi), %rax # sched: [9:1.00]
-; HASWELL-SSE-NEXT:    addq %rcx, %rax # sched: [1:0.25]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_cvtsd2siq:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vcvtsd2si %xmm0, %rcx # sched: [4:1.00]
-; HASWELL-NEXT:    vcvtsd2si (%rdi), %rax # sched: [9:1.00]
-; HASWELL-NEXT:    addq %rcx, %rax # sched: [1:0.25]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_cvtsd2siq:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    cvtsd2si (%rdi), %rax # sched: [9:1.00]
-; BROADWELL-SSE-NEXT:    cvtsd2si %xmm0, %rcx # sched: [4:1.00]
-; BROADWELL-SSE-NEXT:    addq %rcx, %rax # sched: [1:0.25]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_cvtsd2siq:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vcvtsd2si (%rdi), %rax # sched: [9:1.00]
-; BROADWELL-NEXT:    vcvtsd2si %xmm0, %rcx # sched: [4:1.00]
-; BROADWELL-NEXT:    addq %rcx, %rax # sched: [1:0.25]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_cvtsd2siq:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    cvtsd2si %xmm0, %rcx # sched: [6:1.00]
-; SKYLAKE-SSE-NEXT:    cvtsd2si (%rdi), %rax # sched: [11:1.00]
-; SKYLAKE-SSE-NEXT:    addq %rcx, %rax # sched: [1:0.25]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_cvtsd2siq:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vcvtsd2si %xmm0, %rcx # sched: [6:1.00]
-; SKYLAKE-NEXT:    vcvtsd2si (%rdi), %rax # sched: [11:1.00]
-; SKYLAKE-NEXT:    addq %rcx, %rax # sched: [1:0.25]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_cvtsd2siq:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    cvtsd2si %xmm0, %rcx # sched: [6:1.00]
-; SKX-SSE-NEXT:    cvtsd2si (%rdi), %rax # sched: [11:1.00]
-; SKX-SSE-NEXT:    addq %rcx, %rax # sched: [1:0.25]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_cvtsd2siq:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vcvtsd2si %xmm0, %rcx # sched: [6:1.00]
-; SKX-NEXT:    vcvtsd2si (%rdi), %rax # sched: [11:1.00]
-; SKX-NEXT:    addq %rcx, %rax # sched: [1:0.25]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_cvtsd2siq:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    cvtsd2si (%rdi), %rax # sched: [18:1.00]
-; BDVER2-SSE-NEXT:    cvtsd2si %xmm0, %rcx # sched: [13:1.00]
-; BDVER2-SSE-NEXT:    addq %rcx, %rax # sched: [1:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_cvtsd2siq:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vcvtsd2si (%rdi), %rax # sched: [18:1.00]
-; BDVER2-NEXT:    vcvtsd2si %xmm0, %rcx # sched: [13:1.00]
-; BDVER2-NEXT:    addq %rcx, %rax # sched: [1:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_cvtsd2siq:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    cvtsd2si (%rdi), %rax # sched: [12:1.00]
-; BTVER2-SSE-NEXT:    cvtsd2si %xmm0, %rcx # sched: [7:1.00]
-; BTVER2-SSE-NEXT:    addq %rcx, %rax # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_cvtsd2siq:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vcvtsd2si (%rdi), %rax # sched: [12:1.00]
-; BTVER2-NEXT:    vcvtsd2si %xmm0, %rcx # sched: [7:1.00]
-; BTVER2-NEXT:    addq %rcx, %rax # sched: [1:0.50]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_cvtsd2siq:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    cvtsd2si (%rdi), %rax # sched: [12:1.00]
-; ZNVER1-SSE-NEXT:    cvtsd2si %xmm0, %rcx # sched: [5:1.00]
-; ZNVER1-SSE-NEXT:    addq %rcx, %rax # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_cvtsd2siq:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vcvtsd2si (%rdi), %rax # sched: [12:1.00]
-; ZNVER1-NEXT:    vcvtsd2si %xmm0, %rcx # sched: [5:1.00]
-; ZNVER1-NEXT:    addq %rcx, %rax # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = insertelement <2 x double> undef, double %a0, i32 0
-  %2 = call i64 @llvm.x86.sse2.cvtsd2si64(<2 x double> %1)
-  %3 = load double, double *%a1, align 8
-  %4 = insertelement <2 x double> undef, double %3, i32 0
-  %5 = call i64 @llvm.x86.sse2.cvtsd2si64(<2 x double> %4)
-  %6 = add i64 %2, %5
-  ret i64 %6
-}
-declare i64 @llvm.x86.sse2.cvtsd2si64(<2 x double>) nounwind readnone
-
-define float @test_cvtsd2ss(double %a0, double *%a1) {
-; GENERIC-LABEL: test_cvtsd2ss:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    cvtsd2ss %xmm0, %xmm1 # sched: [4:1.00]
-; GENERIC-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero sched: [6:0.50]
-; GENERIC-NEXT:    cvtsd2ss %xmm0, %xmm0 # sched: [4:1.00]
-; GENERIC-NEXT:    addss %xmm1, %xmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_cvtsd2ss:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    movsd {{.*#+}} xmm1 = mem[0],zero sched: [1:1.00]
-; ATOM-NEXT:    cvtsd2ss %xmm0, %xmm2 # sched: [6:3.00]
-; ATOM-NEXT:    xorps %xmm0, %xmm0 # sched: [1:0.50]
-; ATOM-NEXT:    cvtsd2ss %xmm1, %xmm0 # sched: [6:3.00]
-; ATOM-NEXT:    addss %xmm2, %xmm0 # sched: [5:5.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_cvtsd2ss:
-; SLM:       # %bb.0:
-; SLM-NEXT:    cvtsd2ss %xmm0, %xmm1 # sched: [4:0.50]
-; SLM-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero sched: [3:1.00]
-; SLM-NEXT:    cvtsd2ss %xmm0, %xmm0 # sched: [4:0.50]
-; SLM-NEXT:    addss %xmm1, %xmm0 # sched: [3:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_cvtsd2ss:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    cvtsd2ss %xmm0, %xmm1 # sched: [4:1.00]
-; SANDY-SSE-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero sched: [6:0.50]
-; SANDY-SSE-NEXT:    cvtsd2ss %xmm0, %xmm0 # sched: [4:1.00]
-; SANDY-SSE-NEXT:    addss %xmm1, %xmm0 # sched: [3:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_cvtsd2ss:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vcvtsd2ss %xmm0, %xmm0, %xmm0 # sched: [4:1.00]
-; SANDY-NEXT:    vmovsd {{.*#+}} xmm1 = mem[0],zero sched: [6:0.50]
-; SANDY-NEXT:    vcvtsd2ss %xmm1, %xmm1, %xmm1 # sched: [4:1.00]
-; SANDY-NEXT:    vaddss %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_cvtsd2ss:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    cvtsd2ss %xmm0, %xmm1 # sched: [4:1.00]
-; HASWELL-SSE-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero sched: [5:0.50]
-; HASWELL-SSE-NEXT:    cvtsd2ss %xmm0, %xmm0 # sched: [4:1.00]
-; HASWELL-SSE-NEXT:    addss %xmm1, %xmm0 # sched: [3:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_cvtsd2ss:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vcvtsd2ss %xmm0, %xmm0, %xmm0 # sched: [4:1.00]
-; HASWELL-NEXT:    vmovsd {{.*#+}} xmm1 = mem[0],zero sched: [5:0.50]
-; HASWELL-NEXT:    vcvtsd2ss %xmm1, %xmm1, %xmm1 # sched: [4:1.00]
-; HASWELL-NEXT:    vaddss %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_cvtsd2ss:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    cvtsd2ss %xmm0, %xmm1 # sched: [4:1.00]
-; BROADWELL-SSE-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero sched: [5:0.50]
-; BROADWELL-SSE-NEXT:    cvtsd2ss %xmm0, %xmm0 # sched: [4:1.00]
-; BROADWELL-SSE-NEXT:    addss %xmm1, %xmm0 # sched: [3:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_cvtsd2ss:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vcvtsd2ss %xmm0, %xmm0, %xmm0 # sched: [4:1.00]
-; BROADWELL-NEXT:    vmovsd {{.*#+}} xmm1 = mem[0],zero sched: [5:0.50]
-; BROADWELL-NEXT:    vcvtsd2ss %xmm1, %xmm1, %xmm1 # sched: [4:1.00]
-; BROADWELL-NEXT:    vaddss %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_cvtsd2ss:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    cvtsd2ss %xmm0, %xmm1 # sched: [5:1.00]
-; SKYLAKE-SSE-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero sched: [5:0.50]
-; SKYLAKE-SSE-NEXT:    cvtsd2ss %xmm0, %xmm0 # sched: [5:1.00]
-; SKYLAKE-SSE-NEXT:    addss %xmm1, %xmm0 # sched: [4:0.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_cvtsd2ss:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vcvtsd2ss %xmm0, %xmm0, %xmm0 # sched: [5:1.00]
-; SKYLAKE-NEXT:    vmovsd {{.*#+}} xmm1 = mem[0],zero sched: [5:0.50]
-; SKYLAKE-NEXT:    vcvtsd2ss %xmm1, %xmm1, %xmm1 # sched: [5:1.00]
-; SKYLAKE-NEXT:    vaddss %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_cvtsd2ss:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    cvtsd2ss %xmm0, %xmm1 # sched: [5:1.00]
-; SKX-SSE-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero sched: [5:0.50]
-; SKX-SSE-NEXT:    cvtsd2ss %xmm0, %xmm0 # sched: [5:1.00]
-; SKX-SSE-NEXT:    addss %xmm1, %xmm0 # sched: [4:0.50]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_cvtsd2ss:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vcvtsd2ss %xmm0, %xmm0, %xmm0 # sched: [5:1.00]
-; SKX-NEXT:    vmovsd {{.*#+}} xmm1 = mem[0],zero sched: [5:0.50]
-; SKX-NEXT:    vcvtsd2ss %xmm1, %xmm1, %xmm1 # sched: [5:1.00]
-; SKX-NEXT:    vaddss %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_cvtsd2ss:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    cvtsd2ss %xmm0, %xmm1 # sched: [4:1.00]
-; BDVER2-SSE-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero sched: [5:0.50]
-; BDVER2-SSE-NEXT:    cvtsd2ss %xmm0, %xmm0 # sched: [4:1.00]
-; BDVER2-SSE-NEXT:    addss %xmm1, %xmm0 # sched: [5:1.00]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_cvtsd2ss:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vmovsd {{.*#+}} xmm1 = mem[0],zero sched: [5:0.50]
-; BDVER2-NEXT:    vcvtsd2ss %xmm0, %xmm0, %xmm0 # sched: [4:1.00]
-; BDVER2-NEXT:    vcvtsd2ss %xmm1, %xmm1, %xmm1 # sched: [4:1.00]
-; BDVER2-NEXT:    vaddss %xmm1, %xmm0, %xmm0 # sched: [5:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_cvtsd2ss:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    cvtsd2ss %xmm0, %xmm1 # sched: [7:2.00]
-; BTVER2-SSE-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero sched: [5:1.00]
-; BTVER2-SSE-NEXT:    cvtsd2ss %xmm0, %xmm0 # sched: [7:2.00]
-; BTVER2-SSE-NEXT:    addss %xmm1, %xmm0 # sched: [3:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_cvtsd2ss:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vcvtsd2ss %xmm0, %xmm0, %xmm0 # sched: [7:2.00]
-; BTVER2-NEXT:    vmovsd {{.*#+}} xmm1 = mem[0],zero sched: [5:1.00]
-; BTVER2-NEXT:    vcvtsd2ss %xmm1, %xmm1, %xmm1 # sched: [7:2.00]
-; BTVER2-NEXT:    vaddss %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_cvtsd2ss:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    cvtsd2ss %xmm0, %xmm1 # sched: [4:1.00]
-; ZNVER1-SSE-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    cvtsd2ss %xmm0, %xmm0 # sched: [4:1.00]
-; ZNVER1-SSE-NEXT:    addss %xmm1, %xmm0 # sched: [3:1.00]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_cvtsd2ss:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vmovsd {{.*#+}} xmm1 = mem[0],zero sched: [8:0.50]
-; ZNVER1-NEXT:    vcvtsd2ss %xmm0, %xmm0, %xmm0 # sched: [4:1.00]
-; ZNVER1-NEXT:    vcvtsd2ss %xmm1, %xmm1, %xmm1 # sched: [4:1.00]
-; ZNVER1-NEXT:    vaddss %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = fptrunc double %a0 to float
-  %2 = load double, double *%a1, align 8
-  %3 = fptrunc double %2 to float
-  %4 = fadd float %1, %3
-  ret float %4
-}
-
-define double @test_cvtsi2sd(i32 %a0, i32 *%a1) {
-; GENERIC-LABEL: test_cvtsi2sd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    cvtsi2sdl %edi, %xmm1 # sched: [4:1.00]
-; GENERIC-NEXT:    cvtsi2sdl (%rsi), %xmm0 # sched: [9:1.00]
-; GENERIC-NEXT:    addsd %xmm1, %xmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_cvtsi2sd:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    cvtsi2sdl (%rsi), %xmm0 # sched: [7:3.50]
-; ATOM-NEXT:    cvtsi2sdl %edi, %xmm1 # sched: [6:3.00]
-; ATOM-NEXT:    addsd %xmm1, %xmm0 # sched: [5:5.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_cvtsi2sd:
-; SLM:       # %bb.0:
-; SLM-NEXT:    cvtsi2sdl (%rsi), %xmm0 # sched: [7:1.00]
-; SLM-NEXT:    cvtsi2sdl %edi, %xmm1 # sched: [4:0.50]
-; SLM-NEXT:    addsd %xmm1, %xmm0 # sched: [3:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_cvtsi2sd:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    cvtsi2sdl %edi, %xmm1 # sched: [4:1.00]
-; SANDY-SSE-NEXT:    cvtsi2sdl (%rsi), %xmm0 # sched: [9:1.00]
-; SANDY-SSE-NEXT:    addsd %xmm1, %xmm0 # sched: [3:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_cvtsi2sd:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vcvtsi2sdl %edi, %xmm0, %xmm0 # sched: [4:1.00]
-; SANDY-NEXT:    vcvtsi2sdl (%rsi), %xmm1, %xmm1 # sched: [9:1.00]
-; SANDY-NEXT:    vaddsd %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_cvtsi2sd:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    cvtsi2sdl %edi, %xmm1 # sched: [4:1.00]
-; HASWELL-SSE-NEXT:    cvtsi2sdl (%rsi), %xmm0 # sched: [9:1.00]
-; HASWELL-SSE-NEXT:    addsd %xmm1, %xmm0 # sched: [3:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_cvtsi2sd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vcvtsi2sdl %edi, %xmm0, %xmm0 # sched: [4:1.00]
-; HASWELL-NEXT:    vcvtsi2sdl (%rsi), %xmm1, %xmm1 # sched: [9:1.00]
-; HASWELL-NEXT:    vaddsd %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_cvtsi2sd:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    cvtsi2sdl %edi, %xmm1 # sched: [4:1.00]
-; BROADWELL-SSE-NEXT:    cvtsi2sdl (%rsi), %xmm0 # sched: [9:1.00]
-; BROADWELL-SSE-NEXT:    addsd %xmm1, %xmm0 # sched: [3:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_cvtsi2sd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vcvtsi2sdl %edi, %xmm0, %xmm0 # sched: [4:1.00]
-; BROADWELL-NEXT:    vcvtsi2sdl (%rsi), %xmm1, %xmm1 # sched: [9:1.00]
-; BROADWELL-NEXT:    vaddsd %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_cvtsi2sd:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    cvtsi2sdl %edi, %xmm1 # sched: [5:1.00]
-; SKYLAKE-SSE-NEXT:    cvtsi2sdl (%rsi), %xmm0 # sched: [9:1.00]
-; SKYLAKE-SSE-NEXT:    addsd %xmm1, %xmm0 # sched: [4:0.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_cvtsi2sd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vcvtsi2sdl %edi, %xmm0, %xmm0 # sched: [5:1.00]
-; SKYLAKE-NEXT:    vcvtsi2sdl (%rsi), %xmm1, %xmm1 # sched: [9:1.00]
-; SKYLAKE-NEXT:    vaddsd %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_cvtsi2sd:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    cvtsi2sdl %edi, %xmm1 # sched: [5:1.00]
-; SKX-SSE-NEXT:    cvtsi2sdl (%rsi), %xmm0 # sched: [9:1.00]
-; SKX-SSE-NEXT:    addsd %xmm1, %xmm0 # sched: [4:0.50]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_cvtsi2sd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vcvtsi2sdl %edi, %xmm0, %xmm0 # sched: [5:1.00]
-; SKX-NEXT:    vcvtsi2sdl (%rsi), %xmm1, %xmm1 # sched: [9:1.00]
-; SKX-NEXT:    vaddsd %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_cvtsi2sd:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    cvtsi2sdl (%rsi), %xmm0 # sched: [9:1.00]
-; BDVER2-SSE-NEXT:    cvtsi2sdl %edi, %xmm1 # sched: [4:1.00]
-; BDVER2-SSE-NEXT:    addsd %xmm1, %xmm0 # sched: [5:1.00]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_cvtsi2sd:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vcvtsi2sdl %edi, %xmm0, %xmm0 # sched: [4:1.00]
-; BDVER2-NEXT:    vcvtsi2sdl (%rsi), %xmm1, %xmm1 # sched: [9:1.00]
-; BDVER2-NEXT:    vaddsd %xmm1, %xmm0, %xmm0 # sched: [5:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_cvtsi2sd:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    cvtsi2sdl (%rsi), %xmm0 # sched: [14:1.00]
-; BTVER2-SSE-NEXT:    cvtsi2sdl %edi, %xmm1 # sched: [9:1.00]
-; BTVER2-SSE-NEXT:    addsd %xmm1, %xmm0 # sched: [3:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_cvtsi2sd:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vcvtsi2sdl %edi, %xmm0, %xmm0 # sched: [9:1.00]
-; BTVER2-NEXT:    vcvtsi2sdl (%rsi), %xmm1, %xmm1 # sched: [14:1.00]
-; BTVER2-NEXT:    vaddsd %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_cvtsi2sd:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    cvtsi2sdl (%rsi), %xmm0 # sched: [12:1.00]
-; ZNVER1-SSE-NEXT:    cvtsi2sdl %edi, %xmm1 # sched: [5:1.00]
-; ZNVER1-SSE-NEXT:    addsd %xmm1, %xmm0 # sched: [3:1.00]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_cvtsi2sd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vcvtsi2sdl %edi, %xmm0, %xmm0 # sched: [5:1.00]
-; ZNVER1-NEXT:    vcvtsi2sdl (%rsi), %xmm1, %xmm1 # sched: [12:1.00]
-; ZNVER1-NEXT:    vaddsd %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = sitofp i32 %a0 to double
-  %2 = load i32, i32 *%a1, align 8
-  %3 = sitofp i32 %2 to double
-  %4 = fadd double %1, %3
-  ret double %4
-}
-
-define double @test_cvtsi2sdq(i64 %a0, i64 *%a1) {
-; GENERIC-LABEL: test_cvtsi2sdq:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    cvtsi2sdq %rdi, %xmm1 # sched: [4:1.00]
-; GENERIC-NEXT:    cvtsi2sdq (%rsi), %xmm0 # sched: [9:1.00]
-; GENERIC-NEXT:    addsd %xmm1, %xmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_cvtsi2sdq:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    cvtsi2sdq (%rsi), %xmm0 # sched: [7:3.50]
-; ATOM-NEXT:    cvtsi2sdq %rdi, %xmm1 # sched: [6:3.00]
-; ATOM-NEXT:    addsd %xmm1, %xmm0 # sched: [5:5.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_cvtsi2sdq:
-; SLM:       # %bb.0:
-; SLM-NEXT:    cvtsi2sdq (%rsi), %xmm0 # sched: [7:1.00]
-; SLM-NEXT:    cvtsi2sdq %rdi, %xmm1 # sched: [4:0.50]
-; SLM-NEXT:    addsd %xmm1, %xmm0 # sched: [3:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_cvtsi2sdq:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    cvtsi2sdq %rdi, %xmm1 # sched: [4:1.00]
-; SANDY-SSE-NEXT:    cvtsi2sdq (%rsi), %xmm0 # sched: [9:1.00]
-; SANDY-SSE-NEXT:    addsd %xmm1, %xmm0 # sched: [3:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_cvtsi2sdq:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vcvtsi2sdq %rdi, %xmm0, %xmm0 # sched: [4:1.00]
-; SANDY-NEXT:    vcvtsi2sdq (%rsi), %xmm1, %xmm1 # sched: [9:1.00]
-; SANDY-NEXT:    vaddsd %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_cvtsi2sdq:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    cvtsi2sdq %rdi, %xmm1 # sched: [4:1.00]
-; HASWELL-SSE-NEXT:    cvtsi2sdq (%rsi), %xmm0 # sched: [9:1.00]
-; HASWELL-SSE-NEXT:    addsd %xmm1, %xmm0 # sched: [3:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_cvtsi2sdq:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vcvtsi2sdq %rdi, %xmm0, %xmm0 # sched: [4:1.00]
-; HASWELL-NEXT:    vcvtsi2sdq (%rsi), %xmm1, %xmm1 # sched: [9:1.00]
-; HASWELL-NEXT:    vaddsd %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_cvtsi2sdq:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    cvtsi2sdq %rdi, %xmm1 # sched: [4:1.00]
-; BROADWELL-SSE-NEXT:    cvtsi2sdq (%rsi), %xmm0 # sched: [9:1.00]
-; BROADWELL-SSE-NEXT:    addsd %xmm1, %xmm0 # sched: [3:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_cvtsi2sdq:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vcvtsi2sdq %rdi, %xmm0, %xmm0 # sched: [4:1.00]
-; BROADWELL-NEXT:    vcvtsi2sdq (%rsi), %xmm1, %xmm1 # sched: [9:1.00]
-; BROADWELL-NEXT:    vaddsd %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_cvtsi2sdq:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    cvtsi2sdq %rdi, %xmm1 # sched: [5:1.00]
-; SKYLAKE-SSE-NEXT:    cvtsi2sdq (%rsi), %xmm0 # sched: [9:1.00]
-; SKYLAKE-SSE-NEXT:    addsd %xmm1, %xmm0 # sched: [4:0.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_cvtsi2sdq:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vcvtsi2sdq %rdi, %xmm0, %xmm0 # sched: [5:1.00]
-; SKYLAKE-NEXT:    vcvtsi2sdq (%rsi), %xmm1, %xmm1 # sched: [9:1.00]
-; SKYLAKE-NEXT:    vaddsd %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_cvtsi2sdq:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    cvtsi2sdq %rdi, %xmm1 # sched: [5:1.00]
-; SKX-SSE-NEXT:    cvtsi2sdq (%rsi), %xmm0 # sched: [9:1.00]
-; SKX-SSE-NEXT:    addsd %xmm1, %xmm0 # sched: [4:0.50]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_cvtsi2sdq:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vcvtsi2sdq %rdi, %xmm0, %xmm0 # sched: [5:1.00]
-; SKX-NEXT:    vcvtsi2sdq (%rsi), %xmm1, %xmm1 # sched: [9:1.00]
-; SKX-NEXT:    vaddsd %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_cvtsi2sdq:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    cvtsi2sdq %rdi, %xmm1 # sched: [13:1.00]
-; BDVER2-SSE-NEXT:    cvtsi2sdq (%rsi), %xmm0 # sched: [9:1.00]
-; BDVER2-SSE-NEXT:    addsd %xmm1, %xmm0 # sched: [5:1.00]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_cvtsi2sdq:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vcvtsi2sdq %rdi, %xmm0, %xmm0 # sched: [4:1.00]
-; BDVER2-NEXT:    vcvtsi2sdq (%rsi), %xmm1, %xmm1 # sched: [9:1.00]
-; BDVER2-NEXT:    vaddsd %xmm1, %xmm0, %xmm0 # sched: [5:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_cvtsi2sdq:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    cvtsi2sdq (%rsi), %xmm0 # sched: [14:1.00]
-; BTVER2-SSE-NEXT:    cvtsi2sdq %rdi, %xmm1 # sched: [9:1.00]
-; BTVER2-SSE-NEXT:    addsd %xmm1, %xmm0 # sched: [3:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_cvtsi2sdq:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vcvtsi2sdq %rdi, %xmm0, %xmm0 # sched: [9:1.00]
-; BTVER2-NEXT:    vcvtsi2sdq (%rsi), %xmm1, %xmm1 # sched: [14:1.00]
-; BTVER2-NEXT:    vaddsd %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_cvtsi2sdq:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    cvtsi2sdq (%rsi), %xmm0 # sched: [12:1.00]
-; ZNVER1-SSE-NEXT:    cvtsi2sdq %rdi, %xmm1 # sched: [5:1.00]
-; ZNVER1-SSE-NEXT:    addsd %xmm1, %xmm0 # sched: [3:1.00]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_cvtsi2sdq:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vcvtsi2sdq %rdi, %xmm0, %xmm0 # sched: [5:1.00]
-; ZNVER1-NEXT:    vcvtsi2sdq (%rsi), %xmm1, %xmm1 # sched: [12:1.00]
-; ZNVER1-NEXT:    vaddsd %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = sitofp i64 %a0 to double
-  %2 = load i64, i64 *%a1, align 8
-  %3 = sitofp i64 %2 to double
-  %4 = fadd double %1, %3
-  ret double %4
-}
-
-; TODO - cvtss2sd_m
-
-define double @test_cvtss2sd(float %a0, float *%a1) {
-; GENERIC-LABEL: test_cvtss2sd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    cvtss2sd %xmm0, %xmm1 # sched: [1:1.00]
-; GENERIC-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero sched: [6:0.50]
-; GENERIC-NEXT:    cvtss2sd %xmm0, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    addsd %xmm1, %xmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_cvtss2sd:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero sched: [1:1.00]
-; ATOM-NEXT:    cvtss2sd %xmm0, %xmm2 # sched: [6:3.00]
-; ATOM-NEXT:    xorps %xmm0, %xmm0 # sched: [1:0.50]
-; ATOM-NEXT:    cvtss2sd %xmm1, %xmm0 # sched: [6:3.00]
-; ATOM-NEXT:    addsd %xmm2, %xmm0 # sched: [5:5.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_cvtss2sd:
-; SLM:       # %bb.0:
-; SLM-NEXT:    cvtss2sd %xmm0, %xmm1 # sched: [4:0.50]
-; SLM-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero sched: [3:1.00]
-; SLM-NEXT:    cvtss2sd %xmm0, %xmm0 # sched: [4:0.50]
-; SLM-NEXT:    addsd %xmm1, %xmm0 # sched: [3:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_cvtss2sd:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    cvtss2sd %xmm0, %xmm1 # sched: [1:1.00]
-; SANDY-SSE-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero sched: [6:0.50]
-; SANDY-SSE-NEXT:    cvtss2sd %xmm0, %xmm0 # sched: [1:1.00]
-; SANDY-SSE-NEXT:    addsd %xmm1, %xmm0 # sched: [3:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_cvtss2sd:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vcvtss2sd %xmm0, %xmm0, %xmm0 # sched: [1:1.00]
-; SANDY-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero sched: [6:0.50]
-; SANDY-NEXT:    vcvtss2sd %xmm1, %xmm1, %xmm1 # sched: [1:1.00]
-; SANDY-NEXT:    vaddsd %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_cvtss2sd:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    cvtss2sd %xmm0, %xmm1 # sched: [2:1.00]
-; HASWELL-SSE-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero sched: [5:0.50]
-; HASWELL-SSE-NEXT:    cvtss2sd %xmm0, %xmm0 # sched: [2:1.00]
-; HASWELL-SSE-NEXT:    addsd %xmm1, %xmm0 # sched: [3:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_cvtss2sd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vcvtss2sd %xmm0, %xmm0, %xmm0 # sched: [2:1.00]
-; HASWELL-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero sched: [5:0.50]
-; HASWELL-NEXT:    vcvtss2sd %xmm1, %xmm1, %xmm1 # sched: [2:1.00]
-; HASWELL-NEXT:    vaddsd %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_cvtss2sd:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    cvtss2sd %xmm0, %xmm1 # sched: [2:1.00]
-; BROADWELL-SSE-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero sched: [5:0.50]
-; BROADWELL-SSE-NEXT:    cvtss2sd %xmm0, %xmm0 # sched: [2:1.00]
-; BROADWELL-SSE-NEXT:    addsd %xmm1, %xmm0 # sched: [3:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_cvtss2sd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vcvtss2sd %xmm0, %xmm0, %xmm0 # sched: [2:1.00]
-; BROADWELL-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero sched: [5:0.50]
-; BROADWELL-NEXT:    vcvtss2sd %xmm1, %xmm1, %xmm1 # sched: [2:1.00]
-; BROADWELL-NEXT:    vaddsd %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_cvtss2sd:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    cvtss2sd %xmm0, %xmm1 # sched: [5:1.00]
-; SKYLAKE-SSE-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero sched: [5:0.50]
-; SKYLAKE-SSE-NEXT:    cvtss2sd %xmm0, %xmm0 # sched: [5:1.00]
-; SKYLAKE-SSE-NEXT:    addsd %xmm1, %xmm0 # sched: [4:0.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_cvtss2sd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vcvtss2sd %xmm0, %xmm0, %xmm0 # sched: [5:1.00]
-; SKYLAKE-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero sched: [5:0.50]
-; SKYLAKE-NEXT:    vcvtss2sd %xmm1, %xmm1, %xmm1 # sched: [5:1.00]
-; SKYLAKE-NEXT:    vaddsd %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_cvtss2sd:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    cvtss2sd %xmm0, %xmm1 # sched: [5:1.00]
-; SKX-SSE-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero sched: [5:0.50]
-; SKX-SSE-NEXT:    cvtss2sd %xmm0, %xmm0 # sched: [5:1.00]
-; SKX-SSE-NEXT:    addsd %xmm1, %xmm0 # sched: [4:0.50]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_cvtss2sd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vcvtss2sd %xmm0, %xmm0, %xmm0 # sched: [5:1.00]
-; SKX-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero sched: [5:0.50]
-; SKX-NEXT:    vcvtss2sd %xmm1, %xmm1, %xmm1 # sched: [5:1.00]
-; SKX-NEXT:    vaddsd %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_cvtss2sd:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    cvtss2sd %xmm0, %xmm1 # sched: [4:1.00]
-; BDVER2-SSE-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero sched: [5:0.50]
-; BDVER2-SSE-NEXT:    cvtss2sd %xmm0, %xmm0 # sched: [4:1.00]
-; BDVER2-SSE-NEXT:    addsd %xmm1, %xmm0 # sched: [5:1.00]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_cvtss2sd:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero sched: [5:0.50]
-; BDVER2-NEXT:    vcvtss2sd %xmm0, %xmm0, %xmm0 # sched: [4:1.00]
-; BDVER2-NEXT:    vcvtss2sd %xmm1, %xmm1, %xmm1 # sched: [4:1.00]
-; BDVER2-NEXT:    vaddsd %xmm1, %xmm0, %xmm0 # sched: [5:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_cvtss2sd:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    cvtss2sd %xmm0, %xmm1 # sched: [7:2.00]
-; BTVER2-SSE-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero sched: [5:1.00]
-; BTVER2-SSE-NEXT:    cvtss2sd %xmm0, %xmm0 # sched: [7:2.00]
-; BTVER2-SSE-NEXT:    addsd %xmm1, %xmm0 # sched: [3:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_cvtss2sd:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vcvtss2sd %xmm0, %xmm0, %xmm0 # sched: [7:2.00]
-; BTVER2-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero sched: [5:1.00]
-; BTVER2-NEXT:    vcvtss2sd %xmm1, %xmm1, %xmm1 # sched: [7:2.00]
-; BTVER2-NEXT:    vaddsd %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_cvtss2sd:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    cvtss2sd %xmm0, %xmm1 # sched: [4:1.00]
-; ZNVER1-SSE-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    cvtss2sd %xmm0, %xmm0 # sched: [4:1.00]
-; ZNVER1-SSE-NEXT:    addsd %xmm1, %xmm0 # sched: [3:1.00]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_cvtss2sd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero sched: [8:0.50]
-; ZNVER1-NEXT:    vcvtss2sd %xmm0, %xmm0, %xmm0 # sched: [4:1.00]
-; ZNVER1-NEXT:    vcvtss2sd %xmm1, %xmm1, %xmm1 # sched: [4:1.00]
-; ZNVER1-NEXT:    vaddsd %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = fpext float %a0 to double
-  %2 = load float, float *%a1, align 4
-  %3 = fpext float %2 to double
-  %4 = fadd double %1, %3
-  ret double %4
-}
-
-define <4 x i32> @test_cvttpd2dq(<2 x double> %a0, <2 x double> *%a1) {
-; GENERIC-LABEL: test_cvttpd2dq:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    cvttpd2dq %xmm0, %xmm1 # sched: [4:1.00]
-; GENERIC-NEXT:    cvttpd2dq (%rdi), %xmm0 # sched: [10:1.00]
-; GENERIC-NEXT:    paddd %xmm1, %xmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_cvttpd2dq:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    cvttpd2dq (%rdi), %xmm1 # sched: [8:4.00]
-; ATOM-NEXT:    cvttpd2dq %xmm0, %xmm0 # sched: [7:3.50]
-; ATOM-NEXT:    paddd %xmm0, %xmm1 # sched: [1:0.50]
-; ATOM-NEXT:    movdqa %xmm1, %xmm0 # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_cvttpd2dq:
-; SLM:       # %bb.0:
-; SLM-NEXT:    cvttpd2dq %xmm0, %xmm1 # sched: [4:0.50]
-; SLM-NEXT:    cvttpd2dq (%rdi), %xmm0 # sched: [7:1.00]
-; SLM-NEXT:    paddd %xmm1, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_cvttpd2dq:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    cvttpd2dq %xmm0, %xmm1 # sched: [4:1.00]
-; SANDY-SSE-NEXT:    cvttpd2dq (%rdi), %xmm0 # sched: [10:1.00]
-; SANDY-SSE-NEXT:    paddd %xmm1, %xmm0 # sched: [1:0.50]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_cvttpd2dq:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vcvttpd2dq %xmm0, %xmm0 # sched: [4:1.00]
-; SANDY-NEXT:    vcvttpd2dqx (%rdi), %xmm1 # sched: [10:1.00]
-; SANDY-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_cvttpd2dq:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    cvttpd2dq %xmm0, %xmm1 # sched: [4:1.00]
-; HASWELL-SSE-NEXT:    cvttpd2dq (%rdi), %xmm0 # sched: [10:1.00]
-; HASWELL-SSE-NEXT:    paddd %xmm1, %xmm0 # sched: [1:0.50]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_cvttpd2dq:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vcvttpd2dq %xmm0, %xmm0 # sched: [4:1.00]
-; HASWELL-NEXT:    vcvttpd2dqx (%rdi), %xmm1 # sched: [8:1.00]
-; HASWELL-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_cvttpd2dq:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    cvttpd2dq (%rdi), %xmm1 # sched: [9:1.00]
-; BROADWELL-SSE-NEXT:    cvttpd2dq %xmm0, %xmm0 # sched: [4:1.00]
-; BROADWELL-SSE-NEXT:    paddd %xmm0, %xmm1 # sched: [1:0.50]
-; BROADWELL-SSE-NEXT:    movdqa %xmm1, %xmm0 # sched: [1:0.33]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_cvttpd2dq:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vcvttpd2dq %xmm0, %xmm0 # sched: [4:1.00]
-; BROADWELL-NEXT:    vcvttpd2dqx (%rdi), %xmm1 # sched: [8:1.00]
-; BROADWELL-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_cvttpd2dq:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    cvttpd2dq %xmm0, %xmm1 # sched: [5:1.00]
-; SKYLAKE-SSE-NEXT:    cvttpd2dq (%rdi), %xmm0 # sched: [11:1.00]
-; SKYLAKE-SSE-NEXT:    paddd %xmm1, %xmm0 # sched: [1:0.33]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_cvttpd2dq:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vcvttpd2dq %xmm0, %xmm0 # sched: [5:1.00]
-; SKYLAKE-NEXT:    vcvttpd2dqx (%rdi), %xmm1 # sched: [8:1.00]
-; SKYLAKE-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_cvttpd2dq:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    cvttpd2dq %xmm0, %xmm1 # sched: [5:1.00]
-; SKX-SSE-NEXT:    cvttpd2dq (%rdi), %xmm0 # sched: [11:1.00]
-; SKX-SSE-NEXT:    paddd %xmm1, %xmm0 # sched: [1:0.33]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_cvttpd2dq:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vcvttpd2dq %xmm0, %xmm0 # sched: [5:1.00]
-; SKX-NEXT:    vcvttpd2dqx (%rdi), %xmm1 # sched: [8:0.50]
-; SKX-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_cvttpd2dq:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    cvttpd2dq %xmm0, %xmm1 # sched: [8:1.00]
-; BDVER2-SSE-NEXT:    cvttpd2dq (%rdi), %xmm0 # sched: [13:1.00]
-; BDVER2-SSE-NEXT:    paddd %xmm1, %xmm0 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_cvttpd2dq:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vcvttpd2dqx (%rdi), %xmm1 # sched: [13:1.00]
-; BDVER2-NEXT:    vcvttpd2dq %xmm0, %xmm0 # sched: [8:1.00]
-; BDVER2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_cvttpd2dq:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    cvttpd2dq %xmm0, %xmm1 # sched: [3:1.00]
-; BTVER2-SSE-NEXT:    cvttpd2dq (%rdi), %xmm0 # sched: [8:1.00]
-; BTVER2-SSE-NEXT:    paddd %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_cvttpd2dq:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vcvttpd2dqx (%rdi), %xmm1 # sched: [8:1.00]
-; BTVER2-NEXT:    vcvttpd2dq %xmm0, %xmm0 # sched: [3:1.00]
-; BTVER2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_cvttpd2dq:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    cvttpd2dq %xmm0, %xmm1 # sched: [5:1.00]
-; ZNVER1-SSE-NEXT:    cvttpd2dq (%rdi), %xmm0 # sched: [12:1.00]
-; ZNVER1-SSE-NEXT:    paddd %xmm1, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_cvttpd2dq:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vcvttpd2dqx (%rdi), %xmm1 # sched: [12:1.00]
-; ZNVER1-NEXT:    vcvttpd2dq %xmm0, %xmm0 # sched: [5:1.00]
-; ZNVER1-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = fptosi <2 x double> %a0 to <2 x i32>
-  %2 = shufflevector <2 x i32> %1, <2 x i32> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-  %3 = load <2 x double>, <2 x double> *%a1, align 16
-  %4 = fptosi <2 x double> %3 to <2 x i32>
-  %5 = shufflevector <2 x i32> %4, <2 x i32> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-  %6 = add <4 x i32> %2, %5
-  ret <4 x i32> %6
-}
-
-define <4 x i32> @test_cvttps2dq(<4 x float> %a0, <4 x float> *%a1) {
-; GENERIC-LABEL: test_cvttps2dq:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    cvttps2dq %xmm0, %xmm1 # sched: [3:1.00]
-; GENERIC-NEXT:    cvttps2dq (%rdi), %xmm0 # sched: [9:1.00]
-; GENERIC-NEXT:    paddd %xmm1, %xmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_cvttps2dq:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    cvttps2dq (%rdi), %xmm1 # sched: [7:3.50]
-; ATOM-NEXT:    cvttps2dq %xmm0, %xmm0 # sched: [6:3.00]
-; ATOM-NEXT:    paddd %xmm0, %xmm1 # sched: [1:0.50]
-; ATOM-NEXT:    movdqa %xmm1, %xmm0 # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_cvttps2dq:
-; SLM:       # %bb.0:
-; SLM-NEXT:    cvttps2dq %xmm0, %xmm1 # sched: [4:0.50]
-; SLM-NEXT:    cvttps2dq (%rdi), %xmm0 # sched: [7:1.00]
-; SLM-NEXT:    paddd %xmm1, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_cvttps2dq:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    cvttps2dq %xmm0, %xmm1 # sched: [3:1.00]
-; SANDY-SSE-NEXT:    cvttps2dq (%rdi), %xmm0 # sched: [9:1.00]
-; SANDY-SSE-NEXT:    paddd %xmm1, %xmm0 # sched: [1:0.50]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_cvttps2dq:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vcvttps2dq %xmm0, %xmm0 # sched: [3:1.00]
-; SANDY-NEXT:    vcvttps2dq (%rdi), %xmm1 # sched: [9:1.00]
-; SANDY-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_cvttps2dq:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    cvttps2dq %xmm0, %xmm1 # sched: [3:1.00]
-; HASWELL-SSE-NEXT:    cvttps2dq (%rdi), %xmm0 # sched: [9:1.00]
-; HASWELL-SSE-NEXT:    paddd %xmm1, %xmm0 # sched: [1:0.50]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_cvttps2dq:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vcvttps2dq %xmm0, %xmm0 # sched: [3:1.00]
-; HASWELL-NEXT:    vcvttps2dq (%rdi), %xmm1 # sched: [9:1.00]
-; HASWELL-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_cvttps2dq:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    cvttps2dq %xmm0, %xmm1 # sched: [3:1.00]
-; BROADWELL-SSE-NEXT:    cvttps2dq (%rdi), %xmm0 # sched: [8:1.00]
-; BROADWELL-SSE-NEXT:    paddd %xmm1, %xmm0 # sched: [1:0.50]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_cvttps2dq:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vcvttps2dq %xmm0, %xmm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    vcvttps2dq (%rdi), %xmm1 # sched: [8:1.00]
-; BROADWELL-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_cvttps2dq:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    cvttps2dq %xmm0, %xmm1 # sched: [4:0.50]
-; SKYLAKE-SSE-NEXT:    cvttps2dq (%rdi), %xmm0 # sched: [10:0.50]
-; SKYLAKE-SSE-NEXT:    paddd %xmm1, %xmm0 # sched: [1:0.33]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_cvttps2dq:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vcvttps2dq %xmm0, %xmm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    vcvttps2dq (%rdi), %xmm1 # sched: [10:0.50]
-; SKYLAKE-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_cvttps2dq:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    cvttps2dq %xmm0, %xmm1 # sched: [4:0.50]
-; SKX-SSE-NEXT:    cvttps2dq (%rdi), %xmm0 # sched: [10:0.50]
-; SKX-SSE-NEXT:    paddd %xmm1, %xmm0 # sched: [1:0.33]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_cvttps2dq:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vcvttps2dq %xmm0, %xmm0 # sched: [4:0.50]
-; SKX-NEXT:    vcvttps2dq (%rdi), %xmm1 # sched: [10:0.50]
-; SKX-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_cvttps2dq:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    cvttps2dq %xmm0, %xmm1 # sched: [4:1.00]
-; BDVER2-SSE-NEXT:    cvttps2dq (%rdi), %xmm0 # sched: [9:1.00]
-; BDVER2-SSE-NEXT:    paddd %xmm1, %xmm0 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_cvttps2dq:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vcvttps2dq (%rdi), %xmm1 # sched: [9:1.00]
-; BDVER2-NEXT:    vcvttps2dq %xmm0, %xmm0 # sched: [4:1.00]
-; BDVER2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_cvttps2dq:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    cvttps2dq %xmm0, %xmm1 # sched: [3:1.00]
-; BTVER2-SSE-NEXT:    cvttps2dq (%rdi), %xmm0 # sched: [8:1.00]
-; BTVER2-SSE-NEXT:    paddd %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_cvttps2dq:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vcvttps2dq (%rdi), %xmm1 # sched: [8:1.00]
-; BTVER2-NEXT:    vcvttps2dq %xmm0, %xmm0 # sched: [3:1.00]
-; BTVER2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_cvttps2dq:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    cvttps2dq %xmm0, %xmm1 # sched: [5:1.00]
-; ZNVER1-SSE-NEXT:    cvttps2dq (%rdi), %xmm0 # sched: [12:1.00]
-; ZNVER1-SSE-NEXT:    paddd %xmm1, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_cvttps2dq:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vcvttps2dq (%rdi), %xmm1 # sched: [12:1.00]
-; ZNVER1-NEXT:    vcvttps2dq %xmm0, %xmm0 # sched: [5:1.00]
-; ZNVER1-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = fptosi <4 x float> %a0 to <4 x i32>
-  %2 = load <4 x float>, <4 x float> *%a1, align 16
-  %3 = fptosi <4 x float> %2 to <4 x i32>
-  %4 = add <4 x i32> %1, %3
-  ret <4 x i32> %4
-}
-
-define i32 @test_cvttsd2si(double %a0, double *%a1) {
-; GENERIC-LABEL: test_cvttsd2si:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    cvttsd2si %xmm0, %ecx # sched: [5:1.00]
-; GENERIC-NEXT:    cvttsd2si (%rdi), %eax # sched: [9:1.00]
-; GENERIC-NEXT:    addl %ecx, %eax # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_cvttsd2si:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    cvttsd2si (%rdi), %eax # sched: [9:4.50]
-; ATOM-NEXT:    cvttsd2si %xmm0, %ecx # sched: [8:4.00]
-; ATOM-NEXT:    addl %ecx, %eax # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_cvttsd2si:
-; SLM:       # %bb.0:
-; SLM-NEXT:    cvttsd2si (%rdi), %eax # sched: [7:1.00]
-; SLM-NEXT:    cvttsd2si %xmm0, %ecx # sched: [4:0.50]
-; SLM-NEXT:    addl %ecx, %eax # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_cvttsd2si:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    cvttsd2si %xmm0, %ecx # sched: [5:1.00]
-; SANDY-SSE-NEXT:    cvttsd2si (%rdi), %eax # sched: [9:1.00]
-; SANDY-SSE-NEXT:    addl %ecx, %eax # sched: [1:0.33]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_cvttsd2si:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vcvttsd2si %xmm0, %ecx # sched: [5:1.00]
-; SANDY-NEXT:    vcvttsd2si (%rdi), %eax # sched: [10:1.00]
-; SANDY-NEXT:    addl %ecx, %eax # sched: [1:0.33]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_cvttsd2si:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    cvttsd2si %xmm0, %ecx # sched: [4:1.00]
-; HASWELL-SSE-NEXT:    cvttsd2si (%rdi), %eax # sched: [9:1.00]
-; HASWELL-SSE-NEXT:    addl %ecx, %eax # sched: [1:0.25]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_cvttsd2si:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vcvttsd2si %xmm0, %ecx # sched: [4:1.00]
-; HASWELL-NEXT:    vcvttsd2si (%rdi), %eax # sched: [9:1.00]
-; HASWELL-NEXT:    addl %ecx, %eax # sched: [1:0.25]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_cvttsd2si:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    cvttsd2si (%rdi), %eax # sched: [9:1.00]
-; BROADWELL-SSE-NEXT:    cvttsd2si %xmm0, %ecx # sched: [4:1.00]
-; BROADWELL-SSE-NEXT:    addl %ecx, %eax # sched: [1:0.25]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_cvttsd2si:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vcvttsd2si (%rdi), %eax # sched: [9:1.00]
-; BROADWELL-NEXT:    vcvttsd2si %xmm0, %ecx # sched: [4:1.00]
-; BROADWELL-NEXT:    addl %ecx, %eax # sched: [1:0.25]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_cvttsd2si:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    cvttsd2si %xmm0, %ecx # sched: [6:1.00]
-; SKYLAKE-SSE-NEXT:    cvttsd2si (%rdi), %eax # sched: [11:1.00]
-; SKYLAKE-SSE-NEXT:    addl %ecx, %eax # sched: [1:0.25]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_cvttsd2si:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vcvttsd2si %xmm0, %ecx # sched: [6:1.00]
-; SKYLAKE-NEXT:    vcvttsd2si (%rdi), %eax # sched: [11:1.00]
-; SKYLAKE-NEXT:    addl %ecx, %eax # sched: [1:0.25]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_cvttsd2si:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    cvttsd2si %xmm0, %ecx # sched: [6:1.00]
-; SKX-SSE-NEXT:    cvttsd2si (%rdi), %eax # sched: [11:1.00]
-; SKX-SSE-NEXT:    addl %ecx, %eax # sched: [1:0.25]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_cvttsd2si:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vcvttsd2si %xmm0, %ecx # sched: [6:1.00]
-; SKX-NEXT:    vcvttsd2si (%rdi), %eax # sched: [11:1.00]
-; SKX-NEXT:    addl %ecx, %eax # sched: [1:0.25]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_cvttsd2si:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    cvttsd2si (%rdi), %eax # sched: [18:1.00]
-; BDVER2-SSE-NEXT:    cvttsd2si %xmm0, %ecx # sched: [13:1.00]
-; BDVER2-SSE-NEXT:    addl %ecx, %eax # sched: [1:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_cvttsd2si:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vcvttsd2si (%rdi), %eax # sched: [18:1.00]
-; BDVER2-NEXT:    vcvttsd2si %xmm0, %ecx # sched: [13:1.00]
-; BDVER2-NEXT:    addl %ecx, %eax # sched: [1:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_cvttsd2si:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    cvttsd2si (%rdi), %eax # sched: [12:1.00]
-; BTVER2-SSE-NEXT:    cvttsd2si %xmm0, %ecx # sched: [7:1.00]
-; BTVER2-SSE-NEXT:    addl %ecx, %eax # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_cvttsd2si:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vcvttsd2si (%rdi), %eax # sched: [12:1.00]
-; BTVER2-NEXT:    vcvttsd2si %xmm0, %ecx # sched: [7:1.00]
-; BTVER2-NEXT:    addl %ecx, %eax # sched: [1:0.50]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_cvttsd2si:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    cvttsd2si (%rdi), %eax # sched: [12:1.00]
-; ZNVER1-SSE-NEXT:    cvttsd2si %xmm0, %ecx # sched: [5:1.00]
-; ZNVER1-SSE-NEXT:    addl %ecx, %eax # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_cvttsd2si:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vcvttsd2si (%rdi), %eax # sched: [12:1.00]
-; ZNVER1-NEXT:    vcvttsd2si %xmm0, %ecx # sched: [5:1.00]
-; ZNVER1-NEXT:    addl %ecx, %eax # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = fptosi double %a0 to i32
-  %2 = load double, double *%a1, align 8
-  %3 = fptosi double %2 to i32
-  %4 = add i32 %1, %3
-  ret i32 %4
-}
-
-define i64 @test_cvttsd2siq(double %a0, double *%a1) {
-; GENERIC-LABEL: test_cvttsd2siq:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    cvttsd2si %xmm0, %rcx # sched: [5:1.00]
-; GENERIC-NEXT:    cvttsd2si (%rdi), %rax # sched: [9:1.00]
-; GENERIC-NEXT:    addq %rcx, %rax # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_cvttsd2siq:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    cvttsd2si (%rdi), %rax # sched: [9:4.50]
-; ATOM-NEXT:    cvttsd2si %xmm0, %rcx # sched: [8:4.00]
-; ATOM-NEXT:    addq %rcx, %rax # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_cvttsd2siq:
-; SLM:       # %bb.0:
-; SLM-NEXT:    cvttsd2si (%rdi), %rax # sched: [7:1.00]
-; SLM-NEXT:    cvttsd2si %xmm0, %rcx # sched: [4:0.50]
-; SLM-NEXT:    addq %rcx, %rax # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_cvttsd2siq:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    cvttsd2si %xmm0, %rcx # sched: [5:1.00]
-; SANDY-SSE-NEXT:    cvttsd2si (%rdi), %rax # sched: [9:1.00]
-; SANDY-SSE-NEXT:    addq %rcx, %rax # sched: [1:0.33]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_cvttsd2siq:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vcvttsd2si %xmm0, %rcx # sched: [5:1.00]
-; SANDY-NEXT:    vcvttsd2si (%rdi), %rax # sched: [10:1.00]
-; SANDY-NEXT:    addq %rcx, %rax # sched: [1:0.33]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_cvttsd2siq:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    cvttsd2si %xmm0, %rcx # sched: [4:1.00]
-; HASWELL-SSE-NEXT:    cvttsd2si (%rdi), %rax # sched: [9:1.00]
-; HASWELL-SSE-NEXT:    addq %rcx, %rax # sched: [1:0.25]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_cvttsd2siq:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vcvttsd2si %xmm0, %rcx # sched: [4:1.00]
-; HASWELL-NEXT:    vcvttsd2si (%rdi), %rax # sched: [9:1.00]
-; HASWELL-NEXT:    addq %rcx, %rax # sched: [1:0.25]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_cvttsd2siq:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    cvttsd2si (%rdi), %rax # sched: [9:1.00]
-; BROADWELL-SSE-NEXT:    cvttsd2si %xmm0, %rcx # sched: [4:1.00]
-; BROADWELL-SSE-NEXT:    addq %rcx, %rax # sched: [1:0.25]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_cvttsd2siq:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vcvttsd2si (%rdi), %rax # sched: [9:1.00]
-; BROADWELL-NEXT:    vcvttsd2si %xmm0, %rcx # sched: [4:1.00]
-; BROADWELL-NEXT:    addq %rcx, %rax # sched: [1:0.25]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_cvttsd2siq:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    cvttsd2si %xmm0, %rcx # sched: [6:1.00]
-; SKYLAKE-SSE-NEXT:    cvttsd2si (%rdi), %rax # sched: [11:1.00]
-; SKYLAKE-SSE-NEXT:    addq %rcx, %rax # sched: [1:0.25]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_cvttsd2siq:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vcvttsd2si %xmm0, %rcx # sched: [6:1.00]
-; SKYLAKE-NEXT:    vcvttsd2si (%rdi), %rax # sched: [11:1.00]
-; SKYLAKE-NEXT:    addq %rcx, %rax # sched: [1:0.25]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_cvttsd2siq:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    cvttsd2si %xmm0, %rcx # sched: [6:1.00]
-; SKX-SSE-NEXT:    cvttsd2si (%rdi), %rax # sched: [11:1.00]
-; SKX-SSE-NEXT:    addq %rcx, %rax # sched: [1:0.25]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_cvttsd2siq:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vcvttsd2si %xmm0, %rcx # sched: [6:1.00]
-; SKX-NEXT:    vcvttsd2si (%rdi), %rax # sched: [11:1.00]
-; SKX-NEXT:    addq %rcx, %rax # sched: [1:0.25]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_cvttsd2siq:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    cvttsd2si (%rdi), %rax # sched: [18:1.00]
-; BDVER2-SSE-NEXT:    cvttsd2si %xmm0, %rcx # sched: [13:1.00]
-; BDVER2-SSE-NEXT:    addq %rcx, %rax # sched: [1:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_cvttsd2siq:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vcvttsd2si (%rdi), %rax # sched: [18:1.00]
-; BDVER2-NEXT:    vcvttsd2si %xmm0, %rcx # sched: [13:1.00]
-; BDVER2-NEXT:    addq %rcx, %rax # sched: [1:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_cvttsd2siq:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    cvttsd2si (%rdi), %rax # sched: [12:1.00]
-; BTVER2-SSE-NEXT:    cvttsd2si %xmm0, %rcx # sched: [7:1.00]
-; BTVER2-SSE-NEXT:    addq %rcx, %rax # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_cvttsd2siq:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vcvttsd2si (%rdi), %rax # sched: [12:1.00]
-; BTVER2-NEXT:    vcvttsd2si %xmm0, %rcx # sched: [7:1.00]
-; BTVER2-NEXT:    addq %rcx, %rax # sched: [1:0.50]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_cvttsd2siq:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    cvttsd2si (%rdi), %rax # sched: [12:1.00]
-; ZNVER1-SSE-NEXT:    cvttsd2si %xmm0, %rcx # sched: [5:1.00]
-; ZNVER1-SSE-NEXT:    addq %rcx, %rax # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_cvttsd2siq:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vcvttsd2si (%rdi), %rax # sched: [12:1.00]
-; ZNVER1-NEXT:    vcvttsd2si %xmm0, %rcx # sched: [5:1.00]
-; ZNVER1-NEXT:    addq %rcx, %rax # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = fptosi double %a0 to i64
-  %2 = load double, double *%a1, align 8
-  %3 = fptosi double %2 to i64
-  %4 = add i64 %1, %3
-  ret i64 %4
-}
-
-define <2 x double> @test_divpd(<2 x double> %a0, <2 x double> %a1, <2 x double> *%a2) {
-; GENERIC-LABEL: test_divpd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    divpd %xmm1, %xmm0 # sched: [22:22.00]
-; GENERIC-NEXT:    divpd (%rdi), %xmm0 # sched: [28:22.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_divpd:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    divpd %xmm1, %xmm0 # sched: [125:62.50]
-; ATOM-NEXT:    divpd (%rdi), %xmm0 # sched: [125:62.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_divpd:
-; SLM:       # %bb.0:
-; SLM-NEXT:    divpd %xmm1, %xmm0 # sched: [69:69.00]
-; SLM-NEXT:    divpd (%rdi), %xmm0 # sched: [72:69.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_divpd:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    divpd %xmm1, %xmm0 # sched: [22:22.00]
-; SANDY-SSE-NEXT:    divpd (%rdi), %xmm0 # sched: [28:22.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_divpd:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vdivpd %xmm1, %xmm0, %xmm0 # sched: [22:22.00]
-; SANDY-NEXT:    vdivpd (%rdi), %xmm0, %xmm0 # sched: [28:22.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_divpd:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    divpd %xmm1, %xmm0 # sched: [20:14.00]
-; HASWELL-SSE-NEXT:    divpd (%rdi), %xmm0 # sched: [26:14.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_divpd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vdivpd %xmm1, %xmm0, %xmm0 # sched: [20:14.00]
-; HASWELL-NEXT:    vdivpd (%rdi), %xmm0, %xmm0 # sched: [26:14.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_divpd:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    divpd %xmm1, %xmm0 # sched: [14:8.00]
-; BROADWELL-SSE-NEXT:    divpd (%rdi), %xmm0 # sched: [19:8.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_divpd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vdivpd %xmm1, %xmm0, %xmm0 # sched: [14:8.00]
-; BROADWELL-NEXT:    vdivpd (%rdi), %xmm0, %xmm0 # sched: [19:8.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_divpd:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    divpd %xmm1, %xmm0 # sched: [14:3.00]
-; SKYLAKE-SSE-NEXT:    divpd (%rdi), %xmm0 # sched: [20:4.00]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_divpd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vdivpd %xmm1, %xmm0, %xmm0 # sched: [14:3.00]
-; SKYLAKE-NEXT:    vdivpd (%rdi), %xmm0, %xmm0 # sched: [20:4.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_divpd:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    divpd %xmm1, %xmm0 # sched: [14:3.00]
-; SKX-SSE-NEXT:    divpd (%rdi), %xmm0 # sched: [20:4.00]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_divpd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vdivpd %xmm1, %xmm0, %xmm0 # sched: [14:3.00]
-; SKX-NEXT:    vdivpd (%rdi), %xmm0, %xmm0 # sched: [20:4.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_divpd:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    divpd %xmm1, %xmm0 # sched: [9:9.50]
-; BDVER2-SSE-NEXT:    divpd (%rdi), %xmm0 # sched: [14:9.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_divpd:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vdivpd %xmm1, %xmm0, %xmm0 # sched: [9:9.50]
-; BDVER2-NEXT:    vdivpd (%rdi), %xmm0, %xmm0 # sched: [14:9.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_divpd:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    divpd %xmm1, %xmm0 # sched: [19:19.00]
-; BTVER2-SSE-NEXT:    divpd (%rdi), %xmm0 # sched: [24:19.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_divpd:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vdivpd %xmm1, %xmm0, %xmm0 # sched: [19:19.00]
-; BTVER2-NEXT:    vdivpd (%rdi), %xmm0, %xmm0 # sched: [24:19.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_divpd:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    divpd %xmm1, %xmm0 # sched: [15:1.00]
-; ZNVER1-SSE-NEXT:    divpd (%rdi), %xmm0 # sched: [22:1.00]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_divpd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vdivpd %xmm1, %xmm0, %xmm0 # sched: [15:1.00]
-; ZNVER1-NEXT:    vdivpd (%rdi), %xmm0, %xmm0 # sched: [22:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = fdiv <2 x double> %a0, %a1
-  %2 = load <2 x double>, <2 x double> *%a2, align 16
-  %3 = fdiv <2 x double> %1, %2
-  ret <2 x double> %3
-}
-
-define double @test_divsd(double %a0, double %a1, double *%a2) {
-; GENERIC-LABEL: test_divsd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    divsd %xmm1, %xmm0 # sched: [22:22.00]
-; GENERIC-NEXT:    divsd (%rdi), %xmm0 # sched: [28:22.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_divsd:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    divsd %xmm1, %xmm0 # sched: [62:31.00]
-; ATOM-NEXT:    divsd (%rdi), %xmm0 # sched: [62:31.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_divsd:
-; SLM:       # %bb.0:
-; SLM-NEXT:    divsd %xmm1, %xmm0 # sched: [34:32.00]
-; SLM-NEXT:    divsd (%rdi), %xmm0 # sched: [37:32.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_divsd:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    divsd %xmm1, %xmm0 # sched: [22:22.00]
-; SANDY-SSE-NEXT:    divsd (%rdi), %xmm0 # sched: [28:22.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_divsd:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vdivsd %xmm1, %xmm0, %xmm0 # sched: [22:22.00]
-; SANDY-NEXT:    vdivsd (%rdi), %xmm0, %xmm0 # sched: [28:22.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_divsd:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    divsd %xmm1, %xmm0 # sched: [20:14.00]
-; HASWELL-SSE-NEXT:    divsd (%rdi), %xmm0 # sched: [25:14.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_divsd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vdivsd %xmm1, %xmm0, %xmm0 # sched: [20:14.00]
-; HASWELL-NEXT:    vdivsd (%rdi), %xmm0, %xmm0 # sched: [25:14.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_divsd:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    divsd %xmm1, %xmm0 # sched: [14:4.00]
-; BROADWELL-SSE-NEXT:    divsd (%rdi), %xmm0 # sched: [19:8.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_divsd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vdivsd %xmm1, %xmm0, %xmm0 # sched: [14:4.00]
-; BROADWELL-NEXT:    vdivsd (%rdi), %xmm0, %xmm0 # sched: [19:8.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_divsd:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    divsd %xmm1, %xmm0 # sched: [14:3.00]
-; SKYLAKE-SSE-NEXT:    divsd (%rdi), %xmm0 # sched: [19:4.00]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_divsd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vdivsd %xmm1, %xmm0, %xmm0 # sched: [14:3.00]
-; SKYLAKE-NEXT:    vdivsd (%rdi), %xmm0, %xmm0 # sched: [19:4.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_divsd:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    divsd %xmm1, %xmm0 # sched: [14:3.00]
-; SKX-SSE-NEXT:    divsd (%rdi), %xmm0 # sched: [19:4.00]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_divsd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vdivsd %xmm1, %xmm0, %xmm0 # sched: [14:3.00]
-; SKX-NEXT:    vdivsd (%rdi), %xmm0, %xmm0 # sched: [19:4.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_divsd:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    divsd %xmm1, %xmm0 # sched: [9:9.50]
-; BDVER2-SSE-NEXT:    divsd (%rdi), %xmm0 # sched: [14:9.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_divsd:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vdivsd %xmm1, %xmm0, %xmm0 # sched: [9:9.50]
-; BDVER2-NEXT:    vdivsd (%rdi), %xmm0, %xmm0 # sched: [14:9.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_divsd:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    divsd %xmm1, %xmm0 # sched: [19:19.00]
-; BTVER2-SSE-NEXT:    divsd (%rdi), %xmm0 # sched: [24:19.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_divsd:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vdivsd %xmm1, %xmm0, %xmm0 # sched: [19:19.00]
-; BTVER2-NEXT:    vdivsd (%rdi), %xmm0, %xmm0 # sched: [24:19.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_divsd:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    divsd %xmm1, %xmm0 # sched: [15:1.00]
-; ZNVER1-SSE-NEXT:    divsd (%rdi), %xmm0 # sched: [22:1.00]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_divsd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vdivsd %xmm1, %xmm0, %xmm0 # sched: [15:1.00]
-; ZNVER1-NEXT:    vdivsd (%rdi), %xmm0, %xmm0 # sched: [22:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = fdiv double %a0, %a1
-  %2 = load double, double *%a2, align 8
-  %3 = fdiv double %1, %2
-  ret double %3
-}
-
-define void @test_lfence() {
-; GENERIC-LABEL: test_lfence:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    lfence # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_lfence:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    lfence # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_lfence:
-; SLM:       # %bb.0:
-; SLM-NEXT:    lfence # sched: [1:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_lfence:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    lfence # sched: [1:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_lfence:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    lfence # sched: [1:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_lfence:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    lfence # sched: [2:0.50]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_lfence:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    lfence # sched: [2:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_lfence:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    lfence # sched: [2:0.50]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_lfence:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    lfence # sched: [2:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_lfence:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    lfence # sched: [2:0.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_lfence:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    lfence # sched: [2:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_lfence:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    lfence # sched: [2:0.50]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_lfence:
-; SKX:       # %bb.0:
-; SKX-NEXT:    lfence # sched: [2:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_lfence:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    lfence # sched: [1:1.00]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_lfence:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    lfence # sched: [1:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_lfence:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    lfence # sched: [1:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_lfence:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    lfence # sched: [1:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_lfence:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    lfence # sched: [1:0.50]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_lfence:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    lfence # sched: [1:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  call void @llvm.x86.sse2.lfence()
-  ret void
-}
-declare void @llvm.x86.sse2.lfence() nounwind readnone
-
-define void @test_mfence() {
-; GENERIC-LABEL: test_mfence:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    mfence # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_mfence:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    mfence # sched: [1:1.00]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_mfence:
-; SLM:       # %bb.0:
-; SLM-NEXT:    mfence # sched: [1:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_mfence:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    mfence # sched: [1:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_mfence:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    mfence # sched: [1:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_mfence:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    mfence # sched: [2:0.50]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_mfence:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    mfence # sched: [2:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_mfence:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    mfence # sched: [2:0.50]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_mfence:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    mfence # sched: [2:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_mfence:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    mfence # sched: [3:0.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_mfence:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    mfence # sched: [3:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_mfence:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    mfence # sched: [3:0.50]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_mfence:
-; SKX:       # %bb.0:
-; SKX-NEXT:    mfence # sched: [3:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_mfence:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    mfence # sched: [1:1.00]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_mfence:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    mfence # sched: [1:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_mfence:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    mfence # sched: [1:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_mfence:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    mfence # sched: [1:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_mfence:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    mfence # sched: [1:0.50]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_mfence:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    mfence # sched: [1:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  call void @llvm.x86.sse2.mfence()
-  ret void
-}
-declare void @llvm.x86.sse2.mfence() nounwind readnone
-
-define void @test_maskmovdqu(<16 x i8> %a0, <16 x i8> %a1, i8* %a2) {
-; GENERIC-LABEL: test_maskmovdqu:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    maskmovdqu %xmm1, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_maskmovdqu:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    maskmovdqu %xmm1, %xmm0 # sched: [2:1.00]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_maskmovdqu:
-; SLM:       # %bb.0:
-; SLM-NEXT:    maskmovdqu %xmm1, %xmm0 # sched: [1:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_maskmovdqu:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    maskmovdqu %xmm1, %xmm0 # sched: [1:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_maskmovdqu:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vmaskmovdqu %xmm1, %xmm0 # sched: [1:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_maskmovdqu:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    maskmovdqu %xmm1, %xmm0 # sched: [1:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_maskmovdqu:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vmaskmovdqu %xmm1, %xmm0 # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_maskmovdqu:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    maskmovdqu %xmm1, %xmm0 # sched: [1:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_maskmovdqu:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vmaskmovdqu %xmm1, %xmm0 # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_maskmovdqu:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    maskmovdqu %xmm1, %xmm0 # sched: [1:1.00]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_maskmovdqu:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vmaskmovdqu %xmm1, %xmm0 # sched: [1:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_maskmovdqu:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    maskmovdqu %xmm1, %xmm0 # sched: [1:1.00]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_maskmovdqu:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmaskmovdqu %xmm1, %xmm0 # sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_maskmovdqu:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    maskmovdqu %xmm1, %xmm0 # sched: [1:1.00]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_maskmovdqu:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vmaskmovdqu %xmm1, %xmm0 # sched: [1:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_maskmovdqu:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    maskmovdqu %xmm1, %xmm0 # sched: [1:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_maskmovdqu:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vmaskmovdqu %xmm1, %xmm0 # sched: [1:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_maskmovdqu:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    maskmovdqu %xmm1, %xmm0 # sched: [100:0.25]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_maskmovdqu:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vmaskmovdqu %xmm1, %xmm0 # sched: [100:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  call void @llvm.x86.sse2.maskmov.dqu(<16 x i8> %a0, <16 x i8> %a1, i8* %a2)
-  ret void
-}
-declare void @llvm.x86.sse2.maskmov.dqu(<16 x i8>, <16 x i8>, i8*) nounwind
-
-define <2 x double> @test_maxpd(<2 x double> %a0, <2 x double> %a1, <2 x double> *%a2) {
-; GENERIC-LABEL: test_maxpd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    maxpd %xmm1, %xmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    maxpd (%rdi), %xmm0 # sched: [9:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_maxpd:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    maxpd %xmm1, %xmm0 # sched: [6:3.00]
-; ATOM-NEXT:    maxpd (%rdi), %xmm0 # sched: [7:3.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_maxpd:
-; SLM:       # %bb.0:
-; SLM-NEXT:    maxpd %xmm1, %xmm0 # sched: [3:1.00]
-; SLM-NEXT:    maxpd (%rdi), %xmm0 # sched: [6:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_maxpd:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    maxpd %xmm1, %xmm0 # sched: [3:1.00]
-; SANDY-SSE-NEXT:    maxpd (%rdi), %xmm0 # sched: [9:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_maxpd:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vmaxpd %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; SANDY-NEXT:    vmaxpd (%rdi), %xmm0, %xmm0 # sched: [9:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_maxpd:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    maxpd %xmm1, %xmm0 # sched: [3:1.00]
-; HASWELL-SSE-NEXT:    maxpd (%rdi), %xmm0 # sched: [9:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_maxpd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vmaxpd %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; HASWELL-NEXT:    vmaxpd (%rdi), %xmm0, %xmm0 # sched: [9:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_maxpd:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    maxpd %xmm1, %xmm0 # sched: [3:1.00]
-; BROADWELL-SSE-NEXT:    maxpd (%rdi), %xmm0 # sched: [8:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_maxpd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vmaxpd %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    vmaxpd (%rdi), %xmm0, %xmm0 # sched: [8:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_maxpd:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    maxpd %xmm1, %xmm0 # sched: [4:0.50]
-; SKYLAKE-SSE-NEXT:    maxpd (%rdi), %xmm0 # sched: [10:0.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_maxpd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vmaxpd %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    vmaxpd (%rdi), %xmm0, %xmm0 # sched: [10:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_maxpd:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    maxpd %xmm1, %xmm0 # sched: [4:0.50]
-; SKX-SSE-NEXT:    maxpd (%rdi), %xmm0 # sched: [10:0.50]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_maxpd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmaxpd %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKX-NEXT:    vmaxpd (%rdi), %xmm0, %xmm0 # sched: [10:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_maxpd:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    maxpd %xmm1, %xmm0 # sched: [2:1.00]
-; BDVER2-SSE-NEXT:    maxpd (%rdi), %xmm0 # sched: [7:1.00]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_maxpd:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vmaxpd %xmm1, %xmm0, %xmm0 # sched: [2:1.00]
-; BDVER2-NEXT:    vmaxpd (%rdi), %xmm0, %xmm0 # sched: [7:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_maxpd:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    maxpd %xmm1, %xmm0 # sched: [2:1.00]
-; BTVER2-SSE-NEXT:    maxpd (%rdi), %xmm0 # sched: [7:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_maxpd:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vmaxpd %xmm1, %xmm0, %xmm0 # sched: [2:1.00]
-; BTVER2-NEXT:    vmaxpd (%rdi), %xmm0, %xmm0 # sched: [7:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_maxpd:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    maxpd %xmm1, %xmm0 # sched: [3:1.00]
-; ZNVER1-SSE-NEXT:    maxpd (%rdi), %xmm0 # sched: [10:1.00]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_maxpd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vmaxpd %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    vmaxpd (%rdi), %xmm0, %xmm0 # sched: [10:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <2 x double> @llvm.x86.sse2.max.pd(<2 x double> %a0, <2 x double> %a1)
-  %2 = load <2 x double>, <2 x double> *%a2, align 16
-  %3 = call <2 x double> @llvm.x86.sse2.max.pd(<2 x double> %1, <2 x double> %2)
-  ret <2 x double> %3
-}
-declare <2 x double> @llvm.x86.sse2.max.pd(<2 x double>, <2 x double>) nounwind readnone
-
-define <2 x double> @test_maxsd(<2 x double> %a0, <2 x double> %a1, <2 x double> *%a2) {
-; GENERIC-LABEL: test_maxsd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    maxsd %xmm1, %xmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    maxsd (%rdi), %xmm0 # sched: [9:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_maxsd:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    maxsd %xmm1, %xmm0 # sched: [5:5.00]
-; ATOM-NEXT:    maxsd (%rdi), %xmm0 # sched: [5:5.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_maxsd:
-; SLM:       # %bb.0:
-; SLM-NEXT:    maxsd %xmm1, %xmm0 # sched: [3:1.00]
-; SLM-NEXT:    maxsd (%rdi), %xmm0 # sched: [6:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_maxsd:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    maxsd %xmm1, %xmm0 # sched: [3:1.00]
-; SANDY-SSE-NEXT:    maxsd (%rdi), %xmm0 # sched: [9:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_maxsd:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vmaxsd %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; SANDY-NEXT:    vmaxsd (%rdi), %xmm0, %xmm0 # sched: [9:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_maxsd:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    maxsd %xmm1, %xmm0 # sched: [3:1.00]
-; HASWELL-SSE-NEXT:    maxsd (%rdi), %xmm0 # sched: [8:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_maxsd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vmaxsd %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; HASWELL-NEXT:    vmaxsd (%rdi), %xmm0, %xmm0 # sched: [8:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_maxsd:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    maxsd %xmm1, %xmm0 # sched: [3:1.00]
-; BROADWELL-SSE-NEXT:    maxsd (%rdi), %xmm0 # sched: [8:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_maxsd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vmaxsd %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    vmaxsd (%rdi), %xmm0, %xmm0 # sched: [8:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_maxsd:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    maxsd %xmm1, %xmm0 # sched: [4:0.50]
-; SKYLAKE-SSE-NEXT:    maxsd (%rdi), %xmm0 # sched: [9:0.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_maxsd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vmaxsd %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    vmaxsd (%rdi), %xmm0, %xmm0 # sched: [9:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_maxsd:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    maxsd %xmm1, %xmm0 # sched: [4:0.50]
-; SKX-SSE-NEXT:    maxsd (%rdi), %xmm0 # sched: [9:0.50]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_maxsd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmaxsd %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKX-NEXT:    vmaxsd (%rdi), %xmm0, %xmm0 # sched: [9:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_maxsd:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    maxsd %xmm1, %xmm0 # sched: [2:1.00]
-; BDVER2-SSE-NEXT:    maxsd (%rdi), %xmm0 # sched: [7:1.00]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_maxsd:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vmaxsd %xmm1, %xmm0, %xmm0 # sched: [2:1.00]
-; BDVER2-NEXT:    vmaxsd (%rdi), %xmm0, %xmm0 # sched: [7:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_maxsd:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    maxsd %xmm1, %xmm0 # sched: [2:1.00]
-; BTVER2-SSE-NEXT:    maxsd (%rdi), %xmm0 # sched: [7:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_maxsd:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vmaxsd %xmm1, %xmm0, %xmm0 # sched: [2:1.00]
-; BTVER2-NEXT:    vmaxsd (%rdi), %xmm0, %xmm0 # sched: [7:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_maxsd:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    maxsd %xmm1, %xmm0 # sched: [3:1.00]
-; ZNVER1-SSE-NEXT:    maxsd (%rdi), %xmm0 # sched: [10:1.00]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_maxsd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vmaxsd %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    vmaxsd (%rdi), %xmm0, %xmm0 # sched: [10:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <2 x double> @llvm.x86.sse2.max.sd(<2 x double> %a0, <2 x double> %a1)
-  %2 = load <2 x double>, <2 x double> *%a2, align 16
-  %3 = call <2 x double> @llvm.x86.sse2.max.sd(<2 x double> %1, <2 x double> %2)
-  ret <2 x double> %3
-}
-declare <2 x double> @llvm.x86.sse2.max.sd(<2 x double>, <2 x double>) nounwind readnone
-
-define <2 x double> @test_minpd(<2 x double> %a0, <2 x double> %a1, <2 x double> *%a2) {
-; GENERIC-LABEL: test_minpd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    minpd %xmm1, %xmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    minpd (%rdi), %xmm0 # sched: [9:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_minpd:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    minpd %xmm1, %xmm0 # sched: [6:3.00]
-; ATOM-NEXT:    minpd (%rdi), %xmm0 # sched: [7:3.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_minpd:
-; SLM:       # %bb.0:
-; SLM-NEXT:    minpd %xmm1, %xmm0 # sched: [3:1.00]
-; SLM-NEXT:    minpd (%rdi), %xmm0 # sched: [6:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_minpd:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    minpd %xmm1, %xmm0 # sched: [3:1.00]
-; SANDY-SSE-NEXT:    minpd (%rdi), %xmm0 # sched: [9:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_minpd:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vminpd %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; SANDY-NEXT:    vminpd (%rdi), %xmm0, %xmm0 # sched: [9:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_minpd:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    minpd %xmm1, %xmm0 # sched: [3:1.00]
-; HASWELL-SSE-NEXT:    minpd (%rdi), %xmm0 # sched: [9:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_minpd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vminpd %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; HASWELL-NEXT:    vminpd (%rdi), %xmm0, %xmm0 # sched: [9:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_minpd:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    minpd %xmm1, %xmm0 # sched: [3:1.00]
-; BROADWELL-SSE-NEXT:    minpd (%rdi), %xmm0 # sched: [8:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_minpd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vminpd %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    vminpd (%rdi), %xmm0, %xmm0 # sched: [8:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_minpd:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    minpd %xmm1, %xmm0 # sched: [4:0.50]
-; SKYLAKE-SSE-NEXT:    minpd (%rdi), %xmm0 # sched: [10:0.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_minpd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vminpd %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    vminpd (%rdi), %xmm0, %xmm0 # sched: [10:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_minpd:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    minpd %xmm1, %xmm0 # sched: [4:0.50]
-; SKX-SSE-NEXT:    minpd (%rdi), %xmm0 # sched: [10:0.50]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_minpd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vminpd %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKX-NEXT:    vminpd (%rdi), %xmm0, %xmm0 # sched: [10:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_minpd:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    minpd %xmm1, %xmm0 # sched: [2:1.00]
-; BDVER2-SSE-NEXT:    minpd (%rdi), %xmm0 # sched: [7:1.00]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_minpd:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vminpd %xmm1, %xmm0, %xmm0 # sched: [2:1.00]
-; BDVER2-NEXT:    vminpd (%rdi), %xmm0, %xmm0 # sched: [7:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_minpd:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    minpd %xmm1, %xmm0 # sched: [2:1.00]
-; BTVER2-SSE-NEXT:    minpd (%rdi), %xmm0 # sched: [7:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_minpd:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vminpd %xmm1, %xmm0, %xmm0 # sched: [2:1.00]
-; BTVER2-NEXT:    vminpd (%rdi), %xmm0, %xmm0 # sched: [7:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_minpd:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    minpd %xmm1, %xmm0 # sched: [3:1.00]
-; ZNVER1-SSE-NEXT:    minpd (%rdi), %xmm0 # sched: [10:1.00]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_minpd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vminpd %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    vminpd (%rdi), %xmm0, %xmm0 # sched: [10:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <2 x double> @llvm.x86.sse2.min.pd(<2 x double> %a0, <2 x double> %a1)
-  %2 = load <2 x double>, <2 x double> *%a2, align 16
-  %3 = call <2 x double> @llvm.x86.sse2.min.pd(<2 x double> %1, <2 x double> %2)
-  ret <2 x double> %3
-}
-declare <2 x double> @llvm.x86.sse2.min.pd(<2 x double>, <2 x double>) nounwind readnone
-
-define <2 x double> @test_minsd(<2 x double> %a0, <2 x double> %a1, <2 x double> *%a2) {
-; GENERIC-LABEL: test_minsd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    minsd %xmm1, %xmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    minsd (%rdi), %xmm0 # sched: [9:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_minsd:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    minsd %xmm1, %xmm0 # sched: [5:5.00]
-; ATOM-NEXT:    minsd (%rdi), %xmm0 # sched: [5:5.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_minsd:
-; SLM:       # %bb.0:
-; SLM-NEXT:    minsd %xmm1, %xmm0 # sched: [3:1.00]
-; SLM-NEXT:    minsd (%rdi), %xmm0 # sched: [6:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_minsd:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    minsd %xmm1, %xmm0 # sched: [3:1.00]
-; SANDY-SSE-NEXT:    minsd (%rdi), %xmm0 # sched: [9:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_minsd:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vminsd %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; SANDY-NEXT:    vminsd (%rdi), %xmm0, %xmm0 # sched: [9:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_minsd:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    minsd %xmm1, %xmm0 # sched: [3:1.00]
-; HASWELL-SSE-NEXT:    minsd (%rdi), %xmm0 # sched: [8:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_minsd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vminsd %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; HASWELL-NEXT:    vminsd (%rdi), %xmm0, %xmm0 # sched: [8:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_minsd:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    minsd %xmm1, %xmm0 # sched: [3:1.00]
-; BROADWELL-SSE-NEXT:    minsd (%rdi), %xmm0 # sched: [8:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_minsd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vminsd %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    vminsd (%rdi), %xmm0, %xmm0 # sched: [8:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_minsd:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    minsd %xmm1, %xmm0 # sched: [4:0.50]
-; SKYLAKE-SSE-NEXT:    minsd (%rdi), %xmm0 # sched: [9:0.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_minsd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vminsd %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    vminsd (%rdi), %xmm0, %xmm0 # sched: [9:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_minsd:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    minsd %xmm1, %xmm0 # sched: [4:0.50]
-; SKX-SSE-NEXT:    minsd (%rdi), %xmm0 # sched: [9:0.50]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_minsd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vminsd %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKX-NEXT:    vminsd (%rdi), %xmm0, %xmm0 # sched: [9:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_minsd:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    minsd %xmm1, %xmm0 # sched: [2:1.00]
-; BDVER2-SSE-NEXT:    minsd (%rdi), %xmm0 # sched: [7:1.00]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_minsd:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vminsd %xmm1, %xmm0, %xmm0 # sched: [2:1.00]
-; BDVER2-NEXT:    vminsd (%rdi), %xmm0, %xmm0 # sched: [7:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_minsd:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    minsd %xmm1, %xmm0 # sched: [2:1.00]
-; BTVER2-SSE-NEXT:    minsd (%rdi), %xmm0 # sched: [7:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_minsd:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vminsd %xmm1, %xmm0, %xmm0 # sched: [2:1.00]
-; BTVER2-NEXT:    vminsd (%rdi), %xmm0, %xmm0 # sched: [7:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_minsd:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    minsd %xmm1, %xmm0 # sched: [3:1.00]
-; ZNVER1-SSE-NEXT:    minsd (%rdi), %xmm0 # sched: [10:1.00]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_minsd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vminsd %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    vminsd (%rdi), %xmm0, %xmm0 # sched: [10:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <2 x double> @llvm.x86.sse2.min.sd(<2 x double> %a0, <2 x double> %a1)
-  %2 = load <2 x double>, <2 x double> *%a2, align 16
-  %3 = call <2 x double> @llvm.x86.sse2.min.sd(<2 x double> %1, <2 x double> %2)
-  ret <2 x double> %3
-}
-declare <2 x double> @llvm.x86.sse2.min.sd(<2 x double>, <2 x double>) nounwind readnone
-
-define void @test_movapd(<2 x double> *%a0, <2 x double> *%a1) {
-; GENERIC-LABEL: test_movapd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    movapd (%rdi), %xmm0 # sched: [6:0.50]
-; GENERIC-NEXT:    addpd %xmm0, %xmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    movapd %xmm0, (%rsi) # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_movapd:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    movapd (%rdi), %xmm0 # sched: [1:1.00]
-; ATOM-NEXT:    addpd %xmm0, %xmm0 # sched: [6:3.00]
-; ATOM-NEXT:    movapd %xmm0, (%rsi) # sched: [1:1.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_movapd:
-; SLM:       # %bb.0:
-; SLM-NEXT:    movapd (%rdi), %xmm0 # sched: [3:1.00]
-; SLM-NEXT:    addpd %xmm0, %xmm0 # sched: [3:1.00]
-; SLM-NEXT:    movapd %xmm0, (%rsi) # sched: [1:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_movapd:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    movapd (%rdi), %xmm0 # sched: [6:0.50]
-; SANDY-SSE-NEXT:    addpd %xmm0, %xmm0 # sched: [3:1.00]
-; SANDY-SSE-NEXT:    movapd %xmm0, (%rsi) # sched: [1:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_movapd:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vmovapd (%rdi), %xmm0 # sched: [6:0.50]
-; SANDY-NEXT:    vaddpd %xmm0, %xmm0, %xmm0 # sched: [3:1.00]
-; SANDY-NEXT:    vmovapd %xmm0, (%rsi) # sched: [1:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_movapd:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    movapd (%rdi), %xmm0 # sched: [6:0.50]
-; HASWELL-SSE-NEXT:    addpd %xmm0, %xmm0 # sched: [3:1.00]
-; HASWELL-SSE-NEXT:    movapd %xmm0, (%rsi) # sched: [1:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_movapd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vmovapd (%rdi), %xmm0 # sched: [6:0.50]
-; HASWELL-NEXT:    vaddpd %xmm0, %xmm0, %xmm0 # sched: [3:1.00]
-; HASWELL-NEXT:    vmovapd %xmm0, (%rsi) # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_movapd:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    movapd (%rdi), %xmm0 # sched: [5:0.50]
-; BROADWELL-SSE-NEXT:    addpd %xmm0, %xmm0 # sched: [3:1.00]
-; BROADWELL-SSE-NEXT:    movapd %xmm0, (%rsi) # sched: [1:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_movapd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vmovapd (%rdi), %xmm0 # sched: [5:0.50]
-; BROADWELL-NEXT:    vaddpd %xmm0, %xmm0, %xmm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    vmovapd %xmm0, (%rsi) # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_movapd:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    movapd (%rdi), %xmm0 # sched: [6:0.50]
-; SKYLAKE-SSE-NEXT:    addpd %xmm0, %xmm0 # sched: [4:0.50]
-; SKYLAKE-SSE-NEXT:    movapd %xmm0, (%rsi) # sched: [1:1.00]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_movapd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vmovapd (%rdi), %xmm0 # sched: [6:0.50]
-; SKYLAKE-NEXT:    vaddpd %xmm0, %xmm0, %xmm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    vmovapd %xmm0, (%rsi) # sched: [1:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_movapd:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    movapd (%rdi), %xmm0 # sched: [6:0.50]
-; SKX-SSE-NEXT:    addpd %xmm0, %xmm0 # sched: [4:0.50]
-; SKX-SSE-NEXT:    movapd %xmm0, (%rsi) # sched: [1:1.00]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_movapd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovapd (%rdi), %xmm0 # sched: [6:0.50]
-; SKX-NEXT:    vaddpd %xmm0, %xmm0, %xmm0 # sched: [4:0.50]
-; SKX-NEXT:    vmovapd %xmm0, (%rsi) # sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_movapd:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    movapd (%rdi), %xmm0 # sched: [5:0.50]
-; BDVER2-SSE-NEXT:    addpd %xmm0, %xmm0 # sched: [5:1.00]
-; BDVER2-SSE-NEXT:    movapd %xmm0, (%rsi) # sched: [1:1.00]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_movapd:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vmovapd (%rdi), %xmm0 # sched: [5:0.50]
-; BDVER2-NEXT:    vaddpd %xmm0, %xmm0, %xmm0 # sched: [5:1.00]
-; BDVER2-NEXT:    vmovapd %xmm0, (%rsi) # sched: [1:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_movapd:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    movapd (%rdi), %xmm0 # sched: [5:1.00]
-; BTVER2-SSE-NEXT:    addpd %xmm0, %xmm0 # sched: [3:1.00]
-; BTVER2-SSE-NEXT:    movapd %xmm0, (%rsi) # sched: [1:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_movapd:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vmovapd (%rdi), %xmm0 # sched: [5:1.00]
-; BTVER2-NEXT:    vaddpd %xmm0, %xmm0, %xmm0 # sched: [3:1.00]
-; BTVER2-NEXT:    vmovapd %xmm0, (%rsi) # sched: [1:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_movapd:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    movapd (%rdi), %xmm0 # sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    addpd %xmm0, %xmm0 # sched: [3:1.00]
-; ZNVER1-SSE-NEXT:    movapd %xmm0, (%rsi) # sched: [1:0.50]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_movapd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vmovapd (%rdi), %xmm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    vaddpd %xmm0, %xmm0, %xmm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    vmovapd %xmm0, (%rsi) # sched: [1:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = load <2 x double>, <2 x double> *%a0, align 16
-  %2 = fadd <2 x double> %1, %1
-  store <2 x double> %2, <2 x double> *%a1, align 16
-  ret void
-}
-
-define void @test_movdqa(<2 x i64> *%a0, <2 x i64> *%a1) {
-; GENERIC-LABEL: test_movdqa:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    movdqa (%rdi), %xmm0 # sched: [6:0.50]
-; GENERIC-NEXT:    paddq %xmm0, %xmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    movdqa %xmm0, (%rsi) # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_movdqa:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    movdqa (%rdi), %xmm0 # sched: [1:1.00]
-; ATOM-NEXT:    paddq %xmm0, %xmm0 # sched: [2:1.00]
-; ATOM-NEXT:    movdqa %xmm0, (%rsi) # sched: [1:1.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_movdqa:
-; SLM:       # %bb.0:
-; SLM-NEXT:    movdqa (%rdi), %xmm0 # sched: [3:1.00]
-; SLM-NEXT:    paddq %xmm0, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    movdqa %xmm0, (%rsi) # sched: [1:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_movdqa:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    movdqa (%rdi), %xmm0 # sched: [6:0.50]
-; SANDY-SSE-NEXT:    paddq %xmm0, %xmm0 # sched: [1:0.50]
-; SANDY-SSE-NEXT:    movdqa %xmm0, (%rsi) # sched: [1:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_movdqa:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vmovdqa (%rdi), %xmm0 # sched: [6:0.50]
-; SANDY-NEXT:    vpaddq %xmm0, %xmm0, %xmm0 # sched: [1:0.50]
-; SANDY-NEXT:    vmovdqa %xmm0, (%rsi) # sched: [1:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_movdqa:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    movdqa (%rdi), %xmm0 # sched: [6:0.50]
-; HASWELL-SSE-NEXT:    paddq %xmm0, %xmm0 # sched: [1:0.50]
-; HASWELL-SSE-NEXT:    movdqa %xmm0, (%rsi) # sched: [1:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_movdqa:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vmovdqa (%rdi), %xmm0 # sched: [6:0.50]
-; HASWELL-NEXT:    vpaddq %xmm0, %xmm0, %xmm0 # sched: [1:0.50]
-; HASWELL-NEXT:    vmovdqa %xmm0, (%rsi) # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_movdqa:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    movdqa (%rdi), %xmm0 # sched: [5:0.50]
-; BROADWELL-SSE-NEXT:    paddq %xmm0, %xmm0 # sched: [1:0.50]
-; BROADWELL-SSE-NEXT:    movdqa %xmm0, (%rsi) # sched: [1:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_movdqa:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vmovdqa (%rdi), %xmm0 # sched: [5:0.50]
-; BROADWELL-NEXT:    vpaddq %xmm0, %xmm0, %xmm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    vmovdqa %xmm0, (%rsi) # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_movdqa:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    movdqa (%rdi), %xmm0 # sched: [6:0.50]
-; SKYLAKE-SSE-NEXT:    paddq %xmm0, %xmm0 # sched: [1:0.33]
-; SKYLAKE-SSE-NEXT:    movdqa %xmm0, (%rsi) # sched: [1:1.00]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_movdqa:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vmovdqa (%rdi), %xmm0 # sched: [6:0.50]
-; SKYLAKE-NEXT:    vpaddq %xmm0, %xmm0, %xmm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    vmovdqa %xmm0, (%rsi) # sched: [1:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_movdqa:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    movdqa (%rdi), %xmm0 # sched: [6:0.50]
-; SKX-SSE-NEXT:    paddq %xmm0, %xmm0 # sched: [1:0.33]
-; SKX-SSE-NEXT:    movdqa %xmm0, (%rsi) # sched: [1:1.00]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_movdqa:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovdqa (%rdi), %xmm0 # sched: [6:0.50]
-; SKX-NEXT:    vpaddq %xmm0, %xmm0, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    vmovdqa %xmm0, (%rsi) # sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_movdqa:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    movdqa (%rdi), %xmm0 # sched: [5:0.50]
-; BDVER2-SSE-NEXT:    paddq %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    movdqa %xmm0, (%rsi) # sched: [1:1.00]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_movdqa:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vmovdqa (%rdi), %xmm0 # sched: [5:0.50]
-; BDVER2-NEXT:    vpaddq %xmm0, %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    vmovdqa %xmm0, (%rsi) # sched: [1:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_movdqa:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    movdqa (%rdi), %xmm0 # sched: [5:1.00]
-; BTVER2-SSE-NEXT:    paddq %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    movdqa %xmm0, (%rsi) # sched: [1:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_movdqa:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vmovdqa (%rdi), %xmm0 # sched: [5:1.00]
-; BTVER2-NEXT:    vpaddq %xmm0, %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    vmovdqa %xmm0, (%rsi) # sched: [1:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_movdqa:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    movdqa (%rdi), %xmm0 # sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    paddq %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    movdqa %xmm0, (%rsi) # sched: [1:0.50]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_movdqa:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vmovdqa (%rdi), %xmm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    vpaddq %xmm0, %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vmovdqa %xmm0, (%rsi) # sched: [1:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = load <2 x i64>, <2 x i64> *%a0, align 16
-  %2 = add <2 x i64> %1, %1
-  store <2 x i64> %2, <2 x i64> *%a1, align 16
-  ret void
-}
-
-define void @test_movdqu(<2 x i64> *%a0, <2 x i64> *%a1) {
-; GENERIC-LABEL: test_movdqu:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    movdqu (%rdi), %xmm0 # sched: [6:0.50]
-; GENERIC-NEXT:    paddq %xmm0, %xmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    movdqu %xmm0, (%rsi) # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_movdqu:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    movdqu (%rdi), %xmm0 # sched: [3:1.50]
-; ATOM-NEXT:    paddq %xmm0, %xmm0 # sched: [2:1.00]
-; ATOM-NEXT:    movdqu %xmm0, (%rsi) # sched: [2:1.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_movdqu:
-; SLM:       # %bb.0:
-; SLM-NEXT:    movdqu (%rdi), %xmm0 # sched: [3:1.00]
-; SLM-NEXT:    paddq %xmm0, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    movdqu %xmm0, (%rsi) # sched: [1:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_movdqu:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    movdqu (%rdi), %xmm0 # sched: [6:0.50]
-; SANDY-SSE-NEXT:    paddq %xmm0, %xmm0 # sched: [1:0.50]
-; SANDY-SSE-NEXT:    movdqu %xmm0, (%rsi) # sched: [1:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_movdqu:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vmovdqu (%rdi), %xmm0 # sched: [6:0.50]
-; SANDY-NEXT:    vpaddq %xmm0, %xmm0, %xmm0 # sched: [1:0.50]
-; SANDY-NEXT:    vmovdqu %xmm0, (%rsi) # sched: [1:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_movdqu:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    movdqu (%rdi), %xmm0 # sched: [6:0.50]
-; HASWELL-SSE-NEXT:    paddq %xmm0, %xmm0 # sched: [1:0.50]
-; HASWELL-SSE-NEXT:    movdqu %xmm0, (%rsi) # sched: [1:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_movdqu:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vmovdqu (%rdi), %xmm0 # sched: [6:0.50]
-; HASWELL-NEXT:    vpaddq %xmm0, %xmm0, %xmm0 # sched: [1:0.50]
-; HASWELL-NEXT:    vmovdqu %xmm0, (%rsi) # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_movdqu:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    movdqu (%rdi), %xmm0 # sched: [5:0.50]
-; BROADWELL-SSE-NEXT:    paddq %xmm0, %xmm0 # sched: [1:0.50]
-; BROADWELL-SSE-NEXT:    movdqu %xmm0, (%rsi) # sched: [1:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_movdqu:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vmovdqu (%rdi), %xmm0 # sched: [5:0.50]
-; BROADWELL-NEXT:    vpaddq %xmm0, %xmm0, %xmm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    vmovdqu %xmm0, (%rsi) # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_movdqu:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    movdqu (%rdi), %xmm0 # sched: [6:0.50]
-; SKYLAKE-SSE-NEXT:    paddq %xmm0, %xmm0 # sched: [1:0.33]
-; SKYLAKE-SSE-NEXT:    movdqu %xmm0, (%rsi) # sched: [1:1.00]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_movdqu:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vmovdqu (%rdi), %xmm0 # sched: [6:0.50]
-; SKYLAKE-NEXT:    vpaddq %xmm0, %xmm0, %xmm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    vmovdqu %xmm0, (%rsi) # sched: [1:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_movdqu:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    movdqu (%rdi), %xmm0 # sched: [6:0.50]
-; SKX-SSE-NEXT:    paddq %xmm0, %xmm0 # sched: [1:0.33]
-; SKX-SSE-NEXT:    movdqu %xmm0, (%rsi) # sched: [1:1.00]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_movdqu:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovdqu (%rdi), %xmm0 # sched: [6:0.50]
-; SKX-NEXT:    vpaddq %xmm0, %xmm0, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    vmovdqu %xmm0, (%rsi) # sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_movdqu:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    movdqu (%rdi), %xmm0 # sched: [5:0.50]
-; BDVER2-SSE-NEXT:    paddq %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    movdqu %xmm0, (%rsi) # sched: [1:1.00]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_movdqu:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vmovdqu (%rdi), %xmm0 # sched: [5:0.50]
-; BDVER2-NEXT:    vpaddq %xmm0, %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    vmovdqu %xmm0, (%rsi) # sched: [1:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_movdqu:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    movdqu (%rdi), %xmm0 # sched: [5:1.00]
-; BTVER2-SSE-NEXT:    paddq %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    movdqu %xmm0, (%rsi) # sched: [1:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_movdqu:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vmovdqu (%rdi), %xmm0 # sched: [5:1.00]
-; BTVER2-NEXT:    vpaddq %xmm0, %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    vmovdqu %xmm0, (%rsi) # sched: [1:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_movdqu:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    movdqu (%rdi), %xmm0 # sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    paddq %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    movdqu %xmm0, (%rsi) # sched: [1:0.50]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_movdqu:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vmovdqu (%rdi), %xmm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    vpaddq %xmm0, %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vmovdqu %xmm0, (%rsi) # sched: [1:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = load <2 x i64>, <2 x i64> *%a0, align 1
-  %2 = add <2 x i64> %1, %1
-  store <2 x i64> %2, <2 x i64> *%a1, align 1
-  ret void
-}
-
-define i32 @test_movd(<4 x i32> %a0, i32 %a1, i32 *%a2) {
-; GENERIC-LABEL: test_movd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    movd %edi, %xmm1 # sched: [1:1.00]
-; GENERIC-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero sched: [6:0.50]
-; GENERIC-NEXT:    paddd %xmm0, %xmm1 # sched: [1:0.50]
-; GENERIC-NEXT:    paddd %xmm0, %xmm2 # sched: [1:0.50]
-; GENERIC-NEXT:    movd %xmm2, %eax # sched: [2:1.00]
-; GENERIC-NEXT:    movd %xmm1, (%rsi) # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_movd:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero sched: [1:1.00]
-; ATOM-NEXT:    paddd %xmm0, %xmm1 # sched: [1:0.50]
-; ATOM-NEXT:    movd %xmm1, %eax # sched: [3:3.00]
-; ATOM-NEXT:    movd %edi, %xmm1 # sched: [1:1.00]
-; ATOM-NEXT:    paddd %xmm0, %xmm1 # sched: [1:0.50]
-; ATOM-NEXT:    movd %xmm1, (%rsi) # sched: [1:1.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_movd:
-; SLM:       # %bb.0:
-; SLM-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero sched: [3:1.00]
-; SLM-NEXT:    movd %edi, %xmm1 # sched: [1:0.50]
-; SLM-NEXT:    paddd %xmm0, %xmm1 # sched: [1:0.50]
-; SLM-NEXT:    movd %xmm1, (%rsi) # sched: [1:1.00]
-; SLM-NEXT:    paddd %xmm0, %xmm2 # sched: [1:0.50]
-; SLM-NEXT:    movd %xmm2, %eax # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_movd:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    movd %edi, %xmm1 # sched: [1:1.00]
-; SANDY-SSE-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero sched: [6:0.50]
-; SANDY-SSE-NEXT:    paddd %xmm0, %xmm1 # sched: [1:0.50]
-; SANDY-SSE-NEXT:    paddd %xmm0, %xmm2 # sched: [1:0.50]
-; SANDY-SSE-NEXT:    movd %xmm2, %eax # sched: [2:1.00]
-; SANDY-SSE-NEXT:    movd %xmm1, (%rsi) # sched: [1:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_movd:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vmovd %edi, %xmm1 # sched: [1:1.00]
-; SANDY-NEXT:    vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero sched: [6:0.50]
-; SANDY-NEXT:    vpaddd %xmm1, %xmm0, %xmm1 # sched: [1:0.50]
-; SANDY-NEXT:    vpaddd %xmm2, %xmm0, %xmm0 # sched: [1:0.50]
-; SANDY-NEXT:    vmovd %xmm0, %eax # sched: [2:1.00]
-; SANDY-NEXT:    vmovd %xmm1, (%rsi) # sched: [1:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_movd:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    movd %edi, %xmm1 # sched: [1:1.00]
-; HASWELL-SSE-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero sched: [5:0.50]
-; HASWELL-SSE-NEXT:    paddd %xmm0, %xmm1 # sched: [1:0.50]
-; HASWELL-SSE-NEXT:    paddd %xmm0, %xmm2 # sched: [1:0.50]
-; HASWELL-SSE-NEXT:    movd %xmm2, %eax # sched: [1:1.00]
-; HASWELL-SSE-NEXT:    movd %xmm1, (%rsi) # sched: [1:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_movd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vmovd %edi, %xmm1 # sched: [1:1.00]
-; HASWELL-NEXT:    vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero sched: [5:0.50]
-; HASWELL-NEXT:    vpaddd %xmm1, %xmm0, %xmm1 # sched: [1:0.50]
-; HASWELL-NEXT:    vpaddd %xmm2, %xmm0, %xmm0 # sched: [1:0.50]
-; HASWELL-NEXT:    vmovd %xmm0, %eax # sched: [1:1.00]
-; HASWELL-NEXT:    vmovd %xmm1, (%rsi) # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_movd:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    movd %edi, %xmm1 # sched: [1:1.00]
-; BROADWELL-SSE-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero sched: [5:0.50]
-; BROADWELL-SSE-NEXT:    paddd %xmm0, %xmm1 # sched: [1:0.50]
-; BROADWELL-SSE-NEXT:    paddd %xmm0, %xmm2 # sched: [1:0.50]
-; BROADWELL-SSE-NEXT:    movd %xmm2, %eax # sched: [1:1.00]
-; BROADWELL-SSE-NEXT:    movd %xmm1, (%rsi) # sched: [1:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_movd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vmovd %edi, %xmm1 # sched: [1:1.00]
-; BROADWELL-NEXT:    vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero sched: [5:0.50]
-; BROADWELL-NEXT:    vpaddd %xmm1, %xmm0, %xmm1 # sched: [1:0.50]
-; BROADWELL-NEXT:    vpaddd %xmm2, %xmm0, %xmm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    vmovd %xmm0, %eax # sched: [1:1.00]
-; BROADWELL-NEXT:    vmovd %xmm1, (%rsi) # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_movd:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    movd %edi, %xmm1 # sched: [1:1.00]
-; SKYLAKE-SSE-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero sched: [5:0.50]
-; SKYLAKE-SSE-NEXT:    paddd %xmm0, %xmm1 # sched: [1:0.33]
-; SKYLAKE-SSE-NEXT:    paddd %xmm0, %xmm2 # sched: [1:0.33]
-; SKYLAKE-SSE-NEXT:    movd %xmm2, %eax # sched: [2:1.00]
-; SKYLAKE-SSE-NEXT:    movd %xmm1, (%rsi) # sched: [1:1.00]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_movd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vmovd %edi, %xmm1 # sched: [1:1.00]
-; SKYLAKE-NEXT:    vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero sched: [5:0.50]
-; SKYLAKE-NEXT:    vpaddd %xmm1, %xmm0, %xmm1 # sched: [1:0.33]
-; SKYLAKE-NEXT:    vpaddd %xmm2, %xmm0, %xmm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    vmovd %xmm0, %eax # sched: [2:1.00]
-; SKYLAKE-NEXT:    vmovd %xmm1, (%rsi) # sched: [1:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_movd:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    movd %edi, %xmm1 # sched: [1:1.00]
-; SKX-SSE-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero sched: [5:0.50]
-; SKX-SSE-NEXT:    paddd %xmm0, %xmm1 # sched: [1:0.33]
-; SKX-SSE-NEXT:    paddd %xmm0, %xmm2 # sched: [1:0.33]
-; SKX-SSE-NEXT:    movd %xmm2, %eax # sched: [2:1.00]
-; SKX-SSE-NEXT:    movd %xmm1, (%rsi) # sched: [1:1.00]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_movd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovd %edi, %xmm1 # sched: [1:1.00]
-; SKX-NEXT:    vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero sched: [5:0.50]
-; SKX-NEXT:    vpaddd %xmm1, %xmm0, %xmm1 # sched: [1:0.33]
-; SKX-NEXT:    vpaddd %xmm2, %xmm0, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    vmovd %xmm0, %eax # sched: [2:1.00]
-; SKX-NEXT:    vmovd %xmm1, (%rsi) # sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_movd:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero sched: [5:0.50]
-; BDVER2-SSE-NEXT:    movd %edi, %xmm1 # sched: [10:0.50]
-; BDVER2-SSE-NEXT:    paddd %xmm0, %xmm2 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    movd %xmm2, %eax # sched: [10:1.00]
-; BDVER2-SSE-NEXT:    paddd %xmm0, %xmm1 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    movd %xmm1, (%rsi) # sched: [2:1.00]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_movd:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vmovd %edi, %xmm1 # sched: [10:0.50]
-; BDVER2-NEXT:    vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero sched: [5:0.50]
-; BDVER2-NEXT:    vpaddd %xmm1, %xmm0, %xmm1 # sched: [2:0.50]
-; BDVER2-NEXT:    vpaddd %xmm2, %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    vmovd %xmm0, %eax # sched: [10:1.00]
-; BDVER2-NEXT:    vmovd %xmm1, (%rsi) # sched: [2:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_movd:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero sched: [5:1.00]
-; BTVER2-SSE-NEXT:    movd %edi, %xmm1 # sched: [8:0.50]
-; BTVER2-SSE-NEXT:    paddd %xmm0, %xmm2 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    movd %xmm2, %eax # sched: [4:1.00]
-; BTVER2-SSE-NEXT:    paddd %xmm0, %xmm1 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    movd %xmm1, (%rsi) # sched: [2:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_movd:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vmovd %edi, %xmm1 # sched: [8:0.50]
-; BTVER2-NEXT:    vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero sched: [5:1.00]
-; BTVER2-NEXT:    vpaddd %xmm1, %xmm0, %xmm1 # sched: [1:0.50]
-; BTVER2-NEXT:    vpaddd %xmm2, %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    vmovd %xmm0, %eax # sched: [4:1.00]
-; BTVER2-NEXT:    vmovd %xmm1, (%rsi) # sched: [2:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_movd:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    movd %edi, %xmm1 # sched: [3:1.00]
-; ZNVER1-SSE-NEXT:    paddd %xmm0, %xmm1 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    movd %xmm1, (%rsi) # sched: [1:0.50]
-; ZNVER1-SSE-NEXT:    paddd %xmm0, %xmm2 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    movd %xmm2, %eax # sched: [2:1.00]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_movd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero sched: [8:0.50]
-; ZNVER1-NEXT:    vmovd %edi, %xmm1 # sched: [3:1.00]
-; ZNVER1-NEXT:    vpaddd %xmm1, %xmm0, %xmm1 # sched: [1:0.25]
-; ZNVER1-NEXT:    vmovd %xmm1, (%rsi) # sched: [1:0.50]
-; ZNVER1-NEXT:    vpaddd %xmm2, %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vmovd %xmm0, %eax # sched: [2:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = insertelement <4 x i32> undef, i32 %a1, i32 0
-  %2 = load i32, i32 *%a2
-  %3 = insertelement <4 x i32> undef, i32 %2, i32 0
-  %4 = add <4 x i32> %a0, %1
-  %5 = add <4 x i32> %a0, %3
-  %6 = extractelement <4 x i32> %4, i32 0
-  %7 = extractelement <4 x i32> %5, i32 0
-  store i32 %6, i32* %a2
-  ret i32 %7
-}
-
-define i64 @test_movd_64(<2 x i64> %a0, i64 %a1, i64 *%a2) {
-; GENERIC-LABEL: test_movd_64:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    movq %rdi, %xmm1 # sched: [1:1.00]
-; GENERIC-NEXT:    movq {{.*#+}} xmm2 = mem[0],zero sched: [6:0.50]
-; GENERIC-NEXT:    paddq %xmm0, %xmm1 # sched: [1:0.50]
-; GENERIC-NEXT:    paddq %xmm0, %xmm2 # sched: [1:0.50]
-; GENERIC-NEXT:    movq %xmm2, %rax # sched: [2:1.00]
-; GENERIC-NEXT:    movq %xmm1, (%rsi) # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_movd_64:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero sched: [1:1.00]
-; ATOM-NEXT:    movq %rdi, %xmm2 # sched: [1:1.00]
-; ATOM-NEXT:    paddq %xmm0, %xmm1 # sched: [2:1.00]
-; ATOM-NEXT:    paddq %xmm0, %xmm2 # sched: [2:1.00]
-; ATOM-NEXT:    movq %xmm1, %rax # sched: [3:3.00]
-; ATOM-NEXT:    movq %xmm2, (%rsi) # sched: [1:1.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_movd_64:
-; SLM:       # %bb.0:
-; SLM-NEXT:    movq {{.*#+}} xmm2 = mem[0],zero sched: [3:1.00]
-; SLM-NEXT:    movq %rdi, %xmm1 # sched: [1:0.50]
-; SLM-NEXT:    paddq %xmm0, %xmm1 # sched: [1:0.50]
-; SLM-NEXT:    movq %xmm1, (%rsi) # sched: [1:1.00]
-; SLM-NEXT:    paddq %xmm0, %xmm2 # sched: [1:0.50]
-; SLM-NEXT:    movq %xmm2, %rax # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_movd_64:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    movq %rdi, %xmm1 # sched: [1:1.00]
-; SANDY-SSE-NEXT:    movq {{.*#+}} xmm2 = mem[0],zero sched: [6:0.50]
-; SANDY-SSE-NEXT:    paddq %xmm0, %xmm1 # sched: [1:0.50]
-; SANDY-SSE-NEXT:    paddq %xmm0, %xmm2 # sched: [1:0.50]
-; SANDY-SSE-NEXT:    movq %xmm2, %rax # sched: [2:1.00]
-; SANDY-SSE-NEXT:    movq %xmm1, (%rsi) # sched: [1:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_movd_64:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vmovq %rdi, %xmm1 # sched: [1:1.00]
-; SANDY-NEXT:    vmovq {{.*#+}} xmm2 = mem[0],zero sched: [6:0.50]
-; SANDY-NEXT:    vpaddq %xmm1, %xmm0, %xmm1 # sched: [1:0.50]
-; SANDY-NEXT:    vpaddq %xmm2, %xmm0, %xmm0 # sched: [1:0.50]
-; SANDY-NEXT:    vmovq %xmm0, %rax # sched: [2:1.00]
-; SANDY-NEXT:    vmovq %xmm1, (%rsi) # sched: [1:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_movd_64:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    movq %rdi, %xmm1 # sched: [1:1.00]
-; HASWELL-SSE-NEXT:    movq {{.*#+}} xmm2 = mem[0],zero sched: [5:0.50]
-; HASWELL-SSE-NEXT:    paddq %xmm0, %xmm1 # sched: [1:0.50]
-; HASWELL-SSE-NEXT:    paddq %xmm0, %xmm2 # sched: [1:0.50]
-; HASWELL-SSE-NEXT:    movq %xmm2, %rax # sched: [1:1.00]
-; HASWELL-SSE-NEXT:    movq %xmm1, (%rsi) # sched: [1:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_movd_64:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vmovq %rdi, %xmm1 # sched: [1:1.00]
-; HASWELL-NEXT:    vmovq {{.*#+}} xmm2 = mem[0],zero sched: [5:0.50]
-; HASWELL-NEXT:    vpaddq %xmm1, %xmm0, %xmm1 # sched: [1:0.50]
-; HASWELL-NEXT:    vpaddq %xmm2, %xmm0, %xmm0 # sched: [1:0.50]
-; HASWELL-NEXT:    vmovq %xmm0, %rax # sched: [1:1.00]
-; HASWELL-NEXT:    vmovq %xmm1, (%rsi) # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_movd_64:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    movq %rdi, %xmm1 # sched: [1:1.00]
-; BROADWELL-SSE-NEXT:    movq {{.*#+}} xmm2 = mem[0],zero sched: [5:0.50]
-; BROADWELL-SSE-NEXT:    paddq %xmm0, %xmm1 # sched: [1:0.50]
-; BROADWELL-SSE-NEXT:    paddq %xmm0, %xmm2 # sched: [1:0.50]
-; BROADWELL-SSE-NEXT:    movq %xmm2, %rax # sched: [1:1.00]
-; BROADWELL-SSE-NEXT:    movq %xmm1, (%rsi) # sched: [1:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_movd_64:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vmovq %rdi, %xmm1 # sched: [1:1.00]
-; BROADWELL-NEXT:    vmovq {{.*#+}} xmm2 = mem[0],zero sched: [5:0.50]
-; BROADWELL-NEXT:    vpaddq %xmm1, %xmm0, %xmm1 # sched: [1:0.50]
-; BROADWELL-NEXT:    vpaddq %xmm2, %xmm0, %xmm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    vmovq %xmm0, %rax # sched: [1:1.00]
-; BROADWELL-NEXT:    vmovq %xmm1, (%rsi) # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_movd_64:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    movq %rdi, %xmm1 # sched: [1:1.00]
-; SKYLAKE-SSE-NEXT:    movq {{.*#+}} xmm2 = mem[0],zero sched: [5:0.50]
-; SKYLAKE-SSE-NEXT:    paddq %xmm0, %xmm1 # sched: [1:0.33]
-; SKYLAKE-SSE-NEXT:    paddq %xmm0, %xmm2 # sched: [1:0.33]
-; SKYLAKE-SSE-NEXT:    movq %xmm2, %rax # sched: [2:1.00]
-; SKYLAKE-SSE-NEXT:    movq %xmm1, (%rsi) # sched: [1:1.00]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_movd_64:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vmovq %rdi, %xmm1 # sched: [1:1.00]
-; SKYLAKE-NEXT:    vmovq {{.*#+}} xmm2 = mem[0],zero sched: [5:0.50]
-; SKYLAKE-NEXT:    vpaddq %xmm1, %xmm0, %xmm1 # sched: [1:0.33]
-; SKYLAKE-NEXT:    vpaddq %xmm2, %xmm0, %xmm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    vmovq %xmm0, %rax # sched: [2:1.00]
-; SKYLAKE-NEXT:    vmovq %xmm1, (%rsi) # sched: [1:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_movd_64:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    movq %rdi, %xmm1 # sched: [1:1.00]
-; SKX-SSE-NEXT:    movq {{.*#+}} xmm2 = mem[0],zero sched: [5:0.50]
-; SKX-SSE-NEXT:    paddq %xmm0, %xmm1 # sched: [1:0.33]
-; SKX-SSE-NEXT:    paddq %xmm0, %xmm2 # sched: [1:0.33]
-; SKX-SSE-NEXT:    movq %xmm2, %rax # sched: [2:1.00]
-; SKX-SSE-NEXT:    movq %xmm1, (%rsi) # sched: [1:1.00]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_movd_64:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovq %rdi, %xmm1 # sched: [1:1.00]
-; SKX-NEXT:    vmovq {{.*#+}} xmm2 = mem[0],zero sched: [5:0.50]
-; SKX-NEXT:    vpaddq %xmm1, %xmm0, %xmm1 # sched: [1:0.33]
-; SKX-NEXT:    vpaddq %xmm2, %xmm0, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    vmovq %xmm0, %rax # sched: [2:1.00]
-; SKX-NEXT:    vmovq %xmm1, (%rsi) # sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_movd_64:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    movq {{.*#+}} xmm2 = mem[0],zero sched: [5:0.50]
-; BDVER2-SSE-NEXT:    movq %rdi, %xmm1 # sched: [10:0.50]
-; BDVER2-SSE-NEXT:    paddq %xmm0, %xmm2 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    movq %xmm2, %rax # sched: [10:1.00]
-; BDVER2-SSE-NEXT:    paddq %xmm0, %xmm1 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    movq %xmm1, (%rsi) # sched: [2:1.00]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_movd_64:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vmovq %rdi, %xmm1 # sched: [10:0.50]
-; BDVER2-NEXT:    vmovq {{.*#+}} xmm2 = mem[0],zero sched: [5:0.50]
-; BDVER2-NEXT:    vpaddq %xmm1, %xmm0, %xmm1 # sched: [2:0.50]
-; BDVER2-NEXT:    vpaddq %xmm2, %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    vmovq %xmm0, %rax # sched: [10:1.00]
-; BDVER2-NEXT:    vmovq %xmm1, (%rsi) # sched: [2:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_movd_64:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    movq {{.*#+}} xmm2 = mem[0],zero sched: [5:1.00]
-; BTVER2-SSE-NEXT:    movq %rdi, %xmm1 # sched: [8:0.50]
-; BTVER2-SSE-NEXT:    paddq %xmm0, %xmm2 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    movq %xmm2, %rax # sched: [4:1.00]
-; BTVER2-SSE-NEXT:    paddq %xmm0, %xmm1 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    movq %xmm1, (%rsi) # sched: [2:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_movd_64:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vmovq %rdi, %xmm1 # sched: [8:0.50]
-; BTVER2-NEXT:    vmovq {{.*#+}} xmm2 = mem[0],zero sched: [5:1.00]
-; BTVER2-NEXT:    vpaddq %xmm1, %xmm0, %xmm1 # sched: [1:0.50]
-; BTVER2-NEXT:    vpaddq %xmm2, %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    vmovq %xmm0, %rax # sched: [4:1.00]
-; BTVER2-NEXT:    vmovq %xmm1, (%rsi) # sched: [2:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_movd_64:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    movq {{.*#+}} xmm2 = mem[0],zero sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    movq %rdi, %xmm1 # sched: [3:1.00]
-; ZNVER1-SSE-NEXT:    paddq %xmm0, %xmm1 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    movq %xmm1, (%rsi) # sched: [1:0.50]
-; ZNVER1-SSE-NEXT:    paddq %xmm0, %xmm2 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    movq %xmm2, %rax # sched: [2:1.00]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_movd_64:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vmovq {{.*#+}} xmm2 = mem[0],zero sched: [8:0.50]
-; ZNVER1-NEXT:    vmovq %rdi, %xmm1 # sched: [3:1.00]
-; ZNVER1-NEXT:    vpaddq %xmm1, %xmm0, %xmm1 # sched: [1:0.25]
-; ZNVER1-NEXT:    vmovq %xmm1, (%rsi) # sched: [1:0.50]
-; ZNVER1-NEXT:    vpaddq %xmm2, %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vmovq %xmm0, %rax # sched: [2:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = insertelement <2 x i64> undef, i64 %a1, i64 0
-  %2 = load i64, i64 *%a2
-  %3 = insertelement <2 x i64> undef, i64 %2, i64 0
-  %4 = add <2 x i64> %a0, %1
-  %5 = add <2 x i64> %a0, %3
-  %6 = extractelement <2 x i64> %4, i64 0
-  %7 = extractelement <2 x i64> %5, i64 0
-  store i64 %6, i64* %a2
-  ret i64 %7
-}
-
-define <2 x double> @test_movhpd(<2 x double> %a0, <2 x double> %a1, x86_mmx *%a2) {
-; GENERIC-LABEL: test_movhpd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    movhpd {{.*#+}} xmm1 = xmm1[0],mem[0] sched: [7:1.00]
-; GENERIC-NEXT:    addpd %xmm1, %xmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    movhpd %xmm0, (%rdi) # sched: [1:1.00]
-; GENERIC-NEXT:    movapd %xmm1, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_movhpd:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    movapd %xmm0, %xmm2 # sched: [1:0.50]
-; ATOM-NEXT:    movhpd {{.*#+}} xmm1 = xmm1[0],mem[0] sched: [1:1.00]
-; ATOM-NEXT:    addpd %xmm1, %xmm2 # sched: [6:3.00]
-; ATOM-NEXT:    movapd %xmm1, %xmm0 # sched: [1:0.50]
-; ATOM-NEXT:    movhpd %xmm2, (%rdi) # sched: [1:1.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_movhpd:
-; SLM:       # %bb.0:
-; SLM-NEXT:    movhpd {{.*#+}} xmm1 = xmm1[0],mem[0] sched: [4:1.00]
-; SLM-NEXT:    addpd %xmm1, %xmm0 # sched: [3:1.00]
-; SLM-NEXT:    movhpd %xmm0, (%rdi) # sched: [1:1.00]
-; SLM-NEXT:    movapd %xmm1, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_movhpd:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    movhpd {{.*#+}} xmm1 = xmm1[0],mem[0] sched: [7:1.00]
-; SANDY-SSE-NEXT:    addpd %xmm1, %xmm0 # sched: [3:1.00]
-; SANDY-SSE-NEXT:    movhpd %xmm0, (%rdi) # sched: [1:1.00]
-; SANDY-SSE-NEXT:    movapd %xmm1, %xmm0 # sched: [1:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_movhpd:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vmovhpd {{.*#+}} xmm1 = xmm1[0],mem[0] sched: [7:1.00]
-; SANDY-NEXT:    vaddpd %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; SANDY-NEXT:    vmovhpd %xmm0, (%rdi) # sched: [1:1.00]
-; SANDY-NEXT:    vmovapd %xmm1, %xmm0 # sched: [1:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_movhpd:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    movhpd {{.*#+}} xmm1 = xmm1[0],mem[0] sched: [6:1.00]
-; HASWELL-SSE-NEXT:    addpd %xmm1, %xmm0 # sched: [3:1.00]
-; HASWELL-SSE-NEXT:    movhpd %xmm0, (%rdi) # sched: [1:1.00]
-; HASWELL-SSE-NEXT:    movapd %xmm1, %xmm0 # sched: [1:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_movhpd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vmovhpd {{.*#+}} xmm1 = xmm1[0],mem[0] sched: [6:1.00]
-; HASWELL-NEXT:    vaddpd %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; HASWELL-NEXT:    vmovhpd %xmm0, (%rdi) # sched: [1:1.00]
-; HASWELL-NEXT:    vmovapd %xmm1, %xmm0 # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_movhpd:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    movhpd {{.*#+}} xmm1 = xmm1[0],mem[0] sched: [6:1.00]
-; BROADWELL-SSE-NEXT:    addpd %xmm1, %xmm0 # sched: [3:1.00]
-; BROADWELL-SSE-NEXT:    movhpd %xmm0, (%rdi) # sched: [1:1.00]
-; BROADWELL-SSE-NEXT:    movapd %xmm1, %xmm0 # sched: [1:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_movhpd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vmovhpd {{.*#+}} xmm1 = xmm1[0],mem[0] sched: [6:1.00]
-; BROADWELL-NEXT:    vaddpd %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    vmovhpd %xmm0, (%rdi) # sched: [1:1.00]
-; BROADWELL-NEXT:    vmovapd %xmm1, %xmm0 # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_movhpd:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    movhpd {{.*#+}} xmm1 = xmm1[0],mem[0] sched: [6:1.00]
-; SKYLAKE-SSE-NEXT:    addpd %xmm1, %xmm0 # sched: [4:0.50]
-; SKYLAKE-SSE-NEXT:    movhpd %xmm0, (%rdi) # sched: [1:1.00]
-; SKYLAKE-SSE-NEXT:    movapd %xmm1, %xmm0 # sched: [1:0.33]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_movhpd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vmovhpd {{.*#+}} xmm1 = xmm1[0],mem[0] sched: [6:1.00]
-; SKYLAKE-NEXT:    vaddpd %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    vmovhpd %xmm0, (%rdi) # sched: [1:1.00]
-; SKYLAKE-NEXT:    vmovapd %xmm1, %xmm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_movhpd:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    movhpd {{.*#+}} xmm1 = xmm1[0],mem[0] sched: [6:1.00]
-; SKX-SSE-NEXT:    addpd %xmm1, %xmm0 # sched: [4:0.50]
-; SKX-SSE-NEXT:    movhpd %xmm0, (%rdi) # sched: [1:1.00]
-; SKX-SSE-NEXT:    movapd %xmm1, %xmm0 # sched: [1:0.33]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_movhpd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovhpd {{.*#+}} xmm1 = xmm1[0],mem[0] sched: [6:1.00]
-; SKX-NEXT:    vaddpd %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKX-NEXT:    vmovhpd %xmm0, (%rdi) # sched: [1:1.00]
-; SKX-NEXT:    vmovapd %xmm1, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_movhpd:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    movhpd {{.*#+}} xmm1 = xmm1[0],mem[0] sched: [7:0.50]
-; BDVER2-SSE-NEXT:    addpd %xmm1, %xmm0 # sched: [5:1.00]
-; BDVER2-SSE-NEXT:    movhpd %xmm0, (%rdi) # sched: [2:1.00]
-; BDVER2-SSE-NEXT:    movapd %xmm1, %xmm0 # sched: [1:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_movhpd:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vmovhpd {{.*#+}} xmm1 = xmm1[0],mem[0] sched: [7:0.50]
-; BDVER2-NEXT:    vaddpd %xmm1, %xmm0, %xmm0 # sched: [5:1.00]
-; BDVER2-NEXT:    vmovhpd %xmm0, (%rdi) # sched: [2:1.00]
-; BDVER2-NEXT:    vmovapd %xmm1, %xmm0 # sched: [1:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_movhpd:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    movhpd {{.*#+}} xmm1 = xmm1[0],mem[0] sched: [6:1.00]
-; BTVER2-SSE-NEXT:    addpd %xmm1, %xmm0 # sched: [3:1.00]
-; BTVER2-SSE-NEXT:    movhpd %xmm0, (%rdi) # sched: [2:1.00]
-; BTVER2-SSE-NEXT:    movapd %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_movhpd:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vmovhpd {{.*#+}} xmm1 = xmm1[0],mem[0] sched: [6:1.00]
-; BTVER2-NEXT:    vaddpd %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; BTVER2-NEXT:    vmovhpd %xmm0, (%rdi) # sched: [2:1.00]
-; BTVER2-NEXT:    vmovapd %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_movhpd:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    movhpd {{.*#+}} xmm1 = xmm1[0],mem[0] sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    addpd %xmm1, %xmm0 # sched: [3:1.00]
-; ZNVER1-SSE-NEXT:    movhpd %xmm0, (%rdi) # sched: [1:0.50]
-; ZNVER1-SSE-NEXT:    movapd %xmm1, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_movhpd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vmovhpd {{.*#+}} xmm1 = xmm1[0],mem[0] sched: [8:0.50]
-; ZNVER1-NEXT:    vaddpd %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    vmovhpd %xmm0, (%rdi) # sched: [1:0.50]
-; ZNVER1-NEXT:    vmovapd %xmm1, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = bitcast x86_mmx* %a2 to double*
-  %2 = load double, double *%1, align 8
-  %3 = insertelement <2 x double> %a1, double %2, i32 1
-  %4 = fadd <2 x double> %a0, %3
-  %5 = extractelement <2 x double> %4, i32 1
-  store double %5, double* %1
-  ret <2 x double> %3
-}
-
-define <2 x double> @test_movlpd(<2 x double> %a0, <2 x double> %a1, x86_mmx *%a2) {
-; GENERIC-LABEL: test_movlpd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    movlpd {{.*#+}} xmm1 = mem[0],xmm1[1] sched: [7:1.00]
-; GENERIC-NEXT:    addpd %xmm1, %xmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    movlpd %xmm0, (%rdi) # sched: [1:1.00]
-; GENERIC-NEXT:    movapd %xmm1, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_movlpd:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    movapd %xmm0, %xmm2 # sched: [1:0.50]
-; ATOM-NEXT:    movlpd {{.*#+}} xmm1 = mem[0],xmm1[1] sched: [1:1.00]
-; ATOM-NEXT:    addpd %xmm1, %xmm2 # sched: [6:3.00]
-; ATOM-NEXT:    movapd %xmm1, %xmm0 # sched: [1:0.50]
-; ATOM-NEXT:    movlpd %xmm2, (%rdi) # sched: [1:1.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_movlpd:
-; SLM:       # %bb.0:
-; SLM-NEXT:    movlpd {{.*#+}} xmm1 = mem[0],xmm1[1] sched: [4:1.00]
-; SLM-NEXT:    addpd %xmm1, %xmm0 # sched: [3:1.00]
-; SLM-NEXT:    movlpd %xmm0, (%rdi) # sched: [1:1.00]
-; SLM-NEXT:    movapd %xmm1, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_movlpd:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    movlpd {{.*#+}} xmm1 = mem[0],xmm1[1] sched: [7:1.00]
-; SANDY-SSE-NEXT:    addpd %xmm1, %xmm0 # sched: [3:1.00]
-; SANDY-SSE-NEXT:    movlpd %xmm0, (%rdi) # sched: [1:1.00]
-; SANDY-SSE-NEXT:    movapd %xmm1, %xmm0 # sched: [1:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_movlpd:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vmovlpd {{.*#+}} xmm1 = mem[0],xmm1[1] sched: [7:1.00]
-; SANDY-NEXT:    vaddpd %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; SANDY-NEXT:    vmovlpd %xmm0, (%rdi) # sched: [1:1.00]
-; SANDY-NEXT:    vmovapd %xmm1, %xmm0 # sched: [1:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_movlpd:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    movlpd {{.*#+}} xmm1 = mem[0],xmm1[1] sched: [6:1.00]
-; HASWELL-SSE-NEXT:    addpd %xmm1, %xmm0 # sched: [3:1.00]
-; HASWELL-SSE-NEXT:    movlpd %xmm0, (%rdi) # sched: [1:1.00]
-; HASWELL-SSE-NEXT:    movapd %xmm1, %xmm0 # sched: [1:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_movlpd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vmovlpd {{.*#+}} xmm1 = mem[0],xmm1[1] sched: [6:1.00]
-; HASWELL-NEXT:    vaddpd %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; HASWELL-NEXT:    vmovlpd %xmm0, (%rdi) # sched: [1:1.00]
-; HASWELL-NEXT:    vmovapd %xmm1, %xmm0 # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_movlpd:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    movlpd {{.*#+}} xmm1 = mem[0],xmm1[1] sched: [6:1.00]
-; BROADWELL-SSE-NEXT:    addpd %xmm1, %xmm0 # sched: [3:1.00]
-; BROADWELL-SSE-NEXT:    movlpd %xmm0, (%rdi) # sched: [1:1.00]
-; BROADWELL-SSE-NEXT:    movapd %xmm1, %xmm0 # sched: [1:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_movlpd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vmovlpd {{.*#+}} xmm1 = mem[0],xmm1[1] sched: [6:1.00]
-; BROADWELL-NEXT:    vaddpd %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    vmovlpd %xmm0, (%rdi) # sched: [1:1.00]
-; BROADWELL-NEXT:    vmovapd %xmm1, %xmm0 # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_movlpd:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    movlpd {{.*#+}} xmm1 = mem[0],xmm1[1] sched: [6:1.00]
-; SKYLAKE-SSE-NEXT:    addpd %xmm1, %xmm0 # sched: [4:0.50]
-; SKYLAKE-SSE-NEXT:    movlpd %xmm0, (%rdi) # sched: [1:1.00]
-; SKYLAKE-SSE-NEXT:    movapd %xmm1, %xmm0 # sched: [1:0.33]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_movlpd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vmovlpd {{.*#+}} xmm1 = mem[0],xmm1[1] sched: [6:1.00]
-; SKYLAKE-NEXT:    vaddpd %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    vmovlpd %xmm0, (%rdi) # sched: [1:1.00]
-; SKYLAKE-NEXT:    vmovapd %xmm1, %xmm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_movlpd:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    movlpd {{.*#+}} xmm1 = mem[0],xmm1[1] sched: [6:1.00]
-; SKX-SSE-NEXT:    addpd %xmm1, %xmm0 # sched: [4:0.50]
-; SKX-SSE-NEXT:    movlpd %xmm0, (%rdi) # sched: [1:1.00]
-; SKX-SSE-NEXT:    movapd %xmm1, %xmm0 # sched: [1:0.33]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_movlpd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovlpd {{.*#+}} xmm1 = mem[0],xmm1[1] sched: [6:1.00]
-; SKX-NEXT:    vaddpd %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKX-NEXT:    vmovlpd %xmm0, (%rdi) # sched: [1:1.00]
-; SKX-NEXT:    vmovapd %xmm1, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_movlpd:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    movlpd {{.*#+}} xmm1 = mem[0],xmm1[1] sched: [7:0.50]
-; BDVER2-SSE-NEXT:    addpd %xmm1, %xmm0 # sched: [5:1.00]
-; BDVER2-SSE-NEXT:    movlpd %xmm0, (%rdi) # sched: [2:1.00]
-; BDVER2-SSE-NEXT:    movapd %xmm1, %xmm0 # sched: [1:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_movlpd:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vmovlpd {{.*#+}} xmm1 = mem[0],xmm1[1] sched: [7:0.50]
-; BDVER2-NEXT:    vaddpd %xmm1, %xmm0, %xmm0 # sched: [5:1.00]
-; BDVER2-NEXT:    vmovlpd %xmm0, (%rdi) # sched: [2:1.00]
-; BDVER2-NEXT:    vmovapd %xmm1, %xmm0 # sched: [1:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_movlpd:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    movlpd {{.*#+}} xmm1 = mem[0],xmm1[1] sched: [6:1.00]
-; BTVER2-SSE-NEXT:    addpd %xmm1, %xmm0 # sched: [3:1.00]
-; BTVER2-SSE-NEXT:    movlpd %xmm0, (%rdi) # sched: [2:1.00]
-; BTVER2-SSE-NEXT:    movapd %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_movlpd:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vmovlpd {{.*#+}} xmm1 = mem[0],xmm1[1] sched: [6:1.00]
-; BTVER2-NEXT:    vaddpd %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; BTVER2-NEXT:    vmovlpd %xmm0, (%rdi) # sched: [2:1.00]
-; BTVER2-NEXT:    vmovapd %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_movlpd:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    movlpd {{.*#+}} xmm1 = mem[0],xmm1[1] sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    addpd %xmm1, %xmm0 # sched: [3:1.00]
-; ZNVER1-SSE-NEXT:    movlpd %xmm0, (%rdi) # sched: [1:0.50]
-; ZNVER1-SSE-NEXT:    movapd %xmm1, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_movlpd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vmovlpd {{.*#+}} xmm1 = mem[0],xmm1[1] sched: [8:0.50]
-; ZNVER1-NEXT:    vaddpd %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    vmovlpd %xmm0, (%rdi) # sched: [1:0.50]
-; ZNVER1-NEXT:    vmovapd %xmm1, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = bitcast x86_mmx* %a2 to double*
-  %2 = load double, double *%1, align 8
-  %3 = insertelement <2 x double> %a1, double %2, i32 0
-  %4 = fadd <2 x double> %a0, %3
-  %5 = extractelement <2 x double> %4, i32 0
-  store double %5, double* %1
-  ret <2 x double> %3
-}
-
-define i32 @test_movmskpd(<2 x double> %a0) {
-; GENERIC-LABEL: test_movmskpd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    movmskpd %xmm0, %eax # sched: [2:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_movmskpd:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    movmskpd %xmm0, %eax # sched: [3:3.00]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_movmskpd:
-; SLM:       # %bb.0:
-; SLM-NEXT:    movmskpd %xmm0, %eax # sched: [4:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_movmskpd:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    movmskpd %xmm0, %eax # sched: [2:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_movmskpd:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vmovmskpd %xmm0, %eax # sched: [2:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_movmskpd:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    movmskpd %xmm0, %eax # sched: [3:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_movmskpd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vmovmskpd %xmm0, %eax # sched: [3:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_movmskpd:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    movmskpd %xmm0, %eax # sched: [3:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_movmskpd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vmovmskpd %xmm0, %eax # sched: [3:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_movmskpd:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    movmskpd %xmm0, %eax # sched: [2:1.00]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_movmskpd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vmovmskpd %xmm0, %eax # sched: [2:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_movmskpd:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    movmskpd %xmm0, %eax # sched: [2:1.00]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_movmskpd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovmskpd %xmm0, %eax # sched: [2:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_movmskpd:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    movmskpd %xmm0, %eax # sched: [10:1.00]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_movmskpd:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vmovmskpd %xmm0, %eax # sched: [10:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_movmskpd:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    movmskpd %xmm0, %eax # sched: [3:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_movmskpd:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vmovmskpd %xmm0, %eax # sched: [3:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_movmskpd:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    movmskpd %xmm0, %eax # sched: [1:1.00]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_movmskpd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vmovmskpd %xmm0, %eax # sched: [1:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call i32 @llvm.x86.sse2.movmsk.pd(<2 x double> %a0)
-  ret i32 %1
-}
-declare i32 @llvm.x86.sse2.movmsk.pd(<2 x double>) nounwind readnone
-
-define void @test_movntdqa(<2 x i64> %a0, <2 x i64> *%a1) {
-; GENERIC-LABEL: test_movntdqa:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    paddq %xmm0, %xmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    movntdq %xmm0, (%rdi) # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_movntdqa:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    paddq %xmm0, %xmm0 # sched: [2:1.00]
-; ATOM-NEXT:    movntdq %xmm0, (%rdi) # sched: [1:1.00]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_movntdqa:
-; SLM:       # %bb.0:
-; SLM-NEXT:    paddq %xmm0, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    movntdq %xmm0, (%rdi) # sched: [1:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_movntdqa:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    paddq %xmm0, %xmm0 # sched: [1:0.50]
-; SANDY-SSE-NEXT:    movntdq %xmm0, (%rdi) # sched: [1:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_movntdqa:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vpaddq %xmm0, %xmm0, %xmm0 # sched: [1:0.50]
-; SANDY-NEXT:    vmovntdq %xmm0, (%rdi) # sched: [1:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_movntdqa:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    paddq %xmm0, %xmm0 # sched: [1:0.50]
-; HASWELL-SSE-NEXT:    movntdq %xmm0, (%rdi) # sched: [1:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_movntdqa:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpaddq %xmm0, %xmm0, %xmm0 # sched: [1:0.50]
-; HASWELL-NEXT:    vmovntdq %xmm0, (%rdi) # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_movntdqa:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    paddq %xmm0, %xmm0 # sched: [1:0.50]
-; BROADWELL-SSE-NEXT:    movntdq %xmm0, (%rdi) # sched: [1:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_movntdqa:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpaddq %xmm0, %xmm0, %xmm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    vmovntdq %xmm0, (%rdi) # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_movntdqa:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    paddq %xmm0, %xmm0 # sched: [1:0.33]
-; SKYLAKE-SSE-NEXT:    movntdq %xmm0, (%rdi) # sched: [1:1.00]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_movntdqa:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpaddq %xmm0, %xmm0, %xmm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    vmovntdq %xmm0, (%rdi) # sched: [1:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_movntdqa:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    paddq %xmm0, %xmm0 # sched: [1:0.33]
-; SKX-SSE-NEXT:    movntdq %xmm0, (%rdi) # sched: [1:1.00]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_movntdqa:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpaddq %xmm0, %xmm0, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    vmovntdq %xmm0, (%rdi) # sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_movntdqa:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    paddq %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    movntdq %xmm0, (%rdi) # sched: [2:1.00]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_movntdqa:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vpaddq %xmm0, %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    vmovntdq %xmm0, (%rdi) # sched: [2:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_movntdqa:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    paddq %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    movntdq %xmm0, (%rdi) # sched: [2:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_movntdqa:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vpaddq %xmm0, %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    vmovntdq %xmm0, (%rdi) # sched: [2:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_movntdqa:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    paddq %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    movntdq %xmm0, (%rdi) # sched: [1:0.50]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_movntdqa:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpaddq %xmm0, %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vmovntdq %xmm0, (%rdi) # sched: [1:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = add <2 x i64> %a0, %a0
-  store <2 x i64> %1, <2 x i64> *%a1, align 16, !nontemporal !0
-  ret void
-}
-
-define void @test_movntpd(<2 x double> %a0, <2 x double> *%a1) {
-; GENERIC-LABEL: test_movntpd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    addpd %xmm0, %xmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    movntpd %xmm0, (%rdi) # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_movntpd:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    addpd %xmm0, %xmm0 # sched: [6:3.00]
-; ATOM-NEXT:    movntpd %xmm0, (%rdi) # sched: [1:1.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_movntpd:
-; SLM:       # %bb.0:
-; SLM-NEXT:    addpd %xmm0, %xmm0 # sched: [3:1.00]
-; SLM-NEXT:    movntpd %xmm0, (%rdi) # sched: [1:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_movntpd:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    addpd %xmm0, %xmm0 # sched: [3:1.00]
-; SANDY-SSE-NEXT:    movntpd %xmm0, (%rdi) # sched: [1:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_movntpd:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vaddpd %xmm0, %xmm0, %xmm0 # sched: [3:1.00]
-; SANDY-NEXT:    vmovntpd %xmm0, (%rdi) # sched: [1:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_movntpd:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    addpd %xmm0, %xmm0 # sched: [3:1.00]
-; HASWELL-SSE-NEXT:    movntpd %xmm0, (%rdi) # sched: [1:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_movntpd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vaddpd %xmm0, %xmm0, %xmm0 # sched: [3:1.00]
-; HASWELL-NEXT:    vmovntpd %xmm0, (%rdi) # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_movntpd:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    addpd %xmm0, %xmm0 # sched: [3:1.00]
-; BROADWELL-SSE-NEXT:    movntpd %xmm0, (%rdi) # sched: [1:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_movntpd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vaddpd %xmm0, %xmm0, %xmm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    vmovntpd %xmm0, (%rdi) # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_movntpd:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    addpd %xmm0, %xmm0 # sched: [4:0.50]
-; SKYLAKE-SSE-NEXT:    movntpd %xmm0, (%rdi) # sched: [1:1.00]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_movntpd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vaddpd %xmm0, %xmm0, %xmm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    vmovntpd %xmm0, (%rdi) # sched: [1:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_movntpd:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    addpd %xmm0, %xmm0 # sched: [4:0.50]
-; SKX-SSE-NEXT:    movntpd %xmm0, (%rdi) # sched: [1:1.00]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_movntpd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vaddpd %xmm0, %xmm0, %xmm0 # sched: [4:0.50]
-; SKX-NEXT:    vmovntpd %xmm0, (%rdi) # sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_movntpd:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    addpd %xmm0, %xmm0 # sched: [5:1.00]
-; BDVER2-SSE-NEXT:    movntpd %xmm0, (%rdi) # sched: [3:1.00]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_movntpd:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vaddpd %xmm0, %xmm0, %xmm0 # sched: [5:1.00]
-; BDVER2-NEXT:    vmovntpd %xmm0, (%rdi) # sched: [3:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_movntpd:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    addpd %xmm0, %xmm0 # sched: [3:1.00]
-; BTVER2-SSE-NEXT:    movntpd %xmm0, (%rdi) # sched: [3:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_movntpd:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vaddpd %xmm0, %xmm0, %xmm0 # sched: [3:1.00]
-; BTVER2-NEXT:    vmovntpd %xmm0, (%rdi) # sched: [3:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_movntpd:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    addpd %xmm0, %xmm0 # sched: [3:1.00]
-; ZNVER1-SSE-NEXT:    movntpd %xmm0, (%rdi) # sched: [1:0.50]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_movntpd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vaddpd %xmm0, %xmm0, %xmm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    vmovntpd %xmm0, (%rdi) # sched: [1:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = fadd <2 x double> %a0, %a0
-  store <2 x double> %1, <2 x double> *%a1, align 16, !nontemporal !0
-  ret void
-}
-
-define <2 x i64> @test_movq_mem(<2 x i64> %a0, i64 *%a1) {
-; GENERIC-LABEL: test_movq_mem:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero sched: [6:0.50]
-; GENERIC-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    movq %xmm0, (%rdi) # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_movq_mem:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero sched: [1:1.00]
-; ATOM-NEXT:    paddq %xmm1, %xmm0 # sched: [2:1.00]
-; ATOM-NEXT:    movq %xmm0, (%rdi) # sched: [1:1.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_movq_mem:
-; SLM:       # %bb.0:
-; SLM-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero sched: [3:1.00]
-; SLM-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    movq %xmm0, (%rdi) # sched: [1:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_movq_mem:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero sched: [6:0.50]
-; SANDY-SSE-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.50]
-; SANDY-SSE-NEXT:    movq %xmm0, (%rdi) # sched: [1:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_movq_mem:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero sched: [6:0.50]
-; SANDY-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SANDY-NEXT:    vmovq %xmm0, (%rdi) # sched: [1:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_movq_mem:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero sched: [5:0.50]
-; HASWELL-SSE-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.50]
-; HASWELL-SSE-NEXT:    movq %xmm0, (%rdi) # sched: [1:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_movq_mem:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero sched: [5:0.50]
-; HASWELL-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; HASWELL-NEXT:    vmovq %xmm0, (%rdi) # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_movq_mem:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero sched: [5:0.50]
-; BROADWELL-SSE-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.50]
-; BROADWELL-SSE-NEXT:    movq %xmm0, (%rdi) # sched: [1:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_movq_mem:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero sched: [5:0.50]
-; BROADWELL-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    vmovq %xmm0, (%rdi) # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_movq_mem:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero sched: [5:0.50]
-; SKYLAKE-SSE-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.33]
-; SKYLAKE-SSE-NEXT:    movq %xmm0, (%rdi) # sched: [1:1.00]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_movq_mem:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero sched: [5:0.50]
-; SKYLAKE-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    vmovq %xmm0, (%rdi) # sched: [1:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_movq_mem:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero sched: [5:0.50]
-; SKX-SSE-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.33]
-; SKX-SSE-NEXT:    movq %xmm0, (%rdi) # sched: [1:1.00]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_movq_mem:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero sched: [5:0.50]
-; SKX-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    vmovq %xmm0, (%rdi) # sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_movq_mem:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero sched: [5:0.50]
-; BDVER2-SSE-NEXT:    paddq %xmm1, %xmm0 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    movq %xmm0, (%rdi) # sched: [2:1.00]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_movq_mem:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero sched: [5:0.50]
-; BDVER2-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    vmovq %xmm0, (%rdi) # sched: [2:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_movq_mem:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero sched: [5:1.00]
-; BTVER2-SSE-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    movq %xmm0, (%rdi) # sched: [2:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_movq_mem:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero sched: [5:1.00]
-; BTVER2-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    vmovq %xmm0, (%rdi) # sched: [2:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_movq_mem:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    movq %xmm0, (%rdi) # sched: [1:0.50]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_movq_mem:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vmovq {{.*#+}} xmm1 = mem[0],zero sched: [8:0.50]
-; ZNVER1-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vmovq %xmm0, (%rdi) # sched: [1:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = load i64, i64* %a1, align 1
-  %2 = insertelement <2 x i64> zeroinitializer, i64 %1, i32 0
-  %3 = add <2 x i64> %a0, %2
-  %4 = extractelement <2 x i64> %3, i32 0
-  store i64 %4, i64 *%a1, align 1
-  ret <2 x i64> %3
-}
-
-define <2 x i64> @test_movq_reg(<2 x i64> %a0, <2 x i64> %a1) {
-; GENERIC-LABEL: test_movq_reg:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    movq {{.*#+}} xmm0 = xmm0[0],zero sched: [1:0.33]
-; GENERIC-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_movq_reg:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    movq {{.*#+}} xmm0 = xmm0[0],zero sched: [1:0.50]
-; ATOM-NEXT:    paddq %xmm1, %xmm0 # sched: [2:1.00]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_movq_reg:
-; SLM:       # %bb.0:
-; SLM-NEXT:    movq {{.*#+}} xmm0 = xmm0[0],zero sched: [1:0.50]
-; SLM-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_movq_reg:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    movq {{.*#+}} xmm0 = xmm0[0],zero sched: [1:0.33]
-; SANDY-SSE-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.50]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_movq_reg:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero sched: [1:0.33]
-; SANDY-NEXT:    vpaddq %xmm0, %xmm1, %xmm0 # sched: [1:0.50]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_movq_reg:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    movq {{.*#+}} xmm0 = xmm0[0],zero sched: [1:0.33]
-; HASWELL-SSE-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.50]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_movq_reg:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero sched: [1:0.33]
-; HASWELL-NEXT:    vpaddq %xmm0, %xmm1, %xmm0 # sched: [1:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_movq_reg:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    movq {{.*#+}} xmm0 = xmm0[0],zero sched: [1:0.33]
-; BROADWELL-SSE-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.50]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_movq_reg:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero sched: [1:0.33]
-; BROADWELL-NEXT:    vpaddq %xmm0, %xmm1, %xmm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_movq_reg:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    movq {{.*#+}} xmm0 = xmm0[0],zero sched: [1:0.33]
-; SKYLAKE-SSE-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.33]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_movq_reg:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero sched: [1:0.33]
-; SKYLAKE-NEXT:    vpaddq %xmm0, %xmm1, %xmm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_movq_reg:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    movq {{.*#+}} xmm0 = xmm0[0],zero sched: [1:0.33]
-; SKX-SSE-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.33]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_movq_reg:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero sched: [1:0.33]
-; SKX-NEXT:    vpaddq %xmm0, %xmm1, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_movq_reg:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    movq {{.*#+}} xmm0 = xmm0[0],zero sched: [2:0.50]
-; BDVER2-SSE-NEXT:    paddq %xmm1, %xmm0 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_movq_reg:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero sched: [2:0.50]
-; BDVER2-NEXT:    vpaddq %xmm0, %xmm1, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_movq_reg:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    movq {{.*#+}} xmm0 = xmm0[0],zero sched: [1:0.50]
-; BTVER2-SSE-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_movq_reg:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero sched: [1:0.50]
-; BTVER2-NEXT:    vpaddq %xmm0, %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_movq_reg:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    movq {{.*#+}} xmm0 = xmm0[0],zero sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_movq_reg:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vmovq {{.*#+}} xmm0 = xmm0[0],zero sched: [1:0.25]
-; ZNVER1-NEXT:    vpaddq %xmm0, %xmm1, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = shufflevector <2 x i64> %a0, <2 x i64> zeroinitializer, <2 x i32> <i32 0, i32 2>
-  %2 = add <2 x i64> %a1, %1
-  ret <2 x i64> %2
-}
-
-define void @test_movsd_mem(double* %a0, double* %a1) {
-; GENERIC-LABEL: test_movsd_mem:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero sched: [6:0.50]
-; GENERIC-NEXT:    addsd %xmm0, %xmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    movsd %xmm0, (%rsi) # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_movsd_mem:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero sched: [1:1.00]
-; ATOM-NEXT:    addsd %xmm0, %xmm0 # sched: [5:5.00]
-; ATOM-NEXT:    movsd %xmm0, (%rsi) # sched: [1:1.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_movsd_mem:
-; SLM:       # %bb.0:
-; SLM-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero sched: [3:1.00]
-; SLM-NEXT:    addsd %xmm0, %xmm0 # sched: [3:1.00]
-; SLM-NEXT:    movsd %xmm0, (%rsi) # sched: [1:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_movsd_mem:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero sched: [6:0.50]
-; SANDY-SSE-NEXT:    addsd %xmm0, %xmm0 # sched: [3:1.00]
-; SANDY-SSE-NEXT:    movsd %xmm0, (%rsi) # sched: [1:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_movsd_mem:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero sched: [6:0.50]
-; SANDY-NEXT:    vaddsd %xmm0, %xmm0, %xmm0 # sched: [3:1.00]
-; SANDY-NEXT:    vmovsd %xmm0, (%rsi) # sched: [1:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_movsd_mem:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero sched: [5:0.50]
-; HASWELL-SSE-NEXT:    addsd %xmm0, %xmm0 # sched: [3:1.00]
-; HASWELL-SSE-NEXT:    movsd %xmm0, (%rsi) # sched: [1:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_movsd_mem:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero sched: [5:0.50]
-; HASWELL-NEXT:    vaddsd %xmm0, %xmm0, %xmm0 # sched: [3:1.00]
-; HASWELL-NEXT:    vmovsd %xmm0, (%rsi) # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_movsd_mem:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero sched: [5:0.50]
-; BROADWELL-SSE-NEXT:    addsd %xmm0, %xmm0 # sched: [3:1.00]
-; BROADWELL-SSE-NEXT:    movsd %xmm0, (%rsi) # sched: [1:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_movsd_mem:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero sched: [5:0.50]
-; BROADWELL-NEXT:    vaddsd %xmm0, %xmm0, %xmm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    vmovsd %xmm0, (%rsi) # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_movsd_mem:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero sched: [5:0.50]
-; SKYLAKE-SSE-NEXT:    addsd %xmm0, %xmm0 # sched: [4:0.50]
-; SKYLAKE-SSE-NEXT:    movsd %xmm0, (%rsi) # sched: [1:1.00]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_movsd_mem:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero sched: [5:0.50]
-; SKYLAKE-NEXT:    vaddsd %xmm0, %xmm0, %xmm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    vmovsd %xmm0, (%rsi) # sched: [1:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_movsd_mem:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero sched: [5:0.50]
-; SKX-SSE-NEXT:    addsd %xmm0, %xmm0 # sched: [4:0.50]
-; SKX-SSE-NEXT:    movsd %xmm0, (%rsi) # sched: [1:1.00]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_movsd_mem:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero sched: [5:0.50]
-; SKX-NEXT:    vaddsd %xmm0, %xmm0, %xmm0 # sched: [4:0.50]
-; SKX-NEXT:    vmovsd %xmm0, (%rsi) # sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_movsd_mem:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero sched: [5:0.50]
-; BDVER2-SSE-NEXT:    addsd %xmm0, %xmm0 # sched: [5:1.00]
-; BDVER2-SSE-NEXT:    movsd %xmm0, (%rsi) # sched: [2:1.00]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_movsd_mem:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero sched: [5:0.50]
-; BDVER2-NEXT:    vaddsd %xmm0, %xmm0, %xmm0 # sched: [5:1.00]
-; BDVER2-NEXT:    vmovsd %xmm0, (%rsi) # sched: [2:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_movsd_mem:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero sched: [5:1.00]
-; BTVER2-SSE-NEXT:    addsd %xmm0, %xmm0 # sched: [3:1.00]
-; BTVER2-SSE-NEXT:    movsd %xmm0, (%rsi) # sched: [2:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_movsd_mem:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero sched: [5:1.00]
-; BTVER2-NEXT:    vaddsd %xmm0, %xmm0, %xmm0 # sched: [3:1.00]
-; BTVER2-NEXT:    vmovsd %xmm0, (%rsi) # sched: [2:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_movsd_mem:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    movsd {{.*#+}} xmm0 = mem[0],zero sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    addsd %xmm0, %xmm0 # sched: [3:1.00]
-; ZNVER1-SSE-NEXT:    movsd %xmm0, (%rsi) # sched: [1:0.50]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_movsd_mem:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero sched: [8:0.50]
-; ZNVER1-NEXT:    vaddsd %xmm0, %xmm0, %xmm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    vmovsd %xmm0, (%rsi) # sched: [1:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = load double, double* %a0, align 1
-  %2 = fadd double %1, %1
-  store double %2, double *%a1, align 1
-  ret void
-}
-
-define <2 x double> @test_movsd_reg(<2 x double> %a0, <2 x double> %a1) {
-; GENERIC-LABEL: test_movsd_reg:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    movlhps {{.*#+}} xmm1 = xmm1[0],xmm0[0] sched: [1:1.00]
-; GENERIC-NEXT:    movaps %xmm1, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_movsd_reg:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    movlhps {{.*#+}} xmm1 = xmm1[0],xmm0[0] sched: [1:1.00]
-; ATOM-NEXT:    movaps %xmm1, %xmm0 # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_movsd_reg:
-; SLM:       # %bb.0:
-; SLM-NEXT:    movlhps {{.*#+}} xmm1 = xmm1[0],xmm0[0] sched: [1:1.00]
-; SLM-NEXT:    movaps %xmm1, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_movsd_reg:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    movlhps {{.*#+}} xmm1 = xmm1[0],xmm0[0] sched: [1:1.00]
-; SANDY-SSE-NEXT:    movaps %xmm1, %xmm0 # sched: [1:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_movsd_reg:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm1[0],xmm0[0] sched: [1:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_movsd_reg:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    movlhps {{.*#+}} xmm1 = xmm1[0],xmm0[0] sched: [1:1.00]
-; HASWELL-SSE-NEXT:    movaps %xmm1, %xmm0 # sched: [1:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_movsd_reg:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm1[0],xmm0[0] sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_movsd_reg:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    movlhps {{.*#+}} xmm1 = xmm1[0],xmm0[0] sched: [1:1.00]
-; BROADWELL-SSE-NEXT:    movaps %xmm1, %xmm0 # sched: [1:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_movsd_reg:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm1[0],xmm0[0] sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_movsd_reg:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    movlhps {{.*#+}} xmm1 = xmm1[0],xmm0[0] sched: [1:1.00]
-; SKYLAKE-SSE-NEXT:    movaps %xmm1, %xmm0 # sched: [1:0.33]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_movsd_reg:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm1[0],xmm0[0] sched: [1:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_movsd_reg:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    movlhps {{.*#+}} xmm1 = xmm1[0],xmm0[0] sched: [1:1.00]
-; SKX-SSE-NEXT:    movaps %xmm1, %xmm0 # sched: [1:0.33]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_movsd_reg:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm1[0],xmm0[0] sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_movsd_reg:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    movlhps {{.*#+}} xmm1 = xmm1[0],xmm0[0] sched: [2:0.50]
-; BDVER2-SSE-NEXT:    movaps %xmm1, %xmm0 # sched: [1:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_movsd_reg:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm1[0],xmm0[0] sched: [2:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_movsd_reg:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    movlhps {{.*#+}} xmm1 = xmm1[0],xmm0[0] sched: [1:0.50]
-; BTVER2-SSE-NEXT:    movaps %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_movsd_reg:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm1[0],xmm0[0] sched: [1:0.50]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_movsd_reg:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    movlhps {{.*#+}} xmm1 = xmm1[0],xmm0[0] sched: [1:0.50]
-; ZNVER1-SSE-NEXT:    movaps %xmm1, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_movsd_reg:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm1[0],xmm0[0] sched: [1:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = shufflevector <2 x double> %a0, <2 x double> %a1, <2 x i32> <i32 2, i32 0>
-  ret <2 x double> %1
-}
-
-define void @test_movupd(<2 x double> *%a0, <2 x double> *%a1) {
-; GENERIC-LABEL: test_movupd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    movupd (%rdi), %xmm0 # sched: [6:0.50]
-; GENERIC-NEXT:    addpd %xmm0, %xmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    movupd %xmm0, (%rsi) # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_movupd:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    movupd (%rdi), %xmm0 # sched: [3:1.50]
-; ATOM-NEXT:    addpd %xmm0, %xmm0 # sched: [6:3.00]
-; ATOM-NEXT:    movupd %xmm0, (%rsi) # sched: [2:1.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_movupd:
-; SLM:       # %bb.0:
-; SLM-NEXT:    movupd (%rdi), %xmm0 # sched: [3:1.00]
-; SLM-NEXT:    addpd %xmm0, %xmm0 # sched: [3:1.00]
-; SLM-NEXT:    movupd %xmm0, (%rsi) # sched: [1:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_movupd:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    movupd (%rdi), %xmm0 # sched: [6:0.50]
-; SANDY-SSE-NEXT:    addpd %xmm0, %xmm0 # sched: [3:1.00]
-; SANDY-SSE-NEXT:    movupd %xmm0, (%rsi) # sched: [1:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_movupd:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vmovupd (%rdi), %xmm0 # sched: [6:0.50]
-; SANDY-NEXT:    vaddpd %xmm0, %xmm0, %xmm0 # sched: [3:1.00]
-; SANDY-NEXT:    vmovupd %xmm0, (%rsi) # sched: [1:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_movupd:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    movupd (%rdi), %xmm0 # sched: [6:0.50]
-; HASWELL-SSE-NEXT:    addpd %xmm0, %xmm0 # sched: [3:1.00]
-; HASWELL-SSE-NEXT:    movupd %xmm0, (%rsi) # sched: [1:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_movupd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vmovupd (%rdi), %xmm0 # sched: [6:0.50]
-; HASWELL-NEXT:    vaddpd %xmm0, %xmm0, %xmm0 # sched: [3:1.00]
-; HASWELL-NEXT:    vmovupd %xmm0, (%rsi) # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_movupd:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    movupd (%rdi), %xmm0 # sched: [5:0.50]
-; BROADWELL-SSE-NEXT:    addpd %xmm0, %xmm0 # sched: [3:1.00]
-; BROADWELL-SSE-NEXT:    movupd %xmm0, (%rsi) # sched: [1:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_movupd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vmovupd (%rdi), %xmm0 # sched: [5:0.50]
-; BROADWELL-NEXT:    vaddpd %xmm0, %xmm0, %xmm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    vmovupd %xmm0, (%rsi) # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_movupd:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    movupd (%rdi), %xmm0 # sched: [6:0.50]
-; SKYLAKE-SSE-NEXT:    addpd %xmm0, %xmm0 # sched: [4:0.50]
-; SKYLAKE-SSE-NEXT:    movupd %xmm0, (%rsi) # sched: [1:1.00]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_movupd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vmovupd (%rdi), %xmm0 # sched: [6:0.50]
-; SKYLAKE-NEXT:    vaddpd %xmm0, %xmm0, %xmm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    vmovupd %xmm0, (%rsi) # sched: [1:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_movupd:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    movupd (%rdi), %xmm0 # sched: [6:0.50]
-; SKX-SSE-NEXT:    addpd %xmm0, %xmm0 # sched: [4:0.50]
-; SKX-SSE-NEXT:    movupd %xmm0, (%rsi) # sched: [1:1.00]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_movupd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovupd (%rdi), %xmm0 # sched: [6:0.50]
-; SKX-NEXT:    vaddpd %xmm0, %xmm0, %xmm0 # sched: [4:0.50]
-; SKX-NEXT:    vmovupd %xmm0, (%rsi) # sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_movupd:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    movupd (%rdi), %xmm0 # sched: [5:0.50]
-; BDVER2-SSE-NEXT:    addpd %xmm0, %xmm0 # sched: [5:1.00]
-; BDVER2-SSE-NEXT:    movupd %xmm0, (%rsi) # sched: [1:1.00]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_movupd:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vmovupd (%rdi), %xmm0 # sched: [5:0.50]
-; BDVER2-NEXT:    vaddpd %xmm0, %xmm0, %xmm0 # sched: [5:1.00]
-; BDVER2-NEXT:    vmovupd %xmm0, (%rsi) # sched: [1:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_movupd:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    movupd (%rdi), %xmm0 # sched: [5:1.00]
-; BTVER2-SSE-NEXT:    addpd %xmm0, %xmm0 # sched: [3:1.00]
-; BTVER2-SSE-NEXT:    movupd %xmm0, (%rsi) # sched: [1:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_movupd:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vmovupd (%rdi), %xmm0 # sched: [5:1.00]
-; BTVER2-NEXT:    vaddpd %xmm0, %xmm0, %xmm0 # sched: [3:1.00]
-; BTVER2-NEXT:    vmovupd %xmm0, (%rsi) # sched: [1:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_movupd:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    movupd (%rdi), %xmm0 # sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    addpd %xmm0, %xmm0 # sched: [3:1.00]
-; ZNVER1-SSE-NEXT:    movupd %xmm0, (%rsi) # sched: [1:0.50]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_movupd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vmovupd (%rdi), %xmm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    vaddpd %xmm0, %xmm0, %xmm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    vmovupd %xmm0, (%rsi) # sched: [1:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = load <2 x double>, <2 x double> *%a0, align 1
-  %2 = fadd <2 x double> %1, %1
-  store <2 x double> %2, <2 x double> *%a1, align 1
-  ret void
-}
-
-define <2 x double> @test_mulpd(<2 x double> %a0, <2 x double> %a1, <2 x double> *%a2) {
-; GENERIC-LABEL: test_mulpd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    mulpd %xmm1, %xmm0 # sched: [5:1.00]
-; GENERIC-NEXT:    mulpd (%rdi), %xmm0 # sched: [11:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_mulpd:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    mulpd %xmm1, %xmm0 # sched: [9:4.50]
-; ATOM-NEXT:    mulpd (%rdi), %xmm0 # sched: [10:5.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_mulpd:
-; SLM:       # %bb.0:
-; SLM-NEXT:    mulpd %xmm1, %xmm0 # sched: [5:2.00]
-; SLM-NEXT:    mulpd (%rdi), %xmm0 # sched: [8:2.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_mulpd:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    mulpd %xmm1, %xmm0 # sched: [5:1.00]
-; SANDY-SSE-NEXT:    mulpd (%rdi), %xmm0 # sched: [11:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_mulpd:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vmulpd %xmm1, %xmm0, %xmm0 # sched: [5:1.00]
-; SANDY-NEXT:    vmulpd (%rdi), %xmm0, %xmm0 # sched: [11:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_mulpd:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    mulpd %xmm1, %xmm0 # sched: [5:0.50]
-; HASWELL-SSE-NEXT:    mulpd (%rdi), %xmm0 # sched: [11:0.50]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_mulpd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vmulpd %xmm1, %xmm0, %xmm0 # sched: [5:0.50]
-; HASWELL-NEXT:    vmulpd (%rdi), %xmm0, %xmm0 # sched: [11:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_mulpd:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    mulpd %xmm1, %xmm0 # sched: [3:0.50]
-; BROADWELL-SSE-NEXT:    mulpd (%rdi), %xmm0 # sched: [8:0.50]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_mulpd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vmulpd %xmm1, %xmm0, %xmm0 # sched: [3:0.50]
-; BROADWELL-NEXT:    vmulpd (%rdi), %xmm0, %xmm0 # sched: [8:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_mulpd:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    mulpd %xmm1, %xmm0 # sched: [4:0.50]
-; SKYLAKE-SSE-NEXT:    mulpd (%rdi), %xmm0 # sched: [10:0.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_mulpd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vmulpd %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    vmulpd (%rdi), %xmm0, %xmm0 # sched: [10:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_mulpd:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    mulpd %xmm1, %xmm0 # sched: [4:0.50]
-; SKX-SSE-NEXT:    mulpd (%rdi), %xmm0 # sched: [10:0.50]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_mulpd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmulpd %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKX-NEXT:    vmulpd (%rdi), %xmm0, %xmm0 # sched: [10:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_mulpd:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    mulpd %xmm1, %xmm0 # sched: [5:1.00]
-; BDVER2-SSE-NEXT:    mulpd (%rdi), %xmm0 # sched: [10:1.00]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_mulpd:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vmulpd %xmm1, %xmm0, %xmm0 # sched: [5:1.00]
-; BDVER2-NEXT:    vmulpd (%rdi), %xmm0, %xmm0 # sched: [10:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_mulpd:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    mulpd %xmm1, %xmm0 # sched: [4:2.00]
-; BTVER2-SSE-NEXT:    mulpd (%rdi), %xmm0 # sched: [9:2.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_mulpd:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vmulpd %xmm1, %xmm0, %xmm0 # sched: [4:2.00]
-; BTVER2-NEXT:    vmulpd (%rdi), %xmm0, %xmm0 # sched: [9:2.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_mulpd:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    mulpd %xmm1, %xmm0 # sched: [3:0.50]
-; ZNVER1-SSE-NEXT:    mulpd (%rdi), %xmm0 # sched: [10:0.50]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_mulpd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vmulpd %xmm1, %xmm0, %xmm0 # sched: [3:0.50]
-; ZNVER1-NEXT:    vmulpd (%rdi), %xmm0, %xmm0 # sched: [10:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = fmul <2 x double> %a0, %a1
-  %2 = load <2 x double>, <2 x double> *%a2, align 16
-  %3 = fmul <2 x double> %1, %2
-  ret <2 x double> %3
-}
-
-define double @test_mulsd(double %a0, double %a1, double *%a2) {
-; GENERIC-LABEL: test_mulsd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    mulsd %xmm1, %xmm0 # sched: [5:1.00]
-; GENERIC-NEXT:    mulsd (%rdi), %xmm0 # sched: [11:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_mulsd:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    mulsd %xmm1, %xmm0 # sched: [5:5.00]
-; ATOM-NEXT:    mulsd (%rdi), %xmm0 # sched: [5:5.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_mulsd:
-; SLM:       # %bb.0:
-; SLM-NEXT:    mulsd %xmm1, %xmm0 # sched: [5:2.00]
-; SLM-NEXT:    mulsd (%rdi), %xmm0 # sched: [8:2.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_mulsd:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    mulsd %xmm1, %xmm0 # sched: [5:1.00]
-; SANDY-SSE-NEXT:    mulsd (%rdi), %xmm0 # sched: [11:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_mulsd:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vmulsd %xmm1, %xmm0, %xmm0 # sched: [5:1.00]
-; SANDY-NEXT:    vmulsd (%rdi), %xmm0, %xmm0 # sched: [11:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_mulsd:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    mulsd %xmm1, %xmm0 # sched: [5:0.50]
-; HASWELL-SSE-NEXT:    mulsd (%rdi), %xmm0 # sched: [10:0.50]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_mulsd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vmulsd %xmm1, %xmm0, %xmm0 # sched: [5:0.50]
-; HASWELL-NEXT:    vmulsd (%rdi), %xmm0, %xmm0 # sched: [10:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_mulsd:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    mulsd %xmm1, %xmm0 # sched: [3:0.50]
-; BROADWELL-SSE-NEXT:    mulsd (%rdi), %xmm0 # sched: [8:0.50]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_mulsd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vmulsd %xmm1, %xmm0, %xmm0 # sched: [3:0.50]
-; BROADWELL-NEXT:    vmulsd (%rdi), %xmm0, %xmm0 # sched: [8:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_mulsd:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    mulsd %xmm1, %xmm0 # sched: [4:0.50]
-; SKYLAKE-SSE-NEXT:    mulsd (%rdi), %xmm0 # sched: [9:0.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_mulsd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vmulsd %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    vmulsd (%rdi), %xmm0, %xmm0 # sched: [9:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_mulsd:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    mulsd %xmm1, %xmm0 # sched: [4:0.50]
-; SKX-SSE-NEXT:    mulsd (%rdi), %xmm0 # sched: [9:0.50]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_mulsd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmulsd %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKX-NEXT:    vmulsd (%rdi), %xmm0, %xmm0 # sched: [9:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_mulsd:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    mulsd %xmm1, %xmm0 # sched: [5:1.00]
-; BDVER2-SSE-NEXT:    mulsd (%rdi), %xmm0 # sched: [10:1.00]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_mulsd:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vmulsd %xmm1, %xmm0, %xmm0 # sched: [5:1.00]
-; BDVER2-NEXT:    vmulsd (%rdi), %xmm0, %xmm0 # sched: [10:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_mulsd:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    mulsd %xmm1, %xmm0 # sched: [4:2.00]
-; BTVER2-SSE-NEXT:    mulsd (%rdi), %xmm0 # sched: [9:2.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_mulsd:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vmulsd %xmm1, %xmm0, %xmm0 # sched: [4:2.00]
-; BTVER2-NEXT:    vmulsd (%rdi), %xmm0, %xmm0 # sched: [9:2.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_mulsd:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    mulsd %xmm1, %xmm0 # sched: [3:0.50]
-; ZNVER1-SSE-NEXT:    mulsd (%rdi), %xmm0 # sched: [10:0.50]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_mulsd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vmulsd %xmm1, %xmm0, %xmm0 # sched: [3:0.50]
-; ZNVER1-NEXT:    vmulsd (%rdi), %xmm0, %xmm0 # sched: [10:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = fmul double %a0, %a1
-  %2 = load double, double *%a2, align 8
-  %3 = fmul double %1, %2
-  ret double %3
-}
-
-define <2 x double> @test_orpd(<2 x double> %a0, <2 x double> %a1, <2 x double> *%a2) {
-; GENERIC-LABEL: test_orpd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    orpd %xmm1, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    orpd (%rdi), %xmm0 # sched: [7:1.00]
-; GENERIC-NEXT:    addpd %xmm1, %xmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_orpd:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    orpd %xmm1, %xmm0 # sched: [1:0.50]
-; ATOM-NEXT:    orpd (%rdi), %xmm0 # sched: [1:1.00]
-; ATOM-NEXT:    addpd %xmm1, %xmm0 # sched: [6:3.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_orpd:
-; SLM:       # %bb.0:
-; SLM-NEXT:    orpd %xmm1, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    orpd (%rdi), %xmm0 # sched: [4:1.00]
-; SLM-NEXT:    addpd %xmm1, %xmm0 # sched: [3:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_orpd:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    orpd %xmm1, %xmm0 # sched: [1:1.00]
-; SANDY-SSE-NEXT:    orpd (%rdi), %xmm0 # sched: [7:1.00]
-; SANDY-SSE-NEXT:    addpd %xmm1, %xmm0 # sched: [3:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_orpd:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vorpd %xmm1, %xmm0, %xmm0 # sched: [1:1.00]
-; SANDY-NEXT:    vorpd (%rdi), %xmm0, %xmm0 # sched: [7:1.00]
-; SANDY-NEXT:    vaddpd %xmm0, %xmm1, %xmm0 # sched: [3:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_orpd:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    orpd %xmm1, %xmm0 # sched: [1:1.00]
-; HASWELL-SSE-NEXT:    orpd (%rdi), %xmm0 # sched: [7:1.00]
-; HASWELL-SSE-NEXT:    addpd %xmm1, %xmm0 # sched: [3:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_orpd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vorpd %xmm1, %xmm0, %xmm0 # sched: [1:1.00]
-; HASWELL-NEXT:    vorpd (%rdi), %xmm0, %xmm0 # sched: [7:1.00]
-; HASWELL-NEXT:    vaddpd %xmm0, %xmm1, %xmm0 # sched: [3:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_orpd:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    orpd %xmm1, %xmm0 # sched: [1:1.00]
-; BROADWELL-SSE-NEXT:    orpd (%rdi), %xmm0 # sched: [6:1.00]
-; BROADWELL-SSE-NEXT:    addpd %xmm1, %xmm0 # sched: [3:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_orpd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vorpd %xmm1, %xmm0, %xmm0 # sched: [1:1.00]
-; BROADWELL-NEXT:    vorpd (%rdi), %xmm0, %xmm0 # sched: [6:1.00]
-; BROADWELL-NEXT:    vaddpd %xmm0, %xmm1, %xmm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_orpd:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    orpd %xmm1, %xmm0 # sched: [1:0.33]
-; SKYLAKE-SSE-NEXT:    orpd (%rdi), %xmm0 # sched: [7:0.50]
-; SKYLAKE-SSE-NEXT:    addpd %xmm1, %xmm0 # sched: [4:0.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_orpd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vorpd %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    vorpd (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKYLAKE-NEXT:    vaddpd %xmm0, %xmm1, %xmm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_orpd:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    orpd %xmm1, %xmm0 # sched: [1:0.33]
-; SKX-SSE-NEXT:    orpd (%rdi), %xmm0 # sched: [7:0.50]
-; SKX-SSE-NEXT:    addpd %xmm1, %xmm0 # sched: [4:0.50]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_orpd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vorpd %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    vorpd (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKX-NEXT:    vaddpd %xmm0, %xmm1, %xmm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_orpd:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    orpd %xmm1, %xmm0 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    orpd (%rdi), %xmm0 # sched: [7:0.50]
-; BDVER2-SSE-NEXT:    addpd %xmm1, %xmm0 # sched: [5:1.00]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_orpd:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vorpd %xmm1, %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    vorpd (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; BDVER2-NEXT:    vaddpd %xmm0, %xmm1, %xmm0 # sched: [5:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_orpd:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    orpd %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    orpd (%rdi), %xmm0 # sched: [6:1.00]
-; BTVER2-SSE-NEXT:    addpd %xmm1, %xmm0 # sched: [3:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_orpd:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vorpd %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    vorpd (%rdi), %xmm0, %xmm0 # sched: [6:1.00]
-; BTVER2-NEXT:    vaddpd %xmm0, %xmm1, %xmm0 # sched: [3:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_orpd:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    orpd %xmm1, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    orpd (%rdi), %xmm0 # sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    addpd %xmm1, %xmm0 # sched: [3:1.00]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_orpd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vorpd %xmm1, %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vorpd (%rdi), %xmm0, %xmm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    vaddpd %xmm0, %xmm1, %xmm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = bitcast <2 x double> %a0 to <4 x i32>
-  %2 = bitcast <2 x double> %a1 to <4 x i32>
-  %3 = or <4 x i32> %1, %2
-  %4 = load <2 x double>, <2 x double> *%a2, align 16
-  %5 = bitcast <2 x double> %4 to <4 x i32>
-  %6 = or <4 x i32> %3, %5
-  %7 = bitcast <4 x i32> %6 to <2 x double>
-  %8 = fadd <2 x double> %a1, %7
-  ret <2 x double> %8
-}
-
-define <8 x i16> @test_packssdw(<4 x i32> %a0, <4 x i32> %a1, <4 x i32> *%a2) {
-; GENERIC-LABEL: test_packssdw:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    packssdw %xmm1, %xmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    packssdw (%rdi), %xmm0 # sched: [7:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_packssdw:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    packssdw %xmm1, %xmm0 # sched: [1:1.00]
-; ATOM-NEXT:    packssdw (%rdi), %xmm0 # sched: [1:1.00]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_packssdw:
-; SLM:       # %bb.0:
-; SLM-NEXT:    packssdw %xmm1, %xmm0 # sched: [1:1.00]
-; SLM-NEXT:    packssdw (%rdi), %xmm0 # sched: [4:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_packssdw:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    packssdw %xmm1, %xmm0 # sched: [1:0.50]
-; SANDY-SSE-NEXT:    packssdw (%rdi), %xmm0 # sched: [7:0.50]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_packssdw:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vpackssdw %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SANDY-NEXT:    vpackssdw (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_packssdw:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    packssdw %xmm1, %xmm0 # sched: [1:1.00]
-; HASWELL-SSE-NEXT:    packssdw (%rdi), %xmm0 # sched: [7:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_packssdw:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpackssdw %xmm1, %xmm0, %xmm0 # sched: [1:1.00]
-; HASWELL-NEXT:    vpackssdw (%rdi), %xmm0, %xmm0 # sched: [7:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_packssdw:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    packssdw %xmm1, %xmm0 # sched: [1:1.00]
-; BROADWELL-SSE-NEXT:    packssdw (%rdi), %xmm0 # sched: [6:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_packssdw:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpackssdw %xmm1, %xmm0, %xmm0 # sched: [1:1.00]
-; BROADWELL-NEXT:    vpackssdw (%rdi), %xmm0, %xmm0 # sched: [6:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_packssdw:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    packssdw %xmm1, %xmm0 # sched: [1:1.00]
-; SKYLAKE-SSE-NEXT:    packssdw (%rdi), %xmm0 # sched: [7:1.00]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_packssdw:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpackssdw %xmm1, %xmm0, %xmm0 # sched: [1:1.00]
-; SKYLAKE-NEXT:    vpackssdw (%rdi), %xmm0, %xmm0 # sched: [7:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_packssdw:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    packssdw %xmm1, %xmm0 # sched: [1:1.00]
-; SKX-SSE-NEXT:    packssdw (%rdi), %xmm0 # sched: [7:1.00]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_packssdw:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpackssdw %xmm1, %xmm0, %xmm0 # sched: [1:1.00]
-; SKX-NEXT:    vpackssdw (%rdi), %xmm0, %xmm0 # sched: [7:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_packssdw:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    packssdw %xmm1, %xmm0 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    packssdw (%rdi), %xmm0 # sched: [7:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_packssdw:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vpackssdw %xmm1, %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    vpackssdw (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_packssdw:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    packssdw %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    packssdw (%rdi), %xmm0 # sched: [6:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_packssdw:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vpackssdw %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    vpackssdw (%rdi), %xmm0, %xmm0 # sched: [6:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_packssdw:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    packssdw %xmm1, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    packssdw (%rdi), %xmm0 # sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_packssdw:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpackssdw %xmm1, %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vpackssdw (%rdi), %xmm0, %xmm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <8 x i16> @llvm.x86.sse2.packssdw.128(<4 x i32> %a0, <4 x i32> %a1)
-  %2 = bitcast <8 x i16> %1 to <4 x i32>
-  %3 = load <4 x i32>, <4 x i32> *%a2, align 16
-  %4 = call <8 x i16> @llvm.x86.sse2.packssdw.128(<4 x i32> %2, <4 x i32> %3)
-  ret <8 x i16> %4
-}
-declare <8 x i16> @llvm.x86.sse2.packssdw.128(<4 x i32>, <4 x i32>) nounwind readnone
-
-define <16 x i8> @test_packsswb(<8 x i16> %a0, <8 x i16> %a1, <8 x i16> *%a2) {
-; GENERIC-LABEL: test_packsswb:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    packsswb %xmm1, %xmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    packsswb (%rdi), %xmm0 # sched: [7:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_packsswb:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    packsswb %xmm1, %xmm0 # sched: [1:1.00]
-; ATOM-NEXT:    packsswb (%rdi), %xmm0 # sched: [1:1.00]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_packsswb:
-; SLM:       # %bb.0:
-; SLM-NEXT:    packsswb %xmm1, %xmm0 # sched: [1:1.00]
-; SLM-NEXT:    packsswb (%rdi), %xmm0 # sched: [4:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_packsswb:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    packsswb %xmm1, %xmm0 # sched: [1:0.50]
-; SANDY-SSE-NEXT:    packsswb (%rdi), %xmm0 # sched: [7:0.50]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_packsswb:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vpacksswb %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SANDY-NEXT:    vpacksswb (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_packsswb:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    packsswb %xmm1, %xmm0 # sched: [1:1.00]
-; HASWELL-SSE-NEXT:    packsswb (%rdi), %xmm0 # sched: [7:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_packsswb:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpacksswb %xmm1, %xmm0, %xmm0 # sched: [1:1.00]
-; HASWELL-NEXT:    vpacksswb (%rdi), %xmm0, %xmm0 # sched: [7:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_packsswb:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    packsswb %xmm1, %xmm0 # sched: [1:1.00]
-; BROADWELL-SSE-NEXT:    packsswb (%rdi), %xmm0 # sched: [6:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_packsswb:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpacksswb %xmm1, %xmm0, %xmm0 # sched: [1:1.00]
-; BROADWELL-NEXT:    vpacksswb (%rdi), %xmm0, %xmm0 # sched: [6:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_packsswb:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    packsswb %xmm1, %xmm0 # sched: [1:1.00]
-; SKYLAKE-SSE-NEXT:    packsswb (%rdi), %xmm0 # sched: [7:1.00]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_packsswb:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpacksswb %xmm1, %xmm0, %xmm0 # sched: [1:1.00]
-; SKYLAKE-NEXT:    vpacksswb (%rdi), %xmm0, %xmm0 # sched: [7:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_packsswb:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    packsswb %xmm1, %xmm0 # sched: [1:1.00]
-; SKX-SSE-NEXT:    packsswb (%rdi), %xmm0 # sched: [7:1.00]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_packsswb:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpacksswb %xmm1, %xmm0, %xmm0 # sched: [1:1.00]
-; SKX-NEXT:    vpacksswb (%rdi), %xmm0, %xmm0 # sched: [7:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_packsswb:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    packsswb %xmm1, %xmm0 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    packsswb (%rdi), %xmm0 # sched: [7:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_packsswb:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vpacksswb %xmm1, %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    vpacksswb (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_packsswb:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    packsswb %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    packsswb (%rdi), %xmm0 # sched: [6:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_packsswb:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vpacksswb %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    vpacksswb (%rdi), %xmm0, %xmm0 # sched: [6:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_packsswb:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    packsswb %xmm1, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    packsswb (%rdi), %xmm0 # sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_packsswb:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpacksswb %xmm1, %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vpacksswb (%rdi), %xmm0, %xmm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <16 x i8> @llvm.x86.sse2.packsswb.128(<8 x i16> %a0, <8 x i16> %a1)
-  %2 = bitcast <16 x i8> %1 to <8 x i16>
-  %3 = load <8 x i16>, <8 x i16> *%a2, align 16
-  %4 = call <16 x i8> @llvm.x86.sse2.packsswb.128(<8 x i16> %2, <8 x i16> %3)
-  ret <16 x i8> %4
-}
-declare <16 x i8> @llvm.x86.sse2.packsswb.128(<8 x i16>, <8 x i16>) nounwind readnone
-
-define <16 x i8> @test_packuswb(<8 x i16> %a0, <8 x i16> %a1, <8 x i16> *%a2) {
-; GENERIC-LABEL: test_packuswb:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    packuswb %xmm1, %xmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    packuswb (%rdi), %xmm0 # sched: [7:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_packuswb:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    packuswb %xmm1, %xmm0 # sched: [1:1.00]
-; ATOM-NEXT:    packuswb (%rdi), %xmm0 # sched: [1:1.00]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_packuswb:
-; SLM:       # %bb.0:
-; SLM-NEXT:    packuswb %xmm1, %xmm0 # sched: [1:1.00]
-; SLM-NEXT:    packuswb (%rdi), %xmm0 # sched: [4:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_packuswb:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    packuswb %xmm1, %xmm0 # sched: [1:0.50]
-; SANDY-SSE-NEXT:    packuswb (%rdi), %xmm0 # sched: [7:0.50]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_packuswb:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SANDY-NEXT:    vpackuswb (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_packuswb:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    packuswb %xmm1, %xmm0 # sched: [1:1.00]
-; HASWELL-SSE-NEXT:    packuswb (%rdi), %xmm0 # sched: [7:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_packuswb:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0 # sched: [1:1.00]
-; HASWELL-NEXT:    vpackuswb (%rdi), %xmm0, %xmm0 # sched: [7:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_packuswb:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    packuswb %xmm1, %xmm0 # sched: [1:1.00]
-; BROADWELL-SSE-NEXT:    packuswb (%rdi), %xmm0 # sched: [6:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_packuswb:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0 # sched: [1:1.00]
-; BROADWELL-NEXT:    vpackuswb (%rdi), %xmm0, %xmm0 # sched: [6:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_packuswb:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    packuswb %xmm1, %xmm0 # sched: [1:1.00]
-; SKYLAKE-SSE-NEXT:    packuswb (%rdi), %xmm0 # sched: [7:1.00]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_packuswb:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0 # sched: [1:1.00]
-; SKYLAKE-NEXT:    vpackuswb (%rdi), %xmm0, %xmm0 # sched: [7:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_packuswb:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    packuswb %xmm1, %xmm0 # sched: [1:1.00]
-; SKX-SSE-NEXT:    packuswb (%rdi), %xmm0 # sched: [7:1.00]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_packuswb:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0 # sched: [1:1.00]
-; SKX-NEXT:    vpackuswb (%rdi), %xmm0, %xmm0 # sched: [7:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_packuswb:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    packuswb %xmm1, %xmm0 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    packuswb (%rdi), %xmm0 # sched: [7:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_packuswb:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    vpackuswb (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_packuswb:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    packuswb %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    packuswb (%rdi), %xmm0 # sched: [6:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_packuswb:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    vpackuswb (%rdi), %xmm0, %xmm0 # sched: [6:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_packuswb:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    packuswb %xmm1, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    packuswb (%rdi), %xmm0 # sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_packuswb:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vpackuswb (%rdi), %xmm0, %xmm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <16 x i8> @llvm.x86.sse2.packuswb.128(<8 x i16> %a0, <8 x i16> %a1)
-  %2 = bitcast <16 x i8> %1 to <8 x i16>
-  %3 = load <8 x i16>, <8 x i16> *%a2, align 16
-  %4 = call <16 x i8> @llvm.x86.sse2.packuswb.128(<8 x i16> %2, <8 x i16> %3)
-  ret <16 x i8> %4
-}
-declare <16 x i8> @llvm.x86.sse2.packuswb.128(<8 x i16>, <8 x i16>) nounwind readnone
-
-define <16 x i8> @test_paddb(<16 x i8> %a0, <16 x i8> %a1, <16 x i8> *%a2) {
-; GENERIC-LABEL: test_paddb:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    paddb %xmm1, %xmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    paddb (%rdi), %xmm0 # sched: [7:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_paddb:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    paddb %xmm1, %xmm0 # sched: [1:0.50]
-; ATOM-NEXT:    paddb (%rdi), %xmm0 # sched: [1:1.00]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_paddb:
-; SLM:       # %bb.0:
-; SLM-NEXT:    paddb %xmm1, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    paddb (%rdi), %xmm0 # sched: [4:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_paddb:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    paddb %xmm1, %xmm0 # sched: [1:0.50]
-; SANDY-SSE-NEXT:    paddb (%rdi), %xmm0 # sched: [7:0.50]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_paddb:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vpaddb %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SANDY-NEXT:    vpaddb (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_paddb:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    paddb %xmm1, %xmm0 # sched: [1:0.50]
-; HASWELL-SSE-NEXT:    paddb (%rdi), %xmm0 # sched: [7:0.50]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_paddb:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpaddb %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; HASWELL-NEXT:    vpaddb (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_paddb:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    paddb %xmm1, %xmm0 # sched: [1:0.50]
-; BROADWELL-SSE-NEXT:    paddb (%rdi), %xmm0 # sched: [6:0.50]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_paddb:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpaddb %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    vpaddb (%rdi), %xmm0, %xmm0 # sched: [6:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_paddb:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    paddb %xmm1, %xmm0 # sched: [1:0.33]
-; SKYLAKE-SSE-NEXT:    paddb (%rdi), %xmm0 # sched: [7:0.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_paddb:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpaddb %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    vpaddb (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_paddb:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    paddb %xmm1, %xmm0 # sched: [1:0.33]
-; SKX-SSE-NEXT:    paddb (%rdi), %xmm0 # sched: [7:0.50]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_paddb:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpaddb %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    vpaddb (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_paddb:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    paddb %xmm1, %xmm0 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    paddb (%rdi), %xmm0 # sched: [7:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_paddb:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vpaddb %xmm1, %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    vpaddb (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_paddb:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    paddb %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    paddb (%rdi), %xmm0 # sched: [6:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_paddb:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vpaddb %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    vpaddb (%rdi), %xmm0, %xmm0 # sched: [6:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_paddb:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    paddb %xmm1, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    paddb (%rdi), %xmm0 # sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_paddb:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpaddb %xmm1, %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vpaddb (%rdi), %xmm0, %xmm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = add <16 x i8> %a0, %a1
-  %2 = load <16 x i8>, <16 x i8> *%a2, align 16
-  %3 = add <16 x i8> %1, %2
-  ret <16 x i8> %3
-}
-
-define <4 x i32> @test_paddd(<4 x i32> %a0, <4 x i32> %a1, <4 x i32> *%a2) {
-; GENERIC-LABEL: test_paddd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    paddd %xmm1, %xmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    paddd (%rdi), %xmm0 # sched: [7:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_paddd:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    paddd %xmm1, %xmm0 # sched: [1:0.50]
-; ATOM-NEXT:    paddd (%rdi), %xmm0 # sched: [1:1.00]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_paddd:
-; SLM:       # %bb.0:
-; SLM-NEXT:    paddd %xmm1, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    paddd (%rdi), %xmm0 # sched: [4:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_paddd:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    paddd %xmm1, %xmm0 # sched: [1:0.50]
-; SANDY-SSE-NEXT:    paddd (%rdi), %xmm0 # sched: [7:0.50]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_paddd:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SANDY-NEXT:    vpaddd (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_paddd:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    paddd %xmm1, %xmm0 # sched: [1:0.50]
-; HASWELL-SSE-NEXT:    paddd (%rdi), %xmm0 # sched: [7:0.50]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_paddd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; HASWELL-NEXT:    vpaddd (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_paddd:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    paddd %xmm1, %xmm0 # sched: [1:0.50]
-; BROADWELL-SSE-NEXT:    paddd (%rdi), %xmm0 # sched: [6:0.50]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_paddd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    vpaddd (%rdi), %xmm0, %xmm0 # sched: [6:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_paddd:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    paddd %xmm1, %xmm0 # sched: [1:0.33]
-; SKYLAKE-SSE-NEXT:    paddd (%rdi), %xmm0 # sched: [7:0.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_paddd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    vpaddd (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_paddd:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    paddd %xmm1, %xmm0 # sched: [1:0.33]
-; SKX-SSE-NEXT:    paddd (%rdi), %xmm0 # sched: [7:0.50]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_paddd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    vpaddd (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_paddd:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    paddd %xmm1, %xmm0 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    paddd (%rdi), %xmm0 # sched: [7:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_paddd:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    vpaddd (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_paddd:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    paddd %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    paddd (%rdi), %xmm0 # sched: [6:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_paddd:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    vpaddd (%rdi), %xmm0, %xmm0 # sched: [6:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_paddd:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    paddd %xmm1, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    paddd (%rdi), %xmm0 # sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_paddd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vpaddd (%rdi), %xmm0, %xmm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = add <4 x i32> %a0, %a1
-  %2 = load <4 x i32>, <4 x i32> *%a2, align 16
-  %3 = add <4 x i32> %1, %2
-  ret <4 x i32> %3
-}
-
-define <2 x i64> @test_paddq(<2 x i64> %a0, <2 x i64> %a1, <2 x i64> *%a2) {
-; GENERIC-LABEL: test_paddq:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    paddq (%rdi), %xmm0 # sched: [7:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_paddq:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    paddq %xmm1, %xmm0 # sched: [2:1.00]
-; ATOM-NEXT:    paddq (%rdi), %xmm0 # sched: [3:1.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_paddq:
-; SLM:       # %bb.0:
-; SLM-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    paddq (%rdi), %xmm0 # sched: [4:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_paddq:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.50]
-; SANDY-SSE-NEXT:    paddq (%rdi), %xmm0 # sched: [7:0.50]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_paddq:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SANDY-NEXT:    vpaddq (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_paddq:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.50]
-; HASWELL-SSE-NEXT:    paddq (%rdi), %xmm0 # sched: [7:0.50]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_paddq:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; HASWELL-NEXT:    vpaddq (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_paddq:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.50]
-; BROADWELL-SSE-NEXT:    paddq (%rdi), %xmm0 # sched: [6:0.50]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_paddq:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    vpaddq (%rdi), %xmm0, %xmm0 # sched: [6:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_paddq:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.33]
-; SKYLAKE-SSE-NEXT:    paddq (%rdi), %xmm0 # sched: [7:0.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_paddq:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    vpaddq (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_paddq:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.33]
-; SKX-SSE-NEXT:    paddq (%rdi), %xmm0 # sched: [7:0.50]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_paddq:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    vpaddq (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_paddq:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    paddq %xmm1, %xmm0 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    paddq (%rdi), %xmm0 # sched: [7:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_paddq:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    vpaddq (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_paddq:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    paddq (%rdi), %xmm0 # sched: [6:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_paddq:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    vpaddq (%rdi), %xmm0, %xmm0 # sched: [6:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_paddq:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    paddq (%rdi), %xmm0 # sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_paddq:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vpaddq (%rdi), %xmm0, %xmm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = add <2 x i64> %a0, %a1
-  %2 = load <2 x i64>, <2 x i64> *%a2, align 16
-  %3 = add <2 x i64> %1, %2
-  ret <2 x i64> %3
-}
-
-define <16 x i8> @test_paddsb(<16 x i8> %a0, <16 x i8> %a1, <16 x i8> *%a2) {
-; GENERIC-LABEL: test_paddsb:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    paddsb %xmm1, %xmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    paddsb (%rdi), %xmm0 # sched: [7:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_paddsb:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    paddsb %xmm1, %xmm0 # sched: [1:0.50]
-; ATOM-NEXT:    paddsb (%rdi), %xmm0 # sched: [1:1.00]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_paddsb:
-; SLM:       # %bb.0:
-; SLM-NEXT:    paddsb %xmm1, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    paddsb (%rdi), %xmm0 # sched: [4:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_paddsb:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    paddsb %xmm1, %xmm0 # sched: [1:0.50]
-; SANDY-SSE-NEXT:    paddsb (%rdi), %xmm0 # sched: [7:0.50]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_paddsb:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vpaddsb %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SANDY-NEXT:    vpaddsb (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_paddsb:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    paddsb %xmm1, %xmm0 # sched: [1:0.50]
-; HASWELL-SSE-NEXT:    paddsb (%rdi), %xmm0 # sched: [7:0.50]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_paddsb:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpaddsb %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; HASWELL-NEXT:    vpaddsb (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_paddsb:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    paddsb %xmm1, %xmm0 # sched: [1:0.50]
-; BROADWELL-SSE-NEXT:    paddsb (%rdi), %xmm0 # sched: [6:0.50]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_paddsb:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpaddsb %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    vpaddsb (%rdi), %xmm0, %xmm0 # sched: [6:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_paddsb:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    paddsb %xmm1, %xmm0 # sched: [1:0.50]
-; SKYLAKE-SSE-NEXT:    paddsb (%rdi), %xmm0 # sched: [7:0.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_paddsb:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpaddsb %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SKYLAKE-NEXT:    vpaddsb (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_paddsb:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    paddsb %xmm1, %xmm0 # sched: [1:0.50]
-; SKX-SSE-NEXT:    paddsb (%rdi), %xmm0 # sched: [7:0.50]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_paddsb:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpaddsb %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SKX-NEXT:    vpaddsb (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_paddsb:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    paddsb %xmm1, %xmm0 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    paddsb (%rdi), %xmm0 # sched: [7:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_paddsb:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vpaddsb %xmm1, %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    vpaddsb (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_paddsb:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    paddsb %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    paddsb (%rdi), %xmm0 # sched: [6:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_paddsb:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vpaddsb %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    vpaddsb (%rdi), %xmm0, %xmm0 # sched: [6:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_paddsb:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    paddsb %xmm1, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    paddsb (%rdi), %xmm0 # sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_paddsb:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpaddsb %xmm1, %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vpaddsb (%rdi), %xmm0, %xmm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <16 x i8> @llvm.x86.sse2.padds.b(<16 x i8> %a0, <16 x i8> %a1)
-  %2 = load <16 x i8>, <16 x i8> *%a2, align 16
-  %3 = call <16 x i8> @llvm.x86.sse2.padds.b(<16 x i8> %1, <16 x i8> %2)
-  ret <16 x i8> %3
-}
-declare <16 x i8> @llvm.x86.sse2.padds.b(<16 x i8>, <16 x i8>) nounwind readnone
-
-define <8 x i16> @test_paddsw(<8 x i16> %a0, <8 x i16> %a1, <8 x i16> *%a2) {
-; GENERIC-LABEL: test_paddsw:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    paddsw %xmm1, %xmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    paddsw (%rdi), %xmm0 # sched: [7:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_paddsw:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    paddsw %xmm1, %xmm0 # sched: [1:0.50]
-; ATOM-NEXT:    paddsw (%rdi), %xmm0 # sched: [1:1.00]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_paddsw:
-; SLM:       # %bb.0:
-; SLM-NEXT:    paddsw %xmm1, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    paddsw (%rdi), %xmm0 # sched: [4:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_paddsw:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    paddsw %xmm1, %xmm0 # sched: [1:0.50]
-; SANDY-SSE-NEXT:    paddsw (%rdi), %xmm0 # sched: [7:0.50]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_paddsw:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vpaddsw %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SANDY-NEXT:    vpaddsw (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_paddsw:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    paddsw %xmm1, %xmm0 # sched: [1:0.50]
-; HASWELL-SSE-NEXT:    paddsw (%rdi), %xmm0 # sched: [7:0.50]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_paddsw:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpaddsw %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; HASWELL-NEXT:    vpaddsw (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_paddsw:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    paddsw %xmm1, %xmm0 # sched: [1:0.50]
-; BROADWELL-SSE-NEXT:    paddsw (%rdi), %xmm0 # sched: [6:0.50]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_paddsw:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpaddsw %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    vpaddsw (%rdi), %xmm0, %xmm0 # sched: [6:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_paddsw:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    paddsw %xmm1, %xmm0 # sched: [1:0.50]
-; SKYLAKE-SSE-NEXT:    paddsw (%rdi), %xmm0 # sched: [7:0.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_paddsw:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpaddsw %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SKYLAKE-NEXT:    vpaddsw (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_paddsw:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    paddsw %xmm1, %xmm0 # sched: [1:0.50]
-; SKX-SSE-NEXT:    paddsw (%rdi), %xmm0 # sched: [7:0.50]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_paddsw:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpaddsw %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SKX-NEXT:    vpaddsw (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_paddsw:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    paddsw %xmm1, %xmm0 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    paddsw (%rdi), %xmm0 # sched: [7:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_paddsw:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vpaddsw %xmm1, %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    vpaddsw (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_paddsw:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    paddsw %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    paddsw (%rdi), %xmm0 # sched: [6:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_paddsw:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vpaddsw %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    vpaddsw (%rdi), %xmm0, %xmm0 # sched: [6:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_paddsw:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    paddsw %xmm1, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    paddsw (%rdi), %xmm0 # sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_paddsw:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpaddsw %xmm1, %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vpaddsw (%rdi), %xmm0, %xmm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <8 x i16> @llvm.x86.sse2.padds.w(<8 x i16> %a0, <8 x i16> %a1)
-  %2 = load <8 x i16>, <8 x i16> *%a2, align 16
-  %3 = call <8 x i16> @llvm.x86.sse2.padds.w(<8 x i16> %1, <8 x i16> %2)
-  ret <8 x i16> %3
-}
-declare <8 x i16> @llvm.x86.sse2.padds.w(<8 x i16>, <8 x i16>) nounwind readnone
-
-define <16 x i8> @test_paddusb(<16 x i8> %a0, <16 x i8> %a1, <16 x i8> *%a2) {
-; GENERIC-LABEL: test_paddusb:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    paddusb %xmm1, %xmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    paddusb (%rdi), %xmm0 # sched: [7:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_paddusb:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    paddusb %xmm1, %xmm0 # sched: [1:0.50]
-; ATOM-NEXT:    paddusb (%rdi), %xmm0 # sched: [1:1.00]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_paddusb:
-; SLM:       # %bb.0:
-; SLM-NEXT:    paddusb %xmm1, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    paddusb (%rdi), %xmm0 # sched: [4:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_paddusb:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    paddusb %xmm1, %xmm0 # sched: [1:0.50]
-; SANDY-SSE-NEXT:    paddusb (%rdi), %xmm0 # sched: [7:0.50]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_paddusb:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vpaddusb %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SANDY-NEXT:    vpaddusb (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_paddusb:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    paddusb %xmm1, %xmm0 # sched: [1:0.50]
-; HASWELL-SSE-NEXT:    paddusb (%rdi), %xmm0 # sched: [7:0.50]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_paddusb:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpaddusb %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; HASWELL-NEXT:    vpaddusb (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_paddusb:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    paddusb %xmm1, %xmm0 # sched: [1:0.50]
-; BROADWELL-SSE-NEXT:    paddusb (%rdi), %xmm0 # sched: [6:0.50]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_paddusb:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpaddusb %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    vpaddusb (%rdi), %xmm0, %xmm0 # sched: [6:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_paddusb:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    paddusb %xmm1, %xmm0 # sched: [1:0.50]
-; SKYLAKE-SSE-NEXT:    paddusb (%rdi), %xmm0 # sched: [7:0.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_paddusb:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpaddusb %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SKYLAKE-NEXT:    vpaddusb (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_paddusb:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    paddusb %xmm1, %xmm0 # sched: [1:0.50]
-; SKX-SSE-NEXT:    paddusb (%rdi), %xmm0 # sched: [7:0.50]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_paddusb:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpaddusb %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SKX-NEXT:    vpaddusb (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_paddusb:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    paddusb %xmm1, %xmm0 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    paddusb (%rdi), %xmm0 # sched: [7:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_paddusb:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vpaddusb %xmm1, %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    vpaddusb (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_paddusb:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    paddusb %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    paddusb (%rdi), %xmm0 # sched: [6:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_paddusb:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vpaddusb %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    vpaddusb (%rdi), %xmm0, %xmm0 # sched: [6:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_paddusb:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    paddusb %xmm1, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    paddusb (%rdi), %xmm0 # sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_paddusb:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpaddusb %xmm1, %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vpaddusb (%rdi), %xmm0, %xmm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <16 x i8> @llvm.x86.sse2.paddus.b(<16 x i8> %a0, <16 x i8> %a1)
-  %2 = load <16 x i8>, <16 x i8> *%a2, align 16
-  %3 = call <16 x i8> @llvm.x86.sse2.paddus.b(<16 x i8> %1, <16 x i8> %2)
-  ret <16 x i8> %3
-}
-declare <16 x i8> @llvm.x86.sse2.paddus.b(<16 x i8>, <16 x i8>) nounwind readnone
-
-define <8 x i16> @test_paddusw(<8 x i16> %a0, <8 x i16> %a1, <8 x i16> *%a2) {
-; GENERIC-LABEL: test_paddusw:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    paddusw %xmm1, %xmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    paddusw (%rdi), %xmm0 # sched: [7:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_paddusw:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    paddusw %xmm1, %xmm0 # sched: [1:0.50]
-; ATOM-NEXT:    paddusw (%rdi), %xmm0 # sched: [1:1.00]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_paddusw:
-; SLM:       # %bb.0:
-; SLM-NEXT:    paddusw %xmm1, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    paddusw (%rdi), %xmm0 # sched: [4:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_paddusw:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    paddusw %xmm1, %xmm0 # sched: [1:0.50]
-; SANDY-SSE-NEXT:    paddusw (%rdi), %xmm0 # sched: [7:0.50]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_paddusw:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vpaddusw %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SANDY-NEXT:    vpaddusw (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_paddusw:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    paddusw %xmm1, %xmm0 # sched: [1:0.50]
-; HASWELL-SSE-NEXT:    paddusw (%rdi), %xmm0 # sched: [7:0.50]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_paddusw:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpaddusw %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; HASWELL-NEXT:    vpaddusw (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_paddusw:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    paddusw %xmm1, %xmm0 # sched: [1:0.50]
-; BROADWELL-SSE-NEXT:    paddusw (%rdi), %xmm0 # sched: [6:0.50]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_paddusw:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpaddusw %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    vpaddusw (%rdi), %xmm0, %xmm0 # sched: [6:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_paddusw:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    paddusw %xmm1, %xmm0 # sched: [1:0.50]
-; SKYLAKE-SSE-NEXT:    paddusw (%rdi), %xmm0 # sched: [7:0.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_paddusw:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpaddusw %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SKYLAKE-NEXT:    vpaddusw (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_paddusw:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    paddusw %xmm1, %xmm0 # sched: [1:0.50]
-; SKX-SSE-NEXT:    paddusw (%rdi), %xmm0 # sched: [7:0.50]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_paddusw:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpaddusw %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SKX-NEXT:    vpaddusw (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_paddusw:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    paddusw %xmm1, %xmm0 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    paddusw (%rdi), %xmm0 # sched: [7:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_paddusw:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vpaddusw %xmm1, %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    vpaddusw (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_paddusw:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    paddusw %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    paddusw (%rdi), %xmm0 # sched: [6:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_paddusw:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vpaddusw %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    vpaddusw (%rdi), %xmm0, %xmm0 # sched: [6:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_paddusw:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    paddusw %xmm1, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    paddusw (%rdi), %xmm0 # sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_paddusw:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpaddusw %xmm1, %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vpaddusw (%rdi), %xmm0, %xmm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <8 x i16> @llvm.x86.sse2.paddus.w(<8 x i16> %a0, <8 x i16> %a1)
-  %2 = load <8 x i16>, <8 x i16> *%a2, align 16
-  %3 = call <8 x i16> @llvm.x86.sse2.paddus.w(<8 x i16> %1, <8 x i16> %2)
-  ret <8 x i16> %3
-}
-declare <8 x i16> @llvm.x86.sse2.paddus.w(<8 x i16>, <8 x i16>) nounwind readnone
-
-define <8 x i16> @test_paddw(<8 x i16> %a0, <8 x i16> %a1, <8 x i16> *%a2) {
-; GENERIC-LABEL: test_paddw:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    paddw %xmm1, %xmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    paddw (%rdi), %xmm0 # sched: [7:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_paddw:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    paddw %xmm1, %xmm0 # sched: [1:0.50]
-; ATOM-NEXT:    paddw (%rdi), %xmm0 # sched: [1:1.00]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_paddw:
-; SLM:       # %bb.0:
-; SLM-NEXT:    paddw %xmm1, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    paddw (%rdi), %xmm0 # sched: [4:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_paddw:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    paddw %xmm1, %xmm0 # sched: [1:0.50]
-; SANDY-SSE-NEXT:    paddw (%rdi), %xmm0 # sched: [7:0.50]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_paddw:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vpaddw %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SANDY-NEXT:    vpaddw (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_paddw:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    paddw %xmm1, %xmm0 # sched: [1:0.50]
-; HASWELL-SSE-NEXT:    paddw (%rdi), %xmm0 # sched: [7:0.50]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_paddw:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpaddw %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; HASWELL-NEXT:    vpaddw (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_paddw:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    paddw %xmm1, %xmm0 # sched: [1:0.50]
-; BROADWELL-SSE-NEXT:    paddw (%rdi), %xmm0 # sched: [6:0.50]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_paddw:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpaddw %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    vpaddw (%rdi), %xmm0, %xmm0 # sched: [6:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_paddw:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    paddw %xmm1, %xmm0 # sched: [1:0.33]
-; SKYLAKE-SSE-NEXT:    paddw (%rdi), %xmm0 # sched: [7:0.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_paddw:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpaddw %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    vpaddw (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_paddw:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    paddw %xmm1, %xmm0 # sched: [1:0.33]
-; SKX-SSE-NEXT:    paddw (%rdi), %xmm0 # sched: [7:0.50]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_paddw:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpaddw %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    vpaddw (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_paddw:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    paddw %xmm1, %xmm0 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    paddw (%rdi), %xmm0 # sched: [7:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_paddw:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vpaddw %xmm1, %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    vpaddw (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_paddw:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    paddw %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    paddw (%rdi), %xmm0 # sched: [6:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_paddw:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vpaddw %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    vpaddw (%rdi), %xmm0, %xmm0 # sched: [6:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_paddw:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    paddw %xmm1, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    paddw (%rdi), %xmm0 # sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_paddw:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpaddw %xmm1, %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vpaddw (%rdi), %xmm0, %xmm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = add <8 x i16> %a0, %a1
-  %2 = load <8 x i16>, <8 x i16> *%a2, align 16
-  %3 = add <8 x i16> %1, %2
-  ret <8 x i16> %3
-}
-
-define <2 x i64> @test_pand(<2 x i64> %a0, <2 x i64> %a1, <2 x i64> *%a2) {
-; GENERIC-LABEL: test_pand:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    pand %xmm1, %xmm0 # sched: [1:0.33]
-; GENERIC-NEXT:    pand (%rdi), %xmm0 # sched: [7:0.50]
-; GENERIC-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_pand:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    pand %xmm1, %xmm0 # sched: [1:0.50]
-; ATOM-NEXT:    pand (%rdi), %xmm0 # sched: [1:1.00]
-; ATOM-NEXT:    paddq %xmm1, %xmm0 # sched: [2:1.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_pand:
-; SLM:       # %bb.0:
-; SLM-NEXT:    pand %xmm1, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    pand (%rdi), %xmm0 # sched: [4:1.00]
-; SLM-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_pand:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    pand %xmm1, %xmm0 # sched: [1:0.33]
-; SANDY-SSE-NEXT:    pand (%rdi), %xmm0 # sched: [7:0.50]
-; SANDY-SSE-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.50]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_pand:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vpand %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SANDY-NEXT:    vpand (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SANDY-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_pand:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    pand %xmm1, %xmm0 # sched: [1:0.33]
-; HASWELL-SSE-NEXT:    pand (%rdi), %xmm0 # sched: [7:0.50]
-; HASWELL-SSE-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.50]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_pand:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpand %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; HASWELL-NEXT:    vpand (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; HASWELL-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_pand:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    pand %xmm1, %xmm0 # sched: [1:0.33]
-; BROADWELL-SSE-NEXT:    pand (%rdi), %xmm0 # sched: [6:0.50]
-; BROADWELL-SSE-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.50]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pand:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpand %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; BROADWELL-NEXT:    vpand (%rdi), %xmm0, %xmm0 # sched: [6:0.50]
-; BROADWELL-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_pand:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    pand %xmm1, %xmm0 # sched: [1:0.33]
-; SKYLAKE-SSE-NEXT:    pand (%rdi), %xmm0 # sched: [7:0.50]
-; SKYLAKE-SSE-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.33]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pand:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpand %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    vpand (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKYLAKE-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_pand:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    pand %xmm1, %xmm0 # sched: [1:0.33]
-; SKX-SSE-NEXT:    pand (%rdi), %xmm0 # sched: [7:0.50]
-; SKX-SSE-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.33]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pand:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpand %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    vpand (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKX-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_pand:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    pand %xmm1, %xmm0 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    pand (%rdi), %xmm0 # sched: [7:0.50]
-; BDVER2-SSE-NEXT:    paddq %xmm1, %xmm0 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_pand:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vpand %xmm1, %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    vpand (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; BDVER2-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_pand:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    pand %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    pand (%rdi), %xmm0 # sched: [6:1.00]
-; BTVER2-SSE-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_pand:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vpand %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    vpand (%rdi), %xmm0, %xmm0 # sched: [6:1.00]
-; BTVER2-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_pand:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    pand %xmm1, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    pand (%rdi), %xmm0 # sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_pand:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpand %xmm1, %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vpand (%rdi), %xmm0, %xmm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = and <2 x i64> %a0, %a1
-  %2 = load <2 x i64>, <2 x i64> *%a2, align 16
-  %3 = and <2 x i64> %1, %2
-  %4 = add <2 x i64> %3, %a1
-  ret <2 x i64> %4
-}
-
-define <2 x i64> @test_pandn(<2 x i64> %a0, <2 x i64> %a1, <2 x i64> *%a2) {
-; GENERIC-LABEL: test_pandn:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    pandn %xmm1, %xmm0 # sched: [1:0.33]
-; GENERIC-NEXT:    movdqa %xmm0, %xmm1 # sched: [1:0.33]
-; GENERIC-NEXT:    pandn (%rdi), %xmm1 # sched: [7:0.50]
-; GENERIC-NEXT:    paddq %xmm0, %xmm1 # sched: [1:0.50]
-; GENERIC-NEXT:    movdqa %xmm1, %xmm0 # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_pandn:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    pandn %xmm1, %xmm0 # sched: [1:0.50]
-; ATOM-NEXT:    movdqa %xmm0, %xmm1 # sched: [1:0.50]
-; ATOM-NEXT:    pandn (%rdi), %xmm1 # sched: [1:1.00]
-; ATOM-NEXT:    paddq %xmm0, %xmm1 # sched: [2:1.00]
-; ATOM-NEXT:    movdqa %xmm1, %xmm0 # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_pandn:
-; SLM:       # %bb.0:
-; SLM-NEXT:    pandn %xmm1, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    movdqa %xmm0, %xmm1 # sched: [1:0.50]
-; SLM-NEXT:    pandn (%rdi), %xmm1 # sched: [4:1.00]
-; SLM-NEXT:    paddq %xmm0, %xmm1 # sched: [1:0.50]
-; SLM-NEXT:    movdqa %xmm1, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_pandn:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    pandn %xmm1, %xmm0 # sched: [1:0.33]
-; SANDY-SSE-NEXT:    movdqa %xmm0, %xmm1 # sched: [1:0.33]
-; SANDY-SSE-NEXT:    pandn (%rdi), %xmm1 # sched: [7:0.50]
-; SANDY-SSE-NEXT:    paddq %xmm0, %xmm1 # sched: [1:0.50]
-; SANDY-SSE-NEXT:    movdqa %xmm1, %xmm0 # sched: [1:0.33]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_pandn:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vpandn %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SANDY-NEXT:    vpandn (%rdi), %xmm0, %xmm1 # sched: [7:0.50]
-; SANDY-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_pandn:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    pandn %xmm1, %xmm0 # sched: [1:0.33]
-; HASWELL-SSE-NEXT:    movdqa %xmm0, %xmm1 # sched: [1:0.33]
-; HASWELL-SSE-NEXT:    pandn (%rdi), %xmm1 # sched: [7:0.50]
-; HASWELL-SSE-NEXT:    paddq %xmm0, %xmm1 # sched: [1:0.50]
-; HASWELL-SSE-NEXT:    movdqa %xmm1, %xmm0 # sched: [1:0.33]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_pandn:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpandn %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; HASWELL-NEXT:    vpandn (%rdi), %xmm0, %xmm1 # sched: [7:0.50]
-; HASWELL-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_pandn:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    pandn %xmm1, %xmm0 # sched: [1:0.33]
-; BROADWELL-SSE-NEXT:    movdqa %xmm0, %xmm1 # sched: [1:0.33]
-; BROADWELL-SSE-NEXT:    pandn (%rdi), %xmm1 # sched: [6:0.50]
-; BROADWELL-SSE-NEXT:    paddq %xmm0, %xmm1 # sched: [1:0.50]
-; BROADWELL-SSE-NEXT:    movdqa %xmm1, %xmm0 # sched: [1:0.33]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pandn:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpandn %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; BROADWELL-NEXT:    vpandn (%rdi), %xmm0, %xmm1 # sched: [6:0.50]
-; BROADWELL-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_pandn:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    pandn %xmm1, %xmm0 # sched: [1:0.33]
-; SKYLAKE-SSE-NEXT:    movdqa %xmm0, %xmm1 # sched: [1:0.33]
-; SKYLAKE-SSE-NEXT:    pandn (%rdi), %xmm1 # sched: [7:0.50]
-; SKYLAKE-SSE-NEXT:    paddq %xmm0, %xmm1 # sched: [1:0.33]
-; SKYLAKE-SSE-NEXT:    movdqa %xmm1, %xmm0 # sched: [1:0.33]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pandn:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpandn %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    vpandn (%rdi), %xmm0, %xmm1 # sched: [7:0.50]
-; SKYLAKE-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_pandn:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    pandn %xmm1, %xmm0 # sched: [1:0.33]
-; SKX-SSE-NEXT:    movdqa %xmm0, %xmm1 # sched: [1:0.33]
-; SKX-SSE-NEXT:    pandn (%rdi), %xmm1 # sched: [7:0.50]
-; SKX-SSE-NEXT:    paddq %xmm0, %xmm1 # sched: [1:0.33]
-; SKX-SSE-NEXT:    movdqa %xmm1, %xmm0 # sched: [1:0.33]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pandn:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpandn %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    vpandn (%rdi), %xmm0, %xmm1 # sched: [7:0.50]
-; SKX-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_pandn:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    pandn %xmm1, %xmm0 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    movdqa %xmm0, %xmm1 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    pandn (%rdi), %xmm1 # sched: [7:0.50]
-; BDVER2-SSE-NEXT:    paddq %xmm0, %xmm1 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    movdqa %xmm1, %xmm0 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_pandn:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vpandn %xmm1, %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    vpandn (%rdi), %xmm0, %xmm1 # sched: [7:0.50]
-; BDVER2-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_pandn:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    pandn %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    movdqa %xmm0, %xmm1 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    pandn (%rdi), %xmm1 # sched: [6:1.00]
-; BTVER2-SSE-NEXT:    paddq %xmm0, %xmm1 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    movdqa %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_pandn:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vpandn %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    vpandn (%rdi), %xmm0, %xmm1 # sched: [6:1.00]
-; BTVER2-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_pandn:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    pandn %xmm1, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    movdqa %xmm0, %xmm1 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    pandn (%rdi), %xmm1 # sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    paddq %xmm0, %xmm1 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    movdqa %xmm1, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_pandn:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpandn %xmm1, %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vpandn (%rdi), %xmm0, %xmm1 # sched: [8:0.50]
-; ZNVER1-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = xor <2 x i64> %a0, <i64 -1, i64 -1>
-  %2 = and <2 x i64> %a1, %1
-  %3 = load <2 x i64>, <2 x i64> *%a2, align 16
-  %4 = xor <2 x i64> %2, <i64 -1, i64 -1>
-  %5 = and <2 x i64> %3, %4
-  %6 = add <2 x i64> %2, %5
-  ret <2 x i64> %6
-}
-
-define <16 x i8> @test_pavgb(<16 x i8> %a0, <16 x i8> %a1, <16 x i8> *%a2) {
-; GENERIC-LABEL: test_pavgb:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    pavgb %xmm1, %xmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    pavgb (%rdi), %xmm0 # sched: [7:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_pavgb:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    pavgb %xmm1, %xmm0 # sched: [1:0.50]
-; ATOM-NEXT:    pavgb (%rdi), %xmm0 # sched: [1:1.00]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_pavgb:
-; SLM:       # %bb.0:
-; SLM-NEXT:    pavgb %xmm1, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    pavgb (%rdi), %xmm0 # sched: [4:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_pavgb:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    pavgb %xmm1, %xmm0 # sched: [1:0.50]
-; SANDY-SSE-NEXT:    pavgb (%rdi), %xmm0 # sched: [7:0.50]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_pavgb:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vpavgb %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SANDY-NEXT:    vpavgb (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_pavgb:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    pavgb %xmm1, %xmm0 # sched: [1:0.50]
-; HASWELL-SSE-NEXT:    pavgb (%rdi), %xmm0 # sched: [7:0.50]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_pavgb:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpavgb %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; HASWELL-NEXT:    vpavgb (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_pavgb:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    pavgb %xmm1, %xmm0 # sched: [1:0.50]
-; BROADWELL-SSE-NEXT:    pavgb (%rdi), %xmm0 # sched: [6:0.50]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pavgb:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpavgb %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    vpavgb (%rdi), %xmm0, %xmm0 # sched: [6:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_pavgb:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    pavgb %xmm1, %xmm0 # sched: [1:0.50]
-; SKYLAKE-SSE-NEXT:    pavgb (%rdi), %xmm0 # sched: [7:0.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pavgb:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpavgb %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SKYLAKE-NEXT:    vpavgb (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_pavgb:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    pavgb %xmm1, %xmm0 # sched: [1:0.50]
-; SKX-SSE-NEXT:    pavgb (%rdi), %xmm0 # sched: [7:0.50]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pavgb:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpavgb %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SKX-NEXT:    vpavgb (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_pavgb:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    pavgb %xmm1, %xmm0 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    pavgb (%rdi), %xmm0 # sched: [7:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_pavgb:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vpavgb %xmm1, %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    vpavgb (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_pavgb:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    pavgb %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    pavgb (%rdi), %xmm0 # sched: [6:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_pavgb:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vpavgb %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    vpavgb (%rdi), %xmm0, %xmm0 # sched: [6:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_pavgb:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    pavgb %xmm1, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    pavgb (%rdi), %xmm0 # sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_pavgb:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpavgb %xmm1, %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vpavgb (%rdi), %xmm0, %xmm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = zext <16 x i8> %a0 to <16 x i16>
-  %2 = zext <16 x i8> %a1 to <16 x i16>
-  %3 = add <16 x i16> %1, %2
-  %4 = add <16 x i16> %3, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
-  %5 = lshr <16 x i16> %4, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
-  %6 = trunc <16 x i16> %5 to <16 x i8>
-  %7 = load <16 x i8>, <16 x i8> *%a2, align 16
-  %8 = zext <16 x i8> %6 to <16 x i16>
-  %9 = zext <16 x i8> %7 to <16 x i16>
-  %10 = add <16 x i16> %8, %9
-  %11 = add <16 x i16> %10, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
-  %12 = lshr <16 x i16> %11, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
-  %13 = trunc <16 x i16> %12 to <16 x i8>
-  ret <16 x i8> %13
-}
-
-define <8 x i16> @test_pavgw(<8 x i16> %a0, <8 x i16> %a1, <8 x i16> *%a2) {
-; GENERIC-LABEL: test_pavgw:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    pavgw %xmm1, %xmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    pavgw (%rdi), %xmm0 # sched: [7:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_pavgw:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    pavgw %xmm1, %xmm0 # sched: [1:0.50]
-; ATOM-NEXT:    pavgw (%rdi), %xmm0 # sched: [1:1.00]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_pavgw:
-; SLM:       # %bb.0:
-; SLM-NEXT:    pavgw %xmm1, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    pavgw (%rdi), %xmm0 # sched: [4:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_pavgw:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    pavgw %xmm1, %xmm0 # sched: [1:0.50]
-; SANDY-SSE-NEXT:    pavgw (%rdi), %xmm0 # sched: [7:0.50]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_pavgw:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vpavgw %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SANDY-NEXT:    vpavgw (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_pavgw:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    pavgw %xmm1, %xmm0 # sched: [1:0.50]
-; HASWELL-SSE-NEXT:    pavgw (%rdi), %xmm0 # sched: [7:0.50]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_pavgw:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpavgw %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; HASWELL-NEXT:    vpavgw (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_pavgw:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    pavgw %xmm1, %xmm0 # sched: [1:0.50]
-; BROADWELL-SSE-NEXT:    pavgw (%rdi), %xmm0 # sched: [6:0.50]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pavgw:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpavgw %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    vpavgw (%rdi), %xmm0, %xmm0 # sched: [6:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_pavgw:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    pavgw %xmm1, %xmm0 # sched: [1:0.50]
-; SKYLAKE-SSE-NEXT:    pavgw (%rdi), %xmm0 # sched: [7:0.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pavgw:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpavgw %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SKYLAKE-NEXT:    vpavgw (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_pavgw:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    pavgw %xmm1, %xmm0 # sched: [1:0.50]
-; SKX-SSE-NEXT:    pavgw (%rdi), %xmm0 # sched: [7:0.50]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pavgw:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpavgw %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SKX-NEXT:    vpavgw (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_pavgw:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    pavgw %xmm1, %xmm0 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    pavgw (%rdi), %xmm0 # sched: [7:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_pavgw:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vpavgw %xmm1, %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    vpavgw (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_pavgw:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    pavgw %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    pavgw (%rdi), %xmm0 # sched: [6:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_pavgw:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vpavgw %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    vpavgw (%rdi), %xmm0, %xmm0 # sched: [6:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_pavgw:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    pavgw %xmm1, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    pavgw (%rdi), %xmm0 # sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_pavgw:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpavgw %xmm1, %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vpavgw (%rdi), %xmm0, %xmm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = zext <8 x i16> %a0 to <8 x i32>
-  %2 = zext <8 x i16> %a1 to <8 x i32>
-  %3 = add <8 x i32> %1, %2
-  %4 = add <8 x i32> %3, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
-  %5 = lshr <8 x i32> %4, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
-  %6 = trunc <8 x i32> %5 to <8 x i16>
-  %7 = load <8 x i16>, <8 x i16> *%a2, align 16
-  %8 = zext <8 x i16> %6 to <8 x i32>
-  %9 = zext <8 x i16> %7 to <8 x i32>
-  %10 = add <8 x i32> %8, %9
-  %11 = add <8 x i32> %10, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
-  %12 = lshr <8 x i32> %11, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
-  %13 = trunc <8 x i32> %12 to <8 x i16>
-  ret <8 x i16> %13
-}
-
-define <16 x i8> @test_pcmpeqb(<16 x i8> %a0, <16 x i8> %a1, <16 x i8> *%a2) {
-; GENERIC-LABEL: test_pcmpeqb:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    pcmpeqb %xmm0, %xmm1 # sched: [1:0.50]
-; GENERIC-NEXT:    pcmpeqb (%rdi), %xmm0 # sched: [7:0.50]
-; GENERIC-NEXT:    por %xmm1, %xmm0 # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_pcmpeqb:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    pcmpeqb %xmm0, %xmm1 # sched: [1:0.50]
-; ATOM-NEXT:    pcmpeqb (%rdi), %xmm0 # sched: [1:1.00]
-; ATOM-NEXT:    por %xmm1, %xmm0 # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_pcmpeqb:
-; SLM:       # %bb.0:
-; SLM-NEXT:    pcmpeqb %xmm0, %xmm1 # sched: [1:0.50]
-; SLM-NEXT:    pcmpeqb (%rdi), %xmm0 # sched: [4:1.00]
-; SLM-NEXT:    por %xmm1, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_pcmpeqb:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    pcmpeqb %xmm0, %xmm1 # sched: [1:0.50]
-; SANDY-SSE-NEXT:    pcmpeqb (%rdi), %xmm0 # sched: [7:0.50]
-; SANDY-SSE-NEXT:    por %xmm1, %xmm0 # sched: [1:0.33]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_pcmpeqb:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vpcmpeqb %xmm1, %xmm0, %xmm1 # sched: [1:0.50]
-; SANDY-NEXT:    vpcmpeqb (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SANDY-NEXT:    vpor %xmm0, %xmm1, %xmm0 # sched: [1:0.33]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_pcmpeqb:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    pcmpeqb %xmm0, %xmm1 # sched: [1:0.50]
-; HASWELL-SSE-NEXT:    pcmpeqb (%rdi), %xmm0 # sched: [7:0.50]
-; HASWELL-SSE-NEXT:    por %xmm1, %xmm0 # sched: [1:0.33]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_pcmpeqb:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpcmpeqb %xmm1, %xmm0, %xmm1 # sched: [1:0.50]
-; HASWELL-NEXT:    vpcmpeqb (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; HASWELL-NEXT:    vpor %xmm0, %xmm1, %xmm0 # sched: [1:0.33]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_pcmpeqb:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    pcmpeqb %xmm0, %xmm1 # sched: [1:0.50]
-; BROADWELL-SSE-NEXT:    pcmpeqb (%rdi), %xmm0 # sched: [6:0.50]
-; BROADWELL-SSE-NEXT:    por %xmm1, %xmm0 # sched: [1:0.33]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pcmpeqb:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpcmpeqb %xmm1, %xmm0, %xmm1 # sched: [1:0.50]
-; BROADWELL-NEXT:    vpcmpeqb (%rdi), %xmm0, %xmm0 # sched: [6:0.50]
-; BROADWELL-NEXT:    vpor %xmm0, %xmm1, %xmm0 # sched: [1:0.33]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_pcmpeqb:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    pcmpeqb %xmm0, %xmm1 # sched: [1:0.50]
-; SKYLAKE-SSE-NEXT:    pcmpeqb (%rdi), %xmm0 # sched: [7:0.50]
-; SKYLAKE-SSE-NEXT:    por %xmm1, %xmm0 # sched: [1:0.33]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pcmpeqb:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpcmpeqb %xmm1, %xmm0, %xmm1 # sched: [1:0.50]
-; SKYLAKE-NEXT:    vpcmpeqb (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKYLAKE-NEXT:    vpor %xmm0, %xmm1, %xmm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_pcmpeqb:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    pcmpeqb %xmm0, %xmm1 # sched: [1:0.50]
-; SKX-SSE-NEXT:    pcmpeqb (%rdi), %xmm0 # sched: [7:0.50]
-; SKX-SSE-NEXT:    por %xmm1, %xmm0 # sched: [1:0.33]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pcmpeqb:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpcmpeqb %xmm1, %xmm0, %xmm1 # sched: [1:0.50]
-; SKX-NEXT:    vpcmpeqb (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKX-NEXT:    vpor %xmm0, %xmm1, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_pcmpeqb:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    pcmpeqb %xmm0, %xmm1 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    pcmpeqb (%rdi), %xmm0 # sched: [7:0.50]
-; BDVER2-SSE-NEXT:    por %xmm1, %xmm0 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_pcmpeqb:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vpcmpeqb %xmm1, %xmm0, %xmm1 # sched: [2:0.50]
-; BDVER2-NEXT:    vpcmpeqb (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; BDVER2-NEXT:    vpor %xmm0, %xmm1, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_pcmpeqb:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    pcmpeqb %xmm0, %xmm1 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    pcmpeqb (%rdi), %xmm0 # sched: [6:1.00]
-; BTVER2-SSE-NEXT:    por %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_pcmpeqb:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vpcmpeqb %xmm1, %xmm0, %xmm1 # sched: [1:0.50]
-; BTVER2-NEXT:    vpcmpeqb (%rdi), %xmm0, %xmm0 # sched: [6:1.00]
-; BTVER2-NEXT:    vpor %xmm0, %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_pcmpeqb:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    pcmpeqb %xmm0, %xmm1 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    pcmpeqb (%rdi), %xmm0 # sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    por %xmm1, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_pcmpeqb:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpcmpeqb %xmm1, %xmm0, %xmm1 # sched: [1:0.25]
-; ZNVER1-NEXT:    vpcmpeqb (%rdi), %xmm0, %xmm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    vpor %xmm0, %xmm1, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = icmp eq <16 x i8> %a0, %a1
-  %2 = load <16 x i8>, <16 x i8> *%a2, align 16
-  %3 = icmp eq <16 x i8> %a0, %2
-  %4 = or <16 x i1> %1, %3
-  %5 = sext <16 x i1> %4 to <16 x i8>
-  ret <16 x i8> %5
-}
-
-define <4 x i32> @test_pcmpeqd(<4 x i32> %a0, <4 x i32> %a1, <4 x i32> *%a2) {
-; GENERIC-LABEL: test_pcmpeqd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    pcmpeqd %xmm0, %xmm1 # sched: [1:0.50]
-; GENERIC-NEXT:    pcmpeqd (%rdi), %xmm0 # sched: [7:0.50]
-; GENERIC-NEXT:    por %xmm1, %xmm0 # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_pcmpeqd:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    pcmpeqd %xmm0, %xmm1 # sched: [1:0.50]
-; ATOM-NEXT:    pcmpeqd (%rdi), %xmm0 # sched: [1:1.00]
-; ATOM-NEXT:    por %xmm1, %xmm0 # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_pcmpeqd:
-; SLM:       # %bb.0:
-; SLM-NEXT:    pcmpeqd %xmm0, %xmm1 # sched: [1:0.50]
-; SLM-NEXT:    pcmpeqd (%rdi), %xmm0 # sched: [4:1.00]
-; SLM-NEXT:    por %xmm1, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_pcmpeqd:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    pcmpeqd %xmm0, %xmm1 # sched: [1:0.50]
-; SANDY-SSE-NEXT:    pcmpeqd (%rdi), %xmm0 # sched: [7:0.50]
-; SANDY-SSE-NEXT:    por %xmm1, %xmm0 # sched: [1:0.33]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_pcmpeqd:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm1 # sched: [1:0.50]
-; SANDY-NEXT:    vpcmpeqd (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SANDY-NEXT:    vpor %xmm0, %xmm1, %xmm0 # sched: [1:0.33]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_pcmpeqd:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    pcmpeqd %xmm0, %xmm1 # sched: [1:0.50]
-; HASWELL-SSE-NEXT:    pcmpeqd (%rdi), %xmm0 # sched: [7:0.50]
-; HASWELL-SSE-NEXT:    por %xmm1, %xmm0 # sched: [1:0.33]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_pcmpeqd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm1 # sched: [1:0.50]
-; HASWELL-NEXT:    vpcmpeqd (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; HASWELL-NEXT:    vpor %xmm0, %xmm1, %xmm0 # sched: [1:0.33]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_pcmpeqd:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    pcmpeqd %xmm0, %xmm1 # sched: [1:0.50]
-; BROADWELL-SSE-NEXT:    pcmpeqd (%rdi), %xmm0 # sched: [6:0.50]
-; BROADWELL-SSE-NEXT:    por %xmm1, %xmm0 # sched: [1:0.33]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pcmpeqd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm1 # sched: [1:0.50]
-; BROADWELL-NEXT:    vpcmpeqd (%rdi), %xmm0, %xmm0 # sched: [6:0.50]
-; BROADWELL-NEXT:    vpor %xmm0, %xmm1, %xmm0 # sched: [1:0.33]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_pcmpeqd:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    pcmpeqd %xmm0, %xmm1 # sched: [1:0.50]
-; SKYLAKE-SSE-NEXT:    pcmpeqd (%rdi), %xmm0 # sched: [7:0.50]
-; SKYLAKE-SSE-NEXT:    por %xmm1, %xmm0 # sched: [1:0.33]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pcmpeqd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm1 # sched: [1:0.50]
-; SKYLAKE-NEXT:    vpcmpeqd (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKYLAKE-NEXT:    vpor %xmm0, %xmm1, %xmm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_pcmpeqd:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    pcmpeqd %xmm0, %xmm1 # sched: [1:0.50]
-; SKX-SSE-NEXT:    pcmpeqd (%rdi), %xmm0 # sched: [7:0.50]
-; SKX-SSE-NEXT:    por %xmm1, %xmm0 # sched: [1:0.33]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pcmpeqd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm1 # sched: [1:0.50]
-; SKX-NEXT:    vpcmpeqd (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKX-NEXT:    vpor %xmm0, %xmm1, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_pcmpeqd:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    pcmpeqd %xmm0, %xmm1 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    pcmpeqd (%rdi), %xmm0 # sched: [7:0.50]
-; BDVER2-SSE-NEXT:    por %xmm1, %xmm0 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_pcmpeqd:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm1 # sched: [2:0.50]
-; BDVER2-NEXT:    vpcmpeqd (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; BDVER2-NEXT:    vpor %xmm0, %xmm1, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_pcmpeqd:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    pcmpeqd %xmm0, %xmm1 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    pcmpeqd (%rdi), %xmm0 # sched: [6:1.00]
-; BTVER2-SSE-NEXT:    por %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_pcmpeqd:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm1 # sched: [1:0.50]
-; BTVER2-NEXT:    vpcmpeqd (%rdi), %xmm0, %xmm0 # sched: [6:1.00]
-; BTVER2-NEXT:    vpor %xmm0, %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_pcmpeqd:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    pcmpeqd %xmm0, %xmm1 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    pcmpeqd (%rdi), %xmm0 # sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    por %xmm1, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_pcmpeqd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm1 # sched: [1:0.25]
-; ZNVER1-NEXT:    vpcmpeqd (%rdi), %xmm0, %xmm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    vpor %xmm0, %xmm1, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = icmp eq <4 x i32> %a0, %a1
-  %2 = load <4 x i32>, <4 x i32> *%a2, align 16
-  %3 = icmp eq <4 x i32> %a0, %2
-  %4 = or <4 x i1> %1, %3
-  %5 = sext <4 x i1> %4 to <4 x i32>
-  ret <4 x i32> %5
-}
-
-define <8 x i16> @test_pcmpeqw(<8 x i16> %a0, <8 x i16> %a1, <8 x i16> *%a2) {
-; GENERIC-LABEL: test_pcmpeqw:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    pcmpeqw %xmm0, %xmm1 # sched: [1:0.50]
-; GENERIC-NEXT:    pcmpeqw (%rdi), %xmm0 # sched: [7:0.50]
-; GENERIC-NEXT:    por %xmm1, %xmm0 # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_pcmpeqw:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    pcmpeqw %xmm0, %xmm1 # sched: [1:0.50]
-; ATOM-NEXT:    pcmpeqw (%rdi), %xmm0 # sched: [1:1.00]
-; ATOM-NEXT:    por %xmm1, %xmm0 # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_pcmpeqw:
-; SLM:       # %bb.0:
-; SLM-NEXT:    pcmpeqw %xmm0, %xmm1 # sched: [1:0.50]
-; SLM-NEXT:    pcmpeqw (%rdi), %xmm0 # sched: [4:1.00]
-; SLM-NEXT:    por %xmm1, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_pcmpeqw:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    pcmpeqw %xmm0, %xmm1 # sched: [1:0.50]
-; SANDY-SSE-NEXT:    pcmpeqw (%rdi), %xmm0 # sched: [7:0.50]
-; SANDY-SSE-NEXT:    por %xmm1, %xmm0 # sched: [1:0.33]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_pcmpeqw:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vpcmpeqw %xmm1, %xmm0, %xmm1 # sched: [1:0.50]
-; SANDY-NEXT:    vpcmpeqw (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SANDY-NEXT:    vpor %xmm0, %xmm1, %xmm0 # sched: [1:0.33]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_pcmpeqw:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    pcmpeqw %xmm0, %xmm1 # sched: [1:0.50]
-; HASWELL-SSE-NEXT:    pcmpeqw (%rdi), %xmm0 # sched: [7:0.50]
-; HASWELL-SSE-NEXT:    por %xmm1, %xmm0 # sched: [1:0.33]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_pcmpeqw:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpcmpeqw %xmm1, %xmm0, %xmm1 # sched: [1:0.50]
-; HASWELL-NEXT:    vpcmpeqw (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; HASWELL-NEXT:    vpor %xmm0, %xmm1, %xmm0 # sched: [1:0.33]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_pcmpeqw:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    pcmpeqw %xmm0, %xmm1 # sched: [1:0.50]
-; BROADWELL-SSE-NEXT:    pcmpeqw (%rdi), %xmm0 # sched: [6:0.50]
-; BROADWELL-SSE-NEXT:    por %xmm1, %xmm0 # sched: [1:0.33]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pcmpeqw:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpcmpeqw %xmm1, %xmm0, %xmm1 # sched: [1:0.50]
-; BROADWELL-NEXT:    vpcmpeqw (%rdi), %xmm0, %xmm0 # sched: [6:0.50]
-; BROADWELL-NEXT:    vpor %xmm0, %xmm1, %xmm0 # sched: [1:0.33]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_pcmpeqw:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    pcmpeqw %xmm0, %xmm1 # sched: [1:0.50]
-; SKYLAKE-SSE-NEXT:    pcmpeqw (%rdi), %xmm0 # sched: [7:0.50]
-; SKYLAKE-SSE-NEXT:    por %xmm1, %xmm0 # sched: [1:0.33]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pcmpeqw:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpcmpeqw %xmm1, %xmm0, %xmm1 # sched: [1:0.50]
-; SKYLAKE-NEXT:    vpcmpeqw (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKYLAKE-NEXT:    vpor %xmm0, %xmm1, %xmm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_pcmpeqw:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    pcmpeqw %xmm0, %xmm1 # sched: [1:0.50]
-; SKX-SSE-NEXT:    pcmpeqw (%rdi), %xmm0 # sched: [7:0.50]
-; SKX-SSE-NEXT:    por %xmm1, %xmm0 # sched: [1:0.33]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pcmpeqw:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpcmpeqw %xmm1, %xmm0, %xmm1 # sched: [1:0.50]
-; SKX-NEXT:    vpcmpeqw (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKX-NEXT:    vpor %xmm0, %xmm1, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_pcmpeqw:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    pcmpeqw %xmm0, %xmm1 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    pcmpeqw (%rdi), %xmm0 # sched: [7:0.50]
-; BDVER2-SSE-NEXT:    por %xmm1, %xmm0 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_pcmpeqw:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vpcmpeqw %xmm1, %xmm0, %xmm1 # sched: [2:0.50]
-; BDVER2-NEXT:    vpcmpeqw (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; BDVER2-NEXT:    vpor %xmm0, %xmm1, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_pcmpeqw:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    pcmpeqw %xmm0, %xmm1 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    pcmpeqw (%rdi), %xmm0 # sched: [6:1.00]
-; BTVER2-SSE-NEXT:    por %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_pcmpeqw:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vpcmpeqw %xmm1, %xmm0, %xmm1 # sched: [1:0.50]
-; BTVER2-NEXT:    vpcmpeqw (%rdi), %xmm0, %xmm0 # sched: [6:1.00]
-; BTVER2-NEXT:    vpor %xmm0, %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_pcmpeqw:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    pcmpeqw %xmm0, %xmm1 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    pcmpeqw (%rdi), %xmm0 # sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    por %xmm1, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_pcmpeqw:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpcmpeqw %xmm1, %xmm0, %xmm1 # sched: [1:0.25]
-; ZNVER1-NEXT:    vpcmpeqw (%rdi), %xmm0, %xmm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    vpor %xmm0, %xmm1, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = icmp eq <8 x i16> %a0, %a1
-  %2 = load <8 x i16>, <8 x i16> *%a2, align 16
-  %3 = icmp eq <8 x i16> %a0, %2
-  %4 = or <8 x i1> %1, %3
-  %5 = sext <8 x i1> %4 to <8 x i16>
-  ret <8 x i16> %5
-}
-
-define <16 x i8> @test_pcmpgtb(<16 x i8> %a0, <16 x i8> %a1, <16 x i8> *%a2) {
-; GENERIC-LABEL: test_pcmpgtb:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    movdqa %xmm0, %xmm2 # sched: [1:0.33]
-; GENERIC-NEXT:    pcmpgtb %xmm1, %xmm2 # sched: [1:0.50]
-; GENERIC-NEXT:    pcmpgtb (%rdi), %xmm0 # sched: [7:0.50]
-; GENERIC-NEXT:    por %xmm2, %xmm0 # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_pcmpgtb:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    movdqa %xmm0, %xmm2 # sched: [1:0.50]
-; ATOM-NEXT:    pcmpgtb (%rdi), %xmm0 # sched: [1:1.00]
-; ATOM-NEXT:    pcmpgtb %xmm1, %xmm2 # sched: [1:0.50]
-; ATOM-NEXT:    por %xmm2, %xmm0 # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_pcmpgtb:
-; SLM:       # %bb.0:
-; SLM-NEXT:    movdqa %xmm0, %xmm2 # sched: [1:0.50]
-; SLM-NEXT:    pcmpgtb (%rdi), %xmm0 # sched: [4:1.00]
-; SLM-NEXT:    pcmpgtb %xmm1, %xmm2 # sched: [1:0.50]
-; SLM-NEXT:    por %xmm2, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_pcmpgtb:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    movdqa %xmm0, %xmm2 # sched: [1:0.33]
-; SANDY-SSE-NEXT:    pcmpgtb %xmm1, %xmm2 # sched: [1:0.50]
-; SANDY-SSE-NEXT:    pcmpgtb (%rdi), %xmm0 # sched: [7:0.50]
-; SANDY-SSE-NEXT:    por %xmm2, %xmm0 # sched: [1:0.33]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_pcmpgtb:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vpcmpgtb %xmm1, %xmm0, %xmm1 # sched: [1:0.50]
-; SANDY-NEXT:    vpcmpgtb (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SANDY-NEXT:    vpor %xmm0, %xmm1, %xmm0 # sched: [1:0.33]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_pcmpgtb:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    movdqa %xmm0, %xmm2 # sched: [1:0.33]
-; HASWELL-SSE-NEXT:    pcmpgtb %xmm1, %xmm2 # sched: [1:0.50]
-; HASWELL-SSE-NEXT:    pcmpgtb (%rdi), %xmm0 # sched: [7:0.50]
-; HASWELL-SSE-NEXT:    por %xmm2, %xmm0 # sched: [1:0.33]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_pcmpgtb:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpcmpgtb %xmm1, %xmm0, %xmm1 # sched: [1:0.50]
-; HASWELL-NEXT:    vpcmpgtb (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; HASWELL-NEXT:    vpor %xmm0, %xmm1, %xmm0 # sched: [1:0.33]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_pcmpgtb:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    movdqa %xmm0, %xmm2 # sched: [1:0.33]
-; BROADWELL-SSE-NEXT:    pcmpgtb %xmm1, %xmm2 # sched: [1:0.50]
-; BROADWELL-SSE-NEXT:    pcmpgtb (%rdi), %xmm0 # sched: [6:0.50]
-; BROADWELL-SSE-NEXT:    por %xmm2, %xmm0 # sched: [1:0.33]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pcmpgtb:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpcmpgtb %xmm1, %xmm0, %xmm1 # sched: [1:0.50]
-; BROADWELL-NEXT:    vpcmpgtb (%rdi), %xmm0, %xmm0 # sched: [6:0.50]
-; BROADWELL-NEXT:    vpor %xmm0, %xmm1, %xmm0 # sched: [1:0.33]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_pcmpgtb:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    movdqa %xmm0, %xmm2 # sched: [1:0.33]
-; SKYLAKE-SSE-NEXT:    pcmpgtb %xmm1, %xmm2 # sched: [1:0.50]
-; SKYLAKE-SSE-NEXT:    pcmpgtb (%rdi), %xmm0 # sched: [7:0.50]
-; SKYLAKE-SSE-NEXT:    por %xmm2, %xmm0 # sched: [1:0.33]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pcmpgtb:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpcmpgtb %xmm1, %xmm0, %xmm1 # sched: [1:0.50]
-; SKYLAKE-NEXT:    vpcmpgtb (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKYLAKE-NEXT:    vpor %xmm0, %xmm1, %xmm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_pcmpgtb:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    movdqa %xmm0, %xmm2 # sched: [1:0.33]
-; SKX-SSE-NEXT:    pcmpgtb %xmm1, %xmm2 # sched: [1:0.50]
-; SKX-SSE-NEXT:    pcmpgtb (%rdi), %xmm0 # sched: [7:0.50]
-; SKX-SSE-NEXT:    por %xmm2, %xmm0 # sched: [1:0.33]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pcmpgtb:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpcmpgtb %xmm1, %xmm0, %xmm1 # sched: [1:0.50]
-; SKX-NEXT:    vpcmpgtb (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKX-NEXT:    vpor %xmm0, %xmm1, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_pcmpgtb:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    movdqa %xmm0, %xmm2 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    pcmpgtb (%rdi), %xmm0 # sched: [7:0.50]
-; BDVER2-SSE-NEXT:    pcmpgtb %xmm1, %xmm2 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    por %xmm2, %xmm0 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_pcmpgtb:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vpcmpgtb %xmm1, %xmm0, %xmm1 # sched: [2:0.50]
-; BDVER2-NEXT:    vpcmpgtb (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; BDVER2-NEXT:    vpor %xmm0, %xmm1, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_pcmpgtb:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    movdqa %xmm0, %xmm2 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    pcmpgtb (%rdi), %xmm0 # sched: [6:1.00]
-; BTVER2-SSE-NEXT:    pcmpgtb %xmm1, %xmm2 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    por %xmm2, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_pcmpgtb:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vpcmpgtb %xmm1, %xmm0, %xmm1 # sched: [1:0.50]
-; BTVER2-NEXT:    vpcmpgtb (%rdi), %xmm0, %xmm0 # sched: [6:1.00]
-; BTVER2-NEXT:    vpor %xmm0, %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_pcmpgtb:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    movdqa %xmm0, %xmm2 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    pcmpgtb (%rdi), %xmm0 # sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    pcmpgtb %xmm1, %xmm2 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    por %xmm2, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_pcmpgtb:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpcmpgtb %xmm1, %xmm0, %xmm1 # sched: [1:0.25]
-; ZNVER1-NEXT:    vpcmpgtb (%rdi), %xmm0, %xmm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    vpor %xmm0, %xmm1, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = icmp sgt <16 x i8> %a0, %a1
-  %2 = load <16 x i8>, <16 x i8> *%a2, align 16
-  %3 = icmp sgt <16 x i8> %a0, %2
-  %4 = or <16 x i1> %1, %3
-  %5 = sext <16 x i1> %4 to <16 x i8>
-  ret <16 x i8> %5
-}
-
-define <4 x i32> @test_pcmpgtd(<4 x i32> %a0, <4 x i32> %a1, <4 x i32> *%a2) {
-; GENERIC-LABEL: test_pcmpgtd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    movdqa %xmm0, %xmm2 # sched: [1:0.33]
-; GENERIC-NEXT:    pcmpgtd %xmm1, %xmm2 # sched: [1:0.50]
-; GENERIC-NEXT:    pcmpeqd (%rdi), %xmm0 # sched: [7:0.50]
-; GENERIC-NEXT:    por %xmm2, %xmm0 # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_pcmpgtd:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    movdqa %xmm0, %xmm2 # sched: [1:0.50]
-; ATOM-NEXT:    pcmpeqd (%rdi), %xmm0 # sched: [1:1.00]
-; ATOM-NEXT:    pcmpgtd %xmm1, %xmm2 # sched: [1:0.50]
-; ATOM-NEXT:    por %xmm2, %xmm0 # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_pcmpgtd:
-; SLM:       # %bb.0:
-; SLM-NEXT:    movdqa %xmm0, %xmm2 # sched: [1:0.50]
-; SLM-NEXT:    pcmpeqd (%rdi), %xmm0 # sched: [4:1.00]
-; SLM-NEXT:    pcmpgtd %xmm1, %xmm2 # sched: [1:0.50]
-; SLM-NEXT:    por %xmm2, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_pcmpgtd:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    movdqa %xmm0, %xmm2 # sched: [1:0.33]
-; SANDY-SSE-NEXT:    pcmpgtd %xmm1, %xmm2 # sched: [1:0.50]
-; SANDY-SSE-NEXT:    pcmpeqd (%rdi), %xmm0 # sched: [7:0.50]
-; SANDY-SSE-NEXT:    por %xmm2, %xmm0 # sched: [1:0.33]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_pcmpgtd:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vpcmpgtd %xmm1, %xmm0, %xmm1 # sched: [1:0.50]
-; SANDY-NEXT:    vpcmpeqd (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SANDY-NEXT:    vpor %xmm0, %xmm1, %xmm0 # sched: [1:0.33]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_pcmpgtd:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    movdqa %xmm0, %xmm2 # sched: [1:0.33]
-; HASWELL-SSE-NEXT:    pcmpgtd %xmm1, %xmm2 # sched: [1:0.50]
-; HASWELL-SSE-NEXT:    pcmpeqd (%rdi), %xmm0 # sched: [7:0.50]
-; HASWELL-SSE-NEXT:    por %xmm2, %xmm0 # sched: [1:0.33]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_pcmpgtd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpcmpgtd %xmm1, %xmm0, %xmm1 # sched: [1:0.50]
-; HASWELL-NEXT:    vpcmpeqd (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; HASWELL-NEXT:    vpor %xmm0, %xmm1, %xmm0 # sched: [1:0.33]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_pcmpgtd:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    movdqa %xmm0, %xmm2 # sched: [1:0.33]
-; BROADWELL-SSE-NEXT:    pcmpgtd %xmm1, %xmm2 # sched: [1:0.50]
-; BROADWELL-SSE-NEXT:    pcmpeqd (%rdi), %xmm0 # sched: [6:0.50]
-; BROADWELL-SSE-NEXT:    por %xmm2, %xmm0 # sched: [1:0.33]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pcmpgtd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpcmpgtd %xmm1, %xmm0, %xmm1 # sched: [1:0.50]
-; BROADWELL-NEXT:    vpcmpeqd (%rdi), %xmm0, %xmm0 # sched: [6:0.50]
-; BROADWELL-NEXT:    vpor %xmm0, %xmm1, %xmm0 # sched: [1:0.33]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_pcmpgtd:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    movdqa %xmm0, %xmm2 # sched: [1:0.33]
-; SKYLAKE-SSE-NEXT:    pcmpgtd %xmm1, %xmm2 # sched: [1:0.50]
-; SKYLAKE-SSE-NEXT:    pcmpeqd (%rdi), %xmm0 # sched: [7:0.50]
-; SKYLAKE-SSE-NEXT:    por %xmm2, %xmm0 # sched: [1:0.33]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pcmpgtd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpcmpgtd %xmm1, %xmm0, %xmm1 # sched: [1:0.50]
-; SKYLAKE-NEXT:    vpcmpeqd (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKYLAKE-NEXT:    vpor %xmm0, %xmm1, %xmm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_pcmpgtd:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    movdqa %xmm0, %xmm2 # sched: [1:0.33]
-; SKX-SSE-NEXT:    pcmpgtd %xmm1, %xmm2 # sched: [1:0.50]
-; SKX-SSE-NEXT:    pcmpeqd (%rdi), %xmm0 # sched: [7:0.50]
-; SKX-SSE-NEXT:    por %xmm2, %xmm0 # sched: [1:0.33]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pcmpgtd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpcmpgtd %xmm1, %xmm0, %xmm1 # sched: [1:0.50]
-; SKX-NEXT:    vpcmpeqd (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKX-NEXT:    vpor %xmm0, %xmm1, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_pcmpgtd:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    movdqa %xmm0, %xmm2 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    pcmpeqd (%rdi), %xmm0 # sched: [7:0.50]
-; BDVER2-SSE-NEXT:    pcmpgtd %xmm1, %xmm2 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    por %xmm2, %xmm0 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_pcmpgtd:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vpcmpgtd %xmm1, %xmm0, %xmm1 # sched: [2:0.50]
-; BDVER2-NEXT:    vpcmpeqd (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; BDVER2-NEXT:    vpor %xmm0, %xmm1, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_pcmpgtd:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    movdqa %xmm0, %xmm2 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    pcmpeqd (%rdi), %xmm0 # sched: [6:1.00]
-; BTVER2-SSE-NEXT:    pcmpgtd %xmm1, %xmm2 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    por %xmm2, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_pcmpgtd:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vpcmpgtd %xmm1, %xmm0, %xmm1 # sched: [1:0.50]
-; BTVER2-NEXT:    vpcmpeqd (%rdi), %xmm0, %xmm0 # sched: [6:1.00]
-; BTVER2-NEXT:    vpor %xmm0, %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_pcmpgtd:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    movdqa %xmm0, %xmm2 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    pcmpeqd (%rdi), %xmm0 # sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    pcmpgtd %xmm1, %xmm2 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    por %xmm2, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_pcmpgtd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpcmpgtd %xmm1, %xmm0, %xmm1 # sched: [1:0.25]
-; ZNVER1-NEXT:    vpcmpeqd (%rdi), %xmm0, %xmm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    vpor %xmm0, %xmm1, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = icmp sgt <4 x i32> %a0, %a1
-  %2 = load <4 x i32>, <4 x i32> *%a2, align 16
-  %3 = icmp eq <4 x i32> %a0, %2
-  %4 = or <4 x i1> %1, %3
-  %5 = sext <4 x i1> %4 to <4 x i32>
-  ret <4 x i32> %5
-}
-
-define <8 x i16> @test_pcmpgtw(<8 x i16> %a0, <8 x i16> %a1, <8 x i16> *%a2) {
-; GENERIC-LABEL: test_pcmpgtw:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    movdqa %xmm0, %xmm2 # sched: [1:0.33]
-; GENERIC-NEXT:    pcmpgtw %xmm1, %xmm2 # sched: [1:0.50]
-; GENERIC-NEXT:    pcmpgtw (%rdi), %xmm0 # sched: [7:0.50]
-; GENERIC-NEXT:    por %xmm2, %xmm0 # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_pcmpgtw:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    movdqa %xmm0, %xmm2 # sched: [1:0.50]
-; ATOM-NEXT:    pcmpgtw (%rdi), %xmm0 # sched: [1:1.00]
-; ATOM-NEXT:    pcmpgtw %xmm1, %xmm2 # sched: [1:0.50]
-; ATOM-NEXT:    por %xmm2, %xmm0 # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_pcmpgtw:
-; SLM:       # %bb.0:
-; SLM-NEXT:    movdqa %xmm0, %xmm2 # sched: [1:0.50]
-; SLM-NEXT:    pcmpgtw (%rdi), %xmm0 # sched: [4:1.00]
-; SLM-NEXT:    pcmpgtw %xmm1, %xmm2 # sched: [1:0.50]
-; SLM-NEXT:    por %xmm2, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_pcmpgtw:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    movdqa %xmm0, %xmm2 # sched: [1:0.33]
-; SANDY-SSE-NEXT:    pcmpgtw %xmm1, %xmm2 # sched: [1:0.50]
-; SANDY-SSE-NEXT:    pcmpgtw (%rdi), %xmm0 # sched: [7:0.50]
-; SANDY-SSE-NEXT:    por %xmm2, %xmm0 # sched: [1:0.33]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_pcmpgtw:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vpcmpgtw %xmm1, %xmm0, %xmm1 # sched: [1:0.50]
-; SANDY-NEXT:    vpcmpgtw (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SANDY-NEXT:    vpor %xmm0, %xmm1, %xmm0 # sched: [1:0.33]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_pcmpgtw:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    movdqa %xmm0, %xmm2 # sched: [1:0.33]
-; HASWELL-SSE-NEXT:    pcmpgtw %xmm1, %xmm2 # sched: [1:0.50]
-; HASWELL-SSE-NEXT:    pcmpgtw (%rdi), %xmm0 # sched: [7:0.50]
-; HASWELL-SSE-NEXT:    por %xmm2, %xmm0 # sched: [1:0.33]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_pcmpgtw:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpcmpgtw %xmm1, %xmm0, %xmm1 # sched: [1:0.50]
-; HASWELL-NEXT:    vpcmpgtw (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; HASWELL-NEXT:    vpor %xmm0, %xmm1, %xmm0 # sched: [1:0.33]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_pcmpgtw:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    movdqa %xmm0, %xmm2 # sched: [1:0.33]
-; BROADWELL-SSE-NEXT:    pcmpgtw %xmm1, %xmm2 # sched: [1:0.50]
-; BROADWELL-SSE-NEXT:    pcmpgtw (%rdi), %xmm0 # sched: [6:0.50]
-; BROADWELL-SSE-NEXT:    por %xmm2, %xmm0 # sched: [1:0.33]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pcmpgtw:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpcmpgtw %xmm1, %xmm0, %xmm1 # sched: [1:0.50]
-; BROADWELL-NEXT:    vpcmpgtw (%rdi), %xmm0, %xmm0 # sched: [6:0.50]
-; BROADWELL-NEXT:    vpor %xmm0, %xmm1, %xmm0 # sched: [1:0.33]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_pcmpgtw:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    movdqa %xmm0, %xmm2 # sched: [1:0.33]
-; SKYLAKE-SSE-NEXT:    pcmpgtw %xmm1, %xmm2 # sched: [1:0.50]
-; SKYLAKE-SSE-NEXT:    pcmpgtw (%rdi), %xmm0 # sched: [7:0.50]
-; SKYLAKE-SSE-NEXT:    por %xmm2, %xmm0 # sched: [1:0.33]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pcmpgtw:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpcmpgtw %xmm1, %xmm0, %xmm1 # sched: [1:0.50]
-; SKYLAKE-NEXT:    vpcmpgtw (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKYLAKE-NEXT:    vpor %xmm0, %xmm1, %xmm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_pcmpgtw:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    movdqa %xmm0, %xmm2 # sched: [1:0.33]
-; SKX-SSE-NEXT:    pcmpgtw %xmm1, %xmm2 # sched: [1:0.50]
-; SKX-SSE-NEXT:    pcmpgtw (%rdi), %xmm0 # sched: [7:0.50]
-; SKX-SSE-NEXT:    por %xmm2, %xmm0 # sched: [1:0.33]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pcmpgtw:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpcmpgtw %xmm1, %xmm0, %xmm1 # sched: [1:0.50]
-; SKX-NEXT:    vpcmpgtw (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKX-NEXT:    vpor %xmm0, %xmm1, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_pcmpgtw:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    movdqa %xmm0, %xmm2 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    pcmpgtw (%rdi), %xmm0 # sched: [7:0.50]
-; BDVER2-SSE-NEXT:    pcmpgtw %xmm1, %xmm2 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    por %xmm2, %xmm0 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_pcmpgtw:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vpcmpgtw %xmm1, %xmm0, %xmm1 # sched: [2:0.50]
-; BDVER2-NEXT:    vpcmpgtw (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; BDVER2-NEXT:    vpor %xmm0, %xmm1, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_pcmpgtw:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    movdqa %xmm0, %xmm2 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    pcmpgtw (%rdi), %xmm0 # sched: [6:1.00]
-; BTVER2-SSE-NEXT:    pcmpgtw %xmm1, %xmm2 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    por %xmm2, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_pcmpgtw:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vpcmpgtw %xmm1, %xmm0, %xmm1 # sched: [1:0.50]
-; BTVER2-NEXT:    vpcmpgtw (%rdi), %xmm0, %xmm0 # sched: [6:1.00]
-; BTVER2-NEXT:    vpor %xmm0, %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_pcmpgtw:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    movdqa %xmm0, %xmm2 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    pcmpgtw (%rdi), %xmm0 # sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    pcmpgtw %xmm1, %xmm2 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    por %xmm2, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_pcmpgtw:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpcmpgtw %xmm1, %xmm0, %xmm1 # sched: [1:0.25]
-; ZNVER1-NEXT:    vpcmpgtw (%rdi), %xmm0, %xmm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    vpor %xmm0, %xmm1, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = icmp sgt <8 x i16> %a0, %a1
-  %2 = load <8 x i16>, <8 x i16> *%a2, align 16
-  %3 = icmp sgt <8 x i16> %a0, %2
-  %4 = or <8 x i1> %1, %3
-  %5 = sext <8 x i1> %4 to <8 x i16>
-  ret <8 x i16> %5
-}
-
-define i16 @test_pextrw(<8 x i16> %a0) {
-; GENERIC-LABEL: test_pextrw:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    pextrw $6, %xmm0, %eax # sched: [3:1.00]
-; GENERIC-NEXT:    # kill: def $ax killed $ax killed $eax
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_pextrw:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    pextrw $6, %xmm0, %eax # sched: [4:2.00]
-; ATOM-NEXT:    # kill: def $ax killed $ax killed $eax
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_pextrw:
-; SLM:       # %bb.0:
-; SLM-NEXT:    pextrw $6, %xmm0, %eax # sched: [1:1.00]
-; SLM-NEXT:    # kill: def $ax killed $ax killed $eax
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_pextrw:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    pextrw $6, %xmm0, %eax # sched: [3:1.00]
-; SANDY-SSE-NEXT:    # kill: def $ax killed $ax killed $eax
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_pextrw:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vpextrw $6, %xmm0, %eax # sched: [3:1.00]
-; SANDY-NEXT:    # kill: def $ax killed $ax killed $eax
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_pextrw:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    pextrw $6, %xmm0, %eax # sched: [2:1.00]
-; HASWELL-SSE-NEXT:    # kill: def $ax killed $ax killed $eax
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_pextrw:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpextrw $6, %xmm0, %eax # sched: [2:1.00]
-; HASWELL-NEXT:    # kill: def $ax killed $ax killed $eax
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_pextrw:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    pextrw $6, %xmm0, %eax # sched: [2:1.00]
-; BROADWELL-SSE-NEXT:    # kill: def $ax killed $ax killed $eax
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pextrw:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpextrw $6, %xmm0, %eax # sched: [2:1.00]
-; BROADWELL-NEXT:    # kill: def $ax killed $ax killed $eax
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_pextrw:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    pextrw $6, %xmm0, %eax # sched: [3:1.00]
-; SKYLAKE-SSE-NEXT:    # kill: def $ax killed $ax killed $eax
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pextrw:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpextrw $6, %xmm0, %eax # sched: [3:1.00]
-; SKYLAKE-NEXT:    # kill: def $ax killed $ax killed $eax
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_pextrw:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    pextrw $6, %xmm0, %eax # sched: [3:1.00]
-; SKX-SSE-NEXT:    # kill: def $ax killed $ax killed $eax
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pextrw:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpextrw $6, %xmm0, %eax # sched: [3:1.00]
-; SKX-NEXT:    # kill: def $ax killed $ax killed $eax
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_pextrw:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    pextrw $6, %xmm0, %eax # sched: [13:1.00]
-; BDVER2-SSE-NEXT:    # kill: def $ax killed $ax killed $eax
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_pextrw:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vpextrw $6, %xmm0, %eax # sched: [13:1.00]
-; BDVER2-NEXT:    # kill: def $ax killed $ax killed $eax
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_pextrw:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    pextrw $6, %xmm0, %eax # sched: [3:1.00]
-; BTVER2-SSE-NEXT:    # kill: def $ax killed $ax killed $eax
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_pextrw:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vpextrw $6, %xmm0, %eax # sched: [3:1.00]
-; BTVER2-NEXT:    # kill: def $ax killed $ax killed $eax
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_pextrw:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    pextrw $6, %xmm0, %eax # sched: [2:2.00]
-; ZNVER1-SSE-NEXT:    # kill: def $ax killed $ax killed $eax
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_pextrw:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpextrw $6, %xmm0, %eax # sched: [2:2.00]
-; ZNVER1-NEXT:    # kill: def $ax killed $ax killed $eax
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = extractelement <8 x i16> %a0, i32 6
-  ret i16 %1
-}
-
-define <8 x i16> @test_pinsrw(<8 x i16> %a0, i16 %a1, i16 *%a2) {
-; GENERIC-LABEL: test_pinsrw:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    pinsrw $1, %edi, %xmm0 # sched: [2:1.00]
-; GENERIC-NEXT:    pinsrw $3, (%rsi), %xmm0 # sched: [7:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_pinsrw:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    pinsrw $1, %edi, %xmm0 # sched: [1:1.00]
-; ATOM-NEXT:    pinsrw $3, (%rsi), %xmm0 # sched: [1:1.00]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_pinsrw:
-; SLM:       # %bb.0:
-; SLM-NEXT:    pinsrw $1, %edi, %xmm0 # sched: [1:1.00]
-; SLM-NEXT:    pinsrw $3, (%rsi), %xmm0 # sched: [4:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_pinsrw:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    pinsrw $1, %edi, %xmm0 # sched: [2:1.00]
-; SANDY-SSE-NEXT:    pinsrw $3, (%rsi), %xmm0 # sched: [7:0.50]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_pinsrw:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vpinsrw $1, %edi, %xmm0, %xmm0 # sched: [2:1.00]
-; SANDY-NEXT:    vpinsrw $3, (%rsi), %xmm0, %xmm0 # sched: [7:0.50]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_pinsrw:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    pinsrw $1, %edi, %xmm0 # sched: [2:2.00]
-; HASWELL-SSE-NEXT:    pinsrw $3, (%rsi), %xmm0 # sched: [6:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_pinsrw:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpinsrw $1, %edi, %xmm0, %xmm0 # sched: [2:2.00]
-; HASWELL-NEXT:    vpinsrw $3, (%rsi), %xmm0, %xmm0 # sched: [6:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_pinsrw:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    pinsrw $1, %edi, %xmm0 # sched: [2:2.00]
-; BROADWELL-SSE-NEXT:    pinsrw $3, (%rsi), %xmm0 # sched: [6:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pinsrw:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpinsrw $1, %edi, %xmm0, %xmm0 # sched: [2:2.00]
-; BROADWELL-NEXT:    vpinsrw $3, (%rsi), %xmm0, %xmm0 # sched: [6:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_pinsrw:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    pinsrw $1, %edi, %xmm0 # sched: [2:2.00]
-; SKYLAKE-SSE-NEXT:    pinsrw $3, (%rsi), %xmm0 # sched: [6:1.00]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pinsrw:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpinsrw $1, %edi, %xmm0, %xmm0 # sched: [2:2.00]
-; SKYLAKE-NEXT:    vpinsrw $3, (%rsi), %xmm0, %xmm0 # sched: [6:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_pinsrw:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    pinsrw $1, %edi, %xmm0 # sched: [2:2.00]
-; SKX-SSE-NEXT:    pinsrw $3, (%rsi), %xmm0 # sched: [6:1.00]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pinsrw:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpinsrw $1, %edi, %xmm0, %xmm0 # sched: [2:2.00]
-; SKX-NEXT:    vpinsrw $3, (%rsi), %xmm0, %xmm0 # sched: [6:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_pinsrw:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    pinsrw $1, %edi, %xmm0 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    pinsrw $3, (%rsi), %xmm0 # sched: [6:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_pinsrw:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vpinsrw $1, %edi, %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    vpinsrw $3, (%rsi), %xmm0, %xmm0 # sched: [6:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_pinsrw:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    pinsrw $1, %edi, %xmm0 # sched: [7:0.50]
-; BTVER2-SSE-NEXT:    pinsrw $3, (%rsi), %xmm0 # sched: [4:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_pinsrw:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vpinsrw $1, %edi, %xmm0, %xmm0 # sched: [7:0.50]
-; BTVER2-NEXT:    vpinsrw $3, (%rsi), %xmm0, %xmm0 # sched: [4:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_pinsrw:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    pinsrw $1, %edi, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    pinsrw $3, (%rsi), %xmm0 # sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_pinsrw:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpinsrw $1, %edi, %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vpinsrw $3, (%rsi), %xmm0, %xmm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = insertelement <8 x i16> %a0, i16 %a1, i32 1
-  %2 = load i16, i16 *%a2
-  %3 = insertelement <8 x i16> %1, i16 %2, i32 3
-  ret <8 x i16> %3
-}
-
-define <4 x i32> @test_pmaddwd(<8 x i16> %a0, <8 x i16> %a1, <8 x i16> *%a2) {
-; GENERIC-LABEL: test_pmaddwd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    pmaddwd %xmm1, %xmm0 # sched: [5:1.00]
-; GENERIC-NEXT:    pmaddwd (%rdi), %xmm0 # sched: [11:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_pmaddwd:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    pmaddwd %xmm1, %xmm0 # sched: [5:5.00]
-; ATOM-NEXT:    pmaddwd (%rdi), %xmm0 # sched: [5:5.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_pmaddwd:
-; SLM:       # %bb.0:
-; SLM-NEXT:    pmaddwd %xmm1, %xmm0 # sched: [4:1.00]
-; SLM-NEXT:    pmaddwd (%rdi), %xmm0 # sched: [7:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_pmaddwd:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    pmaddwd %xmm1, %xmm0 # sched: [5:1.00]
-; SANDY-SSE-NEXT:    pmaddwd (%rdi), %xmm0 # sched: [11:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_pmaddwd:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vpmaddwd %xmm1, %xmm0, %xmm0 # sched: [5:1.00]
-; SANDY-NEXT:    vpmaddwd (%rdi), %xmm0, %xmm0 # sched: [11:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_pmaddwd:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    pmaddwd %xmm1, %xmm0 # sched: [5:1.00]
-; HASWELL-SSE-NEXT:    pmaddwd (%rdi), %xmm0 # sched: [11:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_pmaddwd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpmaddwd %xmm1, %xmm0, %xmm0 # sched: [5:1.00]
-; HASWELL-NEXT:    vpmaddwd (%rdi), %xmm0, %xmm0 # sched: [11:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_pmaddwd:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    pmaddwd %xmm1, %xmm0 # sched: [5:1.00]
-; BROADWELL-SSE-NEXT:    pmaddwd (%rdi), %xmm0 # sched: [10:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pmaddwd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpmaddwd %xmm1, %xmm0, %xmm0 # sched: [5:1.00]
-; BROADWELL-NEXT:    vpmaddwd (%rdi), %xmm0, %xmm0 # sched: [10:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_pmaddwd:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    pmaddwd %xmm1, %xmm0 # sched: [4:0.50]
-; SKYLAKE-SSE-NEXT:    pmaddwd (%rdi), %xmm0 # sched: [10:0.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pmaddwd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpmaddwd %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    vpmaddwd (%rdi), %xmm0, %xmm0 # sched: [10:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_pmaddwd:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    pmaddwd %xmm1, %xmm0 # sched: [4:0.50]
-; SKX-SSE-NEXT:    pmaddwd (%rdi), %xmm0 # sched: [10:0.50]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pmaddwd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpmaddwd %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKX-NEXT:    vpmaddwd (%rdi), %xmm0, %xmm0 # sched: [10:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_pmaddwd:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    pmaddwd %xmm1, %xmm0 # sched: [4:1.00]
-; BDVER2-SSE-NEXT:    pmaddwd (%rdi), %xmm0 # sched: [9:1.00]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_pmaddwd:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vpmaddwd %xmm1, %xmm0, %xmm0 # sched: [4:1.00]
-; BDVER2-NEXT:    vpmaddwd (%rdi), %xmm0, %xmm0 # sched: [9:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_pmaddwd:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    pmaddwd %xmm1, %xmm0 # sched: [2:1.00]
-; BTVER2-SSE-NEXT:    pmaddwd (%rdi), %xmm0 # sched: [7:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_pmaddwd:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vpmaddwd %xmm1, %xmm0, %xmm0 # sched: [2:1.00]
-; BTVER2-NEXT:    vpmaddwd (%rdi), %xmm0, %xmm0 # sched: [7:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_pmaddwd:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    pmaddwd %xmm1, %xmm0 # sched: [4:1.00]
-; ZNVER1-SSE-NEXT:    pmaddwd (%rdi), %xmm0 # sched: [11:1.00]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_pmaddwd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpmaddwd %xmm1, %xmm0, %xmm0 # sched: [4:1.00]
-; ZNVER1-NEXT:    vpmaddwd (%rdi), %xmm0, %xmm0 # sched: [11:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <4 x i32> @llvm.x86.sse2.pmadd.wd(<8 x i16> %a0, <8 x i16> %a1)
-  %2 = bitcast <4 x i32> %1 to <8 x i16>
-  %3 = load <8 x i16>, <8 x i16> *%a2, align 16
-  %4 = call <4 x i32> @llvm.x86.sse2.pmadd.wd(<8 x i16> %2, <8 x i16> %3)
-  ret <4 x i32> %4
-}
-declare <4 x i32> @llvm.x86.sse2.pmadd.wd(<8 x i16>, <8 x i16>) nounwind readnone
-
-define <8 x i16> @test_pmaxsw(<8 x i16> %a0, <8 x i16> %a1, <8 x i16> *%a2) {
-; GENERIC-LABEL: test_pmaxsw:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    pmaxsw %xmm1, %xmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    pmaxsw (%rdi), %xmm0 # sched: [7:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_pmaxsw:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    pmaxsw %xmm1, %xmm0 # sched: [1:0.50]
-; ATOM-NEXT:    pmaxsw (%rdi), %xmm0 # sched: [1:1.00]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_pmaxsw:
-; SLM:       # %bb.0:
-; SLM-NEXT:    pmaxsw %xmm1, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    pmaxsw (%rdi), %xmm0 # sched: [4:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_pmaxsw:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    pmaxsw %xmm1, %xmm0 # sched: [1:0.50]
-; SANDY-SSE-NEXT:    pmaxsw (%rdi), %xmm0 # sched: [7:0.50]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_pmaxsw:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vpmaxsw %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SANDY-NEXT:    vpmaxsw (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_pmaxsw:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    pmaxsw %xmm1, %xmm0 # sched: [1:0.50]
-; HASWELL-SSE-NEXT:    pmaxsw (%rdi), %xmm0 # sched: [7:0.50]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_pmaxsw:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpmaxsw %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; HASWELL-NEXT:    vpmaxsw (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_pmaxsw:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    pmaxsw %xmm1, %xmm0 # sched: [1:0.50]
-; BROADWELL-SSE-NEXT:    pmaxsw (%rdi), %xmm0 # sched: [6:0.50]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pmaxsw:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpmaxsw %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    vpmaxsw (%rdi), %xmm0, %xmm0 # sched: [6:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_pmaxsw:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    pmaxsw %xmm1, %xmm0 # sched: [1:0.50]
-; SKYLAKE-SSE-NEXT:    pmaxsw (%rdi), %xmm0 # sched: [7:0.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pmaxsw:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpmaxsw %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SKYLAKE-NEXT:    vpmaxsw (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_pmaxsw:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    pmaxsw %xmm1, %xmm0 # sched: [1:0.50]
-; SKX-SSE-NEXT:    pmaxsw (%rdi), %xmm0 # sched: [7:0.50]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pmaxsw:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpmaxsw %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SKX-NEXT:    vpmaxsw (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_pmaxsw:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    pmaxsw %xmm1, %xmm0 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    pmaxsw (%rdi), %xmm0 # sched: [7:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_pmaxsw:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vpmaxsw %xmm1, %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    vpmaxsw (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_pmaxsw:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    pmaxsw %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    pmaxsw (%rdi), %xmm0 # sched: [6:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_pmaxsw:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vpmaxsw %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    vpmaxsw (%rdi), %xmm0, %xmm0 # sched: [6:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_pmaxsw:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    pmaxsw %xmm1, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    pmaxsw (%rdi), %xmm0 # sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_pmaxsw:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpmaxsw %xmm1, %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vpmaxsw (%rdi), %xmm0, %xmm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <8 x i16> @llvm.x86.sse2.pmaxs.w(<8 x i16> %a0, <8 x i16> %a1)
-  %2 = load <8 x i16>, <8 x i16> *%a2, align 16
-  %3 = call <8 x i16> @llvm.x86.sse2.pmaxs.w(<8 x i16> %1, <8 x i16> %2)
-  ret <8 x i16> %3
-}
-declare <8 x i16> @llvm.x86.sse2.pmaxs.w(<8 x i16>, <8 x i16>) nounwind readnone
-
-define <16 x i8> @test_pmaxub(<16 x i8> %a0, <16 x i8> %a1, <16 x i8> *%a2) {
-; GENERIC-LABEL: test_pmaxub:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    pmaxub %xmm1, %xmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    pmaxub (%rdi), %xmm0 # sched: [7:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_pmaxub:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    pmaxub %xmm1, %xmm0 # sched: [1:0.50]
-; ATOM-NEXT:    pmaxub (%rdi), %xmm0 # sched: [1:1.00]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_pmaxub:
-; SLM:       # %bb.0:
-; SLM-NEXT:    pmaxub %xmm1, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    pmaxub (%rdi), %xmm0 # sched: [4:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_pmaxub:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    pmaxub %xmm1, %xmm0 # sched: [1:0.50]
-; SANDY-SSE-NEXT:    pmaxub (%rdi), %xmm0 # sched: [7:0.50]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_pmaxub:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vpmaxub %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SANDY-NEXT:    vpmaxub (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_pmaxub:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    pmaxub %xmm1, %xmm0 # sched: [1:0.50]
-; HASWELL-SSE-NEXT:    pmaxub (%rdi), %xmm0 # sched: [7:0.50]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_pmaxub:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpmaxub %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; HASWELL-NEXT:    vpmaxub (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_pmaxub:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    pmaxub %xmm1, %xmm0 # sched: [1:0.50]
-; BROADWELL-SSE-NEXT:    pmaxub (%rdi), %xmm0 # sched: [6:0.50]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pmaxub:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpmaxub %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    vpmaxub (%rdi), %xmm0, %xmm0 # sched: [6:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_pmaxub:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    pmaxub %xmm1, %xmm0 # sched: [1:0.50]
-; SKYLAKE-SSE-NEXT:    pmaxub (%rdi), %xmm0 # sched: [7:0.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pmaxub:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpmaxub %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SKYLAKE-NEXT:    vpmaxub (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_pmaxub:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    pmaxub %xmm1, %xmm0 # sched: [1:0.50]
-; SKX-SSE-NEXT:    pmaxub (%rdi), %xmm0 # sched: [7:0.50]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pmaxub:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpmaxub %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SKX-NEXT:    vpmaxub (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_pmaxub:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    pmaxub %xmm1, %xmm0 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    pmaxub (%rdi), %xmm0 # sched: [7:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_pmaxub:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vpmaxub %xmm1, %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    vpmaxub (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_pmaxub:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    pmaxub %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    pmaxub (%rdi), %xmm0 # sched: [6:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_pmaxub:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vpmaxub %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    vpmaxub (%rdi), %xmm0, %xmm0 # sched: [6:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_pmaxub:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    pmaxub %xmm1, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    pmaxub (%rdi), %xmm0 # sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_pmaxub:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpmaxub %xmm1, %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vpmaxub (%rdi), %xmm0, %xmm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <16 x i8> @llvm.x86.sse2.pmaxu.b(<16 x i8> %a0, <16 x i8> %a1)
-  %2 = load <16 x i8>, <16 x i8> *%a2, align 16
-  %3 = call <16 x i8> @llvm.x86.sse2.pmaxu.b(<16 x i8> %1, <16 x i8> %2)
-  ret <16 x i8> %3
-}
-declare <16 x i8> @llvm.x86.sse2.pmaxu.b(<16 x i8>, <16 x i8>) nounwind readnone
-
-define <8 x i16> @test_pminsw(<8 x i16> %a0, <8 x i16> %a1, <8 x i16> *%a2) {
-; GENERIC-LABEL: test_pminsw:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    pminsw %xmm1, %xmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    pminsw (%rdi), %xmm0 # sched: [7:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_pminsw:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    pminsw %xmm1, %xmm0 # sched: [1:0.50]
-; ATOM-NEXT:    pminsw (%rdi), %xmm0 # sched: [1:1.00]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_pminsw:
-; SLM:       # %bb.0:
-; SLM-NEXT:    pminsw %xmm1, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    pminsw (%rdi), %xmm0 # sched: [4:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_pminsw:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    pminsw %xmm1, %xmm0 # sched: [1:0.50]
-; SANDY-SSE-NEXT:    pminsw (%rdi), %xmm0 # sched: [7:0.50]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_pminsw:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vpminsw %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SANDY-NEXT:    vpminsw (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_pminsw:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    pminsw %xmm1, %xmm0 # sched: [1:0.50]
-; HASWELL-SSE-NEXT:    pminsw (%rdi), %xmm0 # sched: [7:0.50]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_pminsw:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpminsw %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; HASWELL-NEXT:    vpminsw (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_pminsw:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    pminsw %xmm1, %xmm0 # sched: [1:0.50]
-; BROADWELL-SSE-NEXT:    pminsw (%rdi), %xmm0 # sched: [6:0.50]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pminsw:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpminsw %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    vpminsw (%rdi), %xmm0, %xmm0 # sched: [6:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_pminsw:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    pminsw %xmm1, %xmm0 # sched: [1:0.50]
-; SKYLAKE-SSE-NEXT:    pminsw (%rdi), %xmm0 # sched: [7:0.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pminsw:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpminsw %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SKYLAKE-NEXT:    vpminsw (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_pminsw:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    pminsw %xmm1, %xmm0 # sched: [1:0.50]
-; SKX-SSE-NEXT:    pminsw (%rdi), %xmm0 # sched: [7:0.50]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pminsw:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpminsw %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SKX-NEXT:    vpminsw (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_pminsw:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    pminsw %xmm1, %xmm0 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    pminsw (%rdi), %xmm0 # sched: [7:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_pminsw:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vpminsw %xmm1, %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    vpminsw (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_pminsw:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    pminsw %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    pminsw (%rdi), %xmm0 # sched: [6:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_pminsw:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vpminsw %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    vpminsw (%rdi), %xmm0, %xmm0 # sched: [6:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_pminsw:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    pminsw %xmm1, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    pminsw (%rdi), %xmm0 # sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_pminsw:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpminsw %xmm1, %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vpminsw (%rdi), %xmm0, %xmm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <8 x i16> @llvm.x86.sse2.pmins.w(<8 x i16> %a0, <8 x i16> %a1)
-  %2 = load <8 x i16>, <8 x i16> *%a2, align 16
-  %3 = call <8 x i16> @llvm.x86.sse2.pmins.w(<8 x i16> %1, <8 x i16> %2)
-  ret <8 x i16> %3
-}
-declare <8 x i16> @llvm.x86.sse2.pmins.w(<8 x i16>, <8 x i16>) nounwind readnone
-
-define <16 x i8> @test_pminub(<16 x i8> %a0, <16 x i8> %a1, <16 x i8> *%a2) {
-; GENERIC-LABEL: test_pminub:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    pminub %xmm1, %xmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    pminub (%rdi), %xmm0 # sched: [7:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_pminub:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    pminub %xmm1, %xmm0 # sched: [1:0.50]
-; ATOM-NEXT:    pminub (%rdi), %xmm0 # sched: [1:1.00]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_pminub:
-; SLM:       # %bb.0:
-; SLM-NEXT:    pminub %xmm1, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    pminub (%rdi), %xmm0 # sched: [4:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_pminub:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    pminub %xmm1, %xmm0 # sched: [1:0.50]
-; SANDY-SSE-NEXT:    pminub (%rdi), %xmm0 # sched: [7:0.50]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_pminub:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vpminub %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SANDY-NEXT:    vpminub (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_pminub:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    pminub %xmm1, %xmm0 # sched: [1:0.50]
-; HASWELL-SSE-NEXT:    pminub (%rdi), %xmm0 # sched: [7:0.50]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_pminub:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpminub %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; HASWELL-NEXT:    vpminub (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_pminub:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    pminub %xmm1, %xmm0 # sched: [1:0.50]
-; BROADWELL-SSE-NEXT:    pminub (%rdi), %xmm0 # sched: [6:0.50]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pminub:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpminub %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    vpminub (%rdi), %xmm0, %xmm0 # sched: [6:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_pminub:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    pminub %xmm1, %xmm0 # sched: [1:0.50]
-; SKYLAKE-SSE-NEXT:    pminub (%rdi), %xmm0 # sched: [7:0.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pminub:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpminub %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SKYLAKE-NEXT:    vpminub (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_pminub:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    pminub %xmm1, %xmm0 # sched: [1:0.50]
-; SKX-SSE-NEXT:    pminub (%rdi), %xmm0 # sched: [7:0.50]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pminub:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpminub %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SKX-NEXT:    vpminub (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_pminub:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    pminub %xmm1, %xmm0 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    pminub (%rdi), %xmm0 # sched: [7:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_pminub:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vpminub %xmm1, %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    vpminub (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_pminub:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    pminub %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    pminub (%rdi), %xmm0 # sched: [6:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_pminub:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vpminub %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    vpminub (%rdi), %xmm0, %xmm0 # sched: [6:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_pminub:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    pminub %xmm1, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    pminub (%rdi), %xmm0 # sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_pminub:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpminub %xmm1, %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vpminub (%rdi), %xmm0, %xmm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <16 x i8> @llvm.x86.sse2.pminu.b(<16 x i8> %a0, <16 x i8> %a1)
-  %2 = load <16 x i8>, <16 x i8> *%a2, align 16
-  %3 = call <16 x i8> @llvm.x86.sse2.pminu.b(<16 x i8> %1, <16 x i8> %2)
-  ret <16 x i8> %3
-}
-declare <16 x i8> @llvm.x86.sse2.pminu.b(<16 x i8>, <16 x i8>) nounwind readnone
-
-define i32 @test_pmovmskb(<16 x i8> %a0) {
-; GENERIC-LABEL: test_pmovmskb:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    pmovmskb %xmm0, %eax # sched: [2:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_pmovmskb:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    pmovmskb %xmm0, %eax # sched: [3:3.00]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_pmovmskb:
-; SLM:       # %bb.0:
-; SLM-NEXT:    pmovmskb %xmm0, %eax # sched: [4:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_pmovmskb:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    pmovmskb %xmm0, %eax # sched: [2:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_pmovmskb:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vpmovmskb %xmm0, %eax # sched: [2:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_pmovmskb:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    pmovmskb %xmm0, %eax # sched: [3:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_pmovmskb:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpmovmskb %xmm0, %eax # sched: [3:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_pmovmskb:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    pmovmskb %xmm0, %eax # sched: [3:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pmovmskb:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpmovmskb %xmm0, %eax # sched: [3:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_pmovmskb:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    pmovmskb %xmm0, %eax # sched: [2:1.00]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pmovmskb:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpmovmskb %xmm0, %eax # sched: [2:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_pmovmskb:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    pmovmskb %xmm0, %eax # sched: [2:1.00]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pmovmskb:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpmovmskb %xmm0, %eax # sched: [2:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_pmovmskb:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    pmovmskb %xmm0, %eax # sched: [13:1.00]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_pmovmskb:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vpmovmskb %xmm0, %eax # sched: [13:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_pmovmskb:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    pmovmskb %xmm0, %eax # sched: [3:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_pmovmskb:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vpmovmskb %xmm0, %eax # sched: [3:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_pmovmskb:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    pmovmskb %xmm0, %eax # sched: [1:1.00]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_pmovmskb:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpmovmskb %xmm0, %eax # sched: [1:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call i32 @llvm.x86.sse2.pmovmskb.128(<16 x i8> %a0)
-  ret i32 %1
-}
-declare i32 @llvm.x86.sse2.pmovmskb.128(<16 x i8>) nounwind readnone
-
-define <8 x i16> @test_pmulhuw(<8 x i16> %a0, <8 x i16> %a1, <8 x i16> *%a2) {
-; GENERIC-LABEL: test_pmulhuw:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    pmulhuw %xmm1, %xmm0 # sched: [5:1.00]
-; GENERIC-NEXT:    pmulhuw (%rdi), %xmm0 # sched: [11:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_pmulhuw:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    pmulhuw %xmm1, %xmm0 # sched: [5:5.00]
-; ATOM-NEXT:    pmulhuw (%rdi), %xmm0 # sched: [5:5.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_pmulhuw:
-; SLM:       # %bb.0:
-; SLM-NEXT:    pmulhuw %xmm1, %xmm0 # sched: [4:1.00]
-; SLM-NEXT:    pmulhuw (%rdi), %xmm0 # sched: [7:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_pmulhuw:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    pmulhuw %xmm1, %xmm0 # sched: [5:1.00]
-; SANDY-SSE-NEXT:    pmulhuw (%rdi), %xmm0 # sched: [11:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_pmulhuw:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vpmulhuw %xmm1, %xmm0, %xmm0 # sched: [5:1.00]
-; SANDY-NEXT:    vpmulhuw (%rdi), %xmm0, %xmm0 # sched: [11:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_pmulhuw:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    pmulhuw %xmm1, %xmm0 # sched: [5:1.00]
-; HASWELL-SSE-NEXT:    pmulhuw (%rdi), %xmm0 # sched: [11:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_pmulhuw:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpmulhuw %xmm1, %xmm0, %xmm0 # sched: [5:1.00]
-; HASWELL-NEXT:    vpmulhuw (%rdi), %xmm0, %xmm0 # sched: [11:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_pmulhuw:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    pmulhuw %xmm1, %xmm0 # sched: [5:1.00]
-; BROADWELL-SSE-NEXT:    pmulhuw (%rdi), %xmm0 # sched: [10:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pmulhuw:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpmulhuw %xmm1, %xmm0, %xmm0 # sched: [5:1.00]
-; BROADWELL-NEXT:    vpmulhuw (%rdi), %xmm0, %xmm0 # sched: [10:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_pmulhuw:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    pmulhuw %xmm1, %xmm0 # sched: [4:0.50]
-; SKYLAKE-SSE-NEXT:    pmulhuw (%rdi), %xmm0 # sched: [10:0.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pmulhuw:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpmulhuw %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    vpmulhuw (%rdi), %xmm0, %xmm0 # sched: [10:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_pmulhuw:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    pmulhuw %xmm1, %xmm0 # sched: [4:0.50]
-; SKX-SSE-NEXT:    pmulhuw (%rdi), %xmm0 # sched: [10:0.50]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pmulhuw:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpmulhuw %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKX-NEXT:    vpmulhuw (%rdi), %xmm0, %xmm0 # sched: [10:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_pmulhuw:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    pmulhuw %xmm1, %xmm0 # sched: [4:1.00]
-; BDVER2-SSE-NEXT:    pmulhuw (%rdi), %xmm0 # sched: [9:1.00]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_pmulhuw:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vpmulhuw %xmm1, %xmm0, %xmm0 # sched: [4:1.00]
-; BDVER2-NEXT:    vpmulhuw (%rdi), %xmm0, %xmm0 # sched: [9:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_pmulhuw:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    pmulhuw %xmm1, %xmm0 # sched: [2:1.00]
-; BTVER2-SSE-NEXT:    pmulhuw (%rdi), %xmm0 # sched: [7:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_pmulhuw:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vpmulhuw %xmm1, %xmm0, %xmm0 # sched: [2:1.00]
-; BTVER2-NEXT:    vpmulhuw (%rdi), %xmm0, %xmm0 # sched: [7:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_pmulhuw:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    pmulhuw %xmm1, %xmm0 # sched: [4:1.00]
-; ZNVER1-SSE-NEXT:    pmulhuw (%rdi), %xmm0 # sched: [11:1.00]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_pmulhuw:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpmulhuw %xmm1, %xmm0, %xmm0 # sched: [4:1.00]
-; ZNVER1-NEXT:    vpmulhuw (%rdi), %xmm0, %xmm0 # sched: [11:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <8 x i16> @llvm.x86.sse2.pmulhu.w(<8 x i16> %a0, <8 x i16> %a1)
-  %2 = load <8 x i16>, <8 x i16> *%a2, align 16
-  %3 = call <8 x i16> @llvm.x86.sse2.pmulhu.w(<8 x i16> %1, <8 x i16> %2)
-  ret <8 x i16> %3
-}
-declare <8 x i16> @llvm.x86.sse2.pmulhu.w(<8 x i16>, <8 x i16>) nounwind readnone
-
-define <8 x i16> @test_pmulhw(<8 x i16> %a0, <8 x i16> %a1, <8 x i16> *%a2) {
-; GENERIC-LABEL: test_pmulhw:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    pmulhw %xmm1, %xmm0 # sched: [5:1.00]
-; GENERIC-NEXT:    pmulhw (%rdi), %xmm0 # sched: [11:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_pmulhw:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    pmulhw %xmm1, %xmm0 # sched: [5:5.00]
-; ATOM-NEXT:    pmulhw (%rdi), %xmm0 # sched: [5:5.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_pmulhw:
-; SLM:       # %bb.0:
-; SLM-NEXT:    pmulhw %xmm1, %xmm0 # sched: [4:1.00]
-; SLM-NEXT:    pmulhw (%rdi), %xmm0 # sched: [7:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_pmulhw:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    pmulhw %xmm1, %xmm0 # sched: [5:1.00]
-; SANDY-SSE-NEXT:    pmulhw (%rdi), %xmm0 # sched: [11:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_pmulhw:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vpmulhw %xmm1, %xmm0, %xmm0 # sched: [5:1.00]
-; SANDY-NEXT:    vpmulhw (%rdi), %xmm0, %xmm0 # sched: [11:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_pmulhw:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    pmulhw %xmm1, %xmm0 # sched: [5:1.00]
-; HASWELL-SSE-NEXT:    pmulhw (%rdi), %xmm0 # sched: [11:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_pmulhw:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpmulhw %xmm1, %xmm0, %xmm0 # sched: [5:1.00]
-; HASWELL-NEXT:    vpmulhw (%rdi), %xmm0, %xmm0 # sched: [11:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_pmulhw:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    pmulhw %xmm1, %xmm0 # sched: [5:1.00]
-; BROADWELL-SSE-NEXT:    pmulhw (%rdi), %xmm0 # sched: [10:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pmulhw:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpmulhw %xmm1, %xmm0, %xmm0 # sched: [5:1.00]
-; BROADWELL-NEXT:    vpmulhw (%rdi), %xmm0, %xmm0 # sched: [10:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_pmulhw:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    pmulhw %xmm1, %xmm0 # sched: [4:0.50]
-; SKYLAKE-SSE-NEXT:    pmulhw (%rdi), %xmm0 # sched: [10:0.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pmulhw:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpmulhw %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    vpmulhw (%rdi), %xmm0, %xmm0 # sched: [10:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_pmulhw:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    pmulhw %xmm1, %xmm0 # sched: [4:0.50]
-; SKX-SSE-NEXT:    pmulhw (%rdi), %xmm0 # sched: [10:0.50]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pmulhw:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpmulhw %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKX-NEXT:    vpmulhw (%rdi), %xmm0, %xmm0 # sched: [10:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_pmulhw:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    pmulhw %xmm1, %xmm0 # sched: [4:1.00]
-; BDVER2-SSE-NEXT:    pmulhw (%rdi), %xmm0 # sched: [9:1.00]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_pmulhw:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vpmulhw %xmm1, %xmm0, %xmm0 # sched: [4:1.00]
-; BDVER2-NEXT:    vpmulhw (%rdi), %xmm0, %xmm0 # sched: [9:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_pmulhw:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    pmulhw %xmm1, %xmm0 # sched: [2:1.00]
-; BTVER2-SSE-NEXT:    pmulhw (%rdi), %xmm0 # sched: [7:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_pmulhw:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vpmulhw %xmm1, %xmm0, %xmm0 # sched: [2:1.00]
-; BTVER2-NEXT:    vpmulhw (%rdi), %xmm0, %xmm0 # sched: [7:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_pmulhw:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    pmulhw %xmm1, %xmm0 # sched: [4:1.00]
-; ZNVER1-SSE-NEXT:    pmulhw (%rdi), %xmm0 # sched: [11:1.00]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_pmulhw:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpmulhw %xmm1, %xmm0, %xmm0 # sched: [4:1.00]
-; ZNVER1-NEXT:    vpmulhw (%rdi), %xmm0, %xmm0 # sched: [11:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <8 x i16> @llvm.x86.sse2.pmulh.w(<8 x i16> %a0, <8 x i16> %a1)
-  %2 = load <8 x i16>, <8 x i16> *%a2, align 16
-  %3 = call <8 x i16> @llvm.x86.sse2.pmulh.w(<8 x i16> %1, <8 x i16> %2)
-  ret <8 x i16> %3
-}
-declare <8 x i16> @llvm.x86.sse2.pmulh.w(<8 x i16>, <8 x i16>) nounwind readnone
-
-define <8 x i16> @test_pmullw(<8 x i16> %a0, <8 x i16> %a1, <8 x i16> *%a2) {
-; GENERIC-LABEL: test_pmullw:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    pmullw %xmm1, %xmm0 # sched: [5:1.00]
-; GENERIC-NEXT:    pmullw (%rdi), %xmm0 # sched: [11:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_pmullw:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    pmullw %xmm1, %xmm0 # sched: [5:5.00]
-; ATOM-NEXT:    pmullw (%rdi), %xmm0 # sched: [5:5.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_pmullw:
-; SLM:       # %bb.0:
-; SLM-NEXT:    pmullw %xmm1, %xmm0 # sched: [4:1.00]
-; SLM-NEXT:    pmullw (%rdi), %xmm0 # sched: [7:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_pmullw:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    pmullw %xmm1, %xmm0 # sched: [5:1.00]
-; SANDY-SSE-NEXT:    pmullw (%rdi), %xmm0 # sched: [11:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_pmullw:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vpmullw %xmm1, %xmm0, %xmm0 # sched: [5:1.00]
-; SANDY-NEXT:    vpmullw (%rdi), %xmm0, %xmm0 # sched: [11:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_pmullw:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    pmullw %xmm1, %xmm0 # sched: [5:1.00]
-; HASWELL-SSE-NEXT:    pmullw (%rdi), %xmm0 # sched: [11:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_pmullw:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0 # sched: [5:1.00]
-; HASWELL-NEXT:    vpmullw (%rdi), %xmm0, %xmm0 # sched: [11:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_pmullw:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    pmullw %xmm1, %xmm0 # sched: [5:1.00]
-; BROADWELL-SSE-NEXT:    pmullw (%rdi), %xmm0 # sched: [10:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pmullw:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0 # sched: [5:1.00]
-; BROADWELL-NEXT:    vpmullw (%rdi), %xmm0, %xmm0 # sched: [10:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_pmullw:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    pmullw %xmm1, %xmm0 # sched: [4:0.50]
-; SKYLAKE-SSE-NEXT:    pmullw (%rdi), %xmm0 # sched: [10:0.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pmullw:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpmullw %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    vpmullw (%rdi), %xmm0, %xmm0 # sched: [10:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_pmullw:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    pmullw %xmm1, %xmm0 # sched: [4:0.50]
-; SKX-SSE-NEXT:    pmullw (%rdi), %xmm0 # sched: [10:0.50]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pmullw:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpmullw %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKX-NEXT:    vpmullw (%rdi), %xmm0, %xmm0 # sched: [10:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_pmullw:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    pmullw %xmm1, %xmm0 # sched: [4:1.00]
-; BDVER2-SSE-NEXT:    pmullw (%rdi), %xmm0 # sched: [9:1.00]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_pmullw:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vpmullw %xmm1, %xmm0, %xmm0 # sched: [4:1.00]
-; BDVER2-NEXT:    vpmullw (%rdi), %xmm0, %xmm0 # sched: [9:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_pmullw:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    pmullw %xmm1, %xmm0 # sched: [2:1.00]
-; BTVER2-SSE-NEXT:    pmullw (%rdi), %xmm0 # sched: [7:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_pmullw:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vpmullw %xmm1, %xmm0, %xmm0 # sched: [2:1.00]
-; BTVER2-NEXT:    vpmullw (%rdi), %xmm0, %xmm0 # sched: [7:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_pmullw:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    pmullw %xmm1, %xmm0 # sched: [4:1.00]
-; ZNVER1-SSE-NEXT:    pmullw (%rdi), %xmm0 # sched: [11:1.00]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_pmullw:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0 # sched: [4:1.00]
-; ZNVER1-NEXT:    vpmullw (%rdi), %xmm0, %xmm0 # sched: [11:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = mul <8 x i16> %a0, %a1
-  %2 = load <8 x i16>, <8 x i16> *%a2, align 16
-  %3 = mul <8 x i16> %1, %2
-  ret <8 x i16> %3
-}
-
-define <2 x i64> @test_pmuludq(<4 x i32> %a0, <4 x i32> %a1, <4 x i32> *%a2) {
-; GENERIC-LABEL: test_pmuludq:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    pmuludq %xmm1, %xmm0 # sched: [5:1.00]
-; GENERIC-NEXT:    pmuludq (%rdi), %xmm0 # sched: [11:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_pmuludq:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    pmuludq %xmm1, %xmm0 # sched: [5:5.00]
-; ATOM-NEXT:    pmuludq (%rdi), %xmm0 # sched: [5:5.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_pmuludq:
-; SLM:       # %bb.0:
-; SLM-NEXT:    pmuludq %xmm1, %xmm0 # sched: [4:1.00]
-; SLM-NEXT:    pmuludq (%rdi), %xmm0 # sched: [7:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_pmuludq:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    pmuludq %xmm1, %xmm0 # sched: [5:1.00]
-; SANDY-SSE-NEXT:    pmuludq (%rdi), %xmm0 # sched: [11:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_pmuludq:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0 # sched: [5:1.00]
-; SANDY-NEXT:    vpmuludq (%rdi), %xmm0, %xmm0 # sched: [11:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_pmuludq:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    pmuludq %xmm1, %xmm0 # sched: [5:1.00]
-; HASWELL-SSE-NEXT:    pmuludq (%rdi), %xmm0 # sched: [11:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_pmuludq:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0 # sched: [5:1.00]
-; HASWELL-NEXT:    vpmuludq (%rdi), %xmm0, %xmm0 # sched: [11:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_pmuludq:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    pmuludq %xmm1, %xmm0 # sched: [5:1.00]
-; BROADWELL-SSE-NEXT:    pmuludq (%rdi), %xmm0 # sched: [10:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pmuludq:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0 # sched: [5:1.00]
-; BROADWELL-NEXT:    vpmuludq (%rdi), %xmm0, %xmm0 # sched: [10:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_pmuludq:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    pmuludq %xmm1, %xmm0 # sched: [4:0.50]
-; SKYLAKE-SSE-NEXT:    pmuludq (%rdi), %xmm0 # sched: [10:0.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pmuludq:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    vpmuludq (%rdi), %xmm0, %xmm0 # sched: [10:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_pmuludq:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    pmuludq %xmm1, %xmm0 # sched: [4:0.50]
-; SKX-SSE-NEXT:    pmuludq (%rdi), %xmm0 # sched: [10:0.50]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pmuludq:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKX-NEXT:    vpmuludq (%rdi), %xmm0, %xmm0 # sched: [10:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_pmuludq:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    pmuludq %xmm1, %xmm0 # sched: [4:1.00]
-; BDVER2-SSE-NEXT:    pmuludq (%rdi), %xmm0 # sched: [9:1.00]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_pmuludq:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0 # sched: [4:1.00]
-; BDVER2-NEXT:    vpmuludq (%rdi), %xmm0, %xmm0 # sched: [9:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_pmuludq:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    pmuludq %xmm1, %xmm0 # sched: [2:1.00]
-; BTVER2-SSE-NEXT:    pmuludq (%rdi), %xmm0 # sched: [7:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_pmuludq:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0 # sched: [2:1.00]
-; BTVER2-NEXT:    vpmuludq (%rdi), %xmm0, %xmm0 # sched: [7:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_pmuludq:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    pmuludq %xmm1, %xmm0 # sched: [4:1.00]
-; ZNVER1-SSE-NEXT:    pmuludq (%rdi), %xmm0 # sched: [11:1.00]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_pmuludq:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0 # sched: [4:1.00]
-; ZNVER1-NEXT:    vpmuludq (%rdi), %xmm0, %xmm0 # sched: [11:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <2 x i64> @llvm.x86.sse2.pmulu.dq(<4 x i32> %a0, <4 x i32> %a1)
-  %2 = bitcast <2 x i64> %1 to <4 x i32>
-  %3 = load <4 x i32>, <4 x i32> *%a2, align 16
-  %4 = call <2 x i64> @llvm.x86.sse2.pmulu.dq(<4 x i32> %2, <4 x i32> %3)
-  ret <2 x i64> %4
-}
-declare <2 x i64> @llvm.x86.sse2.pmulu.dq(<4 x i32>, <4 x i32>) nounwind readnone
-
-define <2 x i64> @test_por(<2 x i64> %a0, <2 x i64> %a1, <2 x i64> *%a2) {
-; GENERIC-LABEL: test_por:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    por %xmm1, %xmm0 # sched: [1:0.33]
-; GENERIC-NEXT:    por (%rdi), %xmm0 # sched: [7:0.50]
-; GENERIC-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_por:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    por %xmm1, %xmm0 # sched: [1:0.50]
-; ATOM-NEXT:    por (%rdi), %xmm0 # sched: [1:1.00]
-; ATOM-NEXT:    paddq %xmm1, %xmm0 # sched: [2:1.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_por:
-; SLM:       # %bb.0:
-; SLM-NEXT:    por %xmm1, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    por (%rdi), %xmm0 # sched: [4:1.00]
-; SLM-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_por:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    por %xmm1, %xmm0 # sched: [1:0.33]
-; SANDY-SSE-NEXT:    por (%rdi), %xmm0 # sched: [7:0.50]
-; SANDY-SSE-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.50]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_por:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vpor %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SANDY-NEXT:    vpor (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SANDY-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_por:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    por %xmm1, %xmm0 # sched: [1:0.33]
-; HASWELL-SSE-NEXT:    por (%rdi), %xmm0 # sched: [7:0.50]
-; HASWELL-SSE-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.50]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_por:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpor %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; HASWELL-NEXT:    vpor (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; HASWELL-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_por:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    por %xmm1, %xmm0 # sched: [1:0.33]
-; BROADWELL-SSE-NEXT:    por (%rdi), %xmm0 # sched: [6:0.50]
-; BROADWELL-SSE-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.50]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_por:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpor %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; BROADWELL-NEXT:    vpor (%rdi), %xmm0, %xmm0 # sched: [6:0.50]
-; BROADWELL-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_por:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    por %xmm1, %xmm0 # sched: [1:0.33]
-; SKYLAKE-SSE-NEXT:    por (%rdi), %xmm0 # sched: [7:0.50]
-; SKYLAKE-SSE-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.33]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_por:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpor %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    vpor (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKYLAKE-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_por:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    por %xmm1, %xmm0 # sched: [1:0.33]
-; SKX-SSE-NEXT:    por (%rdi), %xmm0 # sched: [7:0.50]
-; SKX-SSE-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.33]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_por:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpor %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    vpor (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKX-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_por:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    por %xmm1, %xmm0 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    por (%rdi), %xmm0 # sched: [7:0.50]
-; BDVER2-SSE-NEXT:    paddq %xmm1, %xmm0 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_por:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vpor %xmm1, %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    vpor (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; BDVER2-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_por:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    por %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    por (%rdi), %xmm0 # sched: [6:1.00]
-; BTVER2-SSE-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_por:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vpor %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    vpor (%rdi), %xmm0, %xmm0 # sched: [6:1.00]
-; BTVER2-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_por:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    por %xmm1, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    por (%rdi), %xmm0 # sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_por:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpor %xmm1, %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vpor (%rdi), %xmm0, %xmm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = or <2 x i64> %a0, %a1
-  %2 = load <2 x i64>, <2 x i64> *%a2, align 16
-  %3 = or <2 x i64> %1, %2
-  %4 = add <2 x i64> %3, %a1
-  ret <2 x i64> %4
-}
-
-define <2 x i64> @test_psadbw(<16 x i8> %a0, <16 x i8> %a1, <16 x i8> *%a2) {
-; GENERIC-LABEL: test_psadbw:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    psadbw %xmm1, %xmm0 # sched: [5:1.00]
-; GENERIC-NEXT:    psadbw (%rdi), %xmm0 # sched: [11:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_psadbw:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    psadbw %xmm1, %xmm0 # sched: [5:5.00]
-; ATOM-NEXT:    psadbw (%rdi), %xmm0 # sched: [5:5.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_psadbw:
-; SLM:       # %bb.0:
-; SLM-NEXT:    psadbw %xmm1, %xmm0 # sched: [4:1.00]
-; SLM-NEXT:    psadbw (%rdi), %xmm0 # sched: [7:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_psadbw:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    psadbw %xmm1, %xmm0 # sched: [5:1.00]
-; SANDY-SSE-NEXT:    psadbw (%rdi), %xmm0 # sched: [11:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_psadbw:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vpsadbw %xmm1, %xmm0, %xmm0 # sched: [5:1.00]
-; SANDY-NEXT:    vpsadbw (%rdi), %xmm0, %xmm0 # sched: [11:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_psadbw:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    psadbw %xmm1, %xmm0 # sched: [5:1.00]
-; HASWELL-SSE-NEXT:    psadbw (%rdi), %xmm0 # sched: [11:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_psadbw:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpsadbw %xmm1, %xmm0, %xmm0 # sched: [5:1.00]
-; HASWELL-NEXT:    vpsadbw (%rdi), %xmm0, %xmm0 # sched: [11:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_psadbw:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    psadbw %xmm1, %xmm0 # sched: [5:1.00]
-; BROADWELL-SSE-NEXT:    psadbw (%rdi), %xmm0 # sched: [10:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_psadbw:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpsadbw %xmm1, %xmm0, %xmm0 # sched: [5:1.00]
-; BROADWELL-NEXT:    vpsadbw (%rdi), %xmm0, %xmm0 # sched: [10:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_psadbw:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    psadbw %xmm1, %xmm0 # sched: [3:1.00]
-; SKYLAKE-SSE-NEXT:    psadbw (%rdi), %xmm0 # sched: [9:1.00]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_psadbw:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpsadbw %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; SKYLAKE-NEXT:    vpsadbw (%rdi), %xmm0, %xmm0 # sched: [9:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_psadbw:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    psadbw %xmm1, %xmm0 # sched: [3:1.00]
-; SKX-SSE-NEXT:    psadbw (%rdi), %xmm0 # sched: [9:1.00]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_psadbw:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpsadbw %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; SKX-NEXT:    vpsadbw (%rdi), %xmm0, %xmm0 # sched: [9:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_psadbw:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    psadbw %xmm1, %xmm0 # sched: [4:0.50]
-; BDVER2-SSE-NEXT:    psadbw (%rdi), %xmm0 # sched: [9:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_psadbw:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vpsadbw %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; BDVER2-NEXT:    vpsadbw (%rdi), %xmm0, %xmm0 # sched: [9:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_psadbw:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    psadbw %xmm1, %xmm0 # sched: [2:0.50]
-; BTVER2-SSE-NEXT:    psadbw (%rdi), %xmm0 # sched: [7:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_psadbw:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vpsadbw %xmm1, %xmm0, %xmm0 # sched: [2:0.50]
-; BTVER2-NEXT:    vpsadbw (%rdi), %xmm0, %xmm0 # sched: [7:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_psadbw:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    psadbw %xmm1, %xmm0 # sched: [3:1.00]
-; ZNVER1-SSE-NEXT:    psadbw (%rdi), %xmm0 # sched: [10:1.00]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_psadbw:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpsadbw %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    vpsadbw (%rdi), %xmm0, %xmm0 # sched: [10:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <2 x i64> @llvm.x86.sse2.psad.bw(<16 x i8> %a0, <16 x i8> %a1)
-  %2 = bitcast <2 x i64> %1 to <16 x i8>
-  %3 = load <16 x i8>, <16 x i8> *%a2, align 16
-  %4 = call <2 x i64> @llvm.x86.sse2.psad.bw(<16 x i8> %2, <16 x i8> %3)
-  ret <2 x i64> %4
-}
-declare <2 x i64> @llvm.x86.sse2.psad.bw(<16 x i8>, <16 x i8>) nounwind readnone
-
-define <4 x i32> @test_pshufd(<4 x i32> %a0, <4 x i32> *%a1) {
-; GENERIC-LABEL: test_pshufd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,0,3,2] sched: [1:0.50]
-; GENERIC-NEXT:    pshufd {{.*#+}} xmm0 = mem[3,2,1,0] sched: [7:0.50]
-; GENERIC-NEXT:    paddd %xmm1, %xmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_pshufd:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,0,3,2] sched: [1:1.00]
-; ATOM-NEXT:    pshufd {{.*#+}} xmm0 = mem[3,2,1,0] sched: [1:1.00]
-; ATOM-NEXT:    paddd %xmm1, %xmm0 # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_pshufd:
-; SLM:       # %bb.0:
-; SLM-NEXT:    pshufd {{.*#+}} xmm1 = mem[3,2,1,0] sched: [4:1.00]
-; SLM-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,0,3,2] sched: [1:1.00]
-; SLM-NEXT:    paddd %xmm0, %xmm1 # sched: [1:0.50]
-; SLM-NEXT:    movdqa %xmm1, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_pshufd:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,0,3,2] sched: [1:0.50]
-; SANDY-SSE-NEXT:    pshufd {{.*#+}} xmm0 = mem[3,2,1,0] sched: [7:0.50]
-; SANDY-SSE-NEXT:    paddd %xmm1, %xmm0 # sched: [1:0.50]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_pshufd:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,0,3,2] sched: [1:0.50]
-; SANDY-NEXT:    vpshufd {{.*#+}} xmm1 = mem[3,2,1,0] sched: [7:0.50]
-; SANDY-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_pshufd:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,0,3,2] sched: [1:1.00]
-; HASWELL-SSE-NEXT:    pshufd {{.*#+}} xmm0 = mem[3,2,1,0] sched: [7:1.00]
-; HASWELL-SSE-NEXT:    paddd %xmm1, %xmm0 # sched: [1:0.50]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_pshufd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,0,3,2] sched: [1:1.00]
-; HASWELL-NEXT:    vpshufd {{.*#+}} xmm1 = mem[3,2,1,0] sched: [7:1.00]
-; HASWELL-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_pshufd:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,0,3,2] sched: [1:1.00]
-; BROADWELL-SSE-NEXT:    pshufd {{.*#+}} xmm0 = mem[3,2,1,0] sched: [6:1.00]
-; BROADWELL-SSE-NEXT:    paddd %xmm1, %xmm0 # sched: [1:0.50]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pshufd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,0,3,2] sched: [1:1.00]
-; BROADWELL-NEXT:    vpshufd {{.*#+}} xmm1 = mem[3,2,1,0] sched: [6:1.00]
-; BROADWELL-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_pshufd:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,0,3,2] sched: [1:1.00]
-; SKYLAKE-SSE-NEXT:    pshufd {{.*#+}} xmm0 = mem[3,2,1,0] sched: [7:1.00]
-; SKYLAKE-SSE-NEXT:    paddd %xmm1, %xmm0 # sched: [1:0.33]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pshufd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,0,3,2] sched: [1:1.00]
-; SKYLAKE-NEXT:    vpshufd {{.*#+}} xmm1 = mem[3,2,1,0] sched: [7:1.00]
-; SKYLAKE-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_pshufd:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,0,3,2] sched: [1:1.00]
-; SKX-SSE-NEXT:    pshufd {{.*#+}} xmm0 = mem[3,2,1,0] sched: [7:1.00]
-; SKX-SSE-NEXT:    paddd %xmm1, %xmm0 # sched: [1:0.33]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pshufd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,0,3,2] sched: [1:1.00]
-; SKX-NEXT:    vpshufd {{.*#+}} xmm1 = mem[3,2,1,0] sched: [7:1.00]
-; SKX-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_pshufd:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,0,3,2] sched: [2:0.50]
-; BDVER2-SSE-NEXT:    pshufd {{.*#+}} xmm0 = mem[3,2,1,0] sched: [7:0.50]
-; BDVER2-SSE-NEXT:    paddd %xmm1, %xmm0 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_pshufd:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vpshufd {{.*#+}} xmm1 = mem[3,2,1,0] sched: [7:0.50]
-; BDVER2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,0,3,2] sched: [2:0.50]
-; BDVER2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_pshufd:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,0,3,2] sched: [1:0.50]
-; BTVER2-SSE-NEXT:    pshufd {{.*#+}} xmm0 = mem[3,2,1,0] sched: [6:1.00]
-; BTVER2-SSE-NEXT:    paddd %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_pshufd:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vpshufd {{.*#+}} xmm1 = mem[3,2,1,0] sched: [6:1.00]
-; BTVER2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,0,3,2] sched: [1:0.50]
-; BTVER2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_pshufd:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,0,3,2] sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    pshufd {{.*#+}} xmm0 = mem[3,2,1,0] sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    paddd %xmm1, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_pshufd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpshufd {{.*#+}} xmm1 = mem[3,2,1,0] sched: [8:0.50]
-; ZNVER1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,0,3,2] sched: [1:0.25]
-; ZNVER1-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = shufflevector <4 x i32> %a0, <4 x i32> undef, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
-  %2 = load <4 x i32>, <4 x i32> *%a1, align 16
-  %3 = shufflevector <4 x i32> %2, <4 x i32> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
-  %4 = add <4 x i32> %1, %3
-  ret <4 x i32> %4
-}
-
-define <8 x i16> @test_pshufhw(<8 x i16> %a0, <8 x i16> *%a1) {
-; GENERIC-LABEL: test_pshufhw:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    pshufhw {{.*#+}} xmm1 = xmm0[0,1,2,3,5,4,7,6] sched: [1:0.50]
-; GENERIC-NEXT:    pshufhw {{.*#+}} xmm0 = mem[0,1,2,3,7,6,5,4] sched: [7:0.50]
-; GENERIC-NEXT:    paddw %xmm1, %xmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_pshufhw:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    pshufhw {{.*#+}} xmm1 = xmm0[0,1,2,3,5,4,7,6] sched: [1:1.00]
-; ATOM-NEXT:    pshufhw {{.*#+}} xmm0 = mem[0,1,2,3,7,6,5,4] sched: [1:1.00]
-; ATOM-NEXT:    paddw %xmm1, %xmm0 # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_pshufhw:
-; SLM:       # %bb.0:
-; SLM-NEXT:    pshufhw {{.*#+}} xmm1 = mem[0,1,2,3,7,6,5,4] sched: [4:1.00]
-; SLM-NEXT:    pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,5,4,7,6] sched: [1:1.00]
-; SLM-NEXT:    paddw %xmm0, %xmm1 # sched: [1:0.50]
-; SLM-NEXT:    movdqa %xmm1, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_pshufhw:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    pshufhw {{.*#+}} xmm1 = xmm0[0,1,2,3,5,4,7,6] sched: [1:0.50]
-; SANDY-SSE-NEXT:    pshufhw {{.*#+}} xmm0 = mem[0,1,2,3,7,6,5,4] sched: [7:0.50]
-; SANDY-SSE-NEXT:    paddw %xmm1, %xmm0 # sched: [1:0.50]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_pshufhw:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,5,4,7,6] sched: [1:0.50]
-; SANDY-NEXT:    vpshufhw {{.*#+}} xmm1 = mem[0,1,2,3,7,6,5,4] sched: [7:0.50]
-; SANDY-NEXT:    vpaddw %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_pshufhw:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    pshufhw {{.*#+}} xmm1 = xmm0[0,1,2,3,5,4,7,6] sched: [1:1.00]
-; HASWELL-SSE-NEXT:    pshufhw {{.*#+}} xmm0 = mem[0,1,2,3,7,6,5,4] sched: [7:1.00]
-; HASWELL-SSE-NEXT:    paddw %xmm1, %xmm0 # sched: [1:0.50]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_pshufhw:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,5,4,7,6] sched: [1:1.00]
-; HASWELL-NEXT:    vpshufhw {{.*#+}} xmm1 = mem[0,1,2,3,7,6,5,4] sched: [7:1.00]
-; HASWELL-NEXT:    vpaddw %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_pshufhw:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    pshufhw {{.*#+}} xmm1 = xmm0[0,1,2,3,5,4,7,6] sched: [1:1.00]
-; BROADWELL-SSE-NEXT:    pshufhw {{.*#+}} xmm0 = mem[0,1,2,3,7,6,5,4] sched: [6:1.00]
-; BROADWELL-SSE-NEXT:    paddw %xmm1, %xmm0 # sched: [1:0.50]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pshufhw:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,5,4,7,6] sched: [1:1.00]
-; BROADWELL-NEXT:    vpshufhw {{.*#+}} xmm1 = mem[0,1,2,3,7,6,5,4] sched: [6:1.00]
-; BROADWELL-NEXT:    vpaddw %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_pshufhw:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    pshufhw {{.*#+}} xmm1 = xmm0[0,1,2,3,5,4,7,6] sched: [1:1.00]
-; SKYLAKE-SSE-NEXT:    pshufhw {{.*#+}} xmm0 = mem[0,1,2,3,7,6,5,4] sched: [7:1.00]
-; SKYLAKE-SSE-NEXT:    paddw %xmm1, %xmm0 # sched: [1:0.33]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pshufhw:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,5,4,7,6] sched: [1:1.00]
-; SKYLAKE-NEXT:    vpshufhw {{.*#+}} xmm1 = mem[0,1,2,3,7,6,5,4] sched: [7:1.00]
-; SKYLAKE-NEXT:    vpaddw %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_pshufhw:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    pshufhw {{.*#+}} xmm1 = xmm0[0,1,2,3,5,4,7,6] sched: [1:1.00]
-; SKX-SSE-NEXT:    pshufhw {{.*#+}} xmm0 = mem[0,1,2,3,7,6,5,4] sched: [7:1.00]
-; SKX-SSE-NEXT:    paddw %xmm1, %xmm0 # sched: [1:0.33]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pshufhw:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,5,4,7,6] sched: [1:1.00]
-; SKX-NEXT:    vpshufhw {{.*#+}} xmm1 = mem[0,1,2,3,7,6,5,4] sched: [7:1.00]
-; SKX-NEXT:    vpaddw %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_pshufhw:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    pshufhw {{.*#+}} xmm1 = xmm0[0,1,2,3,5,4,7,6] sched: [2:0.50]
-; BDVER2-SSE-NEXT:    pshufhw {{.*#+}} xmm0 = mem[0,1,2,3,7,6,5,4] sched: [7:0.50]
-; BDVER2-SSE-NEXT:    paddw %xmm1, %xmm0 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_pshufhw:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vpshufhw {{.*#+}} xmm1 = mem[0,1,2,3,7,6,5,4] sched: [7:0.50]
-; BDVER2-NEXT:    vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,5,4,7,6] sched: [2:0.50]
-; BDVER2-NEXT:    vpaddw %xmm1, %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_pshufhw:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    pshufhw {{.*#+}} xmm1 = xmm0[0,1,2,3,5,4,7,6] sched: [1:0.50]
-; BTVER2-SSE-NEXT:    pshufhw {{.*#+}} xmm0 = mem[0,1,2,3,7,6,5,4] sched: [6:1.00]
-; BTVER2-SSE-NEXT:    paddw %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_pshufhw:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vpshufhw {{.*#+}} xmm1 = mem[0,1,2,3,7,6,5,4] sched: [6:1.00]
-; BTVER2-NEXT:    vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,5,4,7,6] sched: [1:0.50]
-; BTVER2-NEXT:    vpaddw %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_pshufhw:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    pshufhw {{.*#+}} xmm1 = xmm0[0,1,2,3,5,4,7,6] sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    pshufhw {{.*#+}} xmm0 = mem[0,1,2,3,7,6,5,4] sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    paddw %xmm1, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_pshufhw:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpshufhw {{.*#+}} xmm1 = mem[0,1,2,3,7,6,5,4] sched: [8:0.50]
-; ZNVER1-NEXT:    vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,5,4,7,6] sched: [1:0.25]
-; ZNVER1-NEXT:    vpaddw %xmm1, %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = shufflevector <8 x i16> %a0, <8 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 5, i32 4, i32 7, i32 6>
-  %2 = load <8 x i16>, <8 x i16> *%a1, align 16
-  %3 = shufflevector <8 x i16> %2, <8 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 7, i32 6, i32 5, i32 4>
-  %4 = add <8 x i16> %1, %3
-  ret <8 x i16> %4
-}
-
-define <8 x i16> @test_pshuflw(<8 x i16> %a0, <8 x i16> *%a1) {
-; GENERIC-LABEL: test_pshuflw:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    pshuflw {{.*#+}} xmm1 = xmm0[1,0,3,2,4,5,6,7] sched: [1:0.50]
-; GENERIC-NEXT:    pshuflw {{.*#+}} xmm0 = mem[3,2,1,0,4,5,6,7] sched: [7:0.50]
-; GENERIC-NEXT:    paddw %xmm1, %xmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_pshuflw:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    pshuflw {{.*#+}} xmm1 = xmm0[1,0,3,2,4,5,6,7] sched: [1:1.00]
-; ATOM-NEXT:    pshuflw {{.*#+}} xmm0 = mem[3,2,1,0,4,5,6,7] sched: [1:1.00]
-; ATOM-NEXT:    paddw %xmm1, %xmm0 # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_pshuflw:
-; SLM:       # %bb.0:
-; SLM-NEXT:    pshuflw {{.*#+}} xmm1 = mem[3,2,1,0,4,5,6,7] sched: [4:1.00]
-; SLM-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[1,0,3,2,4,5,6,7] sched: [1:1.00]
-; SLM-NEXT:    paddw %xmm0, %xmm1 # sched: [1:0.50]
-; SLM-NEXT:    movdqa %xmm1, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_pshuflw:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    pshuflw {{.*#+}} xmm1 = xmm0[1,0,3,2,4,5,6,7] sched: [1:0.50]
-; SANDY-SSE-NEXT:    pshuflw {{.*#+}} xmm0 = mem[3,2,1,0,4,5,6,7] sched: [7:0.50]
-; SANDY-SSE-NEXT:    paddw %xmm1, %xmm0 # sched: [1:0.50]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_pshuflw:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[1,0,3,2,4,5,6,7] sched: [1:0.50]
-; SANDY-NEXT:    vpshuflw {{.*#+}} xmm1 = mem[3,2,1,0,4,5,6,7] sched: [7:0.50]
-; SANDY-NEXT:    vpaddw %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_pshuflw:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    pshuflw {{.*#+}} xmm1 = xmm0[1,0,3,2,4,5,6,7] sched: [1:1.00]
-; HASWELL-SSE-NEXT:    pshuflw {{.*#+}} xmm0 = mem[3,2,1,0,4,5,6,7] sched: [7:1.00]
-; HASWELL-SSE-NEXT:    paddw %xmm1, %xmm0 # sched: [1:0.50]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_pshuflw:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[1,0,3,2,4,5,6,7] sched: [1:1.00]
-; HASWELL-NEXT:    vpshuflw {{.*#+}} xmm1 = mem[3,2,1,0,4,5,6,7] sched: [7:1.00]
-; HASWELL-NEXT:    vpaddw %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_pshuflw:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    pshuflw {{.*#+}} xmm1 = xmm0[1,0,3,2,4,5,6,7] sched: [1:1.00]
-; BROADWELL-SSE-NEXT:    pshuflw {{.*#+}} xmm0 = mem[3,2,1,0,4,5,6,7] sched: [6:1.00]
-; BROADWELL-SSE-NEXT:    paddw %xmm1, %xmm0 # sched: [1:0.50]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pshuflw:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[1,0,3,2,4,5,6,7] sched: [1:1.00]
-; BROADWELL-NEXT:    vpshuflw {{.*#+}} xmm1 = mem[3,2,1,0,4,5,6,7] sched: [6:1.00]
-; BROADWELL-NEXT:    vpaddw %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_pshuflw:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    pshuflw {{.*#+}} xmm1 = xmm0[1,0,3,2,4,5,6,7] sched: [1:1.00]
-; SKYLAKE-SSE-NEXT:    pshuflw {{.*#+}} xmm0 = mem[3,2,1,0,4,5,6,7] sched: [7:1.00]
-; SKYLAKE-SSE-NEXT:    paddw %xmm1, %xmm0 # sched: [1:0.33]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pshuflw:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[1,0,3,2,4,5,6,7] sched: [1:1.00]
-; SKYLAKE-NEXT:    vpshuflw {{.*#+}} xmm1 = mem[3,2,1,0,4,5,6,7] sched: [7:1.00]
-; SKYLAKE-NEXT:    vpaddw %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_pshuflw:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    pshuflw {{.*#+}} xmm1 = xmm0[1,0,3,2,4,5,6,7] sched: [1:1.00]
-; SKX-SSE-NEXT:    pshuflw {{.*#+}} xmm0 = mem[3,2,1,0,4,5,6,7] sched: [7:1.00]
-; SKX-SSE-NEXT:    paddw %xmm1, %xmm0 # sched: [1:0.33]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pshuflw:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[1,0,3,2,4,5,6,7] sched: [1:1.00]
-; SKX-NEXT:    vpshuflw {{.*#+}} xmm1 = mem[3,2,1,0,4,5,6,7] sched: [7:1.00]
-; SKX-NEXT:    vpaddw %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_pshuflw:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    pshuflw {{.*#+}} xmm1 = xmm0[1,0,3,2,4,5,6,7] sched: [2:0.50]
-; BDVER2-SSE-NEXT:    pshuflw {{.*#+}} xmm0 = mem[3,2,1,0,4,5,6,7] sched: [7:0.50]
-; BDVER2-SSE-NEXT:    paddw %xmm1, %xmm0 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_pshuflw:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vpshuflw {{.*#+}} xmm1 = mem[3,2,1,0,4,5,6,7] sched: [7:0.50]
-; BDVER2-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[1,0,3,2,4,5,6,7] sched: [2:0.50]
-; BDVER2-NEXT:    vpaddw %xmm1, %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_pshuflw:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    pshuflw {{.*#+}} xmm1 = xmm0[1,0,3,2,4,5,6,7] sched: [1:0.50]
-; BTVER2-SSE-NEXT:    pshuflw {{.*#+}} xmm0 = mem[3,2,1,0,4,5,6,7] sched: [6:1.00]
-; BTVER2-SSE-NEXT:    paddw %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_pshuflw:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vpshuflw {{.*#+}} xmm1 = mem[3,2,1,0,4,5,6,7] sched: [6:1.00]
-; BTVER2-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[1,0,3,2,4,5,6,7] sched: [1:0.50]
-; BTVER2-NEXT:    vpaddw %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_pshuflw:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    pshuflw {{.*#+}} xmm1 = xmm0[1,0,3,2,4,5,6,7] sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    pshuflw {{.*#+}} xmm0 = mem[3,2,1,0,4,5,6,7] sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    paddw %xmm1, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_pshuflw:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpshuflw {{.*#+}} xmm1 = mem[3,2,1,0,4,5,6,7] sched: [8:0.50]
-; ZNVER1-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[1,0,3,2,4,5,6,7] sched: [1:0.25]
-; ZNVER1-NEXT:    vpaddw %xmm1, %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = shufflevector <8 x i16> %a0, <8 x i16> undef, <8 x i32> <i32 1, i32 0, i32 3, i32 2, i32 4, i32 5, i32 6, i32 7>
-  %2 = load <8 x i16>, <8 x i16> *%a1, align 16
-  %3 = shufflevector <8 x i16> %2, <8 x i16> undef, <8 x i32> <i32 3, i32 2, i32 1, i32 0, i32 4, i32 5, i32 6, i32 7>
-  %4 = add <8 x i16> %1, %3
-  ret <8 x i16> %4
-}
-
-define <4 x i32> @test_pslld(<4 x i32> %a0, <4 x i32> %a1, <4 x i32> *%a2) {
-; GENERIC-LABEL: test_pslld:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    pslld %xmm1, %xmm0 # sched: [2:1.00]
-; GENERIC-NEXT:    pslld (%rdi), %xmm0 # sched: [8:1.00]
-; GENERIC-NEXT:    pslld $2, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_pslld:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    pslld %xmm1, %xmm0 # sched: [2:1.00]
-; ATOM-NEXT:    pslld (%rdi), %xmm0 # sched: [3:1.50]
-; ATOM-NEXT:    pslld $2, %xmm0 # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_pslld:
-; SLM:       # %bb.0:
-; SLM-NEXT:    pslld %xmm1, %xmm0 # sched: [1:1.00]
-; SLM-NEXT:    pslld (%rdi), %xmm0 # sched: [4:1.00]
-; SLM-NEXT:    pslld $2, %xmm0 # sched: [1:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_pslld:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    pslld %xmm1, %xmm0 # sched: [2:1.00]
-; SANDY-SSE-NEXT:    pslld (%rdi), %xmm0 # sched: [8:1.00]
-; SANDY-SSE-NEXT:    pslld $2, %xmm0 # sched: [1:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_pslld:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vpslld %xmm1, %xmm0, %xmm0 # sched: [2:1.00]
-; SANDY-NEXT:    vpslld (%rdi), %xmm0, %xmm0 # sched: [8:1.00]
-; SANDY-NEXT:    vpslld $2, %xmm0, %xmm0 # sched: [1:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_pslld:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    pslld %xmm1, %xmm0 # sched: [2:1.00]
-; HASWELL-SSE-NEXT:    pslld (%rdi), %xmm0 # sched: [8:1.00]
-; HASWELL-SSE-NEXT:    pslld $2, %xmm0 # sched: [1:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_pslld:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpslld %xmm1, %xmm0, %xmm0 # sched: [2:1.00]
-; HASWELL-NEXT:    vpslld (%rdi), %xmm0, %xmm0 # sched: [8:1.00]
-; HASWELL-NEXT:    vpslld $2, %xmm0, %xmm0 # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_pslld:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    pslld %xmm1, %xmm0 # sched: [2:1.00]
-; BROADWELL-SSE-NEXT:    pslld (%rdi), %xmm0 # sched: [7:1.00]
-; BROADWELL-SSE-NEXT:    pslld $2, %xmm0 # sched: [1:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pslld:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpslld %xmm1, %xmm0, %xmm0 # sched: [2:1.00]
-; BROADWELL-NEXT:    vpslld (%rdi), %xmm0, %xmm0 # sched: [7:1.00]
-; BROADWELL-NEXT:    vpslld $2, %xmm0, %xmm0 # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_pslld:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    pslld %xmm1, %xmm0 # sched: [2:1.00]
-; SKYLAKE-SSE-NEXT:    pslld (%rdi), %xmm0 # sched: [7:0.50]
-; SKYLAKE-SSE-NEXT:    pslld $2, %xmm0 # sched: [1:0.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pslld:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpslld %xmm1, %xmm0, %xmm0 # sched: [2:1.00]
-; SKYLAKE-NEXT:    vpslld (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKYLAKE-NEXT:    vpslld $2, %xmm0, %xmm0 # sched: [1:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_pslld:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    pslld %xmm1, %xmm0 # sched: [2:1.00]
-; SKX-SSE-NEXT:    pslld (%rdi), %xmm0 # sched: [7:0.50]
-; SKX-SSE-NEXT:    pslld $2, %xmm0 # sched: [1:0.50]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pslld:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpslld %xmm1, %xmm0, %xmm0 # sched: [2:1.00]
-; SKX-NEXT:    vpslld (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKX-NEXT:    vpslld $2, %xmm0, %xmm0 # sched: [1:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_pslld:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    pslld %xmm1, %xmm0 # sched: [3:0.50]
-; BDVER2-SSE-NEXT:    pslld (%rdi), %xmm0 # sched: [8:0.50]
-; BDVER2-SSE-NEXT:    pslld $2, %xmm0 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_pslld:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vpslld %xmm1, %xmm0, %xmm0 # sched: [3:0.50]
-; BDVER2-NEXT:    vpslld (%rdi), %xmm0, %xmm0 # sched: [8:0.50]
-; BDVER2-NEXT:    vpslld $2, %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_pslld:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    pslld %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    pslld (%rdi), %xmm0 # sched: [6:1.00]
-; BTVER2-SSE-NEXT:    pslld $2, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_pslld:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vpslld %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    vpslld (%rdi), %xmm0, %xmm0 # sched: [6:1.00]
-; BTVER2-NEXT:    vpslld $2, %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_pslld:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    pslld %xmm1, %xmm0 # sched: [1:1.00]
-; ZNVER1-SSE-NEXT:    pslld (%rdi), %xmm0 # sched: [8:1.00]
-; ZNVER1-SSE-NEXT:    pslld $2, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_pslld:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpslld %xmm1, %xmm0, %xmm0 # sched: [1:1.00]
-; ZNVER1-NEXT:    vpslld (%rdi), %xmm0, %xmm0 # sched: [8:1.00]
-; ZNVER1-NEXT:    vpslld $2, %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <4 x i32> @llvm.x86.sse2.psll.d(<4 x i32> %a0, <4 x i32> %a1)
-  %2 = load <4 x i32>, <4 x i32> *%a2, align 16
-  %3 = call <4 x i32> @llvm.x86.sse2.psll.d(<4 x i32> %1, <4 x i32> %2)
-  %4 = call <4 x i32> @llvm.x86.sse2.pslli.d(<4 x i32> %3, i32 2)
-  ret <4 x i32> %4
-}
-declare <4 x i32> @llvm.x86.sse2.psll.d(<4 x i32>, <4 x i32>) nounwind readnone
-declare <4 x i32> @llvm.x86.sse2.pslli.d(<4 x i32>, i32) nounwind readnone
-
-define <4 x i32> @test_pslldq(<4 x i32> %a0) {
-; GENERIC-LABEL: test_pslldq:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    pslldq {{.*#+}} xmm0 = zero,zero,zero,zero,xmm0[0,1,2,3,4,5,6,7,8,9,10,11] sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_pslldq:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    pslldq {{.*#+}} xmm0 = zero,zero,zero,zero,xmm0[0,1,2,3,4,5,6,7,8,9,10,11] sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_pslldq:
-; SLM:       # %bb.0:
-; SLM-NEXT:    pslldq {{.*#+}} xmm0 = zero,zero,zero,zero,xmm0[0,1,2,3,4,5,6,7,8,9,10,11] sched: [1:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_pslldq:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    pslldq {{.*#+}} xmm0 = zero,zero,zero,zero,xmm0[0,1,2,3,4,5,6,7,8,9,10,11] sched: [1:0.50]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_pslldq:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vpslldq {{.*#+}} xmm0 = zero,zero,zero,zero,xmm0[0,1,2,3,4,5,6,7,8,9,10,11] sched: [1:0.50]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_pslldq:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    pslldq {{.*#+}} xmm0 = zero,zero,zero,zero,xmm0[0,1,2,3,4,5,6,7,8,9,10,11] sched: [1:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_pslldq:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpslldq {{.*#+}} xmm0 = zero,zero,zero,zero,xmm0[0,1,2,3,4,5,6,7,8,9,10,11] sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_pslldq:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    pslldq {{.*#+}} xmm0 = zero,zero,zero,zero,xmm0[0,1,2,3,4,5,6,7,8,9,10,11] sched: [1:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pslldq:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpslldq {{.*#+}} xmm0 = zero,zero,zero,zero,xmm0[0,1,2,3,4,5,6,7,8,9,10,11] sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_pslldq:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    pslldq {{.*#+}} xmm0 = zero,zero,zero,zero,xmm0[0,1,2,3,4,5,6,7,8,9,10,11] sched: [1:1.00]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pslldq:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpslldq {{.*#+}} xmm0 = zero,zero,zero,zero,xmm0[0,1,2,3,4,5,6,7,8,9,10,11] sched: [1:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_pslldq:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    pslldq {{.*#+}} xmm0 = zero,zero,zero,zero,xmm0[0,1,2,3,4,5,6,7,8,9,10,11] sched: [1:1.00]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pslldq:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpslldq {{.*#+}} xmm0 = zero,zero,zero,zero,xmm0[0,1,2,3,4,5,6,7,8,9,10,11] sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_pslldq:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    pslldq {{.*#+}} xmm0 = zero,zero,zero,zero,xmm0[0,1,2,3,4,5,6,7,8,9,10,11] sched: [2:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_pslldq:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vpslldq {{.*#+}} xmm0 = zero,zero,zero,zero,xmm0[0,1,2,3,4,5,6,7,8,9,10,11] sched: [2:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_pslldq:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    pslldq {{.*#+}} xmm0 = zero,zero,zero,zero,xmm0[0,1,2,3,4,5,6,7,8,9,10,11] sched: [1:0.50]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_pslldq:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vpslldq {{.*#+}} xmm0 = zero,zero,zero,zero,xmm0[0,1,2,3,4,5,6,7,8,9,10,11] sched: [1:0.50]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_pslldq:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    pslldq {{.*#+}} xmm0 = zero,zero,zero,zero,xmm0[0,1,2,3,4,5,6,7,8,9,10,11] sched: [1:1.00]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_pslldq:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpslldq {{.*#+}} xmm0 = zero,zero,zero,zero,xmm0[0,1,2,3,4,5,6,7,8,9,10,11] sched: [1:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = shufflevector <4 x i32> %a0, <4 x i32> zeroinitializer, <4 x i32> <i32 4, i32 0, i32 1, i32 2>
-  ret <4 x i32> %1
-}
-
-define <2 x i64> @test_psllq(<2 x i64> %a0, <2 x i64> %a1, <2 x i64> *%a2) {
-; GENERIC-LABEL: test_psllq:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    psllq %xmm1, %xmm0 # sched: [2:1.00]
-; GENERIC-NEXT:    psllq (%rdi), %xmm0 # sched: [8:1.00]
-; GENERIC-NEXT:    psllq $2, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_psllq:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    psllq %xmm1, %xmm0 # sched: [2:1.00]
-; ATOM-NEXT:    psllq (%rdi), %xmm0 # sched: [3:1.50]
-; ATOM-NEXT:    psllq $2, %xmm0 # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_psllq:
-; SLM:       # %bb.0:
-; SLM-NEXT:    psllq %xmm1, %xmm0 # sched: [1:1.00]
-; SLM-NEXT:    psllq (%rdi), %xmm0 # sched: [4:1.00]
-; SLM-NEXT:    psllq $2, %xmm0 # sched: [1:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_psllq:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    psllq %xmm1, %xmm0 # sched: [2:1.00]
-; SANDY-SSE-NEXT:    psllq (%rdi), %xmm0 # sched: [8:1.00]
-; SANDY-SSE-NEXT:    psllq $2, %xmm0 # sched: [1:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_psllq:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vpsllq %xmm1, %xmm0, %xmm0 # sched: [2:1.00]
-; SANDY-NEXT:    vpsllq (%rdi), %xmm0, %xmm0 # sched: [8:1.00]
-; SANDY-NEXT:    vpsllq $2, %xmm0, %xmm0 # sched: [1:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_psllq:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    psllq %xmm1, %xmm0 # sched: [2:1.00]
-; HASWELL-SSE-NEXT:    psllq (%rdi), %xmm0 # sched: [8:1.00]
-; HASWELL-SSE-NEXT:    psllq $2, %xmm0 # sched: [1:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_psllq:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpsllq %xmm1, %xmm0, %xmm0 # sched: [2:1.00]
-; HASWELL-NEXT:    vpsllq (%rdi), %xmm0, %xmm0 # sched: [8:1.00]
-; HASWELL-NEXT:    vpsllq $2, %xmm0, %xmm0 # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_psllq:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    psllq %xmm1, %xmm0 # sched: [2:1.00]
-; BROADWELL-SSE-NEXT:    psllq (%rdi), %xmm0 # sched: [7:1.00]
-; BROADWELL-SSE-NEXT:    psllq $2, %xmm0 # sched: [1:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_psllq:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpsllq %xmm1, %xmm0, %xmm0 # sched: [2:1.00]
-; BROADWELL-NEXT:    vpsllq (%rdi), %xmm0, %xmm0 # sched: [7:1.00]
-; BROADWELL-NEXT:    vpsllq $2, %xmm0, %xmm0 # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_psllq:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    psllq %xmm1, %xmm0 # sched: [2:1.00]
-; SKYLAKE-SSE-NEXT:    psllq (%rdi), %xmm0 # sched: [7:0.50]
-; SKYLAKE-SSE-NEXT:    psllq $2, %xmm0 # sched: [1:0.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_psllq:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpsllq %xmm1, %xmm0, %xmm0 # sched: [2:1.00]
-; SKYLAKE-NEXT:    vpsllq (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKYLAKE-NEXT:    vpsllq $2, %xmm0, %xmm0 # sched: [1:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_psllq:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    psllq %xmm1, %xmm0 # sched: [2:1.00]
-; SKX-SSE-NEXT:    psllq (%rdi), %xmm0 # sched: [7:0.50]
-; SKX-SSE-NEXT:    psllq $2, %xmm0 # sched: [1:0.50]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_psllq:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpsllq %xmm1, %xmm0, %xmm0 # sched: [2:1.00]
-; SKX-NEXT:    vpsllq (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKX-NEXT:    vpsllq $2, %xmm0, %xmm0 # sched: [1:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_psllq:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    psllq %xmm1, %xmm0 # sched: [3:0.50]
-; BDVER2-SSE-NEXT:    psllq (%rdi), %xmm0 # sched: [8:0.50]
-; BDVER2-SSE-NEXT:    psllq $2, %xmm0 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_psllq:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vpsllq %xmm1, %xmm0, %xmm0 # sched: [3:0.50]
-; BDVER2-NEXT:    vpsllq (%rdi), %xmm0, %xmm0 # sched: [8:0.50]
-; BDVER2-NEXT:    vpsllq $2, %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_psllq:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    psllq %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    psllq (%rdi), %xmm0 # sched: [6:1.00]
-; BTVER2-SSE-NEXT:    psllq $2, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_psllq:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vpsllq %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    vpsllq (%rdi), %xmm0, %xmm0 # sched: [6:1.00]
-; BTVER2-NEXT:    vpsllq $2, %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_psllq:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    psllq %xmm1, %xmm0 # sched: [1:1.00]
-; ZNVER1-SSE-NEXT:    psllq (%rdi), %xmm0 # sched: [8:1.00]
-; ZNVER1-SSE-NEXT:    psllq $2, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_psllq:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpsllq %xmm1, %xmm0, %xmm0 # sched: [1:1.00]
-; ZNVER1-NEXT:    vpsllq (%rdi), %xmm0, %xmm0 # sched: [8:1.00]
-; ZNVER1-NEXT:    vpsllq $2, %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <2 x i64> @llvm.x86.sse2.psll.q(<2 x i64> %a0, <2 x i64> %a1)
-  %2 = load <2 x i64>, <2 x i64> *%a2, align 16
-  %3 = call <2 x i64> @llvm.x86.sse2.psll.q(<2 x i64> %1, <2 x i64> %2)
-  %4 = call <2 x i64> @llvm.x86.sse2.pslli.q(<2 x i64> %3, i32 2)
-  ret <2 x i64> %4
-}
-declare <2 x i64> @llvm.x86.sse2.psll.q(<2 x i64>, <2 x i64>) nounwind readnone
-declare <2 x i64> @llvm.x86.sse2.pslli.q(<2 x i64>, i32) nounwind readnone
-
-define <8 x i16> @test_psllw(<8 x i16> %a0, <8 x i16> %a1, <8 x i16> *%a2) {
-; GENERIC-LABEL: test_psllw:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    psllw %xmm1, %xmm0 # sched: [2:1.00]
-; GENERIC-NEXT:    psllw (%rdi), %xmm0 # sched: [8:1.00]
-; GENERIC-NEXT:    psllw $2, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_psllw:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    psllw %xmm1, %xmm0 # sched: [2:1.00]
-; ATOM-NEXT:    psllw (%rdi), %xmm0 # sched: [3:1.50]
-; ATOM-NEXT:    psllw $2, %xmm0 # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_psllw:
-; SLM:       # %bb.0:
-; SLM-NEXT:    psllw %xmm1, %xmm0 # sched: [1:1.00]
-; SLM-NEXT:    psllw (%rdi), %xmm0 # sched: [4:1.00]
-; SLM-NEXT:    psllw $2, %xmm0 # sched: [1:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_psllw:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    psllw %xmm1, %xmm0 # sched: [2:1.00]
-; SANDY-SSE-NEXT:    psllw (%rdi), %xmm0 # sched: [8:1.00]
-; SANDY-SSE-NEXT:    psllw $2, %xmm0 # sched: [1:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_psllw:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vpsllw %xmm1, %xmm0, %xmm0 # sched: [2:1.00]
-; SANDY-NEXT:    vpsllw (%rdi), %xmm0, %xmm0 # sched: [8:1.00]
-; SANDY-NEXT:    vpsllw $2, %xmm0, %xmm0 # sched: [1:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_psllw:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    psllw %xmm1, %xmm0 # sched: [2:1.00]
-; HASWELL-SSE-NEXT:    psllw (%rdi), %xmm0 # sched: [8:1.00]
-; HASWELL-SSE-NEXT:    psllw $2, %xmm0 # sched: [1:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_psllw:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpsllw %xmm1, %xmm0, %xmm0 # sched: [2:1.00]
-; HASWELL-NEXT:    vpsllw (%rdi), %xmm0, %xmm0 # sched: [8:1.00]
-; HASWELL-NEXT:    vpsllw $2, %xmm0, %xmm0 # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_psllw:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    psllw %xmm1, %xmm0 # sched: [2:1.00]
-; BROADWELL-SSE-NEXT:    psllw (%rdi), %xmm0 # sched: [7:1.00]
-; BROADWELL-SSE-NEXT:    psllw $2, %xmm0 # sched: [1:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_psllw:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpsllw %xmm1, %xmm0, %xmm0 # sched: [2:1.00]
-; BROADWELL-NEXT:    vpsllw (%rdi), %xmm0, %xmm0 # sched: [7:1.00]
-; BROADWELL-NEXT:    vpsllw $2, %xmm0, %xmm0 # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_psllw:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    psllw %xmm1, %xmm0 # sched: [2:1.00]
-; SKYLAKE-SSE-NEXT:    psllw (%rdi), %xmm0 # sched: [7:0.50]
-; SKYLAKE-SSE-NEXT:    psllw $2, %xmm0 # sched: [1:0.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_psllw:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpsllw %xmm1, %xmm0, %xmm0 # sched: [2:1.00]
-; SKYLAKE-NEXT:    vpsllw (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKYLAKE-NEXT:    vpsllw $2, %xmm0, %xmm0 # sched: [1:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_psllw:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    psllw %xmm1, %xmm0 # sched: [2:1.00]
-; SKX-SSE-NEXT:    psllw (%rdi), %xmm0 # sched: [7:0.50]
-; SKX-SSE-NEXT:    psllw $2, %xmm0 # sched: [1:0.50]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_psllw:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpsllw %xmm1, %xmm0, %xmm0 # sched: [2:1.00]
-; SKX-NEXT:    vpsllw (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKX-NEXT:    vpsllw $2, %xmm0, %xmm0 # sched: [1:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_psllw:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    psllw %xmm1, %xmm0 # sched: [3:0.50]
-; BDVER2-SSE-NEXT:    psllw (%rdi), %xmm0 # sched: [8:0.50]
-; BDVER2-SSE-NEXT:    psllw $2, %xmm0 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_psllw:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vpsllw %xmm1, %xmm0, %xmm0 # sched: [3:0.50]
-; BDVER2-NEXT:    vpsllw (%rdi), %xmm0, %xmm0 # sched: [8:0.50]
-; BDVER2-NEXT:    vpsllw $2, %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_psllw:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    psllw %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    psllw (%rdi), %xmm0 # sched: [6:1.00]
-; BTVER2-SSE-NEXT:    psllw $2, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_psllw:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vpsllw %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    vpsllw (%rdi), %xmm0, %xmm0 # sched: [6:1.00]
-; BTVER2-NEXT:    vpsllw $2, %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_psllw:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    psllw %xmm1, %xmm0 # sched: [1:1.00]
-; ZNVER1-SSE-NEXT:    psllw (%rdi), %xmm0 # sched: [8:1.00]
-; ZNVER1-SSE-NEXT:    psllw $2, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_psllw:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpsllw %xmm1, %xmm0, %xmm0 # sched: [1:1.00]
-; ZNVER1-NEXT:    vpsllw (%rdi), %xmm0, %xmm0 # sched: [8:1.00]
-; ZNVER1-NEXT:    vpsllw $2, %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <8 x i16> @llvm.x86.sse2.psll.w(<8 x i16> %a0, <8 x i16> %a1)
-  %2 = load <8 x i16>, <8 x i16> *%a2, align 16
-  %3 = call <8 x i16> @llvm.x86.sse2.psll.w(<8 x i16> %1, <8 x i16> %2)
-  %4 = call <8 x i16> @llvm.x86.sse2.pslli.w(<8 x i16> %3, i32 2)
-  ret <8 x i16> %4
-}
-declare <8 x i16> @llvm.x86.sse2.psll.w(<8 x i16>, <8 x i16>) nounwind readnone
-declare <8 x i16> @llvm.x86.sse2.pslli.w(<8 x i16>, i32) nounwind readnone
-
-define <4 x i32> @test_psrad(<4 x i32> %a0, <4 x i32> %a1, <4 x i32> *%a2) {
-; GENERIC-LABEL: test_psrad:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    psrad %xmm1, %xmm0 # sched: [2:1.00]
-; GENERIC-NEXT:    psrad (%rdi), %xmm0 # sched: [8:1.00]
-; GENERIC-NEXT:    psrad $2, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_psrad:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    psrad %xmm1, %xmm0 # sched: [2:1.00]
-; ATOM-NEXT:    psrad (%rdi), %xmm0 # sched: [3:1.50]
-; ATOM-NEXT:    psrad $2, %xmm0 # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_psrad:
-; SLM:       # %bb.0:
-; SLM-NEXT:    psrad %xmm1, %xmm0 # sched: [1:1.00]
-; SLM-NEXT:    psrad (%rdi), %xmm0 # sched: [4:1.00]
-; SLM-NEXT:    psrad $2, %xmm0 # sched: [1:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_psrad:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    psrad %xmm1, %xmm0 # sched: [2:1.00]
-; SANDY-SSE-NEXT:    psrad (%rdi), %xmm0 # sched: [8:1.00]
-; SANDY-SSE-NEXT:    psrad $2, %xmm0 # sched: [1:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_psrad:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vpsrad %xmm1, %xmm0, %xmm0 # sched: [2:1.00]
-; SANDY-NEXT:    vpsrad (%rdi), %xmm0, %xmm0 # sched: [8:1.00]
-; SANDY-NEXT:    vpsrad $2, %xmm0, %xmm0 # sched: [1:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_psrad:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    psrad %xmm1, %xmm0 # sched: [2:1.00]
-; HASWELL-SSE-NEXT:    psrad (%rdi), %xmm0 # sched: [8:1.00]
-; HASWELL-SSE-NEXT:    psrad $2, %xmm0 # sched: [1:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_psrad:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpsrad %xmm1, %xmm0, %xmm0 # sched: [2:1.00]
-; HASWELL-NEXT:    vpsrad (%rdi), %xmm0, %xmm0 # sched: [8:1.00]
-; HASWELL-NEXT:    vpsrad $2, %xmm0, %xmm0 # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_psrad:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    psrad %xmm1, %xmm0 # sched: [2:1.00]
-; BROADWELL-SSE-NEXT:    psrad (%rdi), %xmm0 # sched: [7:1.00]
-; BROADWELL-SSE-NEXT:    psrad $2, %xmm0 # sched: [1:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_psrad:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpsrad %xmm1, %xmm0, %xmm0 # sched: [2:1.00]
-; BROADWELL-NEXT:    vpsrad (%rdi), %xmm0, %xmm0 # sched: [7:1.00]
-; BROADWELL-NEXT:    vpsrad $2, %xmm0, %xmm0 # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_psrad:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    psrad %xmm1, %xmm0 # sched: [2:1.00]
-; SKYLAKE-SSE-NEXT:    psrad (%rdi), %xmm0 # sched: [7:0.50]
-; SKYLAKE-SSE-NEXT:    psrad $2, %xmm0 # sched: [1:0.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_psrad:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpsrad %xmm1, %xmm0, %xmm0 # sched: [2:1.00]
-; SKYLAKE-NEXT:    vpsrad (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKYLAKE-NEXT:    vpsrad $2, %xmm0, %xmm0 # sched: [1:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_psrad:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    psrad %xmm1, %xmm0 # sched: [2:1.00]
-; SKX-SSE-NEXT:    psrad (%rdi), %xmm0 # sched: [7:0.50]
-; SKX-SSE-NEXT:    psrad $2, %xmm0 # sched: [1:0.50]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_psrad:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpsrad %xmm1, %xmm0, %xmm0 # sched: [2:1.00]
-; SKX-NEXT:    vpsrad (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKX-NEXT:    vpsrad $2, %xmm0, %xmm0 # sched: [1:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_psrad:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    psrad %xmm1, %xmm0 # sched: [3:0.50]
-; BDVER2-SSE-NEXT:    psrad (%rdi), %xmm0 # sched: [8:0.50]
-; BDVER2-SSE-NEXT:    psrad $2, %xmm0 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_psrad:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vpsrad %xmm1, %xmm0, %xmm0 # sched: [3:0.50]
-; BDVER2-NEXT:    vpsrad (%rdi), %xmm0, %xmm0 # sched: [8:0.50]
-; BDVER2-NEXT:    vpsrad $2, %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_psrad:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    psrad %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    psrad (%rdi), %xmm0 # sched: [6:1.00]
-; BTVER2-SSE-NEXT:    psrad $2, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_psrad:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vpsrad %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    vpsrad (%rdi), %xmm0, %xmm0 # sched: [6:1.00]
-; BTVER2-NEXT:    vpsrad $2, %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_psrad:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    psrad %xmm1, %xmm0 # sched: [1:1.00]
-; ZNVER1-SSE-NEXT:    psrad (%rdi), %xmm0 # sched: [8:1.00]
-; ZNVER1-SSE-NEXT:    psrad $2, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_psrad:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpsrad %xmm1, %xmm0, %xmm0 # sched: [1:1.00]
-; ZNVER1-NEXT:    vpsrad (%rdi), %xmm0, %xmm0 # sched: [8:1.00]
-; ZNVER1-NEXT:    vpsrad $2, %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <4 x i32> @llvm.x86.sse2.psra.d(<4 x i32> %a0, <4 x i32> %a1)
-  %2 = load <4 x i32>, <4 x i32> *%a2, align 16
-  %3 = call <4 x i32> @llvm.x86.sse2.psra.d(<4 x i32> %1, <4 x i32> %2)
-  %4 = call <4 x i32> @llvm.x86.sse2.psrai.d(<4 x i32> %3, i32 2)
-  ret <4 x i32> %4
-}
-declare <4 x i32> @llvm.x86.sse2.psra.d(<4 x i32>, <4 x i32>) nounwind readnone
-declare <4 x i32> @llvm.x86.sse2.psrai.d(<4 x i32>, i32) nounwind readnone
-
-define <8 x i16> @test_psraw(<8 x i16> %a0, <8 x i16> %a1, <8 x i16> *%a2) {
-; GENERIC-LABEL: test_psraw:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    psraw %xmm1, %xmm0 # sched: [2:1.00]
-; GENERIC-NEXT:    psraw (%rdi), %xmm0 # sched: [8:1.00]
-; GENERIC-NEXT:    psraw $2, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_psraw:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    psraw %xmm1, %xmm0 # sched: [2:1.00]
-; ATOM-NEXT:    psraw (%rdi), %xmm0 # sched: [3:1.50]
-; ATOM-NEXT:    psraw $2, %xmm0 # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_psraw:
-; SLM:       # %bb.0:
-; SLM-NEXT:    psraw %xmm1, %xmm0 # sched: [1:1.00]
-; SLM-NEXT:    psraw (%rdi), %xmm0 # sched: [4:1.00]
-; SLM-NEXT:    psraw $2, %xmm0 # sched: [1:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_psraw:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    psraw %xmm1, %xmm0 # sched: [2:1.00]
-; SANDY-SSE-NEXT:    psraw (%rdi), %xmm0 # sched: [8:1.00]
-; SANDY-SSE-NEXT:    psraw $2, %xmm0 # sched: [1:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_psraw:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vpsraw %xmm1, %xmm0, %xmm0 # sched: [2:1.00]
-; SANDY-NEXT:    vpsraw (%rdi), %xmm0, %xmm0 # sched: [8:1.00]
-; SANDY-NEXT:    vpsraw $2, %xmm0, %xmm0 # sched: [1:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_psraw:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    psraw %xmm1, %xmm0 # sched: [2:1.00]
-; HASWELL-SSE-NEXT:    psraw (%rdi), %xmm0 # sched: [8:1.00]
-; HASWELL-SSE-NEXT:    psraw $2, %xmm0 # sched: [1:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_psraw:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpsraw %xmm1, %xmm0, %xmm0 # sched: [2:1.00]
-; HASWELL-NEXT:    vpsraw (%rdi), %xmm0, %xmm0 # sched: [8:1.00]
-; HASWELL-NEXT:    vpsraw $2, %xmm0, %xmm0 # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_psraw:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    psraw %xmm1, %xmm0 # sched: [2:1.00]
-; BROADWELL-SSE-NEXT:    psraw (%rdi), %xmm0 # sched: [7:1.00]
-; BROADWELL-SSE-NEXT:    psraw $2, %xmm0 # sched: [1:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_psraw:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpsraw %xmm1, %xmm0, %xmm0 # sched: [2:1.00]
-; BROADWELL-NEXT:    vpsraw (%rdi), %xmm0, %xmm0 # sched: [7:1.00]
-; BROADWELL-NEXT:    vpsraw $2, %xmm0, %xmm0 # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_psraw:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    psraw %xmm1, %xmm0 # sched: [2:1.00]
-; SKYLAKE-SSE-NEXT:    psraw (%rdi), %xmm0 # sched: [7:0.50]
-; SKYLAKE-SSE-NEXT:    psraw $2, %xmm0 # sched: [1:0.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_psraw:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpsraw %xmm1, %xmm0, %xmm0 # sched: [2:1.00]
-; SKYLAKE-NEXT:    vpsraw (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKYLAKE-NEXT:    vpsraw $2, %xmm0, %xmm0 # sched: [1:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_psraw:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    psraw %xmm1, %xmm0 # sched: [2:1.00]
-; SKX-SSE-NEXT:    psraw (%rdi), %xmm0 # sched: [7:0.50]
-; SKX-SSE-NEXT:    psraw $2, %xmm0 # sched: [1:0.50]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_psraw:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpsraw %xmm1, %xmm0, %xmm0 # sched: [2:1.00]
-; SKX-NEXT:    vpsraw (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKX-NEXT:    vpsraw $2, %xmm0, %xmm0 # sched: [1:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_psraw:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    psraw %xmm1, %xmm0 # sched: [3:0.50]
-; BDVER2-SSE-NEXT:    psraw (%rdi), %xmm0 # sched: [8:0.50]
-; BDVER2-SSE-NEXT:    psraw $2, %xmm0 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_psraw:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vpsraw %xmm1, %xmm0, %xmm0 # sched: [3:0.50]
-; BDVER2-NEXT:    vpsraw (%rdi), %xmm0, %xmm0 # sched: [8:0.50]
-; BDVER2-NEXT:    vpsraw $2, %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_psraw:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    psraw %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    psraw (%rdi), %xmm0 # sched: [6:1.00]
-; BTVER2-SSE-NEXT:    psraw $2, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_psraw:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vpsraw %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    vpsraw (%rdi), %xmm0, %xmm0 # sched: [6:1.00]
-; BTVER2-NEXT:    vpsraw $2, %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_psraw:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    psraw %xmm1, %xmm0 # sched: [1:1.00]
-; ZNVER1-SSE-NEXT:    psraw (%rdi), %xmm0 # sched: [8:1.00]
-; ZNVER1-SSE-NEXT:    psraw $2, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_psraw:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpsraw %xmm1, %xmm0, %xmm0 # sched: [1:1.00]
-; ZNVER1-NEXT:    vpsraw (%rdi), %xmm0, %xmm0 # sched: [8:1.00]
-; ZNVER1-NEXT:    vpsraw $2, %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <8 x i16> @llvm.x86.sse2.psra.w(<8 x i16> %a0, <8 x i16> %a1)
-  %2 = load <8 x i16>, <8 x i16> *%a2, align 16
-  %3 = call <8 x i16> @llvm.x86.sse2.psra.w(<8 x i16> %1, <8 x i16> %2)
-  %4 = call <8 x i16> @llvm.x86.sse2.psrai.w(<8 x i16> %3, i32 2)
-  ret <8 x i16> %4
-}
-declare <8 x i16> @llvm.x86.sse2.psra.w(<8 x i16>, <8 x i16>) nounwind readnone
-declare <8 x i16> @llvm.x86.sse2.psrai.w(<8 x i16>, i32) nounwind readnone
-
-define <4 x i32> @test_psrld(<4 x i32> %a0, <4 x i32> %a1, <4 x i32> *%a2) {
-; GENERIC-LABEL: test_psrld:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    psrld %xmm1, %xmm0 # sched: [2:1.00]
-; GENERIC-NEXT:    psrld (%rdi), %xmm0 # sched: [8:1.00]
-; GENERIC-NEXT:    psrld $2, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_psrld:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    psrld %xmm1, %xmm0 # sched: [2:1.00]
-; ATOM-NEXT:    psrld (%rdi), %xmm0 # sched: [3:1.50]
-; ATOM-NEXT:    psrld $2, %xmm0 # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_psrld:
-; SLM:       # %bb.0:
-; SLM-NEXT:    psrld %xmm1, %xmm0 # sched: [1:1.00]
-; SLM-NEXT:    psrld (%rdi), %xmm0 # sched: [4:1.00]
-; SLM-NEXT:    psrld $2, %xmm0 # sched: [1:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_psrld:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    psrld %xmm1, %xmm0 # sched: [2:1.00]
-; SANDY-SSE-NEXT:    psrld (%rdi), %xmm0 # sched: [8:1.00]
-; SANDY-SSE-NEXT:    psrld $2, %xmm0 # sched: [1:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_psrld:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vpsrld %xmm1, %xmm0, %xmm0 # sched: [2:1.00]
-; SANDY-NEXT:    vpsrld (%rdi), %xmm0, %xmm0 # sched: [8:1.00]
-; SANDY-NEXT:    vpsrld $2, %xmm0, %xmm0 # sched: [1:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_psrld:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    psrld %xmm1, %xmm0 # sched: [2:1.00]
-; HASWELL-SSE-NEXT:    psrld (%rdi), %xmm0 # sched: [8:1.00]
-; HASWELL-SSE-NEXT:    psrld $2, %xmm0 # sched: [1:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_psrld:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpsrld %xmm1, %xmm0, %xmm0 # sched: [2:1.00]
-; HASWELL-NEXT:    vpsrld (%rdi), %xmm0, %xmm0 # sched: [8:1.00]
-; HASWELL-NEXT:    vpsrld $2, %xmm0, %xmm0 # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_psrld:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    psrld %xmm1, %xmm0 # sched: [2:1.00]
-; BROADWELL-SSE-NEXT:    psrld (%rdi), %xmm0 # sched: [7:1.00]
-; BROADWELL-SSE-NEXT:    psrld $2, %xmm0 # sched: [1:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_psrld:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpsrld %xmm1, %xmm0, %xmm0 # sched: [2:1.00]
-; BROADWELL-NEXT:    vpsrld (%rdi), %xmm0, %xmm0 # sched: [7:1.00]
-; BROADWELL-NEXT:    vpsrld $2, %xmm0, %xmm0 # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_psrld:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    psrld %xmm1, %xmm0 # sched: [2:1.00]
-; SKYLAKE-SSE-NEXT:    psrld (%rdi), %xmm0 # sched: [7:0.50]
-; SKYLAKE-SSE-NEXT:    psrld $2, %xmm0 # sched: [1:0.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_psrld:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpsrld %xmm1, %xmm0, %xmm0 # sched: [2:1.00]
-; SKYLAKE-NEXT:    vpsrld (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKYLAKE-NEXT:    vpsrld $2, %xmm0, %xmm0 # sched: [1:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_psrld:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    psrld %xmm1, %xmm0 # sched: [2:1.00]
-; SKX-SSE-NEXT:    psrld (%rdi), %xmm0 # sched: [7:0.50]
-; SKX-SSE-NEXT:    psrld $2, %xmm0 # sched: [1:0.50]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_psrld:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpsrld %xmm1, %xmm0, %xmm0 # sched: [2:1.00]
-; SKX-NEXT:    vpsrld (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKX-NEXT:    vpsrld $2, %xmm0, %xmm0 # sched: [1:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_psrld:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    psrld %xmm1, %xmm0 # sched: [3:0.50]
-; BDVER2-SSE-NEXT:    psrld (%rdi), %xmm0 # sched: [8:0.50]
-; BDVER2-SSE-NEXT:    psrld $2, %xmm0 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_psrld:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vpsrld %xmm1, %xmm0, %xmm0 # sched: [3:0.50]
-; BDVER2-NEXT:    vpsrld (%rdi), %xmm0, %xmm0 # sched: [8:0.50]
-; BDVER2-NEXT:    vpsrld $2, %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_psrld:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    psrld %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    psrld (%rdi), %xmm0 # sched: [6:1.00]
-; BTVER2-SSE-NEXT:    psrld $2, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_psrld:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vpsrld %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    vpsrld (%rdi), %xmm0, %xmm0 # sched: [6:1.00]
-; BTVER2-NEXT:    vpsrld $2, %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_psrld:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    psrld %xmm1, %xmm0 # sched: [1:1.00]
-; ZNVER1-SSE-NEXT:    psrld (%rdi), %xmm0 # sched: [8:1.00]
-; ZNVER1-SSE-NEXT:    psrld $2, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_psrld:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpsrld %xmm1, %xmm0, %xmm0 # sched: [1:1.00]
-; ZNVER1-NEXT:    vpsrld (%rdi), %xmm0, %xmm0 # sched: [8:1.00]
-; ZNVER1-NEXT:    vpsrld $2, %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <4 x i32> @llvm.x86.sse2.psrl.d(<4 x i32> %a0, <4 x i32> %a1)
-  %2 = load <4 x i32>, <4 x i32> *%a2, align 16
-  %3 = call <4 x i32> @llvm.x86.sse2.psrl.d(<4 x i32> %1, <4 x i32> %2)
-  %4 = call <4 x i32> @llvm.x86.sse2.psrli.d(<4 x i32> %3, i32 2)
-  ret <4 x i32> %4
-}
-declare <4 x i32> @llvm.x86.sse2.psrl.d(<4 x i32>, <4 x i32>) nounwind readnone
-declare <4 x i32> @llvm.x86.sse2.psrli.d(<4 x i32>, i32) nounwind readnone
-
-define <4 x i32> @test_psrldq(<4 x i32> %a0) {
-; GENERIC-LABEL: test_psrldq:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    psrldq {{.*#+}} xmm0 = xmm0[4,5,6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_psrldq:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    psrldq {{.*#+}} xmm0 = xmm0[4,5,6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_psrldq:
-; SLM:       # %bb.0:
-; SLM-NEXT:    psrldq {{.*#+}} xmm0 = xmm0[4,5,6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero sched: [1:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_psrldq:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    psrldq {{.*#+}} xmm0 = xmm0[4,5,6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero sched: [1:0.50]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_psrldq:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vpsrldq {{.*#+}} xmm0 = xmm0[4,5,6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero sched: [1:0.50]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_psrldq:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    psrldq {{.*#+}} xmm0 = xmm0[4,5,6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero sched: [1:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_psrldq:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpsrldq {{.*#+}} xmm0 = xmm0[4,5,6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_psrldq:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    psrldq {{.*#+}} xmm0 = xmm0[4,5,6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero sched: [1:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_psrldq:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpsrldq {{.*#+}} xmm0 = xmm0[4,5,6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_psrldq:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    psrldq {{.*#+}} xmm0 = xmm0[4,5,6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero sched: [1:1.00]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_psrldq:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpsrldq {{.*#+}} xmm0 = xmm0[4,5,6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero sched: [1:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_psrldq:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    psrldq {{.*#+}} xmm0 = xmm0[4,5,6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero sched: [1:1.00]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_psrldq:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpsrldq {{.*#+}} xmm0 = xmm0[4,5,6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero sched: [1:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_psrldq:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    psrldq {{.*#+}} xmm0 = xmm0[4,5,6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero sched: [2:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_psrldq:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vpsrldq {{.*#+}} xmm0 = xmm0[4,5,6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero sched: [2:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_psrldq:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    psrldq {{.*#+}} xmm0 = xmm0[4,5,6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero sched: [1:0.50]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_psrldq:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vpsrldq {{.*#+}} xmm0 = xmm0[4,5,6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero sched: [1:0.50]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_psrldq:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    psrldq {{.*#+}} xmm0 = xmm0[4,5,6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero sched: [1:1.00]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_psrldq:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpsrldq {{.*#+}} xmm0 = xmm0[4,5,6,7,8,9,10,11,12,13,14,15],zero,zero,zero,zero sched: [1:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = shufflevector <4 x i32> %a0, <4 x i32> zeroinitializer, <4 x i32> <i32 1, i32 2, i32 3, i32 4>
-  ret <4 x i32> %1
-}
-
-define <2 x i64> @test_psrlq(<2 x i64> %a0, <2 x i64> %a1, <2 x i64> *%a2) {
-; GENERIC-LABEL: test_psrlq:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    psrlq %xmm1, %xmm0 # sched: [2:1.00]
-; GENERIC-NEXT:    psrlq (%rdi), %xmm0 # sched: [8:1.00]
-; GENERIC-NEXT:    psrlq $2, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_psrlq:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    psrlq %xmm1, %xmm0 # sched: [2:1.00]
-; ATOM-NEXT:    psrlq (%rdi), %xmm0 # sched: [3:1.50]
-; ATOM-NEXT:    psrlq $2, %xmm0 # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_psrlq:
-; SLM:       # %bb.0:
-; SLM-NEXT:    psrlq %xmm1, %xmm0 # sched: [1:1.00]
-; SLM-NEXT:    psrlq (%rdi), %xmm0 # sched: [4:1.00]
-; SLM-NEXT:    psrlq $2, %xmm0 # sched: [1:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_psrlq:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    psrlq %xmm1, %xmm0 # sched: [2:1.00]
-; SANDY-SSE-NEXT:    psrlq (%rdi), %xmm0 # sched: [8:1.00]
-; SANDY-SSE-NEXT:    psrlq $2, %xmm0 # sched: [1:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_psrlq:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vpsrlq %xmm1, %xmm0, %xmm0 # sched: [2:1.00]
-; SANDY-NEXT:    vpsrlq (%rdi), %xmm0, %xmm0 # sched: [8:1.00]
-; SANDY-NEXT:    vpsrlq $2, %xmm0, %xmm0 # sched: [1:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_psrlq:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    psrlq %xmm1, %xmm0 # sched: [2:1.00]
-; HASWELL-SSE-NEXT:    psrlq (%rdi), %xmm0 # sched: [8:1.00]
-; HASWELL-SSE-NEXT:    psrlq $2, %xmm0 # sched: [1:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_psrlq:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpsrlq %xmm1, %xmm0, %xmm0 # sched: [2:1.00]
-; HASWELL-NEXT:    vpsrlq (%rdi), %xmm0, %xmm0 # sched: [8:1.00]
-; HASWELL-NEXT:    vpsrlq $2, %xmm0, %xmm0 # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_psrlq:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    psrlq %xmm1, %xmm0 # sched: [2:1.00]
-; BROADWELL-SSE-NEXT:    psrlq (%rdi), %xmm0 # sched: [7:1.00]
-; BROADWELL-SSE-NEXT:    psrlq $2, %xmm0 # sched: [1:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_psrlq:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpsrlq %xmm1, %xmm0, %xmm0 # sched: [2:1.00]
-; BROADWELL-NEXT:    vpsrlq (%rdi), %xmm0, %xmm0 # sched: [7:1.00]
-; BROADWELL-NEXT:    vpsrlq $2, %xmm0, %xmm0 # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_psrlq:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    psrlq %xmm1, %xmm0 # sched: [2:1.00]
-; SKYLAKE-SSE-NEXT:    psrlq (%rdi), %xmm0 # sched: [7:0.50]
-; SKYLAKE-SSE-NEXT:    psrlq $2, %xmm0 # sched: [1:0.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_psrlq:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpsrlq %xmm1, %xmm0, %xmm0 # sched: [2:1.00]
-; SKYLAKE-NEXT:    vpsrlq (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKYLAKE-NEXT:    vpsrlq $2, %xmm0, %xmm0 # sched: [1:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_psrlq:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    psrlq %xmm1, %xmm0 # sched: [2:1.00]
-; SKX-SSE-NEXT:    psrlq (%rdi), %xmm0 # sched: [7:0.50]
-; SKX-SSE-NEXT:    psrlq $2, %xmm0 # sched: [1:0.50]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_psrlq:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpsrlq %xmm1, %xmm0, %xmm0 # sched: [2:1.00]
-; SKX-NEXT:    vpsrlq (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKX-NEXT:    vpsrlq $2, %xmm0, %xmm0 # sched: [1:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_psrlq:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    psrlq %xmm1, %xmm0 # sched: [3:0.50]
-; BDVER2-SSE-NEXT:    psrlq (%rdi), %xmm0 # sched: [8:0.50]
-; BDVER2-SSE-NEXT:    psrlq $2, %xmm0 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_psrlq:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vpsrlq %xmm1, %xmm0, %xmm0 # sched: [3:0.50]
-; BDVER2-NEXT:    vpsrlq (%rdi), %xmm0, %xmm0 # sched: [8:0.50]
-; BDVER2-NEXT:    vpsrlq $2, %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_psrlq:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    psrlq %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    psrlq (%rdi), %xmm0 # sched: [6:1.00]
-; BTVER2-SSE-NEXT:    psrlq $2, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_psrlq:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vpsrlq %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    vpsrlq (%rdi), %xmm0, %xmm0 # sched: [6:1.00]
-; BTVER2-NEXT:    vpsrlq $2, %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_psrlq:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    psrlq %xmm1, %xmm0 # sched: [1:1.00]
-; ZNVER1-SSE-NEXT:    psrlq (%rdi), %xmm0 # sched: [8:1.00]
-; ZNVER1-SSE-NEXT:    psrlq $2, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_psrlq:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpsrlq %xmm1, %xmm0, %xmm0 # sched: [1:1.00]
-; ZNVER1-NEXT:    vpsrlq (%rdi), %xmm0, %xmm0 # sched: [8:1.00]
-; ZNVER1-NEXT:    vpsrlq $2, %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <2 x i64> @llvm.x86.sse2.psrl.q(<2 x i64> %a0, <2 x i64> %a1)
-  %2 = load <2 x i64>, <2 x i64> *%a2, align 16
-  %3 = call <2 x i64> @llvm.x86.sse2.psrl.q(<2 x i64> %1, <2 x i64> %2)
-  %4 = call <2 x i64> @llvm.x86.sse2.psrli.q(<2 x i64> %3, i32 2)
-  ret <2 x i64> %4
-}
-declare <2 x i64> @llvm.x86.sse2.psrl.q(<2 x i64>, <2 x i64>) nounwind readnone
-declare <2 x i64> @llvm.x86.sse2.psrli.q(<2 x i64>, i32) nounwind readnone
-
-define <8 x i16> @test_psrlw(<8 x i16> %a0, <8 x i16> %a1, <8 x i16> *%a2) {
-; GENERIC-LABEL: test_psrlw:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    psrlw %xmm1, %xmm0 # sched: [2:1.00]
-; GENERIC-NEXT:    psrlw (%rdi), %xmm0 # sched: [8:1.00]
-; GENERIC-NEXT:    psrlw $2, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_psrlw:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    psrlw %xmm1, %xmm0 # sched: [2:1.00]
-; ATOM-NEXT:    psrlw (%rdi), %xmm0 # sched: [3:1.50]
-; ATOM-NEXT:    psrlw $2, %xmm0 # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_psrlw:
-; SLM:       # %bb.0:
-; SLM-NEXT:    psrlw %xmm1, %xmm0 # sched: [1:1.00]
-; SLM-NEXT:    psrlw (%rdi), %xmm0 # sched: [4:1.00]
-; SLM-NEXT:    psrlw $2, %xmm0 # sched: [1:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_psrlw:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    psrlw %xmm1, %xmm0 # sched: [2:1.00]
-; SANDY-SSE-NEXT:    psrlw (%rdi), %xmm0 # sched: [8:1.00]
-; SANDY-SSE-NEXT:    psrlw $2, %xmm0 # sched: [1:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_psrlw:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vpsrlw %xmm1, %xmm0, %xmm0 # sched: [2:1.00]
-; SANDY-NEXT:    vpsrlw (%rdi), %xmm0, %xmm0 # sched: [8:1.00]
-; SANDY-NEXT:    vpsrlw $2, %xmm0, %xmm0 # sched: [1:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_psrlw:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    psrlw %xmm1, %xmm0 # sched: [2:1.00]
-; HASWELL-SSE-NEXT:    psrlw (%rdi), %xmm0 # sched: [8:1.00]
-; HASWELL-SSE-NEXT:    psrlw $2, %xmm0 # sched: [1:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_psrlw:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpsrlw %xmm1, %xmm0, %xmm0 # sched: [2:1.00]
-; HASWELL-NEXT:    vpsrlw (%rdi), %xmm0, %xmm0 # sched: [8:1.00]
-; HASWELL-NEXT:    vpsrlw $2, %xmm0, %xmm0 # sched: [1:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_psrlw:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    psrlw %xmm1, %xmm0 # sched: [2:1.00]
-; BROADWELL-SSE-NEXT:    psrlw (%rdi), %xmm0 # sched: [7:1.00]
-; BROADWELL-SSE-NEXT:    psrlw $2, %xmm0 # sched: [1:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_psrlw:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpsrlw %xmm1, %xmm0, %xmm0 # sched: [2:1.00]
-; BROADWELL-NEXT:    vpsrlw (%rdi), %xmm0, %xmm0 # sched: [7:1.00]
-; BROADWELL-NEXT:    vpsrlw $2, %xmm0, %xmm0 # sched: [1:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_psrlw:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    psrlw %xmm1, %xmm0 # sched: [2:1.00]
-; SKYLAKE-SSE-NEXT:    psrlw (%rdi), %xmm0 # sched: [7:0.50]
-; SKYLAKE-SSE-NEXT:    psrlw $2, %xmm0 # sched: [1:0.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_psrlw:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpsrlw %xmm1, %xmm0, %xmm0 # sched: [2:1.00]
-; SKYLAKE-NEXT:    vpsrlw (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKYLAKE-NEXT:    vpsrlw $2, %xmm0, %xmm0 # sched: [1:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_psrlw:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    psrlw %xmm1, %xmm0 # sched: [2:1.00]
-; SKX-SSE-NEXT:    psrlw (%rdi), %xmm0 # sched: [7:0.50]
-; SKX-SSE-NEXT:    psrlw $2, %xmm0 # sched: [1:0.50]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_psrlw:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpsrlw %xmm1, %xmm0, %xmm0 # sched: [2:1.00]
-; SKX-NEXT:    vpsrlw (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKX-NEXT:    vpsrlw $2, %xmm0, %xmm0 # sched: [1:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_psrlw:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    psrlw %xmm1, %xmm0 # sched: [3:0.50]
-; BDVER2-SSE-NEXT:    psrlw (%rdi), %xmm0 # sched: [8:0.50]
-; BDVER2-SSE-NEXT:    psrlw $2, %xmm0 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_psrlw:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vpsrlw %xmm1, %xmm0, %xmm0 # sched: [3:0.50]
-; BDVER2-NEXT:    vpsrlw (%rdi), %xmm0, %xmm0 # sched: [8:0.50]
-; BDVER2-NEXT:    vpsrlw $2, %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_psrlw:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    psrlw %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    psrlw (%rdi), %xmm0 # sched: [6:1.00]
-; BTVER2-SSE-NEXT:    psrlw $2, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_psrlw:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vpsrlw %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    vpsrlw (%rdi), %xmm0, %xmm0 # sched: [6:1.00]
-; BTVER2-NEXT:    vpsrlw $2, %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_psrlw:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    psrlw %xmm1, %xmm0 # sched: [1:1.00]
-; ZNVER1-SSE-NEXT:    psrlw (%rdi), %xmm0 # sched: [8:1.00]
-; ZNVER1-SSE-NEXT:    psrlw $2, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_psrlw:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpsrlw %xmm1, %xmm0, %xmm0 # sched: [1:1.00]
-; ZNVER1-NEXT:    vpsrlw (%rdi), %xmm0, %xmm0 # sched: [8:1.00]
-; ZNVER1-NEXT:    vpsrlw $2, %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <8 x i16> @llvm.x86.sse2.psrl.w(<8 x i16> %a0, <8 x i16> %a1)
-  %2 = load <8 x i16>, <8 x i16> *%a2, align 16
-  %3 = call <8 x i16> @llvm.x86.sse2.psrl.w(<8 x i16> %1, <8 x i16> %2)
-  %4 = call <8 x i16> @llvm.x86.sse2.psrli.w(<8 x i16> %3, i32 2)
-  ret <8 x i16> %4
-}
-declare <8 x i16> @llvm.x86.sse2.psrl.w(<8 x i16>, <8 x i16>) nounwind readnone
-declare <8 x i16> @llvm.x86.sse2.psrli.w(<8 x i16>, i32) nounwind readnone
-
-define <16 x i8> @test_psubb(<16 x i8> %a0, <16 x i8> %a1, <16 x i8> *%a2) {
-; GENERIC-LABEL: test_psubb:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    psubb %xmm1, %xmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    psubb (%rdi), %xmm0 # sched: [7:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_psubb:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    psubb %xmm1, %xmm0 # sched: [1:0.50]
-; ATOM-NEXT:    psubb (%rdi), %xmm0 # sched: [1:1.00]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_psubb:
-; SLM:       # %bb.0:
-; SLM-NEXT:    psubb %xmm1, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    psubb (%rdi), %xmm0 # sched: [4:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_psubb:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    psubb %xmm1, %xmm0 # sched: [1:0.50]
-; SANDY-SSE-NEXT:    psubb (%rdi), %xmm0 # sched: [7:0.50]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_psubb:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vpsubb %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SANDY-NEXT:    vpsubb (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_psubb:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    psubb %xmm1, %xmm0 # sched: [1:0.50]
-; HASWELL-SSE-NEXT:    psubb (%rdi), %xmm0 # sched: [7:0.50]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_psubb:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpsubb %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; HASWELL-NEXT:    vpsubb (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_psubb:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    psubb %xmm1, %xmm0 # sched: [1:0.50]
-; BROADWELL-SSE-NEXT:    psubb (%rdi), %xmm0 # sched: [6:0.50]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_psubb:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpsubb %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    vpsubb (%rdi), %xmm0, %xmm0 # sched: [6:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_psubb:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    psubb %xmm1, %xmm0 # sched: [1:0.33]
-; SKYLAKE-SSE-NEXT:    psubb (%rdi), %xmm0 # sched: [7:0.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_psubb:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpsubb %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    vpsubb (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_psubb:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    psubb %xmm1, %xmm0 # sched: [1:0.33]
-; SKX-SSE-NEXT:    psubb (%rdi), %xmm0 # sched: [7:0.50]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_psubb:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpsubb %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    vpsubb (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_psubb:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    psubb %xmm1, %xmm0 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    psubb (%rdi), %xmm0 # sched: [7:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_psubb:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vpsubb %xmm1, %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    vpsubb (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_psubb:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    psubb %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    psubb (%rdi), %xmm0 # sched: [6:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_psubb:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vpsubb %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    vpsubb (%rdi), %xmm0, %xmm0 # sched: [6:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_psubb:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    psubb %xmm1, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    psubb (%rdi), %xmm0 # sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_psubb:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpsubb %xmm1, %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vpsubb (%rdi), %xmm0, %xmm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = sub <16 x i8> %a0, %a1
-  %2 = load <16 x i8>, <16 x i8> *%a2, align 16
-  %3 = sub <16 x i8> %1, %2
-  ret <16 x i8> %3
-}
-
-define <4 x i32> @test_psubd(<4 x i32> %a0, <4 x i32> %a1, <4 x i32> *%a2) {
-; GENERIC-LABEL: test_psubd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    psubd %xmm1, %xmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    psubd (%rdi), %xmm0 # sched: [7:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_psubd:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    psubd %xmm1, %xmm0 # sched: [1:0.50]
-; ATOM-NEXT:    psubd (%rdi), %xmm0 # sched: [1:1.00]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_psubd:
-; SLM:       # %bb.0:
-; SLM-NEXT:    psubd %xmm1, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    psubd (%rdi), %xmm0 # sched: [4:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_psubd:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    psubd %xmm1, %xmm0 # sched: [1:0.50]
-; SANDY-SSE-NEXT:    psubd (%rdi), %xmm0 # sched: [7:0.50]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_psubd:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vpsubd %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SANDY-NEXT:    vpsubd (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_psubd:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    psubd %xmm1, %xmm0 # sched: [1:0.50]
-; HASWELL-SSE-NEXT:    psubd (%rdi), %xmm0 # sched: [7:0.50]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_psubd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpsubd %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; HASWELL-NEXT:    vpsubd (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_psubd:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    psubd %xmm1, %xmm0 # sched: [1:0.50]
-; BROADWELL-SSE-NEXT:    psubd (%rdi), %xmm0 # sched: [6:0.50]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_psubd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpsubd %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    vpsubd (%rdi), %xmm0, %xmm0 # sched: [6:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_psubd:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    psubd %xmm1, %xmm0 # sched: [1:0.33]
-; SKYLAKE-SSE-NEXT:    psubd (%rdi), %xmm0 # sched: [7:0.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_psubd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpsubd %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    vpsubd (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_psubd:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    psubd %xmm1, %xmm0 # sched: [1:0.33]
-; SKX-SSE-NEXT:    psubd (%rdi), %xmm0 # sched: [7:0.50]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_psubd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpsubd %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    vpsubd (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_psubd:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    psubd %xmm1, %xmm0 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    psubd (%rdi), %xmm0 # sched: [7:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_psubd:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vpsubd %xmm1, %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    vpsubd (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_psubd:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    psubd %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    psubd (%rdi), %xmm0 # sched: [6:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_psubd:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vpsubd %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    vpsubd (%rdi), %xmm0, %xmm0 # sched: [6:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_psubd:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    psubd %xmm1, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    psubd (%rdi), %xmm0 # sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_psubd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpsubd %xmm1, %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vpsubd (%rdi), %xmm0, %xmm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = sub <4 x i32> %a0, %a1
-  %2 = load <4 x i32>, <4 x i32> *%a2, align 16
-  %3 = sub <4 x i32> %1, %2
-  ret <4 x i32> %3
-}
-
-define <2 x i64> @test_psubq(<2 x i64> %a0, <2 x i64> %a1, <2 x i64> *%a2) {
-; GENERIC-LABEL: test_psubq:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    psubq %xmm1, %xmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    psubq (%rdi), %xmm0 # sched: [7:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_psubq:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    psubq %xmm1, %xmm0 # sched: [2:1.00]
-; ATOM-NEXT:    psubq (%rdi), %xmm0 # sched: [3:1.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_psubq:
-; SLM:       # %bb.0:
-; SLM-NEXT:    psubq %xmm1, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    psubq (%rdi), %xmm0 # sched: [4:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_psubq:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    psubq %xmm1, %xmm0 # sched: [1:0.50]
-; SANDY-SSE-NEXT:    psubq (%rdi), %xmm0 # sched: [7:0.50]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_psubq:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vpsubq %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SANDY-NEXT:    vpsubq (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_psubq:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    psubq %xmm1, %xmm0 # sched: [1:0.50]
-; HASWELL-SSE-NEXT:    psubq (%rdi), %xmm0 # sched: [7:0.50]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_psubq:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpsubq %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; HASWELL-NEXT:    vpsubq (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_psubq:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    psubq %xmm1, %xmm0 # sched: [1:0.50]
-; BROADWELL-SSE-NEXT:    psubq (%rdi), %xmm0 # sched: [6:0.50]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_psubq:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpsubq %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    vpsubq (%rdi), %xmm0, %xmm0 # sched: [6:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_psubq:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    psubq %xmm1, %xmm0 # sched: [1:0.33]
-; SKYLAKE-SSE-NEXT:    psubq (%rdi), %xmm0 # sched: [7:0.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_psubq:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpsubq %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    vpsubq (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_psubq:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    psubq %xmm1, %xmm0 # sched: [1:0.33]
-; SKX-SSE-NEXT:    psubq (%rdi), %xmm0 # sched: [7:0.50]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_psubq:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpsubq %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    vpsubq (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_psubq:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    psubq %xmm1, %xmm0 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    psubq (%rdi), %xmm0 # sched: [7:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_psubq:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vpsubq %xmm1, %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    vpsubq (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_psubq:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    psubq %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    psubq (%rdi), %xmm0 # sched: [6:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_psubq:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vpsubq %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    vpsubq (%rdi), %xmm0, %xmm0 # sched: [6:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_psubq:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    psubq %xmm1, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    psubq (%rdi), %xmm0 # sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_psubq:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpsubq %xmm1, %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vpsubq (%rdi), %xmm0, %xmm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = sub <2 x i64> %a0, %a1
-  %2 = load <2 x i64>, <2 x i64> *%a2, align 16
-  %3 = sub <2 x i64> %1, %2
-  ret <2 x i64> %3
-}
-
-define <16 x i8> @test_psubsb(<16 x i8> %a0, <16 x i8> %a1, <16 x i8> *%a2) {
-; GENERIC-LABEL: test_psubsb:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    psubsb %xmm1, %xmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    psubsb (%rdi), %xmm0 # sched: [7:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_psubsb:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    psubsb %xmm1, %xmm0 # sched: [1:0.50]
-; ATOM-NEXT:    psubsb (%rdi), %xmm0 # sched: [1:1.00]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_psubsb:
-; SLM:       # %bb.0:
-; SLM-NEXT:    psubsb %xmm1, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    psubsb (%rdi), %xmm0 # sched: [4:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_psubsb:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    psubsb %xmm1, %xmm0 # sched: [1:0.50]
-; SANDY-SSE-NEXT:    psubsb (%rdi), %xmm0 # sched: [7:0.50]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_psubsb:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vpsubsb %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SANDY-NEXT:    vpsubsb (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_psubsb:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    psubsb %xmm1, %xmm0 # sched: [1:0.50]
-; HASWELL-SSE-NEXT:    psubsb (%rdi), %xmm0 # sched: [7:0.50]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_psubsb:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpsubsb %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; HASWELL-NEXT:    vpsubsb (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_psubsb:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    psubsb %xmm1, %xmm0 # sched: [1:0.50]
-; BROADWELL-SSE-NEXT:    psubsb (%rdi), %xmm0 # sched: [6:0.50]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_psubsb:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpsubsb %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    vpsubsb (%rdi), %xmm0, %xmm0 # sched: [6:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_psubsb:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    psubsb %xmm1, %xmm0 # sched: [1:0.50]
-; SKYLAKE-SSE-NEXT:    psubsb (%rdi), %xmm0 # sched: [7:0.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_psubsb:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpsubsb %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SKYLAKE-NEXT:    vpsubsb (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_psubsb:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    psubsb %xmm1, %xmm0 # sched: [1:0.50]
-; SKX-SSE-NEXT:    psubsb (%rdi), %xmm0 # sched: [7:0.50]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_psubsb:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpsubsb %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SKX-NEXT:    vpsubsb (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_psubsb:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    psubsb %xmm1, %xmm0 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    psubsb (%rdi), %xmm0 # sched: [7:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_psubsb:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vpsubsb %xmm1, %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    vpsubsb (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_psubsb:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    psubsb %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    psubsb (%rdi), %xmm0 # sched: [6:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_psubsb:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vpsubsb %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    vpsubsb (%rdi), %xmm0, %xmm0 # sched: [6:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_psubsb:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    psubsb %xmm1, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    psubsb (%rdi), %xmm0 # sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_psubsb:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpsubsb %xmm1, %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vpsubsb (%rdi), %xmm0, %xmm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <16 x i8> @llvm.x86.sse2.psubs.b(<16 x i8> %a0, <16 x i8> %a1)
-  %2 = load <16 x i8>, <16 x i8> *%a2, align 16
-  %3 = call <16 x i8> @llvm.x86.sse2.psubs.b(<16 x i8> %1, <16 x i8> %2)
-  ret <16 x i8> %3
-}
-declare <16 x i8> @llvm.x86.sse2.psubs.b(<16 x i8>, <16 x i8>) nounwind readnone
-
-define <8 x i16> @test_psubsw(<8 x i16> %a0, <8 x i16> %a1, <8 x i16> *%a2) {
-; GENERIC-LABEL: test_psubsw:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    psubsw %xmm1, %xmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    psubsw (%rdi), %xmm0 # sched: [7:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_psubsw:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    psubsw %xmm1, %xmm0 # sched: [1:0.50]
-; ATOM-NEXT:    psubsw (%rdi), %xmm0 # sched: [1:1.00]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_psubsw:
-; SLM:       # %bb.0:
-; SLM-NEXT:    psubsw %xmm1, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    psubsw (%rdi), %xmm0 # sched: [4:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_psubsw:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    psubsw %xmm1, %xmm0 # sched: [1:0.50]
-; SANDY-SSE-NEXT:    psubsw (%rdi), %xmm0 # sched: [7:0.50]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_psubsw:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vpsubsw %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SANDY-NEXT:    vpsubsw (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_psubsw:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    psubsw %xmm1, %xmm0 # sched: [1:0.50]
-; HASWELL-SSE-NEXT:    psubsw (%rdi), %xmm0 # sched: [7:0.50]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_psubsw:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpsubsw %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; HASWELL-NEXT:    vpsubsw (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_psubsw:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    psubsw %xmm1, %xmm0 # sched: [1:0.50]
-; BROADWELL-SSE-NEXT:    psubsw (%rdi), %xmm0 # sched: [6:0.50]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_psubsw:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpsubsw %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    vpsubsw (%rdi), %xmm0, %xmm0 # sched: [6:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_psubsw:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    psubsw %xmm1, %xmm0 # sched: [1:0.50]
-; SKYLAKE-SSE-NEXT:    psubsw (%rdi), %xmm0 # sched: [7:0.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_psubsw:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpsubsw %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SKYLAKE-NEXT:    vpsubsw (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_psubsw:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    psubsw %xmm1, %xmm0 # sched: [1:0.50]
-; SKX-SSE-NEXT:    psubsw (%rdi), %xmm0 # sched: [7:0.50]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_psubsw:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpsubsw %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SKX-NEXT:    vpsubsw (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_psubsw:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    psubsw %xmm1, %xmm0 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    psubsw (%rdi), %xmm0 # sched: [7:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_psubsw:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vpsubsw %xmm1, %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    vpsubsw (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_psubsw:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    psubsw %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    psubsw (%rdi), %xmm0 # sched: [6:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_psubsw:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vpsubsw %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    vpsubsw (%rdi), %xmm0, %xmm0 # sched: [6:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_psubsw:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    psubsw %xmm1, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    psubsw (%rdi), %xmm0 # sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_psubsw:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpsubsw %xmm1, %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vpsubsw (%rdi), %xmm0, %xmm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <8 x i16> @llvm.x86.sse2.psubs.w(<8 x i16> %a0, <8 x i16> %a1)
-  %2 = load <8 x i16>, <8 x i16> *%a2, align 16
-  %3 = call <8 x i16> @llvm.x86.sse2.psubs.w(<8 x i16> %1, <8 x i16> %2)
-  ret <8 x i16> %3
-}
-declare <8 x i16> @llvm.x86.sse2.psubs.w(<8 x i16>, <8 x i16>) nounwind readnone
-
-define <16 x i8> @test_psubusb(<16 x i8> %a0, <16 x i8> %a1, <16 x i8> *%a2) {
-; GENERIC-LABEL: test_psubusb:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    psubusb %xmm1, %xmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    psubusb (%rdi), %xmm0 # sched: [7:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_psubusb:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    psubusb %xmm1, %xmm0 # sched: [1:0.50]
-; ATOM-NEXT:    psubusb (%rdi), %xmm0 # sched: [1:1.00]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_psubusb:
-; SLM:       # %bb.0:
-; SLM-NEXT:    psubusb %xmm1, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    psubusb (%rdi), %xmm0 # sched: [4:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_psubusb:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    psubusb %xmm1, %xmm0 # sched: [1:0.50]
-; SANDY-SSE-NEXT:    psubusb (%rdi), %xmm0 # sched: [7:0.50]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_psubusb:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vpsubusb %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SANDY-NEXT:    vpsubusb (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_psubusb:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    psubusb %xmm1, %xmm0 # sched: [1:0.50]
-; HASWELL-SSE-NEXT:    psubusb (%rdi), %xmm0 # sched: [7:0.50]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_psubusb:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpsubusb %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; HASWELL-NEXT:    vpsubusb (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_psubusb:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    psubusb %xmm1, %xmm0 # sched: [1:0.50]
-; BROADWELL-SSE-NEXT:    psubusb (%rdi), %xmm0 # sched: [6:0.50]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_psubusb:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpsubusb %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    vpsubusb (%rdi), %xmm0, %xmm0 # sched: [6:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_psubusb:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    psubusb %xmm1, %xmm0 # sched: [1:0.50]
-; SKYLAKE-SSE-NEXT:    psubusb (%rdi), %xmm0 # sched: [7:0.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_psubusb:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpsubusb %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SKYLAKE-NEXT:    vpsubusb (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_psubusb:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    psubusb %xmm1, %xmm0 # sched: [1:0.50]
-; SKX-SSE-NEXT:    psubusb (%rdi), %xmm0 # sched: [7:0.50]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_psubusb:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpsubusb %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SKX-NEXT:    vpsubusb (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_psubusb:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    psubusb %xmm1, %xmm0 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    psubusb (%rdi), %xmm0 # sched: [7:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_psubusb:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vpsubusb %xmm1, %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    vpsubusb (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_psubusb:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    psubusb %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    psubusb (%rdi), %xmm0 # sched: [6:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_psubusb:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vpsubusb %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    vpsubusb (%rdi), %xmm0, %xmm0 # sched: [6:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_psubusb:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    psubusb %xmm1, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    psubusb (%rdi), %xmm0 # sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_psubusb:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpsubusb %xmm1, %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vpsubusb (%rdi), %xmm0, %xmm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <16 x i8> @llvm.x86.sse2.psubus.b(<16 x i8> %a0, <16 x i8> %a1)
-  %2 = load <16 x i8>, <16 x i8> *%a2, align 16
-  %3 = call <16 x i8> @llvm.x86.sse2.psubus.b(<16 x i8> %1, <16 x i8> %2)
-  ret <16 x i8> %3
-}
-declare <16 x i8> @llvm.x86.sse2.psubus.b(<16 x i8>, <16 x i8>) nounwind readnone
-
-define <8 x i16> @test_psubusw(<8 x i16> %a0, <8 x i16> %a1, <8 x i16> *%a2) {
-; GENERIC-LABEL: test_psubusw:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    psubusw %xmm1, %xmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    psubusw (%rdi), %xmm0 # sched: [7:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_psubusw:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    psubusw %xmm1, %xmm0 # sched: [1:0.50]
-; ATOM-NEXT:    psubusw (%rdi), %xmm0 # sched: [1:1.00]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_psubusw:
-; SLM:       # %bb.0:
-; SLM-NEXT:    psubusw %xmm1, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    psubusw (%rdi), %xmm0 # sched: [4:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_psubusw:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    psubusw %xmm1, %xmm0 # sched: [1:0.50]
-; SANDY-SSE-NEXT:    psubusw (%rdi), %xmm0 # sched: [7:0.50]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_psubusw:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vpsubusw %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SANDY-NEXT:    vpsubusw (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_psubusw:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    psubusw %xmm1, %xmm0 # sched: [1:0.50]
-; HASWELL-SSE-NEXT:    psubusw (%rdi), %xmm0 # sched: [7:0.50]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_psubusw:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpsubusw %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; HASWELL-NEXT:    vpsubusw (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_psubusw:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    psubusw %xmm1, %xmm0 # sched: [1:0.50]
-; BROADWELL-SSE-NEXT:    psubusw (%rdi), %xmm0 # sched: [6:0.50]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_psubusw:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpsubusw %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    vpsubusw (%rdi), %xmm0, %xmm0 # sched: [6:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_psubusw:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    psubusw %xmm1, %xmm0 # sched: [1:0.50]
-; SKYLAKE-SSE-NEXT:    psubusw (%rdi), %xmm0 # sched: [7:0.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_psubusw:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpsubusw %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SKYLAKE-NEXT:    vpsubusw (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_psubusw:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    psubusw %xmm1, %xmm0 # sched: [1:0.50]
-; SKX-SSE-NEXT:    psubusw (%rdi), %xmm0 # sched: [7:0.50]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_psubusw:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpsubusw %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SKX-NEXT:    vpsubusw (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_psubusw:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    psubusw %xmm1, %xmm0 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    psubusw (%rdi), %xmm0 # sched: [7:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_psubusw:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vpsubusw %xmm1, %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    vpsubusw (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_psubusw:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    psubusw %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    psubusw (%rdi), %xmm0 # sched: [6:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_psubusw:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vpsubusw %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    vpsubusw (%rdi), %xmm0, %xmm0 # sched: [6:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_psubusw:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    psubusw %xmm1, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    psubusw (%rdi), %xmm0 # sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_psubusw:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpsubusw %xmm1, %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vpsubusw (%rdi), %xmm0, %xmm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <8 x i16> @llvm.x86.sse2.psubus.w(<8 x i16> %a0, <8 x i16> %a1)
-  %2 = load <8 x i16>, <8 x i16> *%a2, align 16
-  %3 = call <8 x i16> @llvm.x86.sse2.psubus.w(<8 x i16> %1, <8 x i16> %2)
-  ret <8 x i16> %3
-}
-declare <8 x i16> @llvm.x86.sse2.psubus.w(<8 x i16>, <8 x i16>) nounwind readnone
-
-define <8 x i16> @test_psubw(<8 x i16> %a0, <8 x i16> %a1, <8 x i16> *%a2) {
-; GENERIC-LABEL: test_psubw:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    psubw %xmm1, %xmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    psubw (%rdi), %xmm0 # sched: [7:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_psubw:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    psubw %xmm1, %xmm0 # sched: [1:0.50]
-; ATOM-NEXT:    psubw (%rdi), %xmm0 # sched: [1:1.00]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_psubw:
-; SLM:       # %bb.0:
-; SLM-NEXT:    psubw %xmm1, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    psubw (%rdi), %xmm0 # sched: [4:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_psubw:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    psubw %xmm1, %xmm0 # sched: [1:0.50]
-; SANDY-SSE-NEXT:    psubw (%rdi), %xmm0 # sched: [7:0.50]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_psubw:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vpsubw %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SANDY-NEXT:    vpsubw (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_psubw:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    psubw %xmm1, %xmm0 # sched: [1:0.50]
-; HASWELL-SSE-NEXT:    psubw (%rdi), %xmm0 # sched: [7:0.50]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_psubw:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpsubw %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; HASWELL-NEXT:    vpsubw (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_psubw:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    psubw %xmm1, %xmm0 # sched: [1:0.50]
-; BROADWELL-SSE-NEXT:    psubw (%rdi), %xmm0 # sched: [6:0.50]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_psubw:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpsubw %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    vpsubw (%rdi), %xmm0, %xmm0 # sched: [6:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_psubw:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    psubw %xmm1, %xmm0 # sched: [1:0.33]
-; SKYLAKE-SSE-NEXT:    psubw (%rdi), %xmm0 # sched: [7:0.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_psubw:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpsubw %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    vpsubw (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_psubw:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    psubw %xmm1, %xmm0 # sched: [1:0.33]
-; SKX-SSE-NEXT:    psubw (%rdi), %xmm0 # sched: [7:0.50]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_psubw:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpsubw %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    vpsubw (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_psubw:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    psubw %xmm1, %xmm0 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    psubw (%rdi), %xmm0 # sched: [7:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_psubw:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vpsubw %xmm1, %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    vpsubw (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_psubw:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    psubw %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    psubw (%rdi), %xmm0 # sched: [6:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_psubw:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vpsubw %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    vpsubw (%rdi), %xmm0, %xmm0 # sched: [6:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_psubw:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    psubw %xmm1, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    psubw (%rdi), %xmm0 # sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_psubw:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpsubw %xmm1, %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vpsubw (%rdi), %xmm0, %xmm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = sub <8 x i16> %a0, %a1
-  %2 = load <8 x i16>, <8 x i16> *%a2, align 16
-  %3 = sub <8 x i16> %1, %2
-  ret <8 x i16> %3
-}
-
-define <16 x i8> @test_punpckhbw(<16 x i8> %a0, <16 x i8> %a1, <16 x i8> *%a2) {
-; GENERIC-LABEL: test_punpckhbw:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm1[8],xmm0[9],xmm1[9],xmm0[10],xmm1[10],xmm0[11],xmm1[11],xmm0[12],xmm1[12],xmm0[13],xmm1[13],xmm0[14],xmm1[14],xmm0[15],xmm1[15] sched: [1:0.50]
-; GENERIC-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8],mem[8],xmm0[9],mem[9],xmm0[10],mem[10],xmm0[11],mem[11],xmm0[12],mem[12],xmm0[13],mem[13],xmm0[14],mem[14],xmm0[15],mem[15] sched: [7:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_punpckhbw:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm1[8],xmm0[9],xmm1[9],xmm0[10],xmm1[10],xmm0[11],xmm1[11],xmm0[12],xmm1[12],xmm0[13],xmm1[13],xmm0[14],xmm1[14],xmm0[15],xmm1[15] sched: [1:1.00]
-; ATOM-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8],mem[8],xmm0[9],mem[9],xmm0[10],mem[10],xmm0[11],mem[11],xmm0[12],mem[12],xmm0[13],mem[13],xmm0[14],mem[14],xmm0[15],mem[15] sched: [1:1.00]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_punpckhbw:
-; SLM:       # %bb.0:
-; SLM-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm1[8],xmm0[9],xmm1[9],xmm0[10],xmm1[10],xmm0[11],xmm1[11],xmm0[12],xmm1[12],xmm0[13],xmm1[13],xmm0[14],xmm1[14],xmm0[15],xmm1[15] sched: [1:1.00]
-; SLM-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8],mem[8],xmm0[9],mem[9],xmm0[10],mem[10],xmm0[11],mem[11],xmm0[12],mem[12],xmm0[13],mem[13],xmm0[14],mem[14],xmm0[15],mem[15] sched: [4:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_punpckhbw:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm1[8],xmm0[9],xmm1[9],xmm0[10],xmm1[10],xmm0[11],xmm1[11],xmm0[12],xmm1[12],xmm0[13],xmm1[13],xmm0[14],xmm1[14],xmm0[15],xmm1[15] sched: [1:0.50]
-; SANDY-SSE-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8],mem[8],xmm0[9],mem[9],xmm0[10],mem[10],xmm0[11],mem[11],xmm0[12],mem[12],xmm0[13],mem[13],xmm0[14],mem[14],xmm0[15],mem[15] sched: [7:0.50]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_punpckhbw:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm0[8],xmm1[8],xmm0[9],xmm1[9],xmm0[10],xmm1[10],xmm0[11],xmm1[11],xmm0[12],xmm1[12],xmm0[13],xmm1[13],xmm0[14],xmm1[14],xmm0[15],xmm1[15] sched: [1:0.50]
-; SANDY-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm0[8],mem[8],xmm0[9],mem[9],xmm0[10],mem[10],xmm0[11],mem[11],xmm0[12],mem[12],xmm0[13],mem[13],xmm0[14],mem[14],xmm0[15],mem[15] sched: [7:0.50]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_punpckhbw:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm1[8],xmm0[9],xmm1[9],xmm0[10],xmm1[10],xmm0[11],xmm1[11],xmm0[12],xmm1[12],xmm0[13],xmm1[13],xmm0[14],xmm1[14],xmm0[15],xmm1[15] sched: [1:1.00]
-; HASWELL-SSE-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8],mem[8],xmm0[9],mem[9],xmm0[10],mem[10],xmm0[11],mem[11],xmm0[12],mem[12],xmm0[13],mem[13],xmm0[14],mem[14],xmm0[15],mem[15] sched: [7:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_punpckhbw:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm0[8],xmm1[8],xmm0[9],xmm1[9],xmm0[10],xmm1[10],xmm0[11],xmm1[11],xmm0[12],xmm1[12],xmm0[13],xmm1[13],xmm0[14],xmm1[14],xmm0[15],xmm1[15] sched: [1:1.00]
-; HASWELL-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm0[8],mem[8],xmm0[9],mem[9],xmm0[10],mem[10],xmm0[11],mem[11],xmm0[12],mem[12],xmm0[13],mem[13],xmm0[14],mem[14],xmm0[15],mem[15] sched: [7:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_punpckhbw:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm1[8],xmm0[9],xmm1[9],xmm0[10],xmm1[10],xmm0[11],xmm1[11],xmm0[12],xmm1[12],xmm0[13],xmm1[13],xmm0[14],xmm1[14],xmm0[15],xmm1[15] sched: [1:1.00]
-; BROADWELL-SSE-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8],mem[8],xmm0[9],mem[9],xmm0[10],mem[10],xmm0[11],mem[11],xmm0[12],mem[12],xmm0[13],mem[13],xmm0[14],mem[14],xmm0[15],mem[15] sched: [6:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_punpckhbw:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm0[8],xmm1[8],xmm0[9],xmm1[9],xmm0[10],xmm1[10],xmm0[11],xmm1[11],xmm0[12],xmm1[12],xmm0[13],xmm1[13],xmm0[14],xmm1[14],xmm0[15],xmm1[15] sched: [1:1.00]
-; BROADWELL-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm0[8],mem[8],xmm0[9],mem[9],xmm0[10],mem[10],xmm0[11],mem[11],xmm0[12],mem[12],xmm0[13],mem[13],xmm0[14],mem[14],xmm0[15],mem[15] sched: [6:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_punpckhbw:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm1[8],xmm0[9],xmm1[9],xmm0[10],xmm1[10],xmm0[11],xmm1[11],xmm0[12],xmm1[12],xmm0[13],xmm1[13],xmm0[14],xmm1[14],xmm0[15],xmm1[15] sched: [1:1.00]
-; SKYLAKE-SSE-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8],mem[8],xmm0[9],mem[9],xmm0[10],mem[10],xmm0[11],mem[11],xmm0[12],mem[12],xmm0[13],mem[13],xmm0[14],mem[14],xmm0[15],mem[15] sched: [7:1.00]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_punpckhbw:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm0[8],xmm1[8],xmm0[9],xmm1[9],xmm0[10],xmm1[10],xmm0[11],xmm1[11],xmm0[12],xmm1[12],xmm0[13],xmm1[13],xmm0[14],xmm1[14],xmm0[15],xmm1[15] sched: [1:1.00]
-; SKYLAKE-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm0[8],mem[8],xmm0[9],mem[9],xmm0[10],mem[10],xmm0[11],mem[11],xmm0[12],mem[12],xmm0[13],mem[13],xmm0[14],mem[14],xmm0[15],mem[15] sched: [7:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_punpckhbw:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm1[8],xmm0[9],xmm1[9],xmm0[10],xmm1[10],xmm0[11],xmm1[11],xmm0[12],xmm1[12],xmm0[13],xmm1[13],xmm0[14],xmm1[14],xmm0[15],xmm1[15] sched: [1:1.00]
-; SKX-SSE-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8],mem[8],xmm0[9],mem[9],xmm0[10],mem[10],xmm0[11],mem[11],xmm0[12],mem[12],xmm0[13],mem[13],xmm0[14],mem[14],xmm0[15],mem[15] sched: [7:1.00]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_punpckhbw:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm0[8],xmm1[8],xmm0[9],xmm1[9],xmm0[10],xmm1[10],xmm0[11],xmm1[11],xmm0[12],xmm1[12],xmm0[13],xmm1[13],xmm0[14],xmm1[14],xmm0[15],xmm1[15] sched: [1:1.00]
-; SKX-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm0[8],mem[8],xmm0[9],mem[9],xmm0[10],mem[10],xmm0[11],mem[11],xmm0[12],mem[12],xmm0[13],mem[13],xmm0[14],mem[14],xmm0[15],mem[15] sched: [7:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_punpckhbw:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm1[8],xmm0[9],xmm1[9],xmm0[10],xmm1[10],xmm0[11],xmm1[11],xmm0[12],xmm1[12],xmm0[13],xmm1[13],xmm0[14],xmm1[14],xmm0[15],xmm1[15] sched: [2:0.50]
-; BDVER2-SSE-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8],mem[8],xmm0[9],mem[9],xmm0[10],mem[10],xmm0[11],mem[11],xmm0[12],mem[12],xmm0[13],mem[13],xmm0[14],mem[14],xmm0[15],mem[15] sched: [7:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_punpckhbw:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm0[8],xmm1[8],xmm0[9],xmm1[9],xmm0[10],xmm1[10],xmm0[11],xmm1[11],xmm0[12],xmm1[12],xmm0[13],xmm1[13],xmm0[14],xmm1[14],xmm0[15],xmm1[15] sched: [2:0.50]
-; BDVER2-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm0[8],mem[8],xmm0[9],mem[9],xmm0[10],mem[10],xmm0[11],mem[11],xmm0[12],mem[12],xmm0[13],mem[13],xmm0[14],mem[14],xmm0[15],mem[15] sched: [7:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_punpckhbw:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm1[8],xmm0[9],xmm1[9],xmm0[10],xmm1[10],xmm0[11],xmm1[11],xmm0[12],xmm1[12],xmm0[13],xmm1[13],xmm0[14],xmm1[14],xmm0[15],xmm1[15] sched: [1:0.50]
-; BTVER2-SSE-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8],mem[8],xmm0[9],mem[9],xmm0[10],mem[10],xmm0[11],mem[11],xmm0[12],mem[12],xmm0[13],mem[13],xmm0[14],mem[14],xmm0[15],mem[15] sched: [6:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_punpckhbw:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm0[8],xmm1[8],xmm0[9],xmm1[9],xmm0[10],xmm1[10],xmm0[11],xmm1[11],xmm0[12],xmm1[12],xmm0[13],xmm1[13],xmm0[14],xmm1[14],xmm0[15],xmm1[15] sched: [1:0.50]
-; BTVER2-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm0[8],mem[8],xmm0[9],mem[9],xmm0[10],mem[10],xmm0[11],mem[11],xmm0[12],mem[12],xmm0[13],mem[13],xmm0[14],mem[14],xmm0[15],mem[15] sched: [6:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_punpckhbw:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm1[8],xmm0[9],xmm1[9],xmm0[10],xmm1[10],xmm0[11],xmm1[11],xmm0[12],xmm1[12],xmm0[13],xmm1[13],xmm0[14],xmm1[14],xmm0[15],xmm1[15] sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    punpckhbw {{.*#+}} xmm0 = xmm0[8],mem[8],xmm0[9],mem[9],xmm0[10],mem[10],xmm0[11],mem[11],xmm0[12],mem[12],xmm0[13],mem[13],xmm0[14],mem[14],xmm0[15],mem[15] sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_punpckhbw:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm0[8],xmm1[8],xmm0[9],xmm1[9],xmm0[10],xmm1[10],xmm0[11],xmm1[11],xmm0[12],xmm1[12],xmm0[13],xmm1[13],xmm0[14],xmm1[14],xmm0[15],xmm1[15] sched: [1:0.25]
-; ZNVER1-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm0[8],mem[8],xmm0[9],mem[9],xmm0[10],mem[10],xmm0[11],mem[11],xmm0[12],mem[12],xmm0[13],mem[13],xmm0[14],mem[14],xmm0[15],mem[15] sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = shufflevector <16 x i8> %a0, <16 x i8> %a1, <16 x i32> <i32 8, i32 24, i32 9, i32 25, i32 10, i32 26, i32 11, i32 27, i32 12, i32 28, i32 13, i32 29, i32 14, i32 30, i32 15, i32 31>
-  %2 = load <16 x i8>, <16 x i8> *%a2, align 16
-  %3 = shufflevector <16 x i8> %1, <16 x i8> %2, <16 x i32> <i32 8, i32 24, i32 9, i32 25, i32 10, i32 26, i32 11, i32 27, i32 12, i32 28, i32 13, i32 29, i32 14, i32 30, i32 15, i32 31>
-  ret <16 x i8> %3
-}
-
-define <4 x i32> @test_punpckhdq(<4 x i32> %a0, <4 x i32> %a1, <4 x i32> *%a2) {
-; GENERIC-LABEL: test_punpckhdq:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3] sched: [1:0.50]
-; GENERIC-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],mem[2],xmm1[3],mem[3] sched: [7:0.50]
-; GENERIC-NEXT:    paddd %xmm1, %xmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_punpckhdq:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3] sched: [1:1.00]
-; ATOM-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],mem[2],xmm1[3],mem[3] sched: [1:1.00]
-; ATOM-NEXT:    paddd %xmm1, %xmm0 # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_punpckhdq:
-; SLM:       # %bb.0:
-; SLM-NEXT:    punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3] sched: [1:1.00]
-; SLM-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],mem[2],xmm1[3],mem[3] sched: [4:1.00]
-; SLM-NEXT:    paddd %xmm1, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_punpckhdq:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3] sched: [1:0.50]
-; SANDY-SSE-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],mem[2],xmm1[3],mem[3] sched: [7:0.50]
-; SANDY-SSE-NEXT:    paddd %xmm1, %xmm0 # sched: [1:0.50]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_punpckhdq:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vpunpckhdq {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3] sched: [1:0.50]
-; SANDY-NEXT:    vpunpckhdq {{.*#+}} xmm1 = xmm1[2],mem[2],xmm1[3],mem[3] sched: [7:0.50]
-; SANDY-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_punpckhdq:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3] sched: [1:1.00]
-; HASWELL-SSE-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],mem[2],xmm1[3],mem[3] sched: [7:1.00]
-; HASWELL-SSE-NEXT:    paddd %xmm1, %xmm0 # sched: [1:0.50]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_punpckhdq:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpunpckhdq {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3] sched: [1:1.00]
-; HASWELL-NEXT:    vpunpckhdq {{.*#+}} xmm1 = xmm1[2],mem[2],xmm1[3],mem[3] sched: [7:1.00]
-; HASWELL-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_punpckhdq:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3] sched: [1:1.00]
-; BROADWELL-SSE-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],mem[2],xmm1[3],mem[3] sched: [6:1.00]
-; BROADWELL-SSE-NEXT:    paddd %xmm1, %xmm0 # sched: [1:0.50]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_punpckhdq:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpunpckhdq {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3] sched: [1:1.00]
-; BROADWELL-NEXT:    vpunpckhdq {{.*#+}} xmm1 = xmm1[2],mem[2],xmm1[3],mem[3] sched: [6:1.00]
-; BROADWELL-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_punpckhdq:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3] sched: [1:1.00]
-; SKYLAKE-SSE-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],mem[2],xmm1[3],mem[3] sched: [7:1.00]
-; SKYLAKE-SSE-NEXT:    paddd %xmm1, %xmm0 # sched: [1:0.33]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_punpckhdq:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpunpckhdq {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3] sched: [1:1.00]
-; SKYLAKE-NEXT:    vpunpckhdq {{.*#+}} xmm1 = xmm1[2],mem[2],xmm1[3],mem[3] sched: [7:1.00]
-; SKYLAKE-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_punpckhdq:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3] sched: [1:1.00]
-; SKX-SSE-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],mem[2],xmm1[3],mem[3] sched: [7:1.00]
-; SKX-SSE-NEXT:    paddd %xmm1, %xmm0 # sched: [1:0.33]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_punpckhdq:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpunpckhdq {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3] sched: [1:1.00]
-; SKX-NEXT:    vpunpckhdq {{.*#+}} xmm1 = xmm1[2],mem[2],xmm1[3],mem[3] sched: [7:1.00]
-; SKX-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_punpckhdq:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3] sched: [2:0.50]
-; BDVER2-SSE-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],mem[2],xmm1[3],mem[3] sched: [7:0.50]
-; BDVER2-SSE-NEXT:    paddd %xmm1, %xmm0 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_punpckhdq:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vpunpckhdq {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3] sched: [2:0.50]
-; BDVER2-NEXT:    vpunpckhdq {{.*#+}} xmm1 = xmm1[2],mem[2],xmm1[3],mem[3] sched: [7:0.50]
-; BDVER2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_punpckhdq:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3] sched: [1:0.50]
-; BTVER2-SSE-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],mem[2],xmm1[3],mem[3] sched: [6:1.00]
-; BTVER2-SSE-NEXT:    paddd %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_punpckhdq:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vpunpckhdq {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3] sched: [1:0.50]
-; BTVER2-NEXT:    vpunpckhdq {{.*#+}} xmm1 = xmm1[2],mem[2],xmm1[3],mem[3] sched: [6:1.00]
-; BTVER2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_punpckhdq:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3] sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],mem[2],xmm1[3],mem[3] sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    paddd %xmm1, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_punpckhdq:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpunpckhdq {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3] sched: [1:0.25]
-; ZNVER1-NEXT:    vpunpckhdq {{.*#+}} xmm1 = xmm1[2],mem[2],xmm1[3],mem[3] sched: [8:0.50]
-; ZNVER1-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = shufflevector <4 x i32> %a0, <4 x i32> %a1, <4 x i32> <i32 2, i32 6, i32 3, i32 7>
-  %2 = load <4 x i32>, <4 x i32> *%a2, align 16
-  %3 = shufflevector <4 x i32> %a1, <4 x i32> %2, <4 x i32> <i32 2, i32 6, i32 3, i32 7>
-  %4 = add <4 x i32> %1, %3
-  ret <4 x i32> %4
-}
-
-define <2 x i64> @test_punpckhqdq(<2 x i64> %a0, <2 x i64> %a1, <2 x i64> *%a2) {
-; GENERIC-LABEL: test_punpckhqdq:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    punpckhqdq {{.*#+}} xmm0 = xmm0[1],xmm1[1] sched: [1:0.50]
-; GENERIC-NEXT:    punpckhqdq {{.*#+}} xmm1 = xmm1[1],mem[1] sched: [7:0.50]
-; GENERIC-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_punpckhqdq:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    punpckhqdq {{.*#+}} xmm0 = xmm0[1],xmm1[1] sched: [1:1.00]
-; ATOM-NEXT:    punpckhqdq {{.*#+}} xmm1 = xmm1[1],mem[1] sched: [1:1.00]
-; ATOM-NEXT:    paddq %xmm1, %xmm0 # sched: [2:1.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_punpckhqdq:
-; SLM:       # %bb.0:
-; SLM-NEXT:    punpckhqdq {{.*#+}} xmm0 = xmm0[1],xmm1[1] sched: [1:1.00]
-; SLM-NEXT:    punpckhqdq {{.*#+}} xmm1 = xmm1[1],mem[1] sched: [4:1.00]
-; SLM-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_punpckhqdq:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    punpckhqdq {{.*#+}} xmm0 = xmm0[1],xmm1[1] sched: [1:0.50]
-; SANDY-SSE-NEXT:    punpckhqdq {{.*#+}} xmm1 = xmm1[1],mem[1] sched: [7:0.50]
-; SANDY-SSE-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.50]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_punpckhqdq:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vpunpckhqdq {{.*#+}} xmm0 = xmm0[1],xmm1[1] sched: [1:0.50]
-; SANDY-NEXT:    vpunpckhqdq {{.*#+}} xmm1 = xmm1[1],mem[1] sched: [7:0.50]
-; SANDY-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_punpckhqdq:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    punpckhqdq {{.*#+}} xmm0 = xmm0[1],xmm1[1] sched: [1:1.00]
-; HASWELL-SSE-NEXT:    punpckhqdq {{.*#+}} xmm1 = xmm1[1],mem[1] sched: [7:1.00]
-; HASWELL-SSE-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.50]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_punpckhqdq:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpunpckhqdq {{.*#+}} xmm0 = xmm0[1],xmm1[1] sched: [1:1.00]
-; HASWELL-NEXT:    vpunpckhqdq {{.*#+}} xmm1 = xmm1[1],mem[1] sched: [7:1.00]
-; HASWELL-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_punpckhqdq:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    punpckhqdq {{.*#+}} xmm0 = xmm0[1],xmm1[1] sched: [1:1.00]
-; BROADWELL-SSE-NEXT:    punpckhqdq {{.*#+}} xmm1 = xmm1[1],mem[1] sched: [6:1.00]
-; BROADWELL-SSE-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.50]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_punpckhqdq:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpunpckhqdq {{.*#+}} xmm0 = xmm0[1],xmm1[1] sched: [1:1.00]
-; BROADWELL-NEXT:    vpunpckhqdq {{.*#+}} xmm1 = xmm1[1],mem[1] sched: [6:1.00]
-; BROADWELL-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_punpckhqdq:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    punpckhqdq {{.*#+}} xmm0 = xmm0[1],xmm1[1] sched: [1:1.00]
-; SKYLAKE-SSE-NEXT:    punpckhqdq {{.*#+}} xmm1 = xmm1[1],mem[1] sched: [7:1.00]
-; SKYLAKE-SSE-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.33]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_punpckhqdq:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpunpckhqdq {{.*#+}} xmm0 = xmm0[1],xmm1[1] sched: [1:1.00]
-; SKYLAKE-NEXT:    vpunpckhqdq {{.*#+}} xmm1 = xmm1[1],mem[1] sched: [7:1.00]
-; SKYLAKE-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_punpckhqdq:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    punpckhqdq {{.*#+}} xmm0 = xmm0[1],xmm1[1] sched: [1:1.00]
-; SKX-SSE-NEXT:    punpckhqdq {{.*#+}} xmm1 = xmm1[1],mem[1] sched: [7:1.00]
-; SKX-SSE-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.33]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_punpckhqdq:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpunpckhqdq {{.*#+}} xmm0 = xmm0[1],xmm1[1] sched: [1:1.00]
-; SKX-NEXT:    vpunpckhqdq {{.*#+}} xmm1 = xmm1[1],mem[1] sched: [7:1.00]
-; SKX-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_punpckhqdq:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    punpckhqdq {{.*#+}} xmm0 = xmm0[1],xmm1[1] sched: [2:0.50]
-; BDVER2-SSE-NEXT:    punpckhqdq {{.*#+}} xmm1 = xmm1[1],mem[1] sched: [7:0.50]
-; BDVER2-SSE-NEXT:    paddq %xmm1, %xmm0 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_punpckhqdq:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vpunpckhqdq {{.*#+}} xmm0 = xmm0[1],xmm1[1] sched: [2:0.50]
-; BDVER2-NEXT:    vpunpckhqdq {{.*#+}} xmm1 = xmm1[1],mem[1] sched: [7:0.50]
-; BDVER2-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_punpckhqdq:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    punpckhqdq {{.*#+}} xmm0 = xmm0[1],xmm1[1] sched: [1:0.50]
-; BTVER2-SSE-NEXT:    punpckhqdq {{.*#+}} xmm1 = xmm1[1],mem[1] sched: [6:1.00]
-; BTVER2-SSE-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_punpckhqdq:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vpunpckhqdq {{.*#+}} xmm0 = xmm0[1],xmm1[1] sched: [1:0.50]
-; BTVER2-NEXT:    vpunpckhqdq {{.*#+}} xmm1 = xmm1[1],mem[1] sched: [6:1.00]
-; BTVER2-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_punpckhqdq:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    punpckhqdq {{.*#+}} xmm0 = xmm0[1],xmm1[1] sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    punpckhqdq {{.*#+}} xmm1 = xmm1[1],mem[1] sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_punpckhqdq:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpunpckhqdq {{.*#+}} xmm0 = xmm0[1],xmm1[1] sched: [1:0.25]
-; ZNVER1-NEXT:    vpunpckhqdq {{.*#+}} xmm1 = xmm1[1],mem[1] sched: [8:0.50]
-; ZNVER1-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = shufflevector <2 x i64> %a0, <2 x i64> %a1, <2 x i32> <i32 1, i32 3>
-  %2 = load <2 x i64>, <2 x i64> *%a2, align 16
-  %3 = shufflevector <2 x i64> %a1, <2 x i64> %2, <2x i32> <i32 1, i32 3>
-  %4 = add <2 x i64> %1, %3
-  ret <2 x i64> %4
-}
-
-define <8 x i16> @test_punpckhwd(<8 x i16> %a0, <8 x i16> %a1, <8 x i16> *%a2) {
-; GENERIC-LABEL: test_punpckhwd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] sched: [1:0.50]
-; GENERIC-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7] sched: [7:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_punpckhwd:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] sched: [1:1.00]
-; ATOM-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7] sched: [1:1.00]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_punpckhwd:
-; SLM:       # %bb.0:
-; SLM-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] sched: [1:1.00]
-; SLM-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7] sched: [4:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_punpckhwd:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] sched: [1:0.50]
-; SANDY-SSE-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7] sched: [7:0.50]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_punpckhwd:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] sched: [1:0.50]
-; SANDY-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7] sched: [7:0.50]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_punpckhwd:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] sched: [1:1.00]
-; HASWELL-SSE-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7] sched: [7:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_punpckhwd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] sched: [1:1.00]
-; HASWELL-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7] sched: [7:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_punpckhwd:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] sched: [1:1.00]
-; BROADWELL-SSE-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7] sched: [6:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_punpckhwd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] sched: [1:1.00]
-; BROADWELL-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7] sched: [6:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_punpckhwd:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] sched: [1:1.00]
-; SKYLAKE-SSE-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7] sched: [7:1.00]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_punpckhwd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] sched: [1:1.00]
-; SKYLAKE-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7] sched: [7:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_punpckhwd:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] sched: [1:1.00]
-; SKX-SSE-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7] sched: [7:1.00]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_punpckhwd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] sched: [1:1.00]
-; SKX-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7] sched: [7:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_punpckhwd:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] sched: [2:0.50]
-; BDVER2-SSE-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7] sched: [7:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_punpckhwd:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] sched: [2:0.50]
-; BDVER2-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7] sched: [7:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_punpckhwd:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] sched: [1:0.50]
-; BTVER2-SSE-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7] sched: [6:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_punpckhwd:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] sched: [1:0.50]
-; BTVER2-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7] sched: [6:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_punpckhwd:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7] sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_punpckhwd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] sched: [1:0.25]
-; ZNVER1-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7] sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = shufflevector <8 x i16> %a0, <8 x i16> %a1, <8 x i32> <i32 4, i32 12, i32 5, i32 13, i32 6, i32 14, i32 7, i32 15>
-  %2 = load <8 x i16>, <8 x i16> *%a2, align 16
-  %3 = shufflevector <8 x i16> %1, <8 x i16> %2, <8 x i32> <i32 4, i32 12, i32 5, i32 13, i32 6, i32 14, i32 7, i32 15>
-  ret <8 x i16> %3
-}
-
-define <16 x i8> @test_punpcklbw(<16 x i8> %a0, <16 x i8> %a1, <16 x i8> *%a2) {
-; GENERIC-LABEL: test_punpcklbw:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] sched: [1:0.50]
-; GENERIC-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1],xmm0[2],mem[2],xmm0[3],mem[3],xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7] sched: [7:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_punpcklbw:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] sched: [1:1.00]
-; ATOM-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1],xmm0[2],mem[2],xmm0[3],mem[3],xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7] sched: [1:1.00]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_punpcklbw:
-; SLM:       # %bb.0:
-; SLM-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] sched: [1:1.00]
-; SLM-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1],xmm0[2],mem[2],xmm0[3],mem[3],xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7] sched: [4:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_punpcklbw:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] sched: [1:0.50]
-; SANDY-SSE-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1],xmm0[2],mem[2],xmm0[3],mem[3],xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7] sched: [7:0.50]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_punpcklbw:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] sched: [1:0.50]
-; SANDY-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1],xmm0[2],mem[2],xmm0[3],mem[3],xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7] sched: [7:0.50]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_punpcklbw:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] sched: [1:1.00]
-; HASWELL-SSE-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1],xmm0[2],mem[2],xmm0[3],mem[3],xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7] sched: [7:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_punpcklbw:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] sched: [1:1.00]
-; HASWELL-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1],xmm0[2],mem[2],xmm0[3],mem[3],xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7] sched: [7:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_punpcklbw:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] sched: [1:1.00]
-; BROADWELL-SSE-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1],xmm0[2],mem[2],xmm0[3],mem[3],xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7] sched: [6:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_punpcklbw:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] sched: [1:1.00]
-; BROADWELL-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1],xmm0[2],mem[2],xmm0[3],mem[3],xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7] sched: [6:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_punpcklbw:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] sched: [1:1.00]
-; SKYLAKE-SSE-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1],xmm0[2],mem[2],xmm0[3],mem[3],xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7] sched: [7:1.00]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_punpcklbw:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] sched: [1:1.00]
-; SKYLAKE-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1],xmm0[2],mem[2],xmm0[3],mem[3],xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7] sched: [7:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_punpcklbw:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] sched: [1:1.00]
-; SKX-SSE-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1],xmm0[2],mem[2],xmm0[3],mem[3],xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7] sched: [7:1.00]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_punpcklbw:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] sched: [1:1.00]
-; SKX-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1],xmm0[2],mem[2],xmm0[3],mem[3],xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7] sched: [7:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_punpcklbw:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] sched: [2:0.50]
-; BDVER2-SSE-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1],xmm0[2],mem[2],xmm0[3],mem[3],xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7] sched: [7:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_punpcklbw:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] sched: [2:0.50]
-; BDVER2-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1],xmm0[2],mem[2],xmm0[3],mem[3],xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7] sched: [7:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_punpcklbw:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] sched: [1:0.50]
-; BTVER2-SSE-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1],xmm0[2],mem[2],xmm0[3],mem[3],xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7] sched: [6:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_punpcklbw:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] sched: [1:0.50]
-; BTVER2-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1],xmm0[2],mem[2],xmm0[3],mem[3],xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7] sched: [6:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_punpcklbw:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1],xmm0[2],mem[2],xmm0[3],mem[3],xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7] sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_punpcklbw:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] sched: [1:0.25]
-; ZNVER1-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1],xmm0[2],mem[2],xmm0[3],mem[3],xmm0[4],mem[4],xmm0[5],mem[5],xmm0[6],mem[6],xmm0[7],mem[7] sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = shufflevector <16 x i8> %a0, <16 x i8> %a1, <16 x i32> <i32 0, i32 16, i32 1, i32 17, i32 2, i32 18, i32 3, i32 19, i32 4, i32 20, i32 5, i32 21, i32 6, i32 22, i32 7, i32 23>
-  %2 = load <16 x i8>, <16 x i8> *%a2, align 16
-  %3 = shufflevector <16 x i8> %1, <16 x i8> %2, <16 x i32> <i32 0, i32 16, i32 1, i32 17, i32 2, i32 18, i32 3, i32 19, i32 4, i32 20, i32 5, i32 21, i32 6, i32 22, i32 7, i32 23>
-  ret <16 x i8> %3
-}
-
-define <4 x i32> @test_punpckldq(<4 x i32> %a0, <4 x i32> %a1, <4 x i32> *%a2) {
-; GENERIC-LABEL: test_punpckldq:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] sched: [1:0.50]
-; GENERIC-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[1],mem[1] sched: [7:0.50]
-; GENERIC-NEXT:    paddd %xmm1, %xmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_punpckldq:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] sched: [1:1.00]
-; ATOM-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[1],mem[1] sched: [1:1.00]
-; ATOM-NEXT:    paddd %xmm1, %xmm0 # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_punpckldq:
-; SLM:       # %bb.0:
-; SLM-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] sched: [1:1.00]
-; SLM-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[1],mem[1] sched: [4:1.00]
-; SLM-NEXT:    paddd %xmm1, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_punpckldq:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] sched: [1:0.50]
-; SANDY-SSE-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[1],mem[1] sched: [7:0.50]
-; SANDY-SSE-NEXT:    paddd %xmm1, %xmm0 # sched: [1:0.50]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_punpckldq:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] sched: [1:0.50]
-; SANDY-NEXT:    vpunpckldq {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[1],mem[1] sched: [7:0.50]
-; SANDY-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_punpckldq:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] sched: [1:1.00]
-; HASWELL-SSE-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[1],mem[1] sched: [7:1.00]
-; HASWELL-SSE-NEXT:    paddd %xmm1, %xmm0 # sched: [1:0.50]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_punpckldq:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] sched: [1:1.00]
-; HASWELL-NEXT:    vpunpckldq {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[1],mem[1] sched: [7:1.00]
-; HASWELL-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_punpckldq:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] sched: [1:1.00]
-; BROADWELL-SSE-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[1],mem[1] sched: [6:1.00]
-; BROADWELL-SSE-NEXT:    paddd %xmm1, %xmm0 # sched: [1:0.50]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_punpckldq:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] sched: [1:1.00]
-; BROADWELL-NEXT:    vpunpckldq {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[1],mem[1] sched: [6:1.00]
-; BROADWELL-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_punpckldq:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] sched: [1:1.00]
-; SKYLAKE-SSE-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[1],mem[1] sched: [7:1.00]
-; SKYLAKE-SSE-NEXT:    paddd %xmm1, %xmm0 # sched: [1:0.33]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_punpckldq:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] sched: [1:1.00]
-; SKYLAKE-NEXT:    vpunpckldq {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[1],mem[1] sched: [7:1.00]
-; SKYLAKE-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_punpckldq:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] sched: [1:1.00]
-; SKX-SSE-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[1],mem[1] sched: [7:1.00]
-; SKX-SSE-NEXT:    paddd %xmm1, %xmm0 # sched: [1:0.33]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_punpckldq:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] sched: [1:1.00]
-; SKX-NEXT:    vpunpckldq {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[1],mem[1] sched: [7:1.00]
-; SKX-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_punpckldq:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] sched: [2:0.50]
-; BDVER2-SSE-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[1],mem[1] sched: [7:0.50]
-; BDVER2-SSE-NEXT:    paddd %xmm1, %xmm0 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_punpckldq:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] sched: [2:0.50]
-; BDVER2-NEXT:    vpunpckldq {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[1],mem[1] sched: [7:0.50]
-; BDVER2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_punpckldq:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] sched: [1:0.50]
-; BTVER2-SSE-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[1],mem[1] sched: [6:1.00]
-; BTVER2-SSE-NEXT:    paddd %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_punpckldq:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] sched: [1:0.50]
-; BTVER2-NEXT:    vpunpckldq {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[1],mem[1] sched: [6:1.00]
-; BTVER2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_punpckldq:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[1],mem[1] sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    paddd %xmm1, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_punpckldq:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] sched: [1:0.25]
-; ZNVER1-NEXT:    vpunpckldq {{.*#+}} xmm1 = xmm1[0],mem[0],xmm1[1],mem[1] sched: [8:0.50]
-; ZNVER1-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = shufflevector <4 x i32> %a0, <4 x i32> %a1, <4 x i32> <i32 0, i32 4, i32 1, i32 5>
-  %2 = load <4 x i32>, <4 x i32> *%a2, align 16
-  %3 = shufflevector <4 x i32> %a1, <4 x i32> %2, <4 x i32> <i32 0, i32 4, i32 1, i32 5>
-  %4 = add <4 x i32> %1, %3
-  ret <4 x i32> %4
-}
-
-define <2 x i64> @test_punpcklqdq(<2 x i64> %a0, <2 x i64> %a1, <2 x i64> *%a2) {
-; GENERIC-LABEL: test_punpcklqdq:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0] sched: [1:0.50]
-; GENERIC-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],mem[0] sched: [7:0.50]
-; GENERIC-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_punpcklqdq:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0] sched: [1:1.00]
-; ATOM-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],mem[0] sched: [1:1.00]
-; ATOM-NEXT:    paddq %xmm1, %xmm0 # sched: [2:1.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_punpcklqdq:
-; SLM:       # %bb.0:
-; SLM-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0] sched: [1:1.00]
-; SLM-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],mem[0] sched: [4:1.00]
-; SLM-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_punpcklqdq:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0] sched: [1:0.50]
-; SANDY-SSE-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],mem[0] sched: [7:0.50]
-; SANDY-SSE-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.50]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_punpcklqdq:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0] sched: [1:0.50]
-; SANDY-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],mem[0] sched: [7:0.50]
-; SANDY-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_punpcklqdq:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0] sched: [1:1.00]
-; HASWELL-SSE-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],mem[0] sched: [7:1.00]
-; HASWELL-SSE-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.50]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_punpcklqdq:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0] sched: [1:1.00]
-; HASWELL-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],mem[0] sched: [7:1.00]
-; HASWELL-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_punpcklqdq:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0] sched: [1:1.00]
-; BROADWELL-SSE-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],mem[0] sched: [6:1.00]
-; BROADWELL-SSE-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.50]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_punpcklqdq:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0] sched: [1:1.00]
-; BROADWELL-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],mem[0] sched: [6:1.00]
-; BROADWELL-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_punpcklqdq:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0] sched: [1:1.00]
-; SKYLAKE-SSE-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],mem[0] sched: [7:1.00]
-; SKYLAKE-SSE-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.33]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_punpcklqdq:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0] sched: [1:1.00]
-; SKYLAKE-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],mem[0] sched: [7:1.00]
-; SKYLAKE-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_punpcklqdq:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0] sched: [1:1.00]
-; SKX-SSE-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],mem[0] sched: [7:1.00]
-; SKX-SSE-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.33]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_punpcklqdq:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0] sched: [1:1.00]
-; SKX-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],mem[0] sched: [7:1.00]
-; SKX-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_punpcklqdq:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0] sched: [2:0.50]
-; BDVER2-SSE-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],mem[0] sched: [7:0.50]
-; BDVER2-SSE-NEXT:    paddq %xmm1, %xmm0 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_punpcklqdq:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0] sched: [2:0.50]
-; BDVER2-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],mem[0] sched: [7:0.50]
-; BDVER2-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_punpcklqdq:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0] sched: [1:0.50]
-; BTVER2-SSE-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],mem[0] sched: [6:1.00]
-; BTVER2-SSE-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_punpcklqdq:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0] sched: [1:0.50]
-; BTVER2-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],mem[0] sched: [6:1.00]
-; BTVER2-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_punpcklqdq:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0] sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],mem[0] sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_punpcklqdq:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0] sched: [1:0.25]
-; ZNVER1-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],mem[0] sched: [8:0.50]
-; ZNVER1-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = shufflevector <2 x i64> %a0, <2 x i64> %a1, <2 x i32> <i32 0, i32 2>
-  %2 = load <2 x i64>, <2 x i64> *%a2, align 16
-  %3 = shufflevector <2 x i64> %a1, <2 x i64> %2, <2x i32> <i32 0, i32 2>
-  %4 = add <2 x i64> %1, %3
-  ret <2 x i64> %4
-}
-
-define <8 x i16> @test_punpcklwd(<8 x i16> %a0, <8 x i16> %a1, <8 x i16> *%a2) {
-; GENERIC-LABEL: test_punpcklwd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3] sched: [1:0.50]
-; GENERIC-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1],xmm0[2],mem[2],xmm0[3],mem[3] sched: [7:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_punpcklwd:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3] sched: [1:1.00]
-; ATOM-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1],xmm0[2],mem[2],xmm0[3],mem[3] sched: [1:1.00]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_punpcklwd:
-; SLM:       # %bb.0:
-; SLM-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3] sched: [1:1.00]
-; SLM-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1],xmm0[2],mem[2],xmm0[3],mem[3] sched: [4:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_punpcklwd:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3] sched: [1:0.50]
-; SANDY-SSE-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1],xmm0[2],mem[2],xmm0[3],mem[3] sched: [7:0.50]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_punpcklwd:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3] sched: [1:0.50]
-; SANDY-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1],xmm0[2],mem[2],xmm0[3],mem[3] sched: [7:0.50]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_punpcklwd:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3] sched: [1:1.00]
-; HASWELL-SSE-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1],xmm0[2],mem[2],xmm0[3],mem[3] sched: [7:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_punpcklwd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3] sched: [1:1.00]
-; HASWELL-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1],xmm0[2],mem[2],xmm0[3],mem[3] sched: [7:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_punpcklwd:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3] sched: [1:1.00]
-; BROADWELL-SSE-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1],xmm0[2],mem[2],xmm0[3],mem[3] sched: [6:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_punpcklwd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3] sched: [1:1.00]
-; BROADWELL-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1],xmm0[2],mem[2],xmm0[3],mem[3] sched: [6:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_punpcklwd:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3] sched: [1:1.00]
-; SKYLAKE-SSE-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1],xmm0[2],mem[2],xmm0[3],mem[3] sched: [7:1.00]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_punpcklwd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3] sched: [1:1.00]
-; SKYLAKE-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1],xmm0[2],mem[2],xmm0[3],mem[3] sched: [7:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_punpcklwd:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3] sched: [1:1.00]
-; SKX-SSE-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1],xmm0[2],mem[2],xmm0[3],mem[3] sched: [7:1.00]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_punpcklwd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3] sched: [1:1.00]
-; SKX-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1],xmm0[2],mem[2],xmm0[3],mem[3] sched: [7:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_punpcklwd:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3] sched: [2:0.50]
-; BDVER2-SSE-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1],xmm0[2],mem[2],xmm0[3],mem[3] sched: [7:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_punpcklwd:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3] sched: [2:0.50]
-; BDVER2-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1],xmm0[2],mem[2],xmm0[3],mem[3] sched: [7:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_punpcklwd:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3] sched: [1:0.50]
-; BTVER2-SSE-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1],xmm0[2],mem[2],xmm0[3],mem[3] sched: [6:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_punpcklwd:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3] sched: [1:0.50]
-; BTVER2-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1],xmm0[2],mem[2],xmm0[3],mem[3] sched: [6:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_punpcklwd:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3] sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1],xmm0[2],mem[2],xmm0[3],mem[3] sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_punpcklwd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3] sched: [1:0.25]
-; ZNVER1-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1],xmm0[2],mem[2],xmm0[3],mem[3] sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = shufflevector <8 x i16> %a0, <8 x i16> %a1, <8 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11>
-  %2 = load <8 x i16>, <8 x i16> *%a2, align 16
-  %3 = shufflevector <8 x i16> %1, <8 x i16> %2, <8 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11>
-  ret <8 x i16> %3
-}
-
-define <2 x i64> @test_pxor(<2 x i64> %a0, <2 x i64> %a1, <2 x i64> *%a2) {
-; GENERIC-LABEL: test_pxor:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    pxor %xmm1, %xmm0 # sched: [1:0.33]
-; GENERIC-NEXT:    pxor (%rdi), %xmm0 # sched: [7:0.50]
-; GENERIC-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_pxor:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    pxor %xmm1, %xmm0 # sched: [1:0.50]
-; ATOM-NEXT:    pxor (%rdi), %xmm0 # sched: [1:1.00]
-; ATOM-NEXT:    paddq %xmm1, %xmm0 # sched: [2:1.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_pxor:
-; SLM:       # %bb.0:
-; SLM-NEXT:    pxor %xmm1, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    pxor (%rdi), %xmm0 # sched: [4:1.00]
-; SLM-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_pxor:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    pxor %xmm1, %xmm0 # sched: [1:0.33]
-; SANDY-SSE-NEXT:    pxor (%rdi), %xmm0 # sched: [7:0.50]
-; SANDY-SSE-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.50]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_pxor:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vpxor %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SANDY-NEXT:    vpxor (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SANDY-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_pxor:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    pxor %xmm1, %xmm0 # sched: [1:0.33]
-; HASWELL-SSE-NEXT:    pxor (%rdi), %xmm0 # sched: [7:0.50]
-; HASWELL-SSE-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.50]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_pxor:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpxor %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; HASWELL-NEXT:    vpxor (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; HASWELL-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_pxor:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    pxor %xmm1, %xmm0 # sched: [1:0.33]
-; BROADWELL-SSE-NEXT:    pxor (%rdi), %xmm0 # sched: [6:0.50]
-; BROADWELL-SSE-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.50]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pxor:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpxor %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; BROADWELL-NEXT:    vpxor (%rdi), %xmm0, %xmm0 # sched: [6:0.50]
-; BROADWELL-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_pxor:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    pxor %xmm1, %xmm0 # sched: [1:0.33]
-; SKYLAKE-SSE-NEXT:    pxor (%rdi), %xmm0 # sched: [7:0.50]
-; SKYLAKE-SSE-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.33]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pxor:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpxor %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    vpxor (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKYLAKE-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_pxor:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    pxor %xmm1, %xmm0 # sched: [1:0.33]
-; SKX-SSE-NEXT:    pxor (%rdi), %xmm0 # sched: [7:0.50]
-; SKX-SSE-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.33]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pxor:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpxor %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    vpxor (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKX-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_pxor:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    pxor %xmm1, %xmm0 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    pxor (%rdi), %xmm0 # sched: [7:0.50]
-; BDVER2-SSE-NEXT:    paddq %xmm1, %xmm0 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_pxor:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vpxor %xmm1, %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    vpxor (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; BDVER2-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_pxor:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    pxor %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    pxor (%rdi), %xmm0 # sched: [6:1.00]
-; BTVER2-SSE-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_pxor:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vpxor %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    vpxor (%rdi), %xmm0, %xmm0 # sched: [6:1.00]
-; BTVER2-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_pxor:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    pxor %xmm1, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    pxor (%rdi), %xmm0 # sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_pxor:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpxor %xmm1, %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vpxor (%rdi), %xmm0, %xmm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = xor <2 x i64> %a0, %a1
-  %2 = load <2 x i64>, <2 x i64> *%a2, align 16
-  %3 = xor <2 x i64> %1, %2
-  %4 = add <2 x i64> %3, %a1
-  ret <2 x i64> %4
-}
-
-define <2 x double> @test_shufpd(<2 x double> %a0, <2 x double> %a1, <2 x double> *%a2) {
-; GENERIC-LABEL: test_shufpd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    shufpd {{.*#+}} xmm0 = xmm0[1],xmm1[0] sched: [1:1.00]
-; GENERIC-NEXT:    shufpd {{.*#+}} xmm1 = xmm1[1],mem[0] sched: [7:1.00]
-; GENERIC-NEXT:    addpd %xmm1, %xmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_shufpd:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    shufpd {{.*#+}} xmm0 = xmm0[1],xmm1[0] sched: [1:1.00]
-; ATOM-NEXT:    shufpd {{.*#+}} xmm1 = xmm1[1],mem[0] sched: [1:1.00]
-; ATOM-NEXT:    addpd %xmm1, %xmm0 # sched: [6:3.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_shufpd:
-; SLM:       # %bb.0:
-; SLM-NEXT:    shufpd {{.*#+}} xmm0 = xmm0[1],xmm1[0] sched: [1:1.00]
-; SLM-NEXT:    shufpd {{.*#+}} xmm1 = xmm1[1],mem[0] sched: [4:1.00]
-; SLM-NEXT:    addpd %xmm1, %xmm0 # sched: [3:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_shufpd:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    shufpd {{.*#+}} xmm0 = xmm0[1],xmm1[0] sched: [1:1.00]
-; SANDY-SSE-NEXT:    shufpd {{.*#+}} xmm1 = xmm1[1],mem[0] sched: [7:1.00]
-; SANDY-SSE-NEXT:    addpd %xmm1, %xmm0 # sched: [3:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_shufpd:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vshufpd {{.*#+}} xmm0 = xmm0[1],xmm1[0] sched: [1:1.00]
-; SANDY-NEXT:    vshufpd {{.*#+}} xmm1 = xmm1[1],mem[0] sched: [7:1.00]
-; SANDY-NEXT:    vaddpd %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_shufpd:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    shufpd {{.*#+}} xmm0 = xmm0[1],xmm1[0] sched: [1:1.00]
-; HASWELL-SSE-NEXT:    shufpd {{.*#+}} xmm1 = xmm1[1],mem[0] sched: [7:1.00]
-; HASWELL-SSE-NEXT:    addpd %xmm1, %xmm0 # sched: [3:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_shufpd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vshufpd {{.*#+}} xmm0 = xmm0[1],xmm1[0] sched: [1:1.00]
-; HASWELL-NEXT:    vshufpd {{.*#+}} xmm1 = xmm1[1],mem[0] sched: [7:1.00]
-; HASWELL-NEXT:    vaddpd %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_shufpd:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    shufpd {{.*#+}} xmm0 = xmm0[1],xmm1[0] sched: [1:1.00]
-; BROADWELL-SSE-NEXT:    shufpd {{.*#+}} xmm1 = xmm1[1],mem[0] sched: [6:1.00]
-; BROADWELL-SSE-NEXT:    addpd %xmm1, %xmm0 # sched: [3:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_shufpd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vshufpd {{.*#+}} xmm0 = xmm0[1],xmm1[0] sched: [1:1.00]
-; BROADWELL-NEXT:    vshufpd {{.*#+}} xmm1 = xmm1[1],mem[0] sched: [6:1.00]
-; BROADWELL-NEXT:    vaddpd %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_shufpd:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    shufpd {{.*#+}} xmm0 = xmm0[1],xmm1[0] sched: [1:1.00]
-; SKYLAKE-SSE-NEXT:    shufpd {{.*#+}} xmm1 = xmm1[1],mem[0] sched: [7:1.00]
-; SKYLAKE-SSE-NEXT:    addpd %xmm1, %xmm0 # sched: [4:0.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_shufpd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vshufpd {{.*#+}} xmm0 = xmm0[1],xmm1[0] sched: [1:1.00]
-; SKYLAKE-NEXT:    vshufpd {{.*#+}} xmm1 = xmm1[1],mem[0] sched: [7:1.00]
-; SKYLAKE-NEXT:    vaddpd %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_shufpd:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    shufpd {{.*#+}} xmm0 = xmm0[1],xmm1[0] sched: [1:1.00]
-; SKX-SSE-NEXT:    shufpd {{.*#+}} xmm1 = xmm1[1],mem[0] sched: [7:1.00]
-; SKX-SSE-NEXT:    addpd %xmm1, %xmm0 # sched: [4:0.50]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_shufpd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vshufpd {{.*#+}} xmm0 = xmm0[1],xmm1[0] sched: [1:1.00]
-; SKX-NEXT:    vshufpd {{.*#+}} xmm1 = xmm1[1],mem[0] sched: [7:1.00]
-; SKX-NEXT:    vaddpd %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_shufpd:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    shufpd {{.*#+}} xmm0 = xmm0[1],xmm1[0] sched: [2:0.50]
-; BDVER2-SSE-NEXT:    shufpd {{.*#+}} xmm1 = xmm1[1],mem[0] sched: [7:0.50]
-; BDVER2-SSE-NEXT:    addpd %xmm1, %xmm0 # sched: [5:1.00]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_shufpd:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vshufpd {{.*#+}} xmm0 = xmm0[1],xmm1[0] sched: [2:0.50]
-; BDVER2-NEXT:    vshufpd {{.*#+}} xmm1 = xmm1[1],mem[0] sched: [7:0.50]
-; BDVER2-NEXT:    vaddpd %xmm1, %xmm0, %xmm0 # sched: [5:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_shufpd:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    shufpd {{.*#+}} xmm0 = xmm0[1],xmm1[0] sched: [1:0.50]
-; BTVER2-SSE-NEXT:    shufpd {{.*#+}} xmm1 = xmm1[1],mem[0] sched: [6:1.00]
-; BTVER2-SSE-NEXT:    addpd %xmm1, %xmm0 # sched: [3:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_shufpd:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vshufpd {{.*#+}} xmm0 = xmm0[1],xmm1[0] sched: [1:0.50]
-; BTVER2-NEXT:    vshufpd {{.*#+}} xmm1 = xmm1[1],mem[0] sched: [6:1.00]
-; BTVER2-NEXT:    vaddpd %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_shufpd:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    shufpd {{.*#+}} xmm0 = xmm0[1],xmm1[0] sched: [1:0.50]
-; ZNVER1-SSE-NEXT:    shufpd {{.*#+}} xmm1 = xmm1[1],mem[0] sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    addpd %xmm1, %xmm0 # sched: [3:1.00]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_shufpd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vshufpd {{.*#+}} xmm0 = xmm0[1],xmm1[0] sched: [1:0.50]
-; ZNVER1-NEXT:    vshufpd {{.*#+}} xmm1 = xmm1[1],mem[0] sched: [8:0.50]
-; ZNVER1-NEXT:    vaddpd %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = shufflevector <2 x double> %a0, <2 x double> %a1, <2 x i32> <i32 1, i32 2>
-  %2 = load <2 x double>, <2 x double> *%a2, align 16
-  %3 = shufflevector <2 x double> %a1, <2 x double> %2, <2 x i32> <i32 1, i32 2>
-  %4 = fadd <2 x double> %1, %3
-  ret <2 x double> %4
-}
-
-define <2 x double> @test_sqrtpd(<2 x double> %a0, <2 x double> *%a1) {
-; GENERIC-LABEL: test_sqrtpd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    sqrtpd %xmm0, %xmm1 # sched: [21:21.00]
-; GENERIC-NEXT:    sqrtpd (%rdi), %xmm0 # sched: [27:21.00]
-; GENERIC-NEXT:    addpd %xmm1, %xmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_sqrtpd:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    sqrtpd %xmm0, %xmm1 # sched: [125:62.50]
-; ATOM-NEXT:    sqrtpd (%rdi), %xmm0 # sched: [125:62.50]
-; ATOM-NEXT:    addpd %xmm1, %xmm0 # sched: [6:3.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_sqrtpd:
-; SLM:       # %bb.0:
-; SLM-NEXT:    sqrtpd (%rdi), %xmm1 # sched: [74:70.00]
-; SLM-NEXT:    sqrtpd %xmm0, %xmm0 # sched: [71:70.00]
-; SLM-NEXT:    addpd %xmm0, %xmm1 # sched: [3:1.00]
-; SLM-NEXT:    movapd %xmm1, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_sqrtpd:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    sqrtpd %xmm0, %xmm1 # sched: [21:21.00]
-; SANDY-SSE-NEXT:    sqrtpd (%rdi), %xmm0 # sched: [27:21.00]
-; SANDY-SSE-NEXT:    addpd %xmm1, %xmm0 # sched: [3:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_sqrtpd:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vsqrtpd %xmm0, %xmm0 # sched: [21:21.00]
-; SANDY-NEXT:    vsqrtpd (%rdi), %xmm1 # sched: [27:21.00]
-; SANDY-NEXT:    vaddpd %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_sqrtpd:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    sqrtpd %xmm0, %xmm1 # sched: [16:14.00]
-; HASWELL-SSE-NEXT:    sqrtpd (%rdi), %xmm0 # sched: [22:14.00]
-; HASWELL-SSE-NEXT:    addpd %xmm1, %xmm0 # sched: [3:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_sqrtpd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vsqrtpd %xmm0, %xmm0 # sched: [16:14.00]
-; HASWELL-NEXT:    vsqrtpd (%rdi), %xmm1 # sched: [22:14.00]
-; HASWELL-NEXT:    vaddpd %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_sqrtpd:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    sqrtpd %xmm0, %xmm1 # sched: [16:14.00]
-; BROADWELL-SSE-NEXT:    sqrtpd (%rdi), %xmm0 # sched: [21:14.00]
-; BROADWELL-SSE-NEXT:    addpd %xmm1, %xmm0 # sched: [3:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_sqrtpd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vsqrtpd %xmm0, %xmm0 # sched: [16:14.00]
-; BROADWELL-NEXT:    vsqrtpd (%rdi), %xmm1 # sched: [21:14.00]
-; BROADWELL-NEXT:    vaddpd %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_sqrtpd:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    sqrtpd %xmm0, %xmm1 # sched: [18:6.00]
-; SKYLAKE-SSE-NEXT:    sqrtpd (%rdi), %xmm0 # sched: [24:6.00]
-; SKYLAKE-SSE-NEXT:    addpd %xmm1, %xmm0 # sched: [4:0.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_sqrtpd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vsqrtpd %xmm0, %xmm0 # sched: [18:6.00]
-; SKYLAKE-NEXT:    vsqrtpd (%rdi), %xmm1 # sched: [24:6.00]
-; SKYLAKE-NEXT:    vaddpd %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_sqrtpd:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    sqrtpd %xmm0, %xmm1 # sched: [18:6.00]
-; SKX-SSE-NEXT:    sqrtpd (%rdi), %xmm0 # sched: [24:6.00]
-; SKX-SSE-NEXT:    addpd %xmm1, %xmm0 # sched: [4:0.50]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_sqrtpd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vsqrtpd %xmm0, %xmm0 # sched: [18:6.00]
-; SKX-NEXT:    vsqrtpd (%rdi), %xmm1 # sched: [24:6.00]
-; SKX-NEXT:    vaddpd %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_sqrtpd:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    sqrtpd %xmm0, %xmm1 # sched: [9:13.50]
-; BDVER2-SSE-NEXT:    sqrtpd (%rdi), %xmm0 # sched: [14:13.50]
-; BDVER2-SSE-NEXT:    addpd %xmm1, %xmm0 # sched: [5:1.00]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_sqrtpd:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vsqrtpd (%rdi), %xmm1 # sched: [14:13.50]
-; BDVER2-NEXT:    vsqrtpd %xmm0, %xmm0 # sched: [9:13.50]
-; BDVER2-NEXT:    vaddpd %xmm1, %xmm0, %xmm0 # sched: [5:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_sqrtpd:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    sqrtpd %xmm0, %xmm1 # sched: [27:27.00]
-; BTVER2-SSE-NEXT:    sqrtpd (%rdi), %xmm0 # sched: [32:27.00]
-; BTVER2-SSE-NEXT:    addpd %xmm1, %xmm0 # sched: [3:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_sqrtpd:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vsqrtpd (%rdi), %xmm1 # sched: [32:27.00]
-; BTVER2-NEXT:    vsqrtpd %xmm0, %xmm0 # sched: [27:27.00]
-; BTVER2-NEXT:    vaddpd %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_sqrtpd:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    sqrtpd %xmm0, %xmm1 # sched: [20:20.00]
-; ZNVER1-SSE-NEXT:    sqrtpd (%rdi), %xmm0 # sched: [27:20.00]
-; ZNVER1-SSE-NEXT:    addpd %xmm1, %xmm0 # sched: [3:1.00]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_sqrtpd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vsqrtpd (%rdi), %xmm1 # sched: [27:20.00]
-; ZNVER1-NEXT:    vsqrtpd %xmm0, %xmm0 # sched: [20:20.00]
-; ZNVER1-NEXT:    vaddpd %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <2 x double> @llvm.x86.sse2.sqrt.pd(<2 x double> %a0)
-  %2 = load <2 x double>, <2 x double> *%a1, align 16
-  %3 = call <2 x double> @llvm.x86.sse2.sqrt.pd(<2 x double> %2)
-  %4 = fadd <2 x double> %1, %3
-  ret <2 x double> %4
-}
-declare <2 x double> @llvm.x86.sse2.sqrt.pd(<2 x double>) nounwind readnone
-
-; TODO - sqrtsd_m
-
-define <2 x double> @test_sqrtsd(<2 x double> %a0, <2 x double> *%a1) {
-; GENERIC-LABEL: test_sqrtsd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    sqrtsd %xmm0, %xmm0 # sched: [21:21.00]
-; GENERIC-NEXT:    movapd (%rdi), %xmm1 # sched: [6:0.50]
-; GENERIC-NEXT:    sqrtsd %xmm1, %xmm1 # sched: [21:21.00]
-; GENERIC-NEXT:    addpd %xmm1, %xmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_sqrtsd:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    movapd (%rdi), %xmm1 # sched: [1:1.00]
-; ATOM-NEXT:    sqrtsd %xmm0, %xmm0 # sched: [62:31.00]
-; ATOM-NEXT:    sqrtsd %xmm1, %xmm1 # sched: [62:31.00]
-; ATOM-NEXT:    addpd %xmm1, %xmm0 # sched: [6:3.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_sqrtsd:
-; SLM:       # %bb.0:
-; SLM-NEXT:    movapd (%rdi), %xmm1 # sched: [3:1.00]
-; SLM-NEXT:    sqrtsd %xmm0, %xmm0 # sched: [35:35.00]
-; SLM-NEXT:    sqrtsd %xmm1, %xmm1 # sched: [35:35.00]
-; SLM-NEXT:    addpd %xmm1, %xmm0 # sched: [3:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_sqrtsd:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    sqrtsd %xmm0, %xmm0 # sched: [21:21.00]
-; SANDY-SSE-NEXT:    movapd (%rdi), %xmm1 # sched: [6:0.50]
-; SANDY-SSE-NEXT:    sqrtsd %xmm1, %xmm1 # sched: [21:21.00]
-; SANDY-SSE-NEXT:    addpd %xmm1, %xmm0 # sched: [3:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_sqrtsd:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vsqrtsd %xmm0, %xmm0, %xmm0 # sched: [21:21.00]
-; SANDY-NEXT:    vmovapd (%rdi), %xmm1 # sched: [6:0.50]
-; SANDY-NEXT:    vsqrtsd %xmm1, %xmm1, %xmm1 # sched: [21:21.00]
-; SANDY-NEXT:    vaddpd %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_sqrtsd:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    sqrtsd %xmm0, %xmm0 # sched: [16:14.00]
-; HASWELL-SSE-NEXT:    movapd (%rdi), %xmm1 # sched: [6:0.50]
-; HASWELL-SSE-NEXT:    sqrtsd %xmm1, %xmm1 # sched: [16:14.00]
-; HASWELL-SSE-NEXT:    addpd %xmm1, %xmm0 # sched: [3:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_sqrtsd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vsqrtsd %xmm0, %xmm0, %xmm0 # sched: [16:14.00]
-; HASWELL-NEXT:    vmovapd (%rdi), %xmm1 # sched: [6:0.50]
-; HASWELL-NEXT:    vsqrtsd %xmm1, %xmm1, %xmm1 # sched: [16:14.00]
-; HASWELL-NEXT:    vaddpd %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_sqrtsd:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    sqrtsd %xmm0, %xmm0 # sched: [16:8.00]
-; BROADWELL-SSE-NEXT:    movapd (%rdi), %xmm1 # sched: [5:0.50]
-; BROADWELL-SSE-NEXT:    sqrtsd %xmm1, %xmm1 # sched: [16:8.00]
-; BROADWELL-SSE-NEXT:    addpd %xmm1, %xmm0 # sched: [3:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_sqrtsd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vsqrtsd %xmm0, %xmm0, %xmm0 # sched: [16:8.00]
-; BROADWELL-NEXT:    vmovapd (%rdi), %xmm1 # sched: [5:0.50]
-; BROADWELL-NEXT:    vsqrtsd %xmm1, %xmm1, %xmm1 # sched: [16:8.00]
-; BROADWELL-NEXT:    vaddpd %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_sqrtsd:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    sqrtsd %xmm0, %xmm0 # sched: [18:6.00]
-; SKYLAKE-SSE-NEXT:    movapd (%rdi), %xmm1 # sched: [6:0.50]
-; SKYLAKE-SSE-NEXT:    sqrtsd %xmm1, %xmm1 # sched: [18:6.00]
-; SKYLAKE-SSE-NEXT:    addpd %xmm1, %xmm0 # sched: [4:0.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_sqrtsd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vsqrtsd %xmm0, %xmm0, %xmm0 # sched: [18:6.00]
-; SKYLAKE-NEXT:    vmovapd (%rdi), %xmm1 # sched: [6:0.50]
-; SKYLAKE-NEXT:    vsqrtsd %xmm1, %xmm1, %xmm1 # sched: [18:6.00]
-; SKYLAKE-NEXT:    vaddpd %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_sqrtsd:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    sqrtsd %xmm0, %xmm0 # sched: [18:6.00]
-; SKX-SSE-NEXT:    movapd (%rdi), %xmm1 # sched: [6:0.50]
-; SKX-SSE-NEXT:    sqrtsd %xmm1, %xmm1 # sched: [18:6.00]
-; SKX-SSE-NEXT:    addpd %xmm1, %xmm0 # sched: [4:0.50]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_sqrtsd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vsqrtsd %xmm0, %xmm0, %xmm0 # sched: [18:6.00]
-; SKX-NEXT:    vmovapd (%rdi), %xmm1 # sched: [6:0.50]
-; SKX-NEXT:    vsqrtsd %xmm1, %xmm1, %xmm1 # sched: [18:6.00]
-; SKX-NEXT:    vaddpd %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_sqrtsd:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    movapd (%rdi), %xmm1 # sched: [5:0.50]
-; BDVER2-SSE-NEXT:    sqrtsd %xmm0, %xmm0 # sched: [9:13.50]
-; BDVER2-SSE-NEXT:    sqrtsd %xmm1, %xmm1 # sched: [9:13.50]
-; BDVER2-SSE-NEXT:    addpd %xmm1, %xmm0 # sched: [5:1.00]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_sqrtsd:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vmovapd (%rdi), %xmm1 # sched: [5:0.50]
-; BDVER2-NEXT:    vsqrtsd %xmm0, %xmm0, %xmm0 # sched: [9:13.50]
-; BDVER2-NEXT:    vsqrtsd %xmm1, %xmm1, %xmm1 # sched: [9:13.50]
-; BDVER2-NEXT:    vaddpd %xmm1, %xmm0, %xmm0 # sched: [5:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_sqrtsd:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    movapd (%rdi), %xmm1 # sched: [5:1.00]
-; BTVER2-SSE-NEXT:    sqrtsd %xmm0, %xmm0 # sched: [27:27.00]
-; BTVER2-SSE-NEXT:    sqrtsd %xmm1, %xmm1 # sched: [27:27.00]
-; BTVER2-SSE-NEXT:    addpd %xmm1, %xmm0 # sched: [3:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_sqrtsd:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vmovapd (%rdi), %xmm1 # sched: [5:1.00]
-; BTVER2-NEXT:    vsqrtsd %xmm0, %xmm0, %xmm0 # sched: [27:27.00]
-; BTVER2-NEXT:    vsqrtsd %xmm1, %xmm1, %xmm1 # sched: [27:27.00]
-; BTVER2-NEXT:    vaddpd %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_sqrtsd:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    movapd (%rdi), %xmm1 # sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    sqrtsd %xmm0, %xmm0 # sched: [20:20.00]
-; ZNVER1-SSE-NEXT:    sqrtsd %xmm1, %xmm1 # sched: [20:20.00]
-; ZNVER1-SSE-NEXT:    addpd %xmm1, %xmm0 # sched: [3:1.00]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_sqrtsd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vmovapd (%rdi), %xmm1 # sched: [8:0.50]
-; ZNVER1-NEXT:    vsqrtsd %xmm0, %xmm0, %xmm0 # sched: [20:20.00]
-; ZNVER1-NEXT:    vsqrtsd %xmm1, %xmm1, %xmm1 # sched: [20:20.00]
-; ZNVER1-NEXT:    vaddpd %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <2 x double> @llvm.x86.sse2.sqrt.sd(<2 x double> %a0)
-  %2 = load <2 x double>, <2 x double> *%a1, align 16
-  %3 = call <2 x double> @llvm.x86.sse2.sqrt.sd(<2 x double> %2)
-  %4 = fadd <2 x double> %1, %3
-  ret <2 x double> %4
-}
-declare <2 x double> @llvm.x86.sse2.sqrt.sd(<2 x double>) nounwind readnone
-
-define <2 x double> @test_subpd(<2 x double> %a0, <2 x double> %a1, <2 x double> *%a2) {
-; GENERIC-LABEL: test_subpd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    subpd %xmm1, %xmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    subpd (%rdi), %xmm0 # sched: [9:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_subpd:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    subpd %xmm1, %xmm0 # sched: [6:3.00]
-; ATOM-NEXT:    subpd (%rdi), %xmm0 # sched: [7:3.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_subpd:
-; SLM:       # %bb.0:
-; SLM-NEXT:    subpd %xmm1, %xmm0 # sched: [3:1.00]
-; SLM-NEXT:    subpd (%rdi), %xmm0 # sched: [6:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_subpd:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    subpd %xmm1, %xmm0 # sched: [3:1.00]
-; SANDY-SSE-NEXT:    subpd (%rdi), %xmm0 # sched: [9:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_subpd:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vsubpd %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; SANDY-NEXT:    vsubpd (%rdi), %xmm0, %xmm0 # sched: [9:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_subpd:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    subpd %xmm1, %xmm0 # sched: [3:1.00]
-; HASWELL-SSE-NEXT:    subpd (%rdi), %xmm0 # sched: [9:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_subpd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vsubpd %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; HASWELL-NEXT:    vsubpd (%rdi), %xmm0, %xmm0 # sched: [9:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_subpd:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    subpd %xmm1, %xmm0 # sched: [3:1.00]
-; BROADWELL-SSE-NEXT:    subpd (%rdi), %xmm0 # sched: [8:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_subpd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vsubpd %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    vsubpd (%rdi), %xmm0, %xmm0 # sched: [8:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_subpd:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    subpd %xmm1, %xmm0 # sched: [4:0.50]
-; SKYLAKE-SSE-NEXT:    subpd (%rdi), %xmm0 # sched: [10:0.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_subpd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vsubpd %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    vsubpd (%rdi), %xmm0, %xmm0 # sched: [10:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_subpd:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    subpd %xmm1, %xmm0 # sched: [4:0.50]
-; SKX-SSE-NEXT:    subpd (%rdi), %xmm0 # sched: [10:0.50]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_subpd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vsubpd %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKX-NEXT:    vsubpd (%rdi), %xmm0, %xmm0 # sched: [10:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_subpd:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    subpd %xmm1, %xmm0 # sched: [5:1.00]
-; BDVER2-SSE-NEXT:    subpd (%rdi), %xmm0 # sched: [10:1.00]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_subpd:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vsubpd %xmm1, %xmm0, %xmm0 # sched: [5:1.00]
-; BDVER2-NEXT:    vsubpd (%rdi), %xmm0, %xmm0 # sched: [10:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_subpd:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    subpd %xmm1, %xmm0 # sched: [3:1.00]
-; BTVER2-SSE-NEXT:    subpd (%rdi), %xmm0 # sched: [8:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_subpd:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vsubpd %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; BTVER2-NEXT:    vsubpd (%rdi), %xmm0, %xmm0 # sched: [8:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_subpd:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    subpd %xmm1, %xmm0 # sched: [3:1.00]
-; ZNVER1-SSE-NEXT:    subpd (%rdi), %xmm0 # sched: [10:1.00]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_subpd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vsubpd %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    vsubpd (%rdi), %xmm0, %xmm0 # sched: [10:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = fsub <2 x double> %a0, %a1
-  %2 = load <2 x double>, <2 x double> *%a2, align 16
-  %3 = fsub <2 x double> %1, %2
-  ret <2 x double> %3
-}
-
-define double @test_subsd(double %a0, double %a1, double *%a2) {
-; GENERIC-LABEL: test_subsd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    subsd %xmm1, %xmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    subsd (%rdi), %xmm0 # sched: [9:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_subsd:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    subsd %xmm1, %xmm0 # sched: [5:5.00]
-; ATOM-NEXT:    subsd (%rdi), %xmm0 # sched: [5:5.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_subsd:
-; SLM:       # %bb.0:
-; SLM-NEXT:    subsd %xmm1, %xmm0 # sched: [3:1.00]
-; SLM-NEXT:    subsd (%rdi), %xmm0 # sched: [6:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_subsd:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    subsd %xmm1, %xmm0 # sched: [3:1.00]
-; SANDY-SSE-NEXT:    subsd (%rdi), %xmm0 # sched: [9:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_subsd:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vsubsd %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; SANDY-NEXT:    vsubsd (%rdi), %xmm0, %xmm0 # sched: [9:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_subsd:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    subsd %xmm1, %xmm0 # sched: [3:1.00]
-; HASWELL-SSE-NEXT:    subsd (%rdi), %xmm0 # sched: [8:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_subsd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vsubsd %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; HASWELL-NEXT:    vsubsd (%rdi), %xmm0, %xmm0 # sched: [8:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_subsd:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    subsd %xmm1, %xmm0 # sched: [3:1.00]
-; BROADWELL-SSE-NEXT:    subsd (%rdi), %xmm0 # sched: [8:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_subsd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vsubsd %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    vsubsd (%rdi), %xmm0, %xmm0 # sched: [8:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_subsd:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    subsd %xmm1, %xmm0 # sched: [4:0.50]
-; SKYLAKE-SSE-NEXT:    subsd (%rdi), %xmm0 # sched: [9:0.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_subsd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vsubsd %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    vsubsd (%rdi), %xmm0, %xmm0 # sched: [9:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_subsd:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    subsd %xmm1, %xmm0 # sched: [4:0.50]
-; SKX-SSE-NEXT:    subsd (%rdi), %xmm0 # sched: [9:0.50]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_subsd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vsubsd %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKX-NEXT:    vsubsd (%rdi), %xmm0, %xmm0 # sched: [9:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_subsd:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    subsd %xmm1, %xmm0 # sched: [5:1.00]
-; BDVER2-SSE-NEXT:    subsd (%rdi), %xmm0 # sched: [10:1.00]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_subsd:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vsubsd %xmm1, %xmm0, %xmm0 # sched: [5:1.00]
-; BDVER2-NEXT:    vsubsd (%rdi), %xmm0, %xmm0 # sched: [10:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_subsd:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    subsd %xmm1, %xmm0 # sched: [3:1.00]
-; BTVER2-SSE-NEXT:    subsd (%rdi), %xmm0 # sched: [8:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_subsd:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vsubsd %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; BTVER2-NEXT:    vsubsd (%rdi), %xmm0, %xmm0 # sched: [8:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_subsd:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    subsd %xmm1, %xmm0 # sched: [3:1.00]
-; ZNVER1-SSE-NEXT:    subsd (%rdi), %xmm0 # sched: [10:1.00]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_subsd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vsubsd %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    vsubsd (%rdi), %xmm0, %xmm0 # sched: [10:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = fsub double %a0, %a1
-  %2 = load double, double *%a2, align 8
-  %3 = fsub double %1, %2
-  ret double %3
-}
-
-define i32 @test_ucomisd(<2 x double> %a0, <2 x double> %a1, <2 x double> *%a2) {
-; GENERIC-LABEL: test_ucomisd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    ucomisd %xmm1, %xmm0 # sched: [2:1.00]
-; GENERIC-NEXT:    setnp %al # sched: [1:0.50]
-; GENERIC-NEXT:    sete %cl # sched: [1:0.50]
-; GENERIC-NEXT:    andb %al, %cl # sched: [1:0.33]
-; GENERIC-NEXT:    ucomisd (%rdi), %xmm0 # sched: [8:1.00]
-; GENERIC-NEXT:    setnp %al # sched: [1:0.50]
-; GENERIC-NEXT:    sete %dl # sched: [1:0.50]
-; GENERIC-NEXT:    andb %al, %dl # sched: [1:0.33]
-; GENERIC-NEXT:    orb %cl, %dl # sched: [1:0.33]
-; GENERIC-NEXT:    movzbl %dl, %eax # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_ucomisd:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    ucomisd %xmm1, %xmm0 # sched: [9:4.50]
-; ATOM-NEXT:    setnp %al # sched: [1:0.50]
-; ATOM-NEXT:    sete %cl # sched: [1:0.50]
-; ATOM-NEXT:    andb %al, %cl # sched: [1:0.50]
-; ATOM-NEXT:    ucomisd (%rdi), %xmm0 # sched: [10:5.00]
-; ATOM-NEXT:    setnp %al # sched: [1:0.50]
-; ATOM-NEXT:    sete %dl # sched: [1:0.50]
-; ATOM-NEXT:    andb %al, %dl # sched: [1:0.50]
-; ATOM-NEXT:    orb %cl, %dl # sched: [1:0.50]
-; ATOM-NEXT:    movzbl %dl, %eax # sched: [1:1.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_ucomisd:
-; SLM:       # %bb.0:
-; SLM-NEXT:    ucomisd %xmm1, %xmm0 # sched: [3:1.00]
-; SLM-NEXT:    setnp %al # sched: [1:0.50]
-; SLM-NEXT:    sete %cl # sched: [1:0.50]
-; SLM-NEXT:    andb %al, %cl # sched: [1:0.50]
-; SLM-NEXT:    ucomisd (%rdi), %xmm0 # sched: [6:1.00]
-; SLM-NEXT:    setnp %al # sched: [1:0.50]
-; SLM-NEXT:    sete %dl # sched: [1:0.50]
-; SLM-NEXT:    andb %al, %dl # sched: [1:0.50]
-; SLM-NEXT:    orb %cl, %dl # sched: [1:0.50]
-; SLM-NEXT:    movzbl %dl, %eax # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_ucomisd:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    ucomisd %xmm1, %xmm0 # sched: [2:1.00]
-; SANDY-SSE-NEXT:    setnp %al # sched: [1:0.50]
-; SANDY-SSE-NEXT:    sete %cl # sched: [1:0.50]
-; SANDY-SSE-NEXT:    andb %al, %cl # sched: [1:0.33]
-; SANDY-SSE-NEXT:    ucomisd (%rdi), %xmm0 # sched: [8:1.00]
-; SANDY-SSE-NEXT:    setnp %al # sched: [1:0.50]
-; SANDY-SSE-NEXT:    sete %dl # sched: [1:0.50]
-; SANDY-SSE-NEXT:    andb %al, %dl # sched: [1:0.33]
-; SANDY-SSE-NEXT:    orb %cl, %dl # sched: [1:0.33]
-; SANDY-SSE-NEXT:    movzbl %dl, %eax # sched: [1:0.33]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_ucomisd:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vucomisd %xmm1, %xmm0 # sched: [2:1.00]
-; SANDY-NEXT:    setnp %al # sched: [1:0.50]
-; SANDY-NEXT:    sete %cl # sched: [1:0.50]
-; SANDY-NEXT:    andb %al, %cl # sched: [1:0.33]
-; SANDY-NEXT:    vucomisd (%rdi), %xmm0 # sched: [8:1.00]
-; SANDY-NEXT:    setnp %al # sched: [1:0.50]
-; SANDY-NEXT:    sete %dl # sched: [1:0.50]
-; SANDY-NEXT:    andb %al, %dl # sched: [1:0.33]
-; SANDY-NEXT:    orb %cl, %dl # sched: [1:0.33]
-; SANDY-NEXT:    movzbl %dl, %eax # sched: [1:0.33]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_ucomisd:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    ucomisd %xmm1, %xmm0 # sched: [3:1.00]
-; HASWELL-SSE-NEXT:    setnp %al # sched: [1:0.50]
-; HASWELL-SSE-NEXT:    sete %cl # sched: [1:0.50]
-; HASWELL-SSE-NEXT:    andb %al, %cl # sched: [1:0.25]
-; HASWELL-SSE-NEXT:    ucomisd (%rdi), %xmm0 # sched: [8:1.00]
-; HASWELL-SSE-NEXT:    setnp %al # sched: [1:0.50]
-; HASWELL-SSE-NEXT:    sete %dl # sched: [1:0.50]
-; HASWELL-SSE-NEXT:    andb %al, %dl # sched: [1:0.25]
-; HASWELL-SSE-NEXT:    orb %cl, %dl # sched: [1:0.25]
-; HASWELL-SSE-NEXT:    movzbl %dl, %eax # sched: [1:0.25]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_ucomisd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vucomisd %xmm1, %xmm0 # sched: [3:1.00]
-; HASWELL-NEXT:    setnp %al # sched: [1:0.50]
-; HASWELL-NEXT:    sete %cl # sched: [1:0.50]
-; HASWELL-NEXT:    andb %al, %cl # sched: [1:0.25]
-; HASWELL-NEXT:    vucomisd (%rdi), %xmm0 # sched: [8:1.00]
-; HASWELL-NEXT:    setnp %al # sched: [1:0.50]
-; HASWELL-NEXT:    sete %dl # sched: [1:0.50]
-; HASWELL-NEXT:    andb %al, %dl # sched: [1:0.25]
-; HASWELL-NEXT:    orb %cl, %dl # sched: [1:0.25]
-; HASWELL-NEXT:    movzbl %dl, %eax # sched: [1:0.25]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_ucomisd:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    ucomisd %xmm1, %xmm0 # sched: [3:1.00]
-; BROADWELL-SSE-NEXT:    setnp %al # sched: [1:0.50]
-; BROADWELL-SSE-NEXT:    sete %cl # sched: [1:0.50]
-; BROADWELL-SSE-NEXT:    andb %al, %cl # sched: [1:0.25]
-; BROADWELL-SSE-NEXT:    ucomisd (%rdi), %xmm0 # sched: [8:1.00]
-; BROADWELL-SSE-NEXT:    setnp %al # sched: [1:0.50]
-; BROADWELL-SSE-NEXT:    sete %dl # sched: [1:0.50]
-; BROADWELL-SSE-NEXT:    andb %al, %dl # sched: [1:0.25]
-; BROADWELL-SSE-NEXT:    orb %cl, %dl # sched: [1:0.25]
-; BROADWELL-SSE-NEXT:    movzbl %dl, %eax # sched: [1:0.25]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_ucomisd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vucomisd %xmm1, %xmm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    setnp %al # sched: [1:0.50]
-; BROADWELL-NEXT:    sete %cl # sched: [1:0.50]
-; BROADWELL-NEXT:    andb %al, %cl # sched: [1:0.25]
-; BROADWELL-NEXT:    vucomisd (%rdi), %xmm0 # sched: [8:1.00]
-; BROADWELL-NEXT:    setnp %al # sched: [1:0.50]
-; BROADWELL-NEXT:    sete %dl # sched: [1:0.50]
-; BROADWELL-NEXT:    andb %al, %dl # sched: [1:0.25]
-; BROADWELL-NEXT:    orb %cl, %dl # sched: [1:0.25]
-; BROADWELL-NEXT:    movzbl %dl, %eax # sched: [1:0.25]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_ucomisd:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    ucomisd %xmm1, %xmm0 # sched: [2:1.00]
-; SKYLAKE-SSE-NEXT:    setnp %al # sched: [1:0.50]
-; SKYLAKE-SSE-NEXT:    sete %cl # sched: [1:0.50]
-; SKYLAKE-SSE-NEXT:    andb %al, %cl # sched: [1:0.25]
-; SKYLAKE-SSE-NEXT:    ucomisd (%rdi), %xmm0 # sched: [7:1.00]
-; SKYLAKE-SSE-NEXT:    setnp %al # sched: [1:0.50]
-; SKYLAKE-SSE-NEXT:    sete %dl # sched: [1:0.50]
-; SKYLAKE-SSE-NEXT:    andb %al, %dl # sched: [1:0.25]
-; SKYLAKE-SSE-NEXT:    orb %cl, %dl # sched: [1:0.25]
-; SKYLAKE-SSE-NEXT:    movzbl %dl, %eax # sched: [1:0.25]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_ucomisd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vucomisd %xmm1, %xmm0 # sched: [2:1.00]
-; SKYLAKE-NEXT:    setnp %al # sched: [1:0.50]
-; SKYLAKE-NEXT:    sete %cl # sched: [1:0.50]
-; SKYLAKE-NEXT:    andb %al, %cl # sched: [1:0.25]
-; SKYLAKE-NEXT:    vucomisd (%rdi), %xmm0 # sched: [7:1.00]
-; SKYLAKE-NEXT:    setnp %al # sched: [1:0.50]
-; SKYLAKE-NEXT:    sete %dl # sched: [1:0.50]
-; SKYLAKE-NEXT:    andb %al, %dl # sched: [1:0.25]
-; SKYLAKE-NEXT:    orb %cl, %dl # sched: [1:0.25]
-; SKYLAKE-NEXT:    movzbl %dl, %eax # sched: [1:0.25]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_ucomisd:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    ucomisd %xmm1, %xmm0 # sched: [2:1.00]
-; SKX-SSE-NEXT:    setnp %al # sched: [1:0.50]
-; SKX-SSE-NEXT:    sete %cl # sched: [1:0.50]
-; SKX-SSE-NEXT:    andb %al, %cl # sched: [1:0.25]
-; SKX-SSE-NEXT:    ucomisd (%rdi), %xmm0 # sched: [7:1.00]
-; SKX-SSE-NEXT:    setnp %al # sched: [1:0.50]
-; SKX-SSE-NEXT:    sete %dl # sched: [1:0.50]
-; SKX-SSE-NEXT:    andb %al, %dl # sched: [1:0.25]
-; SKX-SSE-NEXT:    orb %cl, %dl # sched: [1:0.25]
-; SKX-SSE-NEXT:    movzbl %dl, %eax # sched: [1:0.25]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_ucomisd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vucomisd %xmm1, %xmm0 # sched: [2:1.00]
-; SKX-NEXT:    setnp %al # sched: [1:0.50]
-; SKX-NEXT:    sete %cl # sched: [1:0.50]
-; SKX-NEXT:    andb %al, %cl # sched: [1:0.25]
-; SKX-NEXT:    vucomisd (%rdi), %xmm0 # sched: [7:1.00]
-; SKX-NEXT:    setnp %al # sched: [1:0.50]
-; SKX-NEXT:    sete %dl # sched: [1:0.50]
-; SKX-NEXT:    andb %al, %dl # sched: [1:0.25]
-; SKX-NEXT:    orb %cl, %dl # sched: [1:0.25]
-; SKX-NEXT:    movzbl %dl, %eax # sched: [1:0.25]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_ucomisd:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    ucomisd %xmm1, %xmm0 # sched: [1:1.00]
-; BDVER2-SSE-NEXT:    setnp %al # sched: [1:0.50]
-; BDVER2-SSE-NEXT:    sete %cl # sched: [1:0.50]
-; BDVER2-SSE-NEXT:    andb %al, %cl # sched: [1:0.50]
-; BDVER2-SSE-NEXT:    ucomisd (%rdi), %xmm0 # sched: [6:1.00]
-; BDVER2-SSE-NEXT:    setnp %al # sched: [1:0.50]
-; BDVER2-SSE-NEXT:    sete %dl # sched: [1:0.50]
-; BDVER2-SSE-NEXT:    andb %al, %dl # sched: [1:0.50]
-; BDVER2-SSE-NEXT:    orb %cl, %dl # sched: [1:0.50]
-; BDVER2-SSE-NEXT:    movzbl %dl, %eax # sched: [1:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_ucomisd:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vucomisd %xmm1, %xmm0 # sched: [1:1.00]
-; BDVER2-NEXT:    setnp %al # sched: [1:0.50]
-; BDVER2-NEXT:    sete %cl # sched: [1:0.50]
-; BDVER2-NEXT:    andb %al, %cl # sched: [1:0.50]
-; BDVER2-NEXT:    vucomisd (%rdi), %xmm0 # sched: [6:1.00]
-; BDVER2-NEXT:    setnp %al # sched: [1:0.50]
-; BDVER2-NEXT:    sete %dl # sched: [1:0.50]
-; BDVER2-NEXT:    andb %al, %dl # sched: [1:0.50]
-; BDVER2-NEXT:    orb %cl, %dl # sched: [1:0.50]
-; BDVER2-NEXT:    movzbl %dl, %eax # sched: [1:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_ucomisd:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    ucomisd %xmm1, %xmm0 # sched: [3:1.00]
-; BTVER2-SSE-NEXT:    setnp %al # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    sete %cl # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    andb %al, %cl # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    ucomisd (%rdi), %xmm0 # sched: [8:1.00]
-; BTVER2-SSE-NEXT:    setnp %al # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    sete %dl # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    andb %al, %dl # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    orb %cl, %dl # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    movzbl %dl, %eax # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_ucomisd:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vucomisd %xmm1, %xmm0 # sched: [3:1.00]
-; BTVER2-NEXT:    setnp %al # sched: [1:0.50]
-; BTVER2-NEXT:    sete %cl # sched: [1:0.50]
-; BTVER2-NEXT:    andb %al, %cl # sched: [1:0.50]
-; BTVER2-NEXT:    vucomisd (%rdi), %xmm0 # sched: [8:1.00]
-; BTVER2-NEXT:    setnp %al # sched: [1:0.50]
-; BTVER2-NEXT:    sete %dl # sched: [1:0.50]
-; BTVER2-NEXT:    andb %al, %dl # sched: [1:0.50]
-; BTVER2-NEXT:    orb %cl, %dl # sched: [1:0.50]
-; BTVER2-NEXT:    movzbl %dl, %eax # sched: [1:0.50]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_ucomisd:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    ucomisd %xmm1, %xmm0 # sched: [3:1.00]
-; ZNVER1-SSE-NEXT:    setnp %al # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    sete %cl # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    andb %al, %cl # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    ucomisd (%rdi), %xmm0 # sched: [10:1.00]
-; ZNVER1-SSE-NEXT:    setnp %al # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    sete %dl # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    andb %al, %dl # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    orb %cl, %dl # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    movzbl %dl, %eax # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_ucomisd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vucomisd %xmm1, %xmm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    setnp %al # sched: [1:0.25]
-; ZNVER1-NEXT:    sete %cl # sched: [1:0.25]
-; ZNVER1-NEXT:    andb %al, %cl # sched: [1:0.25]
-; ZNVER1-NEXT:    vucomisd (%rdi), %xmm0 # sched: [10:1.00]
-; ZNVER1-NEXT:    setnp %al # sched: [1:0.25]
-; ZNVER1-NEXT:    sete %dl # sched: [1:0.25]
-; ZNVER1-NEXT:    andb %al, %dl # sched: [1:0.25]
-; ZNVER1-NEXT:    orb %cl, %dl # sched: [1:0.25]
-; ZNVER1-NEXT:    movzbl %dl, %eax # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call i32 @llvm.x86.sse2.ucomieq.sd(<2 x double> %a0, <2 x double> %a1)
-  %2 = load <2 x double>, <2 x double> *%a2, align 8
-  %3 = call i32 @llvm.x86.sse2.ucomieq.sd(<2 x double> %a0, <2 x double> %2)
-  %4 = or i32 %1, %3
-  ret i32 %4
-}
-declare i32 @llvm.x86.sse2.ucomieq.sd(<2 x double>, <2 x double>) nounwind readnone
-
-define <2 x double> @test_unpckhpd(<2 x double> %a0, <2 x double> %a1, <2 x double> *%a2) {
-; GENERIC-LABEL: test_unpckhpd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    unpckhpd {{.*#+}} xmm0 = xmm0[1],xmm1[1] sched: [1:1.00]
-; GENERIC-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],mem[1] sched: [7:1.00]
-; GENERIC-NEXT:    addpd %xmm1, %xmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_unpckhpd:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    unpckhpd {{.*#+}} xmm0 = xmm0[1],xmm1[1] sched: [1:1.00]
-; ATOM-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],mem[1] sched: [1:1.00]
-; ATOM-NEXT:    addpd %xmm1, %xmm0 # sched: [6:3.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_unpckhpd:
-; SLM:       # %bb.0:
-; SLM-NEXT:    unpckhpd {{.*#+}} xmm0 = xmm0[1],xmm1[1] sched: [1:1.00]
-; SLM-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],mem[1] sched: [4:1.00]
-; SLM-NEXT:    addpd %xmm1, %xmm0 # sched: [3:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_unpckhpd:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    unpckhpd {{.*#+}} xmm0 = xmm0[1],xmm1[1] sched: [1:1.00]
-; SANDY-SSE-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],mem[1] sched: [7:1.00]
-; SANDY-SSE-NEXT:    addpd %xmm1, %xmm0 # sched: [3:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_unpckhpd:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vunpckhpd {{.*#+}} xmm0 = xmm0[1],xmm1[1] sched: [1:1.00]
-; SANDY-NEXT:    vunpckhpd {{.*#+}} xmm1 = xmm1[1],mem[1] sched: [7:1.00]
-; SANDY-NEXT:    vaddpd %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_unpckhpd:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    unpckhpd {{.*#+}} xmm0 = xmm0[1],xmm1[1] sched: [1:1.00]
-; HASWELL-SSE-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],mem[1] sched: [7:1.00]
-; HASWELL-SSE-NEXT:    addpd %xmm1, %xmm0 # sched: [3:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_unpckhpd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vunpckhpd {{.*#+}} xmm0 = xmm0[1],xmm1[1] sched: [1:1.00]
-; HASWELL-NEXT:    vunpckhpd {{.*#+}} xmm1 = xmm1[1],mem[1] sched: [7:1.00]
-; HASWELL-NEXT:    vaddpd %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_unpckhpd:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    unpckhpd {{.*#+}} xmm0 = xmm0[1],xmm1[1] sched: [1:1.00]
-; BROADWELL-SSE-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],mem[1] sched: [6:1.00]
-; BROADWELL-SSE-NEXT:    addpd %xmm1, %xmm0 # sched: [3:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_unpckhpd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vunpckhpd {{.*#+}} xmm0 = xmm0[1],xmm1[1] sched: [1:1.00]
-; BROADWELL-NEXT:    vunpckhpd {{.*#+}} xmm1 = xmm1[1],mem[1] sched: [6:1.00]
-; BROADWELL-NEXT:    vaddpd %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_unpckhpd:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    unpckhpd {{.*#+}} xmm0 = xmm0[1],xmm1[1] sched: [1:1.00]
-; SKYLAKE-SSE-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],mem[1] sched: [7:1.00]
-; SKYLAKE-SSE-NEXT:    addpd %xmm1, %xmm0 # sched: [4:0.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_unpckhpd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vunpckhpd {{.*#+}} xmm0 = xmm0[1],xmm1[1] sched: [1:1.00]
-; SKYLAKE-NEXT:    vunpckhpd {{.*#+}} xmm1 = xmm1[1],mem[1] sched: [7:1.00]
-; SKYLAKE-NEXT:    vaddpd %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_unpckhpd:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    unpckhpd {{.*#+}} xmm0 = xmm0[1],xmm1[1] sched: [1:1.00]
-; SKX-SSE-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],mem[1] sched: [7:1.00]
-; SKX-SSE-NEXT:    addpd %xmm1, %xmm0 # sched: [4:0.50]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_unpckhpd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vunpckhpd {{.*#+}} xmm0 = xmm0[1],xmm1[1] sched: [1:1.00]
-; SKX-NEXT:    vunpckhpd {{.*#+}} xmm1 = xmm1[1],mem[1] sched: [7:1.00]
-; SKX-NEXT:    vaddpd %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_unpckhpd:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    unpckhpd {{.*#+}} xmm0 = xmm0[1],xmm1[1] sched: [2:0.50]
-; BDVER2-SSE-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],mem[1] sched: [7:0.50]
-; BDVER2-SSE-NEXT:    addpd %xmm1, %xmm0 # sched: [5:1.00]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_unpckhpd:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vunpckhpd {{.*#+}} xmm0 = xmm0[1],xmm1[1] sched: [2:0.50]
-; BDVER2-NEXT:    vunpckhpd {{.*#+}} xmm1 = xmm1[1],mem[1] sched: [7:0.50]
-; BDVER2-NEXT:    vaddpd %xmm1, %xmm0, %xmm0 # sched: [5:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_unpckhpd:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    unpckhpd {{.*#+}} xmm0 = xmm0[1],xmm1[1] sched: [1:0.50]
-; BTVER2-SSE-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],mem[1] sched: [6:1.00]
-; BTVER2-SSE-NEXT:    addpd %xmm1, %xmm0 # sched: [3:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_unpckhpd:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vunpckhpd {{.*#+}} xmm0 = xmm0[1],xmm1[1] sched: [1:0.50]
-; BTVER2-NEXT:    vunpckhpd {{.*#+}} xmm1 = xmm1[1],mem[1] sched: [6:1.00]
-; BTVER2-NEXT:    vaddpd %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_unpckhpd:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    unpckhpd {{.*#+}} xmm0 = xmm0[1],xmm1[1] sched: [1:0.50]
-; ZNVER1-SSE-NEXT:    unpckhpd {{.*#+}} xmm1 = xmm1[1],mem[1] sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    addpd %xmm1, %xmm0 # sched: [3:1.00]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_unpckhpd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vunpckhpd {{.*#+}} xmm0 = xmm0[1],xmm1[1] sched: [1:0.50]
-; ZNVER1-NEXT:    vunpckhpd {{.*#+}} xmm1 = xmm1[1],mem[1] sched: [8:0.50]
-; ZNVER1-NEXT:    vaddpd %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = shufflevector <2 x double> %a0, <2 x double> %a1, <2 x i32> <i32 1, i32 3>
-  %2 = load <2 x double>, <2 x double> *%a2, align 16
-  %3 = shufflevector <2 x double> %a1, <2 x double> %2, <2 x i32> <i32 1, i32 3>
-  %4 = fadd <2 x double> %1, %3
-  ret <2 x double> %4
-}
-
-define <2 x double> @test_unpcklpd(<2 x double> %a0, <2 x double> %a1, <2 x double> *%a2) {
-; GENERIC-LABEL: test_unpcklpd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    movapd %xmm0, %xmm2 # sched: [1:1.00]
-; GENERIC-NEXT:    unpcklpd {{.*#+}} xmm2 = xmm2[0],xmm1[0] sched: [1:1.00]
-; GENERIC-NEXT:    unpcklpd {{.*#+}} xmm0 = xmm0[0],mem[0] sched: [7:1.00]
-; GENERIC-NEXT:    addpd %xmm2, %xmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_unpcklpd:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    movapd %xmm0, %xmm2 # sched: [1:0.50]
-; ATOM-NEXT:    unpcklpd {{.*#+}} xmm0 = xmm0[0],mem[0] sched: [1:1.00]
-; ATOM-NEXT:    unpcklpd {{.*#+}} xmm2 = xmm2[0],xmm1[0] sched: [1:1.00]
-; ATOM-NEXT:    addpd %xmm2, %xmm0 # sched: [6:3.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_unpcklpd:
-; SLM:       # %bb.0:
-; SLM-NEXT:    movapd %xmm0, %xmm2 # sched: [1:0.50]
-; SLM-NEXT:    unpcklpd {{.*#+}} xmm0 = xmm0[0],mem[0] sched: [4:1.00]
-; SLM-NEXT:    unpcklpd {{.*#+}} xmm2 = xmm2[0],xmm1[0] sched: [1:1.00]
-; SLM-NEXT:    addpd %xmm2, %xmm0 # sched: [3:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_unpcklpd:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    movapd %xmm0, %xmm2 # sched: [1:1.00]
-; SANDY-SSE-NEXT:    unpcklpd {{.*#+}} xmm2 = xmm2[0],xmm1[0] sched: [1:1.00]
-; SANDY-SSE-NEXT:    unpcklpd {{.*#+}} xmm0 = xmm0[0],mem[0] sched: [7:1.00]
-; SANDY-SSE-NEXT:    addpd %xmm2, %xmm0 # sched: [3:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_unpcklpd:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vunpcklpd {{.*#+}} xmm1 = xmm0[0],xmm1[0] sched: [1:1.00]
-; SANDY-NEXT:    vunpcklpd {{.*#+}} xmm0 = xmm0[0],mem[0] sched: [7:1.00]
-; SANDY-NEXT:    vaddpd %xmm0, %xmm1, %xmm0 # sched: [3:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_unpcklpd:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    movapd %xmm0, %xmm2 # sched: [1:1.00]
-; HASWELL-SSE-NEXT:    unpcklpd {{.*#+}} xmm2 = xmm2[0],xmm1[0] sched: [1:1.00]
-; HASWELL-SSE-NEXT:    unpcklpd {{.*#+}} xmm0 = xmm0[0],mem[0] sched: [7:1.00]
-; HASWELL-SSE-NEXT:    addpd %xmm2, %xmm0 # sched: [3:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_unpcklpd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vunpcklpd {{.*#+}} xmm1 = xmm0[0],xmm1[0] sched: [1:1.00]
-; HASWELL-NEXT:    vunpcklpd {{.*#+}} xmm0 = xmm0[0],mem[0] sched: [7:1.00]
-; HASWELL-NEXT:    vaddpd %xmm0, %xmm1, %xmm0 # sched: [3:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_unpcklpd:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    movapd %xmm0, %xmm2 # sched: [1:1.00]
-; BROADWELL-SSE-NEXT:    unpcklpd {{.*#+}} xmm2 = xmm2[0],xmm1[0] sched: [1:1.00]
-; BROADWELL-SSE-NEXT:    unpcklpd {{.*#+}} xmm0 = xmm0[0],mem[0] sched: [6:1.00]
-; BROADWELL-SSE-NEXT:    addpd %xmm2, %xmm0 # sched: [3:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_unpcklpd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vunpcklpd {{.*#+}} xmm1 = xmm0[0],xmm1[0] sched: [1:1.00]
-; BROADWELL-NEXT:    vunpcklpd {{.*#+}} xmm0 = xmm0[0],mem[0] sched: [6:1.00]
-; BROADWELL-NEXT:    vaddpd %xmm0, %xmm1, %xmm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_unpcklpd:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    movapd %xmm0, %xmm2 # sched: [1:0.33]
-; SKYLAKE-SSE-NEXT:    unpcklpd {{.*#+}} xmm2 = xmm2[0],xmm1[0] sched: [1:1.00]
-; SKYLAKE-SSE-NEXT:    unpcklpd {{.*#+}} xmm0 = xmm0[0],mem[0] sched: [7:1.00]
-; SKYLAKE-SSE-NEXT:    addpd %xmm2, %xmm0 # sched: [4:0.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_unpcklpd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vunpcklpd {{.*#+}} xmm1 = xmm0[0],xmm1[0] sched: [1:1.00]
-; SKYLAKE-NEXT:    vunpcklpd {{.*#+}} xmm0 = xmm0[0],mem[0] sched: [7:1.00]
-; SKYLAKE-NEXT:    vaddpd %xmm0, %xmm1, %xmm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_unpcklpd:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    movapd %xmm0, %xmm2 # sched: [1:0.33]
-; SKX-SSE-NEXT:    unpcklpd {{.*#+}} xmm2 = xmm2[0],xmm1[0] sched: [1:1.00]
-; SKX-SSE-NEXT:    unpcklpd {{.*#+}} xmm0 = xmm0[0],mem[0] sched: [7:1.00]
-; SKX-SSE-NEXT:    addpd %xmm2, %xmm0 # sched: [4:0.50]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_unpcklpd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vunpcklpd {{.*#+}} xmm1 = xmm0[0],xmm1[0] sched: [1:1.00]
-; SKX-NEXT:    vunpcklpd {{.*#+}} xmm0 = xmm0[0],mem[0] sched: [7:1.00]
-; SKX-NEXT:    vaddpd %xmm0, %xmm1, %xmm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_unpcklpd:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    movapd %xmm0, %xmm2 # sched: [1:0.50]
-; BDVER2-SSE-NEXT:    unpcklpd {{.*#+}} xmm0 = xmm0[0],mem[0] sched: [7:0.50]
-; BDVER2-SSE-NEXT:    unpcklpd {{.*#+}} xmm2 = xmm2[0],xmm1[0] sched: [2:0.50]
-; BDVER2-SSE-NEXT:    addpd %xmm2, %xmm0 # sched: [5:1.00]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_unpcklpd:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vunpcklpd {{.*#+}} xmm1 = xmm0[0],xmm1[0] sched: [2:0.50]
-; BDVER2-NEXT:    vunpcklpd {{.*#+}} xmm0 = xmm0[0],mem[0] sched: [7:0.50]
-; BDVER2-NEXT:    vaddpd %xmm0, %xmm1, %xmm0 # sched: [5:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_unpcklpd:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    movapd %xmm0, %xmm2 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    unpcklpd {{.*#+}} xmm0 = xmm0[0],mem[0] sched: [6:1.00]
-; BTVER2-SSE-NEXT:    unpcklpd {{.*#+}} xmm2 = xmm2[0],xmm1[0] sched: [1:0.50]
-; BTVER2-SSE-NEXT:    addpd %xmm2, %xmm0 # sched: [3:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_unpcklpd:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vunpcklpd {{.*#+}} xmm1 = xmm0[0],xmm1[0] sched: [1:0.50]
-; BTVER2-NEXT:    vunpcklpd {{.*#+}} xmm0 = xmm0[0],mem[0] sched: [6:1.00]
-; BTVER2-NEXT:    vaddpd %xmm0, %xmm1, %xmm0 # sched: [3:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_unpcklpd:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    movapd %xmm0, %xmm2 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    unpcklpd {{.*#+}} xmm0 = xmm0[0],mem[0] sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    unpcklpd {{.*#+}} xmm2 = xmm2[0],xmm1[0] sched: [1:0.50]
-; ZNVER1-SSE-NEXT:    addpd %xmm2, %xmm0 # sched: [3:1.00]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_unpcklpd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vunpcklpd {{.*#+}} xmm1 = xmm0[0],xmm1[0] sched: [1:0.50]
-; ZNVER1-NEXT:    vunpcklpd {{.*#+}} xmm0 = xmm0[0],mem[0] sched: [8:0.50]
-; ZNVER1-NEXT:    vaddpd %xmm0, %xmm1, %xmm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = shufflevector <2 x double> %a0, <2 x double> %a1, <2 x i32> <i32 0, i32 2>
-  %2 = load <2 x double>, <2 x double> *%a2, align 16
-  %3 = shufflevector <2 x double> %1, <2 x double> %2, <2 x i32> <i32 0, i32 2>
-  %4 = fadd <2 x double> %1, %3
-  ret <2 x double> %4
-}
-
-define <2 x double> @test_xorpd(<2 x double> %a0, <2 x double> %a1, <2 x double> *%a2) {
-; GENERIC-LABEL: test_xorpd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    xorpd %xmm1, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    xorpd (%rdi), %xmm0 # sched: [7:1.00]
-; GENERIC-NEXT:    addpd %xmm1, %xmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_xorpd:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    xorpd %xmm1, %xmm0 # sched: [1:0.50]
-; ATOM-NEXT:    xorpd (%rdi), %xmm0 # sched: [1:1.00]
-; ATOM-NEXT:    addpd %xmm1, %xmm0 # sched: [6:3.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_xorpd:
-; SLM:       # %bb.0:
-; SLM-NEXT:    xorpd %xmm1, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    xorpd (%rdi), %xmm0 # sched: [4:1.00]
-; SLM-NEXT:    addpd %xmm1, %xmm0 # sched: [3:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_xorpd:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    xorpd %xmm1, %xmm0 # sched: [1:1.00]
-; SANDY-SSE-NEXT:    xorpd (%rdi), %xmm0 # sched: [7:1.00]
-; SANDY-SSE-NEXT:    addpd %xmm1, %xmm0 # sched: [3:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_xorpd:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vxorpd %xmm1, %xmm0, %xmm0 # sched: [1:1.00]
-; SANDY-NEXT:    vxorpd (%rdi), %xmm0, %xmm0 # sched: [7:1.00]
-; SANDY-NEXT:    vaddpd %xmm0, %xmm1, %xmm0 # sched: [3:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_xorpd:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    xorpd %xmm1, %xmm0 # sched: [1:1.00]
-; HASWELL-SSE-NEXT:    xorpd (%rdi), %xmm0 # sched: [7:1.00]
-; HASWELL-SSE-NEXT:    addpd %xmm1, %xmm0 # sched: [3:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_xorpd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vxorpd %xmm1, %xmm0, %xmm0 # sched: [1:1.00]
-; HASWELL-NEXT:    vxorpd (%rdi), %xmm0, %xmm0 # sched: [7:1.00]
-; HASWELL-NEXT:    vaddpd %xmm0, %xmm1, %xmm0 # sched: [3:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_xorpd:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    xorpd %xmm1, %xmm0 # sched: [1:1.00]
-; BROADWELL-SSE-NEXT:    xorpd (%rdi), %xmm0 # sched: [6:1.00]
-; BROADWELL-SSE-NEXT:    addpd %xmm1, %xmm0 # sched: [3:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_xorpd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vxorpd %xmm1, %xmm0, %xmm0 # sched: [1:1.00]
-; BROADWELL-NEXT:    vxorpd (%rdi), %xmm0, %xmm0 # sched: [6:1.00]
-; BROADWELL-NEXT:    vaddpd %xmm0, %xmm1, %xmm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_xorpd:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    xorpd %xmm1, %xmm0 # sched: [1:0.33]
-; SKYLAKE-SSE-NEXT:    xorpd (%rdi), %xmm0 # sched: [7:0.50]
-; SKYLAKE-SSE-NEXT:    addpd %xmm1, %xmm0 # sched: [4:0.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_xorpd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vxorpd %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    vxorpd (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKYLAKE-NEXT:    vaddpd %xmm0, %xmm1, %xmm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_xorpd:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    xorpd %xmm1, %xmm0 # sched: [1:0.33]
-; SKX-SSE-NEXT:    xorpd (%rdi), %xmm0 # sched: [7:0.50]
-; SKX-SSE-NEXT:    addpd %xmm1, %xmm0 # sched: [4:0.50]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_xorpd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vxorpd %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    vxorpd (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKX-NEXT:    vaddpd %xmm0, %xmm1, %xmm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_xorpd:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    xorpd %xmm1, %xmm0 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    xorpd (%rdi), %xmm0 # sched: [7:0.50]
-; BDVER2-SSE-NEXT:    addpd %xmm1, %xmm0 # sched: [5:1.00]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_xorpd:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vxorpd %xmm1, %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    vxorpd (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; BDVER2-NEXT:    vaddpd %xmm0, %xmm1, %xmm0 # sched: [5:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_xorpd:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    xorpd %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    xorpd (%rdi), %xmm0 # sched: [6:1.00]
-; BTVER2-SSE-NEXT:    addpd %xmm1, %xmm0 # sched: [3:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_xorpd:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vxorpd %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    vxorpd (%rdi), %xmm0, %xmm0 # sched: [6:1.00]
-; BTVER2-NEXT:    vaddpd %xmm0, %xmm1, %xmm0 # sched: [3:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_xorpd:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    xorpd %xmm1, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    xorpd (%rdi), %xmm0 # sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    addpd %xmm1, %xmm0 # sched: [3:1.00]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_xorpd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vxorpd %xmm1, %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vxorpd (%rdi), %xmm0, %xmm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    vaddpd %xmm0, %xmm1, %xmm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = bitcast <2 x double> %a0 to <4 x i32>
-  %2 = bitcast <2 x double> %a1 to <4 x i32>
-  %3 = xor <4 x i32> %1, %2
-  %4 = load <2 x double>, <2 x double> *%a2, align 16
-  %5 = bitcast <2 x double> %4 to <4 x i32>
-  %6 = xor <4 x i32> %3, %5
-  %7 = bitcast <4 x i32> %6 to <2 x double>
-  %8 = fadd <2 x double> %a1, %7
-  ret <2 x double> %8
-}
-
-!0 = !{i32 1}
diff --git a/test/CodeGen/X86/sse3-avx-addsub-2.ll b/test/CodeGen/X86/sse3-avx-addsub-2.ll
index 8041647cc14baf8b13d231259b1acc45d44f3d49..c43f293f5aadd25ccf76cb30aaf22ae947c14c3b 100644
--- a/test/CodeGen/X86/sse3-avx-addsub-2.ll
+++ b/test/CodeGen/X86/sse3-avx-addsub-2.ll
@@ -403,17 +403,29 @@ define <4 x float> @test15(<4 x float> %A, <4 x float> %B) {
 ; SSE-NEXT:    movaps %xmm2, %xmm0
 ; SSE-NEXT:    retq
 ;
-; AVX-LABEL: test15:
-; AVX:       # %bb.0:
-; AVX-NEXT:    vmovshdup {{.*#+}} xmm2 = xmm0[1,1,3,3]
-; AVX-NEXT:    vmovshdup {{.*#+}} xmm3 = xmm1[1,1,3,3]
-; AVX-NEXT:    vaddss %xmm3, %xmm2, %xmm2
-; AVX-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,1,2,3]
-; AVX-NEXT:    vpermilps {{.*#+}} xmm1 = xmm1[3,1,2,3]
-; AVX-NEXT:    vaddss %xmm1, %xmm0, %xmm0
-; AVX-NEXT:    vmovsldup {{.*#+}} xmm1 = xmm2[0,0,2,2]
-; AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
-; AVX-NEXT:    retq
+; AVX1-LABEL: test15:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vmovshdup {{.*#+}} xmm2 = xmm0[1,1,3,3]
+; AVX1-NEXT:    vmovshdup {{.*#+}} xmm3 = xmm1[1,1,3,3]
+; AVX1-NEXT:    vaddss %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm1[3,1,2,3]
+; AVX1-NEXT:    vaddss %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vmovsldup {{.*#+}} xmm1 = xmm2[0,0,2,2]
+; AVX1-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
+; AVX1-NEXT:    retq
+;
+; AVX512-LABEL: test15:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vmovshdup {{.*#+}} xmm2 = xmm0[1,1,3,3]
+; AVX512-NEXT:    vmovshdup {{.*#+}} xmm3 = xmm1[1,1,3,3]
+; AVX512-NEXT:    vaddss %xmm3, %xmm2, %xmm2
+; AVX512-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; AVX512-NEXT:    vpermilps {{.*#+}} xmm1 = xmm1[3,1,2,3]
+; AVX512-NEXT:    vaddss %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vbroadcastss %xmm2, %xmm1
+; AVX512-NEXT:    vinsertps {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[0]
+; AVX512-NEXT:    retq
   %1 = extractelement <4 x float> %A, i32 1
   %2 = extractelement <4 x float> %B, i32 1
   %add = fadd float %1, %2
diff --git a/test/CodeGen/X86/sse3-schedule.ll b/test/CodeGen/X86/sse3-schedule.ll
deleted file mode 100644
index 1c3419a35ff3eccc31b567576a41144f0d3e0e20..0000000000000000000000000000000000000000
--- a/test/CodeGen/X86/sse3-schedule.ll
+++ /dev/null
@@ -1,1549 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=x86-64 -mattr=+sse3 | FileCheck %s --check-prefixes=CHECK,GENERIC
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=atom -mattr=-ssse3 | FileCheck %s --check-prefixes=CHECK,ATOM
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=slm -mattr=-ssse3 | FileCheck %s --check-prefixes=CHECK,SLM
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=sandybridge -mattr=-ssse3 | FileCheck %s --check-prefixes=CHECK,SANDY-SSE
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=sandybridge -mattr=-avx2  | FileCheck %s --check-prefixes=CHECK,SANDY
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=ivybridge -mattr=-ssse3 | FileCheck %s --check-prefixes=CHECK,SANDY-SSE
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=ivybridge -mattr=-avx2  | FileCheck %s --check-prefixes=CHECK,SANDY
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=haswell -mattr=-ssse3 | FileCheck %s --check-prefixes=CHECK,HASWELL-SSE
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=haswell -mattr=-avx2  | FileCheck %s --check-prefixes=CHECK,HASWELL
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=broadwell -mattr=-ssse3 | FileCheck %s --check-prefixes=CHECK,BROADWELL-SSE
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=broadwell -mattr=-avx2  | FileCheck %s --check-prefixes=CHECK,BROADWELL
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=skylake -mattr=-ssse3 | FileCheck %s --check-prefixes=CHECK,SKYLAKE-SSE
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=skylake -mattr=-avx2  | FileCheck %s --check-prefixes=CHECK,SKYLAKE
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=skx -mattr=-ssse3 | FileCheck %s --check-prefixes=CHECK,SKX-SSE
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=skx -mattr=-avx2  | FileCheck %s --check-prefixes=CHECK,SKX
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=bdver2 -mattr=-ssse3 | FileCheck %s --check-prefixes=CHECK,BDVER2-SSE
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=bdver2 -mattr=-avx2  | FileCheck %s --check-prefixes=CHECK,BDVER2
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=btver2 -mattr=-ssse3 | FileCheck %s --check-prefixes=CHECK,BTVER2-SSE
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=btver2 -mattr=-avx2  | FileCheck %s --check-prefixes=CHECK,BTVER2
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=znver1 -mattr=-ssse3 | FileCheck %s --check-prefixes=CHECK,ZNVER1-SSE
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=znver1 -mattr=-avx2  | FileCheck %s --check-prefixes=CHECK,ZNVER1
-
-define <2 x double> @test_addsubpd(<2 x double> %a0, <2 x double> %a1, <2 x double> *%a2) {
-; GENERIC-LABEL: test_addsubpd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    addsubpd %xmm1, %xmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    addsubpd (%rdi), %xmm0 # sched: [9:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_addsubpd:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    addsubpd %xmm1, %xmm0 # sched: [6:3.00]
-; ATOM-NEXT:    addsubpd (%rdi), %xmm0 # sched: [7:3.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_addsubpd:
-; SLM:       # %bb.0:
-; SLM-NEXT:    addsubpd %xmm1, %xmm0 # sched: [3:1.00]
-; SLM-NEXT:    addsubpd (%rdi), %xmm0 # sched: [6:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_addsubpd:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    addsubpd %xmm1, %xmm0 # sched: [3:1.00]
-; SANDY-SSE-NEXT:    addsubpd (%rdi), %xmm0 # sched: [9:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_addsubpd:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vaddsubpd %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; SANDY-NEXT:    vaddsubpd (%rdi), %xmm0, %xmm0 # sched: [9:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_addsubpd:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    addsubpd %xmm1, %xmm0 # sched: [3:1.00]
-; HASWELL-SSE-NEXT:    addsubpd (%rdi), %xmm0 # sched: [9:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_addsubpd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vaddsubpd %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; HASWELL-NEXT:    vaddsubpd (%rdi), %xmm0, %xmm0 # sched: [9:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_addsubpd:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    addsubpd %xmm1, %xmm0 # sched: [3:1.00]
-; BROADWELL-SSE-NEXT:    addsubpd (%rdi), %xmm0 # sched: [8:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_addsubpd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vaddsubpd %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    vaddsubpd (%rdi), %xmm0, %xmm0 # sched: [8:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_addsubpd:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    addsubpd %xmm1, %xmm0 # sched: [4:0.50]
-; SKYLAKE-SSE-NEXT:    addsubpd (%rdi), %xmm0 # sched: [10:0.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_addsubpd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vaddsubpd %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    vaddsubpd (%rdi), %xmm0, %xmm0 # sched: [10:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_addsubpd:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    addsubpd %xmm1, %xmm0 # sched: [4:0.50]
-; SKX-SSE-NEXT:    addsubpd (%rdi), %xmm0 # sched: [10:0.50]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_addsubpd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vaddsubpd %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKX-NEXT:    vaddsubpd (%rdi), %xmm0, %xmm0 # sched: [10:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_addsubpd:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    addsubpd %xmm1, %xmm0 # sched: [5:1.00]
-; BDVER2-SSE-NEXT:    addsubpd (%rdi), %xmm0 # sched: [10:1.00]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_addsubpd:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vaddsubpd %xmm1, %xmm0, %xmm0 # sched: [5:1.00]
-; BDVER2-NEXT:    vaddsubpd (%rdi), %xmm0, %xmm0 # sched: [10:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_addsubpd:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    addsubpd %xmm1, %xmm0 # sched: [3:1.00]
-; BTVER2-SSE-NEXT:    addsubpd (%rdi), %xmm0 # sched: [8:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_addsubpd:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vaddsubpd %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; BTVER2-NEXT:    vaddsubpd (%rdi), %xmm0, %xmm0 # sched: [8:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_addsubpd:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    addsubpd %xmm1, %xmm0 # sched: [3:1.00]
-; ZNVER1-SSE-NEXT:    addsubpd (%rdi), %xmm0 # sched: [10:1.00]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_addsubpd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vaddsubpd %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    vaddsubpd (%rdi), %xmm0, %xmm0 # sched: [10:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <2 x double> @llvm.x86.sse3.addsub.pd(<2 x double> %a0, <2 x double> %a1)
-  %2 = load <2 x double>, <2 x double> *%a2, align 16
-  %3 = call <2 x double> @llvm.x86.sse3.addsub.pd(<2 x double> %1, <2 x double> %2)
-  ret <2 x double> %3
-}
-declare <2 x double> @llvm.x86.sse3.addsub.pd(<2 x double>, <2 x double>) nounwind readnone
-
-define <4 x float> @test_addsubps(<4 x float> %a0, <4 x float> %a1, <4 x float> *%a2) {
-; GENERIC-LABEL: test_addsubps:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    addsubps %xmm1, %xmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    addsubps (%rdi), %xmm0 # sched: [9:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_addsubps:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    addsubps %xmm1, %xmm0 # sched: [5:5.00]
-; ATOM-NEXT:    addsubps (%rdi), %xmm0 # sched: [5:5.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_addsubps:
-; SLM:       # %bb.0:
-; SLM-NEXT:    addsubps %xmm1, %xmm0 # sched: [3:1.00]
-; SLM-NEXT:    addsubps (%rdi), %xmm0 # sched: [6:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_addsubps:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    addsubps %xmm1, %xmm0 # sched: [3:1.00]
-; SANDY-SSE-NEXT:    addsubps (%rdi), %xmm0 # sched: [9:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_addsubps:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vaddsubps %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; SANDY-NEXT:    vaddsubps (%rdi), %xmm0, %xmm0 # sched: [9:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_addsubps:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    addsubps %xmm1, %xmm0 # sched: [3:1.00]
-; HASWELL-SSE-NEXT:    addsubps (%rdi), %xmm0 # sched: [9:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_addsubps:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vaddsubps %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; HASWELL-NEXT:    vaddsubps (%rdi), %xmm0, %xmm0 # sched: [9:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_addsubps:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    addsubps %xmm1, %xmm0 # sched: [3:1.00]
-; BROADWELL-SSE-NEXT:    addsubps (%rdi), %xmm0 # sched: [8:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_addsubps:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vaddsubps %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    vaddsubps (%rdi), %xmm0, %xmm0 # sched: [8:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_addsubps:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    addsubps %xmm1, %xmm0 # sched: [4:0.50]
-; SKYLAKE-SSE-NEXT:    addsubps (%rdi), %xmm0 # sched: [10:0.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_addsubps:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vaddsubps %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    vaddsubps (%rdi), %xmm0, %xmm0 # sched: [10:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_addsubps:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    addsubps %xmm1, %xmm0 # sched: [4:0.50]
-; SKX-SSE-NEXT:    addsubps (%rdi), %xmm0 # sched: [10:0.50]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_addsubps:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vaddsubps %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKX-NEXT:    vaddsubps (%rdi), %xmm0, %xmm0 # sched: [10:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_addsubps:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    addsubps %xmm1, %xmm0 # sched: [5:1.00]
-; BDVER2-SSE-NEXT:    addsubps (%rdi), %xmm0 # sched: [10:1.00]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_addsubps:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vaddsubps %xmm1, %xmm0, %xmm0 # sched: [5:1.00]
-; BDVER2-NEXT:    vaddsubps (%rdi), %xmm0, %xmm0 # sched: [10:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_addsubps:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    addsubps %xmm1, %xmm0 # sched: [3:1.00]
-; BTVER2-SSE-NEXT:    addsubps (%rdi), %xmm0 # sched: [8:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_addsubps:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vaddsubps %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; BTVER2-NEXT:    vaddsubps (%rdi), %xmm0, %xmm0 # sched: [8:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_addsubps:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    addsubps %xmm1, %xmm0 # sched: [3:1.00]
-; ZNVER1-SSE-NEXT:    addsubps (%rdi), %xmm0 # sched: [10:1.00]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_addsubps:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vaddsubps %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    vaddsubps (%rdi), %xmm0, %xmm0 # sched: [10:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <4 x float> @llvm.x86.sse3.addsub.ps(<4 x float> %a0, <4 x float> %a1)
-  %2 = load <4 x float>, <4 x float> *%a2, align 16
-  %3 = call <4 x float> @llvm.x86.sse3.addsub.ps(<4 x float> %1, <4 x float> %2)
-  ret <4 x float> %3
-}
-declare <4 x float> @llvm.x86.sse3.addsub.ps(<4 x float>, <4 x float>) nounwind readnone
-
-define <2 x double> @test_haddpd(<2 x double> %a0, <2 x double> %a1, <2 x double> *%a2) {
-; GENERIC-LABEL: test_haddpd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    haddpd %xmm1, %xmm0 # sched: [5:2.00]
-; GENERIC-NEXT:    haddpd (%rdi), %xmm0 # sched: [11:2.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_haddpd:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    haddpd %xmm1, %xmm0 # sched: [8:4.00]
-; ATOM-NEXT:    haddpd (%rdi), %xmm0 # sched: [9:4.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_haddpd:
-; SLM:       # %bb.0:
-; SLM-NEXT:    haddpd %xmm1, %xmm0 # sched: [3:1.00]
-; SLM-NEXT:    haddpd (%rdi), %xmm0 # sched: [6:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_haddpd:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    haddpd %xmm1, %xmm0 # sched: [5:2.00]
-; SANDY-SSE-NEXT:    haddpd (%rdi), %xmm0 # sched: [11:2.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_haddpd:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vhaddpd %xmm1, %xmm0, %xmm0 # sched: [5:2.00]
-; SANDY-NEXT:    vhaddpd (%rdi), %xmm0, %xmm0 # sched: [11:2.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_haddpd:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    haddpd %xmm1, %xmm0 # sched: [5:2.00]
-; HASWELL-SSE-NEXT:    haddpd (%rdi), %xmm0 # sched: [11:2.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_haddpd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vhaddpd %xmm1, %xmm0, %xmm0 # sched: [5:2.00]
-; HASWELL-NEXT:    vhaddpd (%rdi), %xmm0, %xmm0 # sched: [11:2.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_haddpd:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    haddpd %xmm1, %xmm0 # sched: [5:2.00]
-; BROADWELL-SSE-NEXT:    haddpd (%rdi), %xmm0 # sched: [10:2.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_haddpd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vhaddpd %xmm1, %xmm0, %xmm0 # sched: [5:2.00]
-; BROADWELL-NEXT:    vhaddpd (%rdi), %xmm0, %xmm0 # sched: [10:2.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_haddpd:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    haddpd %xmm1, %xmm0 # sched: [6:2.00]
-; SKYLAKE-SSE-NEXT:    haddpd (%rdi), %xmm0 # sched: [12:2.00]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_haddpd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vhaddpd %xmm1, %xmm0, %xmm0 # sched: [6:2.00]
-; SKYLAKE-NEXT:    vhaddpd (%rdi), %xmm0, %xmm0 # sched: [12:2.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_haddpd:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    haddpd %xmm1, %xmm0 # sched: [6:2.00]
-; SKX-SSE-NEXT:    haddpd (%rdi), %xmm0 # sched: [12:2.00]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_haddpd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vhaddpd %xmm1, %xmm0, %xmm0 # sched: [6:2.00]
-; SKX-NEXT:    vhaddpd (%rdi), %xmm0, %xmm0 # sched: [12:2.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_haddpd:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    haddpd %xmm1, %xmm0 # sched: [11:1.00]
-; BDVER2-SSE-NEXT:    haddpd (%rdi), %xmm0 # sched: [16:1.00]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_haddpd:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vhaddpd %xmm1, %xmm0, %xmm0 # sched: [11:1.00]
-; BDVER2-NEXT:    vhaddpd (%rdi), %xmm0, %xmm0 # sched: [16:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_haddpd:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    haddpd %xmm1, %xmm0 # sched: [3:1.00]
-; BTVER2-SSE-NEXT:    haddpd (%rdi), %xmm0 # sched: [8:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_haddpd:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vhaddpd %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; BTVER2-NEXT:    vhaddpd (%rdi), %xmm0, %xmm0 # sched: [8:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_haddpd:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    haddpd %xmm1, %xmm0 # sched: [100:0.25]
-; ZNVER1-SSE-NEXT:    haddpd (%rdi), %xmm0 # sched: [100:0.25]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_haddpd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vhaddpd %xmm1, %xmm0, %xmm0 # sched: [100:0.25]
-; ZNVER1-NEXT:    vhaddpd (%rdi), %xmm0, %xmm0 # sched: [100:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <2 x double> @llvm.x86.sse3.hadd.pd(<2 x double> %a0, <2 x double> %a1)
-  %2 = load <2 x double>, <2 x double> *%a2, align 16
-  %3 = call <2 x double> @llvm.x86.sse3.hadd.pd(<2 x double> %1, <2 x double> %2)
-  ret <2 x double> %3
-}
-declare <2 x double> @llvm.x86.sse3.hadd.pd(<2 x double>, <2 x double>) nounwind readnone
-
-define <4 x float> @test_haddps(<4 x float> %a0, <4 x float> %a1, <4 x float> *%a2) {
-; GENERIC-LABEL: test_haddps:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    haddps %xmm1, %xmm0 # sched: [5:2.00]
-; GENERIC-NEXT:    haddps (%rdi), %xmm0 # sched: [11:2.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_haddps:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    haddps %xmm1, %xmm0 # sched: [8:4.00]
-; ATOM-NEXT:    haddps (%rdi), %xmm0 # sched: [9:4.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_haddps:
-; SLM:       # %bb.0:
-; SLM-NEXT:    haddps %xmm1, %xmm0 # sched: [3:1.00]
-; SLM-NEXT:    haddps (%rdi), %xmm0 # sched: [6:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_haddps:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    haddps %xmm1, %xmm0 # sched: [5:2.00]
-; SANDY-SSE-NEXT:    haddps (%rdi), %xmm0 # sched: [11:2.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_haddps:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vhaddps %xmm1, %xmm0, %xmm0 # sched: [5:2.00]
-; SANDY-NEXT:    vhaddps (%rdi), %xmm0, %xmm0 # sched: [11:2.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_haddps:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    haddps %xmm1, %xmm0 # sched: [5:2.00]
-; HASWELL-SSE-NEXT:    haddps (%rdi), %xmm0 # sched: [11:2.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_haddps:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vhaddps %xmm1, %xmm0, %xmm0 # sched: [5:2.00]
-; HASWELL-NEXT:    vhaddps (%rdi), %xmm0, %xmm0 # sched: [11:2.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_haddps:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    haddps %xmm1, %xmm0 # sched: [5:2.00]
-; BROADWELL-SSE-NEXT:    haddps (%rdi), %xmm0 # sched: [10:2.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_haddps:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vhaddps %xmm1, %xmm0, %xmm0 # sched: [5:2.00]
-; BROADWELL-NEXT:    vhaddps (%rdi), %xmm0, %xmm0 # sched: [10:2.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_haddps:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    haddps %xmm1, %xmm0 # sched: [6:2.00]
-; SKYLAKE-SSE-NEXT:    haddps (%rdi), %xmm0 # sched: [12:2.00]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_haddps:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vhaddps %xmm1, %xmm0, %xmm0 # sched: [6:2.00]
-; SKYLAKE-NEXT:    vhaddps (%rdi), %xmm0, %xmm0 # sched: [12:2.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_haddps:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    haddps %xmm1, %xmm0 # sched: [6:2.00]
-; SKX-SSE-NEXT:    haddps (%rdi), %xmm0 # sched: [12:2.00]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_haddps:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vhaddps %xmm1, %xmm0, %xmm0 # sched: [6:2.00]
-; SKX-NEXT:    vhaddps (%rdi), %xmm0, %xmm0 # sched: [12:2.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_haddps:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    haddps %xmm1, %xmm0 # sched: [11:1.00]
-; BDVER2-SSE-NEXT:    haddps (%rdi), %xmm0 # sched: [16:1.00]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_haddps:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vhaddps %xmm1, %xmm0, %xmm0 # sched: [11:1.00]
-; BDVER2-NEXT:    vhaddps (%rdi), %xmm0, %xmm0 # sched: [16:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_haddps:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    haddps %xmm1, %xmm0 # sched: [3:1.00]
-; BTVER2-SSE-NEXT:    haddps (%rdi), %xmm0 # sched: [8:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_haddps:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vhaddps %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; BTVER2-NEXT:    vhaddps (%rdi), %xmm0, %xmm0 # sched: [8:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_haddps:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    haddps %xmm1, %xmm0 # sched: [100:0.25]
-; ZNVER1-SSE-NEXT:    haddps (%rdi), %xmm0 # sched: [100:0.25]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_haddps:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vhaddps %xmm1, %xmm0, %xmm0 # sched: [100:0.25]
-; ZNVER1-NEXT:    vhaddps (%rdi), %xmm0, %xmm0 # sched: [100:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <4 x float> @llvm.x86.sse3.hadd.ps(<4 x float> %a0, <4 x float> %a1)
-  %2 = load <4 x float>, <4 x float> *%a2, align 16
-  %3 = call <4 x float> @llvm.x86.sse3.hadd.ps(<4 x float> %1, <4 x float> %2)
-  ret <4 x float> %3
-}
-declare <4 x float> @llvm.x86.sse3.hadd.ps(<4 x float>, <4 x float>) nounwind readnone
-
-define <2 x double> @test_hsubpd(<2 x double> %a0, <2 x double> %a1, <2 x double> *%a2) {
-; GENERIC-LABEL: test_hsubpd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    hsubpd %xmm1, %xmm0 # sched: [5:2.00]
-; GENERIC-NEXT:    hsubpd (%rdi), %xmm0 # sched: [11:2.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_hsubpd:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    hsubpd %xmm1, %xmm0 # sched: [8:4.00]
-; ATOM-NEXT:    hsubpd (%rdi), %xmm0 # sched: [9:4.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_hsubpd:
-; SLM:       # %bb.0:
-; SLM-NEXT:    hsubpd %xmm1, %xmm0 # sched: [3:1.00]
-; SLM-NEXT:    hsubpd (%rdi), %xmm0 # sched: [6:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_hsubpd:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    hsubpd %xmm1, %xmm0 # sched: [5:2.00]
-; SANDY-SSE-NEXT:    hsubpd (%rdi), %xmm0 # sched: [11:2.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_hsubpd:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vhsubpd %xmm1, %xmm0, %xmm0 # sched: [5:2.00]
-; SANDY-NEXT:    vhsubpd (%rdi), %xmm0, %xmm0 # sched: [11:2.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_hsubpd:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    hsubpd %xmm1, %xmm0 # sched: [5:2.00]
-; HASWELL-SSE-NEXT:    hsubpd (%rdi), %xmm0 # sched: [11:2.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_hsubpd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vhsubpd %xmm1, %xmm0, %xmm0 # sched: [5:2.00]
-; HASWELL-NEXT:    vhsubpd (%rdi), %xmm0, %xmm0 # sched: [11:2.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_hsubpd:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    hsubpd %xmm1, %xmm0 # sched: [5:2.00]
-; BROADWELL-SSE-NEXT:    hsubpd (%rdi), %xmm0 # sched: [10:2.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_hsubpd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vhsubpd %xmm1, %xmm0, %xmm0 # sched: [5:2.00]
-; BROADWELL-NEXT:    vhsubpd (%rdi), %xmm0, %xmm0 # sched: [10:2.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_hsubpd:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    hsubpd %xmm1, %xmm0 # sched: [6:2.00]
-; SKYLAKE-SSE-NEXT:    hsubpd (%rdi), %xmm0 # sched: [12:2.00]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_hsubpd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vhsubpd %xmm1, %xmm0, %xmm0 # sched: [6:2.00]
-; SKYLAKE-NEXT:    vhsubpd (%rdi), %xmm0, %xmm0 # sched: [12:2.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_hsubpd:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    hsubpd %xmm1, %xmm0 # sched: [6:2.00]
-; SKX-SSE-NEXT:    hsubpd (%rdi), %xmm0 # sched: [12:2.00]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_hsubpd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vhsubpd %xmm1, %xmm0, %xmm0 # sched: [6:2.00]
-; SKX-NEXT:    vhsubpd (%rdi), %xmm0, %xmm0 # sched: [12:2.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_hsubpd:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    hsubpd %xmm1, %xmm0 # sched: [11:1.00]
-; BDVER2-SSE-NEXT:    hsubpd (%rdi), %xmm0 # sched: [16:1.00]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_hsubpd:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vhsubpd %xmm1, %xmm0, %xmm0 # sched: [11:1.00]
-; BDVER2-NEXT:    vhsubpd (%rdi), %xmm0, %xmm0 # sched: [16:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_hsubpd:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    hsubpd %xmm1, %xmm0 # sched: [3:1.00]
-; BTVER2-SSE-NEXT:    hsubpd (%rdi), %xmm0 # sched: [8:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_hsubpd:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vhsubpd %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; BTVER2-NEXT:    vhsubpd (%rdi), %xmm0, %xmm0 # sched: [8:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_hsubpd:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    hsubpd %xmm1, %xmm0 # sched: [100:0.25]
-; ZNVER1-SSE-NEXT:    hsubpd (%rdi), %xmm0 # sched: [100:0.25]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_hsubpd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vhsubpd %xmm1, %xmm0, %xmm0 # sched: [100:0.25]
-; ZNVER1-NEXT:    vhsubpd (%rdi), %xmm0, %xmm0 # sched: [100:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <2 x double> @llvm.x86.sse3.hsub.pd(<2 x double> %a0, <2 x double> %a1)
-  %2 = load <2 x double>, <2 x double> *%a2, align 16
-  %3 = call <2 x double> @llvm.x86.sse3.hsub.pd(<2 x double> %1, <2 x double> %2)
-  ret <2 x double> %3
-}
-declare <2 x double> @llvm.x86.sse3.hsub.pd(<2 x double>, <2 x double>) nounwind readnone
-
-define <4 x float> @test_hsubps(<4 x float> %a0, <4 x float> %a1, <4 x float> *%a2) {
-; GENERIC-LABEL: test_hsubps:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    hsubps %xmm1, %xmm0 # sched: [5:2.00]
-; GENERIC-NEXT:    hsubps (%rdi), %xmm0 # sched: [11:2.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_hsubps:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    hsubps %xmm1, %xmm0 # sched: [8:4.00]
-; ATOM-NEXT:    hsubps (%rdi), %xmm0 # sched: [9:4.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_hsubps:
-; SLM:       # %bb.0:
-; SLM-NEXT:    hsubps %xmm1, %xmm0 # sched: [3:1.00]
-; SLM-NEXT:    hsubps (%rdi), %xmm0 # sched: [6:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_hsubps:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    hsubps %xmm1, %xmm0 # sched: [5:2.00]
-; SANDY-SSE-NEXT:    hsubps (%rdi), %xmm0 # sched: [11:2.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_hsubps:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vhsubps %xmm1, %xmm0, %xmm0 # sched: [5:2.00]
-; SANDY-NEXT:    vhsubps (%rdi), %xmm0, %xmm0 # sched: [11:2.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_hsubps:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    hsubps %xmm1, %xmm0 # sched: [5:2.00]
-; HASWELL-SSE-NEXT:    hsubps (%rdi), %xmm0 # sched: [11:2.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_hsubps:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vhsubps %xmm1, %xmm0, %xmm0 # sched: [5:2.00]
-; HASWELL-NEXT:    vhsubps (%rdi), %xmm0, %xmm0 # sched: [11:2.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_hsubps:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    hsubps %xmm1, %xmm0 # sched: [5:2.00]
-; BROADWELL-SSE-NEXT:    hsubps (%rdi), %xmm0 # sched: [10:2.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_hsubps:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vhsubps %xmm1, %xmm0, %xmm0 # sched: [5:2.00]
-; BROADWELL-NEXT:    vhsubps (%rdi), %xmm0, %xmm0 # sched: [10:2.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_hsubps:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    hsubps %xmm1, %xmm0 # sched: [6:2.00]
-; SKYLAKE-SSE-NEXT:    hsubps (%rdi), %xmm0 # sched: [12:2.00]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_hsubps:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vhsubps %xmm1, %xmm0, %xmm0 # sched: [6:2.00]
-; SKYLAKE-NEXT:    vhsubps (%rdi), %xmm0, %xmm0 # sched: [12:2.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_hsubps:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    hsubps %xmm1, %xmm0 # sched: [6:2.00]
-; SKX-SSE-NEXT:    hsubps (%rdi), %xmm0 # sched: [12:2.00]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_hsubps:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vhsubps %xmm1, %xmm0, %xmm0 # sched: [6:2.00]
-; SKX-NEXT:    vhsubps (%rdi), %xmm0, %xmm0 # sched: [12:2.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_hsubps:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    hsubps %xmm1, %xmm0 # sched: [11:1.00]
-; BDVER2-SSE-NEXT:    hsubps (%rdi), %xmm0 # sched: [16:1.00]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_hsubps:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vhsubps %xmm1, %xmm0, %xmm0 # sched: [11:1.00]
-; BDVER2-NEXT:    vhsubps (%rdi), %xmm0, %xmm0 # sched: [16:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_hsubps:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    hsubps %xmm1, %xmm0 # sched: [3:1.00]
-; BTVER2-SSE-NEXT:    hsubps (%rdi), %xmm0 # sched: [8:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_hsubps:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vhsubps %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; BTVER2-NEXT:    vhsubps (%rdi), %xmm0, %xmm0 # sched: [8:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_hsubps:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    hsubps %xmm1, %xmm0 # sched: [100:0.25]
-; ZNVER1-SSE-NEXT:    hsubps (%rdi), %xmm0 # sched: [100:0.25]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_hsubps:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vhsubps %xmm1, %xmm0, %xmm0 # sched: [100:0.25]
-; ZNVER1-NEXT:    vhsubps (%rdi), %xmm0, %xmm0 # sched: [100:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <4 x float> @llvm.x86.sse3.hsub.ps(<4 x float> %a0, <4 x float> %a1)
-  %2 = load <4 x float>, <4 x float> *%a2, align 16
-  %3 = call <4 x float> @llvm.x86.sse3.hsub.ps(<4 x float> %1, <4 x float> %2)
-  ret <4 x float> %3
-}
-declare <4 x float> @llvm.x86.sse3.hsub.ps(<4 x float>, <4 x float>) nounwind readnone
-
-define <16 x i8> @test_lddqu(i8* %a0) {
-; GENERIC-LABEL: test_lddqu:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    lddqu (%rdi), %xmm0 # sched: [6:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_lddqu:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    lddqu (%rdi), %xmm0 # sched: [3:1.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_lddqu:
-; SLM:       # %bb.0:
-; SLM-NEXT:    lddqu (%rdi), %xmm0 # sched: [3:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_lddqu:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    lddqu (%rdi), %xmm0 # sched: [6:0.50]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_lddqu:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vlddqu (%rdi), %xmm0 # sched: [6:0.50]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_lddqu:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    lddqu (%rdi), %xmm0 # sched: [6:0.50]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_lddqu:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vlddqu (%rdi), %xmm0 # sched: [6:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_lddqu:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    lddqu (%rdi), %xmm0 # sched: [5:0.50]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_lddqu:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vlddqu (%rdi), %xmm0 # sched: [5:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_lddqu:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    lddqu (%rdi), %xmm0 # sched: [6:0.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_lddqu:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vlddqu (%rdi), %xmm0 # sched: [6:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_lddqu:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    lddqu (%rdi), %xmm0 # sched: [6:0.50]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_lddqu:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vlddqu (%rdi), %xmm0 # sched: [6:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_lddqu:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    lddqu (%rdi), %xmm0 # sched: [5:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_lddqu:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vlddqu (%rdi), %xmm0 # sched: [5:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_lddqu:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    lddqu (%rdi), %xmm0 # sched: [5:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_lddqu:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vlddqu (%rdi), %xmm0 # sched: [5:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_lddqu:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    lddqu (%rdi), %xmm0 # sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_lddqu:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vlddqu (%rdi), %xmm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <16 x i8> @llvm.x86.sse3.ldu.dq(i8* %a0)
-  ret <16 x i8> %1
-}
-declare <16 x i8> @llvm.x86.sse3.ldu.dq(i8*) nounwind readonly
-
-define void @test_monitor(i8* %a0, i32 %a1, i32 %a2) {
-; GENERIC-LABEL: test_monitor:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    movl %esi, %ecx # sched: [1:0.33]
-; GENERIC-NEXT:    leaq (%rdi), %rax # sched: [1:0.50]
-; GENERIC-NEXT:    monitor # sched: [100:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_monitor:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    movl %esi, %ecx # sched: [1:0.50]
-; ATOM-NEXT:    leaq (%rdi), %rax # sched: [1:1.00]
-; ATOM-NEXT:    monitor # sched: [45:22.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_monitor:
-; SLM:       # %bb.0:
-; SLM-NEXT:    movl %esi, %ecx # sched: [1:0.50]
-; SLM-NEXT:    leaq (%rdi), %rax # sched: [1:1.00]
-; SLM-NEXT:    monitor # sched: [100:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_monitor:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    movl %esi, %ecx # sched: [1:0.33]
-; SANDY-SSE-NEXT:    leaq (%rdi), %rax # sched: [1:0.50]
-; SANDY-SSE-NEXT:    monitor # sched: [100:0.33]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_monitor:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    movl %esi, %ecx # sched: [1:0.33]
-; SANDY-NEXT:    leaq (%rdi), %rax # sched: [1:0.50]
-; SANDY-NEXT:    monitor # sched: [100:0.33]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_monitor:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    movl %esi, %ecx # sched: [1:0.25]
-; HASWELL-SSE-NEXT:    leaq (%rdi), %rax # sched: [1:0.50]
-; HASWELL-SSE-NEXT:    monitor # sched: [100:0.25]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_monitor:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    movl %esi, %ecx # sched: [1:0.25]
-; HASWELL-NEXT:    leaq (%rdi), %rax # sched: [1:0.50]
-; HASWELL-NEXT:    monitor # sched: [100:0.25]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_monitor:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    movl %esi, %ecx # sched: [1:0.25]
-; BROADWELL-SSE-NEXT:    leaq (%rdi), %rax # sched: [1:0.50]
-; BROADWELL-SSE-NEXT:    monitor # sched: [100:0.25]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_monitor:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    movl %esi, %ecx # sched: [1:0.25]
-; BROADWELL-NEXT:    leaq (%rdi), %rax # sched: [1:0.50]
-; BROADWELL-NEXT:    monitor # sched: [100:0.25]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_monitor:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    movl %esi, %ecx # sched: [1:0.25]
-; SKYLAKE-SSE-NEXT:    leaq (%rdi), %rax # sched: [1:0.50]
-; SKYLAKE-SSE-NEXT:    monitor # sched: [100:0.25]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_monitor:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    movl %esi, %ecx # sched: [1:0.25]
-; SKYLAKE-NEXT:    leaq (%rdi), %rax # sched: [1:0.50]
-; SKYLAKE-NEXT:    monitor # sched: [100:0.25]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_monitor:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    movl %esi, %ecx # sched: [1:0.25]
-; SKX-SSE-NEXT:    leaq (%rdi), %rax # sched: [1:0.50]
-; SKX-SSE-NEXT:    monitor # sched: [100:0.25]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_monitor:
-; SKX:       # %bb.0:
-; SKX-NEXT:    movl %esi, %ecx # sched: [1:0.25]
-; SKX-NEXT:    leaq (%rdi), %rax # sched: [1:0.50]
-; SKX-NEXT:    monitor # sched: [100:0.25]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_monitor:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    leaq (%rdi), %rax # sched: [1:0.50]
-; BDVER2-SSE-NEXT:    movl %esi, %ecx # sched: [1:0.50]
-; BDVER2-SSE-NEXT:    monitor # sched: [100:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_monitor:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    leaq (%rdi), %rax # sched: [1:0.50]
-; BDVER2-NEXT:    movl %esi, %ecx # sched: [1:0.50]
-; BDVER2-NEXT:    monitor # sched: [100:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_monitor:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    movl %esi, %ecx # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    leaq (%rdi), %rax # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    monitor # sched: [100:0.50]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_monitor:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    movl %esi, %ecx # sched: [1:0.50]
-; BTVER2-NEXT:    leaq (%rdi), %rax # sched: [1:0.50]
-; BTVER2-NEXT:    monitor # sched: [100:0.50]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_monitor:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    movl %esi, %ecx # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    leaq (%rdi), %rax # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    monitor # sched: [100:0.25]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_monitor:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    movl %esi, %ecx # sched: [1:0.25]
-; ZNVER1-NEXT:    leaq (%rdi), %rax # sched: [1:0.25]
-; ZNVER1-NEXT:    monitor # sched: [100:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  tail call void @llvm.x86.sse3.monitor(i8* %a0, i32 %a1, i32 %a2)
-  ret void
-}
-declare void @llvm.x86.sse3.monitor(i8*, i32, i32)
-
-define <2 x double> @test_movddup(<2 x double> %a0, <2 x double> *%a1) {
-; GENERIC-LABEL: test_movddup:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    movddup {{.*#+}} xmm1 = xmm0[0,0] sched: [1:1.00]
-; GENERIC-NEXT:    movddup {{.*#+}} xmm0 = mem[0,0] sched: [6:0.50]
-; GENERIC-NEXT:    subpd %xmm1, %xmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_movddup:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    movddup {{.*#+}} xmm1 = xmm0[0,0] sched: [1:1.00]
-; ATOM-NEXT:    movddup {{.*#+}} xmm0 = mem[0,0] sched: [1:1.00]
-; ATOM-NEXT:    subpd %xmm1, %xmm0 # sched: [6:3.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_movddup:
-; SLM:       # %bb.0:
-; SLM-NEXT:    movddup {{.*#+}} xmm1 = mem[0,0] sched: [4:1.00]
-; SLM-NEXT:    movddup {{.*#+}} xmm0 = xmm0[0,0] sched: [1:1.00]
-; SLM-NEXT:    subpd %xmm0, %xmm1 # sched: [3:1.00]
-; SLM-NEXT:    movapd %xmm1, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_movddup:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    movddup {{.*#+}} xmm1 = xmm0[0,0] sched: [1:1.00]
-; SANDY-SSE-NEXT:    movddup {{.*#+}} xmm0 = mem[0,0] sched: [6:0.50]
-; SANDY-SSE-NEXT:    subpd %xmm1, %xmm0 # sched: [3:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_movddup:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vmovddup {{.*#+}} xmm0 = xmm0[0,0] sched: [1:1.00]
-; SANDY-NEXT:    vmovddup {{.*#+}} xmm1 = mem[0,0] sched: [6:0.50]
-; SANDY-NEXT:    vsubpd %xmm0, %xmm1, %xmm0 # sched: [3:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_movddup:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    movddup {{.*#+}} xmm1 = xmm0[0,0] sched: [1:1.00]
-; HASWELL-SSE-NEXT:    movddup {{.*#+}} xmm0 = mem[0,0] sched: [5:0.50]
-; HASWELL-SSE-NEXT:    subpd %xmm1, %xmm0 # sched: [3:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_movddup:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vmovddup {{.*#+}} xmm0 = xmm0[0,0] sched: [1:1.00]
-; HASWELL-NEXT:    vmovddup {{.*#+}} xmm1 = mem[0,0] sched: [5:0.50]
-; HASWELL-NEXT:    vsubpd %xmm0, %xmm1, %xmm0 # sched: [3:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_movddup:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    movddup {{.*#+}} xmm1 = xmm0[0,0] sched: [1:1.00]
-; BROADWELL-SSE-NEXT:    movddup {{.*#+}} xmm0 = mem[0,0] sched: [5:0.50]
-; BROADWELL-SSE-NEXT:    subpd %xmm1, %xmm0 # sched: [3:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_movddup:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vmovddup {{.*#+}} xmm0 = xmm0[0,0] sched: [1:1.00]
-; BROADWELL-NEXT:    vmovddup {{.*#+}} xmm1 = mem[0,0] sched: [5:0.50]
-; BROADWELL-NEXT:    vsubpd %xmm0, %xmm1, %xmm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_movddup:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    movddup {{.*#+}} xmm1 = xmm0[0,0] sched: [1:1.00]
-; SKYLAKE-SSE-NEXT:    movddup {{.*#+}} xmm0 = mem[0,0] sched: [5:0.50]
-; SKYLAKE-SSE-NEXT:    subpd %xmm1, %xmm0 # sched: [4:0.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_movddup:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vmovddup {{.*#+}} xmm0 = xmm0[0,0] sched: [1:1.00]
-; SKYLAKE-NEXT:    vmovddup {{.*#+}} xmm1 = mem[0,0] sched: [5:0.50]
-; SKYLAKE-NEXT:    vsubpd %xmm0, %xmm1, %xmm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_movddup:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    movddup {{.*#+}} xmm1 = xmm0[0,0] sched: [1:1.00]
-; SKX-SSE-NEXT:    movddup {{.*#+}} xmm0 = mem[0,0] sched: [5:0.50]
-; SKX-SSE-NEXT:    subpd %xmm1, %xmm0 # sched: [4:0.50]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_movddup:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovddup {{.*#+}} xmm0 = xmm0[0,0] sched: [1:1.00]
-; SKX-NEXT:    vmovddup {{.*#+}} xmm1 = mem[0,0] sched: [5:0.50]
-; SKX-NEXT:    vsubpd %xmm0, %xmm1, %xmm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_movddup:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    movddup {{.*#+}} xmm1 = xmm0[0,0] sched: [2:0.50]
-; BDVER2-SSE-NEXT:    movddup {{.*#+}} xmm0 = mem[0,0] sched: [7:0.50]
-; BDVER2-SSE-NEXT:    subpd %xmm1, %xmm0 # sched: [5:1.00]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_movddup:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vmovddup {{.*#+}} xmm1 = mem[0,0] sched: [7:0.50]
-; BDVER2-NEXT:    vmovddup {{.*#+}} xmm0 = xmm0[0,0] sched: [2:0.50]
-; BDVER2-NEXT:    vsubpd %xmm0, %xmm1, %xmm0 # sched: [5:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_movddup:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    movddup {{.*#+}} xmm1 = xmm0[0,0] sched: [1:0.50]
-; BTVER2-SSE-NEXT:    movddup {{.*#+}} xmm0 = mem[0,0] sched: [6:1.00]
-; BTVER2-SSE-NEXT:    subpd %xmm1, %xmm0 # sched: [3:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_movddup:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vmovddup {{.*#+}} xmm1 = mem[0,0] sched: [6:1.00]
-; BTVER2-NEXT:    vmovddup {{.*#+}} xmm0 = xmm0[0,0] sched: [1:0.50]
-; BTVER2-NEXT:    vsubpd %xmm0, %xmm1, %xmm0 # sched: [3:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_movddup:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    movddup {{.*#+}} xmm1 = xmm0[0,0] sched: [1:0.50]
-; ZNVER1-SSE-NEXT:    movddup {{.*#+}} xmm0 = mem[0,0] sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    subpd %xmm1, %xmm0 # sched: [3:1.00]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_movddup:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vmovddup {{.*#+}} xmm1 = mem[0,0] sched: [8:0.50]
-; ZNVER1-NEXT:    vmovddup {{.*#+}} xmm0 = xmm0[0,0] sched: [1:0.50]
-; ZNVER1-NEXT:    vsubpd %xmm0, %xmm1, %xmm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = shufflevector <2 x double> %a0, <2 x double> undef, <2 x i32> zeroinitializer
-  %2 = load <2 x double>, <2 x double> *%a1, align 16
-  %3 = shufflevector <2 x double> %2, <2 x double> undef, <2 x i32> zeroinitializer
-  %4 = fsub <2 x double> %3, %1 ; Use fsub to stop the movddup from being folded as a broadcast load in avx512vl.
-  ret <2 x double> %4
-}
-
-define <4 x float> @test_movshdup(<4 x float> %a0, <4 x float> *%a1) {
-; GENERIC-LABEL: test_movshdup:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    movshdup {{.*#+}} xmm1 = xmm0[1,1,3,3] sched: [1:1.00]
-; GENERIC-NEXT:    movshdup {{.*#+}} xmm0 = mem[1,1,3,3] sched: [6:0.50]
-; GENERIC-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_movshdup:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    movshdup {{.*#+}} xmm1 = xmm0[1,1,3,3] sched: [1:1.00]
-; ATOM-NEXT:    movshdup {{.*#+}} xmm0 = mem[1,1,3,3] sched: [1:1.00]
-; ATOM-NEXT:    addps %xmm1, %xmm0 # sched: [5:5.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_movshdup:
-; SLM:       # %bb.0:
-; SLM-NEXT:    movshdup {{.*#+}} xmm1 = mem[1,1,3,3] sched: [4:1.00]
-; SLM-NEXT:    movshdup {{.*#+}} xmm0 = xmm0[1,1,3,3] sched: [1:1.00]
-; SLM-NEXT:    addps %xmm0, %xmm1 # sched: [3:1.00]
-; SLM-NEXT:    movaps %xmm1, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_movshdup:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    movshdup {{.*#+}} xmm1 = xmm0[1,1,3,3] sched: [1:1.00]
-; SANDY-SSE-NEXT:    movshdup {{.*#+}} xmm0 = mem[1,1,3,3] sched: [6:0.50]
-; SANDY-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_movshdup:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vmovshdup {{.*#+}} xmm0 = xmm0[1,1,3,3] sched: [1:1.00]
-; SANDY-NEXT:    vmovshdup {{.*#+}} xmm1 = mem[1,1,3,3] sched: [6:0.50]
-; SANDY-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_movshdup:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    movshdup {{.*#+}} xmm1 = xmm0[1,1,3,3] sched: [1:1.00]
-; HASWELL-SSE-NEXT:    movshdup {{.*#+}} xmm0 = mem[1,1,3,3] sched: [6:0.50]
-; HASWELL-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_movshdup:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vmovshdup {{.*#+}} xmm0 = xmm0[1,1,3,3] sched: [1:1.00]
-; HASWELL-NEXT:    vmovshdup {{.*#+}} xmm1 = mem[1,1,3,3] sched: [6:0.50]
-; HASWELL-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_movshdup:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    movshdup {{.*#+}} xmm1 = xmm0[1,1,3,3] sched: [1:1.00]
-; BROADWELL-SSE-NEXT:    movshdup {{.*#+}} xmm0 = mem[1,1,3,3] sched: [5:0.50]
-; BROADWELL-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_movshdup:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vmovshdup {{.*#+}} xmm0 = xmm0[1,1,3,3] sched: [1:1.00]
-; BROADWELL-NEXT:    vmovshdup {{.*#+}} xmm1 = mem[1,1,3,3] sched: [5:0.50]
-; BROADWELL-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_movshdup:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    movshdup {{.*#+}} xmm1 = xmm0[1,1,3,3] sched: [1:1.00]
-; SKYLAKE-SSE-NEXT:    movshdup {{.*#+}} xmm0 = mem[1,1,3,3] sched: [6:0.50]
-; SKYLAKE-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [4:0.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_movshdup:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vmovshdup {{.*#+}} xmm0 = xmm0[1,1,3,3] sched: [1:1.00]
-; SKYLAKE-NEXT:    vmovshdup {{.*#+}} xmm1 = mem[1,1,3,3] sched: [6:0.50]
-; SKYLAKE-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_movshdup:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    movshdup {{.*#+}} xmm1 = xmm0[1,1,3,3] sched: [1:1.00]
-; SKX-SSE-NEXT:    movshdup {{.*#+}} xmm0 = mem[1,1,3,3] sched: [6:0.50]
-; SKX-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [4:0.50]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_movshdup:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovshdup {{.*#+}} xmm0 = xmm0[1,1,3,3] sched: [1:1.00]
-; SKX-NEXT:    vmovshdup {{.*#+}} xmm1 = mem[1,1,3,3] sched: [6:0.50]
-; SKX-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_movshdup:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    movshdup {{.*#+}} xmm1 = xmm0[1,1,3,3] sched: [2:0.50]
-; BDVER2-SSE-NEXT:    movshdup {{.*#+}} xmm0 = mem[1,1,3,3] sched: [7:0.50]
-; BDVER2-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [5:1.00]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_movshdup:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vmovshdup {{.*#+}} xmm1 = mem[1,1,3,3] sched: [7:0.50]
-; BDVER2-NEXT:    vmovshdup {{.*#+}} xmm0 = xmm0[1,1,3,3] sched: [2:0.50]
-; BDVER2-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [5:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_movshdup:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    movshdup {{.*#+}} xmm1 = xmm0[1,1,3,3] sched: [1:0.50]
-; BTVER2-SSE-NEXT:    movshdup {{.*#+}} xmm0 = mem[1,1,3,3] sched: [6:1.00]
-; BTVER2-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_movshdup:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vmovshdup {{.*#+}} xmm1 = mem[1,1,3,3] sched: [6:1.00]
-; BTVER2-NEXT:    vmovshdup {{.*#+}} xmm0 = xmm0[1,1,3,3] sched: [1:0.50]
-; BTVER2-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_movshdup:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    movshdup {{.*#+}} xmm1 = xmm0[1,1,3,3] sched: [1:0.50]
-; ZNVER1-SSE-NEXT:    movshdup {{.*#+}} xmm0 = mem[1,1,3,3] sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_movshdup:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vmovshdup {{.*#+}} xmm1 = mem[1,1,3,3] sched: [8:0.50]
-; ZNVER1-NEXT:    vmovshdup {{.*#+}} xmm0 = xmm0[1,1,3,3] sched: [1:0.50]
-; ZNVER1-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = shufflevector <4 x float> %a0, <4 x float> undef, <4 x i32> <i32 1, i32 1, i32 3, i32 3>
-  %2 = load <4 x float>, <4 x float> *%a1, align 16
-  %3 = shufflevector <4 x float> %2, <4 x float> undef, <4 x i32> <i32 1, i32 1, i32 3, i32 3>
-  %4 = fadd <4 x float> %1, %3
-  ret <4 x float> %4
-}
-
-define <4 x float> @test_movsldup(<4 x float> %a0, <4 x float> *%a1) {
-; GENERIC-LABEL: test_movsldup:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    movsldup {{.*#+}} xmm1 = xmm0[0,0,2,2] sched: [1:1.00]
-; GENERIC-NEXT:    movsldup {{.*#+}} xmm0 = mem[0,0,2,2] sched: [6:0.50]
-; GENERIC-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_movsldup:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    movsldup {{.*#+}} xmm1 = xmm0[0,0,2,2] sched: [1:1.00]
-; ATOM-NEXT:    movsldup {{.*#+}} xmm0 = mem[0,0,2,2] sched: [1:1.00]
-; ATOM-NEXT:    addps %xmm1, %xmm0 # sched: [5:5.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_movsldup:
-; SLM:       # %bb.0:
-; SLM-NEXT:    movsldup {{.*#+}} xmm1 = mem[0,0,2,2] sched: [4:1.00]
-; SLM-NEXT:    movsldup {{.*#+}} xmm0 = xmm0[0,0,2,2] sched: [1:1.00]
-; SLM-NEXT:    addps %xmm0, %xmm1 # sched: [3:1.00]
-; SLM-NEXT:    movaps %xmm1, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_movsldup:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    movsldup {{.*#+}} xmm1 = xmm0[0,0,2,2] sched: [1:1.00]
-; SANDY-SSE-NEXT:    movsldup {{.*#+}} xmm0 = mem[0,0,2,2] sched: [6:0.50]
-; SANDY-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_movsldup:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vmovsldup {{.*#+}} xmm0 = xmm0[0,0,2,2] sched: [1:1.00]
-; SANDY-NEXT:    vmovsldup {{.*#+}} xmm1 = mem[0,0,2,2] sched: [6:0.50]
-; SANDY-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_movsldup:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    movsldup {{.*#+}} xmm1 = xmm0[0,0,2,2] sched: [1:1.00]
-; HASWELL-SSE-NEXT:    movsldup {{.*#+}} xmm0 = mem[0,0,2,2] sched: [6:0.50]
-; HASWELL-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_movsldup:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vmovsldup {{.*#+}} xmm0 = xmm0[0,0,2,2] sched: [1:1.00]
-; HASWELL-NEXT:    vmovsldup {{.*#+}} xmm1 = mem[0,0,2,2] sched: [6:0.50]
-; HASWELL-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_movsldup:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    movsldup {{.*#+}} xmm1 = xmm0[0,0,2,2] sched: [1:1.00]
-; BROADWELL-SSE-NEXT:    movsldup {{.*#+}} xmm0 = mem[0,0,2,2] sched: [5:0.50]
-; BROADWELL-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_movsldup:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vmovsldup {{.*#+}} xmm0 = xmm0[0,0,2,2] sched: [1:1.00]
-; BROADWELL-NEXT:    vmovsldup {{.*#+}} xmm1 = mem[0,0,2,2] sched: [5:0.50]
-; BROADWELL-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_movsldup:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    movsldup {{.*#+}} xmm1 = xmm0[0,0,2,2] sched: [1:1.00]
-; SKYLAKE-SSE-NEXT:    movsldup {{.*#+}} xmm0 = mem[0,0,2,2] sched: [6:0.50]
-; SKYLAKE-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [4:0.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_movsldup:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vmovsldup {{.*#+}} xmm0 = xmm0[0,0,2,2] sched: [1:1.00]
-; SKYLAKE-NEXT:    vmovsldup {{.*#+}} xmm1 = mem[0,0,2,2] sched: [6:0.50]
-; SKYLAKE-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_movsldup:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    movsldup {{.*#+}} xmm1 = xmm0[0,0,2,2] sched: [1:1.00]
-; SKX-SSE-NEXT:    movsldup {{.*#+}} xmm0 = mem[0,0,2,2] sched: [6:0.50]
-; SKX-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [4:0.50]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_movsldup:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovsldup {{.*#+}} xmm0 = xmm0[0,0,2,2] sched: [1:1.00]
-; SKX-NEXT:    vmovsldup {{.*#+}} xmm1 = mem[0,0,2,2] sched: [6:0.50]
-; SKX-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_movsldup:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    movsldup {{.*#+}} xmm1 = xmm0[0,0,2,2] sched: [2:0.50]
-; BDVER2-SSE-NEXT:    movsldup {{.*#+}} xmm0 = mem[0,0,2,2] sched: [7:0.50]
-; BDVER2-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [5:1.00]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_movsldup:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vmovsldup {{.*#+}} xmm1 = mem[0,0,2,2] sched: [7:0.50]
-; BDVER2-NEXT:    vmovsldup {{.*#+}} xmm0 = xmm0[0,0,2,2] sched: [2:0.50]
-; BDVER2-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [5:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_movsldup:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    movsldup {{.*#+}} xmm1 = xmm0[0,0,2,2] sched: [1:0.50]
-; BTVER2-SSE-NEXT:    movsldup {{.*#+}} xmm0 = mem[0,0,2,2] sched: [6:1.00]
-; BTVER2-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_movsldup:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vmovsldup {{.*#+}} xmm1 = mem[0,0,2,2] sched: [6:1.00]
-; BTVER2-NEXT:    vmovsldup {{.*#+}} xmm0 = xmm0[0,0,2,2] sched: [1:0.50]
-; BTVER2-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_movsldup:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    movsldup {{.*#+}} xmm1 = xmm0[0,0,2,2] sched: [100:0.25]
-; ZNVER1-SSE-NEXT:    movsldup {{.*#+}} xmm0 = mem[0,0,2,2] sched: [100:0.25]
-; ZNVER1-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_movsldup:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vmovsldup {{.*#+}} xmm1 = mem[0,0,2,2] sched: [8:0.50]
-; ZNVER1-NEXT:    vmovsldup {{.*#+}} xmm0 = xmm0[0,0,2,2] sched: [1:0.50]
-; ZNVER1-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = shufflevector <4 x float> %a0, <4 x float> undef, <4 x i32> <i32 0, i32 0, i32 2, i32 2>
-  %2 = load <4 x float>, <4 x float> *%a1, align 16
-  %3 = shufflevector <4 x float> %2, <4 x float> undef, <4 x i32> <i32 0, i32 0, i32 2, i32 2>
-  %4 = fadd <4 x float> %1, %3
-  ret <4 x float> %4
-}
-
-define void @test_mwait(i32 %a0, i32 %a1) {
-; GENERIC-LABEL: test_mwait:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    movl %esi, %eax # sched: [1:0.33]
-; GENERIC-NEXT:    movl %edi, %ecx # sched: [1:0.33]
-; GENERIC-NEXT:    mwait # sched: [100:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_mwait:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    movl %esi, %eax # sched: [1:0.50]
-; ATOM-NEXT:    movl %edi, %ecx # sched: [1:0.50]
-; ATOM-NEXT:    mwait # sched: [46:23.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_mwait:
-; SLM:       # %bb.0:
-; SLM-NEXT:    movl %esi, %eax # sched: [1:0.50]
-; SLM-NEXT:    movl %edi, %ecx # sched: [1:0.50]
-; SLM-NEXT:    mwait # sched: [100:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_mwait:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    movl %esi, %eax # sched: [1:0.33]
-; SANDY-SSE-NEXT:    movl %edi, %ecx # sched: [1:0.33]
-; SANDY-SSE-NEXT:    mwait # sched: [100:0.33]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_mwait:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    movl %esi, %eax # sched: [1:0.33]
-; SANDY-NEXT:    movl %edi, %ecx # sched: [1:0.33]
-; SANDY-NEXT:    mwait # sched: [100:0.33]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_mwait:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    movl %esi, %eax # sched: [1:0.25]
-; HASWELL-SSE-NEXT:    movl %edi, %ecx # sched: [1:0.25]
-; HASWELL-SSE-NEXT:    mwait # sched: [20:2.50]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_mwait:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    movl %esi, %eax # sched: [1:0.25]
-; HASWELL-NEXT:    movl %edi, %ecx # sched: [1:0.25]
-; HASWELL-NEXT:    mwait # sched: [20:2.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_mwait:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    movl %esi, %eax # sched: [1:0.25]
-; BROADWELL-SSE-NEXT:    movl %edi, %ecx # sched: [1:0.25]
-; BROADWELL-SSE-NEXT:    mwait # sched: [100:0.25]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_mwait:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    movl %esi, %eax # sched: [1:0.25]
-; BROADWELL-NEXT:    movl %edi, %ecx # sched: [1:0.25]
-; BROADWELL-NEXT:    mwait # sched: [100:0.25]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_mwait:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    movl %esi, %eax # sched: [1:0.25]
-; SKYLAKE-SSE-NEXT:    movl %edi, %ecx # sched: [1:0.25]
-; SKYLAKE-SSE-NEXT:    mwait # sched: [20:2.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_mwait:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    movl %esi, %eax # sched: [1:0.25]
-; SKYLAKE-NEXT:    movl %edi, %ecx # sched: [1:0.25]
-; SKYLAKE-NEXT:    mwait # sched: [20:2.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_mwait:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    movl %esi, %eax # sched: [1:0.25]
-; SKX-SSE-NEXT:    movl %edi, %ecx # sched: [1:0.25]
-; SKX-SSE-NEXT:    mwait # sched: [20:2.50]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_mwait:
-; SKX:       # %bb.0:
-; SKX-NEXT:    movl %esi, %eax # sched: [1:0.25]
-; SKX-NEXT:    movl %edi, %ecx # sched: [1:0.25]
-; SKX-NEXT:    mwait # sched: [20:2.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_mwait:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    movl %esi, %eax # sched: [1:0.50]
-; BDVER2-SSE-NEXT:    movl %edi, %ecx # sched: [1:0.50]
-; BDVER2-SSE-NEXT:    mwait # sched: [100:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_mwait:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    movl %esi, %eax # sched: [1:0.50]
-; BDVER2-NEXT:    movl %edi, %ecx # sched: [1:0.50]
-; BDVER2-NEXT:    mwait # sched: [100:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_mwait:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    movl %esi, %eax # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    movl %edi, %ecx # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    mwait # sched: [100:0.50]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_mwait:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    movl %esi, %eax # sched: [1:0.50]
-; BTVER2-NEXT:    movl %edi, %ecx # sched: [1:0.50]
-; BTVER2-NEXT:    mwait # sched: [100:0.50]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_mwait:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    movl %esi, %eax # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    movl %edi, %ecx # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    mwait # sched: [100:0.25]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_mwait:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    movl %esi, %eax # sched: [1:0.25]
-; ZNVER1-NEXT:    movl %edi, %ecx # sched: [1:0.25]
-; ZNVER1-NEXT:    mwait # sched: [100:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  tail call void @llvm.x86.sse3.mwait(i32 %a0, i32 %a1)
-  ret void
-}
-declare void @llvm.x86.sse3.mwait(i32, i32)
diff --git a/test/CodeGen/X86/sse41-intrinsics-fast-isel.ll b/test/CodeGen/X86/sse41-intrinsics-fast-isel.ll
index dd82bef1d1113b858ab25221861fe46fd6956208..9990ac00eb0543204be641236fcd1f739dc82672 100644
--- a/test/CodeGen/X86/sse41-intrinsics-fast-isel.ll
+++ b/test/CodeGen/X86/sse41-intrinsics-fast-isel.ll
@@ -832,26 +832,11 @@ declare <8 x i16> @llvm.x86.sse41.mpsadbw(<16 x i8>, <16 x i8>, i8) nounwind rea
 define <2 x i64> @test_mm_mul_epi32(<2 x i64> %a0, <2 x i64> %a1) {
 ; SSE-LABEL: test_mm_mul_epi32:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    movdqa %xmm0, %xmm2
-; SSE-NEXT:    psllq $32, %xmm2
-; SSE-NEXT:    psrad $31, %xmm2
-; SSE-NEXT:    pblendw {{.*#+}} xmm2 = xmm0[0,1],xmm2[2,3],xmm0[4,5],xmm2[6,7]
-; SSE-NEXT:    movdqa %xmm1, %xmm0
-; SSE-NEXT:    psllq $32, %xmm0
-; SSE-NEXT:    psrad $31, %xmm0
-; SSE-NEXT:    pblendw {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3],xmm1[4,5],xmm0[6,7]
-; SSE-NEXT:    pmuldq %xmm0, %xmm2
-; SSE-NEXT:    movdqa %xmm2, %xmm0
+; SSE-NEXT:    pmuldq %xmm1, %xmm0
 ; SSE-NEXT:    ret{{[l|q]}}
 ;
 ; AVX1-LABEL: test_mm_mul_epi32:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpsllq $32, %xmm0, %xmm2
-; AVX1-NEXT:    vpsrad $31, %xmm2, %xmm2
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3],xmm0[4,5],xmm2[6,7]
-; AVX1-NEXT:    vpsllq $32, %xmm1, %xmm2
-; AVX1-NEXT:    vpsrad $31, %xmm2, %xmm2
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
 ; AVX1-NEXT:    vpmuldq %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    ret{{[l|q]}}
 ;
diff --git a/test/CodeGen/X86/sse41-schedule.ll b/test/CodeGen/X86/sse41-schedule.ll
deleted file mode 100644
index ea606463fc17b09d26a912d2e3b76c57bec09cfe..0000000000000000000000000000000000000000
--- a/test/CodeGen/X86/sse41-schedule.ll
+++ /dev/null
@@ -1,6248 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=x86-64 -mattr=+sse4.1 | FileCheck %s --check-prefixes=CHECK,GENERIC
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=slm -mattr=-sse4.2 | FileCheck %s --check-prefixes=CHECK,SLM
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=sandybridge -mattr=-sse4.2 | FileCheck %s --check-prefixes=CHECK,SANDY-SSE
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=sandybridge -mattr=-avx2   | FileCheck %s --check-prefixes=CHECK,SANDY
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=ivybridge -mattr=-sse4.2 | FileCheck %s --check-prefixes=CHECK,SANDY-SSE
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=ivybridge -mattr=-avx2   | FileCheck %s --check-prefixes=CHECK,SANDY
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=haswell -mattr=-sse4.2 | FileCheck %s --check-prefixes=CHECK,HASWELL-SSE
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=haswell -mattr=-avx2   | FileCheck %s --check-prefixes=CHECK,HASWELL
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=broadwell -mattr=-sse4.2 | FileCheck %s --check-prefixes=CHECK,BROADWELL-SSE
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=broadwell -mattr=-avx2   | FileCheck %s --check-prefixes=CHECK,BROADWELL
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=skylake -mattr=-sse4.2 | FileCheck %s --check-prefixes=CHECK,SKYLAKE-SSE
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=skylake -mattr=-avx2   | FileCheck %s --check-prefixes=CHECK,SKYLAKE
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=skx -mattr=-sse4.2 | FileCheck %s --check-prefixes=CHECK,SKX-SSE
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=skx -mattr=-avx2   | FileCheck %s --check-prefixes=CHECK,SKX
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=bdver2 -mattr=-sse4.2 | FileCheck %s --check-prefixes=CHECK,BDVER2-SSE
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=bdver2 -mattr=-avx2,-xop   | FileCheck %s --check-prefixes=CHECK,BDVER2
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=btver2 -mattr=-sse4.2 | FileCheck %s --check-prefixes=CHECK,BTVER2-SSE
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=btver2 -mattr=-avx2   | FileCheck %s --check-prefixes=CHECK,BTVER2
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=znver1 -mattr=-sse4.2 | FileCheck %s --check-prefixes=CHECK,ZNVER1-SSE
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=znver1 -mattr=-avx2   | FileCheck %s --check-prefixes=CHECK,ZNVER1
-
-define <2 x double> @test_blendpd(<2 x double> %a0, <2 x double> %a1, <2 x double> *%a2) {
-; GENERIC-LABEL: test_blendpd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    blendpd {{.*#+}} xmm1 = xmm0[0],xmm1[1] sched: [1:0.50]
-; GENERIC-NEXT:    blendpd {{.*#+}} xmm0 = xmm0[0],mem[1] sched: [7:0.50]
-; GENERIC-NEXT:    addpd %xmm1, %xmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SLM-LABEL: test_blendpd:
-; SLM:       # %bb.0:
-; SLM-NEXT:    blendpd {{.*#+}} xmm1 = xmm0[0],xmm1[1] sched: [1:1.00]
-; SLM-NEXT:    blendpd {{.*#+}} xmm0 = xmm0[0],mem[1] sched: [4:1.00]
-; SLM-NEXT:    addpd %xmm1, %xmm0 # sched: [3:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_blendpd:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    blendpd {{.*#+}} xmm1 = xmm0[0],xmm1[1] sched: [1:0.50]
-; SANDY-SSE-NEXT:    blendpd {{.*#+}} xmm0 = xmm0[0],mem[1] sched: [7:0.50]
-; SANDY-SSE-NEXT:    addpd %xmm1, %xmm0 # sched: [3:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_blendpd:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vblendpd {{.*#+}} xmm1 = xmm0[0],xmm1[1] sched: [1:0.50]
-; SANDY-NEXT:    vblendpd {{.*#+}} xmm0 = xmm0[0],mem[1] sched: [7:0.50]
-; SANDY-NEXT:    vaddpd %xmm0, %xmm1, %xmm0 # sched: [3:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_blendpd:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    blendpd {{.*#+}} xmm1 = xmm0[0],xmm1[1] sched: [1:0.33]
-; HASWELL-SSE-NEXT:    blendpd {{.*#+}} xmm0 = xmm0[0],mem[1] sched: [7:0.50]
-; HASWELL-SSE-NEXT:    addpd %xmm1, %xmm0 # sched: [3:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_blendpd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vblendpd {{.*#+}} xmm1 = xmm0[0],xmm1[1] sched: [1:0.33]
-; HASWELL-NEXT:    vblendpd {{.*#+}} xmm0 = xmm0[0],mem[1] sched: [7:0.50]
-; HASWELL-NEXT:    vaddpd %xmm0, %xmm1, %xmm0 # sched: [3:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_blendpd:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    blendpd {{.*#+}} xmm1 = xmm0[0],xmm1[1] sched: [1:0.33]
-; BROADWELL-SSE-NEXT:    blendpd {{.*#+}} xmm0 = xmm0[0],mem[1] sched: [6:0.50]
-; BROADWELL-SSE-NEXT:    addpd %xmm1, %xmm0 # sched: [3:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_blendpd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vblendpd {{.*#+}} xmm1 = xmm0[0],xmm1[1] sched: [1:0.33]
-; BROADWELL-NEXT:    vblendpd {{.*#+}} xmm0 = xmm0[0],mem[1] sched: [6:0.50]
-; BROADWELL-NEXT:    vaddpd %xmm0, %xmm1, %xmm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_blendpd:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    blendpd {{.*#+}} xmm1 = xmm0[0],xmm1[1] sched: [1:0.33]
-; SKYLAKE-SSE-NEXT:    blendpd {{.*#+}} xmm0 = xmm0[0],mem[1] sched: [7:0.50]
-; SKYLAKE-SSE-NEXT:    addpd %xmm1, %xmm0 # sched: [4:0.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_blendpd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vblendpd {{.*#+}} xmm1 = xmm0[0],xmm1[1] sched: [1:0.33]
-; SKYLAKE-NEXT:    vblendpd {{.*#+}} xmm0 = xmm0[0],mem[1] sched: [7:0.50]
-; SKYLAKE-NEXT:    vaddpd %xmm0, %xmm1, %xmm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_blendpd:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    blendpd {{.*#+}} xmm1 = xmm0[0],xmm1[1] sched: [1:0.33]
-; SKX-SSE-NEXT:    blendpd {{.*#+}} xmm0 = xmm0[0],mem[1] sched: [7:0.50]
-; SKX-SSE-NEXT:    addpd %xmm1, %xmm0 # sched: [4:0.50]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_blendpd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vblendpd {{.*#+}} xmm1 = xmm0[0],xmm1[1] sched: [1:0.33]
-; SKX-NEXT:    vblendpd {{.*#+}} xmm0 = xmm0[0],mem[1] sched: [7:0.50]
-; SKX-NEXT:    vaddpd %xmm0, %xmm1, %xmm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_blendpd:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    blendpd {{.*#+}} xmm1 = xmm0[0],xmm1[1] sched: [2:0.50]
-; BDVER2-SSE-NEXT:    blendpd {{.*#+}} xmm0 = xmm0[0],mem[1] sched: [7:0.50]
-; BDVER2-SSE-NEXT:    addpd %xmm1, %xmm0 # sched: [5:1.00]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_blendpd:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vblendpd {{.*#+}} xmm1 = xmm0[0],xmm1[1] sched: [2:0.50]
-; BDVER2-NEXT:    vblendpd {{.*#+}} xmm0 = xmm0[0],mem[1] sched: [7:0.50]
-; BDVER2-NEXT:    vaddpd %xmm0, %xmm1, %xmm0 # sched: [5:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_blendpd:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    blendpd {{.*#+}} xmm1 = xmm0[0],xmm1[1] sched: [1:0.50]
-; BTVER2-SSE-NEXT:    blendpd {{.*#+}} xmm0 = xmm0[0],mem[1] sched: [6:1.00]
-; BTVER2-SSE-NEXT:    addpd %xmm1, %xmm0 # sched: [3:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_blendpd:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vblendpd {{.*#+}} xmm1 = xmm0[0],xmm1[1] sched: [1:0.50]
-; BTVER2-NEXT:    vblendpd {{.*#+}} xmm0 = xmm0[0],mem[1] sched: [6:1.00]
-; BTVER2-NEXT:    vaddpd %xmm0, %xmm1, %xmm0 # sched: [3:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_blendpd:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    blendpd {{.*#+}} xmm1 = xmm0[0],xmm1[1] sched: [1:0.50]
-; ZNVER1-SSE-NEXT:    blendpd {{.*#+}} xmm0 = xmm0[0],mem[1] sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    addpd %xmm1, %xmm0 # sched: [3:1.00]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_blendpd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vblendpd {{.*#+}} xmm1 = xmm0[0],xmm1[1] sched: [1:0.50]
-; ZNVER1-NEXT:    vblendpd {{.*#+}} xmm0 = xmm0[0],mem[1] sched: [8:0.50]
-; ZNVER1-NEXT:    vaddpd %xmm0, %xmm1, %xmm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = shufflevector <2 x double> %a0, <2 x double> %a1, <2 x i32> <i32 0, i32 3>
-  %2 = load <2 x double>, <2 x double> *%a2, align 16
-  %3 = shufflevector <2 x double> %1, <2 x double> %2, <2 x i32> <i32 0, i32 3>
-  %4 = fadd <2 x double> %1, %3
-  ret <2 x double> %4
-}
-
-define <4 x float> @test_blendps(<4 x float> %a0, <4 x float> %a1, <4 x float> *%a2) {
-; GENERIC-LABEL: test_blendps:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    blendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2],xmm0[3] sched: [1:0.50]
-; GENERIC-NEXT:    blendps {{.*#+}} xmm1 = xmm1[0],mem[1],xmm1[2,3] sched: [7:0.50]
-; GENERIC-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SLM-LABEL: test_blendps:
-; SLM:       # %bb.0:
-; SLM-NEXT:    blendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2],xmm0[3] sched: [1:1.00]
-; SLM-NEXT:    blendps {{.*#+}} xmm1 = xmm1[0],mem[1],xmm1[2,3] sched: [4:1.00]
-; SLM-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_blendps:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    blendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2],xmm0[3] sched: [1:0.50]
-; SANDY-SSE-NEXT:    blendps {{.*#+}} xmm1 = xmm1[0],mem[1],xmm1[2,3] sched: [7:0.50]
-; SANDY-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_blendps:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2],xmm0[3] sched: [1:0.50]
-; SANDY-NEXT:    vblendps {{.*#+}} xmm1 = xmm1[0],mem[1],xmm1[2,3] sched: [7:0.50]
-; SANDY-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_blendps:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    blendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2],xmm0[3] sched: [1:0.33]
-; HASWELL-SSE-NEXT:    blendps {{.*#+}} xmm1 = xmm1[0],mem[1],xmm1[2,3] sched: [7:0.50]
-; HASWELL-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_blendps:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2],xmm0[3] sched: [1:0.33]
-; HASWELL-NEXT:    vblendps {{.*#+}} xmm1 = xmm1[0],mem[1],xmm1[2,3] sched: [7:0.50]
-; HASWELL-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_blendps:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    blendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2],xmm0[3] sched: [1:0.33]
-; BROADWELL-SSE-NEXT:    blendps {{.*#+}} xmm1 = xmm1[0],mem[1],xmm1[2,3] sched: [6:0.50]
-; BROADWELL-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_blendps:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2],xmm0[3] sched: [1:0.33]
-; BROADWELL-NEXT:    vblendps {{.*#+}} xmm1 = xmm1[0],mem[1],xmm1[2,3] sched: [6:0.50]
-; BROADWELL-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_blendps:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    blendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2],xmm0[3] sched: [1:0.33]
-; SKYLAKE-SSE-NEXT:    blendps {{.*#+}} xmm1 = xmm1[0],mem[1],xmm1[2,3] sched: [7:0.50]
-; SKYLAKE-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [4:0.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_blendps:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2],xmm0[3] sched: [1:0.33]
-; SKYLAKE-NEXT:    vblendps {{.*#+}} xmm1 = xmm1[0],mem[1],xmm1[2,3] sched: [7:0.50]
-; SKYLAKE-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_blendps:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    blendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2],xmm0[3] sched: [1:0.33]
-; SKX-SSE-NEXT:    blendps {{.*#+}} xmm1 = xmm1[0],mem[1],xmm1[2,3] sched: [7:0.50]
-; SKX-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [4:0.50]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_blendps:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2],xmm0[3] sched: [1:0.33]
-; SKX-NEXT:    vblendps {{.*#+}} xmm1 = xmm1[0],mem[1],xmm1[2,3] sched: [7:0.50]
-; SKX-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_blendps:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    blendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2],xmm0[3] sched: [2:0.50]
-; BDVER2-SSE-NEXT:    blendps {{.*#+}} xmm1 = xmm1[0],mem[1],xmm1[2,3] sched: [7:0.50]
-; BDVER2-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [5:1.00]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_blendps:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2],xmm0[3] sched: [2:0.50]
-; BDVER2-NEXT:    vblendps {{.*#+}} xmm1 = xmm1[0],mem[1],xmm1[2,3] sched: [7:0.50]
-; BDVER2-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [5:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_blendps:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    blendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2],xmm0[3] sched: [1:0.50]
-; BTVER2-SSE-NEXT:    blendps {{.*#+}} xmm1 = xmm1[0],mem[1],xmm1[2,3] sched: [6:1.00]
-; BTVER2-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_blendps:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2],xmm0[3] sched: [1:0.50]
-; BTVER2-NEXT:    vblendps {{.*#+}} xmm1 = xmm1[0],mem[1],xmm1[2,3] sched: [6:1.00]
-; BTVER2-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_blendps:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    blendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2],xmm0[3] sched: [1:0.50]
-; ZNVER1-SSE-NEXT:    blendps {{.*#+}} xmm1 = xmm1[0],mem[1],xmm1[2,3] sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_blendps:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2],xmm0[3] sched: [1:0.50]
-; ZNVER1-NEXT:    vblendps {{.*#+}} xmm1 = xmm1[0],mem[1],xmm1[2,3] sched: [8:0.50]
-; ZNVER1-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = shufflevector <4 x float> %a0, <4 x float> %a1, <4 x i32> <i32 0, i32 5, i32 6, i32 3>
-  %2 = load <4 x float>, <4 x float> *%a2, align 16
-  %3 = shufflevector <4 x float> %a1, <4 x float> %2, <4 x i32> <i32 0, i32 5, i32 2, i32 3>
-  %4 = fadd <4 x float> %1, %3
-  ret <4 x float> %4
-}
-
-define <2 x double> @test_blendvpd(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, <2 x double> *%a3) {
-; GENERIC-LABEL: test_blendvpd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    movapd %xmm0, %xmm3 # sched: [1:1.00]
-; GENERIC-NEXT:    movaps %xmm2, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    blendvpd %xmm0, %xmm1, %xmm3 # sched: [2:1.00]
-; GENERIC-NEXT:    blendvpd %xmm0, (%rdi), %xmm3 # sched: [8:1.00]
-; GENERIC-NEXT:    movapd %xmm3, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SLM-LABEL: test_blendvpd:
-; SLM:       # %bb.0:
-; SLM-NEXT:    movapd %xmm0, %xmm3 # sched: [1:0.50]
-; SLM-NEXT:    movaps %xmm2, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    blendvpd %xmm0, %xmm1, %xmm3 # sched: [1:1.00]
-; SLM-NEXT:    blendvpd %xmm0, (%rdi), %xmm3 # sched: [4:1.00]
-; SLM-NEXT:    movapd %xmm3, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_blendvpd:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    movapd %xmm0, %xmm3 # sched: [1:1.00]
-; SANDY-SSE-NEXT:    movaps %xmm2, %xmm0 # sched: [1:1.00]
-; SANDY-SSE-NEXT:    blendvpd %xmm0, %xmm1, %xmm3 # sched: [2:1.00]
-; SANDY-SSE-NEXT:    blendvpd %xmm0, (%rdi), %xmm3 # sched: [8:1.00]
-; SANDY-SSE-NEXT:    movapd %xmm3, %xmm0 # sched: [1:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_blendvpd:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vblendvpd %xmm2, %xmm1, %xmm0, %xmm0 # sched: [2:1.00]
-; SANDY-NEXT:    vblendvpd %xmm2, (%rdi), %xmm0, %xmm0 # sched: [8:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_blendvpd:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    movapd %xmm0, %xmm3 # sched: [1:1.00]
-; HASWELL-SSE-NEXT:    movaps %xmm2, %xmm0 # sched: [1:1.00]
-; HASWELL-SSE-NEXT:    blendvpd %xmm0, %xmm1, %xmm3 # sched: [2:2.00]
-; HASWELL-SSE-NEXT:    blendvpd %xmm0, (%rdi), %xmm3 # sched: [8:2.00]
-; HASWELL-SSE-NEXT:    movapd %xmm3, %xmm0 # sched: [1:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_blendvpd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vblendvpd %xmm2, %xmm1, %xmm0, %xmm0 # sched: [2:2.00]
-; HASWELL-NEXT:    vblendvpd %xmm2, (%rdi), %xmm0, %xmm0 # sched: [8:2.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_blendvpd:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    movapd %xmm0, %xmm3 # sched: [1:1.00]
-; BROADWELL-SSE-NEXT:    movaps %xmm2, %xmm0 # sched: [1:1.00]
-; BROADWELL-SSE-NEXT:    blendvpd %xmm0, %xmm1, %xmm3 # sched: [2:2.00]
-; BROADWELL-SSE-NEXT:    blendvpd %xmm0, (%rdi), %xmm3 # sched: [7:2.00]
-; BROADWELL-SSE-NEXT:    movapd %xmm3, %xmm0 # sched: [1:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_blendvpd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vblendvpd %xmm2, %xmm1, %xmm0, %xmm0 # sched: [2:2.00]
-; BROADWELL-NEXT:    vblendvpd %xmm2, (%rdi), %xmm0, %xmm0 # sched: [7:2.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_blendvpd:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    movapd %xmm0, %xmm3 # sched: [1:0.33]
-; SKYLAKE-SSE-NEXT:    movaps %xmm2, %xmm0 # sched: [1:0.33]
-; SKYLAKE-SSE-NEXT:    blendvpd %xmm0, %xmm1, %xmm3 # sched: [2:0.67]
-; SKYLAKE-SSE-NEXT:    blendvpd %xmm0, (%rdi), %xmm3 # sched: [8:0.67]
-; SKYLAKE-SSE-NEXT:    movapd %xmm3, %xmm0 # sched: [1:0.33]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_blendvpd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vblendvpd %xmm2, %xmm1, %xmm0, %xmm0 # sched: [2:0.67]
-; SKYLAKE-NEXT:    vblendvpd %xmm2, (%rdi), %xmm0, %xmm0 # sched: [8:0.67]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_blendvpd:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    movapd %xmm0, %xmm3 # sched: [1:0.33]
-; SKX-SSE-NEXT:    movaps %xmm2, %xmm0 # sched: [1:0.33]
-; SKX-SSE-NEXT:    blendvpd %xmm0, %xmm1, %xmm3 # sched: [2:0.67]
-; SKX-SSE-NEXT:    blendvpd %xmm0, (%rdi), %xmm3 # sched: [8:0.67]
-; SKX-SSE-NEXT:    movapd %xmm3, %xmm0 # sched: [1:0.33]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_blendvpd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vblendvpd %xmm2, %xmm1, %xmm0, %xmm0 # sched: [2:0.67]
-; SKX-NEXT:    vblendvpd %xmm2, (%rdi), %xmm0, %xmm0 # sched: [8:0.67]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_blendvpd:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    movapd %xmm0, %xmm3 # sched: [1:0.50]
-; BDVER2-SSE-NEXT:    movaps %xmm2, %xmm0 # sched: [1:0.50]
-; BDVER2-SSE-NEXT:    blendvpd %xmm0, %xmm1, %xmm3 # sched: [2:2.00]
-; BDVER2-SSE-NEXT:    blendvpd %xmm0, (%rdi), %xmm3 # sched: [7:2.00]
-; BDVER2-SSE-NEXT:    movapd %xmm3, %xmm0 # sched: [1:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_blendvpd:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vblendvpd %xmm2, %xmm1, %xmm0, %xmm0 # sched: [2:2.00]
-; BDVER2-NEXT:    vblendvpd %xmm2, (%rdi), %xmm0, %xmm0 # sched: [7:2.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_blendvpd:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    movapd %xmm0, %xmm3 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    movaps %xmm2, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    blendvpd %xmm0, %xmm1, %xmm3 # sched: [2:2.00]
-; BTVER2-SSE-NEXT:    blendvpd %xmm0, (%rdi), %xmm3 # sched: [7:2.00]
-; BTVER2-SSE-NEXT:    movapd %xmm3, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_blendvpd:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vblendvpd %xmm2, %xmm1, %xmm0, %xmm0 # sched: [2:2.00]
-; BTVER2-NEXT:    vblendvpd %xmm2, (%rdi), %xmm0, %xmm0 # sched: [7:2.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_blendvpd:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    movapd %xmm0, %xmm3 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    movaps %xmm2, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    blendvpd %xmm0, %xmm1, %xmm3 # sched: [1:0.50]
-; ZNVER1-SSE-NEXT:    blendvpd %xmm0, (%rdi), %xmm3 # sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    movapd %xmm3, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_blendvpd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vblendvpd %xmm2, %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; ZNVER1-NEXT:    vblendvpd %xmm2, (%rdi), %xmm0, %xmm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <2 x double> @llvm.x86.sse41.blendvpd(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2)
-  %2 = load <2 x double>, <2 x double> *%a3, align 16
-  %3 = call <2 x double> @llvm.x86.sse41.blendvpd(<2 x double> %1, <2 x double> %2, <2 x double> %a2)
-  ret <2 x double> %3
-}
-declare <2 x double> @llvm.x86.sse41.blendvpd(<2 x double>, <2 x double>, <2 x double>) nounwind readnone
-
-define <4 x float> @test_blendvps(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, <4 x float> *%a3) {
-; GENERIC-LABEL: test_blendvps:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    movaps %xmm0, %xmm3 # sched: [1:1.00]
-; GENERIC-NEXT:    movaps %xmm2, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    blendvps %xmm0, %xmm1, %xmm3 # sched: [2:1.00]
-; GENERIC-NEXT:    blendvps %xmm0, (%rdi), %xmm3 # sched: [8:1.00]
-; GENERIC-NEXT:    movaps %xmm3, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SLM-LABEL: test_blendvps:
-; SLM:       # %bb.0:
-; SLM-NEXT:    movaps %xmm0, %xmm3 # sched: [1:0.50]
-; SLM-NEXT:    movaps %xmm2, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    blendvps %xmm0, %xmm1, %xmm3 # sched: [1:1.00]
-; SLM-NEXT:    blendvps %xmm0, (%rdi), %xmm3 # sched: [4:1.00]
-; SLM-NEXT:    movaps %xmm3, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_blendvps:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    movaps %xmm0, %xmm3 # sched: [1:1.00]
-; SANDY-SSE-NEXT:    movaps %xmm2, %xmm0 # sched: [1:1.00]
-; SANDY-SSE-NEXT:    blendvps %xmm0, %xmm1, %xmm3 # sched: [2:1.00]
-; SANDY-SSE-NEXT:    blendvps %xmm0, (%rdi), %xmm3 # sched: [8:1.00]
-; SANDY-SSE-NEXT:    movaps %xmm3, %xmm0 # sched: [1:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_blendvps:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vblendvps %xmm2, %xmm1, %xmm0, %xmm0 # sched: [2:1.00]
-; SANDY-NEXT:    vblendvps %xmm2, (%rdi), %xmm0, %xmm0 # sched: [8:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_blendvps:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    movaps %xmm0, %xmm3 # sched: [1:1.00]
-; HASWELL-SSE-NEXT:    movaps %xmm2, %xmm0 # sched: [1:1.00]
-; HASWELL-SSE-NEXT:    blendvps %xmm0, %xmm1, %xmm3 # sched: [2:2.00]
-; HASWELL-SSE-NEXT:    blendvps %xmm0, (%rdi), %xmm3 # sched: [8:2.00]
-; HASWELL-SSE-NEXT:    movaps %xmm3, %xmm0 # sched: [1:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_blendvps:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vblendvps %xmm2, %xmm1, %xmm0, %xmm0 # sched: [2:2.00]
-; HASWELL-NEXT:    vblendvps %xmm2, (%rdi), %xmm0, %xmm0 # sched: [8:2.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_blendvps:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    movaps %xmm0, %xmm3 # sched: [1:1.00]
-; BROADWELL-SSE-NEXT:    movaps %xmm2, %xmm0 # sched: [1:1.00]
-; BROADWELL-SSE-NEXT:    blendvps %xmm0, %xmm1, %xmm3 # sched: [2:2.00]
-; BROADWELL-SSE-NEXT:    blendvps %xmm0, (%rdi), %xmm3 # sched: [7:2.00]
-; BROADWELL-SSE-NEXT:    movaps %xmm3, %xmm0 # sched: [1:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_blendvps:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vblendvps %xmm2, %xmm1, %xmm0, %xmm0 # sched: [2:2.00]
-; BROADWELL-NEXT:    vblendvps %xmm2, (%rdi), %xmm0, %xmm0 # sched: [7:2.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_blendvps:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    movaps %xmm0, %xmm3 # sched: [1:0.33]
-; SKYLAKE-SSE-NEXT:    movaps %xmm2, %xmm0 # sched: [1:0.33]
-; SKYLAKE-SSE-NEXT:    blendvps %xmm0, %xmm1, %xmm3 # sched: [2:0.67]
-; SKYLAKE-SSE-NEXT:    blendvps %xmm0, (%rdi), %xmm3 # sched: [8:0.67]
-; SKYLAKE-SSE-NEXT:    movaps %xmm3, %xmm0 # sched: [1:0.33]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_blendvps:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vblendvps %xmm2, %xmm1, %xmm0, %xmm0 # sched: [2:0.67]
-; SKYLAKE-NEXT:    vblendvps %xmm2, (%rdi), %xmm0, %xmm0 # sched: [8:0.67]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_blendvps:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    movaps %xmm0, %xmm3 # sched: [1:0.33]
-; SKX-SSE-NEXT:    movaps %xmm2, %xmm0 # sched: [1:0.33]
-; SKX-SSE-NEXT:    blendvps %xmm0, %xmm1, %xmm3 # sched: [2:0.67]
-; SKX-SSE-NEXT:    blendvps %xmm0, (%rdi), %xmm3 # sched: [8:0.67]
-; SKX-SSE-NEXT:    movaps %xmm3, %xmm0 # sched: [1:0.33]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_blendvps:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vblendvps %xmm2, %xmm1, %xmm0, %xmm0 # sched: [2:0.67]
-; SKX-NEXT:    vblendvps %xmm2, (%rdi), %xmm0, %xmm0 # sched: [8:0.67]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_blendvps:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    movaps %xmm0, %xmm3 # sched: [1:0.50]
-; BDVER2-SSE-NEXT:    movaps %xmm2, %xmm0 # sched: [1:0.50]
-; BDVER2-SSE-NEXT:    blendvps %xmm0, %xmm1, %xmm3 # sched: [2:2.00]
-; BDVER2-SSE-NEXT:    blendvps %xmm0, (%rdi), %xmm3 # sched: [7:2.00]
-; BDVER2-SSE-NEXT:    movaps %xmm3, %xmm0 # sched: [1:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_blendvps:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vblendvps %xmm2, %xmm1, %xmm0, %xmm0 # sched: [2:2.00]
-; BDVER2-NEXT:    vblendvps %xmm2, (%rdi), %xmm0, %xmm0 # sched: [7:2.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_blendvps:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    movaps %xmm0, %xmm3 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    movaps %xmm2, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    blendvps %xmm0, %xmm1, %xmm3 # sched: [2:2.00]
-; BTVER2-SSE-NEXT:    blendvps %xmm0, (%rdi), %xmm3 # sched: [7:2.00]
-; BTVER2-SSE-NEXT:    movaps %xmm3, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_blendvps:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vblendvps %xmm2, %xmm1, %xmm0, %xmm0 # sched: [2:2.00]
-; BTVER2-NEXT:    vblendvps %xmm2, (%rdi), %xmm0, %xmm0 # sched: [7:2.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_blendvps:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    movaps %xmm0, %xmm3 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    movaps %xmm2, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    blendvps %xmm0, %xmm1, %xmm3 # sched: [1:0.50]
-; ZNVER1-SSE-NEXT:    blendvps %xmm0, (%rdi), %xmm3 # sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    movaps %xmm3, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_blendvps:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vblendvps %xmm2, %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; ZNVER1-NEXT:    vblendvps %xmm2, (%rdi), %xmm0, %xmm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <4 x float> @llvm.x86.sse41.blendvps(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2)
-  %2 = load <4 x float>, <4 x float> *%a3
-  %3 = call <4 x float> @llvm.x86.sse41.blendvps(<4 x float> %1, <4 x float> %2, <4 x float> %a2)
-  ret <4 x float> %3
-}
-declare <4 x float> @llvm.x86.sse41.blendvps(<4 x float>, <4 x float>, <4 x float>) nounwind readnone
-
-define <2 x double> @test_dppd(<2 x double> %a0, <2 x double> %a1, <2 x double> *%a2) {
-; GENERIC-LABEL: test_dppd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    dppd $7, %xmm1, %xmm0 # sched: [9:1.00]
-; GENERIC-NEXT:    dppd $7, (%rdi), %xmm0 # sched: [15:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SLM-LABEL: test_dppd:
-; SLM:       # %bb.0:
-; SLM-NEXT:    dppd $7, %xmm1, %xmm0 # sched: [3:1.00]
-; SLM-NEXT:    dppd $7, (%rdi), %xmm0 # sched: [6:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_dppd:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    dppd $7, %xmm1, %xmm0 # sched: [9:1.00]
-; SANDY-SSE-NEXT:    dppd $7, (%rdi), %xmm0 # sched: [15:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_dppd:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vdppd $7, %xmm1, %xmm0, %xmm0 # sched: [9:1.00]
-; SANDY-NEXT:    vdppd $7, (%rdi), %xmm0, %xmm0 # sched: [15:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_dppd:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    dppd $7, %xmm1, %xmm0 # sched: [9:1.00]
-; HASWELL-SSE-NEXT:    dppd $7, (%rdi), %xmm0 # sched: [15:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_dppd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vdppd $7, %xmm1, %xmm0, %xmm0 # sched: [9:1.00]
-; HASWELL-NEXT:    vdppd $7, (%rdi), %xmm0, %xmm0 # sched: [15:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_dppd:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    dppd $7, %xmm1, %xmm0 # sched: [9:1.00]
-; BROADWELL-SSE-NEXT:    dppd $7, (%rdi), %xmm0 # sched: [14:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_dppd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vdppd $7, %xmm1, %xmm0, %xmm0 # sched: [9:1.00]
-; BROADWELL-NEXT:    vdppd $7, (%rdi), %xmm0, %xmm0 # sched: [14:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_dppd:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    dppd $7, %xmm1, %xmm0 # sched: [9:1.00]
-; SKYLAKE-SSE-NEXT:    dppd $7, (%rdi), %xmm0 # sched: [15:1.00]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_dppd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vdppd $7, %xmm1, %xmm0, %xmm0 # sched: [9:1.00]
-; SKYLAKE-NEXT:    vdppd $7, (%rdi), %xmm0, %xmm0 # sched: [15:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_dppd:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    dppd $7, %xmm1, %xmm0 # sched: [9:1.00]
-; SKX-SSE-NEXT:    dppd $7, (%rdi), %xmm0 # sched: [15:1.00]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_dppd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vdppd $7, %xmm1, %xmm0, %xmm0 # sched: [9:1.00]
-; SKX-NEXT:    vdppd $7, (%rdi), %xmm0, %xmm0 # sched: [15:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_dppd:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    dppd $7, %xmm1, %xmm0 # sched: [15:1.50]
-; BDVER2-SSE-NEXT:    dppd $7, (%rdi), %xmm0 # sched: [20:1.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_dppd:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vdppd $7, %xmm1, %xmm0, %xmm0 # sched: [15:1.50]
-; BDVER2-NEXT:    vdppd $7, (%rdi), %xmm0, %xmm0 # sched: [20:1.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_dppd:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    dppd $7, %xmm1, %xmm0 # sched: [9:3.00]
-; BTVER2-SSE-NEXT:    dppd $7, (%rdi), %xmm0 # sched: [14:3.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_dppd:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vdppd $7, %xmm1, %xmm0, %xmm0 # sched: [9:3.00]
-; BTVER2-NEXT:    vdppd $7, (%rdi), %xmm0, %xmm0 # sched: [14:3.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_dppd:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    dppd $7, %xmm1, %xmm0 # sched: [100:0.25]
-; ZNVER1-SSE-NEXT:    dppd $7, (%rdi), %xmm0 # sched: [100:0.25]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_dppd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vdppd $7, %xmm1, %xmm0, %xmm0 # sched: [100:0.25]
-; ZNVER1-NEXT:    vdppd $7, (%rdi), %xmm0, %xmm0 # sched: [100:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <2 x double> @llvm.x86.sse41.dppd(<2 x double> %a0, <2 x double> %a1, i8 7)
-  %2 = load <2 x double>, <2 x double> *%a2, align 16
-  %3 = call <2 x double> @llvm.x86.sse41.dppd(<2 x double> %1, <2 x double> %2, i8 7)
-  ret <2 x double> %3
-}
-declare <2 x double> @llvm.x86.sse41.dppd(<2 x double>, <2 x double>, i8) nounwind readnone
-
-define <4 x float> @test_dpps(<4 x float> %a0, <4 x float> %a1, <4 x float> *%a2) {
-; GENERIC-LABEL: test_dpps:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    dpps $7, %xmm1, %xmm0 # sched: [12:2.00]
-; GENERIC-NEXT:    dpps $7, (%rdi), %xmm0 # sched: [18:2.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SLM-LABEL: test_dpps:
-; SLM:       # %bb.0:
-; SLM-NEXT:    dpps $7, %xmm1, %xmm0 # sched: [3:1.00]
-; SLM-NEXT:    dpps $7, (%rdi), %xmm0 # sched: [6:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_dpps:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    dpps $7, %xmm1, %xmm0 # sched: [12:2.00]
-; SANDY-SSE-NEXT:    dpps $7, (%rdi), %xmm0 # sched: [18:2.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_dpps:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vdpps $7, %xmm1, %xmm0, %xmm0 # sched: [12:2.00]
-; SANDY-NEXT:    vdpps $7, (%rdi), %xmm0, %xmm0 # sched: [18:2.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_dpps:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    dpps $7, %xmm1, %xmm0 # sched: [14:2.00]
-; HASWELL-SSE-NEXT:    dpps $7, (%rdi), %xmm0 # sched: [20:2.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_dpps:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vdpps $7, %xmm1, %xmm0, %xmm0 # sched: [14:2.00]
-; HASWELL-NEXT:    vdpps $7, (%rdi), %xmm0, %xmm0 # sched: [20:2.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_dpps:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    dpps $7, %xmm1, %xmm0 # sched: [14:2.00]
-; BROADWELL-SSE-NEXT:    dpps $7, (%rdi), %xmm0 # sched: [19:2.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_dpps:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vdpps $7, %xmm1, %xmm0, %xmm0 # sched: [14:2.00]
-; BROADWELL-NEXT:    vdpps $7, (%rdi), %xmm0, %xmm0 # sched: [19:2.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_dpps:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    dpps $7, %xmm1, %xmm0 # sched: [13:1.50]
-; SKYLAKE-SSE-NEXT:    dpps $7, (%rdi), %xmm0 # sched: [19:1.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_dpps:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vdpps $7, %xmm1, %xmm0, %xmm0 # sched: [13:1.50]
-; SKYLAKE-NEXT:    vdpps $7, (%rdi), %xmm0, %xmm0 # sched: [19:1.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_dpps:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    dpps $7, %xmm1, %xmm0 # sched: [13:1.33]
-; SKX-SSE-NEXT:    dpps $7, (%rdi), %xmm0 # sched: [19:1.33]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_dpps:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vdpps $7, %xmm1, %xmm0, %xmm0 # sched: [13:1.33]
-; SKX-NEXT:    vdpps $7, (%rdi), %xmm0, %xmm0 # sched: [19:1.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_dpps:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    dpps $7, %xmm1, %xmm0 # sched: [25:1.50]
-; BDVER2-SSE-NEXT:    dpps $7, (%rdi), %xmm0 # sched: [30:1.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_dpps:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vdpps $7, %xmm1, %xmm0, %xmm0 # sched: [25:1.50]
-; BDVER2-NEXT:    vdpps $7, (%rdi), %xmm0, %xmm0 # sched: [30:1.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_dpps:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    dpps $7, %xmm1, %xmm0 # sched: [11:3.00]
-; BTVER2-SSE-NEXT:    dpps $7, (%rdi), %xmm0 # sched: [16:3.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_dpps:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vdpps $7, %xmm1, %xmm0, %xmm0 # sched: [11:3.00]
-; BTVER2-NEXT:    vdpps $7, (%rdi), %xmm0, %xmm0 # sched: [16:3.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_dpps:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    dpps $7, %xmm1, %xmm0 # sched: [100:0.25]
-; ZNVER1-SSE-NEXT:    dpps $7, (%rdi), %xmm0 # sched: [100:0.25]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_dpps:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vdpps $7, %xmm1, %xmm0, %xmm0 # sched: [100:0.25]
-; ZNVER1-NEXT:    vdpps $7, (%rdi), %xmm0, %xmm0 # sched: [100:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <4 x float> @llvm.x86.sse41.dpps(<4 x float> %a0, <4 x float> %a1, i8 7)
-  %2 = load <4 x float>, <4 x float> *%a2, align 16
-  %3 = call <4 x float> @llvm.x86.sse41.dpps(<4 x float> %1, <4 x float> %2, i8 7)
-  ret <4 x float> %3
-}
-declare <4 x float> @llvm.x86.sse41.dpps(<4 x float>, <4 x float>, i8) nounwind readnone
-
-define i32 @test_extractps(<4 x float> %a0, i32 *%a1) {
-; GENERIC-LABEL: test_extractps:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    extractps $3, %xmm0, %eax # sched: [3:1.00]
-; GENERIC-NEXT:    extractps $1, %xmm0, (%rdi) # sched: [5:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SLM-LABEL: test_extractps:
-; SLM:       # %bb.0:
-; SLM-NEXT:    extractps $3, %xmm0, %eax # sched: [1:1.00]
-; SLM-NEXT:    extractps $1, %xmm0, (%rdi) # sched: [4:2.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_extractps:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    extractps $3, %xmm0, %eax # sched: [3:1.00]
-; SANDY-SSE-NEXT:    extractps $1, %xmm0, (%rdi) # sched: [5:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_extractps:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vextractps $3, %xmm0, %eax # sched: [3:1.00]
-; SANDY-NEXT:    vextractps $1, %xmm0, (%rdi) # sched: [5:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_extractps:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    extractps $3, %xmm0, %eax # sched: [2:1.00]
-; HASWELL-SSE-NEXT:    extractps $1, %xmm0, (%rdi) # sched: [2:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_extractps:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vextractps $3, %xmm0, %eax # sched: [2:1.00]
-; HASWELL-NEXT:    vextractps $1, %xmm0, (%rdi) # sched: [2:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_extractps:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    extractps $3, %xmm0, %eax # sched: [2:1.00]
-; BROADWELL-SSE-NEXT:    extractps $1, %xmm0, (%rdi) # sched: [2:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_extractps:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vextractps $3, %xmm0, %eax # sched: [2:1.00]
-; BROADWELL-NEXT:    vextractps $1, %xmm0, (%rdi) # sched: [2:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_extractps:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    extractps $3, %xmm0, %eax # sched: [3:1.00]
-; SKYLAKE-SSE-NEXT:    extractps $1, %xmm0, (%rdi) # sched: [2:1.00]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_extractps:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vextractps $3, %xmm0, %eax # sched: [3:1.00]
-; SKYLAKE-NEXT:    vextractps $1, %xmm0, (%rdi) # sched: [2:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_extractps:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    extractps $3, %xmm0, %eax # sched: [3:1.00]
-; SKX-SSE-NEXT:    extractps $1, %xmm0, (%rdi) # sched: [2:1.00]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_extractps:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vextractps $3, %xmm0, %eax # sched: [3:1.00]
-; SKX-NEXT:    vextractps $1, %xmm0, (%rdi) # sched: [2:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_extractps:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    extractps $3, %xmm0, %eax # sched: [13:1.00]
-; BDVER2-SSE-NEXT:    extractps $1, %xmm0, (%rdi) # sched: [13:1.00]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_extractps:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vextractps $3, %xmm0, %eax # sched: [13:1.00]
-; BDVER2-NEXT:    vextractps $1, %xmm0, (%rdi) # sched: [13:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_extractps:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    extractps $3, %xmm0, %eax # sched: [3:1.00]
-; BTVER2-SSE-NEXT:    extractps $1, %xmm0, (%rdi) # sched: [3:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_extractps:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vextractps $3, %xmm0, %eax # sched: [3:1.00]
-; BTVER2-NEXT:    vextractps $1, %xmm0, (%rdi) # sched: [3:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_extractps:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    extractps $3, %xmm0, %eax # sched: [2:2.00]
-; ZNVER1-SSE-NEXT:    extractps $1, %xmm0, (%rdi) # sched: [5:2.50]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_extractps:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vextractps $3, %xmm0, %eax # sched: [2:2.00]
-; ZNVER1-NEXT:    vextractps $1, %xmm0, (%rdi) # sched: [5:2.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = extractelement <4 x float> %a0, i32 3
-  %2 = extractelement <4 x float> %a0, i32 1
-  %3 = bitcast float %1 to i32
-  %4 = bitcast float %2 to i32
-  store i32 %4, i32 *%a1
-  ret i32 %3
-}
-
-define <4 x float> @test_insertps(<4 x float> %a0, <4 x float> %a1, float *%a2) {
-; GENERIC-LABEL: test_insertps:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    insertps {{.*#+}} xmm0 = zero,xmm1[0],xmm0[2,3] sched: [1:1.00]
-; GENERIC-NEXT:    insertps {{.*#+}} xmm0 = xmm0[0,1,2],mem[0] sched: [7:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SLM-LABEL: test_insertps:
-; SLM:       # %bb.0:
-; SLM-NEXT:    insertps {{.*#+}} xmm0 = zero,xmm1[0],xmm0[2,3] sched: [1:1.00]
-; SLM-NEXT:    insertps {{.*#+}} xmm0 = xmm0[0,1,2],mem[0] sched: [4:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_insertps:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    insertps {{.*#+}} xmm0 = zero,xmm1[0],xmm0[2,3] sched: [1:1.00]
-; SANDY-SSE-NEXT:    insertps {{.*#+}} xmm0 = xmm0[0,1,2],mem[0] sched: [7:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_insertps:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vinsertps {{.*#+}} xmm0 = zero,xmm1[0],xmm0[2,3] sched: [1:1.00]
-; SANDY-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,1,2],mem[0] sched: [7:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_insertps:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    insertps {{.*#+}} xmm0 = zero,xmm1[0],xmm0[2,3] sched: [1:1.00]
-; HASWELL-SSE-NEXT:    insertps {{.*#+}} xmm0 = xmm0[0,1,2],mem[0] sched: [7:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_insertps:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vinsertps {{.*#+}} xmm0 = zero,xmm1[0],xmm0[2,3] sched: [1:1.00]
-; HASWELL-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,1,2],mem[0] sched: [7:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_insertps:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    insertps {{.*#+}} xmm0 = zero,xmm1[0],xmm0[2,3] sched: [1:1.00]
-; BROADWELL-SSE-NEXT:    insertps {{.*#+}} xmm0 = xmm0[0,1,2],mem[0] sched: [6:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_insertps:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vinsertps {{.*#+}} xmm0 = zero,xmm1[0],xmm0[2,3] sched: [1:1.00]
-; BROADWELL-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,1,2],mem[0] sched: [6:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_insertps:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    insertps {{.*#+}} xmm0 = zero,xmm1[0],xmm0[2,3] sched: [1:1.00]
-; SKYLAKE-SSE-NEXT:    insertps {{.*#+}} xmm0 = xmm0[0,1,2],mem[0] sched: [7:1.00]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_insertps:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vinsertps {{.*#+}} xmm0 = zero,xmm1[0],xmm0[2,3] sched: [1:1.00]
-; SKYLAKE-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,1,2],mem[0] sched: [7:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_insertps:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    insertps {{.*#+}} xmm0 = zero,xmm1[0],xmm0[2,3] sched: [1:1.00]
-; SKX-SSE-NEXT:    insertps {{.*#+}} xmm0 = xmm0[0,1,2],mem[0] sched: [7:1.00]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_insertps:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vinsertps {{.*#+}} xmm0 = zero,xmm1[0],xmm0[2,3] sched: [1:1.00]
-; SKX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,1,2],mem[0] sched: [7:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_insertps:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    insertps {{.*#+}} xmm0 = zero,xmm1[0],xmm0[2,3] sched: [2:0.50]
-; BDVER2-SSE-NEXT:    insertps {{.*#+}} xmm0 = xmm0[0,1,2],mem[0] sched: [7:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_insertps:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vinsertps {{.*#+}} xmm0 = zero,xmm1[0],xmm0[2,3] sched: [2:0.50]
-; BDVER2-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,1,2],mem[0] sched: [7:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_insertps:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    insertps {{.*#+}} xmm0 = zero,xmm1[0],xmm0[2,3] sched: [1:0.50]
-; BTVER2-SSE-NEXT:    insertps {{.*#+}} xmm0 = xmm0[0,1,2],mem[0] sched: [6:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_insertps:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vinsertps {{.*#+}} xmm0 = zero,xmm1[0],xmm0[2,3] sched: [1:0.50]
-; BTVER2-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,1,2],mem[0] sched: [6:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_insertps:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    insertps {{.*#+}} xmm0 = zero,xmm1[0],xmm0[2,3] sched: [1:0.50]
-; ZNVER1-SSE-NEXT:    insertps {{.*#+}} xmm0 = xmm0[0,1,2],mem[0] sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_insertps:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vinsertps {{.*#+}} xmm0 = zero,xmm1[0],xmm0[2,3] sched: [1:0.50]
-; ZNVER1-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,1,2],mem[0] sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <4 x float> @llvm.x86.sse41.insertps(<4 x float> %a0, <4 x float> %a1, i8 17)
-  %2 = load float, float *%a2
-  %3 = insertelement <4 x float> %1, float %2, i32 3
-  ret <4 x float> %3
-}
-declare <4 x float> @llvm.x86.sse41.insertps(<4 x float>, <4 x float>, i8) nounwind readnone
-
-define <2 x i64> @test_movntdqa(i8* %a0) {
-; GENERIC-LABEL: test_movntdqa:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    movntdqa (%rdi), %xmm0 # sched: [6:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SLM-LABEL: test_movntdqa:
-; SLM:       # %bb.0:
-; SLM-NEXT:    movntdqa (%rdi), %xmm0 # sched: [3:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_movntdqa:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    movntdqa (%rdi), %xmm0 # sched: [6:0.50]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_movntdqa:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vmovntdqa (%rdi), %xmm0 # sched: [6:0.50]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_movntdqa:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    movntdqa (%rdi), %xmm0 # sched: [6:0.50]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_movntdqa:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vmovntdqa (%rdi), %xmm0 # sched: [6:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_movntdqa:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    movntdqa (%rdi), %xmm0 # sched: [5:0.50]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_movntdqa:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vmovntdqa (%rdi), %xmm0 # sched: [5:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_movntdqa:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    movntdqa (%rdi), %xmm0 # sched: [6:0.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_movntdqa:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vmovntdqa (%rdi), %xmm0 # sched: [6:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_movntdqa:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    movntdqa (%rdi), %xmm0 # sched: [6:0.50]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_movntdqa:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmovntdqa (%rdi), %xmm0 # sched: [6:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_movntdqa:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    movntdqa (%rdi), %xmm0 # sched: [5:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_movntdqa:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vmovntdqa (%rdi), %xmm0 # sched: [5:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_movntdqa:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    movntdqa (%rdi), %xmm0 # sched: [5:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_movntdqa:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vmovntdqa (%rdi), %xmm0 # sched: [5:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_movntdqa:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    movntdqa (%rdi), %xmm0 # sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_movntdqa:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vmovntdqa (%rdi), %xmm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <2 x i64> @llvm.x86.sse41.movntdqa(i8* %a0)
-  ret <2 x i64> %1
-}
-declare <2 x i64> @llvm.x86.sse41.movntdqa(i8*) nounwind readnone
-
-define <8 x i16> @test_mpsadbw(<16 x i8> %a0, <16 x i8> %a1, <16 x i8> *%a2) {
-; GENERIC-LABEL: test_mpsadbw:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    mpsadbw $7, %xmm1, %xmm0 # sched: [7:1.00]
-; GENERIC-NEXT:    mpsadbw $7, (%rdi), %xmm0 # sched: [13:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SLM-LABEL: test_mpsadbw:
-; SLM:       # %bb.0:
-; SLM-NEXT:    mpsadbw $7, %xmm1, %xmm0 # sched: [7:1.00]
-; SLM-NEXT:    mpsadbw $7, (%rdi), %xmm0 # sched: [10:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_mpsadbw:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    mpsadbw $7, %xmm1, %xmm0 # sched: [7:1.00]
-; SANDY-SSE-NEXT:    mpsadbw $7, (%rdi), %xmm0 # sched: [13:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_mpsadbw:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vmpsadbw $7, %xmm1, %xmm0, %xmm0 # sched: [7:1.00]
-; SANDY-NEXT:    vmpsadbw $7, (%rdi), %xmm0, %xmm0 # sched: [13:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_mpsadbw:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    mpsadbw $7, %xmm1, %xmm0 # sched: [7:2.00]
-; HASWELL-SSE-NEXT:    mpsadbw $7, (%rdi), %xmm0 # sched: [13:2.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_mpsadbw:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vmpsadbw $7, %xmm1, %xmm0, %xmm0 # sched: [7:2.00]
-; HASWELL-NEXT:    vmpsadbw $7, (%rdi), %xmm0, %xmm0 # sched: [13:2.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_mpsadbw:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    mpsadbw $7, %xmm1, %xmm0 # sched: [7:2.00]
-; BROADWELL-SSE-NEXT:    mpsadbw $7, (%rdi), %xmm0 # sched: [12:2.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_mpsadbw:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vmpsadbw $7, %xmm1, %xmm0, %xmm0 # sched: [7:2.00]
-; BROADWELL-NEXT:    vmpsadbw $7, (%rdi), %xmm0, %xmm0 # sched: [12:2.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_mpsadbw:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    mpsadbw $7, %xmm1, %xmm0 # sched: [4:2.00]
-; SKYLAKE-SSE-NEXT:    mpsadbw $7, (%rdi), %xmm0 # sched: [10:2.00]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_mpsadbw:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vmpsadbw $7, %xmm1, %xmm0, %xmm0 # sched: [4:2.00]
-; SKYLAKE-NEXT:    vmpsadbw $7, (%rdi), %xmm0, %xmm0 # sched: [10:2.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_mpsadbw:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    mpsadbw $7, %xmm1, %xmm0 # sched: [4:2.00]
-; SKX-SSE-NEXT:    mpsadbw $7, (%rdi), %xmm0 # sched: [10:2.00]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_mpsadbw:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vmpsadbw $7, %xmm1, %xmm0, %xmm0 # sched: [4:2.00]
-; SKX-NEXT:    vmpsadbw $7, (%rdi), %xmm0, %xmm0 # sched: [10:2.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_mpsadbw:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    mpsadbw $7, %xmm1, %xmm0 # sched: [9:2.00]
-; BDVER2-SSE-NEXT:    mpsadbw $7, (%rdi), %xmm0 # sched: [14:2.00]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_mpsadbw:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vmpsadbw $7, %xmm1, %xmm0, %xmm0 # sched: [9:2.00]
-; BDVER2-NEXT:    vmpsadbw $7, (%rdi), %xmm0, %xmm0 # sched: [14:2.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_mpsadbw:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    mpsadbw $7, %xmm1, %xmm0 # sched: [3:2.00]
-; BTVER2-SSE-NEXT:    mpsadbw $7, (%rdi), %xmm0 # sched: [8:2.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_mpsadbw:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vmpsadbw $7, %xmm1, %xmm0, %xmm0 # sched: [3:2.00]
-; BTVER2-NEXT:    vmpsadbw $7, (%rdi), %xmm0, %xmm0 # sched: [8:2.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_mpsadbw:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    mpsadbw $7, %xmm1, %xmm0 # sched: [100:0.25]
-; ZNVER1-SSE-NEXT:    mpsadbw $7, (%rdi), %xmm0 # sched: [100:0.25]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_mpsadbw:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vmpsadbw $7, %xmm1, %xmm0, %xmm0 # sched: [100:0.25]
-; ZNVER1-NEXT:    vmpsadbw $7, (%rdi), %xmm0, %xmm0 # sched: [100:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <8 x i16> @llvm.x86.sse41.mpsadbw(<16 x i8> %a0, <16 x i8> %a1, i8 7)
-  %2 = bitcast <8 x i16> %1 to <16 x i8>
-  %3 = load <16 x i8>, <16 x i8> *%a2, align 16
-  %4 = call <8 x i16> @llvm.x86.sse41.mpsadbw(<16 x i8> %2, <16 x i8> %3, i8 7)
-  ret <8 x i16> %4
-}
-declare <8 x i16> @llvm.x86.sse41.mpsadbw(<16 x i8>, <16 x i8>, i8) nounwind readnone
-
-define <8 x i16> @test_packusdw(<4 x i32> %a0, <4 x i32> %a1, <4 x i32> *%a2) {
-; GENERIC-LABEL: test_packusdw:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    packusdw %xmm1, %xmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    packusdw (%rdi), %xmm0 # sched: [7:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SLM-LABEL: test_packusdw:
-; SLM:       # %bb.0:
-; SLM-NEXT:    packusdw %xmm1, %xmm0 # sched: [1:1.00]
-; SLM-NEXT:    packusdw (%rdi), %xmm0 # sched: [4:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_packusdw:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    packusdw %xmm1, %xmm0 # sched: [1:0.50]
-; SANDY-SSE-NEXT:    packusdw (%rdi), %xmm0 # sched: [7:0.50]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_packusdw:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SANDY-NEXT:    vpackusdw (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_packusdw:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    packusdw %xmm1, %xmm0 # sched: [1:1.00]
-; HASWELL-SSE-NEXT:    packusdw (%rdi), %xmm0 # sched: [7:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_packusdw:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0 # sched: [1:1.00]
-; HASWELL-NEXT:    vpackusdw (%rdi), %xmm0, %xmm0 # sched: [7:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_packusdw:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    packusdw %xmm1, %xmm0 # sched: [1:1.00]
-; BROADWELL-SSE-NEXT:    packusdw (%rdi), %xmm0 # sched: [6:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_packusdw:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0 # sched: [1:1.00]
-; BROADWELL-NEXT:    vpackusdw (%rdi), %xmm0, %xmm0 # sched: [6:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_packusdw:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    packusdw %xmm1, %xmm0 # sched: [1:1.00]
-; SKYLAKE-SSE-NEXT:    packusdw (%rdi), %xmm0 # sched: [7:1.00]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_packusdw:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0 # sched: [1:1.00]
-; SKYLAKE-NEXT:    vpackusdw (%rdi), %xmm0, %xmm0 # sched: [7:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_packusdw:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    packusdw %xmm1, %xmm0 # sched: [1:1.00]
-; SKX-SSE-NEXT:    packusdw (%rdi), %xmm0 # sched: [7:1.00]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_packusdw:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0 # sched: [1:1.00]
-; SKX-NEXT:    vpackusdw (%rdi), %xmm0, %xmm0 # sched: [7:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_packusdw:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    packusdw %xmm1, %xmm0 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    packusdw (%rdi), %xmm0 # sched: [7:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_packusdw:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    vpackusdw (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_packusdw:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    packusdw %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    packusdw (%rdi), %xmm0 # sched: [6:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_packusdw:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    vpackusdw (%rdi), %xmm0, %xmm0 # sched: [6:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_packusdw:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    packusdw %xmm1, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    packusdw (%rdi), %xmm0 # sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_packusdw:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vpackusdw (%rdi), %xmm0, %xmm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <8 x i16> @llvm.x86.sse41.packusdw(<4 x i32> %a0, <4 x i32> %a1)
-  %2 = bitcast <8 x i16> %1 to <4 x i32>
-  %3 = load <4 x i32>, <4 x i32> *%a2, align 16
-  %4 = call <8 x i16> @llvm.x86.sse41.packusdw(<4 x i32> %2, <4 x i32> %3)
-  ret <8 x i16> %4
-}
-declare <8 x i16> @llvm.x86.sse41.packusdw(<4 x i32>, <4 x i32>) nounwind readnone
-
-define <16 x i8> @test_pblendvb(<16 x i8> %a0, <16 x i8> %a1, <16 x i8> %a2, <16 x i8> *%a3) {
-; GENERIC-LABEL: test_pblendvb:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    movdqa %xmm0, %xmm3 # sched: [1:0.33]
-; GENERIC-NEXT:    movaps %xmm2, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    pblendvb %xmm0, %xmm1, %xmm3 # sched: [2:1.00]
-; GENERIC-NEXT:    pblendvb %xmm0, (%rdi), %xmm3 # sched: [8:1.00]
-; GENERIC-NEXT:    movdqa %xmm3, %xmm0 # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SLM-LABEL: test_pblendvb:
-; SLM:       # %bb.0:
-; SLM-NEXT:    movdqa %xmm0, %xmm3 # sched: [1:0.50]
-; SLM-NEXT:    movaps %xmm2, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    pblendvb %xmm0, %xmm1, %xmm3 # sched: [1:1.00]
-; SLM-NEXT:    pblendvb %xmm0, (%rdi), %xmm3 # sched: [4:1.00]
-; SLM-NEXT:    movdqa %xmm3, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_pblendvb:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    movdqa %xmm0, %xmm3 # sched: [1:0.33]
-; SANDY-SSE-NEXT:    movaps %xmm2, %xmm0 # sched: [1:1.00]
-; SANDY-SSE-NEXT:    pblendvb %xmm0, %xmm1, %xmm3 # sched: [2:1.00]
-; SANDY-SSE-NEXT:    pblendvb %xmm0, (%rdi), %xmm3 # sched: [8:1.00]
-; SANDY-SSE-NEXT:    movdqa %xmm3, %xmm0 # sched: [1:0.33]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_pblendvb:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vpblendvb %xmm2, %xmm1, %xmm0, %xmm0 # sched: [2:1.00]
-; SANDY-NEXT:    vpblendvb %xmm2, (%rdi), %xmm0, %xmm0 # sched: [8:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_pblendvb:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    movdqa %xmm0, %xmm3 # sched: [1:0.33]
-; HASWELL-SSE-NEXT:    movaps %xmm2, %xmm0 # sched: [1:1.00]
-; HASWELL-SSE-NEXT:    pblendvb %xmm0, %xmm1, %xmm3 # sched: [2:2.00]
-; HASWELL-SSE-NEXT:    pblendvb %xmm0, (%rdi), %xmm3 # sched: [8:2.00]
-; HASWELL-SSE-NEXT:    movdqa %xmm3, %xmm0 # sched: [1:0.33]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_pblendvb:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpblendvb %xmm2, %xmm1, %xmm0, %xmm0 # sched: [2:2.00]
-; HASWELL-NEXT:    vpblendvb %xmm2, (%rdi), %xmm0, %xmm0 # sched: [8:2.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_pblendvb:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    movdqa %xmm0, %xmm3 # sched: [1:0.33]
-; BROADWELL-SSE-NEXT:    movaps %xmm2, %xmm0 # sched: [1:1.00]
-; BROADWELL-SSE-NEXT:    pblendvb %xmm0, %xmm1, %xmm3 # sched: [2:2.00]
-; BROADWELL-SSE-NEXT:    pblendvb %xmm0, (%rdi), %xmm3 # sched: [7:2.00]
-; BROADWELL-SSE-NEXT:    movdqa %xmm3, %xmm0 # sched: [1:0.33]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pblendvb:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpblendvb %xmm2, %xmm1, %xmm0, %xmm0 # sched: [2:2.00]
-; BROADWELL-NEXT:    vpblendvb %xmm2, (%rdi), %xmm0, %xmm0 # sched: [7:2.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_pblendvb:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    movdqa %xmm0, %xmm3 # sched: [1:0.33]
-; SKYLAKE-SSE-NEXT:    movaps %xmm2, %xmm0 # sched: [1:0.33]
-; SKYLAKE-SSE-NEXT:    pblendvb %xmm0, %xmm1, %xmm3 # sched: [2:0.67]
-; SKYLAKE-SSE-NEXT:    pblendvb %xmm0, (%rdi), %xmm3 # sched: [8:0.67]
-; SKYLAKE-SSE-NEXT:    movdqa %xmm3, %xmm0 # sched: [1:0.33]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pblendvb:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpblendvb %xmm2, %xmm1, %xmm0, %xmm0 # sched: [2:0.67]
-; SKYLAKE-NEXT:    vpblendvb %xmm2, (%rdi), %xmm0, %xmm0 # sched: [8:0.67]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_pblendvb:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    movdqa %xmm0, %xmm3 # sched: [1:0.33]
-; SKX-SSE-NEXT:    movaps %xmm2, %xmm0 # sched: [1:0.33]
-; SKX-SSE-NEXT:    pblendvb %xmm0, %xmm1, %xmm3 # sched: [2:0.67]
-; SKX-SSE-NEXT:    pblendvb %xmm0, (%rdi), %xmm3 # sched: [8:0.67]
-; SKX-SSE-NEXT:    movdqa %xmm3, %xmm0 # sched: [1:0.33]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pblendvb:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpblendvb %xmm2, %xmm1, %xmm0, %xmm0 # sched: [2:0.67]
-; SKX-NEXT:    vpblendvb %xmm2, (%rdi), %xmm0, %xmm0 # sched: [8:0.67]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_pblendvb:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    movdqa %xmm0, %xmm3 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    movaps %xmm2, %xmm0 # sched: [1:0.50]
-; BDVER2-SSE-NEXT:    pblendvb %xmm0, %xmm1, %xmm3 # sched: [2:2.00]
-; BDVER2-SSE-NEXT:    pblendvb %xmm0, (%rdi), %xmm3 # sched: [7:2.00]
-; BDVER2-SSE-NEXT:    movdqa %xmm3, %xmm0 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_pblendvb:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vpblendvb %xmm2, %xmm1, %xmm0, %xmm0 # sched: [2:2.00]
-; BDVER2-NEXT:    vpblendvb %xmm2, (%rdi), %xmm0, %xmm0 # sched: [7:2.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_pblendvb:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    movdqa %xmm0, %xmm3 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    movaps %xmm2, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    pblendvb %xmm0, %xmm1, %xmm3 # sched: [2:2.00]
-; BTVER2-SSE-NEXT:    pblendvb %xmm0, (%rdi), %xmm3 # sched: [7:2.00]
-; BTVER2-SSE-NEXT:    movdqa %xmm3, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_pblendvb:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vpblendvb %xmm2, %xmm1, %xmm0, %xmm0 # sched: [2:2.00]
-; BTVER2-NEXT:    vpblendvb %xmm2, (%rdi), %xmm0, %xmm0 # sched: [7:2.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_pblendvb:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    movdqa %xmm0, %xmm3 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    movaps %xmm2, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    pblendvb %xmm0, %xmm1, %xmm3 # sched: [1:1.00]
-; ZNVER1-SSE-NEXT:    pblendvb %xmm0, (%rdi), %xmm3 # sched: [8:1.00]
-; ZNVER1-SSE-NEXT:    movdqa %xmm3, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_pblendvb:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpblendvb %xmm2, %xmm1, %xmm0, %xmm0 # sched: [1:1.00]
-; ZNVER1-NEXT:    vpblendvb %xmm2, (%rdi), %xmm0, %xmm0 # sched: [8:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <16 x i8> @llvm.x86.sse41.pblendvb(<16 x i8> %a0, <16 x i8> %a1, <16 x i8> %a2)
-  %2 = load <16 x i8>, <16 x i8> *%a3, align 16
-  %3 = call <16 x i8> @llvm.x86.sse41.pblendvb(<16 x i8> %1, <16 x i8> %2, <16 x i8> %a2)
-  ret <16 x i8> %3
-}
-declare <16 x i8> @llvm.x86.sse41.pblendvb(<16 x i8>, <16 x i8>, <16 x i8>) nounwind readnone
-
-define <8 x i16> @test_pblendw(<8 x i16> %a0, <8 x i16> %a1, <8 x i16> *%a2) {
-; GENERIC-LABEL: test_pblendw:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3],xmm0[4],xmm1[5],xmm0[6],xmm1[7] sched: [1:0.50]
-; GENERIC-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1],mem[2,3],xmm1[4,5,6],mem[7] sched: [7:0.50]
-; GENERIC-NEXT:    paddw %xmm1, %xmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SLM-LABEL: test_pblendw:
-; SLM:       # %bb.0:
-; SLM-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3],xmm0[4],xmm1[5],xmm0[6],xmm1[7] sched: [1:1.00]
-; SLM-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1],mem[2,3],xmm1[4,5,6],mem[7] sched: [4:1.00]
-; SLM-NEXT:    paddw %xmm1, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_pblendw:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3],xmm0[4],xmm1[5],xmm0[6],xmm1[7] sched: [1:0.50]
-; SANDY-SSE-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1],mem[2,3],xmm1[4,5,6],mem[7] sched: [7:0.50]
-; SANDY-SSE-NEXT:    paddw %xmm1, %xmm0 # sched: [1:0.50]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_pblendw:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3],xmm0[4],xmm1[5],xmm0[6],xmm1[7] sched: [1:0.50]
-; SANDY-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],mem[2,3],xmm1[4,5,6],mem[7] sched: [7:0.50]
-; SANDY-NEXT:    vpaddw %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_pblendw:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3],xmm0[4],xmm1[5],xmm0[6],xmm1[7] sched: [1:1.00]
-; HASWELL-SSE-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1],mem[2,3],xmm1[4,5,6],mem[7] sched: [7:1.00]
-; HASWELL-SSE-NEXT:    paddw %xmm1, %xmm0 # sched: [1:0.50]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_pblendw:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3],xmm0[4],xmm1[5],xmm0[6],xmm1[7] sched: [1:1.00]
-; HASWELL-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],mem[2,3],xmm1[4,5,6],mem[7] sched: [7:1.00]
-; HASWELL-NEXT:    vpaddw %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_pblendw:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3],xmm0[4],xmm1[5],xmm0[6],xmm1[7] sched: [1:1.00]
-; BROADWELL-SSE-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1],mem[2,3],xmm1[4,5,6],mem[7] sched: [6:1.00]
-; BROADWELL-SSE-NEXT:    paddw %xmm1, %xmm0 # sched: [1:0.50]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pblendw:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3],xmm0[4],xmm1[5],xmm0[6],xmm1[7] sched: [1:1.00]
-; BROADWELL-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],mem[2,3],xmm1[4,5,6],mem[7] sched: [6:1.00]
-; BROADWELL-NEXT:    vpaddw %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_pblendw:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3],xmm0[4],xmm1[5],xmm0[6],xmm1[7] sched: [1:1.00]
-; SKYLAKE-SSE-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1],mem[2,3],xmm1[4,5,6],mem[7] sched: [7:1.00]
-; SKYLAKE-SSE-NEXT:    paddw %xmm1, %xmm0 # sched: [1:0.33]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pblendw:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3],xmm0[4],xmm1[5],xmm0[6],xmm1[7] sched: [1:1.00]
-; SKYLAKE-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],mem[2,3],xmm1[4,5,6],mem[7] sched: [7:1.00]
-; SKYLAKE-NEXT:    vpaddw %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_pblendw:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3],xmm0[4],xmm1[5],xmm0[6],xmm1[7] sched: [1:1.00]
-; SKX-SSE-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1],mem[2,3],xmm1[4,5,6],mem[7] sched: [7:1.00]
-; SKX-SSE-NEXT:    paddw %xmm1, %xmm0 # sched: [1:0.33]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pblendw:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3],xmm0[4],xmm1[5],xmm0[6],xmm1[7] sched: [1:1.00]
-; SKX-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],mem[2,3],xmm1[4,5,6],mem[7] sched: [7:1.00]
-; SKX-NEXT:    vpaddw %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_pblendw:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3],xmm0[4],xmm1[5],xmm0[6],xmm1[7] sched: [2:0.50]
-; BDVER2-SSE-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1],mem[2,3],xmm1[4,5,6],mem[7] sched: [7:0.50]
-; BDVER2-SSE-NEXT:    paddw %xmm1, %xmm0 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_pblendw:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3],xmm0[4],xmm1[5],xmm0[6],xmm1[7] sched: [2:0.50]
-; BDVER2-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],mem[2,3],xmm1[4,5,6],mem[7] sched: [7:0.50]
-; BDVER2-NEXT:    vpaddw %xmm1, %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_pblendw:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3],xmm0[4],xmm1[5],xmm0[6],xmm1[7] sched: [1:0.50]
-; BTVER2-SSE-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1],mem[2,3],xmm1[4,5,6],mem[7] sched: [6:1.00]
-; BTVER2-SSE-NEXT:    paddw %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_pblendw:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3],xmm0[4],xmm1[5],xmm0[6],xmm1[7] sched: [1:0.50]
-; BTVER2-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],mem[2,3],xmm1[4,5,6],mem[7] sched: [6:1.00]
-; BTVER2-NEXT:    vpaddw %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_pblendw:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3],xmm0[4],xmm1[5],xmm0[6],xmm1[7] sched: [1:0.33]
-; ZNVER1-SSE-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1],mem[2,3],xmm1[4,5,6],mem[7] sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    paddw %xmm1, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_pblendw:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3],xmm0[4],xmm1[5],xmm0[6],xmm1[7] sched: [1:0.33]
-; ZNVER1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],mem[2,3],xmm1[4,5,6],mem[7] sched: [8:0.50]
-; ZNVER1-NEXT:    vpaddw %xmm1, %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = shufflevector <8 x i16> %a0, <8 x i16> %a1, <8 x i32> <i32 0, i32 9, i32 2, i32 11, i32 4, i32 13, i32 6, i32 15>
-  %2 = load <8 x i16>, <8 x i16> *%a2, align 16
-  %3 = shufflevector <8 x i16> %a1, <8 x i16> %2, <8 x i32> <i32 0, i32 1, i32 10, i32 11, i32 4, i32 5, i32 6, i32 15>
-  %4 = add <8 x i16> %1, %3
-  ret <8 x i16> %4
-}
-
-define <2 x i64> @test_pcmpeqq(<2 x i64> %a0, <2 x i64> %a1, <2 x i64> *%a2) {
-; GENERIC-LABEL: test_pcmpeqq:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    pcmpeqq %xmm1, %xmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    pcmpeqq (%rdi), %xmm0 # sched: [7:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SLM-LABEL: test_pcmpeqq:
-; SLM:       # %bb.0:
-; SLM-NEXT:    pcmpeqq %xmm1, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    pcmpeqq (%rdi), %xmm0 # sched: [4:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_pcmpeqq:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    pcmpeqq %xmm1, %xmm0 # sched: [1:0.50]
-; SANDY-SSE-NEXT:    pcmpeqq (%rdi), %xmm0 # sched: [7:0.50]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_pcmpeqq:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vpcmpeqq %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SANDY-NEXT:    vpcmpeqq (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_pcmpeqq:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    pcmpeqq %xmm1, %xmm0 # sched: [1:0.50]
-; HASWELL-SSE-NEXT:    pcmpeqq (%rdi), %xmm0 # sched: [7:0.50]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_pcmpeqq:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpcmpeqq %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; HASWELL-NEXT:    vpcmpeqq (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_pcmpeqq:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    pcmpeqq %xmm1, %xmm0 # sched: [1:0.50]
-; BROADWELL-SSE-NEXT:    pcmpeqq (%rdi), %xmm0 # sched: [6:0.50]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pcmpeqq:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpcmpeqq %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    vpcmpeqq (%rdi), %xmm0, %xmm0 # sched: [6:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_pcmpeqq:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    pcmpeqq %xmm1, %xmm0 # sched: [1:0.50]
-; SKYLAKE-SSE-NEXT:    pcmpeqq (%rdi), %xmm0 # sched: [7:0.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pcmpeqq:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpcmpeqq %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SKYLAKE-NEXT:    vpcmpeqq (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_pcmpeqq:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    pcmpeqq %xmm1, %xmm0 # sched: [1:0.50]
-; SKX-SSE-NEXT:    pcmpeqq (%rdi), %xmm0 # sched: [7:0.50]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pcmpeqq:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpcmpeqq %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SKX-NEXT:    vpcmpeqq (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_pcmpeqq:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    pcmpeqq %xmm1, %xmm0 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    pcmpeqq (%rdi), %xmm0 # sched: [7:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_pcmpeqq:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vpcmpeqq %xmm1, %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    vpcmpeqq (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_pcmpeqq:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    pcmpeqq %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    pcmpeqq (%rdi), %xmm0 # sched: [6:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_pcmpeqq:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vpcmpeqq %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    vpcmpeqq (%rdi), %xmm0, %xmm0 # sched: [6:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_pcmpeqq:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    pcmpeqq %xmm1, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    pcmpeqq (%rdi), %xmm0 # sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_pcmpeqq:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpcmpeqq %xmm1, %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vpcmpeqq (%rdi), %xmm0, %xmm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = icmp eq <2 x i64> %a0, %a1
-  %2 = sext <2 x i1> %1 to <2 x i64>
-  %3 = load <2 x i64>, <2 x i64>*%a2, align 16
-  %4 = icmp eq <2 x i64> %2, %3
-  %5 = sext <2 x i1> %4 to <2 x i64>
-  ret <2 x i64> %5
-}
-
-define i32 @test_pextrb(<16 x i8> %a0, i8 *%a1) {
-; GENERIC-LABEL: test_pextrb:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    pextrb $3, %xmm0, %eax # sched: [3:1.00]
-; GENERIC-NEXT:    pextrb $1, %xmm0, (%rdi) # sched: [5:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SLM-LABEL: test_pextrb:
-; SLM:       # %bb.0:
-; SLM-NEXT:    pextrb $3, %xmm0, %eax # sched: [1:1.00]
-; SLM-NEXT:    pextrb $1, %xmm0, (%rdi) # sched: [4:2.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_pextrb:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    pextrb $3, %xmm0, %eax # sched: [3:1.00]
-; SANDY-SSE-NEXT:    pextrb $1, %xmm0, (%rdi) # sched: [5:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_pextrb:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vpextrb $3, %xmm0, %eax # sched: [3:1.00]
-; SANDY-NEXT:    vpextrb $1, %xmm0, (%rdi) # sched: [5:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_pextrb:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    pextrb $3, %xmm0, %eax # sched: [2:1.00]
-; HASWELL-SSE-NEXT:    pextrb $1, %xmm0, (%rdi) # sched: [2:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_pextrb:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpextrb $3, %xmm0, %eax # sched: [2:1.00]
-; HASWELL-NEXT:    vpextrb $1, %xmm0, (%rdi) # sched: [2:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_pextrb:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    pextrb $3, %xmm0, %eax # sched: [2:1.00]
-; BROADWELL-SSE-NEXT:    pextrb $1, %xmm0, (%rdi) # sched: [2:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pextrb:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpextrb $3, %xmm0, %eax # sched: [2:1.00]
-; BROADWELL-NEXT:    vpextrb $1, %xmm0, (%rdi) # sched: [2:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_pextrb:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    pextrb $3, %xmm0, %eax # sched: [3:1.00]
-; SKYLAKE-SSE-NEXT:    pextrb $1, %xmm0, (%rdi) # sched: [2:1.00]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pextrb:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpextrb $3, %xmm0, %eax # sched: [3:1.00]
-; SKYLAKE-NEXT:    vpextrb $1, %xmm0, (%rdi) # sched: [2:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_pextrb:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    pextrb $3, %xmm0, %eax # sched: [3:1.00]
-; SKX-SSE-NEXT:    pextrb $1, %xmm0, (%rdi) # sched: [2:1.00]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pextrb:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpextrb $3, %xmm0, %eax # sched: [3:1.00]
-; SKX-NEXT:    vpextrb $1, %xmm0, (%rdi) # sched: [2:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_pextrb:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    pextrb $3, %xmm0, %eax # sched: [13:1.00]
-; BDVER2-SSE-NEXT:    pextrb $1, %xmm0, (%rdi) # sched: [13:1.00]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_pextrb:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vpextrb $3, %xmm0, %eax # sched: [13:1.00]
-; BDVER2-NEXT:    vpextrb $1, %xmm0, (%rdi) # sched: [13:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_pextrb:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    pextrb $3, %xmm0, %eax # sched: [3:1.00]
-; BTVER2-SSE-NEXT:    pextrb $1, %xmm0, (%rdi) # sched: [3:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_pextrb:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vpextrb $3, %xmm0, %eax # sched: [3:1.00]
-; BTVER2-NEXT:    vpextrb $1, %xmm0, (%rdi) # sched: [3:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_pextrb:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    pextrb $3, %xmm0, %eax # sched: [2:2.00]
-; ZNVER1-SSE-NEXT:    pextrb $1, %xmm0, (%rdi) # sched: [5:3.00]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_pextrb:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpextrb $3, %xmm0, %eax # sched: [2:2.00]
-; ZNVER1-NEXT:    vpextrb $1, %xmm0, (%rdi) # sched: [5:3.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = extractelement <16 x i8> %a0, i32 3
-  %2 = extractelement <16 x i8> %a0, i32 1
-  store i8 %2, i8 *%a1
-  %3 = zext i8 %1 to i32
-  ret i32 %3
-}
-
-define i32 @test_pextrd(<4 x i32> %a0, i32 *%a1) {
-; GENERIC-LABEL: test_pextrd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    paddd %xmm0, %xmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    pextrd $3, %xmm0, %eax # sched: [3:1.00]
-; GENERIC-NEXT:    pextrd $1, %xmm0, (%rdi) # sched: [5:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SLM-LABEL: test_pextrd:
-; SLM:       # %bb.0:
-; SLM-NEXT:    paddd %xmm0, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    pextrd $3, %xmm0, %eax # sched: [1:1.00]
-; SLM-NEXT:    pextrd $1, %xmm0, (%rdi) # sched: [4:2.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_pextrd:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    paddd %xmm0, %xmm0 # sched: [1:0.50]
-; SANDY-SSE-NEXT:    pextrd $3, %xmm0, %eax # sched: [3:1.00]
-; SANDY-SSE-NEXT:    pextrd $1, %xmm0, (%rdi) # sched: [5:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_pextrd:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vpaddd %xmm0, %xmm0, %xmm0 # sched: [1:0.50]
-; SANDY-NEXT:    vpextrd $3, %xmm0, %eax # sched: [3:1.00]
-; SANDY-NEXT:    vpextrd $1, %xmm0, (%rdi) # sched: [5:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_pextrd:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    paddd %xmm0, %xmm0 # sched: [1:0.50]
-; HASWELL-SSE-NEXT:    pextrd $3, %xmm0, %eax # sched: [2:1.00]
-; HASWELL-SSE-NEXT:    pextrd $1, %xmm0, (%rdi) # sched: [2:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_pextrd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpaddd %xmm0, %xmm0, %xmm0 # sched: [1:0.50]
-; HASWELL-NEXT:    vpextrd $3, %xmm0, %eax # sched: [2:1.00]
-; HASWELL-NEXT:    vpextrd $1, %xmm0, (%rdi) # sched: [2:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_pextrd:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    paddd %xmm0, %xmm0 # sched: [1:0.50]
-; BROADWELL-SSE-NEXT:    pextrd $3, %xmm0, %eax # sched: [2:1.00]
-; BROADWELL-SSE-NEXT:    pextrd $1, %xmm0, (%rdi) # sched: [2:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pextrd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpaddd %xmm0, %xmm0, %xmm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    vpextrd $3, %xmm0, %eax # sched: [2:1.00]
-; BROADWELL-NEXT:    vpextrd $1, %xmm0, (%rdi) # sched: [2:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_pextrd:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    paddd %xmm0, %xmm0 # sched: [1:0.33]
-; SKYLAKE-SSE-NEXT:    pextrd $3, %xmm0, %eax # sched: [3:1.00]
-; SKYLAKE-SSE-NEXT:    pextrd $1, %xmm0, (%rdi) # sched: [2:1.00]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pextrd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpaddd %xmm0, %xmm0, %xmm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    vpextrd $3, %xmm0, %eax # sched: [3:1.00]
-; SKYLAKE-NEXT:    vpextrd $1, %xmm0, (%rdi) # sched: [2:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_pextrd:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    paddd %xmm0, %xmm0 # sched: [1:0.33]
-; SKX-SSE-NEXT:    pextrd $3, %xmm0, %eax # sched: [3:1.00]
-; SKX-SSE-NEXT:    pextrd $1, %xmm0, (%rdi) # sched: [2:1.00]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pextrd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpaddd %xmm0, %xmm0, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    vpextrd $3, %xmm0, %eax # sched: [3:1.00]
-; SKX-NEXT:    vpextrd $1, %xmm0, (%rdi) # sched: [2:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_pextrd:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    paddd %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    pextrd $3, %xmm0, %eax # sched: [13:1.00]
-; BDVER2-SSE-NEXT:    pextrd $1, %xmm0, (%rdi) # sched: [13:1.00]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_pextrd:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vpaddd %xmm0, %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    vpextrd $3, %xmm0, %eax # sched: [13:1.00]
-; BDVER2-NEXT:    vpextrd $1, %xmm0, (%rdi) # sched: [13:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_pextrd:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    paddd %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    pextrd $3, %xmm0, %eax # sched: [3:1.00]
-; BTVER2-SSE-NEXT:    pextrd $1, %xmm0, (%rdi) # sched: [3:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_pextrd:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vpaddd %xmm0, %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    vpextrd $3, %xmm0, %eax # sched: [3:1.00]
-; BTVER2-NEXT:    vpextrd $1, %xmm0, (%rdi) # sched: [3:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_pextrd:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    paddd %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    pextrd $3, %xmm0, %eax # sched: [2:2.00]
-; ZNVER1-SSE-NEXT:    pextrd $1, %xmm0, (%rdi) # sched: [5:3.00]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_pextrd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpaddd %xmm0, %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vpextrd $3, %xmm0, %eax # sched: [2:2.00]
-; ZNVER1-NEXT:    vpextrd $1, %xmm0, (%rdi) # sched: [5:3.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = add <4 x i32> %a0, %a0
-  %2 = extractelement <4 x i32> %1, i32 3
-  %3 = extractelement <4 x i32> %1, i32 1
-  store i32 %3, i32 *%a1
-  ret i32 %2
-}
-
-define i64 @test_pextrq(<2 x i64> %a0, <2 x i64> %a1, i64 *%a2) {
-; GENERIC-LABEL: test_pextrq:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    pextrq $1, %xmm0, %rax # sched: [3:1.00]
-; GENERIC-NEXT:    pextrq $1, %xmm0, (%rdi) # sched: [5:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SLM-LABEL: test_pextrq:
-; SLM:       # %bb.0:
-; SLM-NEXT:    pextrq $1, %xmm0, %rax # sched: [1:1.00]
-; SLM-NEXT:    pextrq $1, %xmm0, (%rdi) # sched: [4:2.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_pextrq:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    pextrq $1, %xmm0, %rax # sched: [3:1.00]
-; SANDY-SSE-NEXT:    pextrq $1, %xmm0, (%rdi) # sched: [5:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_pextrq:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vpextrq $1, %xmm0, %rax # sched: [3:1.00]
-; SANDY-NEXT:    vpextrq $1, %xmm0, (%rdi) # sched: [5:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_pextrq:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    pextrq $1, %xmm0, %rax # sched: [2:1.00]
-; HASWELL-SSE-NEXT:    pextrq $1, %xmm0, (%rdi) # sched: [2:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_pextrq:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpextrq $1, %xmm0, %rax # sched: [2:1.00]
-; HASWELL-NEXT:    vpextrq $1, %xmm0, (%rdi) # sched: [2:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_pextrq:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    pextrq $1, %xmm0, %rax # sched: [2:1.00]
-; BROADWELL-SSE-NEXT:    pextrq $1, %xmm0, (%rdi) # sched: [2:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pextrq:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpextrq $1, %xmm0, %rax # sched: [2:1.00]
-; BROADWELL-NEXT:    vpextrq $1, %xmm0, (%rdi) # sched: [2:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_pextrq:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    pextrq $1, %xmm0, %rax # sched: [3:1.00]
-; SKYLAKE-SSE-NEXT:    pextrq $1, %xmm0, (%rdi) # sched: [2:1.00]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pextrq:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpextrq $1, %xmm0, %rax # sched: [3:1.00]
-; SKYLAKE-NEXT:    vpextrq $1, %xmm0, (%rdi) # sched: [2:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_pextrq:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    pextrq $1, %xmm0, %rax # sched: [3:1.00]
-; SKX-SSE-NEXT:    pextrq $1, %xmm0, (%rdi) # sched: [2:1.00]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pextrq:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpextrq $1, %xmm0, %rax # sched: [3:1.00]
-; SKX-NEXT:    vpextrq $1, %xmm0, (%rdi) # sched: [2:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_pextrq:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    pextrq $1, %xmm0, %rax # sched: [13:1.00]
-; BDVER2-SSE-NEXT:    pextrq $1, %xmm0, (%rdi) # sched: [13:1.00]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_pextrq:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vpextrq $1, %xmm0, %rax # sched: [13:1.00]
-; BDVER2-NEXT:    vpextrq $1, %xmm0, (%rdi) # sched: [13:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_pextrq:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    pextrq $1, %xmm0, %rax # sched: [3:1.00]
-; BTVER2-SSE-NEXT:    pextrq $1, %xmm0, (%rdi) # sched: [3:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_pextrq:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vpextrq $1, %xmm0, %rax # sched: [3:1.00]
-; BTVER2-NEXT:    vpextrq $1, %xmm0, (%rdi) # sched: [3:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_pextrq:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    pextrq $1, %xmm0, %rax # sched: [2:2.00]
-; ZNVER1-SSE-NEXT:    pextrq $1, %xmm0, (%rdi) # sched: [5:3.00]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_pextrq:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpextrq $1, %xmm0, %rax # sched: [2:2.00]
-; ZNVER1-NEXT:    vpextrq $1, %xmm0, (%rdi) # sched: [5:3.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = extractelement <2 x i64> %a0, i32 1
-  %2 = extractelement <2 x i64> %a0, i32 1
-  store i64 %2, i64 *%a2
-  ret i64 %1
-}
-
-define i32 @test_pextrw(<8 x i16> %a0, i16 *%a1) {
-; GENERIC-LABEL: test_pextrw:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    pextrw $3, %xmm0, %eax # sched: [3:1.00]
-; GENERIC-NEXT:    pextrw $1, %xmm0, (%rdi) # sched: [5:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SLM-LABEL: test_pextrw:
-; SLM:       # %bb.0:
-; SLM-NEXT:    pextrw $3, %xmm0, %eax # sched: [1:1.00]
-; SLM-NEXT:    pextrw $1, %xmm0, (%rdi) # sched: [4:2.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_pextrw:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    pextrw $3, %xmm0, %eax # sched: [3:1.00]
-; SANDY-SSE-NEXT:    pextrw $1, %xmm0, (%rdi) # sched: [5:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_pextrw:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vpextrw $3, %xmm0, %eax # sched: [3:1.00]
-; SANDY-NEXT:    vpextrw $1, %xmm0, (%rdi) # sched: [5:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_pextrw:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    pextrw $3, %xmm0, %eax # sched: [2:1.00]
-; HASWELL-SSE-NEXT:    pextrw $1, %xmm0, (%rdi) # sched: [2:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_pextrw:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpextrw $3, %xmm0, %eax # sched: [2:1.00]
-; HASWELL-NEXT:    vpextrw $1, %xmm0, (%rdi) # sched: [2:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_pextrw:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    pextrw $3, %xmm0, %eax # sched: [2:1.00]
-; BROADWELL-SSE-NEXT:    pextrw $1, %xmm0, (%rdi) # sched: [2:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pextrw:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpextrw $3, %xmm0, %eax # sched: [2:1.00]
-; BROADWELL-NEXT:    vpextrw $1, %xmm0, (%rdi) # sched: [2:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_pextrw:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    pextrw $3, %xmm0, %eax # sched: [3:1.00]
-; SKYLAKE-SSE-NEXT:    pextrw $1, %xmm0, (%rdi) # sched: [2:1.00]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pextrw:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpextrw $3, %xmm0, %eax # sched: [3:1.00]
-; SKYLAKE-NEXT:    vpextrw $1, %xmm0, (%rdi) # sched: [2:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_pextrw:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    pextrw $3, %xmm0, %eax # sched: [3:1.00]
-; SKX-SSE-NEXT:    pextrw $1, %xmm0, (%rdi) # sched: [2:1.00]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pextrw:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpextrw $3, %xmm0, %eax # sched: [3:1.00]
-; SKX-NEXT:    vpextrw $1, %xmm0, (%rdi) # sched: [2:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_pextrw:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    pextrw $3, %xmm0, %eax # sched: [13:1.00]
-; BDVER2-SSE-NEXT:    pextrw $1, %xmm0, (%rdi) # sched: [13:1.00]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_pextrw:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vpextrw $3, %xmm0, %eax # sched: [13:1.00]
-; BDVER2-NEXT:    vpextrw $1, %xmm0, (%rdi) # sched: [13:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_pextrw:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    pextrw $3, %xmm0, %eax # sched: [3:1.00]
-; BTVER2-SSE-NEXT:    pextrw $1, %xmm0, (%rdi) # sched: [3:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_pextrw:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vpextrw $3, %xmm0, %eax # sched: [3:1.00]
-; BTVER2-NEXT:    vpextrw $1, %xmm0, (%rdi) # sched: [3:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_pextrw:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    pextrw $3, %xmm0, %eax # sched: [2:2.00]
-; ZNVER1-SSE-NEXT:    pextrw $1, %xmm0, (%rdi) # sched: [5:3.00]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_pextrw:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpextrw $3, %xmm0, %eax # sched: [2:2.00]
-; ZNVER1-NEXT:    vpextrw $1, %xmm0, (%rdi) # sched: [5:3.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = extractelement <8 x i16> %a0, i32 3
-  %2 = extractelement <8 x i16> %a0, i32 1
-  store i16 %2, i16 *%a1
-  %3 = zext i16 %1 to i32
-  ret i32 %3
-}
-
-define <8 x i16> @test_phminposuw(<8 x i16> *%a0) {
-; GENERIC-LABEL: test_phminposuw:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    phminposuw (%rdi), %xmm0 # sched: [11:1.00]
-; GENERIC-NEXT:    phminposuw %xmm0, %xmm0 # sched: [5:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SLM-LABEL: test_phminposuw:
-; SLM:       # %bb.0:
-; SLM-NEXT:    phminposuw (%rdi), %xmm0 # sched: [7:1.00]
-; SLM-NEXT:    phminposuw %xmm0, %xmm0 # sched: [4:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_phminposuw:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    phminposuw (%rdi), %xmm0 # sched: [11:1.00]
-; SANDY-SSE-NEXT:    phminposuw %xmm0, %xmm0 # sched: [5:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_phminposuw:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vphminposuw (%rdi), %xmm0 # sched: [11:1.00]
-; SANDY-NEXT:    vphminposuw %xmm0, %xmm0 # sched: [5:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_phminposuw:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    phminposuw (%rdi), %xmm0 # sched: [11:1.00]
-; HASWELL-SSE-NEXT:    phminposuw %xmm0, %xmm0 # sched: [5:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_phminposuw:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vphminposuw (%rdi), %xmm0 # sched: [11:1.00]
-; HASWELL-NEXT:    vphminposuw %xmm0, %xmm0 # sched: [5:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_phminposuw:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    phminposuw (%rdi), %xmm0 # sched: [10:1.00]
-; BROADWELL-SSE-NEXT:    phminposuw %xmm0, %xmm0 # sched: [5:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_phminposuw:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vphminposuw (%rdi), %xmm0 # sched: [10:1.00]
-; BROADWELL-NEXT:    vphminposuw %xmm0, %xmm0 # sched: [5:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_phminposuw:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    phminposuw (%rdi), %xmm0 # sched: [10:0.50]
-; SKYLAKE-SSE-NEXT:    phminposuw %xmm0, %xmm0 # sched: [4:0.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_phminposuw:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vphminposuw (%rdi), %xmm0 # sched: [10:0.50]
-; SKYLAKE-NEXT:    vphminposuw %xmm0, %xmm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_phminposuw:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    phminposuw (%rdi), %xmm0 # sched: [10:1.00]
-; SKX-SSE-NEXT:    phminposuw %xmm0, %xmm0 # sched: [4:1.00]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_phminposuw:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vphminposuw (%rdi), %xmm0 # sched: [10:1.00]
-; SKX-NEXT:    vphminposuw %xmm0, %xmm0 # sched: [4:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_phminposuw:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    phminposuw (%rdi), %xmm0 # sched: [9:1.00]
-; BDVER2-SSE-NEXT:    phminposuw %xmm0, %xmm0 # sched: [4:1.00]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_phminposuw:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vphminposuw (%rdi), %xmm0 # sched: [9:1.00]
-; BDVER2-NEXT:    vphminposuw %xmm0, %xmm0 # sched: [4:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_phminposuw:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    phminposuw (%rdi), %xmm0 # sched: [7:1.00]
-; BTVER2-SSE-NEXT:    phminposuw %xmm0, %xmm0 # sched: [2:0.50]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_phminposuw:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vphminposuw (%rdi), %xmm0 # sched: [7:1.00]
-; BTVER2-NEXT:    vphminposuw %xmm0, %xmm0 # sched: [2:0.50]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_phminposuw:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    phminposuw (%rdi), %xmm0 # sched: [11:1.00]
-; ZNVER1-SSE-NEXT:    phminposuw %xmm0, %xmm0 # sched: [4:1.00]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_phminposuw:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vphminposuw (%rdi), %xmm0 # sched: [11:1.00]
-; ZNVER1-NEXT:    vphminposuw %xmm0, %xmm0 # sched: [4:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = load <8 x i16>, <8 x i16> *%a0, align 16
-  %2 = call <8 x i16> @llvm.x86.sse41.phminposuw(<8 x i16> %1)
-  %3 = call <8 x i16> @llvm.x86.sse41.phminposuw(<8 x i16> %2)
-  ret <8 x i16> %3
-}
-declare <8 x i16> @llvm.x86.sse41.phminposuw(<8 x i16>) nounwind readnone
-
-define <16 x i8> @test_pinsrb(<16 x i8> %a0, i8 %a1, i8 *%a2) {
-; GENERIC-LABEL: test_pinsrb:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    pinsrb $1, %edi, %xmm0 # sched: [2:1.00]
-; GENERIC-NEXT:    pinsrb $3, (%rsi), %xmm0 # sched: [7:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SLM-LABEL: test_pinsrb:
-; SLM:       # %bb.0:
-; SLM-NEXT:    pinsrb $1, %edi, %xmm0 # sched: [1:1.00]
-; SLM-NEXT:    pinsrb $3, (%rsi), %xmm0 # sched: [4:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_pinsrb:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    pinsrb $1, %edi, %xmm0 # sched: [2:1.00]
-; SANDY-SSE-NEXT:    pinsrb $3, (%rsi), %xmm0 # sched: [7:0.50]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_pinsrb:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vpinsrb $1, %edi, %xmm0, %xmm0 # sched: [2:1.00]
-; SANDY-NEXT:    vpinsrb $3, (%rsi), %xmm0, %xmm0 # sched: [7:0.50]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_pinsrb:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    pinsrb $1, %edi, %xmm0 # sched: [2:2.00]
-; HASWELL-SSE-NEXT:    pinsrb $3, (%rsi), %xmm0 # sched: [6:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_pinsrb:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpinsrb $1, %edi, %xmm0, %xmm0 # sched: [2:2.00]
-; HASWELL-NEXT:    vpinsrb $3, (%rsi), %xmm0, %xmm0 # sched: [6:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_pinsrb:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    pinsrb $1, %edi, %xmm0 # sched: [2:2.00]
-; BROADWELL-SSE-NEXT:    pinsrb $3, (%rsi), %xmm0 # sched: [6:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pinsrb:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpinsrb $1, %edi, %xmm0, %xmm0 # sched: [2:2.00]
-; BROADWELL-NEXT:    vpinsrb $3, (%rsi), %xmm0, %xmm0 # sched: [6:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_pinsrb:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    pinsrb $1, %edi, %xmm0 # sched: [2:2.00]
-; SKYLAKE-SSE-NEXT:    pinsrb $3, (%rsi), %xmm0 # sched: [6:1.00]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pinsrb:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpinsrb $1, %edi, %xmm0, %xmm0 # sched: [2:2.00]
-; SKYLAKE-NEXT:    vpinsrb $3, (%rsi), %xmm0, %xmm0 # sched: [6:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_pinsrb:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    pinsrb $1, %edi, %xmm0 # sched: [2:2.00]
-; SKX-SSE-NEXT:    pinsrb $3, (%rsi), %xmm0 # sched: [6:1.00]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pinsrb:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpinsrb $1, %edi, %xmm0, %xmm0 # sched: [2:2.00]
-; SKX-NEXT:    vpinsrb $3, (%rsi), %xmm0, %xmm0 # sched: [6:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_pinsrb:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    pinsrb $1, %edi, %xmm0 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    pinsrb $3, (%rsi), %xmm0 # sched: [6:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_pinsrb:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vpinsrb $1, %edi, %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    vpinsrb $3, (%rsi), %xmm0, %xmm0 # sched: [6:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_pinsrb:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    pinsrb $1, %edi, %xmm0 # sched: [7:0.50]
-; BTVER2-SSE-NEXT:    pinsrb $3, (%rsi), %xmm0 # sched: [4:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_pinsrb:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vpinsrb $1, %edi, %xmm0, %xmm0 # sched: [7:0.50]
-; BTVER2-NEXT:    vpinsrb $3, (%rsi), %xmm0, %xmm0 # sched: [4:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_pinsrb:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    pinsrb $1, %edi, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    pinsrb $3, (%rsi), %xmm0 # sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_pinsrb:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpinsrb $1, %edi, %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vpinsrb $3, (%rsi), %xmm0, %xmm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = insertelement <16 x i8> %a0, i8 %a1, i32 1
-  %2 = load i8, i8 *%a2
-  %3 = insertelement <16 x i8> %1, i8 %2, i32 3
-  ret <16 x i8> %3
-}
-
-define <4 x i32> @test_pinsrd(<4 x i32> %a0, i32 %a1, i32 *%a2) {
-; GENERIC-LABEL: test_pinsrd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    pinsrd $1, %edi, %xmm0 # sched: [2:1.00]
-; GENERIC-NEXT:    pinsrd $3, (%rsi), %xmm0 # sched: [7:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SLM-LABEL: test_pinsrd:
-; SLM:       # %bb.0:
-; SLM-NEXT:    pinsrd $1, %edi, %xmm0 # sched: [1:1.00]
-; SLM-NEXT:    pinsrd $3, (%rsi), %xmm0 # sched: [4:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_pinsrd:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    pinsrd $1, %edi, %xmm0 # sched: [2:1.00]
-; SANDY-SSE-NEXT:    pinsrd $3, (%rsi), %xmm0 # sched: [7:0.50]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_pinsrd:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vpinsrd $1, %edi, %xmm0, %xmm0 # sched: [2:1.00]
-; SANDY-NEXT:    vpinsrd $3, (%rsi), %xmm0, %xmm0 # sched: [7:0.50]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_pinsrd:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    pinsrd $1, %edi, %xmm0 # sched: [2:2.00]
-; HASWELL-SSE-NEXT:    pinsrd $3, (%rsi), %xmm0 # sched: [6:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_pinsrd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpinsrd $1, %edi, %xmm0, %xmm0 # sched: [2:2.00]
-; HASWELL-NEXT:    vpinsrd $3, (%rsi), %xmm0, %xmm0 # sched: [6:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_pinsrd:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    pinsrd $1, %edi, %xmm0 # sched: [2:2.00]
-; BROADWELL-SSE-NEXT:    pinsrd $3, (%rsi), %xmm0 # sched: [6:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pinsrd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpinsrd $1, %edi, %xmm0, %xmm0 # sched: [2:2.00]
-; BROADWELL-NEXT:    vpinsrd $3, (%rsi), %xmm0, %xmm0 # sched: [6:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_pinsrd:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    pinsrd $1, %edi, %xmm0 # sched: [2:2.00]
-; SKYLAKE-SSE-NEXT:    pinsrd $3, (%rsi), %xmm0 # sched: [6:1.00]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pinsrd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpinsrd $1, %edi, %xmm0, %xmm0 # sched: [2:2.00]
-; SKYLAKE-NEXT:    vpinsrd $3, (%rsi), %xmm0, %xmm0 # sched: [6:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_pinsrd:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    pinsrd $1, %edi, %xmm0 # sched: [2:2.00]
-; SKX-SSE-NEXT:    pinsrd $3, (%rsi), %xmm0 # sched: [6:1.00]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pinsrd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpinsrd $1, %edi, %xmm0, %xmm0 # sched: [2:2.00]
-; SKX-NEXT:    vpinsrd $3, (%rsi), %xmm0, %xmm0 # sched: [6:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_pinsrd:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    pinsrd $1, %edi, %xmm0 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    pinsrd $3, (%rsi), %xmm0 # sched: [6:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_pinsrd:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vpinsrd $1, %edi, %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    vpinsrd $3, (%rsi), %xmm0, %xmm0 # sched: [6:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_pinsrd:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    pinsrd $1, %edi, %xmm0 # sched: [7:0.50]
-; BTVER2-SSE-NEXT:    pinsrd $3, (%rsi), %xmm0 # sched: [4:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_pinsrd:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vpinsrd $1, %edi, %xmm0, %xmm0 # sched: [7:0.50]
-; BTVER2-NEXT:    vpinsrd $3, (%rsi), %xmm0, %xmm0 # sched: [4:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_pinsrd:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    pinsrd $1, %edi, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    pinsrd $3, (%rsi), %xmm0 # sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_pinsrd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpinsrd $1, %edi, %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vpinsrd $3, (%rsi), %xmm0, %xmm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = insertelement <4 x i32> %a0, i32 %a1, i32 1
-  %2 = load i32, i32 *%a2
-  %3 = insertelement <4 x i32> %1, i32 %2, i32 3
-  ret <4 x i32> %3
-}
-
-define <2 x i64> @test_pinsrq(<2 x i64> %a0, <2 x i64> %a1, i64 %a2, i64 *%a3) {
-; GENERIC-LABEL: test_pinsrq:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    pinsrq $1, %rdi, %xmm0 # sched: [2:1.00]
-; GENERIC-NEXT:    pinsrq $1, (%rsi), %xmm1 # sched: [7:0.50]
-; GENERIC-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SLM-LABEL: test_pinsrq:
-; SLM:       # %bb.0:
-; SLM-NEXT:    pinsrq $1, (%rsi), %xmm1 # sched: [4:1.00]
-; SLM-NEXT:    pinsrq $1, %rdi, %xmm0 # sched: [1:1.00]
-; SLM-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_pinsrq:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    pinsrq $1, %rdi, %xmm0 # sched: [2:1.00]
-; SANDY-SSE-NEXT:    pinsrq $1, (%rsi), %xmm1 # sched: [7:0.50]
-; SANDY-SSE-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.50]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_pinsrq:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vpinsrq $1, %rdi, %xmm0, %xmm0 # sched: [2:1.00]
-; SANDY-NEXT:    vpinsrq $1, (%rsi), %xmm1, %xmm1 # sched: [7:0.50]
-; SANDY-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_pinsrq:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    pinsrq $1, %rdi, %xmm0 # sched: [2:2.00]
-; HASWELL-SSE-NEXT:    pinsrq $1, (%rsi), %xmm1 # sched: [6:1.00]
-; HASWELL-SSE-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.50]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_pinsrq:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpinsrq $1, %rdi, %xmm0, %xmm0 # sched: [2:2.00]
-; HASWELL-NEXT:    vpinsrq $1, (%rsi), %xmm1, %xmm1 # sched: [6:1.00]
-; HASWELL-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_pinsrq:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    pinsrq $1, %rdi, %xmm0 # sched: [2:2.00]
-; BROADWELL-SSE-NEXT:    pinsrq $1, (%rsi), %xmm1 # sched: [6:1.00]
-; BROADWELL-SSE-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.50]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pinsrq:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpinsrq $1, %rdi, %xmm0, %xmm0 # sched: [2:2.00]
-; BROADWELL-NEXT:    vpinsrq $1, (%rsi), %xmm1, %xmm1 # sched: [6:1.00]
-; BROADWELL-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_pinsrq:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    pinsrq $1, %rdi, %xmm0 # sched: [2:2.00]
-; SKYLAKE-SSE-NEXT:    pinsrq $1, (%rsi), %xmm1 # sched: [6:1.00]
-; SKYLAKE-SSE-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.33]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pinsrq:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpinsrq $1, %rdi, %xmm0, %xmm0 # sched: [2:2.00]
-; SKYLAKE-NEXT:    vpinsrq $1, (%rsi), %xmm1, %xmm1 # sched: [6:1.00]
-; SKYLAKE-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_pinsrq:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    pinsrq $1, %rdi, %xmm0 # sched: [2:2.00]
-; SKX-SSE-NEXT:    pinsrq $1, (%rsi), %xmm1 # sched: [6:1.00]
-; SKX-SSE-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.33]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pinsrq:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpinsrq $1, %rdi, %xmm0, %xmm0 # sched: [2:2.00]
-; SKX-NEXT:    vpinsrq $1, (%rsi), %xmm1, %xmm1 # sched: [6:1.00]
-; SKX-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_pinsrq:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    pinsrq $1, (%rsi), %xmm1 # sched: [6:0.50]
-; BDVER2-SSE-NEXT:    pinsrq $1, %rdi, %xmm0 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    paddq %xmm1, %xmm0 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_pinsrq:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vpinsrq $1, (%rsi), %xmm1, %xmm1 # sched: [6:0.50]
-; BDVER2-NEXT:    vpinsrq $1, %rdi, %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_pinsrq:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    pinsrq $1, %rdi, %xmm0 # sched: [7:0.50]
-; BTVER2-SSE-NEXT:    pinsrq $1, (%rsi), %xmm1 # sched: [4:1.00]
-; BTVER2-SSE-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_pinsrq:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vpinsrq $1, %rdi, %xmm0, %xmm0 # sched: [7:0.50]
-; BTVER2-NEXT:    vpinsrq $1, (%rsi), %xmm1, %xmm1 # sched: [4:1.00]
-; BTVER2-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_pinsrq:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    pinsrq $1, (%rsi), %xmm1 # sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    pinsrq $1, %rdi, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_pinsrq:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpinsrq $1, (%rsi), %xmm1, %xmm1 # sched: [8:0.50]
-; ZNVER1-NEXT:    vpinsrq $1, %rdi, %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = insertelement <2 x i64> %a0, i64 %a2, i32 1
-  %2 = load i64, i64 *%a3
-  %3 = insertelement <2 x i64> %a1, i64 %2, i32 1
-  %4 = add <2 x i64> %1, %3
-  ret <2 x i64> %4
-}
-
-define <16 x i8> @test_pmaxsb(<16 x i8> %a0, <16 x i8> %a1, <16 x i8> *%a2) {
-; GENERIC-LABEL: test_pmaxsb:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    pmaxsb %xmm1, %xmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    pmaxsb (%rdi), %xmm0 # sched: [7:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SLM-LABEL: test_pmaxsb:
-; SLM:       # %bb.0:
-; SLM-NEXT:    pmaxsb %xmm1, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    pmaxsb (%rdi), %xmm0 # sched: [4:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_pmaxsb:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    pmaxsb %xmm1, %xmm0 # sched: [1:0.50]
-; SANDY-SSE-NEXT:    pmaxsb (%rdi), %xmm0 # sched: [7:0.50]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_pmaxsb:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vpmaxsb %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SANDY-NEXT:    vpmaxsb (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_pmaxsb:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    pmaxsb %xmm1, %xmm0 # sched: [1:0.50]
-; HASWELL-SSE-NEXT:    pmaxsb (%rdi), %xmm0 # sched: [7:0.50]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_pmaxsb:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpmaxsb %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; HASWELL-NEXT:    vpmaxsb (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_pmaxsb:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    pmaxsb %xmm1, %xmm0 # sched: [1:0.50]
-; BROADWELL-SSE-NEXT:    pmaxsb (%rdi), %xmm0 # sched: [6:0.50]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pmaxsb:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpmaxsb %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    vpmaxsb (%rdi), %xmm0, %xmm0 # sched: [6:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_pmaxsb:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    pmaxsb %xmm1, %xmm0 # sched: [1:0.50]
-; SKYLAKE-SSE-NEXT:    pmaxsb (%rdi), %xmm0 # sched: [7:0.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pmaxsb:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpmaxsb %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SKYLAKE-NEXT:    vpmaxsb (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_pmaxsb:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    pmaxsb %xmm1, %xmm0 # sched: [1:0.50]
-; SKX-SSE-NEXT:    pmaxsb (%rdi), %xmm0 # sched: [7:0.50]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pmaxsb:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpmaxsb %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SKX-NEXT:    vpmaxsb (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_pmaxsb:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    pmaxsb %xmm1, %xmm0 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    pmaxsb (%rdi), %xmm0 # sched: [7:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_pmaxsb:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vpmaxsb %xmm1, %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    vpmaxsb (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_pmaxsb:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    pmaxsb %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    pmaxsb (%rdi), %xmm0 # sched: [6:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_pmaxsb:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vpmaxsb %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    vpmaxsb (%rdi), %xmm0, %xmm0 # sched: [6:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_pmaxsb:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    pmaxsb %xmm1, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    pmaxsb (%rdi), %xmm0 # sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_pmaxsb:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpmaxsb %xmm1, %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vpmaxsb (%rdi), %xmm0, %xmm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <16 x i8> @llvm.x86.sse41.pmaxsb(<16 x i8> %a0, <16 x i8> %a1)
-  %2 = load <16 x i8>, <16 x i8> *%a2, align 16
-  %3 = call <16 x i8> @llvm.x86.sse41.pmaxsb(<16 x i8> %1, <16 x i8> %2)
-  ret <16 x i8> %3
-}
-declare <16 x i8> @llvm.x86.sse41.pmaxsb(<16 x i8>, <16 x i8>) nounwind readnone
-
-define <4 x i32> @test_pmaxsd(<4 x i32> %a0, <4 x i32> %a1, <4 x i32> *%a2) {
-; GENERIC-LABEL: test_pmaxsd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    pmaxsd %xmm1, %xmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    pmaxsd (%rdi), %xmm0 # sched: [7:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SLM-LABEL: test_pmaxsd:
-; SLM:       # %bb.0:
-; SLM-NEXT:    pmaxsd %xmm1, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    pmaxsd (%rdi), %xmm0 # sched: [4:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_pmaxsd:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    pmaxsd %xmm1, %xmm0 # sched: [1:0.50]
-; SANDY-SSE-NEXT:    pmaxsd (%rdi), %xmm0 # sched: [7:0.50]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_pmaxsd:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SANDY-NEXT:    vpmaxsd (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_pmaxsd:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    pmaxsd %xmm1, %xmm0 # sched: [1:0.50]
-; HASWELL-SSE-NEXT:    pmaxsd (%rdi), %xmm0 # sched: [7:0.50]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_pmaxsd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; HASWELL-NEXT:    vpmaxsd (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_pmaxsd:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    pmaxsd %xmm1, %xmm0 # sched: [1:0.50]
-; BROADWELL-SSE-NEXT:    pmaxsd (%rdi), %xmm0 # sched: [6:0.50]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pmaxsd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    vpmaxsd (%rdi), %xmm0, %xmm0 # sched: [6:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_pmaxsd:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    pmaxsd %xmm1, %xmm0 # sched: [1:0.50]
-; SKYLAKE-SSE-NEXT:    pmaxsd (%rdi), %xmm0 # sched: [7:0.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pmaxsd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SKYLAKE-NEXT:    vpmaxsd (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_pmaxsd:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    pmaxsd %xmm1, %xmm0 # sched: [1:0.50]
-; SKX-SSE-NEXT:    pmaxsd (%rdi), %xmm0 # sched: [7:0.50]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pmaxsd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SKX-NEXT:    vpmaxsd (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_pmaxsd:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    pmaxsd %xmm1, %xmm0 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    pmaxsd (%rdi), %xmm0 # sched: [7:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_pmaxsd:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    vpmaxsd (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_pmaxsd:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    pmaxsd %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    pmaxsd (%rdi), %xmm0 # sched: [6:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_pmaxsd:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    vpmaxsd (%rdi), %xmm0, %xmm0 # sched: [6:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_pmaxsd:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    pmaxsd %xmm1, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    pmaxsd (%rdi), %xmm0 # sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_pmaxsd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpmaxsd %xmm1, %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vpmaxsd (%rdi), %xmm0, %xmm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <4 x i32> @llvm.x86.sse41.pmaxsd(<4 x i32> %a0, <4 x i32> %a1)
-  %2 = load <4 x i32>, <4 x i32> *%a2, align 16
-  %3 = call <4 x i32> @llvm.x86.sse41.pmaxsd(<4 x i32> %1, <4 x i32> %2)
-  ret <4 x i32> %3
-}
-declare <4 x i32> @llvm.x86.sse41.pmaxsd(<4 x i32>, <4 x i32>) nounwind readnone
-
-define <4 x i32> @test_pmaxud(<4 x i32> %a0, <4 x i32> %a1, <4 x i32> *%a2) {
-; GENERIC-LABEL: test_pmaxud:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    pmaxud %xmm1, %xmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    pmaxud (%rdi), %xmm0 # sched: [7:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SLM-LABEL: test_pmaxud:
-; SLM:       # %bb.0:
-; SLM-NEXT:    pmaxud %xmm1, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    pmaxud (%rdi), %xmm0 # sched: [4:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_pmaxud:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    pmaxud %xmm1, %xmm0 # sched: [1:0.50]
-; SANDY-SSE-NEXT:    pmaxud (%rdi), %xmm0 # sched: [7:0.50]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_pmaxud:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SANDY-NEXT:    vpmaxud (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_pmaxud:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    pmaxud %xmm1, %xmm0 # sched: [1:0.50]
-; HASWELL-SSE-NEXT:    pmaxud (%rdi), %xmm0 # sched: [7:0.50]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_pmaxud:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; HASWELL-NEXT:    vpmaxud (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_pmaxud:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    pmaxud %xmm1, %xmm0 # sched: [1:0.50]
-; BROADWELL-SSE-NEXT:    pmaxud (%rdi), %xmm0 # sched: [6:0.50]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pmaxud:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    vpmaxud (%rdi), %xmm0, %xmm0 # sched: [6:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_pmaxud:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    pmaxud %xmm1, %xmm0 # sched: [1:0.50]
-; SKYLAKE-SSE-NEXT:    pmaxud (%rdi), %xmm0 # sched: [7:0.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pmaxud:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SKYLAKE-NEXT:    vpmaxud (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_pmaxud:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    pmaxud %xmm1, %xmm0 # sched: [1:0.50]
-; SKX-SSE-NEXT:    pmaxud (%rdi), %xmm0 # sched: [7:0.50]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pmaxud:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SKX-NEXT:    vpmaxud (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_pmaxud:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    pmaxud %xmm1, %xmm0 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    pmaxud (%rdi), %xmm0 # sched: [7:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_pmaxud:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    vpmaxud (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_pmaxud:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    pmaxud %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    pmaxud (%rdi), %xmm0 # sched: [6:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_pmaxud:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    vpmaxud (%rdi), %xmm0, %xmm0 # sched: [6:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_pmaxud:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    pmaxud %xmm1, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    pmaxud (%rdi), %xmm0 # sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_pmaxud:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vpmaxud (%rdi), %xmm0, %xmm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <4 x i32> @llvm.x86.sse41.pmaxud(<4 x i32> %a0, <4 x i32> %a1)
-  %2 = load <4 x i32>, <4 x i32> *%a2, align 16
-  %3 = call <4 x i32> @llvm.x86.sse41.pmaxud(<4 x i32> %1, <4 x i32> %2)
-  ret <4 x i32> %3
-}
-declare <4 x i32> @llvm.x86.sse41.pmaxud(<4 x i32>, <4 x i32>) nounwind readnone
-
-define <8 x i16> @test_pmaxuw(<8 x i16> %a0, <8 x i16> %a1, <8 x i16> *%a2) {
-; GENERIC-LABEL: test_pmaxuw:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    pmaxuw %xmm1, %xmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    pmaxuw (%rdi), %xmm0 # sched: [7:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SLM-LABEL: test_pmaxuw:
-; SLM:       # %bb.0:
-; SLM-NEXT:    pmaxuw %xmm1, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    pmaxuw (%rdi), %xmm0 # sched: [4:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_pmaxuw:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    pmaxuw %xmm1, %xmm0 # sched: [1:0.50]
-; SANDY-SSE-NEXT:    pmaxuw (%rdi), %xmm0 # sched: [7:0.50]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_pmaxuw:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vpmaxuw %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SANDY-NEXT:    vpmaxuw (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_pmaxuw:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    pmaxuw %xmm1, %xmm0 # sched: [1:0.50]
-; HASWELL-SSE-NEXT:    pmaxuw (%rdi), %xmm0 # sched: [7:0.50]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_pmaxuw:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpmaxuw %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; HASWELL-NEXT:    vpmaxuw (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_pmaxuw:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    pmaxuw %xmm1, %xmm0 # sched: [1:0.50]
-; BROADWELL-SSE-NEXT:    pmaxuw (%rdi), %xmm0 # sched: [6:0.50]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pmaxuw:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpmaxuw %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    vpmaxuw (%rdi), %xmm0, %xmm0 # sched: [6:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_pmaxuw:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    pmaxuw %xmm1, %xmm0 # sched: [1:0.50]
-; SKYLAKE-SSE-NEXT:    pmaxuw (%rdi), %xmm0 # sched: [7:0.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pmaxuw:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpmaxuw %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SKYLAKE-NEXT:    vpmaxuw (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_pmaxuw:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    pmaxuw %xmm1, %xmm0 # sched: [1:0.50]
-; SKX-SSE-NEXT:    pmaxuw (%rdi), %xmm0 # sched: [7:0.50]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pmaxuw:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpmaxuw %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SKX-NEXT:    vpmaxuw (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_pmaxuw:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    pmaxuw %xmm1, %xmm0 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    pmaxuw (%rdi), %xmm0 # sched: [7:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_pmaxuw:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vpmaxuw %xmm1, %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    vpmaxuw (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_pmaxuw:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    pmaxuw %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    pmaxuw (%rdi), %xmm0 # sched: [6:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_pmaxuw:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vpmaxuw %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    vpmaxuw (%rdi), %xmm0, %xmm0 # sched: [6:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_pmaxuw:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    pmaxuw %xmm1, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    pmaxuw (%rdi), %xmm0 # sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_pmaxuw:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpmaxuw %xmm1, %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vpmaxuw (%rdi), %xmm0, %xmm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <8 x i16> @llvm.x86.sse41.pmaxuw(<8 x i16> %a0, <8 x i16> %a1)
-  %2 = load <8 x i16>, <8 x i16> *%a2, align 16
-  %3 = call <8 x i16> @llvm.x86.sse41.pmaxuw(<8 x i16> %1, <8 x i16> %2)
-  ret <8 x i16> %3
-}
-declare <8 x i16> @llvm.x86.sse41.pmaxuw(<8 x i16>, <8 x i16>) nounwind readnone
-
-define <16 x i8> @test_pminsb(<16 x i8> %a0, <16 x i8> %a1, <16 x i8> *%a2) {
-; GENERIC-LABEL: test_pminsb:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    pminsb %xmm1, %xmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    pminsb (%rdi), %xmm0 # sched: [7:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SLM-LABEL: test_pminsb:
-; SLM:       # %bb.0:
-; SLM-NEXT:    pminsb %xmm1, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    pminsb (%rdi), %xmm0 # sched: [4:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_pminsb:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    pminsb %xmm1, %xmm0 # sched: [1:0.50]
-; SANDY-SSE-NEXT:    pminsb (%rdi), %xmm0 # sched: [7:0.50]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_pminsb:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vpminsb %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SANDY-NEXT:    vpminsb (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_pminsb:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    pminsb %xmm1, %xmm0 # sched: [1:0.50]
-; HASWELL-SSE-NEXT:    pminsb (%rdi), %xmm0 # sched: [7:0.50]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_pminsb:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpminsb %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; HASWELL-NEXT:    vpminsb (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_pminsb:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    pminsb %xmm1, %xmm0 # sched: [1:0.50]
-; BROADWELL-SSE-NEXT:    pminsb (%rdi), %xmm0 # sched: [6:0.50]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pminsb:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpminsb %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    vpminsb (%rdi), %xmm0, %xmm0 # sched: [6:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_pminsb:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    pminsb %xmm1, %xmm0 # sched: [1:0.50]
-; SKYLAKE-SSE-NEXT:    pminsb (%rdi), %xmm0 # sched: [7:0.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pminsb:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpminsb %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SKYLAKE-NEXT:    vpminsb (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_pminsb:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    pminsb %xmm1, %xmm0 # sched: [1:0.50]
-; SKX-SSE-NEXT:    pminsb (%rdi), %xmm0 # sched: [7:0.50]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pminsb:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpminsb %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SKX-NEXT:    vpminsb (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_pminsb:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    pminsb %xmm1, %xmm0 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    pminsb (%rdi), %xmm0 # sched: [7:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_pminsb:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vpminsb %xmm1, %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    vpminsb (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_pminsb:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    pminsb %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    pminsb (%rdi), %xmm0 # sched: [6:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_pminsb:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vpminsb %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    vpminsb (%rdi), %xmm0, %xmm0 # sched: [6:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_pminsb:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    pminsb %xmm1, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    pminsb (%rdi), %xmm0 # sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_pminsb:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpminsb %xmm1, %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vpminsb (%rdi), %xmm0, %xmm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <16 x i8> @llvm.x86.sse41.pminsb(<16 x i8> %a0, <16 x i8> %a1)
-  %2 = load <16 x i8>, <16 x i8> *%a2, align 16
-  %3 = call <16 x i8> @llvm.x86.sse41.pminsb(<16 x i8> %1, <16 x i8> %2)
-  ret <16 x i8> %3
-}
-declare <16 x i8> @llvm.x86.sse41.pminsb(<16 x i8>, <16 x i8>) nounwind readnone
-
-define <4 x i32> @test_pminsd(<4 x i32> %a0, <4 x i32> %a1, <4 x i32> *%a2) {
-; GENERIC-LABEL: test_pminsd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    pminsd %xmm1, %xmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    pminsd (%rdi), %xmm0 # sched: [7:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SLM-LABEL: test_pminsd:
-; SLM:       # %bb.0:
-; SLM-NEXT:    pminsd %xmm1, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    pminsd (%rdi), %xmm0 # sched: [4:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_pminsd:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    pminsd %xmm1, %xmm0 # sched: [1:0.50]
-; SANDY-SSE-NEXT:    pminsd (%rdi), %xmm0 # sched: [7:0.50]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_pminsd:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vpminsd %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SANDY-NEXT:    vpminsd (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_pminsd:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    pminsd %xmm1, %xmm0 # sched: [1:0.50]
-; HASWELL-SSE-NEXT:    pminsd (%rdi), %xmm0 # sched: [7:0.50]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_pminsd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpminsd %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; HASWELL-NEXT:    vpminsd (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_pminsd:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    pminsd %xmm1, %xmm0 # sched: [1:0.50]
-; BROADWELL-SSE-NEXT:    pminsd (%rdi), %xmm0 # sched: [6:0.50]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pminsd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpminsd %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    vpminsd (%rdi), %xmm0, %xmm0 # sched: [6:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_pminsd:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    pminsd %xmm1, %xmm0 # sched: [1:0.50]
-; SKYLAKE-SSE-NEXT:    pminsd (%rdi), %xmm0 # sched: [7:0.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pminsd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpminsd %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SKYLAKE-NEXT:    vpminsd (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_pminsd:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    pminsd %xmm1, %xmm0 # sched: [1:0.50]
-; SKX-SSE-NEXT:    pminsd (%rdi), %xmm0 # sched: [7:0.50]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pminsd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpminsd %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SKX-NEXT:    vpminsd (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_pminsd:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    pminsd %xmm1, %xmm0 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    pminsd (%rdi), %xmm0 # sched: [7:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_pminsd:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vpminsd %xmm1, %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    vpminsd (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_pminsd:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    pminsd %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    pminsd (%rdi), %xmm0 # sched: [6:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_pminsd:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vpminsd %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    vpminsd (%rdi), %xmm0, %xmm0 # sched: [6:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_pminsd:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    pminsd %xmm1, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    pminsd (%rdi), %xmm0 # sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_pminsd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpminsd %xmm1, %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vpminsd (%rdi), %xmm0, %xmm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <4 x i32> @llvm.x86.sse41.pminsd(<4 x i32> %a0, <4 x i32> %a1)
-  %2 = load <4 x i32>, <4 x i32> *%a2, align 16
-  %3 = call <4 x i32> @llvm.x86.sse41.pminsd(<4 x i32> %1, <4 x i32> %2)
-  ret <4 x i32> %3
-}
-declare <4 x i32> @llvm.x86.sse41.pminsd(<4 x i32>, <4 x i32>) nounwind readnone
-
-define <4 x i32> @test_pminud(<4 x i32> %a0, <4 x i32> %a1, <4 x i32> *%a2) {
-; GENERIC-LABEL: test_pminud:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    pminud %xmm1, %xmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    pminud (%rdi), %xmm0 # sched: [7:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SLM-LABEL: test_pminud:
-; SLM:       # %bb.0:
-; SLM-NEXT:    pminud %xmm1, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    pminud (%rdi), %xmm0 # sched: [4:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_pminud:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    pminud %xmm1, %xmm0 # sched: [1:0.50]
-; SANDY-SSE-NEXT:    pminud (%rdi), %xmm0 # sched: [7:0.50]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_pminud:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vpminud %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SANDY-NEXT:    vpminud (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_pminud:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    pminud %xmm1, %xmm0 # sched: [1:0.50]
-; HASWELL-SSE-NEXT:    pminud (%rdi), %xmm0 # sched: [7:0.50]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_pminud:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpminud %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; HASWELL-NEXT:    vpminud (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_pminud:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    pminud %xmm1, %xmm0 # sched: [1:0.50]
-; BROADWELL-SSE-NEXT:    pminud (%rdi), %xmm0 # sched: [6:0.50]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pminud:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpminud %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    vpminud (%rdi), %xmm0, %xmm0 # sched: [6:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_pminud:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    pminud %xmm1, %xmm0 # sched: [1:0.50]
-; SKYLAKE-SSE-NEXT:    pminud (%rdi), %xmm0 # sched: [7:0.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pminud:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpminud %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SKYLAKE-NEXT:    vpminud (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_pminud:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    pminud %xmm1, %xmm0 # sched: [1:0.50]
-; SKX-SSE-NEXT:    pminud (%rdi), %xmm0 # sched: [7:0.50]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pminud:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpminud %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SKX-NEXT:    vpminud (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_pminud:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    pminud %xmm1, %xmm0 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    pminud (%rdi), %xmm0 # sched: [7:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_pminud:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vpminud %xmm1, %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    vpminud (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_pminud:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    pminud %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    pminud (%rdi), %xmm0 # sched: [6:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_pminud:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vpminud %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    vpminud (%rdi), %xmm0, %xmm0 # sched: [6:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_pminud:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    pminud %xmm1, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    pminud (%rdi), %xmm0 # sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_pminud:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpminud %xmm1, %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vpminud (%rdi), %xmm0, %xmm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <4 x i32> @llvm.x86.sse41.pminud(<4 x i32> %a0, <4 x i32> %a1)
-  %2 = load <4 x i32>, <4 x i32> *%a2, align 16
-  %3 = call <4 x i32> @llvm.x86.sse41.pminud(<4 x i32> %1, <4 x i32> %2)
-  ret <4 x i32> %3
-}
-declare <4 x i32> @llvm.x86.sse41.pminud(<4 x i32>, <4 x i32>) nounwind readnone
-
-define <8 x i16> @test_pminuw(<8 x i16> %a0, <8 x i16> %a1, <8 x i16> *%a2) {
-; GENERIC-LABEL: test_pminuw:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    pminuw %xmm1, %xmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    pminuw (%rdi), %xmm0 # sched: [7:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SLM-LABEL: test_pminuw:
-; SLM:       # %bb.0:
-; SLM-NEXT:    pminuw %xmm1, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    pminuw (%rdi), %xmm0 # sched: [4:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_pminuw:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    pminuw %xmm1, %xmm0 # sched: [1:0.50]
-; SANDY-SSE-NEXT:    pminuw (%rdi), %xmm0 # sched: [7:0.50]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_pminuw:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vpminuw %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SANDY-NEXT:    vpminuw (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_pminuw:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    pminuw %xmm1, %xmm0 # sched: [1:0.50]
-; HASWELL-SSE-NEXT:    pminuw (%rdi), %xmm0 # sched: [7:0.50]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_pminuw:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpminuw %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; HASWELL-NEXT:    vpminuw (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_pminuw:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    pminuw %xmm1, %xmm0 # sched: [1:0.50]
-; BROADWELL-SSE-NEXT:    pminuw (%rdi), %xmm0 # sched: [6:0.50]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pminuw:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpminuw %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    vpminuw (%rdi), %xmm0, %xmm0 # sched: [6:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_pminuw:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    pminuw %xmm1, %xmm0 # sched: [1:0.50]
-; SKYLAKE-SSE-NEXT:    pminuw (%rdi), %xmm0 # sched: [7:0.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pminuw:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpminuw %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SKYLAKE-NEXT:    vpminuw (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_pminuw:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    pminuw %xmm1, %xmm0 # sched: [1:0.50]
-; SKX-SSE-NEXT:    pminuw (%rdi), %xmm0 # sched: [7:0.50]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pminuw:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpminuw %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SKX-NEXT:    vpminuw (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_pminuw:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    pminuw %xmm1, %xmm0 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    pminuw (%rdi), %xmm0 # sched: [7:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_pminuw:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vpminuw %xmm1, %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    vpminuw (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_pminuw:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    pminuw %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    pminuw (%rdi), %xmm0 # sched: [6:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_pminuw:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vpminuw %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    vpminuw (%rdi), %xmm0, %xmm0 # sched: [6:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_pminuw:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    pminuw %xmm1, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    pminuw (%rdi), %xmm0 # sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_pminuw:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpminuw %xmm1, %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vpminuw (%rdi), %xmm0, %xmm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <8 x i16> @llvm.x86.sse41.pminuw(<8 x i16> %a0, <8 x i16> %a1)
-  %2 = load <8 x i16>, <8 x i16> *%a2, align 16
-  %3 = call <8 x i16> @llvm.x86.sse41.pminuw(<8 x i16> %1, <8 x i16> %2)
-  ret <8 x i16> %3
-}
-declare <8 x i16> @llvm.x86.sse41.pminuw(<8 x i16>, <8 x i16>) nounwind readnone
-
-define <8 x i16> @test_pmovsxbw(<16 x i8> %a0, <8 x i8> *%a1) {
-; GENERIC-LABEL: test_pmovsxbw:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    pmovsxbw %xmm0, %xmm1 # sched: [1:0.50]
-; GENERIC-NEXT:    pmovsxbw (%rdi), %xmm0 # sched: [7:0.50]
-; GENERIC-NEXT:    paddw %xmm1, %xmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SLM-LABEL: test_pmovsxbw:
-; SLM:       # %bb.0:
-; SLM-NEXT:    pmovsxbw (%rdi), %xmm1 # sched: [4:1.00]
-; SLM-NEXT:    pmovsxbw %xmm0, %xmm0 # sched: [1:1.00]
-; SLM-NEXT:    paddw %xmm0, %xmm1 # sched: [1:0.50]
-; SLM-NEXT:    movdqa %xmm1, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_pmovsxbw:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    pmovsxbw %xmm0, %xmm1 # sched: [1:0.50]
-; SANDY-SSE-NEXT:    pmovsxbw (%rdi), %xmm0 # sched: [7:0.50]
-; SANDY-SSE-NEXT:    paddw %xmm1, %xmm0 # sched: [1:0.50]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_pmovsxbw:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vpmovsxbw %xmm0, %xmm0 # sched: [1:0.50]
-; SANDY-NEXT:    vpmovsxbw (%rdi), %xmm1 # sched: [7:0.50]
-; SANDY-NEXT:    vpaddw %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_pmovsxbw:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    pmovsxbw %xmm0, %xmm1 # sched: [1:1.00]
-; HASWELL-SSE-NEXT:    pmovsxbw (%rdi), %xmm0 # sched: [6:1.00]
-; HASWELL-SSE-NEXT:    paddw %xmm1, %xmm0 # sched: [1:0.50]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_pmovsxbw:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpmovsxbw %xmm0, %xmm0 # sched: [1:1.00]
-; HASWELL-NEXT:    vpmovsxbw (%rdi), %xmm1 # sched: [6:1.00]
-; HASWELL-NEXT:    vpaddw %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_pmovsxbw:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    pmovsxbw %xmm0, %xmm1 # sched: [1:1.00]
-; BROADWELL-SSE-NEXT:    pmovsxbw (%rdi), %xmm0 # sched: [6:1.00]
-; BROADWELL-SSE-NEXT:    paddw %xmm1, %xmm0 # sched: [1:0.50]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pmovsxbw:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpmovsxbw %xmm0, %xmm0 # sched: [1:1.00]
-; BROADWELL-NEXT:    vpmovsxbw (%rdi), %xmm1 # sched: [6:1.00]
-; BROADWELL-NEXT:    vpaddw %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_pmovsxbw:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    pmovsxbw %xmm0, %xmm1 # sched: [1:1.00]
-; SKYLAKE-SSE-NEXT:    pmovsxbw (%rdi), %xmm0 # sched: [6:1.00]
-; SKYLAKE-SSE-NEXT:    paddw %xmm1, %xmm0 # sched: [1:0.33]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pmovsxbw:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpmovsxbw %xmm0, %xmm0 # sched: [1:1.00]
-; SKYLAKE-NEXT:    vpmovsxbw (%rdi), %xmm1 # sched: [6:1.00]
-; SKYLAKE-NEXT:    vpaddw %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_pmovsxbw:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    pmovsxbw %xmm0, %xmm1 # sched: [1:1.00]
-; SKX-SSE-NEXT:    pmovsxbw (%rdi), %xmm0 # sched: [6:1.00]
-; SKX-SSE-NEXT:    paddw %xmm1, %xmm0 # sched: [1:0.33]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pmovsxbw:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpmovsxbw %xmm0, %xmm0 # sched: [1:1.00]
-; SKX-NEXT:    vpmovsxbw (%rdi), %xmm1 # sched: [6:1.00]
-; SKX-NEXT:    vpaddw %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_pmovsxbw:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    pmovsxbw %xmm0, %xmm1 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    pmovsxbw (%rdi), %xmm0 # sched: [7:0.50]
-; BDVER2-SSE-NEXT:    paddw %xmm1, %xmm0 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_pmovsxbw:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vpmovsxbw (%rdi), %xmm1 # sched: [7:0.50]
-; BDVER2-NEXT:    vpmovsxbw %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    vpaddw %xmm1, %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_pmovsxbw:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    pmovsxbw %xmm0, %xmm1 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    pmovsxbw (%rdi), %xmm0 # sched: [6:1.00]
-; BTVER2-SSE-NEXT:    paddw %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_pmovsxbw:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vpmovsxbw (%rdi), %xmm1 # sched: [6:1.00]
-; BTVER2-NEXT:    vpmovsxbw %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    vpaddw %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_pmovsxbw:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    pmovsxbw %xmm0, %xmm1 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    pmovsxbw (%rdi), %xmm0 # sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    paddw %xmm1, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_pmovsxbw:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpmovsxbw (%rdi), %xmm1 # sched: [8:0.50]
-; ZNVER1-NEXT:    vpmovsxbw %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vpaddw %xmm1, %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = shufflevector <16 x i8> %a0, <16 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-  %2 = sext <8 x i8> %1 to <8 x i16>
-  %3 = load <8 x i8>, <8 x i8>* %a1, align 1
-  %4 = sext <8 x i8> %3 to <8 x i16>
-  %5 = add <8 x i16> %2, %4
-  ret <8 x i16> %5
-}
-
-define <4 x i32> @test_pmovsxbd(<16 x i8> %a0, <4 x i8> *%a1) {
-; GENERIC-LABEL: test_pmovsxbd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    pmovsxbd %xmm0, %xmm1 # sched: [1:0.50]
-; GENERIC-NEXT:    pmovsxbd (%rdi), %xmm0 # sched: [7:0.50]
-; GENERIC-NEXT:    paddd %xmm1, %xmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SLM-LABEL: test_pmovsxbd:
-; SLM:       # %bb.0:
-; SLM-NEXT:    pmovsxbd (%rdi), %xmm1 # sched: [4:1.00]
-; SLM-NEXT:    pmovsxbd %xmm0, %xmm0 # sched: [1:1.00]
-; SLM-NEXT:    paddd %xmm0, %xmm1 # sched: [1:0.50]
-; SLM-NEXT:    movdqa %xmm1, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_pmovsxbd:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    pmovsxbd %xmm0, %xmm1 # sched: [1:0.50]
-; SANDY-SSE-NEXT:    pmovsxbd (%rdi), %xmm0 # sched: [7:0.50]
-; SANDY-SSE-NEXT:    paddd %xmm1, %xmm0 # sched: [1:0.50]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_pmovsxbd:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vpmovsxbd %xmm0, %xmm0 # sched: [1:0.50]
-; SANDY-NEXT:    vpmovsxbd (%rdi), %xmm1 # sched: [7:0.50]
-; SANDY-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_pmovsxbd:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    pmovsxbd %xmm0, %xmm1 # sched: [1:1.00]
-; HASWELL-SSE-NEXT:    pmovsxbd (%rdi), %xmm0 # sched: [6:1.00]
-; HASWELL-SSE-NEXT:    paddd %xmm1, %xmm0 # sched: [1:0.50]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_pmovsxbd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpmovsxbd %xmm0, %xmm0 # sched: [1:1.00]
-; HASWELL-NEXT:    vpmovsxbd (%rdi), %xmm1 # sched: [6:1.00]
-; HASWELL-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_pmovsxbd:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    pmovsxbd %xmm0, %xmm1 # sched: [1:1.00]
-; BROADWELL-SSE-NEXT:    pmovsxbd (%rdi), %xmm0 # sched: [6:1.00]
-; BROADWELL-SSE-NEXT:    paddd %xmm1, %xmm0 # sched: [1:0.50]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pmovsxbd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpmovsxbd %xmm0, %xmm0 # sched: [1:1.00]
-; BROADWELL-NEXT:    vpmovsxbd (%rdi), %xmm1 # sched: [6:1.00]
-; BROADWELL-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_pmovsxbd:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    pmovsxbd %xmm0, %xmm1 # sched: [1:1.00]
-; SKYLAKE-SSE-NEXT:    pmovsxbd (%rdi), %xmm0 # sched: [6:1.00]
-; SKYLAKE-SSE-NEXT:    paddd %xmm1, %xmm0 # sched: [1:0.33]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pmovsxbd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpmovsxbd %xmm0, %xmm0 # sched: [1:1.00]
-; SKYLAKE-NEXT:    vpmovsxbd (%rdi), %xmm1 # sched: [6:1.00]
-; SKYLAKE-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_pmovsxbd:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    pmovsxbd %xmm0, %xmm1 # sched: [1:1.00]
-; SKX-SSE-NEXT:    pmovsxbd (%rdi), %xmm0 # sched: [6:1.00]
-; SKX-SSE-NEXT:    paddd %xmm1, %xmm0 # sched: [1:0.33]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pmovsxbd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpmovsxbd %xmm0, %xmm0 # sched: [1:1.00]
-; SKX-NEXT:    vpmovsxbd (%rdi), %xmm1 # sched: [6:1.00]
-; SKX-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_pmovsxbd:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    pmovsxbd %xmm0, %xmm1 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    pmovsxbd (%rdi), %xmm0 # sched: [7:0.50]
-; BDVER2-SSE-NEXT:    paddd %xmm1, %xmm0 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_pmovsxbd:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vpmovsxbd (%rdi), %xmm1 # sched: [7:0.50]
-; BDVER2-NEXT:    vpmovsxbd %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_pmovsxbd:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    pmovsxbd %xmm0, %xmm1 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    pmovsxbd (%rdi), %xmm0 # sched: [6:1.00]
-; BTVER2-SSE-NEXT:    paddd %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_pmovsxbd:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vpmovsxbd (%rdi), %xmm1 # sched: [6:1.00]
-; BTVER2-NEXT:    vpmovsxbd %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_pmovsxbd:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    pmovsxbd %xmm0, %xmm1 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    pmovsxbd (%rdi), %xmm0 # sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    paddd %xmm1, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_pmovsxbd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpmovsxbd (%rdi), %xmm1 # sched: [8:0.50]
-; ZNVER1-NEXT:    vpmovsxbd %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = shufflevector <16 x i8> %a0, <16 x i8> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-  %2 = sext <4 x i8> %1 to <4 x i32>
-  %3 = load <4 x i8>, <4 x i8>* %a1, align 1
-  %4 = sext <4 x i8> %3 to <4 x i32>
-  %5 = add <4 x i32> %2, %4
-  ret <4 x i32> %5
-}
-
-define <2 x i64> @test_pmovsxbq(<16 x i8> %a0, <2 x i8> *%a1) {
-; GENERIC-LABEL: test_pmovsxbq:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    pmovsxbq %xmm0, %xmm1 # sched: [1:0.50]
-; GENERIC-NEXT:    pmovsxbq (%rdi), %xmm0 # sched: [7:0.50]
-; GENERIC-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SLM-LABEL: test_pmovsxbq:
-; SLM:       # %bb.0:
-; SLM-NEXT:    pmovsxbq (%rdi), %xmm1 # sched: [4:1.00]
-; SLM-NEXT:    pmovsxbq %xmm0, %xmm0 # sched: [1:1.00]
-; SLM-NEXT:    paddq %xmm0, %xmm1 # sched: [1:0.50]
-; SLM-NEXT:    movdqa %xmm1, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_pmovsxbq:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    pmovsxbq %xmm0, %xmm1 # sched: [1:0.50]
-; SANDY-SSE-NEXT:    pmovsxbq (%rdi), %xmm0 # sched: [7:0.50]
-; SANDY-SSE-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.50]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_pmovsxbq:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vpmovsxbq %xmm0, %xmm0 # sched: [1:0.50]
-; SANDY-NEXT:    vpmovsxbq (%rdi), %xmm1 # sched: [7:0.50]
-; SANDY-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_pmovsxbq:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    pmovsxbq %xmm0, %xmm1 # sched: [1:1.00]
-; HASWELL-SSE-NEXT:    pmovsxbq (%rdi), %xmm0 # sched: [6:1.00]
-; HASWELL-SSE-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.50]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_pmovsxbq:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpmovsxbq %xmm0, %xmm0 # sched: [1:1.00]
-; HASWELL-NEXT:    vpmovsxbq (%rdi), %xmm1 # sched: [6:1.00]
-; HASWELL-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_pmovsxbq:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    pmovsxbq %xmm0, %xmm1 # sched: [1:1.00]
-; BROADWELL-SSE-NEXT:    pmovsxbq (%rdi), %xmm0 # sched: [6:1.00]
-; BROADWELL-SSE-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.50]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pmovsxbq:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpmovsxbq %xmm0, %xmm0 # sched: [1:1.00]
-; BROADWELL-NEXT:    vpmovsxbq (%rdi), %xmm1 # sched: [6:1.00]
-; BROADWELL-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_pmovsxbq:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    pmovsxbq %xmm0, %xmm1 # sched: [1:1.00]
-; SKYLAKE-SSE-NEXT:    pmovsxbq (%rdi), %xmm0 # sched: [6:1.00]
-; SKYLAKE-SSE-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.33]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pmovsxbq:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpmovsxbq %xmm0, %xmm0 # sched: [1:1.00]
-; SKYLAKE-NEXT:    vpmovsxbq (%rdi), %xmm1 # sched: [6:1.00]
-; SKYLAKE-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_pmovsxbq:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    pmovsxbq %xmm0, %xmm1 # sched: [1:1.00]
-; SKX-SSE-NEXT:    pmovsxbq (%rdi), %xmm0 # sched: [6:1.00]
-; SKX-SSE-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.33]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pmovsxbq:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpmovsxbq %xmm0, %xmm0 # sched: [1:1.00]
-; SKX-NEXT:    vpmovsxbq (%rdi), %xmm1 # sched: [6:1.00]
-; SKX-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_pmovsxbq:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    pmovsxbq %xmm0, %xmm1 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    pmovsxbq (%rdi), %xmm0 # sched: [7:0.50]
-; BDVER2-SSE-NEXT:    paddq %xmm1, %xmm0 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_pmovsxbq:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vpmovsxbq (%rdi), %xmm1 # sched: [7:0.50]
-; BDVER2-NEXT:    vpmovsxbq %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_pmovsxbq:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    pmovsxbq %xmm0, %xmm1 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    pmovsxbq (%rdi), %xmm0 # sched: [6:1.00]
-; BTVER2-SSE-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_pmovsxbq:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vpmovsxbq (%rdi), %xmm1 # sched: [6:1.00]
-; BTVER2-NEXT:    vpmovsxbq %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_pmovsxbq:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    pmovsxbq %xmm0, %xmm1 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    pmovsxbq (%rdi), %xmm0 # sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_pmovsxbq:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpmovsxbq (%rdi), %xmm1 # sched: [8:0.50]
-; ZNVER1-NEXT:    vpmovsxbq %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = shufflevector <16 x i8> %a0, <16 x i8> undef, <2 x i32> <i32 0, i32 1>
-  %2 = sext <2 x i8> %1 to <2 x i64>
-  %3 = load <2 x i8>, <2 x i8>* %a1, align 1
-  %4 = sext <2 x i8> %3 to <2 x i64>
-  %5 = add <2 x i64> %2, %4
-  ret <2 x i64> %5
-}
-
-define <2 x i64> @test_pmovsxdq(<4 x i32> %a0, <2 x i32> *%a1) {
-; GENERIC-LABEL: test_pmovsxdq:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    pmovsxdq %xmm0, %xmm1 # sched: [1:0.50]
-; GENERIC-NEXT:    pmovsxdq (%rdi), %xmm0 # sched: [7:0.50]
-; GENERIC-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SLM-LABEL: test_pmovsxdq:
-; SLM:       # %bb.0:
-; SLM-NEXT:    pmovsxdq (%rdi), %xmm1 # sched: [4:1.00]
-; SLM-NEXT:    pmovsxdq %xmm0, %xmm0 # sched: [1:1.00]
-; SLM-NEXT:    paddq %xmm0, %xmm1 # sched: [1:0.50]
-; SLM-NEXT:    movdqa %xmm1, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_pmovsxdq:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    pmovsxdq %xmm0, %xmm1 # sched: [1:0.50]
-; SANDY-SSE-NEXT:    pmovsxdq (%rdi), %xmm0 # sched: [7:0.50]
-; SANDY-SSE-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.50]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_pmovsxdq:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vpmovsxdq %xmm0, %xmm0 # sched: [1:0.50]
-; SANDY-NEXT:    vpmovsxdq (%rdi), %xmm1 # sched: [7:0.50]
-; SANDY-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_pmovsxdq:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    pmovsxdq %xmm0, %xmm1 # sched: [1:1.00]
-; HASWELL-SSE-NEXT:    pmovsxdq (%rdi), %xmm0 # sched: [6:1.00]
-; HASWELL-SSE-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.50]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_pmovsxdq:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpmovsxdq %xmm0, %xmm0 # sched: [1:1.00]
-; HASWELL-NEXT:    vpmovsxdq (%rdi), %xmm1 # sched: [6:1.00]
-; HASWELL-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_pmovsxdq:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    pmovsxdq %xmm0, %xmm1 # sched: [1:1.00]
-; BROADWELL-SSE-NEXT:    pmovsxdq (%rdi), %xmm0 # sched: [6:1.00]
-; BROADWELL-SSE-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.50]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pmovsxdq:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpmovsxdq %xmm0, %xmm0 # sched: [1:1.00]
-; BROADWELL-NEXT:    vpmovsxdq (%rdi), %xmm1 # sched: [6:1.00]
-; BROADWELL-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_pmovsxdq:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    pmovsxdq %xmm0, %xmm1 # sched: [1:1.00]
-; SKYLAKE-SSE-NEXT:    pmovsxdq (%rdi), %xmm0 # sched: [6:1.00]
-; SKYLAKE-SSE-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.33]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pmovsxdq:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpmovsxdq %xmm0, %xmm0 # sched: [1:1.00]
-; SKYLAKE-NEXT:    vpmovsxdq (%rdi), %xmm1 # sched: [6:1.00]
-; SKYLAKE-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_pmovsxdq:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    pmovsxdq %xmm0, %xmm1 # sched: [1:1.00]
-; SKX-SSE-NEXT:    pmovsxdq (%rdi), %xmm0 # sched: [6:1.00]
-; SKX-SSE-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.33]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pmovsxdq:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpmovsxdq %xmm0, %xmm0 # sched: [1:1.00]
-; SKX-NEXT:    vpmovsxdq (%rdi), %xmm1 # sched: [6:1.00]
-; SKX-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_pmovsxdq:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    pmovsxdq %xmm0, %xmm1 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    pmovsxdq (%rdi), %xmm0 # sched: [7:0.50]
-; BDVER2-SSE-NEXT:    paddq %xmm1, %xmm0 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_pmovsxdq:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vpmovsxdq (%rdi), %xmm1 # sched: [7:0.50]
-; BDVER2-NEXT:    vpmovsxdq %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_pmovsxdq:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    pmovsxdq %xmm0, %xmm1 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    pmovsxdq (%rdi), %xmm0 # sched: [6:1.00]
-; BTVER2-SSE-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_pmovsxdq:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vpmovsxdq (%rdi), %xmm1 # sched: [6:1.00]
-; BTVER2-NEXT:    vpmovsxdq %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_pmovsxdq:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    pmovsxdq %xmm0, %xmm1 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    pmovsxdq (%rdi), %xmm0 # sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_pmovsxdq:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpmovsxdq (%rdi), %xmm1 # sched: [8:0.50]
-; ZNVER1-NEXT:    vpmovsxdq %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = shufflevector <4 x i32> %a0, <4 x i32> undef, <2 x i32> <i32 0, i32 1>
-  %2 = sext <2 x i32> %1 to <2 x i64>
-  %3 = load <2 x i32>, <2 x i32>* %a1, align 1
-  %4 = sext <2 x i32> %3 to <2 x i64>
-  %5 = add <2 x i64> %2, %4
-  ret <2 x i64> %5
-}
-
-define <4 x i32> @test_pmovsxwd(<8 x i16> %a0, <4 x i16> *%a1) {
-; GENERIC-LABEL: test_pmovsxwd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    pmovsxwd %xmm0, %xmm1 # sched: [1:0.50]
-; GENERIC-NEXT:    pmovsxwd (%rdi), %xmm0 # sched: [7:0.50]
-; GENERIC-NEXT:    paddd %xmm1, %xmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SLM-LABEL: test_pmovsxwd:
-; SLM:       # %bb.0:
-; SLM-NEXT:    pmovsxwd (%rdi), %xmm1 # sched: [4:1.00]
-; SLM-NEXT:    pmovsxwd %xmm0, %xmm0 # sched: [1:1.00]
-; SLM-NEXT:    paddd %xmm0, %xmm1 # sched: [1:0.50]
-; SLM-NEXT:    movdqa %xmm1, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_pmovsxwd:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    pmovsxwd %xmm0, %xmm1 # sched: [1:0.50]
-; SANDY-SSE-NEXT:    pmovsxwd (%rdi), %xmm0 # sched: [7:0.50]
-; SANDY-SSE-NEXT:    paddd %xmm1, %xmm0 # sched: [1:0.50]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_pmovsxwd:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vpmovsxwd %xmm0, %xmm0 # sched: [1:0.50]
-; SANDY-NEXT:    vpmovsxwd (%rdi), %xmm1 # sched: [7:0.50]
-; SANDY-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_pmovsxwd:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    pmovsxwd %xmm0, %xmm1 # sched: [1:1.00]
-; HASWELL-SSE-NEXT:    pmovsxwd (%rdi), %xmm0 # sched: [6:1.00]
-; HASWELL-SSE-NEXT:    paddd %xmm1, %xmm0 # sched: [1:0.50]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_pmovsxwd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpmovsxwd %xmm0, %xmm0 # sched: [1:1.00]
-; HASWELL-NEXT:    vpmovsxwd (%rdi), %xmm1 # sched: [6:1.00]
-; HASWELL-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_pmovsxwd:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    pmovsxwd %xmm0, %xmm1 # sched: [1:1.00]
-; BROADWELL-SSE-NEXT:    pmovsxwd (%rdi), %xmm0 # sched: [6:1.00]
-; BROADWELL-SSE-NEXT:    paddd %xmm1, %xmm0 # sched: [1:0.50]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pmovsxwd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpmovsxwd %xmm0, %xmm0 # sched: [1:1.00]
-; BROADWELL-NEXT:    vpmovsxwd (%rdi), %xmm1 # sched: [6:1.00]
-; BROADWELL-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_pmovsxwd:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    pmovsxwd %xmm0, %xmm1 # sched: [1:1.00]
-; SKYLAKE-SSE-NEXT:    pmovsxwd (%rdi), %xmm0 # sched: [6:1.00]
-; SKYLAKE-SSE-NEXT:    paddd %xmm1, %xmm0 # sched: [1:0.33]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pmovsxwd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpmovsxwd %xmm0, %xmm0 # sched: [1:1.00]
-; SKYLAKE-NEXT:    vpmovsxwd (%rdi), %xmm1 # sched: [6:1.00]
-; SKYLAKE-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_pmovsxwd:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    pmovsxwd %xmm0, %xmm1 # sched: [1:1.00]
-; SKX-SSE-NEXT:    pmovsxwd (%rdi), %xmm0 # sched: [6:1.00]
-; SKX-SSE-NEXT:    paddd %xmm1, %xmm0 # sched: [1:0.33]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pmovsxwd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpmovsxwd %xmm0, %xmm0 # sched: [1:1.00]
-; SKX-NEXT:    vpmovsxwd (%rdi), %xmm1 # sched: [6:1.00]
-; SKX-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_pmovsxwd:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    pmovsxwd %xmm0, %xmm1 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    pmovsxwd (%rdi), %xmm0 # sched: [7:0.50]
-; BDVER2-SSE-NEXT:    paddd %xmm1, %xmm0 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_pmovsxwd:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vpmovsxwd (%rdi), %xmm1 # sched: [7:0.50]
-; BDVER2-NEXT:    vpmovsxwd %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_pmovsxwd:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    pmovsxwd %xmm0, %xmm1 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    pmovsxwd (%rdi), %xmm0 # sched: [6:1.00]
-; BTVER2-SSE-NEXT:    paddd %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_pmovsxwd:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vpmovsxwd (%rdi), %xmm1 # sched: [6:1.00]
-; BTVER2-NEXT:    vpmovsxwd %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_pmovsxwd:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    pmovsxwd %xmm0, %xmm1 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    pmovsxwd (%rdi), %xmm0 # sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    paddd %xmm1, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_pmovsxwd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpmovsxwd (%rdi), %xmm1 # sched: [8:0.50]
-; ZNVER1-NEXT:    vpmovsxwd %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = shufflevector <8 x i16> %a0, <8 x i16> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-  %2 = sext <4 x i16> %1 to <4 x i32>
-  %3 = load <4 x i16>, <4 x i16>* %a1, align 1
-  %4 = sext <4 x i16> %3 to <4 x i32>
-  %5 = add <4 x i32> %2, %4
-  ret <4 x i32> %5
-}
-
-define <2 x i64> @test_pmovsxwq(<8 x i16> %a0, <2 x i16> *%a1) {
-; GENERIC-LABEL: test_pmovsxwq:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    pmovsxwq %xmm0, %xmm1 # sched: [1:0.50]
-; GENERIC-NEXT:    pmovsxwq (%rdi), %xmm0 # sched: [7:0.50]
-; GENERIC-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SLM-LABEL: test_pmovsxwq:
-; SLM:       # %bb.0:
-; SLM-NEXT:    pmovsxwq (%rdi), %xmm1 # sched: [4:1.00]
-; SLM-NEXT:    pmovsxwq %xmm0, %xmm0 # sched: [1:1.00]
-; SLM-NEXT:    paddq %xmm0, %xmm1 # sched: [1:0.50]
-; SLM-NEXT:    movdqa %xmm1, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_pmovsxwq:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    pmovsxwq %xmm0, %xmm1 # sched: [1:0.50]
-; SANDY-SSE-NEXT:    pmovsxwq (%rdi), %xmm0 # sched: [7:0.50]
-; SANDY-SSE-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.50]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_pmovsxwq:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vpmovsxwq %xmm0, %xmm0 # sched: [1:0.50]
-; SANDY-NEXT:    vpmovsxwq (%rdi), %xmm1 # sched: [7:0.50]
-; SANDY-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_pmovsxwq:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    pmovsxwq %xmm0, %xmm1 # sched: [1:1.00]
-; HASWELL-SSE-NEXT:    pmovsxwq (%rdi), %xmm0 # sched: [6:1.00]
-; HASWELL-SSE-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.50]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_pmovsxwq:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpmovsxwq %xmm0, %xmm0 # sched: [1:1.00]
-; HASWELL-NEXT:    vpmovsxwq (%rdi), %xmm1 # sched: [6:1.00]
-; HASWELL-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_pmovsxwq:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    pmovsxwq %xmm0, %xmm1 # sched: [1:1.00]
-; BROADWELL-SSE-NEXT:    pmovsxwq (%rdi), %xmm0 # sched: [6:1.00]
-; BROADWELL-SSE-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.50]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pmovsxwq:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpmovsxwq %xmm0, %xmm0 # sched: [1:1.00]
-; BROADWELL-NEXT:    vpmovsxwq (%rdi), %xmm1 # sched: [6:1.00]
-; BROADWELL-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_pmovsxwq:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    pmovsxwq %xmm0, %xmm1 # sched: [1:1.00]
-; SKYLAKE-SSE-NEXT:    pmovsxwq (%rdi), %xmm0 # sched: [6:1.00]
-; SKYLAKE-SSE-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.33]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pmovsxwq:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpmovsxwq %xmm0, %xmm0 # sched: [1:1.00]
-; SKYLAKE-NEXT:    vpmovsxwq (%rdi), %xmm1 # sched: [6:1.00]
-; SKYLAKE-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_pmovsxwq:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    pmovsxwq %xmm0, %xmm1 # sched: [1:1.00]
-; SKX-SSE-NEXT:    pmovsxwq (%rdi), %xmm0 # sched: [6:1.00]
-; SKX-SSE-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.33]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pmovsxwq:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpmovsxwq %xmm0, %xmm0 # sched: [1:1.00]
-; SKX-NEXT:    vpmovsxwq (%rdi), %xmm1 # sched: [6:1.00]
-; SKX-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_pmovsxwq:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    pmovsxwq %xmm0, %xmm1 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    pmovsxwq (%rdi), %xmm0 # sched: [7:0.50]
-; BDVER2-SSE-NEXT:    paddq %xmm1, %xmm0 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_pmovsxwq:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vpmovsxwq (%rdi), %xmm1 # sched: [7:0.50]
-; BDVER2-NEXT:    vpmovsxwq %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_pmovsxwq:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    pmovsxwq %xmm0, %xmm1 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    pmovsxwq (%rdi), %xmm0 # sched: [6:1.00]
-; BTVER2-SSE-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_pmovsxwq:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vpmovsxwq (%rdi), %xmm1 # sched: [6:1.00]
-; BTVER2-NEXT:    vpmovsxwq %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_pmovsxwq:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    pmovsxwq %xmm0, %xmm1 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    pmovsxwq (%rdi), %xmm0 # sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_pmovsxwq:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpmovsxwq (%rdi), %xmm1 # sched: [8:0.50]
-; ZNVER1-NEXT:    vpmovsxwq %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = shufflevector <8 x i16> %a0, <8 x i16> undef, <2 x i32> <i32 0, i32 1>
-  %2 = sext <2 x i16> %1 to <2 x i64>
-  %3 = load <2 x i16>, <2 x i16>* %a1, align 1
-  %4 = sext <2 x i16> %3 to <2 x i64>
-  %5 = add <2 x i64> %2, %4
-  ret <2 x i64> %5
-}
-
-define <8 x i16> @test_pmovzxbw(<16 x i8> %a0, <8 x i8> *%a1) {
-; GENERIC-LABEL: test_pmovzxbw:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    pmovzxbw {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero sched: [1:0.50]
-; GENERIC-NEXT:    pmovzxbw {{.*#+}} xmm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero sched: [7:0.50]
-; GENERIC-NEXT:    paddw %xmm1, %xmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SLM-LABEL: test_pmovzxbw:
-; SLM:       # %bb.0:
-; SLM-NEXT:    pmovzxbw {{.*#+}} xmm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero sched: [4:1.00]
-; SLM-NEXT:    pmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero sched: [1:1.00]
-; SLM-NEXT:    paddw %xmm0, %xmm1 # sched: [1:0.50]
-; SLM-NEXT:    movdqa %xmm1, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_pmovzxbw:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    pmovzxbw {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero sched: [1:0.50]
-; SANDY-SSE-NEXT:    pmovzxbw {{.*#+}} xmm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero sched: [7:0.50]
-; SANDY-SSE-NEXT:    paddw %xmm1, %xmm0 # sched: [1:0.50]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_pmovzxbw:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero sched: [1:0.50]
-; SANDY-NEXT:    vpmovzxbw {{.*#+}} xmm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero sched: [7:0.50]
-; SANDY-NEXT:    vpaddw %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_pmovzxbw:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    pmovzxbw {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero sched: [1:1.00]
-; HASWELL-SSE-NEXT:    pmovzxbw {{.*#+}} xmm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero sched: [6:1.00]
-; HASWELL-SSE-NEXT:    paddw %xmm1, %xmm0 # sched: [1:0.50]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_pmovzxbw:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero sched: [1:1.00]
-; HASWELL-NEXT:    vpmovzxbw {{.*#+}} xmm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero sched: [6:1.00]
-; HASWELL-NEXT:    vpaddw %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_pmovzxbw:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    pmovzxbw {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero sched: [1:1.00]
-; BROADWELL-SSE-NEXT:    pmovzxbw {{.*#+}} xmm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero sched: [6:1.00]
-; BROADWELL-SSE-NEXT:    paddw %xmm1, %xmm0 # sched: [1:0.50]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pmovzxbw:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero sched: [1:1.00]
-; BROADWELL-NEXT:    vpmovzxbw {{.*#+}} xmm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero sched: [6:1.00]
-; BROADWELL-NEXT:    vpaddw %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_pmovzxbw:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    pmovzxbw {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero sched: [1:1.00]
-; SKYLAKE-SSE-NEXT:    pmovzxbw {{.*#+}} xmm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero sched: [6:1.00]
-; SKYLAKE-SSE-NEXT:    paddw %xmm1, %xmm0 # sched: [1:0.33]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pmovzxbw:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero sched: [1:1.00]
-; SKYLAKE-NEXT:    vpmovzxbw {{.*#+}} xmm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero sched: [6:1.00]
-; SKYLAKE-NEXT:    vpaddw %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_pmovzxbw:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    pmovzxbw {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero sched: [1:1.00]
-; SKX-SSE-NEXT:    pmovzxbw {{.*#+}} xmm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero sched: [6:1.00]
-; SKX-SSE-NEXT:    paddw %xmm1, %xmm0 # sched: [1:0.33]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pmovzxbw:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero sched: [1:1.00]
-; SKX-NEXT:    vpmovzxbw {{.*#+}} xmm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero sched: [6:1.00]
-; SKX-NEXT:    vpaddw %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_pmovzxbw:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    pmovzxbw {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero sched: [2:0.50]
-; BDVER2-SSE-NEXT:    pmovzxbw {{.*#+}} xmm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero sched: [7:0.50]
-; BDVER2-SSE-NEXT:    paddw %xmm1, %xmm0 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_pmovzxbw:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vpmovzxbw {{.*#+}} xmm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero sched: [7:0.50]
-; BDVER2-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero sched: [2:0.50]
-; BDVER2-NEXT:    vpaddw %xmm1, %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_pmovzxbw:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    pmovzxbw {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero sched: [1:0.50]
-; BTVER2-SSE-NEXT:    pmovzxbw {{.*#+}} xmm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero sched: [6:1.00]
-; BTVER2-SSE-NEXT:    paddw %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_pmovzxbw:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vpmovzxbw {{.*#+}} xmm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero sched: [6:1.00]
-; BTVER2-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero sched: [1:0.50]
-; BTVER2-NEXT:    vpaddw %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_pmovzxbw:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    pmovzxbw {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    pmovzxbw {{.*#+}} xmm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    paddw %xmm1, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_pmovzxbw:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpmovzxbw {{.*#+}} xmm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero sched: [8:0.50]
-; ZNVER1-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero sched: [1:0.25]
-; ZNVER1-NEXT:    vpaddw %xmm1, %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = shufflevector <16 x i8> %a0, <16 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-  %2 = zext <8 x i8> %1 to <8 x i16>
-  %3 = load <8 x i8>, <8 x i8>* %a1, align 1
-  %4 = zext <8 x i8> %3 to <8 x i16>
-  %5 = add <8 x i16> %2, %4
-  ret <8 x i16> %5
-}
-
-define <4 x i32> @test_pmovzxbd(<16 x i8> %a0, <4 x i8> *%a1) {
-; GENERIC-LABEL: test_pmovzxbd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    pmovzxbd {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero sched: [1:0.50]
-; GENERIC-NEXT:    pmovzxbd {{.*#+}} xmm0 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero sched: [7:0.50]
-; GENERIC-NEXT:    paddd %xmm1, %xmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SLM-LABEL: test_pmovzxbd:
-; SLM:       # %bb.0:
-; SLM-NEXT:    pmovzxbd {{.*#+}} xmm1 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero sched: [4:1.00]
-; SLM-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero sched: [1:1.00]
-; SLM-NEXT:    paddd %xmm0, %xmm1 # sched: [1:0.50]
-; SLM-NEXT:    movdqa %xmm1, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_pmovzxbd:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    pmovzxbd {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero sched: [1:0.50]
-; SANDY-SSE-NEXT:    pmovzxbd {{.*#+}} xmm0 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero sched: [7:0.50]
-; SANDY-SSE-NEXT:    paddd %xmm1, %xmm0 # sched: [1:0.50]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_pmovzxbd:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero sched: [1:0.50]
-; SANDY-NEXT:    vpmovzxbd {{.*#+}} xmm1 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero sched: [7:0.50]
-; SANDY-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_pmovzxbd:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    pmovzxbd {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero sched: [1:1.00]
-; HASWELL-SSE-NEXT:    pmovzxbd {{.*#+}} xmm0 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero sched: [6:1.00]
-; HASWELL-SSE-NEXT:    paddd %xmm1, %xmm0 # sched: [1:0.50]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_pmovzxbd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero sched: [1:1.00]
-; HASWELL-NEXT:    vpmovzxbd {{.*#+}} xmm1 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero sched: [6:1.00]
-; HASWELL-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_pmovzxbd:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    pmovzxbd {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero sched: [1:1.00]
-; BROADWELL-SSE-NEXT:    pmovzxbd {{.*#+}} xmm0 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero sched: [6:1.00]
-; BROADWELL-SSE-NEXT:    paddd %xmm1, %xmm0 # sched: [1:0.50]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pmovzxbd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero sched: [1:1.00]
-; BROADWELL-NEXT:    vpmovzxbd {{.*#+}} xmm1 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero sched: [6:1.00]
-; BROADWELL-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_pmovzxbd:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    pmovzxbd {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero sched: [1:1.00]
-; SKYLAKE-SSE-NEXT:    pmovzxbd {{.*#+}} xmm0 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero sched: [6:1.00]
-; SKYLAKE-SSE-NEXT:    paddd %xmm1, %xmm0 # sched: [1:0.33]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pmovzxbd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero sched: [1:1.00]
-; SKYLAKE-NEXT:    vpmovzxbd {{.*#+}} xmm1 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero sched: [6:1.00]
-; SKYLAKE-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_pmovzxbd:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    pmovzxbd {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero sched: [1:1.00]
-; SKX-SSE-NEXT:    pmovzxbd {{.*#+}} xmm0 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero sched: [6:1.00]
-; SKX-SSE-NEXT:    paddd %xmm1, %xmm0 # sched: [1:0.33]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pmovzxbd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero sched: [1:1.00]
-; SKX-NEXT:    vpmovzxbd {{.*#+}} xmm1 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero sched: [6:1.00]
-; SKX-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_pmovzxbd:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    pmovzxbd {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero sched: [2:0.50]
-; BDVER2-SSE-NEXT:    pmovzxbd {{.*#+}} xmm0 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero sched: [7:0.50]
-; BDVER2-SSE-NEXT:    paddd %xmm1, %xmm0 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_pmovzxbd:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vpmovzxbd {{.*#+}} xmm1 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero sched: [7:0.50]
-; BDVER2-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero sched: [2:0.50]
-; BDVER2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_pmovzxbd:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    pmovzxbd {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero sched: [1:0.50]
-; BTVER2-SSE-NEXT:    pmovzxbd {{.*#+}} xmm0 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero sched: [6:1.00]
-; BTVER2-SSE-NEXT:    paddd %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_pmovzxbd:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vpmovzxbd {{.*#+}} xmm1 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero sched: [6:1.00]
-; BTVER2-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero sched: [1:0.50]
-; BTVER2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_pmovzxbd:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    pmovzxbd {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    pmovzxbd {{.*#+}} xmm0 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    paddd %xmm1, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_pmovzxbd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpmovzxbd {{.*#+}} xmm1 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero sched: [8:0.50]
-; ZNVER1-NEXT:    vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero sched: [1:0.25]
-; ZNVER1-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = shufflevector <16 x i8> %a0, <16 x i8> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-  %2 = zext <4 x i8> %1 to <4 x i32>
-  %3 = load <4 x i8>, <4 x i8>* %a1, align 1
-  %4 = zext <4 x i8> %3 to <4 x i32>
-  %5 = add <4 x i32> %2, %4
-  ret <4 x i32> %5
-}
-
-define <2 x i64> @test_pmovzxbq(<16 x i8> %a0, <2 x i8> *%a1) {
-; GENERIC-LABEL: test_pmovzxbq:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    pmovzxbq {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero sched: [1:0.50]
-; GENERIC-NEXT:    pmovzxbq {{.*#+}} xmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero sched: [7:0.50]
-; GENERIC-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SLM-LABEL: test_pmovzxbq:
-; SLM:       # %bb.0:
-; SLM-NEXT:    pmovzxbq {{.*#+}} xmm1 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero sched: [4:1.00]
-; SLM-NEXT:    pmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero sched: [1:1.00]
-; SLM-NEXT:    paddq %xmm0, %xmm1 # sched: [1:0.50]
-; SLM-NEXT:    movdqa %xmm1, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_pmovzxbq:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    pmovzxbq {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero sched: [1:0.50]
-; SANDY-SSE-NEXT:    pmovzxbq {{.*#+}} xmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero sched: [7:0.50]
-; SANDY-SSE-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.50]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_pmovzxbq:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vpmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero sched: [1:0.50]
-; SANDY-NEXT:    vpmovzxbq {{.*#+}} xmm1 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero sched: [7:0.50]
-; SANDY-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_pmovzxbq:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    pmovzxbq {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero sched: [1:1.00]
-; HASWELL-SSE-NEXT:    pmovzxbq {{.*#+}} xmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero sched: [6:1.00]
-; HASWELL-SSE-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.50]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_pmovzxbq:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero sched: [1:1.00]
-; HASWELL-NEXT:    vpmovzxbq {{.*#+}} xmm1 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero sched: [6:1.00]
-; HASWELL-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_pmovzxbq:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    pmovzxbq {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero sched: [1:1.00]
-; BROADWELL-SSE-NEXT:    pmovzxbq {{.*#+}} xmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero sched: [6:1.00]
-; BROADWELL-SSE-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.50]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pmovzxbq:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero sched: [1:1.00]
-; BROADWELL-NEXT:    vpmovzxbq {{.*#+}} xmm1 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero sched: [6:1.00]
-; BROADWELL-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_pmovzxbq:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    pmovzxbq {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero sched: [1:1.00]
-; SKYLAKE-SSE-NEXT:    pmovzxbq {{.*#+}} xmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero sched: [6:1.00]
-; SKYLAKE-SSE-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.33]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pmovzxbq:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero sched: [1:1.00]
-; SKYLAKE-NEXT:    vpmovzxbq {{.*#+}} xmm1 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero sched: [6:1.00]
-; SKYLAKE-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_pmovzxbq:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    pmovzxbq {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero sched: [1:1.00]
-; SKX-SSE-NEXT:    pmovzxbq {{.*#+}} xmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero sched: [6:1.00]
-; SKX-SSE-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.33]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pmovzxbq:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero sched: [1:1.00]
-; SKX-NEXT:    vpmovzxbq {{.*#+}} xmm1 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero sched: [6:1.00]
-; SKX-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_pmovzxbq:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    pmovzxbq {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero sched: [2:0.50]
-; BDVER2-SSE-NEXT:    pmovzxbq {{.*#+}} xmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero sched: [7:0.50]
-; BDVER2-SSE-NEXT:    paddq %xmm1, %xmm0 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_pmovzxbq:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vpmovzxbq {{.*#+}} xmm1 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero sched: [7:0.50]
-; BDVER2-NEXT:    vpmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero sched: [2:0.50]
-; BDVER2-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_pmovzxbq:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    pmovzxbq {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero sched: [1:0.50]
-; BTVER2-SSE-NEXT:    pmovzxbq {{.*#+}} xmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero sched: [6:1.00]
-; BTVER2-SSE-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_pmovzxbq:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vpmovzxbq {{.*#+}} xmm1 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero sched: [6:1.00]
-; BTVER2-NEXT:    vpmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero sched: [1:0.50]
-; BTVER2-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_pmovzxbq:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    pmovzxbq {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    pmovzxbq {{.*#+}} xmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_pmovzxbq:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpmovzxbq {{.*#+}} xmm1 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero sched: [8:0.50]
-; ZNVER1-NEXT:    vpmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero sched: [1:0.25]
-; ZNVER1-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = shufflevector <16 x i8> %a0, <16 x i8> undef, <2 x i32> <i32 0, i32 1>
-  %2 = zext <2 x i8> %1 to <2 x i64>
-  %3 = load <2 x i8>, <2 x i8>* %a1, align 1
-  %4 = zext <2 x i8> %3 to <2 x i64>
-  %5 = add <2 x i64> %2, %4
-  ret <2 x i64> %5
-}
-
-define <2 x i64> @test_pmovzxdq(<4 x i32> %a0, <2 x i32> *%a1) {
-; GENERIC-LABEL: test_pmovzxdq:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    pmovzxdq {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero sched: [1:0.50]
-; GENERIC-NEXT:    pmovzxdq {{.*#+}} xmm0 = mem[0],zero,mem[1],zero sched: [7:0.50]
-; GENERIC-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SLM-LABEL: test_pmovzxdq:
-; SLM:       # %bb.0:
-; SLM-NEXT:    pmovzxdq {{.*#+}} xmm1 = mem[0],zero,mem[1],zero sched: [4:1.00]
-; SLM-NEXT:    pmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero sched: [1:1.00]
-; SLM-NEXT:    paddq %xmm0, %xmm1 # sched: [1:0.50]
-; SLM-NEXT:    movdqa %xmm1, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_pmovzxdq:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    pmovzxdq {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero sched: [1:0.50]
-; SANDY-SSE-NEXT:    pmovzxdq {{.*#+}} xmm0 = mem[0],zero,mem[1],zero sched: [7:0.50]
-; SANDY-SSE-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.50]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_pmovzxdq:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero sched: [1:0.50]
-; SANDY-NEXT:    vpmovzxdq {{.*#+}} xmm1 = mem[0],zero,mem[1],zero sched: [7:0.50]
-; SANDY-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_pmovzxdq:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    pmovzxdq {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero sched: [1:1.00]
-; HASWELL-SSE-NEXT:    pmovzxdq {{.*#+}} xmm0 = mem[0],zero,mem[1],zero sched: [6:1.00]
-; HASWELL-SSE-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.50]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_pmovzxdq:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero sched: [1:1.00]
-; HASWELL-NEXT:    vpmovzxdq {{.*#+}} xmm1 = mem[0],zero,mem[1],zero sched: [6:1.00]
-; HASWELL-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_pmovzxdq:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    pmovzxdq {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero sched: [1:1.00]
-; BROADWELL-SSE-NEXT:    pmovzxdq {{.*#+}} xmm0 = mem[0],zero,mem[1],zero sched: [6:1.00]
-; BROADWELL-SSE-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.50]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pmovzxdq:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero sched: [1:1.00]
-; BROADWELL-NEXT:    vpmovzxdq {{.*#+}} xmm1 = mem[0],zero,mem[1],zero sched: [6:1.00]
-; BROADWELL-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_pmovzxdq:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    pmovzxdq {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero sched: [1:1.00]
-; SKYLAKE-SSE-NEXT:    pmovzxdq {{.*#+}} xmm0 = mem[0],zero,mem[1],zero sched: [6:1.00]
-; SKYLAKE-SSE-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.33]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pmovzxdq:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero sched: [1:1.00]
-; SKYLAKE-NEXT:    vpmovzxdq {{.*#+}} xmm1 = mem[0],zero,mem[1],zero sched: [6:1.00]
-; SKYLAKE-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_pmovzxdq:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    pmovzxdq {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero sched: [1:1.00]
-; SKX-SSE-NEXT:    pmovzxdq {{.*#+}} xmm0 = mem[0],zero,mem[1],zero sched: [6:1.00]
-; SKX-SSE-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.33]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pmovzxdq:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero sched: [1:1.00]
-; SKX-NEXT:    vpmovzxdq {{.*#+}} xmm1 = mem[0],zero,mem[1],zero sched: [6:1.00]
-; SKX-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_pmovzxdq:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    pmovzxdq {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero sched: [2:0.50]
-; BDVER2-SSE-NEXT:    pmovzxdq {{.*#+}} xmm0 = mem[0],zero,mem[1],zero sched: [7:0.50]
-; BDVER2-SSE-NEXT:    paddq %xmm1, %xmm0 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_pmovzxdq:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vpmovzxdq {{.*#+}} xmm1 = mem[0],zero,mem[1],zero sched: [7:0.50]
-; BDVER2-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero sched: [2:0.50]
-; BDVER2-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_pmovzxdq:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    pmovzxdq {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero sched: [1:0.50]
-; BTVER2-SSE-NEXT:    pmovzxdq {{.*#+}} xmm0 = mem[0],zero,mem[1],zero sched: [6:1.00]
-; BTVER2-SSE-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_pmovzxdq:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vpmovzxdq {{.*#+}} xmm1 = mem[0],zero,mem[1],zero sched: [6:1.00]
-; BTVER2-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero sched: [1:0.50]
-; BTVER2-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_pmovzxdq:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    pmovzxdq {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    pmovzxdq {{.*#+}} xmm0 = mem[0],zero,mem[1],zero sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_pmovzxdq:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpmovzxdq {{.*#+}} xmm1 = mem[0],zero,mem[1],zero sched: [8:0.50]
-; ZNVER1-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero sched: [1:0.25]
-; ZNVER1-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = shufflevector <4 x i32> %a0, <4 x i32> undef, <2 x i32> <i32 0, i32 1>
-  %2 = zext <2 x i32> %1 to <2 x i64>
-  %3 = load <2 x i32>, <2 x i32>* %a1, align 1
-  %4 = zext <2 x i32> %3 to <2 x i64>
-  %5 = add <2 x i64> %2, %4
-  ret <2 x i64> %5
-}
-
-define <4 x i32> @test_pmovzxwd(<8 x i16> %a0, <4 x i16> *%a1) {
-; GENERIC-LABEL: test_pmovzxwd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    pmovzxwd {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero sched: [1:0.50]
-; GENERIC-NEXT:    pmovzxwd {{.*#+}} xmm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero sched: [7:0.50]
-; GENERIC-NEXT:    paddd %xmm1, %xmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SLM-LABEL: test_pmovzxwd:
-; SLM:       # %bb.0:
-; SLM-NEXT:    pmovzxwd {{.*#+}} xmm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero sched: [4:1.00]
-; SLM-NEXT:    pmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero sched: [1:1.00]
-; SLM-NEXT:    paddd %xmm0, %xmm1 # sched: [1:0.50]
-; SLM-NEXT:    movdqa %xmm1, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_pmovzxwd:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    pmovzxwd {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero sched: [1:0.50]
-; SANDY-SSE-NEXT:    pmovzxwd {{.*#+}} xmm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero sched: [7:0.50]
-; SANDY-SSE-NEXT:    paddd %xmm1, %xmm0 # sched: [1:0.50]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_pmovzxwd:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero sched: [1:0.50]
-; SANDY-NEXT:    vpmovzxwd {{.*#+}} xmm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero sched: [7:0.50]
-; SANDY-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_pmovzxwd:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    pmovzxwd {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero sched: [1:1.00]
-; HASWELL-SSE-NEXT:    pmovzxwd {{.*#+}} xmm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero sched: [6:1.00]
-; HASWELL-SSE-NEXT:    paddd %xmm1, %xmm0 # sched: [1:0.50]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_pmovzxwd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero sched: [1:1.00]
-; HASWELL-NEXT:    vpmovzxwd {{.*#+}} xmm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero sched: [6:1.00]
-; HASWELL-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_pmovzxwd:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    pmovzxwd {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero sched: [1:1.00]
-; BROADWELL-SSE-NEXT:    pmovzxwd {{.*#+}} xmm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero sched: [6:1.00]
-; BROADWELL-SSE-NEXT:    paddd %xmm1, %xmm0 # sched: [1:0.50]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pmovzxwd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero sched: [1:1.00]
-; BROADWELL-NEXT:    vpmovzxwd {{.*#+}} xmm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero sched: [6:1.00]
-; BROADWELL-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_pmovzxwd:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    pmovzxwd {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero sched: [1:1.00]
-; SKYLAKE-SSE-NEXT:    pmovzxwd {{.*#+}} xmm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero sched: [6:1.00]
-; SKYLAKE-SSE-NEXT:    paddd %xmm1, %xmm0 # sched: [1:0.33]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pmovzxwd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero sched: [1:1.00]
-; SKYLAKE-NEXT:    vpmovzxwd {{.*#+}} xmm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero sched: [6:1.00]
-; SKYLAKE-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_pmovzxwd:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    pmovzxwd {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero sched: [1:1.00]
-; SKX-SSE-NEXT:    pmovzxwd {{.*#+}} xmm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero sched: [6:1.00]
-; SKX-SSE-NEXT:    paddd %xmm1, %xmm0 # sched: [1:0.33]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pmovzxwd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero sched: [1:1.00]
-; SKX-NEXT:    vpmovzxwd {{.*#+}} xmm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero sched: [6:1.00]
-; SKX-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_pmovzxwd:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    pmovzxwd {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero sched: [2:0.50]
-; BDVER2-SSE-NEXT:    pmovzxwd {{.*#+}} xmm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero sched: [7:0.50]
-; BDVER2-SSE-NEXT:    paddd %xmm1, %xmm0 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_pmovzxwd:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vpmovzxwd {{.*#+}} xmm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero sched: [7:0.50]
-; BDVER2-NEXT:    vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero sched: [2:0.50]
-; BDVER2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_pmovzxwd:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    pmovzxwd {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero sched: [1:0.50]
-; BTVER2-SSE-NEXT:    pmovzxwd {{.*#+}} xmm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero sched: [6:1.00]
-; BTVER2-SSE-NEXT:    paddd %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_pmovzxwd:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vpmovzxwd {{.*#+}} xmm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero sched: [6:1.00]
-; BTVER2-NEXT:    vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero sched: [1:0.50]
-; BTVER2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_pmovzxwd:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    pmovzxwd {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    pmovzxwd {{.*#+}} xmm0 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    paddd %xmm1, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_pmovzxwd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpmovzxwd {{.*#+}} xmm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero sched: [8:0.50]
-; ZNVER1-NEXT:    vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero sched: [1:0.25]
-; ZNVER1-NEXT:    vpaddd %xmm1, %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = shufflevector <8 x i16> %a0, <8 x i16> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
-  %2 = zext <4 x i16> %1 to <4 x i32>
-  %3 = load <4 x i16>, <4 x i16>* %a1, align 1
-  %4 = zext <4 x i16> %3 to <4 x i32>
-  %5 = add <4 x i32> %2, %4
-  ret <4 x i32> %5
-}
-
-define <2 x i64> @test_pmovzxwq(<8 x i16> %a0, <2 x i16> *%a1) {
-; GENERIC-LABEL: test_pmovzxwq:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    pmovzxwq {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero sched: [1:0.50]
-; GENERIC-NEXT:    pmovzxwq {{.*#+}} xmm0 = mem[0],zero,zero,zero,mem[1],zero,zero,zero sched: [7:0.50]
-; GENERIC-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SLM-LABEL: test_pmovzxwq:
-; SLM:       # %bb.0:
-; SLM-NEXT:    pmovzxwq {{.*#+}} xmm1 = mem[0],zero,zero,zero,mem[1],zero,zero,zero sched: [4:1.00]
-; SLM-NEXT:    pmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero sched: [1:1.00]
-; SLM-NEXT:    paddq %xmm0, %xmm1 # sched: [1:0.50]
-; SLM-NEXT:    movdqa %xmm1, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_pmovzxwq:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    pmovzxwq {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero sched: [1:0.50]
-; SANDY-SSE-NEXT:    pmovzxwq {{.*#+}} xmm0 = mem[0],zero,zero,zero,mem[1],zero,zero,zero sched: [7:0.50]
-; SANDY-SSE-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.50]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_pmovzxwq:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero sched: [1:0.50]
-; SANDY-NEXT:    vpmovzxwq {{.*#+}} xmm1 = mem[0],zero,zero,zero,mem[1],zero,zero,zero sched: [7:0.50]
-; SANDY-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_pmovzxwq:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    pmovzxwq {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero sched: [1:1.00]
-; HASWELL-SSE-NEXT:    pmovzxwq {{.*#+}} xmm0 = mem[0],zero,zero,zero,mem[1],zero,zero,zero sched: [6:1.00]
-; HASWELL-SSE-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.50]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_pmovzxwq:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero sched: [1:1.00]
-; HASWELL-NEXT:    vpmovzxwq {{.*#+}} xmm1 = mem[0],zero,zero,zero,mem[1],zero,zero,zero sched: [6:1.00]
-; HASWELL-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_pmovzxwq:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    pmovzxwq {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero sched: [1:1.00]
-; BROADWELL-SSE-NEXT:    pmovzxwq {{.*#+}} xmm0 = mem[0],zero,zero,zero,mem[1],zero,zero,zero sched: [6:1.00]
-; BROADWELL-SSE-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.50]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pmovzxwq:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero sched: [1:1.00]
-; BROADWELL-NEXT:    vpmovzxwq {{.*#+}} xmm1 = mem[0],zero,zero,zero,mem[1],zero,zero,zero sched: [6:1.00]
-; BROADWELL-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_pmovzxwq:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    pmovzxwq {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero sched: [1:1.00]
-; SKYLAKE-SSE-NEXT:    pmovzxwq {{.*#+}} xmm0 = mem[0],zero,zero,zero,mem[1],zero,zero,zero sched: [6:1.00]
-; SKYLAKE-SSE-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.33]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pmovzxwq:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero sched: [1:1.00]
-; SKYLAKE-NEXT:    vpmovzxwq {{.*#+}} xmm1 = mem[0],zero,zero,zero,mem[1],zero,zero,zero sched: [6:1.00]
-; SKYLAKE-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_pmovzxwq:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    pmovzxwq {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero sched: [1:1.00]
-; SKX-SSE-NEXT:    pmovzxwq {{.*#+}} xmm0 = mem[0],zero,zero,zero,mem[1],zero,zero,zero sched: [6:1.00]
-; SKX-SSE-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.33]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pmovzxwq:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero sched: [1:1.00]
-; SKX-NEXT:    vpmovzxwq {{.*#+}} xmm1 = mem[0],zero,zero,zero,mem[1],zero,zero,zero sched: [6:1.00]
-; SKX-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_pmovzxwq:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    pmovzxwq {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero sched: [2:0.50]
-; BDVER2-SSE-NEXT:    pmovzxwq {{.*#+}} xmm0 = mem[0],zero,zero,zero,mem[1],zero,zero,zero sched: [7:0.50]
-; BDVER2-SSE-NEXT:    paddq %xmm1, %xmm0 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_pmovzxwq:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vpmovzxwq {{.*#+}} xmm1 = mem[0],zero,zero,zero,mem[1],zero,zero,zero sched: [7:0.50]
-; BDVER2-NEXT:    vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero sched: [2:0.50]
-; BDVER2-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_pmovzxwq:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    pmovzxwq {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero sched: [1:0.50]
-; BTVER2-SSE-NEXT:    pmovzxwq {{.*#+}} xmm0 = mem[0],zero,zero,zero,mem[1],zero,zero,zero sched: [6:1.00]
-; BTVER2-SSE-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_pmovzxwq:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vpmovzxwq {{.*#+}} xmm1 = mem[0],zero,zero,zero,mem[1],zero,zero,zero sched: [6:1.00]
-; BTVER2-NEXT:    vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero sched: [1:0.50]
-; BTVER2-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_pmovzxwq:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    pmovzxwq {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    pmovzxwq {{.*#+}} xmm0 = mem[0],zero,zero,zero,mem[1],zero,zero,zero sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    paddq %xmm1, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_pmovzxwq:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpmovzxwq {{.*#+}} xmm1 = mem[0],zero,zero,zero,mem[1],zero,zero,zero sched: [8:0.50]
-; ZNVER1-NEXT:    vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero sched: [1:0.25]
-; ZNVER1-NEXT:    vpaddq %xmm1, %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = shufflevector <8 x i16> %a0, <8 x i16> undef, <2 x i32> <i32 0, i32 1>
-  %2 = zext <2 x i16> %1 to <2 x i64>
-  %3 = load <2 x i16>, <2 x i16>* %a1, align 1
-  %4 = zext <2 x i16> %3 to <2 x i64>
-  %5 = add <2 x i64> %2, %4
-  ret <2 x i64> %5
-}
-
-define <2 x i64> @test_pmuldq(<4 x i32> %a0, <4 x i32> %a1, <4 x i32> %a2, <4 x i32> *%a3) {
-; GENERIC-LABEL: test_pmuldq:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    pmuldq %xmm1, %xmm0 # sched: [5:1.00]
-; GENERIC-NEXT:    pmuldq (%rdi), %xmm2 # sched: [11:1.00]
-; GENERIC-NEXT:    por %xmm2, %xmm0 # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SLM-LABEL: test_pmuldq:
-; SLM:       # %bb.0:
-; SLM-NEXT:    pmuldq (%rdi), %xmm2 # sched: [7:1.00]
-; SLM-NEXT:    pmuldq %xmm1, %xmm0 # sched: [4:1.00]
-; SLM-NEXT:    por %xmm2, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_pmuldq:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    pmuldq %xmm1, %xmm0 # sched: [5:1.00]
-; SANDY-SSE-NEXT:    pmuldq (%rdi), %xmm2 # sched: [11:1.00]
-; SANDY-SSE-NEXT:    por %xmm2, %xmm0 # sched: [1:0.33]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_pmuldq:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vpmuldq %xmm1, %xmm0, %xmm0 # sched: [5:1.00]
-; SANDY-NEXT:    vpmuldq (%rdi), %xmm2, %xmm1 # sched: [11:1.00]
-; SANDY-NEXT:    vpor %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_pmuldq:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    pmuldq %xmm1, %xmm0 # sched: [5:1.00]
-; HASWELL-SSE-NEXT:    pmuldq (%rdi), %xmm2 # sched: [11:1.00]
-; HASWELL-SSE-NEXT:    por %xmm2, %xmm0 # sched: [1:0.33]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_pmuldq:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpmuldq %xmm1, %xmm0, %xmm0 # sched: [5:1.00]
-; HASWELL-NEXT:    vpmuldq (%rdi), %xmm2, %xmm1 # sched: [11:1.00]
-; HASWELL-NEXT:    vpor %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_pmuldq:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    pmuldq %xmm1, %xmm0 # sched: [5:1.00]
-; BROADWELL-SSE-NEXT:    pmuldq (%rdi), %xmm2 # sched: [10:1.00]
-; BROADWELL-SSE-NEXT:    por %xmm2, %xmm0 # sched: [1:0.33]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pmuldq:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpmuldq %xmm1, %xmm0, %xmm0 # sched: [5:1.00]
-; BROADWELL-NEXT:    vpmuldq (%rdi), %xmm2, %xmm1 # sched: [10:1.00]
-; BROADWELL-NEXT:    vpor %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_pmuldq:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    pmuldq %xmm1, %xmm0 # sched: [4:0.50]
-; SKYLAKE-SSE-NEXT:    pmuldq (%rdi), %xmm2 # sched: [10:0.50]
-; SKYLAKE-SSE-NEXT:    por %xmm2, %xmm0 # sched: [1:0.33]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pmuldq:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpmuldq %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    vpmuldq (%rdi), %xmm2, %xmm1 # sched: [10:0.50]
-; SKYLAKE-NEXT:    vpor %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_pmuldq:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    pmuldq %xmm1, %xmm0 # sched: [4:0.50]
-; SKX-SSE-NEXT:    pmuldq (%rdi), %xmm2 # sched: [10:0.50]
-; SKX-SSE-NEXT:    por %xmm2, %xmm0 # sched: [1:0.33]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pmuldq:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpmuldq %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKX-NEXT:    vpmuldq (%rdi), %xmm2, %xmm1 # sched: [10:0.50]
-; SKX-NEXT:    vpor %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_pmuldq:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    pmuldq (%rdi), %xmm2 # sched: [9:1.00]
-; BDVER2-SSE-NEXT:    pmuldq %xmm1, %xmm0 # sched: [4:1.00]
-; BDVER2-SSE-NEXT:    por %xmm2, %xmm0 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_pmuldq:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vpmuldq (%rdi), %xmm2, %xmm2 # sched: [9:1.00]
-; BDVER2-NEXT:    vpmuldq %xmm1, %xmm0, %xmm0 # sched: [4:1.00]
-; BDVER2-NEXT:    vpor %xmm2, %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_pmuldq:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    pmuldq (%rdi), %xmm2 # sched: [7:1.00]
-; BTVER2-SSE-NEXT:    pmuldq %xmm1, %xmm0 # sched: [2:1.00]
-; BTVER2-SSE-NEXT:    por %xmm2, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_pmuldq:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vpmuldq (%rdi), %xmm2, %xmm2 # sched: [7:1.00]
-; BTVER2-NEXT:    vpmuldq %xmm1, %xmm0, %xmm0 # sched: [2:1.00]
-; BTVER2-NEXT:    vpor %xmm2, %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_pmuldq:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    pmuldq (%rdi), %xmm2 # sched: [11:1.00]
-; ZNVER1-SSE-NEXT:    pmuldq %xmm1, %xmm0 # sched: [4:1.00]
-; ZNVER1-SSE-NEXT:    por %xmm2, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_pmuldq:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpmuldq (%rdi), %xmm2, %xmm2 # sched: [11:1.00]
-; ZNVER1-NEXT:    vpmuldq %xmm1, %xmm0, %xmm0 # sched: [4:1.00]
-; ZNVER1-NEXT:    vpor %xmm2, %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <2 x i64> @llvm.x86.sse41.pmuldq(<4 x i32> %a0, <4 x i32> %a1)
-  %2 = load <4 x i32>, <4 x i32> *%a3, align 16
-  %3 = call <2 x i64> @llvm.x86.sse41.pmuldq(<4 x i32> %a2, <4 x i32> %2)
-  %4 = or <2 x i64> %1, %3
-  ret <2 x i64> %4
-}
-declare <2 x i64> @llvm.x86.sse41.pmuldq(<4 x i32>, <4 x i32>) nounwind readnone
-
-define <4 x i32> @test_pmulld(<4 x i32> %a0, <4 x i32> %a1, <4 x i32> *%a2) {
-; GENERIC-LABEL: test_pmulld:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    pmulld %xmm1, %xmm0 # sched: [5:1.00]
-; GENERIC-NEXT:    pmulld (%rdi), %xmm0 # sched: [11:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SLM-LABEL: test_pmulld:
-; SLM:       # %bb.0:
-; SLM-NEXT:    pmulld %xmm1, %xmm0 # sched: [4:1.00]
-; SLM-NEXT:    pmulld (%rdi), %xmm0 # sched: [7:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_pmulld:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    pmulld %xmm1, %xmm0 # sched: [5:1.00]
-; SANDY-SSE-NEXT:    pmulld (%rdi), %xmm0 # sched: [11:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_pmulld:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vpmulld %xmm1, %xmm0, %xmm0 # sched: [5:1.00]
-; SANDY-NEXT:    vpmulld (%rdi), %xmm0, %xmm0 # sched: [11:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_pmulld:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    pmulld %xmm1, %xmm0 # sched: [10:2.00]
-; HASWELL-SSE-NEXT:    pmulld (%rdi), %xmm0 # sched: [16:2.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_pmulld:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpmulld %xmm1, %xmm0, %xmm0 # sched: [10:2.00]
-; HASWELL-NEXT:    vpmulld (%rdi), %xmm0, %xmm0 # sched: [16:2.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_pmulld:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    pmulld %xmm1, %xmm0 # sched: [10:2.00]
-; BROADWELL-SSE-NEXT:    pmulld (%rdi), %xmm0 # sched: [15:2.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pmulld:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpmulld %xmm1, %xmm0, %xmm0 # sched: [10:2.00]
-; BROADWELL-NEXT:    vpmulld (%rdi), %xmm0, %xmm0 # sched: [15:2.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_pmulld:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    pmulld %xmm1, %xmm0 # sched: [10:1.00]
-; SKYLAKE-SSE-NEXT:    pmulld (%rdi), %xmm0 # sched: [16:1.00]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pmulld:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpmulld %xmm1, %xmm0, %xmm0 # sched: [10:1.00]
-; SKYLAKE-NEXT:    vpmulld (%rdi), %xmm0, %xmm0 # sched: [16:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_pmulld:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    pmulld %xmm1, %xmm0 # sched: [10:1.00]
-; SKX-SSE-NEXT:    pmulld (%rdi), %xmm0 # sched: [16:1.00]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pmulld:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpmulld %xmm1, %xmm0, %xmm0 # sched: [10:1.00]
-; SKX-NEXT:    vpmulld (%rdi), %xmm0, %xmm0 # sched: [16:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_pmulld:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    pmulld %xmm1, %xmm0 # sched: [5:2.00]
-; BDVER2-SSE-NEXT:    pmulld (%rdi), %xmm0 # sched: [10:2.00]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_pmulld:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vpmulld %xmm1, %xmm0, %xmm0 # sched: [5:2.00]
-; BDVER2-NEXT:    vpmulld (%rdi), %xmm0, %xmm0 # sched: [10:2.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_pmulld:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    pmulld %xmm1, %xmm0 # sched: [4:2.00]
-; BTVER2-SSE-NEXT:    pmulld (%rdi), %xmm0 # sched: [9:2.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_pmulld:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vpmulld %xmm1, %xmm0, %xmm0 # sched: [4:2.00]
-; BTVER2-NEXT:    vpmulld (%rdi), %xmm0, %xmm0 # sched: [9:2.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_pmulld:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    pmulld %xmm1, %xmm0 # sched: [4:1.00]
-; ZNVER1-SSE-NEXT:    pmulld (%rdi), %xmm0 # sched: [11:1.00]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_pmulld:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpmulld %xmm1, %xmm0, %xmm0 # sched: [4:1.00]
-; ZNVER1-NEXT:    vpmulld (%rdi), %xmm0, %xmm0 # sched: [11:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = mul <4 x i32> %a0, %a1
-  %2 = load <4 x i32>, <4 x i32> *%a2, align 16
-  %3 = mul <4 x i32> %1, %2
-  ret <4 x i32> %3
-}
-
-define i32 @test_ptest(<2 x i64> %a0, <2 x i64> %a1, <2 x i64> *%a2) {
-; GENERIC-LABEL: test_ptest:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    ptest %xmm1, %xmm0 # sched: [2:1.00]
-; GENERIC-NEXT:    setb %al # sched: [1:0.50]
-; GENERIC-NEXT:    ptest (%rdi), %xmm0 # sched: [8:1.00]
-; GENERIC-NEXT:    setb %cl # sched: [1:0.50]
-; GENERIC-NEXT:    andb %al, %cl # sched: [1:0.33]
-; GENERIC-NEXT:    movzbl %cl, %eax # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SLM-LABEL: test_ptest:
-; SLM:       # %bb.0:
-; SLM-NEXT:    ptest %xmm1, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    setb %al # sched: [1:0.50]
-; SLM-NEXT:    ptest (%rdi), %xmm0 # sched: [4:1.00]
-; SLM-NEXT:    setb %cl # sched: [1:0.50]
-; SLM-NEXT:    andb %al, %cl # sched: [1:0.50]
-; SLM-NEXT:    movzbl %cl, %eax # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_ptest:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    ptest %xmm1, %xmm0 # sched: [2:1.00]
-; SANDY-SSE-NEXT:    setb %al # sched: [1:0.50]
-; SANDY-SSE-NEXT:    ptest (%rdi), %xmm0 # sched: [8:1.00]
-; SANDY-SSE-NEXT:    setb %cl # sched: [1:0.50]
-; SANDY-SSE-NEXT:    andb %al, %cl # sched: [1:0.33]
-; SANDY-SSE-NEXT:    movzbl %cl, %eax # sched: [1:0.33]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_ptest:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vptest %xmm1, %xmm0 # sched: [2:1.00]
-; SANDY-NEXT:    setb %al # sched: [1:0.50]
-; SANDY-NEXT:    vptest (%rdi), %xmm0 # sched: [8:1.00]
-; SANDY-NEXT:    setb %cl # sched: [1:0.50]
-; SANDY-NEXT:    andb %al, %cl # sched: [1:0.33]
-; SANDY-NEXT:    movzbl %cl, %eax # sched: [1:0.33]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_ptest:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    ptest %xmm1, %xmm0 # sched: [2:1.00]
-; HASWELL-SSE-NEXT:    setb %al # sched: [1:0.50]
-; HASWELL-SSE-NEXT:    ptest (%rdi), %xmm0 # sched: [8:1.00]
-; HASWELL-SSE-NEXT:    setb %cl # sched: [1:0.50]
-; HASWELL-SSE-NEXT:    andb %al, %cl # sched: [1:0.25]
-; HASWELL-SSE-NEXT:    movzbl %cl, %eax # sched: [1:0.25]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_ptest:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vptest %xmm1, %xmm0 # sched: [2:1.00]
-; HASWELL-NEXT:    setb %al # sched: [1:0.50]
-; HASWELL-NEXT:    vptest (%rdi), %xmm0 # sched: [8:1.00]
-; HASWELL-NEXT:    setb %cl # sched: [1:0.50]
-; HASWELL-NEXT:    andb %al, %cl # sched: [1:0.25]
-; HASWELL-NEXT:    movzbl %cl, %eax # sched: [1:0.25]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_ptest:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    ptest %xmm1, %xmm0 # sched: [2:1.00]
-; BROADWELL-SSE-NEXT:    setb %al # sched: [1:0.50]
-; BROADWELL-SSE-NEXT:    ptest (%rdi), %xmm0 # sched: [7:1.00]
-; BROADWELL-SSE-NEXT:    setb %cl # sched: [1:0.50]
-; BROADWELL-SSE-NEXT:    andb %al, %cl # sched: [1:0.25]
-; BROADWELL-SSE-NEXT:    movzbl %cl, %eax # sched: [1:0.25]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_ptest:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vptest %xmm1, %xmm0 # sched: [2:1.00]
-; BROADWELL-NEXT:    setb %al # sched: [1:0.50]
-; BROADWELL-NEXT:    vptest (%rdi), %xmm0 # sched: [7:1.00]
-; BROADWELL-NEXT:    setb %cl # sched: [1:0.50]
-; BROADWELL-NEXT:    andb %al, %cl # sched: [1:0.25]
-; BROADWELL-NEXT:    movzbl %cl, %eax # sched: [1:0.25]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_ptest:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    ptest %xmm1, %xmm0 # sched: [3:1.00]
-; SKYLAKE-SSE-NEXT:    setb %al # sched: [1:0.50]
-; SKYLAKE-SSE-NEXT:    ptest (%rdi), %xmm0 # sched: [9:1.00]
-; SKYLAKE-SSE-NEXT:    setb %cl # sched: [1:0.50]
-; SKYLAKE-SSE-NEXT:    andb %al, %cl # sched: [1:0.25]
-; SKYLAKE-SSE-NEXT:    movzbl %cl, %eax # sched: [1:0.25]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_ptest:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vptest %xmm1, %xmm0 # sched: [3:1.00]
-; SKYLAKE-NEXT:    setb %al # sched: [1:0.50]
-; SKYLAKE-NEXT:    vptest (%rdi), %xmm0 # sched: [9:1.00]
-; SKYLAKE-NEXT:    setb %cl # sched: [1:0.50]
-; SKYLAKE-NEXT:    andb %al, %cl # sched: [1:0.25]
-; SKYLAKE-NEXT:    movzbl %cl, %eax # sched: [1:0.25]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_ptest:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    ptest %xmm1, %xmm0 # sched: [3:1.00]
-; SKX-SSE-NEXT:    setb %al # sched: [1:0.50]
-; SKX-SSE-NEXT:    ptest (%rdi), %xmm0 # sched: [9:1.00]
-; SKX-SSE-NEXT:    setb %cl # sched: [1:0.50]
-; SKX-SSE-NEXT:    andb %al, %cl # sched: [1:0.25]
-; SKX-SSE-NEXT:    movzbl %cl, %eax # sched: [1:0.25]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_ptest:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vptest %xmm1, %xmm0 # sched: [3:1.00]
-; SKX-NEXT:    setb %al # sched: [1:0.50]
-; SKX-NEXT:    vptest (%rdi), %xmm0 # sched: [9:1.00]
-; SKX-NEXT:    setb %cl # sched: [1:0.50]
-; SKX-NEXT:    andb %al, %cl # sched: [1:0.25]
-; SKX-NEXT:    movzbl %cl, %eax # sched: [1:0.25]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_ptest:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    ptest %xmm1, %xmm0 # sched: [1:1.00]
-; BDVER2-SSE-NEXT:    setb %al # sched: [1:0.50]
-; BDVER2-SSE-NEXT:    ptest (%rdi), %xmm0 # sched: [6:1.00]
-; BDVER2-SSE-NEXT:    setb %cl # sched: [1:0.50]
-; BDVER2-SSE-NEXT:    andb %al, %cl # sched: [1:0.50]
-; BDVER2-SSE-NEXT:    movzbl %cl, %eax # sched: [1:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_ptest:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vptest %xmm1, %xmm0 # sched: [1:1.00]
-; BDVER2-NEXT:    setb %al # sched: [1:0.50]
-; BDVER2-NEXT:    vptest (%rdi), %xmm0 # sched: [6:1.00]
-; BDVER2-NEXT:    setb %cl # sched: [1:0.50]
-; BDVER2-NEXT:    andb %al, %cl # sched: [1:0.50]
-; BDVER2-NEXT:    movzbl %cl, %eax # sched: [1:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_ptest:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    ptest %xmm1, %xmm0 # sched: [3:1.00]
-; BTVER2-SSE-NEXT:    setb %al # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    ptest (%rdi), %xmm0 # sched: [8:1.00]
-; BTVER2-SSE-NEXT:    setb %cl # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    andb %al, %cl # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    movzbl %cl, %eax # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_ptest:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vptest %xmm1, %xmm0 # sched: [3:1.00]
-; BTVER2-NEXT:    setb %al # sched: [1:0.50]
-; BTVER2-NEXT:    vptest (%rdi), %xmm0 # sched: [8:1.00]
-; BTVER2-NEXT:    setb %cl # sched: [1:0.50]
-; BTVER2-NEXT:    andb %al, %cl # sched: [1:0.50]
-; BTVER2-NEXT:    movzbl %cl, %eax # sched: [1:0.50]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_ptest:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    ptest %xmm1, %xmm0 # sched: [1:1.00]
-; ZNVER1-SSE-NEXT:    setb %al # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    ptest (%rdi), %xmm0 # sched: [8:1.00]
-; ZNVER1-SSE-NEXT:    setb %cl # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    andb %al, %cl # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    movzbl %cl, %eax # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_ptest:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vptest %xmm1, %xmm0 # sched: [1:1.00]
-; ZNVER1-NEXT:    setb %al # sched: [1:0.25]
-; ZNVER1-NEXT:    vptest (%rdi), %xmm0 # sched: [8:1.00]
-; ZNVER1-NEXT:    setb %cl # sched: [1:0.25]
-; ZNVER1-NEXT:    andb %al, %cl # sched: [1:0.25]
-; ZNVER1-NEXT:    movzbl %cl, %eax # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call i32 @llvm.x86.sse41.ptestc(<2 x i64> %a0, <2 x i64> %a1)
-  %2 = load <2 x i64>, <2 x i64> *%a2, align 16
-  %3 = call i32 @llvm.x86.sse41.ptestc(<2 x i64> %a0, <2 x i64> %2)
-  %4 = and i32 %1, %3
-  ret i32 %4
-}
-declare i32 @llvm.x86.sse41.ptestc(<2 x i64>, <2 x i64>) nounwind readnone
-
-define <2 x double> @test_roundpd(<2 x double> %a0, <2 x double> *%a1) {
-; GENERIC-LABEL: test_roundpd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    roundpd $7, %xmm0, %xmm1 # sched: [3:1.00]
-; GENERIC-NEXT:    roundpd $7, (%rdi), %xmm0 # sched: [9:1.00]
-; GENERIC-NEXT:    addpd %xmm1, %xmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SLM-LABEL: test_roundpd:
-; SLM:       # %bb.0:
-; SLM-NEXT:    roundpd $7, (%rdi), %xmm1 # sched: [6:1.00]
-; SLM-NEXT:    roundpd $7, %xmm0, %xmm0 # sched: [3:1.00]
-; SLM-NEXT:    addpd %xmm0, %xmm1 # sched: [3:1.00]
-; SLM-NEXT:    movapd %xmm1, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_roundpd:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    roundpd $7, %xmm0, %xmm1 # sched: [3:1.00]
-; SANDY-SSE-NEXT:    roundpd $7, (%rdi), %xmm0 # sched: [9:1.00]
-; SANDY-SSE-NEXT:    addpd %xmm1, %xmm0 # sched: [3:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_roundpd:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vroundpd $7, %xmm0, %xmm0 # sched: [3:1.00]
-; SANDY-NEXT:    vroundpd $7, (%rdi), %xmm1 # sched: [9:1.00]
-; SANDY-NEXT:    vaddpd %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_roundpd:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    roundpd $7, %xmm0, %xmm1 # sched: [6:0.50]
-; HASWELL-SSE-NEXT:    roundpd $7, (%rdi), %xmm0 # sched: [12:2.00]
-; HASWELL-SSE-NEXT:    addpd %xmm1, %xmm0 # sched: [3:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_roundpd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vroundpd $7, %xmm0, %xmm0 # sched: [6:0.50]
-; HASWELL-NEXT:    vroundpd $7, (%rdi), %xmm1 # sched: [12:2.00]
-; HASWELL-NEXT:    vaddpd %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_roundpd:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    roundpd $7, (%rdi), %xmm1 # sched: [11:2.00]
-; BROADWELL-SSE-NEXT:    roundpd $7, %xmm0, %xmm0 # sched: [6:0.50]
-; BROADWELL-SSE-NEXT:    addpd %xmm0, %xmm1 # sched: [3:1.00]
-; BROADWELL-SSE-NEXT:    movapd %xmm1, %xmm0 # sched: [1:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_roundpd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vroundpd $7, (%rdi), %xmm1 # sched: [11:2.00]
-; BROADWELL-NEXT:    vroundpd $7, %xmm0, %xmm0 # sched: [6:0.50]
-; BROADWELL-NEXT:    vaddpd %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_roundpd:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    roundpd $7, %xmm0, %xmm1 # sched: [8:1.00]
-; SKYLAKE-SSE-NEXT:    roundpd $7, (%rdi), %xmm0 # sched: [14:1.00]
-; SKYLAKE-SSE-NEXT:    addpd %xmm1, %xmm0 # sched: [4:0.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_roundpd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vroundpd $7, %xmm0, %xmm0 # sched: [8:1.00]
-; SKYLAKE-NEXT:    vroundpd $7, (%rdi), %xmm1 # sched: [14:1.00]
-; SKYLAKE-NEXT:    vaddpd %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_roundpd:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    roundpd $7, %xmm0, %xmm1 # sched: [8:1.00]
-; SKX-SSE-NEXT:    roundpd $7, (%rdi), %xmm0 # sched: [14:1.00]
-; SKX-SSE-NEXT:    addpd %xmm1, %xmm0 # sched: [4:0.50]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_roundpd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vroundpd $7, %xmm0, %xmm0 # sched: [8:1.00]
-; SKX-NEXT:    vroundpd $7, (%rdi), %xmm1 # sched: [14:1.00]
-; SKX-NEXT:    vaddpd %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_roundpd:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    roundpd $7, %xmm0, %xmm1 # sched: [4:1.00]
-; BDVER2-SSE-NEXT:    roundpd $7, (%rdi), %xmm0 # sched: [9:1.00]
-; BDVER2-SSE-NEXT:    addpd %xmm1, %xmm0 # sched: [5:1.00]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_roundpd:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vroundpd $7, (%rdi), %xmm1 # sched: [9:1.00]
-; BDVER2-NEXT:    vroundpd $7, %xmm0, %xmm0 # sched: [4:1.00]
-; BDVER2-NEXT:    vaddpd %xmm1, %xmm0, %xmm0 # sched: [5:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_roundpd:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    roundpd $7, %xmm0, %xmm1 # sched: [3:1.00]
-; BTVER2-SSE-NEXT:    roundpd $7, (%rdi), %xmm0 # sched: [8:1.00]
-; BTVER2-SSE-NEXT:    addpd %xmm1, %xmm0 # sched: [3:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_roundpd:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vroundpd $7, (%rdi), %xmm1 # sched: [8:1.00]
-; BTVER2-NEXT:    vroundpd $7, %xmm0, %xmm0 # sched: [3:1.00]
-; BTVER2-NEXT:    vaddpd %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_roundpd:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    roundpd $7, %xmm0, %xmm1 # sched: [4:1.00]
-; ZNVER1-SSE-NEXT:    roundpd $7, (%rdi), %xmm0 # sched: [11:1.00]
-; ZNVER1-SSE-NEXT:    addpd %xmm1, %xmm0 # sched: [3:1.00]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_roundpd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vroundpd $7, (%rdi), %xmm1 # sched: [11:1.00]
-; ZNVER1-NEXT:    vroundpd $7, %xmm0, %xmm0 # sched: [4:1.00]
-; ZNVER1-NEXT:    vaddpd %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <2 x double> @llvm.x86.sse41.round.pd(<2 x double> %a0, i32 7)
-  %2 = load <2 x double>, <2 x double> *%a1, align 16
-  %3 = call <2 x double> @llvm.x86.sse41.round.pd(<2 x double> %2, i32 7)
-  %4 = fadd <2 x double> %1, %3
-  ret <2 x double> %4
-}
-declare <2 x double> @llvm.x86.sse41.round.pd(<2 x double>, i32) nounwind readnone
-
-define <4 x float> @test_roundps(<4 x float> %a0, <4 x float> *%a1) {
-; GENERIC-LABEL: test_roundps:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    roundps $7, %xmm0, %xmm1 # sched: [3:1.00]
-; GENERIC-NEXT:    roundps $7, (%rdi), %xmm0 # sched: [9:1.00]
-; GENERIC-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SLM-LABEL: test_roundps:
-; SLM:       # %bb.0:
-; SLM-NEXT:    roundps $7, (%rdi), %xmm1 # sched: [6:1.00]
-; SLM-NEXT:    roundps $7, %xmm0, %xmm0 # sched: [3:1.00]
-; SLM-NEXT:    addps %xmm0, %xmm1 # sched: [3:1.00]
-; SLM-NEXT:    movaps %xmm1, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_roundps:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    roundps $7, %xmm0, %xmm1 # sched: [3:1.00]
-; SANDY-SSE-NEXT:    roundps $7, (%rdi), %xmm0 # sched: [9:1.00]
-; SANDY-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_roundps:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vroundps $7, %xmm0, %xmm0 # sched: [3:1.00]
-; SANDY-NEXT:    vroundps $7, (%rdi), %xmm1 # sched: [9:1.00]
-; SANDY-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_roundps:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    roundps $7, %xmm0, %xmm1 # sched: [6:0.50]
-; HASWELL-SSE-NEXT:    roundps $7, (%rdi), %xmm0 # sched: [12:2.00]
-; HASWELL-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_roundps:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vroundps $7, %xmm0, %xmm0 # sched: [6:0.50]
-; HASWELL-NEXT:    vroundps $7, (%rdi), %xmm1 # sched: [12:2.00]
-; HASWELL-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_roundps:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    roundps $7, (%rdi), %xmm1 # sched: [11:2.00]
-; BROADWELL-SSE-NEXT:    roundps $7, %xmm0, %xmm0 # sched: [6:0.50]
-; BROADWELL-SSE-NEXT:    addps %xmm0, %xmm1 # sched: [3:1.00]
-; BROADWELL-SSE-NEXT:    movaps %xmm1, %xmm0 # sched: [1:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_roundps:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vroundps $7, (%rdi), %xmm1 # sched: [11:2.00]
-; BROADWELL-NEXT:    vroundps $7, %xmm0, %xmm0 # sched: [6:0.50]
-; BROADWELL-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_roundps:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    roundps $7, %xmm0, %xmm1 # sched: [8:1.00]
-; SKYLAKE-SSE-NEXT:    roundps $7, (%rdi), %xmm0 # sched: [14:1.00]
-; SKYLAKE-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [4:0.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_roundps:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vroundps $7, %xmm0, %xmm0 # sched: [8:1.00]
-; SKYLAKE-NEXT:    vroundps $7, (%rdi), %xmm1 # sched: [14:1.00]
-; SKYLAKE-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_roundps:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    roundps $7, %xmm0, %xmm1 # sched: [8:1.00]
-; SKX-SSE-NEXT:    roundps $7, (%rdi), %xmm0 # sched: [14:1.00]
-; SKX-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [4:0.50]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_roundps:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vroundps $7, %xmm0, %xmm0 # sched: [8:1.00]
-; SKX-NEXT:    vroundps $7, (%rdi), %xmm1 # sched: [14:1.00]
-; SKX-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_roundps:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    roundps $7, %xmm0, %xmm1 # sched: [4:1.00]
-; BDVER2-SSE-NEXT:    roundps $7, (%rdi), %xmm0 # sched: [9:1.00]
-; BDVER2-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [5:1.00]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_roundps:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vroundps $7, (%rdi), %xmm1 # sched: [9:1.00]
-; BDVER2-NEXT:    vroundps $7, %xmm0, %xmm0 # sched: [4:1.00]
-; BDVER2-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [5:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_roundps:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    roundps $7, %xmm0, %xmm1 # sched: [3:1.00]
-; BTVER2-SSE-NEXT:    roundps $7, (%rdi), %xmm0 # sched: [8:1.00]
-; BTVER2-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_roundps:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vroundps $7, (%rdi), %xmm1 # sched: [8:1.00]
-; BTVER2-NEXT:    vroundps $7, %xmm0, %xmm0 # sched: [3:1.00]
-; BTVER2-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_roundps:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    roundps $7, %xmm0, %xmm1 # sched: [4:1.00]
-; ZNVER1-SSE-NEXT:    roundps $7, (%rdi), %xmm0 # sched: [11:1.00]
-; ZNVER1-SSE-NEXT:    addps %xmm1, %xmm0 # sched: [3:1.00]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_roundps:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vroundps $7, (%rdi), %xmm1 # sched: [11:1.00]
-; ZNVER1-NEXT:    vroundps $7, %xmm0, %xmm0 # sched: [4:1.00]
-; ZNVER1-NEXT:    vaddps %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <4 x float> @llvm.x86.sse41.round.ps(<4 x float> %a0, i32 7)
-  %2 = load <4 x float>, <4 x float> *%a1, align 16
-  %3 = call <4 x float> @llvm.x86.sse41.round.ps(<4 x float> %2, i32 7)
-  %4 = fadd <4 x float> %1, %3
-  ret <4 x float> %4
-}
-declare <4 x float> @llvm.x86.sse41.round.ps(<4 x float>, i32) nounwind readnone
-
-define <2 x double> @test_roundsd(<2 x double> %a0, <2 x double> %a1, <2 x double> *%a2) {
-; GENERIC-LABEL: test_roundsd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    movapd %xmm0, %xmm2 # sched: [1:1.00]
-; GENERIC-NEXT:    roundsd $7, %xmm1, %xmm2 # sched: [3:1.00]
-; GENERIC-NEXT:    roundsd $7, (%rdi), %xmm0 # sched: [9:1.00]
-; GENERIC-NEXT:    addpd %xmm2, %xmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SLM-LABEL: test_roundsd:
-; SLM:       # %bb.0:
-; SLM-NEXT:    movapd %xmm0, %xmm2 # sched: [1:0.50]
-; SLM-NEXT:    roundsd $7, (%rdi), %xmm0 # sched: [6:1.00]
-; SLM-NEXT:    roundsd $7, %xmm1, %xmm2 # sched: [3:1.00]
-; SLM-NEXT:    addpd %xmm2, %xmm0 # sched: [3:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_roundsd:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    movapd %xmm0, %xmm2 # sched: [1:1.00]
-; SANDY-SSE-NEXT:    roundsd $7, %xmm1, %xmm2 # sched: [3:1.00]
-; SANDY-SSE-NEXT:    roundsd $7, (%rdi), %xmm0 # sched: [9:1.00]
-; SANDY-SSE-NEXT:    addpd %xmm2, %xmm0 # sched: [3:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_roundsd:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vroundsd $7, %xmm1, %xmm0, %xmm1 # sched: [3:1.00]
-; SANDY-NEXT:    vroundsd $7, (%rdi), %xmm0, %xmm0 # sched: [9:1.00]
-; SANDY-NEXT:    vaddpd %xmm0, %xmm1, %xmm0 # sched: [3:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_roundsd:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    movapd %xmm0, %xmm2 # sched: [1:1.00]
-; HASWELL-SSE-NEXT:    roundsd $7, %xmm1, %xmm2 # sched: [6:0.50]
-; HASWELL-SSE-NEXT:    roundsd $7, (%rdi), %xmm0 # sched: [12:2.00]
-; HASWELL-SSE-NEXT:    addpd %xmm2, %xmm0 # sched: [3:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_roundsd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vroundsd $7, %xmm1, %xmm0, %xmm1 # sched: [6:0.50]
-; HASWELL-NEXT:    vroundsd $7, (%rdi), %xmm0, %xmm0 # sched: [12:2.00]
-; HASWELL-NEXT:    vaddpd %xmm0, %xmm1, %xmm0 # sched: [3:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_roundsd:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    movapd %xmm0, %xmm2 # sched: [1:1.00]
-; BROADWELL-SSE-NEXT:    roundsd $7, (%rdi), %xmm0 # sched: [11:2.00]
-; BROADWELL-SSE-NEXT:    roundsd $7, %xmm1, %xmm2 # sched: [6:0.50]
-; BROADWELL-SSE-NEXT:    addpd %xmm2, %xmm0 # sched: [3:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_roundsd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vroundsd $7, (%rdi), %xmm0, %xmm2 # sched: [11:2.00]
-; BROADWELL-NEXT:    vroundsd $7, %xmm1, %xmm0, %xmm0 # sched: [6:0.50]
-; BROADWELL-NEXT:    vaddpd %xmm2, %xmm0, %xmm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_roundsd:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    movapd %xmm0, %xmm2 # sched: [1:0.33]
-; SKYLAKE-SSE-NEXT:    roundsd $7, %xmm1, %xmm2 # sched: [8:1.00]
-; SKYLAKE-SSE-NEXT:    roundsd $7, (%rdi), %xmm0 # sched: [14:1.00]
-; SKYLAKE-SSE-NEXT:    addpd %xmm2, %xmm0 # sched: [4:0.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_roundsd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vroundsd $7, %xmm1, %xmm0, %xmm1 # sched: [8:1.00]
-; SKYLAKE-NEXT:    vroundsd $7, (%rdi), %xmm0, %xmm0 # sched: [14:1.00]
-; SKYLAKE-NEXT:    vaddpd %xmm0, %xmm1, %xmm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_roundsd:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    movapd %xmm0, %xmm2 # sched: [1:0.33]
-; SKX-SSE-NEXT:    roundsd $7, %xmm1, %xmm2 # sched: [8:1.00]
-; SKX-SSE-NEXT:    roundsd $7, (%rdi), %xmm0 # sched: [14:1.00]
-; SKX-SSE-NEXT:    addpd %xmm2, %xmm0 # sched: [4:0.50]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_roundsd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vroundsd $7, %xmm1, %xmm0, %xmm1 # sched: [8:1.00]
-; SKX-NEXT:    vroundsd $7, (%rdi), %xmm0, %xmm0 # sched: [14:1.00]
-; SKX-NEXT:    vaddpd %xmm0, %xmm1, %xmm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_roundsd:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    movapd %xmm0, %xmm2 # sched: [1:0.50]
-; BDVER2-SSE-NEXT:    roundsd $7, (%rdi), %xmm0 # sched: [9:1.00]
-; BDVER2-SSE-NEXT:    roundsd $7, %xmm1, %xmm2 # sched: [4:1.00]
-; BDVER2-SSE-NEXT:    addpd %xmm2, %xmm0 # sched: [5:1.00]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_roundsd:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vroundsd $7, %xmm1, %xmm0, %xmm1 # sched: [4:1.00]
-; BDVER2-NEXT:    vroundsd $7, (%rdi), %xmm0, %xmm0 # sched: [9:1.00]
-; BDVER2-NEXT:    vaddpd %xmm0, %xmm1, %xmm0 # sched: [5:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_roundsd:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    movapd %xmm0, %xmm2 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    roundsd $7, (%rdi), %xmm0 # sched: [8:1.00]
-; BTVER2-SSE-NEXT:    roundsd $7, %xmm1, %xmm2 # sched: [3:1.00]
-; BTVER2-SSE-NEXT:    addpd %xmm2, %xmm0 # sched: [3:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_roundsd:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vroundsd $7, %xmm1, %xmm0, %xmm1 # sched: [3:1.00]
-; BTVER2-NEXT:    vroundsd $7, (%rdi), %xmm0, %xmm0 # sched: [8:1.00]
-; BTVER2-NEXT:    vaddpd %xmm0, %xmm1, %xmm0 # sched: [3:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_roundsd:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    movapd %xmm0, %xmm2 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    roundsd $7, (%rdi), %xmm0 # sched: [11:1.00]
-; ZNVER1-SSE-NEXT:    roundsd $7, %xmm1, %xmm2 # sched: [4:1.00]
-; ZNVER1-SSE-NEXT:    addpd %xmm2, %xmm0 # sched: [3:1.00]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_roundsd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vroundsd $7, %xmm1, %xmm0, %xmm1 # sched: [4:1.00]
-; ZNVER1-NEXT:    vroundsd $7, (%rdi), %xmm0, %xmm0 # sched: [11:1.00]
-; ZNVER1-NEXT:    vaddpd %xmm0, %xmm1, %xmm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <2 x double> @llvm.x86.sse41.round.sd(<2 x double> %a0, <2 x double> %a1, i32 7)
-  %2 = load <2 x double>, <2 x double>* %a2, align 16
-  %3 = call <2 x double> @llvm.x86.sse41.round.sd(<2 x double> %a0, <2 x double> %2, i32 7)
-  %4 = fadd <2 x double> %1, %3
-  ret <2 x double> %4
-}
-declare <2 x double> @llvm.x86.sse41.round.sd(<2 x double>, <2 x double>, i32) nounwind readnone
-
-define <4 x float> @test_roundss(<4 x float> %a0, <4 x float> %a1, <4 x float> *%a2) {
-; GENERIC-LABEL: test_roundss:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    movaps %xmm0, %xmm2 # sched: [1:1.00]
-; GENERIC-NEXT:    roundss $7, %xmm1, %xmm2 # sched: [3:1.00]
-; GENERIC-NEXT:    roundss $7, (%rdi), %xmm0 # sched: [9:1.00]
-; GENERIC-NEXT:    addps %xmm2, %xmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SLM-LABEL: test_roundss:
-; SLM:       # %bb.0:
-; SLM-NEXT:    movaps %xmm0, %xmm2 # sched: [1:0.50]
-; SLM-NEXT:    roundss $7, (%rdi), %xmm0 # sched: [6:1.00]
-; SLM-NEXT:    roundss $7, %xmm1, %xmm2 # sched: [3:1.00]
-; SLM-NEXT:    addps %xmm2, %xmm0 # sched: [3:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_roundss:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    movaps %xmm0, %xmm2 # sched: [1:1.00]
-; SANDY-SSE-NEXT:    roundss $7, %xmm1, %xmm2 # sched: [3:1.00]
-; SANDY-SSE-NEXT:    roundss $7, (%rdi), %xmm0 # sched: [9:1.00]
-; SANDY-SSE-NEXT:    addps %xmm2, %xmm0 # sched: [3:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_roundss:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vroundss $7, %xmm1, %xmm0, %xmm1 # sched: [3:1.00]
-; SANDY-NEXT:    vroundss $7, (%rdi), %xmm0, %xmm0 # sched: [9:1.00]
-; SANDY-NEXT:    vaddps %xmm0, %xmm1, %xmm0 # sched: [3:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_roundss:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    movaps %xmm0, %xmm2 # sched: [1:1.00]
-; HASWELL-SSE-NEXT:    roundss $7, %xmm1, %xmm2 # sched: [6:0.50]
-; HASWELL-SSE-NEXT:    roundss $7, (%rdi), %xmm0 # sched: [12:2.00]
-; HASWELL-SSE-NEXT:    addps %xmm2, %xmm0 # sched: [3:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_roundss:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vroundss $7, %xmm1, %xmm0, %xmm1 # sched: [6:0.50]
-; HASWELL-NEXT:    vroundss $7, (%rdi), %xmm0, %xmm0 # sched: [12:2.00]
-; HASWELL-NEXT:    vaddps %xmm0, %xmm1, %xmm0 # sched: [3:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_roundss:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    movaps %xmm0, %xmm2 # sched: [1:1.00]
-; BROADWELL-SSE-NEXT:    roundss $7, (%rdi), %xmm0 # sched: [11:2.00]
-; BROADWELL-SSE-NEXT:    roundss $7, %xmm1, %xmm2 # sched: [6:0.50]
-; BROADWELL-SSE-NEXT:    addps %xmm2, %xmm0 # sched: [3:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_roundss:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vroundss $7, (%rdi), %xmm0, %xmm2 # sched: [11:2.00]
-; BROADWELL-NEXT:    vroundss $7, %xmm1, %xmm0, %xmm0 # sched: [6:0.50]
-; BROADWELL-NEXT:    vaddps %xmm2, %xmm0, %xmm0 # sched: [3:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_roundss:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    movaps %xmm0, %xmm2 # sched: [1:0.33]
-; SKYLAKE-SSE-NEXT:    roundss $7, %xmm1, %xmm2 # sched: [8:1.00]
-; SKYLAKE-SSE-NEXT:    roundss $7, (%rdi), %xmm0 # sched: [14:1.00]
-; SKYLAKE-SSE-NEXT:    addps %xmm2, %xmm0 # sched: [4:0.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_roundss:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vroundss $7, %xmm1, %xmm0, %xmm1 # sched: [8:1.00]
-; SKYLAKE-NEXT:    vroundss $7, (%rdi), %xmm0, %xmm0 # sched: [14:1.00]
-; SKYLAKE-NEXT:    vaddps %xmm0, %xmm1, %xmm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_roundss:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    movaps %xmm0, %xmm2 # sched: [1:0.33]
-; SKX-SSE-NEXT:    roundss $7, %xmm1, %xmm2 # sched: [8:1.00]
-; SKX-SSE-NEXT:    roundss $7, (%rdi), %xmm0 # sched: [14:1.00]
-; SKX-SSE-NEXT:    addps %xmm2, %xmm0 # sched: [4:0.50]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_roundss:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vroundss $7, %xmm1, %xmm0, %xmm1 # sched: [8:1.00]
-; SKX-NEXT:    vroundss $7, (%rdi), %xmm0, %xmm0 # sched: [14:1.00]
-; SKX-NEXT:    vaddps %xmm0, %xmm1, %xmm0 # sched: [4:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_roundss:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    movaps %xmm0, %xmm2 # sched: [1:0.50]
-; BDVER2-SSE-NEXT:    roundss $7, (%rdi), %xmm0 # sched: [9:1.00]
-; BDVER2-SSE-NEXT:    roundss $7, %xmm1, %xmm2 # sched: [4:1.00]
-; BDVER2-SSE-NEXT:    addps %xmm2, %xmm0 # sched: [5:1.00]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_roundss:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vroundss $7, %xmm1, %xmm0, %xmm1 # sched: [4:1.00]
-; BDVER2-NEXT:    vroundss $7, (%rdi), %xmm0, %xmm0 # sched: [9:1.00]
-; BDVER2-NEXT:    vaddps %xmm0, %xmm1, %xmm0 # sched: [5:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_roundss:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    movaps %xmm0, %xmm2 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    roundss $7, (%rdi), %xmm0 # sched: [8:1.00]
-; BTVER2-SSE-NEXT:    roundss $7, %xmm1, %xmm2 # sched: [3:1.00]
-; BTVER2-SSE-NEXT:    addps %xmm2, %xmm0 # sched: [3:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_roundss:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vroundss $7, %xmm1, %xmm0, %xmm1 # sched: [3:1.00]
-; BTVER2-NEXT:    vroundss $7, (%rdi), %xmm0, %xmm0 # sched: [8:1.00]
-; BTVER2-NEXT:    vaddps %xmm0, %xmm1, %xmm0 # sched: [3:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_roundss:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    movaps %xmm0, %xmm2 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    roundss $7, (%rdi), %xmm0 # sched: [11:1.00]
-; ZNVER1-SSE-NEXT:    roundss $7, %xmm1, %xmm2 # sched: [4:1.00]
-; ZNVER1-SSE-NEXT:    addps %xmm2, %xmm0 # sched: [3:1.00]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_roundss:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vroundss $7, %xmm1, %xmm0, %xmm1 # sched: [4:1.00]
-; ZNVER1-NEXT:    vroundss $7, (%rdi), %xmm0, %xmm0 # sched: [11:1.00]
-; ZNVER1-NEXT:    vaddps %xmm0, %xmm1, %xmm0 # sched: [3:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <4 x float> @llvm.x86.sse41.round.ss(<4 x float> %a0, <4 x float> %a1, i32 7)
-  %2 = load <4 x float>, <4 x float> *%a2, align 16
-  %3 = call <4 x float> @llvm.x86.sse41.round.ss(<4 x float> %a0, <4 x float> %2, i32 7)
-  %4 = fadd <4 x float> %1, %3
-  ret <4 x float> %4
-}
-declare <4 x float> @llvm.x86.sse41.round.ss(<4 x float>, <4 x float>, i32) nounwind readnone
diff --git a/test/CodeGen/X86/sse42-schedule.ll b/test/CodeGen/X86/sse42-schedule.ll
deleted file mode 100644
index 97dffb4db0943460d665ae15cb1d68a944469ffd..0000000000000000000000000000000000000000
--- a/test/CodeGen/X86/sse42-schedule.ll
+++ /dev/null
@@ -1,1631 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=x86-64 -mattr=+sse4.2,+pclmul | FileCheck %s --check-prefixes=CHECK,GENERIC
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=slm -mattr=-avx | FileCheck %s --check-prefixes=CHECK,SLM
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=sandybridge -mattr=-avx  | FileCheck %s --check-prefixes=CHECK,SANDY-SSE
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=sandybridge -mattr=-avx2 | FileCheck %s --check-prefixes=CHECK,SANDY
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=ivybridge -mattr=-avx  | FileCheck %s --check-prefixes=CHECK,SANDY-SSE
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=ivybridge -mattr=-avx2 | FileCheck %s --check-prefixes=CHECK,SANDY
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=haswell -mattr=-avx  | FileCheck %s --check-prefixes=CHECK,HASWELL-SSE
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=haswell -mattr=-avx2 | FileCheck %s --check-prefixes=CHECK,HASWELL
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=broadwell -mattr=-avx  | FileCheck %s --check-prefixes=CHECK,BROADWELL-SSE
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=broadwell -mattr=-avx2 | FileCheck %s --check-prefixes=CHECK,BROADWELL
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=skylake -mattr=-avx  | FileCheck %s --check-prefixes=CHECK,SKYLAKE-SSE
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=skylake -mattr=-avx2 | FileCheck %s --check-prefixes=CHECK,SKYLAKE
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=skx -mattr=-avx  | FileCheck %s --check-prefixes=CHECK,SKX-SSE
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=skx -mattr=-avx2 | FileCheck %s --check-prefixes=CHECK,SKX
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=bdver2 -mattr=-avx  | FileCheck %s --check-prefixes=CHECK,BDVER2-SSE
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=bdver2 -mattr=-avx2,-xop | FileCheck %s --check-prefixes=CHECK,BDVER2
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=btver2 -mattr=-avx  | FileCheck %s --check-prefixes=CHECK,BTVER2-SSE
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=btver2 -mattr=-avx2 | FileCheck %s --check-prefixes=CHECK,BTVER2
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=znver1 -mattr=-avx  | FileCheck %s --check-prefixes=CHECK,ZNVER1-SSE
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=znver1 -mattr=-avx2 | FileCheck %s --check-prefixes=CHECK,ZNVER1
-
-define i32 @crc32_32_8(i32 %a0, i8 %a1, i8 *%a2) {
-; GENERIC-LABEL: crc32_32_8:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    movl %edi, %eax # sched: [1:0.33]
-; GENERIC-NEXT:    crc32b %sil, %eax # sched: [3:1.00]
-; GENERIC-NEXT:    crc32b (%rdx), %eax # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SLM-LABEL: crc32_32_8:
-; SLM:       # %bb.0:
-; SLM-NEXT:    movl %edi, %eax # sched: [1:0.50]
-; SLM-NEXT:    crc32b %sil, %eax # sched: [3:1.00]
-; SLM-NEXT:    crc32b (%rdx), %eax # sched: [6:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: crc32_32_8:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    movl %edi, %eax # sched: [1:0.33]
-; SANDY-SSE-NEXT:    crc32b %sil, %eax # sched: [3:1.00]
-; SANDY-SSE-NEXT:    crc32b (%rdx), %eax # sched: [8:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: crc32_32_8:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    movl %edi, %eax # sched: [1:0.33]
-; SANDY-NEXT:    crc32b %sil, %eax # sched: [3:1.00]
-; SANDY-NEXT:    crc32b (%rdx), %eax # sched: [8:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: crc32_32_8:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    movl %edi, %eax # sched: [1:0.25]
-; HASWELL-SSE-NEXT:    crc32b %sil, %eax # sched: [3:1.00]
-; HASWELL-SSE-NEXT:    crc32b (%rdx), %eax # sched: [8:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: crc32_32_8:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    movl %edi, %eax # sched: [1:0.25]
-; HASWELL-NEXT:    crc32b %sil, %eax # sched: [3:1.00]
-; HASWELL-NEXT:    crc32b (%rdx), %eax # sched: [8:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: crc32_32_8:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    movl %edi, %eax # sched: [1:0.25]
-; BROADWELL-SSE-NEXT:    crc32b %sil, %eax # sched: [3:1.00]
-; BROADWELL-SSE-NEXT:    crc32b (%rdx), %eax # sched: [8:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: crc32_32_8:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    movl %edi, %eax # sched: [1:0.25]
-; BROADWELL-NEXT:    crc32b %sil, %eax # sched: [3:1.00]
-; BROADWELL-NEXT:    crc32b (%rdx), %eax # sched: [8:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: crc32_32_8:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    movl %edi, %eax # sched: [1:0.25]
-; SKYLAKE-SSE-NEXT:    crc32b %sil, %eax # sched: [3:1.00]
-; SKYLAKE-SSE-NEXT:    crc32b (%rdx), %eax # sched: [8:1.00]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: crc32_32_8:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    movl %edi, %eax # sched: [1:0.25]
-; SKYLAKE-NEXT:    crc32b %sil, %eax # sched: [3:1.00]
-; SKYLAKE-NEXT:    crc32b (%rdx), %eax # sched: [8:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: crc32_32_8:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    movl %edi, %eax # sched: [1:0.25]
-; SKX-SSE-NEXT:    crc32b %sil, %eax # sched: [3:1.00]
-; SKX-SSE-NEXT:    crc32b (%rdx), %eax # sched: [8:1.00]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: crc32_32_8:
-; SKX:       # %bb.0:
-; SKX-NEXT:    movl %edi, %eax # sched: [1:0.25]
-; SKX-NEXT:    crc32b %sil, %eax # sched: [3:1.00]
-; SKX-NEXT:    crc32b (%rdx), %eax # sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: crc32_32_8:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    movl %edi, %eax # sched: [1:0.50]
-; BDVER2-SSE-NEXT:    crc32b %sil, %eax # sched: [3:2.00]
-; BDVER2-SSE-NEXT:    crc32b (%rdx), %eax # sched: [7:2.00]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: crc32_32_8:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    movl %edi, %eax # sched: [1:0.50]
-; BDVER2-NEXT:    crc32b %sil, %eax # sched: [3:2.00]
-; BDVER2-NEXT:    crc32b (%rdx), %eax # sched: [7:2.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: crc32_32_8:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    movl %edi, %eax # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    crc32b %sil, %eax # sched: [3:2.00]
-; BTVER2-SSE-NEXT:    crc32b (%rdx), %eax # sched: [6:2.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: crc32_32_8:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    movl %edi, %eax # sched: [1:0.50]
-; BTVER2-NEXT:    crc32b %sil, %eax # sched: [3:2.00]
-; BTVER2-NEXT:    crc32b (%rdx), %eax # sched: [6:2.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: crc32_32_8:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    movl %edi, %eax # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    crc32b %sil, %eax # sched: [3:1.00]
-; ZNVER1-SSE-NEXT:    crc32b (%rdx), %eax # sched: [10:1.00]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: crc32_32_8:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    movl %edi, %eax # sched: [1:0.25]
-; ZNVER1-NEXT:    crc32b %sil, %eax # sched: [3:1.00]
-; ZNVER1-NEXT:    crc32b (%rdx), %eax # sched: [10:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call i32 @llvm.x86.sse42.crc32.32.8(i32 %a0, i8 %a1)
-  %2 = load i8, i8 *%a2
-  %3 = call i32 @llvm.x86.sse42.crc32.32.8(i32 %1, i8 %2)
-  ret i32 %3
-}
-declare i32 @llvm.x86.sse42.crc32.32.8(i32, i8) nounwind
-
-define i32 @crc32_32_16(i32 %a0, i16 %a1, i16 *%a2) {
-; GENERIC-LABEL: crc32_32_16:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    movl %edi, %eax # sched: [1:0.33]
-; GENERIC-NEXT:    crc32w %si, %eax # sched: [3:1.00]
-; GENERIC-NEXT:    crc32w (%rdx), %eax # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SLM-LABEL: crc32_32_16:
-; SLM:       # %bb.0:
-; SLM-NEXT:    movl %edi, %eax # sched: [1:0.50]
-; SLM-NEXT:    crc32w %si, %eax # sched: [3:1.00]
-; SLM-NEXT:    crc32w (%rdx), %eax # sched: [6:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: crc32_32_16:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    movl %edi, %eax # sched: [1:0.33]
-; SANDY-SSE-NEXT:    crc32w %si, %eax # sched: [3:1.00]
-; SANDY-SSE-NEXT:    crc32w (%rdx), %eax # sched: [8:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: crc32_32_16:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    movl %edi, %eax # sched: [1:0.33]
-; SANDY-NEXT:    crc32w %si, %eax # sched: [3:1.00]
-; SANDY-NEXT:    crc32w (%rdx), %eax # sched: [8:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: crc32_32_16:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    movl %edi, %eax # sched: [1:0.25]
-; HASWELL-SSE-NEXT:    crc32w %si, %eax # sched: [3:1.00]
-; HASWELL-SSE-NEXT:    crc32w (%rdx), %eax # sched: [8:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: crc32_32_16:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    movl %edi, %eax # sched: [1:0.25]
-; HASWELL-NEXT:    crc32w %si, %eax # sched: [3:1.00]
-; HASWELL-NEXT:    crc32w (%rdx), %eax # sched: [8:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: crc32_32_16:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    movl %edi, %eax # sched: [1:0.25]
-; BROADWELL-SSE-NEXT:    crc32w %si, %eax # sched: [3:1.00]
-; BROADWELL-SSE-NEXT:    crc32w (%rdx), %eax # sched: [8:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: crc32_32_16:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    movl %edi, %eax # sched: [1:0.25]
-; BROADWELL-NEXT:    crc32w %si, %eax # sched: [3:1.00]
-; BROADWELL-NEXT:    crc32w (%rdx), %eax # sched: [8:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: crc32_32_16:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    movl %edi, %eax # sched: [1:0.25]
-; SKYLAKE-SSE-NEXT:    crc32w %si, %eax # sched: [3:1.00]
-; SKYLAKE-SSE-NEXT:    crc32w (%rdx), %eax # sched: [8:1.00]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: crc32_32_16:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    movl %edi, %eax # sched: [1:0.25]
-; SKYLAKE-NEXT:    crc32w %si, %eax # sched: [3:1.00]
-; SKYLAKE-NEXT:    crc32w (%rdx), %eax # sched: [8:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: crc32_32_16:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    movl %edi, %eax # sched: [1:0.25]
-; SKX-SSE-NEXT:    crc32w %si, %eax # sched: [3:1.00]
-; SKX-SSE-NEXT:    crc32w (%rdx), %eax # sched: [8:1.00]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: crc32_32_16:
-; SKX:       # %bb.0:
-; SKX-NEXT:    movl %edi, %eax # sched: [1:0.25]
-; SKX-NEXT:    crc32w %si, %eax # sched: [3:1.00]
-; SKX-NEXT:    crc32w (%rdx), %eax # sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: crc32_32_16:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    movl %edi, %eax # sched: [1:0.50]
-; BDVER2-SSE-NEXT:    crc32w %si, %eax # sched: [5:2.00]
-; BDVER2-SSE-NEXT:    crc32w (%rdx), %eax # sched: [7:2.00]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: crc32_32_16:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    movl %edi, %eax # sched: [1:0.50]
-; BDVER2-NEXT:    crc32w %si, %eax # sched: [5:2.00]
-; BDVER2-NEXT:    crc32w (%rdx), %eax # sched: [7:2.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: crc32_32_16:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    movl %edi, %eax # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    crc32w %si, %eax # sched: [3:2.00]
-; BTVER2-SSE-NEXT:    crc32w (%rdx), %eax # sched: [6:2.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: crc32_32_16:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    movl %edi, %eax # sched: [1:0.50]
-; BTVER2-NEXT:    crc32w %si, %eax # sched: [3:2.00]
-; BTVER2-NEXT:    crc32w (%rdx), %eax # sched: [6:2.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: crc32_32_16:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    movl %edi, %eax # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    crc32w %si, %eax # sched: [3:1.00]
-; ZNVER1-SSE-NEXT:    crc32w (%rdx), %eax # sched: [10:1.00]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: crc32_32_16:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    movl %edi, %eax # sched: [1:0.25]
-; ZNVER1-NEXT:    crc32w %si, %eax # sched: [3:1.00]
-; ZNVER1-NEXT:    crc32w (%rdx), %eax # sched: [10:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call i32 @llvm.x86.sse42.crc32.32.16(i32 %a0, i16 %a1)
-  %2 = load i16, i16 *%a2
-  %3 = call i32 @llvm.x86.sse42.crc32.32.16(i32 %1, i16 %2)
-  ret i32 %3
-}
-declare i32 @llvm.x86.sse42.crc32.32.16(i32, i16) nounwind
-
-define i32 @crc32_32_32(i32 %a0, i32 %a1, i32 *%a2) {
-; GENERIC-LABEL: crc32_32_32:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    movl %edi, %eax # sched: [1:0.33]
-; GENERIC-NEXT:    crc32l %esi, %eax # sched: [3:1.00]
-; GENERIC-NEXT:    crc32l (%rdx), %eax # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SLM-LABEL: crc32_32_32:
-; SLM:       # %bb.0:
-; SLM-NEXT:    movl %edi, %eax # sched: [1:0.50]
-; SLM-NEXT:    crc32l %esi, %eax # sched: [3:1.00]
-; SLM-NEXT:    crc32l (%rdx), %eax # sched: [6:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: crc32_32_32:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    movl %edi, %eax # sched: [1:0.33]
-; SANDY-SSE-NEXT:    crc32l %esi, %eax # sched: [3:1.00]
-; SANDY-SSE-NEXT:    crc32l (%rdx), %eax # sched: [8:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: crc32_32_32:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    movl %edi, %eax # sched: [1:0.33]
-; SANDY-NEXT:    crc32l %esi, %eax # sched: [3:1.00]
-; SANDY-NEXT:    crc32l (%rdx), %eax # sched: [8:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: crc32_32_32:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    movl %edi, %eax # sched: [1:0.25]
-; HASWELL-SSE-NEXT:    crc32l %esi, %eax # sched: [3:1.00]
-; HASWELL-SSE-NEXT:    crc32l (%rdx), %eax # sched: [8:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: crc32_32_32:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    movl %edi, %eax # sched: [1:0.25]
-; HASWELL-NEXT:    crc32l %esi, %eax # sched: [3:1.00]
-; HASWELL-NEXT:    crc32l (%rdx), %eax # sched: [8:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: crc32_32_32:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    movl %edi, %eax # sched: [1:0.25]
-; BROADWELL-SSE-NEXT:    crc32l %esi, %eax # sched: [3:1.00]
-; BROADWELL-SSE-NEXT:    crc32l (%rdx), %eax # sched: [8:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: crc32_32_32:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    movl %edi, %eax # sched: [1:0.25]
-; BROADWELL-NEXT:    crc32l %esi, %eax # sched: [3:1.00]
-; BROADWELL-NEXT:    crc32l (%rdx), %eax # sched: [8:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: crc32_32_32:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    movl %edi, %eax # sched: [1:0.25]
-; SKYLAKE-SSE-NEXT:    crc32l %esi, %eax # sched: [3:1.00]
-; SKYLAKE-SSE-NEXT:    crc32l (%rdx), %eax # sched: [8:1.00]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: crc32_32_32:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    movl %edi, %eax # sched: [1:0.25]
-; SKYLAKE-NEXT:    crc32l %esi, %eax # sched: [3:1.00]
-; SKYLAKE-NEXT:    crc32l (%rdx), %eax # sched: [8:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: crc32_32_32:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    movl %edi, %eax # sched: [1:0.25]
-; SKX-SSE-NEXT:    crc32l %esi, %eax # sched: [3:1.00]
-; SKX-SSE-NEXT:    crc32l (%rdx), %eax # sched: [8:1.00]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: crc32_32_32:
-; SKX:       # %bb.0:
-; SKX-NEXT:    movl %edi, %eax # sched: [1:0.25]
-; SKX-NEXT:    crc32l %esi, %eax # sched: [3:1.00]
-; SKX-NEXT:    crc32l (%rdx), %eax # sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: crc32_32_32:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    movl %edi, %eax # sched: [1:0.50]
-; BDVER2-SSE-NEXT:    crc32l %esi, %eax # sched: [6:2.00]
-; BDVER2-SSE-NEXT:    crc32l (%rdx), %eax # sched: [7:2.00]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: crc32_32_32:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    movl %edi, %eax # sched: [1:0.50]
-; BDVER2-NEXT:    crc32l %esi, %eax # sched: [6:2.00]
-; BDVER2-NEXT:    crc32l (%rdx), %eax # sched: [7:2.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: crc32_32_32:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    movl %edi, %eax # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    crc32l %esi, %eax # sched: [3:2.00]
-; BTVER2-SSE-NEXT:    crc32l (%rdx), %eax # sched: [6:2.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: crc32_32_32:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    movl %edi, %eax # sched: [1:0.50]
-; BTVER2-NEXT:    crc32l %esi, %eax # sched: [3:2.00]
-; BTVER2-NEXT:    crc32l (%rdx), %eax # sched: [6:2.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: crc32_32_32:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    movl %edi, %eax # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    crc32l %esi, %eax # sched: [3:1.00]
-; ZNVER1-SSE-NEXT:    crc32l (%rdx), %eax # sched: [10:1.00]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: crc32_32_32:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    movl %edi, %eax # sched: [1:0.25]
-; ZNVER1-NEXT:    crc32l %esi, %eax # sched: [3:1.00]
-; ZNVER1-NEXT:    crc32l (%rdx), %eax # sched: [10:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call i32 @llvm.x86.sse42.crc32.32.32(i32 %a0, i32 %a1)
-  %2 = load i32, i32 *%a2
-  %3 = call i32 @llvm.x86.sse42.crc32.32.32(i32 %1, i32 %2)
-  ret i32 %3
-}
-declare i32 @llvm.x86.sse42.crc32.32.32(i32, i32) nounwind
-
-define i64 @crc32_64_8(i64 %a0, i8 %a1, i8 *%a2) nounwind {
-; GENERIC-LABEL: crc32_64_8:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    movq %rdi, %rax # sched: [1:0.33]
-; GENERIC-NEXT:    crc32b %sil, %eax # sched: [3:1.00]
-; GENERIC-NEXT:    crc32b (%rdx), %eax # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SLM-LABEL: crc32_64_8:
-; SLM:       # %bb.0:
-; SLM-NEXT:    movq %rdi, %rax # sched: [1:0.50]
-; SLM-NEXT:    crc32b %sil, %eax # sched: [3:1.00]
-; SLM-NEXT:    crc32b (%rdx), %eax # sched: [6:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: crc32_64_8:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    movq %rdi, %rax # sched: [1:0.33]
-; SANDY-SSE-NEXT:    crc32b %sil, %eax # sched: [3:1.00]
-; SANDY-SSE-NEXT:    crc32b (%rdx), %eax # sched: [8:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: crc32_64_8:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    movq %rdi, %rax # sched: [1:0.33]
-; SANDY-NEXT:    crc32b %sil, %eax # sched: [3:1.00]
-; SANDY-NEXT:    crc32b (%rdx), %eax # sched: [8:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: crc32_64_8:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    movq %rdi, %rax # sched: [1:0.25]
-; HASWELL-SSE-NEXT:    crc32b %sil, %eax # sched: [3:1.00]
-; HASWELL-SSE-NEXT:    crc32b (%rdx), %eax # sched: [8:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: crc32_64_8:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    movq %rdi, %rax # sched: [1:0.25]
-; HASWELL-NEXT:    crc32b %sil, %eax # sched: [3:1.00]
-; HASWELL-NEXT:    crc32b (%rdx), %eax # sched: [8:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: crc32_64_8:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    movq %rdi, %rax # sched: [1:0.25]
-; BROADWELL-SSE-NEXT:    crc32b %sil, %eax # sched: [3:1.00]
-; BROADWELL-SSE-NEXT:    crc32b (%rdx), %eax # sched: [8:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: crc32_64_8:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    movq %rdi, %rax # sched: [1:0.25]
-; BROADWELL-NEXT:    crc32b %sil, %eax # sched: [3:1.00]
-; BROADWELL-NEXT:    crc32b (%rdx), %eax # sched: [8:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: crc32_64_8:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    movq %rdi, %rax # sched: [1:0.25]
-; SKYLAKE-SSE-NEXT:    crc32b %sil, %eax # sched: [3:1.00]
-; SKYLAKE-SSE-NEXT:    crc32b (%rdx), %eax # sched: [8:1.00]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: crc32_64_8:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    movq %rdi, %rax # sched: [1:0.25]
-; SKYLAKE-NEXT:    crc32b %sil, %eax # sched: [3:1.00]
-; SKYLAKE-NEXT:    crc32b (%rdx), %eax # sched: [8:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: crc32_64_8:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    movq %rdi, %rax # sched: [1:0.25]
-; SKX-SSE-NEXT:    crc32b %sil, %eax # sched: [3:1.00]
-; SKX-SSE-NEXT:    crc32b (%rdx), %eax # sched: [8:1.00]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: crc32_64_8:
-; SKX:       # %bb.0:
-; SKX-NEXT:    movq %rdi, %rax # sched: [1:0.25]
-; SKX-NEXT:    crc32b %sil, %eax # sched: [3:1.00]
-; SKX-NEXT:    crc32b (%rdx), %eax # sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: crc32_64_8:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    movq %rdi, %rax # sched: [1:0.50]
-; BDVER2-SSE-NEXT:    crc32b %sil, %eax # sched: [3:2.00]
-; BDVER2-SSE-NEXT:    crc32b (%rdx), %eax # sched: [7:2.00]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: crc32_64_8:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    movq %rdi, %rax # sched: [1:0.50]
-; BDVER2-NEXT:    crc32b %sil, %eax # sched: [3:2.00]
-; BDVER2-NEXT:    crc32b (%rdx), %eax # sched: [7:2.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: crc32_64_8:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    movq %rdi, %rax # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    crc32b %sil, %eax # sched: [3:2.00]
-; BTVER2-SSE-NEXT:    crc32b (%rdx), %eax # sched: [6:2.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: crc32_64_8:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    movq %rdi, %rax # sched: [1:0.50]
-; BTVER2-NEXT:    crc32b %sil, %eax # sched: [3:2.00]
-; BTVER2-NEXT:    crc32b (%rdx), %eax # sched: [6:2.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: crc32_64_8:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    movq %rdi, %rax # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    crc32b %sil, %eax # sched: [3:1.00]
-; ZNVER1-SSE-NEXT:    crc32b (%rdx), %eax # sched: [10:1.00]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: crc32_64_8:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    movq %rdi, %rax # sched: [1:0.25]
-; ZNVER1-NEXT:    crc32b %sil, %eax # sched: [3:1.00]
-; ZNVER1-NEXT:    crc32b (%rdx), %eax # sched: [10:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call i64 @llvm.x86.sse42.crc32.64.8(i64 %a0, i8 %a1)
-  %2 = load i8, i8 *%a2
-  %3 = call i64 @llvm.x86.sse42.crc32.64.8(i64 %1, i8 %2)
-  ret i64 %3
-}
-declare i64 @llvm.x86.sse42.crc32.64.8(i64, i8) nounwind
-
-define i64 @crc32_64_64(i64 %a0, i64 %a1, i64 *%a2) {
-; GENERIC-LABEL: crc32_64_64:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    movq %rdi, %rax # sched: [1:0.33]
-; GENERIC-NEXT:    crc32q %rsi, %rax # sched: [3:1.00]
-; GENERIC-NEXT:    crc32q (%rdx), %rax # sched: [8:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SLM-LABEL: crc32_64_64:
-; SLM:       # %bb.0:
-; SLM-NEXT:    movq %rdi, %rax # sched: [1:0.50]
-; SLM-NEXT:    crc32q %rsi, %rax # sched: [3:1.00]
-; SLM-NEXT:    crc32q (%rdx), %rax # sched: [6:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: crc32_64_64:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    movq %rdi, %rax # sched: [1:0.33]
-; SANDY-SSE-NEXT:    crc32q %rsi, %rax # sched: [3:1.00]
-; SANDY-SSE-NEXT:    crc32q (%rdx), %rax # sched: [8:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: crc32_64_64:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    movq %rdi, %rax # sched: [1:0.33]
-; SANDY-NEXT:    crc32q %rsi, %rax # sched: [3:1.00]
-; SANDY-NEXT:    crc32q (%rdx), %rax # sched: [8:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: crc32_64_64:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    movq %rdi, %rax # sched: [1:0.25]
-; HASWELL-SSE-NEXT:    crc32q %rsi, %rax # sched: [3:1.00]
-; HASWELL-SSE-NEXT:    crc32q (%rdx), %rax # sched: [8:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: crc32_64_64:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    movq %rdi, %rax # sched: [1:0.25]
-; HASWELL-NEXT:    crc32q %rsi, %rax # sched: [3:1.00]
-; HASWELL-NEXT:    crc32q (%rdx), %rax # sched: [8:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: crc32_64_64:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    movq %rdi, %rax # sched: [1:0.25]
-; BROADWELL-SSE-NEXT:    crc32q %rsi, %rax # sched: [3:1.00]
-; BROADWELL-SSE-NEXT:    crc32q (%rdx), %rax # sched: [8:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: crc32_64_64:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    movq %rdi, %rax # sched: [1:0.25]
-; BROADWELL-NEXT:    crc32q %rsi, %rax # sched: [3:1.00]
-; BROADWELL-NEXT:    crc32q (%rdx), %rax # sched: [8:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: crc32_64_64:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    movq %rdi, %rax # sched: [1:0.25]
-; SKYLAKE-SSE-NEXT:    crc32q %rsi, %rax # sched: [3:1.00]
-; SKYLAKE-SSE-NEXT:    crc32q (%rdx), %rax # sched: [8:1.00]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: crc32_64_64:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    movq %rdi, %rax # sched: [1:0.25]
-; SKYLAKE-NEXT:    crc32q %rsi, %rax # sched: [3:1.00]
-; SKYLAKE-NEXT:    crc32q (%rdx), %rax # sched: [8:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: crc32_64_64:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    movq %rdi, %rax # sched: [1:0.25]
-; SKX-SSE-NEXT:    crc32q %rsi, %rax # sched: [3:1.00]
-; SKX-SSE-NEXT:    crc32q (%rdx), %rax # sched: [8:1.00]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: crc32_64_64:
-; SKX:       # %bb.0:
-; SKX-NEXT:    movq %rdi, %rax # sched: [1:0.25]
-; SKX-NEXT:    crc32q %rsi, %rax # sched: [3:1.00]
-; SKX-NEXT:    crc32q (%rdx), %rax # sched: [8:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: crc32_64_64:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    movq %rdi, %rax # sched: [1:0.50]
-; BDVER2-SSE-NEXT:    crc32q %rsi, %rax # sched: [10:2.00]
-; BDVER2-SSE-NEXT:    crc32q (%rdx), %rax # sched: [7:2.00]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: crc32_64_64:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    movq %rdi, %rax # sched: [1:0.50]
-; BDVER2-NEXT:    crc32q %rsi, %rax # sched: [10:2.00]
-; BDVER2-NEXT:    crc32q (%rdx), %rax # sched: [7:2.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: crc32_64_64:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    movq %rdi, %rax # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    crc32q %rsi, %rax # sched: [3:2.00]
-; BTVER2-SSE-NEXT:    crc32q (%rdx), %rax # sched: [6:2.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: crc32_64_64:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    movq %rdi, %rax # sched: [1:0.50]
-; BTVER2-NEXT:    crc32q %rsi, %rax # sched: [3:2.00]
-; BTVER2-NEXT:    crc32q (%rdx), %rax # sched: [6:2.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: crc32_64_64:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    movq %rdi, %rax # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    crc32q %rsi, %rax # sched: [3:1.00]
-; ZNVER1-SSE-NEXT:    crc32q (%rdx), %rax # sched: [10:1.00]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: crc32_64_64:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    movq %rdi, %rax # sched: [1:0.25]
-; ZNVER1-NEXT:    crc32q %rsi, %rax # sched: [3:1.00]
-; ZNVER1-NEXT:    crc32q (%rdx), %rax # sched: [10:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call i64 @llvm.x86.sse42.crc32.64.64(i64 %a0, i64 %a1)
-  %2 = load i64, i64 *%a2
-  %3 = call i64 @llvm.x86.sse42.crc32.64.64(i64 %1, i64 %2)
-  ret i64 %3
-}
-declare i64 @llvm.x86.sse42.crc32.64.64(i64, i64) nounwind
-
-define i32 @test_pcmpestri(<16 x i8> %a0, <16 x i8> %a1, <16 x i8> *%a2) {
-; GENERIC-LABEL: test_pcmpestri:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    movl $7, %eax # sched: [1:0.33]
-; GENERIC-NEXT:    movl $7, %edx # sched: [1:0.33]
-; GENERIC-NEXT:    pcmpestri $7, %xmm1, %xmm0 # sched: [4:2.67]
-; GENERIC-NEXT:    movl %ecx, %esi # sched: [1:0.33]
-; GENERIC-NEXT:    movl $7, %eax # sched: [1:0.33]
-; GENERIC-NEXT:    movl $7, %edx # sched: [1:0.33]
-; GENERIC-NEXT:    pcmpestri $7, (%rdi), %xmm0 # sched: [4:2.33]
-; GENERIC-NEXT:    # kill: def $ecx killed $ecx def $rcx
-; GENERIC-NEXT:    leal (%rcx,%rsi), %eax # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SLM-LABEL: test_pcmpestri:
-; SLM:       # %bb.0:
-; SLM-NEXT:    movl $7, %eax # sched: [1:0.50]
-; SLM-NEXT:    movl $7, %edx # sched: [1:0.50]
-; SLM-NEXT:    pcmpestri $7, %xmm1, %xmm0 # sched: [21:21.00]
-; SLM-NEXT:    movl $7, %eax # sched: [1:0.50]
-; SLM-NEXT:    movl $7, %edx # sched: [1:0.50]
-; SLM-NEXT:    movl %ecx, %esi # sched: [1:0.50]
-; SLM-NEXT:    pcmpestri $7, (%rdi), %xmm0 # sched: [21:21.00]
-; SLM-NEXT:    # kill: def $ecx killed $ecx def $rcx
-; SLM-NEXT:    leal (%rcx,%rsi), %eax # sched: [1:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_pcmpestri:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    movl $7, %eax # sched: [1:0.33]
-; SANDY-SSE-NEXT:    movl $7, %edx # sched: [1:0.33]
-; SANDY-SSE-NEXT:    pcmpestri $7, %xmm1, %xmm0 # sched: [4:2.67]
-; SANDY-SSE-NEXT:    movl %ecx, %esi # sched: [1:0.33]
-; SANDY-SSE-NEXT:    movl $7, %eax # sched: [1:0.33]
-; SANDY-SSE-NEXT:    movl $7, %edx # sched: [1:0.33]
-; SANDY-SSE-NEXT:    pcmpestri $7, (%rdi), %xmm0 # sched: [4:2.33]
-; SANDY-SSE-NEXT:    # kill: def $ecx killed $ecx def $rcx
-; SANDY-SSE-NEXT:    leal (%rcx,%rsi), %eax # sched: [1:0.50]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_pcmpestri:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    movl $7, %eax # sched: [1:0.33]
-; SANDY-NEXT:    movl $7, %edx # sched: [1:0.33]
-; SANDY-NEXT:    vpcmpestri $7, %xmm1, %xmm0 # sched: [4:2.67]
-; SANDY-NEXT:    movl %ecx, %esi # sched: [1:0.33]
-; SANDY-NEXT:    movl $7, %eax # sched: [1:0.33]
-; SANDY-NEXT:    movl $7, %edx # sched: [1:0.33]
-; SANDY-NEXT:    vpcmpestri $7, (%rdi), %xmm0 # sched: [4:2.33]
-; SANDY-NEXT:    # kill: def $ecx killed $ecx def $rcx
-; SANDY-NEXT:    leal (%rcx,%rsi), %eax # sched: [1:0.50]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_pcmpestri:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    movl $7, %eax # sched: [1:0.25]
-; HASWELL-SSE-NEXT:    movl $7, %edx # sched: [1:0.25]
-; HASWELL-SSE-NEXT:    pcmpestri $7, %xmm1, %xmm0 # sched: [18:4.00]
-; HASWELL-SSE-NEXT:    movl %ecx, %esi # sched: [1:0.25]
-; HASWELL-SSE-NEXT:    movl $7, %eax # sched: [1:0.25]
-; HASWELL-SSE-NEXT:    movl $7, %edx # sched: [1:0.25]
-; HASWELL-SSE-NEXT:    pcmpestri $7, (%rdi), %xmm0 # sched: [24:4.00]
-; HASWELL-SSE-NEXT:    # kill: def $ecx killed $ecx def $rcx
-; HASWELL-SSE-NEXT:    leal (%rcx,%rsi), %eax # sched: [1:0.50]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_pcmpestri:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    movl $7, %eax # sched: [1:0.25]
-; HASWELL-NEXT:    movl $7, %edx # sched: [1:0.25]
-; HASWELL-NEXT:    vpcmpestri $7, %xmm1, %xmm0 # sched: [18:4.00]
-; HASWELL-NEXT:    movl %ecx, %esi # sched: [1:0.25]
-; HASWELL-NEXT:    movl $7, %eax # sched: [1:0.25]
-; HASWELL-NEXT:    movl $7, %edx # sched: [1:0.25]
-; HASWELL-NEXT:    vpcmpestri $7, (%rdi), %xmm0 # sched: [24:4.00]
-; HASWELL-NEXT:    # kill: def $ecx killed $ecx def $rcx
-; HASWELL-NEXT:    leal (%rcx,%rsi), %eax # sched: [1:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_pcmpestri:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    movl $7, %eax # sched: [1:0.25]
-; BROADWELL-SSE-NEXT:    movl $7, %edx # sched: [1:0.25]
-; BROADWELL-SSE-NEXT:    pcmpestri $7, %xmm1, %xmm0 # sched: [18:4.00]
-; BROADWELL-SSE-NEXT:    movl %ecx, %esi # sched: [1:0.25]
-; BROADWELL-SSE-NEXT:    movl $7, %eax # sched: [1:0.25]
-; BROADWELL-SSE-NEXT:    movl $7, %edx # sched: [1:0.25]
-; BROADWELL-SSE-NEXT:    pcmpestri $7, (%rdi), %xmm0 # sched: [23:4.00]
-; BROADWELL-SSE-NEXT:    # kill: def $ecx killed $ecx def $rcx
-; BROADWELL-SSE-NEXT:    leal (%rcx,%rsi), %eax # sched: [1:0.50]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pcmpestri:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    movl $7, %eax # sched: [1:0.25]
-; BROADWELL-NEXT:    movl $7, %edx # sched: [1:0.25]
-; BROADWELL-NEXT:    vpcmpestri $7, %xmm1, %xmm0 # sched: [18:4.00]
-; BROADWELL-NEXT:    movl %ecx, %esi # sched: [1:0.25]
-; BROADWELL-NEXT:    movl $7, %eax # sched: [1:0.25]
-; BROADWELL-NEXT:    movl $7, %edx # sched: [1:0.25]
-; BROADWELL-NEXT:    vpcmpestri $7, (%rdi), %xmm0 # sched: [23:4.00]
-; BROADWELL-NEXT:    # kill: def $ecx killed $ecx def $rcx
-; BROADWELL-NEXT:    leal (%rcx,%rsi), %eax # sched: [1:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_pcmpestri:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    movl $7, %eax # sched: [1:0.25]
-; SKYLAKE-SSE-NEXT:    movl $7, %edx # sched: [1:0.25]
-; SKYLAKE-SSE-NEXT:    pcmpestri $7, %xmm1, %xmm0 # sched: [18:4.00]
-; SKYLAKE-SSE-NEXT:    movl %ecx, %esi # sched: [1:0.25]
-; SKYLAKE-SSE-NEXT:    movl $7, %eax # sched: [1:0.25]
-; SKYLAKE-SSE-NEXT:    movl $7, %edx # sched: [1:0.25]
-; SKYLAKE-SSE-NEXT:    pcmpestri $7, (%rdi), %xmm0 # sched: [24:4.00]
-; SKYLAKE-SSE-NEXT:    # kill: def $ecx killed $ecx def $rcx
-; SKYLAKE-SSE-NEXT:    leal (%rcx,%rsi), %eax # sched: [1:0.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pcmpestri:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    movl $7, %eax # sched: [1:0.25]
-; SKYLAKE-NEXT:    movl $7, %edx # sched: [1:0.25]
-; SKYLAKE-NEXT:    vpcmpestri $7, %xmm1, %xmm0 # sched: [18:4.00]
-; SKYLAKE-NEXT:    movl %ecx, %esi # sched: [1:0.25]
-; SKYLAKE-NEXT:    movl $7, %eax # sched: [1:0.25]
-; SKYLAKE-NEXT:    movl $7, %edx # sched: [1:0.25]
-; SKYLAKE-NEXT:    vpcmpestri $7, (%rdi), %xmm0 # sched: [24:4.00]
-; SKYLAKE-NEXT:    # kill: def $ecx killed $ecx def $rcx
-; SKYLAKE-NEXT:    leal (%rcx,%rsi), %eax # sched: [1:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_pcmpestri:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    movl $7, %eax # sched: [1:0.25]
-; SKX-SSE-NEXT:    movl $7, %edx # sched: [1:0.25]
-; SKX-SSE-NEXT:    pcmpestri $7, %xmm1, %xmm0 # sched: [18:4.00]
-; SKX-SSE-NEXT:    movl %ecx, %esi # sched: [1:0.25]
-; SKX-SSE-NEXT:    movl $7, %eax # sched: [1:0.25]
-; SKX-SSE-NEXT:    movl $7, %edx # sched: [1:0.25]
-; SKX-SSE-NEXT:    pcmpestri $7, (%rdi), %xmm0 # sched: [24:4.00]
-; SKX-SSE-NEXT:    # kill: def $ecx killed $ecx def $rcx
-; SKX-SSE-NEXT:    leal (%rcx,%rsi), %eax # sched: [1:0.50]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pcmpestri:
-; SKX:       # %bb.0:
-; SKX-NEXT:    movl $7, %eax # sched: [1:0.25]
-; SKX-NEXT:    movl $7, %edx # sched: [1:0.25]
-; SKX-NEXT:    vpcmpestri $7, %xmm1, %xmm0 # sched: [18:4.00]
-; SKX-NEXT:    movl %ecx, %esi # sched: [1:0.25]
-; SKX-NEXT:    movl $7, %eax # sched: [1:0.25]
-; SKX-NEXT:    movl $7, %edx # sched: [1:0.25]
-; SKX-NEXT:    vpcmpestri $7, (%rdi), %xmm0 # sched: [24:4.00]
-; SKX-NEXT:    # kill: def $ecx killed $ecx def $rcx
-; SKX-NEXT:    leal (%rcx,%rsi), %eax # sched: [1:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_pcmpestri:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    movl $7, %eax # sched: [1:0.50]
-; BDVER2-SSE-NEXT:    movl $7, %edx # sched: [1:0.50]
-; BDVER2-SSE-NEXT:    pcmpestri $7, %xmm1, %xmm0 # sched: [15:4.00]
-; BDVER2-SSE-NEXT:    movl $7, %eax # sched: [1:0.50]
-; BDVER2-SSE-NEXT:    movl $7, %edx # sched: [1:0.50]
-; BDVER2-SSE-NEXT:    movl %ecx, %esi # sched: [1:0.50]
-; BDVER2-SSE-NEXT:    pcmpestri $7, (%rdi), %xmm0 # sched: [20:4.50]
-; BDVER2-SSE-NEXT:    # kill: def $ecx killed $ecx def $rcx
-; BDVER2-SSE-NEXT:    leal (%rcx,%rsi), %eax # sched: [1:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_pcmpestri:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    movl $7, %eax # sched: [1:0.50]
-; BDVER2-NEXT:    movl $7, %edx # sched: [1:0.50]
-; BDVER2-NEXT:    vpcmpestri $7, %xmm1, %xmm0 # sched: [15:4.00]
-; BDVER2-NEXT:    movl $7, %eax # sched: [1:0.50]
-; BDVER2-NEXT:    movl $7, %edx # sched: [1:0.50]
-; BDVER2-NEXT:    movl %ecx, %esi # sched: [1:0.50]
-; BDVER2-NEXT:    vpcmpestri $7, (%rdi), %xmm0 # sched: [20:4.50]
-; BDVER2-NEXT:    # kill: def $ecx killed $ecx def $rcx
-; BDVER2-NEXT:    leal (%rcx,%rsi), %eax # sched: [1:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_pcmpestri:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    movl $7, %eax # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    movl $7, %edx # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    pcmpestri $7, %xmm1, %xmm0 # sched: [14:5.00]
-; BTVER2-SSE-NEXT:    movl $7, %eax # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    movl $7, %edx # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    movl %ecx, %esi # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    pcmpestri $7, (%rdi), %xmm0 # sched: [19:5.00]
-; BTVER2-SSE-NEXT:    # kill: def $ecx killed $ecx def $rcx
-; BTVER2-SSE-NEXT:    leal (%rcx,%rsi), %eax # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_pcmpestri:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    movl $7, %eax # sched: [1:0.50]
-; BTVER2-NEXT:    movl $7, %edx # sched: [1:0.50]
-; BTVER2-NEXT:    vpcmpestri $7, %xmm1, %xmm0 # sched: [14:5.00]
-; BTVER2-NEXT:    movl $7, %eax # sched: [1:0.50]
-; BTVER2-NEXT:    movl $7, %edx # sched: [1:0.50]
-; BTVER2-NEXT:    movl %ecx, %esi # sched: [1:0.50]
-; BTVER2-NEXT:    vpcmpestri $7, (%rdi), %xmm0 # sched: [19:5.00]
-; BTVER2-NEXT:    # kill: def $ecx killed $ecx def $rcx
-; BTVER2-NEXT:    leal (%rcx,%rsi), %eax # sched: [1:0.50]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_pcmpestri:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    movl $7, %eax # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    movl $7, %edx # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    pcmpestri $7, %xmm1, %xmm0 # sched: [100:0.25]
-; ZNVER1-SSE-NEXT:    movl $7, %eax # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    movl $7, %edx # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    movl %ecx, %esi # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    pcmpestri $7, (%rdi), %xmm0 # sched: [100:0.25]
-; ZNVER1-SSE-NEXT:    # kill: def $ecx killed $ecx def $rcx
-; ZNVER1-SSE-NEXT:    leal (%rcx,%rsi), %eax # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_pcmpestri:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    movl $7, %eax # sched: [1:0.25]
-; ZNVER1-NEXT:    movl $7, %edx # sched: [1:0.25]
-; ZNVER1-NEXT:    vpcmpestri $7, %xmm1, %xmm0 # sched: [100:0.25]
-; ZNVER1-NEXT:    movl $7, %eax # sched: [1:0.25]
-; ZNVER1-NEXT:    movl $7, %edx # sched: [1:0.25]
-; ZNVER1-NEXT:    movl %ecx, %esi # sched: [1:0.25]
-; ZNVER1-NEXT:    vpcmpestri $7, (%rdi), %xmm0 # sched: [100:0.25]
-; ZNVER1-NEXT:    # kill: def $ecx killed $ecx def $rcx
-; ZNVER1-NEXT:    leal (%rcx,%rsi), %eax # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call i32 @llvm.x86.sse42.pcmpestri128(<16 x i8> %a0, i32 7, <16 x i8> %a1, i32 7, i8 7)
-  %2 = load <16 x i8>, <16 x i8> *%a2, align 16
-  %3 = call i32 @llvm.x86.sse42.pcmpestri128(<16 x i8> %a0, i32 7, <16 x i8> %2, i32 7, i8 7)
-  %4 = add i32 %1, %3
-  ret i32 %4
-}
-declare i32 @llvm.x86.sse42.pcmpestri128(<16 x i8>, i32, <16 x i8>, i32, i8) nounwind readnone
-
-define <16 x i8> @test_pcmpestrm(<16 x i8> %a0, <16 x i8> %a1, <16 x i8> *%a2) {
-; GENERIC-LABEL: test_pcmpestrm:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    movl $7, %eax # sched: [1:0.33]
-; GENERIC-NEXT:    movl $7, %edx # sched: [1:0.33]
-; GENERIC-NEXT:    pcmpestrm $7, %xmm1, %xmm0 # sched: [11:2.67]
-; GENERIC-NEXT:    movl $7, %eax # sched: [1:0.33]
-; GENERIC-NEXT:    movl $7, %edx # sched: [1:0.33]
-; GENERIC-NEXT:    pcmpestrm $7, (%rdi), %xmm0 # sched: [11:2.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SLM-LABEL: test_pcmpestrm:
-; SLM:       # %bb.0:
-; SLM-NEXT:    movl $7, %eax # sched: [1:0.50]
-; SLM-NEXT:    movl $7, %edx # sched: [1:0.50]
-; SLM-NEXT:    pcmpestrm $7, %xmm1, %xmm0 # sched: [17:17.00]
-; SLM-NEXT:    movl $7, %eax # sched: [1:0.50]
-; SLM-NEXT:    movl $7, %edx # sched: [1:0.50]
-; SLM-NEXT:    pcmpestrm $7, (%rdi), %xmm0 # sched: [17:17.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_pcmpestrm:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    movl $7, %eax # sched: [1:0.33]
-; SANDY-SSE-NEXT:    movl $7, %edx # sched: [1:0.33]
-; SANDY-SSE-NEXT:    pcmpestrm $7, %xmm1, %xmm0 # sched: [11:2.67]
-; SANDY-SSE-NEXT:    movl $7, %eax # sched: [1:0.33]
-; SANDY-SSE-NEXT:    movl $7, %edx # sched: [1:0.33]
-; SANDY-SSE-NEXT:    pcmpestrm $7, (%rdi), %xmm0 # sched: [11:2.33]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_pcmpestrm:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    movl $7, %eax # sched: [1:0.33]
-; SANDY-NEXT:    movl $7, %edx # sched: [1:0.33]
-; SANDY-NEXT:    vpcmpestrm $7, %xmm1, %xmm0 # sched: [11:2.67]
-; SANDY-NEXT:    movl $7, %eax # sched: [1:0.33]
-; SANDY-NEXT:    movl $7, %edx # sched: [1:0.33]
-; SANDY-NEXT:    vpcmpestrm $7, (%rdi), %xmm0 # sched: [11:2.33]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_pcmpestrm:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    movl $7, %eax # sched: [1:0.25]
-; HASWELL-SSE-NEXT:    movl $7, %edx # sched: [1:0.25]
-; HASWELL-SSE-NEXT:    pcmpestrm $7, %xmm1, %xmm0 # sched: [19:4.00]
-; HASWELL-SSE-NEXT:    movl $7, %eax # sched: [1:0.25]
-; HASWELL-SSE-NEXT:    movl $7, %edx # sched: [1:0.25]
-; HASWELL-SSE-NEXT:    pcmpestrm $7, (%rdi), %xmm0 # sched: [25:4.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_pcmpestrm:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    movl $7, %eax # sched: [1:0.25]
-; HASWELL-NEXT:    movl $7, %edx # sched: [1:0.25]
-; HASWELL-NEXT:    vpcmpestrm $7, %xmm1, %xmm0 # sched: [19:4.00]
-; HASWELL-NEXT:    movl $7, %eax # sched: [1:0.25]
-; HASWELL-NEXT:    movl $7, %edx # sched: [1:0.25]
-; HASWELL-NEXT:    vpcmpestrm $7, (%rdi), %xmm0 # sched: [25:4.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_pcmpestrm:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    movl $7, %eax # sched: [1:0.25]
-; BROADWELL-SSE-NEXT:    movl $7, %edx # sched: [1:0.25]
-; BROADWELL-SSE-NEXT:    pcmpestrm $7, %xmm1, %xmm0 # sched: [19:4.00]
-; BROADWELL-SSE-NEXT:    movl $7, %eax # sched: [1:0.25]
-; BROADWELL-SSE-NEXT:    movl $7, %edx # sched: [1:0.25]
-; BROADWELL-SSE-NEXT:    pcmpestrm $7, (%rdi), %xmm0 # sched: [24:4.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pcmpestrm:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    movl $7, %eax # sched: [1:0.25]
-; BROADWELL-NEXT:    movl $7, %edx # sched: [1:0.25]
-; BROADWELL-NEXT:    vpcmpestrm $7, %xmm1, %xmm0 # sched: [19:4.00]
-; BROADWELL-NEXT:    movl $7, %eax # sched: [1:0.25]
-; BROADWELL-NEXT:    movl $7, %edx # sched: [1:0.25]
-; BROADWELL-NEXT:    vpcmpestrm $7, (%rdi), %xmm0 # sched: [24:4.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_pcmpestrm:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    movl $7, %eax # sched: [1:0.25]
-; SKYLAKE-SSE-NEXT:    movl $7, %edx # sched: [1:0.25]
-; SKYLAKE-SSE-NEXT:    pcmpestrm $7, %xmm1, %xmm0 # sched: [19:4.00]
-; SKYLAKE-SSE-NEXT:    movl $7, %eax # sched: [1:0.25]
-; SKYLAKE-SSE-NEXT:    movl $7, %edx # sched: [1:0.25]
-; SKYLAKE-SSE-NEXT:    pcmpestrm $7, (%rdi), %xmm0 # sched: [25:4.00]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pcmpestrm:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    movl $7, %eax # sched: [1:0.25]
-; SKYLAKE-NEXT:    movl $7, %edx # sched: [1:0.25]
-; SKYLAKE-NEXT:    vpcmpestrm $7, %xmm1, %xmm0 # sched: [19:4.00]
-; SKYLAKE-NEXT:    movl $7, %eax # sched: [1:0.25]
-; SKYLAKE-NEXT:    movl $7, %edx # sched: [1:0.25]
-; SKYLAKE-NEXT:    vpcmpestrm $7, (%rdi), %xmm0 # sched: [25:4.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_pcmpestrm:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    movl $7, %eax # sched: [1:0.25]
-; SKX-SSE-NEXT:    movl $7, %edx # sched: [1:0.25]
-; SKX-SSE-NEXT:    pcmpestrm $7, %xmm1, %xmm0 # sched: [19:4.00]
-; SKX-SSE-NEXT:    movl $7, %eax # sched: [1:0.25]
-; SKX-SSE-NEXT:    movl $7, %edx # sched: [1:0.25]
-; SKX-SSE-NEXT:    pcmpestrm $7, (%rdi), %xmm0 # sched: [25:4.00]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pcmpestrm:
-; SKX:       # %bb.0:
-; SKX-NEXT:    movl $7, %eax # sched: [1:0.25]
-; SKX-NEXT:    movl $7, %edx # sched: [1:0.25]
-; SKX-NEXT:    vpcmpestrm $7, %xmm1, %xmm0 # sched: [19:4.00]
-; SKX-NEXT:    movl $7, %eax # sched: [1:0.25]
-; SKX-NEXT:    movl $7, %edx # sched: [1:0.25]
-; SKX-NEXT:    vpcmpestrm $7, (%rdi), %xmm0 # sched: [25:4.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_pcmpestrm:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    movl $7, %eax # sched: [1:0.50]
-; BDVER2-SSE-NEXT:    movl $7, %edx # sched: [1:0.50]
-; BDVER2-SSE-NEXT:    pcmpestrm $7, %xmm1, %xmm0 # sched: [10:4.00]
-; BDVER2-SSE-NEXT:    movl $7, %eax # sched: [1:0.50]
-; BDVER2-SSE-NEXT:    movl $7, %edx # sched: [1:0.50]
-; BDVER2-SSE-NEXT:    pcmpestrm $7, (%rdi), %xmm0 # sched: [15:4.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_pcmpestrm:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    movl $7, %eax # sched: [1:0.50]
-; BDVER2-NEXT:    movl $7, %edx # sched: [1:0.50]
-; BDVER2-NEXT:    vpcmpestrm $7, %xmm1, %xmm0 # sched: [10:4.00]
-; BDVER2-NEXT:    movl $7, %eax # sched: [1:0.50]
-; BDVER2-NEXT:    movl $7, %edx # sched: [1:0.50]
-; BDVER2-NEXT:    vpcmpestrm $7, (%rdi), %xmm0 # sched: [15:4.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_pcmpestrm:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    movl $7, %eax # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    movl $7, %edx # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    pcmpestrm $7, %xmm1, %xmm0 # sched: [14:5.00]
-; BTVER2-SSE-NEXT:    movl $7, %eax # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    movl $7, %edx # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    pcmpestrm $7, (%rdi), %xmm0 # sched: [19:5.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_pcmpestrm:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    movl $7, %eax # sched: [1:0.50]
-; BTVER2-NEXT:    movl $7, %edx # sched: [1:0.50]
-; BTVER2-NEXT:    vpcmpestrm $7, %xmm1, %xmm0 # sched: [14:5.00]
-; BTVER2-NEXT:    movl $7, %eax # sched: [1:0.50]
-; BTVER2-NEXT:    movl $7, %edx # sched: [1:0.50]
-; BTVER2-NEXT:    vpcmpestrm $7, (%rdi), %xmm0 # sched: [19:5.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_pcmpestrm:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    movl $7, %eax # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    movl $7, %edx # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    pcmpestrm $7, %xmm1, %xmm0 # sched: [100:0.25]
-; ZNVER1-SSE-NEXT:    movl $7, %eax # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    movl $7, %edx # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    pcmpestrm $7, (%rdi), %xmm0 # sched: [100:0.25]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_pcmpestrm:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    movl $7, %eax # sched: [1:0.25]
-; ZNVER1-NEXT:    movl $7, %edx # sched: [1:0.25]
-; ZNVER1-NEXT:    vpcmpestrm $7, %xmm1, %xmm0 # sched: [100:0.25]
-; ZNVER1-NEXT:    movl $7, %eax # sched: [1:0.25]
-; ZNVER1-NEXT:    movl $7, %edx # sched: [1:0.25]
-; ZNVER1-NEXT:    vpcmpestrm $7, (%rdi), %xmm0 # sched: [100:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <16 x i8> @llvm.x86.sse42.pcmpestrm128(<16 x i8> %a0, i32 7, <16 x i8> %a1, i32 7, i8 7)
-  %2 = load <16 x i8>, <16 x i8> *%a2, align 16
-  %3 = call <16 x i8> @llvm.x86.sse42.pcmpestrm128(<16 x i8> %1, i32 7, <16 x i8> %2, i32 7, i8 7)
-  ret <16 x i8> %3
-}
-declare <16 x i8> @llvm.x86.sse42.pcmpestrm128(<16 x i8>, i32, <16 x i8>, i32, i8) nounwind readnone
-
-define i32 @test_pcmpistri(<16 x i8> %a0, <16 x i8> %a1, <16 x i8> *%a2) {
-; GENERIC-LABEL: test_pcmpistri:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    pcmpistri $7, %xmm1, %xmm0 # sched: [11:3.00]
-; GENERIC-NEXT:    movl %ecx, %eax # sched: [1:0.33]
-; GENERIC-NEXT:    pcmpistri $7, (%rdi), %xmm0 # sched: [17:3.00]
-; GENERIC-NEXT:    # kill: def $ecx killed $ecx def $rcx
-; GENERIC-NEXT:    leal (%rcx,%rax), %eax # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SLM-LABEL: test_pcmpistri:
-; SLM:       # %bb.0:
-; SLM-NEXT:    pcmpistri $7, %xmm1, %xmm0 # sched: [17:17.00]
-; SLM-NEXT:    movl %ecx, %eax # sched: [1:0.50]
-; SLM-NEXT:    pcmpistri $7, (%rdi), %xmm0 # sched: [17:17.00]
-; SLM-NEXT:    # kill: def $ecx killed $ecx def $rcx
-; SLM-NEXT:    leal (%rcx,%rax), %eax # sched: [1:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_pcmpistri:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    pcmpistri $7, %xmm1, %xmm0 # sched: [11:3.00]
-; SANDY-SSE-NEXT:    movl %ecx, %eax # sched: [1:0.33]
-; SANDY-SSE-NEXT:    pcmpistri $7, (%rdi), %xmm0 # sched: [17:3.00]
-; SANDY-SSE-NEXT:    # kill: def $ecx killed $ecx def $rcx
-; SANDY-SSE-NEXT:    leal (%rcx,%rax), %eax # sched: [1:0.50]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_pcmpistri:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vpcmpistri $7, %xmm1, %xmm0 # sched: [11:3.00]
-; SANDY-NEXT:    movl %ecx, %eax # sched: [1:0.33]
-; SANDY-NEXT:    vpcmpistri $7, (%rdi), %xmm0 # sched: [17:3.00]
-; SANDY-NEXT:    # kill: def $ecx killed $ecx def $rcx
-; SANDY-NEXT:    leal (%rcx,%rax), %eax # sched: [1:0.50]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_pcmpistri:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    pcmpistri $7, %xmm1, %xmm0 # sched: [11:3.00]
-; HASWELL-SSE-NEXT:    movl %ecx, %eax # sched: [1:0.25]
-; HASWELL-SSE-NEXT:    pcmpistri $7, (%rdi), %xmm0 # sched: [17:3.00]
-; HASWELL-SSE-NEXT:    # kill: def $ecx killed $ecx def $rcx
-; HASWELL-SSE-NEXT:    leal (%rcx,%rax), %eax # sched: [1:0.50]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_pcmpistri:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpcmpistri $7, %xmm1, %xmm0 # sched: [11:3.00]
-; HASWELL-NEXT:    movl %ecx, %eax # sched: [1:0.25]
-; HASWELL-NEXT:    vpcmpistri $7, (%rdi), %xmm0 # sched: [17:3.00]
-; HASWELL-NEXT:    # kill: def $ecx killed $ecx def $rcx
-; HASWELL-NEXT:    leal (%rcx,%rax), %eax # sched: [1:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_pcmpistri:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    pcmpistri $7, %xmm1, %xmm0 # sched: [11:3.00]
-; BROADWELL-SSE-NEXT:    movl %ecx, %eax # sched: [1:0.25]
-; BROADWELL-SSE-NEXT:    pcmpistri $7, (%rdi), %xmm0 # sched: [16:3.00]
-; BROADWELL-SSE-NEXT:    # kill: def $ecx killed $ecx def $rcx
-; BROADWELL-SSE-NEXT:    leal (%rcx,%rax), %eax # sched: [1:0.50]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pcmpistri:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpcmpistri $7, %xmm1, %xmm0 # sched: [11:3.00]
-; BROADWELL-NEXT:    movl %ecx, %eax # sched: [1:0.25]
-; BROADWELL-NEXT:    vpcmpistri $7, (%rdi), %xmm0 # sched: [16:3.00]
-; BROADWELL-NEXT:    # kill: def $ecx killed $ecx def $rcx
-; BROADWELL-NEXT:    leal (%rcx,%rax), %eax # sched: [1:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_pcmpistri:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    pcmpistri $7, %xmm1, %xmm0 # sched: [10:3.00]
-; SKYLAKE-SSE-NEXT:    movl %ecx, %eax # sched: [1:0.25]
-; SKYLAKE-SSE-NEXT:    pcmpistri $7, (%rdi), %xmm0 # sched: [16:3.00]
-; SKYLAKE-SSE-NEXT:    # kill: def $ecx killed $ecx def $rcx
-; SKYLAKE-SSE-NEXT:    leal (%rcx,%rax), %eax # sched: [1:0.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pcmpistri:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpcmpistri $7, %xmm1, %xmm0 # sched: [10:3.00]
-; SKYLAKE-NEXT:    movl %ecx, %eax # sched: [1:0.25]
-; SKYLAKE-NEXT:    vpcmpistri $7, (%rdi), %xmm0 # sched: [16:3.00]
-; SKYLAKE-NEXT:    # kill: def $ecx killed $ecx def $rcx
-; SKYLAKE-NEXT:    leal (%rcx,%rax), %eax # sched: [1:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_pcmpistri:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    pcmpistri $7, %xmm1, %xmm0 # sched: [10:3.00]
-; SKX-SSE-NEXT:    movl %ecx, %eax # sched: [1:0.25]
-; SKX-SSE-NEXT:    pcmpistri $7, (%rdi), %xmm0 # sched: [16:3.00]
-; SKX-SSE-NEXT:    # kill: def $ecx killed $ecx def $rcx
-; SKX-SSE-NEXT:    leal (%rcx,%rax), %eax # sched: [1:0.50]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pcmpistri:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpcmpistri $7, %xmm1, %xmm0 # sched: [10:3.00]
-; SKX-NEXT:    movl %ecx, %eax # sched: [1:0.25]
-; SKX-NEXT:    vpcmpistri $7, (%rdi), %xmm0 # sched: [16:3.00]
-; SKX-NEXT:    # kill: def $ecx killed $ecx def $rcx
-; SKX-NEXT:    leal (%rcx,%rax), %eax # sched: [1:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_pcmpistri:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    pcmpistri $7, %xmm1, %xmm0 # sched: [14:1.00]
-; BDVER2-SSE-NEXT:    movl %ecx, %eax # sched: [1:0.50]
-; BDVER2-SSE-NEXT:    pcmpistri $7, (%rdi), %xmm0 # sched: [19:1.00]
-; BDVER2-SSE-NEXT:    # kill: def $ecx killed $ecx def $rcx
-; BDVER2-SSE-NEXT:    leal (%rcx,%rax), %eax # sched: [1:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_pcmpistri:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vpcmpistri $7, %xmm1, %xmm0 # sched: [14:1.00]
-; BDVER2-NEXT:    movl %ecx, %eax # sched: [1:0.50]
-; BDVER2-NEXT:    vpcmpistri $7, (%rdi), %xmm0 # sched: [19:1.00]
-; BDVER2-NEXT:    # kill: def $ecx killed $ecx def $rcx
-; BDVER2-NEXT:    leal (%rcx,%rax), %eax # sched: [1:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_pcmpistri:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    pcmpistri $7, %xmm1, %xmm0 # sched: [7:2.00]
-; BTVER2-SSE-NEXT:    movl %ecx, %eax # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    pcmpistri $7, (%rdi), %xmm0 # sched: [12:2.00]
-; BTVER2-SSE-NEXT:    # kill: def $ecx killed $ecx def $rcx
-; BTVER2-SSE-NEXT:    leal (%rcx,%rax), %eax # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_pcmpistri:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vpcmpistri $7, %xmm1, %xmm0 # sched: [7:2.00]
-; BTVER2-NEXT:    movl %ecx, %eax # sched: [1:0.50]
-; BTVER2-NEXT:    vpcmpistri $7, (%rdi), %xmm0 # sched: [12:2.00]
-; BTVER2-NEXT:    # kill: def $ecx killed $ecx def $rcx
-; BTVER2-NEXT:    leal (%rcx,%rax), %eax # sched: [1:0.50]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_pcmpistri:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    pcmpistri $7, %xmm1, %xmm0 # sched: [100:0.25]
-; ZNVER1-SSE-NEXT:    movl %ecx, %eax # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    pcmpistri $7, (%rdi), %xmm0 # sched: [100:0.25]
-; ZNVER1-SSE-NEXT:    # kill: def $ecx killed $ecx def $rcx
-; ZNVER1-SSE-NEXT:    leal (%rcx,%rax), %eax # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_pcmpistri:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpcmpistri $7, %xmm1, %xmm0 # sched: [100:0.25]
-; ZNVER1-NEXT:    movl %ecx, %eax # sched: [1:0.25]
-; ZNVER1-NEXT:    vpcmpistri $7, (%rdi), %xmm0 # sched: [100:0.25]
-; ZNVER1-NEXT:    # kill: def $ecx killed $ecx def $rcx
-; ZNVER1-NEXT:    leal (%rcx,%rax), %eax # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call i32 @llvm.x86.sse42.pcmpistri128(<16 x i8> %a0, <16 x i8> %a1, i8 7)
-  %2 = load <16 x i8>, <16 x i8> *%a2, align 16
-  %3 = call i32 @llvm.x86.sse42.pcmpistri128(<16 x i8> %a0, <16 x i8> %2, i8 7)
-  %4 = add i32 %1, %3
-  ret i32 %4
-}
-declare i32 @llvm.x86.sse42.pcmpistri128(<16 x i8>, <16 x i8>, i8) nounwind readnone
-
-define <16 x i8> @test_pcmpistrm(<16 x i8> %a0, <16 x i8> %a1, <16 x i8> *%a2) {
-; GENERIC-LABEL: test_pcmpistrm:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    pcmpistrm $7, %xmm1, %xmm0 # sched: [11:3.00]
-; GENERIC-NEXT:    pcmpistrm $7, (%rdi), %xmm0 # sched: [17:3.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SLM-LABEL: test_pcmpistrm:
-; SLM:       # %bb.0:
-; SLM-NEXT:    pcmpistrm $7, %xmm1, %xmm0 # sched: [13:13.00]
-; SLM-NEXT:    pcmpistrm $7, (%rdi), %xmm0 # sched: [13:13.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_pcmpistrm:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    pcmpistrm $7, %xmm1, %xmm0 # sched: [11:3.00]
-; SANDY-SSE-NEXT:    pcmpistrm $7, (%rdi), %xmm0 # sched: [17:3.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_pcmpistrm:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vpcmpistrm $7, %xmm1, %xmm0 # sched: [11:3.00]
-; SANDY-NEXT:    vpcmpistrm $7, (%rdi), %xmm0 # sched: [17:3.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_pcmpistrm:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    pcmpistrm $7, %xmm1, %xmm0 # sched: [11:3.00]
-; HASWELL-SSE-NEXT:    pcmpistrm $7, (%rdi), %xmm0 # sched: [17:3.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_pcmpistrm:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpcmpistrm $7, %xmm1, %xmm0 # sched: [11:3.00]
-; HASWELL-NEXT:    vpcmpistrm $7, (%rdi), %xmm0 # sched: [17:3.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_pcmpistrm:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    pcmpistrm $7, %xmm1, %xmm0 # sched: [11:3.00]
-; BROADWELL-SSE-NEXT:    pcmpistrm $7, (%rdi), %xmm0 # sched: [16:3.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pcmpistrm:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpcmpistrm $7, %xmm1, %xmm0 # sched: [11:3.00]
-; BROADWELL-NEXT:    vpcmpistrm $7, (%rdi), %xmm0 # sched: [16:3.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_pcmpistrm:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    pcmpistrm $7, %xmm1, %xmm0 # sched: [10:3.00]
-; SKYLAKE-SSE-NEXT:    pcmpistrm $7, (%rdi), %xmm0 # sched: [16:3.00]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pcmpistrm:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpcmpistrm $7, %xmm1, %xmm0 # sched: [10:3.00]
-; SKYLAKE-NEXT:    vpcmpistrm $7, (%rdi), %xmm0 # sched: [16:3.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_pcmpistrm:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    pcmpistrm $7, %xmm1, %xmm0 # sched: [10:3.00]
-; SKX-SSE-NEXT:    pcmpistrm $7, (%rdi), %xmm0 # sched: [16:3.00]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pcmpistrm:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpcmpistrm $7, %xmm1, %xmm0 # sched: [10:3.00]
-; SKX-NEXT:    vpcmpistrm $7, (%rdi), %xmm0 # sched: [16:3.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_pcmpistrm:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    pcmpistrm $7, %xmm1, %xmm0 # sched: [6:1.00]
-; BDVER2-SSE-NEXT:    pcmpistrm $7, (%rdi), %xmm0 # sched: [11:1.00]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_pcmpistrm:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vpcmpistrm $7, %xmm1, %xmm0 # sched: [6:1.00]
-; BDVER2-NEXT:    vpcmpistrm $7, (%rdi), %xmm0 # sched: [11:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_pcmpistrm:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    pcmpistrm $7, %xmm1, %xmm0 # sched: [8:2.00]
-; BTVER2-SSE-NEXT:    pcmpistrm $7, (%rdi), %xmm0 # sched: [13:2.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_pcmpistrm:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vpcmpistrm $7, %xmm1, %xmm0 # sched: [8:2.00]
-; BTVER2-NEXT:    vpcmpistrm $7, (%rdi), %xmm0 # sched: [13:2.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_pcmpistrm:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    pcmpistrm $7, %xmm1, %xmm0 # sched: [100:0.25]
-; ZNVER1-SSE-NEXT:    pcmpistrm $7, (%rdi), %xmm0 # sched: [100:0.25]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_pcmpistrm:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpcmpistrm $7, %xmm1, %xmm0 # sched: [100:0.25]
-; ZNVER1-NEXT:    vpcmpistrm $7, (%rdi), %xmm0 # sched: [100:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <16 x i8> @llvm.x86.sse42.pcmpistrm128(<16 x i8> %a0, <16 x i8> %a1, i8 7)
-  %2 = load <16 x i8>, <16 x i8> *%a2, align 16
-  %3 = call <16 x i8> @llvm.x86.sse42.pcmpistrm128(<16 x i8> %1, <16 x i8> %2, i8 7)
-  ret <16 x i8> %3
-}
-declare <16 x i8> @llvm.x86.sse42.pcmpistrm128(<16 x i8>, <16 x i8>, i8) nounwind readnone
-
-define <2 x i64> @test_pcmpgtq(<2 x i64> %a0, <2 x i64> %a1, <2 x i64> *%a2) {
-; GENERIC-LABEL: test_pcmpgtq:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    pcmpgtq %xmm1, %xmm0 # sched: [5:1.00]
-; GENERIC-NEXT:    pcmpgtq (%rdi), %xmm0 # sched: [11:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SLM-LABEL: test_pcmpgtq:
-; SLM:       # %bb.0:
-; SLM-NEXT:    pcmpgtq %xmm1, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    pcmpgtq (%rdi), %xmm0 # sched: [4:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_pcmpgtq:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    pcmpgtq %xmm1, %xmm0 # sched: [5:1.00]
-; SANDY-SSE-NEXT:    pcmpgtq (%rdi), %xmm0 # sched: [11:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_pcmpgtq:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vpcmpgtq %xmm1, %xmm0, %xmm0 # sched: [5:1.00]
-; SANDY-NEXT:    vpcmpgtq (%rdi), %xmm0, %xmm0 # sched: [11:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_pcmpgtq:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    pcmpgtq %xmm1, %xmm0 # sched: [5:1.00]
-; HASWELL-SSE-NEXT:    pcmpgtq (%rdi), %xmm0 # sched: [11:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_pcmpgtq:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpcmpgtq %xmm1, %xmm0, %xmm0 # sched: [5:1.00]
-; HASWELL-NEXT:    vpcmpgtq (%rdi), %xmm0, %xmm0 # sched: [11:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_pcmpgtq:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    pcmpgtq %xmm1, %xmm0 # sched: [5:1.00]
-; BROADWELL-SSE-NEXT:    pcmpgtq (%rdi), %xmm0 # sched: [10:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pcmpgtq:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpcmpgtq %xmm1, %xmm0, %xmm0 # sched: [5:1.00]
-; BROADWELL-NEXT:    vpcmpgtq (%rdi), %xmm0, %xmm0 # sched: [10:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_pcmpgtq:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    pcmpgtq %xmm1, %xmm0 # sched: [3:1.00]
-; SKYLAKE-SSE-NEXT:    pcmpgtq (%rdi), %xmm0 # sched: [9:1.00]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pcmpgtq:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpcmpgtq %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; SKYLAKE-NEXT:    vpcmpgtq (%rdi), %xmm0, %xmm0 # sched: [9:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_pcmpgtq:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    pcmpgtq %xmm1, %xmm0 # sched: [3:1.00]
-; SKX-SSE-NEXT:    pcmpgtq (%rdi), %xmm0 # sched: [9:1.00]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pcmpgtq:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpcmpgtq %xmm1, %xmm0, %xmm0 # sched: [3:1.00]
-; SKX-NEXT:    vpcmpgtq (%rdi), %xmm0, %xmm0 # sched: [9:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_pcmpgtq:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    pcmpgtq %xmm1, %xmm0 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    pcmpgtq (%rdi), %xmm0 # sched: [7:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_pcmpgtq:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vpcmpgtq %xmm1, %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    vpcmpgtq (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_pcmpgtq:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    pcmpgtq %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    pcmpgtq (%rdi), %xmm0 # sched: [6:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_pcmpgtq:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vpcmpgtq %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    vpcmpgtq (%rdi), %xmm0, %xmm0 # sched: [6:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_pcmpgtq:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    pcmpgtq %xmm1, %xmm0 # sched: [1:0.50]
-; ZNVER1-SSE-NEXT:    pcmpgtq (%rdi), %xmm0 # sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_pcmpgtq:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpcmpgtq %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; ZNVER1-NEXT:    vpcmpgtq (%rdi), %xmm0, %xmm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = icmp sgt <2 x i64> %a0, %a1
-  %2 = sext <2 x i1> %1 to <2 x i64>
-  %3 = load <2 x i64>, <2 x i64>*%a2, align 16
-  %4 = icmp sgt <2 x i64> %2, %3
-  %5 = sext <2 x i1> %4 to <2 x i64>
-  ret <2 x i64> %5
-}
-
-define <2 x i64> @test_pclmulqdq(<2 x i64> %a0, <2 x i64> %a1, <2 x i64> *%a2) {
-; GENERIC-LABEL: test_pclmulqdq:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    pclmulqdq $0, %xmm1, %xmm0 # sched: [14:6.00]
-; GENERIC-NEXT:    pclmulqdq $0, (%rdi), %xmm0 # sched: [14:5.67]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; SLM-LABEL: test_pclmulqdq:
-; SLM:       # %bb.0:
-; SLM-NEXT:    pclmulqdq $0, %xmm1, %xmm0 # sched: [10:10.00]
-; SLM-NEXT:    pclmulqdq $0, (%rdi), %xmm0 # sched: [10:10.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_pclmulqdq:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    pclmulqdq $0, %xmm1, %xmm0 # sched: [14:6.00]
-; SANDY-SSE-NEXT:    pclmulqdq $0, (%rdi), %xmm0 # sched: [14:5.67]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_pclmulqdq:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vpclmulqdq $0, %xmm1, %xmm0, %xmm0 # sched: [14:6.00]
-; SANDY-NEXT:    vpclmulqdq $0, (%rdi), %xmm0, %xmm0 # sched: [14:5.67]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_pclmulqdq:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    pclmulqdq $0, %xmm1, %xmm0 # sched: [11:2.00]
-; HASWELL-SSE-NEXT:    pclmulqdq $0, (%rdi), %xmm0 # sched: [17:2.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_pclmulqdq:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpclmulqdq $0, %xmm1, %xmm0, %xmm0 # sched: [11:2.00]
-; HASWELL-NEXT:    vpclmulqdq $0, (%rdi), %xmm0, %xmm0 # sched: [17:2.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_pclmulqdq:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    pclmulqdq $0, %xmm1, %xmm0 # sched: [5:1.00]
-; BROADWELL-SSE-NEXT:    pclmulqdq $0, (%rdi), %xmm0 # sched: [10:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pclmulqdq:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpclmulqdq $0, %xmm1, %xmm0, %xmm0 # sched: [5:1.00]
-; BROADWELL-NEXT:    vpclmulqdq $0, (%rdi), %xmm0, %xmm0 # sched: [10:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_pclmulqdq:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    pclmulqdq $0, %xmm1, %xmm0 # sched: [6:1.00]
-; SKYLAKE-SSE-NEXT:    pclmulqdq $0, (%rdi), %xmm0 # sched: [12:1.00]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pclmulqdq:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpclmulqdq $0, %xmm1, %xmm0, %xmm0 # sched: [6:1.00]
-; SKYLAKE-NEXT:    vpclmulqdq $0, (%rdi), %xmm0, %xmm0 # sched: [12:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_pclmulqdq:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    pclmulqdq $0, %xmm1, %xmm0 # sched: [6:1.00]
-; SKX-SSE-NEXT:    pclmulqdq $0, (%rdi), %xmm0 # sched: [12:1.00]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pclmulqdq:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpclmulqdq $0, %xmm1, %xmm0, %xmm0 # sched: [6:1.00]
-; SKX-NEXT:    vpclmulqdq $0, (%rdi), %xmm0, %xmm0 # sched: [12:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_pclmulqdq:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    pclmulqdq $0, %xmm1, %xmm0 # sched: [12:1.00]
-; BDVER2-SSE-NEXT:    pclmulqdq $0, (%rdi), %xmm0 # sched: [17:1.00]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_pclmulqdq:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vpclmulqdq $0, %xmm1, %xmm0, %xmm0 # sched: [13:1.00]
-; BDVER2-NEXT:    vpclmulqdq $0, (%rdi), %xmm0, %xmm0 # sched: [17:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_pclmulqdq:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    pclmulqdq $0, %xmm1, %xmm0 # sched: [2:1.00]
-; BTVER2-SSE-NEXT:    pclmulqdq $0, (%rdi), %xmm0 # sched: [7:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_pclmulqdq:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vpclmulqdq $0, %xmm1, %xmm0, %xmm0 # sched: [2:1.00]
-; BTVER2-NEXT:    vpclmulqdq $0, (%rdi), %xmm0, %xmm0 # sched: [7:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_pclmulqdq:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    pclmulqdq $0, %xmm1, %xmm0 # sched: [100:0.25]
-; ZNVER1-SSE-NEXT:    pclmulqdq $0, (%rdi), %xmm0 # sched: [100:0.25]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_pclmulqdq:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpclmulqdq $0, %xmm1, %xmm0, %xmm0 # sched: [100:0.25]
-; ZNVER1-NEXT:    vpclmulqdq $0, (%rdi), %xmm0, %xmm0 # sched: [100:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = load <2 x i64>, <2 x i64> *%a2, align 16
-  %2 = call <2 x i64> @llvm.x86.pclmulqdq(<2 x i64> %a0, <2 x i64> %a1, i8 0)
-  %3 = call <2 x i64> @llvm.x86.pclmulqdq(<2 x i64> %1, <2 x i64> %2, i8 0)
-  ret <2 x i64> %3
-}
-declare <2 x i64> @llvm.x86.pclmulqdq(<2 x i64>, <2 x i64>, i8)
diff --git a/test/CodeGen/X86/sse4a-schedule.ll b/test/CodeGen/X86/sse4a-schedule.ll
deleted file mode 100644
index ad76845a73d26c9adda4d3501d3d2b9b1913e8c5..0000000000000000000000000000000000000000
--- a/test/CodeGen/X86/sse4a-schedule.ll
+++ /dev/null
@@ -1,156 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=x86-64 -mattr=+sse4a | FileCheck %s --check-prefix=GENERIC
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=bdver2 | FileCheck %s --check-prefix=BDVER2
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=btver2 | FileCheck %s --check-prefix=BTVER2
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=znver1 | FileCheck %s --check-prefix=ZNVER1
-
-define <2 x i64> @test_extrq(<2 x i64> %a0, <16 x i8> %a1) {
-; GENERIC-LABEL: test_extrq:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    extrq %xmm1, %xmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER2-LABEL: test_extrq:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    extrq %xmm1, %xmm0 # sched: [3:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_extrq:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    extrq %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_extrq:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    extrq %xmm1, %xmm0 # sched: [2:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = tail call <2 x i64> @llvm.x86.sse4a.extrq(<2 x i64> %a0, <16 x i8> %a1)
-  ret <2 x i64> %1
-}
-declare <2 x i64> @llvm.x86.sse4a.extrq(<2 x i64>, <16 x i8>)
-
-define <2 x i64> @test_extrqi(<2 x i64> %a0) {
-; GENERIC-LABEL: test_extrqi:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    extrq $2, $3, %xmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER2-LABEL: test_extrqi:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    extrq $2, $3, %xmm0 # sched: [3:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_extrqi:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    extrq $2, $3, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_extrqi:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    extrq $2, $3, %xmm0 # sched: [2:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = tail call <2 x i64> @llvm.x86.sse4a.extrqi(<2 x i64> %a0, i8 3, i8 2)
-  ret <2 x i64> %1
-}
-declare <2 x i64> @llvm.x86.sse4a.extrqi(<2 x i64>, i8, i8)
-
-define <2 x i64> @test_insertq(<2 x i64> %a0, <2 x i64> %a1) {
-; GENERIC-LABEL: test_insertq:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    insertq %xmm1, %xmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER2-LABEL: test_insertq:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    insertq %xmm1, %xmm0 # sched: [3:2.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_insertq:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    insertq %xmm1, %xmm0 # sched: [2:2.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_insertq:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    insertq %xmm1, %xmm0 # sched: [4:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = tail call <2 x i64> @llvm.x86.sse4a.insertq(<2 x i64> %a0, <2 x i64> %a1)
-  ret <2 x i64> %1
-}
-declare <2 x i64> @llvm.x86.sse4a.insertq(<2 x i64>, <2 x i64>)
-
-define <2 x i64> @test_insertqi(<2 x i64> %a0, <2 x i64> %a1) {
-; GENERIC-LABEL: test_insertqi:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    insertq $6, $5, %xmm1, %xmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER2-LABEL: test_insertqi:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    insertq $6, $5, %xmm1, %xmm0 # sched: [3:2.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_insertqi:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    insertq $6, $5, %xmm1, %xmm0 # sched: [2:2.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_insertqi:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    insertq $6, $5, %xmm1, %xmm0 # sched: [4:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = tail call <2 x i64> @llvm.x86.sse4a.insertqi(<2 x i64> %a0, <2 x i64> %a1, i8 5, i8 6)
-  ret <2 x i64> %1
-}
-declare <2 x i64> @llvm.x86.sse4a.insertqi(<2 x i64>, <2 x i64>, i8, i8)
-
-define void @test_movntsd(i8* %p, <2 x double> %a) {
-; GENERIC-LABEL: test_movntsd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    movntsd %xmm0, (%rdi) # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER2-LABEL: test_movntsd:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    movntsd %xmm0, (%rdi) # sched: [3:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_movntsd:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    movntsd %xmm0, (%rdi) # sched: [3:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_movntsd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    movntsd %xmm0, (%rdi) # sched: [8:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  tail call void @llvm.x86.sse4a.movnt.sd(i8* %p, <2 x double> %a)
-  ret void
-}
-declare void @llvm.x86.sse4a.movnt.sd(i8*, <2 x double>)
-
-define void @test_movntss(i8* %p, <4 x float> %a) {
-; GENERIC-LABEL: test_movntss:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    movntss %xmm0, (%rdi) # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER2-LABEL: test_movntss:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    movntss %xmm0, (%rdi) # sched: [3:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_movntss:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    movntss %xmm0, (%rdi) # sched: [3:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_movntss:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    movntss %xmm0, (%rdi) # sched: [8:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  tail call void @llvm.x86.sse4a.movnt.ss(i8* %p, <4 x float> %a)
-  ret void
-}
-declare void @llvm.x86.sse4a.movnt.ss(i8*, <4 x float>)
-
diff --git a/test/CodeGen/X86/ssp-data-layout.ll b/test/CodeGen/X86/ssp-data-layout.ll
index e954d9c1042a827fb97590a9fe976a732d0ef999..409dd7cb853242def7f4990572356c2e5f5e64dd 100644
--- a/test/CodeGen/X86/ssp-data-layout.ll
+++ b/test/CodeGen/X86/ssp-data-layout.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -stack-symbol-ordering=0 -disable-fp-elim -mtriple=x86_64-pc-linux-gnu -mcpu=corei7 -o - | FileCheck %s
+; RUN: llc < %s -stack-symbol-ordering=0 -frame-pointer=all -mtriple=x86_64-pc-linux-gnu -mcpu=corei7 -o - | FileCheck %s
 ;  This test is fairly fragile.  The goal is to ensure that "large" stack
 ;  objects are allocated closest to the stack protector (i.e., farthest away 
 ;  from the Stack Pointer.)  In standard SSP mode this means that large (>=
@@ -11,7 +11,7 @@
 ;  and that the groups have the correct relative stack offset.  The ordering
 ;  within a group is not relevant to this test.  Unfortunately, there is not
 ;  an elegant way to do this, so just match the offset for each object.
-; RUN: llc < %s -disable-fp-elim -mtriple=x86_64-unknown-unknown -O0 -mcpu=corei7 -o - \
+; RUN: llc < %s -frame-pointer=all -mtriple=x86_64-unknown-unknown -O0 -mcpu=corei7 -o - \
 ; RUN:   | FileCheck --check-prefix=FAST-NON-LIN %s
 ; FastISel was not setting the StackProtectorIndex when lowering
 ; Intrinsic::stackprotector and as a result the stack re-arrangement code was
diff --git a/test/CodeGen/X86/ssse3-schedule.ll b/test/CodeGen/X86/ssse3-schedule.ll
deleted file mode 100644
index 5c8bd2dc843ea91e89d06f16bdf980f8cd50c303..0000000000000000000000000000000000000000
--- a/test/CodeGen/X86/ssse3-schedule.ll
+++ /dev/null
@@ -1,2049 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=x86-64 -mattr=+ssse3 | FileCheck %s --check-prefixes=CHECK,GENERIC
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=atom | FileCheck %s --check-prefixes=CHECK,ATOM
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=slm -mattr=-sse4.1 | FileCheck %s --check-prefixes=CHECK,SLM
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=sandybridge -mattr=-sse4.1 | FileCheck %s --check-prefixes=CHECK,SANDY-SSE
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=sandybridge -mattr=-avx2   | FileCheck %s --check-prefixes=CHECK,SANDY
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=ivybridge -mattr=-sse4.1 | FileCheck %s --check-prefixes=CHECK,SANDY-SSE
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=ivybridge -mattr=-avx2   | FileCheck %s --check-prefixes=CHECK,SANDY
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=haswell -mattr=-sse4.1 | FileCheck %s --check-prefixes=CHECK,HASWELL-SSE
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=haswell -mattr=-avx2   | FileCheck %s --check-prefixes=CHECK,HASWELL
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=broadwell -mattr=-sse4.1 | FileCheck %s --check-prefixes=CHECK,BROADWELL-SSE
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=broadwell -mattr=-avx2   | FileCheck %s --check-prefixes=CHECK,BROADWELL
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=skylake -mattr=-sse4.1 | FileCheck %s --check-prefixes=CHECK,SKYLAKE-SSE
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=skylake -mattr=-avx2   | FileCheck %s --check-prefixes=CHECK,SKYLAKE
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=skx -mattr=-sse4.1 | FileCheck %s --check-prefixes=CHECK,SKX-SSE
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=skx -mattr=-avx2   | FileCheck %s --check-prefixes=CHECK,SKX
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=bdver2 -mattr=-sse4.1 | FileCheck %s --check-prefixes=CHECK,BDVER2-SSE
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=bdver2 -mattr=-avx2   | FileCheck %s --check-prefixes=CHECK,BDVER2
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=btver2 -mattr=-sse4.1 | FileCheck %s --check-prefixes=CHECK,BTVER2-SSE
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=btver2 -mattr=-avx2   | FileCheck %s --check-prefixes=CHECK,BTVER2
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=znver1 -mattr=-sse4.1 | FileCheck %s --check-prefixes=CHECK,ZNVER1-SSE
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=znver1 -mattr=-avx2   | FileCheck %s --check-prefixes=CHECK,ZNVER1
-
-define <16 x i8> @test_pabsb(<16 x i8> %a0, <16 x i8> *%a1) {
-; GENERIC-LABEL: test_pabsb:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    pabsb %xmm0, %xmm1 # sched: [1:0.50]
-; GENERIC-NEXT:    pabsb (%rdi), %xmm0 # sched: [7:0.50]
-; GENERIC-NEXT:    por %xmm1, %xmm0 # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_pabsb:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    pabsb %xmm0, %xmm1 # sched: [1:0.50]
-; ATOM-NEXT:    pabsb (%rdi), %xmm0 # sched: [1:1.00]
-; ATOM-NEXT:    por %xmm1, %xmm0 # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_pabsb:
-; SLM:       # %bb.0:
-; SLM-NEXT:    pabsb %xmm0, %xmm1 # sched: [1:0.50]
-; SLM-NEXT:    pabsb (%rdi), %xmm0 # sched: [4:1.00]
-; SLM-NEXT:    por %xmm1, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_pabsb:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    pabsb %xmm0, %xmm1 # sched: [1:0.50]
-; SANDY-SSE-NEXT:    pabsb (%rdi), %xmm0 # sched: [7:0.50]
-; SANDY-SSE-NEXT:    por %xmm1, %xmm0 # sched: [1:0.33]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_pabsb:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vpabsb %xmm0, %xmm0 # sched: [1:0.50]
-; SANDY-NEXT:    vpabsb (%rdi), %xmm1 # sched: [7:0.50]
-; SANDY-NEXT:    vpor %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_pabsb:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    pabsb %xmm0, %xmm1 # sched: [1:0.50]
-; HASWELL-SSE-NEXT:    pabsb (%rdi), %xmm0 # sched: [7:0.50]
-; HASWELL-SSE-NEXT:    por %xmm1, %xmm0 # sched: [1:0.33]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_pabsb:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpabsb %xmm0, %xmm0 # sched: [1:0.50]
-; HASWELL-NEXT:    vpabsb (%rdi), %xmm1 # sched: [7:0.50]
-; HASWELL-NEXT:    vpor %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_pabsb:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    pabsb %xmm0, %xmm1 # sched: [1:0.50]
-; BROADWELL-SSE-NEXT:    pabsb (%rdi), %xmm0 # sched: [6:0.50]
-; BROADWELL-SSE-NEXT:    por %xmm1, %xmm0 # sched: [1:0.33]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pabsb:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpabsb %xmm0, %xmm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    vpabsb (%rdi), %xmm1 # sched: [6:0.50]
-; BROADWELL-NEXT:    vpor %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_pabsb:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    pabsb %xmm0, %xmm1 # sched: [1:0.50]
-; SKYLAKE-SSE-NEXT:    pabsb (%rdi), %xmm0 # sched: [7:0.50]
-; SKYLAKE-SSE-NEXT:    por %xmm1, %xmm0 # sched: [1:0.33]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pabsb:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpabsb %xmm0, %xmm0 # sched: [1:0.50]
-; SKYLAKE-NEXT:    vpabsb (%rdi), %xmm1 # sched: [7:0.50]
-; SKYLAKE-NEXT:    vpor %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_pabsb:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    pabsb %xmm0, %xmm1 # sched: [1:0.50]
-; SKX-SSE-NEXT:    pabsb (%rdi), %xmm0 # sched: [7:0.50]
-; SKX-SSE-NEXT:    por %xmm1, %xmm0 # sched: [1:0.33]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pabsb:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpabsb %xmm0, %xmm0 # sched: [1:0.50]
-; SKX-NEXT:    vpabsb (%rdi), %xmm1 # sched: [7:0.50]
-; SKX-NEXT:    vpor %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_pabsb:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    pabsb %xmm0, %xmm1 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    pabsb (%rdi), %xmm0 # sched: [7:0.50]
-; BDVER2-SSE-NEXT:    por %xmm1, %xmm0 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_pabsb:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vpabsb (%rdi), %xmm1 # sched: [7:0.50]
-; BDVER2-NEXT:    vpabsb %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    vpor %xmm1, %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_pabsb:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    pabsb %xmm0, %xmm1 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    pabsb (%rdi), %xmm0 # sched: [6:1.00]
-; BTVER2-SSE-NEXT:    por %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_pabsb:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vpabsb (%rdi), %xmm1 # sched: [6:1.00]
-; BTVER2-NEXT:    vpabsb %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    vpor %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_pabsb:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    pabsb %xmm0, %xmm1 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    pabsb (%rdi), %xmm0 # sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    por %xmm1, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_pabsb:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpabsb (%rdi), %xmm1 # sched: [8:0.50]
-; ZNVER1-NEXT:    vpabsb %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vpor %xmm1, %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <16 x i8> @llvm.x86.ssse3.pabs.b.128(<16 x i8> %a0)
-  %2 = load <16 x i8>, <16 x i8> *%a1, align 16
-  %3 = call <16 x i8> @llvm.x86.ssse3.pabs.b.128(<16 x i8> %2)
-  %4 = or <16 x i8> %1, %3
-  ret <16 x i8> %4
-}
-declare <16 x i8> @llvm.x86.ssse3.pabs.b.128(<16 x i8>) nounwind readnone
-
-define <4 x i32> @test_pabsd(<4 x i32> %a0, <4 x i32> *%a1) {
-; GENERIC-LABEL: test_pabsd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    pabsd %xmm0, %xmm1 # sched: [1:0.50]
-; GENERIC-NEXT:    pabsd (%rdi), %xmm0 # sched: [7:0.50]
-; GENERIC-NEXT:    por %xmm1, %xmm0 # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_pabsd:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    pabsd %xmm0, %xmm1 # sched: [1:0.50]
-; ATOM-NEXT:    pabsd (%rdi), %xmm0 # sched: [1:1.00]
-; ATOM-NEXT:    por %xmm1, %xmm0 # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_pabsd:
-; SLM:       # %bb.0:
-; SLM-NEXT:    pabsd %xmm0, %xmm1 # sched: [1:0.50]
-; SLM-NEXT:    pabsd (%rdi), %xmm0 # sched: [4:1.00]
-; SLM-NEXT:    por %xmm1, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_pabsd:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    pabsd %xmm0, %xmm1 # sched: [1:0.50]
-; SANDY-SSE-NEXT:    pabsd (%rdi), %xmm0 # sched: [7:0.50]
-; SANDY-SSE-NEXT:    por %xmm1, %xmm0 # sched: [1:0.33]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_pabsd:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vpabsd %xmm0, %xmm0 # sched: [1:0.50]
-; SANDY-NEXT:    vpabsd (%rdi), %xmm1 # sched: [7:0.50]
-; SANDY-NEXT:    vpor %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_pabsd:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    pabsd %xmm0, %xmm1 # sched: [1:0.50]
-; HASWELL-SSE-NEXT:    pabsd (%rdi), %xmm0 # sched: [7:0.50]
-; HASWELL-SSE-NEXT:    por %xmm1, %xmm0 # sched: [1:0.33]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_pabsd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpabsd %xmm0, %xmm0 # sched: [1:0.50]
-; HASWELL-NEXT:    vpabsd (%rdi), %xmm1 # sched: [7:0.50]
-; HASWELL-NEXT:    vpor %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_pabsd:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    pabsd %xmm0, %xmm1 # sched: [1:0.50]
-; BROADWELL-SSE-NEXT:    pabsd (%rdi), %xmm0 # sched: [6:0.50]
-; BROADWELL-SSE-NEXT:    por %xmm1, %xmm0 # sched: [1:0.33]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pabsd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpabsd %xmm0, %xmm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    vpabsd (%rdi), %xmm1 # sched: [6:0.50]
-; BROADWELL-NEXT:    vpor %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_pabsd:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    pabsd %xmm0, %xmm1 # sched: [1:0.50]
-; SKYLAKE-SSE-NEXT:    pabsd (%rdi), %xmm0 # sched: [7:0.50]
-; SKYLAKE-SSE-NEXT:    por %xmm1, %xmm0 # sched: [1:0.33]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pabsd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpabsd %xmm0, %xmm0 # sched: [1:0.50]
-; SKYLAKE-NEXT:    vpabsd (%rdi), %xmm1 # sched: [7:0.50]
-; SKYLAKE-NEXT:    vpor %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_pabsd:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    pabsd %xmm0, %xmm1 # sched: [1:0.50]
-; SKX-SSE-NEXT:    pabsd (%rdi), %xmm0 # sched: [7:0.50]
-; SKX-SSE-NEXT:    por %xmm1, %xmm0 # sched: [1:0.33]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pabsd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpabsd %xmm0, %xmm0 # sched: [1:0.50]
-; SKX-NEXT:    vpabsd (%rdi), %xmm1 # sched: [7:0.50]
-; SKX-NEXT:    vpor %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_pabsd:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    pabsd %xmm0, %xmm1 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    pabsd (%rdi), %xmm0 # sched: [7:0.50]
-; BDVER2-SSE-NEXT:    por %xmm1, %xmm0 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_pabsd:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vpabsd (%rdi), %xmm1 # sched: [7:0.50]
-; BDVER2-NEXT:    vpabsd %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    vpor %xmm1, %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_pabsd:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    pabsd %xmm0, %xmm1 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    pabsd (%rdi), %xmm0 # sched: [6:1.00]
-; BTVER2-SSE-NEXT:    por %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_pabsd:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vpabsd (%rdi), %xmm1 # sched: [6:1.00]
-; BTVER2-NEXT:    vpabsd %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    vpor %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_pabsd:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    pabsd %xmm0, %xmm1 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    pabsd (%rdi), %xmm0 # sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    por %xmm1, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_pabsd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpabsd (%rdi), %xmm1 # sched: [8:0.50]
-; ZNVER1-NEXT:    vpabsd %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vpor %xmm1, %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <4 x i32> @llvm.x86.ssse3.pabs.d.128(<4 x i32> %a0)
-  %2 = load <4 x i32>, <4 x i32> *%a1, align 16
-  %3 = call <4 x i32> @llvm.x86.ssse3.pabs.d.128(<4 x i32> %2)
-  %4 = or <4 x i32> %1, %3
-  ret <4 x i32> %4
-}
-declare <4 x i32> @llvm.x86.ssse3.pabs.d.128(<4 x i32>) nounwind readnone
-
-define <8 x i16> @test_pabsw(<8 x i16> %a0, <8 x i16> *%a1) {
-; GENERIC-LABEL: test_pabsw:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    pabsw %xmm0, %xmm1 # sched: [1:0.50]
-; GENERIC-NEXT:    pabsw (%rdi), %xmm0 # sched: [7:0.50]
-; GENERIC-NEXT:    por %xmm1, %xmm0 # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_pabsw:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    pabsw %xmm0, %xmm1 # sched: [1:0.50]
-; ATOM-NEXT:    pabsw (%rdi), %xmm0 # sched: [1:1.00]
-; ATOM-NEXT:    por %xmm1, %xmm0 # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_pabsw:
-; SLM:       # %bb.0:
-; SLM-NEXT:    pabsw %xmm0, %xmm1 # sched: [1:0.50]
-; SLM-NEXT:    pabsw (%rdi), %xmm0 # sched: [4:1.00]
-; SLM-NEXT:    por %xmm1, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_pabsw:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    pabsw %xmm0, %xmm1 # sched: [1:0.50]
-; SANDY-SSE-NEXT:    pabsw (%rdi), %xmm0 # sched: [7:0.50]
-; SANDY-SSE-NEXT:    por %xmm1, %xmm0 # sched: [1:0.33]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_pabsw:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vpabsw %xmm0, %xmm0 # sched: [1:0.50]
-; SANDY-NEXT:    vpabsw (%rdi), %xmm1 # sched: [7:0.50]
-; SANDY-NEXT:    vpor %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_pabsw:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    pabsw %xmm0, %xmm1 # sched: [1:0.50]
-; HASWELL-SSE-NEXT:    pabsw (%rdi), %xmm0 # sched: [7:0.50]
-; HASWELL-SSE-NEXT:    por %xmm1, %xmm0 # sched: [1:0.33]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_pabsw:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpabsw %xmm0, %xmm0 # sched: [1:0.50]
-; HASWELL-NEXT:    vpabsw (%rdi), %xmm1 # sched: [7:0.50]
-; HASWELL-NEXT:    vpor %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_pabsw:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    pabsw %xmm0, %xmm1 # sched: [1:0.50]
-; BROADWELL-SSE-NEXT:    pabsw (%rdi), %xmm0 # sched: [6:0.50]
-; BROADWELL-SSE-NEXT:    por %xmm1, %xmm0 # sched: [1:0.33]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pabsw:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpabsw %xmm0, %xmm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    vpabsw (%rdi), %xmm1 # sched: [6:0.50]
-; BROADWELL-NEXT:    vpor %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_pabsw:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    pabsw %xmm0, %xmm1 # sched: [1:0.50]
-; SKYLAKE-SSE-NEXT:    pabsw (%rdi), %xmm0 # sched: [7:0.50]
-; SKYLAKE-SSE-NEXT:    por %xmm1, %xmm0 # sched: [1:0.33]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pabsw:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpabsw %xmm0, %xmm0 # sched: [1:0.50]
-; SKYLAKE-NEXT:    vpabsw (%rdi), %xmm1 # sched: [7:0.50]
-; SKYLAKE-NEXT:    vpor %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_pabsw:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    pabsw %xmm0, %xmm1 # sched: [1:0.50]
-; SKX-SSE-NEXT:    pabsw (%rdi), %xmm0 # sched: [7:0.50]
-; SKX-SSE-NEXT:    por %xmm1, %xmm0 # sched: [1:0.33]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pabsw:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpabsw %xmm0, %xmm0 # sched: [1:0.50]
-; SKX-NEXT:    vpabsw (%rdi), %xmm1 # sched: [7:0.50]
-; SKX-NEXT:    vpor %xmm1, %xmm0, %xmm0 # sched: [1:0.33]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_pabsw:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    pabsw %xmm0, %xmm1 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    pabsw (%rdi), %xmm0 # sched: [7:0.50]
-; BDVER2-SSE-NEXT:    por %xmm1, %xmm0 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_pabsw:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vpabsw (%rdi), %xmm1 # sched: [7:0.50]
-; BDVER2-NEXT:    vpabsw %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    vpor %xmm1, %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_pabsw:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    pabsw %xmm0, %xmm1 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    pabsw (%rdi), %xmm0 # sched: [6:1.00]
-; BTVER2-SSE-NEXT:    por %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_pabsw:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vpabsw (%rdi), %xmm1 # sched: [6:1.00]
-; BTVER2-NEXT:    vpabsw %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    vpor %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_pabsw:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    pabsw %xmm0, %xmm1 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    pabsw (%rdi), %xmm0 # sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    por %xmm1, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_pabsw:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpabsw (%rdi), %xmm1 # sched: [8:0.50]
-; ZNVER1-NEXT:    vpabsw %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vpor %xmm1, %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <8 x i16> @llvm.x86.ssse3.pabs.w.128(<8 x i16> %a0)
-  %2 = load <8 x i16>, <8 x i16> *%a1, align 16
-  %3 = call <8 x i16> @llvm.x86.ssse3.pabs.w.128(<8 x i16> %2)
-  %4 = or <8 x i16> %1, %3
-  ret <8 x i16> %4
-}
-declare <8 x i16> @llvm.x86.ssse3.pabs.w.128(<8 x i16>) nounwind readnone
-
-define <8 x i16> @test_palignr(<8 x i16> %a0, <8 x i16> %a1, <8 x i16> *%a2) {
-; GENERIC-LABEL: test_palignr:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    palignr {{.*#+}} xmm1 = xmm0[6,7,8,9,10,11,12,13,14,15],xmm1[0,1,2,3,4,5] sched: [1:0.50]
-; GENERIC-NEXT:    palignr {{.*#+}} xmm1 = mem[14,15],xmm1[0,1,2,3,4,5,6,7,8,9,10,11,12,13] sched: [7:0.50]
-; GENERIC-NEXT:    movdqa %xmm1, %xmm0 # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_palignr:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    palignr {{.*#+}} xmm1 = xmm0[6,7,8,9,10,11,12,13,14,15],xmm1[0,1,2,3,4,5] sched: [1:1.00]
-; ATOM-NEXT:    palignr {{.*#+}} xmm1 = mem[14,15],xmm1[0,1,2,3,4,5,6,7,8,9,10,11,12,13] sched: [1:1.00]
-; ATOM-NEXT:    movdqa %xmm1, %xmm0 # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_palignr:
-; SLM:       # %bb.0:
-; SLM-NEXT:    palignr {{.*#+}} xmm1 = xmm0[6,7,8,9,10,11,12,13,14,15],xmm1[0,1,2,3,4,5] sched: [1:1.00]
-; SLM-NEXT:    palignr {{.*#+}} xmm1 = mem[14,15],xmm1[0,1,2,3,4,5,6,7,8,9,10,11,12,13] sched: [4:1.00]
-; SLM-NEXT:    movdqa %xmm1, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_palignr:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    palignr {{.*#+}} xmm1 = xmm0[6,7,8,9,10,11,12,13,14,15],xmm1[0,1,2,3,4,5] sched: [1:0.50]
-; SANDY-SSE-NEXT:    palignr {{.*#+}} xmm1 = mem[14,15],xmm1[0,1,2,3,4,5,6,7,8,9,10,11,12,13] sched: [7:0.50]
-; SANDY-SSE-NEXT:    movdqa %xmm1, %xmm0 # sched: [1:0.33]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_palignr:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vpalignr {{.*#+}} xmm0 = xmm0[6,7,8,9,10,11,12,13,14,15],xmm1[0,1,2,3,4,5] sched: [1:0.50]
-; SANDY-NEXT:    vpalignr {{.*#+}} xmm0 = mem[14,15],xmm0[0,1,2,3,4,5,6,7,8,9,10,11,12,13] sched: [7:0.50]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_palignr:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    palignr {{.*#+}} xmm1 = xmm0[6,7,8,9,10,11,12,13,14,15],xmm1[0,1,2,3,4,5] sched: [1:1.00]
-; HASWELL-SSE-NEXT:    palignr {{.*#+}} xmm1 = mem[14,15],xmm1[0,1,2,3,4,5,6,7,8,9,10,11,12,13] sched: [7:1.00]
-; HASWELL-SSE-NEXT:    movdqa %xmm1, %xmm0 # sched: [1:0.33]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_palignr:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpalignr {{.*#+}} xmm0 = xmm0[6,7,8,9,10,11,12,13,14,15],xmm1[0,1,2,3,4,5] sched: [1:1.00]
-; HASWELL-NEXT:    vpalignr {{.*#+}} xmm0 = mem[14,15],xmm0[0,1,2,3,4,5,6,7,8,9,10,11,12,13] sched: [7:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_palignr:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    palignr {{.*#+}} xmm1 = xmm0[6,7,8,9,10,11,12,13,14,15],xmm1[0,1,2,3,4,5] sched: [1:1.00]
-; BROADWELL-SSE-NEXT:    palignr {{.*#+}} xmm1 = mem[14,15],xmm1[0,1,2,3,4,5,6,7,8,9,10,11,12,13] sched: [6:1.00]
-; BROADWELL-SSE-NEXT:    movdqa %xmm1, %xmm0 # sched: [1:0.33]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_palignr:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpalignr {{.*#+}} xmm0 = xmm0[6,7,8,9,10,11,12,13,14,15],xmm1[0,1,2,3,4,5] sched: [1:1.00]
-; BROADWELL-NEXT:    vpalignr {{.*#+}} xmm0 = mem[14,15],xmm0[0,1,2,3,4,5,6,7,8,9,10,11,12,13] sched: [6:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_palignr:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    palignr {{.*#+}} xmm1 = xmm0[6,7,8,9,10,11,12,13,14,15],xmm1[0,1,2,3,4,5] sched: [1:1.00]
-; SKYLAKE-SSE-NEXT:    palignr {{.*#+}} xmm1 = mem[14,15],xmm1[0,1,2,3,4,5,6,7,8,9,10,11,12,13] sched: [7:1.00]
-; SKYLAKE-SSE-NEXT:    movdqa %xmm1, %xmm0 # sched: [1:0.33]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_palignr:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpalignr {{.*#+}} xmm0 = xmm0[6,7,8,9,10,11,12,13,14,15],xmm1[0,1,2,3,4,5] sched: [1:1.00]
-; SKYLAKE-NEXT:    vpalignr {{.*#+}} xmm0 = mem[14,15],xmm0[0,1,2,3,4,5,6,7,8,9,10,11,12,13] sched: [7:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_palignr:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    palignr {{.*#+}} xmm1 = xmm0[6,7,8,9,10,11,12,13,14,15],xmm1[0,1,2,3,4,5] sched: [1:1.00]
-; SKX-SSE-NEXT:    palignr {{.*#+}} xmm1 = mem[14,15],xmm1[0,1,2,3,4,5,6,7,8,9,10,11,12,13] sched: [7:1.00]
-; SKX-SSE-NEXT:    movdqa %xmm1, %xmm0 # sched: [1:0.33]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_palignr:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpalignr {{.*#+}} xmm0 = xmm0[6,7,8,9,10,11,12,13,14,15],xmm1[0,1,2,3,4,5] sched: [1:1.00]
-; SKX-NEXT:    vpalignr {{.*#+}} xmm0 = mem[14,15],xmm0[0,1,2,3,4,5,6,7,8,9,10,11,12,13] sched: [7:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_palignr:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    palignr {{.*#+}} xmm1 = xmm0[6,7,8,9,10,11,12,13,14,15],xmm1[0,1,2,3,4,5] sched: [2:0.50]
-; BDVER2-SSE-NEXT:    palignr {{.*#+}} xmm1 = mem[14,15],xmm1[0,1,2,3,4,5,6,7,8,9,10,11,12,13] sched: [7:0.50]
-; BDVER2-SSE-NEXT:    movdqa %xmm1, %xmm0 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_palignr:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vpalignr {{.*#+}} xmm0 = xmm0[6,7,8,9,10,11,12,13,14,15],xmm1[0,1,2,3,4,5] sched: [2:0.50]
-; BDVER2-NEXT:    vpalignr {{.*#+}} xmm0 = mem[14,15],xmm0[0,1,2,3,4,5,6,7,8,9,10,11,12,13] sched: [7:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_palignr:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    palignr {{.*#+}} xmm1 = xmm0[6,7,8,9,10,11,12,13,14,15],xmm1[0,1,2,3,4,5] sched: [1:0.50]
-; BTVER2-SSE-NEXT:    palignr {{.*#+}} xmm1 = mem[14,15],xmm1[0,1,2,3,4,5,6,7,8,9,10,11,12,13] sched: [6:1.00]
-; BTVER2-SSE-NEXT:    movdqa %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_palignr:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vpalignr {{.*#+}} xmm0 = xmm0[6,7,8,9,10,11,12,13,14,15],xmm1[0,1,2,3,4,5] sched: [1:0.50]
-; BTVER2-NEXT:    vpalignr {{.*#+}} xmm0 = mem[14,15],xmm0[0,1,2,3,4,5,6,7,8,9,10,11,12,13] sched: [6:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_palignr:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    palignr {{.*#+}} xmm1 = xmm0[6,7,8,9,10,11,12,13,14,15],xmm1[0,1,2,3,4,5] sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    palignr {{.*#+}} xmm1 = mem[14,15],xmm1[0,1,2,3,4,5,6,7,8,9,10,11,12,13] sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    movdqa %xmm1, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_palignr:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpalignr {{.*#+}} xmm0 = xmm0[6,7,8,9,10,11,12,13,14,15],xmm1[0,1,2,3,4,5] sched: [1:0.25]
-; ZNVER1-NEXT:    vpalignr {{.*#+}} xmm0 = mem[14,15],xmm0[0,1,2,3,4,5,6,7,8,9,10,11,12,13] sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = shufflevector <8 x i16> %a0, <8 x i16> %a1, <8 x i32> <i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10>
-  %2 = load <8 x i16>, <8 x i16> *%a2, align 16
-  %3 = shufflevector <8 x i16> %2, <8 x i16> %1, <8 x i32> <i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14>
-  ret <8 x i16> %3
-}
-
-define <4 x i32> @test_phaddd(<4 x i32> %a0, <4 x i32> %a1, <4 x i32> *%a2) {
-; GENERIC-LABEL: test_phaddd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    phaddd %xmm1, %xmm0 # sched: [3:1.50]
-; GENERIC-NEXT:    phaddd (%rdi), %xmm0 # sched: [9:1.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_phaddd:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    phaddd %xmm1, %xmm0 # sched: [3:1.50]
-; ATOM-NEXT:    phaddd (%rdi), %xmm0 # sched: [4:2.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_phaddd:
-; SLM:       # %bb.0:
-; SLM-NEXT:    phaddd %xmm1, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    phaddd (%rdi), %xmm0 # sched: [4:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_phaddd:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    phaddd %xmm1, %xmm0 # sched: [3:1.50]
-; SANDY-SSE-NEXT:    phaddd (%rdi), %xmm0 # sched: [9:1.50]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_phaddd:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vphaddd %xmm1, %xmm0, %xmm0 # sched: [3:1.50]
-; SANDY-NEXT:    vphaddd (%rdi), %xmm0, %xmm0 # sched: [9:1.50]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_phaddd:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    phaddd %xmm1, %xmm0 # sched: [3:2.00]
-; HASWELL-SSE-NEXT:    phaddd (%rdi), %xmm0 # sched: [9:2.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_phaddd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vphaddd %xmm1, %xmm0, %xmm0 # sched: [3:2.00]
-; HASWELL-NEXT:    vphaddd (%rdi), %xmm0, %xmm0 # sched: [9:2.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_phaddd:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    phaddd %xmm1, %xmm0 # sched: [3:2.00]
-; BROADWELL-SSE-NEXT:    phaddd (%rdi), %xmm0 # sched: [8:2.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_phaddd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vphaddd %xmm1, %xmm0, %xmm0 # sched: [3:2.00]
-; BROADWELL-NEXT:    vphaddd (%rdi), %xmm0, %xmm0 # sched: [8:2.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_phaddd:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    phaddd %xmm1, %xmm0 # sched: [3:2.00]
-; SKYLAKE-SSE-NEXT:    phaddd (%rdi), %xmm0 # sched: [9:2.00]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_phaddd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vphaddd %xmm1, %xmm0, %xmm0 # sched: [3:2.00]
-; SKYLAKE-NEXT:    vphaddd (%rdi), %xmm0, %xmm0 # sched: [9:2.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_phaddd:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    phaddd %xmm1, %xmm0 # sched: [3:2.00]
-; SKX-SSE-NEXT:    phaddd (%rdi), %xmm0 # sched: [9:2.00]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_phaddd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vphaddd %xmm1, %xmm0, %xmm0 # sched: [3:2.00]
-; SKX-NEXT:    vphaddd (%rdi), %xmm0, %xmm0 # sched: [9:2.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_phaddd:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    phaddd %xmm1, %xmm0 # sched: [5:0.50]
-; BDVER2-SSE-NEXT:    phaddd (%rdi), %xmm0 # sched: [10:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_phaddd:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vphaddd %xmm1, %xmm0, %xmm0 # sched: [5:0.50]
-; BDVER2-NEXT:    vphaddd (%rdi), %xmm0, %xmm0 # sched: [10:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_phaddd:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    phaddd %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    phaddd (%rdi), %xmm0 # sched: [6:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_phaddd:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vphaddd %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    vphaddd (%rdi), %xmm0, %xmm0 # sched: [6:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_phaddd:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    phaddd %xmm1, %xmm0 # sched: [100:0.25]
-; ZNVER1-SSE-NEXT:    phaddd (%rdi), %xmm0 # sched: [100:0.25]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_phaddd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vphaddd %xmm1, %xmm0, %xmm0 # sched: [100:0.25]
-; ZNVER1-NEXT:    vphaddd (%rdi), %xmm0, %xmm0 # sched: [100:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <4 x i32> @llvm.x86.ssse3.phadd.d.128(<4 x i32> %a0, <4 x i32> %a1)
-  %2 = load <4 x i32>, <4 x i32> *%a2, align 16
-  %3 = call <4 x i32> @llvm.x86.ssse3.phadd.d.128(<4 x i32> %1, <4 x i32> %2)
-  ret <4 x i32> %3
-}
-declare <4 x i32> @llvm.x86.ssse3.phadd.d.128(<4 x i32>, <4 x i32>) nounwind readnone
-
-define <8 x i16> @test_phaddsw(<8 x i16> %a0, <8 x i16> %a1, <8 x i16> *%a2) {
-; GENERIC-LABEL: test_phaddsw:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    phaddsw %xmm1, %xmm0 # sched: [3:1.50]
-; GENERIC-NEXT:    phaddsw (%rdi), %xmm0 # sched: [9:1.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_phaddsw:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    phaddsw %xmm1, %xmm0 # sched: [7:3.50]
-; ATOM-NEXT:    phaddsw (%rdi), %xmm0 # sched: [8:4.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_phaddsw:
-; SLM:       # %bb.0:
-; SLM-NEXT:    phaddsw %xmm1, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    phaddsw (%rdi), %xmm0 # sched: [4:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_phaddsw:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    phaddsw %xmm1, %xmm0 # sched: [3:1.50]
-; SANDY-SSE-NEXT:    phaddsw (%rdi), %xmm0 # sched: [9:1.50]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_phaddsw:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vphaddsw %xmm1, %xmm0, %xmm0 # sched: [3:1.50]
-; SANDY-NEXT:    vphaddsw (%rdi), %xmm0, %xmm0 # sched: [9:1.50]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_phaddsw:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    phaddsw %xmm1, %xmm0 # sched: [3:2.00]
-; HASWELL-SSE-NEXT:    phaddsw (%rdi), %xmm0 # sched: [9:2.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_phaddsw:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vphaddsw %xmm1, %xmm0, %xmm0 # sched: [3:2.00]
-; HASWELL-NEXT:    vphaddsw (%rdi), %xmm0, %xmm0 # sched: [9:2.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_phaddsw:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    phaddsw %xmm1, %xmm0 # sched: [3:2.00]
-; BROADWELL-SSE-NEXT:    phaddsw (%rdi), %xmm0 # sched: [8:2.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_phaddsw:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vphaddsw %xmm1, %xmm0, %xmm0 # sched: [3:2.00]
-; BROADWELL-NEXT:    vphaddsw (%rdi), %xmm0, %xmm0 # sched: [8:2.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_phaddsw:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    phaddsw %xmm1, %xmm0 # sched: [3:2.00]
-; SKYLAKE-SSE-NEXT:    phaddsw (%rdi), %xmm0 # sched: [9:2.00]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_phaddsw:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vphaddsw %xmm1, %xmm0, %xmm0 # sched: [3:2.00]
-; SKYLAKE-NEXT:    vphaddsw (%rdi), %xmm0, %xmm0 # sched: [9:2.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_phaddsw:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    phaddsw %xmm1, %xmm0 # sched: [3:2.00]
-; SKX-SSE-NEXT:    phaddsw (%rdi), %xmm0 # sched: [9:2.00]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_phaddsw:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vphaddsw %xmm1, %xmm0, %xmm0 # sched: [3:2.00]
-; SKX-NEXT:    vphaddsw (%rdi), %xmm0, %xmm0 # sched: [9:2.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_phaddsw:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    phaddsw %xmm1, %xmm0 # sched: [5:0.50]
-; BDVER2-SSE-NEXT:    phaddsw (%rdi), %xmm0 # sched: [10:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_phaddsw:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vphaddsw %xmm1, %xmm0, %xmm0 # sched: [5:0.50]
-; BDVER2-NEXT:    vphaddsw (%rdi), %xmm0, %xmm0 # sched: [10:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_phaddsw:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    phaddsw %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    phaddsw (%rdi), %xmm0 # sched: [6:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_phaddsw:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vphaddsw %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    vphaddsw (%rdi), %xmm0, %xmm0 # sched: [6:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_phaddsw:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    phaddsw %xmm1, %xmm0 # sched: [100:0.25]
-; ZNVER1-SSE-NEXT:    phaddsw (%rdi), %xmm0 # sched: [100:0.25]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_phaddsw:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vphaddsw %xmm1, %xmm0, %xmm0 # sched: [100:0.25]
-; ZNVER1-NEXT:    vphaddsw (%rdi), %xmm0, %xmm0 # sched: [100:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <8 x i16> @llvm.x86.ssse3.phadd.sw.128(<8 x i16> %a0, <8 x i16> %a1)
-  %2 = load <8 x i16>, <8 x i16> *%a2, align 16
-  %3 = call <8 x i16> @llvm.x86.ssse3.phadd.sw.128(<8 x i16> %1, <8 x i16> %2)
-  ret <8 x i16> %3
-}
-declare <8 x i16> @llvm.x86.ssse3.phadd.sw.128(<8 x i16>, <8 x i16>) nounwind readnone
-
-define <8 x i16> @test_phaddw(<8 x i16> %a0, <8 x i16> %a1, <8 x i16> *%a2) {
-; GENERIC-LABEL: test_phaddw:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    phaddw %xmm1, %xmm0 # sched: [3:1.50]
-; GENERIC-NEXT:    phaddw (%rdi), %xmm0 # sched: [9:1.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_phaddw:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    phaddw %xmm1, %xmm0 # sched: [7:3.50]
-; ATOM-NEXT:    phaddw (%rdi), %xmm0 # sched: [8:4.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_phaddw:
-; SLM:       # %bb.0:
-; SLM-NEXT:    phaddw %xmm1, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    phaddw (%rdi), %xmm0 # sched: [4:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_phaddw:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    phaddw %xmm1, %xmm0 # sched: [3:1.50]
-; SANDY-SSE-NEXT:    phaddw (%rdi), %xmm0 # sched: [9:1.50]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_phaddw:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vphaddw %xmm1, %xmm0, %xmm0 # sched: [3:1.50]
-; SANDY-NEXT:    vphaddw (%rdi), %xmm0, %xmm0 # sched: [9:1.50]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_phaddw:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    phaddw %xmm1, %xmm0 # sched: [3:2.00]
-; HASWELL-SSE-NEXT:    phaddw (%rdi), %xmm0 # sched: [9:2.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_phaddw:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vphaddw %xmm1, %xmm0, %xmm0 # sched: [3:2.00]
-; HASWELL-NEXT:    vphaddw (%rdi), %xmm0, %xmm0 # sched: [9:2.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_phaddw:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    phaddw %xmm1, %xmm0 # sched: [3:2.00]
-; BROADWELL-SSE-NEXT:    phaddw (%rdi), %xmm0 # sched: [8:2.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_phaddw:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vphaddw %xmm1, %xmm0, %xmm0 # sched: [3:2.00]
-; BROADWELL-NEXT:    vphaddw (%rdi), %xmm0, %xmm0 # sched: [8:2.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_phaddw:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    phaddw %xmm1, %xmm0 # sched: [3:2.00]
-; SKYLAKE-SSE-NEXT:    phaddw (%rdi), %xmm0 # sched: [9:2.00]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_phaddw:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vphaddw %xmm1, %xmm0, %xmm0 # sched: [3:2.00]
-; SKYLAKE-NEXT:    vphaddw (%rdi), %xmm0, %xmm0 # sched: [9:2.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_phaddw:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    phaddw %xmm1, %xmm0 # sched: [3:2.00]
-; SKX-SSE-NEXT:    phaddw (%rdi), %xmm0 # sched: [9:2.00]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_phaddw:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vphaddw %xmm1, %xmm0, %xmm0 # sched: [3:2.00]
-; SKX-NEXT:    vphaddw (%rdi), %xmm0, %xmm0 # sched: [9:2.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_phaddw:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    phaddw %xmm1, %xmm0 # sched: [5:0.50]
-; BDVER2-SSE-NEXT:    phaddw (%rdi), %xmm0 # sched: [10:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_phaddw:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vphaddw %xmm1, %xmm0, %xmm0 # sched: [5:0.50]
-; BDVER2-NEXT:    vphaddw (%rdi), %xmm0, %xmm0 # sched: [10:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_phaddw:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    phaddw %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    phaddw (%rdi), %xmm0 # sched: [6:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_phaddw:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vphaddw %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    vphaddw (%rdi), %xmm0, %xmm0 # sched: [6:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_phaddw:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    phaddw %xmm1, %xmm0 # sched: [100:0.25]
-; ZNVER1-SSE-NEXT:    phaddw (%rdi), %xmm0 # sched: [100:0.25]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_phaddw:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vphaddw %xmm1, %xmm0, %xmm0 # sched: [100:0.25]
-; ZNVER1-NEXT:    vphaddw (%rdi), %xmm0, %xmm0 # sched: [100:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <8 x i16> @llvm.x86.ssse3.phadd.w.128(<8 x i16> %a0, <8 x i16> %a1)
-  %2 = load <8 x i16>, <8 x i16> *%a2, align 16
-  %3 = call <8 x i16> @llvm.x86.ssse3.phadd.w.128(<8 x i16> %1, <8 x i16> %2)
-  ret <8 x i16> %3
-}
-declare <8 x i16> @llvm.x86.ssse3.phadd.w.128(<8 x i16>, <8 x i16>) nounwind readnone
-
-define <4 x i32> @test_phsubd(<4 x i32> %a0, <4 x i32> %a1, <4 x i32> *%a2) {
-; GENERIC-LABEL: test_phsubd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    phsubd %xmm1, %xmm0 # sched: [3:1.50]
-; GENERIC-NEXT:    phsubd (%rdi), %xmm0 # sched: [9:1.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_phsubd:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    phsubd %xmm1, %xmm0 # sched: [3:1.50]
-; ATOM-NEXT:    phsubd (%rdi), %xmm0 # sched: [4:2.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_phsubd:
-; SLM:       # %bb.0:
-; SLM-NEXT:    phsubd %xmm1, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    phsubd (%rdi), %xmm0 # sched: [4:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_phsubd:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    phsubd %xmm1, %xmm0 # sched: [3:1.50]
-; SANDY-SSE-NEXT:    phsubd (%rdi), %xmm0 # sched: [9:1.50]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_phsubd:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vphsubd %xmm1, %xmm0, %xmm0 # sched: [3:1.50]
-; SANDY-NEXT:    vphsubd (%rdi), %xmm0, %xmm0 # sched: [9:1.50]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_phsubd:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    phsubd %xmm1, %xmm0 # sched: [3:2.00]
-; HASWELL-SSE-NEXT:    phsubd (%rdi), %xmm0 # sched: [9:2.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_phsubd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vphsubd %xmm1, %xmm0, %xmm0 # sched: [3:2.00]
-; HASWELL-NEXT:    vphsubd (%rdi), %xmm0, %xmm0 # sched: [9:2.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_phsubd:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    phsubd %xmm1, %xmm0 # sched: [3:2.00]
-; BROADWELL-SSE-NEXT:    phsubd (%rdi), %xmm0 # sched: [8:2.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_phsubd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vphsubd %xmm1, %xmm0, %xmm0 # sched: [3:2.00]
-; BROADWELL-NEXT:    vphsubd (%rdi), %xmm0, %xmm0 # sched: [8:2.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_phsubd:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    phsubd %xmm1, %xmm0 # sched: [3:2.00]
-; SKYLAKE-SSE-NEXT:    phsubd (%rdi), %xmm0 # sched: [9:2.00]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_phsubd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vphsubd %xmm1, %xmm0, %xmm0 # sched: [3:2.00]
-; SKYLAKE-NEXT:    vphsubd (%rdi), %xmm0, %xmm0 # sched: [9:2.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_phsubd:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    phsubd %xmm1, %xmm0 # sched: [3:2.00]
-; SKX-SSE-NEXT:    phsubd (%rdi), %xmm0 # sched: [9:2.00]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_phsubd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vphsubd %xmm1, %xmm0, %xmm0 # sched: [3:2.00]
-; SKX-NEXT:    vphsubd (%rdi), %xmm0, %xmm0 # sched: [9:2.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_phsubd:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    phsubd %xmm1, %xmm0 # sched: [5:0.50]
-; BDVER2-SSE-NEXT:    phsubd (%rdi), %xmm0 # sched: [10:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_phsubd:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vphsubd %xmm1, %xmm0, %xmm0 # sched: [5:0.50]
-; BDVER2-NEXT:    vphsubd (%rdi), %xmm0, %xmm0 # sched: [10:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_phsubd:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    phsubd %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    phsubd (%rdi), %xmm0 # sched: [6:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_phsubd:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vphsubd %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    vphsubd (%rdi), %xmm0, %xmm0 # sched: [6:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_phsubd:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    phsubd %xmm1, %xmm0 # sched: [100:0.25]
-; ZNVER1-SSE-NEXT:    phsubd (%rdi), %xmm0 # sched: [100:0.25]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_phsubd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vphsubd %xmm1, %xmm0, %xmm0 # sched: [100:0.25]
-; ZNVER1-NEXT:    vphsubd (%rdi), %xmm0, %xmm0 # sched: [100:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <4 x i32> @llvm.x86.ssse3.phsub.d.128(<4 x i32> %a0, <4 x i32> %a1)
-  %2 = load <4 x i32>, <4 x i32> *%a2, align 16
-  %3 = call <4 x i32> @llvm.x86.ssse3.phsub.d.128(<4 x i32> %1, <4 x i32> %2)
-  ret <4 x i32> %3
-}
-declare <4 x i32> @llvm.x86.ssse3.phsub.d.128(<4 x i32>, <4 x i32>) nounwind readnone
-
-define <8 x i16> @test_phsubsw(<8 x i16> %a0, <8 x i16> %a1, <8 x i16> *%a2) {
-; GENERIC-LABEL: test_phsubsw:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    phsubsw %xmm1, %xmm0 # sched: [3:1.50]
-; GENERIC-NEXT:    phsubsw (%rdi), %xmm0 # sched: [9:1.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_phsubsw:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    phsubsw %xmm1, %xmm0 # sched: [7:3.50]
-; ATOM-NEXT:    phsubsw (%rdi), %xmm0 # sched: [8:4.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_phsubsw:
-; SLM:       # %bb.0:
-; SLM-NEXT:    phsubsw %xmm1, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    phsubsw (%rdi), %xmm0 # sched: [4:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_phsubsw:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    phsubsw %xmm1, %xmm0 # sched: [3:1.50]
-; SANDY-SSE-NEXT:    phsubsw (%rdi), %xmm0 # sched: [9:1.50]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_phsubsw:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vphsubsw %xmm1, %xmm0, %xmm0 # sched: [3:1.50]
-; SANDY-NEXT:    vphsubsw (%rdi), %xmm0, %xmm0 # sched: [9:1.50]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_phsubsw:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    phsubsw %xmm1, %xmm0 # sched: [3:2.00]
-; HASWELL-SSE-NEXT:    phsubsw (%rdi), %xmm0 # sched: [9:2.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_phsubsw:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vphsubsw %xmm1, %xmm0, %xmm0 # sched: [3:2.00]
-; HASWELL-NEXT:    vphsubsw (%rdi), %xmm0, %xmm0 # sched: [9:2.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_phsubsw:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    phsubsw %xmm1, %xmm0 # sched: [3:2.00]
-; BROADWELL-SSE-NEXT:    phsubsw (%rdi), %xmm0 # sched: [8:2.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_phsubsw:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vphsubsw %xmm1, %xmm0, %xmm0 # sched: [3:2.00]
-; BROADWELL-NEXT:    vphsubsw (%rdi), %xmm0, %xmm0 # sched: [8:2.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_phsubsw:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    phsubsw %xmm1, %xmm0 # sched: [3:2.00]
-; SKYLAKE-SSE-NEXT:    phsubsw (%rdi), %xmm0 # sched: [9:2.00]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_phsubsw:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vphsubsw %xmm1, %xmm0, %xmm0 # sched: [3:2.00]
-; SKYLAKE-NEXT:    vphsubsw (%rdi), %xmm0, %xmm0 # sched: [9:2.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_phsubsw:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    phsubsw %xmm1, %xmm0 # sched: [3:2.00]
-; SKX-SSE-NEXT:    phsubsw (%rdi), %xmm0 # sched: [9:2.00]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_phsubsw:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vphsubsw %xmm1, %xmm0, %xmm0 # sched: [3:2.00]
-; SKX-NEXT:    vphsubsw (%rdi), %xmm0, %xmm0 # sched: [9:2.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_phsubsw:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    phsubsw %xmm1, %xmm0 # sched: [5:0.50]
-; BDVER2-SSE-NEXT:    phsubsw (%rdi), %xmm0 # sched: [10:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_phsubsw:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vphsubsw %xmm1, %xmm0, %xmm0 # sched: [5:0.50]
-; BDVER2-NEXT:    vphsubsw (%rdi), %xmm0, %xmm0 # sched: [10:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_phsubsw:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    phsubsw %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    phsubsw (%rdi), %xmm0 # sched: [6:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_phsubsw:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vphsubsw %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    vphsubsw (%rdi), %xmm0, %xmm0 # sched: [6:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_phsubsw:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    phsubsw %xmm1, %xmm0 # sched: [100:0.25]
-; ZNVER1-SSE-NEXT:    phsubsw (%rdi), %xmm0 # sched: [100:0.25]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_phsubsw:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vphsubsw %xmm1, %xmm0, %xmm0 # sched: [100:0.25]
-; ZNVER1-NEXT:    vphsubsw (%rdi), %xmm0, %xmm0 # sched: [100:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <8 x i16> @llvm.x86.ssse3.phsub.sw.128(<8 x i16> %a0, <8 x i16> %a1)
-  %2 = load <8 x i16>, <8 x i16> *%a2, align 16
-  %3 = call <8 x i16> @llvm.x86.ssse3.phsub.sw.128(<8 x i16> %1, <8 x i16> %2)
-  ret <8 x i16> %3
-}
-declare <8 x i16> @llvm.x86.ssse3.phsub.sw.128(<8 x i16>, <8 x i16>) nounwind readnone
-
-define <8 x i16> @test_phsubw(<8 x i16> %a0, <8 x i16> %a1, <8 x i16> *%a2) {
-; GENERIC-LABEL: test_phsubw:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    phsubw %xmm1, %xmm0 # sched: [3:1.50]
-; GENERIC-NEXT:    phsubw (%rdi), %xmm0 # sched: [9:1.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_phsubw:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    phsubw %xmm1, %xmm0 # sched: [7:3.50]
-; ATOM-NEXT:    phsubw (%rdi), %xmm0 # sched: [8:4.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_phsubw:
-; SLM:       # %bb.0:
-; SLM-NEXT:    phsubw %xmm1, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    phsubw (%rdi), %xmm0 # sched: [4:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_phsubw:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    phsubw %xmm1, %xmm0 # sched: [3:1.50]
-; SANDY-SSE-NEXT:    phsubw (%rdi), %xmm0 # sched: [9:1.50]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_phsubw:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vphsubw %xmm1, %xmm0, %xmm0 # sched: [3:1.50]
-; SANDY-NEXT:    vphsubw (%rdi), %xmm0, %xmm0 # sched: [9:1.50]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_phsubw:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    phsubw %xmm1, %xmm0 # sched: [3:2.00]
-; HASWELL-SSE-NEXT:    phsubw (%rdi), %xmm0 # sched: [9:2.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_phsubw:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vphsubw %xmm1, %xmm0, %xmm0 # sched: [3:2.00]
-; HASWELL-NEXT:    vphsubw (%rdi), %xmm0, %xmm0 # sched: [9:2.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_phsubw:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    phsubw %xmm1, %xmm0 # sched: [3:2.00]
-; BROADWELL-SSE-NEXT:    phsubw (%rdi), %xmm0 # sched: [8:2.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_phsubw:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vphsubw %xmm1, %xmm0, %xmm0 # sched: [3:2.00]
-; BROADWELL-NEXT:    vphsubw (%rdi), %xmm0, %xmm0 # sched: [8:2.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_phsubw:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    phsubw %xmm1, %xmm0 # sched: [3:2.00]
-; SKYLAKE-SSE-NEXT:    phsubw (%rdi), %xmm0 # sched: [9:2.00]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_phsubw:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vphsubw %xmm1, %xmm0, %xmm0 # sched: [3:2.00]
-; SKYLAKE-NEXT:    vphsubw (%rdi), %xmm0, %xmm0 # sched: [9:2.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_phsubw:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    phsubw %xmm1, %xmm0 # sched: [3:2.00]
-; SKX-SSE-NEXT:    phsubw (%rdi), %xmm0 # sched: [9:2.00]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_phsubw:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vphsubw %xmm1, %xmm0, %xmm0 # sched: [3:2.00]
-; SKX-NEXT:    vphsubw (%rdi), %xmm0, %xmm0 # sched: [9:2.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_phsubw:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    phsubw %xmm1, %xmm0 # sched: [5:0.50]
-; BDVER2-SSE-NEXT:    phsubw (%rdi), %xmm0 # sched: [10:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_phsubw:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vphsubw %xmm1, %xmm0, %xmm0 # sched: [5:0.50]
-; BDVER2-NEXT:    vphsubw (%rdi), %xmm0, %xmm0 # sched: [10:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_phsubw:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    phsubw %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    phsubw (%rdi), %xmm0 # sched: [6:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_phsubw:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vphsubw %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    vphsubw (%rdi), %xmm0, %xmm0 # sched: [6:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_phsubw:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    phsubw %xmm1, %xmm0 # sched: [100:0.25]
-; ZNVER1-SSE-NEXT:    phsubw (%rdi), %xmm0 # sched: [100:0.25]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_phsubw:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vphsubw %xmm1, %xmm0, %xmm0 # sched: [100:0.25]
-; ZNVER1-NEXT:    vphsubw (%rdi), %xmm0, %xmm0 # sched: [100:0.25]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <8 x i16> @llvm.x86.ssse3.phsub.w.128(<8 x i16> %a0, <8 x i16> %a1)
-  %2 = load <8 x i16>, <8 x i16> *%a2, align 16
-  %3 = call <8 x i16> @llvm.x86.ssse3.phsub.w.128(<8 x i16> %1, <8 x i16> %2)
-  ret <8 x i16> %3
-}
-declare <8 x i16> @llvm.x86.ssse3.phsub.w.128(<8 x i16>, <8 x i16>) nounwind readnone
-
-define <8 x i16> @test_pmaddubsw(<16 x i8> %a0, <16 x i8> %a1, <16 x i8> *%a2) {
-; GENERIC-LABEL: test_pmaddubsw:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    pmaddubsw %xmm1, %xmm0 # sched: [5:1.00]
-; GENERIC-NEXT:    pmaddubsw (%rdi), %xmm0 # sched: [11:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_pmaddubsw:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    pmaddubsw %xmm1, %xmm0 # sched: [5:5.00]
-; ATOM-NEXT:    pmaddubsw (%rdi), %xmm0 # sched: [5:5.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_pmaddubsw:
-; SLM:       # %bb.0:
-; SLM-NEXT:    pmaddubsw %xmm1, %xmm0 # sched: [4:1.00]
-; SLM-NEXT:    pmaddubsw (%rdi), %xmm0 # sched: [7:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_pmaddubsw:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    pmaddubsw %xmm1, %xmm0 # sched: [5:1.00]
-; SANDY-SSE-NEXT:    pmaddubsw (%rdi), %xmm0 # sched: [11:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_pmaddubsw:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vpmaddubsw %xmm1, %xmm0, %xmm0 # sched: [5:1.00]
-; SANDY-NEXT:    vpmaddubsw (%rdi), %xmm0, %xmm0 # sched: [11:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_pmaddubsw:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    pmaddubsw %xmm1, %xmm0 # sched: [5:1.00]
-; HASWELL-SSE-NEXT:    pmaddubsw (%rdi), %xmm0 # sched: [11:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_pmaddubsw:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpmaddubsw %xmm1, %xmm0, %xmm0 # sched: [5:1.00]
-; HASWELL-NEXT:    vpmaddubsw (%rdi), %xmm0, %xmm0 # sched: [11:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_pmaddubsw:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    pmaddubsw %xmm1, %xmm0 # sched: [5:1.00]
-; BROADWELL-SSE-NEXT:    pmaddubsw (%rdi), %xmm0 # sched: [10:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pmaddubsw:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpmaddubsw %xmm1, %xmm0, %xmm0 # sched: [5:1.00]
-; BROADWELL-NEXT:    vpmaddubsw (%rdi), %xmm0, %xmm0 # sched: [10:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_pmaddubsw:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    pmaddubsw %xmm1, %xmm0 # sched: [4:0.50]
-; SKYLAKE-SSE-NEXT:    pmaddubsw (%rdi), %xmm0 # sched: [10:0.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pmaddubsw:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpmaddubsw %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    vpmaddubsw (%rdi), %xmm0, %xmm0 # sched: [10:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_pmaddubsw:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    pmaddubsw %xmm1, %xmm0 # sched: [4:0.50]
-; SKX-SSE-NEXT:    pmaddubsw (%rdi), %xmm0 # sched: [10:0.50]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pmaddubsw:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpmaddubsw %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKX-NEXT:    vpmaddubsw (%rdi), %xmm0, %xmm0 # sched: [10:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_pmaddubsw:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    pmaddubsw %xmm1, %xmm0 # sched: [4:1.00]
-; BDVER2-SSE-NEXT:    pmaddubsw (%rdi), %xmm0 # sched: [9:1.00]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_pmaddubsw:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vpmaddubsw %xmm1, %xmm0, %xmm0 # sched: [4:1.00]
-; BDVER2-NEXT:    vpmaddubsw (%rdi), %xmm0, %xmm0 # sched: [9:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_pmaddubsw:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    pmaddubsw %xmm1, %xmm0 # sched: [2:1.00]
-; BTVER2-SSE-NEXT:    pmaddubsw (%rdi), %xmm0 # sched: [7:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_pmaddubsw:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vpmaddubsw %xmm1, %xmm0, %xmm0 # sched: [2:1.00]
-; BTVER2-NEXT:    vpmaddubsw (%rdi), %xmm0, %xmm0 # sched: [7:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_pmaddubsw:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    pmaddubsw %xmm1, %xmm0 # sched: [4:1.00]
-; ZNVER1-SSE-NEXT:    pmaddubsw (%rdi), %xmm0 # sched: [11:1.00]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_pmaddubsw:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpmaddubsw %xmm1, %xmm0, %xmm0 # sched: [4:1.00]
-; ZNVER1-NEXT:    vpmaddubsw (%rdi), %xmm0, %xmm0 # sched: [11:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <8 x i16> @llvm.x86.ssse3.pmadd.ub.sw.128(<16 x i8> %a0, <16 x i8> %a1)
-  %2 = load <16 x i8>, <16 x i8> *%a2, align 16
-  %3 = bitcast <8 x i16> %1 to <16 x i8>
-  %4 = call <8 x i16> @llvm.x86.ssse3.pmadd.ub.sw.128(<16 x i8> %3, <16 x i8> %2)
-  ret <8 x i16> %4
-}
-declare <8 x i16> @llvm.x86.ssse3.pmadd.ub.sw.128(<16 x i8>, <16 x i8>) nounwind readnone
-
-define <8 x i16> @test_pmulhrsw(<8 x i16> %a0, <8 x i16> %a1, <8 x i16> *%a2) {
-; GENERIC-LABEL: test_pmulhrsw:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    pmulhrsw %xmm1, %xmm0 # sched: [5:1.00]
-; GENERIC-NEXT:    pmulhrsw (%rdi), %xmm0 # sched: [11:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_pmulhrsw:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    pmulhrsw %xmm1, %xmm0 # sched: [5:5.00]
-; ATOM-NEXT:    pmulhrsw (%rdi), %xmm0 # sched: [5:5.00]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_pmulhrsw:
-; SLM:       # %bb.0:
-; SLM-NEXT:    pmulhrsw %xmm1, %xmm0 # sched: [4:1.00]
-; SLM-NEXT:    pmulhrsw (%rdi), %xmm0 # sched: [7:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_pmulhrsw:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    pmulhrsw %xmm1, %xmm0 # sched: [5:1.00]
-; SANDY-SSE-NEXT:    pmulhrsw (%rdi), %xmm0 # sched: [11:1.00]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_pmulhrsw:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vpmulhrsw %xmm1, %xmm0, %xmm0 # sched: [5:1.00]
-; SANDY-NEXT:    vpmulhrsw (%rdi), %xmm0, %xmm0 # sched: [11:1.00]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_pmulhrsw:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    pmulhrsw %xmm1, %xmm0 # sched: [5:1.00]
-; HASWELL-SSE-NEXT:    pmulhrsw (%rdi), %xmm0 # sched: [11:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_pmulhrsw:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpmulhrsw %xmm1, %xmm0, %xmm0 # sched: [5:1.00]
-; HASWELL-NEXT:    vpmulhrsw (%rdi), %xmm0, %xmm0 # sched: [11:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_pmulhrsw:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    pmulhrsw %xmm1, %xmm0 # sched: [5:1.00]
-; BROADWELL-SSE-NEXT:    pmulhrsw (%rdi), %xmm0 # sched: [10:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pmulhrsw:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpmulhrsw %xmm1, %xmm0, %xmm0 # sched: [5:1.00]
-; BROADWELL-NEXT:    vpmulhrsw (%rdi), %xmm0, %xmm0 # sched: [10:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_pmulhrsw:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    pmulhrsw %xmm1, %xmm0 # sched: [4:0.50]
-; SKYLAKE-SSE-NEXT:    pmulhrsw (%rdi), %xmm0 # sched: [10:0.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pmulhrsw:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpmulhrsw %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKYLAKE-NEXT:    vpmulhrsw (%rdi), %xmm0, %xmm0 # sched: [10:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_pmulhrsw:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    pmulhrsw %xmm1, %xmm0 # sched: [4:0.50]
-; SKX-SSE-NEXT:    pmulhrsw (%rdi), %xmm0 # sched: [10:0.50]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pmulhrsw:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpmulhrsw %xmm1, %xmm0, %xmm0 # sched: [4:0.50]
-; SKX-NEXT:    vpmulhrsw (%rdi), %xmm0, %xmm0 # sched: [10:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_pmulhrsw:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    pmulhrsw %xmm1, %xmm0 # sched: [4:1.00]
-; BDVER2-SSE-NEXT:    pmulhrsw (%rdi), %xmm0 # sched: [9:1.00]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_pmulhrsw:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vpmulhrsw %xmm1, %xmm0, %xmm0 # sched: [4:1.00]
-; BDVER2-NEXT:    vpmulhrsw (%rdi), %xmm0, %xmm0 # sched: [9:1.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_pmulhrsw:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    pmulhrsw %xmm1, %xmm0 # sched: [2:1.00]
-; BTVER2-SSE-NEXT:    pmulhrsw (%rdi), %xmm0 # sched: [7:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_pmulhrsw:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vpmulhrsw %xmm1, %xmm0, %xmm0 # sched: [2:1.00]
-; BTVER2-NEXT:    vpmulhrsw (%rdi), %xmm0, %xmm0 # sched: [7:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_pmulhrsw:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    pmulhrsw %xmm1, %xmm0 # sched: [4:1.00]
-; ZNVER1-SSE-NEXT:    pmulhrsw (%rdi), %xmm0 # sched: [11:1.00]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_pmulhrsw:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpmulhrsw %xmm1, %xmm0, %xmm0 # sched: [4:1.00]
-; ZNVER1-NEXT:    vpmulhrsw (%rdi), %xmm0, %xmm0 # sched: [11:1.00]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <8 x i16> @llvm.x86.ssse3.pmul.hr.sw.128(<8 x i16> %a0, <8 x i16> %a1)
-  %2 = load <8 x i16>, <8 x i16> *%a2, align 16
-  %3 = call <8 x i16> @llvm.x86.ssse3.pmul.hr.sw.128(<8 x i16> %1, <8 x i16> %2)
-  ret <8 x i16> %3
-}
-declare <8 x i16> @llvm.x86.ssse3.pmul.hr.sw.128(<8 x i16>, <8 x i16>) nounwind readnone
-
-define <16 x i8> @test_pshufb(<16 x i8> %a0, <16 x i8> %a1, <16 x i8> *%a2) {
-; GENERIC-LABEL: test_pshufb:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    pshufb %xmm1, %xmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    pshufb (%rdi), %xmm0 # sched: [7:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_pshufb:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    pshufb %xmm1, %xmm0 # sched: [4:2.00]
-; ATOM-NEXT:    pshufb (%rdi), %xmm0 # sched: [5:2.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_pshufb:
-; SLM:       # %bb.0:
-; SLM-NEXT:    pshufb %xmm1, %xmm0 # sched: [1:1.00]
-; SLM-NEXT:    pshufb (%rdi), %xmm0 # sched: [4:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_pshufb:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    pshufb %xmm1, %xmm0 # sched: [1:0.50]
-; SANDY-SSE-NEXT:    pshufb (%rdi), %xmm0 # sched: [7:0.50]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_pshufb:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vpshufb %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SANDY-NEXT:    vpshufb (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_pshufb:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    pshufb %xmm1, %xmm0 # sched: [1:1.00]
-; HASWELL-SSE-NEXT:    pshufb (%rdi), %xmm0 # sched: [7:1.00]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_pshufb:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpshufb %xmm1, %xmm0, %xmm0 # sched: [1:1.00]
-; HASWELL-NEXT:    vpshufb (%rdi), %xmm0, %xmm0 # sched: [7:1.00]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_pshufb:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    pshufb %xmm1, %xmm0 # sched: [1:1.00]
-; BROADWELL-SSE-NEXT:    pshufb (%rdi), %xmm0 # sched: [6:1.00]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_pshufb:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpshufb %xmm1, %xmm0, %xmm0 # sched: [1:1.00]
-; BROADWELL-NEXT:    vpshufb (%rdi), %xmm0, %xmm0 # sched: [6:1.00]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_pshufb:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    pshufb %xmm1, %xmm0 # sched: [1:1.00]
-; SKYLAKE-SSE-NEXT:    pshufb (%rdi), %xmm0 # sched: [7:1.00]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_pshufb:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpshufb %xmm1, %xmm0, %xmm0 # sched: [1:1.00]
-; SKYLAKE-NEXT:    vpshufb (%rdi), %xmm0, %xmm0 # sched: [7:1.00]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_pshufb:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    pshufb %xmm1, %xmm0 # sched: [1:1.00]
-; SKX-SSE-NEXT:    pshufb (%rdi), %xmm0 # sched: [7:1.00]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_pshufb:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpshufb %xmm1, %xmm0, %xmm0 # sched: [1:1.00]
-; SKX-NEXT:    vpshufb (%rdi), %xmm0, %xmm0 # sched: [7:1.00]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_pshufb:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    pshufb %xmm1, %xmm0 # sched: [3:2.00]
-; BDVER2-SSE-NEXT:    pshufb (%rdi), %xmm0 # sched: [8:2.00]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_pshufb:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vpshufb %xmm1, %xmm0, %xmm0 # sched: [3:2.00]
-; BDVER2-NEXT:    vpshufb (%rdi), %xmm0, %xmm0 # sched: [8:2.00]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_pshufb:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    pshufb %xmm1, %xmm0 # sched: [2:2.00]
-; BTVER2-SSE-NEXT:    pshufb (%rdi), %xmm0 # sched: [7:2.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_pshufb:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vpshufb %xmm1, %xmm0, %xmm0 # sched: [2:2.00]
-; BTVER2-NEXT:    vpshufb (%rdi), %xmm0, %xmm0 # sched: [7:2.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_pshufb:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    pshufb %xmm1, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    pshufb (%rdi), %xmm0 # sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_pshufb:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpshufb %xmm1, %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vpshufb (%rdi), %xmm0, %xmm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <16 x i8> @llvm.x86.ssse3.pshuf.b.128(<16 x i8> %a0, <16 x i8> %a1)
-  %2 = load <16 x i8>, <16 x i8> *%a2, align 16
-  %3 = call <16 x i8> @llvm.x86.ssse3.pshuf.b.128(<16 x i8> %1, <16 x i8> %2)
-  ret <16 x i8> %3
-}
-declare <16 x i8> @llvm.x86.ssse3.pshuf.b.128(<16 x i8>, <16 x i8>) nounwind readnone
-
-define <16 x i8> @test_psignb(<16 x i8> %a0, <16 x i8> %a1, <16 x i8> *%a2) {
-; GENERIC-LABEL: test_psignb:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    psignb %xmm1, %xmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    psignb (%rdi), %xmm0 # sched: [7:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_psignb:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    psignb %xmm1, %xmm0 # sched: [1:0.50]
-; ATOM-NEXT:    psignb (%rdi), %xmm0 # sched: [1:1.00]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_psignb:
-; SLM:       # %bb.0:
-; SLM-NEXT:    psignb %xmm1, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    psignb (%rdi), %xmm0 # sched: [4:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_psignb:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    psignb %xmm1, %xmm0 # sched: [1:0.50]
-; SANDY-SSE-NEXT:    psignb (%rdi), %xmm0 # sched: [7:0.50]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_psignb:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vpsignb %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SANDY-NEXT:    vpsignb (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_psignb:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    psignb %xmm1, %xmm0 # sched: [1:0.50]
-; HASWELL-SSE-NEXT:    psignb (%rdi), %xmm0 # sched: [7:0.50]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_psignb:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpsignb %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; HASWELL-NEXT:    vpsignb (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_psignb:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    psignb %xmm1, %xmm0 # sched: [1:0.50]
-; BROADWELL-SSE-NEXT:    psignb (%rdi), %xmm0 # sched: [6:0.50]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_psignb:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpsignb %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    vpsignb (%rdi), %xmm0, %xmm0 # sched: [6:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_psignb:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    psignb %xmm1, %xmm0 # sched: [1:0.50]
-; SKYLAKE-SSE-NEXT:    psignb (%rdi), %xmm0 # sched: [7:0.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_psignb:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpsignb %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SKYLAKE-NEXT:    vpsignb (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_psignb:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    psignb %xmm1, %xmm0 # sched: [1:0.50]
-; SKX-SSE-NEXT:    psignb (%rdi), %xmm0 # sched: [7:0.50]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_psignb:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpsignb %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SKX-NEXT:    vpsignb (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_psignb:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    psignb %xmm1, %xmm0 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    psignb (%rdi), %xmm0 # sched: [7:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_psignb:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vpsignb %xmm1, %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    vpsignb (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_psignb:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    psignb %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    psignb (%rdi), %xmm0 # sched: [6:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_psignb:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vpsignb %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    vpsignb (%rdi), %xmm0, %xmm0 # sched: [6:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_psignb:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    psignb %xmm1, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    psignb (%rdi), %xmm0 # sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_psignb:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpsignb %xmm1, %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vpsignb (%rdi), %xmm0, %xmm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <16 x i8> @llvm.x86.ssse3.psign.b.128(<16 x i8> %a0, <16 x i8> %a1)
-  %2 = load <16 x i8>, <16 x i8> *%a2, align 16
-  %3 = call <16 x i8> @llvm.x86.ssse3.psign.b.128(<16 x i8> %1, <16 x i8> %2)
-  ret <16 x i8> %3
-}
-declare <16 x i8> @llvm.x86.ssse3.psign.b.128(<16 x i8>, <16 x i8>) nounwind readnone
-
-define <4 x i32> @test_psignd(<4 x i32> %a0, <4 x i32> %a1, <4 x i32> *%a2) {
-; GENERIC-LABEL: test_psignd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    psignd %xmm1, %xmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    psignd (%rdi), %xmm0 # sched: [7:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_psignd:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    psignd %xmm1, %xmm0 # sched: [1:0.50]
-; ATOM-NEXT:    psignd (%rdi), %xmm0 # sched: [1:1.00]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_psignd:
-; SLM:       # %bb.0:
-; SLM-NEXT:    psignd %xmm1, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    psignd (%rdi), %xmm0 # sched: [4:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_psignd:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    psignd %xmm1, %xmm0 # sched: [1:0.50]
-; SANDY-SSE-NEXT:    psignd (%rdi), %xmm0 # sched: [7:0.50]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_psignd:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vpsignd %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SANDY-NEXT:    vpsignd (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_psignd:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    psignd %xmm1, %xmm0 # sched: [1:0.50]
-; HASWELL-SSE-NEXT:    psignd (%rdi), %xmm0 # sched: [7:0.50]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_psignd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpsignd %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; HASWELL-NEXT:    vpsignd (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_psignd:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    psignd %xmm1, %xmm0 # sched: [1:0.50]
-; BROADWELL-SSE-NEXT:    psignd (%rdi), %xmm0 # sched: [6:0.50]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_psignd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpsignd %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    vpsignd (%rdi), %xmm0, %xmm0 # sched: [6:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_psignd:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    psignd %xmm1, %xmm0 # sched: [1:0.50]
-; SKYLAKE-SSE-NEXT:    psignd (%rdi), %xmm0 # sched: [7:0.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_psignd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpsignd %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SKYLAKE-NEXT:    vpsignd (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_psignd:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    psignd %xmm1, %xmm0 # sched: [1:0.50]
-; SKX-SSE-NEXT:    psignd (%rdi), %xmm0 # sched: [7:0.50]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_psignd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpsignd %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SKX-NEXT:    vpsignd (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_psignd:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    psignd %xmm1, %xmm0 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    psignd (%rdi), %xmm0 # sched: [7:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_psignd:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vpsignd %xmm1, %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    vpsignd (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_psignd:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    psignd %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    psignd (%rdi), %xmm0 # sched: [6:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_psignd:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vpsignd %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    vpsignd (%rdi), %xmm0, %xmm0 # sched: [6:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_psignd:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    psignd %xmm1, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    psignd (%rdi), %xmm0 # sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_psignd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpsignd %xmm1, %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vpsignd (%rdi), %xmm0, %xmm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <4 x i32> @llvm.x86.ssse3.psign.d.128(<4 x i32> %a0, <4 x i32> %a1)
-  %2 = load <4 x i32>, <4 x i32> *%a2, align 16
-  %3 = call <4 x i32> @llvm.x86.ssse3.psign.d.128(<4 x i32> %1, <4 x i32> %2)
-  ret <4 x i32> %3
-}
-declare <4 x i32> @llvm.x86.ssse3.psign.d.128(<4 x i32>, <4 x i32>) nounwind readnone
-
-define <8 x i16> @test_psignw(<8 x i16> %a0, <8 x i16> %a1, <8 x i16> *%a2) {
-; GENERIC-LABEL: test_psignw:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    psignw %xmm1, %xmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    psignw (%rdi), %xmm0 # sched: [7:0.50]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; ATOM-LABEL: test_psignw:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    psignw %xmm1, %xmm0 # sched: [1:0.50]
-; ATOM-NEXT:    psignw (%rdi), %xmm0 # sched: [1:1.00]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    nop # sched: [1:0.50]
-; ATOM-NEXT:    retq # sched: [79:39.50]
-;
-; SLM-LABEL: test_psignw:
-; SLM:       # %bb.0:
-; SLM-NEXT:    psignw %xmm1, %xmm0 # sched: [1:0.50]
-; SLM-NEXT:    psignw (%rdi), %xmm0 # sched: [4:1.00]
-; SLM-NEXT:    retq # sched: [4:1.00]
-;
-; SANDY-SSE-LABEL: test_psignw:
-; SANDY-SSE:       # %bb.0:
-; SANDY-SSE-NEXT:    psignw %xmm1, %xmm0 # sched: [1:0.50]
-; SANDY-SSE-NEXT:    psignw (%rdi), %xmm0 # sched: [7:0.50]
-; SANDY-SSE-NEXT:    retq # sched: [1:1.00]
-;
-; SANDY-LABEL: test_psignw:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    vpsignw %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SANDY-NEXT:    vpsignw (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SANDY-NEXT:    retq # sched: [1:1.00]
-;
-; HASWELL-SSE-LABEL: test_psignw:
-; HASWELL-SSE:       # %bb.0:
-; HASWELL-SSE-NEXT:    psignw %xmm1, %xmm0 # sched: [1:0.50]
-; HASWELL-SSE-NEXT:    psignw (%rdi), %xmm0 # sched: [7:0.50]
-; HASWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; HASWELL-LABEL: test_psignw:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    vpsignw %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; HASWELL-NEXT:    vpsignw (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; HASWELL-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-SSE-LABEL: test_psignw:
-; BROADWELL-SSE:       # %bb.0:
-; BROADWELL-SSE-NEXT:    psignw %xmm1, %xmm0 # sched: [1:0.50]
-; BROADWELL-SSE-NEXT:    psignw (%rdi), %xmm0 # sched: [6:0.50]
-; BROADWELL-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_psignw:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    vpsignw %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BROADWELL-NEXT:    vpsignw (%rdi), %xmm0, %xmm0 # sched: [6:0.50]
-; BROADWELL-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-SSE-LABEL: test_psignw:
-; SKYLAKE-SSE:       # %bb.0:
-; SKYLAKE-SSE-NEXT:    psignw %xmm1, %xmm0 # sched: [1:0.50]
-; SKYLAKE-SSE-NEXT:    psignw (%rdi), %xmm0 # sched: [7:0.50]
-; SKYLAKE-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKYLAKE-LABEL: test_psignw:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    vpsignw %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SKYLAKE-NEXT:    vpsignw (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKYLAKE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-SSE-LABEL: test_psignw:
-; SKX-SSE:       # %bb.0:
-; SKX-SSE-NEXT:    psignw %xmm1, %xmm0 # sched: [1:0.50]
-; SKX-SSE-NEXT:    psignw (%rdi), %xmm0 # sched: [7:0.50]
-; SKX-SSE-NEXT:    retq # sched: [7:1.00]
-;
-; SKX-LABEL: test_psignw:
-; SKX:       # %bb.0:
-; SKX-NEXT:    vpsignw %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; SKX-NEXT:    vpsignw (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; SKX-NEXT:    retq # sched: [7:1.00]
-;
-; BDVER2-SSE-LABEL: test_psignw:
-; BDVER2-SSE:       # %bb.0:
-; BDVER2-SSE-NEXT:    psignw %xmm1, %xmm0 # sched: [2:0.50]
-; BDVER2-SSE-NEXT:    psignw (%rdi), %xmm0 # sched: [7:0.50]
-; BDVER2-SSE-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER2-LABEL: test_psignw:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    vpsignw %xmm1, %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER2-NEXT:    vpsignw (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BTVER2-SSE-LABEL: test_psignw:
-; BTVER2-SSE:       # %bb.0:
-; BTVER2-SSE-NEXT:    psignw %xmm1, %xmm0 # sched: [1:0.50]
-; BTVER2-SSE-NEXT:    psignw (%rdi), %xmm0 # sched: [6:1.00]
-; BTVER2-SSE-NEXT:    retq # sched: [4:1.00]
-;
-; BTVER2-LABEL: test_psignw:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    vpsignw %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; BTVER2-NEXT:    vpsignw (%rdi), %xmm0, %xmm0 # sched: [6:1.00]
-; BTVER2-NEXT:    retq # sched: [4:1.00]
-;
-; ZNVER1-SSE-LABEL: test_psignw:
-; ZNVER1-SSE:       # %bb.0:
-; ZNVER1-SSE-NEXT:    psignw %xmm1, %xmm0 # sched: [1:0.25]
-; ZNVER1-SSE-NEXT:    psignw (%rdi), %xmm0 # sched: [8:0.50]
-; ZNVER1-SSE-NEXT:    retq # sched: [1:0.50]
-;
-; ZNVER1-LABEL: test_psignw:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    vpsignw %xmm1, %xmm0, %xmm0 # sched: [1:0.25]
-; ZNVER1-NEXT:    vpsignw (%rdi), %xmm0, %xmm0 # sched: [8:0.50]
-; ZNVER1-NEXT:    retq # sched: [1:0.50]
-  %1 = call <8 x i16> @llvm.x86.ssse3.psign.w.128(<8 x i16> %a0, <8 x i16> %a1)
-  %2 = load <8 x i16>, <8 x i16> *%a2, align 16
-  %3 = call <8 x i16> @llvm.x86.ssse3.psign.w.128(<8 x i16> %1, <8 x i16> %2)
-  ret <8 x i16> %3
-}
-declare <8 x i16> @llvm.x86.ssse3.psign.w.128(<8 x i16>, <8 x i16>) nounwind readnone
diff --git a/test/CodeGen/X86/ssub_sat.ll b/test/CodeGen/X86/ssub_sat.ll
index 6d9a534fad90f7229550c69cabb29bebaca46dcd..7fda35a3f7b661b5379e4953e3d816c98736a12f 100644
--- a/test/CodeGen/X86/ssub_sat.ll
+++ b/test/CodeGen/X86/ssub_sat.ll
@@ -1,267 +1,215 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mcpu=generic -mtriple=x86_64-linux | FileCheck %s
-; RUN: llc < %s -mcpu=generic -mtriple=i686 -mattr=cmov | FileCheck %s --check-prefix=CHECK32
+; RUN: llc < %s -mtriple=i686 -mattr=cmov | FileCheck %s --check-prefixes=CHECK,X86
+; RUN: llc < %s -mtriple=x86_64-linux | FileCheck %s --check-prefixes=CHECK,X64
 
 declare  i4  @llvm.ssub.sat.i4   (i4,  i4)
 declare  i32 @llvm.ssub.sat.i32  (i32, i32)
 declare  i64 @llvm.ssub.sat.i64  (i64, i64)
 declare  <4 x i32> @llvm.ssub.sat.v4i32(<4 x i32>, <4 x i32>)
 
-define i32 @func(i32 %x, i32 %y) {
-; CHECK-LABEL: func:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    xorl %eax, %eax
-; CHECK-NEXT:    movl %edi, %ecx
-; CHECK-NEXT:    subl %esi, %ecx
-; CHECK-NEXT:    setns %al
-; CHECK-NEXT:    addl $2147483647, %eax # imm = 0x7FFFFFFF
-; CHECK-NEXT:    subl %esi, %edi
-; CHECK-NEXT:    cmovnol %edi, %eax
-; CHECK-NEXT:    retq
+define i32 @func(i32 %x, i32 %y) nounwind {
+; X86-LABEL: func:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    xorl %ecx, %ecx
+; X86-NEXT:    movl %eax, %esi
+; X86-NEXT:    subl %edx, %esi
+; X86-NEXT:    setns %cl
+; X86-NEXT:    addl $2147483647, %ecx # imm = 0x7FFFFFFF
+; X86-NEXT:    subl %edx, %eax
+; X86-NEXT:    cmovol %ecx, %eax
+; X86-NEXT:    popl %esi
+; X86-NEXT:    retl
 ;
-; CHECK32-LABEL: func:
-; CHECK32:       # %bb.0:
-; CHECK32-NEXT:    pushl %esi
-; CHECK32-NEXT:    .cfi_def_cfa_offset 8
-; CHECK32-NEXT:    .cfi_offset %esi, -8
-; CHECK32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; CHECK32-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; CHECK32-NEXT:    xorl %ecx, %ecx
-; CHECK32-NEXT:    movl %eax, %esi
-; CHECK32-NEXT:    subl %edx, %esi
-; CHECK32-NEXT:    setns %cl
-; CHECK32-NEXT:    addl $2147483647, %ecx # imm = 0x7FFFFFFF
-; CHECK32-NEXT:    subl %edx, %eax
-; CHECK32-NEXT:    cmovol %ecx, %eax
-; CHECK32-NEXT:    popl %esi
-; CHECK32-NEXT:    .cfi_def_cfa_offset 4
-; CHECK32-NEXT:    retl
+; X64-LABEL: func:
+; X64:       # %bb.0:
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    movl %edi, %ecx
+; X64-NEXT:    subl %esi, %ecx
+; X64-NEXT:    setns %al
+; X64-NEXT:    addl $2147483647, %eax # imm = 0x7FFFFFFF
+; X64-NEXT:    subl %esi, %edi
+; X64-NEXT:    cmovnol %edi, %eax
+; X64-NEXT:    retq
   %tmp = call i32 @llvm.ssub.sat.i32(i32 %x, i32 %y);
   ret i32 %tmp;
 }
 
-define i64 @func2(i64 %x, i64 %y) {
-; CHECK-LABEL: func2:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    xorl %ecx, %ecx
-; CHECK-NEXT:    movq %rdi, %rax
-; CHECK-NEXT:    subq %rsi, %rax
-; CHECK-NEXT:    setns %cl
-; CHECK-NEXT:    movabsq $9223372036854775807, %rax # imm = 0x7FFFFFFFFFFFFFFF
-; CHECK-NEXT:    addq %rcx, %rax
-; CHECK-NEXT:    subq %rsi, %rdi
-; CHECK-NEXT:    cmovnoq %rdi, %rax
-; CHECK-NEXT:    retq
+define i64 @func2(i64 %x, i64 %y) nounwind {
+; X86-LABEL: func2:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %ebp
+; X86-NEXT:    pushl %ebx
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ebx
+; X86-NEXT:    subl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    movl %ebx, %ebp
+; X86-NEXT:    sbbl %esi, %ebp
+; X86-NEXT:    movl %ebp, %eax
+; X86-NEXT:    sarl $31, %eax
+; X86-NEXT:    xorl %ecx, %ecx
+; X86-NEXT:    testl %ebp, %ebp
+; X86-NEXT:    setns %cl
+; X86-NEXT:    movl %ecx, %edx
+; X86-NEXT:    addl $2147483647, %edx # imm = 0x7FFFFFFF
+; X86-NEXT:    testl %ebx, %ebx
+; X86-NEXT:    setns %bl
+; X86-NEXT:    cmpb %cl, %bl
+; X86-NEXT:    setne %cl
+; X86-NEXT:    testl %esi, %esi
+; X86-NEXT:    setns %ch
+; X86-NEXT:    cmpb %ch, %bl
+; X86-NEXT:    setne %ch
+; X86-NEXT:    testb %cl, %ch
+; X86-NEXT:    cmovel %ebp, %edx
+; X86-NEXT:    cmovel %edi, %eax
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    popl %ebx
+; X86-NEXT:    popl %ebp
+; X86-NEXT:    retl
 ;
-; CHECK32-LABEL: func2:
-; CHECK32:       # %bb.0:
-; CHECK32-NEXT:    pushl %ebp
-; CHECK32-NEXT:    .cfi_def_cfa_offset 8
-; CHECK32-NEXT:    pushl %ebx
-; CHECK32-NEXT:    .cfi_def_cfa_offset 12
-; CHECK32-NEXT:    pushl %edi
-; CHECK32-NEXT:    .cfi_def_cfa_offset 16
-; CHECK32-NEXT:    pushl %esi
-; CHECK32-NEXT:    .cfi_def_cfa_offset 20
-; CHECK32-NEXT:    .cfi_offset %esi, -20
-; CHECK32-NEXT:    .cfi_offset %edi, -16
-; CHECK32-NEXT:    .cfi_offset %ebx, -12
-; CHECK32-NEXT:    .cfi_offset %ebp, -8
-; CHECK32-NEXT:    movl {{[0-9]+}}(%esp), %esi
-; CHECK32-NEXT:    movl {{[0-9]+}}(%esp), %edi
-; CHECK32-NEXT:    movl {{[0-9]+}}(%esp), %ebx
-; CHECK32-NEXT:    subl {{[0-9]+}}(%esp), %edi
-; CHECK32-NEXT:    movl %ebx, %ebp
-; CHECK32-NEXT:    sbbl %esi, %ebp
-; CHECK32-NEXT:    movl %ebp, %eax
-; CHECK32-NEXT:    sarl $31, %eax
-; CHECK32-NEXT:    xorl %ecx, %ecx
-; CHECK32-NEXT:    testl %ebp, %ebp
-; CHECK32-NEXT:    setns %cl
-; CHECK32-NEXT:    movl %ecx, %edx
-; CHECK32-NEXT:    addl $2147483647, %edx # imm = 0x7FFFFFFF
-; CHECK32-NEXT:    testl %ebx, %ebx
-; CHECK32-NEXT:    setns %bl
-; CHECK32-NEXT:    cmpb %cl, %bl
-; CHECK32-NEXT:    setne %cl
-; CHECK32-NEXT:    testl %esi, %esi
-; CHECK32-NEXT:    setns %ch
-; CHECK32-NEXT:    cmpb %ch, %bl
-; CHECK32-NEXT:    setne %ch
-; CHECK32-NEXT:    testb %cl, %ch
-; CHECK32-NEXT:    cmovel %ebp, %edx
-; CHECK32-NEXT:    cmovel %edi, %eax
-; CHECK32-NEXT:    popl %esi
-; CHECK32-NEXT:    .cfi_def_cfa_offset 16
-; CHECK32-NEXT:    popl %edi
-; CHECK32-NEXT:    .cfi_def_cfa_offset 12
-; CHECK32-NEXT:    popl %ebx
-; CHECK32-NEXT:    .cfi_def_cfa_offset 8
-; CHECK32-NEXT:    popl %ebp
-; CHECK32-NEXT:    .cfi_def_cfa_offset 4
-; CHECK32-NEXT:    retl
+; X64-LABEL: func2:
+; X64:       # %bb.0:
+; X64-NEXT:    xorl %ecx, %ecx
+; X64-NEXT:    movq %rdi, %rax
+; X64-NEXT:    subq %rsi, %rax
+; X64-NEXT:    setns %cl
+; X64-NEXT:    movabsq $9223372036854775807, %rax # imm = 0x7FFFFFFFFFFFFFFF
+; X64-NEXT:    addq %rcx, %rax
+; X64-NEXT:    subq %rsi, %rdi
+; X64-NEXT:    cmovnoq %rdi, %rax
+; X64-NEXT:    retq
   %tmp = call i64 @llvm.ssub.sat.i64(i64 %x, i64 %y);
   ret i64 %tmp;
 }
 
-define i4 @func3(i4 %x, i4 %y) {
-; CHECK-LABEL: func3:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    movl %edi, %eax
-; CHECK-NEXT:    shlb $4, %sil
-; CHECK-NEXT:    shlb $4, %al
-; CHECK-NEXT:    movl %eax, %ecx
-; CHECK-NEXT:    subb %sil, %cl
-; CHECK-NEXT:    setns %cl
-; CHECK-NEXT:    subb %sil, %al
-; CHECK-NEXT:    jno .LBB2_2
-; CHECK-NEXT:  # %bb.1:
-; CHECK-NEXT:    addb $127, %cl
-; CHECK-NEXT:    movl %ecx, %eax
-; CHECK-NEXT:  .LBB2_2:
-; CHECK-NEXT:    sarb $4, %al
-; CHECK-NEXT:    # kill: def $al killed $al killed $eax
-; CHECK-NEXT:    retq
+define i4 @func3(i4 %x, i4 %y) nounwind {
+; X86-LABEL: func3:
+; X86:       # %bb.0:
+; X86-NEXT:    movb {{[0-9]+}}(%esp), %al
+; X86-NEXT:    movb {{[0-9]+}}(%esp), %dl
+; X86-NEXT:    shlb $4, %dl
+; X86-NEXT:    shlb $4, %al
+; X86-NEXT:    movl %eax, %ecx
+; X86-NEXT:    subb %dl, %cl
+; X86-NEXT:    setns %cl
+; X86-NEXT:    subb %dl, %al
+; X86-NEXT:    jno .LBB2_2
+; X86-NEXT:  # %bb.1:
+; X86-NEXT:    addb $127, %cl
+; X86-NEXT:    movl %ecx, %eax
+; X86-NEXT:  .LBB2_2:
+; X86-NEXT:    sarb $4, %al
+; X86-NEXT:    retl
 ;
-; CHECK32-LABEL: func3:
-; CHECK32:       # %bb.0:
-; CHECK32-NEXT:    movb {{[0-9]+}}(%esp), %al
-; CHECK32-NEXT:    movb {{[0-9]+}}(%esp), %dl
-; CHECK32-NEXT:    shlb $4, %dl
-; CHECK32-NEXT:    shlb $4, %al
-; CHECK32-NEXT:    movl %eax, %ecx
-; CHECK32-NEXT:    subb %dl, %cl
-; CHECK32-NEXT:    setns %cl
-; CHECK32-NEXT:    subb %dl, %al
-; CHECK32-NEXT:    jno .LBB2_2
-; CHECK32-NEXT:  # %bb.1:
-; CHECK32-NEXT:    addb $127, %cl
-; CHECK32-NEXT:    movl %ecx, %eax
-; CHECK32-NEXT:  .LBB2_2:
-; CHECK32-NEXT:    sarb $4, %al
-; CHECK32-NEXT:    retl
+; X64-LABEL: func3:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    shlb $4, %sil
+; X64-NEXT:    shlb $4, %al
+; X64-NEXT:    movl %eax, %ecx
+; X64-NEXT:    subb %sil, %cl
+; X64-NEXT:    setns %cl
+; X64-NEXT:    subb %sil, %al
+; X64-NEXT:    jno .LBB2_2
+; X64-NEXT:  # %bb.1:
+; X64-NEXT:    addb $127, %cl
+; X64-NEXT:    movl %ecx, %eax
+; X64-NEXT:  .LBB2_2:
+; X64-NEXT:    sarb $4, %al
+; X64-NEXT:    # kill: def $al killed $al killed $eax
+; X64-NEXT:    retq
   %tmp = call i4 @llvm.ssub.sat.i4(i4 %x, i4 %y);
   ret i4 %tmp;
 }
 
-define <4 x i32> @vec(<4 x i32> %x, <4 x i32> %y) {
-; CHECK-LABEL: vec:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,2,3]
-; CHECK-NEXT:    movd %xmm2, %ecx
-; CHECK-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,2,3]
-; CHECK-NEXT:    movd %xmm2, %r8d
-; CHECK-NEXT:    xorl %edx, %edx
-; CHECK-NEXT:    movl %r8d, %esi
-; CHECK-NEXT:    subl %ecx, %esi
-; CHECK-NEXT:    setns %dl
-; CHECK-NEXT:    addl $2147483647, %edx # imm = 0x7FFFFFFF
-; CHECK-NEXT:    subl %ecx, %r8d
-; CHECK-NEXT:    cmovol %edx, %r8d
-; CHECK-NEXT:    movd %xmm1, %edx
-; CHECK-NEXT:    movd %xmm0, %ecx
-; CHECK-NEXT:    xorl %esi, %esi
-; CHECK-NEXT:    movl %ecx, %edi
-; CHECK-NEXT:    subl %edx, %edi
-; CHECK-NEXT:    setns %sil
-; CHECK-NEXT:    addl $2147483647, %esi # imm = 0x7FFFFFFF
-; CHECK-NEXT:    subl %edx, %ecx
-; CHECK-NEXT:    cmovol %esi, %ecx
-; CHECK-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,0,1]
-; CHECK-NEXT:    movd %xmm2, %edx
-; CHECK-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
-; CHECK-NEXT:    movd %xmm2, %eax
-; CHECK-NEXT:    xorl %edi, %edi
-; CHECK-NEXT:    movl %eax, %esi
-; CHECK-NEXT:    subl %edx, %esi
-; CHECK-NEXT:    setns %dil
-; CHECK-NEXT:    addl $2147483647, %edi # imm = 0x7FFFFFFF
-; CHECK-NEXT:    subl %edx, %eax
-; CHECK-NEXT:    cmovol %edi, %eax
-; CHECK-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[3,1,2,3]
-; CHECK-NEXT:    movd %xmm1, %r9d
-; CHECK-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
-; CHECK-NEXT:    movd %xmm0, %edx
-; CHECK-NEXT:    xorl %edi, %edi
-; CHECK-NEXT:    movl %edx, %esi
-; CHECK-NEXT:    subl %r9d, %esi
-; CHECK-NEXT:    setns %dil
-; CHECK-NEXT:    addl $2147483647, %edi # imm = 0x7FFFFFFF
-; CHECK-NEXT:    subl %r9d, %edx
-; CHECK-NEXT:    cmovol %edi, %edx
-; CHECK-NEXT:    movd %edx, %xmm0
-; CHECK-NEXT:    movd %eax, %xmm1
-; CHECK-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
-; CHECK-NEXT:    movd %ecx, %xmm0
-; CHECK-NEXT:    movd %r8d, %xmm2
-; CHECK-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
-; CHECK-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; CHECK-NEXT:    retq
+define <4 x i32> @vec(<4 x i32> %x, <4 x i32> %y) nounwind {
+; X86-LABEL: vec:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %ebp
+; X86-NEXT:    pushl %ebx
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    xorl %eax, %eax
+; X86-NEXT:    movl %ecx, %esi
+; X86-NEXT:    subl %edx, %esi
+; X86-NEXT:    setns %al
+; X86-NEXT:    addl $2147483647, %eax # imm = 0x7FFFFFFF
+; X86-NEXT:    subl %edx, %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    cmovol %eax, %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    xorl %eax, %eax
+; X86-NEXT:    movl %edx, %edi
+; X86-NEXT:    subl %esi, %edi
+; X86-NEXT:    setns %al
+; X86-NEXT:    addl $2147483647, %eax # imm = 0x7FFFFFFF
+; X86-NEXT:    subl %esi, %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    cmovol %eax, %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    xorl %eax, %eax
+; X86-NEXT:    movl %esi, %ebx
+; X86-NEXT:    subl %edi, %ebx
+; X86-NEXT:    setns %al
+; X86-NEXT:    addl $2147483647, %eax # imm = 0x7FFFFFFF
+; X86-NEXT:    subl %edi, %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    cmovol %eax, %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    xorl %ebx, %ebx
+; X86-NEXT:    movl %edi, %ebp
+; X86-NEXT:    subl %eax, %ebp
+; X86-NEXT:    setns %bl
+; X86-NEXT:    addl $2147483647, %ebx # imm = 0x7FFFFFFF
+; X86-NEXT:    subl %eax, %edi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmovol %ebx, %edi
+; X86-NEXT:    movl %ecx, 12(%eax)
+; X86-NEXT:    movl %edx, 8(%eax)
+; X86-NEXT:    movl %esi, 4(%eax)
+; X86-NEXT:    movl %edi, (%eax)
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    popl %ebx
+; X86-NEXT:    popl %ebp
+; X86-NEXT:    retl $4
 ;
-; CHECK32-LABEL: vec:
-; CHECK32:       # %bb.0:
-; CHECK32-NEXT:    pushl %ebp
-; CHECK32-NEXT:    .cfi_def_cfa_offset 8
-; CHECK32-NEXT:    pushl %ebx
-; CHECK32-NEXT:    .cfi_def_cfa_offset 12
-; CHECK32-NEXT:    pushl %edi
-; CHECK32-NEXT:    .cfi_def_cfa_offset 16
-; CHECK32-NEXT:    pushl %esi
-; CHECK32-NEXT:    .cfi_def_cfa_offset 20
-; CHECK32-NEXT:    .cfi_offset %esi, -20
-; CHECK32-NEXT:    .cfi_offset %edi, -16
-; CHECK32-NEXT:    .cfi_offset %ebx, -12
-; CHECK32-NEXT:    .cfi_offset %ebp, -8
-; CHECK32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; CHECK32-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; CHECK32-NEXT:    xorl %eax, %eax
-; CHECK32-NEXT:    movl %ecx, %esi
-; CHECK32-NEXT:    subl %edx, %esi
-; CHECK32-NEXT:    setns %al
-; CHECK32-NEXT:    addl $2147483647, %eax # imm = 0x7FFFFFFF
-; CHECK32-NEXT:    subl %edx, %ecx
-; CHECK32-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; CHECK32-NEXT:    cmovol %eax, %ecx
-; CHECK32-NEXT:    movl {{[0-9]+}}(%esp), %esi
-; CHECK32-NEXT:    xorl %eax, %eax
-; CHECK32-NEXT:    movl %edx, %edi
-; CHECK32-NEXT:    subl %esi, %edi
-; CHECK32-NEXT:    setns %al
-; CHECK32-NEXT:    addl $2147483647, %eax # imm = 0x7FFFFFFF
-; CHECK32-NEXT:    subl %esi, %edx
-; CHECK32-NEXT:    movl {{[0-9]+}}(%esp), %esi
-; CHECK32-NEXT:    cmovol %eax, %edx
-; CHECK32-NEXT:    movl {{[0-9]+}}(%esp), %edi
-; CHECK32-NEXT:    xorl %eax, %eax
-; CHECK32-NEXT:    movl %esi, %ebx
-; CHECK32-NEXT:    subl %edi, %ebx
-; CHECK32-NEXT:    setns %al
-; CHECK32-NEXT:    addl $2147483647, %eax # imm = 0x7FFFFFFF
-; CHECK32-NEXT:    subl %edi, %esi
-; CHECK32-NEXT:    movl {{[0-9]+}}(%esp), %edi
-; CHECK32-NEXT:    cmovol %eax, %esi
-; CHECK32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; CHECK32-NEXT:    xorl %ebx, %ebx
-; CHECK32-NEXT:    movl %edi, %ebp
-; CHECK32-NEXT:    subl %eax, %ebp
-; CHECK32-NEXT:    setns %bl
-; CHECK32-NEXT:    addl $2147483647, %ebx # imm = 0x7FFFFFFF
-; CHECK32-NEXT:    subl %eax, %edi
-; CHECK32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; CHECK32-NEXT:    cmovol %ebx, %edi
-; CHECK32-NEXT:    movl %ecx, 12(%eax)
-; CHECK32-NEXT:    movl %edx, 8(%eax)
-; CHECK32-NEXT:    movl %esi, 4(%eax)
-; CHECK32-NEXT:    movl %edi, (%eax)
-; CHECK32-NEXT:    popl %esi
-; CHECK32-NEXT:    .cfi_def_cfa_offset 16
-; CHECK32-NEXT:    popl %edi
-; CHECK32-NEXT:    .cfi_def_cfa_offset 12
-; CHECK32-NEXT:    popl %ebx
-; CHECK32-NEXT:    .cfi_def_cfa_offset 8
-; CHECK32-NEXT:    popl %ebp
-; CHECK32-NEXT:    .cfi_def_cfa_offset 4
-; CHECK32-NEXT:    retl $4
+; X64-LABEL: vec:
+; X64:       # %bb.0:
+; X64-NEXT:    movdqa %xmm0, %xmm2
+; X64-NEXT:    pxor %xmm3, %xmm3
+; X64-NEXT:    pxor %xmm0, %xmm0
+; X64-NEXT:    pcmpgtd %xmm1, %xmm0
+; X64-NEXT:    pcmpeqd %xmm4, %xmm4
+; X64-NEXT:    pxor %xmm4, %xmm0
+; X64-NEXT:    pxor %xmm5, %xmm5
+; X64-NEXT:    pcmpgtd %xmm2, %xmm5
+; X64-NEXT:    pxor %xmm4, %xmm5
+; X64-NEXT:    pcmpeqd %xmm5, %xmm0
+; X64-NEXT:    psubd %xmm1, %xmm2
+; X64-NEXT:    pcmpgtd %xmm2, %xmm3
+; X64-NEXT:    movdqa %xmm3, %xmm1
+; X64-NEXT:    pxor %xmm4, %xmm1
+; X64-NEXT:    pcmpeqd %xmm5, %xmm1
+; X64-NEXT:    pxor %xmm4, %xmm1
+; X64-NEXT:    pandn %xmm1, %xmm0
+; X64-NEXT:    movdqa %xmm3, %xmm1
+; X64-NEXT:    pandn {{.*}}(%rip), %xmm1
+; X64-NEXT:    psrld $1, %xmm3
+; X64-NEXT:    por %xmm1, %xmm3
+; X64-NEXT:    pand %xmm0, %xmm3
+; X64-NEXT:    pandn %xmm2, %xmm0
+; X64-NEXT:    por %xmm3, %xmm0
+; X64-NEXT:    retq
   %tmp = call <4 x i32> @llvm.ssub.sat.v4i32(<4 x i32> %x, <4 x i32> %y);
   ret <4 x i32> %tmp;
 }
diff --git a/test/CodeGen/X86/ssub_sat_vec.ll b/test/CodeGen/X86/ssub_sat_vec.ll
index 28a8ee27a456b246239934dbcfadb2a5912d16c8..5ba7b173c51cd42dbf335520fe408910df51483c 100644
--- a/test/CodeGen/X86/ssub_sat_vec.ll
+++ b/test/CodeGen/X86/ssub_sat_vec.ll
@@ -26,8 +26,14 @@ declare <32 x i16> @llvm.ssub.sat.v32i16(<32 x i16>, <32 x i16>)
 declare <16 x i1> @llvm.ssub.sat.v16i1(<16 x i1>, <16 x i1>)
 declare <16 x i4> @llvm.ssub.sat.v16i4(<16 x i4>, <16 x i4>)
 
-declare <4 x i32> @llvm.ssub.sat.v4i32(<4 x i32>, <4 x i32>)
 declare <2 x i32> @llvm.ssub.sat.v2i32(<2 x i32>, <2 x i32>)
+declare <4 x i32> @llvm.ssub.sat.v4i32(<4 x i32>, <4 x i32>)
+declare <8 x i32> @llvm.ssub.sat.v8i32(<8 x i32>, <8 x i32>)
+declare <16 x i32> @llvm.ssub.sat.v16i32(<16 x i32>, <16 x i32>)
+declare <2 x i64> @llvm.ssub.sat.v2i64(<2 x i64>, <2 x i64>)
+declare <4 x i64> @llvm.ssub.sat.v4i64(<4 x i64>, <4 x i64>)
+declare <8 x i64> @llvm.ssub.sat.v8i64(<8 x i64>, <8 x i64>)
+
 declare <4 x i24> @llvm.ssub.sat.v4i24(<4 x i24>, <4 x i24>)
 declare <2 x i128> @llvm.ssub.sat.v2i128(<2 x i128>, <2 x i128>)
 
@@ -651,330 +657,230 @@ define <16 x i1> @v16i1(<16 x i1> %x, <16 x i1> %y) nounwind {
 
 ; Expanded
 
-define <4 x i32> @v4i32(<4 x i32> %x, <4 x i32> %y) nounwind {
-; SSE2-LABEL: v4i32:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,2,3]
-; SSE2-NEXT:    movd %xmm2, %ecx
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,2,3]
-; SSE2-NEXT:    movd %xmm2, %r8d
-; SSE2-NEXT:    xorl %edx, %edx
-; SSE2-NEXT:    movl %r8d, %esi
-; SSE2-NEXT:    subl %ecx, %esi
-; SSE2-NEXT:    setns %dl
-; SSE2-NEXT:    addl $2147483647, %edx # imm = 0x7FFFFFFF
-; SSE2-NEXT:    subl %ecx, %r8d
-; SSE2-NEXT:    cmovol %edx, %r8d
-; SSE2-NEXT:    movd %xmm1, %edx
-; SSE2-NEXT:    movd %xmm0, %ecx
-; SSE2-NEXT:    xorl %esi, %esi
-; SSE2-NEXT:    movl %ecx, %edi
-; SSE2-NEXT:    subl %edx, %edi
-; SSE2-NEXT:    setns %sil
-; SSE2-NEXT:    addl $2147483647, %esi # imm = 0x7FFFFFFF
-; SSE2-NEXT:    subl %edx, %ecx
-; SSE2-NEXT:    cmovol %esi, %ecx
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,0,1]
-; SSE2-NEXT:    movd %xmm2, %edx
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
-; SSE2-NEXT:    movd %xmm2, %eax
-; SSE2-NEXT:    xorl %edi, %edi
-; SSE2-NEXT:    movl %eax, %esi
-; SSE2-NEXT:    subl %edx, %esi
-; SSE2-NEXT:    setns %dil
-; SSE2-NEXT:    addl $2147483647, %edi # imm = 0x7FFFFFFF
-; SSE2-NEXT:    subl %edx, %eax
-; SSE2-NEXT:    cmovol %edi, %eax
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[3,1,2,3]
-; SSE2-NEXT:    movd %xmm1, %r9d
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
-; SSE2-NEXT:    movd %xmm0, %edx
-; SSE2-NEXT:    xorl %edi, %edi
-; SSE2-NEXT:    movl %edx, %esi
-; SSE2-NEXT:    subl %r9d, %esi
-; SSE2-NEXT:    setns %dil
-; SSE2-NEXT:    addl $2147483647, %edi # imm = 0x7FFFFFFF
-; SSE2-NEXT:    subl %r9d, %edx
-; SSE2-NEXT:    cmovol %edi, %edx
-; SSE2-NEXT:    movd %edx, %xmm0
-; SSE2-NEXT:    movd %eax, %xmm1
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
-; SSE2-NEXT:    movd %ecx, %xmm0
-; SSE2-NEXT:    movd %r8d, %xmm2
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
-; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; SSE2-NEXT:    retq
-;
-; SSSE3-LABEL: v4i32:
-; SSSE3:       # %bb.0:
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,2,3]
-; SSSE3-NEXT:    movd %xmm2, %ecx
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,2,3]
-; SSSE3-NEXT:    movd %xmm2, %r8d
-; SSSE3-NEXT:    xorl %edx, %edx
-; SSSE3-NEXT:    movl %r8d, %esi
-; SSSE3-NEXT:    subl %ecx, %esi
-; SSSE3-NEXT:    setns %dl
-; SSSE3-NEXT:    addl $2147483647, %edx # imm = 0x7FFFFFFF
-; SSSE3-NEXT:    subl %ecx, %r8d
-; SSSE3-NEXT:    cmovol %edx, %r8d
-; SSSE3-NEXT:    movd %xmm1, %edx
-; SSSE3-NEXT:    movd %xmm0, %ecx
-; SSSE3-NEXT:    xorl %esi, %esi
-; SSSE3-NEXT:    movl %ecx, %edi
-; SSSE3-NEXT:    subl %edx, %edi
-; SSSE3-NEXT:    setns %sil
-; SSSE3-NEXT:    addl $2147483647, %esi # imm = 0x7FFFFFFF
-; SSSE3-NEXT:    subl %edx, %ecx
-; SSSE3-NEXT:    cmovol %esi, %ecx
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,0,1]
-; SSSE3-NEXT:    movd %xmm2, %edx
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
-; SSSE3-NEXT:    movd %xmm2, %eax
-; SSSE3-NEXT:    xorl %edi, %edi
-; SSSE3-NEXT:    movl %eax, %esi
-; SSSE3-NEXT:    subl %edx, %esi
-; SSSE3-NEXT:    setns %dil
-; SSSE3-NEXT:    addl $2147483647, %edi # imm = 0x7FFFFFFF
-; SSSE3-NEXT:    subl %edx, %eax
-; SSSE3-NEXT:    cmovol %edi, %eax
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[3,1,2,3]
-; SSSE3-NEXT:    movd %xmm1, %r9d
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
-; SSSE3-NEXT:    movd %xmm0, %edx
-; SSSE3-NEXT:    xorl %edi, %edi
-; SSSE3-NEXT:    movl %edx, %esi
-; SSSE3-NEXT:    subl %r9d, %esi
-; SSSE3-NEXT:    setns %dil
-; SSSE3-NEXT:    addl $2147483647, %edi # imm = 0x7FFFFFFF
-; SSSE3-NEXT:    subl %r9d, %edx
-; SSSE3-NEXT:    cmovol %edi, %edx
-; SSSE3-NEXT:    movd %edx, %xmm0
-; SSSE3-NEXT:    movd %eax, %xmm1
-; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
-; SSSE3-NEXT:    movd %ecx, %xmm0
-; SSSE3-NEXT:    movd %r8d, %xmm2
-; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
-; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; SSSE3-NEXT:    retq
-;
-; SSE41-LABEL: v4i32:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    pextrd $3, %xmm1, %ecx
-; SSE41-NEXT:    pextrd $3, %xmm0, %r8d
-; SSE41-NEXT:    xorl %edx, %edx
-; SSE41-NEXT:    movl %r8d, %esi
-; SSE41-NEXT:    subl %ecx, %esi
-; SSE41-NEXT:    setns %dl
-; SSE41-NEXT:    addl $2147483647, %edx # imm = 0x7FFFFFFF
-; SSE41-NEXT:    subl %ecx, %r8d
-; SSE41-NEXT:    cmovol %edx, %r8d
-; SSE41-NEXT:    pextrd $2, %xmm1, %edx
-; SSE41-NEXT:    pextrd $2, %xmm0, %ecx
-; SSE41-NEXT:    xorl %esi, %esi
-; SSE41-NEXT:    movl %ecx, %edi
-; SSE41-NEXT:    subl %edx, %edi
-; SSE41-NEXT:    setns %sil
-; SSE41-NEXT:    addl $2147483647, %esi # imm = 0x7FFFFFFF
-; SSE41-NEXT:    subl %edx, %ecx
-; SSE41-NEXT:    cmovol %esi, %ecx
-; SSE41-NEXT:    movd %xmm1, %edx
-; SSE41-NEXT:    movd %xmm0, %eax
-; SSE41-NEXT:    xorl %edi, %edi
-; SSE41-NEXT:    movl %eax, %esi
-; SSE41-NEXT:    subl %edx, %esi
-; SSE41-NEXT:    setns %dil
-; SSE41-NEXT:    addl $2147483647, %edi # imm = 0x7FFFFFFF
-; SSE41-NEXT:    subl %edx, %eax
-; SSE41-NEXT:    cmovol %edi, %eax
-; SSE41-NEXT:    pextrd $1, %xmm1, %r9d
-; SSE41-NEXT:    pextrd $1, %xmm0, %edx
-; SSE41-NEXT:    xorl %edi, %edi
-; SSE41-NEXT:    movl %edx, %esi
-; SSE41-NEXT:    subl %r9d, %esi
-; SSE41-NEXT:    setns %dil
-; SSE41-NEXT:    addl $2147483647, %edi # imm = 0x7FFFFFFF
-; SSE41-NEXT:    subl %r9d, %edx
-; SSE41-NEXT:    cmovol %edi, %edx
-; SSE41-NEXT:    movd %eax, %xmm0
-; SSE41-NEXT:    pinsrd $1, %edx, %xmm0
-; SSE41-NEXT:    pinsrd $2, %ecx, %xmm0
-; SSE41-NEXT:    pinsrd $3, %r8d, %xmm0
-; SSE41-NEXT:    retq
-;
-; AVX-LABEL: v4i32:
-; AVX:       # %bb.0:
-; AVX-NEXT:    vpextrd $3, %xmm1, %ecx
-; AVX-NEXT:    vpextrd $3, %xmm0, %r9d
-; AVX-NEXT:    xorl %edx, %edx
-; AVX-NEXT:    movl %r9d, %esi
-; AVX-NEXT:    subl %ecx, %esi
-; AVX-NEXT:    setns %dl
-; AVX-NEXT:    addl $2147483647, %edx # imm = 0x7FFFFFFF
-; AVX-NEXT:    subl %ecx, %r9d
-; AVX-NEXT:    cmovol %edx, %r9d
-; AVX-NEXT:    vpextrd $2, %xmm1, %edx
-; AVX-NEXT:    vpextrd $2, %xmm0, %ecx
-; AVX-NEXT:    xorl %esi, %esi
-; AVX-NEXT:    movl %ecx, %edi
-; AVX-NEXT:    subl %edx, %edi
-; AVX-NEXT:    setns %sil
-; AVX-NEXT:    addl $2147483647, %esi # imm = 0x7FFFFFFF
-; AVX-NEXT:    subl %edx, %ecx
-; AVX-NEXT:    cmovol %esi, %ecx
-; AVX-NEXT:    vmovd %xmm1, %r8d
-; AVX-NEXT:    vmovd %xmm0, %edx
-; AVX-NEXT:    xorl %edi, %edi
-; AVX-NEXT:    movl %edx, %esi
-; AVX-NEXT:    subl %r8d, %esi
-; AVX-NEXT:    setns %dil
-; AVX-NEXT:    addl $2147483647, %edi # imm = 0x7FFFFFFF
-; AVX-NEXT:    subl %r8d, %edx
-; AVX-NEXT:    cmovol %edi, %edx
-; AVX-NEXT:    vpextrd $1, %xmm1, %r8d
-; AVX-NEXT:    vpextrd $1, %xmm0, %eax
-; AVX-NEXT:    xorl %esi, %esi
-; AVX-NEXT:    movl %eax, %edi
-; AVX-NEXT:    subl %r8d, %edi
-; AVX-NEXT:    setns %sil
-; AVX-NEXT:    addl $2147483647, %esi # imm = 0x7FFFFFFF
-; AVX-NEXT:    subl %r8d, %eax
-; AVX-NEXT:    cmovol %esi, %eax
-; AVX-NEXT:    vmovd %edx, %xmm0
-; AVX-NEXT:    vpinsrd $1, %eax, %xmm0, %xmm0
-; AVX-NEXT:    vpinsrd $2, %ecx, %xmm0, %xmm0
-; AVX-NEXT:    vpinsrd $3, %r9d, %xmm0, %xmm0
-; AVX-NEXT:    retq
-  %z = call <4 x i32> @llvm.ssub.sat.v4i32(<4 x i32> %x, <4 x i32> %y)
-  ret <4 x i32> %z
-}
-
 define <2 x i32> @v2i32(<2 x i32> %x, <2 x i32> %y) nounwind {
 ; SSE2-LABEL: v2i32:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    psllq $32, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,0,1]
-; SSE2-NEXT:    movq %xmm2, %rax
+; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648]
 ; SSE2-NEXT:    psllq $32, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
-; SSE2-NEXT:    movq %xmm2, %rcx
-; SSE2-NEXT:    xorl %edx, %edx
-; SSE2-NEXT:    movq %rcx, %rsi
-; SSE2-NEXT:    subq %rax, %rsi
-; SSE2-NEXT:    setns %dl
-; SSE2-NEXT:    movabsq $9223372036854775807, %r8 # imm = 0x7FFFFFFFFFFFFFFF
-; SSE2-NEXT:    addq %r8, %rdx
-; SSE2-NEXT:    subq %rax, %rcx
-; SSE2-NEXT:    cmovoq %rdx, %rcx
-; SSE2-NEXT:    movq %xmm1, %rax
-; SSE2-NEXT:    movq %xmm0, %rsi
-; SSE2-NEXT:    xorl %edi, %edi
-; SSE2-NEXT:    movq %rsi, %rdx
-; SSE2-NEXT:    subq %rax, %rdx
-; SSE2-NEXT:    setns %dil
-; SSE2-NEXT:    addq %r8, %rdi
-; SSE2-NEXT:    subq %rax, %rsi
-; SSE2-NEXT:    cmovoq %rdi, %rsi
-; SSE2-NEXT:    movq %rsi, %xmm1
-; SSE2-NEXT:    movq %rcx, %xmm0
-; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,3,2,3]
-; SSE2-NEXT:    psrad $31, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,3,2,3]
+; SSE2-NEXT:    movdqa %xmm0, %xmm3
+; SSE2-NEXT:    psubq %xmm1, %xmm0
+; SSE2-NEXT:    pxor %xmm2, %xmm1
+; SSE2-NEXT:    movdqa %xmm2, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm1, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm2, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSE2-NEXT:    pand %xmm5, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSE2-NEXT:    por %xmm1, %xmm4
+; SSE2-NEXT:    pcmpeqd %xmm5, %xmm5
+; SSE2-NEXT:    pxor %xmm5, %xmm4
+; SSE2-NEXT:    pxor %xmm2, %xmm3
+; SSE2-NEXT:    movdqa %xmm2, %xmm1
+; SSE2-NEXT:    pcmpgtd %xmm3, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm1[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm2, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
+; SSE2-NEXT:    pand %xmm6, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm1[1,1,3,3]
+; SSE2-NEXT:    por %xmm3, %xmm6
+; SSE2-NEXT:    pxor %xmm5, %xmm6
+; SSE2-NEXT:    pcmpeqd %xmm6, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm4[1,0,3,2]
+; SSE2-NEXT:    pand %xmm4, %xmm1
+; SSE2-NEXT:    movdqa %xmm0, %xmm3
+; SSE2-NEXT:    pxor %xmm2, %xmm3
+; SSE2-NEXT:    movdqa %xmm2, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm3, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm4[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm2, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
+; SSE2-NEXT:    pand %xmm7, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSE2-NEXT:    por %xmm3, %xmm4
+; SSE2-NEXT:    pxor %xmm5, %xmm4
+; SSE2-NEXT:    pcmpeqd %xmm6, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[1,0,3,2]
+; SSE2-NEXT:    pand %xmm4, %xmm3
+; SSE2-NEXT:    pxor %xmm5, %xmm3
+; SSE2-NEXT:    pandn %xmm3, %xmm1
+; SSE2-NEXT:    movdqa %xmm1, %xmm3
+; SSE2-NEXT:    pandn %xmm0, %xmm3
+; SSE2-NEXT:    por %xmm2, %xmm0
+; SSE2-NEXT:    movdqa %xmm2, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm0, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm2, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE2-NEXT:    pand %xmm5, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm4[1,1,3,3]
+; SSE2-NEXT:    por %xmm0, %xmm2
+; SSE2-NEXT:    movdqa %xmm2, %xmm0
+; SSE2-NEXT:    pandn {{.*}}(%rip), %xmm0
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm2
+; SSE2-NEXT:    por %xmm0, %xmm2
+; SSE2-NEXT:    pand %xmm1, %xmm2
+; SSE2-NEXT:    por %xmm3, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,3,2,3]
+; SSE2-NEXT:    psrad $31, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,3,2,3]
 ; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
 ; SSE2-NEXT:    retq
 ;
 ; SSSE3-LABEL: v2i32:
 ; SSSE3:       # %bb.0:
 ; SSSE3-NEXT:    psllq $32, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,0,1]
-; SSSE3-NEXT:    movq %xmm2, %rax
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648]
 ; SSSE3-NEXT:    psllq $32, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
-; SSSE3-NEXT:    movq %xmm2, %rcx
-; SSSE3-NEXT:    xorl %edx, %edx
-; SSSE3-NEXT:    movq %rcx, %rsi
-; SSSE3-NEXT:    subq %rax, %rsi
-; SSSE3-NEXT:    setns %dl
-; SSSE3-NEXT:    movabsq $9223372036854775807, %r8 # imm = 0x7FFFFFFFFFFFFFFF
-; SSSE3-NEXT:    addq %r8, %rdx
-; SSSE3-NEXT:    subq %rax, %rcx
-; SSSE3-NEXT:    cmovoq %rdx, %rcx
-; SSSE3-NEXT:    movq %xmm1, %rax
-; SSSE3-NEXT:    movq %xmm0, %rsi
-; SSSE3-NEXT:    xorl %edi, %edi
-; SSSE3-NEXT:    movq %rsi, %rdx
-; SSSE3-NEXT:    subq %rax, %rdx
-; SSSE3-NEXT:    setns %dil
-; SSSE3-NEXT:    addq %r8, %rdi
-; SSSE3-NEXT:    subq %rax, %rsi
-; SSSE3-NEXT:    cmovoq %rdi, %rsi
-; SSSE3-NEXT:    movq %rsi, %xmm1
-; SSSE3-NEXT:    movq %rcx, %xmm0
-; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,3,2,3]
-; SSSE3-NEXT:    psrad $31, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,3,2,3]
+; SSSE3-NEXT:    movdqa %xmm0, %xmm3
+; SSSE3-NEXT:    psubq %xmm1, %xmm0
+; SSSE3-NEXT:    pxor %xmm2, %xmm1
+; SSSE3-NEXT:    movdqa %xmm2, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm5, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    por %xmm1, %xmm4
+; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm5
+; SSSE3-NEXT:    pxor %xmm5, %xmm4
+; SSSE3-NEXT:    pxor %xmm2, %xmm3
+; SSSE3-NEXT:    movdqa %xmm2, %xmm1
+; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm1[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm3
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm6, %xmm3
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm1[1,1,3,3]
+; SSSE3-NEXT:    por %xmm3, %xmm6
+; SSSE3-NEXT:    pxor %xmm5, %xmm6
+; SSSE3-NEXT:    pcmpeqd %xmm6, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm4[1,0,3,2]
+; SSSE3-NEXT:    pand %xmm4, %xmm1
+; SSSE3-NEXT:    movdqa %xmm0, %xmm3
+; SSSE3-NEXT:    pxor %xmm2, %xmm3
+; SSSE3-NEXT:    movdqa %xmm2, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm4[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm3
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm7, %xmm3
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    por %xmm3, %xmm4
+; SSSE3-NEXT:    pxor %xmm5, %xmm4
+; SSSE3-NEXT:    pcmpeqd %xmm6, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[1,0,3,2]
+; SSSE3-NEXT:    pand %xmm4, %xmm3
+; SSSE3-NEXT:    pxor %xmm5, %xmm3
+; SSSE3-NEXT:    pandn %xmm3, %xmm1
+; SSSE3-NEXT:    movdqa %xmm1, %xmm3
+; SSSE3-NEXT:    pandn %xmm0, %xmm3
+; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    movdqa %xmm2, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm5, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    por %xmm0, %xmm2
+; SSSE3-NEXT:    movdqa %xmm2, %xmm0
+; SSSE3-NEXT:    pandn {{.*}}(%rip), %xmm0
+; SSSE3-NEXT:    pand {{.*}}(%rip), %xmm2
+; SSSE3-NEXT:    por %xmm0, %xmm2
+; SSSE3-NEXT:    pand %xmm1, %xmm2
+; SSSE3-NEXT:    por %xmm3, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,3,2,3]
+; SSSE3-NEXT:    psrad $31, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,3,2,3]
 ; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: v2i32:
 ; SSE41:       # %bb.0:
+; SSE41-NEXT:    movdqa %xmm0, %xmm2
 ; SSE41-NEXT:    psllq $32, %xmm1
-; SSE41-NEXT:    movq %xmm1, %rax
-; SSE41-NEXT:    psllq $32, %xmm0
-; SSE41-NEXT:    movq %xmm0, %rcx
-; SSE41-NEXT:    xorl %edx, %edx
-; SSE41-NEXT:    movq %rcx, %rsi
-; SSE41-NEXT:    subq %rax, %rsi
-; SSE41-NEXT:    setns %dl
-; SSE41-NEXT:    movabsq $9223372036854775807, %r8 # imm = 0x7FFFFFFFFFFFFFFF
-; SSE41-NEXT:    addq %r8, %rdx
-; SSE41-NEXT:    subq %rax, %rcx
-; SSE41-NEXT:    cmovoq %rdx, %rcx
-; SSE41-NEXT:    pextrq $1, %xmm1, %rax
-; SSE41-NEXT:    pextrq $1, %xmm0, %rsi
-; SSE41-NEXT:    xorl %edi, %edi
-; SSE41-NEXT:    movq %rsi, %rdx
-; SSE41-NEXT:    subq %rax, %rdx
-; SSE41-NEXT:    setns %dil
-; SSE41-NEXT:    addq %r8, %rdi
-; SSE41-NEXT:    subq %rax, %rsi
-; SSE41-NEXT:    cmovoq %rdi, %rsi
-; SSE41-NEXT:    movq %rsi, %xmm1
-; SSE41-NEXT:    movq %rcx, %xmm0
-; SSE41-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
-; SSE41-NEXT:    psrad $31, %xmm0
-; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3],xmm1[4,5],xmm0[6,7]
+; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483648,2147483648]
+; SSE41-NEXT:    psllq $32, %xmm2
+; SSE41-NEXT:    movdqa %xmm2, %xmm3
+; SSE41-NEXT:    psubq %xmm1, %xmm2
+; SSE41-NEXT:    pxor %xmm0, %xmm1
+; SSE41-NEXT:    movdqa %xmm0, %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm1, %xmm4
+; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
+; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm1[1,1,3,3]
+; SSE41-NEXT:    pand %xmm5, %xmm6
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm4[1,1,3,3]
+; SSE41-NEXT:    por %xmm6, %xmm1
+; SSE41-NEXT:    pcmpeqd %xmm4, %xmm4
+; SSE41-NEXT:    pxor %xmm4, %xmm1
+; SSE41-NEXT:    pxor %xmm0, %xmm3
+; SSE41-NEXT:    movdqa %xmm0, %xmm5
+; SSE41-NEXT:    pcmpgtd %xmm3, %xmm5
+; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm3
+; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
+; SSE41-NEXT:    pand %xmm6, %xmm3
+; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
+; SSE41-NEXT:    por %xmm3, %xmm5
+; SSE41-NEXT:    pxor %xmm4, %xmm5
+; SSE41-NEXT:    pcmpeqq %xmm5, %xmm1
+; SSE41-NEXT:    movdqa %xmm2, %xmm3
+; SSE41-NEXT:    pxor %xmm0, %xmm3
+; SSE41-NEXT:    movdqa %xmm0, %xmm6
+; SSE41-NEXT:    pcmpgtd %xmm3, %xmm6
+; SSE41-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm3
+; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
+; SSE41-NEXT:    pand %xmm7, %xmm3
+; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
+; SSE41-NEXT:    por %xmm3, %xmm6
+; SSE41-NEXT:    pxor %xmm4, %xmm6
+; SSE41-NEXT:    pcmpeqq %xmm5, %xmm6
+; SSE41-NEXT:    pxor %xmm4, %xmm6
+; SSE41-NEXT:    pandn %xmm6, %xmm1
+; SSE41-NEXT:    movdqa %xmm2, %xmm3
+; SSE41-NEXT:    por %xmm0, %xmm3
+; SSE41-NEXT:    movdqa %xmm0, %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm3, %xmm4
+; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm3
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
+; SSE41-NEXT:    pand %xmm5, %xmm0
+; SSE41-NEXT:    por %xmm4, %xmm0
+; SSE41-NEXT:    movapd {{.*#+}} xmm3 = [9223372036854775808,9223372036854775808]
+; SSE41-NEXT:    blendvpd %xmm0, {{.*}}(%rip), %xmm3
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm3, %xmm2
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,3,3]
+; SSE41-NEXT:    psrad $31, %xmm2
+; SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm0[0,1],xmm2[2,3],xmm0[4,5],xmm2[6,7]
+; SSE41-NEXT:    movdqa %xmm2, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: v2i32:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vpsllq $32, %xmm1, %xmm1
-; AVX1-NEXT:    vmovq %xmm1, %rax
+; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm2, %xmm3
+; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
+; AVX1-NEXT:    vpxor %xmm4, %xmm3, %xmm3
 ; AVX1-NEXT:    vpsllq $32, %xmm0, %xmm0
-; AVX1-NEXT:    vmovq %xmm0, %rcx
-; AVX1-NEXT:    xorl %edx, %edx
-; AVX1-NEXT:    movq %rcx, %rsi
-; AVX1-NEXT:    subq %rax, %rsi
-; AVX1-NEXT:    setns %dl
-; AVX1-NEXT:    movabsq $9223372036854775807, %r8 # imm = 0x7FFFFFFFFFFFFFFF
-; AVX1-NEXT:    addq %r8, %rdx
-; AVX1-NEXT:    subq %rax, %rcx
-; AVX1-NEXT:    cmovoq %rdx, %rcx
-; AVX1-NEXT:    vpextrq $1, %xmm1, %rax
-; AVX1-NEXT:    vpextrq $1, %xmm0, %rsi
-; AVX1-NEXT:    xorl %edi, %edi
-; AVX1-NEXT:    movq %rsi, %rdx
-; AVX1-NEXT:    subq %rax, %rdx
-; AVX1-NEXT:    setns %dil
-; AVX1-NEXT:    addq %r8, %rdi
-; AVX1-NEXT:    subq %rax, %rsi
-; AVX1-NEXT:    cmovoq %rdi, %rsi
-; AVX1-NEXT:    vmovq %rsi, %xmm0
-; AVX1-NEXT:    vmovq %rcx, %xmm1
-; AVX1-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; AVX1-NEXT:    vpcmpgtq %xmm0, %xmm2, %xmm5
+; AVX1-NEXT:    vpxor %xmm4, %xmm5, %xmm5
+; AVX1-NEXT:    vpcmpeqq %xmm3, %xmm5, %xmm3
+; AVX1-NEXT:    vpsubq %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpcmpgtq %xmm0, %xmm2, %xmm1
+; AVX1-NEXT:    vpxor %xmm4, %xmm1, %xmm1
+; AVX1-NEXT:    vpcmpeqq %xmm1, %xmm5, %xmm1
+; AVX1-NEXT:    vpxor %xmm4, %xmm1, %xmm1
+; AVX1-NEXT:    vpandn %xmm1, %xmm3, %xmm1
+; AVX1-NEXT:    vmovapd {{.*#+}} xmm2 = [9223372036854775808,9223372036854775808]
+; AVX1-NEXT:    vblendvpd %xmm0, {{.*}}(%rip), %xmm2, %xmm2
+; AVX1-NEXT:    vblendvpd %xmm1, %xmm2, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrad $31, %xmm0, %xmm1
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
 ; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3],xmm0[4,5],xmm1[6,7]
@@ -983,29 +889,23 @@ define <2 x i32> @v2i32(<2 x i32> %x, <2 x i32> %y) nounwind {
 ; AVX2-LABEL: v2i32:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpsllq $32, %xmm1, %xmm1
-; AVX2-NEXT:    vmovq %xmm1, %rax
+; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX2-NEXT:    vpcmpgtq %xmm1, %xmm2, %xmm3
+; AVX2-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
+; AVX2-NEXT:    vpxor %xmm4, %xmm3, %xmm3
 ; AVX2-NEXT:    vpsllq $32, %xmm0, %xmm0
-; AVX2-NEXT:    vmovq %xmm0, %rcx
-; AVX2-NEXT:    xorl %edx, %edx
-; AVX2-NEXT:    movq %rcx, %rsi
-; AVX2-NEXT:    subq %rax, %rsi
-; AVX2-NEXT:    setns %dl
-; AVX2-NEXT:    movabsq $9223372036854775807, %r8 # imm = 0x7FFFFFFFFFFFFFFF
-; AVX2-NEXT:    addq %r8, %rdx
-; AVX2-NEXT:    subq %rax, %rcx
-; AVX2-NEXT:    cmovoq %rdx, %rcx
-; AVX2-NEXT:    vpextrq $1, %xmm1, %rax
-; AVX2-NEXT:    vpextrq $1, %xmm0, %rsi
-; AVX2-NEXT:    xorl %edi, %edi
-; AVX2-NEXT:    movq %rsi, %rdx
-; AVX2-NEXT:    subq %rax, %rdx
-; AVX2-NEXT:    setns %dil
-; AVX2-NEXT:    addq %r8, %rdi
-; AVX2-NEXT:    subq %rax, %rsi
-; AVX2-NEXT:    cmovoq %rdi, %rsi
-; AVX2-NEXT:    vmovq %rsi, %xmm0
-; AVX2-NEXT:    vmovq %rcx, %xmm1
-; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; AVX2-NEXT:    vpcmpgtq %xmm0, %xmm2, %xmm5
+; AVX2-NEXT:    vpxor %xmm4, %xmm5, %xmm5
+; AVX2-NEXT:    vpcmpeqq %xmm3, %xmm5, %xmm3
+; AVX2-NEXT:    vpsubq %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpcmpgtq %xmm0, %xmm2, %xmm1
+; AVX2-NEXT:    vpxor %xmm4, %xmm1, %xmm1
+; AVX2-NEXT:    vpcmpeqq %xmm1, %xmm5, %xmm1
+; AVX2-NEXT:    vpxor %xmm4, %xmm1, %xmm1
+; AVX2-NEXT:    vpandn %xmm1, %xmm3, %xmm1
+; AVX2-NEXT:    vmovapd {{.*#+}} xmm2 = [9223372036854775808,9223372036854775808]
+; AVX2-NEXT:    vblendvpd %xmm0, {{.*}}(%rip), %xmm2, %xmm2
+; AVX2-NEXT:    vblendvpd %xmm1, %xmm2, %xmm0, %xmm0
 ; AVX2-NEXT:    vpsrad $31, %xmm0, %xmm1
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
 ; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3]
@@ -1014,243 +914,2121 @@ define <2 x i32> @v2i32(<2 x i32> %x, <2 x i32> %y) nounwind {
 ; AVX512-LABEL: v2i32:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpsllq $32, %xmm1, %xmm1
-; AVX512-NEXT:    vmovq %xmm1, %rax
+; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX512-NEXT:    vpcmpnltq %xmm2, %xmm1, %k0
 ; AVX512-NEXT:    vpsllq $32, %xmm0, %xmm0
-; AVX512-NEXT:    vmovq %xmm0, %rcx
-; AVX512-NEXT:    xorl %edx, %edx
-; AVX512-NEXT:    movq %rcx, %rsi
-; AVX512-NEXT:    subq %rax, %rsi
-; AVX512-NEXT:    setns %dl
-; AVX512-NEXT:    movabsq $9223372036854775807, %r8 # imm = 0x7FFFFFFFFFFFFFFF
-; AVX512-NEXT:    addq %r8, %rdx
-; AVX512-NEXT:    subq %rax, %rcx
-; AVX512-NEXT:    cmovoq %rdx, %rcx
-; AVX512-NEXT:    vpextrq $1, %xmm1, %rax
-; AVX512-NEXT:    vpextrq $1, %xmm0, %rsi
-; AVX512-NEXT:    xorl %edi, %edi
-; AVX512-NEXT:    movq %rsi, %rdx
-; AVX512-NEXT:    subq %rax, %rdx
-; AVX512-NEXT:    setns %dil
-; AVX512-NEXT:    addq %r8, %rdi
-; AVX512-NEXT:    subq %rax, %rsi
-; AVX512-NEXT:    cmovoq %rdi, %rsi
-; AVX512-NEXT:    vmovq %rsi, %xmm0
-; AVX512-NEXT:    vmovq %rcx, %xmm1
-; AVX512-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; AVX512-NEXT:    vpcmpnltq %xmm2, %xmm0, %k1
+; AVX512-NEXT:    kxorw %k0, %k1, %k0
+; AVX512-NEXT:    vpsubq %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vpcmpnltq %xmm2, %xmm0, %k2
+; AVX512-NEXT:    kxorw %k2, %k1, %k1
+; AVX512-NEXT:    kandw %k1, %k0, %k1
+; AVX512-NEXT:    vpcmpgtq %xmm0, %xmm2, %k2
+; AVX512-NEXT:    vmovdqa {{.*#+}} xmm1 = [9223372036854775808,9223372036854775808]
+; AVX512-NEXT:    vmovdqa64 {{.*}}(%rip), %xmm1 {%k2}
+; AVX512-NEXT:    vmovdqa64 %xmm1, %xmm0 {%k1}
 ; AVX512-NEXT:    vpsraq $32, %xmm0, %xmm0
 ; AVX512-NEXT:    retq
   %z = call <2 x i32> @llvm.ssub.sat.v2i32(<2 x i32> %x, <2 x i32> %y)
   ret <2 x i32> %z
 }
 
-define <4 x i24> @v4i24(<4 x i24> %x, <4 x i24> %y) nounwind {
-; SSE2-LABEL: v4i24:
+define <4 x i32> @v4i32(<4 x i32> %x, <4 x i32> %y) nounwind {
+; SSE2-LABEL: v4i32:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pslld $8, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,2,3]
-; SSE2-NEXT:    movd %xmm2, %ecx
-; SSE2-NEXT:    pslld $8, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,2,3]
-; SSE2-NEXT:    movd %xmm2, %r8d
-; SSE2-NEXT:    xorl %edx, %edx
-; SSE2-NEXT:    movl %r8d, %esi
-; SSE2-NEXT:    subl %ecx, %esi
-; SSE2-NEXT:    setns %dl
-; SSE2-NEXT:    addl $2147483647, %edx # imm = 0x7FFFFFFF
-; SSE2-NEXT:    subl %ecx, %r8d
-; SSE2-NEXT:    cmovol %edx, %r8d
-; SSE2-NEXT:    movd %xmm1, %edx
-; SSE2-NEXT:    movd %xmm0, %ecx
-; SSE2-NEXT:    xorl %esi, %esi
-; SSE2-NEXT:    movl %ecx, %edi
-; SSE2-NEXT:    subl %edx, %edi
-; SSE2-NEXT:    setns %sil
-; SSE2-NEXT:    addl $2147483647, %esi # imm = 0x7FFFFFFF
-; SSE2-NEXT:    subl %edx, %ecx
-; SSE2-NEXT:    cmovol %esi, %ecx
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,0,1]
-; SSE2-NEXT:    movd %xmm2, %edx
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
-; SSE2-NEXT:    movd %xmm2, %eax
-; SSE2-NEXT:    xorl %edi, %edi
-; SSE2-NEXT:    movl %eax, %esi
-; SSE2-NEXT:    subl %edx, %esi
-; SSE2-NEXT:    setns %dil
-; SSE2-NEXT:    addl $2147483647, %edi # imm = 0x7FFFFFFF
-; SSE2-NEXT:    subl %edx, %eax
-; SSE2-NEXT:    cmovol %edi, %eax
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[3,1,2,3]
-; SSE2-NEXT:    movd %xmm1, %r9d
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
-; SSE2-NEXT:    movd %xmm0, %edx
-; SSE2-NEXT:    xorl %edi, %edi
-; SSE2-NEXT:    movl %edx, %esi
-; SSE2-NEXT:    subl %r9d, %esi
-; SSE2-NEXT:    setns %dil
-; SSE2-NEXT:    addl $2147483647, %edi # imm = 0x7FFFFFFF
-; SSE2-NEXT:    subl %r9d, %edx
-; SSE2-NEXT:    cmovol %edi, %edx
-; SSE2-NEXT:    movd %edx, %xmm0
-; SSE2-NEXT:    movd %eax, %xmm1
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
-; SSE2-NEXT:    movd %ecx, %xmm0
-; SSE2-NEXT:    movd %r8d, %xmm2
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
-; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; SSE2-NEXT:    psrad $8, %xmm0
+; SSE2-NEXT:    movdqa %xmm0, %xmm2
+; SSE2-NEXT:    pxor %xmm3, %xmm3
+; SSE2-NEXT:    pxor %xmm0, %xmm0
+; SSE2-NEXT:    pcmpgtd %xmm1, %xmm0
+; SSE2-NEXT:    pcmpeqd %xmm4, %xmm4
+; SSE2-NEXT:    pxor %xmm4, %xmm0
+; SSE2-NEXT:    pxor %xmm5, %xmm5
+; SSE2-NEXT:    pcmpgtd %xmm2, %xmm5
+; SSE2-NEXT:    pxor %xmm4, %xmm5
+; SSE2-NEXT:    pcmpeqd %xmm5, %xmm0
+; SSE2-NEXT:    psubd %xmm1, %xmm2
+; SSE2-NEXT:    pcmpgtd %xmm2, %xmm3
+; SSE2-NEXT:    movdqa %xmm3, %xmm1
+; SSE2-NEXT:    pxor %xmm4, %xmm1
+; SSE2-NEXT:    pcmpeqd %xmm5, %xmm1
+; SSE2-NEXT:    pxor %xmm4, %xmm1
+; SSE2-NEXT:    pandn %xmm1, %xmm0
+; SSE2-NEXT:    movdqa %xmm3, %xmm1
+; SSE2-NEXT:    pandn {{.*}}(%rip), %xmm1
+; SSE2-NEXT:    psrld $1, %xmm3
+; SSE2-NEXT:    por %xmm1, %xmm3
+; SSE2-NEXT:    pand %xmm0, %xmm3
+; SSE2-NEXT:    pandn %xmm2, %xmm0
+; SSE2-NEXT:    por %xmm3, %xmm0
 ; SSE2-NEXT:    retq
 ;
-; SSSE3-LABEL: v4i24:
+; SSSE3-LABEL: v4i32:
 ; SSSE3:       # %bb.0:
-; SSSE3-NEXT:    pslld $8, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,2,3]
-; SSSE3-NEXT:    movd %xmm2, %ecx
-; SSSE3-NEXT:    pslld $8, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,2,3]
-; SSSE3-NEXT:    movd %xmm2, %r8d
-; SSSE3-NEXT:    xorl %edx, %edx
-; SSSE3-NEXT:    movl %r8d, %esi
-; SSSE3-NEXT:    subl %ecx, %esi
-; SSSE3-NEXT:    setns %dl
-; SSSE3-NEXT:    addl $2147483647, %edx # imm = 0x7FFFFFFF
-; SSSE3-NEXT:    subl %ecx, %r8d
-; SSSE3-NEXT:    cmovol %edx, %r8d
-; SSSE3-NEXT:    movd %xmm1, %edx
-; SSSE3-NEXT:    movd %xmm0, %ecx
-; SSSE3-NEXT:    xorl %esi, %esi
-; SSSE3-NEXT:    movl %ecx, %edi
-; SSSE3-NEXT:    subl %edx, %edi
-; SSSE3-NEXT:    setns %sil
-; SSSE3-NEXT:    addl $2147483647, %esi # imm = 0x7FFFFFFF
-; SSSE3-NEXT:    subl %edx, %ecx
-; SSSE3-NEXT:    cmovol %esi, %ecx
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[2,3,0,1]
-; SSSE3-NEXT:    movd %xmm2, %edx
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[2,3,0,1]
-; SSSE3-NEXT:    movd %xmm2, %eax
-; SSSE3-NEXT:    xorl %edi, %edi
-; SSSE3-NEXT:    movl %eax, %esi
-; SSSE3-NEXT:    subl %edx, %esi
-; SSSE3-NEXT:    setns %dil
-; SSSE3-NEXT:    addl $2147483647, %edi # imm = 0x7FFFFFFF
-; SSSE3-NEXT:    subl %edx, %eax
-; SSSE3-NEXT:    cmovol %edi, %eax
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[3,1,2,3]
-; SSSE3-NEXT:    movd %xmm1, %r9d
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
-; SSSE3-NEXT:    movd %xmm0, %edx
-; SSSE3-NEXT:    xorl %edi, %edi
-; SSSE3-NEXT:    movl %edx, %esi
-; SSSE3-NEXT:    subl %r9d, %esi
-; SSSE3-NEXT:    setns %dil
-; SSSE3-NEXT:    addl $2147483647, %edi # imm = 0x7FFFFFFF
-; SSSE3-NEXT:    subl %r9d, %edx
-; SSSE3-NEXT:    cmovol %edi, %edx
-; SSSE3-NEXT:    movd %edx, %xmm0
-; SSSE3-NEXT:    movd %eax, %xmm1
-; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
-; SSSE3-NEXT:    movd %ecx, %xmm0
-; SSSE3-NEXT:    movd %r8d, %xmm2
-; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
-; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; SSSE3-NEXT:    psrad $8, %xmm0
+; SSSE3-NEXT:    movdqa %xmm0, %xmm2
+; SSSE3-NEXT:    pxor %xmm3, %xmm3
+; SSSE3-NEXT:    pxor %xmm0, %xmm0
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm0
+; SSSE3-NEXT:    pcmpeqd %xmm4, %xmm4
+; SSSE3-NEXT:    pxor %xmm4, %xmm0
+; SSSE3-NEXT:    pxor %xmm5, %xmm5
+; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm5
+; SSSE3-NEXT:    pxor %xmm4, %xmm5
+; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm0
+; SSSE3-NEXT:    psubd %xmm1, %xmm2
+; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm3
+; SSSE3-NEXT:    movdqa %xmm3, %xmm1
+; SSSE3-NEXT:    pxor %xmm4, %xmm1
+; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm1
+; SSSE3-NEXT:    pxor %xmm4, %xmm1
+; SSSE3-NEXT:    pandn %xmm1, %xmm0
+; SSSE3-NEXT:    movdqa %xmm3, %xmm1
+; SSSE3-NEXT:    pandn {{.*}}(%rip), %xmm1
+; SSSE3-NEXT:    psrld $1, %xmm3
+; SSSE3-NEXT:    por %xmm1, %xmm3
+; SSSE3-NEXT:    pand %xmm0, %xmm3
+; SSSE3-NEXT:    pandn %xmm2, %xmm0
+; SSSE3-NEXT:    por %xmm3, %xmm0
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: v4i32:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movdqa %xmm0, %xmm2
+; SSE41-NEXT:    pxor %xmm0, %xmm0
+; SSE41-NEXT:    pxor %xmm3, %xmm3
+; SSE41-NEXT:    pcmpgtd %xmm1, %xmm3
+; SSE41-NEXT:    pcmpeqd %xmm4, %xmm4
+; SSE41-NEXT:    pxor %xmm4, %xmm3
+; SSE41-NEXT:    pxor %xmm5, %xmm5
+; SSE41-NEXT:    pcmpgtd %xmm2, %xmm5
+; SSE41-NEXT:    pxor %xmm4, %xmm5
+; SSE41-NEXT:    pcmpeqd %xmm5, %xmm3
+; SSE41-NEXT:    psubd %xmm1, %xmm2
+; SSE41-NEXT:    pcmpgtd %xmm2, %xmm0
+; SSE41-NEXT:    pxor %xmm4, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm5, %xmm0
+; SSE41-NEXT:    pxor %xmm4, %xmm0
+; SSE41-NEXT:    pandn %xmm0, %xmm3
+; SSE41-NEXT:    movaps {{.*#+}} xmm1 = [2147483648,2147483648,2147483648,2147483648]
+; SSE41-NEXT:    movdqa %xmm2, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, {{.*}}(%rip), %xmm1
+; SSE41-NEXT:    movdqa %xmm3, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm1, %xmm2
+; SSE41-NEXT:    movaps %xmm2, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: v4i32:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm2, %xmm3
+; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
+; AVX1-NEXT:    vpxor %xmm4, %xmm3, %xmm3
+; AVX1-NEXT:    vpcmpgtd %xmm0, %xmm2, %xmm5
+; AVX1-NEXT:    vpxor %xmm4, %xmm5, %xmm5
+; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm5, %xmm3
+; AVX1-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpcmpgtd %xmm0, %xmm2, %xmm1
+; AVX1-NEXT:    vpxor %xmm4, %xmm1, %xmm1
+; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm5, %xmm1
+; AVX1-NEXT:    vpxor %xmm4, %xmm1, %xmm1
+; AVX1-NEXT:    vpandn %xmm1, %xmm3, %xmm1
+; AVX1-NEXT:    vmovaps {{.*#+}} xmm2 = [2147483648,2147483648,2147483648,2147483648]
+; AVX1-NEXT:    vblendvps %xmm0, {{.*}}(%rip), %xmm2, %xmm2
+; AVX1-NEXT:    vblendvps %xmm1, %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: v4i32:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX2-NEXT:    vpcmpgtd %xmm1, %xmm2, %xmm3
+; AVX2-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
+; AVX2-NEXT:    vpxor %xmm4, %xmm3, %xmm3
+; AVX2-NEXT:    vpcmpgtd %xmm0, %xmm2, %xmm5
+; AVX2-NEXT:    vpxor %xmm4, %xmm5, %xmm5
+; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm5, %xmm3
+; AVX2-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpcmpgtd %xmm0, %xmm2, %xmm1
+; AVX2-NEXT:    vpxor %xmm4, %xmm1, %xmm1
+; AVX2-NEXT:    vpcmpeqd %xmm1, %xmm5, %xmm1
+; AVX2-NEXT:    vpxor %xmm4, %xmm1, %xmm1
+; AVX2-NEXT:    vpandn %xmm1, %xmm3, %xmm1
+; AVX2-NEXT:    vbroadcastss {{.*#+}} xmm2 = [2147483647,2147483647,2147483647,2147483647]
+; AVX2-NEXT:    vbroadcastss {{.*#+}} xmm3 = [2147483648,2147483648,2147483648,2147483648]
+; AVX2-NEXT:    vblendvps %xmm0, %xmm2, %xmm3, %xmm2
+; AVX2-NEXT:    vblendvps %xmm1, %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: v4i32:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX512-NEXT:    vpcmpnltd %xmm2, %xmm1, %k0
+; AVX512-NEXT:    vpcmpnltd %xmm2, %xmm0, %k1
+; AVX512-NEXT:    kxorw %k0, %k1, %k0
+; AVX512-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vpcmpnltd %xmm2, %xmm0, %k2
+; AVX512-NEXT:    kxorw %k2, %k1, %k1
+; AVX512-NEXT:    kandw %k1, %k0, %k1
+; AVX512-NEXT:    vpcmpgtd %xmm0, %xmm2, %k2
+; AVX512-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [2147483648,2147483648,2147483648,2147483648]
+; AVX512-NEXT:    vpbroadcastd {{.*}}(%rip), %xmm1 {%k2}
+; AVX512-NEXT:    vmovdqa32 %xmm1, %xmm0 {%k1}
+; AVX512-NEXT:    retq
+  %z = call <4 x i32> @llvm.ssub.sat.v4i32(<4 x i32> %x, <4 x i32> %y)
+  ret <4 x i32> %z
+}
+
+define <8 x i32> @v8i32(<8 x i32> %x, <8 x i32> %y) nounwind {
+; SSE2-LABEL: v8i32:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa %xmm0, %xmm4
+; SSE2-NEXT:    pxor %xmm5, %xmm5
+; SSE2-NEXT:    pxor %xmm0, %xmm0
+; SSE2-NEXT:    pcmpgtd %xmm2, %xmm0
+; SSE2-NEXT:    pcmpeqd %xmm8, %xmm8
+; SSE2-NEXT:    pxor %xmm8, %xmm0
+; SSE2-NEXT:    pxor %xmm7, %xmm7
+; SSE2-NEXT:    pcmpgtd %xmm4, %xmm7
+; SSE2-NEXT:    pxor %xmm8, %xmm7
+; SSE2-NEXT:    pcmpeqd %xmm7, %xmm0
+; SSE2-NEXT:    psubd %xmm2, %xmm4
+; SSE2-NEXT:    pxor %xmm6, %xmm6
+; SSE2-NEXT:    pcmpgtd %xmm4, %xmm6
+; SSE2-NEXT:    movdqa %xmm6, %xmm2
+; SSE2-NEXT:    pxor %xmm8, %xmm2
+; SSE2-NEXT:    pcmpeqd %xmm7, %xmm2
+; SSE2-NEXT:    pxor %xmm8, %xmm2
+; SSE2-NEXT:    pandn %xmm2, %xmm0
+; SSE2-NEXT:    movdqa {{.*#+}} xmm7 = [2147483648,2147483648,2147483648,2147483648]
+; SSE2-NEXT:    movdqa %xmm6, %xmm2
+; SSE2-NEXT:    pandn %xmm7, %xmm2
+; SSE2-NEXT:    psrld $1, %xmm6
+; SSE2-NEXT:    por %xmm2, %xmm6
+; SSE2-NEXT:    pand %xmm0, %xmm6
+; SSE2-NEXT:    pandn %xmm4, %xmm0
+; SSE2-NEXT:    por %xmm6, %xmm0
+; SSE2-NEXT:    pxor %xmm2, %xmm2
+; SSE2-NEXT:    pcmpgtd %xmm3, %xmm2
+; SSE2-NEXT:    pxor %xmm8, %xmm2
+; SSE2-NEXT:    pxor %xmm4, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm1, %xmm4
+; SSE2-NEXT:    pxor %xmm8, %xmm4
+; SSE2-NEXT:    pcmpeqd %xmm4, %xmm2
+; SSE2-NEXT:    psubd %xmm3, %xmm1
+; SSE2-NEXT:    pcmpgtd %xmm1, %xmm5
+; SSE2-NEXT:    movdqa %xmm5, %xmm3
+; SSE2-NEXT:    pxor %xmm8, %xmm3
+; SSE2-NEXT:    pcmpeqd %xmm4, %xmm3
+; SSE2-NEXT:    pxor %xmm8, %xmm3
+; SSE2-NEXT:    pandn %xmm3, %xmm2
+; SSE2-NEXT:    movdqa %xmm5, %xmm3
+; SSE2-NEXT:    pandn %xmm7, %xmm3
+; SSE2-NEXT:    psrld $1, %xmm5
+; SSE2-NEXT:    por %xmm3, %xmm5
+; SSE2-NEXT:    pand %xmm2, %xmm5
+; SSE2-NEXT:    pandn %xmm1, %xmm2
+; SSE2-NEXT:    por %xmm5, %xmm2
+; SSE2-NEXT:    movdqa %xmm2, %xmm1
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: v8i32:
+; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    movdqa %xmm0, %xmm4
+; SSSE3-NEXT:    pxor %xmm5, %xmm5
+; SSSE3-NEXT:    pxor %xmm0, %xmm0
+; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm0
+; SSSE3-NEXT:    pcmpeqd %xmm8, %xmm8
+; SSSE3-NEXT:    pxor %xmm8, %xmm0
+; SSSE3-NEXT:    pxor %xmm7, %xmm7
+; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm7
+; SSSE3-NEXT:    pxor %xmm8, %xmm7
+; SSSE3-NEXT:    pcmpeqd %xmm7, %xmm0
+; SSSE3-NEXT:    psubd %xmm2, %xmm4
+; SSSE3-NEXT:    pxor %xmm6, %xmm6
+; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm6
+; SSSE3-NEXT:    movdqa %xmm6, %xmm2
+; SSSE3-NEXT:    pxor %xmm8, %xmm2
+; SSSE3-NEXT:    pcmpeqd %xmm7, %xmm2
+; SSSE3-NEXT:    pxor %xmm8, %xmm2
+; SSSE3-NEXT:    pandn %xmm2, %xmm0
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm7 = [2147483648,2147483648,2147483648,2147483648]
+; SSSE3-NEXT:    movdqa %xmm6, %xmm2
+; SSSE3-NEXT:    pandn %xmm7, %xmm2
+; SSSE3-NEXT:    psrld $1, %xmm6
+; SSSE3-NEXT:    por %xmm2, %xmm6
+; SSSE3-NEXT:    pand %xmm0, %xmm6
+; SSSE3-NEXT:    pandn %xmm4, %xmm0
+; SSSE3-NEXT:    por %xmm6, %xmm0
+; SSSE3-NEXT:    pxor %xmm2, %xmm2
+; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm2
+; SSSE3-NEXT:    pxor %xmm8, %xmm2
+; SSSE3-NEXT:    pxor %xmm4, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm4
+; SSSE3-NEXT:    pxor %xmm8, %xmm4
+; SSSE3-NEXT:    pcmpeqd %xmm4, %xmm2
+; SSSE3-NEXT:    psubd %xmm3, %xmm1
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm5
+; SSSE3-NEXT:    movdqa %xmm5, %xmm3
+; SSSE3-NEXT:    pxor %xmm8, %xmm3
+; SSSE3-NEXT:    pcmpeqd %xmm4, %xmm3
+; SSSE3-NEXT:    pxor %xmm8, %xmm3
+; SSSE3-NEXT:    pandn %xmm3, %xmm2
+; SSSE3-NEXT:    movdqa %xmm5, %xmm3
+; SSSE3-NEXT:    pandn %xmm7, %xmm3
+; SSSE3-NEXT:    psrld $1, %xmm5
+; SSSE3-NEXT:    por %xmm3, %xmm5
+; SSSE3-NEXT:    pand %xmm2, %xmm5
+; SSSE3-NEXT:    pandn %xmm1, %xmm2
+; SSSE3-NEXT:    por %xmm5, %xmm2
+; SSSE3-NEXT:    movdqa %xmm2, %xmm1
 ; SSSE3-NEXT:    retq
 ;
-; SSE41-LABEL: v4i24:
+; SSE41-LABEL: v8i32:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    pslld $8, %xmm1
-; SSE41-NEXT:    pextrd $3, %xmm1, %ecx
-; SSE41-NEXT:    pslld $8, %xmm0
-; SSE41-NEXT:    pextrd $3, %xmm0, %r8d
-; SSE41-NEXT:    xorl %edx, %edx
-; SSE41-NEXT:    movl %r8d, %esi
-; SSE41-NEXT:    subl %ecx, %esi
-; SSE41-NEXT:    setns %dl
-; SSE41-NEXT:    addl $2147483647, %edx # imm = 0x7FFFFFFF
-; SSE41-NEXT:    subl %ecx, %r8d
-; SSE41-NEXT:    cmovol %edx, %r8d
-; SSE41-NEXT:    pextrd $2, %xmm1, %edx
-; SSE41-NEXT:    pextrd $2, %xmm0, %ecx
-; SSE41-NEXT:    xorl %esi, %esi
-; SSE41-NEXT:    movl %ecx, %edi
-; SSE41-NEXT:    subl %edx, %edi
-; SSE41-NEXT:    setns %sil
-; SSE41-NEXT:    addl $2147483647, %esi # imm = 0x7FFFFFFF
-; SSE41-NEXT:    subl %edx, %ecx
-; SSE41-NEXT:    cmovol %esi, %ecx
-; SSE41-NEXT:    movd %xmm1, %edx
-; SSE41-NEXT:    movd %xmm0, %eax
-; SSE41-NEXT:    xorl %edi, %edi
-; SSE41-NEXT:    movl %eax, %esi
-; SSE41-NEXT:    subl %edx, %esi
-; SSE41-NEXT:    setns %dil
-; SSE41-NEXT:    addl $2147483647, %edi # imm = 0x7FFFFFFF
-; SSE41-NEXT:    subl %edx, %eax
-; SSE41-NEXT:    cmovol %edi, %eax
-; SSE41-NEXT:    pextrd $1, %xmm1, %r9d
-; SSE41-NEXT:    pextrd $1, %xmm0, %edx
-; SSE41-NEXT:    xorl %edi, %edi
-; SSE41-NEXT:    movl %edx, %esi
-; SSE41-NEXT:    subl %r9d, %esi
-; SSE41-NEXT:    setns %dil
-; SSE41-NEXT:    addl $2147483647, %edi # imm = 0x7FFFFFFF
-; SSE41-NEXT:    subl %r9d, %edx
-; SSE41-NEXT:    cmovol %edi, %edx
-; SSE41-NEXT:    movd %eax, %xmm0
-; SSE41-NEXT:    pinsrd $1, %edx, %xmm0
-; SSE41-NEXT:    pinsrd $2, %ecx, %xmm0
-; SSE41-NEXT:    pinsrd $3, %r8d, %xmm0
-; SSE41-NEXT:    psrad $8, %xmm0
+; SSE41-NEXT:    movdqa %xmm0, %xmm4
+; SSE41-NEXT:    pxor %xmm6, %xmm6
+; SSE41-NEXT:    pxor %xmm5, %xmm5
+; SSE41-NEXT:    pcmpgtd %xmm2, %xmm5
+; SSE41-NEXT:    pcmpeqd %xmm8, %xmm8
+; SSE41-NEXT:    pxor %xmm8, %xmm5
+; SSE41-NEXT:    pxor %xmm0, %xmm0
+; SSE41-NEXT:    pcmpgtd %xmm4, %xmm0
+; SSE41-NEXT:    pxor %xmm8, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm5
+; SSE41-NEXT:    psubd %xmm2, %xmm4
+; SSE41-NEXT:    pxor %xmm2, %xmm2
+; SSE41-NEXT:    pcmpgtd %xmm4, %xmm2
+; SSE41-NEXT:    pxor %xmm8, %xmm2
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm2
+; SSE41-NEXT:    pxor %xmm8, %xmm2
+; SSE41-NEXT:    pandn %xmm2, %xmm5
+; SSE41-NEXT:    movaps {{.*#+}} xmm9 = [2147483647,2147483647,2147483647,2147483647]
+; SSE41-NEXT:    movaps {{.*#+}} xmm7 = [2147483648,2147483648,2147483648,2147483648]
+; SSE41-NEXT:    movaps %xmm7, %xmm2
+; SSE41-NEXT:    movdqa %xmm4, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm9, %xmm2
+; SSE41-NEXT:    movdqa %xmm5, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm2, %xmm4
+; SSE41-NEXT:    xorps %xmm2, %xmm2
+; SSE41-NEXT:    pcmpgtd %xmm3, %xmm2
+; SSE41-NEXT:    pxor %xmm8, %xmm2
+; SSE41-NEXT:    pxor %xmm0, %xmm0
+; SSE41-NEXT:    pcmpgtd %xmm1, %xmm0
+; SSE41-NEXT:    pxor %xmm8, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm2
+; SSE41-NEXT:    psubd %xmm3, %xmm1
+; SSE41-NEXT:    pcmpgtd %xmm1, %xmm6
+; SSE41-NEXT:    pxor %xmm8, %xmm6
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm6
+; SSE41-NEXT:    pxor %xmm8, %xmm6
+; SSE41-NEXT:    pandn %xmm6, %xmm2
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm9, %xmm7
+; SSE41-NEXT:    movdqa %xmm2, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm7, %xmm1
+; SSE41-NEXT:    movaps %xmm4, %xmm0
 ; SSE41-NEXT:    retq
 ;
-; AVX-LABEL: v4i24:
-; AVX:       # %bb.0:
-; AVX-NEXT:    vpslld $8, %xmm1, %xmm1
-; AVX-NEXT:    vpextrd $3, %xmm1, %ecx
-; AVX-NEXT:    vpslld $8, %xmm0, %xmm0
-; AVX-NEXT:    vpextrd $3, %xmm0, %r9d
-; AVX-NEXT:    xorl %edx, %edx
-; AVX-NEXT:    movl %r9d, %esi
-; AVX-NEXT:    subl %ecx, %esi
-; AVX-NEXT:    setns %dl
-; AVX-NEXT:    addl $2147483647, %edx # imm = 0x7FFFFFFF
-; AVX-NEXT:    subl %ecx, %r9d
-; AVX-NEXT:    cmovol %edx, %r9d
-; AVX-NEXT:    vpextrd $2, %xmm1, %edx
-; AVX-NEXT:    vpextrd $2, %xmm0, %ecx
-; AVX-NEXT:    xorl %esi, %esi
-; AVX-NEXT:    movl %ecx, %edi
-; AVX-NEXT:    subl %edx, %edi
-; AVX-NEXT:    setns %sil
-; AVX-NEXT:    addl $2147483647, %esi # imm = 0x7FFFFFFF
-; AVX-NEXT:    subl %edx, %ecx
-; AVX-NEXT:    cmovol %esi, %ecx
-; AVX-NEXT:    vmovd %xmm1, %r8d
-; AVX-NEXT:    vmovd %xmm0, %edx
-; AVX-NEXT:    xorl %edi, %edi
-; AVX-NEXT:    movl %edx, %esi
-; AVX-NEXT:    subl %r8d, %esi
-; AVX-NEXT:    setns %dil
-; AVX-NEXT:    addl $2147483647, %edi # imm = 0x7FFFFFFF
-; AVX-NEXT:    subl %r8d, %edx
-; AVX-NEXT:    cmovol %edi, %edx
-; AVX-NEXT:    vpextrd $1, %xmm1, %r8d
-; AVX-NEXT:    vpextrd $1, %xmm0, %eax
-; AVX-NEXT:    xorl %esi, %esi
-; AVX-NEXT:    movl %eax, %edi
-; AVX-NEXT:    subl %r8d, %edi
-; AVX-NEXT:    setns %sil
-; AVX-NEXT:    addl $2147483647, %esi # imm = 0x7FFFFFFF
-; AVX-NEXT:    subl %r8d, %eax
-; AVX-NEXT:    cmovol %esi, %eax
-; AVX-NEXT:    vmovd %edx, %xmm0
-; AVX-NEXT:    vpinsrd $1, %eax, %xmm0, %xmm0
-; AVX-NEXT:    vpinsrd $2, %ecx, %xmm0, %xmm0
-; AVX-NEXT:    vpinsrd $3, %r9d, %xmm0, %xmm0
-; AVX-NEXT:    vpsrad $8, %xmm0, %xmm0
-; AVX-NEXT:    retq
-  %z = call <4 x i24> @llvm.ssub.sat.v4i24(<4 x i24> %x, <4 x i24> %y)
-  ret <4 x i24> %z
+; AVX1-LABEL: v8i32:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm9
+; AVX1-NEXT:    vpxor %xmm10, %xmm10, %xmm10
+; AVX1-NEXT:    vpcmpgtd %xmm9, %xmm10, %xmm4
+; AVX1-NEXT:    vpcmpeqd %xmm5, %xmm5, %xmm5
+; AVX1-NEXT:    vpxor %xmm5, %xmm4, %xmm4
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm6
+; AVX1-NEXT:    vpcmpgtd %xmm6, %xmm10, %xmm7
+; AVX1-NEXT:    vpxor %xmm5, %xmm7, %xmm7
+; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm7, %xmm4
+; AVX1-NEXT:    vpxor %xmm5, %xmm4, %xmm8
+; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm10, %xmm4
+; AVX1-NEXT:    vpxor %xmm5, %xmm4, %xmm4
+; AVX1-NEXT:    vpcmpgtd %xmm0, %xmm10, %xmm2
+; AVX1-NEXT:    vpxor %xmm5, %xmm2, %xmm2
+; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm2, %xmm4
+; AVX1-NEXT:    vpxor %xmm5, %xmm4, %xmm4
+; AVX1-NEXT:    vinsertf128 $1, %xmm8, %ymm4, %ymm8
+; AVX1-NEXT:    vpsubd %xmm9, %xmm6, %xmm6
+; AVX1-NEXT:    vpcmpgtd %xmm6, %xmm10, %xmm4
+; AVX1-NEXT:    vpxor %xmm5, %xmm4, %xmm3
+; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm7, %xmm3
+; AVX1-NEXT:    vpxor %xmm5, %xmm3, %xmm3
+; AVX1-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpcmpgtd %xmm0, %xmm10, %xmm1
+; AVX1-NEXT:    vpxor %xmm5, %xmm1, %xmm1
+; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm2, %xmm1
+; AVX1-NEXT:    vpxor %xmm5, %xmm1, %xmm1
+; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm1, %ymm1
+; AVX1-NEXT:    vandps %ymm1, %ymm8, %ymm1
+; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm0, %ymm2
+; AVX1-NEXT:    vmovaps {{.*#+}} ymm3 = [2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648]
+; AVX1-NEXT:    vblendvps %ymm2, {{.*}}(%rip), %ymm3, %ymm2
+; AVX1-NEXT:    vinsertf128 $1, %xmm6, %ymm0, %ymm0
+; AVX1-NEXT:    vblendvps %ymm1, %ymm2, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: v8i32:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX2-NEXT:    vpcmpgtd %ymm1, %ymm2, %ymm3
+; AVX2-NEXT:    vpcmpeqd %ymm4, %ymm4, %ymm4
+; AVX2-NEXT:    vpxor %ymm4, %ymm3, %ymm3
+; AVX2-NEXT:    vpcmpgtd %ymm0, %ymm2, %ymm5
+; AVX2-NEXT:    vpxor %ymm4, %ymm5, %ymm5
+; AVX2-NEXT:    vpcmpeqd %ymm3, %ymm5, %ymm3
+; AVX2-NEXT:    vpsubd %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpcmpgtd %ymm0, %ymm2, %ymm1
+; AVX2-NEXT:    vpxor %ymm4, %ymm1, %ymm1
+; AVX2-NEXT:    vpcmpeqd %ymm1, %ymm5, %ymm1
+; AVX2-NEXT:    vpxor %ymm4, %ymm1, %ymm1
+; AVX2-NEXT:    vpandn %ymm1, %ymm3, %ymm1
+; AVX2-NEXT:    vbroadcastss {{.*#+}} ymm2 = [2147483647,2147483647,2147483647,2147483647,2147483647,2147483647,2147483647,2147483647]
+; AVX2-NEXT:    vbroadcastss {{.*#+}} ymm3 = [2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648]
+; AVX2-NEXT:    vblendvps %ymm0, %ymm2, %ymm3, %ymm2
+; AVX2-NEXT:    vblendvps %ymm1, %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: v8i32:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX512-NEXT:    vpcmpnltd %ymm2, %ymm1, %k0
+; AVX512-NEXT:    vpcmpnltd %ymm2, %ymm0, %k1
+; AVX512-NEXT:    kxorw %k0, %k1, %k0
+; AVX512-NEXT:    vpsubd %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpcmpnltd %ymm2, %ymm0, %k2
+; AVX512-NEXT:    kxorw %k2, %k1, %k1
+; AVX512-NEXT:    kandw %k1, %k0, %k1
+; AVX512-NEXT:    vpcmpgtd %ymm0, %ymm2, %k2
+; AVX512-NEXT:    vpbroadcastd {{.*#+}} ymm1 = [2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648]
+; AVX512-NEXT:    vpbroadcastd {{.*}}(%rip), %ymm1 {%k2}
+; AVX512-NEXT:    vmovdqa32 %ymm1, %ymm0 {%k1}
+; AVX512-NEXT:    retq
+  %z = call <8 x i32> @llvm.ssub.sat.v8i32(<8 x i32> %x, <8 x i32> %y)
+  ret <8 x i32> %z
+}
+
+define <16 x i32> @v16i32(<16 x i32> %x, <16 x i32> %y) nounwind {
+; SSE2-LABEL: v16i32:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa %xmm1, %xmm8
+; SSE2-NEXT:    movdqa %xmm0, %xmm12
+; SSE2-NEXT:    pxor %xmm9, %xmm9
+; SSE2-NEXT:    pxor %xmm0, %xmm0
+; SSE2-NEXT:    pcmpgtd %xmm4, %xmm0
+; SSE2-NEXT:    pcmpeqd %xmm10, %xmm10
+; SSE2-NEXT:    pxor %xmm10, %xmm0
+; SSE2-NEXT:    pxor %xmm11, %xmm11
+; SSE2-NEXT:    pcmpgtd %xmm12, %xmm11
+; SSE2-NEXT:    pxor %xmm10, %xmm11
+; SSE2-NEXT:    pcmpeqd %xmm11, %xmm0
+; SSE2-NEXT:    psubd %xmm4, %xmm12
+; SSE2-NEXT:    pxor %xmm1, %xmm1
+; SSE2-NEXT:    pcmpgtd %xmm12, %xmm1
+; SSE2-NEXT:    movdqa %xmm1, %xmm4
+; SSE2-NEXT:    pxor %xmm10, %xmm4
+; SSE2-NEXT:    pcmpeqd %xmm11, %xmm4
+; SSE2-NEXT:    pxor %xmm10, %xmm4
+; SSE2-NEXT:    pandn %xmm4, %xmm0
+; SSE2-NEXT:    movdqa {{.*#+}} xmm11 = [2147483648,2147483648,2147483648,2147483648]
+; SSE2-NEXT:    movdqa %xmm1, %xmm4
+; SSE2-NEXT:    pandn %xmm11, %xmm4
+; SSE2-NEXT:    psrld $1, %xmm1
+; SSE2-NEXT:    por %xmm4, %xmm1
+; SSE2-NEXT:    pand %xmm0, %xmm1
+; SSE2-NEXT:    pandn %xmm12, %xmm0
+; SSE2-NEXT:    por %xmm1, %xmm0
+; SSE2-NEXT:    pxor %xmm1, %xmm1
+; SSE2-NEXT:    pcmpgtd %xmm5, %xmm1
+; SSE2-NEXT:    pxor %xmm10, %xmm1
+; SSE2-NEXT:    pxor %xmm12, %xmm12
+; SSE2-NEXT:    pcmpgtd %xmm8, %xmm12
+; SSE2-NEXT:    pxor %xmm10, %xmm12
+; SSE2-NEXT:    pcmpeqd %xmm12, %xmm1
+; SSE2-NEXT:    psubd %xmm5, %xmm8
+; SSE2-NEXT:    pxor %xmm4, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm8, %xmm4
+; SSE2-NEXT:    movdqa %xmm4, %xmm5
+; SSE2-NEXT:    pxor %xmm10, %xmm5
+; SSE2-NEXT:    pcmpeqd %xmm12, %xmm5
+; SSE2-NEXT:    pxor %xmm10, %xmm5
+; SSE2-NEXT:    pandn %xmm5, %xmm1
+; SSE2-NEXT:    movdqa %xmm4, %xmm5
+; SSE2-NEXT:    pandn %xmm11, %xmm5
+; SSE2-NEXT:    psrld $1, %xmm4
+; SSE2-NEXT:    por %xmm5, %xmm4
+; SSE2-NEXT:    pand %xmm1, %xmm4
+; SSE2-NEXT:    pandn %xmm8, %xmm1
+; SSE2-NEXT:    por %xmm4, %xmm1
+; SSE2-NEXT:    pxor %xmm8, %xmm8
+; SSE2-NEXT:    pcmpgtd %xmm6, %xmm8
+; SSE2-NEXT:    pxor %xmm10, %xmm8
+; SSE2-NEXT:    pxor %xmm5, %xmm5
+; SSE2-NEXT:    pcmpgtd %xmm2, %xmm5
+; SSE2-NEXT:    pxor %xmm10, %xmm5
+; SSE2-NEXT:    pcmpeqd %xmm5, %xmm8
+; SSE2-NEXT:    psubd %xmm6, %xmm2
+; SSE2-NEXT:    pxor %xmm4, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm2, %xmm4
+; SSE2-NEXT:    movdqa %xmm4, %xmm6
+; SSE2-NEXT:    pxor %xmm10, %xmm6
+; SSE2-NEXT:    pcmpeqd %xmm5, %xmm6
+; SSE2-NEXT:    pxor %xmm10, %xmm6
+; SSE2-NEXT:    pandn %xmm6, %xmm8
+; SSE2-NEXT:    movdqa %xmm4, %xmm5
+; SSE2-NEXT:    pandn %xmm11, %xmm5
+; SSE2-NEXT:    psrld $1, %xmm4
+; SSE2-NEXT:    por %xmm5, %xmm4
+; SSE2-NEXT:    pand %xmm8, %xmm4
+; SSE2-NEXT:    pandn %xmm2, %xmm8
+; SSE2-NEXT:    por %xmm4, %xmm8
+; SSE2-NEXT:    pxor %xmm5, %xmm5
+; SSE2-NEXT:    pcmpgtd %xmm7, %xmm5
+; SSE2-NEXT:    pxor %xmm10, %xmm5
+; SSE2-NEXT:    pxor %xmm2, %xmm2
+; SSE2-NEXT:    pcmpgtd %xmm3, %xmm2
+; SSE2-NEXT:    pxor %xmm10, %xmm2
+; SSE2-NEXT:    pcmpeqd %xmm2, %xmm5
+; SSE2-NEXT:    psubd %xmm7, %xmm3
+; SSE2-NEXT:    pcmpgtd %xmm3, %xmm9
+; SSE2-NEXT:    movdqa %xmm9, %xmm4
+; SSE2-NEXT:    pxor %xmm10, %xmm4
+; SSE2-NEXT:    pcmpeqd %xmm2, %xmm4
+; SSE2-NEXT:    pxor %xmm10, %xmm4
+; SSE2-NEXT:    pandn %xmm4, %xmm5
+; SSE2-NEXT:    movdqa %xmm9, %xmm2
+; SSE2-NEXT:    pandn %xmm11, %xmm2
+; SSE2-NEXT:    psrld $1, %xmm9
+; SSE2-NEXT:    por %xmm2, %xmm9
+; SSE2-NEXT:    pand %xmm5, %xmm9
+; SSE2-NEXT:    pandn %xmm3, %xmm5
+; SSE2-NEXT:    por %xmm9, %xmm5
+; SSE2-NEXT:    movdqa %xmm8, %xmm2
+; SSE2-NEXT:    movdqa %xmm5, %xmm3
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: v16i32:
+; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    movdqa %xmm1, %xmm8
+; SSSE3-NEXT:    movdqa %xmm0, %xmm12
+; SSSE3-NEXT:    pxor %xmm9, %xmm9
+; SSSE3-NEXT:    pxor %xmm0, %xmm0
+; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm0
+; SSSE3-NEXT:    pcmpeqd %xmm10, %xmm10
+; SSSE3-NEXT:    pxor %xmm10, %xmm0
+; SSSE3-NEXT:    pxor %xmm11, %xmm11
+; SSSE3-NEXT:    pcmpgtd %xmm12, %xmm11
+; SSSE3-NEXT:    pxor %xmm10, %xmm11
+; SSSE3-NEXT:    pcmpeqd %xmm11, %xmm0
+; SSSE3-NEXT:    psubd %xmm4, %xmm12
+; SSSE3-NEXT:    pxor %xmm1, %xmm1
+; SSSE3-NEXT:    pcmpgtd %xmm12, %xmm1
+; SSSE3-NEXT:    movdqa %xmm1, %xmm4
+; SSSE3-NEXT:    pxor %xmm10, %xmm4
+; SSSE3-NEXT:    pcmpeqd %xmm11, %xmm4
+; SSSE3-NEXT:    pxor %xmm10, %xmm4
+; SSSE3-NEXT:    pandn %xmm4, %xmm0
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm11 = [2147483648,2147483648,2147483648,2147483648]
+; SSSE3-NEXT:    movdqa %xmm1, %xmm4
+; SSSE3-NEXT:    pandn %xmm11, %xmm4
+; SSSE3-NEXT:    psrld $1, %xmm1
+; SSSE3-NEXT:    por %xmm4, %xmm1
+; SSSE3-NEXT:    pand %xmm0, %xmm1
+; SSSE3-NEXT:    pandn %xmm12, %xmm0
+; SSSE3-NEXT:    por %xmm1, %xmm0
+; SSSE3-NEXT:    pxor %xmm1, %xmm1
+; SSSE3-NEXT:    pcmpgtd %xmm5, %xmm1
+; SSSE3-NEXT:    pxor %xmm10, %xmm1
+; SSSE3-NEXT:    pxor %xmm12, %xmm12
+; SSSE3-NEXT:    pcmpgtd %xmm8, %xmm12
+; SSSE3-NEXT:    pxor %xmm10, %xmm12
+; SSSE3-NEXT:    pcmpeqd %xmm12, %xmm1
+; SSSE3-NEXT:    psubd %xmm5, %xmm8
+; SSSE3-NEXT:    pxor %xmm4, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm8, %xmm4
+; SSSE3-NEXT:    movdqa %xmm4, %xmm5
+; SSSE3-NEXT:    pxor %xmm10, %xmm5
+; SSSE3-NEXT:    pcmpeqd %xmm12, %xmm5
+; SSSE3-NEXT:    pxor %xmm10, %xmm5
+; SSSE3-NEXT:    pandn %xmm5, %xmm1
+; SSSE3-NEXT:    movdqa %xmm4, %xmm5
+; SSSE3-NEXT:    pandn %xmm11, %xmm5
+; SSSE3-NEXT:    psrld $1, %xmm4
+; SSSE3-NEXT:    por %xmm5, %xmm4
+; SSSE3-NEXT:    pand %xmm1, %xmm4
+; SSSE3-NEXT:    pandn %xmm8, %xmm1
+; SSSE3-NEXT:    por %xmm4, %xmm1
+; SSSE3-NEXT:    pxor %xmm8, %xmm8
+; SSSE3-NEXT:    pcmpgtd %xmm6, %xmm8
+; SSSE3-NEXT:    pxor %xmm10, %xmm8
+; SSSE3-NEXT:    pxor %xmm5, %xmm5
+; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm5
+; SSSE3-NEXT:    pxor %xmm10, %xmm5
+; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm8
+; SSSE3-NEXT:    psubd %xmm6, %xmm2
+; SSSE3-NEXT:    pxor %xmm4, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm4
+; SSSE3-NEXT:    movdqa %xmm4, %xmm6
+; SSSE3-NEXT:    pxor %xmm10, %xmm6
+; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm6
+; SSSE3-NEXT:    pxor %xmm10, %xmm6
+; SSSE3-NEXT:    pandn %xmm6, %xmm8
+; SSSE3-NEXT:    movdqa %xmm4, %xmm5
+; SSSE3-NEXT:    pandn %xmm11, %xmm5
+; SSSE3-NEXT:    psrld $1, %xmm4
+; SSSE3-NEXT:    por %xmm5, %xmm4
+; SSSE3-NEXT:    pand %xmm8, %xmm4
+; SSSE3-NEXT:    pandn %xmm2, %xmm8
+; SSSE3-NEXT:    por %xmm4, %xmm8
+; SSSE3-NEXT:    pxor %xmm5, %xmm5
+; SSSE3-NEXT:    pcmpgtd %xmm7, %xmm5
+; SSSE3-NEXT:    pxor %xmm10, %xmm5
+; SSSE3-NEXT:    pxor %xmm2, %xmm2
+; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm2
+; SSSE3-NEXT:    pxor %xmm10, %xmm2
+; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm5
+; SSSE3-NEXT:    psubd %xmm7, %xmm3
+; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm9
+; SSSE3-NEXT:    movdqa %xmm9, %xmm4
+; SSSE3-NEXT:    pxor %xmm10, %xmm4
+; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm4
+; SSSE3-NEXT:    pxor %xmm10, %xmm4
+; SSSE3-NEXT:    pandn %xmm4, %xmm5
+; SSSE3-NEXT:    movdqa %xmm9, %xmm2
+; SSSE3-NEXT:    pandn %xmm11, %xmm2
+; SSSE3-NEXT:    psrld $1, %xmm9
+; SSSE3-NEXT:    por %xmm2, %xmm9
+; SSSE3-NEXT:    pand %xmm5, %xmm9
+; SSSE3-NEXT:    pandn %xmm3, %xmm5
+; SSSE3-NEXT:    por %xmm9, %xmm5
+; SSSE3-NEXT:    movdqa %xmm8, %xmm2
+; SSSE3-NEXT:    movdqa %xmm5, %xmm3
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: v16i32:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movdqa %xmm0, %xmm8
+; SSE41-NEXT:    pxor %xmm10, %xmm10
+; SSE41-NEXT:    pxor %xmm9, %xmm9
+; SSE41-NEXT:    pcmpgtd %xmm4, %xmm9
+; SSE41-NEXT:    pcmpeqd %xmm11, %xmm11
+; SSE41-NEXT:    pxor %xmm11, %xmm9
+; SSE41-NEXT:    pxor %xmm0, %xmm0
+; SSE41-NEXT:    pcmpgtd %xmm8, %xmm0
+; SSE41-NEXT:    pxor %xmm11, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm9
+; SSE41-NEXT:    psubd %xmm4, %xmm8
+; SSE41-NEXT:    pxor %xmm4, %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm8, %xmm4
+; SSE41-NEXT:    pxor %xmm11, %xmm4
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm4
+; SSE41-NEXT:    pxor %xmm11, %xmm4
+; SSE41-NEXT:    pandn %xmm4, %xmm9
+; SSE41-NEXT:    movaps {{.*#+}} xmm13 = [2147483647,2147483647,2147483647,2147483647]
+; SSE41-NEXT:    movaps {{.*#+}} xmm12 = [2147483648,2147483648,2147483648,2147483648]
+; SSE41-NEXT:    movaps %xmm12, %xmm4
+; SSE41-NEXT:    movdqa %xmm8, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm13, %xmm4
+; SSE41-NEXT:    movdqa %xmm9, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm4, %xmm8
+; SSE41-NEXT:    xorps %xmm4, %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm5, %xmm4
+; SSE41-NEXT:    pxor %xmm11, %xmm4
+; SSE41-NEXT:    pxor %xmm0, %xmm0
+; SSE41-NEXT:    pcmpgtd %xmm1, %xmm0
+; SSE41-NEXT:    pxor %xmm11, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm4
+; SSE41-NEXT:    psubd %xmm5, %xmm1
+; SSE41-NEXT:    pxor %xmm5, %xmm5
+; SSE41-NEXT:    pcmpgtd %xmm1, %xmm5
+; SSE41-NEXT:    pxor %xmm11, %xmm5
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm5
+; SSE41-NEXT:    pxor %xmm11, %xmm5
+; SSE41-NEXT:    pandn %xmm5, %xmm4
+; SSE41-NEXT:    movaps %xmm12, %xmm5
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm13, %xmm5
+; SSE41-NEXT:    movdqa %xmm4, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm5, %xmm1
+; SSE41-NEXT:    pxor %xmm4, %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm6, %xmm4
+; SSE41-NEXT:    pxor %xmm11, %xmm4
+; SSE41-NEXT:    pxor %xmm0, %xmm0
+; SSE41-NEXT:    pcmpgtd %xmm2, %xmm0
+; SSE41-NEXT:    pxor %xmm11, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm4
+; SSE41-NEXT:    psubd %xmm6, %xmm2
+; SSE41-NEXT:    xorps %xmm5, %xmm5
+; SSE41-NEXT:    pcmpgtd %xmm2, %xmm5
+; SSE41-NEXT:    pxor %xmm11, %xmm5
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm5
+; SSE41-NEXT:    pxor %xmm11, %xmm5
+; SSE41-NEXT:    pandn %xmm5, %xmm4
+; SSE41-NEXT:    movaps %xmm12, %xmm5
+; SSE41-NEXT:    movdqa %xmm2, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm13, %xmm5
+; SSE41-NEXT:    movdqa %xmm4, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm5, %xmm2
+; SSE41-NEXT:    pxor %xmm4, %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm7, %xmm4
+; SSE41-NEXT:    pxor %xmm11, %xmm4
+; SSE41-NEXT:    pxor %xmm0, %xmm0
+; SSE41-NEXT:    pcmpgtd %xmm3, %xmm0
+; SSE41-NEXT:    pxor %xmm11, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm4
+; SSE41-NEXT:    psubd %xmm7, %xmm3
+; SSE41-NEXT:    pcmpgtd %xmm3, %xmm10
+; SSE41-NEXT:    pxor %xmm11, %xmm10
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm10
+; SSE41-NEXT:    pxor %xmm11, %xmm10
+; SSE41-NEXT:    pandn %xmm10, %xmm4
+; SSE41-NEXT:    movdqa %xmm3, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm13, %xmm12
+; SSE41-NEXT:    movdqa %xmm4, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm12, %xmm3
+; SSE41-NEXT:    movaps %xmm8, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: v16i32:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm8
+; AVX1-NEXT:    vpxor %xmm12, %xmm12, %xmm12
+; AVX1-NEXT:    vpcmpgtd %xmm8, %xmm12, %xmm7
+; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
+; AVX1-NEXT:    vpxor %xmm4, %xmm7, %xmm9
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm6
+; AVX1-NEXT:    vpcmpgtd %xmm6, %xmm12, %xmm7
+; AVX1-NEXT:    vpxor %xmm4, %xmm7, %xmm10
+; AVX1-NEXT:    vpcmpeqd %xmm9, %xmm10, %xmm7
+; AVX1-NEXT:    vpxor %xmm4, %xmm7, %xmm9
+; AVX1-NEXT:    vpcmpgtd %xmm2, %xmm12, %xmm7
+; AVX1-NEXT:    vpxor %xmm4, %xmm7, %xmm11
+; AVX1-NEXT:    vpcmpgtd %xmm0, %xmm12, %xmm7
+; AVX1-NEXT:    vpxor %xmm4, %xmm7, %xmm7
+; AVX1-NEXT:    vpcmpeqd %xmm11, %xmm7, %xmm5
+; AVX1-NEXT:    vpxor %xmm4, %xmm5, %xmm5
+; AVX1-NEXT:    vinsertf128 $1, %xmm9, %ymm5, %ymm9
+; AVX1-NEXT:    vpsubd %xmm8, %xmm6, %xmm8
+; AVX1-NEXT:    vpcmpgtd %xmm8, %xmm12, %xmm5
+; AVX1-NEXT:    vpxor %xmm4, %xmm5, %xmm6
+; AVX1-NEXT:    vpcmpeqd %xmm6, %xmm10, %xmm6
+; AVX1-NEXT:    vpxor %xmm4, %xmm6, %xmm6
+; AVX1-NEXT:    vpsubd %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpcmpgtd %xmm0, %xmm12, %xmm2
+; AVX1-NEXT:    vpxor %xmm4, %xmm2, %xmm2
+; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm7, %xmm2
+; AVX1-NEXT:    vpxor %xmm4, %xmm2, %xmm2
+; AVX1-NEXT:    vinsertf128 $1, %xmm6, %ymm2, %ymm2
+; AVX1-NEXT:    vandps %ymm2, %ymm9, %ymm6
+; AVX1-NEXT:    vinsertf128 $1, %xmm5, %ymm0, %ymm7
+; AVX1-NEXT:    vmovaps {{.*#+}} ymm9 = [2147483647,2147483647,2147483647,2147483647,2147483647,2147483647,2147483647,2147483647]
+; AVX1-NEXT:    vmovaps {{.*#+}} ymm10 = [2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648]
+; AVX1-NEXT:    vblendvps %ymm7, %ymm9, %ymm10, %ymm7
+; AVX1-NEXT:    vinsertf128 $1, %xmm8, %ymm0, %ymm0
+; AVX1-NEXT:    vblendvps %ymm6, %ymm7, %ymm0, %ymm0
+; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm13
+; AVX1-NEXT:    vpcmpgtd %xmm13, %xmm12, %xmm7
+; AVX1-NEXT:    vpxor %xmm4, %xmm7, %xmm7
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; AVX1-NEXT:    vpcmpgtd %xmm2, %xmm12, %xmm5
+; AVX1-NEXT:    vpxor %xmm4, %xmm5, %xmm5
+; AVX1-NEXT:    vpcmpeqd %xmm7, %xmm5, %xmm7
+; AVX1-NEXT:    vpxor %xmm4, %xmm7, %xmm8
+; AVX1-NEXT:    vpcmpgtd %xmm3, %xmm12, %xmm7
+; AVX1-NEXT:    vpxor %xmm4, %xmm7, %xmm11
+; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm12, %xmm7
+; AVX1-NEXT:    vpxor %xmm4, %xmm7, %xmm7
+; AVX1-NEXT:    vpcmpeqd %xmm11, %xmm7, %xmm6
+; AVX1-NEXT:    vpxor %xmm4, %xmm6, %xmm6
+; AVX1-NEXT:    vinsertf128 $1, %xmm8, %ymm6, %ymm8
+; AVX1-NEXT:    vpsubd %xmm13, %xmm2, %xmm11
+; AVX1-NEXT:    vpcmpgtd %xmm11, %xmm12, %xmm6
+; AVX1-NEXT:    vpxor %xmm4, %xmm6, %xmm2
+; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm5, %xmm2
+; AVX1-NEXT:    vpxor %xmm4, %xmm2, %xmm2
+; AVX1-NEXT:    vpsubd %xmm3, %xmm1, %xmm1
+; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm12, %xmm3
+; AVX1-NEXT:    vpxor %xmm4, %xmm3, %xmm3
+; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm7, %xmm3
+; AVX1-NEXT:    vpxor %xmm4, %xmm3, %xmm3
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm3, %ymm2
+; AVX1-NEXT:    vandps %ymm2, %ymm8, %ymm2
+; AVX1-NEXT:    vinsertf128 $1, %xmm6, %ymm1, %ymm3
+; AVX1-NEXT:    vblendvps %ymm3, %ymm9, %ymm10, %ymm3
+; AVX1-NEXT:    vinsertf128 $1, %xmm11, %ymm1, %ymm1
+; AVX1-NEXT:    vblendvps %ymm2, %ymm3, %ymm1, %ymm1
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: v16i32:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpxor %xmm4, %xmm4, %xmm4
+; AVX2-NEXT:    vpcmpgtd %ymm2, %ymm4, %ymm5
+; AVX2-NEXT:    vpcmpeqd %ymm6, %ymm6, %ymm6
+; AVX2-NEXT:    vpxor %ymm6, %ymm5, %ymm5
+; AVX2-NEXT:    vpcmpgtd %ymm0, %ymm4, %ymm7
+; AVX2-NEXT:    vpxor %ymm6, %ymm7, %ymm7
+; AVX2-NEXT:    vpcmpeqd %ymm5, %ymm7, %ymm5
+; AVX2-NEXT:    vpsubd %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vpcmpgtd %ymm0, %ymm4, %ymm2
+; AVX2-NEXT:    vpxor %ymm6, %ymm2, %ymm2
+; AVX2-NEXT:    vpcmpeqd %ymm2, %ymm7, %ymm2
+; AVX2-NEXT:    vpxor %ymm6, %ymm2, %ymm2
+; AVX2-NEXT:    vpandn %ymm2, %ymm5, %ymm2
+; AVX2-NEXT:    vbroadcastss {{.*#+}} ymm5 = [2147483647,2147483647,2147483647,2147483647,2147483647,2147483647,2147483647,2147483647]
+; AVX2-NEXT:    vbroadcastss {{.*#+}} ymm7 = [2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648]
+; AVX2-NEXT:    vblendvps %ymm0, %ymm5, %ymm7, %ymm8
+; AVX2-NEXT:    vblendvps %ymm2, %ymm8, %ymm0, %ymm0
+; AVX2-NEXT:    vpcmpgtd %ymm3, %ymm4, %ymm2
+; AVX2-NEXT:    vpxor %ymm6, %ymm2, %ymm2
+; AVX2-NEXT:    vpcmpgtd %ymm1, %ymm4, %ymm8
+; AVX2-NEXT:    vpxor %ymm6, %ymm8, %ymm8
+; AVX2-NEXT:    vpcmpeqd %ymm2, %ymm8, %ymm2
+; AVX2-NEXT:    vpsubd %ymm3, %ymm1, %ymm1
+; AVX2-NEXT:    vpcmpgtd %ymm1, %ymm4, %ymm3
+; AVX2-NEXT:    vpxor %ymm6, %ymm3, %ymm3
+; AVX2-NEXT:    vpcmpeqd %ymm3, %ymm8, %ymm3
+; AVX2-NEXT:    vpxor %ymm6, %ymm3, %ymm3
+; AVX2-NEXT:    vpandn %ymm3, %ymm2, %ymm2
+; AVX2-NEXT:    vblendvps %ymm1, %ymm5, %ymm7, %ymm3
+; AVX2-NEXT:    vblendvps %ymm2, %ymm3, %ymm1, %ymm1
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: v16i32:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX512-NEXT:    vpcmpnltd %zmm2, %zmm1, %k0
+; AVX512-NEXT:    vpcmpnltd %zmm2, %zmm0, %k1
+; AVX512-NEXT:    kxorw %k0, %k1, %k0
+; AVX512-NEXT:    vpsubd %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpcmpnltd %zmm2, %zmm0, %k2
+; AVX512-NEXT:    kxorw %k2, %k1, %k1
+; AVX512-NEXT:    kandw %k1, %k0, %k1
+; AVX512-NEXT:    vpcmpgtd %zmm0, %zmm2, %k2
+; AVX512-NEXT:    vpbroadcastd {{.*#+}} zmm1 = [2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648]
+; AVX512-NEXT:    vpbroadcastd {{.*}}(%rip), %zmm1 {%k2}
+; AVX512-NEXT:    vmovdqa32 %zmm1, %zmm0 {%k1}
+; AVX512-NEXT:    retq
+  %z = call <16 x i32> @llvm.ssub.sat.v16i32(<16 x i32> %x, <16 x i32> %y)
+  ret <16 x i32> %z
+}
+
+define <2 x i64> @v2i64(<2 x i64> %x, <2 x i64> %y) nounwind {
+; SSE2-LABEL: v2i64:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648]
+; SSE2-NEXT:    movdqa %xmm0, %xmm3
+; SSE2-NEXT:    psubq %xmm1, %xmm0
+; SSE2-NEXT:    pxor %xmm2, %xmm1
+; SSE2-NEXT:    movdqa %xmm2, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm1, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm2, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSE2-NEXT:    pand %xmm5, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSE2-NEXT:    por %xmm1, %xmm4
+; SSE2-NEXT:    pcmpeqd %xmm5, %xmm5
+; SSE2-NEXT:    pxor %xmm5, %xmm4
+; SSE2-NEXT:    pxor %xmm2, %xmm3
+; SSE2-NEXT:    movdqa %xmm2, %xmm1
+; SSE2-NEXT:    pcmpgtd %xmm3, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm1[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm2, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
+; SSE2-NEXT:    pand %xmm6, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm1[1,1,3,3]
+; SSE2-NEXT:    por %xmm3, %xmm6
+; SSE2-NEXT:    pxor %xmm5, %xmm6
+; SSE2-NEXT:    pcmpeqd %xmm6, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm4[1,0,3,2]
+; SSE2-NEXT:    pand %xmm4, %xmm1
+; SSE2-NEXT:    movdqa %xmm0, %xmm3
+; SSE2-NEXT:    pxor %xmm2, %xmm3
+; SSE2-NEXT:    movdqa %xmm2, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm3, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm4[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm2, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[1,1,3,3]
+; SSE2-NEXT:    pand %xmm7, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[1,1,3,3]
+; SSE2-NEXT:    por %xmm2, %xmm3
+; SSE2-NEXT:    movdqa %xmm3, %xmm2
+; SSE2-NEXT:    pxor %xmm5, %xmm2
+; SSE2-NEXT:    pcmpeqd %xmm6, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[1,0,3,2]
+; SSE2-NEXT:    pand %xmm2, %xmm4
+; SSE2-NEXT:    pxor %xmm5, %xmm4
+; SSE2-NEXT:    pandn %xmm4, %xmm1
+; SSE2-NEXT:    movdqa %xmm3, %xmm2
+; SSE2-NEXT:    pandn {{.*}}(%rip), %xmm2
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm3
+; SSE2-NEXT:    por %xmm2, %xmm3
+; SSE2-NEXT:    pand %xmm1, %xmm3
+; SSE2-NEXT:    pandn %xmm0, %xmm1
+; SSE2-NEXT:    por %xmm3, %xmm1
+; SSE2-NEXT:    movdqa %xmm1, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: v2i64:
+; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648]
+; SSSE3-NEXT:    movdqa %xmm0, %xmm3
+; SSSE3-NEXT:    psubq %xmm1, %xmm0
+; SSSE3-NEXT:    pxor %xmm2, %xmm1
+; SSSE3-NEXT:    movdqa %xmm2, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm5, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    por %xmm1, %xmm4
+; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm5
+; SSSE3-NEXT:    pxor %xmm5, %xmm4
+; SSSE3-NEXT:    pxor %xmm2, %xmm3
+; SSSE3-NEXT:    movdqa %xmm2, %xmm1
+; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm1[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm3
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm6, %xmm3
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm1[1,1,3,3]
+; SSSE3-NEXT:    por %xmm3, %xmm6
+; SSSE3-NEXT:    pxor %xmm5, %xmm6
+; SSSE3-NEXT:    pcmpeqd %xmm6, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm4[1,0,3,2]
+; SSSE3-NEXT:    pand %xmm4, %xmm1
+; SSSE3-NEXT:    movdqa %xmm0, %xmm3
+; SSSE3-NEXT:    pxor %xmm2, %xmm3
+; SSSE3-NEXT:    movdqa %xmm2, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm4[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm3
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm7, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    por %xmm2, %xmm3
+; SSSE3-NEXT:    movdqa %xmm3, %xmm2
+; SSSE3-NEXT:    pxor %xmm5, %xmm2
+; SSSE3-NEXT:    pcmpeqd %xmm6, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[1,0,3,2]
+; SSSE3-NEXT:    pand %xmm2, %xmm4
+; SSSE3-NEXT:    pxor %xmm5, %xmm4
+; SSSE3-NEXT:    pandn %xmm4, %xmm1
+; SSSE3-NEXT:    movdqa %xmm3, %xmm2
+; SSSE3-NEXT:    pandn {{.*}}(%rip), %xmm2
+; SSSE3-NEXT:    pand {{.*}}(%rip), %xmm3
+; SSSE3-NEXT:    por %xmm2, %xmm3
+; SSSE3-NEXT:    pand %xmm1, %xmm3
+; SSSE3-NEXT:    pandn %xmm0, %xmm1
+; SSSE3-NEXT:    por %xmm3, %xmm1
+; SSSE3-NEXT:    movdqa %xmm1, %xmm0
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: v2i64:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movdqa %xmm0, %xmm2
+; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483648,2147483648]
+; SSE41-NEXT:    movdqa %xmm2, %xmm3
+; SSE41-NEXT:    psubq %xmm1, %xmm2
+; SSE41-NEXT:    pxor %xmm0, %xmm1
+; SSE41-NEXT:    movdqa %xmm0, %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm1, %xmm4
+; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
+; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm1[1,1,3,3]
+; SSE41-NEXT:    pand %xmm5, %xmm6
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm4[1,1,3,3]
+; SSE41-NEXT:    por %xmm6, %xmm1
+; SSE41-NEXT:    pcmpeqd %xmm4, %xmm4
+; SSE41-NEXT:    pxor %xmm4, %xmm1
+; SSE41-NEXT:    pxor %xmm0, %xmm3
+; SSE41-NEXT:    movdqa %xmm0, %xmm5
+; SSE41-NEXT:    pcmpgtd %xmm3, %xmm5
+; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm3
+; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
+; SSE41-NEXT:    pand %xmm6, %xmm3
+; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
+; SSE41-NEXT:    por %xmm3, %xmm5
+; SSE41-NEXT:    pxor %xmm4, %xmm5
+; SSE41-NEXT:    pcmpeqq %xmm5, %xmm1
+; SSE41-NEXT:    movdqa %xmm2, %xmm3
+; SSE41-NEXT:    pxor %xmm0, %xmm3
+; SSE41-NEXT:    movdqa %xmm0, %xmm6
+; SSE41-NEXT:    pcmpgtd %xmm3, %xmm6
+; SSE41-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm3
+; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
+; SSE41-NEXT:    pand %xmm7, %xmm3
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[1,1,3,3]
+; SSE41-NEXT:    por %xmm3, %xmm0
+; SSE41-NEXT:    movdqa %xmm0, %xmm3
+; SSE41-NEXT:    pxor %xmm4, %xmm3
+; SSE41-NEXT:    pcmpeqq %xmm5, %xmm3
+; SSE41-NEXT:    pxor %xmm4, %xmm3
+; SSE41-NEXT:    pandn %xmm3, %xmm1
+; SSE41-NEXT:    movapd {{.*#+}} xmm3 = [9223372036854775808,9223372036854775808]
+; SSE41-NEXT:    blendvpd %xmm0, {{.*}}(%rip), %xmm3
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm3, %xmm2
+; SSE41-NEXT:    movapd %xmm2, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: v2i64:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm2, %xmm3
+; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
+; AVX1-NEXT:    vpxor %xmm4, %xmm3, %xmm3
+; AVX1-NEXT:    vpcmpgtq %xmm0, %xmm2, %xmm5
+; AVX1-NEXT:    vpxor %xmm4, %xmm5, %xmm5
+; AVX1-NEXT:    vpcmpeqq %xmm3, %xmm5, %xmm3
+; AVX1-NEXT:    vpsubq %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpcmpgtq %xmm0, %xmm2, %xmm1
+; AVX1-NEXT:    vpxor %xmm4, %xmm1, %xmm1
+; AVX1-NEXT:    vpcmpeqq %xmm1, %xmm5, %xmm1
+; AVX1-NEXT:    vpxor %xmm4, %xmm1, %xmm1
+; AVX1-NEXT:    vpandn %xmm1, %xmm3, %xmm1
+; AVX1-NEXT:    vmovapd {{.*#+}} xmm2 = [9223372036854775808,9223372036854775808]
+; AVX1-NEXT:    vblendvpd %xmm0, {{.*}}(%rip), %xmm2, %xmm2
+; AVX1-NEXT:    vblendvpd %xmm1, %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: v2i64:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX2-NEXT:    vpcmpgtq %xmm1, %xmm2, %xmm3
+; AVX2-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
+; AVX2-NEXT:    vpxor %xmm4, %xmm3, %xmm3
+; AVX2-NEXT:    vpcmpgtq %xmm0, %xmm2, %xmm5
+; AVX2-NEXT:    vpxor %xmm4, %xmm5, %xmm5
+; AVX2-NEXT:    vpcmpeqq %xmm3, %xmm5, %xmm3
+; AVX2-NEXT:    vpsubq %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpcmpgtq %xmm0, %xmm2, %xmm1
+; AVX2-NEXT:    vpxor %xmm4, %xmm1, %xmm1
+; AVX2-NEXT:    vpcmpeqq %xmm1, %xmm5, %xmm1
+; AVX2-NEXT:    vpxor %xmm4, %xmm1, %xmm1
+; AVX2-NEXT:    vpandn %xmm1, %xmm3, %xmm1
+; AVX2-NEXT:    vmovapd {{.*#+}} xmm2 = [9223372036854775808,9223372036854775808]
+; AVX2-NEXT:    vblendvpd %xmm0, {{.*}}(%rip), %xmm2, %xmm2
+; AVX2-NEXT:    vblendvpd %xmm1, %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: v2i64:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX512-NEXT:    vpcmpnltq %xmm2, %xmm1, %k0
+; AVX512-NEXT:    vpcmpnltq %xmm2, %xmm0, %k1
+; AVX512-NEXT:    kxorw %k0, %k1, %k0
+; AVX512-NEXT:    vpsubq %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vpcmpnltq %xmm2, %xmm0, %k2
+; AVX512-NEXT:    kxorw %k2, %k1, %k1
+; AVX512-NEXT:    kandw %k1, %k0, %k1
+; AVX512-NEXT:    vpcmpgtq %xmm0, %xmm2, %k2
+; AVX512-NEXT:    vmovdqa {{.*#+}} xmm1 = [9223372036854775808,9223372036854775808]
+; AVX512-NEXT:    vmovdqa64 {{.*}}(%rip), %xmm1 {%k2}
+; AVX512-NEXT:    vmovdqa64 %xmm1, %xmm0 {%k1}
+; AVX512-NEXT:    retq
+  %z = call <2 x i64> @llvm.ssub.sat.v2i64(<2 x i64> %x, <2 x i64> %y)
+  ret <2 x i64> %z
+}
+
+define <4 x i64> @v4i64(<4 x i64> %x, <4 x i64> %y) nounwind {
+; SSE2-LABEL: v4i64:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa %xmm0, %xmm10
+; SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [2147483648,2147483648]
+; SSE2-NEXT:    psubq %xmm2, %xmm10
+; SSE2-NEXT:    pxor %xmm5, %xmm2
+; SSE2-NEXT:    movdqa %xmm5, %xmm6
+; SSE2-NEXT:    pcmpgtd %xmm2, %xmm6
+; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm5, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; SSE2-NEXT:    pand %xmm7, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[1,1,3,3]
+; SSE2-NEXT:    por %xmm2, %xmm7
+; SSE2-NEXT:    pcmpeqd %xmm9, %xmm9
+; SSE2-NEXT:    pxor %xmm9, %xmm7
+; SSE2-NEXT:    pxor %xmm5, %xmm0
+; SSE2-NEXT:    movdqa %xmm5, %xmm2
+; SSE2-NEXT:    pcmpgtd %xmm0, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm2[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm5, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE2-NEXT:    pand %xmm6, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; SSE2-NEXT:    por %xmm0, %xmm2
+; SSE2-NEXT:    pxor %xmm9, %xmm2
+; SSE2-NEXT:    pcmpeqd %xmm2, %xmm7
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm7[1,0,3,2]
+; SSE2-NEXT:    pand %xmm7, %xmm0
+; SSE2-NEXT:    movdqa %xmm10, %xmm6
+; SSE2-NEXT:    pxor %xmm5, %xmm6
+; SSE2-NEXT:    movdqa %xmm5, %xmm7
+; SSE2-NEXT:    pcmpgtd %xmm6, %xmm7
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm7[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm5, %xmm6
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
+; SSE2-NEXT:    pand %xmm4, %xmm6
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm7[1,1,3,3]
+; SSE2-NEXT:    por %xmm6, %xmm4
+; SSE2-NEXT:    movdqa %xmm4, %xmm6
+; SSE2-NEXT:    pxor %xmm9, %xmm6
+; SSE2-NEXT:    pcmpeqd %xmm2, %xmm6
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm6[1,0,3,2]
+; SSE2-NEXT:    pand %xmm6, %xmm2
+; SSE2-NEXT:    pxor %xmm9, %xmm2
+; SSE2-NEXT:    pandn %xmm2, %xmm0
+; SSE2-NEXT:    movdqa {{.*#+}} xmm8 = [9223372036854775808,9223372036854775808]
+; SSE2-NEXT:    movdqa %xmm4, %xmm2
+; SSE2-NEXT:    pandn %xmm8, %xmm2
+; SSE2-NEXT:    movdqa {{.*#+}} xmm11 = [9223372036854775807,9223372036854775807]
+; SSE2-NEXT:    pand %xmm11, %xmm4
+; SSE2-NEXT:    por %xmm2, %xmm4
+; SSE2-NEXT:    pand %xmm0, %xmm4
+; SSE2-NEXT:    pandn %xmm10, %xmm0
+; SSE2-NEXT:    por %xmm4, %xmm0
+; SSE2-NEXT:    movdqa %xmm1, %xmm2
+; SSE2-NEXT:    psubq %xmm3, %xmm1
+; SSE2-NEXT:    pxor %xmm5, %xmm3
+; SSE2-NEXT:    movdqa %xmm5, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm3, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm5, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
+; SSE2-NEXT:    pand %xmm6, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSE2-NEXT:    por %xmm3, %xmm4
+; SSE2-NEXT:    pxor %xmm9, %xmm4
+; SSE2-NEXT:    pxor %xmm5, %xmm2
+; SSE2-NEXT:    movdqa %xmm5, %xmm3
+; SSE2-NEXT:    pcmpgtd %xmm2, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm3[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm5, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; SSE2-NEXT:    pand %xmm6, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
+; SSE2-NEXT:    por %xmm2, %xmm3
+; SSE2-NEXT:    pxor %xmm9, %xmm3
+; SSE2-NEXT:    pcmpeqd %xmm3, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm4[1,0,3,2]
+; SSE2-NEXT:    pand %xmm4, %xmm2
+; SSE2-NEXT:    movdqa %xmm1, %xmm4
+; SSE2-NEXT:    pxor %xmm5, %xmm4
+; SSE2-NEXT:    movdqa %xmm5, %xmm6
+; SSE2-NEXT:    pcmpgtd %xmm4, %xmm6
+; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm5, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSE2-NEXT:    pand %xmm7, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm6[1,1,3,3]
+; SSE2-NEXT:    por %xmm4, %xmm5
+; SSE2-NEXT:    movdqa %xmm5, %xmm4
+; SSE2-NEXT:    pxor %xmm9, %xmm4
+; SSE2-NEXT:    pcmpeqd %xmm3, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[1,0,3,2]
+; SSE2-NEXT:    pand %xmm4, %xmm3
+; SSE2-NEXT:    pxor %xmm9, %xmm3
+; SSE2-NEXT:    pandn %xmm3, %xmm2
+; SSE2-NEXT:    movdqa %xmm5, %xmm3
+; SSE2-NEXT:    pandn %xmm8, %xmm3
+; SSE2-NEXT:    pand %xmm11, %xmm5
+; SSE2-NEXT:    por %xmm3, %xmm5
+; SSE2-NEXT:    pand %xmm2, %xmm5
+; SSE2-NEXT:    pandn %xmm1, %xmm2
+; SSE2-NEXT:    por %xmm5, %xmm2
+; SSE2-NEXT:    movdqa %xmm2, %xmm1
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: v4i64:
+; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    movdqa %xmm0, %xmm10
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm5 = [2147483648,2147483648]
+; SSSE3-NEXT:    psubq %xmm2, %xmm10
+; SSSE3-NEXT:    pxor %xmm5, %xmm2
+; SSSE3-NEXT:    movdqa %xmm5, %xmm6
+; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm6
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm7, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[1,1,3,3]
+; SSSE3-NEXT:    por %xmm2, %xmm7
+; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm9
+; SSSE3-NEXT:    pxor %xmm9, %xmm7
+; SSSE3-NEXT:    pxor %xmm5, %xmm0
+; SSSE3-NEXT:    movdqa %xmm5, %xmm2
+; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm2[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm6, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; SSSE3-NEXT:    por %xmm0, %xmm2
+; SSSE3-NEXT:    pxor %xmm9, %xmm2
+; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm7
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm7[1,0,3,2]
+; SSSE3-NEXT:    pand %xmm7, %xmm0
+; SSSE3-NEXT:    movdqa %xmm10, %xmm6
+; SSSE3-NEXT:    pxor %xmm5, %xmm6
+; SSSE3-NEXT:    movdqa %xmm5, %xmm7
+; SSSE3-NEXT:    pcmpgtd %xmm6, %xmm7
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm7[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm6
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm4, %xmm6
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm7[1,1,3,3]
+; SSSE3-NEXT:    por %xmm6, %xmm4
+; SSSE3-NEXT:    movdqa %xmm4, %xmm6
+; SSSE3-NEXT:    pxor %xmm9, %xmm6
+; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm6
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm6[1,0,3,2]
+; SSSE3-NEXT:    pand %xmm6, %xmm2
+; SSSE3-NEXT:    pxor %xmm9, %xmm2
+; SSSE3-NEXT:    pandn %xmm2, %xmm0
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm8 = [9223372036854775808,9223372036854775808]
+; SSSE3-NEXT:    movdqa %xmm4, %xmm2
+; SSSE3-NEXT:    pandn %xmm8, %xmm2
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm11 = [9223372036854775807,9223372036854775807]
+; SSSE3-NEXT:    pand %xmm11, %xmm4
+; SSSE3-NEXT:    por %xmm2, %xmm4
+; SSSE3-NEXT:    pand %xmm0, %xmm4
+; SSSE3-NEXT:    pandn %xmm10, %xmm0
+; SSSE3-NEXT:    por %xmm4, %xmm0
+; SSSE3-NEXT:    movdqa %xmm1, %xmm2
+; SSSE3-NEXT:    psubq %xmm3, %xmm1
+; SSSE3-NEXT:    pxor %xmm5, %xmm3
+; SSSE3-NEXT:    movdqa %xmm5, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm3
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm6, %xmm3
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    por %xmm3, %xmm4
+; SSSE3-NEXT:    pxor %xmm9, %xmm4
+; SSSE3-NEXT:    pxor %xmm5, %xmm2
+; SSSE3-NEXT:    movdqa %xmm5, %xmm3
+; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm3
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm3[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm6, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
+; SSSE3-NEXT:    por %xmm2, %xmm3
+; SSSE3-NEXT:    pxor %xmm9, %xmm3
+; SSSE3-NEXT:    pcmpeqd %xmm3, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm4[1,0,3,2]
+; SSSE3-NEXT:    pand %xmm4, %xmm2
+; SSSE3-NEXT:    movdqa %xmm1, %xmm4
+; SSSE3-NEXT:    pxor %xmm5, %xmm4
+; SSSE3-NEXT:    movdqa %xmm5, %xmm6
+; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm6
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm7, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm6[1,1,3,3]
+; SSSE3-NEXT:    por %xmm4, %xmm5
+; SSSE3-NEXT:    movdqa %xmm5, %xmm4
+; SSSE3-NEXT:    pxor %xmm9, %xmm4
+; SSSE3-NEXT:    pcmpeqd %xmm3, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[1,0,3,2]
+; SSSE3-NEXT:    pand %xmm4, %xmm3
+; SSSE3-NEXT:    pxor %xmm9, %xmm3
+; SSSE3-NEXT:    pandn %xmm3, %xmm2
+; SSSE3-NEXT:    movdqa %xmm5, %xmm3
+; SSSE3-NEXT:    pandn %xmm8, %xmm3
+; SSSE3-NEXT:    pand %xmm11, %xmm5
+; SSSE3-NEXT:    por %xmm3, %xmm5
+; SSSE3-NEXT:    pand %xmm2, %xmm5
+; SSSE3-NEXT:    pandn %xmm1, %xmm2
+; SSSE3-NEXT:    por %xmm5, %xmm2
+; SSSE3-NEXT:    movdqa %xmm2, %xmm1
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: v4i64:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movdqa %xmm0, %xmm9
+; SSE41-NEXT:    movdqa {{.*#+}} xmm11 = [2147483648,2147483648]
+; SSE41-NEXT:    psubq %xmm2, %xmm9
+; SSE41-NEXT:    pxor %xmm11, %xmm2
+; SSE41-NEXT:    movdqa %xmm11, %xmm6
+; SSE41-NEXT:    pcmpgtd %xmm2, %xmm6
+; SSE41-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
+; SSE41-NEXT:    pcmpeqd %xmm11, %xmm2
+; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[1,1,3,3]
+; SSE41-NEXT:    pand %xmm7, %xmm4
+; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm6[1,1,3,3]
+; SSE41-NEXT:    por %xmm4, %xmm2
+; SSE41-NEXT:    pcmpeqd %xmm10, %xmm10
+; SSE41-NEXT:    pxor %xmm10, %xmm2
+; SSE41-NEXT:    pxor %xmm11, %xmm0
+; SSE41-NEXT:    movdqa %xmm11, %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm4
+; SSE41-NEXT:    pshufd {{.*#+}} xmm7 = xmm4[0,0,2,2]
+; SSE41-NEXT:    pcmpeqd %xmm11, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE41-NEXT:    pand %xmm7, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSE41-NEXT:    por %xmm0, %xmm4
+; SSE41-NEXT:    pxor %xmm10, %xmm4
+; SSE41-NEXT:    pcmpeqq %xmm4, %xmm2
+; SSE41-NEXT:    movdqa %xmm9, %xmm0
+; SSE41-NEXT:    pxor %xmm11, %xmm0
+; SSE41-NEXT:    movdqa %xmm11, %xmm7
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm7
+; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm7[0,0,2,2]
+; SSE41-NEXT:    pcmpeqd %xmm11, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm0[1,1,3,3]
+; SSE41-NEXT:    pand %xmm6, %xmm5
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm7[1,1,3,3]
+; SSE41-NEXT:    por %xmm5, %xmm0
+; SSE41-NEXT:    movdqa %xmm0, %xmm5
+; SSE41-NEXT:    pxor %xmm10, %xmm5
+; SSE41-NEXT:    pcmpeqq %xmm4, %xmm5
+; SSE41-NEXT:    pxor %xmm10, %xmm5
+; SSE41-NEXT:    pandn %xmm5, %xmm2
+; SSE41-NEXT:    movapd {{.*#+}} xmm8 = [9223372036854775807,9223372036854775807]
+; SSE41-NEXT:    movapd {{.*#+}} xmm7 = [9223372036854775808,9223372036854775808]
+; SSE41-NEXT:    movapd %xmm7, %xmm4
+; SSE41-NEXT:    blendvpd %xmm0, %xmm8, %xmm4
+; SSE41-NEXT:    movdqa %xmm2, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm4, %xmm9
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    psubq %xmm3, %xmm1
+; SSE41-NEXT:    pxor %xmm11, %xmm3
+; SSE41-NEXT:    movdqa %xmm11, %xmm2
+; SSE41-NEXT:    pcmpgtd %xmm3, %xmm2
+; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[0,0,2,2]
+; SSE41-NEXT:    pcmpeqd %xmm11, %xmm3
+; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
+; SSE41-NEXT:    pand %xmm4, %xmm3
+; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; SSE41-NEXT:    por %xmm3, %xmm2
+; SSE41-NEXT:    pxor %xmm10, %xmm2
+; SSE41-NEXT:    pxor %xmm11, %xmm0
+; SSE41-NEXT:    movdqa %xmm11, %xmm3
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm3
+; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[0,0,2,2]
+; SSE41-NEXT:    pcmpeqd %xmm11, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE41-NEXT:    pand %xmm4, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
+; SSE41-NEXT:    por %xmm0, %xmm3
+; SSE41-NEXT:    pxor %xmm10, %xmm3
+; SSE41-NEXT:    pcmpeqq %xmm3, %xmm2
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    pxor %xmm11, %xmm0
+; SSE41-NEXT:    movdqa %xmm11, %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm4
+; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSE41-NEXT:    pcmpeqd %xmm11, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm0[1,1,3,3]
+; SSE41-NEXT:    pand %xmm5, %xmm6
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[1,1,3,3]
+; SSE41-NEXT:    por %xmm6, %xmm0
+; SSE41-NEXT:    movdqa %xmm0, %xmm4
+; SSE41-NEXT:    pxor %xmm10, %xmm4
+; SSE41-NEXT:    pcmpeqq %xmm3, %xmm4
+; SSE41-NEXT:    pxor %xmm10, %xmm4
+; SSE41-NEXT:    pandn %xmm4, %xmm2
+; SSE41-NEXT:    blendvpd %xmm0, %xmm8, %xmm7
+; SSE41-NEXT:    movdqa %xmm2, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm7, %xmm1
+; SSE41-NEXT:    movapd %xmm9, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: v4i64:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm9
+; AVX1-NEXT:    vpxor %xmm10, %xmm10, %xmm10
+; AVX1-NEXT:    vpcmpgtq %xmm9, %xmm10, %xmm4
+; AVX1-NEXT:    vpcmpeqd %xmm5, %xmm5, %xmm5
+; AVX1-NEXT:    vpxor %xmm5, %xmm4, %xmm4
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm6
+; AVX1-NEXT:    vpcmpgtq %xmm6, %xmm10, %xmm7
+; AVX1-NEXT:    vpxor %xmm5, %xmm7, %xmm7
+; AVX1-NEXT:    vpcmpeqq %xmm4, %xmm7, %xmm4
+; AVX1-NEXT:    vpxor %xmm5, %xmm4, %xmm8
+; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm10, %xmm4
+; AVX1-NEXT:    vpxor %xmm5, %xmm4, %xmm4
+; AVX1-NEXT:    vpcmpgtq %xmm0, %xmm10, %xmm2
+; AVX1-NEXT:    vpxor %xmm5, %xmm2, %xmm2
+; AVX1-NEXT:    vpcmpeqq %xmm4, %xmm2, %xmm4
+; AVX1-NEXT:    vpxor %xmm5, %xmm4, %xmm4
+; AVX1-NEXT:    vinsertf128 $1, %xmm8, %ymm4, %ymm8
+; AVX1-NEXT:    vpsubq %xmm9, %xmm6, %xmm6
+; AVX1-NEXT:    vpcmpgtq %xmm6, %xmm10, %xmm4
+; AVX1-NEXT:    vpxor %xmm5, %xmm4, %xmm3
+; AVX1-NEXT:    vpcmpeqq %xmm3, %xmm7, %xmm3
+; AVX1-NEXT:    vpxor %xmm5, %xmm3, %xmm3
+; AVX1-NEXT:    vpsubq %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpcmpgtq %xmm0, %xmm10, %xmm1
+; AVX1-NEXT:    vpxor %xmm5, %xmm1, %xmm1
+; AVX1-NEXT:    vpcmpeqq %xmm1, %xmm2, %xmm1
+; AVX1-NEXT:    vpxor %xmm5, %xmm1, %xmm1
+; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm1, %ymm1
+; AVX1-NEXT:    vandpd %ymm1, %ymm8, %ymm1
+; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm0, %ymm2
+; AVX1-NEXT:    vmovapd {{.*#+}} ymm3 = [9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808]
+; AVX1-NEXT:    vblendvpd %ymm2, {{.*}}(%rip), %ymm3, %ymm2
+; AVX1-NEXT:    vinsertf128 $1, %xmm6, %ymm0, %ymm0
+; AVX1-NEXT:    vblendvpd %ymm1, %ymm2, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: v4i64:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX2-NEXT:    vpcmpgtq %ymm1, %ymm2, %ymm3
+; AVX2-NEXT:    vpcmpeqd %ymm4, %ymm4, %ymm4
+; AVX2-NEXT:    vpxor %ymm4, %ymm3, %ymm3
+; AVX2-NEXT:    vpcmpgtq %ymm0, %ymm2, %ymm5
+; AVX2-NEXT:    vpxor %ymm4, %ymm5, %ymm5
+; AVX2-NEXT:    vpcmpeqq %ymm3, %ymm5, %ymm3
+; AVX2-NEXT:    vpsubq %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpcmpgtq %ymm0, %ymm2, %ymm1
+; AVX2-NEXT:    vpxor %ymm4, %ymm1, %ymm1
+; AVX2-NEXT:    vpcmpeqq %ymm1, %ymm5, %ymm1
+; AVX2-NEXT:    vpxor %ymm4, %ymm1, %ymm1
+; AVX2-NEXT:    vpandn %ymm1, %ymm3, %ymm1
+; AVX2-NEXT:    vbroadcastsd {{.*#+}} ymm2 = [9223372036854775807,9223372036854775807,9223372036854775807,9223372036854775807]
+; AVX2-NEXT:    vbroadcastsd {{.*#+}} ymm3 = [9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808]
+; AVX2-NEXT:    vblendvpd %ymm0, %ymm2, %ymm3, %ymm2
+; AVX2-NEXT:    vblendvpd %ymm1, %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: v4i64:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX512-NEXT:    vpcmpnltq %ymm2, %ymm1, %k0
+; AVX512-NEXT:    vpcmpnltq %ymm2, %ymm0, %k1
+; AVX512-NEXT:    kxorw %k0, %k1, %k0
+; AVX512-NEXT:    vpsubq %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpcmpnltq %ymm2, %ymm0, %k2
+; AVX512-NEXT:    kxorw %k2, %k1, %k1
+; AVX512-NEXT:    kandw %k1, %k0, %k1
+; AVX512-NEXT:    vpcmpgtq %ymm0, %ymm2, %k2
+; AVX512-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808]
+; AVX512-NEXT:    vpbroadcastq {{.*}}(%rip), %ymm1 {%k2}
+; AVX512-NEXT:    vmovdqa64 %ymm1, %ymm0 {%k1}
+; AVX512-NEXT:    retq
+  %z = call <4 x i64> @llvm.ssub.sat.v4i64(<4 x i64> %x, <4 x i64> %y)
+  ret <4 x i64> %z
+}
+
+define <8 x i64> @v8i64(<8 x i64> %x, <8 x i64> %y) nounwind {
+; SSE2-LABEL: v8i64:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa %xmm1, %xmm8
+; SSE2-NEXT:    movdqa %xmm0, %xmm13
+; SSE2-NEXT:    movdqa {{.*#+}} xmm9 = [2147483648,2147483648]
+; SSE2-NEXT:    psubq %xmm4, %xmm13
+; SSE2-NEXT:    pxor %xmm9, %xmm4
+; SSE2-NEXT:    movdqa %xmm9, %xmm1
+; SSE2-NEXT:    pcmpgtd %xmm4, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm10 = xmm1[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm9, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSE2-NEXT:    pand %xmm10, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSE2-NEXT:    por %xmm4, %xmm1
+; SSE2-NEXT:    pcmpeqd %xmm10, %xmm10
+; SSE2-NEXT:    pxor %xmm10, %xmm1
+; SSE2-NEXT:    pxor %xmm9, %xmm0
+; SSE2-NEXT:    movdqa %xmm9, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm0, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm11 = xmm4[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm9, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE2-NEXT:    pand %xmm11, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm15 = xmm4[1,1,3,3]
+; SSE2-NEXT:    por %xmm0, %xmm15
+; SSE2-NEXT:    pxor %xmm10, %xmm15
+; SSE2-NEXT:    pcmpeqd %xmm15, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,0,3,2]
+; SSE2-NEXT:    pand %xmm1, %xmm0
+; SSE2-NEXT:    movdqa %xmm13, %xmm1
+; SSE2-NEXT:    pxor %xmm9, %xmm1
+; SSE2-NEXT:    movdqa %xmm9, %xmm11
+; SSE2-NEXT:    pcmpgtd %xmm1, %xmm11
+; SSE2-NEXT:    pshufd {{.*#+}} xmm12 = xmm11[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm9, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm14 = xmm1[1,1,3,3]
+; SSE2-NEXT:    pand %xmm12, %xmm14
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm11[1,1,3,3]
+; SSE2-NEXT:    por %xmm14, %xmm1
+; SSE2-NEXT:    movdqa %xmm1, %xmm11
+; SSE2-NEXT:    pxor %xmm10, %xmm11
+; SSE2-NEXT:    pcmpeqd %xmm15, %xmm11
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm11[1,0,3,2]
+; SSE2-NEXT:    pand %xmm11, %xmm4
+; SSE2-NEXT:    pxor %xmm10, %xmm4
+; SSE2-NEXT:    pandn %xmm4, %xmm0
+; SSE2-NEXT:    movdqa {{.*#+}} xmm11 = [9223372036854775808,9223372036854775808]
+; SSE2-NEXT:    movdqa %xmm1, %xmm4
+; SSE2-NEXT:    pandn %xmm11, %xmm4
+; SSE2-NEXT:    movdqa {{.*#+}} xmm12 = [9223372036854775807,9223372036854775807]
+; SSE2-NEXT:    pand %xmm12, %xmm1
+; SSE2-NEXT:    por %xmm4, %xmm1
+; SSE2-NEXT:    pand %xmm0, %xmm1
+; SSE2-NEXT:    pandn %xmm13, %xmm0
+; SSE2-NEXT:    por %xmm1, %xmm0
+; SSE2-NEXT:    movdqa %xmm8, %xmm1
+; SSE2-NEXT:    psubq %xmm5, %xmm8
+; SSE2-NEXT:    pxor %xmm9, %xmm5
+; SSE2-NEXT:    movdqa %xmm9, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm5, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm13 = xmm4[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm9, %xmm5
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
+; SSE2-NEXT:    pand %xmm13, %xmm5
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSE2-NEXT:    por %xmm5, %xmm4
+; SSE2-NEXT:    pxor %xmm10, %xmm4
+; SSE2-NEXT:    pxor %xmm9, %xmm1
+; SSE2-NEXT:    movdqa %xmm9, %xmm5
+; SSE2-NEXT:    pcmpgtd %xmm1, %xmm5
+; SSE2-NEXT:    pshufd {{.*#+}} xmm13 = xmm5[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm9, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSE2-NEXT:    pand %xmm13, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
+; SSE2-NEXT:    por %xmm1, %xmm5
+; SSE2-NEXT:    pxor %xmm10, %xmm5
+; SSE2-NEXT:    pcmpeqd %xmm5, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm4[1,0,3,2]
+; SSE2-NEXT:    pand %xmm4, %xmm1
+; SSE2-NEXT:    movdqa %xmm8, %xmm4
+; SSE2-NEXT:    pxor %xmm9, %xmm4
+; SSE2-NEXT:    movdqa %xmm9, %xmm13
+; SSE2-NEXT:    pcmpgtd %xmm4, %xmm13
+; SSE2-NEXT:    pshufd {{.*#+}} xmm14 = xmm13[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm9, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm15 = xmm4[1,1,3,3]
+; SSE2-NEXT:    pand %xmm14, %xmm15
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm13[1,1,3,3]
+; SSE2-NEXT:    por %xmm15, %xmm4
+; SSE2-NEXT:    movdqa %xmm4, %xmm13
+; SSE2-NEXT:    pxor %xmm10, %xmm13
+; SSE2-NEXT:    pcmpeqd %xmm5, %xmm13
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm13[1,0,3,2]
+; SSE2-NEXT:    pand %xmm13, %xmm5
+; SSE2-NEXT:    pxor %xmm10, %xmm5
+; SSE2-NEXT:    pandn %xmm5, %xmm1
+; SSE2-NEXT:    movdqa %xmm4, %xmm5
+; SSE2-NEXT:    pandn %xmm11, %xmm5
+; SSE2-NEXT:    pand %xmm12, %xmm4
+; SSE2-NEXT:    por %xmm5, %xmm4
+; SSE2-NEXT:    pand %xmm1, %xmm4
+; SSE2-NEXT:    pandn %xmm8, %xmm1
+; SSE2-NEXT:    por %xmm4, %xmm1
+; SSE2-NEXT:    movdqa %xmm2, %xmm4
+; SSE2-NEXT:    psubq %xmm6, %xmm2
+; SSE2-NEXT:    pxor %xmm9, %xmm6
+; SSE2-NEXT:    movdqa %xmm9, %xmm5
+; SSE2-NEXT:    pcmpgtd %xmm6, %xmm5
+; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm5[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm9, %xmm6
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
+; SSE2-NEXT:    pand %xmm8, %xmm6
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
+; SSE2-NEXT:    por %xmm6, %xmm5
+; SSE2-NEXT:    pxor %xmm10, %xmm5
+; SSE2-NEXT:    pxor %xmm9, %xmm4
+; SSE2-NEXT:    movdqa %xmm9, %xmm6
+; SSE2-NEXT:    pcmpgtd %xmm4, %xmm6
+; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm6[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm9, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSE2-NEXT:    pand %xmm8, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
+; SSE2-NEXT:    por %xmm4, %xmm6
+; SSE2-NEXT:    pxor %xmm10, %xmm6
+; SSE2-NEXT:    pcmpeqd %xmm6, %xmm5
+; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm5[1,0,3,2]
+; SSE2-NEXT:    pand %xmm5, %xmm8
+; SSE2-NEXT:    movdqa %xmm2, %xmm5
+; SSE2-NEXT:    pxor %xmm9, %xmm5
+; SSE2-NEXT:    movdqa %xmm9, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm5, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm13 = xmm4[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm9, %xmm5
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
+; SSE2-NEXT:    pand %xmm13, %xmm5
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSE2-NEXT:    por %xmm5, %xmm4
+; SSE2-NEXT:    movdqa %xmm4, %xmm5
+; SSE2-NEXT:    pxor %xmm10, %xmm5
+; SSE2-NEXT:    pcmpeqd %xmm6, %xmm5
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[1,0,3,2]
+; SSE2-NEXT:    pand %xmm5, %xmm6
+; SSE2-NEXT:    pxor %xmm10, %xmm6
+; SSE2-NEXT:    pandn %xmm6, %xmm8
+; SSE2-NEXT:    movdqa %xmm4, %xmm5
+; SSE2-NEXT:    pandn %xmm11, %xmm5
+; SSE2-NEXT:    pand %xmm12, %xmm4
+; SSE2-NEXT:    por %xmm5, %xmm4
+; SSE2-NEXT:    pand %xmm8, %xmm4
+; SSE2-NEXT:    pandn %xmm2, %xmm8
+; SSE2-NEXT:    por %xmm4, %xmm8
+; SSE2-NEXT:    movdqa %xmm3, %xmm2
+; SSE2-NEXT:    psubq %xmm7, %xmm3
+; SSE2-NEXT:    pxor %xmm9, %xmm7
+; SSE2-NEXT:    movdqa %xmm9, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm7, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm9, %xmm7
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm7[1,1,3,3]
+; SSE2-NEXT:    pand %xmm5, %xmm6
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSE2-NEXT:    por %xmm6, %xmm4
+; SSE2-NEXT:    pxor %xmm10, %xmm4
+; SSE2-NEXT:    pxor %xmm9, %xmm2
+; SSE2-NEXT:    movdqa %xmm9, %xmm5
+; SSE2-NEXT:    pcmpgtd %xmm2, %xmm5
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm9, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; SSE2-NEXT:    pand %xmm6, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[1,1,3,3]
+; SSE2-NEXT:    por %xmm2, %xmm6
+; SSE2-NEXT:    pxor %xmm10, %xmm6
+; SSE2-NEXT:    pcmpeqd %xmm6, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[1,0,3,2]
+; SSE2-NEXT:    pand %xmm4, %xmm5
+; SSE2-NEXT:    movdqa %xmm3, %xmm2
+; SSE2-NEXT:    pxor %xmm9, %xmm2
+; SSE2-NEXT:    movdqa %xmm9, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm2, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm4[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm9, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; SSE2-NEXT:    pand %xmm7, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSE2-NEXT:    por %xmm2, %xmm4
+; SSE2-NEXT:    movdqa %xmm4, %xmm2
+; SSE2-NEXT:    pxor %xmm10, %xmm2
+; SSE2-NEXT:    pcmpeqd %xmm6, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm2[1,0,3,2]
+; SSE2-NEXT:    pand %xmm2, %xmm6
+; SSE2-NEXT:    pxor %xmm10, %xmm6
+; SSE2-NEXT:    pandn %xmm6, %xmm5
+; SSE2-NEXT:    movdqa %xmm4, %xmm2
+; SSE2-NEXT:    pandn %xmm11, %xmm2
+; SSE2-NEXT:    pand %xmm12, %xmm4
+; SSE2-NEXT:    por %xmm2, %xmm4
+; SSE2-NEXT:    pand %xmm5, %xmm4
+; SSE2-NEXT:    pandn %xmm3, %xmm5
+; SSE2-NEXT:    por %xmm4, %xmm5
+; SSE2-NEXT:    movdqa %xmm8, %xmm2
+; SSE2-NEXT:    movdqa %xmm5, %xmm3
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: v8i64:
+; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    movdqa %xmm1, %xmm8
+; SSSE3-NEXT:    movdqa %xmm0, %xmm13
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm9 = [2147483648,2147483648]
+; SSSE3-NEXT:    psubq %xmm4, %xmm13
+; SSSE3-NEXT:    pxor %xmm9, %xmm4
+; SSSE3-NEXT:    movdqa %xmm9, %xmm1
+; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm10 = xmm1[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm10, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSSE3-NEXT:    por %xmm4, %xmm1
+; SSSE3-NEXT:    pcmpeqd %xmm10, %xmm10
+; SSSE3-NEXT:    pxor %xmm10, %xmm1
+; SSSE3-NEXT:    pxor %xmm9, %xmm0
+; SSSE3-NEXT:    movdqa %xmm9, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm11 = xmm4[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm11, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm15 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    por %xmm0, %xmm15
+; SSSE3-NEXT:    pxor %xmm10, %xmm15
+; SSSE3-NEXT:    pcmpeqd %xmm15, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,0,3,2]
+; SSSE3-NEXT:    pand %xmm1, %xmm0
+; SSSE3-NEXT:    movdqa %xmm13, %xmm1
+; SSSE3-NEXT:    pxor %xmm9, %xmm1
+; SSSE3-NEXT:    movdqa %xmm9, %xmm11
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm11
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm12 = xmm11[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm14 = xmm1[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm12, %xmm14
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm11[1,1,3,3]
+; SSSE3-NEXT:    por %xmm14, %xmm1
+; SSSE3-NEXT:    movdqa %xmm1, %xmm11
+; SSSE3-NEXT:    pxor %xmm10, %xmm11
+; SSSE3-NEXT:    pcmpeqd %xmm15, %xmm11
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm11[1,0,3,2]
+; SSSE3-NEXT:    pand %xmm11, %xmm4
+; SSSE3-NEXT:    pxor %xmm10, %xmm4
+; SSSE3-NEXT:    pandn %xmm4, %xmm0
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm11 = [9223372036854775808,9223372036854775808]
+; SSSE3-NEXT:    movdqa %xmm1, %xmm4
+; SSSE3-NEXT:    pandn %xmm11, %xmm4
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm12 = [9223372036854775807,9223372036854775807]
+; SSSE3-NEXT:    pand %xmm12, %xmm1
+; SSSE3-NEXT:    por %xmm4, %xmm1
+; SSSE3-NEXT:    pand %xmm0, %xmm1
+; SSSE3-NEXT:    pandn %xmm13, %xmm0
+; SSSE3-NEXT:    por %xmm1, %xmm0
+; SSSE3-NEXT:    movdqa %xmm8, %xmm1
+; SSSE3-NEXT:    psubq %xmm5, %xmm8
+; SSSE3-NEXT:    pxor %xmm9, %xmm5
+; SSSE3-NEXT:    movdqa %xmm9, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm5, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm13 = xmm4[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm5
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm13, %xmm5
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    por %xmm5, %xmm4
+; SSSE3-NEXT:    pxor %xmm10, %xmm4
+; SSSE3-NEXT:    pxor %xmm9, %xmm1
+; SSSE3-NEXT:    movdqa %xmm9, %xmm5
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm5
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm13 = xmm5[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm13, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
+; SSSE3-NEXT:    por %xmm1, %xmm5
+; SSSE3-NEXT:    pxor %xmm10, %xmm5
+; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm4[1,0,3,2]
+; SSSE3-NEXT:    pand %xmm4, %xmm1
+; SSSE3-NEXT:    movdqa %xmm8, %xmm4
+; SSSE3-NEXT:    pxor %xmm9, %xmm4
+; SSSE3-NEXT:    movdqa %xmm9, %xmm13
+; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm13
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm14 = xmm13[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm15 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm14, %xmm15
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm13[1,1,3,3]
+; SSSE3-NEXT:    por %xmm15, %xmm4
+; SSSE3-NEXT:    movdqa %xmm4, %xmm13
+; SSSE3-NEXT:    pxor %xmm10, %xmm13
+; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm13
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm13[1,0,3,2]
+; SSSE3-NEXT:    pand %xmm13, %xmm5
+; SSSE3-NEXT:    pxor %xmm10, %xmm5
+; SSSE3-NEXT:    pandn %xmm5, %xmm1
+; SSSE3-NEXT:    movdqa %xmm4, %xmm5
+; SSSE3-NEXT:    pandn %xmm11, %xmm5
+; SSSE3-NEXT:    pand %xmm12, %xmm4
+; SSSE3-NEXT:    por %xmm5, %xmm4
+; SSSE3-NEXT:    pand %xmm1, %xmm4
+; SSSE3-NEXT:    pandn %xmm8, %xmm1
+; SSSE3-NEXT:    por %xmm4, %xmm1
+; SSSE3-NEXT:    movdqa %xmm2, %xmm4
+; SSSE3-NEXT:    psubq %xmm6, %xmm2
+; SSSE3-NEXT:    pxor %xmm9, %xmm6
+; SSSE3-NEXT:    movdqa %xmm9, %xmm5
+; SSSE3-NEXT:    pcmpgtd %xmm6, %xmm5
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm8 = xmm5[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm6
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm8, %xmm6
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
+; SSSE3-NEXT:    por %xmm6, %xmm5
+; SSSE3-NEXT:    pxor %xmm10, %xmm5
+; SSSE3-NEXT:    pxor %xmm9, %xmm4
+; SSSE3-NEXT:    movdqa %xmm9, %xmm6
+; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm6
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm8 = xmm6[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm8, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
+; SSSE3-NEXT:    por %xmm4, %xmm6
+; SSSE3-NEXT:    pxor %xmm10, %xmm6
+; SSSE3-NEXT:    pcmpeqd %xmm6, %xmm5
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm8 = xmm5[1,0,3,2]
+; SSSE3-NEXT:    pand %xmm5, %xmm8
+; SSSE3-NEXT:    movdqa %xmm2, %xmm5
+; SSSE3-NEXT:    pxor %xmm9, %xmm5
+; SSSE3-NEXT:    movdqa %xmm9, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm5, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm13 = xmm4[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm5
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm13, %xmm5
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    por %xmm5, %xmm4
+; SSSE3-NEXT:    movdqa %xmm4, %xmm5
+; SSSE3-NEXT:    pxor %xmm10, %xmm5
+; SSSE3-NEXT:    pcmpeqd %xmm6, %xmm5
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[1,0,3,2]
+; SSSE3-NEXT:    pand %xmm5, %xmm6
+; SSSE3-NEXT:    pxor %xmm10, %xmm6
+; SSSE3-NEXT:    pandn %xmm6, %xmm8
+; SSSE3-NEXT:    movdqa %xmm4, %xmm5
+; SSSE3-NEXT:    pandn %xmm11, %xmm5
+; SSSE3-NEXT:    pand %xmm12, %xmm4
+; SSSE3-NEXT:    por %xmm5, %xmm4
+; SSSE3-NEXT:    pand %xmm8, %xmm4
+; SSSE3-NEXT:    pandn %xmm2, %xmm8
+; SSSE3-NEXT:    por %xmm4, %xmm8
+; SSSE3-NEXT:    movdqa %xmm3, %xmm2
+; SSSE3-NEXT:    psubq %xmm7, %xmm3
+; SSSE3-NEXT:    pxor %xmm9, %xmm7
+; SSSE3-NEXT:    movdqa %xmm9, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm7, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm7
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm7[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm5, %xmm6
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    por %xmm6, %xmm4
+; SSSE3-NEXT:    pxor %xmm10, %xmm4
+; SSSE3-NEXT:    pxor %xmm9, %xmm2
+; SSSE3-NEXT:    movdqa %xmm9, %xmm5
+; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm5
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm6, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[1,1,3,3]
+; SSSE3-NEXT:    por %xmm2, %xmm6
+; SSSE3-NEXT:    pxor %xmm10, %xmm6
+; SSSE3-NEXT:    pcmpeqd %xmm6, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[1,0,3,2]
+; SSSE3-NEXT:    pand %xmm4, %xmm5
+; SSSE3-NEXT:    movdqa %xmm3, %xmm2
+; SSSE3-NEXT:    pxor %xmm9, %xmm2
+; SSSE3-NEXT:    movdqa %xmm9, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm4[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm7, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    por %xmm2, %xmm4
+; SSSE3-NEXT:    movdqa %xmm4, %xmm2
+; SSSE3-NEXT:    pxor %xmm10, %xmm2
+; SSSE3-NEXT:    pcmpeqd %xmm6, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm2[1,0,3,2]
+; SSSE3-NEXT:    pand %xmm2, %xmm6
+; SSSE3-NEXT:    pxor %xmm10, %xmm6
+; SSSE3-NEXT:    pandn %xmm6, %xmm5
+; SSSE3-NEXT:    movdqa %xmm4, %xmm2
+; SSSE3-NEXT:    pandn %xmm11, %xmm2
+; SSSE3-NEXT:    pand %xmm12, %xmm4
+; SSSE3-NEXT:    por %xmm2, %xmm4
+; SSSE3-NEXT:    pand %xmm5, %xmm4
+; SSSE3-NEXT:    pandn %xmm3, %xmm5
+; SSSE3-NEXT:    por %xmm4, %xmm5
+; SSSE3-NEXT:    movdqa %xmm8, %xmm2
+; SSSE3-NEXT:    movdqa %xmm5, %xmm3
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: v8i64:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movdqa %xmm0, %xmm8
+; SSE41-NEXT:    movdqa {{.*#+}} xmm9 = [2147483648,2147483648]
+; SSE41-NEXT:    movdqa %xmm0, %xmm11
+; SSE41-NEXT:    psubq %xmm4, %xmm8
+; SSE41-NEXT:    pxor %xmm9, %xmm4
+; SSE41-NEXT:    movdqa %xmm9, %xmm0
+; SSE41-NEXT:    pcmpgtd %xmm4, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm10 = xmm0[0,0,2,2]
+; SSE41-NEXT:    pcmpeqd %xmm9, %xmm4
+; SSE41-NEXT:    pshufd {{.*#+}} xmm12 = xmm4[1,1,3,3]
+; SSE41-NEXT:    pand %xmm10, %xmm12
+; SSE41-NEXT:    pshufd {{.*#+}} xmm15 = xmm0[1,1,3,3]
+; SSE41-NEXT:    por %xmm12, %xmm15
+; SSE41-NEXT:    pcmpeqd %xmm10, %xmm10
+; SSE41-NEXT:    pxor %xmm10, %xmm15
+; SSE41-NEXT:    pxor %xmm9, %xmm11
+; SSE41-NEXT:    movdqa %xmm9, %xmm0
+; SSE41-NEXT:    pcmpgtd %xmm11, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm12 = xmm0[0,0,2,2]
+; SSE41-NEXT:    pcmpeqd %xmm9, %xmm11
+; SSE41-NEXT:    pshufd {{.*#+}} xmm11 = xmm11[1,1,3,3]
+; SSE41-NEXT:    pand %xmm12, %xmm11
+; SSE41-NEXT:    pshufd {{.*#+}} xmm12 = xmm0[1,1,3,3]
+; SSE41-NEXT:    por %xmm11, %xmm12
+; SSE41-NEXT:    pxor %xmm10, %xmm12
+; SSE41-NEXT:    pcmpeqq %xmm12, %xmm15
+; SSE41-NEXT:    movdqa %xmm8, %xmm0
+; SSE41-NEXT:    pxor %xmm9, %xmm0
+; SSE41-NEXT:    movdqa %xmm9, %xmm11
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm11
+; SSE41-NEXT:    pshufd {{.*#+}} xmm13 = xmm11[0,0,2,2]
+; SSE41-NEXT:    pcmpeqd %xmm9, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm14 = xmm0[1,1,3,3]
+; SSE41-NEXT:    pand %xmm13, %xmm14
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm11[1,1,3,3]
+; SSE41-NEXT:    por %xmm14, %xmm0
+; SSE41-NEXT:    movdqa %xmm0, %xmm4
+; SSE41-NEXT:    pxor %xmm10, %xmm4
+; SSE41-NEXT:    pcmpeqq %xmm12, %xmm4
+; SSE41-NEXT:    pxor %xmm10, %xmm4
+; SSE41-NEXT:    pandn %xmm4, %xmm15
+; SSE41-NEXT:    movapd {{.*#+}} xmm12 = [9223372036854775807,9223372036854775807]
+; SSE41-NEXT:    movapd {{.*#+}} xmm11 = [9223372036854775808,9223372036854775808]
+; SSE41-NEXT:    movapd %xmm11, %xmm4
+; SSE41-NEXT:    blendvpd %xmm0, %xmm12, %xmm4
+; SSE41-NEXT:    movdqa %xmm15, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm4, %xmm8
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    psubq %xmm5, %xmm1
+; SSE41-NEXT:    pxor %xmm9, %xmm5
+; SSE41-NEXT:    movdqa %xmm9, %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm5, %xmm4
+; SSE41-NEXT:    pshufd {{.*#+}} xmm13 = xmm4[0,0,2,2]
+; SSE41-NEXT:    pcmpeqd %xmm9, %xmm5
+; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
+; SSE41-NEXT:    pand %xmm13, %xmm5
+; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSE41-NEXT:    por %xmm5, %xmm4
+; SSE41-NEXT:    pxor %xmm10, %xmm4
+; SSE41-NEXT:    pxor %xmm9, %xmm0
+; SSE41-NEXT:    movdqa %xmm9, %xmm5
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm5
+; SSE41-NEXT:    pshufd {{.*#+}} xmm13 = xmm5[0,0,2,2]
+; SSE41-NEXT:    pcmpeqd %xmm9, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE41-NEXT:    pand %xmm13, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm14 = xmm5[1,1,3,3]
+; SSE41-NEXT:    por %xmm0, %xmm14
+; SSE41-NEXT:    pxor %xmm10, %xmm14
+; SSE41-NEXT:    pcmpeqq %xmm14, %xmm4
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    pxor %xmm9, %xmm0
+; SSE41-NEXT:    movdqa %xmm9, %xmm13
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm13
+; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm13[0,0,2,2]
+; SSE41-NEXT:    pcmpeqd %xmm9, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm15 = xmm0[1,1,3,3]
+; SSE41-NEXT:    pand %xmm5, %xmm15
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm13[1,1,3,3]
+; SSE41-NEXT:    por %xmm15, %xmm0
+; SSE41-NEXT:    movdqa %xmm0, %xmm5
+; SSE41-NEXT:    pxor %xmm10, %xmm5
+; SSE41-NEXT:    pcmpeqq %xmm14, %xmm5
+; SSE41-NEXT:    pxor %xmm10, %xmm5
+; SSE41-NEXT:    pandn %xmm5, %xmm4
+; SSE41-NEXT:    movapd %xmm11, %xmm5
+; SSE41-NEXT:    blendvpd %xmm0, %xmm12, %xmm5
+; SSE41-NEXT:    movdqa %xmm4, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm5, %xmm1
+; SSE41-NEXT:    movdqa %xmm2, %xmm0
+; SSE41-NEXT:    psubq %xmm6, %xmm2
+; SSE41-NEXT:    pxor %xmm9, %xmm6
+; SSE41-NEXT:    movdqa %xmm9, %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm6, %xmm4
+; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSE41-NEXT:    pcmpeqd %xmm9, %xmm6
+; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
+; SSE41-NEXT:    pand %xmm5, %xmm6
+; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSE41-NEXT:    por %xmm6, %xmm4
+; SSE41-NEXT:    pxor %xmm10, %xmm4
+; SSE41-NEXT:    pxor %xmm9, %xmm0
+; SSE41-NEXT:    movdqa %xmm9, %xmm5
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm5
+; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
+; SSE41-NEXT:    pcmpeqd %xmm9, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE41-NEXT:    pand %xmm6, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
+; SSE41-NEXT:    por %xmm0, %xmm5
+; SSE41-NEXT:    pxor %xmm10, %xmm5
+; SSE41-NEXT:    pcmpeqq %xmm5, %xmm4
+; SSE41-NEXT:    movdqa %xmm2, %xmm0
+; SSE41-NEXT:    pxor %xmm9, %xmm0
+; SSE41-NEXT:    movdqa %xmm9, %xmm6
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm6
+; SSE41-NEXT:    pshufd {{.*#+}} xmm13 = xmm6[0,0,2,2]
+; SSE41-NEXT:    pcmpeqd %xmm9, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm14 = xmm0[1,1,3,3]
+; SSE41-NEXT:    pand %xmm13, %xmm14
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[1,1,3,3]
+; SSE41-NEXT:    por %xmm14, %xmm0
+; SSE41-NEXT:    movdqa %xmm0, %xmm6
+; SSE41-NEXT:    pxor %xmm10, %xmm6
+; SSE41-NEXT:    pcmpeqq %xmm5, %xmm6
+; SSE41-NEXT:    pxor %xmm10, %xmm6
+; SSE41-NEXT:    pandn %xmm6, %xmm4
+; SSE41-NEXT:    movapd %xmm11, %xmm5
+; SSE41-NEXT:    blendvpd %xmm0, %xmm12, %xmm5
+; SSE41-NEXT:    movdqa %xmm4, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm5, %xmm2
+; SSE41-NEXT:    movdqa %xmm3, %xmm0
+; SSE41-NEXT:    psubq %xmm7, %xmm3
+; SSE41-NEXT:    pxor %xmm9, %xmm7
+; SSE41-NEXT:    movdqa %xmm9, %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm7, %xmm4
+; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSE41-NEXT:    pcmpeqd %xmm9, %xmm7
+; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm7[1,1,3,3]
+; SSE41-NEXT:    pand %xmm5, %xmm6
+; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSE41-NEXT:    por %xmm6, %xmm4
+; SSE41-NEXT:    pxor %xmm10, %xmm4
+; SSE41-NEXT:    pxor %xmm9, %xmm0
+; SSE41-NEXT:    movdqa %xmm9, %xmm5
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm5
+; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
+; SSE41-NEXT:    pcmpeqd %xmm9, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE41-NEXT:    pand %xmm6, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
+; SSE41-NEXT:    por %xmm0, %xmm5
+; SSE41-NEXT:    pxor %xmm10, %xmm5
+; SSE41-NEXT:    pcmpeqq %xmm5, %xmm4
+; SSE41-NEXT:    movdqa %xmm3, %xmm0
+; SSE41-NEXT:    pxor %xmm9, %xmm0
+; SSE41-NEXT:    movdqa %xmm9, %xmm6
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm6
+; SSE41-NEXT:    pshufd {{.*#+}} xmm13 = xmm6[0,0,2,2]
+; SSE41-NEXT:    pcmpeqd %xmm9, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm7 = xmm0[1,1,3,3]
+; SSE41-NEXT:    pand %xmm13, %xmm7
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[1,1,3,3]
+; SSE41-NEXT:    por %xmm7, %xmm0
+; SSE41-NEXT:    movdqa %xmm0, %xmm6
+; SSE41-NEXT:    pxor %xmm10, %xmm6
+; SSE41-NEXT:    pcmpeqq %xmm5, %xmm6
+; SSE41-NEXT:    pxor %xmm10, %xmm6
+; SSE41-NEXT:    pandn %xmm6, %xmm4
+; SSE41-NEXT:    blendvpd %xmm0, %xmm12, %xmm11
+; SSE41-NEXT:    movdqa %xmm4, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm11, %xmm3
+; SSE41-NEXT:    movapd %xmm8, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: v8i64:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm8
+; AVX1-NEXT:    vpxor %xmm12, %xmm12, %xmm12
+; AVX1-NEXT:    vpcmpgtq %xmm8, %xmm12, %xmm7
+; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
+; AVX1-NEXT:    vpxor %xmm4, %xmm7, %xmm9
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm6
+; AVX1-NEXT:    vpcmpgtq %xmm6, %xmm12, %xmm7
+; AVX1-NEXT:    vpxor %xmm4, %xmm7, %xmm10
+; AVX1-NEXT:    vpcmpeqq %xmm9, %xmm10, %xmm7
+; AVX1-NEXT:    vpxor %xmm4, %xmm7, %xmm9
+; AVX1-NEXT:    vpcmpgtq %xmm2, %xmm12, %xmm7
+; AVX1-NEXT:    vpxor %xmm4, %xmm7, %xmm11
+; AVX1-NEXT:    vpcmpgtq %xmm0, %xmm12, %xmm7
+; AVX1-NEXT:    vpxor %xmm4, %xmm7, %xmm7
+; AVX1-NEXT:    vpcmpeqq %xmm11, %xmm7, %xmm5
+; AVX1-NEXT:    vpxor %xmm4, %xmm5, %xmm5
+; AVX1-NEXT:    vinsertf128 $1, %xmm9, %ymm5, %ymm9
+; AVX1-NEXT:    vpsubq %xmm8, %xmm6, %xmm8
+; AVX1-NEXT:    vpcmpgtq %xmm8, %xmm12, %xmm5
+; AVX1-NEXT:    vpxor %xmm4, %xmm5, %xmm6
+; AVX1-NEXT:    vpcmpeqq %xmm6, %xmm10, %xmm6
+; AVX1-NEXT:    vpxor %xmm4, %xmm6, %xmm6
+; AVX1-NEXT:    vpsubq %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpcmpgtq %xmm0, %xmm12, %xmm2
+; AVX1-NEXT:    vpxor %xmm4, %xmm2, %xmm2
+; AVX1-NEXT:    vpcmpeqq %xmm2, %xmm7, %xmm2
+; AVX1-NEXT:    vpxor %xmm4, %xmm2, %xmm2
+; AVX1-NEXT:    vinsertf128 $1, %xmm6, %ymm2, %ymm2
+; AVX1-NEXT:    vandpd %ymm2, %ymm9, %ymm6
+; AVX1-NEXT:    vinsertf128 $1, %xmm5, %ymm0, %ymm7
+; AVX1-NEXT:    vmovapd {{.*#+}} ymm9 = [9223372036854775807,9223372036854775807,9223372036854775807,9223372036854775807]
+; AVX1-NEXT:    vmovapd {{.*#+}} ymm10 = [9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808]
+; AVX1-NEXT:    vblendvpd %ymm7, %ymm9, %ymm10, %ymm7
+; AVX1-NEXT:    vinsertf128 $1, %xmm8, %ymm0, %ymm0
+; AVX1-NEXT:    vblendvpd %ymm6, %ymm7, %ymm0, %ymm0
+; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm13
+; AVX1-NEXT:    vpcmpgtq %xmm13, %xmm12, %xmm7
+; AVX1-NEXT:    vpxor %xmm4, %xmm7, %xmm7
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; AVX1-NEXT:    vpcmpgtq %xmm2, %xmm12, %xmm5
+; AVX1-NEXT:    vpxor %xmm4, %xmm5, %xmm5
+; AVX1-NEXT:    vpcmpeqq %xmm7, %xmm5, %xmm7
+; AVX1-NEXT:    vpxor %xmm4, %xmm7, %xmm8
+; AVX1-NEXT:    vpcmpgtq %xmm3, %xmm12, %xmm7
+; AVX1-NEXT:    vpxor %xmm4, %xmm7, %xmm11
+; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm12, %xmm7
+; AVX1-NEXT:    vpxor %xmm4, %xmm7, %xmm7
+; AVX1-NEXT:    vpcmpeqq %xmm11, %xmm7, %xmm6
+; AVX1-NEXT:    vpxor %xmm4, %xmm6, %xmm6
+; AVX1-NEXT:    vinsertf128 $1, %xmm8, %ymm6, %ymm8
+; AVX1-NEXT:    vpsubq %xmm13, %xmm2, %xmm11
+; AVX1-NEXT:    vpcmpgtq %xmm11, %xmm12, %xmm6
+; AVX1-NEXT:    vpxor %xmm4, %xmm6, %xmm2
+; AVX1-NEXT:    vpcmpeqq %xmm2, %xmm5, %xmm2
+; AVX1-NEXT:    vpxor %xmm4, %xmm2, %xmm2
+; AVX1-NEXT:    vpsubq %xmm3, %xmm1, %xmm1
+; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm12, %xmm3
+; AVX1-NEXT:    vpxor %xmm4, %xmm3, %xmm3
+; AVX1-NEXT:    vpcmpeqq %xmm3, %xmm7, %xmm3
+; AVX1-NEXT:    vpxor %xmm4, %xmm3, %xmm3
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm3, %ymm2
+; AVX1-NEXT:    vandpd %ymm2, %ymm8, %ymm2
+; AVX1-NEXT:    vinsertf128 $1, %xmm6, %ymm1, %ymm3
+; AVX1-NEXT:    vblendvpd %ymm3, %ymm9, %ymm10, %ymm3
+; AVX1-NEXT:    vinsertf128 $1, %xmm11, %ymm1, %ymm1
+; AVX1-NEXT:    vblendvpd %ymm2, %ymm3, %ymm1, %ymm1
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: v8i64:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpxor %xmm4, %xmm4, %xmm4
+; AVX2-NEXT:    vpcmpgtq %ymm2, %ymm4, %ymm5
+; AVX2-NEXT:    vpcmpeqd %ymm6, %ymm6, %ymm6
+; AVX2-NEXT:    vpxor %ymm6, %ymm5, %ymm5
+; AVX2-NEXT:    vpcmpgtq %ymm0, %ymm4, %ymm7
+; AVX2-NEXT:    vpxor %ymm6, %ymm7, %ymm7
+; AVX2-NEXT:    vpcmpeqq %ymm5, %ymm7, %ymm5
+; AVX2-NEXT:    vpsubq %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vpcmpgtq %ymm0, %ymm4, %ymm2
+; AVX2-NEXT:    vpxor %ymm6, %ymm2, %ymm2
+; AVX2-NEXT:    vpcmpeqq %ymm2, %ymm7, %ymm2
+; AVX2-NEXT:    vpxor %ymm6, %ymm2, %ymm2
+; AVX2-NEXT:    vpandn %ymm2, %ymm5, %ymm2
+; AVX2-NEXT:    vbroadcastsd {{.*#+}} ymm5 = [9223372036854775807,9223372036854775807,9223372036854775807,9223372036854775807]
+; AVX2-NEXT:    vbroadcastsd {{.*#+}} ymm7 = [9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808]
+; AVX2-NEXT:    vblendvpd %ymm0, %ymm5, %ymm7, %ymm8
+; AVX2-NEXT:    vblendvpd %ymm2, %ymm8, %ymm0, %ymm0
+; AVX2-NEXT:    vpcmpgtq %ymm3, %ymm4, %ymm2
+; AVX2-NEXT:    vpxor %ymm6, %ymm2, %ymm2
+; AVX2-NEXT:    vpcmpgtq %ymm1, %ymm4, %ymm8
+; AVX2-NEXT:    vpxor %ymm6, %ymm8, %ymm8
+; AVX2-NEXT:    vpcmpeqq %ymm2, %ymm8, %ymm2
+; AVX2-NEXT:    vpsubq %ymm3, %ymm1, %ymm1
+; AVX2-NEXT:    vpcmpgtq %ymm1, %ymm4, %ymm3
+; AVX2-NEXT:    vpxor %ymm6, %ymm3, %ymm3
+; AVX2-NEXT:    vpcmpeqq %ymm3, %ymm8, %ymm3
+; AVX2-NEXT:    vpxor %ymm6, %ymm3, %ymm3
+; AVX2-NEXT:    vpandn %ymm3, %ymm2, %ymm2
+; AVX2-NEXT:    vblendvpd %ymm1, %ymm5, %ymm7, %ymm3
+; AVX2-NEXT:    vblendvpd %ymm2, %ymm3, %ymm1, %ymm1
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: v8i64:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX512-NEXT:    vpcmpnltq %zmm2, %zmm1, %k0
+; AVX512-NEXT:    vpcmpnltq %zmm2, %zmm0, %k1
+; AVX512-NEXT:    kxorw %k0, %k1, %k0
+; AVX512-NEXT:    vpsubq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpcmpnltq %zmm2, %zmm0, %k2
+; AVX512-NEXT:    kxorw %k2, %k1, %k1
+; AVX512-NEXT:    kandw %k1, %k0, %k1
+; AVX512-NEXT:    vpcmpgtq %zmm0, %zmm2, %k2
+; AVX512-NEXT:    vpbroadcastq {{.*#+}} zmm1 = [9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808]
+; AVX512-NEXT:    vpbroadcastq {{.*}}(%rip), %zmm1 {%k2}
+; AVX512-NEXT:    vmovdqa64 %zmm1, %zmm0 {%k1}
+; AVX512-NEXT:    retq
+  %z = call <8 x i64> @llvm.ssub.sat.v8i64(<8 x i64> %x, <8 x i64> %y)
+  ret <8 x i64> %z
 }
 
 define <2 x i128> @v2i128(<2 x i128> %x, <2 x i128> %y) nounwind {
diff --git a/test/CodeGen/X86/stack-folding-fp-avx512vl.ll b/test/CodeGen/X86/stack-folding-fp-avx512vl.ll
index fd8bed3c82e796b1ec386a5a984253a3d5c61d58..0cb72d16245be5bf2cc7d7f43d1b106fbc3bdf01 100644
--- a/test/CodeGen/X86/stack-folding-fp-avx512vl.ll
+++ b/test/CodeGen/X86/stack-folding-fp-avx512vl.ll
@@ -639,73 +639,65 @@ define <4 x float> @stack_fold_vpermt2ps(<4 x float> %x0, <4 x i32> %x1, <4 x fl
   ;CHECK-LABEL: stack_fold_vpermt2ps
   ;CHECK:       vpermt2ps {{-?[0-9]*}}(%rsp), %xmm1, %xmm0 # 16-byte Folded Reload
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
-  %res = call <4 x float> @llvm.x86.avx512.mask.vpermi2var.ps.128(<4 x float> %x0, <4 x i32> %x1, <4 x float> %x2, i8 -1)
-  ret <4 x float> %res
+  %2 = call <4 x float> @llvm.x86.avx512.vpermi2var.ps.128(<4 x float> %x0, <4 x i32> %x1, <4 x float> %x2)
+  ret <4 x float> %2
 }
-declare <4 x float> @llvm.x86.avx512.mask.vpermi2var.ps.128(<4 x float>, <4 x i32>, <4 x float>, i8)
 
 define <4 x float> @stack_fold_vpermi2ps(<4 x i32> %x0, <4 x float> %x1, <4 x float> %x2) {
   ;CHECK-LABEL: stack_fold_vpermi2ps
   ;CHECK:       vpermi2ps {{-?[0-9]*}}(%rsp), %xmm1, %xmm0 # 16-byte Folded Reload
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
-  %res = call <4 x float> @llvm.x86.avx512.mask.vpermt2var.ps.128(<4 x i32> %x0, <4 x float> %x1, <4 x float> %x2, i8 -1)
-  ret <4 x float> %res
+  %2 = call <4 x float> @llvm.x86.avx512.vpermi2var.ps.128(<4 x float> %x1, <4 x i32> %x0, <4 x float> %x2)
+  ret <4 x float> %2
 }
-declare <4 x float> @llvm.x86.avx512.mask.vpermt2var.ps.128(<4 x i32>, <4 x float>, <4 x float>, i8)
 
 define <2 x double> @stack_fold_vpermt2pd(<2 x double> %x0, <2 x i64> %x1, <2 x double> %x2) {
   ;CHECK-LABEL: stack_fold_vpermt2pd
   ;CHECK:       vpermt2pd {{-?[0-9]*}}(%rsp), %xmm1, %xmm0 # 16-byte Folded Reload
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
-  %res = call <2 x double> @llvm.x86.avx512.mask.vpermi2var.pd.128(<2 x double> %x0, <2 x i64> %x1, <2 x double> %x2, i8 -1)
-  ret <2 x double> %res
+  %2 = call <2 x double> @llvm.x86.avx512.vpermi2var.pd.128(<2 x double> %x0, <2 x i64> %x1, <2 x double> %x2)
+  ret <2 x double> %2
 }
-declare <2 x double> @llvm.x86.avx512.mask.vpermi2var.pd.128(<2 x double>, <2 x i64>, <2 x double>, i8)
 
 define <2 x double> @stack_fold_vpermi2pd(<2 x i64> %x0, <2 x double> %x1, <2 x double> %x2) {
   ;CHECK-LABEL: stack_fold_vpermi2pd
   ;CHECK:       vpermi2pd {{-?[0-9]*}}(%rsp), %xmm1, %xmm0 # 16-byte Folded Reload
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
-  %res = call <2 x double> @llvm.x86.avx512.mask.vpermt2var.pd.128(<2 x i64> %x0, <2 x double> %x1, <2 x double> %x2, i8 -1)
-  ret <2 x double> %res
+  %2 = call <2 x double> @llvm.x86.avx512.vpermi2var.pd.128(<2 x double> %x1, <2 x i64> %x0, <2 x double> %x2)
+  ret <2 x double> %2
 }
-declare <2 x double> @llvm.x86.avx512.mask.vpermt2var.pd.128(<2 x i64>, <2 x double>, <2 x double>, i8)
 
 define <8 x float> @stack_fold_vpermt2ps_ymm(<8 x float> %x0, <8 x i32> %x1, <8 x float> %x2) {
   ;CHECK-LABEL: stack_fold_vpermt2ps_ymm
   ;CHECK:       vpermt2ps {{-?[0-9]*}}(%rsp), %ymm1, %ymm0 # 32-byte Folded Reload
   %1 = tail call <4 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
-  %res = call <8 x float> @llvm.x86.avx512.mask.vpermi2var.ps.256(<8 x float> %x0, <8 x i32> %x1, <8 x float> %x2, i8 -1)
-  ret <8 x float> %res
+  %2 = call <8 x float> @llvm.x86.avx512.vpermi2var.ps.256(<8 x float> %x0, <8 x i32> %x1, <8 x float> %x2)
+  ret <8 x float> %2
 }
-declare <8 x float> @llvm.x86.avx512.mask.vpermi2var.ps.256(<8 x float>, <8 x i32>, <8 x float>, i8)
 
 define <8 x float> @stack_fold_vpermi2ps_ymm(<8 x i32> %x0, <8 x float> %x1, <8 x float> %x2) {
   ;CHECK-LABEL: stack_fold_vpermi2ps_ymm
   ;CHECK:       vpermi2ps {{-?[0-9]*}}(%rsp), %ymm1, %ymm0 # 32-byte Folded Reload
   %1 = tail call <4 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
-  %res = call <8 x float> @llvm.x86.avx512.mask.vpermt2var.ps.256(<8 x i32> %x0, <8 x float> %x1, <8 x float> %x2, i8 -1)
-  ret <8 x float> %res
+  %2 = call <8 x float> @llvm.x86.avx512.vpermi2var.ps.256(<8 x float> %x1, <8 x i32> %x0, <8 x float> %x2)
+  ret <8 x float> %2
 }
-declare <8 x float> @llvm.x86.avx512.mask.vpermt2var.ps.256(<8 x i32>, <8 x float>, <8 x float>, i8)
 
 define <4 x double> @stack_fold_vpermt2pd_ymm(<4 x double> %x0, <4 x i64> %x1, <4 x double> %x2) {
   ;CHECK-LABEL: stack_fold_vpermt2pd_ymm
   ;CHECK:       vpermt2pd {{-?[0-9]*}}(%rsp), %ymm1, %ymm0 # 32-byte Folded Reload
   %1 = tail call <4 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
-  %res = call <4 x double> @llvm.x86.avx512.mask.vpermi2var.pd.256(<4 x double> %x0, <4 x i64> %x1, <4 x double> %x2, i8 -1)
-  ret <4 x double> %res
+  %2 = call <4 x double> @llvm.x86.avx512.vpermi2var.pd.256(<4 x double> %x0, <4 x i64> %x1, <4 x double> %x2)
+  ret <4 x double> %2
 }
-declare <4 x double> @llvm.x86.avx512.mask.vpermi2var.pd.256(<4 x double>, <4 x i64>, <4 x double>, i8)
 
 define <4 x double> @stack_fold_vpermi2pd_ymm(<4 x i64> %x0, <4 x double> %x1, <4 x double> %x2) {
   ;CHECK-LABEL: stack_fold_vpermi2pd_ymm
   ;CHECK:       vpermi2pd {{-?[0-9]*}}(%rsp), %ymm1, %ymm0 # 32-byte Folded Reload
   %1 = tail call <4 x i64> asm sideeffect "nop", "=x,~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
-  %res = call <4 x double> @llvm.x86.avx512.mask.vpermt2var.pd.256(<4 x i64> %x0, <4 x double> %x1, <4 x double> %x2, i8 -1)
-  ret <4 x double> %res
+  %2 = call <4 x double> @llvm.x86.avx512.vpermi2var.pd.256(<4 x double> %x1, <4 x i64> %x0, <4 x double> %x2)
+  ret <4 x double> %2
 }
-declare <4 x double> @llvm.x86.avx512.mask.vpermt2var.pd.256(<4 x i64>, <4 x double>, <4 x double>, i8)
 
 define <4 x double> @stack_fold_permpd(<4 x double> %a0) {
   ;CHECK-LABEL: stack_fold_permpd
@@ -815,5 +807,10 @@ define <8 x float> @stack_fold_permilpsvar_ymm_maskz(<8 x float> %a0, <8 x i32>
   ret <8 x float> %4
 }
 
+declare <4 x float> @llvm.x86.avx512.vpermi2var.ps.128(<4 x float>, <4 x i32>, <4 x float>)
+declare <2 x double> @llvm.x86.avx512.vpermi2var.pd.128(<2 x double>, <2 x i64>, <2 x double>)
+declare <8 x float> @llvm.x86.avx512.vpermi2var.ps.256(<8 x float>, <8 x i32>, <8 x float>)
+declare <4 x double> @llvm.x86.avx512.vpermi2var.pd.256(<4 x double>, <4 x i64>, <4 x double>)
+
 attributes #0 = { "unsafe-fp-math"="false" }
 attributes #1 = { "unsafe-fp-math"="true" }
diff --git a/test/CodeGen/X86/stack-folding-fp-sse42.ll b/test/CodeGen/X86/stack-folding-fp-sse42.ll
index 3604aaa35fc7c318515227eb6bd3a7a483fb4234..4599c4d931a23d0c735c9f038c9305ee99bbd660 100644
--- a/test/CodeGen/X86/stack-folding-fp-sse42.ll
+++ b/test/CodeGen/X86/stack-folding-fp-sse42.ll
@@ -1091,19 +1091,17 @@ define <2 x double> @stack_fold_sqrtpd(<2 x double> %a0) {
   ;CHECK-LABEL: stack_fold_sqrtpd
   ;CHECK:       sqrtpd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
-  %2 = call <2 x double> @llvm.x86.sse2.sqrt.pd(<2 x double> %a0)
+  %2 = call <2 x double> @llvm.sqrt.v2f64(<2 x double> %a0)
   ret <2 x double> %2
 }
-declare <2 x double> @llvm.x86.sse2.sqrt.pd(<2 x double>) nounwind readnone
 
 define <4 x float> @stack_fold_sqrtps(<4 x float> %a0) {
   ;CHECK-LABEL: stack_fold_sqrtps
   ;CHECK:       sqrtps {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
-  %2 = call <4 x float> @llvm.x86.sse.sqrt.ps(<4 x float> %a0)
+  %2 = call <4 x float> @llvm.sqrt.v4f32(<4 x float> %a0)
   ret <4 x float> %2
 }
-declare <4 x float> @llvm.x86.sse.sqrt.ps(<4 x float>) nounwind readnone
 
 define double @stack_fold_sqrtsd(double %a0) optsize {
   ;CHECK-LABEL: stack_fold_sqrtsd
@@ -1118,12 +1116,13 @@ define <2 x double> @stack_fold_sqrtsd_int(<2 x double> %a0, <2 x double> %a1) o
   ;CHECK-LABEL: stack_fold_sqrtsd_int
   ;CHECK:       sqrtsd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
-  %2 = call <2 x double> @llvm.x86.sse2.sqrt.sd(<2 x double> %a1)
-  %3 = extractelement <2 x double> %2, i32 0
-  %4 = insertelement <2 x double> %a0, double %3, i32 0
-  ret <2 x double> %4
+  %2 = extractelement <2 x double> %a1, i64 0
+  %3 = call double @llvm.sqrt.f64(double %2)
+  %4 = insertelement <2 x double> %a1, double %3, i64 0
+  %5 = extractelement <2 x double> %4, i32 0
+  %6 = insertelement <2 x double> %a0, double %5, i32 0
+  ret <2 x double> %6
 }
-declare <2 x double> @llvm.x86.sse2.sqrt.sd(<2 x double>) nounwind readnone
 
 define float @stack_fold_sqrtss(float %a0) minsize {
   ;CHECK-LABEL: stack_fold_sqrtss
@@ -1302,5 +1301,8 @@ define <4 x float> @stack_fold_xorps(<4 x float> %a0, <4 x float> %a1) {
   ret <4 x float> %6
 }
 
+declare <2 x double> @llvm.sqrt.v2f64(<2 x double>)
+declare <4 x float> @llvm.sqrt.v4f32(<4 x float>)
+
 attributes #0 = { "unsafe-fp-math"="false" }
 attributes #1 = { "unsafe-fp-math"="true" }
diff --git a/test/CodeGen/X86/stack-folding-int-avx1.ll b/test/CodeGen/X86/stack-folding-int-avx1.ll
index 91bf9e2876387f6b95bdaeef57071d31cb86137b..04d62d4bc25e1984d615abaa4843d69125c6c1e5 100644
--- a/test/CodeGen/X86/stack-folding-int-avx1.ll
+++ b/test/CodeGen/X86/stack-folding-int-avx1.ll
@@ -205,37 +205,37 @@ define <16 x i8> @stack_fold_paddsb(<16 x i8> %a0, <16 x i8> %a1) {
   ;CHECK-LABEL: stack_fold_paddsb
   ;CHECK:       vpaddsb {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
-  %2 = call <16 x i8> @llvm.x86.sse2.padds.b(<16 x i8> %a0, <16 x i8> %a1)
+  %2 = call <16 x i8> @llvm.sadd.sat.v16i8(<16 x i8> %a0, <16 x i8> %a1)
   ret <16 x i8> %2
 }
-declare <16 x i8> @llvm.x86.sse2.padds.b(<16 x i8>, <16 x i8>) nounwind readnone
+declare <16 x i8> @llvm.sadd.sat.v16i8(<16 x i8>, <16 x i8>) nounwind readnone
 
 define <8 x i16> @stack_fold_paddsw(<8 x i16> %a0, <8 x i16> %a1) {
   ;CHECK-LABEL: stack_fold_paddsw
   ;CHECK:       vpaddsw {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
-  %2 = call <8 x i16> @llvm.x86.sse2.padds.w(<8 x i16> %a0, <8 x i16> %a1)
+  %2 = call <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16> %a0, <8 x i16> %a1)
   ret <8 x i16> %2
 }
-declare <8 x i16> @llvm.x86.sse2.padds.w(<8 x i16>, <8 x i16>) nounwind readnone
+declare <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16>, <8 x i16>) nounwind readnone
 
 define <16 x i8> @stack_fold_paddusb(<16 x i8> %a0, <16 x i8> %a1) {
   ;CHECK-LABEL: stack_fold_paddusb
   ;CHECK:       vpaddusb {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
-  %2 = call <16 x i8> @llvm.x86.sse2.paddus.b(<16 x i8> %a0, <16 x i8> %a1)
+  %2 = call <16 x i8> @llvm.uadd.sat.v16i8(<16 x i8> %a0, <16 x i8> %a1)
   ret <16 x i8> %2
 }
-declare <16 x i8> @llvm.x86.sse2.paddus.b(<16 x i8>, <16 x i8>) nounwind readnone
+declare <16 x i8> @llvm.uadd.sat.v16i8(<16 x i8>, <16 x i8>) nounwind readnone
 
 define <8 x i16> @stack_fold_paddusw(<8 x i16> %a0, <8 x i16> %a1) {
   ;CHECK-LABEL: stack_fold_paddusw
   ;CHECK:       vpaddusw {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
-  %2 = call <8 x i16> @llvm.x86.sse2.paddus.w(<8 x i16> %a0, <8 x i16> %a1)
+  %2 = call <8 x i16> @llvm.uadd.sat.v8i16(<8 x i16> %a0, <8 x i16> %a1)
   ret <8 x i16> %2
 }
-declare <8 x i16> @llvm.x86.sse2.paddus.w(<8 x i16>, <8 x i16>) nounwind readnone
+declare <8 x i16> @llvm.uadd.sat.v8i16(<8 x i16>, <8 x i16>) nounwind readnone
 
 define <8 x i16> @stack_fold_paddw(<8 x i16> %a0, <8 x i16> %a1) {
   ;CHECK-LABEL: stack_fold_paddw
@@ -678,7 +678,6 @@ define <8 x i16> @stack_fold_pminuw(<8 x i16> %a0, <8 x i16> %a1) {
   %3 = select <8 x i1> %2, <8 x i16> %a0, <8 x i16> %a1
   ret <8 x i16> %3
 }
-declare <8 x i16> @llvm.x86.sse41.pminuw(<8 x i16>, <8 x i16>) nounwind readnone
 
 define <2 x i64> @stack_fold_pmuldq(<4 x i32> %a0, <4 x i32> %a1) {
   ;CHECK-LABEL: stack_fold_pmuldq
@@ -930,37 +929,37 @@ define <16 x i8> @stack_fold_psubsb(<16 x i8> %a0, <16 x i8> %a1) {
   ;CHECK-LABEL: stack_fold_psubsb
   ;CHECK:       vpsubsb {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
-  %2 = call <16 x i8> @llvm.x86.sse2.psubs.b(<16 x i8> %a0, <16 x i8> %a1)
+  %2 = call <16 x i8> @llvm.ssub.sat.v16i8(<16 x i8> %a0, <16 x i8> %a1)
   ret <16 x i8> %2
 }
-declare <16 x i8> @llvm.x86.sse2.psubs.b(<16 x i8>, <16 x i8>) nounwind readnone
+declare <16 x i8> @llvm.ssub.sat.v16i8(<16 x i8>, <16 x i8>) nounwind readnone
 
 define <8 x i16> @stack_fold_psubsw(<8 x i16> %a0, <8 x i16> %a1) {
   ;CHECK-LABEL: stack_fold_psubsw
   ;CHECK:       vpsubsw {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
-  %2 = call <8 x i16> @llvm.x86.sse2.psubs.w(<8 x i16> %a0, <8 x i16> %a1)
+  %2 = call <8 x i16> @llvm.ssub.sat.v8i16(<8 x i16> %a0, <8 x i16> %a1)
   ret <8 x i16> %2
 }
-declare <8 x i16> @llvm.x86.sse2.psubs.w(<8 x i16>, <8 x i16>) nounwind readnone
+declare <8 x i16> @llvm.ssub.sat.v8i16(<8 x i16>, <8 x i16>) nounwind readnone
 
 define <16 x i8> @stack_fold_psubusb(<16 x i8> %a0, <16 x i8> %a1) {
   ;CHECK-LABEL: stack_fold_psubusb
   ;CHECK:       vpsubusb {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
-  %2 = call <16 x i8> @llvm.x86.sse2.psubus.b(<16 x i8> %a0, <16 x i8> %a1)
+  %2 = call <16 x i8> @llvm.usub.sat.v16i8(<16 x i8> %a0, <16 x i8> %a1)
   ret <16 x i8> %2
 }
-declare <16 x i8> @llvm.x86.sse2.psubus.b(<16 x i8>, <16 x i8>) nounwind readnone
+declare <16 x i8> @llvm.usub.sat.v16i8(<16 x i8>, <16 x i8>) nounwind readnone
 
 define <8 x i16> @stack_fold_psubusw(<8 x i16> %a0, <8 x i16> %a1) {
   ;CHECK-LABEL: stack_fold_psubusw
   ;CHECK:       vpsubusw {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
-  %2 = call <8 x i16> @llvm.x86.sse2.psubus.w(<8 x i16> %a0, <8 x i16> %a1)
+  %2 = call <8 x i16> @llvm.usub.sat.v8i16(<8 x i16> %a0, <8 x i16> %a1)
   ret <8 x i16> %2
 }
-declare <8 x i16> @llvm.x86.sse2.psubus.w(<8 x i16>, <8 x i16>) nounwind readnone
+declare <8 x i16> @llvm.usub.sat.v8i16(<8 x i16>, <8 x i16>) nounwind readnone
 
 define <8 x i16> @stack_fold_psubw(<8 x i16> %a0, <8 x i16> %a1) {
   ;CHECK-LABEL: stack_fold_psubw
diff --git a/test/CodeGen/X86/stack-folding-int-avx2.ll b/test/CodeGen/X86/stack-folding-int-avx2.ll
index 9335acb90c0d6700becc86fa841d74f4fc386aa6..a96179843c97f3c1aedb779ef4d9e2c76397702d 100644
--- a/test/CodeGen/X86/stack-folding-int-avx2.ll
+++ b/test/CodeGen/X86/stack-folding-int-avx2.ll
@@ -161,37 +161,37 @@ define <32 x i8> @stack_fold_paddsb(<32 x i8> %a0, <32 x i8> %a1) {
   ;CHECK-LABEL: stack_fold_paddsb
   ;CHECK:       vpaddsb {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
-  %2 = call <32 x i8> @llvm.x86.avx2.padds.b(<32 x i8> %a0, <32 x i8> %a1)
+  %2 = call <32 x i8> @llvm.sadd.sat.v32i8(<32 x i8> %a0, <32 x i8> %a1)
   ret <32 x i8> %2
 }
-declare <32 x i8> @llvm.x86.avx2.padds.b(<32 x i8>, <32 x i8>) nounwind readnone
+declare <32 x i8> @llvm.sadd.sat.v32i8(<32 x i8>, <32 x i8>) nounwind readnone
 
 define <16 x i16> @stack_fold_paddsw(<16 x i16> %a0, <16 x i16> %a1) {
   ;CHECK-LABEL: stack_fold_paddsw
   ;CHECK:       vpaddsw {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
-  %2 = call <16 x i16> @llvm.x86.avx2.padds.w(<16 x i16> %a0, <16 x i16> %a1)
+  %2 = call <16 x i16> @llvm.sadd.sat.v16i16(<16 x i16> %a0, <16 x i16> %a1)
   ret <16 x i16> %2
 }
-declare <16 x i16> @llvm.x86.avx2.padds.w(<16 x i16>, <16 x i16>) nounwind readnone
+declare <16 x i16> @llvm.sadd.sat.v16i16(<16 x i16>, <16 x i16>) nounwind readnone
 
 define <32 x i8> @stack_fold_paddusb(<32 x i8> %a0, <32 x i8> %a1) {
   ;CHECK-LABEL: stack_fold_paddusb
   ;CHECK:       vpaddusb {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
-  %2 = call <32 x i8> @llvm.x86.avx2.paddus.b(<32 x i8> %a0, <32 x i8> %a1)
+  %2 = call <32 x i8> @llvm.uadd.sat.v32i8(<32 x i8> %a0, <32 x i8> %a1)
   ret <32 x i8> %2
 }
-declare <32 x i8> @llvm.x86.avx2.paddus.b(<32 x i8>, <32 x i8>) nounwind readnone
+declare <32 x i8> @llvm.uadd.sat.v32i8(<32 x i8>, <32 x i8>) nounwind readnone
 
 define <16 x i16> @stack_fold_paddusw(<16 x i16> %a0, <16 x i16> %a1) {
   ;CHECK-LABEL: stack_fold_paddusw
   ;CHECK:       vpaddusw {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
-  %2 = call <16 x i16> @llvm.x86.avx2.paddus.w(<16 x i16> %a0, <16 x i16> %a1)
+  %2 = call <16 x i16> @llvm.uadd.sat.v16i16(<16 x i16> %a0, <16 x i16> %a1)
   ret <16 x i16> %2
 }
-declare <16 x i16> @llvm.x86.avx2.paddus.w(<16 x i16>, <16 x i16>) nounwind readnone
+declare <16 x i16> @llvm.uadd.sat.v16i16(<16 x i16>, <16 x i16>) nounwind readnone
 
 define <16 x i16> @stack_fold_paddw(<16 x i16> %a0, <16 x i16> %a1) {
   ;CHECK-LABEL: stack_fold_paddw
@@ -1109,37 +1109,37 @@ define <32 x i8> @stack_fold_psubsb(<32 x i8> %a0, <32 x i8> %a1) {
   ;CHECK-LABEL: stack_fold_psubsb
   ;CHECK:       vpsubsb {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
-  %2 = call <32 x i8> @llvm.x86.avx2.psubs.b(<32 x i8> %a0, <32 x i8> %a1)
+  %2 = call <32 x i8> @llvm.ssub.sat.v32i8(<32 x i8> %a0, <32 x i8> %a1)
   ret <32 x i8> %2
 }
-declare <32 x i8> @llvm.x86.avx2.psubs.b(<32 x i8>, <32 x i8>) nounwind readnone
+declare <32 x i8> @llvm.ssub.sat.v32i8(<32 x i8>, <32 x i8>) nounwind readnone
 
 define <16 x i16> @stack_fold_psubsw(<16 x i16> %a0, <16 x i16> %a1) {
   ;CHECK-LABEL: stack_fold_psubsw
   ;CHECK:       vpsubsw {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
-  %2 = call <16 x i16> @llvm.x86.avx2.psubs.w(<16 x i16> %a0, <16 x i16> %a1)
+  %2 = call <16 x i16> @llvm.ssub.sat.v16i16(<16 x i16> %a0, <16 x i16> %a1)
   ret <16 x i16> %2
 }
-declare <16 x i16> @llvm.x86.avx2.psubs.w(<16 x i16>, <16 x i16>) nounwind readnone
+declare <16 x i16> @llvm.ssub.sat.v16i16(<16 x i16>, <16 x i16>) nounwind readnone
 
 define <32 x i8> @stack_fold_psubusb(<32 x i8> %a0, <32 x i8> %a1) {
   ;CHECK-LABEL: stack_fold_psubusb
   ;CHECK:       vpsubusb {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
-  %2 = call <32 x i8> @llvm.x86.avx2.psubus.b(<32 x i8> %a0, <32 x i8> %a1)
+  %2 = call <32 x i8> @llvm.usub.sat.v32i8(<32 x i8> %a0, <32 x i8> %a1)
   ret <32 x i8> %2
 }
-declare <32 x i8> @llvm.x86.avx2.psubus.b(<32 x i8>, <32 x i8>) nounwind readnone
+declare <32 x i8> @llvm.usub.sat.v32i8(<32 x i8>, <32 x i8>) nounwind readnone
 
 define <16 x i16> @stack_fold_psubusw(<16 x i16> %a0, <16 x i16> %a1) {
   ;CHECK-LABEL: stack_fold_psubusw
   ;CHECK:       vpsubusw {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
-  %2 = call <16 x i16> @llvm.x86.avx2.psubus.w(<16 x i16> %a0, <16 x i16> %a1)
+  %2 = call <16 x i16> @llvm.usub.sat.v16i16(<16 x i16> %a0, <16 x i16> %a1)
   ret <16 x i16> %2
 }
-declare <16 x i16> @llvm.x86.avx2.psubus.w(<16 x i16>, <16 x i16>) nounwind readnone
+declare <16 x i16> @llvm.usub.sat.v16i16(<16 x i16>, <16 x i16>) nounwind readnone
 
 define <16 x i16> @stack_fold_psubw(<16 x i16> %a0, <16 x i16> %a1) {
   ;CHECK-LABEL: stack_fold_psubw
diff --git a/test/CodeGen/X86/stack-folding-int-avx512.ll b/test/CodeGen/X86/stack-folding-int-avx512.ll
index 5464775a43a5ba4d264bbfd8140002d69cb89c6a..a899475c9d80fe7b2f67054f48541759d175493d 100644
--- a/test/CodeGen/X86/stack-folding-int-avx512.ll
+++ b/test/CodeGen/X86/stack-folding-int-avx512.ll
@@ -20,7 +20,7 @@ define <16 x i32> @stack_fold_valignd_mask(<16 x i32> %a, <16 x i32> %b, <16 x i
   ;CHECK-LABEL: stack_fold_valignd_mask
   ;CHECK:   valignd $1, {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 64-byte Folded Reload
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
-  %2 = shufflevector <16 x i32> %a, <16 x i32> %b, <16 x i32><i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16>
+  %2 = shufflevector <16 x i32> %a, <16 x i32> %b, <16 x i32> <i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16>
   %3 = bitcast i16 %mask to <16 x i1>
   %4 = load <16 x i32>, <16 x i32>* %passthru
   %5 = select <16 x i1> %3, <16 x i32> %2, <16 x i32> %4
@@ -31,7 +31,7 @@ define <16 x i32> @stack_fold_valignd_maskz(<16 x i32> %a, <16 x i32> %b, i16 %m
   ;CHECK-LABEL: stack_fold_valignd_maskz
   ;CHECK:   valignd $1, {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 64-byte Folded Reload
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
-  %2 = shufflevector <16 x i32> %a, <16 x i32> %b, <16 x i32><i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16>
+  %2 = shufflevector <16 x i32> %a, <16 x i32> %b, <16 x i32> <i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16>
   %3 = bitcast i16 %mask to <16 x i1>
   %4 = select <16 x i1> %3, <16 x i32> %2, <16 x i32> zeroinitializer
   ret <16 x i32> %4
@@ -74,7 +74,7 @@ define <64 x i8> @stack_fold_pavgb(<64 x i8> %a0, <64 x i8> %a1) {
   %3 = zext <64 x i8> %a1 to <64 x i16>
   %4 = add <64 x i16> %2, %3
   %5 = add <64 x i16> %4, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
-  %6 = lshr <64 x i16> %5, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1,   i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
+  %6 = lshr <64 x i16> %5, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
   %7 = trunc <64 x i16> %6 to <64 x i8>
   ret <64 x i8> %7
 }
@@ -87,7 +87,7 @@ define <64 x i8> @stack_fold_pavgb_mask(<64 x i8>* %passthru, <64 x i8> %a0, <64
   %3 = zext <64 x i8> %a0 to <64 x i16>
   %4 = zext <64 x i8> %a1 to <64 x i16>
   %5 = add <64 x i16> %3, %4
-  %6 = add <64 x i16> %5, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1,  i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
+  %6 = add <64 x i16> %5, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
   %7 = lshr <64 x i16> %6, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
   %8 = trunc <64 x i16> %7 to <64 x i8>
   %9 = bitcast i64 %mask to <64 x i1>
@@ -102,7 +102,7 @@ define <64 x i8> @stack_fold_pavgb_maskz(<64 x i8> %a0, <64 x i8> %a1, i64 %mask
   %2 = zext <64 x i8> %a0 to <64 x i16>
   %3 = zext <64 x i8> %a1 to <64 x i16>
   %4 = add <64 x i16> %2, %3
-  %5 = add <64 x i16> %4, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1,  i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
+  %5 = add <64 x i16> %4, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
   %6 = lshr <64 x i16> %5, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
   %7 = trunc <64 x i16> %6 to <64 x i8>
   %8 = bitcast i64 %mask to <64 x i1>
@@ -118,7 +118,7 @@ define <32 x i16> @stack_fold_pavgw(<32 x i16> %a0, <32 x i16> %a1) {
   %3 = zext <32 x i16> %a1 to <32 x i32>
   %4 = add <32 x i32> %2, %3
   %5 = add <32 x i32> %4, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
-  %6 = lshr <32 x i32> %5, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1,   i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  %6 = lshr <32 x i32> %5, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
   %7 = trunc <32 x i32> %6 to <32 x i16>
   ret <32 x i16> %7
 }
@@ -131,8 +131,8 @@ define <32 x i16> @stack_fold_pavgw_mask(<32 x i16>* %passthru, <32 x i16> %a0,
   %3 = zext <32 x i16> %a0 to <32 x i32>
   %4 = zext <32 x i16> %a1 to <32 x i32>
   %5 = add <32 x i32> %3, %4
-  %6 = add <32 x i32> %5, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1,    i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
-  %7 = lshr <32 x i32> %6, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1,   i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  %6 = add <32 x i32> %5, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  %7 = lshr <32 x i32> %6, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
   %8 = trunc <32 x i32> %7 to <32 x i16>
   %9 = bitcast i32 %mask to <32 x i1>
   %10 = select <32 x i1> %9, <32 x i16> %8, <32 x i16> %2
@@ -146,8 +146,8 @@ define <32 x i16> @stack_fold_pavgw_maskz(<32 x i16> %a0, <32 x i16> %a1, i32 %m
   %2 = zext <32 x i16> %a0 to <32 x i32>
   %3 = zext <32 x i16> %a1 to <32 x i32>
   %4 = add <32 x i32> %2, %3
-  %5 = add <32 x i32> %4, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1,    i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
-  %6 = lshr <32 x i32> %5, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1,   i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  %5 = add <32 x i32> %4, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  %6 = lshr <32 x i32> %5, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
   %7 = trunc <32 x i32> %6 to <32 x i16>
   %8 = bitcast i32 %mask to <32 x i1>
   %9 = select <32 x i1> %8, <32 x i16> %7, <32 x i16> zeroinitializer
@@ -223,7 +223,6 @@ define <64 x i8> @stack_fold_pabsb(<64 x i8> %a0) {
   %4 = select <64 x i1> %2, <64 x i8> %a0, <64 x i8> %3
   ret <64 x i8> %4
 }
-declare <64 x i8> @llvm.x86.avx512.mask.pabs.b.512(<64 x i8>, <64 x i8>, i64) nounwind readnone
 
 define <64 x i8> @stack_fold_pabsb_mask(<64 x i8> %passthru, <64 x i8> %a0, i64 %mask) {
   ;CHECK-LABEL: stack_fold_pabsb_mask
@@ -458,25 +457,23 @@ define <64 x i8> @stack_fold_paddsb(<64 x i8> %a0, <64 x i8> %a1) {
   ;CHECK-LABEL: stack_fold_paddsb
   ;CHECK:       vpaddsb {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
-  %2 = call <64 x i8> @llvm.x86.avx512.mask.padds.b.512(<64 x i8> %a0, <64 x i8> %a1, <64 x i8> undef, i64 -1)
+  %2 = call <64 x i8> @llvm.sadd.sat.v64i8(<64 x i8> %a0, <64 x i8> %a1)
   ret <64 x i8> %2
 }
-declare <64 x i8> @llvm.x86.avx512.mask.padds.b.512(<64 x i8>, <64 x i8>, <64 x i8>, i64) nounwind readnone
 
 define <32 x i16> @stack_fold_paddsw(<32 x i16> %a0, <32 x i16> %a1) {
   ;CHECK-LABEL: stack_fold_paddsw
   ;CHECK:       vpaddsw {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
-  %2 = call <32 x i16> @llvm.x86.avx512.mask.padds.w.512(<32 x i16> %a0, <32 x i16> %a1, <32 x i16> undef, i32 -1)
+  %2 = call <32 x i16> @llvm.sadd.sat.v32i16(<32 x i16> %a0, <32 x i16> %a1)
   ret <32 x i16> %2
 }
-declare <32 x i16> @llvm.x86.avx512.mask.padds.w.512(<32 x i16>, <32 x i16>, <32 x i16>, i32)
 
 define <64 x i8> @stack_fold_paddusb(<64 x i8> %a0, <64 x i8> %a1) {
   ;CHECK-LABEL: stack_fold_paddusb
   ;CHECK:       vpaddusb {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
-  %2 = call <64 x i8> @llvm.x86.avx512.mask.paddus.b.512(<64 x i8> %a0, <64 x i8> %a1, <64 x i8> undef, i64 -1)
+  %2 = call <64 x i8> @llvm.uadd.sat.v64i8(<64 x i8> %a0, <64 x i8> %a1)
   ret <64 x i8> %2
 }
 declare <64 x i8> @llvm.x86.avx512.mask.paddus.b.512(<64 x i8>, <64 x i8>, <64 x i8>, i64) nounwind readnone
@@ -485,7 +482,7 @@ define <32 x i16> @stack_fold_paddusw(<32 x i16> %a0, <32 x i16> %a1) {
   ;CHECK-LABEL: stack_fold_paddusw
   ;CHECK:       vpaddusw {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
-  %2 = call <32 x i16> @llvm.x86.avx512.mask.paddus.w.512(<32 x i16> %a0, <32 x i16> %a1, <32 x i16> undef, i32 -1)
+  %2 = call <32 x i16> @llvm.uadd.sat.v32i16(<32 x i16> %a0, <32 x i16> %a1)
   ret <32 x i16> %2
 }
 declare <32 x i16> @llvm.x86.avx512.mask.paddus.w.512(<32 x i16>, <32 x i16>, <32 x i16>, i32)
@@ -531,7 +528,7 @@ define <16 x i32> @stack_fold_vpconflictd(<16 x i32> %a0) {
   ;CHECK-LABEL: stack_fold_vpconflictd
   ;CHECK:       vpconflictd {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
-  %2 = call <16 x i32> @llvm.x86.avx512.mask.conflict.d.512(<16 x i32> %a0, <16 x i32> undef, i16 -1)
+  %2 = call <16 x i32> @llvm.x86.avx512.conflict.d.512(<16 x i32> %a0)
   ret <16 x i32> %2
 }
 declare <16 x i32> @llvm.x86.avx512.mask.conflict.d.512(<16 x i32>, <16 x i32>, i16) nounwind readonly
@@ -540,7 +537,7 @@ define <8 x i64> @stack_fold_vpconflictq(<8 x i64> %a0) {
   ;CHECK-LABEL: stack_fold_vpconflictq
   ;CHECK:       vpconflictq {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
-  %2 = call <8 x i64> @llvm.x86.avx512.mask.conflict.q.512(<8 x i64> %a0, <8 x i64> undef, i8 -1)
+  %2 = call <8 x i64> @llvm.x86.avx512.conflict.q.512(<8 x i64> %a0)
   ret <8 x i64> %2
 }
 declare <8 x i64> @llvm.x86.avx512.mask.conflict.q.512(<8 x i64>, <8 x i64>, i8) nounwind readnone
@@ -892,19 +889,17 @@ define <16 x i32> @stack_fold_vplzcntd(<16 x i32> %a0) {
   ;CHECK-LABEL: stack_fold_vplzcntd
   ;CHECK:       vplzcntd {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
-  %2 = call <16 x i32> @llvm.ctlz.v16i32(<16 x i32> %a0)
+  %2 = call <16 x i32> @llvm.ctlz.v16i32(<16 x i32> %a0, i1 false)
   ret <16 x i32> %2
 }
-declare <16 x i32> @llvm.ctlz.v16i32(<16 x i32>) nounwind readonly
 
 define <8 x i64> @stack_fold_vplzcntq(<8 x i64> %a0) {
   ;CHECK-LABEL: stack_fold_vplzcntq
   ;CHECK:       vplzcntq {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
-  %2 = call <8 x i64> @llvm.ctlz.v8i64(<8 x i64> %a0)
+  %2 = call <8 x i64> @llvm.ctlz.v8i64(<8 x i64> %a0, i1 false)
   ret <8 x i64> %2
 }
-declare <8 x i64> @llvm.ctlz.v8i64(<8 x i64>) nounwind readnone
 
 define <32 x i16> @stack_fold_pmaddubsw_zmm(<64 x i8> %a0, <64 x i8> %a1) {
   ;CHECK-LABEL: stack_fold_pmaddubsw_zmm
@@ -999,7 +994,7 @@ define <2 x i64> @stack_fold_movq_load(<2 x i64> %a0) {
 define <8 x i32> @stack_fold_vpmovqd(<8 x i64> %a0) {
   ;CHECK-LABEL: stack_fold_vpmovqd
   ;CHECK:       vpmovqd %zmm0, {{-?[0-9]*}}(%rsp) # 32-byte Folded Spill
-  %1 = call <8 x i32> @llvm.x86.avx512.mask.pmov.qd.512(<8 x i64> %a0, <8 x i32> undef, i8 -1)
+  %1 = trunc <8 x i64> %a0 to <8 x i32>
   %2 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   ret <8 x i32> %1
 }
@@ -1017,7 +1012,7 @@ declare <8 x i16> @llvm.x86.avx512.mask.pmov.qw.512(<8 x i64>, <8 x i16>, i8)
 define <32 x i8> @stack_fold_vpmovwb(<32 x i16> %a0) {
   ;CHECK-LABEL: stack_fold_vpmovwb
   ;CHECK:       vpmovwb %zmm0, {{-?[0-9]*}}(%rsp) # 32-byte Folded Spill
-  %1 = call <32 x i8> @llvm.x86.avx512.mask.pmov.wb.512(<32 x i16> %a0, <32 x i8> undef, i32 -1)
+  %1 = trunc <32 x i16> %a0 to <32 x i8>
   %2 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   ret <32 x i8> %1
 }
@@ -1743,37 +1738,33 @@ define <64 x i8> @stack_fold_psubsb(<64 x i8> %a0, <64 x i8> %a1) {
   ;CHECK-LABEL: stack_fold_psubsb
   ;CHECK:       vpsubsb {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
-  %2 = call <64 x i8> @llvm.x86.avx512.mask.psubs.b.512(<64 x i8> %a0, <64 x i8> %a1, <64 x i8> undef, i64 -1)
+  %2 = call <64 x i8> @llvm.ssub.sat.v64i8(<64 x i8> %a0, <64 x i8> %a1)
   ret <64 x i8> %2
 }
-declare <64 x i8> @llvm.x86.avx512.mask.psubs.b.512(<64 x i8>, <64 x i8>, <64 x i8>, i64) nounwind readnone
 
 define <32 x i16> @stack_fold_psubsw(<32 x i16> %a0, <32 x i16> %a1) {
   ;CHECK-LABEL: stack_fold_psubsw
   ;CHECK:       vpsubsw {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
-  %2 = call <32 x i16> @llvm.x86.avx512.mask.psubs.w.512(<32 x i16> %a0, <32 x i16> %a1, <32 x i16> undef, i32 -1)
+  %2 = call <32 x i16> @llvm.ssub.sat.v32i16(<32 x i16> %a0, <32 x i16> %a1)
   ret <32 x i16> %2
 }
-declare <32 x i16> @llvm.x86.avx512.mask.psubs.w.512(<32 x i16>, <32 x i16>, <32 x i16>, i32)
 
 define <64 x i8> @stack_fold_psubusb(<64 x i8> %a0, <64 x i8> %a1) {
   ;CHECK-LABEL: stack_fold_psubusb
   ;CHECK:       vpsubusb {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
-  %2 = call <64 x i8> @llvm.x86.avx512.mask.psubus.b.512(<64 x i8> %a0, <64 x i8> %a1, <64 x i8> undef, i64 -1)
+  %2 = call <64 x i8> @llvm.usub.sat.v64i8(<64 x i8> %a0, <64 x i8> %a1)
   ret <64 x i8> %2
 }
-declare <64 x i8> @llvm.x86.avx512.mask.psubus.b.512(<64 x i8>, <64 x i8>, <64 x i8>, i64) nounwind readnone
 
 define <32 x i16> @stack_fold_psubusw(<32 x i16> %a0, <32 x i16> %a1) {
   ;CHECK-LABEL: stack_fold_psubusw
   ;CHECK:       vpsubusw {{-?[0-9]*}}(%rsp), {{%zmm[0-9][0-9]*}}, {{%zmm[0-9][0-9]*}} {{.*#+}} 64-byte Folded Reload
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
-  %2 = call <32 x i16> @llvm.x86.avx512.mask.psubus.w.512(<32 x i16> %a0, <32 x i16> %a1, <32 x i16> undef, i32 -1)
+  %2 = call <32 x i16> @llvm.usub.sat.v32i16(<32 x i16> %a0, <32 x i16> %a1)
   ret <32 x i16> %2
 }
-declare <32 x i16> @llvm.x86.avx512.mask.psubus.w.512(<32 x i16>, <32 x i16>, <32 x i16>, i32)
 
 define <32 x i16> @stack_fold_psubw(<32 x i16> %a0, <32 x i16> %a1) {
   ;CHECK-LABEL: stack_fold_psubw
@@ -1883,3 +1874,16 @@ define <64 x i8> @stack_fold_punpckhbw_maskz_zmm(<64 x i8> %a0, <64 x i8> %a1, i
   %4 = select <64 x i1> %3, <64 x i8> %2, <64 x i8> zeroinitializer
   ret <64 x i8> %4
 }
+
+declare <64 x i8> @llvm.sadd.sat.v64i8(<64 x i8>, <64 x i8>)
+declare <32 x i16> @llvm.sadd.sat.v32i16(<32 x i16>, <32 x i16>)
+declare <64 x i8> @llvm.uadd.sat.v64i8(<64 x i8>, <64 x i8>)
+declare <32 x i16> @llvm.uadd.sat.v32i16(<32 x i16>, <32 x i16>)
+declare <16 x i32> @llvm.x86.avx512.conflict.d.512(<16 x i32>)
+declare <8 x i64> @llvm.x86.avx512.conflict.q.512(<8 x i64>)
+declare <16 x i32> @llvm.ctlz.v16i32(<16 x i32>, i1)
+declare <8 x i64> @llvm.ctlz.v8i64(<8 x i64>, i1)
+declare <64 x i8> @llvm.ssub.sat.v64i8(<64 x i8>, <64 x i8>)
+declare <32 x i16> @llvm.ssub.sat.v32i16(<32 x i16>, <32 x i16>)
+declare <64 x i8> @llvm.usub.sat.v64i8(<64 x i8>, <64 x i8>)
+declare <32 x i16> @llvm.usub.sat.v32i16(<32 x i16>, <32 x i16>)
diff --git a/test/CodeGen/X86/stack-folding-int-avx512vl.ll b/test/CodeGen/X86/stack-folding-int-avx512vl.ll
index 8d8676f0f9e7f5273eca525abebfcba308fb41c1..67a02d5b9b6e9e025f592fcf0f075c945f31c3ad 100644
--- a/test/CodeGen/X86/stack-folding-int-avx512vl.ll
+++ b/test/CodeGen/X86/stack-folding-int-avx512vl.ll
@@ -12,7 +12,7 @@ define <8 x i32> @stack_fold_valignd_ymm(<8 x i32> %a, <8 x i32> %b) {
   ;CHECK-LABEL: stack_fold_valignd_ymm
   ;CHECK:   valignd $1, {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
-  %2 = shufflevector <8 x i32> %a, <8 x i32> %b, <8 x i32><i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8>
+  %2 = shufflevector <8 x i32> %a, <8 x i32> %b, <8 x i32> <i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8>
   ret <8 x i32> %2
 }
 
@@ -20,7 +20,7 @@ define <8 x i32> @stack_fold_valignd_ymm_mask(<8 x i32> %a, <8 x i32> %b, <8 x i
   ;CHECK-LABEL: stack_fold_valignd_ymm_mask
   ;CHECK:   valignd $1, {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{{%k[0-7]}}} {{.*#+}} 32-byte Folded Reload
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
-  %2 = shufflevector <8 x i32> %a, <8 x i32> %b, <8 x i32><i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8>
+  %2 = shufflevector <8 x i32> %a, <8 x i32> %b, <8 x i32> <i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8>
   %3 = bitcast i8 %mask to <8 x i1>
   %4 = load <8 x i32>, <8 x i32>* %passthru
   %5 = select <8 x i1> %3, <8 x i32> %2, <8 x i32> %4
@@ -31,7 +31,7 @@ define <8 x i32> @stack_fold_valignd_ymm_maskz(<8 x i32> %a, <8 x i32> %b, i8 %m
   ;CHECK-LABEL: stack_fold_valignd_ymm_maskz
   ;CHECK:   valignd $1, {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{{%k[0-7]}}} {z} {{.*#+}} 32-byte Folded Reload
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
-  %2 = shufflevector <8 x i32> %a, <8 x i32> %b, <8 x i32><i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8>
+  %2 = shufflevector <8 x i32> %a, <8 x i32> %b, <8 x i32> <i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8>
   %3 = bitcast i8 %mask to <8 x i1>
   %4 = select <8 x i1> %3, <8 x i32> %2, <8 x i32> zeroinitializer
   ret <8 x i32> %4
@@ -65,8 +65,8 @@ define <32 x i8> @stack_fold_pavgb_ymm(<32 x i8> %a0, <32 x i8> %a1) {
   %2 = zext <32 x i8> %a0 to <32 x i16>
   %3 = zext <32 x i8> %a1 to <32 x i16>
   %4 = add <32 x i16> %2, %3
-  %5 = add <32 x i16> %4, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1,    i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
-  %6 = lshr <32 x i16> %5, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1,   i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
+  %5 = add <32 x i16> %4, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
+  %6 = lshr <32 x i16> %5, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
   %7 = trunc <32 x i16> %6 to <32 x i8>
   ret <32 x i8> %7
 }
@@ -88,50 +88,46 @@ define <16 x i16> @stack_fold_pavgw_ymm(<16 x i16> %a0, <16 x i16> %a1) {
   ;CHECK-LABEL: stack_fold_pavgw_ymm
   ;CHECK:       vpavgw {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
- %2 = zext <16 x i16> %a0 to <16 x i32>
- %3 = zext <16 x i16> %a1 to <16 x i32>
- %4 = add <16 x i32> %2, %3
- %5 = add <16 x i32> %4, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
- %6 = lshr <16 x i32> %5, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
- %7 = trunc <16 x i32> %6 to <16 x i16>
- ret <16 x i16> %7
+  %2 = zext <16 x i16> %a0 to <16 x i32>
+  %3 = zext <16 x i16> %a1 to <16 x i32>
+  %4 = add <16 x i32> %2, %3
+  %5 = add <16 x i32> %4, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  %6 = lshr <16 x i32> %5, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+  %7 = trunc <16 x i32> %6 to <16 x i16>
+  ret <16 x i16> %7
 }
 
 define <4 x i32> @stack_fold_vpconflictd(<4 x i32> %a0) {
   ;CHECK-LABEL: stack_fold_vpconflictd
   ;CHECK:       vpconflictd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
-  %2 = call <4 x i32> @llvm.x86.avx512.mask.conflict.d.128(<4 x i32> %a0, <4 x i32> undef, i8 -1)
+  %2 = call <4 x i32> @llvm.x86.avx512.conflict.d.128(<4 x i32> %a0)
   ret <4 x i32> %2
 }
-declare <4 x i32> @llvm.x86.avx512.mask.conflict.d.128(<4 x i32>, <4 x i32>, i8) nounwind readonly
 
 define <8 x i32> @stack_fold_vpconflictd_ymm(<8 x i32> %a0) {
   ;CHECK-LABEL: stack_fold_vpconflictd_ymm
   ;CHECK:       vpconflictd {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
-  %2 = call <8 x i32> @llvm.x86.avx512.mask.conflict.d.256(<8 x i32> %a0, <8 x i32> undef, i8 -1)
+  %2 = call <8 x i32> @llvm.x86.avx512.conflict.d.256(<8 x i32> %a0)
   ret <8 x i32> %2
 }
-declare <8 x i32> @llvm.x86.avx512.mask.conflict.d.256(<8 x i32>, <8 x i32>, i8) nounwind readonly
 
 define <2 x i64> @stack_fold_vpconflictq(<2 x i64> %a0) {
   ;CHECK-LABEL: stack_fold_vpconflictq
   ;CHECK:       vpconflictq {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
-  %2 = call <2 x i64> @llvm.x86.avx512.mask.conflict.q.128(<2 x i64> %a0, <2 x i64> undef, i8 -1)
+  %2 = call <2 x i64> @llvm.x86.avx512.conflict.q.128(<2 x i64> %a0)
   ret <2 x i64> %2
 }
-declare <2 x i64> @llvm.x86.avx512.mask.conflict.q.128(<2 x i64>, <2 x i64>, i8) nounwind readnone
 
 define <4 x i64> @stack_fold_vpconflictq_ymm(<4 x i64> %a0) {
   ;CHECK-LABEL: stack_fold_vpconflictq_ymm
   ;CHECK:       vpconflictq {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
-  %2 = call <4 x i64> @llvm.x86.avx512.mask.conflict.q.256(<4 x i64> %a0, <4 x i64> undef, i8 -1)
+  %2 = call <4 x i64> @llvm.x86.avx512.conflict.q.256(<4 x i64> %a0)
   ret <4 x i64> %2
 }
-declare <4 x i64> @llvm.x86.avx512.mask.conflict.q.256(<4 x i64>, <4 x i64>, i8) nounwind readnone
 
 define <4 x i32> @stack_fold_extracti32x4(<8 x i16> %a0, <8 x i32> %a1) {
   ;CHECK-LABEL: stack_fold_extracti32x4
@@ -421,73 +417,73 @@ define <16 x i8> @stack_fold_paddsb(<16 x i8> %a0, <16 x i8> %a1) {
   ;CHECK-LABEL: stack_fold_paddsb
   ;CHECK:       vpaddsb {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
-  %2 = call <16 x i8> @llvm.x86.sse2.padds.b(<16 x i8> %a0, <16 x i8> %a1)
+  %2 = call <16 x i8> @llvm.sadd.sat.v16i8(<16 x i8> %a0, <16 x i8> %a1)
   ret <16 x i8> %2
 }
-declare <16 x i8> @llvm.x86.sse2.padds.b(<16 x i8>, <16 x i8>) nounwind readnone
+declare <16 x i8> @llvm.sadd.sat.v16i8(<16 x i8>, <16 x i8>) nounwind readnone
 
 define <32 x i8> @stack_fold_paddsb_ymm(<32 x i8> %a0, <32 x i8> %a1) {
   ;CHECK-LABEL: stack_fold_paddsb_ymm
   ;CHECK:       vpaddsb {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
-  %2 = call <32 x i8> @llvm.x86.avx2.padds.b(<32 x i8> %a0, <32 x i8> %a1)
+  %2 = call <32 x i8> @llvm.sadd.sat.v32i8(<32 x i8> %a0, <32 x i8> %a1)
   ret <32 x i8> %2
 }
-declare <32 x i8> @llvm.x86.avx2.padds.b(<32 x i8>, <32 x i8>) nounwind readnone
+declare <32 x i8> @llvm.sadd.sat.v32i8(<32 x i8>, <32 x i8>) nounwind readnone
 
 define <8 x i16> @stack_fold_paddsw(<8 x i16> %a0, <8 x i16> %a1) {
   ;CHECK-LABEL: stack_fold_paddsw
   ;CHECK:       vpaddsw {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
-  %2 = call <8 x i16> @llvm.x86.sse2.padds.w(<8 x i16> %a0, <8 x i16> %a1)
+  %2 = call <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16> %a0, <8 x i16> %a1)
   ret <8 x i16> %2
 }
-declare <8 x i16> @llvm.x86.sse2.padds.w(<8 x i16>, <8 x i16>) nounwind readnone
+declare <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16>, <8 x i16>) nounwind readnone
 
 define <16 x i16> @stack_fold_paddsw_ymm(<16 x i16> %a0, <16 x i16> %a1) {
   ;CHECK-LABEL: stack_fold_paddsw_ymm
   ;CHECK:       vpaddsw {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
-  %2 = call <16 x i16> @llvm.x86.avx2.padds.w(<16 x i16> %a0, <16 x i16> %a1)
+  %2 = call <16 x i16> @llvm.sadd.sat.v16i16(<16 x i16> %a0, <16 x i16> %a1)
   ret <16 x i16> %2
 }
-declare <16 x i16> @llvm.x86.avx2.padds.w(<16 x i16>, <16 x i16>) nounwind readnone
+declare <16 x i16> @llvm.sadd.sat.v16i16(<16 x i16>, <16 x i16>) nounwind readnone
 
 define <16 x i8> @stack_fold_paddusb(<16 x i8> %a0, <16 x i8> %a1) {
   ;CHECK-LABEL: stack_fold_paddusb
   ;CHECK:       vpaddusb {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
-  %2 = call <16 x i8> @llvm.x86.sse2.paddus.b(<16 x i8> %a0, <16 x i8> %a1)
+  %2 = call <16 x i8> @llvm.uadd.sat.v16i8(<16 x i8> %a0, <16 x i8> %a1)
   ret <16 x i8> %2
 }
-declare <16 x i8> @llvm.x86.sse2.paddus.b(<16 x i8>, <16 x i8>) nounwind readnone
+declare <16 x i8> @llvm.uadd.sat.v16i8(<16 x i8>, <16 x i8>) nounwind readnone
 
 define <32 x i8> @stack_fold_paddusb_ymm(<32 x i8> %a0, <32 x i8> %a1) {
   ;CHECK-LABEL: stack_fold_paddusb_ymm
   ;CHECK:       vpaddusb {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
-  %2 = call <32 x i8> @llvm.x86.avx2.paddus.b(<32 x i8> %a0, <32 x i8> %a1)
+  %2 = call <32 x i8> @llvm.uadd.sat.v32i8(<32 x i8> %a0, <32 x i8> %a1)
   ret <32 x i8> %2
 }
-declare <32 x i8> @llvm.x86.avx2.paddus.b(<32 x i8>, <32 x i8>) nounwind readnone
+declare <32 x i8> @llvm.uadd.sat.v32i8(<32 x i8>, <32 x i8>) nounwind readnone
 
 define <8 x i16> @stack_fold_paddusw(<8 x i16> %a0, <8 x i16> %a1) {
   ;CHECK-LABEL: stack_fold_paddusw
   ;CHECK:       vpaddusw {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
-  %2 = call <8 x i16> @llvm.x86.sse2.paddus.w(<8 x i16> %a0, <8 x i16> %a1)
+  %2 = call <8 x i16> @llvm.uadd.sat.v8i16(<8 x i16> %a0, <8 x i16> %a1)
   ret <8 x i16> %2
 }
-declare <8 x i16> @llvm.x86.sse2.paddus.w(<8 x i16>, <8 x i16>) nounwind readnone
+declare <8 x i16> @llvm.uadd.sat.v8i16(<8 x i16>, <8 x i16>) nounwind readnone
 
 define <16 x i16> @stack_fold_paddusw_ymm(<16 x i16> %a0, <16 x i16> %a1) {
   ;CHECK-LABEL: stack_fold_paddusw_ymm
   ;CHECK:       vpaddusw {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
-  %2 = call <16 x i16> @llvm.x86.avx2.paddus.w(<16 x i16> %a0, <16 x i16> %a1)
+  %2 = call <16 x i16> @llvm.uadd.sat.v16i16(<16 x i16> %a0, <16 x i16> %a1)
   ret <16 x i16> %2
 }
-declare <16 x i16> @llvm.x86.avx2.paddus.w(<16 x i16>, <16 x i16>) nounwind readnone
+declare <16 x i16> @llvm.uadd.sat.v16i16(<16 x i16>, <16 x i16>) nounwind readnone
 
 define <8 x i16> @stack_fold_paddw(<8 x i16> %a0, <8 x i16> %a1) {
   ;CHECK-LABEL: stack_fold_paddw
@@ -766,37 +762,33 @@ define <4 x i32> @stack_fold_vplzcntd(<4 x i32> %a0) {
   ;CHECK-LABEL: stack_fold_vplzcntd
   ;CHECK:       vplzcntd {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
-  %2 = call <4 x i32> @llvm.ctlz.v4i32(<4 x i32> %a0)
+  %2 = call <4 x i32> @llvm.ctlz.v4i32(<4 x i32> %a0, i1 false)
   ret <4 x i32> %2
 }
-declare <4 x i32> @llvm.ctlz.v4i32(<4 x i32>) nounwind readonly
 
 define <8 x i32> @stack_fold_vplzcntd_ymm(<8 x i32> %a0) {
   ;CHECK-LABEL: stack_fold_vplzcntd_ymm
   ;CHECK:       vplzcntd {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
-  %2 = call <8 x i32> @llvm.ctlz.v8i32(<8 x i32> %a0)
+  %2 = call <8 x i32> @llvm.ctlz.v8i32(<8 x i32> %a0, i1 false)
   ret <8 x i32> %2
 }
-declare <8 x i32> @llvm.ctlz.v8i32(<8 x i32>) nounwind readonly
 
 define <2 x i64> @stack_fold_vplzcntq(<2 x i64> %a0) {
   ;CHECK-LABEL: stack_fold_vplzcntq
   ;CHECK:       vplzcntq {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
-  %2 = call <2 x i64> @llvm.ctlz.v2i64(<2 x i64> %a0)
+  %2 = call <2 x i64> @llvm.ctlz.v2i64(<2 x i64> %a0, i1 false)
   ret <2 x i64> %2
 }
-declare <2 x i64> @llvm.ctlz.v2i64(<2 x i64>) nounwind readnone
 
 define <4 x i64> @stack_fold_vplzcntq_ymm(<4 x i64> %a0) {
   ;CHECK-LABEL: stack_fold_vplzcntq_ymm
   ;CHECK:       vplzcntq {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
-  %2 = call <4 x i64> @llvm.ctlz.v4i64(<4 x i64> %a0)
+  %2 = call <4 x i64> @llvm.ctlz.v4i64(<4 x i64> %a0, i1 false)
   ret <4 x i64> %2
 }
-declare <4 x i64> @llvm.ctlz.v4i64(<4 x i64>) nounwind readnone
 
 define <8 x i16> @stack_fold_pmaddubsw(<16 x i8> %a0, <16 x i8> %a1) {
   ;CHECK-LABEL: stack_fold_pmaddubsw
@@ -1275,7 +1267,7 @@ declare <8 x i16> @llvm.x86.avx512.mask.pmov.dw.256(<8 x i32>, <8 x i16>, i8)
 define <4 x i32> @stack_fold_vpmovqd(<4 x i64> %a0) {
   ;CHECK-LABEL: stack_fold_vpmovqd
   ;CHECK:       vpmovqd %ymm0, {{-?[0-9]*}}(%rsp) # 16-byte Folded Spill
-  %1 = call <4 x i32> @llvm.x86.avx512.mask.pmov.qd.256(<4 x i64> %a0, <4 x i32> undef, i8 -1)
+  %1 = trunc <4 x i64> %a0 to <4 x i32>
   %2 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   ret <4 x i32> %1
 }
@@ -1284,7 +1276,7 @@ declare <4 x i32> @llvm.x86.avx512.mask.pmov.qd.256(<4 x i64>, <4 x i32>, i8)
 define <16 x i8> @stack_fold_vpmovwb(<16 x i16> %a0) {
   ;CHECK-LABEL: stack_fold_vpmovwb
   ;CHECK:       vpmovwb %ymm0, {{-?[0-9]*}}(%rsp) # 16-byte Folded Spill
-  %1 = call <16 x i8> @llvm.x86.avx512.mask.pmov.wb.256(<16 x i16> %a0, <16 x i8> undef, i16 -1)
+  %1 = trunc <16 x i16> %a0 to <16 x i8>
   %2 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
   ret <16 x i8> %1
 }
@@ -2355,73 +2347,73 @@ define <16 x i8> @stack_fold_psubsb(<16 x i8> %a0, <16 x i8> %a1) {
   ;CHECK-LABEL: stack_fold_psubsb
   ;CHECK:       vpsubsb {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
-  %2 = call <16 x i8> @llvm.x86.sse2.psubs.b(<16 x i8> %a0, <16 x i8> %a1)
+  %2 = call <16 x i8> @llvm.ssub.sat.v16i8(<16 x i8> %a0, <16 x i8> %a1)
   ret <16 x i8> %2
 }
-declare <16 x i8> @llvm.x86.sse2.psubs.b(<16 x i8>, <16 x i8>) nounwind readnone
+declare <16 x i8> @llvm.ssub.sat.v16i8(<16 x i8>, <16 x i8>) nounwind readnone
 
 define <32 x i8> @stack_fold_psubsb_ymm(<32 x i8> %a0, <32 x i8> %a1) {
   ;CHECK-LABEL: stack_fold_psubsb_ymm
   ;CHECK:       vpsubsb {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
-  %2 = call <32 x i8> @llvm.x86.avx2.psubs.b(<32 x i8> %a0, <32 x i8> %a1)
+  %2 = call <32 x i8> @llvm.ssub.sat.v32i8(<32 x i8> %a0, <32 x i8> %a1)
   ret <32 x i8> %2
 }
-declare <32 x i8> @llvm.x86.avx2.psubs.b(<32 x i8>, <32 x i8>) nounwind readnone
+declare <32 x i8> @llvm.ssub.sat.v32i8(<32 x i8>, <32 x i8>) nounwind readnone
 
 define <8 x i16> @stack_fold_psubsw(<8 x i16> %a0, <8 x i16> %a1) {
   ;CHECK-LABEL: stack_fold_psubsw
   ;CHECK:       vpsubsw {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
-  %2 = call <8 x i16> @llvm.x86.sse2.psubs.w(<8 x i16> %a0, <8 x i16> %a1)
+  %2 = call <8 x i16> @llvm.ssub.sat.v8i16(<8 x i16> %a0, <8 x i16> %a1)
   ret <8 x i16> %2
 }
-declare <8 x i16> @llvm.x86.sse2.psubs.w(<8 x i16>, <8 x i16>) nounwind readnone
+declare <8 x i16> @llvm.ssub.sat.v8i16(<8 x i16>, <8 x i16>) nounwind readnone
 
 define <16 x i16> @stack_fold_psubsw_ymm(<16 x i16> %a0, <16 x i16> %a1) {
   ;CHECK-LABEL: stack_fold_psubsw_ymm
   ;CHECK:       vpsubsw {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
-  %2 = call <16 x i16> @llvm.x86.avx2.psubs.w(<16 x i16> %a0, <16 x i16> %a1)
+  %2 = call <16 x i16> @llvm.ssub.sat.v16i16(<16 x i16> %a0, <16 x i16> %a1)
   ret <16 x i16> %2
 }
-declare <16 x i16> @llvm.x86.avx2.psubs.w(<16 x i16>, <16 x i16>) nounwind readnone
+declare <16 x i16> @llvm.ssub.sat.v16i16(<16 x i16>, <16 x i16>) nounwind readnone
 
 define <16 x i8> @stack_fold_psubusb(<16 x i8> %a0, <16 x i8> %a1) {
   ;CHECK-LABEL: stack_fold_psubusb
   ;CHECK:       vpsubusb {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
-  %2 = call <16 x i8> @llvm.x86.sse2.psubus.b(<16 x i8> %a0, <16 x i8> %a1)
+  %2 = call <16 x i8> @llvm.usub.sat.v16i8(<16 x i8> %a0, <16 x i8> %a1)
   ret <16 x i8> %2
 }
-declare <16 x i8> @llvm.x86.sse2.psubus.b(<16 x i8>, <16 x i8>) nounwind readnone
+declare <16 x i8> @llvm.usub.sat.v16i8(<16 x i8>, <16 x i8>) nounwind readnone
 
 define <32 x i8> @stack_fold_psubusb_ymm(<32 x i8> %a0, <32 x i8> %a1) {
   ;CHECK-LABEL: stack_fold_psubusb_ymm
   ;CHECK:       vpsubusb {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
-  %2 = call <32 x i8> @llvm.x86.avx2.psubus.b(<32 x i8> %a0, <32 x i8> %a1)
+  %2 = call <32 x i8> @llvm.usub.sat.v32i8(<32 x i8> %a0, <32 x i8> %a1)
   ret <32 x i8> %2
 }
-declare <32 x i8> @llvm.x86.avx2.psubus.b(<32 x i8>, <32 x i8>) nounwind readnone
+declare <32 x i8> @llvm.usub.sat.v32i8(<32 x i8>, <32 x i8>) nounwind readnone
 
 define <8 x i16> @stack_fold_psubusw(<8 x i16> %a0, <8 x i16> %a1) {
   ;CHECK-LABEL: stack_fold_psubusw
   ;CHECK:       vpsubusw {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}}, {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
-  %2 = call <8 x i16> @llvm.x86.sse2.psubus.w(<8 x i16> %a0, <8 x i16> %a1)
+  %2 = call <8 x i16> @llvm.usub.sat.v8i16(<8 x i16> %a0, <8 x i16> %a1)
   ret <8 x i16> %2
 }
-declare <8 x i16> @llvm.x86.sse2.psubus.w(<8 x i16>, <8 x i16>) nounwind readnone
+declare <8 x i16> @llvm.usub.sat.v8i16(<8 x i16>, <8 x i16>) nounwind readnone
 
 define <16 x i16> @stack_fold_psubusw_ymm(<16 x i16> %a0, <16 x i16> %a1) {
   ;CHECK-LABEL: stack_fold_psubusw_ymm
   ;CHECK:       vpsubusw {{-?[0-9]*}}(%rsp), {{%ymm[0-9][0-9]*}}, {{%ymm[0-9][0-9]*}} {{.*#+}} 32-byte Folded Reload
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
-  %2 = call <16 x i16> @llvm.x86.avx2.psubus.w(<16 x i16> %a0, <16 x i16> %a1)
+  %2 = call <16 x i16> @llvm.usub.sat.v16i16(<16 x i16> %a0, <16 x i16> %a1)
   ret <16 x i16> %2
 }
-declare <16 x i16> @llvm.x86.avx2.psubus.w(<16 x i16>, <16 x i16>) nounwind readnone
+declare <16 x i16> @llvm.usub.sat.v16i16(<16 x i16>, <16 x i16>) nounwind readnone
 
 define <8 x i16> @stack_fold_psubw(<8 x i16> %a0, <8 x i16> %a1) {
   ;CHECK-LABEL: stack_fold_psubw
@@ -2519,3 +2511,12 @@ define <8 x i32> @stack_fold_shufi32x4_maskz(<8 x i32> %a, <8 x i32> %b, i8 %mas
   %4 = select <8 x i1> %3, <8 x i32> %2, <8 x i32> zeroinitializer
   ret <8 x i32> %4
 }
+
+declare <4 x i32> @llvm.x86.avx512.conflict.d.128(<4 x i32>)
+declare <8 x i32> @llvm.x86.avx512.conflict.d.256(<8 x i32>)
+declare <2 x i64> @llvm.x86.avx512.conflict.q.128(<2 x i64>)
+declare <4 x i64> @llvm.x86.avx512.conflict.q.256(<4 x i64>)
+declare <4 x i32> @llvm.ctlz.v4i32(<4 x i32>, i1)
+declare <8 x i32> @llvm.ctlz.v8i32(<8 x i32>, i1)
+declare <2 x i64> @llvm.ctlz.v2i64(<2 x i64>, i1)
+declare <4 x i64> @llvm.ctlz.v4i64(<4 x i64>, i1)
diff --git a/test/CodeGen/X86/stack-folding-int-sse42.ll b/test/CodeGen/X86/stack-folding-int-sse42.ll
index 3c5aeb7841fb54953c6bd095fc02b66c12e82500..c498e2bfb6b91c078baedf636f0cb72052e55db6 100644
--- a/test/CodeGen/X86/stack-folding-int-sse42.ll
+++ b/test/CodeGen/X86/stack-folding-int-sse42.ll
@@ -241,37 +241,37 @@ define <16 x i8> @stack_fold_paddsb(<16 x i8> %a0, <16 x i8> %a1) {
   ;CHECK-LABEL: stack_fold_paddsb
   ;CHECK:       paddsb {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
-  %2 = call <16 x i8> @llvm.x86.sse2.padds.b(<16 x i8> %a0, <16 x i8> %a1)
+  %2 = call <16 x i8> @llvm.sadd.sat.v16i8(<16 x i8> %a0, <16 x i8> %a1)
   ret <16 x i8> %2
 }
-declare <16 x i8> @llvm.x86.sse2.padds.b(<16 x i8>, <16 x i8>) nounwind readnone
+declare <16 x i8> @llvm.sadd.sat.v16i8(<16 x i8>, <16 x i8>) nounwind readnone
 
 define <8 x i16> @stack_fold_paddsw(<8 x i16> %a0, <8 x i16> %a1) {
   ;CHECK-LABEL: stack_fold_paddsw
   ;CHECK:       paddsw {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
-  %2 = call <8 x i16> @llvm.x86.sse2.padds.w(<8 x i16> %a0, <8 x i16> %a1)
+  %2 = call <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16> %a0, <8 x i16> %a1)
   ret <8 x i16> %2
 }
-declare <8 x i16> @llvm.x86.sse2.padds.w(<8 x i16>, <8 x i16>) nounwind readnone
+declare <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16>, <8 x i16>) nounwind readnone
 
 define <16 x i8> @stack_fold_paddusb(<16 x i8> %a0, <16 x i8> %a1) {
   ;CHECK-LABEL: stack_fold_paddusb
   ;CHECK:       paddusb {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
-  %2 = call <16 x i8> @llvm.x86.sse2.paddus.b(<16 x i8> %a0, <16 x i8> %a1)
+  %2 = call <16 x i8> @llvm.uadd.sat.v16i8(<16 x i8> %a0, <16 x i8> %a1)
   ret <16 x i8> %2
 }
-declare <16 x i8> @llvm.x86.sse2.paddus.b(<16 x i8>, <16 x i8>) nounwind readnone
+declare <16 x i8> @llvm.uadd.sat.v16i8(<16 x i8>, <16 x i8>) nounwind readnone
 
 define <8 x i16> @stack_fold_paddusw(<8 x i16> %a0, <8 x i16> %a1) {
   ;CHECK-LABEL: stack_fold_paddusw
   ;CHECK:       paddusw {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
-  %2 = call <8 x i16> @llvm.x86.sse2.paddus.w(<8 x i16> %a0, <8 x i16> %a1)
+  %2 = call <8 x i16> @llvm.uadd.sat.v8i16(<8 x i16> %a0, <8 x i16> %a1)
   ret <8 x i16> %2
 }
-declare <8 x i16> @llvm.x86.sse2.paddus.w(<8 x i16>, <8 x i16>) nounwind readnone
+declare <8 x i16> @llvm.uadd.sat.v8i16(<8 x i16>, <8 x i16>) nounwind readnone
 
 define <8 x i16> @stack_fold_paddw(<8 x i16> %a0, <8 x i16> %a1) {
   ;CHECK-LABEL: stack_fold_paddw
@@ -1084,37 +1084,37 @@ define <16 x i8> @stack_fold_psubsb(<16 x i8> %a0, <16 x i8> %a1) {
   ;CHECK-LABEL: stack_fold_psubsb
   ;CHECK:       psubsb {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
-  %2 = call <16 x i8> @llvm.x86.sse2.psubs.b(<16 x i8> %a0, <16 x i8> %a1)
+  %2 = call <16 x i8> @llvm.ssub.sat.v16i8(<16 x i8> %a0, <16 x i8> %a1)
   ret <16 x i8> %2
 }
-declare <16 x i8> @llvm.x86.sse2.psubs.b(<16 x i8>, <16 x i8>) nounwind readnone
+declare <16 x i8> @llvm.ssub.sat.v16i8(<16 x i8>, <16 x i8>) nounwind readnone
 
 define <8 x i16> @stack_fold_psubsw(<8 x i16> %a0, <8 x i16> %a1) {
   ;CHECK-LABEL: stack_fold_psubsw
   ;CHECK:       psubsw {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
-  %2 = call <8 x i16> @llvm.x86.sse2.psubs.w(<8 x i16> %a0, <8 x i16> %a1)
+  %2 = call <8 x i16> @llvm.ssub.sat.v8i16(<8 x i16> %a0, <8 x i16> %a1)
   ret <8 x i16> %2
 }
-declare <8 x i16> @llvm.x86.sse2.psubs.w(<8 x i16>, <8 x i16>) nounwind readnone
+declare <8 x i16> @llvm.ssub.sat.v8i16(<8 x i16>, <8 x i16>) nounwind readnone
 
 define <16 x i8> @stack_fold_psubusb(<16 x i8> %a0, <16 x i8> %a1) {
   ;CHECK-LABEL: stack_fold_psubusb
   ;CHECK:       psubusb {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
-  %2 = call <16 x i8> @llvm.x86.sse2.psubus.b(<16 x i8> %a0, <16 x i8> %a1)
+  %2 = call <16 x i8> @llvm.usub.sat.v16i8(<16 x i8> %a0, <16 x i8> %a1)
   ret <16 x i8> %2
 }
-declare <16 x i8> @llvm.x86.sse2.psubus.b(<16 x i8>, <16 x i8>) nounwind readnone
+declare <16 x i8> @llvm.usub.sat.v16i8(<16 x i8>, <16 x i8>) nounwind readnone
 
 define <8 x i16> @stack_fold_psubusw(<8 x i16> %a0, <8 x i16> %a1) {
   ;CHECK-LABEL: stack_fold_psubusw
   ;CHECK:       psubusw {{-?[0-9]*}}(%rsp), {{%xmm[0-9][0-9]*}} {{.*#+}} 16-byte Folded Reload
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
-  %2 = call <8 x i16> @llvm.x86.sse2.psubus.w(<8 x i16> %a0, <8 x i16> %a1)
+  %2 = call <8 x i16> @llvm.usub.sat.v8i16(<8 x i16> %a0, <8 x i16> %a1)
   ret <8 x i16> %2
 }
-declare <8 x i16> @llvm.x86.sse2.psubus.w(<8 x i16>, <8 x i16>) nounwind readnone
+declare <8 x i16> @llvm.usub.sat.v8i16(<8 x i16>, <8 x i16>) nounwind readnone
 
 define <8 x i16> @stack_fold_psubw(<8 x i16> %a0, <8 x i16> %a1) {
   ;CHECK-LABEL: stack_fold_psubw
diff --git a/test/CodeGen/X86/statepoint-allocas.ll b/test/CodeGen/X86/statepoint-allocas.ll
index bd820e0b83d35b39515d187fdb1cc36ea1bde623..c7bae8ff18c109419bb22e9e52b1e86a861d5df6 100644
--- a/test/CodeGen/X86/statepoint-allocas.ll
+++ b/test/CodeGen/X86/statepoint-allocas.ll
@@ -1,23 +1,28 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -verify-machineinstrs < %s | FileCheck %s
 ; Check that we can lower a use of an alloca both as a deopt value (where the
 ; exact meaning is up to the consumer of the stackmap) and as an explicit spill
-; slot used for GC.  
+; slot used for GC.
 
 target datalayout = "e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-pc-linux-gnu"
 
 declare zeroext i1 @return_i1()
 
-; Can we handle an explicit relocation slot (in the form of an alloca) given 
+; Can we handle an explicit relocation slot (in the form of an alloca) given
 ; to the statepoint?
 define i32 addrspace(1)* @test(i32 addrspace(1)* %ptr) gc "statepoint-example" {
-; CHECK-LABEL: test
-; CHECK: pushq  %rax
-; CHECK: movq   %rdi, (%rsp)
-; CHECK: callq return_i1
-; CHECK: movq   (%rsp), %rax
-; CHECK: popq   %rcx
-; CHECK: retq
+; CHECK-LABEL: test:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    movq %rdi, (%rsp)
+; CHECK-NEXT:    callq return_i1
+; CHECK-NEXT:  .Ltmp0:
+; CHECK-NEXT:    movq (%rsp), %rax
+; CHECK-NEXT:    popq %rcx
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
 entry:
   %alloca = alloca i32 addrspace(1)*, align 8
   store i32 addrspace(1)* %ptr, i32 addrspace(1)** %alloca
@@ -26,15 +31,19 @@ entry:
   ret i32 addrspace(1)* %rel
 }
 
-; Can we handle an alloca as a deopt value?  
+; Can we handle an alloca as a deopt value?
 define i32 addrspace(1)* @test2(i32 addrspace(1)* %ptr) gc "statepoint-example" {
-; CHECK-LABEL: test2
-; CHECK: pushq  %rax
-; CHECK: movq   %rdi, (%rsp)
-; CHECK: callq return_i1
-; CHECK: xorl   %eax, %eax
-; CHECK: popq   %rcx
-; CHECK: retq
+; CHECK-LABEL: test2:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    movq %rdi, (%rsp)
+; CHECK-NEXT:    callq return_i1
+; CHECK-NEXT:  .Ltmp1:
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    popq %rcx
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
 entry:
   %alloca = alloca i32 addrspace(1)*, align 8
   store i32 addrspace(1)* %ptr, i32 addrspace(1)** %alloca
diff --git a/test/CodeGen/X86/statepoint-call-lowering.ll b/test/CodeGen/X86/statepoint-call-lowering.ll
index 2020b8d43bf63a15bb21fff5c7d75251ab19faae..bc312b71f6b4ba6bd34ba29fb2cef7f89704fa48 100644
--- a/test/CodeGen/X86/statepoint-call-lowering.ll
+++ b/test/CodeGen/X86/statepoint-call-lowering.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -verify-machineinstrs < %s | FileCheck %s
 ; This file contains a collection of basic tests to ensure we didn't
 ; screw up normal call lowering when there are no deopt or gc arguments.
@@ -15,13 +16,17 @@ declare %struct @return_struct()
 declare void @varargf(i32, ...)
 
 define i1 @test_i1_return() gc "statepoint-example" {
-; CHECK-LABEL: test_i1_return
+; CHECK-LABEL: test_i1_return:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    callq return_i1
+; CHECK-NEXT:  .Ltmp0:
+; CHECK-NEXT:    popq %rcx
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
 ; This is just checking that a i1 gets lowered normally when there's no extra
 ; state arguments to the statepoint
-; CHECK: pushq %rax
-; CHECK: callq return_i1
-; CHECK: popq %rcx
-; CHECK: retq
 entry:
   %safepoint_token = tail call token (i64, i32, i1 ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i1f(i64 0, i32 0, i1 ()* @return_i1, i32 0, i32 0, i32 0, i32 0)
   %call1 = call zeroext i1 @llvm.experimental.gc.result.i1(token %safepoint_token)
@@ -29,11 +34,15 @@ entry:
 }
 
 define i32 @test_i32_return() gc "statepoint-example" {
-; CHECK-LABEL: test_i32_return
-; CHECK: pushq %rax
-; CHECK: callq return_i32
-; CHECK: popq %rcx
-; CHECK: retq
+; CHECK-LABEL: test_i32_return:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    callq return_i32
+; CHECK-NEXT:  .Ltmp1:
+; CHECK-NEXT:    popq %rcx
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
 entry:
   %safepoint_token = tail call token (i64, i32, i32 ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i32f(i64 0, i32 0, i32 ()* @return_i32, i32 0, i32 0, i32 0, i32 0)
   %call1 = call zeroext i32 @llvm.experimental.gc.result.i32(token %safepoint_token)
@@ -41,11 +50,15 @@ entry:
 }
 
 define i32* @test_i32ptr_return() gc "statepoint-example" {
-; CHECK-LABEL: test_i32ptr_return
-; CHECK: pushq %rax
-; CHECK: callq return_i32ptr
-; CHECK: popq %rcx
-; CHECK: retq
+; CHECK-LABEL: test_i32ptr_return:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    callq return_i32ptr
+; CHECK-NEXT:  .Ltmp2:
+; CHECK-NEXT:    popq %rcx
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
 entry:
   %safepoint_token = tail call token (i64, i32, i32* ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_p0i32f(i64 0, i32 0, i32* ()* @return_i32ptr, i32 0, i32 0, i32 0, i32 0)
   %call1 = call i32* @llvm.experimental.gc.result.p0i32(token %safepoint_token)
@@ -53,11 +66,15 @@ entry:
 }
 
 define float @test_float_return() gc "statepoint-example" {
-; CHECK-LABEL: test_float_return
-; CHECK: pushq %rax
-; CHECK: callq return_float
-; CHECK: popq %rax
-; CHECK: retq
+; CHECK-LABEL: test_float_return:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    callq return_float
+; CHECK-NEXT:  .Ltmp3:
+; CHECK-NEXT:    popq %rax
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
 entry:
   %safepoint_token = tail call token (i64, i32, float ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_f32f(i64 0, i32 0, float ()* @return_float, i32 0, i32 0, i32 0, i32 0)
   %call1 = call float @llvm.experimental.gc.result.f32(token %safepoint_token)
@@ -65,11 +82,15 @@ entry:
 }
 
 define %struct @test_struct_return() gc "statepoint-example" {
-; CHECK-LABEL: test_struct_return
-; CHECK: pushq %rax
-; CHECK: callq return_struct
-; CHECK: popq %rcx
-; CHECK: retq
+; CHECK-LABEL: test_struct_return:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    callq return_struct
+; CHECK-NEXT:  .Ltmp4:
+; CHECK-NEXT:    popq %rcx
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
 entry:
   %safepoint_token = tail call token (i64, i32, %struct ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_structf(i64 0, i32 0, %struct ()* @return_struct, i32 0, i32 0, i32 0, i32 0)
   %call1 = call %struct @llvm.experimental.gc.result.struct(token %safepoint_token)
@@ -77,14 +98,17 @@ entry:
 }
 
 define i1 @test_relocate(i32 addrspace(1)* %a) gc "statepoint-example" {
-; CHECK-LABEL: test_relocate
+; CHECK-LABEL: test_relocate:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    movq %rdi, (%rsp)
+; CHECK-NEXT:    callq return_i1
+; CHECK-NEXT:  .Ltmp5:
+; CHECK-NEXT:    popq %rcx
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
 ; Check that an ununsed relocate has no code-generation impact
-; CHECK: pushq %rax
-; CHECK: callq return_i1
-; CHECK-NEXT: .Ltmp5:
-; CHECK-NEXT: popq %rcx
-; CHECK-NEXT: .cfi_def_cfa_offset 8
-; CHECK-NEXT: retq
 entry:
   %safepoint_token = tail call token (i64, i32, i1 ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i1f(i64 0, i32 0, i1 ()* @return_i1, i32 0, i32 0, i32 0, i32 0, i32 addrspace(1)* %a)
   %call1 = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %safepoint_token,  i32 7, i32 7)
@@ -93,9 +117,18 @@ entry:
 }
 
 define void @test_void_vararg() gc "statepoint-example" {
-; CHECK-LABEL: test_void_vararg
+; CHECK-LABEL: test_void_vararg:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    movl $42, %edi
+; CHECK-NEXT:    movl $43, %esi
+; CHECK-NEXT:    callq varargf
+; CHECK-NEXT:  .Ltmp6:
+; CHECK-NEXT:    popq %rax
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
 ; Check a statepoint wrapping a *void* returning vararg function works
-; CHECK: callq varargf
 entry:
   %safepoint_token = tail call token (i64, i32, void (i32, ...)*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidi32varargf(i64 0, i32 0, void (i32, ...)* @varargf, i32 2, i32 0, i32 42, i32 43, i32 0, i32 0)
   ;; if we try to use the result from a statepoint wrapping a
@@ -104,12 +137,17 @@ entry:
 }
 
 define i1 @test_i1_return_patchable() gc "statepoint-example" {
-; CHECK-LABEL: test_i1_return_patchable
+; CHECK-LABEL: test_i1_return_patchable:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    nopl (%rax)
+; CHECK-NEXT:  .Ltmp7:
+; CHECK-NEXT:    popq %rcx
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
 ; A patchable variant of test_i1_return
-; CHECK: pushq %rax
-; CHECK: nopl
-; CHECK: popq %rcx
-; CHECK: retq
 entry:
   %safepoint_token = tail call token (i64, i32, i1 ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i1f(i64 0, i32 3, i1 ()*null, i32 0, i32 0, i32 0, i32 0)
   %call1 = call zeroext i1 @llvm.experimental.gc.result.i1(token %safepoint_token)
@@ -119,13 +157,38 @@ entry:
 declare void @consume(i32 addrspace(1)* %obj)
 
 define i1 @test_cross_bb(i32 addrspace(1)* %a, i1 %external_cond) gc "statepoint-example" {
-; CHECK-LABEL: test_cross_bb
-; CHECK: movq
-; CHECK: callq return_i1
-; CHECK: %left
-; CHECK: movq
-; CHECK-NEXT: callq consume
-; CHECK: retq
+; CHECK-LABEL: test_cross_bb:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    .cfi_offset %rbx, -24
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    movl %esi, %ebp
+; CHECK-NEXT:    movq %rdi, (%rsp)
+; CHECK-NEXT:    callq return_i1
+; CHECK-NEXT:  .Ltmp8:
+; CHECK-NEXT:    testb $1, %bpl
+; CHECK-NEXT:    je .LBB8_2
+; CHECK-NEXT:  # %bb.1: # %left
+; CHECK-NEXT:    movl %eax, %ebx
+; CHECK-NEXT:    movq (%rsp), %rdi
+; CHECK-NEXT:    callq consume
+; CHECK-NEXT:    movl %ebx, %eax
+; CHECK-NEXT:    jmp .LBB8_3
+; CHECK-NEXT:  .LBB8_2: # %right
+; CHECK-NEXT:    movb $1, %al
+; CHECK-NEXT:  .LBB8_3: # %right
+; CHECK-NEXT:    addq $8, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
 entry:
   %safepoint_token = tail call token (i64, i32, i1 ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i1f(i64 0, i32 0, i1 ()* @return_i1, i32 0, i32 0, i32 0, i32 0, i32 addrspace(1)* %a)
   br i1 %external_cond, label %left, label %right
@@ -145,17 +208,34 @@ right:
 declare void @consume_attributes(i32, i8* nest, i32, %struct2* byval)
 
 define void @test_attributes(%struct2* byval %s) gc "statepoint-example" {
+; CHECK-LABEL: test_attributes:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    subq $8, %rsp
+; CHECK-NEXT:    .cfi_adjust_cfa_offset 8
+; CHECK-NEXT:    movq {{[0-9]+}}(%rsp), %rax
+; CHECK-NEXT:    movq {{[0-9]+}}(%rsp), %rcx
+; CHECK-NEXT:    movq {{[0-9]+}}(%rsp), %rdx
+; CHECK-NEXT:    movl $42, %edi
+; CHECK-NEXT:    xorl %r10d, %r10d
+; CHECK-NEXT:    movl $17, %esi
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    .cfi_adjust_cfa_offset 8
+; CHECK-NEXT:    pushq %rdx
+; CHECK-NEXT:    .cfi_adjust_cfa_offset 8
+; CHECK-NEXT:    pushq %rcx
+; CHECK-NEXT:    .cfi_adjust_cfa_offset 8
+; CHECK-NEXT:    callq consume_attributes
+; CHECK-NEXT:  .Ltmp9:
+; CHECK-NEXT:    addq $32, %rsp
+; CHECK-NEXT:    .cfi_adjust_cfa_offset -32
+; CHECK-NEXT:    popq %rax
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
 entry:
-; CHECK-LABEL: test_attributes
 ; Check that arguments with attributes are lowered correctly.
 ; We call a function that has a nest argument and a byval argument.
-; CHECK: movl $42, %edi
-; CHECK: xorl %r10d, %r10d
-; CHECK: movl $17, %esi
-; CHECK: pushq
-; CHECK: pushq
-; CHECK: pushq
-; CHECK: callq consume_attributes
   %statepoint_token = call token (i64, i32, void (i32, i8*, i32, %struct2*)*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidi32p0i8i32p0s_struct2sf(i64 0, i32 0, void (i32, i8*, i32, %struct2*)* @consume_attributes, i32 4, i32 0, i32 42, i8* nest null, i32 17, %struct2* byval %s, i32 0, i32 0)
   ret void
 }
diff --git a/test/CodeGen/X86/statepoint-far-call.ll b/test/CodeGen/X86/statepoint-far-call.ll
index 9f9b684efae838e9dab1deff00a8ef2dd61b0ed4..487829857f88fd4c705dfde6cb130643d43505d2 100644
--- a/test/CodeGen/X86/statepoint-far-call.ll
+++ b/test/CodeGen/X86/statepoint-far-call.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -verify-machineinstrs < %s | FileCheck %s
 ; Test to check that Statepoints with X64 far-immediate targets
 ; are lowered correctly to an indirect call via a scratch register.
@@ -6,15 +7,19 @@ target datalayout = "e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-pc-win64"
 
 define void @test_far_call() gc "statepoint-example" {
-; CHECK-LABEL: test_far_call
-; CHECK: pushq %rax
-; CHECK: movabsq $140727162896504, %rax 
-; CHECK: callq *%rax
-; CHECK: popq %rax
-; CHECK: retq
+; CHECK-LABEL: test_far_call:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    movabsq $140727162896504, %rax # imm = 0x7FFD988E0078
+; CHECK-NEXT:    callq *%rax
+; CHECK-NEXT:  .Ltmp0:
+; CHECK-NEXT:    popq %rax
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
 
 entry:
-  %safepoint_token = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* inttoptr (i64 140727162896504 to void ()*), i32 0, i32 0, i32 0, i32 0)  
+  %safepoint_token = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* inttoptr (i64 140727162896504 to void ()*), i32 0, i32 0, i32 0, i32 0)
   ret void
 }
 
diff --git a/test/CodeGen/X86/statepoint-invoke.ll b/test/CodeGen/X86/statepoint-invoke.ll
index d4934e3b37967476a7507e02101912a8a60f9717..e0edac6e4e6e6316b5aa2b9844f50624c43e60c8 100644
--- a/test/CodeGen/X86/statepoint-invoke.ll
+++ b/test/CodeGen/X86/statepoint-invoke.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -verify-machineinstrs < %s 2>&1 | FileCheck %s
 
 target triple = "x86_64-pc-linux-gnu"
@@ -8,29 +9,43 @@ declare i64 addrspace(1)* @"some_other_call"(i64 addrspace(1)*)
 declare i32 @"personality_function"()
 
 define i64 addrspace(1)* @test_basic(i64 addrspace(1)* %obj,
+; CHECK-LABEL: test_basic:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    subq $24, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    movq %rdi, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    movq %rsi, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:  .Ltmp0:
+; CHECK-NEXT:    callq some_call
+; CHECK-NEXT:  .Ltmp3:
+; CHECK-NEXT:  .Ltmp1:
+; CHECK-NEXT:  # %bb.1: # %invoke_safepoint_normal_dest
+; CHECK-NEXT:    movq {{[0-9]+}}(%rsp), %rax
+; CHECK-NEXT:    addq $24, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
+; CHECK-NEXT:  .LBB0_2: # %exceptional_return
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:  .Ltmp2:
+; CHECK-NEXT:    movq {{[0-9]+}}(%rsp), %rax
+; CHECK-NEXT:    addq $24, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
                                      i64 addrspace(1)* %obj1)
 gc "statepoint-example" personality i32 ()* @"personality_function" {
 entry:
-  ; CHECK: Ltmp{{[0-9]+}}:
-  ; CHECK: callq some_call
-  ; CHECK: Ltmp{{[0-9]+}}:
   %0 = invoke token (i64, i32, void (i64 addrspace(1)*)*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidp1i64f(i64 0, i32 0, void (i64 addrspace(1)*)* @some_call, i32 1, i32 0, i64 addrspace(1)* %obj, i32 0, i32 5, i32 0, i32 -1, i32 0, i32 0, i32 0, i64 addrspace(1)* %obj, i64 addrspace(1)* %obj1)
           to label %invoke_safepoint_normal_dest unwind label %exceptional_return
 
 invoke_safepoint_normal_dest:
-  ; CHECK: movq
   %obj.relocated = call coldcc i64 addrspace(1)* @llvm.experimental.gc.relocate.p1i64(token %0, i32 13, i32 13)
   %obj1.relocated = call coldcc i64 addrspace(1)* @llvm.experimental.gc.relocate.p1i64(token %0, i32 14, i32 14)
   br label %normal_return
 
 normal_return:
-  ; CHECK: retq
   ret i64 addrspace(1)* %obj.relocated
 
 exceptional_return:
-  ; CHECK: Ltmp{{[0-9]+}}:
-  ; CHECK: movq
-  ; CHECK: retq
   %landing_pad = landingpad token
           cleanup
   %obj.relocated1 = call coldcc i64 addrspace(1)* @llvm.experimental.gc.relocate.p1i64(token %landing_pad, i32 13, i32 13)
@@ -44,24 +59,37 @@ exceptional_return:
 ; CHECK: .p2align 4
 
 define i64 addrspace(1)* @test_result(i64 addrspace(1)* %obj,
+; CHECK-LABEL: test_result:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    movq %rdi, (%rsp)
+; CHECK-NEXT:  .Ltmp4:
+; CHECK-NEXT:    callq some_other_call
+; CHECK-NEXT:  .Ltmp7:
+; CHECK-NEXT:  .Ltmp5:
+; CHECK-NEXT:  # %bb.1: # %normal_return
+; CHECK-NEXT:    popq %rcx
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
+; CHECK-NEXT:  .LBB1_2: # %exceptional_return
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:  .Ltmp6:
+; CHECK-NEXT:    movq (%rsp), %rax
+; CHECK-NEXT:    popq %rcx
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
                                       i64 addrspace(1)* %obj1)
   gc "statepoint-example" personality i32 ()* @personality_function {
 entry:
-  ; CHECK: .Ltmp{{[0-9]+}}:
-  ; CHECK: callq some_other_call
-  ; CHECK: .Ltmp{{[0-9]+}}:
   %0 = invoke token (i64, i32, i64 addrspace(1)* (i64 addrspace(1)*)*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_p1i64p1i64f(i64 0, i32 0, i64 addrspace(1)* (i64 addrspace(1)*)* @some_other_call, i32 1, i32 0, i64 addrspace(1)* %obj, i32 0, i32 5, i32 0, i32 -1, i32 0, i32 0, i32 0, i64 addrspace(1)* %obj, i64 addrspace(1)* %obj1)
           to label %normal_return unwind label %exceptional_return
 
 normal_return:
-  ; CHECK: popq 
-  ; CHECK: retq
   %ret_val = call i64 addrspace(1)* @llvm.experimental.gc.result.p1i64(token %0)
   ret i64 addrspace(1)* %ret_val
 
 exceptional_return:
-  ; CHECK: .Ltmp{{[0-9]+}}:
-  ; CHECK: movq
   %landing_pad = landingpad token
           cleanup
   %obj.relocated = call coldcc i64 addrspace(1)* @llvm.experimental.gc.relocate.p1i64(token %landing_pad, i32 13, i32 13)
@@ -74,44 +102,80 @@ exceptional_return:
 ; CHECK: .p2align 4
 
 define i64 addrspace(1)* @test_same_val(i1 %cond, i64 addrspace(1)* %val1, i64 addrspace(1)* %val2, i64 addrspace(1)* %val3)
+; CHECK-LABEL: test_same_val:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    subq $16, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    .cfi_offset %rbx, -16
+; CHECK-NEXT:    movl %edi, %ebx
+; CHECK-NEXT:    testb $1, %bl
+; CHECK-NEXT:    je .LBB2_3
+; CHECK-NEXT:  # %bb.1: # %left
+; CHECK-NEXT:    movq %rsi, (%rsp)
+; CHECK-NEXT:    movq %rdx, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:  .Ltmp11:
+; CHECK-NEXT:    movq %rsi, %rdi
+; CHECK-NEXT:    callq some_call
+; CHECK-NEXT:  .Ltmp14:
+; CHECK-NEXT:  .Ltmp12:
+; CHECK-NEXT:  # %bb.2: # %left.relocs
+; CHECK-NEXT:    movq (%rsp), %rax
+; CHECK-NEXT:    movq {{[0-9]+}}(%rsp), %rcx
+; CHECK-NEXT:    jmp .LBB2_5
+; CHECK-NEXT:  .LBB2_3: # %right
+; CHECK-NEXT:    movq %rdx, (%rsp)
+; CHECK-NEXT:    movq %rcx, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:  .Ltmp8:
+; CHECK-NEXT:    movq %rsi, %rdi
+; CHECK-NEXT:    callq some_call
+; CHECK-NEXT:  .Ltmp15:
+; CHECK-NEXT:  .Ltmp9:
+; CHECK-NEXT:  # %bb.4: # %right.relocs
+; CHECK-NEXT:    movq (%rsp), %rcx
+; CHECK-NEXT:    movq {{[0-9]+}}(%rsp), %rax
+; CHECK-NEXT:  .LBB2_5: # %normal_return
+; CHECK-NEXT:    testb $1, %bl
+; CHECK-NEXT:    cmoveq %rcx, %rax
+; CHECK-NEXT:  .LBB2_6: # %normal_return
+; CHECK-NEXT:    addq $16, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
+; CHECK-NEXT:  .LBB2_9: # %exceptional_return.right
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:  .Ltmp10:
+; CHECK-NEXT:    movq (%rsp), %rax
+; CHECK-NEXT:    jmp .LBB2_6
+; CHECK-NEXT:  .LBB2_7: # %exceptional_return.left
+; CHECK-NEXT:  .Ltmp13:
+; CHECK-NEXT:    movq (%rsp), %rax
+; CHECK-NEXT:    jmp .LBB2_6
   gc "statepoint-example" personality i32 ()* @"personality_function" {
 entry:
   br i1 %cond, label %left, label %right
 
 left:
-  ; CHECK-LABEL: %left
-  ; CHECK: movq %rdx, 8(%rsp)
-  ; CHECK: movq
-  ; CHECK: callq some_call
   %sp1 = invoke token (i64, i32, void (i64 addrspace(1)*)*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidp1i64f(i64 0, i32 0, void (i64 addrspace(1)*)* @some_call, i32 1, i32 0, i64 addrspace(1)* %val1, i32 0, i32 5, i32 0, i32 -1, i32 0, i32 0, i32 0, i64 addrspace(1)* %val1, i64 addrspace(1)* %val2)
            to label %left.relocs unwind label %exceptional_return.left
 
 left.relocs:
-  ; CHECK: movq (%rsp),
-  ; CHECK: movq 8(%rsp), [[REGVAL2:%[a-z]+]]
   %val1.relocated = call coldcc i64 addrspace(1)* @llvm.experimental.gc.relocate.p1i64(token %sp1, i32 13, i32 13)
   %val2.relocated_left = call coldcc i64 addrspace(1)* @llvm.experimental.gc.relocate.p1i64(token %sp1, i32 14, i32 14)
   br label %normal_return
 
 right:
-  ; CHECK-LABEL: %right
-  ; CHECK: movq %rdx, (%rsp)
-  ; CHECK: movq
-  ; CHECK: callq some_call
   %sp2 = invoke token (i64, i32, void (i64 addrspace(1)*)*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidp1i64f(i64 0, i32 0, void (i64 addrspace(1)*)* @some_call, i32 1, i32 0, i64 addrspace(1)* %val1, i32 0, i32 5, i32 0, i32 -1, i32 0, i32 0, i32 0, i64 addrspace(1)* %val2, i64 addrspace(1)* %val3)
            to label %right.relocs unwind label %exceptional_return.right
 
 right.relocs:
-  ; CHECK: movq (%rsp), [[REGVAL2]]
-  ; CHECK: movq
   %val2.relocated_right = call coldcc i64 addrspace(1)* @llvm.experimental.gc.relocate.p1i64(token %sp2, i32 13, i32 13)
   %val3.relocated = call coldcc i64 addrspace(1)* @llvm.experimental.gc.relocate.p1i64(token %sp2, i32 14, i32 14)
   br label %normal_return
 
 normal_return:
-  ; CHECK-LABEL: %normal_return
-  ; CHECK: cmoveq {{.*}}[[REGVAL2]]{{.*}}
-  ; CHECK: retq
   %a1 = phi i64 addrspace(1)* [%val1.relocated, %left.relocs], [%val3.relocated, %right.relocs]
   %a2 = phi i64 addrspace(1)* [%val2.relocated_left, %left.relocs], [%val2.relocated_right, %right.relocs]
   %ret = select i1 %cond, i64 addrspace(1)* %a1, i64 addrspace(1)* %a2
@@ -131,19 +195,29 @@ exceptional_return.right:
 }
 
 define i64 addrspace(1)* @test_null_undef(i64 addrspace(1)* %val1)
-       gc "statepoint-example" personality i32 ()* @"personality_function" {
 ; CHECK-LABEL: test_null_undef:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:  .Ltmp16:
+; CHECK-NEXT:    callq some_call
+; CHECK-NEXT:  .Ltmp19:
+; CHECK-NEXT:  .Ltmp17:
+; CHECK-NEXT:  .LBB3_1: # %normal_return
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    popq %rcx
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
+; CHECK-NEXT:  .LBB3_2: # %exceptional_return
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:  .Ltmp18:
+; CHECK-NEXT:    jmp .LBB3_1
+       gc "statepoint-example" personality i32 ()* @"personality_function" {
 entry:
-  ; CHECK: callq some_call
   %sp1 = invoke token (i64, i32, void (i64 addrspace(1)*)*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidp1i64f(i64 0, i32 0, void (i64 addrspace(1)*)* @some_call, i32 1, i32 0, i64 addrspace(1)* %val1, i32 0, i32 5, i32 0, i32 -1, i32 0, i32 0, i32 0, i64 addrspace(1)* null, i64 addrspace(1)* undef)
            to label %normal_return unwind label %exceptional_return
 
 normal_return:
-  ; CHECK-LABEL: %normal_return
-  ; CHECK: xorl %eax, %eax
-  ; CHECK-NEXT: popq
-  ; CHECK-NEXT: .cfi_def_cfa_offset 8
-  ; CHECK-NEXT: retq
   %null.relocated = call coldcc i64 addrspace(1)* @llvm.experimental.gc.relocate.p1i64(token %sp1, i32 13, i32 13)
   %undef.relocated = call coldcc i64 addrspace(1)* @llvm.experimental.gc.relocate.p1i64(token %sp1, i32 14, i32 14)
   ret i64 addrspace(1)* %null.relocated
@@ -157,30 +231,40 @@ exceptional_return:
 }
 
 define i64 addrspace(1)* @test_alloca_and_const(i64 addrspace(1)* %val1)
-       gc "statepoint-example" personality i32 ()* @"personality_function" {
 ; CHECK-LABEL: test_alloca_and_const:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:  .Ltmp20:
+; CHECK-NEXT:    callq some_call
+; CHECK-NEXT:  .Ltmp23:
+; CHECK-NEXT:  .Ltmp21:
+; CHECK-NEXT:  # %bb.1: # %normal_return
+; CHECK-NEXT:    leaq {{[0-9]+}}(%rsp), %rax
+; CHECK-NEXT:    popq %rcx
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
+; CHECK-NEXT:  .LBB4_2: # %exceptional_return
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:  .Ltmp22:
+; CHECK-NEXT:    movl $15, %eax
+; CHECK-NEXT:    popq %rcx
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
+       gc "statepoint-example" personality i32 ()* @"personality_function" {
 entry:
   %a = alloca i32
   %aa = addrspacecast i32* %a to i32 addrspace(1)*
   %c = inttoptr i64 15 to i64 addrspace(1)*
-  ; CHECK: callq
   %sp = invoke token (i64, i32, void (i64 addrspace(1)*)*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidp1i64f(i64 0, i32 0, void (i64 addrspace(1)*)* @some_call, i32 1, i32 0, i64 addrspace(1)* %val1, i32 0, i32 5, i32 0, i32 -1, i32 0, i32 0, i32 0, i32 addrspace(1)* %aa, i64 addrspace(1)* %c)
            to label %normal_return unwind label %exceptional_return
 
 normal_return:
-  ; CHECK: leaq
-  ; CHECK-NEXT: popq
-  ; CHECK-NEXT: .cfi_def_cfa_offset 8
-  ; CHECK-NEXT: retq
   %aa.rel = call coldcc i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %sp, i32 13, i32 13)
   %aa.converted = bitcast i32 addrspace(1)* %aa.rel to i64 addrspace(1)*
   ret i64 addrspace(1)* %aa.converted
 
 exceptional_return:
-  ; CHECK: movl	$15
-  ; CHECK-NEXT: popq
-  ; CHECK-NEXT: .cfi_def_cfa_offset 8
-  ; CHECK-NEXT: retq
   %landing_pad = landingpad token
           cleanup
   %aa.rel2 = call coldcc i64 addrspace(1)* @llvm.experimental.gc.relocate.p1i64(token %landing_pad, i32 14, i32 14)
diff --git a/test/CodeGen/X86/statepoint-live-in.ll b/test/CodeGen/X86/statepoint-live-in.ll
index 69affe2a9fcbe0efb16cf62a2518c9fb3d4a42ea..acc12f8e6283d4da61c4f02420dd73780a172f38 100644
--- a/test/CodeGen/X86/statepoint-live-in.ll
+++ b/test/CodeGen/X86/statepoint-live-in.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -verify-machineinstrs -O3 < %s | FileCheck %s
+; RUN: llc -verify-machineinstrs -O3 -restrict-statepoint-remat < %s | FileCheck %s
 target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.11.0"
 
@@ -128,6 +128,511 @@ entry:
   ret void
 }
 
+; A variant of test6 where values are not directly foldable from stack slots.
+; This stresses our rematerialization handling.
+define void @test7(i32 %a, i32 %b, i32 %c, i32 %d, i32 %e, i32 %f, i32 %g, i32 %h, i32 %i, i32 %j, i32 %k, i32 %l, i32 %m, i32 %n, i32 %o, i32 %p, i32 %q, i32 %r, i32 %s, i32 %t, i32 %u, i32 %v, i32 %w, i32 %x, i32 %y, i32 %z) gc "statepoint-example" {
+; The code for this is terrible, check simply for correctness for the moment
+; CHECK-LABEL: test7:
+; CHECK:       ## %bb.0: ## %entry
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    pushq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    pushq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 56
+; CHECK-NEXT:    subq $88, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 144
+; CHECK-NEXT:    .cfi_offset %rbx, -56
+; CHECK-NEXT:    .cfi_offset %r12, -48
+; CHECK-NEXT:    .cfi_offset %r13, -40
+; CHECK-NEXT:    .cfi_offset %r14, -32
+; CHECK-NEXT:    .cfi_offset %r15, -24
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    movl {{[0-9]+}}(%rsp), %eax
+; CHECK-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) ## 8-byte Spill
+; CHECK-NEXT:    movl {{[0-9]+}}(%rsp), %eax
+; CHECK-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) ## 8-byte Spill
+; CHECK-NEXT:    movl {{[0-9]+}}(%rsp), %eax
+; CHECK-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) ## 8-byte Spill
+; CHECK-NEXT:    movl {{[0-9]+}}(%rsp), %eax
+; CHECK-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) ## 8-byte Spill
+; CHECK-NEXT:    movl {{[0-9]+}}(%rsp), %eax
+; CHECK-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) ## 8-byte Spill
+; CHECK-NEXT:    movl {{[0-9]+}}(%rsp), %eax
+; CHECK-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) ## 8-byte Spill
+; CHECK-NEXT:    movl {{[0-9]+}}(%rsp), %eax
+; CHECK-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) ## 8-byte Spill
+; CHECK-NEXT:    movl {{[0-9]+}}(%rsp), %eax
+; CHECK-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) ## 8-byte Spill
+; CHECK-NEXT:    movl {{[0-9]+}}(%rsp), %eax
+; CHECK-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) ## 8-byte Spill
+; CHECK-NEXT:    movl %edi, %edi
+; CHECK-NEXT:    movl %esi, %esi
+; CHECK-NEXT:    movl %edx, %edx
+; CHECK-NEXT:    movl %ecx, %ecx
+; CHECK-NEXT:    movl %r8d, %r8d
+; CHECK-NEXT:    movl %r9d, %r9d
+; CHECK-NEXT:    movl {{[0-9]+}}(%rsp), %eax
+; CHECK-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) ## 8-byte Spill
+; CHECK-NEXT:    movl {{[0-9]+}}(%rsp), %eax
+; CHECK-NEXT:    movq %rax, (%rsp) ## 8-byte Spill
+; CHECK-NEXT:    movl {{[0-9]+}}(%rsp), %ebp
+; CHECK-NEXT:    movl {{[0-9]+}}(%rsp), %r13d
+; CHECK-NEXT:    movl {{[0-9]+}}(%rsp), %r12d
+; CHECK-NEXT:    movl {{[0-9]+}}(%rsp), %r15d
+; CHECK-NEXT:    movl {{[0-9]+}}(%rsp), %r14d
+; CHECK-NEXT:    movl {{[0-9]+}}(%rsp), %ebx
+; CHECK-NEXT:    movl {{[0-9]+}}(%rsp), %r11d
+; CHECK-NEXT:    movl {{[0-9]+}}(%rsp), %r10d
+; CHECK-NEXT:    movl {{[0-9]+}}(%rsp), %eax
+; CHECK-NEXT:    callq _bar ## 88-byte Folded Reload
+; CHECK-NEXT:  Ltmp8:
+; CHECK-NEXT:    addq $88, %rsp
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    popq %r12
+; CHECK-NEXT:    popq %r13
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    retq
+entry:
+  %a64 = zext i32 %a to i64
+  %b64 = zext i32 %b to i64
+  %c64 = zext i32 %c to i64
+  %d64 = zext i32 %d to i64
+  %e64 = zext i32 %e to i64
+  %f64 = zext i32 %f to i64
+  %g64 = zext i32 %g to i64
+  %h64 = zext i32 %h to i64
+  %i64 = zext i32 %i to i64
+  %j64 = zext i32 %j to i64
+  %k64 = zext i32 %k to i64
+  %l64 = zext i32 %l to i64
+  %m64 = zext i32 %m to i64
+  %n64 = zext i32 %n to i64
+  %o64 = zext i32 %o to i64
+  %p64 = zext i32 %p to i64
+  %q64 = zext i32 %q to i64
+  %r64 = zext i32 %r to i64
+  %s64 = zext i32 %s to i64
+  %t64 = zext i32 %t to i64
+  %u64 = zext i32 %u to i64
+  %v64 = zext i32 %v to i64
+  %w64 = zext i32 %w to i64
+  %x64 = zext i32 %x to i64
+  %y64 = zext i32 %y to i64
+  %z64 = zext i32 %z to i64
+  %statepoint_token1 = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @bar, i32 0, i32 2, i32 0, i32 26, i64 %a64, i64 %b64, i64 %c64, i64 %d64, i64 %e64, i64 %f64, i64 %g64, i64 %h64, i64 %i64, i64 %j64, i64 %k64, i64 %l64, i64 %m64, i64 %n64, i64 %o64, i64 %p64, i64 %q64, i64 %r64, i64 %s64, i64 %t64, i64 %u64, i64 %v64, i64 %w64, i64 %x64, i64 %y64, i64 %z64)
+  ret void
+}
+
+; a variant of test7 with mixed types chosen to exercise register aliases
+define void @test8(i32 %a, i32 %b, i32 %c, i32 %d, i32 %e, i32 %f, i32 %g, i32 %h, i32 %i, i32 %j, i32 %k, i32 %l, i32 %m, i32 %n, i32 %o, i32 %p, i32 %q, i32 %r, i32 %s, i32 %t, i32 %u, i32 %v, i32 %w, i32 %x, i32 %y, i32 %z) gc "statepoint-example" {
+; The code for this is terrible, check simply for correctness for the moment
+; CHECK-LABEL: test8:
+; CHECK:       ## %bb.0: ## %entry
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    pushq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    pushq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 56
+; CHECK-NEXT:    subq $72, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 128
+; CHECK-NEXT:    .cfi_offset %rbx, -56
+; CHECK-NEXT:    .cfi_offset %r12, -48
+; CHECK-NEXT:    .cfi_offset %r13, -40
+; CHECK-NEXT:    .cfi_offset %r14, -32
+; CHECK-NEXT:    .cfi_offset %r15, -24
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    movl %r9d, %r10d
+; CHECK-NEXT:    movl %r8d, %r9d
+; CHECK-NEXT:    movl {{[0-9]+}}(%rsp), %eax
+; CHECK-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) ## 8-byte Spill
+; CHECK-NEXT:    movl {{[0-9]+}}(%rsp), %eax
+; CHECK-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) ## 8-byte Spill
+; CHECK-NEXT:    movl {{[0-9]+}}(%rsp), %eax
+; CHECK-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) ## 8-byte Spill
+; CHECK-NEXT:    movl {{[0-9]+}}(%rsp), %eax
+; CHECK-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) ## 8-byte Spill
+; CHECK-NEXT:    movl {{[0-9]+}}(%rsp), %eax
+; CHECK-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) ## 8-byte Spill
+; CHECK-NEXT:    movl {{[0-9]+}}(%rsp), %eax
+; CHECK-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) ## 8-byte Spill
+; CHECK-NEXT:    movl {{[0-9]+}}(%rsp), %eax
+; CHECK-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) ## 8-byte Spill
+; CHECK-NEXT:    movl {{[0-9]+}}(%rsp), %eax
+; CHECK-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) ## 8-byte Spill
+; CHECK-NEXT:    movl {{[0-9]+}}(%rsp), %eax
+; CHECK-NEXT:    movq %rax, (%rsp) ## 8-byte Spill
+; CHECK-NEXT:    movl {{[0-9]+}}(%rsp), %ebp
+; CHECK-NEXT:    movl {{[0-9]+}}(%rsp), %r13d
+; CHECK-NEXT:    movl {{[0-9]+}}(%rsp), %r12d
+; CHECK-NEXT:    movl {{[0-9]+}}(%rsp), %r15d
+; CHECK-NEXT:    movl {{[0-9]+}}(%rsp), %r14d
+; CHECK-NEXT:    movl {{[0-9]+}}(%rsp), %ebx
+; CHECK-NEXT:    movl {{[0-9]+}}(%rsp), %r11d
+; CHECK-NEXT:    movl {{[0-9]+}}(%rsp), %r8d
+; CHECK-NEXT:    movl {{[0-9]+}}(%rsp), %eax
+; CHECK-NEXT:    callq _bar ## 72-byte Folded Reload
+; CHECK-NEXT:  Ltmp9:
+; CHECK-NEXT:    addq $72, %rsp
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    popq %r12
+; CHECK-NEXT:    popq %r13
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    retq
+entry:
+  %a8 = trunc i32 %a to i8
+  %b8 = trunc i32 %b to i8
+  %c8 = trunc i32 %c to i8
+  %d8 = trunc i32 %d to i8
+  %e16 = trunc i32 %e to i16
+  %f16 = trunc i32 %f to i16
+  %g16 = trunc i32 %g to i16
+  %h16 = trunc i32 %h to i16
+  %i64 = zext i32 %i to i64
+  %j64 = zext i32 %j to i64
+  %k64 = zext i32 %k to i64
+  %l64 = zext i32 %l to i64
+  %m64 = zext i32 %m to i64
+  %n64 = zext i32 %n to i64
+  %o64 = zext i32 %o to i64
+  %p64 = zext i32 %p to i64
+  %q64 = zext i32 %q to i64
+  %r64 = zext i32 %r to i64
+  %s64 = zext i32 %s to i64
+  %t64 = zext i32 %t to i64
+  %u64 = zext i32 %u to i64
+  %v64 = zext i32 %v to i64
+  %w64 = zext i32 %w to i64
+  %x64 = zext i32 %x to i64
+  %y64 = zext i32 %y to i64
+  %z64 = zext i32 %z to i64
+  %statepoint_token1 = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @bar, i32 0, i32 2, i32 0, i32 26, i8 %a8, i8 %b8, i8 %c8, i8 %d8, i16 %e16, i16 %f16, i16 %g16, i16 %h16, i64 %i64, i64 %j64, i64 %k64, i64 %l64, i64 %m64, i64 %n64, i64 %o64, i64 %p64, i64 %q64, i64 %r64, i64 %s64, i64 %t64, i64 %u64, i64 %v64, i64 %w64, i64 %x64, i64 %y64, i64 %z64)
+  ret void
+}
+
+; Test perfect forwarding of argument registers and stack slots to the
+; deopt bundle uses
+define void @test9(i32 %a, i32 %b, i32 %c, i32 %d, i32 %e, i32 %f, i32 %g, i32 %h, i32 %i, i32 %j, i32 %k, i32 %l, i32 %m, i32 %n, i32 %o, i32 %p, i32 %q, i32 %r, i32 %s, i32 %t, i32 %u, i32 %v, i32 %w, i32 %x, i32 %y, i32 %z) gc "statepoint-example" {
+; CHECK-LABEL: test9:
+; CHECK:       ## %bb.0: ## %entry
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    callq _bar
+; CHECK-NEXT:  Ltmp10:
+; CHECK-NEXT:    popq %rax
+; CHECK-NEXT:    retq
+
+entry:
+  %statepoint_token1 = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @bar, i32 0, i32 2, i32 0, i32 26, i32 %a, i32 %b, i32 %c, i32 %d, i32 %e, i32 %f, i32 %g, i32 %h, i32 %i, i32 %j, i32 %k, i32 %l, i32 %m, i32 %n, i32 %o, i32 %p, i32 %q, i32 %r, i32 %s, i32 %t, i32 %u, i32 %v, i32 %w, i32 %x, i32 %y, i32 %z)
+  ret void
+}
+
+; Test enough folding of argument slots when we have one call which clobbers
+; registers before a second which needs them - i.e. we must do something with
+; arguments originally passed in registers
+define void @test10(i32 %a, i32 %b, i32 %c, i32 %d, i32 %e, i32 %f, i32 %g, i32 %h, i32 %i, i32 %j, i32 %k, i32 %l, i32 %m, i32 %n, i32 %o, i32 %p, i32 %q, i32 %r, i32 %s, i32 %t, i32 %u, i32 %v, i32 %w, i32 %x, i32 %y, i32 %z) gc "statepoint-example" {
+; FIXME (minor): It would be better to just spill (and fold reload) for
+; argument registers then spill and fill all the CSRs.
+; CHECK-LABEL: test10:
+; CHECK:       ## %bb.0: ## %entry
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    pushq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    pushq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 56
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    .cfi_def_cfa_offset 64
+; CHECK-NEXT:    .cfi_offset %rbx, -56
+; CHECK-NEXT:    .cfi_offset %r12, -48
+; CHECK-NEXT:    .cfi_offset %r13, -40
+; CHECK-NEXT:    .cfi_offset %r14, -32
+; CHECK-NEXT:    .cfi_offset %r15, -24
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    movl %r9d, %r15d
+; CHECK-NEXT:    movl %r8d, %r14d
+; CHECK-NEXT:    movl %ecx, %r12d
+; CHECK-NEXT:    movl %edx, %r13d
+; CHECK-NEXT:    movl %esi, %ebx
+; CHECK-NEXT:    movl %edi, %ebp
+; CHECK-NEXT:    callq _bar
+; CHECK-NEXT:  Ltmp11:
+; CHECK-NEXT:    callq _bar
+; CHECK-NEXT:  Ltmp12:
+; CHECK-NEXT:    addq $8, %rsp
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    popq %r12
+; CHECK-NEXT:    popq %r13
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    retq
+
+entry:
+  %statepoint_token1 = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @bar, i32 0, i32 2, i32 0, i32 26, i32 %a, i32 %b, i32 %c, i32 %d, i32 %e, i32 %f, i32 %g, i32 %h, i32 %i, i32 %j, i32 %k, i32 %l, i32 %m, i32 %n, i32 %o, i32 %p, i32 %q, i32 %r, i32 %s, i32 %t, i32 %u, i32 %v, i32 %w, i32 %x, i32 %y, i32 %z)
+  %statepoint_token2 = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @bar, i32 0, i32 2, i32 0, i32 26, i32 %a, i32 %b, i32 %c, i32 %d, i32 %e, i32 %f, i32 %g, i32 %h, i32 %i, i32 %j, i32 %k, i32 %l, i32 %m, i32 %n, i32 %o, i32 %p, i32 %q, i32 %r, i32 %s, i32 %t, i32 %u, i32 %v, i32 %w, i32 %x, i32 %y, i32 %z)
+  ret void
+}
+
+; Check that we can remat some uses of a def despite not remating before the
+; statepoint user.
+define i64 @test11(i32 %a, i32 %b, i32 %c, i32 %d, i32 %e, i32 %f, i32 %g, i32 %h, i32 %i, i32 %j, i32 %k, i32 %l, i32 %m, i32 %n, i32 %o, i32 %p, i32 %q, i32 %r, i32 %s, i32 %t, i32 %u, i32 %v, i32 %w, i32 %x, i32 %y, i32 %z) gc "statepoint-example" {
+; FIXME: The codegen for this is correct, but horrible.  Lots of room for
+; improvement if we so desire.
+; CHECK-LABEL: test11:
+; CHECK:       ## %bb.0: ## %entry
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    .cfi_def_cfa_offset 24
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    pushq %r13
+; CHECK-NEXT:    .cfi_def_cfa_offset 40
+; CHECK-NEXT:    pushq %r12
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    .cfi_def_cfa_offset 56
+; CHECK-NEXT:    subq $168, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 224
+; CHECK-NEXT:    .cfi_offset %rbx, -56
+; CHECK-NEXT:    .cfi_offset %r12, -48
+; CHECK-NEXT:    .cfi_offset %r13, -40
+; CHECK-NEXT:    .cfi_offset %r14, -32
+; CHECK-NEXT:    .cfi_offset %r15, -24
+; CHECK-NEXT:    .cfi_offset %rbp, -16
+; CHECK-NEXT:    movl {{[0-9]+}}(%rsp), %eax
+; CHECK-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) ## 8-byte Spill
+; CHECK-NEXT:    movl {{[0-9]+}}(%rsp), %eax
+; CHECK-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) ## 8-byte Spill
+; CHECK-NEXT:    movl {{[0-9]+}}(%rsp), %eax
+; CHECK-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) ## 8-byte Spill
+; CHECK-NEXT:    movl {{[0-9]+}}(%rsp), %eax
+; CHECK-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) ## 8-byte Spill
+; CHECK-NEXT:    movl {{[0-9]+}}(%rsp), %eax
+; CHECK-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) ## 8-byte Spill
+; CHECK-NEXT:    movl {{[0-9]+}}(%rsp), %eax
+; CHECK-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) ## 8-byte Spill
+; CHECK-NEXT:    movl {{[0-9]+}}(%rsp), %eax
+; CHECK-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) ## 8-byte Spill
+; CHECK-NEXT:    movl {{[0-9]+}}(%rsp), %eax
+; CHECK-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) ## 8-byte Spill
+; CHECK-NEXT:    movl {{[0-9]+}}(%rsp), %eax
+; CHECK-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) ## 8-byte Spill
+; CHECK-NEXT:    movl %edi, %ebx
+; CHECK-NEXT:    movl %esi, %r15d
+; CHECK-NEXT:    movl %edx, %r12d
+; CHECK-NEXT:    movl %ecx, %r13d
+; CHECK-NEXT:    movl %r8d, %ebp
+; CHECK-NEXT:    movl %r9d, %r14d
+; CHECK-NEXT:    movl {{[0-9]+}}(%rsp), %eax
+; CHECK-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) ## 8-byte Spill
+; CHECK-NEXT:    movl {{[0-9]+}}(%rsp), %eax
+; CHECK-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) ## 8-byte Spill
+; CHECK-NEXT:    movl {{[0-9]+}}(%rsp), %eax
+; CHECK-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) ## 8-byte Spill
+; CHECK-NEXT:    movl {{[0-9]+}}(%rsp), %eax
+; CHECK-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) ## 8-byte Spill
+; CHECK-NEXT:    movl {{[0-9]+}}(%rsp), %eax
+; CHECK-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) ## 8-byte Spill
+; CHECK-NEXT:    movl {{[0-9]+}}(%rsp), %eax
+; CHECK-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) ## 8-byte Spill
+; CHECK-NEXT:    movl {{[0-9]+}}(%rsp), %eax
+; CHECK-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) ## 8-byte Spill
+; CHECK-NEXT:    movl {{[0-9]+}}(%rsp), %eax
+; CHECK-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) ## 8-byte Spill
+; CHECK-NEXT:    movl {{[0-9]+}}(%rsp), %eax
+; CHECK-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) ## 8-byte Spill
+; CHECK-NEXT:    movl {{[0-9]+}}(%rsp), %eax
+; CHECK-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) ## 8-byte Spill
+; CHECK-NEXT:    movl {{[0-9]+}}(%rsp), %eax
+; CHECK-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) ## 8-byte Spill
+; CHECK-NEXT:    callq _bar ## 160-byte Folded Reload
+; CHECK-NEXT:  Ltmp13:
+; CHECK-NEXT:    addq %r15, %rbx
+; CHECK-NEXT:    addq %r12, %rbx
+; CHECK-NEXT:    addq %r13, %rbx
+; CHECK-NEXT:    addq %rbp, %rbx
+; CHECK-NEXT:    addq %r14, %rbx
+; CHECK-NEXT:    movl {{[0-9]+}}(%rsp), %eax
+; CHECK-NEXT:    addq %rax, %rbx
+; CHECK-NEXT:    movl {{[0-9]+}}(%rsp), %eax
+; CHECK-NEXT:    addq %rax, %rbx
+; CHECK-NEXT:    movl {{[0-9]+}}(%rsp), %eax
+; CHECK-NEXT:    addq %rax, %rbx
+; CHECK-NEXT:    movl {{[0-9]+}}(%rsp), %eax
+; CHECK-NEXT:    addq %rax, %rbx
+; CHECK-NEXT:    movl {{[0-9]+}}(%rsp), %eax
+; CHECK-NEXT:    addq %rax, %rbx
+; CHECK-NEXT:    movl {{[0-9]+}}(%rsp), %eax
+; CHECK-NEXT:    addq %rax, %rbx
+; CHECK-NEXT:    movl {{[0-9]+}}(%rsp), %eax
+; CHECK-NEXT:    addq %rax, %rbx
+; CHECK-NEXT:    movl {{[0-9]+}}(%rsp), %eax
+; CHECK-NEXT:    addq %rax, %rbx
+; CHECK-NEXT:    movl {{[0-9]+}}(%rsp), %eax
+; CHECK-NEXT:    addq %rax, %rbx
+; CHECK-NEXT:    movl {{[0-9]+}}(%rsp), %eax
+; CHECK-NEXT:    addq %rax, %rbx
+; CHECK-NEXT:    movl {{[0-9]+}}(%rsp), %eax
+; CHECK-NEXT:    addq %rax, %rbx
+; CHECK-NEXT:    movl {{[0-9]+}}(%rsp), %eax
+; CHECK-NEXT:    addq %rax, %rbx
+; CHECK-NEXT:    movl {{[0-9]+}}(%rsp), %eax
+; CHECK-NEXT:    addq %rax, %rbx
+; CHECK-NEXT:    movl {{[0-9]+}}(%rsp), %eax
+; CHECK-NEXT:    addq %rax, %rbx
+; CHECK-NEXT:    movl {{[0-9]+}}(%rsp), %eax
+; CHECK-NEXT:    addq %rax, %rbx
+; CHECK-NEXT:    movl {{[0-9]+}}(%rsp), %eax
+; CHECK-NEXT:    addq %rax, %rbx
+; CHECK-NEXT:    movl {{[0-9]+}}(%rsp), %eax
+; CHECK-NEXT:    addq %rax, %rbx
+; CHECK-NEXT:    movl {{[0-9]+}}(%rsp), %eax
+; CHECK-NEXT:    addq %rax, %rbx
+; CHECK-NEXT:    movl {{[0-9]+}}(%rsp), %eax
+; CHECK-NEXT:    addq %rax, %rbx
+; CHECK-NEXT:    movl {{[0-9]+}}(%rsp), %eax
+; CHECK-NEXT:    addq %rax, %rbx
+; CHECK-NEXT:    movq %rbx, %rax
+; CHECK-NEXT:    addq $168, %rsp
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    popq %r12
+; CHECK-NEXT:    popq %r13
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    retq
+
+entry:
+  %a64 = zext i32 %a to i64
+  %b64 = zext i32 %b to i64
+  %c64 = zext i32 %c to i64
+  %d64 = zext i32 %d to i64
+  %e64 = zext i32 %e to i64
+  %f64 = zext i32 %f to i64
+  %g64 = zext i32 %g to i64
+  %h64 = zext i32 %h to i64
+  %i64 = zext i32 %i to i64
+  %j64 = zext i32 %j to i64
+  %k64 = zext i32 %k to i64
+  %l64 = zext i32 %l to i64
+  %m64 = zext i32 %m to i64
+  %n64 = zext i32 %n to i64
+  %o64 = zext i32 %o to i64
+  %p64 = zext i32 %p to i64
+  %q64 = zext i32 %q to i64
+  %r64 = zext i32 %r to i64
+  %s64 = zext i32 %s to i64
+  %t64 = zext i32 %t to i64
+  %u64 = zext i32 %u to i64
+  %v64 = zext i32 %v to i64
+  %w64 = zext i32 %w to i64
+  %x64 = zext i32 %x to i64
+  %y64 = zext i32 %y to i64
+  %z64 = zext i32 %z to i64
+  call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @bar, i32 0, i32 2, i64 0, i64 26, i64 %a64, i64 %b64, i64 %c64, i64 %d64, i64 %e64, i64 %f64, i64 %g64, i64 %h64, i64 %i64, i64 %j64, i64 %k64, i64 %l64, i64 %m64, i64 %n64, i64 %o64, i64 %p64, i64 %q64, i64 %r64, i64 %s64, i64 %t64, i64 %u64, i64 %v64, i64 %w64, i64 %x64, i64 %y64, i64 %z64)
+  %addab = add i64 %a64, %b64
+  %addc = add i64 %addab, %c64
+  %addd = add i64 %addc, %d64
+  %adde = add i64 %addd, %e64
+  %addf = add i64 %adde, %f64
+  %addg = add i64 %addf, %g64
+  %addh = add i64 %addg, %h64
+  %addi = add i64 %addh, %i64
+  %addj = add i64 %addi, %j64
+  %addk = add i64 %addj, %k64
+  %addl = add i64 %addk, %l64
+  %addm = add i64 %addl, %m64
+  %addn = add i64 %addm, %n64
+  %addo = add i64 %addn, %o64
+  %addp = add i64 %addo, %p64
+  %addq = add i64 %addp, %q64
+  %addr = add i64 %addq, %r64
+  %adds = add i64 %addr, %s64
+  %addt = add i64 %adds, %t64
+  %addu = add i64 %addt, %u64
+  %addv = add i64 %addu, %v64
+  %addw = add i64 %addv, %w64
+  %addx = add i64 %addw, %x64
+  %addy = add i64 %addx, %y64
+  %addz = add i64 %addy, %z64
+  ret i64 %addz
+}
+
+; Demonstrate address of a function (w/o spilling)
+define void @addr_func() gc "statepoint-example" {
+; CHECK-LABEL: addr_func:
+; CHECK:       ## %bb.0: ## %entry
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    movq _bar@{{.*}}(%rip), %rax
+; CHECK-NEXT:    callq _bar
+; CHECK-NEXT:  Ltmp14:
+; CHECK-NEXT:    popq %rax
+; CHECK-NEXT:    retq
+entry:
+  %statepoint_token1 = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @bar, i32 0, i32 2, i64 0, i64 3, void ()* @bar, void ()* @bar, void ()* @bar)
+  ret void
+}
+
+; Demonstrate address of a global (w/o spilling)
+@G = external global i32
+define void @addr_global() gc "statepoint-example" {
+; CHECK-LABEL: addr_global:
+; CHECK:       ## %bb.0: ## %entry
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    movq _G@{{.*}}(%rip), %rax
+; CHECK-NEXT:    callq _bar
+; CHECK-NEXT:  Ltmp15:
+; CHECK-NEXT:    popq %rax
+; CHECK-NEXT:    retq
+entry:
+  %statepoint_token1 = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @bar, i32 0, i32 2, i64 0, i64 3, i32* @G, i32* @G, i32* @G)
+  ret void
+}
+
+define void @addr_alloca(i32 %v) gc "statepoint-example" {
+; CHECK-LABEL: addr_alloca:
+; CHECK:       ## %bb.0: ## %entry
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    movl %edi, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    callq _bar
+; CHECK-NEXT:  Ltmp16:
+; CHECK-NEXT:    popq %rax
+; CHECK-NEXT:    retq
+entry:
+  %a = alloca i32
+  store i32 %v, i32* %a
+  %statepoint_token1 = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @bar, i32 0, i32 2, i64 0, i64 3, i32* %a, i32* %a, i32* %a)
+  ret void
+}
+
 
 ; CHECK: Ltmp0-_test1
 ; CHECK:      .byte	1
@@ -167,6 +672,5 @@ entry:
 declare token @llvm.experimental.gc.statepoint.p0f_isVoidf(i64, i32, void ()*, i32, i32, ...)
 declare i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token, i32, i32)
 
-
 attributes #0 = { "deopt-lowering"="live-in" }
 attributes #1 = { "deopt-lowering"="live-through" }
diff --git a/test/CodeGen/X86/statepoint-uniqueing.ll b/test/CodeGen/X86/statepoint-uniqueing.ll
index a5fa1f2d99c9a3034079376c7e5587cc91f45294..ae7c076ab12e8199a49daa2d71b91fccc3f4026b 100644
--- a/test/CodeGen/X86/statepoint-uniqueing.ll
+++ b/test/CodeGen/X86/statepoint-uniqueing.ll
@@ -1,31 +1,94 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -verify-machineinstrs < %s | FileCheck %s
 
-; Checks for a crash we had when two gc.relocate calls would
-; relocating identical values
-
 target datalayout = "e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-pc-linux-gnu"
 
+declare void @use(...)
 declare void @f()
 declare token @llvm.experimental.gc.statepoint.p0f_isVoidf(i64, i32, void ()*, i32, i32, ...)
 declare i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token, i32, i32) #3
 declare i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token, i32, i32) #3
 
+;; Two gc.relocates of the same input, should require only a single spill/fill
 define void @test_gcrelocate_uniqueing(i32 addrspace(1)* %ptr) gc "statepoint-example" {
-; CHECK-LABEL: test_gcrelocate_uniqueing
+; CHECK-LABEL: test_gcrelocate_uniqueing:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    subq $24, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    movq %rdi, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    callq f
+; CHECK-NEXT:  .Ltmp0:
+; CHECK-NEXT:    movq {{[0-9]+}}(%rsp), %rdi
+; CHECK-NEXT:    movq %rdi, %rsi
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    callq use
+; CHECK-NEXT:    addq $24, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %tok = tail call token (i64, i32, void ()*, i32, i32, ...)
       @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @f, i32 0, i32 0, i32 0, i32 2, i32 addrspace(1)* %ptr, i32 undef, i32 addrspace(1)* %ptr, i32 addrspace(1)* %ptr)
   %a = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %tok, i32 9, i32 9)
   %b = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %tok, i32 10, i32 10)
+  call void (...) @use(i32 addrspace(1)* %a, i32 addrspace(1)* %b)
   ret void
 }
 
+;; Two gc.relocates of a bitcasted pointer should only require a single spill/fill
 define void @test_gcptr_uniqueing(i32 addrspace(1)* %ptr) gc "statepoint-example" {
-; CHECK-LABEL: test_gcptr_uniqueing
+; CHECK-LABEL: test_gcptr_uniqueing:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    subq $24, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    movq %rdi, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    callq f
+; CHECK-NEXT:  .Ltmp1:
+; CHECK-NEXT:    movq {{[0-9]+}}(%rsp), %rdi
+; CHECK-NEXT:    movq %rdi, %rsi
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    callq use
+; CHECK-NEXT:    addq $24, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
   %ptr2 = bitcast i32 addrspace(1)* %ptr to i8 addrspace(1)*
   %tok = tail call token (i64, i32, void ()*, i32, i32, ...)
       @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @f, i32 0, i32 0, i32 0, i32 2, i32 addrspace(1)* %ptr, i32 undef, i32 addrspace(1)* %ptr, i8 addrspace(1)* %ptr2)
   %a = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %tok, i32 9, i32 9)
   %b = call i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token %tok, i32 10, i32 10)
+  call void (...) @use(i32 addrspace(1)* %a, i8 addrspace(1)* %b)
+  ret void
+}
+
+;; A GC value is not dead, and does need to be spill (but not filed) if
+;; that same value is also in the deopt list.
+define void @test_deopt_use(i32 addrspace(1)* %ptr) gc "statepoint-example" {
+; CHECK-LABEL: test_deopt_use:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    subq $24, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    movq %rdi, {{[0-9]+}}(%rsp)
+; CHECK-NEXT:    callq f
+; CHECK-NEXT:  .Ltmp2:
+; CHECK-NEXT:    addq $24, %rsp
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
+  tail call token (i64, i32, void ()*, i32, i32, ...)
+      @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @f, i32 0, i32 0, i32 0, i32 2, i32 addrspace(1)* %ptr, i32 undef, i32 addrspace(1)* %ptr)
+  ret void
+}
+
+;; A GC value which is truely unused does not need to spilled or filled.
+define void @test_dse(i32 addrspace(1)* %ptr) gc "statepoint-example" {
+; CHECK-LABEL: test_dse:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    callq f
+; CHECK-NEXT:  .Ltmp3:
+; CHECK-NEXT:    popq %rax
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
+  tail call token (i64, i32, void ()*, i32, i32, ...)
+      @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 0, i32 0, void ()* @f, i32 0, i32 0, i32 0, i32 2, i32 undef, i32 undef, i32 addrspace(1)* %ptr)
   ret void
 }
diff --git a/test/CodeGen/X86/sub-with-overflow.ll b/test/CodeGen/X86/sub-with-overflow.ll
index fa00d6f61c70b4cc409893cde45fe5bc26595b74..0bcf2d8a565f19f3a680b0487eb5ad04731bfe49 100644
--- a/test/CodeGen/X86/sub-with-overflow.ll
+++ b/test/CodeGen/X86/sub-with-overflow.ll
@@ -1,9 +1,29 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=i686-linux | FileCheck %s
 
 @ok = internal constant [4 x i8] c"%d\0A\00"
 @no = internal constant [4 x i8] c"no\0A\00"
 
 define i1 @func1(i32 %v1, i32 %v2) nounwind {
+; CHECK-LABEL: func1:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    subl $12, %esp
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    subl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    jno .LBB0_1
+; CHECK-NEXT:  # %bb.2: # %overflow
+; CHECK-NEXT:    movl $no, (%esp)
+; CHECK-NEXT:    calll printf
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    addl $12, %esp
+; CHECK-NEXT:    retl
+; CHECK-NEXT:  .LBB0_1: # %normal
+; CHECK-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; CHECK-NEXT:    movl $ok, (%esp)
+; CHECK-NEXT:    calll printf
+; CHECK-NEXT:    movb $1, %al
+; CHECK-NEXT:    addl $12, %esp
+; CHECK-NEXT:    retl
 entry:
   %t = call {i32, i1} @llvm.ssub.with.overflow.i32(i32 %v1, i32 %v2)
   %sum = extractvalue {i32, i1} %t, 0
@@ -18,12 +38,28 @@ overflow:
   %t2 = tail call i32 (i8*, ...) @printf( i8* getelementptr ([4 x i8], [4 x i8]* @no, i32 0, i32 0) ) nounwind
   ret i1 false
 
-; CHECK-LABEL: func1:
-; CHECK: subl 20(%esp)
-; CHECK-NEXT: jno
 }
 
 define i1 @func2(i32 %v1, i32 %v2) nounwind {
+; CHECK-LABEL: func2:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    subl $12, %esp
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    subl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    jae .LBB1_1
+; CHECK-NEXT:  # %bb.2: # %carry
+; CHECK-NEXT:    movl $no, (%esp)
+; CHECK-NEXT:    calll printf
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    addl $12, %esp
+; CHECK-NEXT:    retl
+; CHECK-NEXT:  .LBB1_1: # %normal
+; CHECK-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; CHECK-NEXT:    movl $ok, (%esp)
+; CHECK-NEXT:    calll printf
+; CHECK-NEXT:    movb $1, %al
+; CHECK-NEXT:    addl $12, %esp
+; CHECK-NEXT:    retl
 entry:
   %t = call {i32, i1} @llvm.usub.with.overflow.i32(i32 %v1, i32 %v2)
   %sum = extractvalue {i32, i1} %t, 0
@@ -38,9 +74,6 @@ carry:
   %t2 = tail call i32 (i8*, ...) @printf( i8* getelementptr ([4 x i8], [4 x i8]* @no, i32 0, i32 0) ) nounwind
   ret i1 false
 
-; CHECK-LABEL: func2:
-; CHECK: subl 20(%esp)
-; CHECK-NEXT: jae
 }
 
 declare i32 @printf(i8*, ...) nounwind
@@ -48,12 +81,14 @@ declare {i32, i1} @llvm.ssub.with.overflow.i32(i32, i32)
 declare {i32, i1} @llvm.usub.with.overflow.i32(i32, i32)
 
 define i1 @func3(i32 %x) nounwind {
+; CHECK-LABEL: func3:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    cmpl $1, {{[0-9]+}}(%esp)
+; CHECK-NEXT:    seto %al
+; CHECK-NEXT:    retl
 entry:
   %t = call {i32, i1} @llvm.ssub.with.overflow.i32(i32 %x, i32 1)
   %obit = extractvalue {i32, i1} %t, 1
   ret i1 %obit
 
-; CHECK-LABEL: func3:
-; CHECK: decl
-; CHECK-NEXT: seto
 }
diff --git a/test/CodeGen/X86/subvector-broadcast.ll b/test/CodeGen/X86/subvector-broadcast.ll
index a05288ac0318c1a2108a09deb884d3a833115ced..0e66f60dbc5544458f35a634282bdf90679930ad 100644
--- a/test/CodeGen/X86/subvector-broadcast.ll
+++ b/test/CodeGen/X86/subvector-broadcast.ll
@@ -1,14 +1,14 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+avx | FileCheck %s --check-prefix=X32 --check-prefix=X32-AVX --check-prefix=X32-AVX1
-; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefix=X32 --check-prefix=X32-AVX --check-prefix=X32-AVX2
-; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+avx512vl | FileCheck %s --check-prefix=X32 --check-prefix=X32-AVX512 --check-prefix=X32-AVX512F
-; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+avx512bw,+avx512vl | FileCheck %s --check-prefix=X32 --check-prefix=X32-AVX512 --check-prefix=X32-AVX512BW
-; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+avx512dq,+avx512vl | FileCheck %s --check-prefix=X32 --check-prefix=X32-AVX512 --check-prefix=X32-AVX512DQ
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefix=X64 --check-prefix=X64-AVX --check-prefix=X64-AVX1
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefix=X64 --check-prefix=X64-AVX --check-prefix=X64-AVX2
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512vl | FileCheck %s --check-prefix=X64 --check-prefix=X64-AVX512 --check-prefix=X64-AVX512F
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw,+avx512vl | FileCheck %s --check-prefix=X64 --check-prefix=X64-AVX512 --check-prefix=X64-AVX512BW
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512dq,+avx512vl | FileCheck %s --check-prefix=X64 --check-prefix=X64-AVX512 --check-prefix=X64-AVX512DQ
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+avx | FileCheck %s --check-prefixes=X32,X32-AVX,X32-AVX1
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefixes=X32,X32-AVX,X32-AVX2
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+avx512vl | FileCheck %s --check-prefixes=X32,X32-AVX512,X32-AVX512F
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+avx512bw,+avx512vl | FileCheck %s --check-prefixes=X32,X32-AVX512,X32-AVX512BW
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+avx512dq,+avx512vl | FileCheck %s --check-prefixes=X32,X32-AVX512,X32-AVX512DQ
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefixes=X64,X64-AVX,X64-AVX1
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefixes=X64,X64-AVX,X64-AVX2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512vl | FileCheck %s --check-prefixes=X64,X64-AVX512,X64-AVX512F
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw,+avx512vl | FileCheck %s --check-prefixes=X64,X64-AVX512,X64-AVX512BW
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512dq,+avx512vl | FileCheck %s --check-prefixes=X64,X64-AVX512,X64-AVX512DQ
 
 ;
 ; Subvector Load + Broadcast
@@ -1550,3 +1550,236 @@ define <64 x i8> @reg_broadcast_32i8_64i8(<32 x i8> %a0) nounwind {
  %1 = shufflevector <32 x i8> %a0, <32 x i8> undef, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
  ret <64 x i8> %1
 }
+
+;
+; PR34394
+;
+
+define <4 x i32> @test_2xi32_to_4xi32_mem(<2 x i32>* %vp) {
+; X32-LABEL: test_2xi32_to_4xi32_mem:
+; X32:       # %bb.0:
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT:    vmovddup {{.*#+}} xmm0 = mem[0,0]
+; X32-NEXT:    retl
+;
+; X64-LABEL: test_2xi32_to_4xi32_mem:
+; X64:       # %bb.0:
+; X64-NEXT:    vmovddup {{.*#+}} xmm0 = mem[0,0]
+; X64-NEXT:    retq
+  %vec = load <2 x i32>, <2 x i32>* %vp
+  %res = shufflevector <2 x i32> %vec, <2 x i32> undef, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
+  ret <4 x i32> %res
+}
+
+define <8 x i32> @test_2xi32_to_8xi32_mem(<2 x i32>* %vp) {
+; X32-LABEL: test_2xi32_to_8xi32_mem:
+; X32:       # %bb.0:
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT:    vbroadcastsd (%eax), %ymm0
+; X32-NEXT:    retl
+;
+; X64-LABEL: test_2xi32_to_8xi32_mem:
+; X64:       # %bb.0:
+; X64-NEXT:    vbroadcastsd (%rdi), %ymm0
+; X64-NEXT:    retq
+  %vec = load <2 x i32>, <2 x i32>* %vp
+  %res = shufflevector <2 x i32> %vec, <2 x i32> undef, <8 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1>
+  ret <8 x i32> %res
+}
+
+define <16 x i32> @test_2xi32_to_16xi32_mem(<2 x i32>* %vp) {
+; X32-AVX-LABEL: test_2xi32_to_16xi32_mem:
+; X32-AVX:       # %bb.0:
+; X32-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-AVX-NEXT:    vbroadcastsd (%eax), %ymm0
+; X32-AVX-NEXT:    vmovaps %ymm0, %ymm1
+; X32-AVX-NEXT:    retl
+;
+; X32-AVX512-LABEL: test_2xi32_to_16xi32_mem:
+; X32-AVX512:       # %bb.0:
+; X32-AVX512-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-AVX512-NEXT:    vbroadcastsd (%eax), %zmm0
+; X32-AVX512-NEXT:    retl
+;
+; X64-AVX-LABEL: test_2xi32_to_16xi32_mem:
+; X64-AVX:       # %bb.0:
+; X64-AVX-NEXT:    vbroadcastsd (%rdi), %ymm0
+; X64-AVX-NEXT:    vmovaps %ymm0, %ymm1
+; X64-AVX-NEXT:    retq
+;
+; X64-AVX512-LABEL: test_2xi32_to_16xi32_mem:
+; X64-AVX512:       # %bb.0:
+; X64-AVX512-NEXT:    vbroadcastsd (%rdi), %zmm0
+; X64-AVX512-NEXT:    retq
+  %vec = load <2 x i32>, <2 x i32>* %vp
+  %res = shufflevector <2 x i32> %vec, <2 x i32> undef, <16 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1>
+  ret <16 x i32> %res
+}
+
+;
+; PR34041
+;
+
+define <4 x double> @broadcast_v4f64_f64_u000(double* %p) {
+; X32-LABEL: broadcast_v4f64_f64_u000:
+; X32:       # %bb.0:
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT:    vbroadcastsd (%eax), %ymm0
+; X32-NEXT:    retl
+;
+; X64-LABEL: broadcast_v4f64_f64_u000:
+; X64:       # %bb.0:
+; X64-NEXT:    vbroadcastsd (%rdi), %ymm0
+; X64-NEXT:    retq
+  %s = load double, double* %p
+  %vec = insertelement <2 x double> undef, double %s, i32 0
+  %res = shufflevector <2 x double> %vec, <2 x double> undef, <4 x i32> <i32 1, i32 0, i32 0, i32 0>
+  ret <4 x double> %res
+}
+
+define <4 x double> @broadcast_v4f64_v2f64_4u61(<2 x double>* %vp, <4 x double> %default) {
+; X32-LABEL: broadcast_v4f64_v2f64_4u61:
+; X32:       # %bb.0:
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT:    vinsertf128 $1, (%eax), %ymm0, %ymm1
+; X32-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
+; X32-NEXT:    retl
+;
+; X64-LABEL: broadcast_v4f64_v2f64_4u61:
+; X64:       # %bb.0:
+; X64-NEXT:    vinsertf128 $1, (%rdi), %ymm0, %ymm1
+; X64-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
+; X64-NEXT:    retq
+  %vec = load <2 x double>, <2 x double>* %vp
+  %shuf = shufflevector <2 x double> %vec, <2 x double> undef, <4 x i32> <i32 0, i32 3, i32 undef, i32 1>
+  %res = select <4 x i1> <i1 0, i1 1, i1 0, i1 1>, <4 x double> %shuf, <4 x double> %default
+  ret <4 x double> %res
+}
+
+define <8 x float> @broadcast_v8f32_v2f32_u1uu0uEu(<2 x float>* %vp, <8 x float> %default) {
+; X32-AVX1-LABEL: broadcast_v8f32_v2f32_u1uu0uEu:
+; X32-AVX1:       # %bb.0:
+; X32-AVX1-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-AVX1-NEXT:    vmovsd {{.*#+}} xmm1 = mem[0],zero
+; X32-AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm1, %ymm1
+; X32-AVX1-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,3],ymm1[4,5],ymm0[6,7]
+; X32-AVX1-NEXT:    retl
+;
+; X32-AVX2-LABEL: broadcast_v8f32_v2f32_u1uu0uEu:
+; X32-AVX2:       # %bb.0:
+; X32-AVX2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-AVX2-NEXT:    vmovsd {{.*#+}} xmm1 = mem[0],zero
+; X32-AVX2-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,3],ymm1[4,5],ymm0[6,7]
+; X32-AVX2-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[0,1,0,3]
+; X32-AVX2-NEXT:    retl
+;
+; X32-AVX512-LABEL: broadcast_v8f32_v2f32_u1uu0uEu:
+; X32-AVX512:       # %bb.0:
+; X32-AVX512-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-AVX512-NEXT:    vmovsd {{.*#+}} xmm1 = mem[0],zero
+; X32-AVX512-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,3],ymm1[4,5],ymm0[6,7]
+; X32-AVX512-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[0,1,0,3]
+; X32-AVX512-NEXT:    retl
+;
+; X64-AVX1-LABEL: broadcast_v8f32_v2f32_u1uu0uEu:
+; X64-AVX1:       # %bb.0:
+; X64-AVX1-NEXT:    vmovsd {{.*#+}} xmm1 = mem[0],zero
+; X64-AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm1, %ymm1
+; X64-AVX1-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,3],ymm1[4,5],ymm0[6,7]
+; X64-AVX1-NEXT:    retq
+;
+; X64-AVX2-LABEL: broadcast_v8f32_v2f32_u1uu0uEu:
+; X64-AVX2:       # %bb.0:
+; X64-AVX2-NEXT:    vmovsd {{.*#+}} xmm1 = mem[0],zero
+; X64-AVX2-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,3],ymm1[4,5],ymm0[6,7]
+; X64-AVX2-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[0,1,0,3]
+; X64-AVX2-NEXT:    retq
+;
+; X64-AVX512-LABEL: broadcast_v8f32_v2f32_u1uu0uEu:
+; X64-AVX512:       # %bb.0:
+; X64-AVX512-NEXT:    vmovsd {{.*#+}} xmm1 = mem[0],zero
+; X64-AVX512-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,3],ymm1[4,5],ymm0[6,7]
+; X64-AVX512-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[0,1,0,3]
+; X64-AVX512-NEXT:    retq
+  %vec = load <2 x float>, <2 x float>* %vp
+  %shuf = shufflevector <2 x float> %vec, <2 x float> undef, <8 x i32> <i32 undef, i32 1, i32 undef, i32 undef, i32 0, i32 2, i32 3, i32 undef>
+  %res = select <8 x i1> <i1 1, i1 1, i1 1, i1 1, i1 1, i1 1, i1 0, i1 1>, <8 x float> %shuf, <8 x float> %default
+  ret <8 x float> %res
+}
+
+define <8 x double> @broadcast_v8f64_v2f64_u1u10101(<2 x double>* %vp) {
+; X32-AVX1-LABEL: broadcast_v8f64_v2f64_u1u10101:
+; X32-AVX1:       # %bb.0:
+; X32-AVX1-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-AVX1-NEXT:    vbroadcastf128 {{.*#+}} ymm0 = mem[0,1,0,1]
+; X32-AVX1-NEXT:    vmovaps %ymm0, %ymm1
+; X32-AVX1-NEXT:    retl
+;
+; X32-AVX2-LABEL: broadcast_v8f64_v2f64_u1u10101:
+; X32-AVX2:       # %bb.0:
+; X32-AVX2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-AVX2-NEXT:    vmovaps (%eax), %xmm0
+; X32-AVX2-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm1
+; X32-AVX2-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[0,1,2,1]
+; X32-AVX2-NEXT:    retl
+;
+; X32-AVX512-LABEL: broadcast_v8f64_v2f64_u1u10101:
+; X32-AVX512:       # %bb.0:
+; X32-AVX512-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-AVX512-NEXT:    vbroadcastf32x4 {{.*#+}} zmm0 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; X32-AVX512-NEXT:    retl
+;
+; X64-AVX1-LABEL: broadcast_v8f64_v2f64_u1u10101:
+; X64-AVX1:       # %bb.0:
+; X64-AVX1-NEXT:    vbroadcastf128 {{.*#+}} ymm0 = mem[0,1,0,1]
+; X64-AVX1-NEXT:    vmovaps %ymm0, %ymm1
+; X64-AVX1-NEXT:    retq
+;
+; X64-AVX2-LABEL: broadcast_v8f64_v2f64_u1u10101:
+; X64-AVX2:       # %bb.0:
+; X64-AVX2-NEXT:    vmovaps (%rdi), %xmm0
+; X64-AVX2-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm1
+; X64-AVX2-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[0,1,2,1]
+; X64-AVX2-NEXT:    retq
+;
+; X64-AVX512-LABEL: broadcast_v8f64_v2f64_u1u10101:
+; X64-AVX512:       # %bb.0:
+; X64-AVX512-NEXT:    vbroadcastf32x4 {{.*#+}} zmm0 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; X64-AVX512-NEXT:    retq
+  %vec = load <2 x double>, <2 x double>* %vp
+  %res = shufflevector <2 x double> %vec, <2 x double> undef, <8 x i32> <i32 3, i32 1, i32 undef, i32 1, i32 0, i32 1, i32 0, i32 1>
+  ret <8 x double> %res
+}
+
+define <8 x double> @broadcast_v8f64_v2f64_0uuu0101(<2 x double>* %vp) {
+; X32-AVX-LABEL: broadcast_v8f64_v2f64_0uuu0101:
+; X32-AVX:       # %bb.0:
+; X32-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-AVX-NEXT:    vmovaps (%eax), %xmm0
+; X32-AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm1
+; X32-AVX-NEXT:    retl
+;
+; X32-AVX512-LABEL: broadcast_v8f64_v2f64_0uuu0101:
+; X32-AVX512:       # %bb.0:
+; X32-AVX512-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-AVX512-NEXT:    vmovaps (%eax), %xmm0
+; X32-AVX512-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm1
+; X32-AVX512-NEXT:    vinsertf64x4 $1, %ymm1, %zmm0, %zmm0
+; X32-AVX512-NEXT:    retl
+;
+; X64-AVX-LABEL: broadcast_v8f64_v2f64_0uuu0101:
+; X64-AVX:       # %bb.0:
+; X64-AVX-NEXT:    vmovaps (%rdi), %xmm0
+; X64-AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm1
+; X64-AVX-NEXT:    retq
+;
+; X64-AVX512-LABEL: broadcast_v8f64_v2f64_0uuu0101:
+; X64-AVX512:       # %bb.0:
+; X64-AVX512-NEXT:    vmovaps (%rdi), %xmm0
+; X64-AVX512-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm1
+; X64-AVX512-NEXT:    vinsertf64x4 $1, %ymm1, %zmm0, %zmm0
+; X64-AVX512-NEXT:    retq
+  %vec = load <2 x double>, <2 x double>* %vp
+  %res = shufflevector <2 x double> %vec, <2 x double> undef, <8 x i32> <i32 0, i32 undef, i32 undef, i32 undef, i32 0, i32 1, i32 0, i32 1>
+  ret <8 x double> %res
+}
diff --git a/test/CodeGen/X86/swap.ll b/test/CodeGen/X86/swap.ll
new file mode 100644
index 0000000000000000000000000000000000000000..e74ed248c60caeb3d5f4904c9161b849da4e4833
--- /dev/null
+++ b/test/CodeGen/X86/swap.ll
@@ -0,0 +1,211 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mcpu=haswell | FileCheck %s -check-prefix=NOAA
+; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mcpu=haswell -combiner-global-alias-analysis=1 | FileCheck %s -check-prefix=AA
+
+declare void @llvm.lifetime.start.p0i8(i64, i8* nocapture)
+declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture writeonly, i8* nocapture readonly, i64, i1)
+declare void @llvm.lifetime.end.p0i8(i64, i8* nocapture)
+
+%struct.S = type { [16 x i8] }
+
+define dso_local void @_Z4SwapP1SS0_(%struct.S* nocapture %a, %struct.S* nocapture %b) local_unnamed_addr {
+; NOAA-LABEL: _Z4SwapP1SS0_:
+; NOAA:       # %bb.0: # %entry
+; NOAA-NEXT:    vmovups (%rdi), %xmm0
+; NOAA-NEXT:    vmovaps %xmm0, -{{[0-9]+}}(%rsp)
+; NOAA-NEXT:    vmovups (%rsi), %xmm0
+; NOAA-NEXT:    vmovups %xmm0, (%rdi)
+; NOAA-NEXT:    vmovaps -{{[0-9]+}}(%rsp), %xmm0
+; NOAA-NEXT:    vmovups %xmm0, (%rsi)
+; NOAA-NEXT:    retq
+;
+; AA-LABEL: _Z4SwapP1SS0_:
+; AA:       # %bb.0: # %entry
+; AA-NEXT:    vmovups (%rdi), %xmm0
+; AA-NEXT:    vmovaps %xmm0, -{{[0-9]+}}(%rsp)
+; AA-NEXT:    vmovups (%rsi), %xmm1
+; AA-NEXT:    vmovups %xmm1, (%rdi)
+; AA-NEXT:    vmovups %xmm0, (%rsi)
+; AA-NEXT:    retq
+entry:
+  %tmp.sroa.0 = alloca [16 x i8], align 1
+  %tmp.sroa.0.0..sroa_idx6 = getelementptr inbounds [16 x i8], [16 x i8]* %tmp.sroa.0, i64 0, i64 0
+  call void @llvm.lifetime.start.p0i8(i64 16, i8* nonnull %tmp.sroa.0.0..sroa_idx6)
+  %tmp.sroa.0.0..sroa_idx1 = getelementptr inbounds %struct.S, %struct.S* %a, i64 0, i32 0, i64 0
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* nonnull align 1 %tmp.sroa.0.0..sroa_idx6, i8* align 1 %tmp.sroa.0.0..sroa_idx1, i64 16, i1 false)
+  %0 = getelementptr inbounds %struct.S, %struct.S* %b, i64 0, i32 0, i64 0
+  tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 1 %tmp.sroa.0.0..sroa_idx1, i8* align 1 %0, i64 16, i1 false), !tbaa.struct !2
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 1 %0, i8* nonnull align 1 %tmp.sroa.0.0..sroa_idx6, i64 16, i1 false)
+  call void @llvm.lifetime.end.p0i8(i64 16, i8* nonnull %tmp.sroa.0.0..sroa_idx6)
+  ret void
+}
+
+define dso_local void @onealloc_noreadback(i8* nocapture %a, i8* nocapture %b) local_unnamed_addr {
+; NOAA-LABEL: onealloc_noreadback:
+; NOAA:       # %bb.0: # %entry
+; NOAA-NEXT:    vmovups (%rdi), %xmm0
+; NOAA-NEXT:    vmovaps %xmm0, -{{[0-9]+}}(%rsp)
+; NOAA-NEXT:    vmovups (%rsi), %xmm0
+; NOAA-NEXT:    vmovaps %xmm0, -{{[0-9]+}}(%rsp)
+; NOAA-NEXT:    retq
+;
+; AA-LABEL: onealloc_noreadback:
+; AA:       # %bb.0: # %entry
+; AA-NEXT:    vmovups (%rdi), %xmm0
+; AA-NEXT:    vmovaps %xmm0, -{{[0-9]+}}(%rsp)
+; AA-NEXT:    vmovups (%rsi), %xmm0
+; AA-NEXT:    vmovaps %xmm0, -{{[0-9]+}}(%rsp)
+; AA-NEXT:    retq
+entry:
+  %alloc = alloca [16 x i8], i8 2, align 1
+  %part1 = getelementptr inbounds [16 x i8], [16 x i8]* %alloc, i64 0, i64 0
+  %part2 = getelementptr inbounds [16 x i8], [16 x i8]* %alloc, i64 1, i64 0
+  call void @llvm.lifetime.start.p0i8(i64 16, i8* nonnull %part1)
+  call void @llvm.lifetime.start.p0i8(i64 16, i8* nonnull %part2)
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* nonnull align 1 %part1, i8* align 1 %a, i64 16, i1 false)
+  tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* nonnull align 1 %part2, i8* align 1 %b, i64 16, i1 false)
+  call void @llvm.lifetime.end.p0i8(i64 16, i8* nonnull %part1)
+  call void @llvm.lifetime.end.p0i8(i64 16, i8* nonnull %part2)
+  ret void
+}
+
+
+define dso_local void @twoallocs_trivial(i8* nocapture %a, i8* nocapture %b) local_unnamed_addr {
+; NOAA-LABEL: twoallocs_trivial:
+; NOAA:       # %bb.0: # %entry
+; NOAA-NEXT:    vmovups (%rdi), %xmm0
+; NOAA-NEXT:    vmovaps %xmm0, -{{[0-9]+}}(%rsp)
+; NOAA-NEXT:    vmovups (%rsi), %xmm0
+; NOAA-NEXT:    vmovaps %xmm0, -{{[0-9]+}}(%rsp)
+; NOAA-NEXT:    retq
+;
+; AA-LABEL: twoallocs_trivial:
+; AA:       # %bb.0: # %entry
+; AA-NEXT:    vmovups (%rdi), %xmm0
+; AA-NEXT:    vmovaps %xmm0, -{{[0-9]+}}(%rsp)
+; AA-NEXT:    vmovups (%rsi), %xmm0
+; AA-NEXT:    vmovaps %xmm0, -{{[0-9]+}}(%rsp)
+; AA-NEXT:    retq
+entry:
+  %alloc1 = alloca [16 x i8], align 1
+  %alloc2 = alloca [16 x i8], align 1
+  %part1 = getelementptr inbounds [16 x i8], [16 x i8]* %alloc1, i64 0, i64 0
+  %part2 = getelementptr inbounds [16 x i8], [16 x i8]* %alloc2, i64 0, i64 0
+  call void @llvm.lifetime.start.p0i8(i64 16, i8* nonnull %part1)
+  call void @llvm.lifetime.start.p0i8(i64 16, i8* nonnull %part2)
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* nonnull align 1 %part1, i8* align 1 %a, i64 16, i1 false)
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* nonnull align 1 %part2, i8* align 1 %b, i64 16, i1 false)
+  call void @llvm.lifetime.end.p0i8(i64 16, i8* nonnull %part1)
+  call void @llvm.lifetime.end.p0i8(i64 16, i8* nonnull %part2)
+  ret void
+}
+
+define dso_local void @twoallocs(i8* nocapture %a, i8* nocapture %b) local_unnamed_addr {
+; NOAA-LABEL: twoallocs:
+; NOAA:       # %bb.0: # %entry
+; NOAA-NEXT:    vmovups (%rdi), %xmm0
+; NOAA-NEXT:    vmovaps %xmm0, -{{[0-9]+}}(%rsp)
+; NOAA-NEXT:    vmovaps -{{[0-9]+}}(%rsp), %xmm0
+; NOAA-NEXT:    vmovups %xmm0, (%rsi)
+; NOAA-NEXT:    retq
+;
+; AA-LABEL: twoallocs:
+; AA:       # %bb.0: # %entry
+; AA-NEXT:    vmovups (%rdi), %xmm0
+; AA-NEXT:    vmovaps %xmm0, -{{[0-9]+}}(%rsp)
+; AA-NEXT:    vmovaps -{{[0-9]+}}(%rsp), %xmm0
+; AA-NEXT:    vmovups %xmm0, (%rsi)
+; AA-NEXT:    retq
+entry:
+  %alloc1 = alloca [16 x i8], align 1
+  %alloc2 = alloca [16 x i8], align 1
+  %part1 = getelementptr inbounds [16 x i8], [16 x i8]* %alloc1, i64 0, i64 0
+  %part2 = getelementptr inbounds [16 x i8], [16 x i8]* %alloc2, i64 0, i64 0
+  %part2_alias = getelementptr inbounds [16 x i8], [16 x i8]* %alloc2, i64 0, i64 0
+  call void @llvm.lifetime.start.p0i8(i64 16, i8* nonnull %part1)
+  call void @llvm.lifetime.start.p0i8(i64 16, i8* nonnull %part2)
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* nonnull align 1 %part2, i8* align 1 %a, i64 16, i1 false)
+  call void @llvm.lifetime.end.p0i8(i64 16, i8* nonnull %part1)
+  tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* nonnull align 1 %b, i8* align 1 %part2_alias, i64 16, i1 false)
+  call void @llvm.lifetime.end.p0i8(i64 16, i8* nonnull %part2)
+  ret void
+}
+
+define dso_local void @onealloc_readback_1(i8* nocapture %a, i8* nocapture %b) local_unnamed_addr {
+; NOAA-LABEL: onealloc_readback_1:
+; NOAA:       # %bb.0: # %entry
+; NOAA-NEXT:    vmovups (%rdi), %xmm0
+; NOAA-NEXT:    vmovaps %xmm0, -{{[0-9]+}}(%rsp)
+; NOAA-NEXT:    vmovups (%rsi), %xmm0
+; NOAA-NEXT:    vmovaps %xmm0, -{{[0-9]+}}(%rsp)
+; NOAA-NEXT:    vmovaps -{{[0-9]+}}(%rsp), %xmm0
+; NOAA-NEXT:    vmovups %xmm0, (%rdi)
+; NOAA-NEXT:    retq
+;
+; AA-LABEL: onealloc_readback_1:
+; AA:       # %bb.0: # %entry
+; AA-NEXT:    vmovups (%rdi), %xmm0
+; AA-NEXT:    vmovaps %xmm0, -{{[0-9]+}}(%rsp)
+; AA-NEXT:    vmovups (%rsi), %xmm0
+; AA-NEXT:    vmovaps %xmm0, -{{[0-9]+}}(%rsp)
+; AA-NEXT:    vmovaps -{{[0-9]+}}(%rsp), %xmm0
+; AA-NEXT:    vmovups %xmm0, (%rdi)
+; AA-NEXT:    retq
+entry:
+  %alloc = alloca [16 x i8], i8 2, align 1
+  %part2 = getelementptr inbounds [16 x i8], [16 x i8]* %alloc, i64 0, i64 0
+  %part1 = getelementptr inbounds [16 x i8], [16 x i8]* %alloc, i64 1, i64 0
+  call void @llvm.lifetime.start.p0i8(i64 16, i8* nonnull %part1)
+  call void @llvm.lifetime.start.p0i8(i64 16, i8* nonnull %part2)
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* nonnull align 1 %part1, i8* align 1 %a, i64 16, i1 false)
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* nonnull align 1 %part2, i8* align 1 %b, i64 16, i1 false)
+  call void @llvm.lifetime.end.p0i8(i64 16, i8* nonnull %part1)
+  tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* nonnull align 1 %a, i8* align 1 %part2, i64 16, i1 false)
+  call void @llvm.lifetime.end.p0i8(i64 16, i8* nonnull %part2)
+  ret void
+}
+
+define dso_local void @onealloc_readback_2(i8* nocapture %a, i8* nocapture %b) local_unnamed_addr {
+; NOAA-LABEL: onealloc_readback_2:
+; NOAA:       # %bb.0: # %entry
+; NOAA-NEXT:    vmovups (%rdi), %xmm0
+; NOAA-NEXT:    vmovaps %xmm0, -{{[0-9]+}}(%rsp)
+; NOAA-NEXT:    vmovups (%rsi), %xmm0
+; NOAA-NEXT:    vmovaps %xmm0, -{{[0-9]+}}(%rsp)
+; NOAA-NEXT:    vmovaps -{{[0-9]+}}(%rsp), %xmm0
+; NOAA-NEXT:    vmovups %xmm0, (%rdi)
+; NOAA-NEXT:    retq
+;
+; AA-LABEL: onealloc_readback_2:
+; AA:       # %bb.0: # %entry
+; AA-NEXT:    vmovups (%rdi), %xmm0
+; AA-NEXT:    vmovaps %xmm0, -{{[0-9]+}}(%rsp)
+; AA-NEXT:    vmovups (%rsi), %xmm0
+; AA-NEXT:    vmovaps %xmm0, -{{[0-9]+}}(%rsp)
+; AA-NEXT:    vmovaps -{{[0-9]+}}(%rsp), %xmm0
+; AA-NEXT:    vmovups %xmm0, (%rdi)
+; AA-NEXT:    retq
+entry:
+  %alloc = alloca [16 x i8], i8 2, align 1
+  %part1 = getelementptr inbounds [16 x i8], [16 x i8]* %alloc, i64 0, i64 0
+  %part2 = getelementptr inbounds [16 x i8], [16 x i8]* %alloc, i64 1, i64 0
+  call void @llvm.lifetime.start.p0i8(i64 16, i8* nonnull %part1)
+  call void @llvm.lifetime.start.p0i8(i64 16, i8* nonnull %part2)
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* nonnull align 1 %part1, i8* align 1 %a, i64 16, i1 false)
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* nonnull align 1 %part2, i8* align 1 %b, i64 16, i1 false)
+  call void @llvm.lifetime.end.p0i8(i64 16, i8* nonnull %part1)
+  tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* nonnull align 1 %a, i8* align 1 %part2, i64 16, i1 false)
+  call void @llvm.lifetime.end.p0i8(i64 16, i8* nonnull %part2)
+  ret void
+}
+
+
+!llvm.module.flags = !{!0}
+!llvm.ident = !{!1}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{!"clang version 9.0.0 (trunk 352631) (llvm/trunk 352632)"}
+!2 = !{i64 0, i64 16, !3}
+!3 = !{!4, !4, i64 0}
+!4 = !{!"omnipotent char", !5, i64 0}
+!5 = !{!"Simple C++ TBAA"}
diff --git a/test/CodeGen/X86/tbm-schedule.ll b/test/CodeGen/X86/tbm-schedule.ll
deleted file mode 100644
index b8f9bb08f3e3e7de5eedf2f1ea0e10c5b258d55c..0000000000000000000000000000000000000000
--- a/test/CodeGen/X86/tbm-schedule.ll
+++ /dev/null
@@ -1,773 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=x86-64 -mattr=+tbm | FileCheck %s --check-prefix=GENERIC
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=bdver2 | FileCheck %s --check-prefix=BDVER --check-prefix=BDVER2
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=bdver3 | FileCheck %s --check-prefix=BDVER --check-prefix=BDVER3
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=bdver4 | FileCheck %s --check-prefix=BDVER --check-prefix=BDVER4
-
-define i32 @test_x86_tbm_bextri_u32(i32 %a0, i32* nocapture %p1) nounwind {
-; GENERIC-LABEL: test_x86_tbm_bextri_u32:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    bextrl $3076, %edi, %ecx # imm = 0xC04
-; GENERIC-NEXT:    # sched: [2:1.00]
-; GENERIC-NEXT:    bextrl $3076, (%rsi), %eax # imm = 0xC04
-; GENERIC-NEXT:    # sched: [7:1.00]
-; GENERIC-NEXT:    addl %ecx, %eax # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER2-LABEL: test_x86_tbm_bextri_u32:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    bextrl $3076, (%rsi), %eax # imm = 0xC04
-; BDVER2-NEXT:    # sched: [6:0.50]
-; BDVER2-NEXT:    bextrl $3076, %edi, %ecx # imm = 0xC04
-; BDVER2-NEXT:    # sched: [2:0.50]
-; BDVER2-NEXT:    addl %ecx, %eax # sched: [1:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER3-LABEL: test_x86_tbm_bextri_u32:
-; BDVER3:       # %bb.0:
-; BDVER3-NEXT:    bextrl $3076, %edi, %ecx # imm = 0xC04
-; BDVER3-NEXT:    bextrl $3076, (%rsi), %eax # imm = 0xC04
-; BDVER3-NEXT:    addl %ecx, %eax
-; BDVER3-NEXT:    retq
-;
-; BDVER4-LABEL: test_x86_tbm_bextri_u32:
-; BDVER4:       # %bb.0:
-; BDVER4-NEXT:    bextrl $3076, %edi, %ecx # imm = 0xC04
-; BDVER4-NEXT:    bextrl $3076, (%rsi), %eax # imm = 0xC04
-; BDVER4-NEXT:    addl %ecx, %eax
-; BDVER4-NEXT:    retq
-  %a1 = load i32, i32* %p1
-  %r0 = lshr i32 %a0, 4
-  %m0 = lshr i32 %a1, 4
-  %r1 = and i32 %r0, 4095
-  %m1 = and i32 %m0, 4095
-  %res = add i32 %r1, %m1
-  ret i32 %res
-}
-
-define i64 @test_x86_tbm_bextri_u64(i64 %a0, i64* nocapture %p1) nounwind {
-; GENERIC-LABEL: test_x86_tbm_bextri_u64:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    bextrl $3076, %edi, %ecx # imm = 0xC04
-; GENERIC-NEXT:    # sched: [2:1.00]
-; GENERIC-NEXT:    bextrl $3076, (%rsi), %eax # imm = 0xC04
-; GENERIC-NEXT:    # sched: [7:1.00]
-; GENERIC-NEXT:    addq %rcx, %rax # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER2-LABEL: test_x86_tbm_bextri_u64:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    bextrl $3076, (%rsi), %eax # imm = 0xC04
-; BDVER2-NEXT:    # sched: [6:0.50]
-; BDVER2-NEXT:    bextrl $3076, %edi, %ecx # imm = 0xC04
-; BDVER2-NEXT:    # sched: [2:0.50]
-; BDVER2-NEXT:    addq %rcx, %rax # sched: [1:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER3-LABEL: test_x86_tbm_bextri_u64:
-; BDVER3:       # %bb.0:
-; BDVER3-NEXT:    bextrl $3076, %edi, %ecx # imm = 0xC04
-; BDVER3-NEXT:    bextrl $3076, (%rsi), %eax # imm = 0xC04
-; BDVER3-NEXT:    addq %rcx, %rax
-; BDVER3-NEXT:    retq
-;
-; BDVER4-LABEL: test_x86_tbm_bextri_u64:
-; BDVER4:       # %bb.0:
-; BDVER4-NEXT:    bextrl $3076, %edi, %ecx # imm = 0xC04
-; BDVER4-NEXT:    bextrl $3076, (%rsi), %eax # imm = 0xC04
-; BDVER4-NEXT:    addq %rcx, %rax
-; BDVER4-NEXT:    retq
-  %a1 = load i64, i64* %p1
-  %r0 = lshr i64 %a0, 4
-  %m0 = lshr i64 %a1, 4
-  %r1 = and i64 %r0, 4095
-  %m1 = and i64 %m0, 4095
-  %res = add i64 %r1, %m1
-  ret i64 %res
-}
-
-define i32 @test_x86_tbm_blcfill_u32(i32 %a0, i32* nocapture %p1) nounwind {
-; GENERIC-LABEL: test_x86_tbm_blcfill_u32:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    blcfilll %edi, %ecx # sched: [1:0.33]
-; GENERIC-NEXT:    blcfilll (%rsi), %eax # sched: [6:0.50]
-; GENERIC-NEXT:    addl %ecx, %eax # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER2-LABEL: test_x86_tbm_blcfill_u32:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    blcfilll (%rsi), %eax # sched: [6:0.50]
-; BDVER2-NEXT:    blcfilll %edi, %ecx # sched: [2:0.50]
-; BDVER2-NEXT:    addl %ecx, %eax # sched: [1:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER3-LABEL: test_x86_tbm_blcfill_u32:
-; BDVER3:       # %bb.0:
-; BDVER3-NEXT:    blcfilll %edi, %ecx
-; BDVER3-NEXT:    blcfilll (%rsi), %eax
-; BDVER3-NEXT:    addl %ecx, %eax
-; BDVER3-NEXT:    retq
-;
-; BDVER4-LABEL: test_x86_tbm_blcfill_u32:
-; BDVER4:       # %bb.0:
-; BDVER4-NEXT:    blcfilll %edi, %ecx
-; BDVER4-NEXT:    blcfilll (%rsi), %eax
-; BDVER4-NEXT:    addl %ecx, %eax
-; BDVER4-NEXT:    retq
-  %a1 = load i32, i32* %p1
-  %r0 = add i32 %a0, 1
-  %m0 = add i32 %a1, 1
-  %r1 = and i32 %r0, %a0
-  %m1 = and i32 %m0, %a1
-  %res = add i32 %r1, %m1
-  ret i32 %res
-}
-
-define i64 @test_x86_tbm_blcfill_u64(i64 %a0, i64* nocapture %p1) nounwind {
-; GENERIC-LABEL: test_x86_tbm_blcfill_u64:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    blcfillq %rdi, %rcx # sched: [1:0.33]
-; GENERIC-NEXT:    blcfillq (%rsi), %rax # sched: [6:0.50]
-; GENERIC-NEXT:    addq %rcx, %rax # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER2-LABEL: test_x86_tbm_blcfill_u64:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    blcfillq (%rsi), %rax # sched: [6:0.50]
-; BDVER2-NEXT:    blcfillq %rdi, %rcx # sched: [2:0.50]
-; BDVER2-NEXT:    addq %rcx, %rax # sched: [1:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER3-LABEL: test_x86_tbm_blcfill_u64:
-; BDVER3:       # %bb.0:
-; BDVER3-NEXT:    blcfillq %rdi, %rcx
-; BDVER3-NEXT:    blcfillq (%rsi), %rax
-; BDVER3-NEXT:    addq %rcx, %rax
-; BDVER3-NEXT:    retq
-;
-; BDVER4-LABEL: test_x86_tbm_blcfill_u64:
-; BDVER4:       # %bb.0:
-; BDVER4-NEXT:    blcfillq %rdi, %rcx
-; BDVER4-NEXT:    blcfillq (%rsi), %rax
-; BDVER4-NEXT:    addq %rcx, %rax
-; BDVER4-NEXT:    retq
-  %a1 = load i64, i64* %p1
-  %r0 = add i64 %a0, 1
-  %m0 = add i64 %a1, 1
-  %r1 = and i64 %r0, %a0
-  %m1 = and i64 %m0, %a1
-  %res = add i64 %r1, %m1
-  ret i64 %res
-}
-
-define i32 @test_x86_tbm_blci_u32(i32 %a0, i32* nocapture %p1) nounwind {
-; GENERIC-LABEL: test_x86_tbm_blci_u32:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    blcil %edi, %ecx # sched: [1:0.33]
-; GENERIC-NEXT:    blcil (%rsi), %eax # sched: [6:0.50]
-; GENERIC-NEXT:    addl %ecx, %eax # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER2-LABEL: test_x86_tbm_blci_u32:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    blcil (%rsi), %eax # sched: [6:0.50]
-; BDVER2-NEXT:    blcil %edi, %ecx # sched: [2:0.50]
-; BDVER2-NEXT:    addl %ecx, %eax # sched: [1:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER3-LABEL: test_x86_tbm_blci_u32:
-; BDVER3:       # %bb.0:
-; BDVER3-NEXT:    blcil %edi, %ecx
-; BDVER3-NEXT:    blcil (%rsi), %eax
-; BDVER3-NEXT:    addl %ecx, %eax
-; BDVER3-NEXT:    retq
-;
-; BDVER4-LABEL: test_x86_tbm_blci_u32:
-; BDVER4:       # %bb.0:
-; BDVER4-NEXT:    blcil %edi, %ecx
-; BDVER4-NEXT:    blcil (%rsi), %eax
-; BDVER4-NEXT:    addl %ecx, %eax
-; BDVER4-NEXT:    retq
-  %a1 = load i32, i32* %p1
-  %r0 = add i32 1, %a0
-  %m0 = add i32 1, %a1
-  %r1 = xor i32 %r0, -1
-  %m1 = xor i32 %m0, -1
-  %r2 = or i32 %r1, %a0
-  %m2 = or i32 %m1, %a1
-  %res = add i32 %r2, %m2
-  ret i32 %res
-}
-
-define i64 @test_x86_tbm_blci_u64(i64 %a0, i64* nocapture %p1) nounwind {
-; GENERIC-LABEL: test_x86_tbm_blci_u64:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    blciq %rdi, %rcx # sched: [1:0.33]
-; GENERIC-NEXT:    blciq (%rsi), %rax # sched: [6:0.50]
-; GENERIC-NEXT:    addq %rcx, %rax # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER2-LABEL: test_x86_tbm_blci_u64:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    blciq (%rsi), %rax # sched: [6:0.50]
-; BDVER2-NEXT:    blciq %rdi, %rcx # sched: [2:0.50]
-; BDVER2-NEXT:    addq %rcx, %rax # sched: [1:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER3-LABEL: test_x86_tbm_blci_u64:
-; BDVER3:       # %bb.0:
-; BDVER3-NEXT:    blciq %rdi, %rcx
-; BDVER3-NEXT:    blciq (%rsi), %rax
-; BDVER3-NEXT:    addq %rcx, %rax
-; BDVER3-NEXT:    retq
-;
-; BDVER4-LABEL: test_x86_tbm_blci_u64:
-; BDVER4:       # %bb.0:
-; BDVER4-NEXT:    blciq %rdi, %rcx
-; BDVER4-NEXT:    blciq (%rsi), %rax
-; BDVER4-NEXT:    addq %rcx, %rax
-; BDVER4-NEXT:    retq
-  %a1 = load i64, i64* %p1
-  %r0 = add i64 1, %a0
-  %m0 = add i64 1, %a1
-  %r1 = xor i64 %r0, -1
-  %m1 = xor i64 %m0, -1
-  %r2 = or i64 %r1, %a0
-  %m2 = or i64 %m1, %a1
-  %res = add i64 %r2, %m2
-  ret i64 %res
-}
-
-define i32 @test_x86_tbm_blcic_u32(i32 %a0, i32* nocapture %p1) nounwind {
-; GENERIC-LABEL: test_x86_tbm_blcic_u32:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    blcicl %edi, %ecx # sched: [1:0.33]
-; GENERIC-NEXT:    blcicl (%rsi), %eax # sched: [6:0.50]
-; GENERIC-NEXT:    addl %ecx, %eax # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER2-LABEL: test_x86_tbm_blcic_u32:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    blcicl (%rsi), %eax # sched: [6:0.50]
-; BDVER2-NEXT:    blcicl %edi, %ecx # sched: [2:0.50]
-; BDVER2-NEXT:    addl %ecx, %eax # sched: [1:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER3-LABEL: test_x86_tbm_blcic_u32:
-; BDVER3:       # %bb.0:
-; BDVER3-NEXT:    blcicl %edi, %ecx
-; BDVER3-NEXT:    blcicl (%rsi), %eax
-; BDVER3-NEXT:    addl %ecx, %eax
-; BDVER3-NEXT:    retq
-;
-; BDVER4-LABEL: test_x86_tbm_blcic_u32:
-; BDVER4:       # %bb.0:
-; BDVER4-NEXT:    blcicl %edi, %ecx
-; BDVER4-NEXT:    blcicl (%rsi), %eax
-; BDVER4-NEXT:    addl %ecx, %eax
-; BDVER4-NEXT:    retq
-  %a1 = load i32, i32* %p1
-  %r0 = xor i32 %a0, -1
-  %m0 = xor i32 %a1, -1
-  %r1 = add i32 %a0, 1
-  %m1 = add i32 %a1, 1
-  %r2 = and i32 %r1, %r0
-  %m2 = and i32 %m1, %m0
-  %res = add i32 %r2, %m2
-  ret i32 %res
-}
-
-define i64 @test_x86_tbm_blcic_u64(i64 %a0, i64* nocapture %p1) nounwind {
-; GENERIC-LABEL: test_x86_tbm_blcic_u64:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    blcicq %rdi, %rcx # sched: [1:0.33]
-; GENERIC-NEXT:    blcicq (%rsi), %rax # sched: [6:0.50]
-; GENERIC-NEXT:    addq %rcx, %rax # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER2-LABEL: test_x86_tbm_blcic_u64:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    blcicq (%rsi), %rax # sched: [6:0.50]
-; BDVER2-NEXT:    blcicq %rdi, %rcx # sched: [2:0.50]
-; BDVER2-NEXT:    addq %rcx, %rax # sched: [1:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER3-LABEL: test_x86_tbm_blcic_u64:
-; BDVER3:       # %bb.0:
-; BDVER3-NEXT:    blcicq %rdi, %rcx
-; BDVER3-NEXT:    blcicq (%rsi), %rax
-; BDVER3-NEXT:    addq %rcx, %rax
-; BDVER3-NEXT:    retq
-;
-; BDVER4-LABEL: test_x86_tbm_blcic_u64:
-; BDVER4:       # %bb.0:
-; BDVER4-NEXT:    blcicq %rdi, %rcx
-; BDVER4-NEXT:    blcicq (%rsi), %rax
-; BDVER4-NEXT:    addq %rcx, %rax
-; BDVER4-NEXT:    retq
-  %a1 = load i64, i64* %p1
-  %r0 = xor i64 %a0, -1
-  %m0 = xor i64 %a1, -1
-  %r1 = add i64 %a0, 1
-  %m1 = add i64 %a1, 1
-  %r2 = and i64 %r1, %r0
-  %m2 = and i64 %m1, %m0
-  %res = add i64 %r2, %m2
-  ret i64 %res
-}
-
-define i32 @test_x86_tbm_blcmsk_u32(i32 %a0, i32* nocapture %p1) nounwind {
-; GENERIC-LABEL: test_x86_tbm_blcmsk_u32:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    blcmskl %edi, %ecx # sched: [1:0.33]
-; GENERIC-NEXT:    blcmskl (%rsi), %eax # sched: [6:0.50]
-; GENERIC-NEXT:    addl %ecx, %eax # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER2-LABEL: test_x86_tbm_blcmsk_u32:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    blcmskl (%rsi), %eax # sched: [6:0.50]
-; BDVER2-NEXT:    blcmskl %edi, %ecx # sched: [2:0.50]
-; BDVER2-NEXT:    addl %ecx, %eax # sched: [1:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER3-LABEL: test_x86_tbm_blcmsk_u32:
-; BDVER3:       # %bb.0:
-; BDVER3-NEXT:    blcmskl %edi, %ecx
-; BDVER3-NEXT:    blcmskl (%rsi), %eax
-; BDVER3-NEXT:    addl %ecx, %eax
-; BDVER3-NEXT:    retq
-;
-; BDVER4-LABEL: test_x86_tbm_blcmsk_u32:
-; BDVER4:       # %bb.0:
-; BDVER4-NEXT:    blcmskl %edi, %ecx
-; BDVER4-NEXT:    blcmskl (%rsi), %eax
-; BDVER4-NEXT:    addl %ecx, %eax
-; BDVER4-NEXT:    retq
-  %a1 = load i32, i32* %p1
-  %r0 = add i32 %a0, 1
-  %m0 = add i32 %a1, 1
-  %r1 = xor i32 %r0, %a0
-  %m1 = xor i32 %m0, %a1
-  %res = add i32 %r1, %m1
-  ret i32 %res
-}
-
-define i64 @test_x86_tbm_blcmsk_u64(i64 %a0, i64* nocapture %p1) nounwind {
-; GENERIC-LABEL: test_x86_tbm_blcmsk_u64:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    blcmskq %rdi, %rcx # sched: [1:0.33]
-; GENERIC-NEXT:    blcmskq (%rsi), %rax # sched: [6:0.50]
-; GENERIC-NEXT:    addq %rcx, %rax # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER2-LABEL: test_x86_tbm_blcmsk_u64:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    blcmskq (%rsi), %rax # sched: [6:0.50]
-; BDVER2-NEXT:    blcmskq %rdi, %rcx # sched: [2:0.50]
-; BDVER2-NEXT:    addq %rcx, %rax # sched: [1:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER3-LABEL: test_x86_tbm_blcmsk_u64:
-; BDVER3:       # %bb.0:
-; BDVER3-NEXT:    blcmskq %rdi, %rcx
-; BDVER3-NEXT:    blcmskq (%rsi), %rax
-; BDVER3-NEXT:    addq %rcx, %rax
-; BDVER3-NEXT:    retq
-;
-; BDVER4-LABEL: test_x86_tbm_blcmsk_u64:
-; BDVER4:       # %bb.0:
-; BDVER4-NEXT:    blcmskq %rdi, %rcx
-; BDVER4-NEXT:    blcmskq (%rsi), %rax
-; BDVER4-NEXT:    addq %rcx, %rax
-; BDVER4-NEXT:    retq
-  %a1 = load i64, i64* %p1
-  %r0 = add i64 %a0, 1
-  %m0 = add i64 %a1, 1
-  %r1 = xor i64 %r0, %a0
-  %m1 = xor i64 %m0, %a1
-  %res = add i64 %r1, %m1
-  ret i64 %res
-}
-
-define i32 @test_x86_tbm_blcs_u32(i32 %a0, i32* nocapture %p1) nounwind {
-; GENERIC-LABEL: test_x86_tbm_blcs_u32:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    blcsl %edi, %ecx # sched: [1:0.33]
-; GENERIC-NEXT:    blcsl (%rsi), %eax # sched: [6:0.50]
-; GENERIC-NEXT:    addl %ecx, %eax # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER2-LABEL: test_x86_tbm_blcs_u32:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    blcsl (%rsi), %eax # sched: [6:0.50]
-; BDVER2-NEXT:    blcsl %edi, %ecx # sched: [2:0.50]
-; BDVER2-NEXT:    addl %ecx, %eax # sched: [1:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER3-LABEL: test_x86_tbm_blcs_u32:
-; BDVER3:       # %bb.0:
-; BDVER3-NEXT:    blcsl %edi, %ecx
-; BDVER3-NEXT:    blcsl (%rsi), %eax
-; BDVER3-NEXT:    addl %ecx, %eax
-; BDVER3-NEXT:    retq
-;
-; BDVER4-LABEL: test_x86_tbm_blcs_u32:
-; BDVER4:       # %bb.0:
-; BDVER4-NEXT:    blcsl %edi, %ecx
-; BDVER4-NEXT:    blcsl (%rsi), %eax
-; BDVER4-NEXT:    addl %ecx, %eax
-; BDVER4-NEXT:    retq
-  %a1 = load i32, i32* %p1
-  %r0 = add i32 %a0, 1
-  %m0 = add i32 %a1, 1
-  %r1 = or i32 %r0, %a0
-  %m1 = or i32 %m0, %a1
-  %res = add i32 %r1, %m1
-  ret i32 %res
-}
-
-define i64 @test_x86_tbm_blcs_u64(i64 %a0, i64* nocapture %p1) nounwind {
-; GENERIC-LABEL: test_x86_tbm_blcs_u64:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    blcsq %rdi, %rcx # sched: [1:0.33]
-; GENERIC-NEXT:    blcsq (%rsi), %rax # sched: [6:0.50]
-; GENERIC-NEXT:    addq %rcx, %rax # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER2-LABEL: test_x86_tbm_blcs_u64:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    blcsq (%rsi), %rax # sched: [6:0.50]
-; BDVER2-NEXT:    blcsq %rdi, %rcx # sched: [2:0.50]
-; BDVER2-NEXT:    addq %rcx, %rax # sched: [1:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER3-LABEL: test_x86_tbm_blcs_u64:
-; BDVER3:       # %bb.0:
-; BDVER3-NEXT:    blcsq %rdi, %rcx
-; BDVER3-NEXT:    blcsq (%rsi), %rax
-; BDVER3-NEXT:    addq %rcx, %rax
-; BDVER3-NEXT:    retq
-;
-; BDVER4-LABEL: test_x86_tbm_blcs_u64:
-; BDVER4:       # %bb.0:
-; BDVER4-NEXT:    blcsq %rdi, %rcx
-; BDVER4-NEXT:    blcsq (%rsi), %rax
-; BDVER4-NEXT:    addq %rcx, %rax
-; BDVER4-NEXT:    retq
-  %a1 = load i64, i64* %p1
-  %r0 = add i64 %a0, 1
-  %m0 = add i64 %a1, 1
-  %r1 = or i64 %r0, %a0
-  %m1 = or i64 %m0, %a1
-  %res = add i64 %r1, %m1
-  ret i64 %res
-}
-
-define i32 @test_x86_tbm_blsfill_u32(i32 %a0, i32* nocapture %p1) nounwind {
-; GENERIC-LABEL: test_x86_tbm_blsfill_u32:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    blsfilll %edi, %ecx # sched: [1:0.33]
-; GENERIC-NEXT:    blsfilll (%rsi), %eax # sched: [6:0.50]
-; GENERIC-NEXT:    addl %ecx, %eax # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER2-LABEL: test_x86_tbm_blsfill_u32:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    blsfilll (%rsi), %eax # sched: [6:0.50]
-; BDVER2-NEXT:    blsfilll %edi, %ecx # sched: [2:0.50]
-; BDVER2-NEXT:    addl %ecx, %eax # sched: [1:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER3-LABEL: test_x86_tbm_blsfill_u32:
-; BDVER3:       # %bb.0:
-; BDVER3-NEXT:    blsfilll %edi, %ecx
-; BDVER3-NEXT:    blsfilll (%rsi), %eax
-; BDVER3-NEXT:    addl %ecx, %eax
-; BDVER3-NEXT:    retq
-;
-; BDVER4-LABEL: test_x86_tbm_blsfill_u32:
-; BDVER4:       # %bb.0:
-; BDVER4-NEXT:    blsfilll %edi, %ecx
-; BDVER4-NEXT:    blsfilll (%rsi), %eax
-; BDVER4-NEXT:    addl %ecx, %eax
-; BDVER4-NEXT:    retq
-  %a1 = load i32, i32* %p1
-  %r0 = add i32 %a0, -1
-  %m0 = add i32 %a1, -1
-  %r1 = or i32 %r0, %a0
-  %m1 = or i32 %m0, %a1
-  %res = add i32 %r1, %m1
-  ret i32 %res
-}
-
-define i64 @test_x86_tbm_blsfill_u64(i64 %a0, i64* nocapture %p1) nounwind {
-; GENERIC-LABEL: test_x86_tbm_blsfill_u64:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    blsfillq %rdi, %rcx # sched: [1:0.33]
-; GENERIC-NEXT:    blsfillq (%rsi), %rax # sched: [6:0.50]
-; GENERIC-NEXT:    addq %rcx, %rax # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER2-LABEL: test_x86_tbm_blsfill_u64:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    blsfillq (%rsi), %rax # sched: [6:0.50]
-; BDVER2-NEXT:    blsfillq %rdi, %rcx # sched: [2:0.50]
-; BDVER2-NEXT:    addq %rcx, %rax # sched: [1:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER3-LABEL: test_x86_tbm_blsfill_u64:
-; BDVER3:       # %bb.0:
-; BDVER3-NEXT:    blsfillq %rdi, %rcx
-; BDVER3-NEXT:    blsfillq (%rsi), %rax
-; BDVER3-NEXT:    addq %rcx, %rax
-; BDVER3-NEXT:    retq
-;
-; BDVER4-LABEL: test_x86_tbm_blsfill_u64:
-; BDVER4:       # %bb.0:
-; BDVER4-NEXT:    blsfillq %rdi, %rcx
-; BDVER4-NEXT:    blsfillq (%rsi), %rax
-; BDVER4-NEXT:    addq %rcx, %rax
-; BDVER4-NEXT:    retq
-  %a1 = load i64, i64* %p1
-  %r0 = add i64 %a0, -1
-  %m0 = add i64 %a1, -1
-  %r1 = or i64 %r0, %a0
-  %m1 = or i64 %m0, %a1
-  %res = add i64 %r1, %m1
-  ret i64 %res
-}
-
-define i32 @test_x86_tbm_blsic_u32(i32 %a0, i32* nocapture %p1) nounwind {
-; GENERIC-LABEL: test_x86_tbm_blsic_u32:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    blsicl %edi, %ecx # sched: [1:0.33]
-; GENERIC-NEXT:    blsicl (%rsi), %eax # sched: [6:0.50]
-; GENERIC-NEXT:    addl %ecx, %eax # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER2-LABEL: test_x86_tbm_blsic_u32:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    blsicl (%rsi), %eax # sched: [6:0.50]
-; BDVER2-NEXT:    blsicl %edi, %ecx # sched: [2:0.50]
-; BDVER2-NEXT:    addl %ecx, %eax # sched: [1:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER3-LABEL: test_x86_tbm_blsic_u32:
-; BDVER3:       # %bb.0:
-; BDVER3-NEXT:    blsicl %edi, %ecx
-; BDVER3-NEXT:    blsicl (%rsi), %eax
-; BDVER3-NEXT:    addl %ecx, %eax
-; BDVER3-NEXT:    retq
-;
-; BDVER4-LABEL: test_x86_tbm_blsic_u32:
-; BDVER4:       # %bb.0:
-; BDVER4-NEXT:    blsicl %edi, %ecx
-; BDVER4-NEXT:    blsicl (%rsi), %eax
-; BDVER4-NEXT:    addl %ecx, %eax
-; BDVER4-NEXT:    retq
-  %a1 = load i32, i32* %p1
-  %r0 = xor i32 %a0, -1
-  %m0 = xor i32 %a1, -1
-  %r1 = add i32 %a0, -1
-  %m1 = add i32 %a1, -1
-  %r2 = or i32 %r0, %r1
-  %m2 = or i32 %m0, %m1
-  %res = add i32 %r2, %m2
-  ret i32 %res
-}
-
-define i64 @test_x86_tbm_blsic_u64(i64 %a0, i64* nocapture %p1) nounwind {
-; GENERIC-LABEL: test_x86_tbm_blsic_u64:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    blsicq %rdi, %rcx # sched: [1:0.33]
-; GENERIC-NEXT:    blsicq (%rsi), %rax # sched: [6:0.50]
-; GENERIC-NEXT:    addq %rcx, %rax # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER2-LABEL: test_x86_tbm_blsic_u64:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    blsicq (%rsi), %rax # sched: [6:0.50]
-; BDVER2-NEXT:    blsicq %rdi, %rcx # sched: [2:0.50]
-; BDVER2-NEXT:    addq %rcx, %rax # sched: [1:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER3-LABEL: test_x86_tbm_blsic_u64:
-; BDVER3:       # %bb.0:
-; BDVER3-NEXT:    blsicq %rdi, %rcx
-; BDVER3-NEXT:    blsicq (%rsi), %rax
-; BDVER3-NEXT:    addq %rcx, %rax
-; BDVER3-NEXT:    retq
-;
-; BDVER4-LABEL: test_x86_tbm_blsic_u64:
-; BDVER4:       # %bb.0:
-; BDVER4-NEXT:    blsicq %rdi, %rcx
-; BDVER4-NEXT:    blsicq (%rsi), %rax
-; BDVER4-NEXT:    addq %rcx, %rax
-; BDVER4-NEXT:    retq
-  %a1 = load i64, i64* %p1
-  %r0 = xor i64 %a0, -1
-  %m0 = xor i64 %a1, -1
-  %r1 = add i64 %a0, -1
-  %m1 = add i64 %a1, -1
-  %r2 = or i64 %r0, %r1
-  %m2 = or i64 %m0, %m1
-  %res = add i64 %r2, %m2
-  ret i64 %res
-}
-
-define i32 @test_x86_tbm_t1mskc_u32(i32 %a0, i32* nocapture %p1) nounwind {
-; GENERIC-LABEL: test_x86_tbm_t1mskc_u32:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    t1mskcl %edi, %ecx # sched: [1:0.33]
-; GENERIC-NEXT:    t1mskcl (%rsi), %eax # sched: [6:0.50]
-; GENERIC-NEXT:    addl %ecx, %eax # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER2-LABEL: test_x86_tbm_t1mskc_u32:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    t1mskcl (%rsi), %eax # sched: [6:0.50]
-; BDVER2-NEXT:    t1mskcl %edi, %ecx # sched: [2:0.50]
-; BDVER2-NEXT:    addl %ecx, %eax # sched: [1:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER3-LABEL: test_x86_tbm_t1mskc_u32:
-; BDVER3:       # %bb.0:
-; BDVER3-NEXT:    t1mskcl %edi, %ecx
-; BDVER3-NEXT:    t1mskcl (%rsi), %eax
-; BDVER3-NEXT:    addl %ecx, %eax
-; BDVER3-NEXT:    retq
-;
-; BDVER4-LABEL: test_x86_tbm_t1mskc_u32:
-; BDVER4:       # %bb.0:
-; BDVER4-NEXT:    t1mskcl %edi, %ecx
-; BDVER4-NEXT:    t1mskcl (%rsi), %eax
-; BDVER4-NEXT:    addl %ecx, %eax
-; BDVER4-NEXT:    retq
-  %a1 = load i32, i32* %p1
-  %r0 = xor i32 %a0, -1
-  %m0 = xor i32 %a1, -1
-  %r1 = add i32 %a0, 1
-  %m1 = add i32 %a1, 1
-  %r2 = or i32 %r0, %r1
-  %m2 = or i32 %m0, %m1
-  %res = add i32 %r2, %m2
-  ret i32 %res
-}
-
-define i64 @test_x86_tbm_t1mskc_u64(i64 %a0, i64* nocapture %p1) nounwind {
-; GENERIC-LABEL: test_x86_tbm_t1mskc_u64:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    t1mskcq %rdi, %rcx # sched: [1:0.33]
-; GENERIC-NEXT:    t1mskcq (%rsi), %rax # sched: [6:0.50]
-; GENERIC-NEXT:    addq %rcx, %rax # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER2-LABEL: test_x86_tbm_t1mskc_u64:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    t1mskcq (%rsi), %rax # sched: [6:0.50]
-; BDVER2-NEXT:    t1mskcq %rdi, %rcx # sched: [2:0.50]
-; BDVER2-NEXT:    addq %rcx, %rax # sched: [1:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER3-LABEL: test_x86_tbm_t1mskc_u64:
-; BDVER3:       # %bb.0:
-; BDVER3-NEXT:    t1mskcq %rdi, %rcx
-; BDVER3-NEXT:    t1mskcq (%rsi), %rax
-; BDVER3-NEXT:    addq %rcx, %rax
-; BDVER3-NEXT:    retq
-;
-; BDVER4-LABEL: test_x86_tbm_t1mskc_u64:
-; BDVER4:       # %bb.0:
-; BDVER4-NEXT:    t1mskcq %rdi, %rcx
-; BDVER4-NEXT:    t1mskcq (%rsi), %rax
-; BDVER4-NEXT:    addq %rcx, %rax
-; BDVER4-NEXT:    retq
-  %a1 = load i64, i64* %p1
-  %r0 = xor i64 %a0, -1
-  %m0 = xor i64 %a1, -1
-  %r1 = add i64 %a0, 1
-  %m1 = add i64 %a1, 1
-  %r2 = or i64 %r0, %r1
-  %m2 = or i64 %m0, %m1
-  %res = add i64 %r2, %m2
-  ret i64 %res
-}
-
-define i32 @test_x86_tbm_tzmsk_u32(i32 %a0, i32* nocapture %p1) nounwind {
-; GENERIC-LABEL: test_x86_tbm_tzmsk_u32:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    tzmskl %edi, %ecx # sched: [1:0.33]
-; GENERIC-NEXT:    tzmskl (%rsi), %eax # sched: [6:0.50]
-; GENERIC-NEXT:    addl %ecx, %eax # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER2-LABEL: test_x86_tbm_tzmsk_u32:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    tzmskl (%rsi), %eax # sched: [6:0.50]
-; BDVER2-NEXT:    tzmskl %edi, %ecx # sched: [2:0.50]
-; BDVER2-NEXT:    addl %ecx, %eax # sched: [1:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER3-LABEL: test_x86_tbm_tzmsk_u32:
-; BDVER3:       # %bb.0:
-; BDVER3-NEXT:    tzmskl %edi, %ecx
-; BDVER3-NEXT:    tzmskl (%rsi), %eax
-; BDVER3-NEXT:    addl %ecx, %eax
-; BDVER3-NEXT:    retq
-;
-; BDVER4-LABEL: test_x86_tbm_tzmsk_u32:
-; BDVER4:       # %bb.0:
-; BDVER4-NEXT:    tzmskl %edi, %ecx
-; BDVER4-NEXT:    tzmskl (%rsi), %eax
-; BDVER4-NEXT:    addl %ecx, %eax
-; BDVER4-NEXT:    retq
-  %a1 = load i32, i32* %p1
-  %r0 = xor i32 %a0, -1
-  %m0 = xor i32 %a1, -1
-  %r1 = add i32 %a0, -1
-  %m1 = add i32 %a1, -1
-  %r2 = and i32 %r0, %r1
-  %m2 = and i32 %m0, %m1
-  %res = add i32 %r2, %m2
-  ret i32 %res
-}
-
-define i64 @test_x86_tbm_tzmsk_u64(i64 %a0, i64* nocapture %p1) nounwind {
-; GENERIC-LABEL: test_x86_tbm_tzmsk_u64:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    tzmskq %rdi, %rcx # sched: [1:0.33]
-; GENERIC-NEXT:    tzmskq (%rsi), %rax # sched: [6:0.50]
-; GENERIC-NEXT:    addq %rcx, %rax # sched: [1:0.33]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER2-LABEL: test_x86_tbm_tzmsk_u64:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    tzmskq (%rsi), %rax # sched: [6:0.50]
-; BDVER2-NEXT:    tzmskq %rdi, %rcx # sched: [2:0.50]
-; BDVER2-NEXT:    addq %rcx, %rax # sched: [1:0.50]
-; BDVER2-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER3-LABEL: test_x86_tbm_tzmsk_u64:
-; BDVER3:       # %bb.0:
-; BDVER3-NEXT:    tzmskq %rdi, %rcx
-; BDVER3-NEXT:    tzmskq (%rsi), %rax
-; BDVER3-NEXT:    addq %rcx, %rax
-; BDVER3-NEXT:    retq
-;
-; BDVER4-LABEL: test_x86_tbm_tzmsk_u64:
-; BDVER4:       # %bb.0:
-; BDVER4-NEXT:    tzmskq %rdi, %rcx
-; BDVER4-NEXT:    tzmskq (%rsi), %rax
-; BDVER4-NEXT:    addq %rcx, %rax
-; BDVER4-NEXT:    retq
-  %a1 = load i64, i64* %p1
-  %r0 = xor i64 %a0, -1
-  %m0 = xor i64 %a1, -1
-  %r1 = add i64 %a0, -1
-  %m1 = add i64 %a1, -1
-  %r2 = and i64 %r0, %r1
-  %m2 = and i64 %m0, %m1
-  %res = add i64 %r2, %m2
-  ret i64 %res
-}
diff --git a/test/CodeGen/X86/tbm_patterns.ll b/test/CodeGen/X86/tbm_patterns.ll
index 7ebc5dd0d129aaff17b53a739eb38ee3ea1206ab..f7bfb5bec8c9cadd88b557aefaa9877df22e3a6f 100644
--- a/test/CodeGen/X86/tbm_patterns.ll
+++ b/test/CodeGen/X86/tbm_patterns.ll
@@ -150,9 +150,7 @@ define i32 @test_x86_tbm_blcfill_u32_z2(i32 %a, i32 %b, i32 %c) nounwind {
 ; CHECK-LABEL: test_x86_tbm_blcfill_u32_z2:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    movl %esi, %eax
-; CHECK-NEXT:    # kill: def $edi killed $edi def $rdi
-; CHECK-NEXT:    leal 1(%rdi), %ecx
-; CHECK-NEXT:    testl %edi, %ecx
+; CHECK-NEXT:    blcfilll %edi, %ecx
 ; CHECK-NEXT:    cmovnel %edx, %eax
 ; CHECK-NEXT:    retq
   %t0 = add i32 %a, 1
@@ -189,8 +187,7 @@ define i64 @test_x86_tbm_blcfill_u64_z2(i64 %a, i64 %b, i64 %c) nounwind {
 ; CHECK-LABEL: test_x86_tbm_blcfill_u64_z2:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    movq %rsi, %rax
-; CHECK-NEXT:    leaq 1(%rdi), %rcx
-; CHECK-NEXT:    testq %rdi, %rcx
+; CHECK-NEXT:    blcfillq %rdi, %rcx
 ; CHECK-NEXT:    cmovneq %rdx, %rax
 ; CHECK-NEXT:    retq
   %t0 = add i64 %a, 1
@@ -229,10 +226,7 @@ define i32 @test_x86_tbm_blci_u32_z2(i32 %a, i32 %b, i32 %c) nounwind {
 ; CHECK-LABEL: test_x86_tbm_blci_u32_z2:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    movl %esi, %eax
-; CHECK-NEXT:    # kill: def $edi killed $edi def $rdi
-; CHECK-NEXT:    leal 1(%rdi), %ecx
-; CHECK-NEXT:    notl %ecx
-; CHECK-NEXT:    orl %edi, %ecx
+; CHECK-NEXT:    blcil %edi, %ecx
 ; CHECK-NEXT:    cmovnel %edx, %eax
 ; CHECK-NEXT:    retq
   %t0 = add i32 1, %a
@@ -272,9 +266,7 @@ define i64 @test_x86_tbm_blci_u64_z2(i64 %a, i64 %b, i64 %c) nounwind {
 ; CHECK-LABEL: test_x86_tbm_blci_u64_z2:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    movq %rsi, %rax
-; CHECK-NEXT:    leaq 1(%rdi), %rcx
-; CHECK-NEXT:    notq %rcx
-; CHECK-NEXT:    orq %rdi, %rcx
+; CHECK-NEXT:    blciq %rdi, %rcx
 ; CHECK-NEXT:    cmovneq %rdx, %rax
 ; CHECK-NEXT:    retq
   %t0 = add i64 1, %a
@@ -334,10 +326,7 @@ define i32 @test_x86_tbm_blcic_u32_z2(i32 %a, i32 %b, i32 %c) nounwind {
 ; CHECK-LABEL: test_x86_tbm_blcic_u32_z2:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    movl %esi, %eax
-; CHECK-NEXT:    movl %edi, %ecx
-; CHECK-NEXT:    notl %ecx
-; CHECK-NEXT:    incl %edi
-; CHECK-NEXT:    testl %ecx, %edi
+; CHECK-NEXT:    blcicl %edi, %ecx
 ; CHECK-NEXT:    cmovnel %edx, %eax
 ; CHECK-NEXT:    retq
   %t0 = xor i32 %a, -1
@@ -377,10 +366,7 @@ define i64 @test_x86_tbm_blcic_u64_z2(i64 %a, i64 %b, i64 %c) nounwind {
 ; CHECK-LABEL: test_x86_tbm_blcic_u64_z2:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    movq %rsi, %rax
-; CHECK-NEXT:    movq %rdi, %rcx
-; CHECK-NEXT:    notq %rcx
-; CHECK-NEXT:    incq %rdi
-; CHECK-NEXT:    testq %rcx, %rdi
+; CHECK-NEXT:    blcicq %rdi, %rcx
 ; CHECK-NEXT:    cmovneq %rdx, %rax
 ; CHECK-NEXT:    retq
   %t0 = xor i64 %a, -1
@@ -418,9 +404,7 @@ define i32 @test_x86_tbm_blcmsk_u32_z2(i32 %a, i32 %b, i32 %c) nounwind {
 ; CHECK-LABEL: test_x86_tbm_blcmsk_u32_z2:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    movl %esi, %eax
-; CHECK-NEXT:    # kill: def $edi killed $edi def $rdi
-; CHECK-NEXT:    leal 1(%rdi), %ecx
-; CHECK-NEXT:    xorl %edi, %ecx
+; CHECK-NEXT:    blcmskl %edi, %ecx
 ; CHECK-NEXT:    cmovnel %edx, %eax
 ; CHECK-NEXT:    retq
   %t0 = add i32 %a, 1
@@ -457,8 +441,7 @@ define i64 @test_x86_tbm_blcmsk_u64_z2(i64 %a, i64 %b, i64 %c) nounwind {
 ; CHECK-LABEL: test_x86_tbm_blcmsk_u64_z2:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    movq %rsi, %rax
-; CHECK-NEXT:    leaq 1(%rdi), %rcx
-; CHECK-NEXT:    xorq %rdi, %rcx
+; CHECK-NEXT:    blcmskq %rdi, %rcx
 ; CHECK-NEXT:    cmovneq %rdx, %rax
 ; CHECK-NEXT:    retq
   %t0 = add i64 %a, 1
@@ -495,9 +478,7 @@ define i32 @test_x86_tbm_blcs_u32_z2(i32 %a, i32 %b, i32 %c) nounwind {
 ; CHECK-LABEL: test_x86_tbm_blcs_u32_z2:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    movl %esi, %eax
-; CHECK-NEXT:    # kill: def $edi killed $edi def $rdi
-; CHECK-NEXT:    leal 1(%rdi), %ecx
-; CHECK-NEXT:    orl %edi, %ecx
+; CHECK-NEXT:    blcsl %edi, %ecx
 ; CHECK-NEXT:    cmovnel %edx, %eax
 ; CHECK-NEXT:    retq
   %t0 = add i32 %a, 1
@@ -534,8 +515,7 @@ define i64 @test_x86_tbm_blcs_u64_z2(i64 %a, i64 %b, i64 %c) nounwind {
 ; CHECK-LABEL: test_x86_tbm_blcs_u64_z2:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    movq %rsi, %rax
-; CHECK-NEXT:    leaq 1(%rdi), %rcx
-; CHECK-NEXT:    orq %rdi, %rcx
+; CHECK-NEXT:    blcsq %rdi, %rcx
 ; CHECK-NEXT:    cmovneq %rdx, %rax
 ; CHECK-NEXT:    retq
   %t0 = add i64 %a, 1
@@ -572,9 +552,7 @@ define i32 @test_x86_tbm_blsfill_u32_z2(i32 %a, i32 %b, i32 %c) nounwind {
 ; CHECK-LABEL: test_x86_tbm_blsfill_u32_z2:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    movl %esi, %eax
-; CHECK-NEXT:    # kill: def $edi killed $edi def $rdi
-; CHECK-NEXT:    leal -1(%rdi), %ecx
-; CHECK-NEXT:    orl %edi, %ecx
+; CHECK-NEXT:    blsfilll %edi, %ecx
 ; CHECK-NEXT:    cmovnel %edx, %eax
 ; CHECK-NEXT:    retq
   %t0 = add i32 %a, -1
@@ -611,8 +589,7 @@ define i64 @test_x86_tbm_blsfill_u64_z2(i64 %a, i64 %b, i64 %c) nounwind {
 ; CHECK-LABEL: test_x86_tbm_blsfill_u64_z2:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    movq %rsi, %rax
-; CHECK-NEXT:    leaq -1(%rdi), %rcx
-; CHECK-NEXT:    orq %rdi, %rcx
+; CHECK-NEXT:    blsfillq %rdi, %rcx
 ; CHECK-NEXT:    cmovneq %rdx, %rax
 ; CHECK-NEXT:    retq
   %t0 = add i64 %a, -1
@@ -651,10 +628,7 @@ define i32 @test_x86_tbm_blsic_u32_z2(i32 %a, i32 %b, i32 %c) nounwind {
 ; CHECK-LABEL: test_x86_tbm_blsic_u32_z2:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    movl %esi, %eax
-; CHECK-NEXT:    movl %edi, %ecx
-; CHECK-NEXT:    notl %ecx
-; CHECK-NEXT:    decl %edi
-; CHECK-NEXT:    orl %ecx, %edi
+; CHECK-NEXT:    blsicl %edi, %ecx
 ; CHECK-NEXT:    cmovnel %edx, %eax
 ; CHECK-NEXT:    retq
   %t0 = xor i32 %a, -1
@@ -694,10 +668,7 @@ define i64 @test_x86_tbm_blsic_u64_z2(i64 %a, i64 %b, i64 %c) nounwind {
 ; CHECK-LABEL: test_x86_tbm_blsic_u64_z2:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    movq %rsi, %rax
-; CHECK-NEXT:    movq %rdi, %rcx
-; CHECK-NEXT:    notq %rcx
-; CHECK-NEXT:    decq %rdi
-; CHECK-NEXT:    orq %rcx, %rdi
+; CHECK-NEXT:    blsicq %rdi, %rcx
 ; CHECK-NEXT:    cmovneq %rdx, %rax
 ; CHECK-NEXT:    retq
   %t0 = xor i64 %a, -1
@@ -737,10 +708,7 @@ define i32 @test_x86_tbm_t1mskc_u32_z2(i32 %a, i32 %b, i32 %c) nounwind {
 ; CHECK-LABEL: test_x86_tbm_t1mskc_u32_z2:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    movl %esi, %eax
-; CHECK-NEXT:    movl %edi, %ecx
-; CHECK-NEXT:    notl %ecx
-; CHECK-NEXT:    incl %edi
-; CHECK-NEXT:    orl %ecx, %edi
+; CHECK-NEXT:    t1mskcl %edi, %ecx
 ; CHECK-NEXT:    cmovnel %edx, %eax
 ; CHECK-NEXT:    retq
   %t0 = xor i32 %a, -1
@@ -780,10 +748,7 @@ define i64 @test_x86_tbm_t1mskc_u64_z2(i64 %a, i64 %b, i64 %c) nounwind {
 ; CHECK-LABEL: test_x86_tbm_t1mskc_u64_z2:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    movq %rsi, %rax
-; CHECK-NEXT:    movq %rdi, %rcx
-; CHECK-NEXT:    notq %rcx
-; CHECK-NEXT:    incq %rdi
-; CHECK-NEXT:    orq %rcx, %rdi
+; CHECK-NEXT:    t1mskcq %rdi, %rcx
 ; CHECK-NEXT:    cmovneq %rdx, %rax
 ; CHECK-NEXT:    retq
   %t0 = xor i64 %a, -1
@@ -823,10 +788,7 @@ define i32 @test_x86_tbm_tzmsk_u32_z2(i32 %a, i32 %b, i32 %c) nounwind {
 ; CHECK-LABEL: test_x86_tbm_tzmsk_u32_z2:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    movl %esi, %eax
-; CHECK-NEXT:    movl %edi, %ecx
-; CHECK-NEXT:    notl %ecx
-; CHECK-NEXT:    decl %edi
-; CHECK-NEXT:    testl %edi, %ecx
+; CHECK-NEXT:    tzmskl %edi, %ecx
 ; CHECK-NEXT:    cmovnel %edx, %eax
 ; CHECK-NEXT:    retq
   %t0 = xor i32 %a, -1
@@ -866,10 +828,7 @@ define i64 @test_x86_tbm_tzmsk_u64_z2(i64 %a, i64 %b, i64 %c) nounwind {
 ; CHECK-LABEL: test_x86_tbm_tzmsk_u64_z2:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    movq %rsi, %rax
-; CHECK-NEXT:    movq %rdi, %rcx
-; CHECK-NEXT:    notq %rcx
-; CHECK-NEXT:    decq %rdi
-; CHECK-NEXT:    testq %rdi, %rcx
+; CHECK-NEXT:    tzmskq %rdi, %rcx
 ; CHECK-NEXT:    cmovneq %rdx, %rax
 ; CHECK-NEXT:    retq
   %t0 = xor i64 %a, -1
diff --git a/test/CodeGen/X86/tls-addr-non-leaf-function.ll b/test/CodeGen/X86/tls-addr-non-leaf-function.ll
index b9cab65465b87055fce432aa6454f936830892c6..6ebced191e0662020eeb5e1d95ca648d5e7c5efd 100644
--- a/test/CodeGen/X86/tls-addr-non-leaf-function.ll
+++ b/test/CodeGen/X86/tls-addr-non-leaf-function.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -relocation-model=pic -O2 -disable-fp-elim -o - | FileCheck %s
+; RUN: llc < %s -relocation-model=pic -O2 -frame-pointer=all -o - | FileCheck %s
 ; RUN: llc < %s -relocation-model=pic -O2 -o - | FileCheck %s
 
 ; This test runs twice with different options regarding the frame pointer:
diff --git a/test/CodeGen/X86/trunc-subvector.ll b/test/CodeGen/X86/trunc-subvector.ll
index 0355241ae5ffae7d2e6516a73760d67f34728d13..abd4fb45ea9d560ef5881706b1216f74d8fdb5ce 100644
--- a/test/CodeGen/X86/trunc-subvector.ll
+++ b/test/CodeGen/X86/trunc-subvector.ll
@@ -108,11 +108,9 @@ define <2 x i32> @test5(<8 x i32> %v) {
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpmovsxdq %xmm0, %ymm1
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm0
-; AVX2-NEXT:    vpbroadcastd %xmm0, %ymm0
-; AVX2-NEXT:    vpmovsxdq %xmm0, %ymm0
-; AVX2-NEXT:    vpalignr {{.*#+}} ymm0 = ymm1[8,9,10,11,12,13,14,15],ymm0[0,1,2,3,4,5,6,7],ymm1[24,25,26,27,28,29,30,31],ymm0[16,17,18,19,20,21,22,23]
-; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,1,2,3]
-; AVX2-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX2-NEXT:    vpmovsxdq %xmm0, %xmm0
+; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm1
+; AVX2-NEXT:    vpalignr {{.*#+}} xmm0 = xmm1[8,9,10,11,12,13,14,15],xmm0[0,1,2,3,4,5,6,7]
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
@@ -228,11 +226,9 @@ define <2 x i32> @test10(<8 x i32> %v) {
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpmovzxdq {{.*#+}} ymm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm0
-; AVX2-NEXT:    vpbroadcastd %xmm0, %ymm0
-; AVX2-NEXT:    vpmovzxdq {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
-; AVX2-NEXT:    vpalignr {{.*#+}} ymm0 = ymm1[8,9,10,11,12,13,14,15],ymm0[0,1,2,3,4,5,6,7],ymm1[24,25,26,27,28,29,30,31],ymm0[16,17,18,19,20,21,22,23]
-; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,1,2,3]
-; AVX2-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX2-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
+; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm1
+; AVX2-NEXT:    vpalignr {{.*#+}} xmm0 = xmm1[8,9,10,11,12,13,14,15],xmm0[0,1,2,3,4,5,6,7]
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
diff --git a/test/CodeGen/X86/twoaddr-dbg-value.mir b/test/CodeGen/X86/twoaddr-dbg-value.mir
new file mode 100644
index 0000000000000000000000000000000000000000..f2d6c5df58224ed1d465955211d1f2115eef7bbd
--- /dev/null
+++ b/test/CodeGen/X86/twoaddr-dbg-value.mir
@@ -0,0 +1,27 @@
+# RUN: llc -run-pass=livevars,twoaddressinstruction -mtriple=x86_64-- -o - %s | FileCheck %s
+---
+name:            foo
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $edi
+
+    %0:gr32 = COPY killed $edi
+    %1:gr32 = COPY killed %0
+    %4:gr32 = XOR32ri8 %1, 1, implicit-def dead $eflags
+    DBG_VALUE %4
+    %5:gr32 = COPY %4
+    PUSH32r killed %1, implicit-def $esp, implicit $esp
+    $eax = COPY killed %5
+    RETQ implicit killed $eax
+
+...
+
+# Verify that the DBG_VALUE instruction does not inhibit
+# TwoAddressInstructionPass::rescheduleMIBelowKill optimization
+
+# CHECK: PUSH32r %1, implicit-def $esp, implicit $esp
+# CHECK-NEXT: %2:gr32 = COPY killed %1
+# CHECK-NEXT: %2:gr32 = XOR32ri8 %2, 1, implicit-def dead $eflags
+# CHECK-NEXT: DBG_VALUE %2
+# CHECK-NEXT: %3:gr32 = COPY killed %2
diff --git a/test/CodeGen/X86/uadd_sat.ll b/test/CodeGen/X86/uadd_sat.ll
index f72d036288d045fb7a2309552efad1c6cfc8f232..677db07e196a215cbac21f1a014abba591a709f8 100644
--- a/test/CodeGen/X86/uadd_sat.ll
+++ b/test/CodeGen/X86/uadd_sat.ll
@@ -1,157 +1,129 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mcpu=generic -mtriple=x86_64-linux | FileCheck %s
-; RUN: llc < %s -mcpu=generic -mtriple=i686 -mattr=cmov | FileCheck %s --check-prefix=CHECK32
+; RUN: llc < %s -mtriple=i686 -mattr=cmov | FileCheck %s --check-prefixes=CHECK,X86
+; RUN: llc < %s -mtriple=x86_64-linux | FileCheck %s --check-prefixes=CHECK,X64
 
 declare  i4  @llvm.uadd.sat.i4   (i4,  i4)
 declare  i32 @llvm.uadd.sat.i32  (i32, i32)
 declare  i64 @llvm.uadd.sat.i64  (i64, i64)
 declare  <4 x i32> @llvm.uadd.sat.v4i32(<4 x i32>, <4 x i32>)
 
-define i32 @func(i32 %x, i32 %y) {
-; CHECK-LABEL: func:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    addl %esi, %edi
-; CHECK-NEXT:    movl $-1, %eax
-; CHECK-NEXT:    cmovael %edi, %eax
-; CHECK-NEXT:    retq
+define i32 @func(i32 %x, i32 %y) nounwind {
+; X86-LABEL: func:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    addl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl $-1, %eax
+; X86-NEXT:    cmovael %ecx, %eax
+; X86-NEXT:    retl
 ;
-; CHECK32-LABEL: func:
-; CHECK32:       # %bb.0:
-; CHECK32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; CHECK32-NEXT:    addl {{[0-9]+}}(%esp), %ecx
-; CHECK32-NEXT:    movl $-1, %eax
-; CHECK32-NEXT:    cmovael %ecx, %eax
-; CHECK32-NEXT:    retl
+; X64-LABEL: func:
+; X64:       # %bb.0:
+; X64-NEXT:    addl %esi, %edi
+; X64-NEXT:    movl $-1, %eax
+; X64-NEXT:    cmovael %edi, %eax
+; X64-NEXT:    retq
   %tmp = call i32 @llvm.uadd.sat.i32(i32 %x, i32 %y);
   ret i32 %tmp;
 }
 
-define i64 @func2(i64 %x, i64 %y) {
-; CHECK-LABEL: func2:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    addq %rsi, %rdi
-; CHECK-NEXT:    movq $-1, %rax
-; CHECK-NEXT:    cmovaeq %rdi, %rax
-; CHECK-NEXT:    retq
+define i64 @func2(i64 %x, i64 %y) nounwind {
+; X86-LABEL: func2:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    addl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    adcl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl $-1, %ecx
+; X86-NEXT:    cmovbl %ecx, %edx
+; X86-NEXT:    cmovbl %ecx, %eax
+; X86-NEXT:    retl
 ;
-; CHECK32-LABEL: func2:
-; CHECK32:       # %bb.0:
-; CHECK32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; CHECK32-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; CHECK32-NEXT:    addl {{[0-9]+}}(%esp), %eax
-; CHECK32-NEXT:    adcl {{[0-9]+}}(%esp), %edx
-; CHECK32-NEXT:    movl $-1, %ecx
-; CHECK32-NEXT:    cmovbl %ecx, %edx
-; CHECK32-NEXT:    cmovbl %ecx, %eax
-; CHECK32-NEXT:    retl
+; X64-LABEL: func2:
+; X64:       # %bb.0:
+; X64-NEXT:    addq %rsi, %rdi
+; X64-NEXT:    movq $-1, %rax
+; X64-NEXT:    cmovaeq %rdi, %rax
+; X64-NEXT:    retq
   %tmp = call i64 @llvm.uadd.sat.i64(i64 %x, i64 %y);
   ret i64 %tmp;
 }
 
-define i4 @func3(i4 %x, i4 %y) {
-; CHECK-LABEL: func3:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    shlb $4, %sil
-; CHECK-NEXT:    shlb $4, %dil
-; CHECK-NEXT:    addb %sil, %dil
-; CHECK-NEXT:    movb $-1, %al
-; CHECK-NEXT:    jb .LBB2_2
-; CHECK-NEXT:  # %bb.1:
-; CHECK-NEXT:    movl %edi, %eax
-; CHECK-NEXT:  .LBB2_2:
-; CHECK-NEXT:    shrb $4, %al
-; CHECK-NEXT:    retq
+define i4 @func3(i4 %x, i4 %y) nounwind {
+; X86-LABEL: func3:
+; X86:       # %bb.0:
+; X86-NEXT:    movb {{[0-9]+}}(%esp), %cl
+; X86-NEXT:    movb {{[0-9]+}}(%esp), %al
+; X86-NEXT:    shlb $4, %al
+; X86-NEXT:    shlb $4, %cl
+; X86-NEXT:    addb %al, %cl
+; X86-NEXT:    movb $-1, %al
+; X86-NEXT:    jb .LBB2_2
+; X86-NEXT:  # %bb.1:
+; X86-NEXT:    movl %ecx, %eax
+; X86-NEXT:  .LBB2_2:
+; X86-NEXT:    shrb $4, %al
+; X86-NEXT:    retl
 ;
-; CHECK32-LABEL: func3:
-; CHECK32:       # %bb.0:
-; CHECK32-NEXT:    movb {{[0-9]+}}(%esp), %cl
-; CHECK32-NEXT:    movb {{[0-9]+}}(%esp), %al
-; CHECK32-NEXT:    shlb $4, %al
-; CHECK32-NEXT:    shlb $4, %cl
-; CHECK32-NEXT:    addb %al, %cl
-; CHECK32-NEXT:    movb $-1, %al
-; CHECK32-NEXT:    jb .LBB2_2
-; CHECK32-NEXT:  # %bb.1:
-; CHECK32-NEXT:    movl %ecx, %eax
-; CHECK32-NEXT:  .LBB2_2:
-; CHECK32-NEXT:    shrb $4, %al
-; CHECK32-NEXT:    retl
+; X64-LABEL: func3:
+; X64:       # %bb.0:
+; X64-NEXT:    shlb $4, %sil
+; X64-NEXT:    shlb $4, %dil
+; X64-NEXT:    addb %sil, %dil
+; X64-NEXT:    movb $-1, %al
+; X64-NEXT:    jb .LBB2_2
+; X64-NEXT:  # %bb.1:
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:  .LBB2_2:
+; X64-NEXT:    shrb $4, %al
+; X64-NEXT:    retq
   %tmp = call i4 @llvm.uadd.sat.i4(i4 %x, i4 %y);
   ret i4 %tmp;
 }
 
-define <4 x i32> @vec(<4 x i32> %x, <4 x i32> %y) {
-; CHECK-LABEL: vec:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[3,1,2,3]
-; CHECK-NEXT:    movd %xmm2, %eax
-; CHECK-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[3,1,2,3]
-; CHECK-NEXT:    movd %xmm2, %ecx
-; CHECK-NEXT:    addl %eax, %ecx
-; CHECK-NEXT:    movl $-1, %eax
-; CHECK-NEXT:    cmovbl %eax, %ecx
-; CHECK-NEXT:    movd %ecx, %xmm2
-; CHECK-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[2,3,0,1]
-; CHECK-NEXT:    movd %xmm3, %ecx
-; CHECK-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[2,3,0,1]
-; CHECK-NEXT:    movd %xmm3, %edx
-; CHECK-NEXT:    addl %ecx, %edx
-; CHECK-NEXT:    cmovbl %eax, %edx
-; CHECK-NEXT:    movd %edx, %xmm3
-; CHECK-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
-; CHECK-NEXT:    movd %xmm1, %ecx
-; CHECK-NEXT:    movd %xmm0, %edx
-; CHECK-NEXT:    addl %ecx, %edx
-; CHECK-NEXT:    cmovbl %eax, %edx
-; CHECK-NEXT:    movd %edx, %xmm2
-; CHECK-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,2,3]
-; CHECK-NEXT:    movd %xmm1, %ecx
-; CHECK-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
-; CHECK-NEXT:    movd %xmm0, %edx
-; CHECK-NEXT:    addl %ecx, %edx
-; CHECK-NEXT:    cmovbl %eax, %edx
-; CHECK-NEXT:    movd %edx, %xmm0
-; CHECK-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
-; CHECK-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; CHECK-NEXT:    movdqa %xmm2, %xmm0
-; CHECK-NEXT:    retq
+define <4 x i32> @vec(<4 x i32> %x, <4 x i32> %y) nounwind {
+; X86-LABEL: vec:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %ebx
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    addl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    movl $-1, %ebx
+; X86-NEXT:    cmovbl %ebx, %edi
+; X86-NEXT:    addl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    cmovbl %ebx, %esi
+; X86-NEXT:    addl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    cmovbl %ebx, %edx
+; X86-NEXT:    addl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    cmovbl %ebx, %ecx
+; X86-NEXT:    movl %ecx, 12(%eax)
+; X86-NEXT:    movl %edx, 8(%eax)
+; X86-NEXT:    movl %esi, 4(%eax)
+; X86-NEXT:    movl %edi, (%eax)
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    popl %ebx
+; X86-NEXT:    retl $4
 ;
-; CHECK32-LABEL: vec:
-; CHECK32:       # %bb.0:
-; CHECK32-NEXT:    pushl %ebx
-; CHECK32-NEXT:    .cfi_def_cfa_offset 8
-; CHECK32-NEXT:    pushl %edi
-; CHECK32-NEXT:    .cfi_def_cfa_offset 12
-; CHECK32-NEXT:    pushl %esi
-; CHECK32-NEXT:    .cfi_def_cfa_offset 16
-; CHECK32-NEXT:    .cfi_offset %esi, -16
-; CHECK32-NEXT:    .cfi_offset %edi, -12
-; CHECK32-NEXT:    .cfi_offset %ebx, -8
-; CHECK32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; CHECK32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; CHECK32-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; CHECK32-NEXT:    movl {{[0-9]+}}(%esp), %esi
-; CHECK32-NEXT:    movl {{[0-9]+}}(%esp), %edi
-; CHECK32-NEXT:    addl {{[0-9]+}}(%esp), %edi
-; CHECK32-NEXT:    movl $-1, %ebx
-; CHECK32-NEXT:    cmovbl %ebx, %edi
-; CHECK32-NEXT:    addl {{[0-9]+}}(%esp), %esi
-; CHECK32-NEXT:    cmovbl %ebx, %esi
-; CHECK32-NEXT:    addl {{[0-9]+}}(%esp), %edx
-; CHECK32-NEXT:    cmovbl %ebx, %edx
-; CHECK32-NEXT:    addl {{[0-9]+}}(%esp), %ecx
-; CHECK32-NEXT:    cmovbl %ebx, %ecx
-; CHECK32-NEXT:    movl %ecx, 12(%eax)
-; CHECK32-NEXT:    movl %edx, 8(%eax)
-; CHECK32-NEXT:    movl %esi, 4(%eax)
-; CHECK32-NEXT:    movl %edi, (%eax)
-; CHECK32-NEXT:    popl %esi
-; CHECK32-NEXT:    .cfi_def_cfa_offset 12
-; CHECK32-NEXT:    popl %edi
-; CHECK32-NEXT:    .cfi_def_cfa_offset 8
-; CHECK32-NEXT:    popl %ebx
-; CHECK32-NEXT:    .cfi_def_cfa_offset 4
-; CHECK32-NEXT:    retl $4
+; X64-LABEL: vec:
+; X64:       # %bb.0:
+; X64-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648,2147483648,2147483648]
+; X64-NEXT:    pxor %xmm0, %xmm2
+; X64-NEXT:    movdqa {{.*#+}} xmm3 = [2147483647,2147483647,2147483647,2147483647]
+; X64-NEXT:    pxor %xmm1, %xmm3
+; X64-NEXT:    pcmpgtd %xmm2, %xmm3
+; X64-NEXT:    pand %xmm3, %xmm0
+; X64-NEXT:    pcmpeqd %xmm2, %xmm2
+; X64-NEXT:    pxor %xmm3, %xmm2
+; X64-NEXT:    movdqa %xmm1, %xmm3
+; X64-NEXT:    pandn %xmm2, %xmm3
+; X64-NEXT:    por %xmm3, %xmm0
+; X64-NEXT:    paddd %xmm1, %xmm0
+; X64-NEXT:    retq
   %tmp = call <4 x i32> @llvm.uadd.sat.v4i32(<4 x i32> %x, <4 x i32> %y);
   ret <4 x i32> %tmp;
 }
diff --git a/test/CodeGen/X86/uadd_sat_vec.ll b/test/CodeGen/X86/uadd_sat_vec.ll
index 98f6d0f19237c226192a2e504a1ba8c92134519b..c52c489bcaddeba0fa5b7565af615fb6d18f6ccb 100644
--- a/test/CodeGen/X86/uadd_sat_vec.ll
+++ b/test/CodeGen/X86/uadd_sat_vec.ll
@@ -26,8 +26,14 @@ declare <32 x i16> @llvm.uadd.sat.v32i16(<32 x i16>, <32 x i16>)
 declare <16 x i1> @llvm.uadd.sat.v16i1(<16 x i1>, <16 x i1>)
 declare <16 x i4> @llvm.uadd.sat.v16i4(<16 x i4>, <16 x i4>)
 
-declare <4 x i32> @llvm.uadd.sat.v4i32(<4 x i32>, <4 x i32>)
 declare <2 x i32> @llvm.uadd.sat.v2i32(<2 x i32>, <2 x i32>)
+declare <4 x i32> @llvm.uadd.sat.v4i32(<4 x i32>, <4 x i32>)
+declare <8 x i32> @llvm.uadd.sat.v8i32(<8 x i32>, <8 x i32>)
+declare <16 x i32> @llvm.uadd.sat.v16i32(<16 x i32>, <16 x i32>)
+declare <2 x i64> @llvm.uadd.sat.v2i64(<2 x i64>, <2 x i64>)
+declare <4 x i64> @llvm.uadd.sat.v4i64(<4 x i64>, <4 x i64>)
+declare <8 x i64> @llvm.uadd.sat.v8i64(<8 x i64>, <8 x i64>)
+
 declare <4 x i24> @llvm.uadd.sat.v4i24(<4 x i24>, <4 x i24>)
 declare <2 x i128> @llvm.uadd.sat.v2i128(<2 x i128>, <2 x i128>)
 
@@ -626,352 +632,1016 @@ define <16 x i1> @v16i1(<16 x i1> %x, <16 x i1> %y) nounwind {
 
 ; Expanded
 
+define <2 x i32> @v2i32(<2 x i32> %x, <2 x i32> %y) nounwind {
+; SSE2-LABEL: v2i32:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    psllq $32, %xmm0
+; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [9223372039002259456,9223372039002259456]
+; SSE2-NEXT:    pxor %xmm0, %xmm2
+; SSE2-NEXT:    psllq $32, %xmm1
+; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [9223372034707292159,9223372034707292159]
+; SSE2-NEXT:    pxor %xmm1, %xmm3
+; SSE2-NEXT:    movdqa %xmm3, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm2, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm2, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[1,1,3,3]
+; SSE2-NEXT:    pand %xmm5, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[1,1,3,3]
+; SSE2-NEXT:    por %xmm2, %xmm3
+; SSE2-NEXT:    pand %xmm3, %xmm0
+; SSE2-NEXT:    pcmpeqd %xmm2, %xmm2
+; SSE2-NEXT:    pxor %xmm3, %xmm2
+; SSE2-NEXT:    movdqa %xmm1, %xmm3
+; SSE2-NEXT:    pandn %xmm2, %xmm3
+; SSE2-NEXT:    por %xmm3, %xmm0
+; SSE2-NEXT:    paddq %xmm1, %xmm0
+; SSE2-NEXT:    psrlq $32, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: v2i32:
+; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    psllq $32, %xmm0
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm2 = [9223372039002259456,9223372039002259456]
+; SSSE3-NEXT:    pxor %xmm0, %xmm2
+; SSSE3-NEXT:    psllq $32, %xmm1
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm3 = [9223372034707292159,9223372034707292159]
+; SSSE3-NEXT:    pxor %xmm1, %xmm3
+; SSSE3-NEXT:    movdqa %xmm3, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm3
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm5, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    por %xmm2, %xmm3
+; SSSE3-NEXT:    pand %xmm3, %xmm0
+; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm2
+; SSSE3-NEXT:    pxor %xmm3, %xmm2
+; SSSE3-NEXT:    movdqa %xmm1, %xmm3
+; SSSE3-NEXT:    pandn %xmm2, %xmm3
+; SSSE3-NEXT:    por %xmm3, %xmm0
+; SSSE3-NEXT:    paddq %xmm1, %xmm0
+; SSSE3-NEXT:    psrlq $32, %xmm0
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: v2i32:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movdqa %xmm0, %xmm2
+; SSE41-NEXT:    psllq $32, %xmm2
+; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [9223372039002259456,9223372039002259456]
+; SSE41-NEXT:    pxor %xmm2, %xmm0
+; SSE41-NEXT:    psllq $32, %xmm1
+; SSE41-NEXT:    movdqa {{.*#+}} xmm3 = [9223372034707292159,9223372034707292159]
+; SSE41-NEXT:    pxor %xmm1, %xmm3
+; SSE41-NEXT:    movdqa %xmm3, %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm4
+; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm3
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
+; SSE41-NEXT:    pand %xmm5, %xmm0
+; SSE41-NEXT:    por %xmm4, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm3, %xmm3
+; SSE41-NEXT:    pxor %xmm1, %xmm3
+; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm3
+; SSE41-NEXT:    paddq %xmm1, %xmm3
+; SSE41-NEXT:    psrlq $32, %xmm3
+; SSE41-NEXT:    movdqa %xmm3, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: v2i32:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpsllq $32, %xmm1, %xmm1
+; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
+; AVX1-NEXT:    vpxor %xmm2, %xmm1, %xmm2
+; AVX1-NEXT:    vpsllq $32, %xmm0, %xmm0
+; AVX1-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm3
+; AVX1-NEXT:    vpxor {{.*}}(%rip), %xmm1, %xmm4
+; AVX1-NEXT:    vpcmpgtq %xmm3, %xmm4, %xmm3
+; AVX1-NEXT:    vblendvpd %xmm3, %xmm0, %xmm2, %xmm0
+; AVX1-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpsrlq $32, %xmm0, %xmm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: v2i32:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpsllq $32, %xmm1, %xmm1
+; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
+; AVX2-NEXT:    vpxor %xmm2, %xmm1, %xmm2
+; AVX2-NEXT:    vpsllq $32, %xmm0, %xmm0
+; AVX2-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm3
+; AVX2-NEXT:    vpxor {{.*}}(%rip), %xmm1, %xmm4
+; AVX2-NEXT:    vpcmpgtq %xmm3, %xmm4, %xmm3
+; AVX2-NEXT:    vblendvpd %xmm3, %xmm0, %xmm2, %xmm0
+; AVX2-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpsrlq $32, %xmm0, %xmm0
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: v2i32:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpsllq $32, %xmm0, %xmm0
+; AVX512-NEXT:    vpsllq $32, %xmm1, %xmm1
+; AVX512-NEXT:    vmovdqa %xmm1, %xmm2
+; AVX512-NEXT:    vpternlogq $15, %xmm1, %xmm1, %xmm2
+; AVX512-NEXT:    vpminuq %xmm2, %xmm0, %xmm0
+; AVX512-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vpsrlq $32, %xmm0, %xmm0
+; AVX512-NEXT:    retq
+  %z = call <2 x i32> @llvm.uadd.sat.v2i32(<2 x i32> %x, <2 x i32> %y)
+  ret <2 x i32> %z
+}
+
 define <4 x i32> @v4i32(<4 x i32> %x, <4 x i32> %y) nounwind {
 ; SSE2-LABEL: v4i32:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[3,1,2,3]
-; SSE2-NEXT:    movd %xmm2, %eax
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[3,1,2,3]
-; SSE2-NEXT:    movd %xmm2, %ecx
-; SSE2-NEXT:    addl %eax, %ecx
-; SSE2-NEXT:    movl $-1, %eax
-; SSE2-NEXT:    cmovbl %eax, %ecx
-; SSE2-NEXT:    movd %ecx, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[2,3,0,1]
-; SSE2-NEXT:    movd %xmm3, %ecx
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[2,3,0,1]
-; SSE2-NEXT:    movd %xmm3, %edx
-; SSE2-NEXT:    addl %ecx, %edx
-; SSE2-NEXT:    cmovbl %eax, %edx
-; SSE2-NEXT:    movd %edx, %xmm3
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
-; SSE2-NEXT:    movd %xmm1, %ecx
-; SSE2-NEXT:    movd %xmm0, %edx
-; SSE2-NEXT:    addl %ecx, %edx
-; SSE2-NEXT:    cmovbl %eax, %edx
-; SSE2-NEXT:    movd %edx, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,2,3]
-; SSE2-NEXT:    movd %xmm1, %ecx
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
-; SSE2-NEXT:    movd %xmm0, %edx
-; SSE2-NEXT:    addl %ecx, %edx
-; SSE2-NEXT:    cmovbl %eax, %edx
-; SSE2-NEXT:    movd %edx, %xmm0
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
-; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; SSE2-NEXT:    movdqa %xmm2, %xmm0
+; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648,2147483648,2147483648]
+; SSE2-NEXT:    pxor %xmm0, %xmm2
+; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [2147483647,2147483647,2147483647,2147483647]
+; SSE2-NEXT:    pxor %xmm1, %xmm3
+; SSE2-NEXT:    pcmpgtd %xmm2, %xmm3
+; SSE2-NEXT:    pand %xmm3, %xmm0
+; SSE2-NEXT:    pcmpeqd %xmm2, %xmm2
+; SSE2-NEXT:    pxor %xmm3, %xmm2
+; SSE2-NEXT:    movdqa %xmm1, %xmm3
+; SSE2-NEXT:    pandn %xmm2, %xmm3
+; SSE2-NEXT:    por %xmm3, %xmm0
+; SSE2-NEXT:    paddd %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSSE3-LABEL: v4i32:
 ; SSSE3:       # %bb.0:
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[3,1,2,3]
-; SSSE3-NEXT:    movd %xmm2, %eax
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[3,1,2,3]
-; SSSE3-NEXT:    movd %xmm2, %ecx
-; SSSE3-NEXT:    addl %eax, %ecx
-; SSSE3-NEXT:    movl $-1, %eax
-; SSSE3-NEXT:    cmovbl %eax, %ecx
-; SSSE3-NEXT:    movd %ecx, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[2,3,0,1]
-; SSSE3-NEXT:    movd %xmm3, %ecx
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[2,3,0,1]
-; SSSE3-NEXT:    movd %xmm3, %edx
-; SSSE3-NEXT:    addl %ecx, %edx
-; SSSE3-NEXT:    cmovbl %eax, %edx
-; SSSE3-NEXT:    movd %edx, %xmm3
-; SSSE3-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
-; SSSE3-NEXT:    movd %xmm1, %ecx
-; SSSE3-NEXT:    movd %xmm0, %edx
-; SSSE3-NEXT:    addl %ecx, %edx
-; SSSE3-NEXT:    cmovbl %eax, %edx
-; SSSE3-NEXT:    movd %edx, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,2,3]
-; SSSE3-NEXT:    movd %xmm1, %ecx
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
-; SSSE3-NEXT:    movd %xmm0, %edx
-; SSSE3-NEXT:    addl %ecx, %edx
-; SSSE3-NEXT:    cmovbl %eax, %edx
-; SSSE3-NEXT:    movd %edx, %xmm0
-; SSSE3-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
-; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; SSSE3-NEXT:    movdqa %xmm2, %xmm0
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648,2147483648,2147483648]
+; SSSE3-NEXT:    pxor %xmm0, %xmm2
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm3 = [2147483647,2147483647,2147483647,2147483647]
+; SSSE3-NEXT:    pxor %xmm1, %xmm3
+; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm3
+; SSSE3-NEXT:    pand %xmm3, %xmm0
+; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm2
+; SSSE3-NEXT:    pxor %xmm3, %xmm2
+; SSSE3-NEXT:    movdqa %xmm1, %xmm3
+; SSSE3-NEXT:    pandn %xmm2, %xmm3
+; SSSE3-NEXT:    por %xmm3, %xmm0
+; SSSE3-NEXT:    paddd %xmm1, %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: v4i32:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    pextrd $1, %xmm1, %eax
-; SSE41-NEXT:    pextrd $1, %xmm0, %ecx
-; SSE41-NEXT:    addl %eax, %ecx
-; SSE41-NEXT:    movl $-1, %eax
-; SSE41-NEXT:    cmovbl %eax, %ecx
-; SSE41-NEXT:    movd %xmm1, %edx
-; SSE41-NEXT:    movd %xmm0, %esi
-; SSE41-NEXT:    addl %edx, %esi
-; SSE41-NEXT:    cmovbl %eax, %esi
-; SSE41-NEXT:    movd %esi, %xmm2
-; SSE41-NEXT:    pinsrd $1, %ecx, %xmm2
-; SSE41-NEXT:    pextrd $2, %xmm1, %ecx
-; SSE41-NEXT:    pextrd $2, %xmm0, %edx
-; SSE41-NEXT:    addl %ecx, %edx
-; SSE41-NEXT:    cmovbl %eax, %edx
-; SSE41-NEXT:    pinsrd $2, %edx, %xmm2
-; SSE41-NEXT:    pextrd $3, %xmm1, %ecx
-; SSE41-NEXT:    pextrd $3, %xmm0, %edx
-; SSE41-NEXT:    addl %ecx, %edx
-; SSE41-NEXT:    cmovbl %eax, %edx
-; SSE41-NEXT:    pinsrd $3, %edx, %xmm2
-; SSE41-NEXT:    movdqa %xmm2, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm2, %xmm2
+; SSE41-NEXT:    pxor %xmm1, %xmm2
+; SSE41-NEXT:    pminud %xmm2, %xmm0
+; SSE41-NEXT:    paddd %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
-; AVX-LABEL: v4i32:
-; AVX:       # %bb.0:
-; AVX-NEXT:    vpextrd $1, %xmm1, %eax
-; AVX-NEXT:    vpextrd $1, %xmm0, %ecx
-; AVX-NEXT:    addl %eax, %ecx
-; AVX-NEXT:    movl $-1, %eax
-; AVX-NEXT:    cmovbl %eax, %ecx
-; AVX-NEXT:    vmovd %xmm1, %edx
-; AVX-NEXT:    vmovd %xmm0, %esi
-; AVX-NEXT:    addl %edx, %esi
-; AVX-NEXT:    cmovbl %eax, %esi
-; AVX-NEXT:    vmovd %esi, %xmm2
-; AVX-NEXT:    vpinsrd $1, %ecx, %xmm2, %xmm2
-; AVX-NEXT:    vpextrd $2, %xmm1, %ecx
-; AVX-NEXT:    vpextrd $2, %xmm0, %edx
-; AVX-NEXT:    addl %ecx, %edx
-; AVX-NEXT:    cmovbl %eax, %edx
-; AVX-NEXT:    vpinsrd $2, %edx, %xmm2, %xmm2
-; AVX-NEXT:    vpextrd $3, %xmm1, %ecx
-; AVX-NEXT:    vpextrd $3, %xmm0, %edx
-; AVX-NEXT:    addl %ecx, %edx
-; AVX-NEXT:    cmovbl %eax, %edx
-; AVX-NEXT:    vpinsrd $3, %edx, %xmm2, %xmm0
-; AVX-NEXT:    retq
+; AVX1-LABEL: v4i32:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
+; AVX1-NEXT:    vpxor %xmm2, %xmm1, %xmm2
+; AVX1-NEXT:    vpminud %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: v4i32:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
+; AVX2-NEXT:    vpxor %xmm2, %xmm1, %xmm2
+; AVX2-NEXT:    vpminud %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: v4i32:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vmovdqa %xmm1, %xmm2
+; AVX512-NEXT:    vpternlogq $15, %xmm1, %xmm1, %xmm2
+; AVX512-NEXT:    vpminud %xmm2, %xmm0, %xmm0
+; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    retq
   %z = call <4 x i32> @llvm.uadd.sat.v4i32(<4 x i32> %x, <4 x i32> %y)
   ret <4 x i32> %z
 }
 
-define <2 x i32> @v2i32(<2 x i32> %x, <2 x i32> %y) nounwind {
-; SSE2-LABEL: v2i32:
+define <8 x i32> @v8i32(<8 x i32> %x, <8 x i32> %y) nounwind {
+; SSE2-LABEL: v8i32:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    psllq $32, %xmm1
-; SSE2-NEXT:    movq %xmm1, %rax
-; SSE2-NEXT:    psllq $32, %xmm0
-; SSE2-NEXT:    movq %xmm0, %rcx
-; SSE2-NEXT:    addq %rax, %rcx
-; SSE2-NEXT:    movq $-1, %rax
-; SSE2-NEXT:    cmovbq %rax, %rcx
-; SSE2-NEXT:    movq %rcx, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
-; SSE2-NEXT:    movq %xmm1, %rcx
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; SSE2-NEXT:    movq %xmm0, %rdx
-; SSE2-NEXT:    addq %rcx, %rdx
-; SSE2-NEXT:    cmovbq %rax, %rdx
-; SSE2-NEXT:    movq %rdx, %xmm0
-; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm0[0]
-; SSE2-NEXT:    psrlq $32, %xmm2
-; SSE2-NEXT:    movdqa %xmm2, %xmm0
+; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [2147483648,2147483648,2147483648,2147483648]
+; SSE2-NEXT:    movdqa %xmm0, %xmm5
+; SSE2-NEXT:    pxor %xmm4, %xmm5
+; SSE2-NEXT:    movdqa {{.*#+}} xmm6 = [2147483647,2147483647,2147483647,2147483647]
+; SSE2-NEXT:    movdqa %xmm2, %xmm7
+; SSE2-NEXT:    pxor %xmm6, %xmm7
+; SSE2-NEXT:    pcmpgtd %xmm5, %xmm7
+; SSE2-NEXT:    pand %xmm7, %xmm0
+; SSE2-NEXT:    pcmpeqd %xmm8, %xmm8
+; SSE2-NEXT:    pxor %xmm8, %xmm7
+; SSE2-NEXT:    movdqa %xmm2, %xmm5
+; SSE2-NEXT:    pandn %xmm7, %xmm5
+; SSE2-NEXT:    por %xmm5, %xmm0
+; SSE2-NEXT:    paddd %xmm2, %xmm0
+; SSE2-NEXT:    pxor %xmm1, %xmm4
+; SSE2-NEXT:    pxor %xmm3, %xmm6
+; SSE2-NEXT:    pcmpgtd %xmm4, %xmm6
+; SSE2-NEXT:    pand %xmm6, %xmm1
+; SSE2-NEXT:    pxor %xmm8, %xmm6
+; SSE2-NEXT:    movdqa %xmm3, %xmm2
+; SSE2-NEXT:    pandn %xmm6, %xmm2
+; SSE2-NEXT:    por %xmm2, %xmm1
+; SSE2-NEXT:    paddd %xmm3, %xmm1
 ; SSE2-NEXT:    retq
 ;
-; SSSE3-LABEL: v2i32:
+; SSSE3-LABEL: v8i32:
 ; SSSE3:       # %bb.0:
-; SSSE3-NEXT:    psllq $32, %xmm1
-; SSSE3-NEXT:    movq %xmm1, %rax
-; SSSE3-NEXT:    psllq $32, %xmm0
-; SSSE3-NEXT:    movq %xmm0, %rcx
-; SSSE3-NEXT:    addq %rax, %rcx
-; SSSE3-NEXT:    movq $-1, %rax
-; SSSE3-NEXT:    cmovbq %rax, %rcx
-; SSSE3-NEXT:    movq %rcx, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
-; SSSE3-NEXT:    movq %xmm1, %rcx
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; SSSE3-NEXT:    movq %xmm0, %rdx
-; SSSE3-NEXT:    addq %rcx, %rdx
-; SSSE3-NEXT:    cmovbq %rax, %rdx
-; SSSE3-NEXT:    movq %rdx, %xmm0
-; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm0[0]
-; SSSE3-NEXT:    psrlq $32, %xmm2
-; SSSE3-NEXT:    movdqa %xmm2, %xmm0
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm4 = [2147483648,2147483648,2147483648,2147483648]
+; SSSE3-NEXT:    movdqa %xmm0, %xmm5
+; SSSE3-NEXT:    pxor %xmm4, %xmm5
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm6 = [2147483647,2147483647,2147483647,2147483647]
+; SSSE3-NEXT:    movdqa %xmm2, %xmm7
+; SSSE3-NEXT:    pxor %xmm6, %xmm7
+; SSSE3-NEXT:    pcmpgtd %xmm5, %xmm7
+; SSSE3-NEXT:    pand %xmm7, %xmm0
+; SSSE3-NEXT:    pcmpeqd %xmm8, %xmm8
+; SSSE3-NEXT:    pxor %xmm8, %xmm7
+; SSSE3-NEXT:    movdqa %xmm2, %xmm5
+; SSSE3-NEXT:    pandn %xmm7, %xmm5
+; SSSE3-NEXT:    por %xmm5, %xmm0
+; SSSE3-NEXT:    paddd %xmm2, %xmm0
+; SSSE3-NEXT:    pxor %xmm1, %xmm4
+; SSSE3-NEXT:    pxor %xmm3, %xmm6
+; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm6
+; SSSE3-NEXT:    pand %xmm6, %xmm1
+; SSSE3-NEXT:    pxor %xmm8, %xmm6
+; SSSE3-NEXT:    movdqa %xmm3, %xmm2
+; SSSE3-NEXT:    pandn %xmm6, %xmm2
+; SSSE3-NEXT:    por %xmm2, %xmm1
+; SSSE3-NEXT:    paddd %xmm3, %xmm1
 ; SSSE3-NEXT:    retq
 ;
-; SSE41-LABEL: v2i32:
+; SSE41-LABEL: v8i32:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    psllq $32, %xmm1
-; SSE41-NEXT:    pextrq $1, %xmm1, %rax
-; SSE41-NEXT:    psllq $32, %xmm0
-; SSE41-NEXT:    pextrq $1, %xmm0, %rcx
-; SSE41-NEXT:    addq %rax, %rcx
-; SSE41-NEXT:    movq $-1, %rax
-; SSE41-NEXT:    cmovbq %rax, %rcx
-; SSE41-NEXT:    movq %rcx, %xmm2
-; SSE41-NEXT:    movq %xmm1, %rcx
-; SSE41-NEXT:    movq %xmm0, %rdx
-; SSE41-NEXT:    addq %rcx, %rdx
-; SSE41-NEXT:    cmovbq %rax, %rdx
-; SSE41-NEXT:    movq %rdx, %xmm0
-; SSE41-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
-; SSE41-NEXT:    psrlq $32, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm4, %xmm4
+; SSE41-NEXT:    movdqa %xmm2, %xmm5
+; SSE41-NEXT:    pxor %xmm4, %xmm5
+; SSE41-NEXT:    pminud %xmm5, %xmm0
+; SSE41-NEXT:    paddd %xmm2, %xmm0
+; SSE41-NEXT:    pxor %xmm3, %xmm4
+; SSE41-NEXT:    pminud %xmm4, %xmm1
+; SSE41-NEXT:    paddd %xmm3, %xmm1
 ; SSE41-NEXT:    retq
 ;
-; AVX-LABEL: v2i32:
-; AVX:       # %bb.0:
-; AVX-NEXT:    vpsllq $32, %xmm1, %xmm1
-; AVX-NEXT:    vpextrq $1, %xmm1, %rax
-; AVX-NEXT:    vpsllq $32, %xmm0, %xmm0
-; AVX-NEXT:    vpextrq $1, %xmm0, %rcx
-; AVX-NEXT:    addq %rax, %rcx
-; AVX-NEXT:    movq $-1, %rax
-; AVX-NEXT:    cmovbq %rax, %rcx
-; AVX-NEXT:    vmovq %rcx, %xmm2
-; AVX-NEXT:    vmovq %xmm1, %rcx
-; AVX-NEXT:    vmovq %xmm0, %rdx
-; AVX-NEXT:    addq %rcx, %rdx
-; AVX-NEXT:    cmovbq %rax, %rdx
-; AVX-NEXT:    vmovq %rdx, %xmm0
-; AVX-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
-; AVX-NEXT:    vpsrlq $32, %xmm0, %xmm0
-; AVX-NEXT:    retq
-  %z = call <2 x i32> @llvm.uadd.sat.v2i32(<2 x i32> %x, <2 x i32> %y)
-  ret <2 x i32> %z
+; AVX1-LABEL: v8i32:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vxorps %xmm2, %xmm2, %xmm2
+; AVX1-NEXT:    vcmptrueps %ymm2, %ymm2, %ymm2
+; AVX1-NEXT:    vxorps %ymm2, %ymm1, %ymm2
+; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm3
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
+; AVX1-NEXT:    vpminud %xmm3, %xmm4, %xmm3
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm4
+; AVX1-NEXT:    vpaddd %xmm4, %xmm3, %xmm3
+; AVX1-NEXT:    vpminud %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: v8i32:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpcmpeqd %ymm2, %ymm2, %ymm2
+; AVX2-NEXT:    vpxor %ymm2, %ymm1, %ymm2
+; AVX2-NEXT:    vpminud %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vpaddd %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: v8i32:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vmovdqa %ymm1, %ymm2
+; AVX512-NEXT:    vpternlogq $15, %ymm1, %ymm1, %ymm2
+; AVX512-NEXT:    vpminud %ymm2, %ymm0, %ymm0
+; AVX512-NEXT:    vpaddd %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    retq
+  %z = call <8 x i32> @llvm.uadd.sat.v8i32(<8 x i32> %x, <8 x i32> %y)
+  ret <8 x i32> %z
+}
+
+define <16 x i32> @v16i32(<16 x i32> %x, <16 x i32> %y) nounwind {
+; SSE2-LABEL: v16i32:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa {{.*#+}} xmm9 = [2147483648,2147483648,2147483648,2147483648]
+; SSE2-NEXT:    movdqa %xmm0, %xmm10
+; SSE2-NEXT:    pxor %xmm9, %xmm10
+; SSE2-NEXT:    movdqa {{.*#+}} xmm8 = [2147483647,2147483647,2147483647,2147483647]
+; SSE2-NEXT:    movdqa %xmm4, %xmm11
+; SSE2-NEXT:    pxor %xmm8, %xmm11
+; SSE2-NEXT:    pcmpgtd %xmm10, %xmm11
+; SSE2-NEXT:    pand %xmm11, %xmm0
+; SSE2-NEXT:    pcmpeqd %xmm10, %xmm10
+; SSE2-NEXT:    pxor %xmm10, %xmm11
+; SSE2-NEXT:    movdqa %xmm4, %xmm12
+; SSE2-NEXT:    pandn %xmm11, %xmm12
+; SSE2-NEXT:    por %xmm12, %xmm0
+; SSE2-NEXT:    paddd %xmm4, %xmm0
+; SSE2-NEXT:    movdqa %xmm1, %xmm11
+; SSE2-NEXT:    pxor %xmm9, %xmm11
+; SSE2-NEXT:    movdqa %xmm5, %xmm12
+; SSE2-NEXT:    pxor %xmm8, %xmm12
+; SSE2-NEXT:    pcmpgtd %xmm11, %xmm12
+; SSE2-NEXT:    pand %xmm12, %xmm1
+; SSE2-NEXT:    pxor %xmm10, %xmm12
+; SSE2-NEXT:    movdqa %xmm5, %xmm4
+; SSE2-NEXT:    pandn %xmm12, %xmm4
+; SSE2-NEXT:    por %xmm4, %xmm1
+; SSE2-NEXT:    paddd %xmm5, %xmm1
+; SSE2-NEXT:    movdqa %xmm2, %xmm4
+; SSE2-NEXT:    pxor %xmm9, %xmm4
+; SSE2-NEXT:    movdqa %xmm6, %xmm5
+; SSE2-NEXT:    pxor %xmm8, %xmm5
+; SSE2-NEXT:    pcmpgtd %xmm4, %xmm5
+; SSE2-NEXT:    pand %xmm5, %xmm2
+; SSE2-NEXT:    pxor %xmm10, %xmm5
+; SSE2-NEXT:    movdqa %xmm6, %xmm4
+; SSE2-NEXT:    pandn %xmm5, %xmm4
+; SSE2-NEXT:    por %xmm4, %xmm2
+; SSE2-NEXT:    paddd %xmm6, %xmm2
+; SSE2-NEXT:    pxor %xmm3, %xmm9
+; SSE2-NEXT:    pxor %xmm7, %xmm8
+; SSE2-NEXT:    pcmpgtd %xmm9, %xmm8
+; SSE2-NEXT:    pand %xmm8, %xmm3
+; SSE2-NEXT:    pxor %xmm10, %xmm8
+; SSE2-NEXT:    movdqa %xmm7, %xmm4
+; SSE2-NEXT:    pandn %xmm8, %xmm4
+; SSE2-NEXT:    por %xmm4, %xmm3
+; SSE2-NEXT:    paddd %xmm7, %xmm3
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: v16i32:
+; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm9 = [2147483648,2147483648,2147483648,2147483648]
+; SSSE3-NEXT:    movdqa %xmm0, %xmm10
+; SSSE3-NEXT:    pxor %xmm9, %xmm10
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm8 = [2147483647,2147483647,2147483647,2147483647]
+; SSSE3-NEXT:    movdqa %xmm4, %xmm11
+; SSSE3-NEXT:    pxor %xmm8, %xmm11
+; SSSE3-NEXT:    pcmpgtd %xmm10, %xmm11
+; SSSE3-NEXT:    pand %xmm11, %xmm0
+; SSSE3-NEXT:    pcmpeqd %xmm10, %xmm10
+; SSSE3-NEXT:    pxor %xmm10, %xmm11
+; SSSE3-NEXT:    movdqa %xmm4, %xmm12
+; SSSE3-NEXT:    pandn %xmm11, %xmm12
+; SSSE3-NEXT:    por %xmm12, %xmm0
+; SSSE3-NEXT:    paddd %xmm4, %xmm0
+; SSSE3-NEXT:    movdqa %xmm1, %xmm11
+; SSSE3-NEXT:    pxor %xmm9, %xmm11
+; SSSE3-NEXT:    movdqa %xmm5, %xmm12
+; SSSE3-NEXT:    pxor %xmm8, %xmm12
+; SSSE3-NEXT:    pcmpgtd %xmm11, %xmm12
+; SSSE3-NEXT:    pand %xmm12, %xmm1
+; SSSE3-NEXT:    pxor %xmm10, %xmm12
+; SSSE3-NEXT:    movdqa %xmm5, %xmm4
+; SSSE3-NEXT:    pandn %xmm12, %xmm4
+; SSSE3-NEXT:    por %xmm4, %xmm1
+; SSSE3-NEXT:    paddd %xmm5, %xmm1
+; SSSE3-NEXT:    movdqa %xmm2, %xmm4
+; SSSE3-NEXT:    pxor %xmm9, %xmm4
+; SSSE3-NEXT:    movdqa %xmm6, %xmm5
+; SSSE3-NEXT:    pxor %xmm8, %xmm5
+; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm5
+; SSSE3-NEXT:    pand %xmm5, %xmm2
+; SSSE3-NEXT:    pxor %xmm10, %xmm5
+; SSSE3-NEXT:    movdqa %xmm6, %xmm4
+; SSSE3-NEXT:    pandn %xmm5, %xmm4
+; SSSE3-NEXT:    por %xmm4, %xmm2
+; SSSE3-NEXT:    paddd %xmm6, %xmm2
+; SSSE3-NEXT:    pxor %xmm3, %xmm9
+; SSSE3-NEXT:    pxor %xmm7, %xmm8
+; SSSE3-NEXT:    pcmpgtd %xmm9, %xmm8
+; SSSE3-NEXT:    pand %xmm8, %xmm3
+; SSSE3-NEXT:    pxor %xmm10, %xmm8
+; SSSE3-NEXT:    movdqa %xmm7, %xmm4
+; SSSE3-NEXT:    pandn %xmm8, %xmm4
+; SSSE3-NEXT:    por %xmm4, %xmm3
+; SSSE3-NEXT:    paddd %xmm7, %xmm3
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: v16i32:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    pcmpeqd %xmm8, %xmm8
+; SSE41-NEXT:    movdqa %xmm4, %xmm9
+; SSE41-NEXT:    pxor %xmm8, %xmm9
+; SSE41-NEXT:    pminud %xmm9, %xmm0
+; SSE41-NEXT:    paddd %xmm4, %xmm0
+; SSE41-NEXT:    movdqa %xmm5, %xmm4
+; SSE41-NEXT:    pxor %xmm8, %xmm4
+; SSE41-NEXT:    pminud %xmm4, %xmm1
+; SSE41-NEXT:    paddd %xmm5, %xmm1
+; SSE41-NEXT:    movdqa %xmm6, %xmm4
+; SSE41-NEXT:    pxor %xmm8, %xmm4
+; SSE41-NEXT:    pminud %xmm4, %xmm2
+; SSE41-NEXT:    paddd %xmm6, %xmm2
+; SSE41-NEXT:    pxor %xmm7, %xmm8
+; SSE41-NEXT:    pminud %xmm8, %xmm3
+; SSE41-NEXT:    paddd %xmm7, %xmm3
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: v16i32:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vxorps %xmm4, %xmm4, %xmm4
+; AVX1-NEXT:    vcmptrueps %ymm4, %ymm4, %ymm4
+; AVX1-NEXT:    vxorps %ymm4, %ymm2, %ymm5
+; AVX1-NEXT:    vextractf128 $1, %ymm5, %xmm6
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm7
+; AVX1-NEXT:    vpminud %xmm6, %xmm7, %xmm6
+; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm7
+; AVX1-NEXT:    vpaddd %xmm7, %xmm6, %xmm6
+; AVX1-NEXT:    vpminud %xmm5, %xmm0, %xmm0
+; AVX1-NEXT:    vpaddd %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm6, %ymm0, %ymm0
+; AVX1-NEXT:    vxorps %ymm4, %ymm3, %ymm2
+; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm4
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm5
+; AVX1-NEXT:    vpminud %xmm4, %xmm5, %xmm4
+; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm5
+; AVX1-NEXT:    vpaddd %xmm5, %xmm4, %xmm4
+; AVX1-NEXT:    vpminud %xmm2, %xmm1, %xmm1
+; AVX1-NEXT:    vpaddd %xmm3, %xmm1, %xmm1
+; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm1, %ymm1
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: v16i32:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpcmpeqd %ymm4, %ymm4, %ymm4
+; AVX2-NEXT:    vpxor %ymm4, %ymm2, %ymm5
+; AVX2-NEXT:    vpminud %ymm5, %ymm0, %ymm0
+; AVX2-NEXT:    vpaddd %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vpxor %ymm4, %ymm3, %ymm2
+; AVX2-NEXT:    vpminud %ymm2, %ymm1, %ymm1
+; AVX2-NEXT:    vpaddd %ymm3, %ymm1, %ymm1
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: v16i32:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vmovdqa64 %zmm1, %zmm2
+; AVX512-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm2
+; AVX512-NEXT:    vpminud %zmm2, %zmm0, %zmm0
+; AVX512-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    retq
+  %z = call <16 x i32> @llvm.uadd.sat.v16i32(<16 x i32> %x, <16 x i32> %y)
+  ret <16 x i32> %z
+}
+
+define <2 x i64> @v2i64(<2 x i64> %x, <2 x i64> %y) nounwind {
+; SSE2-LABEL: v2i64:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [9223372039002259456,9223372039002259456]
+; SSE2-NEXT:    pxor %xmm0, %xmm2
+; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [9223372034707292159,9223372034707292159]
+; SSE2-NEXT:    pxor %xmm1, %xmm3
+; SSE2-NEXT:    movdqa %xmm3, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm2, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm2, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[1,1,3,3]
+; SSE2-NEXT:    pand %xmm5, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[1,1,3,3]
+; SSE2-NEXT:    por %xmm2, %xmm3
+; SSE2-NEXT:    pand %xmm3, %xmm0
+; SSE2-NEXT:    pcmpeqd %xmm2, %xmm2
+; SSE2-NEXT:    pxor %xmm3, %xmm2
+; SSE2-NEXT:    movdqa %xmm1, %xmm3
+; SSE2-NEXT:    pandn %xmm2, %xmm3
+; SSE2-NEXT:    por %xmm3, %xmm0
+; SSE2-NEXT:    paddq %xmm1, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: v2i64:
+; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm2 = [9223372039002259456,9223372039002259456]
+; SSSE3-NEXT:    pxor %xmm0, %xmm2
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm3 = [9223372034707292159,9223372034707292159]
+; SSSE3-NEXT:    pxor %xmm1, %xmm3
+; SSSE3-NEXT:    movdqa %xmm3, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm3
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm5, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    por %xmm2, %xmm3
+; SSSE3-NEXT:    pand %xmm3, %xmm0
+; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm2
+; SSSE3-NEXT:    pxor %xmm3, %xmm2
+; SSSE3-NEXT:    movdqa %xmm1, %xmm3
+; SSSE3-NEXT:    pandn %xmm2, %xmm3
+; SSSE3-NEXT:    por %xmm3, %xmm0
+; SSSE3-NEXT:    paddq %xmm1, %xmm0
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: v2i64:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movdqa %xmm0, %xmm2
+; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [9223372039002259456,9223372039002259456]
+; SSE41-NEXT:    pxor %xmm2, %xmm0
+; SSE41-NEXT:    movdqa {{.*#+}} xmm3 = [9223372034707292159,9223372034707292159]
+; SSE41-NEXT:    pxor %xmm1, %xmm3
+; SSE41-NEXT:    movdqa %xmm3, %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm4
+; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm3
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
+; SSE41-NEXT:    pand %xmm5, %xmm0
+; SSE41-NEXT:    por %xmm4, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm3, %xmm3
+; SSE41-NEXT:    pxor %xmm1, %xmm3
+; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm3
+; SSE41-NEXT:    paddq %xmm1, %xmm3
+; SSE41-NEXT:    movdqa %xmm3, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: v2i64:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
+; AVX1-NEXT:    vpxor %xmm2, %xmm1, %xmm2
+; AVX1-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm3
+; AVX1-NEXT:    vpxor {{.*}}(%rip), %xmm1, %xmm4
+; AVX1-NEXT:    vpcmpgtq %xmm3, %xmm4, %xmm3
+; AVX1-NEXT:    vblendvpd %xmm3, %xmm0, %xmm2, %xmm0
+; AVX1-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: v2i64:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
+; AVX2-NEXT:    vpxor %xmm2, %xmm1, %xmm2
+; AVX2-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm3
+; AVX2-NEXT:    vpxor {{.*}}(%rip), %xmm1, %xmm4
+; AVX2-NEXT:    vpcmpgtq %xmm3, %xmm4, %xmm3
+; AVX2-NEXT:    vblendvpd %xmm3, %xmm0, %xmm2, %xmm0
+; AVX2-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: v2i64:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vmovdqa %xmm1, %xmm2
+; AVX512-NEXT:    vpternlogq $15, %xmm1, %xmm1, %xmm2
+; AVX512-NEXT:    vpminuq %xmm2, %xmm0, %xmm0
+; AVX512-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    retq
+  %z = call <2 x i64> @llvm.uadd.sat.v2i64(<2 x i64> %x, <2 x i64> %y)
+  ret <2 x i64> %z
 }
 
-define <4 x i24> @v4i24(<4 x i24> %x, <4 x i24> %y) nounwind {
-; SSE2-LABEL: v4i24:
+define <4 x i64> @v4i64(<4 x i64> %x, <4 x i64> %y) nounwind {
+; SSE2-LABEL: v4i64:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pslld $8, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[3,1,2,3]
-; SSE2-NEXT:    movd %xmm2, %eax
-; SSE2-NEXT:    pslld $8, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[3,1,2,3]
-; SSE2-NEXT:    movd %xmm2, %ecx
-; SSE2-NEXT:    addl %eax, %ecx
-; SSE2-NEXT:    movl $-1, %eax
-; SSE2-NEXT:    cmovbl %eax, %ecx
-; SSE2-NEXT:    movd %ecx, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[2,3,0,1]
-; SSE2-NEXT:    movd %xmm3, %ecx
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[2,3,0,1]
-; SSE2-NEXT:    movd %xmm3, %edx
-; SSE2-NEXT:    addl %ecx, %edx
-; SSE2-NEXT:    cmovbl %eax, %edx
-; SSE2-NEXT:    movd %edx, %xmm3
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
-; SSE2-NEXT:    movd %xmm1, %ecx
-; SSE2-NEXT:    movd %xmm0, %edx
-; SSE2-NEXT:    addl %ecx, %edx
-; SSE2-NEXT:    cmovbl %eax, %edx
-; SSE2-NEXT:    movd %edx, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,2,3]
-; SSE2-NEXT:    movd %xmm1, %ecx
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
-; SSE2-NEXT:    movd %xmm0, %edx
-; SSE2-NEXT:    addl %ecx, %edx
-; SSE2-NEXT:    cmovbl %eax, %edx
-; SSE2-NEXT:    movd %edx, %xmm0
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
-; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; SSE2-NEXT:    psrld $8, %xmm2
-; SSE2-NEXT:    movdqa %xmm2, %xmm0
+; SSE2-NEXT:    movdqa {{.*#+}} xmm8 = [9223372039002259456,9223372039002259456]
+; SSE2-NEXT:    movdqa %xmm0, %xmm6
+; SSE2-NEXT:    pxor %xmm8, %xmm6
+; SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [9223372034707292159,9223372034707292159]
+; SSE2-NEXT:    movdqa %xmm2, %xmm7
+; SSE2-NEXT:    pxor %xmm5, %xmm7
+; SSE2-NEXT:    movdqa %xmm7, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm6, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm9 = xmm4[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm6, %xmm7
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm7[1,1,3,3]
+; SSE2-NEXT:    pand %xmm9, %xmm6
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSE2-NEXT:    por %xmm6, %xmm4
+; SSE2-NEXT:    pand %xmm4, %xmm0
+; SSE2-NEXT:    pcmpeqd %xmm6, %xmm6
+; SSE2-NEXT:    pxor %xmm6, %xmm4
+; SSE2-NEXT:    movdqa %xmm2, %xmm7
+; SSE2-NEXT:    pandn %xmm4, %xmm7
+; SSE2-NEXT:    por %xmm7, %xmm0
+; SSE2-NEXT:    paddq %xmm2, %xmm0
+; SSE2-NEXT:    pxor %xmm1, %xmm8
+; SSE2-NEXT:    pxor %xmm3, %xmm5
+; SSE2-NEXT:    movdqa %xmm5, %xmm2
+; SSE2-NEXT:    pcmpgtd %xmm8, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm8, %xmm5
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
+; SSE2-NEXT:    pand %xmm4, %xmm5
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; SSE2-NEXT:    por %xmm5, %xmm2
+; SSE2-NEXT:    pand %xmm2, %xmm1
+; SSE2-NEXT:    pxor %xmm6, %xmm2
+; SSE2-NEXT:    movdqa %xmm3, %xmm4
+; SSE2-NEXT:    pandn %xmm2, %xmm4
+; SSE2-NEXT:    por %xmm4, %xmm1
+; SSE2-NEXT:    paddq %xmm3, %xmm1
 ; SSE2-NEXT:    retq
 ;
-; SSSE3-LABEL: v4i24:
+; SSSE3-LABEL: v4i64:
 ; SSSE3:       # %bb.0:
-; SSSE3-NEXT:    pslld $8, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[3,1,2,3]
-; SSSE3-NEXT:    movd %xmm2, %eax
-; SSSE3-NEXT:    pslld $8, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[3,1,2,3]
-; SSSE3-NEXT:    movd %xmm2, %ecx
-; SSSE3-NEXT:    addl %eax, %ecx
-; SSSE3-NEXT:    movl $-1, %eax
-; SSSE3-NEXT:    cmovbl %eax, %ecx
-; SSSE3-NEXT:    movd %ecx, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[2,3,0,1]
-; SSSE3-NEXT:    movd %xmm3, %ecx
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[2,3,0,1]
-; SSSE3-NEXT:    movd %xmm3, %edx
-; SSSE3-NEXT:    addl %ecx, %edx
-; SSSE3-NEXT:    cmovbl %eax, %edx
-; SSSE3-NEXT:    movd %edx, %xmm3
-; SSSE3-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
-; SSSE3-NEXT:    movd %xmm1, %ecx
-; SSSE3-NEXT:    movd %xmm0, %edx
-; SSSE3-NEXT:    addl %ecx, %edx
-; SSSE3-NEXT:    cmovbl %eax, %edx
-; SSSE3-NEXT:    movd %edx, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,2,3]
-; SSSE3-NEXT:    movd %xmm1, %ecx
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
-; SSSE3-NEXT:    movd %xmm0, %edx
-; SSSE3-NEXT:    addl %ecx, %edx
-; SSSE3-NEXT:    cmovbl %eax, %edx
-; SSSE3-NEXT:    movd %edx, %xmm0
-; SSSE3-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
-; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; SSSE3-NEXT:    psrld $8, %xmm2
-; SSSE3-NEXT:    movdqa %xmm2, %xmm0
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm8 = [9223372039002259456,9223372039002259456]
+; SSSE3-NEXT:    movdqa %xmm0, %xmm6
+; SSSE3-NEXT:    pxor %xmm8, %xmm6
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm5 = [9223372034707292159,9223372034707292159]
+; SSSE3-NEXT:    movdqa %xmm2, %xmm7
+; SSSE3-NEXT:    pxor %xmm5, %xmm7
+; SSSE3-NEXT:    movdqa %xmm7, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm6, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm9 = xmm4[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm6, %xmm7
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm7[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm9, %xmm6
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    por %xmm6, %xmm4
+; SSSE3-NEXT:    pand %xmm4, %xmm0
+; SSSE3-NEXT:    pcmpeqd %xmm6, %xmm6
+; SSSE3-NEXT:    pxor %xmm6, %xmm4
+; SSSE3-NEXT:    movdqa %xmm2, %xmm7
+; SSSE3-NEXT:    pandn %xmm4, %xmm7
+; SSSE3-NEXT:    por %xmm7, %xmm0
+; SSSE3-NEXT:    paddq %xmm2, %xmm0
+; SSSE3-NEXT:    pxor %xmm1, %xmm8
+; SSSE3-NEXT:    pxor %xmm3, %xmm5
+; SSSE3-NEXT:    movdqa %xmm5, %xmm2
+; SSSE3-NEXT:    pcmpgtd %xmm8, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm8, %xmm5
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm4, %xmm5
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; SSSE3-NEXT:    por %xmm5, %xmm2
+; SSSE3-NEXT:    pand %xmm2, %xmm1
+; SSSE3-NEXT:    pxor %xmm6, %xmm2
+; SSSE3-NEXT:    movdqa %xmm3, %xmm4
+; SSSE3-NEXT:    pandn %xmm2, %xmm4
+; SSSE3-NEXT:    por %xmm4, %xmm1
+; SSSE3-NEXT:    paddq %xmm3, %xmm1
 ; SSSE3-NEXT:    retq
 ;
-; SSE41-LABEL: v4i24:
+; SSE41-LABEL: v4i64:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    pslld $8, %xmm1
-; SSE41-NEXT:    pextrd $1, %xmm1, %eax
-; SSE41-NEXT:    pslld $8, %xmm0
-; SSE41-NEXT:    pextrd $1, %xmm0, %ecx
-; SSE41-NEXT:    addl %eax, %ecx
-; SSE41-NEXT:    movl $-1, %eax
-; SSE41-NEXT:    cmovbl %eax, %ecx
-; SSE41-NEXT:    movd %xmm1, %edx
-; SSE41-NEXT:    movd %xmm0, %esi
-; SSE41-NEXT:    addl %edx, %esi
-; SSE41-NEXT:    cmovbl %eax, %esi
-; SSE41-NEXT:    movd %esi, %xmm2
-; SSE41-NEXT:    pinsrd $1, %ecx, %xmm2
-; SSE41-NEXT:    pextrd $2, %xmm1, %ecx
-; SSE41-NEXT:    pextrd $2, %xmm0, %edx
-; SSE41-NEXT:    addl %ecx, %edx
-; SSE41-NEXT:    cmovbl %eax, %edx
-; SSE41-NEXT:    pinsrd $2, %edx, %xmm2
-; SSE41-NEXT:    pextrd $3, %xmm1, %ecx
-; SSE41-NEXT:    pextrd $3, %xmm0, %edx
-; SSE41-NEXT:    addl %ecx, %edx
-; SSE41-NEXT:    cmovbl %eax, %edx
-; SSE41-NEXT:    pinsrd $3, %edx, %xmm2
-; SSE41-NEXT:    psrld $8, %xmm2
+; SSE41-NEXT:    movdqa %xmm0, %xmm8
+; SSE41-NEXT:    movdqa {{.*#+}} xmm7 = [9223372039002259456,9223372039002259456]
+; SSE41-NEXT:    pxor %xmm7, %xmm0
+; SSE41-NEXT:    movdqa {{.*#+}} xmm4 = [9223372034707292159,9223372034707292159]
+; SSE41-NEXT:    movdqa %xmm2, %xmm5
+; SSE41-NEXT:    pxor %xmm4, %xmm5
+; SSE41-NEXT:    movdqa %xmm5, %xmm6
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm6
+; SSE41-NEXT:    pshufd {{.*#+}} xmm9 = xmm6[0,0,2,2]
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm5
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm5[1,1,3,3]
+; SSE41-NEXT:    pand %xmm9, %xmm0
+; SSE41-NEXT:    por %xmm6, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm5, %xmm5
+; SSE41-NEXT:    movdqa %xmm2, %xmm6
+; SSE41-NEXT:    pxor %xmm5, %xmm6
+; SSE41-NEXT:    blendvpd %xmm0, %xmm8, %xmm6
+; SSE41-NEXT:    paddq %xmm2, %xmm6
+; SSE41-NEXT:    pxor %xmm1, %xmm7
+; SSE41-NEXT:    pxor %xmm3, %xmm4
+; SSE41-NEXT:    movdqa %xmm4, %xmm2
+; SSE41-NEXT:    pcmpgtd %xmm7, %xmm2
+; SSE41-NEXT:    pshufd {{.*#+}} xmm8 = xmm2[0,0,2,2]
+; SSE41-NEXT:    pcmpeqd %xmm7, %xmm4
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[1,1,3,3]
+; SSE41-NEXT:    pand %xmm8, %xmm0
+; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    pxor %xmm3, %xmm5
+; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm5
+; SSE41-NEXT:    paddq %xmm3, %xmm5
+; SSE41-NEXT:    movdqa %xmm6, %xmm0
+; SSE41-NEXT:    movdqa %xmm5, %xmm1
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: v4i64:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [9223372036854775808,9223372036854775808]
+; AVX1-NEXT:    vpxor %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vxorps %xmm4, %xmm4, %xmm4
+; AVX1-NEXT:    vcmptrueps %ymm4, %ymm4, %ymm4
+; AVX1-NEXT:    vxorps %ymm4, %ymm1, %ymm4
+; AVX1-NEXT:    vextractf128 $1, %ymm4, %xmm5
+; AVX1-NEXT:    vpxor %xmm3, %xmm5, %xmm5
+; AVX1-NEXT:    vpcmpgtq %xmm2, %xmm5, %xmm2
+; AVX1-NEXT:    vpxor %xmm3, %xmm0, %xmm5
+; AVX1-NEXT:    vxorps %xmm3, %xmm4, %xmm3
+; AVX1-NEXT:    vpcmpgtq %xmm5, %xmm3, %xmm3
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm3, %ymm2
+; AVX1-NEXT:    vblendvpd %ymm2, %ymm0, %ymm4, %ymm0
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm3
+; AVX1-NEXT:    vpaddq %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: v4i64:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808]
+; AVX2-NEXT:    vpxor %ymm2, %ymm0, %ymm2
+; AVX2-NEXT:    vpbroadcastq {{.*#+}} ymm3 = [9223372036854775807,9223372036854775807,9223372036854775807,9223372036854775807]
+; AVX2-NEXT:    vpxor %ymm3, %ymm1, %ymm3
+; AVX2-NEXT:    vpcmpgtq %ymm2, %ymm3, %ymm2
+; AVX2-NEXT:    vpcmpeqd %ymm3, %ymm3, %ymm3
+; AVX2-NEXT:    vpxor %ymm3, %ymm1, %ymm3
+; AVX2-NEXT:    vblendvpd %ymm2, %ymm0, %ymm3, %ymm0
+; AVX2-NEXT:    vpaddq %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: v4i64:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vmovdqa %ymm1, %ymm2
+; AVX512-NEXT:    vpternlogq $15, %ymm1, %ymm1, %ymm2
+; AVX512-NEXT:    vpminuq %ymm2, %ymm0, %ymm0
+; AVX512-NEXT:    vpaddq %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    retq
+  %z = call <4 x i64> @llvm.uadd.sat.v4i64(<4 x i64> %x, <4 x i64> %y)
+  ret <4 x i64> %z
+}
+
+define <8 x i64> @v8i64(<8 x i64> %x, <8 x i64> %y) nounwind {
+; SSE2-LABEL: v8i64:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa {{.*#+}} xmm8 = [9223372039002259456,9223372039002259456]
+; SSE2-NEXT:    movdqa %xmm0, %xmm10
+; SSE2-NEXT:    pxor %xmm8, %xmm10
+; SSE2-NEXT:    movdqa {{.*#+}} xmm9 = [9223372034707292159,9223372034707292159]
+; SSE2-NEXT:    movdqa %xmm4, %xmm11
+; SSE2-NEXT:    pxor %xmm9, %xmm11
+; SSE2-NEXT:    movdqa %xmm11, %xmm12
+; SSE2-NEXT:    pcmpgtd %xmm10, %xmm12
+; SSE2-NEXT:    pshufd {{.*#+}} xmm13 = xmm12[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm10, %xmm11
+; SSE2-NEXT:    pshufd {{.*#+}} xmm10 = xmm11[1,1,3,3]
+; SSE2-NEXT:    pand %xmm13, %xmm10
+; SSE2-NEXT:    pshufd {{.*#+}} xmm11 = xmm12[1,1,3,3]
+; SSE2-NEXT:    por %xmm10, %xmm11
+; SSE2-NEXT:    pand %xmm11, %xmm0
+; SSE2-NEXT:    pcmpeqd %xmm10, %xmm10
+; SSE2-NEXT:    pxor %xmm10, %xmm11
+; SSE2-NEXT:    movdqa %xmm4, %xmm12
+; SSE2-NEXT:    pandn %xmm11, %xmm12
+; SSE2-NEXT:    por %xmm12, %xmm0
+; SSE2-NEXT:    paddq %xmm4, %xmm0
+; SSE2-NEXT:    movdqa %xmm1, %xmm11
+; SSE2-NEXT:    pxor %xmm8, %xmm11
+; SSE2-NEXT:    movdqa %xmm5, %xmm4
+; SSE2-NEXT:    pxor %xmm9, %xmm4
+; SSE2-NEXT:    movdqa %xmm4, %xmm12
+; SSE2-NEXT:    pcmpgtd %xmm11, %xmm12
+; SSE2-NEXT:    pshufd {{.*#+}} xmm13 = xmm12[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm11, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm11 = xmm4[1,1,3,3]
+; SSE2-NEXT:    pand %xmm13, %xmm11
+; SSE2-NEXT:    pshufd {{.*#+}} xmm12 = xmm12[1,1,3,3]
+; SSE2-NEXT:    por %xmm11, %xmm12
+; SSE2-NEXT:    pand %xmm12, %xmm1
+; SSE2-NEXT:    pxor %xmm10, %xmm12
+; SSE2-NEXT:    movdqa %xmm5, %xmm4
+; SSE2-NEXT:    pandn %xmm12, %xmm4
+; SSE2-NEXT:    por %xmm4, %xmm1
+; SSE2-NEXT:    paddq %xmm5, %xmm1
+; SSE2-NEXT:    movdqa %xmm2, %xmm4
+; SSE2-NEXT:    pxor %xmm8, %xmm4
+; SSE2-NEXT:    movdqa %xmm6, %xmm5
+; SSE2-NEXT:    pxor %xmm9, %xmm5
+; SSE2-NEXT:    movdqa %xmm5, %xmm11
+; SSE2-NEXT:    pcmpgtd %xmm4, %xmm11
+; SSE2-NEXT:    pshufd {{.*#+}} xmm12 = xmm11[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm4, %xmm5
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm5[1,1,3,3]
+; SSE2-NEXT:    pand %xmm12, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm11[1,1,3,3]
+; SSE2-NEXT:    por %xmm4, %xmm5
+; SSE2-NEXT:    pand %xmm5, %xmm2
+; SSE2-NEXT:    pxor %xmm10, %xmm5
+; SSE2-NEXT:    movdqa %xmm6, %xmm4
+; SSE2-NEXT:    pandn %xmm5, %xmm4
+; SSE2-NEXT:    por %xmm4, %xmm2
+; SSE2-NEXT:    paddq %xmm6, %xmm2
+; SSE2-NEXT:    pxor %xmm3, %xmm8
+; SSE2-NEXT:    pxor %xmm7, %xmm9
+; SSE2-NEXT:    movdqa %xmm9, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm8, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm8, %xmm9
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm9[1,1,3,3]
+; SSE2-NEXT:    pand %xmm5, %xmm6
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSE2-NEXT:    por %xmm6, %xmm4
+; SSE2-NEXT:    pand %xmm4, %xmm3
+; SSE2-NEXT:    pxor %xmm10, %xmm4
+; SSE2-NEXT:    movdqa %xmm7, %xmm5
+; SSE2-NEXT:    pandn %xmm4, %xmm5
+; SSE2-NEXT:    por %xmm5, %xmm3
+; SSE2-NEXT:    paddq %xmm7, %xmm3
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: v8i64:
+; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm8 = [9223372039002259456,9223372039002259456]
+; SSSE3-NEXT:    movdqa %xmm0, %xmm10
+; SSSE3-NEXT:    pxor %xmm8, %xmm10
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm9 = [9223372034707292159,9223372034707292159]
+; SSSE3-NEXT:    movdqa %xmm4, %xmm11
+; SSSE3-NEXT:    pxor %xmm9, %xmm11
+; SSSE3-NEXT:    movdqa %xmm11, %xmm12
+; SSSE3-NEXT:    pcmpgtd %xmm10, %xmm12
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm13 = xmm12[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm10, %xmm11
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm10 = xmm11[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm13, %xmm10
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm11 = xmm12[1,1,3,3]
+; SSSE3-NEXT:    por %xmm10, %xmm11
+; SSSE3-NEXT:    pand %xmm11, %xmm0
+; SSSE3-NEXT:    pcmpeqd %xmm10, %xmm10
+; SSSE3-NEXT:    pxor %xmm10, %xmm11
+; SSSE3-NEXT:    movdqa %xmm4, %xmm12
+; SSSE3-NEXT:    pandn %xmm11, %xmm12
+; SSSE3-NEXT:    por %xmm12, %xmm0
+; SSSE3-NEXT:    paddq %xmm4, %xmm0
+; SSSE3-NEXT:    movdqa %xmm1, %xmm11
+; SSSE3-NEXT:    pxor %xmm8, %xmm11
+; SSSE3-NEXT:    movdqa %xmm5, %xmm4
+; SSSE3-NEXT:    pxor %xmm9, %xmm4
+; SSSE3-NEXT:    movdqa %xmm4, %xmm12
+; SSSE3-NEXT:    pcmpgtd %xmm11, %xmm12
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm13 = xmm12[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm11, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm11 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm13, %xmm11
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm12 = xmm12[1,1,3,3]
+; SSSE3-NEXT:    por %xmm11, %xmm12
+; SSSE3-NEXT:    pand %xmm12, %xmm1
+; SSSE3-NEXT:    pxor %xmm10, %xmm12
+; SSSE3-NEXT:    movdqa %xmm5, %xmm4
+; SSSE3-NEXT:    pandn %xmm12, %xmm4
+; SSSE3-NEXT:    por %xmm4, %xmm1
+; SSSE3-NEXT:    paddq %xmm5, %xmm1
+; SSSE3-NEXT:    movdqa %xmm2, %xmm4
+; SSSE3-NEXT:    pxor %xmm8, %xmm4
+; SSSE3-NEXT:    movdqa %xmm6, %xmm5
+; SSSE3-NEXT:    pxor %xmm9, %xmm5
+; SSSE3-NEXT:    movdqa %xmm5, %xmm11
+; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm11
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm12 = xmm11[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm4, %xmm5
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm5[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm12, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm11[1,1,3,3]
+; SSSE3-NEXT:    por %xmm4, %xmm5
+; SSSE3-NEXT:    pand %xmm5, %xmm2
+; SSSE3-NEXT:    pxor %xmm10, %xmm5
+; SSSE3-NEXT:    movdqa %xmm6, %xmm4
+; SSSE3-NEXT:    pandn %xmm5, %xmm4
+; SSSE3-NEXT:    por %xmm4, %xmm2
+; SSSE3-NEXT:    paddq %xmm6, %xmm2
+; SSSE3-NEXT:    pxor %xmm3, %xmm8
+; SSSE3-NEXT:    pxor %xmm7, %xmm9
+; SSSE3-NEXT:    movdqa %xmm9, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm8, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm8, %xmm9
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm9[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm5, %xmm6
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    por %xmm6, %xmm4
+; SSSE3-NEXT:    pand %xmm4, %xmm3
+; SSSE3-NEXT:    pxor %xmm10, %xmm4
+; SSSE3-NEXT:    movdqa %xmm7, %xmm5
+; SSSE3-NEXT:    pandn %xmm4, %xmm5
+; SSSE3-NEXT:    por %xmm5, %xmm3
+; SSSE3-NEXT:    paddq %xmm7, %xmm3
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: v8i64:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movdqa %xmm1, %xmm9
+; SSE41-NEXT:    movdqa %xmm0, %xmm13
+; SSE41-NEXT:    movdqa {{.*#+}} xmm11 = [9223372039002259456,9223372039002259456]
+; SSE41-NEXT:    pxor %xmm11, %xmm0
+; SSE41-NEXT:    movdqa {{.*#+}} xmm12 = [9223372034707292159,9223372034707292159]
+; SSE41-NEXT:    movdqa %xmm4, %xmm1
+; SSE41-NEXT:    pxor %xmm12, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm8
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm8
+; SSE41-NEXT:    pshufd {{.*#+}} xmm10 = xmm8[0,0,2,2]
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
+; SSE41-NEXT:    pand %xmm10, %xmm0
+; SSE41-NEXT:    por %xmm8, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm10, %xmm10
+; SSE41-NEXT:    movdqa %xmm4, %xmm8
+; SSE41-NEXT:    pxor %xmm10, %xmm8
+; SSE41-NEXT:    blendvpd %xmm0, %xmm13, %xmm8
+; SSE41-NEXT:    paddq %xmm4, %xmm8
+; SSE41-NEXT:    movdqa %xmm9, %xmm0
+; SSE41-NEXT:    pxor %xmm11, %xmm0
+; SSE41-NEXT:    movdqa %xmm5, %xmm1
+; SSE41-NEXT:    pxor %xmm12, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm4
+; SSE41-NEXT:    pshufd {{.*#+}} xmm13 = xmm4[0,0,2,2]
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
+; SSE41-NEXT:    pand %xmm13, %xmm0
+; SSE41-NEXT:    por %xmm4, %xmm0
+; SSE41-NEXT:    movdqa %xmm5, %xmm1
+; SSE41-NEXT:    pxor %xmm10, %xmm1
+; SSE41-NEXT:    blendvpd %xmm0, %xmm9, %xmm1
+; SSE41-NEXT:    paddq %xmm5, %xmm1
 ; SSE41-NEXT:    movdqa %xmm2, %xmm0
+; SSE41-NEXT:    pxor %xmm11, %xmm0
+; SSE41-NEXT:    movdqa %xmm6, %xmm4
+; SSE41-NEXT:    pxor %xmm12, %xmm4
+; SSE41-NEXT:    movdqa %xmm4, %xmm5
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm5
+; SSE41-NEXT:    pshufd {{.*#+}} xmm9 = xmm5[0,0,2,2]
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm4
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[1,1,3,3]
+; SSE41-NEXT:    pand %xmm9, %xmm0
+; SSE41-NEXT:    por %xmm5, %xmm0
+; SSE41-NEXT:    movdqa %xmm6, %xmm4
+; SSE41-NEXT:    pxor %xmm10, %xmm4
+; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm4
+; SSE41-NEXT:    paddq %xmm6, %xmm4
+; SSE41-NEXT:    pxor %xmm3, %xmm11
+; SSE41-NEXT:    pxor %xmm7, %xmm12
+; SSE41-NEXT:    movdqa %xmm12, %xmm2
+; SSE41-NEXT:    pcmpgtd %xmm11, %xmm2
+; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm2[0,0,2,2]
+; SSE41-NEXT:    pcmpeqd %xmm11, %xmm12
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm12[1,1,3,3]
+; SSE41-NEXT:    pand %xmm5, %xmm0
+; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    pxor %xmm7, %xmm10
+; SSE41-NEXT:    blendvpd %xmm0, %xmm3, %xmm10
+; SSE41-NEXT:    paddq %xmm7, %xmm10
+; SSE41-NEXT:    movdqa %xmm8, %xmm0
+; SSE41-NEXT:    movdqa %xmm4, %xmm2
+; SSE41-NEXT:    movdqa %xmm10, %xmm3
 ; SSE41-NEXT:    retq
 ;
-; AVX-LABEL: v4i24:
-; AVX:       # %bb.0:
-; AVX-NEXT:    vpslld $8, %xmm1, %xmm1
-; AVX-NEXT:    vpextrd $1, %xmm1, %eax
-; AVX-NEXT:    vpslld $8, %xmm0, %xmm0
-; AVX-NEXT:    vpextrd $1, %xmm0, %ecx
-; AVX-NEXT:    addl %eax, %ecx
-; AVX-NEXT:    movl $-1, %eax
-; AVX-NEXT:    cmovbl %eax, %ecx
-; AVX-NEXT:    vmovd %xmm1, %edx
-; AVX-NEXT:    vmovd %xmm0, %esi
-; AVX-NEXT:    addl %edx, %esi
-; AVX-NEXT:    cmovbl %eax, %esi
-; AVX-NEXT:    vmovd %esi, %xmm2
-; AVX-NEXT:    vpinsrd $1, %ecx, %xmm2, %xmm2
-; AVX-NEXT:    vpextrd $2, %xmm1, %ecx
-; AVX-NEXT:    vpextrd $2, %xmm0, %edx
-; AVX-NEXT:    addl %ecx, %edx
-; AVX-NEXT:    cmovbl %eax, %edx
-; AVX-NEXT:    vpinsrd $2, %edx, %xmm2, %xmm2
-; AVX-NEXT:    vpextrd $3, %xmm1, %ecx
-; AVX-NEXT:    vpextrd $3, %xmm0, %edx
-; AVX-NEXT:    addl %ecx, %edx
-; AVX-NEXT:    cmovbl %eax, %edx
-; AVX-NEXT:    vpinsrd $3, %edx, %xmm2, %xmm0
-; AVX-NEXT:    vpsrld $8, %xmm0, %xmm0
-; AVX-NEXT:    retq
-  %z = call <4 x i24> @llvm.uadd.sat.v4i24(<4 x i24> %x, <4 x i24> %y)
-  ret <4 x i24> %z
+; AVX1-LABEL: v8i64:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm5
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = [9223372036854775808,9223372036854775808]
+; AVX1-NEXT:    vpxor %xmm4, %xmm5, %xmm5
+; AVX1-NEXT:    vxorps %xmm6, %xmm6, %xmm6
+; AVX1-NEXT:    vcmptrueps %ymm6, %ymm6, %ymm8
+; AVX1-NEXT:    vxorps %ymm8, %ymm2, %ymm7
+; AVX1-NEXT:    vextractf128 $1, %ymm7, %xmm6
+; AVX1-NEXT:    vpxor %xmm4, %xmm6, %xmm6
+; AVX1-NEXT:    vpcmpgtq %xmm5, %xmm6, %xmm9
+; AVX1-NEXT:    vpxor %xmm4, %xmm0, %xmm6
+; AVX1-NEXT:    vxorps %xmm4, %xmm7, %xmm5
+; AVX1-NEXT:    vpcmpgtq %xmm6, %xmm5, %xmm5
+; AVX1-NEXT:    vinsertf128 $1, %xmm9, %ymm5, %ymm5
+; AVX1-NEXT:    vblendvpd %ymm5, %ymm0, %ymm7, %ymm0
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm5
+; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm6
+; AVX1-NEXT:    vpaddq %xmm6, %xmm5, %xmm5
+; AVX1-NEXT:    vpaddq %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm5, %ymm0, %ymm0
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; AVX1-NEXT:    vpxor %xmm4, %xmm2, %xmm2
+; AVX1-NEXT:    vxorps %ymm8, %ymm3, %ymm5
+; AVX1-NEXT:    vextractf128 $1, %ymm5, %xmm6
+; AVX1-NEXT:    vpxor %xmm4, %xmm6, %xmm6
+; AVX1-NEXT:    vpcmpgtq %xmm2, %xmm6, %xmm2
+; AVX1-NEXT:    vpxor %xmm4, %xmm1, %xmm6
+; AVX1-NEXT:    vxorps %xmm4, %xmm5, %xmm4
+; AVX1-NEXT:    vpcmpgtq %xmm6, %xmm4, %xmm4
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm4, %ymm2
+; AVX1-NEXT:    vblendvpd %ymm2, %ymm1, %ymm5, %ymm1
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm4
+; AVX1-NEXT:    vpaddq %xmm4, %xmm2, %xmm2
+; AVX1-NEXT:    vpaddq %xmm3, %xmm1, %xmm1
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: v8i64:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpbroadcastq {{.*#+}} ymm4 = [9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808]
+; AVX2-NEXT:    vpxor %ymm4, %ymm0, %ymm5
+; AVX2-NEXT:    vpbroadcastq {{.*#+}} ymm6 = [9223372036854775807,9223372036854775807,9223372036854775807,9223372036854775807]
+; AVX2-NEXT:    vpxor %ymm6, %ymm2, %ymm7
+; AVX2-NEXT:    vpcmpgtq %ymm5, %ymm7, %ymm5
+; AVX2-NEXT:    vpcmpeqd %ymm7, %ymm7, %ymm7
+; AVX2-NEXT:    vpxor %ymm7, %ymm2, %ymm8
+; AVX2-NEXT:    vblendvpd %ymm5, %ymm0, %ymm8, %ymm0
+; AVX2-NEXT:    vpaddq %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vpxor %ymm4, %ymm1, %ymm2
+; AVX2-NEXT:    vpxor %ymm6, %ymm3, %ymm4
+; AVX2-NEXT:    vpcmpgtq %ymm2, %ymm4, %ymm2
+; AVX2-NEXT:    vpxor %ymm7, %ymm3, %ymm4
+; AVX2-NEXT:    vblendvpd %ymm2, %ymm1, %ymm4, %ymm1
+; AVX2-NEXT:    vpaddq %ymm3, %ymm1, %ymm1
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: v8i64:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vmovdqa64 %zmm1, %zmm2
+; AVX512-NEXT:    vpternlogq $15, %zmm1, %zmm1, %zmm2
+; AVX512-NEXT:    vpminuq %zmm2, %zmm0, %zmm0
+; AVX512-NEXT:    vpaddq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    retq
+  %z = call <8 x i64> @llvm.uadd.sat.v8i64(<8 x i64> %x, <8 x i64> %y)
+  ret <8 x i64> %z
 }
 
 define <2 x i128> @v2i128(<2 x i128> %x, <2 x i128> %y) nounwind {
diff --git a/test/CodeGen/X86/umul-with-overflow.ll b/test/CodeGen/X86/umul-with-overflow.ll
index 72b1fcceb92469d4b9dfd8107c6849d25c874b56..64a8933346e86ee900f9861fdd3063fbe7e0a6dd 100644
--- a/test/CodeGen/X86/umul-with-overflow.ll
+++ b/test/CodeGen/X86/umul-with-overflow.ll
@@ -7,9 +7,8 @@ declare {i32, i1} @llvm.umul.with.overflow.i32(i32 %a, i32 %b)
 define zeroext i1 @a(i32 %x)  nounwind {
 ; X86-LABEL: a:
 ; X86:       # %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    movl $3, %ecx
-; X86-NEXT:    mull %ecx
+; X86-NEXT:    movl $3, %eax
+; X86-NEXT:    mull {{[0-9]+}}(%esp)
 ; X86-NEXT:    seto %al
 ; X86-NEXT:    retl
 ;
diff --git a/test/CodeGen/X86/umul_fix.ll b/test/CodeGen/X86/umul_fix.ll
new file mode 100644
index 0000000000000000000000000000000000000000..6f7e22245b75051fac4f1e9262ceb39a356af44e
--- /dev/null
+++ b/test/CodeGen/X86/umul_fix.ll
@@ -0,0 +1,393 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-linux | FileCheck %s --check-prefix=X64
+; RUN: llc < %s -mtriple=i686 -mattr=cmov | FileCheck %s --check-prefix=X86
+
+declare  i4  @llvm.umul.fix.i4   (i4,  i4, i32)
+declare  i32 @llvm.umul.fix.i32  (i32, i32, i32)
+declare  i64 @llvm.umul.fix.i64  (i64, i64, i32)
+declare  <4 x i32> @llvm.umul.fix.v4i32(<4 x i32>, <4 x i32>, i32)
+
+define i32 @func(i32 %x, i32 %y) nounwind {
+; X64-LABEL: func:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %esi, %eax
+; X64-NEXT:    movl %edi, %ecx
+; X64-NEXT:    imulq %rax, %rcx
+; X64-NEXT:    movq %rcx, %rax
+; X64-NEXT:    shrq $32, %rax
+; X64-NEXT:    shldl $30, %ecx, %eax
+; X64-NEXT:    # kill: def $eax killed $eax killed $rax
+; X64-NEXT:    retq
+;
+; X86-LABEL: func:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    mull {{[0-9]+}}(%esp)
+; X86-NEXT:    shrdl $2, %edx, %eax
+; X86-NEXT:    retl
+  %tmp = call i32 @llvm.umul.fix.i32(i32 %x, i32 %y, i32 2);
+  ret i32 %tmp;
+}
+
+define i64 @func2(i64 %x, i64 %y) nounwind {
+; X64-LABEL: func2:
+; X64:       # %bb.0:
+; X64-NEXT:    movq %rdi, %rax
+; X64-NEXT:    mulq %rsi
+; X64-NEXT:    shrdq $2, %rdx, %rax
+; X64-NEXT:    retq
+;
+; X86-LABEL: func2:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %ebp
+; X86-NEXT:    pushl %ebx
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movl %ecx, %eax
+; X86-NEXT:    mull {{[0-9]+}}(%esp)
+; X86-NEXT:    movl %edx, %edi
+; X86-NEXT:    movl %eax, %ebx
+; X86-NEXT:    movl %ecx, %eax
+; X86-NEXT:    mull {{[0-9]+}}(%esp)
+; X86-NEXT:    movl %eax, %ecx
+; X86-NEXT:    movl %edx, %ebp
+; X86-NEXT:    addl %ebx, %ebp
+; X86-NEXT:    adcl $0, %edi
+; X86-NEXT:    movl %esi, %eax
+; X86-NEXT:    mull {{[0-9]+}}(%esp)
+; X86-NEXT:    addl %ebp, %eax
+; X86-NEXT:    adcl %edi, %edx
+; X86-NEXT:    imull {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    addl %esi, %edx
+; X86-NEXT:    shldl $30, %eax, %edx
+; X86-NEXT:    shldl $30, %ecx, %eax
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    popl %ebx
+; X86-NEXT:    popl %ebp
+; X86-NEXT:    retl
+  %tmp = call i64 @llvm.umul.fix.i64(i64 %x, i64 %y, i32 2);
+  ret i64 %tmp;
+}
+
+define i4 @func3(i4 %x, i4 %y) nounwind {
+; X64-LABEL: func3:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    andl $15, %esi
+; X64-NEXT:    andl $15, %eax
+; X64-NEXT:    imull %esi, %eax
+; X64-NEXT:    shrb $2, %al
+; X64-NEXT:    # kill: def $al killed $al killed $eax
+; X64-NEXT:    retq
+;
+; X86-LABEL: func3:
+; X86:       # %bb.0:
+; X86-NEXT:    movb {{[0-9]+}}(%esp), %al
+; X86-NEXT:    andb $15, %al
+; X86-NEXT:    movb {{[0-9]+}}(%esp), %cl
+; X86-NEXT:    andb $15, %cl
+; X86-NEXT:    movzbl %cl, %ecx
+; X86-NEXT:    movzbl %al, %eax
+; X86-NEXT:    imull %ecx, %eax
+; X86-NEXT:    shrb $2, %al
+; X86-NEXT:    # kill: def $al killed $al killed $eax
+; X86-NEXT:    retl
+  %tmp = call i4 @llvm.umul.fix.i4(i4 %x, i4 %y, i32 2);
+  ret i4 %tmp;
+}
+
+define <4 x i32> @vec(<4 x i32> %x, <4 x i32> %y) nounwind {
+; X64-LABEL: vec:
+; X64:       # %bb.0:
+; X64-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
+; X64-NEXT:    pmuludq %xmm1, %xmm0
+; X64-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[0,2,2,3]
+; X64-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; X64-NEXT:    pmuludq %xmm2, %xmm1
+; X64-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[0,2,2,3]
+; X64-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
+; X64-NEXT:    psrld $2, %xmm3
+; X64-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,3,2,3]
+; X64-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,3,2,3]
+; X64-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; X64-NEXT:    pslld $30, %xmm0
+; X64-NEXT:    por %xmm3, %xmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: vec:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %ebp
+; X86-NEXT:    pushl %ebx
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ebx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    mull {{[0-9]+}}(%esp)
+; X86-NEXT:    movl %edx, %ebp
+; X86-NEXT:    shldl $30, %eax, %ebp
+; X86-NEXT:    movl %ebx, %eax
+; X86-NEXT:    mull {{[0-9]+}}(%esp)
+; X86-NEXT:    movl %edx, %ebx
+; X86-NEXT:    shldl $30, %eax, %ebx
+; X86-NEXT:    movl %edi, %eax
+; X86-NEXT:    mull {{[0-9]+}}(%esp)
+; X86-NEXT:    movl %edx, %edi
+; X86-NEXT:    shldl $30, %eax, %edi
+; X86-NEXT:    movl %esi, %eax
+; X86-NEXT:    mull {{[0-9]+}}(%esp)
+; X86-NEXT:    shldl $30, %eax, %edx
+; X86-NEXT:    movl %edx, 12(%ecx)
+; X86-NEXT:    movl %edi, 8(%ecx)
+; X86-NEXT:    movl %ebx, 4(%ecx)
+; X86-NEXT:    movl %ebp, (%ecx)
+; X86-NEXT:    movl %ecx, %eax
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    popl %ebx
+; X86-NEXT:    popl %ebp
+; X86-NEXT:    retl $4
+  %tmp = call <4 x i32> @llvm.umul.fix.v4i32(<4 x i32> %x, <4 x i32> %y, i32 2);
+  ret <4 x i32> %tmp;
+}
+
+; These result in regular integer multiplication
+define i32 @func4(i32 %x, i32 %y) nounwind {
+; X64-LABEL: func4:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    imull %esi, %eax
+; X64-NEXT:    retq
+;
+; X86-LABEL: func4:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    imull {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    retl
+  %tmp = call i32 @llvm.umul.fix.i32(i32 %x, i32 %y, i32 0);
+  ret i32 %tmp;
+}
+
+define i64 @func5(i64 %x, i64 %y) nounwind {
+; X64-LABEL: func5:
+; X64:       # %bb.0:
+; X64-NEXT:    movq %rdi, %rax
+; X64-NEXT:    imulq %rsi, %rax
+; X64-NEXT:    retq
+;
+; X86-LABEL: func5:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movl %ecx, %eax
+; X86-NEXT:    mull %esi
+; X86-NEXT:    imull {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    addl %ecx, %edx
+; X86-NEXT:    imull {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    addl %esi, %edx
+; X86-NEXT:    popl %esi
+; X86-NEXT:    retl
+  %tmp = call i64 @llvm.umul.fix.i64(i64 %x, i64 %y, i32 0);
+  ret i64 %tmp;
+}
+
+define i4 @func6(i4 %x, i4 %y) nounwind {
+; X64-LABEL: func6:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    andb $15, %al
+; X64-NEXT:    andb $15, %sil
+; X64-NEXT:    # kill: def $al killed $al killed $eax
+; X64-NEXT:    mulb %sil
+; X64-NEXT:    retq
+;
+; X86-LABEL: func6:
+; X86:       # %bb.0:
+; X86-NEXT:    movb {{[0-9]+}}(%esp), %al
+; X86-NEXT:    andb $15, %al
+; X86-NEXT:    movb {{[0-9]+}}(%esp), %cl
+; X86-NEXT:    andb $15, %cl
+; X86-NEXT:    mulb %cl
+; X86-NEXT:    retl
+  %tmp = call i4 @llvm.umul.fix.i4(i4 %x, i4 %y, i32 0);
+  ret i4 %tmp;
+}
+
+define <4 x i32> @vec2(<4 x i32> %x, <4 x i32> %y) nounwind {
+; X64-LABEL: vec2:
+; X64:       # %bb.0:
+; X64-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
+; X64-NEXT:    pmuludq %xmm1, %xmm0
+; X64-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; X64-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; X64-NEXT:    pmuludq %xmm2, %xmm1
+; X64-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; X64-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; X64-NEXT:    retq
+;
+; X86-LABEL: vec2:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    imull {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    imull {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    imull {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    imull {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl %ecx, 12(%eax)
+; X86-NEXT:    movl %edx, 8(%eax)
+; X86-NEXT:    movl %esi, 4(%eax)
+; X86-NEXT:    movl %edi, (%eax)
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    retl $4
+  %tmp = call <4 x i32> @llvm.umul.fix.v4i32(<4 x i32> %x, <4 x i32> %y, i32 0);
+  ret <4 x i32> %tmp;
+}
+
+define i64 @func7(i64 %x, i64 %y) nounwind {
+; X64-LABEL: func7:
+; X64:       # %bb.0:
+; X64-NEXT:    movq %rdi, %rax
+; X64-NEXT:    mulq %rsi
+; X64-NEXT:    shrdq $32, %rdx, %rax
+; X64-NEXT:    retq
+;
+; X86-LABEL: func7:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %ebp
+; X86-NEXT:    pushl %ebx
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ebp
+; X86-NEXT:    movl %ecx, %eax
+; X86-NEXT:    mull {{[0-9]+}}(%esp)
+; X86-NEXT:    movl %edx, %edi
+; X86-NEXT:    movl %eax, %ebx
+; X86-NEXT:    movl %ecx, %eax
+; X86-NEXT:    mull %ebp
+; X86-NEXT:    addl %edx, %ebx
+; X86-NEXT:    adcl $0, %edi
+; X86-NEXT:    movl %esi, %eax
+; X86-NEXT:    mull %ebp
+; X86-NEXT:    addl %ebx, %eax
+; X86-NEXT:    adcl %edi, %edx
+; X86-NEXT:    imull {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    addl %esi, %edx
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    popl %ebx
+; X86-NEXT:    popl %ebp
+; X86-NEXT:    retl
+  %tmp = call i64 @llvm.umul.fix.i64(i64 %x, i64 %y, i32 32);
+  ret i64 %tmp;
+}
+
+define i64 @func8(i64 %x, i64 %y) nounwind {
+; X64-LABEL: func8:
+; X64:       # %bb.0:
+; X64-NEXT:    movq %rdi, %rax
+; X64-NEXT:    mulq %rsi
+; X64-NEXT:    shrdq $63, %rdx, %rax
+; X64-NEXT:    retq
+;
+; X86-LABEL: func8:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %ebp
+; X86-NEXT:    pushl %ebx
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ebp
+; X86-NEXT:    movl %ecx, %eax
+; X86-NEXT:    mull %ebp
+; X86-NEXT:    movl %edx, %edi
+; X86-NEXT:    movl %eax, %ebx
+; X86-NEXT:    movl %ecx, %eax
+; X86-NEXT:    mull %esi
+; X86-NEXT:    movl %edx, %ecx
+; X86-NEXT:    addl %ebx, %ecx
+; X86-NEXT:    adcl $0, %edi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    mull %ebp
+; X86-NEXT:    movl %edx, %ebx
+; X86-NEXT:    movl %eax, %ebp
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    mull %esi
+; X86-NEXT:    addl %ecx, %eax
+; X86-NEXT:    adcl %edi, %edx
+; X86-NEXT:    adcl $0, %ebx
+; X86-NEXT:    addl %ebp, %edx
+; X86-NEXT:    adcl $0, %ebx
+; X86-NEXT:    shldl $1, %edx, %ebx
+; X86-NEXT:    shrdl $31, %edx, %eax
+; X86-NEXT:    movl %ebx, %edx
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    popl %ebx
+; X86-NEXT:    popl %ebp
+; X86-NEXT:    retl
+  %tmp = call i64 @llvm.umul.fix.i64(i64 %x, i64 %y, i32 63);
+  ret i64 %tmp;
+}
+
+define i64 @func9(i64 %x, i64 %y) nounwind {
+; X64-LABEL: func9:
+; X64:       # %bb.0:
+; X64-NEXT:    movq %rdi, %rax
+; X64-NEXT:    mulq %rsi
+; X64-NEXT:    movq %rdx, %rax
+; X64-NEXT:    retq
+;
+; X86-LABEL: func9:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %ebp
+; X86-NEXT:    pushl %ebx
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ebp
+; X86-NEXT:    movl %ecx, %eax
+; X86-NEXT:    mull %ebp
+; X86-NEXT:    movl %edx, %edi
+; X86-NEXT:    movl %eax, %ebx
+; X86-NEXT:    movl %ecx, %eax
+; X86-NEXT:    mull %esi
+; X86-NEXT:    movl %edx, %ecx
+; X86-NEXT:    addl %ebx, %ecx
+; X86-NEXT:    adcl $0, %edi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    mull %ebp
+; X86-NEXT:    movl %edx, %ebx
+; X86-NEXT:    movl %eax, %ebp
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    mull %esi
+; X86-NEXT:    addl %ecx, %eax
+; X86-NEXT:    adcl %edi, %edx
+; X86-NEXT:    adcl $0, %ebx
+; X86-NEXT:    addl %edx, %ebp
+; X86-NEXT:    adcl $0, %ebx
+; X86-NEXT:    movl %ebp, %eax
+; X86-NEXT:    movl %ebx, %edx
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    popl %ebx
+; X86-NEXT:    popl %ebp
+; X86-NEXT:    retl
+  %tmp = call i64 @llvm.umul.fix.i64(i64 %x, i64 %y, i32 64);
+  ret i64 %tmp;
+}
diff --git a/test/CodeGen/X86/urem-seteq-vec-splat.ll b/test/CodeGen/X86/urem-seteq-vec-splat.ll
index b22cbee8988f84b783fcb88d34f32a4adbf90905..22fc26e9065f60494db1126e726651a81a6b37ab 100644
--- a/test/CodeGen/X86/urem-seteq-vec-splat.ll
+++ b/test/CodeGen/X86/urem-seteq-vec-splat.ll
@@ -98,101 +98,36 @@ define <4 x i32> @test_urem_odd_vec_i32(<4 x i32> %X) nounwind readnone {
   ret <4 x i32> %ret
 }
 
-; Like test_urem_odd_vec_i32, but with 4 x i16 vectors.
-define <4 x i16> @test_urem_odd_vec_i16(<4 x i16> %X) nounwind readnone {
-; CHECK-SSE2-LABEL: test_urem_odd_vec_i16:
-; CHECK-SSE2:       # %bb.0:
-; CHECK-SSE2-NEXT:    pand {{.*}}(%rip), %xmm0
-; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [3435973837,3435973837,3435973837,3435973837]
-; CHECK-SSE2-NEXT:    movdqa %xmm0, %xmm2
-; CHECK-SSE2-NEXT:    pmuludq %xmm1, %xmm2
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,3,2,3]
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
-; CHECK-SSE2-NEXT:    pmuludq %xmm1, %xmm3
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,3,2,3]
-; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
-; CHECK-SSE2-NEXT:    psrld $2, %xmm2
-; CHECK-SSE2-NEXT:    movdqa %xmm2, %xmm1
-; CHECK-SSE2-NEXT:    pslld $2, %xmm1
-; CHECK-SSE2-NEXT:    paddd %xmm2, %xmm1
-; CHECK-SSE2-NEXT:    psubd %xmm1, %xmm0
-; CHECK-SSE2-NEXT:    pxor %xmm1, %xmm1
-; CHECK-SSE2-NEXT:    pcmpeqd %xmm1, %xmm0
-; CHECK-SSE2-NEXT:    psrld $31, %xmm0
-; CHECK-SSE2-NEXT:    retq
-;
-; CHECK-SSE41-LABEL: test_urem_odd_vec_i16:
-; CHECK-SSE41:       # %bb.0:
-; CHECK-SSE41-NEXT:    pxor %xmm1, %xmm1
-; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3],xmm0[4],xmm1[5],xmm0[6],xmm1[7]
-; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
-; CHECK-SSE41-NEXT:    movdqa {{.*#+}} xmm3 = [3435973837,3435973837,3435973837,3435973837]
-; CHECK-SSE41-NEXT:    pmuludq %xmm3, %xmm2
-; CHECK-SSE41-NEXT:    pmuludq %xmm0, %xmm3
-; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm3 = xmm3[0,1],xmm2[2,3],xmm3[4,5],xmm2[6,7]
-; CHECK-SSE41-NEXT:    psrld $2, %xmm3
-; CHECK-SSE41-NEXT:    pmulld {{.*}}(%rip), %xmm3
-; CHECK-SSE41-NEXT:    psubd %xmm3, %xmm0
-; CHECK-SSE41-NEXT:    pcmpeqd %xmm1, %xmm0
-; CHECK-SSE41-NEXT:    psrld $31, %xmm0
-; CHECK-SSE41-NEXT:    retq
-;
-; CHECK-AVX1-LABEL: test_urem_odd_vec_i16:
-; CHECK-AVX1:       # %bb.0:
-; CHECK-AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3],xmm0[4],xmm1[5],xmm0[6],xmm1[7]
-; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
-; CHECK-AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [3435973837,3435973837,3435973837,3435973837]
-; CHECK-AVX1-NEXT:    vpmuludq %xmm3, %xmm2, %xmm2
-; CHECK-AVX1-NEXT:    vpmuludq %xmm3, %xmm0, %xmm3
-; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0,1],xmm2[2,3],xmm3[4,5],xmm2[6,7]
-; CHECK-AVX1-NEXT:    vpsrld $2, %xmm2, %xmm2
-; CHECK-AVX1-NEXT:    vpmulld {{.*}}(%rip), %xmm2, %xmm2
-; CHECK-AVX1-NEXT:    vpsubd %xmm2, %xmm0, %xmm0
-; CHECK-AVX1-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
-; CHECK-AVX1-NEXT:    vpsrld $31, %xmm0, %xmm0
-; CHECK-AVX1-NEXT:    retq
-;
-; CHECK-AVX2-LABEL: test_urem_odd_vec_i16:
-; CHECK-AVX2:       # %bb.0:
-; CHECK-AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; CHECK-AVX2-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3],xmm0[4],xmm1[5],xmm0[6],xmm1[7]
-; CHECK-AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
-; CHECK-AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm3 = [3435973837,3435973837,3435973837,3435973837]
-; CHECK-AVX2-NEXT:    vpmuludq %xmm3, %xmm2, %xmm2
-; CHECK-AVX2-NEXT:    vpmuludq %xmm3, %xmm0, %xmm3
-; CHECK-AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; CHECK-AVX2-NEXT:    vpblendd {{.*#+}} xmm2 = xmm3[0],xmm2[1],xmm3[2],xmm2[3]
-; CHECK-AVX2-NEXT:    vpsrld $2, %xmm2, %xmm2
-; CHECK-AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm3 = [5,5,5,5]
-; CHECK-AVX2-NEXT:    vpmulld %xmm3, %xmm2, %xmm2
-; CHECK-AVX2-NEXT:    vpsubd %xmm2, %xmm0, %xmm0
-; CHECK-AVX2-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
-; CHECK-AVX2-NEXT:    vpsrld $31, %xmm0, %xmm0
-; CHECK-AVX2-NEXT:    retq
+; Like test_urem_odd_vec_i32, but with 8 x i16 vectors.
+define <8 x i16> @test_urem_odd_vec_i16(<8 x i16> %X) nounwind readnone {
+; CHECK-SSE-LABEL: test_urem_odd_vec_i16:
+; CHECK-SSE:       # %bb.0:
+; CHECK-SSE-NEXT:    movdqa {{.*#+}} xmm1 = [52429,52429,52429,52429,52429,52429,52429,52429]
+; CHECK-SSE-NEXT:    pmulhuw %xmm0, %xmm1
+; CHECK-SSE-NEXT:    psrlw $2, %xmm1
+; CHECK-SSE-NEXT:    pmullw {{.*}}(%rip), %xmm1
+; CHECK-SSE-NEXT:    psubw %xmm1, %xmm0
+; CHECK-SSE-NEXT:    pxor %xmm1, %xmm1
+; CHECK-SSE-NEXT:    pcmpeqw %xmm1, %xmm0
+; CHECK-SSE-NEXT:    psrlw $15, %xmm0
+; CHECK-SSE-NEXT:    retq
 ;
-; CHECK-AVX512VL-LABEL: test_urem_odd_vec_i16:
-; CHECK-AVX512VL:       # %bb.0:
-; CHECK-AVX512VL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; CHECK-AVX512VL-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3],xmm0[4],xmm1[5],xmm0[6],xmm1[7]
-; CHECK-AVX512VL-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
-; CHECK-AVX512VL-NEXT:    vpbroadcastd {{.*#+}} xmm3 = [3435973837,3435973837,3435973837,3435973837]
-; CHECK-AVX512VL-NEXT:    vpmuludq %xmm3, %xmm2, %xmm2
-; CHECK-AVX512VL-NEXT:    vpmuludq %xmm3, %xmm0, %xmm3
-; CHECK-AVX512VL-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; CHECK-AVX512VL-NEXT:    vpblendd {{.*#+}} xmm2 = xmm3[0],xmm2[1],xmm3[2],xmm2[3]
-; CHECK-AVX512VL-NEXT:    vpsrld $2, %xmm2, %xmm2
-; CHECK-AVX512VL-NEXT:    vpmulld {{.*}}(%rip){1to4}, %xmm2, %xmm2
-; CHECK-AVX512VL-NEXT:    vpsubd %xmm2, %xmm0, %xmm0
-; CHECK-AVX512VL-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
-; CHECK-AVX512VL-NEXT:    vpsrld $31, %xmm0, %xmm0
-; CHECK-AVX512VL-NEXT:    retq
-  %urem = urem <4 x i16> %X, <i16 5, i16 5, i16 5, i16 5>
-  %cmp = icmp eq <4 x i16> %urem, <i16 0, i16 0, i16 0, i16 0>
-  %ret = zext <4 x i1> %cmp to <4 x i16>
-  ret <4 x i16> %ret
+; CHECK-AVX-LABEL: test_urem_odd_vec_i16:
+; CHECK-AVX:       # %bb.0:
+; CHECK-AVX-NEXT:    vpmulhuw {{.*}}(%rip), %xmm0, %xmm1
+; CHECK-AVX-NEXT:    vpsrlw $2, %xmm1, %xmm1
+; CHECK-AVX-NEXT:    vpmullw {{.*}}(%rip), %xmm1, %xmm1
+; CHECK-AVX-NEXT:    vpsubw %xmm1, %xmm0, %xmm0
+; CHECK-AVX-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; CHECK-AVX-NEXT:    vpcmpeqw %xmm1, %xmm0, %xmm0
+; CHECK-AVX-NEXT:    vpsrlw $15, %xmm0, %xmm0
+; CHECK-AVX-NEXT:    retq
+  %urem = urem <8 x i16> %X, <i16 5, i16 5, i16 5, i16 5,
+                              i16 5, i16 5, i16 5, i16 5>
+  %cmp = icmp eq <8 x i16> %urem, <i16 0, i16 0, i16 0, i16 0,
+                                   i16 0, i16 0, i16 0, i16 0>
+  %ret = zext <8 x i1> %cmp to <8 x i16>
+  ret <8 x i16> %ret
 }
 
 ; Tests BuildUREMEqFold for 4 x i32 splat vectors with even divisor.
@@ -299,111 +234,38 @@ define <4 x i32> @test_urem_even_vec_i32(<4 x i32> %X) nounwind readnone {
   ret <4 x i32> %ret
 }
 
-; Like test_urem_even_vec_i32, but with 4 x i16 vectors.
-define <4 x i16> @test_urem_even_vec_i16(<4 x i16> %X) nounwind readnone {
-; CHECK-SSE2-LABEL: test_urem_even_vec_i16:
-; CHECK-SSE2:       # %bb.0:
-; CHECK-SSE2-NEXT:    pand {{.*}}(%rip), %xmm0
-; CHECK-SSE2-NEXT:    movdqa %xmm0, %xmm1
-; CHECK-SSE2-NEXT:    psrld $1, %xmm1
-; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [2454267027,2454267027,2454267027,2454267027]
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[1,1,3,3]
-; CHECK-SSE2-NEXT:    pmuludq %xmm2, %xmm1
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,3,2,3]
-; CHECK-SSE2-NEXT:    pmuludq %xmm2, %xmm3
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[1,3,2,3]
-; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
-; CHECK-SSE2-NEXT:    psrld $2, %xmm1
-; CHECK-SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [14,14,14,14]
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[1,1,3,3]
-; CHECK-SSE2-NEXT:    pmuludq %xmm2, %xmm1
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
-; CHECK-SSE2-NEXT:    pmuludq %xmm2, %xmm3
-; CHECK-SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,2,2,3]
-; CHECK-SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
-; CHECK-SSE2-NEXT:    psubd %xmm1, %xmm0
-; CHECK-SSE2-NEXT:    pxor %xmm1, %xmm1
-; CHECK-SSE2-NEXT:    pcmpeqd %xmm1, %xmm0
-; CHECK-SSE2-NEXT:    psrld $31, %xmm0
-; CHECK-SSE2-NEXT:    retq
-;
-; CHECK-SSE41-LABEL: test_urem_even_vec_i16:
-; CHECK-SSE41:       # %bb.0:
-; CHECK-SSE41-NEXT:    pxor %xmm1, %xmm1
-; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3],xmm0[4],xmm1[5],xmm0[6],xmm1[7]
-; CHECK-SSE41-NEXT:    movdqa %xmm0, %xmm2
-; CHECK-SSE41-NEXT:    psrld $1, %xmm2
-; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[1,1,3,3]
-; CHECK-SSE41-NEXT:    movdqa {{.*#+}} xmm4 = [2454267027,2454267027,2454267027,2454267027]
-; CHECK-SSE41-NEXT:    pmuludq %xmm4, %xmm3
-; CHECK-SSE41-NEXT:    pmuludq %xmm4, %xmm2
-; CHECK-SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; CHECK-SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm2[0,1],xmm3[2,3],xmm2[4,5],xmm3[6,7]
-; CHECK-SSE41-NEXT:    psrld $2, %xmm2
-; CHECK-SSE41-NEXT:    pmulld {{.*}}(%rip), %xmm2
-; CHECK-SSE41-NEXT:    psubd %xmm2, %xmm0
-; CHECK-SSE41-NEXT:    pcmpeqd %xmm1, %xmm0
-; CHECK-SSE41-NEXT:    psrld $31, %xmm0
-; CHECK-SSE41-NEXT:    retq
-;
-; CHECK-AVX1-LABEL: test_urem_even_vec_i16:
-; CHECK-AVX1:       # %bb.0:
-; CHECK-AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3],xmm0[4],xmm1[5],xmm0[6],xmm1[7]
-; CHECK-AVX1-NEXT:    vpsrld $1, %xmm0, %xmm2
-; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm2[1,1,3,3]
-; CHECK-AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = [2454267027,2454267027,2454267027,2454267027]
-; CHECK-AVX1-NEXT:    vpmuludq %xmm4, %xmm3, %xmm3
-; CHECK-AVX1-NEXT:    vpmuludq %xmm4, %xmm2, %xmm2
-; CHECK-AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; CHECK-AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0,1],xmm3[2,3],xmm2[4,5],xmm3[6,7]
-; CHECK-AVX1-NEXT:    vpsrld $2, %xmm2, %xmm2
-; CHECK-AVX1-NEXT:    vpmulld {{.*}}(%rip), %xmm2, %xmm2
-; CHECK-AVX1-NEXT:    vpsubd %xmm2, %xmm0, %xmm0
-; CHECK-AVX1-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
-; CHECK-AVX1-NEXT:    vpsrld $31, %xmm0, %xmm0
-; CHECK-AVX1-NEXT:    retq
-;
-; CHECK-AVX2-LABEL: test_urem_even_vec_i16:
-; CHECK-AVX2:       # %bb.0:
-; CHECK-AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; CHECK-AVX2-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3],xmm0[4],xmm1[5],xmm0[6],xmm1[7]
-; CHECK-AVX2-NEXT:    vpsrld $1, %xmm0, %xmm2
-; CHECK-AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm2[1,1,3,3]
-; CHECK-AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm4 = [2454267027,2454267027,2454267027,2454267027]
-; CHECK-AVX2-NEXT:    vpmuludq %xmm4, %xmm3, %xmm3
-; CHECK-AVX2-NEXT:    vpmuludq %xmm4, %xmm2, %xmm2
-; CHECK-AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; CHECK-AVX2-NEXT:    vpblendd {{.*#+}} xmm2 = xmm2[0],xmm3[1],xmm2[2],xmm3[3]
-; CHECK-AVX2-NEXT:    vpsrld $2, %xmm2, %xmm2
-; CHECK-AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm3 = [14,14,14,14]
-; CHECK-AVX2-NEXT:    vpmulld %xmm3, %xmm2, %xmm2
-; CHECK-AVX2-NEXT:    vpsubd %xmm2, %xmm0, %xmm0
-; CHECK-AVX2-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
-; CHECK-AVX2-NEXT:    vpsrld $31, %xmm0, %xmm0
-; CHECK-AVX2-NEXT:    retq
+; Like test_urem_even_vec_i32, but with 8 x i16 vectors.
+define <8 x i16> @test_urem_even_vec_i16(<8 x i16> %X) nounwind readnone {
+; CHECK-SSE-LABEL: test_urem_even_vec_i16:
+; CHECK-SSE:       # %bb.0:
+; CHECK-SSE-NEXT:    movdqa %xmm0, %xmm1
+; CHECK-SSE-NEXT:    psrlw $1, %xmm1
+; CHECK-SSE-NEXT:    pmulhuw {{.*}}(%rip), %xmm1
+; CHECK-SSE-NEXT:    psrlw $1, %xmm1
+; CHECK-SSE-NEXT:    pmullw {{.*}}(%rip), %xmm1
+; CHECK-SSE-NEXT:    psubw %xmm1, %xmm0
+; CHECK-SSE-NEXT:    pxor %xmm1, %xmm1
+; CHECK-SSE-NEXT:    pcmpeqw %xmm1, %xmm0
+; CHECK-SSE-NEXT:    psrlw $15, %xmm0
+; CHECK-SSE-NEXT:    retq
 ;
-; CHECK-AVX512VL-LABEL: test_urem_even_vec_i16:
-; CHECK-AVX512VL:       # %bb.0:
-; CHECK-AVX512VL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; CHECK-AVX512VL-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3],xmm0[4],xmm1[5],xmm0[6],xmm1[7]
-; CHECK-AVX512VL-NEXT:    vpsrld $1, %xmm0, %xmm2
-; CHECK-AVX512VL-NEXT:    vpshufd {{.*#+}} xmm3 = xmm2[1,1,3,3]
-; CHECK-AVX512VL-NEXT:    vpbroadcastd {{.*#+}} xmm4 = [2454267027,2454267027,2454267027,2454267027]
-; CHECK-AVX512VL-NEXT:    vpmuludq %xmm4, %xmm3, %xmm3
-; CHECK-AVX512VL-NEXT:    vpmuludq %xmm4, %xmm2, %xmm2
-; CHECK-AVX512VL-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; CHECK-AVX512VL-NEXT:    vpblendd {{.*#+}} xmm2 = xmm2[0],xmm3[1],xmm2[2],xmm3[3]
-; CHECK-AVX512VL-NEXT:    vpsrld $2, %xmm2, %xmm2
-; CHECK-AVX512VL-NEXT:    vpmulld {{.*}}(%rip){1to4}, %xmm2, %xmm2
-; CHECK-AVX512VL-NEXT:    vpsubd %xmm2, %xmm0, %xmm0
-; CHECK-AVX512VL-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
-; CHECK-AVX512VL-NEXT:    vpsrld $31, %xmm0, %xmm0
-; CHECK-AVX512VL-NEXT:    retq
-  %urem = urem <4 x i16> %X, <i16 14, i16 14, i16 14, i16 14>
-  %cmp = icmp eq <4 x i16> %urem, <i16 0, i16 0, i16 0, i16 0>
-  %ret = zext <4 x i1> %cmp to <4 x i16>
-  ret <4 x i16> %ret
+; CHECK-AVX-LABEL: test_urem_even_vec_i16:
+; CHECK-AVX:       # %bb.0:
+; CHECK-AVX-NEXT:    vpsrlw $1, %xmm0, %xmm1
+; CHECK-AVX-NEXT:    vpmulhuw {{.*}}(%rip), %xmm1, %xmm1
+; CHECK-AVX-NEXT:    vpsrlw $1, %xmm1, %xmm1
+; CHECK-AVX-NEXT:    vpmullw {{.*}}(%rip), %xmm1, %xmm1
+; CHECK-AVX-NEXT:    vpsubw %xmm1, %xmm0, %xmm0
+; CHECK-AVX-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; CHECK-AVX-NEXT:    vpcmpeqw %xmm1, %xmm0, %xmm0
+; CHECK-AVX-NEXT:    vpsrlw $15, %xmm0, %xmm0
+; CHECK-AVX-NEXT:    retq
+  %urem = urem <8 x i16> %X, <i16 14, i16 14, i16 14, i16 14,
+                              i16 14, i16 14, i16 14, i16 14>
+  %cmp = icmp eq <8 x i16> %urem, <i16 0, i16 0, i16 0, i16 0,
+                                   i16 0, i16 0, i16 0, i16 0>
+  %ret = zext <8 x i1> %cmp to <8 x i16>
+  ret <8 x i16> %ret
 }
 
 ; We should not proceed with this fold if the divisor is 1 or -1
diff --git a/test/CodeGen/X86/usub_sat.ll b/test/CodeGen/X86/usub_sat.ll
index 1c9a5c56c250f551a7170b981caf72fdc6aecda6..ef822faafebaec7ff64a1be304fde812a162ea17 100644
--- a/test/CodeGen/X86/usub_sat.ll
+++ b/test/CodeGen/X86/usub_sat.ll
@@ -1,158 +1,127 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mcpu=generic -mtriple=x86_64-linux | FileCheck %s
-; RUN: llc < %s -mcpu=generic -mtriple=i686 -mattr=cmov | FileCheck %s --check-prefix=CHECK32
+; RUN: llc < %s -mtriple=i686 -mattr=cmov | FileCheck %s --check-prefixes=CHECK,X86
+; RUN: llc < %s -mtriple=x86_64-linux | FileCheck %s --check-prefixes=CHECK,X64
 
 declare  i4  @llvm.usub.sat.i4   (i4,  i4)
 declare  i32 @llvm.usub.sat.i32  (i32, i32)
 declare  i64 @llvm.usub.sat.i64  (i64, i64)
 declare  <4 x i32> @llvm.usub.sat.v4i32(<4 x i32>, <4 x i32>)
 
-define i32 @func(i32 %x, i32 %y) {
-; CHECK-LABEL: func:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    xorl %eax, %eax
-; CHECK-NEXT:    subl %esi, %edi
-; CHECK-NEXT:    cmovael %edi, %eax
-; CHECK-NEXT:    retq
+define i32 @func(i32 %x, i32 %y) nounwind {
+; X86-LABEL: func:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    xorl %ecx, %ecx
+; X86-NEXT:    subl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    cmovbl %ecx, %eax
+; X86-NEXT:    retl
 ;
-; CHECK32-LABEL: func:
-; CHECK32:       # %bb.0:
-; CHECK32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; CHECK32-NEXT:    xorl %ecx, %ecx
-; CHECK32-NEXT:    subl {{[0-9]+}}(%esp), %eax
-; CHECK32-NEXT:    cmovbl %ecx, %eax
-; CHECK32-NEXT:    retl
+; X64-LABEL: func:
+; X64:       # %bb.0:
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    subl %esi, %edi
+; X64-NEXT:    cmovael %edi, %eax
+; X64-NEXT:    retq
   %tmp = call i32 @llvm.usub.sat.i32(i32 %x, i32 %y);
   ret i32 %tmp;
 }
 
-define i64 @func2(i64 %x, i64 %y) {
-; CHECK-LABEL: func2:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    xorl %eax, %eax
-; CHECK-NEXT:    subq %rsi, %rdi
-; CHECK-NEXT:    cmovaeq %rdi, %rax
-; CHECK-NEXT:    retq
+define i64 @func2(i64 %x, i64 %y) nounwind {
+; X86-LABEL: func2:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    xorl %ecx, %ecx
+; X86-NEXT:    subl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    sbbl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    cmovbl %ecx, %edx
+; X86-NEXT:    cmovbl %ecx, %eax
+; X86-NEXT:    retl
 ;
-; CHECK32-LABEL: func2:
-; CHECK32:       # %bb.0:
-; CHECK32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; CHECK32-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; CHECK32-NEXT:    xorl %ecx, %ecx
-; CHECK32-NEXT:    subl {{[0-9]+}}(%esp), %eax
-; CHECK32-NEXT:    sbbl {{[0-9]+}}(%esp), %edx
-; CHECK32-NEXT:    cmovbl %ecx, %edx
-; CHECK32-NEXT:    cmovbl %ecx, %eax
-; CHECK32-NEXT:    retl
+; X64-LABEL: func2:
+; X64:       # %bb.0:
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    subq %rsi, %rdi
+; X64-NEXT:    cmovaeq %rdi, %rax
+; X64-NEXT:    retq
   %tmp = call i64 @llvm.usub.sat.i64(i64 %x, i64 %y);
   ret i64 %tmp;
 }
 
-define i4 @func3(i4 %x, i4 %y) {
-; CHECK-LABEL: func3:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    movl %edi, %eax
-; CHECK-NEXT:    shlb $4, %sil
-; CHECK-NEXT:    shlb $4, %al
-; CHECK-NEXT:    subb %sil, %al
-; CHECK-NEXT:    jae .LBB2_2
-; CHECK-NEXT:  # %bb.1:
-; CHECK-NEXT:    xorl %eax, %eax
-; CHECK-NEXT:  .LBB2_2:
-; CHECK-NEXT:    shrb $4, %al
-; CHECK-NEXT:    # kill: def $al killed $al killed $eax
-; CHECK-NEXT:    retq
+define i4 @func3(i4 %x, i4 %y) nounwind {
+; X86-LABEL: func3:
+; X86:       # %bb.0:
+; X86-NEXT:    movb {{[0-9]+}}(%esp), %al
+; X86-NEXT:    movb {{[0-9]+}}(%esp), %cl
+; X86-NEXT:    shlb $4, %cl
+; X86-NEXT:    shlb $4, %al
+; X86-NEXT:    subb %cl, %al
+; X86-NEXT:    jae .LBB2_2
+; X86-NEXT:  # %bb.1:
+; X86-NEXT:    xorl %eax, %eax
+; X86-NEXT:  .LBB2_2:
+; X86-NEXT:    shrb $4, %al
+; X86-NEXT:    # kill: def $al killed $al killed $eax
+; X86-NEXT:    retl
 ;
-; CHECK32-LABEL: func3:
-; CHECK32:       # %bb.0:
-; CHECK32-NEXT:    movb {{[0-9]+}}(%esp), %al
-; CHECK32-NEXT:    movb {{[0-9]+}}(%esp), %cl
-; CHECK32-NEXT:    shlb $4, %cl
-; CHECK32-NEXT:    shlb $4, %al
-; CHECK32-NEXT:    subb %cl, %al
-; CHECK32-NEXT:    jae .LBB2_2
-; CHECK32-NEXT:  # %bb.1:
-; CHECK32-NEXT:    xorl %eax, %eax
-; CHECK32-NEXT:  .LBB2_2:
-; CHECK32-NEXT:    shrb $4, %al
-; CHECK32-NEXT:    # kill: def $al killed $al killed $eax
-; CHECK32-NEXT:    retl
+; X64-LABEL: func3:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    shlb $4, %sil
+; X64-NEXT:    shlb $4, %al
+; X64-NEXT:    subb %sil, %al
+; X64-NEXT:    jae .LBB2_2
+; X64-NEXT:  # %bb.1:
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:  .LBB2_2:
+; X64-NEXT:    shrb $4, %al
+; X64-NEXT:    # kill: def $al killed $al killed $eax
+; X64-NEXT:    retq
   %tmp = call i4 @llvm.usub.sat.i4(i4 %x, i4 %y);
   ret i4 %tmp;
 }
 
-define <4 x i32> @vec(<4 x i32> %x, <4 x i32> %y) {
-; CHECK-LABEL: vec:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[3,1,2,3]
-; CHECK-NEXT:    movd %xmm2, %eax
-; CHECK-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[3,1,2,3]
-; CHECK-NEXT:    movd %xmm2, %ecx
-; CHECK-NEXT:    xorl %edx, %edx
-; CHECK-NEXT:    subl %eax, %ecx
-; CHECK-NEXT:    cmovbl %edx, %ecx
-; CHECK-NEXT:    movd %ecx, %xmm2
-; CHECK-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[2,3,0,1]
-; CHECK-NEXT:    movd %xmm3, %eax
-; CHECK-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[2,3,0,1]
-; CHECK-NEXT:    movd %xmm3, %ecx
-; CHECK-NEXT:    subl %eax, %ecx
-; CHECK-NEXT:    cmovbl %edx, %ecx
-; CHECK-NEXT:    movd %ecx, %xmm3
-; CHECK-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
-; CHECK-NEXT:    movd %xmm1, %eax
-; CHECK-NEXT:    movd %xmm0, %ecx
-; CHECK-NEXT:    subl %eax, %ecx
-; CHECK-NEXT:    cmovbl %edx, %ecx
-; CHECK-NEXT:    movd %ecx, %xmm2
-; CHECK-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,2,3]
-; CHECK-NEXT:    movd %xmm1, %eax
-; CHECK-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
-; CHECK-NEXT:    movd %xmm0, %ecx
-; CHECK-NEXT:    subl %eax, %ecx
-; CHECK-NEXT:    cmovbl %edx, %ecx
-; CHECK-NEXT:    movd %ecx, %xmm0
-; CHECK-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
-; CHECK-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; CHECK-NEXT:    movdqa %xmm2, %xmm0
-; CHECK-NEXT:    retq
+define <4 x i32> @vec(<4 x i32> %x, <4 x i32> %y) nounwind {
+; X86-LABEL: vec:
+; X86:       # %bb.0:
+; X86-NEXT:    pushl %ebx
+; X86-NEXT:    pushl %edi
+; X86-NEXT:    pushl %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    xorl %ebx, %ebx
+; X86-NEXT:    subl {{[0-9]+}}(%esp), %edi
+; X86-NEXT:    cmovbl %ebx, %edi
+; X86-NEXT:    subl {{[0-9]+}}(%esp), %esi
+; X86-NEXT:    cmovbl %ebx, %esi
+; X86-NEXT:    subl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    cmovbl %ebx, %edx
+; X86-NEXT:    subl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    cmovbl %ebx, %ecx
+; X86-NEXT:    movl %ecx, 12(%eax)
+; X86-NEXT:    movl %edx, 8(%eax)
+; X86-NEXT:    movl %esi, 4(%eax)
+; X86-NEXT:    movl %edi, (%eax)
+; X86-NEXT:    popl %esi
+; X86-NEXT:    popl %edi
+; X86-NEXT:    popl %ebx
+; X86-NEXT:    retl $4
 ;
-; CHECK32-LABEL: vec:
-; CHECK32:       # %bb.0:
-; CHECK32-NEXT:    pushl %ebx
-; CHECK32-NEXT:    .cfi_def_cfa_offset 8
-; CHECK32-NEXT:    pushl %edi
-; CHECK32-NEXT:    .cfi_def_cfa_offset 12
-; CHECK32-NEXT:    pushl %esi
-; CHECK32-NEXT:    .cfi_def_cfa_offset 16
-; CHECK32-NEXT:    .cfi_offset %esi, -16
-; CHECK32-NEXT:    .cfi_offset %edi, -12
-; CHECK32-NEXT:    .cfi_offset %ebx, -8
-; CHECK32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; CHECK32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; CHECK32-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; CHECK32-NEXT:    movl {{[0-9]+}}(%esp), %esi
-; CHECK32-NEXT:    movl {{[0-9]+}}(%esp), %edi
-; CHECK32-NEXT:    xorl %ebx, %ebx
-; CHECK32-NEXT:    subl {{[0-9]+}}(%esp), %edi
-; CHECK32-NEXT:    cmovbl %ebx, %edi
-; CHECK32-NEXT:    subl {{[0-9]+}}(%esp), %esi
-; CHECK32-NEXT:    cmovbl %ebx, %esi
-; CHECK32-NEXT:    subl {{[0-9]+}}(%esp), %edx
-; CHECK32-NEXT:    cmovbl %ebx, %edx
-; CHECK32-NEXT:    subl {{[0-9]+}}(%esp), %ecx
-; CHECK32-NEXT:    cmovbl %ebx, %ecx
-; CHECK32-NEXT:    movl %ecx, 12(%eax)
-; CHECK32-NEXT:    movl %edx, 8(%eax)
-; CHECK32-NEXT:    movl %esi, 4(%eax)
-; CHECK32-NEXT:    movl %edi, (%eax)
-; CHECK32-NEXT:    popl %esi
-; CHECK32-NEXT:    .cfi_def_cfa_offset 12
-; CHECK32-NEXT:    popl %edi
-; CHECK32-NEXT:    .cfi_def_cfa_offset 8
-; CHECK32-NEXT:    popl %ebx
-; CHECK32-NEXT:    .cfi_def_cfa_offset 4
-; CHECK32-NEXT:    retl $4
+; X64-LABEL: vec:
+; X64:       # %bb.0:
+; X64-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648,2147483648,2147483648]
+; X64-NEXT:    movdqa %xmm1, %xmm3
+; X64-NEXT:    pxor %xmm2, %xmm3
+; X64-NEXT:    pxor %xmm0, %xmm2
+; X64-NEXT:    pcmpgtd %xmm3, %xmm2
+; X64-NEXT:    pand %xmm2, %xmm0
+; X64-NEXT:    pandn %xmm1, %xmm2
+; X64-NEXT:    por %xmm2, %xmm0
+; X64-NEXT:    psubd %xmm1, %xmm0
+; X64-NEXT:    retq
   %tmp = call <4 x i32> @llvm.usub.sat.v4i32(<4 x i32> %x, <4 x i32> %y);
   ret <4 x i32> %tmp;
 }
diff --git a/test/CodeGen/X86/usub_sat_vec.ll b/test/CodeGen/X86/usub_sat_vec.ll
index 47a95423ee762cf5fb4bd08bb709cf911a7484aa..72c0c51ab741cddf629dcb4c77fbc729a2d386b1 100644
--- a/test/CodeGen/X86/usub_sat_vec.ll
+++ b/test/CodeGen/X86/usub_sat_vec.ll
@@ -26,8 +26,14 @@ declare <32 x i16> @llvm.usub.sat.v32i16(<32 x i16>, <32 x i16>)
 declare <16 x i1> @llvm.usub.sat.v16i1(<16 x i1>, <16 x i1>)
 declare <16 x i4> @llvm.usub.sat.v16i4(<16 x i4>, <16 x i4>)
 
-declare <4 x i32> @llvm.usub.sat.v4i32(<4 x i32>, <4 x i32>)
 declare <2 x i32> @llvm.usub.sat.v2i32(<2 x i32>, <2 x i32>)
+declare <4 x i32> @llvm.usub.sat.v4i32(<4 x i32>, <4 x i32>)
+declare <8 x i32> @llvm.usub.sat.v8i32(<8 x i32>, <8 x i32>)
+declare <16 x i32> @llvm.usub.sat.v16i32(<16 x i32>, <16 x i32>)
+declare <2 x i64> @llvm.usub.sat.v2i64(<2 x i64>, <2 x i64>)
+declare <4 x i64> @llvm.usub.sat.v4i64(<4 x i64>, <4 x i64>)
+declare <8 x i64> @llvm.usub.sat.v8i64(<8 x i64>, <8 x i64>)
+
 declare <4 x i24> @llvm.usub.sat.v4i24(<4 x i24>, <4 x i24>)
 declare <2 x i128> @llvm.usub.sat.v2i128(<2 x i128>, <2 x i128>)
 
@@ -624,352 +630,862 @@ define <16 x i1> @v16i1(<16 x i1> %x, <16 x i1> %y) nounwind {
 
 ; Expanded
 
+define <2 x i32> @v2i32(<2 x i32> %x, <2 x i32> %y) nounwind {
+; SSE2-LABEL: v2i32:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    psllq $32, %xmm1
+; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [9223372039002259456,9223372039002259456]
+; SSE2-NEXT:    movdqa %xmm1, %xmm3
+; SSE2-NEXT:    pxor %xmm2, %xmm3
+; SSE2-NEXT:    psllq $32, %xmm0
+; SSE2-NEXT:    pxor %xmm0, %xmm2
+; SSE2-NEXT:    movdqa %xmm2, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm3, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm3, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; SSE2-NEXT:    pand %xmm5, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[1,1,3,3]
+; SSE2-NEXT:    por %xmm2, %xmm3
+; SSE2-NEXT:    pand %xmm3, %xmm0
+; SSE2-NEXT:    pandn %xmm1, %xmm3
+; SSE2-NEXT:    por %xmm3, %xmm0
+; SSE2-NEXT:    psubq %xmm1, %xmm0
+; SSE2-NEXT:    psrlq $32, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: v2i32:
+; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    psllq $32, %xmm1
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm2 = [9223372039002259456,9223372039002259456]
+; SSSE3-NEXT:    movdqa %xmm1, %xmm3
+; SSSE3-NEXT:    pxor %xmm2, %xmm3
+; SSSE3-NEXT:    psllq $32, %xmm0
+; SSSE3-NEXT:    pxor %xmm0, %xmm2
+; SSSE3-NEXT:    movdqa %xmm2, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm3, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm5, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    por %xmm2, %xmm3
+; SSSE3-NEXT:    pand %xmm3, %xmm0
+; SSSE3-NEXT:    pandn %xmm1, %xmm3
+; SSSE3-NEXT:    por %xmm3, %xmm0
+; SSSE3-NEXT:    psubq %xmm1, %xmm0
+; SSSE3-NEXT:    psrlq $32, %xmm0
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: v2i32:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movdqa %xmm0, %xmm2
+; SSE41-NEXT:    psllq $32, %xmm1
+; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [9223372039002259456,9223372039002259456]
+; SSE41-NEXT:    movdqa %xmm1, %xmm3
+; SSE41-NEXT:    pxor %xmm0, %xmm3
+; SSE41-NEXT:    psllq $32, %xmm2
+; SSE41-NEXT:    pxor %xmm2, %xmm0
+; SSE41-NEXT:    movdqa %xmm0, %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm3, %xmm4
+; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSE41-NEXT:    pcmpeqd %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE41-NEXT:    pand %xmm5, %xmm0
+; SSE41-NEXT:    por %xmm4, %xmm0
+; SSE41-NEXT:    movdqa %xmm1, %xmm3
+; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm3
+; SSE41-NEXT:    psubq %xmm1, %xmm3
+; SSE41-NEXT:    psrlq $32, %xmm3
+; SSE41-NEXT:    movdqa %xmm3, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: v2i32:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpsllq $32, %xmm1, %xmm1
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [9223372036854775808,9223372036854775808]
+; AVX1-NEXT:    vpxor %xmm2, %xmm1, %xmm3
+; AVX1-NEXT:    vpsllq $32, %xmm0, %xmm0
+; AVX1-NEXT:    vpxor %xmm2, %xmm0, %xmm2
+; AVX1-NEXT:    vpcmpgtq %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
+; AVX1-NEXT:    vpsubq %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpsrlq $32, %xmm0, %xmm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: v2i32:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpsllq $32, %xmm1, %xmm1
+; AVX2-NEXT:    vmovdqa {{.*#+}} xmm2 = [9223372036854775808,9223372036854775808]
+; AVX2-NEXT:    vpxor %xmm2, %xmm1, %xmm3
+; AVX2-NEXT:    vpsllq $32, %xmm0, %xmm0
+; AVX2-NEXT:    vpxor %xmm2, %xmm0, %xmm2
+; AVX2-NEXT:    vpcmpgtq %xmm3, %xmm2, %xmm2
+; AVX2-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
+; AVX2-NEXT:    vpsubq %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpsrlq $32, %xmm0, %xmm0
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: v2i32:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpsllq $32, %xmm1, %xmm1
+; AVX512-NEXT:    vpsllq $32, %xmm0, %xmm0
+; AVX512-NEXT:    vpmaxuq %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vpsubq %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vpsrlq $32, %xmm0, %xmm0
+; AVX512-NEXT:    retq
+  %z = call <2 x i32> @llvm.usub.sat.v2i32(<2 x i32> %x, <2 x i32> %y)
+  ret <2 x i32> %z
+}
+
 define <4 x i32> @v4i32(<4 x i32> %x, <4 x i32> %y) nounwind {
 ; SSE2-LABEL: v4i32:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[3,1,2,3]
-; SSE2-NEXT:    movd %xmm2, %eax
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[3,1,2,3]
-; SSE2-NEXT:    movd %xmm2, %ecx
-; SSE2-NEXT:    xorl %edx, %edx
-; SSE2-NEXT:    subl %eax, %ecx
-; SSE2-NEXT:    cmovbl %edx, %ecx
-; SSE2-NEXT:    movd %ecx, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[2,3,0,1]
-; SSE2-NEXT:    movd %xmm3, %eax
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[2,3,0,1]
-; SSE2-NEXT:    movd %xmm3, %ecx
-; SSE2-NEXT:    subl %eax, %ecx
-; SSE2-NEXT:    cmovbl %edx, %ecx
-; SSE2-NEXT:    movd %ecx, %xmm3
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
-; SSE2-NEXT:    movd %xmm1, %eax
-; SSE2-NEXT:    movd %xmm0, %ecx
-; SSE2-NEXT:    subl %eax, %ecx
-; SSE2-NEXT:    cmovbl %edx, %ecx
-; SSE2-NEXT:    movd %ecx, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,2,3]
-; SSE2-NEXT:    movd %xmm1, %eax
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
-; SSE2-NEXT:    movd %xmm0, %ecx
-; SSE2-NEXT:    subl %eax, %ecx
-; SSE2-NEXT:    cmovbl %edx, %ecx
-; SSE2-NEXT:    movd %ecx, %xmm0
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
-; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; SSE2-NEXT:    movdqa %xmm2, %xmm0
+; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648,2147483648,2147483648]
+; SSE2-NEXT:    movdqa %xmm1, %xmm3
+; SSE2-NEXT:    pxor %xmm2, %xmm3
+; SSE2-NEXT:    pxor %xmm0, %xmm2
+; SSE2-NEXT:    pcmpgtd %xmm3, %xmm2
+; SSE2-NEXT:    pand %xmm2, %xmm0
+; SSE2-NEXT:    pandn %xmm1, %xmm2
+; SSE2-NEXT:    por %xmm2, %xmm0
+; SSE2-NEXT:    psubd %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSSE3-LABEL: v4i32:
 ; SSSE3:       # %bb.0:
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[3,1,2,3]
-; SSSE3-NEXT:    movd %xmm2, %eax
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[3,1,2,3]
-; SSSE3-NEXT:    movd %xmm2, %ecx
-; SSSE3-NEXT:    xorl %edx, %edx
-; SSSE3-NEXT:    subl %eax, %ecx
-; SSSE3-NEXT:    cmovbl %edx, %ecx
-; SSSE3-NEXT:    movd %ecx, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[2,3,0,1]
-; SSSE3-NEXT:    movd %xmm3, %eax
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[2,3,0,1]
-; SSSE3-NEXT:    movd %xmm3, %ecx
-; SSSE3-NEXT:    subl %eax, %ecx
-; SSSE3-NEXT:    cmovbl %edx, %ecx
-; SSSE3-NEXT:    movd %ecx, %xmm3
-; SSSE3-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
-; SSSE3-NEXT:    movd %xmm1, %eax
-; SSSE3-NEXT:    movd %xmm0, %ecx
-; SSSE3-NEXT:    subl %eax, %ecx
-; SSSE3-NEXT:    cmovbl %edx, %ecx
-; SSSE3-NEXT:    movd %ecx, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,2,3]
-; SSSE3-NEXT:    movd %xmm1, %eax
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
-; SSSE3-NEXT:    movd %xmm0, %ecx
-; SSSE3-NEXT:    subl %eax, %ecx
-; SSSE3-NEXT:    cmovbl %edx, %ecx
-; SSSE3-NEXT:    movd %ecx, %xmm0
-; SSSE3-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
-; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; SSSE3-NEXT:    movdqa %xmm2, %xmm0
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648,2147483648,2147483648]
+; SSSE3-NEXT:    movdqa %xmm1, %xmm3
+; SSSE3-NEXT:    pxor %xmm2, %xmm3
+; SSSE3-NEXT:    pxor %xmm0, %xmm2
+; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm2
+; SSSE3-NEXT:    pand %xmm2, %xmm0
+; SSSE3-NEXT:    pandn %xmm1, %xmm2
+; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    psubd %xmm1, %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: v4i32:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    pextrd $1, %xmm1, %eax
-; SSE41-NEXT:    pextrd $1, %xmm0, %ecx
-; SSE41-NEXT:    xorl %edx, %edx
-; SSE41-NEXT:    subl %eax, %ecx
-; SSE41-NEXT:    cmovbl %edx, %ecx
-; SSE41-NEXT:    movd %xmm1, %eax
-; SSE41-NEXT:    movd %xmm0, %esi
-; SSE41-NEXT:    subl %eax, %esi
-; SSE41-NEXT:    cmovbl %edx, %esi
-; SSE41-NEXT:    movd %esi, %xmm2
-; SSE41-NEXT:    pinsrd $1, %ecx, %xmm2
-; SSE41-NEXT:    pextrd $2, %xmm1, %eax
-; SSE41-NEXT:    pextrd $2, %xmm0, %ecx
-; SSE41-NEXT:    subl %eax, %ecx
-; SSE41-NEXT:    cmovbl %edx, %ecx
-; SSE41-NEXT:    pinsrd $2, %ecx, %xmm2
-; SSE41-NEXT:    pextrd $3, %xmm1, %eax
-; SSE41-NEXT:    pextrd $3, %xmm0, %ecx
-; SSE41-NEXT:    subl %eax, %ecx
-; SSE41-NEXT:    cmovbl %edx, %ecx
-; SSE41-NEXT:    pinsrd $3, %ecx, %xmm2
-; SSE41-NEXT:    movdqa %xmm2, %xmm0
+; SSE41-NEXT:    pmaxud %xmm1, %xmm0
+; SSE41-NEXT:    psubd %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX-LABEL: v4i32:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpextrd $1, %xmm1, %eax
-; AVX-NEXT:    vpextrd $1, %xmm0, %ecx
-; AVX-NEXT:    xorl %edx, %edx
-; AVX-NEXT:    subl %eax, %ecx
-; AVX-NEXT:    cmovbl %edx, %ecx
-; AVX-NEXT:    vmovd %xmm1, %eax
-; AVX-NEXT:    vmovd %xmm0, %esi
-; AVX-NEXT:    subl %eax, %esi
-; AVX-NEXT:    cmovbl %edx, %esi
-; AVX-NEXT:    vmovd %esi, %xmm2
-; AVX-NEXT:    vpinsrd $1, %ecx, %xmm2, %xmm2
-; AVX-NEXT:    vpextrd $2, %xmm1, %eax
-; AVX-NEXT:    vpextrd $2, %xmm0, %ecx
-; AVX-NEXT:    subl %eax, %ecx
-; AVX-NEXT:    cmovbl %edx, %ecx
-; AVX-NEXT:    vpinsrd $2, %ecx, %xmm2, %xmm2
-; AVX-NEXT:    vpextrd $3, %xmm1, %eax
-; AVX-NEXT:    vpextrd $3, %xmm0, %ecx
-; AVX-NEXT:    subl %eax, %ecx
-; AVX-NEXT:    cmovbl %edx, %ecx
-; AVX-NEXT:    vpinsrd $3, %ecx, %xmm2, %xmm0
+; AVX-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    retq
   %z = call <4 x i32> @llvm.usub.sat.v4i32(<4 x i32> %x, <4 x i32> %y)
   ret <4 x i32> %z
 }
 
-define <2 x i32> @v2i32(<2 x i32> %x, <2 x i32> %y) nounwind {
-; SSE2-LABEL: v2i32:
+define <8 x i32> @v8i32(<8 x i32> %x, <8 x i32> %y) nounwind {
+; SSE2-LABEL: v8i32:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    psllq $32, %xmm1
-; SSE2-NEXT:    movq %xmm1, %rax
-; SSE2-NEXT:    psllq $32, %xmm0
-; SSE2-NEXT:    movq %xmm0, %rcx
-; SSE2-NEXT:    xorl %edx, %edx
-; SSE2-NEXT:    subq %rax, %rcx
-; SSE2-NEXT:    cmovbq %rdx, %rcx
-; SSE2-NEXT:    movq %rcx, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
-; SSE2-NEXT:    movq %xmm1, %rax
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; SSE2-NEXT:    movq %xmm0, %rcx
-; SSE2-NEXT:    subq %rax, %rcx
-; SSE2-NEXT:    cmovbq %rdx, %rcx
-; SSE2-NEXT:    movq %rcx, %xmm0
-; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm0[0]
-; SSE2-NEXT:    psrlq $32, %xmm2
-; SSE2-NEXT:    movdqa %xmm2, %xmm0
+; SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [2147483648,2147483648,2147483648,2147483648]
+; SSE2-NEXT:    movdqa %xmm2, %xmm6
+; SSE2-NEXT:    pxor %xmm5, %xmm6
+; SSE2-NEXT:    movdqa %xmm0, %xmm4
+; SSE2-NEXT:    pxor %xmm5, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm6, %xmm4
+; SSE2-NEXT:    pand %xmm4, %xmm0
+; SSE2-NEXT:    pandn %xmm2, %xmm4
+; SSE2-NEXT:    por %xmm0, %xmm4
+; SSE2-NEXT:    psubd %xmm2, %xmm4
+; SSE2-NEXT:    movdqa %xmm3, %xmm0
+; SSE2-NEXT:    pxor %xmm5, %xmm0
+; SSE2-NEXT:    pxor %xmm1, %xmm5
+; SSE2-NEXT:    pcmpgtd %xmm0, %xmm5
+; SSE2-NEXT:    pand %xmm5, %xmm1
+; SSE2-NEXT:    pandn %xmm3, %xmm5
+; SSE2-NEXT:    por %xmm5, %xmm1
+; SSE2-NEXT:    psubd %xmm3, %xmm1
+; SSE2-NEXT:    movdqa %xmm4, %xmm0
 ; SSE2-NEXT:    retq
 ;
-; SSSE3-LABEL: v2i32:
+; SSSE3-LABEL: v8i32:
 ; SSSE3:       # %bb.0:
-; SSSE3-NEXT:    psllq $32, %xmm1
-; SSSE3-NEXT:    movq %xmm1, %rax
-; SSSE3-NEXT:    psllq $32, %xmm0
-; SSSE3-NEXT:    movq %xmm0, %rcx
-; SSSE3-NEXT:    xorl %edx, %edx
-; SSSE3-NEXT:    subq %rax, %rcx
-; SSSE3-NEXT:    cmovbq %rdx, %rcx
-; SSSE3-NEXT:    movq %rcx, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
-; SSSE3-NEXT:    movq %xmm1, %rax
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; SSSE3-NEXT:    movq %xmm0, %rcx
-; SSSE3-NEXT:    subq %rax, %rcx
-; SSSE3-NEXT:    cmovbq %rdx, %rcx
-; SSSE3-NEXT:    movq %rcx, %xmm0
-; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm0[0]
-; SSSE3-NEXT:    psrlq $32, %xmm2
-; SSSE3-NEXT:    movdqa %xmm2, %xmm0
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm5 = [2147483648,2147483648,2147483648,2147483648]
+; SSSE3-NEXT:    movdqa %xmm2, %xmm6
+; SSSE3-NEXT:    pxor %xmm5, %xmm6
+; SSSE3-NEXT:    movdqa %xmm0, %xmm4
+; SSSE3-NEXT:    pxor %xmm5, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm6, %xmm4
+; SSSE3-NEXT:    pand %xmm4, %xmm0
+; SSSE3-NEXT:    pandn %xmm2, %xmm4
+; SSSE3-NEXT:    por %xmm0, %xmm4
+; SSSE3-NEXT:    psubd %xmm2, %xmm4
+; SSSE3-NEXT:    movdqa %xmm3, %xmm0
+; SSSE3-NEXT:    pxor %xmm5, %xmm0
+; SSSE3-NEXT:    pxor %xmm1, %xmm5
+; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm5
+; SSSE3-NEXT:    pand %xmm5, %xmm1
+; SSSE3-NEXT:    pandn %xmm3, %xmm5
+; SSSE3-NEXT:    por %xmm5, %xmm1
+; SSSE3-NEXT:    psubd %xmm3, %xmm1
+; SSSE3-NEXT:    movdqa %xmm4, %xmm0
 ; SSSE3-NEXT:    retq
 ;
-; SSE41-LABEL: v2i32:
+; SSE41-LABEL: v8i32:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    psllq $32, %xmm1
-; SSE41-NEXT:    pextrq $1, %xmm1, %rax
-; SSE41-NEXT:    psllq $32, %xmm0
-; SSE41-NEXT:    pextrq $1, %xmm0, %rcx
-; SSE41-NEXT:    xorl %edx, %edx
-; SSE41-NEXT:    subq %rax, %rcx
-; SSE41-NEXT:    cmovbq %rdx, %rcx
-; SSE41-NEXT:    movq %rcx, %xmm2
-; SSE41-NEXT:    movq %xmm1, %rax
-; SSE41-NEXT:    movq %xmm0, %rcx
-; SSE41-NEXT:    subq %rax, %rcx
-; SSE41-NEXT:    cmovbq %rdx, %rcx
-; SSE41-NEXT:    movq %rcx, %xmm0
-; SSE41-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
-; SSE41-NEXT:    psrlq $32, %xmm0
+; SSE41-NEXT:    pmaxud %xmm2, %xmm0
+; SSE41-NEXT:    psubd %xmm2, %xmm0
+; SSE41-NEXT:    pmaxud %xmm3, %xmm1
+; SSE41-NEXT:    psubd %xmm3, %xmm1
 ; SSE41-NEXT:    retq
 ;
-; AVX-LABEL: v2i32:
-; AVX:       # %bb.0:
-; AVX-NEXT:    vpsllq $32, %xmm1, %xmm1
-; AVX-NEXT:    vpextrq $1, %xmm1, %rax
-; AVX-NEXT:    vpsllq $32, %xmm0, %xmm0
-; AVX-NEXT:    vpextrq $1, %xmm0, %rcx
-; AVX-NEXT:    xorl %edx, %edx
-; AVX-NEXT:    subq %rax, %rcx
-; AVX-NEXT:    cmovbq %rdx, %rcx
-; AVX-NEXT:    vmovq %rcx, %xmm2
-; AVX-NEXT:    vmovq %xmm1, %rax
-; AVX-NEXT:    vmovq %xmm0, %rcx
-; AVX-NEXT:    subq %rax, %rcx
-; AVX-NEXT:    cmovbq %rdx, %rcx
-; AVX-NEXT:    vmovq %rcx, %xmm0
-; AVX-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
-; AVX-NEXT:    vpsrlq $32, %xmm0, %xmm0
-; AVX-NEXT:    retq
-  %z = call <2 x i32> @llvm.usub.sat.v2i32(<2 x i32> %x, <2 x i32> %y)
-  ret <2 x i32> %z
+; AVX1-LABEL: v8i32:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
+; AVX1-NEXT:    vpmaxud %xmm2, %xmm3, %xmm3
+; AVX1-NEXT:    vpsubd %xmm2, %xmm3, %xmm2
+; AVX1-NEXT:    vpmaxud %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: v8i32:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpmaxud %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpsubd %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: v8i32:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpmaxud %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpsubd %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    retq
+  %z = call <8 x i32> @llvm.usub.sat.v8i32(<8 x i32> %x, <8 x i32> %y)
+  ret <8 x i32> %z
 }
 
-define <4 x i24> @v4i24(<4 x i24> %x, <4 x i24> %y) nounwind {
-; SSE2-LABEL: v4i24:
+define <16 x i32> @v16i32(<16 x i32> %x, <16 x i32> %y) nounwind {
+; SSE2-LABEL: v16i32:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pslld $8, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[3,1,2,3]
-; SSE2-NEXT:    movd %xmm2, %eax
-; SSE2-NEXT:    pslld $8, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[3,1,2,3]
-; SSE2-NEXT:    movd %xmm2, %ecx
-; SSE2-NEXT:    xorl %edx, %edx
-; SSE2-NEXT:    subl %eax, %ecx
-; SSE2-NEXT:    cmovbl %edx, %ecx
-; SSE2-NEXT:    movd %ecx, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[2,3,0,1]
-; SSE2-NEXT:    movd %xmm3, %eax
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[2,3,0,1]
-; SSE2-NEXT:    movd %xmm3, %ecx
-; SSE2-NEXT:    subl %eax, %ecx
-; SSE2-NEXT:    cmovbl %edx, %ecx
-; SSE2-NEXT:    movd %ecx, %xmm3
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
-; SSE2-NEXT:    movd %xmm1, %eax
-; SSE2-NEXT:    movd %xmm0, %ecx
-; SSE2-NEXT:    subl %eax, %ecx
-; SSE2-NEXT:    cmovbl %edx, %ecx
-; SSE2-NEXT:    movd %ecx, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,2,3]
-; SSE2-NEXT:    movd %xmm1, %eax
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
-; SSE2-NEXT:    movd %xmm0, %ecx
-; SSE2-NEXT:    subl %eax, %ecx
-; SSE2-NEXT:    cmovbl %edx, %ecx
-; SSE2-NEXT:    movd %ecx, %xmm0
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
-; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; SSE2-NEXT:    psrld $8, %xmm2
-; SSE2-NEXT:    movdqa %xmm2, %xmm0
+; SSE2-NEXT:    movdqa %xmm1, %xmm8
+; SSE2-NEXT:    movdqa %xmm0, %xmm10
+; SSE2-NEXT:    movdqa {{.*#+}} xmm9 = [2147483648,2147483648,2147483648,2147483648]
+; SSE2-NEXT:    movdqa %xmm4, %xmm1
+; SSE2-NEXT:    pxor %xmm9, %xmm1
+; SSE2-NEXT:    pxor %xmm9, %xmm0
+; SSE2-NEXT:    pcmpgtd %xmm1, %xmm0
+; SSE2-NEXT:    pand %xmm0, %xmm10
+; SSE2-NEXT:    pandn %xmm4, %xmm0
+; SSE2-NEXT:    por %xmm10, %xmm0
+; SSE2-NEXT:    psubd %xmm4, %xmm0
+; SSE2-NEXT:    movdqa %xmm5, %xmm4
+; SSE2-NEXT:    pxor %xmm9, %xmm4
+; SSE2-NEXT:    movdqa %xmm8, %xmm1
+; SSE2-NEXT:    pxor %xmm9, %xmm1
+; SSE2-NEXT:    pcmpgtd %xmm4, %xmm1
+; SSE2-NEXT:    pand %xmm1, %xmm8
+; SSE2-NEXT:    pandn %xmm5, %xmm1
+; SSE2-NEXT:    por %xmm8, %xmm1
+; SSE2-NEXT:    psubd %xmm5, %xmm1
+; SSE2-NEXT:    movdqa %xmm6, %xmm5
+; SSE2-NEXT:    pxor %xmm9, %xmm5
+; SSE2-NEXT:    movdqa %xmm2, %xmm4
+; SSE2-NEXT:    pxor %xmm9, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm5, %xmm4
+; SSE2-NEXT:    pand %xmm4, %xmm2
+; SSE2-NEXT:    pandn %xmm6, %xmm4
+; SSE2-NEXT:    por %xmm2, %xmm4
+; SSE2-NEXT:    psubd %xmm6, %xmm4
+; SSE2-NEXT:    movdqa %xmm7, %xmm2
+; SSE2-NEXT:    pxor %xmm9, %xmm2
+; SSE2-NEXT:    pxor %xmm3, %xmm9
+; SSE2-NEXT:    pcmpgtd %xmm2, %xmm9
+; SSE2-NEXT:    pand %xmm9, %xmm3
+; SSE2-NEXT:    pandn %xmm7, %xmm9
+; SSE2-NEXT:    por %xmm9, %xmm3
+; SSE2-NEXT:    psubd %xmm7, %xmm3
+; SSE2-NEXT:    movdqa %xmm4, %xmm2
 ; SSE2-NEXT:    retq
 ;
-; SSSE3-LABEL: v4i24:
+; SSSE3-LABEL: v16i32:
 ; SSSE3:       # %bb.0:
-; SSSE3-NEXT:    pslld $8, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[3,1,2,3]
-; SSSE3-NEXT:    movd %xmm2, %eax
-; SSSE3-NEXT:    pslld $8, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[3,1,2,3]
-; SSSE3-NEXT:    movd %xmm2, %ecx
-; SSSE3-NEXT:    xorl %edx, %edx
-; SSSE3-NEXT:    subl %eax, %ecx
-; SSSE3-NEXT:    cmovbl %edx, %ecx
-; SSSE3-NEXT:    movd %ecx, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[2,3,0,1]
-; SSSE3-NEXT:    movd %xmm3, %eax
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[2,3,0,1]
-; SSSE3-NEXT:    movd %xmm3, %ecx
-; SSSE3-NEXT:    subl %eax, %ecx
-; SSSE3-NEXT:    cmovbl %edx, %ecx
-; SSSE3-NEXT:    movd %ecx, %xmm3
-; SSSE3-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
-; SSSE3-NEXT:    movd %xmm1, %eax
-; SSSE3-NEXT:    movd %xmm0, %ecx
-; SSSE3-NEXT:    subl %eax, %ecx
-; SSSE3-NEXT:    cmovbl %edx, %ecx
-; SSSE3-NEXT:    movd %ecx, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,2,3]
-; SSSE3-NEXT:    movd %xmm1, %eax
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
-; SSSE3-NEXT:    movd %xmm0, %ecx
-; SSSE3-NEXT:    subl %eax, %ecx
-; SSSE3-NEXT:    cmovbl %edx, %ecx
-; SSSE3-NEXT:    movd %ecx, %xmm0
-; SSSE3-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
-; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; SSSE3-NEXT:    psrld $8, %xmm2
-; SSSE3-NEXT:    movdqa %xmm2, %xmm0
+; SSSE3-NEXT:    movdqa %xmm1, %xmm8
+; SSSE3-NEXT:    movdqa %xmm0, %xmm10
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm9 = [2147483648,2147483648,2147483648,2147483648]
+; SSSE3-NEXT:    movdqa %xmm4, %xmm1
+; SSSE3-NEXT:    pxor %xmm9, %xmm1
+; SSSE3-NEXT:    pxor %xmm9, %xmm0
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm0
+; SSSE3-NEXT:    pand %xmm0, %xmm10
+; SSSE3-NEXT:    pandn %xmm4, %xmm0
+; SSSE3-NEXT:    por %xmm10, %xmm0
+; SSSE3-NEXT:    psubd %xmm4, %xmm0
+; SSSE3-NEXT:    movdqa %xmm5, %xmm4
+; SSSE3-NEXT:    pxor %xmm9, %xmm4
+; SSSE3-NEXT:    movdqa %xmm8, %xmm1
+; SSSE3-NEXT:    pxor %xmm9, %xmm1
+; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm1
+; SSSE3-NEXT:    pand %xmm1, %xmm8
+; SSSE3-NEXT:    pandn %xmm5, %xmm1
+; SSSE3-NEXT:    por %xmm8, %xmm1
+; SSSE3-NEXT:    psubd %xmm5, %xmm1
+; SSSE3-NEXT:    movdqa %xmm6, %xmm5
+; SSSE3-NEXT:    pxor %xmm9, %xmm5
+; SSSE3-NEXT:    movdqa %xmm2, %xmm4
+; SSSE3-NEXT:    pxor %xmm9, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm5, %xmm4
+; SSSE3-NEXT:    pand %xmm4, %xmm2
+; SSSE3-NEXT:    pandn %xmm6, %xmm4
+; SSSE3-NEXT:    por %xmm2, %xmm4
+; SSSE3-NEXT:    psubd %xmm6, %xmm4
+; SSSE3-NEXT:    movdqa %xmm7, %xmm2
+; SSSE3-NEXT:    pxor %xmm9, %xmm2
+; SSSE3-NEXT:    pxor %xmm3, %xmm9
+; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm9
+; SSSE3-NEXT:    pand %xmm9, %xmm3
+; SSSE3-NEXT:    pandn %xmm7, %xmm9
+; SSSE3-NEXT:    por %xmm9, %xmm3
+; SSSE3-NEXT:    psubd %xmm7, %xmm3
+; SSSE3-NEXT:    movdqa %xmm4, %xmm2
 ; SSSE3-NEXT:    retq
 ;
-; SSE41-LABEL: v4i24:
+; SSE41-LABEL: v16i32:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    pslld $8, %xmm1
-; SSE41-NEXT:    pextrd $1, %xmm1, %eax
-; SSE41-NEXT:    pslld $8, %xmm0
-; SSE41-NEXT:    pextrd $1, %xmm0, %ecx
-; SSE41-NEXT:    xorl %edx, %edx
-; SSE41-NEXT:    subl %eax, %ecx
-; SSE41-NEXT:    cmovbl %edx, %ecx
-; SSE41-NEXT:    movd %xmm1, %eax
-; SSE41-NEXT:    movd %xmm0, %esi
-; SSE41-NEXT:    subl %eax, %esi
-; SSE41-NEXT:    cmovbl %edx, %esi
-; SSE41-NEXT:    movd %esi, %xmm2
-; SSE41-NEXT:    pinsrd $1, %ecx, %xmm2
-; SSE41-NEXT:    pextrd $2, %xmm1, %eax
-; SSE41-NEXT:    pextrd $2, %xmm0, %ecx
-; SSE41-NEXT:    subl %eax, %ecx
-; SSE41-NEXT:    cmovbl %edx, %ecx
-; SSE41-NEXT:    pinsrd $2, %ecx, %xmm2
-; SSE41-NEXT:    pextrd $3, %xmm1, %eax
-; SSE41-NEXT:    pextrd $3, %xmm0, %ecx
-; SSE41-NEXT:    subl %eax, %ecx
-; SSE41-NEXT:    cmovbl %edx, %ecx
-; SSE41-NEXT:    pinsrd $3, %ecx, %xmm2
-; SSE41-NEXT:    psrld $8, %xmm2
-; SSE41-NEXT:    movdqa %xmm2, %xmm0
+; SSE41-NEXT:    pmaxud %xmm4, %xmm0
+; SSE41-NEXT:    psubd %xmm4, %xmm0
+; SSE41-NEXT:    pmaxud %xmm5, %xmm1
+; SSE41-NEXT:    psubd %xmm5, %xmm1
+; SSE41-NEXT:    pmaxud %xmm6, %xmm2
+; SSE41-NEXT:    psubd %xmm6, %xmm2
+; SSE41-NEXT:    pmaxud %xmm7, %xmm3
+; SSE41-NEXT:    psubd %xmm7, %xmm3
 ; SSE41-NEXT:    retq
 ;
-; AVX-LABEL: v4i24:
-; AVX:       # %bb.0:
-; AVX-NEXT:    vpslld $8, %xmm1, %xmm1
-; AVX-NEXT:    vpextrd $1, %xmm1, %eax
-; AVX-NEXT:    vpslld $8, %xmm0, %xmm0
-; AVX-NEXT:    vpextrd $1, %xmm0, %ecx
-; AVX-NEXT:    xorl %edx, %edx
-; AVX-NEXT:    subl %eax, %ecx
-; AVX-NEXT:    cmovbl %edx, %ecx
-; AVX-NEXT:    vmovd %xmm1, %eax
-; AVX-NEXT:    vmovd %xmm0, %esi
-; AVX-NEXT:    subl %eax, %esi
-; AVX-NEXT:    cmovbl %edx, %esi
-; AVX-NEXT:    vmovd %esi, %xmm2
-; AVX-NEXT:    vpinsrd $1, %ecx, %xmm2, %xmm2
-; AVX-NEXT:    vpextrd $2, %xmm1, %eax
-; AVX-NEXT:    vpextrd $2, %xmm0, %ecx
-; AVX-NEXT:    subl %eax, %ecx
-; AVX-NEXT:    cmovbl %edx, %ecx
-; AVX-NEXT:    vpinsrd $2, %ecx, %xmm2, %xmm2
-; AVX-NEXT:    vpextrd $3, %xmm1, %eax
-; AVX-NEXT:    vpextrd $3, %xmm0, %ecx
-; AVX-NEXT:    subl %eax, %ecx
-; AVX-NEXT:    cmovbl %edx, %ecx
-; AVX-NEXT:    vpinsrd $3, %ecx, %xmm2, %xmm0
-; AVX-NEXT:    vpsrld $8, %xmm0, %xmm0
-; AVX-NEXT:    retq
-  %z = call <4 x i24> @llvm.usub.sat.v4i24(<4 x i24> %x, <4 x i24> %y)
-  ret <4 x i24> %z
+; AVX1-LABEL: v16i32:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm4
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm5
+; AVX1-NEXT:    vpmaxud %xmm4, %xmm5, %xmm5
+; AVX1-NEXT:    vpsubd %xmm4, %xmm5, %xmm4
+; AVX1-NEXT:    vpmaxud %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpsubd %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm0, %ymm0
+; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm2
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm4
+; AVX1-NEXT:    vpmaxud %xmm2, %xmm4, %xmm4
+; AVX1-NEXT:    vpsubd %xmm2, %xmm4, %xmm2
+; AVX1-NEXT:    vpmaxud %xmm3, %xmm1, %xmm1
+; AVX1-NEXT:    vpsubd %xmm3, %xmm1, %xmm1
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: v16i32:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpmaxud %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vpsubd %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vpmaxud %ymm3, %ymm1, %ymm1
+; AVX2-NEXT:    vpsubd %ymm3, %ymm1, %ymm1
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: v16i32:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpmaxud %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpsubd %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    retq
+  %z = call <16 x i32> @llvm.usub.sat.v16i32(<16 x i32> %x, <16 x i32> %y)
+  ret <16 x i32> %z
+}
+
+define <2 x i64> @v2i64(<2 x i64> %x, <2 x i64> %y) nounwind {
+; SSE2-LABEL: v2i64:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [9223372039002259456,9223372039002259456]
+; SSE2-NEXT:    movdqa %xmm1, %xmm3
+; SSE2-NEXT:    pxor %xmm2, %xmm3
+; SSE2-NEXT:    pxor %xmm0, %xmm2
+; SSE2-NEXT:    movdqa %xmm2, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm3, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm3, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; SSE2-NEXT:    pand %xmm5, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[1,1,3,3]
+; SSE2-NEXT:    por %xmm2, %xmm3
+; SSE2-NEXT:    pand %xmm3, %xmm0
+; SSE2-NEXT:    pandn %xmm1, %xmm3
+; SSE2-NEXT:    por %xmm3, %xmm0
+; SSE2-NEXT:    psubq %xmm1, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: v2i64:
+; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm2 = [9223372039002259456,9223372039002259456]
+; SSSE3-NEXT:    movdqa %xmm1, %xmm3
+; SSSE3-NEXT:    pxor %xmm2, %xmm3
+; SSSE3-NEXT:    pxor %xmm0, %xmm2
+; SSSE3-NEXT:    movdqa %xmm2, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm3, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm5, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    por %xmm2, %xmm3
+; SSSE3-NEXT:    pand %xmm3, %xmm0
+; SSSE3-NEXT:    pandn %xmm1, %xmm3
+; SSSE3-NEXT:    por %xmm3, %xmm0
+; SSSE3-NEXT:    psubq %xmm1, %xmm0
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: v2i64:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movdqa %xmm0, %xmm2
+; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [9223372039002259456,9223372039002259456]
+; SSE41-NEXT:    movdqa %xmm1, %xmm3
+; SSE41-NEXT:    pxor %xmm0, %xmm3
+; SSE41-NEXT:    pxor %xmm2, %xmm0
+; SSE41-NEXT:    movdqa %xmm0, %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm3, %xmm4
+; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSE41-NEXT:    pcmpeqd %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE41-NEXT:    pand %xmm5, %xmm0
+; SSE41-NEXT:    por %xmm4, %xmm0
+; SSE41-NEXT:    movdqa %xmm1, %xmm3
+; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm3
+; SSE41-NEXT:    psubq %xmm1, %xmm3
+; SSE41-NEXT:    movdqa %xmm3, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: v2i64:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [9223372036854775808,9223372036854775808]
+; AVX1-NEXT:    vpxor %xmm2, %xmm1, %xmm3
+; AVX1-NEXT:    vpxor %xmm2, %xmm0, %xmm2
+; AVX1-NEXT:    vpcmpgtq %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
+; AVX1-NEXT:    vpsubq %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: v2i64:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vmovdqa {{.*#+}} xmm2 = [9223372036854775808,9223372036854775808]
+; AVX2-NEXT:    vpxor %xmm2, %xmm1, %xmm3
+; AVX2-NEXT:    vpxor %xmm2, %xmm0, %xmm2
+; AVX2-NEXT:    vpcmpgtq %xmm3, %xmm2, %xmm2
+; AVX2-NEXT:    vblendvpd %xmm2, %xmm0, %xmm1, %xmm0
+; AVX2-NEXT:    vpsubq %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: v2i64:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpmaxuq %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vpsubq %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    retq
+  %z = call <2 x i64> @llvm.usub.sat.v2i64(<2 x i64> %x, <2 x i64> %y)
+  ret <2 x i64> %z
+}
+
+define <4 x i64> @v4i64(<4 x i64> %x, <4 x i64> %y) nounwind {
+; SSE2-LABEL: v4i64:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [9223372039002259456,9223372039002259456]
+; SSE2-NEXT:    movdqa %xmm2, %xmm5
+; SSE2-NEXT:    pxor %xmm4, %xmm5
+; SSE2-NEXT:    movdqa %xmm0, %xmm6
+; SSE2-NEXT:    pxor %xmm4, %xmm6
+; SSE2-NEXT:    movdqa %xmm6, %xmm7
+; SSE2-NEXT:    pcmpgtd %xmm5, %xmm7
+; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm7[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm5, %xmm6
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm6[1,1,3,3]
+; SSE2-NEXT:    pand %xmm8, %xmm5
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm7[1,1,3,3]
+; SSE2-NEXT:    por %xmm5, %xmm6
+; SSE2-NEXT:    pand %xmm6, %xmm0
+; SSE2-NEXT:    pandn %xmm2, %xmm6
+; SSE2-NEXT:    por %xmm6, %xmm0
+; SSE2-NEXT:    psubq %xmm2, %xmm0
+; SSE2-NEXT:    movdqa %xmm3, %xmm2
+; SSE2-NEXT:    pxor %xmm4, %xmm2
+; SSE2-NEXT:    pxor %xmm1, %xmm4
+; SSE2-NEXT:    movdqa %xmm4, %xmm5
+; SSE2-NEXT:    pcmpgtd %xmm2, %xmm5
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm2, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm4[1,1,3,3]
+; SSE2-NEXT:    pand %xmm6, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm5[1,1,3,3]
+; SSE2-NEXT:    por %xmm2, %xmm4
+; SSE2-NEXT:    pand %xmm4, %xmm1
+; SSE2-NEXT:    pandn %xmm3, %xmm4
+; SSE2-NEXT:    por %xmm4, %xmm1
+; SSE2-NEXT:    psubq %xmm3, %xmm1
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: v4i64:
+; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm4 = [9223372039002259456,9223372039002259456]
+; SSSE3-NEXT:    movdqa %xmm2, %xmm5
+; SSSE3-NEXT:    pxor %xmm4, %xmm5
+; SSSE3-NEXT:    movdqa %xmm0, %xmm6
+; SSSE3-NEXT:    pxor %xmm4, %xmm6
+; SSSE3-NEXT:    movdqa %xmm6, %xmm7
+; SSSE3-NEXT:    pcmpgtd %xmm5, %xmm7
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm8 = xmm7[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm6
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm6[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm8, %xmm5
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm7[1,1,3,3]
+; SSSE3-NEXT:    por %xmm5, %xmm6
+; SSSE3-NEXT:    pand %xmm6, %xmm0
+; SSSE3-NEXT:    pandn %xmm2, %xmm6
+; SSSE3-NEXT:    por %xmm6, %xmm0
+; SSSE3-NEXT:    psubq %xmm2, %xmm0
+; SSSE3-NEXT:    movdqa %xmm3, %xmm2
+; SSSE3-NEXT:    pxor %xmm4, %xmm2
+; SSSE3-NEXT:    pxor %xmm1, %xmm4
+; SSSE3-NEXT:    movdqa %xmm4, %xmm5
+; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm5
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm6, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm5[1,1,3,3]
+; SSSE3-NEXT:    por %xmm2, %xmm4
+; SSSE3-NEXT:    pand %xmm4, %xmm1
+; SSSE3-NEXT:    pandn %xmm3, %xmm4
+; SSSE3-NEXT:    por %xmm4, %xmm1
+; SSSE3-NEXT:    psubq %xmm3, %xmm1
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: v4i64:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movdqa %xmm0, %xmm4
+; SSE41-NEXT:    movdqa {{.*#+}} xmm6 = [9223372039002259456,9223372039002259456]
+; SSE41-NEXT:    movdqa %xmm2, %xmm5
+; SSE41-NEXT:    pxor %xmm6, %xmm5
+; SSE41-NEXT:    movdqa %xmm0, %xmm7
+; SSE41-NEXT:    pxor %xmm6, %xmm7
+; SSE41-NEXT:    movdqa %xmm7, %xmm0
+; SSE41-NEXT:    pcmpgtd %xmm5, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm8 = xmm0[0,0,2,2]
+; SSE41-NEXT:    pcmpeqd %xmm5, %xmm7
+; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm7[1,1,3,3]
+; SSE41-NEXT:    pand %xmm8, %xmm5
+; SSE41-NEXT:    por %xmm5, %xmm0
+; SSE41-NEXT:    movdqa %xmm2, %xmm5
+; SSE41-NEXT:    blendvpd %xmm0, %xmm4, %xmm5
+; SSE41-NEXT:    psubq %xmm2, %xmm5
+; SSE41-NEXT:    movdqa %xmm3, %xmm0
+; SSE41-NEXT:    pxor %xmm6, %xmm0
+; SSE41-NEXT:    pxor %xmm1, %xmm6
+; SSE41-NEXT:    movdqa %xmm6, %xmm2
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm2
+; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[0,0,2,2]
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm6
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[1,1,3,3]
+; SSE41-NEXT:    pand %xmm4, %xmm0
+; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    movdqa %xmm3, %xmm2
+; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm2
+; SSE41-NEXT:    psubq %xmm3, %xmm2
+; SSE41-NEXT:    movdqa %xmm5, %xmm0
+; SSE41-NEXT:    movdqa %xmm2, %xmm1
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: v4i64:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [9223372036854775808,9223372036854775808]
+; AVX1-NEXT:    vpxor %xmm3, %xmm2, %xmm4
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm5
+; AVX1-NEXT:    vpxor %xmm3, %xmm5, %xmm5
+; AVX1-NEXT:    vpcmpgtq %xmm4, %xmm5, %xmm4
+; AVX1-NEXT:    vpxor %xmm3, %xmm1, %xmm5
+; AVX1-NEXT:    vpxor %xmm3, %xmm0, %xmm3
+; AVX1-NEXT:    vpcmpgtq %xmm5, %xmm3, %xmm3
+; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm3, %ymm3
+; AVX1-NEXT:    vblendvpd %ymm3, %ymm0, %ymm1, %ymm0
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
+; AVX1-NEXT:    vpsubq %xmm2, %xmm3, %xmm2
+; AVX1-NEXT:    vpsubq %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: v4i64:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808]
+; AVX2-NEXT:    vpxor %ymm2, %ymm1, %ymm3
+; AVX2-NEXT:    vpxor %ymm2, %ymm0, %ymm2
+; AVX2-NEXT:    vpcmpgtq %ymm3, %ymm2, %ymm2
+; AVX2-NEXT:    vblendvpd %ymm2, %ymm0, %ymm1, %ymm0
+; AVX2-NEXT:    vpsubq %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: v4i64:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpmaxuq %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpsubq %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    retq
+  %z = call <4 x i64> @llvm.usub.sat.v4i64(<4 x i64> %x, <4 x i64> %y)
+  ret <4 x i64> %z
+}
+
+define <8 x i64> @v8i64(<8 x i64> %x, <8 x i64> %y) nounwind {
+; SSE2-LABEL: v8i64:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa {{.*#+}} xmm8 = [9223372039002259456,9223372039002259456]
+; SSE2-NEXT:    movdqa %xmm4, %xmm9
+; SSE2-NEXT:    pxor %xmm8, %xmm9
+; SSE2-NEXT:    movdqa %xmm0, %xmm10
+; SSE2-NEXT:    pxor %xmm8, %xmm10
+; SSE2-NEXT:    movdqa %xmm10, %xmm11
+; SSE2-NEXT:    pcmpgtd %xmm9, %xmm11
+; SSE2-NEXT:    pshufd {{.*#+}} xmm12 = xmm11[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm9, %xmm10
+; SSE2-NEXT:    pshufd {{.*#+}} xmm9 = xmm10[1,1,3,3]
+; SSE2-NEXT:    pand %xmm12, %xmm9
+; SSE2-NEXT:    pshufd {{.*#+}} xmm10 = xmm11[1,1,3,3]
+; SSE2-NEXT:    por %xmm9, %xmm10
+; SSE2-NEXT:    pand %xmm10, %xmm0
+; SSE2-NEXT:    pandn %xmm4, %xmm10
+; SSE2-NEXT:    por %xmm10, %xmm0
+; SSE2-NEXT:    psubq %xmm4, %xmm0
+; SSE2-NEXT:    movdqa %xmm5, %xmm9
+; SSE2-NEXT:    pxor %xmm8, %xmm9
+; SSE2-NEXT:    movdqa %xmm1, %xmm4
+; SSE2-NEXT:    pxor %xmm8, %xmm4
+; SSE2-NEXT:    movdqa %xmm4, %xmm10
+; SSE2-NEXT:    pcmpgtd %xmm9, %xmm10
+; SSE2-NEXT:    pshufd {{.*#+}} xmm11 = xmm10[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm9, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm9 = xmm4[1,1,3,3]
+; SSE2-NEXT:    pand %xmm11, %xmm9
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm10[1,1,3,3]
+; SSE2-NEXT:    por %xmm9, %xmm4
+; SSE2-NEXT:    pand %xmm4, %xmm1
+; SSE2-NEXT:    pandn %xmm5, %xmm4
+; SSE2-NEXT:    por %xmm4, %xmm1
+; SSE2-NEXT:    psubq %xmm5, %xmm1
+; SSE2-NEXT:    movdqa %xmm6, %xmm4
+; SSE2-NEXT:    pxor %xmm8, %xmm4
+; SSE2-NEXT:    movdqa %xmm2, %xmm5
+; SSE2-NEXT:    pxor %xmm8, %xmm5
+; SSE2-NEXT:    movdqa %xmm5, %xmm9
+; SSE2-NEXT:    pcmpgtd %xmm4, %xmm9
+; SSE2-NEXT:    pshufd {{.*#+}} xmm10 = xmm9[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm4, %xmm5
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm5[1,1,3,3]
+; SSE2-NEXT:    pand %xmm10, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm9[1,1,3,3]
+; SSE2-NEXT:    por %xmm4, %xmm5
+; SSE2-NEXT:    pand %xmm5, %xmm2
+; SSE2-NEXT:    pandn %xmm6, %xmm5
+; SSE2-NEXT:    por %xmm5, %xmm2
+; SSE2-NEXT:    psubq %xmm6, %xmm2
+; SSE2-NEXT:    movdqa %xmm7, %xmm4
+; SSE2-NEXT:    pxor %xmm8, %xmm4
+; SSE2-NEXT:    pxor %xmm3, %xmm8
+; SSE2-NEXT:    movdqa %xmm8, %xmm5
+; SSE2-NEXT:    pcmpgtd %xmm4, %xmm5
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm4, %xmm8
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm8[1,1,3,3]
+; SSE2-NEXT:    pand %xmm6, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
+; SSE2-NEXT:    por %xmm4, %xmm5
+; SSE2-NEXT:    pand %xmm5, %xmm3
+; SSE2-NEXT:    pandn %xmm7, %xmm5
+; SSE2-NEXT:    por %xmm5, %xmm3
+; SSE2-NEXT:    psubq %xmm7, %xmm3
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: v8i64:
+; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm8 = [9223372039002259456,9223372039002259456]
+; SSSE3-NEXT:    movdqa %xmm4, %xmm9
+; SSSE3-NEXT:    pxor %xmm8, %xmm9
+; SSSE3-NEXT:    movdqa %xmm0, %xmm10
+; SSSE3-NEXT:    pxor %xmm8, %xmm10
+; SSSE3-NEXT:    movdqa %xmm10, %xmm11
+; SSSE3-NEXT:    pcmpgtd %xmm9, %xmm11
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm12 = xmm11[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm10
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm9 = xmm10[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm12, %xmm9
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm10 = xmm11[1,1,3,3]
+; SSSE3-NEXT:    por %xmm9, %xmm10
+; SSSE3-NEXT:    pand %xmm10, %xmm0
+; SSSE3-NEXT:    pandn %xmm4, %xmm10
+; SSSE3-NEXT:    por %xmm10, %xmm0
+; SSSE3-NEXT:    psubq %xmm4, %xmm0
+; SSSE3-NEXT:    movdqa %xmm5, %xmm9
+; SSSE3-NEXT:    pxor %xmm8, %xmm9
+; SSSE3-NEXT:    movdqa %xmm1, %xmm4
+; SSSE3-NEXT:    pxor %xmm8, %xmm4
+; SSSE3-NEXT:    movdqa %xmm4, %xmm10
+; SSSE3-NEXT:    pcmpgtd %xmm9, %xmm10
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm11 = xmm10[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm9 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm11, %xmm9
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm10[1,1,3,3]
+; SSSE3-NEXT:    por %xmm9, %xmm4
+; SSSE3-NEXT:    pand %xmm4, %xmm1
+; SSSE3-NEXT:    pandn %xmm5, %xmm4
+; SSSE3-NEXT:    por %xmm4, %xmm1
+; SSSE3-NEXT:    psubq %xmm5, %xmm1
+; SSSE3-NEXT:    movdqa %xmm6, %xmm4
+; SSSE3-NEXT:    pxor %xmm8, %xmm4
+; SSSE3-NEXT:    movdqa %xmm2, %xmm5
+; SSSE3-NEXT:    pxor %xmm8, %xmm5
+; SSSE3-NEXT:    movdqa %xmm5, %xmm9
+; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm9
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm10 = xmm9[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm4, %xmm5
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm5[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm10, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm9[1,1,3,3]
+; SSSE3-NEXT:    por %xmm4, %xmm5
+; SSSE3-NEXT:    pand %xmm5, %xmm2
+; SSSE3-NEXT:    pandn %xmm6, %xmm5
+; SSSE3-NEXT:    por %xmm5, %xmm2
+; SSSE3-NEXT:    psubq %xmm6, %xmm2
+; SSSE3-NEXT:    movdqa %xmm7, %xmm4
+; SSSE3-NEXT:    pxor %xmm8, %xmm4
+; SSSE3-NEXT:    pxor %xmm3, %xmm8
+; SSSE3-NEXT:    movdqa %xmm8, %xmm5
+; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm5
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm4, %xmm8
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm8[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm6, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
+; SSSE3-NEXT:    por %xmm4, %xmm5
+; SSSE3-NEXT:    pand %xmm5, %xmm3
+; SSSE3-NEXT:    pandn %xmm7, %xmm5
+; SSSE3-NEXT:    por %xmm5, %xmm3
+; SSSE3-NEXT:    psubq %xmm7, %xmm3
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: v8i64:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movdqa %xmm1, %xmm8
+; SSE41-NEXT:    movdqa %xmm0, %xmm11
+; SSE41-NEXT:    movdqa {{.*#+}} xmm10 = [9223372039002259456,9223372039002259456]
+; SSE41-NEXT:    movdqa %xmm4, %xmm9
+; SSE41-NEXT:    pxor %xmm10, %xmm9
+; SSE41-NEXT:    movdqa %xmm0, %xmm1
+; SSE41-NEXT:    pxor %xmm10, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    pcmpgtd %xmm9, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm12 = xmm0[0,0,2,2]
+; SSE41-NEXT:    pcmpeqd %xmm9, %xmm1
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSE41-NEXT:    pand %xmm12, %xmm1
+; SSE41-NEXT:    por %xmm1, %xmm0
+; SSE41-NEXT:    movdqa %xmm4, %xmm9
+; SSE41-NEXT:    blendvpd %xmm0, %xmm11, %xmm9
+; SSE41-NEXT:    psubq %xmm4, %xmm9
+; SSE41-NEXT:    movdqa %xmm5, %xmm0
+; SSE41-NEXT:    pxor %xmm10, %xmm0
+; SSE41-NEXT:    movdqa %xmm8, %xmm1
+; SSE41-NEXT:    pxor %xmm10, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm4
+; SSE41-NEXT:    pshufd {{.*#+}} xmm11 = xmm4[0,0,2,2]
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,1,3,3]
+; SSE41-NEXT:    pand %xmm11, %xmm0
+; SSE41-NEXT:    por %xmm4, %xmm0
+; SSE41-NEXT:    movdqa %xmm5, %xmm1
+; SSE41-NEXT:    blendvpd %xmm0, %xmm8, %xmm1
+; SSE41-NEXT:    psubq %xmm5, %xmm1
+; SSE41-NEXT:    movdqa %xmm6, %xmm0
+; SSE41-NEXT:    pxor %xmm10, %xmm0
+; SSE41-NEXT:    movdqa %xmm2, %xmm4
+; SSE41-NEXT:    pxor %xmm10, %xmm4
+; SSE41-NEXT:    movdqa %xmm4, %xmm5
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm5
+; SSE41-NEXT:    pshufd {{.*#+}} xmm8 = xmm5[0,0,2,2]
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm4
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[1,1,3,3]
+; SSE41-NEXT:    pand %xmm8, %xmm0
+; SSE41-NEXT:    por %xmm5, %xmm0
+; SSE41-NEXT:    movdqa %xmm6, %xmm4
+; SSE41-NEXT:    blendvpd %xmm0, %xmm2, %xmm4
+; SSE41-NEXT:    psubq %xmm6, %xmm4
+; SSE41-NEXT:    movdqa %xmm7, %xmm0
+; SSE41-NEXT:    pxor %xmm10, %xmm0
+; SSE41-NEXT:    pxor %xmm3, %xmm10
+; SSE41-NEXT:    movdqa %xmm10, %xmm2
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm2
+; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm2[0,0,2,2]
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm10
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm10[1,1,3,3]
+; SSE41-NEXT:    pand %xmm5, %xmm0
+; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    movdqa %xmm7, %xmm5
+; SSE41-NEXT:    blendvpd %xmm0, %xmm3, %xmm5
+; SSE41-NEXT:    psubq %xmm7, %xmm5
+; SSE41-NEXT:    movdqa %xmm9, %xmm0
+; SSE41-NEXT:    movdqa %xmm4, %xmm2
+; SSE41-NEXT:    movdqa %xmm5, %xmm3
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: v8i64:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm4
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm5 = [9223372036854775808,9223372036854775808]
+; AVX1-NEXT:    vpxor %xmm5, %xmm4, %xmm6
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm7
+; AVX1-NEXT:    vpxor %xmm5, %xmm7, %xmm7
+; AVX1-NEXT:    vpcmpgtq %xmm6, %xmm7, %xmm8
+; AVX1-NEXT:    vpxor %xmm5, %xmm2, %xmm7
+; AVX1-NEXT:    vpxor %xmm5, %xmm0, %xmm6
+; AVX1-NEXT:    vpcmpgtq %xmm7, %xmm6, %xmm6
+; AVX1-NEXT:    vinsertf128 $1, %xmm8, %ymm6, %ymm6
+; AVX1-NEXT:    vblendvpd %ymm6, %ymm0, %ymm2, %ymm0
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm6
+; AVX1-NEXT:    vpsubq %xmm4, %xmm6, %xmm4
+; AVX1-NEXT:    vpsubq %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm0, %ymm0
+; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm2
+; AVX1-NEXT:    vpxor %xmm5, %xmm2, %xmm4
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm6
+; AVX1-NEXT:    vpxor %xmm5, %xmm6, %xmm6
+; AVX1-NEXT:    vpcmpgtq %xmm4, %xmm6, %xmm4
+; AVX1-NEXT:    vpxor %xmm5, %xmm3, %xmm6
+; AVX1-NEXT:    vpxor %xmm5, %xmm1, %xmm5
+; AVX1-NEXT:    vpcmpgtq %xmm6, %xmm5, %xmm5
+; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm5, %ymm4
+; AVX1-NEXT:    vblendvpd %ymm4, %ymm1, %ymm3, %ymm1
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm4
+; AVX1-NEXT:    vpsubq %xmm2, %xmm4, %xmm2
+; AVX1-NEXT:    vpsubq %xmm3, %xmm1, %xmm1
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: v8i64:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpbroadcastq {{.*#+}} ymm4 = [9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808]
+; AVX2-NEXT:    vpxor %ymm4, %ymm2, %ymm5
+; AVX2-NEXT:    vpxor %ymm4, %ymm0, %ymm6
+; AVX2-NEXT:    vpcmpgtq %ymm5, %ymm6, %ymm5
+; AVX2-NEXT:    vblendvpd %ymm5, %ymm0, %ymm2, %ymm0
+; AVX2-NEXT:    vpsubq %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vpxor %ymm4, %ymm3, %ymm2
+; AVX2-NEXT:    vpxor %ymm4, %ymm1, %ymm4
+; AVX2-NEXT:    vpcmpgtq %ymm2, %ymm4, %ymm2
+; AVX2-NEXT:    vblendvpd %ymm2, %ymm1, %ymm3, %ymm1
+; AVX2-NEXT:    vpsubq %ymm3, %ymm1, %ymm1
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: v8i64:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpmaxuq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpsubq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    retq
+  %z = call <8 x i64> @llvm.usub.sat.v8i64(<8 x i64> %x, <8 x i64> %y)
+  ret <8 x i64> %z
 }
 
 define <2 x i128> @v2i128(<2 x i128> %x, <2 x i128> %y) nounwind {
diff --git a/test/CodeGen/X86/vec-copysign.ll b/test/CodeGen/X86/vec-copysign.ll
index 852ebcd3e45216be26973fdf4ba6c70d215aa25c..78ceb52db54df870144ee69b7b783af63cd560aa 100644
--- a/test/CodeGen/X86/vec-copysign.ll
+++ b/test/CodeGen/X86/vec-copysign.ll
@@ -1,5 +1,5 @@
-; RUN: llc < %s -mtriple=x86_64-apple-macosx10.10.0 -mattr=+sse2 | FileCheck %s --check-prefix=SSE2 --check-prefix=CHECK
-; RUN: llc < %s -mtriple=x86_64-apple-macosx10.10.0 -mattr=+avx | FileCheck %s --check-prefix=AVX --check-prefix=CHECK
+; RUN: llc < %s -mtriple=x86_64-apple-macosx10.10.0 -mattr=+sse2 | FileCheck %s --check-prefixes=CHECK,SSE2
+; RUN: llc < %s -mtriple=x86_64-apple-macosx10.10.0 -mattr=+avx | FileCheck %s --check-prefixes=CHECK,AVX
 
 ; Assertions have been enhanced from utils/update_llc_test_checks.py to show the constant pool values.
 ; Use a macosx triple to make sure the format of those constant strings is exact.
@@ -35,12 +35,6 @@ define <4 x float> @v4f32(<4 x float> %a, <4 x float> %b) nounwind {
   ret <4 x float> %tmp
 }
 
-; SSE2:       [[SIGNMASK2:L.+]]:
-; SSE2-NEXT:  .long 2147483648
-; SSE2-NEXT:  .long 2147483648
-; SSE2-NEXT:  .long 2147483648
-; SSE2-NEXT:  .long 2147483648
-
 ; SSE2:       [[MAGMASK2:L.+]]:
 ; SSE2-NEXT:  .long 2147483647
 ; SSE2-NEXT:  .long 2147483647
@@ -70,14 +64,14 @@ define <4 x float> @v4f32(<4 x float> %a, <4 x float> %b) nounwind {
 define <8 x float> @v8f32(<8 x float> %a, <8 x float> %b) nounwind {
 ; SSE2-LABEL: v8f32:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    movaps [[SIGNMASK2]](%rip), %xmm4
-; SSE2-NEXT:    andps %xmm4, %xmm2
-; SSE2-NEXT:    movaps [[MAGMASK2]](%rip), %xmm5
-; SSE2-NEXT:    andps %xmm5, %xmm0
-; SSE2-NEXT:    orps %xmm2, %xmm0
-; SSE2-NEXT:    andps %xmm4, %xmm3
-; SSE2-NEXT:    andps %xmm5, %xmm1
-; SSE2-NEXT:    orps %xmm3, %xmm1
+; SSE2-NEXT:    movaps [[MAGMASK2]](%rip), %xmm4
+; SSE2-NEXT:    movaps %xmm4, %xmm5
+; SSE2-NEXT:    andnps %xmm2, %xmm5
+; SSE2-NEXT:    andps %xmm4, %xmm0
+; SSE2-NEXT:    orps %xmm5, %xmm0
+; SSE2-NEXT:    andps %xmm4, %xmm1
+; SSE2-NEXT:    andnps %xmm3, %xmm4
+; SSE2-NEXT:    orps %xmm4, %xmm1
 ; SSE2-NEXT:    retq
 ;
 ; AVX-LABEL: v8f32:
@@ -118,10 +112,6 @@ define <2 x double> @v2f64(<2 x double> %a, <2 x double> %b) nounwind {
   ret <2 x double> %tmp
 }
 
-; SSE2:        [[SIGNMASK4:L.+]]:
-; SSE2-NEXT:   .quad -9223372036854775808
-; SSE2-NEXT:   .quad -9223372036854775808
-
 ; SSE2:        [[MAGMASK4:L.+]]:
 ; SSE2-NEXT:   .quad 9223372036854775807
 ; SSE2-NEXT:   .quad 9223372036854775807
@@ -141,14 +131,14 @@ define <2 x double> @v2f64(<2 x double> %a, <2 x double> %b) nounwind {
 define <4 x double> @v4f64(<4 x double> %a, <4 x double> %b) nounwind {
 ; SSE2-LABEL: v4f64:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    movaps [[SIGNMASK4]](%rip), %xmm4
-; SSE2-NEXT:    andps %xmm4, %xmm2
-; SSE2-NEXT:    movaps [[MAGMASK4]](%rip), %xmm5
-; SSE2-NEXT:    andps %xmm5, %xmm0
-; SSE2-NEXT:    orps %xmm2, %xmm0
-; SSE2-NEXT:    andps %xmm4, %xmm3
-; SSE2-NEXT:    andps %xmm5, %xmm1
-; SSE2-NEXT:    orps %xmm3, %xmm1
+; SSE2-NEXT:    movaps [[MAGMASK4]](%rip), %xmm4
+; SSE2-NEXT:    movaps %xmm4, %xmm5
+; SSE2-NEXT:    andnps %xmm2, %xmm5
+; SSE2-NEXT:    andps %xmm4, %xmm0
+; SSE2-NEXT:    orps %xmm5, %xmm0
+; SSE2-NEXT:    andps %xmm4, %xmm1
+; SSE2-NEXT:    andnps %xmm3, %xmm4
+; SSE2-NEXT:    orps %xmm4, %xmm1
 ; SSE2-NEXT:    retq
 ;
 ; AVX-LABEL: v4f64:
diff --git a/test/CodeGen/X86/vec_cast.ll b/test/CodeGen/X86/vec_cast.ll
index 23870bcf5654aa3684eae1c813c8d14241180eb2..6e9a16768951179ecbfdc46943097f710ecec52f 100644
--- a/test/CodeGen/X86/vec_cast.ll
+++ b/test/CodeGen/X86/vec_cast.ll
@@ -14,11 +14,10 @@ define <8 x i32> @a(<8 x i16> %a) nounwind {
 ;
 ; CHECK-WIN-LABEL: a:
 ; CHECK-WIN:       # %bb.0:
-; CHECK-WIN-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
-; CHECK-WIN-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
+; CHECK-WIN-NEXT:    movdqa (%rcx), %xmm1
+; CHECK-WIN-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
 ; CHECK-WIN-NEXT:    psrad $16, %xmm0
-; CHECK-WIN-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero
-; CHECK-WIN-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3]
+; CHECK-WIN-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
 ; CHECK-WIN-NEXT:    psrad $16, %xmm1
 ; CHECK-WIN-NEXT:    retq
   %c = sext <8 x i16> %a to <8 x i32>
diff --git a/test/CodeGen/X86/vec_cast3.ll b/test/CodeGen/X86/vec_cast3.ll
index e4e6aa52ff56c85c911acb026f8f0c3b590cd631..e4ff93a228161cedcce17b8235966edc8fc7dda1 100644
--- a/test/CodeGen/X86/vec_cast3.ll
+++ b/test/CodeGen/X86/vec_cast3.ll
@@ -236,3 +236,26 @@ define <2 x i32> @cvt_v2f32_v2u32(<2 x float> %src) {
   %res = fptoui <2 x float> %src to <2 x i32>
   ret <2 x i32> %res
 }
+
+define <32 x i8> @PR40146(<4 x i64> %x) {
+; CHECK-LABEL: PR40146:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; CHECK-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; CHECK-NEXT:    vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; CHECK-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; CHECK-NEXT:    retl
+;
+; CHECK-WIDE-LABEL: PR40146:
+; CHECK-WIDE:       ## %bb.0:
+; CHECK-WIDE-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; CHECK-WIDE-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; CHECK-WIDE-NEXT:    vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; CHECK-WIDE-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; CHECK-WIDE-NEXT:    retl
+  %perm = shufflevector <4 x i64> %x, <4 x i64> undef, <4 x i32> <i32 0, i32 undef, i32 1, i32 undef>
+  %t1 = bitcast <4 x i64> %perm to <32 x i8>
+  %t2 = shufflevector <32 x i8> %t1, <32 x i8> <i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef>, <32 x i32> <i32 0, i32 32, i32 1, i32 32, i32 2, i32 32, i32 3, i32 32, i32 4, i32 32, i32 5, i32 32, i32 6, i32 32, i32 7, i32 32, i32 16, i32 48, i32 17, i32 48, i32 18, i32 48, i32 19, i32 48, i32 20, i32 48, i32 21, i32 48, i32 22, i32 48, i32 23, i32 48>
+  ret <32 x i8> %t2
+}
+
diff --git a/test/CodeGen/X86/vec_int_to_fp.ll b/test/CodeGen/X86/vec_int_to_fp.ll
index 1ccd636a7c172eafe444f37a6f27f97214a63869..1f2749fd0994747fac9ec463d72e459e2007a0ee 100644
--- a/test/CodeGen/X86/vec_int_to_fp.ll
+++ b/test/CodeGen/X86/vec_int_to_fp.ll
@@ -4278,13 +4278,12 @@ define <8 x float> @sitofp_load_8i32_to_8f32(<8 x i32> *%a) {
 define <8 x float> @sitofp_load_8i16_to_8f32(<8 x i16> *%a) {
 ; SSE2-LABEL: sitofp_load_8i16_to_8f32:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
-; SSE2-NEXT:    psrad $16, %xmm1
-; SSE2-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
+; SSE2-NEXT:    movdqa (%rdi), %xmm1
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
 ; SSE2-NEXT:    psrad $16, %xmm0
 ; SSE2-NEXT:    cvtdq2ps %xmm0, %xmm0
+; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    psrad $16, %xmm1
 ; SSE2-NEXT:    cvtdq2ps %xmm1, %xmm1
 ; SSE2-NEXT:    retq
 ;
@@ -4323,15 +4322,13 @@ define <8 x float> @sitofp_load_8i16_to_8f32(<8 x i16> *%a) {
 define <8 x float> @sitofp_load_8i8_to_8f32(<8 x i8> *%a) {
 ; SSE2-LABEL: sitofp_load_8i8_to_8f32:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
-; SSE2-NEXT:    psrad $24, %xmm1
-; SSE2-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
+; SSE2-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
 ; SSE2-NEXT:    psrad $24, %xmm0
 ; SSE2-NEXT:    cvtdq2ps %xmm0, %xmm0
+; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    psrad $24, %xmm1
 ; SSE2-NEXT:    cvtdq2ps %xmm1, %xmm1
 ; SSE2-NEXT:    retq
 ;
@@ -4345,10 +4342,8 @@ define <8 x float> @sitofp_load_8i8_to_8f32(<8 x i8> *%a) {
 ;
 ; AVX1-LABEL: sitofp_load_8i8_to_8f32:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpmovsxbw (%rdi), %xmm0
-; AVX1-NEXT:    vpmovsxwd %xmm0, %xmm1
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; AVX1-NEXT:    vpmovsxwd %xmm0, %xmm0
+; AVX1-NEXT:    vpmovsxbd 4(%rdi), %xmm0
+; AVX1-NEXT:    vpmovsxbd (%rdi), %xmm1
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; AVX1-NEXT:    vcvtdq2ps %ymm0, %ymm0
 ; AVX1-NEXT:    retq
@@ -5555,3 +5550,292 @@ define <4 x float> @sitofp_i64_to_4f32(<4 x float> %a0, i64 %a1) nounwind {
   %res = insertelement <4 x float> %a0, float %cvt, i32 0
   ret <4 x float> %res
 }
+
+; Extract from int vector and convert to FP.
+
+define float @extract0_sitofp_v4i32_f32(<4 x i32> %x) nounwind {
+; SSE-LABEL: extract0_sitofp_v4i32_f32:
+; SSE:       # %bb.0:
+; SSE-NEXT:    cvtdq2ps %xmm0, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: extract0_sitofp_v4i32_f32:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vcvtdq2ps %xmm0, %xmm0
+; AVX-NEXT:    retq
+  %e = extractelement <4 x i32> %x, i32 0
+  %r = sitofp i32 %e to float
+  ret float %r
+}
+
+define float @extract0_sitofp_v4i32_f32i_multiuse1(<4 x i32> %x) nounwind {
+; SSE-LABEL: extract0_sitofp_v4i32_f32i_multiuse1:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movd %xmm0, %eax
+; SSE-NEXT:    cvtdq2ps %xmm0, %xmm0
+; SSE-NEXT:    incl %eax
+; SSE-NEXT:    cvtsi2ssl %eax, %xmm1
+; SSE-NEXT:    divss %xmm1, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: extract0_sitofp_v4i32_f32i_multiuse1:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovd %xmm0, %eax
+; AVX-NEXT:    vcvtdq2ps %xmm0, %xmm0
+; AVX-NEXT:    incl %eax
+; AVX-NEXT:    vcvtsi2ssl %eax, %xmm1, %xmm1
+; AVX-NEXT:    vdivss %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    retq
+  %e = extractelement <4 x i32> %x, i32 0
+  %f = sitofp i32 %e to float
+  %e1 = add i32 %e, 1
+  %f1 = sitofp i32 %e1 to float
+  %r = fdiv float %f, %f1
+  ret float %r
+}
+
+define float @extract0_sitofp_v4i32_f32_multiuse2(<4 x i32> %x, i32* %p) nounwind {
+; SSE-LABEL: extract0_sitofp_v4i32_f32_multiuse2:
+; SSE:       # %bb.0:
+; SSE-NEXT:    cvtdq2ps %xmm0, %xmm1
+; SSE-NEXT:    movss %xmm0, (%rdi)
+; SSE-NEXT:    movaps %xmm1, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: extract0_sitofp_v4i32_f32_multiuse2:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vcvtdq2ps %xmm0, %xmm1
+; AVX-NEXT:    vmovss %xmm0, (%rdi)
+; AVX-NEXT:    vmovaps %xmm1, %xmm0
+; AVX-NEXT:    retq
+  %e = extractelement <4 x i32> %x, i32 0
+  %r = sitofp i32 %e to float
+  store i32 %e, i32* %p
+  ret float %r
+}
+
+define double @extract0_sitofp_v4i32_f64(<4 x i32> %x) nounwind {
+; SSE-LABEL: extract0_sitofp_v4i32_f64:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movd %xmm0, %eax
+; SSE-NEXT:    xorps %xmm0, %xmm0
+; SSE-NEXT:    cvtsi2sdl %eax, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: extract0_sitofp_v4i32_f64:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vcvtdq2pd %xmm0, %xmm0
+; AVX-NEXT:    retq
+  %e = extractelement <4 x i32> %x, i32 0
+  %r = sitofp i32 %e to double
+  ret double %r
+}
+
+define float @extract0_uitofp_v4i32_f32(<4 x i32> %x) nounwind {
+; SSE-LABEL: extract0_uitofp_v4i32_f32:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movd %xmm0, %eax
+; SSE-NEXT:    xorps %xmm0, %xmm0
+; SSE-NEXT:    cvtsi2ssq %rax, %xmm0
+; SSE-NEXT:    retq
+;
+; VEX-LABEL: extract0_uitofp_v4i32_f32:
+; VEX:       # %bb.0:
+; VEX-NEXT:    vmovd %xmm0, %eax
+; VEX-NEXT:    vcvtsi2ssq %rax, %xmm1, %xmm0
+; VEX-NEXT:    retq
+;
+; AVX512F-LABEL: extract0_uitofp_v4i32_f32:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512F-NEXT:    vcvtudq2ps %zmm0, %zmm0
+; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: extract0_uitofp_v4i32_f32:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vcvtudq2ps %xmm0, %xmm0
+; AVX512VL-NEXT:    retq
+;
+; AVX512DQ-LABEL: extract0_uitofp_v4i32_f32:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512DQ-NEXT:    vcvtudq2ps %zmm0, %zmm0
+; AVX512DQ-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512DQ-NEXT:    vzeroupper
+; AVX512DQ-NEXT:    retq
+;
+; AVX512VLDQ-LABEL: extract0_uitofp_v4i32_f32:
+; AVX512VLDQ:       # %bb.0:
+; AVX512VLDQ-NEXT:    vcvtudq2ps %xmm0, %xmm0
+; AVX512VLDQ-NEXT:    retq
+  %e = extractelement <4 x i32> %x, i32 0
+  %r = uitofp i32 %e to float
+  ret float %r
+}
+
+define double @extract0_uitofp_v4i32_f64(<4 x i32> %x) nounwind {
+; SSE-LABEL: extract0_uitofp_v4i32_f64:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movd %xmm0, %eax
+; SSE-NEXT:    xorps %xmm0, %xmm0
+; SSE-NEXT:    cvtsi2sdq %rax, %xmm0
+; SSE-NEXT:    retq
+;
+; VEX-LABEL: extract0_uitofp_v4i32_f64:
+; VEX:       # %bb.0:
+; VEX-NEXT:    vmovd %xmm0, %eax
+; VEX-NEXT:    vcvtsi2sdq %rax, %xmm1, %xmm0
+; VEX-NEXT:    retq
+;
+; AVX512F-LABEL: extract0_uitofp_v4i32_f64:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
+; AVX512F-NEXT:    vcvtudq2pd %ymm0, %zmm0
+; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: extract0_uitofp_v4i32_f64:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vcvtudq2pd %xmm0, %ymm0
+; AVX512VL-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX512VL-NEXT:    vzeroupper
+; AVX512VL-NEXT:    retq
+;
+; AVX512DQ-LABEL: extract0_uitofp_v4i32_f64:
+; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
+; AVX512DQ-NEXT:    vcvtudq2pd %ymm0, %zmm0
+; AVX512DQ-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512DQ-NEXT:    vzeroupper
+; AVX512DQ-NEXT:    retq
+;
+; AVX512VLDQ-LABEL: extract0_uitofp_v4i32_f64:
+; AVX512VLDQ:       # %bb.0:
+; AVX512VLDQ-NEXT:    vcvtudq2pd %xmm0, %ymm0
+; AVX512VLDQ-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX512VLDQ-NEXT:    vzeroupper
+; AVX512VLDQ-NEXT:    retq
+  %e = extractelement <4 x i32> %x, i32 0
+  %r = uitofp i32 %e to double
+  ret double %r
+}
+
+; Extract non-zero element from int vector and convert to FP.
+
+define float @extract3_sitofp_v4i32_f32(<4 x i32> %x) nounwind {
+; SSE2-LABEL: extract3_sitofp_v4i32_f32:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE2-NEXT:    cvtdq2ps %xmm0, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: extract3_sitofp_v4i32_f32:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    extractps $3, %xmm0, %eax
+; SSE41-NEXT:    xorps %xmm0, %xmm0
+; SSE41-NEXT:    cvtsi2ssl %eax, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: extract3_sitofp_v4i32_f32:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vextractps $3, %xmm0, %eax
+; AVX-NEXT:    vcvtsi2ssl %eax, %xmm1, %xmm0
+; AVX-NEXT:    retq
+  %e = extractelement <4 x i32> %x, i32 3
+  %r = sitofp i32 %e to float
+  ret float %r
+}
+
+define double @extract3_sitofp_v4i32_f64(<4 x i32> %x) nounwind {
+; SSE2-LABEL: extract3_sitofp_v4i32_f64:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE2-NEXT:    movd %xmm0, %eax
+; SSE2-NEXT:    xorps %xmm0, %xmm0
+; SSE2-NEXT:    cvtsi2sdl %eax, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: extract3_sitofp_v4i32_f64:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    extractps $3, %xmm0, %eax
+; SSE41-NEXT:    xorps %xmm0, %xmm0
+; SSE41-NEXT:    cvtsi2sdl %eax, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: extract3_sitofp_v4i32_f64:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vextractps $3, %xmm0, %eax
+; AVX-NEXT:    vcvtsi2sdl %eax, %xmm1, %xmm0
+; AVX-NEXT:    retq
+  %e = extractelement <4 x i32> %x, i32 3
+  %r = sitofp i32 %e to double
+  ret double %r
+}
+
+define float @extract3_uitofp_v4i32_f32(<4 x i32> %x) nounwind {
+; SSE2-LABEL: extract3_uitofp_v4i32_f32:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE2-NEXT:    movd %xmm0, %eax
+; SSE2-NEXT:    xorps %xmm0, %xmm0
+; SSE2-NEXT:    cvtsi2ssq %rax, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: extract3_uitofp_v4i32_f32:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    extractps $3, %xmm0, %eax
+; SSE41-NEXT:    xorps %xmm0, %xmm0
+; SSE41-NEXT:    cvtsi2ssq %rax, %xmm0
+; SSE41-NEXT:    retq
+;
+; VEX-LABEL: extract3_uitofp_v4i32_f32:
+; VEX:       # %bb.0:
+; VEX-NEXT:    vextractps $3, %xmm0, %eax
+; VEX-NEXT:    vcvtsi2ssq %rax, %xmm1, %xmm0
+; VEX-NEXT:    retq
+;
+; AVX512-LABEL: extract3_uitofp_v4i32_f32:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vextractps $3, %xmm0, %eax
+; AVX512-NEXT:    vcvtusi2ssl %eax, %xmm1, %xmm0
+; AVX512-NEXT:    retq
+  %e = extractelement <4 x i32> %x, i32 3
+  %r = uitofp i32 %e to float
+  ret float %r
+}
+
+define double @extract3_uitofp_v4i32_f64(<4 x i32> %x) nounwind {
+; SSE2-LABEL: extract3_uitofp_v4i32_f64:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[3,1,2,3]
+; SSE2-NEXT:    movd %xmm0, %eax
+; SSE2-NEXT:    xorps %xmm0, %xmm0
+; SSE2-NEXT:    cvtsi2sdq %rax, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSE41-LABEL: extract3_uitofp_v4i32_f64:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    extractps $3, %xmm0, %eax
+; SSE41-NEXT:    xorps %xmm0, %xmm0
+; SSE41-NEXT:    cvtsi2sdq %rax, %xmm0
+; SSE41-NEXT:    retq
+;
+; VEX-LABEL: extract3_uitofp_v4i32_f64:
+; VEX:       # %bb.0:
+; VEX-NEXT:    vextractps $3, %xmm0, %eax
+; VEX-NEXT:    vcvtsi2sdq %rax, %xmm1, %xmm0
+; VEX-NEXT:    retq
+;
+; AVX512-LABEL: extract3_uitofp_v4i32_f64:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vextractps $3, %xmm0, %eax
+; AVX512-NEXT:    vcvtusi2sdl %eax, %xmm1, %xmm0
+; AVX512-NEXT:    retq
+  %e = extractelement <4 x i32> %x, i32 3
+  %r = uitofp i32 %e to double
+  ret double %r
+}
+
diff --git a/test/CodeGen/X86/vec_saddo.ll b/test/CodeGen/X86/vec_saddo.ll
new file mode 100644
index 0000000000000000000000000000000000000000..dbdc412f81d12f2198c7268d099bc8e4a79bd6f2
--- /dev/null
+++ b/test/CodeGen/X86/vec_saddo.ll
@@ -0,0 +1,2028 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefixes=SSE,SSE2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+ssse3 | FileCheck %s --check-prefixes=SSE,SSSE3
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefixes=SSE,SSE41
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefixes=AVX,AVX1
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefixes=AVX,AVX2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw,+avx512vl,+fast-variable-shuffle | FileCheck %s --check-prefixes=AVX,AVX512
+
+declare {<1 x i32>, <1 x i1>} @llvm.sadd.with.overflow.v1i32(<1 x i32>, <1 x i32>)
+declare {<2 x i32>, <2 x i1>} @llvm.sadd.with.overflow.v2i32(<2 x i32>, <2 x i32>)
+declare {<3 x i32>, <3 x i1>} @llvm.sadd.with.overflow.v3i32(<3 x i32>, <3 x i32>)
+declare {<4 x i32>, <4 x i1>} @llvm.sadd.with.overflow.v4i32(<4 x i32>, <4 x i32>)
+declare {<6 x i32>, <6 x i1>} @llvm.sadd.with.overflow.v6i32(<6 x i32>, <6 x i32>)
+declare {<8 x i32>, <8 x i1>} @llvm.sadd.with.overflow.v8i32(<8 x i32>, <8 x i32>)
+declare {<16 x i32>, <16 x i1>} @llvm.sadd.with.overflow.v16i32(<16 x i32>, <16 x i32>)
+
+declare {<16 x i8>, <16 x i1>} @llvm.sadd.with.overflow.v16i8(<16 x i8>, <16 x i8>)
+declare {<8 x i16>, <8 x i1>} @llvm.sadd.with.overflow.v8i16(<8 x i16>, <8 x i16>)
+declare {<2 x i64>, <2 x i1>} @llvm.sadd.with.overflow.v2i64(<2 x i64>, <2 x i64>)
+
+declare {<4 x i24>, <4 x i1>} @llvm.sadd.with.overflow.v4i24(<4 x i24>, <4 x i24>)
+declare {<4 x i1>, <4 x i1>} @llvm.sadd.with.overflow.v4i1(<4 x i1>, <4 x i1>)
+declare {<2 x i128>, <2 x i1>} @llvm.sadd.with.overflow.v2i128(<2 x i128>, <2 x i128>)
+
+define <1 x i32> @saddo_v1i32(<1 x i32> %a0, <1 x i32> %a1, <1 x i32>* %p2) nounwind {
+; SSE-LABEL: saddo_v1i32:
+; SSE:       # %bb.0:
+; SSE-NEXT:    xorl %eax, %eax
+; SSE-NEXT:    addl %esi, %edi
+; SSE-NEXT:    seto %al
+; SSE-NEXT:    negl %eax
+; SSE-NEXT:    movl %edi, (%rdx)
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: saddo_v1i32:
+; AVX:       # %bb.0:
+; AVX-NEXT:    xorl %eax, %eax
+; AVX-NEXT:    addl %esi, %edi
+; AVX-NEXT:    seto %al
+; AVX-NEXT:    negl %eax
+; AVX-NEXT:    movl %edi, (%rdx)
+; AVX-NEXT:    retq
+  %t = call {<1 x i32>, <1 x i1>} @llvm.sadd.with.overflow.v1i32(<1 x i32> %a0, <1 x i32> %a1)
+  %val = extractvalue {<1 x i32>, <1 x i1>} %t, 0
+  %obit = extractvalue {<1 x i32>, <1 x i1>} %t, 1
+  %res = sext <1 x i1> %obit to <1 x i32>
+  store <1 x i32> %val, <1 x i32>* %p2
+  ret <1 x i32> %res
+}
+
+define <2 x i32> @saddo_v2i32(<2 x i32> %a0, <2 x i32> %a1, <2 x i32>* %p2) nounwind {
+; SSE2-LABEL: saddo_v2i32:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    psllq $32, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,3,2,3]
+; SSE2-NEXT:    psrad $31, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,3,2,3]
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; SSE2-NEXT:    psllq $32, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,3,2,3]
+; SSE2-NEXT:    psrad $31, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,3,2,3]
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSE2-NEXT:    paddq %xmm2, %xmm1
+; SSE2-NEXT:    movdqa %xmm1, %xmm0
+; SSE2-NEXT:    psllq $32, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,3,2,3]
+; SSE2-NEXT:    psrad $31, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,3,2,3]
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
+; SSE2-NEXT:    pcmpeqd %xmm1, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[1,0,3,2]
+; SSE2-NEXT:    pand %xmm2, %xmm3
+; SSE2-NEXT:    pcmpeqd %xmm0, %xmm0
+; SSE2-NEXT:    pxor %xmm3, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; SSE2-NEXT:    movq %xmm1, (%rdi)
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: saddo_v2i32:
+; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    psllq $32, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,3,2,3]
+; SSSE3-NEXT:    psrad $31, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,3,2,3]
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; SSSE3-NEXT:    psllq $32, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,3,2,3]
+; SSSE3-NEXT:    psrad $31, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,3,2,3]
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSSE3-NEXT:    paddq %xmm2, %xmm1
+; SSSE3-NEXT:    movdqa %xmm1, %xmm0
+; SSSE3-NEXT:    psllq $32, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,3,2,3]
+; SSSE3-NEXT:    psrad $31, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,3,2,3]
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
+; SSSE3-NEXT:    pcmpeqd %xmm1, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[1,0,3,2]
+; SSSE3-NEXT:    pand %xmm2, %xmm3
+; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm0
+; SSSE3-NEXT:    pxor %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; SSSE3-NEXT:    movq %xmm1, (%rdi)
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: saddo_v2i32:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movdqa %xmm1, %xmm2
+; SSE41-NEXT:    psllq $32, %xmm2
+; SSE41-NEXT:    psrad $31, %xmm2
+; SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
+; SSE41-NEXT:    movdqa %xmm0, %xmm1
+; SSE41-NEXT:    psllq $32, %xmm1
+; SSE41-NEXT:    psrad $31, %xmm1
+; SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm0[0,1],xmm1[2,3],xmm0[4,5],xmm1[6,7]
+; SSE41-NEXT:    paddq %xmm2, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    psllq $32, %xmm0
+; SSE41-NEXT:    psrad $31, %xmm0
+; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3],xmm1[4,5],xmm0[6,7]
+; SSE41-NEXT:    pcmpeqq %xmm1, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm2, %xmm2
+; SSE41-NEXT:    pxor %xmm2, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; SSE41-NEXT:    movq %xmm1, (%rdi)
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: saddo_v2i32:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpsllq $32, %xmm1, %xmm2
+; AVX1-NEXT:    vpsrad $31, %xmm2, %xmm2
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
+; AVX1-NEXT:    vpsllq $32, %xmm0, %xmm2
+; AVX1-NEXT:    vpsrad $31, %xmm2, %xmm2
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3],xmm0[4,5],xmm2[6,7]
+; AVX1-NEXT:    vpaddq %xmm1, %xmm0, %xmm1
+; AVX1-NEXT:    vpsllq $32, %xmm1, %xmm0
+; AVX1-NEXT:    vpsrad $31, %xmm0, %xmm0
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3],xmm1[4,5],xmm0[6,7]
+; AVX1-NEXT:    vpcmpeqq %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
+; AVX1-NEXT:    vpxor %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; AVX1-NEXT:    vmovq %xmm1, (%rdi)
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: saddo_v2i32:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpsllq $32, %xmm1, %xmm2
+; AVX2-NEXT:    vpsrad $31, %xmm2, %xmm2
+; AVX2-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3]
+; AVX2-NEXT:    vpsllq $32, %xmm0, %xmm2
+; AVX2-NEXT:    vpsrad $31, %xmm2, %xmm2
+; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2],xmm2[3]
+; AVX2-NEXT:    vpaddq %xmm1, %xmm0, %xmm1
+; AVX2-NEXT:    vpsllq $32, %xmm1, %xmm0
+; AVX2-NEXT:    vpsrad $31, %xmm0, %xmm0
+; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm1[0],xmm0[1],xmm1[2],xmm0[3]
+; AVX2-NEXT:    vpcmpeqq %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
+; AVX2-NEXT:    vpxor %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; AVX2-NEXT:    vmovq %xmm1, (%rdi)
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: saddo_v2i32:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpsllq $32, %xmm1, %xmm1
+; AVX512-NEXT:    vpsraq $32, %xmm1, %xmm1
+; AVX512-NEXT:    vpsllq $32, %xmm0, %xmm0
+; AVX512-NEXT:    vpsraq $32, %xmm0, %xmm0
+; AVX512-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vpsllq $32, %xmm0, %xmm1
+; AVX512-NEXT:    vpsraq $32, %xmm1, %xmm1
+; AVX512-NEXT:    vpmovqd %xmm0, (%rdi)
+; AVX512-NEXT:    vpcmpeqq %xmm0, %xmm1, %xmm0
+; AVX512-NEXT:    vpternlogq $15, %xmm0, %xmm0, %xmm0
+; AVX512-NEXT:    retq
+  %t = call {<2 x i32>, <2 x i1>} @llvm.sadd.with.overflow.v2i32(<2 x i32> %a0, <2 x i32> %a1)
+  %val = extractvalue {<2 x i32>, <2 x i1>} %t, 0
+  %obit = extractvalue {<2 x i32>, <2 x i1>} %t, 1
+  %res = sext <2 x i1> %obit to <2 x i32>
+  store <2 x i32> %val, <2 x i32>* %p2
+  ret <2 x i32> %res
+}
+
+define <3 x i32> @saddo_v3i32(<3 x i32> %a0, <3 x i32> %a1, <3 x i32>* %p2) nounwind {
+; SSE2-LABEL: saddo_v3i32:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    pxor %xmm2, %xmm2
+; SSE2-NEXT:    pxor %xmm3, %xmm3
+; SSE2-NEXT:    pcmpgtd %xmm1, %xmm3
+; SSE2-NEXT:    pcmpeqd %xmm4, %xmm4
+; SSE2-NEXT:    pxor %xmm4, %xmm3
+; SSE2-NEXT:    pxor %xmm5, %xmm5
+; SSE2-NEXT:    pcmpgtd %xmm0, %xmm5
+; SSE2-NEXT:    pxor %xmm4, %xmm5
+; SSE2-NEXT:    pcmpeqd %xmm5, %xmm3
+; SSE2-NEXT:    paddd %xmm1, %xmm0
+; SSE2-NEXT:    pcmpgtd %xmm0, %xmm2
+; SSE2-NEXT:    pxor %xmm4, %xmm2
+; SSE2-NEXT:    pcmpeqd %xmm5, %xmm2
+; SSE2-NEXT:    pandn %xmm3, %xmm2
+; SSE2-NEXT:    movq %xmm0, (%rdi)
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE2-NEXT:    movd %xmm0, 8(%rdi)
+; SSE2-NEXT:    movdqa %xmm2, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: saddo_v3i32:
+; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    pxor %xmm2, %xmm2
+; SSSE3-NEXT:    pxor %xmm3, %xmm3
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm3
+; SSSE3-NEXT:    pcmpeqd %xmm4, %xmm4
+; SSSE3-NEXT:    pxor %xmm4, %xmm3
+; SSSE3-NEXT:    pxor %xmm5, %xmm5
+; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm5
+; SSSE3-NEXT:    pxor %xmm4, %xmm5
+; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm3
+; SSSE3-NEXT:    paddd %xmm1, %xmm0
+; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm2
+; SSSE3-NEXT:    pxor %xmm4, %xmm2
+; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm2
+; SSSE3-NEXT:    pandn %xmm3, %xmm2
+; SSSE3-NEXT:    movq %xmm0, (%rdi)
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSSE3-NEXT:    movd %xmm0, 8(%rdi)
+; SSSE3-NEXT:    movdqa %xmm2, %xmm0
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: saddo_v3i32:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    pxor %xmm2, %xmm2
+; SSE41-NEXT:    pxor %xmm3, %xmm3
+; SSE41-NEXT:    pcmpgtd %xmm1, %xmm3
+; SSE41-NEXT:    pcmpeqd %xmm4, %xmm4
+; SSE41-NEXT:    pxor %xmm4, %xmm3
+; SSE41-NEXT:    pxor %xmm5, %xmm5
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm5
+; SSE41-NEXT:    pxor %xmm4, %xmm5
+; SSE41-NEXT:    pcmpeqd %xmm5, %xmm3
+; SSE41-NEXT:    paddd %xmm1, %xmm0
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm2
+; SSE41-NEXT:    pxor %xmm4, %xmm2
+; SSE41-NEXT:    pcmpeqd %xmm5, %xmm2
+; SSE41-NEXT:    pandn %xmm3, %xmm2
+; SSE41-NEXT:    pextrd $2, %xmm0, 8(%rdi)
+; SSE41-NEXT:    movq %xmm0, (%rdi)
+; SSE41-NEXT:    movdqa %xmm2, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: saddo_v3i32:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm2, %xmm3
+; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
+; AVX1-NEXT:    vpxor %xmm4, %xmm3, %xmm3
+; AVX1-NEXT:    vpcmpgtd %xmm0, %xmm2, %xmm5
+; AVX1-NEXT:    vpxor %xmm4, %xmm5, %xmm5
+; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm5, %xmm3
+; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm1
+; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm2, %xmm0
+; AVX1-NEXT:    vpxor %xmm4, %xmm0, %xmm0
+; AVX1-NEXT:    vpcmpeqd %xmm0, %xmm5, %xmm0
+; AVX1-NEXT:    vpandn %xmm3, %xmm0, %xmm0
+; AVX1-NEXT:    vpextrd $2, %xmm1, 8(%rdi)
+; AVX1-NEXT:    vmovq %xmm1, (%rdi)
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: saddo_v3i32:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX2-NEXT:    vpcmpgtd %xmm1, %xmm2, %xmm3
+; AVX2-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
+; AVX2-NEXT:    vpxor %xmm4, %xmm3, %xmm3
+; AVX2-NEXT:    vpcmpgtd %xmm0, %xmm2, %xmm5
+; AVX2-NEXT:    vpxor %xmm4, %xmm5, %xmm5
+; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm5, %xmm3
+; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm1
+; AVX2-NEXT:    vpcmpgtd %xmm1, %xmm2, %xmm0
+; AVX2-NEXT:    vpxor %xmm4, %xmm0, %xmm0
+; AVX2-NEXT:    vpcmpeqd %xmm0, %xmm5, %xmm0
+; AVX2-NEXT:    vpandn %xmm3, %xmm0, %xmm0
+; AVX2-NEXT:    vpextrd $2, %xmm1, 8(%rdi)
+; AVX2-NEXT:    vmovq %xmm1, (%rdi)
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: saddo_v3i32:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX512-NEXT:    vpcmpnltd %xmm2, %xmm1, %k0
+; AVX512-NEXT:    vpcmpnltd %xmm2, %xmm0, %k1
+; AVX512-NEXT:    kxorw %k0, %k1, %k0
+; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm1
+; AVX512-NEXT:    vpcmpnltd %xmm2, %xmm1, %k2
+; AVX512-NEXT:    kxorw %k2, %k1, %k1
+; AVX512-NEXT:    kandnw %k1, %k0, %k1
+; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
+; AVX512-NEXT:    vpextrd $2, %xmm1, 8(%rdi)
+; AVX512-NEXT:    vmovq %xmm1, (%rdi)
+; AVX512-NEXT:    retq
+  %t = call {<3 x i32>, <3 x i1>} @llvm.sadd.with.overflow.v3i32(<3 x i32> %a0, <3 x i32> %a1)
+  %val = extractvalue {<3 x i32>, <3 x i1>} %t, 0
+  %obit = extractvalue {<3 x i32>, <3 x i1>} %t, 1
+  %res = sext <3 x i1> %obit to <3 x i32>
+  store <3 x i32> %val, <3 x i32>* %p2
+  ret <3 x i32> %res
+}
+
+define <4 x i32> @saddo_v4i32(<4 x i32> %a0, <4 x i32> %a1, <4 x i32>* %p2) nounwind {
+; SSE-LABEL: saddo_v4i32:
+; SSE:       # %bb.0:
+; SSE-NEXT:    pxor %xmm2, %xmm2
+; SSE-NEXT:    pxor %xmm3, %xmm3
+; SSE-NEXT:    pcmpgtd %xmm1, %xmm3
+; SSE-NEXT:    pcmpeqd %xmm4, %xmm4
+; SSE-NEXT:    pxor %xmm4, %xmm3
+; SSE-NEXT:    pxor %xmm5, %xmm5
+; SSE-NEXT:    pcmpgtd %xmm0, %xmm5
+; SSE-NEXT:    pxor %xmm4, %xmm5
+; SSE-NEXT:    pcmpeqd %xmm5, %xmm3
+; SSE-NEXT:    paddd %xmm1, %xmm0
+; SSE-NEXT:    pcmpgtd %xmm0, %xmm2
+; SSE-NEXT:    pxor %xmm4, %xmm2
+; SSE-NEXT:    pcmpeqd %xmm5, %xmm2
+; SSE-NEXT:    pandn %xmm3, %xmm2
+; SSE-NEXT:    movdqa %xmm0, (%rdi)
+; SSE-NEXT:    movdqa %xmm2, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX1-LABEL: saddo_v4i32:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm2, %xmm3
+; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
+; AVX1-NEXT:    vpxor %xmm4, %xmm3, %xmm3
+; AVX1-NEXT:    vpcmpgtd %xmm0, %xmm2, %xmm5
+; AVX1-NEXT:    vpxor %xmm4, %xmm5, %xmm5
+; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm5, %xmm3
+; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm1
+; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm2, %xmm0
+; AVX1-NEXT:    vpxor %xmm4, %xmm0, %xmm0
+; AVX1-NEXT:    vpcmpeqd %xmm0, %xmm5, %xmm0
+; AVX1-NEXT:    vpandn %xmm3, %xmm0, %xmm0
+; AVX1-NEXT:    vmovdqa %xmm1, (%rdi)
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: saddo_v4i32:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX2-NEXT:    vpcmpgtd %xmm1, %xmm2, %xmm3
+; AVX2-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
+; AVX2-NEXT:    vpxor %xmm4, %xmm3, %xmm3
+; AVX2-NEXT:    vpcmpgtd %xmm0, %xmm2, %xmm5
+; AVX2-NEXT:    vpxor %xmm4, %xmm5, %xmm5
+; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm5, %xmm3
+; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm1
+; AVX2-NEXT:    vpcmpgtd %xmm1, %xmm2, %xmm0
+; AVX2-NEXT:    vpxor %xmm4, %xmm0, %xmm0
+; AVX2-NEXT:    vpcmpeqd %xmm0, %xmm5, %xmm0
+; AVX2-NEXT:    vpandn %xmm3, %xmm0, %xmm0
+; AVX2-NEXT:    vmovdqa %xmm1, (%rdi)
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: saddo_v4i32:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX512-NEXT:    vpcmpnltd %xmm2, %xmm1, %k0
+; AVX512-NEXT:    vpcmpnltd %xmm2, %xmm0, %k1
+; AVX512-NEXT:    kxorw %k0, %k1, %k0
+; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm1
+; AVX512-NEXT:    vpcmpnltd %xmm2, %xmm1, %k2
+; AVX512-NEXT:    kxorw %k2, %k1, %k1
+; AVX512-NEXT:    kandnw %k1, %k0, %k1
+; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
+; AVX512-NEXT:    vmovdqa %xmm1, (%rdi)
+; AVX512-NEXT:    retq
+  %t = call {<4 x i32>, <4 x i1>} @llvm.sadd.with.overflow.v4i32(<4 x i32> %a0, <4 x i32> %a1)
+  %val = extractvalue {<4 x i32>, <4 x i1>} %t, 0
+  %obit = extractvalue {<4 x i32>, <4 x i1>} %t, 1
+  %res = sext <4 x i1> %obit to <4 x i32>
+  store <4 x i32> %val, <4 x i32>* %p2
+  ret <4 x i32> %res
+}
+
+define <6 x i32> @saddo_v6i32(<6 x i32> %a0, <6 x i32> %a1, <6 x i32>* %p2) nounwind {
+; SSE2-LABEL: saddo_v6i32:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movq %rdi, %rax
+; SSE2-NEXT:    movd %r8d, %xmm0
+; SSE2-NEXT:    movd %ecx, %xmm1
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSE2-NEXT:    movd %edx, %xmm0
+; SSE2-NEXT:    movd %esi, %xmm4
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1]
+; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm1[0]
+; SSE2-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSE2-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; SSE2-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
+; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; SSE2-NEXT:    movd %r9d, %xmm3
+; SSE2-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
+; SSE2-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; SSE2-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
+; SSE2-NEXT:    movq {{[0-9]+}}(%rsp), %rcx
+; SSE2-NEXT:    pxor %xmm2, %xmm2
+; SSE2-NEXT:    pxor %xmm6, %xmm6
+; SSE2-NEXT:    pcmpgtd %xmm0, %xmm6
+; SSE2-NEXT:    pcmpeqd %xmm5, %xmm5
+; SSE2-NEXT:    pxor %xmm5, %xmm6
+; SSE2-NEXT:    pxor %xmm7, %xmm7
+; SSE2-NEXT:    pcmpgtd %xmm4, %xmm7
+; SSE2-NEXT:    pxor %xmm5, %xmm7
+; SSE2-NEXT:    pcmpeqd %xmm7, %xmm6
+; SSE2-NEXT:    paddd %xmm4, %xmm0
+; SSE2-NEXT:    pxor %xmm4, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm0, %xmm4
+; SSE2-NEXT:    pxor %xmm5, %xmm4
+; SSE2-NEXT:    pcmpeqd %xmm7, %xmm4
+; SSE2-NEXT:    pandn %xmm6, %xmm4
+; SSE2-NEXT:    pxor %xmm6, %xmm6
+; SSE2-NEXT:    pcmpgtd %xmm1, %xmm6
+; SSE2-NEXT:    pxor %xmm5, %xmm6
+; SSE2-NEXT:    pxor %xmm7, %xmm7
+; SSE2-NEXT:    pcmpgtd %xmm3, %xmm7
+; SSE2-NEXT:    pxor %xmm5, %xmm7
+; SSE2-NEXT:    pcmpeqd %xmm7, %xmm6
+; SSE2-NEXT:    paddd %xmm3, %xmm1
+; SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
+; SSE2-NEXT:    pxor %xmm5, %xmm2
+; SSE2-NEXT:    pcmpeqd %xmm7, %xmm2
+; SSE2-NEXT:    pandn %xmm6, %xmm2
+; SSE2-NEXT:    movq %xmm1, 16(%rcx)
+; SSE2-NEXT:    movdqa %xmm0, (%rcx)
+; SSE2-NEXT:    movq %xmm2, 16(%rdi)
+; SSE2-NEXT:    movdqa %xmm4, (%rdi)
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: saddo_v6i32:
+; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    movq %rdi, %rax
+; SSSE3-NEXT:    movd %r8d, %xmm0
+; SSSE3-NEXT:    movd %ecx, %xmm1
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSSE3-NEXT:    movd %edx, %xmm0
+; SSSE3-NEXT:    movd %esi, %xmm4
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1]
+; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm1[0]
+; SSSE3-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSSE3-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSSE3-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; SSSE3-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
+; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; SSSE3-NEXT:    movd %r9d, %xmm3
+; SSSE3-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
+; SSSE3-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; SSSE3-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
+; SSSE3-NEXT:    movq {{[0-9]+}}(%rsp), %rcx
+; SSSE3-NEXT:    pxor %xmm2, %xmm2
+; SSSE3-NEXT:    pxor %xmm6, %xmm6
+; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm6
+; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm5
+; SSSE3-NEXT:    pxor %xmm5, %xmm6
+; SSSE3-NEXT:    pxor %xmm7, %xmm7
+; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm7
+; SSSE3-NEXT:    pxor %xmm5, %xmm7
+; SSSE3-NEXT:    pcmpeqd %xmm7, %xmm6
+; SSSE3-NEXT:    paddd %xmm4, %xmm0
+; SSSE3-NEXT:    pxor %xmm4, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm4
+; SSSE3-NEXT:    pxor %xmm5, %xmm4
+; SSSE3-NEXT:    pcmpeqd %xmm7, %xmm4
+; SSSE3-NEXT:    pandn %xmm6, %xmm4
+; SSSE3-NEXT:    pxor %xmm6, %xmm6
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm6
+; SSSE3-NEXT:    pxor %xmm5, %xmm6
+; SSSE3-NEXT:    pxor %xmm7, %xmm7
+; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm7
+; SSSE3-NEXT:    pxor %xmm5, %xmm7
+; SSSE3-NEXT:    pcmpeqd %xmm7, %xmm6
+; SSSE3-NEXT:    paddd %xmm3, %xmm1
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm2
+; SSSE3-NEXT:    pxor %xmm5, %xmm2
+; SSSE3-NEXT:    pcmpeqd %xmm7, %xmm2
+; SSSE3-NEXT:    pandn %xmm6, %xmm2
+; SSSE3-NEXT:    movq %xmm1, 16(%rcx)
+; SSSE3-NEXT:    movdqa %xmm0, (%rcx)
+; SSSE3-NEXT:    movq %xmm2, 16(%rdi)
+; SSSE3-NEXT:    movdqa %xmm4, (%rdi)
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: saddo_v6i32:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movq %rdi, %rax
+; SSE41-NEXT:    movd %esi, %xmm4
+; SSE41-NEXT:    pinsrd $1, %edx, %xmm4
+; SSE41-NEXT:    pinsrd $2, %ecx, %xmm4
+; SSE41-NEXT:    pinsrd $3, %r8d, %xmm4
+; SSE41-NEXT:    movd %r9d, %xmm2
+; SSE41-NEXT:    pinsrd $1, {{[0-9]+}}(%rsp), %xmm2
+; SSE41-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE41-NEXT:    pinsrd $1, {{[0-9]+}}(%rsp), %xmm0
+; SSE41-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SSE41-NEXT:    pinsrd $1, {{[0-9]+}}(%rsp), %xmm1
+; SSE41-NEXT:    pinsrd $2, {{[0-9]+}}(%rsp), %xmm1
+; SSE41-NEXT:    pinsrd $3, {{[0-9]+}}(%rsp), %xmm1
+; SSE41-NEXT:    movq {{[0-9]+}}(%rsp), %rcx
+; SSE41-NEXT:    pxor %xmm3, %xmm3
+; SSE41-NEXT:    pxor %xmm6, %xmm6
+; SSE41-NEXT:    pcmpgtd %xmm1, %xmm6
+; SSE41-NEXT:    pcmpeqd %xmm5, %xmm5
+; SSE41-NEXT:    pxor %xmm5, %xmm6
+; SSE41-NEXT:    pxor %xmm7, %xmm7
+; SSE41-NEXT:    pcmpgtd %xmm4, %xmm7
+; SSE41-NEXT:    pxor %xmm5, %xmm7
+; SSE41-NEXT:    pcmpeqd %xmm7, %xmm6
+; SSE41-NEXT:    paddd %xmm4, %xmm1
+; SSE41-NEXT:    pxor %xmm4, %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm1, %xmm4
+; SSE41-NEXT:    pxor %xmm5, %xmm4
+; SSE41-NEXT:    pcmpeqd %xmm7, %xmm4
+; SSE41-NEXT:    pandn %xmm6, %xmm4
+; SSE41-NEXT:    pxor %xmm6, %xmm6
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm6
+; SSE41-NEXT:    pxor %xmm5, %xmm6
+; SSE41-NEXT:    pxor %xmm7, %xmm7
+; SSE41-NEXT:    pcmpgtd %xmm2, %xmm7
+; SSE41-NEXT:    pxor %xmm5, %xmm7
+; SSE41-NEXT:    pcmpeqd %xmm7, %xmm6
+; SSE41-NEXT:    paddd %xmm2, %xmm0
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm3
+; SSE41-NEXT:    pxor %xmm5, %xmm3
+; SSE41-NEXT:    pcmpeqd %xmm7, %xmm3
+; SSE41-NEXT:    pandn %xmm6, %xmm3
+; SSE41-NEXT:    movq %xmm0, 16(%rcx)
+; SSE41-NEXT:    movdqa %xmm1, (%rcx)
+; SSE41-NEXT:    movq %xmm3, 16(%rdi)
+; SSE41-NEXT:    movdqa %xmm4, (%rdi)
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: saddo_v6i32:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; AVX1-NEXT:    vpcmpgtd %xmm2, %xmm3, %xmm4
+; AVX1-NEXT:    vpcmpeqd %xmm5, %xmm5, %xmm5
+; AVX1-NEXT:    vpxor %xmm5, %xmm4, %xmm4
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm6
+; AVX1-NEXT:    vpcmpgtd %xmm6, %xmm3, %xmm7
+; AVX1-NEXT:    vpxor %xmm5, %xmm7, %xmm7
+; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm7, %xmm8
+; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm3, %xmm4
+; AVX1-NEXT:    vpxor %xmm5, %xmm4, %xmm9
+; AVX1-NEXT:    vpcmpgtd %xmm0, %xmm3, %xmm4
+; AVX1-NEXT:    vpxor %xmm5, %xmm4, %xmm4
+; AVX1-NEXT:    vpcmpeqd %xmm9, %xmm4, %xmm9
+; AVX1-NEXT:    vinsertf128 $1, %xmm8, %ymm9, %ymm8
+; AVX1-NEXT:    vpaddd %xmm2, %xmm6, %xmm2
+; AVX1-NEXT:    vpcmpgtd %xmm2, %xmm3, %xmm6
+; AVX1-NEXT:    vpxor %xmm5, %xmm6, %xmm6
+; AVX1-NEXT:    vpcmpeqd %xmm6, %xmm7, %xmm6
+; AVX1-NEXT:    vpxor %xmm5, %xmm6, %xmm6
+; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm1
+; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm3, %xmm0
+; AVX1-NEXT:    vpxor %xmm5, %xmm0, %xmm0
+; AVX1-NEXT:    vpcmpeqd %xmm0, %xmm4, %xmm0
+; AVX1-NEXT:    vpxor %xmm5, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm6, %ymm0, %ymm0
+; AVX1-NEXT:    vandps %ymm0, %ymm8, %ymm0
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
+; AVX1-NEXT:    vpackssdw %xmm3, %xmm0, %xmm0
+; AVX1-NEXT:    vpmovsxwd %xmm0, %xmm3
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpmovsxwd %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm3, %ymm0
+; AVX1-NEXT:    vmovq %xmm2, 16(%rdi)
+; AVX1-NEXT:    vmovdqa %xmm1, (%rdi)
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: saddo_v6i32:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX2-NEXT:    vpcmpgtd %ymm1, %ymm2, %ymm3
+; AVX2-NEXT:    vpcmpeqd %ymm4, %ymm4, %ymm4
+; AVX2-NEXT:    vpxor %ymm4, %ymm3, %ymm3
+; AVX2-NEXT:    vpcmpgtd %ymm0, %ymm2, %ymm5
+; AVX2-NEXT:    vpxor %ymm4, %ymm5, %ymm5
+; AVX2-NEXT:    vpcmpeqd %ymm3, %ymm5, %ymm3
+; AVX2-NEXT:    vpaddd %ymm1, %ymm0, %ymm1
+; AVX2-NEXT:    vpcmpgtd %ymm1, %ymm2, %ymm0
+; AVX2-NEXT:    vpxor %ymm4, %ymm0, %ymm0
+; AVX2-NEXT:    vpcmpeqd %ymm0, %ymm5, %ymm0
+; AVX2-NEXT:    vpandn %ymm3, %ymm0, %ymm0
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm2
+; AVX2-NEXT:    vpackssdw %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    vpmovsxwd %xmm0, %ymm0
+; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm2
+; AVX2-NEXT:    vmovq %xmm2, 16(%rdi)
+; AVX2-NEXT:    vmovdqa %xmm1, (%rdi)
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: saddo_v6i32:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX512-NEXT:    vpcmpnltd %ymm2, %ymm1, %k0
+; AVX512-NEXT:    vpcmpnltd %ymm2, %ymm0, %k1
+; AVX512-NEXT:    kxorw %k0, %k1, %k0
+; AVX512-NEXT:    vpaddd %ymm1, %ymm0, %ymm1
+; AVX512-NEXT:    vpcmpnltd %ymm2, %ymm1, %k2
+; AVX512-NEXT:    kxorw %k2, %k1, %k1
+; AVX512-NEXT:    kandnw %k1, %k0, %k1
+; AVX512-NEXT:    vpcmpeqd %ymm0, %ymm0, %ymm0
+; AVX512-NEXT:    vmovdqa32 %ymm0, %ymm0 {%k1} {z}
+; AVX512-NEXT:    vextracti128 $1, %ymm1, %xmm2
+; AVX512-NEXT:    vmovq %xmm2, 16(%rdi)
+; AVX512-NEXT:    vmovdqa %xmm1, (%rdi)
+; AVX512-NEXT:    retq
+  %t = call {<6 x i32>, <6 x i1>} @llvm.sadd.with.overflow.v6i32(<6 x i32> %a0, <6 x i32> %a1)
+  %val = extractvalue {<6 x i32>, <6 x i1>} %t, 0
+  %obit = extractvalue {<6 x i32>, <6 x i1>} %t, 1
+  %res = sext <6 x i1> %obit to <6 x i32>
+  store <6 x i32> %val, <6 x i32>* %p2
+  ret <6 x i32> %res
+}
+
+define <8 x i32> @saddo_v8i32(<8 x i32> %a0, <8 x i32> %a1, <8 x i32>* %p2) nounwind {
+; SSE-LABEL: saddo_v8i32:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movdqa %xmm1, %xmm4
+; SSE-NEXT:    pxor %xmm1, %xmm1
+; SSE-NEXT:    pxor %xmm6, %xmm6
+; SSE-NEXT:    pcmpgtd %xmm2, %xmm6
+; SSE-NEXT:    pcmpeqd %xmm5, %xmm5
+; SSE-NEXT:    pxor %xmm5, %xmm6
+; SSE-NEXT:    pxor %xmm7, %xmm7
+; SSE-NEXT:    pcmpgtd %xmm0, %xmm7
+; SSE-NEXT:    pxor %xmm5, %xmm7
+; SSE-NEXT:    pcmpeqd %xmm7, %xmm6
+; SSE-NEXT:    paddd %xmm2, %xmm0
+; SSE-NEXT:    pxor %xmm2, %xmm2
+; SSE-NEXT:    pcmpgtd %xmm0, %xmm2
+; SSE-NEXT:    pxor %xmm5, %xmm2
+; SSE-NEXT:    pcmpeqd %xmm7, %xmm2
+; SSE-NEXT:    pandn %xmm6, %xmm2
+; SSE-NEXT:    pxor %xmm6, %xmm6
+; SSE-NEXT:    pcmpgtd %xmm3, %xmm6
+; SSE-NEXT:    pxor %xmm5, %xmm6
+; SSE-NEXT:    pxor %xmm7, %xmm7
+; SSE-NEXT:    pcmpgtd %xmm4, %xmm7
+; SSE-NEXT:    pxor %xmm5, %xmm7
+; SSE-NEXT:    pcmpeqd %xmm7, %xmm6
+; SSE-NEXT:    paddd %xmm3, %xmm4
+; SSE-NEXT:    pcmpgtd %xmm4, %xmm1
+; SSE-NEXT:    pxor %xmm5, %xmm1
+; SSE-NEXT:    pcmpeqd %xmm7, %xmm1
+; SSE-NEXT:    pandn %xmm6, %xmm1
+; SSE-NEXT:    movdqa %xmm4, 16(%rdi)
+; SSE-NEXT:    movdqa %xmm0, (%rdi)
+; SSE-NEXT:    movdqa %xmm2, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX1-LABEL: saddo_v8i32:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; AVX1-NEXT:    vpcmpgtd %xmm2, %xmm3, %xmm4
+; AVX1-NEXT:    vpcmpeqd %xmm5, %xmm5, %xmm5
+; AVX1-NEXT:    vpxor %xmm5, %xmm4, %xmm4
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm6
+; AVX1-NEXT:    vpcmpgtd %xmm6, %xmm3, %xmm7
+; AVX1-NEXT:    vpxor %xmm5, %xmm7, %xmm7
+; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm7, %xmm8
+; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm3, %xmm4
+; AVX1-NEXT:    vpxor %xmm5, %xmm4, %xmm9
+; AVX1-NEXT:    vpcmpgtd %xmm0, %xmm3, %xmm4
+; AVX1-NEXT:    vpxor %xmm5, %xmm4, %xmm4
+; AVX1-NEXT:    vpcmpeqd %xmm9, %xmm4, %xmm9
+; AVX1-NEXT:    vinsertf128 $1, %xmm8, %ymm9, %ymm8
+; AVX1-NEXT:    vpaddd %xmm2, %xmm6, %xmm2
+; AVX1-NEXT:    vpcmpgtd %xmm2, %xmm3, %xmm6
+; AVX1-NEXT:    vpxor %xmm5, %xmm6, %xmm6
+; AVX1-NEXT:    vpcmpeqd %xmm6, %xmm7, %xmm6
+; AVX1-NEXT:    vpxor %xmm5, %xmm6, %xmm6
+; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpcmpgtd %xmm0, %xmm3, %xmm1
+; AVX1-NEXT:    vpxor %xmm5, %xmm1, %xmm1
+; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm4, %xmm1
+; AVX1-NEXT:    vpxor %xmm5, %xmm1, %xmm1
+; AVX1-NEXT:    vinsertf128 $1, %xmm6, %ymm1, %ymm1
+; AVX1-NEXT:    vandps %ymm1, %ymm8, %ymm1
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm3
+; AVX1-NEXT:    vpackssdw %xmm3, %xmm1, %xmm1
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm2
+; AVX1-NEXT:    vpmovsxwd %xmm1, %xmm0
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; AVX1-NEXT:    vpmovsxwd %xmm1, %xmm1
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    vmovaps %ymm2, (%rdi)
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: saddo_v8i32:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX2-NEXT:    vpcmpgtd %ymm1, %ymm2, %ymm3
+; AVX2-NEXT:    vpcmpeqd %ymm4, %ymm4, %ymm4
+; AVX2-NEXT:    vpxor %ymm4, %ymm3, %ymm3
+; AVX2-NEXT:    vpcmpgtd %ymm0, %ymm2, %ymm5
+; AVX2-NEXT:    vpxor %ymm4, %ymm5, %ymm5
+; AVX2-NEXT:    vpcmpeqd %ymm3, %ymm5, %ymm3
+; AVX2-NEXT:    vpaddd %ymm1, %ymm0, %ymm1
+; AVX2-NEXT:    vpcmpgtd %ymm1, %ymm2, %ymm0
+; AVX2-NEXT:    vpxor %ymm4, %ymm0, %ymm0
+; AVX2-NEXT:    vpcmpeqd %ymm0, %ymm5, %ymm0
+; AVX2-NEXT:    vpandn %ymm3, %ymm0, %ymm0
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm2
+; AVX2-NEXT:    vpackssdw %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    vpmovsxwd %xmm0, %ymm0
+; AVX2-NEXT:    vmovdqa %ymm1, (%rdi)
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: saddo_v8i32:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX512-NEXT:    vpcmpnltd %ymm2, %ymm1, %k0
+; AVX512-NEXT:    vpcmpnltd %ymm2, %ymm0, %k1
+; AVX512-NEXT:    kxorw %k0, %k1, %k0
+; AVX512-NEXT:    vpaddd %ymm1, %ymm0, %ymm1
+; AVX512-NEXT:    vpcmpnltd %ymm2, %ymm1, %k2
+; AVX512-NEXT:    kxorw %k2, %k1, %k1
+; AVX512-NEXT:    kandnw %k1, %k0, %k1
+; AVX512-NEXT:    vpcmpeqd %ymm0, %ymm0, %ymm0
+; AVX512-NEXT:    vmovdqa32 %ymm0, %ymm0 {%k1} {z}
+; AVX512-NEXT:    vmovdqa %ymm1, (%rdi)
+; AVX512-NEXT:    retq
+  %t = call {<8 x i32>, <8 x i1>} @llvm.sadd.with.overflow.v8i32(<8 x i32> %a0, <8 x i32> %a1)
+  %val = extractvalue {<8 x i32>, <8 x i1>} %t, 0
+  %obit = extractvalue {<8 x i32>, <8 x i1>} %t, 1
+  %res = sext <8 x i1> %obit to <8 x i32>
+  store <8 x i32> %val, <8 x i32>* %p2
+  ret <8 x i32> %res
+}
+
+define <16 x i32> @saddo_v16i32(<16 x i32> %a0, <16 x i32> %a1, <16 x i32>* %p2) nounwind {
+; SSE-LABEL: saddo_v16i32:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movdqa %xmm3, %xmm8
+; SSE-NEXT:    pxor %xmm3, %xmm3
+; SSE-NEXT:    pxor %xmm11, %xmm11
+; SSE-NEXT:    pcmpgtd %xmm4, %xmm11
+; SSE-NEXT:    pcmpeqd %xmm10, %xmm10
+; SSE-NEXT:    pxor %xmm10, %xmm11
+; SSE-NEXT:    pxor %xmm12, %xmm12
+; SSE-NEXT:    pcmpgtd %xmm0, %xmm12
+; SSE-NEXT:    pxor %xmm10, %xmm12
+; SSE-NEXT:    pcmpeqd %xmm12, %xmm11
+; SSE-NEXT:    paddd %xmm4, %xmm0
+; SSE-NEXT:    pxor %xmm9, %xmm9
+; SSE-NEXT:    pcmpgtd %xmm0, %xmm9
+; SSE-NEXT:    pxor %xmm10, %xmm9
+; SSE-NEXT:    pcmpeqd %xmm12, %xmm9
+; SSE-NEXT:    pandn %xmm11, %xmm9
+; SSE-NEXT:    pxor %xmm12, %xmm12
+; SSE-NEXT:    pcmpgtd %xmm5, %xmm12
+; SSE-NEXT:    pxor %xmm10, %xmm12
+; SSE-NEXT:    pxor %xmm4, %xmm4
+; SSE-NEXT:    pcmpgtd %xmm1, %xmm4
+; SSE-NEXT:    pxor %xmm10, %xmm4
+; SSE-NEXT:    pcmpeqd %xmm4, %xmm12
+; SSE-NEXT:    paddd %xmm5, %xmm1
+; SSE-NEXT:    pxor %xmm11, %xmm11
+; SSE-NEXT:    pcmpgtd %xmm1, %xmm11
+; SSE-NEXT:    pxor %xmm10, %xmm11
+; SSE-NEXT:    pcmpeqd %xmm4, %xmm11
+; SSE-NEXT:    pandn %xmm12, %xmm11
+; SSE-NEXT:    pxor %xmm4, %xmm4
+; SSE-NEXT:    pcmpgtd %xmm6, %xmm4
+; SSE-NEXT:    pxor %xmm10, %xmm4
+; SSE-NEXT:    pxor %xmm5, %xmm5
+; SSE-NEXT:    pcmpgtd %xmm2, %xmm5
+; SSE-NEXT:    pxor %xmm10, %xmm5
+; SSE-NEXT:    pcmpeqd %xmm5, %xmm4
+; SSE-NEXT:    paddd %xmm6, %xmm2
+; SSE-NEXT:    pxor %xmm6, %xmm6
+; SSE-NEXT:    pcmpgtd %xmm2, %xmm6
+; SSE-NEXT:    pxor %xmm10, %xmm6
+; SSE-NEXT:    pcmpeqd %xmm5, %xmm6
+; SSE-NEXT:    pandn %xmm4, %xmm6
+; SSE-NEXT:    pxor %xmm4, %xmm4
+; SSE-NEXT:    pcmpgtd %xmm7, %xmm4
+; SSE-NEXT:    pxor %xmm10, %xmm4
+; SSE-NEXT:    pxor %xmm5, %xmm5
+; SSE-NEXT:    pcmpgtd %xmm8, %xmm5
+; SSE-NEXT:    pxor %xmm10, %xmm5
+; SSE-NEXT:    pcmpeqd %xmm5, %xmm4
+; SSE-NEXT:    paddd %xmm7, %xmm8
+; SSE-NEXT:    pcmpgtd %xmm8, %xmm3
+; SSE-NEXT:    pxor %xmm10, %xmm3
+; SSE-NEXT:    pcmpeqd %xmm5, %xmm3
+; SSE-NEXT:    pandn %xmm4, %xmm3
+; SSE-NEXT:    movdqa %xmm8, 48(%rdi)
+; SSE-NEXT:    movdqa %xmm2, 32(%rdi)
+; SSE-NEXT:    movdqa %xmm1, 16(%rdi)
+; SSE-NEXT:    movdqa %xmm0, (%rdi)
+; SSE-NEXT:    movdqa %xmm9, %xmm0
+; SSE-NEXT:    movdqa %xmm11, %xmm1
+; SSE-NEXT:    movdqa %xmm6, %xmm2
+; SSE-NEXT:    retq
+;
+; AVX1-LABEL: saddo_v16i32:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm9
+; AVX1-NEXT:    vpxor %xmm5, %xmm5, %xmm5
+; AVX1-NEXT:    vpcmpgtd %xmm9, %xmm5, %xmm7
+; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
+; AVX1-NEXT:    vpxor %xmm4, %xmm7, %xmm8
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm7
+; AVX1-NEXT:    vpcmpgtd %xmm7, %xmm5, %xmm6
+; AVX1-NEXT:    vpxor %xmm4, %xmm6, %xmm10
+; AVX1-NEXT:    vpcmpeqd %xmm8, %xmm10, %xmm8
+; AVX1-NEXT:    vpcmpgtd %xmm3, %xmm5, %xmm6
+; AVX1-NEXT:    vpxor %xmm4, %xmm6, %xmm11
+; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm5, %xmm6
+; AVX1-NEXT:    vpxor %xmm4, %xmm6, %xmm6
+; AVX1-NEXT:    vpcmpeqd %xmm11, %xmm6, %xmm11
+; AVX1-NEXT:    vinsertf128 $1, %xmm8, %ymm11, %ymm8
+; AVX1-NEXT:    vpaddd %xmm9, %xmm7, %xmm9
+; AVX1-NEXT:    vpcmpgtd %xmm9, %xmm5, %xmm7
+; AVX1-NEXT:    vpxor %xmm4, %xmm7, %xmm7
+; AVX1-NEXT:    vpcmpeqd %xmm7, %xmm10, %xmm7
+; AVX1-NEXT:    vpxor %xmm4, %xmm7, %xmm7
+; AVX1-NEXT:    vpaddd %xmm3, %xmm1, %xmm10
+; AVX1-NEXT:    vpcmpgtd %xmm10, %xmm5, %xmm3
+; AVX1-NEXT:    vpxor %xmm4, %xmm3, %xmm3
+; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm6, %xmm3
+; AVX1-NEXT:    vpxor %xmm4, %xmm3, %xmm3
+; AVX1-NEXT:    vinsertf128 $1, %xmm7, %ymm3, %ymm3
+; AVX1-NEXT:    vandps %ymm3, %ymm8, %ymm3
+; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm6
+; AVX1-NEXT:    vpackssdw %xmm6, %xmm3, %xmm8
+; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm6
+; AVX1-NEXT:    vpcmpgtd %xmm6, %xmm5, %xmm7
+; AVX1-NEXT:    vpxor %xmm4, %xmm7, %xmm7
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm5, %xmm3
+; AVX1-NEXT:    vpxor %xmm4, %xmm3, %xmm3
+; AVX1-NEXT:    vpcmpeqd %xmm7, %xmm3, %xmm11
+; AVX1-NEXT:    vpcmpgtd %xmm2, %xmm5, %xmm7
+; AVX1-NEXT:    vpxor %xmm4, %xmm7, %xmm12
+; AVX1-NEXT:    vpcmpgtd %xmm0, %xmm5, %xmm7
+; AVX1-NEXT:    vpxor %xmm4, %xmm7, %xmm7
+; AVX1-NEXT:    vpcmpeqd %xmm12, %xmm7, %xmm12
+; AVX1-NEXT:    vinsertf128 $1, %xmm11, %ymm12, %ymm11
+; AVX1-NEXT:    vpaddd %xmm6, %xmm1, %xmm1
+; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm5, %xmm6
+; AVX1-NEXT:    vpxor %xmm4, %xmm6, %xmm6
+; AVX1-NEXT:    vpcmpeqd %xmm6, %xmm3, %xmm3
+; AVX1-NEXT:    vpxor %xmm4, %xmm3, %xmm3
+; AVX1-NEXT:    vpaddd %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpcmpgtd %xmm0, %xmm5, %xmm2
+; AVX1-NEXT:    vpxor %xmm4, %xmm2, %xmm2
+; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm7, %xmm2
+; AVX1-NEXT:    vpxor %xmm4, %xmm2, %xmm2
+; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm2, %ymm2
+; AVX1-NEXT:    vandps %ymm2, %ymm11, %ymm2
+; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm3
+; AVX1-NEXT:    vpackssdw %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm3
+; AVX1-NEXT:    vinsertf128 $1, %xmm9, %ymm10, %ymm4
+; AVX1-NEXT:    vpmovsxwd %xmm2, %xmm0
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm2[2,3,0,1]
+; AVX1-NEXT:    vpmovsxwd %xmm1, %xmm1
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    vpmovsxwd %xmm8, %xmm1
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm8[2,3,0,1]
+; AVX1-NEXT:    vpmovsxwd %xmm2, %xmm2
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
+; AVX1-NEXT:    vmovaps %ymm4, 32(%rdi)
+; AVX1-NEXT:    vmovaps %ymm3, (%rdi)
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: saddo_v16i32:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpxor %xmm4, %xmm4, %xmm4
+; AVX2-NEXT:    vpcmpgtd %ymm3, %ymm4, %ymm5
+; AVX2-NEXT:    vpcmpeqd %ymm6, %ymm6, %ymm6
+; AVX2-NEXT:    vpxor %ymm6, %ymm5, %ymm5
+; AVX2-NEXT:    vpcmpgtd %ymm1, %ymm4, %ymm7
+; AVX2-NEXT:    vpxor %ymm6, %ymm7, %ymm7
+; AVX2-NEXT:    vpcmpeqd %ymm5, %ymm7, %ymm5
+; AVX2-NEXT:    vpaddd %ymm3, %ymm1, %ymm3
+; AVX2-NEXT:    vpcmpgtd %ymm3, %ymm4, %ymm1
+; AVX2-NEXT:    vpxor %ymm6, %ymm1, %ymm1
+; AVX2-NEXT:    vpcmpeqd %ymm1, %ymm7, %ymm1
+; AVX2-NEXT:    vpandn %ymm5, %ymm1, %ymm1
+; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm5
+; AVX2-NEXT:    vpackssdw %xmm5, %xmm1, %xmm1
+; AVX2-NEXT:    vpcmpgtd %ymm2, %ymm4, %ymm5
+; AVX2-NEXT:    vpxor %ymm6, %ymm5, %ymm5
+; AVX2-NEXT:    vpcmpgtd %ymm0, %ymm4, %ymm7
+; AVX2-NEXT:    vpxor %ymm6, %ymm7, %ymm7
+; AVX2-NEXT:    vpcmpeqd %ymm5, %ymm7, %ymm5
+; AVX2-NEXT:    vpaddd %ymm2, %ymm0, %ymm2
+; AVX2-NEXT:    vpcmpgtd %ymm2, %ymm4, %ymm0
+; AVX2-NEXT:    vpxor %ymm6, %ymm0, %ymm0
+; AVX2-NEXT:    vpcmpeqd %ymm0, %ymm7, %ymm0
+; AVX2-NEXT:    vpandn %ymm5, %ymm0, %ymm0
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm4
+; AVX2-NEXT:    vpackssdw %xmm4, %xmm0, %xmm0
+; AVX2-NEXT:    vpmovsxwd %xmm0, %ymm0
+; AVX2-NEXT:    vpmovsxwd %xmm1, %ymm1
+; AVX2-NEXT:    vmovdqa %ymm3, 32(%rdi)
+; AVX2-NEXT:    vmovdqa %ymm2, (%rdi)
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: saddo_v16i32:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX512-NEXT:    vpcmpnltd %zmm2, %zmm1, %k0
+; AVX512-NEXT:    vpcmpnltd %zmm2, %zmm0, %k1
+; AVX512-NEXT:    kxorw %k0, %k1, %k0
+; AVX512-NEXT:    vpaddd %zmm1, %zmm0, %zmm1
+; AVX512-NEXT:    vpcmpnltd %zmm2, %zmm1, %k2
+; AVX512-NEXT:    kxorw %k2, %k1, %k1
+; AVX512-NEXT:    kandnw %k1, %k0, %k1
+; AVX512-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; AVX512-NEXT:    vmovdqa64 %zmm1, (%rdi)
+; AVX512-NEXT:    retq
+  %t = call {<16 x i32>, <16 x i1>} @llvm.sadd.with.overflow.v16i32(<16 x i32> %a0, <16 x i32> %a1)
+  %val = extractvalue {<16 x i32>, <16 x i1>} %t, 0
+  %obit = extractvalue {<16 x i32>, <16 x i1>} %t, 1
+  %res = sext <16 x i1> %obit to <16 x i32>
+  store <16 x i32> %val, <16 x i32>* %p2
+  ret <16 x i32> %res
+}
+
+define <16 x i32> @saddo_v16i8(<16 x i8> %a0, <16 x i8> %a1, <16 x i8>* %p2) nounwind {
+; SSE2-LABEL: saddo_v16i8:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    pxor %xmm3, %xmm3
+; SSE2-NEXT:    pxor %xmm2, %xmm2
+; SSE2-NEXT:    pcmpgtb %xmm1, %xmm2
+; SSE2-NEXT:    pcmpeqd %xmm4, %xmm4
+; SSE2-NEXT:    pxor %xmm4, %xmm2
+; SSE2-NEXT:    pxor %xmm5, %xmm5
+; SSE2-NEXT:    pcmpgtb %xmm0, %xmm5
+; SSE2-NEXT:    pxor %xmm4, %xmm5
+; SSE2-NEXT:    pcmpeqb %xmm5, %xmm2
+; SSE2-NEXT:    paddb %xmm1, %xmm0
+; SSE2-NEXT:    pcmpgtb %xmm0, %xmm3
+; SSE2-NEXT:    pxor %xmm4, %xmm3
+; SSE2-NEXT:    pcmpeqb %xmm5, %xmm3
+; SSE2-NEXT:    pandn %xmm2, %xmm3
+; SSE2-NEXT:    movdqa %xmm3, %xmm1
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSE2-NEXT:    movdqa %xmm1, %xmm4
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3]
+; SSE2-NEXT:    pslld $31, %xmm4
+; SSE2-NEXT:    psrad $31, %xmm4
+; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSE2-NEXT:    pslld $31, %xmm1
+; SSE2-NEXT:    psrad $31, %xmm1
+; SSE2-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm0[8],xmm3[9],xmm0[9],xmm3[10],xmm0[10],xmm3[11],xmm0[11],xmm3[12],xmm0[12],xmm3[13],xmm0[13],xmm3[14],xmm0[14],xmm3[15],xmm0[15]
+; SSE2-NEXT:    movdqa %xmm3, %xmm2
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
+; SSE2-NEXT:    pslld $31, %xmm2
+; SSE2-NEXT:    psrad $31, %xmm2
+; SSE2-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
+; SSE2-NEXT:    pslld $31, %xmm3
+; SSE2-NEXT:    psrad $31, %xmm3
+; SSE2-NEXT:    movdqa %xmm0, (%rdi)
+; SSE2-NEXT:    movdqa %xmm4, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: saddo_v16i8:
+; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    pxor %xmm3, %xmm3
+; SSSE3-NEXT:    pxor %xmm2, %xmm2
+; SSSE3-NEXT:    pcmpgtb %xmm1, %xmm2
+; SSSE3-NEXT:    pcmpeqd %xmm4, %xmm4
+; SSSE3-NEXT:    pxor %xmm4, %xmm2
+; SSSE3-NEXT:    pxor %xmm5, %xmm5
+; SSSE3-NEXT:    pcmpgtb %xmm0, %xmm5
+; SSSE3-NEXT:    pxor %xmm4, %xmm5
+; SSSE3-NEXT:    pcmpeqb %xmm5, %xmm2
+; SSSE3-NEXT:    paddb %xmm1, %xmm0
+; SSSE3-NEXT:    pcmpgtb %xmm0, %xmm3
+; SSSE3-NEXT:    pxor %xmm4, %xmm3
+; SSSE3-NEXT:    pcmpeqb %xmm5, %xmm3
+; SSSE3-NEXT:    pandn %xmm2, %xmm3
+; SSSE3-NEXT:    movdqa %xmm3, %xmm1
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSSE3-NEXT:    movdqa %xmm1, %xmm4
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3]
+; SSSE3-NEXT:    pslld $31, %xmm4
+; SSSE3-NEXT:    psrad $31, %xmm4
+; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSSE3-NEXT:    pslld $31, %xmm1
+; SSSE3-NEXT:    psrad $31, %xmm1
+; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm0[8],xmm3[9],xmm0[9],xmm3[10],xmm0[10],xmm3[11],xmm0[11],xmm3[12],xmm0[12],xmm3[13],xmm0[13],xmm3[14],xmm0[14],xmm3[15],xmm0[15]
+; SSSE3-NEXT:    movdqa %xmm3, %xmm2
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
+; SSSE3-NEXT:    pslld $31, %xmm2
+; SSSE3-NEXT:    psrad $31, %xmm2
+; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
+; SSSE3-NEXT:    pslld $31, %xmm3
+; SSSE3-NEXT:    psrad $31, %xmm3
+; SSSE3-NEXT:    movdqa %xmm0, (%rdi)
+; SSSE3-NEXT:    movdqa %xmm4, %xmm0
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: saddo_v16i8:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    pxor %xmm3, %xmm3
+; SSE41-NEXT:    pxor %xmm2, %xmm2
+; SSE41-NEXT:    pcmpgtb %xmm1, %xmm2
+; SSE41-NEXT:    pcmpeqd %xmm4, %xmm4
+; SSE41-NEXT:    pxor %xmm4, %xmm2
+; SSE41-NEXT:    pxor %xmm5, %xmm5
+; SSE41-NEXT:    pcmpgtb %xmm0, %xmm5
+; SSE41-NEXT:    pxor %xmm4, %xmm5
+; SSE41-NEXT:    pcmpeqb %xmm5, %xmm2
+; SSE41-NEXT:    paddb %xmm1, %xmm0
+; SSE41-NEXT:    pcmpgtb %xmm0, %xmm3
+; SSE41-NEXT:    pxor %xmm4, %xmm3
+; SSE41-NEXT:    pcmpeqb %xmm5, %xmm3
+; SSE41-NEXT:    pandn %xmm2, %xmm3
+; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm4 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
+; SSE41-NEXT:    pslld $31, %xmm4
+; SSE41-NEXT:    psrad $31, %xmm4
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,2,3]
+; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
+; SSE41-NEXT:    pslld $31, %xmm1
+; SSE41-NEXT:    psrad $31, %xmm1
+; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[2,3,0,1]
+; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero
+; SSE41-NEXT:    pslld $31, %xmm2
+; SSE41-NEXT:    psrad $31, %xmm2
+; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[3,1,2,3]
+; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
+; SSE41-NEXT:    pslld $31, %xmm3
+; SSE41-NEXT:    psrad $31, %xmm3
+; SSE41-NEXT:    movdqa %xmm0, (%rdi)
+; SSE41-NEXT:    movdqa %xmm4, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: saddo_v16i8:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX1-NEXT:    vpcmpgtb %xmm1, %xmm2, %xmm3
+; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
+; AVX1-NEXT:    vpxor %xmm4, %xmm3, %xmm3
+; AVX1-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm5
+; AVX1-NEXT:    vpxor %xmm4, %xmm5, %xmm5
+; AVX1-NEXT:    vpcmpeqb %xmm3, %xmm5, %xmm3
+; AVX1-NEXT:    vpaddb %xmm1, %xmm0, %xmm6
+; AVX1-NEXT:    vpcmpgtb %xmm6, %xmm2, %xmm0
+; AVX1-NEXT:    vpxor %xmm4, %xmm0, %xmm0
+; AVX1-NEXT:    vpcmpeqb %xmm0, %xmm5, %xmm0
+; AVX1-NEXT:    vpandn %xmm3, %xmm0, %xmm1
+; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; AVX1-NEXT:    vpmovzxwd {{.*#+}} xmm2 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; AVX1-NEXT:    vpslld $31, %xmm2, %xmm2
+; AVX1-NEXT:    vpsrad $31, %xmm2, %xmm2
+; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm0[4,4,5,5,6,6,7,7]
+; AVX1-NEXT:    vpslld $31, %xmm0, %xmm0
+; AVX1-NEXT:    vpsrad $31, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm2, %ymm0
+; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
+; AVX1-NEXT:    vpmovzxwd {{.*#+}} xmm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
+; AVX1-NEXT:    vpslld $31, %xmm2, %xmm2
+; AVX1-NEXT:    vpsrad $31, %xmm2, %xmm2
+; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; AVX1-NEXT:    vpslld $31, %xmm1, %xmm1
+; AVX1-NEXT:    vpsrad $31, %xmm1, %xmm1
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm1
+; AVX1-NEXT:    vmovdqa %xmm6, (%rdi)
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: saddo_v16i8:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX2-NEXT:    vpcmpgtb %xmm1, %xmm2, %xmm3
+; AVX2-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
+; AVX2-NEXT:    vpxor %xmm4, %xmm3, %xmm3
+; AVX2-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm5
+; AVX2-NEXT:    vpxor %xmm4, %xmm5, %xmm5
+; AVX2-NEXT:    vpcmpeqb %xmm3, %xmm5, %xmm3
+; AVX2-NEXT:    vpaddb %xmm1, %xmm0, %xmm6
+; AVX2-NEXT:    vpcmpgtb %xmm6, %xmm2, %xmm0
+; AVX2-NEXT:    vpxor %xmm4, %xmm0, %xmm0
+; AVX2-NEXT:    vpcmpeqb %xmm0, %xmm5, %xmm0
+; AVX2-NEXT:    vpandn %xmm3, %xmm0, %xmm1
+; AVX2-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX2-NEXT:    vpslld $31, %ymm0, %ymm0
+; AVX2-NEXT:    vpsrad $31, %ymm0, %ymm0
+; AVX2-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
+; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; AVX2-NEXT:    vpslld $31, %ymm1, %ymm1
+; AVX2-NEXT:    vpsrad $31, %ymm1, %ymm1
+; AVX2-NEXT:    vmovdqa %xmm6, (%rdi)
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: saddo_v16i8:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX512-NEXT:    vpcmpnltb %xmm2, %xmm1, %k0
+; AVX512-NEXT:    vpcmpnltb %xmm2, %xmm0, %k1
+; AVX512-NEXT:    kxorw %k0, %k1, %k0
+; AVX512-NEXT:    vpaddb %xmm1, %xmm0, %xmm1
+; AVX512-NEXT:    vpcmpnltb %xmm2, %xmm1, %k2
+; AVX512-NEXT:    kxorw %k2, %k1, %k1
+; AVX512-NEXT:    kandnw %k1, %k0, %k1
+; AVX512-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; AVX512-NEXT:    vmovdqa %xmm1, (%rdi)
+; AVX512-NEXT:    retq
+  %t = call {<16 x i8>, <16 x i1>} @llvm.sadd.with.overflow.v16i8(<16 x i8> %a0, <16 x i8> %a1)
+  %val = extractvalue {<16 x i8>, <16 x i1>} %t, 0
+  %obit = extractvalue {<16 x i8>, <16 x i1>} %t, 1
+  %res = sext <16 x i1> %obit to <16 x i32>
+  store <16 x i8> %val, <16 x i8>* %p2
+  ret <16 x i32> %res
+}
+
+define <8 x i32> @saddo_v8i16(<8 x i16> %a0, <8 x i16> %a1, <8 x i16>* %p2) nounwind {
+; SSE2-LABEL: saddo_v8i16:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa %xmm1, %xmm2
+; SSE2-NEXT:    pxor %xmm1, %xmm1
+; SSE2-NEXT:    pxor %xmm3, %xmm3
+; SSE2-NEXT:    pcmpgtw %xmm2, %xmm3
+; SSE2-NEXT:    pcmpeqd %xmm4, %xmm4
+; SSE2-NEXT:    pxor %xmm4, %xmm3
+; SSE2-NEXT:    pxor %xmm5, %xmm5
+; SSE2-NEXT:    pcmpgtw %xmm0, %xmm5
+; SSE2-NEXT:    pxor %xmm4, %xmm5
+; SSE2-NEXT:    pcmpeqw %xmm5, %xmm3
+; SSE2-NEXT:    paddw %xmm2, %xmm0
+; SSE2-NEXT:    pcmpgtw %xmm0, %xmm1
+; SSE2-NEXT:    pxor %xmm4, %xmm1
+; SSE2-NEXT:    pcmpeqw %xmm5, %xmm1
+; SSE2-NEXT:    pandn %xmm3, %xmm1
+; SSE2-NEXT:    movdqa %xmm1, %xmm2
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
+; SSE2-NEXT:    pslld $31, %xmm2
+; SSE2-NEXT:    psrad $31, %xmm2
+; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSE2-NEXT:    pslld $31, %xmm1
+; SSE2-NEXT:    psrad $31, %xmm1
+; SSE2-NEXT:    movdqa %xmm0, (%rdi)
+; SSE2-NEXT:    movdqa %xmm2, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: saddo_v8i16:
+; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    movdqa %xmm1, %xmm2
+; SSSE3-NEXT:    pxor %xmm1, %xmm1
+; SSSE3-NEXT:    pxor %xmm3, %xmm3
+; SSSE3-NEXT:    pcmpgtw %xmm2, %xmm3
+; SSSE3-NEXT:    pcmpeqd %xmm4, %xmm4
+; SSSE3-NEXT:    pxor %xmm4, %xmm3
+; SSSE3-NEXT:    pxor %xmm5, %xmm5
+; SSSE3-NEXT:    pcmpgtw %xmm0, %xmm5
+; SSSE3-NEXT:    pxor %xmm4, %xmm5
+; SSSE3-NEXT:    pcmpeqw %xmm5, %xmm3
+; SSSE3-NEXT:    paddw %xmm2, %xmm0
+; SSSE3-NEXT:    pcmpgtw %xmm0, %xmm1
+; SSSE3-NEXT:    pxor %xmm4, %xmm1
+; SSSE3-NEXT:    pcmpeqw %xmm5, %xmm1
+; SSSE3-NEXT:    pandn %xmm3, %xmm1
+; SSSE3-NEXT:    movdqa %xmm1, %xmm2
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
+; SSSE3-NEXT:    pslld $31, %xmm2
+; SSSE3-NEXT:    psrad $31, %xmm2
+; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSSE3-NEXT:    pslld $31, %xmm1
+; SSSE3-NEXT:    psrad $31, %xmm1
+; SSSE3-NEXT:    movdqa %xmm0, (%rdi)
+; SSSE3-NEXT:    movdqa %xmm2, %xmm0
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: saddo_v8i16:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movdqa %xmm1, %xmm2
+; SSE41-NEXT:    pxor %xmm1, %xmm1
+; SSE41-NEXT:    pxor %xmm3, %xmm3
+; SSE41-NEXT:    pcmpgtw %xmm2, %xmm3
+; SSE41-NEXT:    pcmpeqd %xmm4, %xmm4
+; SSE41-NEXT:    pxor %xmm4, %xmm3
+; SSE41-NEXT:    pxor %xmm5, %xmm5
+; SSE41-NEXT:    pcmpgtw %xmm0, %xmm5
+; SSE41-NEXT:    pxor %xmm4, %xmm5
+; SSE41-NEXT:    pcmpeqw %xmm5, %xmm3
+; SSE41-NEXT:    paddw %xmm2, %xmm0
+; SSE41-NEXT:    pcmpgtw %xmm0, %xmm1
+; SSE41-NEXT:    pxor %xmm4, %xmm1
+; SSE41-NEXT:    pcmpeqw %xmm5, %xmm1
+; SSE41-NEXT:    pandn %xmm3, %xmm1
+; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
+; SSE41-NEXT:    pslld $31, %xmm2
+; SSE41-NEXT:    psrad $31, %xmm2
+; SSE41-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSE41-NEXT:    pslld $31, %xmm1
+; SSE41-NEXT:    psrad $31, %xmm1
+; SSE41-NEXT:    movdqa %xmm0, (%rdi)
+; SSE41-NEXT:    movdqa %xmm2, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: saddo_v8i16:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX1-NEXT:    vpcmpgtw %xmm1, %xmm2, %xmm3
+; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
+; AVX1-NEXT:    vpxor %xmm4, %xmm3, %xmm3
+; AVX1-NEXT:    vpcmpgtw %xmm0, %xmm2, %xmm5
+; AVX1-NEXT:    vpxor %xmm4, %xmm5, %xmm5
+; AVX1-NEXT:    vpcmpeqw %xmm3, %xmm5, %xmm3
+; AVX1-NEXT:    vpaddw %xmm1, %xmm0, %xmm1
+; AVX1-NEXT:    vpcmpgtw %xmm1, %xmm2, %xmm0
+; AVX1-NEXT:    vpxor %xmm4, %xmm0, %xmm0
+; AVX1-NEXT:    vpcmpeqw %xmm0, %xmm5, %xmm0
+; AVX1-NEXT:    vpandn %xmm3, %xmm0, %xmm0
+; AVX1-NEXT:    vpmovsxwd %xmm0, %xmm2
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpmovsxwd %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm2, %ymm0
+; AVX1-NEXT:    vmovdqa %xmm1, (%rdi)
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: saddo_v8i16:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX2-NEXT:    vpcmpgtw %xmm1, %xmm2, %xmm3
+; AVX2-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
+; AVX2-NEXT:    vpxor %xmm4, %xmm3, %xmm3
+; AVX2-NEXT:    vpcmpgtw %xmm0, %xmm2, %xmm5
+; AVX2-NEXT:    vpxor %xmm4, %xmm5, %xmm5
+; AVX2-NEXT:    vpcmpeqw %xmm3, %xmm5, %xmm3
+; AVX2-NEXT:    vpaddw %xmm1, %xmm0, %xmm1
+; AVX2-NEXT:    vpcmpgtw %xmm1, %xmm2, %xmm0
+; AVX2-NEXT:    vpxor %xmm4, %xmm0, %xmm0
+; AVX2-NEXT:    vpcmpeqw %xmm0, %xmm5, %xmm0
+; AVX2-NEXT:    vpandn %xmm3, %xmm0, %xmm0
+; AVX2-NEXT:    vpmovsxwd %xmm0, %ymm0
+; AVX2-NEXT:    vmovdqa %xmm1, (%rdi)
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: saddo_v8i16:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX512-NEXT:    vpcmpnltw %xmm2, %xmm1, %k0
+; AVX512-NEXT:    vpcmpnltw %xmm2, %xmm0, %k1
+; AVX512-NEXT:    kxorw %k0, %k1, %k0
+; AVX512-NEXT:    vpaddw %xmm1, %xmm0, %xmm1
+; AVX512-NEXT:    vpcmpnltw %xmm2, %xmm1, %k2
+; AVX512-NEXT:    kxorw %k2, %k1, %k1
+; AVX512-NEXT:    kandnw %k1, %k0, %k1
+; AVX512-NEXT:    vpcmpeqd %ymm0, %ymm0, %ymm0
+; AVX512-NEXT:    vmovdqa32 %ymm0, %ymm0 {%k1} {z}
+; AVX512-NEXT:    vmovdqa %xmm1, (%rdi)
+; AVX512-NEXT:    retq
+  %t = call {<8 x i16>, <8 x i1>} @llvm.sadd.with.overflow.v8i16(<8 x i16> %a0, <8 x i16> %a1)
+  %val = extractvalue {<8 x i16>, <8 x i1>} %t, 0
+  %obit = extractvalue {<8 x i16>, <8 x i1>} %t, 1
+  %res = sext <8 x i1> %obit to <8 x i32>
+  store <8 x i16> %val, <8 x i16>* %p2
+  ret <8 x i32> %res
+}
+
+define <2 x i32> @saddo_v2i64(<2 x i64> %a0, <2 x i64> %a1, <2 x i64>* %p2) nounwind {
+; SSE2-LABEL: saddo_v2i64:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648]
+; SSE2-NEXT:    movdqa %xmm0, %xmm3
+; SSE2-NEXT:    paddq %xmm1, %xmm0
+; SSE2-NEXT:    pxor %xmm2, %xmm1
+; SSE2-NEXT:    movdqa %xmm2, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm1, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm2, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSE2-NEXT:    pand %xmm5, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSE2-NEXT:    por %xmm1, %xmm4
+; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
+; SSE2-NEXT:    pxor %xmm1, %xmm4
+; SSE2-NEXT:    pxor %xmm2, %xmm3
+; SSE2-NEXT:    movdqa %xmm2, %xmm5
+; SSE2-NEXT:    pcmpgtd %xmm3, %xmm5
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm2, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
+; SSE2-NEXT:    pand %xmm6, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
+; SSE2-NEXT:    por %xmm3, %xmm5
+; SSE2-NEXT:    pxor %xmm1, %xmm5
+; SSE2-NEXT:    pcmpeqd %xmm5, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[1,0,3,2]
+; SSE2-NEXT:    pand %xmm4, %xmm3
+; SSE2-NEXT:    movdqa %xmm0, (%rdi)
+; SSE2-NEXT:    pxor %xmm2, %xmm0
+; SSE2-NEXT:    movdqa %xmm2, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm0, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm2, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE2-NEXT:    pand %xmm6, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm4[1,1,3,3]
+; SSE2-NEXT:    por %xmm0, %xmm2
+; SSE2-NEXT:    pxor %xmm1, %xmm2
+; SSE2-NEXT:    pcmpeqd %xmm5, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,0,3,2]
+; SSE2-NEXT:    pand %xmm2, %xmm0
+; SSE2-NEXT:    pandn %xmm3, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: saddo_v2i64:
+; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648]
+; SSSE3-NEXT:    movdqa %xmm0, %xmm3
+; SSSE3-NEXT:    paddq %xmm1, %xmm0
+; SSSE3-NEXT:    pxor %xmm2, %xmm1
+; SSSE3-NEXT:    movdqa %xmm2, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm5, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    por %xmm1, %xmm4
+; SSSE3-NEXT:    pcmpeqd %xmm1, %xmm1
+; SSSE3-NEXT:    pxor %xmm1, %xmm4
+; SSSE3-NEXT:    pxor %xmm2, %xmm3
+; SSSE3-NEXT:    movdqa %xmm2, %xmm5
+; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm5
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm3
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm6, %xmm3
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
+; SSSE3-NEXT:    por %xmm3, %xmm5
+; SSSE3-NEXT:    pxor %xmm1, %xmm5
+; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[1,0,3,2]
+; SSSE3-NEXT:    pand %xmm4, %xmm3
+; SSSE3-NEXT:    movdqa %xmm0, (%rdi)
+; SSSE3-NEXT:    pxor %xmm2, %xmm0
+; SSSE3-NEXT:    movdqa %xmm2, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm6, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    por %xmm0, %xmm2
+; SSSE3-NEXT:    pxor %xmm1, %xmm2
+; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,0,3,2]
+; SSSE3-NEXT:    pand %xmm2, %xmm0
+; SSSE3-NEXT:    pandn %xmm3, %xmm0
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: saddo_v2i64:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648]
+; SSE41-NEXT:    movdqa %xmm0, %xmm3
+; SSE41-NEXT:    paddq %xmm1, %xmm0
+; SSE41-NEXT:    pxor %xmm2, %xmm1
+; SSE41-NEXT:    movdqa %xmm2, %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm1, %xmm4
+; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSE41-NEXT:    pcmpeqd %xmm2, %xmm1
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSE41-NEXT:    pand %xmm5, %xmm1
+; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSE41-NEXT:    por %xmm1, %xmm4
+; SSE41-NEXT:    pcmpeqd %xmm1, %xmm1
+; SSE41-NEXT:    pxor %xmm1, %xmm4
+; SSE41-NEXT:    pxor %xmm2, %xmm3
+; SSE41-NEXT:    movdqa %xmm2, %xmm5
+; SSE41-NEXT:    pcmpgtd %xmm3, %xmm5
+; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
+; SSE41-NEXT:    pcmpeqd %xmm2, %xmm3
+; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
+; SSE41-NEXT:    pand %xmm6, %xmm3
+; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
+; SSE41-NEXT:    por %xmm3, %xmm5
+; SSE41-NEXT:    pxor %xmm1, %xmm5
+; SSE41-NEXT:    pcmpeqq %xmm5, %xmm4
+; SSE41-NEXT:    movdqa %xmm0, (%rdi)
+; SSE41-NEXT:    pxor %xmm2, %xmm0
+; SSE41-NEXT:    movdqa %xmm2, %xmm3
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm3
+; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm3[0,0,2,2]
+; SSE41-NEXT:    pcmpeqd %xmm2, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
+; SSE41-NEXT:    pand %xmm6, %xmm2
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
+; SSE41-NEXT:    por %xmm2, %xmm0
+; SSE41-NEXT:    pxor %xmm1, %xmm0
+; SSE41-NEXT:    pcmpeqq %xmm5, %xmm0
+; SSE41-NEXT:    pandn %xmm4, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: saddo_v2i64:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm2, %xmm3
+; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
+; AVX1-NEXT:    vpxor %xmm4, %xmm3, %xmm3
+; AVX1-NEXT:    vpcmpgtq %xmm0, %xmm2, %xmm5
+; AVX1-NEXT:    vpxor %xmm4, %xmm5, %xmm5
+; AVX1-NEXT:    vpcmpeqq %xmm3, %xmm5, %xmm3
+; AVX1-NEXT:    vpaddq %xmm1, %xmm0, %xmm1
+; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm2, %xmm0
+; AVX1-NEXT:    vpxor %xmm4, %xmm0, %xmm0
+; AVX1-NEXT:    vpcmpeqq %xmm0, %xmm5, %xmm0
+; AVX1-NEXT:    vpandn %xmm3, %xmm0, %xmm0
+; AVX1-NEXT:    vmovdqa %xmm1, (%rdi)
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: saddo_v2i64:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX2-NEXT:    vpcmpgtq %xmm1, %xmm2, %xmm3
+; AVX2-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
+; AVX2-NEXT:    vpxor %xmm4, %xmm3, %xmm3
+; AVX2-NEXT:    vpcmpgtq %xmm0, %xmm2, %xmm5
+; AVX2-NEXT:    vpxor %xmm4, %xmm5, %xmm5
+; AVX2-NEXT:    vpcmpeqq %xmm3, %xmm5, %xmm3
+; AVX2-NEXT:    vpaddq %xmm1, %xmm0, %xmm1
+; AVX2-NEXT:    vpcmpgtq %xmm1, %xmm2, %xmm0
+; AVX2-NEXT:    vpxor %xmm4, %xmm0, %xmm0
+; AVX2-NEXT:    vpcmpeqq %xmm0, %xmm5, %xmm0
+; AVX2-NEXT:    vpandn %xmm3, %xmm0, %xmm0
+; AVX2-NEXT:    vmovdqa %xmm1, (%rdi)
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: saddo_v2i64:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX512-NEXT:    vpcmpnltq %xmm2, %xmm1, %k0
+; AVX512-NEXT:    vpcmpnltq %xmm2, %xmm0, %k1
+; AVX512-NEXT:    kxorw %k0, %k1, %k0
+; AVX512-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vpcmpnltq %xmm2, %xmm0, %k2
+; AVX512-NEXT:    kxorw %k2, %k1, %k1
+; AVX512-NEXT:    kandnw %k1, %k0, %k1
+; AVX512-NEXT:    vmovdqa %xmm0, (%rdi)
+; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX512-NEXT:    vmovdqa64 %xmm0, %xmm0 {%k1} {z}
+; AVX512-NEXT:    retq
+  %t = call {<2 x i64>, <2 x i1>} @llvm.sadd.with.overflow.v2i64(<2 x i64> %a0, <2 x i64> %a1)
+  %val = extractvalue {<2 x i64>, <2 x i1>} %t, 0
+  %obit = extractvalue {<2 x i64>, <2 x i1>} %t, 1
+  %res = sext <2 x i1> %obit to <2 x i32>
+  store <2 x i64> %val, <2 x i64>* %p2
+  ret <2 x i32> %res
+}
+
+define <4 x i32> @saddo_v4i24(<4 x i24> %a0, <4 x i24> %a1, <4 x i24>* %p2) nounwind {
+; SSE2-LABEL: saddo_v4i24:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa %xmm0, %xmm2
+; SSE2-NEXT:    pslld $8, %xmm1
+; SSE2-NEXT:    psrad $8, %xmm1
+; SSE2-NEXT:    pslld $8, %xmm2
+; SSE2-NEXT:    psrad $8, %xmm2
+; SSE2-NEXT:    paddd %xmm1, %xmm2
+; SSE2-NEXT:    movdqa %xmm2, %xmm0
+; SSE2-NEXT:    pslld $8, %xmm0
+; SSE2-NEXT:    psrad $8, %xmm0
+; SSE2-NEXT:    pcmpeqd %xmm2, %xmm0
+; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
+; SSE2-NEXT:    pxor %xmm1, %xmm0
+; SSE2-NEXT:    movd %xmm2, %eax
+; SSE2-NEXT:    movw %ax, (%rdi)
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[3,1,2,3]
+; SSE2-NEXT:    movd %xmm1, %ecx
+; SSE2-NEXT:    movw %cx, 9(%rdi)
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[2,3,0,1]
+; SSE2-NEXT:    movd %xmm1, %edx
+; SSE2-NEXT:    movw %dx, 6(%rdi)
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,1,2,3]
+; SSE2-NEXT:    movd %xmm1, %esi
+; SSE2-NEXT:    movw %si, 3(%rdi)
+; SSE2-NEXT:    shrl $16, %eax
+; SSE2-NEXT:    movb %al, 2(%rdi)
+; SSE2-NEXT:    shrl $16, %ecx
+; SSE2-NEXT:    movb %cl, 11(%rdi)
+; SSE2-NEXT:    shrl $16, %edx
+; SSE2-NEXT:    movb %dl, 8(%rdi)
+; SSE2-NEXT:    shrl $16, %esi
+; SSE2-NEXT:    movb %sil, 5(%rdi)
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: saddo_v4i24:
+; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    movdqa %xmm0, %xmm2
+; SSSE3-NEXT:    pslld $8, %xmm1
+; SSSE3-NEXT:    psrad $8, %xmm1
+; SSSE3-NEXT:    pslld $8, %xmm2
+; SSSE3-NEXT:    psrad $8, %xmm2
+; SSSE3-NEXT:    paddd %xmm1, %xmm2
+; SSSE3-NEXT:    movdqa %xmm2, %xmm0
+; SSSE3-NEXT:    pslld $8, %xmm0
+; SSSE3-NEXT:    psrad $8, %xmm0
+; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm0
+; SSSE3-NEXT:    pcmpeqd %xmm1, %xmm1
+; SSSE3-NEXT:    pxor %xmm1, %xmm0
+; SSSE3-NEXT:    movd %xmm2, %eax
+; SSSE3-NEXT:    movw %ax, (%rdi)
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[3,1,2,3]
+; SSSE3-NEXT:    movd %xmm1, %ecx
+; SSSE3-NEXT:    movw %cx, 9(%rdi)
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[2,3,0,1]
+; SSSE3-NEXT:    movd %xmm1, %edx
+; SSSE3-NEXT:    movw %dx, 6(%rdi)
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,1,2,3]
+; SSSE3-NEXT:    movd %xmm1, %esi
+; SSSE3-NEXT:    movw %si, 3(%rdi)
+; SSSE3-NEXT:    shrl $16, %eax
+; SSSE3-NEXT:    movb %al, 2(%rdi)
+; SSSE3-NEXT:    shrl $16, %ecx
+; SSSE3-NEXT:    movb %cl, 11(%rdi)
+; SSSE3-NEXT:    shrl $16, %edx
+; SSSE3-NEXT:    movb %dl, 8(%rdi)
+; SSSE3-NEXT:    shrl $16, %esi
+; SSSE3-NEXT:    movb %sil, 5(%rdi)
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: saddo_v4i24:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movdqa %xmm0, %xmm2
+; SSE41-NEXT:    pslld $8, %xmm1
+; SSE41-NEXT:    psrad $8, %xmm1
+; SSE41-NEXT:    pslld $8, %xmm2
+; SSE41-NEXT:    psrad $8, %xmm2
+; SSE41-NEXT:    paddd %xmm1, %xmm2
+; SSE41-NEXT:    movdqa %xmm2, %xmm0
+; SSE41-NEXT:    pslld $8, %xmm0
+; SSE41-NEXT:    psrad $8, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm2, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm1, %xmm1
+; SSE41-NEXT:    pxor %xmm1, %xmm0
+; SSE41-NEXT:    pextrd $3, %xmm2, %eax
+; SSE41-NEXT:    movw %ax, 9(%rdi)
+; SSE41-NEXT:    pextrd $2, %xmm2, %ecx
+; SSE41-NEXT:    movw %cx, 6(%rdi)
+; SSE41-NEXT:    pextrd $1, %xmm2, %edx
+; SSE41-NEXT:    movw %dx, 3(%rdi)
+; SSE41-NEXT:    movd %xmm2, %esi
+; SSE41-NEXT:    movw %si, (%rdi)
+; SSE41-NEXT:    shrl $16, %eax
+; SSE41-NEXT:    movb %al, 11(%rdi)
+; SSE41-NEXT:    shrl $16, %ecx
+; SSE41-NEXT:    movb %cl, 8(%rdi)
+; SSE41-NEXT:    shrl $16, %edx
+; SSE41-NEXT:    movb %dl, 5(%rdi)
+; SSE41-NEXT:    shrl $16, %esi
+; SSE41-NEXT:    movb %sil, 2(%rdi)
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: saddo_v4i24:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpslld $8, %xmm1, %xmm1
+; AVX1-NEXT:    vpsrad $8, %xmm1, %xmm1
+; AVX1-NEXT:    vpslld $8, %xmm0, %xmm0
+; AVX1-NEXT:    vpsrad $8, %xmm0, %xmm0
+; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm1
+; AVX1-NEXT:    vpslld $8, %xmm1, %xmm0
+; AVX1-NEXT:    vpsrad $8, %xmm0, %xmm0
+; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
+; AVX1-NEXT:    vpxor %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpextrd $3, %xmm1, %eax
+; AVX1-NEXT:    movw %ax, 9(%rdi)
+; AVX1-NEXT:    vpextrd $2, %xmm1, %ecx
+; AVX1-NEXT:    movw %cx, 6(%rdi)
+; AVX1-NEXT:    vpextrd $1, %xmm1, %edx
+; AVX1-NEXT:    movw %dx, 3(%rdi)
+; AVX1-NEXT:    vmovd %xmm1, %esi
+; AVX1-NEXT:    movw %si, (%rdi)
+; AVX1-NEXT:    shrl $16, %eax
+; AVX1-NEXT:    movb %al, 11(%rdi)
+; AVX1-NEXT:    shrl $16, %ecx
+; AVX1-NEXT:    movb %cl, 8(%rdi)
+; AVX1-NEXT:    shrl $16, %edx
+; AVX1-NEXT:    movb %dl, 5(%rdi)
+; AVX1-NEXT:    shrl $16, %esi
+; AVX1-NEXT:    movb %sil, 2(%rdi)
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: saddo_v4i24:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpslld $8, %xmm1, %xmm1
+; AVX2-NEXT:    vpsrad $8, %xmm1, %xmm1
+; AVX2-NEXT:    vpslld $8, %xmm0, %xmm0
+; AVX2-NEXT:    vpsrad $8, %xmm0, %xmm0
+; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm1
+; AVX2-NEXT:    vpslld $8, %xmm1, %xmm0
+; AVX2-NEXT:    vpsrad $8, %xmm0, %xmm0
+; AVX2-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
+; AVX2-NEXT:    vpxor %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    vpextrd $3, %xmm1, %eax
+; AVX2-NEXT:    movw %ax, 9(%rdi)
+; AVX2-NEXT:    vpextrd $2, %xmm1, %ecx
+; AVX2-NEXT:    movw %cx, 6(%rdi)
+; AVX2-NEXT:    vpextrd $1, %xmm1, %edx
+; AVX2-NEXT:    movw %dx, 3(%rdi)
+; AVX2-NEXT:    vmovd %xmm1, %esi
+; AVX2-NEXT:    movw %si, (%rdi)
+; AVX2-NEXT:    shrl $16, %eax
+; AVX2-NEXT:    movb %al, 11(%rdi)
+; AVX2-NEXT:    shrl $16, %ecx
+; AVX2-NEXT:    movb %cl, 8(%rdi)
+; AVX2-NEXT:    shrl $16, %edx
+; AVX2-NEXT:    movb %dl, 5(%rdi)
+; AVX2-NEXT:    shrl $16, %esi
+; AVX2-NEXT:    movb %sil, 2(%rdi)
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: saddo_v4i24:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpslld $8, %xmm1, %xmm1
+; AVX512-NEXT:    vpsrad $8, %xmm1, %xmm1
+; AVX512-NEXT:    vpslld $8, %xmm0, %xmm0
+; AVX512-NEXT:    vpsrad $8, %xmm0, %xmm0
+; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm1
+; AVX512-NEXT:    vpslld $8, %xmm1, %xmm0
+; AVX512-NEXT:    vpsrad $8, %xmm0, %xmm0
+; AVX512-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vpternlogq $15, %xmm0, %xmm0, %xmm0
+; AVX512-NEXT:    vpextrd $3, %xmm1, %eax
+; AVX512-NEXT:    movw %ax, 9(%rdi)
+; AVX512-NEXT:    vpextrd $2, %xmm1, %ecx
+; AVX512-NEXT:    movw %cx, 6(%rdi)
+; AVX512-NEXT:    vpextrd $1, %xmm1, %edx
+; AVX512-NEXT:    movw %dx, 3(%rdi)
+; AVX512-NEXT:    vmovd %xmm1, %esi
+; AVX512-NEXT:    movw %si, (%rdi)
+; AVX512-NEXT:    shrl $16, %eax
+; AVX512-NEXT:    movb %al, 11(%rdi)
+; AVX512-NEXT:    shrl $16, %ecx
+; AVX512-NEXT:    movb %cl, 8(%rdi)
+; AVX512-NEXT:    shrl $16, %edx
+; AVX512-NEXT:    movb %dl, 5(%rdi)
+; AVX512-NEXT:    shrl $16, %esi
+; AVX512-NEXT:    movb %sil, 2(%rdi)
+; AVX512-NEXT:    retq
+  %t = call {<4 x i24>, <4 x i1>} @llvm.sadd.with.overflow.v4i24(<4 x i24> %a0, <4 x i24> %a1)
+  %val = extractvalue {<4 x i24>, <4 x i1>} %t, 0
+  %obit = extractvalue {<4 x i24>, <4 x i1>} %t, 1
+  %res = sext <4 x i1> %obit to <4 x i32>
+  store <4 x i24> %val, <4 x i24>* %p2
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @saddo_v4i1(<4 x i1> %a0, <4 x i1> %a1, <4 x i1>* %p2) nounwind {
+; SSE-LABEL: saddo_v4i1:
+; SSE:       # %bb.0:
+; SSE-NEXT:    pslld $31, %xmm1
+; SSE-NEXT:    psrad $31, %xmm1
+; SSE-NEXT:    pslld $31, %xmm0
+; SSE-NEXT:    psrad $31, %xmm0
+; SSE-NEXT:    paddd %xmm1, %xmm0
+; SSE-NEXT:    movdqa %xmm0, %xmm1
+; SSE-NEXT:    pslld $31, %xmm1
+; SSE-NEXT:    psrad $31, %xmm1
+; SSE-NEXT:    pcmpeqd %xmm1, %xmm0
+; SSE-NEXT:    pcmpeqd %xmm2, %xmm2
+; SSE-NEXT:    pxor %xmm2, %xmm0
+; SSE-NEXT:    movmskps %xmm1, %eax
+; SSE-NEXT:    movb %al, (%rdi)
+; SSE-NEXT:    retq
+;
+; AVX1-LABEL: saddo_v4i1:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpslld $31, %xmm1, %xmm1
+; AVX1-NEXT:    vpsrad $31, %xmm1, %xmm1
+; AVX1-NEXT:    vpslld $31, %xmm0, %xmm0
+; AVX1-NEXT:    vpsrad $31, %xmm0, %xmm0
+; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpslld $31, %xmm0, %xmm1
+; AVX1-NEXT:    vpsrad $31, %xmm1, %xmm1
+; AVX1-NEXT:    vpcmpeqd %xmm0, %xmm1, %xmm0
+; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
+; AVX1-NEXT:    vpxor %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vmovmskps %xmm1, %eax
+; AVX1-NEXT:    movb %al, (%rdi)
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: saddo_v4i1:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpslld $31, %xmm1, %xmm1
+; AVX2-NEXT:    vpsrad $31, %xmm1, %xmm1
+; AVX2-NEXT:    vpslld $31, %xmm0, %xmm0
+; AVX2-NEXT:    vpsrad $31, %xmm0, %xmm0
+; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpslld $31, %xmm0, %xmm1
+; AVX2-NEXT:    vpsrad $31, %xmm1, %xmm1
+; AVX2-NEXT:    vpcmpeqd %xmm0, %xmm1, %xmm0
+; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
+; AVX2-NEXT:    vpxor %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    vmovmskps %xmm1, %eax
+; AVX2-NEXT:    movb %al, (%rdi)
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: saddo_v4i1:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpslld $31, %xmm0, %xmm0
+; AVX512-NEXT:    vptestmd %xmm0, %xmm0, %k0
+; AVX512-NEXT:    vptestnmd %xmm0, %xmm0, %k1
+; AVX512-NEXT:    vpslld $31, %xmm1, %xmm0
+; AVX512-NEXT:    vptestmd %xmm0, %xmm0, %k2
+; AVX512-NEXT:    kxorw %k2, %k1, %k3
+; AVX512-NEXT:    kxorw %k2, %k0, %k0
+; AVX512-NEXT:    kxorw %k0, %k1, %k1
+; AVX512-NEXT:    kandnw %k3, %k1, %k1
+; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
+; AVX512-NEXT:    kmovd %k0, %eax
+; AVX512-NEXT:    movb %al, (%rdi)
+; AVX512-NEXT:    retq
+  %t = call {<4 x i1>, <4 x i1>} @llvm.sadd.with.overflow.v4i1(<4 x i1> %a0, <4 x i1> %a1)
+  %val = extractvalue {<4 x i1>, <4 x i1>} %t, 0
+  %obit = extractvalue {<4 x i1>, <4 x i1>} %t, 1
+  %res = sext <4 x i1> %obit to <4 x i32>
+  store <4 x i1> %val, <4 x i1>* %p2
+  ret <4 x i32> %res
+}
+
+define <2 x i32> @saddo_v2i128(<2 x i128> %a0, <2 x i128> %a1, <2 x i128>* %p2) nounwind {
+; SSE2-LABEL: saddo_v2i128:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    pushq %rbp
+; SSE2-NEXT:    pushq %rbx
+; SSE2-NEXT:    movq {{[0-9]+}}(%rsp), %r10
+; SSE2-NEXT:    movq {{[0-9]+}}(%rsp), %r11
+; SSE2-NEXT:    addq {{[0-9]+}}(%rsp), %rdx
+; SSE2-NEXT:    movq %rcx, %rax
+; SSE2-NEXT:    adcq %r11, %rax
+; SSE2-NEXT:    setns %bl
+; SSE2-NEXT:    testq %rcx, %rcx
+; SSE2-NEXT:    setns %cl
+; SSE2-NEXT:    cmpb %bl, %cl
+; SSE2-NEXT:    setne %bpl
+; SSE2-NEXT:    testq %r11, %r11
+; SSE2-NEXT:    setns %bl
+; SSE2-NEXT:    cmpb %bl, %cl
+; SSE2-NEXT:    sete %cl
+; SSE2-NEXT:    andb %bpl, %cl
+; SSE2-NEXT:    movzbl %cl, %ebp
+; SSE2-NEXT:    testq %r9, %r9
+; SSE2-NEXT:    setns %bl
+; SSE2-NEXT:    testq %rsi, %rsi
+; SSE2-NEXT:    setns %cl
+; SSE2-NEXT:    cmpb %bl, %cl
+; SSE2-NEXT:    sete %r11b
+; SSE2-NEXT:    addq %r8, %rdi
+; SSE2-NEXT:    adcq %r9, %rsi
+; SSE2-NEXT:    setns %bl
+; SSE2-NEXT:    cmpb %bl, %cl
+; SSE2-NEXT:    setne %cl
+; SSE2-NEXT:    andb %r11b, %cl
+; SSE2-NEXT:    movzbl %cl, %ecx
+; SSE2-NEXT:    movd %ecx, %xmm0
+; SSE2-NEXT:    pinsrw $4, %ebp, %xmm0
+; SSE2-NEXT:    movq %rdx, 16(%r10)
+; SSE2-NEXT:    movq %rdi, (%r10)
+; SSE2-NEXT:    movq %rax, 24(%r10)
+; SSE2-NEXT:    movq %rsi, 8(%r10)
+; SSE2-NEXT:    psllq $63, %xmm0
+; SSE2-NEXT:    psrad $31, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE2-NEXT:    popq %rbx
+; SSE2-NEXT:    popq %rbp
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: saddo_v2i128:
+; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    pushq %rbp
+; SSSE3-NEXT:    pushq %rbx
+; SSSE3-NEXT:    movq {{[0-9]+}}(%rsp), %r10
+; SSSE3-NEXT:    movq {{[0-9]+}}(%rsp), %r11
+; SSSE3-NEXT:    addq {{[0-9]+}}(%rsp), %rdx
+; SSSE3-NEXT:    movq %rcx, %rax
+; SSSE3-NEXT:    adcq %r11, %rax
+; SSSE3-NEXT:    setns %bl
+; SSSE3-NEXT:    testq %rcx, %rcx
+; SSSE3-NEXT:    setns %cl
+; SSSE3-NEXT:    cmpb %bl, %cl
+; SSSE3-NEXT:    setne %bpl
+; SSSE3-NEXT:    testq %r11, %r11
+; SSSE3-NEXT:    setns %bl
+; SSSE3-NEXT:    cmpb %bl, %cl
+; SSSE3-NEXT:    sete %cl
+; SSSE3-NEXT:    andb %bpl, %cl
+; SSSE3-NEXT:    movzbl %cl, %ebp
+; SSSE3-NEXT:    testq %r9, %r9
+; SSSE3-NEXT:    setns %bl
+; SSSE3-NEXT:    testq %rsi, %rsi
+; SSSE3-NEXT:    setns %cl
+; SSSE3-NEXT:    cmpb %bl, %cl
+; SSSE3-NEXT:    sete %r11b
+; SSSE3-NEXT:    addq %r8, %rdi
+; SSSE3-NEXT:    adcq %r9, %rsi
+; SSSE3-NEXT:    setns %bl
+; SSSE3-NEXT:    cmpb %bl, %cl
+; SSSE3-NEXT:    setne %cl
+; SSSE3-NEXT:    andb %r11b, %cl
+; SSSE3-NEXT:    movzbl %cl, %ecx
+; SSSE3-NEXT:    movd %ecx, %xmm0
+; SSSE3-NEXT:    pinsrw $4, %ebp, %xmm0
+; SSSE3-NEXT:    movq %rdx, 16(%r10)
+; SSSE3-NEXT:    movq %rdi, (%r10)
+; SSSE3-NEXT:    movq %rax, 24(%r10)
+; SSSE3-NEXT:    movq %rsi, 8(%r10)
+; SSSE3-NEXT:    psllq $63, %xmm0
+; SSSE3-NEXT:    psrad $31, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSSE3-NEXT:    popq %rbx
+; SSSE3-NEXT:    popq %rbp
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: saddo_v2i128:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    pushq %rbp
+; SSE41-NEXT:    pushq %rbx
+; SSE41-NEXT:    movq {{[0-9]+}}(%rsp), %r10
+; SSE41-NEXT:    movq {{[0-9]+}}(%rsp), %r11
+; SSE41-NEXT:    addq {{[0-9]+}}(%rsp), %rdx
+; SSE41-NEXT:    movq %rcx, %rax
+; SSE41-NEXT:    adcq %r11, %rax
+; SSE41-NEXT:    setns %bl
+; SSE41-NEXT:    testq %rcx, %rcx
+; SSE41-NEXT:    setns %cl
+; SSE41-NEXT:    cmpb %bl, %cl
+; SSE41-NEXT:    setne %bpl
+; SSE41-NEXT:    testq %r11, %r11
+; SSE41-NEXT:    setns %bl
+; SSE41-NEXT:    cmpb %bl, %cl
+; SSE41-NEXT:    sete %cl
+; SSE41-NEXT:    andb %bpl, %cl
+; SSE41-NEXT:    movzbl %cl, %ebp
+; SSE41-NEXT:    testq %r9, %r9
+; SSE41-NEXT:    setns %bl
+; SSE41-NEXT:    testq %rsi, %rsi
+; SSE41-NEXT:    setns %cl
+; SSE41-NEXT:    cmpb %bl, %cl
+; SSE41-NEXT:    sete %r11b
+; SSE41-NEXT:    addq %r8, %rdi
+; SSE41-NEXT:    adcq %r9, %rsi
+; SSE41-NEXT:    setns %bl
+; SSE41-NEXT:    cmpb %bl, %cl
+; SSE41-NEXT:    setne %cl
+; SSE41-NEXT:    andb %r11b, %cl
+; SSE41-NEXT:    movzbl %cl, %ecx
+; SSE41-NEXT:    movd %ecx, %xmm0
+; SSE41-NEXT:    pinsrb $8, %ebp, %xmm0
+; SSE41-NEXT:    movq %rdx, 16(%r10)
+; SSE41-NEXT:    movq %rdi, (%r10)
+; SSE41-NEXT:    movq %rax, 24(%r10)
+; SSE41-NEXT:    movq %rsi, 8(%r10)
+; SSE41-NEXT:    psllq $63, %xmm0
+; SSE41-NEXT:    psrad $31, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE41-NEXT:    popq %rbx
+; SSE41-NEXT:    popq %rbp
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: saddo_v2i128:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    pushq %rbp
+; AVX1-NEXT:    pushq %rbx
+; AVX1-NEXT:    movq {{[0-9]+}}(%rsp), %r10
+; AVX1-NEXT:    movq {{[0-9]+}}(%rsp), %r11
+; AVX1-NEXT:    addq {{[0-9]+}}(%rsp), %rdx
+; AVX1-NEXT:    movq %rcx, %rax
+; AVX1-NEXT:    adcq %r11, %rax
+; AVX1-NEXT:    setns %bl
+; AVX1-NEXT:    testq %rcx, %rcx
+; AVX1-NEXT:    setns %cl
+; AVX1-NEXT:    cmpb %bl, %cl
+; AVX1-NEXT:    setne %bpl
+; AVX1-NEXT:    testq %r11, %r11
+; AVX1-NEXT:    setns %bl
+; AVX1-NEXT:    cmpb %bl, %cl
+; AVX1-NEXT:    sete %cl
+; AVX1-NEXT:    andb %bpl, %cl
+; AVX1-NEXT:    movzbl %cl, %ebp
+; AVX1-NEXT:    testq %r9, %r9
+; AVX1-NEXT:    setns %bl
+; AVX1-NEXT:    testq %rsi, %rsi
+; AVX1-NEXT:    setns %cl
+; AVX1-NEXT:    cmpb %bl, %cl
+; AVX1-NEXT:    sete %r11b
+; AVX1-NEXT:    addq %r8, %rdi
+; AVX1-NEXT:    adcq %r9, %rsi
+; AVX1-NEXT:    setns %bl
+; AVX1-NEXT:    cmpb %bl, %cl
+; AVX1-NEXT:    setne %cl
+; AVX1-NEXT:    andb %r11b, %cl
+; AVX1-NEXT:    movzbl %cl, %ecx
+; AVX1-NEXT:    vmovd %ecx, %xmm0
+; AVX1-NEXT:    vpinsrb $8, %ebp, %xmm0, %xmm0
+; AVX1-NEXT:    movq %rdx, 16(%r10)
+; AVX1-NEXT:    movq %rdi, (%r10)
+; AVX1-NEXT:    movq %rax, 24(%r10)
+; AVX1-NEXT:    movq %rsi, 8(%r10)
+; AVX1-NEXT:    vpsllq $63, %xmm0, %xmm0
+; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX1-NEXT:    vpcmpgtq %xmm0, %xmm1, %xmm0
+; AVX1-NEXT:    popq %rbx
+; AVX1-NEXT:    popq %rbp
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: saddo_v2i128:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    pushq %rbp
+; AVX2-NEXT:    pushq %rbx
+; AVX2-NEXT:    movq {{[0-9]+}}(%rsp), %r10
+; AVX2-NEXT:    movq {{[0-9]+}}(%rsp), %r11
+; AVX2-NEXT:    addq {{[0-9]+}}(%rsp), %rdx
+; AVX2-NEXT:    movq %rcx, %rax
+; AVX2-NEXT:    adcq %r11, %rax
+; AVX2-NEXT:    setns %bl
+; AVX2-NEXT:    testq %rcx, %rcx
+; AVX2-NEXT:    setns %cl
+; AVX2-NEXT:    cmpb %bl, %cl
+; AVX2-NEXT:    setne %bpl
+; AVX2-NEXT:    testq %r11, %r11
+; AVX2-NEXT:    setns %bl
+; AVX2-NEXT:    cmpb %bl, %cl
+; AVX2-NEXT:    sete %cl
+; AVX2-NEXT:    andb %bpl, %cl
+; AVX2-NEXT:    movzbl %cl, %ebp
+; AVX2-NEXT:    testq %r9, %r9
+; AVX2-NEXT:    setns %bl
+; AVX2-NEXT:    testq %rsi, %rsi
+; AVX2-NEXT:    setns %cl
+; AVX2-NEXT:    cmpb %bl, %cl
+; AVX2-NEXT:    sete %r11b
+; AVX2-NEXT:    addq %r8, %rdi
+; AVX2-NEXT:    adcq %r9, %rsi
+; AVX2-NEXT:    setns %bl
+; AVX2-NEXT:    cmpb %bl, %cl
+; AVX2-NEXT:    setne %cl
+; AVX2-NEXT:    andb %r11b, %cl
+; AVX2-NEXT:    movzbl %cl, %ecx
+; AVX2-NEXT:    vmovd %ecx, %xmm0
+; AVX2-NEXT:    vpinsrb $8, %ebp, %xmm0, %xmm0
+; AVX2-NEXT:    movq %rdx, 16(%r10)
+; AVX2-NEXT:    movq %rdi, (%r10)
+; AVX2-NEXT:    movq %rax, 24(%r10)
+; AVX2-NEXT:    movq %rsi, 8(%r10)
+; AVX2-NEXT:    vpsllq $63, %xmm0, %xmm0
+; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX2-NEXT:    vpcmpgtq %xmm0, %xmm1, %xmm0
+; AVX2-NEXT:    popq %rbx
+; AVX2-NEXT:    popq %rbp
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: saddo_v2i128:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    pushq %r14
+; AVX512-NEXT:    pushq %rbx
+; AVX512-NEXT:    movq {{[0-9]+}}(%rsp), %r10
+; AVX512-NEXT:    movq {{[0-9]+}}(%rsp), %r11
+; AVX512-NEXT:    addq {{[0-9]+}}(%rsp), %rdx
+; AVX512-NEXT:    movq %rcx, %r14
+; AVX512-NEXT:    adcq %r11, %r14
+; AVX512-NEXT:    setns %bl
+; AVX512-NEXT:    testq %rcx, %rcx
+; AVX512-NEXT:    setns %cl
+; AVX512-NEXT:    cmpb %bl, %cl
+; AVX512-NEXT:    setne %bl
+; AVX512-NEXT:    testq %r11, %r11
+; AVX512-NEXT:    setns %al
+; AVX512-NEXT:    cmpb %al, %cl
+; AVX512-NEXT:    sete %al
+; AVX512-NEXT:    andb %bl, %al
+; AVX512-NEXT:    movb %al, -{{[0-9]+}}(%rsp)
+; AVX512-NEXT:    testq %r9, %r9
+; AVX512-NEXT:    setns %al
+; AVX512-NEXT:    testq %rsi, %rsi
+; AVX512-NEXT:    setns %cl
+; AVX512-NEXT:    cmpb %al, %cl
+; AVX512-NEXT:    sete %al
+; AVX512-NEXT:    addq %r8, %rdi
+; AVX512-NEXT:    adcq %r9, %rsi
+; AVX512-NEXT:    setns %bl
+; AVX512-NEXT:    cmpb %bl, %cl
+; AVX512-NEXT:    setne %cl
+; AVX512-NEXT:    andb %al, %cl
+; AVX512-NEXT:    movb %cl, -{{[0-9]+}}(%rsp)
+; AVX512-NEXT:    kmovw -{{[0-9]+}}(%rsp), %k1
+; AVX512-NEXT:    movq %rdx, 16(%r10)
+; AVX512-NEXT:    movq %rdi, (%r10)
+; AVX512-NEXT:    movq %r14, 24(%r10)
+; AVX512-NEXT:    movq %rsi, 8(%r10)
+; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX512-NEXT:    vmovdqa64 %xmm0, %xmm0 {%k1} {z}
+; AVX512-NEXT:    popq %rbx
+; AVX512-NEXT:    popq %r14
+; AVX512-NEXT:    retq
+  %t = call {<2 x i128>, <2 x i1>} @llvm.sadd.with.overflow.v2i128(<2 x i128> %a0, <2 x i128> %a1)
+  %val = extractvalue {<2 x i128>, <2 x i1>} %t, 0
+  %obit = extractvalue {<2 x i128>, <2 x i1>} %t, 1
+  %res = sext <2 x i1> %obit to <2 x i32>
+  store <2 x i128> %val, <2 x i128>* %p2
+  ret <2 x i32> %res
+}
diff --git a/test/CodeGen/X86/vec_setcc-2.ll b/test/CodeGen/X86/vec_setcc-2.ll
index 946c9fcdaa32f9e164fa28bc2e5e250e76099568..2a0a166ec9af145f73615a2f758f39aa0f9e3e97 100644
--- a/test/CodeGen/X86/vec_setcc-2.ll
+++ b/test/CodeGen/X86/vec_setcc-2.ll
@@ -511,11 +511,9 @@ define <4 x i1> @ugt_v4i32_nonsplat(<4 x i32> %x) {
 ;
 ; SSE41-LABEL: ugt_v4i32_nonsplat:
 ; SSE41:       ## %bb.0:
-; SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [4294967253,4294967254,4294967255,4294967256]
-; SSE41-NEXT:    pminud %xmm0, %xmm1
+; SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [4294967254,4294967255,4294967256,4294967257]
+; SSE41-NEXT:    pmaxud %xmm0, %xmm1
 ; SSE41-NEXT:    pcmpeqd %xmm1, %xmm0
-; SSE41-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE41-NEXT:    pxor %xmm1, %xmm0
 ; SSE41-NEXT:    retq
   %cmp = icmp ugt <4 x i32> %x, <i32 -43, i32 -42, i32 -41, i32 -40>
   ret <4 x i1> %cmp
diff --git a/test/CodeGen/X86/vec_ssubo.ll b/test/CodeGen/X86/vec_ssubo.ll
new file mode 100644
index 0000000000000000000000000000000000000000..0e1354b939ea9e0fa0fc137cc1e25d16537bf1a8
--- /dev/null
+++ b/test/CodeGen/X86/vec_ssubo.ll
@@ -0,0 +1,2078 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefixes=SSE,SSE2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+ssse3 | FileCheck %s --check-prefixes=SSE,SSSE3
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefixes=SSE,SSE41
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefixes=AVX,AVX1
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefixes=AVX,AVX2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw,+avx512vl,+fast-variable-shuffle | FileCheck %s --check-prefixes=AVX,AVX512
+
+declare {<1 x i32>, <1 x i1>} @llvm.ssub.with.overflow.v1i32(<1 x i32>, <1 x i32>)
+declare {<2 x i32>, <2 x i1>} @llvm.ssub.with.overflow.v2i32(<2 x i32>, <2 x i32>)
+declare {<3 x i32>, <3 x i1>} @llvm.ssub.with.overflow.v3i32(<3 x i32>, <3 x i32>)
+declare {<4 x i32>, <4 x i1>} @llvm.ssub.with.overflow.v4i32(<4 x i32>, <4 x i32>)
+declare {<6 x i32>, <6 x i1>} @llvm.ssub.with.overflow.v6i32(<6 x i32>, <6 x i32>)
+declare {<8 x i32>, <8 x i1>} @llvm.ssub.with.overflow.v8i32(<8 x i32>, <8 x i32>)
+declare {<16 x i32>, <16 x i1>} @llvm.ssub.with.overflow.v16i32(<16 x i32>, <16 x i32>)
+
+declare {<16 x i8>, <16 x i1>} @llvm.ssub.with.overflow.v16i8(<16 x i8>, <16 x i8>)
+declare {<8 x i16>, <8 x i1>} @llvm.ssub.with.overflow.v8i16(<8 x i16>, <8 x i16>)
+declare {<2 x i64>, <2 x i1>} @llvm.ssub.with.overflow.v2i64(<2 x i64>, <2 x i64>)
+
+declare {<4 x i24>, <4 x i1>} @llvm.ssub.with.overflow.v4i24(<4 x i24>, <4 x i24>)
+declare {<4 x i1>, <4 x i1>} @llvm.ssub.with.overflow.v4i1(<4 x i1>, <4 x i1>)
+declare {<2 x i128>, <2 x i1>} @llvm.ssub.with.overflow.v2i128(<2 x i128>, <2 x i128>)
+
+define <1 x i32> @ssubo_v1i32(<1 x i32> %a0, <1 x i32> %a1, <1 x i32>* %p2) nounwind {
+; SSE-LABEL: ssubo_v1i32:
+; SSE:       # %bb.0:
+; SSE-NEXT:    xorl %eax, %eax
+; SSE-NEXT:    subl %esi, %edi
+; SSE-NEXT:    seto %al
+; SSE-NEXT:    negl %eax
+; SSE-NEXT:    movl %edi, (%rdx)
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: ssubo_v1i32:
+; AVX:       # %bb.0:
+; AVX-NEXT:    xorl %eax, %eax
+; AVX-NEXT:    subl %esi, %edi
+; AVX-NEXT:    seto %al
+; AVX-NEXT:    negl %eax
+; AVX-NEXT:    movl %edi, (%rdx)
+; AVX-NEXT:    retq
+  %t = call {<1 x i32>, <1 x i1>} @llvm.ssub.with.overflow.v1i32(<1 x i32> %a0, <1 x i32> %a1)
+  %val = extractvalue {<1 x i32>, <1 x i1>} %t, 0
+  %obit = extractvalue {<1 x i32>, <1 x i1>} %t, 1
+  %res = sext <1 x i1> %obit to <1 x i32>
+  store <1 x i32> %val, <1 x i32>* %p2
+  ret <1 x i32> %res
+}
+
+define <2 x i32> @ssubo_v2i32(<2 x i32> %a0, <2 x i32> %a1, <2 x i32>* %p2) nounwind {
+; SSE2-LABEL: ssubo_v2i32:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    psllq $32, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,3,2,3]
+; SSE2-NEXT:    psrad $31, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,3,2,3]
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; SSE2-NEXT:    psllq $32, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,3,2,3]
+; SSE2-NEXT:    psrad $31, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,3,2,3]
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSE2-NEXT:    psubq %xmm2, %xmm1
+; SSE2-NEXT:    movdqa %xmm1, %xmm0
+; SSE2-NEXT:    psllq $32, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,3,2,3]
+; SSE2-NEXT:    psrad $31, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,3,2,3]
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
+; SSE2-NEXT:    pcmpeqd %xmm1, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[1,0,3,2]
+; SSE2-NEXT:    pand %xmm2, %xmm3
+; SSE2-NEXT:    pcmpeqd %xmm0, %xmm0
+; SSE2-NEXT:    pxor %xmm3, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; SSE2-NEXT:    movq %xmm1, (%rdi)
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: ssubo_v2i32:
+; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    psllq $32, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,3,2,3]
+; SSSE3-NEXT:    psrad $31, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,3,2,3]
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; SSSE3-NEXT:    psllq $32, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,3,2,3]
+; SSSE3-NEXT:    psrad $31, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,3,2,3]
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSSE3-NEXT:    psubq %xmm2, %xmm1
+; SSSE3-NEXT:    movdqa %xmm1, %xmm0
+; SSSE3-NEXT:    psllq $32, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,3,2,3]
+; SSSE3-NEXT:    psrad $31, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,3,2,3]
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
+; SSSE3-NEXT:    pcmpeqd %xmm1, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[1,0,3,2]
+; SSSE3-NEXT:    pand %xmm2, %xmm3
+; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm0
+; SSSE3-NEXT:    pxor %xmm3, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; SSSE3-NEXT:    movq %xmm1, (%rdi)
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: ssubo_v2i32:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movdqa %xmm1, %xmm2
+; SSE41-NEXT:    psllq $32, %xmm2
+; SSE41-NEXT:    psrad $31, %xmm2
+; SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
+; SSE41-NEXT:    movdqa %xmm0, %xmm1
+; SSE41-NEXT:    psllq $32, %xmm1
+; SSE41-NEXT:    psrad $31, %xmm1
+; SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm0[0,1],xmm1[2,3],xmm0[4,5],xmm1[6,7]
+; SSE41-NEXT:    psubq %xmm2, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    psllq $32, %xmm0
+; SSE41-NEXT:    psrad $31, %xmm0
+; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3],xmm1[4,5],xmm0[6,7]
+; SSE41-NEXT:    pcmpeqq %xmm1, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm2, %xmm2
+; SSE41-NEXT:    pxor %xmm2, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; SSE41-NEXT:    movq %xmm1, (%rdi)
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: ssubo_v2i32:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpsllq $32, %xmm1, %xmm2
+; AVX1-NEXT:    vpsrad $31, %xmm2, %xmm2
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
+; AVX1-NEXT:    vpsllq $32, %xmm0, %xmm2
+; AVX1-NEXT:    vpsrad $31, %xmm2, %xmm2
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3],xmm0[4,5],xmm2[6,7]
+; AVX1-NEXT:    vpsubq %xmm1, %xmm0, %xmm1
+; AVX1-NEXT:    vpsllq $32, %xmm1, %xmm0
+; AVX1-NEXT:    vpsrad $31, %xmm0, %xmm0
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3],xmm1[4,5],xmm0[6,7]
+; AVX1-NEXT:    vpcmpeqq %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
+; AVX1-NEXT:    vpxor %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; AVX1-NEXT:    vmovq %xmm1, (%rdi)
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: ssubo_v2i32:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpsllq $32, %xmm1, %xmm2
+; AVX2-NEXT:    vpsrad $31, %xmm2, %xmm2
+; AVX2-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3]
+; AVX2-NEXT:    vpsllq $32, %xmm0, %xmm2
+; AVX2-NEXT:    vpsrad $31, %xmm2, %xmm2
+; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2],xmm2[3]
+; AVX2-NEXT:    vpsubq %xmm1, %xmm0, %xmm1
+; AVX2-NEXT:    vpsllq $32, %xmm1, %xmm0
+; AVX2-NEXT:    vpsrad $31, %xmm0, %xmm0
+; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm1[0],xmm0[1],xmm1[2],xmm0[3]
+; AVX2-NEXT:    vpcmpeqq %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
+; AVX2-NEXT:    vpxor %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; AVX2-NEXT:    vmovq %xmm1, (%rdi)
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: ssubo_v2i32:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpsllq $32, %xmm1, %xmm1
+; AVX512-NEXT:    vpsraq $32, %xmm1, %xmm1
+; AVX512-NEXT:    vpsllq $32, %xmm0, %xmm0
+; AVX512-NEXT:    vpsraq $32, %xmm0, %xmm0
+; AVX512-NEXT:    vpsubq %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vpsllq $32, %xmm0, %xmm1
+; AVX512-NEXT:    vpsraq $32, %xmm1, %xmm1
+; AVX512-NEXT:    vpmovqd %xmm0, (%rdi)
+; AVX512-NEXT:    vpcmpeqq %xmm0, %xmm1, %xmm0
+; AVX512-NEXT:    vpternlogq $15, %xmm0, %xmm0, %xmm0
+; AVX512-NEXT:    retq
+  %t = call {<2 x i32>, <2 x i1>} @llvm.ssub.with.overflow.v2i32(<2 x i32> %a0, <2 x i32> %a1)
+  %val = extractvalue {<2 x i32>, <2 x i1>} %t, 0
+  %obit = extractvalue {<2 x i32>, <2 x i1>} %t, 1
+  %res = sext <2 x i1> %obit to <2 x i32>
+  store <2 x i32> %val, <2 x i32>* %p2
+  ret <2 x i32> %res
+}
+
+define <3 x i32> @ssubo_v3i32(<3 x i32> %a0, <3 x i32> %a1, <3 x i32>* %p2) nounwind {
+; SSE2-LABEL: ssubo_v3i32:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    pxor %xmm3, %xmm3
+; SSE2-NEXT:    pxor %xmm2, %xmm2
+; SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
+; SSE2-NEXT:    pcmpeqd %xmm4, %xmm4
+; SSE2-NEXT:    pxor %xmm4, %xmm2
+; SSE2-NEXT:    pxor %xmm5, %xmm5
+; SSE2-NEXT:    pcmpgtd %xmm0, %xmm5
+; SSE2-NEXT:    pxor %xmm4, %xmm5
+; SSE2-NEXT:    pcmpeqd %xmm5, %xmm2
+; SSE2-NEXT:    psubd %xmm1, %xmm0
+; SSE2-NEXT:    pcmpgtd %xmm0, %xmm3
+; SSE2-NEXT:    pxor %xmm4, %xmm3
+; SSE2-NEXT:    pcmpeqd %xmm5, %xmm3
+; SSE2-NEXT:    pxor %xmm4, %xmm3
+; SSE2-NEXT:    pandn %xmm3, %xmm2
+; SSE2-NEXT:    movq %xmm0, (%rdi)
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE2-NEXT:    movd %xmm0, 8(%rdi)
+; SSE2-NEXT:    movdqa %xmm2, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: ssubo_v3i32:
+; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    pxor %xmm3, %xmm3
+; SSSE3-NEXT:    pxor %xmm2, %xmm2
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm2
+; SSSE3-NEXT:    pcmpeqd %xmm4, %xmm4
+; SSSE3-NEXT:    pxor %xmm4, %xmm2
+; SSSE3-NEXT:    pxor %xmm5, %xmm5
+; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm5
+; SSSE3-NEXT:    pxor %xmm4, %xmm5
+; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm2
+; SSSE3-NEXT:    psubd %xmm1, %xmm0
+; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm3
+; SSSE3-NEXT:    pxor %xmm4, %xmm3
+; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm3
+; SSSE3-NEXT:    pxor %xmm4, %xmm3
+; SSSE3-NEXT:    pandn %xmm3, %xmm2
+; SSSE3-NEXT:    movq %xmm0, (%rdi)
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSSE3-NEXT:    movd %xmm0, 8(%rdi)
+; SSSE3-NEXT:    movdqa %xmm2, %xmm0
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: ssubo_v3i32:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    pxor %xmm3, %xmm3
+; SSE41-NEXT:    pxor %xmm2, %xmm2
+; SSE41-NEXT:    pcmpgtd %xmm1, %xmm2
+; SSE41-NEXT:    pcmpeqd %xmm4, %xmm4
+; SSE41-NEXT:    pxor %xmm4, %xmm2
+; SSE41-NEXT:    pxor %xmm5, %xmm5
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm5
+; SSE41-NEXT:    pxor %xmm4, %xmm5
+; SSE41-NEXT:    pcmpeqd %xmm5, %xmm2
+; SSE41-NEXT:    psubd %xmm1, %xmm0
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm3
+; SSE41-NEXT:    pxor %xmm4, %xmm3
+; SSE41-NEXT:    pcmpeqd %xmm5, %xmm3
+; SSE41-NEXT:    pxor %xmm4, %xmm3
+; SSE41-NEXT:    pandn %xmm3, %xmm2
+; SSE41-NEXT:    pextrd $2, %xmm0, 8(%rdi)
+; SSE41-NEXT:    movq %xmm0, (%rdi)
+; SSE41-NEXT:    movdqa %xmm2, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: ssubo_v3i32:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm2, %xmm3
+; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
+; AVX1-NEXT:    vpxor %xmm4, %xmm3, %xmm3
+; AVX1-NEXT:    vpcmpgtd %xmm0, %xmm2, %xmm5
+; AVX1-NEXT:    vpxor %xmm4, %xmm5, %xmm5
+; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm5, %xmm3
+; AVX1-NEXT:    vpsubd %xmm1, %xmm0, %xmm1
+; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm2, %xmm0
+; AVX1-NEXT:    vpxor %xmm4, %xmm0, %xmm0
+; AVX1-NEXT:    vpcmpeqd %xmm0, %xmm5, %xmm0
+; AVX1-NEXT:    vpxor %xmm4, %xmm0, %xmm0
+; AVX1-NEXT:    vpandn %xmm0, %xmm3, %xmm0
+; AVX1-NEXT:    vpextrd $2, %xmm1, 8(%rdi)
+; AVX1-NEXT:    vmovq %xmm1, (%rdi)
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: ssubo_v3i32:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX2-NEXT:    vpcmpgtd %xmm1, %xmm2, %xmm3
+; AVX2-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
+; AVX2-NEXT:    vpxor %xmm4, %xmm3, %xmm3
+; AVX2-NEXT:    vpcmpgtd %xmm0, %xmm2, %xmm5
+; AVX2-NEXT:    vpxor %xmm4, %xmm5, %xmm5
+; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm5, %xmm3
+; AVX2-NEXT:    vpsubd %xmm1, %xmm0, %xmm1
+; AVX2-NEXT:    vpcmpgtd %xmm1, %xmm2, %xmm0
+; AVX2-NEXT:    vpxor %xmm4, %xmm0, %xmm0
+; AVX2-NEXT:    vpcmpeqd %xmm0, %xmm5, %xmm0
+; AVX2-NEXT:    vpxor %xmm4, %xmm0, %xmm0
+; AVX2-NEXT:    vpandn %xmm0, %xmm3, %xmm0
+; AVX2-NEXT:    vpextrd $2, %xmm1, 8(%rdi)
+; AVX2-NEXT:    vmovq %xmm1, (%rdi)
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: ssubo_v3i32:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX512-NEXT:    vpcmpnltd %xmm2, %xmm1, %k0
+; AVX512-NEXT:    vpcmpnltd %xmm2, %xmm0, %k1
+; AVX512-NEXT:    kxorw %k0, %k1, %k0
+; AVX512-NEXT:    vpsubd %xmm1, %xmm0, %xmm1
+; AVX512-NEXT:    vpcmpnltd %xmm2, %xmm1, %k2
+; AVX512-NEXT:    kxorw %k2, %k1, %k1
+; AVX512-NEXT:    kandw %k1, %k0, %k1
+; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
+; AVX512-NEXT:    vpextrd $2, %xmm1, 8(%rdi)
+; AVX512-NEXT:    vmovq %xmm1, (%rdi)
+; AVX512-NEXT:    retq
+  %t = call {<3 x i32>, <3 x i1>} @llvm.ssub.with.overflow.v3i32(<3 x i32> %a0, <3 x i32> %a1)
+  %val = extractvalue {<3 x i32>, <3 x i1>} %t, 0
+  %obit = extractvalue {<3 x i32>, <3 x i1>} %t, 1
+  %res = sext <3 x i1> %obit to <3 x i32>
+  store <3 x i32> %val, <3 x i32>* %p2
+  ret <3 x i32> %res
+}
+
+define <4 x i32> @ssubo_v4i32(<4 x i32> %a0, <4 x i32> %a1, <4 x i32>* %p2) nounwind {
+; SSE-LABEL: ssubo_v4i32:
+; SSE:       # %bb.0:
+; SSE-NEXT:    pxor %xmm3, %xmm3
+; SSE-NEXT:    pxor %xmm2, %xmm2
+; SSE-NEXT:    pcmpgtd %xmm1, %xmm2
+; SSE-NEXT:    pcmpeqd %xmm4, %xmm4
+; SSE-NEXT:    pxor %xmm4, %xmm2
+; SSE-NEXT:    pxor %xmm5, %xmm5
+; SSE-NEXT:    pcmpgtd %xmm0, %xmm5
+; SSE-NEXT:    pxor %xmm4, %xmm5
+; SSE-NEXT:    pcmpeqd %xmm5, %xmm2
+; SSE-NEXT:    psubd %xmm1, %xmm0
+; SSE-NEXT:    pcmpgtd %xmm0, %xmm3
+; SSE-NEXT:    pxor %xmm4, %xmm3
+; SSE-NEXT:    pcmpeqd %xmm5, %xmm3
+; SSE-NEXT:    pxor %xmm4, %xmm3
+; SSE-NEXT:    pandn %xmm3, %xmm2
+; SSE-NEXT:    movdqa %xmm0, (%rdi)
+; SSE-NEXT:    movdqa %xmm2, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX1-LABEL: ssubo_v4i32:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm2, %xmm3
+; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
+; AVX1-NEXT:    vpxor %xmm4, %xmm3, %xmm3
+; AVX1-NEXT:    vpcmpgtd %xmm0, %xmm2, %xmm5
+; AVX1-NEXT:    vpxor %xmm4, %xmm5, %xmm5
+; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm5, %xmm3
+; AVX1-NEXT:    vpsubd %xmm1, %xmm0, %xmm1
+; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm2, %xmm0
+; AVX1-NEXT:    vpxor %xmm4, %xmm0, %xmm0
+; AVX1-NEXT:    vpcmpeqd %xmm0, %xmm5, %xmm0
+; AVX1-NEXT:    vpxor %xmm4, %xmm0, %xmm0
+; AVX1-NEXT:    vpandn %xmm0, %xmm3, %xmm0
+; AVX1-NEXT:    vmovdqa %xmm1, (%rdi)
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: ssubo_v4i32:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX2-NEXT:    vpcmpgtd %xmm1, %xmm2, %xmm3
+; AVX2-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
+; AVX2-NEXT:    vpxor %xmm4, %xmm3, %xmm3
+; AVX2-NEXT:    vpcmpgtd %xmm0, %xmm2, %xmm5
+; AVX2-NEXT:    vpxor %xmm4, %xmm5, %xmm5
+; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm5, %xmm3
+; AVX2-NEXT:    vpsubd %xmm1, %xmm0, %xmm1
+; AVX2-NEXT:    vpcmpgtd %xmm1, %xmm2, %xmm0
+; AVX2-NEXT:    vpxor %xmm4, %xmm0, %xmm0
+; AVX2-NEXT:    vpcmpeqd %xmm0, %xmm5, %xmm0
+; AVX2-NEXT:    vpxor %xmm4, %xmm0, %xmm0
+; AVX2-NEXT:    vpandn %xmm0, %xmm3, %xmm0
+; AVX2-NEXT:    vmovdqa %xmm1, (%rdi)
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: ssubo_v4i32:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX512-NEXT:    vpcmpnltd %xmm2, %xmm1, %k0
+; AVX512-NEXT:    vpcmpnltd %xmm2, %xmm0, %k1
+; AVX512-NEXT:    kxorw %k0, %k1, %k0
+; AVX512-NEXT:    vpsubd %xmm1, %xmm0, %xmm1
+; AVX512-NEXT:    vpcmpnltd %xmm2, %xmm1, %k2
+; AVX512-NEXT:    kxorw %k2, %k1, %k1
+; AVX512-NEXT:    kandw %k1, %k0, %k1
+; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
+; AVX512-NEXT:    vmovdqa %xmm1, (%rdi)
+; AVX512-NEXT:    retq
+  %t = call {<4 x i32>, <4 x i1>} @llvm.ssub.with.overflow.v4i32(<4 x i32> %a0, <4 x i32> %a1)
+  %val = extractvalue {<4 x i32>, <4 x i1>} %t, 0
+  %obit = extractvalue {<4 x i32>, <4 x i1>} %t, 1
+  %res = sext <4 x i1> %obit to <4 x i32>
+  store <4 x i32> %val, <4 x i32>* %p2
+  ret <4 x i32> %res
+}
+
+define <6 x i32> @ssubo_v6i32(<6 x i32> %a0, <6 x i32> %a1, <6 x i32>* %p2) nounwind {
+; SSE2-LABEL: ssubo_v6i32:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movq %rdi, %rax
+; SSE2-NEXT:    movd %r8d, %xmm0
+; SSE2-NEXT:    movd %ecx, %xmm1
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSE2-NEXT:    movd %edx, %xmm2
+; SSE2-NEXT:    movd %esi, %xmm0
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
+; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; SSE2-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SSE2-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; SSE2-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SSE2-NEXT:    movd {{.*#+}} xmm6 = mem[0],zero,zero,zero
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm6 = xmm6[0],xmm1[0],xmm6[1],xmm1[1]
+; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm6 = xmm6[0],xmm2[0]
+; SSE2-NEXT:    movd %r9d, %xmm1
+; SSE2-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
+; SSE2-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; SSE2-NEXT:    movd {{.*#+}} xmm4 = mem[0],zero,zero,zero
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1]
+; SSE2-NEXT:    movq {{[0-9]+}}(%rsp), %rcx
+; SSE2-NEXT:    pxor %xmm3, %xmm3
+; SSE2-NEXT:    pxor %xmm2, %xmm2
+; SSE2-NEXT:    pcmpgtd %xmm6, %xmm2
+; SSE2-NEXT:    pcmpeqd %xmm5, %xmm5
+; SSE2-NEXT:    pxor %xmm5, %xmm2
+; SSE2-NEXT:    pxor %xmm7, %xmm7
+; SSE2-NEXT:    pcmpgtd %xmm0, %xmm7
+; SSE2-NEXT:    pxor %xmm5, %xmm7
+; SSE2-NEXT:    pcmpeqd %xmm7, %xmm2
+; SSE2-NEXT:    psubd %xmm6, %xmm0
+; SSE2-NEXT:    pxor %xmm6, %xmm6
+; SSE2-NEXT:    pcmpgtd %xmm0, %xmm6
+; SSE2-NEXT:    pxor %xmm5, %xmm6
+; SSE2-NEXT:    pcmpeqd %xmm7, %xmm6
+; SSE2-NEXT:    pxor %xmm5, %xmm6
+; SSE2-NEXT:    pandn %xmm6, %xmm2
+; SSE2-NEXT:    pxor %xmm6, %xmm6
+; SSE2-NEXT:    pcmpgtd %xmm4, %xmm6
+; SSE2-NEXT:    pxor %xmm5, %xmm6
+; SSE2-NEXT:    pxor %xmm7, %xmm7
+; SSE2-NEXT:    pcmpgtd %xmm1, %xmm7
+; SSE2-NEXT:    pxor %xmm5, %xmm7
+; SSE2-NEXT:    pcmpeqd %xmm7, %xmm6
+; SSE2-NEXT:    psubd %xmm4, %xmm1
+; SSE2-NEXT:    pcmpgtd %xmm1, %xmm3
+; SSE2-NEXT:    pxor %xmm5, %xmm3
+; SSE2-NEXT:    pcmpeqd %xmm7, %xmm3
+; SSE2-NEXT:    pxor %xmm5, %xmm3
+; SSE2-NEXT:    pandn %xmm3, %xmm6
+; SSE2-NEXT:    movq %xmm1, 16(%rcx)
+; SSE2-NEXT:    movdqa %xmm0, (%rcx)
+; SSE2-NEXT:    movq %xmm6, 16(%rdi)
+; SSE2-NEXT:    movdqa %xmm2, (%rdi)
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: ssubo_v6i32:
+; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    movq %rdi, %rax
+; SSSE3-NEXT:    movd %r8d, %xmm0
+; SSSE3-NEXT:    movd %ecx, %xmm1
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSSE3-NEXT:    movd %edx, %xmm2
+; SSSE3-NEXT:    movd %esi, %xmm0
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
+; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; SSSE3-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SSSE3-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; SSSE3-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SSSE3-NEXT:    movd {{.*#+}} xmm6 = mem[0],zero,zero,zero
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm6 = xmm6[0],xmm1[0],xmm6[1],xmm1[1]
+; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm6 = xmm6[0],xmm2[0]
+; SSSE3-NEXT:    movd %r9d, %xmm1
+; SSSE3-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
+; SSSE3-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; SSSE3-NEXT:    movd {{.*#+}} xmm4 = mem[0],zero,zero,zero
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1]
+; SSSE3-NEXT:    movq {{[0-9]+}}(%rsp), %rcx
+; SSSE3-NEXT:    pxor %xmm3, %xmm3
+; SSSE3-NEXT:    pxor %xmm2, %xmm2
+; SSSE3-NEXT:    pcmpgtd %xmm6, %xmm2
+; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm5
+; SSSE3-NEXT:    pxor %xmm5, %xmm2
+; SSSE3-NEXT:    pxor %xmm7, %xmm7
+; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm7
+; SSSE3-NEXT:    pxor %xmm5, %xmm7
+; SSSE3-NEXT:    pcmpeqd %xmm7, %xmm2
+; SSSE3-NEXT:    psubd %xmm6, %xmm0
+; SSSE3-NEXT:    pxor %xmm6, %xmm6
+; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm6
+; SSSE3-NEXT:    pxor %xmm5, %xmm6
+; SSSE3-NEXT:    pcmpeqd %xmm7, %xmm6
+; SSSE3-NEXT:    pxor %xmm5, %xmm6
+; SSSE3-NEXT:    pandn %xmm6, %xmm2
+; SSSE3-NEXT:    pxor %xmm6, %xmm6
+; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm6
+; SSSE3-NEXT:    pxor %xmm5, %xmm6
+; SSSE3-NEXT:    pxor %xmm7, %xmm7
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm7
+; SSSE3-NEXT:    pxor %xmm5, %xmm7
+; SSSE3-NEXT:    pcmpeqd %xmm7, %xmm6
+; SSSE3-NEXT:    psubd %xmm4, %xmm1
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm3
+; SSSE3-NEXT:    pxor %xmm5, %xmm3
+; SSSE3-NEXT:    pcmpeqd %xmm7, %xmm3
+; SSSE3-NEXT:    pxor %xmm5, %xmm3
+; SSSE3-NEXT:    pandn %xmm3, %xmm6
+; SSSE3-NEXT:    movq %xmm1, 16(%rcx)
+; SSSE3-NEXT:    movdqa %xmm0, (%rcx)
+; SSSE3-NEXT:    movq %xmm6, 16(%rdi)
+; SSSE3-NEXT:    movdqa %xmm2, (%rdi)
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: ssubo_v6i32:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movq %rdi, %rax
+; SSE41-NEXT:    movd %esi, %xmm0
+; SSE41-NEXT:    pinsrd $1, %edx, %xmm0
+; SSE41-NEXT:    pinsrd $2, %ecx, %xmm0
+; SSE41-NEXT:    pinsrd $3, %r8d, %xmm0
+; SSE41-NEXT:    movd %r9d, %xmm1
+; SSE41-NEXT:    pinsrd $1, {{[0-9]+}}(%rsp), %xmm1
+; SSE41-NEXT:    movd {{.*#+}} xmm3 = mem[0],zero,zero,zero
+; SSE41-NEXT:    pinsrd $1, {{[0-9]+}}(%rsp), %xmm3
+; SSE41-NEXT:    movd {{.*#+}} xmm6 = mem[0],zero,zero,zero
+; SSE41-NEXT:    pinsrd $1, {{[0-9]+}}(%rsp), %xmm6
+; SSE41-NEXT:    pinsrd $2, {{[0-9]+}}(%rsp), %xmm6
+; SSE41-NEXT:    pinsrd $3, {{[0-9]+}}(%rsp), %xmm6
+; SSE41-NEXT:    movq {{[0-9]+}}(%rsp), %rcx
+; SSE41-NEXT:    pxor %xmm4, %xmm4
+; SSE41-NEXT:    pxor %xmm2, %xmm2
+; SSE41-NEXT:    pcmpgtd %xmm6, %xmm2
+; SSE41-NEXT:    pcmpeqd %xmm5, %xmm5
+; SSE41-NEXT:    pxor %xmm5, %xmm2
+; SSE41-NEXT:    pxor %xmm7, %xmm7
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm7
+; SSE41-NEXT:    pxor %xmm5, %xmm7
+; SSE41-NEXT:    pcmpeqd %xmm7, %xmm2
+; SSE41-NEXT:    psubd %xmm6, %xmm0
+; SSE41-NEXT:    pxor %xmm6, %xmm6
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm6
+; SSE41-NEXT:    pxor %xmm5, %xmm6
+; SSE41-NEXT:    pcmpeqd %xmm7, %xmm6
+; SSE41-NEXT:    pxor %xmm5, %xmm6
+; SSE41-NEXT:    pandn %xmm6, %xmm2
+; SSE41-NEXT:    pxor %xmm6, %xmm6
+; SSE41-NEXT:    pcmpgtd %xmm3, %xmm6
+; SSE41-NEXT:    pxor %xmm5, %xmm6
+; SSE41-NEXT:    pxor %xmm7, %xmm7
+; SSE41-NEXT:    pcmpgtd %xmm1, %xmm7
+; SSE41-NEXT:    pxor %xmm5, %xmm7
+; SSE41-NEXT:    pcmpeqd %xmm7, %xmm6
+; SSE41-NEXT:    psubd %xmm3, %xmm1
+; SSE41-NEXT:    pcmpgtd %xmm1, %xmm4
+; SSE41-NEXT:    pxor %xmm5, %xmm4
+; SSE41-NEXT:    pcmpeqd %xmm7, %xmm4
+; SSE41-NEXT:    pxor %xmm5, %xmm4
+; SSE41-NEXT:    pandn %xmm4, %xmm6
+; SSE41-NEXT:    movq %xmm1, 16(%rcx)
+; SSE41-NEXT:    movdqa %xmm0, (%rcx)
+; SSE41-NEXT:    movq %xmm6, 16(%rdi)
+; SSE41-NEXT:    movdqa %xmm2, (%rdi)
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: ssubo_v6i32:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm9
+; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; AVX1-NEXT:    vpcmpgtd %xmm9, %xmm3, %xmm4
+; AVX1-NEXT:    vpcmpeqd %xmm5, %xmm5, %xmm5
+; AVX1-NEXT:    vpxor %xmm5, %xmm4, %xmm4
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm6
+; AVX1-NEXT:    vpcmpgtd %xmm6, %xmm3, %xmm7
+; AVX1-NEXT:    vpxor %xmm5, %xmm7, %xmm7
+; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm7, %xmm4
+; AVX1-NEXT:    vpxor %xmm5, %xmm4, %xmm8
+; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm3, %xmm4
+; AVX1-NEXT:    vpxor %xmm5, %xmm4, %xmm4
+; AVX1-NEXT:    vpcmpgtd %xmm0, %xmm3, %xmm2
+; AVX1-NEXT:    vpxor %xmm5, %xmm2, %xmm2
+; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm2, %xmm4
+; AVX1-NEXT:    vpxor %xmm5, %xmm4, %xmm4
+; AVX1-NEXT:    vinsertf128 $1, %xmm8, %ymm4, %ymm8
+; AVX1-NEXT:    vpsubd %xmm9, %xmm6, %xmm6
+; AVX1-NEXT:    vpcmpgtd %xmm6, %xmm3, %xmm4
+; AVX1-NEXT:    vpxor %xmm5, %xmm4, %xmm4
+; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm7, %xmm4
+; AVX1-NEXT:    vpxor %xmm5, %xmm4, %xmm4
+; AVX1-NEXT:    vpsubd %xmm1, %xmm0, %xmm1
+; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm3, %xmm0
+; AVX1-NEXT:    vpxor %xmm5, %xmm0, %xmm0
+; AVX1-NEXT:    vpcmpeqd %xmm0, %xmm2, %xmm0
+; AVX1-NEXT:    vpxor %xmm5, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm0, %ymm0
+; AVX1-NEXT:    vandps %ymm0, %ymm8, %ymm0
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; AVX1-NEXT:    vpackssdw %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpmovsxwd %xmm0, %xmm2
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpmovsxwd %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm2, %ymm0
+; AVX1-NEXT:    vmovq %xmm6, 16(%rdi)
+; AVX1-NEXT:    vmovdqa %xmm1, (%rdi)
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: ssubo_v6i32:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX2-NEXT:    vpcmpgtd %ymm1, %ymm2, %ymm3
+; AVX2-NEXT:    vpcmpeqd %ymm4, %ymm4, %ymm4
+; AVX2-NEXT:    vpxor %ymm4, %ymm3, %ymm3
+; AVX2-NEXT:    vpcmpgtd %ymm0, %ymm2, %ymm5
+; AVX2-NEXT:    vpxor %ymm4, %ymm5, %ymm5
+; AVX2-NEXT:    vpcmpeqd %ymm3, %ymm5, %ymm3
+; AVX2-NEXT:    vpsubd %ymm1, %ymm0, %ymm1
+; AVX2-NEXT:    vpcmpgtd %ymm1, %ymm2, %ymm0
+; AVX2-NEXT:    vpxor %ymm4, %ymm0, %ymm0
+; AVX2-NEXT:    vpcmpeqd %ymm0, %ymm5, %ymm0
+; AVX2-NEXT:    vpxor %ymm4, %ymm0, %ymm0
+; AVX2-NEXT:    vpandn %ymm0, %ymm3, %ymm0
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm2
+; AVX2-NEXT:    vpackssdw %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    vpmovsxwd %xmm0, %ymm0
+; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm2
+; AVX2-NEXT:    vmovq %xmm2, 16(%rdi)
+; AVX2-NEXT:    vmovdqa %xmm1, (%rdi)
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: ssubo_v6i32:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX512-NEXT:    vpcmpnltd %ymm2, %ymm1, %k0
+; AVX512-NEXT:    vpcmpnltd %ymm2, %ymm0, %k1
+; AVX512-NEXT:    kxorw %k0, %k1, %k0
+; AVX512-NEXT:    vpsubd %ymm1, %ymm0, %ymm1
+; AVX512-NEXT:    vpcmpnltd %ymm2, %ymm1, %k2
+; AVX512-NEXT:    kxorw %k2, %k1, %k1
+; AVX512-NEXT:    kandw %k1, %k0, %k1
+; AVX512-NEXT:    vpcmpeqd %ymm0, %ymm0, %ymm0
+; AVX512-NEXT:    vmovdqa32 %ymm0, %ymm0 {%k1} {z}
+; AVX512-NEXT:    vextracti128 $1, %ymm1, %xmm2
+; AVX512-NEXT:    vmovq %xmm2, 16(%rdi)
+; AVX512-NEXT:    vmovdqa %xmm1, (%rdi)
+; AVX512-NEXT:    retq
+  %t = call {<6 x i32>, <6 x i1>} @llvm.ssub.with.overflow.v6i32(<6 x i32> %a0, <6 x i32> %a1)
+  %val = extractvalue {<6 x i32>, <6 x i1>} %t, 0
+  %obit = extractvalue {<6 x i32>, <6 x i1>} %t, 1
+  %res = sext <6 x i1> %obit to <6 x i32>
+  store <6 x i32> %val, <6 x i32>* %p2
+  ret <6 x i32> %res
+}
+
+define <8 x i32> @ssubo_v8i32(<8 x i32> %a0, <8 x i32> %a1, <8 x i32>* %p2) nounwind {
+; SSE-LABEL: ssubo_v8i32:
+; SSE:       # %bb.0:
+; SSE-NEXT:    pxor %xmm5, %xmm5
+; SSE-NEXT:    pxor %xmm4, %xmm4
+; SSE-NEXT:    pcmpgtd %xmm2, %xmm4
+; SSE-NEXT:    pcmpeqd %xmm6, %xmm6
+; SSE-NEXT:    pxor %xmm6, %xmm4
+; SSE-NEXT:    pxor %xmm7, %xmm7
+; SSE-NEXT:    pcmpgtd %xmm0, %xmm7
+; SSE-NEXT:    pxor %xmm6, %xmm7
+; SSE-NEXT:    pcmpeqd %xmm7, %xmm4
+; SSE-NEXT:    psubd %xmm2, %xmm0
+; SSE-NEXT:    pxor %xmm2, %xmm2
+; SSE-NEXT:    pcmpgtd %xmm0, %xmm2
+; SSE-NEXT:    pxor %xmm6, %xmm2
+; SSE-NEXT:    pcmpeqd %xmm7, %xmm2
+; SSE-NEXT:    pxor %xmm6, %xmm2
+; SSE-NEXT:    pandn %xmm2, %xmm4
+; SSE-NEXT:    pxor %xmm2, %xmm2
+; SSE-NEXT:    pcmpgtd %xmm3, %xmm2
+; SSE-NEXT:    pxor %xmm6, %xmm2
+; SSE-NEXT:    pxor %xmm7, %xmm7
+; SSE-NEXT:    pcmpgtd %xmm1, %xmm7
+; SSE-NEXT:    pxor %xmm6, %xmm7
+; SSE-NEXT:    pcmpeqd %xmm7, %xmm2
+; SSE-NEXT:    psubd %xmm3, %xmm1
+; SSE-NEXT:    pcmpgtd %xmm1, %xmm5
+; SSE-NEXT:    pxor %xmm6, %xmm5
+; SSE-NEXT:    pcmpeqd %xmm7, %xmm5
+; SSE-NEXT:    pxor %xmm6, %xmm5
+; SSE-NEXT:    pandn %xmm5, %xmm2
+; SSE-NEXT:    movdqa %xmm1, 16(%rdi)
+; SSE-NEXT:    movdqa %xmm0, (%rdi)
+; SSE-NEXT:    movdqa %xmm4, %xmm0
+; SSE-NEXT:    movdqa %xmm2, %xmm1
+; SSE-NEXT:    retq
+;
+; AVX1-LABEL: ssubo_v8i32:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm9
+; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; AVX1-NEXT:    vpcmpgtd %xmm9, %xmm3, %xmm4
+; AVX1-NEXT:    vpcmpeqd %xmm5, %xmm5, %xmm5
+; AVX1-NEXT:    vpxor %xmm5, %xmm4, %xmm4
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm6
+; AVX1-NEXT:    vpcmpgtd %xmm6, %xmm3, %xmm7
+; AVX1-NEXT:    vpxor %xmm5, %xmm7, %xmm7
+; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm7, %xmm4
+; AVX1-NEXT:    vpxor %xmm5, %xmm4, %xmm8
+; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm3, %xmm4
+; AVX1-NEXT:    vpxor %xmm5, %xmm4, %xmm4
+; AVX1-NEXT:    vpcmpgtd %xmm0, %xmm3, %xmm2
+; AVX1-NEXT:    vpxor %xmm5, %xmm2, %xmm2
+; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm2, %xmm4
+; AVX1-NEXT:    vpxor %xmm5, %xmm4, %xmm4
+; AVX1-NEXT:    vinsertf128 $1, %xmm8, %ymm4, %ymm8
+; AVX1-NEXT:    vpsubd %xmm9, %xmm6, %xmm6
+; AVX1-NEXT:    vpcmpgtd %xmm6, %xmm3, %xmm4
+; AVX1-NEXT:    vpxor %xmm5, %xmm4, %xmm4
+; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm7, %xmm4
+; AVX1-NEXT:    vpxor %xmm5, %xmm4, %xmm4
+; AVX1-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpcmpgtd %xmm0, %xmm3, %xmm1
+; AVX1-NEXT:    vpxor %xmm5, %xmm1, %xmm1
+; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm2, %xmm1
+; AVX1-NEXT:    vpxor %xmm5, %xmm1, %xmm1
+; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm1, %ymm1
+; AVX1-NEXT:    vandps %ymm1, %ymm8, %ymm1
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; AVX1-NEXT:    vpackssdw %xmm2, %xmm1, %xmm1
+; AVX1-NEXT:    vinsertf128 $1, %xmm6, %ymm0, %ymm2
+; AVX1-NEXT:    vpmovsxwd %xmm1, %xmm0
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; AVX1-NEXT:    vpmovsxwd %xmm1, %xmm1
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    vmovaps %ymm2, (%rdi)
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: ssubo_v8i32:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX2-NEXT:    vpcmpgtd %ymm1, %ymm2, %ymm3
+; AVX2-NEXT:    vpcmpeqd %ymm4, %ymm4, %ymm4
+; AVX2-NEXT:    vpxor %ymm4, %ymm3, %ymm3
+; AVX2-NEXT:    vpcmpgtd %ymm0, %ymm2, %ymm5
+; AVX2-NEXT:    vpxor %ymm4, %ymm5, %ymm5
+; AVX2-NEXT:    vpcmpeqd %ymm3, %ymm5, %ymm3
+; AVX2-NEXT:    vpsubd %ymm1, %ymm0, %ymm1
+; AVX2-NEXT:    vpcmpgtd %ymm1, %ymm2, %ymm0
+; AVX2-NEXT:    vpxor %ymm4, %ymm0, %ymm0
+; AVX2-NEXT:    vpcmpeqd %ymm0, %ymm5, %ymm0
+; AVX2-NEXT:    vpxor %ymm4, %ymm0, %ymm0
+; AVX2-NEXT:    vpandn %ymm0, %ymm3, %ymm0
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm2
+; AVX2-NEXT:    vpackssdw %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    vpmovsxwd %xmm0, %ymm0
+; AVX2-NEXT:    vmovdqa %ymm1, (%rdi)
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: ssubo_v8i32:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX512-NEXT:    vpcmpnltd %ymm2, %ymm1, %k0
+; AVX512-NEXT:    vpcmpnltd %ymm2, %ymm0, %k1
+; AVX512-NEXT:    kxorw %k0, %k1, %k0
+; AVX512-NEXT:    vpsubd %ymm1, %ymm0, %ymm1
+; AVX512-NEXT:    vpcmpnltd %ymm2, %ymm1, %k2
+; AVX512-NEXT:    kxorw %k2, %k1, %k1
+; AVX512-NEXT:    kandw %k1, %k0, %k1
+; AVX512-NEXT:    vpcmpeqd %ymm0, %ymm0, %ymm0
+; AVX512-NEXT:    vmovdqa32 %ymm0, %ymm0 {%k1} {z}
+; AVX512-NEXT:    vmovdqa %ymm1, (%rdi)
+; AVX512-NEXT:    retq
+  %t = call {<8 x i32>, <8 x i1>} @llvm.ssub.with.overflow.v8i32(<8 x i32> %a0, <8 x i32> %a1)
+  %val = extractvalue {<8 x i32>, <8 x i1>} %t, 0
+  %obit = extractvalue {<8 x i32>, <8 x i1>} %t, 1
+  %res = sext <8 x i1> %obit to <8 x i32>
+  store <8 x i32> %val, <8 x i32>* %p2
+  ret <8 x i32> %res
+}
+
+define <16 x i32> @ssubo_v16i32(<16 x i32> %a0, <16 x i32> %a1, <16 x i32>* %p2) nounwind {
+; SSE-LABEL: ssubo_v16i32:
+; SSE:       # %bb.0:
+; SSE-NEXT:    pxor %xmm10, %xmm10
+; SSE-NEXT:    pxor %xmm8, %xmm8
+; SSE-NEXT:    pcmpgtd %xmm4, %xmm8
+; SSE-NEXT:    pcmpeqd %xmm11, %xmm11
+; SSE-NEXT:    pxor %xmm11, %xmm8
+; SSE-NEXT:    pxor %xmm9, %xmm9
+; SSE-NEXT:    pcmpgtd %xmm0, %xmm9
+; SSE-NEXT:    pxor %xmm11, %xmm9
+; SSE-NEXT:    pcmpeqd %xmm9, %xmm8
+; SSE-NEXT:    psubd %xmm4, %xmm0
+; SSE-NEXT:    pxor %xmm4, %xmm4
+; SSE-NEXT:    pcmpgtd %xmm0, %xmm4
+; SSE-NEXT:    pxor %xmm11, %xmm4
+; SSE-NEXT:    pcmpeqd %xmm9, %xmm4
+; SSE-NEXT:    pxor %xmm11, %xmm4
+; SSE-NEXT:    pandn %xmm4, %xmm8
+; SSE-NEXT:    pxor %xmm9, %xmm9
+; SSE-NEXT:    pcmpgtd %xmm5, %xmm9
+; SSE-NEXT:    pxor %xmm11, %xmm9
+; SSE-NEXT:    pxor %xmm4, %xmm4
+; SSE-NEXT:    pcmpgtd %xmm1, %xmm4
+; SSE-NEXT:    pxor %xmm11, %xmm4
+; SSE-NEXT:    pcmpeqd %xmm4, %xmm9
+; SSE-NEXT:    psubd %xmm5, %xmm1
+; SSE-NEXT:    pxor %xmm5, %xmm5
+; SSE-NEXT:    pcmpgtd %xmm1, %xmm5
+; SSE-NEXT:    pxor %xmm11, %xmm5
+; SSE-NEXT:    pcmpeqd %xmm4, %xmm5
+; SSE-NEXT:    pxor %xmm11, %xmm5
+; SSE-NEXT:    pandn %xmm5, %xmm9
+; SSE-NEXT:    pxor %xmm4, %xmm4
+; SSE-NEXT:    pcmpgtd %xmm6, %xmm4
+; SSE-NEXT:    pxor %xmm11, %xmm4
+; SSE-NEXT:    pxor %xmm5, %xmm5
+; SSE-NEXT:    pcmpgtd %xmm2, %xmm5
+; SSE-NEXT:    pxor %xmm11, %xmm5
+; SSE-NEXT:    pcmpeqd %xmm5, %xmm4
+; SSE-NEXT:    psubd %xmm6, %xmm2
+; SSE-NEXT:    pxor %xmm6, %xmm6
+; SSE-NEXT:    pcmpgtd %xmm2, %xmm6
+; SSE-NEXT:    pxor %xmm11, %xmm6
+; SSE-NEXT:    pcmpeqd %xmm5, %xmm6
+; SSE-NEXT:    pxor %xmm11, %xmm6
+; SSE-NEXT:    pandn %xmm6, %xmm4
+; SSE-NEXT:    pxor %xmm5, %xmm5
+; SSE-NEXT:    pcmpgtd %xmm7, %xmm5
+; SSE-NEXT:    pxor %xmm11, %xmm5
+; SSE-NEXT:    pxor %xmm6, %xmm6
+; SSE-NEXT:    pcmpgtd %xmm3, %xmm6
+; SSE-NEXT:    pxor %xmm11, %xmm6
+; SSE-NEXT:    pcmpeqd %xmm6, %xmm5
+; SSE-NEXT:    psubd %xmm7, %xmm3
+; SSE-NEXT:    pcmpgtd %xmm3, %xmm10
+; SSE-NEXT:    pxor %xmm11, %xmm10
+; SSE-NEXT:    pcmpeqd %xmm6, %xmm10
+; SSE-NEXT:    pxor %xmm11, %xmm10
+; SSE-NEXT:    pandn %xmm10, %xmm5
+; SSE-NEXT:    movdqa %xmm3, 48(%rdi)
+; SSE-NEXT:    movdqa %xmm2, 32(%rdi)
+; SSE-NEXT:    movdqa %xmm1, 16(%rdi)
+; SSE-NEXT:    movdqa %xmm0, (%rdi)
+; SSE-NEXT:    movdqa %xmm8, %xmm0
+; SSE-NEXT:    movdqa %xmm9, %xmm1
+; SSE-NEXT:    movdqa %xmm4, %xmm2
+; SSE-NEXT:    movdqa %xmm5, %xmm3
+; SSE-NEXT:    retq
+;
+; AVX1-LABEL: ssubo_v16i32:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm8
+; AVX1-NEXT:    vpxor %xmm5, %xmm5, %xmm5
+; AVX1-NEXT:    vpcmpgtd %xmm8, %xmm5, %xmm7
+; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
+; AVX1-NEXT:    vpxor %xmm4, %xmm7, %xmm9
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm12
+; AVX1-NEXT:    vpcmpgtd %xmm12, %xmm5, %xmm7
+; AVX1-NEXT:    vpxor %xmm4, %xmm7, %xmm10
+; AVX1-NEXT:    vpcmpeqd %xmm9, %xmm10, %xmm7
+; AVX1-NEXT:    vpxor %xmm4, %xmm7, %xmm9
+; AVX1-NEXT:    vpcmpgtd %xmm3, %xmm5, %xmm7
+; AVX1-NEXT:    vpxor %xmm4, %xmm7, %xmm11
+; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm5, %xmm7
+; AVX1-NEXT:    vpxor %xmm4, %xmm7, %xmm7
+; AVX1-NEXT:    vpcmpeqd %xmm11, %xmm7, %xmm6
+; AVX1-NEXT:    vpxor %xmm4, %xmm6, %xmm6
+; AVX1-NEXT:    vinsertf128 $1, %xmm9, %ymm6, %ymm9
+; AVX1-NEXT:    vpsubd %xmm8, %xmm12, %xmm8
+; AVX1-NEXT:    vpcmpgtd %xmm8, %xmm5, %xmm6
+; AVX1-NEXT:    vpxor %xmm4, %xmm6, %xmm6
+; AVX1-NEXT:    vpcmpeqd %xmm6, %xmm10, %xmm6
+; AVX1-NEXT:    vpxor %xmm4, %xmm6, %xmm6
+; AVX1-NEXT:    vpsubd %xmm3, %xmm1, %xmm10
+; AVX1-NEXT:    vpcmpgtd %xmm10, %xmm5, %xmm3
+; AVX1-NEXT:    vpxor %xmm4, %xmm3, %xmm3
+; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm7, %xmm3
+; AVX1-NEXT:    vpxor %xmm4, %xmm3, %xmm3
+; AVX1-NEXT:    vinsertf128 $1, %xmm6, %ymm3, %ymm3
+; AVX1-NEXT:    vandps %ymm3, %ymm9, %ymm3
+; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm6
+; AVX1-NEXT:    vpackssdw %xmm6, %xmm3, %xmm9
+; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm13
+; AVX1-NEXT:    vpcmpgtd %xmm13, %xmm5, %xmm7
+; AVX1-NEXT:    vpxor %xmm4, %xmm7, %xmm7
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm5, %xmm3
+; AVX1-NEXT:    vpxor %xmm4, %xmm3, %xmm3
+; AVX1-NEXT:    vpcmpeqd %xmm7, %xmm3, %xmm7
+; AVX1-NEXT:    vpxor %xmm4, %xmm7, %xmm11
+; AVX1-NEXT:    vpcmpgtd %xmm2, %xmm5, %xmm7
+; AVX1-NEXT:    vpxor %xmm4, %xmm7, %xmm12
+; AVX1-NEXT:    vpcmpgtd %xmm0, %xmm5, %xmm7
+; AVX1-NEXT:    vpxor %xmm4, %xmm7, %xmm7
+; AVX1-NEXT:    vpcmpeqd %xmm12, %xmm7, %xmm6
+; AVX1-NEXT:    vpxor %xmm4, %xmm6, %xmm6
+; AVX1-NEXT:    vinsertf128 $1, %xmm11, %ymm6, %ymm11
+; AVX1-NEXT:    vpsubd %xmm13, %xmm1, %xmm1
+; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm5, %xmm6
+; AVX1-NEXT:    vpxor %xmm4, %xmm6, %xmm6
+; AVX1-NEXT:    vpcmpeqd %xmm6, %xmm3, %xmm3
+; AVX1-NEXT:    vpxor %xmm4, %xmm3, %xmm3
+; AVX1-NEXT:    vpsubd %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpcmpgtd %xmm0, %xmm5, %xmm2
+; AVX1-NEXT:    vpxor %xmm4, %xmm2, %xmm2
+; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm7, %xmm2
+; AVX1-NEXT:    vpxor %xmm4, %xmm2, %xmm2
+; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm2, %ymm2
+; AVX1-NEXT:    vandps %ymm2, %ymm11, %ymm2
+; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm3
+; AVX1-NEXT:    vpackssdw %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm3
+; AVX1-NEXT:    vinsertf128 $1, %xmm8, %ymm10, %ymm4
+; AVX1-NEXT:    vpmovsxwd %xmm2, %xmm0
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm2[2,3,0,1]
+; AVX1-NEXT:    vpmovsxwd %xmm1, %xmm1
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    vpmovsxwd %xmm9, %xmm1
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm9[2,3,0,1]
+; AVX1-NEXT:    vpmovsxwd %xmm2, %xmm2
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
+; AVX1-NEXT:    vmovaps %ymm4, 32(%rdi)
+; AVX1-NEXT:    vmovaps %ymm3, (%rdi)
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: ssubo_v16i32:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpxor %xmm4, %xmm4, %xmm4
+; AVX2-NEXT:    vpcmpgtd %ymm3, %ymm4, %ymm5
+; AVX2-NEXT:    vpcmpeqd %ymm6, %ymm6, %ymm6
+; AVX2-NEXT:    vpxor %ymm6, %ymm5, %ymm5
+; AVX2-NEXT:    vpcmpgtd %ymm1, %ymm4, %ymm7
+; AVX2-NEXT:    vpxor %ymm6, %ymm7, %ymm7
+; AVX2-NEXT:    vpcmpeqd %ymm5, %ymm7, %ymm5
+; AVX2-NEXT:    vpsubd %ymm3, %ymm1, %ymm3
+; AVX2-NEXT:    vpcmpgtd %ymm3, %ymm4, %ymm1
+; AVX2-NEXT:    vpxor %ymm6, %ymm1, %ymm1
+; AVX2-NEXT:    vpcmpeqd %ymm1, %ymm7, %ymm1
+; AVX2-NEXT:    vpxor %ymm6, %ymm1, %ymm1
+; AVX2-NEXT:    vpandn %ymm1, %ymm5, %ymm1
+; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm5
+; AVX2-NEXT:    vpackssdw %xmm5, %xmm1, %xmm1
+; AVX2-NEXT:    vpcmpgtd %ymm2, %ymm4, %ymm5
+; AVX2-NEXT:    vpxor %ymm6, %ymm5, %ymm5
+; AVX2-NEXT:    vpcmpgtd %ymm0, %ymm4, %ymm7
+; AVX2-NEXT:    vpxor %ymm6, %ymm7, %ymm7
+; AVX2-NEXT:    vpcmpeqd %ymm5, %ymm7, %ymm5
+; AVX2-NEXT:    vpsubd %ymm2, %ymm0, %ymm2
+; AVX2-NEXT:    vpcmpgtd %ymm2, %ymm4, %ymm0
+; AVX2-NEXT:    vpxor %ymm6, %ymm0, %ymm0
+; AVX2-NEXT:    vpcmpeqd %ymm0, %ymm7, %ymm0
+; AVX2-NEXT:    vpxor %ymm6, %ymm0, %ymm0
+; AVX2-NEXT:    vpandn %ymm0, %ymm5, %ymm0
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm4
+; AVX2-NEXT:    vpackssdw %xmm4, %xmm0, %xmm0
+; AVX2-NEXT:    vpmovsxwd %xmm0, %ymm0
+; AVX2-NEXT:    vpmovsxwd %xmm1, %ymm1
+; AVX2-NEXT:    vmovdqa %ymm3, 32(%rdi)
+; AVX2-NEXT:    vmovdqa %ymm2, (%rdi)
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: ssubo_v16i32:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX512-NEXT:    vpcmpnltd %zmm2, %zmm1, %k0
+; AVX512-NEXT:    vpcmpnltd %zmm2, %zmm0, %k1
+; AVX512-NEXT:    kxorw %k0, %k1, %k0
+; AVX512-NEXT:    vpsubd %zmm1, %zmm0, %zmm1
+; AVX512-NEXT:    vpcmpnltd %zmm2, %zmm1, %k2
+; AVX512-NEXT:    kxorw %k2, %k1, %k1
+; AVX512-NEXT:    kandw %k1, %k0, %k1
+; AVX512-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; AVX512-NEXT:    vmovdqa64 %zmm1, (%rdi)
+; AVX512-NEXT:    retq
+  %t = call {<16 x i32>, <16 x i1>} @llvm.ssub.with.overflow.v16i32(<16 x i32> %a0, <16 x i32> %a1)
+  %val = extractvalue {<16 x i32>, <16 x i1>} %t, 0
+  %obit = extractvalue {<16 x i32>, <16 x i1>} %t, 1
+  %res = sext <16 x i1> %obit to <16 x i32>
+  store <16 x i32> %val, <16 x i32>* %p2
+  ret <16 x i32> %res
+}
+
+define <16 x i32> @ssubo_v16i8(<16 x i8> %a0, <16 x i8> %a1, <16 x i8>* %p2) nounwind {
+; SSE2-LABEL: ssubo_v16i8:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    pxor %xmm2, %xmm2
+; SSE2-NEXT:    pxor %xmm3, %xmm3
+; SSE2-NEXT:    pcmpgtb %xmm1, %xmm3
+; SSE2-NEXT:    pcmpeqd %xmm4, %xmm4
+; SSE2-NEXT:    pxor %xmm4, %xmm3
+; SSE2-NEXT:    pxor %xmm5, %xmm5
+; SSE2-NEXT:    pcmpgtb %xmm0, %xmm5
+; SSE2-NEXT:    pxor %xmm4, %xmm5
+; SSE2-NEXT:    pcmpeqb %xmm5, %xmm3
+; SSE2-NEXT:    psubb %xmm1, %xmm0
+; SSE2-NEXT:    pcmpgtb %xmm0, %xmm2
+; SSE2-NEXT:    pxor %xmm4, %xmm2
+; SSE2-NEXT:    pcmpeqb %xmm5, %xmm2
+; SSE2-NEXT:    pxor %xmm4, %xmm2
+; SSE2-NEXT:    pandn %xmm2, %xmm3
+; SSE2-NEXT:    movdqa %xmm3, %xmm1
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSE2-NEXT:    movdqa %xmm1, %xmm4
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3]
+; SSE2-NEXT:    pslld $31, %xmm4
+; SSE2-NEXT:    psrad $31, %xmm4
+; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSE2-NEXT:    pslld $31, %xmm1
+; SSE2-NEXT:    psrad $31, %xmm1
+; SSE2-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm0[8],xmm3[9],xmm0[9],xmm3[10],xmm0[10],xmm3[11],xmm0[11],xmm3[12],xmm0[12],xmm3[13],xmm0[13],xmm3[14],xmm0[14],xmm3[15],xmm0[15]
+; SSE2-NEXT:    movdqa %xmm3, %xmm2
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
+; SSE2-NEXT:    pslld $31, %xmm2
+; SSE2-NEXT:    psrad $31, %xmm2
+; SSE2-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
+; SSE2-NEXT:    pslld $31, %xmm3
+; SSE2-NEXT:    psrad $31, %xmm3
+; SSE2-NEXT:    movdqa %xmm0, (%rdi)
+; SSE2-NEXT:    movdqa %xmm4, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: ssubo_v16i8:
+; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    pxor %xmm2, %xmm2
+; SSSE3-NEXT:    pxor %xmm3, %xmm3
+; SSSE3-NEXT:    pcmpgtb %xmm1, %xmm3
+; SSSE3-NEXT:    pcmpeqd %xmm4, %xmm4
+; SSSE3-NEXT:    pxor %xmm4, %xmm3
+; SSSE3-NEXT:    pxor %xmm5, %xmm5
+; SSSE3-NEXT:    pcmpgtb %xmm0, %xmm5
+; SSSE3-NEXT:    pxor %xmm4, %xmm5
+; SSSE3-NEXT:    pcmpeqb %xmm5, %xmm3
+; SSSE3-NEXT:    psubb %xmm1, %xmm0
+; SSSE3-NEXT:    pcmpgtb %xmm0, %xmm2
+; SSSE3-NEXT:    pxor %xmm4, %xmm2
+; SSSE3-NEXT:    pcmpeqb %xmm5, %xmm2
+; SSSE3-NEXT:    pxor %xmm4, %xmm2
+; SSSE3-NEXT:    pandn %xmm2, %xmm3
+; SSSE3-NEXT:    movdqa %xmm3, %xmm1
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSSE3-NEXT:    movdqa %xmm1, %xmm4
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3]
+; SSSE3-NEXT:    pslld $31, %xmm4
+; SSSE3-NEXT:    psrad $31, %xmm4
+; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSSE3-NEXT:    pslld $31, %xmm1
+; SSSE3-NEXT:    psrad $31, %xmm1
+; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm0[8],xmm3[9],xmm0[9],xmm3[10],xmm0[10],xmm3[11],xmm0[11],xmm3[12],xmm0[12],xmm3[13],xmm0[13],xmm3[14],xmm0[14],xmm3[15],xmm0[15]
+; SSSE3-NEXT:    movdqa %xmm3, %xmm2
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
+; SSSE3-NEXT:    pslld $31, %xmm2
+; SSSE3-NEXT:    psrad $31, %xmm2
+; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
+; SSSE3-NEXT:    pslld $31, %xmm3
+; SSSE3-NEXT:    psrad $31, %xmm3
+; SSSE3-NEXT:    movdqa %xmm0, (%rdi)
+; SSSE3-NEXT:    movdqa %xmm4, %xmm0
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: ssubo_v16i8:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    pxor %xmm2, %xmm2
+; SSE41-NEXT:    pxor %xmm3, %xmm3
+; SSE41-NEXT:    pcmpgtb %xmm1, %xmm3
+; SSE41-NEXT:    pcmpeqd %xmm4, %xmm4
+; SSE41-NEXT:    pxor %xmm4, %xmm3
+; SSE41-NEXT:    pxor %xmm5, %xmm5
+; SSE41-NEXT:    pcmpgtb %xmm0, %xmm5
+; SSE41-NEXT:    pxor %xmm4, %xmm5
+; SSE41-NEXT:    pcmpeqb %xmm5, %xmm3
+; SSE41-NEXT:    psubb %xmm1, %xmm0
+; SSE41-NEXT:    pcmpgtb %xmm0, %xmm2
+; SSE41-NEXT:    pxor %xmm4, %xmm2
+; SSE41-NEXT:    pcmpeqb %xmm5, %xmm2
+; SSE41-NEXT:    pxor %xmm4, %xmm2
+; SSE41-NEXT:    pandn %xmm2, %xmm3
+; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm4 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
+; SSE41-NEXT:    pslld $31, %xmm4
+; SSE41-NEXT:    psrad $31, %xmm4
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,2,3]
+; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
+; SSE41-NEXT:    pslld $31, %xmm1
+; SSE41-NEXT:    psrad $31, %xmm1
+; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[2,3,0,1]
+; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero
+; SSE41-NEXT:    pslld $31, %xmm2
+; SSE41-NEXT:    psrad $31, %xmm2
+; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[3,1,2,3]
+; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
+; SSE41-NEXT:    pslld $31, %xmm3
+; SSE41-NEXT:    psrad $31, %xmm3
+; SSE41-NEXT:    movdqa %xmm0, (%rdi)
+; SSE41-NEXT:    movdqa %xmm4, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: ssubo_v16i8:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX1-NEXT:    vpcmpgtb %xmm1, %xmm2, %xmm3
+; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
+; AVX1-NEXT:    vpxor %xmm4, %xmm3, %xmm3
+; AVX1-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm5
+; AVX1-NEXT:    vpxor %xmm4, %xmm5, %xmm5
+; AVX1-NEXT:    vpcmpeqb %xmm3, %xmm5, %xmm3
+; AVX1-NEXT:    vpsubb %xmm1, %xmm0, %xmm6
+; AVX1-NEXT:    vpcmpgtb %xmm6, %xmm2, %xmm0
+; AVX1-NEXT:    vpxor %xmm4, %xmm0, %xmm0
+; AVX1-NEXT:    vpcmpeqb %xmm0, %xmm5, %xmm0
+; AVX1-NEXT:    vpxor %xmm4, %xmm0, %xmm0
+; AVX1-NEXT:    vpandn %xmm0, %xmm3, %xmm1
+; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; AVX1-NEXT:    vpmovzxwd {{.*#+}} xmm2 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; AVX1-NEXT:    vpslld $31, %xmm2, %xmm2
+; AVX1-NEXT:    vpsrad $31, %xmm2, %xmm2
+; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm0[4,4,5,5,6,6,7,7]
+; AVX1-NEXT:    vpslld $31, %xmm0, %xmm0
+; AVX1-NEXT:    vpsrad $31, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm2, %ymm0
+; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
+; AVX1-NEXT:    vpmovzxwd {{.*#+}} xmm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
+; AVX1-NEXT:    vpslld $31, %xmm2, %xmm2
+; AVX1-NEXT:    vpsrad $31, %xmm2, %xmm2
+; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; AVX1-NEXT:    vpslld $31, %xmm1, %xmm1
+; AVX1-NEXT:    vpsrad $31, %xmm1, %xmm1
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm1
+; AVX1-NEXT:    vmovdqa %xmm6, (%rdi)
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: ssubo_v16i8:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX2-NEXT:    vpcmpgtb %xmm1, %xmm2, %xmm3
+; AVX2-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
+; AVX2-NEXT:    vpxor %xmm4, %xmm3, %xmm3
+; AVX2-NEXT:    vpcmpgtb %xmm0, %xmm2, %xmm5
+; AVX2-NEXT:    vpxor %xmm4, %xmm5, %xmm5
+; AVX2-NEXT:    vpcmpeqb %xmm3, %xmm5, %xmm3
+; AVX2-NEXT:    vpsubb %xmm1, %xmm0, %xmm6
+; AVX2-NEXT:    vpcmpgtb %xmm6, %xmm2, %xmm0
+; AVX2-NEXT:    vpxor %xmm4, %xmm0, %xmm0
+; AVX2-NEXT:    vpcmpeqb %xmm0, %xmm5, %xmm0
+; AVX2-NEXT:    vpxor %xmm4, %xmm0, %xmm0
+; AVX2-NEXT:    vpandn %xmm0, %xmm3, %xmm1
+; AVX2-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX2-NEXT:    vpslld $31, %ymm0, %ymm0
+; AVX2-NEXT:    vpsrad $31, %ymm0, %ymm0
+; AVX2-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
+; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; AVX2-NEXT:    vpslld $31, %ymm1, %ymm1
+; AVX2-NEXT:    vpsrad $31, %ymm1, %ymm1
+; AVX2-NEXT:    vmovdqa %xmm6, (%rdi)
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: ssubo_v16i8:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX512-NEXT:    vpcmpnltb %xmm2, %xmm1, %k0
+; AVX512-NEXT:    vpcmpnltb %xmm2, %xmm0, %k1
+; AVX512-NEXT:    kxorw %k0, %k1, %k0
+; AVX512-NEXT:    vpsubb %xmm1, %xmm0, %xmm1
+; AVX512-NEXT:    vpcmpnltb %xmm2, %xmm1, %k2
+; AVX512-NEXT:    kxorw %k2, %k1, %k1
+; AVX512-NEXT:    kandw %k1, %k0, %k1
+; AVX512-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; AVX512-NEXT:    vmovdqa %xmm1, (%rdi)
+; AVX512-NEXT:    retq
+  %t = call {<16 x i8>, <16 x i1>} @llvm.ssub.with.overflow.v16i8(<16 x i8> %a0, <16 x i8> %a1)
+  %val = extractvalue {<16 x i8>, <16 x i1>} %t, 0
+  %obit = extractvalue {<16 x i8>, <16 x i1>} %t, 1
+  %res = sext <16 x i1> %obit to <16 x i32>
+  store <16 x i8> %val, <16 x i8>* %p2
+  ret <16 x i32> %res
+}
+
+define <8 x i32> @ssubo_v8i16(<8 x i16> %a0, <8 x i16> %a1, <8 x i16>* %p2) nounwind {
+; SSE2-LABEL: ssubo_v8i16:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa %xmm1, %xmm2
+; SSE2-NEXT:    pxor %xmm3, %xmm3
+; SSE2-NEXT:    pxor %xmm1, %xmm1
+; SSE2-NEXT:    pcmpgtw %xmm2, %xmm1
+; SSE2-NEXT:    pcmpeqd %xmm4, %xmm4
+; SSE2-NEXT:    pxor %xmm4, %xmm1
+; SSE2-NEXT:    pxor %xmm5, %xmm5
+; SSE2-NEXT:    pcmpgtw %xmm0, %xmm5
+; SSE2-NEXT:    pxor %xmm4, %xmm5
+; SSE2-NEXT:    pcmpeqw %xmm5, %xmm1
+; SSE2-NEXT:    psubw %xmm2, %xmm0
+; SSE2-NEXT:    pcmpgtw %xmm0, %xmm3
+; SSE2-NEXT:    pxor %xmm4, %xmm3
+; SSE2-NEXT:    pcmpeqw %xmm5, %xmm3
+; SSE2-NEXT:    pxor %xmm4, %xmm3
+; SSE2-NEXT:    pandn %xmm3, %xmm1
+; SSE2-NEXT:    movdqa %xmm1, %xmm2
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
+; SSE2-NEXT:    pslld $31, %xmm2
+; SSE2-NEXT:    psrad $31, %xmm2
+; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSE2-NEXT:    pslld $31, %xmm1
+; SSE2-NEXT:    psrad $31, %xmm1
+; SSE2-NEXT:    movdqa %xmm0, (%rdi)
+; SSE2-NEXT:    movdqa %xmm2, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: ssubo_v8i16:
+; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    movdqa %xmm1, %xmm2
+; SSSE3-NEXT:    pxor %xmm3, %xmm3
+; SSSE3-NEXT:    pxor %xmm1, %xmm1
+; SSSE3-NEXT:    pcmpgtw %xmm2, %xmm1
+; SSSE3-NEXT:    pcmpeqd %xmm4, %xmm4
+; SSSE3-NEXT:    pxor %xmm4, %xmm1
+; SSSE3-NEXT:    pxor %xmm5, %xmm5
+; SSSE3-NEXT:    pcmpgtw %xmm0, %xmm5
+; SSSE3-NEXT:    pxor %xmm4, %xmm5
+; SSSE3-NEXT:    pcmpeqw %xmm5, %xmm1
+; SSSE3-NEXT:    psubw %xmm2, %xmm0
+; SSSE3-NEXT:    pcmpgtw %xmm0, %xmm3
+; SSSE3-NEXT:    pxor %xmm4, %xmm3
+; SSSE3-NEXT:    pcmpeqw %xmm5, %xmm3
+; SSSE3-NEXT:    pxor %xmm4, %xmm3
+; SSSE3-NEXT:    pandn %xmm3, %xmm1
+; SSSE3-NEXT:    movdqa %xmm1, %xmm2
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
+; SSSE3-NEXT:    pslld $31, %xmm2
+; SSSE3-NEXT:    psrad $31, %xmm2
+; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSSE3-NEXT:    pslld $31, %xmm1
+; SSSE3-NEXT:    psrad $31, %xmm1
+; SSSE3-NEXT:    movdqa %xmm0, (%rdi)
+; SSSE3-NEXT:    movdqa %xmm2, %xmm0
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: ssubo_v8i16:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movdqa %xmm1, %xmm2
+; SSE41-NEXT:    pxor %xmm3, %xmm3
+; SSE41-NEXT:    pxor %xmm1, %xmm1
+; SSE41-NEXT:    pcmpgtw %xmm2, %xmm1
+; SSE41-NEXT:    pcmpeqd %xmm4, %xmm4
+; SSE41-NEXT:    pxor %xmm4, %xmm1
+; SSE41-NEXT:    pxor %xmm5, %xmm5
+; SSE41-NEXT:    pcmpgtw %xmm0, %xmm5
+; SSE41-NEXT:    pxor %xmm4, %xmm5
+; SSE41-NEXT:    pcmpeqw %xmm5, %xmm1
+; SSE41-NEXT:    psubw %xmm2, %xmm0
+; SSE41-NEXT:    pcmpgtw %xmm0, %xmm3
+; SSE41-NEXT:    pxor %xmm4, %xmm3
+; SSE41-NEXT:    pcmpeqw %xmm5, %xmm3
+; SSE41-NEXT:    pxor %xmm4, %xmm3
+; SSE41-NEXT:    pandn %xmm3, %xmm1
+; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
+; SSE41-NEXT:    pslld $31, %xmm2
+; SSE41-NEXT:    psrad $31, %xmm2
+; SSE41-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSE41-NEXT:    pslld $31, %xmm1
+; SSE41-NEXT:    psrad $31, %xmm1
+; SSE41-NEXT:    movdqa %xmm0, (%rdi)
+; SSE41-NEXT:    movdqa %xmm2, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: ssubo_v8i16:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX1-NEXT:    vpcmpgtw %xmm1, %xmm2, %xmm3
+; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
+; AVX1-NEXT:    vpxor %xmm4, %xmm3, %xmm3
+; AVX1-NEXT:    vpcmpgtw %xmm0, %xmm2, %xmm5
+; AVX1-NEXT:    vpxor %xmm4, %xmm5, %xmm5
+; AVX1-NEXT:    vpcmpeqw %xmm3, %xmm5, %xmm3
+; AVX1-NEXT:    vpsubw %xmm1, %xmm0, %xmm1
+; AVX1-NEXT:    vpcmpgtw %xmm1, %xmm2, %xmm0
+; AVX1-NEXT:    vpxor %xmm4, %xmm0, %xmm0
+; AVX1-NEXT:    vpcmpeqw %xmm0, %xmm5, %xmm0
+; AVX1-NEXT:    vpxor %xmm4, %xmm0, %xmm0
+; AVX1-NEXT:    vpandn %xmm0, %xmm3, %xmm0
+; AVX1-NEXT:    vpmovsxwd %xmm0, %xmm2
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpmovsxwd %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm2, %ymm0
+; AVX1-NEXT:    vmovdqa %xmm1, (%rdi)
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: ssubo_v8i16:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX2-NEXT:    vpcmpgtw %xmm1, %xmm2, %xmm3
+; AVX2-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
+; AVX2-NEXT:    vpxor %xmm4, %xmm3, %xmm3
+; AVX2-NEXT:    vpcmpgtw %xmm0, %xmm2, %xmm5
+; AVX2-NEXT:    vpxor %xmm4, %xmm5, %xmm5
+; AVX2-NEXT:    vpcmpeqw %xmm3, %xmm5, %xmm3
+; AVX2-NEXT:    vpsubw %xmm1, %xmm0, %xmm1
+; AVX2-NEXT:    vpcmpgtw %xmm1, %xmm2, %xmm0
+; AVX2-NEXT:    vpxor %xmm4, %xmm0, %xmm0
+; AVX2-NEXT:    vpcmpeqw %xmm0, %xmm5, %xmm0
+; AVX2-NEXT:    vpxor %xmm4, %xmm0, %xmm0
+; AVX2-NEXT:    vpandn %xmm0, %xmm3, %xmm0
+; AVX2-NEXT:    vpmovsxwd %xmm0, %ymm0
+; AVX2-NEXT:    vmovdqa %xmm1, (%rdi)
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: ssubo_v8i16:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX512-NEXT:    vpcmpnltw %xmm2, %xmm1, %k0
+; AVX512-NEXT:    vpcmpnltw %xmm2, %xmm0, %k1
+; AVX512-NEXT:    kxorw %k0, %k1, %k0
+; AVX512-NEXT:    vpsubw %xmm1, %xmm0, %xmm1
+; AVX512-NEXT:    vpcmpnltw %xmm2, %xmm1, %k2
+; AVX512-NEXT:    kxorw %k2, %k1, %k1
+; AVX512-NEXT:    kandw %k1, %k0, %k1
+; AVX512-NEXT:    vpcmpeqd %ymm0, %ymm0, %ymm0
+; AVX512-NEXT:    vmovdqa32 %ymm0, %ymm0 {%k1} {z}
+; AVX512-NEXT:    vmovdqa %xmm1, (%rdi)
+; AVX512-NEXT:    retq
+  %t = call {<8 x i16>, <8 x i1>} @llvm.ssub.with.overflow.v8i16(<8 x i16> %a0, <8 x i16> %a1)
+  %val = extractvalue {<8 x i16>, <8 x i1>} %t, 0
+  %obit = extractvalue {<8 x i16>, <8 x i1>} %t, 1
+  %res = sext <8 x i1> %obit to <8 x i32>
+  store <8 x i16> %val, <8 x i16>* %p2
+  ret <8 x i32> %res
+}
+
+define <2 x i32> @ssubo_v2i64(<2 x i64> %a0, <2 x i64> %a1, <2 x i64>* %p2) nounwind {
+; SSE2-LABEL: ssubo_v2i64:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648]
+; SSE2-NEXT:    movdqa %xmm0, %xmm3
+; SSE2-NEXT:    psubq %xmm1, %xmm0
+; SSE2-NEXT:    pxor %xmm2, %xmm1
+; SSE2-NEXT:    movdqa %xmm2, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm1, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm2, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSE2-NEXT:    pand %xmm5, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSE2-NEXT:    por %xmm1, %xmm4
+; SSE2-NEXT:    pcmpeqd %xmm5, %xmm5
+; SSE2-NEXT:    pxor %xmm5, %xmm4
+; SSE2-NEXT:    pxor %xmm2, %xmm3
+; SSE2-NEXT:    movdqa %xmm2, %xmm1
+; SSE2-NEXT:    pcmpgtd %xmm3, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm1[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm2, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
+; SSE2-NEXT:    pand %xmm6, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm1[1,1,3,3]
+; SSE2-NEXT:    por %xmm3, %xmm6
+; SSE2-NEXT:    pxor %xmm5, %xmm6
+; SSE2-NEXT:    pcmpeqd %xmm6, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm4[1,0,3,2]
+; SSE2-NEXT:    pand %xmm4, %xmm1
+; SSE2-NEXT:    movdqa %xmm0, (%rdi)
+; SSE2-NEXT:    pxor %xmm2, %xmm0
+; SSE2-NEXT:    movdqa %xmm2, %xmm3
+; SSE2-NEXT:    pcmpgtd %xmm0, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm2, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE2-NEXT:    pand %xmm4, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[1,1,3,3]
+; SSE2-NEXT:    por %xmm0, %xmm2
+; SSE2-NEXT:    pxor %xmm5, %xmm2
+; SSE2-NEXT:    pcmpeqd %xmm6, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,0,3,2]
+; SSE2-NEXT:    pand %xmm2, %xmm0
+; SSE2-NEXT:    pxor %xmm5, %xmm0
+; SSE2-NEXT:    pandn %xmm0, %xmm1
+; SSE2-NEXT:    movdqa %xmm1, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: ssubo_v2i64:
+; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648]
+; SSSE3-NEXT:    movdqa %xmm0, %xmm3
+; SSSE3-NEXT:    psubq %xmm1, %xmm0
+; SSSE3-NEXT:    pxor %xmm2, %xmm1
+; SSSE3-NEXT:    movdqa %xmm2, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm5, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    por %xmm1, %xmm4
+; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm5
+; SSSE3-NEXT:    pxor %xmm5, %xmm4
+; SSSE3-NEXT:    pxor %xmm2, %xmm3
+; SSSE3-NEXT:    movdqa %xmm2, %xmm1
+; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm1[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm3
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm6, %xmm3
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm1[1,1,3,3]
+; SSSE3-NEXT:    por %xmm3, %xmm6
+; SSSE3-NEXT:    pxor %xmm5, %xmm6
+; SSSE3-NEXT:    pcmpeqd %xmm6, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm4[1,0,3,2]
+; SSSE3-NEXT:    pand %xmm4, %xmm1
+; SSSE3-NEXT:    movdqa %xmm0, (%rdi)
+; SSSE3-NEXT:    pxor %xmm2, %xmm0
+; SSSE3-NEXT:    movdqa %xmm2, %xmm3
+; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm3
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm4, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[1,1,3,3]
+; SSSE3-NEXT:    por %xmm0, %xmm2
+; SSSE3-NEXT:    pxor %xmm5, %xmm2
+; SSSE3-NEXT:    pcmpeqd %xmm6, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,0,3,2]
+; SSSE3-NEXT:    pand %xmm2, %xmm0
+; SSSE3-NEXT:    pxor %xmm5, %xmm0
+; SSSE3-NEXT:    pandn %xmm0, %xmm1
+; SSSE3-NEXT:    movdqa %xmm1, %xmm0
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: ssubo_v2i64:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648]
+; SSE41-NEXT:    movdqa %xmm0, %xmm3
+; SSE41-NEXT:    psubq %xmm1, %xmm0
+; SSE41-NEXT:    pxor %xmm2, %xmm1
+; SSE41-NEXT:    movdqa %xmm2, %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm1, %xmm4
+; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSE41-NEXT:    pcmpeqd %xmm2, %xmm1
+; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm1[1,1,3,3]
+; SSE41-NEXT:    pand %xmm5, %xmm6
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm4[1,1,3,3]
+; SSE41-NEXT:    por %xmm6, %xmm1
+; SSE41-NEXT:    pcmpeqd %xmm4, %xmm4
+; SSE41-NEXT:    pxor %xmm4, %xmm1
+; SSE41-NEXT:    pxor %xmm2, %xmm3
+; SSE41-NEXT:    movdqa %xmm2, %xmm5
+; SSE41-NEXT:    pcmpgtd %xmm3, %xmm5
+; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
+; SSE41-NEXT:    pcmpeqd %xmm2, %xmm3
+; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
+; SSE41-NEXT:    pand %xmm6, %xmm3
+; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
+; SSE41-NEXT:    por %xmm3, %xmm5
+; SSE41-NEXT:    pxor %xmm4, %xmm5
+; SSE41-NEXT:    pcmpeqq %xmm5, %xmm1
+; SSE41-NEXT:    movdqa %xmm0, (%rdi)
+; SSE41-NEXT:    pxor %xmm2, %xmm0
+; SSE41-NEXT:    movdqa %xmm2, %xmm3
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm3
+; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm3[0,0,2,2]
+; SSE41-NEXT:    pcmpeqd %xmm2, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE41-NEXT:    pand %xmm6, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[1,1,3,3]
+; SSE41-NEXT:    por %xmm0, %xmm2
+; SSE41-NEXT:    pxor %xmm4, %xmm2
+; SSE41-NEXT:    pcmpeqq %xmm5, %xmm2
+; SSE41-NEXT:    pxor %xmm4, %xmm2
+; SSE41-NEXT:    pandn %xmm2, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: ssubo_v2i64:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm2, %xmm3
+; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
+; AVX1-NEXT:    vpxor %xmm4, %xmm3, %xmm3
+; AVX1-NEXT:    vpcmpgtq %xmm0, %xmm2, %xmm5
+; AVX1-NEXT:    vpxor %xmm4, %xmm5, %xmm5
+; AVX1-NEXT:    vpcmpeqq %xmm3, %xmm5, %xmm3
+; AVX1-NEXT:    vpsubq %xmm1, %xmm0, %xmm1
+; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm2, %xmm0
+; AVX1-NEXT:    vpxor %xmm4, %xmm0, %xmm0
+; AVX1-NEXT:    vpcmpeqq %xmm0, %xmm5, %xmm0
+; AVX1-NEXT:    vpxor %xmm4, %xmm0, %xmm0
+; AVX1-NEXT:    vpandn %xmm0, %xmm3, %xmm0
+; AVX1-NEXT:    vmovdqa %xmm1, (%rdi)
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: ssubo_v2i64:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX2-NEXT:    vpcmpgtq %xmm1, %xmm2, %xmm3
+; AVX2-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
+; AVX2-NEXT:    vpxor %xmm4, %xmm3, %xmm3
+; AVX2-NEXT:    vpcmpgtq %xmm0, %xmm2, %xmm5
+; AVX2-NEXT:    vpxor %xmm4, %xmm5, %xmm5
+; AVX2-NEXT:    vpcmpeqq %xmm3, %xmm5, %xmm3
+; AVX2-NEXT:    vpsubq %xmm1, %xmm0, %xmm1
+; AVX2-NEXT:    vpcmpgtq %xmm1, %xmm2, %xmm0
+; AVX2-NEXT:    vpxor %xmm4, %xmm0, %xmm0
+; AVX2-NEXT:    vpcmpeqq %xmm0, %xmm5, %xmm0
+; AVX2-NEXT:    vpxor %xmm4, %xmm0, %xmm0
+; AVX2-NEXT:    vpandn %xmm0, %xmm3, %xmm0
+; AVX2-NEXT:    vmovdqa %xmm1, (%rdi)
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: ssubo_v2i64:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX512-NEXT:    vpcmpnltq %xmm2, %xmm1, %k0
+; AVX512-NEXT:    vpcmpnltq %xmm2, %xmm0, %k1
+; AVX512-NEXT:    kxorw %k0, %k1, %k0
+; AVX512-NEXT:    vpsubq %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vpcmpnltq %xmm2, %xmm0, %k2
+; AVX512-NEXT:    kxorw %k2, %k1, %k1
+; AVX512-NEXT:    kandw %k1, %k0, %k1
+; AVX512-NEXT:    vmovdqa %xmm0, (%rdi)
+; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX512-NEXT:    vmovdqa64 %xmm0, %xmm0 {%k1} {z}
+; AVX512-NEXT:    retq
+  %t = call {<2 x i64>, <2 x i1>} @llvm.ssub.with.overflow.v2i64(<2 x i64> %a0, <2 x i64> %a1)
+  %val = extractvalue {<2 x i64>, <2 x i1>} %t, 0
+  %obit = extractvalue {<2 x i64>, <2 x i1>} %t, 1
+  %res = sext <2 x i1> %obit to <2 x i32>
+  store <2 x i64> %val, <2 x i64>* %p2
+  ret <2 x i32> %res
+}
+
+define <4 x i32> @ssubo_v4i24(<4 x i24> %a0, <4 x i24> %a1, <4 x i24>* %p2) nounwind {
+; SSE2-LABEL: ssubo_v4i24:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa %xmm0, %xmm2
+; SSE2-NEXT:    pslld $8, %xmm1
+; SSE2-NEXT:    psrad $8, %xmm1
+; SSE2-NEXT:    pslld $8, %xmm2
+; SSE2-NEXT:    psrad $8, %xmm2
+; SSE2-NEXT:    psubd %xmm1, %xmm2
+; SSE2-NEXT:    movdqa %xmm2, %xmm0
+; SSE2-NEXT:    pslld $8, %xmm0
+; SSE2-NEXT:    psrad $8, %xmm0
+; SSE2-NEXT:    pcmpeqd %xmm2, %xmm0
+; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
+; SSE2-NEXT:    pxor %xmm1, %xmm0
+; SSE2-NEXT:    movd %xmm2, %eax
+; SSE2-NEXT:    movw %ax, (%rdi)
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[3,1,2,3]
+; SSE2-NEXT:    movd %xmm1, %ecx
+; SSE2-NEXT:    movw %cx, 9(%rdi)
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[2,3,0,1]
+; SSE2-NEXT:    movd %xmm1, %edx
+; SSE2-NEXT:    movw %dx, 6(%rdi)
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,1,2,3]
+; SSE2-NEXT:    movd %xmm1, %esi
+; SSE2-NEXT:    movw %si, 3(%rdi)
+; SSE2-NEXT:    shrl $16, %eax
+; SSE2-NEXT:    movb %al, 2(%rdi)
+; SSE2-NEXT:    shrl $16, %ecx
+; SSE2-NEXT:    movb %cl, 11(%rdi)
+; SSE2-NEXT:    shrl $16, %edx
+; SSE2-NEXT:    movb %dl, 8(%rdi)
+; SSE2-NEXT:    shrl $16, %esi
+; SSE2-NEXT:    movb %sil, 5(%rdi)
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: ssubo_v4i24:
+; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    movdqa %xmm0, %xmm2
+; SSSE3-NEXT:    pslld $8, %xmm1
+; SSSE3-NEXT:    psrad $8, %xmm1
+; SSSE3-NEXT:    pslld $8, %xmm2
+; SSSE3-NEXT:    psrad $8, %xmm2
+; SSSE3-NEXT:    psubd %xmm1, %xmm2
+; SSSE3-NEXT:    movdqa %xmm2, %xmm0
+; SSSE3-NEXT:    pslld $8, %xmm0
+; SSSE3-NEXT:    psrad $8, %xmm0
+; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm0
+; SSSE3-NEXT:    pcmpeqd %xmm1, %xmm1
+; SSSE3-NEXT:    pxor %xmm1, %xmm0
+; SSSE3-NEXT:    movd %xmm2, %eax
+; SSSE3-NEXT:    movw %ax, (%rdi)
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[3,1,2,3]
+; SSSE3-NEXT:    movd %xmm1, %ecx
+; SSSE3-NEXT:    movw %cx, 9(%rdi)
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[2,3,0,1]
+; SSSE3-NEXT:    movd %xmm1, %edx
+; SSSE3-NEXT:    movw %dx, 6(%rdi)
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,1,2,3]
+; SSSE3-NEXT:    movd %xmm1, %esi
+; SSSE3-NEXT:    movw %si, 3(%rdi)
+; SSSE3-NEXT:    shrl $16, %eax
+; SSSE3-NEXT:    movb %al, 2(%rdi)
+; SSSE3-NEXT:    shrl $16, %ecx
+; SSSE3-NEXT:    movb %cl, 11(%rdi)
+; SSSE3-NEXT:    shrl $16, %edx
+; SSSE3-NEXT:    movb %dl, 8(%rdi)
+; SSSE3-NEXT:    shrl $16, %esi
+; SSSE3-NEXT:    movb %sil, 5(%rdi)
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: ssubo_v4i24:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movdqa %xmm0, %xmm2
+; SSE41-NEXT:    pslld $8, %xmm1
+; SSE41-NEXT:    psrad $8, %xmm1
+; SSE41-NEXT:    pslld $8, %xmm2
+; SSE41-NEXT:    psrad $8, %xmm2
+; SSE41-NEXT:    psubd %xmm1, %xmm2
+; SSE41-NEXT:    movdqa %xmm2, %xmm0
+; SSE41-NEXT:    pslld $8, %xmm0
+; SSE41-NEXT:    psrad $8, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm2, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm1, %xmm1
+; SSE41-NEXT:    pxor %xmm1, %xmm0
+; SSE41-NEXT:    pextrd $3, %xmm2, %eax
+; SSE41-NEXT:    movw %ax, 9(%rdi)
+; SSE41-NEXT:    pextrd $2, %xmm2, %ecx
+; SSE41-NEXT:    movw %cx, 6(%rdi)
+; SSE41-NEXT:    pextrd $1, %xmm2, %edx
+; SSE41-NEXT:    movw %dx, 3(%rdi)
+; SSE41-NEXT:    movd %xmm2, %esi
+; SSE41-NEXT:    movw %si, (%rdi)
+; SSE41-NEXT:    shrl $16, %eax
+; SSE41-NEXT:    movb %al, 11(%rdi)
+; SSE41-NEXT:    shrl $16, %ecx
+; SSE41-NEXT:    movb %cl, 8(%rdi)
+; SSE41-NEXT:    shrl $16, %edx
+; SSE41-NEXT:    movb %dl, 5(%rdi)
+; SSE41-NEXT:    shrl $16, %esi
+; SSE41-NEXT:    movb %sil, 2(%rdi)
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: ssubo_v4i24:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpslld $8, %xmm1, %xmm1
+; AVX1-NEXT:    vpsrad $8, %xmm1, %xmm1
+; AVX1-NEXT:    vpslld $8, %xmm0, %xmm0
+; AVX1-NEXT:    vpsrad $8, %xmm0, %xmm0
+; AVX1-NEXT:    vpsubd %xmm1, %xmm0, %xmm1
+; AVX1-NEXT:    vpslld $8, %xmm1, %xmm0
+; AVX1-NEXT:    vpsrad $8, %xmm0, %xmm0
+; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
+; AVX1-NEXT:    vpxor %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpextrd $3, %xmm1, %eax
+; AVX1-NEXT:    movw %ax, 9(%rdi)
+; AVX1-NEXT:    vpextrd $2, %xmm1, %ecx
+; AVX1-NEXT:    movw %cx, 6(%rdi)
+; AVX1-NEXT:    vpextrd $1, %xmm1, %edx
+; AVX1-NEXT:    movw %dx, 3(%rdi)
+; AVX1-NEXT:    vmovd %xmm1, %esi
+; AVX1-NEXT:    movw %si, (%rdi)
+; AVX1-NEXT:    shrl $16, %eax
+; AVX1-NEXT:    movb %al, 11(%rdi)
+; AVX1-NEXT:    shrl $16, %ecx
+; AVX1-NEXT:    movb %cl, 8(%rdi)
+; AVX1-NEXT:    shrl $16, %edx
+; AVX1-NEXT:    movb %dl, 5(%rdi)
+; AVX1-NEXT:    shrl $16, %esi
+; AVX1-NEXT:    movb %sil, 2(%rdi)
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: ssubo_v4i24:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpslld $8, %xmm1, %xmm1
+; AVX2-NEXT:    vpsrad $8, %xmm1, %xmm1
+; AVX2-NEXT:    vpslld $8, %xmm0, %xmm0
+; AVX2-NEXT:    vpsrad $8, %xmm0, %xmm0
+; AVX2-NEXT:    vpsubd %xmm1, %xmm0, %xmm1
+; AVX2-NEXT:    vpslld $8, %xmm1, %xmm0
+; AVX2-NEXT:    vpsrad $8, %xmm0, %xmm0
+; AVX2-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
+; AVX2-NEXT:    vpxor %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    vpextrd $3, %xmm1, %eax
+; AVX2-NEXT:    movw %ax, 9(%rdi)
+; AVX2-NEXT:    vpextrd $2, %xmm1, %ecx
+; AVX2-NEXT:    movw %cx, 6(%rdi)
+; AVX2-NEXT:    vpextrd $1, %xmm1, %edx
+; AVX2-NEXT:    movw %dx, 3(%rdi)
+; AVX2-NEXT:    vmovd %xmm1, %esi
+; AVX2-NEXT:    movw %si, (%rdi)
+; AVX2-NEXT:    shrl $16, %eax
+; AVX2-NEXT:    movb %al, 11(%rdi)
+; AVX2-NEXT:    shrl $16, %ecx
+; AVX2-NEXT:    movb %cl, 8(%rdi)
+; AVX2-NEXT:    shrl $16, %edx
+; AVX2-NEXT:    movb %dl, 5(%rdi)
+; AVX2-NEXT:    shrl $16, %esi
+; AVX2-NEXT:    movb %sil, 2(%rdi)
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: ssubo_v4i24:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpslld $8, %xmm1, %xmm1
+; AVX512-NEXT:    vpsrad $8, %xmm1, %xmm1
+; AVX512-NEXT:    vpslld $8, %xmm0, %xmm0
+; AVX512-NEXT:    vpsrad $8, %xmm0, %xmm0
+; AVX512-NEXT:    vpsubd %xmm1, %xmm0, %xmm1
+; AVX512-NEXT:    vpslld $8, %xmm1, %xmm0
+; AVX512-NEXT:    vpsrad $8, %xmm0, %xmm0
+; AVX512-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vpternlogq $15, %xmm0, %xmm0, %xmm0
+; AVX512-NEXT:    vpextrd $3, %xmm1, %eax
+; AVX512-NEXT:    movw %ax, 9(%rdi)
+; AVX512-NEXT:    vpextrd $2, %xmm1, %ecx
+; AVX512-NEXT:    movw %cx, 6(%rdi)
+; AVX512-NEXT:    vpextrd $1, %xmm1, %edx
+; AVX512-NEXT:    movw %dx, 3(%rdi)
+; AVX512-NEXT:    vmovd %xmm1, %esi
+; AVX512-NEXT:    movw %si, (%rdi)
+; AVX512-NEXT:    shrl $16, %eax
+; AVX512-NEXT:    movb %al, 11(%rdi)
+; AVX512-NEXT:    shrl $16, %ecx
+; AVX512-NEXT:    movb %cl, 8(%rdi)
+; AVX512-NEXT:    shrl $16, %edx
+; AVX512-NEXT:    movb %dl, 5(%rdi)
+; AVX512-NEXT:    shrl $16, %esi
+; AVX512-NEXT:    movb %sil, 2(%rdi)
+; AVX512-NEXT:    retq
+  %t = call {<4 x i24>, <4 x i1>} @llvm.ssub.with.overflow.v4i24(<4 x i24> %a0, <4 x i24> %a1)
+  %val = extractvalue {<4 x i24>, <4 x i1>} %t, 0
+  %obit = extractvalue {<4 x i24>, <4 x i1>} %t, 1
+  %res = sext <4 x i1> %obit to <4 x i32>
+  store <4 x i24> %val, <4 x i24>* %p2
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @ssubo_v4i1(<4 x i1> %a0, <4 x i1> %a1, <4 x i1>* %p2) nounwind {
+; SSE-LABEL: ssubo_v4i1:
+; SSE:       # %bb.0:
+; SSE-NEXT:    pslld $31, %xmm1
+; SSE-NEXT:    psrad $31, %xmm1
+; SSE-NEXT:    pslld $31, %xmm0
+; SSE-NEXT:    psrad $31, %xmm0
+; SSE-NEXT:    psubd %xmm1, %xmm0
+; SSE-NEXT:    movdqa %xmm0, %xmm1
+; SSE-NEXT:    pslld $31, %xmm1
+; SSE-NEXT:    psrad $31, %xmm1
+; SSE-NEXT:    pcmpeqd %xmm1, %xmm0
+; SSE-NEXT:    pcmpeqd %xmm2, %xmm2
+; SSE-NEXT:    pxor %xmm2, %xmm0
+; SSE-NEXT:    movmskps %xmm1, %eax
+; SSE-NEXT:    movb %al, (%rdi)
+; SSE-NEXT:    retq
+;
+; AVX1-LABEL: ssubo_v4i1:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpslld $31, %xmm1, %xmm1
+; AVX1-NEXT:    vpsrad $31, %xmm1, %xmm1
+; AVX1-NEXT:    vpslld $31, %xmm0, %xmm0
+; AVX1-NEXT:    vpsrad $31, %xmm0, %xmm0
+; AVX1-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpslld $31, %xmm0, %xmm1
+; AVX1-NEXT:    vpsrad $31, %xmm1, %xmm1
+; AVX1-NEXT:    vpcmpeqd %xmm0, %xmm1, %xmm0
+; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
+; AVX1-NEXT:    vpxor %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vmovmskps %xmm1, %eax
+; AVX1-NEXT:    movb %al, (%rdi)
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: ssubo_v4i1:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpslld $31, %xmm1, %xmm1
+; AVX2-NEXT:    vpsrad $31, %xmm1, %xmm1
+; AVX2-NEXT:    vpslld $31, %xmm0, %xmm0
+; AVX2-NEXT:    vpsrad $31, %xmm0, %xmm0
+; AVX2-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpslld $31, %xmm0, %xmm1
+; AVX2-NEXT:    vpsrad $31, %xmm1, %xmm1
+; AVX2-NEXT:    vpcmpeqd %xmm0, %xmm1, %xmm0
+; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
+; AVX2-NEXT:    vpxor %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    vmovmskps %xmm1, %eax
+; AVX2-NEXT:    movb %al, (%rdi)
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: ssubo_v4i1:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpslld $31, %xmm0, %xmm0
+; AVX512-NEXT:    vptestmd %xmm0, %xmm0, %k0
+; AVX512-NEXT:    vptestnmd %xmm0, %xmm0, %k1
+; AVX512-NEXT:    vpslld $31, %xmm1, %xmm0
+; AVX512-NEXT:    vptestmd %xmm0, %xmm0, %k2
+; AVX512-NEXT:    kxorw %k2, %k1, %k3
+; AVX512-NEXT:    kxorw %k2, %k0, %k0
+; AVX512-NEXT:    kxnorw %k0, %k1, %k1
+; AVX512-NEXT:    kandnw %k1, %k3, %k1
+; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
+; AVX512-NEXT:    kmovd %k0, %eax
+; AVX512-NEXT:    movb %al, (%rdi)
+; AVX512-NEXT:    retq
+  %t = call {<4 x i1>, <4 x i1>} @llvm.ssub.with.overflow.v4i1(<4 x i1> %a0, <4 x i1> %a1)
+  %val = extractvalue {<4 x i1>, <4 x i1>} %t, 0
+  %obit = extractvalue {<4 x i1>, <4 x i1>} %t, 1
+  %res = sext <4 x i1> %obit to <4 x i32>
+  store <4 x i1> %val, <4 x i1>* %p2
+  ret <4 x i32> %res
+}
+
+define <2 x i32> @ssubo_v2i128(<2 x i128> %a0, <2 x i128> %a1, <2 x i128>* %p2) nounwind {
+; SSE2-LABEL: ssubo_v2i128:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    pushq %rbp
+; SSE2-NEXT:    pushq %rbx
+; SSE2-NEXT:    movq {{[0-9]+}}(%rsp), %r10
+; SSE2-NEXT:    movq {{[0-9]+}}(%rsp), %r11
+; SSE2-NEXT:    subq {{[0-9]+}}(%rsp), %rdx
+; SSE2-NEXT:    movq %rcx, %rax
+; SSE2-NEXT:    sbbq %r11, %rax
+; SSE2-NEXT:    setns %bl
+; SSE2-NEXT:    testq %rcx, %rcx
+; SSE2-NEXT:    setns %cl
+; SSE2-NEXT:    cmpb %bl, %cl
+; SSE2-NEXT:    setne %bpl
+; SSE2-NEXT:    testq %r11, %r11
+; SSE2-NEXT:    setns %bl
+; SSE2-NEXT:    cmpb %bl, %cl
+; SSE2-NEXT:    setne %cl
+; SSE2-NEXT:    andb %bpl, %cl
+; SSE2-NEXT:    movzbl %cl, %ebp
+; SSE2-NEXT:    testq %r9, %r9
+; SSE2-NEXT:    setns %bl
+; SSE2-NEXT:    testq %rsi, %rsi
+; SSE2-NEXT:    setns %cl
+; SSE2-NEXT:    cmpb %bl, %cl
+; SSE2-NEXT:    setne %r11b
+; SSE2-NEXT:    subq %r8, %rdi
+; SSE2-NEXT:    sbbq %r9, %rsi
+; SSE2-NEXT:    setns %bl
+; SSE2-NEXT:    cmpb %bl, %cl
+; SSE2-NEXT:    setne %cl
+; SSE2-NEXT:    andb %r11b, %cl
+; SSE2-NEXT:    movzbl %cl, %ecx
+; SSE2-NEXT:    movd %ecx, %xmm0
+; SSE2-NEXT:    pinsrw $4, %ebp, %xmm0
+; SSE2-NEXT:    movq %rdx, 16(%r10)
+; SSE2-NEXT:    movq %rdi, (%r10)
+; SSE2-NEXT:    movq %rax, 24(%r10)
+; SSE2-NEXT:    movq %rsi, 8(%r10)
+; SSE2-NEXT:    psllq $63, %xmm0
+; SSE2-NEXT:    psrad $31, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE2-NEXT:    popq %rbx
+; SSE2-NEXT:    popq %rbp
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: ssubo_v2i128:
+; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    pushq %rbp
+; SSSE3-NEXT:    pushq %rbx
+; SSSE3-NEXT:    movq {{[0-9]+}}(%rsp), %r10
+; SSSE3-NEXT:    movq {{[0-9]+}}(%rsp), %r11
+; SSSE3-NEXT:    subq {{[0-9]+}}(%rsp), %rdx
+; SSSE3-NEXT:    movq %rcx, %rax
+; SSSE3-NEXT:    sbbq %r11, %rax
+; SSSE3-NEXT:    setns %bl
+; SSSE3-NEXT:    testq %rcx, %rcx
+; SSSE3-NEXT:    setns %cl
+; SSSE3-NEXT:    cmpb %bl, %cl
+; SSSE3-NEXT:    setne %bpl
+; SSSE3-NEXT:    testq %r11, %r11
+; SSSE3-NEXT:    setns %bl
+; SSSE3-NEXT:    cmpb %bl, %cl
+; SSSE3-NEXT:    setne %cl
+; SSSE3-NEXT:    andb %bpl, %cl
+; SSSE3-NEXT:    movzbl %cl, %ebp
+; SSSE3-NEXT:    testq %r9, %r9
+; SSSE3-NEXT:    setns %bl
+; SSSE3-NEXT:    testq %rsi, %rsi
+; SSSE3-NEXT:    setns %cl
+; SSSE3-NEXT:    cmpb %bl, %cl
+; SSSE3-NEXT:    setne %r11b
+; SSSE3-NEXT:    subq %r8, %rdi
+; SSSE3-NEXT:    sbbq %r9, %rsi
+; SSSE3-NEXT:    setns %bl
+; SSSE3-NEXT:    cmpb %bl, %cl
+; SSSE3-NEXT:    setne %cl
+; SSSE3-NEXT:    andb %r11b, %cl
+; SSSE3-NEXT:    movzbl %cl, %ecx
+; SSSE3-NEXT:    movd %ecx, %xmm0
+; SSSE3-NEXT:    pinsrw $4, %ebp, %xmm0
+; SSSE3-NEXT:    movq %rdx, 16(%r10)
+; SSSE3-NEXT:    movq %rdi, (%r10)
+; SSSE3-NEXT:    movq %rax, 24(%r10)
+; SSSE3-NEXT:    movq %rsi, 8(%r10)
+; SSSE3-NEXT:    psllq $63, %xmm0
+; SSSE3-NEXT:    psrad $31, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSSE3-NEXT:    popq %rbx
+; SSSE3-NEXT:    popq %rbp
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: ssubo_v2i128:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    pushq %rbp
+; SSE41-NEXT:    pushq %rbx
+; SSE41-NEXT:    movq {{[0-9]+}}(%rsp), %r10
+; SSE41-NEXT:    movq {{[0-9]+}}(%rsp), %r11
+; SSE41-NEXT:    subq {{[0-9]+}}(%rsp), %rdx
+; SSE41-NEXT:    movq %rcx, %rax
+; SSE41-NEXT:    sbbq %r11, %rax
+; SSE41-NEXT:    setns %bl
+; SSE41-NEXT:    testq %rcx, %rcx
+; SSE41-NEXT:    setns %cl
+; SSE41-NEXT:    cmpb %bl, %cl
+; SSE41-NEXT:    setne %bpl
+; SSE41-NEXT:    testq %r11, %r11
+; SSE41-NEXT:    setns %bl
+; SSE41-NEXT:    cmpb %bl, %cl
+; SSE41-NEXT:    setne %cl
+; SSE41-NEXT:    andb %bpl, %cl
+; SSE41-NEXT:    movzbl %cl, %ebp
+; SSE41-NEXT:    testq %r9, %r9
+; SSE41-NEXT:    setns %bl
+; SSE41-NEXT:    testq %rsi, %rsi
+; SSE41-NEXT:    setns %cl
+; SSE41-NEXT:    cmpb %bl, %cl
+; SSE41-NEXT:    setne %r11b
+; SSE41-NEXT:    subq %r8, %rdi
+; SSE41-NEXT:    sbbq %r9, %rsi
+; SSE41-NEXT:    setns %bl
+; SSE41-NEXT:    cmpb %bl, %cl
+; SSE41-NEXT:    setne %cl
+; SSE41-NEXT:    andb %r11b, %cl
+; SSE41-NEXT:    movzbl %cl, %ecx
+; SSE41-NEXT:    movd %ecx, %xmm0
+; SSE41-NEXT:    pinsrb $8, %ebp, %xmm0
+; SSE41-NEXT:    movq %rdx, 16(%r10)
+; SSE41-NEXT:    movq %rdi, (%r10)
+; SSE41-NEXT:    movq %rax, 24(%r10)
+; SSE41-NEXT:    movq %rsi, 8(%r10)
+; SSE41-NEXT:    psllq $63, %xmm0
+; SSE41-NEXT:    psrad $31, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE41-NEXT:    popq %rbx
+; SSE41-NEXT:    popq %rbp
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: ssubo_v2i128:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    pushq %rbp
+; AVX1-NEXT:    pushq %rbx
+; AVX1-NEXT:    movq {{[0-9]+}}(%rsp), %r10
+; AVX1-NEXT:    movq {{[0-9]+}}(%rsp), %r11
+; AVX1-NEXT:    subq {{[0-9]+}}(%rsp), %rdx
+; AVX1-NEXT:    movq %rcx, %rax
+; AVX1-NEXT:    sbbq %r11, %rax
+; AVX1-NEXT:    setns %bl
+; AVX1-NEXT:    testq %rcx, %rcx
+; AVX1-NEXT:    setns %cl
+; AVX1-NEXT:    cmpb %bl, %cl
+; AVX1-NEXT:    setne %bpl
+; AVX1-NEXT:    testq %r11, %r11
+; AVX1-NEXT:    setns %bl
+; AVX1-NEXT:    cmpb %bl, %cl
+; AVX1-NEXT:    setne %cl
+; AVX1-NEXT:    andb %bpl, %cl
+; AVX1-NEXT:    movzbl %cl, %ebp
+; AVX1-NEXT:    testq %r9, %r9
+; AVX1-NEXT:    setns %bl
+; AVX1-NEXT:    testq %rsi, %rsi
+; AVX1-NEXT:    setns %cl
+; AVX1-NEXT:    cmpb %bl, %cl
+; AVX1-NEXT:    setne %r11b
+; AVX1-NEXT:    subq %r8, %rdi
+; AVX1-NEXT:    sbbq %r9, %rsi
+; AVX1-NEXT:    setns %bl
+; AVX1-NEXT:    cmpb %bl, %cl
+; AVX1-NEXT:    setne %cl
+; AVX1-NEXT:    andb %r11b, %cl
+; AVX1-NEXT:    movzbl %cl, %ecx
+; AVX1-NEXT:    vmovd %ecx, %xmm0
+; AVX1-NEXT:    vpinsrb $8, %ebp, %xmm0, %xmm0
+; AVX1-NEXT:    movq %rdx, 16(%r10)
+; AVX1-NEXT:    movq %rdi, (%r10)
+; AVX1-NEXT:    movq %rax, 24(%r10)
+; AVX1-NEXT:    movq %rsi, 8(%r10)
+; AVX1-NEXT:    vpsllq $63, %xmm0, %xmm0
+; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX1-NEXT:    vpcmpgtq %xmm0, %xmm1, %xmm0
+; AVX1-NEXT:    popq %rbx
+; AVX1-NEXT:    popq %rbp
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: ssubo_v2i128:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    pushq %rbp
+; AVX2-NEXT:    pushq %rbx
+; AVX2-NEXT:    movq {{[0-9]+}}(%rsp), %r10
+; AVX2-NEXT:    movq {{[0-9]+}}(%rsp), %r11
+; AVX2-NEXT:    subq {{[0-9]+}}(%rsp), %rdx
+; AVX2-NEXT:    movq %rcx, %rax
+; AVX2-NEXT:    sbbq %r11, %rax
+; AVX2-NEXT:    setns %bl
+; AVX2-NEXT:    testq %rcx, %rcx
+; AVX2-NEXT:    setns %cl
+; AVX2-NEXT:    cmpb %bl, %cl
+; AVX2-NEXT:    setne %bpl
+; AVX2-NEXT:    testq %r11, %r11
+; AVX2-NEXT:    setns %bl
+; AVX2-NEXT:    cmpb %bl, %cl
+; AVX2-NEXT:    setne %cl
+; AVX2-NEXT:    andb %bpl, %cl
+; AVX2-NEXT:    movzbl %cl, %ebp
+; AVX2-NEXT:    testq %r9, %r9
+; AVX2-NEXT:    setns %bl
+; AVX2-NEXT:    testq %rsi, %rsi
+; AVX2-NEXT:    setns %cl
+; AVX2-NEXT:    cmpb %bl, %cl
+; AVX2-NEXT:    setne %r11b
+; AVX2-NEXT:    subq %r8, %rdi
+; AVX2-NEXT:    sbbq %r9, %rsi
+; AVX2-NEXT:    setns %bl
+; AVX2-NEXT:    cmpb %bl, %cl
+; AVX2-NEXT:    setne %cl
+; AVX2-NEXT:    andb %r11b, %cl
+; AVX2-NEXT:    movzbl %cl, %ecx
+; AVX2-NEXT:    vmovd %ecx, %xmm0
+; AVX2-NEXT:    vpinsrb $8, %ebp, %xmm0, %xmm0
+; AVX2-NEXT:    movq %rdx, 16(%r10)
+; AVX2-NEXT:    movq %rdi, (%r10)
+; AVX2-NEXT:    movq %rax, 24(%r10)
+; AVX2-NEXT:    movq %rsi, 8(%r10)
+; AVX2-NEXT:    vpsllq $63, %xmm0, %xmm0
+; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX2-NEXT:    vpcmpgtq %xmm0, %xmm1, %xmm0
+; AVX2-NEXT:    popq %rbx
+; AVX2-NEXT:    popq %rbp
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: ssubo_v2i128:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    pushq %r14
+; AVX512-NEXT:    pushq %rbx
+; AVX512-NEXT:    movq {{[0-9]+}}(%rsp), %r10
+; AVX512-NEXT:    movq {{[0-9]+}}(%rsp), %r11
+; AVX512-NEXT:    subq {{[0-9]+}}(%rsp), %rdx
+; AVX512-NEXT:    movq %rcx, %r14
+; AVX512-NEXT:    sbbq %r11, %r14
+; AVX512-NEXT:    setns %bl
+; AVX512-NEXT:    testq %rcx, %rcx
+; AVX512-NEXT:    setns %cl
+; AVX512-NEXT:    cmpb %bl, %cl
+; AVX512-NEXT:    setne %bl
+; AVX512-NEXT:    testq %r11, %r11
+; AVX512-NEXT:    setns %al
+; AVX512-NEXT:    cmpb %al, %cl
+; AVX512-NEXT:    setne %al
+; AVX512-NEXT:    andb %bl, %al
+; AVX512-NEXT:    movb %al, -{{[0-9]+}}(%rsp)
+; AVX512-NEXT:    testq %r9, %r9
+; AVX512-NEXT:    setns %al
+; AVX512-NEXT:    testq %rsi, %rsi
+; AVX512-NEXT:    setns %cl
+; AVX512-NEXT:    cmpb %al, %cl
+; AVX512-NEXT:    setne %al
+; AVX512-NEXT:    subq %r8, %rdi
+; AVX512-NEXT:    sbbq %r9, %rsi
+; AVX512-NEXT:    setns %bl
+; AVX512-NEXT:    cmpb %bl, %cl
+; AVX512-NEXT:    setne %cl
+; AVX512-NEXT:    andb %al, %cl
+; AVX512-NEXT:    movb %cl, -{{[0-9]+}}(%rsp)
+; AVX512-NEXT:    kmovw -{{[0-9]+}}(%rsp), %k1
+; AVX512-NEXT:    movq %rdx, 16(%r10)
+; AVX512-NEXT:    movq %rdi, (%r10)
+; AVX512-NEXT:    movq %r14, 24(%r10)
+; AVX512-NEXT:    movq %rsi, 8(%r10)
+; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX512-NEXT:    vmovdqa64 %xmm0, %xmm0 {%k1} {z}
+; AVX512-NEXT:    popq %rbx
+; AVX512-NEXT:    popq %r14
+; AVX512-NEXT:    retq
+  %t = call {<2 x i128>, <2 x i1>} @llvm.ssub.with.overflow.v2i128(<2 x i128> %a0, <2 x i128> %a1)
+  %val = extractvalue {<2 x i128>, <2 x i1>} %t, 0
+  %obit = extractvalue {<2 x i128>, <2 x i1>} %t, 1
+  %res = sext <2 x i1> %obit to <2 x i32>
+  store <2 x i128> %val, <2 x i128>* %p2
+  ret <2 x i32> %res
+}
diff --git a/test/CodeGen/X86/vec_uaddo.ll b/test/CodeGen/X86/vec_uaddo.ll
new file mode 100644
index 0000000000000000000000000000000000000000..b040cd9916fee221bb7029e4181723d589ef79ab
--- /dev/null
+++ b/test/CodeGen/X86/vec_uaddo.ll
@@ -0,0 +1,1381 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefixes=SSE,SSE2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+ssse3 | FileCheck %s --check-prefixes=SSE,SSSE3
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefixes=SSE,SSE41
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefixes=AVX,AVX1
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefixes=AVX,AVX2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw,+avx512vl,+fast-variable-shuffle | FileCheck %s --check-prefixes=AVX,AVX512
+
+declare {<1 x i32>, <1 x i1>} @llvm.uadd.with.overflow.v1i32(<1 x i32>, <1 x i32>)
+declare {<2 x i32>, <2 x i1>} @llvm.uadd.with.overflow.v2i32(<2 x i32>, <2 x i32>)
+declare {<3 x i32>, <3 x i1>} @llvm.uadd.with.overflow.v3i32(<3 x i32>, <3 x i32>)
+declare {<4 x i32>, <4 x i1>} @llvm.uadd.with.overflow.v4i32(<4 x i32>, <4 x i32>)
+declare {<6 x i32>, <6 x i1>} @llvm.uadd.with.overflow.v6i32(<6 x i32>, <6 x i32>)
+declare {<8 x i32>, <8 x i1>} @llvm.uadd.with.overflow.v8i32(<8 x i32>, <8 x i32>)
+declare {<16 x i32>, <16 x i1>} @llvm.uadd.with.overflow.v16i32(<16 x i32>, <16 x i32>)
+
+declare {<16 x i8>, <16 x i1>} @llvm.uadd.with.overflow.v16i8(<16 x i8>, <16 x i8>)
+declare {<8 x i16>, <8 x i1>} @llvm.uadd.with.overflow.v8i16(<8 x i16>, <8 x i16>)
+declare {<2 x i64>, <2 x i1>} @llvm.uadd.with.overflow.v2i64(<2 x i64>, <2 x i64>)
+
+declare {<4 x i24>, <4 x i1>} @llvm.uadd.with.overflow.v4i24(<4 x i24>, <4 x i24>)
+declare {<4 x i1>, <4 x i1>} @llvm.uadd.with.overflow.v4i1(<4 x i1>, <4 x i1>)
+declare {<2 x i128>, <2 x i1>} @llvm.uadd.with.overflow.v2i128(<2 x i128>, <2 x i128>)
+
+define <1 x i32> @uaddo_v1i32(<1 x i32> %a0, <1 x i32> %a1, <1 x i32>* %p2) nounwind {
+; SSE-LABEL: uaddo_v1i32:
+; SSE:       # %bb.0:
+; SSE-NEXT:    addl %esi, %edi
+; SSE-NEXT:    sbbl %eax, %eax
+; SSE-NEXT:    movl %edi, (%rdx)
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: uaddo_v1i32:
+; AVX:       # %bb.0:
+; AVX-NEXT:    addl %esi, %edi
+; AVX-NEXT:    sbbl %eax, %eax
+; AVX-NEXT:    movl %edi, (%rdx)
+; AVX-NEXT:    retq
+  %t = call {<1 x i32>, <1 x i1>} @llvm.uadd.with.overflow.v1i32(<1 x i32> %a0, <1 x i32> %a1)
+  %val = extractvalue {<1 x i32>, <1 x i1>} %t, 0
+  %obit = extractvalue {<1 x i32>, <1 x i1>} %t, 1
+  %res = sext <1 x i1> %obit to <1 x i32>
+  store <1 x i32> %val, <1 x i32>* %p2
+  ret <1 x i32> %res
+}
+
+define <2 x i32> @uaddo_v2i32(<2 x i32> %a0, <2 x i32> %a1, <2 x i32>* %p2) nounwind {
+; SSE2-LABEL: uaddo_v2i32:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [4294967295,0,4294967295,0]
+; SSE2-NEXT:    pand %xmm2, %xmm1
+; SSE2-NEXT:    pand %xmm2, %xmm0
+; SSE2-NEXT:    paddq %xmm1, %xmm0
+; SSE2-NEXT:    pand %xmm0, %xmm2
+; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[1,0,3,2]
+; SSE2-NEXT:    pand %xmm2, %xmm3
+; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
+; SSE2-NEXT:    pxor %xmm3, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; SSE2-NEXT:    movq %xmm0, (%rdi)
+; SSE2-NEXT:    movdqa %xmm1, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: uaddo_v2i32:
+; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm2 = [4294967295,0,4294967295,0]
+; SSSE3-NEXT:    pand %xmm2, %xmm1
+; SSSE3-NEXT:    pand %xmm2, %xmm0
+; SSSE3-NEXT:    paddq %xmm1, %xmm0
+; SSSE3-NEXT:    pand %xmm0, %xmm2
+; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[1,0,3,2]
+; SSSE3-NEXT:    pand %xmm2, %xmm3
+; SSSE3-NEXT:    pcmpeqd %xmm1, %xmm1
+; SSSE3-NEXT:    pxor %xmm3, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; SSSE3-NEXT:    movq %xmm0, (%rdi)
+; SSSE3-NEXT:    movdqa %xmm1, %xmm0
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: uaddo_v2i32:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    pxor %xmm2, %xmm2
+; SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
+; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3],xmm0[4,5],xmm2[6,7]
+; SSE41-NEXT:    paddq %xmm1, %xmm0
+; SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm0[0,1],xmm2[2,3],xmm0[4,5],xmm2[6,7]
+; SSE41-NEXT:    pcmpeqq %xmm0, %xmm2
+; SSE41-NEXT:    pcmpeqd %xmm1, %xmm1
+; SSE41-NEXT:    pxor %xmm2, %xmm1
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; SSE41-NEXT:    movq %xmm0, (%rdi)
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: uaddo_v2i32:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3],xmm0[4,5],xmm2[6,7]
+; AVX1-NEXT:    vpaddq %xmm1, %xmm0, %xmm1
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
+; AVX1-NEXT:    vpcmpeqq %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
+; AVX1-NEXT:    vpxor %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; AVX1-NEXT:    vmovq %xmm1, (%rdi)
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: uaddo_v2i32:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX2-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3]
+; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2],xmm2[3]
+; AVX2-NEXT:    vpaddq %xmm1, %xmm0, %xmm1
+; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm1[0],xmm2[1],xmm1[2],xmm2[3]
+; AVX2-NEXT:    vpcmpeqq %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
+; AVX2-NEXT:    vpxor %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; AVX2-NEXT:    vmovq %xmm1, (%rdi)
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: uaddo_v2i32:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX512-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3]
+; AVX512-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2],xmm2[3]
+; AVX512-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vpblendd {{.*#+}} xmm1 = xmm0[0],xmm2[1],xmm0[2],xmm2[3]
+; AVX512-NEXT:    vpmovqd %xmm0, (%rdi)
+; AVX512-NEXT:    vpcmpeqq %xmm0, %xmm1, %xmm0
+; AVX512-NEXT:    vpternlogq $15, %xmm0, %xmm0, %xmm0
+; AVX512-NEXT:    retq
+  %t = call {<2 x i32>, <2 x i1>} @llvm.uadd.with.overflow.v2i32(<2 x i32> %a0, <2 x i32> %a1)
+  %val = extractvalue {<2 x i32>, <2 x i1>} %t, 0
+  %obit = extractvalue {<2 x i32>, <2 x i1>} %t, 1
+  %res = sext <2 x i1> %obit to <2 x i32>
+  store <2 x i32> %val, <2 x i32>* %p2
+  ret <2 x i32> %res
+}
+
+define <3 x i32> @uaddo_v3i32(<3 x i32> %a0, <3 x i32> %a1, <3 x i32>* %p2) nounwind {
+; SSE2-LABEL: uaddo_v3i32:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648,2147483648,2147483648]
+; SSE2-NEXT:    paddd %xmm0, %xmm1
+; SSE2-NEXT:    pxor %xmm2, %xmm0
+; SSE2-NEXT:    pxor %xmm1, %xmm2
+; SSE2-NEXT:    pcmpgtd %xmm2, %xmm0
+; SSE2-NEXT:    movq %xmm1, (%rdi)
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; SSE2-NEXT:    movd %xmm1, 8(%rdi)
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: uaddo_v3i32:
+; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648,2147483648,2147483648]
+; SSSE3-NEXT:    paddd %xmm0, %xmm1
+; SSSE3-NEXT:    pxor %xmm2, %xmm0
+; SSSE3-NEXT:    pxor %xmm1, %xmm2
+; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm0
+; SSSE3-NEXT:    movq %xmm1, (%rdi)
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; SSSE3-NEXT:    movd %xmm1, 8(%rdi)
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: uaddo_v3i32:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    paddd %xmm0, %xmm1
+; SSE41-NEXT:    pmaxud %xmm1, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm1, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm2, %xmm2
+; SSE41-NEXT:    pxor %xmm2, %xmm0
+; SSE41-NEXT:    pextrd $2, %xmm1, 8(%rdi)
+; SSE41-NEXT:    movq %xmm1, (%rdi)
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: uaddo_v3i32:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm1
+; AVX1-NEXT:    vpmaxud %xmm0, %xmm1, %xmm0
+; AVX1-NEXT:    vpcmpeqd %xmm0, %xmm1, %xmm0
+; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
+; AVX1-NEXT:    vpxor %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpextrd $2, %xmm1, 8(%rdi)
+; AVX1-NEXT:    vmovq %xmm1, (%rdi)
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: uaddo_v3i32:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm1
+; AVX2-NEXT:    vpmaxud %xmm0, %xmm1, %xmm0
+; AVX2-NEXT:    vpcmpeqd %xmm0, %xmm1, %xmm0
+; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
+; AVX2-NEXT:    vpxor %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    vpextrd $2, %xmm1, 8(%rdi)
+; AVX2-NEXT:    vmovq %xmm1, (%rdi)
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: uaddo_v3i32:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm1
+; AVX512-NEXT:    vpcmpltud %xmm0, %xmm1, %k1
+; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
+; AVX512-NEXT:    vpextrd $2, %xmm1, 8(%rdi)
+; AVX512-NEXT:    vmovq %xmm1, (%rdi)
+; AVX512-NEXT:    retq
+  %t = call {<3 x i32>, <3 x i1>} @llvm.uadd.with.overflow.v3i32(<3 x i32> %a0, <3 x i32> %a1)
+  %val = extractvalue {<3 x i32>, <3 x i1>} %t, 0
+  %obit = extractvalue {<3 x i32>, <3 x i1>} %t, 1
+  %res = sext <3 x i1> %obit to <3 x i32>
+  store <3 x i32> %val, <3 x i32>* %p2
+  ret <3 x i32> %res
+}
+
+define <4 x i32> @uaddo_v4i32(<4 x i32> %a0, <4 x i32> %a1, <4 x i32>* %p2) nounwind {
+; SSE2-LABEL: uaddo_v4i32:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648,2147483648,2147483648]
+; SSE2-NEXT:    paddd %xmm0, %xmm1
+; SSE2-NEXT:    pxor %xmm2, %xmm0
+; SSE2-NEXT:    pxor %xmm1, %xmm2
+; SSE2-NEXT:    pcmpgtd %xmm2, %xmm0
+; SSE2-NEXT:    movdqa %xmm1, (%rdi)
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: uaddo_v4i32:
+; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648,2147483648,2147483648]
+; SSSE3-NEXT:    paddd %xmm0, %xmm1
+; SSSE3-NEXT:    pxor %xmm2, %xmm0
+; SSSE3-NEXT:    pxor %xmm1, %xmm2
+; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm0
+; SSSE3-NEXT:    movdqa %xmm1, (%rdi)
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: uaddo_v4i32:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    paddd %xmm0, %xmm1
+; SSE41-NEXT:    pmaxud %xmm1, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm1, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm2, %xmm2
+; SSE41-NEXT:    pxor %xmm2, %xmm0
+; SSE41-NEXT:    movdqa %xmm1, (%rdi)
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: uaddo_v4i32:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm1
+; AVX1-NEXT:    vpmaxud %xmm0, %xmm1, %xmm0
+; AVX1-NEXT:    vpcmpeqd %xmm0, %xmm1, %xmm0
+; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
+; AVX1-NEXT:    vpxor %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vmovdqa %xmm1, (%rdi)
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: uaddo_v4i32:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm1
+; AVX2-NEXT:    vpmaxud %xmm0, %xmm1, %xmm0
+; AVX2-NEXT:    vpcmpeqd %xmm0, %xmm1, %xmm0
+; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
+; AVX2-NEXT:    vpxor %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    vmovdqa %xmm1, (%rdi)
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: uaddo_v4i32:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm1
+; AVX512-NEXT:    vpcmpltud %xmm0, %xmm1, %k1
+; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
+; AVX512-NEXT:    vmovdqa %xmm1, (%rdi)
+; AVX512-NEXT:    retq
+  %t = call {<4 x i32>, <4 x i1>} @llvm.uadd.with.overflow.v4i32(<4 x i32> %a0, <4 x i32> %a1)
+  %val = extractvalue {<4 x i32>, <4 x i1>} %t, 0
+  %obit = extractvalue {<4 x i32>, <4 x i1>} %t, 1
+  %res = sext <4 x i1> %obit to <4 x i32>
+  store <4 x i32> %val, <4 x i32>* %p2
+  ret <4 x i32> %res
+}
+
+define <6 x i32> @uaddo_v6i32(<6 x i32> %a0, <6 x i32> %a1, <6 x i32>* %p2) nounwind {
+; SSE2-LABEL: uaddo_v6i32:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movq %rdi, %rax
+; SSE2-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSE2-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; SSE2-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
+; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; SSE2-NEXT:    movd %r8d, %xmm1
+; SSE2-NEXT:    movd %ecx, %xmm2
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; SSE2-NEXT:    movd %edx, %xmm3
+; SSE2-NEXT:    movd %esi, %xmm1
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
+; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
+; SSE2-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; SSE2-NEXT:    movd {{.*#+}} xmm3 = mem[0],zero,zero,zero
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
+; SSE2-NEXT:    movd %r9d, %xmm2
+; SSE2-NEXT:    movd {{.*#+}} xmm4 = mem[0],zero,zero,zero
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1]
+; SSE2-NEXT:    movq {{[0-9]+}}(%rsp), %rcx
+; SSE2-NEXT:    paddd %xmm1, %xmm0
+; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [2147483648,2147483648,2147483648,2147483648]
+; SSE2-NEXT:    movdqa %xmm0, (%rcx)
+; SSE2-NEXT:    pxor %xmm4, %xmm0
+; SSE2-NEXT:    pxor %xmm4, %xmm1
+; SSE2-NEXT:    pcmpgtd %xmm0, %xmm1
+; SSE2-NEXT:    paddd %xmm2, %xmm3
+; SSE2-NEXT:    movq %xmm3, 16(%rcx)
+; SSE2-NEXT:    pxor %xmm4, %xmm3
+; SSE2-NEXT:    pxor %xmm4, %xmm2
+; SSE2-NEXT:    pcmpgtd %xmm3, %xmm2
+; SSE2-NEXT:    movq %xmm2, 16(%rdi)
+; SSE2-NEXT:    movdqa %xmm1, (%rdi)
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: uaddo_v6i32:
+; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    movq %rdi, %rax
+; SSSE3-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSSE3-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSSE3-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; SSSE3-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
+; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; SSSE3-NEXT:    movd %r8d, %xmm1
+; SSSE3-NEXT:    movd %ecx, %xmm2
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; SSSE3-NEXT:    movd %edx, %xmm3
+; SSSE3-NEXT:    movd %esi, %xmm1
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
+; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
+; SSSE3-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; SSSE3-NEXT:    movd {{.*#+}} xmm3 = mem[0],zero,zero,zero
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
+; SSSE3-NEXT:    movd %r9d, %xmm2
+; SSSE3-NEXT:    movd {{.*#+}} xmm4 = mem[0],zero,zero,zero
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1]
+; SSSE3-NEXT:    movq {{[0-9]+}}(%rsp), %rcx
+; SSSE3-NEXT:    paddd %xmm1, %xmm0
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm4 = [2147483648,2147483648,2147483648,2147483648]
+; SSSE3-NEXT:    movdqa %xmm0, (%rcx)
+; SSSE3-NEXT:    pxor %xmm4, %xmm0
+; SSSE3-NEXT:    pxor %xmm4, %xmm1
+; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm1
+; SSSE3-NEXT:    paddd %xmm2, %xmm3
+; SSSE3-NEXT:    movq %xmm3, 16(%rcx)
+; SSSE3-NEXT:    pxor %xmm4, %xmm3
+; SSSE3-NEXT:    pxor %xmm4, %xmm2
+; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm2
+; SSSE3-NEXT:    movq %xmm2, 16(%rdi)
+; SSSE3-NEXT:    movdqa %xmm1, (%rdi)
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: uaddo_v6i32:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movq %rdi, %rax
+; SSE41-NEXT:    movd %esi, %xmm0
+; SSE41-NEXT:    pinsrd $1, %edx, %xmm0
+; SSE41-NEXT:    pinsrd $2, %ecx, %xmm0
+; SSE41-NEXT:    pinsrd $3, %r8d, %xmm0
+; SSE41-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SSE41-NEXT:    pinsrd $1, {{[0-9]+}}(%rsp), %xmm1
+; SSE41-NEXT:    movd %r9d, %xmm2
+; SSE41-NEXT:    pinsrd $1, {{[0-9]+}}(%rsp), %xmm2
+; SSE41-NEXT:    movd {{.*#+}} xmm3 = mem[0],zero,zero,zero
+; SSE41-NEXT:    pinsrd $1, {{[0-9]+}}(%rsp), %xmm3
+; SSE41-NEXT:    pinsrd $2, {{[0-9]+}}(%rsp), %xmm3
+; SSE41-NEXT:    pinsrd $3, {{[0-9]+}}(%rsp), %xmm3
+; SSE41-NEXT:    movq {{[0-9]+}}(%rsp), %rcx
+; SSE41-NEXT:    paddd %xmm0, %xmm3
+; SSE41-NEXT:    pmaxud %xmm3, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm3, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm4, %xmm4
+; SSE41-NEXT:    pxor %xmm4, %xmm0
+; SSE41-NEXT:    paddd %xmm2, %xmm1
+; SSE41-NEXT:    pmaxud %xmm1, %xmm2
+; SSE41-NEXT:    pcmpeqd %xmm1, %xmm2
+; SSE41-NEXT:    pxor %xmm4, %xmm2
+; SSE41-NEXT:    movq %xmm1, 16(%rcx)
+; SSE41-NEXT:    movdqa %xmm3, (%rcx)
+; SSE41-NEXT:    movq %xmm2, 16(%rdi)
+; SSE41-NEXT:    movdqa %xmm0, (%rdi)
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: uaddo_v6i32:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
+; AVX1-NEXT:    vpaddd %xmm2, %xmm3, %xmm2
+; AVX1-NEXT:    vpmaxud %xmm3, %xmm2, %xmm3
+; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm2, %xmm3
+; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
+; AVX1-NEXT:    vpxor %xmm4, %xmm3, %xmm3
+; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm1
+; AVX1-NEXT:    vpmaxud %xmm0, %xmm1, %xmm0
+; AVX1-NEXT:    vpcmpeqd %xmm0, %xmm1, %xmm0
+; AVX1-NEXT:    vpxor %xmm4, %xmm0, %xmm0
+; AVX1-NEXT:    vpackssdw %xmm3, %xmm0, %xmm0
+; AVX1-NEXT:    vpmovsxwd %xmm0, %xmm3
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpmovsxwd %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm3, %ymm0
+; AVX1-NEXT:    vmovq %xmm2, 16(%rdi)
+; AVX1-NEXT:    vmovdqa %xmm1, (%rdi)
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: uaddo_v6i32:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpaddd %ymm1, %ymm0, %ymm1
+; AVX2-NEXT:    vpmaxud %ymm0, %ymm1, %ymm0
+; AVX2-NEXT:    vpcmpeqd %ymm0, %ymm1, %ymm0
+; AVX2-NEXT:    vpcmpeqd %ymm2, %ymm2, %ymm2
+; AVX2-NEXT:    vpxor %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm2
+; AVX2-NEXT:    vpackssdw %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    vpmovsxwd %xmm0, %ymm0
+; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm2
+; AVX2-NEXT:    vmovq %xmm2, 16(%rdi)
+; AVX2-NEXT:    vmovdqa %xmm1, (%rdi)
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: uaddo_v6i32:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpaddd %ymm1, %ymm0, %ymm1
+; AVX512-NEXT:    vpcmpltud %ymm0, %ymm1, %k1
+; AVX512-NEXT:    vpcmpeqd %ymm0, %ymm0, %ymm0
+; AVX512-NEXT:    vmovdqa32 %ymm0, %ymm0 {%k1} {z}
+; AVX512-NEXT:    vextracti128 $1, %ymm1, %xmm2
+; AVX512-NEXT:    vmovq %xmm2, 16(%rdi)
+; AVX512-NEXT:    vmovdqa %xmm1, (%rdi)
+; AVX512-NEXT:    retq
+  %t = call {<6 x i32>, <6 x i1>} @llvm.uadd.with.overflow.v6i32(<6 x i32> %a0, <6 x i32> %a1)
+  %val = extractvalue {<6 x i32>, <6 x i1>} %t, 0
+  %obit = extractvalue {<6 x i32>, <6 x i1>} %t, 1
+  %res = sext <6 x i1> %obit to <6 x i32>
+  store <6 x i32> %val, <6 x i32>* %p2
+  ret <6 x i32> %res
+}
+
+define <8 x i32> @uaddo_v8i32(<8 x i32> %a0, <8 x i32> %a1, <8 x i32>* %p2) nounwind {
+; SSE2-LABEL: uaddo_v8i32:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [2147483648,2147483648,2147483648,2147483648]
+; SSE2-NEXT:    paddd %xmm0, %xmm2
+; SSE2-NEXT:    pxor %xmm4, %xmm0
+; SSE2-NEXT:    movdqa %xmm2, (%rdi)
+; SSE2-NEXT:    pxor %xmm4, %xmm2
+; SSE2-NEXT:    pcmpgtd %xmm2, %xmm0
+; SSE2-NEXT:    paddd %xmm1, %xmm3
+; SSE2-NEXT:    pxor %xmm4, %xmm1
+; SSE2-NEXT:    pxor %xmm3, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm4, %xmm1
+; SSE2-NEXT:    movdqa %xmm3, 16(%rdi)
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: uaddo_v8i32:
+; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm4 = [2147483648,2147483648,2147483648,2147483648]
+; SSSE3-NEXT:    paddd %xmm0, %xmm2
+; SSSE3-NEXT:    pxor %xmm4, %xmm0
+; SSSE3-NEXT:    movdqa %xmm2, (%rdi)
+; SSSE3-NEXT:    pxor %xmm4, %xmm2
+; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm0
+; SSSE3-NEXT:    paddd %xmm1, %xmm3
+; SSSE3-NEXT:    pxor %xmm4, %xmm1
+; SSSE3-NEXT:    pxor %xmm3, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm1
+; SSSE3-NEXT:    movdqa %xmm3, 16(%rdi)
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: uaddo_v8i32:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    paddd %xmm0, %xmm2
+; SSE41-NEXT:    pmaxud %xmm2, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm2, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm4, %xmm4
+; SSE41-NEXT:    pxor %xmm4, %xmm0
+; SSE41-NEXT:    paddd %xmm1, %xmm3
+; SSE41-NEXT:    pmaxud %xmm3, %xmm1
+; SSE41-NEXT:    pcmpeqd %xmm3, %xmm1
+; SSE41-NEXT:    pxor %xmm4, %xmm1
+; SSE41-NEXT:    movdqa %xmm3, 16(%rdi)
+; SSE41-NEXT:    movdqa %xmm2, (%rdi)
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: uaddo_v8i32:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
+; AVX1-NEXT:    vpaddd %xmm2, %xmm3, %xmm2
+; AVX1-NEXT:    vpmaxud %xmm3, %xmm2, %xmm3
+; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm2, %xmm3
+; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
+; AVX1-NEXT:    vpxor %xmm4, %xmm3, %xmm3
+; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm1
+; AVX1-NEXT:    vpmaxud %xmm0, %xmm1, %xmm0
+; AVX1-NEXT:    vpcmpeqd %xmm0, %xmm1, %xmm0
+; AVX1-NEXT:    vpxor %xmm4, %xmm0, %xmm0
+; AVX1-NEXT:    vpackssdw %xmm3, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
+; AVX1-NEXT:    vpmovsxwd %xmm0, %xmm2
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpmovsxwd %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm2, %ymm0
+; AVX1-NEXT:    vmovaps %ymm1, (%rdi)
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: uaddo_v8i32:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpaddd %ymm1, %ymm0, %ymm1
+; AVX2-NEXT:    vpmaxud %ymm0, %ymm1, %ymm0
+; AVX2-NEXT:    vpcmpeqd %ymm0, %ymm1, %ymm0
+; AVX2-NEXT:    vpcmpeqd %ymm2, %ymm2, %ymm2
+; AVX2-NEXT:    vpxor %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm2
+; AVX2-NEXT:    vpackssdw %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    vpmovsxwd %xmm0, %ymm0
+; AVX2-NEXT:    vmovdqa %ymm1, (%rdi)
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: uaddo_v8i32:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpaddd %ymm1, %ymm0, %ymm1
+; AVX512-NEXT:    vpcmpltud %ymm0, %ymm1, %k1
+; AVX512-NEXT:    vpcmpeqd %ymm0, %ymm0, %ymm0
+; AVX512-NEXT:    vmovdqa32 %ymm0, %ymm0 {%k1} {z}
+; AVX512-NEXT:    vmovdqa %ymm1, (%rdi)
+; AVX512-NEXT:    retq
+  %t = call {<8 x i32>, <8 x i1>} @llvm.uadd.with.overflow.v8i32(<8 x i32> %a0, <8 x i32> %a1)
+  %val = extractvalue {<8 x i32>, <8 x i1>} %t, 0
+  %obit = extractvalue {<8 x i32>, <8 x i1>} %t, 1
+  %res = sext <8 x i1> %obit to <8 x i32>
+  store <8 x i32> %val, <8 x i32>* %p2
+  ret <8 x i32> %res
+}
+
+define <16 x i32> @uaddo_v16i32(<16 x i32> %a0, <16 x i32> %a1, <16 x i32>* %p2) nounwind {
+; SSE2-LABEL: uaddo_v16i32:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa {{.*#+}} xmm8 = [2147483648,2147483648,2147483648,2147483648]
+; SSE2-NEXT:    paddd %xmm0, %xmm4
+; SSE2-NEXT:    pxor %xmm8, %xmm0
+; SSE2-NEXT:    movdqa %xmm4, (%rdi)
+; SSE2-NEXT:    pxor %xmm8, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm4, %xmm0
+; SSE2-NEXT:    paddd %xmm1, %xmm5
+; SSE2-NEXT:    pxor %xmm8, %xmm1
+; SSE2-NEXT:    movdqa %xmm5, 16(%rdi)
+; SSE2-NEXT:    pxor %xmm8, %xmm5
+; SSE2-NEXT:    pcmpgtd %xmm5, %xmm1
+; SSE2-NEXT:    paddd %xmm2, %xmm6
+; SSE2-NEXT:    pxor %xmm8, %xmm2
+; SSE2-NEXT:    movdqa %xmm6, 32(%rdi)
+; SSE2-NEXT:    pxor %xmm8, %xmm6
+; SSE2-NEXT:    pcmpgtd %xmm6, %xmm2
+; SSE2-NEXT:    paddd %xmm3, %xmm7
+; SSE2-NEXT:    pxor %xmm8, %xmm3
+; SSE2-NEXT:    pxor %xmm7, %xmm8
+; SSE2-NEXT:    pcmpgtd %xmm8, %xmm3
+; SSE2-NEXT:    movdqa %xmm7, 48(%rdi)
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: uaddo_v16i32:
+; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm8 = [2147483648,2147483648,2147483648,2147483648]
+; SSSE3-NEXT:    paddd %xmm0, %xmm4
+; SSSE3-NEXT:    pxor %xmm8, %xmm0
+; SSSE3-NEXT:    movdqa %xmm4, (%rdi)
+; SSSE3-NEXT:    pxor %xmm8, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm0
+; SSSE3-NEXT:    paddd %xmm1, %xmm5
+; SSSE3-NEXT:    pxor %xmm8, %xmm1
+; SSSE3-NEXT:    movdqa %xmm5, 16(%rdi)
+; SSSE3-NEXT:    pxor %xmm8, %xmm5
+; SSSE3-NEXT:    pcmpgtd %xmm5, %xmm1
+; SSSE3-NEXT:    paddd %xmm2, %xmm6
+; SSSE3-NEXT:    pxor %xmm8, %xmm2
+; SSSE3-NEXT:    movdqa %xmm6, 32(%rdi)
+; SSSE3-NEXT:    pxor %xmm8, %xmm6
+; SSSE3-NEXT:    pcmpgtd %xmm6, %xmm2
+; SSSE3-NEXT:    paddd %xmm3, %xmm7
+; SSSE3-NEXT:    pxor %xmm8, %xmm3
+; SSSE3-NEXT:    pxor %xmm7, %xmm8
+; SSSE3-NEXT:    pcmpgtd %xmm8, %xmm3
+; SSSE3-NEXT:    movdqa %xmm7, 48(%rdi)
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: uaddo_v16i32:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    paddd %xmm0, %xmm4
+; SSE41-NEXT:    pmaxud %xmm4, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm4, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm8, %xmm8
+; SSE41-NEXT:    pxor %xmm8, %xmm0
+; SSE41-NEXT:    paddd %xmm1, %xmm5
+; SSE41-NEXT:    pmaxud %xmm5, %xmm1
+; SSE41-NEXT:    pcmpeqd %xmm5, %xmm1
+; SSE41-NEXT:    pxor %xmm8, %xmm1
+; SSE41-NEXT:    paddd %xmm2, %xmm6
+; SSE41-NEXT:    pmaxud %xmm6, %xmm2
+; SSE41-NEXT:    pcmpeqd %xmm6, %xmm2
+; SSE41-NEXT:    pxor %xmm8, %xmm2
+; SSE41-NEXT:    paddd %xmm3, %xmm7
+; SSE41-NEXT:    pmaxud %xmm7, %xmm3
+; SSE41-NEXT:    pcmpeqd %xmm7, %xmm3
+; SSE41-NEXT:    pxor %xmm8, %xmm3
+; SSE41-NEXT:    movdqa %xmm7, 48(%rdi)
+; SSE41-NEXT:    movdqa %xmm6, 32(%rdi)
+; SSE41-NEXT:    movdqa %xmm5, 16(%rdi)
+; SSE41-NEXT:    movdqa %xmm4, (%rdi)
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: uaddo_v16i32:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm4
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm5
+; AVX1-NEXT:    vpaddd %xmm4, %xmm5, %xmm4
+; AVX1-NEXT:    vpmaxud %xmm5, %xmm4, %xmm5
+; AVX1-NEXT:    vpcmpeqd %xmm5, %xmm4, %xmm5
+; AVX1-NEXT:    vpcmpeqd %xmm6, %xmm6, %xmm6
+; AVX1-NEXT:    vpxor %xmm6, %xmm5, %xmm5
+; AVX1-NEXT:    vpaddd %xmm3, %xmm1, %xmm3
+; AVX1-NEXT:    vpmaxud %xmm1, %xmm3, %xmm1
+; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm3, %xmm1
+; AVX1-NEXT:    vpxor %xmm6, %xmm1, %xmm1
+; AVX1-NEXT:    vpackssdw %xmm5, %xmm1, %xmm1
+; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm5
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm7
+; AVX1-NEXT:    vpaddd %xmm5, %xmm7, %xmm5
+; AVX1-NEXT:    vpmaxud %xmm7, %xmm5, %xmm7
+; AVX1-NEXT:    vpcmpeqd %xmm7, %xmm5, %xmm7
+; AVX1-NEXT:    vpxor %xmm6, %xmm7, %xmm7
+; AVX1-NEXT:    vpaddd %xmm2, %xmm0, %xmm2
+; AVX1-NEXT:    vpmaxud %xmm0, %xmm2, %xmm0
+; AVX1-NEXT:    vpcmpeqd %xmm0, %xmm2, %xmm0
+; AVX1-NEXT:    vpxor %xmm6, %xmm0, %xmm0
+; AVX1-NEXT:    vpackssdw %xmm7, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm5, %ymm2, %ymm2
+; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm3, %ymm3
+; AVX1-NEXT:    vpmovsxwd %xmm0, %xmm4
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpmovsxwd %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm4, %ymm0
+; AVX1-NEXT:    vpmovsxwd %xmm1, %xmm4
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; AVX1-NEXT:    vpmovsxwd %xmm1, %xmm1
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm4, %ymm1
+; AVX1-NEXT:    vmovaps %ymm3, 32(%rdi)
+; AVX1-NEXT:    vmovaps %ymm2, (%rdi)
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: uaddo_v16i32:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpaddd %ymm3, %ymm1, %ymm3
+; AVX2-NEXT:    vpmaxud %ymm1, %ymm3, %ymm1
+; AVX2-NEXT:    vpcmpeqd %ymm1, %ymm3, %ymm1
+; AVX2-NEXT:    vpcmpeqd %ymm4, %ymm4, %ymm4
+; AVX2-NEXT:    vpxor %ymm4, %ymm1, %ymm1
+; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm5
+; AVX2-NEXT:    vpackssdw %xmm5, %xmm1, %xmm1
+; AVX2-NEXT:    vpaddd %ymm2, %ymm0, %ymm2
+; AVX2-NEXT:    vpmaxud %ymm0, %ymm2, %ymm0
+; AVX2-NEXT:    vpcmpeqd %ymm0, %ymm2, %ymm0
+; AVX2-NEXT:    vpxor %ymm4, %ymm0, %ymm0
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm4
+; AVX2-NEXT:    vpackssdw %xmm4, %xmm0, %xmm0
+; AVX2-NEXT:    vpmovsxwd %xmm0, %ymm0
+; AVX2-NEXT:    vpmovsxwd %xmm1, %ymm1
+; AVX2-NEXT:    vmovdqa %ymm3, 32(%rdi)
+; AVX2-NEXT:    vmovdqa %ymm2, (%rdi)
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: uaddo_v16i32:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpaddd %zmm1, %zmm0, %zmm1
+; AVX512-NEXT:    vpcmpltud %zmm0, %zmm1, %k1
+; AVX512-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; AVX512-NEXT:    vmovdqa64 %zmm1, (%rdi)
+; AVX512-NEXT:    retq
+  %t = call {<16 x i32>, <16 x i1>} @llvm.uadd.with.overflow.v16i32(<16 x i32> %a0, <16 x i32> %a1)
+  %val = extractvalue {<16 x i32>, <16 x i1>} %t, 0
+  %obit = extractvalue {<16 x i32>, <16 x i1>} %t, 1
+  %res = sext <16 x i1> %obit to <16 x i32>
+  store <16 x i32> %val, <16 x i32>* %p2
+  ret <16 x i32> %res
+}
+
+define <16 x i32> @uaddo_v16i8(<16 x i8> %a0, <16 x i8> %a1, <16 x i8>* %p2) nounwind {
+; SSE2-LABEL: uaddo_v16i8:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    paddb %xmm0, %xmm1
+; SSE2-NEXT:    pmaxub %xmm1, %xmm0
+; SSE2-NEXT:    pcmpeqb %xmm1, %xmm0
+; SSE2-NEXT:    pcmpeqd %xmm3, %xmm3
+; SSE2-NEXT:    pxor %xmm0, %xmm3
+; SSE2-NEXT:    movdqa %xmm3, %xmm4
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3],xmm4[4],xmm0[4],xmm4[5],xmm0[5],xmm4[6],xmm0[6],xmm4[7],xmm0[7]
+; SSE2-NEXT:    movdqa %xmm4, %xmm0
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
+; SSE2-NEXT:    pslld $31, %xmm0
+; SSE2-NEXT:    psrad $31, %xmm0
+; SSE2-NEXT:    punpckhwd {{.*#+}} xmm4 = xmm4[4],xmm0[4],xmm4[5],xmm0[5],xmm4[6],xmm0[6],xmm4[7],xmm0[7]
+; SSE2-NEXT:    pslld $31, %xmm4
+; SSE2-NEXT:    psrad $31, %xmm4
+; SSE2-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm0[8],xmm3[9],xmm0[9],xmm3[10],xmm0[10],xmm3[11],xmm0[11],xmm3[12],xmm0[12],xmm3[13],xmm0[13],xmm3[14],xmm0[14],xmm3[15],xmm0[15]
+; SSE2-NEXT:    movdqa %xmm3, %xmm2
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
+; SSE2-NEXT:    pslld $31, %xmm2
+; SSE2-NEXT:    psrad $31, %xmm2
+; SSE2-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
+; SSE2-NEXT:    pslld $31, %xmm3
+; SSE2-NEXT:    psrad $31, %xmm3
+; SSE2-NEXT:    movdqa %xmm1, (%rdi)
+; SSE2-NEXT:    movdqa %xmm4, %xmm1
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: uaddo_v16i8:
+; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    paddb %xmm0, %xmm1
+; SSSE3-NEXT:    pmaxub %xmm1, %xmm0
+; SSSE3-NEXT:    pcmpeqb %xmm1, %xmm0
+; SSSE3-NEXT:    pcmpeqd %xmm3, %xmm3
+; SSSE3-NEXT:    pxor %xmm0, %xmm3
+; SSSE3-NEXT:    movdqa %xmm3, %xmm4
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3],xmm4[4],xmm0[4],xmm4[5],xmm0[5],xmm4[6],xmm0[6],xmm4[7],xmm0[7]
+; SSSE3-NEXT:    movdqa %xmm4, %xmm0
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
+; SSSE3-NEXT:    pslld $31, %xmm0
+; SSSE3-NEXT:    psrad $31, %xmm0
+; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm4 = xmm4[4],xmm0[4],xmm4[5],xmm0[5],xmm4[6],xmm0[6],xmm4[7],xmm0[7]
+; SSSE3-NEXT:    pslld $31, %xmm4
+; SSSE3-NEXT:    psrad $31, %xmm4
+; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm0[8],xmm3[9],xmm0[9],xmm3[10],xmm0[10],xmm3[11],xmm0[11],xmm3[12],xmm0[12],xmm3[13],xmm0[13],xmm3[14],xmm0[14],xmm3[15],xmm0[15]
+; SSSE3-NEXT:    movdqa %xmm3, %xmm2
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
+; SSSE3-NEXT:    pslld $31, %xmm2
+; SSSE3-NEXT:    psrad $31, %xmm2
+; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
+; SSSE3-NEXT:    pslld $31, %xmm3
+; SSSE3-NEXT:    psrad $31, %xmm3
+; SSSE3-NEXT:    movdqa %xmm1, (%rdi)
+; SSSE3-NEXT:    movdqa %xmm4, %xmm1
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: uaddo_v16i8:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    paddb %xmm0, %xmm1
+; SSE41-NEXT:    pmaxub %xmm1, %xmm0
+; SSE41-NEXT:    pcmpeqb %xmm1, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm3, %xmm3
+; SSE41-NEXT:    pxor %xmm0, %xmm3
+; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
+; SSE41-NEXT:    pslld $31, %xmm0
+; SSE41-NEXT:    psrad $31, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[1,1,2,3]
+; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm4 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero
+; SSE41-NEXT:    pslld $31, %xmm4
+; SSE41-NEXT:    psrad $31, %xmm4
+; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[2,3,0,1]
+; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero
+; SSE41-NEXT:    pslld $31, %xmm2
+; SSE41-NEXT:    psrad $31, %xmm2
+; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[3,1,2,3]
+; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
+; SSE41-NEXT:    pslld $31, %xmm3
+; SSE41-NEXT:    psrad $31, %xmm3
+; SSE41-NEXT:    movdqa %xmm1, (%rdi)
+; SSE41-NEXT:    movdqa %xmm4, %xmm1
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: uaddo_v16i8:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpaddb %xmm1, %xmm0, %xmm2
+; AVX1-NEXT:    vpmaxub %xmm0, %xmm2, %xmm0
+; AVX1-NEXT:    vpcmpeqb %xmm0, %xmm2, %xmm0
+; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
+; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm1
+; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; AVX1-NEXT:    vpmovzxwd {{.*#+}} xmm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; AVX1-NEXT:    vpslld $31, %xmm3, %xmm3
+; AVX1-NEXT:    vpsrad $31, %xmm3, %xmm3
+; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm0[4,4,5,5,6,6,7,7]
+; AVX1-NEXT:    vpslld $31, %xmm0, %xmm0
+; AVX1-NEXT:    vpsrad $31, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm3, %ymm0
+; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
+; AVX1-NEXT:    vpmovzxwd {{.*#+}} xmm3 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
+; AVX1-NEXT:    vpslld $31, %xmm3, %xmm3
+; AVX1-NEXT:    vpsrad $31, %xmm3, %xmm3
+; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; AVX1-NEXT:    vpslld $31, %xmm1, %xmm1
+; AVX1-NEXT:    vpsrad $31, %xmm1, %xmm1
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm3, %ymm1
+; AVX1-NEXT:    vmovdqa %xmm2, (%rdi)
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: uaddo_v16i8:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpaddb %xmm1, %xmm0, %xmm2
+; AVX2-NEXT:    vpmaxub %xmm0, %xmm2, %xmm0
+; AVX2-NEXT:    vpcmpeqb %xmm0, %xmm2, %xmm0
+; AVX2-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
+; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm1
+; AVX2-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX2-NEXT:    vpslld $31, %ymm0, %ymm0
+; AVX2-NEXT:    vpsrad $31, %ymm0, %ymm0
+; AVX2-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
+; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; AVX2-NEXT:    vpslld $31, %ymm1, %ymm1
+; AVX2-NEXT:    vpsrad $31, %ymm1, %ymm1
+; AVX2-NEXT:    vmovdqa %xmm2, (%rdi)
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: uaddo_v16i8:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpaddb %xmm1, %xmm0, %xmm1
+; AVX512-NEXT:    vpcmpltub %xmm0, %xmm1, %k1
+; AVX512-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; AVX512-NEXT:    vmovdqa %xmm1, (%rdi)
+; AVX512-NEXT:    retq
+  %t = call {<16 x i8>, <16 x i1>} @llvm.uadd.with.overflow.v16i8(<16 x i8> %a0, <16 x i8> %a1)
+  %val = extractvalue {<16 x i8>, <16 x i1>} %t, 0
+  %obit = extractvalue {<16 x i8>, <16 x i1>} %t, 1
+  %res = sext <16 x i1> %obit to <16 x i32>
+  store <16 x i8> %val, <16 x i8>* %p2
+  ret <16 x i32> %res
+}
+
+define <8 x i32> @uaddo_v8i16(<8 x i16> %a0, <8 x i16> %a1, <8 x i16>* %p2) nounwind {
+; SSE2-LABEL: uaddo_v8i16:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa %xmm0, %xmm2
+; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [32768,32768,32768,32768,32768,32768,32768,32768]
+; SSE2-NEXT:    paddw %xmm0, %xmm1
+; SSE2-NEXT:    pxor %xmm3, %xmm2
+; SSE2-NEXT:    pxor %xmm1, %xmm3
+; SSE2-NEXT:    pcmpgtw %xmm3, %xmm2
+; SSE2-NEXT:    movdqa %xmm2, %xmm0
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
+; SSE2-NEXT:    pslld $31, %xmm0
+; SSE2-NEXT:    psrad $31, %xmm0
+; SSE2-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; SSE2-NEXT:    pslld $31, %xmm2
+; SSE2-NEXT:    psrad $31, %xmm2
+; SSE2-NEXT:    movdqa %xmm1, (%rdi)
+; SSE2-NEXT:    movdqa %xmm2, %xmm1
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: uaddo_v8i16:
+; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    movdqa %xmm0, %xmm2
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm3 = [32768,32768,32768,32768,32768,32768,32768,32768]
+; SSSE3-NEXT:    paddw %xmm0, %xmm1
+; SSSE3-NEXT:    pxor %xmm3, %xmm2
+; SSSE3-NEXT:    pxor %xmm1, %xmm3
+; SSSE3-NEXT:    pcmpgtw %xmm3, %xmm2
+; SSSE3-NEXT:    movdqa %xmm2, %xmm0
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
+; SSSE3-NEXT:    pslld $31, %xmm0
+; SSSE3-NEXT:    psrad $31, %xmm0
+; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; SSSE3-NEXT:    pslld $31, %xmm2
+; SSSE3-NEXT:    psrad $31, %xmm2
+; SSSE3-NEXT:    movdqa %xmm1, (%rdi)
+; SSSE3-NEXT:    movdqa %xmm2, %xmm1
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: uaddo_v8i16:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    paddw %xmm0, %xmm1
+; SSE41-NEXT:    pmaxuw %xmm1, %xmm0
+; SSE41-NEXT:    pcmpeqw %xmm1, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm2, %xmm2
+; SSE41-NEXT:    pxor %xmm0, %xmm2
+; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm0 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero
+; SSE41-NEXT:    pslld $31, %xmm0
+; SSE41-NEXT:    psrad $31, %xmm0
+; SSE41-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; SSE41-NEXT:    pslld $31, %xmm2
+; SSE41-NEXT:    psrad $31, %xmm2
+; SSE41-NEXT:    movdqa %xmm1, (%rdi)
+; SSE41-NEXT:    movdqa %xmm2, %xmm1
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: uaddo_v8i16:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpaddw %xmm1, %xmm0, %xmm1
+; AVX1-NEXT:    vpmaxuw %xmm0, %xmm1, %xmm0
+; AVX1-NEXT:    vpcmpeqw %xmm0, %xmm1, %xmm0
+; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
+; AVX1-NEXT:    vpxor %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpmovsxwd %xmm0, %xmm2
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpmovsxwd %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm2, %ymm0
+; AVX1-NEXT:    vmovdqa %xmm1, (%rdi)
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: uaddo_v8i16:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpaddw %xmm1, %xmm0, %xmm1
+; AVX2-NEXT:    vpmaxuw %xmm0, %xmm1, %xmm0
+; AVX2-NEXT:    vpcmpeqw %xmm0, %xmm1, %xmm0
+; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
+; AVX2-NEXT:    vpxor %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    vpmovsxwd %xmm0, %ymm0
+; AVX2-NEXT:    vmovdqa %xmm1, (%rdi)
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: uaddo_v8i16:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpaddw %xmm1, %xmm0, %xmm1
+; AVX512-NEXT:    vpcmpltuw %xmm0, %xmm1, %k1
+; AVX512-NEXT:    vpcmpeqd %ymm0, %ymm0, %ymm0
+; AVX512-NEXT:    vmovdqa32 %ymm0, %ymm0 {%k1} {z}
+; AVX512-NEXT:    vmovdqa %xmm1, (%rdi)
+; AVX512-NEXT:    retq
+  %t = call {<8 x i16>, <8 x i1>} @llvm.uadd.with.overflow.v8i16(<8 x i16> %a0, <8 x i16> %a1)
+  %val = extractvalue {<8 x i16>, <8 x i1>} %t, 0
+  %obit = extractvalue {<8 x i16>, <8 x i1>} %t, 1
+  %res = sext <8 x i1> %obit to <8 x i32>
+  store <8 x i16> %val, <8 x i16>* %p2
+  ret <8 x i32> %res
+}
+
+define <2 x i32> @uaddo_v2i64(<2 x i64> %a0, <2 x i64> %a1, <2 x i64>* %p2) nounwind {
+; SSE-LABEL: uaddo_v2i64:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movdqa {{.*#+}} xmm2 = [9223372039002259456,9223372039002259456]
+; SSE-NEXT:    paddq %xmm0, %xmm1
+; SSE-NEXT:    pxor %xmm2, %xmm0
+; SSE-NEXT:    pxor %xmm1, %xmm2
+; SSE-NEXT:    movdqa %xmm0, %xmm3
+; SSE-NEXT:    pcmpgtd %xmm2, %xmm3
+; SSE-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[0,0,2,2]
+; SSE-NEXT:    pcmpeqd %xmm0, %xmm2
+; SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; SSE-NEXT:    pand %xmm4, %xmm2
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
+; SSE-NEXT:    por %xmm2, %xmm0
+; SSE-NEXT:    movdqa %xmm1, (%rdi)
+; SSE-NEXT:    retq
+;
+; AVX1-LABEL: uaddo_v2i64:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [9223372036854775808,9223372036854775808]
+; AVX1-NEXT:    vpxor %xmm2, %xmm0, %xmm3
+; AVX1-NEXT:    vpaddq %xmm1, %xmm0, %xmm1
+; AVX1-NEXT:    vpxor %xmm2, %xmm1, %xmm0
+; AVX1-NEXT:    vpcmpgtq %xmm0, %xmm3, %xmm0
+; AVX1-NEXT:    vmovdqa %xmm1, (%rdi)
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: uaddo_v2i64:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vmovdqa {{.*#+}} xmm2 = [9223372036854775808,9223372036854775808]
+; AVX2-NEXT:    vpxor %xmm2, %xmm0, %xmm3
+; AVX2-NEXT:    vpaddq %xmm1, %xmm0, %xmm1
+; AVX2-NEXT:    vpxor %xmm2, %xmm1, %xmm0
+; AVX2-NEXT:    vpcmpgtq %xmm0, %xmm3, %xmm0
+; AVX2-NEXT:    vmovdqa %xmm1, (%rdi)
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: uaddo_v2i64:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpaddq %xmm1, %xmm0, %xmm1
+; AVX512-NEXT:    vpcmpltuq %xmm0, %xmm1, %k1
+; AVX512-NEXT:    vmovdqa %xmm1, (%rdi)
+; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX512-NEXT:    vmovdqa64 %xmm0, %xmm0 {%k1} {z}
+; AVX512-NEXT:    retq
+  %t = call {<2 x i64>, <2 x i1>} @llvm.uadd.with.overflow.v2i64(<2 x i64> %a0, <2 x i64> %a1)
+  %val = extractvalue {<2 x i64>, <2 x i1>} %t, 0
+  %obit = extractvalue {<2 x i64>, <2 x i1>} %t, 1
+  %res = sext <2 x i1> %obit to <2 x i32>
+  store <2 x i64> %val, <2 x i64>* %p2
+  ret <2 x i32> %res
+}
+
+define <4 x i32> @uaddo_v4i24(<4 x i24> %a0, <4 x i24> %a1, <4 x i24>* %p2) nounwind {
+; SSE2-LABEL: uaddo_v4i24:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa %xmm0, %xmm2
+; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [255,255,255,0,255,255,255,0,255,255,255,0,255,255,255,0]
+; SSE2-NEXT:    pand %xmm3, %xmm1
+; SSE2-NEXT:    pand %xmm3, %xmm2
+; SSE2-NEXT:    paddd %xmm1, %xmm2
+; SSE2-NEXT:    pand %xmm2, %xmm3
+; SSE2-NEXT:    pcmpeqd %xmm2, %xmm3
+; SSE2-NEXT:    pcmpeqd %xmm0, %xmm0
+; SSE2-NEXT:    pxor %xmm3, %xmm0
+; SSE2-NEXT:    movd %xmm2, %eax
+; SSE2-NEXT:    movw %ax, (%rdi)
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[3,1,2,3]
+; SSE2-NEXT:    movd %xmm1, %ecx
+; SSE2-NEXT:    movw %cx, 9(%rdi)
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[2,3,0,1]
+; SSE2-NEXT:    movd %xmm1, %edx
+; SSE2-NEXT:    movw %dx, 6(%rdi)
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,1,2,3]
+; SSE2-NEXT:    movd %xmm1, %esi
+; SSE2-NEXT:    movw %si, 3(%rdi)
+; SSE2-NEXT:    shrl $16, %eax
+; SSE2-NEXT:    movb %al, 2(%rdi)
+; SSE2-NEXT:    shrl $16, %ecx
+; SSE2-NEXT:    movb %cl, 11(%rdi)
+; SSE2-NEXT:    shrl $16, %edx
+; SSE2-NEXT:    movb %dl, 8(%rdi)
+; SSE2-NEXT:    shrl $16, %esi
+; SSE2-NEXT:    movb %sil, 5(%rdi)
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: uaddo_v4i24:
+; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    movdqa %xmm0, %xmm2
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm3 = [255,255,255,0,255,255,255,0,255,255,255,0,255,255,255,0]
+; SSSE3-NEXT:    pand %xmm3, %xmm1
+; SSSE3-NEXT:    pand %xmm3, %xmm2
+; SSSE3-NEXT:    paddd %xmm1, %xmm2
+; SSSE3-NEXT:    pand %xmm2, %xmm3
+; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm3
+; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm0
+; SSSE3-NEXT:    pxor %xmm3, %xmm0
+; SSSE3-NEXT:    movd %xmm2, %eax
+; SSSE3-NEXT:    movw %ax, (%rdi)
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[3,1,2,3]
+; SSSE3-NEXT:    movd %xmm1, %ecx
+; SSSE3-NEXT:    movw %cx, 9(%rdi)
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[2,3,0,1]
+; SSSE3-NEXT:    movd %xmm1, %edx
+; SSSE3-NEXT:    movw %dx, 6(%rdi)
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,1,2,3]
+; SSSE3-NEXT:    movd %xmm1, %esi
+; SSSE3-NEXT:    movw %si, 3(%rdi)
+; SSSE3-NEXT:    shrl $16, %eax
+; SSSE3-NEXT:    movb %al, 2(%rdi)
+; SSSE3-NEXT:    shrl $16, %ecx
+; SSSE3-NEXT:    movb %cl, 11(%rdi)
+; SSSE3-NEXT:    shrl $16, %edx
+; SSSE3-NEXT:    movb %dl, 8(%rdi)
+; SSSE3-NEXT:    shrl $16, %esi
+; SSSE3-NEXT:    movb %sil, 5(%rdi)
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: uaddo_v4i24:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movdqa {{.*#+}} xmm2 = [255,255,255,0,255,255,255,0,255,255,255,0,255,255,255,0]
+; SSE41-NEXT:    pand %xmm2, %xmm1
+; SSE41-NEXT:    pand %xmm2, %xmm0
+; SSE41-NEXT:    paddd %xmm1, %xmm0
+; SSE41-NEXT:    pand %xmm0, %xmm2
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm2
+; SSE41-NEXT:    pcmpeqd %xmm1, %xmm1
+; SSE41-NEXT:    pxor %xmm2, %xmm1
+; SSE41-NEXT:    pextrd $3, %xmm0, %eax
+; SSE41-NEXT:    movw %ax, 9(%rdi)
+; SSE41-NEXT:    pextrd $2, %xmm0, %ecx
+; SSE41-NEXT:    movw %cx, 6(%rdi)
+; SSE41-NEXT:    pextrd $1, %xmm0, %edx
+; SSE41-NEXT:    movw %dx, 3(%rdi)
+; SSE41-NEXT:    movd %xmm0, %esi
+; SSE41-NEXT:    movw %si, (%rdi)
+; SSE41-NEXT:    shrl $16, %eax
+; SSE41-NEXT:    movb %al, 11(%rdi)
+; SSE41-NEXT:    shrl $16, %ecx
+; SSE41-NEXT:    movb %cl, 8(%rdi)
+; SSE41-NEXT:    shrl $16, %edx
+; SSE41-NEXT:    movb %dl, 5(%rdi)
+; SSE41-NEXT:    shrl $16, %esi
+; SSE41-NEXT:    movb %sil, 2(%rdi)
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: uaddo_v4i24:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vbroadcastss {{.*#+}} xmm2 = [2.35098856E-38,2.35098856E-38,2.35098856E-38,2.35098856E-38]
+; AVX1-NEXT:    vandps %xmm2, %xmm1, %xmm1
+; AVX1-NEXT:    vandps %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm1
+; AVX1-NEXT:    vpand %xmm2, %xmm1, %xmm0
+; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
+; AVX1-NEXT:    vpxor %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpextrd $3, %xmm1, %eax
+; AVX1-NEXT:    movw %ax, 9(%rdi)
+; AVX1-NEXT:    vpextrd $2, %xmm1, %ecx
+; AVX1-NEXT:    movw %cx, 6(%rdi)
+; AVX1-NEXT:    vpextrd $1, %xmm1, %edx
+; AVX1-NEXT:    movw %dx, 3(%rdi)
+; AVX1-NEXT:    vmovd %xmm1, %esi
+; AVX1-NEXT:    movw %si, (%rdi)
+; AVX1-NEXT:    shrl $16, %eax
+; AVX1-NEXT:    movb %al, 11(%rdi)
+; AVX1-NEXT:    shrl $16, %ecx
+; AVX1-NEXT:    movb %cl, 8(%rdi)
+; AVX1-NEXT:    shrl $16, %edx
+; AVX1-NEXT:    movb %dl, 5(%rdi)
+; AVX1-NEXT:    shrl $16, %esi
+; AVX1-NEXT:    movb %sil, 2(%rdi)
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: uaddo_v4i24:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [16777215,16777215,16777215,16777215]
+; AVX2-NEXT:    vpand %xmm2, %xmm1, %xmm1
+; AVX2-NEXT:    vpand %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm1
+; AVX2-NEXT:    vpand %xmm2, %xmm1, %xmm0
+; AVX2-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
+; AVX2-NEXT:    vpxor %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    vpextrd $3, %xmm1, %eax
+; AVX2-NEXT:    movw %ax, 9(%rdi)
+; AVX2-NEXT:    vpextrd $2, %xmm1, %ecx
+; AVX2-NEXT:    movw %cx, 6(%rdi)
+; AVX2-NEXT:    vpextrd $1, %xmm1, %edx
+; AVX2-NEXT:    movw %dx, 3(%rdi)
+; AVX2-NEXT:    vmovd %xmm1, %esi
+; AVX2-NEXT:    movw %si, (%rdi)
+; AVX2-NEXT:    shrl $16, %eax
+; AVX2-NEXT:    movb %al, 11(%rdi)
+; AVX2-NEXT:    shrl $16, %ecx
+; AVX2-NEXT:    movb %cl, 8(%rdi)
+; AVX2-NEXT:    shrl $16, %edx
+; AVX2-NEXT:    movb %dl, 5(%rdi)
+; AVX2-NEXT:    shrl $16, %esi
+; AVX2-NEXT:    movb %sil, 2(%rdi)
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: uaddo_v4i24:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [16777215,16777215,16777215,16777215]
+; AVX512-NEXT:    vpand %xmm2, %xmm1, %xmm1
+; AVX512-NEXT:    vpand %xmm2, %xmm0, %xmm0
+; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm1
+; AVX512-NEXT:    vpand %xmm2, %xmm1, %xmm0
+; AVX512-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vpternlogq $15, %xmm0, %xmm0, %xmm0
+; AVX512-NEXT:    vpextrd $3, %xmm1, %eax
+; AVX512-NEXT:    movw %ax, 9(%rdi)
+; AVX512-NEXT:    vpextrd $2, %xmm1, %ecx
+; AVX512-NEXT:    movw %cx, 6(%rdi)
+; AVX512-NEXT:    vpextrd $1, %xmm1, %edx
+; AVX512-NEXT:    movw %dx, 3(%rdi)
+; AVX512-NEXT:    vmovd %xmm1, %esi
+; AVX512-NEXT:    movw %si, (%rdi)
+; AVX512-NEXT:    shrl $16, %eax
+; AVX512-NEXT:    movb %al, 11(%rdi)
+; AVX512-NEXT:    shrl $16, %ecx
+; AVX512-NEXT:    movb %cl, 8(%rdi)
+; AVX512-NEXT:    shrl $16, %edx
+; AVX512-NEXT:    movb %dl, 5(%rdi)
+; AVX512-NEXT:    shrl $16, %esi
+; AVX512-NEXT:    movb %sil, 2(%rdi)
+; AVX512-NEXT:    retq
+  %t = call {<4 x i24>, <4 x i1>} @llvm.uadd.with.overflow.v4i24(<4 x i24> %a0, <4 x i24> %a1)
+  %val = extractvalue {<4 x i24>, <4 x i1>} %t, 0
+  %obit = extractvalue {<4 x i24>, <4 x i1>} %t, 1
+  %res = sext <4 x i1> %obit to <4 x i32>
+  store <4 x i24> %val, <4 x i24>* %p2
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @uaddo_v4i1(<4 x i1> %a0, <4 x i1> %a1, <4 x i1>* %p2) nounwind {
+; SSE-LABEL: uaddo_v4i1:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movdqa {{.*#+}} xmm2 = [1,1,1,1]
+; SSE-NEXT:    pand %xmm2, %xmm1
+; SSE-NEXT:    pand %xmm2, %xmm0
+; SSE-NEXT:    paddd %xmm1, %xmm0
+; SSE-NEXT:    pand %xmm0, %xmm2
+; SSE-NEXT:    pcmpeqd %xmm0, %xmm2
+; SSE-NEXT:    pcmpeqd %xmm1, %xmm1
+; SSE-NEXT:    pxor %xmm2, %xmm1
+; SSE-NEXT:    pslld $31, %xmm0
+; SSE-NEXT:    movmskps %xmm0, %eax
+; SSE-NEXT:    movb %al, (%rdi)
+; SSE-NEXT:    movdqa %xmm1, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX1-LABEL: uaddo_v4i1:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [1,1,1,1]
+; AVX1-NEXT:    vpand %xmm2, %xmm1, %xmm1
+; AVX1-NEXT:    vpand %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm1
+; AVX1-NEXT:    vpand %xmm2, %xmm1, %xmm0
+; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
+; AVX1-NEXT:    vpxor %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpslld $31, %xmm1, %xmm1
+; AVX1-NEXT:    vmovmskps %xmm1, %eax
+; AVX1-NEXT:    movb %al, (%rdi)
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: uaddo_v4i1:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [1,1,1,1]
+; AVX2-NEXT:    vpand %xmm2, %xmm1, %xmm1
+; AVX2-NEXT:    vpand %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm1
+; AVX2-NEXT:    vpand %xmm2, %xmm1, %xmm0
+; AVX2-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
+; AVX2-NEXT:    vpxor %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    vpslld $31, %xmm1, %xmm1
+; AVX2-NEXT:    vmovmskps %xmm1, %eax
+; AVX2-NEXT:    movb %al, (%rdi)
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: uaddo_v4i1:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpslld $31, %xmm0, %xmm0
+; AVX512-NEXT:    vptestmd %xmm0, %xmm0, %k0
+; AVX512-NEXT:    vpslld $31, %xmm1, %xmm1
+; AVX512-NEXT:    vptestmd %xmm1, %xmm1, %k1
+; AVX512-NEXT:    kxorw %k1, %k0, %k2
+; AVX512-NEXT:    kxnorw %k1, %k0, %k1
+; AVX512-NEXT:    vptestmd %xmm0, %xmm0, %k1 {%k1}
+; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
+; AVX512-NEXT:    kmovd %k2, %eax
+; AVX512-NEXT:    movb %al, (%rdi)
+; AVX512-NEXT:    retq
+  %t = call {<4 x i1>, <4 x i1>} @llvm.uadd.with.overflow.v4i1(<4 x i1> %a0, <4 x i1> %a1)
+  %val = extractvalue {<4 x i1>, <4 x i1>} %t, 0
+  %obit = extractvalue {<4 x i1>, <4 x i1>} %t, 1
+  %res = sext <4 x i1> %obit to <4 x i32>
+  store <4 x i1> %val, <4 x i1>* %p2
+  ret <4 x i32> %res
+}
+
+define <2 x i32> @uaddo_v2i128(<2 x i128> %a0, <2 x i128> %a1, <2 x i128>* %p2) nounwind {
+; SSE2-LABEL: uaddo_v2i128:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movq {{[0-9]+}}(%rsp), %r10
+; SSE2-NEXT:    addq {{[0-9]+}}(%rsp), %rdx
+; SSE2-NEXT:    adcq {{[0-9]+}}(%rsp), %rcx
+; SSE2-NEXT:    setb %al
+; SSE2-NEXT:    movzbl %al, %r11d
+; SSE2-NEXT:    addq %r8, %rdi
+; SSE2-NEXT:    adcq %r9, %rsi
+; SSE2-NEXT:    setb %al
+; SSE2-NEXT:    movzbl %al, %eax
+; SSE2-NEXT:    movd %eax, %xmm0
+; SSE2-NEXT:    pinsrw $4, %r11d, %xmm0
+; SSE2-NEXT:    movq %rdx, 16(%r10)
+; SSE2-NEXT:    movq %rdi, (%r10)
+; SSE2-NEXT:    movq %rcx, 24(%r10)
+; SSE2-NEXT:    movq %rsi, 8(%r10)
+; SSE2-NEXT:    psllq $63, %xmm0
+; SSE2-NEXT:    psrad $31, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: uaddo_v2i128:
+; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    movq {{[0-9]+}}(%rsp), %r10
+; SSSE3-NEXT:    addq {{[0-9]+}}(%rsp), %rdx
+; SSSE3-NEXT:    adcq {{[0-9]+}}(%rsp), %rcx
+; SSSE3-NEXT:    setb %al
+; SSSE3-NEXT:    movzbl %al, %r11d
+; SSSE3-NEXT:    addq %r8, %rdi
+; SSSE3-NEXT:    adcq %r9, %rsi
+; SSSE3-NEXT:    setb %al
+; SSSE3-NEXT:    movzbl %al, %eax
+; SSSE3-NEXT:    movd %eax, %xmm0
+; SSSE3-NEXT:    pinsrw $4, %r11d, %xmm0
+; SSSE3-NEXT:    movq %rdx, 16(%r10)
+; SSSE3-NEXT:    movq %rdi, (%r10)
+; SSSE3-NEXT:    movq %rcx, 24(%r10)
+; SSSE3-NEXT:    movq %rsi, 8(%r10)
+; SSSE3-NEXT:    psllq $63, %xmm0
+; SSSE3-NEXT:    psrad $31, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: uaddo_v2i128:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movq {{[0-9]+}}(%rsp), %r10
+; SSE41-NEXT:    addq {{[0-9]+}}(%rsp), %rdx
+; SSE41-NEXT:    adcq {{[0-9]+}}(%rsp), %rcx
+; SSE41-NEXT:    setb %al
+; SSE41-NEXT:    movzbl %al, %r11d
+; SSE41-NEXT:    addq %r8, %rdi
+; SSE41-NEXT:    adcq %r9, %rsi
+; SSE41-NEXT:    setb %al
+; SSE41-NEXT:    movzbl %al, %eax
+; SSE41-NEXT:    movd %eax, %xmm0
+; SSE41-NEXT:    pinsrb $8, %r11d, %xmm0
+; SSE41-NEXT:    movq %rdx, 16(%r10)
+; SSE41-NEXT:    movq %rdi, (%r10)
+; SSE41-NEXT:    movq %rcx, 24(%r10)
+; SSE41-NEXT:    movq %rsi, 8(%r10)
+; SSE41-NEXT:    psllq $63, %xmm0
+; SSE41-NEXT:    psrad $31, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: uaddo_v2i128:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    movq {{[0-9]+}}(%rsp), %r10
+; AVX1-NEXT:    addq {{[0-9]+}}(%rsp), %rdx
+; AVX1-NEXT:    adcq {{[0-9]+}}(%rsp), %rcx
+; AVX1-NEXT:    setb %al
+; AVX1-NEXT:    movzbl %al, %r11d
+; AVX1-NEXT:    addq %r8, %rdi
+; AVX1-NEXT:    adcq %r9, %rsi
+; AVX1-NEXT:    setb %al
+; AVX1-NEXT:    movzbl %al, %eax
+; AVX1-NEXT:    vmovd %eax, %xmm0
+; AVX1-NEXT:    vpinsrb $8, %r11d, %xmm0, %xmm0
+; AVX1-NEXT:    movq %rdx, 16(%r10)
+; AVX1-NEXT:    movq %rdi, (%r10)
+; AVX1-NEXT:    movq %rcx, 24(%r10)
+; AVX1-NEXT:    movq %rsi, 8(%r10)
+; AVX1-NEXT:    vpsllq $63, %xmm0, %xmm0
+; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX1-NEXT:    vpcmpgtq %xmm0, %xmm1, %xmm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: uaddo_v2i128:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    movq {{[0-9]+}}(%rsp), %r10
+; AVX2-NEXT:    addq {{[0-9]+}}(%rsp), %rdx
+; AVX2-NEXT:    adcq {{[0-9]+}}(%rsp), %rcx
+; AVX2-NEXT:    setb %al
+; AVX2-NEXT:    movzbl %al, %r11d
+; AVX2-NEXT:    addq %r8, %rdi
+; AVX2-NEXT:    adcq %r9, %rsi
+; AVX2-NEXT:    setb %al
+; AVX2-NEXT:    movzbl %al, %eax
+; AVX2-NEXT:    vmovd %eax, %xmm0
+; AVX2-NEXT:    vpinsrb $8, %r11d, %xmm0, %xmm0
+; AVX2-NEXT:    movq %rdx, 16(%r10)
+; AVX2-NEXT:    movq %rdi, (%r10)
+; AVX2-NEXT:    movq %rcx, 24(%r10)
+; AVX2-NEXT:    movq %rsi, 8(%r10)
+; AVX2-NEXT:    vpsllq $63, %xmm0, %xmm0
+; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX2-NEXT:    vpcmpgtq %xmm0, %xmm1, %xmm0
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: uaddo_v2i128:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    movq {{[0-9]+}}(%rsp), %r10
+; AVX512-NEXT:    addq {{[0-9]+}}(%rsp), %rdx
+; AVX512-NEXT:    adcq {{[0-9]+}}(%rsp), %rcx
+; AVX512-NEXT:    setb %al
+; AVX512-NEXT:    movb %al, -{{[0-9]+}}(%rsp)
+; AVX512-NEXT:    addq %r8, %rdi
+; AVX512-NEXT:    adcq %r9, %rsi
+; AVX512-NEXT:    setb %al
+; AVX512-NEXT:    movb %al, -{{[0-9]+}}(%rsp)
+; AVX512-NEXT:    kmovw -{{[0-9]+}}(%rsp), %k1
+; AVX512-NEXT:    movq %rdx, 16(%r10)
+; AVX512-NEXT:    movq %rdi, (%r10)
+; AVX512-NEXT:    movq %rcx, 24(%r10)
+; AVX512-NEXT:    movq %rsi, 8(%r10)
+; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX512-NEXT:    vmovdqa64 %xmm0, %xmm0 {%k1} {z}
+; AVX512-NEXT:    retq
+  %t = call {<2 x i128>, <2 x i1>} @llvm.uadd.with.overflow.v2i128(<2 x i128> %a0, <2 x i128> %a1)
+  %val = extractvalue {<2 x i128>, <2 x i1>} %t, 0
+  %obit = extractvalue {<2 x i128>, <2 x i1>} %t, 1
+  %res = sext <2 x i1> %obit to <2 x i32>
+  store <2 x i128> %val, <2 x i128>* %p2
+  ret <2 x i32> %res
+}
diff --git a/test/CodeGen/X86/vec_usubo.ll b/test/CodeGen/X86/vec_usubo.ll
new file mode 100644
index 0000000000000000000000000000000000000000..10de326c3563790c0daf9c4a9d6b80e1a7d059f2
--- /dev/null
+++ b/test/CodeGen/X86/vec_usubo.ll
@@ -0,0 +1,1422 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefixes=SSE,SSE2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+ssse3 | FileCheck %s --check-prefixes=SSE,SSSE3
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefixes=SSE,SSE41
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefixes=AVX,AVX1
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefixes=AVX,AVX2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw,+avx512vl,+fast-variable-shuffle | FileCheck %s --check-prefixes=AVX,AVX512
+
+declare {<1 x i32>, <1 x i1>} @llvm.usub.with.overflow.v1i32(<1 x i32>, <1 x i32>)
+declare {<2 x i32>, <2 x i1>} @llvm.usub.with.overflow.v2i32(<2 x i32>, <2 x i32>)
+declare {<3 x i32>, <3 x i1>} @llvm.usub.with.overflow.v3i32(<3 x i32>, <3 x i32>)
+declare {<4 x i32>, <4 x i1>} @llvm.usub.with.overflow.v4i32(<4 x i32>, <4 x i32>)
+declare {<6 x i32>, <6 x i1>} @llvm.usub.with.overflow.v6i32(<6 x i32>, <6 x i32>)
+declare {<8 x i32>, <8 x i1>} @llvm.usub.with.overflow.v8i32(<8 x i32>, <8 x i32>)
+declare {<16 x i32>, <16 x i1>} @llvm.usub.with.overflow.v16i32(<16 x i32>, <16 x i32>)
+
+declare {<16 x i8>, <16 x i1>} @llvm.usub.with.overflow.v16i8(<16 x i8>, <16 x i8>)
+declare {<8 x i16>, <8 x i1>} @llvm.usub.with.overflow.v8i16(<8 x i16>, <8 x i16>)
+declare {<2 x i64>, <2 x i1>} @llvm.usub.with.overflow.v2i64(<2 x i64>, <2 x i64>)
+
+declare {<4 x i24>, <4 x i1>} @llvm.usub.with.overflow.v4i24(<4 x i24>, <4 x i24>)
+declare {<4 x i1>, <4 x i1>} @llvm.usub.with.overflow.v4i1(<4 x i1>, <4 x i1>)
+declare {<2 x i128>, <2 x i1>} @llvm.usub.with.overflow.v2i128(<2 x i128>, <2 x i128>)
+
+define <1 x i32> @usubo_v1i32(<1 x i32> %a0, <1 x i32> %a1, <1 x i32>* %p2) nounwind {
+; SSE-LABEL: usubo_v1i32:
+; SSE:       # %bb.0:
+; SSE-NEXT:    subl %esi, %edi
+; SSE-NEXT:    sbbl %eax, %eax
+; SSE-NEXT:    movl %edi, (%rdx)
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: usubo_v1i32:
+; AVX:       # %bb.0:
+; AVX-NEXT:    subl %esi, %edi
+; AVX-NEXT:    sbbl %eax, %eax
+; AVX-NEXT:    movl %edi, (%rdx)
+; AVX-NEXT:    retq
+  %t = call {<1 x i32>, <1 x i1>} @llvm.usub.with.overflow.v1i32(<1 x i32> %a0, <1 x i32> %a1)
+  %val = extractvalue {<1 x i32>, <1 x i1>} %t, 0
+  %obit = extractvalue {<1 x i32>, <1 x i1>} %t, 1
+  %res = sext <1 x i1> %obit to <1 x i32>
+  store <1 x i32> %val, <1 x i32>* %p2
+  ret <1 x i32> %res
+}
+
+define <2 x i32> @usubo_v2i32(<2 x i32> %a0, <2 x i32> %a1, <2 x i32>* %p2) nounwind {
+; SSE2-LABEL: usubo_v2i32:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [4294967295,0,4294967295,0]
+; SSE2-NEXT:    pand %xmm2, %xmm1
+; SSE2-NEXT:    pand %xmm2, %xmm0
+; SSE2-NEXT:    psubq %xmm1, %xmm0
+; SSE2-NEXT:    pand %xmm0, %xmm2
+; SSE2-NEXT:    pcmpeqd %xmm0, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[1,0,3,2]
+; SSE2-NEXT:    pand %xmm2, %xmm3
+; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
+; SSE2-NEXT:    pxor %xmm3, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; SSE2-NEXT:    movq %xmm0, (%rdi)
+; SSE2-NEXT:    movdqa %xmm1, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: usubo_v2i32:
+; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm2 = [4294967295,0,4294967295,0]
+; SSSE3-NEXT:    pand %xmm2, %xmm1
+; SSSE3-NEXT:    pand %xmm2, %xmm0
+; SSSE3-NEXT:    psubq %xmm1, %xmm0
+; SSSE3-NEXT:    pand %xmm0, %xmm2
+; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm2[1,0,3,2]
+; SSSE3-NEXT:    pand %xmm2, %xmm3
+; SSSE3-NEXT:    pcmpeqd %xmm1, %xmm1
+; SSSE3-NEXT:    pxor %xmm3, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; SSSE3-NEXT:    movq %xmm0, (%rdi)
+; SSSE3-NEXT:    movdqa %xmm1, %xmm0
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: usubo_v2i32:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    pxor %xmm2, %xmm2
+; SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
+; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3],xmm0[4,5],xmm2[6,7]
+; SSE41-NEXT:    psubq %xmm1, %xmm0
+; SSE41-NEXT:    pblendw {{.*#+}} xmm2 = xmm0[0,1],xmm2[2,3],xmm0[4,5],xmm2[6,7]
+; SSE41-NEXT:    pcmpeqq %xmm0, %xmm2
+; SSE41-NEXT:    pcmpeqd %xmm1, %xmm1
+; SSE41-NEXT:    pxor %xmm2, %xmm1
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; SSE41-NEXT:    movq %xmm0, (%rdi)
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: usubo_v2i32:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3],xmm0[4,5],xmm2[6,7]
+; AVX1-NEXT:    vpsubq %xmm1, %xmm0, %xmm1
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
+; AVX1-NEXT:    vpcmpeqq %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
+; AVX1-NEXT:    vpxor %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; AVX1-NEXT:    vmovq %xmm1, (%rdi)
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: usubo_v2i32:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX2-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3]
+; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2],xmm2[3]
+; AVX2-NEXT:    vpsubq %xmm1, %xmm0, %xmm1
+; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm1[0],xmm2[1],xmm1[2],xmm2[3]
+; AVX2-NEXT:    vpcmpeqq %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
+; AVX2-NEXT:    vpxor %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; AVX2-NEXT:    vmovq %xmm1, (%rdi)
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: usubo_v2i32:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX512-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3]
+; AVX512-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2],xmm2[3]
+; AVX512-NEXT:    vpsubq %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vpblendd {{.*#+}} xmm1 = xmm0[0],xmm2[1],xmm0[2],xmm2[3]
+; AVX512-NEXT:    vpmovqd %xmm0, (%rdi)
+; AVX512-NEXT:    vpcmpeqq %xmm0, %xmm1, %xmm0
+; AVX512-NEXT:    vpternlogq $15, %xmm0, %xmm0, %xmm0
+; AVX512-NEXT:    retq
+  %t = call {<2 x i32>, <2 x i1>} @llvm.usub.with.overflow.v2i32(<2 x i32> %a0, <2 x i32> %a1)
+  %val = extractvalue {<2 x i32>, <2 x i1>} %t, 0
+  %obit = extractvalue {<2 x i32>, <2 x i1>} %t, 1
+  %res = sext <2 x i1> %obit to <2 x i32>
+  store <2 x i32> %val, <2 x i32>* %p2
+  ret <2 x i32> %res
+}
+
+define <3 x i32> @usubo_v3i32(<3 x i32> %a0, <3 x i32> %a1, <3 x i32>* %p2) nounwind {
+; SSE2-LABEL: usubo_v3i32:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648,2147483648,2147483648]
+; SSE2-NEXT:    movdqa %xmm0, %xmm3
+; SSE2-NEXT:    pxor %xmm2, %xmm3
+; SSE2-NEXT:    psubd %xmm1, %xmm0
+; SSE2-NEXT:    pxor %xmm0, %xmm2
+; SSE2-NEXT:    pcmpgtd %xmm3, %xmm2
+; SSE2-NEXT:    movq %xmm0, (%rdi)
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSE2-NEXT:    movd %xmm0, 8(%rdi)
+; SSE2-NEXT:    movdqa %xmm2, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: usubo_v3i32:
+; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648,2147483648,2147483648]
+; SSSE3-NEXT:    movdqa %xmm0, %xmm3
+; SSSE3-NEXT:    pxor %xmm2, %xmm3
+; SSSE3-NEXT:    psubd %xmm1, %xmm0
+; SSSE3-NEXT:    pxor %xmm0, %xmm2
+; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm2
+; SSSE3-NEXT:    movq %xmm0, (%rdi)
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; SSSE3-NEXT:    movd %xmm0, 8(%rdi)
+; SSSE3-NEXT:    movdqa %xmm2, %xmm0
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: usubo_v3i32:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movdqa %xmm0, %xmm2
+; SSE41-NEXT:    psubd %xmm1, %xmm2
+; SSE41-NEXT:    pminud %xmm2, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm2, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm1, %xmm1
+; SSE41-NEXT:    pxor %xmm1, %xmm0
+; SSE41-NEXT:    pextrd $2, %xmm2, 8(%rdi)
+; SSE41-NEXT:    movq %xmm2, (%rdi)
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: usubo_v3i32:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpsubd %xmm1, %xmm0, %xmm1
+; AVX1-NEXT:    vpminud %xmm0, %xmm1, %xmm0
+; AVX1-NEXT:    vpcmpeqd %xmm0, %xmm1, %xmm0
+; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
+; AVX1-NEXT:    vpxor %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpextrd $2, %xmm1, 8(%rdi)
+; AVX1-NEXT:    vmovq %xmm1, (%rdi)
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: usubo_v3i32:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpsubd %xmm1, %xmm0, %xmm1
+; AVX2-NEXT:    vpminud %xmm0, %xmm1, %xmm0
+; AVX2-NEXT:    vpcmpeqd %xmm0, %xmm1, %xmm0
+; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
+; AVX2-NEXT:    vpxor %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    vpextrd $2, %xmm1, 8(%rdi)
+; AVX2-NEXT:    vmovq %xmm1, (%rdi)
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: usubo_v3i32:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpsubd %xmm1, %xmm0, %xmm1
+; AVX512-NEXT:    vpcmpnleud %xmm0, %xmm1, %k1
+; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
+; AVX512-NEXT:    vpextrd $2, %xmm1, 8(%rdi)
+; AVX512-NEXT:    vmovq %xmm1, (%rdi)
+; AVX512-NEXT:    retq
+  %t = call {<3 x i32>, <3 x i1>} @llvm.usub.with.overflow.v3i32(<3 x i32> %a0, <3 x i32> %a1)
+  %val = extractvalue {<3 x i32>, <3 x i1>} %t, 0
+  %obit = extractvalue {<3 x i32>, <3 x i1>} %t, 1
+  %res = sext <3 x i1> %obit to <3 x i32>
+  store <3 x i32> %val, <3 x i32>* %p2
+  ret <3 x i32> %res
+}
+
+define <4 x i32> @usubo_v4i32(<4 x i32> %a0, <4 x i32> %a1, <4 x i32>* %p2) nounwind {
+; SSE2-LABEL: usubo_v4i32:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648,2147483648,2147483648]
+; SSE2-NEXT:    movdqa %xmm0, %xmm3
+; SSE2-NEXT:    pxor %xmm2, %xmm3
+; SSE2-NEXT:    psubd %xmm1, %xmm0
+; SSE2-NEXT:    pxor %xmm0, %xmm2
+; SSE2-NEXT:    pcmpgtd %xmm3, %xmm2
+; SSE2-NEXT:    movdqa %xmm0, (%rdi)
+; SSE2-NEXT:    movdqa %xmm2, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: usubo_v4i32:
+; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648,2147483648,2147483648]
+; SSSE3-NEXT:    movdqa %xmm0, %xmm3
+; SSSE3-NEXT:    pxor %xmm2, %xmm3
+; SSSE3-NEXT:    psubd %xmm1, %xmm0
+; SSSE3-NEXT:    pxor %xmm0, %xmm2
+; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm2
+; SSSE3-NEXT:    movdqa %xmm0, (%rdi)
+; SSSE3-NEXT:    movdqa %xmm2, %xmm0
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: usubo_v4i32:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movdqa %xmm0, %xmm2
+; SSE41-NEXT:    psubd %xmm1, %xmm2
+; SSE41-NEXT:    pminud %xmm2, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm2, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm1, %xmm1
+; SSE41-NEXT:    pxor %xmm1, %xmm0
+; SSE41-NEXT:    movdqa %xmm2, (%rdi)
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: usubo_v4i32:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpsubd %xmm1, %xmm0, %xmm1
+; AVX1-NEXT:    vpminud %xmm0, %xmm1, %xmm0
+; AVX1-NEXT:    vpcmpeqd %xmm0, %xmm1, %xmm0
+; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
+; AVX1-NEXT:    vpxor %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vmovdqa %xmm1, (%rdi)
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: usubo_v4i32:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpsubd %xmm1, %xmm0, %xmm1
+; AVX2-NEXT:    vpminud %xmm0, %xmm1, %xmm0
+; AVX2-NEXT:    vpcmpeqd %xmm0, %xmm1, %xmm0
+; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
+; AVX2-NEXT:    vpxor %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    vmovdqa %xmm1, (%rdi)
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: usubo_v4i32:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpsubd %xmm1, %xmm0, %xmm1
+; AVX512-NEXT:    vpcmpnleud %xmm0, %xmm1, %k1
+; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
+; AVX512-NEXT:    vmovdqa %xmm1, (%rdi)
+; AVX512-NEXT:    retq
+  %t = call {<4 x i32>, <4 x i1>} @llvm.usub.with.overflow.v4i32(<4 x i32> %a0, <4 x i32> %a1)
+  %val = extractvalue {<4 x i32>, <4 x i1>} %t, 0
+  %obit = extractvalue {<4 x i32>, <4 x i1>} %t, 1
+  %res = sext <4 x i1> %obit to <4 x i32>
+  store <4 x i32> %val, <4 x i32>* %p2
+  ret <4 x i32> %res
+}
+
+define <6 x i32> @usubo_v6i32(<6 x i32> %a0, <6 x i32> %a1, <6 x i32>* %p2) nounwind {
+; SSE2-LABEL: usubo_v6i32:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movq %rdi, %rax
+; SSE2-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSE2-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
+; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm1[0]
+; SSE2-NEXT:    movd %r8d, %xmm0
+; SSE2-NEXT:    movd %ecx, %xmm1
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSE2-NEXT:    movd %edx, %xmm3
+; SSE2-NEXT:    movd %esi, %xmm0
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
+; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; SSE2-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SSE2-NEXT:    movd {{.*#+}} xmm3 = mem[0],zero,zero,zero
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
+; SSE2-NEXT:    movd %r9d, %xmm1
+; SSE2-NEXT:    movd {{.*#+}} xmm4 = mem[0],zero,zero,zero
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1]
+; SSE2-NEXT:    movq {{[0-9]+}}(%rsp), %rcx
+; SSE2-NEXT:    movdqa %xmm0, %xmm4
+; SSE2-NEXT:    psubd %xmm2, %xmm4
+; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648,2147483648,2147483648]
+; SSE2-NEXT:    movdqa %xmm4, (%rcx)
+; SSE2-NEXT:    pxor %xmm2, %xmm4
+; SSE2-NEXT:    pxor %xmm2, %xmm0
+; SSE2-NEXT:    pcmpgtd %xmm0, %xmm4
+; SSE2-NEXT:    movdqa %xmm1, %xmm0
+; SSE2-NEXT:    psubd %xmm3, %xmm0
+; SSE2-NEXT:    movq %xmm0, 16(%rcx)
+; SSE2-NEXT:    pxor %xmm2, %xmm0
+; SSE2-NEXT:    pxor %xmm2, %xmm1
+; SSE2-NEXT:    pcmpgtd %xmm1, %xmm0
+; SSE2-NEXT:    movq %xmm0, 16(%rdi)
+; SSE2-NEXT:    movdqa %xmm4, (%rdi)
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: usubo_v6i32:
+; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    movq %rdi, %rax
+; SSSE3-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSSE3-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSSE3-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSSE3-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
+; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm1[0]
+; SSSE3-NEXT:    movd %r8d, %xmm0
+; SSSE3-NEXT:    movd %ecx, %xmm1
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSSE3-NEXT:    movd %edx, %xmm3
+; SSSE3-NEXT:    movd %esi, %xmm0
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
+; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; SSSE3-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SSSE3-NEXT:    movd {{.*#+}} xmm3 = mem[0],zero,zero,zero
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
+; SSSE3-NEXT:    movd %r9d, %xmm1
+; SSSE3-NEXT:    movd {{.*#+}} xmm4 = mem[0],zero,zero,zero
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1]
+; SSSE3-NEXT:    movq {{[0-9]+}}(%rsp), %rcx
+; SSSE3-NEXT:    movdqa %xmm0, %xmm4
+; SSSE3-NEXT:    psubd %xmm2, %xmm4
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648,2147483648,2147483648]
+; SSSE3-NEXT:    movdqa %xmm4, (%rcx)
+; SSSE3-NEXT:    pxor %xmm2, %xmm4
+; SSSE3-NEXT:    pxor %xmm2, %xmm0
+; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm4
+; SSSE3-NEXT:    movdqa %xmm1, %xmm0
+; SSSE3-NEXT:    psubd %xmm3, %xmm0
+; SSSE3-NEXT:    movq %xmm0, 16(%rcx)
+; SSSE3-NEXT:    pxor %xmm2, %xmm0
+; SSSE3-NEXT:    pxor %xmm2, %xmm1
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm0
+; SSSE3-NEXT:    movq %xmm0, 16(%rdi)
+; SSSE3-NEXT:    movdqa %xmm4, (%rdi)
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: usubo_v6i32:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movq %rdi, %rax
+; SSE41-NEXT:    movd %esi, %xmm0
+; SSE41-NEXT:    pinsrd $1, %edx, %xmm0
+; SSE41-NEXT:    pinsrd $2, %ecx, %xmm0
+; SSE41-NEXT:    pinsrd $3, %r8d, %xmm0
+; SSE41-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SSE41-NEXT:    pinsrd $1, {{[0-9]+}}(%rsp), %xmm1
+; SSE41-NEXT:    movd %r9d, %xmm2
+; SSE41-NEXT:    pinsrd $1, {{[0-9]+}}(%rsp), %xmm2
+; SSE41-NEXT:    movd {{.*#+}} xmm3 = mem[0],zero,zero,zero
+; SSE41-NEXT:    pinsrd $1, {{[0-9]+}}(%rsp), %xmm3
+; SSE41-NEXT:    pinsrd $2, {{[0-9]+}}(%rsp), %xmm3
+; SSE41-NEXT:    pinsrd $3, {{[0-9]+}}(%rsp), %xmm3
+; SSE41-NEXT:    movq {{[0-9]+}}(%rsp), %rcx
+; SSE41-NEXT:    movdqa %xmm0, %xmm4
+; SSE41-NEXT:    psubd %xmm3, %xmm4
+; SSE41-NEXT:    pminud %xmm4, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm4, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm3, %xmm3
+; SSE41-NEXT:    pxor %xmm3, %xmm0
+; SSE41-NEXT:    movdqa %xmm2, %xmm5
+; SSE41-NEXT:    psubd %xmm1, %xmm5
+; SSE41-NEXT:    pminud %xmm5, %xmm2
+; SSE41-NEXT:    pcmpeqd %xmm5, %xmm2
+; SSE41-NEXT:    pxor %xmm3, %xmm2
+; SSE41-NEXT:    movq %xmm5, 16(%rcx)
+; SSE41-NEXT:    movdqa %xmm4, (%rcx)
+; SSE41-NEXT:    movq %xmm2, 16(%rdi)
+; SSE41-NEXT:    movdqa %xmm0, (%rdi)
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: usubo_v6i32:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
+; AVX1-NEXT:    vpsubd %xmm2, %xmm3, %xmm2
+; AVX1-NEXT:    vpminud %xmm3, %xmm2, %xmm3
+; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm2, %xmm3
+; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
+; AVX1-NEXT:    vpxor %xmm4, %xmm3, %xmm3
+; AVX1-NEXT:    vpsubd %xmm1, %xmm0, %xmm1
+; AVX1-NEXT:    vpminud %xmm0, %xmm1, %xmm0
+; AVX1-NEXT:    vpcmpeqd %xmm0, %xmm1, %xmm0
+; AVX1-NEXT:    vpxor %xmm4, %xmm0, %xmm0
+; AVX1-NEXT:    vpackssdw %xmm3, %xmm0, %xmm0
+; AVX1-NEXT:    vpmovsxwd %xmm0, %xmm3
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpmovsxwd %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm3, %ymm0
+; AVX1-NEXT:    vmovq %xmm2, 16(%rdi)
+; AVX1-NEXT:    vmovdqa %xmm1, (%rdi)
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: usubo_v6i32:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpsubd %ymm1, %ymm0, %ymm1
+; AVX2-NEXT:    vpminud %ymm0, %ymm1, %ymm0
+; AVX2-NEXT:    vpcmpeqd %ymm0, %ymm1, %ymm0
+; AVX2-NEXT:    vpcmpeqd %ymm2, %ymm2, %ymm2
+; AVX2-NEXT:    vpxor %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm2
+; AVX2-NEXT:    vpackssdw %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    vpmovsxwd %xmm0, %ymm0
+; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm2
+; AVX2-NEXT:    vmovq %xmm2, 16(%rdi)
+; AVX2-NEXT:    vmovdqa %xmm1, (%rdi)
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: usubo_v6i32:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpsubd %ymm1, %ymm0, %ymm1
+; AVX512-NEXT:    vpcmpnleud %ymm0, %ymm1, %k1
+; AVX512-NEXT:    vpcmpeqd %ymm0, %ymm0, %ymm0
+; AVX512-NEXT:    vmovdqa32 %ymm0, %ymm0 {%k1} {z}
+; AVX512-NEXT:    vextracti128 $1, %ymm1, %xmm2
+; AVX512-NEXT:    vmovq %xmm2, 16(%rdi)
+; AVX512-NEXT:    vmovdqa %xmm1, (%rdi)
+; AVX512-NEXT:    retq
+  %t = call {<6 x i32>, <6 x i1>} @llvm.usub.with.overflow.v6i32(<6 x i32> %a0, <6 x i32> %a1)
+  %val = extractvalue {<6 x i32>, <6 x i1>} %t, 0
+  %obit = extractvalue {<6 x i32>, <6 x i1>} %t, 1
+  %res = sext <6 x i1> %obit to <6 x i32>
+  store <6 x i32> %val, <6 x i32>* %p2
+  ret <6 x i32> %res
+}
+
+define <8 x i32> @usubo_v8i32(<8 x i32> %a0, <8 x i32> %a1, <8 x i32>* %p2) nounwind {
+; SSE2-LABEL: usubo_v8i32:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [2147483648,2147483648,2147483648,2147483648]
+; SSE2-NEXT:    movdqa %xmm0, %xmm5
+; SSE2-NEXT:    pxor %xmm4, %xmm5
+; SSE2-NEXT:    psubd %xmm2, %xmm0
+; SSE2-NEXT:    movdqa %xmm0, (%rdi)
+; SSE2-NEXT:    pxor %xmm4, %xmm0
+; SSE2-NEXT:    pcmpgtd %xmm5, %xmm0
+; SSE2-NEXT:    movdqa %xmm1, %xmm2
+; SSE2-NEXT:    pxor %xmm4, %xmm2
+; SSE2-NEXT:    psubd %xmm3, %xmm1
+; SSE2-NEXT:    pxor %xmm1, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm2, %xmm4
+; SSE2-NEXT:    movdqa %xmm1, 16(%rdi)
+; SSE2-NEXT:    movdqa %xmm4, %xmm1
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: usubo_v8i32:
+; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm4 = [2147483648,2147483648,2147483648,2147483648]
+; SSSE3-NEXT:    movdqa %xmm0, %xmm5
+; SSSE3-NEXT:    pxor %xmm4, %xmm5
+; SSSE3-NEXT:    psubd %xmm2, %xmm0
+; SSSE3-NEXT:    movdqa %xmm0, (%rdi)
+; SSSE3-NEXT:    pxor %xmm4, %xmm0
+; SSSE3-NEXT:    pcmpgtd %xmm5, %xmm0
+; SSSE3-NEXT:    movdqa %xmm1, %xmm2
+; SSSE3-NEXT:    pxor %xmm4, %xmm2
+; SSSE3-NEXT:    psubd %xmm3, %xmm1
+; SSSE3-NEXT:    pxor %xmm1, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm4
+; SSSE3-NEXT:    movdqa %xmm1, 16(%rdi)
+; SSSE3-NEXT:    movdqa %xmm4, %xmm1
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: usubo_v8i32:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movdqa %xmm0, %xmm4
+; SSE41-NEXT:    psubd %xmm2, %xmm4
+; SSE41-NEXT:    pminud %xmm4, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm4, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm2, %xmm2
+; SSE41-NEXT:    pxor %xmm2, %xmm0
+; SSE41-NEXT:    movdqa %xmm1, %xmm5
+; SSE41-NEXT:    psubd %xmm3, %xmm5
+; SSE41-NEXT:    pminud %xmm5, %xmm1
+; SSE41-NEXT:    pcmpeqd %xmm5, %xmm1
+; SSE41-NEXT:    pxor %xmm2, %xmm1
+; SSE41-NEXT:    movdqa %xmm5, 16(%rdi)
+; SSE41-NEXT:    movdqa %xmm4, (%rdi)
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: usubo_v8i32:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
+; AVX1-NEXT:    vpsubd %xmm2, %xmm3, %xmm2
+; AVX1-NEXT:    vpminud %xmm3, %xmm2, %xmm3
+; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm2, %xmm3
+; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
+; AVX1-NEXT:    vpxor %xmm4, %xmm3, %xmm3
+; AVX1-NEXT:    vpsubd %xmm1, %xmm0, %xmm1
+; AVX1-NEXT:    vpminud %xmm0, %xmm1, %xmm0
+; AVX1-NEXT:    vpcmpeqd %xmm0, %xmm1, %xmm0
+; AVX1-NEXT:    vpxor %xmm4, %xmm0, %xmm0
+; AVX1-NEXT:    vpackssdw %xmm3, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
+; AVX1-NEXT:    vpmovsxwd %xmm0, %xmm2
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpmovsxwd %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm2, %ymm0
+; AVX1-NEXT:    vmovaps %ymm1, (%rdi)
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: usubo_v8i32:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpsubd %ymm1, %ymm0, %ymm1
+; AVX2-NEXT:    vpminud %ymm0, %ymm1, %ymm0
+; AVX2-NEXT:    vpcmpeqd %ymm0, %ymm1, %ymm0
+; AVX2-NEXT:    vpcmpeqd %ymm2, %ymm2, %ymm2
+; AVX2-NEXT:    vpxor %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm2
+; AVX2-NEXT:    vpackssdw %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    vpmovsxwd %xmm0, %ymm0
+; AVX2-NEXT:    vmovdqa %ymm1, (%rdi)
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: usubo_v8i32:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpsubd %ymm1, %ymm0, %ymm1
+; AVX512-NEXT:    vpcmpnleud %ymm0, %ymm1, %k1
+; AVX512-NEXT:    vpcmpeqd %ymm0, %ymm0, %ymm0
+; AVX512-NEXT:    vmovdqa32 %ymm0, %ymm0 {%k1} {z}
+; AVX512-NEXT:    vmovdqa %ymm1, (%rdi)
+; AVX512-NEXT:    retq
+  %t = call {<8 x i32>, <8 x i1>} @llvm.usub.with.overflow.v8i32(<8 x i32> %a0, <8 x i32> %a1)
+  %val = extractvalue {<8 x i32>, <8 x i1>} %t, 0
+  %obit = extractvalue {<8 x i32>, <8 x i1>} %t, 1
+  %res = sext <8 x i1> %obit to <8 x i32>
+  store <8 x i32> %val, <8 x i32>* %p2
+  ret <8 x i32> %res
+}
+
+define <16 x i32> @usubo_v16i32(<16 x i32> %a0, <16 x i32> %a1, <16 x i32>* %p2) nounwind {
+; SSE2-LABEL: usubo_v16i32:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa {{.*#+}} xmm8 = [2147483648,2147483648,2147483648,2147483648]
+; SSE2-NEXT:    movdqa %xmm0, %xmm9
+; SSE2-NEXT:    pxor %xmm8, %xmm9
+; SSE2-NEXT:    psubd %xmm4, %xmm0
+; SSE2-NEXT:    movdqa %xmm0, (%rdi)
+; SSE2-NEXT:    pxor %xmm8, %xmm0
+; SSE2-NEXT:    pcmpgtd %xmm9, %xmm0
+; SSE2-NEXT:    movdqa %xmm1, %xmm4
+; SSE2-NEXT:    pxor %xmm8, %xmm4
+; SSE2-NEXT:    psubd %xmm5, %xmm1
+; SSE2-NEXT:    movdqa %xmm1, 16(%rdi)
+; SSE2-NEXT:    pxor %xmm8, %xmm1
+; SSE2-NEXT:    pcmpgtd %xmm4, %xmm1
+; SSE2-NEXT:    movdqa %xmm2, %xmm4
+; SSE2-NEXT:    pxor %xmm8, %xmm4
+; SSE2-NEXT:    psubd %xmm6, %xmm2
+; SSE2-NEXT:    movdqa %xmm2, 32(%rdi)
+; SSE2-NEXT:    pxor %xmm8, %xmm2
+; SSE2-NEXT:    pcmpgtd %xmm4, %xmm2
+; SSE2-NEXT:    movdqa %xmm3, %xmm4
+; SSE2-NEXT:    pxor %xmm8, %xmm4
+; SSE2-NEXT:    psubd %xmm7, %xmm3
+; SSE2-NEXT:    pxor %xmm3, %xmm8
+; SSE2-NEXT:    pcmpgtd %xmm4, %xmm8
+; SSE2-NEXT:    movdqa %xmm3, 48(%rdi)
+; SSE2-NEXT:    movdqa %xmm8, %xmm3
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: usubo_v16i32:
+; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm8 = [2147483648,2147483648,2147483648,2147483648]
+; SSSE3-NEXT:    movdqa %xmm0, %xmm9
+; SSSE3-NEXT:    pxor %xmm8, %xmm9
+; SSSE3-NEXT:    psubd %xmm4, %xmm0
+; SSSE3-NEXT:    movdqa %xmm0, (%rdi)
+; SSSE3-NEXT:    pxor %xmm8, %xmm0
+; SSSE3-NEXT:    pcmpgtd %xmm9, %xmm0
+; SSSE3-NEXT:    movdqa %xmm1, %xmm4
+; SSSE3-NEXT:    pxor %xmm8, %xmm4
+; SSSE3-NEXT:    psubd %xmm5, %xmm1
+; SSSE3-NEXT:    movdqa %xmm1, 16(%rdi)
+; SSSE3-NEXT:    pxor %xmm8, %xmm1
+; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm1
+; SSSE3-NEXT:    movdqa %xmm2, %xmm4
+; SSSE3-NEXT:    pxor %xmm8, %xmm4
+; SSSE3-NEXT:    psubd %xmm6, %xmm2
+; SSSE3-NEXT:    movdqa %xmm2, 32(%rdi)
+; SSSE3-NEXT:    pxor %xmm8, %xmm2
+; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm2
+; SSSE3-NEXT:    movdqa %xmm3, %xmm4
+; SSSE3-NEXT:    pxor %xmm8, %xmm4
+; SSSE3-NEXT:    psubd %xmm7, %xmm3
+; SSSE3-NEXT:    pxor %xmm3, %xmm8
+; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm8
+; SSSE3-NEXT:    movdqa %xmm3, 48(%rdi)
+; SSSE3-NEXT:    movdqa %xmm8, %xmm3
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: usubo_v16i32:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movdqa %xmm0, %xmm8
+; SSE41-NEXT:    psubd %xmm4, %xmm8
+; SSE41-NEXT:    pminud %xmm8, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm8, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm9, %xmm9
+; SSE41-NEXT:    pxor %xmm9, %xmm0
+; SSE41-NEXT:    movdqa %xmm1, %xmm4
+; SSE41-NEXT:    psubd %xmm5, %xmm4
+; SSE41-NEXT:    pminud %xmm4, %xmm1
+; SSE41-NEXT:    pcmpeqd %xmm4, %xmm1
+; SSE41-NEXT:    pxor %xmm9, %xmm1
+; SSE41-NEXT:    movdqa %xmm2, %xmm5
+; SSE41-NEXT:    psubd %xmm6, %xmm5
+; SSE41-NEXT:    pminud %xmm5, %xmm2
+; SSE41-NEXT:    pcmpeqd %xmm5, %xmm2
+; SSE41-NEXT:    pxor %xmm9, %xmm2
+; SSE41-NEXT:    movdqa %xmm3, %xmm6
+; SSE41-NEXT:    psubd %xmm7, %xmm6
+; SSE41-NEXT:    pminud %xmm6, %xmm3
+; SSE41-NEXT:    pcmpeqd %xmm6, %xmm3
+; SSE41-NEXT:    pxor %xmm9, %xmm3
+; SSE41-NEXT:    movdqa %xmm6, 48(%rdi)
+; SSE41-NEXT:    movdqa %xmm5, 32(%rdi)
+; SSE41-NEXT:    movdqa %xmm4, 16(%rdi)
+; SSE41-NEXT:    movdqa %xmm8, (%rdi)
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: usubo_v16i32:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm4
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm5
+; AVX1-NEXT:    vpsubd %xmm4, %xmm5, %xmm4
+; AVX1-NEXT:    vpminud %xmm5, %xmm4, %xmm5
+; AVX1-NEXT:    vpcmpeqd %xmm5, %xmm4, %xmm5
+; AVX1-NEXT:    vpcmpeqd %xmm6, %xmm6, %xmm6
+; AVX1-NEXT:    vpxor %xmm6, %xmm5, %xmm5
+; AVX1-NEXT:    vpsubd %xmm3, %xmm1, %xmm3
+; AVX1-NEXT:    vpminud %xmm1, %xmm3, %xmm1
+; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm3, %xmm1
+; AVX1-NEXT:    vpxor %xmm6, %xmm1, %xmm1
+; AVX1-NEXT:    vpackssdw %xmm5, %xmm1, %xmm1
+; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm5
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm7
+; AVX1-NEXT:    vpsubd %xmm5, %xmm7, %xmm5
+; AVX1-NEXT:    vpminud %xmm7, %xmm5, %xmm7
+; AVX1-NEXT:    vpcmpeqd %xmm7, %xmm5, %xmm7
+; AVX1-NEXT:    vpxor %xmm6, %xmm7, %xmm7
+; AVX1-NEXT:    vpsubd %xmm2, %xmm0, %xmm2
+; AVX1-NEXT:    vpminud %xmm0, %xmm2, %xmm0
+; AVX1-NEXT:    vpcmpeqd %xmm0, %xmm2, %xmm0
+; AVX1-NEXT:    vpxor %xmm6, %xmm0, %xmm0
+; AVX1-NEXT:    vpackssdw %xmm7, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm5, %ymm2, %ymm2
+; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm3, %ymm3
+; AVX1-NEXT:    vpmovsxwd %xmm0, %xmm4
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpmovsxwd %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm4, %ymm0
+; AVX1-NEXT:    vpmovsxwd %xmm1, %xmm4
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; AVX1-NEXT:    vpmovsxwd %xmm1, %xmm1
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm4, %ymm1
+; AVX1-NEXT:    vmovaps %ymm3, 32(%rdi)
+; AVX1-NEXT:    vmovaps %ymm2, (%rdi)
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: usubo_v16i32:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpsubd %ymm3, %ymm1, %ymm3
+; AVX2-NEXT:    vpminud %ymm1, %ymm3, %ymm1
+; AVX2-NEXT:    vpcmpeqd %ymm1, %ymm3, %ymm1
+; AVX2-NEXT:    vpcmpeqd %ymm4, %ymm4, %ymm4
+; AVX2-NEXT:    vpxor %ymm4, %ymm1, %ymm1
+; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm5
+; AVX2-NEXT:    vpackssdw %xmm5, %xmm1, %xmm1
+; AVX2-NEXT:    vpsubd %ymm2, %ymm0, %ymm2
+; AVX2-NEXT:    vpminud %ymm0, %ymm2, %ymm0
+; AVX2-NEXT:    vpcmpeqd %ymm0, %ymm2, %ymm0
+; AVX2-NEXT:    vpxor %ymm4, %ymm0, %ymm0
+; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm4
+; AVX2-NEXT:    vpackssdw %xmm4, %xmm0, %xmm0
+; AVX2-NEXT:    vpmovsxwd %xmm0, %ymm0
+; AVX2-NEXT:    vpmovsxwd %xmm1, %ymm1
+; AVX2-NEXT:    vmovdqa %ymm3, 32(%rdi)
+; AVX2-NEXT:    vmovdqa %ymm2, (%rdi)
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: usubo_v16i32:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpsubd %zmm1, %zmm0, %zmm1
+; AVX512-NEXT:    vpcmpnleud %zmm0, %zmm1, %k1
+; AVX512-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; AVX512-NEXT:    vmovdqa64 %zmm1, (%rdi)
+; AVX512-NEXT:    retq
+  %t = call {<16 x i32>, <16 x i1>} @llvm.usub.with.overflow.v16i32(<16 x i32> %a0, <16 x i32> %a1)
+  %val = extractvalue {<16 x i32>, <16 x i1>} %t, 0
+  %obit = extractvalue {<16 x i32>, <16 x i1>} %t, 1
+  %res = sext <16 x i1> %obit to <16 x i32>
+  store <16 x i32> %val, <16 x i32>* %p2
+  ret <16 x i32> %res
+}
+
+define <16 x i32> @usubo_v16i8(<16 x i8> %a0, <16 x i8> %a1, <16 x i8>* %p2) nounwind {
+; SSE2-LABEL: usubo_v16i8:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa %xmm0, %xmm4
+; SSE2-NEXT:    psubb %xmm1, %xmm4
+; SSE2-NEXT:    pminub %xmm4, %xmm0
+; SSE2-NEXT:    pcmpeqb %xmm4, %xmm0
+; SSE2-NEXT:    pcmpeqd %xmm3, %xmm3
+; SSE2-NEXT:    pxor %xmm0, %xmm3
+; SSE2-NEXT:    movdqa %xmm3, %xmm1
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSE2-NEXT:    movdqa %xmm1, %xmm0
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
+; SSE2-NEXT:    pslld $31, %xmm0
+; SSE2-NEXT:    psrad $31, %xmm0
+; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSE2-NEXT:    pslld $31, %xmm1
+; SSE2-NEXT:    psrad $31, %xmm1
+; SSE2-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm0[8],xmm3[9],xmm0[9],xmm3[10],xmm0[10],xmm3[11],xmm0[11],xmm3[12],xmm0[12],xmm3[13],xmm0[13],xmm3[14],xmm0[14],xmm3[15],xmm0[15]
+; SSE2-NEXT:    movdqa %xmm3, %xmm2
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
+; SSE2-NEXT:    pslld $31, %xmm2
+; SSE2-NEXT:    psrad $31, %xmm2
+; SSE2-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
+; SSE2-NEXT:    pslld $31, %xmm3
+; SSE2-NEXT:    psrad $31, %xmm3
+; SSE2-NEXT:    movdqa %xmm4, (%rdi)
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: usubo_v16i8:
+; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    movdqa %xmm0, %xmm4
+; SSSE3-NEXT:    psubb %xmm1, %xmm4
+; SSSE3-NEXT:    pminub %xmm4, %xmm0
+; SSSE3-NEXT:    pcmpeqb %xmm4, %xmm0
+; SSSE3-NEXT:    pcmpeqd %xmm3, %xmm3
+; SSSE3-NEXT:    pxor %xmm0, %xmm3
+; SSSE3-NEXT:    movdqa %xmm3, %xmm1
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSSE3-NEXT:    movdqa %xmm1, %xmm0
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
+; SSSE3-NEXT:    pslld $31, %xmm0
+; SSSE3-NEXT:    psrad $31, %xmm0
+; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSSE3-NEXT:    pslld $31, %xmm1
+; SSSE3-NEXT:    psrad $31, %xmm1
+; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm0[8],xmm3[9],xmm0[9],xmm3[10],xmm0[10],xmm3[11],xmm0[11],xmm3[12],xmm0[12],xmm3[13],xmm0[13],xmm3[14],xmm0[14],xmm3[15],xmm0[15]
+; SSSE3-NEXT:    movdqa %xmm3, %xmm2
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
+; SSSE3-NEXT:    pslld $31, %xmm2
+; SSSE3-NEXT:    psrad $31, %xmm2
+; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
+; SSSE3-NEXT:    pslld $31, %xmm3
+; SSSE3-NEXT:    psrad $31, %xmm3
+; SSSE3-NEXT:    movdqa %xmm4, (%rdi)
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: usubo_v16i8:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movdqa %xmm0, %xmm4
+; SSE41-NEXT:    psubb %xmm1, %xmm4
+; SSE41-NEXT:    pminub %xmm4, %xmm0
+; SSE41-NEXT:    pcmpeqb %xmm4, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm3, %xmm3
+; SSE41-NEXT:    pxor %xmm0, %xmm3
+; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm0 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
+; SSE41-NEXT:    pslld $31, %xmm0
+; SSE41-NEXT:    psrad $31, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,1,2,3]
+; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
+; SSE41-NEXT:    pslld $31, %xmm1
+; SSE41-NEXT:    psrad $31, %xmm1
+; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[2,3,0,1]
+; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero
+; SSE41-NEXT:    pslld $31, %xmm2
+; SSE41-NEXT:    psrad $31, %xmm2
+; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[3,1,2,3]
+; SSE41-NEXT:    pmovzxbd {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
+; SSE41-NEXT:    pslld $31, %xmm3
+; SSE41-NEXT:    psrad $31, %xmm3
+; SSE41-NEXT:    movdqa %xmm4, (%rdi)
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: usubo_v16i8:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpsubb %xmm1, %xmm0, %xmm2
+; AVX1-NEXT:    vpminub %xmm0, %xmm2, %xmm0
+; AVX1-NEXT:    vpcmpeqb %xmm0, %xmm2, %xmm0
+; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
+; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm1
+; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; AVX1-NEXT:    vpmovzxwd {{.*#+}} xmm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; AVX1-NEXT:    vpslld $31, %xmm3, %xmm3
+; AVX1-NEXT:    vpsrad $31, %xmm3, %xmm3
+; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm0[4,4,5,5,6,6,7,7]
+; AVX1-NEXT:    vpslld $31, %xmm0, %xmm0
+; AVX1-NEXT:    vpsrad $31, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm3, %ymm0
+; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
+; AVX1-NEXT:    vpmovzxwd {{.*#+}} xmm3 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
+; AVX1-NEXT:    vpslld $31, %xmm3, %xmm3
+; AVX1-NEXT:    vpsrad $31, %xmm3, %xmm3
+; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; AVX1-NEXT:    vpslld $31, %xmm1, %xmm1
+; AVX1-NEXT:    vpsrad $31, %xmm1, %xmm1
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm3, %ymm1
+; AVX1-NEXT:    vmovdqa %xmm2, (%rdi)
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: usubo_v16i8:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpsubb %xmm1, %xmm0, %xmm2
+; AVX2-NEXT:    vpminub %xmm0, %xmm2, %xmm0
+; AVX2-NEXT:    vpcmpeqb %xmm0, %xmm2, %xmm0
+; AVX2-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
+; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm1
+; AVX2-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX2-NEXT:    vpslld $31, %ymm0, %ymm0
+; AVX2-NEXT:    vpsrad $31, %ymm0, %ymm0
+; AVX2-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
+; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; AVX2-NEXT:    vpslld $31, %ymm1, %ymm1
+; AVX2-NEXT:    vpsrad $31, %ymm1, %ymm1
+; AVX2-NEXT:    vmovdqa %xmm2, (%rdi)
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: usubo_v16i8:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpsubb %xmm1, %xmm0, %xmm1
+; AVX512-NEXT:    vpcmpnleub %xmm0, %xmm1, %k1
+; AVX512-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; AVX512-NEXT:    vmovdqa %xmm1, (%rdi)
+; AVX512-NEXT:    retq
+  %t = call {<16 x i8>, <16 x i1>} @llvm.usub.with.overflow.v16i8(<16 x i8> %a0, <16 x i8> %a1)
+  %val = extractvalue {<16 x i8>, <16 x i1>} %t, 0
+  %obit = extractvalue {<16 x i8>, <16 x i1>} %t, 1
+  %res = sext <16 x i1> %obit to <16 x i32>
+  store <16 x i8> %val, <16 x i8>* %p2
+  ret <16 x i32> %res
+}
+
+define <8 x i32> @usubo_v8i16(<8 x i16> %a0, <8 x i16> %a1, <8 x i16>* %p2) nounwind {
+; SSE2-LABEL: usubo_v8i16:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [32768,32768,32768,32768,32768,32768,32768,32768]
+; SSE2-NEXT:    movdqa %xmm0, %xmm3
+; SSE2-NEXT:    pxor %xmm2, %xmm3
+; SSE2-NEXT:    psubw %xmm1, %xmm0
+; SSE2-NEXT:    pxor %xmm0, %xmm2
+; SSE2-NEXT:    pcmpgtw %xmm3, %xmm2
+; SSE2-NEXT:    movdqa %xmm2, %xmm1
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
+; SSE2-NEXT:    pslld $31, %xmm1
+; SSE2-NEXT:    psrad $31, %xmm1
+; SSE2-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; SSE2-NEXT:    pslld $31, %xmm2
+; SSE2-NEXT:    psrad $31, %xmm2
+; SSE2-NEXT:    movdqa %xmm0, (%rdi)
+; SSE2-NEXT:    movdqa %xmm1, %xmm0
+; SSE2-NEXT:    movdqa %xmm2, %xmm1
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: usubo_v8i16:
+; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm2 = [32768,32768,32768,32768,32768,32768,32768,32768]
+; SSSE3-NEXT:    movdqa %xmm0, %xmm3
+; SSSE3-NEXT:    pxor %xmm2, %xmm3
+; SSSE3-NEXT:    psubw %xmm1, %xmm0
+; SSSE3-NEXT:    pxor %xmm0, %xmm2
+; SSSE3-NEXT:    pcmpgtw %xmm3, %xmm2
+; SSSE3-NEXT:    movdqa %xmm2, %xmm1
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
+; SSSE3-NEXT:    pslld $31, %xmm1
+; SSSE3-NEXT:    psrad $31, %xmm1
+; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; SSSE3-NEXT:    pslld $31, %xmm2
+; SSSE3-NEXT:    psrad $31, %xmm2
+; SSSE3-NEXT:    movdqa %xmm0, (%rdi)
+; SSSE3-NEXT:    movdqa %xmm1, %xmm0
+; SSSE3-NEXT:    movdqa %xmm2, %xmm1
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: usubo_v8i16:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movdqa %xmm0, %xmm2
+; SSE41-NEXT:    psubw %xmm1, %xmm2
+; SSE41-NEXT:    pminuw %xmm2, %xmm0
+; SSE41-NEXT:    pcmpeqw %xmm2, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm1, %xmm1
+; SSE41-NEXT:    pxor %xmm0, %xmm1
+; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm0 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
+; SSE41-NEXT:    pslld $31, %xmm0
+; SSE41-NEXT:    psrad $31, %xmm0
+; SSE41-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSE41-NEXT:    pslld $31, %xmm1
+; SSE41-NEXT:    psrad $31, %xmm1
+; SSE41-NEXT:    movdqa %xmm2, (%rdi)
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: usubo_v8i16:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpsubw %xmm1, %xmm0, %xmm1
+; AVX1-NEXT:    vpminuw %xmm0, %xmm1, %xmm0
+; AVX1-NEXT:    vpcmpeqw %xmm0, %xmm1, %xmm0
+; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
+; AVX1-NEXT:    vpxor %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpmovsxwd %xmm0, %xmm2
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpmovsxwd %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm2, %ymm0
+; AVX1-NEXT:    vmovdqa %xmm1, (%rdi)
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: usubo_v8i16:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpsubw %xmm1, %xmm0, %xmm1
+; AVX2-NEXT:    vpminuw %xmm0, %xmm1, %xmm0
+; AVX2-NEXT:    vpcmpeqw %xmm0, %xmm1, %xmm0
+; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
+; AVX2-NEXT:    vpxor %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    vpmovsxwd %xmm0, %ymm0
+; AVX2-NEXT:    vmovdqa %xmm1, (%rdi)
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: usubo_v8i16:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpsubw %xmm1, %xmm0, %xmm1
+; AVX512-NEXT:    vpcmpnleuw %xmm0, %xmm1, %k1
+; AVX512-NEXT:    vpcmpeqd %ymm0, %ymm0, %ymm0
+; AVX512-NEXT:    vmovdqa32 %ymm0, %ymm0 {%k1} {z}
+; AVX512-NEXT:    vmovdqa %xmm1, (%rdi)
+; AVX512-NEXT:    retq
+  %t = call {<8 x i16>, <8 x i1>} @llvm.usub.with.overflow.v8i16(<8 x i16> %a0, <8 x i16> %a1)
+  %val = extractvalue {<8 x i16>, <8 x i1>} %t, 0
+  %obit = extractvalue {<8 x i16>, <8 x i1>} %t, 1
+  %res = sext <8 x i1> %obit to <8 x i32>
+  store <8 x i16> %val, <8 x i16>* %p2
+  ret <8 x i32> %res
+}
+
+define <2 x i32> @usubo_v2i64(<2 x i64> %a0, <2 x i64> %a1, <2 x i64>* %p2) nounwind {
+; SSE-LABEL: usubo_v2i64:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movdqa {{.*#+}} xmm2 = [9223372039002259456,9223372039002259456]
+; SSE-NEXT:    movdqa %xmm0, %xmm3
+; SSE-NEXT:    pxor %xmm2, %xmm3
+; SSE-NEXT:    psubq %xmm1, %xmm0
+; SSE-NEXT:    pxor %xmm0, %xmm2
+; SSE-NEXT:    movdqa %xmm2, %xmm1
+; SSE-NEXT:    pcmpgtd %xmm3, %xmm1
+; SSE-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[0,0,2,2]
+; SSE-NEXT:    pcmpeqd %xmm3, %xmm2
+; SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; SSE-NEXT:    pand %xmm4, %xmm2
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSE-NEXT:    por %xmm2, %xmm1
+; SSE-NEXT:    movdqa %xmm0, (%rdi)
+; SSE-NEXT:    movdqa %xmm1, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX1-LABEL: usubo_v2i64:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [9223372036854775808,9223372036854775808]
+; AVX1-NEXT:    vpxor %xmm2, %xmm0, %xmm3
+; AVX1-NEXT:    vpsubq %xmm1, %xmm0, %xmm1
+; AVX1-NEXT:    vpxor %xmm2, %xmm1, %xmm0
+; AVX1-NEXT:    vpcmpgtq %xmm3, %xmm0, %xmm0
+; AVX1-NEXT:    vmovdqa %xmm1, (%rdi)
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: usubo_v2i64:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vmovdqa {{.*#+}} xmm2 = [9223372036854775808,9223372036854775808]
+; AVX2-NEXT:    vpxor %xmm2, %xmm0, %xmm3
+; AVX2-NEXT:    vpsubq %xmm1, %xmm0, %xmm1
+; AVX2-NEXT:    vpxor %xmm2, %xmm1, %xmm0
+; AVX2-NEXT:    vpcmpgtq %xmm3, %xmm0, %xmm0
+; AVX2-NEXT:    vmovdqa %xmm1, (%rdi)
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: usubo_v2i64:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpsubq %xmm1, %xmm0, %xmm1
+; AVX512-NEXT:    vpcmpnleuq %xmm0, %xmm1, %k1
+; AVX512-NEXT:    vmovdqa %xmm1, (%rdi)
+; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX512-NEXT:    vmovdqa64 %xmm0, %xmm0 {%k1} {z}
+; AVX512-NEXT:    retq
+  %t = call {<2 x i64>, <2 x i1>} @llvm.usub.with.overflow.v2i64(<2 x i64> %a0, <2 x i64> %a1)
+  %val = extractvalue {<2 x i64>, <2 x i1>} %t, 0
+  %obit = extractvalue {<2 x i64>, <2 x i1>} %t, 1
+  %res = sext <2 x i1> %obit to <2 x i32>
+  store <2 x i64> %val, <2 x i64>* %p2
+  ret <2 x i32> %res
+}
+
+define <4 x i32> @usubo_v4i24(<4 x i24> %a0, <4 x i24> %a1, <4 x i24>* %p2) nounwind {
+; SSE2-LABEL: usubo_v4i24:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa %xmm0, %xmm2
+; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [255,255,255,0,255,255,255,0,255,255,255,0,255,255,255,0]
+; SSE2-NEXT:    pand %xmm3, %xmm1
+; SSE2-NEXT:    pand %xmm3, %xmm2
+; SSE2-NEXT:    psubd %xmm1, %xmm2
+; SSE2-NEXT:    pand %xmm2, %xmm3
+; SSE2-NEXT:    pcmpeqd %xmm2, %xmm3
+; SSE2-NEXT:    pcmpeqd %xmm0, %xmm0
+; SSE2-NEXT:    pxor %xmm3, %xmm0
+; SSE2-NEXT:    movd %xmm2, %eax
+; SSE2-NEXT:    movw %ax, (%rdi)
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[3,1,2,3]
+; SSE2-NEXT:    movd %xmm1, %ecx
+; SSE2-NEXT:    movw %cx, 9(%rdi)
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[2,3,0,1]
+; SSE2-NEXT:    movd %xmm1, %edx
+; SSE2-NEXT:    movw %dx, 6(%rdi)
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,1,2,3]
+; SSE2-NEXT:    movd %xmm1, %esi
+; SSE2-NEXT:    movw %si, 3(%rdi)
+; SSE2-NEXT:    shrl $16, %eax
+; SSE2-NEXT:    movb %al, 2(%rdi)
+; SSE2-NEXT:    shrl $16, %ecx
+; SSE2-NEXT:    movb %cl, 11(%rdi)
+; SSE2-NEXT:    shrl $16, %edx
+; SSE2-NEXT:    movb %dl, 8(%rdi)
+; SSE2-NEXT:    shrl $16, %esi
+; SSE2-NEXT:    movb %sil, 5(%rdi)
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: usubo_v4i24:
+; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    movdqa %xmm0, %xmm2
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm3 = [255,255,255,0,255,255,255,0,255,255,255,0,255,255,255,0]
+; SSSE3-NEXT:    pand %xmm3, %xmm1
+; SSSE3-NEXT:    pand %xmm3, %xmm2
+; SSSE3-NEXT:    psubd %xmm1, %xmm2
+; SSSE3-NEXT:    pand %xmm2, %xmm3
+; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm3
+; SSSE3-NEXT:    pcmpeqd %xmm0, %xmm0
+; SSSE3-NEXT:    pxor %xmm3, %xmm0
+; SSSE3-NEXT:    movd %xmm2, %eax
+; SSSE3-NEXT:    movw %ax, (%rdi)
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[3,1,2,3]
+; SSSE3-NEXT:    movd %xmm1, %ecx
+; SSSE3-NEXT:    movw %cx, 9(%rdi)
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[2,3,0,1]
+; SSSE3-NEXT:    movd %xmm1, %edx
+; SSSE3-NEXT:    movw %dx, 6(%rdi)
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,1,2,3]
+; SSSE3-NEXT:    movd %xmm1, %esi
+; SSSE3-NEXT:    movw %si, 3(%rdi)
+; SSSE3-NEXT:    shrl $16, %eax
+; SSSE3-NEXT:    movb %al, 2(%rdi)
+; SSSE3-NEXT:    shrl $16, %ecx
+; SSSE3-NEXT:    movb %cl, 11(%rdi)
+; SSSE3-NEXT:    shrl $16, %edx
+; SSSE3-NEXT:    movb %dl, 8(%rdi)
+; SSSE3-NEXT:    shrl $16, %esi
+; SSSE3-NEXT:    movb %sil, 5(%rdi)
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: usubo_v4i24:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movdqa {{.*#+}} xmm2 = [255,255,255,0,255,255,255,0,255,255,255,0,255,255,255,0]
+; SSE41-NEXT:    pand %xmm2, %xmm1
+; SSE41-NEXT:    pand %xmm2, %xmm0
+; SSE41-NEXT:    psubd %xmm1, %xmm0
+; SSE41-NEXT:    pand %xmm0, %xmm2
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm2
+; SSE41-NEXT:    pcmpeqd %xmm1, %xmm1
+; SSE41-NEXT:    pxor %xmm2, %xmm1
+; SSE41-NEXT:    pextrd $3, %xmm0, %eax
+; SSE41-NEXT:    movw %ax, 9(%rdi)
+; SSE41-NEXT:    pextrd $2, %xmm0, %ecx
+; SSE41-NEXT:    movw %cx, 6(%rdi)
+; SSE41-NEXT:    pextrd $1, %xmm0, %edx
+; SSE41-NEXT:    movw %dx, 3(%rdi)
+; SSE41-NEXT:    movd %xmm0, %esi
+; SSE41-NEXT:    movw %si, (%rdi)
+; SSE41-NEXT:    shrl $16, %eax
+; SSE41-NEXT:    movb %al, 11(%rdi)
+; SSE41-NEXT:    shrl $16, %ecx
+; SSE41-NEXT:    movb %cl, 8(%rdi)
+; SSE41-NEXT:    shrl $16, %edx
+; SSE41-NEXT:    movb %dl, 5(%rdi)
+; SSE41-NEXT:    shrl $16, %esi
+; SSE41-NEXT:    movb %sil, 2(%rdi)
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: usubo_v4i24:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vbroadcastss {{.*#+}} xmm2 = [2.35098856E-38,2.35098856E-38,2.35098856E-38,2.35098856E-38]
+; AVX1-NEXT:    vandps %xmm2, %xmm1, %xmm1
+; AVX1-NEXT:    vandps %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpsubd %xmm1, %xmm0, %xmm1
+; AVX1-NEXT:    vpand %xmm2, %xmm1, %xmm0
+; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
+; AVX1-NEXT:    vpxor %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpextrd $3, %xmm1, %eax
+; AVX1-NEXT:    movw %ax, 9(%rdi)
+; AVX1-NEXT:    vpextrd $2, %xmm1, %ecx
+; AVX1-NEXT:    movw %cx, 6(%rdi)
+; AVX1-NEXT:    vpextrd $1, %xmm1, %edx
+; AVX1-NEXT:    movw %dx, 3(%rdi)
+; AVX1-NEXT:    vmovd %xmm1, %esi
+; AVX1-NEXT:    movw %si, (%rdi)
+; AVX1-NEXT:    shrl $16, %eax
+; AVX1-NEXT:    movb %al, 11(%rdi)
+; AVX1-NEXT:    shrl $16, %ecx
+; AVX1-NEXT:    movb %cl, 8(%rdi)
+; AVX1-NEXT:    shrl $16, %edx
+; AVX1-NEXT:    movb %dl, 5(%rdi)
+; AVX1-NEXT:    shrl $16, %esi
+; AVX1-NEXT:    movb %sil, 2(%rdi)
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: usubo_v4i24:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [16777215,16777215,16777215,16777215]
+; AVX2-NEXT:    vpand %xmm2, %xmm1, %xmm1
+; AVX2-NEXT:    vpand %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    vpsubd %xmm1, %xmm0, %xmm1
+; AVX2-NEXT:    vpand %xmm2, %xmm1, %xmm0
+; AVX2-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
+; AVX2-NEXT:    vpxor %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    vpextrd $3, %xmm1, %eax
+; AVX2-NEXT:    movw %ax, 9(%rdi)
+; AVX2-NEXT:    vpextrd $2, %xmm1, %ecx
+; AVX2-NEXT:    movw %cx, 6(%rdi)
+; AVX2-NEXT:    vpextrd $1, %xmm1, %edx
+; AVX2-NEXT:    movw %dx, 3(%rdi)
+; AVX2-NEXT:    vmovd %xmm1, %esi
+; AVX2-NEXT:    movw %si, (%rdi)
+; AVX2-NEXT:    shrl $16, %eax
+; AVX2-NEXT:    movb %al, 11(%rdi)
+; AVX2-NEXT:    shrl $16, %ecx
+; AVX2-NEXT:    movb %cl, 8(%rdi)
+; AVX2-NEXT:    shrl $16, %edx
+; AVX2-NEXT:    movb %dl, 5(%rdi)
+; AVX2-NEXT:    shrl $16, %esi
+; AVX2-NEXT:    movb %sil, 2(%rdi)
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: usubo_v4i24:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [16777215,16777215,16777215,16777215]
+; AVX512-NEXT:    vpand %xmm2, %xmm1, %xmm1
+; AVX512-NEXT:    vpand %xmm2, %xmm0, %xmm0
+; AVX512-NEXT:    vpsubd %xmm1, %xmm0, %xmm1
+; AVX512-NEXT:    vpand %xmm2, %xmm1, %xmm0
+; AVX512-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vpternlogq $15, %xmm0, %xmm0, %xmm0
+; AVX512-NEXT:    vpextrd $3, %xmm1, %eax
+; AVX512-NEXT:    movw %ax, 9(%rdi)
+; AVX512-NEXT:    vpextrd $2, %xmm1, %ecx
+; AVX512-NEXT:    movw %cx, 6(%rdi)
+; AVX512-NEXT:    vpextrd $1, %xmm1, %edx
+; AVX512-NEXT:    movw %dx, 3(%rdi)
+; AVX512-NEXT:    vmovd %xmm1, %esi
+; AVX512-NEXT:    movw %si, (%rdi)
+; AVX512-NEXT:    shrl $16, %eax
+; AVX512-NEXT:    movb %al, 11(%rdi)
+; AVX512-NEXT:    shrl $16, %ecx
+; AVX512-NEXT:    movb %cl, 8(%rdi)
+; AVX512-NEXT:    shrl $16, %edx
+; AVX512-NEXT:    movb %dl, 5(%rdi)
+; AVX512-NEXT:    shrl $16, %esi
+; AVX512-NEXT:    movb %sil, 2(%rdi)
+; AVX512-NEXT:    retq
+  %t = call {<4 x i24>, <4 x i1>} @llvm.usub.with.overflow.v4i24(<4 x i24> %a0, <4 x i24> %a1)
+  %val = extractvalue {<4 x i24>, <4 x i1>} %t, 0
+  %obit = extractvalue {<4 x i24>, <4 x i1>} %t, 1
+  %res = sext <4 x i1> %obit to <4 x i32>
+  store <4 x i24> %val, <4 x i24>* %p2
+  ret <4 x i32> %res
+}
+
+define <4 x i32> @usubo_v4i1(<4 x i1> %a0, <4 x i1> %a1, <4 x i1>* %p2) nounwind {
+; SSE-LABEL: usubo_v4i1:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movdqa {{.*#+}} xmm2 = [1,1,1,1]
+; SSE-NEXT:    pand %xmm2, %xmm1
+; SSE-NEXT:    pand %xmm2, %xmm0
+; SSE-NEXT:    psubd %xmm1, %xmm0
+; SSE-NEXT:    pand %xmm0, %xmm2
+; SSE-NEXT:    pcmpeqd %xmm0, %xmm2
+; SSE-NEXT:    pcmpeqd %xmm1, %xmm1
+; SSE-NEXT:    pxor %xmm2, %xmm1
+; SSE-NEXT:    pslld $31, %xmm0
+; SSE-NEXT:    movmskps %xmm0, %eax
+; SSE-NEXT:    movb %al, (%rdi)
+; SSE-NEXT:    movdqa %xmm1, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX1-LABEL: usubo_v4i1:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [1,1,1,1]
+; AVX1-NEXT:    vpand %xmm2, %xmm1, %xmm1
+; AVX1-NEXT:    vpand %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpsubd %xmm1, %xmm0, %xmm1
+; AVX1-NEXT:    vpand %xmm2, %xmm1, %xmm0
+; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
+; AVX1-NEXT:    vpxor %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpslld $31, %xmm1, %xmm1
+; AVX1-NEXT:    vmovmskps %xmm1, %eax
+; AVX1-NEXT:    movb %al, (%rdi)
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: usubo_v4i1:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [1,1,1,1]
+; AVX2-NEXT:    vpand %xmm2, %xmm1, %xmm1
+; AVX2-NEXT:    vpand %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    vpsubd %xmm1, %xmm0, %xmm1
+; AVX2-NEXT:    vpand %xmm2, %xmm1, %xmm0
+; AVX2-NEXT:    vpcmpeqd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm2, %xmm2
+; AVX2-NEXT:    vpxor %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    vpslld $31, %xmm1, %xmm1
+; AVX2-NEXT:    vmovmskps %xmm1, %eax
+; AVX2-NEXT:    movb %al, (%rdi)
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: usubo_v4i1:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpslld $31, %xmm0, %xmm0
+; AVX512-NEXT:    vptestmd %xmm0, %xmm0, %k0
+; AVX512-NEXT:    vpslld $31, %xmm1, %xmm1
+; AVX512-NEXT:    vptestmd %xmm1, %xmm1, %k1
+; AVX512-NEXT:    kxorw %k1, %k0, %k1
+; AVX512-NEXT:    vptestnmd %xmm0, %xmm0, %k2 {%k1}
+; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k2} {z}
+; AVX512-NEXT:    kmovd %k1, %eax
+; AVX512-NEXT:    movb %al, (%rdi)
+; AVX512-NEXT:    retq
+  %t = call {<4 x i1>, <4 x i1>} @llvm.usub.with.overflow.v4i1(<4 x i1> %a0, <4 x i1> %a1)
+  %val = extractvalue {<4 x i1>, <4 x i1>} %t, 0
+  %obit = extractvalue {<4 x i1>, <4 x i1>} %t, 1
+  %res = sext <4 x i1> %obit to <4 x i32>
+  store <4 x i1> %val, <4 x i1>* %p2
+  ret <4 x i32> %res
+}
+
+define <2 x i32> @usubo_v2i128(<2 x i128> %a0, <2 x i128> %a1, <2 x i128>* %p2) nounwind {
+; SSE2-LABEL: usubo_v2i128:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movq {{[0-9]+}}(%rsp), %r10
+; SSE2-NEXT:    subq {{[0-9]+}}(%rsp), %rdx
+; SSE2-NEXT:    sbbq {{[0-9]+}}(%rsp), %rcx
+; SSE2-NEXT:    setb %al
+; SSE2-NEXT:    movzbl %al, %r11d
+; SSE2-NEXT:    subq %r8, %rdi
+; SSE2-NEXT:    sbbq %r9, %rsi
+; SSE2-NEXT:    setb %al
+; SSE2-NEXT:    movzbl %al, %eax
+; SSE2-NEXT:    movd %eax, %xmm0
+; SSE2-NEXT:    pinsrw $4, %r11d, %xmm0
+; SSE2-NEXT:    movq %rdx, 16(%r10)
+; SSE2-NEXT:    movq %rdi, (%r10)
+; SSE2-NEXT:    movq %rcx, 24(%r10)
+; SSE2-NEXT:    movq %rsi, 8(%r10)
+; SSE2-NEXT:    psllq $63, %xmm0
+; SSE2-NEXT:    psrad $31, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: usubo_v2i128:
+; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    movq {{[0-9]+}}(%rsp), %r10
+; SSSE3-NEXT:    subq {{[0-9]+}}(%rsp), %rdx
+; SSSE3-NEXT:    sbbq {{[0-9]+}}(%rsp), %rcx
+; SSSE3-NEXT:    setb %al
+; SSSE3-NEXT:    movzbl %al, %r11d
+; SSSE3-NEXT:    subq %r8, %rdi
+; SSSE3-NEXT:    sbbq %r9, %rsi
+; SSSE3-NEXT:    setb %al
+; SSSE3-NEXT:    movzbl %al, %eax
+; SSSE3-NEXT:    movd %eax, %xmm0
+; SSSE3-NEXT:    pinsrw $4, %r11d, %xmm0
+; SSSE3-NEXT:    movq %rdx, 16(%r10)
+; SSSE3-NEXT:    movq %rdi, (%r10)
+; SSSE3-NEXT:    movq %rcx, 24(%r10)
+; SSSE3-NEXT:    movq %rsi, 8(%r10)
+; SSSE3-NEXT:    psllq $63, %xmm0
+; SSSE3-NEXT:    psrad $31, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: usubo_v2i128:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movq {{[0-9]+}}(%rsp), %r10
+; SSE41-NEXT:    subq {{[0-9]+}}(%rsp), %rdx
+; SSE41-NEXT:    sbbq {{[0-9]+}}(%rsp), %rcx
+; SSE41-NEXT:    setb %al
+; SSE41-NEXT:    movzbl %al, %r11d
+; SSE41-NEXT:    subq %r8, %rdi
+; SSE41-NEXT:    sbbq %r9, %rsi
+; SSE41-NEXT:    setb %al
+; SSE41-NEXT:    movzbl %al, %eax
+; SSE41-NEXT:    movd %eax, %xmm0
+; SSE41-NEXT:    pinsrb $8, %r11d, %xmm0
+; SSE41-NEXT:    movq %rdx, 16(%r10)
+; SSE41-NEXT:    movq %rdi, (%r10)
+; SSE41-NEXT:    movq %rcx, 24(%r10)
+; SSE41-NEXT:    movq %rsi, 8(%r10)
+; SSE41-NEXT:    psllq $63, %xmm0
+; SSE41-NEXT:    psrad $31, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE41-NEXT:    retq
+;
+; AVX1-LABEL: usubo_v2i128:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    movq {{[0-9]+}}(%rsp), %r10
+; AVX1-NEXT:    subq {{[0-9]+}}(%rsp), %rdx
+; AVX1-NEXT:    sbbq {{[0-9]+}}(%rsp), %rcx
+; AVX1-NEXT:    setb %al
+; AVX1-NEXT:    movzbl %al, %r11d
+; AVX1-NEXT:    subq %r8, %rdi
+; AVX1-NEXT:    sbbq %r9, %rsi
+; AVX1-NEXT:    setb %al
+; AVX1-NEXT:    movzbl %al, %eax
+; AVX1-NEXT:    vmovd %eax, %xmm0
+; AVX1-NEXT:    vpinsrb $8, %r11d, %xmm0, %xmm0
+; AVX1-NEXT:    movq %rdx, 16(%r10)
+; AVX1-NEXT:    movq %rdi, (%r10)
+; AVX1-NEXT:    movq %rcx, 24(%r10)
+; AVX1-NEXT:    movq %rsi, 8(%r10)
+; AVX1-NEXT:    vpsllq $63, %xmm0, %xmm0
+; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX1-NEXT:    vpcmpgtq %xmm0, %xmm1, %xmm0
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: usubo_v2i128:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    movq {{[0-9]+}}(%rsp), %r10
+; AVX2-NEXT:    subq {{[0-9]+}}(%rsp), %rdx
+; AVX2-NEXT:    sbbq {{[0-9]+}}(%rsp), %rcx
+; AVX2-NEXT:    setb %al
+; AVX2-NEXT:    movzbl %al, %r11d
+; AVX2-NEXT:    subq %r8, %rdi
+; AVX2-NEXT:    sbbq %r9, %rsi
+; AVX2-NEXT:    setb %al
+; AVX2-NEXT:    movzbl %al, %eax
+; AVX2-NEXT:    vmovd %eax, %xmm0
+; AVX2-NEXT:    vpinsrb $8, %r11d, %xmm0, %xmm0
+; AVX2-NEXT:    movq %rdx, 16(%r10)
+; AVX2-NEXT:    movq %rdi, (%r10)
+; AVX2-NEXT:    movq %rcx, 24(%r10)
+; AVX2-NEXT:    movq %rsi, 8(%r10)
+; AVX2-NEXT:    vpsllq $63, %xmm0, %xmm0
+; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; AVX2-NEXT:    vpcmpgtq %xmm0, %xmm1, %xmm0
+; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: usubo_v2i128:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    movq {{[0-9]+}}(%rsp), %r10
+; AVX512-NEXT:    subq {{[0-9]+}}(%rsp), %rdx
+; AVX512-NEXT:    sbbq {{[0-9]+}}(%rsp), %rcx
+; AVX512-NEXT:    setb %al
+; AVX512-NEXT:    movb %al, -{{[0-9]+}}(%rsp)
+; AVX512-NEXT:    subq %r8, %rdi
+; AVX512-NEXT:    sbbq %r9, %rsi
+; AVX512-NEXT:    setb %al
+; AVX512-NEXT:    movb %al, -{{[0-9]+}}(%rsp)
+; AVX512-NEXT:    kmovw -{{[0-9]+}}(%rsp), %k1
+; AVX512-NEXT:    movq %rdx, 16(%r10)
+; AVX512-NEXT:    movq %rdi, (%r10)
+; AVX512-NEXT:    movq %rcx, 24(%r10)
+; AVX512-NEXT:    movq %rsi, 8(%r10)
+; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
+; AVX512-NEXT:    vmovdqa64 %xmm0, %xmm0 {%k1} {z}
+; AVX512-NEXT:    retq
+  %t = call {<2 x i128>, <2 x i1>} @llvm.usub.with.overflow.v2i128(<2 x i128> %a0, <2 x i128> %a1)
+  %val = extractvalue {<2 x i128>, <2 x i1>} %t, 0
+  %obit = extractvalue {<2 x i128>, <2 x i1>} %t, 1
+  %res = sext <2 x i1> %obit to <2 x i32>
+  store <2 x i128> %val, <2 x i128>* %p2
+  ret <2 x i32> %res
+}
diff --git a/test/CodeGen/X86/vecloadextract.ll b/test/CodeGen/X86/vecloadextract.ll
new file mode 100755
index 0000000000000000000000000000000000000000..94df350a291a0a58b5c92b4413f65bcdd86ae37b
--- /dev/null
+++ b/test/CodeGen/X86/vecloadextract.ll
@@ -0,0 +1,44 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+;RUN: llc < %s -mtriple=i686 -mattr=sse4.1 -stop-after=expand-isel-pseudos 2>&1 | FileCheck %s
+
+; This test makes sure we discard pointer info when we combine a vector load
+; and a variable extractelement into a scalar load using an index. There's also
+; a test to ensure we don't discard it for the constant index case.
+
+; CHECK: name: const_index
+; CHECK:  bb.0 (%ir-block.0):
+; CHECK:    [[POINTER:%[0-9]+]]:gr32 = MOV32rm %fixed-stack.0, 1, $noreg, 0, $noreg :: (load 4 from %fixed-stack.0)
+; CHECK:    [[LOAD:%[0-9]+]]:gr32 = MOV32rm killed [[POINTER]], 1, $noreg, 4, $noreg :: (load 4 from %ir.v + 4)
+; CHECK:    $eax = COPY [[LOAD]]
+; CHECK:    RET 0, $eax
+define i32 @const_index(<8 x i32>* %v) {
+  %a = load <8 x i32>, <8 x i32>* %v
+  %b = extractelement <8 x i32> %a, i32 1
+  ret i32 %b
+}
+
+; CHECK: name: variable_index
+; CHECK:  bb.0 (%ir-block.0):
+; CHECK:    [[INDEX:%[0-9]+]]:gr32_nosp = MOV32rm %fixed-stack.0, 1, $noreg, 0, $noreg :: (load 4 from %fixed-stack.0)
+; CHECK:    [[POINTER:%[0-9]+]]:gr32 = MOV32rm %fixed-stack.1, 1, $noreg, 0, $noreg :: (load 4 from %fixed-stack.1)
+; CHECK:    [[LOAD:%[0-9]+]]:gr32 = MOV32rm killed [[POINTER]], 4, killed [[INDEX]], 0, $noreg :: (load 4)
+; CHECK:    $eax = COPY [[LOAD]]
+; CHECK:    RET 0, $eax
+define i32 @variable_index(<8 x i32>* %v, i32 %i) {
+  %a = load <8 x i32>, <8 x i32>* %v
+  %b = extractelement <8 x i32> %a, i32 %i
+  ret i32 %b
+}
+
+; CHECK: name: variable_index_with_addrspace
+; CHECK:  bb.0 (%ir-block.0):
+; CHECK:    [[INDEX:%[0-9]+]]:gr32_nosp = MOV32rm %fixed-stack.0, 1, $noreg, 0, $noreg :: (load 4 from %fixed-stack.0)
+; CHECK:    [[POINTER:%[0-9]+]]:gr32 = MOV32rm %fixed-stack.1, 1, $noreg, 0, $noreg :: (load 4 from %fixed-stack.1)
+; CHECK:    [[LOAD:%[0-9]+]]:gr32 = MOV32rm killed [[POINTER]], 4, killed [[INDEX]], 0, $noreg :: (load 4, addrspace 1)
+; CHECK:    $eax = COPY [[LOAD]]
+; CHECK:    RET 0, $eax
+define i32 @variable_index_with_addrspace(<8 x i32> addrspace(1)* %v, i32 %i) {
+  %a = load <8 x i32>, <8 x i32> addrspace(1)* %v
+  %b = extractelement <8 x i32> %a, i32 %i
+  ret i32 %b
+}
diff --git a/test/CodeGen/X86/vector-bitreverse.ll b/test/CodeGen/X86/vector-bitreverse.ll
index b249eed2fc76d1baa99694ab1fc8cae2e6e3f746..df1b5041caf343d526b55cf77efc0eb0275695ed 100644
--- a/test/CodeGen/X86/vector-bitreverse.ll
+++ b/test/CodeGen/X86/vector-bitreverse.ll
@@ -236,16 +236,16 @@ define i64 @test_bitreverse_i64(i64 %a) nounwind {
 define <16 x i8> @test_bitreverse_v16i8(<16 x i8> %a) nounwind {
 ; SSE2-LABEL: test_bitreverse_v16i8:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; SSE2-NEXT:    movdqa %xmm0, %xmm2
-; SSE2-NEXT:    pand %xmm1, %xmm2
+; SSE2-NEXT:    movdqa %xmm0, %xmm1
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; SSE2-NEXT:    movdqa %xmm1, %xmm2
+; SSE2-NEXT:    pand %xmm0, %xmm2
 ; SSE2-NEXT:    psllw $4, %xmm2
-; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
-; SSE2-NEXT:    pand %xmm3, %xmm2
-; SSE2-NEXT:    pand %xmm3, %xmm0
-; SSE2-NEXT:    psrlw $4, %xmm0
-; SSE2-NEXT:    pand %xmm1, %xmm0
-; SSE2-NEXT:    por %xmm2, %xmm0
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm1
+; SSE2-NEXT:    psrlw $4, %xmm1
+; SSE2-NEXT:    pand %xmm0, %xmm1
+; SSE2-NEXT:    pandn %xmm2, %xmm0
+; SSE2-NEXT:    por %xmm1, %xmm0
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [51,51,51,51,51,51,51,51,51,51,51,51,51,51,51,51]
 ; SSE2-NEXT:    pand %xmm0, %xmm1
 ; SSE2-NEXT:    psllw $2, %xmm1
@@ -309,18 +309,17 @@ define <8 x i16> @test_bitreverse_v8i16(<8 x i16> %a) nounwind {
 ; SSE2-NEXT:    pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,5,4,7,6]
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
 ; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[1,0,3,2,4,5,6,7]
-; SSE2-NEXT:    pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,5,4,7,6]
-; SSE2-NEXT:    packuswb %xmm2, %xmm0
-; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; SSE2-NEXT:    movdqa %xmm0, %xmm2
-; SSE2-NEXT:    pand %xmm1, %xmm2
+; SSE2-NEXT:    pshufhw {{.*#+}} xmm1 = xmm0[0,1,2,3,5,4,7,6]
+; SSE2-NEXT:    packuswb %xmm2, %xmm1
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; SSE2-NEXT:    movdqa %xmm1, %xmm2
+; SSE2-NEXT:    pand %xmm0, %xmm2
 ; SSE2-NEXT:    psllw $4, %xmm2
-; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
-; SSE2-NEXT:    pand %xmm3, %xmm2
-; SSE2-NEXT:    pand %xmm3, %xmm0
-; SSE2-NEXT:    psrlw $4, %xmm0
-; SSE2-NEXT:    pand %xmm1, %xmm0
-; SSE2-NEXT:    por %xmm2, %xmm0
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm1
+; SSE2-NEXT:    psrlw $4, %xmm1
+; SSE2-NEXT:    pand %xmm0, %xmm1
+; SSE2-NEXT:    pandn %xmm2, %xmm0
+; SSE2-NEXT:    por %xmm1, %xmm0
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [51,51,51,51,51,51,51,51,51,51,51,51,51,51,51,51]
 ; SSE2-NEXT:    pand %xmm0, %xmm1
 ; SSE2-NEXT:    psllw $2, %xmm1
@@ -386,18 +385,17 @@ define <4 x i32> @test_bitreverse_v4i32(<4 x i32> %a) nounwind {
 ; SSE2-NEXT:    pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,7,6,5,4]
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
 ; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[3,2,1,0,4,5,6,7]
-; SSE2-NEXT:    pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,7,6,5,4]
-; SSE2-NEXT:    packuswb %xmm2, %xmm0
-; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; SSE2-NEXT:    movdqa %xmm0, %xmm2
-; SSE2-NEXT:    pand %xmm1, %xmm2
+; SSE2-NEXT:    pshufhw {{.*#+}} xmm1 = xmm0[0,1,2,3,7,6,5,4]
+; SSE2-NEXT:    packuswb %xmm2, %xmm1
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; SSE2-NEXT:    movdqa %xmm1, %xmm2
+; SSE2-NEXT:    pand %xmm0, %xmm2
 ; SSE2-NEXT:    psllw $4, %xmm2
-; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
-; SSE2-NEXT:    pand %xmm3, %xmm2
-; SSE2-NEXT:    pand %xmm3, %xmm0
-; SSE2-NEXT:    psrlw $4, %xmm0
-; SSE2-NEXT:    pand %xmm1, %xmm0
-; SSE2-NEXT:    por %xmm2, %xmm0
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm1
+; SSE2-NEXT:    psrlw $4, %xmm1
+; SSE2-NEXT:    pand %xmm0, %xmm1
+; SSE2-NEXT:    pandn %xmm2, %xmm0
+; SSE2-NEXT:    por %xmm1, %xmm0
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [51,51,51,51,51,51,51,51,51,51,51,51,51,51,51,51]
 ; SSE2-NEXT:    pand %xmm0, %xmm1
 ; SSE2-NEXT:    psllw $2, %xmm1
@@ -465,18 +463,17 @@ define <2 x i64> @test_bitreverse_v2i64(<2 x i64> %a) nounwind {
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
 ; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[3,2,1,0,4,5,6,7]
-; SSE2-NEXT:    pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,7,6,5,4]
-; SSE2-NEXT:    packuswb %xmm2, %xmm0
-; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; SSE2-NEXT:    movdqa %xmm0, %xmm2
-; SSE2-NEXT:    pand %xmm1, %xmm2
+; SSE2-NEXT:    pshufhw {{.*#+}} xmm1 = xmm0[0,1,2,3,7,6,5,4]
+; SSE2-NEXT:    packuswb %xmm2, %xmm1
+; SSE2-NEXT:    movdqa {{.*#+}} xmm0 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; SSE2-NEXT:    movdqa %xmm1, %xmm2
+; SSE2-NEXT:    pand %xmm0, %xmm2
 ; SSE2-NEXT:    psllw $4, %xmm2
-; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
-; SSE2-NEXT:    pand %xmm3, %xmm2
-; SSE2-NEXT:    pand %xmm3, %xmm0
-; SSE2-NEXT:    psrlw $4, %xmm0
-; SSE2-NEXT:    pand %xmm1, %xmm0
-; SSE2-NEXT:    por %xmm2, %xmm0
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm1
+; SSE2-NEXT:    psrlw $4, %xmm1
+; SSE2-NEXT:    pand %xmm0, %xmm1
+; SSE2-NEXT:    pandn %xmm2, %xmm0
+; SSE2-NEXT:    por %xmm1, %xmm0
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [51,51,51,51,51,51,51,51,51,51,51,51,51,51,51,51]
 ; SSE2-NEXT:    pand %xmm0, %xmm1
 ; SSE2-NEXT:    psllw $2, %xmm1
@@ -535,57 +532,59 @@ define <2 x i64> @test_bitreverse_v2i64(<2 x i64> %a) nounwind {
 define <32 x i8> @test_bitreverse_v32i8(<32 x i8> %a) nounwind {
 ; SSE2-LABEL: test_bitreverse_v32i8:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; SSE2-NEXT:    movdqa %xmm1, %xmm2
+; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
 ; SSE2-NEXT:    movdqa %xmm0, %xmm3
-; SSE2-NEXT:    pand %xmm2, %xmm3
+; SSE2-NEXT:    pand %xmm1, %xmm3
 ; SSE2-NEXT:    psllw $4, %xmm3
-; SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
-; SSE2-NEXT:    pand %xmm5, %xmm3
-; SSE2-NEXT:    pand %xmm5, %xmm0
+; SSE2-NEXT:    movdqa %xmm1, %xmm4
+; SSE2-NEXT:    pandn %xmm3, %xmm4
+; SSE2-NEXT:    movdqa {{.*#+}} xmm8 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
+; SSE2-NEXT:    pand %xmm8, %xmm0
 ; SSE2-NEXT:    psrlw $4, %xmm0
-; SSE2-NEXT:    pand %xmm2, %xmm0
-; SSE2-NEXT:    por %xmm3, %xmm0
+; SSE2-NEXT:    pand %xmm1, %xmm0
+; SSE2-NEXT:    por %xmm4, %xmm0
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [51,51,51,51,51,51,51,51,51,51,51,51,51,51,51,51]
 ; SSE2-NEXT:    movdqa %xmm0, %xmm4
 ; SSE2-NEXT:    pand %xmm3, %xmm4
 ; SSE2-NEXT:    psllw $2, %xmm4
-; SSE2-NEXT:    movdqa {{.*#+}} xmm8 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252]
-; SSE2-NEXT:    pand %xmm8, %xmm4
-; SSE2-NEXT:    movdqa {{.*#+}} xmm9 = [204,204,204,204,204,204,204,204,204,204,204,204,204,204,204,204]
-; SSE2-NEXT:    pand %xmm9, %xmm0
-; SSE2-NEXT:    psrlw $2, %xmm0
-; SSE2-NEXT:    movdqa {{.*#+}} xmm10 = [63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm9 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252]
+; SSE2-NEXT:    pand %xmm9, %xmm4
+; SSE2-NEXT:    movdqa {{.*#+}} xmm10 = [204,204,204,204,204,204,204,204,204,204,204,204,204,204,204,204]
 ; SSE2-NEXT:    pand %xmm10, %xmm0
+; SSE2-NEXT:    psrlw $2, %xmm0
+; SSE2-NEXT:    movdqa {{.*#+}} xmm11 = [63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63]
+; SSE2-NEXT:    pand %xmm11, %xmm0
 ; SSE2-NEXT:    por %xmm4, %xmm0
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [170,170,170,170,170,170,170,170,170,170,170,170,170,170,170,170]
-; SSE2-NEXT:    movdqa %xmm0, %xmm7
-; SSE2-NEXT:    pand %xmm4, %xmm7
-; SSE2-NEXT:    psrlw $1, %xmm7
-; SSE2-NEXT:    movdqa {{.*#+}} xmm11 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
-; SSE2-NEXT:    pand %xmm11, %xmm7
-; SSE2-NEXT:    movdqa {{.*#+}} xmm6 = [85,85,85,85,85,85,85,85,85,85,85,85,85,85,85,85]
-; SSE2-NEXT:    pand %xmm6, %xmm0
+; SSE2-NEXT:    movdqa %xmm0, %xmm5
+; SSE2-NEXT:    pand %xmm4, %xmm5
+; SSE2-NEXT:    psrlw $1, %xmm5
+; SSE2-NEXT:    movdqa {{.*#+}} xmm6 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
+; SSE2-NEXT:    pand %xmm6, %xmm5
+; SSE2-NEXT:    movdqa {{.*#+}} xmm7 = [85,85,85,85,85,85,85,85,85,85,85,85,85,85,85,85]
+; SSE2-NEXT:    pand %xmm7, %xmm0
 ; SSE2-NEXT:    paddb %xmm0, %xmm0
-; SSE2-NEXT:    por %xmm7, %xmm0
-; SSE2-NEXT:    movdqa %xmm1, %xmm7
-; SSE2-NEXT:    pand %xmm2, %xmm7
-; SSE2-NEXT:    psllw $4, %xmm7
-; SSE2-NEXT:    pand %xmm5, %xmm7
-; SSE2-NEXT:    pand %xmm5, %xmm1
-; SSE2-NEXT:    psrlw $4, %xmm1
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    por %xmm7, %xmm1
+; SSE2-NEXT:    por %xmm5, %xmm0
+; SSE2-NEXT:    movdqa %xmm2, %xmm5
+; SSE2-NEXT:    pand %xmm1, %xmm5
+; SSE2-NEXT:    psllw $4, %xmm5
+; SSE2-NEXT:    pand %xmm8, %xmm2
+; SSE2-NEXT:    psrlw $4, %xmm2
+; SSE2-NEXT:    pand %xmm1, %xmm2
+; SSE2-NEXT:    pandn %xmm5, %xmm1
+; SSE2-NEXT:    por %xmm2, %xmm1
 ; SSE2-NEXT:    pand %xmm1, %xmm3
 ; SSE2-NEXT:    psllw $2, %xmm3
-; SSE2-NEXT:    pand %xmm8, %xmm3
-; SSE2-NEXT:    pand %xmm9, %xmm1
-; SSE2-NEXT:    psrlw $2, %xmm1
+; SSE2-NEXT:    pand %xmm9, %xmm3
 ; SSE2-NEXT:    pand %xmm10, %xmm1
+; SSE2-NEXT:    psrlw $2, %xmm1
+; SSE2-NEXT:    pand %xmm11, %xmm1
 ; SSE2-NEXT:    por %xmm3, %xmm1
 ; SSE2-NEXT:    pand %xmm1, %xmm4
 ; SSE2-NEXT:    psrlw $1, %xmm4
-; SSE2-NEXT:    pand %xmm11, %xmm4
-; SSE2-NEXT:    pand %xmm6, %xmm1
+; SSE2-NEXT:    pand %xmm6, %xmm4
+; SSE2-NEXT:    pand %xmm7, %xmm1
 ; SSE2-NEXT:    paddb %xmm1, %xmm1
 ; SSE2-NEXT:    por %xmm4, %xmm1
 ; SSE2-NEXT:    retq
@@ -686,74 +685,76 @@ define <32 x i8> @test_bitreverse_v32i8(<32 x i8> %a) nounwind {
 define <16 x i16> @test_bitreverse_v16i16(<16 x i16> %a) nounwind {
 ; SSE2-LABEL: test_bitreverse_v16i16:
 ; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa %xmm1, %xmm2
 ; SSE2-NEXT:    pxor %xmm4, %xmm4
-; SSE2-NEXT:    movdqa %xmm0, %xmm2
-; SSE2-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm4[8],xmm2[9],xmm4[9],xmm2[10],xmm4[10],xmm2[11],xmm4[11],xmm2[12],xmm4[12],xmm2[13],xmm4[13],xmm2[14],xmm4[14],xmm2[15],xmm4[15]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm2 = xmm2[1,0,3,2,4,5,6,7]
-; SSE2-NEXT:    pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,5,4,7,6]
+; SSE2-NEXT:    movdqa %xmm0, %xmm1
+; SSE2-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm4[8],xmm1[9],xmm4[9],xmm1[10],xmm4[10],xmm1[11],xmm4[11],xmm1[12],xmm4[12],xmm1[13],xmm4[13],xmm1[14],xmm4[14],xmm1[15],xmm4[15]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[1,0,3,2,4,5,6,7]
+; SSE2-NEXT:    pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,5,4,7,6]
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3],xmm0[4],xmm4[4],xmm0[5],xmm4[5],xmm0[6],xmm4[6],xmm0[7],xmm4[7]
 ; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[1,0,3,2,4,5,6,7]
 ; SSE2-NEXT:    pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,5,4,7,6]
-; SSE2-NEXT:    packuswb %xmm2, %xmm0
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; SSE2-NEXT:    packuswb %xmm1, %xmm0
+; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
 ; SSE2-NEXT:    movdqa %xmm0, %xmm3
-; SSE2-NEXT:    pand %xmm2, %xmm3
+; SSE2-NEXT:    pand %xmm1, %xmm3
 ; SSE2-NEXT:    psllw $4, %xmm3
-; SSE2-NEXT:    movdqa {{.*#+}} xmm6 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
-; SSE2-NEXT:    pand %xmm6, %xmm3
-; SSE2-NEXT:    pand %xmm6, %xmm0
+; SSE2-NEXT:    movdqa %xmm1, %xmm5
+; SSE2-NEXT:    pandn %xmm3, %xmm5
+; SSE2-NEXT:    movdqa {{.*#+}} xmm8 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
+; SSE2-NEXT:    pand %xmm8, %xmm0
 ; SSE2-NEXT:    psrlw $4, %xmm0
-; SSE2-NEXT:    pand %xmm2, %xmm0
-; SSE2-NEXT:    por %xmm3, %xmm0
+; SSE2-NEXT:    pand %xmm1, %xmm0
+; SSE2-NEXT:    por %xmm5, %xmm0
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [51,51,51,51,51,51,51,51,51,51,51,51,51,51,51,51]
 ; SSE2-NEXT:    movdqa %xmm0, %xmm5
 ; SSE2-NEXT:    pand %xmm3, %xmm5
 ; SSE2-NEXT:    psllw $2, %xmm5
-; SSE2-NEXT:    movdqa {{.*#+}} xmm8 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252]
-; SSE2-NEXT:    pand %xmm8, %xmm5
-; SSE2-NEXT:    movdqa {{.*#+}} xmm9 = [204,204,204,204,204,204,204,204,204,204,204,204,204,204,204,204]
-; SSE2-NEXT:    pand %xmm9, %xmm0
-; SSE2-NEXT:    psrlw $2, %xmm0
-; SSE2-NEXT:    movdqa {{.*#+}} xmm10 = [63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm9 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252]
+; SSE2-NEXT:    pand %xmm9, %xmm5
+; SSE2-NEXT:    movdqa {{.*#+}} xmm10 = [204,204,204,204,204,204,204,204,204,204,204,204,204,204,204,204]
 ; SSE2-NEXT:    pand %xmm10, %xmm0
+; SSE2-NEXT:    psrlw $2, %xmm0
+; SSE2-NEXT:    movdqa {{.*#+}} xmm11 = [63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63]
+; SSE2-NEXT:    pand %xmm11, %xmm0
 ; SSE2-NEXT:    por %xmm5, %xmm0
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [170,170,170,170,170,170,170,170,170,170,170,170,170,170,170,170]
-; SSE2-NEXT:    movdqa %xmm0, %xmm7
-; SSE2-NEXT:    pand %xmm5, %xmm7
-; SSE2-NEXT:    psrlw $1, %xmm7
-; SSE2-NEXT:    movdqa {{.*#+}} xmm11 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
-; SSE2-NEXT:    pand %xmm11, %xmm7
-; SSE2-NEXT:    movdqa {{.*#+}} xmm12 = [85,85,85,85,85,85,85,85,85,85,85,85,85,85,85,85]
-; SSE2-NEXT:    pand %xmm12, %xmm0
+; SSE2-NEXT:    movdqa %xmm0, %xmm6
+; SSE2-NEXT:    pand %xmm5, %xmm6
+; SSE2-NEXT:    psrlw $1, %xmm6
+; SSE2-NEXT:    movdqa {{.*#+}} xmm12 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
+; SSE2-NEXT:    pand %xmm12, %xmm6
+; SSE2-NEXT:    movdqa {{.*#+}} xmm7 = [85,85,85,85,85,85,85,85,85,85,85,85,85,85,85,85]
+; SSE2-NEXT:    pand %xmm7, %xmm0
 ; SSE2-NEXT:    paddb %xmm0, %xmm0
-; SSE2-NEXT:    por %xmm7, %xmm0
-; SSE2-NEXT:    movdqa %xmm1, %xmm7
-; SSE2-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8],xmm4[8],xmm7[9],xmm4[9],xmm7[10],xmm4[10],xmm7[11],xmm4[11],xmm7[12],xmm4[12],xmm7[13],xmm4[13],xmm7[14],xmm4[14],xmm7[15],xmm4[15]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm7 = xmm7[1,0,3,2,4,5,6,7]
-; SSE2-NEXT:    pshufhw {{.*#+}} xmm7 = xmm7[0,1,2,3,5,4,7,6]
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1],xmm1[2],xmm4[2],xmm1[3],xmm4[3],xmm1[4],xmm4[4],xmm1[5],xmm4[5],xmm1[6],xmm4[6],xmm1[7],xmm4[7]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[1,0,3,2,4,5,6,7]
-; SSE2-NEXT:    pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,5,4,7,6]
-; SSE2-NEXT:    packuswb %xmm7, %xmm1
-; SSE2-NEXT:    movdqa %xmm1, %xmm4
-; SSE2-NEXT:    pand %xmm2, %xmm4
+; SSE2-NEXT:    por %xmm6, %xmm0
+; SSE2-NEXT:    movdqa %xmm2, %xmm6
+; SSE2-NEXT:    punpckhbw {{.*#+}} xmm6 = xmm6[8],xmm4[8],xmm6[9],xmm4[9],xmm6[10],xmm4[10],xmm6[11],xmm4[11],xmm6[12],xmm4[12],xmm6[13],xmm4[13],xmm6[14],xmm4[14],xmm6[15],xmm4[15]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm6 = xmm6[1,0,3,2,4,5,6,7]
+; SSE2-NEXT:    pshufhw {{.*#+}} xmm6 = xmm6[0,1,2,3,5,4,7,6]
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3],xmm2[4],xmm4[4],xmm2[5],xmm4[5],xmm2[6],xmm4[6],xmm2[7],xmm4[7]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm2 = xmm2[1,0,3,2,4,5,6,7]
+; SSE2-NEXT:    pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,5,4,7,6]
+; SSE2-NEXT:    packuswb %xmm6, %xmm2
+; SSE2-NEXT:    movdqa %xmm2, %xmm4
+; SSE2-NEXT:    pand %xmm1, %xmm4
 ; SSE2-NEXT:    psllw $4, %xmm4
-; SSE2-NEXT:    pand %xmm6, %xmm4
-; SSE2-NEXT:    pand %xmm6, %xmm1
-; SSE2-NEXT:    psrlw $4, %xmm1
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    por %xmm4, %xmm1
+; SSE2-NEXT:    pand %xmm8, %xmm2
+; SSE2-NEXT:    psrlw $4, %xmm2
+; SSE2-NEXT:    pand %xmm1, %xmm2
+; SSE2-NEXT:    pandn %xmm4, %xmm1
+; SSE2-NEXT:    por %xmm2, %xmm1
 ; SSE2-NEXT:    pand %xmm1, %xmm3
 ; SSE2-NEXT:    psllw $2, %xmm3
-; SSE2-NEXT:    pand %xmm8, %xmm3
-; SSE2-NEXT:    pand %xmm9, %xmm1
-; SSE2-NEXT:    psrlw $2, %xmm1
+; SSE2-NEXT:    pand %xmm9, %xmm3
 ; SSE2-NEXT:    pand %xmm10, %xmm1
+; SSE2-NEXT:    psrlw $2, %xmm1
+; SSE2-NEXT:    pand %xmm11, %xmm1
 ; SSE2-NEXT:    por %xmm3, %xmm1
 ; SSE2-NEXT:    pand %xmm1, %xmm5
 ; SSE2-NEXT:    psrlw $1, %xmm5
-; SSE2-NEXT:    pand %xmm11, %xmm5
-; SSE2-NEXT:    pand %xmm12, %xmm1
+; SSE2-NEXT:    pand %xmm12, %xmm5
+; SSE2-NEXT:    pand %xmm7, %xmm1
 ; SSE2-NEXT:    paddb %xmm1, %xmm1
 ; SSE2-NEXT:    por %xmm5, %xmm1
 ; SSE2-NEXT:    retq
@@ -862,74 +863,76 @@ define <16 x i16> @test_bitreverse_v16i16(<16 x i16> %a) nounwind {
 define <8 x i32> @test_bitreverse_v8i32(<8 x i32> %a) nounwind {
 ; SSE2-LABEL: test_bitreverse_v8i32:
 ; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa %xmm1, %xmm2
 ; SSE2-NEXT:    pxor %xmm4, %xmm4
-; SSE2-NEXT:    movdqa %xmm0, %xmm2
-; SSE2-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm4[8],xmm2[9],xmm4[9],xmm2[10],xmm4[10],xmm2[11],xmm4[11],xmm2[12],xmm4[12],xmm2[13],xmm4[13],xmm2[14],xmm4[14],xmm2[15],xmm4[15]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm2 = xmm2[3,2,1,0,4,5,6,7]
-; SSE2-NEXT:    pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,7,6,5,4]
+; SSE2-NEXT:    movdqa %xmm0, %xmm1
+; SSE2-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm4[8],xmm1[9],xmm4[9],xmm1[10],xmm4[10],xmm1[11],xmm4[11],xmm1[12],xmm4[12],xmm1[13],xmm4[13],xmm1[14],xmm4[14],xmm1[15],xmm4[15]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[3,2,1,0,4,5,6,7]
+; SSE2-NEXT:    pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,7,6,5,4]
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3],xmm0[4],xmm4[4],xmm0[5],xmm4[5],xmm0[6],xmm4[6],xmm0[7],xmm4[7]
 ; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[3,2,1,0,4,5,6,7]
 ; SSE2-NEXT:    pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,7,6,5,4]
-; SSE2-NEXT:    packuswb %xmm2, %xmm0
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; SSE2-NEXT:    packuswb %xmm1, %xmm0
+; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
 ; SSE2-NEXT:    movdqa %xmm0, %xmm3
-; SSE2-NEXT:    pand %xmm2, %xmm3
+; SSE2-NEXT:    pand %xmm1, %xmm3
 ; SSE2-NEXT:    psllw $4, %xmm3
-; SSE2-NEXT:    movdqa {{.*#+}} xmm6 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
-; SSE2-NEXT:    pand %xmm6, %xmm3
-; SSE2-NEXT:    pand %xmm6, %xmm0
+; SSE2-NEXT:    movdqa %xmm1, %xmm5
+; SSE2-NEXT:    pandn %xmm3, %xmm5
+; SSE2-NEXT:    movdqa {{.*#+}} xmm8 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
+; SSE2-NEXT:    pand %xmm8, %xmm0
 ; SSE2-NEXT:    psrlw $4, %xmm0
-; SSE2-NEXT:    pand %xmm2, %xmm0
-; SSE2-NEXT:    por %xmm3, %xmm0
+; SSE2-NEXT:    pand %xmm1, %xmm0
+; SSE2-NEXT:    por %xmm5, %xmm0
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [51,51,51,51,51,51,51,51,51,51,51,51,51,51,51,51]
 ; SSE2-NEXT:    movdqa %xmm0, %xmm5
 ; SSE2-NEXT:    pand %xmm3, %xmm5
 ; SSE2-NEXT:    psllw $2, %xmm5
-; SSE2-NEXT:    movdqa {{.*#+}} xmm8 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252]
-; SSE2-NEXT:    pand %xmm8, %xmm5
-; SSE2-NEXT:    movdqa {{.*#+}} xmm9 = [204,204,204,204,204,204,204,204,204,204,204,204,204,204,204,204]
-; SSE2-NEXT:    pand %xmm9, %xmm0
-; SSE2-NEXT:    psrlw $2, %xmm0
-; SSE2-NEXT:    movdqa {{.*#+}} xmm10 = [63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm9 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252]
+; SSE2-NEXT:    pand %xmm9, %xmm5
+; SSE2-NEXT:    movdqa {{.*#+}} xmm10 = [204,204,204,204,204,204,204,204,204,204,204,204,204,204,204,204]
 ; SSE2-NEXT:    pand %xmm10, %xmm0
+; SSE2-NEXT:    psrlw $2, %xmm0
+; SSE2-NEXT:    movdqa {{.*#+}} xmm11 = [63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63]
+; SSE2-NEXT:    pand %xmm11, %xmm0
 ; SSE2-NEXT:    por %xmm5, %xmm0
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [170,170,170,170,170,170,170,170,170,170,170,170,170,170,170,170]
-; SSE2-NEXT:    movdqa %xmm0, %xmm7
-; SSE2-NEXT:    pand %xmm5, %xmm7
-; SSE2-NEXT:    psrlw $1, %xmm7
-; SSE2-NEXT:    movdqa {{.*#+}} xmm11 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
-; SSE2-NEXT:    pand %xmm11, %xmm7
-; SSE2-NEXT:    movdqa {{.*#+}} xmm12 = [85,85,85,85,85,85,85,85,85,85,85,85,85,85,85,85]
-; SSE2-NEXT:    pand %xmm12, %xmm0
+; SSE2-NEXT:    movdqa %xmm0, %xmm6
+; SSE2-NEXT:    pand %xmm5, %xmm6
+; SSE2-NEXT:    psrlw $1, %xmm6
+; SSE2-NEXT:    movdqa {{.*#+}} xmm12 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
+; SSE2-NEXT:    pand %xmm12, %xmm6
+; SSE2-NEXT:    movdqa {{.*#+}} xmm7 = [85,85,85,85,85,85,85,85,85,85,85,85,85,85,85,85]
+; SSE2-NEXT:    pand %xmm7, %xmm0
 ; SSE2-NEXT:    paddb %xmm0, %xmm0
-; SSE2-NEXT:    por %xmm7, %xmm0
-; SSE2-NEXT:    movdqa %xmm1, %xmm7
-; SSE2-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8],xmm4[8],xmm7[9],xmm4[9],xmm7[10],xmm4[10],xmm7[11],xmm4[11],xmm7[12],xmm4[12],xmm7[13],xmm4[13],xmm7[14],xmm4[14],xmm7[15],xmm4[15]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm7 = xmm7[3,2,1,0,4,5,6,7]
-; SSE2-NEXT:    pshufhw {{.*#+}} xmm7 = xmm7[0,1,2,3,7,6,5,4]
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1],xmm1[2],xmm4[2],xmm1[3],xmm4[3],xmm1[4],xmm4[4],xmm1[5],xmm4[5],xmm1[6],xmm4[6],xmm1[7],xmm4[7]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[3,2,1,0,4,5,6,7]
-; SSE2-NEXT:    pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,7,6,5,4]
-; SSE2-NEXT:    packuswb %xmm7, %xmm1
-; SSE2-NEXT:    movdqa %xmm1, %xmm4
-; SSE2-NEXT:    pand %xmm2, %xmm4
+; SSE2-NEXT:    por %xmm6, %xmm0
+; SSE2-NEXT:    movdqa %xmm2, %xmm6
+; SSE2-NEXT:    punpckhbw {{.*#+}} xmm6 = xmm6[8],xmm4[8],xmm6[9],xmm4[9],xmm6[10],xmm4[10],xmm6[11],xmm4[11],xmm6[12],xmm4[12],xmm6[13],xmm4[13],xmm6[14],xmm4[14],xmm6[15],xmm4[15]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm6 = xmm6[3,2,1,0,4,5,6,7]
+; SSE2-NEXT:    pshufhw {{.*#+}} xmm6 = xmm6[0,1,2,3,7,6,5,4]
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3],xmm2[4],xmm4[4],xmm2[5],xmm4[5],xmm2[6],xmm4[6],xmm2[7],xmm4[7]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm2 = xmm2[3,2,1,0,4,5,6,7]
+; SSE2-NEXT:    pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,7,6,5,4]
+; SSE2-NEXT:    packuswb %xmm6, %xmm2
+; SSE2-NEXT:    movdqa %xmm2, %xmm4
+; SSE2-NEXT:    pand %xmm1, %xmm4
 ; SSE2-NEXT:    psllw $4, %xmm4
-; SSE2-NEXT:    pand %xmm6, %xmm4
-; SSE2-NEXT:    pand %xmm6, %xmm1
-; SSE2-NEXT:    psrlw $4, %xmm1
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    por %xmm4, %xmm1
+; SSE2-NEXT:    pand %xmm8, %xmm2
+; SSE2-NEXT:    psrlw $4, %xmm2
+; SSE2-NEXT:    pand %xmm1, %xmm2
+; SSE2-NEXT:    pandn %xmm4, %xmm1
+; SSE2-NEXT:    por %xmm2, %xmm1
 ; SSE2-NEXT:    pand %xmm1, %xmm3
 ; SSE2-NEXT:    psllw $2, %xmm3
-; SSE2-NEXT:    pand %xmm8, %xmm3
-; SSE2-NEXT:    pand %xmm9, %xmm1
-; SSE2-NEXT:    psrlw $2, %xmm1
+; SSE2-NEXT:    pand %xmm9, %xmm3
 ; SSE2-NEXT:    pand %xmm10, %xmm1
+; SSE2-NEXT:    psrlw $2, %xmm1
+; SSE2-NEXT:    pand %xmm11, %xmm1
 ; SSE2-NEXT:    por %xmm3, %xmm1
 ; SSE2-NEXT:    pand %xmm1, %xmm5
 ; SSE2-NEXT:    psrlw $1, %xmm5
-; SSE2-NEXT:    pand %xmm11, %xmm5
-; SSE2-NEXT:    pand %xmm12, %xmm1
+; SSE2-NEXT:    pand %xmm12, %xmm5
+; SSE2-NEXT:    pand %xmm7, %xmm1
 ; SSE2-NEXT:    paddb %xmm1, %xmm1
 ; SSE2-NEXT:    por %xmm5, %xmm1
 ; SSE2-NEXT:    retq
@@ -1038,78 +1041,80 @@ define <8 x i32> @test_bitreverse_v8i32(<8 x i32> %a) nounwind {
 define <4 x i64> @test_bitreverse_v4i64(<4 x i64> %a) nounwind {
 ; SSE2-LABEL: test_bitreverse_v4i64:
 ; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa %xmm1, %xmm2
 ; SSE2-NEXT:    pxor %xmm4, %xmm4
-; SSE2-NEXT:    movdqa %xmm0, %xmm2
-; SSE2-NEXT:    punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm4[8],xmm2[9],xmm4[9],xmm2[10],xmm4[10],xmm2[11],xmm4[11],xmm2[12],xmm4[12],xmm2[13],xmm4[13],xmm2[14],xmm4[14],xmm2[15],xmm4[15]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[2,3,0,1]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm2 = xmm2[3,2,1,0,4,5,6,7]
-; SSE2-NEXT:    pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,7,6,5,4]
+; SSE2-NEXT:    movdqa %xmm0, %xmm1
+; SSE2-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm4[8],xmm1[9],xmm4[9],xmm1[10],xmm4[10],xmm1[11],xmm4[11],xmm1[12],xmm4[12],xmm1[13],xmm4[13],xmm1[14],xmm4[14],xmm1[15],xmm4[15]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[3,2,1,0,4,5,6,7]
+; SSE2-NEXT:    pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,7,6,5,4]
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3],xmm0[4],xmm4[4],xmm0[5],xmm4[5],xmm0[6],xmm4[6],xmm0[7],xmm4[7]
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
 ; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[3,2,1,0,4,5,6,7]
 ; SSE2-NEXT:    pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,7,6,5,4]
-; SSE2-NEXT:    packuswb %xmm2, %xmm0
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; SSE2-NEXT:    packuswb %xmm1, %xmm0
+; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
 ; SSE2-NEXT:    movdqa %xmm0, %xmm3
-; SSE2-NEXT:    pand %xmm2, %xmm3
+; SSE2-NEXT:    pand %xmm1, %xmm3
 ; SSE2-NEXT:    psllw $4, %xmm3
-; SSE2-NEXT:    movdqa {{.*#+}} xmm6 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
-; SSE2-NEXT:    pand %xmm6, %xmm3
-; SSE2-NEXT:    pand %xmm6, %xmm0
+; SSE2-NEXT:    movdqa %xmm1, %xmm5
+; SSE2-NEXT:    pandn %xmm3, %xmm5
+; SSE2-NEXT:    movdqa {{.*#+}} xmm8 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
+; SSE2-NEXT:    pand %xmm8, %xmm0
 ; SSE2-NEXT:    psrlw $4, %xmm0
-; SSE2-NEXT:    pand %xmm2, %xmm0
-; SSE2-NEXT:    por %xmm3, %xmm0
+; SSE2-NEXT:    pand %xmm1, %xmm0
+; SSE2-NEXT:    por %xmm5, %xmm0
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [51,51,51,51,51,51,51,51,51,51,51,51,51,51,51,51]
 ; SSE2-NEXT:    movdqa %xmm0, %xmm5
 ; SSE2-NEXT:    pand %xmm3, %xmm5
 ; SSE2-NEXT:    psllw $2, %xmm5
-; SSE2-NEXT:    movdqa {{.*#+}} xmm8 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252]
-; SSE2-NEXT:    pand %xmm8, %xmm5
-; SSE2-NEXT:    movdqa {{.*#+}} xmm9 = [204,204,204,204,204,204,204,204,204,204,204,204,204,204,204,204]
-; SSE2-NEXT:    pand %xmm9, %xmm0
-; SSE2-NEXT:    psrlw $2, %xmm0
-; SSE2-NEXT:    movdqa {{.*#+}} xmm10 = [63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm9 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252]
+; SSE2-NEXT:    pand %xmm9, %xmm5
+; SSE2-NEXT:    movdqa {{.*#+}} xmm10 = [204,204,204,204,204,204,204,204,204,204,204,204,204,204,204,204]
 ; SSE2-NEXT:    pand %xmm10, %xmm0
+; SSE2-NEXT:    psrlw $2, %xmm0
+; SSE2-NEXT:    movdqa {{.*#+}} xmm11 = [63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63]
+; SSE2-NEXT:    pand %xmm11, %xmm0
 ; SSE2-NEXT:    por %xmm5, %xmm0
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [170,170,170,170,170,170,170,170,170,170,170,170,170,170,170,170]
-; SSE2-NEXT:    movdqa %xmm0, %xmm7
-; SSE2-NEXT:    pand %xmm5, %xmm7
-; SSE2-NEXT:    psrlw $1, %xmm7
-; SSE2-NEXT:    movdqa {{.*#+}} xmm11 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
-; SSE2-NEXT:    pand %xmm11, %xmm7
-; SSE2-NEXT:    movdqa {{.*#+}} xmm12 = [85,85,85,85,85,85,85,85,85,85,85,85,85,85,85,85]
-; SSE2-NEXT:    pand %xmm12, %xmm0
+; SSE2-NEXT:    movdqa %xmm0, %xmm6
+; SSE2-NEXT:    pand %xmm5, %xmm6
+; SSE2-NEXT:    psrlw $1, %xmm6
+; SSE2-NEXT:    movdqa {{.*#+}} xmm12 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
+; SSE2-NEXT:    pand %xmm12, %xmm6
+; SSE2-NEXT:    movdqa {{.*#+}} xmm7 = [85,85,85,85,85,85,85,85,85,85,85,85,85,85,85,85]
+; SSE2-NEXT:    pand %xmm7, %xmm0
 ; SSE2-NEXT:    paddb %xmm0, %xmm0
-; SSE2-NEXT:    por %xmm7, %xmm0
-; SSE2-NEXT:    movdqa %xmm1, %xmm7
-; SSE2-NEXT:    punpckhbw {{.*#+}} xmm7 = xmm7[8],xmm4[8],xmm7[9],xmm4[9],xmm7[10],xmm4[10],xmm7[11],xmm4[11],xmm7[12],xmm4[12],xmm7[13],xmm4[13],xmm7[14],xmm4[14],xmm7[15],xmm4[15]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm7[2,3,0,1]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm7 = xmm7[3,2,1,0,4,5,6,7]
-; SSE2-NEXT:    pshufhw {{.*#+}} xmm7 = xmm7[0,1,2,3,7,6,5,4]
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1],xmm1[2],xmm4[2],xmm1[3],xmm4[3],xmm1[4],xmm4[4],xmm1[5],xmm4[5],xmm1[6],xmm4[6],xmm1[7],xmm4[7]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[3,2,1,0,4,5,6,7]
-; SSE2-NEXT:    pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,7,6,5,4]
-; SSE2-NEXT:    packuswb %xmm7, %xmm1
-; SSE2-NEXT:    movdqa %xmm1, %xmm4
-; SSE2-NEXT:    pand %xmm2, %xmm4
+; SSE2-NEXT:    por %xmm6, %xmm0
+; SSE2-NEXT:    movdqa %xmm2, %xmm6
+; SSE2-NEXT:    punpckhbw {{.*#+}} xmm6 = xmm6[8],xmm4[8],xmm6[9],xmm4[9],xmm6[10],xmm4[10],xmm6[11],xmm4[11],xmm6[12],xmm4[12],xmm6[13],xmm4[13],xmm6[14],xmm4[14],xmm6[15],xmm4[15]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[2,3,0,1]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm6 = xmm6[3,2,1,0,4,5,6,7]
+; SSE2-NEXT:    pshufhw {{.*#+}} xmm6 = xmm6[0,1,2,3,7,6,5,4]
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3],xmm2[4],xmm4[4],xmm2[5],xmm4[5],xmm2[6],xmm4[6],xmm2[7],xmm4[7]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[2,3,0,1]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm2 = xmm2[3,2,1,0,4,5,6,7]
+; SSE2-NEXT:    pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,7,6,5,4]
+; SSE2-NEXT:    packuswb %xmm6, %xmm2
+; SSE2-NEXT:    movdqa %xmm2, %xmm4
+; SSE2-NEXT:    pand %xmm1, %xmm4
 ; SSE2-NEXT:    psllw $4, %xmm4
-; SSE2-NEXT:    pand %xmm6, %xmm4
-; SSE2-NEXT:    pand %xmm6, %xmm1
-; SSE2-NEXT:    psrlw $4, %xmm1
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    por %xmm4, %xmm1
+; SSE2-NEXT:    pand %xmm8, %xmm2
+; SSE2-NEXT:    psrlw $4, %xmm2
+; SSE2-NEXT:    pand %xmm1, %xmm2
+; SSE2-NEXT:    pandn %xmm4, %xmm1
+; SSE2-NEXT:    por %xmm2, %xmm1
 ; SSE2-NEXT:    pand %xmm1, %xmm3
 ; SSE2-NEXT:    psllw $2, %xmm3
-; SSE2-NEXT:    pand %xmm8, %xmm3
-; SSE2-NEXT:    pand %xmm9, %xmm1
-; SSE2-NEXT:    psrlw $2, %xmm1
+; SSE2-NEXT:    pand %xmm9, %xmm3
 ; SSE2-NEXT:    pand %xmm10, %xmm1
+; SSE2-NEXT:    psrlw $2, %xmm1
+; SSE2-NEXT:    pand %xmm11, %xmm1
 ; SSE2-NEXT:    por %xmm3, %xmm1
 ; SSE2-NEXT:    pand %xmm1, %xmm5
 ; SSE2-NEXT:    psrlw $1, %xmm5
-; SSE2-NEXT:    pand %xmm11, %xmm5
-; SSE2-NEXT:    pand %xmm12, %xmm1
+; SSE2-NEXT:    pand %xmm12, %xmm5
+; SSE2-NEXT:    pand %xmm7, %xmm1
 ; SSE2-NEXT:    paddb %xmm1, %xmm1
 ; SSE2-NEXT:    por %xmm5, %xmm1
 ; SSE2-NEXT:    retq
@@ -1218,103 +1223,107 @@ define <4 x i64> @test_bitreverse_v4i64(<4 x i64> %a) nounwind {
 define <64 x i8> @test_bitreverse_v64i8(<64 x i8> %a) nounwind {
 ; SSE2-LABEL: test_bitreverse_v64i8:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    movdqa {{.*#+}} xmm13 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; SSE2-NEXT:    movdqa %xmm3, %xmm14
+; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
 ; SSE2-NEXT:    movdqa %xmm0, %xmm5
-; SSE2-NEXT:    pand %xmm13, %xmm5
+; SSE2-NEXT:    pand %xmm3, %xmm5
 ; SSE2-NEXT:    psllw $4, %xmm5
-; SSE2-NEXT:    movdqa {{.*#+}} xmm7 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
-; SSE2-NEXT:    pand %xmm7, %xmm5
-; SSE2-NEXT:    pand %xmm7, %xmm0
+; SSE2-NEXT:    movdqa %xmm3, %xmm6
+; SSE2-NEXT:    pandn %xmm5, %xmm6
+; SSE2-NEXT:    movdqa {{.*#+}} xmm9 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
+; SSE2-NEXT:    pand %xmm9, %xmm0
 ; SSE2-NEXT:    psrlw $4, %xmm0
-; SSE2-NEXT:    pand %xmm13, %xmm0
-; SSE2-NEXT:    por %xmm5, %xmm0
+; SSE2-NEXT:    pand %xmm3, %xmm0
+; SSE2-NEXT:    por %xmm6, %xmm0
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [51,51,51,51,51,51,51,51,51,51,51,51,51,51,51,51]
 ; SSE2-NEXT:    movdqa %xmm0, %xmm6
 ; SSE2-NEXT:    pand %xmm5, %xmm6
 ; SSE2-NEXT:    psllw $2, %xmm6
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm8 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252]
 ; SSE2-NEXT:    pand %xmm8, %xmm6
-; SSE2-NEXT:    movdqa {{.*#+}} xmm9 = [204,204,204,204,204,204,204,204,204,204,204,204,204,204,204,204]
-; SSE2-NEXT:    pand %xmm9, %xmm0
-; SSE2-NEXT:    psrlw $2, %xmm0
-; SSE2-NEXT:    movdqa {{.*#+}} xmm10 = [63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63]
+; SSE2-NEXT:    movdqa {{.*#+}} xmm10 = [204,204,204,204,204,204,204,204,204,204,204,204,204,204,204,204]
 ; SSE2-NEXT:    pand %xmm10, %xmm0
+; SSE2-NEXT:    psrlw $2, %xmm0
+; SSE2-NEXT:    movdqa {{.*#+}} xmm11 = [63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63]
+; SSE2-NEXT:    pand %xmm11, %xmm0
 ; SSE2-NEXT:    por %xmm6, %xmm0
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm6 = [170,170,170,170,170,170,170,170,170,170,170,170,170,170,170,170]
-; SSE2-NEXT:    movdqa %xmm0, %xmm4
-; SSE2-NEXT:    pand %xmm6, %xmm4
-; SSE2-NEXT:    psrlw $1, %xmm4
-; SSE2-NEXT:    movdqa {{.*#+}} xmm11 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
-; SSE2-NEXT:    pand %xmm11, %xmm4
-; SSE2-NEXT:    movdqa {{.*#+}} xmm12 = [85,85,85,85,85,85,85,85,85,85,85,85,85,85,85,85]
-; SSE2-NEXT:    pand %xmm12, %xmm0
+; SSE2-NEXT:    movdqa %xmm0, %xmm7
+; SSE2-NEXT:    pand %xmm6, %xmm7
+; SSE2-NEXT:    psrlw $1, %xmm7
+; SSE2-NEXT:    movdqa {{.*#+}} xmm12 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
+; SSE2-NEXT:    pand %xmm12, %xmm7
+; SSE2-NEXT:    movdqa {{.*#+}} xmm13 = [85,85,85,85,85,85,85,85,85,85,85,85,85,85,85,85]
+; SSE2-NEXT:    pand %xmm13, %xmm0
 ; SSE2-NEXT:    paddb %xmm0, %xmm0
-; SSE2-NEXT:    por %xmm4, %xmm0
-; SSE2-NEXT:    movdqa %xmm1, %xmm4
-; SSE2-NEXT:    pand %xmm13, %xmm4
-; SSE2-NEXT:    psllw $4, %xmm4
-; SSE2-NEXT:    pand %xmm7, %xmm4
-; SSE2-NEXT:    pand %xmm7, %xmm1
+; SSE2-NEXT:    por %xmm7, %xmm0
+; SSE2-NEXT:    movdqa %xmm1, %xmm7
+; SSE2-NEXT:    pand %xmm3, %xmm7
+; SSE2-NEXT:    psllw $4, %xmm7
+; SSE2-NEXT:    movdqa %xmm3, %xmm4
+; SSE2-NEXT:    pandn %xmm7, %xmm4
+; SSE2-NEXT:    pand %xmm9, %xmm1
 ; SSE2-NEXT:    psrlw $4, %xmm1
-; SSE2-NEXT:    pand %xmm13, %xmm1
+; SSE2-NEXT:    pand %xmm3, %xmm1
 ; SSE2-NEXT:    por %xmm4, %xmm1
 ; SSE2-NEXT:    movdqa %xmm1, %xmm4
 ; SSE2-NEXT:    pand %xmm5, %xmm4
 ; SSE2-NEXT:    psllw $2, %xmm4
 ; SSE2-NEXT:    pand %xmm8, %xmm4
-; SSE2-NEXT:    pand %xmm9, %xmm1
-; SSE2-NEXT:    psrlw $2, %xmm1
 ; SSE2-NEXT:    pand %xmm10, %xmm1
+; SSE2-NEXT:    psrlw $2, %xmm1
+; SSE2-NEXT:    pand %xmm11, %xmm1
 ; SSE2-NEXT:    por %xmm4, %xmm1
 ; SSE2-NEXT:    movdqa %xmm1, %xmm4
 ; SSE2-NEXT:    pand %xmm6, %xmm4
 ; SSE2-NEXT:    psrlw $1, %xmm4
-; SSE2-NEXT:    pand %xmm11, %xmm4
-; SSE2-NEXT:    pand %xmm12, %xmm1
+; SSE2-NEXT:    pand %xmm12, %xmm4
+; SSE2-NEXT:    pand %xmm13, %xmm1
 ; SSE2-NEXT:    paddb %xmm1, %xmm1
 ; SSE2-NEXT:    por %xmm4, %xmm1
 ; SSE2-NEXT:    movdqa %xmm2, %xmm4
-; SSE2-NEXT:    pand %xmm13, %xmm4
+; SSE2-NEXT:    pand %xmm3, %xmm4
 ; SSE2-NEXT:    psllw $4, %xmm4
-; SSE2-NEXT:    pand %xmm7, %xmm4
-; SSE2-NEXT:    pand %xmm7, %xmm2
+; SSE2-NEXT:    movdqa %xmm3, %xmm7
+; SSE2-NEXT:    pandn %xmm4, %xmm7
+; SSE2-NEXT:    pand %xmm9, %xmm2
 ; SSE2-NEXT:    psrlw $4, %xmm2
-; SSE2-NEXT:    pand %xmm13, %xmm2
-; SSE2-NEXT:    por %xmm4, %xmm2
+; SSE2-NEXT:    pand %xmm3, %xmm2
+; SSE2-NEXT:    por %xmm7, %xmm2
 ; SSE2-NEXT:    movdqa %xmm2, %xmm4
 ; SSE2-NEXT:    pand %xmm5, %xmm4
 ; SSE2-NEXT:    psllw $2, %xmm4
 ; SSE2-NEXT:    pand %xmm8, %xmm4
-; SSE2-NEXT:    pand %xmm9, %xmm2
-; SSE2-NEXT:    psrlw $2, %xmm2
 ; SSE2-NEXT:    pand %xmm10, %xmm2
+; SSE2-NEXT:    psrlw $2, %xmm2
+; SSE2-NEXT:    pand %xmm11, %xmm2
 ; SSE2-NEXT:    por %xmm4, %xmm2
 ; SSE2-NEXT:    movdqa %xmm2, %xmm4
 ; SSE2-NEXT:    pand %xmm6, %xmm4
 ; SSE2-NEXT:    psrlw $1, %xmm4
-; SSE2-NEXT:    pand %xmm11, %xmm4
-; SSE2-NEXT:    pand %xmm12, %xmm2
+; SSE2-NEXT:    pand %xmm12, %xmm4
+; SSE2-NEXT:    pand %xmm13, %xmm2
 ; SSE2-NEXT:    paddb %xmm2, %xmm2
 ; SSE2-NEXT:    por %xmm4, %xmm2
-; SSE2-NEXT:    movdqa %xmm3, %xmm4
-; SSE2-NEXT:    pand %xmm13, %xmm4
+; SSE2-NEXT:    movdqa %xmm14, %xmm4
+; SSE2-NEXT:    pand %xmm3, %xmm4
 ; SSE2-NEXT:    psllw $4, %xmm4
-; SSE2-NEXT:    pand %xmm7, %xmm4
-; SSE2-NEXT:    pand %xmm7, %xmm3
-; SSE2-NEXT:    psrlw $4, %xmm3
-; SSE2-NEXT:    pand %xmm13, %xmm3
-; SSE2-NEXT:    por %xmm4, %xmm3
+; SSE2-NEXT:    pand %xmm9, %xmm14
+; SSE2-NEXT:    psrlw $4, %xmm14
+; SSE2-NEXT:    pand %xmm3, %xmm14
+; SSE2-NEXT:    pandn %xmm4, %xmm3
+; SSE2-NEXT:    por %xmm14, %xmm3
 ; SSE2-NEXT:    pand %xmm3, %xmm5
 ; SSE2-NEXT:    psllw $2, %xmm5
 ; SSE2-NEXT:    pand %xmm8, %xmm5
-; SSE2-NEXT:    pand %xmm9, %xmm3
-; SSE2-NEXT:    psrlw $2, %xmm3
 ; SSE2-NEXT:    pand %xmm10, %xmm3
+; SSE2-NEXT:    psrlw $2, %xmm3
+; SSE2-NEXT:    pand %xmm11, %xmm3
 ; SSE2-NEXT:    por %xmm5, %xmm3
 ; SSE2-NEXT:    pand %xmm3, %xmm6
 ; SSE2-NEXT:    psrlw $1, %xmm6
-; SSE2-NEXT:    pand %xmm11, %xmm6
-; SSE2-NEXT:    pand %xmm12, %xmm3
+; SSE2-NEXT:    pand %xmm12, %xmm6
+; SSE2-NEXT:    pand %xmm13, %xmm3
 ; SSE2-NEXT:    paddb %xmm3, %xmm3
 ; SSE2-NEXT:    por %xmm6, %xmm3
 ; SSE2-NEXT:    retq
@@ -1481,25 +1490,27 @@ define <64 x i8> @test_bitreverse_v64i8(<64 x i8> %a) nounwind {
 define <32 x i16> @test_bitreverse_v32i16(<32 x i16> %a) nounwind {
 ; SSE2-LABEL: test_bitreverse_v32i16:
 ; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa %xmm3, %xmm15
 ; SSE2-NEXT:    pxor %xmm14, %xmm14
-; SSE2-NEXT:    movdqa %xmm0, %xmm4
-; SSE2-NEXT:    punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm14[8],xmm4[9],xmm14[9],xmm4[10],xmm14[10],xmm4[11],xmm14[11],xmm4[12],xmm14[12],xmm4[13],xmm14[13],xmm4[14],xmm14[14],xmm4[15],xmm14[15]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm4 = xmm4[1,0,3,2,4,5,6,7]
-; SSE2-NEXT:    pshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,5,4,7,6]
+; SSE2-NEXT:    movdqa %xmm0, %xmm3
+; SSE2-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm14[8],xmm3[9],xmm14[9],xmm3[10],xmm14[10],xmm3[11],xmm14[11],xmm3[12],xmm14[12],xmm3[13],xmm14[13],xmm3[14],xmm14[14],xmm3[15],xmm14[15]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm3 = xmm3[1,0,3,2,4,5,6,7]
+; SSE2-NEXT:    pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,5,4,7,6]
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm14[0],xmm0[1],xmm14[1],xmm0[2],xmm14[2],xmm0[3],xmm14[3],xmm0[4],xmm14[4],xmm0[5],xmm14[5],xmm0[6],xmm14[6],xmm0[7],xmm14[7]
 ; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[1,0,3,2,4,5,6,7]
 ; SSE2-NEXT:    pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,5,4,7,6]
-; SSE2-NEXT:    packuswb %xmm4, %xmm0
-; SSE2-NEXT:    movdqa {{.*#+}} xmm8 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; SSE2-NEXT:    packuswb %xmm3, %xmm0
+; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
 ; SSE2-NEXT:    movdqa %xmm0, %xmm5
-; SSE2-NEXT:    pand %xmm8, %xmm5
+; SSE2-NEXT:    pand %xmm3, %xmm5
 ; SSE2-NEXT:    psllw $4, %xmm5
-; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
-; SSE2-NEXT:    pand %xmm4, %xmm5
-; SSE2-NEXT:    pand %xmm4, %xmm0
-; SSE2-NEXT:    psrlw $4, %xmm0
+; SSE2-NEXT:    movdqa %xmm3, %xmm7
+; SSE2-NEXT:    pandn %xmm5, %xmm7
+; SSE2-NEXT:    movdqa {{.*#+}} xmm8 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
 ; SSE2-NEXT:    pand %xmm8, %xmm0
-; SSE2-NEXT:    por %xmm5, %xmm0
+; SSE2-NEXT:    psrlw $4, %xmm0
+; SSE2-NEXT:    pand %xmm3, %xmm0
+; SSE2-NEXT:    por %xmm7, %xmm0
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [51,51,51,51,51,51,51,51,51,51,51,51,51,51,51,51]
 ; SSE2-NEXT:    movdqa %xmm0, %xmm7
 ; SSE2-NEXT:    pand %xmm5, %xmm7
@@ -1531,74 +1542,76 @@ define <32 x i16> @test_bitreverse_v32i16(<32 x i16> %a) nounwind {
 ; SSE2-NEXT:    pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,5,4,7,6]
 ; SSE2-NEXT:    packuswb %xmm6, %xmm1
 ; SSE2-NEXT:    movdqa %xmm1, %xmm6
-; SSE2-NEXT:    pand %xmm8, %xmm6
+; SSE2-NEXT:    pand %xmm3, %xmm6
 ; SSE2-NEXT:    psllw $4, %xmm6
-; SSE2-NEXT:    pand %xmm4, %xmm6
-; SSE2-NEXT:    pand %xmm4, %xmm1
-; SSE2-NEXT:    psrlw $4, %xmm1
+; SSE2-NEXT:    movdqa %xmm3, %xmm4
+; SSE2-NEXT:    pandn %xmm6, %xmm4
 ; SSE2-NEXT:    pand %xmm8, %xmm1
-; SSE2-NEXT:    por %xmm6, %xmm1
-; SSE2-NEXT:    movdqa %xmm1, %xmm6
-; SSE2-NEXT:    pand %xmm5, %xmm6
-; SSE2-NEXT:    psllw $2, %xmm6
-; SSE2-NEXT:    pand %xmm9, %xmm6
+; SSE2-NEXT:    psrlw $4, %xmm1
+; SSE2-NEXT:    pand %xmm3, %xmm1
+; SSE2-NEXT:    por %xmm4, %xmm1
+; SSE2-NEXT:    movdqa %xmm1, %xmm4
+; SSE2-NEXT:    pand %xmm5, %xmm4
+; SSE2-NEXT:    psllw $2, %xmm4
+; SSE2-NEXT:    pand %xmm9, %xmm4
 ; SSE2-NEXT:    pand %xmm10, %xmm1
 ; SSE2-NEXT:    psrlw $2, %xmm1
 ; SSE2-NEXT:    pand %xmm11, %xmm1
-; SSE2-NEXT:    por %xmm6, %xmm1
-; SSE2-NEXT:    movdqa %xmm1, %xmm6
-; SSE2-NEXT:    pand %xmm7, %xmm6
-; SSE2-NEXT:    psrlw $1, %xmm6
-; SSE2-NEXT:    pand %xmm12, %xmm6
+; SSE2-NEXT:    por %xmm4, %xmm1
+; SSE2-NEXT:    movdqa %xmm1, %xmm4
+; SSE2-NEXT:    pand %xmm7, %xmm4
+; SSE2-NEXT:    psrlw $1, %xmm4
+; SSE2-NEXT:    pand %xmm12, %xmm4
 ; SSE2-NEXT:    pand %xmm13, %xmm1
 ; SSE2-NEXT:    paddb %xmm1, %xmm1
-; SSE2-NEXT:    por %xmm6, %xmm1
-; SSE2-NEXT:    movdqa %xmm2, %xmm6
-; SSE2-NEXT:    punpckhbw {{.*#+}} xmm6 = xmm6[8],xmm14[8],xmm6[9],xmm14[9],xmm6[10],xmm14[10],xmm6[11],xmm14[11],xmm6[12],xmm14[12],xmm6[13],xmm14[13],xmm6[14],xmm14[14],xmm6[15],xmm14[15]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm6 = xmm6[1,0,3,2,4,5,6,7]
-; SSE2-NEXT:    pshufhw {{.*#+}} xmm6 = xmm6[0,1,2,3,5,4,7,6]
+; SSE2-NEXT:    por %xmm4, %xmm1
+; SSE2-NEXT:    movdqa %xmm2, %xmm4
+; SSE2-NEXT:    punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm14[8],xmm4[9],xmm14[9],xmm4[10],xmm14[10],xmm4[11],xmm14[11],xmm4[12],xmm14[12],xmm4[13],xmm14[13],xmm4[14],xmm14[14],xmm4[15],xmm14[15]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm4 = xmm4[1,0,3,2,4,5,6,7]
+; SSE2-NEXT:    pshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,5,4,7,6]
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm14[0],xmm2[1],xmm14[1],xmm2[2],xmm14[2],xmm2[3],xmm14[3],xmm2[4],xmm14[4],xmm2[5],xmm14[5],xmm2[6],xmm14[6],xmm2[7],xmm14[7]
 ; SSE2-NEXT:    pshuflw {{.*#+}} xmm2 = xmm2[1,0,3,2,4,5,6,7]
 ; SSE2-NEXT:    pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,5,4,7,6]
-; SSE2-NEXT:    packuswb %xmm6, %xmm2
-; SSE2-NEXT:    movdqa %xmm2, %xmm6
-; SSE2-NEXT:    pand %xmm8, %xmm6
-; SSE2-NEXT:    psllw $4, %xmm6
-; SSE2-NEXT:    pand %xmm4, %xmm6
-; SSE2-NEXT:    pand %xmm4, %xmm2
-; SSE2-NEXT:    psrlw $4, %xmm2
+; SSE2-NEXT:    packuswb %xmm4, %xmm2
+; SSE2-NEXT:    movdqa %xmm2, %xmm4
+; SSE2-NEXT:    pand %xmm3, %xmm4
+; SSE2-NEXT:    psllw $4, %xmm4
+; SSE2-NEXT:    movdqa %xmm3, %xmm6
+; SSE2-NEXT:    pandn %xmm4, %xmm6
 ; SSE2-NEXT:    pand %xmm8, %xmm2
+; SSE2-NEXT:    psrlw $4, %xmm2
+; SSE2-NEXT:    pand %xmm3, %xmm2
 ; SSE2-NEXT:    por %xmm6, %xmm2
-; SSE2-NEXT:    movdqa %xmm2, %xmm6
-; SSE2-NEXT:    pand %xmm5, %xmm6
-; SSE2-NEXT:    psllw $2, %xmm6
-; SSE2-NEXT:    pand %xmm9, %xmm6
+; SSE2-NEXT:    movdqa %xmm2, %xmm4
+; SSE2-NEXT:    pand %xmm5, %xmm4
+; SSE2-NEXT:    psllw $2, %xmm4
+; SSE2-NEXT:    pand %xmm9, %xmm4
 ; SSE2-NEXT:    pand %xmm10, %xmm2
 ; SSE2-NEXT:    psrlw $2, %xmm2
 ; SSE2-NEXT:    pand %xmm11, %xmm2
-; SSE2-NEXT:    por %xmm6, %xmm2
-; SSE2-NEXT:    movdqa %xmm2, %xmm6
-; SSE2-NEXT:    pand %xmm7, %xmm6
-; SSE2-NEXT:    psrlw $1, %xmm6
-; SSE2-NEXT:    pand %xmm12, %xmm6
+; SSE2-NEXT:    por %xmm4, %xmm2
+; SSE2-NEXT:    movdqa %xmm2, %xmm4
+; SSE2-NEXT:    pand %xmm7, %xmm4
+; SSE2-NEXT:    psrlw $1, %xmm4
+; SSE2-NEXT:    pand %xmm12, %xmm4
 ; SSE2-NEXT:    pand %xmm13, %xmm2
 ; SSE2-NEXT:    paddb %xmm2, %xmm2
-; SSE2-NEXT:    por %xmm6, %xmm2
-; SSE2-NEXT:    movdqa %xmm3, %xmm6
-; SSE2-NEXT:    punpckhbw {{.*#+}} xmm6 = xmm6[8],xmm14[8],xmm6[9],xmm14[9],xmm6[10],xmm14[10],xmm6[11],xmm14[11],xmm6[12],xmm14[12],xmm6[13],xmm14[13],xmm6[14],xmm14[14],xmm6[15],xmm14[15]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm6 = xmm6[1,0,3,2,4,5,6,7]
+; SSE2-NEXT:    por %xmm4, %xmm2
+; SSE2-NEXT:    movdqa %xmm15, %xmm4
+; SSE2-NEXT:    punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm14[8],xmm4[9],xmm14[9],xmm4[10],xmm14[10],xmm4[11],xmm14[11],xmm4[12],xmm14[12],xmm4[13],xmm14[13],xmm4[14],xmm14[14],xmm4[15],xmm14[15]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm4 = xmm4[1,0,3,2,4,5,6,7]
+; SSE2-NEXT:    pshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,5,4,7,6]
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm15 = xmm15[0],xmm14[0],xmm15[1],xmm14[1],xmm15[2],xmm14[2],xmm15[3],xmm14[3],xmm15[4],xmm14[4],xmm15[5],xmm14[5],xmm15[6],xmm14[6],xmm15[7],xmm14[7]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm6 = xmm15[1,0,3,2,4,5,6,7]
 ; SSE2-NEXT:    pshufhw {{.*#+}} xmm6 = xmm6[0,1,2,3,5,4,7,6]
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm14[0],xmm3[1],xmm14[1],xmm3[2],xmm14[2],xmm3[3],xmm14[3],xmm3[4],xmm14[4],xmm3[5],xmm14[5],xmm3[6],xmm14[6],xmm3[7],xmm14[7]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm3 = xmm3[1,0,3,2,4,5,6,7]
-; SSE2-NEXT:    pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,5,4,7,6]
-; SSE2-NEXT:    packuswb %xmm6, %xmm3
-; SSE2-NEXT:    movdqa %xmm3, %xmm6
+; SSE2-NEXT:    packuswb %xmm4, %xmm6
+; SSE2-NEXT:    movdqa %xmm6, %xmm4
+; SSE2-NEXT:    pand %xmm3, %xmm4
+; SSE2-NEXT:    psllw $4, %xmm4
 ; SSE2-NEXT:    pand %xmm8, %xmm6
-; SSE2-NEXT:    psllw $4, %xmm6
-; SSE2-NEXT:    pand %xmm4, %xmm6
-; SSE2-NEXT:    pand %xmm4, %xmm3
-; SSE2-NEXT:    psrlw $4, %xmm3
-; SSE2-NEXT:    pand %xmm8, %xmm3
+; SSE2-NEXT:    psrlw $4, %xmm6
+; SSE2-NEXT:    pand %xmm3, %xmm6
+; SSE2-NEXT:    pandn %xmm4, %xmm3
 ; SSE2-NEXT:    por %xmm6, %xmm3
 ; SSE2-NEXT:    pand %xmm3, %xmm5
 ; SSE2-NEXT:    psllw $2, %xmm5
@@ -1795,25 +1808,27 @@ define <32 x i16> @test_bitreverse_v32i16(<32 x i16> %a) nounwind {
 define <16 x i32> @test_bitreverse_v16i32(<16 x i32> %a) nounwind {
 ; SSE2-LABEL: test_bitreverse_v16i32:
 ; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa %xmm3, %xmm15
 ; SSE2-NEXT:    pxor %xmm14, %xmm14
-; SSE2-NEXT:    movdqa %xmm0, %xmm4
-; SSE2-NEXT:    punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm14[8],xmm4[9],xmm14[9],xmm4[10],xmm14[10],xmm4[11],xmm14[11],xmm4[12],xmm14[12],xmm4[13],xmm14[13],xmm4[14],xmm14[14],xmm4[15],xmm14[15]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm4 = xmm4[3,2,1,0,4,5,6,7]
-; SSE2-NEXT:    pshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,7,6,5,4]
+; SSE2-NEXT:    movdqa %xmm0, %xmm3
+; SSE2-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm14[8],xmm3[9],xmm14[9],xmm3[10],xmm14[10],xmm3[11],xmm14[11],xmm3[12],xmm14[12],xmm3[13],xmm14[13],xmm3[14],xmm14[14],xmm3[15],xmm14[15]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm3 = xmm3[3,2,1,0,4,5,6,7]
+; SSE2-NEXT:    pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,7,6,5,4]
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm14[0],xmm0[1],xmm14[1],xmm0[2],xmm14[2],xmm0[3],xmm14[3],xmm0[4],xmm14[4],xmm0[5],xmm14[5],xmm0[6],xmm14[6],xmm0[7],xmm14[7]
 ; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[3,2,1,0,4,5,6,7]
 ; SSE2-NEXT:    pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,7,6,5,4]
-; SSE2-NEXT:    packuswb %xmm4, %xmm0
-; SSE2-NEXT:    movdqa {{.*#+}} xmm8 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; SSE2-NEXT:    packuswb %xmm3, %xmm0
+; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
 ; SSE2-NEXT:    movdqa %xmm0, %xmm5
-; SSE2-NEXT:    pand %xmm8, %xmm5
+; SSE2-NEXT:    pand %xmm3, %xmm5
 ; SSE2-NEXT:    psllw $4, %xmm5
-; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
-; SSE2-NEXT:    pand %xmm4, %xmm5
-; SSE2-NEXT:    pand %xmm4, %xmm0
-; SSE2-NEXT:    psrlw $4, %xmm0
+; SSE2-NEXT:    movdqa %xmm3, %xmm7
+; SSE2-NEXT:    pandn %xmm5, %xmm7
+; SSE2-NEXT:    movdqa {{.*#+}} xmm8 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
 ; SSE2-NEXT:    pand %xmm8, %xmm0
-; SSE2-NEXT:    por %xmm5, %xmm0
+; SSE2-NEXT:    psrlw $4, %xmm0
+; SSE2-NEXT:    pand %xmm3, %xmm0
+; SSE2-NEXT:    por %xmm7, %xmm0
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [51,51,51,51,51,51,51,51,51,51,51,51,51,51,51,51]
 ; SSE2-NEXT:    movdqa %xmm0, %xmm7
 ; SSE2-NEXT:    pand %xmm5, %xmm7
@@ -1845,74 +1860,76 @@ define <16 x i32> @test_bitreverse_v16i32(<16 x i32> %a) nounwind {
 ; SSE2-NEXT:    pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,7,6,5,4]
 ; SSE2-NEXT:    packuswb %xmm6, %xmm1
 ; SSE2-NEXT:    movdqa %xmm1, %xmm6
-; SSE2-NEXT:    pand %xmm8, %xmm6
+; SSE2-NEXT:    pand %xmm3, %xmm6
 ; SSE2-NEXT:    psllw $4, %xmm6
-; SSE2-NEXT:    pand %xmm4, %xmm6
-; SSE2-NEXT:    pand %xmm4, %xmm1
-; SSE2-NEXT:    psrlw $4, %xmm1
+; SSE2-NEXT:    movdqa %xmm3, %xmm4
+; SSE2-NEXT:    pandn %xmm6, %xmm4
 ; SSE2-NEXT:    pand %xmm8, %xmm1
-; SSE2-NEXT:    por %xmm6, %xmm1
-; SSE2-NEXT:    movdqa %xmm1, %xmm6
-; SSE2-NEXT:    pand %xmm5, %xmm6
-; SSE2-NEXT:    psllw $2, %xmm6
-; SSE2-NEXT:    pand %xmm9, %xmm6
+; SSE2-NEXT:    psrlw $4, %xmm1
+; SSE2-NEXT:    pand %xmm3, %xmm1
+; SSE2-NEXT:    por %xmm4, %xmm1
+; SSE2-NEXT:    movdqa %xmm1, %xmm4
+; SSE2-NEXT:    pand %xmm5, %xmm4
+; SSE2-NEXT:    psllw $2, %xmm4
+; SSE2-NEXT:    pand %xmm9, %xmm4
 ; SSE2-NEXT:    pand %xmm10, %xmm1
 ; SSE2-NEXT:    psrlw $2, %xmm1
 ; SSE2-NEXT:    pand %xmm11, %xmm1
-; SSE2-NEXT:    por %xmm6, %xmm1
-; SSE2-NEXT:    movdqa %xmm1, %xmm6
-; SSE2-NEXT:    pand %xmm7, %xmm6
-; SSE2-NEXT:    psrlw $1, %xmm6
-; SSE2-NEXT:    pand %xmm12, %xmm6
+; SSE2-NEXT:    por %xmm4, %xmm1
+; SSE2-NEXT:    movdqa %xmm1, %xmm4
+; SSE2-NEXT:    pand %xmm7, %xmm4
+; SSE2-NEXT:    psrlw $1, %xmm4
+; SSE2-NEXT:    pand %xmm12, %xmm4
 ; SSE2-NEXT:    pand %xmm13, %xmm1
 ; SSE2-NEXT:    paddb %xmm1, %xmm1
-; SSE2-NEXT:    por %xmm6, %xmm1
-; SSE2-NEXT:    movdqa %xmm2, %xmm6
-; SSE2-NEXT:    punpckhbw {{.*#+}} xmm6 = xmm6[8],xmm14[8],xmm6[9],xmm14[9],xmm6[10],xmm14[10],xmm6[11],xmm14[11],xmm6[12],xmm14[12],xmm6[13],xmm14[13],xmm6[14],xmm14[14],xmm6[15],xmm14[15]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm6 = xmm6[3,2,1,0,4,5,6,7]
-; SSE2-NEXT:    pshufhw {{.*#+}} xmm6 = xmm6[0,1,2,3,7,6,5,4]
+; SSE2-NEXT:    por %xmm4, %xmm1
+; SSE2-NEXT:    movdqa %xmm2, %xmm4
+; SSE2-NEXT:    punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm14[8],xmm4[9],xmm14[9],xmm4[10],xmm14[10],xmm4[11],xmm14[11],xmm4[12],xmm14[12],xmm4[13],xmm14[13],xmm4[14],xmm14[14],xmm4[15],xmm14[15]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm4 = xmm4[3,2,1,0,4,5,6,7]
+; SSE2-NEXT:    pshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,7,6,5,4]
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm14[0],xmm2[1],xmm14[1],xmm2[2],xmm14[2],xmm2[3],xmm14[3],xmm2[4],xmm14[4],xmm2[5],xmm14[5],xmm2[6],xmm14[6],xmm2[7],xmm14[7]
 ; SSE2-NEXT:    pshuflw {{.*#+}} xmm2 = xmm2[3,2,1,0,4,5,6,7]
 ; SSE2-NEXT:    pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,7,6,5,4]
-; SSE2-NEXT:    packuswb %xmm6, %xmm2
-; SSE2-NEXT:    movdqa %xmm2, %xmm6
-; SSE2-NEXT:    pand %xmm8, %xmm6
-; SSE2-NEXT:    psllw $4, %xmm6
-; SSE2-NEXT:    pand %xmm4, %xmm6
-; SSE2-NEXT:    pand %xmm4, %xmm2
-; SSE2-NEXT:    psrlw $4, %xmm2
+; SSE2-NEXT:    packuswb %xmm4, %xmm2
+; SSE2-NEXT:    movdqa %xmm2, %xmm4
+; SSE2-NEXT:    pand %xmm3, %xmm4
+; SSE2-NEXT:    psllw $4, %xmm4
+; SSE2-NEXT:    movdqa %xmm3, %xmm6
+; SSE2-NEXT:    pandn %xmm4, %xmm6
 ; SSE2-NEXT:    pand %xmm8, %xmm2
+; SSE2-NEXT:    psrlw $4, %xmm2
+; SSE2-NEXT:    pand %xmm3, %xmm2
 ; SSE2-NEXT:    por %xmm6, %xmm2
-; SSE2-NEXT:    movdqa %xmm2, %xmm6
-; SSE2-NEXT:    pand %xmm5, %xmm6
-; SSE2-NEXT:    psllw $2, %xmm6
-; SSE2-NEXT:    pand %xmm9, %xmm6
+; SSE2-NEXT:    movdqa %xmm2, %xmm4
+; SSE2-NEXT:    pand %xmm5, %xmm4
+; SSE2-NEXT:    psllw $2, %xmm4
+; SSE2-NEXT:    pand %xmm9, %xmm4
 ; SSE2-NEXT:    pand %xmm10, %xmm2
 ; SSE2-NEXT:    psrlw $2, %xmm2
 ; SSE2-NEXT:    pand %xmm11, %xmm2
-; SSE2-NEXT:    por %xmm6, %xmm2
-; SSE2-NEXT:    movdqa %xmm2, %xmm6
-; SSE2-NEXT:    pand %xmm7, %xmm6
-; SSE2-NEXT:    psrlw $1, %xmm6
-; SSE2-NEXT:    pand %xmm12, %xmm6
+; SSE2-NEXT:    por %xmm4, %xmm2
+; SSE2-NEXT:    movdqa %xmm2, %xmm4
+; SSE2-NEXT:    pand %xmm7, %xmm4
+; SSE2-NEXT:    psrlw $1, %xmm4
+; SSE2-NEXT:    pand %xmm12, %xmm4
 ; SSE2-NEXT:    pand %xmm13, %xmm2
 ; SSE2-NEXT:    paddb %xmm2, %xmm2
-; SSE2-NEXT:    por %xmm6, %xmm2
-; SSE2-NEXT:    movdqa %xmm3, %xmm6
-; SSE2-NEXT:    punpckhbw {{.*#+}} xmm6 = xmm6[8],xmm14[8],xmm6[9],xmm14[9],xmm6[10],xmm14[10],xmm6[11],xmm14[11],xmm6[12],xmm14[12],xmm6[13],xmm14[13],xmm6[14],xmm14[14],xmm6[15],xmm14[15]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm6 = xmm6[3,2,1,0,4,5,6,7]
+; SSE2-NEXT:    por %xmm4, %xmm2
+; SSE2-NEXT:    movdqa %xmm15, %xmm4
+; SSE2-NEXT:    punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm14[8],xmm4[9],xmm14[9],xmm4[10],xmm14[10],xmm4[11],xmm14[11],xmm4[12],xmm14[12],xmm4[13],xmm14[13],xmm4[14],xmm14[14],xmm4[15],xmm14[15]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm4 = xmm4[3,2,1,0,4,5,6,7]
+; SSE2-NEXT:    pshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,7,6,5,4]
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm15 = xmm15[0],xmm14[0],xmm15[1],xmm14[1],xmm15[2],xmm14[2],xmm15[3],xmm14[3],xmm15[4],xmm14[4],xmm15[5],xmm14[5],xmm15[6],xmm14[6],xmm15[7],xmm14[7]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm6 = xmm15[3,2,1,0,4,5,6,7]
 ; SSE2-NEXT:    pshufhw {{.*#+}} xmm6 = xmm6[0,1,2,3,7,6,5,4]
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm14[0],xmm3[1],xmm14[1],xmm3[2],xmm14[2],xmm3[3],xmm14[3],xmm3[4],xmm14[4],xmm3[5],xmm14[5],xmm3[6],xmm14[6],xmm3[7],xmm14[7]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm3 = xmm3[3,2,1,0,4,5,6,7]
-; SSE2-NEXT:    pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,7,6,5,4]
-; SSE2-NEXT:    packuswb %xmm6, %xmm3
-; SSE2-NEXT:    movdqa %xmm3, %xmm6
+; SSE2-NEXT:    packuswb %xmm4, %xmm6
+; SSE2-NEXT:    movdqa %xmm6, %xmm4
+; SSE2-NEXT:    pand %xmm3, %xmm4
+; SSE2-NEXT:    psllw $4, %xmm4
 ; SSE2-NEXT:    pand %xmm8, %xmm6
-; SSE2-NEXT:    psllw $4, %xmm6
-; SSE2-NEXT:    pand %xmm4, %xmm6
-; SSE2-NEXT:    pand %xmm4, %xmm3
-; SSE2-NEXT:    psrlw $4, %xmm3
-; SSE2-NEXT:    pand %xmm8, %xmm3
+; SSE2-NEXT:    psrlw $4, %xmm6
+; SSE2-NEXT:    pand %xmm3, %xmm6
+; SSE2-NEXT:    pandn %xmm4, %xmm3
 ; SSE2-NEXT:    por %xmm6, %xmm3
 ; SSE2-NEXT:    pand %xmm3, %xmm5
 ; SSE2-NEXT:    psllw $2, %xmm5
@@ -2115,27 +2132,29 @@ define <16 x i32> @test_bitreverse_v16i32(<16 x i32> %a) nounwind {
 define <8 x i64> @test_bitreverse_v8i64(<8 x i64> %a) nounwind {
 ; SSE2-LABEL: test_bitreverse_v8i64:
 ; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa %xmm3, %xmm15
 ; SSE2-NEXT:    pxor %xmm14, %xmm14
-; SSE2-NEXT:    movdqa %xmm0, %xmm4
-; SSE2-NEXT:    punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm14[8],xmm4[9],xmm14[9],xmm4[10],xmm14[10],xmm4[11],xmm14[11],xmm4[12],xmm14[12],xmm4[13],xmm14[13],xmm4[14],xmm14[14],xmm4[15],xmm14[15]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[2,3,0,1]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm4 = xmm4[3,2,1,0,4,5,6,7]
-; SSE2-NEXT:    pshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,7,6,5,4]
+; SSE2-NEXT:    movdqa %xmm0, %xmm3
+; SSE2-NEXT:    punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm14[8],xmm3[9],xmm14[9],xmm3[10],xmm14[10],xmm3[11],xmm14[11],xmm3[12],xmm14[12],xmm3[13],xmm14[13],xmm3[14],xmm14[14],xmm3[15],xmm14[15]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[2,3,0,1]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm3 = xmm3[3,2,1,0,4,5,6,7]
+; SSE2-NEXT:    pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,7,6,5,4]
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm14[0],xmm0[1],xmm14[1],xmm0[2],xmm14[2],xmm0[3],xmm14[3],xmm0[4],xmm14[4],xmm0[5],xmm14[5],xmm0[6],xmm14[6],xmm0[7],xmm14[7]
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
 ; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[3,2,1,0,4,5,6,7]
 ; SSE2-NEXT:    pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,7,6,5,4]
-; SSE2-NEXT:    packuswb %xmm4, %xmm0
-; SSE2-NEXT:    movdqa {{.*#+}} xmm8 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; SSE2-NEXT:    packuswb %xmm3, %xmm0
+; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
 ; SSE2-NEXT:    movdqa %xmm0, %xmm5
-; SSE2-NEXT:    pand %xmm8, %xmm5
+; SSE2-NEXT:    pand %xmm3, %xmm5
 ; SSE2-NEXT:    psllw $4, %xmm5
-; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
-; SSE2-NEXT:    pand %xmm4, %xmm5
-; SSE2-NEXT:    pand %xmm4, %xmm0
-; SSE2-NEXT:    psrlw $4, %xmm0
+; SSE2-NEXT:    movdqa %xmm3, %xmm7
+; SSE2-NEXT:    pandn %xmm5, %xmm7
+; SSE2-NEXT:    movdqa {{.*#+}} xmm8 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
 ; SSE2-NEXT:    pand %xmm8, %xmm0
-; SSE2-NEXT:    por %xmm5, %xmm0
+; SSE2-NEXT:    psrlw $4, %xmm0
+; SSE2-NEXT:    pand %xmm3, %xmm0
+; SSE2-NEXT:    por %xmm7, %xmm0
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [51,51,51,51,51,51,51,51,51,51,51,51,51,51,51,51]
 ; SSE2-NEXT:    movdqa %xmm0, %xmm7
 ; SSE2-NEXT:    pand %xmm5, %xmm7
@@ -2169,78 +2188,80 @@ define <8 x i64> @test_bitreverse_v8i64(<8 x i64> %a) nounwind {
 ; SSE2-NEXT:    pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,7,6,5,4]
 ; SSE2-NEXT:    packuswb %xmm6, %xmm1
 ; SSE2-NEXT:    movdqa %xmm1, %xmm6
-; SSE2-NEXT:    pand %xmm8, %xmm6
+; SSE2-NEXT:    pand %xmm3, %xmm6
 ; SSE2-NEXT:    psllw $4, %xmm6
-; SSE2-NEXT:    pand %xmm4, %xmm6
-; SSE2-NEXT:    pand %xmm4, %xmm1
-; SSE2-NEXT:    psrlw $4, %xmm1
+; SSE2-NEXT:    movdqa %xmm3, %xmm4
+; SSE2-NEXT:    pandn %xmm6, %xmm4
 ; SSE2-NEXT:    pand %xmm8, %xmm1
-; SSE2-NEXT:    por %xmm6, %xmm1
-; SSE2-NEXT:    movdqa %xmm1, %xmm6
-; SSE2-NEXT:    pand %xmm5, %xmm6
-; SSE2-NEXT:    psllw $2, %xmm6
-; SSE2-NEXT:    pand %xmm9, %xmm6
+; SSE2-NEXT:    psrlw $4, %xmm1
+; SSE2-NEXT:    pand %xmm3, %xmm1
+; SSE2-NEXT:    por %xmm4, %xmm1
+; SSE2-NEXT:    movdqa %xmm1, %xmm4
+; SSE2-NEXT:    pand %xmm5, %xmm4
+; SSE2-NEXT:    psllw $2, %xmm4
+; SSE2-NEXT:    pand %xmm9, %xmm4
 ; SSE2-NEXT:    pand %xmm10, %xmm1
 ; SSE2-NEXT:    psrlw $2, %xmm1
 ; SSE2-NEXT:    pand %xmm11, %xmm1
-; SSE2-NEXT:    por %xmm6, %xmm1
-; SSE2-NEXT:    movdqa %xmm1, %xmm6
-; SSE2-NEXT:    pand %xmm7, %xmm6
-; SSE2-NEXT:    psrlw $1, %xmm6
-; SSE2-NEXT:    pand %xmm12, %xmm6
+; SSE2-NEXT:    por %xmm4, %xmm1
+; SSE2-NEXT:    movdqa %xmm1, %xmm4
+; SSE2-NEXT:    pand %xmm7, %xmm4
+; SSE2-NEXT:    psrlw $1, %xmm4
+; SSE2-NEXT:    pand %xmm12, %xmm4
 ; SSE2-NEXT:    pand %xmm13, %xmm1
 ; SSE2-NEXT:    paddb %xmm1, %xmm1
-; SSE2-NEXT:    por %xmm6, %xmm1
-; SSE2-NEXT:    movdqa %xmm2, %xmm6
-; SSE2-NEXT:    punpckhbw {{.*#+}} xmm6 = xmm6[8],xmm14[8],xmm6[9],xmm14[9],xmm6[10],xmm14[10],xmm6[11],xmm14[11],xmm6[12],xmm14[12],xmm6[13],xmm14[13],xmm6[14],xmm14[14],xmm6[15],xmm14[15]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[2,3,0,1]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm6 = xmm6[3,2,1,0,4,5,6,7]
-; SSE2-NEXT:    pshufhw {{.*#+}} xmm6 = xmm6[0,1,2,3,7,6,5,4]
+; SSE2-NEXT:    por %xmm4, %xmm1
+; SSE2-NEXT:    movdqa %xmm2, %xmm4
+; SSE2-NEXT:    punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm14[8],xmm4[9],xmm14[9],xmm4[10],xmm14[10],xmm4[11],xmm14[11],xmm4[12],xmm14[12],xmm4[13],xmm14[13],xmm4[14],xmm14[14],xmm4[15],xmm14[15]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[2,3,0,1]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm4 = xmm4[3,2,1,0,4,5,6,7]
+; SSE2-NEXT:    pshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,7,6,5,4]
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm14[0],xmm2[1],xmm14[1],xmm2[2],xmm14[2],xmm2[3],xmm14[3],xmm2[4],xmm14[4],xmm2[5],xmm14[5],xmm2[6],xmm14[6],xmm2[7],xmm14[7]
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[2,3,0,1]
 ; SSE2-NEXT:    pshuflw {{.*#+}} xmm2 = xmm2[3,2,1,0,4,5,6,7]
 ; SSE2-NEXT:    pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,7,6,5,4]
-; SSE2-NEXT:    packuswb %xmm6, %xmm2
-; SSE2-NEXT:    movdqa %xmm2, %xmm6
-; SSE2-NEXT:    pand %xmm8, %xmm6
-; SSE2-NEXT:    psllw $4, %xmm6
-; SSE2-NEXT:    pand %xmm4, %xmm6
-; SSE2-NEXT:    pand %xmm4, %xmm2
-; SSE2-NEXT:    psrlw $4, %xmm2
+; SSE2-NEXT:    packuswb %xmm4, %xmm2
+; SSE2-NEXT:    movdqa %xmm2, %xmm4
+; SSE2-NEXT:    pand %xmm3, %xmm4
+; SSE2-NEXT:    psllw $4, %xmm4
+; SSE2-NEXT:    movdqa %xmm3, %xmm6
+; SSE2-NEXT:    pandn %xmm4, %xmm6
 ; SSE2-NEXT:    pand %xmm8, %xmm2
+; SSE2-NEXT:    psrlw $4, %xmm2
+; SSE2-NEXT:    pand %xmm3, %xmm2
 ; SSE2-NEXT:    por %xmm6, %xmm2
-; SSE2-NEXT:    movdqa %xmm2, %xmm6
-; SSE2-NEXT:    pand %xmm5, %xmm6
-; SSE2-NEXT:    psllw $2, %xmm6
-; SSE2-NEXT:    pand %xmm9, %xmm6
+; SSE2-NEXT:    movdqa %xmm2, %xmm4
+; SSE2-NEXT:    pand %xmm5, %xmm4
+; SSE2-NEXT:    psllw $2, %xmm4
+; SSE2-NEXT:    pand %xmm9, %xmm4
 ; SSE2-NEXT:    pand %xmm10, %xmm2
 ; SSE2-NEXT:    psrlw $2, %xmm2
 ; SSE2-NEXT:    pand %xmm11, %xmm2
-; SSE2-NEXT:    por %xmm6, %xmm2
-; SSE2-NEXT:    movdqa %xmm2, %xmm6
-; SSE2-NEXT:    pand %xmm7, %xmm6
-; SSE2-NEXT:    psrlw $1, %xmm6
-; SSE2-NEXT:    pand %xmm12, %xmm6
+; SSE2-NEXT:    por %xmm4, %xmm2
+; SSE2-NEXT:    movdqa %xmm2, %xmm4
+; SSE2-NEXT:    pand %xmm7, %xmm4
+; SSE2-NEXT:    psrlw $1, %xmm4
+; SSE2-NEXT:    pand %xmm12, %xmm4
 ; SSE2-NEXT:    pand %xmm13, %xmm2
 ; SSE2-NEXT:    paddb %xmm2, %xmm2
-; SSE2-NEXT:    por %xmm6, %xmm2
-; SSE2-NEXT:    movdqa %xmm3, %xmm6
-; SSE2-NEXT:    punpckhbw {{.*#+}} xmm6 = xmm6[8],xmm14[8],xmm6[9],xmm14[9],xmm6[10],xmm14[10],xmm6[11],xmm14[11],xmm6[12],xmm14[12],xmm6[13],xmm14[13],xmm6[14],xmm14[14],xmm6[15],xmm14[15]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[2,3,0,1]
+; SSE2-NEXT:    por %xmm4, %xmm2
+; SSE2-NEXT:    movdqa %xmm15, %xmm4
+; SSE2-NEXT:    punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm14[8],xmm4[9],xmm14[9],xmm4[10],xmm14[10],xmm4[11],xmm14[11],xmm4[12],xmm14[12],xmm4[13],xmm14[13],xmm4[14],xmm14[14],xmm4[15],xmm14[15]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[2,3,0,1]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm4 = xmm4[3,2,1,0,4,5,6,7]
+; SSE2-NEXT:    pshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,7,6,5,4]
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm15 = xmm15[0],xmm14[0],xmm15[1],xmm14[1],xmm15[2],xmm14[2],xmm15[3],xmm14[3],xmm15[4],xmm14[4],xmm15[5],xmm14[5],xmm15[6],xmm14[6],xmm15[7],xmm14[7]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm15[2,3,0,1]
 ; SSE2-NEXT:    pshuflw {{.*#+}} xmm6 = xmm6[3,2,1,0,4,5,6,7]
 ; SSE2-NEXT:    pshufhw {{.*#+}} xmm6 = xmm6[0,1,2,3,7,6,5,4]
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm14[0],xmm3[1],xmm14[1],xmm3[2],xmm14[2],xmm3[3],xmm14[3],xmm3[4],xmm14[4],xmm3[5],xmm14[5],xmm3[6],xmm14[6],xmm3[7],xmm14[7]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[2,3,0,1]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm3 = xmm3[3,2,1,0,4,5,6,7]
-; SSE2-NEXT:    pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,7,6,5,4]
-; SSE2-NEXT:    packuswb %xmm6, %xmm3
-; SSE2-NEXT:    movdqa %xmm3, %xmm6
+; SSE2-NEXT:    packuswb %xmm4, %xmm6
+; SSE2-NEXT:    movdqa %xmm6, %xmm4
+; SSE2-NEXT:    pand %xmm3, %xmm4
+; SSE2-NEXT:    psllw $4, %xmm4
 ; SSE2-NEXT:    pand %xmm8, %xmm6
-; SSE2-NEXT:    psllw $4, %xmm6
-; SSE2-NEXT:    pand %xmm4, %xmm6
-; SSE2-NEXT:    pand %xmm4, %xmm3
-; SSE2-NEXT:    psrlw $4, %xmm3
-; SSE2-NEXT:    pand %xmm8, %xmm3
+; SSE2-NEXT:    psrlw $4, %xmm6
+; SSE2-NEXT:    pand %xmm3, %xmm6
+; SSE2-NEXT:    pandn %xmm4, %xmm3
 ; SSE2-NEXT:    por %xmm6, %xmm3
 ; SSE2-NEXT:    pand %xmm3, %xmm5
 ; SSE2-NEXT:    psllw $2, %xmm5
diff --git a/test/CodeGen/X86/vector-compare-all_of.ll b/test/CodeGen/X86/vector-compare-all_of.ll
index fe74d07512b023f0e8ba665fca8855242b650b88..9e6b22340c4f10748c9dee21916c799402befe2a 100644
--- a/test/CodeGen/X86/vector-compare-all_of.ll
+++ b/test/CodeGen/X86/vector-compare-all_of.ll
@@ -62,7 +62,7 @@ define i64 @test_v4f64_sext(<4 x double> %a0, <4 x double> %a1) {
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vcmpltpd %ymm0, %ymm1, %ymm0
 ; AVX512-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vandpd %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vandpd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; AVX512-NEXT:    vandpd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovq %xmm0, %rax
@@ -196,9 +196,9 @@ define i32 @test_v8f32_sext(<8 x float> %a0, <8 x float> %a1) {
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vcmpltps %ymm0, %ymm1, %ymm0
 ; AVX512-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vandps %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vandps %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vandps %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vandps %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512-NEXT:    vandps %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
@@ -342,7 +342,7 @@ define i64 @test_v4i64_sext(<4 x i64> %a0, <4 x i64> %a1) {
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpcmpgtq %ymm1, %ymm0, %ymm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vpand %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovq %xmm0, %rax
@@ -506,9 +506,9 @@ define i32 @test_v8i32_sext(<8 x i32> %a0, <8 x i32> %a1) {
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpcmpgtd %ymm1, %ymm0, %ymm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vpand %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpand %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
@@ -661,12 +661,11 @@ define i16 @test_v16i16_sext(<16 x i16> %a0, <16 x i16> %a1) {
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
 ; AVX1-NEXT:    vpcmpgtw %xmm2, %xmm3, %xmm2
 ; AVX1-NEXT:    vpcmpgtw %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
-; AVX1-NEXT:    vandps %ymm2, %ymm0, %ymm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX1-NEXT:    vandps %ymm1, %ymm0, %ymm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX1-NEXT:    vandps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vpand %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovd %xmm0, %eax
@@ -689,11 +688,11 @@ define i16 @test_v16i16_sext(<16 x i16> %a0, <16 x i16> %a1) {
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpcmpgtw %ymm1, %ymm0, %ymm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vpand %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpand %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX512-NEXT:    vpand %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
@@ -861,14 +860,13 @@ define i8 @test_v32i8_sext(<32 x i8> %a0, <32 x i8> %a1) {
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
 ; AVX1-NEXT:    vpcmpgtb %xmm2, %xmm3, %xmm2
 ; AVX1-NEXT:    vpcmpgtb %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
-; AVX1-NEXT:    vandps %ymm2, %ymm0, %ymm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX1-NEXT:    vandps %ymm1, %ymm0, %ymm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX1-NEXT:    vandps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vpand %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX1-NEXT:    vandps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm1
 ; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpextrb $0, %xmm0, %eax
@@ -891,13 +889,13 @@ define i8 @test_v32i8_sext(<32 x i8> %a0, <32 x i8> %a1) {
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpcmpgtb %ymm1, %ymm0, %ymm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vpand %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpand %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX512-NEXT:    vpand %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX512-NEXT:    vpand %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrlw $8, %xmm0, %xmm1
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpextrb $0, %xmm0, %eax
diff --git a/test/CodeGen/X86/vector-compare-any_of.ll b/test/CodeGen/X86/vector-compare-any_of.ll
index 4b2f98b369841fc4dbc2179b7b664779cb7e6045..1eb6e9edf1365f1d894e21732f14a3e17cd67ec5 100644
--- a/test/CodeGen/X86/vector-compare-any_of.ll
+++ b/test/CodeGen/X86/vector-compare-any_of.ll
@@ -61,7 +61,7 @@ define i64 @test_v4f64_sext(<4 x double> %a0, <4 x double> %a1) {
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vcmpltpd %ymm0, %ymm1, %ymm0
 ; AVX512-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vorpd %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vorpd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; AVX512-NEXT:    vorpd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovq %xmm0, %rax
@@ -191,9 +191,9 @@ define i32 @test_v8f32_sext(<8 x float> %a0, <8 x float> %a1) {
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vcmpltps %ymm0, %ymm1, %ymm0
 ; AVX512-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vorps %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vorps %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vorps %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vorps %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512-NEXT:    vorps %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
@@ -333,7 +333,7 @@ define i64 @test_v4i64_sext(<4 x i64> %a0, <4 x i64> %a1) {
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpcmpgtq %ymm1, %ymm0, %ymm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovq %xmm0, %rax
@@ -492,9 +492,9 @@ define i32 @test_v8i32_sext(<8 x i32> %a0, <8 x i32> %a1) {
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpcmpgtd %ymm1, %ymm0, %ymm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
@@ -641,12 +641,11 @@ define i16 @test_v16i16_sext(<16 x i16> %a0, <16 x i16> %a1) {
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
 ; AVX1-NEXT:    vpcmpgtw %xmm2, %xmm3, %xmm2
 ; AVX1-NEXT:    vpcmpgtw %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
-; AVX1-NEXT:    vorps %ymm2, %ymm0, %ymm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vpor %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX1-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovd %xmm0, %eax
@@ -669,11 +668,11 @@ define i16 @test_v16i16_sext(<16 x i16> %a0, <16 x i16> %a1) {
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpcmpgtw %ymm1, %ymm0, %ymm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX512-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
@@ -835,14 +834,13 @@ define i8 @test_v32i8_sext(<32 x i8> %a0, <32 x i8> %a1) {
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
 ; AVX1-NEXT:    vpcmpgtb %xmm2, %xmm3, %xmm2
 ; AVX1-NEXT:    vpcmpgtb %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
-; AVX1-NEXT:    vorps %ymm2, %ymm0, %ymm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vpor %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm1
 ; AVX1-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpextrb $0, %xmm0, %eax
@@ -865,13 +863,13 @@ define i8 @test_v32i8_sext(<32 x i8> %a0, <32 x i8> %a1) {
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpcmpgtb %ymm1, %ymm0, %ymm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX512-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX512-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrlw $8, %xmm0, %xmm1
 ; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpextrb $0, %xmm0, %eax
diff --git a/test/CodeGen/X86/vector-ext-logic.ll b/test/CodeGen/X86/vector-ext-logic.ll
new file mode 100644
index 0000000000000000000000000000000000000000..01c6c1a2fd528c654a64f5bafa1f3e4698fb849b
--- /dev/null
+++ b/test/CodeGen/X86/vector-ext-logic.ll
@@ -0,0 +1,468 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-- -mattr=+sse2 | FileCheck %s --check-prefixes=ALL,SSE2
+; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx2 | FileCheck %s --check-prefixes=ALL,AVX2
+
+define <8 x i32> @zext_and_v8i32(<8 x i16> %x, <8 x i16> %y) {
+; SSE2-LABEL: zext_and_v8i32:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa %xmm0, %xmm2
+; SSE2-NEXT:    pand %xmm1, %xmm2
+; SSE2-NEXT:    pxor %xmm1, %xmm1
+; SSE2-NEXT:    movdqa %xmm2, %xmm0
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; SSE2-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
+; SSE2-NEXT:    movdqa %xmm2, %xmm1
+; SSE2-NEXT:    retq
+;
+; AVX2-LABEL: zext_and_v8i32:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX2-NEXT:    retq
+  %xz = zext <8 x i16> %x to <8 x i32>
+  %yz = zext <8 x i16> %y to <8 x i32>
+  %r = and <8 x i32> %xz, %yz
+  ret <8 x i32> %r
+}
+
+define <8 x i32> @zext_or_v8i32(<8 x i16> %x, <8 x i16> %y) {
+; SSE2-LABEL: zext_or_v8i32:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa %xmm0, %xmm2
+; SSE2-NEXT:    por %xmm1, %xmm2
+; SSE2-NEXT:    pxor %xmm1, %xmm1
+; SSE2-NEXT:    movdqa %xmm2, %xmm0
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; SSE2-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
+; SSE2-NEXT:    movdqa %xmm2, %xmm1
+; SSE2-NEXT:    retq
+;
+; AVX2-LABEL: zext_or_v8i32:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX2-NEXT:    retq
+  %xz = zext <8 x i16> %x to <8 x i32>
+  %yz = zext <8 x i16> %y to <8 x i32>
+  %r = or <8 x i32> %xz, %yz
+  ret <8 x i32> %r
+}
+
+define <8 x i32> @zext_xor_v8i32(<8 x i16> %x, <8 x i16> %y) {
+; SSE2-LABEL: zext_xor_v8i32:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa %xmm0, %xmm2
+; SSE2-NEXT:    pxor %xmm1, %xmm2
+; SSE2-NEXT:    pxor %xmm1, %xmm1
+; SSE2-NEXT:    movdqa %xmm2, %xmm0
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; SSE2-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
+; SSE2-NEXT:    movdqa %xmm2, %xmm1
+; SSE2-NEXT:    retq
+;
+; AVX2-LABEL: zext_xor_v8i32:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX2-NEXT:    retq
+  %xz = zext <8 x i16> %x to <8 x i32>
+  %yz = zext <8 x i16> %y to <8 x i32>
+  %r = xor <8 x i32> %xz, %yz
+  ret <8 x i32> %r
+}
+
+define <8 x i32> @sext_and_v8i32(<8 x i16> %x, <8 x i16> %y) {
+; SSE2-LABEL: sext_and_v8i32:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    pand %xmm1, %xmm0
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
+; SSE2-NEXT:    psrad $16, %xmm2
+; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSE2-NEXT:    psrad $16, %xmm1
+; SSE2-NEXT:    movdqa %xmm2, %xmm0
+; SSE2-NEXT:    retq
+;
+; AVX2-LABEL: sext_and_v8i32:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpmovsxwd %xmm0, %ymm0
+; AVX2-NEXT:    retq
+  %xs = sext <8 x i16> %x to <8 x i32>
+  %ys = sext <8 x i16> %y to <8 x i32>
+  %r = and <8 x i32> %xs, %ys
+  ret <8 x i32> %r
+}
+
+define <8 x i32> @sext_or_v8i32(<8 x i16> %x, <8 x i16> %y) {
+; SSE2-LABEL: sext_or_v8i32:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    por %xmm1, %xmm0
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
+; SSE2-NEXT:    psrad $16, %xmm2
+; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSE2-NEXT:    psrad $16, %xmm1
+; SSE2-NEXT:    movdqa %xmm2, %xmm0
+; SSE2-NEXT:    retq
+;
+; AVX2-LABEL: sext_or_v8i32:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpmovsxwd %xmm0, %ymm0
+; AVX2-NEXT:    retq
+  %xs = sext <8 x i16> %x to <8 x i32>
+  %ys = sext <8 x i16> %y to <8 x i32>
+  %r = or <8 x i32> %xs, %ys
+  ret <8 x i32> %r
+}
+
+define <8 x i32> @sext_xor_v8i32(<8 x i16> %x, <8 x i16> %y) {
+; SSE2-LABEL: sext_xor_v8i32:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    pxor %xmm1, %xmm0
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
+; SSE2-NEXT:    psrad $16, %xmm2
+; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSE2-NEXT:    psrad $16, %xmm1
+; SSE2-NEXT:    movdqa %xmm2, %xmm0
+; SSE2-NEXT:    retq
+;
+; AVX2-LABEL: sext_xor_v8i32:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpmovsxwd %xmm0, %ymm0
+; AVX2-NEXT:    retq
+  %xs = sext <8 x i16> %x to <8 x i32>
+  %ys = sext <8 x i16> %y to <8 x i32>
+  %r = xor <8 x i32> %xs, %ys
+  ret <8 x i32> %r
+}
+
+define <8 x i16> @zext_and_v8i16(<8 x i8> %x, <8 x i8> %y) {
+; SSE2-LABEL: zext_and_v8i16:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    andps %xmm1, %xmm0
+; SSE2-NEXT:    andps {{.*}}(%rip), %xmm0
+; SSE2-NEXT:    retq
+;
+; AVX2-LABEL: zext_and_v8i16:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vandps %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vandps {{.*}}(%rip), %xmm0, %xmm0
+; AVX2-NEXT:    retq
+  %xz = zext <8 x i8> %x to <8 x i16>
+  %yz = zext <8 x i8> %y to <8 x i16>
+  %r = and <8 x i16> %xz, %yz
+  ret <8 x i16> %r
+}
+
+define <8 x i16> @zext_or_v8i16(<8 x i8> %x, <8 x i8> %y) {
+; SSE2-LABEL: zext_or_v8i16:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    orps %xmm1, %xmm0
+; SSE2-NEXT:    andps {{.*}}(%rip), %xmm0
+; SSE2-NEXT:    retq
+;
+; AVX2-LABEL: zext_or_v8i16:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vorps %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vandps {{.*}}(%rip), %xmm0, %xmm0
+; AVX2-NEXT:    retq
+  %xz = zext <8 x i8> %x to <8 x i16>
+  %yz = zext <8 x i8> %y to <8 x i16>
+  %r = or <8 x i16> %xz, %yz
+  ret <8 x i16> %r
+}
+
+define <8 x i16> @zext_xor_v8i16(<8 x i8> %x, <8 x i8> %y) {
+; SSE2-LABEL: zext_xor_v8i16:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    xorps %xmm1, %xmm0
+; SSE2-NEXT:    andps {{.*}}(%rip), %xmm0
+; SSE2-NEXT:    retq
+;
+; AVX2-LABEL: zext_xor_v8i16:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vxorps %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vandps {{.*}}(%rip), %xmm0, %xmm0
+; AVX2-NEXT:    retq
+  %xz = zext <8 x i8> %x to <8 x i16>
+  %yz = zext <8 x i8> %y to <8 x i16>
+  %r = xor <8 x i16> %xz, %yz
+  ret <8 x i16> %r
+}
+
+define <8 x i16> @sext_and_v8i16(<8 x i8> %x, <8 x i8> %y) {
+; SSE2-LABEL: sext_and_v8i16:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    psllw $8, %xmm0
+; SSE2-NEXT:    psraw $8, %xmm0
+; SSE2-NEXT:    psllw $8, %xmm1
+; SSE2-NEXT:    psraw $8, %xmm1
+; SSE2-NEXT:    pand %xmm1, %xmm0
+; SSE2-NEXT:    retq
+;
+; AVX2-LABEL: sext_and_v8i16:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpsllw $8, %xmm0, %xmm0
+; AVX2-NEXT:    vpsraw $8, %xmm0, %xmm0
+; AVX2-NEXT:    vpsllw $8, %xmm1, %xmm1
+; AVX2-NEXT:    vpsraw $8, %xmm1, %xmm1
+; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    retq
+  %xs = sext <8 x i8> %x to <8 x i16>
+  %ys = sext <8 x i8> %y to <8 x i16>
+  %r = and <8 x i16> %xs, %ys
+  ret <8 x i16> %r
+}
+
+define <8 x i16> @sext_or_v8i16(<8 x i8> %x, <8 x i8> %y) {
+; SSE2-LABEL: sext_or_v8i16:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    psllw $8, %xmm0
+; SSE2-NEXT:    psraw $8, %xmm0
+; SSE2-NEXT:    psllw $8, %xmm1
+; SSE2-NEXT:    psraw $8, %xmm1
+; SSE2-NEXT:    por %xmm1, %xmm0
+; SSE2-NEXT:    retq
+;
+; AVX2-LABEL: sext_or_v8i16:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpsllw $8, %xmm0, %xmm0
+; AVX2-NEXT:    vpsraw $8, %xmm0, %xmm0
+; AVX2-NEXT:    vpsllw $8, %xmm1, %xmm1
+; AVX2-NEXT:    vpsraw $8, %xmm1, %xmm1
+; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    retq
+  %xs = sext <8 x i8> %x to <8 x i16>
+  %ys = sext <8 x i8> %y to <8 x i16>
+  %r = or <8 x i16> %xs, %ys
+  ret <8 x i16> %r
+}
+
+define <8 x i16> @sext_xor_v8i16(<8 x i8> %x, <8 x i8> %y) {
+; SSE2-LABEL: sext_xor_v8i16:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    psllw $8, %xmm0
+; SSE2-NEXT:    psraw $8, %xmm0
+; SSE2-NEXT:    psllw $8, %xmm1
+; SSE2-NEXT:    psraw $8, %xmm1
+; SSE2-NEXT:    pxor %xmm1, %xmm0
+; SSE2-NEXT:    retq
+;
+; AVX2-LABEL: sext_xor_v8i16:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpsllw $8, %xmm0, %xmm0
+; AVX2-NEXT:    vpsraw $8, %xmm0, %xmm0
+; AVX2-NEXT:    vpsllw $8, %xmm1, %xmm1
+; AVX2-NEXT:    vpsraw $8, %xmm1, %xmm1
+; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    retq
+  %xs = sext <8 x i8> %x to <8 x i16>
+  %ys = sext <8 x i8> %y to <8 x i16>
+  %r = xor <8 x i16> %xs, %ys
+  ret <8 x i16> %r
+}
+
+define <8 x i32> @bool_zext_and(<8 x i1> %x, <8 x i1> %y) {
+; SSE2-LABEL: bool_zext_and:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa %xmm0, %xmm3
+; SSE2-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm1
+; SSE2-NEXT:    pxor %xmm4, %xmm4
+; SSE2-NEXT:    movdqa %xmm1, %xmm2
+; SSE2-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm4[4],xmm2[5],xmm4[5],xmm2[6],xmm4[6],xmm2[7],xmm4[7]
+; SSE2-NEXT:    pand %xmm3, %xmm2
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1],xmm1[2],xmm4[2],xmm1[3],xmm4[3]
+; SSE2-NEXT:    pand %xmm1, %xmm0
+; SSE2-NEXT:    movdqa %xmm2, %xmm1
+; SSE2-NEXT:    retq
+;
+; AVX2-LABEL: bool_zext_and:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX2-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+  %xz = zext <8 x i1> %x to <8 x i32>
+  %yz = zext <8 x i1> %y to <8 x i32>
+  %r = and <8 x i32> %xz, %yz
+  ret <8 x i32> %r
+}
+
+define <8 x i32> @bool_zext_or(<8 x i1> %x, <8 x i1> %y) {
+; SSE2-LABEL: bool_zext_or:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [1,1,1,1,1,1,1,1]
+; SSE2-NEXT:    pand %xmm2, %xmm0
+; SSE2-NEXT:    pxor %xmm3, %xmm3
+; SSE2-NEXT:    movdqa %xmm0, %xmm4
+; SSE2-NEXT:    punpckhwd {{.*#+}} xmm4 = xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7]
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
+; SSE2-NEXT:    pand %xmm2, %xmm1
+; SSE2-NEXT:    movdqa %xmm1, %xmm2
+; SSE2-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm3[4],xmm2[5],xmm3[5],xmm2[6],xmm3[6],xmm2[7],xmm3[7]
+; SSE2-NEXT:    por %xmm4, %xmm2
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3]
+; SSE2-NEXT:    por %xmm1, %xmm0
+; SSE2-NEXT:    movdqa %xmm2, %xmm1
+; SSE2-NEXT:    retq
+;
+; AVX2-LABEL: bool_zext_or:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX2-NEXT:    retq
+  %xz = zext <8 x i1> %x to <8 x i32>
+  %yz = zext <8 x i1> %y to <8 x i32>
+  %r = or <8 x i32> %xz, %yz
+  ret <8 x i32> %r
+}
+
+define <8 x i32> @bool_zext_xor(<8 x i1> %x, <8 x i1> %y) {
+; SSE2-LABEL: bool_zext_xor:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [1,1,1,1,1,1,1,1]
+; SSE2-NEXT:    pand %xmm2, %xmm0
+; SSE2-NEXT:    pxor %xmm3, %xmm3
+; SSE2-NEXT:    movdqa %xmm0, %xmm4
+; SSE2-NEXT:    punpckhwd {{.*#+}} xmm4 = xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7]
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
+; SSE2-NEXT:    pand %xmm2, %xmm1
+; SSE2-NEXT:    movdqa %xmm1, %xmm2
+; SSE2-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm3[4],xmm2[5],xmm3[5],xmm2[6],xmm3[6],xmm2[7],xmm3[7]
+; SSE2-NEXT:    pxor %xmm4, %xmm2
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3]
+; SSE2-NEXT:    pxor %xmm1, %xmm0
+; SSE2-NEXT:    movdqa %xmm2, %xmm1
+; SSE2-NEXT:    retq
+;
+; AVX2-LABEL: bool_zext_xor:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX2-NEXT:    retq
+  %xz = zext <8 x i1> %x to <8 x i32>
+  %yz = zext <8 x i1> %y to <8 x i32>
+  %r = xor <8 x i32> %xz, %yz
+  ret <8 x i32> %r
+}
+
+define <8 x i32> @bool_sext_and(<8 x i1> %x, <8 x i1> %y) {
+; SSE2-LABEL: bool_sext_and:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa %xmm1, %xmm3
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3]
+; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSE2-NEXT:    movdqa %xmm0, %xmm2
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
+; SSE2-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    pslld $31, %xmm0
+; SSE2-NEXT:    psrad $31, %xmm0
+; SSE2-NEXT:    pslld $31, %xmm2
+; SSE2-NEXT:    psrad $31, %xmm2
+; SSE2-NEXT:    pslld $31, %xmm1
+; SSE2-NEXT:    psrad $31, %xmm1
+; SSE2-NEXT:    pand %xmm0, %xmm1
+; SSE2-NEXT:    pslld $31, %xmm3
+; SSE2-NEXT:    psrad $31, %xmm3
+; SSE2-NEXT:    pand %xmm3, %xmm2
+; SSE2-NEXT:    movdqa %xmm2, %xmm0
+; SSE2-NEXT:    retq
+;
+; AVX2-LABEL: bool_sext_and:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX2-NEXT:    vpslld $31, %ymm0, %ymm0
+; AVX2-NEXT:    vpsrad $31, %ymm0, %ymm0
+; AVX2-NEXT:    vpslld $31, %ymm1, %ymm1
+; AVX2-NEXT:    vpsrad $31, %ymm1, %ymm1
+; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+  %xs = sext <8 x i1> %x to <8 x i32>
+  %ys = sext <8 x i1> %y to <8 x i32>
+  %r = and <8 x i32> %xs, %ys
+  ret <8 x i32> %r
+}
+
+define <8 x i32> @bool_sext_or(<8 x i1> %x, <8 x i1> %y) {
+; SSE2-LABEL: bool_sext_or:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa %xmm1, %xmm3
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3]
+; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSE2-NEXT:    movdqa %xmm0, %xmm2
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
+; SSE2-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    pslld $31, %xmm0
+; SSE2-NEXT:    psrad $31, %xmm0
+; SSE2-NEXT:    pslld $31, %xmm2
+; SSE2-NEXT:    psrad $31, %xmm2
+; SSE2-NEXT:    pslld $31, %xmm1
+; SSE2-NEXT:    psrad $31, %xmm1
+; SSE2-NEXT:    por %xmm0, %xmm1
+; SSE2-NEXT:    pslld $31, %xmm3
+; SSE2-NEXT:    psrad $31, %xmm3
+; SSE2-NEXT:    por %xmm3, %xmm2
+; SSE2-NEXT:    movdqa %xmm2, %xmm0
+; SSE2-NEXT:    retq
+;
+; AVX2-LABEL: bool_sext_or:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX2-NEXT:    vpslld $31, %ymm0, %ymm0
+; AVX2-NEXT:    vpsrad $31, %ymm0, %ymm0
+; AVX2-NEXT:    vpslld $31, %ymm1, %ymm1
+; AVX2-NEXT:    vpsrad $31, %ymm1, %ymm1
+; AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+  %xs = sext <8 x i1> %x to <8 x i32>
+  %ys = sext <8 x i1> %y to <8 x i32>
+  %r = or <8 x i32> %xs, %ys
+  ret <8 x i32> %r
+}
+
+define <8 x i32> @bool_sext_xor(<8 x i1> %x, <8 x i1> %y) {
+; SSE2-LABEL: bool_sext_xor:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa %xmm1, %xmm3
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3]
+; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; SSE2-NEXT:    movdqa %xmm0, %xmm2
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
+; SSE2-NEXT:    punpckhwd {{.*#+}} xmm0 = xmm0[4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    pslld $31, %xmm0
+; SSE2-NEXT:    psrad $31, %xmm0
+; SSE2-NEXT:    pslld $31, %xmm2
+; SSE2-NEXT:    psrad $31, %xmm2
+; SSE2-NEXT:    pslld $31, %xmm1
+; SSE2-NEXT:    psrad $31, %xmm1
+; SSE2-NEXT:    pxor %xmm0, %xmm1
+; SSE2-NEXT:    pslld $31, %xmm3
+; SSE2-NEXT:    psrad $31, %xmm3
+; SSE2-NEXT:    pxor %xmm3, %xmm2
+; SSE2-NEXT:    movdqa %xmm2, %xmm0
+; SSE2-NEXT:    retq
+;
+; AVX2-LABEL: bool_sext_xor:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX2-NEXT:    vpslld $31, %ymm0, %ymm0
+; AVX2-NEXT:    vpsrad $31, %ymm0, %ymm0
+; AVX2-NEXT:    vpslld $31, %ymm1, %ymm1
+; AVX2-NEXT:    vpsrad $31, %ymm1, %ymm1
+; AVX2-NEXT:    vpxor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+  %xs = sext <8 x i1> %x to <8 x i32>
+  %ys = sext <8 x i1> %y to <8 x i32>
+  %r = xor <8 x i32> %xs, %ys
+  ret <8 x i32> %r
+}
+
diff --git a/test/CodeGen/X86/vector-fshl-128.ll b/test/CodeGen/X86/vector-fshl-128.ll
index eb3dadf0c27561859c17f907babf61a7aa6a34fe..4a51e3341b2555daa3a9063931130becdb564f00 100644
--- a/test/CodeGen/X86/vector-fshl-128.ll
+++ b/test/CodeGen/X86/vector-fshl-128.ll
@@ -6,7 +6,9 @@
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f | FileCheck %s --check-prefixes=AVX512,AVX512F
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512vl | FileCheck %s --check-prefixes=AVX512,AVX512VL
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512bw | FileCheck %s --check-prefixes=AVX512,AVX512BW
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512vbmi2 | FileCheck %s --check-prefixes=AVX512,AVX512VBMI2
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512bw,+avx512vl | FileCheck %s --check-prefixes=AVX512,AVX512VLBW
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512vbmi2,+avx512vl | FileCheck %s --check-prefixes=AVX512,AVX512VLVBMI2
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+xop,+avx | FileCheck %s --check-prefixes=XOP,XOPAVX1
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+xop,+avx2 | FileCheck %s --check-prefixes=XOP,XOPAVX2
 
@@ -152,6 +154,23 @@ define <2 x i64> @var_funnnel_v2i64(<2 x i64> %x, <2 x i64> %y, <2 x i64> %amt)
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: var_funnnel_v2i64:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $xmm2 killed $xmm2 def $zmm2
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} xmm3 = [63,63]
+; AVX512VBMI2-NEXT:    vpand %xmm3, %xmm2, %xmm4
+; AVX512VBMI2-NEXT:    vpsllvq %xmm4, %xmm0, %xmm5
+; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} xmm6 = [64,64]
+; AVX512VBMI2-NEXT:    vpsubq %xmm4, %xmm6, %xmm4
+; AVX512VBMI2-NEXT:    vpsrlvq %xmm4, %xmm1, %xmm1
+; AVX512VBMI2-NEXT:    vpor %xmm1, %xmm5, %xmm1
+; AVX512VBMI2-NEXT:    vptestnmq %zmm3, %zmm2, %k1
+; AVX512VBMI2-NEXT:    vmovdqa64 %zmm0, %zmm1 {%k1}
+; AVX512VBMI2-NEXT:    vmovdqa %xmm1, %xmm0
+; AVX512VBMI2-NEXT:    vzeroupper
+; AVX512VBMI2-NEXT:    retq
+;
 ; AVX512VLBW-LABEL: var_funnnel_v2i64:
 ; AVX512VLBW:       # %bb.0:
 ; AVX512VLBW-NEXT:    vmovdqa {{.*#+}} xmm3 = [63,63]
@@ -166,6 +185,11 @@ define <2 x i64> @var_funnnel_v2i64(<2 x i64> %x, <2 x i64> %y, <2 x i64> %amt)
 ; AVX512VLBW-NEXT:    vmovdqa %xmm1, %xmm0
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VLVBMI2-LABEL: var_funnnel_v2i64:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpshldvq %xmm2, %xmm1, %xmm0
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOPAVX1-LABEL: var_funnnel_v2i64:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
@@ -380,6 +404,23 @@ define <4 x i32> @var_funnnel_v4i32(<4 x i32> %x, <4 x i32> %y, <4 x i32> %amt)
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: var_funnnel_v4i32:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $xmm2 killed $xmm2 def $zmm2
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512VBMI2-NEXT:    vpbroadcastd {{.*#+}} xmm3 = [31,31,31,31]
+; AVX512VBMI2-NEXT:    vpand %xmm3, %xmm2, %xmm4
+; AVX512VBMI2-NEXT:    vpsllvd %xmm4, %xmm0, %xmm5
+; AVX512VBMI2-NEXT:    vpbroadcastd {{.*#+}} xmm6 = [32,32,32,32]
+; AVX512VBMI2-NEXT:    vpsubd %xmm4, %xmm6, %xmm4
+; AVX512VBMI2-NEXT:    vpsrlvd %xmm4, %xmm1, %xmm1
+; AVX512VBMI2-NEXT:    vpor %xmm1, %xmm5, %xmm1
+; AVX512VBMI2-NEXT:    vptestnmd %zmm3, %zmm2, %k1
+; AVX512VBMI2-NEXT:    vmovdqa32 %zmm0, %zmm1 {%k1}
+; AVX512VBMI2-NEXT:    vmovdqa %xmm1, %xmm0
+; AVX512VBMI2-NEXT:    vzeroupper
+; AVX512VBMI2-NEXT:    retq
+;
 ; AVX512VLBW-LABEL: var_funnnel_v4i32:
 ; AVX512VLBW:       # %bb.0:
 ; AVX512VLBW-NEXT:    vpbroadcastd {{.*#+}} xmm3 = [31,31,31,31]
@@ -394,6 +435,11 @@ define <4 x i32> @var_funnnel_v4i32(<4 x i32> %x, <4 x i32> %y, <4 x i32> %amt)
 ; AVX512VLBW-NEXT:    vmovdqa %xmm1, %xmm0
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VLVBMI2-LABEL: var_funnnel_v4i32:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpshldvd %xmm2, %xmm1, %xmm0
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOPAVX1-LABEL: var_funnnel_v4i32:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
@@ -689,6 +735,24 @@ define <8 x i16> @var_funnnel_v8i16(<8 x i16> %x, <8 x i16> %y, <8 x i16> %amt)
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: var_funnnel_v8i16:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $xmm2 killed $xmm2 def $zmm2
+; AVX512VBMI2-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} xmm3 = [15,15,15,15,15,15,15,15]
+; AVX512VBMI2-NEXT:    vpand %xmm3, %xmm2, %xmm4
+; AVX512VBMI2-NEXT:    vpsllvw %zmm4, %zmm0, %zmm5
+; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} xmm6 = [16,16,16,16,16,16,16,16]
+; AVX512VBMI2-NEXT:    vpsubw %xmm4, %xmm6, %xmm4
+; AVX512VBMI2-NEXT:    vpsrlvw %zmm4, %zmm1, %zmm1
+; AVX512VBMI2-NEXT:    vpor %xmm1, %xmm5, %xmm1
+; AVX512VBMI2-NEXT:    vptestnmw %zmm3, %zmm2, %k1
+; AVX512VBMI2-NEXT:    vmovdqu16 %zmm0, %zmm1 {%k1}
+; AVX512VBMI2-NEXT:    vmovdqa %xmm1, %xmm0
+; AVX512VBMI2-NEXT:    vzeroupper
+; AVX512VBMI2-NEXT:    retq
+;
 ; AVX512VLBW-LABEL: var_funnnel_v8i16:
 ; AVX512VLBW:       # %bb.0:
 ; AVX512VLBW-NEXT:    vmovdqa {{.*#+}} xmm3 = [15,15,15,15,15,15,15,15]
@@ -703,6 +767,11 @@ define <8 x i16> @var_funnnel_v8i16(<8 x i16> %x, <8 x i16> %y, <8 x i16> %amt)
 ; AVX512VLBW-NEXT:    vmovdqa %xmm1, %xmm0
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VLVBMI2-LABEL: var_funnnel_v8i16:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpshldvw %xmm2, %xmm1, %xmm0
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOP-LABEL: var_funnnel_v8i16:
 ; XOP:       # %bb.0:
 ; XOP-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
@@ -991,6 +1060,28 @@ define <16 x i8> @var_funnnel_v16i8(<16 x i8> %x, <16 x i8> %y, <16 x i8> %amt)
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: var_funnnel_v16i8:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $xmm2 killed $xmm2 def $zmm2
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512VBMI2-NEXT:    vpmovzxbw {{.*#+}} ymm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
+; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} xmm4 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
+; AVX512VBMI2-NEXT:    vpand %xmm4, %xmm2, %xmm5
+; AVX512VBMI2-NEXT:    vpmovzxbw {{.*#+}} ymm6 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero,xmm5[8],zero,xmm5[9],zero,xmm5[10],zero,xmm5[11],zero,xmm5[12],zero,xmm5[13],zero,xmm5[14],zero,xmm5[15],zero
+; AVX512VBMI2-NEXT:    vpsllvw %zmm6, %zmm3, %zmm3
+; AVX512VBMI2-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
+; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} xmm6 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; AVX512VBMI2-NEXT:    vpsubb %xmm5, %xmm6, %xmm5
+; AVX512VBMI2-NEXT:    vpmovzxbw {{.*#+}} ymm5 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero,xmm5[8],zero,xmm5[9],zero,xmm5[10],zero,xmm5[11],zero,xmm5[12],zero,xmm5[13],zero,xmm5[14],zero,xmm5[15],zero
+; AVX512VBMI2-NEXT:    vpsrlvw %zmm5, %zmm1, %zmm1
+; AVX512VBMI2-NEXT:    vpor %ymm1, %ymm3, %ymm1
+; AVX512VBMI2-NEXT:    vpmovwb %zmm1, %ymm1
+; AVX512VBMI2-NEXT:    vptestnmb %zmm4, %zmm2, %k1
+; AVX512VBMI2-NEXT:    vmovdqu8 %zmm0, %zmm1 {%k1}
+; AVX512VBMI2-NEXT:    vmovdqa %xmm1, %xmm0
+; AVX512VBMI2-NEXT:    vzeroupper
+; AVX512VBMI2-NEXT:    retq
+;
 ; AVX512VLBW-LABEL: var_funnnel_v16i8:
 ; AVX512VLBW:       # %bb.0:
 ; AVX512VLBW-NEXT:    vmovdqa {{.*#+}} xmm3 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
@@ -1011,6 +1102,26 @@ define <16 x i8> @var_funnnel_v16i8(<16 x i8> %x, <16 x i8> %y, <16 x i8> %amt)
 ; AVX512VLBW-NEXT:    vzeroupper
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VLVBMI2-LABEL: var_funnnel_v16i8:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vmovdqa {{.*#+}} xmm3 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
+; AVX512VLVBMI2-NEXT:    vpand %xmm3, %xmm2, %xmm4
+; AVX512VLVBMI2-NEXT:    vpmovzxbw {{.*#+}} ymm5 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero,xmm4[8],zero,xmm4[9],zero,xmm4[10],zero,xmm4[11],zero,xmm4[12],zero,xmm4[13],zero,xmm4[14],zero,xmm4[15],zero
+; AVX512VLVBMI2-NEXT:    vpmovzxbw {{.*#+}} ymm6 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
+; AVX512VLVBMI2-NEXT:    vpsllvw %ymm5, %ymm6, %ymm5
+; AVX512VLVBMI2-NEXT:    vmovdqa {{.*#+}} xmm6 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; AVX512VLVBMI2-NEXT:    vpsubb %xmm4, %xmm6, %xmm4
+; AVX512VLVBMI2-NEXT:    vpmovzxbw {{.*#+}} ymm4 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero,xmm4[8],zero,xmm4[9],zero,xmm4[10],zero,xmm4[11],zero,xmm4[12],zero,xmm4[13],zero,xmm4[14],zero,xmm4[15],zero
+; AVX512VLVBMI2-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
+; AVX512VLVBMI2-NEXT:    vpsrlvw %ymm4, %ymm1, %ymm1
+; AVX512VLVBMI2-NEXT:    vpor %ymm1, %ymm5, %ymm1
+; AVX512VLVBMI2-NEXT:    vpmovwb %ymm1, %xmm1
+; AVX512VLVBMI2-NEXT:    vptestnmb %xmm3, %xmm2, %k1
+; AVX512VLVBMI2-NEXT:    vmovdqu8 %xmm0, %xmm1 {%k1}
+; AVX512VLVBMI2-NEXT:    vmovdqa %xmm1, %xmm0
+; AVX512VLVBMI2-NEXT:    vzeroupper
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOP-LABEL: var_funnnel_v16i8:
 ; XOP:       # %bb.0:
 ; XOP-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
@@ -1210,6 +1321,23 @@ define <2 x i64> @splatvar_funnnel_v2i64(<2 x i64> %x, <2 x i64> %y, <2 x i64> %
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: splatvar_funnnel_v2i64:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512VBMI2-NEXT:    vpbroadcastq %xmm2, %xmm2
+; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} xmm3 = [63,63]
+; AVX512VBMI2-NEXT:    vpand %xmm3, %xmm2, %xmm4
+; AVX512VBMI2-NEXT:    vpsllq %xmm4, %xmm0, %xmm5
+; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} xmm6 = [64,64]
+; AVX512VBMI2-NEXT:    vpsubq %xmm4, %xmm6, %xmm4
+; AVX512VBMI2-NEXT:    vpsrlq %xmm4, %xmm1, %xmm1
+; AVX512VBMI2-NEXT:    vpor %xmm1, %xmm5, %xmm1
+; AVX512VBMI2-NEXT:    vptestnmq %zmm3, %zmm2, %k1
+; AVX512VBMI2-NEXT:    vmovdqa64 %zmm0, %zmm1 {%k1}
+; AVX512VBMI2-NEXT:    vmovdqa %xmm1, %xmm0
+; AVX512VBMI2-NEXT:    vzeroupper
+; AVX512VBMI2-NEXT:    retq
+;
 ; AVX512VLBW-LABEL: splatvar_funnnel_v2i64:
 ; AVX512VLBW:       # %bb.0:
 ; AVX512VLBW-NEXT:    vpbroadcastq %xmm2, %xmm2
@@ -1225,6 +1353,12 @@ define <2 x i64> @splatvar_funnnel_v2i64(<2 x i64> %x, <2 x i64> %y, <2 x i64> %
 ; AVX512VLBW-NEXT:    vmovdqa %xmm1, %xmm0
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VLVBMI2-LABEL: splatvar_funnnel_v2i64:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpbroadcastq %xmm2, %xmm2
+; AVX512VLVBMI2-NEXT:    vpshldvq %xmm2, %xmm1, %xmm0
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOPAVX1-LABEL: splatvar_funnnel_v2i64:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[0,1,0,1]
@@ -1294,12 +1428,13 @@ define <4 x i32> @splatvar_funnnel_v4i32(<4 x i32> %x, <4 x i32> %y, <4 x i32> %
 ; SSE2-NEXT:    movss {{.*#+}} xmm4 = xmm2[0],xmm4[1,2,3]
 ; SSE2-NEXT:    movdqa %xmm0, %xmm5
 ; SSE2-NEXT:    pslld %xmm4, %xmm5
-; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [32,32,32,32]
-; SSE2-NEXT:    psubd %xmm2, %xmm4
-; SSE2-NEXT:    pcmpeqd %xmm3, %xmm2
-; SSE2-NEXT:    movss {{.*#+}} xmm3 = xmm4[0],xmm3[1,2,3]
-; SSE2-NEXT:    psrld %xmm3, %xmm1
+; SSE2-NEXT:    movd %xmm2, %eax
+; SSE2-NEXT:    movl $32, %ecx
+; SSE2-NEXT:    subl %eax, %ecx
+; SSE2-NEXT:    movd %ecx, %xmm4
+; SSE2-NEXT:    psrld %xmm4, %xmm1
 ; SSE2-NEXT:    por %xmm5, %xmm1
+; SSE2-NEXT:    pcmpeqd %xmm3, %xmm2
 ; SSE2-NEXT:    pand %xmm2, %xmm0
 ; SSE2-NEXT:    pandn %xmm1, %xmm2
 ; SSE2-NEXT:    por %xmm2, %xmm0
@@ -1412,6 +1547,25 @@ define <4 x i32> @splatvar_funnnel_v4i32(<4 x i32> %x, <4 x i32> %y, <4 x i32> %
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: splatvar_funnnel_v4i32:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512VBMI2-NEXT:    vpbroadcastd %xmm2, %xmm2
+; AVX512VBMI2-NEXT:    vpbroadcastd {{.*#+}} xmm3 = [31,31,31,31]
+; AVX512VBMI2-NEXT:    vpand %xmm3, %xmm2, %xmm4
+; AVX512VBMI2-NEXT:    vpmovzxdq {{.*#+}} xmm5 = xmm4[0],zero,xmm4[1],zero
+; AVX512VBMI2-NEXT:    vpslld %xmm5, %xmm0, %xmm5
+; AVX512VBMI2-NEXT:    vpbroadcastd {{.*#+}} xmm6 = [32,32,32,32]
+; AVX512VBMI2-NEXT:    vpsubd %xmm4, %xmm6, %xmm4
+; AVX512VBMI2-NEXT:    vpmovzxdq {{.*#+}} xmm4 = xmm4[0],zero,xmm4[1],zero
+; AVX512VBMI2-NEXT:    vpsrld %xmm4, %xmm1, %xmm1
+; AVX512VBMI2-NEXT:    vpor %xmm1, %xmm5, %xmm1
+; AVX512VBMI2-NEXT:    vptestnmd %zmm3, %zmm2, %k1
+; AVX512VBMI2-NEXT:    vmovdqa32 %zmm0, %zmm1 {%k1}
+; AVX512VBMI2-NEXT:    vmovdqa %xmm1, %xmm0
+; AVX512VBMI2-NEXT:    vzeroupper
+; AVX512VBMI2-NEXT:    retq
+;
 ; AVX512VLBW-LABEL: splatvar_funnnel_v4i32:
 ; AVX512VLBW:       # %bb.0:
 ; AVX512VLBW-NEXT:    vpbroadcastd %xmm2, %xmm2
@@ -1429,6 +1583,12 @@ define <4 x i32> @splatvar_funnnel_v4i32(<4 x i32> %x, <4 x i32> %y, <4 x i32> %
 ; AVX512VLBW-NEXT:    vmovdqa %xmm1, %xmm0
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VLVBMI2-LABEL: splatvar_funnnel_v4i32:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpbroadcastd %xmm2, %xmm2
+; AVX512VLVBMI2-NEXT:    vpshldvd %xmm2, %xmm1, %xmm0
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOPAVX1-LABEL: splatvar_funnnel_v4i32:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[0,0,0,0]
@@ -1471,12 +1631,13 @@ define <4 x i32> @splatvar_funnnel_v4i32(<4 x i32> %x, <4 x i32> %y, <4 x i32> %
 ; X32-SSE-NEXT:    movss {{.*#+}} xmm4 = xmm2[0],xmm4[1,2,3]
 ; X32-SSE-NEXT:    movdqa %xmm0, %xmm5
 ; X32-SSE-NEXT:    pslld %xmm4, %xmm5
-; X32-SSE-NEXT:    movdqa {{.*#+}} xmm4 = [32,32,32,32]
-; X32-SSE-NEXT:    psubd %xmm2, %xmm4
-; X32-SSE-NEXT:    pcmpeqd %xmm3, %xmm2
-; X32-SSE-NEXT:    movss {{.*#+}} xmm3 = xmm4[0],xmm3[1,2,3]
-; X32-SSE-NEXT:    psrld %xmm3, %xmm1
+; X32-SSE-NEXT:    movd %xmm2, %eax
+; X32-SSE-NEXT:    movl $32, %ecx
+; X32-SSE-NEXT:    subl %eax, %ecx
+; X32-SSE-NEXT:    movd %ecx, %xmm4
+; X32-SSE-NEXT:    psrld %xmm4, %xmm1
 ; X32-SSE-NEXT:    por %xmm5, %xmm1
+; X32-SSE-NEXT:    pcmpeqd %xmm3, %xmm2
 ; X32-SSE-NEXT:    pand %xmm2, %xmm0
 ; X32-SSE-NEXT:    pandn %xmm1, %xmm2
 ; X32-SSE-NEXT:    por %xmm2, %xmm0
@@ -1613,6 +1774,25 @@ define <8 x i16> @splatvar_funnnel_v8i16(<8 x i16> %x, <8 x i16> %y, <8 x i16> %
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: splatvar_funnnel_v8i16:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512VBMI2-NEXT:    vpbroadcastw %xmm2, %xmm2
+; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} xmm3 = [15,15,15,15,15,15,15,15]
+; AVX512VBMI2-NEXT:    vpand %xmm3, %xmm2, %xmm4
+; AVX512VBMI2-NEXT:    vpmovzxwq {{.*#+}} xmm5 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero
+; AVX512VBMI2-NEXT:    vpsllw %xmm5, %xmm0, %xmm5
+; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} xmm6 = [16,16,16,16,16,16,16,16]
+; AVX512VBMI2-NEXT:    vpsubw %xmm4, %xmm6, %xmm4
+; AVX512VBMI2-NEXT:    vpmovzxwq {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero
+; AVX512VBMI2-NEXT:    vpsrlw %xmm4, %xmm1, %xmm1
+; AVX512VBMI2-NEXT:    vpor %xmm1, %xmm5, %xmm1
+; AVX512VBMI2-NEXT:    vptestnmw %zmm3, %zmm2, %k1
+; AVX512VBMI2-NEXT:    vmovdqu16 %zmm0, %zmm1 {%k1}
+; AVX512VBMI2-NEXT:    vmovdqa %xmm1, %xmm0
+; AVX512VBMI2-NEXT:    vzeroupper
+; AVX512VBMI2-NEXT:    retq
+;
 ; AVX512VLBW-LABEL: splatvar_funnnel_v8i16:
 ; AVX512VLBW:       # %bb.0:
 ; AVX512VLBW-NEXT:    vpbroadcastw %xmm2, %xmm2
@@ -1630,6 +1810,12 @@ define <8 x i16> @splatvar_funnnel_v8i16(<8 x i16> %x, <8 x i16> %y, <8 x i16> %
 ; AVX512VLBW-NEXT:    vmovdqa %xmm1, %xmm0
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VLVBMI2-LABEL: splatvar_funnnel_v8i16:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpbroadcastw %xmm2, %xmm2
+; AVX512VLVBMI2-NEXT:    vpshldvw %xmm2, %xmm1, %xmm0
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOPAVX1-LABEL: splatvar_funnnel_v8i16:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm2 = xmm2[0,0,2,3,4,5,6,7]
@@ -1862,6 +2048,28 @@ define <16 x i8> @splatvar_funnnel_v16i8(<16 x i8> %x, <16 x i8> %y, <16 x i8> %
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: splatvar_funnnel_v16i8:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512VBMI2-NEXT:    vpbroadcastb %xmm2, %xmm2
+; AVX512VBMI2-NEXT:    vpmovzxbw {{.*#+}} ymm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
+; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} xmm4 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
+; AVX512VBMI2-NEXT:    vpand %xmm4, %xmm2, %xmm5
+; AVX512VBMI2-NEXT:    vpmovzxbw {{.*#+}} ymm6 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero,xmm5[8],zero,xmm5[9],zero,xmm5[10],zero,xmm5[11],zero,xmm5[12],zero,xmm5[13],zero,xmm5[14],zero,xmm5[15],zero
+; AVX512VBMI2-NEXT:    vpsllvw %zmm6, %zmm3, %zmm3
+; AVX512VBMI2-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
+; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} xmm6 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; AVX512VBMI2-NEXT:    vpsubb %xmm5, %xmm6, %xmm5
+; AVX512VBMI2-NEXT:    vpmovzxbw {{.*#+}} ymm5 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero,xmm5[8],zero,xmm5[9],zero,xmm5[10],zero,xmm5[11],zero,xmm5[12],zero,xmm5[13],zero,xmm5[14],zero,xmm5[15],zero
+; AVX512VBMI2-NEXT:    vpsrlvw %zmm5, %zmm1, %zmm1
+; AVX512VBMI2-NEXT:    vpor %ymm1, %ymm3, %ymm1
+; AVX512VBMI2-NEXT:    vpmovwb %zmm1, %ymm1
+; AVX512VBMI2-NEXT:    vptestnmb %zmm4, %zmm2, %k1
+; AVX512VBMI2-NEXT:    vmovdqu8 %zmm0, %zmm1 {%k1}
+; AVX512VBMI2-NEXT:    vmovdqa %xmm1, %xmm0
+; AVX512VBMI2-NEXT:    vzeroupper
+; AVX512VBMI2-NEXT:    retq
+;
 ; AVX512VLBW-LABEL: splatvar_funnnel_v16i8:
 ; AVX512VLBW:       # %bb.0:
 ; AVX512VLBW-NEXT:    vpbroadcastb %xmm2, %xmm2
@@ -1883,6 +2091,27 @@ define <16 x i8> @splatvar_funnnel_v16i8(<16 x i8> %x, <16 x i8> %y, <16 x i8> %
 ; AVX512VLBW-NEXT:    vzeroupper
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VLVBMI2-LABEL: splatvar_funnnel_v16i8:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpbroadcastb %xmm2, %xmm2
+; AVX512VLVBMI2-NEXT:    vmovdqa {{.*#+}} xmm3 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
+; AVX512VLVBMI2-NEXT:    vpand %xmm3, %xmm2, %xmm4
+; AVX512VLVBMI2-NEXT:    vpmovzxbw {{.*#+}} ymm5 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero,xmm4[8],zero,xmm4[9],zero,xmm4[10],zero,xmm4[11],zero,xmm4[12],zero,xmm4[13],zero,xmm4[14],zero,xmm4[15],zero
+; AVX512VLVBMI2-NEXT:    vpmovzxbw {{.*#+}} ymm6 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
+; AVX512VLVBMI2-NEXT:    vpsllvw %ymm5, %ymm6, %ymm5
+; AVX512VLVBMI2-NEXT:    vmovdqa {{.*#+}} xmm6 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; AVX512VLVBMI2-NEXT:    vpsubb %xmm4, %xmm6, %xmm4
+; AVX512VLVBMI2-NEXT:    vpmovzxbw {{.*#+}} ymm4 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero,xmm4[8],zero,xmm4[9],zero,xmm4[10],zero,xmm4[11],zero,xmm4[12],zero,xmm4[13],zero,xmm4[14],zero,xmm4[15],zero
+; AVX512VLVBMI2-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
+; AVX512VLVBMI2-NEXT:    vpsrlvw %ymm4, %ymm1, %ymm1
+; AVX512VLVBMI2-NEXT:    vpor %ymm1, %ymm5, %ymm1
+; AVX512VLVBMI2-NEXT:    vpmovwb %ymm1, %xmm1
+; AVX512VLVBMI2-NEXT:    vptestnmb %xmm3, %xmm2, %k1
+; AVX512VLVBMI2-NEXT:    vmovdqu8 %xmm0, %xmm1 {%k1}
+; AVX512VLVBMI2-NEXT:    vmovdqa %xmm1, %xmm0
+; AVX512VLVBMI2-NEXT:    vzeroupper
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOPAVX1-LABEL: splatvar_funnnel_v16i8:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
@@ -1998,12 +2227,45 @@ define <2 x i64> @constant_funnnel_v2i64(<2 x i64> %x, <2 x i64> %y) nounwind {
 ; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    retq
 ;
-; AVX512-LABEL: constant_funnnel_v2i64:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpsrlvq {{.*}}(%rip), %xmm1, %xmm1
-; AVX512-NEXT:    vpsllvq {{.*}}(%rip), %xmm0, %xmm0
-; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    retq
+; AVX512F-LABEL: constant_funnnel_v2i64:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpsrlvq {{.*}}(%rip), %xmm1, %xmm1
+; AVX512F-NEXT:    vpsllvq {{.*}}(%rip), %xmm0, %xmm0
+; AVX512F-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: constant_funnnel_v2i64:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpsrlvq {{.*}}(%rip), %xmm1, %xmm1
+; AVX512VL-NEXT:    vpsllvq {{.*}}(%rip), %xmm0, %xmm0
+; AVX512VL-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: constant_funnnel_v2i64:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpsrlvq {{.*}}(%rip), %xmm1, %xmm1
+; AVX512BW-NEXT:    vpsllvq {{.*}}(%rip), %xmm0, %xmm0
+; AVX512BW-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT:    retq
+;
+; AVX512VBMI2-LABEL: constant_funnnel_v2i64:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    vpsrlvq {{.*}}(%rip), %xmm1, %xmm1
+; AVX512VBMI2-NEXT:    vpsllvq {{.*}}(%rip), %xmm0, %xmm0
+; AVX512VBMI2-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLBW-LABEL: constant_funnnel_v2i64:
+; AVX512VLBW:       # %bb.0:
+; AVX512VLBW-NEXT:    vpsrlvq {{.*}}(%rip), %xmm1, %xmm1
+; AVX512VLBW-NEXT:    vpsllvq {{.*}}(%rip), %xmm0, %xmm0
+; AVX512VLBW-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX512VLBW-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: constant_funnnel_v2i64:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpshldvq {{.*}}(%rip), %xmm1, %xmm0
+; AVX512VLVBMI2-NEXT:    retq
 ;
 ; XOPAVX1-LABEL: constant_funnnel_v2i64:
 ; XOPAVX1:       # %bb.0:
@@ -2095,12 +2357,45 @@ define <4 x i32> @constant_funnnel_v4i32(<4 x i32> %x, <4 x i32> %y) nounwind {
 ; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    retq
 ;
-; AVX512-LABEL: constant_funnnel_v4i32:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpsrlvd {{.*}}(%rip), %xmm1, %xmm1
-; AVX512-NEXT:    vpsllvd {{.*}}(%rip), %xmm0, %xmm0
-; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    retq
+; AVX512F-LABEL: constant_funnnel_v4i32:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpsrlvd {{.*}}(%rip), %xmm1, %xmm1
+; AVX512F-NEXT:    vpsllvd {{.*}}(%rip), %xmm0, %xmm0
+; AVX512F-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: constant_funnnel_v4i32:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpsrlvd {{.*}}(%rip), %xmm1, %xmm1
+; AVX512VL-NEXT:    vpsllvd {{.*}}(%rip), %xmm0, %xmm0
+; AVX512VL-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: constant_funnnel_v4i32:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpsrlvd {{.*}}(%rip), %xmm1, %xmm1
+; AVX512BW-NEXT:    vpsllvd {{.*}}(%rip), %xmm0, %xmm0
+; AVX512BW-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT:    retq
+;
+; AVX512VBMI2-LABEL: constant_funnnel_v4i32:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    vpsrlvd {{.*}}(%rip), %xmm1, %xmm1
+; AVX512VBMI2-NEXT:    vpsllvd {{.*}}(%rip), %xmm0, %xmm0
+; AVX512VBMI2-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLBW-LABEL: constant_funnnel_v4i32:
+; AVX512VLBW:       # %bb.0:
+; AVX512VLBW-NEXT:    vpsrlvd {{.*}}(%rip), %xmm1, %xmm1
+; AVX512VLBW-NEXT:    vpsllvd {{.*}}(%rip), %xmm0, %xmm0
+; AVX512VLBW-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX512VLBW-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: constant_funnnel_v4i32:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpshldvd {{.*}}(%rip), %xmm1, %xmm0
+; AVX512VLVBMI2-NEXT:    retq
 ;
 ; XOPAVX1-LABEL: constant_funnnel_v4i32:
 ; XOPAVX1:       # %bb.0:
@@ -2205,6 +2500,19 @@ define <8 x i16> @constant_funnnel_v8i16(<8 x i16> %x, <8 x i16> %y) nounwind {
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: constant_funnnel_v8i16:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} xmm2 = [16,15,14,13,12,11,10,9]
+; AVX512VBMI2-NEXT:    vpsrlvw %zmm2, %zmm1, %zmm1
+; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} xmm2 = [0,1,2,3,4,5,6,7]
+; AVX512VBMI2-NEXT:    vpsllvw %zmm2, %zmm0, %zmm2
+; AVX512VBMI2-NEXT:    vpor %xmm1, %xmm2, %xmm1
+; AVX512VBMI2-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3,4,5,6,7]
+; AVX512VBMI2-NEXT:    vzeroupper
+; AVX512VBMI2-NEXT:    retq
+;
 ; AVX512VLBW-LABEL: constant_funnnel_v8i16:
 ; AVX512VLBW:       # %bb.0:
 ; AVX512VLBW-NEXT:    vpsrlvw {{.*}}(%rip), %xmm1, %xmm1
@@ -2213,6 +2521,11 @@ define <8 x i16> @constant_funnnel_v8i16(<8 x i16> %x, <8 x i16> %y) nounwind {
 ; AVX512VLBW-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3,4,5,6,7]
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VLVBMI2-LABEL: constant_funnnel_v8i16:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpshldvw {{.*}}(%rip), %xmm1, %xmm0
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOP-LABEL: constant_funnnel_v8i16:
 ; XOP:       # %bb.0:
 ; XOP-NEXT:    vpshlw {{.*}}(%rip), %xmm1, %xmm1
@@ -2382,6 +2695,21 @@ define <16 x i8> @constant_funnnel_v16i8(<16 x i8> %x, <16 x i8> %y) nounwind {
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: constant_funnnel_v16i8:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} ymm2 = [8,7,6,5,4,3,2,1,8,1,2,3,4,5,6,7]
+; AVX512VBMI2-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
+; AVX512VBMI2-NEXT:    vpsrlvw %zmm2, %zmm1, %zmm1
+; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} ymm2 = [0,1,2,3,4,5,6,7,0,7,6,5,4,3,2,1]
+; AVX512VBMI2-NEXT:    vpmovzxbw {{.*#+}} ymm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
+; AVX512VBMI2-NEXT:    vpsllvw %zmm2, %zmm3, %zmm2
+; AVX512VBMI2-NEXT:    vpor %ymm1, %ymm2, %ymm1
+; AVX512VBMI2-NEXT:    vpmovwb %zmm1, %ymm1
+; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} xmm2 = [0,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255]
+; AVX512VBMI2-NEXT:    vpblendvb %xmm2, %xmm1, %xmm0, %xmm0
+; AVX512VBMI2-NEXT:    vzeroupper
+; AVX512VBMI2-NEXT:    retq
+;
 ; AVX512VLBW-LABEL: constant_funnnel_v16i8:
 ; AVX512VLBW:       # %bb.0:
 ; AVX512VLBW-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
@@ -2397,6 +2725,21 @@ define <16 x i8> @constant_funnnel_v16i8(<16 x i8> %x, <16 x i8> %y) nounwind {
 ; AVX512VLBW-NEXT:    vzeroupper
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VLVBMI2-LABEL: constant_funnnel_v16i8:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
+; AVX512VLVBMI2-NEXT:    vpsrlvw {{.*}}(%rip), %ymm1, %ymm1
+; AVX512VLVBMI2-NEXT:    vpmovzxbw {{.*#+}} ymm2 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
+; AVX512VLVBMI2-NEXT:    vpsllvw {{.*}}(%rip), %ymm2, %ymm2
+; AVX512VLVBMI2-NEXT:    vpor %ymm1, %ymm2, %ymm1
+; AVX512VLVBMI2-NEXT:    vpmovwb %ymm1, %xmm1
+; AVX512VLVBMI2-NEXT:    movw $257, %ax # imm = 0x101
+; AVX512VLVBMI2-NEXT:    kmovd %eax, %k1
+; AVX512VLVBMI2-NEXT:    vmovdqu8 %xmm0, %xmm1 {%k1}
+; AVX512VLVBMI2-NEXT:    vmovdqa %xmm1, %xmm0
+; AVX512VLVBMI2-NEXT:    vzeroupper
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOP-LABEL: constant_funnnel_v16i8:
 ; XOP:       # %bb.0:
 ; XOP-NEXT:    vpshlb {{.*}}(%rip), %xmm1, %xmm1
@@ -2459,12 +2802,45 @@ define <2 x i64> @splatconstant_funnnel_v2i64(<2 x i64> %x, <2 x i64> %y) nounwi
 ; AVX-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    retq
 ;
-; AVX512-LABEL: splatconstant_funnnel_v2i64:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpsrlq $50, %xmm1, %xmm1
-; AVX512-NEXT:    vpsllq $14, %xmm0, %xmm0
-; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    retq
+; AVX512F-LABEL: splatconstant_funnnel_v2i64:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpsrlq $50, %xmm1, %xmm1
+; AVX512F-NEXT:    vpsllq $14, %xmm0, %xmm0
+; AVX512F-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: splatconstant_funnnel_v2i64:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpsrlq $50, %xmm1, %xmm1
+; AVX512VL-NEXT:    vpsllq $14, %xmm0, %xmm0
+; AVX512VL-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: splatconstant_funnnel_v2i64:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpsrlq $50, %xmm1, %xmm1
+; AVX512BW-NEXT:    vpsllq $14, %xmm0, %xmm0
+; AVX512BW-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT:    retq
+;
+; AVX512VBMI2-LABEL: splatconstant_funnnel_v2i64:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    vpsrlq $50, %xmm1, %xmm1
+; AVX512VBMI2-NEXT:    vpsllq $14, %xmm0, %xmm0
+; AVX512VBMI2-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLBW-LABEL: splatconstant_funnnel_v2i64:
+; AVX512VLBW:       # %bb.0:
+; AVX512VLBW-NEXT:    vpsrlq $50, %xmm1, %xmm1
+; AVX512VLBW-NEXT:    vpsllq $14, %xmm0, %xmm0
+; AVX512VLBW-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX512VLBW-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: splatconstant_funnnel_v2i64:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpshldq $14, %xmm1, %xmm0, %xmm0
+; AVX512VLVBMI2-NEXT:    retq
 ;
 ; XOP-LABEL: splatconstant_funnnel_v2i64:
 ; XOP:       # %bb.0:
@@ -2498,12 +2874,45 @@ define <4 x i32> @splatconstant_funnnel_v4i32(<4 x i32> %x, <4 x i32> %y) nounwi
 ; AVX-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    retq
 ;
-; AVX512-LABEL: splatconstant_funnnel_v4i32:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpsrld $28, %xmm1, %xmm1
-; AVX512-NEXT:    vpslld $4, %xmm0, %xmm0
-; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    retq
+; AVX512F-LABEL: splatconstant_funnnel_v4i32:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpsrld $28, %xmm1, %xmm1
+; AVX512F-NEXT:    vpslld $4, %xmm0, %xmm0
+; AVX512F-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: splatconstant_funnnel_v4i32:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpsrld $28, %xmm1, %xmm1
+; AVX512VL-NEXT:    vpslld $4, %xmm0, %xmm0
+; AVX512VL-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: splatconstant_funnnel_v4i32:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpsrld $28, %xmm1, %xmm1
+; AVX512BW-NEXT:    vpslld $4, %xmm0, %xmm0
+; AVX512BW-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT:    retq
+;
+; AVX512VBMI2-LABEL: splatconstant_funnnel_v4i32:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    vpsrld $28, %xmm1, %xmm1
+; AVX512VBMI2-NEXT:    vpslld $4, %xmm0, %xmm0
+; AVX512VBMI2-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLBW-LABEL: splatconstant_funnnel_v4i32:
+; AVX512VLBW:       # %bb.0:
+; AVX512VLBW-NEXT:    vpsrld $28, %xmm1, %xmm1
+; AVX512VLBW-NEXT:    vpslld $4, %xmm0, %xmm0
+; AVX512VLBW-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX512VLBW-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: splatconstant_funnnel_v4i32:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpshldd $4, %xmm1, %xmm0, %xmm0
+; AVX512VLVBMI2-NEXT:    retq
 ;
 ; XOP-LABEL: splatconstant_funnnel_v4i32:
 ; XOP:       # %bb.0:
@@ -2537,12 +2946,45 @@ define <8 x i16> @splatconstant_funnnel_v8i16(<8 x i16> %x, <8 x i16> %y) nounwi
 ; AVX-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    retq
 ;
-; AVX512-LABEL: splatconstant_funnnel_v8i16:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpsrlw $9, %xmm1, %xmm1
-; AVX512-NEXT:    vpsllw $7, %xmm0, %xmm0
-; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    retq
+; AVX512F-LABEL: splatconstant_funnnel_v8i16:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpsrlw $9, %xmm1, %xmm1
+; AVX512F-NEXT:    vpsllw $7, %xmm0, %xmm0
+; AVX512F-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: splatconstant_funnnel_v8i16:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpsrlw $9, %xmm1, %xmm1
+; AVX512VL-NEXT:    vpsllw $7, %xmm0, %xmm0
+; AVX512VL-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: splatconstant_funnnel_v8i16:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpsrlw $9, %xmm1, %xmm1
+; AVX512BW-NEXT:    vpsllw $7, %xmm0, %xmm0
+; AVX512BW-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT:    retq
+;
+; AVX512VBMI2-LABEL: splatconstant_funnnel_v8i16:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    vpsrlw $9, %xmm1, %xmm1
+; AVX512VBMI2-NEXT:    vpsllw $7, %xmm0, %xmm0
+; AVX512VBMI2-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLBW-LABEL: splatconstant_funnnel_v8i16:
+; AVX512VLBW:       # %bb.0:
+; AVX512VLBW-NEXT:    vpsrlw $9, %xmm1, %xmm1
+; AVX512VLBW-NEXT:    vpsllw $7, %xmm0, %xmm0
+; AVX512VLBW-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX512VLBW-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: splatconstant_funnnel_v8i16:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpshldw $7, %xmm1, %xmm0, %xmm0
+; AVX512VLVBMI2-NEXT:    retq
 ;
 ; XOP-LABEL: splatconstant_funnnel_v8i16:
 ; XOP:       # %bb.0:
diff --git a/test/CodeGen/X86/vector-fshl-256.ll b/test/CodeGen/X86/vector-fshl-256.ll
index aac4662cff0bf82630135ff00afa4fb54c2569c3..0133d9acca76e0bcead749c9ab9146ea30c42056 100644
--- a/test/CodeGen/X86/vector-fshl-256.ll
+++ b/test/CodeGen/X86/vector-fshl-256.ll
@@ -4,7 +4,9 @@
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f | FileCheck %s --check-prefixes=AVX512,AVX512F
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512vl | FileCheck %s --check-prefixes=AVX512,AVX512VL
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512bw | FileCheck %s --check-prefixes=AVX512,AVX512BW
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512vbmi2 | FileCheck %s --check-prefixes=AVX512,AVX512VBMI2
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512bw,+avx512vl | FileCheck %s --check-prefixes=AVX512,AVX512VLBW
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512vbmi2,+avx512vl | FileCheck %s --check-prefixes=AVX512,AVX512VLVBMI2
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+xop,+avx | FileCheck %s --check-prefixes=XOP,XOPAVX1
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+xop,+avx2 | FileCheck %s --check-prefixes=XOP,XOPAVX2
 
@@ -113,6 +115,22 @@ define <4 x i64> @var_funnnel_v4i64(<4 x i64> %x, <4 x i64> %y, <4 x i64> %amt)
 ; AVX512BW-NEXT:    vmovdqa %ymm1, %ymm0
 ; AVX512BW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: var_funnnel_v4i64:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $ymm2 killed $ymm2 def $zmm2
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512VBMI2-NEXT:    vpbroadcastq {{.*#+}} ymm3 = [63,63,63,63]
+; AVX512VBMI2-NEXT:    vpand %ymm3, %ymm2, %ymm4
+; AVX512VBMI2-NEXT:    vpsllvq %ymm4, %ymm0, %ymm5
+; AVX512VBMI2-NEXT:    vpbroadcastq {{.*#+}} ymm6 = [64,64,64,64]
+; AVX512VBMI2-NEXT:    vpsubq %ymm4, %ymm6, %ymm4
+; AVX512VBMI2-NEXT:    vpsrlvq %ymm4, %ymm1, %ymm1
+; AVX512VBMI2-NEXT:    vpor %ymm1, %ymm5, %ymm1
+; AVX512VBMI2-NEXT:    vptestnmq %zmm3, %zmm2, %k1
+; AVX512VBMI2-NEXT:    vmovdqa64 %zmm0, %zmm1 {%k1}
+; AVX512VBMI2-NEXT:    vmovdqa %ymm1, %ymm0
+; AVX512VBMI2-NEXT:    retq
+;
 ; AVX512VLBW-LABEL: var_funnnel_v4i64:
 ; AVX512VLBW:       # %bb.0:
 ; AVX512VLBW-NEXT:    vpbroadcastq {{.*#+}} ymm3 = [63,63,63,63]
@@ -127,6 +145,11 @@ define <4 x i64> @var_funnnel_v4i64(<4 x i64> %x, <4 x i64> %y, <4 x i64> %amt)
 ; AVX512VLBW-NEXT:    vmovdqa %ymm1, %ymm0
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VLVBMI2-LABEL: var_funnnel_v4i64:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpshldvq %ymm2, %ymm1, %ymm0
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOPAVX1-LABEL: var_funnnel_v4i64:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
@@ -278,6 +301,22 @@ define <8 x i32> @var_funnnel_v8i32(<8 x i32> %x, <8 x i32> %y, <8 x i32> %amt)
 ; AVX512BW-NEXT:    vmovdqa %ymm1, %ymm0
 ; AVX512BW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: var_funnnel_v8i32:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $ymm2 killed $ymm2 def $zmm2
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512VBMI2-NEXT:    vpbroadcastd {{.*#+}} ymm3 = [31,31,31,31,31,31,31,31]
+; AVX512VBMI2-NEXT:    vpand %ymm3, %ymm2, %ymm4
+; AVX512VBMI2-NEXT:    vpsllvd %ymm4, %ymm0, %ymm5
+; AVX512VBMI2-NEXT:    vpbroadcastd {{.*#+}} ymm6 = [32,32,32,32,32,32,32,32]
+; AVX512VBMI2-NEXT:    vpsubd %ymm4, %ymm6, %ymm4
+; AVX512VBMI2-NEXT:    vpsrlvd %ymm4, %ymm1, %ymm1
+; AVX512VBMI2-NEXT:    vpor %ymm1, %ymm5, %ymm1
+; AVX512VBMI2-NEXT:    vptestnmd %zmm3, %zmm2, %k1
+; AVX512VBMI2-NEXT:    vmovdqa32 %zmm0, %zmm1 {%k1}
+; AVX512VBMI2-NEXT:    vmovdqa %ymm1, %ymm0
+; AVX512VBMI2-NEXT:    retq
+;
 ; AVX512VLBW-LABEL: var_funnnel_v8i32:
 ; AVX512VLBW:       # %bb.0:
 ; AVX512VLBW-NEXT:    vpbroadcastd {{.*#+}} ymm3 = [31,31,31,31,31,31,31,31]
@@ -292,6 +331,11 @@ define <8 x i32> @var_funnnel_v8i32(<8 x i32> %x, <8 x i32> %y, <8 x i32> %amt)
 ; AVX512VLBW-NEXT:    vmovdqa %ymm1, %ymm0
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VLVBMI2-LABEL: var_funnnel_v8i32:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpshldvd %ymm2, %ymm1, %ymm0
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOPAVX1-LABEL: var_funnnel_v8i32:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
@@ -485,6 +529,23 @@ define <16 x i16> @var_funnnel_v16i16(<16 x i16> %x, <16 x i16> %y, <16 x i16> %
 ; AVX512BW-NEXT:    vmovdqa %ymm1, %ymm0
 ; AVX512BW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: var_funnnel_v16i16:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $ymm2 killed $ymm2 def $zmm2
+; AVX512VBMI2-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} ymm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; AVX512VBMI2-NEXT:    vpand %ymm3, %ymm2, %ymm4
+; AVX512VBMI2-NEXT:    vpsllvw %zmm4, %zmm0, %zmm5
+; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} ymm6 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
+; AVX512VBMI2-NEXT:    vpsubw %ymm4, %ymm6, %ymm4
+; AVX512VBMI2-NEXT:    vpsrlvw %zmm4, %zmm1, %zmm1
+; AVX512VBMI2-NEXT:    vpor %ymm1, %ymm5, %ymm1
+; AVX512VBMI2-NEXT:    vptestnmw %zmm3, %zmm2, %k1
+; AVX512VBMI2-NEXT:    vmovdqu16 %zmm0, %zmm1 {%k1}
+; AVX512VBMI2-NEXT:    vmovdqa %ymm1, %ymm0
+; AVX512VBMI2-NEXT:    retq
+;
 ; AVX512VLBW-LABEL: var_funnnel_v16i16:
 ; AVX512VLBW:       # %bb.0:
 ; AVX512VLBW-NEXT:    vmovdqa {{.*#+}} ymm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
@@ -499,6 +560,11 @@ define <16 x i16> @var_funnnel_v16i16(<16 x i16> %x, <16 x i16> %y, <16 x i16> %
 ; AVX512VLBW-NEXT:    vmovdqa %ymm1, %ymm0
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VLVBMI2-LABEL: var_funnnel_v16i16:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpshldvw %ymm2, %ymm1, %ymm0
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOPAVX1-LABEL: var_funnnel_v16i16:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
@@ -745,6 +811,27 @@ define <32 x i8> @var_funnnel_v32i8(<32 x i8> %x, <32 x i8> %y, <32 x i8> %amt)
 ; AVX512BW-NEXT:    vmovdqa %ymm1, %ymm0
 ; AVX512BW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: var_funnnel_v32i8:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $ymm2 killed $ymm2 def $zmm2
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} ymm3 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
+; AVX512VBMI2-NEXT:    vpand %ymm3, %ymm2, %ymm4
+; AVX512VBMI2-NEXT:    vpmovzxbw {{.*#+}} zmm5 = ymm4[0],zero,ymm4[1],zero,ymm4[2],zero,ymm4[3],zero,ymm4[4],zero,ymm4[5],zero,ymm4[6],zero,ymm4[7],zero,ymm4[8],zero,ymm4[9],zero,ymm4[10],zero,ymm4[11],zero,ymm4[12],zero,ymm4[13],zero,ymm4[14],zero,ymm4[15],zero,ymm4[16],zero,ymm4[17],zero,ymm4[18],zero,ymm4[19],zero,ymm4[20],zero,ymm4[21],zero,ymm4[22],zero,ymm4[23],zero,ymm4[24],zero,ymm4[25],zero,ymm4[26],zero,ymm4[27],zero,ymm4[28],zero,ymm4[29],zero,ymm4[30],zero,ymm4[31],zero
+; AVX512VBMI2-NEXT:    vpmovzxbw {{.*#+}} zmm6 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero,ymm0[16],zero,ymm0[17],zero,ymm0[18],zero,ymm0[19],zero,ymm0[20],zero,ymm0[21],zero,ymm0[22],zero,ymm0[23],zero,ymm0[24],zero,ymm0[25],zero,ymm0[26],zero,ymm0[27],zero,ymm0[28],zero,ymm0[29],zero,ymm0[30],zero,ymm0[31],zero
+; AVX512VBMI2-NEXT:    vpsllvw %zmm5, %zmm6, %zmm5
+; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} ymm6 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; AVX512VBMI2-NEXT:    vpsubb %ymm4, %ymm6, %ymm4
+; AVX512VBMI2-NEXT:    vpmovzxbw {{.*#+}} zmm4 = ymm4[0],zero,ymm4[1],zero,ymm4[2],zero,ymm4[3],zero,ymm4[4],zero,ymm4[5],zero,ymm4[6],zero,ymm4[7],zero,ymm4[8],zero,ymm4[9],zero,ymm4[10],zero,ymm4[11],zero,ymm4[12],zero,ymm4[13],zero,ymm4[14],zero,ymm4[15],zero,ymm4[16],zero,ymm4[17],zero,ymm4[18],zero,ymm4[19],zero,ymm4[20],zero,ymm4[21],zero,ymm4[22],zero,ymm4[23],zero,ymm4[24],zero,ymm4[25],zero,ymm4[26],zero,ymm4[27],zero,ymm4[28],zero,ymm4[29],zero,ymm4[30],zero,ymm4[31],zero
+; AVX512VBMI2-NEXT:    vpmovzxbw {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero
+; AVX512VBMI2-NEXT:    vpsrlvw %zmm4, %zmm1, %zmm1
+; AVX512VBMI2-NEXT:    vporq %zmm1, %zmm5, %zmm1
+; AVX512VBMI2-NEXT:    vpmovwb %zmm1, %ymm1
+; AVX512VBMI2-NEXT:    vptestnmb %zmm3, %zmm2, %k1
+; AVX512VBMI2-NEXT:    vmovdqu8 %zmm0, %zmm1 {%k1}
+; AVX512VBMI2-NEXT:    vmovdqa %ymm1, %ymm0
+; AVX512VBMI2-NEXT:    retq
+;
 ; AVX512VLBW-LABEL: var_funnnel_v32i8:
 ; AVX512VLBW:       # %bb.0:
 ; AVX512VLBW-NEXT:    vmovdqa {{.*#+}} ymm3 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
@@ -764,6 +851,25 @@ define <32 x i8> @var_funnnel_v32i8(<32 x i8> %x, <32 x i8> %y, <32 x i8> %amt)
 ; AVX512VLBW-NEXT:    vmovdqa %ymm1, %ymm0
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VLVBMI2-LABEL: var_funnnel_v32i8:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vmovdqa {{.*#+}} ymm3 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
+; AVX512VLVBMI2-NEXT:    vpand %ymm3, %ymm2, %ymm4
+; AVX512VLVBMI2-NEXT:    vpmovzxbw {{.*#+}} zmm5 = ymm4[0],zero,ymm4[1],zero,ymm4[2],zero,ymm4[3],zero,ymm4[4],zero,ymm4[5],zero,ymm4[6],zero,ymm4[7],zero,ymm4[8],zero,ymm4[9],zero,ymm4[10],zero,ymm4[11],zero,ymm4[12],zero,ymm4[13],zero,ymm4[14],zero,ymm4[15],zero,ymm4[16],zero,ymm4[17],zero,ymm4[18],zero,ymm4[19],zero,ymm4[20],zero,ymm4[21],zero,ymm4[22],zero,ymm4[23],zero,ymm4[24],zero,ymm4[25],zero,ymm4[26],zero,ymm4[27],zero,ymm4[28],zero,ymm4[29],zero,ymm4[30],zero,ymm4[31],zero
+; AVX512VLVBMI2-NEXT:    vpmovzxbw {{.*#+}} zmm6 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero,ymm0[16],zero,ymm0[17],zero,ymm0[18],zero,ymm0[19],zero,ymm0[20],zero,ymm0[21],zero,ymm0[22],zero,ymm0[23],zero,ymm0[24],zero,ymm0[25],zero,ymm0[26],zero,ymm0[27],zero,ymm0[28],zero,ymm0[29],zero,ymm0[30],zero,ymm0[31],zero
+; AVX512VLVBMI2-NEXT:    vpsllvw %zmm5, %zmm6, %zmm5
+; AVX512VLVBMI2-NEXT:    vmovdqa {{.*#+}} ymm6 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; AVX512VLVBMI2-NEXT:    vpsubb %ymm4, %ymm6, %ymm4
+; AVX512VLVBMI2-NEXT:    vpmovzxbw {{.*#+}} zmm4 = ymm4[0],zero,ymm4[1],zero,ymm4[2],zero,ymm4[3],zero,ymm4[4],zero,ymm4[5],zero,ymm4[6],zero,ymm4[7],zero,ymm4[8],zero,ymm4[9],zero,ymm4[10],zero,ymm4[11],zero,ymm4[12],zero,ymm4[13],zero,ymm4[14],zero,ymm4[15],zero,ymm4[16],zero,ymm4[17],zero,ymm4[18],zero,ymm4[19],zero,ymm4[20],zero,ymm4[21],zero,ymm4[22],zero,ymm4[23],zero,ymm4[24],zero,ymm4[25],zero,ymm4[26],zero,ymm4[27],zero,ymm4[28],zero,ymm4[29],zero,ymm4[30],zero,ymm4[31],zero
+; AVX512VLVBMI2-NEXT:    vpmovzxbw {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero
+; AVX512VLVBMI2-NEXT:    vpsrlvw %zmm4, %zmm1, %zmm1
+; AVX512VLVBMI2-NEXT:    vporq %zmm1, %zmm5, %zmm1
+; AVX512VLVBMI2-NEXT:    vpmovwb %zmm1, %ymm1
+; AVX512VLVBMI2-NEXT:    vptestnmb %ymm3, %ymm2, %k1
+; AVX512VLVBMI2-NEXT:    vmovdqu8 %ymm0, %ymm1 {%k1}
+; AVX512VLVBMI2-NEXT:    vmovdqa %ymm1, %ymm0
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOPAVX1-LABEL: var_funnnel_v32i8:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
@@ -906,6 +1012,22 @@ define <4 x i64> @splatvar_funnnel_v4i64(<4 x i64> %x, <4 x i64> %y, <4 x i64> %
 ; AVX512BW-NEXT:    vmovdqa %ymm1, %ymm0
 ; AVX512BW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: splatvar_funnnel_v4i64:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512VBMI2-NEXT:    vpbroadcastq %xmm2, %ymm2
+; AVX512VBMI2-NEXT:    vpbroadcastq {{.*#+}} ymm3 = [63,63,63,63]
+; AVX512VBMI2-NEXT:    vpand %ymm3, %ymm2, %ymm4
+; AVX512VBMI2-NEXT:    vpsllq %xmm4, %ymm0, %ymm5
+; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} xmm6 = [64,64]
+; AVX512VBMI2-NEXT:    vpsubq %xmm4, %xmm6, %xmm4
+; AVX512VBMI2-NEXT:    vpsrlq %xmm4, %ymm1, %ymm1
+; AVX512VBMI2-NEXT:    vpor %ymm1, %ymm5, %ymm1
+; AVX512VBMI2-NEXT:    vptestnmq %zmm3, %zmm2, %k1
+; AVX512VBMI2-NEXT:    vmovdqa64 %zmm0, %zmm1 {%k1}
+; AVX512VBMI2-NEXT:    vmovdqa %ymm1, %ymm0
+; AVX512VBMI2-NEXT:    retq
+;
 ; AVX512VLBW-LABEL: splatvar_funnnel_v4i64:
 ; AVX512VLBW:       # %bb.0:
 ; AVX512VLBW-NEXT:    vpbroadcastq %xmm2, %ymm2
@@ -921,6 +1043,12 @@ define <4 x i64> @splatvar_funnnel_v4i64(<4 x i64> %x, <4 x i64> %y, <4 x i64> %
 ; AVX512VLBW-NEXT:    vmovdqa %ymm1, %ymm0
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VLVBMI2-LABEL: splatvar_funnnel_v4i64:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpbroadcastq %xmm2, %ymm2
+; AVX512VLVBMI2-NEXT:    vpshldvq %ymm2, %ymm1, %ymm0
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOPAVX1-LABEL: splatvar_funnnel_v4i64:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vmovddup {{.*#+}} xmm2 = xmm2[0,0]
@@ -1064,6 +1192,24 @@ define <8 x i32> @splatvar_funnnel_v8i32(<8 x i32> %x, <8 x i32> %y, <8 x i32> %
 ; AVX512BW-NEXT:    vmovdqa %ymm1, %ymm0
 ; AVX512BW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: splatvar_funnnel_v8i32:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512VBMI2-NEXT:    vpbroadcastd %xmm2, %ymm2
+; AVX512VBMI2-NEXT:    vpbroadcastd {{.*#+}} ymm3 = [31,31,31,31,31,31,31,31]
+; AVX512VBMI2-NEXT:    vpand %ymm3, %ymm2, %ymm4
+; AVX512VBMI2-NEXT:    vpmovzxdq {{.*#+}} xmm5 = xmm4[0],zero,xmm4[1],zero
+; AVX512VBMI2-NEXT:    vpslld %xmm5, %ymm0, %ymm5
+; AVX512VBMI2-NEXT:    vpbroadcastd {{.*#+}} xmm6 = [32,32,32,32]
+; AVX512VBMI2-NEXT:    vpsubd %xmm4, %xmm6, %xmm4
+; AVX512VBMI2-NEXT:    vpmovzxdq {{.*#+}} xmm4 = xmm4[0],zero,xmm4[1],zero
+; AVX512VBMI2-NEXT:    vpsrld %xmm4, %ymm1, %ymm1
+; AVX512VBMI2-NEXT:    vpor %ymm1, %ymm5, %ymm1
+; AVX512VBMI2-NEXT:    vptestnmd %zmm3, %zmm2, %k1
+; AVX512VBMI2-NEXT:    vmovdqa32 %zmm0, %zmm1 {%k1}
+; AVX512VBMI2-NEXT:    vmovdqa %ymm1, %ymm0
+; AVX512VBMI2-NEXT:    retq
+;
 ; AVX512VLBW-LABEL: splatvar_funnnel_v8i32:
 ; AVX512VLBW:       # %bb.0:
 ; AVX512VLBW-NEXT:    vpbroadcastd %xmm2, %ymm2
@@ -1081,6 +1227,12 @@ define <8 x i32> @splatvar_funnnel_v8i32(<8 x i32> %x, <8 x i32> %y, <8 x i32> %
 ; AVX512VLBW-NEXT:    vmovdqa %ymm1, %ymm0
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VLVBMI2-LABEL: splatvar_funnnel_v8i32:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpbroadcastd %xmm2, %ymm2
+; AVX512VLVBMI2-NEXT:    vpshldvd %ymm2, %ymm1, %ymm0
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOPAVX1-LABEL: splatvar_funnnel_v8i32:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vpermilps {{.*#+}} xmm2 = xmm2[0,0,0,0]
@@ -1228,6 +1380,24 @@ define <16 x i16> @splatvar_funnnel_v16i16(<16 x i16> %x, <16 x i16> %y, <16 x i
 ; AVX512BW-NEXT:    vmovdqa %ymm1, %ymm0
 ; AVX512BW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: splatvar_funnnel_v16i16:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512VBMI2-NEXT:    vpbroadcastw %xmm2, %ymm2
+; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} ymm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; AVX512VBMI2-NEXT:    vpand %ymm3, %ymm2, %ymm4
+; AVX512VBMI2-NEXT:    vpmovzxwq {{.*#+}} xmm5 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero
+; AVX512VBMI2-NEXT:    vpsllw %xmm5, %ymm0, %ymm5
+; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} xmm6 = [16,16,16,16,16,16,16,16]
+; AVX512VBMI2-NEXT:    vpsubw %xmm4, %xmm6, %xmm4
+; AVX512VBMI2-NEXT:    vpmovzxwq {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero
+; AVX512VBMI2-NEXT:    vpsrlw %xmm4, %ymm1, %ymm1
+; AVX512VBMI2-NEXT:    vpor %ymm1, %ymm5, %ymm1
+; AVX512VBMI2-NEXT:    vptestnmw %zmm3, %zmm2, %k1
+; AVX512VBMI2-NEXT:    vmovdqu16 %zmm0, %zmm1 {%k1}
+; AVX512VBMI2-NEXT:    vmovdqa %ymm1, %ymm0
+; AVX512VBMI2-NEXT:    retq
+;
 ; AVX512VLBW-LABEL: splatvar_funnnel_v16i16:
 ; AVX512VLBW:       # %bb.0:
 ; AVX512VLBW-NEXT:    vpbroadcastw %xmm2, %ymm2
@@ -1245,6 +1415,12 @@ define <16 x i16> @splatvar_funnnel_v16i16(<16 x i16> %x, <16 x i16> %y, <16 x i
 ; AVX512VLBW-NEXT:    vmovdqa %ymm1, %ymm0
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VLVBMI2-LABEL: splatvar_funnnel_v16i16:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpbroadcastw %xmm2, %ymm2
+; AVX512VLVBMI2-NEXT:    vpshldvw %ymm2, %ymm1, %ymm0
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOPAVX1-LABEL: splatvar_funnnel_v16i16:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm2 = xmm2[0,0,2,3,4,5,6,7]
@@ -1430,6 +1606,27 @@ define <32 x i8> @splatvar_funnnel_v32i8(<32 x i8> %x, <32 x i8> %y, <32 x i8> %
 ; AVX512BW-NEXT:    vmovdqa %ymm1, %ymm0
 ; AVX512BW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: splatvar_funnnel_v32i8:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512VBMI2-NEXT:    vpbroadcastb %xmm2, %ymm2
+; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} ymm3 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
+; AVX512VBMI2-NEXT:    vpand %ymm3, %ymm2, %ymm4
+; AVX512VBMI2-NEXT:    vpmovzxbw {{.*#+}} zmm5 = ymm4[0],zero,ymm4[1],zero,ymm4[2],zero,ymm4[3],zero,ymm4[4],zero,ymm4[5],zero,ymm4[6],zero,ymm4[7],zero,ymm4[8],zero,ymm4[9],zero,ymm4[10],zero,ymm4[11],zero,ymm4[12],zero,ymm4[13],zero,ymm4[14],zero,ymm4[15],zero,ymm4[16],zero,ymm4[17],zero,ymm4[18],zero,ymm4[19],zero,ymm4[20],zero,ymm4[21],zero,ymm4[22],zero,ymm4[23],zero,ymm4[24],zero,ymm4[25],zero,ymm4[26],zero,ymm4[27],zero,ymm4[28],zero,ymm4[29],zero,ymm4[30],zero,ymm4[31],zero
+; AVX512VBMI2-NEXT:    vpmovzxbw {{.*#+}} zmm6 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero,ymm0[16],zero,ymm0[17],zero,ymm0[18],zero,ymm0[19],zero,ymm0[20],zero,ymm0[21],zero,ymm0[22],zero,ymm0[23],zero,ymm0[24],zero,ymm0[25],zero,ymm0[26],zero,ymm0[27],zero,ymm0[28],zero,ymm0[29],zero,ymm0[30],zero,ymm0[31],zero
+; AVX512VBMI2-NEXT:    vpsllvw %zmm5, %zmm6, %zmm5
+; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} ymm6 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; AVX512VBMI2-NEXT:    vpsubb %ymm4, %ymm6, %ymm4
+; AVX512VBMI2-NEXT:    vpmovzxbw {{.*#+}} zmm4 = ymm4[0],zero,ymm4[1],zero,ymm4[2],zero,ymm4[3],zero,ymm4[4],zero,ymm4[5],zero,ymm4[6],zero,ymm4[7],zero,ymm4[8],zero,ymm4[9],zero,ymm4[10],zero,ymm4[11],zero,ymm4[12],zero,ymm4[13],zero,ymm4[14],zero,ymm4[15],zero,ymm4[16],zero,ymm4[17],zero,ymm4[18],zero,ymm4[19],zero,ymm4[20],zero,ymm4[21],zero,ymm4[22],zero,ymm4[23],zero,ymm4[24],zero,ymm4[25],zero,ymm4[26],zero,ymm4[27],zero,ymm4[28],zero,ymm4[29],zero,ymm4[30],zero,ymm4[31],zero
+; AVX512VBMI2-NEXT:    vpmovzxbw {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero
+; AVX512VBMI2-NEXT:    vpsrlvw %zmm4, %zmm1, %zmm1
+; AVX512VBMI2-NEXT:    vporq %zmm1, %zmm5, %zmm1
+; AVX512VBMI2-NEXT:    vpmovwb %zmm1, %ymm1
+; AVX512VBMI2-NEXT:    vptestnmb %zmm3, %zmm2, %k1
+; AVX512VBMI2-NEXT:    vmovdqu8 %zmm0, %zmm1 {%k1}
+; AVX512VBMI2-NEXT:    vmovdqa %ymm1, %ymm0
+; AVX512VBMI2-NEXT:    retq
+;
 ; AVX512VLBW-LABEL: splatvar_funnnel_v32i8:
 ; AVX512VLBW:       # %bb.0:
 ; AVX512VLBW-NEXT:    vpbroadcastb %xmm2, %ymm2
@@ -1450,6 +1647,26 @@ define <32 x i8> @splatvar_funnnel_v32i8(<32 x i8> %x, <32 x i8> %y, <32 x i8> %
 ; AVX512VLBW-NEXT:    vmovdqa %ymm1, %ymm0
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VLVBMI2-LABEL: splatvar_funnnel_v32i8:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpbroadcastb %xmm2, %ymm2
+; AVX512VLVBMI2-NEXT:    vmovdqa {{.*#+}} ymm3 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
+; AVX512VLVBMI2-NEXT:    vpand %ymm3, %ymm2, %ymm4
+; AVX512VLVBMI2-NEXT:    vpmovzxbw {{.*#+}} zmm5 = ymm4[0],zero,ymm4[1],zero,ymm4[2],zero,ymm4[3],zero,ymm4[4],zero,ymm4[5],zero,ymm4[6],zero,ymm4[7],zero,ymm4[8],zero,ymm4[9],zero,ymm4[10],zero,ymm4[11],zero,ymm4[12],zero,ymm4[13],zero,ymm4[14],zero,ymm4[15],zero,ymm4[16],zero,ymm4[17],zero,ymm4[18],zero,ymm4[19],zero,ymm4[20],zero,ymm4[21],zero,ymm4[22],zero,ymm4[23],zero,ymm4[24],zero,ymm4[25],zero,ymm4[26],zero,ymm4[27],zero,ymm4[28],zero,ymm4[29],zero,ymm4[30],zero,ymm4[31],zero
+; AVX512VLVBMI2-NEXT:    vpmovzxbw {{.*#+}} zmm6 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero,ymm0[16],zero,ymm0[17],zero,ymm0[18],zero,ymm0[19],zero,ymm0[20],zero,ymm0[21],zero,ymm0[22],zero,ymm0[23],zero,ymm0[24],zero,ymm0[25],zero,ymm0[26],zero,ymm0[27],zero,ymm0[28],zero,ymm0[29],zero,ymm0[30],zero,ymm0[31],zero
+; AVX512VLVBMI2-NEXT:    vpsllvw %zmm5, %zmm6, %zmm5
+; AVX512VLVBMI2-NEXT:    vmovdqa {{.*#+}} ymm6 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; AVX512VLVBMI2-NEXT:    vpsubb %ymm4, %ymm6, %ymm4
+; AVX512VLVBMI2-NEXT:    vpmovzxbw {{.*#+}} zmm4 = ymm4[0],zero,ymm4[1],zero,ymm4[2],zero,ymm4[3],zero,ymm4[4],zero,ymm4[5],zero,ymm4[6],zero,ymm4[7],zero,ymm4[8],zero,ymm4[9],zero,ymm4[10],zero,ymm4[11],zero,ymm4[12],zero,ymm4[13],zero,ymm4[14],zero,ymm4[15],zero,ymm4[16],zero,ymm4[17],zero,ymm4[18],zero,ymm4[19],zero,ymm4[20],zero,ymm4[21],zero,ymm4[22],zero,ymm4[23],zero,ymm4[24],zero,ymm4[25],zero,ymm4[26],zero,ymm4[27],zero,ymm4[28],zero,ymm4[29],zero,ymm4[30],zero,ymm4[31],zero
+; AVX512VLVBMI2-NEXT:    vpmovzxbw {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero
+; AVX512VLVBMI2-NEXT:    vpsrlvw %zmm4, %zmm1, %zmm1
+; AVX512VLVBMI2-NEXT:    vporq %zmm1, %zmm5, %zmm1
+; AVX512VLVBMI2-NEXT:    vpmovwb %zmm1, %ymm1
+; AVX512VLVBMI2-NEXT:    vptestnmb %ymm3, %ymm2, %k1
+; AVX512VLVBMI2-NEXT:    vmovdqu8 %ymm0, %ymm1 {%k1}
+; AVX512VLVBMI2-NEXT:    vmovdqa %ymm1, %ymm0
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOPAVX1-LABEL: splatvar_funnnel_v32i8:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vpxor %xmm8, %xmm8, %xmm8
@@ -1537,12 +1754,45 @@ define <4 x i64> @constant_funnnel_v4i64(<4 x i64> %x, <4 x i64> %y) nounwind {
 ; AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    retq
 ;
-; AVX512-LABEL: constant_funnnel_v4i64:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpsrlvq {{.*}}(%rip), %ymm1, %ymm1
-; AVX512-NEXT:    vpsllvq {{.*}}(%rip), %ymm0, %ymm0
-; AVX512-NEXT:    vpor %ymm1, %ymm0, %ymm0
-; AVX512-NEXT:    retq
+; AVX512F-LABEL: constant_funnnel_v4i64:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpsrlvq {{.*}}(%rip), %ymm1, %ymm1
+; AVX512F-NEXT:    vpsllvq {{.*}}(%rip), %ymm0, %ymm0
+; AVX512F-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: constant_funnnel_v4i64:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpsrlvq {{.*}}(%rip), %ymm1, %ymm1
+; AVX512VL-NEXT:    vpsllvq {{.*}}(%rip), %ymm0, %ymm0
+; AVX512VL-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: constant_funnnel_v4i64:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpsrlvq {{.*}}(%rip), %ymm1, %ymm1
+; AVX512BW-NEXT:    vpsllvq {{.*}}(%rip), %ymm0, %ymm0
+; AVX512BW-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512BW-NEXT:    retq
+;
+; AVX512VBMI2-LABEL: constant_funnnel_v4i64:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    vpsrlvq {{.*}}(%rip), %ymm1, %ymm1
+; AVX512VBMI2-NEXT:    vpsllvq {{.*}}(%rip), %ymm0, %ymm0
+; AVX512VBMI2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLBW-LABEL: constant_funnnel_v4i64:
+; AVX512VLBW:       # %bb.0:
+; AVX512VLBW-NEXT:    vpsrlvq {{.*}}(%rip), %ymm1, %ymm1
+; AVX512VLBW-NEXT:    vpsllvq {{.*}}(%rip), %ymm0, %ymm0
+; AVX512VLBW-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512VLBW-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: constant_funnnel_v4i64:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpshldvq {{.*}}(%rip), %ymm1, %ymm0
+; AVX512VLVBMI2-NEXT:    retq
 ;
 ; XOPAVX1-LABEL: constant_funnnel_v4i64:
 ; XOPAVX1:       # %bb.0:
@@ -1600,12 +1850,45 @@ define <8 x i32> @constant_funnnel_v8i32(<8 x i32> %x, <8 x i32> %y) nounwind {
 ; AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    retq
 ;
-; AVX512-LABEL: constant_funnnel_v8i32:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpsrlvd {{.*}}(%rip), %ymm1, %ymm1
-; AVX512-NEXT:    vpsllvd {{.*}}(%rip), %ymm0, %ymm0
-; AVX512-NEXT:    vpor %ymm1, %ymm0, %ymm0
-; AVX512-NEXT:    retq
+; AVX512F-LABEL: constant_funnnel_v8i32:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpsrlvd {{.*}}(%rip), %ymm1, %ymm1
+; AVX512F-NEXT:    vpsllvd {{.*}}(%rip), %ymm0, %ymm0
+; AVX512F-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: constant_funnnel_v8i32:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpsrlvd {{.*}}(%rip), %ymm1, %ymm1
+; AVX512VL-NEXT:    vpsllvd {{.*}}(%rip), %ymm0, %ymm0
+; AVX512VL-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: constant_funnnel_v8i32:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpsrlvd {{.*}}(%rip), %ymm1, %ymm1
+; AVX512BW-NEXT:    vpsllvd {{.*}}(%rip), %ymm0, %ymm0
+; AVX512BW-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512BW-NEXT:    retq
+;
+; AVX512VBMI2-LABEL: constant_funnnel_v8i32:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    vpsrlvd {{.*}}(%rip), %ymm1, %ymm1
+; AVX512VBMI2-NEXT:    vpsllvd {{.*}}(%rip), %ymm0, %ymm0
+; AVX512VBMI2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLBW-LABEL: constant_funnnel_v8i32:
+; AVX512VLBW:       # %bb.0:
+; AVX512VLBW-NEXT:    vpsrlvd {{.*}}(%rip), %ymm1, %ymm1
+; AVX512VLBW-NEXT:    vpsllvd {{.*}}(%rip), %ymm0, %ymm0
+; AVX512VLBW-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512VLBW-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: constant_funnnel_v8i32:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpshldvd {{.*}}(%rip), %ymm1, %ymm0
+; AVX512VLVBMI2-NEXT:    retq
 ;
 ; XOPAVX1-LABEL: constant_funnnel_v8i32:
 ; XOPAVX1:       # %bb.0:
@@ -1693,6 +1976,19 @@ define <16 x i16> @constant_funnnel_v16i16(<16 x i16> %x, <16 x i16> %y) nounwin
 ; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
 ; AVX512BW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: constant_funnnel_v16i16:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} ymm2 = [16,15,14,13,12,11,10,9,8,7,6,5,4,3,2,1]
+; AVX512VBMI2-NEXT:    vpsrlvw %zmm2, %zmm1, %zmm1
+; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} ymm2 = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15]
+; AVX512VBMI2-NEXT:    vpsllvw %zmm2, %zmm0, %zmm2
+; AVX512VBMI2-NEXT:    vpor %ymm1, %ymm2, %ymm1
+; AVX512VBMI2-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0],ymm1[1,2,3,4,5,6,7],ymm0[8],ymm1[9,10,11,12,13,14,15]
+; AVX512VBMI2-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX512VBMI2-NEXT:    retq
+;
 ; AVX512VLBW-LABEL: constant_funnnel_v16i16:
 ; AVX512VLBW:       # %bb.0:
 ; AVX512VLBW-NEXT:    vpsrlvw {{.*}}(%rip), %ymm1, %ymm1
@@ -1702,6 +1998,11 @@ define <16 x i16> @constant_funnnel_v16i16(<16 x i16> %x, <16 x i16> %y) nounwin
 ; AVX512VLBW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VLVBMI2-LABEL: constant_funnnel_v16i16:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpshldvw {{.*}}(%rip), %ymm1, %ymm0
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOPAVX1-LABEL: constant_funnnel_v16i16:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vpshlw {{.*}}(%rip), %xmm1, %xmm2
@@ -1866,6 +2167,18 @@ define <32 x i8> @constant_funnnel_v32i8(<32 x i8> %x, <32 x i8> %y) nounwind {
 ; AVX512BW-NEXT:    vpblendvb %ymm2, %ymm1, %ymm0, %ymm0
 ; AVX512BW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: constant_funnnel_v32i8:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    vpmovzxbw {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero
+; AVX512VBMI2-NEXT:    vpsrlvw {{.*}}(%rip), %zmm1, %zmm1
+; AVX512VBMI2-NEXT:    vpmovzxbw {{.*#+}} zmm2 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero,ymm0[16],zero,ymm0[17],zero,ymm0[18],zero,ymm0[19],zero,ymm0[20],zero,ymm0[21],zero,ymm0[22],zero,ymm0[23],zero,ymm0[24],zero,ymm0[25],zero,ymm0[26],zero,ymm0[27],zero,ymm0[28],zero,ymm0[29],zero,ymm0[30],zero,ymm0[31],zero
+; AVX512VBMI2-NEXT:    vpsllvw {{.*}}(%rip), %zmm2, %zmm2
+; AVX512VBMI2-NEXT:    vporq %zmm1, %zmm2, %zmm1
+; AVX512VBMI2-NEXT:    vpmovwb %zmm1, %ymm1
+; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} ymm2 = [0,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255]
+; AVX512VBMI2-NEXT:    vpblendvb %ymm2, %ymm1, %ymm0, %ymm0
+; AVX512VBMI2-NEXT:    retq
+;
 ; AVX512VLBW-LABEL: constant_funnnel_v32i8:
 ; AVX512VLBW:       # %bb.0:
 ; AVX512VLBW-NEXT:    vpmovzxbw {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero
@@ -1880,6 +2193,20 @@ define <32 x i8> @constant_funnnel_v32i8(<32 x i8> %x, <32 x i8> %y) nounwind {
 ; AVX512VLBW-NEXT:    vmovdqa %ymm1, %ymm0
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VLVBMI2-LABEL: constant_funnnel_v32i8:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpmovzxbw {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero
+; AVX512VLVBMI2-NEXT:    vpsrlvw {{.*}}(%rip), %zmm1, %zmm1
+; AVX512VLVBMI2-NEXT:    vpmovzxbw {{.*#+}} zmm2 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero,ymm0[16],zero,ymm0[17],zero,ymm0[18],zero,ymm0[19],zero,ymm0[20],zero,ymm0[21],zero,ymm0[22],zero,ymm0[23],zero,ymm0[24],zero,ymm0[25],zero,ymm0[26],zero,ymm0[27],zero,ymm0[28],zero,ymm0[29],zero,ymm0[30],zero,ymm0[31],zero
+; AVX512VLVBMI2-NEXT:    vpsllvw {{.*}}(%rip), %zmm2, %zmm2
+; AVX512VLVBMI2-NEXT:    vporq %zmm1, %zmm2, %zmm1
+; AVX512VLVBMI2-NEXT:    vpmovwb %zmm1, %ymm1
+; AVX512VLVBMI2-NEXT:    movl $16843009, %eax # imm = 0x1010101
+; AVX512VLVBMI2-NEXT:    kmovd %eax, %k1
+; AVX512VLVBMI2-NEXT:    vmovdqu8 %ymm0, %ymm1 {%k1}
+; AVX512VLVBMI2-NEXT:    vmovdqa %ymm1, %ymm0
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOPAVX1-LABEL: constant_funnnel_v32i8:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
@@ -1942,12 +2269,45 @@ define <4 x i64> @splatconstant_funnnel_v4i64(<4 x i64> %x, <4 x i64> %y) nounwi
 ; AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    retq
 ;
-; AVX512-LABEL: splatconstant_funnnel_v4i64:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpsrlq $50, %ymm1, %ymm1
-; AVX512-NEXT:    vpsllq $14, %ymm0, %ymm0
-; AVX512-NEXT:    vpor %ymm1, %ymm0, %ymm0
-; AVX512-NEXT:    retq
+; AVX512F-LABEL: splatconstant_funnnel_v4i64:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpsrlq $50, %ymm1, %ymm1
+; AVX512F-NEXT:    vpsllq $14, %ymm0, %ymm0
+; AVX512F-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: splatconstant_funnnel_v4i64:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpsrlq $50, %ymm1, %ymm1
+; AVX512VL-NEXT:    vpsllq $14, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: splatconstant_funnnel_v4i64:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpsrlq $50, %ymm1, %ymm1
+; AVX512BW-NEXT:    vpsllq $14, %ymm0, %ymm0
+; AVX512BW-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512BW-NEXT:    retq
+;
+; AVX512VBMI2-LABEL: splatconstant_funnnel_v4i64:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    vpsrlq $50, %ymm1, %ymm1
+; AVX512VBMI2-NEXT:    vpsllq $14, %ymm0, %ymm0
+; AVX512VBMI2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLBW-LABEL: splatconstant_funnnel_v4i64:
+; AVX512VLBW:       # %bb.0:
+; AVX512VLBW-NEXT:    vpsrlq $50, %ymm1, %ymm1
+; AVX512VLBW-NEXT:    vpsllq $14, %ymm0, %ymm0
+; AVX512VLBW-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512VLBW-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: splatconstant_funnnel_v4i64:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpshldq $14, %ymm1, %ymm0, %ymm0
+; AVX512VLVBMI2-NEXT:    retq
 ;
 ; XOPAVX1-LABEL: splatconstant_funnnel_v4i64:
 ; XOPAVX1:       # %bb.0:
@@ -1993,12 +2353,45 @@ define <8 x i32> @splatconstant_funnnel_v8i32(<8 x i32> %x, <8 x i32> %y) nounwi
 ; AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    retq
 ;
-; AVX512-LABEL: splatconstant_funnnel_v8i32:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpsrld $28, %ymm1, %ymm1
-; AVX512-NEXT:    vpslld $4, %ymm0, %ymm0
-; AVX512-NEXT:    vpor %ymm1, %ymm0, %ymm0
-; AVX512-NEXT:    retq
+; AVX512F-LABEL: splatconstant_funnnel_v8i32:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpsrld $28, %ymm1, %ymm1
+; AVX512F-NEXT:    vpslld $4, %ymm0, %ymm0
+; AVX512F-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: splatconstant_funnnel_v8i32:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpsrld $28, %ymm1, %ymm1
+; AVX512VL-NEXT:    vpslld $4, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: splatconstant_funnnel_v8i32:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpsrld $28, %ymm1, %ymm1
+; AVX512BW-NEXT:    vpslld $4, %ymm0, %ymm0
+; AVX512BW-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512BW-NEXT:    retq
+;
+; AVX512VBMI2-LABEL: splatconstant_funnnel_v8i32:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    vpsrld $28, %ymm1, %ymm1
+; AVX512VBMI2-NEXT:    vpslld $4, %ymm0, %ymm0
+; AVX512VBMI2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLBW-LABEL: splatconstant_funnnel_v8i32:
+; AVX512VLBW:       # %bb.0:
+; AVX512VLBW-NEXT:    vpsrld $28, %ymm1, %ymm1
+; AVX512VLBW-NEXT:    vpslld $4, %ymm0, %ymm0
+; AVX512VLBW-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512VLBW-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: splatconstant_funnnel_v8i32:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpshldd $4, %ymm1, %ymm0, %ymm0
+; AVX512VLVBMI2-NEXT:    retq
 ;
 ; XOPAVX1-LABEL: splatconstant_funnnel_v8i32:
 ; XOPAVX1:       # %bb.0:
@@ -2044,12 +2437,45 @@ define <16 x i16> @splatconstant_funnnel_v16i16(<16 x i16> %x, <16 x i16> %y) no
 ; AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    retq
 ;
-; AVX512-LABEL: splatconstant_funnnel_v16i16:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpsrlw $9, %ymm1, %ymm1
-; AVX512-NEXT:    vpsllw $7, %ymm0, %ymm0
-; AVX512-NEXT:    vpor %ymm1, %ymm0, %ymm0
-; AVX512-NEXT:    retq
+; AVX512F-LABEL: splatconstant_funnnel_v16i16:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpsrlw $9, %ymm1, %ymm1
+; AVX512F-NEXT:    vpsllw $7, %ymm0, %ymm0
+; AVX512F-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: splatconstant_funnnel_v16i16:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpsrlw $9, %ymm1, %ymm1
+; AVX512VL-NEXT:    vpsllw $7, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: splatconstant_funnnel_v16i16:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpsrlw $9, %ymm1, %ymm1
+; AVX512BW-NEXT:    vpsllw $7, %ymm0, %ymm0
+; AVX512BW-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512BW-NEXT:    retq
+;
+; AVX512VBMI2-LABEL: splatconstant_funnnel_v16i16:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    vpsrlw $9, %ymm1, %ymm1
+; AVX512VBMI2-NEXT:    vpsllw $7, %ymm0, %ymm0
+; AVX512VBMI2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLBW-LABEL: splatconstant_funnnel_v16i16:
+; AVX512VLBW:       # %bb.0:
+; AVX512VLBW-NEXT:    vpsrlw $9, %ymm1, %ymm1
+; AVX512VLBW-NEXT:    vpsllw $7, %ymm0, %ymm0
+; AVX512VLBW-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512VLBW-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: splatconstant_funnnel_v16i16:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpshldw $7, %ymm1, %ymm0, %ymm0
+; AVX512VLVBMI2-NEXT:    retq
 ;
 ; XOPAVX1-LABEL: splatconstant_funnnel_v16i16:
 ; XOPAVX1:       # %bb.0:
@@ -2130,10 +2556,8 @@ define <32 x i8> @splatconstant_funnnel_v32i8(<32 x i8> %x, <32 x i8> %y) nounwi
 ; XOPAVX2-LABEL: splatconstant_funnnel_v32i8:
 ; XOPAVX2:       # %bb.0:
 ; XOPAVX2-NEXT:    vpsrlw $4, %ymm1, %ymm1
-; XOPAVX2-NEXT:    vpand {{.*}}(%rip), %ymm1, %ymm1
 ; XOPAVX2-NEXT:    vpsllw $4, %ymm0, %ymm0
-; XOPAVX2-NEXT:    vpand {{.*}}(%rip), %ymm0, %ymm0
-; XOPAVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; XOPAVX2-NEXT:    vpcmov {{.*}}(%rip), %ymm1, %ymm0, %ymm0
 ; XOPAVX2-NEXT:    retq
   %res = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %x, <32 x i8> %y, <32 x i8> <i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4>)
   ret <32 x i8> %res
diff --git a/test/CodeGen/X86/vector-fshl-512.ll b/test/CodeGen/X86/vector-fshl-512.ll
index 58b471c1f5b74ffe74cd5a0678c7b2f56a2aca04..cb29f33b9e85ea2665296e39fda625abdffc18fe 100644
--- a/test/CodeGen/X86/vector-fshl-512.ll
+++ b/test/CodeGen/X86/vector-fshl-512.ll
@@ -2,7 +2,9 @@
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f | FileCheck %s --check-prefixes=AVX512,AVX512F
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512vl | FileCheck %s --check-prefixes=AVX512,AVX512VL
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512bw | FileCheck %s --check-prefixes=AVX512,AVX512BW
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512vbmi2 | FileCheck %s --check-prefixes=AVX512,AVX512VBMI2
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512bw,+avx512vl | FileCheck %s --check-prefixes=AVX512,AVX512VLBW
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512vbmi2,+avx512vl | FileCheck %s --check-prefixes=AVX512,AVX512VLVBMI2
 
 declare <8 x i64> @llvm.fshl.v8i64(<8 x i64>, <8 x i64>, <8 x i64>)
 declare <16 x i32> @llvm.fshl.v16i32(<16 x i32>, <16 x i32>, <16 x i32>)
@@ -14,37 +16,141 @@ declare <64 x i8> @llvm.fshl.v64i8(<64 x i8>, <64 x i8>, <64 x i8>)
 ;
 
 define <8 x i64> @var_funnnel_v8i64(<8 x i64> %x, <8 x i64> %y, <8 x i64> %amt) nounwind {
-; AVX512-LABEL: var_funnnel_v8i64:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpbroadcastq {{.*#+}} zmm3 = [63,63,63,63,63,63,63,63]
-; AVX512-NEXT:    vpandq %zmm3, %zmm2, %zmm4
-; AVX512-NEXT:    vpsllvq %zmm4, %zmm0, %zmm5
-; AVX512-NEXT:    vpbroadcastq {{.*#+}} zmm6 = [64,64,64,64,64,64,64,64]
-; AVX512-NEXT:    vpsubq %zmm4, %zmm6, %zmm4
-; AVX512-NEXT:    vpsrlvq %zmm4, %zmm1, %zmm1
-; AVX512-NEXT:    vporq %zmm1, %zmm5, %zmm1
-; AVX512-NEXT:    vptestnmq %zmm3, %zmm2, %k1
-; AVX512-NEXT:    vmovdqa64 %zmm0, %zmm1 {%k1}
-; AVX512-NEXT:    vmovdqa64 %zmm1, %zmm0
-; AVX512-NEXT:    retq
+; AVX512F-LABEL: var_funnnel_v8i64:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpbroadcastq {{.*#+}} zmm3 = [63,63,63,63,63,63,63,63]
+; AVX512F-NEXT:    vpandq %zmm3, %zmm2, %zmm4
+; AVX512F-NEXT:    vpsllvq %zmm4, %zmm0, %zmm5
+; AVX512F-NEXT:    vpbroadcastq {{.*#+}} zmm6 = [64,64,64,64,64,64,64,64]
+; AVX512F-NEXT:    vpsubq %zmm4, %zmm6, %zmm4
+; AVX512F-NEXT:    vpsrlvq %zmm4, %zmm1, %zmm1
+; AVX512F-NEXT:    vporq %zmm1, %zmm5, %zmm1
+; AVX512F-NEXT:    vptestnmq %zmm3, %zmm2, %k1
+; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm1 {%k1}
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm0
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: var_funnnel_v8i64:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpbroadcastq {{.*#+}} zmm3 = [63,63,63,63,63,63,63,63]
+; AVX512VL-NEXT:    vpandq %zmm3, %zmm2, %zmm4
+; AVX512VL-NEXT:    vpsllvq %zmm4, %zmm0, %zmm5
+; AVX512VL-NEXT:    vpbroadcastq {{.*#+}} zmm6 = [64,64,64,64,64,64,64,64]
+; AVX512VL-NEXT:    vpsubq %zmm4, %zmm6, %zmm4
+; AVX512VL-NEXT:    vpsrlvq %zmm4, %zmm1, %zmm1
+; AVX512VL-NEXT:    vporq %zmm1, %zmm5, %zmm1
+; AVX512VL-NEXT:    vptestnmq %zmm3, %zmm2, %k1
+; AVX512VL-NEXT:    vmovdqa64 %zmm0, %zmm1 {%k1}
+; AVX512VL-NEXT:    vmovdqa64 %zmm1, %zmm0
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: var_funnnel_v8i64:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpbroadcastq {{.*#+}} zmm3 = [63,63,63,63,63,63,63,63]
+; AVX512BW-NEXT:    vpandq %zmm3, %zmm2, %zmm4
+; AVX512BW-NEXT:    vpsllvq %zmm4, %zmm0, %zmm5
+; AVX512BW-NEXT:    vpbroadcastq {{.*#+}} zmm6 = [64,64,64,64,64,64,64,64]
+; AVX512BW-NEXT:    vpsubq %zmm4, %zmm6, %zmm4
+; AVX512BW-NEXT:    vpsrlvq %zmm4, %zmm1, %zmm1
+; AVX512BW-NEXT:    vporq %zmm1, %zmm5, %zmm1
+; AVX512BW-NEXT:    vptestnmq %zmm3, %zmm2, %k1
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm1 {%k1}
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm0
+; AVX512BW-NEXT:    retq
+;
+; AVX512VBMI2-LABEL: var_funnnel_v8i64:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    vpshldvq %zmm2, %zmm1, %zmm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLBW-LABEL: var_funnnel_v8i64:
+; AVX512VLBW:       # %bb.0:
+; AVX512VLBW-NEXT:    vpbroadcastq {{.*#+}} zmm3 = [63,63,63,63,63,63,63,63]
+; AVX512VLBW-NEXT:    vpandq %zmm3, %zmm2, %zmm4
+; AVX512VLBW-NEXT:    vpsllvq %zmm4, %zmm0, %zmm5
+; AVX512VLBW-NEXT:    vpbroadcastq {{.*#+}} zmm6 = [64,64,64,64,64,64,64,64]
+; AVX512VLBW-NEXT:    vpsubq %zmm4, %zmm6, %zmm4
+; AVX512VLBW-NEXT:    vpsrlvq %zmm4, %zmm1, %zmm1
+; AVX512VLBW-NEXT:    vporq %zmm1, %zmm5, %zmm1
+; AVX512VLBW-NEXT:    vptestnmq %zmm3, %zmm2, %k1
+; AVX512VLBW-NEXT:    vmovdqa64 %zmm0, %zmm1 {%k1}
+; AVX512VLBW-NEXT:    vmovdqa64 %zmm1, %zmm0
+; AVX512VLBW-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: var_funnnel_v8i64:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpshldvq %zmm2, %zmm1, %zmm0
+; AVX512VLVBMI2-NEXT:    retq
   %res = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %x, <8 x i64> %y, <8 x i64> %amt)
   ret <8 x i64> %res
 }
 
 define <16 x i32> @var_funnnel_v16i32(<16 x i32> %x, <16 x i32> %y, <16 x i32> %amt) nounwind {
-; AVX512-LABEL: var_funnnel_v16i32:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpbroadcastd {{.*#+}} zmm3 = [31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31]
-; AVX512-NEXT:    vpandd %zmm3, %zmm2, %zmm4
-; AVX512-NEXT:    vpsllvd %zmm4, %zmm0, %zmm5
-; AVX512-NEXT:    vpbroadcastd {{.*#+}} zmm6 = [32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32]
-; AVX512-NEXT:    vpsubd %zmm4, %zmm6, %zmm4
-; AVX512-NEXT:    vpsrlvd %zmm4, %zmm1, %zmm1
-; AVX512-NEXT:    vpord %zmm1, %zmm5, %zmm1
-; AVX512-NEXT:    vptestnmd %zmm3, %zmm2, %k1
-; AVX512-NEXT:    vmovdqa32 %zmm0, %zmm1 {%k1}
-; AVX512-NEXT:    vmovdqa64 %zmm1, %zmm0
-; AVX512-NEXT:    retq
+; AVX512F-LABEL: var_funnnel_v16i32:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpbroadcastd {{.*#+}} zmm3 = [31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31]
+; AVX512F-NEXT:    vpandd %zmm3, %zmm2, %zmm4
+; AVX512F-NEXT:    vpsllvd %zmm4, %zmm0, %zmm5
+; AVX512F-NEXT:    vpbroadcastd {{.*#+}} zmm6 = [32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32]
+; AVX512F-NEXT:    vpsubd %zmm4, %zmm6, %zmm4
+; AVX512F-NEXT:    vpsrlvd %zmm4, %zmm1, %zmm1
+; AVX512F-NEXT:    vpord %zmm1, %zmm5, %zmm1
+; AVX512F-NEXT:    vptestnmd %zmm3, %zmm2, %k1
+; AVX512F-NEXT:    vmovdqa32 %zmm0, %zmm1 {%k1}
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm0
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: var_funnnel_v16i32:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpbroadcastd {{.*#+}} zmm3 = [31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31]
+; AVX512VL-NEXT:    vpandd %zmm3, %zmm2, %zmm4
+; AVX512VL-NEXT:    vpsllvd %zmm4, %zmm0, %zmm5
+; AVX512VL-NEXT:    vpbroadcastd {{.*#+}} zmm6 = [32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32]
+; AVX512VL-NEXT:    vpsubd %zmm4, %zmm6, %zmm4
+; AVX512VL-NEXT:    vpsrlvd %zmm4, %zmm1, %zmm1
+; AVX512VL-NEXT:    vpord %zmm1, %zmm5, %zmm1
+; AVX512VL-NEXT:    vptestnmd %zmm3, %zmm2, %k1
+; AVX512VL-NEXT:    vmovdqa32 %zmm0, %zmm1 {%k1}
+; AVX512VL-NEXT:    vmovdqa64 %zmm1, %zmm0
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: var_funnnel_v16i32:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpbroadcastd {{.*#+}} zmm3 = [31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31]
+; AVX512BW-NEXT:    vpandd %zmm3, %zmm2, %zmm4
+; AVX512BW-NEXT:    vpsllvd %zmm4, %zmm0, %zmm5
+; AVX512BW-NEXT:    vpbroadcastd {{.*#+}} zmm6 = [32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32]
+; AVX512BW-NEXT:    vpsubd %zmm4, %zmm6, %zmm4
+; AVX512BW-NEXT:    vpsrlvd %zmm4, %zmm1, %zmm1
+; AVX512BW-NEXT:    vpord %zmm1, %zmm5, %zmm1
+; AVX512BW-NEXT:    vptestnmd %zmm3, %zmm2, %k1
+; AVX512BW-NEXT:    vmovdqa32 %zmm0, %zmm1 {%k1}
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm0
+; AVX512BW-NEXT:    retq
+;
+; AVX512VBMI2-LABEL: var_funnnel_v16i32:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    vpshldvd %zmm2, %zmm1, %zmm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLBW-LABEL: var_funnnel_v16i32:
+; AVX512VLBW:       # %bb.0:
+; AVX512VLBW-NEXT:    vpbroadcastd {{.*#+}} zmm3 = [31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31]
+; AVX512VLBW-NEXT:    vpandd %zmm3, %zmm2, %zmm4
+; AVX512VLBW-NEXT:    vpsllvd %zmm4, %zmm0, %zmm5
+; AVX512VLBW-NEXT:    vpbroadcastd {{.*#+}} zmm6 = [32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32]
+; AVX512VLBW-NEXT:    vpsubd %zmm4, %zmm6, %zmm4
+; AVX512VLBW-NEXT:    vpsrlvd %zmm4, %zmm1, %zmm1
+; AVX512VLBW-NEXT:    vpord %zmm1, %zmm5, %zmm1
+; AVX512VLBW-NEXT:    vptestnmd %zmm3, %zmm2, %k1
+; AVX512VLBW-NEXT:    vmovdqa32 %zmm0, %zmm1 {%k1}
+; AVX512VLBW-NEXT:    vmovdqa64 %zmm1, %zmm0
+; AVX512VLBW-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: var_funnnel_v16i32:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpshldvd %zmm2, %zmm1, %zmm0
+; AVX512VLVBMI2-NEXT:    retq
   %res = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %x, <16 x i32> %y, <16 x i32> %amt)
   ret <16 x i32> %res
 }
@@ -126,6 +232,11 @@ define <32 x i16> @var_funnnel_v32i16(<32 x i16> %x, <32 x i16> %y, <32 x i16> %
 ; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm0
 ; AVX512BW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: var_funnnel_v32i16:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    vpshldvw %zmm2, %zmm1, %zmm0
+; AVX512VBMI2-NEXT:    retq
+;
 ; AVX512VLBW-LABEL: var_funnnel_v32i16:
 ; AVX512VLBW:       # %bb.0:
 ; AVX512VLBW-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
@@ -139,6 +250,11 @@ define <32 x i16> @var_funnnel_v32i16(<32 x i16> %x, <32 x i16> %y, <32 x i16> %
 ; AVX512VLBW-NEXT:    vmovdqu16 %zmm0, %zmm1 {%k1}
 ; AVX512VLBW-NEXT:    vmovdqa64 %zmm1, %zmm0
 ; AVX512VLBW-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: var_funnnel_v32i16:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpshldvw %zmm2, %zmm1, %zmm0
+; AVX512VLVBMI2-NEXT:    retq
   %res = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %x, <32 x i16> %y, <32 x i16> %amt)
   ret <32 x i16> %res
 }
@@ -320,6 +436,46 @@ define <64 x i8> @var_funnnel_v64i8(<64 x i8> %x, <64 x i8> %y, <64 x i8> %amt)
 ; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm0
 ; AVX512BW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: var_funnnel_v64i8:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
+; AVX512VBMI2-NEXT:    vpandq %zmm3, %zmm2, %zmm4
+; AVX512VBMI2-NEXT:    vmovdqa64 {{.*#+}} zmm5 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; AVX512VBMI2-NEXT:    vpsubb %zmm4, %zmm5, %zmm5
+; AVX512VBMI2-NEXT:    vpsllw $5, %zmm5, %zmm5
+; AVX512VBMI2-NEXT:    vpaddb %zmm5, %zmm5, %zmm6
+; AVX512VBMI2-NEXT:    vpmovb2m %zmm6, %k1
+; AVX512VBMI2-NEXT:    vpmovb2m %zmm5, %k2
+; AVX512VBMI2-NEXT:    vpsrlw $4, %zmm1, %zmm5
+; AVX512VBMI2-NEXT:    vpandq {{.*}}(%rip), %zmm5, %zmm5
+; AVX512VBMI2-NEXT:    vmovdqu8 %zmm5, %zmm1 {%k2}
+; AVX512VBMI2-NEXT:    vpsrlw $2, %zmm1, %zmm5
+; AVX512VBMI2-NEXT:    vpandq {{.*}}(%rip), %zmm5, %zmm5
+; AVX512VBMI2-NEXT:    vmovdqu8 %zmm5, %zmm1 {%k1}
+; AVX512VBMI2-NEXT:    vpsrlw $1, %zmm1, %zmm5
+; AVX512VBMI2-NEXT:    vpandq {{.*}}(%rip), %zmm5, %zmm5
+; AVX512VBMI2-NEXT:    vpaddb %zmm6, %zmm6, %zmm6
+; AVX512VBMI2-NEXT:    vpmovb2m %zmm6, %k1
+; AVX512VBMI2-NEXT:    vmovdqu8 %zmm5, %zmm1 {%k1}
+; AVX512VBMI2-NEXT:    vpsllw $5, %zmm4, %zmm4
+; AVX512VBMI2-NEXT:    vpaddb %zmm4, %zmm4, %zmm5
+; AVX512VBMI2-NEXT:    vpmovb2m %zmm5, %k1
+; AVX512VBMI2-NEXT:    vpmovb2m %zmm4, %k2
+; AVX512VBMI2-NEXT:    vpsllw $4, %zmm0, %zmm4
+; AVX512VBMI2-NEXT:    vpandq {{.*}}(%rip), %zmm4, %zmm4
+; AVX512VBMI2-NEXT:    vpblendmb %zmm4, %zmm0, %zmm4 {%k2}
+; AVX512VBMI2-NEXT:    vpsllw $2, %zmm4, %zmm6
+; AVX512VBMI2-NEXT:    vpandq {{.*}}(%rip), %zmm6, %zmm6
+; AVX512VBMI2-NEXT:    vmovdqu8 %zmm6, %zmm4 {%k1}
+; AVX512VBMI2-NEXT:    vpaddb %zmm5, %zmm5, %zmm5
+; AVX512VBMI2-NEXT:    vpmovb2m %zmm5, %k1
+; AVX512VBMI2-NEXT:    vpaddb %zmm4, %zmm4, %zmm4 {%k1}
+; AVX512VBMI2-NEXT:    vporq %zmm1, %zmm4, %zmm1
+; AVX512VBMI2-NEXT:    vptestnmb %zmm3, %zmm2, %k1
+; AVX512VBMI2-NEXT:    vmovdqu8 %zmm0, %zmm1 {%k1}
+; AVX512VBMI2-NEXT:    vmovdqa64 %zmm1, %zmm0
+; AVX512VBMI2-NEXT:    retq
+;
 ; AVX512VLBW-LABEL: var_funnnel_v64i8:
 ; AVX512VLBW:       # %bb.0:
 ; AVX512VLBW-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
@@ -359,6 +515,46 @@ define <64 x i8> @var_funnnel_v64i8(<64 x i8> %x, <64 x i8> %y, <64 x i8> %amt)
 ; AVX512VLBW-NEXT:    vmovdqu8 %zmm0, %zmm1 {%k1}
 ; AVX512VLBW-NEXT:    vmovdqa64 %zmm1, %zmm0
 ; AVX512VLBW-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: var_funnnel_v64i8:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
+; AVX512VLVBMI2-NEXT:    vpandq %zmm3, %zmm2, %zmm4
+; AVX512VLVBMI2-NEXT:    vmovdqa64 {{.*#+}} zmm5 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; AVX512VLVBMI2-NEXT:    vpsubb %zmm4, %zmm5, %zmm5
+; AVX512VLVBMI2-NEXT:    vpsllw $5, %zmm5, %zmm5
+; AVX512VLVBMI2-NEXT:    vpaddb %zmm5, %zmm5, %zmm6
+; AVX512VLVBMI2-NEXT:    vpmovb2m %zmm6, %k1
+; AVX512VLVBMI2-NEXT:    vpmovb2m %zmm5, %k2
+; AVX512VLVBMI2-NEXT:    vpsrlw $4, %zmm1, %zmm5
+; AVX512VLVBMI2-NEXT:    vpandq {{.*}}(%rip), %zmm5, %zmm5
+; AVX512VLVBMI2-NEXT:    vmovdqu8 %zmm5, %zmm1 {%k2}
+; AVX512VLVBMI2-NEXT:    vpsrlw $2, %zmm1, %zmm5
+; AVX512VLVBMI2-NEXT:    vpandq {{.*}}(%rip), %zmm5, %zmm5
+; AVX512VLVBMI2-NEXT:    vmovdqu8 %zmm5, %zmm1 {%k1}
+; AVX512VLVBMI2-NEXT:    vpsrlw $1, %zmm1, %zmm5
+; AVX512VLVBMI2-NEXT:    vpandq {{.*}}(%rip), %zmm5, %zmm5
+; AVX512VLVBMI2-NEXT:    vpaddb %zmm6, %zmm6, %zmm6
+; AVX512VLVBMI2-NEXT:    vpmovb2m %zmm6, %k1
+; AVX512VLVBMI2-NEXT:    vmovdqu8 %zmm5, %zmm1 {%k1}
+; AVX512VLVBMI2-NEXT:    vpsllw $5, %zmm4, %zmm4
+; AVX512VLVBMI2-NEXT:    vpaddb %zmm4, %zmm4, %zmm5
+; AVX512VLVBMI2-NEXT:    vpmovb2m %zmm5, %k1
+; AVX512VLVBMI2-NEXT:    vpmovb2m %zmm4, %k2
+; AVX512VLVBMI2-NEXT:    vpsllw $4, %zmm0, %zmm4
+; AVX512VLVBMI2-NEXT:    vpandq {{.*}}(%rip), %zmm4, %zmm4
+; AVX512VLVBMI2-NEXT:    vpblendmb %zmm4, %zmm0, %zmm4 {%k2}
+; AVX512VLVBMI2-NEXT:    vpsllw $2, %zmm4, %zmm6
+; AVX512VLVBMI2-NEXT:    vpandq {{.*}}(%rip), %zmm6, %zmm6
+; AVX512VLVBMI2-NEXT:    vmovdqu8 %zmm6, %zmm4 {%k1}
+; AVX512VLVBMI2-NEXT:    vpaddb %zmm5, %zmm5, %zmm5
+; AVX512VLVBMI2-NEXT:    vpmovb2m %zmm5, %k1
+; AVX512VLVBMI2-NEXT:    vpaddb %zmm4, %zmm4, %zmm4 {%k1}
+; AVX512VLVBMI2-NEXT:    vporq %zmm1, %zmm4, %zmm1
+; AVX512VLVBMI2-NEXT:    vptestnmb %zmm3, %zmm2, %k1
+; AVX512VLVBMI2-NEXT:    vmovdqu8 %zmm0, %zmm1 {%k1}
+; AVX512VLVBMI2-NEXT:    vmovdqa64 %zmm1, %zmm0
+; AVX512VLVBMI2-NEXT:    retq
   %res = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %x, <64 x i8> %y, <64 x i8> %amt)
   ret <64 x i8> %res
 }
@@ -368,42 +564,162 @@ define <64 x i8> @var_funnnel_v64i8(<64 x i8> %x, <64 x i8> %y, <64 x i8> %amt)
 ;
 
 define <8 x i64> @splatvar_funnnel_v8i64(<8 x i64> %x, <8 x i64> %y, <8 x i64> %amt) nounwind {
-; AVX512-LABEL: splatvar_funnnel_v8i64:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpbroadcastq %xmm2, %zmm2
-; AVX512-NEXT:    vpbroadcastq {{.*#+}} zmm3 = [63,63,63,63,63,63,63,63]
-; AVX512-NEXT:    vpandq %zmm3, %zmm2, %zmm4
-; AVX512-NEXT:    vpsllq %xmm4, %zmm0, %zmm5
-; AVX512-NEXT:    vmovdqa {{.*#+}} xmm6 = [64,64]
-; AVX512-NEXT:    vpsubq %xmm4, %xmm6, %xmm4
-; AVX512-NEXT:    vpsrlq %xmm4, %zmm1, %zmm1
-; AVX512-NEXT:    vporq %zmm1, %zmm5, %zmm1
-; AVX512-NEXT:    vptestnmq %zmm3, %zmm2, %k1
-; AVX512-NEXT:    vmovdqa64 %zmm0, %zmm1 {%k1}
-; AVX512-NEXT:    vmovdqa64 %zmm1, %zmm0
-; AVX512-NEXT:    retq
+; AVX512F-LABEL: splatvar_funnnel_v8i64:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpbroadcastq %xmm2, %zmm2
+; AVX512F-NEXT:    vpbroadcastq {{.*#+}} zmm3 = [63,63,63,63,63,63,63,63]
+; AVX512F-NEXT:    vpandq %zmm3, %zmm2, %zmm4
+; AVX512F-NEXT:    vpsllq %xmm4, %zmm0, %zmm5
+; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm6 = [64,64]
+; AVX512F-NEXT:    vpsubq %xmm4, %xmm6, %xmm4
+; AVX512F-NEXT:    vpsrlq %xmm4, %zmm1, %zmm1
+; AVX512F-NEXT:    vporq %zmm1, %zmm5, %zmm1
+; AVX512F-NEXT:    vptestnmq %zmm3, %zmm2, %k1
+; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm1 {%k1}
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm0
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: splatvar_funnnel_v8i64:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpbroadcastq %xmm2, %zmm2
+; AVX512VL-NEXT:    vpbroadcastq {{.*#+}} zmm3 = [63,63,63,63,63,63,63,63]
+; AVX512VL-NEXT:    vpandq %zmm3, %zmm2, %zmm4
+; AVX512VL-NEXT:    vpsllq %xmm4, %zmm0, %zmm5
+; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm6 = [64,64]
+; AVX512VL-NEXT:    vpsubq %xmm4, %xmm6, %xmm4
+; AVX512VL-NEXT:    vpsrlq %xmm4, %zmm1, %zmm1
+; AVX512VL-NEXT:    vporq %zmm1, %zmm5, %zmm1
+; AVX512VL-NEXT:    vptestnmq %zmm3, %zmm2, %k1
+; AVX512VL-NEXT:    vmovdqa64 %zmm0, %zmm1 {%k1}
+; AVX512VL-NEXT:    vmovdqa64 %zmm1, %zmm0
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: splatvar_funnnel_v8i64:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpbroadcastq %xmm2, %zmm2
+; AVX512BW-NEXT:    vpbroadcastq {{.*#+}} zmm3 = [63,63,63,63,63,63,63,63]
+; AVX512BW-NEXT:    vpandq %zmm3, %zmm2, %zmm4
+; AVX512BW-NEXT:    vpsllq %xmm4, %zmm0, %zmm5
+; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm6 = [64,64]
+; AVX512BW-NEXT:    vpsubq %xmm4, %xmm6, %xmm4
+; AVX512BW-NEXT:    vpsrlq %xmm4, %zmm1, %zmm1
+; AVX512BW-NEXT:    vporq %zmm1, %zmm5, %zmm1
+; AVX512BW-NEXT:    vptestnmq %zmm3, %zmm2, %k1
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm1 {%k1}
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm0
+; AVX512BW-NEXT:    retq
+;
+; AVX512VBMI2-LABEL: splatvar_funnnel_v8i64:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    vpbroadcastq %xmm2, %zmm2
+; AVX512VBMI2-NEXT:    vpshldvq %zmm2, %zmm1, %zmm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLBW-LABEL: splatvar_funnnel_v8i64:
+; AVX512VLBW:       # %bb.0:
+; AVX512VLBW-NEXT:    vpbroadcastq %xmm2, %zmm2
+; AVX512VLBW-NEXT:    vpbroadcastq {{.*#+}} zmm3 = [63,63,63,63,63,63,63,63]
+; AVX512VLBW-NEXT:    vpandq %zmm3, %zmm2, %zmm4
+; AVX512VLBW-NEXT:    vpsllq %xmm4, %zmm0, %zmm5
+; AVX512VLBW-NEXT:    vmovdqa {{.*#+}} xmm6 = [64,64]
+; AVX512VLBW-NEXT:    vpsubq %xmm4, %xmm6, %xmm4
+; AVX512VLBW-NEXT:    vpsrlq %xmm4, %zmm1, %zmm1
+; AVX512VLBW-NEXT:    vporq %zmm1, %zmm5, %zmm1
+; AVX512VLBW-NEXT:    vptestnmq %zmm3, %zmm2, %k1
+; AVX512VLBW-NEXT:    vmovdqa64 %zmm0, %zmm1 {%k1}
+; AVX512VLBW-NEXT:    vmovdqa64 %zmm1, %zmm0
+; AVX512VLBW-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: splatvar_funnnel_v8i64:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpbroadcastq %xmm2, %zmm2
+; AVX512VLVBMI2-NEXT:    vpshldvq %zmm2, %zmm1, %zmm0
+; AVX512VLVBMI2-NEXT:    retq
   %splat = shufflevector <8 x i64> %amt, <8 x i64> undef, <8 x i32> zeroinitializer
   %res = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %x, <8 x i64> %y, <8 x i64> %splat)
   ret <8 x i64> %res
 }
 
 define <16 x i32> @splatvar_funnnel_v16i32(<16 x i32> %x, <16 x i32> %y, <16 x i32> %amt) nounwind {
-; AVX512-LABEL: splatvar_funnnel_v16i32:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpbroadcastd %xmm2, %zmm2
-; AVX512-NEXT:    vpbroadcastd {{.*#+}} zmm3 = [31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31]
-; AVX512-NEXT:    vpandd %zmm3, %zmm2, %zmm4
-; AVX512-NEXT:    vpmovzxdq {{.*#+}} xmm5 = xmm4[0],zero,xmm4[1],zero
-; AVX512-NEXT:    vpslld %xmm5, %zmm0, %zmm5
-; AVX512-NEXT:    vpbroadcastd {{.*#+}} xmm6 = [32,32,32,32]
-; AVX512-NEXT:    vpsubd %xmm4, %xmm6, %xmm4
-; AVX512-NEXT:    vpmovzxdq {{.*#+}} xmm4 = xmm4[0],zero,xmm4[1],zero
-; AVX512-NEXT:    vpsrld %xmm4, %zmm1, %zmm1
-; AVX512-NEXT:    vpord %zmm1, %zmm5, %zmm1
-; AVX512-NEXT:    vptestnmd %zmm3, %zmm2, %k1
-; AVX512-NEXT:    vmovdqa32 %zmm0, %zmm1 {%k1}
-; AVX512-NEXT:    vmovdqa64 %zmm1, %zmm0
-; AVX512-NEXT:    retq
+; AVX512F-LABEL: splatvar_funnnel_v16i32:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpbroadcastd %xmm2, %zmm2
+; AVX512F-NEXT:    vpbroadcastd {{.*#+}} zmm3 = [31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31]
+; AVX512F-NEXT:    vpandd %zmm3, %zmm2, %zmm4
+; AVX512F-NEXT:    vpmovzxdq {{.*#+}} xmm5 = xmm4[0],zero,xmm4[1],zero
+; AVX512F-NEXT:    vpslld %xmm5, %zmm0, %zmm5
+; AVX512F-NEXT:    vpbroadcastd {{.*#+}} xmm6 = [32,32,32,32]
+; AVX512F-NEXT:    vpsubd %xmm4, %xmm6, %xmm4
+; AVX512F-NEXT:    vpmovzxdq {{.*#+}} xmm4 = xmm4[0],zero,xmm4[1],zero
+; AVX512F-NEXT:    vpsrld %xmm4, %zmm1, %zmm1
+; AVX512F-NEXT:    vpord %zmm1, %zmm5, %zmm1
+; AVX512F-NEXT:    vptestnmd %zmm3, %zmm2, %k1
+; AVX512F-NEXT:    vmovdqa32 %zmm0, %zmm1 {%k1}
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm0
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: splatvar_funnnel_v16i32:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpbroadcastd %xmm2, %zmm2
+; AVX512VL-NEXT:    vpbroadcastd {{.*#+}} zmm3 = [31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31]
+; AVX512VL-NEXT:    vpandd %zmm3, %zmm2, %zmm4
+; AVX512VL-NEXT:    vpmovzxdq {{.*#+}} xmm5 = xmm4[0],zero,xmm4[1],zero
+; AVX512VL-NEXT:    vpslld %xmm5, %zmm0, %zmm5
+; AVX512VL-NEXT:    vpbroadcastd {{.*#+}} xmm6 = [32,32,32,32]
+; AVX512VL-NEXT:    vpsubd %xmm4, %xmm6, %xmm4
+; AVX512VL-NEXT:    vpmovzxdq {{.*#+}} xmm4 = xmm4[0],zero,xmm4[1],zero
+; AVX512VL-NEXT:    vpsrld %xmm4, %zmm1, %zmm1
+; AVX512VL-NEXT:    vpord %zmm1, %zmm5, %zmm1
+; AVX512VL-NEXT:    vptestnmd %zmm3, %zmm2, %k1
+; AVX512VL-NEXT:    vmovdqa32 %zmm0, %zmm1 {%k1}
+; AVX512VL-NEXT:    vmovdqa64 %zmm1, %zmm0
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: splatvar_funnnel_v16i32:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpbroadcastd %xmm2, %zmm2
+; AVX512BW-NEXT:    vpbroadcastd {{.*#+}} zmm3 = [31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31]
+; AVX512BW-NEXT:    vpandd %zmm3, %zmm2, %zmm4
+; AVX512BW-NEXT:    vpmovzxdq {{.*#+}} xmm5 = xmm4[0],zero,xmm4[1],zero
+; AVX512BW-NEXT:    vpslld %xmm5, %zmm0, %zmm5
+; AVX512BW-NEXT:    vpbroadcastd {{.*#+}} xmm6 = [32,32,32,32]
+; AVX512BW-NEXT:    vpsubd %xmm4, %xmm6, %xmm4
+; AVX512BW-NEXT:    vpmovzxdq {{.*#+}} xmm4 = xmm4[0],zero,xmm4[1],zero
+; AVX512BW-NEXT:    vpsrld %xmm4, %zmm1, %zmm1
+; AVX512BW-NEXT:    vpord %zmm1, %zmm5, %zmm1
+; AVX512BW-NEXT:    vptestnmd %zmm3, %zmm2, %k1
+; AVX512BW-NEXT:    vmovdqa32 %zmm0, %zmm1 {%k1}
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm0
+; AVX512BW-NEXT:    retq
+;
+; AVX512VBMI2-LABEL: splatvar_funnnel_v16i32:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    vpbroadcastd %xmm2, %zmm2
+; AVX512VBMI2-NEXT:    vpshldvd %zmm2, %zmm1, %zmm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLBW-LABEL: splatvar_funnnel_v16i32:
+; AVX512VLBW:       # %bb.0:
+; AVX512VLBW-NEXT:    vpbroadcastd %xmm2, %zmm2
+; AVX512VLBW-NEXT:    vpbroadcastd {{.*#+}} zmm3 = [31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31]
+; AVX512VLBW-NEXT:    vpandd %zmm3, %zmm2, %zmm4
+; AVX512VLBW-NEXT:    vpmovzxdq {{.*#+}} xmm5 = xmm4[0],zero,xmm4[1],zero
+; AVX512VLBW-NEXT:    vpslld %xmm5, %zmm0, %zmm5
+; AVX512VLBW-NEXT:    vpbroadcastd {{.*#+}} xmm6 = [32,32,32,32]
+; AVX512VLBW-NEXT:    vpsubd %xmm4, %xmm6, %xmm4
+; AVX512VLBW-NEXT:    vpmovzxdq {{.*#+}} xmm4 = xmm4[0],zero,xmm4[1],zero
+; AVX512VLBW-NEXT:    vpsrld %xmm4, %zmm1, %zmm1
+; AVX512VLBW-NEXT:    vpord %zmm1, %zmm5, %zmm1
+; AVX512VLBW-NEXT:    vptestnmd %zmm3, %zmm2, %k1
+; AVX512VLBW-NEXT:    vmovdqa32 %zmm0, %zmm1 {%k1}
+; AVX512VLBW-NEXT:    vmovdqa64 %zmm1, %zmm0
+; AVX512VLBW-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: splatvar_funnnel_v16i32:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpbroadcastd %xmm2, %zmm2
+; AVX512VLVBMI2-NEXT:    vpshldvd %zmm2, %zmm1, %zmm0
+; AVX512VLVBMI2-NEXT:    retq
   %splat = shufflevector <16 x i32> %amt, <16 x i32> undef, <16 x i32> zeroinitializer
   %res = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %x, <16 x i32> %y, <16 x i32> %splat)
   ret <16 x i32> %res
@@ -467,6 +783,12 @@ define <32 x i16> @splatvar_funnnel_v32i16(<32 x i16> %x, <32 x i16> %y, <32 x i
 ; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm0
 ; AVX512BW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: splatvar_funnnel_v32i16:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    vpbroadcastw %xmm2, %zmm2
+; AVX512VBMI2-NEXT:    vpshldvw %zmm2, %zmm1, %zmm0
+; AVX512VBMI2-NEXT:    retq
+;
 ; AVX512VLBW-LABEL: splatvar_funnnel_v32i16:
 ; AVX512VLBW:       # %bb.0:
 ; AVX512VLBW-NEXT:    vpbroadcastw %xmm2, %zmm2
@@ -483,6 +805,12 @@ define <32 x i16> @splatvar_funnnel_v32i16(<32 x i16> %x, <32 x i16> %y, <32 x i
 ; AVX512VLBW-NEXT:    vmovdqu16 %zmm0, %zmm1 {%k1}
 ; AVX512VLBW-NEXT:    vmovdqa64 %zmm1, %zmm0
 ; AVX512VLBW-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: splatvar_funnnel_v32i16:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpbroadcastw %xmm2, %zmm2
+; AVX512VLVBMI2-NEXT:    vpshldvw %zmm2, %zmm1, %zmm0
+; AVX512VLVBMI2-NEXT:    retq
   %splat = shufflevector <32 x i16> %amt, <32 x i16> undef, <32 x i32> zeroinitializer
   %res = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %x, <32 x i16> %y, <32 x i16> %splat)
   ret <32 x i16> %res
@@ -574,6 +902,31 @@ define <64 x i8> @splatvar_funnnel_v64i8(<64 x i8> %x, <64 x i8> %y, <64 x i8> %
 ; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm0
 ; AVX512BW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: splatvar_funnnel_v64i8:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    vpbroadcastb %xmm2, %zmm2
+; AVX512VBMI2-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
+; AVX512VBMI2-NEXT:    vpandq %zmm3, %zmm2, %zmm4
+; AVX512VBMI2-NEXT:    vpmovzxbq {{.*#+}} xmm5 = xmm4[0],zero,zero,zero,zero,zero,zero,zero,xmm4[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512VBMI2-NEXT:    vpsllw %xmm5, %zmm0, %zmm6
+; AVX512VBMI2-NEXT:    vpternlogd $255, %zmm7, %zmm7, %zmm7
+; AVX512VBMI2-NEXT:    vpsllw %xmm5, %zmm7, %zmm5
+; AVX512VBMI2-NEXT:    vpbroadcastb %xmm5, %zmm5
+; AVX512VBMI2-NEXT:    vpandq %zmm5, %zmm6, %zmm5
+; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} xmm6 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; AVX512VBMI2-NEXT:    vpsubb %xmm4, %xmm6, %xmm4
+; AVX512VBMI2-NEXT:    vpmovzxbq {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,zero,zero,zero,zero,xmm4[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512VBMI2-NEXT:    vpsrlw %xmm4, %zmm1, %zmm1
+; AVX512VBMI2-NEXT:    vpsrlw %xmm4, %zmm7, %zmm4
+; AVX512VBMI2-NEXT:    vpsrlw $8, %zmm4, %zmm4
+; AVX512VBMI2-NEXT:    vpbroadcastb %xmm4, %zmm4
+; AVX512VBMI2-NEXT:    vpandq %zmm4, %zmm1, %zmm1
+; AVX512VBMI2-NEXT:    vporq %zmm1, %zmm5, %zmm1
+; AVX512VBMI2-NEXT:    vptestnmb %zmm3, %zmm2, %k1
+; AVX512VBMI2-NEXT:    vmovdqu8 %zmm0, %zmm1 {%k1}
+; AVX512VBMI2-NEXT:    vmovdqa64 %zmm1, %zmm0
+; AVX512VBMI2-NEXT:    retq
+;
 ; AVX512VLBW-LABEL: splatvar_funnnel_v64i8:
 ; AVX512VLBW:       # %bb.0:
 ; AVX512VLBW-NEXT:    vpbroadcastb %xmm2, %zmm2
@@ -598,6 +951,31 @@ define <64 x i8> @splatvar_funnnel_v64i8(<64 x i8> %x, <64 x i8> %y, <64 x i8> %
 ; AVX512VLBW-NEXT:    vmovdqu8 %zmm0, %zmm1 {%k1}
 ; AVX512VLBW-NEXT:    vmovdqa64 %zmm1, %zmm0
 ; AVX512VLBW-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: splatvar_funnnel_v64i8:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpbroadcastb %xmm2, %zmm2
+; AVX512VLVBMI2-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
+; AVX512VLVBMI2-NEXT:    vpandq %zmm3, %zmm2, %zmm4
+; AVX512VLVBMI2-NEXT:    vpmovzxbq {{.*#+}} xmm5 = xmm4[0],zero,zero,zero,zero,zero,zero,zero,xmm4[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512VLVBMI2-NEXT:    vpsllw %xmm5, %zmm0, %zmm6
+; AVX512VLVBMI2-NEXT:    vpternlogd $255, %zmm7, %zmm7, %zmm7
+; AVX512VLVBMI2-NEXT:    vpsllw %xmm5, %zmm7, %zmm5
+; AVX512VLVBMI2-NEXT:    vpbroadcastb %xmm5, %zmm5
+; AVX512VLVBMI2-NEXT:    vpandq %zmm5, %zmm6, %zmm5
+; AVX512VLVBMI2-NEXT:    vmovdqa {{.*#+}} xmm6 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; AVX512VLVBMI2-NEXT:    vpsubb %xmm4, %xmm6, %xmm4
+; AVX512VLVBMI2-NEXT:    vpmovzxbq {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,zero,zero,zero,zero,xmm4[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512VLVBMI2-NEXT:    vpsrlw %xmm4, %zmm1, %zmm1
+; AVX512VLVBMI2-NEXT:    vpsrlw %xmm4, %zmm7, %zmm4
+; AVX512VLVBMI2-NEXT:    vpsrlw $8, %zmm4, %zmm4
+; AVX512VLVBMI2-NEXT:    vpbroadcastb %xmm4, %zmm4
+; AVX512VLVBMI2-NEXT:    vpandq %zmm4, %zmm1, %zmm1
+; AVX512VLVBMI2-NEXT:    vporq %zmm1, %zmm5, %zmm1
+; AVX512VLVBMI2-NEXT:    vptestnmb %zmm3, %zmm2, %k1
+; AVX512VLVBMI2-NEXT:    vmovdqu8 %zmm0, %zmm1 {%k1}
+; AVX512VLVBMI2-NEXT:    vmovdqa64 %zmm1, %zmm0
+; AVX512VLVBMI2-NEXT:    retq
   %splat = shufflevector <64 x i8> %amt, <64 x i8> undef, <64 x i32> zeroinitializer
   %res = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %x, <64 x i8> %y, <64 x i8> %splat)
   ret <64 x i8> %res
@@ -608,23 +986,85 @@ define <64 x i8> @splatvar_funnnel_v64i8(<64 x i8> %x, <64 x i8> %y, <64 x i8> %
 ;
 
 define <8 x i64> @constant_funnnel_v8i64(<8 x i64> %x, <8 x i64> %y) nounwind {
-; AVX512-LABEL: constant_funnnel_v8i64:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpsrlvq {{.*}}(%rip), %zmm1, %zmm1
-; AVX512-NEXT:    vpsllvq {{.*}}(%rip), %zmm0, %zmm0
-; AVX512-NEXT:    vporq %zmm1, %zmm0, %zmm0
-; AVX512-NEXT:    retq
+; AVX512F-LABEL: constant_funnnel_v8i64:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpsrlvq {{.*}}(%rip), %zmm1, %zmm1
+; AVX512F-NEXT:    vpsllvq {{.*}}(%rip), %zmm0, %zmm0
+; AVX512F-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: constant_funnnel_v8i64:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpsrlvq {{.*}}(%rip), %zmm1, %zmm1
+; AVX512VL-NEXT:    vpsllvq {{.*}}(%rip), %zmm0, %zmm0
+; AVX512VL-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: constant_funnnel_v8i64:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpsrlvq {{.*}}(%rip), %zmm1, %zmm1
+; AVX512BW-NEXT:    vpsllvq {{.*}}(%rip), %zmm0, %zmm0
+; AVX512BW-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    retq
+;
+; AVX512VBMI2-LABEL: constant_funnnel_v8i64:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    vpshldvq {{.*}}(%rip), %zmm1, %zmm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLBW-LABEL: constant_funnnel_v8i64:
+; AVX512VLBW:       # %bb.0:
+; AVX512VLBW-NEXT:    vpsrlvq {{.*}}(%rip), %zmm1, %zmm1
+; AVX512VLBW-NEXT:    vpsllvq {{.*}}(%rip), %zmm0, %zmm0
+; AVX512VLBW-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512VLBW-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: constant_funnnel_v8i64:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpshldvq {{.*}}(%rip), %zmm1, %zmm0
+; AVX512VLVBMI2-NEXT:    retq
   %res = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %x, <8 x i64> %y, <8 x i64> <i64 4, i64 14, i64 50, i64 60, i64 4, i64 14, i64 50, i64 60>)
   ret <8 x i64> %res
 }
 
 define <16 x i32> @constant_funnnel_v16i32(<16 x i32> %x, <16 x i32> %y) nounwind {
-; AVX512-LABEL: constant_funnnel_v16i32:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpsrlvd {{.*}}(%rip), %zmm1, %zmm1
-; AVX512-NEXT:    vpsllvd {{.*}}(%rip), %zmm0, %zmm0
-; AVX512-NEXT:    vpord %zmm1, %zmm0, %zmm0
-; AVX512-NEXT:    retq
+; AVX512F-LABEL: constant_funnnel_v16i32:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpsrlvd {{.*}}(%rip), %zmm1, %zmm1
+; AVX512F-NEXT:    vpsllvd {{.*}}(%rip), %zmm0, %zmm0
+; AVX512F-NEXT:    vpord %zmm1, %zmm0, %zmm0
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: constant_funnnel_v16i32:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpsrlvd {{.*}}(%rip), %zmm1, %zmm1
+; AVX512VL-NEXT:    vpsllvd {{.*}}(%rip), %zmm0, %zmm0
+; AVX512VL-NEXT:    vpord %zmm1, %zmm0, %zmm0
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: constant_funnnel_v16i32:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpsrlvd {{.*}}(%rip), %zmm1, %zmm1
+; AVX512BW-NEXT:    vpsllvd {{.*}}(%rip), %zmm0, %zmm0
+; AVX512BW-NEXT:    vpord %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    retq
+;
+; AVX512VBMI2-LABEL: constant_funnnel_v16i32:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    vpshldvd {{.*}}(%rip), %zmm1, %zmm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLBW-LABEL: constant_funnnel_v16i32:
+; AVX512VLBW:       # %bb.0:
+; AVX512VLBW-NEXT:    vpsrlvd {{.*}}(%rip), %zmm1, %zmm1
+; AVX512VLBW-NEXT:    vpsllvd {{.*}}(%rip), %zmm0, %zmm0
+; AVX512VLBW-NEXT:    vpord %zmm1, %zmm0, %zmm0
+; AVX512VLBW-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: constant_funnnel_v16i32:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpshldvd {{.*}}(%rip), %zmm1, %zmm0
+; AVX512VLVBMI2-NEXT:    retq
   %res = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %x, <16 x i32> %y, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>)
   ret <16 x i32> %res
 }
@@ -671,6 +1111,11 @@ define <32 x i16> @constant_funnnel_v32i16(<32 x i16> %x, <32 x i16> %y) nounwin
 ; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm0
 ; AVX512BW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: constant_funnnel_v32i16:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    vpshldvw {{.*}}(%rip), %zmm1, %zmm0
+; AVX512VBMI2-NEXT:    retq
+;
 ; AVX512VLBW-LABEL: constant_funnnel_v32i16:
 ; AVX512VLBW:       # %bb.0:
 ; AVX512VLBW-NEXT:    vpsrlvw {{.*}}(%rip), %zmm1, %zmm1
@@ -681,6 +1126,11 @@ define <32 x i16> @constant_funnnel_v32i16(<32 x i16> %x, <32 x i16> %y) nounwin
 ; AVX512VLBW-NEXT:    vmovdqu16 %zmm0, %zmm1 {%k1}
 ; AVX512VLBW-NEXT:    vmovdqa64 %zmm1, %zmm0
 ; AVX512VLBW-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: constant_funnnel_v32i16:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpshldvw {{.*}}(%rip), %zmm1, %zmm0
+; AVX512VLVBMI2-NEXT:    retq
   %res = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %x, <32 x i16> %y, <32 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15, i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15>)
   ret <32 x i16> %res
 }
@@ -818,6 +1268,37 @@ define <64 x i8> @constant_funnnel_v64i8(<64 x i8> %x, <64 x i8> %y) nounwind {
 ; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm0
 ; AVX512BW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: constant_funnnel_v64i8:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [8192,24640,41088,57536,57344,41152,24704,8256,8192,24640,41088,57536,57344,41152,24704,8256,8192,24640,41088,57536,57344,41152,24704,8256,8192,24640,41088,57536,57344,41152,24704,8256]
+; AVX512VBMI2-NEXT:    vpmovb2m %zmm2, %k1
+; AVX512VBMI2-NEXT:    vpsllw $4, %zmm0, %zmm3
+; AVX512VBMI2-NEXT:    vpandq {{.*}}(%rip), %zmm3, %zmm3
+; AVX512VBMI2-NEXT:    vpblendmb %zmm3, %zmm0, %zmm3 {%k1}
+; AVX512VBMI2-NEXT:    vpsllw $2, %zmm3, %zmm4
+; AVX512VBMI2-NEXT:    vpandq {{.*}}(%rip), %zmm4, %zmm4
+; AVX512VBMI2-NEXT:    vpaddb %zmm2, %zmm2, %zmm2
+; AVX512VBMI2-NEXT:    vpmovb2m %zmm2, %k1
+; AVX512VBMI2-NEXT:    vmovdqu8 %zmm4, %zmm3 {%k1}
+; AVX512VBMI2-NEXT:    vpaddb %zmm2, %zmm2, %zmm2
+; AVX512VBMI2-NEXT:    vpmovb2m %zmm2, %k1
+; AVX512VBMI2-NEXT:    vpaddb %zmm3, %zmm3, %zmm3 {%k1}
+; AVX512VBMI2-NEXT:    vpunpckhbw {{.*#+}} zmm2 = zmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]
+; AVX512VBMI2-NEXT:    vpsrlw $8, %zmm2, %zmm2
+; AVX512VBMI2-NEXT:    vpsllvw {{.*}}(%rip), %zmm2, %zmm2
+; AVX512VBMI2-NEXT:    vpsrlw $8, %zmm2, %zmm2
+; AVX512VBMI2-NEXT:    vpunpcklbw {{.*#+}} zmm1 = zmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
+; AVX512VBMI2-NEXT:    vpsrlw $8, %zmm1, %zmm1
+; AVX512VBMI2-NEXT:    vpsllvw {{.*}}(%rip), %zmm1, %zmm1
+; AVX512VBMI2-NEXT:    vpsrlw $8, %zmm1, %zmm1
+; AVX512VBMI2-NEXT:    vpackuswb %zmm2, %zmm1, %zmm1
+; AVX512VBMI2-NEXT:    vporq %zmm1, %zmm3, %zmm1
+; AVX512VBMI2-NEXT:    movabsq $72340172838076673, %rax # imm = 0x101010101010101
+; AVX512VBMI2-NEXT:    kmovq %rax, %k1
+; AVX512VBMI2-NEXT:    vmovdqu8 %zmm0, %zmm1 {%k1}
+; AVX512VBMI2-NEXT:    vmovdqa64 %zmm1, %zmm0
+; AVX512VBMI2-NEXT:    retq
+;
 ; AVX512VLBW-LABEL: constant_funnnel_v64i8:
 ; AVX512VLBW:       # %bb.0:
 ; AVX512VLBW-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [8192,24640,41088,57536,57344,41152,24704,8256,8192,24640,41088,57536,57344,41152,24704,8256,8192,24640,41088,57536,57344,41152,24704,8256,8192,24640,41088,57536,57344,41152,24704,8256]
@@ -848,6 +1329,37 @@ define <64 x i8> @constant_funnnel_v64i8(<64 x i8> %x, <64 x i8> %y) nounwind {
 ; AVX512VLBW-NEXT:    vmovdqu8 %zmm0, %zmm1 {%k1}
 ; AVX512VLBW-NEXT:    vmovdqa64 %zmm1, %zmm0
 ; AVX512VLBW-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: constant_funnnel_v64i8:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [8192,24640,41088,57536,57344,41152,24704,8256,8192,24640,41088,57536,57344,41152,24704,8256,8192,24640,41088,57536,57344,41152,24704,8256,8192,24640,41088,57536,57344,41152,24704,8256]
+; AVX512VLVBMI2-NEXT:    vpmovb2m %zmm2, %k1
+; AVX512VLVBMI2-NEXT:    vpsllw $4, %zmm0, %zmm3
+; AVX512VLVBMI2-NEXT:    vpandq {{.*}}(%rip), %zmm3, %zmm3
+; AVX512VLVBMI2-NEXT:    vpblendmb %zmm3, %zmm0, %zmm3 {%k1}
+; AVX512VLVBMI2-NEXT:    vpsllw $2, %zmm3, %zmm4
+; AVX512VLVBMI2-NEXT:    vpandq {{.*}}(%rip), %zmm4, %zmm4
+; AVX512VLVBMI2-NEXT:    vpaddb %zmm2, %zmm2, %zmm2
+; AVX512VLVBMI2-NEXT:    vpmovb2m %zmm2, %k1
+; AVX512VLVBMI2-NEXT:    vmovdqu8 %zmm4, %zmm3 {%k1}
+; AVX512VLVBMI2-NEXT:    vpaddb %zmm2, %zmm2, %zmm2
+; AVX512VLVBMI2-NEXT:    vpmovb2m %zmm2, %k1
+; AVX512VLVBMI2-NEXT:    vpaddb %zmm3, %zmm3, %zmm3 {%k1}
+; AVX512VLVBMI2-NEXT:    vpunpckhbw {{.*#+}} zmm2 = zmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]
+; AVX512VLVBMI2-NEXT:    vpsrlw $8, %zmm2, %zmm2
+; AVX512VLVBMI2-NEXT:    vpsllvw {{.*}}(%rip), %zmm2, %zmm2
+; AVX512VLVBMI2-NEXT:    vpsrlw $8, %zmm2, %zmm2
+; AVX512VLVBMI2-NEXT:    vpunpcklbw {{.*#+}} zmm1 = zmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
+; AVX512VLVBMI2-NEXT:    vpsrlw $8, %zmm1, %zmm1
+; AVX512VLVBMI2-NEXT:    vpsllvw {{.*}}(%rip), %zmm1, %zmm1
+; AVX512VLVBMI2-NEXT:    vpsrlw $8, %zmm1, %zmm1
+; AVX512VLVBMI2-NEXT:    vpackuswb %zmm2, %zmm1, %zmm1
+; AVX512VLVBMI2-NEXT:    vporq %zmm1, %zmm3, %zmm1
+; AVX512VLVBMI2-NEXT:    movabsq $72340172838076673, %rax # imm = 0x101010101010101
+; AVX512VLVBMI2-NEXT:    kmovq %rax, %k1
+; AVX512VLVBMI2-NEXT:    vmovdqu8 %zmm0, %zmm1 {%k1}
+; AVX512VLVBMI2-NEXT:    vmovdqa64 %zmm1, %zmm0
+; AVX512VLVBMI2-NEXT:    retq
   %res = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %x, <64 x i8> %y, <64 x i8> <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1>)
   ret <64 x i8> %res
 }
@@ -857,23 +1369,85 @@ define <64 x i8> @constant_funnnel_v64i8(<64 x i8> %x, <64 x i8> %y) nounwind {
 ;
 
 define <8 x i64> @splatconstant_funnnel_v8i64(<8 x i64> %x, <8 x i64> %y) nounwind {
-; AVX512-LABEL: splatconstant_funnnel_v8i64:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpsrlq $50, %zmm1, %zmm1
-; AVX512-NEXT:    vpsllq $14, %zmm0, %zmm0
-; AVX512-NEXT:    vporq %zmm1, %zmm0, %zmm0
-; AVX512-NEXT:    retq
+; AVX512F-LABEL: splatconstant_funnnel_v8i64:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpsrlq $50, %zmm1, %zmm1
+; AVX512F-NEXT:    vpsllq $14, %zmm0, %zmm0
+; AVX512F-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: splatconstant_funnnel_v8i64:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpsrlq $50, %zmm1, %zmm1
+; AVX512VL-NEXT:    vpsllq $14, %zmm0, %zmm0
+; AVX512VL-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: splatconstant_funnnel_v8i64:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpsrlq $50, %zmm1, %zmm1
+; AVX512BW-NEXT:    vpsllq $14, %zmm0, %zmm0
+; AVX512BW-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    retq
+;
+; AVX512VBMI2-LABEL: splatconstant_funnnel_v8i64:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    vpshldq $14, %zmm1, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLBW-LABEL: splatconstant_funnnel_v8i64:
+; AVX512VLBW:       # %bb.0:
+; AVX512VLBW-NEXT:    vpsrlq $50, %zmm1, %zmm1
+; AVX512VLBW-NEXT:    vpsllq $14, %zmm0, %zmm0
+; AVX512VLBW-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512VLBW-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: splatconstant_funnnel_v8i64:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpshldq $14, %zmm1, %zmm0, %zmm0
+; AVX512VLVBMI2-NEXT:    retq
   %res = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %x, <8 x i64> %y, <8 x i64> <i64 14, i64 14, i64 14, i64 14, i64 14, i64 14, i64 14, i64 14>)
   ret <8 x i64> %res
 }
 
 define <16 x i32> @splatconstant_funnnel_v16i32(<16 x i32> %x, <16 x i32> %y) nounwind {
-; AVX512-LABEL: splatconstant_funnnel_v16i32:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpsrld $28, %zmm1, %zmm1
-; AVX512-NEXT:    vpslld $4, %zmm0, %zmm0
-; AVX512-NEXT:    vpord %zmm1, %zmm0, %zmm0
-; AVX512-NEXT:    retq
+; AVX512F-LABEL: splatconstant_funnnel_v16i32:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpsrld $28, %zmm1, %zmm1
+; AVX512F-NEXT:    vpslld $4, %zmm0, %zmm0
+; AVX512F-NEXT:    vpord %zmm1, %zmm0, %zmm0
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: splatconstant_funnnel_v16i32:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpsrld $28, %zmm1, %zmm1
+; AVX512VL-NEXT:    vpslld $4, %zmm0, %zmm0
+; AVX512VL-NEXT:    vpord %zmm1, %zmm0, %zmm0
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: splatconstant_funnnel_v16i32:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpsrld $28, %zmm1, %zmm1
+; AVX512BW-NEXT:    vpslld $4, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpord %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    retq
+;
+; AVX512VBMI2-LABEL: splatconstant_funnnel_v16i32:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    vpshldd $4, %zmm1, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLBW-LABEL: splatconstant_funnnel_v16i32:
+; AVX512VLBW:       # %bb.0:
+; AVX512VLBW-NEXT:    vpsrld $28, %zmm1, %zmm1
+; AVX512VLBW-NEXT:    vpslld $4, %zmm0, %zmm0
+; AVX512VLBW-NEXT:    vpord %zmm1, %zmm0, %zmm0
+; AVX512VLBW-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: splatconstant_funnnel_v16i32:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpshldd $4, %zmm1, %zmm0, %zmm0
+; AVX512VLVBMI2-NEXT:    retq
   %res = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %x, <16 x i32> %y, <16 x i32> <i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4>)
   ret <16 x i32> %res
 }
@@ -906,12 +1480,22 @@ define <32 x i16> @splatconstant_funnnel_v32i16(<32 x i16> %x, <32 x i16> %y) no
 ; AVX512BW-NEXT:    vporq %zmm1, %zmm0, %zmm0
 ; AVX512BW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: splatconstant_funnnel_v32i16:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    vpshldw $7, %zmm1, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    retq
+;
 ; AVX512VLBW-LABEL: splatconstant_funnnel_v32i16:
 ; AVX512VLBW:       # %bb.0:
 ; AVX512VLBW-NEXT:    vpsrlw $9, %zmm1, %zmm1
 ; AVX512VLBW-NEXT:    vpsllw $7, %zmm0, %zmm0
 ; AVX512VLBW-NEXT:    vporq %zmm1, %zmm0, %zmm0
 ; AVX512VLBW-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: splatconstant_funnnel_v32i16:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpshldw $7, %zmm1, %zmm0, %zmm0
+; AVX512VLVBMI2-NEXT:    retq
   %res = call <32 x i16> @llvm.fshl.v32i16(<32 x i16> %x, <32 x i16> %y, <32 x i16> <i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7>)
   ret <32 x i16> %res
 }
@@ -920,32 +1504,30 @@ define <64 x i8> @splatconstant_funnnel_v64i8(<64 x i8> %x, <64 x i8> %y) nounwi
 ; AVX512F-LABEL: splatconstant_funnnel_v64i8:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vpsrlw $4, %ymm2, %ymm2
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm4 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; AVX512F-NEXT:    vpand %ymm4, %ymm2, %ymm2
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm4 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
+; AVX512F-NEXT:    vpandn %ymm2, %ymm4, %ymm2
 ; AVX512F-NEXT:    vpsllw $4, %ymm0, %ymm0
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm5 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
-; AVX512F-NEXT:    vpand %ymm5, %ymm0, %ymm0
+; AVX512F-NEXT:    vpand %ymm4, %ymm0, %ymm0
 ; AVX512F-NEXT:    vpor %ymm2, %ymm0, %ymm0
 ; AVX512F-NEXT:    vpsrlw $4, %ymm3, %ymm2
-; AVX512F-NEXT:    vpand %ymm4, %ymm2, %ymm2
+; AVX512F-NEXT:    vpandn %ymm2, %ymm4, %ymm2
 ; AVX512F-NEXT:    vpsllw $4, %ymm1, %ymm1
-; AVX512F-NEXT:    vpand %ymm5, %ymm1, %ymm1
+; AVX512F-NEXT:    vpand %ymm4, %ymm1, %ymm1
 ; AVX512F-NEXT:    vpor %ymm2, %ymm1, %ymm1
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: splatconstant_funnnel_v64i8:
 ; AVX512VL:       # %bb.0:
 ; AVX512VL-NEXT:    vpsrlw $4, %ymm2, %ymm2
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm4 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; AVX512VL-NEXT:    vpand %ymm4, %ymm2, %ymm2
+; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm4 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
+; AVX512VL-NEXT:    vpandn %ymm2, %ymm4, %ymm2
 ; AVX512VL-NEXT:    vpsllw $4, %ymm0, %ymm0
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm5 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
-; AVX512VL-NEXT:    vpand %ymm5, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpand %ymm4, %ymm0, %ymm0
 ; AVX512VL-NEXT:    vpor %ymm2, %ymm0, %ymm0
 ; AVX512VL-NEXT:    vpsrlw $4, %ymm3, %ymm2
-; AVX512VL-NEXT:    vpand %ymm4, %ymm2, %ymm2
+; AVX512VL-NEXT:    vpandn %ymm2, %ymm4, %ymm2
 ; AVX512VL-NEXT:    vpsllw $4, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpand %ymm5, %ymm1, %ymm1
+; AVX512VL-NEXT:    vpand %ymm4, %ymm1, %ymm1
 ; AVX512VL-NEXT:    vpor %ymm2, %ymm1, %ymm1
 ; AVX512VL-NEXT:    retq
 ;
@@ -958,6 +1540,15 @@ define <64 x i8> @splatconstant_funnnel_v64i8(<64 x i8> %x, <64 x i8> %y) nounwi
 ; AVX512BW-NEXT:    vporq %zmm1, %zmm0, %zmm0
 ; AVX512BW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: splatconstant_funnnel_v64i8:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    vpsrlw $4, %zmm1, %zmm1
+; AVX512VBMI2-NEXT:    vpandq {{.*}}(%rip), %zmm1, %zmm1
+; AVX512VBMI2-NEXT:    vpsllw $4, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    vpandq {{.*}}(%rip), %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    retq
+;
 ; AVX512VLBW-LABEL: splatconstant_funnnel_v64i8:
 ; AVX512VLBW:       # %bb.0:
 ; AVX512VLBW-NEXT:    vpsrlw $4, %zmm1, %zmm1
@@ -966,6 +1557,15 @@ define <64 x i8> @splatconstant_funnnel_v64i8(<64 x i8> %x, <64 x i8> %y) nounwi
 ; AVX512VLBW-NEXT:    vpandq {{.*}}(%rip), %zmm0, %zmm0
 ; AVX512VLBW-NEXT:    vporq %zmm1, %zmm0, %zmm0
 ; AVX512VLBW-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: splatconstant_funnnel_v64i8:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpsrlw $4, %zmm1, %zmm1
+; AVX512VLVBMI2-NEXT:    vpandq {{.*}}(%rip), %zmm1, %zmm1
+; AVX512VLVBMI2-NEXT:    vpsllw $4, %zmm0, %zmm0
+; AVX512VLVBMI2-NEXT:    vpandq {{.*}}(%rip), %zmm0, %zmm0
+; AVX512VLVBMI2-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512VLVBMI2-NEXT:    retq
   %res = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %x, <64 x i8> %y, <64 x i8> <i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4>)
   ret <64 x i8> %res
 }
diff --git a/test/CodeGen/X86/vector-fshl-rot-128.ll b/test/CodeGen/X86/vector-fshl-rot-128.ll
index a358df441cee7e6aa80e7f9330a707e92e3bc069..2e12eeabffa0e16f18600317ba95bac03f56570a 100644
--- a/test/CodeGen/X86/vector-fshl-rot-128.ll
+++ b/test/CodeGen/X86/vector-fshl-rot-128.ll
@@ -95,42 +95,38 @@ define <2 x i64> @var_funnnel_v2i64(<2 x i64> %x, <2 x i64> %amt) nounwind {
 ; AVX2-NEXT:    vpor %xmm0, %xmm3, %xmm0
 ; AVX2-NEXT:    retq
 ;
-; AVX512-LABEL: var_funnnel_v2i64:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vmovdqa {{.*#+}} xmm2 = [63,63]
-; AVX512-NEXT:    vpand %xmm2, %xmm1, %xmm3
-; AVX512-NEXT:    vpsllvq %xmm3, %xmm0, %xmm3
-; AVX512-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX512-NEXT:    vpsubq %xmm1, %xmm4, %xmm1
-; AVX512-NEXT:    vpand %xmm2, %xmm1, %xmm1
-; AVX512-NEXT:    vpsrlvq %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpor %xmm0, %xmm3, %xmm0
-; AVX512-NEXT:    retq
+; AVX512F-LABEL: var_funnnel_v2i64:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512F-NEXT:    vprolvq %zmm1, %zmm0, %zmm0
+; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
 ;
-; XOPAVX1-LABEL: var_funnnel_v2i64:
-; XOPAVX1:       # %bb.0:
-; XOPAVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [63,63]
-; XOPAVX1-NEXT:    vpand %xmm2, %xmm1, %xmm3
-; XOPAVX1-NEXT:    vpshlq %xmm3, %xmm0, %xmm3
-; XOPAVX1-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; XOPAVX1-NEXT:    vpsubq %xmm1, %xmm4, %xmm1
-; XOPAVX1-NEXT:    vpand %xmm2, %xmm1, %xmm1
-; XOPAVX1-NEXT:    vpsubq %xmm1, %xmm4, %xmm1
-; XOPAVX1-NEXT:    vpshlq %xmm1, %xmm0, %xmm0
-; XOPAVX1-NEXT:    vpor %xmm0, %xmm3, %xmm0
-; XOPAVX1-NEXT:    retq
+; AVX512VL-LABEL: var_funnnel_v2i64:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vprolvq %xmm1, %xmm0, %xmm0
+; AVX512VL-NEXT:    retq
 ;
-; XOPAVX2-LABEL: var_funnnel_v2i64:
-; XOPAVX2:       # %bb.0:
-; XOPAVX2-NEXT:    vmovdqa {{.*#+}} xmm2 = [63,63]
-; XOPAVX2-NEXT:    vpand %xmm2, %xmm1, %xmm3
-; XOPAVX2-NEXT:    vpsllvq %xmm3, %xmm0, %xmm3
-; XOPAVX2-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; XOPAVX2-NEXT:    vpsubq %xmm1, %xmm4, %xmm1
-; XOPAVX2-NEXT:    vpand %xmm2, %xmm1, %xmm1
-; XOPAVX2-NEXT:    vpsrlvq %xmm1, %xmm0, %xmm0
-; XOPAVX2-NEXT:    vpor %xmm0, %xmm3, %xmm0
-; XOPAVX2-NEXT:    retq
+; AVX512BW-LABEL: var_funnnel_v2i64:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512BW-NEXT:    vprolvq %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+;
+; AVX512VLBW-LABEL: var_funnnel_v2i64:
+; AVX512VLBW:       # %bb.0:
+; AVX512VLBW-NEXT:    vprolvq %xmm1, %xmm0, %xmm0
+; AVX512VLBW-NEXT:    retq
+;
+; XOP-LABEL: var_funnnel_v2i64:
+; XOP:       # %bb.0:
+; XOP-NEXT:    vprotq %xmm1, %xmm0, %xmm0
+; XOP-NEXT:    retq
 ;
 ; X32-SSE-LABEL: var_funnnel_v2i64:
 ; X32-SSE:       # %bb.0:
@@ -159,180 +155,118 @@ define <2 x i64> @var_funnnel_v2i64(<2 x i64> %x, <2 x i64> %amt) nounwind {
 define <4 x i32> @var_funnnel_v4i32(<4 x i32> %x, <4 x i32> %amt) nounwind {
 ; SSE2-LABEL: var_funnnel_v4i32:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pxor %xmm3, %xmm3
-; SSE2-NEXT:    psubd %xmm1, %xmm3
-; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [31,31,31,31]
-; SSE2-NEXT:    pand %xmm4, %xmm3
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm2 = xmm3[2,3,3,3,4,5,6,7]
-; SSE2-NEXT:    movdqa %xmm0, %xmm5
-; SSE2-NEXT:    psrld %xmm2, %xmm5
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm6 = xmm3[0,1,1,1,4,5,6,7]
-; SSE2-NEXT:    movdqa %xmm0, %xmm2
-; SSE2-NEXT:    psrld %xmm6, %xmm2
-; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm5[0]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[2,3,0,1]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm5 = xmm3[2,3,3,3,4,5,6,7]
-; SSE2-NEXT:    movdqa %xmm0, %xmm6
-; SSE2-NEXT:    psrld %xmm5, %xmm6
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm3 = xmm3[0,1,1,1,4,5,6,7]
-; SSE2-NEXT:    movdqa %xmm0, %xmm5
-; SSE2-NEXT:    psrld %xmm3, %xmm5
-; SSE2-NEXT:    punpckhqdq {{.*#+}} xmm5 = xmm5[1],xmm6[1]
-; SSE2-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,3],xmm5[0,3]
-; SSE2-NEXT:    pand %xmm4, %xmm1
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm1
 ; SSE2-NEXT:    pslld $23, %xmm1
 ; SSE2-NEXT:    paddd {{.*}}(%rip), %xmm1
 ; SSE2-NEXT:    cvttps2dq %xmm1, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
 ; SSE2-NEXT:    pmuludq %xmm1, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,3,2,3]
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pmuludq %xmm3, %xmm1
+; SSE2-NEXT:    pmuludq %xmm2, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,3,2,3]
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
 ; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; SSE2-NEXT:    orps %xmm0, %xmm2
-; SSE2-NEXT:    movaps %xmm2, %xmm0
+; SSE2-NEXT:    por %xmm3, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: var_funnnel_v4i32:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    pxor %xmm2, %xmm2
-; SSE41-NEXT:    psubd %xmm1, %xmm2
-; SSE41-NEXT:    movdqa {{.*#+}} xmm3 = [31,31,31,31]
-; SSE41-NEXT:    pand %xmm3, %xmm2
-; SSE41-NEXT:    pshuflw {{.*#+}} xmm4 = xmm2[2,3,3,3,4,5,6,7]
-; SSE41-NEXT:    movdqa %xmm0, %xmm5
-; SSE41-NEXT:    psrld %xmm4, %xmm5
-; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[2,3,0,1]
-; SSE41-NEXT:    pshuflw {{.*#+}} xmm6 = xmm4[2,3,3,3,4,5,6,7]
-; SSE41-NEXT:    movdqa %xmm0, %xmm7
-; SSE41-NEXT:    psrld %xmm6, %xmm7
-; SSE41-NEXT:    pblendw {{.*#+}} xmm7 = xmm5[0,1,2,3],xmm7[4,5,6,7]
-; SSE41-NEXT:    pshuflw {{.*#+}} xmm2 = xmm2[0,1,1,1,4,5,6,7]
-; SSE41-NEXT:    movdqa %xmm0, %xmm5
-; SSE41-NEXT:    psrld %xmm2, %xmm5
-; SSE41-NEXT:    pshuflw {{.*#+}} xmm2 = xmm4[0,1,1,1,4,5,6,7]
-; SSE41-NEXT:    movdqa %xmm0, %xmm4
-; SSE41-NEXT:    psrld %xmm2, %xmm4
-; SSE41-NEXT:    pblendw {{.*#+}} xmm4 = xmm5[0,1,2,3],xmm4[4,5,6,7]
-; SSE41-NEXT:    pblendw {{.*#+}} xmm4 = xmm4[0,1],xmm7[2,3],xmm4[4,5],xmm7[6,7]
-; SSE41-NEXT:    pand %xmm3, %xmm1
+; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
+; SSE41-NEXT:    pand {{.*}}(%rip), %xmm1
 ; SSE41-NEXT:    pslld $23, %xmm1
 ; SSE41-NEXT:    paddd {{.*}}(%rip), %xmm1
 ; SSE41-NEXT:    cvttps2dq %xmm1, %xmm1
-; SSE41-NEXT:    pmulld %xmm1, %xmm0
-; SSE41-NEXT:    por %xmm4, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[1,1,3,3]
+; SSE41-NEXT:    pmuludq %xmm2, %xmm3
+; SSE41-NEXT:    pmuludq %xmm1, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm3[2,3],xmm1[4,5],xmm3[6,7]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
+; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3],xmm0[4,5],xmm2[6,7]
+; SSE41-NEXT:    por %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: var_funnnel_v4i32:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpsubd %xmm1, %xmm2, %xmm3
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = [31,31,31,31]
-; AVX1-NEXT:    vpand %xmm4, %xmm3, %xmm3
-; AVX1-NEXT:    vpsrldq {{.*#+}} xmm5 = xmm3[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX1-NEXT:    vpsrld %xmm5, %xmm0, %xmm5
-; AVX1-NEXT:    vpsrlq $32, %xmm3, %xmm6
-; AVX1-NEXT:    vpsrld %xmm6, %xmm0, %xmm6
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm5 = xmm6[0,1,2,3],xmm5[4,5,6,7]
-; AVX1-NEXT:    vpunpckhdq {{.*#+}} xmm2 = xmm3[2],xmm2[2],xmm3[3],xmm2[3]
-; AVX1-NEXT:    vpsrld %xmm2, %xmm0, %xmm2
-; AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm3 = xmm3[0],zero,xmm3[1],zero
-; AVX1-NEXT:    vpsrld %xmm3, %xmm0, %xmm3
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0,1,2,3],xmm2[4,5,6,7]
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0,1],xmm5[2,3],xmm2[4,5],xmm5[6,7]
-; AVX1-NEXT:    vpand %xmm4, %xmm1, %xmm1
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
+; AVX1-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
 ; AVX1-NEXT:    vpslld $23, %xmm1, %xmm1
 ; AVX1-NEXT:    vpaddd {{.*}}(%rip), %xmm1, %xmm1
 ; AVX1-NEXT:    vcvttps2dq %xmm1, %xmm1
-; AVX1-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpor %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[1,1,3,3]
+; AVX1-NEXT:    vpmuludq %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[0,0,2,2]
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3],xmm0[4,5],xmm2[6,7]
+; AVX1-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: var_funnnel_v4i32:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [31,31,31,31]
-; AVX2-NEXT:    vpand %xmm2, %xmm1, %xmm3
-; AVX2-NEXT:    vpsllvd %xmm3, %xmm0, %xmm3
-; AVX2-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX2-NEXT:    vpsubd %xmm1, %xmm4, %xmm1
 ; AVX2-NEXT:    vpand %xmm2, %xmm1, %xmm1
+; AVX2-NEXT:    vpsllvd %xmm1, %xmm0, %xmm2
+; AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm3 = [32,32,32,32]
+; AVX2-NEXT:    vpsubd %xmm1, %xmm3, %xmm1
 ; AVX2-NEXT:    vpsrlvd %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpor %xmm0, %xmm3, %xmm0
+; AVX2-NEXT:    vpor %xmm0, %xmm2, %xmm0
 ; AVX2-NEXT:    retq
 ;
-; AVX512-LABEL: var_funnnel_v4i32:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [31,31,31,31]
-; AVX512-NEXT:    vpand %xmm2, %xmm1, %xmm3
-; AVX512-NEXT:    vpsllvd %xmm3, %xmm0, %xmm3
-; AVX512-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX512-NEXT:    vpsubd %xmm1, %xmm4, %xmm1
-; AVX512-NEXT:    vpand %xmm2, %xmm1, %xmm1
-; AVX512-NEXT:    vpsrlvd %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpor %xmm0, %xmm3, %xmm0
-; AVX512-NEXT:    retq
+; AVX512F-LABEL: var_funnnel_v4i32:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512F-NEXT:    vprolvd %zmm1, %zmm0, %zmm0
+; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
 ;
-; XOPAVX1-LABEL: var_funnnel_v4i32:
-; XOPAVX1:       # %bb.0:
-; XOPAVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [31,31,31,31]
-; XOPAVX1-NEXT:    vpand %xmm2, %xmm1, %xmm3
-; XOPAVX1-NEXT:    vpshld %xmm3, %xmm0, %xmm3
-; XOPAVX1-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; XOPAVX1-NEXT:    vpsubd %xmm1, %xmm4, %xmm1
-; XOPAVX1-NEXT:    vpand %xmm2, %xmm1, %xmm1
-; XOPAVX1-NEXT:    vpsubd %xmm1, %xmm4, %xmm1
-; XOPAVX1-NEXT:    vpshld %xmm1, %xmm0, %xmm0
-; XOPAVX1-NEXT:    vpor %xmm0, %xmm3, %xmm0
-; XOPAVX1-NEXT:    retq
+; AVX512VL-LABEL: var_funnnel_v4i32:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vprolvd %xmm1, %xmm0, %xmm0
+; AVX512VL-NEXT:    retq
 ;
-; XOPAVX2-LABEL: var_funnnel_v4i32:
-; XOPAVX2:       # %bb.0:
-; XOPAVX2-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [31,31,31,31]
-; XOPAVX2-NEXT:    vpand %xmm2, %xmm1, %xmm3
-; XOPAVX2-NEXT:    vpsllvd %xmm3, %xmm0, %xmm3
-; XOPAVX2-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; XOPAVX2-NEXT:    vpsubd %xmm1, %xmm4, %xmm1
-; XOPAVX2-NEXT:    vpand %xmm2, %xmm1, %xmm1
-; XOPAVX2-NEXT:    vpsrlvd %xmm1, %xmm0, %xmm0
-; XOPAVX2-NEXT:    vpor %xmm0, %xmm3, %xmm0
-; XOPAVX2-NEXT:    retq
+; AVX512BW-LABEL: var_funnnel_v4i32:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512BW-NEXT:    vprolvd %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+;
+; AVX512VLBW-LABEL: var_funnnel_v4i32:
+; AVX512VLBW:       # %bb.0:
+; AVX512VLBW-NEXT:    vprolvd %xmm1, %xmm0, %xmm0
+; AVX512VLBW-NEXT:    retq
+;
+; XOP-LABEL: var_funnnel_v4i32:
+; XOP:       # %bb.0:
+; XOP-NEXT:    vprotd %xmm1, %xmm0, %xmm0
+; XOP-NEXT:    retq
 ;
 ; X32-SSE-LABEL: var_funnnel_v4i32:
 ; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    pxor %xmm3, %xmm3
-; X32-SSE-NEXT:    psubd %xmm1, %xmm3
-; X32-SSE-NEXT:    movdqa {{.*#+}} xmm4 = [31,31,31,31]
-; X32-SSE-NEXT:    pand %xmm4, %xmm3
-; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm2 = xmm3[2,3,3,3,4,5,6,7]
-; X32-SSE-NEXT:    movdqa %xmm0, %xmm5
-; X32-SSE-NEXT:    psrld %xmm2, %xmm5
-; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm6 = xmm3[0,1,1,1,4,5,6,7]
-; X32-SSE-NEXT:    movdqa %xmm0, %xmm2
-; X32-SSE-NEXT:    psrld %xmm6, %xmm2
-; X32-SSE-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm5[0]
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[2,3,0,1]
-; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm5 = xmm3[2,3,3,3,4,5,6,7]
-; X32-SSE-NEXT:    movdqa %xmm0, %xmm6
-; X32-SSE-NEXT:    psrld %xmm5, %xmm6
-; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm3 = xmm3[0,1,1,1,4,5,6,7]
-; X32-SSE-NEXT:    movdqa %xmm0, %xmm5
-; X32-SSE-NEXT:    psrld %xmm3, %xmm5
-; X32-SSE-NEXT:    punpckhqdq {{.*#+}} xmm5 = xmm5[1],xmm6[1]
-; X32-SSE-NEXT:    shufps {{.*#+}} xmm2 = xmm2[0,3],xmm5[0,3]
-; X32-SSE-NEXT:    pand %xmm4, %xmm1
+; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm1
 ; X32-SSE-NEXT:    pslld $23, %xmm1
 ; X32-SSE-NEXT:    paddd {{\.LCPI.*}}, %xmm1
 ; X32-SSE-NEXT:    cvttps2dq %xmm1, %xmm1
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; X32-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
 ; X32-SSE-NEXT:    pmuludq %xmm1, %xmm0
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; X32-SSE-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,3,2,3]
 ; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; X32-SSE-NEXT:    pmuludq %xmm3, %xmm1
+; X32-SSE-NEXT:    pmuludq %xmm2, %xmm1
+; X32-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,3,2,3]
+; X32-SSE-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
+; X32-SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
 ; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
 ; X32-SSE-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; X32-SSE-NEXT:    orps %xmm0, %xmm2
-; X32-SSE-NEXT:    movaps %xmm2, %xmm0
+; X32-SSE-NEXT:    por %xmm3, %xmm0
 ; X32-SSE-NEXT:    retl
   %res = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %x, <4 x i32> %x, <4 x i32> %amt)
   ret <4 x i32> %res
@@ -341,180 +275,98 @@ define <4 x i32> @var_funnnel_v4i32(<4 x i32> %x, <4 x i32> %amt) nounwind {
 define <8 x i16> @var_funnnel_v8i16(<8 x i16> %x, <8 x i16> %amt) nounwind {
 ; SSE2-LABEL: var_funnnel_v8i16:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pxor %xmm3, %xmm3
-; SSE2-NEXT:    pxor %xmm4, %xmm4
-; SSE2-NEXT:    psubw %xmm1, %xmm4
-; SSE2-NEXT:    psllw $12, %xmm4
-; SSE2-NEXT:    movdqa %xmm4, %xmm2
-; SSE2-NEXT:    psraw $15, %xmm2
-; SSE2-NEXT:    movdqa %xmm0, %xmm5
-; SSE2-NEXT:    psrlw $8, %xmm5
-; SSE2-NEXT:    pand %xmm2, %xmm5
-; SSE2-NEXT:    pandn %xmm0, %xmm2
-; SSE2-NEXT:    por %xmm5, %xmm2
-; SSE2-NEXT:    paddw %xmm4, %xmm4
-; SSE2-NEXT:    movdqa %xmm4, %xmm5
-; SSE2-NEXT:    psraw $15, %xmm5
-; SSE2-NEXT:    movdqa %xmm5, %xmm6
-; SSE2-NEXT:    pandn %xmm2, %xmm6
-; SSE2-NEXT:    psrlw $4, %xmm2
-; SSE2-NEXT:    pand %xmm5, %xmm2
-; SSE2-NEXT:    por %xmm6, %xmm2
-; SSE2-NEXT:    paddw %xmm4, %xmm4
-; SSE2-NEXT:    movdqa %xmm4, %xmm5
-; SSE2-NEXT:    psraw $15, %xmm5
-; SSE2-NEXT:    movdqa %xmm5, %xmm6
-; SSE2-NEXT:    pandn %xmm2, %xmm6
-; SSE2-NEXT:    psrlw $2, %xmm2
-; SSE2-NEXT:    pand %xmm5, %xmm2
-; SSE2-NEXT:    por %xmm6, %xmm2
-; SSE2-NEXT:    paddw %xmm4, %xmm4
-; SSE2-NEXT:    psraw $15, %xmm4
-; SSE2-NEXT:    movdqa %xmm4, %xmm5
-; SSE2-NEXT:    pandn %xmm2, %xmm5
-; SSE2-NEXT:    psrlw $1, %xmm2
-; SSE2-NEXT:    pand %xmm4, %xmm2
 ; SSE2-NEXT:    pand {{.*}}(%rip), %xmm1
-; SSE2-NEXT:    movdqa %xmm1, %xmm4
-; SSE2-NEXT:    punpckhwd {{.*#+}} xmm4 = xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7]
-; SSE2-NEXT:    pslld $23, %xmm4
-; SSE2-NEXT:    movdqa {{.*#+}} xmm6 = [1065353216,1065353216,1065353216,1065353216]
-; SSE2-NEXT:    paddd %xmm6, %xmm4
-; SSE2-NEXT:    cvttps2dq %xmm4, %xmm4
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm4 = xmm4[0,2,2,3,4,5,6,7]
-; SSE2-NEXT:    pshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,4,6,6,7]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[0,2,2,3]
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3]
+; SSE2-NEXT:    pxor %xmm2, %xmm2
+; SSE2-NEXT:    movdqa %xmm1, %xmm3
+; SSE2-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
+; SSE2-NEXT:    pslld $23, %xmm3
+; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [1065353216,1065353216,1065353216,1065353216]
+; SSE2-NEXT:    paddd %xmm4, %xmm3
+; SSE2-NEXT:    cvttps2dq %xmm3, %xmm3
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm3 = xmm3[0,2,2,3,4,5,6,7]
+; SSE2-NEXT:    pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,6,6,7]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3]
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
 ; SSE2-NEXT:    pslld $23, %xmm1
-; SSE2-NEXT:    paddd %xmm6, %xmm1
+; SSE2-NEXT:    paddd %xmm4, %xmm1
 ; SSE2-NEXT:    cvttps2dq %xmm1, %xmm1
 ; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,2,2,3,4,5,6,7]
 ; SSE2-NEXT:    pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,6,6,7]
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
-; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm4[0]
-; SSE2-NEXT:    pmullw %xmm0, %xmm1
-; SSE2-NEXT:    por %xmm5, %xmm1
-; SSE2-NEXT:    por %xmm2, %xmm1
-; SSE2-NEXT:    movdqa %xmm1, %xmm0
+; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm3[0]
+; SSE2-NEXT:    movdqa %xmm0, %xmm2
+; SSE2-NEXT:    pmulhuw %xmm1, %xmm2
+; SSE2-NEXT:    pmullw %xmm1, %xmm0
+; SSE2-NEXT:    por %xmm2, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: var_funnnel_v8i16:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    movdqa %xmm0, %xmm2
-; SSE41-NEXT:    pxor %xmm4, %xmm4
-; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    psubw %xmm1, %xmm0
-; SSE41-NEXT:    movdqa {{.*#+}} xmm6 = [15,15,15,15,15,15,15,15]
-; SSE41-NEXT:    pand %xmm6, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm3
-; SSE41-NEXT:    psllw $12, %xmm3
-; SSE41-NEXT:    psllw $4, %xmm0
-; SSE41-NEXT:    por %xmm3, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm3
-; SSE41-NEXT:    paddw %xmm0, %xmm3
-; SSE41-NEXT:    movdqa %xmm2, %xmm7
-; SSE41-NEXT:    psrlw $8, %xmm7
-; SSE41-NEXT:    movdqa %xmm2, %xmm5
-; SSE41-NEXT:    pblendvb %xmm0, %xmm7, %xmm5
-; SSE41-NEXT:    movdqa %xmm5, %xmm7
-; SSE41-NEXT:    psrlw $4, %xmm7
-; SSE41-NEXT:    movdqa %xmm3, %xmm0
-; SSE41-NEXT:    pblendvb %xmm0, %xmm7, %xmm5
-; SSE41-NEXT:    movdqa %xmm5, %xmm7
-; SSE41-NEXT:    psrlw $2, %xmm7
-; SSE41-NEXT:    paddw %xmm3, %xmm3
-; SSE41-NEXT:    movdqa %xmm3, %xmm0
-; SSE41-NEXT:    pblendvb %xmm0, %xmm7, %xmm5
-; SSE41-NEXT:    movdqa %xmm5, %xmm7
-; SSE41-NEXT:    psrlw $1, %xmm7
-; SSE41-NEXT:    paddw %xmm3, %xmm3
-; SSE41-NEXT:    movdqa %xmm3, %xmm0
-; SSE41-NEXT:    pblendvb %xmm0, %xmm7, %xmm5
-; SSE41-NEXT:    pand %xmm6, %xmm1
-; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm0 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
-; SSE41-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm4[4],xmm1[5],xmm4[5],xmm1[6],xmm4[6],xmm1[7],xmm4[7]
+; SSE41-NEXT:    pand {{.*}}(%rip), %xmm1
+; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
+; SSE41-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
 ; SSE41-NEXT:    pslld $23, %xmm1
 ; SSE41-NEXT:    movdqa {{.*#+}} xmm3 = [1065353216,1065353216,1065353216,1065353216]
 ; SSE41-NEXT:    paddd %xmm3, %xmm1
 ; SSE41-NEXT:    cvttps2dq %xmm1, %xmm1
-; SSE41-NEXT:    pslld $23, %xmm0
-; SSE41-NEXT:    paddd %xmm3, %xmm0
-; SSE41-NEXT:    cvttps2dq %xmm0, %xmm0
-; SSE41-NEXT:    packusdw %xmm1, %xmm0
-; SSE41-NEXT:    pmullw %xmm0, %xmm2
-; SSE41-NEXT:    por %xmm5, %xmm2
-; SSE41-NEXT:    movdqa %xmm2, %xmm0
+; SSE41-NEXT:    pslld $23, %xmm2
+; SSE41-NEXT:    paddd %xmm3, %xmm2
+; SSE41-NEXT:    cvttps2dq %xmm2, %xmm2
+; SSE41-NEXT:    packusdw %xmm1, %xmm2
+; SSE41-NEXT:    movdqa %xmm0, %xmm1
+; SSE41-NEXT:    pmulhuw %xmm2, %xmm1
+; SSE41-NEXT:    pmullw %xmm2, %xmm0
+; SSE41-NEXT:    por %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: var_funnnel_v8i16:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpsubw %xmm1, %xmm2, %xmm3
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = [15,15,15,15,15,15,15,15]
-; AVX1-NEXT:    vpand %xmm4, %xmm3, %xmm3
-; AVX1-NEXT:    vpsllw $12, %xmm3, %xmm5
-; AVX1-NEXT:    vpsllw $4, %xmm3, %xmm3
-; AVX1-NEXT:    vpor %xmm5, %xmm3, %xmm3
-; AVX1-NEXT:    vpaddw %xmm3, %xmm3, %xmm5
-; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm6
-; AVX1-NEXT:    vpblendvb %xmm3, %xmm6, %xmm0, %xmm3
-; AVX1-NEXT:    vpsrlw $4, %xmm3, %xmm6
-; AVX1-NEXT:    vpblendvb %xmm5, %xmm6, %xmm3, %xmm3
-; AVX1-NEXT:    vpsrlw $2, %xmm3, %xmm6
-; AVX1-NEXT:    vpaddw %xmm5, %xmm5, %xmm5
-; AVX1-NEXT:    vpblendvb %xmm5, %xmm6, %xmm3, %xmm3
-; AVX1-NEXT:    vpsrlw $1, %xmm3, %xmm6
-; AVX1-NEXT:    vpaddw %xmm5, %xmm5, %xmm5
-; AVX1-NEXT:    vpblendvb %xmm5, %xmm6, %xmm3, %xmm3
-; AVX1-NEXT:    vpand %xmm4, %xmm1, %xmm1
-; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
+; AVX1-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
 ; AVX1-NEXT:    vpslld $23, %xmm2, %xmm2
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = [1065353216,1065353216,1065353216,1065353216]
-; AVX1-NEXT:    vpaddd %xmm4, %xmm2, %xmm2
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [1065353216,1065353216,1065353216,1065353216]
+; AVX1-NEXT:    vpaddd %xmm3, %xmm2, %xmm2
 ; AVX1-NEXT:    vcvttps2dq %xmm2, %xmm2
 ; AVX1-NEXT:    vpmovzxwd {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
 ; AVX1-NEXT:    vpslld $23, %xmm1, %xmm1
-; AVX1-NEXT:    vpaddd %xmm4, %xmm1, %xmm1
+; AVX1-NEXT:    vpaddd %xmm3, %xmm1, %xmm1
 ; AVX1-NEXT:    vcvttps2dq %xmm1, %xmm1
 ; AVX1-NEXT:    vpackusdw %xmm2, %xmm1, %xmm1
+; AVX1-NEXT:    vpmulhuw %xmm1, %xmm0, %xmm2
 ; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpor %xmm3, %xmm0, %xmm0
+; AVX1-NEXT:    vpor %xmm2, %xmm0, %xmm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: var_funnnel_v8i16:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vmovdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15]
-; AVX2-NEXT:    vpand %xmm2, %xmm1, %xmm3
-; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm3 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero
 ; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; AVX2-NEXT:    vpsllvd %ymm3, %ymm0, %ymm3
-; AVX2-NEXT:    vmovdqa {{.*#+}} ymm4 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15,16,17,20,21,24,25,28,29,24,25,28,29,28,29,30,31]
-; AVX2-NEXT:    vpshufb %ymm4, %ymm3, %ymm3
-; AVX2-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[0,2,2,3]
-; AVX2-NEXT:    vpxor %xmm5, %xmm5, %xmm5
-; AVX2-NEXT:    vpsubw %xmm1, %xmm5, %xmm1
-; AVX2-NEXT:    vpand %xmm2, %xmm1, %xmm1
+; AVX2-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; AVX2-NEXT:    vpsllvd %ymm2, %ymm0, %ymm2
+; AVX2-NEXT:    vmovdqa {{.*#+}} ymm3 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15,16,17,20,21,24,25,28,29,24,25,28,29,28,29,30,31]
+; AVX2-NEXT:    vpshufb %ymm3, %ymm2, %ymm2
+; AVX2-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[0,2,2,3]
+; AVX2-NEXT:    vmovdqa {{.*#+}} xmm4 = [16,16,16,16,16,16,16,16]
+; AVX2-NEXT:    vpsubw %xmm1, %xmm4, %xmm1
 ; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
 ; AVX2-NEXT:    vpsrlvd %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpshufb %ymm4, %ymm0, %ymm0
+; AVX2-NEXT:    vpshufb %ymm3, %ymm0, %ymm0
 ; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
-; AVX2-NEXT:    vpor %xmm0, %xmm3, %xmm0
+; AVX2-NEXT:    vpor %xmm0, %xmm2, %xmm0
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
 ; AVX512F-LABEL: var_funnnel_v8i16:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15]
-; AVX512F-NEXT:    vpand %xmm2, %xmm1, %xmm3
-; AVX512F-NEXT:    vpmovzxwd {{.*#+}} ymm3 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero
 ; AVX512F-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; AVX512F-NEXT:    vpsllvd %ymm3, %ymm0, %ymm3
-; AVX512F-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX512F-NEXT:    vpsubw %xmm1, %xmm4, %xmm1
-; AVX512F-NEXT:    vpand %xmm2, %xmm1, %xmm1
+; AVX512F-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX512F-NEXT:    vpmovzxwd {{.*#+}} ymm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; AVX512F-NEXT:    vpsllvd %ymm2, %ymm0, %ymm2
+; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm3 = [16,16,16,16,16,16,16,16]
+; AVX512F-NEXT:    vpsubw %xmm1, %xmm3, %xmm1
 ; AVX512F-NEXT:    vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
 ; AVX512F-NEXT:    vpsrlvd %ymm1, %ymm0, %ymm0
-; AVX512F-NEXT:    vpor %ymm0, %ymm3, %ymm0
+; AVX512F-NEXT:    vpor %ymm0, %ymm2, %ymm0
 ; AVX512F-NEXT:    vpmovdw %zmm0, %ymm0
 ; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
 ; AVX512F-NEXT:    vzeroupper
@@ -522,17 +374,15 @@ define <8 x i16> @var_funnnel_v8i16(<8 x i16> %x, <8 x i16> %amt) nounwind {
 ;
 ; AVX512VL-LABEL: var_funnnel_v8i16:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15]
-; AVX512VL-NEXT:    vpand %xmm2, %xmm1, %xmm3
-; AVX512VL-NEXT:    vpmovzxwd {{.*#+}} ymm3 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero
 ; AVX512VL-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; AVX512VL-NEXT:    vpsllvd %ymm3, %ymm0, %ymm3
-; AVX512VL-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX512VL-NEXT:    vpsubw %xmm1, %xmm4, %xmm1
-; AVX512VL-NEXT:    vpand %xmm2, %xmm1, %xmm1
+; AVX512VL-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX512VL-NEXT:    vpmovzxwd {{.*#+}} ymm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; AVX512VL-NEXT:    vpsllvd %ymm2, %ymm0, %ymm2
+; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm3 = [16,16,16,16,16,16,16,16]
+; AVX512VL-NEXT:    vpsubw %xmm1, %xmm3, %xmm1
 ; AVX512VL-NEXT:    vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
 ; AVX512VL-NEXT:    vpsrlvd %ymm1, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpor %ymm0, %ymm3, %ymm0
+; AVX512VL-NEXT:    vpor %ymm0, %ymm2, %ymm0
 ; AVX512VL-NEXT:    vpmovdw %ymm0, %xmm0
 ; AVX512VL-NEXT:    vzeroupper
 ; AVX512VL-NEXT:    retq
@@ -540,99 +390,55 @@ define <8 x i16> @var_funnnel_v8i16(<8 x i16> %x, <8 x i16> %amt) nounwind {
 ; AVX512BW-LABEL: var_funnnel_v8i16:
 ; AVX512BW:       # %bb.0:
 ; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15]
-; AVX512BW-NEXT:    vpand %xmm2, %xmm1, %xmm3
-; AVX512BW-NEXT:    vpsllvw %zmm3, %zmm0, %zmm3
-; AVX512BW-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX512BW-NEXT:    vpsubw %xmm1, %xmm4, %xmm1
-; AVX512BW-NEXT:    vpand %xmm2, %xmm1, %xmm1
+; AVX512BW-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX512BW-NEXT:    vpsllvw %zmm1, %zmm0, %zmm2
+; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm3 = [16,16,16,16,16,16,16,16]
+; AVX512BW-NEXT:    vpsubw %xmm1, %xmm3, %xmm1
 ; AVX512BW-NEXT:    vpsrlvw %zmm1, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpor %xmm0, %xmm3, %xmm0
+; AVX512BW-NEXT:    vpor %xmm0, %xmm2, %xmm0
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
 ;
 ; AVX512VLBW-LABEL: var_funnnel_v8i16:
 ; AVX512VLBW:       # %bb.0:
-; AVX512VLBW-NEXT:    vmovdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15]
-; AVX512VLBW-NEXT:    vpand %xmm2, %xmm1, %xmm3
-; AVX512VLBW-NEXT:    vpsllvw %xmm3, %xmm0, %xmm3
-; AVX512VLBW-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX512VLBW-NEXT:    vpsubw %xmm1, %xmm4, %xmm1
-; AVX512VLBW-NEXT:    vpand %xmm2, %xmm1, %xmm1
+; AVX512VLBW-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX512VLBW-NEXT:    vpsllvw %xmm1, %xmm0, %xmm2
+; AVX512VLBW-NEXT:    vmovdqa {{.*#+}} xmm3 = [16,16,16,16,16,16,16,16]
+; AVX512VLBW-NEXT:    vpsubw %xmm1, %xmm3, %xmm1
 ; AVX512VLBW-NEXT:    vpsrlvw %xmm1, %xmm0, %xmm0
-; AVX512VLBW-NEXT:    vpor %xmm0, %xmm3, %xmm0
+; AVX512VLBW-NEXT:    vpor %xmm0, %xmm2, %xmm0
 ; AVX512VLBW-NEXT:    retq
 ;
 ; XOP-LABEL: var_funnnel_v8i16:
 ; XOP:       # %bb.0:
-; XOP-NEXT:    vmovdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15]
-; XOP-NEXT:    vpand %xmm2, %xmm1, %xmm3
-; XOP-NEXT:    vpshlw %xmm3, %xmm0, %xmm3
-; XOP-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; XOP-NEXT:    vpsubw %xmm1, %xmm4, %xmm1
-; XOP-NEXT:    vpand %xmm2, %xmm1, %xmm1
-; XOP-NEXT:    vpsubw %xmm1, %xmm4, %xmm1
-; XOP-NEXT:    vpshlw %xmm1, %xmm0, %xmm0
-; XOP-NEXT:    vpor %xmm0, %xmm3, %xmm0
+; XOP-NEXT:    vprotw %xmm1, %xmm0, %xmm0
 ; XOP-NEXT:    retq
 ;
 ; X32-SSE-LABEL: var_funnnel_v8i16:
 ; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    pxor %xmm3, %xmm3
-; X32-SSE-NEXT:    pxor %xmm4, %xmm4
-; X32-SSE-NEXT:    psubw %xmm1, %xmm4
-; X32-SSE-NEXT:    psllw $12, %xmm4
-; X32-SSE-NEXT:    movdqa %xmm4, %xmm2
-; X32-SSE-NEXT:    psraw $15, %xmm2
-; X32-SSE-NEXT:    movdqa %xmm0, %xmm5
-; X32-SSE-NEXT:    psrlw $8, %xmm5
-; X32-SSE-NEXT:    pand %xmm2, %xmm5
-; X32-SSE-NEXT:    pandn %xmm0, %xmm2
-; X32-SSE-NEXT:    por %xmm5, %xmm2
-; X32-SSE-NEXT:    paddw %xmm4, %xmm4
-; X32-SSE-NEXT:    movdqa %xmm4, %xmm5
-; X32-SSE-NEXT:    psraw $15, %xmm5
-; X32-SSE-NEXT:    movdqa %xmm5, %xmm6
-; X32-SSE-NEXT:    pandn %xmm2, %xmm6
-; X32-SSE-NEXT:    psrlw $4, %xmm2
-; X32-SSE-NEXT:    pand %xmm5, %xmm2
-; X32-SSE-NEXT:    por %xmm6, %xmm2
-; X32-SSE-NEXT:    paddw %xmm4, %xmm4
-; X32-SSE-NEXT:    movdqa %xmm4, %xmm5
-; X32-SSE-NEXT:    psraw $15, %xmm5
-; X32-SSE-NEXT:    movdqa %xmm5, %xmm6
-; X32-SSE-NEXT:    pandn %xmm2, %xmm6
-; X32-SSE-NEXT:    psrlw $2, %xmm2
-; X32-SSE-NEXT:    pand %xmm5, %xmm2
-; X32-SSE-NEXT:    por %xmm6, %xmm2
-; X32-SSE-NEXT:    paddw %xmm4, %xmm4
-; X32-SSE-NEXT:    psraw $15, %xmm4
-; X32-SSE-NEXT:    movdqa %xmm4, %xmm5
-; X32-SSE-NEXT:    pandn %xmm2, %xmm5
-; X32-SSE-NEXT:    psrlw $1, %xmm2
-; X32-SSE-NEXT:    pand %xmm4, %xmm2
 ; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm1
-; X32-SSE-NEXT:    movdqa %xmm1, %xmm4
-; X32-SSE-NEXT:    punpckhwd {{.*#+}} xmm4 = xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7]
-; X32-SSE-NEXT:    pslld $23, %xmm4
-; X32-SSE-NEXT:    movdqa {{.*#+}} xmm6 = [1065353216,1065353216,1065353216,1065353216]
-; X32-SSE-NEXT:    paddd %xmm6, %xmm4
-; X32-SSE-NEXT:    cvttps2dq %xmm4, %xmm4
-; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm4 = xmm4[0,2,2,3,4,5,6,7]
-; X32-SSE-NEXT:    pshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,4,6,6,7]
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[0,2,2,3]
-; X32-SSE-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3]
+; X32-SSE-NEXT:    pxor %xmm2, %xmm2
+; X32-SSE-NEXT:    movdqa %xmm1, %xmm3
+; X32-SSE-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
+; X32-SSE-NEXT:    pslld $23, %xmm3
+; X32-SSE-NEXT:    movdqa {{.*#+}} xmm4 = [1065353216,1065353216,1065353216,1065353216]
+; X32-SSE-NEXT:    paddd %xmm4, %xmm3
+; X32-SSE-NEXT:    cvttps2dq %xmm3, %xmm3
+; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm3 = xmm3[0,2,2,3,4,5,6,7]
+; X32-SSE-NEXT:    pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,6,6,7]
+; X32-SSE-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3]
+; X32-SSE-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
 ; X32-SSE-NEXT:    pslld $23, %xmm1
-; X32-SSE-NEXT:    paddd %xmm6, %xmm1
+; X32-SSE-NEXT:    paddd %xmm4, %xmm1
 ; X32-SSE-NEXT:    cvttps2dq %xmm1, %xmm1
 ; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,2,2,3,4,5,6,7]
 ; X32-SSE-NEXT:    pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,6,6,7]
 ; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
-; X32-SSE-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm4[0]
-; X32-SSE-NEXT:    pmullw %xmm0, %xmm1
-; X32-SSE-NEXT:    por %xmm5, %xmm1
-; X32-SSE-NEXT:    por %xmm2, %xmm1
-; X32-SSE-NEXT:    movdqa %xmm1, %xmm0
+; X32-SSE-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm3[0]
+; X32-SSE-NEXT:    movdqa %xmm0, %xmm2
+; X32-SSE-NEXT:    pmulhuw %xmm1, %xmm2
+; X32-SSE-NEXT:    pmullw %xmm1, %xmm0
+; X32-SSE-NEXT:    por %xmm2, %xmm0
 ; X32-SSE-NEXT:    retl
   %res = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %x, <8 x i16> %x, <8 x i16> %amt)
   ret <8 x i16> %res
@@ -641,149 +447,104 @@ define <8 x i16> @var_funnnel_v8i16(<8 x i16> %x, <8 x i16> %amt) nounwind {
 define <16 x i8> @var_funnnel_v16i8(<16 x i8> %x, <16 x i8> %amt) nounwind {
 ; SSE2-LABEL: var_funnnel_v16i8:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
-; SSE2-NEXT:    pxor %xmm4, %xmm4
-; SSE2-NEXT:    psubb %xmm1, %xmm4
-; SSE2-NEXT:    movdqa %xmm1, %xmm5
-; SSE2-NEXT:    pand %xmm2, %xmm5
-; SSE2-NEXT:    psllw $5, %xmm5
+; SSE2-NEXT:    movdqa %xmm0, %xmm2
+; SSE2-NEXT:    psllw $5, %xmm1
+; SSE2-NEXT:    pxor %xmm0, %xmm0
 ; SSE2-NEXT:    pxor %xmm3, %xmm3
-; SSE2-NEXT:    pxor %xmm6, %xmm6
-; SSE2-NEXT:    pcmpgtb %xmm5, %xmm6
-; SSE2-NEXT:    movdqa %xmm0, %xmm1
-; SSE2-NEXT:    psllw $4, %xmm1
-; SSE2-NEXT:    pand %xmm6, %xmm1
-; SSE2-NEXT:    pandn %xmm0, %xmm6
-; SSE2-NEXT:    pand {{.*}}(%rip), %xmm1
-; SSE2-NEXT:    por %xmm6, %xmm1
-; SSE2-NEXT:    paddb %xmm5, %xmm5
-; SSE2-NEXT:    pxor %xmm6, %xmm6
-; SSE2-NEXT:    pcmpgtb %xmm5, %xmm6
-; SSE2-NEXT:    movdqa %xmm6, %xmm7
-; SSE2-NEXT:    pandn %xmm1, %xmm7
-; SSE2-NEXT:    psllw $2, %xmm1
-; SSE2-NEXT:    pand %xmm6, %xmm1
-; SSE2-NEXT:    pand {{.*}}(%rip), %xmm1
-; SSE2-NEXT:    por %xmm7, %xmm1
-; SSE2-NEXT:    paddb %xmm5, %xmm5
-; SSE2-NEXT:    pxor %xmm6, %xmm6
-; SSE2-NEXT:    pcmpgtb %xmm5, %xmm6
-; SSE2-NEXT:    movdqa %xmm6, %xmm5
-; SSE2-NEXT:    pandn %xmm1, %xmm5
+; SSE2-NEXT:    pcmpgtb %xmm1, %xmm3
+; SSE2-NEXT:    movdqa %xmm2, %xmm4
+; SSE2-NEXT:    psrlw $4, %xmm4
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm4
+; SSE2-NEXT:    movdqa %xmm2, %xmm5
+; SSE2-NEXT:    psllw $4, %xmm5
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm5
+; SSE2-NEXT:    por %xmm4, %xmm5
+; SSE2-NEXT:    pand %xmm3, %xmm5
+; SSE2-NEXT:    pandn %xmm2, %xmm3
+; SSE2-NEXT:    por %xmm5, %xmm3
+; SSE2-NEXT:    movdqa %xmm3, %xmm2
+; SSE2-NEXT:    psrlw $6, %xmm2
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm2
+; SSE2-NEXT:    movdqa %xmm3, %xmm4
+; SSE2-NEXT:    psllw $2, %xmm4
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm4
+; SSE2-NEXT:    por %xmm2, %xmm4
 ; SSE2-NEXT:    paddb %xmm1, %xmm1
-; SSE2-NEXT:    pand %xmm6, %xmm1
-; SSE2-NEXT:    pand %xmm2, %xmm4
-; SSE2-NEXT:    psllw $5, %xmm4
-; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    pcmpgtb %xmm4, %xmm2
-; SSE2-NEXT:    movdqa %xmm2, %xmm6
-; SSE2-NEXT:    pandn %xmm0, %xmm6
-; SSE2-NEXT:    psrlw $4, %xmm0
-; SSE2-NEXT:    pand %xmm2, %xmm0
-; SSE2-NEXT:    pand {{.*}}(%rip), %xmm0
-; SSE2-NEXT:    por %xmm6, %xmm0
-; SSE2-NEXT:    paddb %xmm4, %xmm4
 ; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    pcmpgtb %xmm4, %xmm2
-; SSE2-NEXT:    movdqa %xmm2, %xmm6
-; SSE2-NEXT:    pandn %xmm0, %xmm6
-; SSE2-NEXT:    psrlw $2, %xmm0
-; SSE2-NEXT:    pand %xmm2, %xmm0
-; SSE2-NEXT:    pand {{.*}}(%rip), %xmm0
-; SSE2-NEXT:    por %xmm6, %xmm0
-; SSE2-NEXT:    paddb %xmm4, %xmm4
-; SSE2-NEXT:    pcmpgtb %xmm4, %xmm3
-; SSE2-NEXT:    movdqa %xmm3, %xmm2
-; SSE2-NEXT:    pandn %xmm0, %xmm2
-; SSE2-NEXT:    psrlw $1, %xmm0
-; SSE2-NEXT:    pand %xmm3, %xmm0
-; SSE2-NEXT:    pand {{.*}}(%rip), %xmm0
-; SSE2-NEXT:    por %xmm2, %xmm0
-; SSE2-NEXT:    por %xmm5, %xmm0
-; SSE2-NEXT:    por %xmm1, %xmm0
+; SSE2-NEXT:    pcmpgtb %xmm1, %xmm2
+; SSE2-NEXT:    pand %xmm2, %xmm4
+; SSE2-NEXT:    pandn %xmm3, %xmm2
+; SSE2-NEXT:    por %xmm4, %xmm2
+; SSE2-NEXT:    movdqa %xmm2, %xmm3
+; SSE2-NEXT:    paddb %xmm2, %xmm3
+; SSE2-NEXT:    movdqa %xmm2, %xmm4
+; SSE2-NEXT:    psrlw $7, %xmm4
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm4
+; SSE2-NEXT:    por %xmm3, %xmm4
+; SSE2-NEXT:    paddb %xmm1, %xmm1
+; SSE2-NEXT:    pcmpgtb %xmm1, %xmm0
+; SSE2-NEXT:    pand %xmm0, %xmm4
+; SSE2-NEXT:    pandn %xmm2, %xmm0
+; SSE2-NEXT:    por %xmm4, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: var_funnnel_v16i8:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    movdqa %xmm0, %xmm2
-; SSE41-NEXT:    movdqa {{.*#+}} xmm4 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
-; SSE41-NEXT:    pxor %xmm3, %xmm3
-; SSE41-NEXT:    psubb %xmm1, %xmm3
-; SSE41-NEXT:    movdqa %xmm1, %xmm0
-; SSE41-NEXT:    pand %xmm4, %xmm0
-; SSE41-NEXT:    psllw $5, %xmm0
+; SSE41-NEXT:    movdqa %xmm1, %xmm2
 ; SSE41-NEXT:    movdqa %xmm0, %xmm1
-; SSE41-NEXT:    paddb %xmm0, %xmm1
-; SSE41-NEXT:    movdqa %xmm2, %xmm5
-; SSE41-NEXT:    psllw $4, %xmm5
-; SSE41-NEXT:    pand {{.*}}(%rip), %xmm5
-; SSE41-NEXT:    movdqa %xmm2, %xmm6
-; SSE41-NEXT:    pblendvb %xmm0, %xmm5, %xmm6
-; SSE41-NEXT:    movdqa %xmm6, %xmm5
-; SSE41-NEXT:    psllw $2, %xmm5
-; SSE41-NEXT:    pand {{.*}}(%rip), %xmm5
-; SSE41-NEXT:    movdqa %xmm1, %xmm0
-; SSE41-NEXT:    pblendvb %xmm0, %xmm5, %xmm6
-; SSE41-NEXT:    movdqa %xmm6, %xmm5
-; SSE41-NEXT:    paddb %xmm6, %xmm5
-; SSE41-NEXT:    paddb %xmm1, %xmm1
-; SSE41-NEXT:    movdqa %xmm1, %xmm0
-; SSE41-NEXT:    pblendvb %xmm0, %xmm5, %xmm6
-; SSE41-NEXT:    pand %xmm4, %xmm3
-; SSE41-NEXT:    psllw $5, %xmm3
-; SSE41-NEXT:    movdqa %xmm3, %xmm1
-; SSE41-NEXT:    paddb %xmm3, %xmm1
-; SSE41-NEXT:    movdqa %xmm2, %xmm4
-; SSE41-NEXT:    psrlw $4, %xmm4
-; SSE41-NEXT:    pand {{.*}}(%rip), %xmm4
-; SSE41-NEXT:    movdqa %xmm3, %xmm0
-; SSE41-NEXT:    pblendvb %xmm0, %xmm4, %xmm2
-; SSE41-NEXT:    movdqa %xmm2, %xmm3
-; SSE41-NEXT:    psrlw $2, %xmm3
+; SSE41-NEXT:    psrlw $4, %xmm0
+; SSE41-NEXT:    pand {{.*}}(%rip), %xmm0
+; SSE41-NEXT:    movdqa %xmm1, %xmm3
+; SSE41-NEXT:    psllw $4, %xmm3
 ; SSE41-NEXT:    pand {{.*}}(%rip), %xmm3
+; SSE41-NEXT:    por %xmm0, %xmm3
+; SSE41-NEXT:    psllw $5, %xmm2
+; SSE41-NEXT:    movdqa %xmm2, %xmm0
+; SSE41-NEXT:    pblendvb %xmm0, %xmm3, %xmm1
 ; SSE41-NEXT:    movdqa %xmm1, %xmm0
-; SSE41-NEXT:    pblendvb %xmm0, %xmm3, %xmm2
-; SSE41-NEXT:    movdqa %xmm2, %xmm3
-; SSE41-NEXT:    psrlw $1, %xmm3
+; SSE41-NEXT:    psrlw $6, %xmm0
+; SSE41-NEXT:    pand {{.*}}(%rip), %xmm0
+; SSE41-NEXT:    movdqa %xmm1, %xmm3
+; SSE41-NEXT:    psllw $2, %xmm3
 ; SSE41-NEXT:    pand {{.*}}(%rip), %xmm3
-; SSE41-NEXT:    paddb %xmm1, %xmm1
+; SSE41-NEXT:    por %xmm0, %xmm3
+; SSE41-NEXT:    paddb %xmm2, %xmm2
+; SSE41-NEXT:    movdqa %xmm2, %xmm0
+; SSE41-NEXT:    pblendvb %xmm0, %xmm3, %xmm1
 ; SSE41-NEXT:    movdqa %xmm1, %xmm0
-; SSE41-NEXT:    pblendvb %xmm0, %xmm3, %xmm2
-; SSE41-NEXT:    por %xmm6, %xmm2
+; SSE41-NEXT:    paddb %xmm1, %xmm0
+; SSE41-NEXT:    movdqa %xmm1, %xmm3
+; SSE41-NEXT:    psrlw $7, %xmm3
+; SSE41-NEXT:    pand {{.*}}(%rip), %xmm3
+; SSE41-NEXT:    por %xmm0, %xmm3
+; SSE41-NEXT:    paddb %xmm2, %xmm2
 ; SSE41-NEXT:    movdqa %xmm2, %xmm0
+; SSE41-NEXT:    pblendvb %xmm0, %xmm3, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX-LABEL: var_funnnel_v16i8:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vmovdqa {{.*#+}} xmm2 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
-; AVX-NEXT:    vpand %xmm2, %xmm1, %xmm3
-; AVX-NEXT:    vpsllw $5, %xmm3, %xmm3
-; AVX-NEXT:    vpaddb %xmm3, %xmm3, %xmm4
-; AVX-NEXT:    vpsllw $4, %xmm0, %xmm5
-; AVX-NEXT:    vpand {{.*}}(%rip), %xmm5, %xmm5
-; AVX-NEXT:    vpblendvb %xmm3, %xmm5, %xmm0, %xmm3
-; AVX-NEXT:    vpsllw $2, %xmm3, %xmm5
-; AVX-NEXT:    vpand {{.*}}(%rip), %xmm5, %xmm5
-; AVX-NEXT:    vpblendvb %xmm4, %xmm5, %xmm3, %xmm3
-; AVX-NEXT:    vpaddb %xmm3, %xmm3, %xmm5
-; AVX-NEXT:    vpaddb %xmm4, %xmm4, %xmm4
-; AVX-NEXT:    vpblendvb %xmm4, %xmm5, %xmm3, %xmm3
-; AVX-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX-NEXT:    vpsubb %xmm1, %xmm4, %xmm1
-; AVX-NEXT:    vpand %xmm2, %xmm1, %xmm1
+; AVX-NEXT:    vpsrlw $4, %xmm0, %xmm2
+; AVX-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
+; AVX-NEXT:    vpsllw $4, %xmm0, %xmm3
+; AVX-NEXT:    vpand {{.*}}(%rip), %xmm3, %xmm3
+; AVX-NEXT:    vpor %xmm2, %xmm3, %xmm2
 ; AVX-NEXT:    vpsllw $5, %xmm1, %xmm1
-; AVX-NEXT:    vpaddb %xmm1, %xmm1, %xmm2
-; AVX-NEXT:    vpsrlw $4, %xmm0, %xmm4
-; AVX-NEXT:    vpand {{.*}}(%rip), %xmm4, %xmm4
-; AVX-NEXT:    vpblendvb %xmm1, %xmm4, %xmm0, %xmm0
-; AVX-NEXT:    vpsrlw $2, %xmm0, %xmm1
-; AVX-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
-; AVX-NEXT:    vpblendvb %xmm2, %xmm1, %xmm0, %xmm0
-; AVX-NEXT:    vpsrlw $1, %xmm0, %xmm1
-; AVX-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
-; AVX-NEXT:    vpaddb %xmm2, %xmm2, %xmm2
-; AVX-NEXT:    vpblendvb %xmm2, %xmm1, %xmm0, %xmm0
-; AVX-NEXT:    vpor %xmm0, %xmm3, %xmm0
+; AVX-NEXT:    vpblendvb %xmm1, %xmm2, %xmm0, %xmm0
+; AVX-NEXT:    vpsrlw $6, %xmm0, %xmm2
+; AVX-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
+; AVX-NEXT:    vpsllw $2, %xmm0, %xmm3
+; AVX-NEXT:    vpand {{.*}}(%rip), %xmm3, %xmm3
+; AVX-NEXT:    vpor %xmm2, %xmm3, %xmm2
+; AVX-NEXT:    vpaddb %xmm1, %xmm1, %xmm1
+; AVX-NEXT:    vpblendvb %xmm1, %xmm2, %xmm0, %xmm0
+; AVX-NEXT:    vpaddb %xmm0, %xmm0, %xmm2
+; AVX-NEXT:    vpsrlw $7, %xmm0, %xmm3
+; AVX-NEXT:    vpand {{.*}}(%rip), %xmm3, %xmm3
+; AVX-NEXT:    vpor %xmm3, %xmm2, %xmm2
+; AVX-NEXT:    vpaddb %xmm1, %xmm1, %xmm1
+; AVX-NEXT:    vpblendvb %xmm1, %xmm2, %xmm0, %xmm0
 ; AVX-NEXT:    retq
 ;
 ; AVX512F-LABEL: var_funnnel_v16i8:
@@ -857,79 +618,50 @@ define <16 x i8> @var_funnnel_v16i8(<16 x i8> %x, <16 x i8> %amt) nounwind {
 ;
 ; XOP-LABEL: var_funnnel_v16i8:
 ; XOP:       # %bb.0:
-; XOP-NEXT:    vmovdqa {{.*#+}} xmm2 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
-; XOP-NEXT:    vpand %xmm2, %xmm1, %xmm3
-; XOP-NEXT:    vpshlb %xmm3, %xmm0, %xmm3
-; XOP-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; XOP-NEXT:    vpsubb %xmm1, %xmm4, %xmm1
-; XOP-NEXT:    vpand %xmm2, %xmm1, %xmm1
-; XOP-NEXT:    vpsubb %xmm1, %xmm4, %xmm1
-; XOP-NEXT:    vpshlb %xmm1, %xmm0, %xmm0
-; XOP-NEXT:    vpor %xmm0, %xmm3, %xmm0
+; XOP-NEXT:    vprotb %xmm1, %xmm0, %xmm0
 ; XOP-NEXT:    retq
 ;
 ; X32-SSE-LABEL: var_funnnel_v16i8:
 ; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    movdqa {{.*#+}} xmm2 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
-; X32-SSE-NEXT:    pxor %xmm4, %xmm4
-; X32-SSE-NEXT:    psubb %xmm1, %xmm4
-; X32-SSE-NEXT:    movdqa %xmm1, %xmm5
-; X32-SSE-NEXT:    pand %xmm2, %xmm5
-; X32-SSE-NEXT:    psllw $5, %xmm5
+; X32-SSE-NEXT:    movdqa %xmm0, %xmm2
+; X32-SSE-NEXT:    psllw $5, %xmm1
+; X32-SSE-NEXT:    pxor %xmm0, %xmm0
 ; X32-SSE-NEXT:    pxor %xmm3, %xmm3
-; X32-SSE-NEXT:    pxor %xmm6, %xmm6
-; X32-SSE-NEXT:    pcmpgtb %xmm5, %xmm6
-; X32-SSE-NEXT:    movdqa %xmm0, %xmm1
-; X32-SSE-NEXT:    psllw $4, %xmm1
-; X32-SSE-NEXT:    pand %xmm6, %xmm1
-; X32-SSE-NEXT:    pandn %xmm0, %xmm6
-; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm1
-; X32-SSE-NEXT:    por %xmm6, %xmm1
-; X32-SSE-NEXT:    paddb %xmm5, %xmm5
-; X32-SSE-NEXT:    pxor %xmm6, %xmm6
-; X32-SSE-NEXT:    pcmpgtb %xmm5, %xmm6
-; X32-SSE-NEXT:    movdqa %xmm6, %xmm7
-; X32-SSE-NEXT:    pandn %xmm1, %xmm7
-; X32-SSE-NEXT:    psllw $2, %xmm1
-; X32-SSE-NEXT:    pand %xmm6, %xmm1
-; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm1
-; X32-SSE-NEXT:    por %xmm7, %xmm1
-; X32-SSE-NEXT:    paddb %xmm5, %xmm5
-; X32-SSE-NEXT:    pxor %xmm6, %xmm6
-; X32-SSE-NEXT:    pcmpgtb %xmm5, %xmm6
-; X32-SSE-NEXT:    movdqa %xmm6, %xmm5
-; X32-SSE-NEXT:    pandn %xmm1, %xmm5
+; X32-SSE-NEXT:    pcmpgtb %xmm1, %xmm3
+; X32-SSE-NEXT:    movdqa %xmm2, %xmm4
+; X32-SSE-NEXT:    psrlw $4, %xmm4
+; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm4
+; X32-SSE-NEXT:    movdqa %xmm2, %xmm5
+; X32-SSE-NEXT:    psllw $4, %xmm5
+; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm5
+; X32-SSE-NEXT:    por %xmm4, %xmm5
+; X32-SSE-NEXT:    pand %xmm3, %xmm5
+; X32-SSE-NEXT:    pandn %xmm2, %xmm3
+; X32-SSE-NEXT:    por %xmm5, %xmm3
+; X32-SSE-NEXT:    movdqa %xmm3, %xmm2
+; X32-SSE-NEXT:    psrlw $6, %xmm2
+; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm2
+; X32-SSE-NEXT:    movdqa %xmm3, %xmm4
+; X32-SSE-NEXT:    psllw $2, %xmm4
+; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm4
+; X32-SSE-NEXT:    por %xmm2, %xmm4
 ; X32-SSE-NEXT:    paddb %xmm1, %xmm1
-; X32-SSE-NEXT:    pand %xmm6, %xmm1
-; X32-SSE-NEXT:    pand %xmm2, %xmm4
-; X32-SSE-NEXT:    psllw $5, %xmm4
 ; X32-SSE-NEXT:    pxor %xmm2, %xmm2
-; X32-SSE-NEXT:    pcmpgtb %xmm4, %xmm2
-; X32-SSE-NEXT:    movdqa %xmm2, %xmm6
-; X32-SSE-NEXT:    pandn %xmm0, %xmm6
-; X32-SSE-NEXT:    psrlw $4, %xmm0
-; X32-SSE-NEXT:    pand %xmm2, %xmm0
-; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
-; X32-SSE-NEXT:    por %xmm6, %xmm0
-; X32-SSE-NEXT:    paddb %xmm4, %xmm4
-; X32-SSE-NEXT:    pxor %xmm2, %xmm2
-; X32-SSE-NEXT:    pcmpgtb %xmm4, %xmm2
-; X32-SSE-NEXT:    movdqa %xmm2, %xmm6
-; X32-SSE-NEXT:    pandn %xmm0, %xmm6
-; X32-SSE-NEXT:    psrlw $2, %xmm0
-; X32-SSE-NEXT:    pand %xmm2, %xmm0
-; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
-; X32-SSE-NEXT:    por %xmm6, %xmm0
-; X32-SSE-NEXT:    paddb %xmm4, %xmm4
-; X32-SSE-NEXT:    pcmpgtb %xmm4, %xmm3
-; X32-SSE-NEXT:    movdqa %xmm3, %xmm2
-; X32-SSE-NEXT:    pandn %xmm0, %xmm2
-; X32-SSE-NEXT:    psrlw $1, %xmm0
-; X32-SSE-NEXT:    pand %xmm3, %xmm0
-; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
-; X32-SSE-NEXT:    por %xmm2, %xmm0
-; X32-SSE-NEXT:    por %xmm5, %xmm0
-; X32-SSE-NEXT:    por %xmm1, %xmm0
+; X32-SSE-NEXT:    pcmpgtb %xmm1, %xmm2
+; X32-SSE-NEXT:    pand %xmm2, %xmm4
+; X32-SSE-NEXT:    pandn %xmm3, %xmm2
+; X32-SSE-NEXT:    por %xmm4, %xmm2
+; X32-SSE-NEXT:    movdqa %xmm2, %xmm3
+; X32-SSE-NEXT:    paddb %xmm2, %xmm3
+; X32-SSE-NEXT:    movdqa %xmm2, %xmm4
+; X32-SSE-NEXT:    psrlw $7, %xmm4
+; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm4
+; X32-SSE-NEXT:    por %xmm3, %xmm4
+; X32-SSE-NEXT:    paddb %xmm1, %xmm1
+; X32-SSE-NEXT:    pcmpgtb %xmm1, %xmm0
+; X32-SSE-NEXT:    pand %xmm0, %xmm4
+; X32-SSE-NEXT:    pandn %xmm2, %xmm0
+; X32-SSE-NEXT:    por %xmm4, %xmm0
 ; X32-SSE-NEXT:    retl
   %res = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %x, <16 x i8> %x, <16 x i8> %amt)
   ret <16 x i8> %res
@@ -980,43 +712,46 @@ define <2 x i64> @splatvar_funnnel_v2i64(<2 x i64> %x, <2 x i64> %amt) nounwind
 ; AVX2-NEXT:    vpor %xmm0, %xmm3, %xmm0
 ; AVX2-NEXT:    retq
 ;
-; AVX512-LABEL: splatvar_funnnel_v2i64:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpbroadcastq %xmm1, %xmm1
-; AVX512-NEXT:    vmovdqa {{.*#+}} xmm2 = [63,63]
-; AVX512-NEXT:    vpand %xmm2, %xmm1, %xmm3
-; AVX512-NEXT:    vpsllq %xmm3, %xmm0, %xmm3
-; AVX512-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX512-NEXT:    vpsubq %xmm1, %xmm4, %xmm1
-; AVX512-NEXT:    vpand %xmm2, %xmm1, %xmm1
-; AVX512-NEXT:    vpsrlq %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpor %xmm0, %xmm3, %xmm0
-; AVX512-NEXT:    retq
+; AVX512F-LABEL: splatvar_funnnel_v2i64:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512F-NEXT:    vpbroadcastq %xmm1, %xmm1
+; AVX512F-NEXT:    vprolvq %zmm1, %zmm0, %zmm0
+; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: splatvar_funnnel_v2i64:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpbroadcastq %xmm1, %xmm1
+; AVX512VL-NEXT:    vprolvq %xmm1, %xmm0, %xmm0
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: splatvar_funnnel_v2i64:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512BW-NEXT:    vpbroadcastq %xmm1, %xmm1
+; AVX512BW-NEXT:    vprolvq %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+;
+; AVX512VLBW-LABEL: splatvar_funnnel_v2i64:
+; AVX512VLBW:       # %bb.0:
+; AVX512VLBW-NEXT:    vpbroadcastq %xmm1, %xmm1
+; AVX512VLBW-NEXT:    vprolvq %xmm1, %xmm0, %xmm0
+; AVX512VLBW-NEXT:    retq
 ;
 ; XOPAVX1-LABEL: splatvar_funnnel_v2i64:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,1,0,1]
-; XOPAVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [63,63]
-; XOPAVX1-NEXT:    vpand %xmm2, %xmm1, %xmm3
-; XOPAVX1-NEXT:    vpsllq %xmm3, %xmm0, %xmm3
-; XOPAVX1-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; XOPAVX1-NEXT:    vpsubq %xmm1, %xmm4, %xmm1
-; XOPAVX1-NEXT:    vpand %xmm2, %xmm1, %xmm1
-; XOPAVX1-NEXT:    vpsrlq %xmm1, %xmm0, %xmm0
-; XOPAVX1-NEXT:    vpor %xmm0, %xmm3, %xmm0
+; XOPAVX1-NEXT:    vprotq %xmm1, %xmm0, %xmm0
 ; XOPAVX1-NEXT:    retq
 ;
 ; XOPAVX2-LABEL: splatvar_funnnel_v2i64:
 ; XOPAVX2:       # %bb.0:
 ; XOPAVX2-NEXT:    vpbroadcastq %xmm1, %xmm1
-; XOPAVX2-NEXT:    vmovdqa {{.*#+}} xmm2 = [63,63]
-; XOPAVX2-NEXT:    vpand %xmm2, %xmm1, %xmm3
-; XOPAVX2-NEXT:    vpsllq %xmm3, %xmm0, %xmm3
-; XOPAVX2-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; XOPAVX2-NEXT:    vpsubq %xmm1, %xmm4, %xmm1
-; XOPAVX2-NEXT:    vpand %xmm2, %xmm1, %xmm1
-; XOPAVX2-NEXT:    vpsrlq %xmm1, %xmm0, %xmm0
-; XOPAVX2-NEXT:    vpor %xmm0, %xmm3, %xmm0
+; XOPAVX2-NEXT:    vprotq %xmm1, %xmm0, %xmm0
 ; XOPAVX2-NEXT:    retq
 ;
 ; X32-SSE-LABEL: splatvar_funnnel_v2i64:
@@ -1048,129 +783,113 @@ define <2 x i64> @splatvar_funnnel_v2i64(<2 x i64> %x, <2 x i64> %amt) nounwind
 define <4 x i32> @splatvar_funnnel_v4i32(<4 x i32> %x, <4 x i32> %amt) nounwind {
 ; SSE2-LABEL: splatvar_funnnel_v4i32:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [31,31,31,31]
-; SSE2-NEXT:    pxor %xmm3, %xmm3
-; SSE2-NEXT:    psubd %xmm1, %xmm3
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    xorps %xmm4, %xmm4
-; SSE2-NEXT:    xorps %xmm5, %xmm5
-; SSE2-NEXT:    movss {{.*#+}} xmm5 = xmm1[0],xmm5[1,2,3]
-; SSE2-NEXT:    movdqa %xmm0, %xmm1
-; SSE2-NEXT:    pslld %xmm5, %xmm1
-; SSE2-NEXT:    pand %xmm2, %xmm3
-; SSE2-NEXT:    movss {{.*#+}} xmm4 = xmm3[0],xmm4[1,2,3]
-; SSE2-NEXT:    psrld %xmm4, %xmm0
-; SSE2-NEXT:    por %xmm1, %xmm0
+; SSE2-NEXT:    movd %xmm1, %eax
+; SSE2-NEXT:    andl $31, %eax
+; SSE2-NEXT:    movd %eax, %xmm1
+; SSE2-NEXT:    movdqa %xmm0, %xmm2
+; SSE2-NEXT:    pslld %xmm1, %xmm2
+; SSE2-NEXT:    movl $32, %ecx
+; SSE2-NEXT:    subl %eax, %ecx
+; SSE2-NEXT:    movd %ecx, %xmm1
+; SSE2-NEXT:    psrld %xmm1, %xmm0
+; SSE2-NEXT:    por %xmm2, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: splatvar_funnnel_v4i32:
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
-; SSE41-NEXT:    movdqa {{.*#+}} xmm2 = [31,31,31,31]
-; SSE41-NEXT:    pxor %xmm3, %xmm3
-; SSE41-NEXT:    psubd %xmm1, %xmm3
-; SSE41-NEXT:    pand %xmm2, %xmm1
-; SSE41-NEXT:    pmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero
-; SSE41-NEXT:    movdqa %xmm0, %xmm4
-; SSE41-NEXT:    pslld %xmm1, %xmm4
-; SSE41-NEXT:    pand %xmm2, %xmm3
-; SSE41-NEXT:    pmovzxdq {{.*#+}} xmm1 = xmm3[0],zero,xmm3[1],zero
+; SSE41-NEXT:    pand {{.*}}(%rip), %xmm1
+; SSE41-NEXT:    pmovzxdq {{.*#+}} xmm2 = xmm1[0],zero,xmm1[1],zero
+; SSE41-NEXT:    movdqa %xmm0, %xmm3
+; SSE41-NEXT:    pslld %xmm2, %xmm3
+; SSE41-NEXT:    movdqa {{.*#+}} xmm2 = [32,32,32,32]
+; SSE41-NEXT:    psubd %xmm1, %xmm2
+; SSE41-NEXT:    pmovzxdq {{.*#+}} xmm1 = xmm2[0],zero,xmm2[1],zero
 ; SSE41-NEXT:    psrld %xmm1, %xmm0
-; SSE41-NEXT:    por %xmm4, %xmm0
+; SSE41-NEXT:    por %xmm3, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: splatvar_funnnel_v4i32:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [31,31,31,31]
-; AVX1-NEXT:    vpand %xmm2, %xmm1, %xmm3
-; AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm3 = xmm3[0],zero,xmm3[1],zero
-; AVX1-NEXT:    vpslld %xmm3, %xmm0, %xmm3
-; AVX1-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX1-NEXT:    vpsubd %xmm1, %xmm4, %xmm1
-; AVX1-NEXT:    vpand %xmm2, %xmm1, %xmm1
+; AVX1-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm2 = xmm1[0],zero,xmm1[1],zero
+; AVX1-NEXT:    vpslld %xmm2, %xmm0, %xmm2
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [32,32,32,32]
+; AVX1-NEXT:    vpsubd %xmm1, %xmm3, %xmm1
 ; AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero
 ; AVX1-NEXT:    vpsrld %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpor %xmm0, %xmm3, %xmm0
+; AVX1-NEXT:    vpor %xmm0, %xmm2, %xmm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: splatvar_funnnel_v4i32:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpbroadcastd %xmm1, %xmm1
 ; AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [31,31,31,31]
-; AVX2-NEXT:    vpand %xmm2, %xmm1, %xmm3
-; AVX2-NEXT:    vpmovzxdq {{.*#+}} xmm3 = xmm3[0],zero,xmm3[1],zero
-; AVX2-NEXT:    vpslld %xmm3, %xmm0, %xmm3
-; AVX2-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX2-NEXT:    vpsubd %xmm1, %xmm4, %xmm1
 ; AVX2-NEXT:    vpand %xmm2, %xmm1, %xmm1
+; AVX2-NEXT:    vpmovzxdq {{.*#+}} xmm2 = xmm1[0],zero,xmm1[1],zero
+; AVX2-NEXT:    vpslld %xmm2, %xmm0, %xmm2
+; AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm3 = [32,32,32,32]
+; AVX2-NEXT:    vpsubd %xmm1, %xmm3, %xmm1
 ; AVX2-NEXT:    vpmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero
 ; AVX2-NEXT:    vpsrld %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpor %xmm0, %xmm3, %xmm0
+; AVX2-NEXT:    vpor %xmm0, %xmm2, %xmm0
 ; AVX2-NEXT:    retq
 ;
-; AVX512-LABEL: splatvar_funnnel_v4i32:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpbroadcastd %xmm1, %xmm1
-; AVX512-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [31,31,31,31]
-; AVX512-NEXT:    vpand %xmm2, %xmm1, %xmm3
-; AVX512-NEXT:    vpmovzxdq {{.*#+}} xmm3 = xmm3[0],zero,xmm3[1],zero
-; AVX512-NEXT:    vpslld %xmm3, %xmm0, %xmm3
-; AVX512-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX512-NEXT:    vpsubd %xmm1, %xmm4, %xmm1
-; AVX512-NEXT:    vpand %xmm2, %xmm1, %xmm1
-; AVX512-NEXT:    vpmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero
-; AVX512-NEXT:    vpsrld %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpor %xmm0, %xmm3, %xmm0
-; AVX512-NEXT:    retq
+; AVX512F-LABEL: splatvar_funnnel_v4i32:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512F-NEXT:    vpbroadcastd %xmm1, %xmm1
+; AVX512F-NEXT:    vprolvd %zmm1, %zmm0, %zmm0
+; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: splatvar_funnnel_v4i32:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpbroadcastd %xmm1, %xmm1
+; AVX512VL-NEXT:    vprolvd %xmm1, %xmm0, %xmm0
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: splatvar_funnnel_v4i32:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512BW-NEXT:    vpbroadcastd %xmm1, %xmm1
+; AVX512BW-NEXT:    vprolvd %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+;
+; AVX512VLBW-LABEL: splatvar_funnnel_v4i32:
+; AVX512VLBW:       # %bb.0:
+; AVX512VLBW-NEXT:    vpbroadcastd %xmm1, %xmm1
+; AVX512VLBW-NEXT:    vprolvd %xmm1, %xmm0, %xmm0
+; AVX512VLBW-NEXT:    retq
 ;
 ; XOPAVX1-LABEL: splatvar_funnnel_v4i32:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
-; XOPAVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [31,31,31,31]
-; XOPAVX1-NEXT:    vpand %xmm2, %xmm1, %xmm3
-; XOPAVX1-NEXT:    vpmovzxdq {{.*#+}} xmm3 = xmm3[0],zero,xmm3[1],zero
-; XOPAVX1-NEXT:    vpslld %xmm3, %xmm0, %xmm3
-; XOPAVX1-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; XOPAVX1-NEXT:    vpsubd %xmm1, %xmm4, %xmm1
-; XOPAVX1-NEXT:    vpand %xmm2, %xmm1, %xmm1
-; XOPAVX1-NEXT:    vpmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero
-; XOPAVX1-NEXT:    vpsrld %xmm1, %xmm0, %xmm0
-; XOPAVX1-NEXT:    vpor %xmm0, %xmm3, %xmm0
+; XOPAVX1-NEXT:    vprotd %xmm1, %xmm0, %xmm0
 ; XOPAVX1-NEXT:    retq
 ;
 ; XOPAVX2-LABEL: splatvar_funnnel_v4i32:
 ; XOPAVX2:       # %bb.0:
 ; XOPAVX2-NEXT:    vpbroadcastd %xmm1, %xmm1
-; XOPAVX2-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [31,31,31,31]
-; XOPAVX2-NEXT:    vpand %xmm2, %xmm1, %xmm3
-; XOPAVX2-NEXT:    vpmovzxdq {{.*#+}} xmm3 = xmm3[0],zero,xmm3[1],zero
-; XOPAVX2-NEXT:    vpslld %xmm3, %xmm0, %xmm3
-; XOPAVX2-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; XOPAVX2-NEXT:    vpsubd %xmm1, %xmm4, %xmm1
-; XOPAVX2-NEXT:    vpand %xmm2, %xmm1, %xmm1
-; XOPAVX2-NEXT:    vpmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero
-; XOPAVX2-NEXT:    vpsrld %xmm1, %xmm0, %xmm0
-; XOPAVX2-NEXT:    vpor %xmm0, %xmm3, %xmm0
+; XOPAVX2-NEXT:    vprotd %xmm1, %xmm0, %xmm0
 ; XOPAVX2-NEXT:    retq
 ;
 ; X32-SSE-LABEL: splatvar_funnnel_v4i32:
 ; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
-; X32-SSE-NEXT:    movdqa {{.*#+}} xmm2 = [31,31,31,31]
-; X32-SSE-NEXT:    pxor %xmm3, %xmm3
-; X32-SSE-NEXT:    psubd %xmm1, %xmm3
-; X32-SSE-NEXT:    pand %xmm2, %xmm1
-; X32-SSE-NEXT:    xorps %xmm4, %xmm4
-; X32-SSE-NEXT:    xorps %xmm5, %xmm5
-; X32-SSE-NEXT:    movss {{.*#+}} xmm5 = xmm1[0],xmm5[1,2,3]
-; X32-SSE-NEXT:    movdqa %xmm0, %xmm1
-; X32-SSE-NEXT:    pslld %xmm5, %xmm1
-; X32-SSE-NEXT:    pand %xmm2, %xmm3
-; X32-SSE-NEXT:    movss {{.*#+}} xmm4 = xmm3[0],xmm4[1,2,3]
-; X32-SSE-NEXT:    psrld %xmm4, %xmm0
-; X32-SSE-NEXT:    por %xmm1, %xmm0
+; X32-SSE-NEXT:    movd %xmm1, %eax
+; X32-SSE-NEXT:    andl $31, %eax
+; X32-SSE-NEXT:    movd %eax, %xmm1
+; X32-SSE-NEXT:    movdqa %xmm0, %xmm2
+; X32-SSE-NEXT:    pslld %xmm1, %xmm2
+; X32-SSE-NEXT:    movl $32, %ecx
+; X32-SSE-NEXT:    subl %eax, %ecx
+; X32-SSE-NEXT:    movd %ecx, %xmm1
+; X32-SSE-NEXT:    psrld %xmm1, %xmm0
+; X32-SSE-NEXT:    por %xmm2, %xmm0
 ; X32-SSE-NEXT:    retl
   %splat = shufflevector <4 x i32> %amt, <4 x i32> undef, <4 x i32> zeroinitializer
   %res = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %x, <4 x i32> %x, <4 x i32> %splat)
@@ -1182,132 +901,102 @@ define <8 x i16> @splatvar_funnnel_v8i16(<8 x i16> %x, <8 x i16> %amt) nounwind
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,0,2,3,4,5,6,7]
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15]
-; SSE2-NEXT:    pxor %xmm3, %xmm3
-; SSE2-NEXT:    psubw %xmm1, %xmm3
-; SSE2-NEXT:    pand %xmm2, %xmm1
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm1
+; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [16,16,16,16,16,16,16,16]
+; SSE2-NEXT:    psubw %xmm1, %xmm2
 ; SSE2-NEXT:    pslldq {{.*#+}} xmm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm1[0,1]
 ; SSE2-NEXT:    psrldq {{.*#+}} xmm1 = xmm1[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; SSE2-NEXT:    movdqa %xmm0, %xmm4
-; SSE2-NEXT:    psllw %xmm1, %xmm4
-; SSE2-NEXT:    pand %xmm2, %xmm3
-; SSE2-NEXT:    pslldq {{.*#+}} xmm3 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm3[0,1]
-; SSE2-NEXT:    psrldq {{.*#+}} xmm3 = xmm3[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; SSE2-NEXT:    psrlw %xmm3, %xmm0
-; SSE2-NEXT:    por %xmm4, %xmm0
+; SSE2-NEXT:    movdqa %xmm0, %xmm3
+; SSE2-NEXT:    psllw %xmm1, %xmm3
+; SSE2-NEXT:    pslldq {{.*#+}} xmm2 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm2[0,1]
+; SSE2-NEXT:    psrldq {{.*#+}} xmm2 = xmm2[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; SSE2-NEXT:    psrlw %xmm2, %xmm0
+; SSE2-NEXT:    por %xmm3, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: splatvar_funnnel_v8i16:
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,0,2,3,4,5,6,7]
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
-; SSE41-NEXT:    movdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15]
-; SSE41-NEXT:    pxor %xmm3, %xmm3
-; SSE41-NEXT:    psubw %xmm1, %xmm3
-; SSE41-NEXT:    pand %xmm2, %xmm1
-; SSE41-NEXT:    pmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
-; SSE41-NEXT:    movdqa %xmm0, %xmm4
-; SSE41-NEXT:    psllw %xmm1, %xmm4
-; SSE41-NEXT:    pand %xmm2, %xmm3
-; SSE41-NEXT:    pmovzxwq {{.*#+}} xmm1 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero
+; SSE41-NEXT:    pand {{.*}}(%rip), %xmm1
+; SSE41-NEXT:    pmovzxwq {{.*#+}} xmm2 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; SSE41-NEXT:    movdqa %xmm0, %xmm3
+; SSE41-NEXT:    psllw %xmm2, %xmm3
+; SSE41-NEXT:    movdqa {{.*#+}} xmm2 = [16,16,16,16,16,16,16,16]
+; SSE41-NEXT:    psubw %xmm1, %xmm2
+; SSE41-NEXT:    pmovzxwq {{.*#+}} xmm1 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero
 ; SSE41-NEXT:    psrlw %xmm1, %xmm0
-; SSE41-NEXT:    por %xmm4, %xmm0
+; SSE41-NEXT:    por %xmm3, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: splatvar_funnnel_v8i16:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm1[0,0,2,3,4,5,6,7]
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15]
-; AVX1-NEXT:    vpand %xmm2, %xmm1, %xmm3
-; AVX1-NEXT:    vpmovzxwq {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero
-; AVX1-NEXT:    vpsllw %xmm3, %xmm0, %xmm3
-; AVX1-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX1-NEXT:    vpsubw %xmm1, %xmm4, %xmm1
-; AVX1-NEXT:    vpand %xmm2, %xmm1, %xmm1
+; AVX1-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX1-NEXT:    vpmovzxwq {{.*#+}} xmm2 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; AVX1-NEXT:    vpsllw %xmm2, %xmm0, %xmm2
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [16,16,16,16,16,16,16,16]
+; AVX1-NEXT:    vpsubw %xmm1, %xmm3, %xmm1
 ; AVX1-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
 ; AVX1-NEXT:    vpsrlw %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpor %xmm0, %xmm3, %xmm0
+; AVX1-NEXT:    vpor %xmm0, %xmm2, %xmm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: splatvar_funnnel_v8i16:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpbroadcastw %xmm1, %xmm1
-; AVX2-NEXT:    vmovdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15]
-; AVX2-NEXT:    vpand %xmm2, %xmm1, %xmm3
-; AVX2-NEXT:    vpmovzxwq {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero
-; AVX2-NEXT:    vpsllw %xmm3, %xmm0, %xmm3
-; AVX2-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX2-NEXT:    vpsubw %xmm1, %xmm4, %xmm1
-; AVX2-NEXT:    vpand %xmm2, %xmm1, %xmm1
+; AVX2-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX2-NEXT:    vpmovzxwq {{.*#+}} xmm2 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; AVX2-NEXT:    vpsllw %xmm2, %xmm0, %xmm2
+; AVX2-NEXT:    vmovdqa {{.*#+}} xmm3 = [16,16,16,16,16,16,16,16]
+; AVX2-NEXT:    vpsubw %xmm1, %xmm3, %xmm1
 ; AVX2-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
 ; AVX2-NEXT:    vpsrlw %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpor %xmm0, %xmm3, %xmm0
+; AVX2-NEXT:    vpor %xmm0, %xmm2, %xmm0
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: splatvar_funnnel_v8i16:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpbroadcastw %xmm1, %xmm1
-; AVX512-NEXT:    vmovdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15]
-; AVX512-NEXT:    vpand %xmm2, %xmm1, %xmm3
-; AVX512-NEXT:    vpmovzxwq {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero
-; AVX512-NEXT:    vpsllw %xmm3, %xmm0, %xmm3
-; AVX512-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX512-NEXT:    vpsubw %xmm1, %xmm4, %xmm1
-; AVX512-NEXT:    vpand %xmm2, %xmm1, %xmm1
+; AVX512-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX512-NEXT:    vpmovzxwq {{.*#+}} xmm2 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; AVX512-NEXT:    vpsllw %xmm2, %xmm0, %xmm2
+; AVX512-NEXT:    vmovdqa {{.*#+}} xmm3 = [16,16,16,16,16,16,16,16]
+; AVX512-NEXT:    vpsubw %xmm1, %xmm3, %xmm1
 ; AVX512-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
 ; AVX512-NEXT:    vpsrlw %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpor %xmm0, %xmm3, %xmm0
+; AVX512-NEXT:    vpor %xmm0, %xmm2, %xmm0
 ; AVX512-NEXT:    retq
 ;
 ; XOPAVX1-LABEL: splatvar_funnnel_v8i16:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm1[0,0,2,3,4,5,6,7]
 ; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
-; XOPAVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15]
-; XOPAVX1-NEXT:    vpand %xmm2, %xmm1, %xmm3
-; XOPAVX1-NEXT:    vpmovzxwq {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero
-; XOPAVX1-NEXT:    vpsllw %xmm3, %xmm0, %xmm3
-; XOPAVX1-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; XOPAVX1-NEXT:    vpsubw %xmm1, %xmm4, %xmm1
-; XOPAVX1-NEXT:    vpand %xmm2, %xmm1, %xmm1
-; XOPAVX1-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
-; XOPAVX1-NEXT:    vpsrlw %xmm1, %xmm0, %xmm0
-; XOPAVX1-NEXT:    vpor %xmm0, %xmm3, %xmm0
+; XOPAVX1-NEXT:    vprotw %xmm1, %xmm0, %xmm0
 ; XOPAVX1-NEXT:    retq
 ;
 ; XOPAVX2-LABEL: splatvar_funnnel_v8i16:
 ; XOPAVX2:       # %bb.0:
 ; XOPAVX2-NEXT:    vpbroadcastw %xmm1, %xmm1
-; XOPAVX2-NEXT:    vmovdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15]
-; XOPAVX2-NEXT:    vpand %xmm2, %xmm1, %xmm3
-; XOPAVX2-NEXT:    vpmovzxwq {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero
-; XOPAVX2-NEXT:    vpsllw %xmm3, %xmm0, %xmm3
-; XOPAVX2-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; XOPAVX2-NEXT:    vpsubw %xmm1, %xmm4, %xmm1
-; XOPAVX2-NEXT:    vpand %xmm2, %xmm1, %xmm1
-; XOPAVX2-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
-; XOPAVX2-NEXT:    vpsrlw %xmm1, %xmm0, %xmm0
-; XOPAVX2-NEXT:    vpor %xmm0, %xmm3, %xmm0
+; XOPAVX2-NEXT:    vprotw %xmm1, %xmm0, %xmm0
 ; XOPAVX2-NEXT:    retq
 ;
 ; X32-SSE-LABEL: splatvar_funnnel_v8i16:
 ; X32-SSE:       # %bb.0:
 ; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,0,2,3,4,5,6,7]
 ; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
-; X32-SSE-NEXT:    movdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15]
-; X32-SSE-NEXT:    pxor %xmm3, %xmm3
-; X32-SSE-NEXT:    psubw %xmm1, %xmm3
-; X32-SSE-NEXT:    pand %xmm2, %xmm1
+; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm1
+; X32-SSE-NEXT:    movdqa {{.*#+}} xmm2 = [16,16,16,16,16,16,16,16]
+; X32-SSE-NEXT:    psubw %xmm1, %xmm2
 ; X32-SSE-NEXT:    pslldq {{.*#+}} xmm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm1[0,1]
 ; X32-SSE-NEXT:    psrldq {{.*#+}} xmm1 = xmm1[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; X32-SSE-NEXT:    movdqa %xmm0, %xmm4
-; X32-SSE-NEXT:    psllw %xmm1, %xmm4
-; X32-SSE-NEXT:    pand %xmm2, %xmm3
-; X32-SSE-NEXT:    pslldq {{.*#+}} xmm3 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm3[0,1]
-; X32-SSE-NEXT:    psrldq {{.*#+}} xmm3 = xmm3[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; X32-SSE-NEXT:    psrlw %xmm3, %xmm0
-; X32-SSE-NEXT:    por %xmm4, %xmm0
+; X32-SSE-NEXT:    movdqa %xmm0, %xmm3
+; X32-SSE-NEXT:    psllw %xmm1, %xmm3
+; X32-SSE-NEXT:    pslldq {{.*#+}} xmm2 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm2[0,1]
+; X32-SSE-NEXT:    psrldq {{.*#+}} xmm2 = xmm2[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; X32-SSE-NEXT:    psrlw %xmm2, %xmm0
+; X32-SSE-NEXT:    por %xmm3, %xmm0
 ; X32-SSE-NEXT:    retl
   %splat = shufflevector <8 x i16> %amt, <8 x i16> undef, <8 x i32> zeroinitializer
   %res = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %x, <8 x i16> %x, <8 x i16> %splat)
@@ -1320,30 +1009,28 @@ define <16 x i8> @splatvar_funnnel_v16i8(<16 x i8> %x, <16 x i8> %amt) nounwind
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
 ; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,0,2,3,4,5,6,7]
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
-; SSE2-NEXT:    pxor %xmm3, %xmm3
-; SSE2-NEXT:    psubb %xmm1, %xmm3
-; SSE2-NEXT:    movdqa %xmm1, %xmm4
-; SSE2-NEXT:    pand %xmm2, %xmm4
-; SSE2-NEXT:    pslldq {{.*#+}} xmm4 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm4[0]
-; SSE2-NEXT:    psrldq {{.*#+}} xmm4 = xmm4[15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; SSE2-NEXT:    movdqa %xmm0, %xmm1
-; SSE2-NEXT:    psllw %xmm4, %xmm1
-; SSE2-NEXT:    pcmpeqd %xmm5, %xmm5
-; SSE2-NEXT:    pcmpeqd %xmm6, %xmm6
-; SSE2-NEXT:    psllw %xmm4, %xmm6
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm6 = xmm6[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm4 = xmm6[0,0,2,3,4,5,6,7]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[0,0,0,0]
-; SSE2-NEXT:    pand %xmm4, %xmm1
-; SSE2-NEXT:    pand %xmm2, %xmm3
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm1
+; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; SSE2-NEXT:    psubb %xmm1, %xmm2
+; SSE2-NEXT:    movdqa %xmm1, %xmm3
 ; SSE2-NEXT:    pslldq {{.*#+}} xmm3 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm3[0]
 ; SSE2-NEXT:    psrldq {{.*#+}} xmm3 = xmm3[15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; SSE2-NEXT:    psrlw %xmm3, %xmm0
-; SSE2-NEXT:    psrlw %xmm3, %xmm5
-; SSE2-NEXT:    psrlw $8, %xmm5
+; SSE2-NEXT:    movdqa %xmm0, %xmm1
+; SSE2-NEXT:    psllw %xmm3, %xmm1
+; SSE2-NEXT:    pcmpeqd %xmm4, %xmm4
+; SSE2-NEXT:    pcmpeqd %xmm5, %xmm5
+; SSE2-NEXT:    psllw %xmm3, %xmm5
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm2 = xmm5[0,0,2,3,4,5,6,7]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm3 = xmm5[0,0,2,3,4,5,6,7]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[0,0,0,0]
+; SSE2-NEXT:    pand %xmm3, %xmm1
+; SSE2-NEXT:    pslldq {{.*#+}} xmm2 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm2[0]
+; SSE2-NEXT:    psrldq {{.*#+}} xmm2 = xmm2[15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; SSE2-NEXT:    psrlw %xmm2, %xmm0
+; SSE2-NEXT:    psrlw %xmm2, %xmm4
+; SSE2-NEXT:    psrlw $8, %xmm4
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm2 = xmm4[0,0,2,3,4,5,6,7]
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,0,0,0]
 ; SSE2-NEXT:    pand %xmm0, %xmm2
 ; SSE2-NEXT:    por %xmm2, %xmm1
@@ -1354,25 +1041,23 @@ define <16 x i8> @splatvar_funnnel_v16i8(<16 x i8> %x, <16 x i8> %amt) nounwind
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    pxor %xmm3, %xmm3
 ; SSE41-NEXT:    pshufb %xmm3, %xmm1
-; SSE41-NEXT:    movdqa {{.*#+}} xmm4 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
-; SSE41-NEXT:    movdqa %xmm1, %xmm2
-; SSE41-NEXT:    pand %xmm4, %xmm2
-; SSE41-NEXT:    pmovzxbq {{.*#+}} xmm5 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero
+; SSE41-NEXT:    pand {{.*}}(%rip), %xmm1
+; SSE41-NEXT:    pmovzxbq {{.*#+}} xmm4 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
 ; SSE41-NEXT:    movdqa %xmm0, %xmm2
-; SSE41-NEXT:    psllw %xmm5, %xmm2
+; SSE41-NEXT:    psllw %xmm4, %xmm2
+; SSE41-NEXT:    pcmpeqd %xmm5, %xmm5
 ; SSE41-NEXT:    pcmpeqd %xmm6, %xmm6
-; SSE41-NEXT:    pcmpeqd %xmm7, %xmm7
-; SSE41-NEXT:    psllw %xmm5, %xmm7
-; SSE41-NEXT:    pshufb %xmm3, %xmm7
-; SSE41-NEXT:    pand %xmm7, %xmm2
+; SSE41-NEXT:    psllw %xmm4, %xmm6
+; SSE41-NEXT:    pshufb %xmm3, %xmm6
+; SSE41-NEXT:    pand %xmm6, %xmm2
+; SSE41-NEXT:    movdqa {{.*#+}} xmm3 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
 ; SSE41-NEXT:    psubb %xmm1, %xmm3
-; SSE41-NEXT:    pand %xmm4, %xmm3
 ; SSE41-NEXT:    pmovzxbq {{.*#+}} xmm1 = xmm3[0],zero,zero,zero,zero,zero,zero,zero,xmm3[1],zero,zero,zero,zero,zero,zero,zero
 ; SSE41-NEXT:    psrlw %xmm1, %xmm0
-; SSE41-NEXT:    psrlw %xmm1, %xmm6
-; SSE41-NEXT:    pshufb {{.*#+}} xmm6 = xmm6[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
-; SSE41-NEXT:    pand %xmm0, %xmm6
-; SSE41-NEXT:    por %xmm6, %xmm2
+; SSE41-NEXT:    psrlw %xmm1, %xmm5
+; SSE41-NEXT:    pshufb {{.*#+}} xmm5 = xmm5[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
+; SSE41-NEXT:    pand %xmm0, %xmm5
+; SSE41-NEXT:    por %xmm5, %xmm2
 ; SSE41-NEXT:    movdqa %xmm2, %xmm0
 ; SSE41-NEXT:    retq
 ;
@@ -1380,45 +1065,42 @@ define <16 x i8> @splatvar_funnnel_v16i8(<16 x i8> %x, <16 x i8> %amt) nounwind
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
 ; AVX1-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
-; AVX1-NEXT:    vpand %xmm3, %xmm1, %xmm4
-; AVX1-NEXT:    vpmovzxbq {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,zero,zero,zero,zero,xmm4[1],zero,zero,zero,zero,zero,zero,zero
-; AVX1-NEXT:    vpsllw %xmm4, %xmm0, %xmm5
-; AVX1-NEXT:    vpcmpeqd %xmm6, %xmm6, %xmm6
-; AVX1-NEXT:    vpsllw %xmm4, %xmm6, %xmm4
-; AVX1-NEXT:    vpshufb %xmm2, %xmm4, %xmm4
-; AVX1-NEXT:    vpand %xmm4, %xmm5, %xmm4
-; AVX1-NEXT:    vpsubb %xmm1, %xmm2, %xmm1
-; AVX1-NEXT:    vpand %xmm3, %xmm1, %xmm1
+; AVX1-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX1-NEXT:    vpmovzxbq {{.*#+}} xmm3 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; AVX1-NEXT:    vpsllw %xmm3, %xmm0, %xmm4
+; AVX1-NEXT:    vpcmpeqd %xmm5, %xmm5, %xmm5
+; AVX1-NEXT:    vpsllw %xmm3, %xmm5, %xmm3
+; AVX1-NEXT:    vpshufb %xmm2, %xmm3, %xmm2
+; AVX1-NEXT:    vpand %xmm2, %xmm4, %xmm2
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; AVX1-NEXT:    vpsubb %xmm1, %xmm3, %xmm1
 ; AVX1-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
 ; AVX1-NEXT:    vpsrlw %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpsrlw %xmm1, %xmm6, %xmm1
+; AVX1-NEXT:    vpsrlw %xmm1, %xmm5, %xmm1
 ; AVX1-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
 ; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpor %xmm0, %xmm4, %xmm0
+; AVX1-NEXT:    vpor %xmm0, %xmm2, %xmm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: splatvar_funnnel_v16i8:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpbroadcastb %xmm1, %xmm1
-; AVX2-NEXT:    vmovdqa {{.*#+}} xmm2 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
-; AVX2-NEXT:    vpand %xmm2, %xmm1, %xmm3
-; AVX2-NEXT:    vpmovzxbq {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,zero,zero,zero,zero,xmm3[1],zero,zero,zero,zero,zero,zero,zero
-; AVX2-NEXT:    vpsllw %xmm3, %xmm0, %xmm4
-; AVX2-NEXT:    vpcmpeqd %xmm5, %xmm5, %xmm5
-; AVX2-NEXT:    vpsllw %xmm3, %xmm5, %xmm3
-; AVX2-NEXT:    vpbroadcastb %xmm3, %xmm3
-; AVX2-NEXT:    vpand %xmm3, %xmm4, %xmm3
-; AVX2-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX2-NEXT:    vpsubb %xmm1, %xmm4, %xmm1
-; AVX2-NEXT:    vpand %xmm2, %xmm1, %xmm1
+; AVX2-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX2-NEXT:    vpmovzxbq {{.*#+}} xmm2 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; AVX2-NEXT:    vpsllw %xmm2, %xmm0, %xmm3
+; AVX2-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
+; AVX2-NEXT:    vpsllw %xmm2, %xmm4, %xmm2
+; AVX2-NEXT:    vpbroadcastb %xmm2, %xmm2
+; AVX2-NEXT:    vpand %xmm2, %xmm3, %xmm2
+; AVX2-NEXT:    vmovdqa {{.*#+}} xmm3 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; AVX2-NEXT:    vpsubb %xmm1, %xmm3, %xmm1
 ; AVX2-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
 ; AVX2-NEXT:    vpsrlw %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpsrlw %xmm1, %xmm5, %xmm1
+; AVX2-NEXT:    vpsrlw %xmm1, %xmm4, %xmm1
 ; AVX2-NEXT:    vpsrlw $8, %xmm1, %xmm1
 ; AVX2-NEXT:    vpbroadcastb %xmm1, %xmm1
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpor %xmm0, %xmm3, %xmm0
+; AVX2-NEXT:    vpor %xmm0, %xmm2, %xmm0
 ; AVX2-NEXT:    retq
 ;
 ; AVX512F-LABEL: splatvar_funnnel_v16i8:
@@ -1498,28 +1180,13 @@ define <16 x i8> @splatvar_funnnel_v16i8(<16 x i8> %x, <16 x i8> %amt) nounwind
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
 ; XOPAVX1-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
-; XOPAVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
-; XOPAVX1-NEXT:    vpand %xmm3, %xmm1, %xmm4
-; XOPAVX1-NEXT:    vpshlb %xmm4, %xmm0, %xmm4
-; XOPAVX1-NEXT:    vpsubb %xmm1, %xmm2, %xmm1
-; XOPAVX1-NEXT:    vpand %xmm3, %xmm1, %xmm1
-; XOPAVX1-NEXT:    vpsubb %xmm1, %xmm2, %xmm1
-; XOPAVX1-NEXT:    vpshlb %xmm1, %xmm0, %xmm0
-; XOPAVX1-NEXT:    vpor %xmm0, %xmm4, %xmm0
+; XOPAVX1-NEXT:    vprotb %xmm1, %xmm0, %xmm0
 ; XOPAVX1-NEXT:    retq
 ;
 ; XOPAVX2-LABEL: splatvar_funnnel_v16i8:
 ; XOPAVX2:       # %bb.0:
 ; XOPAVX2-NEXT:    vpbroadcastb %xmm1, %xmm1
-; XOPAVX2-NEXT:    vmovdqa {{.*#+}} xmm2 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
-; XOPAVX2-NEXT:    vpand %xmm2, %xmm1, %xmm3
-; XOPAVX2-NEXT:    vpshlb %xmm3, %xmm0, %xmm3
-; XOPAVX2-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; XOPAVX2-NEXT:    vpsubb %xmm1, %xmm4, %xmm1
-; XOPAVX2-NEXT:    vpand %xmm2, %xmm1, %xmm1
-; XOPAVX2-NEXT:    vpsubb %xmm1, %xmm4, %xmm1
-; XOPAVX2-NEXT:    vpshlb %xmm1, %xmm0, %xmm0
-; XOPAVX2-NEXT:    vpor %xmm0, %xmm3, %xmm0
+; XOPAVX2-NEXT:    vprotb %xmm1, %xmm0, %xmm0
 ; XOPAVX2-NEXT:    retq
 ;
 ; X32-SSE-LABEL: splatvar_funnnel_v16i8:
@@ -1527,30 +1194,28 @@ define <16 x i8> @splatvar_funnnel_v16i8(<16 x i8> %x, <16 x i8> %amt) nounwind
 ; X32-SSE-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
 ; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,0,2,3,4,5,6,7]
 ; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
-; X32-SSE-NEXT:    movdqa {{.*#+}} xmm2 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
-; X32-SSE-NEXT:    pxor %xmm3, %xmm3
-; X32-SSE-NEXT:    psubb %xmm1, %xmm3
-; X32-SSE-NEXT:    movdqa %xmm1, %xmm4
-; X32-SSE-NEXT:    pand %xmm2, %xmm4
-; X32-SSE-NEXT:    pslldq {{.*#+}} xmm4 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm4[0]
-; X32-SSE-NEXT:    psrldq {{.*#+}} xmm4 = xmm4[15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; X32-SSE-NEXT:    movdqa %xmm0, %xmm1
-; X32-SSE-NEXT:    psllw %xmm4, %xmm1
-; X32-SSE-NEXT:    pcmpeqd %xmm5, %xmm5
-; X32-SSE-NEXT:    pcmpeqd %xmm6, %xmm6
-; X32-SSE-NEXT:    psllw %xmm4, %xmm6
-; X32-SSE-NEXT:    punpcklbw {{.*#+}} xmm6 = xmm6[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm4 = xmm6[0,0,2,3,4,5,6,7]
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[0,0,0,0]
-; X32-SSE-NEXT:    pand %xmm4, %xmm1
-; X32-SSE-NEXT:    pand %xmm2, %xmm3
+; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm1
+; X32-SSE-NEXT:    movdqa {{.*#+}} xmm2 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; X32-SSE-NEXT:    psubb %xmm1, %xmm2
+; X32-SSE-NEXT:    movdqa %xmm1, %xmm3
 ; X32-SSE-NEXT:    pslldq {{.*#+}} xmm3 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm3[0]
 ; X32-SSE-NEXT:    psrldq {{.*#+}} xmm3 = xmm3[15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; X32-SSE-NEXT:    psrlw %xmm3, %xmm0
-; X32-SSE-NEXT:    psrlw %xmm3, %xmm5
-; X32-SSE-NEXT:    psrlw $8, %xmm5
+; X32-SSE-NEXT:    movdqa %xmm0, %xmm1
+; X32-SSE-NEXT:    psllw %xmm3, %xmm1
+; X32-SSE-NEXT:    pcmpeqd %xmm4, %xmm4
+; X32-SSE-NEXT:    pcmpeqd %xmm5, %xmm5
+; X32-SSE-NEXT:    psllw %xmm3, %xmm5
 ; X32-SSE-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm2 = xmm5[0,0,2,3,4,5,6,7]
+; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm3 = xmm5[0,0,2,3,4,5,6,7]
+; X32-SSE-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[0,0,0,0]
+; X32-SSE-NEXT:    pand %xmm3, %xmm1
+; X32-SSE-NEXT:    pslldq {{.*#+}} xmm2 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm2[0]
+; X32-SSE-NEXT:    psrldq {{.*#+}} xmm2 = xmm2[15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; X32-SSE-NEXT:    psrlw %xmm2, %xmm0
+; X32-SSE-NEXT:    psrlw %xmm2, %xmm4
+; X32-SSE-NEXT:    psrlw $8, %xmm4
+; X32-SSE-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm2 = xmm4[0,0,2,3,4,5,6,7]
 ; X32-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,0,0,0]
 ; X32-SSE-NEXT:    pand %xmm0, %xmm2
 ; X32-SSE-NEXT:    por %xmm2, %xmm1
@@ -1767,55 +1432,43 @@ define <4 x i32> @constant_funnnel_v4i32(<4 x i32> %x) nounwind {
 }
 
 define <8 x i16> @constant_funnnel_v8i16(<8 x i16> %x) nounwind {
-; SSE2-LABEL: constant_funnnel_v8i16:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [0,65535,65535,65535,65535,65535,65535,65535]
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = <u,2,4,8,16,32,64,128>
-; SSE2-NEXT:    pmulhuw %xmm0, %xmm2
-; SSE2-NEXT:    pand %xmm1, %xmm2
-; SSE2-NEXT:    pandn %xmm0, %xmm1
-; SSE2-NEXT:    pmullw {{.*}}(%rip), %xmm0
-; SSE2-NEXT:    por %xmm1, %xmm0
-; SSE2-NEXT:    por %xmm2, %xmm0
-; SSE2-NEXT:    retq
-;
-; SSE41-LABEL: constant_funnnel_v8i16:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    movdqa {{.*#+}} xmm1 = <u,2,4,8,16,32,64,128>
-; SSE41-NEXT:    pmulhuw %xmm0, %xmm1
-; SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3,4,5,6,7]
-; SSE41-NEXT:    pmullw {{.*}}(%rip), %xmm0
-; SSE41-NEXT:    por %xmm1, %xmm0
-; SSE41-NEXT:    retq
+; SSE-LABEL: constant_funnnel_v8i16:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movdqa {{.*#+}} xmm1 = [1,2,4,8,16,32,64,128]
+; SSE-NEXT:    movdqa %xmm0, %xmm2
+; SSE-NEXT:    pmulhuw %xmm1, %xmm2
+; SSE-NEXT:    pmullw %xmm1, %xmm0
+; SSE-NEXT:    por %xmm2, %xmm0
+; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: constant_funnnel_v8i16:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpmulhuw {{.*}}(%rip), %xmm0, %xmm1
-; AVX-NEXT:    vpblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3,4,5,6,7]
-; AVX-NEXT:    vpmullw {{.*}}(%rip), %xmm0, %xmm0
-; AVX-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vmovdqa {{.*#+}} xmm1 = [1,2,4,8,16,32,64,128]
+; AVX-NEXT:    vpmulhuw %xmm1, %xmm0, %xmm2
+; AVX-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vpor %xmm2, %xmm0, %xmm0
 ; AVX-NEXT:    retq
 ;
 ; AVX512F-LABEL: constant_funnnel_v8i16:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpmulhuw {{.*}}(%rip), %xmm0, %xmm1
-; AVX512F-NEXT:    vpblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3,4,5,6,7]
-; AVX512F-NEXT:    vpmullw {{.*}}(%rip), %xmm0, %xmm0
-; AVX512F-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm1 = [1,2,4,8,16,32,64,128]
+; AVX512F-NEXT:    vpmulhuw %xmm1, %xmm0, %xmm2
+; AVX512F-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
+; AVX512F-NEXT:    vpor %xmm2, %xmm0, %xmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: constant_funnnel_v8i16:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpmulhuw {{.*}}(%rip), %xmm0, %xmm1
-; AVX512VL-NEXT:    vpblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3,4,5,6,7]
-; AVX512VL-NEXT:    vpmullw {{.*}}(%rip), %xmm0, %xmm0
-; AVX512VL-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm1 = [1,2,4,8,16,32,64,128]
+; AVX512VL-NEXT:    vpmulhuw %xmm1, %xmm0, %xmm2
+; AVX512VL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
+; AVX512VL-NEXT:    vpor %xmm2, %xmm0, %xmm0
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: constant_funnnel_v8i16:
 ; AVX512BW:       # %bb.0:
 ; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm1 = [0,15,14,13,12,11,10,9]
+; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm1 = [16,15,14,13,12,11,10,9]
 ; AVX512BW-NEXT:    vpsrlvw %zmm1, %zmm0, %zmm1
 ; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm2 = [0,1,2,3,4,5,6,7]
 ; AVX512BW-NEXT:    vpsllvw %zmm2, %zmm0, %zmm0
@@ -1832,20 +1485,15 @@ define <8 x i16> @constant_funnnel_v8i16(<8 x i16> %x) nounwind {
 ;
 ; XOP-LABEL: constant_funnnel_v8i16:
 ; XOP:       # %bb.0:
-; XOP-NEXT:    vpshlw {{.*}}(%rip), %xmm0, %xmm1
-; XOP-NEXT:    vpshlw {{.*}}(%rip), %xmm0, %xmm0
-; XOP-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; XOP-NEXT:    vprotw {{.*}}(%rip), %xmm0, %xmm0
 ; XOP-NEXT:    retq
 ;
 ; X32-SSE-LABEL: constant_funnnel_v8i16:
 ; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    movdqa {{.*#+}} xmm1 = [0,65535,65535,65535,65535,65535,65535,65535]
-; X32-SSE-NEXT:    movdqa {{.*#+}} xmm2 = <u,2,4,8,16,32,64,128>
-; X32-SSE-NEXT:    pmulhuw %xmm0, %xmm2
-; X32-SSE-NEXT:    pand %xmm1, %xmm2
-; X32-SSE-NEXT:    pandn %xmm0, %xmm1
-; X32-SSE-NEXT:    pmullw {{\.LCPI.*}}, %xmm0
-; X32-SSE-NEXT:    por %xmm1, %xmm0
+; X32-SSE-NEXT:    movdqa {{.*#+}} xmm1 = [1,2,4,8,16,32,64,128]
+; X32-SSE-NEXT:    movdqa %xmm0, %xmm2
+; X32-SSE-NEXT:    pmulhuw %xmm1, %xmm2
+; X32-SSE-NEXT:    pmullw %xmm1, %xmm0
 ; X32-SSE-NEXT:    por %xmm2, %xmm0
 ; X32-SSE-NEXT:    retl
   %res = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %x, <8 x i16> %x, <8 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7>)
@@ -1980,9 +1628,7 @@ define <16 x i8> @constant_funnnel_v16i8(<16 x i8> %x) nounwind {
 ;
 ; XOP-LABEL: constant_funnnel_v16i8:
 ; XOP:       # %bb.0:
-; XOP-NEXT:    vpshlb {{.*}}(%rip), %xmm0, %xmm1
-; XOP-NEXT:    vpshlb {{.*}}(%rip), %xmm0, %xmm0
-; XOP-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; XOP-NEXT:    vprotb {{.*}}(%rip), %xmm0, %xmm0
 ; XOP-NEXT:    retq
 ;
 ; X32-SSE-LABEL: constant_funnnel_v16i8:
diff --git a/test/CodeGen/X86/vector-fshl-rot-256.ll b/test/CodeGen/X86/vector-fshl-rot-256.ll
index 6f72c14f7436d4c5036ca0c4388415083e1e1342..64e5efa4b116506f8d1230b09b91f3ac4c453d40 100644
--- a/test/CodeGen/X86/vector-fshl-rot-256.ll
+++ b/test/CodeGen/X86/vector-fshl-rot-256.ll
@@ -63,51 +63,48 @@ define <4 x i64> @var_funnnel_v4i64(<4 x i64> %x, <4 x i64> %amt) nounwind {
 ; AVX2-NEXT:    vpor %ymm0, %ymm3, %ymm0
 ; AVX2-NEXT:    retq
 ;
-; AVX512-LABEL: var_funnnel_v4i64:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [63,63,63,63]
-; AVX512-NEXT:    vpand %ymm2, %ymm1, %ymm3
-; AVX512-NEXT:    vpsllvq %ymm3, %ymm0, %ymm3
-; AVX512-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX512-NEXT:    vpsubq %ymm1, %ymm4, %ymm1
-; AVX512-NEXT:    vpand %ymm2, %ymm1, %ymm1
-; AVX512-NEXT:    vpsrlvq %ymm1, %ymm0, %ymm0
-; AVX512-NEXT:    vpor %ymm0, %ymm3, %ymm0
-; AVX512-NEXT:    retq
+; AVX512F-LABEL: var_funnnel_v4i64:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512F-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512F-NEXT:    vprolvq %zmm1, %zmm0, %zmm0
+; AVX512F-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: var_funnnel_v4i64:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vprolvq %ymm1, %ymm0, %ymm0
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: var_funnnel_v4i64:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512BW-NEXT:    vprolvq %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
+; AVX512BW-NEXT:    retq
+;
+; AVX512VLBW-LABEL: var_funnnel_v4i64:
+; AVX512VLBW:       # %bb.0:
+; AVX512VLBW-NEXT:    vprolvq %ymm1, %ymm0, %ymm0
+; AVX512VLBW-NEXT:    retq
 ;
 ; XOPAVX1-LABEL: var_funnnel_v4i64:
 ; XOPAVX1:       # %bb.0:
-; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
-; XOPAVX1-NEXT:    vandps {{.*}}(%rip), %ymm1, %ymm3
-; XOPAVX1-NEXT:    vextractf128 $1, %ymm3, %xmm4
-; XOPAVX1-NEXT:    vpshlq %xmm4, %xmm2, %xmm4
-; XOPAVX1-NEXT:    vpshlq %xmm3, %xmm0, %xmm3
-; XOPAVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm3, %ymm3
-; XOPAVX1-NEXT:    vextractf128 $1, %ymm1, %xmm4
-; XOPAVX1-NEXT:    vpxor %xmm5, %xmm5, %xmm5
-; XOPAVX1-NEXT:    vpsubq %xmm4, %xmm5, %xmm4
-; XOPAVX1-NEXT:    vmovdqa {{.*#+}} xmm6 = [63,63]
-; XOPAVX1-NEXT:    vpand %xmm6, %xmm4, %xmm4
-; XOPAVX1-NEXT:    vpsubq %xmm4, %xmm5, %xmm4
-; XOPAVX1-NEXT:    vpshlq %xmm4, %xmm2, %xmm2
-; XOPAVX1-NEXT:    vpsubq %xmm1, %xmm5, %xmm1
-; XOPAVX1-NEXT:    vpand %xmm6, %xmm1, %xmm1
-; XOPAVX1-NEXT:    vpsubq %xmm1, %xmm5, %xmm1
-; XOPAVX1-NEXT:    vpshlq %xmm1, %xmm0, %xmm0
+; XOPAVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
+; XOPAVX1-NEXT:    vprotq %xmm2, %xmm3, %xmm2
+; XOPAVX1-NEXT:    vprotq %xmm1, %xmm0, %xmm0
 ; XOPAVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
-; XOPAVX1-NEXT:    vorps %ymm0, %ymm3, %ymm0
 ; XOPAVX1-NEXT:    retq
 ;
 ; XOPAVX2-LABEL: var_funnnel_v4i64:
 ; XOPAVX2:       # %bb.0:
-; XOPAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [63,63,63,63]
-; XOPAVX2-NEXT:    vpand %ymm2, %ymm1, %ymm3
-; XOPAVX2-NEXT:    vpsllvq %ymm3, %ymm0, %ymm3
-; XOPAVX2-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; XOPAVX2-NEXT:    vpsubq %ymm1, %ymm4, %ymm1
-; XOPAVX2-NEXT:    vpand %ymm2, %ymm1, %ymm1
-; XOPAVX2-NEXT:    vpsrlvq %ymm1, %ymm0, %ymm0
-; XOPAVX2-NEXT:    vpor %ymm0, %ymm3, %ymm0
+; XOPAVX2-NEXT:    vextracti128 $1, %ymm1, %xmm2
+; XOPAVX2-NEXT:    vextracti128 $1, %ymm0, %xmm3
+; XOPAVX2-NEXT:    vprotq %xmm2, %xmm3, %xmm2
+; XOPAVX2-NEXT:    vprotq %xmm1, %xmm0, %xmm0
+; XOPAVX2-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm0
 ; XOPAVX2-NEXT:    retq
   %res = call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %x, <4 x i64> %x, <4 x i64> %amt)
   ret <4 x i64> %res
@@ -117,108 +114,91 @@ define <8 x i32> @var_funnnel_v8i32(<8 x i32> %x, <8 x i32> %amt) nounwind {
 ; AVX1-LABEL: var_funnnel_v8i32:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
-; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpsubd %xmm2, %xmm3, %xmm2
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = [31,31,31,31]
-; AVX1-NEXT:    vpand %xmm4, %xmm2, %xmm5
-; AVX1-NEXT:    vpsrldq {{.*#+}} xmm6 = xmm5[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
-; AVX1-NEXT:    vpsrld %xmm6, %xmm2, %xmm6
-; AVX1-NEXT:    vpsrlq $32, %xmm5, %xmm7
-; AVX1-NEXT:    vpsrld %xmm7, %xmm2, %xmm7
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm6 = xmm7[0,1,2,3],xmm6[4,5,6,7]
-; AVX1-NEXT:    vpunpckhdq {{.*#+}} xmm7 = xmm5[2],xmm3[2],xmm5[3],xmm3[3]
-; AVX1-NEXT:    vpsrld %xmm7, %xmm2, %xmm7
-; AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm5 = xmm5[0],zero,xmm5[1],zero
-; AVX1-NEXT:    vpsrld %xmm5, %xmm2, %xmm5
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm5 = xmm5[0,1,2,3],xmm7[4,5,6,7]
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm5 = xmm5[0,1],xmm6[2,3],xmm5[4,5],xmm6[6,7]
-; AVX1-NEXT:    vpsubd %xmm1, %xmm3, %xmm6
-; AVX1-NEXT:    vpand %xmm4, %xmm6, %xmm4
-; AVX1-NEXT:    vpsrldq {{.*#+}} xmm6 = xmm4[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX1-NEXT:    vpsrld %xmm6, %xmm0, %xmm6
-; AVX1-NEXT:    vpsrlq $32, %xmm4, %xmm7
-; AVX1-NEXT:    vpsrld %xmm7, %xmm0, %xmm7
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm6 = xmm7[0,1,2,3],xmm6[4,5,6,7]
-; AVX1-NEXT:    vpunpckhdq {{.*#+}} xmm3 = xmm4[2],xmm3[2],xmm4[3],xmm3[3]
-; AVX1-NEXT:    vpsrld %xmm3, %xmm0, %xmm3
-; AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm4 = xmm4[0],zero,xmm4[1],zero
-; AVX1-NEXT:    vpsrld %xmm4, %xmm0, %xmm4
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm3 = xmm4[0,1,2,3],xmm3[4,5,6,7]
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm3 = xmm3[0,1],xmm6[2,3],xmm3[4,5],xmm6[6,7]
-; AVX1-NEXT:    vinsertf128 $1, %xmm5, %ymm3, %ymm3
-; AVX1-NEXT:    vandps {{.*}}(%rip), %ymm1, %ymm1
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm4
-; AVX1-NEXT:    vpslld $23, %xmm4, %xmm4
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm5 = [1065353216,1065353216,1065353216,1065353216]
-; AVX1-NEXT:    vpaddd %xmm5, %xmm4, %xmm4
-; AVX1-NEXT:    vcvttps2dq %xmm4, %xmm4
-; AVX1-NEXT:    vpmulld %xmm4, %xmm2, %xmm2
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [31,31,31,31]
+; AVX1-NEXT:    vpand %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vpslld $23, %xmm2, %xmm2
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = [1065353216,1065353216,1065353216,1065353216]
+; AVX1-NEXT:    vpaddd %xmm4, %xmm2, %xmm2
+; AVX1-NEXT:    vcvttps2dq %xmm2, %xmm2
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm2[1,1,3,3]
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm6
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm7 = xmm6[1,1,3,3]
+; AVX1-NEXT:    vpmuludq %xmm5, %xmm7, %xmm5
+; AVX1-NEXT:    vpmuludq %xmm2, %xmm6, %xmm2
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm6 = xmm2[1,1,3,3]
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm6 = xmm6[0,1],xmm5[2,3],xmm6[4,5],xmm5[6,7]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm5 = xmm5[0,0,2,2]
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0,1],xmm5[2,3],xmm2[4,5],xmm5[6,7]
+; AVX1-NEXT:    vpor %xmm6, %xmm2, %xmm2
+; AVX1-NEXT:    vpand %xmm3, %xmm1, %xmm1
 ; AVX1-NEXT:    vpslld $23, %xmm1, %xmm1
-; AVX1-NEXT:    vpaddd %xmm5, %xmm1, %xmm1
+; AVX1-NEXT:    vpaddd %xmm4, %xmm1, %xmm1
 ; AVX1-NEXT:    vcvttps2dq %xmm1, %xmm1
-; AVX1-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[1,1,3,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm0[1,1,3,3]
+; AVX1-NEXT:    vpmuludq %xmm3, %xmm4, %xmm3
+; AVX1-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm3[2,3],xmm1[4,5],xmm3[6,7]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[0,0,2,2]
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm3[2,3],xmm0[4,5],xmm3[6,7]
+; AVX1-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
-; AVX1-NEXT:    vorps %ymm3, %ymm0, %ymm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: var_funnnel_v8i32:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpbroadcastd {{.*#+}} ymm2 = [31,31,31,31,31,31,31,31]
-; AVX2-NEXT:    vpand %ymm2, %ymm1, %ymm3
-; AVX2-NEXT:    vpsllvd %ymm3, %ymm0, %ymm3
-; AVX2-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX2-NEXT:    vpsubd %ymm1, %ymm4, %ymm1
 ; AVX2-NEXT:    vpand %ymm2, %ymm1, %ymm1
+; AVX2-NEXT:    vpsllvd %ymm1, %ymm0, %ymm2
+; AVX2-NEXT:    vpbroadcastd {{.*#+}} ymm3 = [32,32,32,32,32,32,32,32]
+; AVX2-NEXT:    vpsubd %ymm1, %ymm3, %ymm1
 ; AVX2-NEXT:    vpsrlvd %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpor %ymm0, %ymm3, %ymm0
+; AVX2-NEXT:    vpor %ymm0, %ymm2, %ymm0
 ; AVX2-NEXT:    retq
 ;
-; AVX512-LABEL: var_funnnel_v8i32:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpbroadcastd {{.*#+}} ymm2 = [31,31,31,31,31,31,31,31]
-; AVX512-NEXT:    vpand %ymm2, %ymm1, %ymm3
-; AVX512-NEXT:    vpsllvd %ymm3, %ymm0, %ymm3
-; AVX512-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX512-NEXT:    vpsubd %ymm1, %ymm4, %ymm1
-; AVX512-NEXT:    vpand %ymm2, %ymm1, %ymm1
-; AVX512-NEXT:    vpsrlvd %ymm1, %ymm0, %ymm0
-; AVX512-NEXT:    vpor %ymm0, %ymm3, %ymm0
-; AVX512-NEXT:    retq
+; AVX512F-LABEL: var_funnnel_v8i32:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512F-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512F-NEXT:    vprolvd %zmm1, %zmm0, %zmm0
+; AVX512F-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: var_funnnel_v8i32:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vprolvd %ymm1, %ymm0, %ymm0
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: var_funnnel_v8i32:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512BW-NEXT:    vprolvd %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
+; AVX512BW-NEXT:    retq
+;
+; AVX512VLBW-LABEL: var_funnnel_v8i32:
+; AVX512VLBW:       # %bb.0:
+; AVX512VLBW-NEXT:    vprolvd %ymm1, %ymm0, %ymm0
+; AVX512VLBW-NEXT:    retq
 ;
 ; XOPAVX1-LABEL: var_funnnel_v8i32:
 ; XOPAVX1:       # %bb.0:
-; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
-; XOPAVX1-NEXT:    vandps {{.*}}(%rip), %ymm1, %ymm3
-; XOPAVX1-NEXT:    vextractf128 $1, %ymm3, %xmm4
-; XOPAVX1-NEXT:    vpshld %xmm4, %xmm2, %xmm4
-; XOPAVX1-NEXT:    vpshld %xmm3, %xmm0, %xmm3
-; XOPAVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm3, %ymm3
-; XOPAVX1-NEXT:    vextractf128 $1, %ymm1, %xmm4
-; XOPAVX1-NEXT:    vpxor %xmm5, %xmm5, %xmm5
-; XOPAVX1-NEXT:    vpsubd %xmm4, %xmm5, %xmm4
-; XOPAVX1-NEXT:    vmovdqa {{.*#+}} xmm6 = [31,31,31,31]
-; XOPAVX1-NEXT:    vpand %xmm6, %xmm4, %xmm4
-; XOPAVX1-NEXT:    vpsubd %xmm4, %xmm5, %xmm4
-; XOPAVX1-NEXT:    vpshld %xmm4, %xmm2, %xmm2
-; XOPAVX1-NEXT:    vpsubd %xmm1, %xmm5, %xmm1
-; XOPAVX1-NEXT:    vpand %xmm6, %xmm1, %xmm1
-; XOPAVX1-NEXT:    vpsubd %xmm1, %xmm5, %xmm1
-; XOPAVX1-NEXT:    vpshld %xmm1, %xmm0, %xmm0
+; XOPAVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
+; XOPAVX1-NEXT:    vprotd %xmm2, %xmm3, %xmm2
+; XOPAVX1-NEXT:    vprotd %xmm1, %xmm0, %xmm0
 ; XOPAVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
-; XOPAVX1-NEXT:    vorps %ymm0, %ymm3, %ymm0
 ; XOPAVX1-NEXT:    retq
 ;
 ; XOPAVX2-LABEL: var_funnnel_v8i32:
 ; XOPAVX2:       # %bb.0:
-; XOPAVX2-NEXT:    vpbroadcastd {{.*#+}} ymm2 = [31,31,31,31,31,31,31,31]
-; XOPAVX2-NEXT:    vpand %ymm2, %ymm1, %ymm3
-; XOPAVX2-NEXT:    vpsllvd %ymm3, %ymm0, %ymm3
-; XOPAVX2-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; XOPAVX2-NEXT:    vpsubd %ymm1, %ymm4, %ymm1
-; XOPAVX2-NEXT:    vpand %ymm2, %ymm1, %ymm1
-; XOPAVX2-NEXT:    vpsrlvd %ymm1, %ymm0, %ymm0
-; XOPAVX2-NEXT:    vpor %ymm0, %ymm3, %ymm0
+; XOPAVX2-NEXT:    vextracti128 $1, %ymm1, %xmm2
+; XOPAVX2-NEXT:    vextracti128 $1, %ymm0, %xmm3
+; XOPAVX2-NEXT:    vprotd %xmm2, %xmm3, %xmm2
+; XOPAVX2-NEXT:    vprotd %xmm1, %xmm0, %xmm0
+; XOPAVX2-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm0
 ; XOPAVX2-NEXT:    retq
   %res = call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %x, <8 x i32> %x, <8 x i32> %amt)
   ret <8 x i32> %res
@@ -227,196 +207,130 @@ define <8 x i32> @var_funnnel_v8i32(<8 x i32> %x, <8 x i32> %amt) nounwind {
 define <16 x i16> @var_funnnel_v16i16(<16 x i16> %x, <16 x i16> %amt) nounwind {
 ; AVX1-LABEL: var_funnnel_v16i16:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm3
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpsubw %xmm3, %xmm2, %xmm3
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = [15,15,15,15,15,15,15,15]
-; AVX1-NEXT:    vpand %xmm4, %xmm3, %xmm3
-; AVX1-NEXT:    vpsllw $12, %xmm3, %xmm5
-; AVX1-NEXT:    vpsllw $4, %xmm3, %xmm3
-; AVX1-NEXT:    vpor %xmm5, %xmm3, %xmm5
-; AVX1-NEXT:    vpaddw %xmm5, %xmm5, %xmm6
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
-; AVX1-NEXT:    vpsrlw $8, %xmm3, %xmm7
-; AVX1-NEXT:    vpblendvb %xmm5, %xmm7, %xmm3, %xmm5
-; AVX1-NEXT:    vpsrlw $4, %xmm5, %xmm7
-; AVX1-NEXT:    vpblendvb %xmm6, %xmm7, %xmm5, %xmm5
-; AVX1-NEXT:    vpsrlw $2, %xmm5, %xmm7
-; AVX1-NEXT:    vpaddw %xmm6, %xmm6, %xmm6
-; AVX1-NEXT:    vpblendvb %xmm6, %xmm7, %xmm5, %xmm5
-; AVX1-NEXT:    vpsrlw $1, %xmm5, %xmm7
-; AVX1-NEXT:    vpaddw %xmm6, %xmm6, %xmm6
-; AVX1-NEXT:    vpblendvb %xmm6, %xmm7, %xmm5, %xmm5
-; AVX1-NEXT:    vpsubw %xmm1, %xmm2, %xmm6
-; AVX1-NEXT:    vpand %xmm4, %xmm6, %xmm4
-; AVX1-NEXT:    vpsllw $12, %xmm4, %xmm6
-; AVX1-NEXT:    vpsllw $4, %xmm4, %xmm4
-; AVX1-NEXT:    vpor %xmm6, %xmm4, %xmm4
-; AVX1-NEXT:    vpaddw %xmm4, %xmm4, %xmm6
-; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm7
-; AVX1-NEXT:    vpblendvb %xmm4, %xmm7, %xmm0, %xmm4
-; AVX1-NEXT:    vpsrlw $4, %xmm4, %xmm7
-; AVX1-NEXT:    vpblendvb %xmm6, %xmm7, %xmm4, %xmm4
-; AVX1-NEXT:    vpsrlw $2, %xmm4, %xmm7
-; AVX1-NEXT:    vpaddw %xmm6, %xmm6, %xmm6
-; AVX1-NEXT:    vpblendvb %xmm6, %xmm7, %xmm4, %xmm4
-; AVX1-NEXT:    vpsrlw $1, %xmm4, %xmm7
-; AVX1-NEXT:    vpaddw %xmm6, %xmm6, %xmm6
-; AVX1-NEXT:    vpblendvb %xmm6, %xmm7, %xmm4, %xmm4
-; AVX1-NEXT:    vinsertf128 $1, %xmm5, %ymm4, %ymm4
-; AVX1-NEXT:    vandps {{.*}}(%rip), %ymm1, %ymm1
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm5
-; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm6 = xmm5[4],xmm2[4],xmm5[5],xmm2[5],xmm5[6],xmm2[6],xmm5[7],xmm2[7]
-; AVX1-NEXT:    vpslld $23, %xmm6, %xmm6
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm7 = [1065353216,1065353216,1065353216,1065353216]
-; AVX1-NEXT:    vpaddd %xmm7, %xmm6, %xmm6
-; AVX1-NEXT:    vcvttps2dq %xmm6, %xmm6
-; AVX1-NEXT:    vpmovzxwd {{.*#+}} xmm5 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [15,15,15,15,15,15,15,15]
+; AVX1-NEXT:    vpand %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vpxor %xmm4, %xmm4, %xmm4
+; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm5 = xmm2[4],xmm4[4],xmm2[5],xmm4[5],xmm2[6],xmm4[6],xmm2[7],xmm4[7]
 ; AVX1-NEXT:    vpslld $23, %xmm5, %xmm5
-; AVX1-NEXT:    vpaddd %xmm7, %xmm5, %xmm5
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm6 = [1065353216,1065353216,1065353216,1065353216]
+; AVX1-NEXT:    vpaddd %xmm6, %xmm5, %xmm5
 ; AVX1-NEXT:    vcvttps2dq %xmm5, %xmm5
-; AVX1-NEXT:    vpackusdw %xmm6, %xmm5, %xmm5
-; AVX1-NEXT:    vpmullw %xmm5, %xmm3, %xmm3
-; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
+; AVX1-NEXT:    vpmovzxwd {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero
 ; AVX1-NEXT:    vpslld $23, %xmm2, %xmm2
-; AVX1-NEXT:    vpaddd %xmm7, %xmm2, %xmm2
+; AVX1-NEXT:    vpaddd %xmm6, %xmm2, %xmm2
 ; AVX1-NEXT:    vcvttps2dq %xmm2, %xmm2
+; AVX1-NEXT:    vpackusdw %xmm5, %xmm2, %xmm2
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm5
+; AVX1-NEXT:    vpmulhuw %xmm2, %xmm5, %xmm7
+; AVX1-NEXT:    vpmullw %xmm2, %xmm5, %xmm2
+; AVX1-NEXT:    vpor %xmm7, %xmm2, %xmm2
+; AVX1-NEXT:    vpand %xmm3, %xmm1, %xmm1
+; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm1[4],xmm4[4],xmm1[5],xmm4[5],xmm1[6],xmm4[6],xmm1[7],xmm4[7]
+; AVX1-NEXT:    vpslld $23, %xmm3, %xmm3
+; AVX1-NEXT:    vpaddd %xmm6, %xmm3, %xmm3
+; AVX1-NEXT:    vcvttps2dq %xmm3, %xmm3
 ; AVX1-NEXT:    vpmovzxwd {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
 ; AVX1-NEXT:    vpslld $23, %xmm1, %xmm1
-; AVX1-NEXT:    vpaddd %xmm7, %xmm1, %xmm1
+; AVX1-NEXT:    vpaddd %xmm6, %xmm1, %xmm1
 ; AVX1-NEXT:    vcvttps2dq %xmm1, %xmm1
-; AVX1-NEXT:    vpackusdw %xmm2, %xmm1, %xmm1
+; AVX1-NEXT:    vpackusdw %xmm3, %xmm1, %xmm1
+; AVX1-NEXT:    vpmulhuw %xmm1, %xmm0, %xmm3
 ; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm0
-; AVX1-NEXT:    vorps %ymm4, %ymm0, %ymm0
+; AVX1-NEXT:    vpor %xmm3, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: var_funnnel_v16i16:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
 ; AVX2-NEXT:    vpunpckhwd {{.*#+}} ymm3 = ymm2[4],ymm0[4],ymm2[5],ymm0[5],ymm2[6],ymm0[6],ymm2[7],ymm0[7],ymm2[12],ymm0[12],ymm2[13],ymm0[13],ymm2[14],ymm0[14],ymm2[15],ymm0[15]
-; AVX2-NEXT:    vmovdqa {{.*#+}} ymm4 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; AVX2-NEXT:    vpand %ymm4, %ymm1, %ymm5
-; AVX2-NEXT:    vpunpckhwd {{.*#+}} ymm6 = ymm5[4],ymm2[4],ymm5[5],ymm2[5],ymm5[6],ymm2[6],ymm5[7],ymm2[7],ymm5[12],ymm2[12],ymm5[13],ymm2[13],ymm5[14],ymm2[14],ymm5[15],ymm2[15]
-; AVX2-NEXT:    vpsllvd %ymm6, %ymm3, %ymm6
-; AVX2-NEXT:    vpsrld $16, %ymm6, %ymm6
+; AVX2-NEXT:    vpand {{.*}}(%rip), %ymm1, %ymm1
+; AVX2-NEXT:    vpunpckhwd {{.*#+}} ymm4 = ymm1[4],ymm2[4],ymm1[5],ymm2[5],ymm1[6],ymm2[6],ymm1[7],ymm2[7],ymm1[12],ymm2[12],ymm1[13],ymm2[13],ymm1[14],ymm2[14],ymm1[15],ymm2[15]
+; AVX2-NEXT:    vpsllvd %ymm4, %ymm3, %ymm4
+; AVX2-NEXT:    vpsrld $16, %ymm4, %ymm4
 ; AVX2-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm2[0],ymm0[0],ymm2[1],ymm0[1],ymm2[2],ymm0[2],ymm2[3],ymm0[3],ymm2[8],ymm0[8],ymm2[9],ymm0[9],ymm2[10],ymm0[10],ymm2[11],ymm0[11]
-; AVX2-NEXT:    vpunpcklwd {{.*#+}} ymm5 = ymm5[0],ymm2[0],ymm5[1],ymm2[1],ymm5[2],ymm2[2],ymm5[3],ymm2[3],ymm5[8],ymm2[8],ymm5[9],ymm2[9],ymm5[10],ymm2[10],ymm5[11],ymm2[11]
+; AVX2-NEXT:    vpunpcklwd {{.*#+}} ymm5 = ymm1[0],ymm2[0],ymm1[1],ymm2[1],ymm1[2],ymm2[2],ymm1[3],ymm2[3],ymm1[8],ymm2[8],ymm1[9],ymm2[9],ymm1[10],ymm2[10],ymm1[11],ymm2[11]
 ; AVX2-NEXT:    vpsllvd %ymm5, %ymm0, %ymm5
 ; AVX2-NEXT:    vpsrld $16, %ymm5, %ymm5
-; AVX2-NEXT:    vpackusdw %ymm6, %ymm5, %ymm5
-; AVX2-NEXT:    vpsubw %ymm1, %ymm2, %ymm1
-; AVX2-NEXT:    vpand %ymm4, %ymm1, %ymm1
-; AVX2-NEXT:    vpunpckhwd {{.*#+}} ymm4 = ymm1[4],ymm2[4],ymm1[5],ymm2[5],ymm1[6],ymm2[6],ymm1[7],ymm2[7],ymm1[12],ymm2[12],ymm1[13],ymm2[13],ymm1[14],ymm2[14],ymm1[15],ymm2[15]
-; AVX2-NEXT:    vpsrlvd %ymm4, %ymm3, %ymm3
+; AVX2-NEXT:    vpackusdw %ymm4, %ymm5, %ymm4
+; AVX2-NEXT:    vmovdqa {{.*#+}} ymm5 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
+; AVX2-NEXT:    vpsubw %ymm1, %ymm5, %ymm1
+; AVX2-NEXT:    vpunpckhwd {{.*#+}} ymm5 = ymm1[4],ymm2[4],ymm1[5],ymm2[5],ymm1[6],ymm2[6],ymm1[7],ymm2[7],ymm1[12],ymm2[12],ymm1[13],ymm2[13],ymm1[14],ymm2[14],ymm1[15],ymm2[15]
+; AVX2-NEXT:    vpsrlvd %ymm5, %ymm3, %ymm3
 ; AVX2-NEXT:    vpsrld $16, %ymm3, %ymm3
 ; AVX2-NEXT:    vpunpcklwd {{.*#+}} ymm1 = ymm1[0],ymm2[0],ymm1[1],ymm2[1],ymm1[2],ymm2[2],ymm1[3],ymm2[3],ymm1[8],ymm2[8],ymm1[9],ymm2[9],ymm1[10],ymm2[10],ymm1[11],ymm2[11]
 ; AVX2-NEXT:    vpsrlvd %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vpsrld $16, %ymm0, %ymm0
 ; AVX2-NEXT:    vpackusdw %ymm3, %ymm0, %ymm0
-; AVX2-NEXT:    vpor %ymm0, %ymm5, %ymm0
+; AVX2-NEXT:    vpor %ymm0, %ymm4, %ymm0
 ; AVX2-NEXT:    retq
 ;
 ; AVX512F-LABEL: var_funnnel_v16i16:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; AVX512F-NEXT:    vpand %ymm2, %ymm1, %ymm3
-; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm3 = ymm3[0],zero,ymm3[1],zero,ymm3[2],zero,ymm3[3],zero,ymm3[4],zero,ymm3[5],zero,ymm3[6],zero,ymm3[7],zero,ymm3[8],zero,ymm3[9],zero,ymm3[10],zero,ymm3[11],zero,ymm3[12],zero,ymm3[13],zero,ymm3[14],zero,ymm3[15],zero
 ; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
-; AVX512F-NEXT:    vpsllvd %zmm3, %zmm0, %zmm3
-; AVX512F-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX512F-NEXT:    vpsubw %ymm1, %ymm4, %ymm1
-; AVX512F-NEXT:    vpand %ymm2, %ymm1, %ymm1
+; AVX512F-NEXT:    vpand {{.*}}(%rip), %ymm1, %ymm1
+; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm2 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero
+; AVX512F-NEXT:    vpsllvd %zmm2, %zmm0, %zmm2
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm3 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
+; AVX512F-NEXT:    vpsubw %ymm1, %ymm3, %ymm1
 ; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero
 ; AVX512F-NEXT:    vpsrlvd %zmm1, %zmm0, %zmm0
-; AVX512F-NEXT:    vpord %zmm0, %zmm3, %zmm0
+; AVX512F-NEXT:    vpord %zmm0, %zmm2, %zmm0
 ; AVX512F-NEXT:    vpmovdw %zmm0, %ymm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: var_funnnel_v16i16:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; AVX512VL-NEXT:    vpand %ymm2, %ymm1, %ymm3
-; AVX512VL-NEXT:    vpmovzxwd {{.*#+}} zmm3 = ymm3[0],zero,ymm3[1],zero,ymm3[2],zero,ymm3[3],zero,ymm3[4],zero,ymm3[5],zero,ymm3[6],zero,ymm3[7],zero,ymm3[8],zero,ymm3[9],zero,ymm3[10],zero,ymm3[11],zero,ymm3[12],zero,ymm3[13],zero,ymm3[14],zero,ymm3[15],zero
 ; AVX512VL-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
-; AVX512VL-NEXT:    vpsllvd %zmm3, %zmm0, %zmm3
-; AVX512VL-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX512VL-NEXT:    vpsubw %ymm1, %ymm4, %ymm1
-; AVX512VL-NEXT:    vpand %ymm2, %ymm1, %ymm1
+; AVX512VL-NEXT:    vpand {{.*}}(%rip), %ymm1, %ymm1
+; AVX512VL-NEXT:    vpmovzxwd {{.*#+}} zmm2 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero
+; AVX512VL-NEXT:    vpsllvd %zmm2, %zmm0, %zmm2
+; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm3 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
+; AVX512VL-NEXT:    vpsubw %ymm1, %ymm3, %ymm1
 ; AVX512VL-NEXT:    vpmovzxwd {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero
 ; AVX512VL-NEXT:    vpsrlvd %zmm1, %zmm0, %zmm0
-; AVX512VL-NEXT:    vpord %zmm0, %zmm3, %zmm0
+; AVX512VL-NEXT:    vpord %zmm0, %zmm2, %zmm0
 ; AVX512VL-NEXT:    vpmovdw %zmm0, %ymm0
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: var_funnnel_v16i16:
 ; AVX512BW:       # %bb.0:
 ; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; AVX512BW-NEXT:    vmovdqa {{.*#+}} ymm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; AVX512BW-NEXT:    vpand %ymm2, %ymm1, %ymm3
-; AVX512BW-NEXT:    vpsllvw %zmm3, %zmm0, %zmm3
-; AVX512BW-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX512BW-NEXT:    vpsubw %ymm1, %ymm4, %ymm1
-; AVX512BW-NEXT:    vpand %ymm2, %ymm1, %ymm1
+; AVX512BW-NEXT:    vpand {{.*}}(%rip), %ymm1, %ymm1
+; AVX512BW-NEXT:    vpsllvw %zmm1, %zmm0, %zmm2
+; AVX512BW-NEXT:    vmovdqa {{.*#+}} ymm3 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
+; AVX512BW-NEXT:    vpsubw %ymm1, %ymm3, %ymm1
 ; AVX512BW-NEXT:    vpsrlvw %zmm1, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpor %ymm0, %ymm3, %ymm0
+; AVX512BW-NEXT:    vpor %ymm0, %ymm2, %ymm0
 ; AVX512BW-NEXT:    retq
 ;
 ; AVX512VLBW-LABEL: var_funnnel_v16i16:
 ; AVX512VLBW:       # %bb.0:
-; AVX512VLBW-NEXT:    vmovdqa {{.*#+}} ymm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; AVX512VLBW-NEXT:    vpand %ymm2, %ymm1, %ymm3
-; AVX512VLBW-NEXT:    vpsllvw %ymm3, %ymm0, %ymm3
-; AVX512VLBW-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX512VLBW-NEXT:    vpsubw %ymm1, %ymm4, %ymm1
-; AVX512VLBW-NEXT:    vpand %ymm2, %ymm1, %ymm1
+; AVX512VLBW-NEXT:    vpand {{.*}}(%rip), %ymm1, %ymm1
+; AVX512VLBW-NEXT:    vpsllvw %ymm1, %ymm0, %ymm2
+; AVX512VLBW-NEXT:    vmovdqa {{.*#+}} ymm3 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
+; AVX512VLBW-NEXT:    vpsubw %ymm1, %ymm3, %ymm1
 ; AVX512VLBW-NEXT:    vpsrlvw %ymm1, %ymm0, %ymm0
-; AVX512VLBW-NEXT:    vpor %ymm0, %ymm3, %ymm0
+; AVX512VLBW-NEXT:    vpor %ymm0, %ymm2, %ymm0
 ; AVX512VLBW-NEXT:    retq
 ;
 ; XOPAVX1-LABEL: var_funnnel_v16i16:
 ; XOPAVX1:       # %bb.0:
-; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
-; XOPAVX1-NEXT:    vandps {{.*}}(%rip), %ymm1, %ymm3
-; XOPAVX1-NEXT:    vextractf128 $1, %ymm3, %xmm4
-; XOPAVX1-NEXT:    vpshlw %xmm4, %xmm2, %xmm4
-; XOPAVX1-NEXT:    vpshlw %xmm3, %xmm0, %xmm3
-; XOPAVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm3, %ymm3
-; XOPAVX1-NEXT:    vextractf128 $1, %ymm1, %xmm4
-; XOPAVX1-NEXT:    vpxor %xmm5, %xmm5, %xmm5
-; XOPAVX1-NEXT:    vpsubw %xmm4, %xmm5, %xmm4
-; XOPAVX1-NEXT:    vmovdqa {{.*#+}} xmm6 = [15,15,15,15,15,15,15,15]
-; XOPAVX1-NEXT:    vpand %xmm6, %xmm4, %xmm4
-; XOPAVX1-NEXT:    vpsubw %xmm4, %xmm5, %xmm4
-; XOPAVX1-NEXT:    vpshlw %xmm4, %xmm2, %xmm2
-; XOPAVX1-NEXT:    vpsubw %xmm1, %xmm5, %xmm1
-; XOPAVX1-NEXT:    vpand %xmm6, %xmm1, %xmm1
-; XOPAVX1-NEXT:    vpsubw %xmm1, %xmm5, %xmm1
-; XOPAVX1-NEXT:    vpshlw %xmm1, %xmm0, %xmm0
+; XOPAVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
+; XOPAVX1-NEXT:    vprotw %xmm2, %xmm3, %xmm2
+; XOPAVX1-NEXT:    vprotw %xmm1, %xmm0, %xmm0
 ; XOPAVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
-; XOPAVX1-NEXT:    vorps %ymm0, %ymm3, %ymm0
 ; XOPAVX1-NEXT:    retq
 ;
 ; XOPAVX2-LABEL: var_funnnel_v16i16:
 ; XOPAVX2:       # %bb.0:
-; XOPAVX2-NEXT:    vmovdqa {{.*#+}} ymm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; XOPAVX2-NEXT:    vpand %ymm2, %ymm1, %ymm3
-; XOPAVX2-NEXT:    vextracti128 $1, %ymm3, %xmm4
-; XOPAVX2-NEXT:    vextracti128 $1, %ymm0, %xmm5
-; XOPAVX2-NEXT:    vpshlw %xmm4, %xmm5, %xmm4
-; XOPAVX2-NEXT:    vpshlw %xmm3, %xmm0, %xmm3
-; XOPAVX2-NEXT:    vinserti128 $1, %xmm4, %ymm3, %ymm3
-; XOPAVX2-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; XOPAVX2-NEXT:    vpsubw %ymm1, %ymm4, %ymm1
-; XOPAVX2-NEXT:    vpand %ymm2, %ymm1, %ymm1
 ; XOPAVX2-NEXT:    vextracti128 $1, %ymm1, %xmm2
-; XOPAVX2-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; XOPAVX2-NEXT:    vpsubw %xmm2, %xmm4, %xmm2
-; XOPAVX2-NEXT:    vpshlw %xmm2, %xmm5, %xmm2
-; XOPAVX2-NEXT:    vpsubw %xmm1, %xmm4, %xmm1
-; XOPAVX2-NEXT:    vpshlw %xmm1, %xmm0, %xmm0
+; XOPAVX2-NEXT:    vextracti128 $1, %ymm0, %xmm3
+; XOPAVX2-NEXT:    vprotw %xmm2, %xmm3, %xmm2
+; XOPAVX2-NEXT:    vprotw %xmm1, %xmm0, %xmm0
 ; XOPAVX2-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm0
-; XOPAVX2-NEXT:    vpor %ymm0, %ymm3, %ymm0
 ; XOPAVX2-NEXT:    retq
   %res = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %x, <16 x i16> %x, <16 x i16> %amt)
   ret <16 x i16> %res
@@ -427,167 +341,122 @@ define <32 x i8> @var_funnnel_v32i8(<32 x i8> %x, <32 x i8> %amt) nounwind {
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
 ; AVX1-NEXT:    vpsrlw $4, %xmm2, %xmm3
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm8 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; AVX1-NEXT:    vpand %xmm8, %xmm3, %xmm3
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
+; AVX1-NEXT:    vpandn %xmm3, %xmm4, %xmm3
+; AVX1-NEXT:    vpsllw $4, %xmm2, %xmm5
+; AVX1-NEXT:    vpand %xmm4, %xmm5, %xmm5
+; AVX1-NEXT:    vpor %xmm3, %xmm5, %xmm3
 ; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm5
-; AVX1-NEXT:    vpxor %xmm9, %xmm9, %xmm9
-; AVX1-NEXT:    vpsubb %xmm5, %xmm9, %xmm5
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm10 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
-; AVX1-NEXT:    vpand %xmm10, %xmm5, %xmm5
-; AVX1-NEXT:    vpsllw $5, %xmm5, %xmm5
-; AVX1-NEXT:    vpblendvb %xmm5, %xmm3, %xmm2, %xmm3
-; AVX1-NEXT:    vpsrlw $2, %xmm3, %xmm4
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm11 = [63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63]
-; AVX1-NEXT:    vpand %xmm11, %xmm4, %xmm4
-; AVX1-NEXT:    vpaddb %xmm5, %xmm5, %xmm5
-; AVX1-NEXT:    vpblendvb %xmm5, %xmm4, %xmm3, %xmm3
-; AVX1-NEXT:    vpsrlw $1, %xmm3, %xmm4
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm7 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
-; AVX1-NEXT:    vpand %xmm7, %xmm4, %xmm4
-; AVX1-NEXT:    vpaddb %xmm5, %xmm5, %xmm5
-; AVX1-NEXT:    vpblendvb %xmm5, %xmm4, %xmm3, %xmm3
-; AVX1-NEXT:    vpsrlw $4, %xmm0, %xmm4
-; AVX1-NEXT:    vpand %xmm8, %xmm4, %xmm4
-; AVX1-NEXT:    vpsubb %xmm1, %xmm9, %xmm5
-; AVX1-NEXT:    vpand %xmm10, %xmm5, %xmm5
 ; AVX1-NEXT:    vpsllw $5, %xmm5, %xmm5
-; AVX1-NEXT:    vpblendvb %xmm5, %xmm4, %xmm0, %xmm4
-; AVX1-NEXT:    vpsrlw $2, %xmm4, %xmm6
-; AVX1-NEXT:    vpand %xmm11, %xmm6, %xmm6
+; AVX1-NEXT:    vpblendvb %xmm5, %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vpsrlw $6, %xmm2, %xmm3
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm6 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252]
+; AVX1-NEXT:    vpandn %xmm3, %xmm6, %xmm3
+; AVX1-NEXT:    vpsllw $2, %xmm2, %xmm7
+; AVX1-NEXT:    vpand %xmm6, %xmm7, %xmm7
+; AVX1-NEXT:    vpor %xmm3, %xmm7, %xmm3
 ; AVX1-NEXT:    vpaddb %xmm5, %xmm5, %xmm5
-; AVX1-NEXT:    vpblendvb %xmm5, %xmm6, %xmm4, %xmm4
-; AVX1-NEXT:    vpsrlw $1, %xmm4, %xmm6
-; AVX1-NEXT:    vpand %xmm7, %xmm6, %xmm6
+; AVX1-NEXT:    vpblendvb %xmm5, %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vpsrlw $7, %xmm2, %xmm3
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm8 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
+; AVX1-NEXT:    vpand %xmm8, %xmm3, %xmm3
+; AVX1-NEXT:    vpaddb %xmm2, %xmm2, %xmm7
+; AVX1-NEXT:    vpor %xmm3, %xmm7, %xmm3
 ; AVX1-NEXT:    vpaddb %xmm5, %xmm5, %xmm5
-; AVX1-NEXT:    vpblendvb %xmm5, %xmm6, %xmm4, %xmm4
-; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm4, %ymm3
-; AVX1-NEXT:    vpsllw $4, %xmm2, %xmm4
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm5 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
-; AVX1-NEXT:    vpand %xmm5, %xmm4, %xmm4
-; AVX1-NEXT:    vandps {{.*}}(%rip), %ymm1, %ymm1
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm6
-; AVX1-NEXT:    vpsllw $5, %xmm6, %xmm6
-; AVX1-NEXT:    vpblendvb %xmm6, %xmm4, %xmm2, %xmm2
-; AVX1-NEXT:    vpsllw $2, %xmm2, %xmm4
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm7 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252]
-; AVX1-NEXT:    vpand %xmm7, %xmm4, %xmm4
-; AVX1-NEXT:    vpaddb %xmm6, %xmm6, %xmm6
-; AVX1-NEXT:    vpblendvb %xmm6, %xmm4, %xmm2, %xmm2
-; AVX1-NEXT:    vpaddb %xmm2, %xmm2, %xmm4
-; AVX1-NEXT:    vpaddb %xmm6, %xmm6, %xmm6
-; AVX1-NEXT:    vpblendvb %xmm6, %xmm4, %xmm2, %xmm2
-; AVX1-NEXT:    vpsllw $4, %xmm0, %xmm4
-; AVX1-NEXT:    vpand %xmm5, %xmm4, %xmm4
+; AVX1-NEXT:    vpblendvb %xmm5, %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vpsrlw $4, %xmm0, %xmm3
+; AVX1-NEXT:    vpandn %xmm3, %xmm4, %xmm3
+; AVX1-NEXT:    vpsllw $4, %xmm0, %xmm5
+; AVX1-NEXT:    vpand %xmm4, %xmm5, %xmm4
+; AVX1-NEXT:    vpor %xmm3, %xmm4, %xmm3
 ; AVX1-NEXT:    vpsllw $5, %xmm1, %xmm1
-; AVX1-NEXT:    vpblendvb %xmm1, %xmm4, %xmm0, %xmm0
+; AVX1-NEXT:    vpblendvb %xmm1, %xmm3, %xmm0, %xmm0
+; AVX1-NEXT:    vpsrlw $6, %xmm0, %xmm3
+; AVX1-NEXT:    vpandn %xmm3, %xmm6, %xmm3
 ; AVX1-NEXT:    vpsllw $2, %xmm0, %xmm4
-; AVX1-NEXT:    vpand %xmm7, %xmm4, %xmm4
+; AVX1-NEXT:    vpand %xmm6, %xmm4, %xmm4
+; AVX1-NEXT:    vpor %xmm3, %xmm4, %xmm3
 ; AVX1-NEXT:    vpaddb %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    vpblendvb %xmm1, %xmm4, %xmm0, %xmm0
+; AVX1-NEXT:    vpblendvb %xmm1, %xmm3, %xmm0, %xmm0
+; AVX1-NEXT:    vpsrlw $7, %xmm0, %xmm3
+; AVX1-NEXT:    vpand %xmm8, %xmm3, %xmm3
 ; AVX1-NEXT:    vpaddb %xmm0, %xmm0, %xmm4
+; AVX1-NEXT:    vpor %xmm3, %xmm4, %xmm3
 ; AVX1-NEXT:    vpaddb %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    vpblendvb %xmm1, %xmm4, %xmm0, %xmm0
+; AVX1-NEXT:    vpblendvb %xmm1, %xmm3, %xmm0, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
-; AVX1-NEXT:    vorps %ymm3, %ymm0, %ymm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: var_funnnel_v32i8:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vmovdqa {{.*#+}} ymm2 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
-; AVX2-NEXT:    vpand %ymm2, %ymm1, %ymm3
-; AVX2-NEXT:    vpsllw $5, %ymm3, %ymm3
-; AVX2-NEXT:    vpaddb %ymm3, %ymm3, %ymm4
-; AVX2-NEXT:    vpsllw $4, %ymm0, %ymm5
-; AVX2-NEXT:    vpand {{.*}}(%rip), %ymm5, %ymm5
-; AVX2-NEXT:    vpblendvb %ymm3, %ymm5, %ymm0, %ymm3
-; AVX2-NEXT:    vpsllw $2, %ymm3, %ymm5
-; AVX2-NEXT:    vpand {{.*}}(%rip), %ymm5, %ymm5
-; AVX2-NEXT:    vpblendvb %ymm4, %ymm5, %ymm3, %ymm3
-; AVX2-NEXT:    vpaddb %ymm3, %ymm3, %ymm5
-; AVX2-NEXT:    vpaddb %ymm4, %ymm4, %ymm4
-; AVX2-NEXT:    vpblendvb %ymm4, %ymm5, %ymm3, %ymm3
-; AVX2-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX2-NEXT:    vpsubb %ymm1, %ymm4, %ymm1
-; AVX2-NEXT:    vpand %ymm2, %ymm1, %ymm1
+; AVX2-NEXT:    vpsrlw $4, %ymm0, %ymm2
+; AVX2-NEXT:    vpand {{.*}}(%rip), %ymm2, %ymm2
+; AVX2-NEXT:    vpsllw $4, %ymm0, %ymm3
+; AVX2-NEXT:    vpand {{.*}}(%rip), %ymm3, %ymm3
+; AVX2-NEXT:    vpor %ymm2, %ymm3, %ymm2
 ; AVX2-NEXT:    vpsllw $5, %ymm1, %ymm1
-; AVX2-NEXT:    vpaddb %ymm1, %ymm1, %ymm2
-; AVX2-NEXT:    vpsrlw $4, %ymm0, %ymm4
-; AVX2-NEXT:    vpand {{.*}}(%rip), %ymm4, %ymm4
-; AVX2-NEXT:    vpblendvb %ymm1, %ymm4, %ymm0, %ymm0
-; AVX2-NEXT:    vpsrlw $2, %ymm0, %ymm1
-; AVX2-NEXT:    vpand {{.*}}(%rip), %ymm1, %ymm1
-; AVX2-NEXT:    vpblendvb %ymm2, %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpsrlw $1, %ymm0, %ymm1
-; AVX2-NEXT:    vpand {{.*}}(%rip), %ymm1, %ymm1
-; AVX2-NEXT:    vpaddb %ymm2, %ymm2, %ymm2
-; AVX2-NEXT:    vpblendvb %ymm2, %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpor %ymm0, %ymm3, %ymm0
+; AVX2-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vpsrlw $6, %ymm0, %ymm2
+; AVX2-NEXT:    vpand {{.*}}(%rip), %ymm2, %ymm2
+; AVX2-NEXT:    vpsllw $2, %ymm0, %ymm3
+; AVX2-NEXT:    vpand {{.*}}(%rip), %ymm3, %ymm3
+; AVX2-NEXT:    vpor %ymm2, %ymm3, %ymm2
+; AVX2-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
+; AVX2-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vpaddb %ymm0, %ymm0, %ymm2
+; AVX2-NEXT:    vpsrlw $7, %ymm0, %ymm3
+; AVX2-NEXT:    vpand {{.*}}(%rip), %ymm3, %ymm3
+; AVX2-NEXT:    vpor %ymm3, %ymm2, %ymm2
+; AVX2-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
+; AVX2-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
 ; AVX2-NEXT:    retq
 ;
 ; AVX512F-LABEL: var_funnnel_v32i8:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm2 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
-; AVX512F-NEXT:    vpand %ymm2, %ymm1, %ymm3
-; AVX512F-NEXT:    vpsllw $5, %ymm3, %ymm3
-; AVX512F-NEXT:    vpaddb %ymm3, %ymm3, %ymm4
-; AVX512F-NEXT:    vpsllw $4, %ymm0, %ymm5
-; AVX512F-NEXT:    vpand {{.*}}(%rip), %ymm5, %ymm5
-; AVX512F-NEXT:    vpblendvb %ymm3, %ymm5, %ymm0, %ymm3
-; AVX512F-NEXT:    vpsllw $2, %ymm3, %ymm5
-; AVX512F-NEXT:    vpand {{.*}}(%rip), %ymm5, %ymm5
-; AVX512F-NEXT:    vpblendvb %ymm4, %ymm5, %ymm3, %ymm3
-; AVX512F-NEXT:    vpaddb %ymm3, %ymm3, %ymm5
-; AVX512F-NEXT:    vpaddb %ymm4, %ymm4, %ymm4
-; AVX512F-NEXT:    vpblendvb %ymm4, %ymm5, %ymm3, %ymm3
-; AVX512F-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX512F-NEXT:    vpsubb %ymm1, %ymm4, %ymm1
-; AVX512F-NEXT:    vpand %ymm2, %ymm1, %ymm1
+; AVX512F-NEXT:    vpsrlw $4, %ymm0, %ymm2
+; AVX512F-NEXT:    vpand {{.*}}(%rip), %ymm2, %ymm2
+; AVX512F-NEXT:    vpsllw $4, %ymm0, %ymm3
+; AVX512F-NEXT:    vpand {{.*}}(%rip), %ymm3, %ymm3
+; AVX512F-NEXT:    vpor %ymm2, %ymm3, %ymm2
 ; AVX512F-NEXT:    vpsllw $5, %ymm1, %ymm1
-; AVX512F-NEXT:    vpaddb %ymm1, %ymm1, %ymm2
-; AVX512F-NEXT:    vpsrlw $4, %ymm0, %ymm4
-; AVX512F-NEXT:    vpand {{.*}}(%rip), %ymm4, %ymm4
-; AVX512F-NEXT:    vpblendvb %ymm1, %ymm4, %ymm0, %ymm0
-; AVX512F-NEXT:    vpsrlw $2, %ymm0, %ymm1
-; AVX512F-NEXT:    vpand {{.*}}(%rip), %ymm1, %ymm1
-; AVX512F-NEXT:    vpblendvb %ymm2, %ymm1, %ymm0, %ymm0
-; AVX512F-NEXT:    vpsrlw $1, %ymm0, %ymm1
-; AVX512F-NEXT:    vpand {{.*}}(%rip), %ymm1, %ymm1
-; AVX512F-NEXT:    vpaddb %ymm2, %ymm2, %ymm2
-; AVX512F-NEXT:    vpblendvb %ymm2, %ymm1, %ymm0, %ymm0
-; AVX512F-NEXT:    vpor %ymm0, %ymm3, %ymm0
+; AVX512F-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
+; AVX512F-NEXT:    vpsrlw $6, %ymm0, %ymm2
+; AVX512F-NEXT:    vpand {{.*}}(%rip), %ymm2, %ymm2
+; AVX512F-NEXT:    vpsllw $2, %ymm0, %ymm3
+; AVX512F-NEXT:    vpand {{.*}}(%rip), %ymm3, %ymm3
+; AVX512F-NEXT:    vpor %ymm2, %ymm3, %ymm2
+; AVX512F-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
+; AVX512F-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
+; AVX512F-NEXT:    vpaddb %ymm0, %ymm0, %ymm2
+; AVX512F-NEXT:    vpsrlw $7, %ymm0, %ymm3
+; AVX512F-NEXT:    vpand {{.*}}(%rip), %ymm3, %ymm3
+; AVX512F-NEXT:    vpor %ymm3, %ymm2, %ymm2
+; AVX512F-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
+; AVX512F-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: var_funnnel_v32i8:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm2 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
-; AVX512VL-NEXT:    vpand %ymm2, %ymm1, %ymm3
-; AVX512VL-NEXT:    vpsllw $5, %ymm3, %ymm3
-; AVX512VL-NEXT:    vpaddb %ymm3, %ymm3, %ymm4
-; AVX512VL-NEXT:    vpsllw $4, %ymm0, %ymm5
-; AVX512VL-NEXT:    vpand {{.*}}(%rip), %ymm5, %ymm5
-; AVX512VL-NEXT:    vpblendvb %ymm3, %ymm5, %ymm0, %ymm3
-; AVX512VL-NEXT:    vpsllw $2, %ymm3, %ymm5
-; AVX512VL-NEXT:    vpand {{.*}}(%rip), %ymm5, %ymm5
-; AVX512VL-NEXT:    vpblendvb %ymm4, %ymm5, %ymm3, %ymm3
-; AVX512VL-NEXT:    vpaddb %ymm3, %ymm3, %ymm5
-; AVX512VL-NEXT:    vpaddb %ymm4, %ymm4, %ymm4
-; AVX512VL-NEXT:    vpblendvb %ymm4, %ymm5, %ymm3, %ymm3
-; AVX512VL-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX512VL-NEXT:    vpsubb %ymm1, %ymm4, %ymm1
-; AVX512VL-NEXT:    vpand %ymm2, %ymm1, %ymm1
+; AVX512VL-NEXT:    vpsrlw $4, %ymm0, %ymm2
+; AVX512VL-NEXT:    vpand {{.*}}(%rip), %ymm2, %ymm2
+; AVX512VL-NEXT:    vpsllw $4, %ymm0, %ymm3
+; AVX512VL-NEXT:    vpand {{.*}}(%rip), %ymm3, %ymm3
+; AVX512VL-NEXT:    vpor %ymm2, %ymm3, %ymm2
 ; AVX512VL-NEXT:    vpsllw $5, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpaddb %ymm1, %ymm1, %ymm2
-; AVX512VL-NEXT:    vpsrlw $4, %ymm0, %ymm4
-; AVX512VL-NEXT:    vpand {{.*}}(%rip), %ymm4, %ymm4
-; AVX512VL-NEXT:    vpblendvb %ymm1, %ymm4, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpsrlw $2, %ymm0, %ymm1
-; AVX512VL-NEXT:    vpand {{.*}}(%rip), %ymm1, %ymm1
-; AVX512VL-NEXT:    vpblendvb %ymm2, %ymm1, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpsrlw $1, %ymm0, %ymm1
-; AVX512VL-NEXT:    vpand {{.*}}(%rip), %ymm1, %ymm1
-; AVX512VL-NEXT:    vpaddb %ymm2, %ymm2, %ymm2
-; AVX512VL-NEXT:    vpblendvb %ymm2, %ymm1, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpor %ymm0, %ymm3, %ymm0
+; AVX512VL-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpsrlw $6, %ymm0, %ymm2
+; AVX512VL-NEXT:    vpand {{.*}}(%rip), %ymm2, %ymm2
+; AVX512VL-NEXT:    vpsllw $2, %ymm0, %ymm3
+; AVX512VL-NEXT:    vpand {{.*}}(%rip), %ymm3, %ymm3
+; AVX512VL-NEXT:    vpor %ymm2, %ymm3, %ymm2
+; AVX512VL-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
+; AVX512VL-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpaddb %ymm0, %ymm0, %ymm2
+; AVX512VL-NEXT:    vpsrlw $7, %ymm0, %ymm3
+; AVX512VL-NEXT:    vpand {{.*}}(%rip), %ymm3, %ymm3
+; AVX512VL-NEXT:    vpor %ymm3, %ymm2, %ymm2
+; AVX512VL-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
+; AVX512VL-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: var_funnnel_v32i8:
@@ -624,47 +493,20 @@ define <32 x i8> @var_funnnel_v32i8(<32 x i8> %x, <32 x i8> %amt) nounwind {
 ;
 ; XOPAVX1-LABEL: var_funnnel_v32i8:
 ; XOPAVX1:       # %bb.0:
-; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
-; XOPAVX1-NEXT:    vandps {{.*}}(%rip), %ymm1, %ymm3
-; XOPAVX1-NEXT:    vextractf128 $1, %ymm3, %xmm4
-; XOPAVX1-NEXT:    vpshlb %xmm4, %xmm2, %xmm4
-; XOPAVX1-NEXT:    vpshlb %xmm3, %xmm0, %xmm3
-; XOPAVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm3, %ymm3
-; XOPAVX1-NEXT:    vextractf128 $1, %ymm1, %xmm4
-; XOPAVX1-NEXT:    vpxor %xmm5, %xmm5, %xmm5
-; XOPAVX1-NEXT:    vpsubb %xmm4, %xmm5, %xmm4
-; XOPAVX1-NEXT:    vmovdqa {{.*#+}} xmm6 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
-; XOPAVX1-NEXT:    vpand %xmm6, %xmm4, %xmm4
-; XOPAVX1-NEXT:    vpsubb %xmm4, %xmm5, %xmm4
-; XOPAVX1-NEXT:    vpshlb %xmm4, %xmm2, %xmm2
-; XOPAVX1-NEXT:    vpsubb %xmm1, %xmm5, %xmm1
-; XOPAVX1-NEXT:    vpand %xmm6, %xmm1, %xmm1
-; XOPAVX1-NEXT:    vpsubb %xmm1, %xmm5, %xmm1
-; XOPAVX1-NEXT:    vpshlb %xmm1, %xmm0, %xmm0
+; XOPAVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
+; XOPAVX1-NEXT:    vprotb %xmm2, %xmm3, %xmm2
+; XOPAVX1-NEXT:    vprotb %xmm1, %xmm0, %xmm0
 ; XOPAVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
-; XOPAVX1-NEXT:    vorps %ymm0, %ymm3, %ymm0
 ; XOPAVX1-NEXT:    retq
 ;
 ; XOPAVX2-LABEL: var_funnnel_v32i8:
 ; XOPAVX2:       # %bb.0:
-; XOPAVX2-NEXT:    vmovdqa {{.*#+}} ymm2 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
-; XOPAVX2-NEXT:    vpand %ymm2, %ymm1, %ymm3
-; XOPAVX2-NEXT:    vextracti128 $1, %ymm3, %xmm4
-; XOPAVX2-NEXT:    vextracti128 $1, %ymm0, %xmm5
-; XOPAVX2-NEXT:    vpshlb %xmm4, %xmm5, %xmm4
-; XOPAVX2-NEXT:    vpshlb %xmm3, %xmm0, %xmm3
-; XOPAVX2-NEXT:    vinserti128 $1, %xmm4, %ymm3, %ymm3
-; XOPAVX2-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; XOPAVX2-NEXT:    vpsubb %ymm1, %ymm4, %ymm1
-; XOPAVX2-NEXT:    vpand %ymm2, %ymm1, %ymm1
 ; XOPAVX2-NEXT:    vextracti128 $1, %ymm1, %xmm2
-; XOPAVX2-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; XOPAVX2-NEXT:    vpsubb %xmm2, %xmm4, %xmm2
-; XOPAVX2-NEXT:    vpshlb %xmm2, %xmm5, %xmm2
-; XOPAVX2-NEXT:    vpsubb %xmm1, %xmm4, %xmm1
-; XOPAVX2-NEXT:    vpshlb %xmm1, %xmm0, %xmm0
+; XOPAVX2-NEXT:    vextracti128 $1, %ymm0, %xmm3
+; XOPAVX2-NEXT:    vprotb %xmm2, %xmm3, %xmm2
+; XOPAVX2-NEXT:    vprotb %xmm1, %xmm0, %xmm0
 ; XOPAVX2-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm0
-; XOPAVX2-NEXT:    vpor %ymm0, %ymm3, %ymm0
 ; XOPAVX2-NEXT:    retq
   %res = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %x, <32 x i8> %x, <32 x i8> %amt)
   ret <32 x i8> %res
@@ -711,49 +553,51 @@ define <4 x i64> @splatvar_funnnel_v4i64(<4 x i64> %x, <4 x i64> %amt) nounwind
 ; AVX2-NEXT:    vpor %ymm0, %ymm3, %ymm0
 ; AVX2-NEXT:    retq
 ;
-; AVX512-LABEL: splatvar_funnnel_v4i64:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpbroadcastq %xmm1, %ymm1
-; AVX512-NEXT:    vmovdqa {{.*#+}} xmm2 = [63,63]
-; AVX512-NEXT:    vpand %xmm2, %xmm1, %xmm3
-; AVX512-NEXT:    vpsllq %xmm3, %ymm0, %ymm3
-; AVX512-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX512-NEXT:    vpsubq %xmm1, %xmm4, %xmm1
-; AVX512-NEXT:    vpand %xmm2, %xmm1, %xmm1
-; AVX512-NEXT:    vpsrlq %xmm1, %ymm0, %ymm0
-; AVX512-NEXT:    vpor %ymm0, %ymm3, %ymm0
-; AVX512-NEXT:    retq
+; AVX512F-LABEL: splatvar_funnnel_v4i64:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512F-NEXT:    vpbroadcastq %xmm1, %ymm1
+; AVX512F-NEXT:    vprolvq %zmm1, %zmm0, %zmm0
+; AVX512F-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: splatvar_funnnel_v4i64:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpbroadcastq %xmm1, %ymm1
+; AVX512VL-NEXT:    vprolvq %ymm1, %ymm0, %ymm0
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: splatvar_funnnel_v4i64:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512BW-NEXT:    vpbroadcastq %xmm1, %ymm1
+; AVX512BW-NEXT:    vprolvq %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
+; AVX512BW-NEXT:    retq
+;
+; AVX512VLBW-LABEL: splatvar_funnnel_v4i64:
+; AVX512VLBW:       # %bb.0:
+; AVX512VLBW-NEXT:    vpbroadcastq %xmm1, %ymm1
+; AVX512VLBW-NEXT:    vprolvq %ymm1, %ymm0, %ymm0
+; AVX512VLBW-NEXT:    retq
 ;
 ; XOPAVX1-LABEL: splatvar_funnnel_v4i64:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vmovddup {{.*#+}} xmm1 = xmm1[0,0]
-; XOPAVX1-NEXT:    vmovapd {{.*#+}} xmm2 = [63,63]
-; XOPAVX1-NEXT:    vandpd %xmm2, %xmm1, %xmm3
-; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
-; XOPAVX1-NEXT:    vpsllq %xmm3, %xmm4, %xmm5
-; XOPAVX1-NEXT:    vpsllq %xmm3, %xmm0, %xmm3
-; XOPAVX1-NEXT:    vinsertf128 $1, %xmm5, %ymm3, %ymm3
-; XOPAVX1-NEXT:    vpxor %xmm5, %xmm5, %xmm5
-; XOPAVX1-NEXT:    vpsubq %xmm1, %xmm5, %xmm1
-; XOPAVX1-NEXT:    vpand %xmm2, %xmm1, %xmm1
-; XOPAVX1-NEXT:    vpsubq %xmm1, %xmm5, %xmm1
-; XOPAVX1-NEXT:    vpshlq %xmm1, %xmm4, %xmm2
-; XOPAVX1-NEXT:    vpshlq %xmm1, %xmm0, %xmm0
+; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; XOPAVX1-NEXT:    vprotq %xmm1, %xmm2, %xmm2
+; XOPAVX1-NEXT:    vprotq %xmm1, %xmm0, %xmm0
 ; XOPAVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
-; XOPAVX1-NEXT:    vorps %ymm0, %ymm3, %ymm0
 ; XOPAVX1-NEXT:    retq
 ;
 ; XOPAVX2-LABEL: splatvar_funnnel_v4i64:
 ; XOPAVX2:       # %bb.0:
 ; XOPAVX2-NEXT:    vpbroadcastq %xmm1, %ymm1
-; XOPAVX2-NEXT:    vmovdqa {{.*#+}} xmm2 = [63,63]
-; XOPAVX2-NEXT:    vpand %xmm2, %xmm1, %xmm3
-; XOPAVX2-NEXT:    vpsllq %xmm3, %ymm0, %ymm3
-; XOPAVX2-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; XOPAVX2-NEXT:    vpsubq %xmm1, %xmm4, %xmm1
-; XOPAVX2-NEXT:    vpand %xmm2, %xmm1, %xmm1
-; XOPAVX2-NEXT:    vpsrlq %xmm1, %ymm0, %ymm0
-; XOPAVX2-NEXT:    vpor %ymm0, %ymm3, %ymm0
+; XOPAVX2-NEXT:    vextracti128 $1, %ymm0, %xmm2
+; XOPAVX2-NEXT:    vextracti128 $1, %ymm1, %xmm3
+; XOPAVX2-NEXT:    vprotq %xmm3, %xmm2, %xmm2
+; XOPAVX2-NEXT:    vprotq %xmm1, %xmm0, %xmm0
+; XOPAVX2-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm0
 ; XOPAVX2-NEXT:    retq
   %splat = shufflevector <4 x i64> %amt, <4 x i64> undef, <4 x i32> zeroinitializer
   %res = call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %x, <4 x i64> %x, <4 x i64> %splat)
@@ -763,102 +607,81 @@ define <4 x i64> @splatvar_funnnel_v4i64(<4 x i64> %x, <4 x i64> %amt) nounwind
 define <8 x i32> @splatvar_funnnel_v8i32(<8 x i32> %x, <8 x i32> %amt) nounwind {
 ; AVX1-LABEL: splatvar_funnnel_v8i32:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm10 = xmm1[0,0,0,0]
-; AVX1-NEXT:    vpxor %xmm9, %xmm9, %xmm9
-; AVX1-NEXT:    vpsubd %xmm10, %xmm9, %xmm3
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm8 = [31,31,31,31]
-; AVX1-NEXT:    vpand %xmm8, %xmm3, %xmm3
-; AVX1-NEXT:    vpsrldq {{.*#+}} xmm5 = xmm3[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm6
-; AVX1-NEXT:    vpsrld %xmm5, %xmm6, %xmm7
-; AVX1-NEXT:    vpsrlq $32, %xmm3, %xmm4
-; AVX1-NEXT:    vpsrld %xmm4, %xmm6, %xmm2
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm11 = xmm2[0,1,2,3],xmm7[4,5,6,7]
-; AVX1-NEXT:    vpunpckhdq {{.*#+}} xmm7 = xmm3[2],xmm9[2],xmm3[3],xmm9[3]
-; AVX1-NEXT:    vpsrld %xmm7, %xmm6, %xmm1
-; AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm3 = xmm3[0],zero,xmm3[1],zero
-; AVX1-NEXT:    vpsrld %xmm3, %xmm6, %xmm2
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm2[0,1,2,3],xmm1[4,5,6,7]
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm11[2,3],xmm1[4,5],xmm11[6,7]
-; AVX1-NEXT:    vpsrld %xmm5, %xmm0, %xmm2
-; AVX1-NEXT:    vpsrld %xmm4, %xmm0, %xmm4
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm4[0,1,2,3],xmm2[4,5,6,7]
-; AVX1-NEXT:    vpsrld %xmm7, %xmm0, %xmm4
-; AVX1-NEXT:    vpsrld %xmm3, %xmm0, %xmm3
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm3 = xmm3[0,1,2,3],xmm4[4,5,6,7]
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0,1],xmm2[2,3],xmm3[4,5],xmm2[6,7]
-; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm1
-; AVX1-NEXT:    vpand %xmm8, %xmm10, %xmm2
-; AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero
-; AVX1-NEXT:    vpslld %xmm2, %xmm6, %xmm3
-; AVX1-NEXT:    vpslld %xmm2, %xmm0, %xmm0
-; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm0
-; AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; AVX1-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm3 = xmm1[0],zero,xmm1[1],zero
+; AVX1-NEXT:    vpslld %xmm3, %xmm2, %xmm4
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm5 = [32,32,32,32]
+; AVX1-NEXT:    vpsubd %xmm1, %xmm5, %xmm1
+; AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero
+; AVX1-NEXT:    vpsrld %xmm1, %xmm2, %xmm2
+; AVX1-NEXT:    vpor %xmm2, %xmm4, %xmm2
+; AVX1-NEXT:    vpslld %xmm3, %xmm0, %xmm3
+; AVX1-NEXT:    vpsrld %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpor %xmm0, %xmm3, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: splatvar_funnnel_v8i32:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpbroadcastd %xmm1, %ymm1
 ; AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [31,31,31,31]
-; AVX2-NEXT:    vpand %xmm2, %xmm1, %xmm3
-; AVX2-NEXT:    vpmovzxdq {{.*#+}} xmm3 = xmm3[0],zero,xmm3[1],zero
-; AVX2-NEXT:    vpslld %xmm3, %ymm0, %ymm3
-; AVX2-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX2-NEXT:    vpsubd %xmm1, %xmm4, %xmm1
 ; AVX2-NEXT:    vpand %xmm2, %xmm1, %xmm1
+; AVX2-NEXT:    vpmovzxdq {{.*#+}} xmm2 = xmm1[0],zero,xmm1[1],zero
+; AVX2-NEXT:    vpslld %xmm2, %ymm0, %ymm2
+; AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm3 = [32,32,32,32]
+; AVX2-NEXT:    vpsubd %xmm1, %xmm3, %xmm1
 ; AVX2-NEXT:    vpmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero
 ; AVX2-NEXT:    vpsrld %xmm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpor %ymm0, %ymm3, %ymm0
+; AVX2-NEXT:    vpor %ymm0, %ymm2, %ymm0
 ; AVX2-NEXT:    retq
 ;
-; AVX512-LABEL: splatvar_funnnel_v8i32:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpbroadcastd %xmm1, %ymm1
-; AVX512-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [31,31,31,31]
-; AVX512-NEXT:    vpand %xmm2, %xmm1, %xmm3
-; AVX512-NEXT:    vpmovzxdq {{.*#+}} xmm3 = xmm3[0],zero,xmm3[1],zero
-; AVX512-NEXT:    vpslld %xmm3, %ymm0, %ymm3
-; AVX512-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX512-NEXT:    vpsubd %xmm1, %xmm4, %xmm1
-; AVX512-NEXT:    vpand %xmm2, %xmm1, %xmm1
-; AVX512-NEXT:    vpmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero
-; AVX512-NEXT:    vpsrld %xmm1, %ymm0, %ymm0
-; AVX512-NEXT:    vpor %ymm0, %ymm3, %ymm0
-; AVX512-NEXT:    retq
+; AVX512F-LABEL: splatvar_funnnel_v8i32:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512F-NEXT:    vpbroadcastd %xmm1, %ymm1
+; AVX512F-NEXT:    vprolvd %zmm1, %zmm0, %zmm0
+; AVX512F-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: splatvar_funnnel_v8i32:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpbroadcastd %xmm1, %ymm1
+; AVX512VL-NEXT:    vprolvd %ymm1, %ymm0, %ymm0
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: splatvar_funnnel_v8i32:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512BW-NEXT:    vpbroadcastd %xmm1, %ymm1
+; AVX512BW-NEXT:    vprolvd %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
+; AVX512BW-NEXT:    retq
+;
+; AVX512VLBW-LABEL: splatvar_funnnel_v8i32:
+; AVX512VLBW:       # %bb.0:
+; AVX512VLBW-NEXT:    vpbroadcastd %xmm1, %ymm1
+; AVX512VLBW-NEXT:    vprolvd %ymm1, %ymm0, %ymm0
+; AVX512VLBW-NEXT:    retq
 ;
 ; XOPAVX1-LABEL: splatvar_funnnel_v8i32:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
-; XOPAVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [31,31,31,31]
-; XOPAVX1-NEXT:    vpand %xmm2, %xmm1, %xmm3
-; XOPAVX1-NEXT:    vpmovzxdq {{.*#+}} xmm3 = xmm3[0],zero,xmm3[1],zero
-; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
-; XOPAVX1-NEXT:    vpslld %xmm3, %xmm4, %xmm5
-; XOPAVX1-NEXT:    vpslld %xmm3, %xmm0, %xmm3
-; XOPAVX1-NEXT:    vinsertf128 $1, %xmm5, %ymm3, %ymm3
-; XOPAVX1-NEXT:    vpxor %xmm5, %xmm5, %xmm5
-; XOPAVX1-NEXT:    vpsubd %xmm1, %xmm5, %xmm1
-; XOPAVX1-NEXT:    vpand %xmm2, %xmm1, %xmm1
-; XOPAVX1-NEXT:    vpsubd %xmm1, %xmm5, %xmm1
-; XOPAVX1-NEXT:    vpshld %xmm1, %xmm4, %xmm2
-; XOPAVX1-NEXT:    vpshld %xmm1, %xmm0, %xmm0
+; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; XOPAVX1-NEXT:    vprotd %xmm1, %xmm2, %xmm2
+; XOPAVX1-NEXT:    vprotd %xmm1, %xmm0, %xmm0
 ; XOPAVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
-; XOPAVX1-NEXT:    vorps %ymm0, %ymm3, %ymm0
 ; XOPAVX1-NEXT:    retq
 ;
 ; XOPAVX2-LABEL: splatvar_funnnel_v8i32:
 ; XOPAVX2:       # %bb.0:
 ; XOPAVX2-NEXT:    vpbroadcastd %xmm1, %ymm1
-; XOPAVX2-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [31,31,31,31]
-; XOPAVX2-NEXT:    vpand %xmm2, %xmm1, %xmm3
-; XOPAVX2-NEXT:    vpmovzxdq {{.*#+}} xmm3 = xmm3[0],zero,xmm3[1],zero
-; XOPAVX2-NEXT:    vpslld %xmm3, %ymm0, %ymm3
-; XOPAVX2-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; XOPAVX2-NEXT:    vpsubd %xmm1, %xmm4, %xmm1
-; XOPAVX2-NEXT:    vpand %xmm2, %xmm1, %xmm1
-; XOPAVX2-NEXT:    vpmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero
-; XOPAVX2-NEXT:    vpsrld %xmm1, %ymm0, %ymm0
-; XOPAVX2-NEXT:    vpor %ymm0, %ymm3, %ymm0
+; XOPAVX2-NEXT:    vextracti128 $1, %ymm0, %xmm2
+; XOPAVX2-NEXT:    vextracti128 $1, %ymm1, %xmm3
+; XOPAVX2-NEXT:    vprotd %xmm3, %xmm2, %xmm2
+; XOPAVX2-NEXT:    vprotd %xmm1, %xmm0, %xmm0
+; XOPAVX2-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm0
 ; XOPAVX2-NEXT:    retq
   %splat = shufflevector <8 x i32> %amt, <8 x i32> undef, <8 x i32> zeroinitializer
   %res = call <8 x i32> @llvm.fshl.v8i32(<8 x i32> %x, <8 x i32> %x, <8 x i32> %splat)
@@ -869,107 +692,66 @@ define <16 x i16> @splatvar_funnnel_v16i16(<16 x i16> %x, <16 x i16> %amt) nounw
 ; AVX1-LABEL: splatvar_funnnel_v16i16:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm1[0,0,2,3,4,5,6,7]
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm9 = xmm1[0,0,0,0]
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpsubw %xmm9, %xmm2, %xmm2
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm8 = [15,15,15,15,15,15,15,15]
-; AVX1-NEXT:    vpand %xmm8, %xmm2, %xmm2
-; AVX1-NEXT:    vpsllw $12, %xmm2, %xmm4
-; AVX1-NEXT:    vpsllw $4, %xmm2, %xmm2
-; AVX1-NEXT:    vpor %xmm4, %xmm2, %xmm2
-; AVX1-NEXT:    vpaddw %xmm2, %xmm2, %xmm4
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm5
-; AVX1-NEXT:    vpsrlw $8, %xmm5, %xmm6
-; AVX1-NEXT:    vpblendvb %xmm2, %xmm6, %xmm5, %xmm6
-; AVX1-NEXT:    vpsrlw $4, %xmm6, %xmm7
-; AVX1-NEXT:    vpblendvb %xmm4, %xmm7, %xmm6, %xmm6
-; AVX1-NEXT:    vpsrlw $2, %xmm6, %xmm7
-; AVX1-NEXT:    vpaddw %xmm4, %xmm4, %xmm3
-; AVX1-NEXT:    vpblendvb %xmm3, %xmm7, %xmm6, %xmm6
-; AVX1-NEXT:    vpsrlw $1, %xmm6, %xmm7
-; AVX1-NEXT:    vpaddw %xmm3, %xmm3, %xmm1
-; AVX1-NEXT:    vpblendvb %xmm1, %xmm7, %xmm6, %xmm6
-; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm7
-; AVX1-NEXT:    vpblendvb %xmm2, %xmm7, %xmm0, %xmm2
-; AVX1-NEXT:    vpsrlw $4, %xmm2, %xmm7
-; AVX1-NEXT:    vpblendvb %xmm4, %xmm7, %xmm2, %xmm2
-; AVX1-NEXT:    vpsrlw $2, %xmm2, %xmm4
-; AVX1-NEXT:    vpblendvb %xmm3, %xmm4, %xmm2, %xmm2
-; AVX1-NEXT:    vpsrlw $1, %xmm2, %xmm3
-; AVX1-NEXT:    vpblendvb %xmm1, %xmm3, %xmm2, %xmm1
-; AVX1-NEXT:    vinsertf128 $1, %xmm6, %ymm1, %ymm1
-; AVX1-NEXT:    vpand %xmm8, %xmm9, %xmm2
-; AVX1-NEXT:    vpmovzxwq {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero
-; AVX1-NEXT:    vpsllw %xmm2, %xmm5, %xmm3
-; AVX1-NEXT:    vpsllw %xmm2, %xmm0, %xmm0
-; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm0
-; AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; AVX1-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX1-NEXT:    vpmovzxwq {{.*#+}} xmm3 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; AVX1-NEXT:    vpsllw %xmm3, %xmm2, %xmm4
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm5 = [16,16,16,16,16,16,16,16]
+; AVX1-NEXT:    vpsubw %xmm1, %xmm5, %xmm1
+; AVX1-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; AVX1-NEXT:    vpsrlw %xmm1, %xmm2, %xmm2
+; AVX1-NEXT:    vpor %xmm2, %xmm4, %xmm2
+; AVX1-NEXT:    vpsllw %xmm3, %xmm0, %xmm3
+; AVX1-NEXT:    vpsrlw %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpor %xmm0, %xmm3, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: splatvar_funnnel_v16i16:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpbroadcastw %xmm1, %ymm1
-; AVX2-NEXT:    vmovdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15]
-; AVX2-NEXT:    vpand %xmm2, %xmm1, %xmm3
-; AVX2-NEXT:    vpmovzxwq {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero
-; AVX2-NEXT:    vpsllw %xmm3, %ymm0, %ymm3
-; AVX2-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX2-NEXT:    vpsubw %xmm1, %xmm4, %xmm1
-; AVX2-NEXT:    vpand %xmm2, %xmm1, %xmm1
+; AVX2-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX2-NEXT:    vpmovzxwq {{.*#+}} xmm2 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; AVX2-NEXT:    vpsllw %xmm2, %ymm0, %ymm2
+; AVX2-NEXT:    vmovdqa {{.*#+}} xmm3 = [16,16,16,16,16,16,16,16]
+; AVX2-NEXT:    vpsubw %xmm1, %xmm3, %xmm1
 ; AVX2-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
 ; AVX2-NEXT:    vpsrlw %xmm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpor %ymm0, %ymm3, %ymm0
+; AVX2-NEXT:    vpor %ymm0, %ymm2, %ymm0
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: splatvar_funnnel_v16i16:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpbroadcastw %xmm1, %ymm1
-; AVX512-NEXT:    vmovdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15]
-; AVX512-NEXT:    vpand %xmm2, %xmm1, %xmm3
-; AVX512-NEXT:    vpmovzxwq {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero
-; AVX512-NEXT:    vpsllw %xmm3, %ymm0, %ymm3
-; AVX512-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX512-NEXT:    vpsubw %xmm1, %xmm4, %xmm1
-; AVX512-NEXT:    vpand %xmm2, %xmm1, %xmm1
+; AVX512-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX512-NEXT:    vpmovzxwq {{.*#+}} xmm2 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; AVX512-NEXT:    vpsllw %xmm2, %ymm0, %ymm2
+; AVX512-NEXT:    vmovdqa {{.*#+}} xmm3 = [16,16,16,16,16,16,16,16]
+; AVX512-NEXT:    vpsubw %xmm1, %xmm3, %xmm1
 ; AVX512-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
 ; AVX512-NEXT:    vpsrlw %xmm1, %ymm0, %ymm0
-; AVX512-NEXT:    vpor %ymm0, %ymm3, %ymm0
+; AVX512-NEXT:    vpor %ymm0, %ymm2, %ymm0
 ; AVX512-NEXT:    retq
 ;
 ; XOPAVX1-LABEL: splatvar_funnnel_v16i16:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm1[0,0,2,3,4,5,6,7]
 ; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
-; XOPAVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15]
-; XOPAVX1-NEXT:    vpand %xmm2, %xmm1, %xmm3
-; XOPAVX1-NEXT:    vpmovzxwq {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero
-; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
-; XOPAVX1-NEXT:    vpsllw %xmm3, %xmm4, %xmm5
-; XOPAVX1-NEXT:    vpsllw %xmm3, %xmm0, %xmm3
-; XOPAVX1-NEXT:    vinsertf128 $1, %xmm5, %ymm3, %ymm3
-; XOPAVX1-NEXT:    vpxor %xmm5, %xmm5, %xmm5
-; XOPAVX1-NEXT:    vpsubw %xmm1, %xmm5, %xmm1
-; XOPAVX1-NEXT:    vpand %xmm2, %xmm1, %xmm1
-; XOPAVX1-NEXT:    vpsubw %xmm1, %xmm5, %xmm1
-; XOPAVX1-NEXT:    vpshlw %xmm1, %xmm4, %xmm2
-; XOPAVX1-NEXT:    vpshlw %xmm1, %xmm0, %xmm0
+; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; XOPAVX1-NEXT:    vprotw %xmm1, %xmm2, %xmm2
+; XOPAVX1-NEXT:    vprotw %xmm1, %xmm0, %xmm0
 ; XOPAVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
-; XOPAVX1-NEXT:    vorps %ymm0, %ymm3, %ymm0
 ; XOPAVX1-NEXT:    retq
 ;
 ; XOPAVX2-LABEL: splatvar_funnnel_v16i16:
 ; XOPAVX2:       # %bb.0:
 ; XOPAVX2-NEXT:    vpbroadcastw %xmm1, %ymm1
-; XOPAVX2-NEXT:    vmovdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15]
-; XOPAVX2-NEXT:    vpand %xmm2, %xmm1, %xmm3
-; XOPAVX2-NEXT:    vpmovzxwq {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero
-; XOPAVX2-NEXT:    vpsllw %xmm3, %ymm0, %ymm3
-; XOPAVX2-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; XOPAVX2-NEXT:    vpsubw %xmm1, %xmm4, %xmm1
-; XOPAVX2-NEXT:    vpand %xmm2, %xmm1, %xmm1
-; XOPAVX2-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
-; XOPAVX2-NEXT:    vpsrlw %xmm1, %ymm0, %ymm0
-; XOPAVX2-NEXT:    vpor %ymm0, %ymm3, %ymm0
+; XOPAVX2-NEXT:    vextracti128 $1, %ymm0, %xmm2
+; XOPAVX2-NEXT:    vextracti128 $1, %ymm1, %xmm3
+; XOPAVX2-NEXT:    vprotw %xmm3, %xmm2, %xmm2
+; XOPAVX2-NEXT:    vprotw %xmm1, %xmm0, %xmm0
+; XOPAVX2-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm0
 ; XOPAVX2-NEXT:    retq
   %splat = shufflevector <16 x i16> %amt, <16 x i16> undef, <16 x i32> zeroinitializer
   %res = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %x, <16 x i16> %x, <16 x i16> %splat)
@@ -979,117 +761,93 @@ define <16 x i16> @splatvar_funnnel_v16i16(<16 x i16> %x, <16 x i16> %amt) nounw
 define <32 x i8> @splatvar_funnnel_v32i8(<32 x i8> %x, <32 x i8> %amt) nounwind {
 ; AVX1-LABEL: splatvar_funnnel_v32i8:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpxor %xmm9, %xmm9, %xmm9
-; AVX1-NEXT:    vpshufb %xmm9, %xmm1, %xmm3
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vpsrlw $4, %xmm1, %xmm4
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm10 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; AVX1-NEXT:    vpand %xmm10, %xmm4, %xmm4
-; AVX1-NEXT:    vpsubb %xmm3, %xmm9, %xmm6
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm8 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
-; AVX1-NEXT:    vpand %xmm8, %xmm6, %xmm6
-; AVX1-NEXT:    vpsllw $5, %xmm6, %xmm6
-; AVX1-NEXT:    vpblendvb %xmm6, %xmm4, %xmm1, %xmm4
-; AVX1-NEXT:    vpsrlw $2, %xmm4, %xmm7
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm11 = [63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63]
-; AVX1-NEXT:    vpand %xmm11, %xmm7, %xmm7
-; AVX1-NEXT:    vpaddb %xmm6, %xmm6, %xmm5
-; AVX1-NEXT:    vpblendvb %xmm5, %xmm7, %xmm4, %xmm4
-; AVX1-NEXT:    vpsrlw $1, %xmm4, %xmm7
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm12 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
-; AVX1-NEXT:    vpand %xmm12, %xmm7, %xmm7
-; AVX1-NEXT:    vpaddb %xmm5, %xmm5, %xmm2
-; AVX1-NEXT:    vpblendvb %xmm2, %xmm7, %xmm4, %xmm4
-; AVX1-NEXT:    vpsrlw $4, %xmm0, %xmm7
-; AVX1-NEXT:    vpand %xmm10, %xmm7, %xmm7
-; AVX1-NEXT:    vpblendvb %xmm6, %xmm7, %xmm0, %xmm6
-; AVX1-NEXT:    vpsrlw $2, %xmm6, %xmm7
-; AVX1-NEXT:    vpand %xmm11, %xmm7, %xmm7
-; AVX1-NEXT:    vpblendvb %xmm5, %xmm7, %xmm6, %xmm5
-; AVX1-NEXT:    vpsrlw $1, %xmm5, %xmm6
-; AVX1-NEXT:    vpand %xmm12, %xmm6, %xmm6
-; AVX1-NEXT:    vpblendvb %xmm2, %xmm6, %xmm5, %xmm2
-; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm2, %ymm2
-; AVX1-NEXT:    vpand %xmm8, %xmm3, %xmm3
-; AVX1-NEXT:    vpmovzxbq {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,zero,zero,zero,zero,xmm3[1],zero,zero,zero,zero,zero,zero,zero
-; AVX1-NEXT:    vpsllw %xmm3, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
-; AVX1-NEXT:    vpsllw %xmm3, %xmm4, %xmm4
-; AVX1-NEXT:    vpshufb %xmm9, %xmm4, %xmm4
-; AVX1-NEXT:    vpand %xmm4, %xmm1, %xmm1
-; AVX1-NEXT:    vpsllw %xmm3, %xmm0, %xmm0
-; AVX1-NEXT:    vpand %xmm4, %xmm0, %xmm0
-; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
-; AVX1-NEXT:    vorps %ymm2, %ymm0, %ymm0
+; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX1-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
+; AVX1-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX1-NEXT:    vpmovzxbq {{.*#+}} xmm3 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
+; AVX1-NEXT:    vpsllw %xmm3, %xmm4, %xmm5
+; AVX1-NEXT:    vpcmpeqd %xmm6, %xmm6, %xmm6
+; AVX1-NEXT:    vpsllw %xmm3, %xmm6, %xmm7
+; AVX1-NEXT:    vpshufb %xmm2, %xmm7, %xmm2
+; AVX1-NEXT:    vpand %xmm2, %xmm5, %xmm5
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm7 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; AVX1-NEXT:    vpsubb %xmm1, %xmm7, %xmm1
+; AVX1-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; AVX1-NEXT:    vpsrlw %xmm1, %xmm4, %xmm4
+; AVX1-NEXT:    vpsrlw %xmm1, %xmm6, %xmm6
+; AVX1-NEXT:    vpshufb {{.*#+}} xmm6 = xmm6[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
+; AVX1-NEXT:    vpand %xmm6, %xmm4, %xmm4
+; AVX1-NEXT:    vpor %xmm4, %xmm5, %xmm4
+; AVX1-NEXT:    vpsllw %xmm3, %xmm0, %xmm3
+; AVX1-NEXT:    vpand %xmm2, %xmm3, %xmm2
+; AVX1-NEXT:    vpsrlw %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpand %xmm6, %xmm0, %xmm0
+; AVX1-NEXT:    vpor %xmm0, %xmm2, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm0, %ymm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: splatvar_funnnel_v32i8:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpbroadcastb %xmm1, %ymm1
-; AVX2-NEXT:    vmovdqa {{.*#+}} xmm2 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
-; AVX2-NEXT:    vpand %xmm2, %xmm1, %xmm3
-; AVX2-NEXT:    vpmovzxbq {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,zero,zero,zero,zero,xmm3[1],zero,zero,zero,zero,zero,zero,zero
-; AVX2-NEXT:    vpsllw %xmm3, %ymm0, %ymm4
-; AVX2-NEXT:    vpcmpeqd %ymm5, %ymm5, %ymm5
-; AVX2-NEXT:    vpsllw %xmm3, %ymm5, %ymm3
-; AVX2-NEXT:    vpbroadcastb %xmm3, %ymm3
-; AVX2-NEXT:    vpand %ymm3, %ymm4, %ymm3
-; AVX2-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX2-NEXT:    vpsubb %xmm1, %xmm4, %xmm1
-; AVX2-NEXT:    vpand %xmm2, %xmm1, %xmm1
+; AVX2-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX2-NEXT:    vpmovzxbq {{.*#+}} xmm2 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; AVX2-NEXT:    vpsllw %xmm2, %ymm0, %ymm3
+; AVX2-NEXT:    vpcmpeqd %ymm4, %ymm4, %ymm4
+; AVX2-NEXT:    vpsllw %xmm2, %ymm4, %ymm2
+; AVX2-NEXT:    vpbroadcastb %xmm2, %ymm2
+; AVX2-NEXT:    vpand %ymm2, %ymm3, %ymm2
+; AVX2-NEXT:    vmovdqa {{.*#+}} xmm3 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; AVX2-NEXT:    vpsubb %xmm1, %xmm3, %xmm1
 ; AVX2-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
 ; AVX2-NEXT:    vpsrlw %xmm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpsrlw %xmm1, %ymm5, %ymm1
+; AVX2-NEXT:    vpsrlw %xmm1, %ymm4, %ymm1
 ; AVX2-NEXT:    vpsrlw $8, %ymm1, %ymm1
 ; AVX2-NEXT:    vpbroadcastb %xmm1, %ymm1
 ; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpor %ymm0, %ymm3, %ymm0
+; AVX2-NEXT:    vpor %ymm0, %ymm2, %ymm0
 ; AVX2-NEXT:    retq
 ;
 ; AVX512F-LABEL: splatvar_funnnel_v32i8:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vpbroadcastb %xmm1, %ymm1
-; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm2 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
-; AVX512F-NEXT:    vpand %xmm2, %xmm1, %xmm3
-; AVX512F-NEXT:    vpmovzxbq {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,zero,zero,zero,zero,xmm3[1],zero,zero,zero,zero,zero,zero,zero
-; AVX512F-NEXT:    vpsllw %xmm3, %ymm0, %ymm4
-; AVX512F-NEXT:    vpcmpeqd %ymm5, %ymm5, %ymm5
-; AVX512F-NEXT:    vpsllw %xmm3, %ymm5, %ymm3
-; AVX512F-NEXT:    vpbroadcastb %xmm3, %ymm3
-; AVX512F-NEXT:    vpand %ymm3, %ymm4, %ymm3
-; AVX512F-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX512F-NEXT:    vpsubb %xmm1, %xmm4, %xmm1
-; AVX512F-NEXT:    vpand %xmm2, %xmm1, %xmm1
+; AVX512F-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX512F-NEXT:    vpmovzxbq {{.*#+}} xmm2 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512F-NEXT:    vpsllw %xmm2, %ymm0, %ymm3
+; AVX512F-NEXT:    vpcmpeqd %ymm4, %ymm4, %ymm4
+; AVX512F-NEXT:    vpsllw %xmm2, %ymm4, %ymm2
+; AVX512F-NEXT:    vpbroadcastb %xmm2, %ymm2
+; AVX512F-NEXT:    vpand %ymm2, %ymm3, %ymm2
+; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm3 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; AVX512F-NEXT:    vpsubb %xmm1, %xmm3, %xmm1
 ; AVX512F-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
 ; AVX512F-NEXT:    vpsrlw %xmm1, %ymm0, %ymm0
-; AVX512F-NEXT:    vpsrlw %xmm1, %ymm5, %ymm1
+; AVX512F-NEXT:    vpsrlw %xmm1, %ymm4, %ymm1
 ; AVX512F-NEXT:    vpsrlw $8, %ymm1, %ymm1
 ; AVX512F-NEXT:    vpbroadcastb %xmm1, %ymm1
 ; AVX512F-NEXT:    vpand %ymm1, %ymm0, %ymm0
-; AVX512F-NEXT:    vpor %ymm0, %ymm3, %ymm0
+; AVX512F-NEXT:    vpor %ymm0, %ymm2, %ymm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: splatvar_funnnel_v32i8:
 ; AVX512VL:       # %bb.0:
 ; AVX512VL-NEXT:    vpbroadcastb %xmm1, %ymm1
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm2 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
-; AVX512VL-NEXT:    vpand %xmm2, %xmm1, %xmm3
-; AVX512VL-NEXT:    vpmovzxbq {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,zero,zero,zero,zero,xmm3[1],zero,zero,zero,zero,zero,zero,zero
-; AVX512VL-NEXT:    vpsllw %xmm3, %ymm0, %ymm4
-; AVX512VL-NEXT:    vpcmpeqd %ymm5, %ymm5, %ymm5
-; AVX512VL-NEXT:    vpsllw %xmm3, %ymm5, %ymm3
-; AVX512VL-NEXT:    vpbroadcastb %xmm3, %ymm3
-; AVX512VL-NEXT:    vpand %ymm3, %ymm4, %ymm3
-; AVX512VL-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX512VL-NEXT:    vpsubb %xmm1, %xmm4, %xmm1
-; AVX512VL-NEXT:    vpand %xmm2, %xmm1, %xmm1
+; AVX512VL-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX512VL-NEXT:    vpmovzxbq {{.*#+}} xmm2 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512VL-NEXT:    vpsllw %xmm2, %ymm0, %ymm3
+; AVX512VL-NEXT:    vpcmpeqd %ymm4, %ymm4, %ymm4
+; AVX512VL-NEXT:    vpsllw %xmm2, %ymm4, %ymm2
+; AVX512VL-NEXT:    vpbroadcastb %xmm2, %ymm2
+; AVX512VL-NEXT:    vpand %ymm2, %ymm3, %ymm2
+; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm3 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; AVX512VL-NEXT:    vpsubb %xmm1, %xmm3, %xmm1
 ; AVX512VL-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
 ; AVX512VL-NEXT:    vpsrlw %xmm1, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpsrlw %xmm1, %ymm5, %ymm1
+; AVX512VL-NEXT:    vpsrlw %xmm1, %ymm4, %ymm1
 ; AVX512VL-NEXT:    vpsrlw $8, %ymm1, %ymm1
 ; AVX512VL-NEXT:    vpbroadcastb %xmm1, %ymm1
 ; AVX512VL-NEXT:    vpand %ymm1, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpor %ymm0, %ymm3, %ymm0
+; AVX512VL-NEXT:    vpor %ymm0, %ymm2, %ymm0
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: splatvar_funnnel_v32i8:
@@ -1130,43 +888,20 @@ define <32 x i8> @splatvar_funnnel_v32i8(<32 x i8> %x, <32 x i8> %amt) nounwind
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
 ; XOPAVX1-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
-; XOPAVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm1, %ymm3
-; XOPAVX1-NEXT:    vandps {{.*}}(%rip), %ymm3, %ymm3
-; XOPAVX1-NEXT:    vextractf128 $1, %ymm3, %xmm4
-; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm5
-; XOPAVX1-NEXT:    vpshlb %xmm4, %xmm5, %xmm4
-; XOPAVX1-NEXT:    vpshlb %xmm3, %xmm0, %xmm3
-; XOPAVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm3, %ymm3
-; XOPAVX1-NEXT:    vpsubb %xmm1, %xmm2, %xmm1
-; XOPAVX1-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
-; XOPAVX1-NEXT:    vpsubb %xmm1, %xmm2, %xmm1
-; XOPAVX1-NEXT:    vpshlb %xmm1, %xmm5, %xmm2
-; XOPAVX1-NEXT:    vpshlb %xmm1, %xmm0, %xmm0
+; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; XOPAVX1-NEXT:    vprotb %xmm1, %xmm2, %xmm2
+; XOPAVX1-NEXT:    vprotb %xmm1, %xmm0, %xmm0
 ; XOPAVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
-; XOPAVX1-NEXT:    vorps %ymm0, %ymm3, %ymm0
 ; XOPAVX1-NEXT:    retq
 ;
 ; XOPAVX2-LABEL: splatvar_funnnel_v32i8:
 ; XOPAVX2:       # %bb.0:
 ; XOPAVX2-NEXT:    vpbroadcastb %xmm1, %ymm1
-; XOPAVX2-NEXT:    vmovdqa {{.*#+}} ymm2 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
-; XOPAVX2-NEXT:    vpand %ymm2, %ymm1, %ymm3
-; XOPAVX2-NEXT:    vextracti128 $1, %ymm3, %xmm4
-; XOPAVX2-NEXT:    vextracti128 $1, %ymm0, %xmm5
-; XOPAVX2-NEXT:    vpshlb %xmm4, %xmm5, %xmm4
-; XOPAVX2-NEXT:    vpshlb %xmm3, %xmm0, %xmm3
-; XOPAVX2-NEXT:    vinserti128 $1, %xmm4, %ymm3, %ymm3
-; XOPAVX2-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; XOPAVX2-NEXT:    vpsubb %ymm1, %ymm4, %ymm1
-; XOPAVX2-NEXT:    vpand %ymm2, %ymm1, %ymm1
-; XOPAVX2-NEXT:    vextracti128 $1, %ymm1, %xmm2
-; XOPAVX2-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; XOPAVX2-NEXT:    vpsubb %xmm2, %xmm4, %xmm2
-; XOPAVX2-NEXT:    vpshlb %xmm2, %xmm5, %xmm2
-; XOPAVX2-NEXT:    vpsubb %xmm1, %xmm4, %xmm1
-; XOPAVX2-NEXT:    vpshlb %xmm1, %xmm0, %xmm0
+; XOPAVX2-NEXT:    vextracti128 $1, %ymm0, %xmm2
+; XOPAVX2-NEXT:    vextracti128 $1, %ymm1, %xmm3
+; XOPAVX2-NEXT:    vprotb %xmm3, %xmm2, %xmm2
+; XOPAVX2-NEXT:    vprotb %xmm1, %xmm0, %xmm0
 ; XOPAVX2-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm0
-; XOPAVX2-NEXT:    vpor %ymm0, %ymm3, %ymm0
 ; XOPAVX2-NEXT:    retq
   %splat = shufflevector <32 x i8> %amt, <32 x i8> undef, <32 x i32> zeroinitializer
   %res = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %x, <32 x i8> %x, <32 x i8> %splat)
@@ -1334,47 +1069,44 @@ define <16 x i16> @constant_funnnel_v16i16(<16 x i16> %x) nounwind {
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [256,512,1024,2048,4096,8192,16384,32768]
-; AVX1-NEXT:    vpmullw %xmm2, %xmm1, %xmm3
-; AVX1-NEXT:    vpmullw {{.*}}(%rip), %xmm0, %xmm4
-; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm4, %ymm3
-; AVX1-NEXT:    vpmulhuw %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpmulhuw {{.*}}(%rip), %xmm0, %xmm2
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm2[1,2,3,4,5,6,7]
+; AVX1-NEXT:    vpmulhuw %xmm2, %xmm1, %xmm3
+; AVX1-NEXT:    vpmullw %xmm2, %xmm1, %xmm1
+; AVX1-NEXT:    vpor %xmm3, %xmm1, %xmm1
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [1,2,4,8,16,32,64,128]
+; AVX1-NEXT:    vpmulhuw %xmm2, %xmm0, %xmm3
+; AVX1-NEXT:    vpmullw %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpor %xmm3, %xmm0, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
-; AVX1-NEXT:    vorps %ymm0, %ymm3, %ymm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: constant_funnnel_v16i16:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpmulhuw {{.*}}(%rip), %ymm0, %ymm1
-; AVX2-NEXT:    vpblendw {{.*#+}} ymm2 = ymm0[0],ymm1[1,2,3,4,5,6,7],ymm0[8],ymm1[9,10,11,12,13,14,15]
-; AVX2-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
-; AVX2-NEXT:    vpmullw {{.*}}(%rip), %ymm0, %ymm0
-; AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vmovdqa {{.*#+}} ymm1 = [1,2,4,8,16,32,64,128,256,512,1024,2048,4096,8192,16384,32768]
+; AVX2-NEXT:    vpmulhuw %ymm1, %ymm0, %ymm2
+; AVX2-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpor %ymm2, %ymm0, %ymm0
 ; AVX2-NEXT:    retq
 ;
 ; AVX512F-LABEL: constant_funnnel_v16i16:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpmulhuw {{.*}}(%rip), %ymm0, %ymm1
-; AVX512F-NEXT:    vpblendw {{.*#+}} ymm2 = ymm0[0],ymm1[1,2,3,4,5,6,7],ymm0[8],ymm1[9,10,11,12,13,14,15]
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
-; AVX512F-NEXT:    vpmullw {{.*}}(%rip), %ymm0, %ymm0
-; AVX512F-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm1 = [1,2,4,8,16,32,64,128,256,512,1024,2048,4096,8192,16384,32768]
+; AVX512F-NEXT:    vpmulhuw %ymm1, %ymm0, %ymm2
+; AVX512F-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
+; AVX512F-NEXT:    vpor %ymm2, %ymm0, %ymm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: constant_funnnel_v16i16:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpmulhuw {{.*}}(%rip), %ymm0, %ymm1
-; AVX512VL-NEXT:    vpblendw {{.*#+}} ymm2 = ymm0[0],ymm1[1,2,3,4,5,6,7],ymm0[8],ymm1[9,10,11,12,13,14,15]
-; AVX512VL-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
-; AVX512VL-NEXT:    vpmullw {{.*}}(%rip), %ymm0, %ymm0
-; AVX512VL-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm1 = [1,2,4,8,16,32,64,128,256,512,1024,2048,4096,8192,16384,32768]
+; AVX512VL-NEXT:    vpmulhuw %ymm1, %ymm0, %ymm2
+; AVX512VL-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpor %ymm2, %ymm0, %ymm0
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: constant_funnnel_v16i16:
 ; AVX512BW:       # %bb.0:
 ; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; AVX512BW-NEXT:    vmovdqa {{.*#+}} ymm1 = [0,15,14,13,12,11,10,9,8,7,6,5,4,3,2,1]
+; AVX512BW-NEXT:    vmovdqa {{.*#+}} ymm1 = [16,15,14,13,12,11,10,9,8,7,6,5,4,3,2,1]
 ; AVX512BW-NEXT:    vpsrlvw %zmm1, %zmm0, %zmm1
 ; AVX512BW-NEXT:    vmovdqa {{.*#+}} ymm2 = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15]
 ; AVX512BW-NEXT:    vpsllvw %zmm2, %zmm0, %zmm0
@@ -1390,23 +1122,18 @@ define <16 x i16> @constant_funnnel_v16i16(<16 x i16> %x) nounwind {
 ;
 ; XOPAVX1-LABEL: constant_funnnel_v16i16:
 ; XOPAVX1:       # %bb.0:
-; XOPAVX1-NEXT:    vpshlw {{.*}}(%rip), %xmm0, %xmm1
-; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
-; XOPAVX1-NEXT:    vpshlw {{.*}}(%rip), %xmm2, %xmm3
-; XOPAVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm1, %ymm1
-; XOPAVX1-NEXT:    vpshlw {{.*}}(%rip), %xmm0, %xmm0
-; XOPAVX1-NEXT:    vpshlw {{.*}}(%rip), %xmm2, %xmm2
-; XOPAVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
-; XOPAVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
+; XOPAVX1-NEXT:    vprotw {{.*}}(%rip), %xmm0, %xmm1
+; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; XOPAVX1-NEXT:    vprotw {{.*}}(%rip), %xmm0, %xmm0
+; XOPAVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; XOPAVX1-NEXT:    retq
 ;
 ; XOPAVX2-LABEL: constant_funnnel_v16i16:
 ; XOPAVX2:       # %bb.0:
-; XOPAVX2-NEXT:    vpmulhuw {{.*}}(%rip), %ymm0, %ymm1
-; XOPAVX2-NEXT:    vpblendw {{.*#+}} ymm2 = ymm0[0],ymm1[1,2,3,4,5,6,7],ymm0[8],ymm1[9,10,11,12,13,14,15]
-; XOPAVX2-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
-; XOPAVX2-NEXT:    vpmullw {{.*}}(%rip), %ymm0, %ymm0
-; XOPAVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; XOPAVX2-NEXT:    vprotw {{.*}}(%rip), %xmm0, %xmm1
+; XOPAVX2-NEXT:    vextracti128 $1, %ymm0, %xmm0
+; XOPAVX2-NEXT:    vprotw {{.*}}(%rip), %xmm0, %xmm0
+; XOPAVX2-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm0
 ; XOPAVX2-NEXT:    retq
   %res = call <16 x i16> @llvm.fshl.v16i16(<16 x i16> %x, <16 x i16> %x, <16 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15>)
   ret <16 x i16> %res
@@ -1416,41 +1143,41 @@ define <32 x i8> @constant_funnnel_v32i8(<32 x i8> %x) nounwind {
 ; AVX1-LABEL: constant_funnnel_v32i8:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm1[8],xmm2[8],xmm1[9],xmm2[9],xmm1[10],xmm2[10],xmm1[11],xmm2[11],xmm1[12],xmm2[12],xmm1[13],xmm2[13],xmm1[14],xmm2[14],xmm1[15],xmm2[15]
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = [256,128,64,32,16,8,4,2]
-; AVX1-NEXT:    vpmullw %xmm4, %xmm3, %xmm3
+; AVX1-NEXT:    vpxor %xmm8, %xmm8, %xmm8
+; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm1[8],xmm8[8],xmm1[9],xmm8[9],xmm1[10],xmm8[10],xmm1[11],xmm8[11],xmm1[12],xmm8[12],xmm1[13],xmm8[13],xmm1[14],xmm8[14],xmm1[15],xmm8[15]
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm9 = [256,128,64,32,16,8,4,2]
+; AVX1-NEXT:    vpmullw %xmm9, %xmm3, %xmm3
 ; AVX1-NEXT:    vpsrlw $8, %xmm3, %xmm3
 ; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm5 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
 ; AVX1-NEXT:    vmovdqa {{.*#+}} xmm6 = [256,2,4,8,16,32,64,128]
 ; AVX1-NEXT:    vpmullw %xmm6, %xmm5, %xmm7
 ; AVX1-NEXT:    vpsrlw $8, %xmm7, %xmm7
 ; AVX1-NEXT:    vpackuswb %xmm3, %xmm7, %xmm3
-; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm0[8],xmm2[8],xmm0[9],xmm2[9],xmm0[10],xmm2[10],xmm0[11],xmm2[11],xmm0[12],xmm2[12],xmm0[13],xmm2[13],xmm0[14],xmm2[14],xmm0[15],xmm2[15]
-; AVX1-NEXT:    vpmullw %xmm4, %xmm2, %xmm2
-; AVX1-NEXT:    vpsrlw $8, %xmm2, %xmm2
-; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm4 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; AVX1-NEXT:    vpmullw %xmm6, %xmm4, %xmm6
-; AVX1-NEXT:    vpsrlw $8, %xmm6, %xmm6
-; AVX1-NEXT:    vpackuswb %xmm2, %xmm6, %xmm2
-; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm2, %ymm2
 ; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [1,128,64,32,16,8,4,2]
-; AVX1-NEXT:    vpmullw %xmm3, %xmm1, %xmm1
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm6 = [255,255,255,255,255,255,255,255]
-; AVX1-NEXT:    vpand %xmm6, %xmm1, %xmm1
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm7 = [1,2,4,8,16,32,64,128]
-; AVX1-NEXT:    vpmullw %xmm7, %xmm5, %xmm5
-; AVX1-NEXT:    vpand %xmm6, %xmm5, %xmm5
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm7 = [1,128,64,32,16,8,4,2]
+; AVX1-NEXT:    vpmullw %xmm7, %xmm1, %xmm1
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
+; AVX1-NEXT:    vpand %xmm2, %xmm1, %xmm1
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = [1,2,4,8,16,32,64,128]
+; AVX1-NEXT:    vpmullw %xmm4, %xmm5, %xmm5
+; AVX1-NEXT:    vpand %xmm2, %xmm5, %xmm5
 ; AVX1-NEXT:    vpackuswb %xmm1, %xmm5, %xmm1
+; AVX1-NEXT:    vpor %xmm3, %xmm1, %xmm1
+; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm0[8],xmm8[8],xmm0[9],xmm8[9],xmm0[10],xmm8[10],xmm0[11],xmm8[11],xmm0[12],xmm8[12],xmm0[13],xmm8[13],xmm0[14],xmm8[14],xmm0[15],xmm8[15]
+; AVX1-NEXT:    vpmullw %xmm9, %xmm3, %xmm3
+; AVX1-NEXT:    vpsrlw $8, %xmm3, %xmm3
+; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm5 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX1-NEXT:    vpmullw %xmm6, %xmm5, %xmm6
+; AVX1-NEXT:    vpsrlw $8, %xmm6, %xmm6
+; AVX1-NEXT:    vpackuswb %xmm3, %xmm6, %xmm3
 ; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; AVX1-NEXT:    vpmullw %xmm3, %xmm0, %xmm0
-; AVX1-NEXT:    vpand %xmm6, %xmm0, %xmm0
-; AVX1-NEXT:    vpmullw %xmm7, %xmm4, %xmm3
-; AVX1-NEXT:    vpand %xmm6, %xmm3, %xmm3
-; AVX1-NEXT:    vpackuswb %xmm0, %xmm3, %xmm0
+; AVX1-NEXT:    vpmullw %xmm7, %xmm0, %xmm0
+; AVX1-NEXT:    vpand %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpmullw %xmm4, %xmm5, %xmm4
+; AVX1-NEXT:    vpand %xmm2, %xmm4, %xmm2
+; AVX1-NEXT:    vpackuswb %xmm0, %xmm2, %xmm0
+; AVX1-NEXT:    vpor %xmm3, %xmm0, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
-; AVX1-NEXT:    vorps %ymm2, %ymm0, %ymm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: constant_funnnel_v32i8:
@@ -1547,29 +1274,19 @@ define <32 x i8> @constant_funnnel_v32i8(<32 x i8> %x) nounwind {
 ; XOPAVX1-LABEL: constant_funnnel_v32i8:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; XOPAVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [0,249,250,251,252,253,254,255,0,255,254,253,252,251,250,249]
-; XOPAVX1-NEXT:    vpshlb %xmm2, %xmm1, %xmm3
-; XOPAVX1-NEXT:    vpshlb %xmm2, %xmm0, %xmm2
-; XOPAVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm2, %ymm2
-; XOPAVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [0,1,2,3,4,5,6,7,0,7,6,5,4,3,2,1]
-; XOPAVX1-NEXT:    vpshlb %xmm3, %xmm1, %xmm1
-; XOPAVX1-NEXT:    vpshlb %xmm3, %xmm0, %xmm0
+; XOPAVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [0,1,2,3,4,5,6,7,8,7,6,5,4,3,2,1]
+; XOPAVX1-NEXT:    vprotb %xmm2, %xmm1, %xmm1
+; XOPAVX1-NEXT:    vprotb %xmm2, %xmm0, %xmm0
 ; XOPAVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
-; XOPAVX1-NEXT:    vorps %ymm2, %ymm0, %ymm0
 ; XOPAVX1-NEXT:    retq
 ;
 ; XOPAVX2-LABEL: constant_funnnel_v32i8:
 ; XOPAVX2:       # %bb.0:
 ; XOPAVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; XOPAVX2-NEXT:    vmovdqa {{.*#+}} xmm2 = [0,249,250,251,252,253,254,255,0,255,254,253,252,251,250,249]
-; XOPAVX2-NEXT:    vpshlb %xmm2, %xmm1, %xmm3
-; XOPAVX2-NEXT:    vpshlb %xmm2, %xmm0, %xmm2
-; XOPAVX2-NEXT:    vinserti128 $1, %xmm3, %ymm2, %ymm2
-; XOPAVX2-NEXT:    vmovdqa {{.*#+}} xmm3 = [0,1,2,3,4,5,6,7,0,7,6,5,4,3,2,1]
-; XOPAVX2-NEXT:    vpshlb %xmm3, %xmm1, %xmm1
-; XOPAVX2-NEXT:    vpshlb %xmm3, %xmm0, %xmm0
+; XOPAVX2-NEXT:    vmovdqa {{.*#+}} xmm2 = [0,1,2,3,4,5,6,7,8,7,6,5,4,3,2,1]
+; XOPAVX2-NEXT:    vprotb %xmm2, %xmm1, %xmm1
+; XOPAVX2-NEXT:    vprotb %xmm2, %xmm0, %xmm0
 ; XOPAVX2-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
-; XOPAVX2-NEXT:    vpor %ymm2, %ymm0, %ymm0
 ; XOPAVX2-NEXT:    retq
   %res = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %x, <32 x i8> %x, <32 x i8> <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1>)
   ret <32 x i8> %res
@@ -1756,16 +1473,15 @@ define <32 x i8> @splatconstant_funnnel_v32i8(<32 x i8> %x) nounwind {
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; AVX1-NEXT:    vpsrlw $4, %xmm1, %xmm2
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; AVX1-NEXT:    vpand %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
+; AVX1-NEXT:    vpandn %xmm2, %xmm3, %xmm2
 ; AVX1-NEXT:    vpsllw $4, %xmm1, %xmm1
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
-; AVX1-NEXT:    vpand %xmm4, %xmm1, %xmm1
+; AVX1-NEXT:    vpand %xmm3, %xmm1, %xmm1
 ; AVX1-NEXT:    vpor %xmm2, %xmm1, %xmm1
 ; AVX1-NEXT:    vpsrlw $4, %xmm0, %xmm2
-; AVX1-NEXT:    vpand %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vpandn %xmm2, %xmm3, %xmm2
 ; AVX1-NEXT:    vpsllw $4, %xmm0, %xmm0
-; AVX1-NEXT:    vpand %xmm4, %xmm0, %xmm0
+; AVX1-NEXT:    vpand %xmm3, %xmm0, %xmm0
 ; AVX1-NEXT:    vpor %xmm2, %xmm0, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
 ; AVX1-NEXT:    retq
diff --git a/test/CodeGen/X86/vector-fshl-rot-512.ll b/test/CodeGen/X86/vector-fshl-rot-512.ll
index 3510a22e13f23accd77c3326ffbc6eed863118ab..184e98355cdf6e2a1d0d009981da3e296b2d5e33 100644
--- a/test/CodeGen/X86/vector-fshl-rot-512.ll
+++ b/test/CodeGen/X86/vector-fshl-rot-512.ll
@@ -16,14 +16,7 @@ declare <64 x i8> @llvm.fshl.v64i8(<64 x i8>, <64 x i8>, <64 x i8>)
 define <8 x i64> @var_funnnel_v8i64(<8 x i64> %x, <8 x i64> %amt) nounwind {
 ; AVX512-LABEL: var_funnnel_v8i64:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpbroadcastq {{.*#+}} zmm2 = [63,63,63,63,63,63,63,63]
-; AVX512-NEXT:    vpandq %zmm2, %zmm1, %zmm3
-; AVX512-NEXT:    vpsllvq %zmm3, %zmm0, %zmm3
-; AVX512-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX512-NEXT:    vpsubq %zmm1, %zmm4, %zmm1
-; AVX512-NEXT:    vpandq %zmm2, %zmm1, %zmm1
-; AVX512-NEXT:    vpsrlvq %zmm1, %zmm0, %zmm0
-; AVX512-NEXT:    vporq %zmm0, %zmm3, %zmm0
+; AVX512-NEXT:    vprolvq %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    retq
   %res = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %x, <8 x i64> %x, <8 x i64> %amt)
   ret <8 x i64> %res
@@ -32,14 +25,7 @@ define <8 x i64> @var_funnnel_v8i64(<8 x i64> %x, <8 x i64> %amt) nounwind {
 define <16 x i32> @var_funnnel_v16i32(<16 x i32> %x, <16 x i32> %amt) nounwind {
 ; AVX512-LABEL: var_funnnel_v16i32:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpbroadcastd {{.*#+}} zmm2 = [31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31]
-; AVX512-NEXT:    vpandd %zmm2, %zmm1, %zmm3
-; AVX512-NEXT:    vpsllvd %zmm3, %zmm0, %zmm3
-; AVX512-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX512-NEXT:    vpsubd %zmm1, %zmm4, %zmm1
-; AVX512-NEXT:    vpandd %zmm2, %zmm1, %zmm1
-; AVX512-NEXT:    vpsrlvd %zmm1, %zmm0, %zmm0
-; AVX512-NEXT:    vpord %zmm0, %zmm3, %zmm0
+; AVX512-NEXT:    vprolvd %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    retq
   %res = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %x, <16 x i32> %x, <16 x i32> %amt)
   ret <16 x i32> %res
@@ -125,49 +111,47 @@ define <64 x i8> @var_funnnel_v64i8(<64 x i8> %x, <64 x i8> %amt) nounwind {
 ; AVX512F-LABEL: var_funnnel_v64i8:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vpsrlw $4, %ymm0, %ymm4
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm5 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; AVX512F-NEXT:    vpand %ymm5, %ymm4, %ymm4
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm5 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
+; AVX512F-NEXT:    vpandn %ymm4, %ymm5, %ymm4
 ; AVX512F-NEXT:    vpsllw $4, %ymm0, %ymm6
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm7 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
-; AVX512F-NEXT:    vpand %ymm7, %ymm6, %ymm6
+; AVX512F-NEXT:    vpand %ymm5, %ymm6, %ymm6
 ; AVX512F-NEXT:    vpor %ymm4, %ymm6, %ymm4
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm6 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
 ; AVX512F-NEXT:    vpand %ymm6, %ymm2, %ymm2
 ; AVX512F-NEXT:    vpsllw $5, %ymm2, %ymm2
 ; AVX512F-NEXT:    vpblendvb %ymm2, %ymm4, %ymm0, %ymm0
 ; AVX512F-NEXT:    vpsrlw $6, %ymm0, %ymm4
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm8 = [3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3]
-; AVX512F-NEXT:    vpand %ymm8, %ymm4, %ymm4
-; AVX512F-NEXT:    vpsllw $2, %ymm0, %ymm9
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm10 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252]
-; AVX512F-NEXT:    vpand %ymm10, %ymm9, %ymm9
-; AVX512F-NEXT:    vpor %ymm4, %ymm9, %ymm4
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm7 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252]
+; AVX512F-NEXT:    vpandn %ymm4, %ymm7, %ymm4
+; AVX512F-NEXT:    vpsllw $2, %ymm0, %ymm8
+; AVX512F-NEXT:    vpand %ymm7, %ymm8, %ymm8
+; AVX512F-NEXT:    vpor %ymm4, %ymm8, %ymm4
 ; AVX512F-NEXT:    vpaddb %ymm2, %ymm2, %ymm2
 ; AVX512F-NEXT:    vpblendvb %ymm2, %ymm4, %ymm0, %ymm0
 ; AVX512F-NEXT:    vpsrlw $7, %ymm0, %ymm4
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm9 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
-; AVX512F-NEXT:    vpand %ymm9, %ymm4, %ymm4
-; AVX512F-NEXT:    vpaddb %ymm0, %ymm0, %ymm11
-; AVX512F-NEXT:    vpor %ymm4, %ymm11, %ymm4
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm8 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
+; AVX512F-NEXT:    vpand %ymm8, %ymm4, %ymm4
+; AVX512F-NEXT:    vpaddb %ymm0, %ymm0, %ymm9
+; AVX512F-NEXT:    vpor %ymm4, %ymm9, %ymm4
 ; AVX512F-NEXT:    vpaddb %ymm2, %ymm2, %ymm2
 ; AVX512F-NEXT:    vpblendvb %ymm2, %ymm4, %ymm0, %ymm0
 ; AVX512F-NEXT:    vpsrlw $4, %ymm1, %ymm2
-; AVX512F-NEXT:    vpand %ymm5, %ymm2, %ymm2
+; AVX512F-NEXT:    vpandn %ymm2, %ymm5, %ymm2
 ; AVX512F-NEXT:    vpsllw $4, %ymm1, %ymm4
-; AVX512F-NEXT:    vpand %ymm7, %ymm4, %ymm4
+; AVX512F-NEXT:    vpand %ymm5, %ymm4, %ymm4
 ; AVX512F-NEXT:    vpor %ymm2, %ymm4, %ymm2
 ; AVX512F-NEXT:    vpand %ymm6, %ymm3, %ymm3
 ; AVX512F-NEXT:    vpsllw $5, %ymm3, %ymm3
 ; AVX512F-NEXT:    vpblendvb %ymm3, %ymm2, %ymm1, %ymm1
 ; AVX512F-NEXT:    vpsrlw $6, %ymm1, %ymm2
-; AVX512F-NEXT:    vpand %ymm8, %ymm2, %ymm2
+; AVX512F-NEXT:    vpandn %ymm2, %ymm7, %ymm2
 ; AVX512F-NEXT:    vpsllw $2, %ymm1, %ymm4
-; AVX512F-NEXT:    vpand %ymm10, %ymm4, %ymm4
+; AVX512F-NEXT:    vpand %ymm7, %ymm4, %ymm4
 ; AVX512F-NEXT:    vpor %ymm2, %ymm4, %ymm2
 ; AVX512F-NEXT:    vpaddb %ymm3, %ymm3, %ymm3
 ; AVX512F-NEXT:    vpblendvb %ymm3, %ymm2, %ymm1, %ymm1
 ; AVX512F-NEXT:    vpsrlw $7, %ymm1, %ymm2
-; AVX512F-NEXT:    vpand %ymm9, %ymm2, %ymm2
+; AVX512F-NEXT:    vpand %ymm8, %ymm2, %ymm2
 ; AVX512F-NEXT:    vpaddb %ymm1, %ymm1, %ymm4
 ; AVX512F-NEXT:    vpor %ymm2, %ymm4, %ymm2
 ; AVX512F-NEXT:    vpaddb %ymm3, %ymm3, %ymm3
@@ -177,49 +161,47 @@ define <64 x i8> @var_funnnel_v64i8(<64 x i8> %x, <64 x i8> %amt) nounwind {
 ; AVX512VL-LABEL: var_funnnel_v64i8:
 ; AVX512VL:       # %bb.0:
 ; AVX512VL-NEXT:    vpsrlw $4, %ymm0, %ymm4
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm5 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; AVX512VL-NEXT:    vpand %ymm5, %ymm4, %ymm4
+; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm5 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
+; AVX512VL-NEXT:    vpandn %ymm4, %ymm5, %ymm4
 ; AVX512VL-NEXT:    vpsllw $4, %ymm0, %ymm6
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm7 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
-; AVX512VL-NEXT:    vpand %ymm7, %ymm6, %ymm6
+; AVX512VL-NEXT:    vpand %ymm5, %ymm6, %ymm6
 ; AVX512VL-NEXT:    vpor %ymm4, %ymm6, %ymm4
 ; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm6 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
 ; AVX512VL-NEXT:    vpand %ymm6, %ymm2, %ymm2
 ; AVX512VL-NEXT:    vpsllw $5, %ymm2, %ymm2
 ; AVX512VL-NEXT:    vpblendvb %ymm2, %ymm4, %ymm0, %ymm0
 ; AVX512VL-NEXT:    vpsrlw $6, %ymm0, %ymm4
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm8 = [3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3]
-; AVX512VL-NEXT:    vpand %ymm8, %ymm4, %ymm4
-; AVX512VL-NEXT:    vpsllw $2, %ymm0, %ymm9
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm10 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252]
-; AVX512VL-NEXT:    vpand %ymm10, %ymm9, %ymm9
-; AVX512VL-NEXT:    vpor %ymm4, %ymm9, %ymm4
+; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm7 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252]
+; AVX512VL-NEXT:    vpandn %ymm4, %ymm7, %ymm4
+; AVX512VL-NEXT:    vpsllw $2, %ymm0, %ymm8
+; AVX512VL-NEXT:    vpand %ymm7, %ymm8, %ymm8
+; AVX512VL-NEXT:    vpor %ymm4, %ymm8, %ymm4
 ; AVX512VL-NEXT:    vpaddb %ymm2, %ymm2, %ymm2
 ; AVX512VL-NEXT:    vpblendvb %ymm2, %ymm4, %ymm0, %ymm0
 ; AVX512VL-NEXT:    vpsrlw $7, %ymm0, %ymm4
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm9 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
-; AVX512VL-NEXT:    vpand %ymm9, %ymm4, %ymm4
-; AVX512VL-NEXT:    vpaddb %ymm0, %ymm0, %ymm11
-; AVX512VL-NEXT:    vpor %ymm4, %ymm11, %ymm4
+; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm8 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
+; AVX512VL-NEXT:    vpand %ymm8, %ymm4, %ymm4
+; AVX512VL-NEXT:    vpaddb %ymm0, %ymm0, %ymm9
+; AVX512VL-NEXT:    vpor %ymm4, %ymm9, %ymm4
 ; AVX512VL-NEXT:    vpaddb %ymm2, %ymm2, %ymm2
 ; AVX512VL-NEXT:    vpblendvb %ymm2, %ymm4, %ymm0, %ymm0
 ; AVX512VL-NEXT:    vpsrlw $4, %ymm1, %ymm2
-; AVX512VL-NEXT:    vpand %ymm5, %ymm2, %ymm2
+; AVX512VL-NEXT:    vpandn %ymm2, %ymm5, %ymm2
 ; AVX512VL-NEXT:    vpsllw $4, %ymm1, %ymm4
-; AVX512VL-NEXT:    vpand %ymm7, %ymm4, %ymm4
+; AVX512VL-NEXT:    vpand %ymm5, %ymm4, %ymm4
 ; AVX512VL-NEXT:    vpor %ymm2, %ymm4, %ymm2
 ; AVX512VL-NEXT:    vpand %ymm6, %ymm3, %ymm3
 ; AVX512VL-NEXT:    vpsllw $5, %ymm3, %ymm3
 ; AVX512VL-NEXT:    vpblendvb %ymm3, %ymm2, %ymm1, %ymm1
 ; AVX512VL-NEXT:    vpsrlw $6, %ymm1, %ymm2
-; AVX512VL-NEXT:    vpand %ymm8, %ymm2, %ymm2
+; AVX512VL-NEXT:    vpandn %ymm2, %ymm7, %ymm2
 ; AVX512VL-NEXT:    vpsllw $2, %ymm1, %ymm4
-; AVX512VL-NEXT:    vpand %ymm10, %ymm4, %ymm4
+; AVX512VL-NEXT:    vpand %ymm7, %ymm4, %ymm4
 ; AVX512VL-NEXT:    vpor %ymm2, %ymm4, %ymm2
 ; AVX512VL-NEXT:    vpaddb %ymm3, %ymm3, %ymm3
 ; AVX512VL-NEXT:    vpblendvb %ymm3, %ymm2, %ymm1, %ymm1
 ; AVX512VL-NEXT:    vpsrlw $7, %ymm1, %ymm2
-; AVX512VL-NEXT:    vpand %ymm9, %ymm2, %ymm2
+; AVX512VL-NEXT:    vpand %ymm8, %ymm2, %ymm2
 ; AVX512VL-NEXT:    vpaddb %ymm1, %ymm1, %ymm4
 ; AVX512VL-NEXT:    vpor %ymm2, %ymm4, %ymm2
 ; AVX512VL-NEXT:    vpaddb %ymm3, %ymm3, %ymm3
@@ -313,14 +295,7 @@ define <8 x i64> @splatvar_funnnel_v8i64(<8 x i64> %x, <8 x i64> %amt) nounwind
 ; AVX512-LABEL: splatvar_funnnel_v8i64:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpbroadcastq %xmm1, %zmm1
-; AVX512-NEXT:    vmovdqa {{.*#+}} xmm2 = [63,63]
-; AVX512-NEXT:    vpand %xmm2, %xmm1, %xmm3
-; AVX512-NEXT:    vpsllq %xmm3, %zmm0, %zmm3
-; AVX512-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX512-NEXT:    vpsubq %xmm1, %xmm4, %xmm1
-; AVX512-NEXT:    vpand %xmm2, %xmm1, %xmm1
-; AVX512-NEXT:    vpsrlq %xmm1, %zmm0, %zmm0
-; AVX512-NEXT:    vporq %zmm0, %zmm3, %zmm0
+; AVX512-NEXT:    vprolvq %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    retq
   %splat = shufflevector <8 x i64> %amt, <8 x i64> undef, <8 x i32> zeroinitializer
   %res = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %x, <8 x i64> %x, <8 x i64> %splat)
@@ -331,16 +306,7 @@ define <16 x i32> @splatvar_funnnel_v16i32(<16 x i32> %x, <16 x i32> %amt) nounw
 ; AVX512-LABEL: splatvar_funnnel_v16i32:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpbroadcastd %xmm1, %zmm1
-; AVX512-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [31,31,31,31]
-; AVX512-NEXT:    vpand %xmm2, %xmm1, %xmm3
-; AVX512-NEXT:    vpmovzxdq {{.*#+}} xmm3 = xmm3[0],zero,xmm3[1],zero
-; AVX512-NEXT:    vpslld %xmm3, %zmm0, %zmm3
-; AVX512-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX512-NEXT:    vpsubd %xmm1, %xmm4, %xmm1
-; AVX512-NEXT:    vpand %xmm2, %xmm1, %xmm1
-; AVX512-NEXT:    vpmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero
-; AVX512-NEXT:    vpsrld %xmm1, %zmm0, %zmm0
-; AVX512-NEXT:    vpord %zmm0, %zmm3, %zmm0
+; AVX512-NEXT:    vprolvd %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    retq
   %splat = shufflevector <16 x i32> %amt, <16 x i32> undef, <16 x i32> zeroinitializer
   %res = call <16 x i32> @llvm.fshl.v16i32(<16 x i32> %x, <16 x i32> %x, <16 x i32> %splat)
@@ -806,32 +772,30 @@ define <64 x i8> @splatconstant_funnnel_v64i8(<64 x i8> %x) nounwind {
 ; AVX512F-LABEL: splatconstant_funnnel_v64i8:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vpsrlw $4, %ymm0, %ymm2
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; AVX512F-NEXT:    vpand %ymm3, %ymm2, %ymm2
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm3 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
+; AVX512F-NEXT:    vpandn %ymm2, %ymm3, %ymm2
 ; AVX512F-NEXT:    vpsllw $4, %ymm0, %ymm0
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm4 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
-; AVX512F-NEXT:    vpand %ymm4, %ymm0, %ymm0
+; AVX512F-NEXT:    vpand %ymm3, %ymm0, %ymm0
 ; AVX512F-NEXT:    vpor %ymm2, %ymm0, %ymm0
 ; AVX512F-NEXT:    vpsrlw $4, %ymm1, %ymm2
-; AVX512F-NEXT:    vpand %ymm3, %ymm2, %ymm2
+; AVX512F-NEXT:    vpandn %ymm2, %ymm3, %ymm2
 ; AVX512F-NEXT:    vpsllw $4, %ymm1, %ymm1
-; AVX512F-NEXT:    vpand %ymm4, %ymm1, %ymm1
+; AVX512F-NEXT:    vpand %ymm3, %ymm1, %ymm1
 ; AVX512F-NEXT:    vpor %ymm2, %ymm1, %ymm1
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: splatconstant_funnnel_v64i8:
 ; AVX512VL:       # %bb.0:
 ; AVX512VL-NEXT:    vpsrlw $4, %ymm0, %ymm2
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; AVX512VL-NEXT:    vpand %ymm3, %ymm2, %ymm2
+; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm3 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
+; AVX512VL-NEXT:    vpandn %ymm2, %ymm3, %ymm2
 ; AVX512VL-NEXT:    vpsllw $4, %ymm0, %ymm0
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm4 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
-; AVX512VL-NEXT:    vpand %ymm4, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpand %ymm3, %ymm0, %ymm0
 ; AVX512VL-NEXT:    vpor %ymm2, %ymm0, %ymm0
 ; AVX512VL-NEXT:    vpsrlw $4, %ymm1, %ymm2
-; AVX512VL-NEXT:    vpand %ymm3, %ymm2, %ymm2
+; AVX512VL-NEXT:    vpandn %ymm2, %ymm3, %ymm2
 ; AVX512VL-NEXT:    vpsllw $4, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpand %ymm4, %ymm1, %ymm1
+; AVX512VL-NEXT:    vpand %ymm3, %ymm1, %ymm1
 ; AVX512VL-NEXT:    vpor %ymm2, %ymm1, %ymm1
 ; AVX512VL-NEXT:    retq
 ;
diff --git a/test/CodeGen/X86/vector-fshr-128.ll b/test/CodeGen/X86/vector-fshr-128.ll
index 616a3780e346f031affdd11cf2430c18b2f1081d..1f70fc95277d4a3a91f8a262a5957a21ab55262b 100644
--- a/test/CodeGen/X86/vector-fshr-128.ll
+++ b/test/CodeGen/X86/vector-fshr-128.ll
@@ -6,7 +6,9 @@
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f | FileCheck %s --check-prefixes=AVX512,AVX512F
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512vl | FileCheck %s --check-prefixes=AVX512,AVX512VL
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512bw | FileCheck %s --check-prefixes=AVX512,AVX512BW
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512vbmi2 | FileCheck %s --check-prefixes=AVX512,AVX512VBMI2
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512bw,+avx512vl | FileCheck %s --check-prefixes=AVX512,AVX512VLBW
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512vbmi2,+avx512vl | FileCheck %s --check-prefixes=AVX512,AVX512VLVBMI2
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+xop,+avx | FileCheck %s --check-prefixes=XOP,XOPAVX1
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+xop,+avx2 | FileCheck %s --check-prefixes=XOP,XOPAVX2
 
@@ -153,6 +155,23 @@ define <2 x i64> @var_funnnel_v2i64(<2 x i64> %x, <2 x i64> %y, <2 x i64> %amt)
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: var_funnnel_v2i64:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $xmm2 killed $xmm2 def $zmm2
+; AVX512VBMI2-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} xmm3 = [63,63]
+; AVX512VBMI2-NEXT:    vpand %xmm3, %xmm2, %xmm4
+; AVX512VBMI2-NEXT:    vpsrlvq %xmm4, %xmm1, %xmm5
+; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} xmm6 = [64,64]
+; AVX512VBMI2-NEXT:    vpsubq %xmm4, %xmm6, %xmm4
+; AVX512VBMI2-NEXT:    vpsllvq %xmm4, %xmm0, %xmm0
+; AVX512VBMI2-NEXT:    vpor %xmm5, %xmm0, %xmm0
+; AVX512VBMI2-NEXT:    vptestnmq %zmm3, %zmm2, %k1
+; AVX512VBMI2-NEXT:    vmovdqa64 %zmm1, %zmm0 {%k1}
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512VBMI2-NEXT:    vzeroupper
+; AVX512VBMI2-NEXT:    retq
+;
 ; AVX512VLBW-LABEL: var_funnnel_v2i64:
 ; AVX512VLBW:       # %bb.0:
 ; AVX512VLBW-NEXT:    vmovdqa {{.*#+}} xmm3 = [63,63]
@@ -166,6 +185,12 @@ define <2 x i64> @var_funnnel_v2i64(<2 x i64> %x, <2 x i64> %y, <2 x i64> %amt)
 ; AVX512VLBW-NEXT:    vmovdqa64 %xmm1, %xmm0 {%k1}
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VLVBMI2-LABEL: var_funnnel_v2i64:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpshrdvq %xmm2, %xmm0, %xmm1
+; AVX512VLVBMI2-NEXT:    vmovdqa %xmm1, %xmm0
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOPAVX1-LABEL: var_funnnel_v2i64:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
@@ -383,6 +408,23 @@ define <4 x i32> @var_funnnel_v4i32(<4 x i32> %x, <4 x i32> %y, <4 x i32> %amt)
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: var_funnnel_v4i32:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $xmm2 killed $xmm2 def $zmm2
+; AVX512VBMI2-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512VBMI2-NEXT:    vpbroadcastd {{.*#+}} xmm3 = [31,31,31,31]
+; AVX512VBMI2-NEXT:    vpand %xmm3, %xmm2, %xmm4
+; AVX512VBMI2-NEXT:    vpsrlvd %xmm4, %xmm1, %xmm5
+; AVX512VBMI2-NEXT:    vpbroadcastd {{.*#+}} xmm6 = [32,32,32,32]
+; AVX512VBMI2-NEXT:    vpsubd %xmm4, %xmm6, %xmm4
+; AVX512VBMI2-NEXT:    vpsllvd %xmm4, %xmm0, %xmm0
+; AVX512VBMI2-NEXT:    vpor %xmm5, %xmm0, %xmm0
+; AVX512VBMI2-NEXT:    vptestnmd %zmm3, %zmm2, %k1
+; AVX512VBMI2-NEXT:    vmovdqa32 %zmm1, %zmm0 {%k1}
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512VBMI2-NEXT:    vzeroupper
+; AVX512VBMI2-NEXT:    retq
+;
 ; AVX512VLBW-LABEL: var_funnnel_v4i32:
 ; AVX512VLBW:       # %bb.0:
 ; AVX512VLBW-NEXT:    vpbroadcastd {{.*#+}} xmm3 = [31,31,31,31]
@@ -396,6 +438,12 @@ define <4 x i32> @var_funnnel_v4i32(<4 x i32> %x, <4 x i32> %y, <4 x i32> %amt)
 ; AVX512VLBW-NEXT:    vmovdqa32 %xmm1, %xmm0 {%k1}
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VLVBMI2-LABEL: var_funnnel_v4i32:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpshrdvd %xmm2, %xmm0, %xmm1
+; AVX512VLVBMI2-NEXT:    vmovdqa %xmm1, %xmm0
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOPAVX1-LABEL: var_funnnel_v4i32:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
@@ -695,6 +743,24 @@ define <8 x i16> @var_funnnel_v8i16(<8 x i16> %x, <8 x i16> %y, <8 x i16> %amt)
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: var_funnnel_v8i16:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $xmm2 killed $xmm2 def $zmm2
+; AVX512VBMI2-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} xmm3 = [15,15,15,15,15,15,15,15]
+; AVX512VBMI2-NEXT:    vpand %xmm3, %xmm2, %xmm4
+; AVX512VBMI2-NEXT:    vpsrlvw %zmm4, %zmm1, %zmm5
+; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} xmm6 = [16,16,16,16,16,16,16,16]
+; AVX512VBMI2-NEXT:    vpsubw %xmm4, %xmm6, %xmm4
+; AVX512VBMI2-NEXT:    vpsllvw %zmm4, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    vpor %xmm5, %xmm0, %xmm0
+; AVX512VBMI2-NEXT:    vptestnmw %zmm3, %zmm2, %k1
+; AVX512VBMI2-NEXT:    vmovdqu16 %zmm1, %zmm0 {%k1}
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512VBMI2-NEXT:    vzeroupper
+; AVX512VBMI2-NEXT:    retq
+;
 ; AVX512VLBW-LABEL: var_funnnel_v8i16:
 ; AVX512VLBW:       # %bb.0:
 ; AVX512VLBW-NEXT:    vmovdqa {{.*#+}} xmm3 = [15,15,15,15,15,15,15,15]
@@ -708,6 +774,12 @@ define <8 x i16> @var_funnnel_v8i16(<8 x i16> %x, <8 x i16> %y, <8 x i16> %amt)
 ; AVX512VLBW-NEXT:    vmovdqu16 %xmm1, %xmm0 {%k1}
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VLVBMI2-LABEL: var_funnnel_v8i16:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpshrdvw %xmm2, %xmm0, %xmm1
+; AVX512VLVBMI2-NEXT:    vmovdqa %xmm1, %xmm0
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOP-LABEL: var_funnnel_v8i16:
 ; XOP:       # %bb.0:
 ; XOP-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
@@ -1004,6 +1076,28 @@ define <16 x i8> @var_funnnel_v16i8(<16 x i8> %x, <16 x i8> %y, <16 x i8> %amt)
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: var_funnnel_v16i8:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $xmm2 killed $xmm2 def $zmm2
+; AVX512VBMI2-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512VBMI2-NEXT:    vpmovzxbw {{.*#+}} ymm3 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
+; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} xmm4 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
+; AVX512VBMI2-NEXT:    vpand %xmm4, %xmm2, %xmm5
+; AVX512VBMI2-NEXT:    vpmovzxbw {{.*#+}} ymm6 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero,xmm5[8],zero,xmm5[9],zero,xmm5[10],zero,xmm5[11],zero,xmm5[12],zero,xmm5[13],zero,xmm5[14],zero,xmm5[15],zero
+; AVX512VBMI2-NEXT:    vpsrlvw %zmm6, %zmm3, %zmm3
+; AVX512VBMI2-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
+; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} xmm6 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; AVX512VBMI2-NEXT:    vpsubb %xmm5, %xmm6, %xmm5
+; AVX512VBMI2-NEXT:    vpmovzxbw {{.*#+}} ymm5 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero,xmm5[8],zero,xmm5[9],zero,xmm5[10],zero,xmm5[11],zero,xmm5[12],zero,xmm5[13],zero,xmm5[14],zero,xmm5[15],zero
+; AVX512VBMI2-NEXT:    vpsllvw %zmm5, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    vpor %ymm3, %ymm0, %ymm0
+; AVX512VBMI2-NEXT:    vpmovwb %zmm0, %ymm0
+; AVX512VBMI2-NEXT:    vptestnmb %zmm4, %zmm2, %k1
+; AVX512VBMI2-NEXT:    vmovdqu8 %zmm1, %zmm0 {%k1}
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512VBMI2-NEXT:    vzeroupper
+; AVX512VBMI2-NEXT:    retq
+;
 ; AVX512VLBW-LABEL: var_funnnel_v16i8:
 ; AVX512VLBW:       # %bb.0:
 ; AVX512VLBW-NEXT:    vmovdqa {{.*#+}} xmm3 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
@@ -1023,6 +1117,25 @@ define <16 x i8> @var_funnnel_v16i8(<16 x i8> %x, <16 x i8> %y, <16 x i8> %amt)
 ; AVX512VLBW-NEXT:    vzeroupper
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VLVBMI2-LABEL: var_funnnel_v16i8:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vmovdqa {{.*#+}} xmm3 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
+; AVX512VLVBMI2-NEXT:    vpand %xmm3, %xmm2, %xmm4
+; AVX512VLVBMI2-NEXT:    vpmovzxbw {{.*#+}} ymm5 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero,xmm4[8],zero,xmm4[9],zero,xmm4[10],zero,xmm4[11],zero,xmm4[12],zero,xmm4[13],zero,xmm4[14],zero,xmm4[15],zero
+; AVX512VLVBMI2-NEXT:    vpmovzxbw {{.*#+}} ymm6 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
+; AVX512VLVBMI2-NEXT:    vpsrlvw %ymm5, %ymm6, %ymm5
+; AVX512VLVBMI2-NEXT:    vmovdqa {{.*#+}} xmm6 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; AVX512VLVBMI2-NEXT:    vpsubb %xmm4, %xmm6, %xmm4
+; AVX512VLVBMI2-NEXT:    vpmovzxbw {{.*#+}} ymm4 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero,xmm4[8],zero,xmm4[9],zero,xmm4[10],zero,xmm4[11],zero,xmm4[12],zero,xmm4[13],zero,xmm4[14],zero,xmm4[15],zero
+; AVX512VLVBMI2-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
+; AVX512VLVBMI2-NEXT:    vpsllvw %ymm4, %ymm0, %ymm0
+; AVX512VLVBMI2-NEXT:    vpor %ymm5, %ymm0, %ymm0
+; AVX512VLVBMI2-NEXT:    vpmovwb %ymm0, %xmm0
+; AVX512VLVBMI2-NEXT:    vptestnmb %xmm3, %xmm2, %k1
+; AVX512VLVBMI2-NEXT:    vmovdqu8 %xmm1, %xmm0 {%k1}
+; AVX512VLVBMI2-NEXT:    vzeroupper
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOP-LABEL: var_funnnel_v16i8:
 ; XOP:       # %bb.0:
 ; XOP-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
@@ -1225,6 +1338,23 @@ define <2 x i64> @splatvar_funnnel_v2i64(<2 x i64> %x, <2 x i64> %y, <2 x i64> %
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: splatvar_funnnel_v2i64:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512VBMI2-NEXT:    vpbroadcastq %xmm2, %xmm2
+; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} xmm3 = [63,63]
+; AVX512VBMI2-NEXT:    vpand %xmm3, %xmm2, %xmm4
+; AVX512VBMI2-NEXT:    vpsrlq %xmm4, %xmm1, %xmm5
+; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} xmm6 = [64,64]
+; AVX512VBMI2-NEXT:    vpsubq %xmm4, %xmm6, %xmm4
+; AVX512VBMI2-NEXT:    vpsllq %xmm4, %xmm0, %xmm0
+; AVX512VBMI2-NEXT:    vpor %xmm5, %xmm0, %xmm0
+; AVX512VBMI2-NEXT:    vptestnmq %zmm3, %zmm2, %k1
+; AVX512VBMI2-NEXT:    vmovdqa64 %zmm1, %zmm0 {%k1}
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512VBMI2-NEXT:    vzeroupper
+; AVX512VBMI2-NEXT:    retq
+;
 ; AVX512VLBW-LABEL: splatvar_funnnel_v2i64:
 ; AVX512VLBW:       # %bb.0:
 ; AVX512VLBW-NEXT:    vpbroadcastq %xmm2, %xmm2
@@ -1239,6 +1369,13 @@ define <2 x i64> @splatvar_funnnel_v2i64(<2 x i64> %x, <2 x i64> %y, <2 x i64> %
 ; AVX512VLBW-NEXT:    vmovdqa64 %xmm1, %xmm0 {%k1}
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VLVBMI2-LABEL: splatvar_funnnel_v2i64:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpbroadcastq %xmm2, %xmm2
+; AVX512VLVBMI2-NEXT:    vpshrdvq %xmm2, %xmm0, %xmm1
+; AVX512VLVBMI2-NEXT:    vmovdqa %xmm1, %xmm0
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOPAVX1-LABEL: splatvar_funnnel_v2i64:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[0,1,0,1]
@@ -1309,12 +1446,13 @@ define <4 x i32> @splatvar_funnnel_v4i32(<4 x i32> %x, <4 x i32> %y, <4 x i32> %
 ; SSE2-NEXT:    movss {{.*#+}} xmm4 = xmm2[0],xmm4[1,2,3]
 ; SSE2-NEXT:    movdqa %xmm1, %xmm5
 ; SSE2-NEXT:    psrld %xmm4, %xmm5
-; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [32,32,32,32]
-; SSE2-NEXT:    psubd %xmm2, %xmm4
-; SSE2-NEXT:    pcmpeqd %xmm3, %xmm2
-; SSE2-NEXT:    movss {{.*#+}} xmm3 = xmm4[0],xmm3[1,2,3]
-; SSE2-NEXT:    pslld %xmm3, %xmm0
+; SSE2-NEXT:    movd %xmm2, %eax
+; SSE2-NEXT:    movl $32, %ecx
+; SSE2-NEXT:    subl %eax, %ecx
+; SSE2-NEXT:    movd %ecx, %xmm4
+; SSE2-NEXT:    pslld %xmm4, %xmm0
 ; SSE2-NEXT:    por %xmm5, %xmm0
+; SSE2-NEXT:    pcmpeqd %xmm3, %xmm2
 ; SSE2-NEXT:    pand %xmm2, %xmm1
 ; SSE2-NEXT:    pandn %xmm0, %xmm2
 ; SSE2-NEXT:    por %xmm1, %xmm2
@@ -1427,6 +1565,25 @@ define <4 x i32> @splatvar_funnnel_v4i32(<4 x i32> %x, <4 x i32> %y, <4 x i32> %
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: splatvar_funnnel_v4i32:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512VBMI2-NEXT:    vpbroadcastd %xmm2, %xmm2
+; AVX512VBMI2-NEXT:    vpbroadcastd {{.*#+}} xmm3 = [31,31,31,31]
+; AVX512VBMI2-NEXT:    vpand %xmm3, %xmm2, %xmm4
+; AVX512VBMI2-NEXT:    vpmovzxdq {{.*#+}} xmm5 = xmm4[0],zero,xmm4[1],zero
+; AVX512VBMI2-NEXT:    vpsrld %xmm5, %xmm1, %xmm5
+; AVX512VBMI2-NEXT:    vpbroadcastd {{.*#+}} xmm6 = [32,32,32,32]
+; AVX512VBMI2-NEXT:    vpsubd %xmm4, %xmm6, %xmm4
+; AVX512VBMI2-NEXT:    vpmovzxdq {{.*#+}} xmm4 = xmm4[0],zero,xmm4[1],zero
+; AVX512VBMI2-NEXT:    vpslld %xmm4, %xmm0, %xmm0
+; AVX512VBMI2-NEXT:    vpor %xmm5, %xmm0, %xmm0
+; AVX512VBMI2-NEXT:    vptestnmd %zmm3, %zmm2, %k1
+; AVX512VBMI2-NEXT:    vmovdqa32 %zmm1, %zmm0 {%k1}
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512VBMI2-NEXT:    vzeroupper
+; AVX512VBMI2-NEXT:    retq
+;
 ; AVX512VLBW-LABEL: splatvar_funnnel_v4i32:
 ; AVX512VLBW:       # %bb.0:
 ; AVX512VLBW-NEXT:    vpbroadcastd %xmm2, %xmm2
@@ -1443,6 +1600,13 @@ define <4 x i32> @splatvar_funnnel_v4i32(<4 x i32> %x, <4 x i32> %y, <4 x i32> %
 ; AVX512VLBW-NEXT:    vmovdqa32 %xmm1, %xmm0 {%k1}
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VLVBMI2-LABEL: splatvar_funnnel_v4i32:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpbroadcastd %xmm2, %xmm2
+; AVX512VLVBMI2-NEXT:    vpshrdvd %xmm2, %xmm0, %xmm1
+; AVX512VLVBMI2-NEXT:    vmovdqa %xmm1, %xmm0
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOPAVX1-LABEL: splatvar_funnnel_v4i32:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[0,0,0,0]
@@ -1485,12 +1649,13 @@ define <4 x i32> @splatvar_funnnel_v4i32(<4 x i32> %x, <4 x i32> %y, <4 x i32> %
 ; X32-SSE-NEXT:    movss {{.*#+}} xmm4 = xmm2[0],xmm4[1,2,3]
 ; X32-SSE-NEXT:    movdqa %xmm1, %xmm5
 ; X32-SSE-NEXT:    psrld %xmm4, %xmm5
-; X32-SSE-NEXT:    movdqa {{.*#+}} xmm4 = [32,32,32,32]
-; X32-SSE-NEXT:    psubd %xmm2, %xmm4
-; X32-SSE-NEXT:    pcmpeqd %xmm3, %xmm2
-; X32-SSE-NEXT:    movss {{.*#+}} xmm3 = xmm4[0],xmm3[1,2,3]
-; X32-SSE-NEXT:    pslld %xmm3, %xmm0
+; X32-SSE-NEXT:    movd %xmm2, %eax
+; X32-SSE-NEXT:    movl $32, %ecx
+; X32-SSE-NEXT:    subl %eax, %ecx
+; X32-SSE-NEXT:    movd %ecx, %xmm4
+; X32-SSE-NEXT:    pslld %xmm4, %xmm0
 ; X32-SSE-NEXT:    por %xmm5, %xmm0
+; X32-SSE-NEXT:    pcmpeqd %xmm3, %xmm2
 ; X32-SSE-NEXT:    pand %xmm2, %xmm1
 ; X32-SSE-NEXT:    pandn %xmm0, %xmm2
 ; X32-SSE-NEXT:    por %xmm1, %xmm2
@@ -1629,6 +1794,25 @@ define <8 x i16> @splatvar_funnnel_v8i16(<8 x i16> %x, <8 x i16> %y, <8 x i16> %
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: splatvar_funnnel_v8i16:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512VBMI2-NEXT:    vpbroadcastw %xmm2, %xmm2
+; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} xmm3 = [15,15,15,15,15,15,15,15]
+; AVX512VBMI2-NEXT:    vpand %xmm3, %xmm2, %xmm4
+; AVX512VBMI2-NEXT:    vpmovzxwq {{.*#+}} xmm5 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero
+; AVX512VBMI2-NEXT:    vpsrlw %xmm5, %xmm1, %xmm5
+; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} xmm6 = [16,16,16,16,16,16,16,16]
+; AVX512VBMI2-NEXT:    vpsubw %xmm4, %xmm6, %xmm4
+; AVX512VBMI2-NEXT:    vpmovzxwq {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero
+; AVX512VBMI2-NEXT:    vpsllw %xmm4, %xmm0, %xmm0
+; AVX512VBMI2-NEXT:    vpor %xmm5, %xmm0, %xmm0
+; AVX512VBMI2-NEXT:    vptestnmw %zmm3, %zmm2, %k1
+; AVX512VBMI2-NEXT:    vmovdqu16 %zmm1, %zmm0 {%k1}
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512VBMI2-NEXT:    vzeroupper
+; AVX512VBMI2-NEXT:    retq
+;
 ; AVX512VLBW-LABEL: splatvar_funnnel_v8i16:
 ; AVX512VLBW:       # %bb.0:
 ; AVX512VLBW-NEXT:    vpbroadcastw %xmm2, %xmm2
@@ -1645,6 +1829,13 @@ define <8 x i16> @splatvar_funnnel_v8i16(<8 x i16> %x, <8 x i16> %y, <8 x i16> %
 ; AVX512VLBW-NEXT:    vmovdqu16 %xmm1, %xmm0 {%k1}
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VLVBMI2-LABEL: splatvar_funnnel_v8i16:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpbroadcastw %xmm2, %xmm2
+; AVX512VLVBMI2-NEXT:    vpshrdvw %xmm2, %xmm0, %xmm1
+; AVX512VLVBMI2-NEXT:    vmovdqa %xmm1, %xmm0
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOPAVX1-LABEL: splatvar_funnnel_v8i16:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm2 = xmm2[0,0,2,3,4,5,6,7]
@@ -1879,6 +2070,28 @@ define <16 x i8> @splatvar_funnnel_v16i8(<16 x i8> %x, <16 x i8> %y, <16 x i8> %
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: splatvar_funnnel_v16i8:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512VBMI2-NEXT:    vpbroadcastb %xmm2, %xmm2
+; AVX512VBMI2-NEXT:    vpmovzxbw {{.*#+}} ymm3 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
+; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} xmm4 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
+; AVX512VBMI2-NEXT:    vpand %xmm4, %xmm2, %xmm5
+; AVX512VBMI2-NEXT:    vpmovzxbw {{.*#+}} ymm6 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero,xmm5[8],zero,xmm5[9],zero,xmm5[10],zero,xmm5[11],zero,xmm5[12],zero,xmm5[13],zero,xmm5[14],zero,xmm5[15],zero
+; AVX512VBMI2-NEXT:    vpsrlvw %zmm6, %zmm3, %zmm3
+; AVX512VBMI2-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
+; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} xmm6 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; AVX512VBMI2-NEXT:    vpsubb %xmm5, %xmm6, %xmm5
+; AVX512VBMI2-NEXT:    vpmovzxbw {{.*#+}} ymm5 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero,xmm5[8],zero,xmm5[9],zero,xmm5[10],zero,xmm5[11],zero,xmm5[12],zero,xmm5[13],zero,xmm5[14],zero,xmm5[15],zero
+; AVX512VBMI2-NEXT:    vpsllvw %zmm5, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    vpor %ymm3, %ymm0, %ymm0
+; AVX512VBMI2-NEXT:    vpmovwb %zmm0, %ymm0
+; AVX512VBMI2-NEXT:    vptestnmb %zmm4, %zmm2, %k1
+; AVX512VBMI2-NEXT:    vmovdqu8 %zmm1, %zmm0 {%k1}
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512VBMI2-NEXT:    vzeroupper
+; AVX512VBMI2-NEXT:    retq
+;
 ; AVX512VLBW-LABEL: splatvar_funnnel_v16i8:
 ; AVX512VLBW:       # %bb.0:
 ; AVX512VLBW-NEXT:    vpbroadcastb %xmm2, %xmm2
@@ -1899,6 +2112,26 @@ define <16 x i8> @splatvar_funnnel_v16i8(<16 x i8> %x, <16 x i8> %y, <16 x i8> %
 ; AVX512VLBW-NEXT:    vzeroupper
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VLVBMI2-LABEL: splatvar_funnnel_v16i8:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpbroadcastb %xmm2, %xmm2
+; AVX512VLVBMI2-NEXT:    vmovdqa {{.*#+}} xmm3 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
+; AVX512VLVBMI2-NEXT:    vpand %xmm3, %xmm2, %xmm4
+; AVX512VLVBMI2-NEXT:    vpmovzxbw {{.*#+}} ymm5 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero,xmm4[8],zero,xmm4[9],zero,xmm4[10],zero,xmm4[11],zero,xmm4[12],zero,xmm4[13],zero,xmm4[14],zero,xmm4[15],zero
+; AVX512VLVBMI2-NEXT:    vpmovzxbw {{.*#+}} ymm6 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
+; AVX512VLVBMI2-NEXT:    vpsrlvw %ymm5, %ymm6, %ymm5
+; AVX512VLVBMI2-NEXT:    vmovdqa {{.*#+}} xmm6 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; AVX512VLVBMI2-NEXT:    vpsubb %xmm4, %xmm6, %xmm4
+; AVX512VLVBMI2-NEXT:    vpmovzxbw {{.*#+}} ymm4 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero,xmm4[8],zero,xmm4[9],zero,xmm4[10],zero,xmm4[11],zero,xmm4[12],zero,xmm4[13],zero,xmm4[14],zero,xmm4[15],zero
+; AVX512VLVBMI2-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
+; AVX512VLVBMI2-NEXT:    vpsllvw %ymm4, %ymm0, %ymm0
+; AVX512VLVBMI2-NEXT:    vpor %ymm5, %ymm0, %ymm0
+; AVX512VLVBMI2-NEXT:    vpmovwb %ymm0, %xmm0
+; AVX512VLVBMI2-NEXT:    vptestnmb %xmm3, %xmm2, %k1
+; AVX512VLVBMI2-NEXT:    vmovdqu8 %xmm1, %xmm0 {%k1}
+; AVX512VLVBMI2-NEXT:    vzeroupper
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOPAVX1-LABEL: splatvar_funnnel_v16i8:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
@@ -2019,12 +2252,46 @@ define <2 x i64> @constant_funnnel_v2i64(<2 x i64> %x, <2 x i64> %y) nounwind {
 ; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    retq
 ;
-; AVX512-LABEL: constant_funnnel_v2i64:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpsrlvq {{.*}}(%rip), %xmm1, %xmm1
-; AVX512-NEXT:    vpsllvq {{.*}}(%rip), %xmm0, %xmm0
-; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    retq
+; AVX512F-LABEL: constant_funnnel_v2i64:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpsrlvq {{.*}}(%rip), %xmm1, %xmm1
+; AVX512F-NEXT:    vpsllvq {{.*}}(%rip), %xmm0, %xmm0
+; AVX512F-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: constant_funnnel_v2i64:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpsrlvq {{.*}}(%rip), %xmm1, %xmm1
+; AVX512VL-NEXT:    vpsllvq {{.*}}(%rip), %xmm0, %xmm0
+; AVX512VL-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: constant_funnnel_v2i64:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpsrlvq {{.*}}(%rip), %xmm1, %xmm1
+; AVX512BW-NEXT:    vpsllvq {{.*}}(%rip), %xmm0, %xmm0
+; AVX512BW-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT:    retq
+;
+; AVX512VBMI2-LABEL: constant_funnnel_v2i64:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    vpsrlvq {{.*}}(%rip), %xmm1, %xmm1
+; AVX512VBMI2-NEXT:    vpsllvq {{.*}}(%rip), %xmm0, %xmm0
+; AVX512VBMI2-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLBW-LABEL: constant_funnnel_v2i64:
+; AVX512VLBW:       # %bb.0:
+; AVX512VLBW-NEXT:    vpsrlvq {{.*}}(%rip), %xmm1, %xmm1
+; AVX512VLBW-NEXT:    vpsllvq {{.*}}(%rip), %xmm0, %xmm0
+; AVX512VLBW-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX512VLBW-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: constant_funnnel_v2i64:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpshrdvq {{.*}}(%rip), %xmm0, %xmm1
+; AVX512VLVBMI2-NEXT:    vmovdqa %xmm1, %xmm0
+; AVX512VLVBMI2-NEXT:    retq
 ;
 ; XOPAVX1-LABEL: constant_funnnel_v2i64:
 ; XOPAVX1:       # %bb.0:
@@ -2116,12 +2383,46 @@ define <4 x i32> @constant_funnnel_v4i32(<4 x i32> %x, <4 x i32> %y) nounwind {
 ; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    retq
 ;
-; AVX512-LABEL: constant_funnnel_v4i32:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpsrlvd {{.*}}(%rip), %xmm1, %xmm1
-; AVX512-NEXT:    vpsllvd {{.*}}(%rip), %xmm0, %xmm0
-; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    retq
+; AVX512F-LABEL: constant_funnnel_v4i32:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpsrlvd {{.*}}(%rip), %xmm1, %xmm1
+; AVX512F-NEXT:    vpsllvd {{.*}}(%rip), %xmm0, %xmm0
+; AVX512F-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: constant_funnnel_v4i32:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpsrlvd {{.*}}(%rip), %xmm1, %xmm1
+; AVX512VL-NEXT:    vpsllvd {{.*}}(%rip), %xmm0, %xmm0
+; AVX512VL-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: constant_funnnel_v4i32:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpsrlvd {{.*}}(%rip), %xmm1, %xmm1
+; AVX512BW-NEXT:    vpsllvd {{.*}}(%rip), %xmm0, %xmm0
+; AVX512BW-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT:    retq
+;
+; AVX512VBMI2-LABEL: constant_funnnel_v4i32:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    vpsrlvd {{.*}}(%rip), %xmm1, %xmm1
+; AVX512VBMI2-NEXT:    vpsllvd {{.*}}(%rip), %xmm0, %xmm0
+; AVX512VBMI2-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLBW-LABEL: constant_funnnel_v4i32:
+; AVX512VLBW:       # %bb.0:
+; AVX512VLBW-NEXT:    vpsrlvd {{.*}}(%rip), %xmm1, %xmm1
+; AVX512VLBW-NEXT:    vpsllvd {{.*}}(%rip), %xmm0, %xmm0
+; AVX512VLBW-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX512VLBW-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: constant_funnnel_v4i32:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpshrdvd {{.*}}(%rip), %xmm0, %xmm1
+; AVX512VLVBMI2-NEXT:    vmovdqa %xmm1, %xmm0
+; AVX512VLVBMI2-NEXT:    retq
 ;
 ; XOPAVX1-LABEL: constant_funnnel_v4i32:
 ; XOPAVX1:       # %bb.0:
@@ -2229,6 +2530,19 @@ define <8 x i16> @constant_funnnel_v8i16(<8 x i16> %x, <8 x i16> %y) nounwind {
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: constant_funnnel_v8i16:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512VBMI2-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} xmm2 = [0,1,2,3,4,5,6,7]
+; AVX512VBMI2-NEXT:    vpsrlvw %zmm2, %zmm1, %zmm2
+; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} xmm3 = [16,15,14,13,12,11,10,9]
+; AVX512VBMI2-NEXT:    vpsllvw %zmm3, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    vpor %xmm2, %xmm0, %xmm0
+; AVX512VBMI2-NEXT:    vpblendw {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3,4,5,6,7]
+; AVX512VBMI2-NEXT:    vzeroupper
+; AVX512VBMI2-NEXT:    retq
+;
 ; AVX512VLBW-LABEL: constant_funnnel_v8i16:
 ; AVX512VLBW:       # %bb.0:
 ; AVX512VLBW-NEXT:    vpsrlvw {{.*}}(%rip), %xmm1, %xmm2
@@ -2237,6 +2551,12 @@ define <8 x i16> @constant_funnnel_v8i16(<8 x i16> %x, <8 x i16> %y) nounwind {
 ; AVX512VLBW-NEXT:    vpblendw {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3,4,5,6,7]
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VLVBMI2-LABEL: constant_funnnel_v8i16:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpshrdvw {{.*}}(%rip), %xmm0, %xmm1
+; AVX512VLVBMI2-NEXT:    vmovdqa %xmm1, %xmm0
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOP-LABEL: constant_funnnel_v8i16:
 ; XOP:       # %bb.0:
 ; XOP-NEXT:    vpshlw {{.*}}(%rip), %xmm1, %xmm2
@@ -2399,6 +2719,21 @@ define <16 x i8> @constant_funnnel_v16i8(<16 x i8> %x, <16 x i8> %y) nounwind {
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: constant_funnnel_v16i8:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} ymm2 = [0,1,2,3,4,5,6,7,0,7,6,5,4,3,2,1]
+; AVX512VBMI2-NEXT:    vpmovzxbw {{.*#+}} ymm3 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
+; AVX512VBMI2-NEXT:    vpsrlvw %zmm2, %zmm3, %zmm2
+; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} ymm3 = [8,7,6,5,4,3,2,1,8,1,2,3,4,5,6,7]
+; AVX512VBMI2-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
+; AVX512VBMI2-NEXT:    vpsllvw %zmm3, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    vpor %ymm2, %ymm0, %ymm0
+; AVX512VBMI2-NEXT:    vpmovwb %zmm0, %ymm0
+; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} xmm2 = [0,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255]
+; AVX512VBMI2-NEXT:    vpblendvb %xmm2, %xmm0, %xmm1, %xmm0
+; AVX512VBMI2-NEXT:    vzeroupper
+; AVX512VBMI2-NEXT:    retq
+;
 ; AVX512VLBW-LABEL: constant_funnnel_v16i8:
 ; AVX512VLBW:       # %bb.0:
 ; AVX512VLBW-NEXT:    vpmovzxbw {{.*#+}} ymm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
@@ -2413,6 +2748,20 @@ define <16 x i8> @constant_funnnel_v16i8(<16 x i8> %x, <16 x i8> %y) nounwind {
 ; AVX512VLBW-NEXT:    vzeroupper
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VLVBMI2-LABEL: constant_funnnel_v16i8:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpmovzxbw {{.*#+}} ymm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
+; AVX512VLVBMI2-NEXT:    vpsrlvw {{.*}}(%rip), %ymm2, %ymm2
+; AVX512VLVBMI2-NEXT:    vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
+; AVX512VLVBMI2-NEXT:    vpsllvw {{.*}}(%rip), %ymm0, %ymm0
+; AVX512VLVBMI2-NEXT:    vpor %ymm2, %ymm0, %ymm0
+; AVX512VLVBMI2-NEXT:    vpmovwb %ymm0, %xmm0
+; AVX512VLVBMI2-NEXT:    movw $257, %ax # imm = 0x101
+; AVX512VLVBMI2-NEXT:    kmovd %eax, %k1
+; AVX512VLVBMI2-NEXT:    vmovdqu8 %xmm1, %xmm0 {%k1}
+; AVX512VLVBMI2-NEXT:    vzeroupper
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOP-LABEL: constant_funnnel_v16i8:
 ; XOP:       # %bb.0:
 ; XOP-NEXT:    vpshlb {{.*}}(%rip), %xmm1, %xmm2
@@ -2472,12 +2821,45 @@ define <2 x i64> @splatconstant_funnnel_v2i64(<2 x i64> %x, <2 x i64> %y) nounwi
 ; AVX-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    retq
 ;
-; AVX512-LABEL: splatconstant_funnnel_v2i64:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpsrlq $14, %xmm1, %xmm1
-; AVX512-NEXT:    vpsllq $50, %xmm0, %xmm0
-; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    retq
+; AVX512F-LABEL: splatconstant_funnnel_v2i64:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpsrlq $14, %xmm1, %xmm1
+; AVX512F-NEXT:    vpsllq $50, %xmm0, %xmm0
+; AVX512F-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: splatconstant_funnnel_v2i64:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpsrlq $14, %xmm1, %xmm1
+; AVX512VL-NEXT:    vpsllq $50, %xmm0, %xmm0
+; AVX512VL-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: splatconstant_funnnel_v2i64:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpsrlq $14, %xmm1, %xmm1
+; AVX512BW-NEXT:    vpsllq $50, %xmm0, %xmm0
+; AVX512BW-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT:    retq
+;
+; AVX512VBMI2-LABEL: splatconstant_funnnel_v2i64:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    vpsrlq $14, %xmm1, %xmm1
+; AVX512VBMI2-NEXT:    vpsllq $50, %xmm0, %xmm0
+; AVX512VBMI2-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLBW-LABEL: splatconstant_funnnel_v2i64:
+; AVX512VLBW:       # %bb.0:
+; AVX512VLBW-NEXT:    vpsrlq $14, %xmm1, %xmm1
+; AVX512VLBW-NEXT:    vpsllq $50, %xmm0, %xmm0
+; AVX512VLBW-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX512VLBW-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: splatconstant_funnnel_v2i64:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpshrdq $14, %xmm0, %xmm1, %xmm0
+; AVX512VLVBMI2-NEXT:    retq
 ;
 ; XOP-LABEL: splatconstant_funnnel_v2i64:
 ; XOP:       # %bb.0:
@@ -2511,12 +2893,45 @@ define <4 x i32> @splatconstant_funnnel_v4i32(<4 x i32> %x, <4 x i32> %y) nounwi
 ; AVX-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    retq
 ;
-; AVX512-LABEL: splatconstant_funnnel_v4i32:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpsrld $4, %xmm1, %xmm1
-; AVX512-NEXT:    vpslld $28, %xmm0, %xmm0
-; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    retq
+; AVX512F-LABEL: splatconstant_funnnel_v4i32:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpsrld $4, %xmm1, %xmm1
+; AVX512F-NEXT:    vpslld $28, %xmm0, %xmm0
+; AVX512F-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: splatconstant_funnnel_v4i32:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpsrld $4, %xmm1, %xmm1
+; AVX512VL-NEXT:    vpslld $28, %xmm0, %xmm0
+; AVX512VL-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: splatconstant_funnnel_v4i32:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpsrld $4, %xmm1, %xmm1
+; AVX512BW-NEXT:    vpslld $28, %xmm0, %xmm0
+; AVX512BW-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT:    retq
+;
+; AVX512VBMI2-LABEL: splatconstant_funnnel_v4i32:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    vpsrld $4, %xmm1, %xmm1
+; AVX512VBMI2-NEXT:    vpslld $28, %xmm0, %xmm0
+; AVX512VBMI2-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLBW-LABEL: splatconstant_funnnel_v4i32:
+; AVX512VLBW:       # %bb.0:
+; AVX512VLBW-NEXT:    vpsrld $4, %xmm1, %xmm1
+; AVX512VLBW-NEXT:    vpslld $28, %xmm0, %xmm0
+; AVX512VLBW-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX512VLBW-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: splatconstant_funnnel_v4i32:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpshrdd $4, %xmm0, %xmm1, %xmm0
+; AVX512VLVBMI2-NEXT:    retq
 ;
 ; XOP-LABEL: splatconstant_funnnel_v4i32:
 ; XOP:       # %bb.0:
@@ -2550,12 +2965,45 @@ define <8 x i16> @splatconstant_funnnel_v8i16(<8 x i16> %x, <8 x i16> %y) nounwi
 ; AVX-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    retq
 ;
-; AVX512-LABEL: splatconstant_funnnel_v8i16:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpsrlw $7, %xmm1, %xmm1
-; AVX512-NEXT:    vpsllw $9, %xmm0, %xmm0
-; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    retq
+; AVX512F-LABEL: splatconstant_funnnel_v8i16:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpsrlw $7, %xmm1, %xmm1
+; AVX512F-NEXT:    vpsllw $9, %xmm0, %xmm0
+; AVX512F-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: splatconstant_funnnel_v8i16:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpsrlw $7, %xmm1, %xmm1
+; AVX512VL-NEXT:    vpsllw $9, %xmm0, %xmm0
+; AVX512VL-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: splatconstant_funnnel_v8i16:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpsrlw $7, %xmm1, %xmm1
+; AVX512BW-NEXT:    vpsllw $9, %xmm0, %xmm0
+; AVX512BW-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT:    retq
+;
+; AVX512VBMI2-LABEL: splatconstant_funnnel_v8i16:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    vpsrlw $7, %xmm1, %xmm1
+; AVX512VBMI2-NEXT:    vpsllw $9, %xmm0, %xmm0
+; AVX512VBMI2-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLBW-LABEL: splatconstant_funnnel_v8i16:
+; AVX512VLBW:       # %bb.0:
+; AVX512VLBW-NEXT:    vpsrlw $7, %xmm1, %xmm1
+; AVX512VLBW-NEXT:    vpsllw $9, %xmm0, %xmm0
+; AVX512VLBW-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX512VLBW-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: splatconstant_funnnel_v8i16:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpshrdw $7, %xmm0, %xmm1, %xmm0
+; AVX512VLVBMI2-NEXT:    retq
 ;
 ; XOP-LABEL: splatconstant_funnnel_v8i16:
 ; XOP:       # %bb.0:
diff --git a/test/CodeGen/X86/vector-fshr-256.ll b/test/CodeGen/X86/vector-fshr-256.ll
index 9672e8d4b53a901faa34a646d8e1d70c98977d9f..64283a6603b3f451c958f62957118ec444945f87 100644
--- a/test/CodeGen/X86/vector-fshr-256.ll
+++ b/test/CodeGen/X86/vector-fshr-256.ll
@@ -4,7 +4,9 @@
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f | FileCheck %s --check-prefixes=AVX512,AVX512F
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512vl | FileCheck %s --check-prefixes=AVX512,AVX512VL
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512bw | FileCheck %s --check-prefixes=AVX512,AVX512BW
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512vbmi2 | FileCheck %s --check-prefixes=AVX512,AVX512VBMI2
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512bw,+avx512vl | FileCheck %s --check-prefixes=AVX512,AVX512VLBW
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512vbmi2,+avx512vl | FileCheck %s --check-prefixes=AVX512,AVX512VLVBMI2
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+xop,+avx | FileCheck %s --check-prefixes=XOP,XOPAVX1
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+xop,+avx2 | FileCheck %s --check-prefixes=XOP,XOPAVX2
 
@@ -112,6 +114,22 @@ define <4 x i64> @var_funnnel_v4i64(<4 x i64> %x, <4 x i64> %y, <4 x i64> %amt)
 ; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
 ; AVX512BW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: var_funnnel_v4i64:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $ymm2 killed $ymm2 def $zmm2
+; AVX512VBMI2-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512VBMI2-NEXT:    vpbroadcastq {{.*#+}} ymm3 = [63,63,63,63]
+; AVX512VBMI2-NEXT:    vpand %ymm3, %ymm2, %ymm4
+; AVX512VBMI2-NEXT:    vpsrlvq %ymm4, %ymm1, %ymm5
+; AVX512VBMI2-NEXT:    vpbroadcastq {{.*#+}} ymm6 = [64,64,64,64]
+; AVX512VBMI2-NEXT:    vpsubq %ymm4, %ymm6, %ymm4
+; AVX512VBMI2-NEXT:    vpsllvq %ymm4, %ymm0, %ymm0
+; AVX512VBMI2-NEXT:    vpor %ymm5, %ymm0, %ymm0
+; AVX512VBMI2-NEXT:    vptestnmq %zmm3, %zmm2, %k1
+; AVX512VBMI2-NEXT:    vmovdqa64 %zmm1, %zmm0 {%k1}
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
+; AVX512VBMI2-NEXT:    retq
+;
 ; AVX512VLBW-LABEL: var_funnnel_v4i64:
 ; AVX512VLBW:       # %bb.0:
 ; AVX512VLBW-NEXT:    vpbroadcastq {{.*#+}} ymm3 = [63,63,63,63]
@@ -125,6 +143,12 @@ define <4 x i64> @var_funnnel_v4i64(<4 x i64> %x, <4 x i64> %y, <4 x i64> %amt)
 ; AVX512VLBW-NEXT:    vmovdqa64 %ymm1, %ymm0 {%k1}
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VLVBMI2-LABEL: var_funnnel_v4i64:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpshrdvq %ymm2, %ymm0, %ymm1
+; AVX512VLVBMI2-NEXT:    vmovdqa %ymm1, %ymm0
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOPAVX1-LABEL: var_funnnel_v4i64:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vandps {{.*}}(%rip), %ymm2, %ymm2
@@ -277,6 +301,22 @@ define <8 x i32> @var_funnnel_v8i32(<8 x i32> %x, <8 x i32> %y, <8 x i32> %amt)
 ; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
 ; AVX512BW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: var_funnnel_v8i32:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $ymm2 killed $ymm2 def $zmm2
+; AVX512VBMI2-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512VBMI2-NEXT:    vpbroadcastd {{.*#+}} ymm3 = [31,31,31,31,31,31,31,31]
+; AVX512VBMI2-NEXT:    vpand %ymm3, %ymm2, %ymm4
+; AVX512VBMI2-NEXT:    vpsrlvd %ymm4, %ymm1, %ymm5
+; AVX512VBMI2-NEXT:    vpbroadcastd {{.*#+}} ymm6 = [32,32,32,32,32,32,32,32]
+; AVX512VBMI2-NEXT:    vpsubd %ymm4, %ymm6, %ymm4
+; AVX512VBMI2-NEXT:    vpsllvd %ymm4, %ymm0, %ymm0
+; AVX512VBMI2-NEXT:    vpor %ymm5, %ymm0, %ymm0
+; AVX512VBMI2-NEXT:    vptestnmd %zmm3, %zmm2, %k1
+; AVX512VBMI2-NEXT:    vmovdqa32 %zmm1, %zmm0 {%k1}
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
+; AVX512VBMI2-NEXT:    retq
+;
 ; AVX512VLBW-LABEL: var_funnnel_v8i32:
 ; AVX512VLBW:       # %bb.0:
 ; AVX512VLBW-NEXT:    vpbroadcastd {{.*#+}} ymm3 = [31,31,31,31,31,31,31,31]
@@ -290,6 +330,12 @@ define <8 x i32> @var_funnnel_v8i32(<8 x i32> %x, <8 x i32> %y, <8 x i32> %amt)
 ; AVX512VLBW-NEXT:    vmovdqa32 %ymm1, %ymm0 {%k1}
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VLVBMI2-LABEL: var_funnnel_v8i32:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpshrdvd %ymm2, %ymm0, %ymm1
+; AVX512VLVBMI2-NEXT:    vmovdqa %ymm1, %ymm0
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOPAVX1-LABEL: var_funnnel_v8i32:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vandps {{.*}}(%rip), %ymm2, %ymm2
@@ -485,6 +531,23 @@ define <16 x i16> @var_funnnel_v16i16(<16 x i16> %x, <16 x i16> %y, <16 x i16> %
 ; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
 ; AVX512BW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: var_funnnel_v16i16:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $ymm2 killed $ymm2 def $zmm2
+; AVX512VBMI2-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} ymm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; AVX512VBMI2-NEXT:    vpand %ymm3, %ymm2, %ymm4
+; AVX512VBMI2-NEXT:    vpsrlvw %zmm4, %zmm1, %zmm5
+; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} ymm6 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
+; AVX512VBMI2-NEXT:    vpsubw %ymm4, %ymm6, %ymm4
+; AVX512VBMI2-NEXT:    vpsllvw %zmm4, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    vpor %ymm5, %ymm0, %ymm0
+; AVX512VBMI2-NEXT:    vptestnmw %zmm3, %zmm2, %k1
+; AVX512VBMI2-NEXT:    vmovdqu16 %zmm1, %zmm0 {%k1}
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
+; AVX512VBMI2-NEXT:    retq
+;
 ; AVX512VLBW-LABEL: var_funnnel_v16i16:
 ; AVX512VLBW:       # %bb.0:
 ; AVX512VLBW-NEXT:    vmovdqa {{.*#+}} ymm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
@@ -498,6 +561,12 @@ define <16 x i16> @var_funnnel_v16i16(<16 x i16> %x, <16 x i16> %y, <16 x i16> %
 ; AVX512VLBW-NEXT:    vmovdqu16 %ymm1, %ymm0 {%k1}
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VLVBMI2-LABEL: var_funnnel_v16i16:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpshrdvw %ymm2, %ymm0, %ymm1
+; AVX512VLVBMI2-NEXT:    vmovdqa %ymm1, %ymm0
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOPAVX1-LABEL: var_funnnel_v16i16:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vandps {{.*}}(%rip), %ymm2, %ymm2
@@ -746,6 +815,27 @@ define <32 x i8> @var_funnnel_v32i8(<32 x i8> %x, <32 x i8> %y, <32 x i8> %amt)
 ; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
 ; AVX512BW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: var_funnnel_v32i8:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $ymm2 killed $ymm2 def $zmm2
+; AVX512VBMI2-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} ymm3 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
+; AVX512VBMI2-NEXT:    vpand %ymm3, %ymm2, %ymm4
+; AVX512VBMI2-NEXT:    vpmovzxbw {{.*#+}} zmm5 = ymm4[0],zero,ymm4[1],zero,ymm4[2],zero,ymm4[3],zero,ymm4[4],zero,ymm4[5],zero,ymm4[6],zero,ymm4[7],zero,ymm4[8],zero,ymm4[9],zero,ymm4[10],zero,ymm4[11],zero,ymm4[12],zero,ymm4[13],zero,ymm4[14],zero,ymm4[15],zero,ymm4[16],zero,ymm4[17],zero,ymm4[18],zero,ymm4[19],zero,ymm4[20],zero,ymm4[21],zero,ymm4[22],zero,ymm4[23],zero,ymm4[24],zero,ymm4[25],zero,ymm4[26],zero,ymm4[27],zero,ymm4[28],zero,ymm4[29],zero,ymm4[30],zero,ymm4[31],zero
+; AVX512VBMI2-NEXT:    vpmovzxbw {{.*#+}} zmm6 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero
+; AVX512VBMI2-NEXT:    vpsrlvw %zmm5, %zmm6, %zmm5
+; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} ymm6 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; AVX512VBMI2-NEXT:    vpsubb %ymm4, %ymm6, %ymm4
+; AVX512VBMI2-NEXT:    vpmovzxbw {{.*#+}} zmm4 = ymm4[0],zero,ymm4[1],zero,ymm4[2],zero,ymm4[3],zero,ymm4[4],zero,ymm4[5],zero,ymm4[6],zero,ymm4[7],zero,ymm4[8],zero,ymm4[9],zero,ymm4[10],zero,ymm4[11],zero,ymm4[12],zero,ymm4[13],zero,ymm4[14],zero,ymm4[15],zero,ymm4[16],zero,ymm4[17],zero,ymm4[18],zero,ymm4[19],zero,ymm4[20],zero,ymm4[21],zero,ymm4[22],zero,ymm4[23],zero,ymm4[24],zero,ymm4[25],zero,ymm4[26],zero,ymm4[27],zero,ymm4[28],zero,ymm4[29],zero,ymm4[30],zero,ymm4[31],zero
+; AVX512VBMI2-NEXT:    vpmovzxbw {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero,ymm0[16],zero,ymm0[17],zero,ymm0[18],zero,ymm0[19],zero,ymm0[20],zero,ymm0[21],zero,ymm0[22],zero,ymm0[23],zero,ymm0[24],zero,ymm0[25],zero,ymm0[26],zero,ymm0[27],zero,ymm0[28],zero,ymm0[29],zero,ymm0[30],zero,ymm0[31],zero
+; AVX512VBMI2-NEXT:    vpsllvw %zmm4, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    vporq %zmm5, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    vpmovwb %zmm0, %ymm0
+; AVX512VBMI2-NEXT:    vptestnmb %zmm3, %zmm2, %k1
+; AVX512VBMI2-NEXT:    vmovdqu8 %zmm1, %zmm0 {%k1}
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
+; AVX512VBMI2-NEXT:    retq
+;
 ; AVX512VLBW-LABEL: var_funnnel_v32i8:
 ; AVX512VLBW:       # %bb.0:
 ; AVX512VLBW-NEXT:    vmovdqa {{.*#+}} ymm3 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
@@ -764,6 +854,24 @@ define <32 x i8> @var_funnnel_v32i8(<32 x i8> %x, <32 x i8> %y, <32 x i8> %amt)
 ; AVX512VLBW-NEXT:    vmovdqu8 %ymm1, %ymm0 {%k1}
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VLVBMI2-LABEL: var_funnnel_v32i8:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vmovdqa {{.*#+}} ymm3 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
+; AVX512VLVBMI2-NEXT:    vpand %ymm3, %ymm2, %ymm4
+; AVX512VLVBMI2-NEXT:    vpmovzxbw {{.*#+}} zmm5 = ymm4[0],zero,ymm4[1],zero,ymm4[2],zero,ymm4[3],zero,ymm4[4],zero,ymm4[5],zero,ymm4[6],zero,ymm4[7],zero,ymm4[8],zero,ymm4[9],zero,ymm4[10],zero,ymm4[11],zero,ymm4[12],zero,ymm4[13],zero,ymm4[14],zero,ymm4[15],zero,ymm4[16],zero,ymm4[17],zero,ymm4[18],zero,ymm4[19],zero,ymm4[20],zero,ymm4[21],zero,ymm4[22],zero,ymm4[23],zero,ymm4[24],zero,ymm4[25],zero,ymm4[26],zero,ymm4[27],zero,ymm4[28],zero,ymm4[29],zero,ymm4[30],zero,ymm4[31],zero
+; AVX512VLVBMI2-NEXT:    vpmovzxbw {{.*#+}} zmm6 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero
+; AVX512VLVBMI2-NEXT:    vpsrlvw %zmm5, %zmm6, %zmm5
+; AVX512VLVBMI2-NEXT:    vmovdqa {{.*#+}} ymm6 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; AVX512VLVBMI2-NEXT:    vpsubb %ymm4, %ymm6, %ymm4
+; AVX512VLVBMI2-NEXT:    vpmovzxbw {{.*#+}} zmm4 = ymm4[0],zero,ymm4[1],zero,ymm4[2],zero,ymm4[3],zero,ymm4[4],zero,ymm4[5],zero,ymm4[6],zero,ymm4[7],zero,ymm4[8],zero,ymm4[9],zero,ymm4[10],zero,ymm4[11],zero,ymm4[12],zero,ymm4[13],zero,ymm4[14],zero,ymm4[15],zero,ymm4[16],zero,ymm4[17],zero,ymm4[18],zero,ymm4[19],zero,ymm4[20],zero,ymm4[21],zero,ymm4[22],zero,ymm4[23],zero,ymm4[24],zero,ymm4[25],zero,ymm4[26],zero,ymm4[27],zero,ymm4[28],zero,ymm4[29],zero,ymm4[30],zero,ymm4[31],zero
+; AVX512VLVBMI2-NEXT:    vpmovzxbw {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero,ymm0[16],zero,ymm0[17],zero,ymm0[18],zero,ymm0[19],zero,ymm0[20],zero,ymm0[21],zero,ymm0[22],zero,ymm0[23],zero,ymm0[24],zero,ymm0[25],zero,ymm0[26],zero,ymm0[27],zero,ymm0[28],zero,ymm0[29],zero,ymm0[30],zero,ymm0[31],zero
+; AVX512VLVBMI2-NEXT:    vpsllvw %zmm4, %zmm0, %zmm0
+; AVX512VLVBMI2-NEXT:    vporq %zmm5, %zmm0, %zmm0
+; AVX512VLVBMI2-NEXT:    vpmovwb %zmm0, %ymm0
+; AVX512VLVBMI2-NEXT:    vptestnmb %ymm3, %ymm2, %k1
+; AVX512VLVBMI2-NEXT:    vmovdqu8 %ymm1, %ymm0 {%k1}
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOPAVX1-LABEL: var_funnnel_v32i8:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vandps {{.*}}(%rip), %ymm2, %ymm2
@@ -907,6 +1015,22 @@ define <4 x i64> @splatvar_funnnel_v4i64(<4 x i64> %x, <4 x i64> %y, <4 x i64> %
 ; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
 ; AVX512BW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: splatvar_funnnel_v4i64:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512VBMI2-NEXT:    vpbroadcastq %xmm2, %ymm2
+; AVX512VBMI2-NEXT:    vpbroadcastq {{.*#+}} ymm3 = [63,63,63,63]
+; AVX512VBMI2-NEXT:    vpand %ymm3, %ymm2, %ymm4
+; AVX512VBMI2-NEXT:    vpsrlq %xmm4, %ymm1, %ymm5
+; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} xmm6 = [64,64]
+; AVX512VBMI2-NEXT:    vpsubq %xmm4, %xmm6, %xmm4
+; AVX512VBMI2-NEXT:    vpsllq %xmm4, %ymm0, %ymm0
+; AVX512VBMI2-NEXT:    vpor %ymm5, %ymm0, %ymm0
+; AVX512VBMI2-NEXT:    vptestnmq %zmm3, %zmm2, %k1
+; AVX512VBMI2-NEXT:    vmovdqa64 %zmm1, %zmm0 {%k1}
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
+; AVX512VBMI2-NEXT:    retq
+;
 ; AVX512VLBW-LABEL: splatvar_funnnel_v4i64:
 ; AVX512VLBW:       # %bb.0:
 ; AVX512VLBW-NEXT:    vpbroadcastq %xmm2, %ymm2
@@ -921,6 +1045,13 @@ define <4 x i64> @splatvar_funnnel_v4i64(<4 x i64> %x, <4 x i64> %y, <4 x i64> %
 ; AVX512VLBW-NEXT:    vmovdqa64 %ymm1, %ymm0 {%k1}
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VLVBMI2-LABEL: splatvar_funnnel_v4i64:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpbroadcastq %xmm2, %ymm2
+; AVX512VLVBMI2-NEXT:    vpshrdvq %ymm2, %ymm0, %ymm1
+; AVX512VLVBMI2-NEXT:    vmovdqa %ymm1, %ymm0
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOPAVX1-LABEL: splatvar_funnnel_v4i64:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vmovddup {{.*#+}} xmm2 = xmm2[0,0]
@@ -1063,6 +1194,24 @@ define <8 x i32> @splatvar_funnnel_v8i32(<8 x i32> %x, <8 x i32> %y, <8 x i32> %
 ; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
 ; AVX512BW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: splatvar_funnnel_v8i32:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512VBMI2-NEXT:    vpbroadcastd %xmm2, %ymm2
+; AVX512VBMI2-NEXT:    vpbroadcastd {{.*#+}} ymm3 = [31,31,31,31,31,31,31,31]
+; AVX512VBMI2-NEXT:    vpand %ymm3, %ymm2, %ymm4
+; AVX512VBMI2-NEXT:    vpmovzxdq {{.*#+}} xmm5 = xmm4[0],zero,xmm4[1],zero
+; AVX512VBMI2-NEXT:    vpsrld %xmm5, %ymm1, %ymm5
+; AVX512VBMI2-NEXT:    vpbroadcastd {{.*#+}} xmm6 = [32,32,32,32]
+; AVX512VBMI2-NEXT:    vpsubd %xmm4, %xmm6, %xmm4
+; AVX512VBMI2-NEXT:    vpmovzxdq {{.*#+}} xmm4 = xmm4[0],zero,xmm4[1],zero
+; AVX512VBMI2-NEXT:    vpslld %xmm4, %ymm0, %ymm0
+; AVX512VBMI2-NEXT:    vpor %ymm5, %ymm0, %ymm0
+; AVX512VBMI2-NEXT:    vptestnmd %zmm3, %zmm2, %k1
+; AVX512VBMI2-NEXT:    vmovdqa32 %zmm1, %zmm0 {%k1}
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
+; AVX512VBMI2-NEXT:    retq
+;
 ; AVX512VLBW-LABEL: splatvar_funnnel_v8i32:
 ; AVX512VLBW:       # %bb.0:
 ; AVX512VLBW-NEXT:    vpbroadcastd %xmm2, %ymm2
@@ -1079,6 +1228,13 @@ define <8 x i32> @splatvar_funnnel_v8i32(<8 x i32> %x, <8 x i32> %y, <8 x i32> %
 ; AVX512VLBW-NEXT:    vmovdqa32 %ymm1, %ymm0 {%k1}
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VLVBMI2-LABEL: splatvar_funnnel_v8i32:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpbroadcastd %xmm2, %ymm2
+; AVX512VLVBMI2-NEXT:    vpshrdvd %ymm2, %ymm0, %ymm1
+; AVX512VLVBMI2-NEXT:    vmovdqa %ymm1, %ymm0
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOPAVX1-LABEL: splatvar_funnnel_v8i32:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vpermilps {{.*#+}} xmm2 = xmm2[0,0,0,0]
@@ -1226,6 +1382,24 @@ define <16 x i16> @splatvar_funnnel_v16i16(<16 x i16> %x, <16 x i16> %y, <16 x i
 ; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
 ; AVX512BW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: splatvar_funnnel_v16i16:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512VBMI2-NEXT:    vpbroadcastw %xmm2, %ymm2
+; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} ymm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; AVX512VBMI2-NEXT:    vpand %ymm3, %ymm2, %ymm4
+; AVX512VBMI2-NEXT:    vpmovzxwq {{.*#+}} xmm5 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero
+; AVX512VBMI2-NEXT:    vpsrlw %xmm5, %ymm1, %ymm5
+; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} xmm6 = [16,16,16,16,16,16,16,16]
+; AVX512VBMI2-NEXT:    vpsubw %xmm4, %xmm6, %xmm4
+; AVX512VBMI2-NEXT:    vpmovzxwq {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero
+; AVX512VBMI2-NEXT:    vpsllw %xmm4, %ymm0, %ymm0
+; AVX512VBMI2-NEXT:    vpor %ymm5, %ymm0, %ymm0
+; AVX512VBMI2-NEXT:    vptestnmw %zmm3, %zmm2, %k1
+; AVX512VBMI2-NEXT:    vmovdqu16 %zmm1, %zmm0 {%k1}
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
+; AVX512VBMI2-NEXT:    retq
+;
 ; AVX512VLBW-LABEL: splatvar_funnnel_v16i16:
 ; AVX512VLBW:       # %bb.0:
 ; AVX512VLBW-NEXT:    vpbroadcastw %xmm2, %ymm2
@@ -1242,6 +1416,13 @@ define <16 x i16> @splatvar_funnnel_v16i16(<16 x i16> %x, <16 x i16> %y, <16 x i
 ; AVX512VLBW-NEXT:    vmovdqu16 %ymm1, %ymm0 {%k1}
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VLVBMI2-LABEL: splatvar_funnnel_v16i16:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpbroadcastw %xmm2, %ymm2
+; AVX512VLVBMI2-NEXT:    vpshrdvw %ymm2, %ymm0, %ymm1
+; AVX512VLVBMI2-NEXT:    vmovdqa %ymm1, %ymm0
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOPAVX1-LABEL: splatvar_funnnel_v16i16:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm2 = xmm2[0,0,2,3,4,5,6,7]
@@ -1426,6 +1607,27 @@ define <32 x i8> @splatvar_funnnel_v32i8(<32 x i8> %x, <32 x i8> %y, <32 x i8> %
 ; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
 ; AVX512BW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: splatvar_funnnel_v32i8:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512VBMI2-NEXT:    vpbroadcastb %xmm2, %ymm2
+; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} ymm3 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
+; AVX512VBMI2-NEXT:    vpand %ymm3, %ymm2, %ymm4
+; AVX512VBMI2-NEXT:    vpmovzxbw {{.*#+}} zmm5 = ymm4[0],zero,ymm4[1],zero,ymm4[2],zero,ymm4[3],zero,ymm4[4],zero,ymm4[5],zero,ymm4[6],zero,ymm4[7],zero,ymm4[8],zero,ymm4[9],zero,ymm4[10],zero,ymm4[11],zero,ymm4[12],zero,ymm4[13],zero,ymm4[14],zero,ymm4[15],zero,ymm4[16],zero,ymm4[17],zero,ymm4[18],zero,ymm4[19],zero,ymm4[20],zero,ymm4[21],zero,ymm4[22],zero,ymm4[23],zero,ymm4[24],zero,ymm4[25],zero,ymm4[26],zero,ymm4[27],zero,ymm4[28],zero,ymm4[29],zero,ymm4[30],zero,ymm4[31],zero
+; AVX512VBMI2-NEXT:    vpmovzxbw {{.*#+}} zmm6 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero
+; AVX512VBMI2-NEXT:    vpsrlvw %zmm5, %zmm6, %zmm5
+; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} ymm6 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; AVX512VBMI2-NEXT:    vpsubb %ymm4, %ymm6, %ymm4
+; AVX512VBMI2-NEXT:    vpmovzxbw {{.*#+}} zmm4 = ymm4[0],zero,ymm4[1],zero,ymm4[2],zero,ymm4[3],zero,ymm4[4],zero,ymm4[5],zero,ymm4[6],zero,ymm4[7],zero,ymm4[8],zero,ymm4[9],zero,ymm4[10],zero,ymm4[11],zero,ymm4[12],zero,ymm4[13],zero,ymm4[14],zero,ymm4[15],zero,ymm4[16],zero,ymm4[17],zero,ymm4[18],zero,ymm4[19],zero,ymm4[20],zero,ymm4[21],zero,ymm4[22],zero,ymm4[23],zero,ymm4[24],zero,ymm4[25],zero,ymm4[26],zero,ymm4[27],zero,ymm4[28],zero,ymm4[29],zero,ymm4[30],zero,ymm4[31],zero
+; AVX512VBMI2-NEXT:    vpmovzxbw {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero,ymm0[16],zero,ymm0[17],zero,ymm0[18],zero,ymm0[19],zero,ymm0[20],zero,ymm0[21],zero,ymm0[22],zero,ymm0[23],zero,ymm0[24],zero,ymm0[25],zero,ymm0[26],zero,ymm0[27],zero,ymm0[28],zero,ymm0[29],zero,ymm0[30],zero,ymm0[31],zero
+; AVX512VBMI2-NEXT:    vpsllvw %zmm4, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    vporq %zmm5, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    vpmovwb %zmm0, %ymm0
+; AVX512VBMI2-NEXT:    vptestnmb %zmm3, %zmm2, %k1
+; AVX512VBMI2-NEXT:    vmovdqu8 %zmm1, %zmm0 {%k1}
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
+; AVX512VBMI2-NEXT:    retq
+;
 ; AVX512VLBW-LABEL: splatvar_funnnel_v32i8:
 ; AVX512VLBW:       # %bb.0:
 ; AVX512VLBW-NEXT:    vpbroadcastb %xmm2, %ymm2
@@ -1445,6 +1647,25 @@ define <32 x i8> @splatvar_funnnel_v32i8(<32 x i8> %x, <32 x i8> %y, <32 x i8> %
 ; AVX512VLBW-NEXT:    vmovdqu8 %ymm1, %ymm0 {%k1}
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VLVBMI2-LABEL: splatvar_funnnel_v32i8:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpbroadcastb %xmm2, %ymm2
+; AVX512VLVBMI2-NEXT:    vmovdqa {{.*#+}} ymm3 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
+; AVX512VLVBMI2-NEXT:    vpand %ymm3, %ymm2, %ymm4
+; AVX512VLVBMI2-NEXT:    vpmovzxbw {{.*#+}} zmm5 = ymm4[0],zero,ymm4[1],zero,ymm4[2],zero,ymm4[3],zero,ymm4[4],zero,ymm4[5],zero,ymm4[6],zero,ymm4[7],zero,ymm4[8],zero,ymm4[9],zero,ymm4[10],zero,ymm4[11],zero,ymm4[12],zero,ymm4[13],zero,ymm4[14],zero,ymm4[15],zero,ymm4[16],zero,ymm4[17],zero,ymm4[18],zero,ymm4[19],zero,ymm4[20],zero,ymm4[21],zero,ymm4[22],zero,ymm4[23],zero,ymm4[24],zero,ymm4[25],zero,ymm4[26],zero,ymm4[27],zero,ymm4[28],zero,ymm4[29],zero,ymm4[30],zero,ymm4[31],zero
+; AVX512VLVBMI2-NEXT:    vpmovzxbw {{.*#+}} zmm6 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero
+; AVX512VLVBMI2-NEXT:    vpsrlvw %zmm5, %zmm6, %zmm5
+; AVX512VLVBMI2-NEXT:    vmovdqa {{.*#+}} ymm6 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; AVX512VLVBMI2-NEXT:    vpsubb %ymm4, %ymm6, %ymm4
+; AVX512VLVBMI2-NEXT:    vpmovzxbw {{.*#+}} zmm4 = ymm4[0],zero,ymm4[1],zero,ymm4[2],zero,ymm4[3],zero,ymm4[4],zero,ymm4[5],zero,ymm4[6],zero,ymm4[7],zero,ymm4[8],zero,ymm4[9],zero,ymm4[10],zero,ymm4[11],zero,ymm4[12],zero,ymm4[13],zero,ymm4[14],zero,ymm4[15],zero,ymm4[16],zero,ymm4[17],zero,ymm4[18],zero,ymm4[19],zero,ymm4[20],zero,ymm4[21],zero,ymm4[22],zero,ymm4[23],zero,ymm4[24],zero,ymm4[25],zero,ymm4[26],zero,ymm4[27],zero,ymm4[28],zero,ymm4[29],zero,ymm4[30],zero,ymm4[31],zero
+; AVX512VLVBMI2-NEXT:    vpmovzxbw {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero,ymm0[16],zero,ymm0[17],zero,ymm0[18],zero,ymm0[19],zero,ymm0[20],zero,ymm0[21],zero,ymm0[22],zero,ymm0[23],zero,ymm0[24],zero,ymm0[25],zero,ymm0[26],zero,ymm0[27],zero,ymm0[28],zero,ymm0[29],zero,ymm0[30],zero,ymm0[31],zero
+; AVX512VLVBMI2-NEXT:    vpsllvw %zmm4, %zmm0, %zmm0
+; AVX512VLVBMI2-NEXT:    vporq %zmm5, %zmm0, %zmm0
+; AVX512VLVBMI2-NEXT:    vpmovwb %zmm0, %ymm0
+; AVX512VLVBMI2-NEXT:    vptestnmb %ymm3, %ymm2, %k1
+; AVX512VLVBMI2-NEXT:    vmovdqu8 %ymm1, %ymm0 {%k1}
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOPAVX1-LABEL: splatvar_funnnel_v32i8:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
@@ -1534,12 +1755,46 @@ define <4 x i64> @constant_funnnel_v4i64(<4 x i64> %x, <4 x i64> %y) nounwind {
 ; AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    retq
 ;
-; AVX512-LABEL: constant_funnnel_v4i64:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpsrlvq {{.*}}(%rip), %ymm1, %ymm1
-; AVX512-NEXT:    vpsllvq {{.*}}(%rip), %ymm0, %ymm0
-; AVX512-NEXT:    vpor %ymm1, %ymm0, %ymm0
-; AVX512-NEXT:    retq
+; AVX512F-LABEL: constant_funnnel_v4i64:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpsrlvq {{.*}}(%rip), %ymm1, %ymm1
+; AVX512F-NEXT:    vpsllvq {{.*}}(%rip), %ymm0, %ymm0
+; AVX512F-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: constant_funnnel_v4i64:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpsrlvq {{.*}}(%rip), %ymm1, %ymm1
+; AVX512VL-NEXT:    vpsllvq {{.*}}(%rip), %ymm0, %ymm0
+; AVX512VL-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: constant_funnnel_v4i64:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpsrlvq {{.*}}(%rip), %ymm1, %ymm1
+; AVX512BW-NEXT:    vpsllvq {{.*}}(%rip), %ymm0, %ymm0
+; AVX512BW-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512BW-NEXT:    retq
+;
+; AVX512VBMI2-LABEL: constant_funnnel_v4i64:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    vpsrlvq {{.*}}(%rip), %ymm1, %ymm1
+; AVX512VBMI2-NEXT:    vpsllvq {{.*}}(%rip), %ymm0, %ymm0
+; AVX512VBMI2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLBW-LABEL: constant_funnnel_v4i64:
+; AVX512VLBW:       # %bb.0:
+; AVX512VLBW-NEXT:    vpsrlvq {{.*}}(%rip), %ymm1, %ymm1
+; AVX512VLBW-NEXT:    vpsllvq {{.*}}(%rip), %ymm0, %ymm0
+; AVX512VLBW-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512VLBW-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: constant_funnnel_v4i64:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpshrdvq {{.*}}(%rip), %ymm0, %ymm1
+; AVX512VLVBMI2-NEXT:    vmovdqa %ymm1, %ymm0
+; AVX512VLVBMI2-NEXT:    retq
 ;
 ; XOPAVX1-LABEL: constant_funnnel_v4i64:
 ; XOPAVX1:       # %bb.0:
@@ -1597,12 +1852,46 @@ define <8 x i32> @constant_funnnel_v8i32(<8 x i32> %x, <8 x i32> %y) nounwind {
 ; AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    retq
 ;
-; AVX512-LABEL: constant_funnnel_v8i32:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpsrlvd {{.*}}(%rip), %ymm1, %ymm1
-; AVX512-NEXT:    vpsllvd {{.*}}(%rip), %ymm0, %ymm0
-; AVX512-NEXT:    vpor %ymm1, %ymm0, %ymm0
-; AVX512-NEXT:    retq
+; AVX512F-LABEL: constant_funnnel_v8i32:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpsrlvd {{.*}}(%rip), %ymm1, %ymm1
+; AVX512F-NEXT:    vpsllvd {{.*}}(%rip), %ymm0, %ymm0
+; AVX512F-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: constant_funnnel_v8i32:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpsrlvd {{.*}}(%rip), %ymm1, %ymm1
+; AVX512VL-NEXT:    vpsllvd {{.*}}(%rip), %ymm0, %ymm0
+; AVX512VL-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: constant_funnnel_v8i32:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpsrlvd {{.*}}(%rip), %ymm1, %ymm1
+; AVX512BW-NEXT:    vpsllvd {{.*}}(%rip), %ymm0, %ymm0
+; AVX512BW-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512BW-NEXT:    retq
+;
+; AVX512VBMI2-LABEL: constant_funnnel_v8i32:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    vpsrlvd {{.*}}(%rip), %ymm1, %ymm1
+; AVX512VBMI2-NEXT:    vpsllvd {{.*}}(%rip), %ymm0, %ymm0
+; AVX512VBMI2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLBW-LABEL: constant_funnnel_v8i32:
+; AVX512VLBW:       # %bb.0:
+; AVX512VLBW-NEXT:    vpsrlvd {{.*}}(%rip), %ymm1, %ymm1
+; AVX512VLBW-NEXT:    vpsllvd {{.*}}(%rip), %ymm0, %ymm0
+; AVX512VLBW-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512VLBW-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: constant_funnnel_v8i32:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpshrdvd {{.*}}(%rip), %ymm0, %ymm1
+; AVX512VLVBMI2-NEXT:    vmovdqa %ymm1, %ymm0
+; AVX512VLVBMI2-NEXT:    retq
 ;
 ; XOPAVX1-LABEL: constant_funnnel_v8i32:
 ; XOPAVX1:       # %bb.0:
@@ -1697,6 +1986,19 @@ define <16 x i16> @constant_funnnel_v16i16(<16 x i16> %x, <16 x i16> %y) nounwin
 ; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
 ; AVX512BW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: constant_funnnel_v16i16:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512VBMI2-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} ymm2 = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15]
+; AVX512VBMI2-NEXT:    vpsrlvw %zmm2, %zmm1, %zmm2
+; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} ymm3 = [16,15,14,13,12,11,10,9,8,7,6,5,4,3,2,1]
+; AVX512VBMI2-NEXT:    vpsllvw %zmm3, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    vpor %ymm2, %ymm0, %ymm0
+; AVX512VBMI2-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0],ymm0[1,2,3,4,5,6,7],ymm1[8],ymm0[9,10,11,12,13,14,15]
+; AVX512VBMI2-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
+; AVX512VBMI2-NEXT:    retq
+;
 ; AVX512VLBW-LABEL: constant_funnnel_v16i16:
 ; AVX512VLBW:       # %bb.0:
 ; AVX512VLBW-NEXT:    vpsrlvw {{.*}}(%rip), %ymm1, %ymm2
@@ -1706,6 +2008,12 @@ define <16 x i16> @constant_funnnel_v16i16(<16 x i16> %x, <16 x i16> %y) nounwin
 ; AVX512VLBW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VLVBMI2-LABEL: constant_funnnel_v16i16:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpshrdvw {{.*}}(%rip), %ymm0, %ymm1
+; AVX512VLVBMI2-NEXT:    vmovdqa %ymm1, %ymm0
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOPAVX1-LABEL: constant_funnnel_v16i16:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vpshlw {{.*}}(%rip), %xmm1, %xmm2
@@ -1874,6 +2182,18 @@ define <32 x i8> @constant_funnnel_v32i8(<32 x i8> %x, <32 x i8> %y) nounwind {
 ; AVX512BW-NEXT:    vpblendvb %ymm2, %ymm0, %ymm1, %ymm0
 ; AVX512BW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: constant_funnnel_v32i8:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    vpmovzxbw {{.*#+}} zmm2 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero
+; AVX512VBMI2-NEXT:    vpsrlvw {{.*}}(%rip), %zmm2, %zmm2
+; AVX512VBMI2-NEXT:    vpmovzxbw {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero,ymm0[16],zero,ymm0[17],zero,ymm0[18],zero,ymm0[19],zero,ymm0[20],zero,ymm0[21],zero,ymm0[22],zero,ymm0[23],zero,ymm0[24],zero,ymm0[25],zero,ymm0[26],zero,ymm0[27],zero,ymm0[28],zero,ymm0[29],zero,ymm0[30],zero,ymm0[31],zero
+; AVX512VBMI2-NEXT:    vpsllvw {{.*}}(%rip), %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    vporq %zmm2, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    vpmovwb %zmm0, %ymm0
+; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} ymm2 = [0,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255,0,255,255,255,255,255,255,255]
+; AVX512VBMI2-NEXT:    vpblendvb %ymm2, %ymm0, %ymm1, %ymm0
+; AVX512VBMI2-NEXT:    retq
+;
 ; AVX512VLBW-LABEL: constant_funnnel_v32i8:
 ; AVX512VLBW:       # %bb.0:
 ; AVX512VLBW-NEXT:    vpmovzxbw {{.*#+}} zmm2 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero
@@ -1887,6 +2207,19 @@ define <32 x i8> @constant_funnnel_v32i8(<32 x i8> %x, <32 x i8> %y) nounwind {
 ; AVX512VLBW-NEXT:    vmovdqu8 %ymm1, %ymm0 {%k1}
 ; AVX512VLBW-NEXT:    retq
 ;
+; AVX512VLVBMI2-LABEL: constant_funnnel_v32i8:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpmovzxbw {{.*#+}} zmm2 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero
+; AVX512VLVBMI2-NEXT:    vpsrlvw {{.*}}(%rip), %zmm2, %zmm2
+; AVX512VLVBMI2-NEXT:    vpmovzxbw {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero,ymm0[16],zero,ymm0[17],zero,ymm0[18],zero,ymm0[19],zero,ymm0[20],zero,ymm0[21],zero,ymm0[22],zero,ymm0[23],zero,ymm0[24],zero,ymm0[25],zero,ymm0[26],zero,ymm0[27],zero,ymm0[28],zero,ymm0[29],zero,ymm0[30],zero,ymm0[31],zero
+; AVX512VLVBMI2-NEXT:    vpsllvw {{.*}}(%rip), %zmm0, %zmm0
+; AVX512VLVBMI2-NEXT:    vporq %zmm2, %zmm0, %zmm0
+; AVX512VLVBMI2-NEXT:    vpmovwb %zmm0, %ymm0
+; AVX512VLVBMI2-NEXT:    movl $16843009, %eax # imm = 0x1010101
+; AVX512VLVBMI2-NEXT:    kmovd %eax, %k1
+; AVX512VLVBMI2-NEXT:    vmovdqu8 %ymm1, %ymm0 {%k1}
+; AVX512VLVBMI2-NEXT:    retq
+;
 ; XOPAVX1-LABEL: constant_funnnel_v32i8:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
@@ -1949,12 +2282,45 @@ define <4 x i64> @splatconstant_funnnel_v4i64(<4 x i64> %x, <4 x i64> %y) nounwi
 ; AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    retq
 ;
-; AVX512-LABEL: splatconstant_funnnel_v4i64:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpsrlq $14, %ymm1, %ymm1
-; AVX512-NEXT:    vpsllq $50, %ymm0, %ymm0
-; AVX512-NEXT:    vpor %ymm1, %ymm0, %ymm0
-; AVX512-NEXT:    retq
+; AVX512F-LABEL: splatconstant_funnnel_v4i64:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpsrlq $14, %ymm1, %ymm1
+; AVX512F-NEXT:    vpsllq $50, %ymm0, %ymm0
+; AVX512F-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: splatconstant_funnnel_v4i64:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpsrlq $14, %ymm1, %ymm1
+; AVX512VL-NEXT:    vpsllq $50, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: splatconstant_funnnel_v4i64:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpsrlq $14, %ymm1, %ymm1
+; AVX512BW-NEXT:    vpsllq $50, %ymm0, %ymm0
+; AVX512BW-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512BW-NEXT:    retq
+;
+; AVX512VBMI2-LABEL: splatconstant_funnnel_v4i64:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    vpsrlq $14, %ymm1, %ymm1
+; AVX512VBMI2-NEXT:    vpsllq $50, %ymm0, %ymm0
+; AVX512VBMI2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLBW-LABEL: splatconstant_funnnel_v4i64:
+; AVX512VLBW:       # %bb.0:
+; AVX512VLBW-NEXT:    vpsrlq $14, %ymm1, %ymm1
+; AVX512VLBW-NEXT:    vpsllq $50, %ymm0, %ymm0
+; AVX512VLBW-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512VLBW-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: splatconstant_funnnel_v4i64:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpshrdq $14, %ymm0, %ymm1, %ymm0
+; AVX512VLVBMI2-NEXT:    retq
 ;
 ; XOPAVX1-LABEL: splatconstant_funnnel_v4i64:
 ; XOPAVX1:       # %bb.0:
@@ -2000,12 +2366,45 @@ define <8 x i32> @splatconstant_funnnel_v8i32(<8 x i32> %x, <8 x i32> %y) nounwi
 ; AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    retq
 ;
-; AVX512-LABEL: splatconstant_funnnel_v8i32:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpsrld $4, %ymm1, %ymm1
-; AVX512-NEXT:    vpslld $28, %ymm0, %ymm0
-; AVX512-NEXT:    vpor %ymm1, %ymm0, %ymm0
-; AVX512-NEXT:    retq
+; AVX512F-LABEL: splatconstant_funnnel_v8i32:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpsrld $4, %ymm1, %ymm1
+; AVX512F-NEXT:    vpslld $28, %ymm0, %ymm0
+; AVX512F-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: splatconstant_funnnel_v8i32:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpsrld $4, %ymm1, %ymm1
+; AVX512VL-NEXT:    vpslld $28, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: splatconstant_funnnel_v8i32:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpsrld $4, %ymm1, %ymm1
+; AVX512BW-NEXT:    vpslld $28, %ymm0, %ymm0
+; AVX512BW-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512BW-NEXT:    retq
+;
+; AVX512VBMI2-LABEL: splatconstant_funnnel_v8i32:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    vpsrld $4, %ymm1, %ymm1
+; AVX512VBMI2-NEXT:    vpslld $28, %ymm0, %ymm0
+; AVX512VBMI2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLBW-LABEL: splatconstant_funnnel_v8i32:
+; AVX512VLBW:       # %bb.0:
+; AVX512VLBW-NEXT:    vpsrld $4, %ymm1, %ymm1
+; AVX512VLBW-NEXT:    vpslld $28, %ymm0, %ymm0
+; AVX512VLBW-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512VLBW-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: splatconstant_funnnel_v8i32:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpshrdd $4, %ymm0, %ymm1, %ymm0
+; AVX512VLVBMI2-NEXT:    retq
 ;
 ; XOPAVX1-LABEL: splatconstant_funnnel_v8i32:
 ; XOPAVX1:       # %bb.0:
@@ -2051,12 +2450,45 @@ define <16 x i16> @splatconstant_funnnel_v16i16(<16 x i16> %x, <16 x i16> %y) no
 ; AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    retq
 ;
-; AVX512-LABEL: splatconstant_funnnel_v16i16:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpsrlw $7, %ymm1, %ymm1
-; AVX512-NEXT:    vpsllw $9, %ymm0, %ymm0
-; AVX512-NEXT:    vpor %ymm1, %ymm0, %ymm0
-; AVX512-NEXT:    retq
+; AVX512F-LABEL: splatconstant_funnnel_v16i16:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpsrlw $7, %ymm1, %ymm1
+; AVX512F-NEXT:    vpsllw $9, %ymm0, %ymm0
+; AVX512F-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: splatconstant_funnnel_v16i16:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpsrlw $7, %ymm1, %ymm1
+; AVX512VL-NEXT:    vpsllw $9, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: splatconstant_funnnel_v16i16:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpsrlw $7, %ymm1, %ymm1
+; AVX512BW-NEXT:    vpsllw $9, %ymm0, %ymm0
+; AVX512BW-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512BW-NEXT:    retq
+;
+; AVX512VBMI2-LABEL: splatconstant_funnnel_v16i16:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    vpsrlw $7, %ymm1, %ymm1
+; AVX512VBMI2-NEXT:    vpsllw $9, %ymm0, %ymm0
+; AVX512VBMI2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLBW-LABEL: splatconstant_funnnel_v16i16:
+; AVX512VLBW:       # %bb.0:
+; AVX512VLBW-NEXT:    vpsrlw $7, %ymm1, %ymm1
+; AVX512VLBW-NEXT:    vpsllw $9, %ymm0, %ymm0
+; AVX512VLBW-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512VLBW-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: splatconstant_funnnel_v16i16:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpshrdw $7, %ymm0, %ymm1, %ymm0
+; AVX512VLVBMI2-NEXT:    retq
 ;
 ; XOPAVX1-LABEL: splatconstant_funnnel_v16i16:
 ; XOPAVX1:       # %bb.0:
@@ -2137,10 +2569,8 @@ define <32 x i8> @splatconstant_funnnel_v32i8(<32 x i8> %x, <32 x i8> %y) nounwi
 ; XOPAVX2-LABEL: splatconstant_funnnel_v32i8:
 ; XOPAVX2:       # %bb.0:
 ; XOPAVX2-NEXT:    vpsrlw $4, %ymm1, %ymm1
-; XOPAVX2-NEXT:    vpand {{.*}}(%rip), %ymm1, %ymm1
 ; XOPAVX2-NEXT:    vpsllw $4, %ymm0, %ymm0
-; XOPAVX2-NEXT:    vpand {{.*}}(%rip), %ymm0, %ymm0
-; XOPAVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; XOPAVX2-NEXT:    vpcmov {{.*}}(%rip), %ymm1, %ymm0, %ymm0
 ; XOPAVX2-NEXT:    retq
   %res = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %x, <32 x i8> %y, <32 x i8> <i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4>)
   ret <32 x i8> %res
diff --git a/test/CodeGen/X86/vector-fshr-512.ll b/test/CodeGen/X86/vector-fshr-512.ll
index 50a4803b6bf9705df2beafe121a1053f9d46389c..063fd038f225323f2a541d6e5268b3009c0c8ffd 100644
--- a/test/CodeGen/X86/vector-fshr-512.ll
+++ b/test/CodeGen/X86/vector-fshr-512.ll
@@ -2,7 +2,9 @@
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f | FileCheck %s --check-prefixes=AVX512,AVX512F
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512vl | FileCheck %s --check-prefixes=AVX512,AVX512VL
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512bw | FileCheck %s --check-prefixes=AVX512,AVX512BW
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512vbmi2 | FileCheck %s --check-prefixes=AVX512,AVX512VBMI2
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512bw,+avx512vl | FileCheck %s --check-prefixes=AVX512,AVX512VLBW
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f,+avx512vbmi2,+avx512vl | FileCheck %s --check-prefixes=AVX512,AVX512VLVBMI2
 
 declare <8 x i64> @llvm.fshr.v8i64(<8 x i64>, <8 x i64>, <8 x i64>)
 declare <16 x i32> @llvm.fshr.v16i32(<16 x i32>, <16 x i32>, <16 x i32>)
@@ -14,35 +16,137 @@ declare <64 x i8> @llvm.fshr.v64i8(<64 x i8>, <64 x i8>, <64 x i8>)
 ;
 
 define <8 x i64> @var_funnnel_v8i64(<8 x i64> %x, <8 x i64> %y, <8 x i64> %amt) nounwind {
-; AVX512-LABEL: var_funnnel_v8i64:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpbroadcastq {{.*#+}} zmm3 = [63,63,63,63,63,63,63,63]
-; AVX512-NEXT:    vpandq %zmm3, %zmm2, %zmm4
-; AVX512-NEXT:    vpsrlvq %zmm4, %zmm1, %zmm5
-; AVX512-NEXT:    vpbroadcastq {{.*#+}} zmm6 = [64,64,64,64,64,64,64,64]
-; AVX512-NEXT:    vpsubq %zmm4, %zmm6, %zmm4
-; AVX512-NEXT:    vpsllvq %zmm4, %zmm0, %zmm0
-; AVX512-NEXT:    vporq %zmm5, %zmm0, %zmm0
-; AVX512-NEXT:    vptestnmq %zmm3, %zmm2, %k1
-; AVX512-NEXT:    vmovdqa64 %zmm1, %zmm0 {%k1}
-; AVX512-NEXT:    retq
+; AVX512F-LABEL: var_funnnel_v8i64:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpbroadcastq {{.*#+}} zmm3 = [63,63,63,63,63,63,63,63]
+; AVX512F-NEXT:    vpandq %zmm3, %zmm2, %zmm4
+; AVX512F-NEXT:    vpsrlvq %zmm4, %zmm1, %zmm5
+; AVX512F-NEXT:    vpbroadcastq {{.*#+}} zmm6 = [64,64,64,64,64,64,64,64]
+; AVX512F-NEXT:    vpsubq %zmm4, %zmm6, %zmm4
+; AVX512F-NEXT:    vpsllvq %zmm4, %zmm0, %zmm0
+; AVX512F-NEXT:    vporq %zmm5, %zmm0, %zmm0
+; AVX512F-NEXT:    vptestnmq %zmm3, %zmm2, %k1
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm0 {%k1}
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: var_funnnel_v8i64:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpbroadcastq {{.*#+}} zmm3 = [63,63,63,63,63,63,63,63]
+; AVX512VL-NEXT:    vpandq %zmm3, %zmm2, %zmm4
+; AVX512VL-NEXT:    vpsrlvq %zmm4, %zmm1, %zmm5
+; AVX512VL-NEXT:    vpbroadcastq {{.*#+}} zmm6 = [64,64,64,64,64,64,64,64]
+; AVX512VL-NEXT:    vpsubq %zmm4, %zmm6, %zmm4
+; AVX512VL-NEXT:    vpsllvq %zmm4, %zmm0, %zmm0
+; AVX512VL-NEXT:    vporq %zmm5, %zmm0, %zmm0
+; AVX512VL-NEXT:    vptestnmq %zmm3, %zmm2, %k1
+; AVX512VL-NEXT:    vmovdqa64 %zmm1, %zmm0 {%k1}
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: var_funnnel_v8i64:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpbroadcastq {{.*#+}} zmm3 = [63,63,63,63,63,63,63,63]
+; AVX512BW-NEXT:    vpandq %zmm3, %zmm2, %zmm4
+; AVX512BW-NEXT:    vpsrlvq %zmm4, %zmm1, %zmm5
+; AVX512BW-NEXT:    vpbroadcastq {{.*#+}} zmm6 = [64,64,64,64,64,64,64,64]
+; AVX512BW-NEXT:    vpsubq %zmm4, %zmm6, %zmm4
+; AVX512BW-NEXT:    vpsllvq %zmm4, %zmm0, %zmm0
+; AVX512BW-NEXT:    vporq %zmm5, %zmm0, %zmm0
+; AVX512BW-NEXT:    vptestnmq %zmm3, %zmm2, %k1
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm0 {%k1}
+; AVX512BW-NEXT:    retq
+;
+; AVX512VBMI2-LABEL: var_funnnel_v8i64:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    vpshrdvq %zmm2, %zmm0, %zmm1
+; AVX512VBMI2-NEXT:    vmovdqa64 %zmm1, %zmm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLBW-LABEL: var_funnnel_v8i64:
+; AVX512VLBW:       # %bb.0:
+; AVX512VLBW-NEXT:    vpbroadcastq {{.*#+}} zmm3 = [63,63,63,63,63,63,63,63]
+; AVX512VLBW-NEXT:    vpandq %zmm3, %zmm2, %zmm4
+; AVX512VLBW-NEXT:    vpsrlvq %zmm4, %zmm1, %zmm5
+; AVX512VLBW-NEXT:    vpbroadcastq {{.*#+}} zmm6 = [64,64,64,64,64,64,64,64]
+; AVX512VLBW-NEXT:    vpsubq %zmm4, %zmm6, %zmm4
+; AVX512VLBW-NEXT:    vpsllvq %zmm4, %zmm0, %zmm0
+; AVX512VLBW-NEXT:    vporq %zmm5, %zmm0, %zmm0
+; AVX512VLBW-NEXT:    vptestnmq %zmm3, %zmm2, %k1
+; AVX512VLBW-NEXT:    vmovdqa64 %zmm1, %zmm0 {%k1}
+; AVX512VLBW-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: var_funnnel_v8i64:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpshrdvq %zmm2, %zmm0, %zmm1
+; AVX512VLVBMI2-NEXT:    vmovdqa64 %zmm1, %zmm0
+; AVX512VLVBMI2-NEXT:    retq
   %res = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %x, <8 x i64> %y, <8 x i64> %amt)
   ret <8 x i64> %res
 }
 
 define <16 x i32> @var_funnnel_v16i32(<16 x i32> %x, <16 x i32> %y, <16 x i32> %amt) nounwind {
-; AVX512-LABEL: var_funnnel_v16i32:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpbroadcastd {{.*#+}} zmm3 = [31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31]
-; AVX512-NEXT:    vpandd %zmm3, %zmm2, %zmm4
-; AVX512-NEXT:    vpsrlvd %zmm4, %zmm1, %zmm5
-; AVX512-NEXT:    vpbroadcastd {{.*#+}} zmm6 = [32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32]
-; AVX512-NEXT:    vpsubd %zmm4, %zmm6, %zmm4
-; AVX512-NEXT:    vpsllvd %zmm4, %zmm0, %zmm0
-; AVX512-NEXT:    vpord %zmm5, %zmm0, %zmm0
-; AVX512-NEXT:    vptestnmd %zmm3, %zmm2, %k1
-; AVX512-NEXT:    vmovdqa32 %zmm1, %zmm0 {%k1}
-; AVX512-NEXT:    retq
+; AVX512F-LABEL: var_funnnel_v16i32:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpbroadcastd {{.*#+}} zmm3 = [31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31]
+; AVX512F-NEXT:    vpandd %zmm3, %zmm2, %zmm4
+; AVX512F-NEXT:    vpsrlvd %zmm4, %zmm1, %zmm5
+; AVX512F-NEXT:    vpbroadcastd {{.*#+}} zmm6 = [32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32]
+; AVX512F-NEXT:    vpsubd %zmm4, %zmm6, %zmm4
+; AVX512F-NEXT:    vpsllvd %zmm4, %zmm0, %zmm0
+; AVX512F-NEXT:    vpord %zmm5, %zmm0, %zmm0
+; AVX512F-NEXT:    vptestnmd %zmm3, %zmm2, %k1
+; AVX512F-NEXT:    vmovdqa32 %zmm1, %zmm0 {%k1}
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: var_funnnel_v16i32:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpbroadcastd {{.*#+}} zmm3 = [31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31]
+; AVX512VL-NEXT:    vpandd %zmm3, %zmm2, %zmm4
+; AVX512VL-NEXT:    vpsrlvd %zmm4, %zmm1, %zmm5
+; AVX512VL-NEXT:    vpbroadcastd {{.*#+}} zmm6 = [32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32]
+; AVX512VL-NEXT:    vpsubd %zmm4, %zmm6, %zmm4
+; AVX512VL-NEXT:    vpsllvd %zmm4, %zmm0, %zmm0
+; AVX512VL-NEXT:    vpord %zmm5, %zmm0, %zmm0
+; AVX512VL-NEXT:    vptestnmd %zmm3, %zmm2, %k1
+; AVX512VL-NEXT:    vmovdqa32 %zmm1, %zmm0 {%k1}
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: var_funnnel_v16i32:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpbroadcastd {{.*#+}} zmm3 = [31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31]
+; AVX512BW-NEXT:    vpandd %zmm3, %zmm2, %zmm4
+; AVX512BW-NEXT:    vpsrlvd %zmm4, %zmm1, %zmm5
+; AVX512BW-NEXT:    vpbroadcastd {{.*#+}} zmm6 = [32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32]
+; AVX512BW-NEXT:    vpsubd %zmm4, %zmm6, %zmm4
+; AVX512BW-NEXT:    vpsllvd %zmm4, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpord %zmm5, %zmm0, %zmm0
+; AVX512BW-NEXT:    vptestnmd %zmm3, %zmm2, %k1
+; AVX512BW-NEXT:    vmovdqa32 %zmm1, %zmm0 {%k1}
+; AVX512BW-NEXT:    retq
+;
+; AVX512VBMI2-LABEL: var_funnnel_v16i32:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    vpshrdvd %zmm2, %zmm0, %zmm1
+; AVX512VBMI2-NEXT:    vmovdqa64 %zmm1, %zmm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLBW-LABEL: var_funnnel_v16i32:
+; AVX512VLBW:       # %bb.0:
+; AVX512VLBW-NEXT:    vpbroadcastd {{.*#+}} zmm3 = [31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31]
+; AVX512VLBW-NEXT:    vpandd %zmm3, %zmm2, %zmm4
+; AVX512VLBW-NEXT:    vpsrlvd %zmm4, %zmm1, %zmm5
+; AVX512VLBW-NEXT:    vpbroadcastd {{.*#+}} zmm6 = [32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32]
+; AVX512VLBW-NEXT:    vpsubd %zmm4, %zmm6, %zmm4
+; AVX512VLBW-NEXT:    vpsllvd %zmm4, %zmm0, %zmm0
+; AVX512VLBW-NEXT:    vpord %zmm5, %zmm0, %zmm0
+; AVX512VLBW-NEXT:    vptestnmd %zmm3, %zmm2, %k1
+; AVX512VLBW-NEXT:    vmovdqa32 %zmm1, %zmm0 {%k1}
+; AVX512VLBW-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: var_funnnel_v16i32:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpshrdvd %zmm2, %zmm0, %zmm1
+; AVX512VLVBMI2-NEXT:    vmovdqa64 %zmm1, %zmm0
+; AVX512VLVBMI2-NEXT:    retq
   %res = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %x, <16 x i32> %y, <16 x i32> %amt)
   ret <16 x i32> %res
 }
@@ -123,6 +227,12 @@ define <32 x i16> @var_funnnel_v32i16(<32 x i16> %x, <32 x i16> %y, <32 x i16> %
 ; AVX512BW-NEXT:    vmovdqu16 %zmm1, %zmm0 {%k1}
 ; AVX512BW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: var_funnnel_v32i16:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    vpshrdvw %zmm2, %zmm0, %zmm1
+; AVX512VBMI2-NEXT:    vmovdqa64 %zmm1, %zmm0
+; AVX512VBMI2-NEXT:    retq
+;
 ; AVX512VLBW-LABEL: var_funnnel_v32i16:
 ; AVX512VLBW:       # %bb.0:
 ; AVX512VLBW-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
@@ -135,6 +245,12 @@ define <32 x i16> @var_funnnel_v32i16(<32 x i16> %x, <32 x i16> %y, <32 x i16> %
 ; AVX512VLBW-NEXT:    vptestnmw %zmm3, %zmm2, %k1
 ; AVX512VLBW-NEXT:    vmovdqu16 %zmm1, %zmm0 {%k1}
 ; AVX512VLBW-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: var_funnnel_v32i16:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpshrdvw %zmm2, %zmm0, %zmm1
+; AVX512VLVBMI2-NEXT:    vmovdqa64 %zmm1, %zmm0
+; AVX512VLVBMI2-NEXT:    retq
   %res = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %x, <32 x i16> %y, <32 x i16> %amt)
   ret <32 x i16> %res
 }
@@ -315,6 +431,45 @@ define <64 x i8> @var_funnnel_v64i8(<64 x i8> %x, <64 x i8> %y, <64 x i8> %amt)
 ; AVX512BW-NEXT:    vmovdqu8 %zmm1, %zmm0 {%k1}
 ; AVX512BW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: var_funnnel_v64i8:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
+; AVX512VBMI2-NEXT:    vpandq %zmm3, %zmm2, %zmm4
+; AVX512VBMI2-NEXT:    vpsllw $5, %zmm4, %zmm5
+; AVX512VBMI2-NEXT:    vpaddb %zmm5, %zmm5, %zmm6
+; AVX512VBMI2-NEXT:    vpmovb2m %zmm6, %k1
+; AVX512VBMI2-NEXT:    vpmovb2m %zmm5, %k2
+; AVX512VBMI2-NEXT:    vpsrlw $4, %zmm1, %zmm5
+; AVX512VBMI2-NEXT:    vpandq {{.*}}(%rip), %zmm5, %zmm5
+; AVX512VBMI2-NEXT:    vpblendmb %zmm5, %zmm1, %zmm5 {%k2}
+; AVX512VBMI2-NEXT:    vpsrlw $2, %zmm5, %zmm7
+; AVX512VBMI2-NEXT:    vpandq {{.*}}(%rip), %zmm7, %zmm7
+; AVX512VBMI2-NEXT:    vmovdqu8 %zmm7, %zmm5 {%k1}
+; AVX512VBMI2-NEXT:    vpsrlw $1, %zmm5, %zmm7
+; AVX512VBMI2-NEXT:    vpandq {{.*}}(%rip), %zmm7, %zmm7
+; AVX512VBMI2-NEXT:    vpaddb %zmm6, %zmm6, %zmm6
+; AVX512VBMI2-NEXT:    vpmovb2m %zmm6, %k1
+; AVX512VBMI2-NEXT:    vmovdqu8 %zmm7, %zmm5 {%k1}
+; AVX512VBMI2-NEXT:    vmovdqa64 {{.*#+}} zmm6 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; AVX512VBMI2-NEXT:    vpsubb %zmm4, %zmm6, %zmm4
+; AVX512VBMI2-NEXT:    vpsllw $5, %zmm4, %zmm4
+; AVX512VBMI2-NEXT:    vpaddb %zmm4, %zmm4, %zmm6
+; AVX512VBMI2-NEXT:    vpmovb2m %zmm6, %k1
+; AVX512VBMI2-NEXT:    vpmovb2m %zmm4, %k2
+; AVX512VBMI2-NEXT:    vpsllw $4, %zmm0, %zmm4
+; AVX512VBMI2-NEXT:    vpandq {{.*}}(%rip), %zmm4, %zmm4
+; AVX512VBMI2-NEXT:    vmovdqu8 %zmm4, %zmm0 {%k2}
+; AVX512VBMI2-NEXT:    vpsllw $2, %zmm0, %zmm4
+; AVX512VBMI2-NEXT:    vpandq {{.*}}(%rip), %zmm4, %zmm4
+; AVX512VBMI2-NEXT:    vmovdqu8 %zmm4, %zmm0 {%k1}
+; AVX512VBMI2-NEXT:    vpaddb %zmm6, %zmm6, %zmm4
+; AVX512VBMI2-NEXT:    vpmovb2m %zmm4, %k1
+; AVX512VBMI2-NEXT:    vpaddb %zmm0, %zmm0, %zmm0 {%k1}
+; AVX512VBMI2-NEXT:    vporq %zmm5, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    vptestnmb %zmm3, %zmm2, %k1
+; AVX512VBMI2-NEXT:    vmovdqu8 %zmm1, %zmm0 {%k1}
+; AVX512VBMI2-NEXT:    retq
+;
 ; AVX512VLBW-LABEL: var_funnnel_v64i8:
 ; AVX512VLBW:       # %bb.0:
 ; AVX512VLBW-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
@@ -353,6 +508,45 @@ define <64 x i8> @var_funnnel_v64i8(<64 x i8> %x, <64 x i8> %y, <64 x i8> %amt)
 ; AVX512VLBW-NEXT:    vptestnmb %zmm3, %zmm2, %k1
 ; AVX512VLBW-NEXT:    vmovdqu8 %zmm1, %zmm0 {%k1}
 ; AVX512VLBW-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: var_funnnel_v64i8:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
+; AVX512VLVBMI2-NEXT:    vpandq %zmm3, %zmm2, %zmm4
+; AVX512VLVBMI2-NEXT:    vpsllw $5, %zmm4, %zmm5
+; AVX512VLVBMI2-NEXT:    vpaddb %zmm5, %zmm5, %zmm6
+; AVX512VLVBMI2-NEXT:    vpmovb2m %zmm6, %k1
+; AVX512VLVBMI2-NEXT:    vpmovb2m %zmm5, %k2
+; AVX512VLVBMI2-NEXT:    vpsrlw $4, %zmm1, %zmm5
+; AVX512VLVBMI2-NEXT:    vpandq {{.*}}(%rip), %zmm5, %zmm5
+; AVX512VLVBMI2-NEXT:    vpblendmb %zmm5, %zmm1, %zmm5 {%k2}
+; AVX512VLVBMI2-NEXT:    vpsrlw $2, %zmm5, %zmm7
+; AVX512VLVBMI2-NEXT:    vpandq {{.*}}(%rip), %zmm7, %zmm7
+; AVX512VLVBMI2-NEXT:    vmovdqu8 %zmm7, %zmm5 {%k1}
+; AVX512VLVBMI2-NEXT:    vpsrlw $1, %zmm5, %zmm7
+; AVX512VLVBMI2-NEXT:    vpandq {{.*}}(%rip), %zmm7, %zmm7
+; AVX512VLVBMI2-NEXT:    vpaddb %zmm6, %zmm6, %zmm6
+; AVX512VLVBMI2-NEXT:    vpmovb2m %zmm6, %k1
+; AVX512VLVBMI2-NEXT:    vmovdqu8 %zmm7, %zmm5 {%k1}
+; AVX512VLVBMI2-NEXT:    vmovdqa64 {{.*#+}} zmm6 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; AVX512VLVBMI2-NEXT:    vpsubb %zmm4, %zmm6, %zmm4
+; AVX512VLVBMI2-NEXT:    vpsllw $5, %zmm4, %zmm4
+; AVX512VLVBMI2-NEXT:    vpaddb %zmm4, %zmm4, %zmm6
+; AVX512VLVBMI2-NEXT:    vpmovb2m %zmm6, %k1
+; AVX512VLVBMI2-NEXT:    vpmovb2m %zmm4, %k2
+; AVX512VLVBMI2-NEXT:    vpsllw $4, %zmm0, %zmm4
+; AVX512VLVBMI2-NEXT:    vpandq {{.*}}(%rip), %zmm4, %zmm4
+; AVX512VLVBMI2-NEXT:    vmovdqu8 %zmm4, %zmm0 {%k2}
+; AVX512VLVBMI2-NEXT:    vpsllw $2, %zmm0, %zmm4
+; AVX512VLVBMI2-NEXT:    vpandq {{.*}}(%rip), %zmm4, %zmm4
+; AVX512VLVBMI2-NEXT:    vmovdqu8 %zmm4, %zmm0 {%k1}
+; AVX512VLVBMI2-NEXT:    vpaddb %zmm6, %zmm6, %zmm4
+; AVX512VLVBMI2-NEXT:    vpmovb2m %zmm4, %k1
+; AVX512VLVBMI2-NEXT:    vpaddb %zmm0, %zmm0, %zmm0 {%k1}
+; AVX512VLVBMI2-NEXT:    vporq %zmm5, %zmm0, %zmm0
+; AVX512VLVBMI2-NEXT:    vptestnmb %zmm3, %zmm2, %k1
+; AVX512VLVBMI2-NEXT:    vmovdqu8 %zmm1, %zmm0 {%k1}
+; AVX512VLVBMI2-NEXT:    retq
   %res = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %x, <64 x i8> %y, <64 x i8> %amt)
   ret <64 x i8> %res
 }
@@ -362,40 +556,158 @@ define <64 x i8> @var_funnnel_v64i8(<64 x i8> %x, <64 x i8> %y, <64 x i8> %amt)
 ;
 
 define <8 x i64> @splatvar_funnnel_v8i64(<8 x i64> %x, <8 x i64> %y, <8 x i64> %amt) nounwind {
-; AVX512-LABEL: splatvar_funnnel_v8i64:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpbroadcastq %xmm2, %zmm2
-; AVX512-NEXT:    vpbroadcastq {{.*#+}} zmm3 = [63,63,63,63,63,63,63,63]
-; AVX512-NEXT:    vpandq %zmm3, %zmm2, %zmm4
-; AVX512-NEXT:    vpsrlq %xmm4, %zmm1, %zmm5
-; AVX512-NEXT:    vmovdqa {{.*#+}} xmm6 = [64,64]
-; AVX512-NEXT:    vpsubq %xmm4, %xmm6, %xmm4
-; AVX512-NEXT:    vpsllq %xmm4, %zmm0, %zmm0
-; AVX512-NEXT:    vporq %zmm5, %zmm0, %zmm0
-; AVX512-NEXT:    vptestnmq %zmm3, %zmm2, %k1
-; AVX512-NEXT:    vmovdqa64 %zmm1, %zmm0 {%k1}
-; AVX512-NEXT:    retq
+; AVX512F-LABEL: splatvar_funnnel_v8i64:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpbroadcastq %xmm2, %zmm2
+; AVX512F-NEXT:    vpbroadcastq {{.*#+}} zmm3 = [63,63,63,63,63,63,63,63]
+; AVX512F-NEXT:    vpandq %zmm3, %zmm2, %zmm4
+; AVX512F-NEXT:    vpsrlq %xmm4, %zmm1, %zmm5
+; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm6 = [64,64]
+; AVX512F-NEXT:    vpsubq %xmm4, %xmm6, %xmm4
+; AVX512F-NEXT:    vpsllq %xmm4, %zmm0, %zmm0
+; AVX512F-NEXT:    vporq %zmm5, %zmm0, %zmm0
+; AVX512F-NEXT:    vptestnmq %zmm3, %zmm2, %k1
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm0 {%k1}
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: splatvar_funnnel_v8i64:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpbroadcastq %xmm2, %zmm2
+; AVX512VL-NEXT:    vpbroadcastq {{.*#+}} zmm3 = [63,63,63,63,63,63,63,63]
+; AVX512VL-NEXT:    vpandq %zmm3, %zmm2, %zmm4
+; AVX512VL-NEXT:    vpsrlq %xmm4, %zmm1, %zmm5
+; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm6 = [64,64]
+; AVX512VL-NEXT:    vpsubq %xmm4, %xmm6, %xmm4
+; AVX512VL-NEXT:    vpsllq %xmm4, %zmm0, %zmm0
+; AVX512VL-NEXT:    vporq %zmm5, %zmm0, %zmm0
+; AVX512VL-NEXT:    vptestnmq %zmm3, %zmm2, %k1
+; AVX512VL-NEXT:    vmovdqa64 %zmm1, %zmm0 {%k1}
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: splatvar_funnnel_v8i64:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpbroadcastq %xmm2, %zmm2
+; AVX512BW-NEXT:    vpbroadcastq {{.*#+}} zmm3 = [63,63,63,63,63,63,63,63]
+; AVX512BW-NEXT:    vpandq %zmm3, %zmm2, %zmm4
+; AVX512BW-NEXT:    vpsrlq %xmm4, %zmm1, %zmm5
+; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm6 = [64,64]
+; AVX512BW-NEXT:    vpsubq %xmm4, %xmm6, %xmm4
+; AVX512BW-NEXT:    vpsllq %xmm4, %zmm0, %zmm0
+; AVX512BW-NEXT:    vporq %zmm5, %zmm0, %zmm0
+; AVX512BW-NEXT:    vptestnmq %zmm3, %zmm2, %k1
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm0 {%k1}
+; AVX512BW-NEXT:    retq
+;
+; AVX512VBMI2-LABEL: splatvar_funnnel_v8i64:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    vpbroadcastq %xmm2, %zmm2
+; AVX512VBMI2-NEXT:    vpshrdvq %zmm2, %zmm0, %zmm1
+; AVX512VBMI2-NEXT:    vmovdqa64 %zmm1, %zmm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLBW-LABEL: splatvar_funnnel_v8i64:
+; AVX512VLBW:       # %bb.0:
+; AVX512VLBW-NEXT:    vpbroadcastq %xmm2, %zmm2
+; AVX512VLBW-NEXT:    vpbroadcastq {{.*#+}} zmm3 = [63,63,63,63,63,63,63,63]
+; AVX512VLBW-NEXT:    vpandq %zmm3, %zmm2, %zmm4
+; AVX512VLBW-NEXT:    vpsrlq %xmm4, %zmm1, %zmm5
+; AVX512VLBW-NEXT:    vmovdqa {{.*#+}} xmm6 = [64,64]
+; AVX512VLBW-NEXT:    vpsubq %xmm4, %xmm6, %xmm4
+; AVX512VLBW-NEXT:    vpsllq %xmm4, %zmm0, %zmm0
+; AVX512VLBW-NEXT:    vporq %zmm5, %zmm0, %zmm0
+; AVX512VLBW-NEXT:    vptestnmq %zmm3, %zmm2, %k1
+; AVX512VLBW-NEXT:    vmovdqa64 %zmm1, %zmm0 {%k1}
+; AVX512VLBW-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: splatvar_funnnel_v8i64:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpbroadcastq %xmm2, %zmm2
+; AVX512VLVBMI2-NEXT:    vpshrdvq %zmm2, %zmm0, %zmm1
+; AVX512VLVBMI2-NEXT:    vmovdqa64 %zmm1, %zmm0
+; AVX512VLVBMI2-NEXT:    retq
   %splat = shufflevector <8 x i64> %amt, <8 x i64> undef, <8 x i32> zeroinitializer
   %res = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %x, <8 x i64> %y, <8 x i64> %splat)
   ret <8 x i64> %res
 }
 
 define <16 x i32> @splatvar_funnnel_v16i32(<16 x i32> %x, <16 x i32> %y, <16 x i32> %amt) nounwind {
-; AVX512-LABEL: splatvar_funnnel_v16i32:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpbroadcastd %xmm2, %zmm2
-; AVX512-NEXT:    vpbroadcastd {{.*#+}} zmm3 = [31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31]
-; AVX512-NEXT:    vpandd %zmm3, %zmm2, %zmm4
-; AVX512-NEXT:    vpmovzxdq {{.*#+}} xmm5 = xmm4[0],zero,xmm4[1],zero
-; AVX512-NEXT:    vpsrld %xmm5, %zmm1, %zmm5
-; AVX512-NEXT:    vpbroadcastd {{.*#+}} xmm6 = [32,32,32,32]
-; AVX512-NEXT:    vpsubd %xmm4, %xmm6, %xmm4
-; AVX512-NEXT:    vpmovzxdq {{.*#+}} xmm4 = xmm4[0],zero,xmm4[1],zero
-; AVX512-NEXT:    vpslld %xmm4, %zmm0, %zmm0
-; AVX512-NEXT:    vpord %zmm5, %zmm0, %zmm0
-; AVX512-NEXT:    vptestnmd %zmm3, %zmm2, %k1
-; AVX512-NEXT:    vmovdqa32 %zmm1, %zmm0 {%k1}
-; AVX512-NEXT:    retq
+; AVX512F-LABEL: splatvar_funnnel_v16i32:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpbroadcastd %xmm2, %zmm2
+; AVX512F-NEXT:    vpbroadcastd {{.*#+}} zmm3 = [31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31]
+; AVX512F-NEXT:    vpandd %zmm3, %zmm2, %zmm4
+; AVX512F-NEXT:    vpmovzxdq {{.*#+}} xmm5 = xmm4[0],zero,xmm4[1],zero
+; AVX512F-NEXT:    vpsrld %xmm5, %zmm1, %zmm5
+; AVX512F-NEXT:    vpbroadcastd {{.*#+}} xmm6 = [32,32,32,32]
+; AVX512F-NEXT:    vpsubd %xmm4, %xmm6, %xmm4
+; AVX512F-NEXT:    vpmovzxdq {{.*#+}} xmm4 = xmm4[0],zero,xmm4[1],zero
+; AVX512F-NEXT:    vpslld %xmm4, %zmm0, %zmm0
+; AVX512F-NEXT:    vpord %zmm5, %zmm0, %zmm0
+; AVX512F-NEXT:    vptestnmd %zmm3, %zmm2, %k1
+; AVX512F-NEXT:    vmovdqa32 %zmm1, %zmm0 {%k1}
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: splatvar_funnnel_v16i32:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpbroadcastd %xmm2, %zmm2
+; AVX512VL-NEXT:    vpbroadcastd {{.*#+}} zmm3 = [31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31]
+; AVX512VL-NEXT:    vpandd %zmm3, %zmm2, %zmm4
+; AVX512VL-NEXT:    vpmovzxdq {{.*#+}} xmm5 = xmm4[0],zero,xmm4[1],zero
+; AVX512VL-NEXT:    vpsrld %xmm5, %zmm1, %zmm5
+; AVX512VL-NEXT:    vpbroadcastd {{.*#+}} xmm6 = [32,32,32,32]
+; AVX512VL-NEXT:    vpsubd %xmm4, %xmm6, %xmm4
+; AVX512VL-NEXT:    vpmovzxdq {{.*#+}} xmm4 = xmm4[0],zero,xmm4[1],zero
+; AVX512VL-NEXT:    vpslld %xmm4, %zmm0, %zmm0
+; AVX512VL-NEXT:    vpord %zmm5, %zmm0, %zmm0
+; AVX512VL-NEXT:    vptestnmd %zmm3, %zmm2, %k1
+; AVX512VL-NEXT:    vmovdqa32 %zmm1, %zmm0 {%k1}
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: splatvar_funnnel_v16i32:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpbroadcastd %xmm2, %zmm2
+; AVX512BW-NEXT:    vpbroadcastd {{.*#+}} zmm3 = [31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31]
+; AVX512BW-NEXT:    vpandd %zmm3, %zmm2, %zmm4
+; AVX512BW-NEXT:    vpmovzxdq {{.*#+}} xmm5 = xmm4[0],zero,xmm4[1],zero
+; AVX512BW-NEXT:    vpsrld %xmm5, %zmm1, %zmm5
+; AVX512BW-NEXT:    vpbroadcastd {{.*#+}} xmm6 = [32,32,32,32]
+; AVX512BW-NEXT:    vpsubd %xmm4, %xmm6, %xmm4
+; AVX512BW-NEXT:    vpmovzxdq {{.*#+}} xmm4 = xmm4[0],zero,xmm4[1],zero
+; AVX512BW-NEXT:    vpslld %xmm4, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpord %zmm5, %zmm0, %zmm0
+; AVX512BW-NEXT:    vptestnmd %zmm3, %zmm2, %k1
+; AVX512BW-NEXT:    vmovdqa32 %zmm1, %zmm0 {%k1}
+; AVX512BW-NEXT:    retq
+;
+; AVX512VBMI2-LABEL: splatvar_funnnel_v16i32:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    vpbroadcastd %xmm2, %zmm2
+; AVX512VBMI2-NEXT:    vpshrdvd %zmm2, %zmm0, %zmm1
+; AVX512VBMI2-NEXT:    vmovdqa64 %zmm1, %zmm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLBW-LABEL: splatvar_funnnel_v16i32:
+; AVX512VLBW:       # %bb.0:
+; AVX512VLBW-NEXT:    vpbroadcastd %xmm2, %zmm2
+; AVX512VLBW-NEXT:    vpbroadcastd {{.*#+}} zmm3 = [31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31]
+; AVX512VLBW-NEXT:    vpandd %zmm3, %zmm2, %zmm4
+; AVX512VLBW-NEXT:    vpmovzxdq {{.*#+}} xmm5 = xmm4[0],zero,xmm4[1],zero
+; AVX512VLBW-NEXT:    vpsrld %xmm5, %zmm1, %zmm5
+; AVX512VLBW-NEXT:    vpbroadcastd {{.*#+}} xmm6 = [32,32,32,32]
+; AVX512VLBW-NEXT:    vpsubd %xmm4, %xmm6, %xmm4
+; AVX512VLBW-NEXT:    vpmovzxdq {{.*#+}} xmm4 = xmm4[0],zero,xmm4[1],zero
+; AVX512VLBW-NEXT:    vpslld %xmm4, %zmm0, %zmm0
+; AVX512VLBW-NEXT:    vpord %zmm5, %zmm0, %zmm0
+; AVX512VLBW-NEXT:    vptestnmd %zmm3, %zmm2, %k1
+; AVX512VLBW-NEXT:    vmovdqa32 %zmm1, %zmm0 {%k1}
+; AVX512VLBW-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: splatvar_funnnel_v16i32:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpbroadcastd %xmm2, %zmm2
+; AVX512VLVBMI2-NEXT:    vpshrdvd %zmm2, %zmm0, %zmm1
+; AVX512VLVBMI2-NEXT:    vmovdqa64 %zmm1, %zmm0
+; AVX512VLVBMI2-NEXT:    retq
   %splat = shufflevector <16 x i32> %amt, <16 x i32> undef, <16 x i32> zeroinitializer
   %res = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %x, <16 x i32> %y, <16 x i32> %splat)
   ret <16 x i32> %res
@@ -458,6 +770,13 @@ define <32 x i16> @splatvar_funnnel_v32i16(<32 x i16> %x, <32 x i16> %y, <32 x i
 ; AVX512BW-NEXT:    vmovdqu16 %zmm1, %zmm0 {%k1}
 ; AVX512BW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: splatvar_funnnel_v32i16:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    vpbroadcastw %xmm2, %zmm2
+; AVX512VBMI2-NEXT:    vpshrdvw %zmm2, %zmm0, %zmm1
+; AVX512VBMI2-NEXT:    vmovdqa64 %zmm1, %zmm0
+; AVX512VBMI2-NEXT:    retq
+;
 ; AVX512VLBW-LABEL: splatvar_funnnel_v32i16:
 ; AVX512VLBW:       # %bb.0:
 ; AVX512VLBW-NEXT:    vpbroadcastw %xmm2, %zmm2
@@ -473,6 +792,13 @@ define <32 x i16> @splatvar_funnnel_v32i16(<32 x i16> %x, <32 x i16> %y, <32 x i
 ; AVX512VLBW-NEXT:    vptestnmw %zmm3, %zmm2, %k1
 ; AVX512VLBW-NEXT:    vmovdqu16 %zmm1, %zmm0 {%k1}
 ; AVX512VLBW-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: splatvar_funnnel_v32i16:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpbroadcastw %xmm2, %zmm2
+; AVX512VLVBMI2-NEXT:    vpshrdvw %zmm2, %zmm0, %zmm1
+; AVX512VLVBMI2-NEXT:    vmovdqa64 %zmm1, %zmm0
+; AVX512VLVBMI2-NEXT:    retq
   %splat = shufflevector <32 x i16> %amt, <32 x i16> undef, <32 x i32> zeroinitializer
   %res = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %x, <32 x i16> %y, <32 x i16> %splat)
   ret <32 x i16> %res
@@ -563,6 +889,30 @@ define <64 x i8> @splatvar_funnnel_v64i8(<64 x i8> %x, <64 x i8> %y, <64 x i8> %
 ; AVX512BW-NEXT:    vmovdqu8 %zmm1, %zmm0 {%k1}
 ; AVX512BW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: splatvar_funnnel_v64i8:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    vpbroadcastb %xmm2, %zmm2
+; AVX512VBMI2-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
+; AVX512VBMI2-NEXT:    vpandq %zmm3, %zmm2, %zmm4
+; AVX512VBMI2-NEXT:    vpmovzxbq {{.*#+}} xmm5 = xmm4[0],zero,zero,zero,zero,zero,zero,zero,xmm4[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512VBMI2-NEXT:    vpsrlw %xmm5, %zmm1, %zmm6
+; AVX512VBMI2-NEXT:    vpternlogd $255, %zmm7, %zmm7, %zmm7
+; AVX512VBMI2-NEXT:    vpsrlw %xmm5, %zmm7, %zmm5
+; AVX512VBMI2-NEXT:    vpsrlw $8, %zmm5, %zmm5
+; AVX512VBMI2-NEXT:    vpbroadcastb %xmm5, %zmm5
+; AVX512VBMI2-NEXT:    vpandq %zmm5, %zmm6, %zmm5
+; AVX512VBMI2-NEXT:    vmovdqa {{.*#+}} xmm6 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; AVX512VBMI2-NEXT:    vpsubb %xmm4, %xmm6, %xmm4
+; AVX512VBMI2-NEXT:    vpmovzxbq {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,zero,zero,zero,zero,xmm4[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512VBMI2-NEXT:    vpsllw %xmm4, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    vpsllw %xmm4, %zmm7, %zmm4
+; AVX512VBMI2-NEXT:    vpbroadcastb %xmm4, %zmm4
+; AVX512VBMI2-NEXT:    vpandq %zmm4, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    vporq %zmm5, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    vptestnmb %zmm3, %zmm2, %k1
+; AVX512VBMI2-NEXT:    vmovdqu8 %zmm1, %zmm0 {%k1}
+; AVX512VBMI2-NEXT:    retq
+;
 ; AVX512VLBW-LABEL: splatvar_funnnel_v64i8:
 ; AVX512VLBW:       # %bb.0:
 ; AVX512VLBW-NEXT:    vpbroadcastb %xmm2, %zmm2
@@ -586,6 +936,30 @@ define <64 x i8> @splatvar_funnnel_v64i8(<64 x i8> %x, <64 x i8> %y, <64 x i8> %
 ; AVX512VLBW-NEXT:    vptestnmb %zmm3, %zmm2, %k1
 ; AVX512VLBW-NEXT:    vmovdqu8 %zmm1, %zmm0 {%k1}
 ; AVX512VLBW-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: splatvar_funnnel_v64i8:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpbroadcastb %xmm2, %zmm2
+; AVX512VLVBMI2-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
+; AVX512VLVBMI2-NEXT:    vpandq %zmm3, %zmm2, %zmm4
+; AVX512VLVBMI2-NEXT:    vpmovzxbq {{.*#+}} xmm5 = xmm4[0],zero,zero,zero,zero,zero,zero,zero,xmm4[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512VLVBMI2-NEXT:    vpsrlw %xmm5, %zmm1, %zmm6
+; AVX512VLVBMI2-NEXT:    vpternlogd $255, %zmm7, %zmm7, %zmm7
+; AVX512VLVBMI2-NEXT:    vpsrlw %xmm5, %zmm7, %zmm5
+; AVX512VLVBMI2-NEXT:    vpsrlw $8, %zmm5, %zmm5
+; AVX512VLVBMI2-NEXT:    vpbroadcastb %xmm5, %zmm5
+; AVX512VLVBMI2-NEXT:    vpandq %zmm5, %zmm6, %zmm5
+; AVX512VLVBMI2-NEXT:    vmovdqa {{.*#+}} xmm6 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; AVX512VLVBMI2-NEXT:    vpsubb %xmm4, %xmm6, %xmm4
+; AVX512VLVBMI2-NEXT:    vpmovzxbq {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,zero,zero,zero,zero,xmm4[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512VLVBMI2-NEXT:    vpsllw %xmm4, %zmm0, %zmm0
+; AVX512VLVBMI2-NEXT:    vpsllw %xmm4, %zmm7, %zmm4
+; AVX512VLVBMI2-NEXT:    vpbroadcastb %xmm4, %zmm4
+; AVX512VLVBMI2-NEXT:    vpandq %zmm4, %zmm0, %zmm0
+; AVX512VLVBMI2-NEXT:    vporq %zmm5, %zmm0, %zmm0
+; AVX512VLVBMI2-NEXT:    vptestnmb %zmm3, %zmm2, %k1
+; AVX512VLVBMI2-NEXT:    vmovdqu8 %zmm1, %zmm0 {%k1}
+; AVX512VLVBMI2-NEXT:    retq
   %splat = shufflevector <64 x i8> %amt, <64 x i8> undef, <64 x i32> zeroinitializer
   %res = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %x, <64 x i8> %y, <64 x i8> %splat)
   ret <64 x i8> %res
@@ -596,23 +970,89 @@ define <64 x i8> @splatvar_funnnel_v64i8(<64 x i8> %x, <64 x i8> %y, <64 x i8> %
 ;
 
 define <8 x i64> @constant_funnnel_v8i64(<8 x i64> %x, <8 x i64> %y) nounwind {
-; AVX512-LABEL: constant_funnnel_v8i64:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpsrlvq {{.*}}(%rip), %zmm1, %zmm1
-; AVX512-NEXT:    vpsllvq {{.*}}(%rip), %zmm0, %zmm0
-; AVX512-NEXT:    vporq %zmm1, %zmm0, %zmm0
-; AVX512-NEXT:    retq
+; AVX512F-LABEL: constant_funnnel_v8i64:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpsrlvq {{.*}}(%rip), %zmm1, %zmm1
+; AVX512F-NEXT:    vpsllvq {{.*}}(%rip), %zmm0, %zmm0
+; AVX512F-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: constant_funnnel_v8i64:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpsrlvq {{.*}}(%rip), %zmm1, %zmm1
+; AVX512VL-NEXT:    vpsllvq {{.*}}(%rip), %zmm0, %zmm0
+; AVX512VL-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: constant_funnnel_v8i64:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpsrlvq {{.*}}(%rip), %zmm1, %zmm1
+; AVX512BW-NEXT:    vpsllvq {{.*}}(%rip), %zmm0, %zmm0
+; AVX512BW-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    retq
+;
+; AVX512VBMI2-LABEL: constant_funnnel_v8i64:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    vpshrdvq {{.*}}(%rip), %zmm0, %zmm1
+; AVX512VBMI2-NEXT:    vmovdqa64 %zmm1, %zmm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLBW-LABEL: constant_funnnel_v8i64:
+; AVX512VLBW:       # %bb.0:
+; AVX512VLBW-NEXT:    vpsrlvq {{.*}}(%rip), %zmm1, %zmm1
+; AVX512VLBW-NEXT:    vpsllvq {{.*}}(%rip), %zmm0, %zmm0
+; AVX512VLBW-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512VLBW-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: constant_funnnel_v8i64:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpshrdvq {{.*}}(%rip), %zmm0, %zmm1
+; AVX512VLVBMI2-NEXT:    vmovdqa64 %zmm1, %zmm0
+; AVX512VLVBMI2-NEXT:    retq
   %res = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %x, <8 x i64> %y, <8 x i64> <i64 4, i64 14, i64 50, i64 60, i64 4, i64 14, i64 50, i64 60>)
   ret <8 x i64> %res
 }
 
 define <16 x i32> @constant_funnnel_v16i32(<16 x i32> %x, <16 x i32> %y) nounwind {
-; AVX512-LABEL: constant_funnnel_v16i32:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpsrlvd {{.*}}(%rip), %zmm1, %zmm1
-; AVX512-NEXT:    vpsllvd {{.*}}(%rip), %zmm0, %zmm0
-; AVX512-NEXT:    vpord %zmm1, %zmm0, %zmm0
-; AVX512-NEXT:    retq
+; AVX512F-LABEL: constant_funnnel_v16i32:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpsrlvd {{.*}}(%rip), %zmm1, %zmm1
+; AVX512F-NEXT:    vpsllvd {{.*}}(%rip), %zmm0, %zmm0
+; AVX512F-NEXT:    vpord %zmm1, %zmm0, %zmm0
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: constant_funnnel_v16i32:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpsrlvd {{.*}}(%rip), %zmm1, %zmm1
+; AVX512VL-NEXT:    vpsllvd {{.*}}(%rip), %zmm0, %zmm0
+; AVX512VL-NEXT:    vpord %zmm1, %zmm0, %zmm0
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: constant_funnnel_v16i32:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpsrlvd {{.*}}(%rip), %zmm1, %zmm1
+; AVX512BW-NEXT:    vpsllvd {{.*}}(%rip), %zmm0, %zmm0
+; AVX512BW-NEXT:    vpord %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    retq
+;
+; AVX512VBMI2-LABEL: constant_funnnel_v16i32:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    vpshrdvd {{.*}}(%rip), %zmm0, %zmm1
+; AVX512VBMI2-NEXT:    vmovdqa64 %zmm1, %zmm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLBW-LABEL: constant_funnnel_v16i32:
+; AVX512VLBW:       # %bb.0:
+; AVX512VLBW-NEXT:    vpsrlvd {{.*}}(%rip), %zmm1, %zmm1
+; AVX512VLBW-NEXT:    vpsllvd {{.*}}(%rip), %zmm0, %zmm0
+; AVX512VLBW-NEXT:    vpord %zmm1, %zmm0, %zmm0
+; AVX512VLBW-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: constant_funnnel_v16i32:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpshrdvd {{.*}}(%rip), %zmm0, %zmm1
+; AVX512VLVBMI2-NEXT:    vmovdqa64 %zmm1, %zmm0
+; AVX512VLVBMI2-NEXT:    retq
   %res = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %x, <16 x i32> %y, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>)
   ret <16 x i32> %res
 }
@@ -666,6 +1106,12 @@ define <32 x i16> @constant_funnnel_v32i16(<32 x i16> %x, <32 x i16> %y) nounwin
 ; AVX512BW-NEXT:    vmovdqu16 %zmm1, %zmm0 {%k1}
 ; AVX512BW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: constant_funnnel_v32i16:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    vpshrdvw {{.*}}(%rip), %zmm0, %zmm1
+; AVX512VBMI2-NEXT:    vmovdqa64 %zmm1, %zmm0
+; AVX512VBMI2-NEXT:    retq
+;
 ; AVX512VLBW-LABEL: constant_funnnel_v32i16:
 ; AVX512VLBW:       # %bb.0:
 ; AVX512VLBW-NEXT:    vpsrlvw {{.*}}(%rip), %zmm1, %zmm2
@@ -675,6 +1121,12 @@ define <32 x i16> @constant_funnnel_v32i16(<32 x i16> %x, <32 x i16> %y) nounwin
 ; AVX512VLBW-NEXT:    kmovd %eax, %k1
 ; AVX512VLBW-NEXT:    vmovdqu16 %zmm1, %zmm0 {%k1}
 ; AVX512VLBW-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: constant_funnnel_v32i16:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpshrdvw {{.*}}(%rip), %zmm0, %zmm1
+; AVX512VLVBMI2-NEXT:    vmovdqa64 %zmm1, %zmm0
+; AVX512VLVBMI2-NEXT:    retq
   %res = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %x, <32 x i16> %y, <32 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15, i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15>)
   ret <32 x i16> %res
 }
@@ -811,6 +1263,36 @@ define <64 x i8> @constant_funnnel_v64i8(<64 x i8> %x, <64 x i8> %y) nounwind {
 ; AVX512BW-NEXT:    vmovdqu8 %zmm1, %zmm0 {%k1}
 ; AVX512BW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: constant_funnnel_v64i8:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [57600,41152,24704,8256,8448,24640,41088,57536,57600,41152,24704,8256,8448,24640,41088,57536,57600,41152,24704,8256,8448,24640,41088,57536,57600,41152,24704,8256,8448,24640,41088,57536]
+; AVX512VBMI2-NEXT:    vpmovb2m %zmm2, %k1
+; AVX512VBMI2-NEXT:    vpsllw $4, %zmm0, %zmm3
+; AVX512VBMI2-NEXT:    vpandq {{.*}}(%rip), %zmm3, %zmm3
+; AVX512VBMI2-NEXT:    vmovdqu8 %zmm3, %zmm0 {%k1}
+; AVX512VBMI2-NEXT:    vpsllw $2, %zmm0, %zmm3
+; AVX512VBMI2-NEXT:    vpandq {{.*}}(%rip), %zmm3, %zmm3
+; AVX512VBMI2-NEXT:    vpaddb %zmm2, %zmm2, %zmm2
+; AVX512VBMI2-NEXT:    vpmovb2m %zmm2, %k1
+; AVX512VBMI2-NEXT:    vmovdqu8 %zmm3, %zmm0 {%k1}
+; AVX512VBMI2-NEXT:    vpaddb %zmm2, %zmm2, %zmm2
+; AVX512VBMI2-NEXT:    vpmovb2m %zmm2, %k1
+; AVX512VBMI2-NEXT:    vpaddb %zmm0, %zmm0, %zmm0 {%k1}
+; AVX512VBMI2-NEXT:    vpunpckhbw {{.*#+}} zmm2 = zmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]
+; AVX512VBMI2-NEXT:    vpsrlw $8, %zmm2, %zmm2
+; AVX512VBMI2-NEXT:    vpsllvw {{.*}}(%rip), %zmm2, %zmm2
+; AVX512VBMI2-NEXT:    vpsrlw $8, %zmm2, %zmm2
+; AVX512VBMI2-NEXT:    vpunpcklbw {{.*#+}} zmm3 = zmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
+; AVX512VBMI2-NEXT:    vpsrlw $8, %zmm3, %zmm3
+; AVX512VBMI2-NEXT:    vpsllvw {{.*}}(%rip), %zmm3, %zmm3
+; AVX512VBMI2-NEXT:    vpsrlw $8, %zmm3, %zmm3
+; AVX512VBMI2-NEXT:    vpackuswb %zmm2, %zmm3, %zmm2
+; AVX512VBMI2-NEXT:    vporq %zmm2, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    movabsq $72340172838076673, %rax # imm = 0x101010101010101
+; AVX512VBMI2-NEXT:    kmovq %rax, %k1
+; AVX512VBMI2-NEXT:    vmovdqu8 %zmm1, %zmm0 {%k1}
+; AVX512VBMI2-NEXT:    retq
+;
 ; AVX512VLBW-LABEL: constant_funnnel_v64i8:
 ; AVX512VLBW:       # %bb.0:
 ; AVX512VLBW-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [57600,41152,24704,8256,8448,24640,41088,57536,57600,41152,24704,8256,8448,24640,41088,57536,57600,41152,24704,8256,8448,24640,41088,57536,57600,41152,24704,8256,8448,24640,41088,57536]
@@ -840,6 +1322,36 @@ define <64 x i8> @constant_funnnel_v64i8(<64 x i8> %x, <64 x i8> %y) nounwind {
 ; AVX512VLBW-NEXT:    kmovq %rax, %k1
 ; AVX512VLBW-NEXT:    vmovdqu8 %zmm1, %zmm0 {%k1}
 ; AVX512VLBW-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: constant_funnnel_v64i8:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [57600,41152,24704,8256,8448,24640,41088,57536,57600,41152,24704,8256,8448,24640,41088,57536,57600,41152,24704,8256,8448,24640,41088,57536,57600,41152,24704,8256,8448,24640,41088,57536]
+; AVX512VLVBMI2-NEXT:    vpmovb2m %zmm2, %k1
+; AVX512VLVBMI2-NEXT:    vpsllw $4, %zmm0, %zmm3
+; AVX512VLVBMI2-NEXT:    vpandq {{.*}}(%rip), %zmm3, %zmm3
+; AVX512VLVBMI2-NEXT:    vmovdqu8 %zmm3, %zmm0 {%k1}
+; AVX512VLVBMI2-NEXT:    vpsllw $2, %zmm0, %zmm3
+; AVX512VLVBMI2-NEXT:    vpandq {{.*}}(%rip), %zmm3, %zmm3
+; AVX512VLVBMI2-NEXT:    vpaddb %zmm2, %zmm2, %zmm2
+; AVX512VLVBMI2-NEXT:    vpmovb2m %zmm2, %k1
+; AVX512VLVBMI2-NEXT:    vmovdqu8 %zmm3, %zmm0 {%k1}
+; AVX512VLVBMI2-NEXT:    vpaddb %zmm2, %zmm2, %zmm2
+; AVX512VLVBMI2-NEXT:    vpmovb2m %zmm2, %k1
+; AVX512VLVBMI2-NEXT:    vpaddb %zmm0, %zmm0, %zmm0 {%k1}
+; AVX512VLVBMI2-NEXT:    vpunpckhbw {{.*#+}} zmm2 = zmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]
+; AVX512VLVBMI2-NEXT:    vpsrlw $8, %zmm2, %zmm2
+; AVX512VLVBMI2-NEXT:    vpsllvw {{.*}}(%rip), %zmm2, %zmm2
+; AVX512VLVBMI2-NEXT:    vpsrlw $8, %zmm2, %zmm2
+; AVX512VLVBMI2-NEXT:    vpunpcklbw {{.*#+}} zmm3 = zmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
+; AVX512VLVBMI2-NEXT:    vpsrlw $8, %zmm3, %zmm3
+; AVX512VLVBMI2-NEXT:    vpsllvw {{.*}}(%rip), %zmm3, %zmm3
+; AVX512VLVBMI2-NEXT:    vpsrlw $8, %zmm3, %zmm3
+; AVX512VLVBMI2-NEXT:    vpackuswb %zmm2, %zmm3, %zmm2
+; AVX512VLVBMI2-NEXT:    vporq %zmm2, %zmm0, %zmm0
+; AVX512VLVBMI2-NEXT:    movabsq $72340172838076673, %rax # imm = 0x101010101010101
+; AVX512VLVBMI2-NEXT:    kmovq %rax, %k1
+; AVX512VLVBMI2-NEXT:    vmovdqu8 %zmm1, %zmm0 {%k1}
+; AVX512VLVBMI2-NEXT:    retq
   %res = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %x, <64 x i8> %y, <64 x i8> <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1>)
   ret <64 x i8> %res
 }
@@ -849,23 +1361,85 @@ define <64 x i8> @constant_funnnel_v64i8(<64 x i8> %x, <64 x i8> %y) nounwind {
 ;
 
 define <8 x i64> @splatconstant_funnnel_v8i64(<8 x i64> %x, <8 x i64> %y) nounwind {
-; AVX512-LABEL: splatconstant_funnnel_v8i64:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpsrlq $14, %zmm1, %zmm1
-; AVX512-NEXT:    vpsllq $50, %zmm0, %zmm0
-; AVX512-NEXT:    vporq %zmm1, %zmm0, %zmm0
-; AVX512-NEXT:    retq
+; AVX512F-LABEL: splatconstant_funnnel_v8i64:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpsrlq $14, %zmm1, %zmm1
+; AVX512F-NEXT:    vpsllq $50, %zmm0, %zmm0
+; AVX512F-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: splatconstant_funnnel_v8i64:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpsrlq $14, %zmm1, %zmm1
+; AVX512VL-NEXT:    vpsllq $50, %zmm0, %zmm0
+; AVX512VL-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: splatconstant_funnnel_v8i64:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpsrlq $14, %zmm1, %zmm1
+; AVX512BW-NEXT:    vpsllq $50, %zmm0, %zmm0
+; AVX512BW-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    retq
+;
+; AVX512VBMI2-LABEL: splatconstant_funnnel_v8i64:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    vpshrdq $14, %zmm0, %zmm1, %zmm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLBW-LABEL: splatconstant_funnnel_v8i64:
+; AVX512VLBW:       # %bb.0:
+; AVX512VLBW-NEXT:    vpsrlq $14, %zmm1, %zmm1
+; AVX512VLBW-NEXT:    vpsllq $50, %zmm0, %zmm0
+; AVX512VLBW-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512VLBW-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: splatconstant_funnnel_v8i64:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpshrdq $14, %zmm0, %zmm1, %zmm0
+; AVX512VLVBMI2-NEXT:    retq
   %res = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %x, <8 x i64> %y, <8 x i64> <i64 14, i64 14, i64 14, i64 14, i64 14, i64 14, i64 14, i64 14>)
   ret <8 x i64> %res
 }
 
 define <16 x i32> @splatconstant_funnnel_v16i32(<16 x i32> %x, <16 x i32> %y) nounwind {
-; AVX512-LABEL: splatconstant_funnnel_v16i32:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpsrld $4, %zmm1, %zmm1
-; AVX512-NEXT:    vpslld $28, %zmm0, %zmm0
-; AVX512-NEXT:    vpord %zmm1, %zmm0, %zmm0
-; AVX512-NEXT:    retq
+; AVX512F-LABEL: splatconstant_funnnel_v16i32:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpsrld $4, %zmm1, %zmm1
+; AVX512F-NEXT:    vpslld $28, %zmm0, %zmm0
+; AVX512F-NEXT:    vpord %zmm1, %zmm0, %zmm0
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: splatconstant_funnnel_v16i32:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpsrld $4, %zmm1, %zmm1
+; AVX512VL-NEXT:    vpslld $28, %zmm0, %zmm0
+; AVX512VL-NEXT:    vpord %zmm1, %zmm0, %zmm0
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: splatconstant_funnnel_v16i32:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpsrld $4, %zmm1, %zmm1
+; AVX512BW-NEXT:    vpslld $28, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpord %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    retq
+;
+; AVX512VBMI2-LABEL: splatconstant_funnnel_v16i32:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    vpshrdd $4, %zmm0, %zmm1, %zmm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLBW-LABEL: splatconstant_funnnel_v16i32:
+; AVX512VLBW:       # %bb.0:
+; AVX512VLBW-NEXT:    vpsrld $4, %zmm1, %zmm1
+; AVX512VLBW-NEXT:    vpslld $28, %zmm0, %zmm0
+; AVX512VLBW-NEXT:    vpord %zmm1, %zmm0, %zmm0
+; AVX512VLBW-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: splatconstant_funnnel_v16i32:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpshrdd $4, %zmm0, %zmm1, %zmm0
+; AVX512VLVBMI2-NEXT:    retq
   %res = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %x, <16 x i32> %y, <16 x i32> <i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4>)
   ret <16 x i32> %res
 }
@@ -898,12 +1472,22 @@ define <32 x i16> @splatconstant_funnnel_v32i16(<32 x i16> %x, <32 x i16> %y) no
 ; AVX512BW-NEXT:    vporq %zmm1, %zmm0, %zmm0
 ; AVX512BW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: splatconstant_funnnel_v32i16:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    vpshrdw $7, %zmm0, %zmm1, %zmm0
+; AVX512VBMI2-NEXT:    retq
+;
 ; AVX512VLBW-LABEL: splatconstant_funnnel_v32i16:
 ; AVX512VLBW:       # %bb.0:
 ; AVX512VLBW-NEXT:    vpsrlw $7, %zmm1, %zmm1
 ; AVX512VLBW-NEXT:    vpsllw $9, %zmm0, %zmm0
 ; AVX512VLBW-NEXT:    vporq %zmm1, %zmm0, %zmm0
 ; AVX512VLBW-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: splatconstant_funnnel_v32i16:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpshrdw $7, %zmm0, %zmm1, %zmm0
+; AVX512VLVBMI2-NEXT:    retq
   %res = call <32 x i16> @llvm.fshr.v32i16(<32 x i16> %x, <32 x i16> %y, <32 x i16> <i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7, i16 7>)
   ret <32 x i16> %res
 }
@@ -912,32 +1496,30 @@ define <64 x i8> @splatconstant_funnnel_v64i8(<64 x i8> %x, <64 x i8> %y) nounwi
 ; AVX512F-LABEL: splatconstant_funnnel_v64i8:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vpsrlw $4, %ymm2, %ymm2
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm4 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; AVX512F-NEXT:    vpand %ymm4, %ymm2, %ymm2
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm4 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
+; AVX512F-NEXT:    vpandn %ymm2, %ymm4, %ymm2
 ; AVX512F-NEXT:    vpsllw $4, %ymm0, %ymm0
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm5 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
-; AVX512F-NEXT:    vpand %ymm5, %ymm0, %ymm0
+; AVX512F-NEXT:    vpand %ymm4, %ymm0, %ymm0
 ; AVX512F-NEXT:    vpor %ymm2, %ymm0, %ymm0
 ; AVX512F-NEXT:    vpsrlw $4, %ymm3, %ymm2
-; AVX512F-NEXT:    vpand %ymm4, %ymm2, %ymm2
+; AVX512F-NEXT:    vpandn %ymm2, %ymm4, %ymm2
 ; AVX512F-NEXT:    vpsllw $4, %ymm1, %ymm1
-; AVX512F-NEXT:    vpand %ymm5, %ymm1, %ymm1
+; AVX512F-NEXT:    vpand %ymm4, %ymm1, %ymm1
 ; AVX512F-NEXT:    vpor %ymm2, %ymm1, %ymm1
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: splatconstant_funnnel_v64i8:
 ; AVX512VL:       # %bb.0:
 ; AVX512VL-NEXT:    vpsrlw $4, %ymm2, %ymm2
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm4 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; AVX512VL-NEXT:    vpand %ymm4, %ymm2, %ymm2
+; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm4 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
+; AVX512VL-NEXT:    vpandn %ymm2, %ymm4, %ymm2
 ; AVX512VL-NEXT:    vpsllw $4, %ymm0, %ymm0
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm5 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
-; AVX512VL-NEXT:    vpand %ymm5, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpand %ymm4, %ymm0, %ymm0
 ; AVX512VL-NEXT:    vpor %ymm2, %ymm0, %ymm0
 ; AVX512VL-NEXT:    vpsrlw $4, %ymm3, %ymm2
-; AVX512VL-NEXT:    vpand %ymm4, %ymm2, %ymm2
+; AVX512VL-NEXT:    vpandn %ymm2, %ymm4, %ymm2
 ; AVX512VL-NEXT:    vpsllw $4, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpand %ymm5, %ymm1, %ymm1
+; AVX512VL-NEXT:    vpand %ymm4, %ymm1, %ymm1
 ; AVX512VL-NEXT:    vpor %ymm2, %ymm1, %ymm1
 ; AVX512VL-NEXT:    retq
 ;
@@ -950,6 +1532,15 @@ define <64 x i8> @splatconstant_funnnel_v64i8(<64 x i8> %x, <64 x i8> %y) nounwi
 ; AVX512BW-NEXT:    vporq %zmm1, %zmm0, %zmm0
 ; AVX512BW-NEXT:    retq
 ;
+; AVX512VBMI2-LABEL: splatconstant_funnnel_v64i8:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    vpsrlw $4, %zmm1, %zmm1
+; AVX512VBMI2-NEXT:    vpandq {{.*}}(%rip), %zmm1, %zmm1
+; AVX512VBMI2-NEXT:    vpsllw $4, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    vpandq {{.*}}(%rip), %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    retq
+;
 ; AVX512VLBW-LABEL: splatconstant_funnnel_v64i8:
 ; AVX512VLBW:       # %bb.0:
 ; AVX512VLBW-NEXT:    vpsrlw $4, %zmm1, %zmm1
@@ -958,6 +1549,15 @@ define <64 x i8> @splatconstant_funnnel_v64i8(<64 x i8> %x, <64 x i8> %y) nounwi
 ; AVX512VLBW-NEXT:    vpandq {{.*}}(%rip), %zmm0, %zmm0
 ; AVX512VLBW-NEXT:    vporq %zmm1, %zmm0, %zmm0
 ; AVX512VLBW-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: splatconstant_funnnel_v64i8:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpsrlw $4, %zmm1, %zmm1
+; AVX512VLVBMI2-NEXT:    vpandq {{.*}}(%rip), %zmm1, %zmm1
+; AVX512VLVBMI2-NEXT:    vpsllw $4, %zmm0, %zmm0
+; AVX512VLVBMI2-NEXT:    vpandq {{.*}}(%rip), %zmm0, %zmm0
+; AVX512VLVBMI2-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512VLVBMI2-NEXT:    retq
   %res = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %x, <64 x i8> %y, <64 x i8> <i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4>)
   ret <64 x i8> %res
 }
diff --git a/test/CodeGen/X86/vector-fshr-rot-128.ll b/test/CodeGen/X86/vector-fshr-rot-128.ll
index 17246fb1db9b4c3f37bb3039d390bc3ec3b137f9..fc915c2ea8f60f79e02c01ea837ec3c01c6b87a0 100644
--- a/test/CodeGen/X86/vector-fshr-rot-128.ll
+++ b/test/CodeGen/X86/vector-fshr-rot-128.ll
@@ -95,42 +95,40 @@ define <2 x i64> @var_funnnel_v2i64(<2 x i64> %x, <2 x i64> %amt) nounwind {
 ; AVX2-NEXT:    vpor %xmm3, %xmm0, %xmm0
 ; AVX2-NEXT:    retq
 ;
-; AVX512-LABEL: var_funnnel_v2i64:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vmovdqa {{.*#+}} xmm2 = [63,63]
-; AVX512-NEXT:    vpand %xmm2, %xmm1, %xmm3
-; AVX512-NEXT:    vpsrlvq %xmm3, %xmm0, %xmm3
-; AVX512-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX512-NEXT:    vpsubq %xmm1, %xmm4, %xmm1
-; AVX512-NEXT:    vpand %xmm2, %xmm1, %xmm1
-; AVX512-NEXT:    vpsllvq %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpor %xmm3, %xmm0, %xmm0
-; AVX512-NEXT:    retq
+; AVX512F-LABEL: var_funnnel_v2i64:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512F-NEXT:    vprorvq %zmm1, %zmm0, %zmm0
+; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
 ;
-; XOPAVX1-LABEL: var_funnnel_v2i64:
-; XOPAVX1:       # %bb.0:
-; XOPAVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [63,63]
-; XOPAVX1-NEXT:    vpand %xmm2, %xmm1, %xmm3
-; XOPAVX1-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; XOPAVX1-NEXT:    vpsubq %xmm3, %xmm4, %xmm3
-; XOPAVX1-NEXT:    vpshlq %xmm3, %xmm0, %xmm3
-; XOPAVX1-NEXT:    vpsubq %xmm1, %xmm4, %xmm1
-; XOPAVX1-NEXT:    vpand %xmm2, %xmm1, %xmm1
-; XOPAVX1-NEXT:    vpshlq %xmm1, %xmm0, %xmm0
-; XOPAVX1-NEXT:    vpor %xmm3, %xmm0, %xmm0
-; XOPAVX1-NEXT:    retq
+; AVX512VL-LABEL: var_funnnel_v2i64:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vprorvq %xmm1, %xmm0, %xmm0
+; AVX512VL-NEXT:    retq
 ;
-; XOPAVX2-LABEL: var_funnnel_v2i64:
-; XOPAVX2:       # %bb.0:
-; XOPAVX2-NEXT:    vmovdqa {{.*#+}} xmm2 = [63,63]
-; XOPAVX2-NEXT:    vpand %xmm2, %xmm1, %xmm3
-; XOPAVX2-NEXT:    vpsrlvq %xmm3, %xmm0, %xmm3
-; XOPAVX2-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; XOPAVX2-NEXT:    vpsubq %xmm1, %xmm4, %xmm1
-; XOPAVX2-NEXT:    vpand %xmm2, %xmm1, %xmm1
-; XOPAVX2-NEXT:    vpsllvq %xmm1, %xmm0, %xmm0
-; XOPAVX2-NEXT:    vpor %xmm3, %xmm0, %xmm0
-; XOPAVX2-NEXT:    retq
+; AVX512BW-LABEL: var_funnnel_v2i64:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512BW-NEXT:    vprorvq %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+;
+; AVX512VLBW-LABEL: var_funnnel_v2i64:
+; AVX512VLBW:       # %bb.0:
+; AVX512VLBW-NEXT:    vprorvq %xmm1, %xmm0, %xmm0
+; AVX512VLBW-NEXT:    retq
+;
+; XOP-LABEL: var_funnnel_v2i64:
+; XOP:       # %bb.0:
+; XOP-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; XOP-NEXT:    vpsubq %xmm1, %xmm2, %xmm1
+; XOP-NEXT:    vprotq %xmm1, %xmm0, %xmm0
+; XOP-NEXT:    retq
 ;
 ; X32-SSE-LABEL: var_funnnel_v2i64:
 ; X32-SSE:       # %bb.0:
@@ -159,182 +157,130 @@ define <2 x i64> @var_funnnel_v2i64(<2 x i64> %x, <2 x i64> %amt) nounwind {
 define <4 x i32> @var_funnnel_v4i32(<4 x i32> %x, <4 x i32> %amt) nounwind {
 ; SSE2-LABEL: var_funnnel_v4i32:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [31,31,31,31]
-; SSE2-NEXT:    pxor %xmm3, %xmm3
-; SSE2-NEXT:    psubd %xmm1, %xmm3
-; SSE2-NEXT:    movdqa %xmm1, %xmm4
-; SSE2-NEXT:    pand %xmm2, %xmm4
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm4[2,3,3,3,4,5,6,7]
-; SSE2-NEXT:    movdqa %xmm0, %xmm5
-; SSE2-NEXT:    psrld %xmm1, %xmm5
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm6 = xmm4[0,1,1,1,4,5,6,7]
-; SSE2-NEXT:    movdqa %xmm0, %xmm1
-; SSE2-NEXT:    psrld %xmm6, %xmm1
-; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm5[0]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[2,3,0,1]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm5 = xmm4[2,3,3,3,4,5,6,7]
-; SSE2-NEXT:    movdqa %xmm0, %xmm6
-; SSE2-NEXT:    psrld %xmm5, %xmm6
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm4 = xmm4[0,1,1,1,4,5,6,7]
-; SSE2-NEXT:    movdqa %xmm0, %xmm5
-; SSE2-NEXT:    psrld %xmm4, %xmm5
-; SSE2-NEXT:    punpckhqdq {{.*#+}} xmm5 = xmm5[1],xmm6[1]
-; SSE2-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,3],xmm5[0,3]
-; SSE2-NEXT:    pand %xmm2, %xmm3
-; SSE2-NEXT:    pslld $23, %xmm3
-; SSE2-NEXT:    paddd {{.*}}(%rip), %xmm3
-; SSE2-NEXT:    cvttps2dq %xmm3, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
-; SSE2-NEXT:    pmuludq %xmm2, %xmm0
+; SSE2-NEXT:    pxor %xmm2, %xmm2
+; SSE2-NEXT:    psubd %xmm1, %xmm2
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm2
+; SSE2-NEXT:    pslld $23, %xmm2
+; SSE2-NEXT:    paddd {{.*}}(%rip), %xmm2
+; SSE2-NEXT:    cvttps2dq %xmm2, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
+; SSE2-NEXT:    pmuludq %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,3,2,3]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSE2-NEXT:    pmuludq %xmm2, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,3,2,3]
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSE2-NEXT:    pmuludq %xmm3, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
-; SSE2-NEXT:    orps %xmm0, %xmm1
-; SSE2-NEXT:    movaps %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE2-NEXT:    por %xmm3, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: var_funnnel_v4i32:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    movdqa {{.*#+}} xmm2 = [31,31,31,31]
-; SSE41-NEXT:    pxor %xmm3, %xmm3
-; SSE41-NEXT:    psubd %xmm1, %xmm3
-; SSE41-NEXT:    pand %xmm2, %xmm1
-; SSE41-NEXT:    pshuflw {{.*#+}} xmm4 = xmm1[2,3,3,3,4,5,6,7]
-; SSE41-NEXT:    movdqa %xmm0, %xmm5
-; SSE41-NEXT:    psrld %xmm4, %xmm5
-; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[2,3,0,1]
-; SSE41-NEXT:    pshuflw {{.*#+}} xmm6 = xmm4[2,3,3,3,4,5,6,7]
-; SSE41-NEXT:    movdqa %xmm0, %xmm7
-; SSE41-NEXT:    psrld %xmm6, %xmm7
-; SSE41-NEXT:    pblendw {{.*#+}} xmm7 = xmm5[0,1,2,3],xmm7[4,5,6,7]
-; SSE41-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,1,1,1,4,5,6,7]
-; SSE41-NEXT:    movdqa %xmm0, %xmm5
-; SSE41-NEXT:    psrld %xmm1, %xmm5
-; SSE41-NEXT:    pshuflw {{.*#+}} xmm1 = xmm4[0,1,1,1,4,5,6,7]
-; SSE41-NEXT:    movdqa %xmm0, %xmm4
-; SSE41-NEXT:    psrld %xmm1, %xmm4
-; SSE41-NEXT:    pblendw {{.*#+}} xmm4 = xmm5[0,1,2,3],xmm4[4,5,6,7]
-; SSE41-NEXT:    pblendw {{.*#+}} xmm4 = xmm4[0,1],xmm7[2,3],xmm4[4,5],xmm7[6,7]
-; SSE41-NEXT:    pand %xmm2, %xmm3
-; SSE41-NEXT:    pslld $23, %xmm3
-; SSE41-NEXT:    paddd {{.*}}(%rip), %xmm3
-; SSE41-NEXT:    cvttps2dq %xmm3, %xmm1
-; SSE41-NEXT:    pmulld %xmm1, %xmm0
-; SSE41-NEXT:    por %xmm4, %xmm0
+; SSE41-NEXT:    pxor %xmm2, %xmm2
+; SSE41-NEXT:    psubd %xmm1, %xmm2
+; SSE41-NEXT:    pand {{.*}}(%rip), %xmm2
+; SSE41-NEXT:    pslld $23, %xmm2
+; SSE41-NEXT:    paddd {{.*}}(%rip), %xmm2
+; SSE41-NEXT:    cvttps2dq %xmm2, %xmm1
+; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; SSE41-NEXT:    pmuludq %xmm2, %xmm3
+; SSE41-NEXT:    pmuludq %xmm1, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm3[2,3],xmm1[4,5],xmm3[6,7]
+; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[0,0,2,2]
+; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3],xmm0[4,5],xmm2[6,7]
+; SSE41-NEXT:    por %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: var_funnnel_v4i32:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [31,31,31,31]
-; AVX1-NEXT:    vpand %xmm2, %xmm1, %xmm3
-; AVX1-NEXT:    vpsrldq {{.*#+}} xmm4 = xmm3[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX1-NEXT:    vpsrld %xmm4, %xmm0, %xmm4
-; AVX1-NEXT:    vpsrlq $32, %xmm3, %xmm5
-; AVX1-NEXT:    vpsrld %xmm5, %xmm0, %xmm5
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm4 = xmm5[0,1,2,3],xmm4[4,5,6,7]
-; AVX1-NEXT:    vpxor %xmm5, %xmm5, %xmm5
-; AVX1-NEXT:    vpunpckhdq {{.*#+}} xmm6 = xmm3[2],xmm5[2],xmm3[3],xmm5[3]
-; AVX1-NEXT:    vpsrld %xmm6, %xmm0, %xmm6
-; AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm3 = xmm3[0],zero,xmm3[1],zero
-; AVX1-NEXT:    vpsrld %xmm3, %xmm0, %xmm3
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm3 = xmm3[0,1,2,3],xmm6[4,5,6,7]
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm3 = xmm3[0,1],xmm4[2,3],xmm3[4,5],xmm4[6,7]
-; AVX1-NEXT:    vpsubd %xmm1, %xmm5, %xmm1
-; AVX1-NEXT:    vpand %xmm2, %xmm1, %xmm1
+; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX1-NEXT:    vpsubd %xmm1, %xmm2, %xmm1
+; AVX1-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
 ; AVX1-NEXT:    vpslld $23, %xmm1, %xmm1
 ; AVX1-NEXT:    vpaddd {{.*}}(%rip), %xmm1, %xmm1
 ; AVX1-NEXT:    vcvttps2dq %xmm1, %xmm1
-; AVX1-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpor %xmm3, %xmm0, %xmm0
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; AVX1-NEXT:    vpmuludq %xmm2, %xmm3, %xmm2
+; AVX1-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5],xmm2[6,7]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[0,0,2,2]
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3],xmm0[4,5],xmm2[6,7]
+; AVX1-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: var_funnnel_v4i32:
 ; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX2-NEXT:    vpsubd %xmm1, %xmm2, %xmm1
 ; AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [31,31,31,31]
-; AVX2-NEXT:    vpand %xmm2, %xmm1, %xmm3
-; AVX2-NEXT:    vpsrlvd %xmm3, %xmm0, %xmm3
-; AVX2-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX2-NEXT:    vpsubd %xmm1, %xmm4, %xmm1
 ; AVX2-NEXT:    vpand %xmm2, %xmm1, %xmm1
-; AVX2-NEXT:    vpsllvd %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpor %xmm3, %xmm0, %xmm0
+; AVX2-NEXT:    vpsllvd %xmm1, %xmm0, %xmm2
+; AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm3 = [32,32,32,32]
+; AVX2-NEXT:    vpsubd %xmm1, %xmm3, %xmm1
+; AVX2-NEXT:    vpsrlvd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpor %xmm0, %xmm2, %xmm0
 ; AVX2-NEXT:    retq
 ;
-; AVX512-LABEL: var_funnnel_v4i32:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [31,31,31,31]
-; AVX512-NEXT:    vpand %xmm2, %xmm1, %xmm3
-; AVX512-NEXT:    vpsrlvd %xmm3, %xmm0, %xmm3
-; AVX512-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX512-NEXT:    vpsubd %xmm1, %xmm4, %xmm1
-; AVX512-NEXT:    vpand %xmm2, %xmm1, %xmm1
-; AVX512-NEXT:    vpsllvd %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpor %xmm3, %xmm0, %xmm0
-; AVX512-NEXT:    retq
+; AVX512F-LABEL: var_funnnel_v4i32:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512F-NEXT:    vprorvd %zmm1, %zmm0, %zmm0
+; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
 ;
-; XOPAVX1-LABEL: var_funnnel_v4i32:
-; XOPAVX1:       # %bb.0:
-; XOPAVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [31,31,31,31]
-; XOPAVX1-NEXT:    vpand %xmm2, %xmm1, %xmm3
-; XOPAVX1-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; XOPAVX1-NEXT:    vpsubd %xmm3, %xmm4, %xmm3
-; XOPAVX1-NEXT:    vpshld %xmm3, %xmm0, %xmm3
-; XOPAVX1-NEXT:    vpsubd %xmm1, %xmm4, %xmm1
-; XOPAVX1-NEXT:    vpand %xmm2, %xmm1, %xmm1
-; XOPAVX1-NEXT:    vpshld %xmm1, %xmm0, %xmm0
-; XOPAVX1-NEXT:    vpor %xmm3, %xmm0, %xmm0
-; XOPAVX1-NEXT:    retq
+; AVX512VL-LABEL: var_funnnel_v4i32:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vprorvd %xmm1, %xmm0, %xmm0
+; AVX512VL-NEXT:    retq
 ;
-; XOPAVX2-LABEL: var_funnnel_v4i32:
-; XOPAVX2:       # %bb.0:
-; XOPAVX2-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [31,31,31,31]
-; XOPAVX2-NEXT:    vpand %xmm2, %xmm1, %xmm3
-; XOPAVX2-NEXT:    vpsrlvd %xmm3, %xmm0, %xmm3
-; XOPAVX2-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; XOPAVX2-NEXT:    vpsubd %xmm1, %xmm4, %xmm1
-; XOPAVX2-NEXT:    vpand %xmm2, %xmm1, %xmm1
-; XOPAVX2-NEXT:    vpsllvd %xmm1, %xmm0, %xmm0
-; XOPAVX2-NEXT:    vpor %xmm3, %xmm0, %xmm0
-; XOPAVX2-NEXT:    retq
+; AVX512BW-LABEL: var_funnnel_v4i32:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512BW-NEXT:    vprorvd %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+;
+; AVX512VLBW-LABEL: var_funnnel_v4i32:
+; AVX512VLBW:       # %bb.0:
+; AVX512VLBW-NEXT:    vprorvd %xmm1, %xmm0, %xmm0
+; AVX512VLBW-NEXT:    retq
+;
+; XOP-LABEL: var_funnnel_v4i32:
+; XOP:       # %bb.0:
+; XOP-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; XOP-NEXT:    vpsubd %xmm1, %xmm2, %xmm1
+; XOP-NEXT:    vprotd %xmm1, %xmm0, %xmm0
+; XOP-NEXT:    retq
 ;
 ; X32-SSE-LABEL: var_funnnel_v4i32:
 ; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    movdqa {{.*#+}} xmm2 = [31,31,31,31]
-; X32-SSE-NEXT:    pxor %xmm3, %xmm3
-; X32-SSE-NEXT:    psubd %xmm1, %xmm3
-; X32-SSE-NEXT:    movdqa %xmm1, %xmm4
-; X32-SSE-NEXT:    pand %xmm2, %xmm4
-; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm1 = xmm4[2,3,3,3,4,5,6,7]
-; X32-SSE-NEXT:    movdqa %xmm0, %xmm5
-; X32-SSE-NEXT:    psrld %xmm1, %xmm5
-; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm6 = xmm4[0,1,1,1,4,5,6,7]
-; X32-SSE-NEXT:    movdqa %xmm0, %xmm1
-; X32-SSE-NEXT:    psrld %xmm6, %xmm1
-; X32-SSE-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm5[0]
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[2,3,0,1]
-; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm5 = xmm4[2,3,3,3,4,5,6,7]
-; X32-SSE-NEXT:    movdqa %xmm0, %xmm6
-; X32-SSE-NEXT:    psrld %xmm5, %xmm6
-; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm4 = xmm4[0,1,1,1,4,5,6,7]
-; X32-SSE-NEXT:    movdqa %xmm0, %xmm5
-; X32-SSE-NEXT:    psrld %xmm4, %xmm5
-; X32-SSE-NEXT:    punpckhqdq {{.*#+}} xmm5 = xmm5[1],xmm6[1]
-; X32-SSE-NEXT:    shufps {{.*#+}} xmm1 = xmm1[0,3],xmm5[0,3]
-; X32-SSE-NEXT:    pand %xmm2, %xmm3
-; X32-SSE-NEXT:    pslld $23, %xmm3
-; X32-SSE-NEXT:    paddd {{\.LCPI.*}}, %xmm3
-; X32-SSE-NEXT:    cvttps2dq %xmm3, %xmm2
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
-; X32-SSE-NEXT:    pmuludq %xmm2, %xmm0
+; X32-SSE-NEXT:    pxor %xmm2, %xmm2
+; X32-SSE-NEXT:    psubd %xmm1, %xmm2
+; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm2
+; X32-SSE-NEXT:    pslld $23, %xmm2
+; X32-SSE-NEXT:    paddd {{\.LCPI.*}}, %xmm2
+; X32-SSE-NEXT:    cvttps2dq %xmm2, %xmm1
+; X32-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
+; X32-SSE-NEXT:    pmuludq %xmm1, %xmm0
+; X32-SSE-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,3,2,3]
+; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; X32-SSE-NEXT:    pmuludq %xmm2, %xmm1
+; X32-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm1[1,3,2,3]
+; X32-SSE-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
 ; X32-SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; X32-SSE-NEXT:    pmuludq %xmm3, %xmm2
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
-; X32-SSE-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
-; X32-SSE-NEXT:    orps %xmm0, %xmm1
-; X32-SSE-NEXT:    movaps %xmm1, %xmm0
+; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; X32-SSE-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; X32-SSE-NEXT:    por %xmm3, %xmm0
 ; X32-SSE-NEXT:    retl
   %res = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %x, <4 x i32> %x, <4 x i32> %amt)
   ret <4 x i32> %res
@@ -344,180 +290,108 @@ define <8 x i16> @var_funnnel_v8i16(<8 x i16> %x, <8 x i16> %amt) nounwind {
 ; SSE2-LABEL: var_funnnel_v8i16:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    psubw %xmm1, %xmm2
-; SSE2-NEXT:    psllw $12, %xmm1
-; SSE2-NEXT:    movdqa %xmm1, %xmm3
-; SSE2-NEXT:    psraw $15, %xmm3
-; SSE2-NEXT:    movdqa %xmm0, %xmm4
-; SSE2-NEXT:    psrlw $8, %xmm4
-; SSE2-NEXT:    pand %xmm3, %xmm4
-; SSE2-NEXT:    pandn %xmm0, %xmm3
-; SSE2-NEXT:    por %xmm4, %xmm3
-; SSE2-NEXT:    paddw %xmm1, %xmm1
-; SSE2-NEXT:    movdqa %xmm1, %xmm4
-; SSE2-NEXT:    psraw $15, %xmm4
-; SSE2-NEXT:    movdqa %xmm4, %xmm5
-; SSE2-NEXT:    pandn %xmm3, %xmm5
-; SSE2-NEXT:    psrlw $4, %xmm3
-; SSE2-NEXT:    pand %xmm4, %xmm3
-; SSE2-NEXT:    por %xmm5, %xmm3
-; SSE2-NEXT:    paddw %xmm1, %xmm1
-; SSE2-NEXT:    movdqa %xmm1, %xmm4
-; SSE2-NEXT:    psraw $15, %xmm4
-; SSE2-NEXT:    movdqa %xmm4, %xmm5
-; SSE2-NEXT:    pandn %xmm3, %xmm5
-; SSE2-NEXT:    psrlw $2, %xmm3
-; SSE2-NEXT:    pand %xmm4, %xmm3
-; SSE2-NEXT:    por %xmm5, %xmm3
-; SSE2-NEXT:    paddw %xmm1, %xmm1
-; SSE2-NEXT:    psraw $15, %xmm1
-; SSE2-NEXT:    movdqa %xmm1, %xmm4
-; SSE2-NEXT:    pandn %xmm3, %xmm4
-; SSE2-NEXT:    psrlw $1, %xmm3
-; SSE2-NEXT:    pand %xmm1, %xmm3
-; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    pand {{.*}}(%rip), %xmm2
-; SSE2-NEXT:    movdqa %xmm2, %xmm5
-; SSE2-NEXT:    punpckhwd {{.*#+}} xmm5 = xmm5[4],xmm1[4],xmm5[5],xmm1[5],xmm5[6],xmm1[6],xmm5[7],xmm1[7]
-; SSE2-NEXT:    pslld $23, %xmm5
-; SSE2-NEXT:    movdqa {{.*#+}} xmm6 = [1065353216,1065353216,1065353216,1065353216]
-; SSE2-NEXT:    paddd %xmm6, %xmm5
-; SSE2-NEXT:    cvttps2dq %xmm5, %xmm5
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm5 = xmm5[0,2,2,3,4,5,6,7]
-; SSE2-NEXT:    pshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,4,6,6,7]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[0,2,2,3]
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
-; SSE2-NEXT:    pslld $23, %xmm2
-; SSE2-NEXT:    paddd %xmm6, %xmm2
-; SSE2-NEXT:    cvttps2dq %xmm2, %xmm1
+; SSE2-NEXT:    pxor %xmm3, %xmm3
+; SSE2-NEXT:    psubw %xmm1, %xmm3
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm3
+; SSE2-NEXT:    movdqa %xmm3, %xmm1
+; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
+; SSE2-NEXT:    pslld $23, %xmm1
+; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [1065353216,1065353216,1065353216,1065353216]
+; SSE2-NEXT:    paddd %xmm4, %xmm1
+; SSE2-NEXT:    cvttps2dq %xmm1, %xmm1
 ; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,2,2,3,4,5,6,7]
 ; SSE2-NEXT:    pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,6,6,7]
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
-; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm5[0]
-; SSE2-NEXT:    pmullw %xmm0, %xmm1
-; SSE2-NEXT:    por %xmm4, %xmm1
-; SSE2-NEXT:    por %xmm3, %xmm1
-; SSE2-NEXT:    movdqa %xmm1, %xmm0
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
+; SSE2-NEXT:    pslld $23, %xmm3
+; SSE2-NEXT:    paddd %xmm4, %xmm3
+; SSE2-NEXT:    cvttps2dq %xmm3, %xmm2
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm2 = xmm2[0,2,2,3,4,5,6,7]
+; SSE2-NEXT:    pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,6,6,7]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
+; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm1[0]
+; SSE2-NEXT:    movdqa %xmm0, %xmm1
+; SSE2-NEXT:    pmulhuw %xmm2, %xmm1
+; SSE2-NEXT:    pmullw %xmm2, %xmm0
+; SSE2-NEXT:    por %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: var_funnnel_v8i16:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    movdqa %xmm0, %xmm2
-; SSE41-NEXT:    movdqa {{.*#+}} xmm3 = [15,15,15,15,15,15,15,15]
-; SSE41-NEXT:    pxor %xmm4, %xmm4
-; SSE41-NEXT:    psubw %xmm1, %xmm4
-; SSE41-NEXT:    movdqa %xmm1, %xmm0
-; SSE41-NEXT:    pand %xmm3, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm1
-; SSE41-NEXT:    psllw $12, %xmm1
-; SSE41-NEXT:    psllw $4, %xmm0
-; SSE41-NEXT:    por %xmm1, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm1
-; SSE41-NEXT:    paddw %xmm0, %xmm1
-; SSE41-NEXT:    movdqa %xmm2, %xmm5
-; SSE41-NEXT:    psrlw $8, %xmm5
-; SSE41-NEXT:    movdqa %xmm2, %xmm6
-; SSE41-NEXT:    pblendvb %xmm0, %xmm5, %xmm6
-; SSE41-NEXT:    movdqa %xmm6, %xmm5
-; SSE41-NEXT:    psrlw $4, %xmm5
-; SSE41-NEXT:    movdqa %xmm1, %xmm0
-; SSE41-NEXT:    pblendvb %xmm0, %xmm5, %xmm6
-; SSE41-NEXT:    movdqa %xmm6, %xmm5
-; SSE41-NEXT:    psrlw $2, %xmm5
-; SSE41-NEXT:    paddw %xmm1, %xmm1
-; SSE41-NEXT:    movdqa %xmm1, %xmm0
-; SSE41-NEXT:    pblendvb %xmm0, %xmm5, %xmm6
-; SSE41-NEXT:    movdqa %xmm6, %xmm5
-; SSE41-NEXT:    psrlw $1, %xmm5
-; SSE41-NEXT:    paddw %xmm1, %xmm1
-; SSE41-NEXT:    movdqa %xmm1, %xmm0
-; SSE41-NEXT:    pblendvb %xmm0, %xmm5, %xmm6
-; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    pand %xmm3, %xmm4
-; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm1 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero
-; SSE41-NEXT:    punpckhwd {{.*#+}} xmm4 = xmm4[4],xmm0[4],xmm4[5],xmm0[5],xmm4[6],xmm0[6],xmm4[7],xmm0[7]
-; SSE41-NEXT:    pslld $23, %xmm4
-; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [1065353216,1065353216,1065353216,1065353216]
-; SSE41-NEXT:    paddd %xmm0, %xmm4
-; SSE41-NEXT:    cvttps2dq %xmm4, %xmm3
+; SSE41-NEXT:    pxor %xmm2, %xmm2
+; SSE41-NEXT:    pxor %xmm3, %xmm3
+; SSE41-NEXT:    psubw %xmm1, %xmm3
+; SSE41-NEXT:    pand {{.*}}(%rip), %xmm3
+; SSE41-NEXT:    pmovzxwd {{.*#+}} xmm1 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero
+; SSE41-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
+; SSE41-NEXT:    pslld $23, %xmm3
+; SSE41-NEXT:    movdqa {{.*#+}} xmm2 = [1065353216,1065353216,1065353216,1065353216]
+; SSE41-NEXT:    paddd %xmm2, %xmm3
+; SSE41-NEXT:    cvttps2dq %xmm3, %xmm3
 ; SSE41-NEXT:    pslld $23, %xmm1
-; SSE41-NEXT:    paddd %xmm0, %xmm1
-; SSE41-NEXT:    cvttps2dq %xmm1, %xmm0
-; SSE41-NEXT:    packusdw %xmm3, %xmm0
-; SSE41-NEXT:    pmullw %xmm0, %xmm2
-; SSE41-NEXT:    por %xmm6, %xmm2
-; SSE41-NEXT:    movdqa %xmm2, %xmm0
+; SSE41-NEXT:    paddd %xmm2, %xmm1
+; SSE41-NEXT:    cvttps2dq %xmm1, %xmm1
+; SSE41-NEXT:    packusdw %xmm3, %xmm1
+; SSE41-NEXT:    movdqa %xmm0, %xmm2
+; SSE41-NEXT:    pmulhuw %xmm1, %xmm2
+; SSE41-NEXT:    pmullw %xmm1, %xmm0
+; SSE41-NEXT:    por %xmm2, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: var_funnnel_v8i16:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15]
-; AVX1-NEXT:    vpand %xmm2, %xmm1, %xmm3
-; AVX1-NEXT:    vpsllw $12, %xmm3, %xmm4
-; AVX1-NEXT:    vpsllw $4, %xmm3, %xmm3
-; AVX1-NEXT:    vpor %xmm4, %xmm3, %xmm3
-; AVX1-NEXT:    vpaddw %xmm3, %xmm3, %xmm4
-; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm5
-; AVX1-NEXT:    vpblendvb %xmm3, %xmm5, %xmm0, %xmm3
-; AVX1-NEXT:    vpsrlw $4, %xmm3, %xmm5
-; AVX1-NEXT:    vpblendvb %xmm4, %xmm5, %xmm3, %xmm3
-; AVX1-NEXT:    vpsrlw $2, %xmm3, %xmm5
-; AVX1-NEXT:    vpaddw %xmm4, %xmm4, %xmm4
-; AVX1-NEXT:    vpblendvb %xmm4, %xmm5, %xmm3, %xmm3
-; AVX1-NEXT:    vpsrlw $1, %xmm3, %xmm5
-; AVX1-NEXT:    vpaddw %xmm4, %xmm4, %xmm4
-; AVX1-NEXT:    vpblendvb %xmm4, %xmm5, %xmm3, %xmm3
-; AVX1-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX1-NEXT:    vpsubw %xmm1, %xmm4, %xmm1
-; AVX1-NEXT:    vpand %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm1[4],xmm4[4],xmm1[5],xmm4[5],xmm1[6],xmm4[6],xmm1[7],xmm4[7]
+; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX1-NEXT:    vpsubw %xmm1, %xmm2, %xmm1
+; AVX1-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
 ; AVX1-NEXT:    vpslld $23, %xmm2, %xmm2
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = [1065353216,1065353216,1065353216,1065353216]
-; AVX1-NEXT:    vpaddd %xmm4, %xmm2, %xmm2
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [1065353216,1065353216,1065353216,1065353216]
+; AVX1-NEXT:    vpaddd %xmm3, %xmm2, %xmm2
 ; AVX1-NEXT:    vcvttps2dq %xmm2, %xmm2
 ; AVX1-NEXT:    vpmovzxwd {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
 ; AVX1-NEXT:    vpslld $23, %xmm1, %xmm1
-; AVX1-NEXT:    vpaddd %xmm4, %xmm1, %xmm1
+; AVX1-NEXT:    vpaddd %xmm3, %xmm1, %xmm1
 ; AVX1-NEXT:    vcvttps2dq %xmm1, %xmm1
 ; AVX1-NEXT:    vpackusdw %xmm2, %xmm1, %xmm1
+; AVX1-NEXT:    vpmulhuw %xmm1, %xmm0, %xmm2
 ; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpor %xmm3, %xmm0, %xmm0
+; AVX1-NEXT:    vpor %xmm2, %xmm0, %xmm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: var_funnnel_v8i16:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vmovdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15]
-; AVX2-NEXT:    vpand %xmm2, %xmm1, %xmm3
-; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm3 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero
+; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX2-NEXT:    vpsubw %xmm1, %xmm2, %xmm1
+; AVX2-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
 ; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; AVX2-NEXT:    vpsrlvd %ymm3, %ymm0, %ymm3
-; AVX2-NEXT:    vmovdqa {{.*#+}} ymm4 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15,16,17,20,21,24,25,28,29,24,25,28,29,28,29,30,31]
-; AVX2-NEXT:    vpshufb %ymm4, %ymm3, %ymm3
-; AVX2-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[0,2,2,3]
-; AVX2-NEXT:    vpxor %xmm5, %xmm5, %xmm5
-; AVX2-NEXT:    vpsubw %xmm1, %xmm5, %xmm1
-; AVX2-NEXT:    vpand %xmm2, %xmm1, %xmm1
+; AVX2-NEXT:    vpsllvd %ymm2, %ymm0, %ymm2
+; AVX2-NEXT:    vmovdqa {{.*#+}} ymm3 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15,16,17,20,21,24,25,28,29,24,25,28,29,28,29,30,31]
+; AVX2-NEXT:    vpshufb %ymm3, %ymm2, %ymm2
+; AVX2-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[0,2,2,3]
+; AVX2-NEXT:    vmovdqa {{.*#+}} xmm4 = [16,16,16,16,16,16,16,16]
+; AVX2-NEXT:    vpsubw %xmm1, %xmm4, %xmm1
 ; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
-; AVX2-NEXT:    vpsllvd %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpshufb %ymm4, %ymm0, %ymm0
+; AVX2-NEXT:    vpsrlvd %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpshufb %ymm3, %ymm0, %ymm0
 ; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
-; AVX2-NEXT:    vpor %xmm3, %xmm0, %xmm0
+; AVX2-NEXT:    vpor %xmm0, %xmm2, %xmm0
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
 ;
 ; AVX512F-LABEL: var_funnnel_v8i16:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15]
-; AVX512F-NEXT:    vpand %xmm2, %xmm1, %xmm3
-; AVX512F-NEXT:    vpmovzxwd {{.*#+}} ymm3 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero
+; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX512F-NEXT:    vpsubw %xmm1, %xmm2, %xmm1
+; AVX512F-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX512F-NEXT:    vpmovzxwd {{.*#+}} ymm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
 ; AVX512F-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; AVX512F-NEXT:    vpsrlvd %ymm3, %ymm0, %ymm3
-; AVX512F-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX512F-NEXT:    vpsubw %xmm1, %xmm4, %xmm1
-; AVX512F-NEXT:    vpand %xmm2, %xmm1, %xmm1
+; AVX512F-NEXT:    vpsllvd %ymm2, %ymm0, %ymm2
+; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm3 = [16,16,16,16,16,16,16,16]
+; AVX512F-NEXT:    vpsubw %xmm1, %xmm3, %xmm1
 ; AVX512F-NEXT:    vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
-; AVX512F-NEXT:    vpsllvd %ymm1, %ymm0, %ymm0
-; AVX512F-NEXT:    vpor %ymm3, %ymm0, %ymm0
+; AVX512F-NEXT:    vpsrlvd %ymm1, %ymm0, %ymm0
+; AVX512F-NEXT:    vpor %ymm0, %ymm2, %ymm0
 ; AVX512F-NEXT:    vpmovdw %zmm0, %ymm0
 ; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
 ; AVX512F-NEXT:    vzeroupper
@@ -525,17 +399,17 @@ define <8 x i16> @var_funnnel_v8i16(<8 x i16> %x, <8 x i16> %amt) nounwind {
 ;
 ; AVX512VL-LABEL: var_funnnel_v8i16:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15]
-; AVX512VL-NEXT:    vpand %xmm2, %xmm1, %xmm3
-; AVX512VL-NEXT:    vpmovzxwd {{.*#+}} ymm3 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero
+; AVX512VL-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX512VL-NEXT:    vpsubw %xmm1, %xmm2, %xmm1
+; AVX512VL-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX512VL-NEXT:    vpmovzxwd {{.*#+}} ymm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
 ; AVX512VL-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; AVX512VL-NEXT:    vpsrlvd %ymm3, %ymm0, %ymm3
-; AVX512VL-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX512VL-NEXT:    vpsubw %xmm1, %xmm4, %xmm1
-; AVX512VL-NEXT:    vpand %xmm2, %xmm1, %xmm1
+; AVX512VL-NEXT:    vpsllvd %ymm2, %ymm0, %ymm2
+; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm3 = [16,16,16,16,16,16,16,16]
+; AVX512VL-NEXT:    vpsubw %xmm1, %xmm3, %xmm1
 ; AVX512VL-NEXT:    vpmovzxwd {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
-; AVX512VL-NEXT:    vpsllvd %ymm1, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpor %ymm3, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpsrlvd %ymm1, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpor %ymm0, %ymm2, %ymm0
 ; AVX512VL-NEXT:    vpmovdw %ymm0, %xmm0
 ; AVX512VL-NEXT:    vzeroupper
 ; AVX512VL-NEXT:    retq
@@ -543,99 +417,63 @@ define <8 x i16> @var_funnnel_v8i16(<8 x i16> %x, <8 x i16> %amt) nounwind {
 ; AVX512BW-LABEL: var_funnnel_v8i16:
 ; AVX512BW:       # %bb.0:
 ; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15]
-; AVX512BW-NEXT:    vpand %xmm2, %xmm1, %xmm3
-; AVX512BW-NEXT:    vpsrlvw %zmm3, %zmm0, %zmm3
-; AVX512BW-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX512BW-NEXT:    vpsubw %xmm1, %xmm4, %xmm1
-; AVX512BW-NEXT:    vpand %xmm2, %xmm1, %xmm1
-; AVX512BW-NEXT:    vpsllvw %zmm1, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpor %xmm3, %xmm0, %xmm0
+; AVX512BW-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX512BW-NEXT:    vpsubw %xmm1, %xmm2, %xmm1
+; AVX512BW-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX512BW-NEXT:    vpsllvw %zmm1, %zmm0, %zmm2
+; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm3 = [16,16,16,16,16,16,16,16]
+; AVX512BW-NEXT:    vpsubw %xmm1, %xmm3, %xmm1
+; AVX512BW-NEXT:    vpsrlvw %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpor %xmm0, %xmm2, %xmm0
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
 ;
 ; AVX512VLBW-LABEL: var_funnnel_v8i16:
 ; AVX512VLBW:       # %bb.0:
-; AVX512VLBW-NEXT:    vmovdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15]
-; AVX512VLBW-NEXT:    vpand %xmm2, %xmm1, %xmm3
-; AVX512VLBW-NEXT:    vpsrlvw %xmm3, %xmm0, %xmm3
-; AVX512VLBW-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX512VLBW-NEXT:    vpsubw %xmm1, %xmm4, %xmm1
-; AVX512VLBW-NEXT:    vpand %xmm2, %xmm1, %xmm1
-; AVX512VLBW-NEXT:    vpsllvw %xmm1, %xmm0, %xmm0
-; AVX512VLBW-NEXT:    vpor %xmm3, %xmm0, %xmm0
+; AVX512VLBW-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX512VLBW-NEXT:    vpsubw %xmm1, %xmm2, %xmm1
+; AVX512VLBW-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX512VLBW-NEXT:    vpsllvw %xmm1, %xmm0, %xmm2
+; AVX512VLBW-NEXT:    vmovdqa {{.*#+}} xmm3 = [16,16,16,16,16,16,16,16]
+; AVX512VLBW-NEXT:    vpsubw %xmm1, %xmm3, %xmm1
+; AVX512VLBW-NEXT:    vpsrlvw %xmm1, %xmm0, %xmm0
+; AVX512VLBW-NEXT:    vpor %xmm0, %xmm2, %xmm0
 ; AVX512VLBW-NEXT:    retq
 ;
 ; XOP-LABEL: var_funnnel_v8i16:
 ; XOP:       # %bb.0:
-; XOP-NEXT:    vmovdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15]
-; XOP-NEXT:    vpand %xmm2, %xmm1, %xmm3
-; XOP-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; XOP-NEXT:    vpsubw %xmm3, %xmm4, %xmm3
-; XOP-NEXT:    vpshlw %xmm3, %xmm0, %xmm3
-; XOP-NEXT:    vpsubw %xmm1, %xmm4, %xmm1
-; XOP-NEXT:    vpand %xmm2, %xmm1, %xmm1
-; XOP-NEXT:    vpshlw %xmm1, %xmm0, %xmm0
-; XOP-NEXT:    vpor %xmm3, %xmm0, %xmm0
+; XOP-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; XOP-NEXT:    vpsubw %xmm1, %xmm2, %xmm1
+; XOP-NEXT:    vprotw %xmm1, %xmm0, %xmm0
 ; XOP-NEXT:    retq
 ;
 ; X32-SSE-LABEL: var_funnnel_v8i16:
 ; X32-SSE:       # %bb.0:
 ; X32-SSE-NEXT:    pxor %xmm2, %xmm2
-; X32-SSE-NEXT:    psubw %xmm1, %xmm2
-; X32-SSE-NEXT:    psllw $12, %xmm1
-; X32-SSE-NEXT:    movdqa %xmm1, %xmm3
-; X32-SSE-NEXT:    psraw $15, %xmm3
-; X32-SSE-NEXT:    movdqa %xmm0, %xmm4
-; X32-SSE-NEXT:    psrlw $8, %xmm4
-; X32-SSE-NEXT:    pand %xmm3, %xmm4
-; X32-SSE-NEXT:    pandn %xmm0, %xmm3
-; X32-SSE-NEXT:    por %xmm4, %xmm3
-; X32-SSE-NEXT:    paddw %xmm1, %xmm1
-; X32-SSE-NEXT:    movdqa %xmm1, %xmm4
-; X32-SSE-NEXT:    psraw $15, %xmm4
-; X32-SSE-NEXT:    movdqa %xmm4, %xmm5
-; X32-SSE-NEXT:    pandn %xmm3, %xmm5
-; X32-SSE-NEXT:    psrlw $4, %xmm3
-; X32-SSE-NEXT:    pand %xmm4, %xmm3
-; X32-SSE-NEXT:    por %xmm5, %xmm3
-; X32-SSE-NEXT:    paddw %xmm1, %xmm1
-; X32-SSE-NEXT:    movdqa %xmm1, %xmm4
-; X32-SSE-NEXT:    psraw $15, %xmm4
-; X32-SSE-NEXT:    movdqa %xmm4, %xmm5
-; X32-SSE-NEXT:    pandn %xmm3, %xmm5
-; X32-SSE-NEXT:    psrlw $2, %xmm3
-; X32-SSE-NEXT:    pand %xmm4, %xmm3
-; X32-SSE-NEXT:    por %xmm5, %xmm3
-; X32-SSE-NEXT:    paddw %xmm1, %xmm1
-; X32-SSE-NEXT:    psraw $15, %xmm1
-; X32-SSE-NEXT:    movdqa %xmm1, %xmm4
-; X32-SSE-NEXT:    pandn %xmm3, %xmm4
-; X32-SSE-NEXT:    psrlw $1, %xmm3
-; X32-SSE-NEXT:    pand %xmm1, %xmm3
-; X32-SSE-NEXT:    pxor %xmm1, %xmm1
-; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm2
-; X32-SSE-NEXT:    movdqa %xmm2, %xmm5
-; X32-SSE-NEXT:    punpckhwd {{.*#+}} xmm5 = xmm5[4],xmm1[4],xmm5[5],xmm1[5],xmm5[6],xmm1[6],xmm5[7],xmm1[7]
-; X32-SSE-NEXT:    pslld $23, %xmm5
-; X32-SSE-NEXT:    movdqa {{.*#+}} xmm6 = [1065353216,1065353216,1065353216,1065353216]
-; X32-SSE-NEXT:    paddd %xmm6, %xmm5
-; X32-SSE-NEXT:    cvttps2dq %xmm5, %xmm5
-; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm5 = xmm5[0,2,2,3,4,5,6,7]
-; X32-SSE-NEXT:    pshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,4,6,6,7]
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[0,2,2,3]
-; X32-SSE-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
-; X32-SSE-NEXT:    pslld $23, %xmm2
-; X32-SSE-NEXT:    paddd %xmm6, %xmm2
-; X32-SSE-NEXT:    cvttps2dq %xmm2, %xmm1
+; X32-SSE-NEXT:    pxor %xmm3, %xmm3
+; X32-SSE-NEXT:    psubw %xmm1, %xmm3
+; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm3
+; X32-SSE-NEXT:    movdqa %xmm3, %xmm1
+; X32-SSE-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
+; X32-SSE-NEXT:    pslld $23, %xmm1
+; X32-SSE-NEXT:    movdqa {{.*#+}} xmm4 = [1065353216,1065353216,1065353216,1065353216]
+; X32-SSE-NEXT:    paddd %xmm4, %xmm1
+; X32-SSE-NEXT:    cvttps2dq %xmm1, %xmm1
 ; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,2,2,3,4,5,6,7]
 ; X32-SSE-NEXT:    pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,6,6,7]
 ; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
-; X32-SSE-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm5[0]
-; X32-SSE-NEXT:    pmullw %xmm0, %xmm1
-; X32-SSE-NEXT:    por %xmm4, %xmm1
-; X32-SSE-NEXT:    por %xmm3, %xmm1
-; X32-SSE-NEXT:    movdqa %xmm1, %xmm0
+; X32-SSE-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
+; X32-SSE-NEXT:    pslld $23, %xmm3
+; X32-SSE-NEXT:    paddd %xmm4, %xmm3
+; X32-SSE-NEXT:    cvttps2dq %xmm3, %xmm2
+; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm2 = xmm2[0,2,2,3,4,5,6,7]
+; X32-SSE-NEXT:    pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,6,6,7]
+; X32-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
+; X32-SSE-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm1[0]
+; X32-SSE-NEXT:    movdqa %xmm0, %xmm1
+; X32-SSE-NEXT:    pmulhuw %xmm2, %xmm1
+; X32-SSE-NEXT:    pmullw %xmm2, %xmm0
+; X32-SSE-NEXT:    por %xmm1, %xmm0
 ; X32-SSE-NEXT:    retl
   %res = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %x, <8 x i16> %x, <8 x i16> %amt)
   ret <8 x i16> %res
@@ -644,149 +482,106 @@ define <8 x i16> @var_funnnel_v8i16(<8 x i16> %x, <8 x i16> %amt) nounwind {
 define <16 x i8> @var_funnnel_v16i8(<16 x i8> %x, <16 x i8> %amt) nounwind {
 ; SSE2-LABEL: var_funnnel_v16i8:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
-; SSE2-NEXT:    pxor %xmm4, %xmm4
-; SSE2-NEXT:    psubb %xmm1, %xmm4
-; SSE2-NEXT:    movdqa %xmm1, %xmm5
-; SSE2-NEXT:    pand %xmm2, %xmm5
-; SSE2-NEXT:    psllw $5, %xmm5
+; SSE2-NEXT:    movdqa %xmm0, %xmm2
+; SSE2-NEXT:    pxor %xmm0, %xmm0
+; SSE2-NEXT:    pxor %xmm3, %xmm3
+; SSE2-NEXT:    psubb %xmm1, %xmm3
+; SSE2-NEXT:    psllw $5, %xmm3
 ; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    pxor %xmm6, %xmm6
-; SSE2-NEXT:    pcmpgtb %xmm5, %xmm6
-; SSE2-NEXT:    movdqa %xmm0, %xmm3
-; SSE2-NEXT:    psrlw $4, %xmm3
-; SSE2-NEXT:    pand %xmm6, %xmm3
-; SSE2-NEXT:    pandn %xmm0, %xmm6
-; SSE2-NEXT:    pand {{.*}}(%rip), %xmm3
-; SSE2-NEXT:    por %xmm6, %xmm3
-; SSE2-NEXT:    paddb %xmm5, %xmm5
-; SSE2-NEXT:    pxor %xmm6, %xmm6
-; SSE2-NEXT:    pcmpgtb %xmm5, %xmm6
-; SSE2-NEXT:    movdqa %xmm6, %xmm7
-; SSE2-NEXT:    pandn %xmm3, %xmm7
-; SSE2-NEXT:    psrlw $2, %xmm3
-; SSE2-NEXT:    pand %xmm6, %xmm3
-; SSE2-NEXT:    pand {{.*}}(%rip), %xmm3
-; SSE2-NEXT:    por %xmm7, %xmm3
-; SSE2-NEXT:    paddb %xmm5, %xmm5
-; SSE2-NEXT:    pxor %xmm6, %xmm6
-; SSE2-NEXT:    pcmpgtb %xmm5, %xmm6
-; SSE2-NEXT:    movdqa %xmm6, %xmm5
-; SSE2-NEXT:    pandn %xmm3, %xmm5
-; SSE2-NEXT:    psrlw $1, %xmm3
-; SSE2-NEXT:    pand %xmm6, %xmm3
-; SSE2-NEXT:    pand {{.*}}(%rip), %xmm3
-; SSE2-NEXT:    por %xmm5, %xmm3
-; SSE2-NEXT:    pand %xmm2, %xmm4
-; SSE2-NEXT:    psllw $5, %xmm4
-; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    pcmpgtb %xmm4, %xmm2
+; SSE2-NEXT:    pcmpgtb %xmm3, %xmm1
+; SSE2-NEXT:    movdqa %xmm2, %xmm4
+; SSE2-NEXT:    psrlw $4, %xmm4
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm4
 ; SSE2-NEXT:    movdqa %xmm2, %xmm5
-; SSE2-NEXT:    pandn %xmm0, %xmm5
-; SSE2-NEXT:    psllw $4, %xmm0
-; SSE2-NEXT:    pand %xmm2, %xmm0
-; SSE2-NEXT:    pand {{.*}}(%rip), %xmm0
-; SSE2-NEXT:    por %xmm5, %xmm0
-; SSE2-NEXT:    paddb %xmm4, %xmm4
-; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    pcmpgtb %xmm4, %xmm2
-; SSE2-NEXT:    movdqa %xmm2, %xmm5
-; SSE2-NEXT:    pandn %xmm0, %xmm5
-; SSE2-NEXT:    psllw $2, %xmm0
-; SSE2-NEXT:    pand %xmm2, %xmm0
-; SSE2-NEXT:    pand {{.*}}(%rip), %xmm0
-; SSE2-NEXT:    por %xmm5, %xmm0
-; SSE2-NEXT:    paddb %xmm4, %xmm4
-; SSE2-NEXT:    pcmpgtb %xmm4, %xmm1
+; SSE2-NEXT:    psllw $4, %xmm5
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm5
+; SSE2-NEXT:    por %xmm4, %xmm5
+; SSE2-NEXT:    pand %xmm1, %xmm5
+; SSE2-NEXT:    pandn %xmm2, %xmm1
+; SSE2-NEXT:    por %xmm5, %xmm1
 ; SSE2-NEXT:    movdqa %xmm1, %xmm2
-; SSE2-NEXT:    pandn %xmm0, %xmm2
-; SSE2-NEXT:    por %xmm3, %xmm2
-; SSE2-NEXT:    paddb %xmm0, %xmm0
-; SSE2-NEXT:    pand %xmm1, %xmm0
-; SSE2-NEXT:    por %xmm2, %xmm0
+; SSE2-NEXT:    psrlw $6, %xmm2
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm2
+; SSE2-NEXT:    movdqa %xmm1, %xmm4
+; SSE2-NEXT:    psllw $2, %xmm4
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm4
+; SSE2-NEXT:    por %xmm2, %xmm4
+; SSE2-NEXT:    paddb %xmm3, %xmm3
+; SSE2-NEXT:    pxor %xmm2, %xmm2
+; SSE2-NEXT:    pcmpgtb %xmm3, %xmm2
+; SSE2-NEXT:    pand %xmm2, %xmm4
+; SSE2-NEXT:    pandn %xmm1, %xmm2
+; SSE2-NEXT:    por %xmm4, %xmm2
+; SSE2-NEXT:    movdqa %xmm2, %xmm1
+; SSE2-NEXT:    paddb %xmm2, %xmm1
+; SSE2-NEXT:    movdqa %xmm2, %xmm4
+; SSE2-NEXT:    psrlw $7, %xmm4
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm4
+; SSE2-NEXT:    por %xmm1, %xmm4
+; SSE2-NEXT:    paddb %xmm3, %xmm3
+; SSE2-NEXT:    pcmpgtb %xmm3, %xmm0
+; SSE2-NEXT:    pand %xmm0, %xmm4
+; SSE2-NEXT:    pandn %xmm2, %xmm0
+; SSE2-NEXT:    por %xmm4, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: var_funnnel_v16i8:
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    movdqa %xmm0, %xmm2
-; SSE41-NEXT:    movdqa {{.*#+}} xmm4 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
-; SSE41-NEXT:    pxor %xmm3, %xmm3
-; SSE41-NEXT:    psubb %xmm1, %xmm3
-; SSE41-NEXT:    movdqa %xmm1, %xmm0
-; SSE41-NEXT:    pand %xmm4, %xmm0
+; SSE41-NEXT:    psrlw $4, %xmm0
+; SSE41-NEXT:    pand {{.*}}(%rip), %xmm0
+; SSE41-NEXT:    movdqa %xmm2, %xmm3
+; SSE41-NEXT:    psllw $4, %xmm3
+; SSE41-NEXT:    pand {{.*}}(%rip), %xmm3
+; SSE41-NEXT:    por %xmm0, %xmm3
+; SSE41-NEXT:    pxor %xmm0, %xmm0
+; SSE41-NEXT:    psubb %xmm1, %xmm0
 ; SSE41-NEXT:    psllw $5, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm1
-; SSE41-NEXT:    paddb %xmm0, %xmm1
-; SSE41-NEXT:    movdqa %xmm2, %xmm5
-; SSE41-NEXT:    psrlw $4, %xmm5
-; SSE41-NEXT:    pand {{.*}}(%rip), %xmm5
-; SSE41-NEXT:    movdqa %xmm2, %xmm6
-; SSE41-NEXT:    pblendvb %xmm0, %xmm5, %xmm6
-; SSE41-NEXT:    movdqa %xmm6, %xmm5
-; SSE41-NEXT:    psrlw $2, %xmm5
-; SSE41-NEXT:    pand {{.*}}(%rip), %xmm5
-; SSE41-NEXT:    movdqa %xmm1, %xmm0
-; SSE41-NEXT:    pblendvb %xmm0, %xmm5, %xmm6
-; SSE41-NEXT:    movdqa %xmm6, %xmm5
-; SSE41-NEXT:    psrlw $1, %xmm5
-; SSE41-NEXT:    pand {{.*}}(%rip), %xmm5
-; SSE41-NEXT:    paddb %xmm1, %xmm1
-; SSE41-NEXT:    movdqa %xmm1, %xmm0
-; SSE41-NEXT:    pblendvb %xmm0, %xmm5, %xmm6
-; SSE41-NEXT:    pand %xmm4, %xmm3
-; SSE41-NEXT:    psllw $5, %xmm3
-; SSE41-NEXT:    movdqa %xmm3, %xmm1
-; SSE41-NEXT:    paddb %xmm3, %xmm1
-; SSE41-NEXT:    movdqa %xmm2, %xmm4
-; SSE41-NEXT:    psllw $4, %xmm4
-; SSE41-NEXT:    pand {{.*}}(%rip), %xmm4
-; SSE41-NEXT:    movdqa %xmm3, %xmm0
-; SSE41-NEXT:    pblendvb %xmm0, %xmm4, %xmm2
+; SSE41-NEXT:    pblendvb %xmm0, %xmm3, %xmm2
+; SSE41-NEXT:    movdqa %xmm2, %xmm1
+; SSE41-NEXT:    psrlw $6, %xmm1
+; SSE41-NEXT:    pand {{.*}}(%rip), %xmm1
 ; SSE41-NEXT:    movdqa %xmm2, %xmm3
 ; SSE41-NEXT:    psllw $2, %xmm3
 ; SSE41-NEXT:    pand {{.*}}(%rip), %xmm3
-; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    por %xmm1, %xmm3
+; SSE41-NEXT:    paddb %xmm0, %xmm0
 ; SSE41-NEXT:    pblendvb %xmm0, %xmm3, %xmm2
+; SSE41-NEXT:    movdqa %xmm2, %xmm1
+; SSE41-NEXT:    paddb %xmm2, %xmm1
 ; SSE41-NEXT:    movdqa %xmm2, %xmm3
-; SSE41-NEXT:    paddb %xmm2, %xmm3
-; SSE41-NEXT:    paddb %xmm1, %xmm1
-; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    psrlw $7, %xmm3
+; SSE41-NEXT:    pand {{.*}}(%rip), %xmm3
+; SSE41-NEXT:    por %xmm1, %xmm3
+; SSE41-NEXT:    paddb %xmm0, %xmm0
 ; SSE41-NEXT:    pblendvb %xmm0, %xmm3, %xmm2
-; SSE41-NEXT:    por %xmm6, %xmm2
 ; SSE41-NEXT:    movdqa %xmm2, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX-LABEL: var_funnnel_v16i8:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vmovdqa {{.*#+}} xmm2 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
-; AVX-NEXT:    vpand %xmm2, %xmm1, %xmm3
-; AVX-NEXT:    vpsllw $5, %xmm3, %xmm3
-; AVX-NEXT:    vpaddb %xmm3, %xmm3, %xmm4
-; AVX-NEXT:    vpsrlw $4, %xmm0, %xmm5
-; AVX-NEXT:    vpand {{.*}}(%rip), %xmm5, %xmm5
-; AVX-NEXT:    vpblendvb %xmm3, %xmm5, %xmm0, %xmm3
-; AVX-NEXT:    vpsrlw $2, %xmm3, %xmm5
-; AVX-NEXT:    vpand {{.*}}(%rip), %xmm5, %xmm5
-; AVX-NEXT:    vpblendvb %xmm4, %xmm5, %xmm3, %xmm3
-; AVX-NEXT:    vpsrlw $1, %xmm3, %xmm5
-; AVX-NEXT:    vpand {{.*}}(%rip), %xmm5, %xmm5
-; AVX-NEXT:    vpaddb %xmm4, %xmm4, %xmm4
-; AVX-NEXT:    vpblendvb %xmm4, %xmm5, %xmm3, %xmm3
-; AVX-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX-NEXT:    vpsubb %xmm1, %xmm4, %xmm1
-; AVX-NEXT:    vpand %xmm2, %xmm1, %xmm1
+; AVX-NEXT:    vpsrlw $4, %xmm0, %xmm2
+; AVX-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
+; AVX-NEXT:    vpsllw $4, %xmm0, %xmm3
+; AVX-NEXT:    vpand {{.*}}(%rip), %xmm3, %xmm3
+; AVX-NEXT:    vpor %xmm2, %xmm3, %xmm2
+; AVX-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; AVX-NEXT:    vpsubb %xmm1, %xmm3, %xmm1
 ; AVX-NEXT:    vpsllw $5, %xmm1, %xmm1
-; AVX-NEXT:    vpaddb %xmm1, %xmm1, %xmm2
-; AVX-NEXT:    vpsllw $4, %xmm0, %xmm4
-; AVX-NEXT:    vpand {{.*}}(%rip), %xmm4, %xmm4
-; AVX-NEXT:    vpblendvb %xmm1, %xmm4, %xmm0, %xmm0
-; AVX-NEXT:    vpsllw $2, %xmm0, %xmm1
-; AVX-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
-; AVX-NEXT:    vpblendvb %xmm2, %xmm1, %xmm0, %xmm0
-; AVX-NEXT:    vpaddb %xmm0, %xmm0, %xmm1
-; AVX-NEXT:    vpaddb %xmm2, %xmm2, %xmm2
-; AVX-NEXT:    vpblendvb %xmm2, %xmm1, %xmm0, %xmm0
-; AVX-NEXT:    vpor %xmm3, %xmm0, %xmm0
+; AVX-NEXT:    vpblendvb %xmm1, %xmm2, %xmm0, %xmm0
+; AVX-NEXT:    vpsrlw $6, %xmm0, %xmm2
+; AVX-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
+; AVX-NEXT:    vpsllw $2, %xmm0, %xmm3
+; AVX-NEXT:    vpand {{.*}}(%rip), %xmm3, %xmm3
+; AVX-NEXT:    vpor %xmm2, %xmm3, %xmm2
+; AVX-NEXT:    vpaddb %xmm1, %xmm1, %xmm1
+; AVX-NEXT:    vpblendvb %xmm1, %xmm2, %xmm0, %xmm0
+; AVX-NEXT:    vpaddb %xmm0, %xmm0, %xmm2
+; AVX-NEXT:    vpsrlw $7, %xmm0, %xmm3
+; AVX-NEXT:    vpand {{.*}}(%rip), %xmm3, %xmm3
+; AVX-NEXT:    vpor %xmm3, %xmm2, %xmm2
+; AVX-NEXT:    vpaddb %xmm1, %xmm1, %xmm1
+; AVX-NEXT:    vpblendvb %xmm1, %xmm2, %xmm0, %xmm0
 ; AVX-NEXT:    retq
 ;
 ; AVX512F-LABEL: var_funnnel_v16i8:
@@ -860,79 +655,54 @@ define <16 x i8> @var_funnnel_v16i8(<16 x i8> %x, <16 x i8> %amt) nounwind {
 ;
 ; XOP-LABEL: var_funnnel_v16i8:
 ; XOP:       # %bb.0:
-; XOP-NEXT:    vmovdqa {{.*#+}} xmm2 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
-; XOP-NEXT:    vpand %xmm2, %xmm1, %xmm3
-; XOP-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; XOP-NEXT:    vpsubb %xmm3, %xmm4, %xmm3
-; XOP-NEXT:    vpshlb %xmm3, %xmm0, %xmm3
-; XOP-NEXT:    vpsubb %xmm1, %xmm4, %xmm1
-; XOP-NEXT:    vpand %xmm2, %xmm1, %xmm1
-; XOP-NEXT:    vpshlb %xmm1, %xmm0, %xmm0
-; XOP-NEXT:    vpor %xmm3, %xmm0, %xmm0
+; XOP-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; XOP-NEXT:    vpsubb %xmm1, %xmm2, %xmm1
+; XOP-NEXT:    vprotb %xmm1, %xmm0, %xmm0
 ; XOP-NEXT:    retq
 ;
 ; X32-SSE-LABEL: var_funnnel_v16i8:
 ; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    movdqa {{.*#+}} xmm2 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
-; X32-SSE-NEXT:    pxor %xmm4, %xmm4
-; X32-SSE-NEXT:    psubb %xmm1, %xmm4
-; X32-SSE-NEXT:    movdqa %xmm1, %xmm5
-; X32-SSE-NEXT:    pand %xmm2, %xmm5
-; X32-SSE-NEXT:    psllw $5, %xmm5
+; X32-SSE-NEXT:    movdqa %xmm0, %xmm2
+; X32-SSE-NEXT:    pxor %xmm0, %xmm0
+; X32-SSE-NEXT:    pxor %xmm3, %xmm3
+; X32-SSE-NEXT:    psubb %xmm1, %xmm3
+; X32-SSE-NEXT:    psllw $5, %xmm3
 ; X32-SSE-NEXT:    pxor %xmm1, %xmm1
-; X32-SSE-NEXT:    pxor %xmm6, %xmm6
-; X32-SSE-NEXT:    pcmpgtb %xmm5, %xmm6
-; X32-SSE-NEXT:    movdqa %xmm0, %xmm3
-; X32-SSE-NEXT:    psrlw $4, %xmm3
-; X32-SSE-NEXT:    pand %xmm6, %xmm3
-; X32-SSE-NEXT:    pandn %xmm0, %xmm6
-; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm3
-; X32-SSE-NEXT:    por %xmm6, %xmm3
-; X32-SSE-NEXT:    paddb %xmm5, %xmm5
-; X32-SSE-NEXT:    pxor %xmm6, %xmm6
-; X32-SSE-NEXT:    pcmpgtb %xmm5, %xmm6
-; X32-SSE-NEXT:    movdqa %xmm6, %xmm7
-; X32-SSE-NEXT:    pandn %xmm3, %xmm7
-; X32-SSE-NEXT:    psrlw $2, %xmm3
-; X32-SSE-NEXT:    pand %xmm6, %xmm3
-; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm3
-; X32-SSE-NEXT:    por %xmm7, %xmm3
-; X32-SSE-NEXT:    paddb %xmm5, %xmm5
-; X32-SSE-NEXT:    pxor %xmm6, %xmm6
-; X32-SSE-NEXT:    pcmpgtb %xmm5, %xmm6
-; X32-SSE-NEXT:    movdqa %xmm6, %xmm5
-; X32-SSE-NEXT:    pandn %xmm3, %xmm5
-; X32-SSE-NEXT:    psrlw $1, %xmm3
-; X32-SSE-NEXT:    pand %xmm6, %xmm3
-; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm3
-; X32-SSE-NEXT:    por %xmm5, %xmm3
-; X32-SSE-NEXT:    pand %xmm2, %xmm4
-; X32-SSE-NEXT:    psllw $5, %xmm4
-; X32-SSE-NEXT:    pxor %xmm2, %xmm2
-; X32-SSE-NEXT:    pcmpgtb %xmm4, %xmm2
-; X32-SSE-NEXT:    movdqa %xmm2, %xmm5
-; X32-SSE-NEXT:    pandn %xmm0, %xmm5
-; X32-SSE-NEXT:    psllw $4, %xmm0
-; X32-SSE-NEXT:    pand %xmm2, %xmm0
-; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
-; X32-SSE-NEXT:    por %xmm5, %xmm0
-; X32-SSE-NEXT:    paddb %xmm4, %xmm4
-; X32-SSE-NEXT:    pxor %xmm2, %xmm2
-; X32-SSE-NEXT:    pcmpgtb %xmm4, %xmm2
+; X32-SSE-NEXT:    pcmpgtb %xmm3, %xmm1
+; X32-SSE-NEXT:    movdqa %xmm2, %xmm4
+; X32-SSE-NEXT:    psrlw $4, %xmm4
+; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm4
 ; X32-SSE-NEXT:    movdqa %xmm2, %xmm5
-; X32-SSE-NEXT:    pandn %xmm0, %xmm5
-; X32-SSE-NEXT:    psllw $2, %xmm0
-; X32-SSE-NEXT:    pand %xmm2, %xmm0
-; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm0
-; X32-SSE-NEXT:    por %xmm5, %xmm0
-; X32-SSE-NEXT:    paddb %xmm4, %xmm4
-; X32-SSE-NEXT:    pcmpgtb %xmm4, %xmm1
+; X32-SSE-NEXT:    psllw $4, %xmm5
+; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm5
+; X32-SSE-NEXT:    por %xmm4, %xmm5
+; X32-SSE-NEXT:    pand %xmm1, %xmm5
+; X32-SSE-NEXT:    pandn %xmm2, %xmm1
+; X32-SSE-NEXT:    por %xmm5, %xmm1
 ; X32-SSE-NEXT:    movdqa %xmm1, %xmm2
-; X32-SSE-NEXT:    pandn %xmm0, %xmm2
-; X32-SSE-NEXT:    por %xmm3, %xmm2
-; X32-SSE-NEXT:    paddb %xmm0, %xmm0
-; X32-SSE-NEXT:    pand %xmm1, %xmm0
-; X32-SSE-NEXT:    por %xmm2, %xmm0
+; X32-SSE-NEXT:    psrlw $6, %xmm2
+; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm2
+; X32-SSE-NEXT:    movdqa %xmm1, %xmm4
+; X32-SSE-NEXT:    psllw $2, %xmm4
+; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm4
+; X32-SSE-NEXT:    por %xmm2, %xmm4
+; X32-SSE-NEXT:    paddb %xmm3, %xmm3
+; X32-SSE-NEXT:    pxor %xmm2, %xmm2
+; X32-SSE-NEXT:    pcmpgtb %xmm3, %xmm2
+; X32-SSE-NEXT:    pand %xmm2, %xmm4
+; X32-SSE-NEXT:    pandn %xmm1, %xmm2
+; X32-SSE-NEXT:    por %xmm4, %xmm2
+; X32-SSE-NEXT:    movdqa %xmm2, %xmm1
+; X32-SSE-NEXT:    paddb %xmm2, %xmm1
+; X32-SSE-NEXT:    movdqa %xmm2, %xmm4
+; X32-SSE-NEXT:    psrlw $7, %xmm4
+; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm4
+; X32-SSE-NEXT:    por %xmm1, %xmm4
+; X32-SSE-NEXT:    paddb %xmm3, %xmm3
+; X32-SSE-NEXT:    pcmpgtb %xmm3, %xmm0
+; X32-SSE-NEXT:    pand %xmm0, %xmm4
+; X32-SSE-NEXT:    pandn %xmm2, %xmm0
+; X32-SSE-NEXT:    por %xmm4, %xmm0
 ; X32-SSE-NEXT:    retl
   %res = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %x, <16 x i8> %x, <16 x i8> %amt)
   ret <16 x i8> %res
@@ -983,43 +753,50 @@ define <2 x i64> @splatvar_funnnel_v2i64(<2 x i64> %x, <2 x i64> %amt) nounwind
 ; AVX2-NEXT:    vpor %xmm3, %xmm0, %xmm0
 ; AVX2-NEXT:    retq
 ;
-; AVX512-LABEL: splatvar_funnnel_v2i64:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpbroadcastq %xmm1, %xmm1
-; AVX512-NEXT:    vmovdqa {{.*#+}} xmm2 = [63,63]
-; AVX512-NEXT:    vpand %xmm2, %xmm1, %xmm3
-; AVX512-NEXT:    vpsrlq %xmm3, %xmm0, %xmm3
-; AVX512-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX512-NEXT:    vpsubq %xmm1, %xmm4, %xmm1
-; AVX512-NEXT:    vpand %xmm2, %xmm1, %xmm1
-; AVX512-NEXT:    vpsllq %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpor %xmm3, %xmm0, %xmm0
-; AVX512-NEXT:    retq
+; AVX512F-LABEL: splatvar_funnnel_v2i64:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512F-NEXT:    vpbroadcastq %xmm1, %xmm1
+; AVX512F-NEXT:    vprorvq %zmm1, %zmm0, %zmm0
+; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: splatvar_funnnel_v2i64:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpbroadcastq %xmm1, %xmm1
+; AVX512VL-NEXT:    vprorvq %xmm1, %xmm0, %xmm0
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: splatvar_funnnel_v2i64:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512BW-NEXT:    vpbroadcastq %xmm1, %xmm1
+; AVX512BW-NEXT:    vprorvq %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+;
+; AVX512VLBW-LABEL: splatvar_funnnel_v2i64:
+; AVX512VLBW:       # %bb.0:
+; AVX512VLBW-NEXT:    vpbroadcastq %xmm1, %xmm1
+; AVX512VLBW-NEXT:    vprorvq %xmm1, %xmm0, %xmm0
+; AVX512VLBW-NEXT:    retq
 ;
 ; XOPAVX1-LABEL: splatvar_funnnel_v2i64:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,1,0,1]
-; XOPAVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [63,63]
-; XOPAVX1-NEXT:    vpand %xmm2, %xmm1, %xmm3
-; XOPAVX1-NEXT:    vpsrlq %xmm3, %xmm0, %xmm3
-; XOPAVX1-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; XOPAVX1-NEXT:    vpsubq %xmm1, %xmm4, %xmm1
-; XOPAVX1-NEXT:    vpand %xmm2, %xmm1, %xmm1
-; XOPAVX1-NEXT:    vpsllq %xmm1, %xmm0, %xmm0
-; XOPAVX1-NEXT:    vpor %xmm3, %xmm0, %xmm0
+; XOPAVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; XOPAVX1-NEXT:    vpsubq %xmm1, %xmm2, %xmm1
+; XOPAVX1-NEXT:    vprotq %xmm1, %xmm0, %xmm0
 ; XOPAVX1-NEXT:    retq
 ;
 ; XOPAVX2-LABEL: splatvar_funnnel_v2i64:
 ; XOPAVX2:       # %bb.0:
 ; XOPAVX2-NEXT:    vpbroadcastq %xmm1, %xmm1
-; XOPAVX2-NEXT:    vmovdqa {{.*#+}} xmm2 = [63,63]
-; XOPAVX2-NEXT:    vpand %xmm2, %xmm1, %xmm3
-; XOPAVX2-NEXT:    vpsrlq %xmm3, %xmm0, %xmm3
-; XOPAVX2-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; XOPAVX2-NEXT:    vpsubq %xmm1, %xmm4, %xmm1
-; XOPAVX2-NEXT:    vpand %xmm2, %xmm1, %xmm1
-; XOPAVX2-NEXT:    vpsllq %xmm1, %xmm0, %xmm0
-; XOPAVX2-NEXT:    vpor %xmm3, %xmm0, %xmm0
+; XOPAVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; XOPAVX2-NEXT:    vpsubq %xmm1, %xmm2, %xmm1
+; XOPAVX2-NEXT:    vprotq %xmm1, %xmm0, %xmm0
 ; XOPAVX2-NEXT:    retq
 ;
 ; X32-SSE-LABEL: splatvar_funnnel_v2i64:
@@ -1051,129 +828,125 @@ define <2 x i64> @splatvar_funnnel_v2i64(<2 x i64> %x, <2 x i64> %amt) nounwind
 define <4 x i32> @splatvar_funnnel_v4i32(<4 x i32> %x, <4 x i32> %amt) nounwind {
 ; SSE2-LABEL: splatvar_funnnel_v4i32:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [31,31,31,31]
-; SSE2-NEXT:    pxor %xmm3, %xmm3
-; SSE2-NEXT:    psubd %xmm1, %xmm3
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    xorps %xmm4, %xmm4
-; SSE2-NEXT:    xorps %xmm5, %xmm5
-; SSE2-NEXT:    movss {{.*#+}} xmm5 = xmm1[0],xmm5[1,2,3]
-; SSE2-NEXT:    movdqa %xmm0, %xmm1
-; SSE2-NEXT:    psrld %xmm5, %xmm1
-; SSE2-NEXT:    pand %xmm2, %xmm3
-; SSE2-NEXT:    movss {{.*#+}} xmm4 = xmm3[0],xmm4[1,2,3]
-; SSE2-NEXT:    pslld %xmm4, %xmm0
-; SSE2-NEXT:    por %xmm1, %xmm0
+; SSE2-NEXT:    movd %xmm1, %eax
+; SSE2-NEXT:    negl %eax
+; SSE2-NEXT:    andl $31, %eax
+; SSE2-NEXT:    movd %eax, %xmm1
+; SSE2-NEXT:    movdqa %xmm0, %xmm2
+; SSE2-NEXT:    pslld %xmm1, %xmm2
+; SSE2-NEXT:    movl $32, %ecx
+; SSE2-NEXT:    subl %eax, %ecx
+; SSE2-NEXT:    movd %ecx, %xmm1
+; SSE2-NEXT:    psrld %xmm1, %xmm0
+; SSE2-NEXT:    por %xmm2, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: splatvar_funnnel_v4i32:
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
-; SSE41-NEXT:    movdqa {{.*#+}} xmm2 = [31,31,31,31]
-; SSE41-NEXT:    pxor %xmm3, %xmm3
-; SSE41-NEXT:    psubd %xmm1, %xmm3
-; SSE41-NEXT:    pand %xmm2, %xmm1
+; SSE41-NEXT:    pxor %xmm2, %xmm2
+; SSE41-NEXT:    psubd %xmm1, %xmm2
+; SSE41-NEXT:    pand {{.*}}(%rip), %xmm2
+; SSE41-NEXT:    pmovzxdq {{.*#+}} xmm1 = xmm2[0],zero,xmm2[1],zero
+; SSE41-NEXT:    movdqa %xmm0, %xmm3
+; SSE41-NEXT:    pslld %xmm1, %xmm3
+; SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [32,32,32,32]
+; SSE41-NEXT:    psubd %xmm2, %xmm1
 ; SSE41-NEXT:    pmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero
-; SSE41-NEXT:    movdqa %xmm0, %xmm4
-; SSE41-NEXT:    psrld %xmm1, %xmm4
-; SSE41-NEXT:    pand %xmm2, %xmm3
-; SSE41-NEXT:    pmovzxdq {{.*#+}} xmm1 = xmm3[0],zero,xmm3[1],zero
-; SSE41-NEXT:    pslld %xmm1, %xmm0
-; SSE41-NEXT:    por %xmm4, %xmm0
+; SSE41-NEXT:    psrld %xmm1, %xmm0
+; SSE41-NEXT:    por %xmm3, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: splatvar_funnnel_v4i32:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [31,31,31,31]
-; AVX1-NEXT:    vpand %xmm2, %xmm1, %xmm3
-; AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm3 = xmm3[0],zero,xmm3[1],zero
-; AVX1-NEXT:    vpsrld %xmm3, %xmm0, %xmm3
-; AVX1-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX1-NEXT:    vpsubd %xmm1, %xmm4, %xmm1
-; AVX1-NEXT:    vpand %xmm2, %xmm1, %xmm1
+; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX1-NEXT:    vpsubd %xmm1, %xmm2, %xmm1
+; AVX1-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm2 = xmm1[0],zero,xmm1[1],zero
+; AVX1-NEXT:    vpslld %xmm2, %xmm0, %xmm2
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [32,32,32,32]
+; AVX1-NEXT:    vpsubd %xmm1, %xmm3, %xmm1
 ; AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero
-; AVX1-NEXT:    vpslld %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpor %xmm3, %xmm0, %xmm0
+; AVX1-NEXT:    vpsrld %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpor %xmm0, %xmm2, %xmm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: splatvar_funnnel_v4i32:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpbroadcastd %xmm1, %xmm1
+; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX2-NEXT:    vpsubd %xmm1, %xmm2, %xmm1
 ; AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [31,31,31,31]
-; AVX2-NEXT:    vpand %xmm2, %xmm1, %xmm3
-; AVX2-NEXT:    vpmovzxdq {{.*#+}} xmm3 = xmm3[0],zero,xmm3[1],zero
-; AVX2-NEXT:    vpsrld %xmm3, %xmm0, %xmm3
-; AVX2-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX2-NEXT:    vpsubd %xmm1, %xmm4, %xmm1
 ; AVX2-NEXT:    vpand %xmm2, %xmm1, %xmm1
+; AVX2-NEXT:    vpmovzxdq {{.*#+}} xmm2 = xmm1[0],zero,xmm1[1],zero
+; AVX2-NEXT:    vpslld %xmm2, %xmm0, %xmm2
+; AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm3 = [32,32,32,32]
+; AVX2-NEXT:    vpsubd %xmm1, %xmm3, %xmm1
 ; AVX2-NEXT:    vpmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero
-; AVX2-NEXT:    vpslld %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpor %xmm3, %xmm0, %xmm0
+; AVX2-NEXT:    vpsrld %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpor %xmm0, %xmm2, %xmm0
 ; AVX2-NEXT:    retq
 ;
-; AVX512-LABEL: splatvar_funnnel_v4i32:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpbroadcastd %xmm1, %xmm1
-; AVX512-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [31,31,31,31]
-; AVX512-NEXT:    vpand %xmm2, %xmm1, %xmm3
-; AVX512-NEXT:    vpmovzxdq {{.*#+}} xmm3 = xmm3[0],zero,xmm3[1],zero
-; AVX512-NEXT:    vpsrld %xmm3, %xmm0, %xmm3
-; AVX512-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX512-NEXT:    vpsubd %xmm1, %xmm4, %xmm1
-; AVX512-NEXT:    vpand %xmm2, %xmm1, %xmm1
-; AVX512-NEXT:    vpmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero
-; AVX512-NEXT:    vpslld %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpor %xmm3, %xmm0, %xmm0
-; AVX512-NEXT:    retq
+; AVX512F-LABEL: splatvar_funnnel_v4i32:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512F-NEXT:    vpbroadcastd %xmm1, %xmm1
+; AVX512F-NEXT:    vprorvd %zmm1, %zmm0, %zmm0
+; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: splatvar_funnnel_v4i32:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpbroadcastd %xmm1, %xmm1
+; AVX512VL-NEXT:    vprorvd %xmm1, %xmm0, %xmm0
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: splatvar_funnnel_v4i32:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512BW-NEXT:    vpbroadcastd %xmm1, %xmm1
+; AVX512BW-NEXT:    vprorvd %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+;
+; AVX512VLBW-LABEL: splatvar_funnnel_v4i32:
+; AVX512VLBW:       # %bb.0:
+; AVX512VLBW-NEXT:    vpbroadcastd %xmm1, %xmm1
+; AVX512VLBW-NEXT:    vprorvd %xmm1, %xmm0, %xmm0
+; AVX512VLBW-NEXT:    retq
 ;
 ; XOPAVX1-LABEL: splatvar_funnnel_v4i32:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
-; XOPAVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [31,31,31,31]
-; XOPAVX1-NEXT:    vpand %xmm2, %xmm1, %xmm3
-; XOPAVX1-NEXT:    vpmovzxdq {{.*#+}} xmm3 = xmm3[0],zero,xmm3[1],zero
-; XOPAVX1-NEXT:    vpsrld %xmm3, %xmm0, %xmm3
-; XOPAVX1-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; XOPAVX1-NEXT:    vpsubd %xmm1, %xmm4, %xmm1
-; XOPAVX1-NEXT:    vpand %xmm2, %xmm1, %xmm1
-; XOPAVX1-NEXT:    vpmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero
-; XOPAVX1-NEXT:    vpslld %xmm1, %xmm0, %xmm0
-; XOPAVX1-NEXT:    vpor %xmm3, %xmm0, %xmm0
+; XOPAVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; XOPAVX1-NEXT:    vpsubd %xmm1, %xmm2, %xmm1
+; XOPAVX1-NEXT:    vprotd %xmm1, %xmm0, %xmm0
 ; XOPAVX1-NEXT:    retq
 ;
 ; XOPAVX2-LABEL: splatvar_funnnel_v4i32:
 ; XOPAVX2:       # %bb.0:
 ; XOPAVX2-NEXT:    vpbroadcastd %xmm1, %xmm1
-; XOPAVX2-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [31,31,31,31]
-; XOPAVX2-NEXT:    vpand %xmm2, %xmm1, %xmm3
-; XOPAVX2-NEXT:    vpmovzxdq {{.*#+}} xmm3 = xmm3[0],zero,xmm3[1],zero
-; XOPAVX2-NEXT:    vpsrld %xmm3, %xmm0, %xmm3
-; XOPAVX2-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; XOPAVX2-NEXT:    vpsubd %xmm1, %xmm4, %xmm1
-; XOPAVX2-NEXT:    vpand %xmm2, %xmm1, %xmm1
-; XOPAVX2-NEXT:    vpmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero
-; XOPAVX2-NEXT:    vpslld %xmm1, %xmm0, %xmm0
-; XOPAVX2-NEXT:    vpor %xmm3, %xmm0, %xmm0
+; XOPAVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; XOPAVX2-NEXT:    vpsubd %xmm1, %xmm2, %xmm1
+; XOPAVX2-NEXT:    vprotd %xmm1, %xmm0, %xmm0
 ; XOPAVX2-NEXT:    retq
 ;
 ; X32-SSE-LABEL: splatvar_funnnel_v4i32:
 ; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
-; X32-SSE-NEXT:    movdqa {{.*#+}} xmm2 = [31,31,31,31]
-; X32-SSE-NEXT:    pxor %xmm3, %xmm3
-; X32-SSE-NEXT:    psubd %xmm1, %xmm3
-; X32-SSE-NEXT:    pand %xmm2, %xmm1
-; X32-SSE-NEXT:    xorps %xmm4, %xmm4
-; X32-SSE-NEXT:    xorps %xmm5, %xmm5
-; X32-SSE-NEXT:    movss {{.*#+}} xmm5 = xmm1[0],xmm5[1,2,3]
-; X32-SSE-NEXT:    movdqa %xmm0, %xmm1
-; X32-SSE-NEXT:    psrld %xmm5, %xmm1
-; X32-SSE-NEXT:    pand %xmm2, %xmm3
-; X32-SSE-NEXT:    movss {{.*#+}} xmm4 = xmm3[0],xmm4[1,2,3]
-; X32-SSE-NEXT:    pslld %xmm4, %xmm0
-; X32-SSE-NEXT:    por %xmm1, %xmm0
+; X32-SSE-NEXT:    movd %xmm1, %eax
+; X32-SSE-NEXT:    negl %eax
+; X32-SSE-NEXT:    andl $31, %eax
+; X32-SSE-NEXT:    movd %eax, %xmm1
+; X32-SSE-NEXT:    movdqa %xmm0, %xmm2
+; X32-SSE-NEXT:    pslld %xmm1, %xmm2
+; X32-SSE-NEXT:    movl $32, %ecx
+; X32-SSE-NEXT:    subl %eax, %ecx
+; X32-SSE-NEXT:    movd %ecx, %xmm1
+; X32-SSE-NEXT:    psrld %xmm1, %xmm0
+; X32-SSE-NEXT:    por %xmm2, %xmm0
 ; X32-SSE-NEXT:    retl
   %splat = shufflevector <4 x i32> %amt, <4 x i32> undef, <4 x i32> zeroinitializer
   %res = call <4 x i32> @llvm.fshr.v4i32(<4 x i32> %x, <4 x i32> %x, <4 x i32> %splat)
@@ -1185,132 +958,118 @@ define <8 x i16> @splatvar_funnnel_v8i16(<8 x i16> %x, <8 x i16> %amt) nounwind
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,0,2,3,4,5,6,7]
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15]
-; SSE2-NEXT:    pxor %xmm3, %xmm3
-; SSE2-NEXT:    psubw %xmm1, %xmm3
-; SSE2-NEXT:    pand %xmm2, %xmm1
+; SSE2-NEXT:    pxor %xmm2, %xmm2
+; SSE2-NEXT:    psubw %xmm1, %xmm2
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm2
+; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [16,16,16,16,16,16,16,16]
+; SSE2-NEXT:    psubw %xmm2, %xmm1
+; SSE2-NEXT:    pslldq {{.*#+}} xmm2 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm2[0,1]
+; SSE2-NEXT:    psrldq {{.*#+}} xmm2 = xmm2[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; SSE2-NEXT:    movdqa %xmm0, %xmm3
+; SSE2-NEXT:    psllw %xmm2, %xmm3
 ; SSE2-NEXT:    pslldq {{.*#+}} xmm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm1[0,1]
 ; SSE2-NEXT:    psrldq {{.*#+}} xmm1 = xmm1[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; SSE2-NEXT:    movdqa %xmm0, %xmm4
-; SSE2-NEXT:    psrlw %xmm1, %xmm4
-; SSE2-NEXT:    pand %xmm2, %xmm3
-; SSE2-NEXT:    pslldq {{.*#+}} xmm3 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm3[0,1]
-; SSE2-NEXT:    psrldq {{.*#+}} xmm3 = xmm3[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; SSE2-NEXT:    psllw %xmm3, %xmm0
-; SSE2-NEXT:    por %xmm4, %xmm0
+; SSE2-NEXT:    psrlw %xmm1, %xmm0
+; SSE2-NEXT:    por %xmm3, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: splatvar_funnnel_v8i16:
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,0,2,3,4,5,6,7]
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
-; SSE41-NEXT:    movdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15]
-; SSE41-NEXT:    pxor %xmm3, %xmm3
-; SSE41-NEXT:    psubw %xmm1, %xmm3
-; SSE41-NEXT:    pand %xmm2, %xmm1
+; SSE41-NEXT:    pxor %xmm2, %xmm2
+; SSE41-NEXT:    psubw %xmm1, %xmm2
+; SSE41-NEXT:    pand {{.*}}(%rip), %xmm2
+; SSE41-NEXT:    pmovzxwq {{.*#+}} xmm1 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero
+; SSE41-NEXT:    movdqa %xmm0, %xmm3
+; SSE41-NEXT:    psllw %xmm1, %xmm3
+; SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [16,16,16,16,16,16,16,16]
+; SSE41-NEXT:    psubw %xmm2, %xmm1
 ; SSE41-NEXT:    pmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
-; SSE41-NEXT:    movdqa %xmm0, %xmm4
-; SSE41-NEXT:    psrlw %xmm1, %xmm4
-; SSE41-NEXT:    pand %xmm2, %xmm3
-; SSE41-NEXT:    pmovzxwq {{.*#+}} xmm1 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero
-; SSE41-NEXT:    psllw %xmm1, %xmm0
-; SSE41-NEXT:    por %xmm4, %xmm0
+; SSE41-NEXT:    psrlw %xmm1, %xmm0
+; SSE41-NEXT:    por %xmm3, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: splatvar_funnnel_v8i16:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm1[0,0,2,3,4,5,6,7]
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15]
-; AVX1-NEXT:    vpand %xmm2, %xmm1, %xmm3
-; AVX1-NEXT:    vpmovzxwq {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero
-; AVX1-NEXT:    vpsrlw %xmm3, %xmm0, %xmm3
-; AVX1-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX1-NEXT:    vpsubw %xmm1, %xmm4, %xmm1
-; AVX1-NEXT:    vpand %xmm2, %xmm1, %xmm1
+; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX1-NEXT:    vpsubw %xmm1, %xmm2, %xmm1
+; AVX1-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX1-NEXT:    vpmovzxwq {{.*#+}} xmm2 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; AVX1-NEXT:    vpsllw %xmm2, %xmm0, %xmm2
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [16,16,16,16,16,16,16,16]
+; AVX1-NEXT:    vpsubw %xmm1, %xmm3, %xmm1
 ; AVX1-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
-; AVX1-NEXT:    vpsllw %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpor %xmm3, %xmm0, %xmm0
+; AVX1-NEXT:    vpsrlw %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpor %xmm0, %xmm2, %xmm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: splatvar_funnnel_v8i16:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpbroadcastw %xmm1, %xmm1
-; AVX2-NEXT:    vmovdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15]
-; AVX2-NEXT:    vpand %xmm2, %xmm1, %xmm3
-; AVX2-NEXT:    vpmovzxwq {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero
-; AVX2-NEXT:    vpsrlw %xmm3, %xmm0, %xmm3
-; AVX2-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX2-NEXT:    vpsubw %xmm1, %xmm4, %xmm1
-; AVX2-NEXT:    vpand %xmm2, %xmm1, %xmm1
+; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX2-NEXT:    vpsubw %xmm1, %xmm2, %xmm1
+; AVX2-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX2-NEXT:    vpmovzxwq {{.*#+}} xmm2 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; AVX2-NEXT:    vpsllw %xmm2, %xmm0, %xmm2
+; AVX2-NEXT:    vmovdqa {{.*#+}} xmm3 = [16,16,16,16,16,16,16,16]
+; AVX2-NEXT:    vpsubw %xmm1, %xmm3, %xmm1
 ; AVX2-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
-; AVX2-NEXT:    vpsllw %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpor %xmm3, %xmm0, %xmm0
+; AVX2-NEXT:    vpsrlw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpor %xmm0, %xmm2, %xmm0
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: splatvar_funnnel_v8i16:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpbroadcastw %xmm1, %xmm1
-; AVX512-NEXT:    vmovdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15]
-; AVX512-NEXT:    vpand %xmm2, %xmm1, %xmm3
-; AVX512-NEXT:    vpmovzxwq {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero
-; AVX512-NEXT:    vpsrlw %xmm3, %xmm0, %xmm3
-; AVX512-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX512-NEXT:    vpsubw %xmm1, %xmm4, %xmm1
-; AVX512-NEXT:    vpand %xmm2, %xmm1, %xmm1
+; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX512-NEXT:    vpsubw %xmm1, %xmm2, %xmm1
+; AVX512-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX512-NEXT:    vpmovzxwq {{.*#+}} xmm2 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; AVX512-NEXT:    vpsllw %xmm2, %xmm0, %xmm2
+; AVX512-NEXT:    vmovdqa {{.*#+}} xmm3 = [16,16,16,16,16,16,16,16]
+; AVX512-NEXT:    vpsubw %xmm1, %xmm3, %xmm1
 ; AVX512-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
-; AVX512-NEXT:    vpsllw %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpor %xmm3, %xmm0, %xmm0
+; AVX512-NEXT:    vpsrlw %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vpor %xmm0, %xmm2, %xmm0
 ; AVX512-NEXT:    retq
 ;
 ; XOPAVX1-LABEL: splatvar_funnnel_v8i16:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm1[0,0,2,3,4,5,6,7]
 ; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
-; XOPAVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15]
-; XOPAVX1-NEXT:    vpand %xmm2, %xmm1, %xmm3
-; XOPAVX1-NEXT:    vpmovzxwq {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero
-; XOPAVX1-NEXT:    vpsrlw %xmm3, %xmm0, %xmm3
-; XOPAVX1-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; XOPAVX1-NEXT:    vpsubw %xmm1, %xmm4, %xmm1
-; XOPAVX1-NEXT:    vpand %xmm2, %xmm1, %xmm1
-; XOPAVX1-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
-; XOPAVX1-NEXT:    vpsllw %xmm1, %xmm0, %xmm0
-; XOPAVX1-NEXT:    vpor %xmm3, %xmm0, %xmm0
+; XOPAVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; XOPAVX1-NEXT:    vpsubw %xmm1, %xmm2, %xmm1
+; XOPAVX1-NEXT:    vprotw %xmm1, %xmm0, %xmm0
 ; XOPAVX1-NEXT:    retq
 ;
 ; XOPAVX2-LABEL: splatvar_funnnel_v8i16:
 ; XOPAVX2:       # %bb.0:
 ; XOPAVX2-NEXT:    vpbroadcastw %xmm1, %xmm1
-; XOPAVX2-NEXT:    vmovdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15]
-; XOPAVX2-NEXT:    vpand %xmm2, %xmm1, %xmm3
-; XOPAVX2-NEXT:    vpmovzxwq {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero
-; XOPAVX2-NEXT:    vpsrlw %xmm3, %xmm0, %xmm3
-; XOPAVX2-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; XOPAVX2-NEXT:    vpsubw %xmm1, %xmm4, %xmm1
-; XOPAVX2-NEXT:    vpand %xmm2, %xmm1, %xmm1
-; XOPAVX2-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
-; XOPAVX2-NEXT:    vpsllw %xmm1, %xmm0, %xmm0
-; XOPAVX2-NEXT:    vpor %xmm3, %xmm0, %xmm0
+; XOPAVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; XOPAVX2-NEXT:    vpsubw %xmm1, %xmm2, %xmm1
+; XOPAVX2-NEXT:    vprotw %xmm1, %xmm0, %xmm0
 ; XOPAVX2-NEXT:    retq
 ;
 ; X32-SSE-LABEL: splatvar_funnnel_v8i16:
 ; X32-SSE:       # %bb.0:
 ; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,0,2,3,4,5,6,7]
 ; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
-; X32-SSE-NEXT:    movdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15]
-; X32-SSE-NEXT:    pxor %xmm3, %xmm3
-; X32-SSE-NEXT:    psubw %xmm1, %xmm3
-; X32-SSE-NEXT:    pand %xmm2, %xmm1
+; X32-SSE-NEXT:    pxor %xmm2, %xmm2
+; X32-SSE-NEXT:    psubw %xmm1, %xmm2
+; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm2
+; X32-SSE-NEXT:    movdqa {{.*#+}} xmm1 = [16,16,16,16,16,16,16,16]
+; X32-SSE-NEXT:    psubw %xmm2, %xmm1
+; X32-SSE-NEXT:    pslldq {{.*#+}} xmm2 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm2[0,1]
+; X32-SSE-NEXT:    psrldq {{.*#+}} xmm2 = xmm2[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; X32-SSE-NEXT:    movdqa %xmm0, %xmm3
+; X32-SSE-NEXT:    psllw %xmm2, %xmm3
 ; X32-SSE-NEXT:    pslldq {{.*#+}} xmm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm1[0,1]
 ; X32-SSE-NEXT:    psrldq {{.*#+}} xmm1 = xmm1[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; X32-SSE-NEXT:    movdqa %xmm0, %xmm4
-; X32-SSE-NEXT:    psrlw %xmm1, %xmm4
-; X32-SSE-NEXT:    pand %xmm2, %xmm3
-; X32-SSE-NEXT:    pslldq {{.*#+}} xmm3 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm3[0,1]
-; X32-SSE-NEXT:    psrldq {{.*#+}} xmm3 = xmm3[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; X32-SSE-NEXT:    psllw %xmm3, %xmm0
-; X32-SSE-NEXT:    por %xmm4, %xmm0
+; X32-SSE-NEXT:    psrlw %xmm1, %xmm0
+; X32-SSE-NEXT:    por %xmm3, %xmm0
 ; X32-SSE-NEXT:    retl
   %splat = shufflevector <8 x i16> %amt, <8 x i16> undef, <8 x i32> zeroinitializer
   %res = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %x, <8 x i16> %x, <8 x i16> %splat)
@@ -1323,102 +1082,104 @@ define <16 x i8> @splatvar_funnnel_v16i8(<16 x i8> %x, <16 x i8> %amt) nounwind
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
 ; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,0,2,3,4,5,6,7]
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
-; SSE2-NEXT:    pxor %xmm3, %xmm3
-; SSE2-NEXT:    psubb %xmm1, %xmm3
-; SSE2-NEXT:    pand %xmm2, %xmm1
-; SSE2-NEXT:    pslldq {{.*#+}} xmm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm1[0]
-; SSE2-NEXT:    psrldq {{.*#+}} xmm1 = xmm1[15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; SSE2-NEXT:    movdqa %xmm0, %xmm4
-; SSE2-NEXT:    psrlw %xmm1, %xmm4
+; SSE2-NEXT:    pxor %xmm2, %xmm2
+; SSE2-NEXT:    psubb %xmm1, %xmm2
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm2
+; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; SSE2-NEXT:    psubb %xmm2, %xmm3
+; SSE2-NEXT:    pslldq {{.*#+}} xmm2 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm2[0]
+; SSE2-NEXT:    psrldq {{.*#+}} xmm2 = xmm2[15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; SSE2-NEXT:    movdqa %xmm0, %xmm1
+; SSE2-NEXT:    psllw %xmm2, %xmm1
+; SSE2-NEXT:    pcmpeqd %xmm4, %xmm4
 ; SSE2-NEXT:    pcmpeqd %xmm5, %xmm5
-; SSE2-NEXT:    pcmpeqd %xmm6, %xmm6
-; SSE2-NEXT:    psrlw %xmm1, %xmm6
-; SSE2-NEXT:    psrlw $8, %xmm6
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm6 = xmm6[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm6[0,0,2,3,4,5,6,7]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
-; SSE2-NEXT:    pand %xmm4, %xmm1
-; SSE2-NEXT:    pand %xmm2, %xmm3
-; SSE2-NEXT:    pslldq {{.*#+}} xmm3 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm3[0]
-; SSE2-NEXT:    psrldq {{.*#+}} xmm3 = xmm3[15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; SSE2-NEXT:    psllw %xmm3, %xmm0
-; SSE2-NEXT:    psllw %xmm3, %xmm5
+; SSE2-NEXT:    psllw %xmm2, %xmm5
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
 ; SSE2-NEXT:    pshuflw {{.*#+}} xmm2 = xmm5[0,0,2,3,4,5,6,7]
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,0,0,0]
-; SSE2-NEXT:    pand %xmm2, %xmm0
-; SSE2-NEXT:    por %xmm1, %xmm0
+; SSE2-NEXT:    pand %xmm2, %xmm1
+; SSE2-NEXT:    pslldq {{.*#+}} xmm3 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm3[0]
+; SSE2-NEXT:    psrldq {{.*#+}} xmm3 = xmm3[15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; SSE2-NEXT:    psrlw %xmm3, %xmm0
+; SSE2-NEXT:    psrlw %xmm3, %xmm4
+; SSE2-NEXT:    psrlw $8, %xmm4
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm2 = xmm4[0,0,2,3,4,5,6,7]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,0,0,0]
+; SSE2-NEXT:    pand %xmm0, %xmm2
+; SSE2-NEXT:    por %xmm2, %xmm1
+; SSE2-NEXT:    movdqa %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: splatvar_funnnel_v16i8:
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    pxor %xmm2, %xmm2
 ; SSE41-NEXT:    pshufb %xmm2, %xmm1
-; SSE41-NEXT:    movdqa {{.*#+}} xmm3 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
-; SSE41-NEXT:    pxor %xmm4, %xmm4
-; SSE41-NEXT:    psubb %xmm1, %xmm4
-; SSE41-NEXT:    pand %xmm3, %xmm1
-; SSE41-NEXT:    pmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
-; SSE41-NEXT:    movdqa %xmm0, %xmm5
-; SSE41-NEXT:    psrlw %xmm1, %xmm5
+; SSE41-NEXT:    pxor %xmm3, %xmm3
+; SSE41-NEXT:    psubb %xmm1, %xmm3
+; SSE41-NEXT:    pand {{.*}}(%rip), %xmm3
+; SSE41-NEXT:    pmovzxbq {{.*#+}} xmm4 = xmm3[0],zero,zero,zero,zero,zero,zero,zero,xmm3[1],zero,zero,zero,zero,zero,zero,zero
+; SSE41-NEXT:    movdqa %xmm0, %xmm1
+; SSE41-NEXT:    psllw %xmm4, %xmm1
+; SSE41-NEXT:    pcmpeqd %xmm5, %xmm5
 ; SSE41-NEXT:    pcmpeqd %xmm6, %xmm6
-; SSE41-NEXT:    pcmpeqd %xmm7, %xmm7
-; SSE41-NEXT:    psrlw %xmm1, %xmm7
-; SSE41-NEXT:    pshufb {{.*#+}} xmm7 = xmm7[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
-; SSE41-NEXT:    pand %xmm5, %xmm7
-; SSE41-NEXT:    pand %xmm3, %xmm4
-; SSE41-NEXT:    pmovzxbq {{.*#+}} xmm1 = xmm4[0],zero,zero,zero,zero,zero,zero,zero,xmm4[1],zero,zero,zero,zero,zero,zero,zero
-; SSE41-NEXT:    psllw %xmm1, %xmm0
-; SSE41-NEXT:    psllw %xmm1, %xmm6
+; SSE41-NEXT:    psllw %xmm4, %xmm6
 ; SSE41-NEXT:    pshufb %xmm2, %xmm6
-; SSE41-NEXT:    pand %xmm6, %xmm0
-; SSE41-NEXT:    por %xmm7, %xmm0
+; SSE41-NEXT:    pand %xmm6, %xmm1
+; SSE41-NEXT:    movdqa {{.*#+}} xmm2 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; SSE41-NEXT:    psubb %xmm3, %xmm2
+; SSE41-NEXT:    pmovzxbq {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero
+; SSE41-NEXT:    psrlw %xmm2, %xmm0
+; SSE41-NEXT:    psrlw %xmm2, %xmm5
+; SSE41-NEXT:    pshufb {{.*#+}} xmm5 = xmm5[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
+; SSE41-NEXT:    pand %xmm0, %xmm5
+; SSE41-NEXT:    por %xmm5, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: splatvar_funnnel_v16i8:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
 ; AVX1-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
-; AVX1-NEXT:    vpand %xmm3, %xmm1, %xmm4
-; AVX1-NEXT:    vpmovzxbq {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,zero,zero,zero,zero,xmm4[1],zero,zero,zero,zero,zero,zero,zero
-; AVX1-NEXT:    vpsrlw %xmm4, %xmm0, %xmm5
-; AVX1-NEXT:    vpcmpeqd %xmm6, %xmm6, %xmm6
-; AVX1-NEXT:    vpsrlw %xmm4, %xmm6, %xmm4
-; AVX1-NEXT:    vpshufb {{.*#+}} xmm4 = xmm4[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
-; AVX1-NEXT:    vpand %xmm4, %xmm5, %xmm4
 ; AVX1-NEXT:    vpsubb %xmm1, %xmm2, %xmm1
-; AVX1-NEXT:    vpand %xmm3, %xmm1, %xmm1
+; AVX1-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX1-NEXT:    vpmovzxbq {{.*#+}} xmm3 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; AVX1-NEXT:    vpsllw %xmm3, %xmm0, %xmm4
+; AVX1-NEXT:    vpcmpeqd %xmm5, %xmm5, %xmm5
+; AVX1-NEXT:    vpsllw %xmm3, %xmm5, %xmm3
+; AVX1-NEXT:    vpshufb %xmm2, %xmm3, %xmm2
+; AVX1-NEXT:    vpand %xmm2, %xmm4, %xmm2
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; AVX1-NEXT:    vpsubb %xmm1, %xmm3, %xmm1
 ; AVX1-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
-; AVX1-NEXT:    vpsllw %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpsllw %xmm1, %xmm6, %xmm1
-; AVX1-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
+; AVX1-NEXT:    vpsrlw %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpsrlw %xmm1, %xmm5, %xmm1
+; AVX1-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
 ; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpor %xmm4, %xmm0, %xmm0
+; AVX1-NEXT:    vpor %xmm0, %xmm2, %xmm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: splatvar_funnnel_v16i8:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpbroadcastb %xmm1, %xmm1
-; AVX2-NEXT:    vmovdqa {{.*#+}} xmm2 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
-; AVX2-NEXT:    vpand %xmm2, %xmm1, %xmm3
-; AVX2-NEXT:    vpmovzxbq {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,zero,zero,zero,zero,xmm3[1],zero,zero,zero,zero,zero,zero,zero
-; AVX2-NEXT:    vpsrlw %xmm3, %xmm0, %xmm4
-; AVX2-NEXT:    vpcmpeqd %xmm5, %xmm5, %xmm5
-; AVX2-NEXT:    vpsrlw %xmm3, %xmm5, %xmm3
-; AVX2-NEXT:    vpsrlw $8, %xmm3, %xmm3
-; AVX2-NEXT:    vpbroadcastb %xmm3, %xmm3
-; AVX2-NEXT:    vpand %xmm3, %xmm4, %xmm3
-; AVX2-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX2-NEXT:    vpsubb %xmm1, %xmm4, %xmm1
-; AVX2-NEXT:    vpand %xmm2, %xmm1, %xmm1
+; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX2-NEXT:    vpsubb %xmm1, %xmm2, %xmm1
+; AVX2-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX2-NEXT:    vpmovzxbq {{.*#+}} xmm2 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; AVX2-NEXT:    vpsllw %xmm2, %xmm0, %xmm3
+; AVX2-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
+; AVX2-NEXT:    vpsllw %xmm2, %xmm4, %xmm2
+; AVX2-NEXT:    vpbroadcastb %xmm2, %xmm2
+; AVX2-NEXT:    vpand %xmm2, %xmm3, %xmm2
+; AVX2-NEXT:    vmovdqa {{.*#+}} xmm3 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; AVX2-NEXT:    vpsubb %xmm1, %xmm3, %xmm1
 ; AVX2-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
-; AVX2-NEXT:    vpsllw %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpsllw %xmm1, %xmm5, %xmm1
+; AVX2-NEXT:    vpsrlw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpsrlw %xmm1, %xmm4, %xmm1
+; AVX2-NEXT:    vpsrlw $8, %xmm1, %xmm1
 ; AVX2-NEXT:    vpbroadcastb %xmm1, %xmm1
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpor %xmm3, %xmm0, %xmm0
+; AVX2-NEXT:    vpor %xmm0, %xmm2, %xmm0
 ; AVX2-NEXT:    retq
 ;
 ; AVX512F-LABEL: splatvar_funnnel_v16i8:
@@ -1498,28 +1259,16 @@ define <16 x i8> @splatvar_funnnel_v16i8(<16 x i8> %x, <16 x i8> %amt) nounwind
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
 ; XOPAVX1-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
-; XOPAVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
-; XOPAVX1-NEXT:    vpand %xmm3, %xmm1, %xmm4
-; XOPAVX1-NEXT:    vpsubb %xmm4, %xmm2, %xmm4
-; XOPAVX1-NEXT:    vpshlb %xmm4, %xmm0, %xmm4
 ; XOPAVX1-NEXT:    vpsubb %xmm1, %xmm2, %xmm1
-; XOPAVX1-NEXT:    vpand %xmm3, %xmm1, %xmm1
-; XOPAVX1-NEXT:    vpshlb %xmm1, %xmm0, %xmm0
-; XOPAVX1-NEXT:    vpor %xmm4, %xmm0, %xmm0
+; XOPAVX1-NEXT:    vprotb %xmm1, %xmm0, %xmm0
 ; XOPAVX1-NEXT:    retq
 ;
 ; XOPAVX2-LABEL: splatvar_funnnel_v16i8:
 ; XOPAVX2:       # %bb.0:
 ; XOPAVX2-NEXT:    vpbroadcastb %xmm1, %xmm1
-; XOPAVX2-NEXT:    vmovdqa {{.*#+}} xmm2 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
-; XOPAVX2-NEXT:    vpand %xmm2, %xmm1, %xmm3
-; XOPAVX2-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; XOPAVX2-NEXT:    vpsubb %xmm3, %xmm4, %xmm3
-; XOPAVX2-NEXT:    vpshlb %xmm3, %xmm0, %xmm3
-; XOPAVX2-NEXT:    vpsubb %xmm1, %xmm4, %xmm1
-; XOPAVX2-NEXT:    vpand %xmm2, %xmm1, %xmm1
-; XOPAVX2-NEXT:    vpshlb %xmm1, %xmm0, %xmm0
-; XOPAVX2-NEXT:    vpor %xmm3, %xmm0, %xmm0
+; XOPAVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; XOPAVX2-NEXT:    vpsubb %xmm1, %xmm2, %xmm1
+; XOPAVX2-NEXT:    vprotb %xmm1, %xmm0, %xmm0
 ; XOPAVX2-NEXT:    retq
 ;
 ; X32-SSE-LABEL: splatvar_funnnel_v16i8:
@@ -1527,32 +1276,33 @@ define <16 x i8> @splatvar_funnnel_v16i8(<16 x i8> %x, <16 x i8> %amt) nounwind
 ; X32-SSE-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
 ; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm1 = xmm1[0,0,2,3,4,5,6,7]
 ; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
-; X32-SSE-NEXT:    movdqa {{.*#+}} xmm2 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
-; X32-SSE-NEXT:    pxor %xmm3, %xmm3
-; X32-SSE-NEXT:    psubb %xmm1, %xmm3
-; X32-SSE-NEXT:    pand %xmm2, %xmm1
-; X32-SSE-NEXT:    pslldq {{.*#+}} xmm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm1[0]
-; X32-SSE-NEXT:    psrldq {{.*#+}} xmm1 = xmm1[15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; X32-SSE-NEXT:    movdqa %xmm0, %xmm4
-; X32-SSE-NEXT:    psrlw %xmm1, %xmm4
+; X32-SSE-NEXT:    pxor %xmm2, %xmm2
+; X32-SSE-NEXT:    psubb %xmm1, %xmm2
+; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm2
+; X32-SSE-NEXT:    movdqa {{.*#+}} xmm3 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; X32-SSE-NEXT:    psubb %xmm2, %xmm3
+; X32-SSE-NEXT:    pslldq {{.*#+}} xmm2 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm2[0]
+; X32-SSE-NEXT:    psrldq {{.*#+}} xmm2 = xmm2[15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; X32-SSE-NEXT:    movdqa %xmm0, %xmm1
+; X32-SSE-NEXT:    psllw %xmm2, %xmm1
+; X32-SSE-NEXT:    pcmpeqd %xmm4, %xmm4
 ; X32-SSE-NEXT:    pcmpeqd %xmm5, %xmm5
-; X32-SSE-NEXT:    pcmpeqd %xmm6, %xmm6
-; X32-SSE-NEXT:    psrlw %xmm1, %xmm6
-; X32-SSE-NEXT:    psrlw $8, %xmm6
-; X32-SSE-NEXT:    punpcklbw {{.*#+}} xmm6 = xmm6[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm1 = xmm6[0,0,2,3,4,5,6,7]
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
-; X32-SSE-NEXT:    pand %xmm4, %xmm1
-; X32-SSE-NEXT:    pand %xmm2, %xmm3
-; X32-SSE-NEXT:    pslldq {{.*#+}} xmm3 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm3[0]
-; X32-SSE-NEXT:    psrldq {{.*#+}} xmm3 = xmm3[15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; X32-SSE-NEXT:    psllw %xmm3, %xmm0
-; X32-SSE-NEXT:    psllw %xmm3, %xmm5
+; X32-SSE-NEXT:    psllw %xmm2, %xmm5
 ; X32-SSE-NEXT:    punpcklbw {{.*#+}} xmm5 = xmm5[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
 ; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm2 = xmm5[0,0,2,3,4,5,6,7]
 ; X32-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,0,0,0]
-; X32-SSE-NEXT:    pand %xmm2, %xmm0
-; X32-SSE-NEXT:    por %xmm1, %xmm0
+; X32-SSE-NEXT:    pand %xmm2, %xmm1
+; X32-SSE-NEXT:    pslldq {{.*#+}} xmm3 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm3[0]
+; X32-SSE-NEXT:    psrldq {{.*#+}} xmm3 = xmm3[15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; X32-SSE-NEXT:    psrlw %xmm3, %xmm0
+; X32-SSE-NEXT:    psrlw %xmm3, %xmm4
+; X32-SSE-NEXT:    psrlw $8, %xmm4
+; X32-SSE-NEXT:    punpcklbw {{.*#+}} xmm4 = xmm4[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; X32-SSE-NEXT:    pshuflw {{.*#+}} xmm2 = xmm4[0,0,2,3,4,5,6,7]
+; X32-SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[0,0,0,0]
+; X32-SSE-NEXT:    pand %xmm0, %xmm2
+; X32-SSE-NEXT:    por %xmm2, %xmm1
+; X32-SSE-NEXT:    movdqa %xmm1, %xmm0
 ; X32-SSE-NEXT:    retl
   %splat = shufflevector <16 x i8> %amt, <16 x i8> undef, <16 x i32> zeroinitializer
   %res = call <16 x i8> @llvm.fshr.v16i8(<16 x i8> %x, <16 x i8> %x, <16 x i8> %splat)
@@ -1613,29 +1363,29 @@ define <2 x i64> @constant_funnnel_v2i64(<2 x i64> %x) nounwind {
 ; AVX512F-LABEL: constant_funnnel_v2i64:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm1 = [60,50]
-; AVX512F-NEXT:    vprolvq %zmm1, %zmm0, %zmm0
+; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm1 = [4,14]
+; AVX512F-NEXT:    vprorvq %zmm1, %zmm0, %zmm0
 ; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: constant_funnnel_v2i64:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vprolvq {{.*}}(%rip), %xmm0, %xmm0
+; AVX512VL-NEXT:    vprorvq {{.*}}(%rip), %xmm0, %xmm0
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: constant_funnnel_v2i64:
 ; AVX512BW:       # %bb.0:
 ; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm1 = [60,50]
-; AVX512BW-NEXT:    vprolvq %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm1 = [4,14]
+; AVX512BW-NEXT:    vprorvq %zmm1, %zmm0, %zmm0
 ; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
 ;
 ; AVX512VLBW-LABEL: constant_funnnel_v2i64:
 ; AVX512VLBW:       # %bb.0:
-; AVX512VLBW-NEXT:    vprolvq {{.*}}(%rip), %xmm0, %xmm0
+; AVX512VLBW-NEXT:    vprorvq {{.*}}(%rip), %xmm0, %xmm0
 ; AVX512VLBW-NEXT:    retq
 ;
 ; XOP-LABEL: constant_funnnel_v2i64:
@@ -1715,29 +1465,29 @@ define <4 x i32> @constant_funnnel_v4i32(<4 x i32> %x) nounwind {
 ; AVX512F-LABEL: constant_funnnel_v4i32:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm1 = [28,27,26,25]
-; AVX512F-NEXT:    vprolvd %zmm1, %zmm0, %zmm0
+; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm1 = [4,5,6,7]
+; AVX512F-NEXT:    vprorvd %zmm1, %zmm0, %zmm0
 ; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: constant_funnnel_v4i32:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vprolvd {{.*}}(%rip), %xmm0, %xmm0
+; AVX512VL-NEXT:    vprorvd {{.*}}(%rip), %xmm0, %xmm0
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: constant_funnnel_v4i32:
 ; AVX512BW:       # %bb.0:
 ; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm1 = [28,27,26,25]
-; AVX512BW-NEXT:    vprolvd %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm1 = [4,5,6,7]
+; AVX512BW-NEXT:    vprorvd %zmm1, %zmm0, %zmm0
 ; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
 ;
 ; AVX512VLBW-LABEL: constant_funnnel_v4i32:
 ; AVX512VLBW:       # %bb.0:
-; AVX512VLBW-NEXT:    vprolvd {{.*}}(%rip), %xmm0, %xmm0
+; AVX512VLBW-NEXT:    vprorvd {{.*}}(%rip), %xmm0, %xmm0
 ; AVX512VLBW-NEXT:    retq
 ;
 ; XOP-LABEL: constant_funnnel_v4i32:
@@ -1765,55 +1515,43 @@ define <4 x i32> @constant_funnnel_v4i32(<4 x i32> %x) nounwind {
 }
 
 define <8 x i16> @constant_funnnel_v8i16(<8 x i16> %x) nounwind {
-; SSE2-LABEL: constant_funnnel_v8i16:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    movdqa {{.*#+}} xmm1 = [0,65535,65535,65535,65535,65535,65535,65535]
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = <u,32768,16384,8192,4096,2048,1024,512>
-; SSE2-NEXT:    pmulhuw %xmm0, %xmm2
-; SSE2-NEXT:    pand %xmm1, %xmm2
-; SSE2-NEXT:    pandn %xmm0, %xmm1
-; SSE2-NEXT:    pmullw {{.*}}(%rip), %xmm0
-; SSE2-NEXT:    por %xmm1, %xmm0
-; SSE2-NEXT:    por %xmm2, %xmm0
-; SSE2-NEXT:    retq
-;
-; SSE41-LABEL: constant_funnnel_v8i16:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    movdqa {{.*#+}} xmm1 = <u,32768,16384,8192,4096,2048,1024,512>
-; SSE41-NEXT:    pmulhuw %xmm0, %xmm1
-; SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3,4,5,6,7]
-; SSE41-NEXT:    pmullw {{.*}}(%rip), %xmm0
-; SSE41-NEXT:    por %xmm1, %xmm0
-; SSE41-NEXT:    retq
+; SSE-LABEL: constant_funnnel_v8i16:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movdqa {{.*#+}} xmm1 = [1,32768,16384,8192,4096,2048,1024,512]
+; SSE-NEXT:    movdqa %xmm0, %xmm2
+; SSE-NEXT:    pmulhuw %xmm1, %xmm2
+; SSE-NEXT:    pmullw %xmm1, %xmm0
+; SSE-NEXT:    por %xmm2, %xmm0
+; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: constant_funnnel_v8i16:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpmulhuw {{.*}}(%rip), %xmm0, %xmm1
-; AVX-NEXT:    vpblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3,4,5,6,7]
-; AVX-NEXT:    vpmullw {{.*}}(%rip), %xmm0, %xmm0
-; AVX-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vmovdqa {{.*#+}} xmm1 = [1,32768,16384,8192,4096,2048,1024,512]
+; AVX-NEXT:    vpmulhuw %xmm1, %xmm0, %xmm2
+; AVX-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vpor %xmm2, %xmm0, %xmm0
 ; AVX-NEXT:    retq
 ;
 ; AVX512F-LABEL: constant_funnnel_v8i16:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpmulhuw {{.*}}(%rip), %xmm0, %xmm1
-; AVX512F-NEXT:    vpblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3,4,5,6,7]
-; AVX512F-NEXT:    vpmullw {{.*}}(%rip), %xmm0, %xmm0
-; AVX512F-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm1 = [1,32768,16384,8192,4096,2048,1024,512]
+; AVX512F-NEXT:    vpmulhuw %xmm1, %xmm0, %xmm2
+; AVX512F-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
+; AVX512F-NEXT:    vpor %xmm2, %xmm0, %xmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: constant_funnnel_v8i16:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpmulhuw {{.*}}(%rip), %xmm0, %xmm1
-; AVX512VL-NEXT:    vpblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3,4,5,6,7]
-; AVX512VL-NEXT:    vpmullw {{.*}}(%rip), %xmm0, %xmm0
-; AVX512VL-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm1 = [1,32768,16384,8192,4096,2048,1024,512]
+; AVX512VL-NEXT:    vpmulhuw %xmm1, %xmm0, %xmm2
+; AVX512VL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
+; AVX512VL-NEXT:    vpor %xmm2, %xmm0, %xmm0
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: constant_funnnel_v8i16:
 ; AVX512BW:       # %bb.0:
 ; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm1 = [0,1,2,3,4,5,6,7]
+; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm1 = [16,1,2,3,4,5,6,7]
 ; AVX512BW-NEXT:    vpsrlvw %zmm1, %zmm0, %zmm1
 ; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm2 = [0,15,14,13,12,11,10,9]
 ; AVX512BW-NEXT:    vpsllvw %zmm2, %zmm0, %zmm0
@@ -1830,20 +1568,15 @@ define <8 x i16> @constant_funnnel_v8i16(<8 x i16> %x) nounwind {
 ;
 ; XOP-LABEL: constant_funnnel_v8i16:
 ; XOP:       # %bb.0:
-; XOP-NEXT:    vpshlw {{.*}}(%rip), %xmm0, %xmm1
-; XOP-NEXT:    vpshlw {{.*}}(%rip), %xmm0, %xmm0
-; XOP-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; XOP-NEXT:    vprotw {{.*}}(%rip), %xmm0, %xmm0
 ; XOP-NEXT:    retq
 ;
 ; X32-SSE-LABEL: constant_funnnel_v8i16:
 ; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    movdqa {{.*#+}} xmm1 = [0,65535,65535,65535,65535,65535,65535,65535]
-; X32-SSE-NEXT:    movdqa {{.*#+}} xmm2 = <u,32768,16384,8192,4096,2048,1024,512>
-; X32-SSE-NEXT:    pmulhuw %xmm0, %xmm2
-; X32-SSE-NEXT:    pand %xmm1, %xmm2
-; X32-SSE-NEXT:    pandn %xmm0, %xmm1
-; X32-SSE-NEXT:    pmullw {{\.LCPI.*}}, %xmm0
-; X32-SSE-NEXT:    por %xmm1, %xmm0
+; X32-SSE-NEXT:    movdqa {{.*#+}} xmm1 = [1,32768,16384,8192,4096,2048,1024,512]
+; X32-SSE-NEXT:    movdqa %xmm0, %xmm2
+; X32-SSE-NEXT:    pmulhuw %xmm1, %xmm2
+; X32-SSE-NEXT:    pmullw %xmm1, %xmm0
 ; X32-SSE-NEXT:    por %xmm2, %xmm0
 ; X32-SSE-NEXT:    retl
   %res = call <8 x i16> @llvm.fshr.v8i16(<8 x i16> %x, <8 x i16> %x, <8 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7>)
@@ -1978,9 +1711,7 @@ define <16 x i8> @constant_funnnel_v16i8(<16 x i8> %x) nounwind {
 ;
 ; XOP-LABEL: constant_funnnel_v16i8:
 ; XOP:       # %bb.0:
-; XOP-NEXT:    vpshlb {{.*}}(%rip), %xmm0, %xmm1
-; XOP-NEXT:    vpshlb {{.*}}(%rip), %xmm0, %xmm0
-; XOP-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; XOP-NEXT:    vprotb {{.*}}(%rip), %xmm0, %xmm0
 ; XOP-NEXT:    retq
 ;
 ; X32-SSE-LABEL: constant_funnnel_v16i8:
@@ -2033,27 +1764,27 @@ define <2 x i64> @splatconstant_funnnel_v2i64(<2 x i64> %x) nounwind {
 ; AVX512F-LABEL: splatconstant_funnnel_v2i64:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; AVX512F-NEXT:    vprolq $50, %zmm0, %zmm0
+; AVX512F-NEXT:    vprorq $14, %zmm0, %zmm0
 ; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: splatconstant_funnnel_v2i64:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vprolq $50, %xmm0, %xmm0
+; AVX512VL-NEXT:    vprorq $14, %xmm0, %xmm0
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: splatconstant_funnnel_v2i64:
 ; AVX512BW:       # %bb.0:
 ; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; AVX512BW-NEXT:    vprolq $50, %zmm0, %zmm0
+; AVX512BW-NEXT:    vprorq $14, %zmm0, %zmm0
 ; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
 ;
 ; AVX512VLBW-LABEL: splatconstant_funnnel_v2i64:
 ; AVX512VLBW:       # %bb.0:
-; AVX512VLBW-NEXT:    vprolq $50, %xmm0, %xmm0
+; AVX512VLBW-NEXT:    vprorq $14, %xmm0, %xmm0
 ; AVX512VLBW-NEXT:    retq
 ;
 ; XOP-LABEL: splatconstant_funnnel_v2i64:
@@ -2091,27 +1822,27 @@ define <4 x i32> @splatconstant_funnnel_v4i32(<4 x i32> %x) nounwind {
 ; AVX512F-LABEL: splatconstant_funnnel_v4i32:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; AVX512F-NEXT:    vprold $28, %zmm0, %zmm0
+; AVX512F-NEXT:    vprord $4, %zmm0, %zmm0
 ; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: splatconstant_funnnel_v4i32:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vprold $28, %xmm0, %xmm0
+; AVX512VL-NEXT:    vprord $4, %xmm0, %xmm0
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: splatconstant_funnnel_v4i32:
 ; AVX512BW:       # %bb.0:
 ; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; AVX512BW-NEXT:    vprold $28, %zmm0, %zmm0
+; AVX512BW-NEXT:    vprord $4, %zmm0, %zmm0
 ; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
 ;
 ; AVX512VLBW-LABEL: splatconstant_funnnel_v4i32:
 ; AVX512VLBW:       # %bb.0:
-; AVX512VLBW-NEXT:    vprold $28, %xmm0, %xmm0
+; AVX512VLBW-NEXT:    vprord $4, %xmm0, %xmm0
 ; AVX512VLBW-NEXT:    retq
 ;
 ; XOP-LABEL: splatconstant_funnnel_v4i32:
diff --git a/test/CodeGen/X86/vector-fshr-rot-256.ll b/test/CodeGen/X86/vector-fshr-rot-256.ll
index d42d2e54fc34744b73d41389d6e34c34cf4fb572..703d3127d18c15753997f3849bc3188877f588b0 100644
--- a/test/CodeGen/X86/vector-fshr-rot-256.ll
+++ b/test/CodeGen/X86/vector-fshr-rot-256.ll
@@ -63,51 +63,53 @@ define <4 x i64> @var_funnnel_v4i64(<4 x i64> %x, <4 x i64> %amt) nounwind {
 ; AVX2-NEXT:    vpor %ymm3, %ymm0, %ymm0
 ; AVX2-NEXT:    retq
 ;
-; AVX512-LABEL: var_funnnel_v4i64:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [63,63,63,63]
-; AVX512-NEXT:    vpand %ymm2, %ymm1, %ymm3
-; AVX512-NEXT:    vpsrlvq %ymm3, %ymm0, %ymm3
-; AVX512-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX512-NEXT:    vpsubq %ymm1, %ymm4, %ymm1
-; AVX512-NEXT:    vpand %ymm2, %ymm1, %ymm1
-; AVX512-NEXT:    vpsllvq %ymm1, %ymm0, %ymm0
-; AVX512-NEXT:    vpor %ymm3, %ymm0, %ymm0
-; AVX512-NEXT:    retq
+; AVX512F-LABEL: var_funnnel_v4i64:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512F-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512F-NEXT:    vprorvq %zmm1, %zmm0, %zmm0
+; AVX512F-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: var_funnnel_v4i64:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vprorvq %ymm1, %ymm0, %ymm0
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: var_funnnel_v4i64:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512BW-NEXT:    vprorvq %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
+; AVX512BW-NEXT:    retq
+;
+; AVX512VLBW-LABEL: var_funnnel_v4i64:
+; AVX512VLBW:       # %bb.0:
+; AVX512VLBW-NEXT:    vprorvq %ymm1, %ymm0, %ymm0
+; AVX512VLBW-NEXT:    retq
 ;
 ; XOPAVX1-LABEL: var_funnnel_v4i64:
 ; XOPAVX1:       # %bb.0:
-; XOPAVX1-NEXT:    vandps {{.*}}(%rip), %ymm1, %ymm2
-; XOPAVX1-NEXT:    vextractf128 $1, %ymm2, %xmm3
-; XOPAVX1-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; XOPAVX1-NEXT:    vpsubq %xmm3, %xmm4, %xmm3
-; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm5
-; XOPAVX1-NEXT:    vpshlq %xmm3, %xmm5, %xmm3
-; XOPAVX1-NEXT:    vpsubq %xmm2, %xmm4, %xmm2
-; XOPAVX1-NEXT:    vpshlq %xmm2, %xmm0, %xmm2
-; XOPAVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm2, %ymm2
-; XOPAVX1-NEXT:    vextractf128 $1, %ymm1, %xmm3
-; XOPAVX1-NEXT:    vpsubq %xmm3, %xmm4, %xmm3
-; XOPAVX1-NEXT:    vmovdqa {{.*#+}} xmm6 = [63,63]
-; XOPAVX1-NEXT:    vpand %xmm6, %xmm3, %xmm3
-; XOPAVX1-NEXT:    vpshlq %xmm3, %xmm5, %xmm3
-; XOPAVX1-NEXT:    vpsubq %xmm1, %xmm4, %xmm1
-; XOPAVX1-NEXT:    vpand %xmm6, %xmm1, %xmm1
-; XOPAVX1-NEXT:    vpshlq %xmm1, %xmm0, %xmm0
-; XOPAVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm0
-; XOPAVX1-NEXT:    vorps %ymm2, %ymm0, %ymm0
+; XOPAVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; XOPAVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; XOPAVX1-NEXT:    vpsubq %xmm2, %xmm3, %xmm2
+; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
+; XOPAVX1-NEXT:    vprotq %xmm2, %xmm4, %xmm2
+; XOPAVX1-NEXT:    vpsubq %xmm1, %xmm3, %xmm1
+; XOPAVX1-NEXT:    vprotq %xmm1, %xmm0, %xmm0
+; XOPAVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
 ; XOPAVX1-NEXT:    retq
 ;
 ; XOPAVX2-LABEL: var_funnnel_v4i64:
 ; XOPAVX2:       # %bb.0:
-; XOPAVX2-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [63,63,63,63]
-; XOPAVX2-NEXT:    vpand %ymm2, %ymm1, %ymm3
-; XOPAVX2-NEXT:    vpsrlvq %ymm3, %ymm0, %ymm3
-; XOPAVX2-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; XOPAVX2-NEXT:    vpsubq %ymm1, %ymm4, %ymm1
-; XOPAVX2-NEXT:    vpand %ymm2, %ymm1, %ymm1
-; XOPAVX2-NEXT:    vpsllvq %ymm1, %ymm0, %ymm0
-; XOPAVX2-NEXT:    vpor %ymm3, %ymm0, %ymm0
+; XOPAVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; XOPAVX2-NEXT:    vpsubq %ymm1, %ymm2, %ymm1
+; XOPAVX2-NEXT:    vextracti128 $1, %ymm1, %xmm2
+; XOPAVX2-NEXT:    vextracti128 $1, %ymm0, %xmm3
+; XOPAVX2-NEXT:    vprotq %xmm2, %xmm3, %xmm2
+; XOPAVX2-NEXT:    vprotq %xmm1, %xmm0, %xmm0
+; XOPAVX2-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm0
 ; XOPAVX2-NEXT:    retq
   %res = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %x, <4 x i64> %x, <4 x i64> %amt)
   ret <4 x i64> %res
@@ -116,109 +118,102 @@ define <4 x i64> @var_funnnel_v4i64(<4 x i64> %x, <4 x i64> %amt) nounwind {
 define <8 x i32> @var_funnnel_v8i32(<8 x i32> %x, <8 x i32> %amt) nounwind {
 ; AVX1-LABEL: var_funnnel_v8i32:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
-; AVX1-NEXT:    vandps {{.*}}(%rip), %ymm1, %ymm3
-; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm4
-; AVX1-NEXT:    vpsrldq {{.*#+}} xmm5 = xmm4[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX1-NEXT:    vpsrld %xmm5, %xmm2, %xmm5
-; AVX1-NEXT:    vpsrlq $32, %xmm4, %xmm6
-; AVX1-NEXT:    vpsrld %xmm6, %xmm2, %xmm6
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm5 = xmm6[0,1,2,3],xmm5[4,5,6,7]
-; AVX1-NEXT:    vpxor %xmm6, %xmm6, %xmm6
-; AVX1-NEXT:    vpunpckhdq {{.*#+}} xmm7 = xmm4[2],xmm6[2],xmm4[3],xmm6[3]
-; AVX1-NEXT:    vpsrld %xmm7, %xmm2, %xmm7
-; AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm4 = xmm4[0],zero,xmm4[1],zero
-; AVX1-NEXT:    vpsrld %xmm4, %xmm2, %xmm4
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm4 = xmm4[0,1,2,3],xmm7[4,5,6,7]
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm4 = xmm4[0,1],xmm5[2,3],xmm4[4,5],xmm5[6,7]
-; AVX1-NEXT:    vpsrldq {{.*#+}} xmm5 = xmm3[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX1-NEXT:    vpsrld %xmm5, %xmm0, %xmm5
-; AVX1-NEXT:    vpsrlq $32, %xmm3, %xmm7
-; AVX1-NEXT:    vpsrld %xmm7, %xmm0, %xmm7
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm5 = xmm7[0,1,2,3],xmm5[4,5,6,7]
-; AVX1-NEXT:    vpunpckhdq {{.*#+}} xmm7 = xmm3[2],xmm6[2],xmm3[3],xmm6[3]
-; AVX1-NEXT:    vpsrld %xmm7, %xmm0, %xmm7
-; AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm3 = xmm3[0],zero,xmm3[1],zero
-; AVX1-NEXT:    vpsrld %xmm3, %xmm0, %xmm3
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm3 = xmm3[0,1,2,3],xmm7[4,5,6,7]
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm3 = xmm3[0,1],xmm5[2,3],xmm3[4,5],xmm5[6,7]
-; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm3, %ymm3
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm4
-; AVX1-NEXT:    vpsubd %xmm4, %xmm6, %xmm4
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm5 = [31,31,31,31]
-; AVX1-NEXT:    vpand %xmm5, %xmm4, %xmm4
-; AVX1-NEXT:    vpslld $23, %xmm4, %xmm4
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm7 = [1065353216,1065353216,1065353216,1065353216]
-; AVX1-NEXT:    vpaddd %xmm7, %xmm4, %xmm4
-; AVX1-NEXT:    vcvttps2dq %xmm4, %xmm4
-; AVX1-NEXT:    vpmulld %xmm4, %xmm2, %xmm2
-; AVX1-NEXT:    vpsubd %xmm1, %xmm6, %xmm1
-; AVX1-NEXT:    vpand %xmm5, %xmm1, %xmm1
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; AVX1-NEXT:    vpxor %xmm8, %xmm8, %xmm8
+; AVX1-NEXT:    vpsubd %xmm2, %xmm8, %xmm2
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = [31,31,31,31]
+; AVX1-NEXT:    vpand %xmm4, %xmm2, %xmm2
+; AVX1-NEXT:    vpslld $23, %xmm2, %xmm2
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm5 = [1065353216,1065353216,1065353216,1065353216]
+; AVX1-NEXT:    vpaddd %xmm5, %xmm2, %xmm2
+; AVX1-NEXT:    vcvttps2dq %xmm2, %xmm2
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm6 = xmm2[1,1,3,3]
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm7
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm7[1,1,3,3]
+; AVX1-NEXT:    vpmuludq %xmm6, %xmm3, %xmm3
+; AVX1-NEXT:    vpmuludq %xmm2, %xmm7, %xmm2
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm6 = xmm2[1,1,3,3]
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm6 = xmm6[0,1],xmm3[2,3],xmm6[4,5],xmm3[6,7]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[0,0,2,2]
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0,1],xmm3[2,3],xmm2[4,5],xmm3[6,7]
+; AVX1-NEXT:    vpor %xmm6, %xmm2, %xmm2
+; AVX1-NEXT:    vpsubd %xmm1, %xmm8, %xmm1
+; AVX1-NEXT:    vpand %xmm4, %xmm1, %xmm1
 ; AVX1-NEXT:    vpslld $23, %xmm1, %xmm1
-; AVX1-NEXT:    vpaddd %xmm7, %xmm1, %xmm1
+; AVX1-NEXT:    vpaddd %xmm5, %xmm1, %xmm1
 ; AVX1-NEXT:    vcvttps2dq %xmm1, %xmm1
-; AVX1-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[1,1,3,3]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm0[1,1,3,3]
+; AVX1-NEXT:    vpmuludq %xmm3, %xmm4, %xmm3
+; AVX1-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm3[2,3],xmm1[4,5],xmm3[6,7]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[0,0,2,2]
+; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm3[2,3],xmm0[4,5],xmm3[6,7]
+; AVX1-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
-; AVX1-NEXT:    vorps %ymm3, %ymm0, %ymm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: var_funnnel_v8i32:
 ; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX2-NEXT:    vpsubd %ymm1, %ymm2, %ymm1
 ; AVX2-NEXT:    vpbroadcastd {{.*#+}} ymm2 = [31,31,31,31,31,31,31,31]
-; AVX2-NEXT:    vpand %ymm2, %ymm1, %ymm3
-; AVX2-NEXT:    vpsrlvd %ymm3, %ymm0, %ymm3
-; AVX2-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX2-NEXT:    vpsubd %ymm1, %ymm4, %ymm1
 ; AVX2-NEXT:    vpand %ymm2, %ymm1, %ymm1
-; AVX2-NEXT:    vpsllvd %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpor %ymm3, %ymm0, %ymm0
+; AVX2-NEXT:    vpsllvd %ymm1, %ymm0, %ymm2
+; AVX2-NEXT:    vpbroadcastd {{.*#+}} ymm3 = [32,32,32,32,32,32,32,32]
+; AVX2-NEXT:    vpsubd %ymm1, %ymm3, %ymm1
+; AVX2-NEXT:    vpsrlvd %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpor %ymm0, %ymm2, %ymm0
 ; AVX2-NEXT:    retq
 ;
-; AVX512-LABEL: var_funnnel_v8i32:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpbroadcastd {{.*#+}} ymm2 = [31,31,31,31,31,31,31,31]
-; AVX512-NEXT:    vpand %ymm2, %ymm1, %ymm3
-; AVX512-NEXT:    vpsrlvd %ymm3, %ymm0, %ymm3
-; AVX512-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX512-NEXT:    vpsubd %ymm1, %ymm4, %ymm1
-; AVX512-NEXT:    vpand %ymm2, %ymm1, %ymm1
-; AVX512-NEXT:    vpsllvd %ymm1, %ymm0, %ymm0
-; AVX512-NEXT:    vpor %ymm3, %ymm0, %ymm0
-; AVX512-NEXT:    retq
+; AVX512F-LABEL: var_funnnel_v8i32:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512F-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512F-NEXT:    vprorvd %zmm1, %zmm0, %zmm0
+; AVX512F-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: var_funnnel_v8i32:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vprorvd %ymm1, %ymm0, %ymm0
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: var_funnnel_v8i32:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512BW-NEXT:    vprorvd %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
+; AVX512BW-NEXT:    retq
+;
+; AVX512VLBW-LABEL: var_funnnel_v8i32:
+; AVX512VLBW:       # %bb.0:
+; AVX512VLBW-NEXT:    vprorvd %ymm1, %ymm0, %ymm0
+; AVX512VLBW-NEXT:    retq
 ;
 ; XOPAVX1-LABEL: var_funnnel_v8i32:
 ; XOPAVX1:       # %bb.0:
-; XOPAVX1-NEXT:    vandps {{.*}}(%rip), %ymm1, %ymm2
-; XOPAVX1-NEXT:    vextractf128 $1, %ymm2, %xmm3
-; XOPAVX1-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; XOPAVX1-NEXT:    vpsubd %xmm3, %xmm4, %xmm3
-; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm5
-; XOPAVX1-NEXT:    vpshld %xmm3, %xmm5, %xmm3
-; XOPAVX1-NEXT:    vpsubd %xmm2, %xmm4, %xmm2
-; XOPAVX1-NEXT:    vpshld %xmm2, %xmm0, %xmm2
-; XOPAVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm2, %ymm2
-; XOPAVX1-NEXT:    vextractf128 $1, %ymm1, %xmm3
-; XOPAVX1-NEXT:    vpsubd %xmm3, %xmm4, %xmm3
-; XOPAVX1-NEXT:    vmovdqa {{.*#+}} xmm6 = [31,31,31,31]
-; XOPAVX1-NEXT:    vpand %xmm6, %xmm3, %xmm3
-; XOPAVX1-NEXT:    vpshld %xmm3, %xmm5, %xmm3
-; XOPAVX1-NEXT:    vpsubd %xmm1, %xmm4, %xmm1
-; XOPAVX1-NEXT:    vpand %xmm6, %xmm1, %xmm1
-; XOPAVX1-NEXT:    vpshld %xmm1, %xmm0, %xmm0
-; XOPAVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm0
-; XOPAVX1-NEXT:    vorps %ymm2, %ymm0, %ymm0
+; XOPAVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; XOPAVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; XOPAVX1-NEXT:    vpsubd %xmm2, %xmm3, %xmm2
+; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
+; XOPAVX1-NEXT:    vprotd %xmm2, %xmm4, %xmm2
+; XOPAVX1-NEXT:    vpsubd %xmm1, %xmm3, %xmm1
+; XOPAVX1-NEXT:    vprotd %xmm1, %xmm0, %xmm0
+; XOPAVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
 ; XOPAVX1-NEXT:    retq
 ;
 ; XOPAVX2-LABEL: var_funnnel_v8i32:
 ; XOPAVX2:       # %bb.0:
-; XOPAVX2-NEXT:    vpbroadcastd {{.*#+}} ymm2 = [31,31,31,31,31,31,31,31]
-; XOPAVX2-NEXT:    vpand %ymm2, %ymm1, %ymm3
-; XOPAVX2-NEXT:    vpsrlvd %ymm3, %ymm0, %ymm3
-; XOPAVX2-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; XOPAVX2-NEXT:    vpsubd %ymm1, %ymm4, %ymm1
-; XOPAVX2-NEXT:    vpand %ymm2, %ymm1, %ymm1
-; XOPAVX2-NEXT:    vpsllvd %ymm1, %ymm0, %ymm0
-; XOPAVX2-NEXT:    vpor %ymm3, %ymm0, %ymm0
+; XOPAVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; XOPAVX2-NEXT:    vpsubd %ymm1, %ymm2, %ymm1
+; XOPAVX2-NEXT:    vextracti128 $1, %ymm1, %xmm2
+; XOPAVX2-NEXT:    vextracti128 $1, %ymm0, %xmm3
+; XOPAVX2-NEXT:    vprotd %xmm2, %xmm3, %xmm2
+; XOPAVX2-NEXT:    vprotd %xmm1, %xmm0, %xmm0
+; XOPAVX2-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm0
 ; XOPAVX2-NEXT:    retq
   %res = call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %x, <8 x i32> %x, <8 x i32> %amt)
   ret <8 x i32> %res
@@ -227,196 +222,146 @@ define <8 x i32> @var_funnnel_v8i32(<8 x i32> %x, <8 x i32> %amt) nounwind {
 define <16 x i16> @var_funnnel_v16i16(<16 x i16> %x, <16 x i16> %amt) nounwind {
 ; AVX1-LABEL: var_funnnel_v16i16:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vandps {{.*}}(%rip), %ymm1, %ymm3
-; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm2
-; AVX1-NEXT:    vpsllw $12, %xmm2, %xmm4
-; AVX1-NEXT:    vpsllw $4, %xmm2, %xmm2
-; AVX1-NEXT:    vpor %xmm4, %xmm2, %xmm4
-; AVX1-NEXT:    vpaddw %xmm4, %xmm4, %xmm5
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
-; AVX1-NEXT:    vpsrlw $8, %xmm2, %xmm6
-; AVX1-NEXT:    vpblendvb %xmm4, %xmm6, %xmm2, %xmm4
-; AVX1-NEXT:    vpsrlw $4, %xmm4, %xmm6
-; AVX1-NEXT:    vpblendvb %xmm5, %xmm6, %xmm4, %xmm4
-; AVX1-NEXT:    vpsrlw $2, %xmm4, %xmm6
-; AVX1-NEXT:    vpaddw %xmm5, %xmm5, %xmm5
-; AVX1-NEXT:    vpblendvb %xmm5, %xmm6, %xmm4, %xmm4
-; AVX1-NEXT:    vpsrlw $1, %xmm4, %xmm6
-; AVX1-NEXT:    vpaddw %xmm5, %xmm5, %xmm5
-; AVX1-NEXT:    vpblendvb %xmm5, %xmm6, %xmm4, %xmm4
-; AVX1-NEXT:    vpsllw $12, %xmm3, %xmm5
-; AVX1-NEXT:    vpsllw $4, %xmm3, %xmm3
-; AVX1-NEXT:    vpor %xmm5, %xmm3, %xmm3
-; AVX1-NEXT:    vpaddw %xmm3, %xmm3, %xmm5
-; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm6
-; AVX1-NEXT:    vpblendvb %xmm3, %xmm6, %xmm0, %xmm3
-; AVX1-NEXT:    vpsrlw $4, %xmm3, %xmm6
-; AVX1-NEXT:    vpblendvb %xmm5, %xmm6, %xmm3, %xmm3
-; AVX1-NEXT:    vpsrlw $2, %xmm3, %xmm6
-; AVX1-NEXT:    vpaddw %xmm5, %xmm5, %xmm5
-; AVX1-NEXT:    vpblendvb %xmm5, %xmm6, %xmm3, %xmm3
-; AVX1-NEXT:    vpsrlw $1, %xmm3, %xmm6
-; AVX1-NEXT:    vpaddw %xmm5, %xmm5, %xmm5
-; AVX1-NEXT:    vpblendvb %xmm5, %xmm6, %xmm3, %xmm3
-; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm3, %ymm8
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm4
-; AVX1-NEXT:    vpxor %xmm5, %xmm5, %xmm5
-; AVX1-NEXT:    vpsubw %xmm4, %xmm5, %xmm4
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm6 = [15,15,15,15,15,15,15,15]
-; AVX1-NEXT:    vpand %xmm6, %xmm4, %xmm4
-; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm7 = xmm4[4],xmm5[4],xmm4[5],xmm5[5],xmm4[6],xmm5[6],xmm4[7],xmm5[7]
-; AVX1-NEXT:    vpslld $23, %xmm7, %xmm7
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [1065353216,1065353216,1065353216,1065353216]
-; AVX1-NEXT:    vpaddd %xmm3, %xmm7, %xmm7
-; AVX1-NEXT:    vcvttps2dq %xmm7, %xmm7
-; AVX1-NEXT:    vpmovzxwd {{.*#+}} xmm4 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero
-; AVX1-NEXT:    vpslld $23, %xmm4, %xmm4
-; AVX1-NEXT:    vpaddd %xmm3, %xmm4, %xmm4
-; AVX1-NEXT:    vcvttps2dq %xmm4, %xmm4
-; AVX1-NEXT:    vpackusdw %xmm7, %xmm4, %xmm4
-; AVX1-NEXT:    vpmullw %xmm4, %xmm2, %xmm2
-; AVX1-NEXT:    vpsubw %xmm1, %xmm5, %xmm1
-; AVX1-NEXT:    vpand %xmm6, %xmm1, %xmm1
-; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm1[4],xmm5[4],xmm1[5],xmm5[5],xmm1[6],xmm5[6],xmm1[7],xmm5[7]
-; AVX1-NEXT:    vpslld $23, %xmm4, %xmm4
-; AVX1-NEXT:    vpaddd %xmm3, %xmm4, %xmm4
-; AVX1-NEXT:    vcvttps2dq %xmm4, %xmm4
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; AVX1-NEXT:    vpsubw %xmm2, %xmm3, %xmm2
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = [15,15,15,15,15,15,15,15]
+; AVX1-NEXT:    vpand %xmm4, %xmm2, %xmm2
+; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm5 = xmm2[4],xmm3[4],xmm2[5],xmm3[5],xmm2[6],xmm3[6],xmm2[7],xmm3[7]
+; AVX1-NEXT:    vpslld $23, %xmm5, %xmm5
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm6 = [1065353216,1065353216,1065353216,1065353216]
+; AVX1-NEXT:    vpaddd %xmm6, %xmm5, %xmm5
+; AVX1-NEXT:    vcvttps2dq %xmm5, %xmm5
+; AVX1-NEXT:    vpmovzxwd {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero
+; AVX1-NEXT:    vpslld $23, %xmm2, %xmm2
+; AVX1-NEXT:    vpaddd %xmm6, %xmm2, %xmm2
+; AVX1-NEXT:    vcvttps2dq %xmm2, %xmm2
+; AVX1-NEXT:    vpackusdw %xmm5, %xmm2, %xmm2
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm5
+; AVX1-NEXT:    vpmulhuw %xmm2, %xmm5, %xmm7
+; AVX1-NEXT:    vpmullw %xmm2, %xmm5, %xmm2
+; AVX1-NEXT:    vpor %xmm7, %xmm2, %xmm2
+; AVX1-NEXT:    vpsubw %xmm1, %xmm3, %xmm1
+; AVX1-NEXT:    vpand %xmm4, %xmm1, %xmm1
+; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm1[4],xmm3[4],xmm1[5],xmm3[5],xmm1[6],xmm3[6],xmm1[7],xmm3[7]
+; AVX1-NEXT:    vpslld $23, %xmm3, %xmm3
+; AVX1-NEXT:    vpaddd %xmm6, %xmm3, %xmm3
+; AVX1-NEXT:    vcvttps2dq %xmm3, %xmm3
 ; AVX1-NEXT:    vpmovzxwd {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
 ; AVX1-NEXT:    vpslld $23, %xmm1, %xmm1
-; AVX1-NEXT:    vpaddd %xmm3, %xmm1, %xmm1
+; AVX1-NEXT:    vpaddd %xmm6, %xmm1, %xmm1
 ; AVX1-NEXT:    vcvttps2dq %xmm1, %xmm1
-; AVX1-NEXT:    vpackusdw %xmm4, %xmm1, %xmm1
+; AVX1-NEXT:    vpackusdw %xmm3, %xmm1, %xmm1
+; AVX1-NEXT:    vpmulhuw %xmm1, %xmm0, %xmm3
 ; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpor %xmm3, %xmm0, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
-; AVX1-NEXT:    vorps %ymm8, %ymm0, %ymm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: var_funnnel_v16i16:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
 ; AVX2-NEXT:    vpunpckhwd {{.*#+}} ymm3 = ymm2[4],ymm0[4],ymm2[5],ymm0[5],ymm2[6],ymm0[6],ymm2[7],ymm0[7],ymm2[12],ymm0[12],ymm2[13],ymm0[13],ymm2[14],ymm0[14],ymm2[15],ymm0[15]
-; AVX2-NEXT:    vmovdqa {{.*#+}} ymm4 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; AVX2-NEXT:    vpand %ymm4, %ymm1, %ymm5
-; AVX2-NEXT:    vpunpckhwd {{.*#+}} ymm6 = ymm5[4],ymm2[4],ymm5[5],ymm2[5],ymm5[6],ymm2[6],ymm5[7],ymm2[7],ymm5[12],ymm2[12],ymm5[13],ymm2[13],ymm5[14],ymm2[14],ymm5[15],ymm2[15]
-; AVX2-NEXT:    vpsrlvd %ymm6, %ymm3, %ymm6
-; AVX2-NEXT:    vpsrld $16, %ymm6, %ymm6
-; AVX2-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm2[0],ymm0[0],ymm2[1],ymm0[1],ymm2[2],ymm0[2],ymm2[3],ymm0[3],ymm2[8],ymm0[8],ymm2[9],ymm0[9],ymm2[10],ymm0[10],ymm2[11],ymm0[11]
-; AVX2-NEXT:    vpunpcklwd {{.*#+}} ymm5 = ymm5[0],ymm2[0],ymm5[1],ymm2[1],ymm5[2],ymm2[2],ymm5[3],ymm2[3],ymm5[8],ymm2[8],ymm5[9],ymm2[9],ymm5[10],ymm2[10],ymm5[11],ymm2[11]
-; AVX2-NEXT:    vpsrlvd %ymm5, %ymm0, %ymm5
-; AVX2-NEXT:    vpsrld $16, %ymm5, %ymm5
-; AVX2-NEXT:    vpackusdw %ymm6, %ymm5, %ymm5
 ; AVX2-NEXT:    vpsubw %ymm1, %ymm2, %ymm1
-; AVX2-NEXT:    vpand %ymm4, %ymm1, %ymm1
+; AVX2-NEXT:    vpand {{.*}}(%rip), %ymm1, %ymm1
 ; AVX2-NEXT:    vpunpckhwd {{.*#+}} ymm4 = ymm1[4],ymm2[4],ymm1[5],ymm2[5],ymm1[6],ymm2[6],ymm1[7],ymm2[7],ymm1[12],ymm2[12],ymm1[13],ymm2[13],ymm1[14],ymm2[14],ymm1[15],ymm2[15]
-; AVX2-NEXT:    vpsllvd %ymm4, %ymm3, %ymm3
+; AVX2-NEXT:    vpsllvd %ymm4, %ymm3, %ymm4
+; AVX2-NEXT:    vpsrld $16, %ymm4, %ymm4
+; AVX2-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm2[0],ymm0[0],ymm2[1],ymm0[1],ymm2[2],ymm0[2],ymm2[3],ymm0[3],ymm2[8],ymm0[8],ymm2[9],ymm0[9],ymm2[10],ymm0[10],ymm2[11],ymm0[11]
+; AVX2-NEXT:    vpunpcklwd {{.*#+}} ymm5 = ymm1[0],ymm2[0],ymm1[1],ymm2[1],ymm1[2],ymm2[2],ymm1[3],ymm2[3],ymm1[8],ymm2[8],ymm1[9],ymm2[9],ymm1[10],ymm2[10],ymm1[11],ymm2[11]
+; AVX2-NEXT:    vpsllvd %ymm5, %ymm0, %ymm5
+; AVX2-NEXT:    vpsrld $16, %ymm5, %ymm5
+; AVX2-NEXT:    vpackusdw %ymm4, %ymm5, %ymm4
+; AVX2-NEXT:    vmovdqa {{.*#+}} ymm5 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
+; AVX2-NEXT:    vpsubw %ymm1, %ymm5, %ymm1
+; AVX2-NEXT:    vpunpckhwd {{.*#+}} ymm5 = ymm1[4],ymm2[4],ymm1[5],ymm2[5],ymm1[6],ymm2[6],ymm1[7],ymm2[7],ymm1[12],ymm2[12],ymm1[13],ymm2[13],ymm1[14],ymm2[14],ymm1[15],ymm2[15]
+; AVX2-NEXT:    vpsrlvd %ymm5, %ymm3, %ymm3
 ; AVX2-NEXT:    vpsrld $16, %ymm3, %ymm3
 ; AVX2-NEXT:    vpunpcklwd {{.*#+}} ymm1 = ymm1[0],ymm2[0],ymm1[1],ymm2[1],ymm1[2],ymm2[2],ymm1[3],ymm2[3],ymm1[8],ymm2[8],ymm1[9],ymm2[9],ymm1[10],ymm2[10],ymm1[11],ymm2[11]
-; AVX2-NEXT:    vpsllvd %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpsrlvd %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vpsrld $16, %ymm0, %ymm0
 ; AVX2-NEXT:    vpackusdw %ymm3, %ymm0, %ymm0
-; AVX2-NEXT:    vpor %ymm5, %ymm0, %ymm0
+; AVX2-NEXT:    vpor %ymm0, %ymm4, %ymm0
 ; AVX2-NEXT:    retq
 ;
 ; AVX512F-LABEL: var_funnnel_v16i16:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; AVX512F-NEXT:    vpand %ymm2, %ymm1, %ymm3
-; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm3 = ymm3[0],zero,ymm3[1],zero,ymm3[2],zero,ymm3[3],zero,ymm3[4],zero,ymm3[5],zero,ymm3[6],zero,ymm3[7],zero,ymm3[8],zero,ymm3[9],zero,ymm3[10],zero,ymm3[11],zero,ymm3[12],zero,ymm3[13],zero,ymm3[14],zero,ymm3[15],zero
+; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX512F-NEXT:    vpsubw %ymm1, %ymm2, %ymm1
+; AVX512F-NEXT:    vpand {{.*}}(%rip), %ymm1, %ymm1
+; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm2 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero
 ; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
-; AVX512F-NEXT:    vpsrlvd %zmm3, %zmm0, %zmm3
-; AVX512F-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX512F-NEXT:    vpsubw %ymm1, %ymm4, %ymm1
-; AVX512F-NEXT:    vpand %ymm2, %ymm1, %ymm1
+; AVX512F-NEXT:    vpsllvd %zmm2, %zmm0, %zmm2
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm3 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
+; AVX512F-NEXT:    vpsubw %ymm1, %ymm3, %ymm1
 ; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero
-; AVX512F-NEXT:    vpsllvd %zmm1, %zmm0, %zmm0
-; AVX512F-NEXT:    vpord %zmm3, %zmm0, %zmm0
+; AVX512F-NEXT:    vpsrlvd %zmm1, %zmm0, %zmm0
+; AVX512F-NEXT:    vpord %zmm0, %zmm2, %zmm0
 ; AVX512F-NEXT:    vpmovdw %zmm0, %ymm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: var_funnnel_v16i16:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; AVX512VL-NEXT:    vpand %ymm2, %ymm1, %ymm3
-; AVX512VL-NEXT:    vpmovzxwd {{.*#+}} zmm3 = ymm3[0],zero,ymm3[1],zero,ymm3[2],zero,ymm3[3],zero,ymm3[4],zero,ymm3[5],zero,ymm3[6],zero,ymm3[7],zero,ymm3[8],zero,ymm3[9],zero,ymm3[10],zero,ymm3[11],zero,ymm3[12],zero,ymm3[13],zero,ymm3[14],zero,ymm3[15],zero
+; AVX512VL-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX512VL-NEXT:    vpsubw %ymm1, %ymm2, %ymm1
+; AVX512VL-NEXT:    vpand {{.*}}(%rip), %ymm1, %ymm1
+; AVX512VL-NEXT:    vpmovzxwd {{.*#+}} zmm2 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero
 ; AVX512VL-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
-; AVX512VL-NEXT:    vpsrlvd %zmm3, %zmm0, %zmm3
-; AVX512VL-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX512VL-NEXT:    vpsubw %ymm1, %ymm4, %ymm1
-; AVX512VL-NEXT:    vpand %ymm2, %ymm1, %ymm1
+; AVX512VL-NEXT:    vpsllvd %zmm2, %zmm0, %zmm2
+; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm3 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
+; AVX512VL-NEXT:    vpsubw %ymm1, %ymm3, %ymm1
 ; AVX512VL-NEXT:    vpmovzxwd {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero
-; AVX512VL-NEXT:    vpsllvd %zmm1, %zmm0, %zmm0
-; AVX512VL-NEXT:    vpord %zmm3, %zmm0, %zmm0
+; AVX512VL-NEXT:    vpsrlvd %zmm1, %zmm0, %zmm0
+; AVX512VL-NEXT:    vpord %zmm0, %zmm2, %zmm0
 ; AVX512VL-NEXT:    vpmovdw %zmm0, %ymm0
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: var_funnnel_v16i16:
 ; AVX512BW:       # %bb.0:
 ; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; AVX512BW-NEXT:    vmovdqa {{.*#+}} ymm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; AVX512BW-NEXT:    vpand %ymm2, %ymm1, %ymm3
-; AVX512BW-NEXT:    vpsrlvw %zmm3, %zmm0, %zmm3
-; AVX512BW-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX512BW-NEXT:    vpsubw %ymm1, %ymm4, %ymm1
-; AVX512BW-NEXT:    vpand %ymm2, %ymm1, %ymm1
-; AVX512BW-NEXT:    vpsllvw %zmm1, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpor %ymm3, %ymm0, %ymm0
+; AVX512BW-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX512BW-NEXT:    vpsubw %ymm1, %ymm2, %ymm1
+; AVX512BW-NEXT:    vpand {{.*}}(%rip), %ymm1, %ymm1
+; AVX512BW-NEXT:    vpsllvw %zmm1, %zmm0, %zmm2
+; AVX512BW-NEXT:    vmovdqa {{.*#+}} ymm3 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
+; AVX512BW-NEXT:    vpsubw %ymm1, %ymm3, %ymm1
+; AVX512BW-NEXT:    vpsrlvw %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpor %ymm0, %ymm2, %ymm0
 ; AVX512BW-NEXT:    retq
 ;
 ; AVX512VLBW-LABEL: var_funnnel_v16i16:
 ; AVX512VLBW:       # %bb.0:
-; AVX512VLBW-NEXT:    vmovdqa {{.*#+}} ymm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; AVX512VLBW-NEXT:    vpand %ymm2, %ymm1, %ymm3
-; AVX512VLBW-NEXT:    vpsrlvw %ymm3, %ymm0, %ymm3
-; AVX512VLBW-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX512VLBW-NEXT:    vpsubw %ymm1, %ymm4, %ymm1
-; AVX512VLBW-NEXT:    vpand %ymm2, %ymm1, %ymm1
-; AVX512VLBW-NEXT:    vpsllvw %ymm1, %ymm0, %ymm0
-; AVX512VLBW-NEXT:    vpor %ymm3, %ymm0, %ymm0
+; AVX512VLBW-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX512VLBW-NEXT:    vpsubw %ymm1, %ymm2, %ymm1
+; AVX512VLBW-NEXT:    vpand {{.*}}(%rip), %ymm1, %ymm1
+; AVX512VLBW-NEXT:    vpsllvw %ymm1, %ymm0, %ymm2
+; AVX512VLBW-NEXT:    vmovdqa {{.*#+}} ymm3 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
+; AVX512VLBW-NEXT:    vpsubw %ymm1, %ymm3, %ymm1
+; AVX512VLBW-NEXT:    vpsrlvw %ymm1, %ymm0, %ymm0
+; AVX512VLBW-NEXT:    vpor %ymm0, %ymm2, %ymm0
 ; AVX512VLBW-NEXT:    retq
 ;
 ; XOPAVX1-LABEL: var_funnnel_v16i16:
 ; XOPAVX1:       # %bb.0:
-; XOPAVX1-NEXT:    vandps {{.*}}(%rip), %ymm1, %ymm2
-; XOPAVX1-NEXT:    vextractf128 $1, %ymm2, %xmm3
-; XOPAVX1-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; XOPAVX1-NEXT:    vpsubw %xmm3, %xmm4, %xmm3
-; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm5
-; XOPAVX1-NEXT:    vpshlw %xmm3, %xmm5, %xmm3
-; XOPAVX1-NEXT:    vpsubw %xmm2, %xmm4, %xmm2
-; XOPAVX1-NEXT:    vpshlw %xmm2, %xmm0, %xmm2
-; XOPAVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm2, %ymm2
-; XOPAVX1-NEXT:    vextractf128 $1, %ymm1, %xmm3
-; XOPAVX1-NEXT:    vpsubw %xmm3, %xmm4, %xmm3
-; XOPAVX1-NEXT:    vmovdqa {{.*#+}} xmm6 = [15,15,15,15,15,15,15,15]
-; XOPAVX1-NEXT:    vpand %xmm6, %xmm3, %xmm3
-; XOPAVX1-NEXT:    vpshlw %xmm3, %xmm5, %xmm3
-; XOPAVX1-NEXT:    vpsubw %xmm1, %xmm4, %xmm1
-; XOPAVX1-NEXT:    vpand %xmm6, %xmm1, %xmm1
-; XOPAVX1-NEXT:    vpshlw %xmm1, %xmm0, %xmm0
-; XOPAVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm0
-; XOPAVX1-NEXT:    vorps %ymm2, %ymm0, %ymm0
+; XOPAVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; XOPAVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; XOPAVX1-NEXT:    vpsubw %xmm2, %xmm3, %xmm2
+; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
+; XOPAVX1-NEXT:    vprotw %xmm2, %xmm4, %xmm2
+; XOPAVX1-NEXT:    vpsubw %xmm1, %xmm3, %xmm1
+; XOPAVX1-NEXT:    vprotw %xmm1, %xmm0, %xmm0
+; XOPAVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
 ; XOPAVX1-NEXT:    retq
 ;
 ; XOPAVX2-LABEL: var_funnnel_v16i16:
 ; XOPAVX2:       # %bb.0:
-; XOPAVX2-NEXT:    vmovdqa {{.*#+}} ymm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; XOPAVX2-NEXT:    vpand %ymm2, %ymm1, %ymm3
-; XOPAVX2-NEXT:    vextracti128 $1, %ymm3, %xmm4
-; XOPAVX2-NEXT:    vpxor %xmm5, %xmm5, %xmm5
-; XOPAVX2-NEXT:    vpsubw %xmm4, %xmm5, %xmm4
-; XOPAVX2-NEXT:    vextracti128 $1, %ymm0, %xmm6
-; XOPAVX2-NEXT:    vpshlw %xmm4, %xmm6, %xmm4
-; XOPAVX2-NEXT:    vpsubw %xmm3, %xmm5, %xmm3
-; XOPAVX2-NEXT:    vpshlw %xmm3, %xmm0, %xmm3
-; XOPAVX2-NEXT:    vinserti128 $1, %xmm4, %ymm3, %ymm3
-; XOPAVX2-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; XOPAVX2-NEXT:    vpsubw %ymm1, %ymm4, %ymm1
-; XOPAVX2-NEXT:    vpand %ymm2, %ymm1, %ymm1
+; XOPAVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; XOPAVX2-NEXT:    vpsubw %ymm1, %ymm2, %ymm1
 ; XOPAVX2-NEXT:    vextracti128 $1, %ymm1, %xmm2
-; XOPAVX2-NEXT:    vpshlw %xmm2, %xmm6, %xmm2
-; XOPAVX2-NEXT:    vpshlw %xmm1, %xmm0, %xmm0
+; XOPAVX2-NEXT:    vextracti128 $1, %ymm0, %xmm3
+; XOPAVX2-NEXT:    vprotw %xmm2, %xmm3, %xmm2
+; XOPAVX2-NEXT:    vprotw %xmm1, %xmm0, %xmm0
 ; XOPAVX2-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm0
-; XOPAVX2-NEXT:    vpor %ymm3, %ymm0, %ymm0
 ; XOPAVX2-NEXT:    retq
   %res = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %x, <16 x i16> %x, <16 x i16> %amt)
   ret <16 x i16> %res
@@ -426,168 +371,132 @@ define <32 x i8> @var_funnnel_v32i8(<32 x i8> %x, <32 x i8> %amt) nounwind {
 ; AVX1-LABEL: var_funnnel_v32i8:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
-; AVX1-NEXT:    vpsllw $4, %xmm2, %xmm3
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm8 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
-; AVX1-NEXT:    vpand %xmm8, %xmm3, %xmm3
+; AVX1-NEXT:    vpsrlw $4, %xmm2, %xmm3
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
+; AVX1-NEXT:    vpandn %xmm3, %xmm4, %xmm3
+; AVX1-NEXT:    vpsllw $4, %xmm2, %xmm5
+; AVX1-NEXT:    vpand %xmm4, %xmm5, %xmm5
+; AVX1-NEXT:    vpor %xmm3, %xmm5, %xmm3
 ; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm5
-; AVX1-NEXT:    vpxor %xmm9, %xmm9, %xmm9
-; AVX1-NEXT:    vpsubb %xmm5, %xmm9, %xmm5
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm7 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
-; AVX1-NEXT:    vpand %xmm7, %xmm5, %xmm5
-; AVX1-NEXT:    vpsllw $5, %xmm5, %xmm5
-; AVX1-NEXT:    vpblendvb %xmm5, %xmm3, %xmm2, %xmm3
-; AVX1-NEXT:    vpsllw $2, %xmm3, %xmm4
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm6 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252]
-; AVX1-NEXT:    vpand %xmm6, %xmm4, %xmm4
-; AVX1-NEXT:    vpaddb %xmm5, %xmm5, %xmm5
-; AVX1-NEXT:    vpblendvb %xmm5, %xmm4, %xmm3, %xmm3
-; AVX1-NEXT:    vpaddb %xmm3, %xmm3, %xmm4
-; AVX1-NEXT:    vpaddb %xmm5, %xmm5, %xmm5
-; AVX1-NEXT:    vpblendvb %xmm5, %xmm4, %xmm3, %xmm3
-; AVX1-NEXT:    vpsllw $4, %xmm0, %xmm4
-; AVX1-NEXT:    vpand %xmm8, %xmm4, %xmm4
-; AVX1-NEXT:    vpsubb %xmm1, %xmm9, %xmm5
-; AVX1-NEXT:    vpand %xmm7, %xmm5, %xmm5
+; AVX1-NEXT:    vpxor %xmm8, %xmm8, %xmm8
+; AVX1-NEXT:    vpsubb %xmm5, %xmm8, %xmm5
 ; AVX1-NEXT:    vpsllw $5, %xmm5, %xmm5
-; AVX1-NEXT:    vpblendvb %xmm5, %xmm4, %xmm0, %xmm4
-; AVX1-NEXT:    vpsllw $2, %xmm4, %xmm7
-; AVX1-NEXT:    vpand %xmm6, %xmm7, %xmm6
+; AVX1-NEXT:    vpblendvb %xmm5, %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vpsrlw $6, %xmm2, %xmm3
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm7 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252]
+; AVX1-NEXT:    vpandn %xmm3, %xmm7, %xmm3
+; AVX1-NEXT:    vpsllw $2, %xmm2, %xmm6
+; AVX1-NEXT:    vpand %xmm7, %xmm6, %xmm6
+; AVX1-NEXT:    vpor %xmm3, %xmm6, %xmm3
 ; AVX1-NEXT:    vpaddb %xmm5, %xmm5, %xmm5
-; AVX1-NEXT:    vpblendvb %xmm5, %xmm6, %xmm4, %xmm4
-; AVX1-NEXT:    vpaddb %xmm4, %xmm4, %xmm6
+; AVX1-NEXT:    vpblendvb %xmm5, %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vpsrlw $7, %xmm2, %xmm3
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm9 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
+; AVX1-NEXT:    vpand %xmm9, %xmm3, %xmm3
+; AVX1-NEXT:    vpaddb %xmm2, %xmm2, %xmm6
+; AVX1-NEXT:    vpor %xmm3, %xmm6, %xmm3
 ; AVX1-NEXT:    vpaddb %xmm5, %xmm5, %xmm5
-; AVX1-NEXT:    vpblendvb %xmm5, %xmm6, %xmm4, %xmm4
-; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm4, %ymm8
-; AVX1-NEXT:    vpsrlw $4, %xmm2, %xmm4
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm5 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; AVX1-NEXT:    vpand %xmm5, %xmm4, %xmm4
-; AVX1-NEXT:    vandps {{.*}}(%rip), %ymm1, %ymm1
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm6
-; AVX1-NEXT:    vpsllw $5, %xmm6, %xmm6
-; AVX1-NEXT:    vpblendvb %xmm6, %xmm4, %xmm2, %xmm2
-; AVX1-NEXT:    vpsrlw $2, %xmm2, %xmm4
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm7 = [63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63]
-; AVX1-NEXT:    vpand %xmm7, %xmm4, %xmm4
-; AVX1-NEXT:    vpaddb %xmm6, %xmm6, %xmm6
-; AVX1-NEXT:    vpblendvb %xmm6, %xmm4, %xmm2, %xmm2
-; AVX1-NEXT:    vpsrlw $1, %xmm2, %xmm4
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
-; AVX1-NEXT:    vpand %xmm3, %xmm4, %xmm4
-; AVX1-NEXT:    vpaddb %xmm6, %xmm6, %xmm6
-; AVX1-NEXT:    vpblendvb %xmm6, %xmm4, %xmm2, %xmm2
-; AVX1-NEXT:    vpsrlw $4, %xmm0, %xmm4
-; AVX1-NEXT:    vpand %xmm5, %xmm4, %xmm4
+; AVX1-NEXT:    vpblendvb %xmm5, %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vpsrlw $4, %xmm0, %xmm3
+; AVX1-NEXT:    vpandn %xmm3, %xmm4, %xmm3
+; AVX1-NEXT:    vpsllw $4, %xmm0, %xmm5
+; AVX1-NEXT:    vpand %xmm4, %xmm5, %xmm4
+; AVX1-NEXT:    vpor %xmm3, %xmm4, %xmm3
+; AVX1-NEXT:    vpsubb %xmm1, %xmm8, %xmm1
 ; AVX1-NEXT:    vpsllw $5, %xmm1, %xmm1
-; AVX1-NEXT:    vpblendvb %xmm1, %xmm4, %xmm0, %xmm0
-; AVX1-NEXT:    vpsrlw $2, %xmm0, %xmm4
+; AVX1-NEXT:    vpblendvb %xmm1, %xmm3, %xmm0, %xmm0
+; AVX1-NEXT:    vpsrlw $6, %xmm0, %xmm3
+; AVX1-NEXT:    vpandn %xmm3, %xmm7, %xmm3
+; AVX1-NEXT:    vpsllw $2, %xmm0, %xmm4
 ; AVX1-NEXT:    vpand %xmm7, %xmm4, %xmm4
+; AVX1-NEXT:    vpor %xmm3, %xmm4, %xmm3
 ; AVX1-NEXT:    vpaddb %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    vpblendvb %xmm1, %xmm4, %xmm0, %xmm0
-; AVX1-NEXT:    vpsrlw $1, %xmm0, %xmm4
-; AVX1-NEXT:    vpand %xmm3, %xmm4, %xmm3
+; AVX1-NEXT:    vpblendvb %xmm1, %xmm3, %xmm0, %xmm0
+; AVX1-NEXT:    vpsrlw $7, %xmm0, %xmm3
+; AVX1-NEXT:    vpand %xmm9, %xmm3, %xmm3
+; AVX1-NEXT:    vpaddb %xmm0, %xmm0, %xmm4
+; AVX1-NEXT:    vpor %xmm3, %xmm4, %xmm3
 ; AVX1-NEXT:    vpaddb %xmm1, %xmm1, %xmm1
 ; AVX1-NEXT:    vpblendvb %xmm1, %xmm3, %xmm0, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
-; AVX1-NEXT:    vorps %ymm0, %ymm8, %ymm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: var_funnnel_v32i8:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vmovdqa {{.*#+}} ymm2 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
-; AVX2-NEXT:    vpand %ymm2, %ymm1, %ymm3
-; AVX2-NEXT:    vpsllw $5, %ymm3, %ymm3
-; AVX2-NEXT:    vpaddb %ymm3, %ymm3, %ymm4
-; AVX2-NEXT:    vpsrlw $4, %ymm0, %ymm5
-; AVX2-NEXT:    vpand {{.*}}(%rip), %ymm5, %ymm5
-; AVX2-NEXT:    vpblendvb %ymm3, %ymm5, %ymm0, %ymm3
-; AVX2-NEXT:    vpsrlw $2, %ymm3, %ymm5
-; AVX2-NEXT:    vpand {{.*}}(%rip), %ymm5, %ymm5
-; AVX2-NEXT:    vpblendvb %ymm4, %ymm5, %ymm3, %ymm3
-; AVX2-NEXT:    vpsrlw $1, %ymm3, %ymm5
-; AVX2-NEXT:    vpand {{.*}}(%rip), %ymm5, %ymm5
-; AVX2-NEXT:    vpaddb %ymm4, %ymm4, %ymm4
-; AVX2-NEXT:    vpblendvb %ymm4, %ymm5, %ymm3, %ymm3
-; AVX2-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX2-NEXT:    vpsubb %ymm1, %ymm4, %ymm1
-; AVX2-NEXT:    vpand %ymm2, %ymm1, %ymm1
+; AVX2-NEXT:    vpsrlw $4, %ymm0, %ymm2
+; AVX2-NEXT:    vpand {{.*}}(%rip), %ymm2, %ymm2
+; AVX2-NEXT:    vpsllw $4, %ymm0, %ymm3
+; AVX2-NEXT:    vpand {{.*}}(%rip), %ymm3, %ymm3
+; AVX2-NEXT:    vpor %ymm2, %ymm3, %ymm2
+; AVX2-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; AVX2-NEXT:    vpsubb %ymm1, %ymm3, %ymm1
 ; AVX2-NEXT:    vpsllw $5, %ymm1, %ymm1
-; AVX2-NEXT:    vpaddb %ymm1, %ymm1, %ymm2
-; AVX2-NEXT:    vpsllw $4, %ymm0, %ymm4
-; AVX2-NEXT:    vpand {{.*}}(%rip), %ymm4, %ymm4
-; AVX2-NEXT:    vpblendvb %ymm1, %ymm4, %ymm0, %ymm0
-; AVX2-NEXT:    vpsllw $2, %ymm0, %ymm1
-; AVX2-NEXT:    vpand {{.*}}(%rip), %ymm1, %ymm1
-; AVX2-NEXT:    vpblendvb %ymm2, %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpaddb %ymm0, %ymm0, %ymm1
-; AVX2-NEXT:    vpaddb %ymm2, %ymm2, %ymm2
-; AVX2-NEXT:    vpblendvb %ymm2, %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpor %ymm3, %ymm0, %ymm0
+; AVX2-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vpsrlw $6, %ymm0, %ymm2
+; AVX2-NEXT:    vpand {{.*}}(%rip), %ymm2, %ymm2
+; AVX2-NEXT:    vpsllw $2, %ymm0, %ymm3
+; AVX2-NEXT:    vpand {{.*}}(%rip), %ymm3, %ymm3
+; AVX2-NEXT:    vpor %ymm2, %ymm3, %ymm2
+; AVX2-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
+; AVX2-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vpaddb %ymm0, %ymm0, %ymm2
+; AVX2-NEXT:    vpsrlw $7, %ymm0, %ymm3
+; AVX2-NEXT:    vpand {{.*}}(%rip), %ymm3, %ymm3
+; AVX2-NEXT:    vpor %ymm3, %ymm2, %ymm2
+; AVX2-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
+; AVX2-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
 ; AVX2-NEXT:    retq
 ;
 ; AVX512F-LABEL: var_funnnel_v32i8:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm2 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
-; AVX512F-NEXT:    vpand %ymm2, %ymm1, %ymm3
-; AVX512F-NEXT:    vpsllw $5, %ymm3, %ymm3
-; AVX512F-NEXT:    vpaddb %ymm3, %ymm3, %ymm4
-; AVX512F-NEXT:    vpsrlw $4, %ymm0, %ymm5
-; AVX512F-NEXT:    vpand {{.*}}(%rip), %ymm5, %ymm5
-; AVX512F-NEXT:    vpblendvb %ymm3, %ymm5, %ymm0, %ymm3
-; AVX512F-NEXT:    vpsrlw $2, %ymm3, %ymm5
-; AVX512F-NEXT:    vpand {{.*}}(%rip), %ymm5, %ymm5
-; AVX512F-NEXT:    vpblendvb %ymm4, %ymm5, %ymm3, %ymm3
-; AVX512F-NEXT:    vpsrlw $1, %ymm3, %ymm5
-; AVX512F-NEXT:    vpand {{.*}}(%rip), %ymm5, %ymm5
-; AVX512F-NEXT:    vpaddb %ymm4, %ymm4, %ymm4
-; AVX512F-NEXT:    vpblendvb %ymm4, %ymm5, %ymm3, %ymm3
-; AVX512F-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX512F-NEXT:    vpsubb %ymm1, %ymm4, %ymm1
-; AVX512F-NEXT:    vpand %ymm2, %ymm1, %ymm1
+; AVX512F-NEXT:    vpsrlw $4, %ymm0, %ymm2
+; AVX512F-NEXT:    vpand {{.*}}(%rip), %ymm2, %ymm2
+; AVX512F-NEXT:    vpsllw $4, %ymm0, %ymm3
+; AVX512F-NEXT:    vpand {{.*}}(%rip), %ymm3, %ymm3
+; AVX512F-NEXT:    vpor %ymm2, %ymm3, %ymm2
+; AVX512F-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; AVX512F-NEXT:    vpsubb %ymm1, %ymm3, %ymm1
 ; AVX512F-NEXT:    vpsllw $5, %ymm1, %ymm1
-; AVX512F-NEXT:    vpaddb %ymm1, %ymm1, %ymm2
-; AVX512F-NEXT:    vpsllw $4, %ymm0, %ymm4
-; AVX512F-NEXT:    vpand {{.*}}(%rip), %ymm4, %ymm4
-; AVX512F-NEXT:    vpblendvb %ymm1, %ymm4, %ymm0, %ymm0
-; AVX512F-NEXT:    vpsllw $2, %ymm0, %ymm1
-; AVX512F-NEXT:    vpand {{.*}}(%rip), %ymm1, %ymm1
-; AVX512F-NEXT:    vpblendvb %ymm2, %ymm1, %ymm0, %ymm0
-; AVX512F-NEXT:    vpaddb %ymm0, %ymm0, %ymm1
-; AVX512F-NEXT:    vpaddb %ymm2, %ymm2, %ymm2
-; AVX512F-NEXT:    vpblendvb %ymm2, %ymm1, %ymm0, %ymm0
-; AVX512F-NEXT:    vpor %ymm3, %ymm0, %ymm0
+; AVX512F-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
+; AVX512F-NEXT:    vpsrlw $6, %ymm0, %ymm2
+; AVX512F-NEXT:    vpand {{.*}}(%rip), %ymm2, %ymm2
+; AVX512F-NEXT:    vpsllw $2, %ymm0, %ymm3
+; AVX512F-NEXT:    vpand {{.*}}(%rip), %ymm3, %ymm3
+; AVX512F-NEXT:    vpor %ymm2, %ymm3, %ymm2
+; AVX512F-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
+; AVX512F-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
+; AVX512F-NEXT:    vpaddb %ymm0, %ymm0, %ymm2
+; AVX512F-NEXT:    vpsrlw $7, %ymm0, %ymm3
+; AVX512F-NEXT:    vpand {{.*}}(%rip), %ymm3, %ymm3
+; AVX512F-NEXT:    vpor %ymm3, %ymm2, %ymm2
+; AVX512F-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
+; AVX512F-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: var_funnnel_v32i8:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm2 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
-; AVX512VL-NEXT:    vpand %ymm2, %ymm1, %ymm3
-; AVX512VL-NEXT:    vpsllw $5, %ymm3, %ymm3
-; AVX512VL-NEXT:    vpaddb %ymm3, %ymm3, %ymm4
-; AVX512VL-NEXT:    vpsrlw $4, %ymm0, %ymm5
-; AVX512VL-NEXT:    vpand {{.*}}(%rip), %ymm5, %ymm5
-; AVX512VL-NEXT:    vpblendvb %ymm3, %ymm5, %ymm0, %ymm3
-; AVX512VL-NEXT:    vpsrlw $2, %ymm3, %ymm5
-; AVX512VL-NEXT:    vpand {{.*}}(%rip), %ymm5, %ymm5
-; AVX512VL-NEXT:    vpblendvb %ymm4, %ymm5, %ymm3, %ymm3
-; AVX512VL-NEXT:    vpsrlw $1, %ymm3, %ymm5
-; AVX512VL-NEXT:    vpand {{.*}}(%rip), %ymm5, %ymm5
-; AVX512VL-NEXT:    vpaddb %ymm4, %ymm4, %ymm4
-; AVX512VL-NEXT:    vpblendvb %ymm4, %ymm5, %ymm3, %ymm3
-; AVX512VL-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX512VL-NEXT:    vpsubb %ymm1, %ymm4, %ymm1
-; AVX512VL-NEXT:    vpand %ymm2, %ymm1, %ymm1
+; AVX512VL-NEXT:    vpsrlw $4, %ymm0, %ymm2
+; AVX512VL-NEXT:    vpand {{.*}}(%rip), %ymm2, %ymm2
+; AVX512VL-NEXT:    vpsllw $4, %ymm0, %ymm3
+; AVX512VL-NEXT:    vpand {{.*}}(%rip), %ymm3, %ymm3
+; AVX512VL-NEXT:    vpor %ymm2, %ymm3, %ymm2
+; AVX512VL-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; AVX512VL-NEXT:    vpsubb %ymm1, %ymm3, %ymm1
 ; AVX512VL-NEXT:    vpsllw $5, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpaddb %ymm1, %ymm1, %ymm2
-; AVX512VL-NEXT:    vpsllw $4, %ymm0, %ymm4
-; AVX512VL-NEXT:    vpand {{.*}}(%rip), %ymm4, %ymm4
-; AVX512VL-NEXT:    vpblendvb %ymm1, %ymm4, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpsllw $2, %ymm0, %ymm1
-; AVX512VL-NEXT:    vpand {{.*}}(%rip), %ymm1, %ymm1
-; AVX512VL-NEXT:    vpblendvb %ymm2, %ymm1, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpaddb %ymm0, %ymm0, %ymm1
-; AVX512VL-NEXT:    vpaddb %ymm2, %ymm2, %ymm2
-; AVX512VL-NEXT:    vpblendvb %ymm2, %ymm1, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpor %ymm3, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpsrlw $6, %ymm0, %ymm2
+; AVX512VL-NEXT:    vpand {{.*}}(%rip), %ymm2, %ymm2
+; AVX512VL-NEXT:    vpsllw $2, %ymm0, %ymm3
+; AVX512VL-NEXT:    vpand {{.*}}(%rip), %ymm3, %ymm3
+; AVX512VL-NEXT:    vpor %ymm2, %ymm3, %ymm2
+; AVX512VL-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
+; AVX512VL-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpaddb %ymm0, %ymm0, %ymm2
+; AVX512VL-NEXT:    vpsrlw $7, %ymm0, %ymm3
+; AVX512VL-NEXT:    vpand {{.*}}(%rip), %ymm3, %ymm3
+; AVX512VL-NEXT:    vpor %ymm3, %ymm2, %ymm2
+; AVX512VL-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
+; AVX512VL-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: var_funnnel_v32i8:
@@ -624,47 +533,25 @@ define <32 x i8> @var_funnnel_v32i8(<32 x i8> %x, <32 x i8> %amt) nounwind {
 ;
 ; XOPAVX1-LABEL: var_funnnel_v32i8:
 ; XOPAVX1:       # %bb.0:
-; XOPAVX1-NEXT:    vandps {{.*}}(%rip), %ymm1, %ymm2
-; XOPAVX1-NEXT:    vextractf128 $1, %ymm2, %xmm3
-; XOPAVX1-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; XOPAVX1-NEXT:    vpsubb %xmm3, %xmm4, %xmm3
-; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm5
-; XOPAVX1-NEXT:    vpshlb %xmm3, %xmm5, %xmm3
-; XOPAVX1-NEXT:    vpsubb %xmm2, %xmm4, %xmm2
-; XOPAVX1-NEXT:    vpshlb %xmm2, %xmm0, %xmm2
-; XOPAVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm2, %ymm2
-; XOPAVX1-NEXT:    vextractf128 $1, %ymm1, %xmm3
-; XOPAVX1-NEXT:    vpsubb %xmm3, %xmm4, %xmm3
-; XOPAVX1-NEXT:    vmovdqa {{.*#+}} xmm6 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
-; XOPAVX1-NEXT:    vpand %xmm6, %xmm3, %xmm3
-; XOPAVX1-NEXT:    vpshlb %xmm3, %xmm5, %xmm3
-; XOPAVX1-NEXT:    vpsubb %xmm1, %xmm4, %xmm1
-; XOPAVX1-NEXT:    vpand %xmm6, %xmm1, %xmm1
-; XOPAVX1-NEXT:    vpshlb %xmm1, %xmm0, %xmm0
-; XOPAVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm0
-; XOPAVX1-NEXT:    vorps %ymm2, %ymm0, %ymm0
+; XOPAVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
+; XOPAVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; XOPAVX1-NEXT:    vpsubb %xmm2, %xmm3, %xmm2
+; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
+; XOPAVX1-NEXT:    vprotb %xmm2, %xmm4, %xmm2
+; XOPAVX1-NEXT:    vpsubb %xmm1, %xmm3, %xmm1
+; XOPAVX1-NEXT:    vprotb %xmm1, %xmm0, %xmm0
+; XOPAVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
 ; XOPAVX1-NEXT:    retq
 ;
 ; XOPAVX2-LABEL: var_funnnel_v32i8:
 ; XOPAVX2:       # %bb.0:
-; XOPAVX2-NEXT:    vmovdqa {{.*#+}} ymm2 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
-; XOPAVX2-NEXT:    vpand %ymm2, %ymm1, %ymm3
-; XOPAVX2-NEXT:    vextracti128 $1, %ymm3, %xmm4
-; XOPAVX2-NEXT:    vpxor %xmm5, %xmm5, %xmm5
-; XOPAVX2-NEXT:    vpsubb %xmm4, %xmm5, %xmm4
-; XOPAVX2-NEXT:    vextracti128 $1, %ymm0, %xmm6
-; XOPAVX2-NEXT:    vpshlb %xmm4, %xmm6, %xmm4
-; XOPAVX2-NEXT:    vpsubb %xmm3, %xmm5, %xmm3
-; XOPAVX2-NEXT:    vpshlb %xmm3, %xmm0, %xmm3
-; XOPAVX2-NEXT:    vinserti128 $1, %xmm4, %ymm3, %ymm3
-; XOPAVX2-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; XOPAVX2-NEXT:    vpsubb %ymm1, %ymm4, %ymm1
-; XOPAVX2-NEXT:    vpand %ymm2, %ymm1, %ymm1
+; XOPAVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; XOPAVX2-NEXT:    vpsubb %ymm1, %ymm2, %ymm1
 ; XOPAVX2-NEXT:    vextracti128 $1, %ymm1, %xmm2
-; XOPAVX2-NEXT:    vpshlb %xmm2, %xmm6, %xmm2
-; XOPAVX2-NEXT:    vpshlb %xmm1, %xmm0, %xmm0
+; XOPAVX2-NEXT:    vextracti128 $1, %ymm0, %xmm3
+; XOPAVX2-NEXT:    vprotb %xmm2, %xmm3, %xmm2
+; XOPAVX2-NEXT:    vprotb %xmm1, %xmm0, %xmm0
 ; XOPAVX2-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm0
-; XOPAVX2-NEXT:    vpor %ymm3, %ymm0, %ymm0
 ; XOPAVX2-NEXT:    retq
   %res = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %x, <32 x i8> %x, <32 x i8> %amt)
   ret <32 x i8> %res
@@ -711,48 +598,55 @@ define <4 x i64> @splatvar_funnnel_v4i64(<4 x i64> %x, <4 x i64> %amt) nounwind
 ; AVX2-NEXT:    vpor %ymm3, %ymm0, %ymm0
 ; AVX2-NEXT:    retq
 ;
-; AVX512-LABEL: splatvar_funnnel_v4i64:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpbroadcastq %xmm1, %ymm1
-; AVX512-NEXT:    vmovdqa {{.*#+}} xmm2 = [63,63]
-; AVX512-NEXT:    vpand %xmm2, %xmm1, %xmm3
-; AVX512-NEXT:    vpsrlq %xmm3, %ymm0, %ymm3
-; AVX512-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX512-NEXT:    vpsubq %xmm1, %xmm4, %xmm1
-; AVX512-NEXT:    vpand %xmm2, %xmm1, %xmm1
-; AVX512-NEXT:    vpsllq %xmm1, %ymm0, %ymm0
-; AVX512-NEXT:    vpor %ymm3, %ymm0, %ymm0
-; AVX512-NEXT:    retq
+; AVX512F-LABEL: splatvar_funnnel_v4i64:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512F-NEXT:    vpbroadcastq %xmm1, %ymm1
+; AVX512F-NEXT:    vprorvq %zmm1, %zmm0, %zmm0
+; AVX512F-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: splatvar_funnnel_v4i64:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpbroadcastq %xmm1, %ymm1
+; AVX512VL-NEXT:    vprorvq %ymm1, %ymm0, %ymm0
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: splatvar_funnnel_v4i64:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512BW-NEXT:    vpbroadcastq %xmm1, %ymm1
+; AVX512BW-NEXT:    vprorvq %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
+; AVX512BW-NEXT:    retq
+;
+; AVX512VLBW-LABEL: splatvar_funnnel_v4i64:
+; AVX512VLBW:       # %bb.0:
+; AVX512VLBW-NEXT:    vpbroadcastq %xmm1, %ymm1
+; AVX512VLBW-NEXT:    vprorvq %ymm1, %ymm0, %ymm0
+; AVX512VLBW-NEXT:    retq
 ;
 ; XOPAVX1-LABEL: splatvar_funnnel_v4i64:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vmovddup {{.*#+}} xmm1 = xmm1[0,0]
-; XOPAVX1-NEXT:    vmovapd {{.*#+}} xmm2 = [63,63]
-; XOPAVX1-NEXT:    vandpd %xmm2, %xmm1, %xmm3
-; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
-; XOPAVX1-NEXT:    vpsrlq %xmm3, %xmm4, %xmm5
-; XOPAVX1-NEXT:    vpsrlq %xmm3, %xmm0, %xmm3
-; XOPAVX1-NEXT:    vinsertf128 $1, %xmm5, %ymm3, %ymm3
-; XOPAVX1-NEXT:    vpxor %xmm5, %xmm5, %xmm5
-; XOPAVX1-NEXT:    vpsubq %xmm1, %xmm5, %xmm1
-; XOPAVX1-NEXT:    vpand %xmm2, %xmm1, %xmm1
-; XOPAVX1-NEXT:    vpshlq %xmm1, %xmm4, %xmm2
-; XOPAVX1-NEXT:    vpshlq %xmm1, %xmm0, %xmm0
+; XOPAVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; XOPAVX1-NEXT:    vpsubq %xmm1, %xmm2, %xmm1
+; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; XOPAVX1-NEXT:    vprotq %xmm1, %xmm2, %xmm2
+; XOPAVX1-NEXT:    vprotq %xmm1, %xmm0, %xmm0
 ; XOPAVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
-; XOPAVX1-NEXT:    vorps %ymm3, %ymm0, %ymm0
 ; XOPAVX1-NEXT:    retq
 ;
 ; XOPAVX2-LABEL: splatvar_funnnel_v4i64:
 ; XOPAVX2:       # %bb.0:
 ; XOPAVX2-NEXT:    vpbroadcastq %xmm1, %ymm1
-; XOPAVX2-NEXT:    vmovdqa {{.*#+}} xmm2 = [63,63]
-; XOPAVX2-NEXT:    vpand %xmm2, %xmm1, %xmm3
-; XOPAVX2-NEXT:    vpsrlq %xmm3, %ymm0, %ymm3
-; XOPAVX2-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; XOPAVX2-NEXT:    vpsubq %xmm1, %xmm4, %xmm1
-; XOPAVX2-NEXT:    vpand %xmm2, %xmm1, %xmm1
-; XOPAVX2-NEXT:    vpsllq %xmm1, %ymm0, %ymm0
-; XOPAVX2-NEXT:    vpor %ymm3, %ymm0, %ymm0
+; XOPAVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; XOPAVX2-NEXT:    vpsubq %ymm1, %ymm2, %ymm1
+; XOPAVX2-NEXT:    vextracti128 $1, %ymm1, %xmm2
+; XOPAVX2-NEXT:    vextracti128 $1, %ymm0, %xmm3
+; XOPAVX2-NEXT:    vprotq %xmm2, %xmm3, %xmm2
+; XOPAVX2-NEXT:    vprotq %xmm1, %xmm0, %xmm0
+; XOPAVX2-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm0
 ; XOPAVX2-NEXT:    retq
   %splat = shufflevector <4 x i64> %amt, <4 x i64> undef, <4 x i32> zeroinitializer
   %res = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %x, <4 x i64> %x, <4 x i64> %splat)
@@ -763,87 +657,88 @@ define <8 x i32> @splatvar_funnnel_v8i32(<8 x i32> %x, <8 x i32> %amt) nounwind
 ; AVX1-LABEL: splatvar_funnnel_v8i32:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [31,31,31,31]
-; AVX1-NEXT:    vpand %xmm2, %xmm1, %xmm3
-; AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm3 = xmm3[0],zero,xmm3[1],zero
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
-; AVX1-NEXT:    vpsrld %xmm3, %xmm4, %xmm5
-; AVX1-NEXT:    vpsrld %xmm3, %xmm0, %xmm3
-; AVX1-NEXT:    vinsertf128 $1, %xmm5, %ymm3, %ymm3
-; AVX1-NEXT:    vpxor %xmm5, %xmm5, %xmm5
+; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX1-NEXT:    vpsubd %xmm1, %xmm2, %xmm1
+; AVX1-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm2 = xmm1[0],zero,xmm1[1],zero
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
+; AVX1-NEXT:    vpslld %xmm2, %xmm3, %xmm4
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm5 = [32,32,32,32]
 ; AVX1-NEXT:    vpsubd %xmm1, %xmm5, %xmm1
-; AVX1-NEXT:    vpand %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpslld $23, %xmm1, %xmm1
-; AVX1-NEXT:    vpaddd {{.*}}(%rip), %xmm1, %xmm1
-; AVX1-NEXT:    vcvttps2dq %xmm1, %xmm1
-; AVX1-NEXT:    vpmulld %xmm1, %xmm4, %xmm2
-; AVX1-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
-; AVX1-NEXT:    vorps %ymm3, %ymm0, %ymm0
+; AVX1-NEXT:    vpmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero
+; AVX1-NEXT:    vpsrld %xmm1, %xmm3, %xmm3
+; AVX1-NEXT:    vpor %xmm3, %xmm4, %xmm3
+; AVX1-NEXT:    vpslld %xmm2, %xmm0, %xmm2
+; AVX1-NEXT:    vpsrld %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpor %xmm0, %xmm2, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: splatvar_funnnel_v8i32:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpbroadcastd %xmm1, %ymm1
+; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX2-NEXT:    vpsubd %xmm1, %xmm2, %xmm1
 ; AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [31,31,31,31]
-; AVX2-NEXT:    vpand %xmm2, %xmm1, %xmm3
-; AVX2-NEXT:    vpmovzxdq {{.*#+}} xmm3 = xmm3[0],zero,xmm3[1],zero
-; AVX2-NEXT:    vpsrld %xmm3, %ymm0, %ymm3
-; AVX2-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX2-NEXT:    vpsubd %xmm1, %xmm4, %xmm1
 ; AVX2-NEXT:    vpand %xmm2, %xmm1, %xmm1
+; AVX2-NEXT:    vpmovzxdq {{.*#+}} xmm2 = xmm1[0],zero,xmm1[1],zero
+; AVX2-NEXT:    vpslld %xmm2, %ymm0, %ymm2
+; AVX2-NEXT:    vpbroadcastd {{.*#+}} xmm3 = [32,32,32,32]
+; AVX2-NEXT:    vpsubd %xmm1, %xmm3, %xmm1
 ; AVX2-NEXT:    vpmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero
-; AVX2-NEXT:    vpslld %xmm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpor %ymm3, %ymm0, %ymm0
+; AVX2-NEXT:    vpsrld %xmm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpor %ymm0, %ymm2, %ymm0
 ; AVX2-NEXT:    retq
 ;
-; AVX512-LABEL: splatvar_funnnel_v8i32:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpbroadcastd %xmm1, %ymm1
-; AVX512-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [31,31,31,31]
-; AVX512-NEXT:    vpand %xmm2, %xmm1, %xmm3
-; AVX512-NEXT:    vpmovzxdq {{.*#+}} xmm3 = xmm3[0],zero,xmm3[1],zero
-; AVX512-NEXT:    vpsrld %xmm3, %ymm0, %ymm3
-; AVX512-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX512-NEXT:    vpsubd %xmm1, %xmm4, %xmm1
-; AVX512-NEXT:    vpand %xmm2, %xmm1, %xmm1
-; AVX512-NEXT:    vpmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero
-; AVX512-NEXT:    vpslld %xmm1, %ymm0, %ymm0
-; AVX512-NEXT:    vpor %ymm3, %ymm0, %ymm0
-; AVX512-NEXT:    retq
+; AVX512F-LABEL: splatvar_funnnel_v8i32:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512F-NEXT:    vpbroadcastd %xmm1, %ymm1
+; AVX512F-NEXT:    vprorvd %zmm1, %zmm0, %zmm0
+; AVX512F-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: splatvar_funnnel_v8i32:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpbroadcastd %xmm1, %ymm1
+; AVX512VL-NEXT:    vprorvd %ymm1, %ymm0, %ymm0
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: splatvar_funnnel_v8i32:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512BW-NEXT:    vpbroadcastd %xmm1, %ymm1
+; AVX512BW-NEXT:    vprorvd %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
+; AVX512BW-NEXT:    retq
+;
+; AVX512VLBW-LABEL: splatvar_funnnel_v8i32:
+; AVX512VLBW:       # %bb.0:
+; AVX512VLBW-NEXT:    vpbroadcastd %xmm1, %ymm1
+; AVX512VLBW-NEXT:    vprorvd %ymm1, %ymm0, %ymm0
+; AVX512VLBW-NEXT:    retq
 ;
 ; XOPAVX1-LABEL: splatvar_funnnel_v8i32:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
-; XOPAVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [31,31,31,31]
-; XOPAVX1-NEXT:    vpand %xmm2, %xmm1, %xmm3
-; XOPAVX1-NEXT:    vpmovzxdq {{.*#+}} xmm3 = xmm3[0],zero,xmm3[1],zero
-; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
-; XOPAVX1-NEXT:    vpsrld %xmm3, %xmm4, %xmm5
-; XOPAVX1-NEXT:    vpsrld %xmm3, %xmm0, %xmm3
-; XOPAVX1-NEXT:    vinsertf128 $1, %xmm5, %ymm3, %ymm3
-; XOPAVX1-NEXT:    vpxor %xmm5, %xmm5, %xmm5
-; XOPAVX1-NEXT:    vpsubd %xmm1, %xmm5, %xmm1
-; XOPAVX1-NEXT:    vpand %xmm2, %xmm1, %xmm1
-; XOPAVX1-NEXT:    vpshld %xmm1, %xmm4, %xmm2
-; XOPAVX1-NEXT:    vpshld %xmm1, %xmm0, %xmm0
+; XOPAVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; XOPAVX1-NEXT:    vpsubd %xmm1, %xmm2, %xmm1
+; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; XOPAVX1-NEXT:    vprotd %xmm1, %xmm2, %xmm2
+; XOPAVX1-NEXT:    vprotd %xmm1, %xmm0, %xmm0
 ; XOPAVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
-; XOPAVX1-NEXT:    vorps %ymm3, %ymm0, %ymm0
 ; XOPAVX1-NEXT:    retq
 ;
 ; XOPAVX2-LABEL: splatvar_funnnel_v8i32:
 ; XOPAVX2:       # %bb.0:
 ; XOPAVX2-NEXT:    vpbroadcastd %xmm1, %ymm1
-; XOPAVX2-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [31,31,31,31]
-; XOPAVX2-NEXT:    vpand %xmm2, %xmm1, %xmm3
-; XOPAVX2-NEXT:    vpmovzxdq {{.*#+}} xmm3 = xmm3[0],zero,xmm3[1],zero
-; XOPAVX2-NEXT:    vpsrld %xmm3, %ymm0, %ymm3
-; XOPAVX2-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; XOPAVX2-NEXT:    vpsubd %xmm1, %xmm4, %xmm1
-; XOPAVX2-NEXT:    vpand %xmm2, %xmm1, %xmm1
-; XOPAVX2-NEXT:    vpmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero
-; XOPAVX2-NEXT:    vpslld %xmm1, %ymm0, %ymm0
-; XOPAVX2-NEXT:    vpor %ymm3, %ymm0, %ymm0
+; XOPAVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; XOPAVX2-NEXT:    vpsubd %ymm1, %ymm2, %ymm1
+; XOPAVX2-NEXT:    vextracti128 $1, %ymm1, %xmm2
+; XOPAVX2-NEXT:    vextracti128 $1, %ymm0, %xmm3
+; XOPAVX2-NEXT:    vprotd %xmm2, %xmm3, %xmm2
+; XOPAVX2-NEXT:    vprotd %xmm1, %xmm0, %xmm0
+; XOPAVX2-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm0
 ; XOPAVX2-NEXT:    retq
   %splat = shufflevector <8 x i32> %amt, <8 x i32> undef, <8 x i32> zeroinitializer
   %res = call <8 x i32> @llvm.fshr.v8i32(<8 x i32> %x, <8 x i32> %x, <8 x i32> %splat)
@@ -856,94 +751,74 @@ define <16 x i16> @splatvar_funnnel_v16i16(<16 x i16> %x, <16 x i16> %amt) nounw
 ; AVX1-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm1[0,0,2,3,4,5,6,7]
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
 ; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpsubw %xmm1, %xmm2, %xmm3
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = [15,15,15,15,15,15,15,15]
-; AVX1-NEXT:    vpand %xmm4, %xmm3, %xmm3
-; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
-; AVX1-NEXT:    vpslld $23, %xmm2, %xmm2
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm5 = [1065353216,1065353216,1065353216,1065353216]
-; AVX1-NEXT:    vpaddd %xmm5, %xmm2, %xmm2
-; AVX1-NEXT:    vcvttps2dq %xmm2, %xmm2
-; AVX1-NEXT:    vpmovzxwd {{.*#+}} xmm3 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero
-; AVX1-NEXT:    vpslld $23, %xmm3, %xmm3
-; AVX1-NEXT:    vpaddd %xmm5, %xmm3, %xmm3
-; AVX1-NEXT:    vcvttps2dq %xmm3, %xmm3
-; AVX1-NEXT:    vpackusdw %xmm2, %xmm3, %xmm2
+; AVX1-NEXT:    vpsubw %xmm1, %xmm2, %xmm1
+; AVX1-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX1-NEXT:    vpmovzxwq {{.*#+}} xmm2 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
-; AVX1-NEXT:    vpmullw %xmm2, %xmm3, %xmm5
-; AVX1-NEXT:    vpmullw %xmm2, %xmm0, %xmm2
-; AVX1-NEXT:    vinsertf128 $1, %xmm5, %ymm2, %ymm2
-; AVX1-NEXT:    vpand %xmm4, %xmm1, %xmm1
+; AVX1-NEXT:    vpsllw %xmm2, %xmm3, %xmm4
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm5 = [16,16,16,16,16,16,16,16]
+; AVX1-NEXT:    vpsubw %xmm1, %xmm5, %xmm1
 ; AVX1-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
 ; AVX1-NEXT:    vpsrlw %xmm1, %xmm3, %xmm3
+; AVX1-NEXT:    vpor %xmm3, %xmm4, %xmm3
+; AVX1-NEXT:    vpsllw %xmm2, %xmm0, %xmm2
 ; AVX1-NEXT:    vpsrlw %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpor %xmm0, %xmm2, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm0
-; AVX1-NEXT:    vorps %ymm0, %ymm2, %ymm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: splatvar_funnnel_v16i16:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpbroadcastw %xmm1, %ymm1
-; AVX2-NEXT:    vmovdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15]
-; AVX2-NEXT:    vpand %xmm2, %xmm1, %xmm3
-; AVX2-NEXT:    vpmovzxwq {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero
-; AVX2-NEXT:    vpsrlw %xmm3, %ymm0, %ymm3
-; AVX2-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX2-NEXT:    vpsubw %xmm1, %xmm4, %xmm1
-; AVX2-NEXT:    vpand %xmm2, %xmm1, %xmm1
+; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX2-NEXT:    vpsubw %xmm1, %xmm2, %xmm1
+; AVX2-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX2-NEXT:    vpmovzxwq {{.*#+}} xmm2 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; AVX2-NEXT:    vpsllw %xmm2, %ymm0, %ymm2
+; AVX2-NEXT:    vmovdqa {{.*#+}} xmm3 = [16,16,16,16,16,16,16,16]
+; AVX2-NEXT:    vpsubw %xmm1, %xmm3, %xmm1
 ; AVX2-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
-; AVX2-NEXT:    vpsllw %xmm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpor %ymm3, %ymm0, %ymm0
+; AVX2-NEXT:    vpsrlw %xmm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpor %ymm0, %ymm2, %ymm0
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: splatvar_funnnel_v16i16:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpbroadcastw %xmm1, %ymm1
-; AVX512-NEXT:    vmovdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15]
-; AVX512-NEXT:    vpand %xmm2, %xmm1, %xmm3
-; AVX512-NEXT:    vpmovzxwq {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero
-; AVX512-NEXT:    vpsrlw %xmm3, %ymm0, %ymm3
-; AVX512-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX512-NEXT:    vpsubw %xmm1, %xmm4, %xmm1
-; AVX512-NEXT:    vpand %xmm2, %xmm1, %xmm1
+; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX512-NEXT:    vpsubw %xmm1, %xmm2, %xmm1
+; AVX512-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX512-NEXT:    vpmovzxwq {{.*#+}} xmm2 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; AVX512-NEXT:    vpsllw %xmm2, %ymm0, %ymm2
+; AVX512-NEXT:    vmovdqa {{.*#+}} xmm3 = [16,16,16,16,16,16,16,16]
+; AVX512-NEXT:    vpsubw %xmm1, %xmm3, %xmm1
 ; AVX512-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
-; AVX512-NEXT:    vpsllw %xmm1, %ymm0, %ymm0
-; AVX512-NEXT:    vpor %ymm3, %ymm0, %ymm0
+; AVX512-NEXT:    vpsrlw %xmm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpor %ymm0, %ymm2, %ymm0
 ; AVX512-NEXT:    retq
 ;
 ; XOPAVX1-LABEL: splatvar_funnnel_v16i16:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm1[0,0,2,3,4,5,6,7]
 ; XOPAVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
-; XOPAVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15]
-; XOPAVX1-NEXT:    vpand %xmm2, %xmm1, %xmm3
-; XOPAVX1-NEXT:    vpmovzxwq {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero
-; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
-; XOPAVX1-NEXT:    vpsrlw %xmm3, %xmm4, %xmm5
-; XOPAVX1-NEXT:    vpsrlw %xmm3, %xmm0, %xmm3
-; XOPAVX1-NEXT:    vinsertf128 $1, %xmm5, %ymm3, %ymm3
-; XOPAVX1-NEXT:    vpxor %xmm5, %xmm5, %xmm5
-; XOPAVX1-NEXT:    vpsubw %xmm1, %xmm5, %xmm1
-; XOPAVX1-NEXT:    vpand %xmm2, %xmm1, %xmm1
-; XOPAVX1-NEXT:    vpshlw %xmm1, %xmm4, %xmm2
-; XOPAVX1-NEXT:    vpshlw %xmm1, %xmm0, %xmm0
+; XOPAVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; XOPAVX1-NEXT:    vpsubw %xmm1, %xmm2, %xmm1
+; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; XOPAVX1-NEXT:    vprotw %xmm1, %xmm2, %xmm2
+; XOPAVX1-NEXT:    vprotw %xmm1, %xmm0, %xmm0
 ; XOPAVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
-; XOPAVX1-NEXT:    vorps %ymm3, %ymm0, %ymm0
 ; XOPAVX1-NEXT:    retq
 ;
 ; XOPAVX2-LABEL: splatvar_funnnel_v16i16:
 ; XOPAVX2:       # %bb.0:
 ; XOPAVX2-NEXT:    vpbroadcastw %xmm1, %ymm1
-; XOPAVX2-NEXT:    vmovdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15]
-; XOPAVX2-NEXT:    vpand %xmm2, %xmm1, %xmm3
-; XOPAVX2-NEXT:    vpmovzxwq {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero
-; XOPAVX2-NEXT:    vpsrlw %xmm3, %ymm0, %ymm3
-; XOPAVX2-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; XOPAVX2-NEXT:    vpsubw %xmm1, %xmm4, %xmm1
-; XOPAVX2-NEXT:    vpand %xmm2, %xmm1, %xmm1
-; XOPAVX2-NEXT:    vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
-; XOPAVX2-NEXT:    vpsllw %xmm1, %ymm0, %ymm0
-; XOPAVX2-NEXT:    vpor %ymm3, %ymm0, %ymm0
+; XOPAVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; XOPAVX2-NEXT:    vpsubw %ymm1, %ymm2, %ymm1
+; XOPAVX2-NEXT:    vextracti128 $1, %ymm1, %xmm2
+; XOPAVX2-NEXT:    vextracti128 $1, %ymm0, %xmm3
+; XOPAVX2-NEXT:    vprotw %xmm2, %xmm3, %xmm2
+; XOPAVX2-NEXT:    vprotw %xmm1, %xmm0, %xmm0
+; XOPAVX2-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm0
 ; XOPAVX2-NEXT:    retq
   %splat = shufflevector <16 x i16> %amt, <16 x i16> undef, <16 x i32> zeroinitializer
   %res = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %x, <16 x i16> %x, <16 x i16> %splat)
@@ -955,112 +830,98 @@ define <32 x i8> @splatvar_funnnel_v32i8(<32 x i8> %x, <32 x i8> %amt) nounwind
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
 ; AVX1-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
-; AVX1-NEXT:    vpsllw $4, %xmm3, %xmm4
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm9 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
-; AVX1-NEXT:    vpand %xmm9, %xmm4, %xmm4
-; AVX1-NEXT:    vpsubb %xmm1, %xmm2, %xmm2
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm8 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
-; AVX1-NEXT:    vpand %xmm8, %xmm2, %xmm2
-; AVX1-NEXT:    vpsllw $5, %xmm2, %xmm2
-; AVX1-NEXT:    vpblendvb %xmm2, %xmm4, %xmm3, %xmm4
-; AVX1-NEXT:    vpsllw $2, %xmm4, %xmm7
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm10 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252]
-; AVX1-NEXT:    vpand %xmm10, %xmm7, %xmm7
-; AVX1-NEXT:    vpaddb %xmm2, %xmm2, %xmm5
-; AVX1-NEXT:    vpblendvb %xmm5, %xmm7, %xmm4, %xmm4
-; AVX1-NEXT:    vpaddb %xmm4, %xmm4, %xmm7
-; AVX1-NEXT:    vpaddb %xmm5, %xmm5, %xmm6
-; AVX1-NEXT:    vpblendvb %xmm6, %xmm7, %xmm4, %xmm4
-; AVX1-NEXT:    vpsllw $4, %xmm0, %xmm7
-; AVX1-NEXT:    vpand %xmm9, %xmm7, %xmm7
-; AVX1-NEXT:    vpblendvb %xmm2, %xmm7, %xmm0, %xmm2
-; AVX1-NEXT:    vpsllw $2, %xmm2, %xmm7
-; AVX1-NEXT:    vpand %xmm10, %xmm7, %xmm7
-; AVX1-NEXT:    vpblendvb %xmm5, %xmm7, %xmm2, %xmm2
-; AVX1-NEXT:    vpaddb %xmm2, %xmm2, %xmm5
-; AVX1-NEXT:    vpblendvb %xmm6, %xmm5, %xmm2, %xmm2
-; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm2, %ymm2
-; AVX1-NEXT:    vpand %xmm8, %xmm1, %xmm1
+; AVX1-NEXT:    vpsubb %xmm1, %xmm2, %xmm1
+; AVX1-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX1-NEXT:    vpmovzxbq {{.*#+}} xmm3 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
+; AVX1-NEXT:    vpsllw %xmm3, %xmm4, %xmm5
+; AVX1-NEXT:    vpcmpeqd %xmm6, %xmm6, %xmm6
+; AVX1-NEXT:    vpsllw %xmm3, %xmm6, %xmm7
+; AVX1-NEXT:    vpshufb %xmm2, %xmm7, %xmm2
+; AVX1-NEXT:    vpand %xmm2, %xmm5, %xmm5
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm7 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; AVX1-NEXT:    vpsubb %xmm1, %xmm7, %xmm1
 ; AVX1-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
-; AVX1-NEXT:    vpsrlw %xmm1, %xmm3, %xmm3
-; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
 ; AVX1-NEXT:    vpsrlw %xmm1, %xmm4, %xmm4
-; AVX1-NEXT:    vpshufb {{.*#+}} xmm4 = xmm4[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
-; AVX1-NEXT:    vpand %xmm4, %xmm3, %xmm3
+; AVX1-NEXT:    vpsrlw %xmm1, %xmm6, %xmm6
+; AVX1-NEXT:    vpshufb {{.*#+}} xmm6 = xmm6[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
+; AVX1-NEXT:    vpand %xmm6, %xmm4, %xmm4
+; AVX1-NEXT:    vpor %xmm4, %xmm5, %xmm4
+; AVX1-NEXT:    vpsllw %xmm3, %xmm0, %xmm3
+; AVX1-NEXT:    vpand %xmm2, %xmm3, %xmm2
 ; AVX1-NEXT:    vpsrlw %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpand %xmm4, %xmm0, %xmm0
-; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm0
-; AVX1-NEXT:    vorps %ymm0, %ymm2, %ymm0
+; AVX1-NEXT:    vpand %xmm6, %xmm0, %xmm0
+; AVX1-NEXT:    vpor %xmm0, %xmm2, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm0, %ymm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: splatvar_funnnel_v32i8:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpbroadcastb %xmm1, %ymm1
-; AVX2-NEXT:    vmovdqa {{.*#+}} xmm2 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
-; AVX2-NEXT:    vpand %xmm2, %xmm1, %xmm3
-; AVX2-NEXT:    vpmovzxbq {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,zero,zero,zero,zero,xmm3[1],zero,zero,zero,zero,zero,zero,zero
-; AVX2-NEXT:    vpsrlw %xmm3, %ymm0, %ymm4
-; AVX2-NEXT:    vpcmpeqd %ymm5, %ymm5, %ymm5
-; AVX2-NEXT:    vpsrlw %xmm3, %ymm5, %ymm3
-; AVX2-NEXT:    vpsrlw $8, %ymm3, %ymm3
-; AVX2-NEXT:    vpbroadcastb %xmm3, %ymm3
-; AVX2-NEXT:    vpand %ymm3, %ymm4, %ymm3
-; AVX2-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX2-NEXT:    vpsubb %xmm1, %xmm4, %xmm1
-; AVX2-NEXT:    vpand %xmm2, %xmm1, %xmm1
+; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX2-NEXT:    vpsubb %xmm1, %xmm2, %xmm1
+; AVX2-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX2-NEXT:    vpmovzxbq {{.*#+}} xmm2 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; AVX2-NEXT:    vpsllw %xmm2, %ymm0, %ymm3
+; AVX2-NEXT:    vpcmpeqd %ymm4, %ymm4, %ymm4
+; AVX2-NEXT:    vpsllw %xmm2, %ymm4, %ymm2
+; AVX2-NEXT:    vpbroadcastb %xmm2, %ymm2
+; AVX2-NEXT:    vpand %ymm2, %ymm3, %ymm2
+; AVX2-NEXT:    vmovdqa {{.*#+}} xmm3 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; AVX2-NEXT:    vpsubb %xmm1, %xmm3, %xmm1
 ; AVX2-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
-; AVX2-NEXT:    vpsllw %xmm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpsllw %xmm1, %ymm5, %ymm1
+; AVX2-NEXT:    vpsrlw %xmm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpsrlw %xmm1, %ymm4, %ymm1
+; AVX2-NEXT:    vpsrlw $8, %ymm1, %ymm1
 ; AVX2-NEXT:    vpbroadcastb %xmm1, %ymm1
 ; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpor %ymm3, %ymm0, %ymm0
+; AVX2-NEXT:    vpor %ymm0, %ymm2, %ymm0
 ; AVX2-NEXT:    retq
 ;
 ; AVX512F-LABEL: splatvar_funnnel_v32i8:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vpbroadcastb %xmm1, %ymm1
-; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm2 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
-; AVX512F-NEXT:    vpand %xmm2, %xmm1, %xmm3
-; AVX512F-NEXT:    vpmovzxbq {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,zero,zero,zero,zero,xmm3[1],zero,zero,zero,zero,zero,zero,zero
-; AVX512F-NEXT:    vpsrlw %xmm3, %ymm0, %ymm4
-; AVX512F-NEXT:    vpcmpeqd %ymm5, %ymm5, %ymm5
-; AVX512F-NEXT:    vpsrlw %xmm3, %ymm5, %ymm3
-; AVX512F-NEXT:    vpsrlw $8, %ymm3, %ymm3
-; AVX512F-NEXT:    vpbroadcastb %xmm3, %ymm3
-; AVX512F-NEXT:    vpand %ymm3, %ymm4, %ymm3
-; AVX512F-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX512F-NEXT:    vpsubb %xmm1, %xmm4, %xmm1
-; AVX512F-NEXT:    vpand %xmm2, %xmm1, %xmm1
+; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX512F-NEXT:    vpsubb %xmm1, %xmm2, %xmm1
+; AVX512F-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX512F-NEXT:    vpmovzxbq {{.*#+}} xmm2 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512F-NEXT:    vpsllw %xmm2, %ymm0, %ymm3
+; AVX512F-NEXT:    vpcmpeqd %ymm4, %ymm4, %ymm4
+; AVX512F-NEXT:    vpsllw %xmm2, %ymm4, %ymm2
+; AVX512F-NEXT:    vpbroadcastb %xmm2, %ymm2
+; AVX512F-NEXT:    vpand %ymm2, %ymm3, %ymm2
+; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm3 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; AVX512F-NEXT:    vpsubb %xmm1, %xmm3, %xmm1
 ; AVX512F-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
-; AVX512F-NEXT:    vpsllw %xmm1, %ymm0, %ymm0
-; AVX512F-NEXT:    vpsllw %xmm1, %ymm5, %ymm1
+; AVX512F-NEXT:    vpsrlw %xmm1, %ymm0, %ymm0
+; AVX512F-NEXT:    vpsrlw %xmm1, %ymm4, %ymm1
+; AVX512F-NEXT:    vpsrlw $8, %ymm1, %ymm1
 ; AVX512F-NEXT:    vpbroadcastb %xmm1, %ymm1
 ; AVX512F-NEXT:    vpand %ymm1, %ymm0, %ymm0
-; AVX512F-NEXT:    vpor %ymm3, %ymm0, %ymm0
+; AVX512F-NEXT:    vpor %ymm0, %ymm2, %ymm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: splatvar_funnnel_v32i8:
 ; AVX512VL:       # %bb.0:
 ; AVX512VL-NEXT:    vpbroadcastb %xmm1, %ymm1
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm2 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
-; AVX512VL-NEXT:    vpand %xmm2, %xmm1, %xmm3
-; AVX512VL-NEXT:    vpmovzxbq {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,zero,zero,zero,zero,xmm3[1],zero,zero,zero,zero,zero,zero,zero
-; AVX512VL-NEXT:    vpsrlw %xmm3, %ymm0, %ymm4
-; AVX512VL-NEXT:    vpcmpeqd %ymm5, %ymm5, %ymm5
-; AVX512VL-NEXT:    vpsrlw %xmm3, %ymm5, %ymm3
-; AVX512VL-NEXT:    vpsrlw $8, %ymm3, %ymm3
-; AVX512VL-NEXT:    vpbroadcastb %xmm3, %ymm3
-; AVX512VL-NEXT:    vpand %ymm3, %ymm4, %ymm3
-; AVX512VL-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX512VL-NEXT:    vpsubb %xmm1, %xmm4, %xmm1
-; AVX512VL-NEXT:    vpand %xmm2, %xmm1, %xmm1
+; AVX512VL-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX512VL-NEXT:    vpsubb %xmm1, %xmm2, %xmm1
+; AVX512VL-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX512VL-NEXT:    vpmovzxbq {{.*#+}} xmm2 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; AVX512VL-NEXT:    vpsllw %xmm2, %ymm0, %ymm3
+; AVX512VL-NEXT:    vpcmpeqd %ymm4, %ymm4, %ymm4
+; AVX512VL-NEXT:    vpsllw %xmm2, %ymm4, %ymm2
+; AVX512VL-NEXT:    vpbroadcastb %xmm2, %ymm2
+; AVX512VL-NEXT:    vpand %ymm2, %ymm3, %ymm2
+; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm3 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; AVX512VL-NEXT:    vpsubb %xmm1, %xmm3, %xmm1
 ; AVX512VL-NEXT:    vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
-; AVX512VL-NEXT:    vpsllw %xmm1, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpsllw %xmm1, %ymm5, %ymm1
+; AVX512VL-NEXT:    vpsrlw %xmm1, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpsrlw %xmm1, %ymm4, %ymm1
+; AVX512VL-NEXT:    vpsrlw $8, %ymm1, %ymm1
 ; AVX512VL-NEXT:    vpbroadcastb %xmm1, %ymm1
 ; AVX512VL-NEXT:    vpand %ymm1, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpor %ymm3, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpor %ymm0, %ymm2, %ymm0
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: splatvar_funnnel_v32i8:
@@ -1101,44 +962,23 @@ define <32 x i8> @splatvar_funnnel_v32i8(<32 x i8> %x, <32 x i8> %amt) nounwind
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
 ; XOPAVX1-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
-; XOPAVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm1, %ymm3
-; XOPAVX1-NEXT:    vandps {{.*}}(%rip), %ymm3, %ymm3
-; XOPAVX1-NEXT:    vextractf128 $1, %ymm3, %xmm4
-; XOPAVX1-NEXT:    vpsubb %xmm4, %xmm2, %xmm4
-; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm5
-; XOPAVX1-NEXT:    vpshlb %xmm4, %xmm5, %xmm4
-; XOPAVX1-NEXT:    vpsubb %xmm3, %xmm2, %xmm3
-; XOPAVX1-NEXT:    vpshlb %xmm3, %xmm0, %xmm3
-; XOPAVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm3, %ymm3
 ; XOPAVX1-NEXT:    vpsubb %xmm1, %xmm2, %xmm1
-; XOPAVX1-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
-; XOPAVX1-NEXT:    vpshlb %xmm1, %xmm5, %xmm2
-; XOPAVX1-NEXT:    vpshlb %xmm1, %xmm0, %xmm0
+; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; XOPAVX1-NEXT:    vprotb %xmm1, %xmm2, %xmm2
+; XOPAVX1-NEXT:    vprotb %xmm1, %xmm0, %xmm0
 ; XOPAVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
-; XOPAVX1-NEXT:    vorps %ymm3, %ymm0, %ymm0
 ; XOPAVX1-NEXT:    retq
 ;
 ; XOPAVX2-LABEL: splatvar_funnnel_v32i8:
 ; XOPAVX2:       # %bb.0:
 ; XOPAVX2-NEXT:    vpbroadcastb %xmm1, %ymm1
-; XOPAVX2-NEXT:    vmovdqa {{.*#+}} ymm2 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
-; XOPAVX2-NEXT:    vpand %ymm2, %ymm1, %ymm3
-; XOPAVX2-NEXT:    vextracti128 $1, %ymm3, %xmm4
-; XOPAVX2-NEXT:    vpxor %xmm5, %xmm5, %xmm5
-; XOPAVX2-NEXT:    vpsubb %xmm4, %xmm5, %xmm4
-; XOPAVX2-NEXT:    vextracti128 $1, %ymm0, %xmm6
-; XOPAVX2-NEXT:    vpshlb %xmm4, %xmm6, %xmm4
-; XOPAVX2-NEXT:    vpsubb %xmm3, %xmm5, %xmm3
-; XOPAVX2-NEXT:    vpshlb %xmm3, %xmm0, %xmm3
-; XOPAVX2-NEXT:    vinserti128 $1, %xmm4, %ymm3, %ymm3
-; XOPAVX2-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; XOPAVX2-NEXT:    vpsubb %ymm1, %ymm4, %ymm1
-; XOPAVX2-NEXT:    vpand %ymm2, %ymm1, %ymm1
+; XOPAVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; XOPAVX2-NEXT:    vpsubb %ymm1, %ymm2, %ymm1
 ; XOPAVX2-NEXT:    vextracti128 $1, %ymm1, %xmm2
-; XOPAVX2-NEXT:    vpshlb %xmm2, %xmm6, %xmm2
-; XOPAVX2-NEXT:    vpshlb %xmm1, %xmm0, %xmm0
+; XOPAVX2-NEXT:    vextracti128 $1, %ymm0, %xmm3
+; XOPAVX2-NEXT:    vprotb %xmm2, %xmm3, %xmm2
+; XOPAVX2-NEXT:    vprotb %xmm1, %xmm0, %xmm0
 ; XOPAVX2-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm0
-; XOPAVX2-NEXT:    vpor %ymm3, %ymm0, %ymm0
 ; XOPAVX2-NEXT:    retq
   %splat = shufflevector <32 x i8> %amt, <32 x i8> undef, <32 x i32> zeroinitializer
   %res = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %x, <32 x i8> %x, <32 x i8> %splat)
@@ -1180,27 +1020,27 @@ define <4 x i64> @constant_funnnel_v4i64(<4 x i64> %x) nounwind {
 ; AVX512F-LABEL: constant_funnnel_v4i64:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm1 = [60,50,14,4]
-; AVX512F-NEXT:    vprolvq %zmm1, %zmm0, %zmm0
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm1 = [4,14,50,60]
+; AVX512F-NEXT:    vprorvq %zmm1, %zmm0, %zmm0
 ; AVX512F-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: constant_funnnel_v4i64:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vprolvq {{.*}}(%rip), %ymm0, %ymm0
+; AVX512VL-NEXT:    vprorvq {{.*}}(%rip), %ymm0, %ymm0
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: constant_funnnel_v4i64:
 ; AVX512BW:       # %bb.0:
 ; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; AVX512BW-NEXT:    vmovdqa {{.*#+}} ymm1 = [60,50,14,4]
-; AVX512BW-NEXT:    vprolvq %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vmovdqa {{.*#+}} ymm1 = [4,14,50,60]
+; AVX512BW-NEXT:    vprorvq %zmm1, %zmm0, %zmm0
 ; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
 ; AVX512BW-NEXT:    retq
 ;
 ; AVX512VLBW-LABEL: constant_funnnel_v4i64:
 ; AVX512VLBW:       # %bb.0:
-; AVX512VLBW-NEXT:    vprolvq {{.*}}(%rip), %ymm0, %ymm0
+; AVX512VLBW-NEXT:    vprorvq {{.*}}(%rip), %ymm0, %ymm0
 ; AVX512VLBW-NEXT:    retq
 ;
 ; XOPAVX1-LABEL: constant_funnnel_v4i64:
@@ -1259,27 +1099,27 @@ define <8 x i32> @constant_funnnel_v8i32(<8 x i32> %x) nounwind {
 ; AVX512F-LABEL: constant_funnnel_v8i32:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm1 = [28,27,26,25,24,23,22,21]
-; AVX512F-NEXT:    vprolvd %zmm1, %zmm0, %zmm0
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm1 = [4,5,6,7,8,9,10,11]
+; AVX512F-NEXT:    vprorvd %zmm1, %zmm0, %zmm0
 ; AVX512F-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: constant_funnnel_v8i32:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vprolvd {{.*}}(%rip), %ymm0, %ymm0
+; AVX512VL-NEXT:    vprorvd {{.*}}(%rip), %ymm0, %ymm0
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: constant_funnnel_v8i32:
 ; AVX512BW:       # %bb.0:
 ; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; AVX512BW-NEXT:    vmovdqa {{.*#+}} ymm1 = [28,27,26,25,24,23,22,21]
-; AVX512BW-NEXT:    vprolvd %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vmovdqa {{.*#+}} ymm1 = [4,5,6,7,8,9,10,11]
+; AVX512BW-NEXT:    vprorvd %zmm1, %zmm0, %zmm0
 ; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
 ; AVX512BW-NEXT:    retq
 ;
 ; AVX512VLBW-LABEL: constant_funnnel_v8i32:
 ; AVX512VLBW:       # %bb.0:
-; AVX512VLBW-NEXT:    vprolvd {{.*}}(%rip), %ymm0, %ymm0
+; AVX512VLBW-NEXT:    vprorvd {{.*}}(%rip), %ymm0, %ymm0
 ; AVX512VLBW-NEXT:    retq
 ;
 ; XOPAVX1-LABEL: constant_funnnel_v8i32:
@@ -1306,47 +1146,44 @@ define <16 x i16> @constant_funnnel_v16i16(<16 x i16> %x) nounwind {
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [256,128,64,32,16,8,4,2]
-; AVX1-NEXT:    vpmullw %xmm2, %xmm1, %xmm3
-; AVX1-NEXT:    vpmullw {{.*}}(%rip), %xmm0, %xmm4
-; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm4, %ymm3
-; AVX1-NEXT:    vpmulhuw %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vpmulhuw {{.*}}(%rip), %xmm0, %xmm2
-; AVX1-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm2[1,2,3,4,5,6,7]
+; AVX1-NEXT:    vpmulhuw %xmm2, %xmm1, %xmm3
+; AVX1-NEXT:    vpmullw %xmm2, %xmm1, %xmm1
+; AVX1-NEXT:    vpor %xmm3, %xmm1, %xmm1
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [1,32768,16384,8192,4096,2048,1024,512]
+; AVX1-NEXT:    vpmulhuw %xmm2, %xmm0, %xmm3
+; AVX1-NEXT:    vpmullw %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpor %xmm3, %xmm0, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
-; AVX1-NEXT:    vorps %ymm0, %ymm3, %ymm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: constant_funnnel_v16i16:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpmulhuw {{.*}}(%rip), %ymm0, %ymm1
-; AVX2-NEXT:    vpblendw {{.*#+}} ymm2 = ymm0[0],ymm1[1,2,3,4,5,6,7],ymm0[8],ymm1[9,10,11,12,13,14,15]
-; AVX2-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
-; AVX2-NEXT:    vpmullw {{.*}}(%rip), %ymm0, %ymm0
-; AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vmovdqa {{.*#+}} ymm1 = [1,32768,16384,8192,4096,2048,1024,512,256,128,64,32,16,8,4,2]
+; AVX2-NEXT:    vpmulhuw %ymm1, %ymm0, %ymm2
+; AVX2-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpor %ymm2, %ymm0, %ymm0
 ; AVX2-NEXT:    retq
 ;
 ; AVX512F-LABEL: constant_funnnel_v16i16:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpmulhuw {{.*}}(%rip), %ymm0, %ymm1
-; AVX512F-NEXT:    vpblendw {{.*#+}} ymm2 = ymm0[0],ymm1[1,2,3,4,5,6,7],ymm0[8],ymm1[9,10,11,12,13,14,15]
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
-; AVX512F-NEXT:    vpmullw {{.*}}(%rip), %ymm0, %ymm0
-; AVX512F-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm1 = [1,32768,16384,8192,4096,2048,1024,512,256,128,64,32,16,8,4,2]
+; AVX512F-NEXT:    vpmulhuw %ymm1, %ymm0, %ymm2
+; AVX512F-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
+; AVX512F-NEXT:    vpor %ymm2, %ymm0, %ymm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: constant_funnnel_v16i16:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpmulhuw {{.*}}(%rip), %ymm0, %ymm1
-; AVX512VL-NEXT:    vpblendw {{.*#+}} ymm2 = ymm0[0],ymm1[1,2,3,4,5,6,7],ymm0[8],ymm1[9,10,11,12,13,14,15]
-; AVX512VL-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
-; AVX512VL-NEXT:    vpmullw {{.*}}(%rip), %ymm0, %ymm0
-; AVX512VL-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm1 = [1,32768,16384,8192,4096,2048,1024,512,256,128,64,32,16,8,4,2]
+; AVX512VL-NEXT:    vpmulhuw %ymm1, %ymm0, %ymm2
+; AVX512VL-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpor %ymm2, %ymm0, %ymm0
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: constant_funnnel_v16i16:
 ; AVX512BW:       # %bb.0:
 ; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; AVX512BW-NEXT:    vmovdqa {{.*#+}} ymm1 = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15]
+; AVX512BW-NEXT:    vmovdqa {{.*#+}} ymm1 = [16,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15]
 ; AVX512BW-NEXT:    vpsrlvw %zmm1, %zmm0, %zmm1
 ; AVX512BW-NEXT:    vmovdqa {{.*#+}} ymm2 = [0,15,14,13,12,11,10,9,8,7,6,5,4,3,2,1]
 ; AVX512BW-NEXT:    vpsllvw %zmm2, %zmm0, %zmm0
@@ -1362,23 +1199,18 @@ define <16 x i16> @constant_funnnel_v16i16(<16 x i16> %x) nounwind {
 ;
 ; XOPAVX1-LABEL: constant_funnnel_v16i16:
 ; XOPAVX1:       # %bb.0:
-; XOPAVX1-NEXT:    vpshlw {{.*}}(%rip), %xmm0, %xmm1
-; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
-; XOPAVX1-NEXT:    vpshlw {{.*}}(%rip), %xmm2, %xmm3
-; XOPAVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm1, %ymm1
-; XOPAVX1-NEXT:    vpshlw {{.*}}(%rip), %xmm0, %xmm0
-; XOPAVX1-NEXT:    vpshlw {{.*}}(%rip), %xmm2, %xmm2
-; XOPAVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
-; XOPAVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
+; XOPAVX1-NEXT:    vprotw {{.*}}(%rip), %xmm0, %xmm1
+; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; XOPAVX1-NEXT:    vprotw {{.*}}(%rip), %xmm0, %xmm0
+; XOPAVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; XOPAVX1-NEXT:    retq
 ;
 ; XOPAVX2-LABEL: constant_funnnel_v16i16:
 ; XOPAVX2:       # %bb.0:
-; XOPAVX2-NEXT:    vpmulhuw {{.*}}(%rip), %ymm0, %ymm1
-; XOPAVX2-NEXT:    vpblendw {{.*#+}} ymm2 = ymm0[0],ymm1[1,2,3,4,5,6,7],ymm0[8],ymm1[9,10,11,12,13,14,15]
-; XOPAVX2-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
-; XOPAVX2-NEXT:    vpmullw {{.*}}(%rip), %ymm0, %ymm0
-; XOPAVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; XOPAVX2-NEXT:    vprotw {{.*}}(%rip), %xmm0, %xmm1
+; XOPAVX2-NEXT:    vextracti128 $1, %ymm0, %xmm0
+; XOPAVX2-NEXT:    vprotw {{.*}}(%rip), %xmm0, %xmm0
+; XOPAVX2-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm0
 ; XOPAVX2-NEXT:    retq
   %res = call <16 x i16> @llvm.fshr.v16i16(<16 x i16> %x, <16 x i16> %x, <16 x i16> <i16 0, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15>)
   ret <16 x i16> %res
@@ -1388,41 +1220,41 @@ define <32 x i8> @constant_funnnel_v32i8(<32 x i8> %x) nounwind {
 ; AVX1-LABEL: constant_funnnel_v32i8:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm1[8],xmm2[8],xmm1[9],xmm2[9],xmm1[10],xmm2[10],xmm1[11],xmm2[11],xmm1[12],xmm2[12],xmm1[13],xmm2[13],xmm1[14],xmm2[14],xmm1[15],xmm2[15]
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = [256,2,4,8,16,32,64,128]
-; AVX1-NEXT:    vpmullw %xmm4, %xmm3, %xmm3
+; AVX1-NEXT:    vpxor %xmm8, %xmm8, %xmm8
+; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm1[8],xmm8[8],xmm1[9],xmm8[9],xmm1[10],xmm8[10],xmm1[11],xmm8[11],xmm1[12],xmm8[12],xmm1[13],xmm8[13],xmm1[14],xmm8[14],xmm1[15],xmm8[15]
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm9 = [256,2,4,8,16,32,64,128]
+; AVX1-NEXT:    vpmullw %xmm9, %xmm3, %xmm3
 ; AVX1-NEXT:    vpsrlw $8, %xmm3, %xmm3
 ; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm5 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
 ; AVX1-NEXT:    vmovdqa {{.*#+}} xmm6 = [256,128,64,32,16,8,4,2]
 ; AVX1-NEXT:    vpmullw %xmm6, %xmm5, %xmm7
 ; AVX1-NEXT:    vpsrlw $8, %xmm7, %xmm7
 ; AVX1-NEXT:    vpackuswb %xmm3, %xmm7, %xmm3
-; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm2 = xmm0[8],xmm2[8],xmm0[9],xmm2[9],xmm0[10],xmm2[10],xmm0[11],xmm2[11],xmm0[12],xmm2[12],xmm0[13],xmm2[13],xmm0[14],xmm2[14],xmm0[15],xmm2[15]
-; AVX1-NEXT:    vpmullw %xmm4, %xmm2, %xmm2
-; AVX1-NEXT:    vpsrlw $8, %xmm2, %xmm2
-; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm4 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; AVX1-NEXT:    vpmullw %xmm6, %xmm4, %xmm6
-; AVX1-NEXT:    vpsrlw $8, %xmm6, %xmm6
-; AVX1-NEXT:    vpackuswb %xmm2, %xmm6, %xmm2
-; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm2, %ymm2
 ; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm1[8],xmm0[8],xmm1[9],xmm0[9],xmm1[10],xmm0[10],xmm1[11],xmm0[11],xmm1[12],xmm0[12],xmm1[13],xmm0[13],xmm1[14],xmm0[14],xmm1[15],xmm0[15]
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [1,2,4,8,16,32,64,128]
-; AVX1-NEXT:    vpmullw %xmm3, %xmm1, %xmm1
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm6 = [255,255,255,255,255,255,255,255]
-; AVX1-NEXT:    vpand %xmm6, %xmm1, %xmm1
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm7 = [1,128,64,32,16,8,4,2]
-; AVX1-NEXT:    vpmullw %xmm7, %xmm5, %xmm5
-; AVX1-NEXT:    vpand %xmm6, %xmm5, %xmm5
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm7 = [1,2,4,8,16,32,64,128]
+; AVX1-NEXT:    vpmullw %xmm7, %xmm1, %xmm1
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
+; AVX1-NEXT:    vpand %xmm2, %xmm1, %xmm1
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = [1,128,64,32,16,8,4,2]
+; AVX1-NEXT:    vpmullw %xmm4, %xmm5, %xmm5
+; AVX1-NEXT:    vpand %xmm2, %xmm5, %xmm5
 ; AVX1-NEXT:    vpackuswb %xmm1, %xmm5, %xmm1
+; AVX1-NEXT:    vpor %xmm3, %xmm1, %xmm1
+; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm0[8],xmm8[8],xmm0[9],xmm8[9],xmm0[10],xmm8[10],xmm0[11],xmm8[11],xmm0[12],xmm8[12],xmm0[13],xmm8[13],xmm0[14],xmm8[14],xmm0[15],xmm8[15]
+; AVX1-NEXT:    vpmullw %xmm9, %xmm3, %xmm3
+; AVX1-NEXT:    vpsrlw $8, %xmm3, %xmm3
+; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm5 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX1-NEXT:    vpmullw %xmm6, %xmm5, %xmm6
+; AVX1-NEXT:    vpsrlw $8, %xmm6, %xmm6
+; AVX1-NEXT:    vpackuswb %xmm3, %xmm6, %xmm3
 ; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; AVX1-NEXT:    vpmullw %xmm3, %xmm0, %xmm0
-; AVX1-NEXT:    vpand %xmm6, %xmm0, %xmm0
-; AVX1-NEXT:    vpmullw %xmm7, %xmm4, %xmm3
-; AVX1-NEXT:    vpand %xmm6, %xmm3, %xmm3
-; AVX1-NEXT:    vpackuswb %xmm0, %xmm3, %xmm0
+; AVX1-NEXT:    vpmullw %xmm7, %xmm0, %xmm0
+; AVX1-NEXT:    vpand %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpmullw %xmm4, %xmm5, %xmm4
+; AVX1-NEXT:    vpand %xmm2, %xmm4, %xmm2
+; AVX1-NEXT:    vpackuswb %xmm0, %xmm2, %xmm0
+; AVX1-NEXT:    vpor %xmm3, %xmm0, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
-; AVX1-NEXT:    vorps %ymm2, %ymm0, %ymm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: constant_funnnel_v32i8:
@@ -1519,29 +1351,19 @@ define <32 x i8> @constant_funnnel_v32i8(<32 x i8> %x) nounwind {
 ; XOPAVX1-LABEL: constant_funnnel_v32i8:
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; XOPAVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [0,255,254,253,252,251,250,249,0,249,250,251,252,253,254,255]
-; XOPAVX1-NEXT:    vpshlb %xmm2, %xmm1, %xmm3
-; XOPAVX1-NEXT:    vpshlb %xmm2, %xmm0, %xmm2
-; XOPAVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm2, %ymm2
-; XOPAVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [0,7,6,5,4,3,2,1,0,1,2,3,4,5,6,7]
-; XOPAVX1-NEXT:    vpshlb %xmm3, %xmm1, %xmm1
-; XOPAVX1-NEXT:    vpshlb %xmm3, %xmm0, %xmm0
+; XOPAVX1-NEXT:    vmovdqa {{.*#+}} xmm2 = [0,255,254,253,252,251,250,249,248,249,250,251,252,253,254,255]
+; XOPAVX1-NEXT:    vprotb %xmm2, %xmm1, %xmm1
+; XOPAVX1-NEXT:    vprotb %xmm2, %xmm0, %xmm0
 ; XOPAVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
-; XOPAVX1-NEXT:    vorps %ymm2, %ymm0, %ymm0
 ; XOPAVX1-NEXT:    retq
 ;
 ; XOPAVX2-LABEL: constant_funnnel_v32i8:
 ; XOPAVX2:       # %bb.0:
 ; XOPAVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; XOPAVX2-NEXT:    vmovdqa {{.*#+}} xmm2 = [0,255,254,253,252,251,250,249,0,249,250,251,252,253,254,255]
-; XOPAVX2-NEXT:    vpshlb %xmm2, %xmm1, %xmm3
-; XOPAVX2-NEXT:    vpshlb %xmm2, %xmm0, %xmm2
-; XOPAVX2-NEXT:    vinserti128 $1, %xmm3, %ymm2, %ymm2
-; XOPAVX2-NEXT:    vmovdqa {{.*#+}} xmm3 = [0,7,6,5,4,3,2,1,0,1,2,3,4,5,6,7]
-; XOPAVX2-NEXT:    vpshlb %xmm3, %xmm1, %xmm1
-; XOPAVX2-NEXT:    vpshlb %xmm3, %xmm0, %xmm0
+; XOPAVX2-NEXT:    vmovdqa {{.*#+}} xmm2 = [0,255,254,253,252,251,250,249,248,249,250,251,252,253,254,255]
+; XOPAVX2-NEXT:    vprotb %xmm2, %xmm1, %xmm1
+; XOPAVX2-NEXT:    vprotb %xmm2, %xmm0, %xmm0
 ; XOPAVX2-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
-; XOPAVX2-NEXT:    vpor %ymm2, %ymm0, %ymm0
 ; XOPAVX2-NEXT:    retq
   %res = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %x, <32 x i8> %x, <32 x i8> <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1>)
   ret <32 x i8> %res
@@ -1574,25 +1396,25 @@ define <4 x i64> @splatconstant_funnnel_v4i64(<4 x i64> %x) nounwind {
 ; AVX512F-LABEL: splatconstant_funnnel_v4i64:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; AVX512F-NEXT:    vprolq $50, %zmm0, %zmm0
+; AVX512F-NEXT:    vprorq $14, %zmm0, %zmm0
 ; AVX512F-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: splatconstant_funnnel_v4i64:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vprolq $50, %ymm0, %ymm0
+; AVX512VL-NEXT:    vprorq $14, %ymm0, %ymm0
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: splatconstant_funnnel_v4i64:
 ; AVX512BW:       # %bb.0:
 ; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; AVX512BW-NEXT:    vprolq $50, %zmm0, %zmm0
+; AVX512BW-NEXT:    vprorq $14, %zmm0, %zmm0
 ; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
 ; AVX512BW-NEXT:    retq
 ;
 ; AVX512VLBW-LABEL: splatconstant_funnnel_v4i64:
 ; AVX512VLBW:       # %bb.0:
-; AVX512VLBW-NEXT:    vprolq $50, %ymm0, %ymm0
+; AVX512VLBW-NEXT:    vprorq $14, %ymm0, %ymm0
 ; AVX512VLBW-NEXT:    retq
 ;
 ; XOPAVX1-LABEL: splatconstant_funnnel_v4i64:
@@ -1637,25 +1459,25 @@ define <8 x i32> @splatconstant_funnnel_v8i32(<8 x i32> %x) nounwind {
 ; AVX512F-LABEL: splatconstant_funnnel_v8i32:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; AVX512F-NEXT:    vprold $28, %zmm0, %zmm0
+; AVX512F-NEXT:    vprord $4, %zmm0, %zmm0
 ; AVX512F-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: splatconstant_funnnel_v8i32:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vprold $28, %ymm0, %ymm0
+; AVX512VL-NEXT:    vprord $4, %ymm0, %ymm0
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: splatconstant_funnnel_v8i32:
 ; AVX512BW:       # %bb.0:
 ; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; AVX512BW-NEXT:    vprold $28, %zmm0, %zmm0
+; AVX512BW-NEXT:    vprord $4, %zmm0, %zmm0
 ; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
 ; AVX512BW-NEXT:    retq
 ;
 ; AVX512VLBW-LABEL: splatconstant_funnnel_v8i32:
 ; AVX512VLBW:       # %bb.0:
-; AVX512VLBW-NEXT:    vprold $28, %ymm0, %ymm0
+; AVX512VLBW-NEXT:    vprord $4, %ymm0, %ymm0
 ; AVX512VLBW-NEXT:    retq
 ;
 ; XOPAVX1-LABEL: splatconstant_funnnel_v8i32:
@@ -1728,16 +1550,15 @@ define <32 x i8> @splatconstant_funnnel_v32i8(<32 x i8> %x) nounwind {
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; AVX1-NEXT:    vpsrlw $4, %xmm1, %xmm2
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; AVX1-NEXT:    vpand %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
+; AVX1-NEXT:    vpandn %xmm2, %xmm3, %xmm2
 ; AVX1-NEXT:    vpsllw $4, %xmm1, %xmm1
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
-; AVX1-NEXT:    vpand %xmm4, %xmm1, %xmm1
+; AVX1-NEXT:    vpand %xmm3, %xmm1, %xmm1
 ; AVX1-NEXT:    vpor %xmm2, %xmm1, %xmm1
 ; AVX1-NEXT:    vpsrlw $4, %xmm0, %xmm2
-; AVX1-NEXT:    vpand %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vpandn %xmm2, %xmm3, %xmm2
 ; AVX1-NEXT:    vpsllw $4, %xmm0, %xmm0
-; AVX1-NEXT:    vpand %xmm4, %xmm0, %xmm0
+; AVX1-NEXT:    vpand %xmm3, %xmm0, %xmm0
 ; AVX1-NEXT:    vpor %xmm2, %xmm0, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
 ; AVX1-NEXT:    retq
diff --git a/test/CodeGen/X86/vector-fshr-rot-512.ll b/test/CodeGen/X86/vector-fshr-rot-512.ll
index 792f172e4d169b94014281349b1dfe45bd7040f4..20add95642684bc139808e11ff914f04fe1843b5 100644
--- a/test/CodeGen/X86/vector-fshr-rot-512.ll
+++ b/test/CodeGen/X86/vector-fshr-rot-512.ll
@@ -16,14 +16,7 @@ declare <64 x i8> @llvm.fshr.v64i8(<64 x i8>, <64 x i8>, <64 x i8>)
 define <8 x i64> @var_funnnel_v8i64(<8 x i64> %x, <8 x i64> %amt) nounwind {
 ; AVX512-LABEL: var_funnnel_v8i64:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpbroadcastq {{.*#+}} zmm2 = [63,63,63,63,63,63,63,63]
-; AVX512-NEXT:    vpandq %zmm2, %zmm1, %zmm3
-; AVX512-NEXT:    vpsrlvq %zmm3, %zmm0, %zmm3
-; AVX512-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX512-NEXT:    vpsubq %zmm1, %zmm4, %zmm1
-; AVX512-NEXT:    vpandq %zmm2, %zmm1, %zmm1
-; AVX512-NEXT:    vpsllvq %zmm1, %zmm0, %zmm0
-; AVX512-NEXT:    vporq %zmm3, %zmm0, %zmm0
+; AVX512-NEXT:    vprorvq %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    retq
   %res = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %x, <8 x i64> %x, <8 x i64> %amt)
   ret <8 x i64> %res
@@ -32,14 +25,7 @@ define <8 x i64> @var_funnnel_v8i64(<8 x i64> %x, <8 x i64> %amt) nounwind {
 define <16 x i32> @var_funnnel_v16i32(<16 x i32> %x, <16 x i32> %amt) nounwind {
 ; AVX512-LABEL: var_funnnel_v16i32:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpbroadcastd {{.*#+}} zmm2 = [31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31]
-; AVX512-NEXT:    vpandd %zmm2, %zmm1, %zmm3
-; AVX512-NEXT:    vpsrlvd %zmm3, %zmm0, %zmm3
-; AVX512-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX512-NEXT:    vpsubd %zmm1, %zmm4, %zmm1
-; AVX512-NEXT:    vpandd %zmm2, %zmm1, %zmm1
-; AVX512-NEXT:    vpsllvd %zmm1, %zmm0, %zmm0
-; AVX512-NEXT:    vpord %zmm3, %zmm0, %zmm0
+; AVX512-NEXT:    vprorvd %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    retq
   %res = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %x, <16 x i32> %x, <16 x i32> %amt)
   ret <16 x i32> %res
@@ -131,52 +117,50 @@ define <64 x i8> @var_funnnel_v64i8(<64 x i8> %x, <64 x i8> %amt) nounwind {
 ; AVX512F-LABEL: var_funnnel_v64i8:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vpsrlw $4, %ymm0, %ymm4
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm5 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; AVX512F-NEXT:    vpand %ymm5, %ymm4, %ymm4
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm5 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
+; AVX512F-NEXT:    vpandn %ymm4, %ymm5, %ymm4
 ; AVX512F-NEXT:    vpsllw $4, %ymm0, %ymm6
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm7 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
-; AVX512F-NEXT:    vpand %ymm7, %ymm6, %ymm6
+; AVX512F-NEXT:    vpand %ymm5, %ymm6, %ymm6
 ; AVX512F-NEXT:    vpor %ymm4, %ymm6, %ymm4
 ; AVX512F-NEXT:    vpxor %xmm6, %xmm6, %xmm6
 ; AVX512F-NEXT:    vpsubb %ymm2, %ymm6, %ymm2
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm8 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
-; AVX512F-NEXT:    vpand %ymm8, %ymm2, %ymm2
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm7 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
+; AVX512F-NEXT:    vpand %ymm7, %ymm2, %ymm2
 ; AVX512F-NEXT:    vpsllw $5, %ymm2, %ymm2
 ; AVX512F-NEXT:    vpblendvb %ymm2, %ymm4, %ymm0, %ymm0
 ; AVX512F-NEXT:    vpsrlw $6, %ymm0, %ymm4
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm9 = [3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3]
-; AVX512F-NEXT:    vpand %ymm9, %ymm4, %ymm4
-; AVX512F-NEXT:    vpsllw $2, %ymm0, %ymm10
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm11 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252]
-; AVX512F-NEXT:    vpand %ymm11, %ymm10, %ymm10
-; AVX512F-NEXT:    vpor %ymm4, %ymm10, %ymm4
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm8 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252]
+; AVX512F-NEXT:    vpandn %ymm4, %ymm8, %ymm4
+; AVX512F-NEXT:    vpsllw $2, %ymm0, %ymm9
+; AVX512F-NEXT:    vpand %ymm8, %ymm9, %ymm9
+; AVX512F-NEXT:    vpor %ymm4, %ymm9, %ymm4
 ; AVX512F-NEXT:    vpaddb %ymm2, %ymm2, %ymm2
 ; AVX512F-NEXT:    vpblendvb %ymm2, %ymm4, %ymm0, %ymm0
 ; AVX512F-NEXT:    vpsrlw $7, %ymm0, %ymm4
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm10 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
-; AVX512F-NEXT:    vpand %ymm10, %ymm4, %ymm4
-; AVX512F-NEXT:    vpaddb %ymm0, %ymm0, %ymm12
-; AVX512F-NEXT:    vpor %ymm4, %ymm12, %ymm4
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm9 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
+; AVX512F-NEXT:    vpand %ymm9, %ymm4, %ymm4
+; AVX512F-NEXT:    vpaddb %ymm0, %ymm0, %ymm10
+; AVX512F-NEXT:    vpor %ymm4, %ymm10, %ymm4
 ; AVX512F-NEXT:    vpaddb %ymm2, %ymm2, %ymm2
 ; AVX512F-NEXT:    vpblendvb %ymm2, %ymm4, %ymm0, %ymm0
 ; AVX512F-NEXT:    vpsrlw $4, %ymm1, %ymm2
-; AVX512F-NEXT:    vpand %ymm5, %ymm2, %ymm2
+; AVX512F-NEXT:    vpandn %ymm2, %ymm5, %ymm2
 ; AVX512F-NEXT:    vpsllw $4, %ymm1, %ymm4
-; AVX512F-NEXT:    vpand %ymm7, %ymm4, %ymm4
+; AVX512F-NEXT:    vpand %ymm5, %ymm4, %ymm4
 ; AVX512F-NEXT:    vpor %ymm2, %ymm4, %ymm2
 ; AVX512F-NEXT:    vpsubb %ymm3, %ymm6, %ymm3
-; AVX512F-NEXT:    vpand %ymm8, %ymm3, %ymm3
+; AVX512F-NEXT:    vpand %ymm7, %ymm3, %ymm3
 ; AVX512F-NEXT:    vpsllw $5, %ymm3, %ymm3
 ; AVX512F-NEXT:    vpblendvb %ymm3, %ymm2, %ymm1, %ymm1
 ; AVX512F-NEXT:    vpsrlw $6, %ymm1, %ymm2
-; AVX512F-NEXT:    vpand %ymm9, %ymm2, %ymm2
+; AVX512F-NEXT:    vpandn %ymm2, %ymm8, %ymm2
 ; AVX512F-NEXT:    vpsllw $2, %ymm1, %ymm4
-; AVX512F-NEXT:    vpand %ymm11, %ymm4, %ymm4
+; AVX512F-NEXT:    vpand %ymm8, %ymm4, %ymm4
 ; AVX512F-NEXT:    vpor %ymm2, %ymm4, %ymm2
 ; AVX512F-NEXT:    vpaddb %ymm3, %ymm3, %ymm3
 ; AVX512F-NEXT:    vpblendvb %ymm3, %ymm2, %ymm1, %ymm1
 ; AVX512F-NEXT:    vpsrlw $7, %ymm1, %ymm2
-; AVX512F-NEXT:    vpand %ymm10, %ymm2, %ymm2
+; AVX512F-NEXT:    vpand %ymm9, %ymm2, %ymm2
 ; AVX512F-NEXT:    vpaddb %ymm1, %ymm1, %ymm4
 ; AVX512F-NEXT:    vpor %ymm2, %ymm4, %ymm2
 ; AVX512F-NEXT:    vpaddb %ymm3, %ymm3, %ymm3
@@ -186,52 +170,50 @@ define <64 x i8> @var_funnnel_v64i8(<64 x i8> %x, <64 x i8> %amt) nounwind {
 ; AVX512VL-LABEL: var_funnnel_v64i8:
 ; AVX512VL:       # %bb.0:
 ; AVX512VL-NEXT:    vpsrlw $4, %ymm0, %ymm4
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm5 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; AVX512VL-NEXT:    vpand %ymm5, %ymm4, %ymm4
+; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm5 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
+; AVX512VL-NEXT:    vpandn %ymm4, %ymm5, %ymm4
 ; AVX512VL-NEXT:    vpsllw $4, %ymm0, %ymm6
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm7 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
-; AVX512VL-NEXT:    vpand %ymm7, %ymm6, %ymm6
+; AVX512VL-NEXT:    vpand %ymm5, %ymm6, %ymm6
 ; AVX512VL-NEXT:    vpor %ymm4, %ymm6, %ymm4
 ; AVX512VL-NEXT:    vpxor %xmm6, %xmm6, %xmm6
 ; AVX512VL-NEXT:    vpsubb %ymm2, %ymm6, %ymm2
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm8 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
-; AVX512VL-NEXT:    vpand %ymm8, %ymm2, %ymm2
+; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm7 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
+; AVX512VL-NEXT:    vpand %ymm7, %ymm2, %ymm2
 ; AVX512VL-NEXT:    vpsllw $5, %ymm2, %ymm2
 ; AVX512VL-NEXT:    vpblendvb %ymm2, %ymm4, %ymm0, %ymm0
 ; AVX512VL-NEXT:    vpsrlw $6, %ymm0, %ymm4
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm9 = [3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3]
-; AVX512VL-NEXT:    vpand %ymm9, %ymm4, %ymm4
-; AVX512VL-NEXT:    vpsllw $2, %ymm0, %ymm10
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm11 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252]
-; AVX512VL-NEXT:    vpand %ymm11, %ymm10, %ymm10
-; AVX512VL-NEXT:    vpor %ymm4, %ymm10, %ymm4
+; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm8 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252]
+; AVX512VL-NEXT:    vpandn %ymm4, %ymm8, %ymm4
+; AVX512VL-NEXT:    vpsllw $2, %ymm0, %ymm9
+; AVX512VL-NEXT:    vpand %ymm8, %ymm9, %ymm9
+; AVX512VL-NEXT:    vpor %ymm4, %ymm9, %ymm4
 ; AVX512VL-NEXT:    vpaddb %ymm2, %ymm2, %ymm2
 ; AVX512VL-NEXT:    vpblendvb %ymm2, %ymm4, %ymm0, %ymm0
 ; AVX512VL-NEXT:    vpsrlw $7, %ymm0, %ymm4
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm10 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
-; AVX512VL-NEXT:    vpand %ymm10, %ymm4, %ymm4
-; AVX512VL-NEXT:    vpaddb %ymm0, %ymm0, %ymm12
-; AVX512VL-NEXT:    vpor %ymm4, %ymm12, %ymm4
+; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm9 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
+; AVX512VL-NEXT:    vpand %ymm9, %ymm4, %ymm4
+; AVX512VL-NEXT:    vpaddb %ymm0, %ymm0, %ymm10
+; AVX512VL-NEXT:    vpor %ymm4, %ymm10, %ymm4
 ; AVX512VL-NEXT:    vpaddb %ymm2, %ymm2, %ymm2
 ; AVX512VL-NEXT:    vpblendvb %ymm2, %ymm4, %ymm0, %ymm0
 ; AVX512VL-NEXT:    vpsrlw $4, %ymm1, %ymm2
-; AVX512VL-NEXT:    vpand %ymm5, %ymm2, %ymm2
+; AVX512VL-NEXT:    vpandn %ymm2, %ymm5, %ymm2
 ; AVX512VL-NEXT:    vpsllw $4, %ymm1, %ymm4
-; AVX512VL-NEXT:    vpand %ymm7, %ymm4, %ymm4
+; AVX512VL-NEXT:    vpand %ymm5, %ymm4, %ymm4
 ; AVX512VL-NEXT:    vpor %ymm2, %ymm4, %ymm2
 ; AVX512VL-NEXT:    vpsubb %ymm3, %ymm6, %ymm3
-; AVX512VL-NEXT:    vpand %ymm8, %ymm3, %ymm3
+; AVX512VL-NEXT:    vpand %ymm7, %ymm3, %ymm3
 ; AVX512VL-NEXT:    vpsllw $5, %ymm3, %ymm3
 ; AVX512VL-NEXT:    vpblendvb %ymm3, %ymm2, %ymm1, %ymm1
 ; AVX512VL-NEXT:    vpsrlw $6, %ymm1, %ymm2
-; AVX512VL-NEXT:    vpand %ymm9, %ymm2, %ymm2
+; AVX512VL-NEXT:    vpandn %ymm2, %ymm8, %ymm2
 ; AVX512VL-NEXT:    vpsllw $2, %ymm1, %ymm4
-; AVX512VL-NEXT:    vpand %ymm11, %ymm4, %ymm4
+; AVX512VL-NEXT:    vpand %ymm8, %ymm4, %ymm4
 ; AVX512VL-NEXT:    vpor %ymm2, %ymm4, %ymm2
 ; AVX512VL-NEXT:    vpaddb %ymm3, %ymm3, %ymm3
 ; AVX512VL-NEXT:    vpblendvb %ymm3, %ymm2, %ymm1, %ymm1
 ; AVX512VL-NEXT:    vpsrlw $7, %ymm1, %ymm2
-; AVX512VL-NEXT:    vpand %ymm10, %ymm2, %ymm2
+; AVX512VL-NEXT:    vpand %ymm9, %ymm2, %ymm2
 ; AVX512VL-NEXT:    vpaddb %ymm1, %ymm1, %ymm4
 ; AVX512VL-NEXT:    vpor %ymm2, %ymm4, %ymm2
 ; AVX512VL-NEXT:    vpaddb %ymm3, %ymm3, %ymm3
@@ -325,14 +307,7 @@ define <8 x i64> @splatvar_funnnel_v8i64(<8 x i64> %x, <8 x i64> %amt) nounwind
 ; AVX512-LABEL: splatvar_funnnel_v8i64:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpbroadcastq %xmm1, %zmm1
-; AVX512-NEXT:    vmovdqa {{.*#+}} xmm2 = [63,63]
-; AVX512-NEXT:    vpand %xmm2, %xmm1, %xmm3
-; AVX512-NEXT:    vpsrlq %xmm3, %zmm0, %zmm3
-; AVX512-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX512-NEXT:    vpsubq %xmm1, %xmm4, %xmm1
-; AVX512-NEXT:    vpand %xmm2, %xmm1, %xmm1
-; AVX512-NEXT:    vpsllq %xmm1, %zmm0, %zmm0
-; AVX512-NEXT:    vporq %zmm3, %zmm0, %zmm0
+; AVX512-NEXT:    vprorvq %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    retq
   %splat = shufflevector <8 x i64> %amt, <8 x i64> undef, <8 x i32> zeroinitializer
   %res = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %x, <8 x i64> %x, <8 x i64> %splat)
@@ -343,16 +318,7 @@ define <16 x i32> @splatvar_funnnel_v16i32(<16 x i32> %x, <16 x i32> %amt) nounw
 ; AVX512-LABEL: splatvar_funnnel_v16i32:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpbroadcastd %xmm1, %zmm1
-; AVX512-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [31,31,31,31]
-; AVX512-NEXT:    vpand %xmm2, %xmm1, %xmm3
-; AVX512-NEXT:    vpmovzxdq {{.*#+}} xmm3 = xmm3[0],zero,xmm3[1],zero
-; AVX512-NEXT:    vpsrld %xmm3, %zmm0, %zmm3
-; AVX512-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX512-NEXT:    vpsubd %xmm1, %xmm4, %xmm1
-; AVX512-NEXT:    vpand %xmm2, %xmm1, %xmm1
-; AVX512-NEXT:    vpmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero
-; AVX512-NEXT:    vpslld %xmm1, %zmm0, %zmm0
-; AVX512-NEXT:    vpord %zmm3, %zmm0, %zmm0
+; AVX512-NEXT:    vprorvd %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    retq
   %splat = shufflevector <16 x i32> %amt, <16 x i32> undef, <16 x i32> zeroinitializer
   %res = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %x, <16 x i32> %x, <16 x i32> %splat)
@@ -544,7 +510,7 @@ define <64 x i8> @splatvar_funnnel_v64i8(<64 x i8> %x, <64 x i8> %amt) nounwind
 define <8 x i64> @constant_funnnel_v8i64(<8 x i64> %x) nounwind {
 ; AVX512-LABEL: constant_funnnel_v8i64:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vprolvq {{.*}}(%rip), %zmm0, %zmm0
+; AVX512-NEXT:    vprorvq {{.*}}(%rip), %zmm0, %zmm0
 ; AVX512-NEXT:    retq
   %res = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %x, <8 x i64> %x, <8 x i64> <i64 4, i64 14, i64 50, i64 60, i64 4, i64 14, i64 50, i64 60>)
   ret <8 x i64> %res
@@ -553,7 +519,7 @@ define <8 x i64> @constant_funnnel_v8i64(<8 x i64> %x) nounwind {
 define <16 x i32> @constant_funnnel_v16i32(<16 x i32> %x) nounwind {
 ; AVX512-LABEL: constant_funnnel_v16i32:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vprolvd {{.*}}(%rip), %zmm0, %zmm0
+; AVX512-NEXT:    vprorvd {{.*}}(%rip), %zmm0, %zmm0
 ; AVX512-NEXT:    retq
   %res = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %x, <16 x i32> %x, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11>)
   ret <16 x i32> %res
@@ -769,7 +735,7 @@ define <64 x i8> @constant_funnnel_v64i8(<64 x i8> %x) nounwind {
 define <8 x i64> @splatconstant_funnnel_v8i64(<8 x i64> %x) nounwind {
 ; AVX512-LABEL: splatconstant_funnnel_v8i64:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vprolq $50, %zmm0, %zmm0
+; AVX512-NEXT:    vprorq $14, %zmm0, %zmm0
 ; AVX512-NEXT:    retq
   %res = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %x, <8 x i64> %x, <8 x i64> <i64 14, i64 14, i64 14, i64 14, i64 14, i64 14, i64 14, i64 14>)
   ret <8 x i64> %res
@@ -778,7 +744,7 @@ define <8 x i64> @splatconstant_funnnel_v8i64(<8 x i64> %x) nounwind {
 define <16 x i32> @splatconstant_funnnel_v16i32(<16 x i32> %x) nounwind {
 ; AVX512-LABEL: splatconstant_funnnel_v16i32:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vprold $28, %zmm0, %zmm0
+; AVX512-NEXT:    vprord $4, %zmm0, %zmm0
 ; AVX512-NEXT:    retq
   %res = call <16 x i32> @llvm.fshr.v16i32(<16 x i32> %x, <16 x i32> %x, <16 x i32> <i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4, i32 4>)
   ret <16 x i32> %res
@@ -826,32 +792,30 @@ define <64 x i8> @splatconstant_funnnel_v64i8(<64 x i8> %x) nounwind {
 ; AVX512F-LABEL: splatconstant_funnnel_v64i8:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vpsrlw $4, %ymm0, %ymm2
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; AVX512F-NEXT:    vpand %ymm3, %ymm2, %ymm2
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm3 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
+; AVX512F-NEXT:    vpandn %ymm2, %ymm3, %ymm2
 ; AVX512F-NEXT:    vpsllw $4, %ymm0, %ymm0
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm4 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
-; AVX512F-NEXT:    vpand %ymm4, %ymm0, %ymm0
+; AVX512F-NEXT:    vpand %ymm3, %ymm0, %ymm0
 ; AVX512F-NEXT:    vpor %ymm2, %ymm0, %ymm0
 ; AVX512F-NEXT:    vpsrlw $4, %ymm1, %ymm2
-; AVX512F-NEXT:    vpand %ymm3, %ymm2, %ymm2
+; AVX512F-NEXT:    vpandn %ymm2, %ymm3, %ymm2
 ; AVX512F-NEXT:    vpsllw $4, %ymm1, %ymm1
-; AVX512F-NEXT:    vpand %ymm4, %ymm1, %ymm1
+; AVX512F-NEXT:    vpand %ymm3, %ymm1, %ymm1
 ; AVX512F-NEXT:    vpor %ymm2, %ymm1, %ymm1
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: splatconstant_funnnel_v64i8:
 ; AVX512VL:       # %bb.0:
 ; AVX512VL-NEXT:    vpsrlw $4, %ymm0, %ymm2
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; AVX512VL-NEXT:    vpand %ymm3, %ymm2, %ymm2
+; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm3 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
+; AVX512VL-NEXT:    vpandn %ymm2, %ymm3, %ymm2
 ; AVX512VL-NEXT:    vpsllw $4, %ymm0, %ymm0
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm4 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
-; AVX512VL-NEXT:    vpand %ymm4, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpand %ymm3, %ymm0, %ymm0
 ; AVX512VL-NEXT:    vpor %ymm2, %ymm0, %ymm0
 ; AVX512VL-NEXT:    vpsrlw $4, %ymm1, %ymm2
-; AVX512VL-NEXT:    vpand %ymm3, %ymm2, %ymm2
+; AVX512VL-NEXT:    vpandn %ymm2, %ymm3, %ymm2
 ; AVX512VL-NEXT:    vpsllw $4, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpand %ymm4, %ymm1, %ymm1
+; AVX512VL-NEXT:    vpand %ymm3, %ymm1, %ymm1
 ; AVX512VL-NEXT:    vpor %ymm2, %ymm1, %ymm1
 ; AVX512VL-NEXT:    retq
 ;
diff --git a/test/CodeGen/X86/vector-gep.ll b/test/CodeGen/X86/vector-gep.ll
index 6b3e709e40ee89b9d004caa446701a27ac3847b4..8f62fe53825642e75bea361cb36cda4d4064cff1 100644
--- a/test/CodeGen/X86/vector-gep.ll
+++ b/test/CodeGen/X86/vector-gep.ll
@@ -20,9 +20,8 @@ define <4 x i32*> @AGEP0(i32* %ptr) nounwind {
 define i32 @AGEP1(<4 x i32*> %param) nounwind {
 ; CHECK-LABEL: AGEP1:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vpaddd {{\.LCPI.*}}, %xmm0, %xmm0
-; CHECK-NEXT:    vpextrd $3, %xmm0, %eax
-; CHECK-NEXT:    movl (%eax), %eax
+; CHECK-NEXT:    vextractps $3, %xmm0, %eax
+; CHECK-NEXT:    movl 16(%eax), %eax
 ; CHECK-NEXT:    retl
   %A2 = getelementptr i32, <4 x i32*> %param, <4 x i32> <i32 1, i32 2, i32 3, i32 4>
   %k = extractelement <4 x i32*> %A2, i32 3
diff --git a/test/CodeGen/X86/vector-partial-undef.ll b/test/CodeGen/X86/vector-partial-undef.ll
new file mode 100644
index 0000000000000000000000000000000000000000..69f7ebbfe48aa4b175f8db71edf956b13ec04c56
--- /dev/null
+++ b/test/CodeGen/X86/vector-partial-undef.ll
@@ -0,0 +1,165 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-- -mattr=+sse2 | FileCheck %s --check-prefix=SSE
+; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx2 | FileCheck %s --check-prefix=AVX
+
+; xor undef, undef --> 0 because it's not worth fighting to make that return undef?
+
+define <4 x i64> @xor_insert_insert(<2 x i64> %x, <2 x i64> %y) {
+; SSE-LABEL: xor_insert_insert:
+; SSE:       # %bb.0:
+; SSE-NEXT:    xorps %xmm1, %xmm0
+; SSE-NEXT:    xorps %xmm1, %xmm1
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: xor_insert_insert:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vxorps %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    retq
+  %xw = shufflevector <2 x i64> %x, <2 x i64> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+  %yw = shufflevector <2 x i64> %y, <2 x i64> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+  %r = xor <4 x i64> %xw, %yw
+  ret <4 x i64> %r
+}
+
+define <4 x i64> @xor_insert_insert_high_half(<2 x i64> %x, <2 x i64> %y) {
+; SSE-LABEL: xor_insert_insert_high_half:
+; SSE:       # %bb.0:
+; SSE-NEXT:    xorps %xmm0, %xmm1
+; SSE-NEXT:    xorps %xmm0, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: xor_insert_insert_high_half:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vxorps %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; AVX-NEXT:    retq
+  %xw = shufflevector <2 x i64> %x, <2 x i64> undef, <4 x i32> <i32 undef, i32 undef, i32 0, i32 1>
+  %yw = shufflevector <2 x i64> %y, <2 x i64> undef, <4 x i32> <i32 undef, i32 undef, i32 0, i32 1>
+  %r = xor <4 x i64> %xw, %yw
+  ret <4 x i64> %r
+}
+
+; All elements of the add are undefined:
+;  x[0] , x[1] , x[2] , x[3],  u ,   u  ,   u  ,  u
+; +  u  ,   u  ,   u  ,   u , 42 ,  43  ,  44  , 45
+
+define <8 x i32> @add_undef_elts(<4 x i32> %x) {
+; SSE-LABEL: add_undef_elts:
+; SSE:       # %bb.0:
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: add_undef_elts:
+; AVX:       # %bb.0:
+; AVX-NEXT:    retq
+  %extend = shufflevector <4 x i32> %x, <4 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+  %bogus_bo = add <8 x i32> %extend, <i32 undef, i32 undef, i32 undef, i32 undef, i32 42, i32 43, i32 44, i32 12>
+  %arbitrary_shuf = shufflevector <8 x i32> %bogus_bo, <8 x i32> undef, <8 x i32> <i32 6, i32 0, i32 5, i32 4, i32 3, i32 2, i32 1, i32 7>
+  ret <8 x i32> %arbitrary_shuf
+}
+
+; Verify that constant operand 0 for a sub works too.
+
+define <8 x i32> @sub_undef_elts(<4 x i32> %x) {
+; SSE-LABEL: sub_undef_elts:
+; SSE:       # %bb.0:
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: sub_undef_elts:
+; AVX:       # %bb.0:
+; AVX-NEXT:    retq
+  %extend = shufflevector <4 x i32> %x, <4 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+  %bogus_bo = sub <8 x i32> <i32 undef, i32 undef, i32 undef, i32 undef, i32 42, i32 43, i32 44, i32 12>, %extend
+  %arbitrary_shuf = shufflevector <8 x i32> %bogus_bo, <8 x i32> undef, <8 x i32> <i32 1, i32 0, i32 5, i32 4, i32 3, i32 2, i32 6, i32 7>
+  ret <8 x i32> %arbitrary_shuf
+}
+
+; and undef, C --> 0, so this tests that we are tracking known zero lanes.
+
+define <4 x i64> @and_undef_elts(<2 x i64> %x) {
+; SSE-LABEL: and_undef_elts:
+; SSE:       # %bb.0:
+; SSE-NEXT:    xorps %xmm0, %xmm0
+; SSE-NEXT:    xorps %xmm1, %xmm1
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: and_undef_elts:
+; AVX:       # %bb.0:
+; AVX-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
+; AVX-NEXT:    vandps {{.*}}(%rip), %ymm0, %ymm0
+; AVX-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[3,0,1,2]
+; AVX-NEXT:    retq
+  %extend = shufflevector <2 x i64> %x, <2 x i64> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+  %bogus_bo = and <4 x i64> %extend, <i64 undef, i64 undef, i64 42, i64 43>
+  %arbitrary_shuf = shufflevector <4 x i64> %bogus_bo, <4 x i64> undef, <4 x i32> <i32 3, i32 0, i32 1, i32 2>
+  ret <4 x i64> %arbitrary_shuf
+}
+
+; or undef, C --> -1, so this tests that we are tracking known all-ones lanes.
+
+define <4 x i64> @or_undef_elts(<2 x i64> %x) {
+; SSE-LABEL: or_undef_elts:
+; SSE:       # %bb.0:
+; SSE-NEXT:    pcmpeqd %xmm0, %xmm0
+; SSE-NEXT:    pcmpeqd %xmm1, %xmm1
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: or_undef_elts:
+; AVX:       # %bb.0:
+; AVX-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
+; AVX-NEXT:    vorps {{.*}}(%rip), %ymm0, %ymm0
+; AVX-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[3,0,1,2]
+; AVX-NEXT:    retq
+  %extend = shufflevector <2 x i64> %x, <2 x i64> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+  %bogus_bo = or <4 x i64> %extend, <i64 undef, i64 undef, i64 42, i64 43>
+  %arbitrary_shuf = shufflevector <4 x i64> %bogus_bo, <4 x i64> undef, <4 x i32> <i32 3, i32 0, i32 1, i32 2>
+  ret <4 x i64> %arbitrary_shuf
+}
+
+; Verify that this isn't limited to high/low halves.
+
+define <8 x i32> @xor_undef_elts(<4 x i32> %x) {
+; SSE-LABEL: xor_undef_elts:
+; SSE:       # %bb.0:
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: xor_undef_elts:
+; AVX:       # %bb.0:
+; AVX-NEXT:    retq
+  %extend = shufflevector <4 x i32> %x, <4 x i32> undef, <8 x i32> <i32 undef, i32 undef, i32 1, i32 3, i32 0, i32 2, i32 undef, i32 undef>
+  %bogus_bo = xor <8 x i32> %extend, <i32 42, i32 43, i32 undef, i32 undef, i32 undef, i32 undef, i32 44, i32 12>
+  %arbitrary_shuf = shufflevector <8 x i32> %bogus_bo, <8 x i32> undef, <8 x i32> <i32 6, i32 1, i32 5, i32 4, i32 3, i32 2, i32 0, i32 7>
+  ret <8 x i32> %arbitrary_shuf
+}
+
+; Verify that this isn't limited to high/low halves
+; Special case: the undef-ness of the 1st shuffle may be lost if we turn that into vector concat.
+
+define <8 x i32> @xor_undef_elts_alt(<4 x i32> %x) {
+; SSE-LABEL: xor_undef_elts_alt:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movaps %xmm0, %xmm1
+; SSE-NEXT:    movaps {{.*#+}} xmm2 = <u,u,44,12>
+; SSE-NEXT:    xorps %xmm0, %xmm2
+; SSE-NEXT:    xorps {{.*}}(%rip), %xmm1
+; SSE-NEXT:    movaps %xmm1, %xmm0
+; SSE-NEXT:    shufps {{.*#+}} xmm0 = xmm0[1,0],xmm2[2,0]
+; SSE-NEXT:    shufps {{.*#+}} xmm0 = xmm0[2,0],xmm2[1,0]
+; SSE-NEXT:    shufps {{.*#+}} xmm2 = xmm2[3,0],xmm1[0,0]
+; SSE-NEXT:    shufps {{.*#+}} xmm1 = xmm1[3,2],xmm2[2,0]
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: xor_undef_elts_alt:
+; AVX:       # %bb.0:
+; AVX-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
+; AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
+; AVX-NEXT:    vxorps {{.*}}(%rip), %ymm0, %ymm0
+; AVX-NEXT:    vmovaps {{.*#+}} ymm1 = [6,1,5,4,3,2,0,7]
+; AVX-NEXT:    vpermps %ymm0, %ymm1, %ymm0
+; AVX-NEXT:    retq
+  %extend = shufflevector <4 x i32> %x, <4 x i32> undef, <8 x i32> <i32 undef, i32 undef, i32 2, i32 3, i32 0, i32 1, i32 undef, i32 undef>
+  %bogus_bo = xor <8 x i32> %extend, <i32 42, i32 43, i32 undef, i32 undef, i32 undef, i32 undef, i32 44, i32 12>
+  %arbitrary_shuf = shufflevector <8 x i32> %bogus_bo, <8 x i32> undef, <8 x i32> <i32 6, i32 1, i32 5, i32 4, i32 3, i32 2, i32 0, i32 7>
+  ret <8 x i32> %arbitrary_shuf
+}
+
diff --git a/test/CodeGen/X86/vector-reduce-add-widen.ll b/test/CodeGen/X86/vector-reduce-add-widen.ll
index 397c533e2b2712077520cc2701a695cae92475cc..a59eabb177ec5baa0f3c1450d37b744554506782 100644
--- a/test/CodeGen/X86/vector-reduce-add-widen.ll
+++ b/test/CodeGen/X86/vector-reduce-add-widen.ll
@@ -57,7 +57,7 @@ define i64 @test_v4i64(<4 x i64> %a0) {
 ; AVX2-LABEL: test_v4i64:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpaddq %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; AVX2-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovq %xmm0, %rax
@@ -67,7 +67,7 @@ define i64 @test_v4i64(<4 x i64> %a0) {
 ; AVX512-LABEL: test_v4i64:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vpaddq %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; AVX512-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovq %xmm0, %rax
@@ -105,7 +105,7 @@ define i64 @test_v8i64(<8 x i64> %a0) {
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpaddq %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpaddq %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; AVX2-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovq %xmm0, %rax
@@ -117,7 +117,7 @@ define i64 @test_v8i64(<8 x i64> %a0) {
 ; AVX512-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512-NEXT:    vpaddq %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vpaddq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; AVX512-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovq %xmm0, %rax
@@ -167,7 +167,7 @@ define i64 @test_v16i64(<16 x i64> %a0) {
 ; AVX2-NEXT:    vpaddq %ymm1, %ymm2, %ymm1
 ; AVX2-NEXT:    vpaddq %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpaddq %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; AVX2-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovq %xmm0, %rax
@@ -180,7 +180,7 @@ define i64 @test_v16i64(<16 x i64> %a0) {
 ; AVX512-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512-NEXT:    vpaddq %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vpaddq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; AVX512-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovq %xmm0, %rax
@@ -276,9 +276,9 @@ define i32 @test_v8i32(<8 x i32> %a0) {
 ; AVX2-LABEL: test_v8i32:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpaddd %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX2-NEXT:    vpaddd %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
@@ -288,9 +288,9 @@ define i32 @test_v8i32(<8 x i32> %a0) {
 ; AVX512-LABEL: test_v8i32:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vpaddd %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpaddd %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
@@ -332,9 +332,9 @@ define i32 @test_v16i32(<16 x i32> %a0) {
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpaddd %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpaddd %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX2-NEXT:    vpaddd %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
@@ -346,9 +346,9 @@ define i32 @test_v16i32(<16 x i32> %a0) {
 ; AVX512-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
@@ -402,9 +402,9 @@ define i32 @test_v32i32(<32 x i32> %a0) {
 ; AVX2-NEXT:    vpaddd %ymm1, %ymm2, %ymm1
 ; AVX2-NEXT:    vpaddd %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpaddd %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX2-NEXT:    vpaddd %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
@@ -417,9 +417,9 @@ define i32 @test_v32i32(<32 x i32> %a0) {
 ; AVX512-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
@@ -571,11 +571,11 @@ define i16 @test_v16i16(<16 x i16> %a0) {
 ; AVX2-LABEL: test_v16i16:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpaddw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX2-NEXT:    vpaddw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX2-NEXT:    vpaddw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX2-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
@@ -586,11 +586,11 @@ define i16 @test_v16i16(<16 x i16> %a0) {
 ; AVX512-LABEL: test_v16i16:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vpaddw %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpaddw %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX512-NEXT:    vpaddw %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX512-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
@@ -640,11 +640,11 @@ define i16 @test_v32i16(<32 x i16> %a0) {
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpaddw %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpaddw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX2-NEXT:    vpaddw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX2-NEXT:    vpaddw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX2-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
@@ -657,11 +657,11 @@ define i16 @test_v32i16(<32 x i16> %a0) {
 ; AVX512-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512-NEXT:    vpaddw %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vpaddw %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpaddw %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX512-NEXT:    vpaddw %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX512-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
@@ -723,11 +723,11 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 ; AVX2-NEXT:    vpaddw %ymm1, %ymm2, %ymm1
 ; AVX2-NEXT:    vpaddw %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpaddw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX2-NEXT:    vpaddw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX2-NEXT:    vpaddw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX2-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
@@ -741,11 +741,11 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 ; AVX512-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512-NEXT:    vpaddw %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vpaddw %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpaddw %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX512-NEXT:    vpaddw %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX512-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
@@ -1021,13 +1021,13 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; AVX2-LABEL: test_v32i8:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpaddb %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX2-NEXT:    vpaddb %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX2-NEXT:    vpaddb %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX2-NEXT:    vpaddb %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpsrlw $8, %xmm0, %xmm1
 ; AVX2-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpextrb $0, %xmm0, %eax
@@ -1038,13 +1038,13 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; AVX512-LABEL: test_v32i8:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vpaddb %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpaddb %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX512-NEXT:    vpaddb %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX512-NEXT:    vpaddb %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrlw $8, %xmm0, %xmm1
 ; AVX512-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpextrb $0, %xmm0, %eax
@@ -1118,13 +1118,13 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpaddb %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpaddb %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX2-NEXT:    vpaddb %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX2-NEXT:    vpaddb %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX2-NEXT:    vpaddb %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpsrlw $8, %xmm0, %xmm1
 ; AVX2-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpextrb $0, %xmm0, %eax
@@ -1137,13 +1137,13 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; AVX512-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512-NEXT:    vpaddb %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vpaddb %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpaddb %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX512-NEXT:    vpaddb %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX512-NEXT:    vpaddb %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrlw $8, %xmm0, %xmm1
 ; AVX512-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpextrb $0, %xmm0, %eax
@@ -1233,13 +1233,13 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; AVX2-NEXT:    vpaddb %ymm1, %ymm2, %ymm1
 ; AVX2-NEXT:    vpaddb %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpaddb %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX2-NEXT:    vpaddb %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX2-NEXT:    vpaddb %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX2-NEXT:    vpaddb %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpsrlw $8, %xmm0, %xmm1
 ; AVX2-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpextrb $0, %xmm0, %eax
@@ -1253,13 +1253,13 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; AVX512-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512-NEXT:    vpaddb %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vpaddb %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpaddb %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX512-NEXT:    vpaddb %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX512-NEXT:    vpaddb %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrlw $8, %xmm0, %xmm1
 ; AVX512-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpextrb $0, %xmm0, %eax
diff --git a/test/CodeGen/X86/vector-reduce-add.ll b/test/CodeGen/X86/vector-reduce-add.ll
index 5d92c9a62d29486d1b015eb3dc1042e93c4b35e2..4c98680dc4fcb18ae44401400179340dcbdfd391 100644
--- a/test/CodeGen/X86/vector-reduce-add.ll
+++ b/test/CodeGen/X86/vector-reduce-add.ll
@@ -57,7 +57,7 @@ define i64 @test_v4i64(<4 x i64> %a0) {
 ; AVX2-LABEL: test_v4i64:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpaddq %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; AVX2-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovq %xmm0, %rax
@@ -67,7 +67,7 @@ define i64 @test_v4i64(<4 x i64> %a0) {
 ; AVX512-LABEL: test_v4i64:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vpaddq %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; AVX512-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovq %xmm0, %rax
@@ -105,7 +105,7 @@ define i64 @test_v8i64(<8 x i64> %a0) {
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpaddq %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpaddq %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; AVX2-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovq %xmm0, %rax
@@ -117,7 +117,7 @@ define i64 @test_v8i64(<8 x i64> %a0) {
 ; AVX512-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512-NEXT:    vpaddq %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vpaddq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; AVX512-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovq %xmm0, %rax
@@ -167,7 +167,7 @@ define i64 @test_v16i64(<16 x i64> %a0) {
 ; AVX2-NEXT:    vpaddq %ymm1, %ymm2, %ymm1
 ; AVX2-NEXT:    vpaddq %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpaddq %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; AVX2-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovq %xmm0, %rax
@@ -180,7 +180,7 @@ define i64 @test_v16i64(<16 x i64> %a0) {
 ; AVX512-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512-NEXT:    vpaddq %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vpaddq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; AVX512-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovq %xmm0, %rax
@@ -276,9 +276,9 @@ define i32 @test_v8i32(<8 x i32> %a0) {
 ; AVX2-LABEL: test_v8i32:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpaddd %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX2-NEXT:    vpaddd %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
@@ -288,9 +288,9 @@ define i32 @test_v8i32(<8 x i32> %a0) {
 ; AVX512-LABEL: test_v8i32:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vpaddd %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpaddd %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
@@ -332,9 +332,9 @@ define i32 @test_v16i32(<16 x i32> %a0) {
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpaddd %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpaddd %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX2-NEXT:    vpaddd %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
@@ -346,9 +346,9 @@ define i32 @test_v16i32(<16 x i32> %a0) {
 ; AVX512-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
@@ -402,9 +402,9 @@ define i32 @test_v32i32(<32 x i32> %a0) {
 ; AVX2-NEXT:    vpaddd %ymm1, %ymm2, %ymm1
 ; AVX2-NEXT:    vpaddd %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpaddd %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX2-NEXT:    vpaddd %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
@@ -417,9 +417,9 @@ define i32 @test_v32i32(<32 x i32> %a0) {
 ; AVX512-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
@@ -569,11 +569,11 @@ define i16 @test_v16i16(<16 x i16> %a0) {
 ; AVX2-LABEL: test_v16i16:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpaddw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX2-NEXT:    vpaddw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX2-NEXT:    vpaddw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX2-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
@@ -584,11 +584,11 @@ define i16 @test_v16i16(<16 x i16> %a0) {
 ; AVX512-LABEL: test_v16i16:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vpaddw %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpaddw %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX512-NEXT:    vpaddw %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX512-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
@@ -638,11 +638,11 @@ define i16 @test_v32i16(<32 x i16> %a0) {
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpaddw %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpaddw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX2-NEXT:    vpaddw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX2-NEXT:    vpaddw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX2-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
@@ -655,11 +655,11 @@ define i16 @test_v32i16(<32 x i16> %a0) {
 ; AVX512-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512-NEXT:    vpaddw %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vpaddw %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpaddw %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX512-NEXT:    vpaddw %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX512-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
@@ -721,11 +721,11 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 ; AVX2-NEXT:    vpaddw %ymm1, %ymm2, %ymm1
 ; AVX2-NEXT:    vpaddw %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpaddw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX2-NEXT:    vpaddw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX2-NEXT:    vpaddw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX2-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
@@ -739,11 +739,11 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 ; AVX512-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512-NEXT:    vpaddw %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vpaddw %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpaddw %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX512-NEXT:    vpaddw %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX512-NEXT:    vpaddw %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
@@ -1011,13 +1011,13 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; AVX2-LABEL: test_v32i8:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpaddb %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX2-NEXT:    vpaddb %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX2-NEXT:    vpaddb %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX2-NEXT:    vpaddb %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpsrlw $8, %xmm0, %xmm1
 ; AVX2-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpextrb $0, %xmm0, %eax
@@ -1028,13 +1028,13 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; AVX512-LABEL: test_v32i8:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vpaddb %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpaddb %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX512-NEXT:    vpaddb %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX512-NEXT:    vpaddb %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrlw $8, %xmm0, %xmm1
 ; AVX512-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpextrb $0, %xmm0, %eax
@@ -1108,13 +1108,13 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpaddb %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpaddb %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX2-NEXT:    vpaddb %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX2-NEXT:    vpaddb %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX2-NEXT:    vpaddb %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpsrlw $8, %xmm0, %xmm1
 ; AVX2-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpextrb $0, %xmm0, %eax
@@ -1127,13 +1127,13 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; AVX512-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512-NEXT:    vpaddb %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vpaddb %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpaddb %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX512-NEXT:    vpaddb %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX512-NEXT:    vpaddb %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrlw $8, %xmm0, %xmm1
 ; AVX512-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpextrb $0, %xmm0, %eax
@@ -1223,13 +1223,13 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; AVX2-NEXT:    vpaddb %ymm1, %ymm2, %ymm1
 ; AVX2-NEXT:    vpaddb %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpaddb %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX2-NEXT:    vpaddb %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX2-NEXT:    vpaddb %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX2-NEXT:    vpaddb %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpsrlw $8, %xmm0, %xmm1
 ; AVX2-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpextrb $0, %xmm0, %eax
@@ -1243,13 +1243,13 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; AVX512-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512-NEXT:    vpaddb %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vpaddb %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpaddb %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX512-NEXT:    vpaddb %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX512-NEXT:    vpaddb %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrlw $8, %xmm0, %xmm1
 ; AVX512-NEXT:    vpaddb %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpextrb $0, %xmm0, %eax
diff --git a/test/CodeGen/X86/vector-reduce-and-widen.ll b/test/CodeGen/X86/vector-reduce-and-widen.ll
index 13415561cf757ced0702e4ddc0eb4e01a5cad860..a00f81cc63d17603d85ea242bec915b9eb2c082f 100644
--- a/test/CodeGen/X86/vector-reduce-and-widen.ll
+++ b/test/CodeGen/X86/vector-reduce-and-widen.ll
@@ -47,9 +47,9 @@ define i64 @test_v4i64(<4 x i64> %a0) {
 ; AVX1-LABEL: test_v4i64:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vandps %ymm1, %ymm0, %ymm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovq %xmm0, %rax
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
@@ -57,7 +57,7 @@ define i64 @test_v4i64(<4 x i64> %a0) {
 ; AVX2-LABEL: test_v4i64:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovq %xmm0, %rax
@@ -67,7 +67,7 @@ define i64 @test_v4i64(<4 x i64> %a0) {
 ; AVX512-LABEL: test_v4i64:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vpand %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovq %xmm0, %rax
@@ -92,7 +92,7 @@ define i64 @test_v8i64(<8 x i64> %a0) {
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vandps %ymm1, %ymm0, %ymm0
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vandps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovq %xmm0, %rax
@@ -103,7 +103,7 @@ define i64 @test_v8i64(<8 x i64> %a0) {
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovq %xmm0, %rax
@@ -115,7 +115,7 @@ define i64 @test_v8i64(<8 x i64> %a0) {
 ; AVX512-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512-NEXT:    vpandq %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vpandq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovq %xmm0, %rax
@@ -146,7 +146,7 @@ define i64 @test_v16i64(<16 x i64> %a0) {
 ; AVX1-NEXT:    vandps %ymm1, %ymm2, %ymm1
 ; AVX1-NEXT:    vandps %ymm1, %ymm0, %ymm0
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vandps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovq %xmm0, %rax
@@ -159,7 +159,7 @@ define i64 @test_v16i64(<16 x i64> %a0) {
 ; AVX2-NEXT:    vpand %ymm1, %ymm2, %ymm1
 ; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovq %xmm0, %rax
@@ -172,7 +172,7 @@ define i64 @test_v16i64(<16 x i64> %a0) {
 ; AVX512-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512-NEXT:    vpandq %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vpandq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovq %xmm0, %rax
@@ -256,11 +256,11 @@ define i32 @test_v8i32(<8 x i32> %a0) {
 ; AVX1-LABEL: test_v8i32:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vandps %ymm1, %ymm0, %ymm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX1-NEXT:    vandps %ymm1, %ymm0, %ymm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovd %xmm0, %eax
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
@@ -268,9 +268,9 @@ define i32 @test_v8i32(<8 x i32> %a0) {
 ; AVX2-LABEL: test_v8i32:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
@@ -280,9 +280,9 @@ define i32 @test_v8i32(<8 x i32> %a0) {
 ; AVX512-LABEL: test_v8i32:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vpand %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpand %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
@@ -309,9 +309,9 @@ define i32 @test_v16i32(<16 x i32> %a0) {
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vandps %ymm1, %ymm0, %ymm0
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vandps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX1-NEXT:    vandps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovd %xmm0, %eax
@@ -322,9 +322,9 @@ define i32 @test_v16i32(<16 x i32> %a0) {
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
@@ -336,9 +336,9 @@ define i32 @test_v16i32(<16 x i32> %a0) {
 ; AVX512-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512-NEXT:    vpandd %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vpandd %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpandd %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
@@ -371,9 +371,9 @@ define i32 @test_v32i32(<32 x i32> %a0) {
 ; AVX1-NEXT:    vandps %ymm1, %ymm2, %ymm1
 ; AVX1-NEXT:    vandps %ymm1, %ymm0, %ymm0
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vandps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX1-NEXT:    vandps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovd %xmm0, %eax
@@ -386,9 +386,9 @@ define i32 @test_v32i32(<32 x i32> %a0) {
 ; AVX2-NEXT:    vpand %ymm1, %ymm2, %ymm1
 ; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
@@ -401,9 +401,9 @@ define i32 @test_v32i32(<32 x i32> %a0) {
 ; AVX512-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512-NEXT:    vpandd %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vpandd %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpandd %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
@@ -540,11 +540,11 @@ define i16 @test_v16i16(<16 x i16> %a0) {
 ; AVX1-LABEL: test_v16i16:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vandps %ymm1, %ymm0, %ymm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX1-NEXT:    vandps %ymm1, %ymm0, %ymm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX1-NEXT:    vandps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovd %xmm0, %eax
@@ -555,11 +555,11 @@ define i16 @test_v16i16(<16 x i16> %a0) {
 ; AVX2-LABEL: test_v16i16:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
@@ -570,11 +570,11 @@ define i16 @test_v16i16(<16 x i16> %a0) {
 ; AVX512-LABEL: test_v16i16:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vpand %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpand %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX512-NEXT:    vpand %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
@@ -606,11 +606,11 @@ define i16 @test_v32i16(<32 x i16> %a0) {
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vandps %ymm1, %ymm0, %ymm0
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vandps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX1-NEXT:    vandps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX1-NEXT:    vandps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovd %xmm0, %eax
@@ -622,11 +622,11 @@ define i16 @test_v32i16(<32 x i16> %a0) {
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
@@ -639,11 +639,11 @@ define i16 @test_v32i16(<32 x i16> %a0) {
 ; AVX512-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512-NEXT:    vpandq %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vpandq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpandq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX512-NEXT:    vpandq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
@@ -681,11 +681,11 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 ; AVX1-NEXT:    vandps %ymm1, %ymm2, %ymm1
 ; AVX1-NEXT:    vandps %ymm1, %ymm0, %ymm0
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vandps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX1-NEXT:    vandps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX1-NEXT:    vandps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovd %xmm0, %eax
@@ -699,11 +699,11 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 ; AVX2-NEXT:    vpand %ymm1, %ymm2, %ymm1
 ; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
@@ -717,11 +717,11 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 ; AVX512-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512-NEXT:    vpandq %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vpandq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpandq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX512-NEXT:    vpandq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
@@ -980,13 +980,13 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; AVX1-LABEL: test_v32i8:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vandps %ymm1, %ymm0, %ymm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX1-NEXT:    vandps %ymm1, %ymm0, %ymm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX1-NEXT:    vandps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX1-NEXT:    vandps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm1
 ; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpextrb $0, %xmm0, %eax
@@ -997,13 +997,13 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; AVX2-LABEL: test_v32i8:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpsrlw $8, %xmm0, %xmm1
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpextrb $0, %xmm0, %eax
@@ -1014,13 +1014,13 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; AVX512-LABEL: test_v32i8:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vpand %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpand %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX512-NEXT:    vpand %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX512-NEXT:    vpand %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrlw $8, %xmm0, %xmm1
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpextrb $0, %xmm0, %eax
@@ -1074,13 +1074,13 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vandps %ymm1, %ymm0, %ymm0
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vandps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX1-NEXT:    vandps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX1-NEXT:    vandps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX1-NEXT:    vandps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm1
 ; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpextrb $0, %xmm0, %eax
@@ -1092,13 +1092,13 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpsrlw $8, %xmm0, %xmm1
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpextrb $0, %xmm0, %eax
@@ -1111,13 +1111,13 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; AVX512-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512-NEXT:    vpandq %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vpandq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpandq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX512-NEXT:    vpandq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX512-NEXT:    vpandq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrlw $8, %xmm0, %xmm1
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpextrb $0, %xmm0, %eax
@@ -1181,13 +1181,13 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; AVX1-NEXT:    vandps %ymm1, %ymm2, %ymm1
 ; AVX1-NEXT:    vandps %ymm1, %ymm0, %ymm0
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vandps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX1-NEXT:    vandps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX1-NEXT:    vandps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX1-NEXT:    vandps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm1
 ; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpextrb $0, %xmm0, %eax
@@ -1201,13 +1201,13 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; AVX2-NEXT:    vpand %ymm1, %ymm2, %ymm1
 ; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpsrlw $8, %xmm0, %xmm1
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpextrb $0, %xmm0, %eax
@@ -1221,13 +1221,13 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; AVX512-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512-NEXT:    vpandq %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vpandq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpandq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX512-NEXT:    vpandq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX512-NEXT:    vpandq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrlw $8, %xmm0, %xmm1
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpextrb $0, %xmm0, %eax
diff --git a/test/CodeGen/X86/vector-reduce-and.ll b/test/CodeGen/X86/vector-reduce-and.ll
index a3448f545ff06d9cbbe31184f0eb7dafa8021f07..e90307b9841cf3ebed94d31dcf36a7fd94944d5e 100644
--- a/test/CodeGen/X86/vector-reduce-and.ll
+++ b/test/CodeGen/X86/vector-reduce-and.ll
@@ -47,9 +47,9 @@ define i64 @test_v4i64(<4 x i64> %a0) {
 ; AVX1-LABEL: test_v4i64:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vandps %ymm1, %ymm0, %ymm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovq %xmm0, %rax
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
@@ -57,7 +57,7 @@ define i64 @test_v4i64(<4 x i64> %a0) {
 ; AVX2-LABEL: test_v4i64:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovq %xmm0, %rax
@@ -67,7 +67,7 @@ define i64 @test_v4i64(<4 x i64> %a0) {
 ; AVX512-LABEL: test_v4i64:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vpand %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovq %xmm0, %rax
@@ -92,7 +92,7 @@ define i64 @test_v8i64(<8 x i64> %a0) {
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vandps %ymm1, %ymm0, %ymm0
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vandps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovq %xmm0, %rax
@@ -103,7 +103,7 @@ define i64 @test_v8i64(<8 x i64> %a0) {
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovq %xmm0, %rax
@@ -115,7 +115,7 @@ define i64 @test_v8i64(<8 x i64> %a0) {
 ; AVX512-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512-NEXT:    vpandq %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vpandq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovq %xmm0, %rax
@@ -146,7 +146,7 @@ define i64 @test_v16i64(<16 x i64> %a0) {
 ; AVX1-NEXT:    vandps %ymm1, %ymm2, %ymm1
 ; AVX1-NEXT:    vandps %ymm1, %ymm0, %ymm0
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vandps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovq %xmm0, %rax
@@ -159,7 +159,7 @@ define i64 @test_v16i64(<16 x i64> %a0) {
 ; AVX2-NEXT:    vpand %ymm1, %ymm2, %ymm1
 ; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovq %xmm0, %rax
@@ -172,7 +172,7 @@ define i64 @test_v16i64(<16 x i64> %a0) {
 ; AVX512-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512-NEXT:    vpandq %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vpandq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovq %xmm0, %rax
@@ -256,11 +256,11 @@ define i32 @test_v8i32(<8 x i32> %a0) {
 ; AVX1-LABEL: test_v8i32:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vandps %ymm1, %ymm0, %ymm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX1-NEXT:    vandps %ymm1, %ymm0, %ymm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovd %xmm0, %eax
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
@@ -268,9 +268,9 @@ define i32 @test_v8i32(<8 x i32> %a0) {
 ; AVX2-LABEL: test_v8i32:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
@@ -280,9 +280,9 @@ define i32 @test_v8i32(<8 x i32> %a0) {
 ; AVX512-LABEL: test_v8i32:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vpand %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpand %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
@@ -309,9 +309,9 @@ define i32 @test_v16i32(<16 x i32> %a0) {
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vandps %ymm1, %ymm0, %ymm0
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vandps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX1-NEXT:    vandps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovd %xmm0, %eax
@@ -322,9 +322,9 @@ define i32 @test_v16i32(<16 x i32> %a0) {
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
@@ -336,9 +336,9 @@ define i32 @test_v16i32(<16 x i32> %a0) {
 ; AVX512-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512-NEXT:    vpandd %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vpandd %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpandd %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
@@ -371,9 +371,9 @@ define i32 @test_v32i32(<32 x i32> %a0) {
 ; AVX1-NEXT:    vandps %ymm1, %ymm2, %ymm1
 ; AVX1-NEXT:    vandps %ymm1, %ymm0, %ymm0
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vandps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX1-NEXT:    vandps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovd %xmm0, %eax
@@ -386,9 +386,9 @@ define i32 @test_v32i32(<32 x i32> %a0) {
 ; AVX2-NEXT:    vpand %ymm1, %ymm2, %ymm1
 ; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
@@ -401,9 +401,9 @@ define i32 @test_v32i32(<32 x i32> %a0) {
 ; AVX512-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512-NEXT:    vpandd %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vpandd %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpandd %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
@@ -538,11 +538,11 @@ define i16 @test_v16i16(<16 x i16> %a0) {
 ; AVX1-LABEL: test_v16i16:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vandps %ymm1, %ymm0, %ymm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX1-NEXT:    vandps %ymm1, %ymm0, %ymm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX1-NEXT:    vandps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovd %xmm0, %eax
@@ -553,11 +553,11 @@ define i16 @test_v16i16(<16 x i16> %a0) {
 ; AVX2-LABEL: test_v16i16:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
@@ -568,11 +568,11 @@ define i16 @test_v16i16(<16 x i16> %a0) {
 ; AVX512-LABEL: test_v16i16:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vpand %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpand %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX512-NEXT:    vpand %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
@@ -604,11 +604,11 @@ define i16 @test_v32i16(<32 x i16> %a0) {
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vandps %ymm1, %ymm0, %ymm0
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vandps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX1-NEXT:    vandps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX1-NEXT:    vandps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovd %xmm0, %eax
@@ -620,11 +620,11 @@ define i16 @test_v32i16(<32 x i16> %a0) {
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
@@ -637,11 +637,11 @@ define i16 @test_v32i16(<32 x i16> %a0) {
 ; AVX512-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512-NEXT:    vpandq %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vpandq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpandq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX512-NEXT:    vpandq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
@@ -679,11 +679,11 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 ; AVX1-NEXT:    vandps %ymm1, %ymm2, %ymm1
 ; AVX1-NEXT:    vandps %ymm1, %ymm0, %ymm0
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vandps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX1-NEXT:    vandps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX1-NEXT:    vandps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovd %xmm0, %eax
@@ -697,11 +697,11 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 ; AVX2-NEXT:    vpand %ymm1, %ymm2, %ymm1
 ; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
@@ -715,11 +715,11 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 ; AVX512-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512-NEXT:    vpandq %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vpandq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpandq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX512-NEXT:    vpandq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
@@ -970,13 +970,13 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; AVX1-LABEL: test_v32i8:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vandps %ymm1, %ymm0, %ymm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX1-NEXT:    vandps %ymm1, %ymm0, %ymm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX1-NEXT:    vandps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX1-NEXT:    vandps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm1
 ; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpextrb $0, %xmm0, %eax
@@ -987,13 +987,13 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; AVX2-LABEL: test_v32i8:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpsrlw $8, %xmm0, %xmm1
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpextrb $0, %xmm0, %eax
@@ -1004,13 +1004,13 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; AVX512-LABEL: test_v32i8:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vpand %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpand %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX512-NEXT:    vpand %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX512-NEXT:    vpand %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrlw $8, %xmm0, %xmm1
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpextrb $0, %xmm0, %eax
@@ -1064,13 +1064,13 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vandps %ymm1, %ymm0, %ymm0
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vandps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX1-NEXT:    vandps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX1-NEXT:    vandps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX1-NEXT:    vandps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm1
 ; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpextrb $0, %xmm0, %eax
@@ -1082,13 +1082,13 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpsrlw $8, %xmm0, %xmm1
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpextrb $0, %xmm0, %eax
@@ -1101,13 +1101,13 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; AVX512-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512-NEXT:    vpandq %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vpandq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpandq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX512-NEXT:    vpandq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX512-NEXT:    vpandq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrlw $8, %xmm0, %xmm1
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpextrb $0, %xmm0, %eax
@@ -1171,13 +1171,13 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; AVX1-NEXT:    vandps %ymm1, %ymm2, %ymm1
 ; AVX1-NEXT:    vandps %ymm1, %ymm0, %ymm0
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vandps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX1-NEXT:    vandps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX1-NEXT:    vandps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vandps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX1-NEXT:    vandps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm1
 ; AVX1-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpextrb $0, %xmm0, %eax
@@ -1191,13 +1191,13 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; AVX2-NEXT:    vpand %ymm1, %ymm2, %ymm1
 ; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpsrlw $8, %xmm0, %xmm1
 ; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpextrb $0, %xmm0, %eax
@@ -1211,13 +1211,13 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; AVX512-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512-NEXT:    vpandq %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vpandq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpandq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX512-NEXT:    vpandq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX512-NEXT:    vpandq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrlw $8, %xmm0, %xmm1
 ; AVX512-NEXT:    vpand %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpextrb $0, %xmm0, %eax
diff --git a/test/CodeGen/X86/vector-reduce-fadd-fast.ll b/test/CodeGen/X86/vector-reduce-fadd-fast.ll
index 9dadf969a0edf284f14d94633d9ea89aaa895ca2..82ec7adfab3e78020db7eca0571a18c94d11737f 100644
--- a/test/CodeGen/X86/vector-reduce-fadd-fast.ll
+++ b/test/CodeGen/X86/vector-reduce-fadd-fast.ll
@@ -103,9 +103,9 @@ define float @test_v8f32(float %a0, <8 x float> %a1) {
 ; AVX-LABEL: test_v8f32:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vextractf128 $1, %ymm1, %xmm0
-; AVX-NEXT:    vaddps %ymm0, %ymm1, %ymm0
+; AVX-NEXT:    vaddps %xmm0, %xmm1, %xmm0
 ; AVX-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
-; AVX-NEXT:    vaddps %ymm1, %ymm0, %ymm0
+; AVX-NEXT:    vaddps %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
 ; AVX-NEXT:    vaddps %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vzeroupper
@@ -114,9 +114,9 @@ define float @test_v8f32(float %a0, <8 x float> %a1) {
 ; AVX512-LABEL: test_v8f32:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vextractf128 $1, %ymm1, %xmm0
-; AVX512-NEXT:    vaddps %ymm0, %ymm1, %ymm0
+; AVX512-NEXT:    vaddps %xmm0, %xmm1, %xmm0
 ; AVX512-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
-; AVX512-NEXT:    vaddps %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vaddps %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
 ; AVX512-NEXT:    vaddps %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vzeroupper
@@ -155,9 +155,9 @@ define float @test_v16f32(float %a0, <16 x float> %a1) {
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vaddps %ymm2, %ymm1, %ymm0
 ; AVX-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX-NEXT:    vaddps %ymm1, %ymm0, %ymm0
+; AVX-NEXT:    vaddps %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
-; AVX-NEXT:    vaddps %ymm1, %ymm0, %ymm0
+; AVX-NEXT:    vaddps %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
 ; AVX-NEXT:    vaddps %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vzeroupper
@@ -168,9 +168,9 @@ define float @test_v16f32(float %a0, <16 x float> %a1) {
 ; AVX512-NEXT:    vextractf64x4 $1, %zmm1, %ymm0
 ; AVX512-NEXT:    vaddps %zmm0, %zmm1, %zmm0
 ; AVX512-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vaddps %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vaddps %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
-; AVX512-NEXT:    vaddps %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vaddps %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
 ; AVX512-NEXT:    vaddps %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vzeroupper
@@ -279,9 +279,9 @@ define float @test_v8f32_zero(<8 x float> %a0) {
 ; AVX-LABEL: test_v8f32_zero:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX-NEXT:    vaddps %ymm1, %ymm0, %ymm0
+; AVX-NEXT:    vaddps %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
-; AVX-NEXT:    vaddps %ymm1, %ymm0, %ymm0
+; AVX-NEXT:    vaddps %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
 ; AVX-NEXT:    vaddps %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vzeroupper
@@ -290,9 +290,9 @@ define float @test_v8f32_zero(<8 x float> %a0) {
 ; AVX512-LABEL: test_v8f32_zero:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vaddps %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vaddps %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
-; AVX512-NEXT:    vaddps %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vaddps %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
 ; AVX512-NEXT:    vaddps %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vzeroupper
@@ -332,9 +332,9 @@ define float @test_v16f32_zero(<16 x float> %a0) {
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vaddps %ymm1, %ymm0, %ymm0
 ; AVX-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX-NEXT:    vaddps %ymm1, %ymm0, %ymm0
+; AVX-NEXT:    vaddps %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
-; AVX-NEXT:    vaddps %ymm1, %ymm0, %ymm0
+; AVX-NEXT:    vaddps %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
 ; AVX-NEXT:    vaddps %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vzeroupper
@@ -345,9 +345,9 @@ define float @test_v16f32_zero(<16 x float> %a0) {
 ; AVX512-NEXT:    vextractf64x4 $1, %zmm0, %ymm1
 ; AVX512-NEXT:    vaddps %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vaddps %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vaddps %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
-; AVX512-NEXT:    vaddps %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vaddps %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
 ; AVX512-NEXT:    vaddps %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vzeroupper
@@ -456,9 +456,9 @@ define float @test_v8f32_undef(<8 x float> %a0) {
 ; AVX-LABEL: test_v8f32_undef:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX-NEXT:    vaddps %ymm1, %ymm0, %ymm0
+; AVX-NEXT:    vaddps %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
-; AVX-NEXT:    vaddps %ymm1, %ymm0, %ymm0
+; AVX-NEXT:    vaddps %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
 ; AVX-NEXT:    vaddps %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vzeroupper
@@ -467,9 +467,9 @@ define float @test_v8f32_undef(<8 x float> %a0) {
 ; AVX512-LABEL: test_v8f32_undef:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vaddps %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vaddps %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
-; AVX512-NEXT:    vaddps %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vaddps %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
 ; AVX512-NEXT:    vaddps %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vzeroupper
@@ -509,9 +509,9 @@ define float @test_v16f32_undef(<16 x float> %a0) {
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vaddps %ymm1, %ymm0, %ymm0
 ; AVX-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX-NEXT:    vaddps %ymm1, %ymm0, %ymm0
+; AVX-NEXT:    vaddps %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
-; AVX-NEXT:    vaddps %ymm1, %ymm0, %ymm0
+; AVX-NEXT:    vaddps %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
 ; AVX-NEXT:    vaddps %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vzeroupper
@@ -522,9 +522,9 @@ define float @test_v16f32_undef(<16 x float> %a0) {
 ; AVX512-NEXT:    vextractf64x4 $1, %zmm0, %ymm1
 ; AVX512-NEXT:    vaddps %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vaddps %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vaddps %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
-; AVX512-NEXT:    vaddps %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vaddps %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
 ; AVX512-NEXT:    vaddps %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vzeroupper
@@ -572,7 +572,7 @@ define double @test_v4f64(double %a0, <4 x double> %a1) {
 ; AVX-LABEL: test_v4f64:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vextractf128 $1, %ymm1, %xmm0
-; AVX-NEXT:    vaddpd %ymm0, %ymm1, %ymm0
+; AVX-NEXT:    vaddpd %xmm0, %xmm1, %xmm0
 ; AVX-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
 ; AVX-NEXT:    vaddpd %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vzeroupper
@@ -581,7 +581,7 @@ define double @test_v4f64(double %a0, <4 x double> %a1) {
 ; AVX512-LABEL: test_v4f64:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vextractf128 $1, %ymm1, %xmm0
-; AVX512-NEXT:    vaddpd %ymm0, %ymm1, %ymm0
+; AVX512-NEXT:    vaddpd %xmm0, %xmm1, %xmm0
 ; AVX512-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
 ; AVX512-NEXT:    vaddpd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vzeroupper
@@ -605,7 +605,7 @@ define double @test_v8f64(double %a0, <8 x double> %a1) {
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vaddpd %ymm2, %ymm1, %ymm0
 ; AVX-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX-NEXT:    vaddpd %ymm1, %ymm0, %ymm0
+; AVX-NEXT:    vaddpd %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
 ; AVX-NEXT:    vaddpd %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vzeroupper
@@ -616,7 +616,7 @@ define double @test_v8f64(double %a0, <8 x double> %a1) {
 ; AVX512-NEXT:    vextractf64x4 $1, %zmm1, %ymm0
 ; AVX512-NEXT:    vaddpd %zmm0, %zmm1, %zmm0
 ; AVX512-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vaddpd %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vaddpd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
 ; AVX512-NEXT:    vaddpd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vzeroupper
@@ -646,7 +646,7 @@ define double @test_v16f64(double %a0, <16 x double> %a1) {
 ; AVX-NEXT:    vaddpd %ymm3, %ymm1, %ymm1
 ; AVX-NEXT:    vaddpd %ymm0, %ymm1, %ymm0
 ; AVX-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX-NEXT:    vaddpd %ymm1, %ymm0, %ymm0
+; AVX-NEXT:    vaddpd %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
 ; AVX-NEXT:    vaddpd %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vzeroupper
@@ -658,7 +658,7 @@ define double @test_v16f64(double %a0, <16 x double> %a1) {
 ; AVX512-NEXT:    vextractf64x4 $1, %zmm0, %ymm1
 ; AVX512-NEXT:    vaddpd %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vaddpd %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vaddpd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
 ; AVX512-NEXT:    vaddpd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vzeroupper
@@ -708,7 +708,7 @@ define double @test_v4f64_zero(<4 x double> %a0) {
 ; AVX-LABEL: test_v4f64_zero:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX-NEXT:    vaddpd %ymm1, %ymm0, %ymm0
+; AVX-NEXT:    vaddpd %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
 ; AVX-NEXT:    vaddpd %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vzeroupper
@@ -717,7 +717,7 @@ define double @test_v4f64_zero(<4 x double> %a0) {
 ; AVX512-LABEL: test_v4f64_zero:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vaddpd %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vaddpd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
 ; AVX512-NEXT:    vaddpd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vzeroupper
@@ -742,7 +742,7 @@ define double @test_v8f64_zero(<8 x double> %a0) {
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vaddpd %ymm1, %ymm0, %ymm0
 ; AVX-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX-NEXT:    vaddpd %ymm1, %ymm0, %ymm0
+; AVX-NEXT:    vaddpd %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
 ; AVX-NEXT:    vaddpd %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vzeroupper
@@ -753,7 +753,7 @@ define double @test_v8f64_zero(<8 x double> %a0) {
 ; AVX512-NEXT:    vextractf64x4 $1, %zmm0, %ymm1
 ; AVX512-NEXT:    vaddpd %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vaddpd %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vaddpd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
 ; AVX512-NEXT:    vaddpd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vzeroupper
@@ -783,7 +783,7 @@ define double @test_v16f64_zero(<16 x double> %a0) {
 ; AVX-NEXT:    vaddpd %ymm2, %ymm0, %ymm0
 ; AVX-NEXT:    vaddpd %ymm1, %ymm0, %ymm0
 ; AVX-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX-NEXT:    vaddpd %ymm1, %ymm0, %ymm0
+; AVX-NEXT:    vaddpd %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
 ; AVX-NEXT:    vaddpd %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vzeroupper
@@ -795,7 +795,7 @@ define double @test_v16f64_zero(<16 x double> %a0) {
 ; AVX512-NEXT:    vextractf64x4 $1, %zmm0, %ymm1
 ; AVX512-NEXT:    vaddpd %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vaddpd %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vaddpd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
 ; AVX512-NEXT:    vaddpd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vzeroupper
@@ -845,7 +845,7 @@ define double @test_v4f64_undef(<4 x double> %a0) {
 ; AVX-LABEL: test_v4f64_undef:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX-NEXT:    vaddpd %ymm1, %ymm0, %ymm0
+; AVX-NEXT:    vaddpd %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
 ; AVX-NEXT:    vaddpd %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vzeroupper
@@ -854,7 +854,7 @@ define double @test_v4f64_undef(<4 x double> %a0) {
 ; AVX512-LABEL: test_v4f64_undef:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vaddpd %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vaddpd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
 ; AVX512-NEXT:    vaddpd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vzeroupper
@@ -879,7 +879,7 @@ define double @test_v8f64_undef(<8 x double> %a0) {
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vaddpd %ymm1, %ymm0, %ymm0
 ; AVX-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX-NEXT:    vaddpd %ymm1, %ymm0, %ymm0
+; AVX-NEXT:    vaddpd %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
 ; AVX-NEXT:    vaddpd %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vzeroupper
@@ -890,7 +890,7 @@ define double @test_v8f64_undef(<8 x double> %a0) {
 ; AVX512-NEXT:    vextractf64x4 $1, %zmm0, %ymm1
 ; AVX512-NEXT:    vaddpd %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vaddpd %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vaddpd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
 ; AVX512-NEXT:    vaddpd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vzeroupper
@@ -920,7 +920,7 @@ define double @test_v16f64_undef(<16 x double> %a0) {
 ; AVX-NEXT:    vaddpd %ymm2, %ymm0, %ymm0
 ; AVX-NEXT:    vaddpd %ymm1, %ymm0, %ymm0
 ; AVX-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX-NEXT:    vaddpd %ymm1, %ymm0, %ymm0
+; AVX-NEXT:    vaddpd %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
 ; AVX-NEXT:    vaddpd %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vzeroupper
@@ -932,7 +932,7 @@ define double @test_v16f64_undef(<16 x double> %a0) {
 ; AVX512-NEXT:    vextractf64x4 $1, %zmm0, %ymm1
 ; AVX512-NEXT:    vaddpd %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vaddpd %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vaddpd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
 ; AVX512-NEXT:    vaddpd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vzeroupper
diff --git a/test/CodeGen/X86/vector-reduce-fmul-fast.ll b/test/CodeGen/X86/vector-reduce-fmul-fast.ll
index efacbf1e3b4b710907e08b2511026fa5d4b77eb0..971d7f0478e81edba2b1f42734742d01fa2be05e 100644
--- a/test/CodeGen/X86/vector-reduce-fmul-fast.ll
+++ b/test/CodeGen/X86/vector-reduce-fmul-fast.ll
@@ -103,9 +103,9 @@ define float @test_v8f32(float %a0, <8 x float> %a1) {
 ; AVX-LABEL: test_v8f32:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vextractf128 $1, %ymm1, %xmm0
-; AVX-NEXT:    vmulps %ymm0, %ymm1, %ymm0
+; AVX-NEXT:    vmulps %xmm0, %xmm1, %xmm0
 ; AVX-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
-; AVX-NEXT:    vmulps %ymm1, %ymm0, %ymm0
+; AVX-NEXT:    vmulps %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
 ; AVX-NEXT:    vmulps %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vzeroupper
@@ -114,9 +114,9 @@ define float @test_v8f32(float %a0, <8 x float> %a1) {
 ; AVX512-LABEL: test_v8f32:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vextractf128 $1, %ymm1, %xmm0
-; AVX512-NEXT:    vmulps %ymm0, %ymm1, %ymm0
+; AVX512-NEXT:    vmulps %xmm0, %xmm1, %xmm0
 ; AVX512-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
-; AVX512-NEXT:    vmulps %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vmulps %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
 ; AVX512-NEXT:    vmulps %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vzeroupper
@@ -155,9 +155,9 @@ define float @test_v16f32(float %a0, <16 x float> %a1) {
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vmulps %ymm2, %ymm1, %ymm0
 ; AVX-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX-NEXT:    vmulps %ymm1, %ymm0, %ymm0
+; AVX-NEXT:    vmulps %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
-; AVX-NEXT:    vmulps %ymm1, %ymm0, %ymm0
+; AVX-NEXT:    vmulps %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
 ; AVX-NEXT:    vmulps %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vzeroupper
@@ -168,9 +168,9 @@ define float @test_v16f32(float %a0, <16 x float> %a1) {
 ; AVX512-NEXT:    vextractf64x4 $1, %zmm1, %ymm0
 ; AVX512-NEXT:    vmulps %zmm0, %zmm1, %zmm0
 ; AVX512-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vmulps %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vmulps %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
-; AVX512-NEXT:    vmulps %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vmulps %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
 ; AVX512-NEXT:    vmulps %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vzeroupper
@@ -279,9 +279,9 @@ define float @test_v8f32_zero(<8 x float> %a0) {
 ; AVX-LABEL: test_v8f32_zero:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX-NEXT:    vmulps %ymm1, %ymm0, %ymm0
+; AVX-NEXT:    vmulps %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
-; AVX-NEXT:    vmulps %ymm1, %ymm0, %ymm0
+; AVX-NEXT:    vmulps %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
 ; AVX-NEXT:    vmulps %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vzeroupper
@@ -290,9 +290,9 @@ define float @test_v8f32_zero(<8 x float> %a0) {
 ; AVX512-LABEL: test_v8f32_zero:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vmulps %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vmulps %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
-; AVX512-NEXT:    vmulps %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vmulps %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
 ; AVX512-NEXT:    vmulps %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vzeroupper
@@ -332,9 +332,9 @@ define float @test_v16f32_zero(<16 x float> %a0) {
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vmulps %ymm1, %ymm0, %ymm0
 ; AVX-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX-NEXT:    vmulps %ymm1, %ymm0, %ymm0
+; AVX-NEXT:    vmulps %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
-; AVX-NEXT:    vmulps %ymm1, %ymm0, %ymm0
+; AVX-NEXT:    vmulps %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
 ; AVX-NEXT:    vmulps %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vzeroupper
@@ -345,9 +345,9 @@ define float @test_v16f32_zero(<16 x float> %a0) {
 ; AVX512-NEXT:    vextractf64x4 $1, %zmm0, %ymm1
 ; AVX512-NEXT:    vmulps %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vmulps %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vmulps %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
-; AVX512-NEXT:    vmulps %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vmulps %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
 ; AVX512-NEXT:    vmulps %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vzeroupper
@@ -456,9 +456,9 @@ define float @test_v8f32_undef(<8 x float> %a0) {
 ; AVX-LABEL: test_v8f32_undef:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX-NEXT:    vmulps %ymm1, %ymm0, %ymm0
+; AVX-NEXT:    vmulps %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
-; AVX-NEXT:    vmulps %ymm1, %ymm0, %ymm0
+; AVX-NEXT:    vmulps %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
 ; AVX-NEXT:    vmulps %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vzeroupper
@@ -467,9 +467,9 @@ define float @test_v8f32_undef(<8 x float> %a0) {
 ; AVX512-LABEL: test_v8f32_undef:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vmulps %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vmulps %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
-; AVX512-NEXT:    vmulps %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vmulps %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
 ; AVX512-NEXT:    vmulps %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vzeroupper
@@ -509,9 +509,9 @@ define float @test_v16f32_undef(<16 x float> %a0) {
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vmulps %ymm1, %ymm0, %ymm0
 ; AVX-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX-NEXT:    vmulps %ymm1, %ymm0, %ymm0
+; AVX-NEXT:    vmulps %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
-; AVX-NEXT:    vmulps %ymm1, %ymm0, %ymm0
+; AVX-NEXT:    vmulps %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
 ; AVX-NEXT:    vmulps %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vzeroupper
@@ -522,9 +522,9 @@ define float @test_v16f32_undef(<16 x float> %a0) {
 ; AVX512-NEXT:    vextractf64x4 $1, %zmm0, %ymm1
 ; AVX512-NEXT:    vmulps %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vmulps %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vmulps %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
-; AVX512-NEXT:    vmulps %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vmulps %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
 ; AVX512-NEXT:    vmulps %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vzeroupper
@@ -572,7 +572,7 @@ define double @test_v4f64(double %a0, <4 x double> %a1) {
 ; AVX-LABEL: test_v4f64:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vextractf128 $1, %ymm1, %xmm0
-; AVX-NEXT:    vmulpd %ymm0, %ymm1, %ymm0
+; AVX-NEXT:    vmulpd %xmm0, %xmm1, %xmm0
 ; AVX-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
 ; AVX-NEXT:    vmulpd %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vzeroupper
@@ -581,7 +581,7 @@ define double @test_v4f64(double %a0, <4 x double> %a1) {
 ; AVX512-LABEL: test_v4f64:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vextractf128 $1, %ymm1, %xmm0
-; AVX512-NEXT:    vmulpd %ymm0, %ymm1, %ymm0
+; AVX512-NEXT:    vmulpd %xmm0, %xmm1, %xmm0
 ; AVX512-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
 ; AVX512-NEXT:    vmulpd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vzeroupper
@@ -605,7 +605,7 @@ define double @test_v8f64(double %a0, <8 x double> %a1) {
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vmulpd %ymm2, %ymm1, %ymm0
 ; AVX-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX-NEXT:    vmulpd %ymm1, %ymm0, %ymm0
+; AVX-NEXT:    vmulpd %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
 ; AVX-NEXT:    vmulpd %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vzeroupper
@@ -616,7 +616,7 @@ define double @test_v8f64(double %a0, <8 x double> %a1) {
 ; AVX512-NEXT:    vextractf64x4 $1, %zmm1, %ymm0
 ; AVX512-NEXT:    vmulpd %zmm0, %zmm1, %zmm0
 ; AVX512-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vmulpd %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vmulpd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
 ; AVX512-NEXT:    vmulpd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vzeroupper
@@ -646,7 +646,7 @@ define double @test_v16f64(double %a0, <16 x double> %a1) {
 ; AVX-NEXT:    vmulpd %ymm3, %ymm1, %ymm1
 ; AVX-NEXT:    vmulpd %ymm0, %ymm1, %ymm0
 ; AVX-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX-NEXT:    vmulpd %ymm1, %ymm0, %ymm0
+; AVX-NEXT:    vmulpd %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
 ; AVX-NEXT:    vmulpd %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vzeroupper
@@ -658,7 +658,7 @@ define double @test_v16f64(double %a0, <16 x double> %a1) {
 ; AVX512-NEXT:    vextractf64x4 $1, %zmm0, %ymm1
 ; AVX512-NEXT:    vmulpd %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vmulpd %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vmulpd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
 ; AVX512-NEXT:    vmulpd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vzeroupper
@@ -708,7 +708,7 @@ define double @test_v4f64_zero(<4 x double> %a0) {
 ; AVX-LABEL: test_v4f64_zero:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX-NEXT:    vmulpd %ymm1, %ymm0, %ymm0
+; AVX-NEXT:    vmulpd %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
 ; AVX-NEXT:    vmulpd %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vzeroupper
@@ -717,7 +717,7 @@ define double @test_v4f64_zero(<4 x double> %a0) {
 ; AVX512-LABEL: test_v4f64_zero:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vmulpd %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vmulpd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
 ; AVX512-NEXT:    vmulpd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vzeroupper
@@ -742,7 +742,7 @@ define double @test_v8f64_zero(<8 x double> %a0) {
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vmulpd %ymm1, %ymm0, %ymm0
 ; AVX-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX-NEXT:    vmulpd %ymm1, %ymm0, %ymm0
+; AVX-NEXT:    vmulpd %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
 ; AVX-NEXT:    vmulpd %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vzeroupper
@@ -753,7 +753,7 @@ define double @test_v8f64_zero(<8 x double> %a0) {
 ; AVX512-NEXT:    vextractf64x4 $1, %zmm0, %ymm1
 ; AVX512-NEXT:    vmulpd %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vmulpd %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vmulpd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
 ; AVX512-NEXT:    vmulpd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vzeroupper
@@ -783,7 +783,7 @@ define double @test_v16f64_zero(<16 x double> %a0) {
 ; AVX-NEXT:    vmulpd %ymm2, %ymm0, %ymm0
 ; AVX-NEXT:    vmulpd %ymm1, %ymm0, %ymm0
 ; AVX-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX-NEXT:    vmulpd %ymm1, %ymm0, %ymm0
+; AVX-NEXT:    vmulpd %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
 ; AVX-NEXT:    vmulpd %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vzeroupper
@@ -795,7 +795,7 @@ define double @test_v16f64_zero(<16 x double> %a0) {
 ; AVX512-NEXT:    vextractf64x4 $1, %zmm0, %ymm1
 ; AVX512-NEXT:    vmulpd %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vmulpd %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vmulpd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
 ; AVX512-NEXT:    vmulpd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vzeroupper
@@ -845,7 +845,7 @@ define double @test_v4f64_undef(<4 x double> %a0) {
 ; AVX-LABEL: test_v4f64_undef:
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX-NEXT:    vmulpd %ymm1, %ymm0, %ymm0
+; AVX-NEXT:    vmulpd %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
 ; AVX-NEXT:    vmulpd %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vzeroupper
@@ -854,7 +854,7 @@ define double @test_v4f64_undef(<4 x double> %a0) {
 ; AVX512-LABEL: test_v4f64_undef:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vmulpd %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vmulpd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
 ; AVX512-NEXT:    vmulpd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vzeroupper
@@ -879,7 +879,7 @@ define double @test_v8f64_undef(<8 x double> %a0) {
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vmulpd %ymm1, %ymm0, %ymm0
 ; AVX-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX-NEXT:    vmulpd %ymm1, %ymm0, %ymm0
+; AVX-NEXT:    vmulpd %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
 ; AVX-NEXT:    vmulpd %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vzeroupper
@@ -890,7 +890,7 @@ define double @test_v8f64_undef(<8 x double> %a0) {
 ; AVX512-NEXT:    vextractf64x4 $1, %zmm0, %ymm1
 ; AVX512-NEXT:    vmulpd %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vmulpd %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vmulpd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
 ; AVX512-NEXT:    vmulpd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vzeroupper
@@ -920,7 +920,7 @@ define double @test_v16f64_undef(<16 x double> %a0) {
 ; AVX-NEXT:    vmulpd %ymm2, %ymm0, %ymm0
 ; AVX-NEXT:    vmulpd %ymm1, %ymm0, %ymm0
 ; AVX-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX-NEXT:    vmulpd %ymm1, %ymm0, %ymm0
+; AVX-NEXT:    vmulpd %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
 ; AVX-NEXT:    vmulpd %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vzeroupper
@@ -932,7 +932,7 @@ define double @test_v16f64_undef(<16 x double> %a0) {
 ; AVX512-NEXT:    vextractf64x4 $1, %zmm0, %ymm1
 ; AVX512-NEXT:    vmulpd %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vmulpd %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vmulpd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpermilpd {{.*#+}} xmm1 = xmm0[1,0]
 ; AVX512-NEXT:    vmulpd %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vzeroupper
diff --git a/test/CodeGen/X86/vector-reduce-mul-widen.ll b/test/CodeGen/X86/vector-reduce-mul-widen.ll
index 7fbd7245c410714f9b5a55db99b1c0bf60d9168a..360b605da78f76c9db29635bffba1e8ba112cc9d 100644
--- a/test/CodeGen/X86/vector-reduce-mul-widen.ll
+++ b/test/CodeGen/X86/vector-reduce-mul-widen.ll
@@ -131,7 +131,7 @@ define i64 @test_v4i64(<4 x i64> %a0) {
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; AVX1-NEXT:    vpsrlq $32, %xmm0, %xmm2
 ; AVX1-NEXT:    vpmuludq %xmm1, %xmm2, %xmm2
-; AVX1-NEXT:    vpsrlq $32, %xmm1, %xmm3
+; AVX1-NEXT:    vpsrldq {{.*#+}} xmm3 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
 ; AVX1-NEXT:    vpmuludq %xmm3, %xmm0, %xmm3
 ; AVX1-NEXT:    vpaddq %xmm2, %xmm3, %xmm2
 ; AVX1-NEXT:    vpsllq $32, %xmm2, %xmm2
@@ -227,7 +227,7 @@ define i64 @test_v4i64(<4 x i64> %a0) {
 ; AVX512DQVL-LABEL: test_v4i64:
 ; AVX512DQVL:       # %bb.0:
 ; AVX512DQVL-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512DQVL-NEXT:    vpmullq %ymm1, %ymm0, %ymm0
+; AVX512DQVL-NEXT:    vpmullq %xmm1, %xmm0, %xmm0
 ; AVX512DQVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; AVX512DQVL-NEXT:    vpmullq %xmm1, %xmm0, %xmm0
 ; AVX512DQVL-NEXT:    vmovq %xmm0, %rax
@@ -315,7 +315,7 @@ define i64 @test_v8i64(<8 x i64> %a0) {
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; AVX1-NEXT:    vpsrlq $32, %xmm0, %xmm2
 ; AVX1-NEXT:    vpmuludq %xmm1, %xmm2, %xmm2
-; AVX1-NEXT:    vpsrlq $32, %xmm1, %xmm3
+; AVX1-NEXT:    vpsrldq {{.*#+}} xmm3 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
 ; AVX1-NEXT:    vpmuludq %xmm3, %xmm0, %xmm3
 ; AVX1-NEXT:    vpaddq %xmm2, %xmm3, %xmm2
 ; AVX1-NEXT:    vpsllq $32, %xmm2, %xmm2
@@ -440,7 +440,7 @@ define i64 @test_v8i64(<8 x i64> %a0) {
 ; AVX512DQVL-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512DQVL-NEXT:    vpmullq %zmm1, %zmm0, %zmm0
 ; AVX512DQVL-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512DQVL-NEXT:    vpmullq %zmm1, %zmm0, %zmm0
+; AVX512DQVL-NEXT:    vpmullq %xmm1, %xmm0, %xmm0
 ; AVX512DQVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; AVX512DQVL-NEXT:    vpmullq %xmm1, %xmm0, %xmm0
 ; AVX512DQVL-NEXT:    vmovq %xmm0, %rax
@@ -602,7 +602,7 @@ define i64 @test_v16i64(<16 x i64> %a0) {
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; AVX1-NEXT:    vpsrlq $32, %xmm0, %xmm2
 ; AVX1-NEXT:    vpmuludq %xmm1, %xmm2, %xmm2
-; AVX1-NEXT:    vpsrlq $32, %xmm1, %xmm3
+; AVX1-NEXT:    vpsrldq {{.*#+}} xmm3 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
 ; AVX1-NEXT:    vpmuludq %xmm3, %xmm0, %xmm3
 ; AVX1-NEXT:    vpaddq %xmm2, %xmm3, %xmm2
 ; AVX1-NEXT:    vpsllq $32, %xmm2, %xmm2
@@ -761,7 +761,7 @@ define i64 @test_v16i64(<16 x i64> %a0) {
 ; AVX512DQVL-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512DQVL-NEXT:    vpmullq %zmm1, %zmm0, %zmm0
 ; AVX512DQVL-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512DQVL-NEXT:    vpmullq %zmm1, %zmm0, %zmm0
+; AVX512DQVL-NEXT:    vpmullq %xmm1, %xmm0, %xmm0
 ; AVX512DQVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; AVX512DQVL-NEXT:    vpmullq %xmm1, %xmm0, %xmm0
 ; AVX512DQVL-NEXT:    vmovq %xmm0, %rax
@@ -900,9 +900,9 @@ define i32 @test_v8i32(<8 x i32> %a0) {
 ; AVX2-LABEL: test_v8i32:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpmulld %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX2-NEXT:    vpmulld %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX2-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
@@ -912,9 +912,9 @@ define i32 @test_v8i32(<8 x i32> %a0) {
 ; AVX512-LABEL: test_v8i32:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vpmulld %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpmulld %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
@@ -983,9 +983,9 @@ define i32 @test_v16i32(<16 x i32> %a0) {
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpmulld %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpmulld %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX2-NEXT:    vpmulld %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX2-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
@@ -997,9 +997,9 @@ define i32 @test_v16i32(<16 x i32> %a0) {
 ; AVX512-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512-NEXT:    vpmulld %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vpmulld %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpmulld %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
@@ -1092,9 +1092,9 @@ define i32 @test_v32i32(<32 x i32> %a0) {
 ; AVX2-NEXT:    vpmulld %ymm1, %ymm2, %ymm1
 ; AVX2-NEXT:    vpmulld %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpmulld %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX2-NEXT:    vpmulld %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX2-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
@@ -1107,9 +1107,9 @@ define i32 @test_v32i32(<32 x i32> %a0) {
 ; AVX512-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512-NEXT:    vpmulld %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vpmulld %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpmulld %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
@@ -1261,11 +1261,11 @@ define i16 @test_v16i16(<16 x i16> %a0) {
 ; AVX2-LABEL: test_v16i16:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX2-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX2-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
@@ -1276,11 +1276,11 @@ define i16 @test_v16i16(<16 x i16> %a0) {
 ; AVX512-LABEL: test_v16i16:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX512-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX512-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
@@ -1330,11 +1330,11 @@ define i16 @test_v32i16(<32 x i16> %a0) {
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX2-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX2-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
@@ -1347,11 +1347,11 @@ define i16 @test_v32i16(<32 x i16> %a0) {
 ; AVX512BW-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512BW-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512BW-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512BW-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX512BW-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512BW-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX512BW-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512BW-NEXT:    vmovd %xmm0, %eax
@@ -1364,11 +1364,11 @@ define i16 @test_v32i16(<32 x i16> %a0) {
 ; AVX512BWVL-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512BWVL-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
 ; AVX512BWVL-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512BWVL-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
+; AVX512BWVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512BWVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512BWVL-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
+; AVX512BWVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512BWVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX512BWVL-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
+; AVX512BWVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512BWVL-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX512BWVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512BWVL-NEXT:    vmovd %xmm0, %eax
@@ -1380,11 +1380,11 @@ define i16 @test_v32i16(<32 x i16> %a0) {
 ; AVX512DQ:       # %bb.0:
 ; AVX512DQ-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512DQ-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512DQ-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX512DQ-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512DQ-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX512DQ-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512DQ-NEXT:    vmovd %xmm0, %eax
@@ -1396,11 +1396,11 @@ define i16 @test_v32i16(<32 x i16> %a0) {
 ; AVX512DQVL:       # %bb.0:
 ; AVX512DQVL-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
 ; AVX512DQVL-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512DQVL-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
+; AVX512DQVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512DQVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512DQVL-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
+; AVX512DQVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512DQVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX512DQVL-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
+; AVX512DQVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512DQVL-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX512DQVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512DQVL-NEXT:    vmovd %xmm0, %eax
@@ -1462,11 +1462,11 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 ; AVX2-NEXT:    vpmullw %ymm1, %ymm2, %ymm1
 ; AVX2-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX2-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX2-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
@@ -1480,11 +1480,11 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 ; AVX512BW-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512BW-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512BW-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512BW-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX512BW-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512BW-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX512BW-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512BW-NEXT:    vmovd %xmm0, %eax
@@ -1498,11 +1498,11 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 ; AVX512BWVL-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512BWVL-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
 ; AVX512BWVL-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512BWVL-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
+; AVX512BWVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512BWVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512BWVL-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
+; AVX512BWVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512BWVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX512BWVL-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
+; AVX512BWVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512BWVL-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX512BWVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512BWVL-NEXT:    vmovd %xmm0, %eax
@@ -1516,11 +1516,11 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 ; AVX512DQ-NEXT:    vpmullw %ymm1, %ymm2, %ymm1
 ; AVX512DQ-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512DQ-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512DQ-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX512DQ-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512DQ-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX512DQ-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512DQ-NEXT:    vmovd %xmm0, %eax
@@ -1534,11 +1534,11 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 ; AVX512DQVL-NEXT:    vpmullw %ymm1, %ymm2, %ymm1
 ; AVX512DQVL-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
 ; AVX512DQVL-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512DQVL-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
+; AVX512DQVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512DQVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512DQVL-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
+; AVX512DQVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512DQVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX512DQVL-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
+; AVX512DQVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512DQVL-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX512DQVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512DQVL-NEXT:    vmovd %xmm0, %eax
@@ -2070,28 +2070,26 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpand %xmm3, %xmm0, %xmm0
 ; AVX1-NEXT:    vpackuswb %xmm2, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
 ; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
 ; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpand %xmm3, %xmm0, %xmm0
 ; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
 ; AVX1-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[1,1,2,3]
-; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
-; AVX1-NEXT:    vpmullw %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm2 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpmullw %xmm0, %xmm2, %xmm0
 ; AVX1-NEXT:    vpand %xmm3, %xmm0, %xmm0
 ; AVX1-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm2
-; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
-; AVX1-NEXT:    vpmullw %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm2 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpmullw %xmm0, %xmm2, %xmm0
 ; AVX1-NEXT:    vpand %xmm3, %xmm0, %xmm0
 ; AVX1-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm2
 ; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
 ; AVX1-NEXT:    vpmullw %xmm2, %xmm0, %xmm0
 ; AVX1-NEXT:    vpand %xmm3, %xmm0, %xmm0
 ; AVX1-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
@@ -2113,26 +2111,27 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; AVX2-NEXT:    vpmullw %ymm2, %ymm0, %ymm0
 ; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vpackuswb %ymm3, %ymm0, %ymm0
-; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
-; AVX2-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; AVX2-NEXT:    vpmullw %ymm0, %ymm2, %ymm0
+; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm2 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
+; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX2-NEXT:    vpmullw %ymm2, %ymm0, %ymm0
 ; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
 ; AVX2-NEXT:    vpackuswb %ymm2, %ymm0, %ymm0
-; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
-; AVX2-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; AVX2-NEXT:    vpmullw %ymm0, %ymm3, %ymm0
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,2,3]
+; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm3[0],ymm0[0],ymm3[1],ymm0[1],ymm3[2],ymm0[2],ymm3[3],ymm0[3],ymm3[4],ymm0[4],ymm3[5],ymm0[5],ymm3[6],ymm0[6],ymm3[7],ymm0[7],ymm3[16],ymm0[16],ymm3[17],ymm0[17],ymm3[18],ymm0[18],ymm3[19],ymm0[19],ymm3[20],ymm0[20],ymm3[21],ymm0[21],ymm3[22],ymm0[22],ymm3[23],ymm0[23]
+; AVX2-NEXT:    vpmullw %ymm3, %ymm0, %ymm0
 ; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vpackuswb %ymm2, %ymm0, %ymm0
-; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
-; AVX2-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
-; AVX2-NEXT:    vpmullw %ymm0, %ymm3, %ymm0
+; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm3
+; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm3[0],ymm0[0],ymm3[1],ymm0[1],ymm3[2],ymm0[2],ymm3[3],ymm0[3],ymm3[4],ymm0[4],ymm3[5],ymm0[5],ymm3[6],ymm0[6],ymm3[7],ymm0[7],ymm3[16],ymm0[16],ymm3[17],ymm0[17],ymm3[18],ymm0[18],ymm3[19],ymm0[19],ymm3[20],ymm0[20],ymm3[21],ymm0[21],ymm3[22],ymm0[22],ymm3[23],ymm0[23]
+; AVX2-NEXT:    vpmullw %ymm3, %ymm0, %ymm0
 ; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vpackuswb %ymm2, %ymm0, %ymm0
 ; AVX2-NEXT:    vpsrlw $8, %xmm0, %xmm3
 ; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm3[0],ymm0[0],ymm3[1],ymm0[1],ymm3[2],ymm0[2],ymm3[3],ymm0[3],ymm3[4],ymm0[4],ymm3[5],ymm0[5],ymm3[6],ymm0[6],ymm3[7],ymm0[7],ymm3[16],ymm0[16],ymm3[17],ymm0[17],ymm3[18],ymm0[18],ymm3[19],ymm0[19],ymm3[20],ymm0[20],ymm3[21],ymm0[21],ymm3[22],ymm0[22],ymm3[23],ymm0[23]
 ; AVX2-NEXT:    vpmullw %ymm3, %ymm0, %ymm0
 ; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vpackuswb %ymm2, %ymm0, %ymm0
@@ -2210,26 +2209,27 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; AVX512DQ-NEXT:    vpmullw %ymm2, %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vpand %ymm1, %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vpackuswb %ymm3, %ymm0, %ymm0
-; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
-; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; AVX512DQ-NEXT:    vpmullw %ymm0, %ymm2, %ymm0
+; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} ymm2 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
+; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX512DQ-NEXT:    vpmullw %ymm2, %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vpand %ymm1, %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vpxor %xmm2, %xmm2, %xmm2
 ; AVX512DQ-NEXT:    vpackuswb %ymm2, %ymm0, %ymm0
-; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
-; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; AVX512DQ-NEXT:    vpmullw %ymm0, %ymm3, %ymm0
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,2,3]
+; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm3[0],ymm0[0],ymm3[1],ymm0[1],ymm3[2],ymm0[2],ymm3[3],ymm0[3],ymm3[4],ymm0[4],ymm3[5],ymm0[5],ymm3[6],ymm0[6],ymm3[7],ymm0[7],ymm3[16],ymm0[16],ymm3[17],ymm0[17],ymm3[18],ymm0[18],ymm3[19],ymm0[19],ymm3[20],ymm0[20],ymm3[21],ymm0[21],ymm3[22],ymm0[22],ymm3[23],ymm0[23]
+; AVX512DQ-NEXT:    vpmullw %ymm3, %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vpand %ymm1, %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vpackuswb %ymm2, %ymm0, %ymm0
-; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
-; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
-; AVX512DQ-NEXT:    vpmullw %ymm0, %ymm3, %ymm0
+; AVX512DQ-NEXT:    vpsrld $16, %xmm0, %xmm3
+; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm3[0],ymm0[0],ymm3[1],ymm0[1],ymm3[2],ymm0[2],ymm3[3],ymm0[3],ymm3[4],ymm0[4],ymm3[5],ymm0[5],ymm3[6],ymm0[6],ymm3[7],ymm0[7],ymm3[16],ymm0[16],ymm3[17],ymm0[17],ymm3[18],ymm0[18],ymm3[19],ymm0[19],ymm3[20],ymm0[20],ymm3[21],ymm0[21],ymm3[22],ymm0[22],ymm3[23],ymm0[23]
+; AVX512DQ-NEXT:    vpmullw %ymm3, %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vpand %ymm1, %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vpackuswb %ymm2, %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vpsrlw $8, %xmm0, %xmm3
 ; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm3[0],ymm0[0],ymm3[1],ymm0[1],ymm3[2],ymm0[2],ymm3[3],ymm0[3],ymm3[4],ymm0[4],ymm3[5],ymm0[5],ymm3[6],ymm0[6],ymm3[7],ymm0[7],ymm3[16],ymm0[16],ymm3[17],ymm0[17],ymm3[18],ymm0[18],ymm3[19],ymm0[19],ymm3[20],ymm0[20],ymm3[21],ymm0[21],ymm3[22],ymm0[22],ymm3[23],ymm0[23]
 ; AVX512DQ-NEXT:    vpmullw %ymm3, %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vpand %ymm1, %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vpackuswb %ymm2, %ymm0, %ymm0
@@ -2251,26 +2251,27 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; AVX512DQVL-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
 ; AVX512DQVL-NEXT:    vpand %ymm3, %ymm0, %ymm0
 ; AVX512DQVL-NEXT:    vpackuswb %ymm2, %ymm0, %ymm0
-; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
-; AVX512DQVL-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; AVX512DQVL-NEXT:    vpmullw %ymm0, %ymm1, %ymm0
+; AVX512DQVL-NEXT:    vpunpckhbw {{.*#+}} ymm1 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
+; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX512DQVL-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
 ; AVX512DQVL-NEXT:    vpand %ymm3, %ymm0, %ymm0
 ; AVX512DQVL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
 ; AVX512DQVL-NEXT:    vpackuswb %ymm1, %ymm0, %ymm0
-; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
-; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; AVX512DQVL-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; AVX512DQVL-NEXT:    vpmullw %ymm0, %ymm2, %ymm0
+; AVX512DQVL-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[1,1,2,3]
+; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm2[0],ymm0[0],ymm2[1],ymm0[1],ymm2[2],ymm0[2],ymm2[3],ymm0[3],ymm2[4],ymm0[4],ymm2[5],ymm0[5],ymm2[6],ymm0[6],ymm2[7],ymm0[7],ymm2[16],ymm0[16],ymm2[17],ymm0[17],ymm2[18],ymm0[18],ymm2[19],ymm0[19],ymm2[20],ymm0[20],ymm2[21],ymm0[21],ymm2[22],ymm0[22],ymm2[23],ymm0[23]
+; AVX512DQVL-NEXT:    vpmullw %ymm2, %ymm0, %ymm0
 ; AVX512DQVL-NEXT:    vpand %ymm3, %ymm0, %ymm0
 ; AVX512DQVL-NEXT:    vpackuswb %ymm1, %ymm0, %ymm0
-; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
-; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; AVX512DQVL-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
-; AVX512DQVL-NEXT:    vpmullw %ymm0, %ymm2, %ymm0
+; AVX512DQVL-NEXT:    vpsrld $16, %xmm0, %xmm2
+; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm2[0],ymm0[0],ymm2[1],ymm0[1],ymm2[2],ymm0[2],ymm2[3],ymm0[3],ymm2[4],ymm0[4],ymm2[5],ymm0[5],ymm2[6],ymm0[6],ymm2[7],ymm0[7],ymm2[16],ymm0[16],ymm2[17],ymm0[17],ymm2[18],ymm0[18],ymm2[19],ymm0[19],ymm2[20],ymm0[20],ymm2[21],ymm0[21],ymm2[22],ymm0[22],ymm2[23],ymm0[23]
+; AVX512DQVL-NEXT:    vpmullw %ymm2, %ymm0, %ymm0
 ; AVX512DQVL-NEXT:    vpand %ymm3, %ymm0, %ymm0
 ; AVX512DQVL-NEXT:    vpackuswb %ymm1, %ymm0, %ymm0
 ; AVX512DQVL-NEXT:    vpsrlw $8, %xmm0, %xmm2
 ; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm2[0],ymm0[0],ymm2[1],ymm0[1],ymm2[2],ymm0[2],ymm2[3],ymm0[3],ymm2[4],ymm0[4],ymm2[5],ymm0[5],ymm2[6],ymm0[6],ymm2[7],ymm0[7],ymm2[16],ymm0[16],ymm2[17],ymm0[17],ymm2[18],ymm0[18],ymm2[19],ymm0[19],ymm2[20],ymm0[20],ymm2[21],ymm0[21],ymm2[22],ymm0[22],ymm2[23],ymm0[23]
 ; AVX512DQVL-NEXT:    vpmullw %ymm2, %ymm0, %ymm0
 ; AVX512DQVL-NEXT:    vpand %ymm3, %ymm0, %ymm0
 ; AVX512DQVL-NEXT:    vpackuswb %ymm1, %ymm0, %ymm0
@@ -2436,28 +2437,26 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; AVX1-NEXT:    vpmullw %xmm0, %xmm3, %xmm0
 ; AVX1-NEXT:    vpand %xmm2, %xmm0, %xmm0
 ; AVX1-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
 ; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
 ; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpand %xmm2, %xmm0, %xmm0
 ; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
 ; AVX1-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,2,3]
-; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm3 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero
-; AVX1-NEXT:    vpmullw %xmm3, %xmm0, %xmm0
+; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpmullw %xmm0, %xmm3, %xmm0
 ; AVX1-NEXT:    vpand %xmm2, %xmm0, %xmm0
 ; AVX1-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm3
-; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm3 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero
-; AVX1-NEXT:    vpmullw %xmm3, %xmm0, %xmm0
+; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpmullw %xmm0, %xmm3, %xmm0
 ; AVX1-NEXT:    vpand %xmm2, %xmm0, %xmm0
 ; AVX1-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm3
 ; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm3 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero
 ; AVX1-NEXT:    vpmullw %xmm3, %xmm0, %xmm0
 ; AVX1-NEXT:    vpand %xmm2, %xmm0, %xmm0
 ; AVX1-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
@@ -2488,26 +2487,27 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; AVX2-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vpand %ymm2, %ymm0, %ymm0
 ; AVX2-NEXT:    vpackuswb %ymm3, %ymm0, %ymm0
-; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
-; AVX2-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; AVX2-NEXT:    vpmullw %ymm0, %ymm1, %ymm0
+; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm1 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
+; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX2-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vpand %ymm2, %ymm0, %ymm0
 ; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
 ; AVX2-NEXT:    vpackuswb %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
-; AVX2-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; AVX2-NEXT:    vpmullw %ymm0, %ymm3, %ymm0
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,2,3]
+; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm3[0],ymm0[0],ymm3[1],ymm0[1],ymm3[2],ymm0[2],ymm3[3],ymm0[3],ymm3[4],ymm0[4],ymm3[5],ymm0[5],ymm3[6],ymm0[6],ymm3[7],ymm0[7],ymm3[16],ymm0[16],ymm3[17],ymm0[17],ymm3[18],ymm0[18],ymm3[19],ymm0[19],ymm3[20],ymm0[20],ymm3[21],ymm0[21],ymm3[22],ymm0[22],ymm3[23],ymm0[23]
+; AVX2-NEXT:    vpmullw %ymm3, %ymm0, %ymm0
 ; AVX2-NEXT:    vpand %ymm2, %ymm0, %ymm0
 ; AVX2-NEXT:    vpackuswb %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
-; AVX2-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
-; AVX2-NEXT:    vpmullw %ymm0, %ymm3, %ymm0
+; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm3
+; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm3[0],ymm0[0],ymm3[1],ymm0[1],ymm3[2],ymm0[2],ymm3[3],ymm0[3],ymm3[4],ymm0[4],ymm3[5],ymm0[5],ymm3[6],ymm0[6],ymm3[7],ymm0[7],ymm3[16],ymm0[16],ymm3[17],ymm0[17],ymm3[18],ymm0[18],ymm3[19],ymm0[19],ymm3[20],ymm0[20],ymm3[21],ymm0[21],ymm3[22],ymm0[22],ymm3[23],ymm0[23]
+; AVX2-NEXT:    vpmullw %ymm3, %ymm0, %ymm0
 ; AVX2-NEXT:    vpand %ymm2, %ymm0, %ymm0
 ; AVX2-NEXT:    vpackuswb %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vpsrlw $8, %xmm0, %xmm3
 ; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm3[0],ymm0[0],ymm3[1],ymm0[1],ymm3[2],ymm0[2],ymm3[3],ymm0[3],ymm3[4],ymm0[4],ymm3[5],ymm0[5],ymm3[6],ymm0[6],ymm3[7],ymm0[7],ymm3[16],ymm0[16],ymm3[17],ymm0[17],ymm3[18],ymm0[18],ymm3[19],ymm0[19],ymm3[20],ymm0[20],ymm3[21],ymm0[21],ymm3[22],ymm0[22],ymm3[23],ymm0[23]
 ; AVX2-NEXT:    vpmullw %ymm3, %ymm0, %ymm0
 ; AVX2-NEXT:    vpand %ymm2, %ymm0, %ymm0
 ; AVX2-NEXT:    vpackuswb %ymm1, %ymm0, %ymm0
@@ -2539,26 +2539,27 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; AVX512BW-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vpandq %zmm3, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vpackuswb %zmm2, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} zmm1 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
-; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; AVX512BW-NEXT:    vpmullw %zmm0, %zmm1, %zmm0
+; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} zmm1 = zmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]
+; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} zmm0 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
+; AVX512BW-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vpandq %zmm3, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vpxor %xmm1, %xmm1, %xmm1
 ; AVX512BW-NEXT:    vpackuswb %zmm1, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} zmm2 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
-; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; AVX512BW-NEXT:    vpmullw %zmm0, %zmm2, %zmm0
+; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[1,1,2,3]
+; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} zmm0 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
+; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} zmm2 = zmm2[0],zmm0[0],zmm2[1],zmm0[1],zmm2[2],zmm0[2],zmm2[3],zmm0[3],zmm2[4],zmm0[4],zmm2[5],zmm0[5],zmm2[6],zmm0[6],zmm2[7],zmm0[7],zmm2[16],zmm0[16],zmm2[17],zmm0[17],zmm2[18],zmm0[18],zmm2[19],zmm0[19],zmm2[20],zmm0[20],zmm2[21],zmm0[21],zmm2[22],zmm0[22],zmm2[23],zmm0[23],zmm2[32],zmm0[32],zmm2[33],zmm0[33],zmm2[34],zmm0[34],zmm2[35],zmm0[35],zmm2[36],zmm0[36],zmm2[37],zmm0[37],zmm2[38],zmm0[38],zmm2[39],zmm0[39],zmm2[48],zmm0[48],zmm2[49],zmm0[49],zmm2[50],zmm0[50],zmm2[51],zmm0[51],zmm2[52],zmm0[52],zmm2[53],zmm0[53],zmm2[54],zmm0[54],zmm2[55],zmm0[55]
+; AVX512BW-NEXT:    vpmullw %zmm2, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vpandq %zmm3, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vpackuswb %zmm1, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} zmm2 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
-; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
-; AVX512BW-NEXT:    vpmullw %zmm0, %zmm2, %zmm0
+; AVX512BW-NEXT:    vpsrld $16, %xmm0, %xmm2
+; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} zmm0 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
+; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} zmm2 = zmm2[0],zmm0[0],zmm2[1],zmm0[1],zmm2[2],zmm0[2],zmm2[3],zmm0[3],zmm2[4],zmm0[4],zmm2[5],zmm0[5],zmm2[6],zmm0[6],zmm2[7],zmm0[7],zmm2[16],zmm0[16],zmm2[17],zmm0[17],zmm2[18],zmm0[18],zmm2[19],zmm0[19],zmm2[20],zmm0[20],zmm2[21],zmm0[21],zmm2[22],zmm0[22],zmm2[23],zmm0[23],zmm2[32],zmm0[32],zmm2[33],zmm0[33],zmm2[34],zmm0[34],zmm2[35],zmm0[35],zmm2[36],zmm0[36],zmm2[37],zmm0[37],zmm2[38],zmm0[38],zmm2[39],zmm0[39],zmm2[48],zmm0[48],zmm2[49],zmm0[49],zmm2[50],zmm0[50],zmm2[51],zmm0[51],zmm2[52],zmm0[52],zmm2[53],zmm0[53],zmm2[54],zmm0[54],zmm2[55],zmm0[55]
+; AVX512BW-NEXT:    vpmullw %zmm2, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vpandq %zmm3, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vpackuswb %zmm1, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vpsrlw $8, %xmm0, %xmm2
 ; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} zmm0 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
+; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} zmm2 = zmm2[0],zmm0[0],zmm2[1],zmm0[1],zmm2[2],zmm0[2],zmm2[3],zmm0[3],zmm2[4],zmm0[4],zmm2[5],zmm0[5],zmm2[6],zmm0[6],zmm2[7],zmm0[7],zmm2[16],zmm0[16],zmm2[17],zmm0[17],zmm2[18],zmm0[18],zmm2[19],zmm0[19],zmm2[20],zmm0[20],zmm2[21],zmm0[21],zmm2[22],zmm0[22],zmm2[23],zmm0[23],zmm2[32],zmm0[32],zmm2[33],zmm0[33],zmm2[34],zmm0[34],zmm2[35],zmm0[35],zmm2[36],zmm0[36],zmm2[37],zmm0[37],zmm2[38],zmm0[38],zmm2[39],zmm0[39],zmm2[48],zmm0[48],zmm2[49],zmm0[49],zmm2[50],zmm0[50],zmm2[51],zmm0[51],zmm2[52],zmm0[52],zmm2[53],zmm0[53],zmm2[54],zmm0[54],zmm2[55],zmm0[55]
 ; AVX512BW-NEXT:    vpmullw %zmm2, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vpandq %zmm3, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vpackuswb %zmm1, %zmm0, %zmm0
@@ -2590,26 +2591,27 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; AVX512BWVL-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
 ; AVX512BWVL-NEXT:    vpandq %zmm3, %zmm0, %zmm0
 ; AVX512BWVL-NEXT:    vpackuswb %zmm2, %zmm0, %zmm0
-; AVX512BWVL-NEXT:    vpunpcklbw {{.*#+}} zmm1 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
-; AVX512BWVL-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; AVX512BWVL-NEXT:    vpmullw %zmm0, %zmm1, %zmm0
+; AVX512BWVL-NEXT:    vpunpckhbw {{.*#+}} zmm1 = zmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]
+; AVX512BWVL-NEXT:    vpunpcklbw {{.*#+}} zmm0 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
+; AVX512BWVL-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
 ; AVX512BWVL-NEXT:    vpandq %zmm3, %zmm0, %zmm0
 ; AVX512BWVL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
 ; AVX512BWVL-NEXT:    vpackuswb %zmm1, %zmm0, %zmm0
-; AVX512BWVL-NEXT:    vpunpcklbw {{.*#+}} zmm2 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
-; AVX512BWVL-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; AVX512BWVL-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; AVX512BWVL-NEXT:    vpmullw %zmm0, %zmm2, %zmm0
+; AVX512BWVL-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[1,1,2,3]
+; AVX512BWVL-NEXT:    vpunpcklbw {{.*#+}} zmm0 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
+; AVX512BWVL-NEXT:    vpunpcklbw {{.*#+}} zmm2 = zmm2[0],zmm0[0],zmm2[1],zmm0[1],zmm2[2],zmm0[2],zmm2[3],zmm0[3],zmm2[4],zmm0[4],zmm2[5],zmm0[5],zmm2[6],zmm0[6],zmm2[7],zmm0[7],zmm2[16],zmm0[16],zmm2[17],zmm0[17],zmm2[18],zmm0[18],zmm2[19],zmm0[19],zmm2[20],zmm0[20],zmm2[21],zmm0[21],zmm2[22],zmm0[22],zmm2[23],zmm0[23],zmm2[32],zmm0[32],zmm2[33],zmm0[33],zmm2[34],zmm0[34],zmm2[35],zmm0[35],zmm2[36],zmm0[36],zmm2[37],zmm0[37],zmm2[38],zmm0[38],zmm2[39],zmm0[39],zmm2[48],zmm0[48],zmm2[49],zmm0[49],zmm2[50],zmm0[50],zmm2[51],zmm0[51],zmm2[52],zmm0[52],zmm2[53],zmm0[53],zmm2[54],zmm0[54],zmm2[55],zmm0[55]
+; AVX512BWVL-NEXT:    vpmullw %zmm2, %zmm0, %zmm0
 ; AVX512BWVL-NEXT:    vpandq %zmm3, %zmm0, %zmm0
 ; AVX512BWVL-NEXT:    vpackuswb %zmm1, %zmm0, %zmm0
-; AVX512BWVL-NEXT:    vpunpcklbw {{.*#+}} zmm2 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
-; AVX512BWVL-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; AVX512BWVL-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
-; AVX512BWVL-NEXT:    vpmullw %zmm0, %zmm2, %zmm0
+; AVX512BWVL-NEXT:    vpsrld $16, %xmm0, %xmm2
+; AVX512BWVL-NEXT:    vpunpcklbw {{.*#+}} zmm0 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
+; AVX512BWVL-NEXT:    vpunpcklbw {{.*#+}} zmm2 = zmm2[0],zmm0[0],zmm2[1],zmm0[1],zmm2[2],zmm0[2],zmm2[3],zmm0[3],zmm2[4],zmm0[4],zmm2[5],zmm0[5],zmm2[6],zmm0[6],zmm2[7],zmm0[7],zmm2[16],zmm0[16],zmm2[17],zmm0[17],zmm2[18],zmm0[18],zmm2[19],zmm0[19],zmm2[20],zmm0[20],zmm2[21],zmm0[21],zmm2[22],zmm0[22],zmm2[23],zmm0[23],zmm2[32],zmm0[32],zmm2[33],zmm0[33],zmm2[34],zmm0[34],zmm2[35],zmm0[35],zmm2[36],zmm0[36],zmm2[37],zmm0[37],zmm2[38],zmm0[38],zmm2[39],zmm0[39],zmm2[48],zmm0[48],zmm2[49],zmm0[49],zmm2[50],zmm0[50],zmm2[51],zmm0[51],zmm2[52],zmm0[52],zmm2[53],zmm0[53],zmm2[54],zmm0[54],zmm2[55],zmm0[55]
+; AVX512BWVL-NEXT:    vpmullw %zmm2, %zmm0, %zmm0
 ; AVX512BWVL-NEXT:    vpandq %zmm3, %zmm0, %zmm0
 ; AVX512BWVL-NEXT:    vpackuswb %zmm1, %zmm0, %zmm0
 ; AVX512BWVL-NEXT:    vpsrlw $8, %xmm0, %xmm2
 ; AVX512BWVL-NEXT:    vpunpcklbw {{.*#+}} zmm0 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
+; AVX512BWVL-NEXT:    vpunpcklbw {{.*#+}} zmm2 = zmm2[0],zmm0[0],zmm2[1],zmm0[1],zmm2[2],zmm0[2],zmm2[3],zmm0[3],zmm2[4],zmm0[4],zmm2[5],zmm0[5],zmm2[6],zmm0[6],zmm2[7],zmm0[7],zmm2[16],zmm0[16],zmm2[17],zmm0[17],zmm2[18],zmm0[18],zmm2[19],zmm0[19],zmm2[20],zmm0[20],zmm2[21],zmm0[21],zmm2[22],zmm0[22],zmm2[23],zmm0[23],zmm2[32],zmm0[32],zmm2[33],zmm0[33],zmm2[34],zmm0[34],zmm2[35],zmm0[35],zmm2[36],zmm0[36],zmm2[37],zmm0[37],zmm2[38],zmm0[38],zmm2[39],zmm0[39],zmm2[48],zmm0[48],zmm2[49],zmm0[49],zmm2[50],zmm0[50],zmm2[51],zmm0[51],zmm2[52],zmm0[52],zmm2[53],zmm0[53],zmm2[54],zmm0[54],zmm2[55],zmm0[55]
 ; AVX512BWVL-NEXT:    vpmullw %zmm2, %zmm0, %zmm0
 ; AVX512BWVL-NEXT:    vpandq %zmm3, %zmm0, %zmm0
 ; AVX512BWVL-NEXT:    vpackuswb %zmm1, %zmm0, %zmm0
@@ -2640,26 +2642,27 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; AVX512DQ-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vpand %ymm2, %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vpackuswb %ymm3, %ymm0, %ymm0
-; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
-; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; AVX512DQ-NEXT:    vpmullw %ymm0, %ymm1, %ymm0
+; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} ymm1 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
+; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX512DQ-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vpand %ymm2, %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vpxor %xmm1, %xmm1, %xmm1
 ; AVX512DQ-NEXT:    vpackuswb %ymm1, %ymm0, %ymm0
-; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
-; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; AVX512DQ-NEXT:    vpmullw %ymm0, %ymm3, %ymm0
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,2,3]
+; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm3[0],ymm0[0],ymm3[1],ymm0[1],ymm3[2],ymm0[2],ymm3[3],ymm0[3],ymm3[4],ymm0[4],ymm3[5],ymm0[5],ymm3[6],ymm0[6],ymm3[7],ymm0[7],ymm3[16],ymm0[16],ymm3[17],ymm0[17],ymm3[18],ymm0[18],ymm3[19],ymm0[19],ymm3[20],ymm0[20],ymm3[21],ymm0[21],ymm3[22],ymm0[22],ymm3[23],ymm0[23]
+; AVX512DQ-NEXT:    vpmullw %ymm3, %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vpand %ymm2, %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vpackuswb %ymm1, %ymm0, %ymm0
-; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
-; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
-; AVX512DQ-NEXT:    vpmullw %ymm0, %ymm3, %ymm0
+; AVX512DQ-NEXT:    vpsrld $16, %xmm0, %xmm3
+; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm3[0],ymm0[0],ymm3[1],ymm0[1],ymm3[2],ymm0[2],ymm3[3],ymm0[3],ymm3[4],ymm0[4],ymm3[5],ymm0[5],ymm3[6],ymm0[6],ymm3[7],ymm0[7],ymm3[16],ymm0[16],ymm3[17],ymm0[17],ymm3[18],ymm0[18],ymm3[19],ymm0[19],ymm3[20],ymm0[20],ymm3[21],ymm0[21],ymm3[22],ymm0[22],ymm3[23],ymm0[23]
+; AVX512DQ-NEXT:    vpmullw %ymm3, %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vpand %ymm2, %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vpackuswb %ymm1, %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vpsrlw $8, %xmm0, %xmm3
 ; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm3[0],ymm0[0],ymm3[1],ymm0[1],ymm3[2],ymm0[2],ymm3[3],ymm0[3],ymm3[4],ymm0[4],ymm3[5],ymm0[5],ymm3[6],ymm0[6],ymm3[7],ymm0[7],ymm3[16],ymm0[16],ymm3[17],ymm0[17],ymm3[18],ymm0[18],ymm3[19],ymm0[19],ymm3[20],ymm0[20],ymm3[21],ymm0[21],ymm3[22],ymm0[22],ymm3[23],ymm0[23]
 ; AVX512DQ-NEXT:    vpmullw %ymm3, %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vpand %ymm2, %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vpackuswb %ymm1, %ymm0, %ymm0
@@ -2690,26 +2693,27 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; AVX512DQVL-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
 ; AVX512DQVL-NEXT:    vpand %ymm3, %ymm0, %ymm0
 ; AVX512DQVL-NEXT:    vpackuswb %ymm2, %ymm0, %ymm0
-; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
-; AVX512DQVL-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; AVX512DQVL-NEXT:    vpmullw %ymm0, %ymm1, %ymm0
+; AVX512DQVL-NEXT:    vpunpckhbw {{.*#+}} ymm1 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
+; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX512DQVL-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
 ; AVX512DQVL-NEXT:    vpand %ymm3, %ymm0, %ymm0
 ; AVX512DQVL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
 ; AVX512DQVL-NEXT:    vpackuswb %ymm1, %ymm0, %ymm0
-; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
-; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; AVX512DQVL-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; AVX512DQVL-NEXT:    vpmullw %ymm0, %ymm2, %ymm0
+; AVX512DQVL-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[1,1,2,3]
+; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm2[0],ymm0[0],ymm2[1],ymm0[1],ymm2[2],ymm0[2],ymm2[3],ymm0[3],ymm2[4],ymm0[4],ymm2[5],ymm0[5],ymm2[6],ymm0[6],ymm2[7],ymm0[7],ymm2[16],ymm0[16],ymm2[17],ymm0[17],ymm2[18],ymm0[18],ymm2[19],ymm0[19],ymm2[20],ymm0[20],ymm2[21],ymm0[21],ymm2[22],ymm0[22],ymm2[23],ymm0[23]
+; AVX512DQVL-NEXT:    vpmullw %ymm2, %ymm0, %ymm0
 ; AVX512DQVL-NEXT:    vpand %ymm3, %ymm0, %ymm0
 ; AVX512DQVL-NEXT:    vpackuswb %ymm1, %ymm0, %ymm0
-; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
-; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; AVX512DQVL-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
-; AVX512DQVL-NEXT:    vpmullw %ymm0, %ymm2, %ymm0
+; AVX512DQVL-NEXT:    vpsrld $16, %xmm0, %xmm2
+; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm2[0],ymm0[0],ymm2[1],ymm0[1],ymm2[2],ymm0[2],ymm2[3],ymm0[3],ymm2[4],ymm0[4],ymm2[5],ymm0[5],ymm2[6],ymm0[6],ymm2[7],ymm0[7],ymm2[16],ymm0[16],ymm2[17],ymm0[17],ymm2[18],ymm0[18],ymm2[19],ymm0[19],ymm2[20],ymm0[20],ymm2[21],ymm0[21],ymm2[22],ymm0[22],ymm2[23],ymm0[23]
+; AVX512DQVL-NEXT:    vpmullw %ymm2, %ymm0, %ymm0
 ; AVX512DQVL-NEXT:    vpand %ymm3, %ymm0, %ymm0
 ; AVX512DQVL-NEXT:    vpackuswb %ymm1, %ymm0, %ymm0
 ; AVX512DQVL-NEXT:    vpsrlw $8, %xmm0, %xmm2
 ; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm2[0],ymm0[0],ymm2[1],ymm0[1],ymm2[2],ymm0[2],ymm2[3],ymm0[3],ymm2[4],ymm0[4],ymm2[5],ymm0[5],ymm2[6],ymm0[6],ymm2[7],ymm0[7],ymm2[16],ymm0[16],ymm2[17],ymm0[17],ymm2[18],ymm0[18],ymm2[19],ymm0[19],ymm2[20],ymm0[20],ymm2[21],ymm0[21],ymm2[22],ymm0[22],ymm2[23],ymm0[23]
 ; AVX512DQVL-NEXT:    vpmullw %ymm2, %ymm0, %ymm0
 ; AVX512DQVL-NEXT:    vpand %ymm3, %ymm0, %ymm0
 ; AVX512DQVL-NEXT:    vpackuswb %ymm1, %ymm0, %ymm0
@@ -2993,28 +2997,26 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpand %xmm4, %xmm0, %xmm0
 ; AVX1-NEXT:    vpackuswb %xmm2, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
 ; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
 ; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpand %xmm4, %xmm0, %xmm0
 ; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
 ; AVX1-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[1,1,2,3]
-; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
-; AVX1-NEXT:    vpmullw %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm2 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpmullw %xmm0, %xmm2, %xmm0
 ; AVX1-NEXT:    vpand %xmm4, %xmm0, %xmm0
 ; AVX1-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm2
-; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
-; AVX1-NEXT:    vpmullw %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm2 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpmullw %xmm0, %xmm2, %xmm0
 ; AVX1-NEXT:    vpand %xmm4, %xmm0, %xmm0
 ; AVX1-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm2
 ; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
 ; AVX1-NEXT:    vpmullw %xmm2, %xmm0, %xmm0
 ; AVX1-NEXT:    vpand %xmm4, %xmm0, %xmm0
 ; AVX1-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
@@ -3063,26 +3065,27 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; AVX2-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vpand %ymm4, %ymm0, %ymm0
 ; AVX2-NEXT:    vpackuswb %ymm2, %ymm0, %ymm0
-; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
-; AVX2-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; AVX2-NEXT:    vpmullw %ymm0, %ymm1, %ymm0
+; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm1 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
+; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX2-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vpand %ymm4, %ymm0, %ymm0
 ; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
 ; AVX2-NEXT:    vpackuswb %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
-; AVX2-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; AVX2-NEXT:    vpmullw %ymm0, %ymm2, %ymm0
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[1,1,2,3]
+; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm2[0],ymm0[0],ymm2[1],ymm0[1],ymm2[2],ymm0[2],ymm2[3],ymm0[3],ymm2[4],ymm0[4],ymm2[5],ymm0[5],ymm2[6],ymm0[6],ymm2[7],ymm0[7],ymm2[16],ymm0[16],ymm2[17],ymm0[17],ymm2[18],ymm0[18],ymm2[19],ymm0[19],ymm2[20],ymm0[20],ymm2[21],ymm0[21],ymm2[22],ymm0[22],ymm2[23],ymm0[23]
+; AVX2-NEXT:    vpmullw %ymm2, %ymm0, %ymm0
 ; AVX2-NEXT:    vpand %ymm4, %ymm0, %ymm0
 ; AVX2-NEXT:    vpackuswb %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
-; AVX2-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
-; AVX2-NEXT:    vpmullw %ymm0, %ymm2, %ymm0
+; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm2
+; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm2[0],ymm0[0],ymm2[1],ymm0[1],ymm2[2],ymm0[2],ymm2[3],ymm0[3],ymm2[4],ymm0[4],ymm2[5],ymm0[5],ymm2[6],ymm0[6],ymm2[7],ymm0[7],ymm2[16],ymm0[16],ymm2[17],ymm0[17],ymm2[18],ymm0[18],ymm2[19],ymm0[19],ymm2[20],ymm0[20],ymm2[21],ymm0[21],ymm2[22],ymm0[22],ymm2[23],ymm0[23]
+; AVX2-NEXT:    vpmullw %ymm2, %ymm0, %ymm0
 ; AVX2-NEXT:    vpand %ymm4, %ymm0, %ymm0
 ; AVX2-NEXT:    vpackuswb %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vpsrlw $8, %xmm0, %xmm2
 ; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm2[0],ymm0[0],ymm2[1],ymm0[1],ymm2[2],ymm0[2],ymm2[3],ymm0[3],ymm2[4],ymm0[4],ymm2[5],ymm0[5],ymm2[6],ymm0[6],ymm2[7],ymm0[7],ymm2[16],ymm0[16],ymm2[17],ymm0[17],ymm2[18],ymm0[18],ymm2[19],ymm0[19],ymm2[20],ymm0[20],ymm2[21],ymm0[21],ymm2[22],ymm0[22],ymm2[23],ymm0[23]
 ; AVX2-NEXT:    vpmullw %ymm2, %ymm0, %ymm0
 ; AVX2-NEXT:    vpand %ymm4, %ymm0, %ymm0
 ; AVX2-NEXT:    vpackuswb %ymm1, %ymm0, %ymm0
@@ -3095,56 +3098,57 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; AVX512BW:       # %bb.0:
 ; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} zmm2 = zmm1[8],zmm0[8],zmm1[9],zmm0[9],zmm1[10],zmm0[10],zmm1[11],zmm0[11],zmm1[12],zmm0[12],zmm1[13],zmm0[13],zmm1[14],zmm0[14],zmm1[15],zmm0[15],zmm1[24],zmm0[24],zmm1[25],zmm0[25],zmm1[26],zmm0[26],zmm1[27],zmm0[27],zmm1[28],zmm0[28],zmm1[29],zmm0[29],zmm1[30],zmm0[30],zmm1[31],zmm0[31],zmm1[40],zmm0[40],zmm1[41],zmm0[41],zmm1[42],zmm0[42],zmm1[43],zmm0[43],zmm1[44],zmm0[44],zmm1[45],zmm0[45],zmm1[46],zmm0[46],zmm1[47],zmm0[47],zmm1[56],zmm0[56],zmm1[57],zmm0[57],zmm1[58],zmm0[58],zmm1[59],zmm0[59],zmm1[60],zmm0[60],zmm1[61],zmm0[61],zmm1[62],zmm0[62],zmm1[63],zmm0[63]
 ; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} zmm3 = zmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]
-; AVX512BW-NEXT:    vpmullw %zmm2, %zmm3, %zmm3
-; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
-; AVX512BW-NEXT:    vpandq %zmm2, %zmm3, %zmm3
+; AVX512BW-NEXT:    vpmullw %zmm2, %zmm3, %zmm2
+; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; AVX512BW-NEXT:    vpandq %zmm3, %zmm2, %zmm2
 ; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} zmm1 = zmm1[0],zmm0[0],zmm1[1],zmm0[1],zmm1[2],zmm0[2],zmm1[3],zmm0[3],zmm1[4],zmm0[4],zmm1[5],zmm0[5],zmm1[6],zmm0[6],zmm1[7],zmm0[7],zmm1[16],zmm0[16],zmm1[17],zmm0[17],zmm1[18],zmm0[18],zmm1[19],zmm0[19],zmm1[20],zmm0[20],zmm1[21],zmm0[21],zmm1[22],zmm0[22],zmm1[23],zmm0[23],zmm1[32],zmm0[32],zmm1[33],zmm0[33],zmm1[34],zmm0[34],zmm1[35],zmm0[35],zmm1[36],zmm0[36],zmm1[37],zmm0[37],zmm1[38],zmm0[38],zmm1[39],zmm0[39],zmm1[48],zmm0[48],zmm1[49],zmm0[49],zmm1[50],zmm0[50],zmm1[51],zmm0[51],zmm1[52],zmm0[52],zmm1[53],zmm0[53],zmm1[54],zmm0[54],zmm1[55],zmm0[55]
 ; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} zmm0 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
 ; AVX512BW-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpandq %zmm2, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpackuswb %zmm3, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpandq %zmm3, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpackuswb %zmm2, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
-; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} zmm3 = zmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]
+; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} zmm2 = zmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]
 ; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} zmm4 = zmm1[8],zmm0[8],zmm1[9],zmm0[9],zmm1[10],zmm0[10],zmm1[11],zmm0[11],zmm1[12],zmm0[12],zmm1[13],zmm0[13],zmm1[14],zmm0[14],zmm1[15],zmm0[15],zmm1[24],zmm0[24],zmm1[25],zmm0[25],zmm1[26],zmm0[26],zmm1[27],zmm0[27],zmm1[28],zmm0[28],zmm1[29],zmm0[29],zmm1[30],zmm0[30],zmm1[31],zmm0[31],zmm1[40],zmm0[40],zmm1[41],zmm0[41],zmm1[42],zmm0[42],zmm1[43],zmm0[43],zmm1[44],zmm0[44],zmm1[45],zmm0[45],zmm1[46],zmm0[46],zmm1[47],zmm0[47],zmm1[56],zmm0[56],zmm1[57],zmm0[57],zmm1[58],zmm0[58],zmm1[59],zmm0[59],zmm1[60],zmm0[60],zmm1[61],zmm0[61],zmm1[62],zmm0[62],zmm1[63],zmm0[63]
-; AVX512BW-NEXT:    vpmullw %zmm4, %zmm3, %zmm3
-; AVX512BW-NEXT:    vpandq %zmm2, %zmm3, %zmm3
+; AVX512BW-NEXT:    vpmullw %zmm4, %zmm2, %zmm2
+; AVX512BW-NEXT:    vpandq %zmm3, %zmm2, %zmm2
 ; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} zmm0 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
 ; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} zmm1 = zmm1[0],zmm0[0],zmm1[1],zmm0[1],zmm1[2],zmm0[2],zmm1[3],zmm0[3],zmm1[4],zmm0[4],zmm1[5],zmm0[5],zmm1[6],zmm0[6],zmm1[7],zmm0[7],zmm1[16],zmm0[16],zmm1[17],zmm0[17],zmm1[18],zmm0[18],zmm1[19],zmm0[19],zmm1[20],zmm0[20],zmm1[21],zmm0[21],zmm1[22],zmm0[22],zmm1[23],zmm0[23],zmm1[32],zmm0[32],zmm1[33],zmm0[33],zmm1[34],zmm0[34],zmm1[35],zmm0[35],zmm1[36],zmm0[36],zmm1[37],zmm0[37],zmm1[38],zmm0[38],zmm1[39],zmm0[39],zmm1[48],zmm0[48],zmm1[49],zmm0[49],zmm1[50],zmm0[50],zmm1[51],zmm0[51],zmm1[52],zmm0[52],zmm1[53],zmm0[53],zmm1[54],zmm0[54],zmm1[55],zmm0[55]
 ; AVX512BW-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpandq %zmm2, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpackuswb %zmm3, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpandq %zmm3, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpackuswb %zmm2, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} zmm3 = zmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]
+; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} zmm2 = zmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]
 ; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} zmm4 = zmm1[8],zmm0[8],zmm1[9],zmm0[9],zmm1[10],zmm0[10],zmm1[11],zmm0[11],zmm1[12],zmm0[12],zmm1[13],zmm0[13],zmm1[14],zmm0[14],zmm1[15],zmm0[15],zmm1[24],zmm0[24],zmm1[25],zmm0[25],zmm1[26],zmm0[26],zmm1[27],zmm0[27],zmm1[28],zmm0[28],zmm1[29],zmm0[29],zmm1[30],zmm0[30],zmm1[31],zmm0[31],zmm1[40],zmm0[40],zmm1[41],zmm0[41],zmm1[42],zmm0[42],zmm1[43],zmm0[43],zmm1[44],zmm0[44],zmm1[45],zmm0[45],zmm1[46],zmm0[46],zmm1[47],zmm0[47],zmm1[56],zmm0[56],zmm1[57],zmm0[57],zmm1[58],zmm0[58],zmm1[59],zmm0[59],zmm1[60],zmm0[60],zmm1[61],zmm0[61],zmm1[62],zmm0[62],zmm1[63],zmm0[63]
-; AVX512BW-NEXT:    vpmullw %zmm4, %zmm3, %zmm3
-; AVX512BW-NEXT:    vpandq %zmm2, %zmm3, %zmm3
+; AVX512BW-NEXT:    vpmullw %zmm4, %zmm2, %zmm2
+; AVX512BW-NEXT:    vpandq %zmm3, %zmm2, %zmm2
 ; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} zmm0 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
 ; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} zmm1 = zmm1[0],zmm0[0],zmm1[1],zmm0[1],zmm1[2],zmm0[2],zmm1[3],zmm0[3],zmm1[4],zmm0[4],zmm1[5],zmm0[5],zmm1[6],zmm0[6],zmm1[7],zmm0[7],zmm1[16],zmm0[16],zmm1[17],zmm0[17],zmm1[18],zmm0[18],zmm1[19],zmm0[19],zmm1[20],zmm0[20],zmm1[21],zmm0[21],zmm1[22],zmm0[22],zmm1[23],zmm0[23],zmm1[32],zmm0[32],zmm1[33],zmm0[33],zmm1[34],zmm0[34],zmm1[35],zmm0[35],zmm1[36],zmm0[36],zmm1[37],zmm0[37],zmm1[38],zmm0[38],zmm1[39],zmm0[39],zmm1[48],zmm0[48],zmm1[49],zmm0[49],zmm1[50],zmm0[50],zmm1[51],zmm0[51],zmm1[52],zmm0[52],zmm1[53],zmm0[53],zmm1[54],zmm0[54],zmm1[55],zmm0[55]
 ; AVX512BW-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpandq %zmm2, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpackuswb %zmm3, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} zmm1 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
-; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; AVX512BW-NEXT:    vpmullw %zmm0, %zmm1, %zmm0
-; AVX512BW-NEXT:    vpandq %zmm2, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpandq %zmm3, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpackuswb %zmm2, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} zmm1 = zmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]
+; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} zmm0 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
+; AVX512BW-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpandq %zmm3, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vpxor %xmm1, %xmm1, %xmm1
 ; AVX512BW-NEXT:    vpackuswb %zmm1, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} zmm3 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
-; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; AVX512BW-NEXT:    vpmullw %zmm0, %zmm3, %zmm0
-; AVX512BW-NEXT:    vpandq %zmm2, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[1,1,2,3]
+; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} zmm0 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
+; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} zmm2 = zmm2[0],zmm0[0],zmm2[1],zmm0[1],zmm2[2],zmm0[2],zmm2[3],zmm0[3],zmm2[4],zmm0[4],zmm2[5],zmm0[5],zmm2[6],zmm0[6],zmm2[7],zmm0[7],zmm2[16],zmm0[16],zmm2[17],zmm0[17],zmm2[18],zmm0[18],zmm2[19],zmm0[19],zmm2[20],zmm0[20],zmm2[21],zmm0[21],zmm2[22],zmm0[22],zmm2[23],zmm0[23],zmm2[32],zmm0[32],zmm2[33],zmm0[33],zmm2[34],zmm0[34],zmm2[35],zmm0[35],zmm2[36],zmm0[36],zmm2[37],zmm0[37],zmm2[38],zmm0[38],zmm2[39],zmm0[39],zmm2[48],zmm0[48],zmm2[49],zmm0[49],zmm2[50],zmm0[50],zmm2[51],zmm0[51],zmm2[52],zmm0[52],zmm2[53],zmm0[53],zmm2[54],zmm0[54],zmm2[55],zmm0[55]
+; AVX512BW-NEXT:    vpmullw %zmm2, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpandq %zmm3, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vpackuswb %zmm1, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} zmm3 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
-; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
-; AVX512BW-NEXT:    vpmullw %zmm0, %zmm3, %zmm0
-; AVX512BW-NEXT:    vpandq %zmm2, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpsrld $16, %xmm0, %xmm2
+; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} zmm0 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
+; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} zmm2 = zmm2[0],zmm0[0],zmm2[1],zmm0[1],zmm2[2],zmm0[2],zmm2[3],zmm0[3],zmm2[4],zmm0[4],zmm2[5],zmm0[5],zmm2[6],zmm0[6],zmm2[7],zmm0[7],zmm2[16],zmm0[16],zmm2[17],zmm0[17],zmm2[18],zmm0[18],zmm2[19],zmm0[19],zmm2[20],zmm0[20],zmm2[21],zmm0[21],zmm2[22],zmm0[22],zmm2[23],zmm0[23],zmm2[32],zmm0[32],zmm2[33],zmm0[33],zmm2[34],zmm0[34],zmm2[35],zmm0[35],zmm2[36],zmm0[36],zmm2[37],zmm0[37],zmm2[38],zmm0[38],zmm2[39],zmm0[39],zmm2[48],zmm0[48],zmm2[49],zmm0[49],zmm2[50],zmm0[50],zmm2[51],zmm0[51],zmm2[52],zmm0[52],zmm2[53],zmm0[53],zmm2[54],zmm0[54],zmm2[55],zmm0[55]
+; AVX512BW-NEXT:    vpmullw %zmm2, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpandq %zmm3, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vpackuswb %zmm1, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpsrlw $8, %xmm0, %xmm3
+; AVX512BW-NEXT:    vpsrlw $8, %xmm0, %xmm2
 ; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} zmm0 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
-; AVX512BW-NEXT:    vpmullw %zmm3, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpandq %zmm2, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} zmm2 = zmm2[0],zmm0[0],zmm2[1],zmm0[1],zmm2[2],zmm0[2],zmm2[3],zmm0[3],zmm2[4],zmm0[4],zmm2[5],zmm0[5],zmm2[6],zmm0[6],zmm2[7],zmm0[7],zmm2[16],zmm0[16],zmm2[17],zmm0[17],zmm2[18],zmm0[18],zmm2[19],zmm0[19],zmm2[20],zmm0[20],zmm2[21],zmm0[21],zmm2[22],zmm0[22],zmm2[23],zmm0[23],zmm2[32],zmm0[32],zmm2[33],zmm0[33],zmm2[34],zmm0[34],zmm2[35],zmm0[35],zmm2[36],zmm0[36],zmm2[37],zmm0[37],zmm2[38],zmm0[38],zmm2[39],zmm0[39],zmm2[48],zmm0[48],zmm2[49],zmm0[49],zmm2[50],zmm0[50],zmm2[51],zmm0[51],zmm2[52],zmm0[52],zmm2[53],zmm0[53],zmm2[54],zmm0[54],zmm2[55],zmm0[55]
+; AVX512BW-NEXT:    vpmullw %zmm2, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpandq %zmm3, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vpackuswb %zmm1, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vpextrb $0, %xmm0, %eax
 ; AVX512BW-NEXT:    # kill: def $al killed $al killed $eax
@@ -3155,56 +3159,57 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; AVX512BWVL:       # %bb.0:
 ; AVX512BWVL-NEXT:    vpunpckhbw {{.*#+}} zmm2 = zmm1[8],zmm0[8],zmm1[9],zmm0[9],zmm1[10],zmm0[10],zmm1[11],zmm0[11],zmm1[12],zmm0[12],zmm1[13],zmm0[13],zmm1[14],zmm0[14],zmm1[15],zmm0[15],zmm1[24],zmm0[24],zmm1[25],zmm0[25],zmm1[26],zmm0[26],zmm1[27],zmm0[27],zmm1[28],zmm0[28],zmm1[29],zmm0[29],zmm1[30],zmm0[30],zmm1[31],zmm0[31],zmm1[40],zmm0[40],zmm1[41],zmm0[41],zmm1[42],zmm0[42],zmm1[43],zmm0[43],zmm1[44],zmm0[44],zmm1[45],zmm0[45],zmm1[46],zmm0[46],zmm1[47],zmm0[47],zmm1[56],zmm0[56],zmm1[57],zmm0[57],zmm1[58],zmm0[58],zmm1[59],zmm0[59],zmm1[60],zmm0[60],zmm1[61],zmm0[61],zmm1[62],zmm0[62],zmm1[63],zmm0[63]
 ; AVX512BWVL-NEXT:    vpunpckhbw {{.*#+}} zmm3 = zmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]
-; AVX512BWVL-NEXT:    vpmullw %zmm2, %zmm3, %zmm3
-; AVX512BWVL-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
-; AVX512BWVL-NEXT:    vpandq %zmm2, %zmm3, %zmm3
+; AVX512BWVL-NEXT:    vpmullw %zmm2, %zmm3, %zmm2
+; AVX512BWVL-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; AVX512BWVL-NEXT:    vpandq %zmm3, %zmm2, %zmm2
 ; AVX512BWVL-NEXT:    vpunpcklbw {{.*#+}} zmm1 = zmm1[0],zmm0[0],zmm1[1],zmm0[1],zmm1[2],zmm0[2],zmm1[3],zmm0[3],zmm1[4],zmm0[4],zmm1[5],zmm0[5],zmm1[6],zmm0[6],zmm1[7],zmm0[7],zmm1[16],zmm0[16],zmm1[17],zmm0[17],zmm1[18],zmm0[18],zmm1[19],zmm0[19],zmm1[20],zmm0[20],zmm1[21],zmm0[21],zmm1[22],zmm0[22],zmm1[23],zmm0[23],zmm1[32],zmm0[32],zmm1[33],zmm0[33],zmm1[34],zmm0[34],zmm1[35],zmm0[35],zmm1[36],zmm0[36],zmm1[37],zmm0[37],zmm1[38],zmm0[38],zmm1[39],zmm0[39],zmm1[48],zmm0[48],zmm1[49],zmm0[49],zmm1[50],zmm0[50],zmm1[51],zmm0[51],zmm1[52],zmm0[52],zmm1[53],zmm0[53],zmm1[54],zmm0[54],zmm1[55],zmm0[55]
 ; AVX512BWVL-NEXT:    vpunpcklbw {{.*#+}} zmm0 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
 ; AVX512BWVL-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
-; AVX512BWVL-NEXT:    vpandq %zmm2, %zmm0, %zmm0
-; AVX512BWVL-NEXT:    vpackuswb %zmm3, %zmm0, %zmm0
+; AVX512BWVL-NEXT:    vpandq %zmm3, %zmm0, %zmm0
+; AVX512BWVL-NEXT:    vpackuswb %zmm2, %zmm0, %zmm0
 ; AVX512BWVL-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
-; AVX512BWVL-NEXT:    vpunpckhbw {{.*#+}} zmm3 = zmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]
+; AVX512BWVL-NEXT:    vpunpckhbw {{.*#+}} zmm2 = zmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]
 ; AVX512BWVL-NEXT:    vpunpckhbw {{.*#+}} zmm4 = zmm1[8],zmm0[8],zmm1[9],zmm0[9],zmm1[10],zmm0[10],zmm1[11],zmm0[11],zmm1[12],zmm0[12],zmm1[13],zmm0[13],zmm1[14],zmm0[14],zmm1[15],zmm0[15],zmm1[24],zmm0[24],zmm1[25],zmm0[25],zmm1[26],zmm0[26],zmm1[27],zmm0[27],zmm1[28],zmm0[28],zmm1[29],zmm0[29],zmm1[30],zmm0[30],zmm1[31],zmm0[31],zmm1[40],zmm0[40],zmm1[41],zmm0[41],zmm1[42],zmm0[42],zmm1[43],zmm0[43],zmm1[44],zmm0[44],zmm1[45],zmm0[45],zmm1[46],zmm0[46],zmm1[47],zmm0[47],zmm1[56],zmm0[56],zmm1[57],zmm0[57],zmm1[58],zmm0[58],zmm1[59],zmm0[59],zmm1[60],zmm0[60],zmm1[61],zmm0[61],zmm1[62],zmm0[62],zmm1[63],zmm0[63]
-; AVX512BWVL-NEXT:    vpmullw %zmm4, %zmm3, %zmm3
-; AVX512BWVL-NEXT:    vpandq %zmm2, %zmm3, %zmm3
+; AVX512BWVL-NEXT:    vpmullw %zmm4, %zmm2, %zmm2
+; AVX512BWVL-NEXT:    vpandq %zmm3, %zmm2, %zmm2
 ; AVX512BWVL-NEXT:    vpunpcklbw {{.*#+}} zmm0 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
 ; AVX512BWVL-NEXT:    vpunpcklbw {{.*#+}} zmm1 = zmm1[0],zmm0[0],zmm1[1],zmm0[1],zmm1[2],zmm0[2],zmm1[3],zmm0[3],zmm1[4],zmm0[4],zmm1[5],zmm0[5],zmm1[6],zmm0[6],zmm1[7],zmm0[7],zmm1[16],zmm0[16],zmm1[17],zmm0[17],zmm1[18],zmm0[18],zmm1[19],zmm0[19],zmm1[20],zmm0[20],zmm1[21],zmm0[21],zmm1[22],zmm0[22],zmm1[23],zmm0[23],zmm1[32],zmm0[32],zmm1[33],zmm0[33],zmm1[34],zmm0[34],zmm1[35],zmm0[35],zmm1[36],zmm0[36],zmm1[37],zmm0[37],zmm1[38],zmm0[38],zmm1[39],zmm0[39],zmm1[48],zmm0[48],zmm1[49],zmm0[49],zmm1[50],zmm0[50],zmm1[51],zmm0[51],zmm1[52],zmm0[52],zmm1[53],zmm0[53],zmm1[54],zmm0[54],zmm1[55],zmm0[55]
 ; AVX512BWVL-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
-; AVX512BWVL-NEXT:    vpandq %zmm2, %zmm0, %zmm0
-; AVX512BWVL-NEXT:    vpackuswb %zmm3, %zmm0, %zmm0
+; AVX512BWVL-NEXT:    vpandq %zmm3, %zmm0, %zmm0
+; AVX512BWVL-NEXT:    vpackuswb %zmm2, %zmm0, %zmm0
 ; AVX512BWVL-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512BWVL-NEXT:    vpunpckhbw {{.*#+}} zmm3 = zmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]
+; AVX512BWVL-NEXT:    vpunpckhbw {{.*#+}} zmm2 = zmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]
 ; AVX512BWVL-NEXT:    vpunpckhbw {{.*#+}} zmm4 = zmm1[8],zmm0[8],zmm1[9],zmm0[9],zmm1[10],zmm0[10],zmm1[11],zmm0[11],zmm1[12],zmm0[12],zmm1[13],zmm0[13],zmm1[14],zmm0[14],zmm1[15],zmm0[15],zmm1[24],zmm0[24],zmm1[25],zmm0[25],zmm1[26],zmm0[26],zmm1[27],zmm0[27],zmm1[28],zmm0[28],zmm1[29],zmm0[29],zmm1[30],zmm0[30],zmm1[31],zmm0[31],zmm1[40],zmm0[40],zmm1[41],zmm0[41],zmm1[42],zmm0[42],zmm1[43],zmm0[43],zmm1[44],zmm0[44],zmm1[45],zmm0[45],zmm1[46],zmm0[46],zmm1[47],zmm0[47],zmm1[56],zmm0[56],zmm1[57],zmm0[57],zmm1[58],zmm0[58],zmm1[59],zmm0[59],zmm1[60],zmm0[60],zmm1[61],zmm0[61],zmm1[62],zmm0[62],zmm1[63],zmm0[63]
-; AVX512BWVL-NEXT:    vpmullw %zmm4, %zmm3, %zmm3
-; AVX512BWVL-NEXT:    vpandq %zmm2, %zmm3, %zmm3
+; AVX512BWVL-NEXT:    vpmullw %zmm4, %zmm2, %zmm2
+; AVX512BWVL-NEXT:    vpandq %zmm3, %zmm2, %zmm2
 ; AVX512BWVL-NEXT:    vpunpcklbw {{.*#+}} zmm0 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
 ; AVX512BWVL-NEXT:    vpunpcklbw {{.*#+}} zmm1 = zmm1[0],zmm0[0],zmm1[1],zmm0[1],zmm1[2],zmm0[2],zmm1[3],zmm0[3],zmm1[4],zmm0[4],zmm1[5],zmm0[5],zmm1[6],zmm0[6],zmm1[7],zmm0[7],zmm1[16],zmm0[16],zmm1[17],zmm0[17],zmm1[18],zmm0[18],zmm1[19],zmm0[19],zmm1[20],zmm0[20],zmm1[21],zmm0[21],zmm1[22],zmm0[22],zmm1[23],zmm0[23],zmm1[32],zmm0[32],zmm1[33],zmm0[33],zmm1[34],zmm0[34],zmm1[35],zmm0[35],zmm1[36],zmm0[36],zmm1[37],zmm0[37],zmm1[38],zmm0[38],zmm1[39],zmm0[39],zmm1[48],zmm0[48],zmm1[49],zmm0[49],zmm1[50],zmm0[50],zmm1[51],zmm0[51],zmm1[52],zmm0[52],zmm1[53],zmm0[53],zmm1[54],zmm0[54],zmm1[55],zmm0[55]
 ; AVX512BWVL-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
-; AVX512BWVL-NEXT:    vpandq %zmm2, %zmm0, %zmm0
-; AVX512BWVL-NEXT:    vpackuswb %zmm3, %zmm0, %zmm0
-; AVX512BWVL-NEXT:    vpunpcklbw {{.*#+}} zmm1 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
-; AVX512BWVL-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; AVX512BWVL-NEXT:    vpmullw %zmm0, %zmm1, %zmm0
-; AVX512BWVL-NEXT:    vpandq %zmm2, %zmm0, %zmm0
+; AVX512BWVL-NEXT:    vpandq %zmm3, %zmm0, %zmm0
+; AVX512BWVL-NEXT:    vpackuswb %zmm2, %zmm0, %zmm0
+; AVX512BWVL-NEXT:    vpunpckhbw {{.*#+}} zmm1 = zmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]
+; AVX512BWVL-NEXT:    vpunpcklbw {{.*#+}} zmm0 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
+; AVX512BWVL-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
+; AVX512BWVL-NEXT:    vpandq %zmm3, %zmm0, %zmm0
 ; AVX512BWVL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
 ; AVX512BWVL-NEXT:    vpackuswb %zmm1, %zmm0, %zmm0
-; AVX512BWVL-NEXT:    vpunpcklbw {{.*#+}} zmm3 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
-; AVX512BWVL-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; AVX512BWVL-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; AVX512BWVL-NEXT:    vpmullw %zmm0, %zmm3, %zmm0
-; AVX512BWVL-NEXT:    vpandq %zmm2, %zmm0, %zmm0
+; AVX512BWVL-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[1,1,2,3]
+; AVX512BWVL-NEXT:    vpunpcklbw {{.*#+}} zmm0 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
+; AVX512BWVL-NEXT:    vpunpcklbw {{.*#+}} zmm2 = zmm2[0],zmm0[0],zmm2[1],zmm0[1],zmm2[2],zmm0[2],zmm2[3],zmm0[3],zmm2[4],zmm0[4],zmm2[5],zmm0[5],zmm2[6],zmm0[6],zmm2[7],zmm0[7],zmm2[16],zmm0[16],zmm2[17],zmm0[17],zmm2[18],zmm0[18],zmm2[19],zmm0[19],zmm2[20],zmm0[20],zmm2[21],zmm0[21],zmm2[22],zmm0[22],zmm2[23],zmm0[23],zmm2[32],zmm0[32],zmm2[33],zmm0[33],zmm2[34],zmm0[34],zmm2[35],zmm0[35],zmm2[36],zmm0[36],zmm2[37],zmm0[37],zmm2[38],zmm0[38],zmm2[39],zmm0[39],zmm2[48],zmm0[48],zmm2[49],zmm0[49],zmm2[50],zmm0[50],zmm2[51],zmm0[51],zmm2[52],zmm0[52],zmm2[53],zmm0[53],zmm2[54],zmm0[54],zmm2[55],zmm0[55]
+; AVX512BWVL-NEXT:    vpmullw %zmm2, %zmm0, %zmm0
+; AVX512BWVL-NEXT:    vpandq %zmm3, %zmm0, %zmm0
 ; AVX512BWVL-NEXT:    vpackuswb %zmm1, %zmm0, %zmm0
-; AVX512BWVL-NEXT:    vpunpcklbw {{.*#+}} zmm3 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
-; AVX512BWVL-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; AVX512BWVL-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
-; AVX512BWVL-NEXT:    vpmullw %zmm0, %zmm3, %zmm0
-; AVX512BWVL-NEXT:    vpandq %zmm2, %zmm0, %zmm0
+; AVX512BWVL-NEXT:    vpsrld $16, %xmm0, %xmm2
+; AVX512BWVL-NEXT:    vpunpcklbw {{.*#+}} zmm0 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
+; AVX512BWVL-NEXT:    vpunpcklbw {{.*#+}} zmm2 = zmm2[0],zmm0[0],zmm2[1],zmm0[1],zmm2[2],zmm0[2],zmm2[3],zmm0[3],zmm2[4],zmm0[4],zmm2[5],zmm0[5],zmm2[6],zmm0[6],zmm2[7],zmm0[7],zmm2[16],zmm0[16],zmm2[17],zmm0[17],zmm2[18],zmm0[18],zmm2[19],zmm0[19],zmm2[20],zmm0[20],zmm2[21],zmm0[21],zmm2[22],zmm0[22],zmm2[23],zmm0[23],zmm2[32],zmm0[32],zmm2[33],zmm0[33],zmm2[34],zmm0[34],zmm2[35],zmm0[35],zmm2[36],zmm0[36],zmm2[37],zmm0[37],zmm2[38],zmm0[38],zmm2[39],zmm0[39],zmm2[48],zmm0[48],zmm2[49],zmm0[49],zmm2[50],zmm0[50],zmm2[51],zmm0[51],zmm2[52],zmm0[52],zmm2[53],zmm0[53],zmm2[54],zmm0[54],zmm2[55],zmm0[55]
+; AVX512BWVL-NEXT:    vpmullw %zmm2, %zmm0, %zmm0
+; AVX512BWVL-NEXT:    vpandq %zmm3, %zmm0, %zmm0
 ; AVX512BWVL-NEXT:    vpackuswb %zmm1, %zmm0, %zmm0
-; AVX512BWVL-NEXT:    vpsrlw $8, %xmm0, %xmm3
+; AVX512BWVL-NEXT:    vpsrlw $8, %xmm0, %xmm2
 ; AVX512BWVL-NEXT:    vpunpcklbw {{.*#+}} zmm0 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
-; AVX512BWVL-NEXT:    vpmullw %zmm3, %zmm0, %zmm0
-; AVX512BWVL-NEXT:    vpandq %zmm2, %zmm0, %zmm0
+; AVX512BWVL-NEXT:    vpunpcklbw {{.*#+}} zmm2 = zmm2[0],zmm0[0],zmm2[1],zmm0[1],zmm2[2],zmm0[2],zmm2[3],zmm0[3],zmm2[4],zmm0[4],zmm2[5],zmm0[5],zmm2[6],zmm0[6],zmm2[7],zmm0[7],zmm2[16],zmm0[16],zmm2[17],zmm0[17],zmm2[18],zmm0[18],zmm2[19],zmm0[19],zmm2[20],zmm0[20],zmm2[21],zmm0[21],zmm2[22],zmm0[22],zmm2[23],zmm0[23],zmm2[32],zmm0[32],zmm2[33],zmm0[33],zmm2[34],zmm0[34],zmm2[35],zmm0[35],zmm2[36],zmm0[36],zmm2[37],zmm0[37],zmm2[38],zmm0[38],zmm2[39],zmm0[39],zmm2[48],zmm0[48],zmm2[49],zmm0[49],zmm2[50],zmm0[50],zmm2[51],zmm0[51],zmm2[52],zmm0[52],zmm2[53],zmm0[53],zmm2[54],zmm0[54],zmm2[55],zmm0[55]
+; AVX512BWVL-NEXT:    vpmullw %zmm2, %zmm0, %zmm0
+; AVX512BWVL-NEXT:    vpandq %zmm3, %zmm0, %zmm0
 ; AVX512BWVL-NEXT:    vpackuswb %zmm1, %zmm0, %zmm0
 ; AVX512BWVL-NEXT:    vpextrb $0, %xmm0, %eax
 ; AVX512BWVL-NEXT:    # kill: def $al killed $al killed $eax
@@ -3251,26 +3256,27 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; AVX512DQ-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vpand %ymm4, %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vpackuswb %ymm2, %ymm0, %ymm0
-; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
-; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; AVX512DQ-NEXT:    vpmullw %ymm0, %ymm1, %ymm0
+; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} ymm1 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
+; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX512DQ-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vpand %ymm4, %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vpxor %xmm1, %xmm1, %xmm1
 ; AVX512DQ-NEXT:    vpackuswb %ymm1, %ymm0, %ymm0
-; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
-; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; AVX512DQ-NEXT:    vpmullw %ymm0, %ymm2, %ymm0
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[1,1,2,3]
+; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm2[0],ymm0[0],ymm2[1],ymm0[1],ymm2[2],ymm0[2],ymm2[3],ymm0[3],ymm2[4],ymm0[4],ymm2[5],ymm0[5],ymm2[6],ymm0[6],ymm2[7],ymm0[7],ymm2[16],ymm0[16],ymm2[17],ymm0[17],ymm2[18],ymm0[18],ymm2[19],ymm0[19],ymm2[20],ymm0[20],ymm2[21],ymm0[21],ymm2[22],ymm0[22],ymm2[23],ymm0[23]
+; AVX512DQ-NEXT:    vpmullw %ymm2, %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vpand %ymm4, %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vpackuswb %ymm1, %ymm0, %ymm0
-; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
-; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
-; AVX512DQ-NEXT:    vpmullw %ymm0, %ymm2, %ymm0
+; AVX512DQ-NEXT:    vpsrld $16, %xmm0, %xmm2
+; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm2[0],ymm0[0],ymm2[1],ymm0[1],ymm2[2],ymm0[2],ymm2[3],ymm0[3],ymm2[4],ymm0[4],ymm2[5],ymm0[5],ymm2[6],ymm0[6],ymm2[7],ymm0[7],ymm2[16],ymm0[16],ymm2[17],ymm0[17],ymm2[18],ymm0[18],ymm2[19],ymm0[19],ymm2[20],ymm0[20],ymm2[21],ymm0[21],ymm2[22],ymm0[22],ymm2[23],ymm0[23]
+; AVX512DQ-NEXT:    vpmullw %ymm2, %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vpand %ymm4, %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vpackuswb %ymm1, %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vpsrlw $8, %xmm0, %xmm2
 ; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm2[0],ymm0[0],ymm2[1],ymm0[1],ymm2[2],ymm0[2],ymm2[3],ymm0[3],ymm2[4],ymm0[4],ymm2[5],ymm0[5],ymm2[6],ymm0[6],ymm2[7],ymm0[7],ymm2[16],ymm0[16],ymm2[17],ymm0[17],ymm2[18],ymm0[18],ymm2[19],ymm0[19],ymm2[20],ymm0[20],ymm2[21],ymm0[21],ymm2[22],ymm0[22],ymm2[23],ymm0[23]
 ; AVX512DQ-NEXT:    vpmullw %ymm2, %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vpand %ymm4, %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vpackuswb %ymm1, %ymm0, %ymm0
@@ -3283,64 +3289,65 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; AVX512DQVL:       # %bb.0:
 ; AVX512DQVL-NEXT:    vpunpckhbw {{.*#+}} ymm4 = ymm2[8],ymm0[8],ymm2[9],ymm0[9],ymm2[10],ymm0[10],ymm2[11],ymm0[11],ymm2[12],ymm0[12],ymm2[13],ymm0[13],ymm2[14],ymm0[14],ymm2[15],ymm0[15],ymm2[24],ymm0[24],ymm2[25],ymm0[25],ymm2[26],ymm0[26],ymm2[27],ymm0[27],ymm2[28],ymm0[28],ymm2[29],ymm0[29],ymm2[30],ymm0[30],ymm2[31],ymm0[31]
 ; AVX512DQVL-NEXT:    vpunpckhbw {{.*#+}} ymm5 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
-; AVX512DQVL-NEXT:    vpmullw %ymm4, %ymm5, %ymm5
-; AVX512DQVL-NEXT:    vmovdqa {{.*#+}} ymm4 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
-; AVX512DQVL-NEXT:    vpand %ymm4, %ymm5, %ymm5
+; AVX512DQVL-NEXT:    vpmullw %ymm4, %ymm5, %ymm4
+; AVX512DQVL-NEXT:    vmovdqa {{.*#+}} ymm5 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; AVX512DQVL-NEXT:    vpand %ymm5, %ymm4, %ymm4
 ; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm2[0],ymm0[0],ymm2[1],ymm0[1],ymm2[2],ymm0[2],ymm2[3],ymm0[3],ymm2[4],ymm0[4],ymm2[5],ymm0[5],ymm2[6],ymm0[6],ymm2[7],ymm0[7],ymm2[16],ymm0[16],ymm2[17],ymm0[17],ymm2[18],ymm0[18],ymm2[19],ymm0[19],ymm2[20],ymm0[20],ymm2[21],ymm0[21],ymm2[22],ymm0[22],ymm2[23],ymm0[23]
 ; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
 ; AVX512DQVL-NEXT:    vpmullw %ymm2, %ymm0, %ymm0
-; AVX512DQVL-NEXT:    vpand %ymm4, %ymm0, %ymm0
-; AVX512DQVL-NEXT:    vpackuswb %ymm5, %ymm0, %ymm0
+; AVX512DQVL-NEXT:    vpand %ymm5, %ymm0, %ymm0
+; AVX512DQVL-NEXT:    vpackuswb %ymm4, %ymm0, %ymm0
 ; AVX512DQVL-NEXT:    vpunpckhbw {{.*#+}} ymm2 = ymm3[8],ymm0[8],ymm3[9],ymm0[9],ymm3[10],ymm0[10],ymm3[11],ymm0[11],ymm3[12],ymm0[12],ymm3[13],ymm0[13],ymm3[14],ymm0[14],ymm3[15],ymm0[15],ymm3[24],ymm0[24],ymm3[25],ymm0[25],ymm3[26],ymm0[26],ymm3[27],ymm0[27],ymm3[28],ymm0[28],ymm3[29],ymm0[29],ymm3[30],ymm0[30],ymm3[31],ymm0[31]
-; AVX512DQVL-NEXT:    vpunpckhbw {{.*#+}} ymm5 = ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11],ymm1[12],ymm0[12],ymm1[13],ymm0[13],ymm1[14],ymm0[14],ymm1[15],ymm0[15],ymm1[24],ymm0[24],ymm1[25],ymm0[25],ymm1[26],ymm0[26],ymm1[27],ymm0[27],ymm1[28],ymm0[28],ymm1[29],ymm0[29],ymm1[30],ymm0[30],ymm1[31],ymm0[31]
-; AVX512DQVL-NEXT:    vpmullw %ymm2, %ymm5, %ymm2
-; AVX512DQVL-NEXT:    vpand %ymm4, %ymm2, %ymm2
+; AVX512DQVL-NEXT:    vpunpckhbw {{.*#+}} ymm4 = ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11],ymm1[12],ymm0[12],ymm1[13],ymm0[13],ymm1[14],ymm0[14],ymm1[15],ymm0[15],ymm1[24],ymm0[24],ymm1[25],ymm0[25],ymm1[26],ymm0[26],ymm1[27],ymm0[27],ymm1[28],ymm0[28],ymm1[29],ymm0[29],ymm1[30],ymm0[30],ymm1[31],ymm0[31]
+; AVX512DQVL-NEXT:    vpmullw %ymm2, %ymm4, %ymm2
+; AVX512DQVL-NEXT:    vpand %ymm5, %ymm2, %ymm2
 ; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm3[0],ymm0[0],ymm3[1],ymm0[1],ymm3[2],ymm0[2],ymm3[3],ymm0[3],ymm3[4],ymm0[4],ymm3[5],ymm0[5],ymm3[6],ymm0[6],ymm3[7],ymm0[7],ymm3[16],ymm0[16],ymm3[17],ymm0[17],ymm3[18],ymm0[18],ymm3[19],ymm0[19],ymm3[20],ymm0[20],ymm3[21],ymm0[21],ymm3[22],ymm0[22],ymm3[23],ymm0[23]
 ; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[16],ymm0[16],ymm1[17],ymm0[17],ymm1[18],ymm0[18],ymm1[19],ymm0[19],ymm1[20],ymm0[20],ymm1[21],ymm0[21],ymm1[22],ymm0[22],ymm1[23],ymm0[23]
 ; AVX512DQVL-NEXT:    vpmullw %ymm3, %ymm1, %ymm1
-; AVX512DQVL-NEXT:    vpand %ymm4, %ymm1, %ymm1
+; AVX512DQVL-NEXT:    vpand %ymm5, %ymm1, %ymm1
 ; AVX512DQVL-NEXT:    vpackuswb %ymm2, %ymm1, %ymm1
 ; AVX512DQVL-NEXT:    vpunpckhbw {{.*#+}} ymm2 = ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11],ymm1[12],ymm0[12],ymm1[13],ymm0[13],ymm1[14],ymm0[14],ymm1[15],ymm0[15],ymm1[24],ymm0[24],ymm1[25],ymm0[25],ymm1[26],ymm0[26],ymm1[27],ymm0[27],ymm1[28],ymm0[28],ymm1[29],ymm0[29],ymm1[30],ymm0[30],ymm1[31],ymm0[31]
 ; AVX512DQVL-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
 ; AVX512DQVL-NEXT:    vpmullw %ymm2, %ymm3, %ymm2
-; AVX512DQVL-NEXT:    vpand %ymm4, %ymm2, %ymm2
+; AVX512DQVL-NEXT:    vpand %ymm5, %ymm2, %ymm2
 ; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[16],ymm0[16],ymm1[17],ymm0[17],ymm1[18],ymm0[18],ymm1[19],ymm0[19],ymm1[20],ymm0[20],ymm1[21],ymm0[21],ymm1[22],ymm0[22],ymm1[23],ymm0[23]
 ; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
 ; AVX512DQVL-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
-; AVX512DQVL-NEXT:    vpand %ymm4, %ymm0, %ymm0
+; AVX512DQVL-NEXT:    vpand %ymm5, %ymm0, %ymm0
 ; AVX512DQVL-NEXT:    vpackuswb %ymm2, %ymm0, %ymm0
 ; AVX512DQVL-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512DQVL-NEXT:    vpunpckhbw {{.*#+}} ymm2 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
 ; AVX512DQVL-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11],ymm1[12],ymm0[12],ymm1[13],ymm0[13],ymm1[14],ymm0[14],ymm1[15],ymm0[15],ymm1[24],ymm0[24],ymm1[25],ymm0[25],ymm1[26],ymm0[26],ymm1[27],ymm0[27],ymm1[28],ymm0[28],ymm1[29],ymm0[29],ymm1[30],ymm0[30],ymm1[31],ymm0[31]
 ; AVX512DQVL-NEXT:    vpmullw %ymm3, %ymm2, %ymm2
-; AVX512DQVL-NEXT:    vpand %ymm4, %ymm2, %ymm2
+; AVX512DQVL-NEXT:    vpand %ymm5, %ymm2, %ymm2
 ; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
 ; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[16],ymm0[16],ymm1[17],ymm0[17],ymm1[18],ymm0[18],ymm1[19],ymm0[19],ymm1[20],ymm0[20],ymm1[21],ymm0[21],ymm1[22],ymm0[22],ymm1[23],ymm0[23]
 ; AVX512DQVL-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
-; AVX512DQVL-NEXT:    vpand %ymm4, %ymm0, %ymm0
+; AVX512DQVL-NEXT:    vpand %ymm5, %ymm0, %ymm0
 ; AVX512DQVL-NEXT:    vpackuswb %ymm2, %ymm0, %ymm0
-; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
-; AVX512DQVL-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; AVX512DQVL-NEXT:    vpmullw %ymm0, %ymm1, %ymm0
-; AVX512DQVL-NEXT:    vpand %ymm4, %ymm0, %ymm0
+; AVX512DQVL-NEXT:    vpunpckhbw {{.*#+}} ymm1 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
+; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX512DQVL-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
+; AVX512DQVL-NEXT:    vpand %ymm5, %ymm0, %ymm0
 ; AVX512DQVL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
 ; AVX512DQVL-NEXT:    vpackuswb %ymm1, %ymm0, %ymm0
-; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
-; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; AVX512DQVL-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; AVX512DQVL-NEXT:    vpmullw %ymm0, %ymm2, %ymm0
-; AVX512DQVL-NEXT:    vpand %ymm4, %ymm0, %ymm0
+; AVX512DQVL-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[1,1,2,3]
+; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm2[0],ymm0[0],ymm2[1],ymm0[1],ymm2[2],ymm0[2],ymm2[3],ymm0[3],ymm2[4],ymm0[4],ymm2[5],ymm0[5],ymm2[6],ymm0[6],ymm2[7],ymm0[7],ymm2[16],ymm0[16],ymm2[17],ymm0[17],ymm2[18],ymm0[18],ymm2[19],ymm0[19],ymm2[20],ymm0[20],ymm2[21],ymm0[21],ymm2[22],ymm0[22],ymm2[23],ymm0[23]
+; AVX512DQVL-NEXT:    vpmullw %ymm2, %ymm0, %ymm0
+; AVX512DQVL-NEXT:    vpand %ymm5, %ymm0, %ymm0
 ; AVX512DQVL-NEXT:    vpackuswb %ymm1, %ymm0, %ymm0
-; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
-; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; AVX512DQVL-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
-; AVX512DQVL-NEXT:    vpmullw %ymm0, %ymm2, %ymm0
-; AVX512DQVL-NEXT:    vpand %ymm4, %ymm0, %ymm0
+; AVX512DQVL-NEXT:    vpsrld $16, %xmm0, %xmm2
+; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm2[0],ymm0[0],ymm2[1],ymm0[1],ymm2[2],ymm0[2],ymm2[3],ymm0[3],ymm2[4],ymm0[4],ymm2[5],ymm0[5],ymm2[6],ymm0[6],ymm2[7],ymm0[7],ymm2[16],ymm0[16],ymm2[17],ymm0[17],ymm2[18],ymm0[18],ymm2[19],ymm0[19],ymm2[20],ymm0[20],ymm2[21],ymm0[21],ymm2[22],ymm0[22],ymm2[23],ymm0[23]
+; AVX512DQVL-NEXT:    vpmullw %ymm2, %ymm0, %ymm0
+; AVX512DQVL-NEXT:    vpand %ymm5, %ymm0, %ymm0
 ; AVX512DQVL-NEXT:    vpackuswb %ymm1, %ymm0, %ymm0
 ; AVX512DQVL-NEXT:    vpsrlw $8, %xmm0, %xmm2
 ; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm2[0],ymm0[0],ymm2[1],ymm0[1],ymm2[2],ymm0[2],ymm2[3],ymm0[3],ymm2[4],ymm0[4],ymm2[5],ymm0[5],ymm2[6],ymm0[6],ymm2[7],ymm0[7],ymm2[16],ymm0[16],ymm2[17],ymm0[17],ymm2[18],ymm0[18],ymm2[19],ymm0[19],ymm2[20],ymm0[20],ymm2[21],ymm0[21],ymm2[22],ymm0[22],ymm2[23],ymm0[23]
 ; AVX512DQVL-NEXT:    vpmullw %ymm2, %ymm0, %ymm0
-; AVX512DQVL-NEXT:    vpand %ymm4, %ymm0, %ymm0
+; AVX512DQVL-NEXT:    vpand %ymm5, %ymm0, %ymm0
 ; AVX512DQVL-NEXT:    vpackuswb %ymm1, %ymm0, %ymm0
 ; AVX512DQVL-NEXT:    vpextrb $0, %xmm0, %eax
 ; AVX512DQVL-NEXT:    # kill: def $al killed $al killed $eax
diff --git a/test/CodeGen/X86/vector-reduce-mul.ll b/test/CodeGen/X86/vector-reduce-mul.ll
index ea80a5be2a6b9949faca34f5d87fe0ca5857a1d1..32da534d42dd63287bd59c06c18a0c02c4bced1b 100644
--- a/test/CodeGen/X86/vector-reduce-mul.ll
+++ b/test/CodeGen/X86/vector-reduce-mul.ll
@@ -131,7 +131,7 @@ define i64 @test_v4i64(<4 x i64> %a0) {
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; AVX1-NEXT:    vpsrlq $32, %xmm0, %xmm2
 ; AVX1-NEXT:    vpmuludq %xmm1, %xmm2, %xmm2
-; AVX1-NEXT:    vpsrlq $32, %xmm1, %xmm3
+; AVX1-NEXT:    vpsrldq {{.*#+}} xmm3 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
 ; AVX1-NEXT:    vpmuludq %xmm3, %xmm0, %xmm3
 ; AVX1-NEXT:    vpaddq %xmm2, %xmm3, %xmm2
 ; AVX1-NEXT:    vpsllq $32, %xmm2, %xmm2
@@ -227,7 +227,7 @@ define i64 @test_v4i64(<4 x i64> %a0) {
 ; AVX512DQVL-LABEL: test_v4i64:
 ; AVX512DQVL:       # %bb.0:
 ; AVX512DQVL-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512DQVL-NEXT:    vpmullq %ymm1, %ymm0, %ymm0
+; AVX512DQVL-NEXT:    vpmullq %xmm1, %xmm0, %xmm0
 ; AVX512DQVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; AVX512DQVL-NEXT:    vpmullq %xmm1, %xmm0, %xmm0
 ; AVX512DQVL-NEXT:    vmovq %xmm0, %rax
@@ -315,7 +315,7 @@ define i64 @test_v8i64(<8 x i64> %a0) {
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; AVX1-NEXT:    vpsrlq $32, %xmm0, %xmm2
 ; AVX1-NEXT:    vpmuludq %xmm1, %xmm2, %xmm2
-; AVX1-NEXT:    vpsrlq $32, %xmm1, %xmm3
+; AVX1-NEXT:    vpsrldq {{.*#+}} xmm3 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
 ; AVX1-NEXT:    vpmuludq %xmm3, %xmm0, %xmm3
 ; AVX1-NEXT:    vpaddq %xmm2, %xmm3, %xmm2
 ; AVX1-NEXT:    vpsllq $32, %xmm2, %xmm2
@@ -440,7 +440,7 @@ define i64 @test_v8i64(<8 x i64> %a0) {
 ; AVX512DQVL-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512DQVL-NEXT:    vpmullq %zmm1, %zmm0, %zmm0
 ; AVX512DQVL-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512DQVL-NEXT:    vpmullq %zmm1, %zmm0, %zmm0
+; AVX512DQVL-NEXT:    vpmullq %xmm1, %xmm0, %xmm0
 ; AVX512DQVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; AVX512DQVL-NEXT:    vpmullq %xmm1, %xmm0, %xmm0
 ; AVX512DQVL-NEXT:    vmovq %xmm0, %rax
@@ -602,7 +602,7 @@ define i64 @test_v16i64(<16 x i64> %a0) {
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; AVX1-NEXT:    vpsrlq $32, %xmm0, %xmm2
 ; AVX1-NEXT:    vpmuludq %xmm1, %xmm2, %xmm2
-; AVX1-NEXT:    vpsrlq $32, %xmm1, %xmm3
+; AVX1-NEXT:    vpsrldq {{.*#+}} xmm3 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
 ; AVX1-NEXT:    vpmuludq %xmm3, %xmm0, %xmm3
 ; AVX1-NEXT:    vpaddq %xmm2, %xmm3, %xmm2
 ; AVX1-NEXT:    vpsllq $32, %xmm2, %xmm2
@@ -761,7 +761,7 @@ define i64 @test_v16i64(<16 x i64> %a0) {
 ; AVX512DQVL-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512DQVL-NEXT:    vpmullq %zmm1, %zmm0, %zmm0
 ; AVX512DQVL-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512DQVL-NEXT:    vpmullq %zmm1, %zmm0, %zmm0
+; AVX512DQVL-NEXT:    vpmullq %xmm1, %xmm0, %xmm0
 ; AVX512DQVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; AVX512DQVL-NEXT:    vpmullq %xmm1, %xmm0, %xmm0
 ; AVX512DQVL-NEXT:    vmovq %xmm0, %rax
@@ -790,35 +790,12 @@ define i32 @test_v2i32(<2 x i32> %a0) {
 ; AVX-NEXT:    vmovd %xmm0, %eax
 ; AVX-NEXT:    retq
 ;
-; AVX512BW-LABEL: test_v2i32:
-; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512BW-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0
-; AVX512BW-NEXT:    vmovd %xmm0, %eax
-; AVX512BW-NEXT:    retq
-;
-; AVX512BWVL-LABEL: test_v2i32:
-; AVX512BWVL:       # %bb.0:
-; AVX512BWVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512BWVL-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0
-; AVX512BWVL-NEXT:    vmovd %xmm0, %eax
-; AVX512BWVL-NEXT:    retq
-;
-; AVX512DQ-LABEL: test_v2i32:
-; AVX512DQ:       # %bb.0:
-; AVX512DQ-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512DQ-NEXT:    vpmullq %zmm1, %zmm0, %zmm0
-; AVX512DQ-NEXT:    vmovd %xmm0, %eax
-; AVX512DQ-NEXT:    vzeroupper
-; AVX512DQ-NEXT:    retq
-;
-; AVX512DQVL-LABEL: test_v2i32:
-; AVX512DQVL:       # %bb.0:
-; AVX512DQVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512DQVL-NEXT:    vpmullq %xmm1, %xmm0, %xmm0
-; AVX512DQVL-NEXT:    vmovd %xmm0, %eax
-; AVX512DQVL-NEXT:    retq
+; AVX512-LABEL: test_v2i32:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vmovd %xmm0, %eax
+; AVX512-NEXT:    retq
   %1 = call i32 @llvm.experimental.vector.reduce.mul.i32.v2i32(<2 x i32> %a0)
   ret i32 %1
 }
@@ -916,9 +893,9 @@ define i32 @test_v8i32(<8 x i32> %a0) {
 ; AVX2-LABEL: test_v8i32:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpmulld %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX2-NEXT:    vpmulld %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX2-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
@@ -928,9 +905,9 @@ define i32 @test_v8i32(<8 x i32> %a0) {
 ; AVX512-LABEL: test_v8i32:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vpmulld %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpmulld %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
@@ -999,9 +976,9 @@ define i32 @test_v16i32(<16 x i32> %a0) {
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpmulld %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpmulld %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX2-NEXT:    vpmulld %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX2-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
@@ -1013,9 +990,9 @@ define i32 @test_v16i32(<16 x i32> %a0) {
 ; AVX512-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512-NEXT:    vpmulld %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vpmulld %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpmulld %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
@@ -1108,9 +1085,9 @@ define i32 @test_v32i32(<32 x i32> %a0) {
 ; AVX2-NEXT:    vpmulld %ymm1, %ymm2, %ymm1
 ; AVX2-NEXT:    vpmulld %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpmulld %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX2-NEXT:    vpmulld %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX2-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
@@ -1123,9 +1100,9 @@ define i32 @test_v32i32(<32 x i32> %a0) {
 ; AVX512-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512-NEXT:    vpmulld %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vpmulld %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpmulld %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512-NEXT:    vpmulld %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
@@ -1156,39 +1133,13 @@ define i16 @test_v2i16(<2 x i16> %a0) {
 ; AVX-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX-NEXT:    retq
 ;
-; AVX512BW-LABEL: test_v2i16:
-; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512BW-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0
-; AVX512BW-NEXT:    vmovd %xmm0, %eax
-; AVX512BW-NEXT:    # kill: def $ax killed $ax killed $eax
-; AVX512BW-NEXT:    retq
-;
-; AVX512BWVL-LABEL: test_v2i16:
-; AVX512BWVL:       # %bb.0:
-; AVX512BWVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512BWVL-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0
-; AVX512BWVL-NEXT:    vmovd %xmm0, %eax
-; AVX512BWVL-NEXT:    # kill: def $ax killed $ax killed $eax
-; AVX512BWVL-NEXT:    retq
-;
-; AVX512DQ-LABEL: test_v2i16:
-; AVX512DQ:       # %bb.0:
-; AVX512DQ-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512DQ-NEXT:    vpmullq %zmm1, %zmm0, %zmm0
-; AVX512DQ-NEXT:    vmovd %xmm0, %eax
-; AVX512DQ-NEXT:    # kill: def $ax killed $ax killed $eax
-; AVX512DQ-NEXT:    vzeroupper
-; AVX512DQ-NEXT:    retq
-;
-; AVX512DQVL-LABEL: test_v2i16:
-; AVX512DQVL:       # %bb.0:
-; AVX512DQVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512DQVL-NEXT:    vpmullq %xmm1, %xmm0, %xmm0
-; AVX512DQVL-NEXT:    vmovd %xmm0, %eax
-; AVX512DQVL-NEXT:    # kill: def $ax killed $ax killed $eax
-; AVX512DQVL-NEXT:    retq
+; AVX512-LABEL: test_v2i16:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vmovd %xmm0, %eax
+; AVX512-NEXT:    # kill: def $ax killed $ax killed $eax
+; AVX512-NEXT:    retq
   %1 = call i16 @llvm.experimental.vector.reduce.mul.i16.v2i16(<2 x i16> %a0)
   ret i16 %1
 }
@@ -1317,11 +1268,11 @@ define i16 @test_v16i16(<16 x i16> %a0) {
 ; AVX2-LABEL: test_v16i16:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX2-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX2-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
@@ -1332,11 +1283,11 @@ define i16 @test_v16i16(<16 x i16> %a0) {
 ; AVX512-LABEL: test_v16i16:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX512-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX512-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
@@ -1386,11 +1337,11 @@ define i16 @test_v32i16(<32 x i16> %a0) {
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX2-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX2-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
@@ -1403,11 +1354,11 @@ define i16 @test_v32i16(<32 x i16> %a0) {
 ; AVX512BW-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512BW-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512BW-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512BW-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX512BW-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512BW-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX512BW-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512BW-NEXT:    vmovd %xmm0, %eax
@@ -1420,11 +1371,11 @@ define i16 @test_v32i16(<32 x i16> %a0) {
 ; AVX512BWVL-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512BWVL-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
 ; AVX512BWVL-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512BWVL-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
+; AVX512BWVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512BWVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512BWVL-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
+; AVX512BWVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512BWVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX512BWVL-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
+; AVX512BWVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512BWVL-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX512BWVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512BWVL-NEXT:    vmovd %xmm0, %eax
@@ -1436,11 +1387,11 @@ define i16 @test_v32i16(<32 x i16> %a0) {
 ; AVX512DQ:       # %bb.0:
 ; AVX512DQ-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512DQ-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512DQ-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX512DQ-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512DQ-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX512DQ-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512DQ-NEXT:    vmovd %xmm0, %eax
@@ -1452,11 +1403,11 @@ define i16 @test_v32i16(<32 x i16> %a0) {
 ; AVX512DQVL:       # %bb.0:
 ; AVX512DQVL-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
 ; AVX512DQVL-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512DQVL-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
+; AVX512DQVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512DQVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512DQVL-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
+; AVX512DQVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512DQVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX512DQVL-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
+; AVX512DQVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512DQVL-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX512DQVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512DQVL-NEXT:    vmovd %xmm0, %eax
@@ -1518,11 +1469,11 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 ; AVX2-NEXT:    vpmullw %ymm1, %ymm2, %ymm1
 ; AVX2-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX2-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX2-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
@@ -1536,11 +1487,11 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 ; AVX512BW-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512BW-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512BW-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512BW-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX512BW-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512BW-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX512BW-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512BW-NEXT:    vmovd %xmm0, %eax
@@ -1554,11 +1505,11 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 ; AVX512BWVL-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512BWVL-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
 ; AVX512BWVL-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512BWVL-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
+; AVX512BWVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512BWVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512BWVL-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
+; AVX512BWVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512BWVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX512BWVL-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
+; AVX512BWVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512BWVL-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX512BWVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512BWVL-NEXT:    vmovd %xmm0, %eax
@@ -1572,11 +1523,11 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 ; AVX512DQ-NEXT:    vpmullw %ymm1, %ymm2, %ymm1
 ; AVX512DQ-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512DQ-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512DQ-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX512DQ-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
+; AVX512DQ-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512DQ-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX512DQ-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512DQ-NEXT:    vmovd %xmm0, %eax
@@ -1590,11 +1541,11 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 ; AVX512DQVL-NEXT:    vpmullw %ymm1, %ymm2, %ymm1
 ; AVX512DQVL-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
 ; AVX512DQVL-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512DQVL-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
+; AVX512DQVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512DQVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512DQVL-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
+; AVX512DQVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512DQVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX512DQVL-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
+; AVX512DQVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512DQVL-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX512DQVL-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX512DQVL-NEXT:    vmovd %xmm0, %eax
@@ -1634,39 +1585,13 @@ define i8 @test_v2i8(<2 x i8> %a0) {
 ; AVX-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX-NEXT:    retq
 ;
-; AVX512BW-LABEL: test_v2i8:
-; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512BW-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0
-; AVX512BW-NEXT:    vpextrb $0, %xmm0, %eax
-; AVX512BW-NEXT:    # kill: def $al killed $al killed $eax
-; AVX512BW-NEXT:    retq
-;
-; AVX512BWVL-LABEL: test_v2i8:
-; AVX512BWVL:       # %bb.0:
-; AVX512BWVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512BWVL-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0
-; AVX512BWVL-NEXT:    vpextrb $0, %xmm0, %eax
-; AVX512BWVL-NEXT:    # kill: def $al killed $al killed $eax
-; AVX512BWVL-NEXT:    retq
-;
-; AVX512DQ-LABEL: test_v2i8:
-; AVX512DQ:       # %bb.0:
-; AVX512DQ-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512DQ-NEXT:    vpmullq %zmm1, %zmm0, %zmm0
-; AVX512DQ-NEXT:    vpextrb $0, %xmm0, %eax
-; AVX512DQ-NEXT:    # kill: def $al killed $al killed $eax
-; AVX512DQ-NEXT:    vzeroupper
-; AVX512DQ-NEXT:    retq
-;
-; AVX512DQVL-LABEL: test_v2i8:
-; AVX512DQVL:       # %bb.0:
-; AVX512DQVL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512DQVL-NEXT:    vpmullq %xmm1, %xmm0, %xmm0
-; AVX512DQVL-NEXT:    vpextrb $0, %xmm0, %eax
-; AVX512DQVL-NEXT:    # kill: def $al killed $al killed $eax
-; AVX512DQVL-NEXT:    retq
+; AVX512-LABEL: test_v2i8:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX512-NEXT:    vpmuludq %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vpextrb $0, %xmm0, %eax
+; AVX512-NEXT:    # kill: def $al killed $al killed $eax
+; AVX512-NEXT:    retq
   %1 = call i8 @llvm.experimental.vector.reduce.mul.i8.v2i8(<2 x i8> %a0)
   ret i8 %1
 }
@@ -2100,28 +2025,26 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpand %xmm3, %xmm0, %xmm0
 ; AVX1-NEXT:    vpackuswb %xmm2, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
 ; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
 ; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpand %xmm3, %xmm0, %xmm0
 ; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
 ; AVX1-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[1,1,2,3]
-; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
-; AVX1-NEXT:    vpmullw %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm2 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpmullw %xmm0, %xmm2, %xmm0
 ; AVX1-NEXT:    vpand %xmm3, %xmm0, %xmm0
 ; AVX1-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm2
-; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
-; AVX1-NEXT:    vpmullw %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm2 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpmullw %xmm0, %xmm2, %xmm0
 ; AVX1-NEXT:    vpand %xmm3, %xmm0, %xmm0
 ; AVX1-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm2
 ; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
 ; AVX1-NEXT:    vpmullw %xmm2, %xmm0, %xmm0
 ; AVX1-NEXT:    vpand %xmm3, %xmm0, %xmm0
 ; AVX1-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
@@ -2143,26 +2066,27 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; AVX2-NEXT:    vpmullw %ymm2, %ymm0, %ymm0
 ; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vpackuswb %ymm3, %ymm0, %ymm0
-; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
-; AVX2-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; AVX2-NEXT:    vpmullw %ymm0, %ymm2, %ymm0
+; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm2 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
+; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX2-NEXT:    vpmullw %ymm2, %ymm0, %ymm0
 ; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
 ; AVX2-NEXT:    vpackuswb %ymm2, %ymm0, %ymm0
-; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
-; AVX2-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; AVX2-NEXT:    vpmullw %ymm0, %ymm3, %ymm0
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,2,3]
+; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm3[0],ymm0[0],ymm3[1],ymm0[1],ymm3[2],ymm0[2],ymm3[3],ymm0[3],ymm3[4],ymm0[4],ymm3[5],ymm0[5],ymm3[6],ymm0[6],ymm3[7],ymm0[7],ymm3[16],ymm0[16],ymm3[17],ymm0[17],ymm3[18],ymm0[18],ymm3[19],ymm0[19],ymm3[20],ymm0[20],ymm3[21],ymm0[21],ymm3[22],ymm0[22],ymm3[23],ymm0[23]
+; AVX2-NEXT:    vpmullw %ymm3, %ymm0, %ymm0
 ; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vpackuswb %ymm2, %ymm0, %ymm0
-; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
-; AVX2-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
-; AVX2-NEXT:    vpmullw %ymm0, %ymm3, %ymm0
+; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm3
+; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm3[0],ymm0[0],ymm3[1],ymm0[1],ymm3[2],ymm0[2],ymm3[3],ymm0[3],ymm3[4],ymm0[4],ymm3[5],ymm0[5],ymm3[6],ymm0[6],ymm3[7],ymm0[7],ymm3[16],ymm0[16],ymm3[17],ymm0[17],ymm3[18],ymm0[18],ymm3[19],ymm0[19],ymm3[20],ymm0[20],ymm3[21],ymm0[21],ymm3[22],ymm0[22],ymm3[23],ymm0[23]
+; AVX2-NEXT:    vpmullw %ymm3, %ymm0, %ymm0
 ; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vpackuswb %ymm2, %ymm0, %ymm0
 ; AVX2-NEXT:    vpsrlw $8, %xmm0, %xmm3
 ; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm3[0],ymm0[0],ymm3[1],ymm0[1],ymm3[2],ymm0[2],ymm3[3],ymm0[3],ymm3[4],ymm0[4],ymm3[5],ymm0[5],ymm3[6],ymm0[6],ymm3[7],ymm0[7],ymm3[16],ymm0[16],ymm3[17],ymm0[17],ymm3[18],ymm0[18],ymm3[19],ymm0[19],ymm3[20],ymm0[20],ymm3[21],ymm0[21],ymm3[22],ymm0[22],ymm3[23],ymm0[23]
 ; AVX2-NEXT:    vpmullw %ymm3, %ymm0, %ymm0
 ; AVX2-NEXT:    vpand %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vpackuswb %ymm2, %ymm0, %ymm0
@@ -2240,26 +2164,27 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; AVX512DQ-NEXT:    vpmullw %ymm2, %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vpand %ymm1, %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vpackuswb %ymm3, %ymm0, %ymm0
-; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
-; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; AVX512DQ-NEXT:    vpmullw %ymm0, %ymm2, %ymm0
+; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} ymm2 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
+; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX512DQ-NEXT:    vpmullw %ymm2, %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vpand %ymm1, %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vpxor %xmm2, %xmm2, %xmm2
 ; AVX512DQ-NEXT:    vpackuswb %ymm2, %ymm0, %ymm0
-; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
-; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; AVX512DQ-NEXT:    vpmullw %ymm0, %ymm3, %ymm0
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,2,3]
+; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm3[0],ymm0[0],ymm3[1],ymm0[1],ymm3[2],ymm0[2],ymm3[3],ymm0[3],ymm3[4],ymm0[4],ymm3[5],ymm0[5],ymm3[6],ymm0[6],ymm3[7],ymm0[7],ymm3[16],ymm0[16],ymm3[17],ymm0[17],ymm3[18],ymm0[18],ymm3[19],ymm0[19],ymm3[20],ymm0[20],ymm3[21],ymm0[21],ymm3[22],ymm0[22],ymm3[23],ymm0[23]
+; AVX512DQ-NEXT:    vpmullw %ymm3, %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vpand %ymm1, %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vpackuswb %ymm2, %ymm0, %ymm0
-; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
-; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
-; AVX512DQ-NEXT:    vpmullw %ymm0, %ymm3, %ymm0
+; AVX512DQ-NEXT:    vpsrld $16, %xmm0, %xmm3
+; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm3[0],ymm0[0],ymm3[1],ymm0[1],ymm3[2],ymm0[2],ymm3[3],ymm0[3],ymm3[4],ymm0[4],ymm3[5],ymm0[5],ymm3[6],ymm0[6],ymm3[7],ymm0[7],ymm3[16],ymm0[16],ymm3[17],ymm0[17],ymm3[18],ymm0[18],ymm3[19],ymm0[19],ymm3[20],ymm0[20],ymm3[21],ymm0[21],ymm3[22],ymm0[22],ymm3[23],ymm0[23]
+; AVX512DQ-NEXT:    vpmullw %ymm3, %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vpand %ymm1, %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vpackuswb %ymm2, %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vpsrlw $8, %xmm0, %xmm3
 ; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm3[0],ymm0[0],ymm3[1],ymm0[1],ymm3[2],ymm0[2],ymm3[3],ymm0[3],ymm3[4],ymm0[4],ymm3[5],ymm0[5],ymm3[6],ymm0[6],ymm3[7],ymm0[7],ymm3[16],ymm0[16],ymm3[17],ymm0[17],ymm3[18],ymm0[18],ymm3[19],ymm0[19],ymm3[20],ymm0[20],ymm3[21],ymm0[21],ymm3[22],ymm0[22],ymm3[23],ymm0[23]
 ; AVX512DQ-NEXT:    vpmullw %ymm3, %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vpand %ymm1, %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vpackuswb %ymm2, %ymm0, %ymm0
@@ -2281,26 +2206,27 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; AVX512DQVL-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
 ; AVX512DQVL-NEXT:    vpand %ymm3, %ymm0, %ymm0
 ; AVX512DQVL-NEXT:    vpackuswb %ymm2, %ymm0, %ymm0
-; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
-; AVX512DQVL-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; AVX512DQVL-NEXT:    vpmullw %ymm0, %ymm1, %ymm0
+; AVX512DQVL-NEXT:    vpunpckhbw {{.*#+}} ymm1 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
+; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX512DQVL-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
 ; AVX512DQVL-NEXT:    vpand %ymm3, %ymm0, %ymm0
 ; AVX512DQVL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
 ; AVX512DQVL-NEXT:    vpackuswb %ymm1, %ymm0, %ymm0
-; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
-; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; AVX512DQVL-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; AVX512DQVL-NEXT:    vpmullw %ymm0, %ymm2, %ymm0
+; AVX512DQVL-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[1,1,2,3]
+; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm2[0],ymm0[0],ymm2[1],ymm0[1],ymm2[2],ymm0[2],ymm2[3],ymm0[3],ymm2[4],ymm0[4],ymm2[5],ymm0[5],ymm2[6],ymm0[6],ymm2[7],ymm0[7],ymm2[16],ymm0[16],ymm2[17],ymm0[17],ymm2[18],ymm0[18],ymm2[19],ymm0[19],ymm2[20],ymm0[20],ymm2[21],ymm0[21],ymm2[22],ymm0[22],ymm2[23],ymm0[23]
+; AVX512DQVL-NEXT:    vpmullw %ymm2, %ymm0, %ymm0
 ; AVX512DQVL-NEXT:    vpand %ymm3, %ymm0, %ymm0
 ; AVX512DQVL-NEXT:    vpackuswb %ymm1, %ymm0, %ymm0
-; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
-; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; AVX512DQVL-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
-; AVX512DQVL-NEXT:    vpmullw %ymm0, %ymm2, %ymm0
+; AVX512DQVL-NEXT:    vpsrld $16, %xmm0, %xmm2
+; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm2[0],ymm0[0],ymm2[1],ymm0[1],ymm2[2],ymm0[2],ymm2[3],ymm0[3],ymm2[4],ymm0[4],ymm2[5],ymm0[5],ymm2[6],ymm0[6],ymm2[7],ymm0[7],ymm2[16],ymm0[16],ymm2[17],ymm0[17],ymm2[18],ymm0[18],ymm2[19],ymm0[19],ymm2[20],ymm0[20],ymm2[21],ymm0[21],ymm2[22],ymm0[22],ymm2[23],ymm0[23]
+; AVX512DQVL-NEXT:    vpmullw %ymm2, %ymm0, %ymm0
 ; AVX512DQVL-NEXT:    vpand %ymm3, %ymm0, %ymm0
 ; AVX512DQVL-NEXT:    vpackuswb %ymm1, %ymm0, %ymm0
 ; AVX512DQVL-NEXT:    vpsrlw $8, %xmm0, %xmm2
 ; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm2[0],ymm0[0],ymm2[1],ymm0[1],ymm2[2],ymm0[2],ymm2[3],ymm0[3],ymm2[4],ymm0[4],ymm2[5],ymm0[5],ymm2[6],ymm0[6],ymm2[7],ymm0[7],ymm2[16],ymm0[16],ymm2[17],ymm0[17],ymm2[18],ymm0[18],ymm2[19],ymm0[19],ymm2[20],ymm0[20],ymm2[21],ymm0[21],ymm2[22],ymm0[22],ymm2[23],ymm0[23]
 ; AVX512DQVL-NEXT:    vpmullw %ymm2, %ymm0, %ymm0
 ; AVX512DQVL-NEXT:    vpand %ymm3, %ymm0, %ymm0
 ; AVX512DQVL-NEXT:    vpackuswb %ymm1, %ymm0, %ymm0
@@ -2466,28 +2392,26 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; AVX1-NEXT:    vpmullw %xmm0, %xmm3, %xmm0
 ; AVX1-NEXT:    vpand %xmm2, %xmm0, %xmm0
 ; AVX1-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
 ; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
 ; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpand %xmm2, %xmm0, %xmm0
 ; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
 ; AVX1-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,2,3]
-; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm3 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero
-; AVX1-NEXT:    vpmullw %xmm3, %xmm0, %xmm0
+; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpmullw %xmm0, %xmm3, %xmm0
 ; AVX1-NEXT:    vpand %xmm2, %xmm0, %xmm0
 ; AVX1-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm3
-; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm3 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero
-; AVX1-NEXT:    vpmullw %xmm3, %xmm0, %xmm0
+; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpmullw %xmm0, %xmm3, %xmm0
 ; AVX1-NEXT:    vpand %xmm2, %xmm0, %xmm0
 ; AVX1-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm3
 ; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm3 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero
 ; AVX1-NEXT:    vpmullw %xmm3, %xmm0, %xmm0
 ; AVX1-NEXT:    vpand %xmm2, %xmm0, %xmm0
 ; AVX1-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
@@ -2518,26 +2442,27 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; AVX2-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vpand %ymm2, %ymm0, %ymm0
 ; AVX2-NEXT:    vpackuswb %ymm3, %ymm0, %ymm0
-; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
-; AVX2-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; AVX2-NEXT:    vpmullw %ymm0, %ymm1, %ymm0
+; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm1 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
+; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX2-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vpand %ymm2, %ymm0, %ymm0
 ; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
 ; AVX2-NEXT:    vpackuswb %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
-; AVX2-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; AVX2-NEXT:    vpmullw %ymm0, %ymm3, %ymm0
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,2,3]
+; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm3[0],ymm0[0],ymm3[1],ymm0[1],ymm3[2],ymm0[2],ymm3[3],ymm0[3],ymm3[4],ymm0[4],ymm3[5],ymm0[5],ymm3[6],ymm0[6],ymm3[7],ymm0[7],ymm3[16],ymm0[16],ymm3[17],ymm0[17],ymm3[18],ymm0[18],ymm3[19],ymm0[19],ymm3[20],ymm0[20],ymm3[21],ymm0[21],ymm3[22],ymm0[22],ymm3[23],ymm0[23]
+; AVX2-NEXT:    vpmullw %ymm3, %ymm0, %ymm0
 ; AVX2-NEXT:    vpand %ymm2, %ymm0, %ymm0
 ; AVX2-NEXT:    vpackuswb %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
-; AVX2-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
-; AVX2-NEXT:    vpmullw %ymm0, %ymm3, %ymm0
+; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm3
+; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm3[0],ymm0[0],ymm3[1],ymm0[1],ymm3[2],ymm0[2],ymm3[3],ymm0[3],ymm3[4],ymm0[4],ymm3[5],ymm0[5],ymm3[6],ymm0[6],ymm3[7],ymm0[7],ymm3[16],ymm0[16],ymm3[17],ymm0[17],ymm3[18],ymm0[18],ymm3[19],ymm0[19],ymm3[20],ymm0[20],ymm3[21],ymm0[21],ymm3[22],ymm0[22],ymm3[23],ymm0[23]
+; AVX2-NEXT:    vpmullw %ymm3, %ymm0, %ymm0
 ; AVX2-NEXT:    vpand %ymm2, %ymm0, %ymm0
 ; AVX2-NEXT:    vpackuswb %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vpsrlw $8, %xmm0, %xmm3
 ; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm3[0],ymm0[0],ymm3[1],ymm0[1],ymm3[2],ymm0[2],ymm3[3],ymm0[3],ymm3[4],ymm0[4],ymm3[5],ymm0[5],ymm3[6],ymm0[6],ymm3[7],ymm0[7],ymm3[16],ymm0[16],ymm3[17],ymm0[17],ymm3[18],ymm0[18],ymm3[19],ymm0[19],ymm3[20],ymm0[20],ymm3[21],ymm0[21],ymm3[22],ymm0[22],ymm3[23],ymm0[23]
 ; AVX2-NEXT:    vpmullw %ymm3, %ymm0, %ymm0
 ; AVX2-NEXT:    vpand %ymm2, %ymm0, %ymm0
 ; AVX2-NEXT:    vpackuswb %ymm1, %ymm0, %ymm0
@@ -2569,26 +2494,27 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; AVX512BW-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vpandq %zmm3, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vpackuswb %zmm2, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} zmm1 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
-; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; AVX512BW-NEXT:    vpmullw %zmm0, %zmm1, %zmm0
+; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} zmm1 = zmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]
+; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} zmm0 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
+; AVX512BW-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vpandq %zmm3, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vpxor %xmm1, %xmm1, %xmm1
 ; AVX512BW-NEXT:    vpackuswb %zmm1, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} zmm2 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
-; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; AVX512BW-NEXT:    vpmullw %zmm0, %zmm2, %zmm0
+; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[1,1,2,3]
+; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} zmm0 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
+; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} zmm2 = zmm2[0],zmm0[0],zmm2[1],zmm0[1],zmm2[2],zmm0[2],zmm2[3],zmm0[3],zmm2[4],zmm0[4],zmm2[5],zmm0[5],zmm2[6],zmm0[6],zmm2[7],zmm0[7],zmm2[16],zmm0[16],zmm2[17],zmm0[17],zmm2[18],zmm0[18],zmm2[19],zmm0[19],zmm2[20],zmm0[20],zmm2[21],zmm0[21],zmm2[22],zmm0[22],zmm2[23],zmm0[23],zmm2[32],zmm0[32],zmm2[33],zmm0[33],zmm2[34],zmm0[34],zmm2[35],zmm0[35],zmm2[36],zmm0[36],zmm2[37],zmm0[37],zmm2[38],zmm0[38],zmm2[39],zmm0[39],zmm2[48],zmm0[48],zmm2[49],zmm0[49],zmm2[50],zmm0[50],zmm2[51],zmm0[51],zmm2[52],zmm0[52],zmm2[53],zmm0[53],zmm2[54],zmm0[54],zmm2[55],zmm0[55]
+; AVX512BW-NEXT:    vpmullw %zmm2, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vpandq %zmm3, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vpackuswb %zmm1, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} zmm2 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
-; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
-; AVX512BW-NEXT:    vpmullw %zmm0, %zmm2, %zmm0
+; AVX512BW-NEXT:    vpsrld $16, %xmm0, %xmm2
+; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} zmm0 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
+; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} zmm2 = zmm2[0],zmm0[0],zmm2[1],zmm0[1],zmm2[2],zmm0[2],zmm2[3],zmm0[3],zmm2[4],zmm0[4],zmm2[5],zmm0[5],zmm2[6],zmm0[6],zmm2[7],zmm0[7],zmm2[16],zmm0[16],zmm2[17],zmm0[17],zmm2[18],zmm0[18],zmm2[19],zmm0[19],zmm2[20],zmm0[20],zmm2[21],zmm0[21],zmm2[22],zmm0[22],zmm2[23],zmm0[23],zmm2[32],zmm0[32],zmm2[33],zmm0[33],zmm2[34],zmm0[34],zmm2[35],zmm0[35],zmm2[36],zmm0[36],zmm2[37],zmm0[37],zmm2[38],zmm0[38],zmm2[39],zmm0[39],zmm2[48],zmm0[48],zmm2[49],zmm0[49],zmm2[50],zmm0[50],zmm2[51],zmm0[51],zmm2[52],zmm0[52],zmm2[53],zmm0[53],zmm2[54],zmm0[54],zmm2[55],zmm0[55]
+; AVX512BW-NEXT:    vpmullw %zmm2, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vpandq %zmm3, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vpackuswb %zmm1, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vpsrlw $8, %xmm0, %xmm2
 ; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} zmm0 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
+; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} zmm2 = zmm2[0],zmm0[0],zmm2[1],zmm0[1],zmm2[2],zmm0[2],zmm2[3],zmm0[3],zmm2[4],zmm0[4],zmm2[5],zmm0[5],zmm2[6],zmm0[6],zmm2[7],zmm0[7],zmm2[16],zmm0[16],zmm2[17],zmm0[17],zmm2[18],zmm0[18],zmm2[19],zmm0[19],zmm2[20],zmm0[20],zmm2[21],zmm0[21],zmm2[22],zmm0[22],zmm2[23],zmm0[23],zmm2[32],zmm0[32],zmm2[33],zmm0[33],zmm2[34],zmm0[34],zmm2[35],zmm0[35],zmm2[36],zmm0[36],zmm2[37],zmm0[37],zmm2[38],zmm0[38],zmm2[39],zmm0[39],zmm2[48],zmm0[48],zmm2[49],zmm0[49],zmm2[50],zmm0[50],zmm2[51],zmm0[51],zmm2[52],zmm0[52],zmm2[53],zmm0[53],zmm2[54],zmm0[54],zmm2[55],zmm0[55]
 ; AVX512BW-NEXT:    vpmullw %zmm2, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vpandq %zmm3, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vpackuswb %zmm1, %zmm0, %zmm0
@@ -2620,26 +2546,27 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; AVX512BWVL-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
 ; AVX512BWVL-NEXT:    vpandq %zmm3, %zmm0, %zmm0
 ; AVX512BWVL-NEXT:    vpackuswb %zmm2, %zmm0, %zmm0
-; AVX512BWVL-NEXT:    vpunpcklbw {{.*#+}} zmm1 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
-; AVX512BWVL-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; AVX512BWVL-NEXT:    vpmullw %zmm0, %zmm1, %zmm0
+; AVX512BWVL-NEXT:    vpunpckhbw {{.*#+}} zmm1 = zmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]
+; AVX512BWVL-NEXT:    vpunpcklbw {{.*#+}} zmm0 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
+; AVX512BWVL-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
 ; AVX512BWVL-NEXT:    vpandq %zmm3, %zmm0, %zmm0
 ; AVX512BWVL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
 ; AVX512BWVL-NEXT:    vpackuswb %zmm1, %zmm0, %zmm0
-; AVX512BWVL-NEXT:    vpunpcklbw {{.*#+}} zmm2 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
-; AVX512BWVL-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; AVX512BWVL-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; AVX512BWVL-NEXT:    vpmullw %zmm0, %zmm2, %zmm0
+; AVX512BWVL-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[1,1,2,3]
+; AVX512BWVL-NEXT:    vpunpcklbw {{.*#+}} zmm0 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
+; AVX512BWVL-NEXT:    vpunpcklbw {{.*#+}} zmm2 = zmm2[0],zmm0[0],zmm2[1],zmm0[1],zmm2[2],zmm0[2],zmm2[3],zmm0[3],zmm2[4],zmm0[4],zmm2[5],zmm0[5],zmm2[6],zmm0[6],zmm2[7],zmm0[7],zmm2[16],zmm0[16],zmm2[17],zmm0[17],zmm2[18],zmm0[18],zmm2[19],zmm0[19],zmm2[20],zmm0[20],zmm2[21],zmm0[21],zmm2[22],zmm0[22],zmm2[23],zmm0[23],zmm2[32],zmm0[32],zmm2[33],zmm0[33],zmm2[34],zmm0[34],zmm2[35],zmm0[35],zmm2[36],zmm0[36],zmm2[37],zmm0[37],zmm2[38],zmm0[38],zmm2[39],zmm0[39],zmm2[48],zmm0[48],zmm2[49],zmm0[49],zmm2[50],zmm0[50],zmm2[51],zmm0[51],zmm2[52],zmm0[52],zmm2[53],zmm0[53],zmm2[54],zmm0[54],zmm2[55],zmm0[55]
+; AVX512BWVL-NEXT:    vpmullw %zmm2, %zmm0, %zmm0
 ; AVX512BWVL-NEXT:    vpandq %zmm3, %zmm0, %zmm0
 ; AVX512BWVL-NEXT:    vpackuswb %zmm1, %zmm0, %zmm0
-; AVX512BWVL-NEXT:    vpunpcklbw {{.*#+}} zmm2 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
-; AVX512BWVL-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; AVX512BWVL-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
-; AVX512BWVL-NEXT:    vpmullw %zmm0, %zmm2, %zmm0
+; AVX512BWVL-NEXT:    vpsrld $16, %xmm0, %xmm2
+; AVX512BWVL-NEXT:    vpunpcklbw {{.*#+}} zmm0 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
+; AVX512BWVL-NEXT:    vpunpcklbw {{.*#+}} zmm2 = zmm2[0],zmm0[0],zmm2[1],zmm0[1],zmm2[2],zmm0[2],zmm2[3],zmm0[3],zmm2[4],zmm0[4],zmm2[5],zmm0[5],zmm2[6],zmm0[6],zmm2[7],zmm0[7],zmm2[16],zmm0[16],zmm2[17],zmm0[17],zmm2[18],zmm0[18],zmm2[19],zmm0[19],zmm2[20],zmm0[20],zmm2[21],zmm0[21],zmm2[22],zmm0[22],zmm2[23],zmm0[23],zmm2[32],zmm0[32],zmm2[33],zmm0[33],zmm2[34],zmm0[34],zmm2[35],zmm0[35],zmm2[36],zmm0[36],zmm2[37],zmm0[37],zmm2[38],zmm0[38],zmm2[39],zmm0[39],zmm2[48],zmm0[48],zmm2[49],zmm0[49],zmm2[50],zmm0[50],zmm2[51],zmm0[51],zmm2[52],zmm0[52],zmm2[53],zmm0[53],zmm2[54],zmm0[54],zmm2[55],zmm0[55]
+; AVX512BWVL-NEXT:    vpmullw %zmm2, %zmm0, %zmm0
 ; AVX512BWVL-NEXT:    vpandq %zmm3, %zmm0, %zmm0
 ; AVX512BWVL-NEXT:    vpackuswb %zmm1, %zmm0, %zmm0
 ; AVX512BWVL-NEXT:    vpsrlw $8, %xmm0, %xmm2
 ; AVX512BWVL-NEXT:    vpunpcklbw {{.*#+}} zmm0 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
+; AVX512BWVL-NEXT:    vpunpcklbw {{.*#+}} zmm2 = zmm2[0],zmm0[0],zmm2[1],zmm0[1],zmm2[2],zmm0[2],zmm2[3],zmm0[3],zmm2[4],zmm0[4],zmm2[5],zmm0[5],zmm2[6],zmm0[6],zmm2[7],zmm0[7],zmm2[16],zmm0[16],zmm2[17],zmm0[17],zmm2[18],zmm0[18],zmm2[19],zmm0[19],zmm2[20],zmm0[20],zmm2[21],zmm0[21],zmm2[22],zmm0[22],zmm2[23],zmm0[23],zmm2[32],zmm0[32],zmm2[33],zmm0[33],zmm2[34],zmm0[34],zmm2[35],zmm0[35],zmm2[36],zmm0[36],zmm2[37],zmm0[37],zmm2[38],zmm0[38],zmm2[39],zmm0[39],zmm2[48],zmm0[48],zmm2[49],zmm0[49],zmm2[50],zmm0[50],zmm2[51],zmm0[51],zmm2[52],zmm0[52],zmm2[53],zmm0[53],zmm2[54],zmm0[54],zmm2[55],zmm0[55]
 ; AVX512BWVL-NEXT:    vpmullw %zmm2, %zmm0, %zmm0
 ; AVX512BWVL-NEXT:    vpandq %zmm3, %zmm0, %zmm0
 ; AVX512BWVL-NEXT:    vpackuswb %zmm1, %zmm0, %zmm0
@@ -2670,26 +2597,27 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; AVX512DQ-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vpand %ymm2, %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vpackuswb %ymm3, %ymm0, %ymm0
-; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
-; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; AVX512DQ-NEXT:    vpmullw %ymm0, %ymm1, %ymm0
+; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} ymm1 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
+; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX512DQ-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vpand %ymm2, %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vpxor %xmm1, %xmm1, %xmm1
 ; AVX512DQ-NEXT:    vpackuswb %ymm1, %ymm0, %ymm0
-; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
-; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; AVX512DQ-NEXT:    vpmullw %ymm0, %ymm3, %ymm0
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm3 = xmm0[1,1,2,3]
+; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm3[0],ymm0[0],ymm3[1],ymm0[1],ymm3[2],ymm0[2],ymm3[3],ymm0[3],ymm3[4],ymm0[4],ymm3[5],ymm0[5],ymm3[6],ymm0[6],ymm3[7],ymm0[7],ymm3[16],ymm0[16],ymm3[17],ymm0[17],ymm3[18],ymm0[18],ymm3[19],ymm0[19],ymm3[20],ymm0[20],ymm3[21],ymm0[21],ymm3[22],ymm0[22],ymm3[23],ymm0[23]
+; AVX512DQ-NEXT:    vpmullw %ymm3, %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vpand %ymm2, %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vpackuswb %ymm1, %ymm0, %ymm0
-; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
-; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
-; AVX512DQ-NEXT:    vpmullw %ymm0, %ymm3, %ymm0
+; AVX512DQ-NEXT:    vpsrld $16, %xmm0, %xmm3
+; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm3[0],ymm0[0],ymm3[1],ymm0[1],ymm3[2],ymm0[2],ymm3[3],ymm0[3],ymm3[4],ymm0[4],ymm3[5],ymm0[5],ymm3[6],ymm0[6],ymm3[7],ymm0[7],ymm3[16],ymm0[16],ymm3[17],ymm0[17],ymm3[18],ymm0[18],ymm3[19],ymm0[19],ymm3[20],ymm0[20],ymm3[21],ymm0[21],ymm3[22],ymm0[22],ymm3[23],ymm0[23]
+; AVX512DQ-NEXT:    vpmullw %ymm3, %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vpand %ymm2, %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vpackuswb %ymm1, %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vpsrlw $8, %xmm0, %xmm3
 ; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm3[0],ymm0[0],ymm3[1],ymm0[1],ymm3[2],ymm0[2],ymm3[3],ymm0[3],ymm3[4],ymm0[4],ymm3[5],ymm0[5],ymm3[6],ymm0[6],ymm3[7],ymm0[7],ymm3[16],ymm0[16],ymm3[17],ymm0[17],ymm3[18],ymm0[18],ymm3[19],ymm0[19],ymm3[20],ymm0[20],ymm3[21],ymm0[21],ymm3[22],ymm0[22],ymm3[23],ymm0[23]
 ; AVX512DQ-NEXT:    vpmullw %ymm3, %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vpand %ymm2, %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vpackuswb %ymm1, %ymm0, %ymm0
@@ -2720,26 +2648,27 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; AVX512DQVL-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
 ; AVX512DQVL-NEXT:    vpand %ymm3, %ymm0, %ymm0
 ; AVX512DQVL-NEXT:    vpackuswb %ymm2, %ymm0, %ymm0
-; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
-; AVX512DQVL-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; AVX512DQVL-NEXT:    vpmullw %ymm0, %ymm1, %ymm0
+; AVX512DQVL-NEXT:    vpunpckhbw {{.*#+}} ymm1 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
+; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX512DQVL-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
 ; AVX512DQVL-NEXT:    vpand %ymm3, %ymm0, %ymm0
 ; AVX512DQVL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
 ; AVX512DQVL-NEXT:    vpackuswb %ymm1, %ymm0, %ymm0
-; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
-; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; AVX512DQVL-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; AVX512DQVL-NEXT:    vpmullw %ymm0, %ymm2, %ymm0
+; AVX512DQVL-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[1,1,2,3]
+; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm2[0],ymm0[0],ymm2[1],ymm0[1],ymm2[2],ymm0[2],ymm2[3],ymm0[3],ymm2[4],ymm0[4],ymm2[5],ymm0[5],ymm2[6],ymm0[6],ymm2[7],ymm0[7],ymm2[16],ymm0[16],ymm2[17],ymm0[17],ymm2[18],ymm0[18],ymm2[19],ymm0[19],ymm2[20],ymm0[20],ymm2[21],ymm0[21],ymm2[22],ymm0[22],ymm2[23],ymm0[23]
+; AVX512DQVL-NEXT:    vpmullw %ymm2, %ymm0, %ymm0
 ; AVX512DQVL-NEXT:    vpand %ymm3, %ymm0, %ymm0
 ; AVX512DQVL-NEXT:    vpackuswb %ymm1, %ymm0, %ymm0
-; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
-; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; AVX512DQVL-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
-; AVX512DQVL-NEXT:    vpmullw %ymm0, %ymm2, %ymm0
+; AVX512DQVL-NEXT:    vpsrld $16, %xmm0, %xmm2
+; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm2[0],ymm0[0],ymm2[1],ymm0[1],ymm2[2],ymm0[2],ymm2[3],ymm0[3],ymm2[4],ymm0[4],ymm2[5],ymm0[5],ymm2[6],ymm0[6],ymm2[7],ymm0[7],ymm2[16],ymm0[16],ymm2[17],ymm0[17],ymm2[18],ymm0[18],ymm2[19],ymm0[19],ymm2[20],ymm0[20],ymm2[21],ymm0[21],ymm2[22],ymm0[22],ymm2[23],ymm0[23]
+; AVX512DQVL-NEXT:    vpmullw %ymm2, %ymm0, %ymm0
 ; AVX512DQVL-NEXT:    vpand %ymm3, %ymm0, %ymm0
 ; AVX512DQVL-NEXT:    vpackuswb %ymm1, %ymm0, %ymm0
 ; AVX512DQVL-NEXT:    vpsrlw $8, %xmm0, %xmm2
 ; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm2[0],ymm0[0],ymm2[1],ymm0[1],ymm2[2],ymm0[2],ymm2[3],ymm0[3],ymm2[4],ymm0[4],ymm2[5],ymm0[5],ymm2[6],ymm0[6],ymm2[7],ymm0[7],ymm2[16],ymm0[16],ymm2[17],ymm0[17],ymm2[18],ymm0[18],ymm2[19],ymm0[19],ymm2[20],ymm0[20],ymm2[21],ymm0[21],ymm2[22],ymm0[22],ymm2[23],ymm0[23]
 ; AVX512DQVL-NEXT:    vpmullw %ymm2, %ymm0, %ymm0
 ; AVX512DQVL-NEXT:    vpand %ymm3, %ymm0, %ymm0
 ; AVX512DQVL-NEXT:    vpackuswb %ymm1, %ymm0, %ymm0
@@ -3023,28 +2952,26 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpand %xmm4, %xmm0, %xmm0
 ; AVX1-NEXT:    vpackuswb %xmm2, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
 ; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
 ; AVX1-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpand %xmm4, %xmm0, %xmm0
 ; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
 ; AVX1-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[1,1,2,3]
-; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
-; AVX1-NEXT:    vpmullw %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm2 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpmullw %xmm0, %xmm2, %xmm0
 ; AVX1-NEXT:    vpand %xmm4, %xmm0, %xmm0
 ; AVX1-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm2
-; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
-; AVX1-NEXT:    vpmullw %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm2 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpmullw %xmm0, %xmm2, %xmm0
 ; AVX1-NEXT:    vpand %xmm4, %xmm0, %xmm0
 ; AVX1-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm2
 ; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
 ; AVX1-NEXT:    vpmullw %xmm2, %xmm0, %xmm0
 ; AVX1-NEXT:    vpand %xmm4, %xmm0, %xmm0
 ; AVX1-NEXT:    vpackuswb %xmm1, %xmm0, %xmm0
@@ -3093,26 +3020,27 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; AVX2-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vpand %ymm4, %ymm0, %ymm0
 ; AVX2-NEXT:    vpackuswb %ymm2, %ymm0, %ymm0
-; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
-; AVX2-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; AVX2-NEXT:    vpmullw %ymm0, %ymm1, %ymm0
+; AVX2-NEXT:    vpunpckhbw {{.*#+}} ymm1 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
+; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX2-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vpand %ymm4, %ymm0, %ymm0
 ; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
 ; AVX2-NEXT:    vpackuswb %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
-; AVX2-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; AVX2-NEXT:    vpmullw %ymm0, %ymm2, %ymm0
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[1,1,2,3]
+; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm2[0],ymm0[0],ymm2[1],ymm0[1],ymm2[2],ymm0[2],ymm2[3],ymm0[3],ymm2[4],ymm0[4],ymm2[5],ymm0[5],ymm2[6],ymm0[6],ymm2[7],ymm0[7],ymm2[16],ymm0[16],ymm2[17],ymm0[17],ymm2[18],ymm0[18],ymm2[19],ymm0[19],ymm2[20],ymm0[20],ymm2[21],ymm0[21],ymm2[22],ymm0[22],ymm2[23],ymm0[23]
+; AVX2-NEXT:    vpmullw %ymm2, %ymm0, %ymm0
 ; AVX2-NEXT:    vpand %ymm4, %ymm0, %ymm0
 ; AVX2-NEXT:    vpackuswb %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
-; AVX2-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
-; AVX2-NEXT:    vpmullw %ymm0, %ymm2, %ymm0
+; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm2
+; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm2[0],ymm0[0],ymm2[1],ymm0[1],ymm2[2],ymm0[2],ymm2[3],ymm0[3],ymm2[4],ymm0[4],ymm2[5],ymm0[5],ymm2[6],ymm0[6],ymm2[7],ymm0[7],ymm2[16],ymm0[16],ymm2[17],ymm0[17],ymm2[18],ymm0[18],ymm2[19],ymm0[19],ymm2[20],ymm0[20],ymm2[21],ymm0[21],ymm2[22],ymm0[22],ymm2[23],ymm0[23]
+; AVX2-NEXT:    vpmullw %ymm2, %ymm0, %ymm0
 ; AVX2-NEXT:    vpand %ymm4, %ymm0, %ymm0
 ; AVX2-NEXT:    vpackuswb %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vpsrlw $8, %xmm0, %xmm2
 ; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX2-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm2[0],ymm0[0],ymm2[1],ymm0[1],ymm2[2],ymm0[2],ymm2[3],ymm0[3],ymm2[4],ymm0[4],ymm2[5],ymm0[5],ymm2[6],ymm0[6],ymm2[7],ymm0[7],ymm2[16],ymm0[16],ymm2[17],ymm0[17],ymm2[18],ymm0[18],ymm2[19],ymm0[19],ymm2[20],ymm0[20],ymm2[21],ymm0[21],ymm2[22],ymm0[22],ymm2[23],ymm0[23]
 ; AVX2-NEXT:    vpmullw %ymm2, %ymm0, %ymm0
 ; AVX2-NEXT:    vpand %ymm4, %ymm0, %ymm0
 ; AVX2-NEXT:    vpackuswb %ymm1, %ymm0, %ymm0
@@ -3125,56 +3053,57 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; AVX512BW:       # %bb.0:
 ; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} zmm2 = zmm1[8],zmm0[8],zmm1[9],zmm0[9],zmm1[10],zmm0[10],zmm1[11],zmm0[11],zmm1[12],zmm0[12],zmm1[13],zmm0[13],zmm1[14],zmm0[14],zmm1[15],zmm0[15],zmm1[24],zmm0[24],zmm1[25],zmm0[25],zmm1[26],zmm0[26],zmm1[27],zmm0[27],zmm1[28],zmm0[28],zmm1[29],zmm0[29],zmm1[30],zmm0[30],zmm1[31],zmm0[31],zmm1[40],zmm0[40],zmm1[41],zmm0[41],zmm1[42],zmm0[42],zmm1[43],zmm0[43],zmm1[44],zmm0[44],zmm1[45],zmm0[45],zmm1[46],zmm0[46],zmm1[47],zmm0[47],zmm1[56],zmm0[56],zmm1[57],zmm0[57],zmm1[58],zmm0[58],zmm1[59],zmm0[59],zmm1[60],zmm0[60],zmm1[61],zmm0[61],zmm1[62],zmm0[62],zmm1[63],zmm0[63]
 ; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} zmm3 = zmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]
-; AVX512BW-NEXT:    vpmullw %zmm2, %zmm3, %zmm3
-; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
-; AVX512BW-NEXT:    vpandq %zmm2, %zmm3, %zmm3
+; AVX512BW-NEXT:    vpmullw %zmm2, %zmm3, %zmm2
+; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; AVX512BW-NEXT:    vpandq %zmm3, %zmm2, %zmm2
 ; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} zmm1 = zmm1[0],zmm0[0],zmm1[1],zmm0[1],zmm1[2],zmm0[2],zmm1[3],zmm0[3],zmm1[4],zmm0[4],zmm1[5],zmm0[5],zmm1[6],zmm0[6],zmm1[7],zmm0[7],zmm1[16],zmm0[16],zmm1[17],zmm0[17],zmm1[18],zmm0[18],zmm1[19],zmm0[19],zmm1[20],zmm0[20],zmm1[21],zmm0[21],zmm1[22],zmm0[22],zmm1[23],zmm0[23],zmm1[32],zmm0[32],zmm1[33],zmm0[33],zmm1[34],zmm0[34],zmm1[35],zmm0[35],zmm1[36],zmm0[36],zmm1[37],zmm0[37],zmm1[38],zmm0[38],zmm1[39],zmm0[39],zmm1[48],zmm0[48],zmm1[49],zmm0[49],zmm1[50],zmm0[50],zmm1[51],zmm0[51],zmm1[52],zmm0[52],zmm1[53],zmm0[53],zmm1[54],zmm0[54],zmm1[55],zmm0[55]
 ; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} zmm0 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
 ; AVX512BW-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpandq %zmm2, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpackuswb %zmm3, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpandq %zmm3, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpackuswb %zmm2, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
-; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} zmm3 = zmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]
+; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} zmm2 = zmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]
 ; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} zmm4 = zmm1[8],zmm0[8],zmm1[9],zmm0[9],zmm1[10],zmm0[10],zmm1[11],zmm0[11],zmm1[12],zmm0[12],zmm1[13],zmm0[13],zmm1[14],zmm0[14],zmm1[15],zmm0[15],zmm1[24],zmm0[24],zmm1[25],zmm0[25],zmm1[26],zmm0[26],zmm1[27],zmm0[27],zmm1[28],zmm0[28],zmm1[29],zmm0[29],zmm1[30],zmm0[30],zmm1[31],zmm0[31],zmm1[40],zmm0[40],zmm1[41],zmm0[41],zmm1[42],zmm0[42],zmm1[43],zmm0[43],zmm1[44],zmm0[44],zmm1[45],zmm0[45],zmm1[46],zmm0[46],zmm1[47],zmm0[47],zmm1[56],zmm0[56],zmm1[57],zmm0[57],zmm1[58],zmm0[58],zmm1[59],zmm0[59],zmm1[60],zmm0[60],zmm1[61],zmm0[61],zmm1[62],zmm0[62],zmm1[63],zmm0[63]
-; AVX512BW-NEXT:    vpmullw %zmm4, %zmm3, %zmm3
-; AVX512BW-NEXT:    vpandq %zmm2, %zmm3, %zmm3
+; AVX512BW-NEXT:    vpmullw %zmm4, %zmm2, %zmm2
+; AVX512BW-NEXT:    vpandq %zmm3, %zmm2, %zmm2
 ; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} zmm0 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
 ; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} zmm1 = zmm1[0],zmm0[0],zmm1[1],zmm0[1],zmm1[2],zmm0[2],zmm1[3],zmm0[3],zmm1[4],zmm0[4],zmm1[5],zmm0[5],zmm1[6],zmm0[6],zmm1[7],zmm0[7],zmm1[16],zmm0[16],zmm1[17],zmm0[17],zmm1[18],zmm0[18],zmm1[19],zmm0[19],zmm1[20],zmm0[20],zmm1[21],zmm0[21],zmm1[22],zmm0[22],zmm1[23],zmm0[23],zmm1[32],zmm0[32],zmm1[33],zmm0[33],zmm1[34],zmm0[34],zmm1[35],zmm0[35],zmm1[36],zmm0[36],zmm1[37],zmm0[37],zmm1[38],zmm0[38],zmm1[39],zmm0[39],zmm1[48],zmm0[48],zmm1[49],zmm0[49],zmm1[50],zmm0[50],zmm1[51],zmm0[51],zmm1[52],zmm0[52],zmm1[53],zmm0[53],zmm1[54],zmm0[54],zmm1[55],zmm0[55]
 ; AVX512BW-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpandq %zmm2, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpackuswb %zmm3, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpandq %zmm3, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpackuswb %zmm2, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} zmm3 = zmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]
+; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} zmm2 = zmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]
 ; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} zmm4 = zmm1[8],zmm0[8],zmm1[9],zmm0[9],zmm1[10],zmm0[10],zmm1[11],zmm0[11],zmm1[12],zmm0[12],zmm1[13],zmm0[13],zmm1[14],zmm0[14],zmm1[15],zmm0[15],zmm1[24],zmm0[24],zmm1[25],zmm0[25],zmm1[26],zmm0[26],zmm1[27],zmm0[27],zmm1[28],zmm0[28],zmm1[29],zmm0[29],zmm1[30],zmm0[30],zmm1[31],zmm0[31],zmm1[40],zmm0[40],zmm1[41],zmm0[41],zmm1[42],zmm0[42],zmm1[43],zmm0[43],zmm1[44],zmm0[44],zmm1[45],zmm0[45],zmm1[46],zmm0[46],zmm1[47],zmm0[47],zmm1[56],zmm0[56],zmm1[57],zmm0[57],zmm1[58],zmm0[58],zmm1[59],zmm0[59],zmm1[60],zmm0[60],zmm1[61],zmm0[61],zmm1[62],zmm0[62],zmm1[63],zmm0[63]
-; AVX512BW-NEXT:    vpmullw %zmm4, %zmm3, %zmm3
-; AVX512BW-NEXT:    vpandq %zmm2, %zmm3, %zmm3
+; AVX512BW-NEXT:    vpmullw %zmm4, %zmm2, %zmm2
+; AVX512BW-NEXT:    vpandq %zmm3, %zmm2, %zmm2
 ; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} zmm0 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
 ; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} zmm1 = zmm1[0],zmm0[0],zmm1[1],zmm0[1],zmm1[2],zmm0[2],zmm1[3],zmm0[3],zmm1[4],zmm0[4],zmm1[5],zmm0[5],zmm1[6],zmm0[6],zmm1[7],zmm0[7],zmm1[16],zmm0[16],zmm1[17],zmm0[17],zmm1[18],zmm0[18],zmm1[19],zmm0[19],zmm1[20],zmm0[20],zmm1[21],zmm0[21],zmm1[22],zmm0[22],zmm1[23],zmm0[23],zmm1[32],zmm0[32],zmm1[33],zmm0[33],zmm1[34],zmm0[34],zmm1[35],zmm0[35],zmm1[36],zmm0[36],zmm1[37],zmm0[37],zmm1[38],zmm0[38],zmm1[39],zmm0[39],zmm1[48],zmm0[48],zmm1[49],zmm0[49],zmm1[50],zmm0[50],zmm1[51],zmm0[51],zmm1[52],zmm0[52],zmm1[53],zmm0[53],zmm1[54],zmm0[54],zmm1[55],zmm0[55]
 ; AVX512BW-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpandq %zmm2, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpackuswb %zmm3, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} zmm1 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
-; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; AVX512BW-NEXT:    vpmullw %zmm0, %zmm1, %zmm0
-; AVX512BW-NEXT:    vpandq %zmm2, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpandq %zmm3, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpackuswb %zmm2, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpunpckhbw {{.*#+}} zmm1 = zmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]
+; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} zmm0 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
+; AVX512BW-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpandq %zmm3, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vpxor %xmm1, %xmm1, %xmm1
 ; AVX512BW-NEXT:    vpackuswb %zmm1, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} zmm3 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
-; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; AVX512BW-NEXT:    vpmullw %zmm0, %zmm3, %zmm0
-; AVX512BW-NEXT:    vpandq %zmm2, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[1,1,2,3]
+; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} zmm0 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
+; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} zmm2 = zmm2[0],zmm0[0],zmm2[1],zmm0[1],zmm2[2],zmm0[2],zmm2[3],zmm0[3],zmm2[4],zmm0[4],zmm2[5],zmm0[5],zmm2[6],zmm0[6],zmm2[7],zmm0[7],zmm2[16],zmm0[16],zmm2[17],zmm0[17],zmm2[18],zmm0[18],zmm2[19],zmm0[19],zmm2[20],zmm0[20],zmm2[21],zmm0[21],zmm2[22],zmm0[22],zmm2[23],zmm0[23],zmm2[32],zmm0[32],zmm2[33],zmm0[33],zmm2[34],zmm0[34],zmm2[35],zmm0[35],zmm2[36],zmm0[36],zmm2[37],zmm0[37],zmm2[38],zmm0[38],zmm2[39],zmm0[39],zmm2[48],zmm0[48],zmm2[49],zmm0[49],zmm2[50],zmm0[50],zmm2[51],zmm0[51],zmm2[52],zmm0[52],zmm2[53],zmm0[53],zmm2[54],zmm0[54],zmm2[55],zmm0[55]
+; AVX512BW-NEXT:    vpmullw %zmm2, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpandq %zmm3, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vpackuswb %zmm1, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} zmm3 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
-; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; AVX512BW-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
-; AVX512BW-NEXT:    vpmullw %zmm0, %zmm3, %zmm0
-; AVX512BW-NEXT:    vpandq %zmm2, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpsrld $16, %xmm0, %xmm2
+; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} zmm0 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
+; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} zmm2 = zmm2[0],zmm0[0],zmm2[1],zmm0[1],zmm2[2],zmm0[2],zmm2[3],zmm0[3],zmm2[4],zmm0[4],zmm2[5],zmm0[5],zmm2[6],zmm0[6],zmm2[7],zmm0[7],zmm2[16],zmm0[16],zmm2[17],zmm0[17],zmm2[18],zmm0[18],zmm2[19],zmm0[19],zmm2[20],zmm0[20],zmm2[21],zmm0[21],zmm2[22],zmm0[22],zmm2[23],zmm0[23],zmm2[32],zmm0[32],zmm2[33],zmm0[33],zmm2[34],zmm0[34],zmm2[35],zmm0[35],zmm2[36],zmm0[36],zmm2[37],zmm0[37],zmm2[38],zmm0[38],zmm2[39],zmm0[39],zmm2[48],zmm0[48],zmm2[49],zmm0[49],zmm2[50],zmm0[50],zmm2[51],zmm0[51],zmm2[52],zmm0[52],zmm2[53],zmm0[53],zmm2[54],zmm0[54],zmm2[55],zmm0[55]
+; AVX512BW-NEXT:    vpmullw %zmm2, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpandq %zmm3, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vpackuswb %zmm1, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpsrlw $8, %xmm0, %xmm3
+; AVX512BW-NEXT:    vpsrlw $8, %xmm0, %xmm2
 ; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} zmm0 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
-; AVX512BW-NEXT:    vpmullw %zmm3, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpandq %zmm2, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpunpcklbw {{.*#+}} zmm2 = zmm2[0],zmm0[0],zmm2[1],zmm0[1],zmm2[2],zmm0[2],zmm2[3],zmm0[3],zmm2[4],zmm0[4],zmm2[5],zmm0[5],zmm2[6],zmm0[6],zmm2[7],zmm0[7],zmm2[16],zmm0[16],zmm2[17],zmm0[17],zmm2[18],zmm0[18],zmm2[19],zmm0[19],zmm2[20],zmm0[20],zmm2[21],zmm0[21],zmm2[22],zmm0[22],zmm2[23],zmm0[23],zmm2[32],zmm0[32],zmm2[33],zmm0[33],zmm2[34],zmm0[34],zmm2[35],zmm0[35],zmm2[36],zmm0[36],zmm2[37],zmm0[37],zmm2[38],zmm0[38],zmm2[39],zmm0[39],zmm2[48],zmm0[48],zmm2[49],zmm0[49],zmm2[50],zmm0[50],zmm2[51],zmm0[51],zmm2[52],zmm0[52],zmm2[53],zmm0[53],zmm2[54],zmm0[54],zmm2[55],zmm0[55]
+; AVX512BW-NEXT:    vpmullw %zmm2, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpandq %zmm3, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vpackuswb %zmm1, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vpextrb $0, %xmm0, %eax
 ; AVX512BW-NEXT:    # kill: def $al killed $al killed $eax
@@ -3185,56 +3114,57 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; AVX512BWVL:       # %bb.0:
 ; AVX512BWVL-NEXT:    vpunpckhbw {{.*#+}} zmm2 = zmm1[8],zmm0[8],zmm1[9],zmm0[9],zmm1[10],zmm0[10],zmm1[11],zmm0[11],zmm1[12],zmm0[12],zmm1[13],zmm0[13],zmm1[14],zmm0[14],zmm1[15],zmm0[15],zmm1[24],zmm0[24],zmm1[25],zmm0[25],zmm1[26],zmm0[26],zmm1[27],zmm0[27],zmm1[28],zmm0[28],zmm1[29],zmm0[29],zmm1[30],zmm0[30],zmm1[31],zmm0[31],zmm1[40],zmm0[40],zmm1[41],zmm0[41],zmm1[42],zmm0[42],zmm1[43],zmm0[43],zmm1[44],zmm0[44],zmm1[45],zmm0[45],zmm1[46],zmm0[46],zmm1[47],zmm0[47],zmm1[56],zmm0[56],zmm1[57],zmm0[57],zmm1[58],zmm0[58],zmm1[59],zmm0[59],zmm1[60],zmm0[60],zmm1[61],zmm0[61],zmm1[62],zmm0[62],zmm1[63],zmm0[63]
 ; AVX512BWVL-NEXT:    vpunpckhbw {{.*#+}} zmm3 = zmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]
-; AVX512BWVL-NEXT:    vpmullw %zmm2, %zmm3, %zmm3
-; AVX512BWVL-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
-; AVX512BWVL-NEXT:    vpandq %zmm2, %zmm3, %zmm3
+; AVX512BWVL-NEXT:    vpmullw %zmm2, %zmm3, %zmm2
+; AVX512BWVL-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; AVX512BWVL-NEXT:    vpandq %zmm3, %zmm2, %zmm2
 ; AVX512BWVL-NEXT:    vpunpcklbw {{.*#+}} zmm1 = zmm1[0],zmm0[0],zmm1[1],zmm0[1],zmm1[2],zmm0[2],zmm1[3],zmm0[3],zmm1[4],zmm0[4],zmm1[5],zmm0[5],zmm1[6],zmm0[6],zmm1[7],zmm0[7],zmm1[16],zmm0[16],zmm1[17],zmm0[17],zmm1[18],zmm0[18],zmm1[19],zmm0[19],zmm1[20],zmm0[20],zmm1[21],zmm0[21],zmm1[22],zmm0[22],zmm1[23],zmm0[23],zmm1[32],zmm0[32],zmm1[33],zmm0[33],zmm1[34],zmm0[34],zmm1[35],zmm0[35],zmm1[36],zmm0[36],zmm1[37],zmm0[37],zmm1[38],zmm0[38],zmm1[39],zmm0[39],zmm1[48],zmm0[48],zmm1[49],zmm0[49],zmm1[50],zmm0[50],zmm1[51],zmm0[51],zmm1[52],zmm0[52],zmm1[53],zmm0[53],zmm1[54],zmm0[54],zmm1[55],zmm0[55]
 ; AVX512BWVL-NEXT:    vpunpcklbw {{.*#+}} zmm0 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
 ; AVX512BWVL-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
-; AVX512BWVL-NEXT:    vpandq %zmm2, %zmm0, %zmm0
-; AVX512BWVL-NEXT:    vpackuswb %zmm3, %zmm0, %zmm0
+; AVX512BWVL-NEXT:    vpandq %zmm3, %zmm0, %zmm0
+; AVX512BWVL-NEXT:    vpackuswb %zmm2, %zmm0, %zmm0
 ; AVX512BWVL-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
-; AVX512BWVL-NEXT:    vpunpckhbw {{.*#+}} zmm3 = zmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]
+; AVX512BWVL-NEXT:    vpunpckhbw {{.*#+}} zmm2 = zmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]
 ; AVX512BWVL-NEXT:    vpunpckhbw {{.*#+}} zmm4 = zmm1[8],zmm0[8],zmm1[9],zmm0[9],zmm1[10],zmm0[10],zmm1[11],zmm0[11],zmm1[12],zmm0[12],zmm1[13],zmm0[13],zmm1[14],zmm0[14],zmm1[15],zmm0[15],zmm1[24],zmm0[24],zmm1[25],zmm0[25],zmm1[26],zmm0[26],zmm1[27],zmm0[27],zmm1[28],zmm0[28],zmm1[29],zmm0[29],zmm1[30],zmm0[30],zmm1[31],zmm0[31],zmm1[40],zmm0[40],zmm1[41],zmm0[41],zmm1[42],zmm0[42],zmm1[43],zmm0[43],zmm1[44],zmm0[44],zmm1[45],zmm0[45],zmm1[46],zmm0[46],zmm1[47],zmm0[47],zmm1[56],zmm0[56],zmm1[57],zmm0[57],zmm1[58],zmm0[58],zmm1[59],zmm0[59],zmm1[60],zmm0[60],zmm1[61],zmm0[61],zmm1[62],zmm0[62],zmm1[63],zmm0[63]
-; AVX512BWVL-NEXT:    vpmullw %zmm4, %zmm3, %zmm3
-; AVX512BWVL-NEXT:    vpandq %zmm2, %zmm3, %zmm3
+; AVX512BWVL-NEXT:    vpmullw %zmm4, %zmm2, %zmm2
+; AVX512BWVL-NEXT:    vpandq %zmm3, %zmm2, %zmm2
 ; AVX512BWVL-NEXT:    vpunpcklbw {{.*#+}} zmm0 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
 ; AVX512BWVL-NEXT:    vpunpcklbw {{.*#+}} zmm1 = zmm1[0],zmm0[0],zmm1[1],zmm0[1],zmm1[2],zmm0[2],zmm1[3],zmm0[3],zmm1[4],zmm0[4],zmm1[5],zmm0[5],zmm1[6],zmm0[6],zmm1[7],zmm0[7],zmm1[16],zmm0[16],zmm1[17],zmm0[17],zmm1[18],zmm0[18],zmm1[19],zmm0[19],zmm1[20],zmm0[20],zmm1[21],zmm0[21],zmm1[22],zmm0[22],zmm1[23],zmm0[23],zmm1[32],zmm0[32],zmm1[33],zmm0[33],zmm1[34],zmm0[34],zmm1[35],zmm0[35],zmm1[36],zmm0[36],zmm1[37],zmm0[37],zmm1[38],zmm0[38],zmm1[39],zmm0[39],zmm1[48],zmm0[48],zmm1[49],zmm0[49],zmm1[50],zmm0[50],zmm1[51],zmm0[51],zmm1[52],zmm0[52],zmm1[53],zmm0[53],zmm1[54],zmm0[54],zmm1[55],zmm0[55]
 ; AVX512BWVL-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
-; AVX512BWVL-NEXT:    vpandq %zmm2, %zmm0, %zmm0
-; AVX512BWVL-NEXT:    vpackuswb %zmm3, %zmm0, %zmm0
+; AVX512BWVL-NEXT:    vpandq %zmm3, %zmm0, %zmm0
+; AVX512BWVL-NEXT:    vpackuswb %zmm2, %zmm0, %zmm0
 ; AVX512BWVL-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512BWVL-NEXT:    vpunpckhbw {{.*#+}} zmm3 = zmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]
+; AVX512BWVL-NEXT:    vpunpckhbw {{.*#+}} zmm2 = zmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]
 ; AVX512BWVL-NEXT:    vpunpckhbw {{.*#+}} zmm4 = zmm1[8],zmm0[8],zmm1[9],zmm0[9],zmm1[10],zmm0[10],zmm1[11],zmm0[11],zmm1[12],zmm0[12],zmm1[13],zmm0[13],zmm1[14],zmm0[14],zmm1[15],zmm0[15],zmm1[24],zmm0[24],zmm1[25],zmm0[25],zmm1[26],zmm0[26],zmm1[27],zmm0[27],zmm1[28],zmm0[28],zmm1[29],zmm0[29],zmm1[30],zmm0[30],zmm1[31],zmm0[31],zmm1[40],zmm0[40],zmm1[41],zmm0[41],zmm1[42],zmm0[42],zmm1[43],zmm0[43],zmm1[44],zmm0[44],zmm1[45],zmm0[45],zmm1[46],zmm0[46],zmm1[47],zmm0[47],zmm1[56],zmm0[56],zmm1[57],zmm0[57],zmm1[58],zmm0[58],zmm1[59],zmm0[59],zmm1[60],zmm0[60],zmm1[61],zmm0[61],zmm1[62],zmm0[62],zmm1[63],zmm0[63]
-; AVX512BWVL-NEXT:    vpmullw %zmm4, %zmm3, %zmm3
-; AVX512BWVL-NEXT:    vpandq %zmm2, %zmm3, %zmm3
+; AVX512BWVL-NEXT:    vpmullw %zmm4, %zmm2, %zmm2
+; AVX512BWVL-NEXT:    vpandq %zmm3, %zmm2, %zmm2
 ; AVX512BWVL-NEXT:    vpunpcklbw {{.*#+}} zmm0 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
 ; AVX512BWVL-NEXT:    vpunpcklbw {{.*#+}} zmm1 = zmm1[0],zmm0[0],zmm1[1],zmm0[1],zmm1[2],zmm0[2],zmm1[3],zmm0[3],zmm1[4],zmm0[4],zmm1[5],zmm0[5],zmm1[6],zmm0[6],zmm1[7],zmm0[7],zmm1[16],zmm0[16],zmm1[17],zmm0[17],zmm1[18],zmm0[18],zmm1[19],zmm0[19],zmm1[20],zmm0[20],zmm1[21],zmm0[21],zmm1[22],zmm0[22],zmm1[23],zmm0[23],zmm1[32],zmm0[32],zmm1[33],zmm0[33],zmm1[34],zmm0[34],zmm1[35],zmm0[35],zmm1[36],zmm0[36],zmm1[37],zmm0[37],zmm1[38],zmm0[38],zmm1[39],zmm0[39],zmm1[48],zmm0[48],zmm1[49],zmm0[49],zmm1[50],zmm0[50],zmm1[51],zmm0[51],zmm1[52],zmm0[52],zmm1[53],zmm0[53],zmm1[54],zmm0[54],zmm1[55],zmm0[55]
 ; AVX512BWVL-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
-; AVX512BWVL-NEXT:    vpandq %zmm2, %zmm0, %zmm0
-; AVX512BWVL-NEXT:    vpackuswb %zmm3, %zmm0, %zmm0
-; AVX512BWVL-NEXT:    vpunpcklbw {{.*#+}} zmm1 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
-; AVX512BWVL-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; AVX512BWVL-NEXT:    vpmullw %zmm0, %zmm1, %zmm0
-; AVX512BWVL-NEXT:    vpandq %zmm2, %zmm0, %zmm0
+; AVX512BWVL-NEXT:    vpandq %zmm3, %zmm0, %zmm0
+; AVX512BWVL-NEXT:    vpackuswb %zmm2, %zmm0, %zmm0
+; AVX512BWVL-NEXT:    vpunpckhbw {{.*#+}} zmm1 = zmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]
+; AVX512BWVL-NEXT:    vpunpcklbw {{.*#+}} zmm0 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
+; AVX512BWVL-NEXT:    vpmullw %zmm1, %zmm0, %zmm0
+; AVX512BWVL-NEXT:    vpandq %zmm3, %zmm0, %zmm0
 ; AVX512BWVL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
 ; AVX512BWVL-NEXT:    vpackuswb %zmm1, %zmm0, %zmm0
-; AVX512BWVL-NEXT:    vpunpcklbw {{.*#+}} zmm3 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
-; AVX512BWVL-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; AVX512BWVL-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; AVX512BWVL-NEXT:    vpmullw %zmm0, %zmm3, %zmm0
-; AVX512BWVL-NEXT:    vpandq %zmm2, %zmm0, %zmm0
+; AVX512BWVL-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[1,1,2,3]
+; AVX512BWVL-NEXT:    vpunpcklbw {{.*#+}} zmm0 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
+; AVX512BWVL-NEXT:    vpunpcklbw {{.*#+}} zmm2 = zmm2[0],zmm0[0],zmm2[1],zmm0[1],zmm2[2],zmm0[2],zmm2[3],zmm0[3],zmm2[4],zmm0[4],zmm2[5],zmm0[5],zmm2[6],zmm0[6],zmm2[7],zmm0[7],zmm2[16],zmm0[16],zmm2[17],zmm0[17],zmm2[18],zmm0[18],zmm2[19],zmm0[19],zmm2[20],zmm0[20],zmm2[21],zmm0[21],zmm2[22],zmm0[22],zmm2[23],zmm0[23],zmm2[32],zmm0[32],zmm2[33],zmm0[33],zmm2[34],zmm0[34],zmm2[35],zmm0[35],zmm2[36],zmm0[36],zmm2[37],zmm0[37],zmm2[38],zmm0[38],zmm2[39],zmm0[39],zmm2[48],zmm0[48],zmm2[49],zmm0[49],zmm2[50],zmm0[50],zmm2[51],zmm0[51],zmm2[52],zmm0[52],zmm2[53],zmm0[53],zmm2[54],zmm0[54],zmm2[55],zmm0[55]
+; AVX512BWVL-NEXT:    vpmullw %zmm2, %zmm0, %zmm0
+; AVX512BWVL-NEXT:    vpandq %zmm3, %zmm0, %zmm0
 ; AVX512BWVL-NEXT:    vpackuswb %zmm1, %zmm0, %zmm0
-; AVX512BWVL-NEXT:    vpunpcklbw {{.*#+}} zmm3 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
-; AVX512BWVL-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; AVX512BWVL-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
-; AVX512BWVL-NEXT:    vpmullw %zmm0, %zmm3, %zmm0
-; AVX512BWVL-NEXT:    vpandq %zmm2, %zmm0, %zmm0
+; AVX512BWVL-NEXT:    vpsrld $16, %xmm0, %xmm2
+; AVX512BWVL-NEXT:    vpunpcklbw {{.*#+}} zmm0 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
+; AVX512BWVL-NEXT:    vpunpcklbw {{.*#+}} zmm2 = zmm2[0],zmm0[0],zmm2[1],zmm0[1],zmm2[2],zmm0[2],zmm2[3],zmm0[3],zmm2[4],zmm0[4],zmm2[5],zmm0[5],zmm2[6],zmm0[6],zmm2[7],zmm0[7],zmm2[16],zmm0[16],zmm2[17],zmm0[17],zmm2[18],zmm0[18],zmm2[19],zmm0[19],zmm2[20],zmm0[20],zmm2[21],zmm0[21],zmm2[22],zmm0[22],zmm2[23],zmm0[23],zmm2[32],zmm0[32],zmm2[33],zmm0[33],zmm2[34],zmm0[34],zmm2[35],zmm0[35],zmm2[36],zmm0[36],zmm2[37],zmm0[37],zmm2[38],zmm0[38],zmm2[39],zmm0[39],zmm2[48],zmm0[48],zmm2[49],zmm0[49],zmm2[50],zmm0[50],zmm2[51],zmm0[51],zmm2[52],zmm0[52],zmm2[53],zmm0[53],zmm2[54],zmm0[54],zmm2[55],zmm0[55]
+; AVX512BWVL-NEXT:    vpmullw %zmm2, %zmm0, %zmm0
+; AVX512BWVL-NEXT:    vpandq %zmm3, %zmm0, %zmm0
 ; AVX512BWVL-NEXT:    vpackuswb %zmm1, %zmm0, %zmm0
-; AVX512BWVL-NEXT:    vpsrlw $8, %xmm0, %xmm3
+; AVX512BWVL-NEXT:    vpsrlw $8, %xmm0, %xmm2
 ; AVX512BWVL-NEXT:    vpunpcklbw {{.*#+}} zmm0 = zmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23,32,32,33,33,34,34,35,35,36,36,37,37,38,38,39,39,48,48,49,49,50,50,51,51,52,52,53,53,54,54,55,55]
-; AVX512BWVL-NEXT:    vpmullw %zmm3, %zmm0, %zmm0
-; AVX512BWVL-NEXT:    vpandq %zmm2, %zmm0, %zmm0
+; AVX512BWVL-NEXT:    vpunpcklbw {{.*#+}} zmm2 = zmm2[0],zmm0[0],zmm2[1],zmm0[1],zmm2[2],zmm0[2],zmm2[3],zmm0[3],zmm2[4],zmm0[4],zmm2[5],zmm0[5],zmm2[6],zmm0[6],zmm2[7],zmm0[7],zmm2[16],zmm0[16],zmm2[17],zmm0[17],zmm2[18],zmm0[18],zmm2[19],zmm0[19],zmm2[20],zmm0[20],zmm2[21],zmm0[21],zmm2[22],zmm0[22],zmm2[23],zmm0[23],zmm2[32],zmm0[32],zmm2[33],zmm0[33],zmm2[34],zmm0[34],zmm2[35],zmm0[35],zmm2[36],zmm0[36],zmm2[37],zmm0[37],zmm2[38],zmm0[38],zmm2[39],zmm0[39],zmm2[48],zmm0[48],zmm2[49],zmm0[49],zmm2[50],zmm0[50],zmm2[51],zmm0[51],zmm2[52],zmm0[52],zmm2[53],zmm0[53],zmm2[54],zmm0[54],zmm2[55],zmm0[55]
+; AVX512BWVL-NEXT:    vpmullw %zmm2, %zmm0, %zmm0
+; AVX512BWVL-NEXT:    vpandq %zmm3, %zmm0, %zmm0
 ; AVX512BWVL-NEXT:    vpackuswb %zmm1, %zmm0, %zmm0
 ; AVX512BWVL-NEXT:    vpextrb $0, %xmm0, %eax
 ; AVX512BWVL-NEXT:    # kill: def $al killed $al killed $eax
@@ -3281,26 +3211,27 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; AVX512DQ-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vpand %ymm4, %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vpackuswb %ymm2, %ymm0, %ymm0
-; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
-; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; AVX512DQ-NEXT:    vpmullw %ymm0, %ymm1, %ymm0
+; AVX512DQ-NEXT:    vpunpckhbw {{.*#+}} ymm1 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
+; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX512DQ-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vpand %ymm4, %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vpxor %xmm1, %xmm1, %xmm1
 ; AVX512DQ-NEXT:    vpackuswb %ymm1, %ymm0, %ymm0
-; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
-; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; AVX512DQ-NEXT:    vpmullw %ymm0, %ymm2, %ymm0
+; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[1,1,2,3]
+; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm2[0],ymm0[0],ymm2[1],ymm0[1],ymm2[2],ymm0[2],ymm2[3],ymm0[3],ymm2[4],ymm0[4],ymm2[5],ymm0[5],ymm2[6],ymm0[6],ymm2[7],ymm0[7],ymm2[16],ymm0[16],ymm2[17],ymm0[17],ymm2[18],ymm0[18],ymm2[19],ymm0[19],ymm2[20],ymm0[20],ymm2[21],ymm0[21],ymm2[22],ymm0[22],ymm2[23],ymm0[23]
+; AVX512DQ-NEXT:    vpmullw %ymm2, %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vpand %ymm4, %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vpackuswb %ymm1, %ymm0, %ymm0
-; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
-; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; AVX512DQ-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
-; AVX512DQ-NEXT:    vpmullw %ymm0, %ymm2, %ymm0
+; AVX512DQ-NEXT:    vpsrld $16, %xmm0, %xmm2
+; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm2[0],ymm0[0],ymm2[1],ymm0[1],ymm2[2],ymm0[2],ymm2[3],ymm0[3],ymm2[4],ymm0[4],ymm2[5],ymm0[5],ymm2[6],ymm0[6],ymm2[7],ymm0[7],ymm2[16],ymm0[16],ymm2[17],ymm0[17],ymm2[18],ymm0[18],ymm2[19],ymm0[19],ymm2[20],ymm0[20],ymm2[21],ymm0[21],ymm2[22],ymm0[22],ymm2[23],ymm0[23]
+; AVX512DQ-NEXT:    vpmullw %ymm2, %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vpand %ymm4, %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vpackuswb %ymm1, %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vpsrlw $8, %xmm0, %xmm2
 ; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX512DQ-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm2[0],ymm0[0],ymm2[1],ymm0[1],ymm2[2],ymm0[2],ymm2[3],ymm0[3],ymm2[4],ymm0[4],ymm2[5],ymm0[5],ymm2[6],ymm0[6],ymm2[7],ymm0[7],ymm2[16],ymm0[16],ymm2[17],ymm0[17],ymm2[18],ymm0[18],ymm2[19],ymm0[19],ymm2[20],ymm0[20],ymm2[21],ymm0[21],ymm2[22],ymm0[22],ymm2[23],ymm0[23]
 ; AVX512DQ-NEXT:    vpmullw %ymm2, %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vpand %ymm4, %ymm0, %ymm0
 ; AVX512DQ-NEXT:    vpackuswb %ymm1, %ymm0, %ymm0
@@ -3313,64 +3244,65 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; AVX512DQVL:       # %bb.0:
 ; AVX512DQVL-NEXT:    vpunpckhbw {{.*#+}} ymm4 = ymm2[8],ymm0[8],ymm2[9],ymm0[9],ymm2[10],ymm0[10],ymm2[11],ymm0[11],ymm2[12],ymm0[12],ymm2[13],ymm0[13],ymm2[14],ymm0[14],ymm2[15],ymm0[15],ymm2[24],ymm0[24],ymm2[25],ymm0[25],ymm2[26],ymm0[26],ymm2[27],ymm0[27],ymm2[28],ymm0[28],ymm2[29],ymm0[29],ymm2[30],ymm0[30],ymm2[31],ymm0[31]
 ; AVX512DQVL-NEXT:    vpunpckhbw {{.*#+}} ymm5 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
-; AVX512DQVL-NEXT:    vpmullw %ymm4, %ymm5, %ymm5
-; AVX512DQVL-NEXT:    vmovdqa {{.*#+}} ymm4 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
-; AVX512DQVL-NEXT:    vpand %ymm4, %ymm5, %ymm5
+; AVX512DQVL-NEXT:    vpmullw %ymm4, %ymm5, %ymm4
+; AVX512DQVL-NEXT:    vmovdqa {{.*#+}} ymm5 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; AVX512DQVL-NEXT:    vpand %ymm5, %ymm4, %ymm4
 ; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm2[0],ymm0[0],ymm2[1],ymm0[1],ymm2[2],ymm0[2],ymm2[3],ymm0[3],ymm2[4],ymm0[4],ymm2[5],ymm0[5],ymm2[6],ymm0[6],ymm2[7],ymm0[7],ymm2[16],ymm0[16],ymm2[17],ymm0[17],ymm2[18],ymm0[18],ymm2[19],ymm0[19],ymm2[20],ymm0[20],ymm2[21],ymm0[21],ymm2[22],ymm0[22],ymm2[23],ymm0[23]
 ; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
 ; AVX512DQVL-NEXT:    vpmullw %ymm2, %ymm0, %ymm0
-; AVX512DQVL-NEXT:    vpand %ymm4, %ymm0, %ymm0
-; AVX512DQVL-NEXT:    vpackuswb %ymm5, %ymm0, %ymm0
+; AVX512DQVL-NEXT:    vpand %ymm5, %ymm0, %ymm0
+; AVX512DQVL-NEXT:    vpackuswb %ymm4, %ymm0, %ymm0
 ; AVX512DQVL-NEXT:    vpunpckhbw {{.*#+}} ymm2 = ymm3[8],ymm0[8],ymm3[9],ymm0[9],ymm3[10],ymm0[10],ymm3[11],ymm0[11],ymm3[12],ymm0[12],ymm3[13],ymm0[13],ymm3[14],ymm0[14],ymm3[15],ymm0[15],ymm3[24],ymm0[24],ymm3[25],ymm0[25],ymm3[26],ymm0[26],ymm3[27],ymm0[27],ymm3[28],ymm0[28],ymm3[29],ymm0[29],ymm3[30],ymm0[30],ymm3[31],ymm0[31]
-; AVX512DQVL-NEXT:    vpunpckhbw {{.*#+}} ymm5 = ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11],ymm1[12],ymm0[12],ymm1[13],ymm0[13],ymm1[14],ymm0[14],ymm1[15],ymm0[15],ymm1[24],ymm0[24],ymm1[25],ymm0[25],ymm1[26],ymm0[26],ymm1[27],ymm0[27],ymm1[28],ymm0[28],ymm1[29],ymm0[29],ymm1[30],ymm0[30],ymm1[31],ymm0[31]
-; AVX512DQVL-NEXT:    vpmullw %ymm2, %ymm5, %ymm2
-; AVX512DQVL-NEXT:    vpand %ymm4, %ymm2, %ymm2
+; AVX512DQVL-NEXT:    vpunpckhbw {{.*#+}} ymm4 = ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11],ymm1[12],ymm0[12],ymm1[13],ymm0[13],ymm1[14],ymm0[14],ymm1[15],ymm0[15],ymm1[24],ymm0[24],ymm1[25],ymm0[25],ymm1[26],ymm0[26],ymm1[27],ymm0[27],ymm1[28],ymm0[28],ymm1[29],ymm0[29],ymm1[30],ymm0[30],ymm1[31],ymm0[31]
+; AVX512DQVL-NEXT:    vpmullw %ymm2, %ymm4, %ymm2
+; AVX512DQVL-NEXT:    vpand %ymm5, %ymm2, %ymm2
 ; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} ymm3 = ymm3[0],ymm0[0],ymm3[1],ymm0[1],ymm3[2],ymm0[2],ymm3[3],ymm0[3],ymm3[4],ymm0[4],ymm3[5],ymm0[5],ymm3[6],ymm0[6],ymm3[7],ymm0[7],ymm3[16],ymm0[16],ymm3[17],ymm0[17],ymm3[18],ymm0[18],ymm3[19],ymm0[19],ymm3[20],ymm0[20],ymm3[21],ymm0[21],ymm3[22],ymm0[22],ymm3[23],ymm0[23]
 ; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[16],ymm0[16],ymm1[17],ymm0[17],ymm1[18],ymm0[18],ymm1[19],ymm0[19],ymm1[20],ymm0[20],ymm1[21],ymm0[21],ymm1[22],ymm0[22],ymm1[23],ymm0[23]
 ; AVX512DQVL-NEXT:    vpmullw %ymm3, %ymm1, %ymm1
-; AVX512DQVL-NEXT:    vpand %ymm4, %ymm1, %ymm1
+; AVX512DQVL-NEXT:    vpand %ymm5, %ymm1, %ymm1
 ; AVX512DQVL-NEXT:    vpackuswb %ymm2, %ymm1, %ymm1
 ; AVX512DQVL-NEXT:    vpunpckhbw {{.*#+}} ymm2 = ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11],ymm1[12],ymm0[12],ymm1[13],ymm0[13],ymm1[14],ymm0[14],ymm1[15],ymm0[15],ymm1[24],ymm0[24],ymm1[25],ymm0[25],ymm1[26],ymm0[26],ymm1[27],ymm0[27],ymm1[28],ymm0[28],ymm1[29],ymm0[29],ymm1[30],ymm0[30],ymm1[31],ymm0[31]
 ; AVX512DQVL-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
 ; AVX512DQVL-NEXT:    vpmullw %ymm2, %ymm3, %ymm2
-; AVX512DQVL-NEXT:    vpand %ymm4, %ymm2, %ymm2
+; AVX512DQVL-NEXT:    vpand %ymm5, %ymm2, %ymm2
 ; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[16],ymm0[16],ymm1[17],ymm0[17],ymm1[18],ymm0[18],ymm1[19],ymm0[19],ymm1[20],ymm0[20],ymm1[21],ymm0[21],ymm1[22],ymm0[22],ymm1[23],ymm0[23]
 ; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
 ; AVX512DQVL-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
-; AVX512DQVL-NEXT:    vpand %ymm4, %ymm0, %ymm0
+; AVX512DQVL-NEXT:    vpand %ymm5, %ymm0, %ymm0
 ; AVX512DQVL-NEXT:    vpackuswb %ymm2, %ymm0, %ymm0
 ; AVX512DQVL-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512DQVL-NEXT:    vpunpckhbw {{.*#+}} ymm2 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
 ; AVX512DQVL-NEXT:    vpunpckhbw {{.*#+}} ymm3 = ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11],ymm1[12],ymm0[12],ymm1[13],ymm0[13],ymm1[14],ymm0[14],ymm1[15],ymm0[15],ymm1[24],ymm0[24],ymm1[25],ymm0[25],ymm1[26],ymm0[26],ymm1[27],ymm0[27],ymm1[28],ymm0[28],ymm1[29],ymm0[29],ymm1[30],ymm0[30],ymm1[31],ymm0[31]
 ; AVX512DQVL-NEXT:    vpmullw %ymm3, %ymm2, %ymm2
-; AVX512DQVL-NEXT:    vpand %ymm4, %ymm2, %ymm2
+; AVX512DQVL-NEXT:    vpand %ymm5, %ymm2, %ymm2
 ; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
 ; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[16],ymm0[16],ymm1[17],ymm0[17],ymm1[18],ymm0[18],ymm1[19],ymm0[19],ymm1[20],ymm0[20],ymm1[21],ymm0[21],ymm1[22],ymm0[22],ymm1[23],ymm0[23]
 ; AVX512DQVL-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
-; AVX512DQVL-NEXT:    vpand %ymm4, %ymm0, %ymm0
+; AVX512DQVL-NEXT:    vpand %ymm5, %ymm0, %ymm0
 ; AVX512DQVL-NEXT:    vpackuswb %ymm2, %ymm0, %ymm0
-; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} ymm1 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
-; AVX512DQVL-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; AVX512DQVL-NEXT:    vpmullw %ymm0, %ymm1, %ymm0
-; AVX512DQVL-NEXT:    vpand %ymm4, %ymm0, %ymm0
+; AVX512DQVL-NEXT:    vpunpckhbw {{.*#+}} ymm1 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
+; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX512DQVL-NEXT:    vpmullw %ymm1, %ymm0, %ymm0
+; AVX512DQVL-NEXT:    vpand %ymm5, %ymm0, %ymm0
 ; AVX512DQVL-NEXT:    vpxor %xmm1, %xmm1, %xmm1
 ; AVX512DQVL-NEXT:    vpackuswb %ymm1, %ymm0, %ymm0
-; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
-; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; AVX512DQVL-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; AVX512DQVL-NEXT:    vpmullw %ymm0, %ymm2, %ymm0
-; AVX512DQVL-NEXT:    vpand %ymm4, %ymm0, %ymm0
+; AVX512DQVL-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[1,1,2,3]
+; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm2[0],ymm0[0],ymm2[1],ymm0[1],ymm2[2],ymm0[2],ymm2[3],ymm0[3],ymm2[4],ymm0[4],ymm2[5],ymm0[5],ymm2[6],ymm0[6],ymm2[7],ymm0[7],ymm2[16],ymm0[16],ymm2[17],ymm0[17],ymm2[18],ymm0[18],ymm2[19],ymm0[19],ymm2[20],ymm0[20],ymm2[21],ymm0[21],ymm2[22],ymm0[22],ymm2[23],ymm0[23]
+; AVX512DQVL-NEXT:    vpmullw %ymm2, %ymm0, %ymm0
+; AVX512DQVL-NEXT:    vpand %ymm5, %ymm0, %ymm0
 ; AVX512DQVL-NEXT:    vpackuswb %ymm1, %ymm0, %ymm0
-; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
-; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; AVX512DQVL-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
-; AVX512DQVL-NEXT:    vpmullw %ymm0, %ymm2, %ymm0
-; AVX512DQVL-NEXT:    vpand %ymm4, %ymm0, %ymm0
+; AVX512DQVL-NEXT:    vpsrld $16, %xmm0, %xmm2
+; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm2[0],ymm0[0],ymm2[1],ymm0[1],ymm2[2],ymm0[2],ymm2[3],ymm0[3],ymm2[4],ymm0[4],ymm2[5],ymm0[5],ymm2[6],ymm0[6],ymm2[7],ymm0[7],ymm2[16],ymm0[16],ymm2[17],ymm0[17],ymm2[18],ymm0[18],ymm2[19],ymm0[19],ymm2[20],ymm0[20],ymm2[21],ymm0[21],ymm2[22],ymm0[22],ymm2[23],ymm0[23]
+; AVX512DQVL-NEXT:    vpmullw %ymm2, %ymm0, %ymm0
+; AVX512DQVL-NEXT:    vpand %ymm5, %ymm0, %ymm0
 ; AVX512DQVL-NEXT:    vpackuswb %ymm1, %ymm0, %ymm0
 ; AVX512DQVL-NEXT:    vpsrlw $8, %xmm0, %xmm2
 ; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
+; AVX512DQVL-NEXT:    vpunpcklbw {{.*#+}} ymm2 = ymm2[0],ymm0[0],ymm2[1],ymm0[1],ymm2[2],ymm0[2],ymm2[3],ymm0[3],ymm2[4],ymm0[4],ymm2[5],ymm0[5],ymm2[6],ymm0[6],ymm2[7],ymm0[7],ymm2[16],ymm0[16],ymm2[17],ymm0[17],ymm2[18],ymm0[18],ymm2[19],ymm0[19],ymm2[20],ymm0[20],ymm2[21],ymm0[21],ymm2[22],ymm0[22],ymm2[23],ymm0[23]
 ; AVX512DQVL-NEXT:    vpmullw %ymm2, %ymm0, %ymm0
-; AVX512DQVL-NEXT:    vpand %ymm4, %ymm0, %ymm0
+; AVX512DQVL-NEXT:    vpand %ymm5, %ymm0, %ymm0
 ; AVX512DQVL-NEXT:    vpackuswb %ymm1, %ymm0, %ymm0
 ; AVX512DQVL-NEXT:    vpextrb $0, %xmm0, %eax
 ; AVX512DQVL-NEXT:    # kill: def $al killed $al killed $eax
diff --git a/test/CodeGen/X86/vector-reduce-or-widen.ll b/test/CodeGen/X86/vector-reduce-or-widen.ll
index 4de56b0b805f5ee0e68b96888a76bef7922169c6..ef060acfb67829daa406fc7d3063f3b4307370d2 100644
--- a/test/CodeGen/X86/vector-reduce-or-widen.ll
+++ b/test/CodeGen/X86/vector-reduce-or-widen.ll
@@ -47,9 +47,9 @@ define i64 @test_v4i64(<4 x i64> %a0) {
 ; AVX1-LABEL: test_v4i64:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX1-NEXT:    vorps %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovq %xmm0, %rax
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
@@ -57,7 +57,7 @@ define i64 @test_v4i64(<4 x i64> %a0) {
 ; AVX2-LABEL: test_v4i64:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovq %xmm0, %rax
@@ -67,7 +67,7 @@ define i64 @test_v4i64(<4 x i64> %a0) {
 ; AVX512-LABEL: test_v4i64:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovq %xmm0, %rax
@@ -92,7 +92,7 @@ define i64 @test_v8i64(<8 x i64> %a0) {
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; AVX1-NEXT:    vorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovq %xmm0, %rax
@@ -103,7 +103,7 @@ define i64 @test_v8i64(<8 x i64> %a0) {
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovq %xmm0, %rax
@@ -115,7 +115,7 @@ define i64 @test_v8i64(<8 x i64> %a0) {
 ; AVX512-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512-NEXT:    vporq %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovq %xmm0, %rax
@@ -146,7 +146,7 @@ define i64 @test_v16i64(<16 x i64> %a0) {
 ; AVX1-NEXT:    vorps %ymm1, %ymm2, %ymm1
 ; AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; AVX1-NEXT:    vorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovq %xmm0, %rax
@@ -159,7 +159,7 @@ define i64 @test_v16i64(<16 x i64> %a0) {
 ; AVX2-NEXT:    vpor %ymm1, %ymm2, %ymm1
 ; AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovq %xmm0, %rax
@@ -172,7 +172,7 @@ define i64 @test_v16i64(<16 x i64> %a0) {
 ; AVX512-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512-NEXT:    vporq %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovq %xmm0, %rax
@@ -256,11 +256,11 @@ define i32 @test_v8i32(<8 x i32> %a0) {
 ; AVX1-LABEL: test_v8i32:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX1-NEXT:    vorps %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovd %xmm0, %eax
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
@@ -268,9 +268,9 @@ define i32 @test_v8i32(<8 x i32> %a0) {
 ; AVX2-LABEL: test_v8i32:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
@@ -280,9 +280,9 @@ define i32 @test_v8i32(<8 x i32> %a0) {
 ; AVX512-LABEL: test_v8i32:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
@@ -309,9 +309,9 @@ define i32 @test_v16i32(<16 x i32> %a0) {
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX1-NEXT:    vorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovd %xmm0, %eax
@@ -322,9 +322,9 @@ define i32 @test_v16i32(<16 x i32> %a0) {
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
@@ -336,9 +336,9 @@ define i32 @test_v16i32(<16 x i32> %a0) {
 ; AVX512-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512-NEXT:    vpord %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vpord %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpord %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
@@ -371,9 +371,9 @@ define i32 @test_v32i32(<32 x i32> %a0) {
 ; AVX1-NEXT:    vorps %ymm1, %ymm2, %ymm1
 ; AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX1-NEXT:    vorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovd %xmm0, %eax
@@ -386,9 +386,9 @@ define i32 @test_v32i32(<32 x i32> %a0) {
 ; AVX2-NEXT:    vpor %ymm1, %ymm2, %ymm1
 ; AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
@@ -401,9 +401,9 @@ define i32 @test_v32i32(<32 x i32> %a0) {
 ; AVX512-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512-NEXT:    vpord %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vpord %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpord %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
@@ -540,11 +540,11 @@ define i16 @test_v16i16(<16 x i16> %a0) {
 ; AVX1-LABEL: test_v16i16:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX1-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovd %xmm0, %eax
@@ -555,11 +555,11 @@ define i16 @test_v16i16(<16 x i16> %a0) {
 ; AVX2-LABEL: test_v16i16:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
@@ -570,11 +570,11 @@ define i16 @test_v16i16(<16 x i16> %a0) {
 ; AVX512-LABEL: test_v16i16:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX512-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
@@ -606,11 +606,11 @@ define i16 @test_v32i16(<32 x i16> %a0) {
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX1-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovd %xmm0, %eax
@@ -622,11 +622,11 @@ define i16 @test_v32i16(<32 x i16> %a0) {
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
@@ -639,11 +639,11 @@ define i16 @test_v32i16(<32 x i16> %a0) {
 ; AVX512-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512-NEXT:    vporq %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX512-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
@@ -681,11 +681,11 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 ; AVX1-NEXT:    vorps %ymm1, %ymm2, %ymm1
 ; AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX1-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovd %xmm0, %eax
@@ -699,11 +699,11 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 ; AVX2-NEXT:    vpor %ymm1, %ymm2, %ymm1
 ; AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
@@ -717,11 +717,11 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 ; AVX512-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512-NEXT:    vporq %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX512-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
@@ -980,13 +980,13 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; AVX1-LABEL: test_v32i8:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm1
 ; AVX1-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpextrb $0, %xmm0, %eax
@@ -997,13 +997,13 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; AVX2-LABEL: test_v32i8:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpsrlw $8, %xmm0, %xmm1
 ; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpextrb $0, %xmm0, %eax
@@ -1014,13 +1014,13 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; AVX512-LABEL: test_v32i8:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX512-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX512-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrlw $8, %xmm0, %xmm1
 ; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpextrb $0, %xmm0, %eax
@@ -1074,13 +1074,13 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm1
 ; AVX1-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpextrb $0, %xmm0, %eax
@@ -1092,13 +1092,13 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpsrlw $8, %xmm0, %xmm1
 ; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpextrb $0, %xmm0, %eax
@@ -1111,13 +1111,13 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; AVX512-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512-NEXT:    vporq %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX512-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX512-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrlw $8, %xmm0, %xmm1
 ; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpextrb $0, %xmm0, %eax
@@ -1181,13 +1181,13 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; AVX1-NEXT:    vorps %ymm1, %ymm2, %ymm1
 ; AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm1
 ; AVX1-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpextrb $0, %xmm0, %eax
@@ -1201,13 +1201,13 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; AVX2-NEXT:    vpor %ymm1, %ymm2, %ymm1
 ; AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpsrlw $8, %xmm0, %xmm1
 ; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpextrb $0, %xmm0, %eax
@@ -1221,13 +1221,13 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; AVX512-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512-NEXT:    vporq %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX512-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX512-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrlw $8, %xmm0, %xmm1
 ; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpextrb $0, %xmm0, %eax
diff --git a/test/CodeGen/X86/vector-reduce-or.ll b/test/CodeGen/X86/vector-reduce-or.ll
index 1cb6af66bf4651f8075c274d8c4a5f10df242cb6..f9c08cb0e987996d20722487fc0f45836321354e 100644
--- a/test/CodeGen/X86/vector-reduce-or.ll
+++ b/test/CodeGen/X86/vector-reduce-or.ll
@@ -47,9 +47,9 @@ define i64 @test_v4i64(<4 x i64> %a0) {
 ; AVX1-LABEL: test_v4i64:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX1-NEXT:    vorps %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovq %xmm0, %rax
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
@@ -57,7 +57,7 @@ define i64 @test_v4i64(<4 x i64> %a0) {
 ; AVX2-LABEL: test_v4i64:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovq %xmm0, %rax
@@ -67,7 +67,7 @@ define i64 @test_v4i64(<4 x i64> %a0) {
 ; AVX512-LABEL: test_v4i64:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovq %xmm0, %rax
@@ -92,7 +92,7 @@ define i64 @test_v8i64(<8 x i64> %a0) {
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; AVX1-NEXT:    vorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovq %xmm0, %rax
@@ -103,7 +103,7 @@ define i64 @test_v8i64(<8 x i64> %a0) {
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovq %xmm0, %rax
@@ -115,7 +115,7 @@ define i64 @test_v8i64(<8 x i64> %a0) {
 ; AVX512-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512-NEXT:    vporq %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovq %xmm0, %rax
@@ -146,7 +146,7 @@ define i64 @test_v16i64(<16 x i64> %a0) {
 ; AVX1-NEXT:    vorps %ymm1, %ymm2, %ymm1
 ; AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; AVX1-NEXT:    vorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovq %xmm0, %rax
@@ -159,7 +159,7 @@ define i64 @test_v16i64(<16 x i64> %a0) {
 ; AVX2-NEXT:    vpor %ymm1, %ymm2, %ymm1
 ; AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovq %xmm0, %rax
@@ -172,7 +172,7 @@ define i64 @test_v16i64(<16 x i64> %a0) {
 ; AVX512-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512-NEXT:    vporq %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovq %xmm0, %rax
@@ -256,11 +256,11 @@ define i32 @test_v8i32(<8 x i32> %a0) {
 ; AVX1-LABEL: test_v8i32:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX1-NEXT:    vorps %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovd %xmm0, %eax
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
@@ -268,9 +268,9 @@ define i32 @test_v8i32(<8 x i32> %a0) {
 ; AVX2-LABEL: test_v8i32:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
@@ -280,9 +280,9 @@ define i32 @test_v8i32(<8 x i32> %a0) {
 ; AVX512-LABEL: test_v8i32:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
@@ -309,9 +309,9 @@ define i32 @test_v16i32(<16 x i32> %a0) {
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX1-NEXT:    vorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovd %xmm0, %eax
@@ -322,9 +322,9 @@ define i32 @test_v16i32(<16 x i32> %a0) {
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
@@ -336,9 +336,9 @@ define i32 @test_v16i32(<16 x i32> %a0) {
 ; AVX512-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512-NEXT:    vpord %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vpord %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpord %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
@@ -371,9 +371,9 @@ define i32 @test_v32i32(<32 x i32> %a0) {
 ; AVX1-NEXT:    vorps %ymm1, %ymm2, %ymm1
 ; AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX1-NEXT:    vorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovd %xmm0, %eax
@@ -386,9 +386,9 @@ define i32 @test_v32i32(<32 x i32> %a0) {
 ; AVX2-NEXT:    vpor %ymm1, %ymm2, %ymm1
 ; AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
@@ -401,9 +401,9 @@ define i32 @test_v32i32(<32 x i32> %a0) {
 ; AVX512-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512-NEXT:    vpord %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vpord %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpord %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
@@ -538,11 +538,11 @@ define i16 @test_v16i16(<16 x i16> %a0) {
 ; AVX1-LABEL: test_v16i16:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX1-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovd %xmm0, %eax
@@ -553,11 +553,11 @@ define i16 @test_v16i16(<16 x i16> %a0) {
 ; AVX2-LABEL: test_v16i16:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
@@ -568,11 +568,11 @@ define i16 @test_v16i16(<16 x i16> %a0) {
 ; AVX512-LABEL: test_v16i16:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX512-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
@@ -604,11 +604,11 @@ define i16 @test_v32i16(<32 x i16> %a0) {
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX1-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovd %xmm0, %eax
@@ -620,11 +620,11 @@ define i16 @test_v32i16(<32 x i16> %a0) {
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
@@ -637,11 +637,11 @@ define i16 @test_v32i16(<32 x i16> %a0) {
 ; AVX512-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512-NEXT:    vporq %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX512-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
@@ -679,11 +679,11 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 ; AVX1-NEXT:    vorps %ymm1, %ymm2, %ymm1
 ; AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX1-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovd %xmm0, %eax
@@ -697,11 +697,11 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 ; AVX2-NEXT:    vpor %ymm1, %ymm2, %ymm1
 ; AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
@@ -715,11 +715,11 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 ; AVX512-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512-NEXT:    vporq %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX512-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
@@ -970,13 +970,13 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; AVX1-LABEL: test_v32i8:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm1
 ; AVX1-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpextrb $0, %xmm0, %eax
@@ -987,13 +987,13 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; AVX2-LABEL: test_v32i8:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpsrlw $8, %xmm0, %xmm1
 ; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpextrb $0, %xmm0, %eax
@@ -1004,13 +1004,13 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; AVX512-LABEL: test_v32i8:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX512-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX512-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrlw $8, %xmm0, %xmm1
 ; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpextrb $0, %xmm0, %eax
@@ -1064,13 +1064,13 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm1
 ; AVX1-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpextrb $0, %xmm0, %eax
@@ -1082,13 +1082,13 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpsrlw $8, %xmm0, %xmm1
 ; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpextrb $0, %xmm0, %eax
@@ -1101,13 +1101,13 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; AVX512-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512-NEXT:    vporq %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX512-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX512-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrlw $8, %xmm0, %xmm1
 ; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpextrb $0, %xmm0, %eax
@@ -1171,13 +1171,13 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; AVX1-NEXT:    vorps %ymm1, %ymm2, %ymm1
 ; AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX1-NEXT:    vorps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm1
 ; AVX1-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpextrb $0, %xmm0, %eax
@@ -1191,13 +1191,13 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; AVX2-NEXT:    vpor %ymm1, %ymm2, %ymm1
 ; AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpsrlw $8, %xmm0, %xmm1
 ; AVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpextrb $0, %xmm0, %eax
@@ -1211,13 +1211,13 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; AVX512-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512-NEXT:    vporq %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX512-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX512-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrlw $8, %xmm0, %xmm1
 ; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpextrb $0, %xmm0, %eax
diff --git a/test/CodeGen/X86/vector-reduce-smax-widen.ll b/test/CodeGen/X86/vector-reduce-smax-widen.ll
index b31c2cf5dbb3ec0073ee0c8ce69f5851eae1a4d4..7be51758b1ca708153eac1a94c5eda85946148c7 100644
--- a/test/CodeGen/X86/vector-reduce-smax-widen.ll
+++ b/test/CodeGen/X86/vector-reduce-smax-widen.ll
@@ -1120,31 +1120,28 @@ define i16 @test_v8i16(<8 x i16> %a0) {
 ;
 ; SSE41-LABEL: test_v8i16:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [32767,32767,32767,32767,32767,32767,32767,32767]
-; SSE41-NEXT:    pxor %xmm1, %xmm0
+; SSE41-NEXT:    pxor {{.*}}(%rip), %xmm0
 ; SSE41-NEXT:    phminposuw %xmm0, %xmm0
-; SSE41-NEXT:    pxor %xmm1, %xmm0
 ; SSE41-NEXT:    movd %xmm0, %eax
+; SSE41-NEXT:    xorl $32767, %eax # imm = 0x7FFF
 ; SSE41-NEXT:    # kill: def $ax killed $ax killed $eax
 ; SSE41-NEXT:    retq
 ;
 ; AVX-LABEL: test_v8i16:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vmovdqa {{.*#+}} xmm1 = [32767,32767,32767,32767,32767,32767,32767,32767]
-; AVX-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
 ; AVX-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vmovd %xmm0, %eax
+; AVX-NEXT:    xorl $32767, %eax # imm = 0x7FFF
 ; AVX-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX-NEXT:    retq
 ;
 ; AVX512-LABEL: test_v8i16:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vmovdqa {{.*#+}} xmm1 = [32767,32767,32767,32767,32767,32767,32767,32767]
-; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
 ; AVX512-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
+; AVX512-NEXT:    xorl $32767, %eax # imm = 0x7FFF
 ; AVX512-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX512-NEXT:    retq
   %1 = call i16 @llvm.experimental.vector.reduce.smax.i16.v8i16(<8 x i16> %a0)
@@ -1169,11 +1166,10 @@ define i16 @test_v16i16(<16 x i16> %a0) {
 ; SSE41-LABEL: test_v16i16:
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    pmaxsw %xmm1, %xmm0
-; SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [32767,32767,32767,32767,32767,32767,32767,32767]
-; SSE41-NEXT:    pxor %xmm1, %xmm0
+; SSE41-NEXT:    pxor {{.*}}(%rip), %xmm0
 ; SSE41-NEXT:    phminposuw %xmm0, %xmm0
-; SSE41-NEXT:    pxor %xmm1, %xmm0
 ; SSE41-NEXT:    movd %xmm0, %eax
+; SSE41-NEXT:    xorl $32767, %eax # imm = 0x7FFF
 ; SSE41-NEXT:    # kill: def $ax killed $ax killed $eax
 ; SSE41-NEXT:    retq
 ;
@@ -1181,11 +1177,10 @@ define i16 @test_v16i16(<16 x i16> %a0) {
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; AVX1-NEXT:    vpmaxsw %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm1 = [32767,32767,32767,32767,32767,32767,32767,32767]
-; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
 ; AVX1-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovd %xmm0, %eax
+; AVX1-NEXT:    xorl $32767, %eax # imm = 0x7FFF
 ; AVX1-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
@@ -1194,11 +1189,10 @@ define i16 @test_v16i16(<16 x i16> %a0) {
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpmaxsw %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vmovdqa {{.*#+}} xmm1 = [32767,32767,32767,32767,32767,32767,32767,32767]
-; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
 ; AVX2-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
+; AVX2-NEXT:    xorl $32767, %eax # imm = 0x7FFF
 ; AVX2-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
@@ -1207,11 +1201,10 @@ define i16 @test_v16i16(<16 x i16> %a0) {
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512-NEXT:    vpmaxsw %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vmovdqa {{.*#+}} xmm1 = [32767,32767,32767,32767,32767,32767,32767,32767]
-; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
 ; AVX512-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
+; AVX512-NEXT:    xorl $32767, %eax # imm = 0x7FFF
 ; AVX512-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
@@ -1241,11 +1234,10 @@ define i16 @test_v32i16(<32 x i16> %a0) {
 ; SSE41-NEXT:    pmaxsw %xmm3, %xmm1
 ; SSE41-NEXT:    pmaxsw %xmm2, %xmm0
 ; SSE41-NEXT:    pmaxsw %xmm1, %xmm0
-; SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [32767,32767,32767,32767,32767,32767,32767,32767]
-; SSE41-NEXT:    pxor %xmm1, %xmm0
+; SSE41-NEXT:    pxor {{.*}}(%rip), %xmm0
 ; SSE41-NEXT:    phminposuw %xmm0, %xmm0
-; SSE41-NEXT:    pxor %xmm1, %xmm0
 ; SSE41-NEXT:    movd %xmm0, %eax
+; SSE41-NEXT:    xorl $32767, %eax # imm = 0x7FFF
 ; SSE41-NEXT:    # kill: def $ax killed $ax killed $eax
 ; SSE41-NEXT:    retq
 ;
@@ -1256,11 +1248,10 @@ define i16 @test_v32i16(<32 x i16> %a0) {
 ; AVX1-NEXT:    vpmaxsw %xmm2, %xmm3, %xmm2
 ; AVX1-NEXT:    vpmaxsw %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpmaxsw %xmm2, %xmm0, %xmm0
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm1 = [32767,32767,32767,32767,32767,32767,32767,32767]
-; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
 ; AVX1-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovd %xmm0, %eax
+; AVX1-NEXT:    xorl $32767, %eax # imm = 0x7FFF
 ; AVX1-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
@@ -1270,11 +1261,10 @@ define i16 @test_v32i16(<32 x i16> %a0) {
 ; AVX2-NEXT:    vpmaxsw %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpmaxsw %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vmovdqa {{.*#+}} xmm1 = [32767,32767,32767,32767,32767,32767,32767,32767]
-; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
 ; AVX2-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
+; AVX2-NEXT:    xorl $32767, %eax # imm = 0x7FFF
 ; AVX2-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
@@ -1285,11 +1275,10 @@ define i16 @test_v32i16(<32 x i16> %a0) {
 ; AVX512-NEXT:    vpmaxsw %ymm1, %ymm0, %ymm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512-NEXT:    vpmaxsw %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vmovdqa {{.*#+}} xmm1 = [32767,32767,32767,32767,32767,32767,32767,32767]
-; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
 ; AVX512-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
+; AVX512-NEXT:    xorl $32767, %eax # imm = 0x7FFF
 ; AVX512-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
@@ -1327,11 +1316,10 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 ; SSE41-NEXT:    pmaxsw %xmm4, %xmm0
 ; SSE41-NEXT:    pmaxsw %xmm2, %xmm0
 ; SSE41-NEXT:    pmaxsw %xmm1, %xmm0
-; SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [32767,32767,32767,32767,32767,32767,32767,32767]
-; SSE41-NEXT:    pxor %xmm1, %xmm0
+; SSE41-NEXT:    pxor {{.*}}(%rip), %xmm0
 ; SSE41-NEXT:    phminposuw %xmm0, %xmm0
-; SSE41-NEXT:    pxor %xmm1, %xmm0
 ; SSE41-NEXT:    movd %xmm0, %eax
+; SSE41-NEXT:    xorl $32767, %eax # imm = 0x7FFF
 ; SSE41-NEXT:    # kill: def $ax killed $ax killed $eax
 ; SSE41-NEXT:    retq
 ;
@@ -1348,11 +1336,10 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 ; AVX1-NEXT:    vpmaxsw %xmm2, %xmm0, %xmm0
 ; AVX1-NEXT:    vpmaxsw %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpmaxsw %xmm4, %xmm0, %xmm0
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm1 = [32767,32767,32767,32767,32767,32767,32767,32767]
-; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
 ; AVX1-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovd %xmm0, %eax
+; AVX1-NEXT:    xorl $32767, %eax # imm = 0x7FFF
 ; AVX1-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
@@ -1364,11 +1351,10 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 ; AVX2-NEXT:    vpmaxsw %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpmaxsw %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vmovdqa {{.*#+}} xmm1 = [32767,32767,32767,32767,32767,32767,32767,32767]
-; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
 ; AVX2-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
+; AVX2-NEXT:    xorl $32767, %eax # imm = 0x7FFF
 ; AVX2-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
@@ -1380,11 +1366,10 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 ; AVX512-NEXT:    vpmaxsw %ymm1, %ymm0, %ymm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512-NEXT:    vpmaxsw %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vmovdqa {{.*#+}} xmm1 = [32767,32767,32767,32767,32767,32767,32767,32767]
-; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
 ; AVX512-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
+; AVX512-NEXT:    xorl $32767, %eax # imm = 0x7FFF
 ; AVX512-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
@@ -1597,38 +1582,35 @@ define i8 @test_v16i8(<16 x i8> %a0) {
 ;
 ; SSE41-LABEL: test_v16i8:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
-; SSE41-NEXT:    pxor %xmm1, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm2
-; SSE41-NEXT:    psrlw $8, %xmm2
-; SSE41-NEXT:    pminub %xmm0, %xmm2
-; SSE41-NEXT:    phminposuw %xmm2, %xmm0
-; SSE41-NEXT:    pxor %xmm1, %xmm0
+; SSE41-NEXT:    pxor {{.*}}(%rip), %xmm0
+; SSE41-NEXT:    movdqa %xmm0, %xmm1
+; SSE41-NEXT:    psrlw $8, %xmm1
+; SSE41-NEXT:    pminub %xmm0, %xmm1
+; SSE41-NEXT:    phminposuw %xmm1, %xmm0
 ; SSE41-NEXT:    pextrb $0, %xmm0, %eax
+; SSE41-NEXT:    xorb $127, %al
 ; SSE41-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE41-NEXT:    retq
 ;
 ; AVX-LABEL: test_v16i8:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vmovdqa {{.*#+}} xmm1 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
-; AVX-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX-NEXT:    vpsrlw $8, %xmm0, %xmm2
-; AVX-NEXT:    vpminub %xmm2, %xmm0, %xmm0
+; AVX-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
+; AVX-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; AVX-NEXT:    vpminub %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpextrb $0, %xmm0, %eax
+; AVX-NEXT:    xorb $127, %al
 ; AVX-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX-NEXT:    retq
 ;
 ; AVX512-LABEL: test_v16i8:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vmovdqa {{.*#+}} xmm1 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
-; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpsrlw $8, %xmm0, %xmm2
-; AVX512-NEXT:    vpminub %xmm2, %xmm0, %xmm0
+; AVX512-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
+; AVX512-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; AVX512-NEXT:    vpminub %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpextrb $0, %xmm0, %eax
+; AVX512-NEXT:    xorb $127, %al
 ; AVX512-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX512-NEXT:    retq
   %1 = call i8 @llvm.experimental.vector.reduce.smax.i8.v16i8(<16 x i8> %a0)
@@ -1676,14 +1658,13 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; SSE41-LABEL: test_v32i8:
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    pmaxsb %xmm1, %xmm0
-; SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
-; SSE41-NEXT:    pxor %xmm1, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm2
-; SSE41-NEXT:    psrlw $8, %xmm2
-; SSE41-NEXT:    pminub %xmm0, %xmm2
-; SSE41-NEXT:    phminposuw %xmm2, %xmm0
-; SSE41-NEXT:    pxor %xmm1, %xmm0
+; SSE41-NEXT:    pxor {{.*}}(%rip), %xmm0
+; SSE41-NEXT:    movdqa %xmm0, %xmm1
+; SSE41-NEXT:    psrlw $8, %xmm1
+; SSE41-NEXT:    pminub %xmm0, %xmm1
+; SSE41-NEXT:    phminposuw %xmm1, %xmm0
 ; SSE41-NEXT:    pextrb $0, %xmm0, %eax
+; SSE41-NEXT:    xorb $127, %al
 ; SSE41-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE41-NEXT:    retq
 ;
@@ -1691,13 +1672,12 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; AVX1-NEXT:    vpmaxsb %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm1 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
-; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm2
-; AVX1-NEXT:    vpminub %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
+; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; AVX1-NEXT:    vpminub %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpextrb $0, %xmm0, %eax
+; AVX1-NEXT:    xorb $127, %al
 ; AVX1-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
@@ -1706,13 +1686,12 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpmaxsb %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vmovdqa {{.*#+}} xmm1 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
-; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpsrlw $8, %xmm0, %xmm2
-; AVX2-NEXT:    vpminub %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
+; AVX2-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; AVX2-NEXT:    vpminub %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpextrb $0, %xmm0, %eax
+; AVX2-NEXT:    xorb $127, %al
 ; AVX2-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
@@ -1721,13 +1700,12 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512-NEXT:    vpmaxsb %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vmovdqa {{.*#+}} xmm1 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
-; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpsrlw $8, %xmm0, %xmm2
-; AVX512-NEXT:    vpminub %xmm2, %xmm0, %xmm0
+; AVX512-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
+; AVX512-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; AVX512-NEXT:    vpminub %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpextrb $0, %xmm0, %eax
+; AVX512-NEXT:    xorb $127, %al
 ; AVX512-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
@@ -1788,14 +1766,13 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; SSE41-NEXT:    pmaxsb %xmm3, %xmm1
 ; SSE41-NEXT:    pmaxsb %xmm2, %xmm0
 ; SSE41-NEXT:    pmaxsb %xmm1, %xmm0
-; SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
-; SSE41-NEXT:    pxor %xmm1, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm2
-; SSE41-NEXT:    psrlw $8, %xmm2
-; SSE41-NEXT:    pminub %xmm0, %xmm2
-; SSE41-NEXT:    phminposuw %xmm2, %xmm0
-; SSE41-NEXT:    pxor %xmm1, %xmm0
+; SSE41-NEXT:    pxor {{.*}}(%rip), %xmm0
+; SSE41-NEXT:    movdqa %xmm0, %xmm1
+; SSE41-NEXT:    psrlw $8, %xmm1
+; SSE41-NEXT:    pminub %xmm0, %xmm1
+; SSE41-NEXT:    phminposuw %xmm1, %xmm0
 ; SSE41-NEXT:    pextrb $0, %xmm0, %eax
+; SSE41-NEXT:    xorb $127, %al
 ; SSE41-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE41-NEXT:    retq
 ;
@@ -1806,13 +1783,12 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; AVX1-NEXT:    vpmaxsb %xmm2, %xmm3, %xmm2
 ; AVX1-NEXT:    vpmaxsb %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpmaxsb %xmm2, %xmm0, %xmm0
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm1 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
-; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm2
-; AVX1-NEXT:    vpminub %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
+; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; AVX1-NEXT:    vpminub %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpextrb $0, %xmm0, %eax
+; AVX1-NEXT:    xorb $127, %al
 ; AVX1-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
@@ -1822,13 +1798,12 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; AVX2-NEXT:    vpmaxsb %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpmaxsb %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vmovdqa {{.*#+}} xmm1 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
-; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpsrlw $8, %xmm0, %xmm2
-; AVX2-NEXT:    vpminub %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
+; AVX2-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; AVX2-NEXT:    vpminub %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpextrb $0, %xmm0, %eax
+; AVX2-NEXT:    xorb $127, %al
 ; AVX2-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
@@ -1839,13 +1814,12 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; AVX512-NEXT:    vpmaxsb %ymm1, %ymm0, %ymm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512-NEXT:    vpmaxsb %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vmovdqa {{.*#+}} xmm1 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
-; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpsrlw $8, %xmm0, %xmm2
-; AVX512-NEXT:    vpminub %xmm2, %xmm0, %xmm0
+; AVX512-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
+; AVX512-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; AVX512-NEXT:    vpminub %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpextrb $0, %xmm0, %eax
+; AVX512-NEXT:    xorb $127, %al
 ; AVX512-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
@@ -1930,14 +1904,13 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; SSE41-NEXT:    pmaxsb %xmm4, %xmm0
 ; SSE41-NEXT:    pmaxsb %xmm2, %xmm0
 ; SSE41-NEXT:    pmaxsb %xmm1, %xmm0
-; SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
-; SSE41-NEXT:    pxor %xmm1, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm2
-; SSE41-NEXT:    psrlw $8, %xmm2
-; SSE41-NEXT:    pminub %xmm0, %xmm2
-; SSE41-NEXT:    phminposuw %xmm2, %xmm0
-; SSE41-NEXT:    pxor %xmm1, %xmm0
+; SSE41-NEXT:    pxor {{.*}}(%rip), %xmm0
+; SSE41-NEXT:    movdqa %xmm0, %xmm1
+; SSE41-NEXT:    psrlw $8, %xmm1
+; SSE41-NEXT:    pminub %xmm0, %xmm1
+; SSE41-NEXT:    phminposuw %xmm1, %xmm0
 ; SSE41-NEXT:    pextrb $0, %xmm0, %eax
+; SSE41-NEXT:    xorb $127, %al
 ; SSE41-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE41-NEXT:    retq
 ;
@@ -1954,13 +1927,12 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; AVX1-NEXT:    vpmaxsb %xmm2, %xmm0, %xmm0
 ; AVX1-NEXT:    vpmaxsb %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpmaxsb %xmm4, %xmm0, %xmm0
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm1 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
-; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm2
-; AVX1-NEXT:    vpminub %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
+; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; AVX1-NEXT:    vpminub %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpextrb $0, %xmm0, %eax
+; AVX1-NEXT:    xorb $127, %al
 ; AVX1-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
@@ -1972,13 +1944,12 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; AVX2-NEXT:    vpmaxsb %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpmaxsb %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vmovdqa {{.*#+}} xmm1 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
-; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpsrlw $8, %xmm0, %xmm2
-; AVX2-NEXT:    vpminub %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
+; AVX2-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; AVX2-NEXT:    vpminub %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpextrb $0, %xmm0, %eax
+; AVX2-NEXT:    xorb $127, %al
 ; AVX2-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
@@ -1990,13 +1961,12 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; AVX512-NEXT:    vpmaxsb %ymm1, %ymm0, %ymm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512-NEXT:    vpmaxsb %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vmovdqa {{.*#+}} xmm1 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
-; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpsrlw $8, %xmm0, %xmm2
-; AVX512-NEXT:    vpminub %xmm2, %xmm0, %xmm0
+; AVX512-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
+; AVX512-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; AVX512-NEXT:    vpminub %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpextrb $0, %xmm0, %eax
+; AVX512-NEXT:    xorb $127, %al
 ; AVX512-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
diff --git a/test/CodeGen/X86/vector-reduce-smax.ll b/test/CodeGen/X86/vector-reduce-smax.ll
index 01f453bd982d59413cb2dd904285e8be678a6e24..162af26bbd0cdda36d65ba3f8f1544bf27418212 100644
--- a/test/CodeGen/X86/vector-reduce-smax.ll
+++ b/test/CodeGen/X86/vector-reduce-smax.ll
@@ -709,17 +709,16 @@ define i32 @test_v2i32(<2 x i32> %a0) {
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[0,2,2,3]
 ; SSE41-NEXT:    psrad $31, %xmm3
 ; SSE41-NEXT:    pblendw {{.*#+}} xmm3 = xmm2[0,1],xmm3[2,3],xmm2[4,5],xmm3[6,7]
-; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483648,2147483648]
-; SSE41-NEXT:    movdqa %xmm3, %xmm2
-; SSE41-NEXT:    pxor %xmm0, %xmm2
-; SSE41-NEXT:    pxor %xmm1, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm2, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm2, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm5, %xmm0
-; SSE41-NEXT:    por %xmm4, %xmm0
+; SSE41-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648]
+; SSE41-NEXT:    movdqa %xmm3, %xmm0
+; SSE41-NEXT:    pxor %xmm2, %xmm0
+; SSE41-NEXT:    pxor %xmm1, %xmm2
+; SSE41-NEXT:    movdqa %xmm2, %xmm4
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm2
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,0,2,2]
+; SSE41-NEXT:    pand %xmm4, %xmm0
+; SSE41-NEXT:    por %xmm2, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm3
 ; SSE41-NEXT:    movd %xmm3, %eax
 ; SSE41-NEXT:    retq
@@ -1171,12 +1170,11 @@ define i16 @test_v2i16(<2 x i16> %a0) {
 ; SSE41-NEXT:    pxor %xmm0, %xmm2
 ; SSE41-NEXT:    pxor %xmm1, %xmm0
 ; SSE41-NEXT:    movdqa %xmm2, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm2, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm5, %xmm0
-; SSE41-NEXT:    por %xmm4, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm2
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,0,2,2]
+; SSE41-NEXT:    pand %xmm4, %xmm0
+; SSE41-NEXT:    por %xmm2, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm3, %xmm1
 ; SSE41-NEXT:    movd %xmm1, %eax
 ; SSE41-NEXT:    # kill: def $ax killed $ax killed $eax
@@ -1340,31 +1338,28 @@ define i16 @test_v8i16(<8 x i16> %a0) {
 ;
 ; SSE41-LABEL: test_v8i16:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [32767,32767,32767,32767,32767,32767,32767,32767]
-; SSE41-NEXT:    pxor %xmm1, %xmm0
+; SSE41-NEXT:    pxor {{.*}}(%rip), %xmm0
 ; SSE41-NEXT:    phminposuw %xmm0, %xmm0
-; SSE41-NEXT:    pxor %xmm1, %xmm0
 ; SSE41-NEXT:    movd %xmm0, %eax
+; SSE41-NEXT:    xorl $32767, %eax # imm = 0x7FFF
 ; SSE41-NEXT:    # kill: def $ax killed $ax killed $eax
 ; SSE41-NEXT:    retq
 ;
 ; AVX-LABEL: test_v8i16:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vmovdqa {{.*#+}} xmm1 = [32767,32767,32767,32767,32767,32767,32767,32767]
-; AVX-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
 ; AVX-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vmovd %xmm0, %eax
+; AVX-NEXT:    xorl $32767, %eax # imm = 0x7FFF
 ; AVX-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX-NEXT:    retq
 ;
 ; AVX512-LABEL: test_v8i16:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vmovdqa {{.*#+}} xmm1 = [32767,32767,32767,32767,32767,32767,32767,32767]
-; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
 ; AVX512-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
+; AVX512-NEXT:    xorl $32767, %eax # imm = 0x7FFF
 ; AVX512-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX512-NEXT:    retq
   %1 = call i16 @llvm.experimental.vector.reduce.smax.i16.v8i16(<8 x i16> %a0)
@@ -1389,11 +1384,10 @@ define i16 @test_v16i16(<16 x i16> %a0) {
 ; SSE41-LABEL: test_v16i16:
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    pmaxsw %xmm1, %xmm0
-; SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [32767,32767,32767,32767,32767,32767,32767,32767]
-; SSE41-NEXT:    pxor %xmm1, %xmm0
+; SSE41-NEXT:    pxor {{.*}}(%rip), %xmm0
 ; SSE41-NEXT:    phminposuw %xmm0, %xmm0
-; SSE41-NEXT:    pxor %xmm1, %xmm0
 ; SSE41-NEXT:    movd %xmm0, %eax
+; SSE41-NEXT:    xorl $32767, %eax # imm = 0x7FFF
 ; SSE41-NEXT:    # kill: def $ax killed $ax killed $eax
 ; SSE41-NEXT:    retq
 ;
@@ -1401,11 +1395,10 @@ define i16 @test_v16i16(<16 x i16> %a0) {
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; AVX1-NEXT:    vpmaxsw %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm1 = [32767,32767,32767,32767,32767,32767,32767,32767]
-; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
 ; AVX1-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovd %xmm0, %eax
+; AVX1-NEXT:    xorl $32767, %eax # imm = 0x7FFF
 ; AVX1-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
@@ -1414,11 +1407,10 @@ define i16 @test_v16i16(<16 x i16> %a0) {
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpmaxsw %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vmovdqa {{.*#+}} xmm1 = [32767,32767,32767,32767,32767,32767,32767,32767]
-; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
 ; AVX2-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
+; AVX2-NEXT:    xorl $32767, %eax # imm = 0x7FFF
 ; AVX2-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
@@ -1427,11 +1419,10 @@ define i16 @test_v16i16(<16 x i16> %a0) {
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512-NEXT:    vpmaxsw %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vmovdqa {{.*#+}} xmm1 = [32767,32767,32767,32767,32767,32767,32767,32767]
-; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
 ; AVX512-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
+; AVX512-NEXT:    xorl $32767, %eax # imm = 0x7FFF
 ; AVX512-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
@@ -1461,11 +1452,10 @@ define i16 @test_v32i16(<32 x i16> %a0) {
 ; SSE41-NEXT:    pmaxsw %xmm3, %xmm1
 ; SSE41-NEXT:    pmaxsw %xmm2, %xmm0
 ; SSE41-NEXT:    pmaxsw %xmm1, %xmm0
-; SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [32767,32767,32767,32767,32767,32767,32767,32767]
-; SSE41-NEXT:    pxor %xmm1, %xmm0
+; SSE41-NEXT:    pxor {{.*}}(%rip), %xmm0
 ; SSE41-NEXT:    phminposuw %xmm0, %xmm0
-; SSE41-NEXT:    pxor %xmm1, %xmm0
 ; SSE41-NEXT:    movd %xmm0, %eax
+; SSE41-NEXT:    xorl $32767, %eax # imm = 0x7FFF
 ; SSE41-NEXT:    # kill: def $ax killed $ax killed $eax
 ; SSE41-NEXT:    retq
 ;
@@ -1476,11 +1466,10 @@ define i16 @test_v32i16(<32 x i16> %a0) {
 ; AVX1-NEXT:    vpmaxsw %xmm2, %xmm3, %xmm2
 ; AVX1-NEXT:    vpmaxsw %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpmaxsw %xmm2, %xmm0, %xmm0
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm1 = [32767,32767,32767,32767,32767,32767,32767,32767]
-; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
 ; AVX1-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovd %xmm0, %eax
+; AVX1-NEXT:    xorl $32767, %eax # imm = 0x7FFF
 ; AVX1-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
@@ -1490,11 +1479,10 @@ define i16 @test_v32i16(<32 x i16> %a0) {
 ; AVX2-NEXT:    vpmaxsw %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpmaxsw %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vmovdqa {{.*#+}} xmm1 = [32767,32767,32767,32767,32767,32767,32767,32767]
-; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
 ; AVX2-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
+; AVX2-NEXT:    xorl $32767, %eax # imm = 0x7FFF
 ; AVX2-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
@@ -1505,11 +1493,10 @@ define i16 @test_v32i16(<32 x i16> %a0) {
 ; AVX512-NEXT:    vpmaxsw %ymm1, %ymm0, %ymm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512-NEXT:    vpmaxsw %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vmovdqa {{.*#+}} xmm1 = [32767,32767,32767,32767,32767,32767,32767,32767]
-; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
 ; AVX512-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
+; AVX512-NEXT:    xorl $32767, %eax # imm = 0x7FFF
 ; AVX512-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
@@ -1547,11 +1534,10 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 ; SSE41-NEXT:    pmaxsw %xmm4, %xmm0
 ; SSE41-NEXT:    pmaxsw %xmm2, %xmm0
 ; SSE41-NEXT:    pmaxsw %xmm1, %xmm0
-; SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [32767,32767,32767,32767,32767,32767,32767,32767]
-; SSE41-NEXT:    pxor %xmm1, %xmm0
+; SSE41-NEXT:    pxor {{.*}}(%rip), %xmm0
 ; SSE41-NEXT:    phminposuw %xmm0, %xmm0
-; SSE41-NEXT:    pxor %xmm1, %xmm0
 ; SSE41-NEXT:    movd %xmm0, %eax
+; SSE41-NEXT:    xorl $32767, %eax # imm = 0x7FFF
 ; SSE41-NEXT:    # kill: def $ax killed $ax killed $eax
 ; SSE41-NEXT:    retq
 ;
@@ -1568,11 +1554,10 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 ; AVX1-NEXT:    vpmaxsw %xmm2, %xmm0, %xmm0
 ; AVX1-NEXT:    vpmaxsw %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpmaxsw %xmm4, %xmm0, %xmm0
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm1 = [32767,32767,32767,32767,32767,32767,32767,32767]
-; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
 ; AVX1-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovd %xmm0, %eax
+; AVX1-NEXT:    xorl $32767, %eax # imm = 0x7FFF
 ; AVX1-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
@@ -1584,11 +1569,10 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 ; AVX2-NEXT:    vpmaxsw %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpmaxsw %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vmovdqa {{.*#+}} xmm1 = [32767,32767,32767,32767,32767,32767,32767,32767]
-; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
 ; AVX2-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
+; AVX2-NEXT:    xorl $32767, %eax # imm = 0x7FFF
 ; AVX2-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
@@ -1600,11 +1584,10 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 ; AVX512-NEXT:    vpmaxsw %ymm1, %ymm0, %ymm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512-NEXT:    vpmaxsw %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vmovdqa {{.*#+}} xmm1 = [32767,32767,32767,32767,32767,32767,32767,32767]
-; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
 ; AVX512-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
+; AVX512-NEXT:    xorl $32767, %eax # imm = 0x7FFF
 ; AVX512-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
@@ -1673,12 +1656,11 @@ define i8 @test_v2i8(<2 x i8> %a0) {
 ; SSE41-NEXT:    pxor %xmm0, %xmm2
 ; SSE41-NEXT:    pxor %xmm1, %xmm0
 ; SSE41-NEXT:    movdqa %xmm2, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm2, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm5, %xmm0
-; SSE41-NEXT:    por %xmm4, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm2
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,0,2,2]
+; SSE41-NEXT:    pand %xmm4, %xmm0
+; SSE41-NEXT:    por %xmm2, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm3, %xmm1
 ; SSE41-NEXT:    pextrb $0, %xmm1, %eax
 ; SSE41-NEXT:    # kill: def $al killed $al killed $eax
@@ -1947,38 +1929,35 @@ define i8 @test_v16i8(<16 x i8> %a0) {
 ;
 ; SSE41-LABEL: test_v16i8:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
-; SSE41-NEXT:    pxor %xmm1, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm2
-; SSE41-NEXT:    psrlw $8, %xmm2
-; SSE41-NEXT:    pminub %xmm0, %xmm2
-; SSE41-NEXT:    phminposuw %xmm2, %xmm0
-; SSE41-NEXT:    pxor %xmm1, %xmm0
+; SSE41-NEXT:    pxor {{.*}}(%rip), %xmm0
+; SSE41-NEXT:    movdqa %xmm0, %xmm1
+; SSE41-NEXT:    psrlw $8, %xmm1
+; SSE41-NEXT:    pminub %xmm0, %xmm1
+; SSE41-NEXT:    phminposuw %xmm1, %xmm0
 ; SSE41-NEXT:    pextrb $0, %xmm0, %eax
+; SSE41-NEXT:    xorb $127, %al
 ; SSE41-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE41-NEXT:    retq
 ;
 ; AVX-LABEL: test_v16i8:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vmovdqa {{.*#+}} xmm1 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
-; AVX-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX-NEXT:    vpsrlw $8, %xmm0, %xmm2
-; AVX-NEXT:    vpminub %xmm2, %xmm0, %xmm0
+; AVX-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
+; AVX-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; AVX-NEXT:    vpminub %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpextrb $0, %xmm0, %eax
+; AVX-NEXT:    xorb $127, %al
 ; AVX-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX-NEXT:    retq
 ;
 ; AVX512-LABEL: test_v16i8:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vmovdqa {{.*#+}} xmm1 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
-; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpsrlw $8, %xmm0, %xmm2
-; AVX512-NEXT:    vpminub %xmm2, %xmm0, %xmm0
+; AVX512-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
+; AVX512-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; AVX512-NEXT:    vpminub %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpextrb $0, %xmm0, %eax
+; AVX512-NEXT:    xorb $127, %al
 ; AVX512-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX512-NEXT:    retq
   %1 = call i8 @llvm.experimental.vector.reduce.smax.i8.v16i8(<16 x i8> %a0)
@@ -2026,14 +2005,13 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; SSE41-LABEL: test_v32i8:
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    pmaxsb %xmm1, %xmm0
-; SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
-; SSE41-NEXT:    pxor %xmm1, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm2
-; SSE41-NEXT:    psrlw $8, %xmm2
-; SSE41-NEXT:    pminub %xmm0, %xmm2
-; SSE41-NEXT:    phminposuw %xmm2, %xmm0
-; SSE41-NEXT:    pxor %xmm1, %xmm0
+; SSE41-NEXT:    pxor {{.*}}(%rip), %xmm0
+; SSE41-NEXT:    movdqa %xmm0, %xmm1
+; SSE41-NEXT:    psrlw $8, %xmm1
+; SSE41-NEXT:    pminub %xmm0, %xmm1
+; SSE41-NEXT:    phminposuw %xmm1, %xmm0
 ; SSE41-NEXT:    pextrb $0, %xmm0, %eax
+; SSE41-NEXT:    xorb $127, %al
 ; SSE41-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE41-NEXT:    retq
 ;
@@ -2041,13 +2019,12 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; AVX1-NEXT:    vpmaxsb %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm1 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
-; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm2
-; AVX1-NEXT:    vpminub %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
+; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; AVX1-NEXT:    vpminub %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpextrb $0, %xmm0, %eax
+; AVX1-NEXT:    xorb $127, %al
 ; AVX1-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
@@ -2056,13 +2033,12 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpmaxsb %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vmovdqa {{.*#+}} xmm1 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
-; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpsrlw $8, %xmm0, %xmm2
-; AVX2-NEXT:    vpminub %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
+; AVX2-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; AVX2-NEXT:    vpminub %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpextrb $0, %xmm0, %eax
+; AVX2-NEXT:    xorb $127, %al
 ; AVX2-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
@@ -2071,13 +2047,12 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512-NEXT:    vpmaxsb %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vmovdqa {{.*#+}} xmm1 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
-; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpsrlw $8, %xmm0, %xmm2
-; AVX512-NEXT:    vpminub %xmm2, %xmm0, %xmm0
+; AVX512-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
+; AVX512-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; AVX512-NEXT:    vpminub %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpextrb $0, %xmm0, %eax
+; AVX512-NEXT:    xorb $127, %al
 ; AVX512-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
@@ -2138,14 +2113,13 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; SSE41-NEXT:    pmaxsb %xmm3, %xmm1
 ; SSE41-NEXT:    pmaxsb %xmm2, %xmm0
 ; SSE41-NEXT:    pmaxsb %xmm1, %xmm0
-; SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
-; SSE41-NEXT:    pxor %xmm1, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm2
-; SSE41-NEXT:    psrlw $8, %xmm2
-; SSE41-NEXT:    pminub %xmm0, %xmm2
-; SSE41-NEXT:    phminposuw %xmm2, %xmm0
-; SSE41-NEXT:    pxor %xmm1, %xmm0
+; SSE41-NEXT:    pxor {{.*}}(%rip), %xmm0
+; SSE41-NEXT:    movdqa %xmm0, %xmm1
+; SSE41-NEXT:    psrlw $8, %xmm1
+; SSE41-NEXT:    pminub %xmm0, %xmm1
+; SSE41-NEXT:    phminposuw %xmm1, %xmm0
 ; SSE41-NEXT:    pextrb $0, %xmm0, %eax
+; SSE41-NEXT:    xorb $127, %al
 ; SSE41-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE41-NEXT:    retq
 ;
@@ -2156,13 +2130,12 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; AVX1-NEXT:    vpmaxsb %xmm2, %xmm3, %xmm2
 ; AVX1-NEXT:    vpmaxsb %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpmaxsb %xmm2, %xmm0, %xmm0
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm1 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
-; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm2
-; AVX1-NEXT:    vpminub %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
+; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; AVX1-NEXT:    vpminub %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpextrb $0, %xmm0, %eax
+; AVX1-NEXT:    xorb $127, %al
 ; AVX1-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
@@ -2172,13 +2145,12 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; AVX2-NEXT:    vpmaxsb %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpmaxsb %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vmovdqa {{.*#+}} xmm1 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
-; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpsrlw $8, %xmm0, %xmm2
-; AVX2-NEXT:    vpminub %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
+; AVX2-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; AVX2-NEXT:    vpminub %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpextrb $0, %xmm0, %eax
+; AVX2-NEXT:    xorb $127, %al
 ; AVX2-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
@@ -2189,13 +2161,12 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; AVX512-NEXT:    vpmaxsb %ymm1, %ymm0, %ymm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512-NEXT:    vpmaxsb %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vmovdqa {{.*#+}} xmm1 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
-; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpsrlw $8, %xmm0, %xmm2
-; AVX512-NEXT:    vpminub %xmm2, %xmm0, %xmm0
+; AVX512-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
+; AVX512-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; AVX512-NEXT:    vpminub %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpextrb $0, %xmm0, %eax
+; AVX512-NEXT:    xorb $127, %al
 ; AVX512-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
@@ -2280,14 +2251,13 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; SSE41-NEXT:    pmaxsb %xmm4, %xmm0
 ; SSE41-NEXT:    pmaxsb %xmm2, %xmm0
 ; SSE41-NEXT:    pmaxsb %xmm1, %xmm0
-; SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
-; SSE41-NEXT:    pxor %xmm1, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm2
-; SSE41-NEXT:    psrlw $8, %xmm2
-; SSE41-NEXT:    pminub %xmm0, %xmm2
-; SSE41-NEXT:    phminposuw %xmm2, %xmm0
-; SSE41-NEXT:    pxor %xmm1, %xmm0
+; SSE41-NEXT:    pxor {{.*}}(%rip), %xmm0
+; SSE41-NEXT:    movdqa %xmm0, %xmm1
+; SSE41-NEXT:    psrlw $8, %xmm1
+; SSE41-NEXT:    pminub %xmm0, %xmm1
+; SSE41-NEXT:    phminposuw %xmm1, %xmm0
 ; SSE41-NEXT:    pextrb $0, %xmm0, %eax
+; SSE41-NEXT:    xorb $127, %al
 ; SSE41-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE41-NEXT:    retq
 ;
@@ -2304,13 +2274,12 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; AVX1-NEXT:    vpmaxsb %xmm2, %xmm0, %xmm0
 ; AVX1-NEXT:    vpmaxsb %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpmaxsb %xmm4, %xmm0, %xmm0
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm1 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
-; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm2
-; AVX1-NEXT:    vpminub %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
+; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; AVX1-NEXT:    vpminub %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpextrb $0, %xmm0, %eax
+; AVX1-NEXT:    xorb $127, %al
 ; AVX1-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
@@ -2322,13 +2291,12 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; AVX2-NEXT:    vpmaxsb %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpmaxsb %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vmovdqa {{.*#+}} xmm1 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
-; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpsrlw $8, %xmm0, %xmm2
-; AVX2-NEXT:    vpminub %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
+; AVX2-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; AVX2-NEXT:    vpminub %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpextrb $0, %xmm0, %eax
+; AVX2-NEXT:    xorb $127, %al
 ; AVX2-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
@@ -2340,13 +2308,12 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; AVX512-NEXT:    vpmaxsb %ymm1, %ymm0, %ymm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512-NEXT:    vpmaxsb %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vmovdqa {{.*#+}} xmm1 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
-; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpsrlw $8, %xmm0, %xmm2
-; AVX512-NEXT:    vpminub %xmm2, %xmm0, %xmm0
+; AVX512-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
+; AVX512-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; AVX512-NEXT:    vpminub %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpextrb $0, %xmm0, %eax
+; AVX512-NEXT:    xorb $127, %al
 ; AVX512-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
diff --git a/test/CodeGen/X86/vector-reduce-smin-widen.ll b/test/CodeGen/X86/vector-reduce-smin-widen.ll
index 140df76e8ec2d9f2a37f7c54c228eb053f6fff75..dcc522ad2881ed35040825a50d8515578e691095 100644
--- a/test/CodeGen/X86/vector-reduce-smin-widen.ll
+++ b/test/CodeGen/X86/vector-reduce-smin-widen.ll
@@ -1119,31 +1119,28 @@ define i16 @test_v8i16(<8 x i16> %a0) {
 ;
 ; SSE41-LABEL: test_v8i16:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [32768,32768,32768,32768,32768,32768,32768,32768]
-; SSE41-NEXT:    pxor %xmm1, %xmm0
+; SSE41-NEXT:    pxor {{.*}}(%rip), %xmm0
 ; SSE41-NEXT:    phminposuw %xmm0, %xmm0
-; SSE41-NEXT:    pxor %xmm1, %xmm0
 ; SSE41-NEXT:    movd %xmm0, %eax
+; SSE41-NEXT:    xorl $32768, %eax # imm = 0x8000
 ; SSE41-NEXT:    # kill: def $ax killed $ax killed $eax
 ; SSE41-NEXT:    retq
 ;
 ; AVX-LABEL: test_v8i16:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vmovdqa {{.*#+}} xmm1 = [32768,32768,32768,32768,32768,32768,32768,32768]
-; AVX-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
 ; AVX-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vmovd %xmm0, %eax
+; AVX-NEXT:    xorl $32768, %eax # imm = 0x8000
 ; AVX-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX-NEXT:    retq
 ;
 ; AVX512-LABEL: test_v8i16:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vmovdqa {{.*#+}} xmm1 = [32768,32768,32768,32768,32768,32768,32768,32768]
-; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
 ; AVX512-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
+; AVX512-NEXT:    xorl $32768, %eax # imm = 0x8000
 ; AVX512-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX512-NEXT:    retq
   %1 = call i16 @llvm.experimental.vector.reduce.smin.i16.v8i16(<8 x i16> %a0)
@@ -1168,11 +1165,10 @@ define i16 @test_v16i16(<16 x i16> %a0) {
 ; SSE41-LABEL: test_v16i16:
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    pminsw %xmm1, %xmm0
-; SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [32768,32768,32768,32768,32768,32768,32768,32768]
-; SSE41-NEXT:    pxor %xmm1, %xmm0
+; SSE41-NEXT:    pxor {{.*}}(%rip), %xmm0
 ; SSE41-NEXT:    phminposuw %xmm0, %xmm0
-; SSE41-NEXT:    pxor %xmm1, %xmm0
 ; SSE41-NEXT:    movd %xmm0, %eax
+; SSE41-NEXT:    xorl $32768, %eax # imm = 0x8000
 ; SSE41-NEXT:    # kill: def $ax killed $ax killed $eax
 ; SSE41-NEXT:    retq
 ;
@@ -1180,11 +1176,10 @@ define i16 @test_v16i16(<16 x i16> %a0) {
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; AVX1-NEXT:    vpminsw %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm1 = [32768,32768,32768,32768,32768,32768,32768,32768]
-; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
 ; AVX1-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovd %xmm0, %eax
+; AVX1-NEXT:    xorl $32768, %eax # imm = 0x8000
 ; AVX1-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
@@ -1193,11 +1188,10 @@ define i16 @test_v16i16(<16 x i16> %a0) {
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpminsw %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vmovdqa {{.*#+}} xmm1 = [32768,32768,32768,32768,32768,32768,32768,32768]
-; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
 ; AVX2-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
+; AVX2-NEXT:    xorl $32768, %eax # imm = 0x8000
 ; AVX2-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
@@ -1206,11 +1200,10 @@ define i16 @test_v16i16(<16 x i16> %a0) {
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512-NEXT:    vpminsw %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vmovdqa {{.*#+}} xmm1 = [32768,32768,32768,32768,32768,32768,32768,32768]
-; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
 ; AVX512-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
+; AVX512-NEXT:    xorl $32768, %eax # imm = 0x8000
 ; AVX512-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
@@ -1240,11 +1233,10 @@ define i16 @test_v32i16(<32 x i16> %a0) {
 ; SSE41-NEXT:    pminsw %xmm3, %xmm1
 ; SSE41-NEXT:    pminsw %xmm2, %xmm0
 ; SSE41-NEXT:    pminsw %xmm1, %xmm0
-; SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [32768,32768,32768,32768,32768,32768,32768,32768]
-; SSE41-NEXT:    pxor %xmm1, %xmm0
+; SSE41-NEXT:    pxor {{.*}}(%rip), %xmm0
 ; SSE41-NEXT:    phminposuw %xmm0, %xmm0
-; SSE41-NEXT:    pxor %xmm1, %xmm0
 ; SSE41-NEXT:    movd %xmm0, %eax
+; SSE41-NEXT:    xorl $32768, %eax # imm = 0x8000
 ; SSE41-NEXT:    # kill: def $ax killed $ax killed $eax
 ; SSE41-NEXT:    retq
 ;
@@ -1255,11 +1247,10 @@ define i16 @test_v32i16(<32 x i16> %a0) {
 ; AVX1-NEXT:    vpminsw %xmm2, %xmm3, %xmm2
 ; AVX1-NEXT:    vpminsw %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpminsw %xmm2, %xmm0, %xmm0
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm1 = [32768,32768,32768,32768,32768,32768,32768,32768]
-; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
 ; AVX1-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovd %xmm0, %eax
+; AVX1-NEXT:    xorl $32768, %eax # imm = 0x8000
 ; AVX1-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
@@ -1269,11 +1260,10 @@ define i16 @test_v32i16(<32 x i16> %a0) {
 ; AVX2-NEXT:    vpminsw %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpminsw %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vmovdqa {{.*#+}} xmm1 = [32768,32768,32768,32768,32768,32768,32768,32768]
-; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
 ; AVX2-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
+; AVX2-NEXT:    xorl $32768, %eax # imm = 0x8000
 ; AVX2-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
@@ -1284,11 +1274,10 @@ define i16 @test_v32i16(<32 x i16> %a0) {
 ; AVX512-NEXT:    vpminsw %ymm1, %ymm0, %ymm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512-NEXT:    vpminsw %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vmovdqa {{.*#+}} xmm1 = [32768,32768,32768,32768,32768,32768,32768,32768]
-; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
 ; AVX512-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
+; AVX512-NEXT:    xorl $32768, %eax # imm = 0x8000
 ; AVX512-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
@@ -1326,11 +1315,10 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 ; SSE41-NEXT:    pminsw %xmm4, %xmm0
 ; SSE41-NEXT:    pminsw %xmm2, %xmm0
 ; SSE41-NEXT:    pminsw %xmm1, %xmm0
-; SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [32768,32768,32768,32768,32768,32768,32768,32768]
-; SSE41-NEXT:    pxor %xmm1, %xmm0
+; SSE41-NEXT:    pxor {{.*}}(%rip), %xmm0
 ; SSE41-NEXT:    phminposuw %xmm0, %xmm0
-; SSE41-NEXT:    pxor %xmm1, %xmm0
 ; SSE41-NEXT:    movd %xmm0, %eax
+; SSE41-NEXT:    xorl $32768, %eax # imm = 0x8000
 ; SSE41-NEXT:    # kill: def $ax killed $ax killed $eax
 ; SSE41-NEXT:    retq
 ;
@@ -1347,11 +1335,10 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 ; AVX1-NEXT:    vpminsw %xmm2, %xmm0, %xmm0
 ; AVX1-NEXT:    vpminsw %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpminsw %xmm4, %xmm0, %xmm0
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm1 = [32768,32768,32768,32768,32768,32768,32768,32768]
-; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
 ; AVX1-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovd %xmm0, %eax
+; AVX1-NEXT:    xorl $32768, %eax # imm = 0x8000
 ; AVX1-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
@@ -1363,11 +1350,10 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 ; AVX2-NEXT:    vpminsw %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpminsw %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vmovdqa {{.*#+}} xmm1 = [32768,32768,32768,32768,32768,32768,32768,32768]
-; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
 ; AVX2-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
+; AVX2-NEXT:    xorl $32768, %eax # imm = 0x8000
 ; AVX2-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
@@ -1379,11 +1365,10 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 ; AVX512-NEXT:    vpminsw %ymm1, %ymm0, %ymm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512-NEXT:    vpminsw %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vmovdqa {{.*#+}} xmm1 = [32768,32768,32768,32768,32768,32768,32768,32768]
-; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
 ; AVX512-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
+; AVX512-NEXT:    xorl $32768, %eax # imm = 0x8000
 ; AVX512-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
@@ -1596,38 +1581,35 @@ define i8 @test_v16i8(<16 x i8> %a0) {
 ;
 ; SSE41-LABEL: test_v16i8:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128]
-; SSE41-NEXT:    pxor %xmm1, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm2
-; SSE41-NEXT:    psrlw $8, %xmm2
-; SSE41-NEXT:    pminub %xmm0, %xmm2
-; SSE41-NEXT:    phminposuw %xmm2, %xmm0
-; SSE41-NEXT:    pxor %xmm1, %xmm0
+; SSE41-NEXT:    pxor {{.*}}(%rip), %xmm0
+; SSE41-NEXT:    movdqa %xmm0, %xmm1
+; SSE41-NEXT:    psrlw $8, %xmm1
+; SSE41-NEXT:    pminub %xmm0, %xmm1
+; SSE41-NEXT:    phminposuw %xmm1, %xmm0
 ; SSE41-NEXT:    pextrb $0, %xmm0, %eax
+; SSE41-NEXT:    xorb $-128, %al
 ; SSE41-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE41-NEXT:    retq
 ;
 ; AVX-LABEL: test_v16i8:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vmovdqa {{.*#+}} xmm1 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128]
-; AVX-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX-NEXT:    vpsrlw $8, %xmm0, %xmm2
-; AVX-NEXT:    vpminub %xmm2, %xmm0, %xmm0
+; AVX-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
+; AVX-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; AVX-NEXT:    vpminub %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpextrb $0, %xmm0, %eax
+; AVX-NEXT:    xorb $-128, %al
 ; AVX-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX-NEXT:    retq
 ;
 ; AVX512-LABEL: test_v16i8:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vmovdqa {{.*#+}} xmm1 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128]
-; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpsrlw $8, %xmm0, %xmm2
-; AVX512-NEXT:    vpminub %xmm2, %xmm0, %xmm0
+; AVX512-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
+; AVX512-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; AVX512-NEXT:    vpminub %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpextrb $0, %xmm0, %eax
+; AVX512-NEXT:    xorb $-128, %al
 ; AVX512-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX512-NEXT:    retq
   %1 = call i8 @llvm.experimental.vector.reduce.smin.i8.v16i8(<16 x i8> %a0)
@@ -1675,14 +1657,13 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; SSE41-LABEL: test_v32i8:
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    pminsb %xmm1, %xmm0
-; SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128]
-; SSE41-NEXT:    pxor %xmm1, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm2
-; SSE41-NEXT:    psrlw $8, %xmm2
-; SSE41-NEXT:    pminub %xmm0, %xmm2
-; SSE41-NEXT:    phminposuw %xmm2, %xmm0
-; SSE41-NEXT:    pxor %xmm1, %xmm0
+; SSE41-NEXT:    pxor {{.*}}(%rip), %xmm0
+; SSE41-NEXT:    movdqa %xmm0, %xmm1
+; SSE41-NEXT:    psrlw $8, %xmm1
+; SSE41-NEXT:    pminub %xmm0, %xmm1
+; SSE41-NEXT:    phminposuw %xmm1, %xmm0
 ; SSE41-NEXT:    pextrb $0, %xmm0, %eax
+; SSE41-NEXT:    xorb $-128, %al
 ; SSE41-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE41-NEXT:    retq
 ;
@@ -1690,13 +1671,12 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; AVX1-NEXT:    vpminsb %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm1 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128]
-; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm2
-; AVX1-NEXT:    vpminub %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
+; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; AVX1-NEXT:    vpminub %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpextrb $0, %xmm0, %eax
+; AVX1-NEXT:    xorb $-128, %al
 ; AVX1-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
@@ -1705,13 +1685,12 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpminsb %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vmovdqa {{.*#+}} xmm1 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128]
-; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpsrlw $8, %xmm0, %xmm2
-; AVX2-NEXT:    vpminub %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
+; AVX2-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; AVX2-NEXT:    vpminub %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpextrb $0, %xmm0, %eax
+; AVX2-NEXT:    xorb $-128, %al
 ; AVX2-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
@@ -1720,13 +1699,12 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512-NEXT:    vpminsb %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vmovdqa {{.*#+}} xmm1 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128]
-; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpsrlw $8, %xmm0, %xmm2
-; AVX512-NEXT:    vpminub %xmm2, %xmm0, %xmm0
+; AVX512-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
+; AVX512-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; AVX512-NEXT:    vpminub %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpextrb $0, %xmm0, %eax
+; AVX512-NEXT:    xorb $-128, %al
 ; AVX512-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
@@ -1787,14 +1765,13 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; SSE41-NEXT:    pminsb %xmm3, %xmm1
 ; SSE41-NEXT:    pminsb %xmm2, %xmm0
 ; SSE41-NEXT:    pminsb %xmm1, %xmm0
-; SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128]
-; SSE41-NEXT:    pxor %xmm1, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm2
-; SSE41-NEXT:    psrlw $8, %xmm2
-; SSE41-NEXT:    pminub %xmm0, %xmm2
-; SSE41-NEXT:    phminposuw %xmm2, %xmm0
-; SSE41-NEXT:    pxor %xmm1, %xmm0
+; SSE41-NEXT:    pxor {{.*}}(%rip), %xmm0
+; SSE41-NEXT:    movdqa %xmm0, %xmm1
+; SSE41-NEXT:    psrlw $8, %xmm1
+; SSE41-NEXT:    pminub %xmm0, %xmm1
+; SSE41-NEXT:    phminposuw %xmm1, %xmm0
 ; SSE41-NEXT:    pextrb $0, %xmm0, %eax
+; SSE41-NEXT:    xorb $-128, %al
 ; SSE41-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE41-NEXT:    retq
 ;
@@ -1805,13 +1782,12 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; AVX1-NEXT:    vpminsb %xmm2, %xmm3, %xmm2
 ; AVX1-NEXT:    vpminsb %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpminsb %xmm2, %xmm0, %xmm0
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm1 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128]
-; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm2
-; AVX1-NEXT:    vpminub %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
+; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; AVX1-NEXT:    vpminub %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpextrb $0, %xmm0, %eax
+; AVX1-NEXT:    xorb $-128, %al
 ; AVX1-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
@@ -1821,13 +1797,12 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; AVX2-NEXT:    vpminsb %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpminsb %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vmovdqa {{.*#+}} xmm1 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128]
-; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpsrlw $8, %xmm0, %xmm2
-; AVX2-NEXT:    vpminub %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
+; AVX2-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; AVX2-NEXT:    vpminub %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpextrb $0, %xmm0, %eax
+; AVX2-NEXT:    xorb $-128, %al
 ; AVX2-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
@@ -1838,13 +1813,12 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; AVX512-NEXT:    vpminsb %ymm1, %ymm0, %ymm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512-NEXT:    vpminsb %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vmovdqa {{.*#+}} xmm1 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128]
-; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpsrlw $8, %xmm0, %xmm2
-; AVX512-NEXT:    vpminub %xmm2, %xmm0, %xmm0
+; AVX512-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
+; AVX512-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; AVX512-NEXT:    vpminub %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpextrb $0, %xmm0, %eax
+; AVX512-NEXT:    xorb $-128, %al
 ; AVX512-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
@@ -1929,14 +1903,13 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; SSE41-NEXT:    pminsb %xmm4, %xmm0
 ; SSE41-NEXT:    pminsb %xmm2, %xmm0
 ; SSE41-NEXT:    pminsb %xmm1, %xmm0
-; SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128]
-; SSE41-NEXT:    pxor %xmm1, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm2
-; SSE41-NEXT:    psrlw $8, %xmm2
-; SSE41-NEXT:    pminub %xmm0, %xmm2
-; SSE41-NEXT:    phminposuw %xmm2, %xmm0
-; SSE41-NEXT:    pxor %xmm1, %xmm0
+; SSE41-NEXT:    pxor {{.*}}(%rip), %xmm0
+; SSE41-NEXT:    movdqa %xmm0, %xmm1
+; SSE41-NEXT:    psrlw $8, %xmm1
+; SSE41-NEXT:    pminub %xmm0, %xmm1
+; SSE41-NEXT:    phminposuw %xmm1, %xmm0
 ; SSE41-NEXT:    pextrb $0, %xmm0, %eax
+; SSE41-NEXT:    xorb $-128, %al
 ; SSE41-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE41-NEXT:    retq
 ;
@@ -1953,13 +1926,12 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; AVX1-NEXT:    vpminsb %xmm2, %xmm0, %xmm0
 ; AVX1-NEXT:    vpminsb %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpminsb %xmm4, %xmm0, %xmm0
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm1 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128]
-; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm2
-; AVX1-NEXT:    vpminub %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
+; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; AVX1-NEXT:    vpminub %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpextrb $0, %xmm0, %eax
+; AVX1-NEXT:    xorb $-128, %al
 ; AVX1-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
@@ -1971,13 +1943,12 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; AVX2-NEXT:    vpminsb %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpminsb %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vmovdqa {{.*#+}} xmm1 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128]
-; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpsrlw $8, %xmm0, %xmm2
-; AVX2-NEXT:    vpminub %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
+; AVX2-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; AVX2-NEXT:    vpminub %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpextrb $0, %xmm0, %eax
+; AVX2-NEXT:    xorb $-128, %al
 ; AVX2-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
@@ -1989,13 +1960,12 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; AVX512-NEXT:    vpminsb %ymm1, %ymm0, %ymm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512-NEXT:    vpminsb %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vmovdqa {{.*#+}} xmm1 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128]
-; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpsrlw $8, %xmm0, %xmm2
-; AVX512-NEXT:    vpminub %xmm2, %xmm0, %xmm0
+; AVX512-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
+; AVX512-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; AVX512-NEXT:    vpminub %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpextrb $0, %xmm0, %eax
+; AVX512-NEXT:    xorb $-128, %al
 ; AVX512-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
diff --git a/test/CodeGen/X86/vector-reduce-smin.ll b/test/CodeGen/X86/vector-reduce-smin.ll
index e8b6a242b8bf83387d3043ad21b273e2063e808b..b27812e0c7acb4dd6783ed19876ac367a14421f6 100644
--- a/test/CodeGen/X86/vector-reduce-smin.ll
+++ b/test/CodeGen/X86/vector-reduce-smin.ll
@@ -708,17 +708,16 @@ define i32 @test_v2i32(<2 x i32> %a0) {
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[0,2,2,3]
 ; SSE41-NEXT:    psrad $31, %xmm3
 ; SSE41-NEXT:    pblendw {{.*#+}} xmm3 = xmm2[0,1],xmm3[2,3],xmm2[4,5],xmm3[6,7]
-; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483648,2147483648]
-; SSE41-NEXT:    movdqa %xmm1, %xmm2
-; SSE41-NEXT:    pxor %xmm0, %xmm2
-; SSE41-NEXT:    pxor %xmm3, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm2, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm2, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm5, %xmm0
-; SSE41-NEXT:    por %xmm4, %xmm0
+; SSE41-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648]
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    pxor %xmm2, %xmm0
+; SSE41-NEXT:    pxor %xmm3, %xmm2
+; SSE41-NEXT:    movdqa %xmm2, %xmm4
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm2
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,0,2,2]
+; SSE41-NEXT:    pand %xmm4, %xmm0
+; SSE41-NEXT:    por %xmm2, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm3
 ; SSE41-NEXT:    movd %xmm3, %eax
 ; SSE41-NEXT:    retq
@@ -1165,17 +1164,16 @@ define i16 @test_v2i16(<2 x i16> %a0) {
 ; SSE41-NEXT:    psrad $16, %xmm1
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
 ; SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm0[2,3],xmm1[4,5],xmm0[6,7]
-; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483648,2147483648]
-; SSE41-NEXT:    movdqa %xmm3, %xmm2
-; SSE41-NEXT:    pxor %xmm0, %xmm2
-; SSE41-NEXT:    pxor %xmm1, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm2, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm2, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm5, %xmm0
-; SSE41-NEXT:    por %xmm4, %xmm0
+; SSE41-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648]
+; SSE41-NEXT:    movdqa %xmm3, %xmm0
+; SSE41-NEXT:    pxor %xmm2, %xmm0
+; SSE41-NEXT:    pxor %xmm1, %xmm2
+; SSE41-NEXT:    movdqa %xmm2, %xmm4
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm2
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,0,2,2]
+; SSE41-NEXT:    pand %xmm4, %xmm0
+; SSE41-NEXT:    por %xmm2, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm3, %xmm1
 ; SSE41-NEXT:    movd %xmm1, %eax
 ; SSE41-NEXT:    # kill: def $ax killed $ax killed $eax
@@ -1339,31 +1337,28 @@ define i16 @test_v8i16(<8 x i16> %a0) {
 ;
 ; SSE41-LABEL: test_v8i16:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [32768,32768,32768,32768,32768,32768,32768,32768]
-; SSE41-NEXT:    pxor %xmm1, %xmm0
+; SSE41-NEXT:    pxor {{.*}}(%rip), %xmm0
 ; SSE41-NEXT:    phminposuw %xmm0, %xmm0
-; SSE41-NEXT:    pxor %xmm1, %xmm0
 ; SSE41-NEXT:    movd %xmm0, %eax
+; SSE41-NEXT:    xorl $32768, %eax # imm = 0x8000
 ; SSE41-NEXT:    # kill: def $ax killed $ax killed $eax
 ; SSE41-NEXT:    retq
 ;
 ; AVX-LABEL: test_v8i16:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vmovdqa {{.*#+}} xmm1 = [32768,32768,32768,32768,32768,32768,32768,32768]
-; AVX-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
 ; AVX-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vmovd %xmm0, %eax
+; AVX-NEXT:    xorl $32768, %eax # imm = 0x8000
 ; AVX-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX-NEXT:    retq
 ;
 ; AVX512-LABEL: test_v8i16:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vmovdqa {{.*#+}} xmm1 = [32768,32768,32768,32768,32768,32768,32768,32768]
-; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
 ; AVX512-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
+; AVX512-NEXT:    xorl $32768, %eax # imm = 0x8000
 ; AVX512-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX512-NEXT:    retq
   %1 = call i16 @llvm.experimental.vector.reduce.smin.i16.v8i16(<8 x i16> %a0)
@@ -1388,11 +1383,10 @@ define i16 @test_v16i16(<16 x i16> %a0) {
 ; SSE41-LABEL: test_v16i16:
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    pminsw %xmm1, %xmm0
-; SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [32768,32768,32768,32768,32768,32768,32768,32768]
-; SSE41-NEXT:    pxor %xmm1, %xmm0
+; SSE41-NEXT:    pxor {{.*}}(%rip), %xmm0
 ; SSE41-NEXT:    phminposuw %xmm0, %xmm0
-; SSE41-NEXT:    pxor %xmm1, %xmm0
 ; SSE41-NEXT:    movd %xmm0, %eax
+; SSE41-NEXT:    xorl $32768, %eax # imm = 0x8000
 ; SSE41-NEXT:    # kill: def $ax killed $ax killed $eax
 ; SSE41-NEXT:    retq
 ;
@@ -1400,11 +1394,10 @@ define i16 @test_v16i16(<16 x i16> %a0) {
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; AVX1-NEXT:    vpminsw %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm1 = [32768,32768,32768,32768,32768,32768,32768,32768]
-; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
 ; AVX1-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovd %xmm0, %eax
+; AVX1-NEXT:    xorl $32768, %eax # imm = 0x8000
 ; AVX1-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
@@ -1413,11 +1406,10 @@ define i16 @test_v16i16(<16 x i16> %a0) {
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpminsw %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vmovdqa {{.*#+}} xmm1 = [32768,32768,32768,32768,32768,32768,32768,32768]
-; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
 ; AVX2-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
+; AVX2-NEXT:    xorl $32768, %eax # imm = 0x8000
 ; AVX2-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
@@ -1426,11 +1418,10 @@ define i16 @test_v16i16(<16 x i16> %a0) {
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512-NEXT:    vpminsw %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vmovdqa {{.*#+}} xmm1 = [32768,32768,32768,32768,32768,32768,32768,32768]
-; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
 ; AVX512-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
+; AVX512-NEXT:    xorl $32768, %eax # imm = 0x8000
 ; AVX512-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
@@ -1460,11 +1451,10 @@ define i16 @test_v32i16(<32 x i16> %a0) {
 ; SSE41-NEXT:    pminsw %xmm3, %xmm1
 ; SSE41-NEXT:    pminsw %xmm2, %xmm0
 ; SSE41-NEXT:    pminsw %xmm1, %xmm0
-; SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [32768,32768,32768,32768,32768,32768,32768,32768]
-; SSE41-NEXT:    pxor %xmm1, %xmm0
+; SSE41-NEXT:    pxor {{.*}}(%rip), %xmm0
 ; SSE41-NEXT:    phminposuw %xmm0, %xmm0
-; SSE41-NEXT:    pxor %xmm1, %xmm0
 ; SSE41-NEXT:    movd %xmm0, %eax
+; SSE41-NEXT:    xorl $32768, %eax # imm = 0x8000
 ; SSE41-NEXT:    # kill: def $ax killed $ax killed $eax
 ; SSE41-NEXT:    retq
 ;
@@ -1475,11 +1465,10 @@ define i16 @test_v32i16(<32 x i16> %a0) {
 ; AVX1-NEXT:    vpminsw %xmm2, %xmm3, %xmm2
 ; AVX1-NEXT:    vpminsw %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpminsw %xmm2, %xmm0, %xmm0
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm1 = [32768,32768,32768,32768,32768,32768,32768,32768]
-; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
 ; AVX1-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovd %xmm0, %eax
+; AVX1-NEXT:    xorl $32768, %eax # imm = 0x8000
 ; AVX1-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
@@ -1489,11 +1478,10 @@ define i16 @test_v32i16(<32 x i16> %a0) {
 ; AVX2-NEXT:    vpminsw %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpminsw %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vmovdqa {{.*#+}} xmm1 = [32768,32768,32768,32768,32768,32768,32768,32768]
-; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
 ; AVX2-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
+; AVX2-NEXT:    xorl $32768, %eax # imm = 0x8000
 ; AVX2-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
@@ -1504,11 +1492,10 @@ define i16 @test_v32i16(<32 x i16> %a0) {
 ; AVX512-NEXT:    vpminsw %ymm1, %ymm0, %ymm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512-NEXT:    vpminsw %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vmovdqa {{.*#+}} xmm1 = [32768,32768,32768,32768,32768,32768,32768,32768]
-; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
 ; AVX512-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
+; AVX512-NEXT:    xorl $32768, %eax # imm = 0x8000
 ; AVX512-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
@@ -1546,11 +1533,10 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 ; SSE41-NEXT:    pminsw %xmm4, %xmm0
 ; SSE41-NEXT:    pminsw %xmm2, %xmm0
 ; SSE41-NEXT:    pminsw %xmm1, %xmm0
-; SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [32768,32768,32768,32768,32768,32768,32768,32768]
-; SSE41-NEXT:    pxor %xmm1, %xmm0
+; SSE41-NEXT:    pxor {{.*}}(%rip), %xmm0
 ; SSE41-NEXT:    phminposuw %xmm0, %xmm0
-; SSE41-NEXT:    pxor %xmm1, %xmm0
 ; SSE41-NEXT:    movd %xmm0, %eax
+; SSE41-NEXT:    xorl $32768, %eax # imm = 0x8000
 ; SSE41-NEXT:    # kill: def $ax killed $ax killed $eax
 ; SSE41-NEXT:    retq
 ;
@@ -1567,11 +1553,10 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 ; AVX1-NEXT:    vpminsw %xmm2, %xmm0, %xmm0
 ; AVX1-NEXT:    vpminsw %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpminsw %xmm4, %xmm0, %xmm0
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm1 = [32768,32768,32768,32768,32768,32768,32768,32768]
-; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
 ; AVX1-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovd %xmm0, %eax
+; AVX1-NEXT:    xorl $32768, %eax # imm = 0x8000
 ; AVX1-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
@@ -1583,11 +1568,10 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 ; AVX2-NEXT:    vpminsw %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpminsw %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vmovdqa {{.*#+}} xmm1 = [32768,32768,32768,32768,32768,32768,32768,32768]
-; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
 ; AVX2-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
+; AVX2-NEXT:    xorl $32768, %eax # imm = 0x8000
 ; AVX2-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
@@ -1599,11 +1583,10 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 ; AVX512-NEXT:    vpminsw %ymm1, %ymm0, %ymm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512-NEXT:    vpminsw %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vmovdqa {{.*#+}} xmm1 = [32768,32768,32768,32768,32768,32768,32768,32768]
-; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; AVX512-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
 ; AVX512-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
+; AVX512-NEXT:    xorl $32768, %eax # imm = 0x8000
 ; AVX512-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
@@ -1667,17 +1650,16 @@ define i8 @test_v2i8(<2 x i8> %a0) {
 ; SSE41-NEXT:    psrad $24, %xmm1
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
 ; SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm1[0,1],xmm0[2,3],xmm1[4,5],xmm0[6,7]
-; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483648,2147483648]
-; SSE41-NEXT:    movdqa %xmm3, %xmm2
-; SSE41-NEXT:    pxor %xmm0, %xmm2
-; SSE41-NEXT:    pxor %xmm1, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm2, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm2, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm5, %xmm0
-; SSE41-NEXT:    por %xmm4, %xmm0
+; SSE41-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648]
+; SSE41-NEXT:    movdqa %xmm3, %xmm0
+; SSE41-NEXT:    pxor %xmm2, %xmm0
+; SSE41-NEXT:    pxor %xmm1, %xmm2
+; SSE41-NEXT:    movdqa %xmm2, %xmm4
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm2
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,0,2,2]
+; SSE41-NEXT:    pand %xmm4, %xmm0
+; SSE41-NEXT:    por %xmm2, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm3, %xmm1
 ; SSE41-NEXT:    pextrb $0, %xmm1, %eax
 ; SSE41-NEXT:    # kill: def $al killed $al killed $eax
@@ -1946,38 +1928,35 @@ define i8 @test_v16i8(<16 x i8> %a0) {
 ;
 ; SSE41-LABEL: test_v16i8:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128]
-; SSE41-NEXT:    pxor %xmm1, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm2
-; SSE41-NEXT:    psrlw $8, %xmm2
-; SSE41-NEXT:    pminub %xmm0, %xmm2
-; SSE41-NEXT:    phminposuw %xmm2, %xmm0
-; SSE41-NEXT:    pxor %xmm1, %xmm0
+; SSE41-NEXT:    pxor {{.*}}(%rip), %xmm0
+; SSE41-NEXT:    movdqa %xmm0, %xmm1
+; SSE41-NEXT:    psrlw $8, %xmm1
+; SSE41-NEXT:    pminub %xmm0, %xmm1
+; SSE41-NEXT:    phminposuw %xmm1, %xmm0
 ; SSE41-NEXT:    pextrb $0, %xmm0, %eax
+; SSE41-NEXT:    xorb $-128, %al
 ; SSE41-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE41-NEXT:    retq
 ;
 ; AVX-LABEL: test_v16i8:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vmovdqa {{.*#+}} xmm1 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128]
-; AVX-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX-NEXT:    vpsrlw $8, %xmm0, %xmm2
-; AVX-NEXT:    vpminub %xmm2, %xmm0, %xmm0
+; AVX-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
+; AVX-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; AVX-NEXT:    vpminub %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpextrb $0, %xmm0, %eax
+; AVX-NEXT:    xorb $-128, %al
 ; AVX-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX-NEXT:    retq
 ;
 ; AVX512-LABEL: test_v16i8:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vmovdqa {{.*#+}} xmm1 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128]
-; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpsrlw $8, %xmm0, %xmm2
-; AVX512-NEXT:    vpminub %xmm2, %xmm0, %xmm0
+; AVX512-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
+; AVX512-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; AVX512-NEXT:    vpminub %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpextrb $0, %xmm0, %eax
+; AVX512-NEXT:    xorb $-128, %al
 ; AVX512-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX512-NEXT:    retq
   %1 = call i8 @llvm.experimental.vector.reduce.smin.i8.v16i8(<16 x i8> %a0)
@@ -2025,14 +2004,13 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; SSE41-LABEL: test_v32i8:
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    pminsb %xmm1, %xmm0
-; SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128]
-; SSE41-NEXT:    pxor %xmm1, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm2
-; SSE41-NEXT:    psrlw $8, %xmm2
-; SSE41-NEXT:    pminub %xmm0, %xmm2
-; SSE41-NEXT:    phminposuw %xmm2, %xmm0
-; SSE41-NEXT:    pxor %xmm1, %xmm0
+; SSE41-NEXT:    pxor {{.*}}(%rip), %xmm0
+; SSE41-NEXT:    movdqa %xmm0, %xmm1
+; SSE41-NEXT:    psrlw $8, %xmm1
+; SSE41-NEXT:    pminub %xmm0, %xmm1
+; SSE41-NEXT:    phminposuw %xmm1, %xmm0
 ; SSE41-NEXT:    pextrb $0, %xmm0, %eax
+; SSE41-NEXT:    xorb $-128, %al
 ; SSE41-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE41-NEXT:    retq
 ;
@@ -2040,13 +2018,12 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; AVX1-NEXT:    vpminsb %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm1 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128]
-; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm2
-; AVX1-NEXT:    vpminub %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
+; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; AVX1-NEXT:    vpminub %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpextrb $0, %xmm0, %eax
+; AVX1-NEXT:    xorb $-128, %al
 ; AVX1-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
@@ -2055,13 +2032,12 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpminsb %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vmovdqa {{.*#+}} xmm1 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128]
-; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpsrlw $8, %xmm0, %xmm2
-; AVX2-NEXT:    vpminub %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
+; AVX2-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; AVX2-NEXT:    vpminub %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpextrb $0, %xmm0, %eax
+; AVX2-NEXT:    xorb $-128, %al
 ; AVX2-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
@@ -2070,13 +2046,12 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512-NEXT:    vpminsb %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vmovdqa {{.*#+}} xmm1 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128]
-; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpsrlw $8, %xmm0, %xmm2
-; AVX512-NEXT:    vpminub %xmm2, %xmm0, %xmm0
+; AVX512-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
+; AVX512-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; AVX512-NEXT:    vpminub %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpextrb $0, %xmm0, %eax
+; AVX512-NEXT:    xorb $-128, %al
 ; AVX512-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
@@ -2137,14 +2112,13 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; SSE41-NEXT:    pminsb %xmm3, %xmm1
 ; SSE41-NEXT:    pminsb %xmm2, %xmm0
 ; SSE41-NEXT:    pminsb %xmm1, %xmm0
-; SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128]
-; SSE41-NEXT:    pxor %xmm1, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm2
-; SSE41-NEXT:    psrlw $8, %xmm2
-; SSE41-NEXT:    pminub %xmm0, %xmm2
-; SSE41-NEXT:    phminposuw %xmm2, %xmm0
-; SSE41-NEXT:    pxor %xmm1, %xmm0
+; SSE41-NEXT:    pxor {{.*}}(%rip), %xmm0
+; SSE41-NEXT:    movdqa %xmm0, %xmm1
+; SSE41-NEXT:    psrlw $8, %xmm1
+; SSE41-NEXT:    pminub %xmm0, %xmm1
+; SSE41-NEXT:    phminposuw %xmm1, %xmm0
 ; SSE41-NEXT:    pextrb $0, %xmm0, %eax
+; SSE41-NEXT:    xorb $-128, %al
 ; SSE41-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE41-NEXT:    retq
 ;
@@ -2155,13 +2129,12 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; AVX1-NEXT:    vpminsb %xmm2, %xmm3, %xmm2
 ; AVX1-NEXT:    vpminsb %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpminsb %xmm2, %xmm0, %xmm0
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm1 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128]
-; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm2
-; AVX1-NEXT:    vpminub %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
+; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; AVX1-NEXT:    vpminub %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpextrb $0, %xmm0, %eax
+; AVX1-NEXT:    xorb $-128, %al
 ; AVX1-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
@@ -2171,13 +2144,12 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; AVX2-NEXT:    vpminsb %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpminsb %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vmovdqa {{.*#+}} xmm1 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128]
-; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpsrlw $8, %xmm0, %xmm2
-; AVX2-NEXT:    vpminub %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
+; AVX2-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; AVX2-NEXT:    vpminub %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpextrb $0, %xmm0, %eax
+; AVX2-NEXT:    xorb $-128, %al
 ; AVX2-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
@@ -2188,13 +2160,12 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; AVX512-NEXT:    vpminsb %ymm1, %ymm0, %ymm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512-NEXT:    vpminsb %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vmovdqa {{.*#+}} xmm1 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128]
-; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpsrlw $8, %xmm0, %xmm2
-; AVX512-NEXT:    vpminub %xmm2, %xmm0, %xmm0
+; AVX512-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
+; AVX512-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; AVX512-NEXT:    vpminub %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpextrb $0, %xmm0, %eax
+; AVX512-NEXT:    xorb $-128, %al
 ; AVX512-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
@@ -2279,14 +2250,13 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; SSE41-NEXT:    pminsb %xmm4, %xmm0
 ; SSE41-NEXT:    pminsb %xmm2, %xmm0
 ; SSE41-NEXT:    pminsb %xmm1, %xmm0
-; SSE41-NEXT:    movdqa {{.*#+}} xmm1 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128]
-; SSE41-NEXT:    pxor %xmm1, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm2
-; SSE41-NEXT:    psrlw $8, %xmm2
-; SSE41-NEXT:    pminub %xmm0, %xmm2
-; SSE41-NEXT:    phminposuw %xmm2, %xmm0
-; SSE41-NEXT:    pxor %xmm1, %xmm0
+; SSE41-NEXT:    pxor {{.*}}(%rip), %xmm0
+; SSE41-NEXT:    movdqa %xmm0, %xmm1
+; SSE41-NEXT:    psrlw $8, %xmm1
+; SSE41-NEXT:    pminub %xmm0, %xmm1
+; SSE41-NEXT:    phminposuw %xmm1, %xmm0
 ; SSE41-NEXT:    pextrb $0, %xmm0, %eax
+; SSE41-NEXT:    xorb $-128, %al
 ; SSE41-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE41-NEXT:    retq
 ;
@@ -2303,13 +2273,12 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; AVX1-NEXT:    vpminsb %xmm2, %xmm0, %xmm0
 ; AVX1-NEXT:    vpminsb %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpminsb %xmm4, %xmm0, %xmm0
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm1 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128]
-; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm2
-; AVX1-NEXT:    vpminub %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
+; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; AVX1-NEXT:    vpminub %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpextrb $0, %xmm0, %eax
+; AVX1-NEXT:    xorb $-128, %al
 ; AVX1-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
@@ -2321,13 +2290,12 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; AVX2-NEXT:    vpminsb %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-NEXT:    vpminsb %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vmovdqa {{.*#+}} xmm1 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128]
-; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpsrlw $8, %xmm0, %xmm2
-; AVX2-NEXT:    vpminub %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
+; AVX2-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; AVX2-NEXT:    vpminub %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpextrb $0, %xmm0, %eax
+; AVX2-NEXT:    xorb $-128, %al
 ; AVX2-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
@@ -2339,13 +2307,12 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; AVX512-NEXT:    vpminsb %ymm1, %ymm0, %ymm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512-NEXT:    vpminsb %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vmovdqa {{.*#+}} xmm1 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128]
-; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpsrlw $8, %xmm0, %xmm2
-; AVX512-NEXT:    vpminub %xmm2, %xmm0, %xmm0
+; AVX512-NEXT:    vpxor {{.*}}(%rip), %xmm0, %xmm0
+; AVX512-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; AVX512-NEXT:    vpminub %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpextrb $0, %xmm0, %eax
+; AVX512-NEXT:    xorb $-128, %al
 ; AVX512-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
diff --git a/test/CodeGen/X86/vector-reduce-umax-widen.ll b/test/CodeGen/X86/vector-reduce-umax-widen.ll
index 3b6cdad368277c17d9bfb1d73dece02273942e4d..46c95994b5199679f3cbf8010ee42896ea7578f3 100644
--- a/test/CodeGen/X86/vector-reduce-umax-widen.ll
+++ b/test/CodeGen/X86/vector-reduce-umax-widen.ll
@@ -1166,8 +1166,8 @@ define i16 @test_v2i16(<2 x i16> %a0) {
 ; SSE2-NEXT:    pxor %xmm2, %xmm0
 ; SSE2-NEXT:    pxor %xmm2, %xmm1
 ; SSE2-NEXT:    pmaxsw %xmm0, %xmm1
-; SSE2-NEXT:    pxor %xmm2, %xmm1
 ; SSE2-NEXT:    movd %xmm1, %eax
+; SSE2-NEXT:    xorl $32768, %eax # imm = 0x8000
 ; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
 ; SSE2-NEXT:    retq
 ;
@@ -1212,8 +1212,8 @@ define i16 @test_v4i16(<4 x i16> %a0) {
 ; SSE2-NEXT:    psrld $16, %xmm0
 ; SSE2-NEXT:    pxor %xmm2, %xmm0
 ; SSE2-NEXT:    pmaxsw %xmm1, %xmm0
-; SSE2-NEXT:    pxor %xmm2, %xmm0
 ; SSE2-NEXT:    movd %xmm0, %eax
+; SSE2-NEXT:    xorl $32768, %eax # imm = 0x8000
 ; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
 ; SSE2-NEXT:    retq
 ;
@@ -1269,18 +1269,18 @@ define i16 @test_v8i16(<8 x i16> %a0) {
 ; SSE2-NEXT:    psrld $16, %xmm1
 ; SSE2-NEXT:    pxor %xmm2, %xmm1
 ; SSE2-NEXT:    pmaxsw %xmm0, %xmm1
-; SSE2-NEXT:    pxor %xmm2, %xmm1
 ; SSE2-NEXT:    movd %xmm1, %eax
+; SSE2-NEXT:    xorl $32768, %eax # imm = 0x8000
 ; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: test_v8i16:
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE41-NEXT:    pxor %xmm1, %xmm0
-; SSE41-NEXT:    phminposuw %xmm0, %xmm0
-; SSE41-NEXT:    pxor %xmm1, %xmm0
+; SSE41-NEXT:    pxor %xmm0, %xmm1
+; SSE41-NEXT:    phminposuw %xmm1, %xmm0
 ; SSE41-NEXT:    movd %xmm0, %eax
+; SSE41-NEXT:    notl %eax
 ; SSE41-NEXT:    # kill: def $ax killed $ax killed $eax
 ; SSE41-NEXT:    retq
 ;
@@ -1289,8 +1289,8 @@ define i16 @test_v8i16(<8 x i16> %a0) {
 ; AVX-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
 ; AVX-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vmovd %xmm0, %eax
+; AVX-NEXT:    notl %eax
 ; AVX-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX-NEXT:    retq
 ;
@@ -1299,8 +1299,8 @@ define i16 @test_v8i16(<8 x i16> %a0) {
 ; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
 ; AVX512BW-NEXT:    vpternlogq $15, %zmm0, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX512BW-NEXT:    vpternlogq $15, %zmm0, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vmovd %xmm0, %eax
+; AVX512BW-NEXT:    notl %eax
 ; AVX512BW-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
@@ -1309,8 +1309,8 @@ define i16 @test_v8i16(<8 x i16> %a0) {
 ; AVX512VL:       # %bb.0:
 ; AVX512VL-NEXT:    vpternlogq $15, %xmm0, %xmm0, %xmm0
 ; AVX512VL-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX512VL-NEXT:    vpternlogq $15, %xmm0, %xmm0, %xmm0
 ; AVX512VL-NEXT:    vmovd %xmm0, %eax
+; AVX512VL-NEXT:    notl %eax
 ; AVX512VL-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX512VL-NEXT:    retq
   %1 = call i16 @llvm.experimental.vector.reduce.umax.i16.v8i16(<8 x i16> %a0)
@@ -1339,8 +1339,8 @@ define i16 @test_v16i16(<16 x i16> %a0) {
 ; SSE2-NEXT:    psrld $16, %xmm1
 ; SSE2-NEXT:    pxor %xmm2, %xmm1
 ; SSE2-NEXT:    pmaxsw %xmm0, %xmm1
-; SSE2-NEXT:    pxor %xmm2, %xmm1
 ; SSE2-NEXT:    movd %xmm1, %eax
+; SSE2-NEXT:    xorl $32768, %eax # imm = 0x8000
 ; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
 ; SSE2-NEXT:    retq
 ;
@@ -1348,10 +1348,10 @@ define i16 @test_v16i16(<16 x i16> %a0) {
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    pmaxuw %xmm1, %xmm0
 ; SSE41-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE41-NEXT:    pxor %xmm1, %xmm0
-; SSE41-NEXT:    phminposuw %xmm0, %xmm0
-; SSE41-NEXT:    pxor %xmm1, %xmm0
+; SSE41-NEXT:    pxor %xmm0, %xmm1
+; SSE41-NEXT:    phminposuw %xmm1, %xmm0
 ; SSE41-NEXT:    movd %xmm0, %eax
+; SSE41-NEXT:    notl %eax
 ; SSE41-NEXT:    # kill: def $ax killed $ax killed $eax
 ; SSE41-NEXT:    retq
 ;
@@ -1362,8 +1362,8 @@ define i16 @test_v16i16(<16 x i16> %a0) {
 ; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
 ; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovd %xmm0, %eax
+; AVX1-NEXT:    notl %eax
 ; AVX1-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
@@ -1375,8 +1375,8 @@ define i16 @test_v16i16(<16 x i16> %a0) {
 ; AVX2-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
 ; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
+; AVX2-NEXT:    notl %eax
 ; AVX2-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
@@ -1387,8 +1387,8 @@ define i16 @test_v16i16(<16 x i16> %a0) {
 ; AVX512BW-NEXT:    vpmaxuw %xmm1, %xmm0, %xmm0
 ; AVX512BW-NEXT:    vpternlogq $15, %zmm0, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX512BW-NEXT:    vpternlogq $15, %zmm0, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vmovd %xmm0, %eax
+; AVX512BW-NEXT:    notl %eax
 ; AVX512BW-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
@@ -1399,8 +1399,8 @@ define i16 @test_v16i16(<16 x i16> %a0) {
 ; AVX512VL-NEXT:    vpmaxuw %xmm1, %xmm0, %xmm0
 ; AVX512VL-NEXT:    vpternlogq $15, %xmm0, %xmm0, %xmm0
 ; AVX512VL-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX512VL-NEXT:    vpternlogq $15, %xmm0, %xmm0, %xmm0
 ; AVX512VL-NEXT:    vmovd %xmm0, %eax
+; AVX512VL-NEXT:    notl %eax
 ; AVX512VL-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX512VL-NEXT:    vzeroupper
 ; AVX512VL-NEXT:    retq
@@ -1434,8 +1434,8 @@ define i16 @test_v32i16(<32 x i16> %a0) {
 ; SSE2-NEXT:    psrld $16, %xmm1
 ; SSE2-NEXT:    pxor %xmm4, %xmm1
 ; SSE2-NEXT:    pmaxsw %xmm0, %xmm1
-; SSE2-NEXT:    pxor %xmm4, %xmm1
 ; SSE2-NEXT:    movd %xmm1, %eax
+; SSE2-NEXT:    xorl $32768, %eax # imm = 0x8000
 ; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
 ; SSE2-NEXT:    retq
 ;
@@ -1445,10 +1445,10 @@ define i16 @test_v32i16(<32 x i16> %a0) {
 ; SSE41-NEXT:    pmaxuw %xmm2, %xmm0
 ; SSE41-NEXT:    pmaxuw %xmm1, %xmm0
 ; SSE41-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE41-NEXT:    pxor %xmm1, %xmm0
-; SSE41-NEXT:    phminposuw %xmm0, %xmm0
-; SSE41-NEXT:    pxor %xmm1, %xmm0
+; SSE41-NEXT:    pxor %xmm0, %xmm1
+; SSE41-NEXT:    phminposuw %xmm1, %xmm0
 ; SSE41-NEXT:    movd %xmm0, %eax
+; SSE41-NEXT:    notl %eax
 ; SSE41-NEXT:    # kill: def $ax killed $ax killed $eax
 ; SSE41-NEXT:    retq
 ;
@@ -1462,8 +1462,8 @@ define i16 @test_v32i16(<32 x i16> %a0) {
 ; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
 ; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovd %xmm0, %eax
+; AVX1-NEXT:    notl %eax
 ; AVX1-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
@@ -1476,8 +1476,8 @@ define i16 @test_v32i16(<32 x i16> %a0) {
 ; AVX2-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
 ; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
+; AVX2-NEXT:    notl %eax
 ; AVX2-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
@@ -1490,8 +1490,8 @@ define i16 @test_v32i16(<32 x i16> %a0) {
 ; AVX512BW-NEXT:    vpmaxuw %xmm1, %xmm0, %xmm0
 ; AVX512BW-NEXT:    vpternlogq $15, %zmm0, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX512BW-NEXT:    vpternlogq $15, %zmm0, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vmovd %xmm0, %eax
+; AVX512BW-NEXT:    notl %eax
 ; AVX512BW-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
@@ -1504,8 +1504,8 @@ define i16 @test_v32i16(<32 x i16> %a0) {
 ; AVX512VL-NEXT:    vpmaxuw %xmm1, %xmm0, %xmm0
 ; AVX512VL-NEXT:    vpternlogq $15, %xmm0, %xmm0, %xmm0
 ; AVX512VL-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX512VL-NEXT:    vpternlogq $15, %xmm0, %xmm0, %xmm0
 ; AVX512VL-NEXT:    vmovd %xmm0, %eax
+; AVX512VL-NEXT:    notl %eax
 ; AVX512VL-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX512VL-NEXT:    vzeroupper
 ; AVX512VL-NEXT:    retq
@@ -1547,8 +1547,8 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 ; SSE2-NEXT:    psrld $16, %xmm0
 ; SSE2-NEXT:    pxor %xmm8, %xmm0
 ; SSE2-NEXT:    pmaxsw %xmm1, %xmm0
-; SSE2-NEXT:    pxor %xmm8, %xmm0
 ; SSE2-NEXT:    movd %xmm0, %eax
+; SSE2-NEXT:    xorl $32768, %eax # imm = 0x8000
 ; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
 ; SSE2-NEXT:    retq
 ;
@@ -1562,10 +1562,10 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 ; SSE41-NEXT:    pmaxuw %xmm2, %xmm0
 ; SSE41-NEXT:    pmaxuw %xmm1, %xmm0
 ; SSE41-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE41-NEXT:    pxor %xmm1, %xmm0
-; SSE41-NEXT:    phminposuw %xmm0, %xmm0
-; SSE41-NEXT:    pxor %xmm1, %xmm0
+; SSE41-NEXT:    pxor %xmm0, %xmm1
+; SSE41-NEXT:    phminposuw %xmm1, %xmm0
 ; SSE41-NEXT:    movd %xmm0, %eax
+; SSE41-NEXT:    notl %eax
 ; SSE41-NEXT:    # kill: def $ax killed $ax killed $eax
 ; SSE41-NEXT:    retq
 ;
@@ -1585,8 +1585,8 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 ; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
 ; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovd %xmm0, %eax
+; AVX1-NEXT:    notl %eax
 ; AVX1-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
@@ -1601,8 +1601,8 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 ; AVX2-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
 ; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
+; AVX2-NEXT:    notl %eax
 ; AVX2-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
@@ -1616,8 +1616,8 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 ; AVX512BW-NEXT:    vpmaxuw %xmm1, %xmm0, %xmm0
 ; AVX512BW-NEXT:    vpternlogq $15, %zmm0, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX512BW-NEXT:    vpternlogq $15, %zmm0, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vmovd %xmm0, %eax
+; AVX512BW-NEXT:    notl %eax
 ; AVX512BW-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
@@ -1631,8 +1631,8 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 ; AVX512VL-NEXT:    vpmaxuw %xmm1, %xmm0, %xmm0
 ; AVX512VL-NEXT:    vpternlogq $15, %xmm0, %xmm0, %xmm0
 ; AVX512VL-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX512VL-NEXT:    vpternlogq $15, %xmm0, %xmm0, %xmm0
 ; AVX512VL-NEXT:    vmovd %xmm0, %eax
+; AVX512VL-NEXT:    notl %eax
 ; AVX512VL-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX512VL-NEXT:    vzeroupper
 ; AVX512VL-NEXT:    retq
@@ -1806,13 +1806,13 @@ define i8 @test_v16i8(<16 x i8> %a0) {
 ; SSE41-LABEL: test_v16i8:
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE41-NEXT:    pxor %xmm1, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm2
-; SSE41-NEXT:    psrlw $8, %xmm2
-; SSE41-NEXT:    pminub %xmm0, %xmm2
-; SSE41-NEXT:    phminposuw %xmm2, %xmm0
-; SSE41-NEXT:    pxor %xmm1, %xmm0
+; SSE41-NEXT:    pxor %xmm0, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    psrlw $8, %xmm0
+; SSE41-NEXT:    pminub %xmm1, %xmm0
+; SSE41-NEXT:    phminposuw %xmm0, %xmm0
 ; SSE41-NEXT:    pextrb $0, %xmm0, %eax
+; SSE41-NEXT:    notb %al
 ; SSE41-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE41-NEXT:    retq
 ;
@@ -1820,11 +1820,11 @@ define i8 @test_v16i8(<16 x i8> %a0) {
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
 ; AVX-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX-NEXT:    vpsrlw $8, %xmm0, %xmm2
-; AVX-NEXT:    vpminub %xmm2, %xmm0, %xmm0
+; AVX-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; AVX-NEXT:    vpminub %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpextrb $0, %xmm0, %eax
+; AVX-NEXT:    notb %al
 ; AVX-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX-NEXT:    retq
 ;
@@ -1835,8 +1835,8 @@ define i8 @test_v16i8(<16 x i8> %a0) {
 ; AVX512BW-NEXT:    vpsrlw $8, %xmm0, %xmm1
 ; AVX512BW-NEXT:    vpminub %xmm1, %xmm0, %xmm0
 ; AVX512BW-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX512BW-NEXT:    vpternlogq $15, %zmm0, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vpextrb $0, %xmm0, %eax
+; AVX512BW-NEXT:    notb %al
 ; AVX512BW-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
@@ -1847,8 +1847,8 @@ define i8 @test_v16i8(<16 x i8> %a0) {
 ; AVX512VL-NEXT:    vpsrlw $8, %xmm0, %xmm1
 ; AVX512VL-NEXT:    vpminub %xmm1, %xmm0, %xmm0
 ; AVX512VL-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX512VL-NEXT:    vpternlogq $15, %xmm0, %xmm0, %xmm0
 ; AVX512VL-NEXT:    vpextrb $0, %xmm0, %eax
+; AVX512VL-NEXT:    notb %al
 ; AVX512VL-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX512VL-NEXT:    retq
   %1 = call i8 @llvm.experimental.vector.reduce.umax.i8.v16i8(<16 x i8> %a0)
@@ -1877,13 +1877,13 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    pmaxub %xmm1, %xmm0
 ; SSE41-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE41-NEXT:    pxor %xmm1, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm2
-; SSE41-NEXT:    psrlw $8, %xmm2
-; SSE41-NEXT:    pminub %xmm0, %xmm2
-; SSE41-NEXT:    phminposuw %xmm2, %xmm0
-; SSE41-NEXT:    pxor %xmm1, %xmm0
+; SSE41-NEXT:    pxor %xmm0, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    psrlw $8, %xmm0
+; SSE41-NEXT:    pminub %xmm1, %xmm0
+; SSE41-NEXT:    phminposuw %xmm0, %xmm0
 ; SSE41-NEXT:    pextrb $0, %xmm0, %eax
+; SSE41-NEXT:    notb %al
 ; SSE41-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE41-NEXT:    retq
 ;
@@ -1893,11 +1893,11 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; AVX1-NEXT:    vpmaxub %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
 ; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm2
-; AVX1-NEXT:    vpminub %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; AVX1-NEXT:    vpminub %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpextrb $0, %xmm0, %eax
+; AVX1-NEXT:    notb %al
 ; AVX1-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
@@ -1908,11 +1908,11 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; AVX2-NEXT:    vpmaxub %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
 ; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpsrlw $8, %xmm0, %xmm2
-; AVX2-NEXT:    vpminub %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; AVX2-NEXT:    vpminub %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpextrb $0, %xmm0, %eax
+; AVX2-NEXT:    notb %al
 ; AVX2-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
@@ -1925,8 +1925,8 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; AVX512BW-NEXT:    vpsrlw $8, %xmm0, %xmm1
 ; AVX512BW-NEXT:    vpminub %xmm1, %xmm0, %xmm0
 ; AVX512BW-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX512BW-NEXT:    vpternlogq $15, %zmm0, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vpextrb $0, %xmm0, %eax
+; AVX512BW-NEXT:    notb %al
 ; AVX512BW-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
@@ -1939,8 +1939,8 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; AVX512VL-NEXT:    vpsrlw $8, %xmm0, %xmm1
 ; AVX512VL-NEXT:    vpminub %xmm1, %xmm0, %xmm0
 ; AVX512VL-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX512VL-NEXT:    vpternlogq $15, %xmm0, %xmm0, %xmm0
 ; AVX512VL-NEXT:    vpextrb $0, %xmm0, %eax
+; AVX512VL-NEXT:    notb %al
 ; AVX512VL-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX512VL-NEXT:    vzeroupper
 ; AVX512VL-NEXT:    retq
@@ -1974,13 +1974,13 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; SSE41-NEXT:    pmaxub %xmm2, %xmm0
 ; SSE41-NEXT:    pmaxub %xmm1, %xmm0
 ; SSE41-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE41-NEXT:    pxor %xmm1, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm2
-; SSE41-NEXT:    psrlw $8, %xmm2
-; SSE41-NEXT:    pminub %xmm0, %xmm2
-; SSE41-NEXT:    phminposuw %xmm2, %xmm0
-; SSE41-NEXT:    pxor %xmm1, %xmm0
+; SSE41-NEXT:    pxor %xmm0, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    psrlw $8, %xmm0
+; SSE41-NEXT:    pminub %xmm1, %xmm0
+; SSE41-NEXT:    phminposuw %xmm0, %xmm0
 ; SSE41-NEXT:    pextrb $0, %xmm0, %eax
+; SSE41-NEXT:    notb %al
 ; SSE41-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE41-NEXT:    retq
 ;
@@ -1993,11 +1993,11 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; AVX1-NEXT:    vpmaxub %xmm2, %xmm0, %xmm0
 ; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
 ; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm2
-; AVX1-NEXT:    vpminub %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; AVX1-NEXT:    vpminub %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpextrb $0, %xmm0, %eax
+; AVX1-NEXT:    notb %al
 ; AVX1-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
@@ -2009,11 +2009,11 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; AVX2-NEXT:    vpmaxub %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
 ; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpsrlw $8, %xmm0, %xmm2
-; AVX2-NEXT:    vpminub %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; AVX2-NEXT:    vpminub %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpextrb $0, %xmm0, %eax
+; AVX2-NEXT:    notb %al
 ; AVX2-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
@@ -2028,8 +2028,8 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; AVX512BW-NEXT:    vpsrlw $8, %xmm0, %xmm1
 ; AVX512BW-NEXT:    vpminub %xmm1, %xmm0, %xmm0
 ; AVX512BW-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX512BW-NEXT:    vpternlogq $15, %zmm0, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vpextrb $0, %xmm0, %eax
+; AVX512BW-NEXT:    notb %al
 ; AVX512BW-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
@@ -2044,8 +2044,8 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; AVX512VL-NEXT:    vpsrlw $8, %xmm0, %xmm1
 ; AVX512VL-NEXT:    vpminub %xmm1, %xmm0, %xmm0
 ; AVX512VL-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX512VL-NEXT:    vpternlogq $15, %xmm0, %xmm0, %xmm0
 ; AVX512VL-NEXT:    vpextrb $0, %xmm0, %eax
+; AVX512VL-NEXT:    notb %al
 ; AVX512VL-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX512VL-NEXT:    vzeroupper
 ; AVX512VL-NEXT:    retq
@@ -2087,13 +2087,13 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; SSE41-NEXT:    pmaxub %xmm2, %xmm0
 ; SSE41-NEXT:    pmaxub %xmm1, %xmm0
 ; SSE41-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE41-NEXT:    pxor %xmm1, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm2
-; SSE41-NEXT:    psrlw $8, %xmm2
-; SSE41-NEXT:    pminub %xmm0, %xmm2
-; SSE41-NEXT:    phminposuw %xmm2, %xmm0
-; SSE41-NEXT:    pxor %xmm1, %xmm0
+; SSE41-NEXT:    pxor %xmm0, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    psrlw $8, %xmm0
+; SSE41-NEXT:    pminub %xmm1, %xmm0
+; SSE41-NEXT:    phminposuw %xmm0, %xmm0
 ; SSE41-NEXT:    pextrb $0, %xmm0, %eax
+; SSE41-NEXT:    notb %al
 ; SSE41-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE41-NEXT:    retq
 ;
@@ -2112,11 +2112,11 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; AVX1-NEXT:    vpmaxub %xmm4, %xmm0, %xmm0
 ; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
 ; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm2
-; AVX1-NEXT:    vpminub %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; AVX1-NEXT:    vpminub %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpextrb $0, %xmm0, %eax
+; AVX1-NEXT:    notb %al
 ; AVX1-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
@@ -2130,11 +2130,11 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; AVX2-NEXT:    vpmaxub %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
 ; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpsrlw $8, %xmm0, %xmm2
-; AVX2-NEXT:    vpminub %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; AVX2-NEXT:    vpminub %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpextrb $0, %xmm0, %eax
+; AVX2-NEXT:    notb %al
 ; AVX2-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
@@ -2150,8 +2150,8 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; AVX512BW-NEXT:    vpsrlw $8, %xmm0, %xmm1
 ; AVX512BW-NEXT:    vpminub %xmm1, %xmm0, %xmm0
 ; AVX512BW-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX512BW-NEXT:    vpternlogq $15, %zmm0, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vpextrb $0, %xmm0, %eax
+; AVX512BW-NEXT:    notb %al
 ; AVX512BW-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
@@ -2167,8 +2167,8 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; AVX512VL-NEXT:    vpsrlw $8, %xmm0, %xmm1
 ; AVX512VL-NEXT:    vpminub %xmm1, %xmm0, %xmm0
 ; AVX512VL-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX512VL-NEXT:    vpternlogq $15, %xmm0, %xmm0, %xmm0
 ; AVX512VL-NEXT:    vpextrb $0, %xmm0, %eax
+; AVX512VL-NEXT:    notb %al
 ; AVX512VL-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX512VL-NEXT:    vzeroupper
 ; AVX512VL-NEXT:    retq
diff --git a/test/CodeGen/X86/vector-reduce-umax.ll b/test/CodeGen/X86/vector-reduce-umax.ll
index ffc98bb289d762543ff792ef1a6a020620cd2f8e..32ea3f5c083c08c5340a333135ff1d38b306e227 100644
--- a/test/CodeGen/X86/vector-reduce-umax.ll
+++ b/test/CodeGen/X86/vector-reduce-umax.ll
@@ -760,12 +760,11 @@ define i32 @test_v2i32(<2 x i32> %a0) {
 ; SSE41-NEXT:    pxor %xmm0, %xmm3
 ; SSE41-NEXT:    pxor %xmm2, %xmm0
 ; SSE41-NEXT:    movdqa %xmm3, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm3, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm5, %xmm0
-; SSE41-NEXT:    por %xmm4, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm3
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[0,0,2,2]
+; SSE41-NEXT:    pand %xmm4, %xmm0
+; SSE41-NEXT:    por %xmm3, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm2
 ; SSE41-NEXT:    movd %xmm2, %eax
 ; SSE41-NEXT:    retq
@@ -1248,12 +1247,11 @@ define i16 @test_v2i16(<2 x i16> %a0) {
 ; SSE41-NEXT:    por %xmm0, %xmm3
 ; SSE41-NEXT:    por %xmm2, %xmm0
 ; SSE41-NEXT:    movdqa %xmm3, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm3, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm5, %xmm0
-; SSE41-NEXT:    por %xmm4, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm3
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[0,0,2,2]
+; SSE41-NEXT:    pand %xmm4, %xmm0
+; SSE41-NEXT:    por %xmm3, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm2
 ; SSE41-NEXT:    movd %xmm2, %eax
 ; SSE41-NEXT:    # kill: def $ax killed $ax killed $eax
@@ -1389,18 +1387,18 @@ define i16 @test_v8i16(<8 x i16> %a0) {
 ; SSE2-NEXT:    psrld $16, %xmm1
 ; SSE2-NEXT:    pxor %xmm2, %xmm1
 ; SSE2-NEXT:    pmaxsw %xmm0, %xmm1
-; SSE2-NEXT:    pxor %xmm2, %xmm1
 ; SSE2-NEXT:    movd %xmm1, %eax
+; SSE2-NEXT:    xorl $32768, %eax # imm = 0x8000
 ; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: test_v8i16:
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE41-NEXT:    pxor %xmm1, %xmm0
-; SSE41-NEXT:    phminposuw %xmm0, %xmm0
-; SSE41-NEXT:    pxor %xmm1, %xmm0
+; SSE41-NEXT:    pxor %xmm0, %xmm1
+; SSE41-NEXT:    phminposuw %xmm1, %xmm0
 ; SSE41-NEXT:    movd %xmm0, %eax
+; SSE41-NEXT:    notl %eax
 ; SSE41-NEXT:    # kill: def $ax killed $ax killed $eax
 ; SSE41-NEXT:    retq
 ;
@@ -1409,8 +1407,8 @@ define i16 @test_v8i16(<8 x i16> %a0) {
 ; AVX-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
 ; AVX-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vmovd %xmm0, %eax
+; AVX-NEXT:    notl %eax
 ; AVX-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX-NEXT:    retq
 ;
@@ -1419,8 +1417,8 @@ define i16 @test_v8i16(<8 x i16> %a0) {
 ; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
 ; AVX512BW-NEXT:    vpternlogq $15, %zmm0, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX512BW-NEXT:    vpternlogq $15, %zmm0, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vmovd %xmm0, %eax
+; AVX512BW-NEXT:    notl %eax
 ; AVX512BW-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
@@ -1429,8 +1427,8 @@ define i16 @test_v8i16(<8 x i16> %a0) {
 ; AVX512VL:       # %bb.0:
 ; AVX512VL-NEXT:    vpternlogq $15, %xmm0, %xmm0, %xmm0
 ; AVX512VL-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX512VL-NEXT:    vpternlogq $15, %xmm0, %xmm0, %xmm0
 ; AVX512VL-NEXT:    vmovd %xmm0, %eax
+; AVX512VL-NEXT:    notl %eax
 ; AVX512VL-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX512VL-NEXT:    retq
   %1 = call i16 @llvm.experimental.vector.reduce.umax.i16.v8i16(<8 x i16> %a0)
@@ -1459,8 +1457,8 @@ define i16 @test_v16i16(<16 x i16> %a0) {
 ; SSE2-NEXT:    psrld $16, %xmm1
 ; SSE2-NEXT:    pxor %xmm2, %xmm1
 ; SSE2-NEXT:    pmaxsw %xmm0, %xmm1
-; SSE2-NEXT:    pxor %xmm2, %xmm1
 ; SSE2-NEXT:    movd %xmm1, %eax
+; SSE2-NEXT:    xorl $32768, %eax # imm = 0x8000
 ; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
 ; SSE2-NEXT:    retq
 ;
@@ -1468,10 +1466,10 @@ define i16 @test_v16i16(<16 x i16> %a0) {
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    pmaxuw %xmm1, %xmm0
 ; SSE41-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE41-NEXT:    pxor %xmm1, %xmm0
-; SSE41-NEXT:    phminposuw %xmm0, %xmm0
-; SSE41-NEXT:    pxor %xmm1, %xmm0
+; SSE41-NEXT:    pxor %xmm0, %xmm1
+; SSE41-NEXT:    phminposuw %xmm1, %xmm0
 ; SSE41-NEXT:    movd %xmm0, %eax
+; SSE41-NEXT:    notl %eax
 ; SSE41-NEXT:    # kill: def $ax killed $ax killed $eax
 ; SSE41-NEXT:    retq
 ;
@@ -1482,8 +1480,8 @@ define i16 @test_v16i16(<16 x i16> %a0) {
 ; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
 ; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovd %xmm0, %eax
+; AVX1-NEXT:    notl %eax
 ; AVX1-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
@@ -1495,8 +1493,8 @@ define i16 @test_v16i16(<16 x i16> %a0) {
 ; AVX2-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
 ; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
+; AVX2-NEXT:    notl %eax
 ; AVX2-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
@@ -1507,8 +1505,8 @@ define i16 @test_v16i16(<16 x i16> %a0) {
 ; AVX512BW-NEXT:    vpmaxuw %xmm1, %xmm0, %xmm0
 ; AVX512BW-NEXT:    vpternlogq $15, %zmm0, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX512BW-NEXT:    vpternlogq $15, %zmm0, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vmovd %xmm0, %eax
+; AVX512BW-NEXT:    notl %eax
 ; AVX512BW-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
@@ -1519,8 +1517,8 @@ define i16 @test_v16i16(<16 x i16> %a0) {
 ; AVX512VL-NEXT:    vpmaxuw %xmm1, %xmm0, %xmm0
 ; AVX512VL-NEXT:    vpternlogq $15, %xmm0, %xmm0, %xmm0
 ; AVX512VL-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX512VL-NEXT:    vpternlogq $15, %xmm0, %xmm0, %xmm0
 ; AVX512VL-NEXT:    vmovd %xmm0, %eax
+; AVX512VL-NEXT:    notl %eax
 ; AVX512VL-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX512VL-NEXT:    vzeroupper
 ; AVX512VL-NEXT:    retq
@@ -1554,8 +1552,8 @@ define i16 @test_v32i16(<32 x i16> %a0) {
 ; SSE2-NEXT:    psrld $16, %xmm1
 ; SSE2-NEXT:    pxor %xmm4, %xmm1
 ; SSE2-NEXT:    pmaxsw %xmm0, %xmm1
-; SSE2-NEXT:    pxor %xmm4, %xmm1
 ; SSE2-NEXT:    movd %xmm1, %eax
+; SSE2-NEXT:    xorl $32768, %eax # imm = 0x8000
 ; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
 ; SSE2-NEXT:    retq
 ;
@@ -1565,10 +1563,10 @@ define i16 @test_v32i16(<32 x i16> %a0) {
 ; SSE41-NEXT:    pmaxuw %xmm2, %xmm0
 ; SSE41-NEXT:    pmaxuw %xmm1, %xmm0
 ; SSE41-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE41-NEXT:    pxor %xmm1, %xmm0
-; SSE41-NEXT:    phminposuw %xmm0, %xmm0
-; SSE41-NEXT:    pxor %xmm1, %xmm0
+; SSE41-NEXT:    pxor %xmm0, %xmm1
+; SSE41-NEXT:    phminposuw %xmm1, %xmm0
 ; SSE41-NEXT:    movd %xmm0, %eax
+; SSE41-NEXT:    notl %eax
 ; SSE41-NEXT:    # kill: def $ax killed $ax killed $eax
 ; SSE41-NEXT:    retq
 ;
@@ -1582,8 +1580,8 @@ define i16 @test_v32i16(<32 x i16> %a0) {
 ; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
 ; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovd %xmm0, %eax
+; AVX1-NEXT:    notl %eax
 ; AVX1-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
@@ -1596,8 +1594,8 @@ define i16 @test_v32i16(<32 x i16> %a0) {
 ; AVX2-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
 ; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
+; AVX2-NEXT:    notl %eax
 ; AVX2-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
@@ -1610,8 +1608,8 @@ define i16 @test_v32i16(<32 x i16> %a0) {
 ; AVX512BW-NEXT:    vpmaxuw %xmm1, %xmm0, %xmm0
 ; AVX512BW-NEXT:    vpternlogq $15, %zmm0, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX512BW-NEXT:    vpternlogq $15, %zmm0, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vmovd %xmm0, %eax
+; AVX512BW-NEXT:    notl %eax
 ; AVX512BW-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
@@ -1624,8 +1622,8 @@ define i16 @test_v32i16(<32 x i16> %a0) {
 ; AVX512VL-NEXT:    vpmaxuw %xmm1, %xmm0, %xmm0
 ; AVX512VL-NEXT:    vpternlogq $15, %xmm0, %xmm0, %xmm0
 ; AVX512VL-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX512VL-NEXT:    vpternlogq $15, %xmm0, %xmm0, %xmm0
 ; AVX512VL-NEXT:    vmovd %xmm0, %eax
+; AVX512VL-NEXT:    notl %eax
 ; AVX512VL-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX512VL-NEXT:    vzeroupper
 ; AVX512VL-NEXT:    retq
@@ -1667,8 +1665,8 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 ; SSE2-NEXT:    psrld $16, %xmm0
 ; SSE2-NEXT:    pxor %xmm8, %xmm0
 ; SSE2-NEXT:    pmaxsw %xmm1, %xmm0
-; SSE2-NEXT:    pxor %xmm8, %xmm0
 ; SSE2-NEXT:    movd %xmm0, %eax
+; SSE2-NEXT:    xorl $32768, %eax # imm = 0x8000
 ; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
 ; SSE2-NEXT:    retq
 ;
@@ -1682,10 +1680,10 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 ; SSE41-NEXT:    pmaxuw %xmm2, %xmm0
 ; SSE41-NEXT:    pmaxuw %xmm1, %xmm0
 ; SSE41-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE41-NEXT:    pxor %xmm1, %xmm0
-; SSE41-NEXT:    phminposuw %xmm0, %xmm0
-; SSE41-NEXT:    pxor %xmm1, %xmm0
+; SSE41-NEXT:    pxor %xmm0, %xmm1
+; SSE41-NEXT:    phminposuw %xmm1, %xmm0
 ; SSE41-NEXT:    movd %xmm0, %eax
+; SSE41-NEXT:    notl %eax
 ; SSE41-NEXT:    # kill: def $ax killed $ax killed $eax
 ; SSE41-NEXT:    retq
 ;
@@ -1705,8 +1703,8 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 ; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
 ; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovd %xmm0, %eax
+; AVX1-NEXT:    notl %eax
 ; AVX1-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
@@ -1721,8 +1719,8 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 ; AVX2-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
 ; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
+; AVX2-NEXT:    notl %eax
 ; AVX2-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
@@ -1736,8 +1734,8 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 ; AVX512BW-NEXT:    vpmaxuw %xmm1, %xmm0, %xmm0
 ; AVX512BW-NEXT:    vpternlogq $15, %zmm0, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX512BW-NEXT:    vpternlogq $15, %zmm0, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vmovd %xmm0, %eax
+; AVX512BW-NEXT:    notl %eax
 ; AVX512BW-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
@@ -1751,8 +1749,8 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 ; AVX512VL-NEXT:    vpmaxuw %xmm1, %xmm0, %xmm0
 ; AVX512VL-NEXT:    vpternlogq $15, %xmm0, %xmm0, %xmm0
 ; AVX512VL-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX512VL-NEXT:    vpternlogq $15, %xmm0, %xmm0, %xmm0
 ; AVX512VL-NEXT:    vmovd %xmm0, %eax
+; AVX512VL-NEXT:    notl %eax
 ; AVX512VL-NEXT:    # kill: def $ax killed $ax killed $eax
 ; AVX512VL-NEXT:    vzeroupper
 ; AVX512VL-NEXT:    retq
@@ -1803,12 +1801,11 @@ define i8 @test_v2i8(<2 x i8> %a0) {
 ; SSE41-NEXT:    por %xmm3, %xmm4
 ; SSE41-NEXT:    por %xmm2, %xmm3
 ; SSE41-NEXT:    movdqa %xmm4, %xmm0
-; SSE41-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm0[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm4, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pand %xmm5, %xmm3
-; SSE41-NEXT:    por %xmm3, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm3, %xmm0
+; SSE41-NEXT:    pcmpgtd %xmm3, %xmm4
+; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[0,0,2,2]
+; SSE41-NEXT:    pand %xmm3, %xmm0
+; SSE41-NEXT:    por %xmm4, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm2
 ; SSE41-NEXT:    pextrb $0, %xmm2, %eax
 ; SSE41-NEXT:    # kill: def $al killed $al killed $eax
@@ -2052,13 +2049,13 @@ define i8 @test_v16i8(<16 x i8> %a0) {
 ; SSE41-LABEL: test_v16i8:
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE41-NEXT:    pxor %xmm1, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm2
-; SSE41-NEXT:    psrlw $8, %xmm2
-; SSE41-NEXT:    pminub %xmm0, %xmm2
-; SSE41-NEXT:    phminposuw %xmm2, %xmm0
-; SSE41-NEXT:    pxor %xmm1, %xmm0
+; SSE41-NEXT:    pxor %xmm0, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    psrlw $8, %xmm0
+; SSE41-NEXT:    pminub %xmm1, %xmm0
+; SSE41-NEXT:    phminposuw %xmm0, %xmm0
 ; SSE41-NEXT:    pextrb $0, %xmm0, %eax
+; SSE41-NEXT:    notb %al
 ; SSE41-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE41-NEXT:    retq
 ;
@@ -2066,11 +2063,11 @@ define i8 @test_v16i8(<16 x i8> %a0) {
 ; AVX:       # %bb.0:
 ; AVX-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
 ; AVX-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX-NEXT:    vpsrlw $8, %xmm0, %xmm2
-; AVX-NEXT:    vpminub %xmm2, %xmm0, %xmm0
+; AVX-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; AVX-NEXT:    vpminub %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    vpextrb $0, %xmm0, %eax
+; AVX-NEXT:    notb %al
 ; AVX-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX-NEXT:    retq
 ;
@@ -2081,8 +2078,8 @@ define i8 @test_v16i8(<16 x i8> %a0) {
 ; AVX512BW-NEXT:    vpsrlw $8, %xmm0, %xmm1
 ; AVX512BW-NEXT:    vpminub %xmm1, %xmm0, %xmm0
 ; AVX512BW-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX512BW-NEXT:    vpternlogq $15, %zmm0, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vpextrb $0, %xmm0, %eax
+; AVX512BW-NEXT:    notb %al
 ; AVX512BW-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
@@ -2093,8 +2090,8 @@ define i8 @test_v16i8(<16 x i8> %a0) {
 ; AVX512VL-NEXT:    vpsrlw $8, %xmm0, %xmm1
 ; AVX512VL-NEXT:    vpminub %xmm1, %xmm0, %xmm0
 ; AVX512VL-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX512VL-NEXT:    vpternlogq $15, %xmm0, %xmm0, %xmm0
 ; AVX512VL-NEXT:    vpextrb $0, %xmm0, %eax
+; AVX512VL-NEXT:    notb %al
 ; AVX512VL-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX512VL-NEXT:    retq
   %1 = call i8 @llvm.experimental.vector.reduce.umax.i8.v16i8(<16 x i8> %a0)
@@ -2123,13 +2120,13 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    pmaxub %xmm1, %xmm0
 ; SSE41-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE41-NEXT:    pxor %xmm1, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm2
-; SSE41-NEXT:    psrlw $8, %xmm2
-; SSE41-NEXT:    pminub %xmm0, %xmm2
-; SSE41-NEXT:    phminposuw %xmm2, %xmm0
-; SSE41-NEXT:    pxor %xmm1, %xmm0
+; SSE41-NEXT:    pxor %xmm0, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    psrlw $8, %xmm0
+; SSE41-NEXT:    pminub %xmm1, %xmm0
+; SSE41-NEXT:    phminposuw %xmm0, %xmm0
 ; SSE41-NEXT:    pextrb $0, %xmm0, %eax
+; SSE41-NEXT:    notb %al
 ; SSE41-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE41-NEXT:    retq
 ;
@@ -2139,11 +2136,11 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; AVX1-NEXT:    vpmaxub %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
 ; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm2
-; AVX1-NEXT:    vpminub %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; AVX1-NEXT:    vpminub %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpextrb $0, %xmm0, %eax
+; AVX1-NEXT:    notb %al
 ; AVX1-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
@@ -2154,11 +2151,11 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; AVX2-NEXT:    vpmaxub %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
 ; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpsrlw $8, %xmm0, %xmm2
-; AVX2-NEXT:    vpminub %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; AVX2-NEXT:    vpminub %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpextrb $0, %xmm0, %eax
+; AVX2-NEXT:    notb %al
 ; AVX2-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
@@ -2171,8 +2168,8 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; AVX512BW-NEXT:    vpsrlw $8, %xmm0, %xmm1
 ; AVX512BW-NEXT:    vpminub %xmm1, %xmm0, %xmm0
 ; AVX512BW-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX512BW-NEXT:    vpternlogq $15, %zmm0, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vpextrb $0, %xmm0, %eax
+; AVX512BW-NEXT:    notb %al
 ; AVX512BW-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
@@ -2185,8 +2182,8 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; AVX512VL-NEXT:    vpsrlw $8, %xmm0, %xmm1
 ; AVX512VL-NEXT:    vpminub %xmm1, %xmm0, %xmm0
 ; AVX512VL-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX512VL-NEXT:    vpternlogq $15, %xmm0, %xmm0, %xmm0
 ; AVX512VL-NEXT:    vpextrb $0, %xmm0, %eax
+; AVX512VL-NEXT:    notb %al
 ; AVX512VL-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX512VL-NEXT:    vzeroupper
 ; AVX512VL-NEXT:    retq
@@ -2220,13 +2217,13 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; SSE41-NEXT:    pmaxub %xmm2, %xmm0
 ; SSE41-NEXT:    pmaxub %xmm1, %xmm0
 ; SSE41-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE41-NEXT:    pxor %xmm1, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm2
-; SSE41-NEXT:    psrlw $8, %xmm2
-; SSE41-NEXT:    pminub %xmm0, %xmm2
-; SSE41-NEXT:    phminposuw %xmm2, %xmm0
-; SSE41-NEXT:    pxor %xmm1, %xmm0
+; SSE41-NEXT:    pxor %xmm0, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    psrlw $8, %xmm0
+; SSE41-NEXT:    pminub %xmm1, %xmm0
+; SSE41-NEXT:    phminposuw %xmm0, %xmm0
 ; SSE41-NEXT:    pextrb $0, %xmm0, %eax
+; SSE41-NEXT:    notb %al
 ; SSE41-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE41-NEXT:    retq
 ;
@@ -2239,11 +2236,11 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; AVX1-NEXT:    vpmaxub %xmm2, %xmm0, %xmm0
 ; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
 ; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm2
-; AVX1-NEXT:    vpminub %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; AVX1-NEXT:    vpminub %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpextrb $0, %xmm0, %eax
+; AVX1-NEXT:    notb %al
 ; AVX1-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
@@ -2255,11 +2252,11 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; AVX2-NEXT:    vpmaxub %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
 ; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpsrlw $8, %xmm0, %xmm2
-; AVX2-NEXT:    vpminub %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; AVX2-NEXT:    vpminub %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpextrb $0, %xmm0, %eax
+; AVX2-NEXT:    notb %al
 ; AVX2-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
@@ -2274,8 +2271,8 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; AVX512BW-NEXT:    vpsrlw $8, %xmm0, %xmm1
 ; AVX512BW-NEXT:    vpminub %xmm1, %xmm0, %xmm0
 ; AVX512BW-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX512BW-NEXT:    vpternlogq $15, %zmm0, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vpextrb $0, %xmm0, %eax
+; AVX512BW-NEXT:    notb %al
 ; AVX512BW-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
@@ -2290,8 +2287,8 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; AVX512VL-NEXT:    vpsrlw $8, %xmm0, %xmm1
 ; AVX512VL-NEXT:    vpminub %xmm1, %xmm0, %xmm0
 ; AVX512VL-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX512VL-NEXT:    vpternlogq $15, %xmm0, %xmm0, %xmm0
 ; AVX512VL-NEXT:    vpextrb $0, %xmm0, %eax
+; AVX512VL-NEXT:    notb %al
 ; AVX512VL-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX512VL-NEXT:    vzeroupper
 ; AVX512VL-NEXT:    retq
@@ -2333,13 +2330,13 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; SSE41-NEXT:    pmaxub %xmm2, %xmm0
 ; SSE41-NEXT:    pmaxub %xmm1, %xmm0
 ; SSE41-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE41-NEXT:    pxor %xmm1, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm2
-; SSE41-NEXT:    psrlw $8, %xmm2
-; SSE41-NEXT:    pminub %xmm0, %xmm2
-; SSE41-NEXT:    phminposuw %xmm2, %xmm0
-; SSE41-NEXT:    pxor %xmm1, %xmm0
+; SSE41-NEXT:    pxor %xmm0, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    psrlw $8, %xmm0
+; SSE41-NEXT:    pminub %xmm1, %xmm0
+; SSE41-NEXT:    phminposuw %xmm0, %xmm0
 ; SSE41-NEXT:    pextrb $0, %xmm0, %eax
+; SSE41-NEXT:    notb %al
 ; SSE41-NEXT:    # kill: def $al killed $al killed $eax
 ; SSE41-NEXT:    retq
 ;
@@ -2358,11 +2355,11 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; AVX1-NEXT:    vpmaxub %xmm4, %xmm0, %xmm0
 ; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
 ; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm2
-; AVX1-NEXT:    vpminub %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; AVX1-NEXT:    vpminub %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpextrb $0, %xmm0, %eax
+; AVX1-NEXT:    notb %al
 ; AVX1-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
@@ -2376,11 +2373,11 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; AVX2-NEXT:    vpmaxub %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
 ; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpsrlw $8, %xmm0, %xmm2
-; AVX2-NEXT:    vpminub %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    vpsrlw $8, %xmm0, %xmm1
+; AVX2-NEXT:    vpminub %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpextrb $0, %xmm0, %eax
+; AVX2-NEXT:    notb %al
 ; AVX2-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
@@ -2396,8 +2393,8 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; AVX512BW-NEXT:    vpsrlw $8, %xmm0, %xmm1
 ; AVX512BW-NEXT:    vpminub %xmm1, %xmm0, %xmm0
 ; AVX512BW-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX512BW-NEXT:    vpternlogq $15, %zmm0, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vpextrb $0, %xmm0, %eax
+; AVX512BW-NEXT:    notb %al
 ; AVX512BW-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
@@ -2413,8 +2410,8 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; AVX512VL-NEXT:    vpsrlw $8, %xmm0, %xmm1
 ; AVX512VL-NEXT:    vpminub %xmm1, %xmm0, %xmm0
 ; AVX512VL-NEXT:    vphminposuw %xmm0, %xmm0
-; AVX512VL-NEXT:    vpternlogq $15, %xmm0, %xmm0, %xmm0
 ; AVX512VL-NEXT:    vpextrb $0, %xmm0, %eax
+; AVX512VL-NEXT:    notb %al
 ; AVX512VL-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX512VL-NEXT:    vzeroupper
 ; AVX512VL-NEXT:    retq
diff --git a/test/CodeGen/X86/vector-reduce-umin-widen.ll b/test/CodeGen/X86/vector-reduce-umin-widen.ll
index 51d2a25130d0a39cf0f69b797800d55317baef0c..9fac3bb10240dada3350cd3eb604b35010901b16 100644
--- a/test/CodeGen/X86/vector-reduce-umin-widen.ll
+++ b/test/CodeGen/X86/vector-reduce-umin-widen.ll
@@ -1165,8 +1165,8 @@ define i16 @test_v2i16(<2 x i16> %a0) {
 ; SSE2-NEXT:    pxor %xmm2, %xmm0
 ; SSE2-NEXT:    pxor %xmm2, %xmm1
 ; SSE2-NEXT:    pminsw %xmm0, %xmm1
-; SSE2-NEXT:    pxor %xmm2, %xmm1
 ; SSE2-NEXT:    movd %xmm1, %eax
+; SSE2-NEXT:    xorl $32768, %eax # imm = 0x8000
 ; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
 ; SSE2-NEXT:    retq
 ;
@@ -1211,8 +1211,8 @@ define i16 @test_v4i16(<4 x i16> %a0) {
 ; SSE2-NEXT:    psrld $16, %xmm0
 ; SSE2-NEXT:    pxor %xmm2, %xmm0
 ; SSE2-NEXT:    pminsw %xmm1, %xmm0
-; SSE2-NEXT:    pxor %xmm2, %xmm0
 ; SSE2-NEXT:    movd %xmm0, %eax
+; SSE2-NEXT:    xorl $32768, %eax # imm = 0x8000
 ; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
 ; SSE2-NEXT:    retq
 ;
@@ -1268,8 +1268,8 @@ define i16 @test_v8i16(<8 x i16> %a0) {
 ; SSE2-NEXT:    psrld $16, %xmm1
 ; SSE2-NEXT:    pxor %xmm2, %xmm1
 ; SSE2-NEXT:    pminsw %xmm0, %xmm1
-; SSE2-NEXT:    pxor %xmm2, %xmm1
 ; SSE2-NEXT:    movd %xmm1, %eax
+; SSE2-NEXT:    xorl $32768, %eax # imm = 0x8000
 ; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
 ; SSE2-NEXT:    retq
 ;
@@ -1319,8 +1319,8 @@ define i16 @test_v16i16(<16 x i16> %a0) {
 ; SSE2-NEXT:    psrld $16, %xmm1
 ; SSE2-NEXT:    pxor %xmm2, %xmm1
 ; SSE2-NEXT:    pminsw %xmm0, %xmm1
-; SSE2-NEXT:    pxor %xmm2, %xmm1
 ; SSE2-NEXT:    movd %xmm1, %eax
+; SSE2-NEXT:    xorl $32768, %eax # imm = 0x8000
 ; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
 ; SSE2-NEXT:    retq
 ;
@@ -1391,8 +1391,8 @@ define i16 @test_v32i16(<32 x i16> %a0) {
 ; SSE2-NEXT:    psrld $16, %xmm1
 ; SSE2-NEXT:    pxor %xmm4, %xmm1
 ; SSE2-NEXT:    pminsw %xmm0, %xmm1
-; SSE2-NEXT:    pxor %xmm4, %xmm1
 ; SSE2-NEXT:    movd %xmm1, %eax
+; SSE2-NEXT:    xorl $32768, %eax # imm = 0x8000
 ; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
 ; SSE2-NEXT:    retq
 ;
@@ -1479,8 +1479,8 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 ; SSE2-NEXT:    psrld $16, %xmm0
 ; SSE2-NEXT:    pxor %xmm8, %xmm0
 ; SSE2-NEXT:    pminsw %xmm1, %xmm0
-; SSE2-NEXT:    pxor %xmm8, %xmm0
 ; SSE2-NEXT:    movd %xmm0, %eax
+; SSE2-NEXT:    xorl $32768, %eax # imm = 0x8000
 ; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
 ; SSE2-NEXT:    retq
 ;
diff --git a/test/CodeGen/X86/vector-reduce-umin.ll b/test/CodeGen/X86/vector-reduce-umin.ll
index a2116377563fb9a5f4007ae834e495f93c56c09f..e3adb9f4aa12eba6879ed9b6f1b6e4087a070f2a 100644
--- a/test/CodeGen/X86/vector-reduce-umin.ll
+++ b/test/CodeGen/X86/vector-reduce-umin.ll
@@ -754,17 +754,16 @@ define i32 @test_v2i32(<2 x i32> %a0) {
 ; SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm0[0,1],xmm1[2,3],xmm0[4,5],xmm1[6,7]
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
 ; SSE41-NEXT:    pmovzxdq {{.*#+}} xmm2 = xmm0[0],zero,xmm0[1],zero
-; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483648,2147483648]
-; SSE41-NEXT:    movdqa %xmm1, %xmm3
-; SSE41-NEXT:    pxor %xmm0, %xmm3
-; SSE41-NEXT:    pxor %xmm2, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm3, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm3, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm5, %xmm0
-; SSE41-NEXT:    por %xmm4, %xmm0
+; SSE41-NEXT:    movdqa {{.*#+}} xmm3 = [2147483648,2147483648]
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    pxor %xmm3, %xmm0
+; SSE41-NEXT:    pxor %xmm2, %xmm3
+; SSE41-NEXT:    movdqa %xmm3, %xmm4
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm3
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[0,0,2,2]
+; SSE41-NEXT:    pand %xmm4, %xmm0
+; SSE41-NEXT:    por %xmm3, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm2
 ; SSE41-NEXT:    movd %xmm2, %eax
 ; SSE41-NEXT:    retq
@@ -1242,17 +1241,16 @@ define i16 @test_v2i16(<2 x i16> %a0) {
 ; SSE41-NEXT:    pblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3],xmm0[4],xmm1[5,6,7]
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
 ; SSE41-NEXT:    pmovzxwq {{.*#+}} xmm2 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
-; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483648,2147483648]
-; SSE41-NEXT:    movdqa %xmm1, %xmm3
-; SSE41-NEXT:    por %xmm0, %xmm3
-; SSE41-NEXT:    por %xmm2, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm3, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm3, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm5, %xmm0
-; SSE41-NEXT:    por %xmm4, %xmm0
+; SSE41-NEXT:    movdqa {{.*#+}} xmm3 = [2147483648,2147483648]
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    por %xmm3, %xmm0
+; SSE41-NEXT:    por %xmm2, %xmm3
+; SSE41-NEXT:    movdqa %xmm3, %xmm4
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm3
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[0,0,2,2]
+; SSE41-NEXT:    pand %xmm4, %xmm0
+; SSE41-NEXT:    por %xmm3, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm2
 ; SSE41-NEXT:    movd %xmm2, %eax
 ; SSE41-NEXT:    # kill: def $ax killed $ax killed $eax
@@ -1388,8 +1386,8 @@ define i16 @test_v8i16(<8 x i16> %a0) {
 ; SSE2-NEXT:    psrld $16, %xmm1
 ; SSE2-NEXT:    pxor %xmm2, %xmm1
 ; SSE2-NEXT:    pminsw %xmm0, %xmm1
-; SSE2-NEXT:    pxor %xmm2, %xmm1
 ; SSE2-NEXT:    movd %xmm1, %eax
+; SSE2-NEXT:    xorl $32768, %eax # imm = 0x8000
 ; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
 ; SSE2-NEXT:    retq
 ;
@@ -1439,8 +1437,8 @@ define i16 @test_v16i16(<16 x i16> %a0) {
 ; SSE2-NEXT:    psrld $16, %xmm1
 ; SSE2-NEXT:    pxor %xmm2, %xmm1
 ; SSE2-NEXT:    pminsw %xmm0, %xmm1
-; SSE2-NEXT:    pxor %xmm2, %xmm1
 ; SSE2-NEXT:    movd %xmm1, %eax
+; SSE2-NEXT:    xorl $32768, %eax # imm = 0x8000
 ; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
 ; SSE2-NEXT:    retq
 ;
@@ -1511,8 +1509,8 @@ define i16 @test_v32i16(<32 x i16> %a0) {
 ; SSE2-NEXT:    psrld $16, %xmm1
 ; SSE2-NEXT:    pxor %xmm4, %xmm1
 ; SSE2-NEXT:    pminsw %xmm0, %xmm1
-; SSE2-NEXT:    pxor %xmm4, %xmm1
 ; SSE2-NEXT:    movd %xmm1, %eax
+; SSE2-NEXT:    xorl $32768, %eax # imm = 0x8000
 ; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
 ; SSE2-NEXT:    retq
 ;
@@ -1599,8 +1597,8 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 ; SSE2-NEXT:    psrld $16, %xmm0
 ; SSE2-NEXT:    pxor %xmm8, %xmm0
 ; SSE2-NEXT:    pminsw %xmm1, %xmm0
-; SSE2-NEXT:    pxor %xmm8, %xmm0
 ; SSE2-NEXT:    movd %xmm0, %eax
+; SSE2-NEXT:    xorl $32768, %eax # imm = 0x8000
 ; SSE2-NEXT:    # kill: def $ax killed $ax killed $eax
 ; SSE2-NEXT:    retq
 ;
@@ -1704,17 +1702,16 @@ define i8 @test_v2i8(<2 x i8> %a0) {
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
 ; SSE41-NEXT:    pand {{.*}}(%rip), %xmm1
 ; SSE41-NEXT:    pmovzxbq {{.*#+}} xmm2 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero
-; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483648,2147483648]
-; SSE41-NEXT:    movdqa %xmm1, %xmm3
-; SSE41-NEXT:    por %xmm0, %xmm3
-; SSE41-NEXT:    por %xmm2, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm3, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm3, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm5, %xmm0
-; SSE41-NEXT:    por %xmm4, %xmm0
+; SSE41-NEXT:    movdqa {{.*#+}} xmm3 = [2147483648,2147483648]
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    por %xmm3, %xmm0
+; SSE41-NEXT:    por %xmm2, %xmm3
+; SSE41-NEXT:    movdqa %xmm3, %xmm4
+; SSE41-NEXT:    pcmpeqd %xmm0, %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm3
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[0,0,2,2]
+; SSE41-NEXT:    pand %xmm4, %xmm0
+; SSE41-NEXT:    por %xmm3, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm2
 ; SSE41-NEXT:    pextrb $0, %xmm2, %eax
 ; SSE41-NEXT:    # kill: def $al killed $al killed $eax
diff --git a/test/CodeGen/X86/vector-reduce-xor-widen.ll b/test/CodeGen/X86/vector-reduce-xor-widen.ll
index 4e17200b3332eb3ed196cc5f8d875f49b3a057ad..62bbba55f5ca16b0e64624a2eb4aa2fe50293846 100644
--- a/test/CodeGen/X86/vector-reduce-xor-widen.ll
+++ b/test/CodeGen/X86/vector-reduce-xor-widen.ll
@@ -47,9 +47,9 @@ define i64 @test_v4i64(<4 x i64> %a0) {
 ; AVX1-LABEL: test_v4i64:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vxorps %ymm1, %ymm0, %ymm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX1-NEXT:    vxorps %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovq %xmm0, %rax
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
@@ -57,7 +57,7 @@ define i64 @test_v4i64(<4 x i64> %a0) {
 ; AVX2-LABEL: test_v4i64:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpxor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovq %xmm0, %rax
@@ -67,7 +67,7 @@ define i64 @test_v4i64(<4 x i64> %a0) {
 ; AVX512-LABEL: test_v4i64:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vpxor %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovq %xmm0, %rax
@@ -92,7 +92,7 @@ define i64 @test_v8i64(<8 x i64> %a0) {
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vxorps %ymm1, %ymm0, %ymm0
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vxorps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vxorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; AVX1-NEXT:    vxorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovq %xmm0, %rax
@@ -103,7 +103,7 @@ define i64 @test_v8i64(<8 x i64> %a0) {
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpxor %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpxor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovq %xmm0, %rax
@@ -115,7 +115,7 @@ define i64 @test_v8i64(<8 x i64> %a0) {
 ; AVX512-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512-NEXT:    vpxorq %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vpxorq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovq %xmm0, %rax
@@ -146,7 +146,7 @@ define i64 @test_v16i64(<16 x i64> %a0) {
 ; AVX1-NEXT:    vxorps %ymm1, %ymm2, %ymm1
 ; AVX1-NEXT:    vxorps %ymm1, %ymm0, %ymm0
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vxorps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vxorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; AVX1-NEXT:    vxorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovq %xmm0, %rax
@@ -159,7 +159,7 @@ define i64 @test_v16i64(<16 x i64> %a0) {
 ; AVX2-NEXT:    vpxor %ymm1, %ymm2, %ymm1
 ; AVX2-NEXT:    vpxor %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpxor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovq %xmm0, %rax
@@ -172,7 +172,7 @@ define i64 @test_v16i64(<16 x i64> %a0) {
 ; AVX512-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512-NEXT:    vpxorq %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vpxorq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovq %xmm0, %rax
@@ -256,11 +256,11 @@ define i32 @test_v8i32(<8 x i32> %a0) {
 ; AVX1-LABEL: test_v8i32:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vxorps %ymm1, %ymm0, %ymm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX1-NEXT:    vxorps %ymm1, %ymm0, %ymm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX1-NEXT:    vxorps %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovd %xmm0, %eax
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
@@ -268,9 +268,9 @@ define i32 @test_v8i32(<8 x i32> %a0) {
 ; AVX2-LABEL: test_v8i32:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpxor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX2-NEXT:    vpxor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
@@ -280,9 +280,9 @@ define i32 @test_v8i32(<8 x i32> %a0) {
 ; AVX512-LABEL: test_v8i32:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vpxor %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpxor %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
@@ -309,9 +309,9 @@ define i32 @test_v16i32(<16 x i32> %a0) {
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vxorps %ymm1, %ymm0, %ymm0
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vxorps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vxorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX1-NEXT:    vxorps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vxorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX1-NEXT:    vxorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovd %xmm0, %eax
@@ -322,9 +322,9 @@ define i32 @test_v16i32(<16 x i32> %a0) {
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpxor %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpxor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX2-NEXT:    vpxor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
@@ -336,9 +336,9 @@ define i32 @test_v16i32(<16 x i32> %a0) {
 ; AVX512-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512-NEXT:    vpxord %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vpxord %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpxord %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
@@ -371,9 +371,9 @@ define i32 @test_v32i32(<32 x i32> %a0) {
 ; AVX1-NEXT:    vxorps %ymm1, %ymm2, %ymm1
 ; AVX1-NEXT:    vxorps %ymm1, %ymm0, %ymm0
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vxorps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vxorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX1-NEXT:    vxorps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vxorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX1-NEXT:    vxorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovd %xmm0, %eax
@@ -386,9 +386,9 @@ define i32 @test_v32i32(<32 x i32> %a0) {
 ; AVX2-NEXT:    vpxor %ymm1, %ymm2, %ymm1
 ; AVX2-NEXT:    vpxor %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpxor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX2-NEXT:    vpxor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
@@ -401,9 +401,9 @@ define i32 @test_v32i32(<32 x i32> %a0) {
 ; AVX512-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512-NEXT:    vpxord %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vpxord %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpxord %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
@@ -540,11 +540,11 @@ define i16 @test_v16i16(<16 x i16> %a0) {
 ; AVX1-LABEL: test_v16i16:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vxorps %ymm1, %ymm0, %ymm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX1-NEXT:    vxorps %ymm1, %ymm0, %ymm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX1-NEXT:    vxorps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovd %xmm0, %eax
@@ -555,11 +555,11 @@ define i16 @test_v16i16(<16 x i16> %a0) {
 ; AVX2-LABEL: test_v16i16:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpxor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX2-NEXT:    vpxor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX2-NEXT:    vpxor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
@@ -570,11 +570,11 @@ define i16 @test_v16i16(<16 x i16> %a0) {
 ; AVX512-LABEL: test_v16i16:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vpxor %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpxor %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX512-NEXT:    vpxor %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
@@ -606,11 +606,11 @@ define i16 @test_v32i16(<32 x i16> %a0) {
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vxorps %ymm1, %ymm0, %ymm0
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vxorps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vxorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX1-NEXT:    vxorps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vxorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX1-NEXT:    vxorps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vxorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovd %xmm0, %eax
@@ -622,11 +622,11 @@ define i16 @test_v32i16(<32 x i16> %a0) {
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpxor %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpxor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX2-NEXT:    vpxor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX2-NEXT:    vpxor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
@@ -639,11 +639,11 @@ define i16 @test_v32i16(<32 x i16> %a0) {
 ; AVX512-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512-NEXT:    vpxorq %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vpxorq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpxorq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX512-NEXT:    vpxorq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
@@ -681,11 +681,11 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 ; AVX1-NEXT:    vxorps %ymm1, %ymm2, %ymm1
 ; AVX1-NEXT:    vxorps %ymm1, %ymm0, %ymm0
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vxorps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vxorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX1-NEXT:    vxorps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vxorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX1-NEXT:    vxorps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vxorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovd %xmm0, %eax
@@ -699,11 +699,11 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 ; AVX2-NEXT:    vpxor %ymm1, %ymm2, %ymm1
 ; AVX2-NEXT:    vpxor %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpxor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX2-NEXT:    vpxor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX2-NEXT:    vpxor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
@@ -717,11 +717,11 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 ; AVX512-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512-NEXT:    vpxorq %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vpxorq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpxorq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX512-NEXT:    vpxorq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
@@ -980,13 +980,13 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; AVX1-LABEL: test_v32i8:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vxorps %ymm1, %ymm0, %ymm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX1-NEXT:    vxorps %ymm1, %ymm0, %ymm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX1-NEXT:    vxorps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX1-NEXT:    vxorps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm1
 ; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpextrb $0, %xmm0, %eax
@@ -997,13 +997,13 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; AVX2-LABEL: test_v32i8:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpxor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX2-NEXT:    vpxor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX2-NEXT:    vpxor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX2-NEXT:    vpxor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpsrlw $8, %xmm0, %xmm1
 ; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpextrb $0, %xmm0, %eax
@@ -1014,13 +1014,13 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; AVX512-LABEL: test_v32i8:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vpxor %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpxor %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX512-NEXT:    vpxor %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX512-NEXT:    vpxor %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrlw $8, %xmm0, %xmm1
 ; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpextrb $0, %xmm0, %eax
@@ -1074,13 +1074,13 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vxorps %ymm1, %ymm0, %ymm0
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vxorps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vxorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX1-NEXT:    vxorps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vxorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX1-NEXT:    vxorps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vxorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX1-NEXT:    vxorps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm1
 ; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpextrb $0, %xmm0, %eax
@@ -1092,13 +1092,13 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpxor %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpxor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX2-NEXT:    vpxor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX2-NEXT:    vpxor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX2-NEXT:    vpxor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpsrlw $8, %xmm0, %xmm1
 ; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpextrb $0, %xmm0, %eax
@@ -1111,13 +1111,13 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; AVX512-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512-NEXT:    vpxorq %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vpxorq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpxorq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX512-NEXT:    vpxorq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX512-NEXT:    vpxorq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrlw $8, %xmm0, %xmm1
 ; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpextrb $0, %xmm0, %eax
@@ -1181,13 +1181,13 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; AVX1-NEXT:    vxorps %ymm1, %ymm2, %ymm1
 ; AVX1-NEXT:    vxorps %ymm1, %ymm0, %ymm0
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vxorps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vxorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX1-NEXT:    vxorps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vxorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX1-NEXT:    vxorps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vxorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX1-NEXT:    vxorps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm1
 ; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpextrb $0, %xmm0, %eax
@@ -1201,13 +1201,13 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; AVX2-NEXT:    vpxor %ymm1, %ymm2, %ymm1
 ; AVX2-NEXT:    vpxor %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpxor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX2-NEXT:    vpxor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX2-NEXT:    vpxor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX2-NEXT:    vpxor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpsrlw $8, %xmm0, %xmm1
 ; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpextrb $0, %xmm0, %eax
@@ -1221,13 +1221,13 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; AVX512-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512-NEXT:    vpxorq %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vpxorq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpxorq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX512-NEXT:    vpxorq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX512-NEXT:    vpxorq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrlw $8, %xmm0, %xmm1
 ; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpextrb $0, %xmm0, %eax
diff --git a/test/CodeGen/X86/vector-reduce-xor.ll b/test/CodeGen/X86/vector-reduce-xor.ll
index d98ee3882152584412d7c86d07043117d1d2796d..870fed03e265651def0bd2d3c779357651c9f947 100644
--- a/test/CodeGen/X86/vector-reduce-xor.ll
+++ b/test/CodeGen/X86/vector-reduce-xor.ll
@@ -47,9 +47,9 @@ define i64 @test_v4i64(<4 x i64> %a0) {
 ; AVX1-LABEL: test_v4i64:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vxorps %ymm1, %ymm0, %ymm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX1-NEXT:    vxorps %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovq %xmm0, %rax
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
@@ -57,7 +57,7 @@ define i64 @test_v4i64(<4 x i64> %a0) {
 ; AVX2-LABEL: test_v4i64:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpxor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovq %xmm0, %rax
@@ -67,7 +67,7 @@ define i64 @test_v4i64(<4 x i64> %a0) {
 ; AVX512-LABEL: test_v4i64:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vpxor %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovq %xmm0, %rax
@@ -92,7 +92,7 @@ define i64 @test_v8i64(<8 x i64> %a0) {
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vxorps %ymm1, %ymm0, %ymm0
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vxorps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vxorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; AVX1-NEXT:    vxorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovq %xmm0, %rax
@@ -103,7 +103,7 @@ define i64 @test_v8i64(<8 x i64> %a0) {
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpxor %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpxor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovq %xmm0, %rax
@@ -115,7 +115,7 @@ define i64 @test_v8i64(<8 x i64> %a0) {
 ; AVX512-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512-NEXT:    vpxorq %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vpxorq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovq %xmm0, %rax
@@ -146,7 +146,7 @@ define i64 @test_v16i64(<16 x i64> %a0) {
 ; AVX1-NEXT:    vxorps %ymm1, %ymm2, %ymm1
 ; AVX1-NEXT:    vxorps %ymm1, %ymm0, %ymm0
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vxorps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vxorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; AVX1-NEXT:    vxorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovq %xmm0, %rax
@@ -159,7 +159,7 @@ define i64 @test_v16i64(<16 x i64> %a0) {
 ; AVX2-NEXT:    vpxor %ymm1, %ymm2, %ymm1
 ; AVX2-NEXT:    vpxor %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpxor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovq %xmm0, %rax
@@ -172,7 +172,7 @@ define i64 @test_v16i64(<16 x i64> %a0) {
 ; AVX512-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512-NEXT:    vpxorq %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vpxorq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
 ; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovq %xmm0, %rax
@@ -256,11 +256,11 @@ define i32 @test_v8i32(<8 x i32> %a0) {
 ; AVX1-LABEL: test_v8i32:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vxorps %ymm1, %ymm0, %ymm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX1-NEXT:    vxorps %ymm1, %ymm0, %ymm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX1-NEXT:    vxorps %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovd %xmm0, %eax
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
@@ -268,9 +268,9 @@ define i32 @test_v8i32(<8 x i32> %a0) {
 ; AVX2-LABEL: test_v8i32:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpxor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX2-NEXT:    vpxor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
@@ -280,9 +280,9 @@ define i32 @test_v8i32(<8 x i32> %a0) {
 ; AVX512-LABEL: test_v8i32:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vpxor %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpxor %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
@@ -309,9 +309,9 @@ define i32 @test_v16i32(<16 x i32> %a0) {
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vxorps %ymm1, %ymm0, %ymm0
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vxorps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vxorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX1-NEXT:    vxorps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vxorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX1-NEXT:    vxorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovd %xmm0, %eax
@@ -322,9 +322,9 @@ define i32 @test_v16i32(<16 x i32> %a0) {
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpxor %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpxor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX2-NEXT:    vpxor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
@@ -336,9 +336,9 @@ define i32 @test_v16i32(<16 x i32> %a0) {
 ; AVX512-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512-NEXT:    vpxord %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vpxord %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpxord %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
@@ -371,9 +371,9 @@ define i32 @test_v32i32(<32 x i32> %a0) {
 ; AVX1-NEXT:    vxorps %ymm1, %ymm2, %ymm1
 ; AVX1-NEXT:    vxorps %ymm1, %ymm0, %ymm0
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vxorps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vxorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX1-NEXT:    vxorps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vxorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX1-NEXT:    vxorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovd %xmm0, %eax
@@ -386,9 +386,9 @@ define i32 @test_v32i32(<32 x i32> %a0) {
 ; AVX2-NEXT:    vpxor %ymm1, %ymm2, %ymm1
 ; AVX2-NEXT:    vpxor %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpxor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX2-NEXT:    vpxor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
@@ -401,9 +401,9 @@ define i32 @test_v32i32(<32 x i32> %a0) {
 ; AVX512-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512-NEXT:    vpxord %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vpxord %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpxord %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
 ; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
@@ -538,11 +538,11 @@ define i16 @test_v16i16(<16 x i16> %a0) {
 ; AVX1-LABEL: test_v16i16:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vxorps %ymm1, %ymm0, %ymm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX1-NEXT:    vxorps %ymm1, %ymm0, %ymm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX1-NEXT:    vxorps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovd %xmm0, %eax
@@ -553,11 +553,11 @@ define i16 @test_v16i16(<16 x i16> %a0) {
 ; AVX2-LABEL: test_v16i16:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpxor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX2-NEXT:    vpxor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX2-NEXT:    vpxor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
@@ -568,11 +568,11 @@ define i16 @test_v16i16(<16 x i16> %a0) {
 ; AVX512-LABEL: test_v16i16:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vpxor %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpxor %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX512-NEXT:    vpxor %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
@@ -604,11 +604,11 @@ define i16 @test_v32i16(<32 x i16> %a0) {
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vxorps %ymm1, %ymm0, %ymm0
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vxorps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vxorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX1-NEXT:    vxorps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vxorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX1-NEXT:    vxorps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vxorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovd %xmm0, %eax
@@ -620,11 +620,11 @@ define i16 @test_v32i16(<32 x i16> %a0) {
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpxor %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpxor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX2-NEXT:    vpxor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX2-NEXT:    vpxor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
@@ -637,11 +637,11 @@ define i16 @test_v32i16(<32 x i16> %a0) {
 ; AVX512-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512-NEXT:    vpxorq %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vpxorq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpxorq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX512-NEXT:    vpxorq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
@@ -679,11 +679,11 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 ; AVX1-NEXT:    vxorps %ymm1, %ymm2, %ymm1
 ; AVX1-NEXT:    vxorps %ymm1, %ymm0, %ymm0
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vxorps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vxorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX1-NEXT:    vxorps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vxorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX1-NEXT:    vxorps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vxorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vmovd %xmm0, %eax
@@ -697,11 +697,11 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 ; AVX2-NEXT:    vpxor %ymm1, %ymm2, %ymm1
 ; AVX2-NEXT:    vpxor %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpxor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX2-NEXT:    vpxor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX2-NEXT:    vpxor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vmovd %xmm0, %eax
@@ -715,11 +715,11 @@ define i16 @test_v64i16(<64 x i16> %a0) {
 ; AVX512-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512-NEXT:    vpxorq %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vpxorq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpxorq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX512-NEXT:    vpxorq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
 ; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovd %xmm0, %eax
@@ -970,13 +970,13 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; AVX1-LABEL: test_v32i8:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vxorps %ymm1, %ymm0, %ymm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX1-NEXT:    vxorps %ymm1, %ymm0, %ymm0
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX1-NEXT:    vxorps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
+; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX1-NEXT:    vxorps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm1
 ; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpextrb $0, %xmm0, %eax
@@ -987,13 +987,13 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; AVX2-LABEL: test_v32i8:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpxor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX2-NEXT:    vpxor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX2-NEXT:    vpxor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX2-NEXT:    vpxor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpsrlw $8, %xmm0, %xmm1
 ; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpextrb $0, %xmm0, %eax
@@ -1004,13 +1004,13 @@ define i8 @test_v32i8(<32 x i8> %a0) {
 ; AVX512-LABEL: test_v32i8:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vpxor %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpxor %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX512-NEXT:    vpxor %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX512-NEXT:    vpxor %ymm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrlw $8, %xmm0, %xmm1
 ; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpextrb $0, %xmm0, %eax
@@ -1064,13 +1064,13 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vxorps %ymm1, %ymm0, %ymm0
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vxorps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vxorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX1-NEXT:    vxorps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vxorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX1-NEXT:    vxorps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vxorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX1-NEXT:    vxorps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm1
 ; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpextrb $0, %xmm0, %eax
@@ -1082,13 +1082,13 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpxor %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpxor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX2-NEXT:    vpxor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX2-NEXT:    vpxor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX2-NEXT:    vpxor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpsrlw $8, %xmm0, %xmm1
 ; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpextrb $0, %xmm0, %eax
@@ -1101,13 +1101,13 @@ define i8 @test_v64i8(<64 x i8> %a0) {
 ; AVX512-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512-NEXT:    vpxorq %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vpxorq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpxorq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX512-NEXT:    vpxorq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX512-NEXT:    vpxorq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrlw $8, %xmm0, %xmm1
 ; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpextrb $0, %xmm0, %eax
@@ -1171,13 +1171,13 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; AVX1-NEXT:    vxorps %ymm1, %ymm2, %ymm1
 ; AVX1-NEXT:    vxorps %ymm1, %ymm0, %ymm0
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vxorps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vxorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX1-NEXT:    vxorps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vxorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX1-NEXT:    vxorps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vxorps %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX1-NEXT:    vxorps %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrlw $8, %xmm0, %xmm1
 ; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpextrb $0, %xmm0, %eax
@@ -1191,13 +1191,13 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; AVX2-NEXT:    vpxor %ymm1, %ymm2, %ymm1
 ; AVX2-NEXT:    vpxor %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT:    vpxor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX2-NEXT:    vpxor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX2-NEXT:    vpxor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX2-NEXT:    vpxor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpsrlw $8, %xmm0, %xmm1
 ; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpextrb $0, %xmm0, %eax
@@ -1211,13 +1211,13 @@ define i8 @test_v128i8(<128 x i8> %a0) {
 ; AVX512-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512-NEXT:    vpxorq %zmm1, %zmm0, %zmm0
 ; AVX512-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT:    vpxorq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1]
-; AVX512-NEXT:    vpxorq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
-; AVX512-NEXT:    vpxorq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrld $16, %xmm0, %xmm1
-; AVX512-NEXT:    vpxorq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpsrlw $8, %xmm0, %xmm1
 ; AVX512-NEXT:    vpxor %xmm1, %xmm0, %xmm0
 ; AVX512-NEXT:    vpextrb $0, %xmm0, %eax
diff --git a/test/CodeGen/X86/vector-rotate-128.ll b/test/CodeGen/X86/vector-rotate-128.ll
index 6258229392c4e266f917982061656871e3b79dbd..4c528a55209832652e9d424fcd2335b3d2b6f1a8 100644
--- a/test/CodeGen/X86/vector-rotate-128.ll
+++ b/test/CodeGen/X86/vector-rotate-128.ll
@@ -538,50 +538,30 @@ define <16 x i8> @var_rotate_v16i8(<16 x i8> %a, <16 x i8> %b) nounwind {
 ;
 ; AVX512F-LABEL: var_rotate_v16i8:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpsrlw $4, %xmm0, %xmm2
-; AVX512F-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
-; AVX512F-NEXT:    vpsllw $4, %xmm0, %xmm3
-; AVX512F-NEXT:    vpand {{.*}}(%rip), %xmm3, %xmm3
-; AVX512F-NEXT:    vpor %xmm2, %xmm3, %xmm2
-; AVX512F-NEXT:    vpsllw $5, %xmm1, %xmm1
-; AVX512F-NEXT:    vpblendvb %xmm1, %xmm2, %xmm0, %xmm0
-; AVX512F-NEXT:    vpsrlw $6, %xmm0, %xmm2
-; AVX512F-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
-; AVX512F-NEXT:    vpsllw $2, %xmm0, %xmm3
-; AVX512F-NEXT:    vpand {{.*}}(%rip), %xmm3, %xmm3
-; AVX512F-NEXT:    vpor %xmm2, %xmm3, %xmm2
-; AVX512F-NEXT:    vpaddb %xmm1, %xmm1, %xmm1
-; AVX512F-NEXT:    vpblendvb %xmm1, %xmm2, %xmm0, %xmm0
-; AVX512F-NEXT:    vpaddb %xmm0, %xmm0, %xmm2
-; AVX512F-NEXT:    vpsrlw $7, %xmm0, %xmm3
-; AVX512F-NEXT:    vpand {{.*}}(%rip), %xmm3, %xmm3
-; AVX512F-NEXT:    vpor %xmm3, %xmm2, %xmm2
-; AVX512F-NEXT:    vpaddb %xmm1, %xmm1, %xmm1
-; AVX512F-NEXT:    vpblendvb %xmm1, %xmm2, %xmm0, %xmm0
+; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm2 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; AVX512F-NEXT:    vpsubb %xmm1, %xmm2, %xmm2
+; AVX512F-NEXT:    vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
+; AVX512F-NEXT:    vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
+; AVX512F-NEXT:    vpsllvd %zmm1, %zmm0, %zmm1
+; AVX512F-NEXT:    vpmovzxbd {{.*#+}} zmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero,xmm2[4],zero,zero,zero,xmm2[5],zero,zero,zero,xmm2[6],zero,zero,zero,xmm2[7],zero,zero,zero,xmm2[8],zero,zero,zero,xmm2[9],zero,zero,zero,xmm2[10],zero,zero,zero,xmm2[11],zero,zero,zero,xmm2[12],zero,zero,zero,xmm2[13],zero,zero,zero,xmm2[14],zero,zero,zero,xmm2[15],zero,zero,zero
+; AVX512F-NEXT:    vpsrlvd %zmm2, %zmm0, %zmm0
+; AVX512F-NEXT:    vpord %zmm0, %zmm1, %zmm0
+; AVX512F-NEXT:    vpmovdb %zmm0, %xmm0
+; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: var_rotate_v16i8:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpsrlw $4, %xmm0, %xmm2
-; AVX512VL-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
-; AVX512VL-NEXT:    vpsllw $4, %xmm0, %xmm3
-; AVX512VL-NEXT:    vpand {{.*}}(%rip), %xmm3, %xmm3
-; AVX512VL-NEXT:    vpor %xmm2, %xmm3, %xmm2
-; AVX512VL-NEXT:    vpsllw $5, %xmm1, %xmm1
-; AVX512VL-NEXT:    vpblendvb %xmm1, %xmm2, %xmm0, %xmm0
-; AVX512VL-NEXT:    vpsrlw $6, %xmm0, %xmm2
-; AVX512VL-NEXT:    vpand {{.*}}(%rip), %xmm2, %xmm2
-; AVX512VL-NEXT:    vpsllw $2, %xmm0, %xmm3
-; AVX512VL-NEXT:    vpand {{.*}}(%rip), %xmm3, %xmm3
-; AVX512VL-NEXT:    vpor %xmm2, %xmm3, %xmm2
-; AVX512VL-NEXT:    vpaddb %xmm1, %xmm1, %xmm1
-; AVX512VL-NEXT:    vpblendvb %xmm1, %xmm2, %xmm0, %xmm0
-; AVX512VL-NEXT:    vpaddb %xmm0, %xmm0, %xmm2
-; AVX512VL-NEXT:    vpsrlw $7, %xmm0, %xmm3
-; AVX512VL-NEXT:    vpand {{.*}}(%rip), %xmm3, %xmm3
-; AVX512VL-NEXT:    vpor %xmm3, %xmm2, %xmm2
-; AVX512VL-NEXT:    vpaddb %xmm1, %xmm1, %xmm1
-; AVX512VL-NEXT:    vpblendvb %xmm1, %xmm2, %xmm0, %xmm0
+; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm2 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; AVX512VL-NEXT:    vpsubb %xmm1, %xmm2, %xmm2
+; AVX512VL-NEXT:    vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
+; AVX512VL-NEXT:    vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
+; AVX512VL-NEXT:    vpsllvd %zmm1, %zmm0, %zmm1
+; AVX512VL-NEXT:    vpmovzxbd {{.*#+}} zmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero,xmm2[4],zero,zero,zero,xmm2[5],zero,zero,zero,xmm2[6],zero,zero,zero,xmm2[7],zero,zero,zero,xmm2[8],zero,zero,zero,xmm2[9],zero,zero,zero,xmm2[10],zero,zero,zero,xmm2[11],zero,zero,zero,xmm2[12],zero,zero,zero,xmm2[13],zero,zero,zero,xmm2[14],zero,zero,zero,xmm2[15],zero,zero,zero
+; AVX512VL-NEXT:    vpsrlvd %zmm2, %zmm0, %zmm0
+; AVX512VL-NEXT:    vpord %zmm0, %zmm1, %zmm0
+; AVX512VL-NEXT:    vpmovdb %zmm0, %xmm0
+; AVX512VL-NEXT:    vzeroupper
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: var_rotate_v16i8:
@@ -758,18 +738,16 @@ define <2 x i64> @splatvar_rotate_v2i64(<2 x i64> %a, <2 x i64> %b) nounwind {
 define <4 x i32> @splatvar_rotate_v4i32(<4 x i32> %a, <4 x i32> %b) nounwind {
 ; SSE2-LABEL: splatvar_rotate_v4i32:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
-; SSE2-NEXT:    pand {{.*}}(%rip), %xmm1
-; SSE2-NEXT:    xorps %xmm2, %xmm2
-; SSE2-NEXT:    xorps %xmm3, %xmm3
-; SSE2-NEXT:    movss {{.*#+}} xmm3 = xmm1[0],xmm3[1,2,3]
-; SSE2-NEXT:    movdqa %xmm0, %xmm4
-; SSE2-NEXT:    pslld %xmm3, %xmm4
-; SSE2-NEXT:    movdqa {{.*#+}} xmm3 = [32,32,32,32]
-; SSE2-NEXT:    psubd %xmm1, %xmm3
-; SSE2-NEXT:    movss {{.*#+}} xmm2 = xmm3[0],xmm2[1,2,3]
-; SSE2-NEXT:    psrld %xmm2, %xmm0
-; SSE2-NEXT:    por %xmm4, %xmm0
+; SSE2-NEXT:    movd %xmm1, %eax
+; SSE2-NEXT:    andl $31, %eax
+; SSE2-NEXT:    movd %eax, %xmm1
+; SSE2-NEXT:    movdqa %xmm0, %xmm2
+; SSE2-NEXT:    pslld %xmm1, %xmm2
+; SSE2-NEXT:    movl $32, %ecx
+; SSE2-NEXT:    subl %eax, %ecx
+; SSE2-NEXT:    movd %ecx, %xmm1
+; SSE2-NEXT:    psrld %xmm1, %xmm0
+; SSE2-NEXT:    por %xmm2, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSE41-LABEL: splatvar_rotate_v4i32:
@@ -857,18 +835,16 @@ define <4 x i32> @splatvar_rotate_v4i32(<4 x i32> %a, <4 x i32> %b) nounwind {
 ;
 ; X32-SSE-LABEL: splatvar_rotate_v4i32:
 ; X32-SSE:       # %bb.0:
-; X32-SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
-; X32-SSE-NEXT:    pand {{\.LCPI.*}}, %xmm1
-; X32-SSE-NEXT:    xorps %xmm2, %xmm2
-; X32-SSE-NEXT:    xorps %xmm3, %xmm3
-; X32-SSE-NEXT:    movss {{.*#+}} xmm3 = xmm1[0],xmm3[1,2,3]
-; X32-SSE-NEXT:    movdqa %xmm0, %xmm4
-; X32-SSE-NEXT:    pslld %xmm3, %xmm4
-; X32-SSE-NEXT:    movdqa {{.*#+}} xmm3 = [32,32,32,32]
-; X32-SSE-NEXT:    psubd %xmm1, %xmm3
-; X32-SSE-NEXT:    movss {{.*#+}} xmm2 = xmm3[0],xmm2[1,2,3]
-; X32-SSE-NEXT:    psrld %xmm2, %xmm0
-; X32-SSE-NEXT:    por %xmm4, %xmm0
+; X32-SSE-NEXT:    movd %xmm1, %eax
+; X32-SSE-NEXT:    andl $31, %eax
+; X32-SSE-NEXT:    movd %eax, %xmm1
+; X32-SSE-NEXT:    movdqa %xmm0, %xmm2
+; X32-SSE-NEXT:    pslld %xmm1, %xmm2
+; X32-SSE-NEXT:    movl $32, %ecx
+; X32-SSE-NEXT:    subl %eax, %ecx
+; X32-SSE-NEXT:    movd %ecx, %xmm1
+; X32-SSE-NEXT:    psrld %xmm1, %xmm0
+; X32-SSE-NEXT:    por %xmm2, %xmm0
 ; X32-SSE-NEXT:    retl
   %splat = shufflevector <4 x i32> %b, <4 x i32> undef, <4 x i32> zeroinitializer
   %splat32 = sub <4 x i32> <i32 32, i32 32, i32 32, i32 32>, %splat
@@ -1091,15 +1067,14 @@ define <16 x i8> @splatvar_rotate_v16i8(<16 x i8> %a, <16 x i8> %b) nounwind {
 ; AVX512F-LABEL: splatvar_rotate_v16i8:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vpbroadcastb %xmm1, %xmm1
+; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm2 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; AVX512F-NEXT:    vpsubb %xmm1, %xmm2, %xmm2
 ; AVX512F-NEXT:    vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
-; AVX512F-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
-; AVX512F-NEXT:    vpmovzxbd {{.*#+}} zmm2 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
-; AVX512F-NEXT:    vpsllvd %zmm2, %zmm0, %zmm2
-; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm3 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
-; AVX512F-NEXT:    vpsubb %xmm1, %xmm3, %xmm1
 ; AVX512F-NEXT:    vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
-; AVX512F-NEXT:    vpsrlvd %zmm1, %zmm0, %zmm0
-; AVX512F-NEXT:    vpord %zmm0, %zmm2, %zmm0
+; AVX512F-NEXT:    vpsllvd %zmm1, %zmm0, %zmm1
+; AVX512F-NEXT:    vpmovzxbd {{.*#+}} zmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero,xmm2[4],zero,zero,zero,xmm2[5],zero,zero,zero,xmm2[6],zero,zero,zero,xmm2[7],zero,zero,zero,xmm2[8],zero,zero,zero,xmm2[9],zero,zero,zero,xmm2[10],zero,zero,zero,xmm2[11],zero,zero,zero,xmm2[12],zero,zero,zero,xmm2[13],zero,zero,zero,xmm2[14],zero,zero,zero,xmm2[15],zero,zero,zero
+; AVX512F-NEXT:    vpsrlvd %zmm2, %zmm0, %zmm0
+; AVX512F-NEXT:    vpord %zmm0, %zmm1, %zmm0
 ; AVX512F-NEXT:    vpmovdb %zmm0, %xmm0
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
@@ -1107,15 +1082,14 @@ define <16 x i8> @splatvar_rotate_v16i8(<16 x i8> %a, <16 x i8> %b) nounwind {
 ; AVX512VL-LABEL: splatvar_rotate_v16i8:
 ; AVX512VL:       # %bb.0:
 ; AVX512VL-NEXT:    vpbroadcastb %xmm1, %xmm1
+; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm2 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
+; AVX512VL-NEXT:    vpsubb %xmm1, %xmm2, %xmm2
 ; AVX512VL-NEXT:    vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
-; AVX512VL-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
-; AVX512VL-NEXT:    vpmovzxbd {{.*#+}} zmm2 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
-; AVX512VL-NEXT:    vpsllvd %zmm2, %zmm0, %zmm2
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} xmm3 = [8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8]
-; AVX512VL-NEXT:    vpsubb %xmm1, %xmm3, %xmm1
 ; AVX512VL-NEXT:    vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
-; AVX512VL-NEXT:    vpsrlvd %zmm1, %zmm0, %zmm0
-; AVX512VL-NEXT:    vpord %zmm0, %zmm2, %zmm0
+; AVX512VL-NEXT:    vpsllvd %zmm1, %zmm0, %zmm1
+; AVX512VL-NEXT:    vpmovzxbd {{.*#+}} zmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero,xmm2[4],zero,zero,zero,xmm2[5],zero,zero,zero,xmm2[6],zero,zero,zero,xmm2[7],zero,zero,zero,xmm2[8],zero,zero,zero,xmm2[9],zero,zero,zero,xmm2[10],zero,zero,zero,xmm2[11],zero,zero,zero,xmm2[12],zero,zero,zero,xmm2[13],zero,zero,zero,xmm2[14],zero,zero,zero,xmm2[15],zero,zero,zero
+; AVX512VL-NEXT:    vpsrlvd %zmm2, %zmm0, %zmm0
+; AVX512VL-NEXT:    vpord %zmm0, %zmm1, %zmm0
 ; AVX512VL-NEXT:    vpmovdb %zmm0, %xmm0
 ; AVX512VL-NEXT:    vzeroupper
 ; AVX512VL-NEXT:    retq
@@ -1570,9 +1544,9 @@ define <16 x i8> @constant_rotate_v16i8(<16 x i8> %a) nounwind {
 ; AVX512F-LABEL: constant_rotate_v16i8:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
-; AVX512F-NEXT:    vpsrlvd {{.*}}(%rip), %zmm0, %zmm1
-; AVX512F-NEXT:    vpsllvd {{.*}}(%rip), %zmm0, %zmm0
-; AVX512F-NEXT:    vpord %zmm1, %zmm0, %zmm0
+; AVX512F-NEXT:    vpsllvd {{.*}}(%rip), %zmm0, %zmm1
+; AVX512F-NEXT:    vpsrlvd {{.*}}(%rip), %zmm0, %zmm0
+; AVX512F-NEXT:    vpord %zmm0, %zmm1, %zmm0
 ; AVX512F-NEXT:    vpmovdb %zmm0, %xmm0
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
@@ -1580,9 +1554,9 @@ define <16 x i8> @constant_rotate_v16i8(<16 x i8> %a) nounwind {
 ; AVX512VL-LABEL: constant_rotate_v16i8:
 ; AVX512VL:       # %bb.0:
 ; AVX512VL-NEXT:    vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
-; AVX512VL-NEXT:    vpsrlvd {{.*}}(%rip), %zmm0, %zmm1
-; AVX512VL-NEXT:    vpsllvd {{.*}}(%rip), %zmm0, %zmm0
-; AVX512VL-NEXT:    vpord %zmm1, %zmm0, %zmm0
+; AVX512VL-NEXT:    vpsllvd {{.*}}(%rip), %zmm0, %zmm1
+; AVX512VL-NEXT:    vpsrlvd {{.*}}(%rip), %zmm0, %zmm0
+; AVX512VL-NEXT:    vpord %zmm0, %zmm1, %zmm0
 ; AVX512VL-NEXT:    vpmovdb %zmm0, %xmm0
 ; AVX512VL-NEXT:    vzeroupper
 ; AVX512VL-NEXT:    retq
@@ -1829,41 +1803,14 @@ define <16 x i8> @splatconstant_rotate_v16i8(<16 x i8> %a) nounwind {
 ; AVX-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    retq
 ;
-; AVX512F-LABEL: splatconstant_rotate_v16i8:
-; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpsrlw $4, %xmm0, %xmm1
-; AVX512F-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
-; AVX512F-NEXT:    vpsllw $4, %xmm0, %xmm0
-; AVX512F-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
-; AVX512F-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX512F-NEXT:    retq
-;
-; AVX512VL-LABEL: splatconstant_rotate_v16i8:
-; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpsrlw $4, %xmm0, %xmm1
-; AVX512VL-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
-; AVX512VL-NEXT:    vpsllw $4, %xmm0, %xmm0
-; AVX512VL-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
-; AVX512VL-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX512VL-NEXT:    retq
-;
-; AVX512BW-LABEL: splatconstant_rotate_v16i8:
-; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vpsllw $4, %xmm0, %xmm1
-; AVX512BW-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
-; AVX512BW-NEXT:    vpsrlw $4, %xmm0, %xmm0
-; AVX512BW-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
-; AVX512BW-NEXT:    vpor %xmm0, %xmm1, %xmm0
-; AVX512BW-NEXT:    retq
-;
-; AVX512VLBW-LABEL: splatconstant_rotate_v16i8:
-; AVX512VLBW:       # %bb.0:
-; AVX512VLBW-NEXT:    vpsllw $4, %xmm0, %xmm1
-; AVX512VLBW-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
-; AVX512VLBW-NEXT:    vpsrlw $4, %xmm0, %xmm0
-; AVX512VLBW-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
-; AVX512VLBW-NEXT:    vpor %xmm0, %xmm1, %xmm0
-; AVX512VLBW-NEXT:    retq
+; AVX512-LABEL: splatconstant_rotate_v16i8:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpsllw $4, %xmm0, %xmm1
+; AVX512-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX512-NEXT:    vpsrlw $4, %xmm0, %xmm0
+; AVX512-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; AVX512-NEXT:    vpor %xmm0, %xmm1, %xmm0
+; AVX512-NEXT:    retq
 ;
 ; XOP-LABEL: splatconstant_rotate_v16i8:
 ; XOP:       # %bb.0:
@@ -2087,43 +2034,14 @@ define <16 x i8> @splatconstant_rotate_mask_v16i8(<16 x i8> %a) nounwind {
 ; AVX-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
 ; AVX-NEXT:    retq
 ;
-; AVX512F-LABEL: splatconstant_rotate_mask_v16i8:
-; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpsrlw $4, %xmm0, %xmm1
-; AVX512F-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
-; AVX512F-NEXT:    vpsllw $4, %xmm0, %xmm0
-; AVX512F-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
-; AVX512F-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX512F-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
-; AVX512F-NEXT:    retq
-;
-; AVX512VL-LABEL: splatconstant_rotate_mask_v16i8:
-; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpsrlw $4, %xmm0, %xmm1
-; AVX512VL-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
-; AVX512VL-NEXT:    vpsllw $4, %xmm0, %xmm0
-; AVX512VL-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
-; AVX512VL-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX512VL-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
-; AVX512VL-NEXT:    retq
-;
-; AVX512BW-LABEL: splatconstant_rotate_mask_v16i8:
-; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vpsllw $4, %xmm0, %xmm1
-; AVX512BW-NEXT:    vpsrlw $4, %xmm0, %xmm0
-; AVX512BW-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
-; AVX512BW-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
-; AVX512BW-NEXT:    vpor %xmm0, %xmm1, %xmm0
-; AVX512BW-NEXT:    retq
-;
-; AVX512VLBW-LABEL: splatconstant_rotate_mask_v16i8:
-; AVX512VLBW:       # %bb.0:
-; AVX512VLBW-NEXT:    vpsllw $4, %xmm0, %xmm1
-; AVX512VLBW-NEXT:    vpsrlw $4, %xmm0, %xmm0
-; AVX512VLBW-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
-; AVX512VLBW-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
-; AVX512VLBW-NEXT:    vpor %xmm0, %xmm1, %xmm0
-; AVX512VLBW-NEXT:    retq
+; AVX512-LABEL: splatconstant_rotate_mask_v16i8:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpsllw $4, %xmm0, %xmm1
+; AVX512-NEXT:    vpsrlw $4, %xmm0, %xmm0
+; AVX512-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; AVX512-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX512-NEXT:    vpor %xmm0, %xmm1, %xmm0
+; AVX512-NEXT:    retq
 ;
 ; XOP-LABEL: splatconstant_rotate_mask_v16i8:
 ; XOP:       # %bb.0:
diff --git a/test/CodeGen/X86/vector-rotate-256.ll b/test/CodeGen/X86/vector-rotate-256.ll
index b0d9cd9a3cb6456b99151f649c6806c756e13db5..31e2853ddb266e5c6d6d64e5c6553f38a1daa45a 100644
--- a/test/CodeGen/X86/vector-rotate-256.ll
+++ b/test/CodeGen/X86/vector-rotate-256.ll
@@ -337,47 +337,45 @@ define <32 x i8> @var_rotate_v32i8(<32 x i8> %a, <32 x i8> %b) nounwind {
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
 ; AVX1-NEXT:    vpsrlw $4, %xmm2, %xmm3
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm8 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; AVX1-NEXT:    vpand %xmm8, %xmm3, %xmm3
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
+; AVX1-NEXT:    vpandn %xmm3, %xmm4, %xmm3
 ; AVX1-NEXT:    vpsllw $4, %xmm2, %xmm5
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm9 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
-; AVX1-NEXT:    vpand %xmm9, %xmm5, %xmm5
+; AVX1-NEXT:    vpand %xmm4, %xmm5, %xmm5
 ; AVX1-NEXT:    vpor %xmm3, %xmm5, %xmm3
 ; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm5
 ; AVX1-NEXT:    vpsllw $5, %xmm5, %xmm5
 ; AVX1-NEXT:    vpblendvb %xmm5, %xmm3, %xmm2, %xmm2
 ; AVX1-NEXT:    vpsrlw $6, %xmm2, %xmm3
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm10 = [3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3]
-; AVX1-NEXT:    vpand %xmm10, %xmm3, %xmm3
-; AVX1-NEXT:    vpsllw $2, %xmm2, %xmm4
 ; AVX1-NEXT:    vmovdqa {{.*#+}} xmm6 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252]
-; AVX1-NEXT:    vpand %xmm6, %xmm4, %xmm4
-; AVX1-NEXT:    vpor %xmm3, %xmm4, %xmm3
-; AVX1-NEXT:    vpaddb %xmm5, %xmm5, %xmm4
-; AVX1-NEXT:    vpblendvb %xmm4, %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vpandn %xmm3, %xmm6, %xmm3
+; AVX1-NEXT:    vpsllw $2, %xmm2, %xmm7
+; AVX1-NEXT:    vpand %xmm6, %xmm7, %xmm7
+; AVX1-NEXT:    vpor %xmm3, %xmm7, %xmm3
+; AVX1-NEXT:    vpaddb %xmm5, %xmm5, %xmm5
+; AVX1-NEXT:    vpblendvb %xmm5, %xmm3, %xmm2, %xmm2
 ; AVX1-NEXT:    vpsrlw $7, %xmm2, %xmm3
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm5 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
-; AVX1-NEXT:    vpand %xmm5, %xmm3, %xmm3
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm8 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
+; AVX1-NEXT:    vpand %xmm8, %xmm3, %xmm3
 ; AVX1-NEXT:    vpaddb %xmm2, %xmm2, %xmm7
 ; AVX1-NEXT:    vpor %xmm3, %xmm7, %xmm3
-; AVX1-NEXT:    vpaddb %xmm4, %xmm4, %xmm4
-; AVX1-NEXT:    vpblendvb %xmm4, %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vpaddb %xmm5, %xmm5, %xmm5
+; AVX1-NEXT:    vpblendvb %xmm5, %xmm3, %xmm2, %xmm2
 ; AVX1-NEXT:    vpsrlw $4, %xmm0, %xmm3
-; AVX1-NEXT:    vpand %xmm8, %xmm3, %xmm3
-; AVX1-NEXT:    vpsllw $4, %xmm0, %xmm4
-; AVX1-NEXT:    vpand %xmm9, %xmm4, %xmm4
+; AVX1-NEXT:    vpandn %xmm3, %xmm4, %xmm3
+; AVX1-NEXT:    vpsllw $4, %xmm0, %xmm5
+; AVX1-NEXT:    vpand %xmm4, %xmm5, %xmm4
 ; AVX1-NEXT:    vpor %xmm3, %xmm4, %xmm3
 ; AVX1-NEXT:    vpsllw $5, %xmm1, %xmm1
 ; AVX1-NEXT:    vpblendvb %xmm1, %xmm3, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrlw $6, %xmm0, %xmm3
-; AVX1-NEXT:    vpand %xmm10, %xmm3, %xmm3
+; AVX1-NEXT:    vpandn %xmm3, %xmm6, %xmm3
 ; AVX1-NEXT:    vpsllw $2, %xmm0, %xmm4
 ; AVX1-NEXT:    vpand %xmm6, %xmm4, %xmm4
 ; AVX1-NEXT:    vpor %xmm3, %xmm4, %xmm3
 ; AVX1-NEXT:    vpaddb %xmm1, %xmm1, %xmm1
 ; AVX1-NEXT:    vpblendvb %xmm1, %xmm3, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrlw $7, %xmm0, %xmm3
-; AVX1-NEXT:    vpand %xmm5, %xmm3, %xmm3
+; AVX1-NEXT:    vpand %xmm8, %xmm3, %xmm3
 ; AVX1-NEXT:    vpaddb %xmm0, %xmm0, %xmm4
 ; AVX1-NEXT:    vpor %xmm3, %xmm4, %xmm3
 ; AVX1-NEXT:    vpaddb %xmm1, %xmm1, %xmm1
@@ -1476,16 +1474,15 @@ define <32 x i8> @splatconstant_rotate_v32i8(<32 x i8> %a) nounwind {
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; AVX1-NEXT:    vpsrlw $4, %xmm1, %xmm2
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; AVX1-NEXT:    vpand %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
+; AVX1-NEXT:    vpandn %xmm2, %xmm3, %xmm2
 ; AVX1-NEXT:    vpsllw $4, %xmm1, %xmm1
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
-; AVX1-NEXT:    vpand %xmm4, %xmm1, %xmm1
+; AVX1-NEXT:    vpand %xmm3, %xmm1, %xmm1
 ; AVX1-NEXT:    vpor %xmm2, %xmm1, %xmm1
 ; AVX1-NEXT:    vpsrlw $4, %xmm0, %xmm2
-; AVX1-NEXT:    vpand %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vpandn %xmm2, %xmm3, %xmm2
 ; AVX1-NEXT:    vpsllw $4, %xmm0, %xmm0
-; AVX1-NEXT:    vpand %xmm4, %xmm0, %xmm0
+; AVX1-NEXT:    vpand %xmm3, %xmm0, %xmm0
 ; AVX1-NEXT:    vpor %xmm2, %xmm0, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
 ; AVX1-NEXT:    retq
@@ -1760,16 +1757,15 @@ define <32 x i8> @splatconstant_rotate_mask_v32i8(<32 x i8> %a) nounwind {
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; AVX1-NEXT:    vpsrlw $4, %xmm1, %xmm2
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; AVX1-NEXT:    vpand %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vmovdqa {{.*#+}} xmm3 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
+; AVX1-NEXT:    vpandn %xmm2, %xmm3, %xmm2
 ; AVX1-NEXT:    vpsllw $4, %xmm1, %xmm1
-; AVX1-NEXT:    vmovdqa {{.*#+}} xmm4 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
-; AVX1-NEXT:    vpand %xmm4, %xmm1, %xmm1
+; AVX1-NEXT:    vpand %xmm3, %xmm1, %xmm1
 ; AVX1-NEXT:    vpor %xmm2, %xmm1, %xmm1
 ; AVX1-NEXT:    vpsrlw $4, %xmm0, %xmm2
-; AVX1-NEXT:    vpand %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vpandn %xmm2, %xmm3, %xmm2
 ; AVX1-NEXT:    vpsllw $4, %xmm0, %xmm0
-; AVX1-NEXT:    vpand %xmm4, %xmm0, %xmm0
+; AVX1-NEXT:    vpand %xmm3, %xmm0, %xmm0
 ; AVX1-NEXT:    vpor %xmm2, %xmm0, %xmm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
 ; AVX1-NEXT:    vandps {{.*}}(%rip), %ymm0, %ymm0
diff --git a/test/CodeGen/X86/vector-rotate-512.ll b/test/CodeGen/X86/vector-rotate-512.ll
index 896d6c07387ef926b24d46eaeecc361c9df39c5b..1d177c1047529741b01a488dc8fd65baa33d9efe 100644
--- a/test/CodeGen/X86/vector-rotate-512.ll
+++ b/test/CodeGen/X86/vector-rotate-512.ll
@@ -109,46 +109,44 @@ define <64 x i8> @var_rotate_v64i8(<64 x i8> %a, <64 x i8> %b) nounwind {
 ; AVX512F-LABEL: var_rotate_v64i8:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vpsrlw $4, %ymm0, %ymm4
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm5 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; AVX512F-NEXT:    vpand %ymm5, %ymm4, %ymm4
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm5 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
+; AVX512F-NEXT:    vpandn %ymm4, %ymm5, %ymm4
 ; AVX512F-NEXT:    vpsllw $4, %ymm0, %ymm6
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm7 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
-; AVX512F-NEXT:    vpand %ymm7, %ymm6, %ymm6
+; AVX512F-NEXT:    vpand %ymm5, %ymm6, %ymm6
 ; AVX512F-NEXT:    vpor %ymm4, %ymm6, %ymm4
 ; AVX512F-NEXT:    vpsllw $5, %ymm2, %ymm2
 ; AVX512F-NEXT:    vpblendvb %ymm2, %ymm4, %ymm0, %ymm0
 ; AVX512F-NEXT:    vpsrlw $6, %ymm0, %ymm4
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm6 = [3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3]
-; AVX512F-NEXT:    vpand %ymm6, %ymm4, %ymm4
-; AVX512F-NEXT:    vpsllw $2, %ymm0, %ymm8
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm9 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252]
-; AVX512F-NEXT:    vpand %ymm9, %ymm8, %ymm8
-; AVX512F-NEXT:    vpor %ymm4, %ymm8, %ymm4
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm6 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252]
+; AVX512F-NEXT:    vpandn %ymm4, %ymm6, %ymm4
+; AVX512F-NEXT:    vpsllw $2, %ymm0, %ymm7
+; AVX512F-NEXT:    vpand %ymm6, %ymm7, %ymm7
+; AVX512F-NEXT:    vpor %ymm4, %ymm7, %ymm4
 ; AVX512F-NEXT:    vpaddb %ymm2, %ymm2, %ymm2
 ; AVX512F-NEXT:    vpblendvb %ymm2, %ymm4, %ymm0, %ymm0
 ; AVX512F-NEXT:    vpsrlw $7, %ymm0, %ymm4
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm8 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
-; AVX512F-NEXT:    vpand %ymm8, %ymm4, %ymm4
-; AVX512F-NEXT:    vpaddb %ymm0, %ymm0, %ymm10
-; AVX512F-NEXT:    vpor %ymm4, %ymm10, %ymm4
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm7 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
+; AVX512F-NEXT:    vpand %ymm7, %ymm4, %ymm4
+; AVX512F-NEXT:    vpaddb %ymm0, %ymm0, %ymm8
+; AVX512F-NEXT:    vpor %ymm4, %ymm8, %ymm4
 ; AVX512F-NEXT:    vpaddb %ymm2, %ymm2, %ymm2
 ; AVX512F-NEXT:    vpblendvb %ymm2, %ymm4, %ymm0, %ymm0
 ; AVX512F-NEXT:    vpsrlw $4, %ymm1, %ymm2
-; AVX512F-NEXT:    vpand %ymm5, %ymm2, %ymm2
+; AVX512F-NEXT:    vpandn %ymm2, %ymm5, %ymm2
 ; AVX512F-NEXT:    vpsllw $4, %ymm1, %ymm4
-; AVX512F-NEXT:    vpand %ymm7, %ymm4, %ymm4
+; AVX512F-NEXT:    vpand %ymm5, %ymm4, %ymm4
 ; AVX512F-NEXT:    vpor %ymm2, %ymm4, %ymm2
 ; AVX512F-NEXT:    vpsllw $5, %ymm3, %ymm3
 ; AVX512F-NEXT:    vpblendvb %ymm3, %ymm2, %ymm1, %ymm1
 ; AVX512F-NEXT:    vpsrlw $6, %ymm1, %ymm2
-; AVX512F-NEXT:    vpand %ymm6, %ymm2, %ymm2
+; AVX512F-NEXT:    vpandn %ymm2, %ymm6, %ymm2
 ; AVX512F-NEXT:    vpsllw $2, %ymm1, %ymm4
-; AVX512F-NEXT:    vpand %ymm9, %ymm4, %ymm4
+; AVX512F-NEXT:    vpand %ymm6, %ymm4, %ymm4
 ; AVX512F-NEXT:    vpor %ymm2, %ymm4, %ymm2
 ; AVX512F-NEXT:    vpaddb %ymm3, %ymm3, %ymm3
 ; AVX512F-NEXT:    vpblendvb %ymm3, %ymm2, %ymm1, %ymm1
 ; AVX512F-NEXT:    vpsrlw $7, %ymm1, %ymm2
-; AVX512F-NEXT:    vpand %ymm8, %ymm2, %ymm2
+; AVX512F-NEXT:    vpand %ymm7, %ymm2, %ymm2
 ; AVX512F-NEXT:    vpaddb %ymm1, %ymm1, %ymm4
 ; AVX512F-NEXT:    vpor %ymm2, %ymm4, %ymm2
 ; AVX512F-NEXT:    vpaddb %ymm3, %ymm3, %ymm3
@@ -158,46 +156,44 @@ define <64 x i8> @var_rotate_v64i8(<64 x i8> %a, <64 x i8> %b) nounwind {
 ; AVX512VL-LABEL: var_rotate_v64i8:
 ; AVX512VL:       # %bb.0:
 ; AVX512VL-NEXT:    vpsrlw $4, %ymm0, %ymm4
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm5 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; AVX512VL-NEXT:    vpand %ymm5, %ymm4, %ymm4
+; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm5 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
+; AVX512VL-NEXT:    vpandn %ymm4, %ymm5, %ymm4
 ; AVX512VL-NEXT:    vpsllw $4, %ymm0, %ymm6
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm7 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
-; AVX512VL-NEXT:    vpand %ymm7, %ymm6, %ymm6
+; AVX512VL-NEXT:    vpand %ymm5, %ymm6, %ymm6
 ; AVX512VL-NEXT:    vpor %ymm4, %ymm6, %ymm4
 ; AVX512VL-NEXT:    vpsllw $5, %ymm2, %ymm2
 ; AVX512VL-NEXT:    vpblendvb %ymm2, %ymm4, %ymm0, %ymm0
 ; AVX512VL-NEXT:    vpsrlw $6, %ymm0, %ymm4
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm6 = [3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3]
-; AVX512VL-NEXT:    vpand %ymm6, %ymm4, %ymm4
-; AVX512VL-NEXT:    vpsllw $2, %ymm0, %ymm8
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm9 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252]
-; AVX512VL-NEXT:    vpand %ymm9, %ymm8, %ymm8
-; AVX512VL-NEXT:    vpor %ymm4, %ymm8, %ymm4
+; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm6 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252]
+; AVX512VL-NEXT:    vpandn %ymm4, %ymm6, %ymm4
+; AVX512VL-NEXT:    vpsllw $2, %ymm0, %ymm7
+; AVX512VL-NEXT:    vpand %ymm6, %ymm7, %ymm7
+; AVX512VL-NEXT:    vpor %ymm4, %ymm7, %ymm4
 ; AVX512VL-NEXT:    vpaddb %ymm2, %ymm2, %ymm2
 ; AVX512VL-NEXT:    vpblendvb %ymm2, %ymm4, %ymm0, %ymm0
 ; AVX512VL-NEXT:    vpsrlw $7, %ymm0, %ymm4
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm8 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
-; AVX512VL-NEXT:    vpand %ymm8, %ymm4, %ymm4
-; AVX512VL-NEXT:    vpaddb %ymm0, %ymm0, %ymm10
-; AVX512VL-NEXT:    vpor %ymm4, %ymm10, %ymm4
+; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm7 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
+; AVX512VL-NEXT:    vpand %ymm7, %ymm4, %ymm4
+; AVX512VL-NEXT:    vpaddb %ymm0, %ymm0, %ymm8
+; AVX512VL-NEXT:    vpor %ymm4, %ymm8, %ymm4
 ; AVX512VL-NEXT:    vpaddb %ymm2, %ymm2, %ymm2
 ; AVX512VL-NEXT:    vpblendvb %ymm2, %ymm4, %ymm0, %ymm0
 ; AVX512VL-NEXT:    vpsrlw $4, %ymm1, %ymm2
-; AVX512VL-NEXT:    vpand %ymm5, %ymm2, %ymm2
+; AVX512VL-NEXT:    vpandn %ymm2, %ymm5, %ymm2
 ; AVX512VL-NEXT:    vpsllw $4, %ymm1, %ymm4
-; AVX512VL-NEXT:    vpand %ymm7, %ymm4, %ymm4
+; AVX512VL-NEXT:    vpand %ymm5, %ymm4, %ymm4
 ; AVX512VL-NEXT:    vpor %ymm2, %ymm4, %ymm2
 ; AVX512VL-NEXT:    vpsllw $5, %ymm3, %ymm3
 ; AVX512VL-NEXT:    vpblendvb %ymm3, %ymm2, %ymm1, %ymm1
 ; AVX512VL-NEXT:    vpsrlw $6, %ymm1, %ymm2
-; AVX512VL-NEXT:    vpand %ymm6, %ymm2, %ymm2
+; AVX512VL-NEXT:    vpandn %ymm2, %ymm6, %ymm2
 ; AVX512VL-NEXT:    vpsllw $2, %ymm1, %ymm4
-; AVX512VL-NEXT:    vpand %ymm9, %ymm4, %ymm4
+; AVX512VL-NEXT:    vpand %ymm6, %ymm4, %ymm4
 ; AVX512VL-NEXT:    vpor %ymm2, %ymm4, %ymm2
 ; AVX512VL-NEXT:    vpaddb %ymm3, %ymm3, %ymm3
 ; AVX512VL-NEXT:    vpblendvb %ymm3, %ymm2, %ymm1, %ymm1
 ; AVX512VL-NEXT:    vpsrlw $7, %ymm1, %ymm2
-; AVX512VL-NEXT:    vpand %ymm8, %ymm2, %ymm2
+; AVX512VL-NEXT:    vpand %ymm7, %ymm2, %ymm2
 ; AVX512VL-NEXT:    vpaddb %ymm1, %ymm1, %ymm4
 ; AVX512VL-NEXT:    vpor %ymm2, %ymm4, %ymm2
 ; AVX512VL-NEXT:    vpaddb %ymm3, %ymm3, %ymm3
@@ -767,32 +763,30 @@ define <64 x i8> @splatconstant_rotate_v64i8(<64 x i8> %a) nounwind {
 ; AVX512F-LABEL: splatconstant_rotate_v64i8:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vpsrlw $4, %ymm0, %ymm2
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; AVX512F-NEXT:    vpand %ymm3, %ymm2, %ymm2
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm3 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
+; AVX512F-NEXT:    vpandn %ymm2, %ymm3, %ymm2
 ; AVX512F-NEXT:    vpsllw $4, %ymm0, %ymm0
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm4 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
-; AVX512F-NEXT:    vpand %ymm4, %ymm0, %ymm0
+; AVX512F-NEXT:    vpand %ymm3, %ymm0, %ymm0
 ; AVX512F-NEXT:    vpor %ymm2, %ymm0, %ymm0
 ; AVX512F-NEXT:    vpsrlw $4, %ymm1, %ymm2
-; AVX512F-NEXT:    vpand %ymm3, %ymm2, %ymm2
+; AVX512F-NEXT:    vpandn %ymm2, %ymm3, %ymm2
 ; AVX512F-NEXT:    vpsllw $4, %ymm1, %ymm1
-; AVX512F-NEXT:    vpand %ymm4, %ymm1, %ymm1
+; AVX512F-NEXT:    vpand %ymm3, %ymm1, %ymm1
 ; AVX512F-NEXT:    vpor %ymm2, %ymm1, %ymm1
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: splatconstant_rotate_v64i8:
 ; AVX512VL:       # %bb.0:
 ; AVX512VL-NEXT:    vpsrlw $4, %ymm0, %ymm2
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; AVX512VL-NEXT:    vpand %ymm3, %ymm2, %ymm2
+; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm3 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
+; AVX512VL-NEXT:    vpandn %ymm2, %ymm3, %ymm2
 ; AVX512VL-NEXT:    vpsllw $4, %ymm0, %ymm0
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm4 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
-; AVX512VL-NEXT:    vpand %ymm4, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpand %ymm3, %ymm0, %ymm0
 ; AVX512VL-NEXT:    vpor %ymm2, %ymm0, %ymm0
 ; AVX512VL-NEXT:    vpsrlw $4, %ymm1, %ymm2
-; AVX512VL-NEXT:    vpand %ymm3, %ymm2, %ymm2
+; AVX512VL-NEXT:    vpandn %ymm2, %ymm3, %ymm2
 ; AVX512VL-NEXT:    vpsllw $4, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpand %ymm4, %ymm1, %ymm1
+; AVX512VL-NEXT:    vpand %ymm3, %ymm1, %ymm1
 ; AVX512VL-NEXT:    vpor %ymm2, %ymm1, %ymm1
 ; AVX512VL-NEXT:    retq
 ;
@@ -907,38 +901,36 @@ define <64 x i8> @splatconstant_rotate_mask_v64i8(<64 x i8> %a) nounwind {
 ; AVX512F-LABEL: splatconstant_rotate_mask_v64i8:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vpsrlw $4, %ymm0, %ymm2
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; AVX512F-NEXT:    vpand %ymm3, %ymm2, %ymm2
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm3 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
+; AVX512F-NEXT:    vpandn %ymm2, %ymm3, %ymm2
 ; AVX512F-NEXT:    vpsllw $4, %ymm0, %ymm0
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm4 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
-; AVX512F-NEXT:    vpand %ymm4, %ymm0, %ymm0
+; AVX512F-NEXT:    vpand %ymm3, %ymm0, %ymm0
 ; AVX512F-NEXT:    vpor %ymm2, %ymm0, %ymm0
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm2 = [39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39]
 ; AVX512F-NEXT:    vpand %ymm2, %ymm0, %ymm0
-; AVX512F-NEXT:    vpsrlw $4, %ymm1, %ymm5
-; AVX512F-NEXT:    vpand %ymm3, %ymm5, %ymm3
+; AVX512F-NEXT:    vpsrlw $4, %ymm1, %ymm4
+; AVX512F-NEXT:    vpandn %ymm4, %ymm3, %ymm4
 ; AVX512F-NEXT:    vpsllw $4, %ymm1, %ymm1
-; AVX512F-NEXT:    vpand %ymm4, %ymm1, %ymm1
-; AVX512F-NEXT:    vpor %ymm3, %ymm1, %ymm1
+; AVX512F-NEXT:    vpand %ymm3, %ymm1, %ymm1
+; AVX512F-NEXT:    vpor %ymm4, %ymm1, %ymm1
 ; AVX512F-NEXT:    vpand %ymm2, %ymm1, %ymm1
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VL-LABEL: splatconstant_rotate_mask_v64i8:
 ; AVX512VL:       # %bb.0:
 ; AVX512VL-NEXT:    vpsrlw $4, %ymm0, %ymm2
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; AVX512VL-NEXT:    vpand %ymm3, %ymm2, %ymm2
+; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm3 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
+; AVX512VL-NEXT:    vpandn %ymm2, %ymm3, %ymm2
 ; AVX512VL-NEXT:    vpsllw $4, %ymm0, %ymm0
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm4 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
-; AVX512VL-NEXT:    vpand %ymm4, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpand %ymm3, %ymm0, %ymm0
 ; AVX512VL-NEXT:    vpor %ymm2, %ymm0, %ymm0
 ; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm2 = [39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39]
 ; AVX512VL-NEXT:    vpand %ymm2, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpsrlw $4, %ymm1, %ymm5
-; AVX512VL-NEXT:    vpand %ymm3, %ymm5, %ymm3
+; AVX512VL-NEXT:    vpsrlw $4, %ymm1, %ymm4
+; AVX512VL-NEXT:    vpandn %ymm4, %ymm3, %ymm4
 ; AVX512VL-NEXT:    vpsllw $4, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpand %ymm4, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpor %ymm3, %ymm1, %ymm1
+; AVX512VL-NEXT:    vpand %ymm3, %ymm1, %ymm1
+; AVX512VL-NEXT:    vpor %ymm4, %ymm1, %ymm1
 ; AVX512VL-NEXT:    vpand %ymm2, %ymm1, %ymm1
 ; AVX512VL-NEXT:    retq
 ;
diff --git a/test/CodeGen/X86/vector-sext-widen.ll b/test/CodeGen/X86/vector-sext-widen.ll
index 895d0e57bbe9ebaa1dfab4d4af27958b8902945a..5cd814dbfdfb24a75cb63c164a180a9123509452 100644
--- a/test/CodeGen/X86/vector-sext-widen.ll
+++ b/test/CodeGen/X86/vector-sext-widen.ll
@@ -1388,23 +1388,13 @@ define <2 x i64> @load_sext_2i1_to_2i64(<2 x i1> *%ptr) {
 ; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
 ; AVX2-NEXT:    retq
 ;
-; AVX512F-LABEL: load_sext_2i1_to_2i64:
-; AVX512F:       # %bb.0: # %entry
-; AVX512F-NEXT:    movzbl (%rdi), %eax
-; AVX512F-NEXT:    kmovw %eax, %k1
-; AVX512F-NEXT:    vpternlogq $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
-; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
-; AVX512F-NEXT:    vzeroupper
-; AVX512F-NEXT:    retq
-;
-; AVX512BW-LABEL: load_sext_2i1_to_2i64:
-; AVX512BW:       # %bb.0: # %entry
-; AVX512BW-NEXT:    movzbl (%rdi), %eax
-; AVX512BW-NEXT:    kmovd %eax, %k1
-; AVX512BW-NEXT:    vpternlogq $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
-; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
-; AVX512BW-NEXT:    vzeroupper
-; AVX512BW-NEXT:    retq
+; AVX512-LABEL: load_sext_2i1_to_2i64:
+; AVX512:       # %bb.0: # %entry
+; AVX512-NEXT:    kmovw (%rdi), %k1
+; AVX512-NEXT:    vpternlogq $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; AVX512-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512-NEXT:    vzeroupper
+; AVX512-NEXT:    retq
 ;
 ; X32-SSE2-LABEL: load_sext_2i1_to_2i64:
 ; X32-SSE2:       # %bb.0: # %entry
@@ -1608,23 +1598,13 @@ define <4 x i32> @load_sext_4i1_to_4i32(<4 x i1> *%ptr) {
 ; AVX2-NEXT:    vpinsrd $3, %eax, %xmm0, %xmm0
 ; AVX2-NEXT:    retq
 ;
-; AVX512F-LABEL: load_sext_4i1_to_4i32:
-; AVX512F:       # %bb.0: # %entry
-; AVX512F-NEXT:    movzbl (%rdi), %eax
-; AVX512F-NEXT:    kmovw %eax, %k1
-; AVX512F-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
-; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
-; AVX512F-NEXT:    vzeroupper
-; AVX512F-NEXT:    retq
-;
-; AVX512BW-LABEL: load_sext_4i1_to_4i32:
-; AVX512BW:       # %bb.0: # %entry
-; AVX512BW-NEXT:    movzbl (%rdi), %eax
-; AVX512BW-NEXT:    kmovd %eax, %k1
-; AVX512BW-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
-; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
-; AVX512BW-NEXT:    vzeroupper
-; AVX512BW-NEXT:    retq
+; AVX512-LABEL: load_sext_4i1_to_4i32:
+; AVX512:       # %bb.0: # %entry
+; AVX512-NEXT:    kmovw (%rdi), %k1
+; AVX512-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; AVX512-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512-NEXT:    vzeroupper
+; AVX512-NEXT:    retq
 ;
 ; X32-SSE2-LABEL: load_sext_4i1_to_4i32:
 ; X32-SSE2:       # %bb.0: # %entry
@@ -1846,21 +1826,12 @@ define <4 x i64> @load_sext_4i1_to_4i64(<4 x i1> *%ptr) {
 ; AVX2-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm0
 ; AVX2-NEXT:    retq
 ;
-; AVX512F-LABEL: load_sext_4i1_to_4i64:
-; AVX512F:       # %bb.0: # %entry
-; AVX512F-NEXT:    movzbl (%rdi), %eax
-; AVX512F-NEXT:    kmovw %eax, %k1
-; AVX512F-NEXT:    vpternlogq $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
-; AVX512F-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
-; AVX512F-NEXT:    retq
-;
-; AVX512BW-LABEL: load_sext_4i1_to_4i64:
-; AVX512BW:       # %bb.0: # %entry
-; AVX512BW-NEXT:    movzbl (%rdi), %eax
-; AVX512BW-NEXT:    kmovd %eax, %k1
-; AVX512BW-NEXT:    vpternlogq $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
-; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
-; AVX512BW-NEXT:    retq
+; AVX512-LABEL: load_sext_4i1_to_4i64:
+; AVX512:       # %bb.0: # %entry
+; AVX512-NEXT:    kmovw (%rdi), %k1
+; AVX512-NEXT:    vpternlogq $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; AVX512-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
+; AVX512-NEXT:    retq
 ;
 ; X32-SSE2-LABEL: load_sext_4i1_to_4i64:
 ; X32-SSE2:       # %bb.0: # %entry
@@ -1921,30 +1892,28 @@ entry:
 define <4 x i64> @load_sext_4i8_to_4i64(<4 x i8> *%ptr) {
 ; SSE2-LABEL: load_sext_4i8_to_4i64:
 ; SSE2:       # %bb.0: # %entry
-; SSE2-NEXT:    movsbq 1(%rdi), %rax
-; SSE2-NEXT:    movq %rax, %xmm1
-; SSE2-NEXT:    movsbq (%rdi), %rax
-; SSE2-NEXT:    movq %rax, %xmm0
-; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; SSE2-NEXT:    movsbq 3(%rdi), %rax
-; SSE2-NEXT:    movq %rax, %xmm2
-; SSE2-NEXT:    movsbq 2(%rdi), %rax
-; SSE2-NEXT:    movq %rax, %xmm1
-; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
+; SSE2-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
+; SSE2-NEXT:    psrad $24, %xmm1
+; SSE2-NEXT:    pxor %xmm2, %xmm2
+; SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
+; SSE2-NEXT:    movdqa %xmm1, %xmm0
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
+; SSE2-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]
 ; SSE2-NEXT:    retq
 ;
 ; SSSE3-LABEL: load_sext_4i8_to_4i64:
 ; SSSE3:       # %bb.0: # %entry
-; SSSE3-NEXT:    movsbq 1(%rdi), %rax
-; SSSE3-NEXT:    movq %rax, %xmm1
-; SSSE3-NEXT:    movsbq (%rdi), %rax
-; SSSE3-NEXT:    movq %rax, %xmm0
-; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; SSSE3-NEXT:    movsbq 3(%rdi), %rax
-; SSSE3-NEXT:    movq %rax, %xmm2
-; SSSE3-NEXT:    movsbq 2(%rdi), %rax
-; SSSE3-NEXT:    movq %rax, %xmm1
-; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
+; SSSE3-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
+; SSSE3-NEXT:    psrad $24, %xmm1
+; SSSE3-NEXT:    pxor %xmm2, %xmm2
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm2
+; SSSE3-NEXT:    movdqa %xmm1, %xmm0
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
+; SSSE3-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: load_sext_4i8_to_4i64:
@@ -1999,20 +1968,24 @@ entry:
 define <2 x i64> @load_sext_4i8_to_4i64_extract(<4 x i8> *%ptr) {
 ; SSE2-LABEL: load_sext_4i8_to_4i64_extract:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    movsbq 3(%rdi), %rax
-; SSE2-NEXT:    movq %rax, %xmm1
-; SSE2-NEXT:    movsbq 2(%rdi), %rax
-; SSE2-NEXT:    movq %rax, %xmm0
-; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; SSE2-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
+; SSE2-NEXT:    psrad $24, %xmm0
+; SSE2-NEXT:    pxor %xmm1, %xmm1
+; SSE2-NEXT:    pcmpgtd %xmm0, %xmm1
+; SSE2-NEXT:    punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
 ; SSE2-NEXT:    retq
 ;
 ; SSSE3-LABEL: load_sext_4i8_to_4i64_extract:
 ; SSSE3:       # %bb.0:
-; SSSE3-NEXT:    movsbq 3(%rdi), %rax
-; SSSE3-NEXT:    movq %rax, %xmm1
-; SSSE3-NEXT:    movsbq 2(%rdi), %rax
-; SSSE3-NEXT:    movq %rax, %xmm0
-; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; SSSE3-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
+; SSSE3-NEXT:    psrad $24, %xmm0
+; SSSE3-NEXT:    pxor %xmm1, %xmm1
+; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm1
+; SSSE3-NEXT:    punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: load_sext_4i8_to_4i64_extract:
@@ -2254,8 +2227,7 @@ define <8 x i16> @load_sext_8i1_to_8i16(<8 x i1> *%ptr) {
 ;
 ; AVX512F-LABEL: load_sext_8i1_to_8i16:
 ; AVX512F:       # %bb.0: # %entry
-; AVX512F-NEXT:    movzbl (%rdi), %eax
-; AVX512F-NEXT:    kmovw %eax, %k1
+; AVX512F-NEXT:    kmovw (%rdi), %k1
 ; AVX512F-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
 ; AVX512F-NEXT:    vpmovdw %zmm0, %ymm0
 ; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
@@ -2264,8 +2236,7 @@ define <8 x i16> @load_sext_8i1_to_8i16(<8 x i1> *%ptr) {
 ;
 ; AVX512BW-LABEL: load_sext_8i1_to_8i16:
 ; AVX512BW:       # %bb.0: # %entry
-; AVX512BW-NEXT:    movzbl (%rdi), %eax
-; AVX512BW-NEXT:    kmovd %eax, %k0
+; AVX512BW-NEXT:    kmovw (%rdi), %k0
 ; AVX512BW-NEXT:    vpmovm2w %k0, %zmm0
 ; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
 ; AVX512BW-NEXT:    vzeroupper
@@ -2402,50 +2373,42 @@ entry:
 define <8 x i64> @load_sext_8i8_to_8i64(<8 x i8> *%ptr) {
 ; SSE2-LABEL: load_sext_8i8_to_8i64:
 ; SSE2:       # %bb.0: # %entry
-; SSE2-NEXT:    movsbq 1(%rdi), %rax
-; SSE2-NEXT:    movq %rax, %xmm1
-; SSE2-NEXT:    movsbq (%rdi), %rax
-; SSE2-NEXT:    movq %rax, %xmm0
-; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; SSE2-NEXT:    movsbq 3(%rdi), %rax
-; SSE2-NEXT:    movq %rax, %xmm2
-; SSE2-NEXT:    movsbq 2(%rdi), %rax
-; SSE2-NEXT:    movq %rax, %xmm1
-; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
-; SSE2-NEXT:    movsbq 5(%rdi), %rax
-; SSE2-NEXT:    movq %rax, %xmm3
-; SSE2-NEXT:    movsbq 4(%rdi), %rax
-; SSE2-NEXT:    movq %rax, %xmm2
-; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; SSE2-NEXT:    movsbq 7(%rdi), %rax
-; SSE2-NEXT:    movq %rax, %xmm4
-; SSE2-NEXT:    movsbq 6(%rdi), %rax
-; SSE2-NEXT:    movq %rax, %xmm3
-; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm4[0]
+; SSE2-NEXT:    movq {{.*#+}} xmm2 = mem[0],zero
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
+; SSE2-NEXT:    psrad $24, %xmm1
+; SSE2-NEXT:    pxor %xmm4, %xmm4
+; SSE2-NEXT:    pxor %xmm3, %xmm3
+; SSE2-NEXT:    pcmpgtd %xmm1, %xmm3
+; SSE2-NEXT:    movdqa %xmm1, %xmm0
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
+; SSE2-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm3[2],xmm1[3],xmm3[3]
+; SSE2-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
+; SSE2-NEXT:    psrad $24, %xmm3
+; SSE2-NEXT:    pcmpgtd %xmm3, %xmm4
+; SSE2-NEXT:    movdqa %xmm3, %xmm2
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1]
+; SSE2-NEXT:    punpckhdq {{.*#+}} xmm3 = xmm3[2],xmm4[2],xmm3[3],xmm4[3]
 ; SSE2-NEXT:    retq
 ;
 ; SSSE3-LABEL: load_sext_8i8_to_8i64:
 ; SSSE3:       # %bb.0: # %entry
-; SSSE3-NEXT:    movsbq 1(%rdi), %rax
-; SSSE3-NEXT:    movq %rax, %xmm1
-; SSSE3-NEXT:    movsbq (%rdi), %rax
-; SSSE3-NEXT:    movq %rax, %xmm0
-; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; SSSE3-NEXT:    movsbq 3(%rdi), %rax
-; SSSE3-NEXT:    movq %rax, %xmm2
-; SSSE3-NEXT:    movsbq 2(%rdi), %rax
-; SSSE3-NEXT:    movq %rax, %xmm1
-; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
-; SSSE3-NEXT:    movsbq 5(%rdi), %rax
-; SSSE3-NEXT:    movq %rax, %xmm3
-; SSSE3-NEXT:    movsbq 4(%rdi), %rax
-; SSSE3-NEXT:    movq %rax, %xmm2
-; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; SSSE3-NEXT:    movsbq 7(%rdi), %rax
-; SSSE3-NEXT:    movq %rax, %xmm4
-; SSSE3-NEXT:    movsbq 6(%rdi), %rax
-; SSSE3-NEXT:    movq %rax, %xmm3
-; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm4[0]
+; SSSE3-NEXT:    movq {{.*#+}} xmm2 = mem[0],zero
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
+; SSSE3-NEXT:    psrad $24, %xmm1
+; SSSE3-NEXT:    pxor %xmm4, %xmm4
+; SSSE3-NEXT:    pxor %xmm3, %xmm3
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm3
+; SSSE3-NEXT:    movdqa %xmm1, %xmm0
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
+; SSSE3-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm3[2],xmm1[3],xmm3[3]
+; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
+; SSSE3-NEXT:    psrad $24, %xmm3
+; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm4
+; SSSE3-NEXT:    movdqa %xmm3, %xmm2
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1]
+; SSSE3-NEXT:    punpckhdq {{.*#+}} xmm3 = xmm3[2],xmm4[2],xmm3[3],xmm4[3]
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: load_sext_8i8_to_8i64:
@@ -2717,21 +2680,12 @@ define <8 x i32> @load_sext_8i1_to_8i32(<8 x i1> *%ptr) {
 ; AVX2-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm0
 ; AVX2-NEXT:    retq
 ;
-; AVX512F-LABEL: load_sext_8i1_to_8i32:
-; AVX512F:       # %bb.0: # %entry
-; AVX512F-NEXT:    movzbl (%rdi), %eax
-; AVX512F-NEXT:    kmovw %eax, %k1
-; AVX512F-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
-; AVX512F-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
-; AVX512F-NEXT:    retq
-;
-; AVX512BW-LABEL: load_sext_8i1_to_8i32:
-; AVX512BW:       # %bb.0: # %entry
-; AVX512BW-NEXT:    movzbl (%rdi), %eax
-; AVX512BW-NEXT:    kmovd %eax, %k1
-; AVX512BW-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
-; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
-; AVX512BW-NEXT:    retq
+; AVX512-LABEL: load_sext_8i1_to_8i32:
+; AVX512:       # %bb.0: # %entry
+; AVX512-NEXT:    kmovw (%rdi), %k1
+; AVX512-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; AVX512-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
+; AVX512-NEXT:    retq
 ;
 ; X32-SSE2-LABEL: load_sext_8i1_to_8i32:
 ; X32-SSE2:       # %bb.0: # %entry
diff --git a/test/CodeGen/X86/vector-sext.ll b/test/CodeGen/X86/vector-sext.ll
index 42fd2d1c74827dd5842281781b76c1327954370d..a5784f326d62e780a1066965f9f8f2a7c32fb795 100644
--- a/test/CodeGen/X86/vector-sext.ll
+++ b/test/CodeGen/X86/vector-sext.ll
@@ -493,33 +493,25 @@ define <4 x i64> @sext_16i8_to_4i64(<16 x i8> %A) nounwind uwtable readnone ssp
 ; SSE2-LABEL: sext_16i8_to_4i64:
 ; SSE2:       # %bb.0: # %entry
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
-; SSE2-NEXT:    pxor %xmm3, %xmm3
-; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    pcmpgtd %xmm2, %xmm1
-; SSE2-NEXT:    psrad $24, %xmm2
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm0[0,2,2,3,4,5,6,7]
-; SSE2-NEXT:    pcmpgtd %xmm1, %xmm3
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
 ; SSE2-NEXT:    psrad $24, %xmm1
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
-; SSE2-NEXT:    movdqa %xmm2, %xmm0
+; SSE2-NEXT:    pxor %xmm2, %xmm2
+; SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
+; SSE2-NEXT:    movdqa %xmm1, %xmm0
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
+; SSE2-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]
 ; SSE2-NEXT:    retq
 ;
 ; SSSE3-LABEL: sext_16i8_to_4i64:
 ; SSSE3:       # %bb.0: # %entry
 ; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
-; SSSE3-NEXT:    pxor %xmm3, %xmm3
-; SSSE3-NEXT:    pxor %xmm1, %xmm1
-; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm1
-; SSSE3-NEXT:    psrad $24, %xmm2
-; SSSE3-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
-; SSSE3-NEXT:    pshuflw {{.*#+}} xmm1 = xmm0[0,2,2,3,4,5,6,7]
-; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm3
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
 ; SSSE3-NEXT:    psrad $24, %xmm1
-; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
-; SSSE3-NEXT:    movdqa %xmm2, %xmm0
+; SSSE3-NEXT:    pxor %xmm2, %xmm2
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm2
+; SSSE3-NEXT:    movdqa %xmm1, %xmm0
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
+; SSSE3-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: sext_16i8_to_4i64:
@@ -551,17 +543,13 @@ define <4 x i64> @sext_16i8_to_4i64(<16 x i8> %A) nounwind uwtable readnone ssp
 ; X32-SSE2-LABEL: sext_16i8_to_4i64:
 ; X32-SSE2:       # %bb.0: # %entry
 ; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
-; X32-SSE2-NEXT:    pxor %xmm3, %xmm3
-; X32-SSE2-NEXT:    pxor %xmm1, %xmm1
-; X32-SSE2-NEXT:    pcmpgtd %xmm2, %xmm1
-; X32-SSE2-NEXT:    psrad $24, %xmm2
-; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
-; X32-SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm0[0,2,2,3,4,5,6,7]
-; X32-SSE2-NEXT:    pcmpgtd %xmm1, %xmm3
+; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
 ; X32-SSE2-NEXT:    psrad $24, %xmm1
-; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
-; X32-SSE2-NEXT:    movdqa %xmm2, %xmm0
+; X32-SSE2-NEXT:    pxor %xmm2, %xmm2
+; X32-SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
+; X32-SSE2-NEXT:    movdqa %xmm1, %xmm0
+; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
+; X32-SSE2-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]
 ; X32-SSE2-NEXT:    retl
 ;
 ; X32-SSE41-LABEL: sext_16i8_to_4i64:
@@ -580,57 +568,41 @@ entry:
 define <8 x i64> @sext_16i8_to_8i64(<16 x i8> %A) nounwind uwtable readnone ssp {
 ; SSE2-LABEL: sext_16i8_to_8i64:
 ; SSE2:       # %bb.0: # %entry
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,2,3]
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3]
-; SSE2-NEXT:    pxor %xmm5, %xmm5
-; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    pcmpgtd %xmm4, %xmm1
-; SSE2-NEXT:    psrad $24, %xmm4
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1]
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
-; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    pcmpgtd %xmm2, %xmm1
-; SSE2-NEXT:    psrad $24, %xmm2
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm0[0,2,2,3,4,5,6,7]
-; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    pcmpgtd %xmm1, %xmm0
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
 ; SSE2-NEXT:    psrad $24, %xmm1
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm3 = xmm3[0,2,2,3,4,5,6,7]
-; SSE2-NEXT:    pcmpgtd %xmm3, %xmm5
+; SSE2-NEXT:    pxor %xmm5, %xmm5
+; SSE2-NEXT:    pxor %xmm2, %xmm2
+; SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
+; SSE2-NEXT:    movdqa %xmm1, %xmm4
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1]
+; SSE2-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]
+; SSE2-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
 ; SSE2-NEXT:    psrad $24, %xmm3
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1]
+; SSE2-NEXT:    pcmpgtd %xmm3, %xmm5
+; SSE2-NEXT:    movdqa %xmm3, %xmm2
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1]
+; SSE2-NEXT:    punpckhdq {{.*#+}} xmm3 = xmm3[2],xmm5[2],xmm3[3],xmm5[3]
 ; SSE2-NEXT:    movdqa %xmm4, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSSE3-LABEL: sext_16i8_to_8i64:
 ; SSSE3:       # %bb.0: # %entry
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,2,3]
 ; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3]
-; SSSE3-NEXT:    pxor %xmm5, %xmm5
-; SSSE3-NEXT:    pxor %xmm1, %xmm1
-; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm1
-; SSSE3-NEXT:    psrad $24, %xmm4
-; SSSE3-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1]
-; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
-; SSSE3-NEXT:    pxor %xmm1, %xmm1
-; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm1
-; SSSE3-NEXT:    psrad $24, %xmm2
-; SSSE3-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
-; SSSE3-NEXT:    pshuflw {{.*#+}} xmm1 = xmm0[0,2,2,3,4,5,6,7]
-; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm0
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
 ; SSSE3-NEXT:    psrad $24, %xmm1
-; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
-; SSSE3-NEXT:    pshuflw {{.*#+}} xmm3 = xmm3[0,2,2,3,4,5,6,7]
-; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm5
+; SSSE3-NEXT:    pxor %xmm5, %xmm5
+; SSSE3-NEXT:    pxor %xmm2, %xmm2
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm2
+; SSSE3-NEXT:    movdqa %xmm1, %xmm4
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1]
+; SSSE3-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]
+; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
 ; SSSE3-NEXT:    psrad $24, %xmm3
-; SSSE3-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1]
+; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm5
+; SSSE3-NEXT:    movdqa %xmm3, %xmm2
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1]
+; SSSE3-NEXT:    punpckhdq {{.*#+}} xmm3 = xmm3[2],xmm5[2],xmm3[3],xmm5[3]
 ; SSSE3-NEXT:    movdqa %xmm4, %xmm0
 ; SSSE3-NEXT:    retq
 ;
@@ -676,29 +648,21 @@ define <8 x i64> @sext_16i8_to_8i64(<16 x i8> %A) nounwind uwtable readnone ssp
 ;
 ; X32-SSE2-LABEL: sext_16i8_to_8i64:
 ; X32-SSE2:       # %bb.0: # %entry
-; X32-SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm0[1,1,2,3]
 ; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3]
-; X32-SSE2-NEXT:    pxor %xmm5, %xmm5
-; X32-SSE2-NEXT:    pxor %xmm1, %xmm1
-; X32-SSE2-NEXT:    pcmpgtd %xmm4, %xmm1
-; X32-SSE2-NEXT:    psrad $24, %xmm4
-; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1]
-; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
-; X32-SSE2-NEXT:    pxor %xmm1, %xmm1
-; X32-SSE2-NEXT:    pcmpgtd %xmm2, %xmm1
-; X32-SSE2-NEXT:    psrad $24, %xmm2
-; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
-; X32-SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm0[0,2,2,3,4,5,6,7]
-; X32-SSE2-NEXT:    pxor %xmm0, %xmm0
-; X32-SSE2-NEXT:    pcmpgtd %xmm1, %xmm0
+; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
 ; X32-SSE2-NEXT:    psrad $24, %xmm1
-; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
-; X32-SSE2-NEXT:    pshuflw {{.*#+}} xmm3 = xmm3[0,2,2,3,4,5,6,7]
-; X32-SSE2-NEXT:    pcmpgtd %xmm3, %xmm5
+; X32-SSE2-NEXT:    pxor %xmm5, %xmm5
+; X32-SSE2-NEXT:    pxor %xmm2, %xmm2
+; X32-SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
+; X32-SSE2-NEXT:    movdqa %xmm1, %xmm4
+; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1]
+; X32-SSE2-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]
+; X32-SSE2-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
 ; X32-SSE2-NEXT:    psrad $24, %xmm3
-; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1]
+; X32-SSE2-NEXT:    pcmpgtd %xmm3, %xmm5
+; X32-SSE2-NEXT:    movdqa %xmm3, %xmm2
+; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1]
+; X32-SSE2-NEXT:    punpckhdq {{.*#+}} xmm3 = xmm3[2],xmm5[2],xmm3[3],xmm5[3]
 ; X32-SSE2-NEXT:    movdqa %xmm4, %xmm0
 ; X32-SSE2-NEXT:    retl
 ;
@@ -974,32 +938,24 @@ entry:
 define <4 x i64> @sext_8i16_to_4i64(<8 x i16> %A) nounwind uwtable readnone ssp {
 ; SSE2-LABEL: sext_8i16_to_4i64:
 ; SSE2:       # %bb.0: # %entry
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
-; SSE2-NEXT:    pxor %xmm3, %xmm3
-; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    pcmpgtd %xmm2, %xmm1
-; SSE2-NEXT:    psrad $16, %xmm2
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm0[0,2,2,3,4,5,6,7]
-; SSE2-NEXT:    pcmpgtd %xmm1, %xmm3
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
 ; SSE2-NEXT:    psrad $16, %xmm1
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
-; SSE2-NEXT:    movdqa %xmm2, %xmm0
+; SSE2-NEXT:    pxor %xmm2, %xmm2
+; SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
+; SSE2-NEXT:    movdqa %xmm1, %xmm0
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
+; SSE2-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]
 ; SSE2-NEXT:    retq
 ;
 ; SSSE3-LABEL: sext_8i16_to_4i64:
 ; SSSE3:       # %bb.0: # %entry
-; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
-; SSSE3-NEXT:    pxor %xmm3, %xmm3
-; SSSE3-NEXT:    pxor %xmm1, %xmm1
-; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm1
-; SSSE3-NEXT:    psrad $16, %xmm2
-; SSSE3-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
-; SSSE3-NEXT:    pshuflw {{.*#+}} xmm1 = xmm0[0,2,2,3,4,5,6,7]
-; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm3
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
 ; SSSE3-NEXT:    psrad $16, %xmm1
-; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
-; SSSE3-NEXT:    movdqa %xmm2, %xmm0
+; SSSE3-NEXT:    pxor %xmm2, %xmm2
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm2
+; SSSE3-NEXT:    movdqa %xmm1, %xmm0
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
+; SSSE3-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: sext_8i16_to_4i64:
@@ -1030,17 +986,13 @@ define <4 x i64> @sext_8i16_to_4i64(<8 x i16> %A) nounwind uwtable readnone ssp
 ;
 ; X32-SSE2-LABEL: sext_8i16_to_4i64:
 ; X32-SSE2:       # %bb.0: # %entry
-; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
-; X32-SSE2-NEXT:    pxor %xmm3, %xmm3
-; X32-SSE2-NEXT:    pxor %xmm1, %xmm1
-; X32-SSE2-NEXT:    pcmpgtd %xmm2, %xmm1
-; X32-SSE2-NEXT:    psrad $16, %xmm2
-; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
-; X32-SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm0[0,2,2,3,4,5,6,7]
-; X32-SSE2-NEXT:    pcmpgtd %xmm1, %xmm3
+; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
 ; X32-SSE2-NEXT:    psrad $16, %xmm1
-; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
-; X32-SSE2-NEXT:    movdqa %xmm2, %xmm0
+; X32-SSE2-NEXT:    pxor %xmm2, %xmm2
+; X32-SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
+; X32-SSE2-NEXT:    movdqa %xmm1, %xmm0
+; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
+; X32-SSE2-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]
 ; X32-SSE2-NEXT:    retl
 ;
 ; X32-SSE41-LABEL: sext_8i16_to_4i64:
@@ -1059,53 +1011,39 @@ entry:
 define <8 x i64> @sext_8i16_to_8i64(<8 x i16> %A) nounwind uwtable readnone ssp {
 ; SSE2-LABEL: sext_8i16_to_8i64:
 ; SSE2:       # %bb.0: # %entry
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3]
-; SSE2-NEXT:    pxor %xmm5, %xmm5
-; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    pcmpgtd %xmm4, %xmm1
-; SSE2-NEXT:    psrad $16, %xmm4
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1]
-; SSE2-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
-; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    pcmpgtd %xmm2, %xmm1
-; SSE2-NEXT:    psrad $16, %xmm2
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm0[0,2,2,3,4,5,6,7]
-; SSE2-NEXT:    pxor %xmm3, %xmm3
-; SSE2-NEXT:    pcmpgtd %xmm1, %xmm3
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
 ; SSE2-NEXT:    psrad $16, %xmm1
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; SSE2-NEXT:    pshuflw {{.*#+}} xmm3 = xmm0[0,2,2,3,4,5,6,7]
-; SSE2-NEXT:    pcmpgtd %xmm3, %xmm5
+; SSE2-NEXT:    pxor %xmm5, %xmm5
+; SSE2-NEXT:    pxor %xmm2, %xmm2
+; SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
+; SSE2-NEXT:    movdqa %xmm1, %xmm4
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1]
+; SSE2-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]
+; SSE2-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
 ; SSE2-NEXT:    psrad $16, %xmm3
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1]
+; SSE2-NEXT:    pcmpgtd %xmm3, %xmm5
+; SSE2-NEXT:    movdqa %xmm3, %xmm2
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1]
+; SSE2-NEXT:    punpckhdq {{.*#+}} xmm3 = xmm3[2],xmm5[2],xmm3[3],xmm5[3]
 ; SSE2-NEXT:    movdqa %xmm4, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSSE3-LABEL: sext_8i16_to_8i64:
 ; SSSE3:       # %bb.0: # %entry
-; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3]
-; SSSE3-NEXT:    pxor %xmm5, %xmm5
-; SSSE3-NEXT:    pxor %xmm1, %xmm1
-; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm1
-; SSSE3-NEXT:    psrad $16, %xmm4
-; SSSE3-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1]
-; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
-; SSSE3-NEXT:    pxor %xmm1, %xmm1
-; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm1
-; SSSE3-NEXT:    psrad $16, %xmm2
-; SSSE3-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
-; SSSE3-NEXT:    pshuflw {{.*#+}} xmm1 = xmm0[0,2,2,3,4,5,6,7]
-; SSSE3-NEXT:    pxor %xmm3, %xmm3
-; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm3
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
 ; SSSE3-NEXT:    psrad $16, %xmm1
-; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; SSSE3-NEXT:    pshuflw {{.*#+}} xmm3 = xmm0[0,2,2,3,4,5,6,7]
-; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm5
+; SSSE3-NEXT:    pxor %xmm5, %xmm5
+; SSSE3-NEXT:    pxor %xmm2, %xmm2
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm2
+; SSSE3-NEXT:    movdqa %xmm1, %xmm4
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1]
+; SSSE3-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]
+; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
 ; SSSE3-NEXT:    psrad $16, %xmm3
-; SSSE3-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1]
+; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm5
+; SSSE3-NEXT:    movdqa %xmm3, %xmm2
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1]
+; SSSE3-NEXT:    punpckhdq {{.*#+}} xmm3 = xmm3[2],xmm5[2],xmm3[3],xmm5[3]
 ; SSSE3-NEXT:    movdqa %xmm4, %xmm0
 ; SSSE3-NEXT:    retq
 ;
@@ -1150,27 +1088,20 @@ define <8 x i64> @sext_8i16_to_8i64(<8 x i16> %A) nounwind uwtable readnone ssp
 ;
 ; X32-SSE2-LABEL: sext_8i16_to_8i64:
 ; X32-SSE2:       # %bb.0: # %entry
-; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3]
-; X32-SSE2-NEXT:    pxor %xmm5, %xmm5
-; X32-SSE2-NEXT:    pxor %xmm1, %xmm1
-; X32-SSE2-NEXT:    pcmpgtd %xmm4, %xmm1
-; X32-SSE2-NEXT:    psrad $16, %xmm4
-; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1]
-; X32-SSE2-NEXT:    punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
-; X32-SSE2-NEXT:    pxor %xmm1, %xmm1
-; X32-SSE2-NEXT:    pcmpgtd %xmm2, %xmm1
-; X32-SSE2-NEXT:    psrad $16, %xmm2
-; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
-; X32-SSE2-NEXT:    pshuflw {{.*#+}} xmm1 = xmm0[0,2,2,3,4,5,6,7]
-; X32-SSE2-NEXT:    pxor %xmm3, %xmm3
-; X32-SSE2-NEXT:    pcmpgtd %xmm1, %xmm3
+; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
 ; X32-SSE2-NEXT:    psrad $16, %xmm1
-; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
-; X32-SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; X32-SSE2-NEXT:    pshuflw {{.*#+}} xmm3 = xmm0[0,2,2,3,4,5,6,7]
-; X32-SSE2-NEXT:    pcmpgtd %xmm3, %xmm5
+; X32-SSE2-NEXT:    pxor %xmm5, %xmm5
+; X32-SSE2-NEXT:    pxor %xmm2, %xmm2
+; X32-SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
+; X32-SSE2-NEXT:    movdqa %xmm1, %xmm4
+; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1]
+; X32-SSE2-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]
+; X32-SSE2-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
 ; X32-SSE2-NEXT:    psrad $16, %xmm3
-; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1]
+; X32-SSE2-NEXT:    pcmpgtd %xmm3, %xmm5
+; X32-SSE2-NEXT:    movdqa %xmm3, %xmm2
+; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1]
+; X32-SSE2-NEXT:    punpckhdq {{.*#+}} xmm3 = xmm3[2],xmm5[2],xmm3[3],xmm5[3]
 ; X32-SSE2-NEXT:    movdqa %xmm4, %xmm0
 ; X32-SSE2-NEXT:    retl
 ;
@@ -1457,23 +1388,13 @@ define <2 x i64> @load_sext_2i1_to_2i64(<2 x i1> *%ptr) {
 ; AVX2-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
 ; AVX2-NEXT:    retq
 ;
-; AVX512F-LABEL: load_sext_2i1_to_2i64:
-; AVX512F:       # %bb.0: # %entry
-; AVX512F-NEXT:    movzbl (%rdi), %eax
-; AVX512F-NEXT:    kmovw %eax, %k1
-; AVX512F-NEXT:    vpternlogq $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
-; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
-; AVX512F-NEXT:    vzeroupper
-; AVX512F-NEXT:    retq
-;
-; AVX512BW-LABEL: load_sext_2i1_to_2i64:
-; AVX512BW:       # %bb.0: # %entry
-; AVX512BW-NEXT:    movzbl (%rdi), %eax
-; AVX512BW-NEXT:    kmovd %eax, %k1
-; AVX512BW-NEXT:    vpternlogq $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
-; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
-; AVX512BW-NEXT:    vzeroupper
-; AVX512BW-NEXT:    retq
+; AVX512-LABEL: load_sext_2i1_to_2i64:
+; AVX512:       # %bb.0: # %entry
+; AVX512-NEXT:    kmovw (%rdi), %k1
+; AVX512-NEXT:    vpternlogq $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; AVX512-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512-NEXT:    vzeroupper
+; AVX512-NEXT:    retq
 ;
 ; X32-SSE2-LABEL: load_sext_2i1_to_2i64:
 ; X32-SSE2:       # %bb.0: # %entry
@@ -1677,23 +1598,13 @@ define <4 x i32> @load_sext_4i1_to_4i32(<4 x i1> *%ptr) {
 ; AVX2-NEXT:    vpinsrd $3, %eax, %xmm0, %xmm0
 ; AVX2-NEXT:    retq
 ;
-; AVX512F-LABEL: load_sext_4i1_to_4i32:
-; AVX512F:       # %bb.0: # %entry
-; AVX512F-NEXT:    movzbl (%rdi), %eax
-; AVX512F-NEXT:    kmovw %eax, %k1
-; AVX512F-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
-; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
-; AVX512F-NEXT:    vzeroupper
-; AVX512F-NEXT:    retq
-;
-; AVX512BW-LABEL: load_sext_4i1_to_4i32:
-; AVX512BW:       # %bb.0: # %entry
-; AVX512BW-NEXT:    movzbl (%rdi), %eax
-; AVX512BW-NEXT:    kmovd %eax, %k1
-; AVX512BW-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
-; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
-; AVX512BW-NEXT:    vzeroupper
-; AVX512BW-NEXT:    retq
+; AVX512-LABEL: load_sext_4i1_to_4i32:
+; AVX512:       # %bb.0: # %entry
+; AVX512-NEXT:    kmovw (%rdi), %k1
+; AVX512-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; AVX512-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512-NEXT:    vzeroupper
+; AVX512-NEXT:    retq
 ;
 ; X32-SSE2-LABEL: load_sext_4i1_to_4i32:
 ; X32-SSE2:       # %bb.0: # %entry
@@ -1915,21 +1826,12 @@ define <4 x i64> @load_sext_4i1_to_4i64(<4 x i1> *%ptr) {
 ; AVX2-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm0
 ; AVX2-NEXT:    retq
 ;
-; AVX512F-LABEL: load_sext_4i1_to_4i64:
-; AVX512F:       # %bb.0: # %entry
-; AVX512F-NEXT:    movzbl (%rdi), %eax
-; AVX512F-NEXT:    kmovw %eax, %k1
-; AVX512F-NEXT:    vpternlogq $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
-; AVX512F-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
-; AVX512F-NEXT:    retq
-;
-; AVX512BW-LABEL: load_sext_4i1_to_4i64:
-; AVX512BW:       # %bb.0: # %entry
-; AVX512BW-NEXT:    movzbl (%rdi), %eax
-; AVX512BW-NEXT:    kmovd %eax, %k1
-; AVX512BW-NEXT:    vpternlogq $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
-; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
-; AVX512BW-NEXT:    retq
+; AVX512-LABEL: load_sext_4i1_to_4i64:
+; AVX512:       # %bb.0: # %entry
+; AVX512-NEXT:    kmovw (%rdi), %k1
+; AVX512-NEXT:    vpternlogq $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; AVX512-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
+; AVX512-NEXT:    retq
 ;
 ; X32-SSE2-LABEL: load_sext_4i1_to_4i64:
 ; X32-SSE2:       # %bb.0: # %entry
@@ -1990,30 +1892,28 @@ entry:
 define <4 x i64> @load_sext_4i8_to_4i64(<4 x i8> *%ptr) {
 ; SSE2-LABEL: load_sext_4i8_to_4i64:
 ; SSE2:       # %bb.0: # %entry
-; SSE2-NEXT:    movsbq 1(%rdi), %rax
-; SSE2-NEXT:    movq %rax, %xmm1
-; SSE2-NEXT:    movsbq (%rdi), %rax
-; SSE2-NEXT:    movq %rax, %xmm0
-; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; SSE2-NEXT:    movsbq 3(%rdi), %rax
-; SSE2-NEXT:    movq %rax, %xmm2
-; SSE2-NEXT:    movsbq 2(%rdi), %rax
-; SSE2-NEXT:    movq %rax, %xmm1
-; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
+; SSE2-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
+; SSE2-NEXT:    psrad $24, %xmm1
+; SSE2-NEXT:    pxor %xmm2, %xmm2
+; SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
+; SSE2-NEXT:    movdqa %xmm1, %xmm0
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
+; SSE2-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]
 ; SSE2-NEXT:    retq
 ;
 ; SSSE3-LABEL: load_sext_4i8_to_4i64:
 ; SSSE3:       # %bb.0: # %entry
-; SSSE3-NEXT:    movsbq 1(%rdi), %rax
-; SSSE3-NEXT:    movq %rax, %xmm1
-; SSSE3-NEXT:    movsbq (%rdi), %rax
-; SSSE3-NEXT:    movq %rax, %xmm0
-; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; SSSE3-NEXT:    movsbq 3(%rdi), %rax
-; SSSE3-NEXT:    movq %rax, %xmm2
-; SSSE3-NEXT:    movsbq 2(%rdi), %rax
-; SSSE3-NEXT:    movq %rax, %xmm1
-; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
+; SSSE3-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
+; SSSE3-NEXT:    psrad $24, %xmm1
+; SSSE3-NEXT:    pxor %xmm2, %xmm2
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm2
+; SSSE3-NEXT:    movdqa %xmm1, %xmm0
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
+; SSSE3-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: load_sext_4i8_to_4i64:
@@ -2024,10 +1924,8 @@ define <4 x i64> @load_sext_4i8_to_4i64(<4 x i8> *%ptr) {
 ;
 ; AVX1-LABEL: load_sext_4i8_to_4i64:
 ; AVX1:       # %bb.0: # %entry
-; AVX1-NEXT:    vpmovsxbd (%rdi), %xmm0
-; AVX1-NEXT:    vpmovsxdq %xmm0, %xmm1
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; AVX1-NEXT:    vpmovsxdq %xmm0, %xmm0
+; AVX1-NEXT:    vpmovsxbq 2(%rdi), %xmm0
+; AVX1-NEXT:    vpmovsxbq (%rdi), %xmm1
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; AVX1-NEXT:    retq
 ;
@@ -2044,28 +1942,15 @@ define <4 x i64> @load_sext_4i8_to_4i64(<4 x i8> *%ptr) {
 ; X32-SSE2-LABEL: load_sext_4i8_to_4i64:
 ; X32-SSE2:       # %bb.0: # %entry
 ; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-SSE2-NEXT:    movsbl 1(%eax), %ecx
-; X32-SSE2-NEXT:    movd %ecx, %xmm1
-; X32-SSE2-NEXT:    sarl $31, %ecx
-; X32-SSE2-NEXT:    movd %ecx, %xmm0
-; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
-; X32-SSE2-NEXT:    movsbl (%eax), %ecx
-; X32-SSE2-NEXT:    movd %ecx, %xmm0
-; X32-SSE2-NEXT:    sarl $31, %ecx
-; X32-SSE2-NEXT:    movd %ecx, %xmm2
+; X32-SSE2-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
+; X32-SSE2-NEXT:    psrad $24, %xmm1
+; X32-SSE2-NEXT:    pxor %xmm2, %xmm2
+; X32-SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
+; X32-SSE2-NEXT:    movdqa %xmm1, %xmm0
 ; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
-; X32-SSE2-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; X32-SSE2-NEXT:    movsbl 3(%eax), %ecx
-; X32-SSE2-NEXT:    movd %ecx, %xmm2
-; X32-SSE2-NEXT:    sarl $31, %ecx
-; X32-SSE2-NEXT:    movd %ecx, %xmm1
-; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
-; X32-SSE2-NEXT:    movsbl 2(%eax), %eax
-; X32-SSE2-NEXT:    movd %eax, %xmm1
-; X32-SSE2-NEXT:    sarl $31, %eax
-; X32-SSE2-NEXT:    movd %eax, %xmm3
-; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
-; X32-SSE2-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
+; X32-SSE2-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]
 ; X32-SSE2-NEXT:    retl
 ;
 ; X32-SSE41-LABEL: load_sext_4i8_to_4i64:
@@ -2083,20 +1968,24 @@ entry:
 define <2 x i64> @load_sext_4i8_to_4i64_extract(<4 x i8> *%ptr) {
 ; SSE2-LABEL: load_sext_4i8_to_4i64_extract:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    movsbq 3(%rdi), %rax
-; SSE2-NEXT:    movq %rax, %xmm1
-; SSE2-NEXT:    movsbq 2(%rdi), %rax
-; SSE2-NEXT:    movq %rax, %xmm0
-; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; SSE2-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
+; SSE2-NEXT:    psrad $24, %xmm0
+; SSE2-NEXT:    pxor %xmm1, %xmm1
+; SSE2-NEXT:    pcmpgtd %xmm0, %xmm1
+; SSE2-NEXT:    punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
 ; SSE2-NEXT:    retq
 ;
 ; SSSE3-LABEL: load_sext_4i8_to_4i64_extract:
 ; SSSE3:       # %bb.0:
-; SSSE3-NEXT:    movsbq 3(%rdi), %rax
-; SSSE3-NEXT:    movq %rax, %xmm1
-; SSSE3-NEXT:    movsbq 2(%rdi), %rax
-; SSSE3-NEXT:    movq %rax, %xmm0
-; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; SSSE3-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
+; SSSE3-NEXT:    psrad $24, %xmm0
+; SSSE3-NEXT:    pxor %xmm1, %xmm1
+; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm1
+; SSSE3-NEXT:    punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: load_sext_4i8_to_4i64_extract:
@@ -2106,9 +1995,7 @@ define <2 x i64> @load_sext_4i8_to_4i64_extract(<4 x i8> *%ptr) {
 ;
 ; AVX1-LABEL: load_sext_4i8_to_4i64_extract:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpmovsxbd (%rdi), %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; AVX1-NEXT:    vpmovsxdq %xmm0, %xmm0
+; AVX1-NEXT:    vpmovsxbq 2(%rdi), %xmm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: load_sext_4i8_to_4i64_extract:
@@ -2128,17 +2015,13 @@ define <2 x i64> @load_sext_4i8_to_4i64_extract(<4 x i8> *%ptr) {
 ; X32-SSE2-LABEL: load_sext_4i8_to_4i64_extract:
 ; X32-SSE2:       # %bb.0:
 ; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-SSE2-NEXT:    movsbl 3(%eax), %ecx
-; X32-SSE2-NEXT:    movd %ecx, %xmm1
-; X32-SSE2-NEXT:    sarl $31, %ecx
-; X32-SSE2-NEXT:    movd %ecx, %xmm0
-; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
-; X32-SSE2-NEXT:    movsbl 2(%eax), %eax
-; X32-SSE2-NEXT:    movd %eax, %xmm0
-; X32-SSE2-NEXT:    sarl $31, %eax
-; X32-SSE2-NEXT:    movd %eax, %xmm2
-; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
-; X32-SSE2-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; X32-SSE2-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
+; X32-SSE2-NEXT:    psrad $24, %xmm0
+; X32-SSE2-NEXT:    pxor %xmm1, %xmm1
+; X32-SSE2-NEXT:    pcmpgtd %xmm0, %xmm1
+; X32-SSE2-NEXT:    punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
 ; X32-SSE2-NEXT:    retl
 ;
 ; X32-SSE41-LABEL: load_sext_4i8_to_4i64_extract:
@@ -2344,8 +2227,7 @@ define <8 x i16> @load_sext_8i1_to_8i16(<8 x i1> *%ptr) {
 ;
 ; AVX512F-LABEL: load_sext_8i1_to_8i16:
 ; AVX512F:       # %bb.0: # %entry
-; AVX512F-NEXT:    movzbl (%rdi), %eax
-; AVX512F-NEXT:    kmovw %eax, %k1
+; AVX512F-NEXT:    kmovw (%rdi), %k1
 ; AVX512F-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
 ; AVX512F-NEXT:    vpmovdw %zmm0, %ymm0
 ; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
@@ -2354,8 +2236,7 @@ define <8 x i16> @load_sext_8i1_to_8i16(<8 x i1> *%ptr) {
 ;
 ; AVX512BW-LABEL: load_sext_8i1_to_8i16:
 ; AVX512BW:       # %bb.0: # %entry
-; AVX512BW-NEXT:    movzbl (%rdi), %eax
-; AVX512BW-NEXT:    kmovd %eax, %k0
+; AVX512BW-NEXT:    kmovw (%rdi), %k0
 ; AVX512BW-NEXT:    vpmovm2w %k0, %zmm0
 ; AVX512BW-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
 ; AVX512BW-NEXT:    vzeroupper
@@ -2492,50 +2373,42 @@ entry:
 define <8 x i64> @load_sext_8i8_to_8i64(<8 x i8> *%ptr) {
 ; SSE2-LABEL: load_sext_8i8_to_8i64:
 ; SSE2:       # %bb.0: # %entry
-; SSE2-NEXT:    movsbq 1(%rdi), %rax
-; SSE2-NEXT:    movq %rax, %xmm1
-; SSE2-NEXT:    movsbq (%rdi), %rax
-; SSE2-NEXT:    movq %rax, %xmm0
-; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; SSE2-NEXT:    movsbq 3(%rdi), %rax
-; SSE2-NEXT:    movq %rax, %xmm2
-; SSE2-NEXT:    movsbq 2(%rdi), %rax
-; SSE2-NEXT:    movq %rax, %xmm1
-; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
-; SSE2-NEXT:    movsbq 5(%rdi), %rax
-; SSE2-NEXT:    movq %rax, %xmm3
-; SSE2-NEXT:    movsbq 4(%rdi), %rax
-; SSE2-NEXT:    movq %rax, %xmm2
-; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; SSE2-NEXT:    movsbq 7(%rdi), %rax
-; SSE2-NEXT:    movq %rax, %xmm4
-; SSE2-NEXT:    movsbq 6(%rdi), %rax
-; SSE2-NEXT:    movq %rax, %xmm3
-; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm4[0]
+; SSE2-NEXT:    movq {{.*#+}} xmm2 = mem[0],zero
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
+; SSE2-NEXT:    psrad $24, %xmm1
+; SSE2-NEXT:    pxor %xmm4, %xmm4
+; SSE2-NEXT:    pxor %xmm3, %xmm3
+; SSE2-NEXT:    pcmpgtd %xmm1, %xmm3
+; SSE2-NEXT:    movdqa %xmm1, %xmm0
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
+; SSE2-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm3[2],xmm1[3],xmm3[3]
+; SSE2-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
+; SSE2-NEXT:    psrad $24, %xmm3
+; SSE2-NEXT:    pcmpgtd %xmm3, %xmm4
+; SSE2-NEXT:    movdqa %xmm3, %xmm2
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1]
+; SSE2-NEXT:    punpckhdq {{.*#+}} xmm3 = xmm3[2],xmm4[2],xmm3[3],xmm4[3]
 ; SSE2-NEXT:    retq
 ;
 ; SSSE3-LABEL: load_sext_8i8_to_8i64:
 ; SSSE3:       # %bb.0: # %entry
-; SSSE3-NEXT:    movsbq 1(%rdi), %rax
-; SSSE3-NEXT:    movq %rax, %xmm1
-; SSSE3-NEXT:    movsbq (%rdi), %rax
-; SSSE3-NEXT:    movq %rax, %xmm0
-; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; SSSE3-NEXT:    movsbq 3(%rdi), %rax
-; SSSE3-NEXT:    movq %rax, %xmm2
-; SSSE3-NEXT:    movsbq 2(%rdi), %rax
-; SSSE3-NEXT:    movq %rax, %xmm1
-; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
-; SSSE3-NEXT:    movsbq 5(%rdi), %rax
-; SSSE3-NEXT:    movq %rax, %xmm3
-; SSSE3-NEXT:    movsbq 4(%rdi), %rax
-; SSSE3-NEXT:    movq %rax, %xmm2
-; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; SSSE3-NEXT:    movsbq 7(%rdi), %rax
-; SSSE3-NEXT:    movq %rax, %xmm4
-; SSSE3-NEXT:    movsbq 6(%rdi), %rax
-; SSSE3-NEXT:    movq %rax, %xmm3
-; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm4[0]
+; SSSE3-NEXT:    movq {{.*#+}} xmm2 = mem[0],zero
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
+; SSSE3-NEXT:    psrad $24, %xmm1
+; SSSE3-NEXT:    pxor %xmm4, %xmm4
+; SSSE3-NEXT:    pxor %xmm3, %xmm3
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm3
+; SSSE3-NEXT:    movdqa %xmm1, %xmm0
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
+; SSSE3-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm3[2],xmm1[3],xmm3[3]
+; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
+; SSSE3-NEXT:    psrad $24, %xmm3
+; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm4
+; SSSE3-NEXT:    movdqa %xmm3, %xmm2
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1]
+; SSSE3-NEXT:    punpckhdq {{.*#+}} xmm3 = xmm3[2],xmm4[2],xmm3[3],xmm4[3]
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: load_sext_8i8_to_8i64:
@@ -2548,15 +2421,11 @@ define <8 x i64> @load_sext_8i8_to_8i64(<8 x i8> *%ptr) {
 ;
 ; AVX1-LABEL: load_sext_8i8_to_8i64:
 ; AVX1:       # %bb.0: # %entry
-; AVX1-NEXT:    vpmovsxbd (%rdi), %xmm0
-; AVX1-NEXT:    vpmovsxdq %xmm0, %xmm1
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; AVX1-NEXT:    vpmovsxdq %xmm0, %xmm0
-; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
-; AVX1-NEXT:    vpmovsxbd 4(%rdi), %xmm1
-; AVX1-NEXT:    vpmovsxdq %xmm1, %xmm2
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
-; AVX1-NEXT:    vpmovsxdq %xmm1, %xmm1
+; AVX1-NEXT:    vpmovsxbq 6(%rdi), %xmm1
+; AVX1-NEXT:    vpmovsxbq 4(%rdi), %xmm2
+; AVX1-NEXT:    vpmovsxbq 2(%rdi), %xmm0
+; AVX1-NEXT:    vpmovsxbq (%rdi), %xmm3
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm3, %ymm0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm1
 ; AVX1-NEXT:    retq
 ;
@@ -2574,50 +2443,22 @@ define <8 x i64> @load_sext_8i8_to_8i64(<8 x i8> *%ptr) {
 ; X32-SSE2-LABEL: load_sext_8i8_to_8i64:
 ; X32-SSE2:       # %bb.0: # %entry
 ; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-SSE2-NEXT:    movsbl 1(%eax), %ecx
-; X32-SSE2-NEXT:    movd %ecx, %xmm1
-; X32-SSE2-NEXT:    sarl $31, %ecx
-; X32-SSE2-NEXT:    movd %ecx, %xmm0
-; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
-; X32-SSE2-NEXT:    movsbl (%eax), %ecx
-; X32-SSE2-NEXT:    movd %ecx, %xmm0
-; X32-SSE2-NEXT:    sarl $31, %ecx
-; X32-SSE2-NEXT:    movd %ecx, %xmm2
-; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
-; X32-SSE2-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; X32-SSE2-NEXT:    movsbl 3(%eax), %ecx
-; X32-SSE2-NEXT:    movd %ecx, %xmm2
-; X32-SSE2-NEXT:    sarl $31, %ecx
-; X32-SSE2-NEXT:    movd %ecx, %xmm1
-; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
-; X32-SSE2-NEXT:    movsbl 2(%eax), %ecx
-; X32-SSE2-NEXT:    movd %ecx, %xmm1
-; X32-SSE2-NEXT:    sarl $31, %ecx
-; X32-SSE2-NEXT:    movd %ecx, %xmm3
-; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
-; X32-SSE2-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
-; X32-SSE2-NEXT:    movsbl 5(%eax), %ecx
-; X32-SSE2-NEXT:    movd %ecx, %xmm3
-; X32-SSE2-NEXT:    sarl $31, %ecx
-; X32-SSE2-NEXT:    movd %ecx, %xmm2
-; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
-; X32-SSE2-NEXT:    movsbl 4(%eax), %ecx
-; X32-SSE2-NEXT:    movd %ecx, %xmm2
-; X32-SSE2-NEXT:    sarl $31, %ecx
-; X32-SSE2-NEXT:    movd %ecx, %xmm4
+; X32-SSE2-NEXT:    movq {{.*#+}} xmm2 = mem[0],zero
+; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
+; X32-SSE2-NEXT:    psrad $24, %xmm1
+; X32-SSE2-NEXT:    pxor %xmm4, %xmm4
+; X32-SSE2-NEXT:    pxor %xmm3, %xmm3
+; X32-SSE2-NEXT:    pcmpgtd %xmm1, %xmm3
+; X32-SSE2-NEXT:    movdqa %xmm1, %xmm0
+; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
+; X32-SSE2-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm3[2],xmm1[3],xmm3[3]
+; X32-SSE2-NEXT:    punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
+; X32-SSE2-NEXT:    psrad $24, %xmm3
+; X32-SSE2-NEXT:    pcmpgtd %xmm3, %xmm4
+; X32-SSE2-NEXT:    movdqa %xmm3, %xmm2
 ; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1]
-; X32-SSE2-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; X32-SSE2-NEXT:    movsbl 7(%eax), %ecx
-; X32-SSE2-NEXT:    movd %ecx, %xmm4
-; X32-SSE2-NEXT:    sarl $31, %ecx
-; X32-SSE2-NEXT:    movd %ecx, %xmm3
-; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
-; X32-SSE2-NEXT:    movsbl 6(%eax), %eax
-; X32-SSE2-NEXT:    movd %eax, %xmm3
-; X32-SSE2-NEXT:    sarl $31, %eax
-; X32-SSE2-NEXT:    movd %eax, %xmm5
-; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1]
-; X32-SSE2-NEXT:    punpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm4[0]
+; X32-SSE2-NEXT:    punpckhdq {{.*#+}} xmm3 = xmm3[2],xmm4[2],xmm3[3],xmm4[3]
 ; X32-SSE2-NEXT:    retl
 ;
 ; X32-SSE41-LABEL: load_sext_8i8_to_8i64:
@@ -2839,21 +2680,12 @@ define <8 x i32> @load_sext_8i1_to_8i32(<8 x i1> *%ptr) {
 ; AVX2-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm0
 ; AVX2-NEXT:    retq
 ;
-; AVX512F-LABEL: load_sext_8i1_to_8i32:
-; AVX512F:       # %bb.0: # %entry
-; AVX512F-NEXT:    movzbl (%rdi), %eax
-; AVX512F-NEXT:    kmovw %eax, %k1
-; AVX512F-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
-; AVX512F-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
-; AVX512F-NEXT:    retq
-;
-; AVX512BW-LABEL: load_sext_8i1_to_8i32:
-; AVX512BW:       # %bb.0: # %entry
-; AVX512BW-NEXT:    movzbl (%rdi), %eax
-; AVX512BW-NEXT:    kmovd %eax, %k1
-; AVX512BW-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
-; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
-; AVX512BW-NEXT:    retq
+; AVX512-LABEL: load_sext_8i1_to_8i32:
+; AVX512:       # %bb.0: # %entry
+; AVX512-NEXT:    kmovw (%rdi), %k1
+; AVX512-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; AVX512-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
+; AVX512-NEXT:    retq
 ;
 ; X32-SSE2-LABEL: load_sext_8i1_to_8i32:
 ; X32-SSE2:       # %bb.0: # %entry
@@ -2951,25 +2783,21 @@ entry:
 define <8 x i32> @load_sext_8i8_to_8i32(<8 x i8> *%ptr) {
 ; SSE2-LABEL: load_sext_8i8_to_8i32:
 ; SSE2:       # %bb.0: # %entry
-; SSE2-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
-; SSE2-NEXT:    psrad $24, %xmm0
-; SSE2-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SSE2-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero
 ; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3]
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; SSE2-NEXT:    psrad $24, %xmm0
+; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
 ; SSE2-NEXT:    psrad $24, %xmm1
 ; SSE2-NEXT:    retq
 ;
 ; SSSE3-LABEL: load_sext_8i8_to_8i32:
 ; SSSE3:       # %bb.0: # %entry
-; SSSE3-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
-; SSSE3-NEXT:    psrad $24, %xmm0
-; SSSE3-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SSSE3-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero
 ; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3]
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; SSSE3-NEXT:    psrad $24, %xmm0
+; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
 ; SSSE3-NEXT:    psrad $24, %xmm1
 ; SSSE3-NEXT:    retq
 ;
@@ -2981,10 +2809,8 @@ define <8 x i32> @load_sext_8i8_to_8i32(<8 x i8> *%ptr) {
 ;
 ; AVX1-LABEL: load_sext_8i8_to_8i32:
 ; AVX1:       # %bb.0: # %entry
-; AVX1-NEXT:    vpmovsxbw (%rdi), %xmm0
-; AVX1-NEXT:    vpmovsxwd %xmm0, %xmm1
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; AVX1-NEXT:    vpmovsxwd %xmm0, %xmm0
+; AVX1-NEXT:    vpmovsxbd 4(%rdi), %xmm0
+; AVX1-NEXT:    vpmovsxbd (%rdi), %xmm1
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; AVX1-NEXT:    retq
 ;
@@ -3001,13 +2827,11 @@ define <8 x i32> @load_sext_8i8_to_8i32(<8 x i8> *%ptr) {
 ; X32-SSE2-LABEL: load_sext_8i8_to_8i32:
 ; X32-SSE2:       # %bb.0: # %entry
 ; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-SSE2-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
-; X32-SSE2-NEXT:    psrad $24, %xmm0
-; X32-SSE2-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; X32-SSE2-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero
 ; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3]
+; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; X32-SSE2-NEXT:    psrad $24, %xmm0
+; X32-SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
 ; X32-SSE2-NEXT:    psrad $24, %xmm1
 ; X32-SSE2-NEXT:    retl
 ;
@@ -5296,21 +5120,19 @@ entry:
 define <16 x i16> @load_sext_16i8_to_16i16(<16 x i8> *%ptr) {
 ; SSE2-LABEL: load_sext_16i8_to_16i16:
 ; SSE2:       # %bb.0: # %entry
-; SSE2-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    movdqa (%rdi), %xmm1
+; SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
 ; SSE2-NEXT:    psraw $8, %xmm0
-; SSE2-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero
-; SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE2-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
 ; SSE2-NEXT:    psraw $8, %xmm1
 ; SSE2-NEXT:    retq
 ;
 ; SSSE3-LABEL: load_sext_16i8_to_16i16:
 ; SSSE3:       # %bb.0: # %entry
-; SSSE3-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
-; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSSE3-NEXT:    movdqa (%rdi), %xmm1
+; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
 ; SSSE3-NEXT:    psraw $8, %xmm0
-; SSSE3-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero
-; SSSE3-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSSE3-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
 ; SSSE3-NEXT:    psraw $8, %xmm1
 ; SSSE3-NEXT:    retq
 ;
@@ -5340,11 +5162,10 @@ define <16 x i16> @load_sext_16i8_to_16i16(<16 x i8> *%ptr) {
 ; X32-SSE2-LABEL: load_sext_16i8_to_16i16:
 ; X32-SSE2:       # %bb.0: # %entry
 ; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-SSE2-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
-; X32-SSE2-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero
-; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; X32-SSE2-NEXT:    movdqa (%eax), %xmm1
+; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
 ; X32-SSE2-NEXT:    psraw $8, %xmm0
-; X32-SSE2-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; X32-SSE2-NEXT:    punpckhbw {{.*#+}} xmm1 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
 ; X32-SSE2-NEXT:    psraw $8, %xmm1
 ; X32-SSE2-NEXT:    retl
 ;
@@ -5460,30 +5281,26 @@ entry:
 define <4 x i64> @load_sext_4i16_to_4i64(<4 x i16> *%ptr) {
 ; SSE2-LABEL: load_sext_4i16_to_4i64:
 ; SSE2:       # %bb.0: # %entry
-; SSE2-NEXT:    movswq 2(%rdi), %rax
-; SSE2-NEXT:    movq %rax, %xmm1
-; SSE2-NEXT:    movswq (%rdi), %rax
-; SSE2-NEXT:    movq %rax, %xmm0
-; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; SSE2-NEXT:    movswq 6(%rdi), %rax
-; SSE2-NEXT:    movq %rax, %xmm2
-; SSE2-NEXT:    movswq 4(%rdi), %rax
-; SSE2-NEXT:    movq %rax, %xmm1
-; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
+; SSE2-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
+; SSE2-NEXT:    psrad $16, %xmm1
+; SSE2-NEXT:    pxor %xmm2, %xmm2
+; SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
+; SSE2-NEXT:    movdqa %xmm1, %xmm0
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
+; SSE2-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]
 ; SSE2-NEXT:    retq
 ;
 ; SSSE3-LABEL: load_sext_4i16_to_4i64:
 ; SSSE3:       # %bb.0: # %entry
-; SSSE3-NEXT:    movswq 2(%rdi), %rax
-; SSSE3-NEXT:    movq %rax, %xmm1
-; SSSE3-NEXT:    movswq (%rdi), %rax
-; SSSE3-NEXT:    movq %rax, %xmm0
-; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; SSSE3-NEXT:    movswq 6(%rdi), %rax
-; SSSE3-NEXT:    movq %rax, %xmm2
-; SSSE3-NEXT:    movswq 4(%rdi), %rax
-; SSSE3-NEXT:    movq %rax, %xmm1
-; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
+; SSSE3-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
+; SSSE3-NEXT:    psrad $16, %xmm1
+; SSSE3-NEXT:    pxor %xmm2, %xmm2
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm2
+; SSSE3-NEXT:    movdqa %xmm1, %xmm0
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
+; SSSE3-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: load_sext_4i16_to_4i64:
@@ -5494,10 +5311,8 @@ define <4 x i64> @load_sext_4i16_to_4i64(<4 x i16> *%ptr) {
 ;
 ; AVX1-LABEL: load_sext_4i16_to_4i64:
 ; AVX1:       # %bb.0: # %entry
-; AVX1-NEXT:    vpmovsxwd (%rdi), %xmm0
-; AVX1-NEXT:    vpmovsxdq %xmm0, %xmm1
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; AVX1-NEXT:    vpmovsxdq %xmm0, %xmm0
+; AVX1-NEXT:    vpmovsxwq 4(%rdi), %xmm0
+; AVX1-NEXT:    vpmovsxwq (%rdi), %xmm1
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; AVX1-NEXT:    retq
 ;
@@ -5514,28 +5329,14 @@ define <4 x i64> @load_sext_4i16_to_4i64(<4 x i16> *%ptr) {
 ; X32-SSE2-LABEL: load_sext_4i16_to_4i64:
 ; X32-SSE2:       # %bb.0: # %entry
 ; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-SSE2-NEXT:    movswl 2(%eax), %ecx
-; X32-SSE2-NEXT:    movd %ecx, %xmm1
-; X32-SSE2-NEXT:    sarl $31, %ecx
-; X32-SSE2-NEXT:    movd %ecx, %xmm0
-; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
-; X32-SSE2-NEXT:    movswl (%eax), %ecx
-; X32-SSE2-NEXT:    movd %ecx, %xmm0
-; X32-SSE2-NEXT:    sarl $31, %ecx
-; X32-SSE2-NEXT:    movd %ecx, %xmm2
+; X32-SSE2-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
+; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
+; X32-SSE2-NEXT:    psrad $16, %xmm1
+; X32-SSE2-NEXT:    pxor %xmm2, %xmm2
+; X32-SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
+; X32-SSE2-NEXT:    movdqa %xmm1, %xmm0
 ; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
-; X32-SSE2-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; X32-SSE2-NEXT:    movswl 6(%eax), %ecx
-; X32-SSE2-NEXT:    movd %ecx, %xmm2
-; X32-SSE2-NEXT:    sarl $31, %ecx
-; X32-SSE2-NEXT:    movd %ecx, %xmm1
-; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
-; X32-SSE2-NEXT:    movswl 4(%eax), %eax
-; X32-SSE2-NEXT:    movd %eax, %xmm1
-; X32-SSE2-NEXT:    sarl $31, %eax
-; X32-SSE2-NEXT:    movd %eax, %xmm3
-; X32-SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
-; X32-SSE2-NEXT:    punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
+; X32-SSE2-NEXT:    punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]
 ; X32-SSE2-NEXT:    retl
 ;
 ; X32-SSE41-LABEL: load_sext_4i16_to_4i64:
@@ -5553,21 +5354,19 @@ entry:
 define <8 x i32> @load_sext_8i16_to_8i32(<8 x i16> *%ptr) {
 ; SSE2-LABEL: load_sext_8i16_to_8i32:
 ; SSE2:       # %bb.0: # %entry
-; SSE2-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
+; SSE2-NEXT:    movdqa (%rdi), %xmm1
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
 ; SSE2-NEXT:    psrad $16, %xmm0
-; SSE2-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero
-; SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3]
+; SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
 ; SSE2-NEXT:    psrad $16, %xmm1
 ; SSE2-NEXT:    retq
 ;
 ; SSSE3-LABEL: load_sext_8i16_to_8i32:
 ; SSSE3:       # %bb.0: # %entry
-; SSSE3-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
-; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
+; SSSE3-NEXT:    movdqa (%rdi), %xmm1
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
 ; SSSE3-NEXT:    psrad $16, %xmm0
-; SSSE3-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero
-; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3]
+; SSSE3-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
 ; SSSE3-NEXT:    psrad $16, %xmm1
 ; SSSE3-NEXT:    retq
 ;
@@ -5597,11 +5396,10 @@ define <8 x i32> @load_sext_8i16_to_8i32(<8 x i16> *%ptr) {
 ; X32-SSE2-LABEL: load_sext_8i16_to_8i32:
 ; X32-SSE2:       # %bb.0: # %entry
 ; X32-SSE2-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-SSE2-NEXT:    movq {{.*#+}} xmm0 = mem[0],zero
-; X32-SSE2-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero
-; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
+; X32-SSE2-NEXT:    movdqa (%eax), %xmm1
+; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
 ; X32-SSE2-NEXT:    psrad $16, %xmm0
-; X32-SSE2-NEXT:    punpcklwd {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3]
+; X32-SSE2-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
 ; X32-SSE2-NEXT:    psrad $16, %xmm1
 ; X32-SSE2-NEXT:    retl
 ;
diff --git a/test/CodeGen/X86/vector-shift-lshr-sub128.ll b/test/CodeGen/X86/vector-shift-lshr-sub128.ll
index b321ca9af74aacd04870fcee15b128f9add67bfb..8b0ae09b964cc261ea49636d7e7c5d2fc9de57c8 100644
--- a/test/CodeGen/X86/vector-shift-lshr-sub128.ll
+++ b/test/CodeGen/X86/vector-shift-lshr-sub128.ll
@@ -702,8 +702,8 @@ define <2 x i8> @var_shift_v2i8(<2 x i8> %a, <2 x i8> %b) nounwind {
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vmovddup {{.*#+}} xmm2 = [1.2598673968951787E-321,1.2598673968951787E-321]
 ; AVX1-NEXT:    # xmm2 = mem[0,0]
-; AVX1-NEXT:    vandpd %xmm2, %xmm1, %xmm1
-; AVX1-NEXT:    vandpd %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vandps %xmm2, %xmm1, %xmm1
+; AVX1-NEXT:    vandps %xmm2, %xmm0, %xmm0
 ; AVX1-NEXT:    vpsrlq %xmm1, %xmm0, %xmm2
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
 ; AVX1-NEXT:    vpsrlq %xmm1, %xmm0, %xmm0
@@ -722,9 +722,9 @@ define <2 x i8> @var_shift_v2i8(<2 x i8> %a, <2 x i8> %b) nounwind {
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vmovddup {{.*#+}} xmm2 = [1.2598673968951787E-321,1.2598673968951787E-321]
 ; XOPAVX1-NEXT:    # xmm2 = mem[0,0]
-; XOPAVX1-NEXT:    vandpd %xmm2, %xmm0, %xmm0
-; XOPAVX1-NEXT:    vandpd %xmm2, %xmm1, %xmm1
-; XOPAVX1-NEXT:    vxorpd %xmm2, %xmm2, %xmm2
+; XOPAVX1-NEXT:    vandps %xmm2, %xmm0, %xmm0
+; XOPAVX1-NEXT:    vandps %xmm2, %xmm1, %xmm1
+; XOPAVX1-NEXT:    vxorps %xmm2, %xmm2, %xmm2
 ; XOPAVX1-NEXT:    vpsubq %xmm1, %xmm2, %xmm1
 ; XOPAVX1-NEXT:    vpshlq %xmm1, %xmm0, %xmm0
 ; XOPAVX1-NEXT:    retq
@@ -1489,7 +1489,7 @@ define <2 x i8> @splatvar_shift_v2i8(<2 x i8> %a, <2 x i8> %b) nounwind {
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vmovddup {{.*#+}} xmm2 = [1.2598673968951787E-321,1.2598673968951787E-321]
 ; AVX1-NEXT:    # xmm2 = mem[0,0]
-; AVX1-NEXT:    vandpd %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vandps %xmm2, %xmm0, %xmm0
 ; AVX1-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[0],zero,zero,zero,zero,zero,zero,zero
 ; AVX1-NEXT:    vpsrlq %xmm1, %xmm0, %xmm2
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
@@ -1509,9 +1509,9 @@ define <2 x i8> @splatvar_shift_v2i8(<2 x i8> %a, <2 x i8> %b) nounwind {
 ; XOPAVX1:       # %bb.0:
 ; XOPAVX1-NEXT:    vmovddup {{.*#+}} xmm2 = [1.2598673968951787E-321,1.2598673968951787E-321]
 ; XOPAVX1-NEXT:    # xmm2 = mem[0,0]
-; XOPAVX1-NEXT:    vandpd %xmm2, %xmm0, %xmm0
+; XOPAVX1-NEXT:    vandps %xmm2, %xmm0, %xmm0
 ; XOPAVX1-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[0],zero,zero,zero,zero,zero,zero,zero
-; XOPAVX1-NEXT:    vxorpd %xmm2, %xmm2, %xmm2
+; XOPAVX1-NEXT:    vxorps %xmm2, %xmm2, %xmm2
 ; XOPAVX1-NEXT:    vpsubq %xmm1, %xmm2, %xmm1
 ; XOPAVX1-NEXT:    vpshlq %xmm1, %xmm0, %xmm0
 ; XOPAVX1-NEXT:    retq
diff --git a/test/CodeGen/X86/vector-shuffle-128-unpck.ll b/test/CodeGen/X86/vector-shuffle-128-unpck.ll
new file mode 100644
index 0000000000000000000000000000000000000000..47d9c41e01997b2a43abc550cb9cc6cb6c125335
--- /dev/null
+++ b/test/CodeGen/X86/vector-shuffle-128-unpck.ll
@@ -0,0 +1,223 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefix=ALL --check-prefix=AVX1OR2 --check-prefix=AVX1
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefix=ALL --check-prefix=AVX1OR2 --check-prefix=AVX2OR512VL --check-prefix=AVX2 --check-prefix=AVX2-SLOW
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2,+fast-variable-shuffle | FileCheck %s --check-prefix=ALL --check-prefix=AVX1OR2 --check-prefix=AVX2OR512VL --check-prefix=AVX2 --check-prefix=AVX2-FAST
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512vl,+avx512dq | FileCheck %s --check-prefix=ALL --check-prefix=AVX2OR512VL --check-prefix=AVX512VL --check-prefix=AVX512VL-SLOW
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512vl,+avx512dq,+fast-variable-shuffle | FileCheck %s --check-prefix=ALL --check-prefix=AVX2OR512VL --check-prefix=AVX512VL --check-prefix=AVX512VL-FAST
+
+define <2 x i64> @unpckh_unary_extracted_v4i64(<4 x i64> %x) {
+; AVX1-LABEL: unpckh_unary_extracted_v4i64:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX1-NEXT:    vunpckhpd {{.*#+}} xmm0 = xmm0[1],xmm1[1]
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    retq
+;
+; AVX2OR512VL-LABEL: unpckh_unary_extracted_v4i64:
+; AVX2OR512VL:       # %bb.0:
+; AVX2OR512VL-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[1,3,2,3]
+; AVX2OR512VL-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX2OR512VL-NEXT:    vzeroupper
+; AVX2OR512VL-NEXT:    retq
+  %extrl = shufflevector <4 x i64> %x, <4 x i64> undef, <2 x i32> <i32 0, i32 1>
+  %extrh = shufflevector <4 x i64> %x, <4 x i64> undef, <2 x i32> <i32 2, i32 3>
+  %r = shufflevector <2 x i64> %extrl, <2 x i64> %extrh, <2 x i32> <i32 1, i32 3>
+  ret <2 x i64> %r
+}
+
+define <2 x double> @unpckh_unary_extracted_v8f64(<4 x double> %x) {
+; AVX1-LABEL: unpckh_unary_extracted_v8f64:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX1-NEXT:    vunpckhpd {{.*#+}} xmm0 = xmm0[1],xmm1[1]
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    retq
+;
+; AVX2OR512VL-LABEL: unpckh_unary_extracted_v8f64:
+; AVX2OR512VL:       # %bb.0:
+; AVX2OR512VL-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[1,3,2,3]
+; AVX2OR512VL-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX2OR512VL-NEXT:    vzeroupper
+; AVX2OR512VL-NEXT:    retq
+  %extrl = shufflevector <4 x double> %x, <4 x double> undef, <2 x i32> <i32 0, i32 1>
+  %extrh = shufflevector <4 x double> %x, <4 x double> undef, <2 x i32> <i32 2, i32 3>
+  %r = shufflevector <2 x double> %extrl, <2 x double> %extrh, <2 x i32> <i32 1, i32 3>
+  ret <2 x double> %r
+}
+
+; vpermps requires a constant load for the index op. It's unlikely to be profitable.
+
+define <4 x i32> @unpckh_unary_extracted_v8i32(<8 x i32> %x) {
+; ALL-LABEL: unpckh_unary_extracted_v8i32:
+; ALL:       # %bb.0:
+; ALL-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; ALL-NEXT:    vunpckhps {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; ALL-NEXT:    vzeroupper
+; ALL-NEXT:    retq
+  %extrl = shufflevector <8 x i32> %x, <8 x i32> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %extrh = shufflevector <8 x i32> %x, <8 x i32> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  %r = shufflevector <4 x i32> %extrl, <4 x i32> %extrh, <4 x i32> <i32 2, i32 6, i32 3, i32 7>
+  ret <4 x i32> %r
+}
+
+define <4 x float> @unpckh_unary_extracted_v8f32(<8 x float> %x) {
+; ALL-LABEL: unpckh_unary_extracted_v8f32:
+; ALL:       # %bb.0:
+; ALL-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; ALL-NEXT:    vunpckhps {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; ALL-NEXT:    vzeroupper
+; ALL-NEXT:    retq
+  %extrl = shufflevector <8 x float> %x, <8 x float> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %extrh = shufflevector <8 x float> %x, <8 x float> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  %r = shufflevector <4 x float> %extrl, <4 x float> %extrh, <4 x i32> <i32 2, i32 6, i32 3, i32 7>
+  ret <4 x float> %r
+}
+
+define <8 x i16> @unpckh_unary_extracted_v16i16(<16 x i16> %x) {
+; AVX1-LABEL: unpckh_unary_extracted_v16i16:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    retq
+;
+; AVX2OR512VL-LABEL: unpckh_unary_extracted_v16i16:
+; AVX2OR512VL:       # %bb.0:
+; AVX2OR512VL-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX2OR512VL-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; AVX2OR512VL-NEXT:    vzeroupper
+; AVX2OR512VL-NEXT:    retq
+  %extrl = shufflevector <16 x i16> %x, <16 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %extrh = shufflevector <16 x i16> %x, <16 x i16> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  %r = shufflevector <8 x i16> %extrl, <8 x i16> %extrh, <8 x i32> <i32 4, i32 12, i32 5, i32 13, i32 6, i32 14, i32 7, i32 15>
+  ret <8 x i16> %r
+}
+
+define <16 x i8> @unpckh_unary_extracted_v32i8(<32 x i8> %x) {
+; AVX1-LABEL: unpckh_unary_extracted_v32i8:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm0[8],xmm1[8],xmm0[9],xmm1[9],xmm0[10],xmm1[10],xmm0[11],xmm1[11],xmm0[12],xmm1[12],xmm0[13],xmm1[13],xmm0[14],xmm1[14],xmm0[15],xmm1[15]
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    retq
+;
+; AVX2OR512VL-LABEL: unpckh_unary_extracted_v32i8:
+; AVX2OR512VL:       # %bb.0:
+; AVX2OR512VL-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX2OR512VL-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm0[8],xmm1[8],xmm0[9],xmm1[9],xmm0[10],xmm1[10],xmm0[11],xmm1[11],xmm0[12],xmm1[12],xmm0[13],xmm1[13],xmm0[14],xmm1[14],xmm0[15],xmm1[15]
+; AVX2OR512VL-NEXT:    vzeroupper
+; AVX2OR512VL-NEXT:    retq
+  %extrl = shufflevector <32 x i8> %x, <32 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  %extrh = shufflevector <32 x i8> %x, <32 x i8> undef, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+  %r = shufflevector <16 x i8> %extrl, <16 x i8> %extrh, <16 x i32> <i32 8, i32 24, i32 9, i32 25, i32 10, i32 26, i32 11, i32 27, i32 12, i32 28, i32 13, i32 29, i32 14, i32 30, i32 15, i32 31>
+  ret <16 x i8> %r
+}
+
+define <2 x i64> @unpckl_unary_extracted_v4i64(<4 x i64> %x) {
+; AVX1-LABEL: unpckl_unary_extracted_v4i64:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX1-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    retq
+;
+; AVX2OR512VL-LABEL: unpckl_unary_extracted_v4i64:
+; AVX2OR512VL:       # %bb.0:
+; AVX2OR512VL-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[0,2,2,3]
+; AVX2OR512VL-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX2OR512VL-NEXT:    vzeroupper
+; AVX2OR512VL-NEXT:    retq
+  %extrl = shufflevector <4 x i64> %x, <4 x i64> undef, <2 x i32> <i32 0, i32 1>
+  %extrh = shufflevector <4 x i64> %x, <4 x i64> undef, <2 x i32> <i32 2, i32 3>
+  %r = shufflevector <2 x i64> %extrl, <2 x i64> %extrh, <2 x i32> <i32 0, i32 2>
+  ret <2 x i64> %r
+}
+
+define <2 x double> @unpckl_unary_extracted_v8f64(<4 x double> %x) {
+; AVX1-LABEL: unpckl_unary_extracted_v8f64:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX1-NEXT:    vmovlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    retq
+;
+; AVX2OR512VL-LABEL: unpckl_unary_extracted_v8f64:
+; AVX2OR512VL:       # %bb.0:
+; AVX2OR512VL-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[0,2,2,3]
+; AVX2OR512VL-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
+; AVX2OR512VL-NEXT:    vzeroupper
+; AVX2OR512VL-NEXT:    retq
+  %extrl = shufflevector <4 x double> %x, <4 x double> undef, <2 x i32> <i32 0, i32 1>
+  %extrh = shufflevector <4 x double> %x, <4 x double> undef, <2 x i32> <i32 2, i32 3>
+  %r = shufflevector <2 x double> %extrl, <2 x double> %extrh, <2 x i32> <i32 0, i32 2>
+  ret <2 x double> %r
+}
+
+; vpermps requires a constant load for the index op. It's unlikely to be profitable.
+
+define <4 x i32> @unpckl_unary_extracted_v8i32(<8 x i32> %x) {
+; ALL-LABEL: unpckl_unary_extracted_v8i32:
+; ALL:       # %bb.0:
+; ALL-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; ALL-NEXT:    vunpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; ALL-NEXT:    vzeroupper
+; ALL-NEXT:    retq
+  %extrl = shufflevector <8 x i32> %x, <8 x i32> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %extrh = shufflevector <8 x i32> %x, <8 x i32> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  %r = shufflevector <4 x i32> %extrl, <4 x i32> %extrh, <4 x i32> <i32 0, i32 4, i32 1, i32 5>
+  ret <4 x i32> %r
+}
+
+define <4 x float> @unpckl_unary_extracted_v8f32(<8 x float> %x) {
+; ALL-LABEL: unpckl_unary_extracted_v8f32:
+; ALL:       # %bb.0:
+; ALL-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; ALL-NEXT:    vunpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; ALL-NEXT:    vzeroupper
+; ALL-NEXT:    retq
+  %extrl = shufflevector <8 x float> %x, <8 x float> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %extrh = shufflevector <8 x float> %x, <8 x float> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+  %r = shufflevector <4 x float> %extrl, <4 x float> %extrh, <4 x i32> <i32 0, i32 4, i32 1, i32 5>
+  ret <4 x float> %r
+}
+
+define <8 x i16> @unpckl_unary_extracted_v16i16(<16 x i16> %x) {
+; AVX1-LABEL: unpckl_unary_extracted_v16i16:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX1-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    retq
+;
+; AVX2OR512VL-LABEL: unpckl_unary_extracted_v16i16:
+; AVX2OR512VL:       # %bb.0:
+; AVX2OR512VL-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX2OR512VL-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; AVX2OR512VL-NEXT:    vzeroupper
+; AVX2OR512VL-NEXT:    retq
+  %extrl = shufflevector <16 x i16> %x, <16 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %extrh = shufflevector <16 x i16> %x, <16 x i16> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  %r = shufflevector <8 x i16> %extrl, <8 x i16> %extrh, <8 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11>
+  ret <8 x i16> %r
+}
+
+define <16 x i8> @unpckl_unary_extracted_v32i8(<32 x i8> %x) {
+; AVX1-LABEL: unpckl_unary_extracted_v32i8:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    retq
+;
+; AVX2OR512VL-LABEL: unpckl_unary_extracted_v32i8:
+; AVX2OR512VL:       # %bb.0:
+; AVX2OR512VL-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX2OR512VL-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; AVX2OR512VL-NEXT:    vzeroupper
+; AVX2OR512VL-NEXT:    retq
+  %extrl = shufflevector <32 x i8> %x, <32 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  %extrh = shufflevector <32 x i8> %x, <32 x i8> undef, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+  %r = shufflevector <16 x i8> %extrl, <16 x i8> %extrh, <16 x i32> <i32 0, i32 16, i32 1, i32 17, i32 2, i32 18, i32 3, i32 19, i32 4, i32 20, i32 5, i32 21, i32 6, i32 22, i32 7, i32 23>
+  ret <16 x i8> %r
+}
+
diff --git a/test/CodeGen/X86/vector-shuffle-128-v16.ll b/test/CodeGen/X86/vector-shuffle-128-v16.ll
index d9790aa9a75b99a86c5efd83f3fa40ea16f6a374..f718bbe7a31b76321fc1939e26af46bfec7f5016 100644
--- a/test/CodeGen/X86/vector-shuffle-128-v16.ll
+++ b/test/CodeGen/X86/vector-shuffle-128-v16.ll
@@ -1532,6 +1532,96 @@ define <16 x i8> @shuffle_v16i8_07_zz_zz_zz_zz_zz_uu_uu_15_uu_uu_uu_uu_uu_zz_zz(
   ret <16 x i8> %shuffle
 }
 
+define <16 x i8> @shuffle_v16i8_zz_zz_zz_zz_01_02_03_04_05_06_zz_zz_zz_zz_zz_zz(<16 x i8> %a) {
+; SSE2-LABEL: shuffle_v16i8_zz_zz_zz_zz_01_02_03_04_05_06_zz_zz_zz_zz_zz_zz:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    pslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1,2,3,4,5,6]
+; SSE2-NEXT:    psrldq {{.*#+}} xmm0 = xmm0[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; SSE2-NEXT:    pslldq {{.*#+}} xmm0 = zero,zero,zero,zero,xmm0[0,1,2,3,4,5,6,7,8,9,10,11]
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: shuffle_v16i8_zz_zz_zz_zz_01_02_03_04_05_06_zz_zz_zz_zz_zz_zz:
+; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    pshufb {{.*#+}} xmm0 = zero,zero,zero,zero,xmm0[1,2,3,4,5,6],zero,zero,zero,zero,zero,zero
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: shuffle_v16i8_zz_zz_zz_zz_01_02_03_04_05_06_zz_zz_zz_zz_zz_zz:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    pshufb {{.*#+}} xmm0 = zero,zero,zero,zero,xmm0[1,2,3,4,5,6],zero,zero,zero,zero,zero,zero
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: shuffle_v16i8_zz_zz_zz_zz_01_02_03_04_05_06_zz_zz_zz_zz_zz_zz:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vpshufb {{.*#+}} xmm0 = zero,zero,zero,zero,xmm0[1,2,3,4,5,6],zero,zero,zero,zero,zero,zero
+; AVX-NEXT:    retq
+  %shuffle = shufflevector <16 x i8> %a, <16 x i8> <i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef>, <16 x i32> <i32 16, i32 16, i32 16, i32 16, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>
+  ret <16 x i8> %shuffle
+}
+
+define <16 x i8> @shuffle_v16i8_01_02_03_04_05_06_zz_zz_zz_zz_zz_zz_zz_zz_zz_zz(<16 x i8> %a) {
+; SSE-LABEL: shuffle_v16i8_01_02_03_04_05_06_zz_zz_zz_zz_zz_zz_zz_zz_zz_zz:
+; SSE:       # %bb.0:
+; SSE-NEXT:    pslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1,2,3,4,5,6]
+; SSE-NEXT:    psrldq {{.*#+}} xmm0 = xmm0[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; SSE-NEXT:    retq
+;
+; AVX1-LABEL: shuffle_v16i8_01_02_03_04_05_06_zz_zz_zz_zz_zz_zz_zz_zz_zz_zz:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1,2,3,4,5,6]
+; AVX1-NEXT:    vpsrldq {{.*#+}} xmm0 = xmm0[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX1-NEXT:    retq
+;
+; AVX2-SLOW-LABEL: shuffle_v16i8_01_02_03_04_05_06_zz_zz_zz_zz_zz_zz_zz_zz_zz_zz:
+; AVX2-SLOW:       # %bb.0:
+; AVX2-SLOW-NEXT:    vpslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1,2,3,4,5,6]
+; AVX2-SLOW-NEXT:    vpsrldq {{.*#+}} xmm0 = xmm0[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX2-SLOW-NEXT:    retq
+;
+; AVX2-FAST-LABEL: shuffle_v16i8_01_02_03_04_05_06_zz_zz_zz_zz_zz_zz_zz_zz_zz_zz:
+; AVX2-FAST:       # %bb.0:
+; AVX2-FAST-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[1,2,3,4,5,6],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX2-FAST-NEXT:    retq
+;
+; AVX512VL-LABEL: shuffle_v16i8_01_02_03_04_05_06_zz_zz_zz_zz_zz_zz_zz_zz_zz_zz:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[1,2,3,4,5,6],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX512VL-NEXT:    retq
+  %shuffle = shufflevector <16 x i8> %a, <16 x i8> <i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef>, <16 x i32> <i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16>
+  ret <16 x i8> %shuffle
+}
+
+define <16 x i8> @shuffle_v16i8_zz_zz_zz_zz_zz_zz_zz_zz_zz_zz_01_02_03_04_05_06(<16 x i8> %a) {
+; SSE-LABEL: shuffle_v16i8_zz_zz_zz_zz_zz_zz_zz_zz_zz_zz_01_02_03_04_05_06:
+; SSE:       # %bb.0:
+; SSE-NEXT:    psrldq {{.*#+}} xmm0 = xmm0[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15],zero
+; SSE-NEXT:    pslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1,2,3,4,5]
+; SSE-NEXT:    retq
+;
+; AVX1-LABEL: shuffle_v16i8_zz_zz_zz_zz_zz_zz_zz_zz_zz_zz_01_02_03_04_05_06:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpsrldq {{.*#+}} xmm0 = xmm0[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15],zero
+; AVX1-NEXT:    vpslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1,2,3,4,5]
+; AVX1-NEXT:    retq
+;
+; AVX2-SLOW-LABEL: shuffle_v16i8_zz_zz_zz_zz_zz_zz_zz_zz_zz_zz_01_02_03_04_05_06:
+; AVX2-SLOW:       # %bb.0:
+; AVX2-SLOW-NEXT:    vpsrldq {{.*#+}} xmm0 = xmm0[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15],zero
+; AVX2-SLOW-NEXT:    vpslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1,2,3,4,5]
+; AVX2-SLOW-NEXT:    retq
+;
+; AVX2-FAST-LABEL: shuffle_v16i8_zz_zz_zz_zz_zz_zz_zz_zz_zz_zz_01_02_03_04_05_06:
+; AVX2-FAST:       # %bb.0:
+; AVX2-FAST-NEXT:    vpshufb {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[1,2,3,4,5,6]
+; AVX2-FAST-NEXT:    retq
+;
+; AVX512VL-LABEL: shuffle_v16i8_zz_zz_zz_zz_zz_zz_zz_zz_zz_zz_01_02_03_04_05_06:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpshufb {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[1,2,3,4,5,6]
+; AVX512VL-NEXT:    retq
+  %shuffle = shufflevector <16 x i8> %a, <16 x i8> <i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef>, <16 x i32> <i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6>
+  ret <16 x i8> %shuffle
+}
+
 define <16 x i8> @PR12412(<16 x i8> %inval1, <16 x i8> %inval2) {
 ; SSE2-LABEL: PR12412:
 ; SSE2:       # %bb.0: # %entry
diff --git a/test/CodeGen/X86/vector-shuffle-128-v2.ll b/test/CodeGen/X86/vector-shuffle-128-v2.ll
index ffcaad8ee5103258935219d66937b2fcf89b6861..7905d9d45f1110bfe8f838ed7f50398b825f38c5 100644
--- a/test/CodeGen/X86/vector-shuffle-128-v2.ll
+++ b/test/CodeGen/X86/vector-shuffle-128-v2.ll
@@ -20,12 +20,12 @@ define <2 x i64> @shuffle_v2i64_00(<2 x i64> %a, <2 x i64> %b) {
 ;
 ; AVX2-LABEL: shuffle_v2i64_00:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpbroadcastq %xmm0, %xmm0
+; AVX2-NEXT:    vmovddup {{.*#+}} xmm0 = xmm0[0,0]
 ; AVX2-NEXT:    retq
 ;
 ; AVX512VL-LABEL: shuffle_v2i64_00:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpbroadcastq %xmm0, %xmm0
+; AVX512VL-NEXT:    vmovddup {{.*#+}} xmm0 = xmm0[0,0]
 ; AVX512VL-NEXT:    retq
   %shuffle = shufflevector <2 x i64> %a, <2 x i64> %b, <2 x i32> <i32 0, i32 0>
   ret <2 x i64> %shuffle
@@ -69,12 +69,12 @@ define <2 x i64> @shuffle_v2i64_22(<2 x i64> %a, <2 x i64> %b) {
 ;
 ; AVX2-LABEL: shuffle_v2i64_22:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpbroadcastq %xmm1, %xmm0
+; AVX2-NEXT:    vmovddup {{.*#+}} xmm0 = xmm1[0,0]
 ; AVX2-NEXT:    retq
 ;
 ; AVX512VL-LABEL: shuffle_v2i64_22:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpbroadcastq %xmm1, %xmm0
+; AVX512VL-NEXT:    vmovddup {{.*#+}} xmm0 = xmm1[0,0]
 ; AVX512VL-NEXT:    retq
   %shuffle = shufflevector <2 x i64> %a, <2 x i64> %b, <2 x i32> <i32 2, i32 2>
   ret <2 x i64> %shuffle
@@ -1264,21 +1264,10 @@ define <2 x i64> @insert_dup_mem_v2i64(i64* %ptr) {
 ; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,1,0,1]
 ; SSE-NEXT:    retq
 ;
-; AVX1-LABEL: insert_dup_mem_v2i64:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,1,0,1]
-; AVX1-NEXT:    retq
-;
-; AVX2-LABEL: insert_dup_mem_v2i64:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpbroadcastq (%rdi), %xmm0
-; AVX2-NEXT:    retq
-;
-; AVX512VL-LABEL: insert_dup_mem_v2i64:
-; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpbroadcastq (%rdi), %xmm0
-; AVX512VL-NEXT:    retq
+; AVX-LABEL: insert_dup_mem_v2i64:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovddup {{.*#+}} xmm0 = mem[0,0]
+; AVX-NEXT:    retq
   %tmp = load i64, i64* %ptr, align 1
   %tmp1 = insertelement <2 x i64> undef, i64 %tmp, i32 0
   %tmp2 = shufflevector <2 x i64> %tmp1, <2 x i64> undef, <2 x i32> zeroinitializer
diff --git a/test/CodeGen/X86/vector-shuffle-128-v4.ll b/test/CodeGen/X86/vector-shuffle-128-v4.ll
index 149fe3d9dcb983fbd1b99d238471f4335bc82459..211c57c63ac7a6b654f909d503fed821b5c7e732 100644
--- a/test/CodeGen/X86/vector-shuffle-128-v4.ll
+++ b/test/CodeGen/X86/vector-shuffle-128-v4.ll
@@ -539,9 +539,9 @@ define <4 x i32> @shuffle_v4i32_0451(<4 x i32> %a, <4 x i32> %b) {
 ;
 ; AVX2OR512VL-LABEL: shuffle_v4i32_0451:
 ; AVX2OR512VL:       # %bb.0:
-; AVX2OR512VL-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,0,1,1]
-; AVX2OR512VL-NEXT:    vpbroadcastq %xmm0, %xmm0
-; AVX2OR512VL-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1,2],xmm0[3]
+; AVX2OR512VL-NEXT:    vpermilps {{.*#+}} xmm1 = xmm1[0,0,1,1]
+; AVX2OR512VL-NEXT:    vmovddup {{.*#+}} xmm0 = xmm0[0,0]
+; AVX2OR512VL-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1,2],xmm0[3]
 ; AVX2OR512VL-NEXT:    retq
   %shuffle = shufflevector <4 x i32> %a, <4 x i32> %b, <4 x i32> <i32 0, i32 4, i32 5, i32 1>
   ret <4 x i32> %shuffle
@@ -595,9 +595,9 @@ define <4 x i32> @shuffle_v4i32_4015(<4 x i32> %a, <4 x i32> %b) {
 ;
 ; AVX2OR512VL-LABEL: shuffle_v4i32_4015:
 ; AVX2OR512VL:       # %bb.0:
-; AVX2OR512VL-NEXT:    vpbroadcastq %xmm1, %xmm1
-; AVX2OR512VL-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,1,1]
-; AVX2OR512VL-NEXT:    vpblendd {{.*#+}} xmm0 = xmm1[0],xmm0[1,2],xmm1[3]
+; AVX2OR512VL-NEXT:    vmovddup {{.*#+}} xmm1 = xmm1[0,0]
+; AVX2OR512VL-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,0,1,1]
+; AVX2OR512VL-NEXT:    vblendps {{.*#+}} xmm0 = xmm1[0],xmm0[1,2],xmm1[3]
 ; AVX2OR512VL-NEXT:    retq
   %shuffle = shufflevector <4 x i32> %a, <4 x i32> %b, <4 x i32> <i32 4, i32 0, i32 1, i32 5>
   ret <4 x i32> %shuffle
diff --git a/test/CodeGen/X86/vector-shuffle-128-v8.ll b/test/CodeGen/X86/vector-shuffle-128-v8.ll
index 316b0157b190e1da015807e3b7965aa5d2dd87f7..592086f93154e7d6c5f2c06eaad3338187bb322b 100644
--- a/test/CodeGen/X86/vector-shuffle-128-v8.ll
+++ b/test/CodeGen/X86/vector-shuffle-128-v8.ll
@@ -2475,6 +2475,122 @@ define <8 x i16> @shuffle_v8i16_8012345u(<8 x i16> %a) {
   ret <8 x i16> %shuffle
 }
 
+; PR40306
+define <8 x i16> @shuffle_v8i16_9zzzuuuu(<8 x i16> %x) {
+; SSE-LABEL: shuffle_v8i16_9zzzuuuu:
+; SSE:       # %bb.0:
+; SSE-NEXT:    pslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1,2,3]
+; SSE-NEXT:    psrldq {{.*#+}} xmm0 = xmm0[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; SSE-NEXT:    retq
+;
+; AVX1-LABEL: shuffle_v8i16_9zzzuuuu:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1,2,3]
+; AVX1-NEXT:    vpsrldq {{.*#+}} xmm0 = xmm0[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX1-NEXT:    retq
+;
+; AVX2-SLOW-LABEL: shuffle_v8i16_9zzzuuuu:
+; AVX2-SLOW:       # %bb.0:
+; AVX2-SLOW-NEXT:    vpslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1,2,3]
+; AVX2-SLOW-NEXT:    vpsrldq {{.*#+}} xmm0 = xmm0[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX2-SLOW-NEXT:    retq
+;
+; AVX2-FAST-LABEL: shuffle_v8i16_9zzzuuuu:
+; AVX2-FAST:       # %bb.0:
+; AVX2-FAST-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[2,3],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX2-FAST-NEXT:    retq
+;
+; AVX512VL-SLOW-LABEL: shuffle_v8i16_9zzzuuuu:
+; AVX512VL-SLOW:       # %bb.0:
+; AVX512VL-SLOW-NEXT:    vpslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1,2,3]
+; AVX512VL-SLOW-NEXT:    vpsrldq {{.*#+}} xmm0 = xmm0[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX512VL-SLOW-NEXT:    retq
+;
+; AVX512VL-FAST-LABEL: shuffle_v8i16_9zzzuuuu:
+; AVX512VL-FAST:       # %bb.0:
+; AVX512VL-FAST-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[2,3],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX512VL-FAST-NEXT:    retq
+  %r = shufflevector <8 x i16> zeroinitializer, <8 x i16> %x, <8 x i32> <i32 9, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+  ret <8 x i16> %r
+}
+
+; PR40318
+define <8 x i16> @shuffle_v8i16_2zzzuuuu(<8 x i16> %x) {
+; SSE-LABEL: shuffle_v8i16_2zzzuuuu:
+; SSE:       # %bb.0:
+; SSE-NEXT:    pslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1,2,3,4,5]
+; SSE-NEXT:    psrldq {{.*#+}} xmm0 = xmm0[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; SSE-NEXT:    retq
+;
+; AVX1-LABEL: shuffle_v8i16_2zzzuuuu:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1,2,3,4,5]
+; AVX1-NEXT:    vpsrldq {{.*#+}} xmm0 = xmm0[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX1-NEXT:    retq
+;
+; AVX2-SLOW-LABEL: shuffle_v8i16_2zzzuuuu:
+; AVX2-SLOW:       # %bb.0:
+; AVX2-SLOW-NEXT:    vpslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1,2,3,4,5]
+; AVX2-SLOW-NEXT:    vpsrldq {{.*#+}} xmm0 = xmm0[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX2-SLOW-NEXT:    retq
+;
+; AVX2-FAST-LABEL: shuffle_v8i16_2zzzuuuu:
+; AVX2-FAST:       # %bb.0:
+; AVX2-FAST-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[4,5],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX2-FAST-NEXT:    retq
+;
+; AVX512VL-SLOW-LABEL: shuffle_v8i16_2zzzuuuu:
+; AVX512VL-SLOW:       # %bb.0:
+; AVX512VL-SLOW-NEXT:    vpslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1,2,3,4,5]
+; AVX512VL-SLOW-NEXT:    vpsrldq {{.*#+}} xmm0 = xmm0[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX512VL-SLOW-NEXT:    retq
+;
+; AVX512VL-FAST-LABEL: shuffle_v8i16_2zzzuuuu:
+; AVX512VL-FAST:       # %bb.0:
+; AVX512VL-FAST-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[4,5],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX512VL-FAST-NEXT:    retq
+  %r = shufflevector <8 x i16> %x, <8 x i16> zeroinitializer, <8 x i32> <i32 2, i32 9, i32 10, i32 11, i32 undef, i32 undef, i32 undef, i32 undef>
+  ret <8 x i16> %r
+}
+
+define <8 x i16> @shuffle_v8i16_3uu6zzzz(<8 x i16> %x) {
+; SSE-LABEL: shuffle_v8i16_3uu6zzzz:
+; SSE:       # %bb.0:
+; SSE-NEXT:    pslldq {{.*#+}} xmm0 = zero,zero,xmm0[0,1,2,3,4,5,6,7,8,9,10,11,12,13]
+; SSE-NEXT:    psrldq {{.*#+}} xmm0 = xmm0[8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero
+; SSE-NEXT:    retq
+;
+; AVX1-LABEL: shuffle_v8i16_3uu6zzzz:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vpslldq {{.*#+}} xmm0 = zero,zero,xmm0[0,1,2,3,4,5,6,7,8,9,10,11,12,13]
+; AVX1-NEXT:    vpsrldq {{.*#+}} xmm0 = xmm0[8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero
+; AVX1-NEXT:    retq
+;
+; AVX2-SLOW-LABEL: shuffle_v8i16_3uu6zzzz:
+; AVX2-SLOW:       # %bb.0:
+; AVX2-SLOW-NEXT:    vpslldq {{.*#+}} xmm0 = zero,zero,xmm0[0,1,2,3,4,5,6,7,8,9,10,11,12,13]
+; AVX2-SLOW-NEXT:    vpsrldq {{.*#+}} xmm0 = xmm0[8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero
+; AVX2-SLOW-NEXT:    retq
+;
+; AVX2-FAST-LABEL: shuffle_v8i16_3uu6zzzz:
+; AVX2-FAST:       # %bb.0:
+; AVX2-FAST-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[6,7,8,9,10,11,12,13],zero,zero,zero,zero,zero,zero,zero,zero
+; AVX2-FAST-NEXT:    retq
+;
+; AVX512VL-SLOW-LABEL: shuffle_v8i16_3uu6zzzz:
+; AVX512VL-SLOW:       # %bb.0:
+; AVX512VL-SLOW-NEXT:    vpslldq {{.*#+}} xmm0 = zero,zero,xmm0[0,1,2,3,4,5,6,7,8,9,10,11,12,13]
+; AVX512VL-SLOW-NEXT:    vpsrldq {{.*#+}} xmm0 = xmm0[8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero
+; AVX512VL-SLOW-NEXT:    retq
+;
+; AVX512VL-FAST-LABEL: shuffle_v8i16_3uu6zzzz:
+; AVX512VL-FAST:       # %bb.0:
+; AVX512VL-FAST-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[6,7,8,9,10,11,12,13],zero,zero,zero,zero,zero,zero,zero,zero
+; AVX512VL-FAST-NEXT:    retq
+  %r = shufflevector <8 x i16> %x, <8 x i16> zeroinitializer, <8 x i32> <i32 3, i32 undef, i32 undef, i32 6, i32 8, i32 8, i32 8, i32 8>
+  ret <8 x i16> %r
+}
+
 define <8 x i16> @mask_v8i16_012345ef(<8 x i16> %a, <8 x i16> %b) {
 ; SSE2-LABEL: mask_v8i16_012345ef:
 ; SSE2:       # %bb.0:
diff --git a/test/CodeGen/X86/vector-shuffle-256-v16.ll b/test/CodeGen/X86/vector-shuffle-256-v16.ll
index 461246d80a824d9b2db2d9886564e27b7292fe9f..96200e42e94c56373e33d6dcc80e0afbc5b0c1a5 100644
--- a/test/CodeGen/X86/vector-shuffle-256-v16.ll
+++ b/test/CodeGen/X86/vector-shuffle-256-v16.ll
@@ -40,8 +40,9 @@ define <16 x i16> @shuffle_v16i16_00_00_00_00_00_00_00_00_00_00_00_00_00_00_01_0
 ;
 ; AVX2-FAST-LABEL: shuffle_v16i16_00_00_00_00_00_00_00_00_00_00_00_00_00_00_01_00:
 ; AVX2-FAST:       # %bb.0:
-; AVX2-FAST-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,0,1,0,1,0,1,0,1,0,1,2,3,0,1]
-; AVX2-FAST-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,0,0,1]
+; AVX2-FAST-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,1,0,4,5,6,7]
+; AVX2-FAST-NEXT:    vmovdqa {{.*#+}} ymm1 = [0,0,0,0,0,0,0,1]
+; AVX2-FAST-NEXT:    vpermd %ymm0, %ymm1, %ymm0
 ; AVX2-FAST-NEXT:    retq
 ;
 ; AVX512VL-SLOW-LABEL: shuffle_v16i16_00_00_00_00_00_00_00_00_00_00_00_00_00_00_01_00:
@@ -53,8 +54,8 @@ define <16 x i16> @shuffle_v16i16_00_00_00_00_00_00_00_00_00_00_00_00_00_00_01_0
 ;
 ; AVX512VL-FAST-LABEL: shuffle_v16i16_00_00_00_00_00_00_00_00_00_00_00_00_00_00_01_00:
 ; AVX512VL-FAST:       # %bb.0:
-; AVX512VL-FAST-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,0,1,0,1,0,1,0,1,0,1,2,3,0,1]
-; AVX512VL-FAST-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,0,0,1]
+; AVX512VL-FAST-NEXT:    vmovdqa {{.*#+}} ymm1 = [0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0]
+; AVX512VL-FAST-NEXT:    vpermw %ymm0, %ymm1, %ymm0
 ; AVX512VL-FAST-NEXT:    retq
   %shuffle = shufflevector <16 x i16> %a, <16 x i16> %b, <16 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 1, i32 0>
   ret <16 x i16> %shuffle
@@ -79,8 +80,9 @@ define <16 x i16> @shuffle_v16i16_00_00_00_00_00_00_00_00_00_00_00_00_00_02_00_0
 ;
 ; AVX2-FAST-LABEL: shuffle_v16i16_00_00_00_00_00_00_00_00_00_00_00_00_00_02_00_00:
 ; AVX2-FAST:       # %bb.0:
-; AVX2-FAST-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,0,1,0,1,0,1,0,1,4,5,0,1,0,1]
-; AVX2-FAST-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,0,0,1]
+; AVX2-FAST-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,0,2,4,5,6,7]
+; AVX2-FAST-NEXT:    vmovdqa {{.*#+}} ymm1 = [0,0,0,0,0,0,1,0]
+; AVX2-FAST-NEXT:    vpermd %ymm0, %ymm1, %ymm0
 ; AVX2-FAST-NEXT:    retq
 ;
 ; AVX512VL-SLOW-LABEL: shuffle_v16i16_00_00_00_00_00_00_00_00_00_00_00_00_00_02_00_00:
@@ -92,8 +94,8 @@ define <16 x i16> @shuffle_v16i16_00_00_00_00_00_00_00_00_00_00_00_00_00_02_00_0
 ;
 ; AVX512VL-FAST-LABEL: shuffle_v16i16_00_00_00_00_00_00_00_00_00_00_00_00_00_02_00_00:
 ; AVX512VL-FAST:       # %bb.0:
-; AVX512VL-FAST-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,0,1,0,1,0,1,0,1,4,5,0,1,0,1]
-; AVX512VL-FAST-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,0,0,1]
+; AVX512VL-FAST-NEXT:    vmovdqa {{.*#+}} ymm1 = [0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0]
+; AVX512VL-FAST-NEXT:    vpermw %ymm0, %ymm1, %ymm0
 ; AVX512VL-FAST-NEXT:    retq
   %shuffle = shufflevector <16 x i16> %a, <16 x i16> %b, <16 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 2, i32 0, i32 0>
   ret <16 x i16> %shuffle
@@ -118,8 +120,9 @@ define <16 x i16> @shuffle_v16i16_00_00_00_00_00_00_00_00_00_00_00_00_03_00_00_0
 ;
 ; AVX2-FAST-LABEL: shuffle_v16i16_00_00_00_00_00_00_00_00_00_00_00_00_03_00_00_00:
 ; AVX2-FAST:       # %bb.0:
-; AVX2-FAST-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,0,1,0,1,0,1,6,7,0,1,0,1,0,1]
-; AVX2-FAST-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,0,0,1]
+; AVX2-FAST-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,3,0,4,5,6,7]
+; AVX2-FAST-NEXT:    vmovdqa {{.*#+}} ymm1 = [0,0,0,0,0,0,1,0]
+; AVX2-FAST-NEXT:    vpermd %ymm0, %ymm1, %ymm0
 ; AVX2-FAST-NEXT:    retq
 ;
 ; AVX512VL-SLOW-LABEL: shuffle_v16i16_00_00_00_00_00_00_00_00_00_00_00_00_03_00_00_00:
@@ -131,8 +134,8 @@ define <16 x i16> @shuffle_v16i16_00_00_00_00_00_00_00_00_00_00_00_00_03_00_00_0
 ;
 ; AVX512VL-FAST-LABEL: shuffle_v16i16_00_00_00_00_00_00_00_00_00_00_00_00_03_00_00_00:
 ; AVX512VL-FAST:       # %bb.0:
-; AVX512VL-FAST-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,0,1,0,1,0,1,6,7,0,1,0,1,0,1]
-; AVX512VL-FAST-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,0,0,1]
+; AVX512VL-FAST-NEXT:    vmovdqa {{.*#+}} ymm1 = [0,0,0,0,0,0,0,0,0,0,0,0,3,0,0,0]
+; AVX512VL-FAST-NEXT:    vpermw %ymm0, %ymm1, %ymm0
 ; AVX512VL-FAST-NEXT:    retq
   %shuffle = shufflevector <16 x i16> %a, <16 x i16> %b, <16 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 3, i32 0, i32 0, i32 0>
   ret <16 x i16> %shuffle
@@ -147,11 +150,33 @@ define <16 x i16> @shuffle_v16i16_00_00_00_00_00_00_00_00_00_00_00_04_00_00_00_0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; AVX1-NEXT:    retq
 ;
-; AVX2OR512VL-LABEL: shuffle_v16i16_00_00_00_00_00_00_00_00_00_00_00_04_00_00_00_00:
-; AVX2OR512VL:       # %bb.0:
-; AVX2OR512VL-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,0,1,0,1,0,1,0,1,0,1,0,1,8,9]
-; AVX2OR512VL-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,0,1,0]
-; AVX2OR512VL-NEXT:    retq
+; AVX2-SLOW-LABEL: shuffle_v16i16_00_00_00_00_00_00_00_00_00_00_00_04_00_00_00_00:
+; AVX2-SLOW:       # %bb.0:
+; AVX2-SLOW-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,1,2,0]
+; AVX2-SLOW-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7]
+; AVX2-SLOW-NEXT:    vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,6,6,6,4]
+; AVX2-SLOW-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,0,1,0]
+; AVX2-SLOW-NEXT:    retq
+;
+; AVX2-FAST-LABEL: shuffle_v16i16_00_00_00_00_00_00_00_00_00_00_00_04_00_00_00_00:
+; AVX2-FAST:       # %bb.0:
+; AVX2-FAST-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,0,1,0,1,0,1,0,1,0,1,0,1,8,9]
+; AVX2-FAST-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,0,1,0]
+; AVX2-FAST-NEXT:    retq
+;
+; AVX512VL-SLOW-LABEL: shuffle_v16i16_00_00_00_00_00_00_00_00_00_00_00_04_00_00_00_00:
+; AVX512VL-SLOW:       # %bb.0:
+; AVX512VL-SLOW-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,1,2,0]
+; AVX512VL-SLOW-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7]
+; AVX512VL-SLOW-NEXT:    vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,6,6,6,4]
+; AVX512VL-SLOW-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,0,1,0]
+; AVX512VL-SLOW-NEXT:    retq
+;
+; AVX512VL-FAST-LABEL: shuffle_v16i16_00_00_00_00_00_00_00_00_00_00_00_04_00_00_00_00:
+; AVX512VL-FAST:       # %bb.0:
+; AVX512VL-FAST-NEXT:    vmovdqa {{.*#+}} ymm1 = [0,0,0,0,0,0,0,0,0,0,0,4,0,0,0,0]
+; AVX512VL-FAST-NEXT:    vpermw %ymm0, %ymm1, %ymm0
+; AVX512VL-FAST-NEXT:    retq
   %shuffle = shufflevector <16 x i16> %a, <16 x i16> %b, <16 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 4, i32 0, i32 0, i32 0, i32 0>
   ret <16 x i16> %shuffle
 }
@@ -165,11 +190,33 @@ define <16 x i16> @shuffle_v16i16_00_00_00_00_00_00_00_00_00_00_05_00_00_00_00_0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; AVX1-NEXT:    retq
 ;
-; AVX2OR512VL-LABEL: shuffle_v16i16_00_00_00_00_00_00_00_00_00_00_05_00_00_00_00_00:
-; AVX2OR512VL:       # %bb.0:
-; AVX2OR512VL-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,0,1,0,1,0,1,0,1,0,1,10,11,0,1]
-; AVX2OR512VL-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,0,1,0]
-; AVX2OR512VL-NEXT:    retq
+; AVX2-SLOW-LABEL: shuffle_v16i16_00_00_00_00_00_00_00_00_00_00_05_00_00_00_00_00:
+; AVX2-SLOW:       # %bb.0:
+; AVX2-SLOW-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,1,2,0]
+; AVX2-SLOW-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7]
+; AVX2-SLOW-NEXT:    vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,6,6,5,6]
+; AVX2-SLOW-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,0,1,0]
+; AVX2-SLOW-NEXT:    retq
+;
+; AVX2-FAST-LABEL: shuffle_v16i16_00_00_00_00_00_00_00_00_00_00_05_00_00_00_00_00:
+; AVX2-FAST:       # %bb.0:
+; AVX2-FAST-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,0,1,0,1,0,1,0,1,0,1,10,11,0,1]
+; AVX2-FAST-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,0,1,0]
+; AVX2-FAST-NEXT:    retq
+;
+; AVX512VL-SLOW-LABEL: shuffle_v16i16_00_00_00_00_00_00_00_00_00_00_05_00_00_00_00_00:
+; AVX512VL-SLOW:       # %bb.0:
+; AVX512VL-SLOW-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,1,2,0]
+; AVX512VL-SLOW-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7]
+; AVX512VL-SLOW-NEXT:    vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,6,6,5,6]
+; AVX512VL-SLOW-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,0,1,0]
+; AVX512VL-SLOW-NEXT:    retq
+;
+; AVX512VL-FAST-LABEL: shuffle_v16i16_00_00_00_00_00_00_00_00_00_00_05_00_00_00_00_00:
+; AVX512VL-FAST:       # %bb.0:
+; AVX512VL-FAST-NEXT:    vmovdqa {{.*#+}} ymm1 = [0,0,0,0,0,0,0,0,0,0,5,0,0,0,0,0]
+; AVX512VL-FAST-NEXT:    vpermw %ymm0, %ymm1, %ymm0
+; AVX512VL-FAST-NEXT:    retq
   %shuffle = shufflevector <16 x i16> %a, <16 x i16> %b, <16 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 5, i32 0, i32 0, i32 0, i32 0, i32 0>
   ret <16 x i16> %shuffle
 }
@@ -183,11 +230,33 @@ define <16 x i16> @shuffle_v16i16_00_00_00_00_00_00_00_00_00_06_00_00_00_00_00_0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; AVX1-NEXT:    retq
 ;
-; AVX2OR512VL-LABEL: shuffle_v16i16_00_00_00_00_00_00_00_00_00_06_00_00_00_00_00_00:
-; AVX2OR512VL:       # %bb.0:
-; AVX2OR512VL-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,0,1,0,1,0,1,0,1,12,13,0,1,0,1]
-; AVX2OR512VL-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,0,1,0]
-; AVX2OR512VL-NEXT:    retq
+; AVX2-SLOW-LABEL: shuffle_v16i16_00_00_00_00_00_00_00_00_00_06_00_00_00_00_00_00:
+; AVX2-SLOW:       # %bb.0:
+; AVX2-SLOW-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,1,0,3]
+; AVX2-SLOW-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7]
+; AVX2-SLOW-NEXT:    vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,6,4,4]
+; AVX2-SLOW-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,0,1,0]
+; AVX2-SLOW-NEXT:    retq
+;
+; AVX2-FAST-LABEL: shuffle_v16i16_00_00_00_00_00_00_00_00_00_06_00_00_00_00_00_00:
+; AVX2-FAST:       # %bb.0:
+; AVX2-FAST-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,0,1,0,1,0,1,0,1,12,13,0,1,0,1]
+; AVX2-FAST-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,0,1,0]
+; AVX2-FAST-NEXT:    retq
+;
+; AVX512VL-SLOW-LABEL: shuffle_v16i16_00_00_00_00_00_00_00_00_00_06_00_00_00_00_00_00:
+; AVX512VL-SLOW:       # %bb.0:
+; AVX512VL-SLOW-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,1,0,3]
+; AVX512VL-SLOW-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7]
+; AVX512VL-SLOW-NEXT:    vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,6,4,4]
+; AVX512VL-SLOW-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,0,1,0]
+; AVX512VL-SLOW-NEXT:    retq
+;
+; AVX512VL-FAST-LABEL: shuffle_v16i16_00_00_00_00_00_00_00_00_00_06_00_00_00_00_00_00:
+; AVX512VL-FAST:       # %bb.0:
+; AVX512VL-FAST-NEXT:    vmovdqa {{.*#+}} ymm1 = [0,0,0,0,0,0,0,0,0,6,0,0,0,0,0,0]
+; AVX512VL-FAST-NEXT:    vpermw %ymm0, %ymm1, %ymm0
+; AVX512VL-FAST-NEXT:    retq
   %shuffle = shufflevector <16 x i16> %a, <16 x i16> %b, <16 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 6, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0>
   ret <16 x i16> %shuffle
 }
@@ -201,11 +270,33 @@ define <16 x i16> @shuffle_v16i16_00_00_00_00_00_00_00_00_07_00_00_00_00_00_00_0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm0
 ; AVX1-NEXT:    retq
 ;
-; AVX2OR512VL-LABEL: shuffle_v16i16_00_00_00_00_00_00_00_00_07_00_00_00_00_00_00_00:
-; AVX2OR512VL:       # %bb.0:
-; AVX2OR512VL-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,0,1,0,1,0,1,14,15,0,1,0,1,0,1]
-; AVX2OR512VL-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,0,1,0]
-; AVX2OR512VL-NEXT:    retq
+; AVX2-SLOW-LABEL: shuffle_v16i16_00_00_00_00_00_00_00_00_07_00_00_00_00_00_00_00:
+; AVX2-SLOW:       # %bb.0:
+; AVX2-SLOW-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,1,0,3]
+; AVX2-SLOW-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7]
+; AVX2-SLOW-NEXT:    vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,7,4,4,4]
+; AVX2-SLOW-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,0,1,0]
+; AVX2-SLOW-NEXT:    retq
+;
+; AVX2-FAST-LABEL: shuffle_v16i16_00_00_00_00_00_00_00_00_07_00_00_00_00_00_00_00:
+; AVX2-FAST:       # %bb.0:
+; AVX2-FAST-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,0,1,0,1,0,1,14,15,0,1,0,1,0,1]
+; AVX2-FAST-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,0,1,0]
+; AVX2-FAST-NEXT:    retq
+;
+; AVX512VL-SLOW-LABEL: shuffle_v16i16_00_00_00_00_00_00_00_00_07_00_00_00_00_00_00_00:
+; AVX512VL-SLOW:       # %bb.0:
+; AVX512VL-SLOW-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,1,0,3]
+; AVX512VL-SLOW-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7]
+; AVX512VL-SLOW-NEXT:    vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,7,4,4,4]
+; AVX512VL-SLOW-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,0,1,0]
+; AVX512VL-SLOW-NEXT:    retq
+;
+; AVX512VL-FAST-LABEL: shuffle_v16i16_00_00_00_00_00_00_00_00_07_00_00_00_00_00_00_00:
+; AVX512VL-FAST:       # %bb.0:
+; AVX512VL-FAST-NEXT:    vmovdqa {{.*#+}} ymm1 = [0,0,0,0,0,0,0,0,7,0,0,0,0,0,0,0]
+; AVX512VL-FAST-NEXT:    vpermw %ymm0, %ymm1, %ymm0
+; AVX512VL-FAST-NEXT:    retq
   %shuffle = shufflevector <16 x i16> %a, <16 x i16> %b, <16 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 7, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0>
   ret <16 x i16> %shuffle
 }
@@ -685,8 +776,9 @@ define <16 x i16> @shuffle_v16i16_00_00_00_00_00_00_01_00_00_00_00_00_00_00_01_0
 ;
 ; AVX2-FAST-LABEL: shuffle_v16i16_00_00_00_00_00_00_01_00_00_00_00_00_00_00_01_00:
 ; AVX2-FAST:       # %bb.0:
-; AVX2-FAST-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,0,1,0,1,0,1,0,1,0,1,2,3,0,1]
-; AVX2-FAST-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,1,0,1]
+; AVX2-FAST-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,1,0,4,5,6,7]
+; AVX2-FAST-NEXT:    vmovdqa {{.*#+}} ymm1 = [0,0,0,1,0,0,0,1]
+; AVX2-FAST-NEXT:    vpermd %ymm0, %ymm1, %ymm0
 ; AVX2-FAST-NEXT:    retq
 ;
 ; AVX512VL-SLOW-LABEL: shuffle_v16i16_00_00_00_00_00_00_01_00_00_00_00_00_00_00_01_00:
@@ -698,8 +790,8 @@ define <16 x i16> @shuffle_v16i16_00_00_00_00_00_00_01_00_00_00_00_00_00_00_01_0
 ;
 ; AVX512VL-FAST-LABEL: shuffle_v16i16_00_00_00_00_00_00_01_00_00_00_00_00_00_00_01_00:
 ; AVX512VL-FAST:       # %bb.0:
-; AVX512VL-FAST-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,0,1,0,1,0,1,0,1,0,1,2,3,0,1]
-; AVX512VL-FAST-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,1,0,1]
+; AVX512VL-FAST-NEXT:    vmovdqa {{.*#+}} ymm1 = [0,0,0,0,0,0,1,0,0,0,0,0,0,0,1,0]
+; AVX512VL-FAST-NEXT:    vpermw %ymm0, %ymm1, %ymm0
 ; AVX512VL-FAST-NEXT:    retq
   %shuffle = shufflevector <16 x i16> %a, <16 x i16> %b, <16 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 1, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 1, i32 0>
   ret <16 x i16> %shuffle
@@ -722,8 +814,9 @@ define <16 x i16> @shuffle_v16i16_00_00_00_00_00_02_00_00_00_00_00_00_00_02_00_0
 ;
 ; AVX2-FAST-LABEL: shuffle_v16i16_00_00_00_00_00_02_00_00_00_00_00_00_00_02_00_00:
 ; AVX2-FAST:       # %bb.0:
-; AVX2-FAST-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,0,1,0,1,0,1,0,1,4,5,0,1,0,1]
-; AVX2-FAST-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,1,0,1]
+; AVX2-FAST-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,0,2,4,5,6,7]
+; AVX2-FAST-NEXT:    vmovdqa {{.*#+}} ymm1 = [0,0,1,0,0,0,1,0]
+; AVX2-FAST-NEXT:    vpermd %ymm0, %ymm1, %ymm0
 ; AVX2-FAST-NEXT:    retq
 ;
 ; AVX512VL-SLOW-LABEL: shuffle_v16i16_00_00_00_00_00_02_00_00_00_00_00_00_00_02_00_00:
@@ -735,8 +828,8 @@ define <16 x i16> @shuffle_v16i16_00_00_00_00_00_02_00_00_00_00_00_00_00_02_00_0
 ;
 ; AVX512VL-FAST-LABEL: shuffle_v16i16_00_00_00_00_00_02_00_00_00_00_00_00_00_02_00_00:
 ; AVX512VL-FAST:       # %bb.0:
-; AVX512VL-FAST-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,0,1,0,1,0,1,0,1,4,5,0,1,0,1]
-; AVX512VL-FAST-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,1,0,1]
+; AVX512VL-FAST-NEXT:    vmovdqa {{.*#+}} ymm1 = [0,0,0,0,0,2,0,0,0,0,0,0,0,2,0,0]
+; AVX512VL-FAST-NEXT:    vpermw %ymm0, %ymm1, %ymm0
 ; AVX512VL-FAST-NEXT:    retq
   %shuffle = shufflevector <16 x i16> %a, <16 x i16> %b, <16 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 2, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 2, i32 0, i32 0>
   ret <16 x i16> %shuffle
@@ -759,8 +852,9 @@ define <16 x i16> @shuffle_v16i16_00_00_00_00_03_00_00_00_00_00_00_00_03_00_00_0
 ;
 ; AVX2-FAST-LABEL: shuffle_v16i16_00_00_00_00_03_00_00_00_00_00_00_00_03_00_00_00:
 ; AVX2-FAST:       # %bb.0:
-; AVX2-FAST-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,0,1,0,1,0,1,6,7,0,1,0,1,0,1]
-; AVX2-FAST-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,1,0,1]
+; AVX2-FAST-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,3,0,4,5,6,7]
+; AVX2-FAST-NEXT:    vmovdqa {{.*#+}} ymm1 = [0,0,1,0,0,0,1,0]
+; AVX2-FAST-NEXT:    vpermd %ymm0, %ymm1, %ymm0
 ; AVX2-FAST-NEXT:    retq
 ;
 ; AVX512VL-SLOW-LABEL: shuffle_v16i16_00_00_00_00_03_00_00_00_00_00_00_00_03_00_00_00:
@@ -772,8 +866,8 @@ define <16 x i16> @shuffle_v16i16_00_00_00_00_03_00_00_00_00_00_00_00_03_00_00_0
 ;
 ; AVX512VL-FAST-LABEL: shuffle_v16i16_00_00_00_00_03_00_00_00_00_00_00_00_03_00_00_00:
 ; AVX512VL-FAST:       # %bb.0:
-; AVX512VL-FAST-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,0,1,0,1,0,1,6,7,0,1,0,1,0,1]
-; AVX512VL-FAST-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,1,0,1]
+; AVX512VL-FAST-NEXT:    vmovdqa {{.*#+}} ymm1 = [0,0,0,0,3,0,0,0,0,0,0,0,3,0,0,0]
+; AVX512VL-FAST-NEXT:    vpermw %ymm0, %ymm1, %ymm0
 ; AVX512VL-FAST-NEXT:    retq
   %shuffle = shufflevector <16 x i16> %a, <16 x i16> %b, <16 x i32> <i32 0, i32 0, i32 0, i32 0, i32 3, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 3, i32 0, i32 0, i32 0>
   ret <16 x i16> %shuffle
@@ -786,11 +880,33 @@ define <16 x i16> @shuffle_v16i16_00_00_00_04_00_00_00_00_00_00_00_04_00_00_00_0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
 ; AVX1-NEXT:    retq
 ;
-; AVX2OR512VL-LABEL: shuffle_v16i16_00_00_00_04_00_00_00_00_00_00_00_04_00_00_00_00:
-; AVX2OR512VL:       # %bb.0:
-; AVX2OR512VL-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,0,1,0,1,8,9,0,1,0,1,0,1,0,1]
-; AVX2OR512VL-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,1,0,1]
-; AVX2OR512VL-NEXT:    retq
+; AVX2-SLOW-LABEL: shuffle_v16i16_00_00_00_04_00_00_00_00_00_00_00_04_00_00_00_00:
+; AVX2-SLOW:       # %bb.0:
+; AVX2-SLOW-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,0,3]
+; AVX2-SLOW-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,0,2,4,5,6,7]
+; AVX2-SLOW-NEXT:    vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,4,4,4]
+; AVX2-SLOW-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,1,0,1]
+; AVX2-SLOW-NEXT:    retq
+;
+; AVX2-FAST-LABEL: shuffle_v16i16_00_00_00_04_00_00_00_00_00_00_00_04_00_00_00_00:
+; AVX2-FAST:       # %bb.0:
+; AVX2-FAST-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,0,1,0,1,8,9,0,1,0,1,0,1,0,1]
+; AVX2-FAST-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,1,0,1]
+; AVX2-FAST-NEXT:    retq
+;
+; AVX512VL-SLOW-LABEL: shuffle_v16i16_00_00_00_04_00_00_00_00_00_00_00_04_00_00_00_00:
+; AVX512VL-SLOW:       # %bb.0:
+; AVX512VL-SLOW-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,0,3]
+; AVX512VL-SLOW-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,0,2,4,5,6,7]
+; AVX512VL-SLOW-NEXT:    vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,4,4,4]
+; AVX512VL-SLOW-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,1,0,1]
+; AVX512VL-SLOW-NEXT:    retq
+;
+; AVX512VL-FAST-LABEL: shuffle_v16i16_00_00_00_04_00_00_00_00_00_00_00_04_00_00_00_00:
+; AVX512VL-FAST:       # %bb.0:
+; AVX512VL-FAST-NEXT:    vmovdqa {{.*#+}} ymm1 = [0,0,0,4,0,0,0,0,0,0,0,4,0,0,0,0]
+; AVX512VL-FAST-NEXT:    vpermw %ymm0, %ymm1, %ymm0
+; AVX512VL-FAST-NEXT:    retq
   %shuffle = shufflevector <16 x i16> %a, <16 x i16> %b, <16 x i32> <i32 0, i32 0, i32 0, i32 4, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 4, i32 0, i32 0, i32 0, i32 0>
   ret <16 x i16> %shuffle
 }
@@ -802,11 +918,33 @@ define <16 x i16> @shuffle_v16i16_00_00_05_00_00_00_00_00_00_00_05_00_00_00_00_0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
 ; AVX1-NEXT:    retq
 ;
-; AVX2OR512VL-LABEL: shuffle_v16i16_00_00_05_00_00_00_00_00_00_00_05_00_00_00_00_00:
-; AVX2OR512VL:       # %bb.0:
-; AVX2OR512VL-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,0,1,10,11,0,1,0,1,0,1,0,1,0,1]
-; AVX2OR512VL-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,1,0,1]
-; AVX2OR512VL-NEXT:    retq
+; AVX2-SLOW-LABEL: shuffle_v16i16_00_00_05_00_00_00_00_00_00_00_05_00_00_00_00_00:
+; AVX2-SLOW:       # %bb.0:
+; AVX2-SLOW-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,0,3]
+; AVX2-SLOW-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,3,0,4,5,6,7]
+; AVX2-SLOW-NEXT:    vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,4,4,4]
+; AVX2-SLOW-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,1,0,1]
+; AVX2-SLOW-NEXT:    retq
+;
+; AVX2-FAST-LABEL: shuffle_v16i16_00_00_05_00_00_00_00_00_00_00_05_00_00_00_00_00:
+; AVX2-FAST:       # %bb.0:
+; AVX2-FAST-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,0,1,10,11,0,1,0,1,0,1,0,1,0,1]
+; AVX2-FAST-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,1,0,1]
+; AVX2-FAST-NEXT:    retq
+;
+; AVX512VL-SLOW-LABEL: shuffle_v16i16_00_00_05_00_00_00_00_00_00_00_05_00_00_00_00_00:
+; AVX512VL-SLOW:       # %bb.0:
+; AVX512VL-SLOW-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,0,3]
+; AVX512VL-SLOW-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,0,3,0,4,5,6,7]
+; AVX512VL-SLOW-NEXT:    vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,4,4,4]
+; AVX512VL-SLOW-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,1,0,1]
+; AVX512VL-SLOW-NEXT:    retq
+;
+; AVX512VL-FAST-LABEL: shuffle_v16i16_00_00_05_00_00_00_00_00_00_00_05_00_00_00_00_00:
+; AVX512VL-FAST:       # %bb.0:
+; AVX512VL-FAST-NEXT:    vmovdqa {{.*#+}} ymm1 = [0,0,5,0,0,0,0,0,0,0,5,0,0,0,0,0]
+; AVX512VL-FAST-NEXT:    vpermw %ymm0, %ymm1, %ymm0
+; AVX512VL-FAST-NEXT:    retq
   %shuffle = shufflevector <16 x i16> %a, <16 x i16> %b, <16 x i32> <i32 0, i32 0, i32 5, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 5, i32 0, i32 0, i32 0, i32 0, i32 0>
   ret <16 x i16> %shuffle
 }
@@ -818,11 +956,33 @@ define <16 x i16> @shuffle_v16i16_00_06_00_00_00_00_00_00_00_06_00_00_00_00_00_0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
 ; AVX1-NEXT:    retq
 ;
-; AVX2OR512VL-LABEL: shuffle_v16i16_00_06_00_00_00_00_00_00_00_06_00_00_00_00_00_00:
-; AVX2OR512VL:       # %bb.0:
-; AVX2OR512VL-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,12,13,0,1,0,1,0,1,0,1,0,1,0,1]
-; AVX2OR512VL-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,1,0,1]
-; AVX2OR512VL-NEXT:    retq
+; AVX2-SLOW-LABEL: shuffle_v16i16_00_06_00_00_00_00_00_00_00_06_00_00_00_00_00_00:
+; AVX2-SLOW:       # %bb.0:
+; AVX2-SLOW-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,3,0,3]
+; AVX2-SLOW-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,0,0,4,5,6,7]
+; AVX2-SLOW-NEXT:    vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,4,4,4]
+; AVX2-SLOW-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,1,0,1]
+; AVX2-SLOW-NEXT:    retq
+;
+; AVX2-FAST-LABEL: shuffle_v16i16_00_06_00_00_00_00_00_00_00_06_00_00_00_00_00_00:
+; AVX2-FAST:       # %bb.0:
+; AVX2-FAST-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,12,13,0,1,0,1,0,1,0,1,0,1,0,1]
+; AVX2-FAST-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,1,0,1]
+; AVX2-FAST-NEXT:    retq
+;
+; AVX512VL-SLOW-LABEL: shuffle_v16i16_00_06_00_00_00_00_00_00_00_06_00_00_00_00_00_00:
+; AVX512VL-SLOW:       # %bb.0:
+; AVX512VL-SLOW-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,3,0,3]
+; AVX512VL-SLOW-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,0,0,4,5,6,7]
+; AVX512VL-SLOW-NEXT:    vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,4,4,4]
+; AVX512VL-SLOW-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,1,0,1]
+; AVX512VL-SLOW-NEXT:    retq
+;
+; AVX512VL-FAST-LABEL: shuffle_v16i16_00_06_00_00_00_00_00_00_00_06_00_00_00_00_00_00:
+; AVX512VL-FAST:       # %bb.0:
+; AVX512VL-FAST-NEXT:    vmovdqa {{.*#+}} ymm1 = [0,6,0,0,0,0,0,0,0,6,0,0,0,0,0,0]
+; AVX512VL-FAST-NEXT:    vpermw %ymm0, %ymm1, %ymm0
+; AVX512VL-FAST-NEXT:    retq
   %shuffle = shufflevector <16 x i16> %a, <16 x i16> %b, <16 x i32> <i32 0, i32 6, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 6, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0>
   ret <16 x i16> %shuffle
 }
@@ -834,11 +994,33 @@ define <16 x i16> @shuffle_v16i16_07_00_00_00_00_00_00_00_07_00_00_00_00_00_00_0
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
 ; AVX1-NEXT:    retq
 ;
-; AVX2OR512VL-LABEL: shuffle_v16i16_07_00_00_00_00_00_00_00_07_00_00_00_00_00_00_00:
-; AVX2OR512VL:       # %bb.0:
-; AVX2OR512VL-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[14,15,0,1,0,1,0,1,0,1,0,1,0,1,0,1]
-; AVX2OR512VL-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,1,0,1]
-; AVX2OR512VL-NEXT:    retq
+; AVX2-SLOW-LABEL: shuffle_v16i16_07_00_00_00_00_00_00_00_07_00_00_00_00_00_00_00:
+; AVX2-SLOW:       # %bb.0:
+; AVX2-SLOW-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,3,0,3]
+; AVX2-SLOW-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[3,0,0,0,4,5,6,7]
+; AVX2-SLOW-NEXT:    vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,4,4,4]
+; AVX2-SLOW-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,1,0,1]
+; AVX2-SLOW-NEXT:    retq
+;
+; AVX2-FAST-LABEL: shuffle_v16i16_07_00_00_00_00_00_00_00_07_00_00_00_00_00_00_00:
+; AVX2-FAST:       # %bb.0:
+; AVX2-FAST-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[14,15,0,1,0,1,0,1,0,1,0,1,0,1,0,1]
+; AVX2-FAST-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,1,0,1]
+; AVX2-FAST-NEXT:    retq
+;
+; AVX512VL-SLOW-LABEL: shuffle_v16i16_07_00_00_00_00_00_00_00_07_00_00_00_00_00_00_00:
+; AVX512VL-SLOW:       # %bb.0:
+; AVX512VL-SLOW-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,3,0,3]
+; AVX512VL-SLOW-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[3,0,0,0,4,5,6,7]
+; AVX512VL-SLOW-NEXT:    vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,4,4,4]
+; AVX512VL-SLOW-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,1,0,1]
+; AVX512VL-SLOW-NEXT:    retq
+;
+; AVX512VL-FAST-LABEL: shuffle_v16i16_07_00_00_00_00_00_00_00_07_00_00_00_00_00_00_00:
+; AVX512VL-FAST:       # %bb.0:
+; AVX512VL-FAST-NEXT:    vmovdqa {{.*#+}} ymm1 = [7,0,0,0,0,0,0,0,7,0,0,0,0,0,0,0]
+; AVX512VL-FAST-NEXT:    vpermw %ymm0, %ymm1, %ymm0
+; AVX512VL-FAST-NEXT:    retq
   %shuffle = shufflevector <16 x i16> %a, <16 x i16> %b, <16 x i32> <i32 7, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 7, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0>
   ret <16 x i16> %shuffle
 }
@@ -4145,8 +4327,7 @@ define <16 x i16> @shuffle_v16i16_u_u_u_u_u_u_u_u_3_3_3_3_3_3_3_3(<16 x i16> %a,
 ; AVX2-SLOW-LABEL: shuffle_v16i16_u_u_u_u_u_u_u_u_3_3_3_3_3_3_3_3:
 ; AVX2-SLOW:       # %bb.0:
 ; AVX2-SLOW-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[3,3,2,3,4,5,6,7]
-; AVX2-SLOW-NEXT:    vpbroadcastd %xmm0, %xmm0
-; AVX2-SLOW-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
+; AVX2-SLOW-NEXT:    vpbroadcastd %xmm0, %ymm0
 ; AVX2-SLOW-NEXT:    retq
 ;
 ; AVX2-FAST-LABEL: shuffle_v16i16_u_u_u_u_u_u_u_u_3_3_3_3_3_3_3_3:
@@ -4158,8 +4339,7 @@ define <16 x i16> @shuffle_v16i16_u_u_u_u_u_u_u_u_3_3_3_3_3_3_3_3(<16 x i16> %a,
 ; AVX512VL-SLOW-LABEL: shuffle_v16i16_u_u_u_u_u_u_u_u_3_3_3_3_3_3_3_3:
 ; AVX512VL-SLOW:       # %bb.0:
 ; AVX512VL-SLOW-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[3,3,2,3,4,5,6,7]
-; AVX512VL-SLOW-NEXT:    vpbroadcastd %xmm0, %xmm0
-; AVX512VL-SLOW-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
+; AVX512VL-SLOW-NEXT:    vpbroadcastd %xmm0, %ymm0
 ; AVX512VL-SLOW-NEXT:    retq
 ;
 ; AVX512VL-FAST-LABEL: shuffle_v16i16_u_u_u_u_u_u_u_u_3_3_3_3_3_3_3_3:
@@ -4492,17 +4672,31 @@ define <16 x i16> @PR34369(<16 x i16> %vec, <16 x i16> %mask) {
 ; AVX1-NEXT:    vandps %ymm0, %ymm1, %ymm0
 ; AVX1-NEXT:    retq
 ;
-; AVX2-LABEL: PR34369:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpermq {{.*#+}} ymm2 = ymm0[2,3,0,1]
-; AVX2-NEXT:    vpshufb {{.*#+}} xmm2 = xmm2[8,9,10,11,4,5,10,11,8,9,10,11,4,5,4,5]
-; AVX2-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[6,7,0,1,0,1,u,u,10,11,4,5,4,5,u,u,30,31,16,17,28,29,16,17,18,19,20,21,24,25,24,25]
-; AVX2-NEXT:    vpblendw {{.*#+}} ymm2 = ymm0[0,1,2],ymm2[3],ymm0[4,5,6],ymm2[7],ymm0[8,9,10],ymm2[11],ymm0[12,13,14],ymm2[15]
-; AVX2-NEXT:    vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3],ymm0[4,5,6,7]
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpeqw %ymm2, %ymm1, %ymm1
-; AVX2-NEXT:    vpand %ymm0, %ymm1, %ymm0
-; AVX2-NEXT:    retq
+; AVX2-SLOW-LABEL: PR34369:
+; AVX2-SLOW:       # %bb.0:
+; AVX2-SLOW-NEXT:    vpermq {{.*#+}} ymm2 = ymm0[2,3,0,1]
+; AVX2-SLOW-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,1,2,1]
+; AVX2-SLOW-NEXT:    vpshuflw {{.*#+}} xmm2 = xmm2[0,1,2,1,4,5,6,7]
+; AVX2-SLOW-NEXT:    vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,5,6,6]
+; AVX2-SLOW-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[6,7,0,1,0,1,u,u,10,11,4,5,4,5,u,u,30,31,16,17,28,29,16,17,18,19,20,21,24,25,24,25]
+; AVX2-SLOW-NEXT:    vpblendw {{.*#+}} ymm2 = ymm0[0,1,2],ymm2[3],ymm0[4,5,6],ymm2[7],ymm0[8,9,10],ymm2[11],ymm0[12,13,14],ymm2[15]
+; AVX2-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3],ymm0[4,5,6,7]
+; AVX2-SLOW-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX2-SLOW-NEXT:    vpcmpeqw %ymm2, %ymm1, %ymm1
+; AVX2-SLOW-NEXT:    vpand %ymm0, %ymm1, %ymm0
+; AVX2-SLOW-NEXT:    retq
+;
+; AVX2-FAST-LABEL: PR34369:
+; AVX2-FAST:       # %bb.0:
+; AVX2-FAST-NEXT:    vpermq {{.*#+}} ymm2 = ymm0[2,3,0,1]
+; AVX2-FAST-NEXT:    vpshufb {{.*#+}} xmm2 = xmm2[8,9,10,11,4,5,10,11,8,9,10,11,4,5,4,5]
+; AVX2-FAST-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[6,7,0,1,0,1,u,u,10,11,4,5,4,5,u,u,30,31,16,17,28,29,16,17,18,19,20,21,24,25,24,25]
+; AVX2-FAST-NEXT:    vpblendw {{.*#+}} ymm2 = ymm0[0,1,2],ymm2[3],ymm0[4,5,6],ymm2[7],ymm0[8,9,10],ymm2[11],ymm0[12,13,14],ymm2[15]
+; AVX2-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3],ymm0[4,5,6,7]
+; AVX2-FAST-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX2-FAST-NEXT:    vpcmpeqw %ymm2, %ymm1, %ymm1
+; AVX2-FAST-NEXT:    vpand %ymm0, %ymm1, %ymm0
+; AVX2-FAST-NEXT:    retq
 ;
 ; AVX512VL-LABEL: PR34369:
 ; AVX512VL:       # %bb.0:
@@ -4605,3 +4799,20 @@ define <16 x i16> @insert_dup_elt3_mem_v16i16_i32(i32* %ptr) #0 {
   %tmp3 = shufflevector <8 x i16> %tmp2, <8 x i16> undef, <16 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
   ret <16 x i16> %tmp3
 }
+
+define <16 x i16> @unpckh_v16i16(<16 x i16> %x, <16 x i16> %y) {
+; AVX1-LABEL: unpckh_v16i16:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm1
+; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; AVX1-NEXT:    retq
+;
+; AVX2OR512VL-LABEL: unpckh_v16i16:
+; AVX2OR512VL:       # %bb.0:
+; AVX2OR512VL-NEXT:    vextracti128 $1, %ymm1, %xmm1
+; AVX2OR512VL-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; AVX2OR512VL-NEXT:    retq
+  %unpckh = shufflevector <16 x i16> %x, <16 x i16> %y, <16 x i32> <i32 4, i32 28, i32 5, i32 29, i32 6, i32 30, i32 7, i32 31, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  ret <16 x i16> %unpckh
+}
+
diff --git a/test/CodeGen/X86/vector-shuffle-256-v32.ll b/test/CodeGen/X86/vector-shuffle-256-v32.ll
index ba0e2086f4888d8d07e176b9a2d592290180cdc8..d268364fda86232fc72dd373eec352777d7b9575 100644
--- a/test/CodeGen/X86/vector-shuffle-256-v32.ll
+++ b/test/CodeGen/X86/vector-shuffle-256-v32.ll
@@ -3321,3 +3321,20 @@ define <32 x i8> @zeroable_src_to_zext(<32 x i8> %a0) {
   %2 = shufflevector <32 x i8> %1, <32 x i8> <i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef>, <32 x i32> <i32 8, i32 9, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 10, i32 11, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 20, i32 21, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48, i32 22, i32 23, i32 48, i32 48, i32 48, i32 48, i32 48, i32 48>
   ret <32 x i8> %2
 }
+
+define <32 x i8> @unpckh_v32i8(<32 x i8> %x, <32 x i8> %y) {
+; AVX1-LABEL: unpckh_v32i8:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm1
+; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm0[8],xmm1[8],xmm0[9],xmm1[9],xmm0[10],xmm1[10],xmm0[11],xmm1[11],xmm0[12],xmm1[12],xmm0[13],xmm1[13],xmm0[14],xmm1[14],xmm0[15],xmm1[15]
+; AVX1-NEXT:    retq
+;
+; AVX2OR512VL-LABEL: unpckh_v32i8:
+; AVX2OR512VL:       # %bb.0:
+; AVX2OR512VL-NEXT:    vextracti128 $1, %ymm1, %xmm1
+; AVX2OR512VL-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm0[8],xmm1[8],xmm0[9],xmm1[9],xmm0[10],xmm1[10],xmm0[11],xmm1[11],xmm0[12],xmm1[12],xmm0[13],xmm1[13],xmm0[14],xmm1[14],xmm0[15],xmm1[15]
+; AVX2OR512VL-NEXT:    retq
+  %unpckh = shufflevector <32 x i8> %x, <32 x i8> %y, <32 x i32> <i32 8, i32 56, i32 9, i32 57, i32 10, i32 58, i32 11, i32 59, i32 12, i32 60, i32 13, i32 61, i32 14, i32 62, i32 15, i32 63, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  ret <32 x i8> %unpckh
+}
+
diff --git a/test/CodeGen/X86/vector-shuffle-256-v4.ll b/test/CodeGen/X86/vector-shuffle-256-v4.ll
index b3750b74ad34019cf5828f605735bc89cd96d097..3d0244f708a8b6c0077ba06915c41068d5214d81 100644
--- a/test/CodeGen/X86/vector-shuffle-256-v4.ll
+++ b/test/CodeGen/X86/vector-shuffle-256-v4.ll
@@ -49,7 +49,7 @@ define <4 x double> @shuffle_v4f64_0020(<4 x double> %a, <4 x double> %b) {
 ; AVX1-LABEL: shuffle_v4f64_0020:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vunpcklpd {{.*#+}} xmm1 = xmm1[0],xmm0[0]
+; AVX1-NEXT:    vmovlhps {{.*#+}} xmm1 = xmm1[0],xmm0[0]
 ; AVX1-NEXT:    vmovddup {{.*#+}} xmm0 = xmm0[0,0]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
 ; AVX1-NEXT:    retq
@@ -321,7 +321,7 @@ define <4 x double> @shuffle_v4f64_0423(<4 x double> %a, <4 x double> %b) {
 ; ALL-LABEL: shuffle_v4f64_0423:
 ; ALL:       # %bb.0:
 ; ALL-NEXT:    vmovddup {{.*#+}} xmm1 = xmm1[0,0]
-; ALL-NEXT:    vblendpd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2,3]
+; ALL-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5,6,7]
 ; ALL-NEXT:    retq
   %shuffle = shufflevector <4 x double> %a, <4 x double> %b, <4 x i32> <i32 0, i32 4, i32 2, i32 3>
   ret <4 x double> %shuffle
@@ -923,14 +923,14 @@ define <4 x i64> @shuffle_v4i64_0412(<4 x i64> %a, <4 x i64> %b) {
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
 ; AVX1-NEXT:    vpalignr {{.*#+}} xmm2 = xmm0[8,9,10,11,12,13,14,15],xmm2[0,1,2,3,4,5,6,7]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
-; AVX1-NEXT:    vblendpd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2,3]
+; AVX1-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5,6,7]
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: shuffle_v4i64_0412:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpbroadcastq %xmm1, %xmm1
-; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,1,1,2]
-; AVX2-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5,6,7]
+; AVX2-NEXT:    vmovddup {{.*#+}} xmm1 = xmm1[0,0]
+; AVX2-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[0,1,1,2]
+; AVX2-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5,6,7]
 ; AVX2-NEXT:    retq
 ;
 ; AVX512VL-LABEL: shuffle_v4i64_0412:
@@ -1903,3 +1903,24 @@ define <8 x i32> @shuffle_v8i32_0zzzzzzz_optsize(<8 x i32> %a) optsize {
   %b = shufflevector <8 x i32> %a, <8 x i32> zeroinitializer, <8 x i32> <i32 0, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
   ret <8 x i32> %b
 }
+
+define <4 x i64> @unpckh_v4i64(<4 x i64> %x, <4 x i64> %y) {
+; ALL-LABEL: unpckh_v4i64:
+; ALL:       # %bb.0:
+; ALL-NEXT:    vextractf128 $1, %ymm1, %xmm1
+; ALL-NEXT:    vunpckhpd {{.*#+}} xmm0 = xmm0[1],xmm1[1]
+; ALL-NEXT:    retq
+  %unpckh = shufflevector <4 x i64> %x, <4 x i64> %y, <4 x i32> <i32 1, i32 7, i32 undef, i32 undef>
+  ret <4 x i64> %unpckh
+}
+
+define <4 x double> @unpckh_v4f64(<4 x double> %x, <4 x double> %y) {
+; ALL-LABEL: unpckh_v4f64:
+; ALL:       # %bb.0:
+; ALL-NEXT:    vextractf128 $1, %ymm1, %xmm1
+; ALL-NEXT:    vunpckhpd {{.*#+}} xmm0 = xmm0[1],xmm1[1]
+; ALL-NEXT:    retq
+  %unpckh = shufflevector <4 x double> %x, <4 x double> %y, <4 x i32> <i32 1, i32 7, i32 undef, i32 undef>
+  ret <4 x double> %unpckh
+}
+
diff --git a/test/CodeGen/X86/vector-shuffle-256-v8.ll b/test/CodeGen/X86/vector-shuffle-256-v8.ll
index addf2d2563fc59bd593e7b771de3d642840dda88..064dbf089139b6ee60140adc5f47442a18b75ac5 100644
--- a/test/CodeGen/X86/vector-shuffle-256-v8.ll
+++ b/test/CodeGen/X86/vector-shuffle-256-v8.ll
@@ -1526,9 +1526,8 @@ define <8 x i32> @shuffle_v8i32_08192a3b(<8 x i32> %a, <8 x i32> %b) {
 ;
 ; AVX512VL-LABEL: shuffle_v8i32_08192a3b:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpmovzxdq {{.*#+}} ymm2 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm0 = [0,8,2,9,4,10,6,11]
-; AVX512VL-NEXT:    vpermi2d %ymm1, %ymm2, %ymm0
+; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm2 = [0,8,1,9,2,10,3,11]
+; AVX512VL-NEXT:    vpermt2d %ymm1, %ymm2, %ymm0
 ; AVX512VL-NEXT:    retq
   %shuffle = shufflevector <8 x i32> %a, <8 x i32> %b, <8 x i32> <i32 0, i32 8, i32 1, i32 9, i32 2, i32 10, i32 3, i32 11>
   ret <8 x i32> %shuffle
@@ -1555,9 +1554,9 @@ define <8 x i32> @shuffle_v8i32_08991abb(<8 x i32> %a, <8 x i32> %b) {
 ;
 ; AVX512VL-LABEL: shuffle_v8i32_08991abb:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpmovzxdq {{.*#+}} xmm2 = xmm0[0],zero,xmm0[1],zero
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm0 = [8,0,1,1,10,2,3,3]
-; AVX512VL-NEXT:    vpermi2d %ymm2, %ymm1, %ymm0
+; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm2 = [8,0,1,1,9,2,3,3]
+; AVX512VL-NEXT:    vpermi2d %ymm0, %ymm1, %ymm2
+; AVX512VL-NEXT:    vmovdqa %ymm2, %ymm0
 ; AVX512VL-NEXT:    retq
   %shuffle = shufflevector <8 x i32> %a, <8 x i32> %b, <8 x i32> <i32 0, i32 8, i32 9, i32 9, i32 1, i32 10, i32 11, i32 11>
   ret <8 x i32> %shuffle
@@ -1572,11 +1571,23 @@ define <8 x i32> @shuffle_v8i32_091b2d3f(<8 x i32> %a, <8 x i32> %b) {
 ; AVX1-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2],ymm1[3],ymm0[4],ymm1[5],ymm0[6],ymm1[7]
 ; AVX1-NEXT:    retq
 ;
-; AVX2OR512VL-LABEL: shuffle_v8i32_091b2d3f:
-; AVX2OR512VL:       # %bb.0:
-; AVX2OR512VL-NEXT:    vpmovzxdq {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
-; AVX2OR512VL-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2],ymm1[3],ymm0[4],ymm1[5],ymm0[6],ymm1[7]
-; AVX2OR512VL-NEXT:    retq
+; AVX2-LABEL: shuffle_v8i32_091b2d3f:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpmovzxdq {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; AVX2-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2],ymm1[3],ymm0[4],ymm1[5],ymm0[6],ymm1[7]
+; AVX2-NEXT:    retq
+;
+; AVX512VL-SLOW-LABEL: shuffle_v8i32_091b2d3f:
+; AVX512VL-SLOW:       # %bb.0:
+; AVX512VL-SLOW-NEXT:    vpmovzxdq {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; AVX512VL-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2],ymm1[3],ymm0[4],ymm1[5],ymm0[6],ymm1[7]
+; AVX512VL-SLOW-NEXT:    retq
+;
+; AVX512VL-FAST-LABEL: shuffle_v8i32_091b2d3f:
+; AVX512VL-FAST:       # %bb.0:
+; AVX512VL-FAST-NEXT:    vmovdqa {{.*#+}} ymm2 = [0,9,1,11,2,13,3,15]
+; AVX512VL-FAST-NEXT:    vpermt2d %ymm1, %ymm2, %ymm0
+; AVX512VL-FAST-NEXT:    retq
   %shuffle = shufflevector <8 x i32> %a, <8 x i32> %b, <8 x i32> <i32 0, i32 9, i32 1, i32 11, i32 2, i32 13, i32 3, i32 15>
   ret <8 x i32> %shuffle
 }
@@ -1596,19 +1607,12 @@ define <8 x i32> @shuffle_v8i32_09ab1def(<8 x i32> %a, <8 x i32> %b) {
 ; AVX2-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0],ymm1[1,2,3],ymm0[4],ymm1[5,6,7]
 ; AVX2-NEXT:    retq
 ;
-; AVX512VL-SLOW-LABEL: shuffle_v8i32_09ab1def:
-; AVX512VL-SLOW:       # %bb.0:
-; AVX512VL-SLOW-NEXT:    vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
-; AVX512VL-SLOW-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,1,1,3]
-; AVX512VL-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0],ymm1[1,2,3],ymm0[4],ymm1[5,6,7]
-; AVX512VL-SLOW-NEXT:    retq
-;
-; AVX512VL-FAST-LABEL: shuffle_v8i32_09ab1def:
-; AVX512VL-FAST:       # %bb.0:
-; AVX512VL-FAST-NEXT:    vpmovzxdq {{.*#+}} xmm2 = xmm0[0],zero,xmm0[1],zero
-; AVX512VL-FAST-NEXT:    vmovdqa {{.*#+}} ymm0 = [8,1,2,3,10,5,6,7]
-; AVX512VL-FAST-NEXT:    vpermi2d %ymm2, %ymm1, %ymm0
-; AVX512VL-FAST-NEXT:    retq
+; AVX512VL-LABEL: shuffle_v8i32_09ab1def:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm2 = [8,1,2,3,9,5,6,7]
+; AVX512VL-NEXT:    vpermi2d %ymm0, %ymm1, %ymm2
+; AVX512VL-NEXT:    vmovdqa %ymm2, %ymm0
+; AVX512VL-NEXT:    retq
   %shuffle = shufflevector <8 x i32> %a, <8 x i32> %b, <8 x i32> <i32 0, i32 9, i32 10, i32 11, i32 1, i32 13, i32 14, i32 15>
   ret <8 x i32> %shuffle
 }
@@ -2360,6 +2364,36 @@ define <8 x i32> @shuffle_v8i32_5555uuuu(<8 x i32> %a, <8 x i32> %b) {
   ret <8 x i32> %shuffle
 }
 
+define <8 x i32> @shuffle_v8i32_0dcd3f14(<8 x i32> %a, <8 x i32> %b) {
+; AVX1-LABEL: shuffle_v8i32_0dcd3f14:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; AVX1-NEXT:    vblendps {{.*#+}} xmm2 = xmm2[0],xmm0[1,2,3]
+; AVX1-NEXT:    vpermilps {{.*#+}} xmm2 = xmm2[3,1,1,0]
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; AVX1-NEXT:    vperm2f128 {{.*#+}} ymm1 = ymm1[2,3,2,3]
+; AVX1-NEXT:    vpermilpd {{.*#+}} ymm1 = ymm1[0,0,3,2]
+; AVX1-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm1[1,2,3],ymm0[4],ymm1[5],ymm0[6,7]
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: shuffle_v8i32_0dcd3f14:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vmovaps {{.*#+}} ymm2 = <0,u,u,u,3,u,1,4>
+; AVX2-NEXT:    vpermps %ymm0, %ymm2, %ymm0
+; AVX2-NEXT:    vpermpd {{.*#+}} ymm1 = ymm1[2,2,3,3]
+; AVX2-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm1[1,2,3],ymm0[4],ymm1[5],ymm0[6,7]
+; AVX2-NEXT:    retq
+;
+; AVX512VL-LABEL: shuffle_v8i32_0dcd3f14:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm2 = [8,5,4,5,11,7,9,12]
+; AVX512VL-NEXT:    vpermi2d %ymm0, %ymm1, %ymm2
+; AVX512VL-NEXT:    vmovdqa %ymm2, %ymm0
+; AVX512VL-NEXT:    retq
+  %shuffle = shufflevector <8 x i32> %a, <8 x i32> %b, <8 x i32> <i32 0, i32 13, i32 12, i32 13, i32 3, i32 15, i32 1, i32 4>
+  ret <8 x i32> %shuffle
+}
+
 ; PR32453
 define <8 x i32> @shuffle_v8i32_uuuuuu7u(<8 x i32> %a, <8 x i32> %b) nounwind {
 ; AVX1-LABEL: shuffle_v8i32_uuuuuu7u:
@@ -2877,3 +2911,131 @@ entry:
   %tmp6 = insertelement <8 x float> %tmp5, float %z, i32 5
   ret <8 x float> %tmp6
 }
+
+; PR40434: https://bugs.llvm.org/show_bug.cgi?id=40434
+
+define <8 x i32> @unpckh_v8i32(<8 x i32> %x, <8 x i32> %y) {
+; ALL-LABEL: unpckh_v8i32:
+; ALL:       # %bb.0:
+; ALL-NEXT:    vextractf128 $1, %ymm1, %xmm1
+; ALL-NEXT:    vunpckhps {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; ALL-NEXT:    retq
+  %unpckh = shufflevector <8 x i32> %x, <8 x i32> %y, <8 x i32> <i32 2, i32 14, i32 3, i32 15, i32 undef, i32 undef, i32 undef, i32 undef>
+  ret <8 x i32> %unpckh
+}
+
+; Same as above but with floats.
+
+define <8 x float> @unpckh_v8f32(<8 x float> %x, <8 x float> %y) {
+; ALL-LABEL: unpckh_v8f32:
+; ALL:       # %bb.0:
+; ALL-NEXT:    vextractf128 $1, %ymm1, %xmm1
+; ALL-NEXT:    vunpckhps {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; ALL-NEXT:    retq
+  %unpckh = shufflevector <8 x float> %x, <8 x float> %y, <8 x i32> <i32 2, i32 14, i32 3, i32 15, i32 undef, i32 undef, i32 undef, i32 undef>
+  ret <8 x float> %unpckh
+}
+
+; Alternate form of the above - make sure we don't have conflicting transforms.
+
+define <8 x i32> @blend_perm_v8i32(<8 x i32> %x, <8 x i32> %y) {
+; ALL-LABEL: blend_perm_v8i32:
+; ALL:       # %bb.0:
+; ALL-NEXT:    vextractf128 $1, %ymm1, %xmm1
+; ALL-NEXT:    vunpckhps {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; ALL-NEXT:    retq
+  %unpckh = shufflevector <8 x i32> %x, <8 x i32> %y, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 14, i32 15>
+  %r = shufflevector <8 x i32> %unpckh, <8 x i32> undef, <8 x i32> <i32 2, i32 6, i32 3, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
+  ret <8 x i32> %r
+}
+
+; Same as above but with floats.
+
+define <8 x float> @blend_perm_v8f32(<8 x float> %x, <8 x float> %y) {
+; ALL-LABEL: blend_perm_v8f32:
+; ALL:       # %bb.0:
+; ALL-NEXT:    vextractf128 $1, %ymm1, %xmm1
+; ALL-NEXT:    vunpckhps {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; ALL-NEXT:    retq
+  %unpckh = shufflevector <8 x float> %x, <8 x float> %y, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 14, i32 15>
+  %r = shufflevector <8 x float> %unpckh, <8 x float> undef, <8 x i32> <i32 2, i32 6, i32 3, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
+  ret <8 x float> %r
+}
+
+; Another variation of the above - make sure we don't have conflicting transforms.
+
+define <8 x i32> @unpckh_v8i32_unary(<8 x i32> %x) {
+; ALL-LABEL: unpckh_v8i32_unary:
+; ALL:       # %bb.0:
+; ALL-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; ALL-NEXT:    vunpckhps {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; ALL-NEXT:    retq
+  %r = shufflevector <8 x i32> %x, <8 x i32> undef, <8 x i32> <i32 2, i32 6, i32 3, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
+  ret <8 x i32> %r
+}
+
+; Same as above but with floats.
+
+define <8 x float> @unpckh_v8f32_unary(<8 x float> %x) {
+; ALL-LABEL: unpckh_v8f32_unary:
+; ALL:       # %bb.0:
+; ALL-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; ALL-NEXT:    vunpckhps {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; ALL-NEXT:    retq
+  %r = shufflevector <8 x float> %x, <8 x float> undef, <8 x i32> <i32 2, i32 6, i32 3, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
+  ret <8 x float> %r
+}
+
+; FIXME: Why are integer and FP (below) lowering different for AVX1?
+
+define <8 x i32> @lowhalf_v8i32(<8 x i32> %x, <8 x i32> %y) {
+; AVX1-LABEL: lowhalf_v8i32:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm1
+; AVX1-NEXT:    vpermilps {{.*#+}} xmm1 = xmm1[0,2,2,2]
+; AVX1-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[2,2,3,3]
+; AVX1-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2],xmm1[3]
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: lowhalf_v8i32:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
+; AVX2-NEXT:    vmovaps {{.*#+}} ymm1 = <2,6,3,6,u,u,u,u>
+; AVX2-NEXT:    vpermps %ymm0, %ymm1, %ymm0
+; AVX2-NEXT:    retq
+;
+; AVX512VL-LABEL: lowhalf_v8i32:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm2 = <2,14,3,14,u,u,u,u>
+; AVX512VL-NEXT:    vpermt2d %ymm1, %ymm2, %ymm0
+; AVX512VL-NEXT:    retq
+  %r = shufflevector <8 x i32> %x, <8 x i32> %y, <8 x i32> <i32 2, i32 14, i32 3, i32 14, i32 undef, i32 undef, i32 undef, i32 undef>
+  ret <8 x i32> %r
+}
+
+; FIXME: AVX1 lowering is better than AVX2 (and AVX512?)
+
+define <8 x float> @lowhalf_v8f32(<8 x float> %x, <8 x float> %y) {
+; AVX1-LABEL: lowhalf_v8f32:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm1
+; AVX1-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[2,3],xmm1[2,2]
+; AVX1-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,2,1,3]
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: lowhalf_v8f32:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
+; AVX2-NEXT:    vmovaps {{.*#+}} ymm1 = <2,6,3,6,u,u,u,u>
+; AVX2-NEXT:    vpermps %ymm0, %ymm1, %ymm0
+; AVX2-NEXT:    retq
+;
+; AVX512VL-LABEL: lowhalf_v8f32:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vmovaps {{.*#+}} ymm2 = <2,14,3,14,u,u,u,u>
+; AVX512VL-NEXT:    vpermt2ps %ymm1, %ymm2, %ymm0
+; AVX512VL-NEXT:    retq
+  %r = shufflevector <8 x float> %x, <8 x float> %y, <8 x i32> <i32 2, i32 14, i32 3, i32 14, i32 undef, i32 undef, i32 undef, i32 undef>
+  ret <8 x float> %r
+}
+
diff --git a/test/CodeGen/X86/vector-shuffle-512-v16.ll b/test/CodeGen/X86/vector-shuffle-512-v16.ll
index b7ea04d23df96502840550457f88b475cc3a9060..18aa9322912cf8f3c8e26bf3504c29df4bc7c937 100644
--- a/test/CodeGen/X86/vector-shuffle-512-v16.ll
+++ b/test/CodeGen/X86/vector-shuffle-512-v16.ll
@@ -311,12 +311,12 @@ define <4 x i32> @test_v16i32_0_1_2_12 (<16 x i32> %v) {
 define <4 x i32> @test_v16i32_0_4_8_12(<16 x i32> %v) {
 ; ALL-LABEL: test_v16i32_0_4_8_12:
 ; ALL:       # %bb.0:
-; ALL-NEXT:    vextractf64x4 $1, %zmm0, %ymm1
+; ALL-NEXT:    vextractf128 $1, %ymm0, %xmm1
+; ALL-NEXT:    vunpcklps {{.*#+}} xmm1 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; ALL-NEXT:    vextractf64x4 $1, %zmm0, %ymm0
 ; ALL-NEXT:    vmovaps {{.*#+}} ymm2 = <u,u,0,4,u,u,u,u>
-; ALL-NEXT:    vpermps %ymm1, %ymm2, %ymm1
-; ALL-NEXT:    vmovaps {{.*#+}} ymm2 = <0,4,u,u,u,u,u,u>
 ; ALL-NEXT:    vpermps %ymm0, %ymm2, %ymm0
-; ALL-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3]
+; ALL-NEXT:    vblendps {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
 ; ALL-NEXT:    vzeroupper
 ; ALL-NEXT:    retq
   %res = shufflevector <16 x i32> %v, <16 x i32> undef, <4 x i32> <i32 0, i32 4, i32 8, i32 12>
@@ -338,7 +338,7 @@ define <8 x float> @shuffle_v16f32_extract_256(float* %RET, float* %a) {
 define <8 x float> @test_v16f32_0_1_2_3_4_6_7_10 (<16 x float> %v) {
 ; ALL-LABEL: test_v16f32_0_1_2_3_4_6_7_10:
 ; ALL:       # %bb.0:
-; ALL-NEXT:    vextractf64x4 $1, %zmm0, %ymm1
+; ALL-NEXT:    vextractf32x4 $2, %zmm0, %xmm1
 ; ALL-NEXT:    vmovsldup {{.*#+}} xmm1 = xmm1[0,0,2,2]
 ; ALL-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm1
 ; ALL-NEXT:    vpermilps {{.*#+}} ymm0 = ymm0[0,1,2,3,4,6,7,u]
diff --git a/test/CodeGen/X86/vector-shuffle-512-v64.ll b/test/CodeGen/X86/vector-shuffle-512-v64.ll
index 741861bc6c6c53958c3515349731585dc50fd2a3..cadb725ea7f3b7d97ad95f02bf3a950c1fc7adcd 100644
--- a/test/CodeGen/X86/vector-shuffle-512-v64.ll
+++ b/test/CodeGen/X86/vector-shuffle-512-v64.ll
@@ -444,13 +444,8 @@ define <64 x i8> @shuffle_v64i8_63_zz_61_zz_59_zz_57_zz_55_zz_53_zz_51_zz_49_zz_
 ;
 ; AVX512BW-LABEL: shuffle_v64i8_63_zz_61_zz_59_zz_57_zz_55_zz_53_zz_51_zz_49_zz_47_zz_45_zz_43_zz_41_zz_39_zz_37_zz_35_zz_33_zz_31_zz_29_zz_27_zz_25_zz_23_zz_21_zz_19_zz_17_zz_15_zz_13_zz_11_zz_9_zz_7_zz_5_zz_3_zz_1_zz:
 ; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vpermq {{.*#+}} ymm1 = ymm0[2,3,0,1]
-; AVX512BW-NEXT:    vmovdqa {{.*#+}} ymm2 = [15,128,13,128,11,128,9,128,7,128,5,128,3,128,1,128,15,128,13,128,11,128,9,128,7,128,5,128,3,128,1,128]
-; AVX512BW-NEXT:    vpshufb %ymm2, %ymm1, %ymm1
-; AVX512BW-NEXT:    vextracti64x4 $1, %zmm0, %ymm0
-; AVX512BW-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,3,0,1]
-; AVX512BW-NEXT:    vpshufb %ymm2, %ymm0, %ymm0
-; AVX512BW-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm0[6,7,4,5,2,3,0,1]
+; AVX512BW-NEXT:    vpshufb {{.*#+}} zmm0 = zmm0[15],zero,zmm0[13],zero,zmm0[11],zero,zmm0[9],zero,zmm0[7],zero,zmm0[5],zero,zmm0[3],zero,zmm0[1],zero,zmm0[31],zero,zmm0[29],zero,zmm0[27],zero,zmm0[25],zero,zmm0[23],zero,zmm0[21],zero,zmm0[19],zero,zmm0[17],zero,zmm0[47],zero,zmm0[45],zero,zmm0[43],zero,zmm0[41],zero,zmm0[39],zero,zmm0[37],zero,zmm0[35],zero,zmm0[33],zero,zmm0[63],zero,zmm0[61],zero,zmm0[59],zero,zmm0[57],zero,zmm0[55],zero,zmm0[53],zero,zmm0[51],zero,zmm0[49],zero
 ; AVX512BW-NEXT:    retq
 ;
 ; AVX512DQ-LABEL: shuffle_v64i8_63_zz_61_zz_59_zz_57_zz_55_zz_53_zz_51_zz_49_zz_47_zz_45_zz_43_zz_41_zz_39_zz_37_zz_35_zz_33_zz_31_zz_29_zz_27_zz_25_zz_23_zz_21_zz_19_zz_17_zz_15_zz_13_zz_11_zz_9_zz_7_zz_5_zz_3_zz_1_zz:
@@ -490,13 +485,12 @@ define <64 x i8> @shuffle_v64i8_63_64_61_66_59_68_57_70_55_72_53_74_51_76_49_78_
 ; AVX512BW-LABEL: shuffle_v64i8_63_64_61_66_59_68_57_70_55_72_53_74_51_76_49_78_47_80_45_82_43_84_41_86_39_88_37_90_35_92_33_94_31_96_29_98_27_100_25_102_23_104_21_106_19_108_17_110_15_112_13_114_11_116_9_118_7_120_5_122_3_124_1_126:
 ; AVX512BW:       # %bb.0:
 ; AVX512BW-NEXT:    vextracti64x4 $1, %zmm1, %ymm2
-; AVX512BW-NEXT:    vpermq {{.*#+}} ymm3 = ymm0[2,3,0,1]
+; AVX512BW-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm0[6,7,4,5,2,3,0,1]
+; AVX512BW-NEXT:    vextracti64x4 $1, %zmm0, %ymm3
 ; AVX512BW-NEXT:    vpbroadcastw {{.*#+}} ymm4 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
 ; AVX512BW-NEXT:    vpblendvb %ymm4, %ymm2, %ymm3, %ymm2
 ; AVX512BW-NEXT:    vmovdqa {{.*#+}} ymm3 = [15,0,13,2,11,4,9,6,7,8,5,10,3,12,1,14,15,0,13,2,11,4,9,6,7,8,5,10,3,12,1,14]
 ; AVX512BW-NEXT:    vpshufb %ymm3, %ymm2, %ymm2
-; AVX512BW-NEXT:    vextracti64x4 $1, %zmm0, %ymm0
-; AVX512BW-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[2,3,0,1]
 ; AVX512BW-NEXT:    vpblendvb %ymm4, %ymm1, %ymm0, %ymm0
 ; AVX512BW-NEXT:    vpshufb %ymm3, %ymm0, %ymm0
 ; AVX512BW-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
@@ -541,13 +535,9 @@ define <64 x i8> @shuffle_v64i8_shift_00_02_04_06_08_10_12_14_16_18_20_22_24_26_
 ; AVX512BW:       # %bb.0:
 ; AVX512BW-NEXT:    vpsrlw $8, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vpsrlw $8, %zmm1, %zmm1
-; AVX512BW-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
-; AVX512BW-NEXT:    vpackuswb %ymm2, %ymm0, %ymm0
-; AVX512BW-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
-; AVX512BW-NEXT:    vextracti64x4 $1, %zmm1, %ymm2
-; AVX512BW-NEXT:    vpackuswb %ymm2, %ymm1, %ymm1
-; AVX512BW-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,2,1,3]
-; AVX512BW-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vshufi64x2 {{.*#+}} zmm2 = zmm0[2,3,6,7],zmm1[2,3,6,7]
+; AVX512BW-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,4,5],zmm1[0,1,4,5]
+; AVX512BW-NEXT:    vpackuswb %zmm2, %zmm0, %zmm0
 ; AVX512BW-NEXT:    retq
 ;
 ; AVX512DQ-LABEL: shuffle_v64i8_shift_00_02_04_06_08_10_12_14_16_18_20_22_24_26_28_30_32_34_36_38_40_42_44_46_48_50_52_54_56_58_60_62_64_66_68_70_72_74_76_78_80_82_84_86_88_90_92_94_96_98_100_102_104_106_108_110_112_114_116_118_120_122_124_126:
diff --git a/test/CodeGen/X86/vector-shuffle-512-v8.ll b/test/CodeGen/X86/vector-shuffle-512-v8.ll
index dce7f02605207a63f54fb2d48268802ce50ac3c1..e628f8814a07d16b4d4d5feb7078a464fe6fb261 100644
--- a/test/CodeGen/X86/vector-shuffle-512-v8.ll
+++ b/test/CodeGen/X86/vector-shuffle-512-v8.ll
@@ -2290,10 +2290,9 @@ define <4 x i64> @test_v8i64_1257 (<8 x i64> %v) {
 define <2 x i64> @test_v8i64_2_5 (<8 x i64> %v) {
 ; ALL-LABEL: test_v8i64_2_5:
 ; ALL:       # %bb.0:
-; ALL-NEXT:    vextractf64x4 $1, %zmm0, %ymm1
-; ALL-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5,6,7]
-; ALL-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[2,1,2,3]
-; ALL-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
+; ALL-NEXT:    vextractf32x4 $2, %zmm0, %xmm1
+; ALL-NEXT:    vextractf128 $1, %ymm0, %xmm0
+; ALL-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3]
 ; ALL-NEXT:    vzeroupper
 ; ALL-NEXT:    ret{{[l|q]}}
   %res = shufflevector <8 x i64> %v, <8 x i64> undef, <2 x i32> <i32 2, i32 5>
diff --git a/test/CodeGen/X86/vector-shuffle-avx512.ll b/test/CodeGen/X86/vector-shuffle-avx512.ll
index e00e376549dbca1a179e559f6f93aba3489329f9..2092b3bf4530aa4f2b6a812ae9a4d155bf956484 100644
--- a/test/CodeGen/X86/vector-shuffle-avx512.ll
+++ b/test/CodeGen/X86/vector-shuffle-avx512.ll
@@ -190,9 +190,10 @@ define <4 x i64> @expand4(<2 x i64> %a ) {
 define <8 x float> @expand5(<4 x float> %a ) {
 ; SKX64-LABEL: expand5:
 ; SKX64:       # %bb.0:
-; SKX64-NEXT:    vbroadcastss %xmm0, %ymm0
+; SKX64-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
 ; SKX64-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; SKX64-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0],ymm0[1],ymm1[2],ymm0[3],ymm1[4],ymm0[5],ymm1[6],ymm0[7]
+; SKX64-NEXT:    vmovaps {{.*#+}} ymm2 = [8,0,10,0,12,0,14,0]
+; SKX64-NEXT:    vpermt2ps %ymm1, %ymm2, %ymm0
 ; SKX64-NEXT:    retq
 ;
 ; KNL64-LABEL: expand5:
@@ -204,9 +205,10 @@ define <8 x float> @expand5(<4 x float> %a ) {
 ;
 ; SKX32-LABEL: expand5:
 ; SKX32:       # %bb.0:
-; SKX32-NEXT:    vbroadcastss %xmm0, %ymm0
+; SKX32-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
 ; SKX32-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; SKX32-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0],ymm0[1],ymm1[2],ymm0[3],ymm1[4],ymm0[5],ymm1[6],ymm0[7]
+; SKX32-NEXT:    vmovaps {{.*#+}} ymm2 = [8,0,10,0,12,0,14,0]
+; SKX32-NEXT:    vpermt2ps %ymm1, %ymm2, %ymm0
 ; SKX32-NEXT:    retl
 ;
 ; KNL32-LABEL: expand5:
@@ -541,8 +543,9 @@ define <8 x float> @expand14(<4 x float> %a) {
 define <8 x float> @expand15(<4 x float> %a) {
 ; SKX64-LABEL: expand15:
 ; SKX64:       # %bb.0:
-; SKX64-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,1,1,3]
-; SKX64-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[0,0,1,3]
+; SKX64-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
+; SKX64-NEXT:    vmovaps {{.*#+}} ymm1 = <0,1,0,1,1,3,u,u>
+; SKX64-NEXT:    vpermps %ymm0, %ymm1, %ymm0
 ; SKX64-NEXT:    vblendps {{.*#+}} ymm0 = mem[0,1],ymm0[2],mem[3],ymm0[4],mem[5,6,7]
 ; SKX64-NEXT:    retq
 ;
@@ -555,8 +558,9 @@ define <8 x float> @expand15(<4 x float> %a) {
 ;
 ; SKX32-LABEL: expand15:
 ; SKX32:       # %bb.0:
-; SKX32-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,1,1,3]
-; SKX32-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[0,0,1,3]
+; SKX32-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
+; SKX32-NEXT:    vmovaps {{.*#+}} ymm1 = <0,1,0,1,1,3,u,u>
+; SKX32-NEXT:    vpermps %ymm0, %ymm1, %ymm0
 ; SKX32-NEXT:    vblendps {{.*#+}} ymm0 = mem[0,1],ymm0[2],mem[3],ymm0[4],mem[5,6,7]
 ; SKX32-NEXT:    retl
 ;
diff --git a/test/CodeGen/X86/vector-shuffle-combining-avx.ll b/test/CodeGen/X86/vector-shuffle-combining-avx.ll
index 03fe8c49b4ffc017d08ba6649abcf7bf53a3344b..5651174c4dc9d92fa53a9c2ee86458ff5769c1f7 100644
--- a/test/CodeGen/X86/vector-shuffle-combining-avx.ll
+++ b/test/CodeGen/X86/vector-shuffle-combining-avx.ll
@@ -1,10 +1,10 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=i686-unknown -mattr=+avx | FileCheck %s --check-prefix=X32 --check-prefix=X32-AVX --check-prefix=X32-AVX1
-; RUN: llc < %s -mtriple=i686-unknown -mattr=+avx2 | FileCheck %s --check-prefix=X32 --check-prefix=X32-AVX --check-prefix=X32-AVX2
-; RUN: llc < %s -mtriple=i686-unknown -mattr=+avx512f | FileCheck %s --check-prefix=X32 --check-prefix=X32-AVX512
-; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx | FileCheck %s --check-prefix=X64 --check-prefix=X64-AVX --check-prefix=X64-AVX1
-; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx2 | FileCheck %s --check-prefix=X64 --check-prefix=X64-AVX --check-prefix=X64-AVX2
-; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx512f | FileCheck %s --check-prefix=X64 --check-prefix=X64-AVX512
+; RUN: llc < %s -mtriple=i686-unknown -mattr=+avx | FileCheck %s --check-prefixes=CHECK,X86,AVX,X86-AVX,AVX1,X86-AVX1
+; RUN: llc < %s -mtriple=i686-unknown -mattr=+avx2 | FileCheck %s --check-prefixes=CHECK,X86,AVX,X86-AVX,AVX2,X86-AVX2
+; RUN: llc < %s -mtriple=i686-unknown -mattr=+avx512f | FileCheck %s --check-prefixes=CHECK,X86,AVX512,X86-AVX512
+; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx | FileCheck %s --check-prefixes=CHECK,X64,AVX,X64-AVX,AVX1,X64-AVX1
+; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx2 | FileCheck %s --check-prefixes=CHECK,X64,AVX,X64-AVX,AVX2,X64-AVX2
+; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx512f | FileCheck %s --check-prefixes=CHECK,X64,AVX512,X64-AVX512
 ;
 ; Combine tests involving AVX target shuffles
 
@@ -23,37 +23,28 @@ declare <8 x float> @llvm.x86.avx.vperm2f128.ps.256(<8 x float>, <8 x float>, i8
 declare <4 x double> @llvm.x86.avx.vperm2f128.pd.256(<4 x double>, <4 x double>, i8)
 
 define <4 x float> @combine_vpermilvar_4f32_identity(<4 x float> %a0) {
-; X32-LABEL: combine_vpermilvar_4f32_identity:
-; X32:       # %bb.0:
-; X32-NEXT:    retl
-;
-; X64-LABEL: combine_vpermilvar_4f32_identity:
-; X64:       # %bb.0:
-; X64-NEXT:    retq
+; CHECK-LABEL: combine_vpermilvar_4f32_identity:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ret{{[l|q]}}
   %1 = tail call <4 x float> @llvm.x86.avx.vpermilvar.ps(<4 x float> %a0, <4 x i32> <i32 3, i32 2, i32 1, i32 0>)
   %2 = tail call <4 x float> @llvm.x86.avx.vpermilvar.ps(<4 x float>  %1, <4 x i32> <i32 3, i32 2, i32 1, i32 0>)
   ret <4 x float> %2
 }
 
 define <4 x float> @combine_vpermilvar_4f32_movddup(<4 x float> %a0) {
-; X32-LABEL: combine_vpermilvar_4f32_movddup:
-; X32:       # %bb.0:
-; X32-NEXT:    vmovddup {{.*#+}} xmm0 = xmm0[0,0]
-; X32-NEXT:    retl
-;
-; X64-LABEL: combine_vpermilvar_4f32_movddup:
-; X64:       # %bb.0:
-; X64-NEXT:    vmovddup {{.*#+}} xmm0 = xmm0[0,0]
-; X64-NEXT:    retq
+; CHECK-LABEL: combine_vpermilvar_4f32_movddup:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovddup {{.*#+}} xmm0 = xmm0[0,0]
+; CHECK-NEXT:    ret{{[l|q]}}
   %1 = tail call <4 x float> @llvm.x86.avx.vpermilvar.ps(<4 x float> %a0, <4 x i32> <i32 0, i32 1, i32 0, i32 1>)
   ret <4 x float> %1
 }
 define <4 x float> @combine_vpermilvar_4f32_movddup_load(<4 x float> *%a0) {
-; X32-LABEL: combine_vpermilvar_4f32_movddup_load:
-; X32:       # %bb.0:
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    vmovddup {{.*#+}} xmm0 = mem[0,0]
-; X32-NEXT:    retl
+; X86-LABEL: combine_vpermilvar_4f32_movddup_load:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    vmovddup {{.*#+}} xmm0 = mem[0,0]
+; X86-NEXT:    retl
 ;
 ; X64-LABEL: combine_vpermilvar_4f32_movddup_load:
 ; X64:       # %bb.0:
@@ -65,119 +56,75 @@ define <4 x float> @combine_vpermilvar_4f32_movddup_load(<4 x float> *%a0) {
 }
 
 define <4 x float> @combine_vpermilvar_4f32_movshdup(<4 x float> %a0) {
-; X32-LABEL: combine_vpermilvar_4f32_movshdup:
-; X32:       # %bb.0:
-; X32-NEXT:    vmovshdup {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; X32-NEXT:    retl
-;
-; X64-LABEL: combine_vpermilvar_4f32_movshdup:
-; X64:       # %bb.0:
-; X64-NEXT:    vmovshdup {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; X64-NEXT:    retq
+; CHECK-LABEL: combine_vpermilvar_4f32_movshdup:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovshdup {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; CHECK-NEXT:    ret{{[l|q]}}
   %1 = tail call <4 x float> @llvm.x86.avx.vpermilvar.ps(<4 x float> %a0, <4 x i32> <i32 undef, i32 1, i32 3, i32 3>)
   ret <4 x float> %1
 }
 
 define <4 x float> @combine_vpermilvar_4f32_movsldup(<4 x float> %a0) {
-; X32-LABEL: combine_vpermilvar_4f32_movsldup:
-; X32:       # %bb.0:
-; X32-NEXT:    vmovsldup {{.*#+}} xmm0 = xmm0[0,0,2,2]
-; X32-NEXT:    retl
-;
-; X64-LABEL: combine_vpermilvar_4f32_movsldup:
-; X64:       # %bb.0:
-; X64-NEXT:    vmovsldup {{.*#+}} xmm0 = xmm0[0,0,2,2]
-; X64-NEXT:    retq
+; CHECK-LABEL: combine_vpermilvar_4f32_movsldup:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovsldup {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; CHECK-NEXT:    ret{{[l|q]}}
   %1 = tail call <4 x float> @llvm.x86.avx.vpermilvar.ps(<4 x float> %a0, <4 x i32> <i32 0, i32 0, i32 2, i32 undef>)
   ret <4 x float> %1
 }
 
 define <4 x float> @combine_vpermilvar_4f32_unpckh(<4 x float> %a0) {
-; X32-LABEL: combine_vpermilvar_4f32_unpckh:
-; X32:       # %bb.0:
-; X32-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[2,2,3,3]
-; X32-NEXT:    retl
-;
-; X64-LABEL: combine_vpermilvar_4f32_unpckh:
-; X64:       # %bb.0:
-; X64-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[2,2,3,3]
-; X64-NEXT:    retq
+; CHECK-LABEL: combine_vpermilvar_4f32_unpckh:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[2,2,3,3]
+; CHECK-NEXT:    ret{{[l|q]}}
   %1 = tail call <4 x float> @llvm.x86.avx.vpermilvar.ps(<4 x float> %a0, <4 x i32> <i32 2, i32 2, i32 3, i32 3>)
   ret <4 x float> %1
 }
 
 define <4 x float> @combine_vpermilvar_4f32_unpckl(<4 x float> %a0) {
-; X32-LABEL: combine_vpermilvar_4f32_unpckl:
-; X32:       # %bb.0:
-; X32-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,0,1,1]
-; X32-NEXT:    retl
-;
-; X64-LABEL: combine_vpermilvar_4f32_unpckl:
-; X64:       # %bb.0:
-; X64-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,0,1,1]
-; X64-NEXT:    retq
+; CHECK-LABEL: combine_vpermilvar_4f32_unpckl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,0,1,1]
+; CHECK-NEXT:    ret{{[l|q]}}
   %1 = tail call <4 x float> @llvm.x86.avx.vpermilvar.ps(<4 x float> %a0, <4 x i32> <i32 0, i32 0, i32 1, i32 1>)
   ret <4 x float> %1
 }
 
 define <8 x float> @combine_vpermilvar_8f32_identity(<8 x float> %a0) {
-; X32-LABEL: combine_vpermilvar_8f32_identity:
-; X32:       # %bb.0:
-; X32-NEXT:    retl
-;
-; X64-LABEL: combine_vpermilvar_8f32_identity:
-; X64:       # %bb.0:
-; X64-NEXT:    retq
+; CHECK-LABEL: combine_vpermilvar_8f32_identity:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ret{{[l|q]}}
   %1 = tail call <8 x float> @llvm.x86.avx.vpermilvar.ps.256(<8 x float> %a0, <8 x i32> <i32 3, i32 2, i32 1, i32 0, i32 2, i32 3, i32 0, i32 undef>)
   %2 = tail call <8 x float> @llvm.x86.avx.vpermilvar.ps.256(<8 x float>  %1, <8 x i32> <i32 3, i32 2, i32 1, i32 0, i32 2, i32 3, i32 0, i32 1>)
   ret <8 x float> %2
 }
 
 define <8 x float> @combine_vpermilvar_8f32_10326u4u(<8 x float> %a0) {
-; X32-LABEL: combine_vpermilvar_8f32_10326u4u:
-; X32:       # %bb.0:
-; X32-NEXT:    vpermilps {{.*#+}} ymm0 = ymm0[1,0,3,2,6,u,4,u]
-; X32-NEXT:    retl
-;
-; X64-LABEL: combine_vpermilvar_8f32_10326u4u:
-; X64:       # %bb.0:
-; X64-NEXT:    vpermilps {{.*#+}} ymm0 = ymm0[1,0,3,2,6,u,4,u]
-; X64-NEXT:    retq
+; CHECK-LABEL: combine_vpermilvar_8f32_10326u4u:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpermilps {{.*#+}} ymm0 = ymm0[1,0,3,2,6,u,4,u]
+; CHECK-NEXT:    ret{{[l|q]}}
   %1 = tail call <8 x float> @llvm.x86.avx.vpermilvar.ps.256(<8 x float> %a0, <8 x i32> <i32 3, i32 2, i32 1, i32 0, i32 0, i32 1, i32 2, i32 undef>)
   %2 = tail call <8 x float> @llvm.x86.avx.vpermilvar.ps.256(<8 x float>  %1, <8 x i32> <i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 undef>)
   ret <8 x float> %2
 }
 
 define <8 x float> @combine_vpermilvar_vperm2f128_8f32(<8 x float> %a0) {
-; X32-AVX1-LABEL: combine_vpermilvar_vperm2f128_8f32:
-; X32-AVX1:       # %bb.0:
-; X32-AVX1-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm0[2,3,0,1]
-; X32-AVX1-NEXT:    retl
-;
-; X32-AVX2-LABEL: combine_vpermilvar_vperm2f128_8f32:
-; X32-AVX2:       # %bb.0:
-; X32-AVX2-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[2,3,0,1]
-; X32-AVX2-NEXT:    retl
-;
-; X32-AVX512-LABEL: combine_vpermilvar_vperm2f128_8f32:
-; X32-AVX512:       # %bb.0:
-; X32-AVX512-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[2,3,0,1]
-; X32-AVX512-NEXT:    retl
-;
-; X64-AVX1-LABEL: combine_vpermilvar_vperm2f128_8f32:
-; X64-AVX1:       # %bb.0:
-; X64-AVX1-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm0[2,3,0,1]
-; X64-AVX1-NEXT:    retq
-;
-; X64-AVX2-LABEL: combine_vpermilvar_vperm2f128_8f32:
-; X64-AVX2:       # %bb.0:
-; X64-AVX2-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[2,3,0,1]
-; X64-AVX2-NEXT:    retq
-;
-; X64-AVX512-LABEL: combine_vpermilvar_vperm2f128_8f32:
-; X64-AVX512:       # %bb.0:
-; X64-AVX512-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[2,3,0,1]
-; X64-AVX512-NEXT:    retq
+; AVX1-LABEL: combine_vpermilvar_vperm2f128_8f32:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm0[2,3,0,1]
+; AVX1-NEXT:    ret{{[l|q]}}
+;
+; AVX2-LABEL: combine_vpermilvar_vperm2f128_8f32:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[2,3,0,1]
+; AVX2-NEXT:    ret{{[l|q]}}
+;
+; AVX512-LABEL: combine_vpermilvar_vperm2f128_8f32:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[2,3,0,1]
+; AVX512-NEXT:    ret{{[l|q]}}
   %1 = tail call <8 x float> @llvm.x86.avx.vpermilvar.ps.256(<8 x float> %a0, <8 x i32> <i32 3, i32 2, i32 1, i32 0, i32 3, i32 2, i32 1, i32 0>)
   %2 = shufflevector <8 x float> %1, <8 x float> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3>
   %3 = tail call <8 x float> @llvm.x86.avx.vpermilvar.ps.256(<8 x float>  %2, <8 x i32> <i32 3, i32 2, i32 1, i32 0, i32 3, i32 2, i32 1, i32 0>)
@@ -185,15 +132,10 @@ define <8 x float> @combine_vpermilvar_vperm2f128_8f32(<8 x float> %a0) {
 }
 
 define <8 x float> @combine_vpermilvar_vperm2f128_zero_8f32(<8 x float> %a0) {
-; X32-LABEL: combine_vpermilvar_vperm2f128_zero_8f32:
-; X32:       # %bb.0:
-; X32-NEXT:    vperm2f128 {{.*#+}} ymm0 = zero,zero,ymm0[0,1]
-; X32-NEXT:    retl
-;
-; X64-LABEL: combine_vpermilvar_vperm2f128_zero_8f32:
-; X64:       # %bb.0:
-; X64-NEXT:    vperm2f128 {{.*#+}} ymm0 = zero,zero,ymm0[0,1]
-; X64-NEXT:    retq
+; CHECK-LABEL: combine_vpermilvar_vperm2f128_zero_8f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vperm2f128 {{.*#+}} ymm0 = zero,zero,ymm0[0,1]
+; CHECK-NEXT:    ret{{[l|q]}}
   %1 = tail call <8 x float> @llvm.x86.avx.vpermilvar.ps.256(<8 x float> %a0, <8 x i32> <i32 3, i32 2, i32 1, i32 0, i32 3, i32 2, i32 1, i32 0>)
   %2 = shufflevector <8 x float> %1, <8 x float> zeroinitializer, <8 x i32> <i32 8, i32 8, i32 8, i32 8, i32 0, i32 1, i32 2, i32 3>
   %3 = tail call <8 x float> @llvm.x86.avx.vpermilvar.ps.256(<8 x float>  %2, <8 x i32> <i32 3, i32 2, i32 1, i32 0, i32 3, i32 2, i32 1, i32 0>)
@@ -201,19 +143,12 @@ define <8 x float> @combine_vpermilvar_vperm2f128_zero_8f32(<8 x float> %a0) {
 }
 
 define <4 x double> @combine_vperm2f128_vpermilvar_as_vpblendpd(<4 x double> %a0) {
-; X32-LABEL: combine_vperm2f128_vpermilvar_as_vpblendpd:
-; X32:       # %bb.0:
-; X32-NEXT:    vpermilpd {{.*#+}} ymm0 = ymm0[1,0,3,2]
-; X32-NEXT:    vmovapd %xmm0, %xmm0
-; X32-NEXT:    vpermilpd {{.*#+}} ymm0 = ymm0[1,0,3,2]
-; X32-NEXT:    retl
-;
-; X64-LABEL: combine_vperm2f128_vpermilvar_as_vpblendpd:
-; X64:       # %bb.0:
-; X64-NEXT:    vpermilpd {{.*#+}} ymm0 = ymm0[1,0,3,2]
-; X64-NEXT:    vmovapd %xmm0, %xmm0
-; X64-NEXT:    vpermilpd {{.*#+}} ymm0 = ymm0[1,0,3,2]
-; X64-NEXT:    retq
+; CHECK-LABEL: combine_vperm2f128_vpermilvar_as_vpblendpd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpermilpd {{.*#+}} ymm0 = ymm0[1,0,3,2]
+; CHECK-NEXT:    vmovapd %xmm0, %xmm0
+; CHECK-NEXT:    vpermilpd {{.*#+}} ymm0 = ymm0[1,0,3,2]
+; CHECK-NEXT:    ret{{[l|q]}}
   %1 = tail call <4 x double> @llvm.x86.avx.vpermilvar.pd.256(<4 x double> %a0, <4 x i64> <i64 2, i64 0, i64 2, i64 0>)
   %2 = shufflevector <4 x double> %1, <4 x double> zeroinitializer, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
   %3 = tail call <4 x double> @llvm.x86.avx.vpermilvar.pd.256(<4 x double> %2, <4 x i64> <i64 2, i64 0, i64 2, i64 0>)
@@ -221,24 +156,19 @@ define <4 x double> @combine_vperm2f128_vpermilvar_as_vpblendpd(<4 x double> %a0
 }
 
 define <8 x float> @combine_vpermilvar_8f32_movddup(<8 x float> %a0) {
-; X32-LABEL: combine_vpermilvar_8f32_movddup:
-; X32:       # %bb.0:
-; X32-NEXT:    vmovddup {{.*#+}} ymm0 = ymm0[0,0,2,2]
-; X32-NEXT:    retl
-;
-; X64-LABEL: combine_vpermilvar_8f32_movddup:
-; X64:       # %bb.0:
-; X64-NEXT:    vmovddup {{.*#+}} ymm0 = ymm0[0,0,2,2]
-; X64-NEXT:    retq
+; CHECK-LABEL: combine_vpermilvar_8f32_movddup:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovddup {{.*#+}} ymm0 = ymm0[0,0,2,2]
+; CHECK-NEXT:    ret{{[l|q]}}
   %1 = tail call <8 x float> @llvm.x86.avx.vpermilvar.ps.256(<8 x float> %a0, <8 x i32> <i32 0, i32 1, i32 0, i32 1, i32 4, i32 5, i32 4, i32 5>)
   ret <8 x float> %1
 }
 define <8 x float> @combine_vpermilvar_8f32_movddup_load(<8 x float> *%a0) {
-; X32-LABEL: combine_vpermilvar_8f32_movddup_load:
-; X32:       # %bb.0:
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    vmovddup {{.*#+}} ymm0 = mem[0,0,2,2]
-; X32-NEXT:    retl
+; X86-LABEL: combine_vpermilvar_8f32_movddup_load:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    vmovddup {{.*#+}} ymm0 = mem[0,0,2,2]
+; X86-NEXT:    retl
 ;
 ; X64-LABEL: combine_vpermilvar_8f32_movddup_load:
 ; X64:       # %bb.0:
@@ -250,97 +180,64 @@ define <8 x float> @combine_vpermilvar_8f32_movddup_load(<8 x float> *%a0) {
 }
 
 define <8 x float> @combine_vpermilvar_8f32_movshdup(<8 x float> %a0) {
-; X32-LABEL: combine_vpermilvar_8f32_movshdup:
-; X32:       # %bb.0:
-; X32-NEXT:    vmovshdup {{.*#+}} ymm0 = ymm0[1,1,3,3,5,5,7,7]
-; X32-NEXT:    retl
-;
-; X64-LABEL: combine_vpermilvar_8f32_movshdup:
-; X64:       # %bb.0:
-; X64-NEXT:    vmovshdup {{.*#+}} ymm0 = ymm0[1,1,3,3,5,5,7,7]
-; X64-NEXT:    retq
+; CHECK-LABEL: combine_vpermilvar_8f32_movshdup:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovshdup {{.*#+}} ymm0 = ymm0[1,1,3,3,5,5,7,7]
+; CHECK-NEXT:    ret{{[l|q]}}
   %1 = tail call <8 x float> @llvm.x86.avx.vpermilvar.ps.256(<8 x float> %a0, <8 x i32> <i32 1, i32 1, i32 3, i32 3, i32 undef, i32 5, i32 7, i32 7>)
   ret <8 x float> %1
 }
 
 define <8 x float> @combine_vpermilvar_8f32_movsldup(<8 x float> %a0) {
-; X32-LABEL: combine_vpermilvar_8f32_movsldup:
-; X32:       # %bb.0:
-; X32-NEXT:    vmovsldup {{.*#+}} ymm0 = ymm0[0,0,2,2,4,4,6,6]
-; X32-NEXT:    retl
-;
-; X64-LABEL: combine_vpermilvar_8f32_movsldup:
-; X64:       # %bb.0:
-; X64-NEXT:    vmovsldup {{.*#+}} ymm0 = ymm0[0,0,2,2,4,4,6,6]
-; X64-NEXT:    retq
+; CHECK-LABEL: combine_vpermilvar_8f32_movsldup:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovsldup {{.*#+}} ymm0 = ymm0[0,0,2,2,4,4,6,6]
+; CHECK-NEXT:    ret{{[l|q]}}
   %1 = tail call <8 x float> @llvm.x86.avx.vpermilvar.ps.256(<8 x float> %a0, <8 x i32> <i32 0, i32 0, i32 2, i32 2, i32 4, i32 4, i32 6, i32 6>)
   ret <8 x float> %1
 }
 
 define <2 x double> @combine_vpermilvar_2f64_identity(<2 x double> %a0) {
-; X32-LABEL: combine_vpermilvar_2f64_identity:
-; X32:       # %bb.0:
-; X32-NEXT:    retl
-;
-; X64-LABEL: combine_vpermilvar_2f64_identity:
-; X64:       # %bb.0:
-; X64-NEXT:    retq
+; CHECK-LABEL: combine_vpermilvar_2f64_identity:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ret{{[l|q]}}
   %1 = tail call <2 x double> @llvm.x86.avx.vpermilvar.pd(<2 x double> %a0, <2 x i64> <i64 2, i64 0>)
   %2 = tail call <2 x double> @llvm.x86.avx.vpermilvar.pd(<2 x double>  %1, <2 x i64> <i64 2, i64 0>)
   ret <2 x double> %2
 }
 
 define <2 x double> @combine_vpermilvar_2f64_movddup(<2 x double> %a0) {
-; X32-LABEL: combine_vpermilvar_2f64_movddup:
-; X32:       # %bb.0:
-; X32-NEXT:    vmovddup {{.*#+}} xmm0 = xmm0[0,0]
-; X32-NEXT:    retl
-;
-; X64-LABEL: combine_vpermilvar_2f64_movddup:
-; X64:       # %bb.0:
-; X64-NEXT:    vmovddup {{.*#+}} xmm0 = xmm0[0,0]
-; X64-NEXT:    retq
+; CHECK-LABEL: combine_vpermilvar_2f64_movddup:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovddup {{.*#+}} xmm0 = xmm0[0,0]
+; CHECK-NEXT:    ret{{[l|q]}}
   %1 = tail call <2 x double> @llvm.x86.avx.vpermilvar.pd(<2 x double> %a0, <2 x i64> <i64 0, i64 0>)
   ret <2 x double> %1
 }
 
 define <4 x double> @combine_vpermilvar_4f64_identity(<4 x double> %a0) {
-; X32-LABEL: combine_vpermilvar_4f64_identity:
-; X32:       # %bb.0:
-; X32-NEXT:    retl
-;
-; X64-LABEL: combine_vpermilvar_4f64_identity:
-; X64:       # %bb.0:
-; X64-NEXT:    retq
+; CHECK-LABEL: combine_vpermilvar_4f64_identity:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ret{{[l|q]}}
   %1 = tail call <4 x double> @llvm.x86.avx.vpermilvar.pd.256(<4 x double> %a0, <4 x i64> <i64 2, i64 0, i64 2, i64 0>)
   %2 = tail call <4 x double> @llvm.x86.avx.vpermilvar.pd.256(<4 x double>  %1, <4 x i64> <i64 2, i64 0, i64 2, i64 0>)
   ret <4 x double> %2
 }
 
 define <4 x double> @combine_vpermilvar_4f64_movddup(<4 x double> %a0) {
-; X32-LABEL: combine_vpermilvar_4f64_movddup:
-; X32:       # %bb.0:
-; X32-NEXT:    vmovddup {{.*#+}} ymm0 = ymm0[0,0,2,2]
-; X32-NEXT:    retl
-;
-; X64-LABEL: combine_vpermilvar_4f64_movddup:
-; X64:       # %bb.0:
-; X64-NEXT:    vmovddup {{.*#+}} ymm0 = ymm0[0,0,2,2]
-; X64-NEXT:    retq
+; CHECK-LABEL: combine_vpermilvar_4f64_movddup:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovddup {{.*#+}} ymm0 = ymm0[0,0,2,2]
+; CHECK-NEXT:    ret{{[l|q]}}
   %1 = tail call <4 x double> @llvm.x86.avx.vpermilvar.pd.256(<4 x double> %a0, <4 x i64> <i64 0, i64 0, i64 4, i64 4>)
   ret <4 x double> %1
 }
 
 define <4 x float> @combine_vpermilvar_4f32_4stage(<4 x float> %a0) {
-; X32-LABEL: combine_vpermilvar_4f32_4stage:
-; X32:       # %bb.0:
-; X32-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[2,0,3,1]
-; X32-NEXT:    retl
-;
-; X64-LABEL: combine_vpermilvar_4f32_4stage:
-; X64:       # %bb.0:
-; X64-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[2,0,3,1]
-; X64-NEXT:    retq
+; CHECK-LABEL: combine_vpermilvar_4f32_4stage:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[2,0,3,1]
+; CHECK-NEXT:    ret{{[l|q]}}
   %1 = tail call <4 x float> @llvm.x86.avx.vpermilvar.ps(<4 x float> %a0, <4 x i32> <i32 3, i32 2, i32 1, i32 0>)
   %2 = tail call <4 x float> @llvm.x86.avx.vpermilvar.ps(<4 x float>  %1, <4 x i32> <i32 2, i32 3, i32 0, i32 1>)
   %3 = tail call <4 x float> @llvm.x86.avx.vpermilvar.ps(<4 x float>  %2, <4 x i32> <i32 0, i32 2, i32 1, i32 3>)
@@ -349,15 +246,10 @@ define <4 x float> @combine_vpermilvar_4f32_4stage(<4 x float> %a0) {
 }
 
 define <8 x float> @combine_vpermilvar_8f32_4stage(<8 x float> %a0) {
-; X32-LABEL: combine_vpermilvar_8f32_4stage:
-; X32:       # %bb.0:
-; X32-NEXT:    vpermilps {{.*#+}} ymm0 = ymm0[2,0,3,1,6,4,7,5]
-; X32-NEXT:    retl
-;
-; X64-LABEL: combine_vpermilvar_8f32_4stage:
-; X64:       # %bb.0:
-; X64-NEXT:    vpermilps {{.*#+}} ymm0 = ymm0[2,0,3,1,6,4,7,5]
-; X64-NEXT:    retq
+; CHECK-LABEL: combine_vpermilvar_8f32_4stage:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpermilps {{.*#+}} ymm0 = ymm0[2,0,3,1,6,4,7,5]
+; CHECK-NEXT:    ret{{[l|q]}}
   %1 = tail call <8 x float> @llvm.x86.avx.vpermilvar.ps.256(<8 x float> %a0, <8 x i32> <i32 3, i32 2, i32 1, i32 0, i32 3, i32 2, i32 1, i32 0>)
   %2 = tail call <8 x float> @llvm.x86.avx.vpermilvar.ps.256(<8 x float>  %1, <8 x i32> <i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1>)
   %3 = tail call <8 x float> @llvm.x86.avx.vpermilvar.ps.256(<8 x float>  %2, <8 x i32> <i32 0, i32 2, i32 1, i32 3, i32 0, i32 2, i32 1, i32 3>)
@@ -366,119 +258,94 @@ define <8 x float> @combine_vpermilvar_8f32_4stage(<8 x float> %a0) {
 }
 
 define <4 x float> @combine_vpermilvar_4f32_as_insertps(<4 x float> %a0) {
-; X32-LABEL: combine_vpermilvar_4f32_as_insertps:
-; X32:       # %bb.0:
-; X32-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[1],zero,xmm0[2],zero
-; X32-NEXT:    retl
-;
-; X64-LABEL: combine_vpermilvar_4f32_as_insertps:
-; X64:       # %bb.0:
-; X64-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[1],zero,xmm0[2],zero
-; X64-NEXT:    retq
+; CHECK-LABEL: combine_vpermilvar_4f32_as_insertps:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[1],zero,xmm0[2],zero
+; CHECK-NEXT:    ret{{[l|q]}}
   %1 = call <4 x float> @llvm.x86.avx.vpermilvar.ps(<4 x float> %a0, <4 x i32> <i32 3, i32 2, i32 1, i32 0>)
   %2 = shufflevector <4 x float> %1, <4 x float> zeroinitializer, <4 x i32> <i32 2, i32 4, i32 1, i32 4>
   ret <4 x float> %2
 }
 
 define <2 x double> @constant_fold_vpermilvar_pd() {
-; X32-LABEL: constant_fold_vpermilvar_pd:
-; X32:       # %bb.0:
-; X32-NEXT:    vmovaps {{.*#+}} xmm0 = [2.0E+0,1.0E+0]
-; X32-NEXT:    retl
-;
-; X64-LABEL: constant_fold_vpermilvar_pd:
-; X64:       # %bb.0:
-; X64-NEXT:    vmovaps {{.*#+}} xmm0 = [2.0E+0,1.0E+0]
-; X64-NEXT:    retq
+; CHECK-LABEL: constant_fold_vpermilvar_pd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps {{.*#+}} xmm0 = [2.0E+0,1.0E+0]
+; CHECK-NEXT:    ret{{[l|q]}}
   %1 = call <2 x double> @llvm.x86.avx.vpermilvar.pd(<2 x double> <double 1.0, double 2.0>, <2 x i64> <i64 2, i64 0>)
   ret <2 x double> %1
 }
 
 define <4 x double> @constant_fold_vpermilvar_pd_256() {
-; X32-LABEL: constant_fold_vpermilvar_pd_256:
-; X32:       # %bb.0:
-; X32-NEXT:    vmovaps {{.*#+}} ymm0 = [2.0E+0,1.0E+0,3.0E+0,4.0E+0]
-; X32-NEXT:    retl
-;
-; X64-LABEL: constant_fold_vpermilvar_pd_256:
-; X64:       # %bb.0:
-; X64-NEXT:    vmovaps {{.*#+}} ymm0 = [2.0E+0,1.0E+0,3.0E+0,4.0E+0]
-; X64-NEXT:    retq
+; CHECK-LABEL: constant_fold_vpermilvar_pd_256:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps {{.*#+}} ymm0 = [2.0E+0,1.0E+0,3.0E+0,4.0E+0]
+; CHECK-NEXT:    ret{{[l|q]}}
   %1 = call <4 x double> @llvm.x86.avx.vpermilvar.pd.256(<4 x double> <double 1.0, double 2.0, double 3.0, double 4.0>, <4 x i64> <i64 2, i64 0, i64 0, i64 2>)
   ret <4 x double> %1
 }
 
 define <4 x float> @constant_fold_vpermilvar_ps() {
-; X32-LABEL: constant_fold_vpermilvar_ps:
-; X32:       # %bb.0:
-; X32-NEXT:    vmovaps {{.*#+}} xmm0 = [4.0E+0,1.0E+0,3.0E+0,2.0E+0]
-; X32-NEXT:    retl
-;
-; X64-LABEL: constant_fold_vpermilvar_ps:
-; X64:       # %bb.0:
-; X64-NEXT:    vmovaps {{.*#+}} xmm0 = [4.0E+0,1.0E+0,3.0E+0,2.0E+0]
-; X64-NEXT:    retq
+; CHECK-LABEL: constant_fold_vpermilvar_ps:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps {{.*#+}} xmm0 = [4.0E+0,1.0E+0,3.0E+0,2.0E+0]
+; CHECK-NEXT:    ret{{[l|q]}}
   %1 = call <4 x float> @llvm.x86.avx.vpermilvar.ps(<4 x float> <float 1.0, float 2.0, float 3.0, float 4.0>, <4 x i32> <i32 3, i32 0, i32 2, i32 1>)
   ret <4 x float> %1
 }
 
 define <8 x float> @constant_fold_vpermilvar_ps_256() {
-; X32-LABEL: constant_fold_vpermilvar_ps_256:
-; X32:       # %bb.0:
-; X32-NEXT:    vmovaps {{.*#+}} ymm0 = [1.0E+0,1.0E+0,3.0E+0,2.0E+0,5.0E+0,6.0E+0,6.0E+0,6.0E+0]
-; X32-NEXT:    retl
-;
-; X64-LABEL: constant_fold_vpermilvar_ps_256:
-; X64:       # %bb.0:
-; X64-NEXT:    vmovaps {{.*#+}} ymm0 = [1.0E+0,1.0E+0,3.0E+0,2.0E+0,5.0E+0,6.0E+0,6.0E+0,6.0E+0]
-; X64-NEXT:    retq
+; CHECK-LABEL: constant_fold_vpermilvar_ps_256:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps {{.*#+}} ymm0 = [1.0E+0,1.0E+0,3.0E+0,2.0E+0,5.0E+0,6.0E+0,6.0E+0,6.0E+0]
+; CHECK-NEXT:    ret{{[l|q]}}
   %1 = call <8 x float> @llvm.x86.avx.vpermilvar.ps.256(<8 x float> <float 1.0, float 2.0, float 3.0, float 4.0, float 5.0, float 6.0, float 7.0, float 8.0>, <8 x i32> <i32 4, i32 0, i32 2, i32 1, i32 0, i32 1, i32 1, i32 1>)
   ret <8 x float> %1
 }
 
 define void @PR39483() {
-; X32-AVX1-LABEL: PR39483:
-; X32-AVX1:       # %bb.0: # %entry
-; X32-AVX1-NEXT:    vmovups 32, %ymm0
-; X32-AVX1-NEXT:    vmovups 64, %xmm1
-; X32-AVX1-NEXT:    vshufps {{.*#+}} xmm1 = xmm1[0,1],mem[0,3]
-; X32-AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm1
-; X32-AVX1-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1],mem[2],ymm0[3,4],mem[5],ymm0[6,7]
-; X32-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
-; X32-AVX1-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2,3]
-; X32-AVX1-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[2,1,0,3]
-; X32-AVX1-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm2[1,0]
-; X32-AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
-; X32-AVX1-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm1[5,6,7]
-; X32-AVX1-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; X32-AVX1-NEXT:    vmulps %ymm1, %ymm0, %ymm0
-; X32-AVX1-NEXT:    vaddps %ymm1, %ymm0, %ymm0
-; X32-AVX1-NEXT:    vmovups %ymm0, (%eax)
-;
-; X32-AVX2-LABEL: PR39483:
-; X32-AVX2:       # %bb.0: # %entry
-; X32-AVX2-NEXT:    vmovups 32, %ymm0
-; X32-AVX2-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1],mem[2],ymm0[3,4],mem[5],ymm0[6,7]
-; X32-AVX2-NEXT:    vmovaps {{.*#+}} ymm1 = <2,5,0,3,6,u,u,u>
-; X32-AVX2-NEXT:    vpermps %ymm0, %ymm1, %ymm0
-; X32-AVX2-NEXT:    vpermilps {{.*#+}} ymm1 = mem[0,1,0,3,4,5,4,7]
-; X32-AVX2-NEXT:    vpermpd {{.*#+}} ymm1 = ymm1[0,1,0,3]
-; X32-AVX2-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm1[5,6,7]
-; X32-AVX2-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; X32-AVX2-NEXT:    vmulps %ymm1, %ymm0, %ymm0
-; X32-AVX2-NEXT:    vaddps %ymm1, %ymm0, %ymm0
-; X32-AVX2-NEXT:    vmovups %ymm0, (%eax)
-;
-; X32-AVX512-LABEL: PR39483:
-; X32-AVX512:       # %bb.0: # %entry
-; X32-AVX512-NEXT:    vmovups 0, %zmm0
-; X32-AVX512-NEXT:    vmovups 64, %ymm1
-; X32-AVX512-NEXT:    vmovaps {{.*#+}} zmm2 = <2,5,8,11,14,17,20,23,u,u,u,u,u,u,u,u>
-; X32-AVX512-NEXT:    vpermi2ps %zmm1, %zmm0, %zmm2
-; X32-AVX512-NEXT:    vxorps %xmm0, %xmm0, %xmm0
-; X32-AVX512-NEXT:    vmulps %ymm0, %ymm2, %ymm1
-; X32-AVX512-NEXT:    vaddps %ymm0, %ymm1, %ymm0
-; X32-AVX512-NEXT:    vmovups %ymm0, (%eax)
+; X86-AVX1-LABEL: PR39483:
+; X86-AVX1:       # %bb.0: # %entry
+; X86-AVX1-NEXT:    vmovups 32, %ymm0
+; X86-AVX1-NEXT:    vmovups 64, %xmm1
+; X86-AVX1-NEXT:    vshufps {{.*#+}} xmm1 = xmm1[0,1],mem[0,3]
+; X86-AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm1
+; X86-AVX1-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1],mem[2],ymm0[3,4],mem[5],ymm0[6,7]
+; X86-AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
+; X86-AVX1-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2,3]
+; X86-AVX1-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[2,1,0,3]
+; X86-AVX1-NEXT:    vpermilpd {{.*#+}} xmm2 = xmm2[1,0]
+; X86-AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; X86-AVX1-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm1[5,6,7]
+; X86-AVX1-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; X86-AVX1-NEXT:    vmulps %ymm1, %ymm0, %ymm0
+; X86-AVX1-NEXT:    vaddps %ymm1, %ymm0, %ymm0
+; X86-AVX1-NEXT:    vmovups %ymm0, (%eax)
+;
+; X86-AVX2-LABEL: PR39483:
+; X86-AVX2:       # %bb.0: # %entry
+; X86-AVX2-NEXT:    vmovups 32, %ymm0
+; X86-AVX2-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1],mem[2],ymm0[3,4],mem[5],ymm0[6,7]
+; X86-AVX2-NEXT:    vmovaps {{.*#+}} ymm1 = <2,5,0,3,6,u,u,u>
+; X86-AVX2-NEXT:    vpermps %ymm0, %ymm1, %ymm0
+; X86-AVX2-NEXT:    vpermilps {{.*#+}} ymm1 = mem[0,1,0,3,4,5,4,7]
+; X86-AVX2-NEXT:    vpermpd {{.*#+}} ymm1 = ymm1[0,1,0,3]
+; X86-AVX2-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm1[5,6,7]
+; X86-AVX2-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; X86-AVX2-NEXT:    vmulps %ymm1, %ymm0, %ymm0
+; X86-AVX2-NEXT:    vaddps %ymm1, %ymm0, %ymm0
+; X86-AVX2-NEXT:    vmovups %ymm0, (%eax)
+;
+; X86-AVX512-LABEL: PR39483:
+; X86-AVX512:       # %bb.0: # %entry
+; X86-AVX512-NEXT:    vmovups 0, %zmm0
+; X86-AVX512-NEXT:    vmovups 64, %ymm1
+; X86-AVX512-NEXT:    vmovaps {{.*#+}} zmm2 = <2,5,8,11,14,17,20,23,u,u,u,u,u,u,u,u>
+; X86-AVX512-NEXT:    vpermi2ps %zmm1, %zmm0, %zmm2
+; X86-AVX512-NEXT:    vxorps %xmm0, %xmm0, %xmm0
+; X86-AVX512-NEXT:    vmulps %ymm0, %ymm2, %ymm1
+; X86-AVX512-NEXT:    vaddps %ymm0, %ymm1, %ymm0
+; X86-AVX512-NEXT:    vmovups %ymm0, (%eax)
 ;
 ; X64-AVX1-LABEL: PR39483:
 ; X64-AVX1:       # %bb.0: # %entry
diff --git a/test/CodeGen/X86/vector-shuffle-combining-avx2.ll b/test/CodeGen/X86/vector-shuffle-combining-avx2.ll
index 2ea0f1ab3e7188e1c7a01358c54c93b61d702b33..963fb98f56a4c45068c4ec739964b730a1a7ee4e 100644
--- a/test/CodeGen/X86/vector-shuffle-combining-avx2.ll
+++ b/test/CodeGen/X86/vector-shuffle-combining-avx2.ll
@@ -141,7 +141,6 @@ define <16 x i8> @combine_pshufb_as_vpbroadcastb128(<16 x i8> %a) {
 define <32 x i8> @combine_pshufb_as_vpbroadcastb256(<2 x i64> %a) {
 ; CHECK-LABEL: combine_pshufb_as_vpbroadcastb256:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
 ; CHECK-NEXT:    vpbroadcastb %xmm0, %ymm0
 ; CHECK-NEXT:    ret{{[l|q]}}
   %1 = shufflevector <2 x i64> %a, <2 x i64> undef, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
@@ -165,7 +164,6 @@ define <16 x i8> @combine_pshufb_as_vpbroadcastw128(<16 x i8> %a) {
 define <32 x i8> @combine_pshufb_as_vpbroadcastw256(<2 x i64> %a) {
 ; CHECK-LABEL: combine_pshufb_as_vpbroadcastw256:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
 ; CHECK-NEXT:    vpbroadcastw %xmm0, %ymm0
 ; CHECK-NEXT:    ret{{[l|q]}}
   %1 = shufflevector <2 x i64> %a, <2 x i64> undef, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
@@ -197,14 +195,12 @@ define <16 x i8> @combine_pshufb_as_vpbroadcastd128(<16 x i8> %a) {
 define <8 x i32> @combine_permd_as_vpbroadcastd256(<4 x i32> %a) {
 ; X86-LABEL: combine_permd_as_vpbroadcastd256:
 ; X86:       # %bb.0:
-; X86-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
 ; X86-NEXT:    vpbroadcastd %xmm0, %ymm0
 ; X86-NEXT:    vpaddd {{\.LCPI.*}}, %ymm0, %ymm0
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: combine_permd_as_vpbroadcastd256:
 ; X64:       # %bb.0:
-; X64-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
 ; X64-NEXT:    vpbroadcastd %xmm0, %ymm0
 ; X64-NEXT:    vpaddd {{.*}}(%rip), %ymm0, %ymm0
 ; X64-NEXT:    retq
@@ -217,7 +213,7 @@ define <8 x i32> @combine_permd_as_vpbroadcastd256(<4 x i32> %a) {
 define <16 x i8> @combine_pshufb_as_vpbroadcastq128(<16 x i8> %a) {
 ; CHECK-LABEL: combine_pshufb_as_vpbroadcastq128:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vpbroadcastq %xmm0, %xmm0
+; CHECK-NEXT:    vmovddup {{.*#+}} xmm0 = xmm0[0,0]
 ; CHECK-NEXT:    ret{{[l|q]}}
   %1 = call <16 x i8> @llvm.x86.ssse3.pshuf.b.128(<16 x i8> %a, <16 x i8> <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7>)
   ret <16 x i8> %1
@@ -226,14 +222,12 @@ define <16 x i8> @combine_pshufb_as_vpbroadcastq128(<16 x i8> %a) {
 define <8 x i32> @combine_permd_as_vpbroadcastq256(<4 x i32> %a) {
 ; X86-LABEL: combine_permd_as_vpbroadcastq256:
 ; X86:       # %bb.0:
-; X86-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
 ; X86-NEXT:    vpbroadcastq %xmm0, %ymm0
 ; X86-NEXT:    vpaddd {{\.LCPI.*}}, %ymm0, %ymm0
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: combine_permd_as_vpbroadcastq256:
 ; X64:       # %bb.0:
-; X64-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
 ; X64-NEXT:    vpbroadcastq %xmm0, %ymm0
 ; X64-NEXT:    vpaddd {{.*}}(%rip), %ymm0, %ymm0
 ; X64-NEXT:    retq
@@ -257,7 +251,6 @@ define <4 x float> @combine_pshufb_as_vpbroadcastss128(<4 x float> %a) {
 define <8 x float> @combine_permps_as_vpbroadcastss256(<4 x float> %a) {
 ; CHECK-LABEL: combine_permps_as_vpbroadcastss256:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
 ; CHECK-NEXT:    vbroadcastss %xmm0, %ymm0
 ; CHECK-NEXT:    ret{{[l|q]}}
   %1 = shufflevector <4 x float> %a, <4 x float> undef, <8 x i32> <i32 0, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
@@ -268,7 +261,6 @@ define <8 x float> @combine_permps_as_vpbroadcastss256(<4 x float> %a) {
 define <4 x double> @combine_permps_as_vpbroadcastsd256(<2 x double> %a) {
 ; CHECK-LABEL: combine_permps_as_vpbroadcastsd256:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
 ; CHECK-NEXT:    vbroadcastsd %xmm0, %ymm0
 ; CHECK-NEXT:    ret{{[l|q]}}
   %1 = shufflevector <2 x double> %a, <2 x double> undef, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
@@ -313,7 +305,6 @@ define <4 x float> @combine_vpbroadcast_pshufb_as_vpbroadcastss128(<4 x float> %
 define <8 x float> @combine_vpbroadcast_permd_as_vpbroadcastss256(<4 x float> %a) {
 ; CHECK-LABEL: combine_vpbroadcast_permd_as_vpbroadcastss256:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
 ; CHECK-NEXT:    vbroadcastss %xmm0, %ymm0
 ; CHECK-NEXT:    ret{{[l|q]}}
   %1 = shufflevector <4 x float> %a, <4 x float> undef, <8 x i32> zeroinitializer
@@ -324,7 +315,6 @@ define <8 x float> @combine_vpbroadcast_permd_as_vpbroadcastss256(<4 x float> %a
 define <4 x double> @combine_vpbroadcast_permd_as_vpbroadcastsd256(<2 x double> %a) {
 ; CHECK-LABEL: combine_vpbroadcast_permd_as_vpbroadcastsd256:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
 ; CHECK-NEXT:    vbroadcastsd %xmm0, %ymm0
 ; CHECK-NEXT:    ret{{[l|q]}}
   %1 = shufflevector <2 x double> %a, <2 x double> undef, <4 x i32> zeroinitializer
@@ -611,10 +601,44 @@ define <32 x i8> @shuffle_combine_packuswb_pshufb(<16 x i16> %a0, <16 x i16> %a1
 }
 declare <32 x i8> @llvm.x86.avx2.packuswb(<16 x i16>, <16 x i16>) nounwind readnone
 
+define <32 x i8> @combine_pshufb_as_packsswb(<16 x i16> %a0, <16 x i16> %a1) nounwind {
+; CHECK-LABEL: combine_pshufb_as_packsswb:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpsraw $11, %ymm0, %ymm0
+; CHECK-NEXT:    vpsraw $11, %ymm1, %ymm1
+; CHECK-NEXT:    vpacksswb %ymm1, %ymm0, %ymm0
+; CHECK-NEXT:    ret{{[l|q]}}
+  %1 = ashr <16 x i16> %a0, <i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11>
+  %2 = ashr <16 x i16> %a1, <i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11>
+  %3 = bitcast <16 x i16> %1 to <32 x i8>
+  %4 = bitcast <16 x i16> %2 to <32 x i8>
+  %5 = tail call <32 x i8> @llvm.x86.avx2.pshuf.b(<32 x i8> %3, <32 x i8> <i8 0, i8 2, i8 4, i8 6, i8 8, i8 10, i8 12, i8 14, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 0, i8 2, i8 4, i8 6, i8 8, i8 10, i8 12, i8 14, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>)
+  %6 = tail call <32 x i8> @llvm.x86.avx2.pshuf.b(<32 x i8> %4, <32 x i8> <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 0, i8 2, i8 4, i8 6, i8 8, i8 10, i8 12, i8 14, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 0, i8 2, i8 4, i8 6, i8 8, i8 10, i8 12, i8 14>)
+  %7 = or <32 x i8> %5, %6
+  ret <32 x i8> %7
+}
+
+define <32 x i8> @combine_pshufb_as_packuswb(<16 x i16> %a0, <16 x i16> %a1) nounwind {
+; CHECK-LABEL: combine_pshufb_as_packuswb:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpsrlw $11, %ymm0, %ymm0
+; CHECK-NEXT:    vpsrlw $11, %ymm1, %ymm1
+; CHECK-NEXT:    vpackuswb %ymm1, %ymm0, %ymm0
+; CHECK-NEXT:    ret{{[l|q]}}
+  %1 = lshr <16 x i16> %a0, <i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11>
+  %2 = lshr <16 x i16> %a1, <i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11>
+  %3 = bitcast <16 x i16> %1 to <32 x i8>
+  %4 = bitcast <16 x i16> %2 to <32 x i8>
+  %5 = tail call <32 x i8> @llvm.x86.avx2.pshuf.b(<32 x i8> %3, <32 x i8> <i8 0, i8 2, i8 4, i8 6, i8 8, i8 10, i8 12, i8 14, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 0, i8 2, i8 4, i8 6, i8 8, i8 10, i8 12, i8 14, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>)
+  %6 = tail call <32 x i8> @llvm.x86.avx2.pshuf.b(<32 x i8> %4, <32 x i8> <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 0, i8 2, i8 4, i8 6, i8 8, i8 10, i8 12, i8 14, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 0, i8 2, i8 4, i8 6, i8 8, i8 10, i8 12, i8 14>)
+  %7 = or <32 x i8> %5, %6
+  ret <32 x i8> %7
+}
+
 define <16 x i8> @combine_pshufb_insertion_as_broadcast_v2i64(i64 %a0) {
 ; X86-LABEL: combine_pshufb_insertion_as_broadcast_v2i64:
 ; X86:       # %bb.0:
-; X86-NEXT:    vpbroadcastq {{[0-9]+}}(%esp), %xmm0
+; X86-NEXT:    vmovddup {{.*#+}} xmm0 = mem[0,0]
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: combine_pshufb_insertion_as_broadcast_v2i64:
@@ -631,7 +655,8 @@ define <16 x i8> @combine_pshufb_insertion_as_broadcast_v2i64(i64 %a0) {
 define <8 x i32> @combine_permd_insertion_as_broadcast_v4i64(i64 %a0) {
 ; X86-LABEL: combine_permd_insertion_as_broadcast_v4i64:
 ; X86:       # %bb.0:
-; X86-NEXT:    vbroadcastsd {{[0-9]+}}(%esp), %ymm0
+; X86-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; X86-NEXT:    vbroadcastsd %xmm0, %ymm0
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: combine_permd_insertion_as_broadcast_v4i64:
@@ -735,3 +760,29 @@ entry:
   %shuf2 = shufflevector <8 x float> %inp1, <8 x float> %shuf1, <8 x i32> <i32 15, i32 10, i32 7, i32 2, i32 12, i32 undef, i32 3, i32 2>
   ret <8 x float> %shuf2
 }
+
+define void @packss_zext_v8i1() {
+; X86-LABEL: packss_zext_v8i1:
+; X86:       # %bb.0:
+; X86-NEXT:    vxorps %xmm0, %xmm0, %xmm0
+; X86-NEXT:    vmovups %ymm0, (%eax)
+; X86-NEXT:    vzeroupper
+; X86-NEXT:    retl
+;
+; X64-LABEL: packss_zext_v8i1:
+; X64:       # %bb.0:
+; X64-NEXT:    vxorps %xmm0, %xmm0, %xmm0
+; X64-NEXT:    vmovups %ymm0, (%rax)
+; X64-NEXT:    vzeroupper
+; X64-NEXT:    retq
+  %tmp0 = icmp sgt <8 x i32> undef, undef
+  %tmp1 = zext <8 x i1> %tmp0 to <8 x i32>
+  %tmp2 = shufflevector <8 x i32> %tmp1, <8 x i32> zeroinitializer, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  %tmp3 = trunc <16 x i32> %tmp2 to <16 x i16>
+  %tmp4 = add <16 x i16> zeroinitializer, %tmp3
+  %tmp6 = sext <16 x i16> %tmp4 to <16 x i32>
+  %tmp10 = shufflevector <16 x i32> %tmp6, <16 x i32> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 12, i32 13, i32 14, i32 15>
+  %tmp11 = tail call <16 x i16> @llvm.x86.avx2.packssdw(<8 x i32> undef, <8 x i32> %tmp10)
+  store <16 x i16> %tmp11, <16 x i16>* undef, align 2
+  ret void
+}
diff --git a/test/CodeGen/X86/vector-shuffle-combining-avx512bw.ll b/test/CodeGen/X86/vector-shuffle-combining-avx512bw.ll
index b8d3824f1caf56e889b85d4011c8f804bb121a8a..6a295ba8cc5314c0608d57e8cdb5111952b472a3 100644
--- a/test/CodeGen/X86/vector-shuffle-combining-avx512bw.ll
+++ b/test/CodeGen/X86/vector-shuffle-combining-avx512bw.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=i686-unknown -mattr=+avx512bw | FileCheck %s --check-prefix=X32
-; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx512bw | FileCheck %s --check-prefix=X64
+; RUN: llc < %s -mtriple=i686-unknown -mattr=+avx512bw | FileCheck %s --check-prefixes=CHECK,X86
+; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx512bw | FileCheck %s --check-prefixes=CHECK,X64
 
 declare <64 x i8> @llvm.x86.avx512.mask.pshuf.b.512(<64 x i8>, <64 x i8>, <64 x i8>, i64)
 
@@ -26,28 +26,24 @@ declare <16 x i32> @llvm.x86.avx512.mask.vpermi2var.d.512(<16 x i32>, <16 x i32>
 declare <32 x i16> @llvm.x86.avx512.mask.vpermi2var.hi.512(<32 x i16>, <32 x i16>, <32 x i16>, i32)
 
 define <8 x double> @combine_permvar_8f64_identity(<8 x double> %x0, <8 x double> %x1) {
-; X32-LABEL: combine_permvar_8f64_identity:
-; X32:       # %bb.0:
-; X32-NEXT:    retl
-;
-; X64-LABEL: combine_permvar_8f64_identity:
-; X64:       # %bb.0:
-; X64-NEXT:    retq
+; CHECK-LABEL: combine_permvar_8f64_identity:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ret{{[l|q]}}
   %1 = call <8 x double> @llvm.x86.avx512.permvar.df.512(<8 x double> %x0, <8 x i64> <i64 7, i64 6, i64 5, i64 4, i64 3, i64 2, i64 1, i64 0>)
   %2 = call <8 x double> @llvm.x86.avx512.permvar.df.512(<8 x double> %1, <8 x i64> <i64 7, i64 14, i64 5, i64 12, i64 3, i64 10, i64 1, i64 8>)
   ret <8 x double> %2
 }
 define <8 x double> @combine_permvar_8f64_identity_mask(<8 x double> %x0, <8 x double> %x1, i8 %m) {
-; X32-LABEL: combine_permvar_8f64_identity_mask:
-; X32:       # %bb.0:
-; X32-NEXT:    vmovapd {{.*#+}} zmm2 = [7,0,6,0,5,0,4,0,3,0,2,0,1,0,0,0]
-; X32-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    kmovd %eax, %k1
-; X32-NEXT:    vpermpd %zmm0, %zmm2, %zmm1 {%k1}
-; X32-NEXT:    vmovapd {{.*#+}} zmm0 = [7,0,14,0,5,0,12,0,3,0,10,0,1,0,8,0]
-; X32-NEXT:    vpermpd %zmm1, %zmm0, %zmm1 {%k1}
-; X32-NEXT:    vmovapd %zmm1, %zmm0
-; X32-NEXT:    retl
+; X86-LABEL: combine_permvar_8f64_identity_mask:
+; X86:       # %bb.0:
+; X86-NEXT:    vmovapd {{.*#+}} zmm2 = [7,0,6,0,5,0,4,0,3,0,2,0,1,0,0,0]
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovd %eax, %k1
+; X86-NEXT:    vpermpd %zmm0, %zmm2, %zmm1 {%k1}
+; X86-NEXT:    vmovapd {{.*#+}} zmm0 = [7,0,14,0,5,0,12,0,3,0,10,0,1,0,8,0]
+; X86-NEXT:    vpermpd %zmm1, %zmm0, %zmm1 {%k1}
+; X86-NEXT:    vmovapd %zmm1, %zmm0
+; X86-NEXT:    retl
 ;
 ; X64-LABEL: combine_permvar_8f64_identity_mask:
 ; X64:       # %bb.0:
@@ -68,28 +64,24 @@ define <8 x double> @combine_permvar_8f64_identity_mask(<8 x double> %x0, <8 x d
 }
 
 define <8 x i64> @combine_permvar_8i64_identity(<8 x i64> %x0, <8 x i64> %x1) {
-; X32-LABEL: combine_permvar_8i64_identity:
-; X32:       # %bb.0:
-; X32-NEXT:    retl
-;
-; X64-LABEL: combine_permvar_8i64_identity:
-; X64:       # %bb.0:
-; X64-NEXT:    retq
+; CHECK-LABEL: combine_permvar_8i64_identity:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ret{{[l|q]}}
   %1 = call <8 x i64> @llvm.x86.avx512.permvar.di.512(<8 x i64> %x0, <8 x i64> <i64 7, i64 6, i64 5, i64 4, i64 3, i64 2, i64 1, i64 0>)
   %2 = call <8 x i64> @llvm.x86.avx512.permvar.di.512(<8 x i64> %1, <8 x i64> <i64 7, i64 14, i64 5, i64 12, i64 3, i64 10, i64 1, i64 8>)
   ret <8 x i64> %2
 }
 define <8 x i64> @combine_permvar_8i64_identity_mask(<8 x i64> %x0, <8 x i64> %x1, i8 %m) {
-; X32-LABEL: combine_permvar_8i64_identity_mask:
-; X32:       # %bb.0:
-; X32-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [7,0,6,0,5,0,4,0,3,0,2,0,1,0,0,0]
-; X32-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    kmovd %eax, %k1
-; X32-NEXT:    vpermq %zmm0, %zmm2, %zmm1 {%k1}
-; X32-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [7,0,14,0,5,0,12,0,3,0,10,0,1,0,8,0]
-; X32-NEXT:    vpermq %zmm1, %zmm0, %zmm1 {%k1}
-; X32-NEXT:    vmovdqa64 %zmm1, %zmm0
-; X32-NEXT:    retl
+; X86-LABEL: combine_permvar_8i64_identity_mask:
+; X86:       # %bb.0:
+; X86-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [7,0,6,0,5,0,4,0,3,0,2,0,1,0,0,0]
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovd %eax, %k1
+; X86-NEXT:    vpermq %zmm0, %zmm2, %zmm1 {%k1}
+; X86-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [7,0,14,0,5,0,12,0,3,0,10,0,1,0,8,0]
+; X86-NEXT:    vpermq %zmm1, %zmm0, %zmm1 {%k1}
+; X86-NEXT:    vmovdqa64 %zmm1, %zmm0
+; X86-NEXT:    retl
 ;
 ; X64-LABEL: combine_permvar_8i64_identity_mask:
 ; X64:       # %bb.0:
@@ -110,27 +102,23 @@ define <8 x i64> @combine_permvar_8i64_identity_mask(<8 x i64> %x0, <8 x i64> %x
 }
 
 define <8 x double> @combine_vpermt2var_8f64_identity(<8 x double> %x0, <8 x double> %x1) {
-; X32-LABEL: combine_vpermt2var_8f64_identity:
-; X32:       # %bb.0:
-; X32-NEXT:    retl
-;
-; X64-LABEL: combine_vpermt2var_8f64_identity:
-; X64:       # %bb.0:
-; X64-NEXT:    retq
+; CHECK-LABEL: combine_vpermt2var_8f64_identity:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ret{{[l|q]}}
   %res0 = call <8 x double> @llvm.x86.avx512.maskz.vpermt2var.pd.512(<8 x i64> <i64 7, i64 6, i64 5, i64 4, i64 3, i64 2, i64 1, i64 0>, <8 x double> %x0, <8 x double> %x1, i8 -1)
   %res1 = call <8 x double> @llvm.x86.avx512.maskz.vpermt2var.pd.512(<8 x i64> <i64 7, i64 14, i64 5, i64 12, i64 3, i64 10, i64 1, i64 8>, <8 x double> %res0, <8 x double> %res0, i8 -1)
   ret <8 x double> %res1
 }
 define <8 x double> @combine_vpermt2var_8f64_identity_mask(<8 x double> %x0, <8 x double> %x1, i8 %m) {
-; X32-LABEL: combine_vpermt2var_8f64_identity_mask:
-; X32:       # %bb.0:
-; X32-NEXT:    vmovapd {{.*#+}} zmm1 = [7,0,6,0,5,0,4,0,3,0,2,0,1,0,0,0]
-; X32-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    kmovd %eax, %k1
-; X32-NEXT:    vpermi2pd %zmm0, %zmm0, %zmm1 {%k1} {z}
-; X32-NEXT:    vmovapd {{.*#+}} zmm0 = [7,0,14,0,5,0,12,0,3,0,10,0,1,0,8,0]
-; X32-NEXT:    vpermi2pd %zmm1, %zmm1, %zmm0 {%k1} {z}
-; X32-NEXT:    retl
+; X86-LABEL: combine_vpermt2var_8f64_identity_mask:
+; X86:       # %bb.0:
+; X86-NEXT:    vmovapd {{.*#+}} zmm1 = [7,0,6,0,5,0,4,0,3,0,2,0,1,0,0,0]
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovd %eax, %k1
+; X86-NEXT:    vpermi2pd %zmm0, %zmm0, %zmm1 {%k1} {z}
+; X86-NEXT:    vmovapd {{.*#+}} zmm0 = [7,0,14,0,5,0,12,0,3,0,10,0,1,0,8,0]
+; X86-NEXT:    vpermi2pd %zmm1, %zmm1, %zmm0 {%k1} {z}
+; X86-NEXT:    retl
 ;
 ; X64-LABEL: combine_vpermt2var_8f64_identity_mask:
 ; X64:       # %bb.0:
@@ -146,24 +134,19 @@ define <8 x double> @combine_vpermt2var_8f64_identity_mask(<8 x double> %x0, <8
 }
 
 define <8 x double> @combine_vpermt2var_8f64_movddup(<8 x double> %x0, <8 x double> %x1) {
-; X32-LABEL: combine_vpermt2var_8f64_movddup:
-; X32:       # %bb.0:
-; X32-NEXT:    vmovddup {{.*#+}} zmm0 = zmm0[0,0,2,2,4,4,6,6]
-; X32-NEXT:    retl
-;
-; X64-LABEL: combine_vpermt2var_8f64_movddup:
-; X64:       # %bb.0:
-; X64-NEXT:    vmovddup {{.*#+}} zmm0 = zmm0[0,0,2,2,4,4,6,6]
-; X64-NEXT:    retq
+; CHECK-LABEL: combine_vpermt2var_8f64_movddup:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovddup {{.*#+}} zmm0 = zmm0[0,0,2,2,4,4,6,6]
+; CHECK-NEXT:    ret{{[l|q]}}
   %res0 = call <8 x double> @llvm.x86.avx512.maskz.vpermt2var.pd.512(<8 x i64> <i64 0, i64 0, i64 2, i64 2, i64 4, i64 4, i64 undef, i64 undef>, <8 x double> %x0, <8 x double> %x1, i8 -1)
   ret <8 x double> %res0
 }
 define <8 x double> @combine_vpermt2var_8f64_movddup_load(<8 x double> *%p0, <8 x double> %x1) {
-; X32-LABEL: combine_vpermt2var_8f64_movddup_load:
-; X32:       # %bb.0:
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    vmovddup {{.*#+}} zmm0 = mem[0,0,2,2,4,4,6,6]
-; X32-NEXT:    retl
+; X86-LABEL: combine_vpermt2var_8f64_movddup_load:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    vmovddup {{.*#+}} zmm0 = mem[0,0,2,2,4,4,6,6]
+; X86-NEXT:    retl
 ;
 ; X64-LABEL: combine_vpermt2var_8f64_movddup_load:
 ; X64:       # %bb.0:
@@ -174,12 +157,12 @@ define <8 x double> @combine_vpermt2var_8f64_movddup_load(<8 x double> *%p0, <8
   ret <8 x double> %res0
 }
 define <8 x double> @combine_vpermt2var_8f64_movddup_mask(<8 x double> %x0, <8 x double> %x1, i8 %m) {
-; X32-LABEL: combine_vpermt2var_8f64_movddup_mask:
-; X32:       # %bb.0:
-; X32-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    kmovd %eax, %k1
-; X32-NEXT:    vmovddup {{.*#+}} zmm0 {%k1} {z} = zmm0[0,0,2,2,4,4,6,6]
-; X32-NEXT:    retl
+; X86-LABEL: combine_vpermt2var_8f64_movddup_mask:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovd %eax, %k1
+; X86-NEXT:    vmovddup {{.*#+}} zmm0 {%k1} {z} = zmm0[0,0,2,2,4,4,6,6]
+; X86-NEXT:    retl
 ;
 ; X64-LABEL: combine_vpermt2var_8f64_movddup_mask:
 ; X64:       # %bb.0:
@@ -191,27 +174,23 @@ define <8 x double> @combine_vpermt2var_8f64_movddup_mask(<8 x double> %x0, <8 x
 }
 
 define <8 x i64> @combine_vpermt2var_8i64_identity(<8 x i64> %x0, <8 x i64> %x1) {
-; X32-LABEL: combine_vpermt2var_8i64_identity:
-; X32:       # %bb.0:
-; X32-NEXT:    retl
-;
-; X64-LABEL: combine_vpermt2var_8i64_identity:
-; X64:       # %bb.0:
-; X64-NEXT:    retq
+; CHECK-LABEL: combine_vpermt2var_8i64_identity:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ret{{[l|q]}}
   %res0 = call <8 x i64> @llvm.x86.avx512.maskz.vpermt2var.q.512(<8 x i64> <i64 undef, i64 6, i64 5, i64 4, i64 3, i64 2, i64 1, i64 0>, <8 x i64> %x0, <8 x i64> %x1, i8 -1)
   %res1 = call <8 x i64> @llvm.x86.avx512.maskz.vpermt2var.q.512(<8 x i64> <i64 undef, i64 14, i64 5, i64 12, i64 3, i64 10, i64 1, i64 8>, <8 x i64> %res0, <8 x i64> %res0, i8 -1)
   ret <8 x i64> %res1
 }
 define <8 x i64> @combine_vpermt2var_8i64_identity_mask(<8 x i64> %x0, <8 x i64> %x1, i8 %m) {
-; X32-LABEL: combine_vpermt2var_8i64_identity_mask:
-; X32:       # %bb.0:
-; X32-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [7,0,6,0,5,0,4,0,3,0,2,0,1,0,0,0]
-; X32-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    kmovd %eax, %k1
-; X32-NEXT:    vpermi2q %zmm0, %zmm0, %zmm1 {%k1} {z}
-; X32-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [7,0,14,0,5,0,12,0,3,0,10,0,1,0,8,0]
-; X32-NEXT:    vpermi2q %zmm1, %zmm1, %zmm0 {%k1} {z}
-; X32-NEXT:    retl
+; X86-LABEL: combine_vpermt2var_8i64_identity_mask:
+; X86:       # %bb.0:
+; X86-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [7,0,6,0,5,0,4,0,3,0,2,0,1,0,0,0]
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovd %eax, %k1
+; X86-NEXT:    vpermi2q %zmm0, %zmm0, %zmm1 {%k1} {z}
+; X86-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [7,0,14,0,5,0,12,0,3,0,10,0,1,0,8,0]
+; X86-NEXT:    vpermi2q %zmm1, %zmm1, %zmm0 {%k1} {z}
+; X86-NEXT:    retl
 ;
 ; X64-LABEL: combine_vpermt2var_8i64_identity_mask:
 ; X64:       # %bb.0:
@@ -227,26 +206,22 @@ define <8 x i64> @combine_vpermt2var_8i64_identity_mask(<8 x i64> %x0, <8 x i64>
 }
 
 define <16 x float> @combine_vpermt2var_16f32_identity(<16 x float> %x0, <16 x float> %x1) {
-; X32-LABEL: combine_vpermt2var_16f32_identity:
-; X32:       # %bb.0:
-; X32-NEXT:    retl
-;
-; X64-LABEL: combine_vpermt2var_16f32_identity:
-; X64:       # %bb.0:
-; X64-NEXT:    retq
+; CHECK-LABEL: combine_vpermt2var_16f32_identity:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ret{{[l|q]}}
   %res0 = call <16 x float> @llvm.x86.avx512.maskz.vpermt2var.ps.512(<16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>, <16 x float> %x0, <16 x float> %x1, i16 -1)
   %res1 = call <16 x float> @llvm.x86.avx512.maskz.vpermt2var.ps.512(<16 x i32> <i32 15, i32 30, i32 13, i32 28, i32 11, i32 26, i32 9, i32 24, i32 7, i32 22, i32 5, i32 20, i32 3, i32 18, i32 1, i32 16>, <16 x float> %res0, <16 x float> %res0, i16 -1)
   ret <16 x float> %res1
 }
 define <16 x float> @combine_vpermt2var_16f32_identity_mask(<16 x float> %x0, <16 x float> %x1, i16 %m) {
-; X32-LABEL: combine_vpermt2var_16f32_identity_mask:
-; X32:       # %bb.0:
-; X32-NEXT:    vmovaps {{.*#+}} zmm1 = [15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0]
-; X32-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
-; X32-NEXT:    vpermi2ps %zmm0, %zmm0, %zmm1 {%k1} {z}
-; X32-NEXT:    vmovaps {{.*#+}} zmm0 = [15,30,13,28,11,26,9,24,7,22,5,20,3,18,1,16]
-; X32-NEXT:    vpermi2ps %zmm1, %zmm1, %zmm0 {%k1} {z}
-; X32-NEXT:    retl
+; X86-LABEL: combine_vpermt2var_16f32_identity_mask:
+; X86:       # %bb.0:
+; X86-NEXT:    vmovaps {{.*#+}} zmm1 = [15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0]
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vpermi2ps %zmm0, %zmm0, %zmm1 {%k1} {z}
+; X86-NEXT:    vmovaps {{.*#+}} zmm0 = [15,30,13,28,11,26,9,24,7,22,5,20,3,18,1,16]
+; X86-NEXT:    vpermi2ps %zmm1, %zmm1, %zmm0 {%k1} {z}
+; X86-NEXT:    retl
 ;
 ; X64-LABEL: combine_vpermt2var_16f32_identity_mask:
 ; X64:       # %bb.0:
@@ -262,28 +237,22 @@ define <16 x float> @combine_vpermt2var_16f32_identity_mask(<16 x float> %x0, <1
 }
 
 define <16 x float> @combine_vpermt2var_16f32_vmovddup(<16 x float> %x0, <16 x float> %x1) {
-; X32-LABEL: combine_vpermt2var_16f32_vmovddup:
-; X32:       # %bb.0:
-; X32-NEXT:    vmovaps {{.*#+}} zmm1 = [0,1,0,1,4,5,4,5,8,9,8,9,12,13,12,13]
-; X32-NEXT:    vpermt2ps %zmm0, %zmm1, %zmm0
-; X32-NEXT:    retl
-;
-; X64-LABEL: combine_vpermt2var_16f32_vmovddup:
-; X64:       # %bb.0:
-; X64-NEXT:    vmovaps {{.*#+}} zmm1 = [0,1,0,1,4,5,4,5,8,9,8,9,12,13,12,13]
-; X64-NEXT:    vpermt2ps %zmm0, %zmm1, %zmm0
-; X64-NEXT:    retq
+; CHECK-LABEL: combine_vpermt2var_16f32_vmovddup:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps {{.*#+}} zmm1 = [0,1,0,1,4,5,4,5,8,9,8,9,12,13,12,13]
+; CHECK-NEXT:    vpermt2ps %zmm0, %zmm1, %zmm0
+; CHECK-NEXT:    ret{{[l|q]}}
   %res0 = call <16 x float> @llvm.x86.avx512.maskz.vpermt2var.ps.512(<16 x i32> <i32 0, i32 1, i32 0, i32 1, i32 4, i32 5, i32 4, i32 5, i32 8, i32 9, i32 8, i32 9, i32 12, i32 13, i32 12, i32 13>, <16 x float> %x0, <16 x float> %x1, i16 -1)
   ret <16 x float> %res0
 }
 define <16 x float> @combine_vpermt2var_16f32_vmovddup_load(<16 x float> *%p0, <16 x float> %x1) {
-; X32-LABEL: combine_vpermt2var_16f32_vmovddup_load:
-; X32:       # %bb.0:
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    vmovaps (%eax), %zmm1
-; X32-NEXT:    vmovaps {{.*#+}} zmm0 = [0,1,0,1,4,5,4,5,8,9,8,9,12,13,12,13]
-; X32-NEXT:    vpermi2ps %zmm0, %zmm1, %zmm0
-; X32-NEXT:    retl
+; X86-LABEL: combine_vpermt2var_16f32_vmovddup_load:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    vmovaps (%eax), %zmm1
+; X86-NEXT:    vmovaps {{.*#+}} zmm0 = [0,1,0,1,4,5,4,5,8,9,8,9,12,13,12,13]
+; X86-NEXT:    vpermi2ps %zmm0, %zmm1, %zmm0
+; X86-NEXT:    retl
 ;
 ; X64-LABEL: combine_vpermt2var_16f32_vmovddup_load:
 ; X64:       # %bb.0:
@@ -296,12 +265,12 @@ define <16 x float> @combine_vpermt2var_16f32_vmovddup_load(<16 x float> *%p0, <
   ret <16 x float> %res0
 }
 define <16 x float> @combine_vpermt2var_16f32_vmovddup_mask(<16 x float> %x0, <16 x float> %x1, i16 %m) {
-; X32-LABEL: combine_vpermt2var_16f32_vmovddup_mask:
-; X32:       # %bb.0:
-; X32-NEXT:    vmovaps {{.*#+}} zmm1 = [0,1,0,1,4,5,4,5,8,9,8,9,12,13,12,13]
-; X32-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
-; X32-NEXT:    vpermt2ps %zmm0, %zmm1, %zmm0 {%k1} {z}
-; X32-NEXT:    retl
+; X86-LABEL: combine_vpermt2var_16f32_vmovddup_mask:
+; X86:       # %bb.0:
+; X86-NEXT:    vmovaps {{.*#+}} zmm1 = [0,1,0,1,4,5,4,5,8,9,8,9,12,13,12,13]
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vpermt2ps %zmm0, %zmm1, %zmm0 {%k1} {z}
+; X86-NEXT:    retl
 ;
 ; X64-LABEL: combine_vpermt2var_16f32_vmovddup_mask:
 ; X64:       # %bb.0:
@@ -313,14 +282,14 @@ define <16 x float> @combine_vpermt2var_16f32_vmovddup_mask(<16 x float> %x0, <1
   ret <16 x float> %res0
 }
 define <16 x float> @combine_vpermt2var_16f32_vmovddup_mask_load(<16 x float> *%p0, <16 x float> %x1, i16 %m) {
-; X32-LABEL: combine_vpermt2var_16f32_vmovddup_mask_load:
-; X32:       # %bb.0:
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    vmovaps (%eax), %zmm1
-; X32-NEXT:    vmovaps {{.*#+}} zmm0 = [0,1,0,1,4,5,4,5,8,9,8,9,12,13,12,13]
-; X32-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
-; X32-NEXT:    vpermi2ps %zmm0, %zmm1, %zmm0 {%k1} {z}
-; X32-NEXT:    retl
+; X86-LABEL: combine_vpermt2var_16f32_vmovddup_mask_load:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    vmovaps (%eax), %zmm1
+; X86-NEXT:    vmovaps {{.*#+}} zmm0 = [0,1,0,1,4,5,4,5,8,9,8,9,12,13,12,13]
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vpermi2ps %zmm0, %zmm1, %zmm0 {%k1} {z}
+; X86-NEXT:    retl
 ;
 ; X64-LABEL: combine_vpermt2var_16f32_vmovddup_mask_load:
 ; X64:       # %bb.0:
@@ -335,24 +304,19 @@ define <16 x float> @combine_vpermt2var_16f32_vmovddup_mask_load(<16 x float> *%
 }
 
 define <16 x float> @combine_vpermt2var_16f32_vmovshdup(<16 x float> %x0, <16 x float> %x1) {
-; X32-LABEL: combine_vpermt2var_16f32_vmovshdup:
-; X32:       # %bb.0:
-; X32-NEXT:    vmovshdup {{.*#+}} zmm0 = zmm0[1,1,3,3,5,5,7,7,9,9,11,11,13,13,15,15]
-; X32-NEXT:    retl
-;
-; X64-LABEL: combine_vpermt2var_16f32_vmovshdup:
-; X64:       # %bb.0:
-; X64-NEXT:    vmovshdup {{.*#+}} zmm0 = zmm0[1,1,3,3,5,5,7,7,9,9,11,11,13,13,15,15]
-; X64-NEXT:    retq
+; CHECK-LABEL: combine_vpermt2var_16f32_vmovshdup:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovshdup {{.*#+}} zmm0 = zmm0[1,1,3,3,5,5,7,7,9,9,11,11,13,13,15,15]
+; CHECK-NEXT:    ret{{[l|q]}}
   %res0 = call <16 x float> @llvm.x86.avx512.maskz.vpermt2var.ps.512(<16 x i32> <i32 1, i32 1, i32 3, i32 3, i32 5, i32 5, i32 7, i32 7, i32 9, i32 9, i32 11, i32 11, i32 13, i32 13, i32 15, i32 15>, <16 x float> %x0, <16 x float> %x1, i16 -1)
   ret <16 x float> %res0
 }
 define <16 x float> @combine_vpermt2var_16f32_vmovshdup_load(<16 x float> *%p0, <16 x float> %x1) {
-; X32-LABEL: combine_vpermt2var_16f32_vmovshdup_load:
-; X32:       # %bb.0:
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    vmovshdup {{.*#+}} zmm0 = mem[1,1,3,3,5,5,7,7,9,9,11,11,13,13,15,15]
-; X32-NEXT:    retl
+; X86-LABEL: combine_vpermt2var_16f32_vmovshdup_load:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    vmovshdup {{.*#+}} zmm0 = mem[1,1,3,3,5,5,7,7,9,9,11,11,13,13,15,15]
+; X86-NEXT:    retl
 ;
 ; X64-LABEL: combine_vpermt2var_16f32_vmovshdup_load:
 ; X64:       # %bb.0:
@@ -363,11 +327,11 @@ define <16 x float> @combine_vpermt2var_16f32_vmovshdup_load(<16 x float> *%p0,
   ret <16 x float> %res0
 }
 define <16 x float> @combine_vpermt2var_16f32_vmovshdup_mask(<16 x float> %x0, <16 x float> %x1, i16 %m) {
-; X32-LABEL: combine_vpermt2var_16f32_vmovshdup_mask:
-; X32:       # %bb.0:
-; X32-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
-; X32-NEXT:    vmovshdup {{.*#+}} zmm0 {%k1} {z} = zmm0[1,1,3,3,5,5,7,7,9,9,11,11,13,13,15,15]
-; X32-NEXT:    retl
+; X86-LABEL: combine_vpermt2var_16f32_vmovshdup_mask:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vmovshdup {{.*#+}} zmm0 {%k1} {z} = zmm0[1,1,3,3,5,5,7,7,9,9,11,11,13,13,15,15]
+; X86-NEXT:    retl
 ;
 ; X64-LABEL: combine_vpermt2var_16f32_vmovshdup_mask:
 ; X64:       # %bb.0:
@@ -379,24 +343,19 @@ define <16 x float> @combine_vpermt2var_16f32_vmovshdup_mask(<16 x float> %x0, <
 }
 
 define <16 x float> @combine_vpermt2var_16f32_vmovsldup(<16 x float> %x0, <16 x float> %x1) {
-; X32-LABEL: combine_vpermt2var_16f32_vmovsldup:
-; X32:       # %bb.0:
-; X32-NEXT:    vmovsldup {{.*#+}} zmm0 = zmm0[0,0,2,2,4,4,6,6,8,8,10,10,12,12,14,14]
-; X32-NEXT:    retl
-;
-; X64-LABEL: combine_vpermt2var_16f32_vmovsldup:
-; X64:       # %bb.0:
-; X64-NEXT:    vmovsldup {{.*#+}} zmm0 = zmm0[0,0,2,2,4,4,6,6,8,8,10,10,12,12,14,14]
-; X64-NEXT:    retq
+; CHECK-LABEL: combine_vpermt2var_16f32_vmovsldup:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovsldup {{.*#+}} zmm0 = zmm0[0,0,2,2,4,4,6,6,8,8,10,10,12,12,14,14]
+; CHECK-NEXT:    ret{{[l|q]}}
   %res0 = call <16 x float> @llvm.x86.avx512.maskz.vpermt2var.ps.512(<16 x i32> <i32 0, i32 0, i32 2, i32 2, i32 4, i32 4, i32 6, i32 6, i32 8, i32 8, i32 10, i32 10, i32 12, i32 12, i32 14, i32 14>, <16 x float> %x0, <16 x float> %x1, i16 -1)
   ret <16 x float> %res0
 }
 define <16 x float> @combine_vpermt2var_16f32_vmovsldup_load(<16 x float> *%p0, <16 x float> %x1) {
-; X32-LABEL: combine_vpermt2var_16f32_vmovsldup_load:
-; X32:       # %bb.0:
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    vmovsldup {{.*#+}} zmm0 = mem[0,0,2,2,4,4,6,6,8,8,10,10,12,12,14,14]
-; X32-NEXT:    retl
+; X86-LABEL: combine_vpermt2var_16f32_vmovsldup_load:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    vmovsldup {{.*#+}} zmm0 = mem[0,0,2,2,4,4,6,6,8,8,10,10,12,12,14,14]
+; X86-NEXT:    retl
 ;
 ; X64-LABEL: combine_vpermt2var_16f32_vmovsldup_load:
 ; X64:       # %bb.0:
@@ -407,11 +366,11 @@ define <16 x float> @combine_vpermt2var_16f32_vmovsldup_load(<16 x float> *%p0,
   ret <16 x float> %res0
 }
 define <16 x float> @combine_vpermt2var_16f32_vmovsldup_mask(<16 x float> %x0, <16 x float> %x1, i16 %m) {
-; X32-LABEL: combine_vpermt2var_16f32_vmovsldup_mask:
-; X32:       # %bb.0:
-; X32-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
-; X32-NEXT:    vmovsldup {{.*#+}} zmm0 {%k1} {z} = zmm0[0,0,2,2,4,4,6,6,8,8,10,10,12,12,14,14]
-; X32-NEXT:    retl
+; X86-LABEL: combine_vpermt2var_16f32_vmovsldup_mask:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vmovsldup {{.*#+}} zmm0 {%k1} {z} = zmm0[0,0,2,2,4,4,6,6,8,8,10,10,12,12,14,14]
+; X86-NEXT:    retl
 ;
 ; X64-LABEL: combine_vpermt2var_16f32_vmovsldup_mask:
 ; X64:       # %bb.0:
@@ -422,12 +381,12 @@ define <16 x float> @combine_vpermt2var_16f32_vmovsldup_mask(<16 x float> %x0, <
   ret <16 x float> %res0
 }
 define <16 x float> @combine_vpermt2var_16f32_vmovsldup_mask_load(<16 x float> *%p0, <16 x float> %x1, i16 %m) {
-; X32-LABEL: combine_vpermt2var_16f32_vmovsldup_mask_load:
-; X32:       # %bb.0:
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
-; X32-NEXT:    vmovsldup {{.*#+}} zmm0 {%k1} {z} = mem[0,0,2,2,4,4,6,6,8,8,10,10,12,12,14,14]
-; X32-NEXT:    retl
+; X86-LABEL: combine_vpermt2var_16f32_vmovsldup_mask_load:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vmovsldup {{.*#+}} zmm0 {%k1} {z} = mem[0,0,2,2,4,4,6,6,8,8,10,10,12,12,14,14]
+; X86-NEXT:    retl
 ;
 ; X64-LABEL: combine_vpermt2var_16f32_vmovsldup_mask_load:
 ; X64:       # %bb.0:
@@ -440,24 +399,19 @@ define <16 x float> @combine_vpermt2var_16f32_vmovsldup_mask_load(<16 x float> *
 }
 
 define <16 x float> @combine_vpermt2var_16f32_vpermilps(<16 x float> %x0, <16 x float> %x1) {
-; X32-LABEL: combine_vpermt2var_16f32_vpermilps:
-; X32:       # %bb.0:
-; X32-NEXT:    vpermilps {{.*#+}} zmm0 = zmm0[3,2,1,0,7,6,5,4,11,10,9,8,15,14,13,12]
-; X32-NEXT:    retl
-;
-; X64-LABEL: combine_vpermt2var_16f32_vpermilps:
-; X64:       # %bb.0:
-; X64-NEXT:    vpermilps {{.*#+}} zmm0 = zmm0[3,2,1,0,7,6,5,4,11,10,9,8,15,14,13,12]
-; X64-NEXT:    retq
+; CHECK-LABEL: combine_vpermt2var_16f32_vpermilps:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpermilps {{.*#+}} zmm0 = zmm0[3,2,1,0,7,6,5,4,11,10,9,8,15,14,13,12]
+; CHECK-NEXT:    ret{{[l|q]}}
   %res0 = call <16 x float> @llvm.x86.avx512.maskz.vpermt2var.ps.512(<16 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4, i32 11, i32 10, i32 9, i32 8, i32 15, i32 14, i32 13, i32 12>, <16 x float> %x0, <16 x float> %x1, i16 -1)
   ret <16 x float> %res0
 }
 define <16 x float> @combine_vpermt2var_16f32_vpermilps_load(<16 x float> *%p0, <16 x float> %x1) {
-; X32-LABEL: combine_vpermt2var_16f32_vpermilps_load:
-; X32:       # %bb.0:
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    vpermilps {{.*#+}} zmm0 = mem[3,2,1,0,7,6,5,4,11,10,9,8,15,14,13,12]
-; X32-NEXT:    retl
+; X86-LABEL: combine_vpermt2var_16f32_vpermilps_load:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    vpermilps {{.*#+}} zmm0 = mem[3,2,1,0,7,6,5,4,11,10,9,8,15,14,13,12]
+; X86-NEXT:    retl
 ;
 ; X64-LABEL: combine_vpermt2var_16f32_vpermilps_load:
 ; X64:       # %bb.0:
@@ -468,11 +422,11 @@ define <16 x float> @combine_vpermt2var_16f32_vpermilps_load(<16 x float> *%p0,
   ret <16 x float> %res0
 }
 define <16 x float> @combine_vpermt2var_16f32_vpermilps_mask(<16 x float> %x0, <16 x float> %x1, i16 %m) {
-; X32-LABEL: combine_vpermt2var_16f32_vpermilps_mask:
-; X32:       # %bb.0:
-; X32-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
-; X32-NEXT:    vpermilps {{.*#+}} zmm0 {%k1} {z} = zmm0[3,2,1,0,7,6,5,4,11,10,9,8,15,14,13,12]
-; X32-NEXT:    retl
+; X86-LABEL: combine_vpermt2var_16f32_vpermilps_mask:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vpermilps {{.*#+}} zmm0 {%k1} {z} = zmm0[3,2,1,0,7,6,5,4,11,10,9,8,15,14,13,12]
+; X86-NEXT:    retl
 ;
 ; X64-LABEL: combine_vpermt2var_16f32_vpermilps_mask:
 ; X64:       # %bb.0:
@@ -483,12 +437,12 @@ define <16 x float> @combine_vpermt2var_16f32_vpermilps_mask(<16 x float> %x0, <
   ret <16 x float> %res0
 }
 define <16 x float> @combine_vpermt2var_16f32_vpermilps_mask_load(<16 x float> *%p0, <16 x float> %x1, i16 %m) {
-; X32-LABEL: combine_vpermt2var_16f32_vpermilps_mask_load:
-; X32:       # %bb.0:
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
-; X32-NEXT:    vpermilps {{.*#+}} zmm0 {%k1} {z} = mem[3,2,1,0,7,6,5,4,11,10,9,8,15,14,13,12]
-; X32-NEXT:    retl
+; X86-LABEL: combine_vpermt2var_16f32_vpermilps_mask_load:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vpermilps {{.*#+}} zmm0 {%k1} {z} = mem[3,2,1,0,7,6,5,4,11,10,9,8,15,14,13,12]
+; X86-NEXT:    retl
 ;
 ; X64-LABEL: combine_vpermt2var_16f32_vpermilps_mask_load:
 ; X64:       # %bb.0:
@@ -501,26 +455,22 @@ define <16 x float> @combine_vpermt2var_16f32_vpermilps_mask_load(<16 x float> *
 }
 
 define <16 x i32> @combine_vpermt2var_16i32_identity(<16 x i32> %x0, <16 x i32> %x1) {
-; X32-LABEL: combine_vpermt2var_16i32_identity:
-; X32:       # %bb.0:
-; X32-NEXT:    retl
-;
-; X64-LABEL: combine_vpermt2var_16i32_identity:
-; X64:       # %bb.0:
-; X64-NEXT:    retq
+; CHECK-LABEL: combine_vpermt2var_16i32_identity:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ret{{[l|q]}}
   %res0 = call <16 x i32> @llvm.x86.avx512.maskz.vpermt2var.d.512(<16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 undef>, <16 x i32> %x0, <16 x i32> %x1, i16 -1)
   %res1 = call <16 x i32> @llvm.x86.avx512.maskz.vpermt2var.d.512(<16 x i32> <i32 15, i32 30, i32 13, i32 28, i32 undef, i32 26, i32 9, i32 24, i32 7, i32 22, i32 5, i32 20, i32 3, i32 18, i32 1, i32 16>, <16 x i32> %res0, <16 x i32> %res0, i16 -1)
   ret <16 x i32> %res1
 }
 define <16 x i32> @combine_vpermt2var_16i32_identity_mask(<16 x i32> %x0, <16 x i32> %x1, i16 %m) {
-; X32-LABEL: combine_vpermt2var_16i32_identity_mask:
-; X32:       # %bb.0:
-; X32-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0]
-; X32-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
-; X32-NEXT:    vpermi2d %zmm0, %zmm0, %zmm1 {%k1} {z}
-; X32-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [15,30,13,28,11,26,9,24,7,22,5,20,3,18,1,16]
-; X32-NEXT:    vpermi2d %zmm1, %zmm1, %zmm0 {%k1} {z}
-; X32-NEXT:    retl
+; X86-LABEL: combine_vpermt2var_16i32_identity_mask:
+; X86:       # %bb.0:
+; X86-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0]
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vpermi2d %zmm0, %zmm0, %zmm1 {%k1} {z}
+; X86-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [15,30,13,28,11,26,9,24,7,22,5,20,3,18,1,16]
+; X86-NEXT:    vpermi2d %zmm1, %zmm1, %zmm0 {%k1} {z}
+; X86-NEXT:    retl
 ;
 ; X64-LABEL: combine_vpermt2var_16i32_identity_mask:
 ; X64:       # %bb.0:
@@ -536,26 +486,22 @@ define <16 x i32> @combine_vpermt2var_16i32_identity_mask(<16 x i32> %x0, <16 x
 }
 
 define <32 x i16> @combine_vpermt2var_32i16_identity(<32 x i16> %x0, <32 x i16> %x1) {
-; X32-LABEL: combine_vpermt2var_32i16_identity:
-; X32:       # %bb.0:
-; X32-NEXT:    retl
-;
-; X64-LABEL: combine_vpermt2var_32i16_identity:
-; X64:       # %bb.0:
-; X64-NEXT:    retq
+; CHECK-LABEL: combine_vpermt2var_32i16_identity:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ret{{[l|q]}}
   %res0 = call <32 x i16> @llvm.x86.avx512.maskz.vpermt2var.hi.512(<32 x i16> <i16 31, i16 30, i16 29, i16 28, i16 27, i16 26, i16 25, i16 24, i16 23, i16 22, i16 21, i16 20, i16 19, i16 18, i16 17, i16 16, i16 15, i16 14, i16 13, i16 12, i16 11, i16 10, i16 9, i16 8, i16 7, i16 6, i16 5, i16 4, i16 3, i16 2, i16 1, i16 0>, <32 x i16> %x0, <32 x i16> %x1, i32 -1)
   %res1 = call <32 x i16> @llvm.x86.avx512.maskz.vpermt2var.hi.512(<32 x i16> <i16 63, i16 30, i16 61, i16 28, i16 59, i16 26, i16 57, i16 24, i16 55, i16 22, i16 53, i16 20, i16 51, i16 18, i16 49, i16 16, i16 47, i16 46, i16 13, i16 44, i16 11, i16 42, i16 9, i16 40, i16 7, i16 38, i16 5, i16 36, i16 3, i16 34, i16 1, i16 32>, <32 x i16> %res0, <32 x i16> %res0, i32 -1)
   ret <32 x i16> %res1
 }
 define <32 x i16> @combine_vpermt2var_32i16_identity_mask(<32 x i16> %x0, <32 x i16> %x1, i32 %m) {
-; X32-LABEL: combine_vpermt2var_32i16_identity_mask:
-; X32:       # %bb.0:
-; X32-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [31,30,29,28,27,26,25,24,23,22,21,20,19,18,17,16,15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0]
-; X32-NEXT:    kmovd {{[0-9]+}}(%esp), %k1
-; X32-NEXT:    vpermi2w %zmm0, %zmm0, %zmm1 {%k1} {z}
-; X32-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [63,30,61,28,59,26,57,24,55,22,53,20,51,18,49,16,47,46,13,44,11,42,9,40,7,38,5,36,3,34,1,32]
-; X32-NEXT:    vpermi2w %zmm1, %zmm1, %zmm0 {%k1} {z}
-; X32-NEXT:    retl
+; X86-LABEL: combine_vpermt2var_32i16_identity_mask:
+; X86:       # %bb.0:
+; X86-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [31,30,29,28,27,26,25,24,23,22,21,20,19,18,17,16,15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0]
+; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vpermi2w %zmm0, %zmm0, %zmm1 {%k1} {z}
+; X86-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [63,30,61,28,59,26,57,24,55,22,53,20,51,18,49,16,47,46,13,44,11,42,9,40,7,38,5,36,3,34,1,32]
+; X86-NEXT:    vpermi2w %zmm1, %zmm1, %zmm0 {%k1} {z}
+; X86-NEXT:    retl
 ;
 ; X64-LABEL: combine_vpermt2var_32i16_identity_mask:
 ; X64:       # %bb.0:
@@ -571,13 +517,9 @@ define <32 x i16> @combine_vpermt2var_32i16_identity_mask(<32 x i16> %x0, <32 x
 }
 
 define <64 x i8> @combine_pshufb_identity(<64 x i8> %x0) {
-; X32-LABEL: combine_pshufb_identity:
-; X32:       # %bb.0:
-; X32-NEXT:    retl
-;
-; X64-LABEL: combine_pshufb_identity:
-; X64:       # %bb.0:
-; X64-NEXT:    retq
+; CHECK-LABEL: combine_pshufb_identity:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ret{{[l|q]}}
   %select = bitcast <8 x i64> <i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1, i64 -1> to <64 x i8>
   %mask = bitcast <16 x i32> <i32 202182159, i32 134810123, i32 67438087, i32 66051, i32 202182159, i32 undef, i32 67438087, i32 66051, i32 202182159, i32 134810123, i32 67438087, i32 66051, i32 202182159, i32 134810123, i32 67438087, i32 66051> to <64 x i8>
   %res0 = call <64 x i8> @llvm.x86.avx512.mask.pshuf.b.512(<64 x i8> %x0, <64 x i8> %mask, <64 x i8> %select, i64 -1)
@@ -585,16 +527,16 @@ define <64 x i8> @combine_pshufb_identity(<64 x i8> %x0) {
   ret <64 x i8> %res1
 }
 define <64 x i8> @combine_pshufb_identity_mask(<64 x i8> %x0, i64 %m) {
-; X32-LABEL: combine_pshufb_identity_mask:
-; X32:       # %bb.0:
-; X32-NEXT:    vpternlogd $255, %zmm1, %zmm1, %zmm1
-; X32-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0,15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0,15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0,15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0]
-; X32-NEXT:    kmovq {{[0-9]+}}(%esp), %k1
-; X32-NEXT:    vpternlogd $255, %zmm3, %zmm3, %zmm3
-; X32-NEXT:    vpshufb %zmm2, %zmm0, %zmm3 {%k1}
-; X32-NEXT:    vpshufb %zmm2, %zmm3, %zmm1 {%k1}
-; X32-NEXT:    vmovdqa64 %zmm1, %zmm0
-; X32-NEXT:    retl
+; X86-LABEL: combine_pshufb_identity_mask:
+; X86:       # %bb.0:
+; X86-NEXT:    vpternlogd $255, %zmm1, %zmm1, %zmm1
+; X86-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0,15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0,15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0,15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0]
+; X86-NEXT:    kmovq {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vpternlogd $255, %zmm3, %zmm3, %zmm3
+; X86-NEXT:    vpshufb %zmm2, %zmm0, %zmm3 {%k1}
+; X86-NEXT:    vpshufb %zmm2, %zmm3, %zmm1 {%k1}
+; X86-NEXT:    vmovdqa64 %zmm1, %zmm0
+; X86-NEXT:    retl
 ;
 ; X64-LABEL: combine_pshufb_identity_mask:
 ; X64:       # %bb.0:
@@ -614,68 +556,48 @@ define <64 x i8> @combine_pshufb_identity_mask(<64 x i8> %x0, i64 %m) {
 }
 
 define <32 x i16> @combine_permvar_as_vpbroadcastw512(<32 x i16> %x0) {
-; X32-LABEL: combine_permvar_as_vpbroadcastw512:
-; X32:       # %bb.0:
-; X32-NEXT:    vpbroadcastw %xmm0, %zmm0
-; X32-NEXT:    retl
-;
-; X64-LABEL: combine_permvar_as_vpbroadcastw512:
-; X64:       # %bb.0:
-; X64-NEXT:    vpbroadcastw %xmm0, %zmm0
-; X64-NEXT:    retq
+; CHECK-LABEL: combine_permvar_as_vpbroadcastw512:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpbroadcastw %xmm0, %zmm0
+; CHECK-NEXT:    ret{{[l|q]}}
   %1 = call <32 x i16> @llvm.x86.avx512.permvar.hi.512(<32 x i16> %x0, <32 x i16> zeroinitializer)
   ret <32 x i16> %1
 }
 
 define <16 x i32> @combine_permvar_as_vpbroadcastd512(<16 x i32> %x0) {
-; X32-LABEL: combine_permvar_as_vpbroadcastd512:
-; X32:       # %bb.0:
-; X32-NEXT:    vbroadcastss %xmm0, %zmm0
-; X32-NEXT:    retl
-;
-; X64-LABEL: combine_permvar_as_vpbroadcastd512:
-; X64:       # %bb.0:
-; X64-NEXT:    vbroadcastss %xmm0, %zmm0
-; X64-NEXT:    retq
+; CHECK-LABEL: combine_permvar_as_vpbroadcastd512:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vbroadcastss %xmm0, %zmm0
+; CHECK-NEXT:    ret{{[l|q]}}
   %1 = call <16 x i32> @llvm.x86.avx512.permvar.si.512(<16 x i32> %x0, <16 x i32> zeroinitializer)
   ret <16 x i32> %1
 }
 
 define <8 x i64> @combine_permvar_as_vpbroadcastq512(<8 x i64> %x0) {
-; X32-LABEL: combine_permvar_as_vpbroadcastq512:
-; X32:       # %bb.0:
-; X32-NEXT:    vbroadcastsd %xmm0, %zmm0
-; X32-NEXT:    retl
-;
-; X64-LABEL: combine_permvar_as_vpbroadcastq512:
-; X64:       # %bb.0:
-; X64-NEXT:    vbroadcastsd %xmm0, %zmm0
-; X64-NEXT:    retq
+; CHECK-LABEL: combine_permvar_as_vpbroadcastq512:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vbroadcastsd %xmm0, %zmm0
+; CHECK-NEXT:    ret{{[l|q]}}
   %1 = call <8 x i64> @llvm.x86.avx512.permvar.di.512(<8 x i64> %x0, <8 x i64> zeroinitializer)
   ret <8 x i64> %1
 }
 
 define <8 x i64> @combine_permvar_8i64_as_permq(<8 x i64> %x0, <8 x i64> %x1) {
-; X32-LABEL: combine_permvar_8i64_as_permq:
-; X32:       # %bb.0:
-; X32-NEXT:    vpermpd {{.*#+}} zmm0 = zmm0[3,2,1,0,7,6,5,4]
-; X32-NEXT:    retl
-;
-; X64-LABEL: combine_permvar_8i64_as_permq:
-; X64:       # %bb.0:
-; X64-NEXT:    vpermpd {{.*#+}} zmm0 = zmm0[3,2,1,0,7,6,5,4]
-; X64-NEXT:    retq
+; CHECK-LABEL: combine_permvar_8i64_as_permq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpermpd {{.*#+}} zmm0 = zmm0[3,2,1,0,7,6,5,4]
+; CHECK-NEXT:    ret{{[l|q]}}
   %1 = call <8 x i64> @llvm.x86.avx512.permvar.di.512(<8 x i64> %x0, <8 x i64> <i64 3, i64 2, i64 1, i64 undef, i64 undef, i64 6, i64 5, i64 4>)
   ret <8 x i64> %1
 }
 define <8 x i64> @combine_permvar_8i64_as_permq_mask(<8 x i64> %x0, <8 x i64> %x1, i8 %m) {
-; X32-LABEL: combine_permvar_8i64_as_permq_mask:
-; X32:       # %bb.0:
-; X32-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    kmovd %eax, %k1
-; X32-NEXT:    vpermq {{.*#+}} zmm1 {%k1} = zmm0[3,2,1,0,7,6,5,4]
-; X32-NEXT:    vmovdqa64 %zmm1, %zmm0
-; X32-NEXT:    retl
+; X86-LABEL: combine_permvar_8i64_as_permq_mask:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovd %eax, %k1
+; X86-NEXT:    vpermq {{.*#+}} zmm1 {%k1} = zmm0[3,2,1,0,7,6,5,4]
+; X86-NEXT:    vmovdqa64 %zmm1, %zmm0
+; X86-NEXT:    retl
 ;
 ; X64-LABEL: combine_permvar_8i64_as_permq_mask:
 ; X64:       # %bb.0:
@@ -690,26 +612,21 @@ define <8 x i64> @combine_permvar_8i64_as_permq_mask(<8 x i64> %x0, <8 x i64> %x
 }
 
 define <8 x double> @combine_permvar_8f64_as_permpd(<8 x double> %x0, <8 x double> %x1) {
-; X32-LABEL: combine_permvar_8f64_as_permpd:
-; X32:       # %bb.0:
-; X32-NEXT:    vpermpd {{.*#+}} zmm0 = zmm0[3,2,1,0,7,6,5,4]
-; X32-NEXT:    retl
-;
-; X64-LABEL: combine_permvar_8f64_as_permpd:
-; X64:       # %bb.0:
-; X64-NEXT:    vpermpd {{.*#+}} zmm0 = zmm0[3,2,1,0,7,6,5,4]
-; X64-NEXT:    retq
+; CHECK-LABEL: combine_permvar_8f64_as_permpd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpermpd {{.*#+}} zmm0 = zmm0[3,2,1,0,7,6,5,4]
+; CHECK-NEXT:    ret{{[l|q]}}
   %1 = call <8 x double> @llvm.x86.avx512.permvar.df.512(<8 x double> %x0, <8 x i64> <i64 3, i64 2, i64 1, i64 undef, i64 undef, i64 6, i64 5, i64 4>)
   ret <8 x double> %1
 }
 define <8 x double> @combine_permvar_8f64_as_permpd_mask(<8 x double> %x0, <8 x double> %x1, i8 %m) {
-; X32-LABEL: combine_permvar_8f64_as_permpd_mask:
-; X32:       # %bb.0:
-; X32-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    kmovd %eax, %k1
-; X32-NEXT:    vpermpd {{.*#+}} zmm1 {%k1} = zmm0[3,2,1,0,7,6,5,4]
-; X32-NEXT:    vmovapd %zmm1, %zmm0
-; X32-NEXT:    retl
+; X86-LABEL: combine_permvar_8f64_as_permpd_mask:
+; X86:       # %bb.0:
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    kmovd %eax, %k1
+; X86-NEXT:    vpermpd {{.*#+}} zmm1 {%k1} = zmm0[3,2,1,0,7,6,5,4]
+; X86-NEXT:    vmovapd %zmm1, %zmm0
+; X86-NEXT:    retl
 ;
 ; X64-LABEL: combine_permvar_8f64_as_permpd_mask:
 ; X64:       # %bb.0:
@@ -724,39 +641,29 @@ define <8 x double> @combine_permvar_8f64_as_permpd_mask(<8 x double> %x0, <8 x
 }
 
 define <16 x float> @combine_vpermilvar_16f32_230146759A8BCFDE(<16 x float> %x0) {
-; X32-LABEL: combine_vpermilvar_16f32_230146759A8BCFDE:
-; X32:       # %bb.0:
-; X32-NEXT:    vpermilps {{.*#+}} zmm0 = zmm0[2,3,0,1,4,6,7,5,9,10,8,11,12,15,13,14]
-; X32-NEXT:    retl
-;
-; X64-LABEL: combine_vpermilvar_16f32_230146759A8BCFDE:
-; X64:       # %bb.0:
-; X64-NEXT:    vpermilps {{.*#+}} zmm0 = zmm0[2,3,0,1,4,6,7,5,9,10,8,11,12,15,13,14]
-; X64-NEXT:    retq
+; CHECK-LABEL: combine_vpermilvar_16f32_230146759A8BCFDE:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpermilps {{.*#+}} zmm0 = zmm0[2,3,0,1,4,6,7,5,9,10,8,11,12,15,13,14]
+; CHECK-NEXT:    ret{{[l|q]}}
   %res0 = call <16 x float> @llvm.x86.avx512.mask.vpermilvar.ps.512(<16 x float> %x0, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 3, i32 2, i32 1, i32 0, i32 2, i32 3, i32 0, i32 1, i32 1, i32 0, i32 3, i32 2>, <16 x float> undef, i16 -1)
   %res1 = call <16 x float> @llvm.x86.avx512.mask.vpermilvar.ps.512(<16 x float> %res0, <16 x i32> <i32 2, i32 3, i32 0, i32 1, i32 3, i32 1, i32 0, i32 2, i32 3, i32 0, i32 2, i32 1, i32 1, i32 2, i32 0, i32 3>, <16 x float> undef, i16 -1)
   ret <16 x float> %res1
 }
 
 define <64 x i8> @combine_pshufb_as_pslldq(<64 x i8> %a0) {
-; X32-LABEL: combine_pshufb_as_pslldq:
-; X32:       # %bb.0:
-; X32-NEXT:    vpshufb {{.*#+}} zmm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zmm0[0,1,2,3,4,5],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zmm0[16,17,18,19,20,21],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zmm0[32,33,34,35,36,37],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zmm0[48,49,50,51,52,53]
-; X32-NEXT:    retl
-;
-; X64-LABEL: combine_pshufb_as_pslldq:
-; X64:       # %bb.0:
-; X64-NEXT:    vpshufb {{.*#+}} zmm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zmm0[0,1,2,3,4,5],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zmm0[16,17,18,19,20,21],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zmm0[32,33,34,35,36,37],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zmm0[48,49,50,51,52,53]
-; X64-NEXT:    retq
+; CHECK-LABEL: combine_pshufb_as_pslldq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpshufb {{.*#+}} zmm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zmm0[0,1,2,3,4,5],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zmm0[16,17,18,19,20,21],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zmm0[32,33,34,35,36,37],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zmm0[48,49,50,51,52,53]
+; CHECK-NEXT:    ret{{[l|q]}}
   %res0 = call <64 x i8> @llvm.x86.avx512.mask.pshuf.b.512(<64 x i8> %a0, <64 x i8> <i8 128, i8 128, i8 128, i8 128, i8 128, i8 128, i8 128, i8 128, i8 128, i8 128, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 128, i8 128, i8 128, i8 128, i8 128, i8 128, i8 128, i8 128, i8 128, i8 128, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 128, i8 128, i8 128, i8 128, i8 128, i8 128, i8 128, i8 128, i8 128, i8 128, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 128, i8 128, i8 128, i8 128, i8 128, i8 128, i8 128, i8 128, i8 128, i8 128, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5>, <64 x i8> undef, i64 -1)
   ret <64 x i8> %res0
 }
 define <64 x i8> @combine_pshufb_as_pslldq_mask(<64 x i8> %a0, i64 %m) {
-; X32-LABEL: combine_pshufb_as_pslldq_mask:
-; X32:       # %bb.0:
-; X32-NEXT:    kmovq {{[0-9]+}}(%esp), %k1
-; X32-NEXT:    vpshufb {{.*#+}} zmm0 {%k1} {z} = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zmm0[0,1,2,3,4,5],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zmm0[16,17,18,19,20,21],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zmm0[32,33,34,35,36,37],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zmm0[48,49,50,51,52,53]
-; X32-NEXT:    retl
+; X86-LABEL: combine_pshufb_as_pslldq_mask:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovq {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vpshufb {{.*#+}} zmm0 {%k1} {z} = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zmm0[0,1,2,3,4,5],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zmm0[16,17,18,19,20,21],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zmm0[32,33,34,35,36,37],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zmm0[48,49,50,51,52,53]
+; X86-NEXT:    retl
 ;
 ; X64-LABEL: combine_pshufb_as_pslldq_mask:
 ; X64:       # %bb.0:
@@ -768,24 +675,19 @@ define <64 x i8> @combine_pshufb_as_pslldq_mask(<64 x i8> %a0, i64 %m) {
 }
 
 define <64 x i8> @combine_pshufb_as_psrldq(<64 x i8> %a0) {
-; X32-LABEL: combine_pshufb_as_psrldq:
-; X32:       # %bb.0:
-; X32-NEXT:    vpshufb {{.*#+}} zmm0 = zmm0[15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zmm0[31],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zmm0[47],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zmm0[63],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; X32-NEXT:    retl
-;
-; X64-LABEL: combine_pshufb_as_psrldq:
-; X64:       # %bb.0:
-; X64-NEXT:    vpshufb {{.*#+}} zmm0 = zmm0[15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zmm0[31],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zmm0[47],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zmm0[63],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; X64-NEXT:    retq
+; CHECK-LABEL: combine_pshufb_as_psrldq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpshufb {{.*#+}} zmm0 = zmm0[15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zmm0[31],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zmm0[47],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zmm0[63],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; CHECK-NEXT:    ret{{[l|q]}}
   %res0 = call <64 x i8> @llvm.x86.avx512.mask.pshuf.b.512(<64 x i8> %a0, <64 x i8> <i8 15, i8 128, i8 128, i8 128, i8 128, i8 128, i8 128, i8 128, i8 128, i8 128, i8 128, i8 128, i8 128, i8 128, i8 128, i8 128, i8 15, i8 128, i8 128, i8 128, i8 128, i8 128, i8 128, i8 128, i8 128, i8 128, i8 128, i8 128, i8 128, i8 128, i8 128, i8 128, i8 15, i8 128, i8 128, i8 128, i8 128, i8 128, i8 128, i8 128, i8 128, i8 128, i8 128, i8 128, i8 128, i8 128, i8 128, i8 128, i8 15, i8 128, i8 128, i8 128, i8 128, i8 128, i8 128, i8 128, i8 128, i8 128, i8 128, i8 128, i8 128, i8 128, i8 128, i8 128>, <64 x i8> undef, i64 -1)
   ret <64 x i8> %res0
 }
 define <64 x i8> @combine_pshufb_as_psrldq_mask(<64 x i8> %a0, i64 %m) {
-; X32-LABEL: combine_pshufb_as_psrldq_mask:
-; X32:       # %bb.0:
-; X32-NEXT:    kmovq {{[0-9]+}}(%esp), %k1
-; X32-NEXT:    vpshufb {{.*#+}} zmm0 {%k1} {z} = zmm0[15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zmm0[31],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zmm0[47],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zmm0[63],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; X32-NEXT:    retl
+; X86-LABEL: combine_pshufb_as_psrldq_mask:
+; X86:       # %bb.0:
+; X86-NEXT:    kmovq {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vpshufb {{.*#+}} zmm0 {%k1} {z} = zmm0[15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zmm0[31],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zmm0[47],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zmm0[63],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; X86-NEXT:    retl
 ;
 ; X64-LABEL: combine_pshufb_as_psrldq_mask:
 ; X64:       # %bb.0:
@@ -797,161 +699,145 @@ define <64 x i8> @combine_pshufb_as_psrldq_mask(<64 x i8> %a0, i64 %m) {
 }
 
 define <32 x i16> @combine_permvar_as_pshuflw(<32 x i16> %a0) {
-; X32-LABEL: combine_permvar_as_pshuflw:
-; X32:       # %bb.0:
-; X32-NEXT:    vpshuflw {{.*#+}} zmm0 = zmm0[1,0,3,2,4,5,6,7,9,8,11,10,12,13,14,15,17,16,19,18,20,21,22,23,25,24,27,26,28,29,30,31]
-; X32-NEXT:    retl
-;
-; X64-LABEL: combine_permvar_as_pshuflw:
-; X64:       # %bb.0:
-; X64-NEXT:    vpshuflw {{.*#+}} zmm0 = zmm0[1,0,3,2,4,5,6,7,9,8,11,10,12,13,14,15,17,16,19,18,20,21,22,23,25,24,27,26,28,29,30,31]
-; X64-NEXT:    retq
+; CHECK-LABEL: combine_permvar_as_pshuflw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpshuflw {{.*#+}} zmm0 = zmm0[1,0,3,2,4,5,6,7,9,8,11,10,12,13,14,15,17,16,19,18,20,21,22,23,25,24,27,26,28,29,30,31]
+; CHECK-NEXT:    ret{{[l|q]}}
   %1 = call <32 x i16> @llvm.x86.avx512.permvar.hi.512(<32 x i16> %a0, <32 x i16> <i16 1, i16 0, i16 3, i16 2, i16 4, i16 5, i16 6, i16 7, i16 9, i16 8, i16 11, i16 10, i16 12, i16 13, i16 14, i16 15, i16 17, i16 16, i16 19, i16 18, i16 20, i16 21, i16 22, i16 23, i16 25, i16 24, i16 27, i16 26, i16 28, i16 29, i16 30, i16 31>)
   ret <32 x i16> %1
 }
 
 define <32 x i16> @combine_pshufb_as_pshufhw(<32 x i16> %a0) {
-; X32-LABEL: combine_pshufb_as_pshufhw:
-; X32:       # %bb.0:
-; X32-NEXT:    vpshufhw {{.*#+}} zmm0 = zmm0[0,1,2,3,5,4,7,6,8,9,10,11,13,12,15,14,16,17,18,19,21,20,23,22,24,25,26,27,29,28,31,30]
-; X32-NEXT:    retl
-;
-; X64-LABEL: combine_pshufb_as_pshufhw:
-; X64:       # %bb.0:
-; X64-NEXT:    vpshufhw {{.*#+}} zmm0 = zmm0[0,1,2,3,5,4,7,6,8,9,10,11,13,12,15,14,16,17,18,19,21,20,23,22,24,25,26,27,29,28,31,30]
-; X64-NEXT:    retq
+; CHECK-LABEL: combine_pshufb_as_pshufhw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpshufhw {{.*#+}} zmm0 = zmm0[0,1,2,3,5,4,7,6,8,9,10,11,13,12,15,14,16,17,18,19,21,20,23,22,24,25,26,27,29,28,31,30]
+; CHECK-NEXT:    ret{{[l|q]}}
   %1 = call <32 x i16> @llvm.x86.avx512.permvar.hi.512(<32 x i16> %a0, <32 x i16> <i16 0, i16 1, i16 2, i16 3, i16 5, i16 4, i16 7, i16 6, i16 8, i16 9, i16 10, i16 11, i16 13, i16 12, i16 15, i16 14, i16 16, i16 17, i16 18, i16 19, i16 21, i16 20, i16 23, i16 22, i16 24, i16 25, i16 26, i16 27, i16 29, i16 28, i16 31, i16 30>)
   ret <32 x i16> %1
 }
 
+define <64 x i8> @combine_pshufb_as_packsswb(<32 x i16> %a0, <32 x i16> %a1) nounwind {
+; CHECK-LABEL: combine_pshufb_as_packsswb:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpsraw $11, %zmm0, %zmm0
+; CHECK-NEXT:    vpsraw $11, %zmm1, %zmm1
+; CHECK-NEXT:    vpacksswb %zmm1, %zmm0, %zmm0
+; CHECK-NEXT:    ret{{[l|q]}}
+  %1 = ashr <32 x i16> %a0, <i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11>
+  %2 = ashr <32 x i16> %a1, <i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11>
+  %3 = bitcast <32 x i16> %1 to <64 x i8>
+  %4 = bitcast <32 x i16> %2 to <64 x i8>
+  %5 = tail call <64 x i8> @llvm.x86.avx512.mask.pshuf.b.512(<64 x i8> %3, <64 x i8> <i8 0, i8 2, i8 4, i8 6, i8 8, i8 10, i8 12, i8 14, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 0, i8 2, i8 4, i8 6, i8 8, i8 10, i8 12, i8 14, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 0, i8 2, i8 4, i8 6, i8 8, i8 10, i8 12, i8 14, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 0, i8 2, i8 4, i8 6, i8 8, i8 10, i8 12, i8 14, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>, <64 x i8> undef, i64 -1)
+  %6 = tail call <64 x i8> @llvm.x86.avx512.mask.pshuf.b.512(<64 x i8> %4, <64 x i8> <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 0, i8 2, i8 4, i8 6, i8 8, i8 10, i8 12, i8 14, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 0, i8 2, i8 4, i8 6, i8 8, i8 10, i8 12, i8 14, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 0, i8 2, i8 4, i8 6, i8 8, i8 10, i8 12, i8 14, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 0, i8 2, i8 4, i8 6, i8 8, i8 10, i8 12, i8 14>, <64 x i8> undef, i64 -1)
+  %7 = or <64 x i8> %5, %6
+  ret <64 x i8> %7
+}
+
+define <64 x i8> @combine_pshufb_as_packuswb(<32 x i16> %a0, <32 x i16> %a1) nounwind {
+; CHECK-LABEL: combine_pshufb_as_packuswb:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpsrlw $11, %zmm0, %zmm0
+; CHECK-NEXT:    vpsrlw $11, %zmm1, %zmm1
+; CHECK-NEXT:    vpackuswb %zmm1, %zmm0, %zmm0
+; CHECK-NEXT:    ret{{[l|q]}}
+  %1 = lshr <32 x i16> %a0, <i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11>
+  %2 = lshr <32 x i16> %a1, <i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11, i16 11>
+  %3 = bitcast <32 x i16> %1 to <64 x i8>
+  %4 = bitcast <32 x i16> %2 to <64 x i8>
+  %5 = tail call <64 x i8> @llvm.x86.avx512.mask.pshuf.b.512(<64 x i8> %3, <64 x i8> <i8 0, i8 2, i8 4, i8 6, i8 8, i8 10, i8 12, i8 14, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 0, i8 2, i8 4, i8 6, i8 8, i8 10, i8 12, i8 14, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 0, i8 2, i8 4, i8 6, i8 8, i8 10, i8 12, i8 14, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 0, i8 2, i8 4, i8 6, i8 8, i8 10, i8 12, i8 14, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>, <64 x i8> undef, i64 -1)
+  %6 = tail call <64 x i8> @llvm.x86.avx512.mask.pshuf.b.512(<64 x i8> %4, <64 x i8> <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 0, i8 2, i8 4, i8 6, i8 8, i8 10, i8 12, i8 14, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 0, i8 2, i8 4, i8 6, i8 8, i8 10, i8 12, i8 14, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 0, i8 2, i8 4, i8 6, i8 8, i8 10, i8 12, i8 14, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 0, i8 2, i8 4, i8 6, i8 8, i8 10, i8 12, i8 14>, <64 x i8> undef, i64 -1)
+  %7 = or <64 x i8> %5, %6
+  ret <64 x i8> %7
+}
+
 define <32 x i16> @combine_vpermi2var_32i16_as_pshufb(<32 x i16> %a0) {
-; X32-LABEL: combine_vpermi2var_32i16_as_pshufb:
-; X32:       # %bb.0:
-; X32-NEXT:    vpshufb {{.*#+}} zmm0 = zmm0[2,3,0,1,6,7,4,5,10,11,8,9,14,15,12,13,18,19,16,17,22,23,20,21,26,27,24,25,30,31,28,29,34,35,32,33,38,39,36,37,42,43,40,41,46,47,44,45,50,51,48,49,54,55,52,53,58,59,56,57,62,63,60,61]
-; X32-NEXT:    retl
-;
-; X64-LABEL: combine_vpermi2var_32i16_as_pshufb:
-; X64:       # %bb.0:
-; X64-NEXT:    vpshufb {{.*#+}} zmm0 = zmm0[2,3,0,1,6,7,4,5,10,11,8,9,14,15,12,13,18,19,16,17,22,23,20,21,26,27,24,25,30,31,28,29,34,35,32,33,38,39,36,37,42,43,40,41,46,47,44,45,50,51,48,49,54,55,52,53,58,59,56,57,62,63,60,61]
-; X64-NEXT:    retq
+; CHECK-LABEL: combine_vpermi2var_32i16_as_pshufb:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpshufb {{.*#+}} zmm0 = zmm0[2,3,0,1,6,7,4,5,10,11,8,9,14,15,12,13,18,19,16,17,22,23,20,21,26,27,24,25,30,31,28,29,34,35,32,33,38,39,36,37,42,43,40,41,46,47,44,45,50,51,48,49,54,55,52,53,58,59,56,57,62,63,60,61]
+; CHECK-NEXT:    ret{{[l|q]}}
   %1 = call <32 x i16> @llvm.x86.avx512.permvar.hi.512(<32 x i16> %a0, <32 x i16> <i16 1, i16 0, i16 3, i16 2, i16 4, i16 5, i16 6, i16 7, i16 9, i16 8, i16 11, i16 10, i16 12, i16 13, i16 14, i16 15, i16 17, i16 16, i16 19, i16 18, i16 20, i16 21, i16 22, i16 23, i16 25, i16 24, i16 27, i16 26, i16 28, i16 29, i16 30, i16 31>)
   %2 = call <32 x i16> @llvm.x86.avx512.permvar.hi.512(<32 x i16> %1, <32 x i16> <i16 0, i16 1, i16 2, i16 3, i16 5, i16 4, i16 7, i16 6, i16 8, i16 9, i16 10, i16 11, i16 13, i16 12, i16 15, i16 14, i16 16, i16 17, i16 18, i16 19, i16 21, i16 20, i16 23, i16 22, i16 24, i16 25, i16 26, i16 27, i16 29, i16 28, i16 31, i16 30>)
   ret <32 x i16> %2
 }
 
 define <8 x double> @combine_vpermi2var_8f64_identity(<8 x double> %x0, <8 x double> %x1) {
-; X32-LABEL: combine_vpermi2var_8f64_identity:
-; X32:       # %bb.0:
-; X32-NEXT:    retl
-;
-; X64-LABEL: combine_vpermi2var_8f64_identity:
-; X64:       # %bb.0:
-; X64-NEXT:    retq
+; CHECK-LABEL: combine_vpermi2var_8f64_identity:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ret{{[l|q]}}
   %res0 = call <8 x double> @llvm.x86.avx512.mask.vpermi2var.pd.512(<8 x double> %x0, <8 x i64> <i64 7, i64 6, i64 5, i64 4, i64 3, i64 2, i64 1, i64 0>, <8 x double> %x1, i8 -1)
   %res1 = call <8 x double> @llvm.x86.avx512.mask.vpermi2var.pd.512(<8 x double> %res0, <8 x i64> <i64 7, i64 14, i64 5, i64 12, i64 3, i64 10, i64 1, i64 8>, <8 x double> %res0, i8 -1)
   ret <8 x double> %res1
 }
 
 define <8 x double> @combine_vpermi2var_8f64_as_shufpd(<8 x double> %x0, <8 x double> %x1) {
-; X32-LABEL: combine_vpermi2var_8f64_as_shufpd:
-; X32:       # %bb.0:
-; X32-NEXT:    vshufpd {{.*#+}} zmm0 = zmm0[1],zmm1[0],zmm0[2],zmm1[2],zmm0[5],zmm1[5],zmm0[6],zmm1[7]
-; X32-NEXT:    retl
-;
-; X64-LABEL: combine_vpermi2var_8f64_as_shufpd:
-; X64:       # %bb.0:
-; X64-NEXT:    vshufpd {{.*#+}} zmm0 = zmm0[1],zmm1[0],zmm0[2],zmm1[2],zmm0[5],zmm1[5],zmm0[6],zmm1[7]
-; X64-NEXT:    retq
+; CHECK-LABEL: combine_vpermi2var_8f64_as_shufpd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vshufpd {{.*#+}} zmm0 = zmm0[1],zmm1[0],zmm0[2],zmm1[2],zmm0[5],zmm1[5],zmm0[6],zmm1[7]
+; CHECK-NEXT:    ret{{[l|q]}}
   %1 = call <8 x double> @llvm.x86.avx512.mask.vpermi2var.pd.512(<8 x double> %x0, <8 x i64> <i64 1, i64 8, i64 2, i64 10, i64 5, i64 13, i64 6, i64 15>, <8 x double> %x1, i8 -1)
   ret <8 x double> %1
 }
 
 define <8 x i64> @combine_vpermi2var_8i64_identity(<8 x i64> %x0, <8 x i64> %x1) {
-; X32-LABEL: combine_vpermi2var_8i64_identity:
-; X32:       # %bb.0:
-; X32-NEXT:    retl
-;
-; X64-LABEL: combine_vpermi2var_8i64_identity:
-; X64:       # %bb.0:
-; X64-NEXT:    retq
+; CHECK-LABEL: combine_vpermi2var_8i64_identity:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ret{{[l|q]}}
   %res0 = call <8 x i64> @llvm.x86.avx512.mask.vpermi2var.q.512(<8 x i64> %x0, <8 x i64> <i64 undef, i64 6, i64 5, i64 4, i64 3, i64 2, i64 1, i64 0>, <8 x i64> %x1, i8 -1)
   %res1 = call <8 x i64> @llvm.x86.avx512.mask.vpermi2var.q.512(<8 x i64> %res0, <8 x i64> <i64 undef, i64 14, i64 5, i64 12, i64 3, i64 10, i64 1, i64 8>, <8 x i64> %res0, i8 -1)
   ret <8 x i64> %res1
 }
 
 define <16 x float> @combine_vpermi2var_16f32_identity(<16 x float> %x0, <16 x float> %x1) {
-; X32-LABEL: combine_vpermi2var_16f32_identity:
-; X32:       # %bb.0:
-; X32-NEXT:    retl
-;
-; X64-LABEL: combine_vpermi2var_16f32_identity:
-; X64:       # %bb.0:
-; X64-NEXT:    retq
+; CHECK-LABEL: combine_vpermi2var_16f32_identity:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ret{{[l|q]}}
   %res0 = call <16 x float> @llvm.x86.avx512.mask.vpermi2var.ps.512(<16 x float> %x0, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>, <16 x float> %x1, i16 -1)
   %res1 = call <16 x float> @llvm.x86.avx512.mask.vpermi2var.ps.512(<16 x float> %res0, <16 x i32> <i32 15, i32 30, i32 13, i32 28, i32 11, i32 26, i32 9, i32 24, i32 7, i32 22, i32 5, i32 20, i32 3, i32 18, i32 1, i32 16>, <16 x float> %res0, i16 -1)
   ret <16 x float> %res1
 }
 
 define <16 x i32> @combine_vpermi2var_16i32_identity(<16 x i32> %x0, <16 x i32> %x1) {
-; X32-LABEL: combine_vpermi2var_16i32_identity:
-; X32:       # %bb.0:
-; X32-NEXT:    retl
-;
-; X64-LABEL: combine_vpermi2var_16i32_identity:
-; X64:       # %bb.0:
-; X64-NEXT:    retq
+; CHECK-LABEL: combine_vpermi2var_16i32_identity:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ret{{[l|q]}}
   %res0 = call <16 x i32> @llvm.x86.avx512.mask.vpermi2var.d.512(<16 x i32> %x0, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 undef>, <16 x i32> %x1, i16 -1)
   %res1 = call <16 x i32> @llvm.x86.avx512.mask.vpermi2var.d.512(<16 x i32> %res0, <16 x i32> <i32 15, i32 30, i32 13, i32 28, i32 undef, i32 26, i32 9, i32 24, i32 7, i32 22, i32 5, i32 20, i32 3, i32 18, i32 1, i32 16>, <16 x i32> %res0, i16 -1)
   ret <16 x i32> %res1
 }
 
 define <16 x float> @combine_vpermt2var_vpermi2var_16f32_as_unpckhps(<16 x float> %a0, <16 x float> %a1) {
-; X32-LABEL: combine_vpermt2var_vpermi2var_16f32_as_unpckhps:
-; X32:       # %bb.0:
-; X32-NEXT:    vunpckhps {{.*#+}} zmm0 = zmm1[2],zmm0[2],zmm1[3],zmm0[3],zmm1[6],zmm0[6],zmm1[7],zmm0[7],zmm1[10],zmm0[10],zmm1[11],zmm0[11],zmm1[14],zmm0[14],zmm1[15],zmm0[15]
-; X32-NEXT:    retl
-;
-; X64-LABEL: combine_vpermt2var_vpermi2var_16f32_as_unpckhps:
-; X64:       # %bb.0:
-; X64-NEXT:    vunpckhps {{.*#+}} zmm0 = zmm1[2],zmm0[2],zmm1[3],zmm0[3],zmm1[6],zmm0[6],zmm1[7],zmm0[7],zmm1[10],zmm0[10],zmm1[11],zmm0[11],zmm1[14],zmm0[14],zmm1[15],zmm0[15]
-; X64-NEXT:    retq
+; CHECK-LABEL: combine_vpermt2var_vpermi2var_16f32_as_unpckhps:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vunpckhps {{.*#+}} zmm0 = zmm1[2],zmm0[2],zmm1[3],zmm0[3],zmm1[6],zmm0[6],zmm1[7],zmm0[7],zmm1[10],zmm0[10],zmm1[11],zmm0[11],zmm1[14],zmm0[14],zmm1[15],zmm0[15]
+; CHECK-NEXT:    ret{{[l|q]}}
   %res0 = call <16 x float> @llvm.x86.avx512.mask.vpermi2var.ps.512(<16 x float> %a0, <16 x i32> <i32 18, i32 2, i32 19, i32 3, i32 22, i32 6, i32 23, i32 7, i32 26, i32 10, i32 27, i32 11, i32 30, i32 14, i32 31, i32 15>, <16 x float> %a1, i16 -1)
   ret <16 x float> %res0
 }
 
 define <16 x i32> @vpermt2var_vpermi2var_16i32_as_unpckldq(<16 x i32> %a0, <16 x i32> %a1) {
-; X32-LABEL: vpermt2var_vpermi2var_16i32_as_unpckldq:
-; X32:       # %bb.0:
-; X32-NEXT:    vunpcklps {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[1],zmm1[1],zmm0[4],zmm1[4],zmm0[5],zmm1[5],zmm0[8],zmm1[8],zmm0[9],zmm1[9],zmm0[12],zmm1[12],zmm0[13],zmm1[13]
-; X32-NEXT:    retl
-;
-; X64-LABEL: vpermt2var_vpermi2var_16i32_as_unpckldq:
-; X64:       # %bb.0:
-; X64-NEXT:    vunpcklps {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[1],zmm1[1],zmm0[4],zmm1[4],zmm0[5],zmm1[5],zmm0[8],zmm1[8],zmm0[9],zmm1[9],zmm0[12],zmm1[12],zmm0[13],zmm1[13]
-; X64-NEXT:    retq
+; CHECK-LABEL: vpermt2var_vpermi2var_16i32_as_unpckldq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vunpcklps {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[1],zmm1[1],zmm0[4],zmm1[4],zmm0[5],zmm1[5],zmm0[8],zmm1[8],zmm0[9],zmm1[9],zmm0[12],zmm1[12],zmm0[13],zmm1[13]
+; CHECK-NEXT:    ret{{[l|q]}}
   %res0 = call <16 x i32> @llvm.x86.avx512.mask.vpermi2var.d.512(<16 x i32> %a0, <16 x i32> <i32 0, i32 16, i32 1, i32 17, i32 4, i32 20, i32 5, i32 21, i32 8, i32 24, i32 9, i32 25, i32 12, i32 28, i32 13, i32 29>, <16 x i32> %a1, i16 -1)
   ret <16 x i32> %res0
 }
 
 define <32 x i16> @combine_vpermi2var_32i16_identity(<32 x i16> %x0, <32 x i16> %x1) {
-; X32-LABEL: combine_vpermi2var_32i16_identity:
-; X32:       # %bb.0:
-; X32-NEXT:    retl
-;
-; X64-LABEL: combine_vpermi2var_32i16_identity:
-; X64:       # %bb.0:
-; X64-NEXT:    retq
+; CHECK-LABEL: combine_vpermi2var_32i16_identity:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ret{{[l|q]}}
   %res0 = call <32 x i16> @llvm.x86.avx512.mask.vpermi2var.hi.512(<32 x i16> %x0, <32 x i16> <i16 31, i16 30, i16 29, i16 28, i16 27, i16 26, i16 25, i16 24, i16 23, i16 22, i16 21, i16 20, i16 19, i16 18, i16 17, i16 16, i16 15, i16 14, i16 13, i16 12, i16 11, i16 10, i16 9, i16 8, i16 7, i16 6, i16 5, i16 4, i16 3, i16 2, i16 1, i16 0>, <32 x i16> %x1, i32 -1)
   %res1 = call <32 x i16> @llvm.x86.avx512.mask.vpermi2var.hi.512(<32 x i16> %res0, <32 x i16> <i16 63, i16 30, i16 61, i16 28, i16 59, i16 26, i16 57, i16 24, i16 55, i16 22, i16 53, i16 20, i16 51, i16 18, i16 49, i16 16, i16 47, i16 46, i16 13, i16 44, i16 11, i16 42, i16 9, i16 40, i16 7, i16 38, i16 5, i16 36, i16 3, i16 34, i16 1, i16 32>, <32 x i16> %res0, i32 -1)
   ret <32 x i16> %res1
 }
 
 define <8 x double> @combine_vpermi2var_8f64_as_vpermpd(<8 x double> %x0, <8 x double> %x1) {
-; X32-LABEL: combine_vpermi2var_8f64_as_vpermpd:
-; X32:       # %bb.0:
-; X32-NEXT:    vmovaps {{.*#+}} zmm1 = [7,0,6,0,5,0,4,0,3,0,2,0,1,0,0,0]
-; X32-NEXT:    vpermpd %zmm0, %zmm1, %zmm0
-; X32-NEXT:    retl
+; X86-LABEL: combine_vpermi2var_8f64_as_vpermpd:
+; X86:       # %bb.0:
+; X86-NEXT:    vmovaps {{.*#+}} zmm1 = [7,0,6,0,5,0,4,0,3,0,2,0,1,0,0,0]
+; X86-NEXT:    vpermpd %zmm0, %zmm1, %zmm0
+; X86-NEXT:    retl
 ;
 ; X64-LABEL: combine_vpermi2var_8f64_as_vpermpd:
 ; X64:       # %bb.0:
@@ -964,11 +850,11 @@ define <8 x double> @combine_vpermi2var_8f64_as_vpermpd(<8 x double> %x0, <8 x d
 }
 
 define <8 x i64> @combine_vpermt2var_8i64_as_vpermq(<8 x i64> %x0, <8 x i64> %x1) {
-; X32-LABEL: combine_vpermt2var_8i64_as_vpermq:
-; X32:       # %bb.0:
-; X32-NEXT:    vmovaps {{.*#+}} zmm1 = [7,0,6,0,5,0,4,0,3,0,2,0,1,0,0,0]
-; X32-NEXT:    vpermpd %zmm0, %zmm1, %zmm0
-; X32-NEXT:    retl
+; X86-LABEL: combine_vpermt2var_8i64_as_vpermq:
+; X86:       # %bb.0:
+; X86-NEXT:    vmovaps {{.*#+}} zmm1 = [7,0,6,0,5,0,4,0,3,0,2,0,1,0,0,0]
+; X86-NEXT:    vpermpd %zmm0, %zmm1, %zmm0
+; X86-NEXT:    retl
 ;
 ; X64-LABEL: combine_vpermt2var_8i64_as_vpermq:
 ; X64:       # %bb.0:
@@ -981,63 +867,45 @@ define <8 x i64> @combine_vpermt2var_8i64_as_vpermq(<8 x i64> %x0, <8 x i64> %x1
 }
 
 define <16 x float> @combine_vpermi2var_16f32_as_vpermps(<16 x float> %x0, <16 x float> %x1) {
-; X32-LABEL: combine_vpermi2var_16f32_as_vpermps:
-; X32:       # %bb.0:
-; X32-NEXT:    vmovaps {{.*#+}} zmm1 = [7,7,5,5,3,3,1,1,15,15,13,13,11,11,9,9]
-; X32-NEXT:    vpermps %zmm0, %zmm1, %zmm0
-; X32-NEXT:    retl
-;
-; X64-LABEL: combine_vpermi2var_16f32_as_vpermps:
-; X64:       # %bb.0:
-; X64-NEXT:    vmovaps {{.*#+}} zmm1 = [7,7,5,5,3,3,1,1,15,15,13,13,11,11,9,9]
-; X64-NEXT:    vpermps %zmm0, %zmm1, %zmm0
-; X64-NEXT:    retq
+; CHECK-LABEL: combine_vpermi2var_16f32_as_vpermps:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps {{.*#+}} zmm1 = [7,7,5,5,3,3,1,1,15,15,13,13,11,11,9,9]
+; CHECK-NEXT:    vpermps %zmm0, %zmm1, %zmm0
+; CHECK-NEXT:    ret{{[l|q]}}
   %res0 = call <16 x float> @llvm.x86.avx512.mask.vpermi2var.ps.512(<16 x float> %x0, <16 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8>, <16 x float> %x1, i16 -1)
   %res1 = call <16 x float> @llvm.x86.avx512.mask.vpermi2var.ps.512(<16 x float> %res0, <16 x i32> <i32 0, i32 16, i32 2, i32 18, i32 4, i32 20, i32 6, i32 22, i32 8, i32 24, i32 10, i32 26, i32 12, i32 28, i32 14, i32 30>, <16 x float> %res0, i16 -1)
   ret <16 x float> %res1
 }
 
 define <16 x i32> @combine_vpermt2var_16i32_as_vpermd(<16 x i32> %x0, <16 x i32> %x1) {
-; X32-LABEL: combine_vpermt2var_16i32_as_vpermd:
-; X32:       # %bb.0:
-; X32-NEXT:    vmovaps {{.*#+}} zmm1 = [7,7,5,5,3,3,1,1,15,15,13,13,11,11,9,9]
-; X32-NEXT:    vpermps %zmm0, %zmm1, %zmm0
-; X32-NEXT:    retl
-;
-; X64-LABEL: combine_vpermt2var_16i32_as_vpermd:
-; X64:       # %bb.0:
-; X64-NEXT:    vmovaps {{.*#+}} zmm1 = [7,7,5,5,3,3,1,1,15,15,13,13,11,11,9,9]
-; X64-NEXT:    vpermps %zmm0, %zmm1, %zmm0
-; X64-NEXT:    retq
+; CHECK-LABEL: combine_vpermt2var_16i32_as_vpermd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps {{.*#+}} zmm1 = [7,7,5,5,3,3,1,1,15,15,13,13,11,11,9,9]
+; CHECK-NEXT:    vpermps %zmm0, %zmm1, %zmm0
+; CHECK-NEXT:    ret{{[l|q]}}
   %res0 = call <16 x i32> @llvm.x86.avx512.maskz.vpermt2var.d.512(<16 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0, i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8>, <16 x i32> %x0, <16 x i32> %x1, i16 -1)
   %res1 = call <16 x i32> @llvm.x86.avx512.maskz.vpermt2var.d.512(<16 x i32> <i32 0, i32 16, i32 2, i32 18, i32 4, i32 20, i32 6, i32 22, i32 8, i32 24, i32 10, i32 26, i32 12, i32 28, i32 14, i32 30>, <16 x i32> %res0, <16 x i32> %res0, i16 -1)
   ret <16 x i32> %res1
 }
 
 define <32 x i16> @combine_vpermi2var_32i16_as_permw(<32 x i16> %x0, <32 x i16> %x1) {
-; X32-LABEL: combine_vpermi2var_32i16_as_permw:
-; X32:       # %bb.0:
-; X32-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [15,16,14,17,13,18,12,19,11,20,10,21,9,22,8,23,7,24,6,25,5,26,4,27,3,28,2,29,1,30,0,31]
-; X32-NEXT:    vpermw %zmm0, %zmm1, %zmm0
-; X32-NEXT:    retl
-;
-; X64-LABEL: combine_vpermi2var_32i16_as_permw:
-; X64:       # %bb.0:
-; X64-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [15,16,14,17,13,18,12,19,11,20,10,21,9,22,8,23,7,24,6,25,5,26,4,27,3,28,2,29,1,30,0,31]
-; X64-NEXT:    vpermw %zmm0, %zmm1, %zmm0
-; X64-NEXT:    retq
+; CHECK-LABEL: combine_vpermi2var_32i16_as_permw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [15,16,14,17,13,18,12,19,11,20,10,21,9,22,8,23,7,24,6,25,5,26,4,27,3,28,2,29,1,30,0,31]
+; CHECK-NEXT:    vpermw %zmm0, %zmm1, %zmm0
+; CHECK-NEXT:    ret{{[l|q]}}
   %res0 = call <32 x i16> @llvm.x86.avx512.mask.vpermi2var.hi.512(<32 x i16> %x0, <32 x i16> <i16 15, i16 14, i16 13, i16 12, i16 11, i16 10, i16 9, i16 8, i16 7, i16 6, i16 5, i16 4, i16 3, i16 2, i16 1, i16 0, i16 31, i16 30, i16 29, i16 28, i16 27, i16 26, i16 25, i16 24, i16 23, i16 22, i16 21, i16 20, i16 19, i16 18, i16 17, i16 16>, <32 x i16> %x1, i32 -1)
   %res1 = call <32 x i16> @llvm.x86.avx512.mask.vpermi2var.hi.512(<32 x i16> %res0, <32 x i16> <i16 0, i16 31, i16 1, i16 30, i16 2, i16 29, i16 3, i16 28, i16 4, i16 27, i16 5, i16 26, i16 6, i16 25, i16 7, i16 24, i16 8, i16 23, i16 9, i16 22, i16 10, i16 21, i16 11, i16 20, i16 12, i16 19, i16 13, i16 18, i16 14, i16 17, i16 15, i16 16>, <32 x i16> %res0, i32 -1)
   ret <32 x i16> %res1
 }
 
 define <8 x double> @combine_vpermi2var_vpermt2var_8f64_as_vperm2(<8 x double> %x0, <8 x double> %x1) {
-; X32-LABEL: combine_vpermi2var_vpermt2var_8f64_as_vperm2:
-; X32:       # %bb.0:
-; X32-NEXT:    vmovapd {{.*#+}} zmm2 = [4,0,14,0,3,0,12,0,7,0,8,0,0,0,15,0]
-; X32-NEXT:    vpermi2pd %zmm0, %zmm1, %zmm2
-; X32-NEXT:    vmovapd %zmm2, %zmm0
-; X32-NEXT:    retl
+; X86-LABEL: combine_vpermi2var_vpermt2var_8f64_as_vperm2:
+; X86:       # %bb.0:
+; X86-NEXT:    vmovapd {{.*#+}} zmm2 = [4,0,14,0,3,0,12,0,7,0,8,0,0,0,15,0]
+; X86-NEXT:    vpermi2pd %zmm0, %zmm1, %zmm2
+; X86-NEXT:    vmovapd %zmm2, %zmm0
+; X86-NEXT:    retl
 ;
 ; X64-LABEL: combine_vpermi2var_vpermt2var_8f64_as_vperm2:
 ; X64:       # %bb.0:
@@ -1051,48 +919,35 @@ define <8 x double> @combine_vpermi2var_vpermt2var_8f64_as_vperm2(<8 x double> %
 }
 
 define <16 x i32> @combine_vpermi2var_vpermt2var_16i32_as_vpermd(<16 x i32> %x0, <16 x i32> %x1) {
-; X32-LABEL: combine_vpermi2var_vpermt2var_16i32_as_vpermd:
-; X32:       # %bb.0:
-; X32-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [0,31,2,2,4,29,6,27,8,25,10,23,12,21,14,19]
-; X32-NEXT:    vpermt2d %zmm1, %zmm2, %zmm0
-; X32-NEXT:    retl
-;
-; X64-LABEL: combine_vpermi2var_vpermt2var_16i32_as_vpermd:
-; X64:       # %bb.0:
-; X64-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [0,31,2,2,4,29,6,27,8,25,10,23,12,21,14,19]
-; X64-NEXT:    vpermt2d %zmm1, %zmm2, %zmm0
-; X64-NEXT:    retq
+; CHECK-LABEL: combine_vpermi2var_vpermt2var_16i32_as_vpermd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [0,31,2,2,4,29,6,27,8,25,10,23,12,21,14,19]
+; CHECK-NEXT:    vpermt2d %zmm1, %zmm2, %zmm0
+; CHECK-NEXT:    ret{{[l|q]}}
   %res0 = call <16 x i32> @llvm.x86.avx512.mask.vpermi2var.d.512(<16 x i32> %x0, <16 x i32> <i32 0, i32 31, i32 2, i32 29, i32 4, i32 27, i32 6, i32 25, i32 8, i32 23, i32 10, i32 21, i32 12, i32 19, i32 14, i32 17>, <16 x i32> %x1, i16 -1)
   %res1 = call <16 x i32> @llvm.x86.avx512.maskz.vpermt2var.d.512(<16 x i32> <i32 0, i32 17, i32 2, i32 18, i32 4, i32 19, i32 6, i32 21, i32 8, i32 23, i32 10, i32 25, i32 12, i32 27, i32 14, i32 29>, <16 x i32> %res0, <16 x i32> %res0, i16 -1)
   ret <16 x i32> %res1
 }
 
 define <32 x i16> @combine_vpermt2var_vpermi2var_32i16_as_permw(<32 x i16> %x0, <32 x i16> %x1) {
-; X32-LABEL: combine_vpermt2var_vpermi2var_32i16_as_permw:
-; X32:       # %bb.0:
-; X32-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [17,39,19,38,21,37,23,36,25,35,27,34,29,33,31,32,1,47,3,46,5,45,7,44,9,43,11,42,13,41,15,40]
-; X32-NEXT:    vpermi2w %zmm0, %zmm1, %zmm2
-; X32-NEXT:    vmovdqa64 %zmm2, %zmm0
-; X32-NEXT:    retl
-;
-; X64-LABEL: combine_vpermt2var_vpermi2var_32i16_as_permw:
-; X64:       # %bb.0:
-; X64-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [17,39,19,38,21,37,23,36,25,35,27,34,29,33,31,32,1,47,3,46,5,45,7,44,9,43,11,42,13,41,15,40]
-; X64-NEXT:    vpermi2w %zmm0, %zmm1, %zmm2
-; X64-NEXT:    vmovdqa64 %zmm2, %zmm0
-; X64-NEXT:    retq
+; CHECK-LABEL: combine_vpermt2var_vpermi2var_32i16_as_permw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [17,39,19,38,21,37,23,36,25,35,27,34,29,33,31,32,1,47,3,46,5,45,7,44,9,43,11,42,13,41,15,40]
+; CHECK-NEXT:    vpermi2w %zmm0, %zmm1, %zmm2
+; CHECK-NEXT:    vmovdqa64 %zmm2, %zmm0
+; CHECK-NEXT:    ret{{[l|q]}}
   %res0 = call <32 x i16> @llvm.x86.avx512.maskz.vpermt2var.hi.512(<32 x i16> <i16 0, i16 63, i16 1, i16 61, i16 2, i16 59, i16 3, i16 57, i16 4, i16 55, i16 5, i16 53, i16 6, i16 51, i16 7, i16 49, i16 8, i16 47, i16 9, i16 45, i16 10, i16 43, i16 11, i16 41, i16 12, i16 39, i16 13, i16 37, i16 14, i16 35, i16 15, i16 33>, <32 x i16> %x0, <32 x i16> %x1, i32 -1)
   %res1 = call <32 x i16> @llvm.x86.avx512.mask.vpermi2var.hi.512(<32 x i16> %res0, <32 x i16> <i16 15, i16 14, i16 13, i16 12, i16 11, i16 10, i16 9, i16 8, i16 7, i16 6, i16 5, i16 4, i16 3, i16 2, i16 1, i16 0, i16 31, i16 30, i16 29, i16 28, i16 27, i16 26, i16 25, i16 24, i16 23, i16 22, i16 21, i16 20, i16 19, i16 18, i16 17, i16 16>, <32 x i16> %res0, i32 -1)
   ret <32 x i16> %res1
 }
 
 define <8 x double> @combine_vpermi2var_vpermvar_8f64_as_vperm2_zero(<8 x double> %x0) {
-; X32-LABEL: combine_vpermi2var_vpermvar_8f64_as_vperm2_zero:
-; X32:       # %bb.0:
-; X32-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
-; X32-NEXT:    vmovapd {{.*#+}} zmm2 = [8,0,3,0,10,0,11,0,1,0,7,0,14,0,5,0]
-; X32-NEXT:    vpermt2pd %zmm1, %zmm2, %zmm0
-; X32-NEXT:    retl
+; X86-LABEL: combine_vpermi2var_vpermvar_8f64_as_vperm2_zero:
+; X86:       # %bb.0:
+; X86-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
+; X86-NEXT:    vmovapd {{.*#+}} zmm2 = [8,0,3,0,10,0,11,0,1,0,7,0,14,0,5,0]
+; X86-NEXT:    vpermt2pd %zmm1, %zmm2, %zmm0
+; X86-NEXT:    retl
 ;
 ; X64-LABEL: combine_vpermi2var_vpermvar_8f64_as_vperm2_zero:
 ; X64:       # %bb.0:
@@ -1106,34 +961,27 @@ define <8 x double> @combine_vpermi2var_vpermvar_8f64_as_vperm2_zero(<8 x double
 }
 
 define <16 x float> @combine_vpermi2var_vpermvar_16f32_as_vperm2_zero(<16 x float> %x0) {
-; X32-LABEL: combine_vpermi2var_vpermvar_16f32_as_vperm2_zero:
-; X32:       # %bb.0:
-; X32-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; X32-NEXT:    vmovaps {{.*#+}} zmm2 = [0,13,1,12,4,9,22,12,4,25,26,9,5,29,30,8]
-; X32-NEXT:    vpermt2ps %zmm1, %zmm2, %zmm0
-; X32-NEXT:    retl
-;
-; X64-LABEL: combine_vpermi2var_vpermvar_16f32_as_vperm2_zero:
-; X64:       # %bb.0:
-; X64-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; X64-NEXT:    vmovaps {{.*#+}} zmm2 = [0,13,1,12,4,9,22,12,4,25,26,9,5,29,30,8]
-; X64-NEXT:    vpermt2ps %zmm1, %zmm2, %zmm0
-; X64-NEXT:    retq
+; CHECK-LABEL: combine_vpermi2var_vpermvar_16f32_as_vperm2_zero:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; CHECK-NEXT:    vmovaps {{.*#+}} zmm2 = [0,13,1,12,4,9,22,12,4,25,26,9,5,29,30,8]
+; CHECK-NEXT:    vpermt2ps %zmm1, %zmm2, %zmm0
+; CHECK-NEXT:    ret{{[l|q]}}
   %res0 = shufflevector <16 x float> %x0, <16 x float> zeroinitializer, <16 x i32> <i32 0, i32 16, i32 1, i32 17, i32 4, i32 20, i32 5, i32 21, i32 8, i32 24, i32 9, i32 25, i32 12, i32 28, i32 13, i32 29>
   %res1 = call <16 x float> @llvm.x86.avx512.mask.vpermi2var.ps.512(<16 x float> %res0, <16 x i32> <i32 0, i32 14, i32 2, i32 12, i32 4, i32 10, i32 3, i32 12, i32 4, i32 11, i32 5, i32 10, i32 6, i32 9, i32 7, i32 8>, <16 x float> %res0, i16 -1)
   ret <16 x float> %res1
 }
 
 define <8 x i64> @combine_vpermvar_insertion_as_broadcast_v8i64(i64 %a0) {
-; X32-LABEL: combine_vpermvar_insertion_as_broadcast_v8i64:
-; X32:       # %bb.0:
-; X32-NEXT:    vbroadcastsd {{[0-9]+}}(%esp), %zmm0
-; X32-NEXT:    retl
+; X86-LABEL: combine_vpermvar_insertion_as_broadcast_v8i64:
+; X86:       # %bb.0:
+; X86-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; X86-NEXT:    vbroadcastsd %xmm0, %zmm0
+; X86-NEXT:    retl
 ;
 ; X64-LABEL: combine_vpermvar_insertion_as_broadcast_v8i64:
 ; X64:       # %bb.0:
-; X64-NEXT:    vmovq %rdi, %xmm0
-; X64-NEXT:    vpbroadcastq %xmm0, %zmm0
+; X64-NEXT:    vpbroadcastq %rdi, %zmm0
 ; X64-NEXT:    retq
   %1 = insertelement <8 x i64> undef, i64 %a0, i32 0
   %2 = call <8 x i64> @llvm.x86.avx512.permvar.di.512(<8 x i64> %1, <8 x i64> zeroinitializer)
diff --git a/test/CodeGen/X86/vector-shuffle-combining-avx512bwvl.ll b/test/CodeGen/X86/vector-shuffle-combining-avx512bwvl.ll
index c4ab922d068ceacd01fc555675f44cb4606a1c1b..fd41c9f089e65f31ab3018637509eac27967ae2e 100644
--- a/test/CodeGen/X86/vector-shuffle-combining-avx512bwvl.ll
+++ b/test/CodeGen/X86/vector-shuffle-combining-avx512bwvl.ll
@@ -1,31 +1,27 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=i686-unknown -mattr=+avx512bw,+avx512vl | FileCheck %s --check-prefix=X32
-; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx512bw,+avx512vl | FileCheck %s --check-prefix=X64
+; RUN: llc < %s -mtriple=i686-unknown -mattr=+avx512bw,+avx512vl | FileCheck %s --check-prefixes=CHECK,X86
+; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx512bw,+avx512vl | FileCheck %s --check-prefixes=CHECK,X64
 
 declare <16 x i16> @llvm.x86.avx512.maskz.vpermt2var.hi.256(<16 x i16>, <16 x i16>, <16 x i16>, i16)
 declare <16 x i16> @llvm.x86.avx512.mask.vpermi2var.hi.256(<16 x i16>, <16 x i16>, <16 x i16>, i16)
 
 define <16 x i16> @combine_vpermt2var_16i16_identity(<16 x i16> %x0, <16 x i16> %x1) {
-; X32-LABEL: combine_vpermt2var_16i16_identity:
-; X32:       # %bb.0:
-; X32-NEXT:    retl
-;
-; X64-LABEL: combine_vpermt2var_16i16_identity:
-; X64:       # %bb.0:
-; X64-NEXT:    retq
+; CHECK-LABEL: combine_vpermt2var_16i16_identity:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ret{{[l|q]}}
   %res0 = call <16 x i16> @llvm.x86.avx512.maskz.vpermt2var.hi.256(<16 x i16> <i16 15, i16 14, i16 13, i16 12, i16 11, i16 10, i16 9, i16 8, i16 7, i16 6, i16 5, i16 4, i16 3, i16 2, i16 1, i16 0>, <16 x i16> %x0, <16 x i16> %x1, i16 -1)
   %res1 = call <16 x i16> @llvm.x86.avx512.maskz.vpermt2var.hi.256(<16 x i16> <i16 15, i16 30, i16 13, i16 28, i16 11, i16 26, i16 9, i16 24, i16 7, i16 22, i16 5, i16 20, i16 3, i16 18, i16 1, i16 16>, <16 x i16> %res0, <16 x i16> %res0, i16 -1)
   ret <16 x i16> %res1
 }
 define <16 x i16> @combine_vpermt2var_16i16_identity_mask(<16 x i16> %x0, <16 x i16> %x1, i16 %m) {
-; X32-LABEL: combine_vpermt2var_16i16_identity_mask:
-; X32:       # %bb.0:
-; X32-NEXT:    vmovdqa {{.*#+}} ymm1 = [15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0]
-; X32-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
-; X32-NEXT:    vpermi2w %ymm0, %ymm0, %ymm1 {%k1} {z}
-; X32-NEXT:    vmovdqa {{.*#+}} ymm0 = [15,30,13,28,11,26,9,24,7,22,5,20,3,18,1,16]
-; X32-NEXT:    vpermi2w %ymm1, %ymm1, %ymm0 {%k1} {z}
-; X32-NEXT:    retl
+; X86-LABEL: combine_vpermt2var_16i16_identity_mask:
+; X86:       # %bb.0:
+; X86-NEXT:    vmovdqa {{.*#+}} ymm1 = [15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0]
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vpermi2w %ymm0, %ymm0, %ymm1 {%k1} {z}
+; X86-NEXT:    vmovdqa {{.*#+}} ymm0 = [15,30,13,28,11,26,9,24,7,22,5,20,3,18,1,16]
+; X86-NEXT:    vpermi2w %ymm1, %ymm1, %ymm0 {%k1} {z}
+; X86-NEXT:    retl
 ;
 ; X64-LABEL: combine_vpermt2var_16i16_identity_mask:
 ; X64:       # %bb.0:
@@ -41,63 +37,41 @@ define <16 x i16> @combine_vpermt2var_16i16_identity_mask(<16 x i16> %x0, <16 x
 }
 
 define <16 x i16> @combine_vpermi2var_16i16_as_permw(<16 x i16> %x0, <16 x i16> %x1) {
-; X32-LABEL: combine_vpermi2var_16i16_as_permw:
-; X32:       # %bb.0:
-; X32-NEXT:    vmovdqa {{.*#+}} ymm1 = [15,0,14,1,13,2,12,3,11,4,10,5,9,6,8,7]
-; X32-NEXT:    vpermw %ymm0, %ymm1, %ymm0
-; X32-NEXT:    retl
-;
-; X64-LABEL: combine_vpermi2var_16i16_as_permw:
-; X64:       # %bb.0:
-; X64-NEXT:    vmovdqa {{.*#+}} ymm1 = [15,0,14,1,13,2,12,3,11,4,10,5,9,6,8,7]
-; X64-NEXT:    vpermw %ymm0, %ymm1, %ymm0
-; X64-NEXT:    retq
+; CHECK-LABEL: combine_vpermi2var_16i16_as_permw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovdqa {{.*#+}} ymm1 = [15,0,14,1,13,2,12,3,11,4,10,5,9,6,8,7]
+; CHECK-NEXT:    vpermw %ymm0, %ymm1, %ymm0
+; CHECK-NEXT:    ret{{[l|q]}}
   %res0 = call <16 x i16> @llvm.x86.avx512.mask.vpermi2var.hi.256(<16 x i16> %x0, <16 x i16> <i16 15, i16 14, i16 13, i16 12, i16 11, i16 10, i16 9, i16 8, i16 7, i16 6, i16 5, i16 4, i16 3, i16 2, i16 1, i16 0>, <16 x i16> %x1, i16 -1)
   %res1 = call <16 x i16> @llvm.x86.avx512.mask.vpermi2var.hi.256(<16 x i16> %res0, <16 x i16> <i16 0, i16 15, i16 1, i16 14, i16 2, i16 13, i16 3, i16 12, i16 4, i16 11, i16 5, i16 10, i16 6, i16 9, i16 7, i16 8>, <16 x i16> %res0, i16 -1)
   ret <16 x i16> %res1
 }
 
 define <16 x i16> @combine_vpermt2var_vpermi2var_16i16_as_vperm2(<16 x i16> %x0, <16 x i16> %x1) {
-; X32-LABEL: combine_vpermt2var_vpermi2var_16i16_as_vperm2:
-; X32:       # %bb.0:
-; X32-NEXT:    vmovdqa {{.*#+}} ymm2 = [0,31,2,2,4,29,6,27,8,25,10,23,12,21,14,19]
-; X32-NEXT:    vpermt2w %ymm1, %ymm2, %ymm0
-; X32-NEXT:    retl
-;
-; X64-LABEL: combine_vpermt2var_vpermi2var_16i16_as_vperm2:
-; X64:       # %bb.0:
-; X64-NEXT:    vmovdqa {{.*#+}} ymm2 = [0,31,2,2,4,29,6,27,8,25,10,23,12,21,14,19]
-; X64-NEXT:    vpermt2w %ymm1, %ymm2, %ymm0
-; X64-NEXT:    retq
+; CHECK-LABEL: combine_vpermt2var_vpermi2var_16i16_as_vperm2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovdqa {{.*#+}} ymm2 = [0,31,2,2,4,29,6,27,8,25,10,23,12,21,14,19]
+; CHECK-NEXT:    vpermt2w %ymm1, %ymm2, %ymm0
+; CHECK-NEXT:    ret{{[l|q]}}
   %res0 = call <16 x i16> @llvm.x86.avx512.mask.vpermi2var.hi.256(<16 x i16> %x0, <16 x i16> <i16 0, i16 31, i16 2, i16 29, i16 4, i16 27, i16 6, i16 25, i16 8, i16 23, i16 10, i16 21, i16 12, i16 19, i16 14, i16 17>, <16 x i16> %x1, i16 -1)
   %res1 = call <16 x i16> @llvm.x86.avx512.maskz.vpermt2var.hi.256(<16 x i16> <i16 0, i16 17, i16 2, i16 18, i16 4, i16 19, i16 6, i16 21, i16 8, i16 23, i16 10, i16 25, i16 12, i16 27, i16 14, i16 29>, <16 x i16> %res0, <16 x i16> %res0, i16 -1)
   ret <16 x i16> %res1
 }
 
 define <16 x i16> @combine_vpermt2var_vpermi2var_16i16_as_unpckhwd(<16 x i16> %a0, <16 x i16> %a1) {
-; X32-LABEL: combine_vpermt2var_vpermi2var_16i16_as_unpckhwd:
-; X32:       # %bb.0:
-; X32-NEXT:    vpunpckhwd {{.*#+}} ymm0 = ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[12],ymm0[12],ymm1[13],ymm0[13],ymm1[14],ymm0[14],ymm1[15],ymm0[15]
-; X32-NEXT:    retl
-;
-; X64-LABEL: combine_vpermt2var_vpermi2var_16i16_as_unpckhwd:
-; X64:       # %bb.0:
-; X64-NEXT:    vpunpckhwd {{.*#+}} ymm0 = ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[12],ymm0[12],ymm1[13],ymm0[13],ymm1[14],ymm0[14],ymm1[15],ymm0[15]
-; X64-NEXT:    retq
+; CHECK-LABEL: combine_vpermt2var_vpermi2var_16i16_as_unpckhwd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpunpckhwd {{.*#+}} ymm0 = ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[12],ymm0[12],ymm1[13],ymm0[13],ymm1[14],ymm0[14],ymm1[15],ymm0[15]
+; CHECK-NEXT:    ret{{[l|q]}}
   %res0 = call <16 x i16> @llvm.x86.avx512.mask.vpermi2var.hi.256(<16 x i16> %a0, <16 x i16> <i16 20, i16 4, i16 21, i16 5, i16 22, i16 6, i16 23, i16 7, i16 28, i16 12, i16 29, i16 13, i16 30, i16 14, i16 31, i16 15>, <16 x i16> %a1, i16 -1)
   ret <16 x i16> %res0
 }
 
 define <16 x i16> @combine_vpermt2var_vpermi2var_16i16_as_unpcklwd(<16 x i16> %a0, <16 x i16> %a1) {
-; X32-LABEL: combine_vpermt2var_vpermi2var_16i16_as_unpcklwd:
-; X32:       # %bb.0:
-; X32-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[8],ymm1[8],ymm0[9],ymm1[9],ymm0[10],ymm1[10],ymm0[11],ymm1[11]
-; X32-NEXT:    retl
-;
-; X64-LABEL: combine_vpermt2var_vpermi2var_16i16_as_unpcklwd:
-; X64:       # %bb.0:
-; X64-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[8],ymm1[8],ymm0[9],ymm1[9],ymm0[10],ymm1[10],ymm0[11],ymm1[11]
-; X64-NEXT:    retq
+; CHECK-LABEL: combine_vpermt2var_vpermi2var_16i16_as_unpcklwd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpunpcklwd {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[8],ymm1[8],ymm0[9],ymm1[9],ymm0[10],ymm1[10],ymm0[11],ymm1[11]
+; CHECK-NEXT:    ret{{[l|q]}}
   %res0 = call <16 x i16> @llvm.x86.avx512.maskz.vpermt2var.hi.256(<16 x i16> <i16 0, i16 16, i16 1, i16 17, i16 2, i16 18, i16 3, i16 19, i16 8, i16 24, i16 9, i16 25, i16 10, i16 26, i16 11, i16 27>, <16 x i16> %a0, <16 x i16> %a1, i16 -1)
   ret <16 x i16> %res0
 }
diff --git a/test/CodeGen/X86/vector-shuffle-combining-avx512vbmi.ll b/test/CodeGen/X86/vector-shuffle-combining-avx512vbmi.ll
index 5350ddab5ef7dc126acf7b7da56275208e92af3b..0e0dfecb22d7695493aabd699ff6f60764bdde62 100644
--- a/test/CodeGen/X86/vector-shuffle-combining-avx512vbmi.ll
+++ b/test/CodeGen/X86/vector-shuffle-combining-avx512vbmi.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=i686-unknown -mattr=+avx512vbmi,+avx512vl | FileCheck %s --check-prefix=X32
-; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx512vbmi,+avx512vl | FileCheck %s --check-prefix=X64
+; RUN: llc < %s -mtriple=i686-unknown -mattr=+avx512vbmi,+avx512vl | FileCheck %s --check-prefixes=CHECK,X86
+; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx512vbmi,+avx512vl | FileCheck %s --check-prefixes=CHECK,X64
 
 declare <16 x i8> @llvm.x86.avx512.mask.vpermi2var.qi.128(<16 x i8>, <16 x i8>, <16 x i8>, i16)
 declare <16 x i8> @llvm.x86.avx512.mask.vpermt2var.qi.128(<16 x i8>, <16 x i8>, <16 x i8>, i16)
@@ -19,26 +19,22 @@ declare <32 x i8> @llvm.x86.avx2.pshuf.b(<32 x i8>, <32 x i8>)
 declare <64 x i8> @llvm.x86.avx512.mask.pshuf.b.512(<64 x i8>, <64 x i8>, <64 x i8>, i64)
 
 define <16 x i8> @combine_vpermt2var_16i8_identity(<16 x i8> %x0, <16 x i8> %x1) {
-; X32-LABEL: combine_vpermt2var_16i8_identity:
-; X32:       # %bb.0:
-; X32-NEXT:    retl
-;
-; X64-LABEL: combine_vpermt2var_16i8_identity:
-; X64:       # %bb.0:
-; X64-NEXT:    retq
+; CHECK-LABEL: combine_vpermt2var_16i8_identity:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ret{{[l|q]}}
   %res0 = call <16 x i8> @llvm.x86.avx512.maskz.vpermt2var.qi.128(<16 x i8> <i8 15, i8 14, i8 13, i8 12, i8 11, i8 10, i8 9, i8 8, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0>, <16 x i8> %x0, <16 x i8> %x1, i16 -1)
   %res1 = call <16 x i8> @llvm.x86.avx512.maskz.vpermt2var.qi.128(<16 x i8> <i8 15, i8 30, i8 13, i8 28, i8 11, i8 26, i8 9, i8 24, i8 7, i8 22, i8 5, i8 20, i8 3, i8 18, i8 1, i8 16>, <16 x i8> %res0, <16 x i8> %res0, i16 -1)
   ret <16 x i8> %res1
 }
 define <16 x i8> @combine_vpermt2var_16i8_identity_mask(<16 x i8> %x0, <16 x i8> %x1, i16 %m) {
-; X32-LABEL: combine_vpermt2var_16i8_identity_mask:
-; X32:       # %bb.0:
-; X32-NEXT:    vmovdqa {{.*#+}} xmm1 = [15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0]
-; X32-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
-; X32-NEXT:    vpermi2b %xmm0, %xmm0, %xmm1 {%k1} {z}
-; X32-NEXT:    vmovdqa {{.*#+}} xmm0 = [15,30,13,28,11,26,9,24,7,22,5,20,3,18,1,16]
-; X32-NEXT:    vpermi2b %xmm1, %xmm1, %xmm0 {%k1} {z}
-; X32-NEXT:    retl
+; X86-LABEL: combine_vpermt2var_16i8_identity_mask:
+; X86:       # %bb.0:
+; X86-NEXT:    vmovdqa {{.*#+}} xmm1 = [15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0]
+; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
+; X86-NEXT:    vpermi2b %xmm0, %xmm0, %xmm1 {%k1} {z}
+; X86-NEXT:    vmovdqa {{.*#+}} xmm0 = [15,30,13,28,11,26,9,24,7,22,5,20,3,18,1,16]
+; X86-NEXT:    vpermi2b %xmm1, %xmm1, %xmm0 {%k1} {z}
+; X86-NEXT:    retl
 ;
 ; X64-LABEL: combine_vpermt2var_16i8_identity_mask:
 ; X64:       # %bb.0:
@@ -54,100 +50,63 @@ define <16 x i8> @combine_vpermt2var_16i8_identity_mask(<16 x i8> %x0, <16 x i8>
 }
 
 define <16 x i8> @combine_vpermi2var_16i8_as_vpshufb(<16 x i8> %x0, <16 x i8> %x1) {
-; X32-LABEL: combine_vpermi2var_16i8_as_vpshufb:
-; X32:       # %bb.0:
-; X32-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[15,0,14,1,13,2,12,3,11,4,10,5,9,6,8,7]
-; X32-NEXT:    retl
-;
-; X64-LABEL: combine_vpermi2var_16i8_as_vpshufb:
-; X64:       # %bb.0:
-; X64-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[15,0,14,1,13,2,12,3,11,4,10,5,9,6,8,7]
-; X64-NEXT:    retq
+; CHECK-LABEL: combine_vpermi2var_16i8_as_vpshufb:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[15,0,14,1,13,2,12,3,11,4,10,5,9,6,8,7]
+; CHECK-NEXT:    ret{{[l|q]}}
   %res0 = call <16 x i8> @llvm.x86.avx512.mask.vpermi2var.qi.128(<16 x i8> %x0, <16 x i8> <i8 15, i8 14, i8 13, i8 12, i8 11, i8 10, i8 9, i8 8, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0>, <16 x i8> %x1, i16 -1)
   %res1 = call <16 x i8> @llvm.x86.avx512.mask.vpermi2var.qi.128(<16 x i8> %res0, <16 x i8> <i8 0, i8 15, i8 1, i8 14, i8 2, i8 13, i8 3, i8 12, i8 4, i8 11, i8 5, i8 10, i8 6, i8 9, i8 7, i8 8>, <16 x i8> %res0, i16 -1)
   ret <16 x i8> %res1
 }
 define <32 x i8> @combine_vpermi2var_32i8_as_vpermb(<32 x i8> %x0, <32 x i8> %x1) {
-; X32-LABEL: combine_vpermi2var_32i8_as_vpermb:
-; X32:       # %bb.0:
-; X32-NEXT:    vmovdqa {{.*#+}} ymm1 = [0,0,1,23,2,22,3,21,4,22,5,21,6,20,7,19,0,0,1,23,2,22,3,21,4,22,5,21,6,20,7,19]
-; X32-NEXT:    vpermb %ymm0, %ymm1, %ymm0
-; X32-NEXT:    retl
-;
-; X64-LABEL: combine_vpermi2var_32i8_as_vpermb:
-; X64:       # %bb.0:
-; X64-NEXT:    vmovdqa {{.*#+}} ymm1 = [0,0,1,23,2,22,3,21,4,22,5,21,6,20,7,19,0,0,1,23,2,22,3,21,4,22,5,21,6,20,7,19]
-; X64-NEXT:    vpermb %ymm0, %ymm1, %ymm0
-; X64-NEXT:    retq
+; CHECK-LABEL: combine_vpermi2var_32i8_as_vpermb:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovdqa {{.*#+}} ymm1 = [0,0,1,23,2,22,3,21,4,22,5,21,6,20,7,19,0,0,1,23,2,22,3,21,4,22,5,21,6,20,7,19]
+; CHECK-NEXT:    vpermb %ymm0, %ymm1, %ymm0
+; CHECK-NEXT:    ret{{[l|q]}}
   %res0 = shufflevector <32 x i8> %x0, <32 x i8> %x1, <32 x i32> <i32 0, i32 32, i32 1, i32 33, i32 2, i32 34, i32 3, i32 35, i32 4, i32 36, i32 5, i32 37, i32 6, i32 38, i32 7, i32 39, i32 16, i32 48, i32 17, i32 49, i32 18, i32 50, i32 19, i32 51, i32 20, i32 52, i32 21, i32 53, i32 22, i32 54, i32 23, i32 55>
   %res1 = call <32 x i8> @llvm.x86.avx512.mask.vpermi2var.qi.256(<32 x i8> %res0, <32 x i8> <i8 0, i8 32, i8 2, i8 30, i8 4, i8 28, i8 6, i8 26, i8 8, i8 28, i8 10, i8 26, i8 12, i8 24, i8 14, i8 22, i8 0, i8 32, i8 2, i8 30, i8 4, i8 28, i8 6, i8 26, i8 8, i8 28, i8 10, i8 26, i8 12, i8 24, i8 14, i8 22>, <32 x i8> %res0, i32 -1)
   ret <32 x i8> %res1
 }
 define <64 x i8> @combine_vpermi2var_64i8_as_vpermb(<64 x i8> %x0, <64 x i8> %x1) {
-; X32-LABEL: combine_vpermi2var_64i8_as_vpermb:
-; X32:       # %bb.0:
-; X32-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [0,32,1,23,2,22,3,21,4,22,5,21,6,20,7,19,0,32,1,23,2,22,3,21,4,22,5,21,6,20,7,19,0,32,1,23,2,22,3,21,4,22,5,21,6,20,7,19,0,32,1,23,2,22,3,21,4,22,5,21,6,20,7,19]
-; X32-NEXT:    vpermb %zmm0, %zmm1, %zmm0
-; X32-NEXT:    retl
-;
-; X64-LABEL: combine_vpermi2var_64i8_as_vpermb:
-; X64:       # %bb.0:
-; X64-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [0,32,1,23,2,22,3,21,4,22,5,21,6,20,7,19,0,32,1,23,2,22,3,21,4,22,5,21,6,20,7,19,0,32,1,23,2,22,3,21,4,22,5,21,6,20,7,19,0,32,1,23,2,22,3,21,4,22,5,21,6,20,7,19]
-; X64-NEXT:    vpermb %zmm0, %zmm1, %zmm0
-; X64-NEXT:    retq
+; CHECK-LABEL: combine_vpermi2var_64i8_as_vpermb:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [0,32,1,23,2,22,3,21,4,22,5,21,6,20,7,19,0,32,1,23,2,22,3,21,4,22,5,21,6,20,7,19,0,32,1,23,2,22,3,21,4,22,5,21,6,20,7,19,0,32,1,23,2,22,3,21,4,22,5,21,6,20,7,19]
+; CHECK-NEXT:    vpermb %zmm0, %zmm1, %zmm0
+; CHECK-NEXT:    ret{{[l|q]}}
   %res0 = shufflevector <64 x i8> %x0, <64 x i8> %x1, <64 x i32> <i32 0, i32 64, i32 1, i32 65, i32 2, i32 66, i32 3, i32 67, i32 4, i32 68, i32 5, i32 69, i32 6, i32 70, i32 7, i32 71, i32 16, i32 80, i32 17, i32 81, i32 18, i32 82, i32 19, i32 83, i32 20, i32 84, i32 21, i32 85, i32 22, i32 86, i32 23, i32 87, i32 32, i32 96, i32 33, i32 97, i32 34, i32 98, i32 35, i32 99, i32 36, i32 100, i32 37, i32 101, i32 38, i32 102, i32 39, i32 103, i32 48, i32 112, i32 49, i32 113, i32 50, i32 114, i32 51, i32 115, i32 52, i32 116, i32 53, i32 117, i32 54, i32 118, i32 55, i32 119>
   %res1 = call <64 x i8> @llvm.x86.avx512.mask.vpermi2var.qi.512(<64 x i8> %res0, <64 x i8> <i8 0, i8 32, i8 2, i8 30, i8 4, i8 28, i8 6, i8 26, i8 8, i8 28, i8 10, i8 26, i8 12, i8 24, i8 14, i8 22, i8 0, i8 32, i8 2, i8 30, i8 4, i8 28, i8 6, i8 26, i8 8, i8 28, i8 10, i8 26, i8 12, i8 24, i8 14, i8 22, i8 0, i8 32, i8 2, i8 30, i8 4, i8 28, i8 6, i8 26, i8 8, i8 28, i8 10, i8 26, i8 12, i8 24, i8 14, i8 22, i8 0, i8 32, i8 2, i8 30, i8 4, i8 28, i8 6, i8 26, i8 8, i8 28, i8 10, i8 26, i8 12, i8 24, i8 14, i8 22>, <64 x i8> %res0, i64 -1)
   ret <64 x i8> %res1
 }
 
 define <16 x i8> @combine_vpermt2var_vpermi2var_16i8_as_vperm2(<16 x i8> %x0, <16 x i8> %x1) {
-; X32-LABEL: combine_vpermt2var_vpermi2var_16i8_as_vperm2:
-; X32:       # %bb.0:
-; X32-NEXT:    vmovdqa {{.*#+}} xmm2 = [0,31,2,29,4,27,6,25,8,23,10,21,12,19,14,17]
-; X32-NEXT:    vpermi2b %xmm1, %xmm0, %xmm2
-; X32-NEXT:    vmovdqa {{.*#+}} xmm0 = [0,17,2,18,4,19,6,21,8,23,10,25,12,27,14,29]
-; X32-NEXT:    vpermi2b %xmm2, %xmm2, %xmm0
-; X32-NEXT:    retl
-;
-; X64-LABEL: combine_vpermt2var_vpermi2var_16i8_as_vperm2:
-; X64:       # %bb.0:
-; X64-NEXT:    vmovdqa {{.*#+}} xmm2 = [0,31,2,29,4,27,6,25,8,23,10,21,12,19,14,17]
-; X64-NEXT:    vpermi2b %xmm1, %xmm0, %xmm2
-; X64-NEXT:    vmovdqa {{.*#+}} xmm0 = [0,17,2,18,4,19,6,21,8,23,10,25,12,27,14,29]
-; X64-NEXT:    vpermi2b %xmm2, %xmm2, %xmm0
-; X64-NEXT:    retq
+; CHECK-LABEL: combine_vpermt2var_vpermi2var_16i8_as_vperm2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovdqa {{.*#+}} xmm2 = [0,31,2,29,4,27,6,25,8,23,10,21,12,19,14,17]
+; CHECK-NEXT:    vpermi2b %xmm1, %xmm0, %xmm2
+; CHECK-NEXT:    vmovdqa {{.*#+}} xmm0 = [0,17,2,18,4,19,6,21,8,23,10,25,12,27,14,29]
+; CHECK-NEXT:    vpermi2b %xmm2, %xmm2, %xmm0
+; CHECK-NEXT:    ret{{[l|q]}}
   %res0 = call <16 x i8> @llvm.x86.avx512.mask.vpermi2var.qi.128(<16 x i8> %x0, <16 x i8> <i8 0, i8 31, i8 2, i8 29, i8 4, i8 27, i8 6, i8 25, i8 8, i8 23, i8 10, i8 21, i8 12, i8 19, i8 14, i8 17>, <16 x i8> %x1, i16 -1)
   %res1 = call <16 x i8> @llvm.x86.avx512.maskz.vpermt2var.qi.128(<16 x i8> <i8 0, i8 17, i8 2, i8 18, i8 4, i8 19, i8 6, i8 21, i8 8, i8 23, i8 10, i8 25, i8 12, i8 27, i8 14, i8 29>, <16 x i8> %res0, <16 x i8> %res0, i16 -1)
   ret <16 x i8> %res1
 }
 define <32 x i8> @combine_vpermi2var_32i8_as_vperm2(<32 x i8> %x0, <32 x i8> %x1) {
-; X32-LABEL: combine_vpermi2var_32i8_as_vperm2:
-; X32:       # %bb.0:
-; X32-NEXT:    vmovdqa {{.*#+}} ymm2 = [0,32,1,23,2,22,3,21,4,22,5,21,6,20,7,19,0,32,1,23,2,22,3,21,4,22,5,21,6,20,7,19]
-; X32-NEXT:    vpermt2b %ymm1, %ymm2, %ymm0
-; X32-NEXT:    retl
-;
-; X64-LABEL: combine_vpermi2var_32i8_as_vperm2:
-; X64:       # %bb.0:
-; X64-NEXT:    vmovdqa {{.*#+}} ymm2 = [0,32,1,23,2,22,3,21,4,22,5,21,6,20,7,19,0,32,1,23,2,22,3,21,4,22,5,21,6,20,7,19]
-; X64-NEXT:    vpermt2b %ymm1, %ymm2, %ymm0
-; X64-NEXT:    retq
+; CHECK-LABEL: combine_vpermi2var_32i8_as_vperm2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovdqa {{.*#+}} ymm2 = [0,32,1,23,2,22,3,21,4,22,5,21,6,20,7,19,0,32,1,23,2,22,3,21,4,22,5,21,6,20,7,19]
+; CHECK-NEXT:    vpermt2b %ymm1, %ymm2, %ymm0
+; CHECK-NEXT:    ret{{[l|q]}}
   %res0 = shufflevector <32 x i8> %x0, <32 x i8> %x1, <32 x i32> <i32 0, i32 32, i32 1, i32 33, i32 2, i32 34, i32 3, i32 35, i32 4, i32 36, i32 5, i32 37, i32 6, i32 38, i32 7, i32 39, i32 16, i32 48, i32 17, i32 49, i32 18, i32 50, i32 19, i32 51, i32 20, i32 52, i32 21, i32 53, i32 22, i32 54, i32 23, i32 55>
   %res1 = call <32 x i8> @llvm.x86.avx512.mask.vpermi2var.qi.256(<32 x i8> %res0, <32 x i8> <i8 0, i8 32, i8 2, i8 30, i8 4, i8 28, i8 6, i8 26, i8 8, i8 28, i8 10, i8 26, i8 12, i8 24, i8 14, i8 22, i8 0, i8 32, i8 2, i8 30, i8 4, i8 28, i8 6, i8 26, i8 8, i8 28, i8 10, i8 26, i8 12, i8 24, i8 14, i8 22>, <32 x i8> %x1, i32 -1)
   ret <32 x i8> %res1
 }
 define <64 x i8> @combine_vpermi2var_64i8_as_vperm2(<64 x i8> %x0, <64 x i8> %x1) {
-; X32-LABEL: combine_vpermi2var_64i8_as_vperm2:
-; X32:       # %bb.0:
-; X32-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [0,80,1,70,2,54,3,49,4,36,5,23,6,18,7,5,0,90,1,100,2,110,3,120,4,22,5,21,6,20,7,19,0,32,1,23,2,22,3,21,4,22,5,21,6,20,7,19,0,32,1,23,2,22,3,21,4,22,5,21,6,20,7,19]
-; X32-NEXT:    vpermt2b %zmm1, %zmm2, %zmm0
-; X32-NEXT:    retl
-;
-; X64-LABEL: combine_vpermi2var_64i8_as_vperm2:
-; X64:       # %bb.0:
-; X64-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [0,80,1,70,2,54,3,49,4,36,5,23,6,18,7,5,0,90,1,100,2,110,3,120,4,22,5,21,6,20,7,19,0,32,1,23,2,22,3,21,4,22,5,21,6,20,7,19,0,32,1,23,2,22,3,21,4,22,5,21,6,20,7,19]
-; X64-NEXT:    vpermt2b %zmm1, %zmm2, %zmm0
-; X64-NEXT:    retq
+; CHECK-LABEL: combine_vpermi2var_64i8_as_vperm2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [0,80,1,70,2,54,3,49,4,36,5,23,6,18,7,5,0,90,1,100,2,110,3,120,4,22,5,21,6,20,7,19,0,32,1,23,2,22,3,21,4,22,5,21,6,20,7,19,0,32,1,23,2,22,3,21,4,22,5,21,6,20,7,19]
+; CHECK-NEXT:    vpermt2b %zmm1, %zmm2, %zmm0
+; CHECK-NEXT:    ret{{[l|q]}}
   %res0 = shufflevector <64 x i8> %x0, <64 x i8> %x1, <64 x i32> <i32 0, i32 64, i32 1, i32 65, i32 2, i32 66, i32 3, i32 67, i32 4, i32 68, i32 5, i32 69, i32 6, i32 70, i32 7, i32 71, i32 16, i32 80, i32 17, i32 81, i32 18, i32 82, i32 19, i32 83, i32 20, i32 84, i32 21, i32 85, i32 22, i32 86, i32 23, i32 87, i32 32, i32 96, i32 33, i32 97, i32 34, i32 98, i32 35, i32 99, i32 36, i32 100, i32 37, i32 101, i32 38, i32 102, i32 39, i32 103, i32 48, i32 112, i32 49, i32 113, i32 50, i32 114, i32 51, i32 115, i32 52, i32 116, i32 53, i32 117, i32 54, i32 118, i32 55, i32 119>
   %res1 = call <64 x i8> @llvm.x86.avx512.mask.vpermi2var.qi.512(<64 x i8> %res0, <64 x i8> <i8 0, i8 80, i8 2, i8 70, i8 4, i8 60, i8 6, i8 50, i8 8, i8 40, i8 10, i8 30, i8 12, i8 20, i8 14, i8 10, i8 0, i8 90, i8 2, i8 100, i8 4, i8 110, i8 6, i8 120, i8 8, i8 28, i8 10, i8 26, i8 12, i8 24, i8 14, i8 22, i8 0, i8 32, i8 2, i8 30, i8 4, i8 28, i8 6, i8 26, i8 8, i8 28, i8 10, i8 26, i8 12, i8 24, i8 14, i8 22, i8 0, i8 32, i8 2, i8 30, i8 4, i8 28, i8 6, i8 26, i8 8, i8 28, i8 10, i8 26, i8 12, i8 24, i8 14, i8 22>, <64 x i8> %x1, i64 -1)
   ret <64 x i8> %res1
diff --git a/test/CodeGen/X86/vector-shuffle-combining-sse41.ll b/test/CodeGen/X86/vector-shuffle-combining-sse41.ll
index 27ccdefe4d50b3c332a059d87c4a9914b3f597e5..60c3c86663640b7e9c1cdf4ed0fa7f00eeebf29f 100644
--- a/test/CodeGen/X86/vector-shuffle-combining-sse41.ll
+++ b/test/CodeGen/X86/vector-shuffle-combining-sse41.ll
@@ -1,9 +1,9 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+sse4.1 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSE41
-; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX1
-; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx2 | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX2
-; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx512f | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX512F
-;
+; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+sse4.1 | FileCheck %s --check-prefixes=CHECK,SSE,SSE41
+; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx | FileCheck %s --check-prefixes=CHECK,AVX,AVX1
+; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx2 | FileCheck %s --check-prefixes=CHECK,AVX,AVX2
+; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx512f | FileCheck %s --check-prefixes=CHECK,AVX,AVX512F
+
 ; Combine tests involving SSE41 target shuffles (BLEND,INSERTPS,MOVZX)
 
 declare <16 x i8> @llvm.x86.ssse3.pshuf.b.128(<16 x i8>, <16 x i8>)
diff --git a/test/CodeGen/X86/vector-shuffle-combining-sse4a.ll b/test/CodeGen/X86/vector-shuffle-combining-sse4a.ll
index 1b701f8515ee2a3912af1516b39f22ad3a9e3282..7fe64031a2a46d7f5b5cc27f03e292240728cab5 100644
--- a/test/CodeGen/X86/vector-shuffle-combining-sse4a.ll
+++ b/test/CodeGen/X86/vector-shuffle-combining-sse4a.ll
@@ -1,28 +1,28 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+ssse3,+sse4a | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSSE3
-; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+sse4.2,+sse4a | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSE42
-; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx,+sse4a| FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX1
-; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx2,+sse4a | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX2
+; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+ssse3,+sse4a | FileCheck %s --check-prefixes=CHECK,SSE,SSSE3
+; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+sse4.2,+sse4a | FileCheck %s --check-prefixes=CHECK,SSE,SSE42
+; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx,+sse4a| FileCheck %s --check-prefixes=CHECK,AVX,AVX1
+; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx2,+sse4a | FileCheck %s --check-prefixes=CHECK,AVX,AVX2
 ;
 ; Combine tests involving SSE4A target shuffles (EXTRQI,INSERTQI)
 
 declare <16 x i8> @llvm.x86.ssse3.pshuf.b.128(<16 x i8>, <16 x i8>)
 
 define <16 x i8> @combine_extrqi_pshufb_16i8(<16 x i8> %a0) {
-; ALL-LABEL: combine_extrqi_pshufb_16i8:
-; ALL:       # %bb.0:
-; ALL-NEXT:    extrq {{.*#+}} xmm0 = xmm0[1,2],zero,zero,zero,zero,zero,zero,xmm0[u,u,u,u,u,u,u,u]
-; ALL-NEXT:    retq
+; CHECK-LABEL: combine_extrqi_pshufb_16i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    extrq {{.*#+}} xmm0 = xmm0[1,2],zero,zero,zero,zero,zero,zero,xmm0[u,u,u,u,u,u,u,u]
+; CHECK-NEXT:    retq
   %1 = shufflevector <16 x i8> %a0, <16 x i8> zeroinitializer, <16 x i32> <i32 1, i32 2, i32 16, i32 16, i32 16, i32 16, i32 16, i32 16, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
   %2 = tail call <16 x i8> @llvm.x86.ssse3.pshuf.b.128(<16 x i8> %1, <16 x i8> <i8 0, i8 1, i8 2, i8 3, i8 4, i8 255, i8 255, i8 255, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef>)
   ret <16 x i8> %2
 }
 
 define <8 x i16> @combine_extrqi_pshufb_8i16(<8 x i16> %a0) {
-; ALL-LABEL: combine_extrqi_pshufb_8i16:
-; ALL:       # %bb.0:
-; ALL-NEXT:    extrq {{.*#+}} xmm0 = xmm0[2,3],zero,zero,zero,zero,zero,zero,xmm0[u,u,u,u,u,u,u,u]
-; ALL-NEXT:    retq
+; CHECK-LABEL: combine_extrqi_pshufb_8i16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    extrq {{.*#+}} xmm0 = xmm0[2,3],zero,zero,zero,zero,zero,zero,xmm0[u,u,u,u,u,u,u,u]
+; CHECK-NEXT:    retq
   %1 = shufflevector <8 x i16> %a0, <8 x i16> zeroinitializer, <8 x i32> <i32 1, i32 2, i32 8, i32 8, i32 undef, i32 undef, i32 undef, i32 undef>
   %2 = bitcast <8 x i16> %1 to <16 x i8>
   %3 = tail call <16 x i8> @llvm.x86.ssse3.pshuf.b.128(<16 x i8> %2, <16 x i8> <i8 0, i8 1, i8 255, i8 255, i8 255, i8 255, i8 255, i8 255, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef>)
@@ -75,10 +75,10 @@ define <8 x i16> @combine_insertqi_pshufb_8i16(<8 x i16> %a0, <8 x i16> %a1) {
 }
 
 define <16 x i8> @combine_pshufb_insertqi_pshufb(<16 x i8> %a0, <16 x i8> %a1) {
-; ALL-LABEL: combine_pshufb_insertqi_pshufb:
-; ALL:       # %bb.0:
-; ALL-NEXT:    insertq {{.*#+}} xmm0 = xmm0[0],xmm1[0,1],xmm0[3,4,5,6,7,u,u,u,u,u,u,u,u]
-; ALL-NEXT:    retq
+; CHECK-LABEL: combine_pshufb_insertqi_pshufb:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    insertq {{.*#+}} xmm0 = xmm0[0],xmm1[0,1],xmm0[3,4,5,6,7,u,u,u,u,u,u,u,u]
+; CHECK-NEXT:    retq
   %1 = tail call <16 x i8> @llvm.x86.ssse3.pshuf.b.128(<16 x i8> %a0, <16 x i8> <i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef>)
   %2 = shufflevector <16 x i8> %1, <16 x i8> %a1, <16 x i32> <i32 0, i32 16, i32 17, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
   %3 = tail call <16 x i8> @llvm.x86.ssse3.pshuf.b.128(<16 x i8> %2, <16 x i8> <i8 7, i8 1, i8 2, i8 4, i8 3, i8 undef, i8 undef, i8 0, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef, i8 undef>)
diff --git a/test/CodeGen/X86/vector-shuffle-combining-ssse3.ll b/test/CodeGen/X86/vector-shuffle-combining-ssse3.ll
index f32f87b7a1461bf2f25272be0a980d2664947048..a531bf68c71d07039ad3b0d16c783a217650ad91 100644
--- a/test/CodeGen/X86/vector-shuffle-combining-ssse3.ll
+++ b/test/CodeGen/X86/vector-shuffle-combining-ssse3.ll
@@ -1,9 +1,9 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+ssse3 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSSE3
-; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+sse4.1 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSE41
-; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX1
-; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx2 | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX2
-; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx512f | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX512F
+; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+ssse3 | FileCheck %s --check-prefixes=CHECK,SSE,SSSE3
+; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+sse4.1 | FileCheck %s --check-prefixes=CHECK,SSE,SSE41
+; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx | FileCheck %s --check-prefixes=CHECK,AVX,AVX1
+; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx2 | FileCheck %s --check-prefixes=CHECK,AVX,AVX2
+; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx512f | FileCheck %s --check-prefixes=CHECK,AVX,AVX512F
 ;
 ; Combine tests involving SSE3/SSSE3 target shuffles (MOVDDUP, MOVSHDUP, MOVSLDUP, PSHUFB)
 
@@ -457,9 +457,9 @@ define <16 x i8> @combine_pshufb_as_unary_unpckhwd(<16 x i8> %a0) {
 }
 
 define <8 x i16> @combine_pshufb_as_unpacklo_undef(<16 x i8> %a0) {
-; ALL-LABEL: combine_pshufb_as_unpacklo_undef:
-; ALL:       # %bb.0:
-; ALL-NEXT:    retq
+; CHECK-LABEL: combine_pshufb_as_unpacklo_undef:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    retq
   %1 = call <16 x i8> @llvm.x86.ssse3.pshuf.b.128(<16 x i8> %a0, <16 x i8> <i8 undef, i8 undef, i8 0, i8 1, i8 undef, i8 undef, i8 2, i8 3, i8 undef, i8 undef, i8 4, i8 5, i8 undef, i8 undef, i8 6, i8 7>)
   %2 = bitcast <16 x i8> %1 to <8 x i16>
   %3 = shufflevector <8 x i16> %2, <8 x i16> undef, <8 x i32> <i32 0, i32 0, i32 2, i32 2, i32 4, i32 4, i32 6, i32 6>
@@ -467,9 +467,9 @@ define <8 x i16> @combine_pshufb_as_unpacklo_undef(<16 x i8> %a0) {
 }
 
 define <16 x i8> @combine_pshufb_as_unpackhi_undef(<16 x i8> %a0) {
-; ALL-LABEL: combine_pshufb_as_unpackhi_undef:
-; ALL:       # %bb.0:
-; ALL-NEXT:    retq
+; CHECK-LABEL: combine_pshufb_as_unpackhi_undef:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    retq
   %1 = call <16 x i8> @llvm.x86.ssse3.pshuf.b.128(<16 x i8> %a0, <16 x i8> <i8 8, i8 undef, i8 9, i8 undef, i8 10, i8 undef, i8 11, i8 undef, i8 12, i8 undef, i8 13, i8 undef, i8 14, i8 undef, i8 15, i8 undef>)
   %2 = shufflevector <16 x i8> %1, <16 x i8> undef, <16 x i32> <i32 1, i32 1, i32 3, i32 3, i32 5, i32 5, i32 7, i32 7, i32 9, i32 9, i32 11, i32 11, i32 13, i32 13, i32 15, i32 15>
   ret <16 x i8> %2
diff --git a/test/CodeGen/X86/vector-shuffle-combining-xop.ll b/test/CodeGen/X86/vector-shuffle-combining-xop.ll
index 5fe0a2b460b9d1948f6b82f10ef85c4112cbdad8..4500b63041d0be26100b662cef1952548d34c930 100644
--- a/test/CodeGen/X86/vector-shuffle-combining-xop.ll
+++ b/test/CodeGen/X86/vector-shuffle-combining-xop.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+avx,+xop | FileCheck %s --check-prefix=X32 --check-prefix=X86AVX
-; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+avx2,+xop | FileCheck %s --check-prefix=X32 --check-prefix=X86AVX2
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx,+xop | FileCheck %s --check-prefix=X64 --check-prefix=X64AVX
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2,+xop | FileCheck %s --check-prefix=X64 --check-prefix=X64AVX2
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+avx,+xop | FileCheck %s --check-prefixes=CHECK,X86,AVX,X86-AVX
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+avx2,+xop | FileCheck %s --check-prefixes=CHECK,X86,AVX2,X86-AVX2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx,+xop | FileCheck %s --check-prefixes=CHECK,X64,AVX,X64-AVX
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2,+xop | FileCheck %s --check-prefixes=CHECK,X64,AVX2,X64-AVX2
 
 declare <2 x double> @llvm.x86.xop.vpermil2pd(<2 x double>, <2 x double>, <2 x i64>, i8) nounwind readnone
 declare <4 x double> @llvm.x86.xop.vpermil2pd.256(<4 x double>, <4 x double>, <4 x i64>, i8) nounwind readnone
@@ -13,211 +13,138 @@ declare <8 x float> @llvm.x86.xop.vpermil2ps.256(<8 x float>, <8 x float>, <8 x
 declare <16 x i8> @llvm.x86.xop.vpperm(<16 x i8>, <16 x i8>, <16 x i8>) nounwind readnone
 
 define <2 x double> @combine_vpermil2pd_identity(<2 x double> %a0, <2 x double> %a1) {
-; X32-LABEL: combine_vpermil2pd_identity:
-; X32:       # %bb.0:
-; X32-NEXT:    vmovaps %xmm1, %xmm0
-; X32-NEXT:    retl
-;
-; X64-LABEL: combine_vpermil2pd_identity:
-; X64:       # %bb.0:
-; X64-NEXT:    vmovaps %xmm1, %xmm0
-; X64-NEXT:    retq
+; CHECK-LABEL: combine_vpermil2pd_identity:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, %xmm0
+; CHECK-NEXT:    ret{{[l|q]}}
   %res0 = call <2 x double> @llvm.x86.xop.vpermil2pd(<2 x double> %a1, <2 x double> %a0, <2 x i64> <i64 2, i64 0>, i8 0)
   %res1 = call <2 x double> @llvm.x86.xop.vpermil2pd(<2 x double> %res0, <2 x double> undef, <2 x i64> <i64 2, i64 0>, i8 0)
   ret <2 x double> %res1
 }
 
 define <4 x double> @combine_vpermil2pd256_identity(<4 x double> %a0, <4 x double> %a1) {
-; X32-LABEL: combine_vpermil2pd256_identity:
-; X32:       # %bb.0:
-; X32-NEXT:    vmovaps %ymm1, %ymm0
-; X32-NEXT:    retl
-;
-; X64-LABEL: combine_vpermil2pd256_identity:
-; X64:       # %bb.0:
-; X64-NEXT:    vmovaps %ymm1, %ymm0
-; X64-NEXT:    retq
+; CHECK-LABEL: combine_vpermil2pd256_identity:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %ymm1, %ymm0
+; CHECK-NEXT:    ret{{[l|q]}}
   %res0 = call <4 x double> @llvm.x86.xop.vpermil2pd.256(<4 x double> %a1, <4 x double> %a0, <4 x i64> <i64 2, i64 0, i64 2, i64 0>, i8 0)
   %res1 = call <4 x double> @llvm.x86.xop.vpermil2pd.256(<4 x double> %res0, <4 x double> undef, <4 x i64> <i64 2, i64 0, i64 2, i64 0>, i8 0)
   ret <4 x double> %res1
 }
 
 define <4 x double> @combine_vpermil2pd256_0z73(<4 x double> %a0, <4 x double> %a1) {
-; X32-LABEL: combine_vpermil2pd256_0z73:
-; X32:       # %bb.0:
-; X32-NEXT:    vpermil2pd {{.*#+}} ymm0 = ymm0[0],zero,ymm1[3],ymm0[3]
-; X32-NEXT:    retl
-;
-; X64-LABEL: combine_vpermil2pd256_0z73:
-; X64:       # %bb.0:
-; X64-NEXT:    vpermil2pd {{.*#+}} ymm0 = ymm0[0],zero,ymm1[3],ymm0[3]
-; X64-NEXT:    retq
+; CHECK-LABEL: combine_vpermil2pd256_0z73:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpermil2pd {{.*#+}} ymm0 = ymm0[0],zero,ymm1[3],ymm0[3]
+; CHECK-NEXT:    ret{{[l|q]}}
   %res0 = shufflevector <4 x double> %a0, <4 x double> %a1, <4 x i32> <i32 0, i32 undef, i32 7, i32 3>
   %res1 = shufflevector <4 x double> %res0, <4 x double> zeroinitializer, <4 x i32> <i32 0, i32 7, i32 2, i32 3>
   ret <4 x double> %res1
 }
 
 define <4 x float> @combine_vpermil2ps_identity(<4 x float> %a0, <4 x float> %a1) {
-; X32-LABEL: combine_vpermil2ps_identity:
-; X32:       # %bb.0:
-; X32-NEXT:    vmovaps %xmm1, %xmm0
-; X32-NEXT:    retl
-;
-; X64-LABEL: combine_vpermil2ps_identity:
-; X64:       # %bb.0:
-; X64-NEXT:    vmovaps %xmm1, %xmm0
-; X64-NEXT:    retq
+; CHECK-LABEL: combine_vpermil2ps_identity:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, %xmm0
+; CHECK-NEXT:    ret{{[l|q]}}
   %res0 = call <4 x float> @llvm.x86.xop.vpermil2ps(<4 x float> %a1, <4 x float> %a0, <4 x i32> <i32 3, i32 2, i32 1, i32 0>, i8 0)
   %res1 = call <4 x float> @llvm.x86.xop.vpermil2ps(<4 x float> %res0, <4 x float> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>, i8 0)
   ret <4 x float> %res1
 }
 
 define <4 x float> @combine_vpermil2ps_1z74(<4 x float> %a0, <4 x float> %a1) {
-; X32-LABEL: combine_vpermil2ps_1z74:
-; X32:       # %bb.0:
-; X32-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[1,1],xmm1[3,0]
-; X32-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; X32-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2,3]
-; X32-NEXT:    retl
-;
-; X64-LABEL: combine_vpermil2ps_1z74:
-; X64:       # %bb.0:
-; X64-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[1,1],xmm1[3,0]
-; X64-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; X64-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2,3]
-; X64-NEXT:    retq
+; CHECK-LABEL: combine_vpermil2ps_1z74:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[1,1],xmm1[3,0]
+; CHECK-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; CHECK-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2,3]
+; CHECK-NEXT:    ret{{[l|q]}}
   %res0 = call <4 x float> @llvm.x86.xop.vpermil2ps(<4 x float> %a0, <4 x float> %a1, <4 x i32> <i32 1, i32 1, i32 7, i32 4>, i8 0)
   %res1 = shufflevector <4 x float> %res0, <4 x float> zeroinitializer, <4 x i32> <i32 0, i32 7, i32 2, i32 3>
   ret <4 x float> %res1
 }
 
 define <4 x float> @combine_vpermil2ps_02zu(<4 x float> %a0, <4 x float> %a1) {
-; X32-LABEL: combine_vpermil2ps_02zu:
-; X32:       # %bb.0:
-; X32-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,2],zero,zero
-; X32-NEXT:    retl
-;
-; X64-LABEL: combine_vpermil2ps_02zu:
-; X64:       # %bb.0:
-; X64-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,2],zero,zero
-; X64-NEXT:    retq
+; CHECK-LABEL: combine_vpermil2ps_02zu:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,2],zero,zero
+; CHECK-NEXT:    ret{{[l|q]}}
   %res0 = call <4 x float> @llvm.x86.xop.vpermil2ps(<4 x float> %a0, <4 x float> zeroinitializer, <4 x i32> <i32 0, i32 2, i32 4, i32 undef>, i8 0)
   ret <4 x float> %res0
 }
 
 define <8 x float> @combine_vpermil2ps256_identity(<8 x float> %a0, <8 x float> %a1) {
-; X32-LABEL: combine_vpermil2ps256_identity:
-; X32:       # %bb.0:
-; X32-NEXT:    vmovaps %ymm1, %ymm0
-; X32-NEXT:    retl
-;
-; X64-LABEL: combine_vpermil2ps256_identity:
-; X64:       # %bb.0:
-; X64-NEXT:    vmovaps %ymm1, %ymm0
-; X64-NEXT:    retq
+; CHECK-LABEL: combine_vpermil2ps256_identity:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %ymm1, %ymm0
+; CHECK-NEXT:    ret{{[l|q]}}
   %res0 = call <8 x float> @llvm.x86.xop.vpermil2ps.256(<8 x float> %a1, <8 x float> %a0, <8 x i32> <i32 3, i32 2, i32 1, i32 0, i32 1, i32 0, i32 3, i32 2>, i8 0)
   %res1 = call <8 x float> @llvm.x86.xop.vpermil2ps.256(<8 x float> %res0, <8 x float> undef, <8 x i32> <i32 3, i32 2, i32 1, i32 0, i32 1, i32 0, i32 3, i32 2>, i8 0)
   ret <8 x float> %res1
 }
 
 define <8 x float> @combine_vpermil2ps256_08z945Az(<8 x float> %a0, <8 x float> %a1) {
-; X32-LABEL: combine_vpermil2ps256_08z945Az:
-; X32:       # %bb.0:
-; X32-NEXT:    vpermil2ps {{.*#+}} ymm0 = ymm0[0],ymm1[0],zero,ymm1[1],ymm0[4,5],ymm1[6],zero
-; X32-NEXT:    retl
-;
-; X64-LABEL: combine_vpermil2ps256_08z945Az:
-; X64:       # %bb.0:
-; X64-NEXT:    vpermil2ps {{.*#+}} ymm0 = ymm0[0],ymm1[0],zero,ymm1[1],ymm0[4,5],ymm1[6],zero
-; X64-NEXT:    retq
+; CHECK-LABEL: combine_vpermil2ps256_08z945Az:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpermil2ps {{.*#+}} ymm0 = ymm0[0],ymm1[0],zero,ymm1[1],ymm0[4,5],ymm1[6],zero
+; CHECK-NEXT:    ret{{[l|q]}}
   %res0 = call <8 x float> @llvm.x86.xop.vpermil2ps.256(<8 x float> %a0, <8 x float> %a1, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 0, i32 1, i32 6, i32 7>, i8 0)
   %res1 = shufflevector <8 x float> %res0, <8 x float> zeroinitializer, <8 x i32> <i32 0, i32 1, i32 8, i32 3, i32 4, i32 5, i32 6, i32 8>
   ret <8 x float> %res1
 }
 
 define <8 x float> @combine_vpermil2ps256_zero(<8 x float> %a0, <8 x float> %a1) {
-; X32-LABEL: combine_vpermil2ps256_zero:
-; X32:       # %bb.0:
-; X32-NEXT:    vxorps %xmm0, %xmm0, %xmm0
-; X32-NEXT:    retl
-;
-; X64-LABEL: combine_vpermil2ps256_zero:
-; X64:       # %bb.0:
-; X64-NEXT:    vxorps %xmm0, %xmm0, %xmm0
-; X64-NEXT:    retq
+; CHECK-LABEL: combine_vpermil2ps256_zero:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vxorps %xmm0, %xmm0, %xmm0
+; CHECK-NEXT:    ret{{[l|q]}}
   %res0 = call <8 x float> @llvm.x86.xop.vpermil2ps.256(<8 x float> %a1, <8 x float> %a0, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 8, i32 9, i32 10, i32 11>, i8 2)
   ret <8 x float> %res0
 }
 
 define <4 x float> @combine_vpermil2ps_blend_with_zero(<4 x float> %a0, <4 x float> %a1) {
-; X32-LABEL: combine_vpermil2ps_blend_with_zero:
-; X32:       # %bb.0:
-; X32-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; X32-NEXT:    vblendps {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
-; X32-NEXT:    retl
-;
-; X64-LABEL: combine_vpermil2ps_blend_with_zero:
-; X64:       # %bb.0:
-; X64-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; X64-NEXT:    vblendps {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
-; X64-NEXT:    retq
+; CHECK-LABEL: combine_vpermil2ps_blend_with_zero:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; CHECK-NEXT:    vblendps {{.*#+}} xmm0 = xmm1[0],xmm0[1,2,3]
+; CHECK-NEXT:    ret{{[l|q]}}
   %res0 = call <4 x float> @llvm.x86.xop.vpermil2ps(<4 x float> %a0, <4 x float> %a1, <4 x i32> <i32 8, i32 1, i32 2, i32 3>, i8 2)
   ret <4 x float> %res0
 }
 
 define <2 x double> @combine_vpermil2pd_as_shufpd(<2 x double> %a0, <2 x double> %a1) {
-; X32-LABEL: combine_vpermil2pd_as_shufpd:
-; X32:       # %bb.0:
-; X32-NEXT:    vshufpd {{.*#+}} xmm0 = xmm0[1],xmm1[0]
-; X32-NEXT:    retl
-;
-; X64-LABEL: combine_vpermil2pd_as_shufpd:
-; X64:       # %bb.0:
-; X64-NEXT:    vshufpd {{.*#+}} xmm0 = xmm0[1],xmm1[0]
-; X64-NEXT:    retq
+; CHECK-LABEL: combine_vpermil2pd_as_shufpd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vshufpd {{.*#+}} xmm0 = xmm0[1],xmm1[0]
+; CHECK-NEXT:    ret{{[l|q]}}
   %res0 = call <2 x double> @llvm.x86.xop.vpermil2pd(<2 x double> %a0, <2 x double> %a1, <2 x i64> <i64 2, i64 4>, i8 0)
   ret <2 x double> %res0
 }
 
 define <4 x double> @combine_vpermil2pd256_as_shufpd(<4 x double> %a0, <4 x double> %a1) {
-; X32-LABEL: combine_vpermil2pd256_as_shufpd:
-; X32:       # %bb.0:
-; X32-NEXT:    vshufpd {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[3],ymm1[3]
-; X32-NEXT:    retl
-;
-; X64-LABEL: combine_vpermil2pd256_as_shufpd:
-; X64:       # %bb.0:
-; X64-NEXT:    vshufpd {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[3],ymm1[3]
-; X64-NEXT:    retq
+; CHECK-LABEL: combine_vpermil2pd256_as_shufpd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vshufpd {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[3],ymm1[3]
+; CHECK-NEXT:    ret{{[l|q]}}
   %res0 = call <4 x double> @llvm.x86.xop.vpermil2pd.256(<4 x double> %a0, <4 x double> %a1, <4 x i64> <i64 0, i64 4, i64 2, i64 7>, i8 0)
   ret <4 x double> %res0
 }
 
 define <16 x i8> @combine_vpperm_identity(<16 x i8> %a0, <16 x i8> %a1) {
-; X32-LABEL: combine_vpperm_identity:
-; X32:       # %bb.0:
-; X32-NEXT:    vmovaps %xmm1, %xmm0
-; X32-NEXT:    retl
-;
-; X64-LABEL: combine_vpperm_identity:
-; X64:       # %bb.0:
-; X64-NEXT:    vmovaps %xmm1, %xmm0
-; X64-NEXT:    retq
+; CHECK-LABEL: combine_vpperm_identity:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, %xmm0
+; CHECK-NEXT:    ret{{[l|q]}}
   %res0 = call <16 x i8> @llvm.x86.xop.vpperm(<16 x i8> %a0, <16 x i8> %a1, <16 x i8> <i8 31, i8 30, i8 29, i8 28, i8 27, i8 26, i8 25, i8 24, i8 23, i8 22, i8 21, i8 20, i8 19, i8 18, i8 17, i8 16>)
   %res1 = call <16 x i8> @llvm.x86.xop.vpperm(<16 x i8> %res0, <16 x i8> undef, <16 x i8> <i8 15, i8 14, i8 13, i8 12, i8 11, i8 10, i8 9, i8 8, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0>)
   ret <16 x i8> %res1
 }
 
 define <16 x i8> @combine_vpperm_zero(<16 x i8> %a0, <16 x i8> %a1) {
-; X32-LABEL: combine_vpperm_zero:
-; X32:       # %bb.0:
-; X32-NEXT:    vxorps %xmm0, %xmm0, %xmm0
-; X32-NEXT:    retl
-;
-; X64-LABEL: combine_vpperm_zero:
-; X64:       # %bb.0:
-; X64-NEXT:    vxorps %xmm0, %xmm0, %xmm0
-; X64-NEXT:    retq
+; CHECK-LABEL: combine_vpperm_zero:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vxorps %xmm0, %xmm0, %xmm0
+; CHECK-NEXT:    ret{{[l|q]}}
   %res0 = call <16 x i8> @llvm.x86.xop.vpperm(<16 x i8> %a0, <16 x i8> %a1, <16 x i8> <i8 128, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0>)
   %res1 = call <16 x i8> @llvm.x86.xop.vpperm(<16 x i8> %res0, <16 x i8> undef, <16 x i8> <i8 0, i8 128, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0>)
   %res2 = call <16 x i8> @llvm.x86.xop.vpperm(<16 x i8> %res1, <16 x i8> undef, <16 x i8> <i8 0, i8 1, i8 128, i8 128, i8 128, i8 128, i8 128, i8 128, i8 128, i8 128, i8 128, i8 128, i8 128, i8 128, i8 128, i8 128>)
@@ -225,10 +152,10 @@ define <16 x i8> @combine_vpperm_zero(<16 x i8> %a0, <16 x i8> %a1) {
 }
 
 define <16 x i8> @combine_vpperm_identity_bitcast(<16 x i8> %a0, <16 x i8> %a1) {
-; X32-LABEL: combine_vpperm_identity_bitcast:
-; X32:       # %bb.0:
-; X32-NEXT:    vpaddq {{\.LCPI.*}}, %xmm0, %xmm0
-; X32-NEXT:    retl
+; X86-LABEL: combine_vpperm_identity_bitcast:
+; X86:       # %bb.0:
+; X86-NEXT:    vpaddq {{\.LCPI.*}}, %xmm0, %xmm0
+; X86-NEXT:    retl
 ;
 ; X64-LABEL: combine_vpperm_identity_bitcast:
 ; X64:       # %bb.0:
@@ -244,73 +171,47 @@ define <16 x i8> @combine_vpperm_identity_bitcast(<16 x i8> %a0, <16 x i8> %a1)
 }
 
 define <16 x i8> @combine_vpperm_as_blend_with_zero(<16 x i8> %a0, <16 x i8> %a1) {
-; X32-LABEL: combine_vpperm_as_blend_with_zero:
-; X32:       # %bb.0:
-; X32-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; X32-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2,3],xmm1[4,5,6,7]
-; X32-NEXT:    retl
-;
-; X64-LABEL: combine_vpperm_as_blend_with_zero:
-; X64:       # %bb.0:
-; X64-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; X64-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2,3],xmm1[4,5,6,7]
-; X64-NEXT:    retq
+; CHECK-LABEL: combine_vpperm_as_blend_with_zero:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; CHECK-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2,3],xmm1[4,5,6,7]
+; CHECK-NEXT:    ret{{[l|q]}}
   %res0 = call <16 x i8> @llvm.x86.xop.vpperm(<16 x i8> %a0, <16 x i8> %a1, <16 x i8> <i8 0, i8 1, i8 128, i8 129, i8 4, i8 5, i8 6, i8 7, i8 130, i8 131, i8 132, i8 133, i8 134, i8 135, i8 136, i8 137>)
   ret <16 x i8> %res0
 }
 
 define <16 x i8> @combine_vpperm_as_unary_unpckhbw(<16 x i8> %a0, <16 x i8> %a1) {
-; X32-LABEL: combine_vpperm_as_unary_unpckhbw:
-; X32:       # %bb.0:
-; X32-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; X32-NEXT:    retl
-;
-; X64-LABEL: combine_vpperm_as_unary_unpckhbw:
-; X64:       # %bb.0:
-; X64-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; X64-NEXT:    retq
+; CHECK-LABEL: combine_vpperm_as_unary_unpckhbw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; CHECK-NEXT:    ret{{[l|q]}}
   %res0 = call <16 x i8> @llvm.x86.xop.vpperm(<16 x i8> %a0, <16 x i8> %a0, <16 x i8> <i8 8, i8 undef, i8 9, i8 25, i8 10, i8 26, i8 11, i8 27, i8 12, i8 28, i8 13, i8 29, i8 14, i8 30, i8 15, i8 31>)
   ret <16 x i8> %res0
 }
 
 define <16 x i8> @combine_vpperm_as_unpckhbw(<16 x i8> %a0, <16 x i8> %a1) {
-; X32-LABEL: combine_vpperm_as_unpckhbw:
-; X32:       # %bb.0:
-; X32-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm0[8],xmm1[8],xmm0[9],xmm1[9],xmm0[10],xmm1[10],xmm0[11],xmm1[11],xmm0[12],xmm1[12],xmm0[13],xmm1[13],xmm0[14],xmm1[14],xmm0[15],xmm1[15]
-; X32-NEXT:    retl
-;
-; X64-LABEL: combine_vpperm_as_unpckhbw:
-; X64:       # %bb.0:
-; X64-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm0[8],xmm1[8],xmm0[9],xmm1[9],xmm0[10],xmm1[10],xmm0[11],xmm1[11],xmm0[12],xmm1[12],xmm0[13],xmm1[13],xmm0[14],xmm1[14],xmm0[15],xmm1[15]
-; X64-NEXT:    retq
+; CHECK-LABEL: combine_vpperm_as_unpckhbw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm0[8],xmm1[8],xmm0[9],xmm1[9],xmm0[10],xmm1[10],xmm0[11],xmm1[11],xmm0[12],xmm1[12],xmm0[13],xmm1[13],xmm0[14],xmm1[14],xmm0[15],xmm1[15]
+; CHECK-NEXT:    ret{{[l|q]}}
   %res0 = call <16 x i8> @llvm.x86.xop.vpperm(<16 x i8> %a0, <16 x i8> %a1, <16 x i8> <i8 8, i8 24, i8 9, i8 25, i8 10, i8 26, i8 11, i8 27, i8 12, i8 28, i8 13, i8 29, i8 14, i8 30, i8 15, i8 31>)
   ret <16 x i8> %res0
 }
 
 define <16 x i8> @combine_vpperm_as_unpcklbw(<16 x i8> %a0, <16 x i8> %a1) {
-; X32-LABEL: combine_vpperm_as_unpcklbw:
-; X32:       # %bb.0:
-; X32-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
-; X32-NEXT:    retl
-;
-; X64-LABEL: combine_vpperm_as_unpcklbw:
-; X64:       # %bb.0:
-; X64-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
-; X64-NEXT:    retq
+; CHECK-LABEL: combine_vpperm_as_unpcklbw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; CHECK-NEXT:    ret{{[l|q]}}
   %res0 = call <16 x i8> @llvm.x86.xop.vpperm(<16 x i8> %a0, <16 x i8> %a1, <16 x i8> <i8 16, i8 0, i8 17, i8 1, i8 18, i8 2, i8 19, i8 3, i8 20, i8 4, i8 21, i8 5, i8 22, i8 6, i8 23, i8 7>)
   ret <16 x i8> %res0
 }
 
 define <4 x i32> @combine_vpperm_10zz32BA(<4 x i32> %a0, <4 x i32> %a1) {
-; X32-LABEL: combine_vpperm_10zz32BA:
-; X32:       # %bb.0:
-; X32-NEXT:    vpperm {{.*#+}} xmm0 = xmm0[2,3,0,1],zero,zero,zero,zero,xmm0[6,7,4,5],xmm1[6,7,4,5]
-; X32-NEXT:    retl
-;
-; X64-LABEL: combine_vpperm_10zz32BA:
-; X64:       # %bb.0:
-; X64-NEXT:    vpperm {{.*#+}} xmm0 = xmm0[2,3,0,1],zero,zero,zero,zero,xmm0[6,7,4,5],xmm1[6,7,4,5]
-; X64-NEXT:    retq
+; CHECK-LABEL: combine_vpperm_10zz32BA:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpperm {{.*#+}} xmm0 = xmm0[2,3,0,1],zero,zero,zero,zero,xmm0[6,7,4,5],xmm1[6,7,4,5]
+; CHECK-NEXT:    ret{{[l|q]}}
   %res0 = shufflevector <4 x i32> %a0, <4 x i32> %a1, <4 x i32> <i32 0, i32 4, i32 1, i32 5>
   %res1 = bitcast <4 x i32> %res0 to <16 x i8>
   %res2 = call <16 x i8> @llvm.x86.xop.vpperm(<16 x i8> %res1, <16 x i8> undef, <16 x i8> <i8 2, i8 3, i8 0, i8 1, i8 128, i8 128, i8 128, i8 128, i8 10, i8 11, i8 8, i8 9, i8 14, i8 15, i8 12, i8 13>)
@@ -320,35 +221,34 @@ define <4 x i32> @combine_vpperm_10zz32BA(<4 x i32> %a0, <4 x i32> %a1) {
 
 ; FIXME: Duplicated load in i686
 define void @buildvector_v4f32_0404(float %a, float %b, <4 x float>* %ptr) {
-; X86AVX-LABEL: buildvector_v4f32_0404:
-; X86AVX:       # %bb.0:
-; X86AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
-; X86AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,1],mem[0],xmm0[3]
-; X86AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,1,2],mem[0]
-; X86AVX-NEXT:    vmovaps %xmm0, (%eax)
-; X86AVX-NEXT:    retl
+; X86-AVX-LABEL: buildvector_v4f32_0404:
+; X86-AVX:       # %bb.0:
+; X86-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-AVX-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; X86-AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,1],mem[0],xmm0[3]
+; X86-AVX-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0,1,2],mem[0]
+; X86-AVX-NEXT:    vmovaps %xmm0, (%eax)
+; X86-AVX-NEXT:    retl
 ;
-; X86AVX2-LABEL: buildvector_v4f32_0404:
-; X86AVX2:       # %bb.0:
-; X86AVX2-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86AVX2-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
-; X86AVX2-NEXT:    vmovddup {{.*#+}} xmm0 = xmm0[0,0]
-; X86AVX2-NEXT:    vmovapd %xmm0, (%eax)
-; X86AVX2-NEXT:    retl
+; X86-AVX2-LABEL: buildvector_v4f32_0404:
+; X86-AVX2:       # %bb.0:
+; X86-AVX2-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-AVX2-NEXT:    vmovddup {{.*#+}} xmm0 = mem[0,0]
+; X86-AVX2-NEXT:    vmovaps %xmm0, (%eax)
+; X86-AVX2-NEXT:    retl
 ;
-; X64AVX-LABEL: buildvector_v4f32_0404:
-; X64AVX:       # %bb.0:
-; X64AVX-NEXT:    vpermil2ps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[0],xmm1[0]
-; X64AVX-NEXT:    vmovaps %xmm0, (%rdi)
-; X64AVX-NEXT:    retq
+; X64-AVX-LABEL: buildvector_v4f32_0404:
+; X64-AVX:       # %bb.0:
+; X64-AVX-NEXT:    vpermil2ps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[0],xmm1[0]
+; X64-AVX-NEXT:    vmovaps %xmm0, (%rdi)
+; X64-AVX-NEXT:    retq
 ;
-; X64AVX2-LABEL: buildvector_v4f32_0404:
-; X64AVX2:       # %bb.0:
-; X64AVX2-NEXT:    vunpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; X64AVX2-NEXT:    vmovddup {{.*#+}} xmm0 = xmm0[0,0]
-; X64AVX2-NEXT:    vmovapd %xmm0, (%rdi)
-; X64AVX2-NEXT:    retq
+; X64-AVX2-LABEL: buildvector_v4f32_0404:
+; X64-AVX2:       # %bb.0:
+; X64-AVX2-NEXT:    vunpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; X64-AVX2-NEXT:    vmovddup {{.*#+}} xmm0 = xmm0[0,0]
+; X64-AVX2-NEXT:    vmovaps %xmm0, (%rdi)
+; X64-AVX2-NEXT:    retq
   %v0 = insertelement <4 x float> undef, float %a, i32 0
   %v1 = insertelement <4 x float> %v0,   float %b, i32 1
   %v2 = insertelement <4 x float> %v1,   float %a, i32 2
@@ -358,13 +258,13 @@ define void @buildvector_v4f32_0404(float %a, float %b, <4 x float>* %ptr) {
 }
 
 define void @buildvector_v4f32_07z6(float %a, <4 x float> %b, <4 x float>* %ptr) {
-; X32-LABEL: buildvector_v4f32_07z6:
-; X32:       # %bb.0:
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; X32-NEXT:    vpermil2ps {{.*#+}} xmm0 = xmm1[0],xmm0[3],zero,xmm0[2]
-; X32-NEXT:    vmovaps %xmm0, (%eax)
-; X32-NEXT:    retl
+; X86-LABEL: buildvector_v4f32_07z6:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; X86-NEXT:    vpermil2ps {{.*#+}} xmm0 = xmm1[0],xmm0[3],zero,xmm0[2]
+; X86-NEXT:    vmovaps %xmm0, (%eax)
+; X86-NEXT:    retl
 ;
 ; X64-LABEL: buildvector_v4f32_07z6:
 ; X64:       # %bb.0:
@@ -382,82 +282,57 @@ define void @buildvector_v4f32_07z6(float %a, <4 x float> %b, <4 x float>* %ptr)
 }
 
 define <2 x double> @constant_fold_vpermil2pd() {
-; X32-LABEL: constant_fold_vpermil2pd:
-; X32:       # %bb.0:
-; X32-NEXT:    vmovaps {{.*#+}} xmm0 = [-2.0E+0,2.0E+0]
-; X32-NEXT:    retl
-;
-; X64-LABEL: constant_fold_vpermil2pd:
-; X64:       # %bb.0:
-; X64-NEXT:    vmovaps {{.*#+}} xmm0 = [-2.0E+0,2.0E+0]
-; X64-NEXT:    retq
+; CHECK-LABEL: constant_fold_vpermil2pd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps {{.*#+}} xmm0 = [-2.0E+0,2.0E+0]
+; CHECK-NEXT:    ret{{[l|q]}}
   %1 = call <2 x double> @llvm.x86.xop.vpermil2pd(<2 x double> <double 1.0, double 2.0>, <2 x double> <double -2.0, double -1.0>, <2 x i64> <i64 4, i64 2>, i8 2)
   ret <2 x double> %1
 }
 
 define <4 x double> @constant_fold_vpermil2pd_256() {
-; X32-LABEL: constant_fold_vpermil2pd_256:
-; X32:       # %bb.0:
-; X32-NEXT:    vmovaps {{.*#+}} ymm0 = [-4.0E+0,0.0E+0,4.0E+0,3.0E+0]
-; X32-NEXT:    retl
-;
-; X64-LABEL: constant_fold_vpermil2pd_256:
-; X64:       # %bb.0:
-; X64-NEXT:    vmovaps {{.*#+}} ymm0 = [-4.0E+0,0.0E+0,4.0E+0,3.0E+0]
-; X64-NEXT:    retq
+; CHECK-LABEL: constant_fold_vpermil2pd_256:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps {{.*#+}} ymm0 = [-4.0E+0,0.0E+0,4.0E+0,3.0E+0]
+; CHECK-NEXT:    ret{{[l|q]}}
   %1 = call <4 x double> @llvm.x86.xop.vpermil2pd.256(<4 x double> <double 1.0, double 2.0, double 3.0, double 4.0>, <4 x double> <double -4.0, double -3.0, double -2.0, double -1.0>, <4 x i64> <i64 4, i64 8, i64 2, i64 0>, i8 2)
   ret <4 x double> %1
 }
 
 define <4 x float> @constant_fold_vpermil2ps() {
-; X32-LABEL: constant_fold_vpermil2ps:
-; X32:       # %bb.0:
-; X32-NEXT:    vmovaps {{.*#+}} xmm0 = [-4.0E+0,1.0E+0,3.0E+0,0.0E+0]
-; X32-NEXT:    retl
-;
-; X64-LABEL: constant_fold_vpermil2ps:
-; X64:       # %bb.0:
-; X64-NEXT:    vmovaps {{.*#+}} xmm0 = [-4.0E+0,1.0E+0,3.0E+0,0.0E+0]
-; X64-NEXT:    retq
+; CHECK-LABEL: constant_fold_vpermil2ps:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps {{.*#+}} xmm0 = [-4.0E+0,1.0E+0,3.0E+0,0.0E+0]
+; CHECK-NEXT:    ret{{[l|q]}}
   %1 = call <4 x float> @llvm.x86.xop.vpermil2ps(<4 x float> <float 1.0, float 2.0, float 3.0, float 4.0>, <4 x float> <float -4.0, float -3.0, float -2.0, float -1.0>, <4 x i32> <i32 4, i32 0, i32 2, i32 8>, i8 2)
   ret <4 x float> %1
 }
 
 define <8 x float> @constant_fold_vpermil2ps_256() {
-; X32-LABEL: constant_fold_vpermil2ps_256:
-; X32:       # %bb.0:
-; X32-NEXT:    vmovaps {{.*#+}} ymm0 = [-8.0E+0,1.0E+0,3.0E+0,0.0E+0,5.0E+0,0.0E+0,5.0E+0,7.0E+0]
-; X32-NEXT:    retl
-;
-; X64-LABEL: constant_fold_vpermil2ps_256:
-; X64:       # %bb.0:
-; X64-NEXT:    vmovaps {{.*#+}} ymm0 = [-8.0E+0,1.0E+0,3.0E+0,0.0E+0,5.0E+0,0.0E+0,5.0E+0,7.0E+0]
-; X64-NEXT:    retq
+; CHECK-LABEL: constant_fold_vpermil2ps_256:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps {{.*#+}} ymm0 = [-8.0E+0,1.0E+0,3.0E+0,0.0E+0,5.0E+0,0.0E+0,5.0E+0,7.0E+0]
+; CHECK-NEXT:    ret{{[l|q]}}
   %1 = call <8 x float> @llvm.x86.xop.vpermil2ps.256(<8 x float> <float 1.0, float 2.0, float 3.0, float 4.0, float 5.0, float 6.0, float 7.0, float 8.0>, <8 x float> <float -8.0, float -7.0, float -6.0, float -5.0, float -4.0, float -3.0, float -2.0, float -1.0>, <8 x i32> <i32 4, i32 0, i32 2, i32 8, i32 0, i32 8, i32 0, i32 2>, i8 2)
   ret <8 x float> %1
 }
 
 define <16 x i8> @constant_fold_vpperm() {
-; X32-LABEL: constant_fold_vpperm:
-; X32:       # %bb.0:
-; X32-NEXT:    vmovaps {{.*#+}} xmm0 = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15]
-; X32-NEXT:    retl
-;
-; X64-LABEL: constant_fold_vpperm:
-; X64:       # %bb.0:
-; X64-NEXT:    vmovaps {{.*#+}} xmm0 = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15]
-; X64-NEXT:    retq
+; CHECK-LABEL: constant_fold_vpperm:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps {{.*#+}} xmm0 = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15]
+; CHECK-NEXT:    ret{{[l|q]}}
   %1 = call <16 x i8> @llvm.x86.xop.vpperm(<16 x i8> <i8 0, i8 -1, i8 -2, i8 -3, i8 -4, i8 -5, i8 -6, i8 -7, i8 -8, i8 -9, i8 -10, i8 -11, i8 -12, i8 -13, i8 -14, i8 -15>, <16 x i8> <i8 15, i8 14, i8 13, i8 12, i8 11, i8 10, i8 9, i8 8, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0>, <16 x i8> <i8 31, i8 30, i8 29, i8 28, i8 27, i8 26, i8 25, i8 24, i8 23, i8 22, i8 21, i8 20, i8 19, i8 18, i8 17, i8 16>)
   ret <16 x i8> %1
 }
 
 define <4 x float> @PR31296(i8* %in) {
-; X32-LABEL: PR31296:
-; X32:       # %bb.0: # %entry
-; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; X32-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0],zero,zero,mem[0]
-; X32-NEXT:    retl
+; X86-LABEL: PR31296:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; X86-NEXT:    vinsertps {{.*#+}} xmm0 = xmm0[0],zero,zero,mem[0]
+; X86-NEXT:    retl
 ;
 ; X64-LABEL: PR31296:
 ; X64:       # %bb.0: # %entry
diff --git a/test/CodeGen/X86/vector-shuffle-combining.ll b/test/CodeGen/X86/vector-shuffle-combining.ll
index 199a05eb2b8fcfd50b562ae7de514ea4ee0803e0..5833c3c390d322abe076dc7fe4ca3a58273a0cf1 100644
--- a/test/CodeGen/X86/vector-shuffle-combining.ll
+++ b/test/CodeGen/X86/vector-shuffle-combining.ll
@@ -1,10 +1,10 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mcpu=x86-64 -mattr=+sse2 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSE2
-; RUN: llc < %s -mcpu=x86-64 -mattr=+ssse3 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSSE3
-; RUN: llc < %s -mcpu=x86-64 -mattr=+sse4.1 | FileCheck %s --check-prefix=ALL --check-prefix=SSE --check-prefix=SSE41
-; RUN: llc < %s -mcpu=x86-64 -mattr=+avx | FileCheck %s --check-prefix=ALL --check-prefix=AVX --check-prefix=AVX1
-; RUN: llc < %s -mcpu=x86-64 -mattr=+avx2 | FileCheck %s --check-prefixes=ALL,AVX,AVX2,AVX2-SLOW
-; RUN: llc < %s -mcpu=x86-64 -mattr=+avx2,+fast-variable-shuffle | FileCheck %s --check-prefixes=ALL,AVX,AVX2,AVX2-FAST
+; RUN: llc < %s -mcpu=x86-64 -mattr=+sse2 | FileCheck %s --check-prefixes=CHECK,SSE,SSE2
+; RUN: llc < %s -mcpu=x86-64 -mattr=+ssse3 | FileCheck %s --check-prefixes=CHECK,SSE,SSSE3
+; RUN: llc < %s -mcpu=x86-64 -mattr=+sse4.1 | FileCheck %s --check-prefixes=CHECK,SSE,SSE41
+; RUN: llc < %s -mcpu=x86-64 -mattr=+avx | FileCheck %s --check-prefixes=CHECK,AVX,AVX1
+; RUN: llc < %s -mcpu=x86-64 -mattr=+avx2 | FileCheck %s --check-prefixes=CHECK,AVX,AVX2,AVX2-SLOW
+; RUN: llc < %s -mcpu=x86-64 -mattr=+avx2,+fast-variable-shuffle | FileCheck %s --check-prefixes=CHECK,AVX,AVX2,AVX2-FAST
 ;
 ; Verify that the DAG combiner correctly folds bitwise operations across
 ; shuffles, nested shuffles with undef, pairs of nested shuffles, and other
@@ -18,9 +18,9 @@ declare <8 x i16> @llvm.x86.sse2.pshufl.w(<8 x i16>, i8)
 declare <8 x i16> @llvm.x86.sse2.pshufh.w(<8 x i16>, i8)
 
 define <4 x i32> @combine_pshufd1(<4 x i32> %a) {
-; ALL-LABEL: combine_pshufd1:
-; ALL:       # %bb.0: # %entry
-; ALL-NEXT:    retq
+; CHECK-LABEL: combine_pshufd1:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    retq
 entry:
   %b = call <4 x i32> @llvm.x86.sse2.pshuf.d(<4 x i32> %a, i8 27)
   %c = call <4 x i32> @llvm.x86.sse2.pshuf.d(<4 x i32> %b, i8 27)
@@ -28,9 +28,9 @@ entry:
 }
 
 define <4 x i32> @combine_pshufd2(<4 x i32> %a) {
-; ALL-LABEL: combine_pshufd2:
-; ALL:       # %bb.0: # %entry
-; ALL-NEXT:    retq
+; CHECK-LABEL: combine_pshufd2:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    retq
 entry:
   %b = call <4 x i32> @llvm.x86.sse2.pshuf.d(<4 x i32> %a, i8 27)
   %b.cast = bitcast <4 x i32> %b to <8 x i16>
@@ -41,9 +41,9 @@ entry:
 }
 
 define <4 x i32> @combine_pshufd3(<4 x i32> %a) {
-; ALL-LABEL: combine_pshufd3:
-; ALL:       # %bb.0: # %entry
-; ALL-NEXT:    retq
+; CHECK-LABEL: combine_pshufd3:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    retq
 entry:
   %b = call <4 x i32> @llvm.x86.sse2.pshuf.d(<4 x i32> %a, i8 27)
   %b.cast = bitcast <4 x i32> %b to <8 x i16>
@@ -113,9 +113,9 @@ entry:
 }
 
 define <8 x i16> @combine_pshuflw1(<8 x i16> %a) {
-; ALL-LABEL: combine_pshuflw1:
-; ALL:       # %bb.0: # %entry
-; ALL-NEXT:    retq
+; CHECK-LABEL: combine_pshuflw1:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    retq
 entry:
   %b = call <8 x i16> @llvm.x86.sse2.pshufl.w(<8 x i16> %a, i8 27)
   %c = call <8 x i16> @llvm.x86.sse2.pshufl.w(<8 x i16> %b, i8 27)
@@ -123,9 +123,9 @@ entry:
 }
 
 define <8 x i16> @combine_pshuflw2(<8 x i16> %a) {
-; ALL-LABEL: combine_pshuflw2:
-; ALL:       # %bb.0: # %entry
-; ALL-NEXT:    retq
+; CHECK-LABEL: combine_pshuflw2:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    retq
 entry:
   %b = call <8 x i16> @llvm.x86.sse2.pshufl.w(<8 x i16> %a, i8 27)
   %c = call <8 x i16> @llvm.x86.sse2.pshufh.w(<8 x i16> %b, i8 -28)
@@ -677,7 +677,7 @@ define <4 x i32> @combine_nested_undef_test4(<4 x i32> %A, <4 x i32> %B) {
 ;
 ; AVX2-LABEL: combine_nested_undef_test4:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpbroadcastq %xmm0, %xmm0
+; AVX2-NEXT:    vmovddup {{.*#+}} xmm0 = xmm0[0,0]
 ; AVX2-NEXT:    retq
   %1 = shufflevector <4 x i32> %A, <4 x i32> %B, <4 x i32> <i32 0, i32 4, i32 7, i32 1>
   %2 = shufflevector <4 x i32> %1, <4 x i32> undef, <4 x i32> <i32 4, i32 4, i32 0, i32 3>
@@ -811,9 +811,9 @@ define <4 x i32> @combine_nested_undef_test12(<4 x i32> %A, <4 x i32> %B) {
 
 ; The following pair of shuffles is folded into vector %A.
 define <4 x i32> @combine_nested_undef_test13(<4 x i32> %A, <4 x i32> %B) {
-; ALL-LABEL: combine_nested_undef_test13:
-; ALL:       # %bb.0:
-; ALL-NEXT:    retq
+; CHECK-LABEL: combine_nested_undef_test13:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    retq
   %1 = shufflevector <4 x i32> %A, <4 x i32> %B, <4 x i32> <i32 1, i32 4, i32 2, i32 6>
   %2 = shufflevector <4 x i32> %1, <4 x i32> undef, <4 x i32> <i32 4, i32 0, i32 2, i32 4>
   ret <4 x i32> %2
@@ -1044,8 +1044,8 @@ define <4 x i32> @combine_nested_undef_test21(<4 x i32> %A, <4 x i32> %B) {
 ;
 ; AVX2-LABEL: combine_nested_undef_test21:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpblendd {{.*#+}} xmm0 = xmm1[0],xmm0[1],xmm1[2,3]
-; AVX2-NEXT:    vpbroadcastq %xmm0, %xmm0
+; AVX2-NEXT:    vblendps {{.*#+}} xmm0 = xmm1[0],xmm0[1],xmm1[2,3]
+; AVX2-NEXT:    vmovddup {{.*#+}} xmm0 = xmm0[0,0]
 ; AVX2-NEXT:    retq
   %1 = shufflevector <4 x i32> %A, <4 x i32> %B, <4 x i32> <i32 4, i32 1, i32 3, i32 1>
   %2 = shufflevector <4 x i32> %1, <4 x i32> undef, <4 x i32> <i32 0, i32 1, i32 0, i32 3>
@@ -1114,7 +1114,7 @@ define <4 x i32> @combine_nested_undef_test25(<4 x i32> %A, <4 x i32> %B) {
 ;
 ; AVX2-LABEL: combine_nested_undef_test25:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpbroadcastq %xmm0, %xmm0
+; AVX2-NEXT:    vmovddup {{.*#+}} xmm0 = xmm0[0,0]
 ; AVX2-NEXT:    retq
   %1 = shufflevector <4 x i32> %B, <4 x i32> %A, <4 x i32> <i32 1, i32 5, i32 2, i32 4>
   %2 = shufflevector <4 x i32> %1, <4 x i32> undef, <4 x i32> <i32 3, i32 1, i32 3, i32 1>
@@ -1149,7 +1149,7 @@ define <4 x i32> @combine_nested_undef_test27(<4 x i32> %A, <4 x i32> %B) {
 ;
 ; AVX2-LABEL: combine_nested_undef_test27:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpbroadcastq %xmm0, %xmm0
+; AVX2-NEXT:    vmovddup {{.*#+}} xmm0 = xmm0[0,0]
 ; AVX2-NEXT:    retq
   %1 = shufflevector <4 x i32> %B, <4 x i32> %A, <4 x i32> <i32 2, i32 1, i32 5, i32 4>
   %2 = shufflevector <4 x i32> %1, <4 x i32> undef, <4 x i32> <i32 3, i32 2, i32 3, i32 2>
@@ -1371,9 +1371,9 @@ define <4 x i32> @combine_test10(<4 x i32> %a, <4 x i32> %b) {
 }
 
 define <4 x float> @combine_test11(<4 x float> %a, <4 x float> %b) {
-; ALL-LABEL: combine_test11:
-; ALL:       # %bb.0:
-; ALL-NEXT:    retq
+; CHECK-LABEL: combine_test11:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    retq
   %1 = shufflevector <4 x float> %a, <4 x float> %b, <4 x i32> <i32 4, i32 1, i32 6, i32 3>
   %2 = shufflevector <4 x float> %1, <4 x float> %a, <4 x i32> <i32 4, i32 1, i32 6, i32 3>
   ret <4 x float> %2
@@ -1464,9 +1464,9 @@ define <4 x float> @combine_test15(<4 x float> %a, <4 x float> %b) {
 }
 
 define <4 x i32> @combine_test16(<4 x i32> %a, <4 x i32> %b) {
-; ALL-LABEL: combine_test16:
-; ALL:       # %bb.0:
-; ALL-NEXT:    retq
+; CHECK-LABEL: combine_test16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    retq
   %1 = shufflevector <4 x i32> %a, <4 x i32> %b, <4 x i32> <i32 4, i32 1, i32 6, i32 3>
   %2 = shufflevector <4 x i32> %1, <4 x i32> %a, <4 x i32> <i32 4, i32 1, i32 6, i32 3>
   ret <4 x i32> %2
@@ -2161,9 +2161,9 @@ define <4 x float> @combine_undef_input_test5(<4 x float> %a, <4 x float> %b) {
 ;  (shuffle(shuffle A, Undef, M0), A, M1) -> (shuffle A, Undef, M2)
 
 define <4 x float> @combine_undef_input_test6(<4 x float> %a) {
-; ALL-LABEL: combine_undef_input_test6:
-; ALL:       # %bb.0:
-; ALL-NEXT:    retq
+; CHECK-LABEL: combine_undef_input_test6:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    retq
   %1 = shufflevector <4 x float> %a, <4 x float> undef, <4 x i32> <i32 4, i32 2, i32 3, i32 1>
   %2 = shufflevector <4 x float> %1, <4 x float> %a, <4 x i32> <i32 4, i32 5, i32 1, i32 2>
   ret <4 x float> %2
@@ -2235,9 +2235,9 @@ define <4 x float> @combine_undef_input_test9(<4 x float> %a) {
 }
 
 define <4 x float> @combine_undef_input_test10(<4 x float> %a) {
-; ALL-LABEL: combine_undef_input_test10:
-; ALL:       # %bb.0:
-; ALL-NEXT:    retq
+; CHECK-LABEL: combine_undef_input_test10:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    retq
   %1 = shufflevector <4 x float> %a, <4 x float> undef, <4 x i32> <i32 0, i32 4, i32 1, i32 3>
   %2 = shufflevector <4 x float> %1, <4 x float> %a, <4 x i32> <i32 0, i32 2, i32 6, i32 7>
   ret <4 x float> %2
@@ -2351,9 +2351,9 @@ define <4 x float> @combine_undef_input_test15(<4 x float> %a, <4 x float> %b) {
 ; combined into a single legal shuffle operation.
 
 define <4 x float> @combine_undef_input_test16(<4 x float> %a) {
-; ALL-LABEL: combine_undef_input_test16:
-; ALL:       # %bb.0:
-; ALL-NEXT:    retq
+; CHECK-LABEL: combine_undef_input_test16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    retq
   %1 = shufflevector <4 x float> %a, <4 x float> undef, <4 x i32> <i32 4, i32 2, i32 3, i32 1>
   %2 = shufflevector <4 x float> %a, <4 x float> %1, <4 x i32> <i32 0, i32 1, i32 5, i32 3>
   ret <4 x float> %2
@@ -2425,9 +2425,9 @@ define <4 x float> @combine_undef_input_test19(<4 x float> %a) {
 }
 
 define <4 x float> @combine_undef_input_test20(<4 x float> %a) {
-; ALL-LABEL: combine_undef_input_test20:
-; ALL:       # %bb.0:
-; ALL-NEXT:    retq
+; CHECK-LABEL: combine_undef_input_test20:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    retq
   %1 = shufflevector <4 x float> %a, <4 x float> undef, <4 x i32> <i32 0, i32 4, i32 1, i32 3>
   %2 = shufflevector <4 x float> %a, <4 x float> %1, <4 x i32> <i32 4, i32 6, i32 2, i32 3>
   ret <4 x float> %2
diff --git a/test/CodeGen/X86/vector-shuffle-sse4a.ll b/test/CodeGen/X86/vector-shuffle-sse4a.ll
index 501d91572605cfae4e31d3b4a47829a9effcbce5..25c44964a55117af6b6549ce8973f94a2aab0e88 100644
--- a/test/CodeGen/X86/vector-shuffle-sse4a.ll
+++ b/test/CodeGen/X86/vector-shuffle-sse4a.ll
@@ -410,8 +410,9 @@ define <16 x i8> @shuffle_uu_0_5_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu(<16 x i8
 define <16 x i8> @shuffle_uu_16_4_16_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu(<16 x i8> %v) {
 ; AMD10H-LABEL: shuffle_uu_16_4_16_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu:
 ; AMD10H:       # %bb.0:
-; AMD10H-NEXT:    psrlq $16, %xmm0
-; AMD10H-NEXT:    pand {{.*}}(%rip), %xmm0
+; AMD10H-NEXT:    pslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1,2,3,4]
+; AMD10H-NEXT:    psrldq {{.*#+}} xmm0 = xmm0[15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AMD10H-NEXT:    pslldq {{.*#+}} xmm0 = zero,zero,xmm0[0,1,2,3,4,5,6,7,8,9,10,11,12,13]
 ; AMD10H-NEXT:    retq
 ;
 ; BTVER1-LABEL: shuffle_uu_16_4_16_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu_uu:
diff --git a/test/CodeGen/X86/vector-trunc-math-widen.ll b/test/CodeGen/X86/vector-trunc-math-widen.ll
index 1cda401a82dfa7491833e9e5c32c0a433911d77f..553562e5e82bd58936381b5e5b4629b5c1e3288f 100644
--- a/test/CodeGen/X86/vector-trunc-math-widen.ll
+++ b/test/CodeGen/X86/vector-trunc-math-widen.ll
@@ -681,21 +681,21 @@ define <16 x i8> @trunc_add_const_v16i64_v16i8(<16 x i64> %a0) nounwind {
 ; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm4
 ; AVX1-NEXT:    vmovddup {{.*#+}} xmm5 = [1.2598673968951787E-321,1.2598673968951787E-321]
 ; AVX1-NEXT:    # xmm5 = mem[0,0]
-; AVX1-NEXT:    vandpd %xmm5, %xmm4, %xmm4
-; AVX1-NEXT:    vandpd %xmm5, %xmm3, %xmm3
+; AVX1-NEXT:    vandps %xmm5, %xmm4, %xmm4
+; AVX1-NEXT:    vandps %xmm5, %xmm3, %xmm3
 ; AVX1-NEXT:    vpackusdw %xmm4, %xmm3, %xmm3
 ; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm4
-; AVX1-NEXT:    vandpd %xmm5, %xmm4, %xmm4
-; AVX1-NEXT:    vandpd %xmm5, %xmm2, %xmm2
+; AVX1-NEXT:    vandps %xmm5, %xmm4, %xmm4
+; AVX1-NEXT:    vandps %xmm5, %xmm2, %xmm2
 ; AVX1-NEXT:    vpackusdw %xmm4, %xmm2, %xmm2
 ; AVX1-NEXT:    vpackusdw %xmm3, %xmm2, %xmm2
 ; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm3
-; AVX1-NEXT:    vandpd %xmm5, %xmm3, %xmm3
-; AVX1-NEXT:    vandpd %xmm5, %xmm1, %xmm1
+; AVX1-NEXT:    vandps %xmm5, %xmm3, %xmm3
+; AVX1-NEXT:    vandps %xmm5, %xmm1, %xmm1
 ; AVX1-NEXT:    vpackusdw %xmm3, %xmm1, %xmm1
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
-; AVX1-NEXT:    vandpd %xmm5, %xmm3, %xmm3
-; AVX1-NEXT:    vandpd %xmm5, %xmm0, %xmm0
+; AVX1-NEXT:    vandps %xmm5, %xmm3, %xmm3
+; AVX1-NEXT:    vandps %xmm5, %xmm0, %xmm0
 ; AVX1-NEXT:    vpackusdw %xmm3, %xmm0, %xmm0
 ; AVX1-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpackuswb %xmm2, %xmm0, %xmm0
@@ -1518,21 +1518,21 @@ define <16 x i8> @trunc_sub_const_v16i64_v16i8(<16 x i64> %a0) nounwind {
 ; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm4
 ; AVX1-NEXT:    vmovddup {{.*#+}} xmm5 = [1.2598673968951787E-321,1.2598673968951787E-321]
 ; AVX1-NEXT:    # xmm5 = mem[0,0]
-; AVX1-NEXT:    vandpd %xmm5, %xmm4, %xmm4
-; AVX1-NEXT:    vandpd %xmm5, %xmm3, %xmm3
+; AVX1-NEXT:    vandps %xmm5, %xmm4, %xmm4
+; AVX1-NEXT:    vandps %xmm5, %xmm3, %xmm3
 ; AVX1-NEXT:    vpackusdw %xmm4, %xmm3, %xmm3
 ; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm4
-; AVX1-NEXT:    vandpd %xmm5, %xmm4, %xmm4
-; AVX1-NEXT:    vandpd %xmm5, %xmm2, %xmm2
+; AVX1-NEXT:    vandps %xmm5, %xmm4, %xmm4
+; AVX1-NEXT:    vandps %xmm5, %xmm2, %xmm2
 ; AVX1-NEXT:    vpackusdw %xmm4, %xmm2, %xmm2
 ; AVX1-NEXT:    vpackusdw %xmm3, %xmm2, %xmm2
 ; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm3
-; AVX1-NEXT:    vandpd %xmm5, %xmm3, %xmm3
-; AVX1-NEXT:    vandpd %xmm5, %xmm1, %xmm1
+; AVX1-NEXT:    vandps %xmm5, %xmm3, %xmm3
+; AVX1-NEXT:    vandps %xmm5, %xmm1, %xmm1
 ; AVX1-NEXT:    vpackusdw %xmm3, %xmm1, %xmm1
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
-; AVX1-NEXT:    vandpd %xmm5, %xmm3, %xmm3
-; AVX1-NEXT:    vandpd %xmm5, %xmm0, %xmm0
+; AVX1-NEXT:    vandps %xmm5, %xmm3, %xmm3
+; AVX1-NEXT:    vandps %xmm5, %xmm0, %xmm0
 ; AVX1-NEXT:    vpackusdw %xmm3, %xmm0, %xmm0
 ; AVX1-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpackuswb %xmm2, %xmm0, %xmm0
@@ -3119,28 +3119,28 @@ define <16 x i8> @trunc_and_v16i64_v16i8(<16 x i64> %a0, <16 x i64> %a1) nounwin
 ;
 ; AVX1-LABEL: trunc_and_v16i64_v16i8:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vandpd %ymm4, %ymm0, %ymm0
-; AVX1-NEXT:    vandpd %ymm5, %ymm1, %ymm1
-; AVX1-NEXT:    vandpd %ymm6, %ymm2, %ymm2
-; AVX1-NEXT:    vandpd %ymm7, %ymm3, %ymm3
+; AVX1-NEXT:    vandps %ymm4, %ymm0, %ymm0
+; AVX1-NEXT:    vandps %ymm5, %ymm1, %ymm1
+; AVX1-NEXT:    vandps %ymm6, %ymm2, %ymm2
+; AVX1-NEXT:    vandps %ymm7, %ymm3, %ymm3
 ; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm4
 ; AVX1-NEXT:    vmovddup {{.*#+}} xmm5 = [1.2598673968951787E-321,1.2598673968951787E-321]
 ; AVX1-NEXT:    # xmm5 = mem[0,0]
-; AVX1-NEXT:    vandpd %xmm5, %xmm4, %xmm4
-; AVX1-NEXT:    vandpd %xmm5, %xmm3, %xmm3
+; AVX1-NEXT:    vandps %xmm5, %xmm4, %xmm4
+; AVX1-NEXT:    vandps %xmm5, %xmm3, %xmm3
 ; AVX1-NEXT:    vpackusdw %xmm4, %xmm3, %xmm3
 ; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm4
-; AVX1-NEXT:    vandpd %xmm5, %xmm4, %xmm4
-; AVX1-NEXT:    vandpd %xmm5, %xmm2, %xmm2
+; AVX1-NEXT:    vandps %xmm5, %xmm4, %xmm4
+; AVX1-NEXT:    vandps %xmm5, %xmm2, %xmm2
 ; AVX1-NEXT:    vpackusdw %xmm4, %xmm2, %xmm2
 ; AVX1-NEXT:    vpackusdw %xmm3, %xmm2, %xmm2
 ; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm3
-; AVX1-NEXT:    vandpd %xmm5, %xmm3, %xmm3
-; AVX1-NEXT:    vandpd %xmm5, %xmm1, %xmm1
+; AVX1-NEXT:    vandps %xmm5, %xmm3, %xmm3
+; AVX1-NEXT:    vandps %xmm5, %xmm1, %xmm1
 ; AVX1-NEXT:    vpackusdw %xmm3, %xmm1, %xmm1
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
-; AVX1-NEXT:    vandpd %xmm5, %xmm3, %xmm3
-; AVX1-NEXT:    vandpd %xmm5, %xmm0, %xmm0
+; AVX1-NEXT:    vandps %xmm5, %xmm3, %xmm3
+; AVX1-NEXT:    vandps %xmm5, %xmm0, %xmm0
 ; AVX1-NEXT:    vpackusdw %xmm3, %xmm0, %xmm0
 ; AVX1-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpackuswb %xmm2, %xmm0, %xmm0
@@ -3518,21 +3518,21 @@ define <16 x i8> @trunc_and_const_v16i64_v16i8(<16 x i64> %a0) nounwind {
 ; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm4
 ; AVX1-NEXT:    vmovddup {{.*#+}} xmm5 = [1.2598673968951787E-321,1.2598673968951787E-321]
 ; AVX1-NEXT:    # xmm5 = mem[0,0]
-; AVX1-NEXT:    vandpd %xmm5, %xmm4, %xmm4
-; AVX1-NEXT:    vandpd %xmm5, %xmm3, %xmm3
+; AVX1-NEXT:    vandps %xmm5, %xmm4, %xmm4
+; AVX1-NEXT:    vandps %xmm5, %xmm3, %xmm3
 ; AVX1-NEXT:    vpackusdw %xmm4, %xmm3, %xmm3
 ; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm4
-; AVX1-NEXT:    vandpd %xmm5, %xmm4, %xmm4
-; AVX1-NEXT:    vandpd %xmm5, %xmm2, %xmm2
+; AVX1-NEXT:    vandps %xmm5, %xmm4, %xmm4
+; AVX1-NEXT:    vandps %xmm5, %xmm2, %xmm2
 ; AVX1-NEXT:    vpackusdw %xmm4, %xmm2, %xmm2
 ; AVX1-NEXT:    vpackusdw %xmm3, %xmm2, %xmm2
 ; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm3
-; AVX1-NEXT:    vandpd %xmm5, %xmm3, %xmm3
-; AVX1-NEXT:    vandpd %xmm5, %xmm1, %xmm1
+; AVX1-NEXT:    vandps %xmm5, %xmm3, %xmm3
+; AVX1-NEXT:    vandps %xmm5, %xmm1, %xmm1
 ; AVX1-NEXT:    vpackusdw %xmm3, %xmm1, %xmm1
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
-; AVX1-NEXT:    vandpd %xmm5, %xmm3, %xmm3
-; AVX1-NEXT:    vandpd %xmm5, %xmm0, %xmm0
+; AVX1-NEXT:    vandps %xmm5, %xmm3, %xmm3
+; AVX1-NEXT:    vandps %xmm5, %xmm0, %xmm0
 ; AVX1-NEXT:    vpackusdw %xmm3, %xmm0, %xmm0
 ; AVX1-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpackuswb %xmm2, %xmm0, %xmm0
@@ -3915,28 +3915,28 @@ define <16 x i8> @trunc_xor_v16i64_v16i8(<16 x i64> %a0, <16 x i64> %a1) nounwin
 ;
 ; AVX1-LABEL: trunc_xor_v16i64_v16i8:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vxorpd %ymm4, %ymm0, %ymm0
-; AVX1-NEXT:    vxorpd %ymm5, %ymm1, %ymm1
-; AVX1-NEXT:    vxorpd %ymm6, %ymm2, %ymm2
-; AVX1-NEXT:    vxorpd %ymm7, %ymm3, %ymm3
+; AVX1-NEXT:    vxorps %ymm4, %ymm0, %ymm0
+; AVX1-NEXT:    vxorps %ymm5, %ymm1, %ymm1
+; AVX1-NEXT:    vxorps %ymm6, %ymm2, %ymm2
+; AVX1-NEXT:    vxorps %ymm7, %ymm3, %ymm3
 ; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm4
 ; AVX1-NEXT:    vmovddup {{.*#+}} xmm5 = [1.2598673968951787E-321,1.2598673968951787E-321]
 ; AVX1-NEXT:    # xmm5 = mem[0,0]
-; AVX1-NEXT:    vandpd %xmm5, %xmm4, %xmm4
-; AVX1-NEXT:    vandpd %xmm5, %xmm3, %xmm3
+; AVX1-NEXT:    vandps %xmm5, %xmm4, %xmm4
+; AVX1-NEXT:    vandps %xmm5, %xmm3, %xmm3
 ; AVX1-NEXT:    vpackusdw %xmm4, %xmm3, %xmm3
 ; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm4
-; AVX1-NEXT:    vandpd %xmm5, %xmm4, %xmm4
-; AVX1-NEXT:    vandpd %xmm5, %xmm2, %xmm2
+; AVX1-NEXT:    vandps %xmm5, %xmm4, %xmm4
+; AVX1-NEXT:    vandps %xmm5, %xmm2, %xmm2
 ; AVX1-NEXT:    vpackusdw %xmm4, %xmm2, %xmm2
 ; AVX1-NEXT:    vpackusdw %xmm3, %xmm2, %xmm2
 ; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm3
-; AVX1-NEXT:    vandpd %xmm5, %xmm3, %xmm3
-; AVX1-NEXT:    vandpd %xmm5, %xmm1, %xmm1
+; AVX1-NEXT:    vandps %xmm5, %xmm3, %xmm3
+; AVX1-NEXT:    vandps %xmm5, %xmm1, %xmm1
 ; AVX1-NEXT:    vpackusdw %xmm3, %xmm1, %xmm1
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
-; AVX1-NEXT:    vandpd %xmm5, %xmm3, %xmm3
-; AVX1-NEXT:    vandpd %xmm5, %xmm0, %xmm0
+; AVX1-NEXT:    vandps %xmm5, %xmm3, %xmm3
+; AVX1-NEXT:    vandps %xmm5, %xmm0, %xmm0
 ; AVX1-NEXT:    vpackusdw %xmm3, %xmm0, %xmm0
 ; AVX1-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpackuswb %xmm2, %xmm0, %xmm0
@@ -4314,21 +4314,21 @@ define <16 x i8> @trunc_xor_const_v16i64_v16i8(<16 x i64> %a0) nounwind {
 ; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm4
 ; AVX1-NEXT:    vmovddup {{.*#+}} xmm5 = [1.2598673968951787E-321,1.2598673968951787E-321]
 ; AVX1-NEXT:    # xmm5 = mem[0,0]
-; AVX1-NEXT:    vandpd %xmm5, %xmm4, %xmm4
-; AVX1-NEXT:    vandpd %xmm5, %xmm3, %xmm3
+; AVX1-NEXT:    vandps %xmm5, %xmm4, %xmm4
+; AVX1-NEXT:    vandps %xmm5, %xmm3, %xmm3
 ; AVX1-NEXT:    vpackusdw %xmm4, %xmm3, %xmm3
 ; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm4
-; AVX1-NEXT:    vandpd %xmm5, %xmm4, %xmm4
-; AVX1-NEXT:    vandpd %xmm5, %xmm2, %xmm2
+; AVX1-NEXT:    vandps %xmm5, %xmm4, %xmm4
+; AVX1-NEXT:    vandps %xmm5, %xmm2, %xmm2
 ; AVX1-NEXT:    vpackusdw %xmm4, %xmm2, %xmm2
 ; AVX1-NEXT:    vpackusdw %xmm3, %xmm2, %xmm2
 ; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm3
-; AVX1-NEXT:    vandpd %xmm5, %xmm3, %xmm3
-; AVX1-NEXT:    vandpd %xmm5, %xmm1, %xmm1
+; AVX1-NEXT:    vandps %xmm5, %xmm3, %xmm3
+; AVX1-NEXT:    vandps %xmm5, %xmm1, %xmm1
 ; AVX1-NEXT:    vpackusdw %xmm3, %xmm1, %xmm1
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
-; AVX1-NEXT:    vandpd %xmm5, %xmm3, %xmm3
-; AVX1-NEXT:    vandpd %xmm5, %xmm0, %xmm0
+; AVX1-NEXT:    vandps %xmm5, %xmm3, %xmm3
+; AVX1-NEXT:    vandps %xmm5, %xmm0, %xmm0
 ; AVX1-NEXT:    vpackusdw %xmm3, %xmm0, %xmm0
 ; AVX1-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpackuswb %xmm2, %xmm0, %xmm0
@@ -4711,28 +4711,28 @@ define <16 x i8> @trunc_or_v16i64_v16i8(<16 x i64> %a0, <16 x i64> %a1) nounwind
 ;
 ; AVX1-LABEL: trunc_or_v16i64_v16i8:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vorpd %ymm4, %ymm0, %ymm0
-; AVX1-NEXT:    vorpd %ymm5, %ymm1, %ymm1
-; AVX1-NEXT:    vorpd %ymm6, %ymm2, %ymm2
-; AVX1-NEXT:    vorpd %ymm7, %ymm3, %ymm3
+; AVX1-NEXT:    vorps %ymm4, %ymm0, %ymm0
+; AVX1-NEXT:    vorps %ymm5, %ymm1, %ymm1
+; AVX1-NEXT:    vorps %ymm6, %ymm2, %ymm2
+; AVX1-NEXT:    vorps %ymm7, %ymm3, %ymm3
 ; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm4
 ; AVX1-NEXT:    vmovddup {{.*#+}} xmm5 = [1.2598673968951787E-321,1.2598673968951787E-321]
 ; AVX1-NEXT:    # xmm5 = mem[0,0]
-; AVX1-NEXT:    vandpd %xmm5, %xmm4, %xmm4
-; AVX1-NEXT:    vandpd %xmm5, %xmm3, %xmm3
+; AVX1-NEXT:    vandps %xmm5, %xmm4, %xmm4
+; AVX1-NEXT:    vandps %xmm5, %xmm3, %xmm3
 ; AVX1-NEXT:    vpackusdw %xmm4, %xmm3, %xmm3
 ; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm4
-; AVX1-NEXT:    vandpd %xmm5, %xmm4, %xmm4
-; AVX1-NEXT:    vandpd %xmm5, %xmm2, %xmm2
+; AVX1-NEXT:    vandps %xmm5, %xmm4, %xmm4
+; AVX1-NEXT:    vandps %xmm5, %xmm2, %xmm2
 ; AVX1-NEXT:    vpackusdw %xmm4, %xmm2, %xmm2
 ; AVX1-NEXT:    vpackusdw %xmm3, %xmm2, %xmm2
 ; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm3
-; AVX1-NEXT:    vandpd %xmm5, %xmm3, %xmm3
-; AVX1-NEXT:    vandpd %xmm5, %xmm1, %xmm1
+; AVX1-NEXT:    vandps %xmm5, %xmm3, %xmm3
+; AVX1-NEXT:    vandps %xmm5, %xmm1, %xmm1
 ; AVX1-NEXT:    vpackusdw %xmm3, %xmm1, %xmm1
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
-; AVX1-NEXT:    vandpd %xmm5, %xmm3, %xmm3
-; AVX1-NEXT:    vandpd %xmm5, %xmm0, %xmm0
+; AVX1-NEXT:    vandps %xmm5, %xmm3, %xmm3
+; AVX1-NEXT:    vandps %xmm5, %xmm0, %xmm0
 ; AVX1-NEXT:    vpackusdw %xmm3, %xmm0, %xmm0
 ; AVX1-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpackuswb %xmm2, %xmm0, %xmm0
@@ -5110,21 +5110,21 @@ define <16 x i8> @trunc_or_const_v16i64_v16i8(<16 x i64> %a0) nounwind {
 ; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm4
 ; AVX1-NEXT:    vmovddup {{.*#+}} xmm5 = [1.2598673968951787E-321,1.2598673968951787E-321]
 ; AVX1-NEXT:    # xmm5 = mem[0,0]
-; AVX1-NEXT:    vandpd %xmm5, %xmm4, %xmm4
-; AVX1-NEXT:    vandpd %xmm5, %xmm3, %xmm3
+; AVX1-NEXT:    vandps %xmm5, %xmm4, %xmm4
+; AVX1-NEXT:    vandps %xmm5, %xmm3, %xmm3
 ; AVX1-NEXT:    vpackusdw %xmm4, %xmm3, %xmm3
 ; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm4
-; AVX1-NEXT:    vandpd %xmm5, %xmm4, %xmm4
-; AVX1-NEXT:    vandpd %xmm5, %xmm2, %xmm2
+; AVX1-NEXT:    vandps %xmm5, %xmm4, %xmm4
+; AVX1-NEXT:    vandps %xmm5, %xmm2, %xmm2
 ; AVX1-NEXT:    vpackusdw %xmm4, %xmm2, %xmm2
 ; AVX1-NEXT:    vpackusdw %xmm3, %xmm2, %xmm2
 ; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm3
-; AVX1-NEXT:    vandpd %xmm5, %xmm3, %xmm3
-; AVX1-NEXT:    vandpd %xmm5, %xmm1, %xmm1
+; AVX1-NEXT:    vandps %xmm5, %xmm3, %xmm3
+; AVX1-NEXT:    vandps %xmm5, %xmm1, %xmm1
 ; AVX1-NEXT:    vpackusdw %xmm3, %xmm1, %xmm1
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
-; AVX1-NEXT:    vandpd %xmm5, %xmm3, %xmm3
-; AVX1-NEXT:    vandpd %xmm5, %xmm0, %xmm0
+; AVX1-NEXT:    vandps %xmm5, %xmm3, %xmm3
+; AVX1-NEXT:    vandps %xmm5, %xmm0, %xmm0
 ; AVX1-NEXT:    vpackusdw %xmm3, %xmm0, %xmm0
 ; AVX1-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpackuswb %xmm2, %xmm0, %xmm0
diff --git a/test/CodeGen/X86/vector-trunc-math.ll b/test/CodeGen/X86/vector-trunc-math.ll
index b06f3294e0a27375a580c2cddba4ec494977c387..fdc558aae934d9213a26d7a9caeba51215cef278 100644
--- a/test/CodeGen/X86/vector-trunc-math.ll
+++ b/test/CodeGen/X86/vector-trunc-math.ll
@@ -681,21 +681,21 @@ define <16 x i8> @trunc_add_const_v16i64_v16i8(<16 x i64> %a0) nounwind {
 ; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm4
 ; AVX1-NEXT:    vmovddup {{.*#+}} xmm5 = [1.2598673968951787E-321,1.2598673968951787E-321]
 ; AVX1-NEXT:    # xmm5 = mem[0,0]
-; AVX1-NEXT:    vandpd %xmm5, %xmm4, %xmm4
-; AVX1-NEXT:    vandpd %xmm5, %xmm3, %xmm3
+; AVX1-NEXT:    vandps %xmm5, %xmm4, %xmm4
+; AVX1-NEXT:    vandps %xmm5, %xmm3, %xmm3
 ; AVX1-NEXT:    vpackusdw %xmm4, %xmm3, %xmm3
 ; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm4
-; AVX1-NEXT:    vandpd %xmm5, %xmm4, %xmm4
-; AVX1-NEXT:    vandpd %xmm5, %xmm2, %xmm2
+; AVX1-NEXT:    vandps %xmm5, %xmm4, %xmm4
+; AVX1-NEXT:    vandps %xmm5, %xmm2, %xmm2
 ; AVX1-NEXT:    vpackusdw %xmm4, %xmm2, %xmm2
 ; AVX1-NEXT:    vpackusdw %xmm3, %xmm2, %xmm2
 ; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm3
-; AVX1-NEXT:    vandpd %xmm5, %xmm3, %xmm3
-; AVX1-NEXT:    vandpd %xmm5, %xmm1, %xmm1
+; AVX1-NEXT:    vandps %xmm5, %xmm3, %xmm3
+; AVX1-NEXT:    vandps %xmm5, %xmm1, %xmm1
 ; AVX1-NEXT:    vpackusdw %xmm3, %xmm1, %xmm1
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
-; AVX1-NEXT:    vandpd %xmm5, %xmm3, %xmm3
-; AVX1-NEXT:    vandpd %xmm5, %xmm0, %xmm0
+; AVX1-NEXT:    vandps %xmm5, %xmm3, %xmm3
+; AVX1-NEXT:    vandps %xmm5, %xmm0, %xmm0
 ; AVX1-NEXT:    vpackusdw %xmm3, %xmm0, %xmm0
 ; AVX1-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpackuswb %xmm2, %xmm0, %xmm0
@@ -1518,21 +1518,21 @@ define <16 x i8> @trunc_sub_const_v16i64_v16i8(<16 x i64> %a0) nounwind {
 ; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm4
 ; AVX1-NEXT:    vmovddup {{.*#+}} xmm5 = [1.2598673968951787E-321,1.2598673968951787E-321]
 ; AVX1-NEXT:    # xmm5 = mem[0,0]
-; AVX1-NEXT:    vandpd %xmm5, %xmm4, %xmm4
-; AVX1-NEXT:    vandpd %xmm5, %xmm3, %xmm3
+; AVX1-NEXT:    vandps %xmm5, %xmm4, %xmm4
+; AVX1-NEXT:    vandps %xmm5, %xmm3, %xmm3
 ; AVX1-NEXT:    vpackusdw %xmm4, %xmm3, %xmm3
 ; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm4
-; AVX1-NEXT:    vandpd %xmm5, %xmm4, %xmm4
-; AVX1-NEXT:    vandpd %xmm5, %xmm2, %xmm2
+; AVX1-NEXT:    vandps %xmm5, %xmm4, %xmm4
+; AVX1-NEXT:    vandps %xmm5, %xmm2, %xmm2
 ; AVX1-NEXT:    vpackusdw %xmm4, %xmm2, %xmm2
 ; AVX1-NEXT:    vpackusdw %xmm3, %xmm2, %xmm2
 ; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm3
-; AVX1-NEXT:    vandpd %xmm5, %xmm3, %xmm3
-; AVX1-NEXT:    vandpd %xmm5, %xmm1, %xmm1
+; AVX1-NEXT:    vandps %xmm5, %xmm3, %xmm3
+; AVX1-NEXT:    vandps %xmm5, %xmm1, %xmm1
 ; AVX1-NEXT:    vpackusdw %xmm3, %xmm1, %xmm1
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
-; AVX1-NEXT:    vandpd %xmm5, %xmm3, %xmm3
-; AVX1-NEXT:    vandpd %xmm5, %xmm0, %xmm0
+; AVX1-NEXT:    vandps %xmm5, %xmm3, %xmm3
+; AVX1-NEXT:    vandps %xmm5, %xmm0, %xmm0
 ; AVX1-NEXT:    vpackusdw %xmm3, %xmm0, %xmm0
 ; AVX1-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpackuswb %xmm2, %xmm0, %xmm0
@@ -3119,28 +3119,28 @@ define <16 x i8> @trunc_and_v16i64_v16i8(<16 x i64> %a0, <16 x i64> %a1) nounwin
 ;
 ; AVX1-LABEL: trunc_and_v16i64_v16i8:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vandpd %ymm4, %ymm0, %ymm0
-; AVX1-NEXT:    vandpd %ymm5, %ymm1, %ymm1
-; AVX1-NEXT:    vandpd %ymm6, %ymm2, %ymm2
-; AVX1-NEXT:    vandpd %ymm7, %ymm3, %ymm3
+; AVX1-NEXT:    vandps %ymm4, %ymm0, %ymm0
+; AVX1-NEXT:    vandps %ymm5, %ymm1, %ymm1
+; AVX1-NEXT:    vandps %ymm6, %ymm2, %ymm2
+; AVX1-NEXT:    vandps %ymm7, %ymm3, %ymm3
 ; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm4
 ; AVX1-NEXT:    vmovddup {{.*#+}} xmm5 = [1.2598673968951787E-321,1.2598673968951787E-321]
 ; AVX1-NEXT:    # xmm5 = mem[0,0]
-; AVX1-NEXT:    vandpd %xmm5, %xmm4, %xmm4
-; AVX1-NEXT:    vandpd %xmm5, %xmm3, %xmm3
+; AVX1-NEXT:    vandps %xmm5, %xmm4, %xmm4
+; AVX1-NEXT:    vandps %xmm5, %xmm3, %xmm3
 ; AVX1-NEXT:    vpackusdw %xmm4, %xmm3, %xmm3
 ; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm4
-; AVX1-NEXT:    vandpd %xmm5, %xmm4, %xmm4
-; AVX1-NEXT:    vandpd %xmm5, %xmm2, %xmm2
+; AVX1-NEXT:    vandps %xmm5, %xmm4, %xmm4
+; AVX1-NEXT:    vandps %xmm5, %xmm2, %xmm2
 ; AVX1-NEXT:    vpackusdw %xmm4, %xmm2, %xmm2
 ; AVX1-NEXT:    vpackusdw %xmm3, %xmm2, %xmm2
 ; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm3
-; AVX1-NEXT:    vandpd %xmm5, %xmm3, %xmm3
-; AVX1-NEXT:    vandpd %xmm5, %xmm1, %xmm1
+; AVX1-NEXT:    vandps %xmm5, %xmm3, %xmm3
+; AVX1-NEXT:    vandps %xmm5, %xmm1, %xmm1
 ; AVX1-NEXT:    vpackusdw %xmm3, %xmm1, %xmm1
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
-; AVX1-NEXT:    vandpd %xmm5, %xmm3, %xmm3
-; AVX1-NEXT:    vandpd %xmm5, %xmm0, %xmm0
+; AVX1-NEXT:    vandps %xmm5, %xmm3, %xmm3
+; AVX1-NEXT:    vandps %xmm5, %xmm0, %xmm0
 ; AVX1-NEXT:    vpackusdw %xmm3, %xmm0, %xmm0
 ; AVX1-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpackuswb %xmm2, %xmm0, %xmm0
@@ -3518,21 +3518,21 @@ define <16 x i8> @trunc_and_const_v16i64_v16i8(<16 x i64> %a0) nounwind {
 ; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm4
 ; AVX1-NEXT:    vmovddup {{.*#+}} xmm5 = [1.2598673968951787E-321,1.2598673968951787E-321]
 ; AVX1-NEXT:    # xmm5 = mem[0,0]
-; AVX1-NEXT:    vandpd %xmm5, %xmm4, %xmm4
-; AVX1-NEXT:    vandpd %xmm5, %xmm3, %xmm3
+; AVX1-NEXT:    vandps %xmm5, %xmm4, %xmm4
+; AVX1-NEXT:    vandps %xmm5, %xmm3, %xmm3
 ; AVX1-NEXT:    vpackusdw %xmm4, %xmm3, %xmm3
 ; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm4
-; AVX1-NEXT:    vandpd %xmm5, %xmm4, %xmm4
-; AVX1-NEXT:    vandpd %xmm5, %xmm2, %xmm2
+; AVX1-NEXT:    vandps %xmm5, %xmm4, %xmm4
+; AVX1-NEXT:    vandps %xmm5, %xmm2, %xmm2
 ; AVX1-NEXT:    vpackusdw %xmm4, %xmm2, %xmm2
 ; AVX1-NEXT:    vpackusdw %xmm3, %xmm2, %xmm2
 ; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm3
-; AVX1-NEXT:    vandpd %xmm5, %xmm3, %xmm3
-; AVX1-NEXT:    vandpd %xmm5, %xmm1, %xmm1
+; AVX1-NEXT:    vandps %xmm5, %xmm3, %xmm3
+; AVX1-NEXT:    vandps %xmm5, %xmm1, %xmm1
 ; AVX1-NEXT:    vpackusdw %xmm3, %xmm1, %xmm1
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
-; AVX1-NEXT:    vandpd %xmm5, %xmm3, %xmm3
-; AVX1-NEXT:    vandpd %xmm5, %xmm0, %xmm0
+; AVX1-NEXT:    vandps %xmm5, %xmm3, %xmm3
+; AVX1-NEXT:    vandps %xmm5, %xmm0, %xmm0
 ; AVX1-NEXT:    vpackusdw %xmm3, %xmm0, %xmm0
 ; AVX1-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpackuswb %xmm2, %xmm0, %xmm0
@@ -3915,28 +3915,28 @@ define <16 x i8> @trunc_xor_v16i64_v16i8(<16 x i64> %a0, <16 x i64> %a1) nounwin
 ;
 ; AVX1-LABEL: trunc_xor_v16i64_v16i8:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vxorpd %ymm4, %ymm0, %ymm0
-; AVX1-NEXT:    vxorpd %ymm5, %ymm1, %ymm1
-; AVX1-NEXT:    vxorpd %ymm6, %ymm2, %ymm2
-; AVX1-NEXT:    vxorpd %ymm7, %ymm3, %ymm3
+; AVX1-NEXT:    vxorps %ymm4, %ymm0, %ymm0
+; AVX1-NEXT:    vxorps %ymm5, %ymm1, %ymm1
+; AVX1-NEXT:    vxorps %ymm6, %ymm2, %ymm2
+; AVX1-NEXT:    vxorps %ymm7, %ymm3, %ymm3
 ; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm4
 ; AVX1-NEXT:    vmovddup {{.*#+}} xmm5 = [1.2598673968951787E-321,1.2598673968951787E-321]
 ; AVX1-NEXT:    # xmm5 = mem[0,0]
-; AVX1-NEXT:    vandpd %xmm5, %xmm4, %xmm4
-; AVX1-NEXT:    vandpd %xmm5, %xmm3, %xmm3
+; AVX1-NEXT:    vandps %xmm5, %xmm4, %xmm4
+; AVX1-NEXT:    vandps %xmm5, %xmm3, %xmm3
 ; AVX1-NEXT:    vpackusdw %xmm4, %xmm3, %xmm3
 ; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm4
-; AVX1-NEXT:    vandpd %xmm5, %xmm4, %xmm4
-; AVX1-NEXT:    vandpd %xmm5, %xmm2, %xmm2
+; AVX1-NEXT:    vandps %xmm5, %xmm4, %xmm4
+; AVX1-NEXT:    vandps %xmm5, %xmm2, %xmm2
 ; AVX1-NEXT:    vpackusdw %xmm4, %xmm2, %xmm2
 ; AVX1-NEXT:    vpackusdw %xmm3, %xmm2, %xmm2
 ; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm3
-; AVX1-NEXT:    vandpd %xmm5, %xmm3, %xmm3
-; AVX1-NEXT:    vandpd %xmm5, %xmm1, %xmm1
+; AVX1-NEXT:    vandps %xmm5, %xmm3, %xmm3
+; AVX1-NEXT:    vandps %xmm5, %xmm1, %xmm1
 ; AVX1-NEXT:    vpackusdw %xmm3, %xmm1, %xmm1
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
-; AVX1-NEXT:    vandpd %xmm5, %xmm3, %xmm3
-; AVX1-NEXT:    vandpd %xmm5, %xmm0, %xmm0
+; AVX1-NEXT:    vandps %xmm5, %xmm3, %xmm3
+; AVX1-NEXT:    vandps %xmm5, %xmm0, %xmm0
 ; AVX1-NEXT:    vpackusdw %xmm3, %xmm0, %xmm0
 ; AVX1-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpackuswb %xmm2, %xmm0, %xmm0
@@ -4314,21 +4314,21 @@ define <16 x i8> @trunc_xor_const_v16i64_v16i8(<16 x i64> %a0) nounwind {
 ; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm4
 ; AVX1-NEXT:    vmovddup {{.*#+}} xmm5 = [1.2598673968951787E-321,1.2598673968951787E-321]
 ; AVX1-NEXT:    # xmm5 = mem[0,0]
-; AVX1-NEXT:    vandpd %xmm5, %xmm4, %xmm4
-; AVX1-NEXT:    vandpd %xmm5, %xmm3, %xmm3
+; AVX1-NEXT:    vandps %xmm5, %xmm4, %xmm4
+; AVX1-NEXT:    vandps %xmm5, %xmm3, %xmm3
 ; AVX1-NEXT:    vpackusdw %xmm4, %xmm3, %xmm3
 ; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm4
-; AVX1-NEXT:    vandpd %xmm5, %xmm4, %xmm4
-; AVX1-NEXT:    vandpd %xmm5, %xmm2, %xmm2
+; AVX1-NEXT:    vandps %xmm5, %xmm4, %xmm4
+; AVX1-NEXT:    vandps %xmm5, %xmm2, %xmm2
 ; AVX1-NEXT:    vpackusdw %xmm4, %xmm2, %xmm2
 ; AVX1-NEXT:    vpackusdw %xmm3, %xmm2, %xmm2
 ; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm3
-; AVX1-NEXT:    vandpd %xmm5, %xmm3, %xmm3
-; AVX1-NEXT:    vandpd %xmm5, %xmm1, %xmm1
+; AVX1-NEXT:    vandps %xmm5, %xmm3, %xmm3
+; AVX1-NEXT:    vandps %xmm5, %xmm1, %xmm1
 ; AVX1-NEXT:    vpackusdw %xmm3, %xmm1, %xmm1
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
-; AVX1-NEXT:    vandpd %xmm5, %xmm3, %xmm3
-; AVX1-NEXT:    vandpd %xmm5, %xmm0, %xmm0
+; AVX1-NEXT:    vandps %xmm5, %xmm3, %xmm3
+; AVX1-NEXT:    vandps %xmm5, %xmm0, %xmm0
 ; AVX1-NEXT:    vpackusdw %xmm3, %xmm0, %xmm0
 ; AVX1-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpackuswb %xmm2, %xmm0, %xmm0
@@ -4711,28 +4711,28 @@ define <16 x i8> @trunc_or_v16i64_v16i8(<16 x i64> %a0, <16 x i64> %a1) nounwind
 ;
 ; AVX1-LABEL: trunc_or_v16i64_v16i8:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vorpd %ymm4, %ymm0, %ymm0
-; AVX1-NEXT:    vorpd %ymm5, %ymm1, %ymm1
-; AVX1-NEXT:    vorpd %ymm6, %ymm2, %ymm2
-; AVX1-NEXT:    vorpd %ymm7, %ymm3, %ymm3
+; AVX1-NEXT:    vorps %ymm4, %ymm0, %ymm0
+; AVX1-NEXT:    vorps %ymm5, %ymm1, %ymm1
+; AVX1-NEXT:    vorps %ymm6, %ymm2, %ymm2
+; AVX1-NEXT:    vorps %ymm7, %ymm3, %ymm3
 ; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm4
 ; AVX1-NEXT:    vmovddup {{.*#+}} xmm5 = [1.2598673968951787E-321,1.2598673968951787E-321]
 ; AVX1-NEXT:    # xmm5 = mem[0,0]
-; AVX1-NEXT:    vandpd %xmm5, %xmm4, %xmm4
-; AVX1-NEXT:    vandpd %xmm5, %xmm3, %xmm3
+; AVX1-NEXT:    vandps %xmm5, %xmm4, %xmm4
+; AVX1-NEXT:    vandps %xmm5, %xmm3, %xmm3
 ; AVX1-NEXT:    vpackusdw %xmm4, %xmm3, %xmm3
 ; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm4
-; AVX1-NEXT:    vandpd %xmm5, %xmm4, %xmm4
-; AVX1-NEXT:    vandpd %xmm5, %xmm2, %xmm2
+; AVX1-NEXT:    vandps %xmm5, %xmm4, %xmm4
+; AVX1-NEXT:    vandps %xmm5, %xmm2, %xmm2
 ; AVX1-NEXT:    vpackusdw %xmm4, %xmm2, %xmm2
 ; AVX1-NEXT:    vpackusdw %xmm3, %xmm2, %xmm2
 ; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm3
-; AVX1-NEXT:    vandpd %xmm5, %xmm3, %xmm3
-; AVX1-NEXT:    vandpd %xmm5, %xmm1, %xmm1
+; AVX1-NEXT:    vandps %xmm5, %xmm3, %xmm3
+; AVX1-NEXT:    vandps %xmm5, %xmm1, %xmm1
 ; AVX1-NEXT:    vpackusdw %xmm3, %xmm1, %xmm1
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
-; AVX1-NEXT:    vandpd %xmm5, %xmm3, %xmm3
-; AVX1-NEXT:    vandpd %xmm5, %xmm0, %xmm0
+; AVX1-NEXT:    vandps %xmm5, %xmm3, %xmm3
+; AVX1-NEXT:    vandps %xmm5, %xmm0, %xmm0
 ; AVX1-NEXT:    vpackusdw %xmm3, %xmm0, %xmm0
 ; AVX1-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpackuswb %xmm2, %xmm0, %xmm0
@@ -5110,21 +5110,21 @@ define <16 x i8> @trunc_or_const_v16i64_v16i8(<16 x i64> %a0) nounwind {
 ; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm4
 ; AVX1-NEXT:    vmovddup {{.*#+}} xmm5 = [1.2598673968951787E-321,1.2598673968951787E-321]
 ; AVX1-NEXT:    # xmm5 = mem[0,0]
-; AVX1-NEXT:    vandpd %xmm5, %xmm4, %xmm4
-; AVX1-NEXT:    vandpd %xmm5, %xmm3, %xmm3
+; AVX1-NEXT:    vandps %xmm5, %xmm4, %xmm4
+; AVX1-NEXT:    vandps %xmm5, %xmm3, %xmm3
 ; AVX1-NEXT:    vpackusdw %xmm4, %xmm3, %xmm3
 ; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm4
-; AVX1-NEXT:    vandpd %xmm5, %xmm4, %xmm4
-; AVX1-NEXT:    vandpd %xmm5, %xmm2, %xmm2
+; AVX1-NEXT:    vandps %xmm5, %xmm4, %xmm4
+; AVX1-NEXT:    vandps %xmm5, %xmm2, %xmm2
 ; AVX1-NEXT:    vpackusdw %xmm4, %xmm2, %xmm2
 ; AVX1-NEXT:    vpackusdw %xmm3, %xmm2, %xmm2
 ; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm3
-; AVX1-NEXT:    vandpd %xmm5, %xmm3, %xmm3
-; AVX1-NEXT:    vandpd %xmm5, %xmm1, %xmm1
+; AVX1-NEXT:    vandps %xmm5, %xmm3, %xmm3
+; AVX1-NEXT:    vandps %xmm5, %xmm1, %xmm1
 ; AVX1-NEXT:    vpackusdw %xmm3, %xmm1, %xmm1
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
-; AVX1-NEXT:    vandpd %xmm5, %xmm3, %xmm3
-; AVX1-NEXT:    vandpd %xmm5, %xmm0, %xmm0
+; AVX1-NEXT:    vandps %xmm5, %xmm3, %xmm3
+; AVX1-NEXT:    vandps %xmm5, %xmm0, %xmm0
 ; AVX1-NEXT:    vpackusdw %xmm3, %xmm0, %xmm0
 ; AVX1-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpackuswb %xmm2, %xmm0, %xmm0
diff --git a/test/CodeGen/X86/vector-trunc-widen.ll b/test/CodeGen/X86/vector-trunc-widen.ll
index 497393786aa99085e5f84cea1183b1d5eddf2a3b..c3f5729b137dad4ed0fe98f5aa307d4950a9529d 100644
--- a/test/CodeGen/X86/vector-trunc-widen.ll
+++ b/test/CodeGen/X86/vector-trunc-widen.ll
@@ -288,12 +288,12 @@ define void @trunc8i64_8i8(<8 x i64> %a) {
 ; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
 ; AVX1-NEXT:    vmovddup {{.*#+}} xmm3 = [1.2598673968951787E-321,1.2598673968951787E-321]
 ; AVX1-NEXT:    # xmm3 = mem[0,0]
-; AVX1-NEXT:    vandpd %xmm3, %xmm2, %xmm2
-; AVX1-NEXT:    vandpd %xmm3, %xmm1, %xmm1
+; AVX1-NEXT:    vandps %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vandps %xmm3, %xmm1, %xmm1
 ; AVX1-NEXT:    vpackusdw %xmm2, %xmm1, %xmm1
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
-; AVX1-NEXT:    vandpd %xmm3, %xmm2, %xmm2
-; AVX1-NEXT:    vandpd %xmm3, %xmm0, %xmm0
+; AVX1-NEXT:    vandps %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vandps %xmm3, %xmm0, %xmm0
 ; AVX1-NEXT:    vpackusdw %xmm2, %xmm0, %xmm0
 ; AVX1-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpackuswb %xmm0, %xmm0, %xmm0
@@ -468,11 +468,7 @@ entry:
 define <8 x i16> @trunc8i32_8i16_lshr(<8 x i32> %a) {
 ; SSE2-LABEL: trunc8i32_8i16_lshr:
 ; SSE2:       # %bb.0: # %entry
-; SSE2-NEXT:    psrld $16, %xmm0
-; SSE2-NEXT:    psrld $16, %xmm1
-; SSE2-NEXT:    pslld $16, %xmm1
 ; SSE2-NEXT:    psrad $16, %xmm1
-; SSE2-NEXT:    pslld $16, %xmm0
 ; SSE2-NEXT:    psrad $16, %xmm0
 ; SSE2-NEXT:    packssdw %xmm1, %xmm0
 ; SSE2-NEXT:    retq
@@ -767,18 +763,10 @@ entry:
 define void @trunc16i32_16i16_lshr(<16 x i32> %a) {
 ; SSE2-LABEL: trunc16i32_16i16_lshr:
 ; SSE2:       # %bb.0: # %entry
-; SSE2-NEXT:    psrld $16, %xmm2
-; SSE2-NEXT:    psrld $16, %xmm3
-; SSE2-NEXT:    psrld $16, %xmm0
-; SSE2-NEXT:    psrld $16, %xmm1
-; SSE2-NEXT:    pslld $16, %xmm1
 ; SSE2-NEXT:    psrad $16, %xmm1
-; SSE2-NEXT:    pslld $16, %xmm0
 ; SSE2-NEXT:    psrad $16, %xmm0
 ; SSE2-NEXT:    packssdw %xmm1, %xmm0
-; SSE2-NEXT:    pslld $16, %xmm3
 ; SSE2-NEXT:    psrad $16, %xmm3
-; SSE2-NEXT:    pslld $16, %xmm2
 ; SSE2-NEXT:    psrad $16, %xmm2
 ; SSE2-NEXT:    packssdw %xmm3, %xmm2
 ; SSE2-NEXT:    movdqu %xmm2, (%rax)
@@ -787,18 +775,10 @@ define void @trunc16i32_16i16_lshr(<16 x i32> %a) {
 ;
 ; SSSE3-LABEL: trunc16i32_16i16_lshr:
 ; SSSE3:       # %bb.0: # %entry
-; SSSE3-NEXT:    psrld $16, %xmm2
-; SSSE3-NEXT:    psrld $16, %xmm3
-; SSSE3-NEXT:    psrld $16, %xmm0
-; SSSE3-NEXT:    psrld $16, %xmm1
-; SSSE3-NEXT:    pslld $16, %xmm1
 ; SSSE3-NEXT:    psrad $16, %xmm1
-; SSSE3-NEXT:    pslld $16, %xmm0
 ; SSSE3-NEXT:    psrad $16, %xmm0
 ; SSSE3-NEXT:    packssdw %xmm1, %xmm0
-; SSSE3-NEXT:    pslld $16, %xmm3
 ; SSSE3-NEXT:    psrad $16, %xmm3
-; SSSE3-NEXT:    pslld $16, %xmm2
 ; SSSE3-NEXT:    psrad $16, %xmm2
 ; SSSE3-NEXT:    packssdw %xmm3, %xmm2
 ; SSSE3-NEXT:    movdqu %xmm2, (%rax)
diff --git a/test/CodeGen/X86/vector-trunc.ll b/test/CodeGen/X86/vector-trunc.ll
index c17a618f28b0fc25d0a2beaaef10d17f77f18aa2..98362b1176226d4be0bfe09ba89591b9ca961b1f 100644
--- a/test/CodeGen/X86/vector-trunc.ll
+++ b/test/CodeGen/X86/vector-trunc.ll
@@ -288,12 +288,12 @@ define void @trunc8i64_8i8(<8 x i64> %a) {
 ; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
 ; AVX1-NEXT:    vmovddup {{.*#+}} xmm3 = [1.2598673968951787E-321,1.2598673968951787E-321]
 ; AVX1-NEXT:    # xmm3 = mem[0,0]
-; AVX1-NEXT:    vandpd %xmm3, %xmm2, %xmm2
-; AVX1-NEXT:    vandpd %xmm3, %xmm1, %xmm1
+; AVX1-NEXT:    vandps %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vandps %xmm3, %xmm1, %xmm1
 ; AVX1-NEXT:    vpackusdw %xmm2, %xmm1, %xmm1
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
-; AVX1-NEXT:    vandpd %xmm3, %xmm2, %xmm2
-; AVX1-NEXT:    vandpd %xmm3, %xmm0, %xmm0
+; AVX1-NEXT:    vandps %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vandps %xmm3, %xmm0, %xmm0
 ; AVX1-NEXT:    vpackusdw %xmm2, %xmm0, %xmm0
 ; AVX1-NEXT:    vpackusdw %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    vpackuswb %xmm0, %xmm0, %xmm0
@@ -478,11 +478,7 @@ entry:
 define <8 x i16> @trunc8i32_8i16_lshr(<8 x i32> %a) {
 ; SSE2-LABEL: trunc8i32_8i16_lshr:
 ; SSE2:       # %bb.0: # %entry
-; SSE2-NEXT:    psrld $16, %xmm0
-; SSE2-NEXT:    psrld $16, %xmm1
-; SSE2-NEXT:    pslld $16, %xmm1
 ; SSE2-NEXT:    psrad $16, %xmm1
-; SSE2-NEXT:    pslld $16, %xmm0
 ; SSE2-NEXT:    psrad $16, %xmm0
 ; SSE2-NEXT:    packssdw %xmm1, %xmm0
 ; SSE2-NEXT:    retq
@@ -777,18 +773,10 @@ entry:
 define void @trunc16i32_16i16_lshr(<16 x i32> %a) {
 ; SSE2-LABEL: trunc16i32_16i16_lshr:
 ; SSE2:       # %bb.0: # %entry
-; SSE2-NEXT:    psrld $16, %xmm2
-; SSE2-NEXT:    psrld $16, %xmm3
-; SSE2-NEXT:    psrld $16, %xmm0
-; SSE2-NEXT:    psrld $16, %xmm1
-; SSE2-NEXT:    pslld $16, %xmm1
 ; SSE2-NEXT:    psrad $16, %xmm1
-; SSE2-NEXT:    pslld $16, %xmm0
 ; SSE2-NEXT:    psrad $16, %xmm0
 ; SSE2-NEXT:    packssdw %xmm1, %xmm0
-; SSE2-NEXT:    pslld $16, %xmm3
 ; SSE2-NEXT:    psrad $16, %xmm3
-; SSE2-NEXT:    pslld $16, %xmm2
 ; SSE2-NEXT:    psrad $16, %xmm2
 ; SSE2-NEXT:    packssdw %xmm3, %xmm2
 ; SSE2-NEXT:    movdqu %xmm2, (%rax)
@@ -797,18 +785,10 @@ define void @trunc16i32_16i16_lshr(<16 x i32> %a) {
 ;
 ; SSSE3-LABEL: trunc16i32_16i16_lshr:
 ; SSSE3:       # %bb.0: # %entry
-; SSSE3-NEXT:    psrld $16, %xmm2
-; SSSE3-NEXT:    psrld $16, %xmm3
-; SSSE3-NEXT:    psrld $16, %xmm0
-; SSSE3-NEXT:    psrld $16, %xmm1
-; SSSE3-NEXT:    pslld $16, %xmm1
 ; SSSE3-NEXT:    psrad $16, %xmm1
-; SSSE3-NEXT:    pslld $16, %xmm0
 ; SSSE3-NEXT:    psrad $16, %xmm0
 ; SSSE3-NEXT:    packssdw %xmm1, %xmm0
-; SSSE3-NEXT:    pslld $16, %xmm3
 ; SSSE3-NEXT:    psrad $16, %xmm3
-; SSSE3-NEXT:    pslld $16, %xmm2
 ; SSSE3-NEXT:    psrad $16, %xmm2
 ; SSSE3-NEXT:    packssdw %xmm3, %xmm2
 ; SSSE3-NEXT:    movdqu %xmm2, (%rax)
diff --git a/test/CodeGen/X86/verifier-generic-extend-truncate.mir b/test/CodeGen/X86/verifier-generic-extend-truncate.mir
index 802a59f2863c189309f07d9c3d89b5076e2e5539..28bca8845ce05389d72f19c750c0973ecfa52c40 100644
--- a/test/CodeGen/X86/verifier-generic-extend-truncate.mir
+++ b/test/CodeGen/X86/verifier-generic-extend-truncate.mir
@@ -5,12 +5,12 @@
 # CHECK-NEXT: - basic block: %bb.1
 # CHECK-NEXT: - instruction: %t_p:_(s32) = G_TRUNC %p:_(p0)
 
-# CHECK: Bad machine code: Generic extend/truncate must be all-vector or all-scalar
+# CHECK: Bad machine code: operand types must be all-vector or all-scalar
 # CHECK-NEXT: - function:    bad_generic_extends_and_truncates
 # CHECK-NEXT: - basic block: %bb.2
 # CHECK-NEXT: - instruction: %se_i32:_(<2 x s64>) = G_SEXT %i32:_(s32)
 
-# CHECK: Bad machine code: Generic vector extend/truncate must preserve number of lanes
+# CHECK: Bad machine code: operand types must preserve number of vector elements
 # CHECK-NEXT: - function:    bad_generic_extends_and_truncates
 # CHECK-NEXT: - basic block: %bb.3
 # CHECK-NEXT: - instruction: %ze_v2i32:_(<4 x s64>) = G_ZEXT %v2i32:_(<2 x s32>)
@@ -31,7 +31,7 @@
 # CHECK-NEXT: - basic block: %bb.6
 # CHECK-NEXT: - instruction: %ze_v2i128:_(<4 x p0>) = G_ZEXT %v2i128:_(<2 x s128>)
 
-# CHECK: Bad machine code: Generic vector extend/truncate must preserve number of lanes
+# CHECK: Bad machine code: operand types must preserve number of vector elements
 # CHECK-NEXT: - function:    bad_generic_extends_and_truncates
 # CHECK-NEXT: - basic block: %bb.6
 # CHECK-NEXT: - instruction: %ze_v2i128:_(<4 x p0>) = G_ZEXT %v2i128:_(<2 x s128>)
@@ -47,7 +47,7 @@
 # CHECK-NEXT: - basic block: %bb.6
 # CHECK-NEXT: - instruction: %fe_v2f128:_(p0) = G_FPEXT %v2f128:_(<2 x s128>)
 
-# CHECK: Bad machine code: Generic extend/truncate must be all-vector or all-scalar
+# CHECK: Bad machine code: operand types must be all-vector or all-scalar
 # CHECK-NEXT: - function:    bad_generic_extends_and_truncates
 # CHECK-NEXT: - basic block: %bb.6
 # CHECK-NEXT: - instruction: %fe_v2f128:_(p0) = G_FPEXT %v2f128:_(<2 x s128>)
diff --git a/test/CodeGen/X86/viabs.ll b/test/CodeGen/X86/viabs.ll
index af7e7964173f71440bf8083eb72f4dccbf5a7bb3..cbb6a9aec6ee77505d068fbf0a4aaf0e270a8a12 100644
--- a/test/CodeGen/X86/viabs.ll
+++ b/test/CodeGen/X86/viabs.ll
@@ -1,10 +1,11 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+sse2     | FileCheck %s --check-prefix=SSE --check-prefix=SSE2
-; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+ssse3    | FileCheck %s --check-prefix=SSE --check-prefix=SSSE3
-; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx      | FileCheck %s --check-prefix=AVX --check-prefix=AVX1
-; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx2     | FileCheck %s --check-prefix=AVX --check-prefix=AVX2
-; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx512vl --show-mc-encoding | FileCheck %s --check-prefix=AVX --check-prefix=AVX512 --check-prefix=AVX512F --check-prefix=AVX512VL
-; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx512vl,+avx512bw --show-mc-encoding | FileCheck %s --check-prefix=AVX --check-prefix=AVX512 --check-prefix=AVX512BW
+; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+sse2     | FileCheck %s --check-prefixes=SSE,SSE2
+; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+ssse3    | FileCheck %s --check-prefixes=SSE,SSSE3
+; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+sse4.1   | FileCheck %s --check-prefixes=SSE,SSE41
+; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx      | FileCheck %s --check-prefixes=AVX,AVX1
+; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx2     | FileCheck %s --check-prefixes=AVX,AVX2
+; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx512vl --show-mc-encoding | FileCheck %s --check-prefixes=AVX,AVX512,AVX512F,AVX512VL
+; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx512vl,+avx512bw --show-mc-encoding | FileCheck %s --check-prefixes=AVX,AVX512,AVX512BW
 
 define <4 x i32> @test_abs_gt_v4i32(<4 x i32> %a) nounwind {
 ; SSE2-LABEL: test_abs_gt_v4i32:
@@ -20,6 +21,11 @@ define <4 x i32> @test_abs_gt_v4i32(<4 x i32> %a) nounwind {
 ; SSSE3-NEXT:    pabsd %xmm0, %xmm0
 ; SSSE3-NEXT:    retq
 ;
+; SSE41-LABEL: test_abs_gt_v4i32:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    pabsd %xmm0, %xmm0
+; SSE41-NEXT:    retq
+;
 ; AVX1-LABEL: test_abs_gt_v4i32:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vpabsd %xmm0, %xmm0
@@ -54,6 +60,11 @@ define <4 x i32> @test_abs_ge_v4i32(<4 x i32> %a) nounwind {
 ; SSSE3-NEXT:    pabsd %xmm0, %xmm0
 ; SSSE3-NEXT:    retq
 ;
+; SSE41-LABEL: test_abs_ge_v4i32:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    pabsd %xmm0, %xmm0
+; SSE41-NEXT:    retq
+;
 ; AVX1-LABEL: test_abs_ge_v4i32:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vpabsd %xmm0, %xmm0
@@ -88,6 +99,11 @@ define <8 x i16> @test_abs_gt_v8i16(<8 x i16> %a) nounwind {
 ; SSSE3-NEXT:    pabsw %xmm0, %xmm0
 ; SSSE3-NEXT:    retq
 ;
+; SSE41-LABEL: test_abs_gt_v8i16:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    pabsw %xmm0, %xmm0
+; SSE41-NEXT:    retq
+;
 ; AVX1-LABEL: test_abs_gt_v8i16:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vpabsw %xmm0, %xmm0
@@ -127,6 +143,11 @@ define <16 x i8> @test_abs_lt_v16i8(<16 x i8> %a) nounwind {
 ; SSSE3-NEXT:    pabsb %xmm0, %xmm0
 ; SSSE3-NEXT:    retq
 ;
+; SSE41-LABEL: test_abs_lt_v16i8:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    pabsb %xmm0, %xmm0
+; SSE41-NEXT:    retq
+;
 ; AVX1-LABEL: test_abs_lt_v16i8:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vpabsb %xmm0, %xmm0
@@ -166,6 +187,11 @@ define <4 x i32> @test_abs_le_v4i32(<4 x i32> %a) nounwind {
 ; SSSE3-NEXT:    pabsd %xmm0, %xmm0
 ; SSSE3-NEXT:    retq
 ;
+; SSE41-LABEL: test_abs_le_v4i32:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    pabsd %xmm0, %xmm0
+; SSE41-NEXT:    retq
+;
 ; AVX1-LABEL: test_abs_le_v4i32:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vpabsd %xmm0, %xmm0
@@ -205,6 +231,12 @@ define <8 x i32> @test_abs_gt_v8i32(<8 x i32> %a) nounwind {
 ; SSSE3-NEXT:    pabsd %xmm1, %xmm1
 ; SSSE3-NEXT:    retq
 ;
+; SSE41-LABEL: test_abs_gt_v8i32:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    pabsd %xmm0, %xmm0
+; SSE41-NEXT:    pabsd %xmm1, %xmm1
+; SSE41-NEXT:    retq
+;
 ; AVX1-LABEL: test_abs_gt_v8i32:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vpabsd %xmm0, %xmm1
@@ -247,6 +279,12 @@ define <8 x i32> @test_abs_ge_v8i32(<8 x i32> %a) nounwind {
 ; SSSE3-NEXT:    pabsd %xmm1, %xmm1
 ; SSSE3-NEXT:    retq
 ;
+; SSE41-LABEL: test_abs_ge_v8i32:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    pabsd %xmm0, %xmm0
+; SSE41-NEXT:    pabsd %xmm1, %xmm1
+; SSE41-NEXT:    retq
+;
 ; AVX1-LABEL: test_abs_ge_v8i32:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vpabsd %xmm0, %xmm1
@@ -289,6 +327,12 @@ define <16 x i16> @test_abs_gt_v16i16(<16 x i16> %a) nounwind {
 ; SSSE3-NEXT:    pabsw %xmm1, %xmm1
 ; SSSE3-NEXT:    retq
 ;
+; SSE41-LABEL: test_abs_gt_v16i16:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    pabsw %xmm0, %xmm0
+; SSE41-NEXT:    pabsw %xmm1, %xmm1
+; SSE41-NEXT:    retq
+;
 ; AVX1-LABEL: test_abs_gt_v16i16:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vpabsw %xmm0, %xmm1
@@ -336,6 +380,12 @@ define <32 x i8> @test_abs_lt_v32i8(<32 x i8> %a) nounwind {
 ; SSSE3-NEXT:    pabsb %xmm1, %xmm1
 ; SSSE3-NEXT:    retq
 ;
+; SSE41-LABEL: test_abs_lt_v32i8:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    pabsb %xmm0, %xmm0
+; SSE41-NEXT:    pabsb %xmm1, %xmm1
+; SSE41-NEXT:    retq
+;
 ; AVX1-LABEL: test_abs_lt_v32i8:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vpabsb %xmm0, %xmm1
@@ -383,6 +433,12 @@ define <8 x i32> @test_abs_le_v8i32(<8 x i32> %a) nounwind {
 ; SSSE3-NEXT:    pabsd %xmm1, %xmm1
 ; SSSE3-NEXT:    retq
 ;
+; SSE41-LABEL: test_abs_le_v8i32:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    pabsd %xmm0, %xmm0
+; SSE41-NEXT:    pabsd %xmm1, %xmm1
+; SSE41-NEXT:    retq
+;
 ; AVX1-LABEL: test_abs_le_v8i32:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vpabsd %xmm0, %xmm1
@@ -435,6 +491,14 @@ define <16 x i32> @test_abs_le_16i32(<16 x i32> %a) nounwind {
 ; SSSE3-NEXT:    pabsd %xmm3, %xmm3
 ; SSSE3-NEXT:    retq
 ;
+; SSE41-LABEL: test_abs_le_16i32:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    pabsd %xmm0, %xmm0
+; SSE41-NEXT:    pabsd %xmm1, %xmm1
+; SSE41-NEXT:    pabsd %xmm2, %xmm2
+; SSE41-NEXT:    pabsd %xmm3, %xmm3
+; SSE41-NEXT:    retq
+;
 ; AVX1-LABEL: test_abs_le_16i32:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vpabsd %xmm0, %xmm2
@@ -464,29 +528,43 @@ define <16 x i32> @test_abs_le_16i32(<16 x i32> %a) nounwind {
 }
 
 define <2 x i64> @test_abs_ge_v2i64(<2 x i64> %a) nounwind {
-; SSE-LABEL: test_abs_ge_v2i64:
-; SSE:       # %bb.0:
-; SSE-NEXT:    movdqa %xmm0, %xmm1
-; SSE-NEXT:    psrad $31, %xmm1
-; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSE-NEXT:    paddq %xmm1, %xmm0
-; SSE-NEXT:    pxor %xmm1, %xmm0
-; SSE-NEXT:    retq
+; SSE2-LABEL: test_abs_ge_v2i64:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa %xmm0, %xmm1
+; SSE2-NEXT:    psrad $31, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSE2-NEXT:    paddq %xmm1, %xmm0
+; SSE2-NEXT:    pxor %xmm1, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: test_abs_ge_v2i64:
+; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    movdqa %xmm0, %xmm1
+; SSSE3-NEXT:    psrad $31, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSSE3-NEXT:    paddq %xmm1, %xmm0
+; SSSE3-NEXT:    pxor %xmm1, %xmm0
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: test_abs_ge_v2i64:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    pxor %xmm1, %xmm1
+; SSE41-NEXT:    psubq %xmm0, %xmm1
+; SSE41-NEXT:    blendvpd %xmm0, %xmm1, %xmm0
+; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: test_abs_ge_v2i64:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpgtq %xmm0, %xmm1, %xmm1
-; AVX1-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpsubq %xmm0, %xmm1, %xmm1
+; AVX1-NEXT:    vblendvpd %xmm0, %xmm1, %xmm0, %xmm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: test_abs_ge_v2i64:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT:    vpcmpgtq %xmm0, %xmm1, %xmm1
-; AVX2-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpxor %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpsubq %xmm0, %xmm1, %xmm1
+; AVX2-NEXT:    vblendvpd %xmm0, %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: test_abs_ge_v2i64:
@@ -500,39 +578,62 @@ define <2 x i64> @test_abs_ge_v2i64(<2 x i64> %a) nounwind {
 }
 
 define <4 x i64> @test_abs_gt_v4i64(<4 x i64> %a) nounwind {
-; SSE-LABEL: test_abs_gt_v4i64:
-; SSE:       # %bb.0:
-; SSE-NEXT:    movdqa %xmm0, %xmm2
-; SSE-NEXT:    psrad $31, %xmm2
-; SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSE-NEXT:    paddq %xmm2, %xmm0
-; SSE-NEXT:    pxor %xmm2, %xmm0
-; SSE-NEXT:    movdqa %xmm1, %xmm2
-; SSE-NEXT:    psrad $31, %xmm2
-; SSE-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSE-NEXT:    paddq %xmm2, %xmm1
-; SSE-NEXT:    pxor %xmm2, %xmm1
-; SSE-NEXT:    retq
+; SSE2-LABEL: test_abs_gt_v4i64:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa %xmm0, %xmm2
+; SSE2-NEXT:    psrad $31, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; SSE2-NEXT:    paddq %xmm2, %xmm0
+; SSE2-NEXT:    pxor %xmm2, %xmm0
+; SSE2-NEXT:    movdqa %xmm1, %xmm2
+; SSE2-NEXT:    psrad $31, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; SSE2-NEXT:    paddq %xmm2, %xmm1
+; SSE2-NEXT:    pxor %xmm2, %xmm1
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: test_abs_gt_v4i64:
+; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    movdqa %xmm0, %xmm2
+; SSSE3-NEXT:    psrad $31, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; SSSE3-NEXT:    paddq %xmm2, %xmm0
+; SSSE3-NEXT:    pxor %xmm2, %xmm0
+; SSSE3-NEXT:    movdqa %xmm1, %xmm2
+; SSSE3-NEXT:    psrad $31, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; SSSE3-NEXT:    paddq %xmm2, %xmm1
+; SSSE3-NEXT:    pxor %xmm2, %xmm1
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: test_abs_gt_v4i64:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movdqa %xmm0, %xmm2
+; SSE41-NEXT:    pxor %xmm3, %xmm3
+; SSE41-NEXT:    pxor %xmm4, %xmm4
+; SSE41-NEXT:    psubq %xmm0, %xmm4
+; SSE41-NEXT:    blendvpd %xmm0, %xmm4, %xmm2
+; SSE41-NEXT:    psubq %xmm1, %xmm3
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm3, %xmm1
+; SSE41-NEXT:    movapd %xmm2, %xmm0
+; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: test_abs_gt_v4i64:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
 ; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm2, %xmm3
-; AVX1-NEXT:    vpcmpgtq %xmm0, %xmm2, %xmm2
-; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm2, %ymm4
-; AVX1-NEXT:    vpaddq %xmm3, %xmm1, %xmm1
-; AVX1-NEXT:    vpaddq %xmm2, %xmm0, %xmm0
-; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
-; AVX1-NEXT:    vxorps %ymm4, %ymm0, %ymm0
+; AVX1-NEXT:    vpsubq %xmm1, %xmm2, %xmm1
+; AVX1-NEXT:    vpsubq %xmm0, %xmm2, %xmm2
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm1
+; AVX1-NEXT:    vblendvpd %ymm0, %ymm1, %ymm0, %ymm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: test_abs_gt_v4i64:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT:    vpcmpgtq %ymm0, %ymm1, %ymm1
-; AVX2-NEXT:    vpaddq %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpxor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpsubq %ymm0, %ymm1, %ymm1
+; AVX2-NEXT:    vblendvpd %ymm0, %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: test_abs_gt_v4i64:
@@ -546,60 +647,97 @@ define <4 x i64> @test_abs_gt_v4i64(<4 x i64> %a) nounwind {
 }
 
 define <8 x i64> @test_abs_le_v8i64(<8 x i64> %a) nounwind {
-; SSE-LABEL: test_abs_le_v8i64:
-; SSE:       # %bb.0:
-; SSE-NEXT:    movdqa %xmm0, %xmm4
-; SSE-NEXT:    psrad $31, %xmm4
-; SSE-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSE-NEXT:    paddq %xmm4, %xmm0
-; SSE-NEXT:    pxor %xmm4, %xmm0
-; SSE-NEXT:    movdqa %xmm1, %xmm4
-; SSE-NEXT:    psrad $31, %xmm4
-; SSE-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSE-NEXT:    paddq %xmm4, %xmm1
-; SSE-NEXT:    pxor %xmm4, %xmm1
-; SSE-NEXT:    movdqa %xmm2, %xmm4
-; SSE-NEXT:    psrad $31, %xmm4
-; SSE-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSE-NEXT:    paddq %xmm4, %xmm2
-; SSE-NEXT:    pxor %xmm4, %xmm2
-; SSE-NEXT:    movdqa %xmm3, %xmm4
-; SSE-NEXT:    psrad $31, %xmm4
-; SSE-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSE-NEXT:    paddq %xmm4, %xmm3
-; SSE-NEXT:    pxor %xmm4, %xmm3
-; SSE-NEXT:    retq
+; SSE2-LABEL: test_abs_le_v8i64:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqa %xmm0, %xmm4
+; SSE2-NEXT:    psrad $31, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSE2-NEXT:    paddq %xmm4, %xmm0
+; SSE2-NEXT:    pxor %xmm4, %xmm0
+; SSE2-NEXT:    movdqa %xmm1, %xmm4
+; SSE2-NEXT:    psrad $31, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSE2-NEXT:    paddq %xmm4, %xmm1
+; SSE2-NEXT:    pxor %xmm4, %xmm1
+; SSE2-NEXT:    movdqa %xmm2, %xmm4
+; SSE2-NEXT:    psrad $31, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSE2-NEXT:    paddq %xmm4, %xmm2
+; SSE2-NEXT:    pxor %xmm4, %xmm2
+; SSE2-NEXT:    movdqa %xmm3, %xmm4
+; SSE2-NEXT:    psrad $31, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSE2-NEXT:    paddq %xmm4, %xmm3
+; SSE2-NEXT:    pxor %xmm4, %xmm3
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: test_abs_le_v8i64:
+; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    movdqa %xmm0, %xmm4
+; SSSE3-NEXT:    psrad $31, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    paddq %xmm4, %xmm0
+; SSSE3-NEXT:    pxor %xmm4, %xmm0
+; SSSE3-NEXT:    movdqa %xmm1, %xmm4
+; SSSE3-NEXT:    psrad $31, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    paddq %xmm4, %xmm1
+; SSSE3-NEXT:    pxor %xmm4, %xmm1
+; SSSE3-NEXT:    movdqa %xmm2, %xmm4
+; SSSE3-NEXT:    psrad $31, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    paddq %xmm4, %xmm2
+; SSSE3-NEXT:    pxor %xmm4, %xmm2
+; SSSE3-NEXT:    movdqa %xmm3, %xmm4
+; SSSE3-NEXT:    psrad $31, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    paddq %xmm4, %xmm3
+; SSSE3-NEXT:    pxor %xmm4, %xmm3
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: test_abs_le_v8i64:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movdqa %xmm0, %xmm4
+; SSE41-NEXT:    pxor %xmm5, %xmm5
+; SSE41-NEXT:    pxor %xmm6, %xmm6
+; SSE41-NEXT:    psubq %xmm0, %xmm6
+; SSE41-NEXT:    blendvpd %xmm0, %xmm6, %xmm4
+; SSE41-NEXT:    pxor %xmm6, %xmm6
+; SSE41-NEXT:    psubq %xmm1, %xmm6
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm6, %xmm1
+; SSE41-NEXT:    pxor %xmm6, %xmm6
+; SSE41-NEXT:    psubq %xmm2, %xmm6
+; SSE41-NEXT:    movdqa %xmm2, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm6, %xmm2
+; SSE41-NEXT:    psubq %xmm3, %xmm5
+; SSE41-NEXT:    movdqa %xmm3, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm5, %xmm3
+; SSE41-NEXT:    movapd %xmm4, %xmm0
+; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: test_abs_le_v8i64:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
 ; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpcmpgtq %xmm2, %xmm3, %xmm4
-; AVX1-NEXT:    vpcmpgtq %xmm0, %xmm3, %xmm5
-; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm5, %ymm6
-; AVX1-NEXT:    vpaddq %xmm4, %xmm2, %xmm2
-; AVX1-NEXT:    vpaddq %xmm5, %xmm0, %xmm0
-; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
-; AVX1-NEXT:    vxorps %ymm6, %ymm0, %ymm0
+; AVX1-NEXT:    vpsubq %xmm2, %xmm3, %xmm2
+; AVX1-NEXT:    vpsubq %xmm0, %xmm3, %xmm4
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm4, %ymm2
+; AVX1-NEXT:    vblendvpd %ymm0, %ymm2, %ymm0, %ymm0
 ; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
-; AVX1-NEXT:    vpcmpgtq %xmm2, %xmm3, %xmm4
-; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm3, %xmm3
-; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm3, %ymm5
-; AVX1-NEXT:    vpaddq %xmm4, %xmm2, %xmm2
-; AVX1-NEXT:    vpaddq %xmm3, %xmm1, %xmm1
-; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
-; AVX1-NEXT:    vxorps %ymm5, %ymm1, %ymm1
+; AVX1-NEXT:    vpsubq %xmm2, %xmm3, %xmm2
+; AVX1-NEXT:    vpsubq %xmm1, %xmm3, %xmm3
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm3, %ymm2
+; AVX1-NEXT:    vblendvpd %ymm1, %ymm2, %ymm1, %ymm1
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: test_abs_le_v8i64:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpgtq %ymm0, %ymm2, %ymm3
-; AVX2-NEXT:    vpaddq %ymm3, %ymm0, %ymm0
-; AVX2-NEXT:    vpxor %ymm3, %ymm0, %ymm0
-; AVX2-NEXT:    vpcmpgtq %ymm1, %ymm2, %ymm2
-; AVX2-NEXT:    vpaddq %ymm2, %ymm1, %ymm1
-; AVX2-NEXT:    vpxor %ymm2, %ymm1, %ymm1
+; AVX2-NEXT:    vpsubq %ymm0, %ymm2, %ymm3
+; AVX2-NEXT:    vblendvpd %ymm0, %ymm3, %ymm0, %ymm0
+; AVX2-NEXT:    vpsubq %ymm1, %ymm2, %ymm2
+; AVX2-NEXT:    vblendvpd %ymm1, %ymm2, %ymm1, %ymm1
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: test_abs_le_v8i64:
@@ -613,55 +751,103 @@ define <8 x i64> @test_abs_le_v8i64(<8 x i64> %a) nounwind {
 }
 
 define <8 x i64> @test_abs_le_v8i64_fold(<8 x i64>* %a.ptr) nounwind {
-; SSE-LABEL: test_abs_le_v8i64_fold:
-; SSE:       # %bb.0:
-; SSE-NEXT:    movdqu (%rdi), %xmm0
-; SSE-NEXT:    movdqu 16(%rdi), %xmm1
-; SSE-NEXT:    movdqu 32(%rdi), %xmm2
-; SSE-NEXT:    movdqu 48(%rdi), %xmm3
-; SSE-NEXT:    movdqa %xmm0, %xmm4
-; SSE-NEXT:    psrad $31, %xmm4
-; SSE-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSE-NEXT:    paddq %xmm4, %xmm0
-; SSE-NEXT:    pxor %xmm4, %xmm0
-; SSE-NEXT:    movdqa %xmm1, %xmm4
-; SSE-NEXT:    psrad $31, %xmm4
-; SSE-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSE-NEXT:    paddq %xmm4, %xmm1
-; SSE-NEXT:    pxor %xmm4, %xmm1
-; SSE-NEXT:    movdqa %xmm2, %xmm4
-; SSE-NEXT:    psrad $31, %xmm4
-; SSE-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSE-NEXT:    paddq %xmm4, %xmm2
-; SSE-NEXT:    pxor %xmm4, %xmm2
-; SSE-NEXT:    movdqa %xmm3, %xmm4
-; SSE-NEXT:    psrad $31, %xmm4
-; SSE-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSE-NEXT:    paddq %xmm4, %xmm3
-; SSE-NEXT:    pxor %xmm4, %xmm3
-; SSE-NEXT:    retq
+; SSE2-LABEL: test_abs_le_v8i64_fold:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movdqu (%rdi), %xmm0
+; SSE2-NEXT:    movdqu 16(%rdi), %xmm1
+; SSE2-NEXT:    movdqu 32(%rdi), %xmm2
+; SSE2-NEXT:    movdqu 48(%rdi), %xmm3
+; SSE2-NEXT:    movdqa %xmm0, %xmm4
+; SSE2-NEXT:    psrad $31, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSE2-NEXT:    paddq %xmm4, %xmm0
+; SSE2-NEXT:    pxor %xmm4, %xmm0
+; SSE2-NEXT:    movdqa %xmm1, %xmm4
+; SSE2-NEXT:    psrad $31, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSE2-NEXT:    paddq %xmm4, %xmm1
+; SSE2-NEXT:    pxor %xmm4, %xmm1
+; SSE2-NEXT:    movdqa %xmm2, %xmm4
+; SSE2-NEXT:    psrad $31, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSE2-NEXT:    paddq %xmm4, %xmm2
+; SSE2-NEXT:    pxor %xmm4, %xmm2
+; SSE2-NEXT:    movdqa %xmm3, %xmm4
+; SSE2-NEXT:    psrad $31, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSE2-NEXT:    paddq %xmm4, %xmm3
+; SSE2-NEXT:    pxor %xmm4, %xmm3
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: test_abs_le_v8i64_fold:
+; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    movdqu (%rdi), %xmm0
+; SSSE3-NEXT:    movdqu 16(%rdi), %xmm1
+; SSSE3-NEXT:    movdqu 32(%rdi), %xmm2
+; SSSE3-NEXT:    movdqu 48(%rdi), %xmm3
+; SSSE3-NEXT:    movdqa %xmm0, %xmm4
+; SSSE3-NEXT:    psrad $31, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    paddq %xmm4, %xmm0
+; SSSE3-NEXT:    pxor %xmm4, %xmm0
+; SSSE3-NEXT:    movdqa %xmm1, %xmm4
+; SSSE3-NEXT:    psrad $31, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    paddq %xmm4, %xmm1
+; SSSE3-NEXT:    pxor %xmm4, %xmm1
+; SSSE3-NEXT:    movdqa %xmm2, %xmm4
+; SSSE3-NEXT:    psrad $31, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    paddq %xmm4, %xmm2
+; SSSE3-NEXT:    pxor %xmm4, %xmm2
+; SSSE3-NEXT:    movdqa %xmm3, %xmm4
+; SSSE3-NEXT:    psrad $31, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    paddq %xmm4, %xmm3
+; SSSE3-NEXT:    pxor %xmm4, %xmm3
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: test_abs_le_v8i64_fold:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movdqu (%rdi), %xmm1
+; SSE41-NEXT:    movdqu 16(%rdi), %xmm2
+; SSE41-NEXT:    movdqu 32(%rdi), %xmm3
+; SSE41-NEXT:    movdqu 48(%rdi), %xmm4
+; SSE41-NEXT:    pxor %xmm5, %xmm5
+; SSE41-NEXT:    pxor %xmm6, %xmm6
+; SSE41-NEXT:    psubq %xmm1, %xmm6
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm6, %xmm1
+; SSE41-NEXT:    pxor %xmm6, %xmm6
+; SSE41-NEXT:    psubq %xmm2, %xmm6
+; SSE41-NEXT:    movdqa %xmm2, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm6, %xmm2
+; SSE41-NEXT:    pxor %xmm6, %xmm6
+; SSE41-NEXT:    psubq %xmm3, %xmm6
+; SSE41-NEXT:    movdqa %xmm3, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm6, %xmm3
+; SSE41-NEXT:    psubq %xmm4, %xmm5
+; SSE41-NEXT:    movdqa %xmm4, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm5, %xmm4
+; SSE41-NEXT:    movapd %xmm1, %xmm0
+; SSE41-NEXT:    movapd %xmm2, %xmm1
+; SSE41-NEXT:    movapd %xmm3, %xmm2
+; SSE41-NEXT:    movapd %xmm4, %xmm3
+; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: test_abs_le_v8i64_fold:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vmovdqu (%rdi), %xmm0
-; AVX1-NEXT:    vmovdqu 16(%rdi), %xmm1
-; AVX1-NEXT:    vmovdqu 32(%rdi), %xmm2
-; AVX1-NEXT:    vmovdqu 48(%rdi), %xmm3
-; AVX1-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm4, %xmm5
-; AVX1-NEXT:    vpcmpgtq %xmm0, %xmm4, %xmm6
-; AVX1-NEXT:    vinsertf128 $1, %xmm5, %ymm6, %ymm7
-; AVX1-NEXT:    vpaddq %xmm5, %xmm1, %xmm1
-; AVX1-NEXT:    vpaddq %xmm6, %xmm0, %xmm0
-; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
-; AVX1-NEXT:    vxorps %ymm7, %ymm0, %ymm0
-; AVX1-NEXT:    vpcmpgtq %xmm3, %xmm4, %xmm1
-; AVX1-NEXT:    vpcmpgtq %xmm2, %xmm4, %xmm4
-; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm4, %ymm5
-; AVX1-NEXT:    vpaddq %xmm1, %xmm3, %xmm1
-; AVX1-NEXT:    vpaddq %xmm4, %xmm2, %xmm2
-; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm1
-; AVX1-NEXT:    vxorps %ymm5, %ymm1, %ymm1
+; AVX1-NEXT:    vmovupd (%rdi), %ymm0
+; AVX1-NEXT:    vmovupd 32(%rdi), %ymm1
+; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX1-NEXT:    vpsubq 16(%rdi), %xmm2, %xmm3
+; AVX1-NEXT:    vpsubq (%rdi), %xmm2, %xmm4
+; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm4, %ymm3
+; AVX1-NEXT:    vblendvpd %ymm0, %ymm3, %ymm0, %ymm0
+; AVX1-NEXT:    vpsubq 48(%rdi), %xmm2, %xmm3
+; AVX1-NEXT:    vpsubq 32(%rdi), %xmm2, %xmm2
+; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm2, %ymm2
+; AVX1-NEXT:    vblendvpd %ymm1, %ymm2, %ymm1, %ymm1
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: test_abs_le_v8i64_fold:
@@ -669,12 +855,10 @@ define <8 x i64> @test_abs_le_v8i64_fold(<8 x i64>* %a.ptr) nounwind {
 ; AVX2-NEXT:    vmovdqu (%rdi), %ymm0
 ; AVX2-NEXT:    vmovdqu 32(%rdi), %ymm1
 ; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpgtq %ymm0, %ymm2, %ymm3
-; AVX2-NEXT:    vpaddq %ymm3, %ymm0, %ymm0
-; AVX2-NEXT:    vpxor %ymm3, %ymm0, %ymm0
-; AVX2-NEXT:    vpcmpgtq %ymm1, %ymm2, %ymm2
-; AVX2-NEXT:    vpaddq %ymm2, %ymm1, %ymm1
-; AVX2-NEXT:    vpxor %ymm2, %ymm1, %ymm1
+; AVX2-NEXT:    vpsubq %ymm0, %ymm2, %ymm3
+; AVX2-NEXT:    vblendvpd %ymm0, %ymm3, %ymm0, %ymm0
+; AVX2-NEXT:    vpsubq %ymm1, %ymm2, %ymm2
+; AVX2-NEXT:    vblendvpd %ymm1, %ymm2, %ymm1, %ymm1
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: test_abs_le_v8i64_fold:
@@ -717,6 +901,14 @@ define <64 x i8> @test_abs_lt_v64i8(<64 x i8> %a) nounwind {
 ; SSSE3-NEXT:    pabsb %xmm3, %xmm3
 ; SSSE3-NEXT:    retq
 ;
+; SSE41-LABEL: test_abs_lt_v64i8:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    pabsb %xmm0, %xmm0
+; SSE41-NEXT:    pabsb %xmm1, %xmm1
+; SSE41-NEXT:    pabsb %xmm2, %xmm2
+; SSE41-NEXT:    pabsb %xmm3, %xmm3
+; SSE41-NEXT:    retq
+;
 ; AVX1-LABEL: test_abs_lt_v64i8:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vpabsb %xmm0, %xmm2
@@ -780,6 +972,14 @@ define <32 x i16> @test_abs_gt_v32i16(<32 x i16> %a) nounwind {
 ; SSSE3-NEXT:    pabsw %xmm3, %xmm3
 ; SSSE3-NEXT:    retq
 ;
+; SSE41-LABEL: test_abs_gt_v32i16:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    pabsw %xmm0, %xmm0
+; SSE41-NEXT:    pabsw %xmm1, %xmm1
+; SSE41-NEXT:    pabsw %xmm2, %xmm2
+; SSE41-NEXT:    pabsw %xmm3, %xmm3
+; SSE41-NEXT:    retq
+;
 ; AVX1-LABEL: test_abs_gt_v32i16:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vpabsw %xmm0, %xmm2
diff --git a/test/CodeGen/X86/vpshufbitqbm-intrinsics-upgrade.ll b/test/CodeGen/X86/vpshufbitqbm-intrinsics-upgrade.ll
new file mode 100644
index 0000000000000000000000000000000000000000..d06d8a95a31af30ad58404112c88c33f8eb22a17
--- /dev/null
+++ b/test/CodeGen/X86/vpshufbitqbm-intrinsics-upgrade.ll
@@ -0,0 +1,44 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-apple-darwin -mattr=+avx512bitalg,+avx512vl | FileCheck %s
+
+declare i16 @llvm.x86.avx512.mask.vpshufbitqmb.128(<16 x i8> %a, <16 x i8> %b, i16 %mask)
+define i16 @test_vpshufbitqmb_128(<16 x i8> %a, <16 x i8> %b, <16 x i8> %c, <16 x i8> %d) {
+; CHECK-LABEL: test_vpshufbitqmb_128:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    vpshufbitqmb %xmm3, %xmm2, %k1
+; CHECK-NEXT:    vpshufbitqmb %xmm1, %xmm0, %k0 {%k1}
+; CHECK-NEXT:    kmovd %k0, %eax
+; CHECK-NEXT:    ## kill: def $ax killed $ax killed $eax
+; CHECK-NEXT:    retq
+  %res = call i16 @llvm.x86.avx512.mask.vpshufbitqmb.128(<16 x i8> %a, <16 x i8> %b, i16 -1)
+  %res2 = call i16 @llvm.x86.avx512.mask.vpshufbitqmb.128(<16 x i8> %c, <16 x i8> %d, i16 %res)
+  ret i16 %res2
+}
+
+declare i32 @llvm.x86.avx512.mask.vpshufbitqmb.256(<32 x i8> %a, <32 x i8> %b, i32 %mask)
+define i32 @test_vpshufbitqmb_256(<32 x i8> %a, <32 x i8> %b, <32 x i8> %c, <32 x i8> %d) {
+; CHECK-LABEL: test_vpshufbitqmb_256:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    vpshufbitqmb %ymm3, %ymm2, %k1
+; CHECK-NEXT:    vpshufbitqmb %ymm1, %ymm0, %k0 {%k1}
+; CHECK-NEXT:    kmovd %k0, %eax
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+  %res = call i32 @llvm.x86.avx512.mask.vpshufbitqmb.256(<32 x i8> %a, <32 x i8> %b, i32 -1)
+  %res2 = call i32 @llvm.x86.avx512.mask.vpshufbitqmb.256(<32 x i8> %c, <32 x i8> %d, i32 %res)
+  ret i32 %res2
+}
+
+declare i64 @llvm.x86.avx512.mask.vpshufbitqmb.512(<64 x i8> %a, <64 x i8> %b, i64 %mask)
+define i64 @test_vpshufbitqmb_512(<64 x i8> %a, <64 x i8> %b, <64 x i8> %c, <64 x i8> %d) {
+; CHECK-LABEL: test_vpshufbitqmb_512:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    vpshufbitqmb %zmm3, %zmm2, %k1
+; CHECK-NEXT:    vpshufbitqmb %zmm1, %zmm0, %k0 {%k1}
+; CHECK-NEXT:    kmovq %k0, %rax
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+  %res = call i64 @llvm.x86.avx512.mask.vpshufbitqmb.512(<64 x i8> %a, <64 x i8> %b, i64 -1)
+  %res2 = call i64 @llvm.x86.avx512.mask.vpshufbitqmb.512(<64 x i8> %c, <64 x i8> %d, i64 %res)
+  ret i64 %res2
+}
diff --git a/test/CodeGen/X86/vpshufbitqbm-intrinsics.ll b/test/CodeGen/X86/vpshufbitqbm-intrinsics.ll
index a80ed2f174e2d31915eb0c111a9bbaa5941f4be7..f83f2df0c35a740cba374731deca7c0f8c85ad54 100644
--- a/test/CodeGen/X86/vpshufbitqbm-intrinsics.ll
+++ b/test/CodeGen/X86/vpshufbitqbm-intrinsics.ll
@@ -1,41 +1,51 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=x86_64-apple-darwin -mattr=+avx512bitalg,+avx512vl | FileCheck %s
 
-declare i16 @llvm.x86.avx512.mask.vpshufbitqmb.128(<16 x i8> %a, <16 x i8> %b, i16 %mask)
-define i16 @test_vpshufbitqmb_128(<16 x i8> %a, <16 x i8> %b, i16 %mask) {
+define i16 @test_vpshufbitqmb_128(<16 x i8> %a, <16 x i8> %b, <16 x i8> %c, <16 x i8> %d) {
 ; CHECK-LABEL: test_vpshufbitqmb_128:
 ; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovd %edi, %k1
-; CHECK-NEXT:    vpshufbitqmb %xmm1, %xmm0, %k0 {%k1}
+; CHECK-NEXT:    vpshufbitqmb %xmm1, %xmm0, %k1
+; CHECK-NEXT:    vpshufbitqmb %xmm3, %xmm2, %k0 {%k1}
 ; CHECK-NEXT:    kmovd %k0, %eax
 ; CHECK-NEXT:    ## kill: def $ax killed $ax killed $eax
 ; CHECK-NEXT:    retq
-  %res = call i16 @llvm.x86.avx512.mask.vpshufbitqmb.128(<16 x i8> %a, <16 x i8> %b, i16 %mask)
-  ret i16 %res
+  %tmp = call <16 x i1> @llvm.x86.avx512.vpshufbitqmb.128(<16 x i8> %a, <16 x i8> %b)
+  %tmp1 = call <16 x i1> @llvm.x86.avx512.vpshufbitqmb.128(<16 x i8> %c, <16 x i8> %d)
+  %tmp2 = and <16 x i1> %tmp, %tmp1
+  %tmp3 = bitcast <16 x i1> %tmp2 to i16
+  ret i16 %tmp3
 }
 
-declare i32 @llvm.x86.avx512.mask.vpshufbitqmb.256(<32 x i8> %a, <32 x i8> %b, i32 %mask)
-define i32 @test_vpshufbitqmb_256(<32 x i8> %a, <32 x i8> %b, i32 %mask) {
+define i32 @test_vpshufbitqmb_256(<32 x i8> %a, <32 x i8> %b, <32 x i8> %c, <32 x i8> %d) {
 ; CHECK-LABEL: test_vpshufbitqmb_256:
 ; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovd %edi, %k1
-; CHECK-NEXT:    vpshufbitqmb %ymm1, %ymm0, %k0 {%k1}
+; CHECK-NEXT:    vpshufbitqmb %ymm1, %ymm0, %k1
+; CHECK-NEXT:    vpshufbitqmb %ymm3, %ymm2, %k0 {%k1}
 ; CHECK-NEXT:    kmovd %k0, %eax
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retq
-  %res = call i32 @llvm.x86.avx512.mask.vpshufbitqmb.256(<32 x i8> %a, <32 x i8> %b, i32 %mask)
-  ret i32 %res
+  %tmp = call <32 x i1> @llvm.x86.avx512.vpshufbitqmb.256(<32 x i8> %a, <32 x i8> %b)
+  %tmp1 = call <32 x i1> @llvm.x86.avx512.vpshufbitqmb.256(<32 x i8> %c, <32 x i8> %d)
+  %tmp2 = and <32 x i1> %tmp, %tmp1
+  %tmp3 = bitcast <32 x i1> %tmp2 to i32
+  ret i32 %tmp3
 }
 
-declare i64 @llvm.x86.avx512.mask.vpshufbitqmb.512(<64 x i8> %a, <64 x i8> %b, i64 %mask)
-define i64 @test_vpshufbitqmb_512(<64 x i8> %a, <64 x i8> %b, i64 %mask) {
+define i64 @test_vpshufbitqmb_512(<64 x i8> %a, <64 x i8> %b, <64 x i8> %c, <64 x i8> %d) {
 ; CHECK-LABEL: test_vpshufbitqmb_512:
 ; CHECK:       ## %bb.0:
-; CHECK-NEXT:    kmovq %rdi, %k1
-; CHECK-NEXT:    vpshufbitqmb %zmm1, %zmm0, %k0 {%k1}
+; CHECK-NEXT:    vpshufbitqmb %zmm1, %zmm0, %k1
+; CHECK-NEXT:    vpshufbitqmb %zmm3, %zmm2, %k0 {%k1}
 ; CHECK-NEXT:    kmovq %k0, %rax
 ; CHECK-NEXT:    vzeroupper
 ; CHECK-NEXT:    retq
-  %res = call i64 @llvm.x86.avx512.mask.vpshufbitqmb.512(<64 x i8> %a, <64 x i8> %b, i64 %mask)
-  ret i64 %res
+  %tmp = call <64 x i1> @llvm.x86.avx512.vpshufbitqmb.512(<64 x i8> %a, <64 x i8> %b)
+  %tmp1 = call <64 x i1> @llvm.x86.avx512.vpshufbitqmb.512(<64 x i8> %c, <64 x i8> %d)
+  %tmp2 = and <64 x i1> %tmp, %tmp1
+  %tmp3 = bitcast <64 x i1> %tmp2 to i64
+  ret i64 %tmp3
 }
+
+declare <16 x i1> @llvm.x86.avx512.vpshufbitqmb.128(<16 x i8>, <16 x i8>)
+declare <32 x i1> @llvm.x86.avx512.vpshufbitqmb.256(<32 x i8>, <32 x i8>)
+declare <64 x i1> @llvm.x86.avx512.vpshufbitqmb.512(<64 x i8>, <64 x i8>)
diff --git a/test/CodeGen/X86/vsel-cmp-load.ll b/test/CodeGen/X86/vsel-cmp-load.ll
index d317377a93a372e08cd7420dc0b78f677c3180db..cbbbd6300fcc1d536f4309095b4bb4f9430fabc6 100644
--- a/test/CodeGen/X86/vsel-cmp-load.ll
+++ b/test/CodeGen/X86/vsel-cmp-load.ll
@@ -11,10 +11,9 @@ define <8 x i32> @eq_zero(<8 x i8>* %p, <8 x i32> %x, <8 x i32> %y) {
 ; AVX1-NEXT:    vpmovzxbw {{.*#+}} xmm2 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
 ; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
 ; AVX1-NEXT:    vpcmpeqw %xmm3, %xmm2, %xmm2
-; AVX1-NEXT:    vpmovzxwd {{.*#+}} xmm3 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero
-; AVX1-NEXT:    vpslld $24, %xmm3, %xmm3
-; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
-; AVX1-NEXT:    vpslld $24, %xmm2, %xmm2
+; AVX1-NEXT:    vpmovsxwd %xmm2, %xmm3
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,3,0,1]
+; AVX1-NEXT:    vpmovsxwd %xmm2, %xmm2
 ; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm3, %ymm2
 ; AVX1-NEXT:    vblendvps %ymm2, %ymm0, %ymm1, %ymm0
 ; AVX1-NEXT:    retq
@@ -181,10 +180,9 @@ define <8 x float> @ne_zero_fp_select(<8 x i8>* %p, <8 x float> %x, <8 x float>
 ; AVX1-NEXT:    vpcmpeqw %xmm3, %xmm2, %xmm2
 ; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
 ; AVX1-NEXT:    vpxor %xmm3, %xmm2, %xmm2
-; AVX1-NEXT:    vpmovzxwd {{.*#+}} xmm3 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero
-; AVX1-NEXT:    vpslld $24, %xmm3, %xmm3
-; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
-; AVX1-NEXT:    vpslld $24, %xmm2, %xmm2
+; AVX1-NEXT:    vpmovsxwd %xmm2, %xmm3
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,3,0,1]
+; AVX1-NEXT:    vpmovsxwd %xmm2, %xmm2
 ; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm3, %ymm2
 ; AVX1-NEXT:    vblendvps %ymm2, %ymm0, %ymm1, %ymm0
 ; AVX1-NEXT:    retq
@@ -215,12 +213,11 @@ define <4 x double> @sgt_zero_fp_select(<4 x i8>* %p, <4 x double> %x, <4 x doub
 ; AVX1-LABEL: sgt_zero_fp_select:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vpmovsxbd (%rdi), %xmm2
+; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
+; AVX1-NEXT:    vpcmpgtd %xmm3, %xmm2, %xmm2
 ; AVX1-NEXT:    vpmovsxdq %xmm2, %xmm3
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,3,0,1]
 ; AVX1-NEXT:    vpmovsxdq %xmm2, %xmm2
-; AVX1-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX1-NEXT:    vpcmpgtq %xmm4, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpgtq %xmm4, %xmm3, %xmm3
 ; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm3, %ymm2
 ; AVX1-NEXT:    vblendvpd %ymm2, %ymm0, %ymm1, %ymm0
 ; AVX1-NEXT:    retq
diff --git a/test/CodeGen/X86/vselect.ll b/test/CodeGen/X86/vselect.ll
index 5c19a01db81dd63b3560632eb4a04800e4a23cb6..f29e53d0143af2bfccb2cfb903c82e0b5a2559a6 100644
--- a/test/CodeGen/X86/vselect.ll
+++ b/test/CodeGen/X86/vselect.ll
@@ -610,3 +610,36 @@ define <2 x i32> @simplify_select(i32 %x, <2 x i1> %z) {
   ret <2 x i32> %r
 }
 
+; Test to make sure we don't try to insert a new setcc to swap the operands
+; of select with all zeros LHS if the setcc has additional users.
+define void @vselect_allzeros_LHS_multiple_use_setcc(<4 x i32> %x, <4 x i32> %y, <4 x i32> %z, <4 x i32>* %p1, <4 x i32>* %p2) {
+; SSE-LABEL: vselect_allzeros_LHS_multiple_use_setcc:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movdqa {{.*#+}} xmm3 = [1,2,4,8]
+; SSE-NEXT:    pand %xmm3, %xmm0
+; SSE-NEXT:    pcmpeqd %xmm3, %xmm0
+; SSE-NEXT:    movdqa %xmm0, %xmm3
+; SSE-NEXT:    pandn %xmm1, %xmm3
+; SSE-NEXT:    pand %xmm2, %xmm0
+; SSE-NEXT:    movdqa %xmm3, (%rdi)
+; SSE-NEXT:    movdqa %xmm0, (%rsi)
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: vselect_allzeros_LHS_multiple_use_setcc:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovdqa {{.*#+}} xmm3 = [1,2,4,8]
+; AVX-NEXT:    vpand %xmm3, %xmm0, %xmm0
+; AVX-NEXT:    vpcmpeqd %xmm3, %xmm0, %xmm0
+; AVX-NEXT:    vpandn %xmm1, %xmm0, %xmm1
+; AVX-NEXT:    vpand %xmm2, %xmm0, %xmm0
+; AVX-NEXT:    vmovdqa %xmm1, (%rdi)
+; AVX-NEXT:    vmovdqa %xmm0, (%rsi)
+; AVX-NEXT:    retq
+  %and = and <4 x i32> %x, <i32 1, i32 2, i32 4, i32 8>
+  %cond = icmp ne <4 x i32> %and, zeroinitializer
+  %sel1 = select <4 x i1> %cond, <4 x i32> zeroinitializer, <4 x i32> %y
+  %sel2 = select <4 x i1> %cond, <4 x i32> %z, <4 x i32> zeroinitializer
+  store <4 x i32> %sel1, <4 x i32>* %p1
+  store <4 x i32> %sel2, <4 x i32>* %p2
+  ret void
+}
diff --git a/test/CodeGen/X86/widened-broadcast.ll b/test/CodeGen/X86/widened-broadcast.ll
index 167128ae002b55825dca76c6de59f75c55e10fa0..ae4f901b1575b3d7ce6efb3a77c793d7494fa358 100644
--- a/test/CodeGen/X86/widened-broadcast.ll
+++ b/test/CodeGen/X86/widened-broadcast.ll
@@ -101,12 +101,12 @@ define <4 x i32> @load_splat_4i32_4i32_0101(<4 x i32>* %ptr) nounwind uwtable re
 ;
 ; AVX2-LABEL: load_splat_4i32_4i32_0101:
 ; AVX2:       # %bb.0: # %entry
-; AVX2-NEXT:    vpbroadcastq (%rdi), %xmm0
+; AVX2-NEXT:    vmovddup {{.*#+}} xmm0 = mem[0,0]
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: load_splat_4i32_4i32_0101:
 ; AVX512:       # %bb.0: # %entry
-; AVX512-NEXT:    vpbroadcastq (%rdi), %xmm0
+; AVX512-NEXT:    vmovddup {{.*#+}} xmm0 = mem[0,0]
 ; AVX512-NEXT:    retq
 entry:
   %ld = load <4 x i32>, <4 x i32>* %ptr
@@ -198,12 +198,12 @@ define <8 x i16> @load_splat_8i16_8i16_01230123(<8 x i16>* %ptr) nounwind uwtabl
 ;
 ; AVX2-LABEL: load_splat_8i16_8i16_01230123:
 ; AVX2:       # %bb.0: # %entry
-; AVX2-NEXT:    vpbroadcastq (%rdi), %xmm0
+; AVX2-NEXT:    vmovddup {{.*#+}} xmm0 = mem[0,0]
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: load_splat_8i16_8i16_01230123:
 ; AVX512:       # %bb.0: # %entry
-; AVX512-NEXT:    vpbroadcastq (%rdi), %xmm0
+; AVX512-NEXT:    vmovddup {{.*#+}} xmm0 = mem[0,0]
 ; AVX512-NEXT:    retq
 entry:
   %ld = load <8 x i16>, <8 x i16>* %ptr
@@ -368,12 +368,12 @@ define <16 x i8> @load_splat_16i8_16i8_0123456701234567(<16 x i8>* %ptr) nounwin
 ;
 ; AVX2-LABEL: load_splat_16i8_16i8_0123456701234567:
 ; AVX2:       # %bb.0: # %entry
-; AVX2-NEXT:    vpbroadcastq (%rdi), %xmm0
+; AVX2-NEXT:    vmovddup {{.*#+}} xmm0 = mem[0,0]
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: load_splat_16i8_16i8_0123456701234567:
 ; AVX512:       # %bb.0: # %entry
-; AVX512-NEXT:    vpbroadcastq (%rdi), %xmm0
+; AVX512-NEXT:    vmovddup {{.*#+}} xmm0 = mem[0,0]
 ; AVX512-NEXT:    retq
 entry:
   %ld = load <16 x i8>, <16 x i8>* %ptr
@@ -580,21 +580,10 @@ define <4 x i32> @load_splat_4i32_2i32_0101(<2 x i32>* %vp) {
 ; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,1,0,1]
 ; SSE-NEXT:    retq
 ;
-; AVX1-LABEL: load_splat_4i32_2i32_0101:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,1,0,1]
-; AVX1-NEXT:    retq
-;
-; AVX2-LABEL: load_splat_4i32_2i32_0101:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpbroadcastq (%rdi), %xmm0
-; AVX2-NEXT:    retq
-;
-; AVX512-LABEL: load_splat_4i32_2i32_0101:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpbroadcastq (%rdi), %xmm0
-; AVX512-NEXT:    retq
+; AVX-LABEL: load_splat_4i32_2i32_0101:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovddup {{.*#+}} xmm0 = mem[0,0]
+; AVX-NEXT:    retq
   %vec = load <2 x i32>, <2 x i32>* %vp
   %res = shufflevector <2 x i32> %vec, <2 x i32> undef, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
   ret <4 x i32> %res
@@ -608,22 +597,10 @@ define <8 x i32> @load_splat_8i32_2i32_0101(<2 x i32>* %vp) {
 ; SSE-NEXT:    movdqa %xmm0, %xmm1
 ; SSE-NEXT:    retq
 ;
-; AVX1-LABEL: load_splat_8i32_2i32_0101:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,1,0,1]
-; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
-; AVX1-NEXT:    retq
-;
-; AVX2-LABEL: load_splat_8i32_2i32_0101:
-; AVX2:       # %bb.0:
-; AVX2-NEXT:    vbroadcastsd (%rdi), %ymm0
-; AVX2-NEXT:    retq
-;
-; AVX512-LABEL: load_splat_8i32_2i32_0101:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vbroadcastsd (%rdi), %ymm0
-; AVX512-NEXT:    retq
+; AVX-LABEL: load_splat_8i32_2i32_0101:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vbroadcastsd (%rdi), %ymm0
+; AVX-NEXT:    retq
   %vec = load <2 x i32>, <2 x i32>* %vp
   %res = shufflevector <2 x i32> %vec, <2 x i32> undef, <8 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1>
   ret <8 x i32> %res
@@ -641,9 +618,7 @@ define <16 x i32> @load_splat_16i32_2i32_0101(<2 x i32>* %vp) {
 ;
 ; AVX1-LABEL: load_splat_16i32_2i32_0101:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
-; AVX1-NEXT:    vpermilps {{.*#+}} xmm0 = xmm0[0,1,0,1]
-; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
+; AVX1-NEXT:    vbroadcastsd (%rdi), %ymm0
 ; AVX1-NEXT:    vmovaps %ymm0, %ymm1
 ; AVX1-NEXT:    retq
 ;
@@ -655,9 +630,7 @@ define <16 x i32> @load_splat_16i32_2i32_0101(<2 x i32>* %vp) {
 ;
 ; AVX512-LABEL: load_splat_16i32_2i32_0101:
 ; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpmovzxdq {{.*#+}} xmm0 = mem[0],zero,mem[1],zero
-; AVX512-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [0,2,0,2,0,2,0,2,0,2,0,2,0,2,0,2]
-; AVX512-NEXT:    vpermd %zmm0, %zmm1, %zmm0
+; AVX512-NEXT:    vbroadcastsd (%rdi), %zmm0
 ; AVX512-NEXT:    retq
   %vec = load <2 x i32>, <2 x i32>* %vp
   %res = shufflevector <2 x i32> %vec, <2 x i32> undef, <16 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1>
diff --git a/test/CodeGen/X86/win32-seh-catchpad.ll b/test/CodeGen/X86/win32-seh-catchpad.ll
index 995a4b9eaf2e334327da0606e110e95f7bdd1645..f60195374937e2015d7d5ed20a60d255f3677e9c 100644
--- a/test/CodeGen/X86/win32-seh-catchpad.ll
+++ b/test/CodeGen/X86/win32-seh-catchpad.ll
@@ -53,7 +53,7 @@ invoke.cont:                                      ; preds = %entry
 define internal i32 @try_except_filter_catchall() #0 {
 entry:
   %0 = call i8* @llvm.frameaddress(i32 1)
-  %1 = call i8* @llvm.x86.seh.recoverfp(i8* bitcast (void ()* @try_except to i8*), i8* %0)
+  %1 = call i8* @llvm.eh.recoverfp(i8* bitcast (void ()* @try_except to i8*), i8* %0)
   %2 = call i8* @llvm.localrecover(i8* bitcast (void ()* @try_except to i8*), i8* %1, i32 0)
   %__exception_code = bitcast i8* %2 to i32*
   %3 = getelementptr inbounds i8, i8* %0, i32 -20
@@ -169,7 +169,7 @@ declare void @crash() #0
 define internal i32 @nested_exceptions_filter_catchall() #0 {
 entry:
   %0 = call i8* @llvm.frameaddress(i32 1)
-  %1 = call i8* @llvm.x86.seh.recoverfp(i8* bitcast (void ()* @nested_exceptions to i8*), i8* %0)
+  %1 = call i8* @llvm.eh.recoverfp(i8* bitcast (void ()* @nested_exceptions to i8*), i8* %0)
   %2 = call i8* @llvm.localrecover(i8* bitcast (void ()* @nested_exceptions to i8*), i8* %1, i32 0)
   %__exception_code3 = bitcast i8* %2 to i32*
   %3 = getelementptr inbounds i8, i8* %0, i32 -20
@@ -213,7 +213,7 @@ __except:
 declare i8* @llvm.frameaddress(i32) #1
 
 ; Function Attrs: nounwind readnone
-declare i8* @llvm.x86.seh.recoverfp(i8*, i8*) #1
+declare i8* @llvm.eh.recoverfp(i8*, i8*) #1
 
 ; Function Attrs: nounwind readnone
 declare i8* @llvm.localrecover(i8*, i8*, i32) #1
diff --git a/test/CodeGen/X86/x86-interleaved-access.ll b/test/CodeGen/X86/x86-interleaved-access.ll
index 261dc3c8379d8d6d6e5215a2bbf6818ef237769d..bff39467c1eb2ff134bf8c4d2cbaef235a0e1982 100644
--- a/test/CodeGen/X86/x86-interleaved-access.ll
+++ b/test/CodeGen/X86/x86-interleaved-access.ll
@@ -264,21 +264,21 @@ define void @interleaved_store_vf32_i8_stride4(<32 x i8> %x1, <32 x i8> %x2, <32
 ; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm6, %ymm3
 ; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm7[4],xmm1[4],xmm7[5],xmm1[5],xmm7[6],xmm1[6],xmm7[7],xmm1[7]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm1
-; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm4[4],xmm5[4],xmm4[5],xmm5[5],xmm4[6],xmm5[6],xmm4[7],xmm5[7]
-; AVX1-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm9[0],xmm0[0],xmm9[1],xmm0[1],xmm9[2],xmm0[2],xmm9[3],xmm0[3]
-; AVX1-NEXT:    vpunpcklwd {{.*#+}} xmm6 = xmm8[0],xmm2[0],xmm8[1],xmm2[1],xmm8[2],xmm2[2],xmm8[3],xmm2[3]
-; AVX1-NEXT:    vinsertf128 $1, %xmm5, %ymm6, %ymm5
+; AVX1-NEXT:    vperm2f128 {{.*#+}} ymm1 = ymm3[2,3],ymm1[2,3]
+; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm4[4],xmm5[4],xmm4[5],xmm5[5],xmm4[6],xmm5[6],xmm4[7],xmm5[7]
+; AVX1-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm9[0],xmm0[0],xmm9[1],xmm0[1],xmm9[2],xmm0[2],xmm9[3],xmm0[3]
+; AVX1-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm8[0],xmm2[0],xmm8[1],xmm2[1],xmm8[2],xmm2[2],xmm8[3],xmm2[3]
+; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm5, %ymm4
 ; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm9[4],xmm0[4],xmm9[5],xmm0[5],xmm9[6],xmm0[6],xmm9[7],xmm0[7]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
+; AVX1-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm4[2,3],ymm0[2,3]
 ; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm8[4],xmm2[4],xmm8[5],xmm2[5],xmm8[6],xmm2[6],xmm8[7],xmm2[7]
-; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm3, %ymm4
+; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm6, %ymm3
 ; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm5, %ymm2
-; AVX1-NEXT:    vperm2f128 {{.*#+}} ymm1 = ymm3[2,3],ymm1[2,3]
-; AVX1-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm5[2,3],ymm0[2,3]
+; AVX1-NEXT:    vmovaps %ymm2, 32(%rdi)
+; AVX1-NEXT:    vmovaps %ymm3, (%rdi)
 ; AVX1-NEXT:    vmovaps %ymm0, 96(%rdi)
 ; AVX1-NEXT:    vmovaps %ymm1, 64(%rdi)
-; AVX1-NEXT:    vmovaps %ymm2, 32(%rdi)
-; AVX1-NEXT:    vmovaps %ymm4, (%rdi)
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
 ;
@@ -1563,81 +1563,71 @@ define void @interleaved_store_vf64_i8_stride4(<64 x i8> %a, <64 x i8> %b, <64 x
 ; AVX1-LABEL: interleaved_store_vf64_i8_stride4:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm8 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
-; AVX1-NEXT:    vmovdqa %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm11
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm12
-; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm8 = xmm12[0],xmm11[0],xmm12[1],xmm11[1],xmm12[2],xmm11[2],xmm12[3],xmm11[3],xmm12[4],xmm11[4],xmm12[5],xmm11[5],xmm12[6],xmm11[6],xmm12[7],xmm11[7]
-; AVX1-NEXT:    vmovdqa %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm8 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3],xmm1[4],xmm3[4],xmm1[5],xmm3[5],xmm1[6],xmm3[6],xmm1[7],xmm3[7]
+; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm9 = xmm12[0],xmm11[0],xmm12[1],xmm11[1],xmm12[2],xmm11[2],xmm12[3],xmm11[3],xmm12[4],xmm11[4],xmm12[5],xmm11[5],xmm12[6],xmm11[6],xmm12[7],xmm11[7]
+; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm10 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3],xmm1[4],xmm3[4],xmm1[5],xmm3[5],xmm1[6],xmm3[6],xmm1[7],xmm3[7]
 ; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm13
 ; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm14
 ; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm15 = xmm14[0],xmm13[0],xmm14[1],xmm13[1],xmm14[2],xmm13[2],xmm14[3],xmm13[3],xmm14[4],xmm13[4],xmm14[5],xmm13[5],xmm14[6],xmm13[6],xmm14[7],xmm13[7]
 ; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm0[8],xmm2[8],xmm0[9],xmm2[9],xmm0[10],xmm2[10],xmm0[11],xmm2[11],xmm0[12],xmm2[12],xmm0[13],xmm2[13],xmm0[14],xmm2[14],xmm0[15],xmm2[15]
 ; AVX1-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm12[8],xmm11[8],xmm12[9],xmm11[9],xmm12[10],xmm11[10],xmm12[11],xmm11[11],xmm12[12],xmm11[12],xmm12[13],xmm11[13],xmm12[14],xmm11[14],xmm12[15],xmm11[15]
-; AVX1-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm1[8],xmm3[8],xmm1[9],xmm3[9],xmm1[10],xmm3[10],xmm1[11],xmm3[11],xmm1[12],xmm3[12],xmm1[13],xmm3[13],xmm1[14],xmm3[14],xmm1[15],xmm3[15]
-; AVX1-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm11 = xmm12[8],xmm11[8],xmm12[9],xmm11[9],xmm12[10],xmm11[10],xmm12[11],xmm11[11],xmm12[12],xmm11[12],xmm12[13],xmm11[13],xmm12[14],xmm11[14],xmm12[15],xmm11[15]
+; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm12 = xmm1[8],xmm3[8],xmm1[9],xmm3[9],xmm1[10],xmm3[10],xmm1[11],xmm3[11],xmm1[12],xmm3[12],xmm1[13],xmm3[13],xmm1[14],xmm3[14],xmm1[15],xmm3[15]
 ; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm13 = xmm14[8],xmm13[8],xmm14[9],xmm13[9],xmm14[10],xmm13[10],xmm14[11],xmm13[11],xmm14[12],xmm13[12],xmm14[13],xmm13[13],xmm14[14],xmm13[14],xmm14[15],xmm13[15]
 ; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm14 = xmm4[0],xmm6[0],xmm4[1],xmm6[1],xmm4[2],xmm6[2],xmm4[3],xmm6[3],xmm4[4],xmm6[4],xmm4[5],xmm6[5],xmm4[6],xmm6[6],xmm4[7],xmm6[7]
 ; AVX1-NEXT:    vextractf128 $1, %ymm6, %xmm1
 ; AVX1-NEXT:    vextractf128 $1, %ymm4, %xmm3
-; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm9 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3],xmm3[4],xmm1[4],xmm3[5],xmm1[5],xmm3[6],xmm1[6],xmm3[7],xmm1[7]
-; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm12 = xmm4[8],xmm6[8],xmm4[9],xmm6[9],xmm4[10],xmm6[10],xmm4[11],xmm6[11],xmm4[12],xmm6[12],xmm4[13],xmm6[13],xmm4[14],xmm6[14],xmm4[15],xmm6[15]
+; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm2 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3],xmm3[4],xmm1[4],xmm3[5],xmm1[5],xmm3[6],xmm1[6],xmm3[7],xmm1[7]
+; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm4 = xmm4[8],xmm6[8],xmm4[9],xmm6[9],xmm4[10],xmm6[10],xmm4[11],xmm6[11],xmm4[12],xmm6[12],xmm4[13],xmm6[13],xmm4[14],xmm6[14],xmm4[15],xmm6[15]
 ; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm6 = xmm5[0],xmm7[0],xmm5[1],xmm7[1],xmm5[2],xmm7[2],xmm5[3],xmm7[3],xmm5[4],xmm7[4],xmm5[5],xmm7[5],xmm5[6],xmm7[6],xmm5[7],xmm7[7]
-; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm10 = xmm3[8],xmm1[8],xmm3[9],xmm1[9],xmm3[10],xmm1[10],xmm3[11],xmm1[11],xmm3[12],xmm1[12],xmm3[13],xmm1[13],xmm3[14],xmm1[14],xmm3[15],xmm1[15]
+; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm3[8],xmm1[8],xmm3[9],xmm1[9],xmm3[10],xmm1[10],xmm3[11],xmm1[11],xmm3[12],xmm1[12],xmm3[13],xmm1[13],xmm3[14],xmm1[14],xmm3[15],xmm1[15]
 ; AVX1-NEXT:    vextractf128 $1, %ymm7, %xmm3
 ; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm7 = xmm5[8],xmm7[8],xmm5[9],xmm7[9],xmm5[10],xmm7[10],xmm5[11],xmm7[11],xmm5[12],xmm7[12],xmm5[13],xmm7[13],xmm5[14],xmm7[14],xmm5[15],xmm7[15]
 ; AVX1-NEXT:    vextractf128 $1, %ymm5, %xmm5
-; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm4 = xmm5[0],xmm3[0],xmm5[1],xmm3[1],xmm5[2],xmm3[2],xmm5[3],xmm3[3],xmm5[4],xmm3[4],xmm5[5],xmm3[5],xmm5[6],xmm3[6],xmm5[7],xmm3[7]
+; AVX1-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm5[0],xmm3[0],xmm5[1],xmm3[1],xmm5[2],xmm3[2],xmm5[3],xmm3[3],xmm5[4],xmm3[4],xmm5[5],xmm3[5],xmm5[6],xmm3[6],xmm5[7],xmm3[7]
 ; AVX1-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm5[8],xmm3[8],xmm5[9],xmm3[9],xmm5[10],xmm3[10],xmm5[11],xmm3[11],xmm5[12],xmm3[12],xmm5[13],xmm3[13],xmm5[14],xmm3[14],xmm5[15],xmm3[15]
-; AVX1-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm15[0],xmm4[0],xmm15[1],xmm4[1],xmm15[2],xmm4[2],xmm15[3],xmm4[3]
-; AVX1-NEXT:    vmovdqa %xmm8, %xmm2
-; AVX1-NEXT:    vpunpcklwd {{.*#+}} xmm11 = xmm8[0],xmm6[0],xmm8[1],xmm6[1],xmm8[2],xmm6[2],xmm8[3],xmm6[3]
-; AVX1-NEXT:    vinsertf128 $1, %xmm5, %ymm11, %ymm11
-; AVX1-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; AVX1-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm0[0],xmm9[0],xmm0[1],xmm9[1],xmm0[2],xmm9[2],xmm0[3],xmm9[3]
-; AVX1-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
-; AVX1-NEXT:    vpunpcklwd {{.*#+}} xmm8 = xmm1[0],xmm14[0],xmm1[1],xmm14[1],xmm1[2],xmm14[2],xmm1[3],xmm14[3]
-; AVX1-NEXT:    vinsertf128 $1, %xmm5, %ymm8, %ymm5
-; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm15[4],xmm4[4],xmm15[5],xmm4[5],xmm15[6],xmm4[6],xmm15[7],xmm4[7]
-; AVX1-NEXT:    vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm8 = xmm2[4],xmm6[4],xmm2[5],xmm6[5],xmm2[6],xmm6[6],xmm2[7],xmm6[7]
-; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm15 = xmm0[4],xmm9[4],xmm0[5],xmm9[5],xmm0[6],xmm9[6],xmm0[7],xmm9[7]
-; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm14 = xmm1[4],xmm14[4],xmm1[5],xmm14[5],xmm1[6],xmm14[6],xmm1[7],xmm14[7]
-; AVX1-NEXT:    vpunpcklwd {{.*#+}} xmm6 = xmm13[0],xmm3[0],xmm13[1],xmm3[1],xmm13[2],xmm3[2],xmm13[3],xmm3[3]
-; AVX1-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
-; AVX1-NEXT:    vpunpcklwd {{.*#+}} xmm9 = xmm4[0],xmm7[0],xmm4[1],xmm7[1],xmm4[2],xmm7[2],xmm4[3],xmm7[3]
+; AVX1-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm15[0],xmm0[0],xmm15[1],xmm0[1],xmm15[2],xmm0[2],xmm15[3],xmm0[3]
+; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm15[4],xmm0[4],xmm15[5],xmm0[5],xmm15[6],xmm0[6],xmm15[7],xmm0[7]
+; AVX1-NEXT:    vpunpcklwd {{.*#+}} xmm15 = xmm10[0],xmm6[0],xmm10[1],xmm6[1],xmm10[2],xmm6[2],xmm10[3],xmm6[3]
+; AVX1-NEXT:    vinsertf128 $1, %xmm5, %ymm15, %ymm5
+; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
+; AVX1-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm5[2,3],ymm0[2,3]
+; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm5 = xmm10[4],xmm6[4],xmm10[5],xmm6[5],xmm10[6],xmm6[6],xmm10[7],xmm6[7]
+; AVX1-NEXT:    vpunpcklwd {{.*#+}} xmm6 = xmm9[0],xmm2[0],xmm9[1],xmm2[1],xmm9[2],xmm2[2],xmm9[3],xmm2[3]
+; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm9[4],xmm2[4],xmm9[5],xmm2[5],xmm9[6],xmm2[6],xmm9[7],xmm2[7]
+; AVX1-NEXT:    vpunpcklwd {{.*#+}} xmm9 = xmm8[0],xmm14[0],xmm8[1],xmm14[1],xmm8[2],xmm14[2],xmm8[3],xmm14[3]
 ; AVX1-NEXT:    vinsertf128 $1, %xmm6, %ymm9, %ymm6
-; AVX1-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; AVX1-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm0[0],xmm10[0],xmm0[1],xmm10[1],xmm0[2],xmm10[2],xmm0[3],xmm10[3]
-; AVX1-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
-; AVX1-NEXT:    vpunpcklwd {{.*#+}} xmm9 = xmm1[0],xmm12[0],xmm1[1],xmm12[1],xmm1[2],xmm12[2],xmm1[3],xmm12[3]
-; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm9, %ymm2
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm2
+; AVX1-NEXT:    vperm2f128 {{.*#+}} ymm10 = ymm6[2,3],ymm2[2,3]
+; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm14 = xmm8[4],xmm14[4],xmm8[5],xmm14[5],xmm8[6],xmm14[6],xmm8[7],xmm14[7]
+; AVX1-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm13[0],xmm3[0],xmm13[1],xmm3[1],xmm13[2],xmm3[2],xmm13[3],xmm3[3]
 ; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm13[4],xmm3[4],xmm13[5],xmm3[5],xmm13[6],xmm3[6],xmm13[7],xmm3[7]
-; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm7 = xmm4[4],xmm7[4],xmm4[5],xmm7[5],xmm4[6],xmm7[6],xmm4[7],xmm7[7]
-; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm0[4],xmm10[4],xmm0[5],xmm10[5],xmm0[6],xmm10[6],xmm0[7],xmm10[7]
-; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm1[4],xmm12[4],xmm1[5],xmm12[5],xmm1[6],xmm12[6],xmm1[7],xmm12[7]
-; AVX1-NEXT:    vinsertf128 $1, %xmm15, %ymm0, %ymm9
-; AVX1-NEXT:    vinsertf128 $1, %xmm14, %ymm5, %ymm10
-; AVX1-NEXT:    vperm2f128 {{.*#+}} ymm5 = ymm5[2,3],ymm9[2,3]
-; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm1
-; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm2, %ymm4
-; AVX1-NEXT:    vperm2f128 {{.*#+}} ymm1 = ymm2[2,3],ymm1[2,3]
-; AVX1-NEXT:    vinsertf128 $1, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm2 # 16-byte Folded Reload
-; AVX1-NEXT:    vinsertf128 $1, %xmm8, %ymm11, %ymm8
-; AVX1-NEXT:    vperm2f128 {{.*#+}} ymm2 = ymm11[2,3],ymm2[2,3]
+; AVX1-NEXT:    vpunpcklwd {{.*#+}} xmm8 = xmm12[0],xmm7[0],xmm12[1],xmm7[1],xmm12[2],xmm7[2],xmm12[3],xmm7[3]
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm8, %ymm2
 ; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm3
-; AVX1-NEXT:    vinsertf128 $1, %xmm7, %ymm6, %ymm7
-; AVX1-NEXT:    vperm2f128 {{.*#+}} ymm3 = ymm6[2,3],ymm3[2,3]
-; AVX1-NEXT:    vmovaps %ymm3, 224(%rdi)
-; AVX1-NEXT:    vmovaps %ymm2, 192(%rdi)
-; AVX1-NEXT:    vmovaps %ymm7, 160(%rdi)
-; AVX1-NEXT:    vmovaps %ymm8, 128(%rdi)
-; AVX1-NEXT:    vmovaps %ymm1, 96(%rdi)
-; AVX1-NEXT:    vmovaps %ymm5, 64(%rdi)
+; AVX1-NEXT:    vperm2f128 {{.*#+}} ymm2 = ymm2[2,3],ymm3[2,3]
+; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm12[4],xmm7[4],xmm12[5],xmm7[5],xmm12[6],xmm7[6],xmm12[7],xmm7[7]
+; AVX1-NEXT:    vpunpcklwd {{.*#+}} xmm7 = xmm11[0],xmm1[0],xmm11[1],xmm1[1],xmm11[2],xmm1[2],xmm11[3],xmm1[3]
+; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm11[4],xmm1[4],xmm11[5],xmm1[5],xmm11[6],xmm1[6],xmm11[7],xmm1[7]
+; AVX1-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
+; AVX1-NEXT:    vpunpcklwd {{.*#+}} xmm11 = xmm6[0],xmm4[0],xmm6[1],xmm4[1],xmm6[2],xmm4[2],xmm6[3],xmm4[3]
+; AVX1-NEXT:    vinsertf128 $1, %xmm7, %ymm11, %ymm7
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm1
+; AVX1-NEXT:    vperm2f128 {{.*#+}} ymm1 = ymm7[2,3],ymm1[2,3]
+; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm6[4],xmm4[4],xmm6[5],xmm4[5],xmm6[6],xmm4[6],xmm6[7],xmm4[7]
+; AVX1-NEXT:    vinsertf128 $1, %xmm14, %ymm9, %ymm6
+; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm11, %ymm4
+; AVX1-NEXT:    vinsertf128 $1, %xmm5, %ymm15, %ymm5
+; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm8, %ymm3
+; AVX1-NEXT:    vmovaps %ymm3, 160(%rdi)
+; AVX1-NEXT:    vmovaps %ymm5, 128(%rdi)
 ; AVX1-NEXT:    vmovaps %ymm4, 32(%rdi)
-; AVX1-NEXT:    vmovaps %ymm10, (%rdi)
+; AVX1-NEXT:    vmovaps %ymm6, (%rdi)
+; AVX1-NEXT:    vmovaps %ymm2, 224(%rdi)
+; AVX1-NEXT:    vmovaps %ymm0, 192(%rdi)
+; AVX1-NEXT:    vmovaps %ymm1, 96(%rdi)
+; AVX1-NEXT:    vmovaps %ymm10, 64(%rdi)
 ; AVX1-NEXT:    vzeroupper
 ; AVX1-NEXT:    retq
 ;
diff --git a/test/CodeGen/X86/x87-schedule.ll b/test/CodeGen/X86/x87-schedule.ll
deleted file mode 100644
index 1921f8c75a3d10ce302c3470f5d19f082d70aed2..0000000000000000000000000000000000000000
--- a/test/CodeGen/X86/x87-schedule.ll
+++ /dev/null
@@ -1,6420 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=i686-unknown-unknown -print-schedule -mcpu=i686 | FileCheck %s --check-prefix=CHECK --check-prefix=GENERIC
-; RUN: llc < %s -mtriple=i686-unknown-unknown -print-schedule -mcpu=atom | FileCheck %s --check-prefix=CHECK --check-prefix=ATOM
-; RUN: llc < %s -mtriple=i686-unknown-unknown -print-schedule -mcpu=slm | FileCheck %s --check-prefix=CHECK --check-prefix=SLM
-; RUN: llc < %s -mtriple=i686-unknown-unknown -print-schedule -mcpu=sandybridge | FileCheck %s --check-prefix=CHECK --check-prefix=SANDY
-; RUN: llc < %s -mtriple=i686-unknown-unknown -print-schedule -mcpu=ivybridge | FileCheck %s --check-prefix=CHECK --check-prefix=SANDY
-; RUN: llc < %s -mtriple=i686-unknown-unknown -print-schedule -mcpu=haswell | FileCheck %s --check-prefix=CHECK --check-prefix=HASWELL
-; RUN: llc < %s -mtriple=i686-unknown-unknown -print-schedule -mcpu=broadwell | FileCheck %s --check-prefix=CHECK --check-prefix=BROADWELL
-; RUN: llc < %s -mtriple=i686-unknown-unknown -print-schedule -mcpu=skylake | FileCheck %s --check-prefix=CHECK --check-prefix=SKYLAKE
-; RUN: llc < %s -mtriple=i686-unknown-unknown -print-schedule -mcpu=skx | FileCheck %s --check-prefix=CHECK --check-prefix=SKX
-; RUN: llc < %s -mtriple=i686-unknown-unknown -print-schedule -mcpu=bdver2 | FileCheck %s --check-prefix=CHECK --check-prefix=BDVER2
-; RUN: llc < %s -mtriple=i686-unknown-unknown -print-schedule -mcpu=btver2 | FileCheck %s --check-prefix=CHECK --check-prefix=BTVER2
-; RUN: llc < %s -mtriple=i686-unknown-unknown -print-schedule -mcpu=znver1 | FileCheck %s --check-prefix=CHECK --check-prefix=ZNVER1
-
-define void @test_f2xm1() optsize {
-; GENERIC-LABEL: test_f2xm1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    f2xm1
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retl
-;
-; ATOM-LABEL: test_f2xm1:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    f2xm1 # sched: [99:49.50]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retl # sched: [79:39.50]
-;
-; SLM-LABEL: test_f2xm1:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    f2xm1 # sched: [100:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retl # sched: [4:1.00]
-;
-; SANDY-LABEL: test_f2xm1:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    f2xm1 # sched: [100:0.33]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retl # sched: [6:1.00]
-;
-; HASWELL-LABEL: test_f2xm1:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    f2xm1 # sched: [100:0.25]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retl # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_f2xm1:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    f2xm1 # sched: [100:0.25]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retl # sched: [6:0.50]
-;
-; SKYLAKE-LABEL: test_f2xm1:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    f2xm1 # sched: [100:0.25]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retl # sched: [6:0.50]
-;
-; SKX-LABEL: test_f2xm1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    f2xm1 # sched: [100:0.25]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retl # sched: [6:0.50]
-;
-; BDVER2-LABEL: test_f2xm1:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    f2xm1 # sched: [100:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retl # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_f2xm1:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    f2xm1 # sched: [100:0.50]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retl # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_f2xm1:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    f2xm1 # sched: [100:0.25]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retl # sched: [1:0.50]
-  tail call void asm sideeffect "f2xm1", ""() nounwind
-  ret void
-}
-
-define void @test_fabs() optsize {
-; GENERIC-LABEL: test_fabs:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    fabs
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retl
-;
-; ATOM-LABEL: test_fabs:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    fabs # sched: [1:1.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retl # sched: [79:39.50]
-;
-; SLM-LABEL: test_fabs:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    fabs # sched: [1:0.50]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retl # sched: [4:1.00]
-;
-; SANDY-LABEL: test_fabs:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    fabs # sched: [1:1.00]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retl # sched: [6:1.00]
-;
-; HASWELL-LABEL: test_fabs:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    fabs # sched: [1:1.00]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retl # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_fabs:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    fabs # sched: [1:1.00]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retl # sched: [6:0.50]
-;
-; SKYLAKE-LABEL: test_fabs:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    fabs # sched: [1:1.00]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retl # sched: [6:0.50]
-;
-; SKX-LABEL: test_fabs:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    fabs # sched: [1:1.00]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retl # sched: [6:0.50]
-;
-; BDVER2-LABEL: test_fabs:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    fabs # sched: [1:1.00]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retl # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_fabs:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    fabs # sched: [2:1.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retl # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_fabs:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    fabs # sched: [2:1.00]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retl # sched: [1:0.50]
-  tail call void asm sideeffect "fabs", ""() nounwind
-  ret void
-}
-
-define void @test_fadd(float *%a0, double *%a1) optsize {
-; GENERIC-LABEL: test_fadd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; GENERIC-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    fadd %st(0), %st(1)
-; GENERIC-NEXT:    fadd %st(2)
-; GENERIC-NEXT:    fadds (%ecx)
-; GENERIC-NEXT:    faddl (%eax)
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retl
-;
-; ATOM-LABEL: test_fadd:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [1:1.00]
-; ATOM-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [1:1.00]
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    fadd %st(0), %st(1) # sched: [5:5.00]
-; ATOM-NEXT:    fadd %st(2) # sched: [5:5.00]
-; ATOM-NEXT:    fadds (%ecx) # sched: [5:5.00]
-; ATOM-NEXT:    faddl (%eax) # sched: [5:5.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retl # sched: [79:39.50]
-;
-; SLM-LABEL: test_fadd:
-; SLM:       # %bb.0:
-; SLM-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [3:1.00]
-; SLM-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [3:1.00]
-; SLM-NEXT:    #APP
-; SLM-NEXT:    fadd %st(0), %st(1) # sched: [3:1.00]
-; SLM-NEXT:    fadd %st(2) # sched: [3:1.00]
-; SLM-NEXT:    fadds (%ecx) # sched: [6:1.00]
-; SLM-NEXT:    faddl (%eax) # sched: [6:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retl # sched: [4:1.00]
-;
-; SANDY-LABEL: test_fadd:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; SANDY-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    fadd %st(0), %st(1) # sched: [3:1.00]
-; SANDY-NEXT:    fadd %st(2) # sched: [3:1.00]
-; SANDY-NEXT:    fadds (%ecx) # sched: [10:1.00]
-; SANDY-NEXT:    faddl (%eax) # sched: [10:1.00]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retl # sched: [6:1.00]
-;
-; HASWELL-LABEL: test_fadd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; HASWELL-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    fadd %st(0), %st(1) # sched: [3:1.00]
-; HASWELL-NEXT:    fadd %st(2) # sched: [3:1.00]
-; HASWELL-NEXT:    fadds (%ecx) # sched: [10:1.00]
-; HASWELL-NEXT:    faddl (%eax) # sched: [10:1.00]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retl # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_fadd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; BROADWELL-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    fadd %st(0), %st(1) # sched: [3:1.00]
-; BROADWELL-NEXT:    fadd %st(2) # sched: [3:1.00]
-; BROADWELL-NEXT:    fadds (%ecx) # sched: [9:1.00]
-; BROADWELL-NEXT:    faddl (%eax) # sched: [9:1.00]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retl # sched: [6:0.50]
-;
-; SKYLAKE-LABEL: test_fadd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; SKYLAKE-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    fadd %st(0), %st(1) # sched: [3:1.00]
-; SKYLAKE-NEXT:    fadd %st(2) # sched: [3:1.00]
-; SKYLAKE-NEXT:    fadds (%ecx) # sched: [10:1.00]
-; SKYLAKE-NEXT:    faddl (%eax) # sched: [10:1.00]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retl # sched: [6:0.50]
-;
-; SKX-LABEL: test_fadd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; SKX-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; SKX-NEXT:    #APP
-; SKX-NEXT:    fadd %st(0), %st(1) # sched: [3:1.00]
-; SKX-NEXT:    fadd %st(2) # sched: [3:1.00]
-; SKX-NEXT:    fadds (%ecx) # sched: [10:1.00]
-; SKX-NEXT:    faddl (%eax) # sched: [10:1.00]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retl # sched: [6:0.50]
-;
-; BDVER2-LABEL: test_fadd:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; BDVER2-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    fadd %st(0), %st(1) # sched: [5:1.00]
-; BDVER2-NEXT:    fadd %st(2) # sched: [5:1.00]
-; BDVER2-NEXT:    fadds (%ecx) # sched: [10:1.00]
-; BDVER2-NEXT:    faddl (%eax) # sched: [10:1.00]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retl # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_fadd:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:1.00]
-; BTVER2-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:1.00]
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    fadd %st(0), %st(1) # sched: [3:1.00]
-; BTVER2-NEXT:    fadd %st(2) # sched: [3:1.00]
-; BTVER2-NEXT:    fadds (%ecx) # sched: [8:1.00]
-; BTVER2-NEXT:    faddl (%eax) # sched: [8:1.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retl # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_fadd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [8:0.50]
-; ZNVER1-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [8:0.50]
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    fadd %st(0), %st(1) # sched: [3:1.00]
-; ZNVER1-NEXT:    fadd %st(2) # sched: [3:1.00]
-; ZNVER1-NEXT:    fadds (%ecx) # sched: [10:1.00]
-; ZNVER1-NEXT:    faddl (%eax) # sched: [10:1.00]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retl # sched: [1:0.50]
-  tail call void asm sideeffect "fadd %st(0), %st(1) \0A\09 fadd %st(2), %st(0) \0A\09 fadds $0 \0A\09 faddl $1", "*m,*m"(float *%a0, double *%a1) nounwind
-  ret void
-}
-
-define void @test_faddp_fiadd(i16 *%a0, i32 *%a1) optsize {
-; GENERIC-LABEL: test_faddp_fiadd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; GENERIC-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    faddp %st(1)
-; GENERIC-NEXT:    faddp %st(2)
-; GENERIC-NEXT:    fiadds (%ecx)
-; GENERIC-NEXT:    fiaddl (%eax)
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retl
-;
-; ATOM-LABEL: test_faddp_fiadd:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [1:1.00]
-; ATOM-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [1:1.00]
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    faddp %st(1) # sched: [5:5.00]
-; ATOM-NEXT:    faddp %st(2) # sched: [5:5.00]
-; ATOM-NEXT:    fiadds (%ecx) # sched: [5:5.00]
-; ATOM-NEXT:    fiaddl (%eax) # sched: [5:5.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retl # sched: [79:39.50]
-;
-; SLM-LABEL: test_faddp_fiadd:
-; SLM:       # %bb.0:
-; SLM-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [3:1.00]
-; SLM-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [3:1.00]
-; SLM-NEXT:    #APP
-; SLM-NEXT:    faddp %st(1) # sched: [3:1.00]
-; SLM-NEXT:    faddp %st(2) # sched: [3:1.00]
-; SLM-NEXT:    fiadds (%ecx) # sched: [6:1.00]
-; SLM-NEXT:    fiaddl (%eax) # sched: [6:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retl # sched: [4:1.00]
-;
-; SANDY-LABEL: test_faddp_fiadd:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; SANDY-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    faddp %st(1) # sched: [3:1.00]
-; SANDY-NEXT:    faddp %st(2) # sched: [3:1.00]
-; SANDY-NEXT:    fiadds (%ecx) # sched: [13:2.00]
-; SANDY-NEXT:    fiaddl (%eax) # sched: [13:2.00]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retl # sched: [6:1.00]
-;
-; HASWELL-LABEL: test_faddp_fiadd:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; HASWELL-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    faddp %st(1) # sched: [3:1.00]
-; HASWELL-NEXT:    faddp %st(2) # sched: [3:1.00]
-; HASWELL-NEXT:    fiadds (%ecx) # sched: [13:2.00]
-; HASWELL-NEXT:    fiaddl (%eax) # sched: [13:2.00]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retl # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_faddp_fiadd:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; BROADWELL-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    faddp %st(1) # sched: [3:1.00]
-; BROADWELL-NEXT:    faddp %st(2) # sched: [3:1.00]
-; BROADWELL-NEXT:    fiadds (%ecx) # sched: [12:2.00]
-; BROADWELL-NEXT:    fiaddl (%eax) # sched: [12:2.00]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retl # sched: [6:0.50]
-;
-; SKYLAKE-LABEL: test_faddp_fiadd:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; SKYLAKE-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    faddp %st(1) # sched: [3:1.00]
-; SKYLAKE-NEXT:    faddp %st(2) # sched: [3:1.00]
-; SKYLAKE-NEXT:    fiadds (%ecx) # sched: [13:2.00]
-; SKYLAKE-NEXT:    fiaddl (%eax) # sched: [13:2.00]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retl # sched: [6:0.50]
-;
-; SKX-LABEL: test_faddp_fiadd:
-; SKX:       # %bb.0:
-; SKX-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; SKX-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; SKX-NEXT:    #APP
-; SKX-NEXT:    faddp %st(1) # sched: [3:1.00]
-; SKX-NEXT:    faddp %st(2) # sched: [3:1.00]
-; SKX-NEXT:    fiadds (%ecx) # sched: [13:2.00]
-; SKX-NEXT:    fiaddl (%eax) # sched: [13:2.00]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retl # sched: [6:0.50]
-;
-; BDVER2-LABEL: test_faddp_fiadd:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; BDVER2-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    faddp %st(1) # sched: [5:1.00]
-; BDVER2-NEXT:    faddp %st(2) # sched: [5:1.00]
-; BDVER2-NEXT:    fiadds (%ecx) # sched: [10:1.00]
-; BDVER2-NEXT:    fiaddl (%eax) # sched: [10:1.00]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retl # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_faddp_fiadd:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:1.00]
-; BTVER2-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:1.00]
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    faddp %st(1) # sched: [3:1.00]
-; BTVER2-NEXT:    faddp %st(2) # sched: [3:1.00]
-; BTVER2-NEXT:    fiadds (%ecx) # sched: [8:1.00]
-; BTVER2-NEXT:    fiaddl (%eax) # sched: [8:1.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retl # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_faddp_fiadd:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [8:0.50]
-; ZNVER1-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [8:0.50]
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    faddp %st(1) # sched: [3:1.00]
-; ZNVER1-NEXT:    faddp %st(2) # sched: [3:1.00]
-; ZNVER1-NEXT:    fiadds (%ecx) # sched: [10:1.00]
-; ZNVER1-NEXT:    fiaddl (%eax) # sched: [10:1.00]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retl # sched: [1:0.50]
-  tail call void asm sideeffect "faddp \0A\09 faddp %st(2), %st(0) \0A\09 fiadds $0 \0A\09 fiaddl $1", "*m,*m"(i16 *%a0, i32 *%a1) nounwind
-  ret void
-}
-
-define void @test_fbld_fbstp(i8* %a0) optsize {
-; GENERIC-LABEL: test_fbld_fbstp:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    fbld (%eax)
-; GENERIC-NEXT:    fbstp (%eax)
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retl
-;
-; ATOM-LABEL: test_fbld_fbstp:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [1:1.00]
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    fbld (%eax) # sched: [100:0.50]
-; ATOM-NEXT:    fbstp (%eax) # sched: [100:0.50]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retl # sched: [79:39.50]
-;
-; SLM-LABEL: test_fbld_fbstp:
-; SLM:       # %bb.0:
-; SLM-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [3:1.00]
-; SLM-NEXT:    #APP
-; SLM-NEXT:    fbld (%eax) # sched: [100:1.00]
-; SLM-NEXT:    fbstp (%eax) # sched: [100:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retl # sched: [4:1.00]
-;
-; SANDY-LABEL: test_fbld_fbstp:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    fbld (%eax) # sched: [100:0.33]
-; SANDY-NEXT:    fbstp (%eax) # sched: [100:0.33]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retl # sched: [6:1.00]
-;
-; HASWELL-LABEL: test_fbld_fbstp:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    fbld (%eax) # sched: [47:10.75]
-; HASWELL-NEXT:    fbstp (%eax) # sched: [1:1.00]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retl # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_fbld_fbstp:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    fbld (%eax) # sched: [100:0.25]
-; BROADWELL-NEXT:    fbstp (%eax) # sched: [1:1.00]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retl # sched: [6:0.50]
-;
-; SKYLAKE-LABEL: test_fbld_fbstp:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    fbld (%eax) # sched: [100:0.25]
-; SKYLAKE-NEXT:    fbstp (%eax) # sched: [1:1.00]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retl # sched: [6:0.50]
-;
-; SKX-LABEL: test_fbld_fbstp:
-; SKX:       # %bb.0:
-; SKX-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; SKX-NEXT:    #APP
-; SKX-NEXT:    fbld (%eax) # sched: [100:0.25]
-; SKX-NEXT:    fbstp (%eax) # sched: [1:1.00]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retl # sched: [6:0.50]
-;
-; BDVER2-LABEL: test_fbld_fbstp:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    fbld (%eax) # sched: [100:0.50]
-; BDVER2-NEXT:    fbstp (%eax) # sched: [100:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retl # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_fbld_fbstp:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:1.00]
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    fbld (%eax) # sched: [100:0.50]
-; BTVER2-NEXT:    fbstp (%eax) # sched: [100:0.50]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retl # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_fbld_fbstp:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [8:0.50]
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    fbld (%eax) # sched: [100:0.25]
-; ZNVER1-NEXT:    fbstp (%eax) # sched: [100:0.25]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retl # sched: [1:0.50]
-  tail call void asm sideeffect "fbld $0 \0A\09 fbstp $0", "*m"(i8 *%a0) nounwind
-  ret void
-}
-
-define void @test_fchs() optsize {
-; GENERIC-LABEL: test_fchs:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    fchs
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retl
-;
-; ATOM-LABEL: test_fchs:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    fchs # sched: [1:1.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retl # sched: [79:39.50]
-;
-; SLM-LABEL: test_fchs:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    fchs # sched: [1:0.50]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retl # sched: [4:1.00]
-;
-; SANDY-LABEL: test_fchs:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    fchs # sched: [1:1.00]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retl # sched: [6:1.00]
-;
-; HASWELL-LABEL: test_fchs:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    fchs # sched: [1:1.00]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retl # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_fchs:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    fchs # sched: [1:1.00]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retl # sched: [6:0.50]
-;
-; SKYLAKE-LABEL: test_fchs:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    fchs # sched: [1:1.00]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retl # sched: [6:0.50]
-;
-; SKX-LABEL: test_fchs:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    fchs # sched: [1:1.00]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retl # sched: [6:0.50]
-;
-; BDVER2-LABEL: test_fchs:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    fchs # sched: [1:1.00]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retl # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_fchs:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    fchs # sched: [2:1.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retl # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_fchs:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    fchs # sched: [1:1.00]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retl # sched: [1:0.50]
-  tail call void asm sideeffect "fchs", ""() nounwind
-  ret void
-}
-
-define void @test_fclex() optsize {
-; GENERIC-LABEL: test_fclex:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    wait
-; GENERIC-NEXT:    fnclex
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retl
-;
-; ATOM-LABEL: test_fclex:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    wait # sched: [1:0.50]
-; ATOM-NEXT:    fnclex # sched: [25:12.50]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retl # sched: [79:39.50]
-;
-; SLM-LABEL: test_fclex:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    wait # sched: [100:1.00]
-; SLM-NEXT:    fnclex # sched: [100:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retl # sched: [4:1.00]
-;
-; SANDY-LABEL: test_fclex:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    wait # sched: [100:0.33]
-; SANDY-NEXT:    fnclex # sched: [100:0.33]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retl # sched: [6:1.00]
-;
-; HASWELL-LABEL: test_fclex:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    wait # sched: [2:0.50]
-; HASWELL-NEXT:    fnclex # sched: [4:1.00]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retl # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_fclex:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    wait # sched: [2:0.50]
-; BROADWELL-NEXT:    fnclex # sched: [4:1.00]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retl # sched: [6:0.50]
-;
-; SKYLAKE-LABEL: test_fclex:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    wait # sched: [2:0.50]
-; SKYLAKE-NEXT:    fnclex # sched: [4:1.00]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retl # sched: [6:0.50]
-;
-; SKX-LABEL: test_fclex:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    wait # sched: [2:0.50]
-; SKX-NEXT:    fnclex # sched: [4:1.00]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retl # sched: [6:0.50]
-;
-; BDVER2-LABEL: test_fclex:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    wait # sched: [100:0.50]
-; BDVER2-NEXT:    fnclex # sched: [100:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retl # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_fclex:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    wait # sched: [100:0.50]
-; BTVER2-NEXT:    fnclex # sched: [100:0.50]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retl # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_fclex:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    wait # sched: [1:1.00]
-; ZNVER1-NEXT:    fnclex # sched: [100:0.25]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retl # sched: [1:0.50]
-  tail call void asm sideeffect "fclex", ""() nounwind
-  ret void
-}
-
-define void @test_fnclex() optsize {
-; GENERIC-LABEL: test_fnclex:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    fnclex
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retl
-;
-; ATOM-LABEL: test_fnclex:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    fnclex # sched: [25:12.50]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retl # sched: [79:39.50]
-;
-; SLM-LABEL: test_fnclex:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    fnclex # sched: [100:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retl # sched: [4:1.00]
-;
-; SANDY-LABEL: test_fnclex:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    fnclex # sched: [100:0.33]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retl # sched: [6:1.00]
-;
-; HASWELL-LABEL: test_fnclex:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    fnclex # sched: [4:1.00]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retl # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_fnclex:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    fnclex # sched: [4:1.00]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retl # sched: [6:0.50]
-;
-; SKYLAKE-LABEL: test_fnclex:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    fnclex # sched: [4:1.00]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retl # sched: [6:0.50]
-;
-; SKX-LABEL: test_fnclex:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    fnclex # sched: [4:1.00]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retl # sched: [6:0.50]
-;
-; BDVER2-LABEL: test_fnclex:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    fnclex # sched: [100:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retl # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_fnclex:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    fnclex # sched: [100:0.50]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retl # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_fnclex:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    fnclex # sched: [100:0.25]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retl # sched: [1:0.50]
-  tail call void asm sideeffect "fnclex", ""() nounwind
-  ret void
-}
-
-define void @test_fcmov() optsize {
-; GENERIC-LABEL: test_fcmov:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    fcmovb %st(1), %st(0)
-; GENERIC-NEXT:    fcmovbe %st(1), %st(0)
-; GENERIC-NEXT:    fcmove %st(1), %st(0)
-; GENERIC-NEXT:    fcmovnb %st(1), %st(0)
-; GENERIC-NEXT:    fcmovnbe %st(1), %st(0)
-; GENERIC-NEXT:    fcmovne %st(1), %st(0)
-; GENERIC-NEXT:    fcmovnu %st(1), %st(0)
-; GENERIC-NEXT:    fcmovu %st(1), %st(0)
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retl
-;
-; ATOM-LABEL: test_fcmov:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    fcmovb %st(1), %st(0) # sched: [9:4.50]
-; ATOM-NEXT:    fcmovbe %st(1), %st(0) # sched: [9:4.50]
-; ATOM-NEXT:    fcmove %st(1), %st(0) # sched: [9:4.50]
-; ATOM-NEXT:    fcmovnb %st(1), %st(0) # sched: [9:4.50]
-; ATOM-NEXT:    fcmovnbe %st(1), %st(0) # sched: [9:4.50]
-; ATOM-NEXT:    fcmovne %st(1), %st(0) # sched: [9:4.50]
-; ATOM-NEXT:    fcmovnu %st(1), %st(0) # sched: [9:4.50]
-; ATOM-NEXT:    fcmovu %st(1), %st(0) # sched: [9:4.50]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retl # sched: [79:39.50]
-;
-; SLM-LABEL: test_fcmov:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    fcmovb %st(1), %st(0) # sched: [3:1.00]
-; SLM-NEXT:    fcmovbe %st(1), %st(0) # sched: [3:1.00]
-; SLM-NEXT:    fcmove %st(1), %st(0) # sched: [3:1.00]
-; SLM-NEXT:    fcmovnb %st(1), %st(0) # sched: [3:1.00]
-; SLM-NEXT:    fcmovnbe %st(1), %st(0) # sched: [3:1.00]
-; SLM-NEXT:    fcmovne %st(1), %st(0) # sched: [3:1.00]
-; SLM-NEXT:    fcmovnu %st(1), %st(0) # sched: [3:1.00]
-; SLM-NEXT:    fcmovu %st(1), %st(0) # sched: [3:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retl # sched: [4:1.00]
-;
-; SANDY-LABEL: test_fcmov:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    fcmovb %st(1), %st(0) # sched: [3:2.00]
-; SANDY-NEXT:    fcmovbe %st(1), %st(0) # sched: [3:2.00]
-; SANDY-NEXT:    fcmove %st(1), %st(0) # sched: [3:2.00]
-; SANDY-NEXT:    fcmovnb %st(1), %st(0) # sched: [3:2.00]
-; SANDY-NEXT:    fcmovnbe %st(1), %st(0) # sched: [3:2.00]
-; SANDY-NEXT:    fcmovne %st(1), %st(0) # sched: [3:2.00]
-; SANDY-NEXT:    fcmovnu %st(1), %st(0) # sched: [3:2.00]
-; SANDY-NEXT:    fcmovu %st(1), %st(0) # sched: [3:2.00]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retl # sched: [6:1.00]
-;
-; HASWELL-LABEL: test_fcmov:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    fcmovb %st(1), %st(0) # sched: [3:1.00]
-; HASWELL-NEXT:    fcmovbe %st(1), %st(0) # sched: [3:1.00]
-; HASWELL-NEXT:    fcmove %st(1), %st(0) # sched: [3:1.00]
-; HASWELL-NEXT:    fcmovnb %st(1), %st(0) # sched: [3:1.00]
-; HASWELL-NEXT:    fcmovnbe %st(1), %st(0) # sched: [3:1.00]
-; HASWELL-NEXT:    fcmovne %st(1), %st(0) # sched: [3:1.00]
-; HASWELL-NEXT:    fcmovnu %st(1), %st(0) # sched: [3:1.00]
-; HASWELL-NEXT:    fcmovu %st(1), %st(0) # sched: [3:1.00]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retl # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_fcmov:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    fcmovb %st(1), %st(0) # sched: [3:1.00]
-; BROADWELL-NEXT:    fcmovbe %st(1), %st(0) # sched: [3:1.00]
-; BROADWELL-NEXT:    fcmove %st(1), %st(0) # sched: [3:1.00]
-; BROADWELL-NEXT:    fcmovnb %st(1), %st(0) # sched: [3:1.00]
-; BROADWELL-NEXT:    fcmovnbe %st(1), %st(0) # sched: [3:1.00]
-; BROADWELL-NEXT:    fcmovne %st(1), %st(0) # sched: [3:1.00]
-; BROADWELL-NEXT:    fcmovnu %st(1), %st(0) # sched: [3:1.00]
-; BROADWELL-NEXT:    fcmovu %st(1), %st(0) # sched: [3:1.00]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retl # sched: [6:0.50]
-;
-; SKYLAKE-LABEL: test_fcmov:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    fcmovb %st(1), %st(0) # sched: [3:1.00]
-; SKYLAKE-NEXT:    fcmovbe %st(1), %st(0) # sched: [3:1.00]
-; SKYLAKE-NEXT:    fcmove %st(1), %st(0) # sched: [3:1.00]
-; SKYLAKE-NEXT:    fcmovnb %st(1), %st(0) # sched: [3:1.00]
-; SKYLAKE-NEXT:    fcmovnbe %st(1), %st(0) # sched: [3:1.00]
-; SKYLAKE-NEXT:    fcmovne %st(1), %st(0) # sched: [3:1.00]
-; SKYLAKE-NEXT:    fcmovnu %st(1), %st(0) # sched: [3:1.00]
-; SKYLAKE-NEXT:    fcmovu %st(1), %st(0) # sched: [3:1.00]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retl # sched: [6:0.50]
-;
-; SKX-LABEL: test_fcmov:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    fcmovb %st(1), %st(0) # sched: [3:1.00]
-; SKX-NEXT:    fcmovbe %st(1), %st(0) # sched: [3:1.00]
-; SKX-NEXT:    fcmove %st(1), %st(0) # sched: [3:1.00]
-; SKX-NEXT:    fcmovnb %st(1), %st(0) # sched: [3:1.00]
-; SKX-NEXT:    fcmovnbe %st(1), %st(0) # sched: [3:1.00]
-; SKX-NEXT:    fcmovne %st(1), %st(0) # sched: [3:1.00]
-; SKX-NEXT:    fcmovnu %st(1), %st(0) # sched: [3:1.00]
-; SKX-NEXT:    fcmovu %st(1), %st(0) # sched: [3:1.00]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retl # sched: [6:0.50]
-;
-; BDVER2-LABEL: test_fcmov:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    fcmovb %st(1), %st(0) # sched: [1:1.00]
-; BDVER2-NEXT:    fcmovbe %st(1), %st(0) # sched: [1:1.00]
-; BDVER2-NEXT:    fcmove %st(1), %st(0) # sched: [1:1.00]
-; BDVER2-NEXT:    fcmovnb %st(1), %st(0) # sched: [1:1.00]
-; BDVER2-NEXT:    fcmovnbe %st(1), %st(0) # sched: [1:1.00]
-; BDVER2-NEXT:    fcmovne %st(1), %st(0) # sched: [1:1.00]
-; BDVER2-NEXT:    fcmovnu %st(1), %st(0) # sched: [1:1.00]
-; BDVER2-NEXT:    fcmovu %st(1), %st(0) # sched: [1:1.00]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retl # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_fcmov:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    fcmovb %st(1), %st(0) # sched: [3:1.00]
-; BTVER2-NEXT:    fcmovbe %st(1), %st(0) # sched: [3:1.00]
-; BTVER2-NEXT:    fcmove %st(1), %st(0) # sched: [3:1.00]
-; BTVER2-NEXT:    fcmovnb %st(1), %st(0) # sched: [3:1.00]
-; BTVER2-NEXT:    fcmovnbe %st(1), %st(0) # sched: [3:1.00]
-; BTVER2-NEXT:    fcmovne %st(1), %st(0) # sched: [3:1.00]
-; BTVER2-NEXT:    fcmovnu %st(1), %st(0) # sched: [3:1.00]
-; BTVER2-NEXT:    fcmovu %st(1), %st(0) # sched: [3:1.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retl # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_fcmov:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    fcmovb %st(1), %st(0) # sched: [100:0.25]
-; ZNVER1-NEXT:    fcmovbe %st(1), %st(0) # sched: [100:0.25]
-; ZNVER1-NEXT:    fcmove %st(1), %st(0) # sched: [100:0.25]
-; ZNVER1-NEXT:    fcmovnb %st(1), %st(0) # sched: [100:0.25]
-; ZNVER1-NEXT:    fcmovnbe %st(1), %st(0) # sched: [100:0.25]
-; ZNVER1-NEXT:    fcmovne %st(1), %st(0) # sched: [100:0.25]
-; ZNVER1-NEXT:    fcmovnu %st(1), %st(0) # sched: [100:0.25]
-; ZNVER1-NEXT:    fcmovu %st(1), %st(0) # sched: [100:0.25]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retl # sched: [1:0.50]
-  tail call void asm sideeffect "fcmovb %st(1), %st(0) \0A\09 fcmovbe %st(1), %st(0) \0A\09 fcmove %st(1), %st(0) \0A\09 fcmovnb %st(1), %st(0) \0A\09 fcmovnbe %st(1), %st(0) \0A\09 fcmovne %st(1), %st(0) \0A\09 fcmovnu %st(1), %st(0) \0A\09 fcmovu %st(1), %st(0)", ""() nounwind
-  ret void
-}
-
-define void @test_fcom(float *%a0, double *%a1) optsize {
-; GENERIC-LABEL: test_fcom:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; GENERIC-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    fcom %st(1)
-; GENERIC-NEXT:    fcom %st(3)
-; GENERIC-NEXT:    fcoms (%ecx)
-; GENERIC-NEXT:    fcoml (%eax)
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retl
-;
-; ATOM-LABEL: test_fcom:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [1:1.00]
-; ATOM-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [1:1.00]
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    fcom %st(1) # sched: [5:5.00]
-; ATOM-NEXT:    fcom %st(3) # sched: [5:5.00]
-; ATOM-NEXT:    fcoms (%ecx) # sched: [5:5.00]
-; ATOM-NEXT:    fcoml (%eax) # sched: [5:5.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retl # sched: [79:39.50]
-;
-; SLM-LABEL: test_fcom:
-; SLM:       # %bb.0:
-; SLM-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [3:1.00]
-; SLM-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [3:1.00]
-; SLM-NEXT:    #APP
-; SLM-NEXT:    fcom %st(1) # sched: [3:1.00]
-; SLM-NEXT:    fcom %st(3) # sched: [3:1.00]
-; SLM-NEXT:    fcoms (%ecx) # sched: [6:1.00]
-; SLM-NEXT:    fcoml (%eax) # sched: [6:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retl # sched: [4:1.00]
-;
-; SANDY-LABEL: test_fcom:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; SANDY-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    fcom %st(1) # sched: [1:1.00]
-; SANDY-NEXT:    fcom %st(3) # sched: [1:1.00]
-; SANDY-NEXT:    fcoms (%ecx) # sched: [8:1.00]
-; SANDY-NEXT:    fcoml (%eax) # sched: [8:1.00]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retl # sched: [6:1.00]
-;
-; HASWELL-LABEL: test_fcom:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; HASWELL-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    fcom %st(1) # sched: [1:1.00]
-; HASWELL-NEXT:    fcom %st(3) # sched: [1:1.00]
-; HASWELL-NEXT:    fcoms (%ecx) # sched: [8:1.00]
-; HASWELL-NEXT:    fcoml (%eax) # sched: [8:1.00]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retl # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_fcom:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; BROADWELL-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    fcom %st(1) # sched: [1:1.00]
-; BROADWELL-NEXT:    fcom %st(3) # sched: [1:1.00]
-; BROADWELL-NEXT:    fcoms (%ecx) # sched: [7:1.00]
-; BROADWELL-NEXT:    fcoml (%eax) # sched: [7:1.00]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retl # sched: [6:0.50]
-;
-; SKYLAKE-LABEL: test_fcom:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; SKYLAKE-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    fcom %st(1) # sched: [1:1.00]
-; SKYLAKE-NEXT:    fcom %st(3) # sched: [1:1.00]
-; SKYLAKE-NEXT:    fcoms (%ecx) # sched: [8:1.00]
-; SKYLAKE-NEXT:    fcoml (%eax) # sched: [8:1.00]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retl # sched: [6:0.50]
-;
-; SKX-LABEL: test_fcom:
-; SKX:       # %bb.0:
-; SKX-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; SKX-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; SKX-NEXT:    #APP
-; SKX-NEXT:    fcom %st(1) # sched: [1:1.00]
-; SKX-NEXT:    fcom %st(3) # sched: [1:1.00]
-; SKX-NEXT:    fcoms (%ecx) # sched: [8:1.00]
-; SKX-NEXT:    fcoml (%eax) # sched: [8:1.00]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retl # sched: [6:0.50]
-;
-; BDVER2-LABEL: test_fcom:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; BDVER2-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    fcom %st(1) # sched: [1:1.00]
-; BDVER2-NEXT:    fcom %st(3) # sched: [1:1.00]
-; BDVER2-NEXT:    fcoms (%ecx) # sched: [6:1.00]
-; BDVER2-NEXT:    fcoml (%eax) # sched: [6:1.00]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retl # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_fcom:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:1.00]
-; BTVER2-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:1.00]
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    fcom %st(1) # sched: [3:1.00]
-; BTVER2-NEXT:    fcom %st(3) # sched: [3:1.00]
-; BTVER2-NEXT:    fcoms (%ecx) # sched: [8:1.00]
-; BTVER2-NEXT:    fcoml (%eax) # sched: [8:1.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retl # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_fcom:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [8:0.50]
-; ZNVER1-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [8:0.50]
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    fcom %st(1) # sched: [1:1.00]
-; ZNVER1-NEXT:    fcom %st(3) # sched: [1:1.00]
-; ZNVER1-NEXT:    fcoms (%ecx) # sched: [8:1.00]
-; ZNVER1-NEXT:    fcoml (%eax) # sched: [8:1.00]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retl # sched: [1:0.50]
-  tail call void asm sideeffect "fcom \0A\09 fcom %st(3) \0A\09 fcoms $0 \0A\09 fcoml $1", "*m,*m"(float *%a0, double *%a1) nounwind
-  ret void
-}
-
-define void @test_fcomp_fcompp(float *%a0, double *%a1) optsize {
-; GENERIC-LABEL: test_fcomp_fcompp:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; GENERIC-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    fcomp %st(1)
-; GENERIC-NEXT:    fcomp %st(3)
-; GENERIC-NEXT:    fcomps (%ecx)
-; GENERIC-NEXT:    fcompl (%eax)
-; GENERIC-NEXT:    fcompp
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retl
-;
-; ATOM-LABEL: test_fcomp_fcompp:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [1:1.00]
-; ATOM-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [1:1.00]
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    fcomp %st(1) # sched: [5:5.00]
-; ATOM-NEXT:    fcomp %st(3) # sched: [5:5.00]
-; ATOM-NEXT:    fcomps (%ecx) # sched: [5:5.00]
-; ATOM-NEXT:    fcompl (%eax) # sched: [5:5.00]
-; ATOM-NEXT:    fcompp # sched: [1:1.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retl # sched: [79:39.50]
-;
-; SLM-LABEL: test_fcomp_fcompp:
-; SLM:       # %bb.0:
-; SLM-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [3:1.00]
-; SLM-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [3:1.00]
-; SLM-NEXT:    #APP
-; SLM-NEXT:    fcomp %st(1) # sched: [3:1.00]
-; SLM-NEXT:    fcomp %st(3) # sched: [3:1.00]
-; SLM-NEXT:    fcomps (%ecx) # sched: [6:1.00]
-; SLM-NEXT:    fcompl (%eax) # sched: [6:1.00]
-; SLM-NEXT:    fcompp # sched: [100:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retl # sched: [4:1.00]
-;
-; SANDY-LABEL: test_fcomp_fcompp:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; SANDY-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    fcomp %st(1) # sched: [1:1.00]
-; SANDY-NEXT:    fcomp %st(3) # sched: [1:1.00]
-; SANDY-NEXT:    fcomps (%ecx) # sched: [8:1.00]
-; SANDY-NEXT:    fcompl (%eax) # sched: [8:1.00]
-; SANDY-NEXT:    fcompp # sched: [100:0.33]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retl # sched: [6:1.00]
-;
-; HASWELL-LABEL: test_fcomp_fcompp:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; HASWELL-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    fcomp %st(1) # sched: [1:1.00]
-; HASWELL-NEXT:    fcomp %st(3) # sched: [1:1.00]
-; HASWELL-NEXT:    fcomps (%ecx) # sched: [8:1.00]
-; HASWELL-NEXT:    fcompl (%eax) # sched: [8:1.00]
-; HASWELL-NEXT:    fcompp # sched: [1:0.50]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retl # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_fcomp_fcompp:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; BROADWELL-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    fcomp %st(1) # sched: [1:1.00]
-; BROADWELL-NEXT:    fcomp %st(3) # sched: [1:1.00]
-; BROADWELL-NEXT:    fcomps (%ecx) # sched: [7:1.00]
-; BROADWELL-NEXT:    fcompl (%eax) # sched: [7:1.00]
-; BROADWELL-NEXT:    fcompp # sched: [100:0.25]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retl # sched: [6:0.50]
-;
-; SKYLAKE-LABEL: test_fcomp_fcompp:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; SKYLAKE-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    fcomp %st(1) # sched: [1:1.00]
-; SKYLAKE-NEXT:    fcomp %st(3) # sched: [1:1.00]
-; SKYLAKE-NEXT:    fcomps (%ecx) # sched: [8:1.00]
-; SKYLAKE-NEXT:    fcompl (%eax) # sched: [8:1.00]
-; SKYLAKE-NEXT:    fcompp # sched: [100:0.25]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retl # sched: [6:0.50]
-;
-; SKX-LABEL: test_fcomp_fcompp:
-; SKX:       # %bb.0:
-; SKX-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; SKX-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; SKX-NEXT:    #APP
-; SKX-NEXT:    fcomp %st(1) # sched: [1:1.00]
-; SKX-NEXT:    fcomp %st(3) # sched: [1:1.00]
-; SKX-NEXT:    fcomps (%ecx) # sched: [8:1.00]
-; SKX-NEXT:    fcompl (%eax) # sched: [8:1.00]
-; SKX-NEXT:    fcompp # sched: [100:0.25]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retl # sched: [6:0.50]
-;
-; BDVER2-LABEL: test_fcomp_fcompp:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; BDVER2-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    fcomp %st(1) # sched: [1:1.00]
-; BDVER2-NEXT:    fcomp %st(3) # sched: [1:1.00]
-; BDVER2-NEXT:    fcomps (%ecx) # sched: [6:1.00]
-; BDVER2-NEXT:    fcompl (%eax) # sched: [6:1.00]
-; BDVER2-NEXT:    fcompp # sched: [100:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retl # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_fcomp_fcompp:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:1.00]
-; BTVER2-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:1.00]
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    fcomp %st(1) # sched: [3:1.00]
-; BTVER2-NEXT:    fcomp %st(3) # sched: [3:1.00]
-; BTVER2-NEXT:    fcomps (%ecx) # sched: [8:1.00]
-; BTVER2-NEXT:    fcompl (%eax) # sched: [8:1.00]
-; BTVER2-NEXT:    fcompp # sched: [100:0.50]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retl # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_fcomp_fcompp:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [8:0.50]
-; ZNVER1-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [8:0.50]
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    fcomp %st(1) # sched: [1:1.00]
-; ZNVER1-NEXT:    fcomp %st(3) # sched: [1:1.00]
-; ZNVER1-NEXT:    fcomps (%ecx) # sched: [8:1.00]
-; ZNVER1-NEXT:    fcompl (%eax) # sched: [8:1.00]
-; ZNVER1-NEXT:    fcompp # sched: [1:1.00]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retl # sched: [1:0.50]
-  tail call void asm sideeffect "fcomp \0A\09 fcomp %st(3) \0A\09 fcomps $0 \0A\09 fcompl $1 \0A\09 fcompp", "*m,*m"(float *%a0, double *%a1) nounwind
-  ret void
-}
-
-define void @test_fcomi_fcomip() optsize {
-; GENERIC-LABEL: test_fcomi_fcomip:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    fcomi %st(3)
-; GENERIC-NEXT:    fcompi %st(3)
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retl
-;
-; ATOM-LABEL: test_fcomi_fcomip:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    fcomi %st(3) # sched: [9:4.50]
-; ATOM-NEXT:    fcompi %st(3) # sched: [9:4.50]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retl # sched: [79:39.50]
-;
-; SLM-LABEL: test_fcomi_fcomip:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    fcomi %st(3) # sched: [3:1.00]
-; SLM-NEXT:    fcompi %st(3) # sched: [3:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retl # sched: [4:1.00]
-;
-; SANDY-LABEL: test_fcomi_fcomip:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    fcomi %st(3) # sched: [3:1.00]
-; SANDY-NEXT:    fcompi %st(3) # sched: [3:1.00]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retl # sched: [6:1.00]
-;
-; HASWELL-LABEL: test_fcomi_fcomip:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    fcomi %st(3) # sched: [1:0.50]
-; HASWELL-NEXT:    fcompi %st(3) # sched: [1:0.50]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retl # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_fcomi_fcomip:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    fcomi %st(3) # sched: [3:1.00]
-; BROADWELL-NEXT:    fcompi %st(3) # sched: [3:1.00]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retl # sched: [6:0.50]
-;
-; SKYLAKE-LABEL: test_fcomi_fcomip:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    fcomi %st(3) # sched: [2:1.00]
-; SKYLAKE-NEXT:    fcompi %st(3) # sched: [2:1.00]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retl # sched: [6:0.50]
-;
-; SKX-LABEL: test_fcomi_fcomip:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    fcomi %st(3) # sched: [2:1.00]
-; SKX-NEXT:    fcompi %st(3) # sched: [2:1.00]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retl # sched: [6:0.50]
-;
-; BDVER2-LABEL: test_fcomi_fcomip:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    fcomi %st(3) # sched: [1:1.00]
-; BDVER2-NEXT:    fcompi %st(3) # sched: [1:1.00]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retl # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_fcomi_fcomip:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    fcomi %st(3) # sched: [3:1.00]
-; BTVER2-NEXT:    fcompi %st(3) # sched: [3:1.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retl # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_fcomi_fcomip:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    fcomi %st(3) # sched: [9:0.50]
-; ZNVER1-NEXT:    fcompi %st(3) # sched: [9:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retl # sched: [1:0.50]
-  tail call void asm sideeffect "fcomi %st(3) \0A\09 fcomip %st(3)", ""() nounwind
-  ret void
-}
-
-define void @test_fcos() optsize {
-; GENERIC-LABEL: test_fcos:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    fcos
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retl
-;
-; ATOM-LABEL: test_fcos:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    fcos # sched: [174:87.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retl # sched: [79:39.50]
-;
-; SLM-LABEL: test_fcos:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    fcos # sched: [100:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retl # sched: [4:1.00]
-;
-; SANDY-LABEL: test_fcos:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    fcos # sched: [100:0.33]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retl # sched: [6:1.00]
-;
-; HASWELL-LABEL: test_fcos:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    fcos # sched: [100:0.25]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retl # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_fcos:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    fcos # sched: [100:0.25]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retl # sched: [6:0.50]
-;
-; SKYLAKE-LABEL: test_fcos:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    fcos # sched: [100:0.25]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retl # sched: [6:0.50]
-;
-; SKX-LABEL: test_fcos:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    fcos # sched: [100:0.25]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retl # sched: [6:0.50]
-;
-; BDVER2-LABEL: test_fcos:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    fcos # sched: [100:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retl # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_fcos:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    fcos # sched: [100:0.50]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retl # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_fcos:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    fcos # sched: [100:0.25]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retl # sched: [1:0.50]
-  tail call void asm sideeffect "fcos", ""() nounwind
-  ret void
-}
-
-define void @test_fdecstp() optsize {
-; GENERIC-LABEL: test_fdecstp:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    fdecstp
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retl
-;
-; ATOM-LABEL: test_fdecstp:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    fdecstp # sched: [1:0.50]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retl # sched: [79:39.50]
-;
-; SLM-LABEL: test_fdecstp:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    fdecstp # sched: [100:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retl # sched: [4:1.00]
-;
-; SANDY-LABEL: test_fdecstp:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    fdecstp # sched: [1:1.00]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retl # sched: [6:1.00]
-;
-; HASWELL-LABEL: test_fdecstp:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    fdecstp # sched: [2:1.00]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retl # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_fdecstp:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    fdecstp # sched: [2:1.00]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retl # sched: [6:0.50]
-;
-; SKYLAKE-LABEL: test_fdecstp:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    fdecstp # sched: [2:1.00]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retl # sched: [6:0.50]
-;
-; SKX-LABEL: test_fdecstp:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    fdecstp # sched: [2:1.00]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retl # sched: [6:0.50]
-;
-; BDVER2-LABEL: test_fdecstp:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    fdecstp # sched: [100:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retl # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_fdecstp:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    fdecstp # sched: [100:0.50]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retl # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_fdecstp:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    fdecstp # sched: [11:1.00]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retl # sched: [1:0.50]
-  tail call void asm sideeffect "fdecstp", ""() nounwind
-  ret void
-}
-
-define void @test_fdiv(float *%a0, double *%a1) optsize {
-; GENERIC-LABEL: test_fdiv:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; GENERIC-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    fdiv %st(0), %st(1)
-; GENERIC-NEXT:    fdiv %st(2)
-; GENERIC-NEXT:    fdivs (%ecx)
-; GENERIC-NEXT:    fdivl (%eax)
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retl
-;
-; ATOM-LABEL: test_fdiv:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [1:1.00]
-; ATOM-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [1:1.00]
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    fdiv %st(0), %st(1) # sched: [34:17.00]
-; ATOM-NEXT:    fdiv %st(2) # sched: [34:17.00]
-; ATOM-NEXT:    fdivs (%ecx) # sched: [34:17.00]
-; ATOM-NEXT:    fdivl (%eax) # sched: [34:17.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retl # sched: [79:39.50]
-;
-; SLM-LABEL: test_fdiv:
-; SLM:       # %bb.0:
-; SLM-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [3:1.00]
-; SLM-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [3:1.00]
-; SLM-NEXT:    #APP
-; SLM-NEXT:    fdiv %st(0), %st(1) # sched: [19:17.00]
-; SLM-NEXT:    fdiv %st(2) # sched: [19:17.00]
-; SLM-NEXT:    fdivs (%ecx) # sched: [22:17.00]
-; SLM-NEXT:    fdivl (%eax) # sched: [22:17.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retl # sched: [4:1.00]
-;
-; SANDY-LABEL: test_fdiv:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; SANDY-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    fdiv %st(0), %st(1) # sched: [14:14.00]
-; SANDY-NEXT:    fdiv %st(2) # sched: [14:14.00]
-; SANDY-NEXT:    fdivs (%ecx) # sched: [31:1.00]
-; SANDY-NEXT:    fdivl (%eax) # sched: [31:1.00]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retl # sched: [6:1.00]
-;
-; HASWELL-LABEL: test_fdiv:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; HASWELL-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    fdiv %st(0), %st(1) # sched: [24:1.00]
-; HASWELL-NEXT:    fdiv %st(2) # sched: [20:1.00]
-; HASWELL-NEXT:    fdivs (%ecx) # sched: [31:1.00]
-; HASWELL-NEXT:    fdivl (%eax) # sched: [31:1.00]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retl # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_fdiv:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; BROADWELL-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    fdiv %st(0), %st(1) # sched: [15:1.00]
-; BROADWELL-NEXT:    fdiv %st(2) # sched: [20:1.00]
-; BROADWELL-NEXT:    fdivs (%ecx) # sched: [21:1.00]
-; BROADWELL-NEXT:    fdivl (%eax) # sched: [21:1.00]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retl # sched: [6:0.50]
-;
-; SKYLAKE-LABEL: test_fdiv:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; SKYLAKE-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    fdiv %st(0), %st(1) # sched: [15:1.00]
-; SKYLAKE-NEXT:    fdiv %st(2) # sched: [20:1.00]
-; SKYLAKE-NEXT:    fdivs (%ecx) # sched: [22:1.00]
-; SKYLAKE-NEXT:    fdivl (%eax) # sched: [22:1.00]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retl # sched: [6:0.50]
-;
-; SKX-LABEL: test_fdiv:
-; SKX:       # %bb.0:
-; SKX-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; SKX-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; SKX-NEXT:    #APP
-; SKX-NEXT:    fdiv %st(0), %st(1) # sched: [15:1.00]
-; SKX-NEXT:    fdiv %st(2) # sched: [20:1.00]
-; SKX-NEXT:    fdivs (%ecx) # sched: [22:1.00]
-; SKX-NEXT:    fdivl (%eax) # sched: [22:1.00]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retl # sched: [6:0.50]
-;
-; BDVER2-LABEL: test_fdiv:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; BDVER2-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    fdiv %st(0), %st(1) # sched: [9:9.50]
-; BDVER2-NEXT:    fdiv %st(2) # sched: [9:9.50]
-; BDVER2-NEXT:    fdivs (%ecx) # sched: [14:9.50]
-; BDVER2-NEXT:    fdivl (%eax) # sched: [14:9.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retl # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_fdiv:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:1.00]
-; BTVER2-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:1.00]
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    fdiv %st(0), %st(1) # sched: [19:19.00]
-; BTVER2-NEXT:    fdiv %st(2) # sched: [19:19.00]
-; BTVER2-NEXT:    fdivs (%ecx) # sched: [24:19.00]
-; BTVER2-NEXT:    fdivl (%eax) # sched: [24:19.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retl # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_fdiv:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [8:0.50]
-; ZNVER1-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [8:0.50]
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    fdiv %st(0), %st(1) # sched: [15:1.00]
-; ZNVER1-NEXT:    fdiv %st(2) # sched: [15:1.00]
-; ZNVER1-NEXT:    fdivs (%ecx) # sched: [22:1.00]
-; ZNVER1-NEXT:    fdivl (%eax) # sched: [22:1.00]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retl # sched: [1:0.50]
-  tail call void asm sideeffect "fdiv %st(0), %st(1) \0A\09 fdiv %st(2), %st(0) \0A\09 fdivs $0 \0A\09 fdivl $1", "*m,*m"(float *%a0, double *%a1) nounwind
-  ret void
-}
-
-define void @test_fdivp_fidiv(i16 *%a0, i32 *%a1) optsize {
-; GENERIC-LABEL: test_fdivp_fidiv:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; GENERIC-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    fdivp %st(1)
-; GENERIC-NEXT:    fdivp %st(2)
-; GENERIC-NEXT:    fidivs (%ecx)
-; GENERIC-NEXT:    fidivl (%eax)
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retl
-;
-; ATOM-LABEL: test_fdivp_fidiv:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [1:1.00]
-; ATOM-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [1:1.00]
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    fdivp %st(1) # sched: [34:17.00]
-; ATOM-NEXT:    fdivp %st(2) # sched: [34:17.00]
-; ATOM-NEXT:    fidivs (%ecx) # sched: [34:17.00]
-; ATOM-NEXT:    fidivl (%eax) # sched: [34:17.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retl # sched: [79:39.50]
-;
-; SLM-LABEL: test_fdivp_fidiv:
-; SLM:       # %bb.0:
-; SLM-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [3:1.00]
-; SLM-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [3:1.00]
-; SLM-NEXT:    #APP
-; SLM-NEXT:    fdivp %st(1) # sched: [19:17.00]
-; SLM-NEXT:    fdivp %st(2) # sched: [19:17.00]
-; SLM-NEXT:    fidivs (%ecx) # sched: [22:17.00]
-; SLM-NEXT:    fidivl (%eax) # sched: [22:17.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retl # sched: [4:1.00]
-;
-; SANDY-LABEL: test_fdivp_fidiv:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; SANDY-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    fdivp %st(1) # sched: [14:14.00]
-; SANDY-NEXT:    fdivp %st(2) # sched: [14:14.00]
-; SANDY-NEXT:    fidivs (%ecx) # sched: [34:1.00]
-; SANDY-NEXT:    fidivl (%eax) # sched: [34:1.00]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retl # sched: [6:1.00]
-;
-; HASWELL-LABEL: test_fdivp_fidiv:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; HASWELL-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    fdivp %st(1) # sched: [24:1.00]
-; HASWELL-NEXT:    fdivp %st(2) # sched: [24:1.00]
-; HASWELL-NEXT:    fidivs (%ecx) # sched: [34:1.00]
-; HASWELL-NEXT:    fidivl (%eax) # sched: [34:1.00]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retl # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_fdivp_fidiv:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; BROADWELL-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    fdivp %st(1) # sched: [15:1.00]
-; BROADWELL-NEXT:    fdivp %st(2) # sched: [15:1.00]
-; BROADWELL-NEXT:    fidivs (%ecx) # sched: [24:1.00]
-; BROADWELL-NEXT:    fidivl (%eax) # sched: [24:1.00]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retl # sched: [6:0.50]
-;
-; SKYLAKE-LABEL: test_fdivp_fidiv:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; SKYLAKE-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    fdivp %st(1) # sched: [15:1.00]
-; SKYLAKE-NEXT:    fdivp %st(2) # sched: [15:1.00]
-; SKYLAKE-NEXT:    fidivs (%ecx) # sched: [25:1.00]
-; SKYLAKE-NEXT:    fidivl (%eax) # sched: [25:1.00]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retl # sched: [6:0.50]
-;
-; SKX-LABEL: test_fdivp_fidiv:
-; SKX:       # %bb.0:
-; SKX-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; SKX-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; SKX-NEXT:    #APP
-; SKX-NEXT:    fdivp %st(1) # sched: [15:1.00]
-; SKX-NEXT:    fdivp %st(2) # sched: [15:1.00]
-; SKX-NEXT:    fidivs (%ecx) # sched: [25:1.00]
-; SKX-NEXT:    fidivl (%eax) # sched: [25:1.00]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retl # sched: [6:0.50]
-;
-; BDVER2-LABEL: test_fdivp_fidiv:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; BDVER2-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    fdivp %st(1) # sched: [9:9.50]
-; BDVER2-NEXT:    fdivp %st(2) # sched: [9:9.50]
-; BDVER2-NEXT:    fidivs (%ecx) # sched: [14:9.50]
-; BDVER2-NEXT:    fidivl (%eax) # sched: [14:9.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retl # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_fdivp_fidiv:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:1.00]
-; BTVER2-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:1.00]
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    fdivp %st(1) # sched: [19:19.00]
-; BTVER2-NEXT:    fdivp %st(2) # sched: [19:19.00]
-; BTVER2-NEXT:    fidivs (%ecx) # sched: [24:19.00]
-; BTVER2-NEXT:    fidivl (%eax) # sched: [24:19.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retl # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_fdivp_fidiv:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [8:0.50]
-; ZNVER1-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [8:0.50]
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    fdivp %st(1) # sched: [15:1.00]
-; ZNVER1-NEXT:    fdivp %st(2) # sched: [15:1.00]
-; ZNVER1-NEXT:    fidivs (%ecx) # sched: [22:1.00]
-; ZNVER1-NEXT:    fidivl (%eax) # sched: [22:1.00]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retl # sched: [1:0.50]
-  tail call void asm sideeffect "fdivp \0A\09 fdivp %st(2), %st(0) \0A\09 fidivs $0 \0A\09 fidivl $1", "*m,*m"(i16 *%a0, i32 *%a1) nounwind
-  ret void
-}
-
-define void @test_fdivr(float *%a0, double *%a1) optsize {
-; GENERIC-LABEL: test_fdivr:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; GENERIC-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    fdivr %st(0), %st(1)
-; GENERIC-NEXT:    fdivr %st(2)
-; GENERIC-NEXT:    fdivrs (%ecx)
-; GENERIC-NEXT:    fdivrl (%eax)
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retl
-;
-; ATOM-LABEL: test_fdivr:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [1:1.00]
-; ATOM-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [1:1.00]
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    fdivr %st(0), %st(1) # sched: [34:17.00]
-; ATOM-NEXT:    fdivr %st(2) # sched: [34:17.00]
-; ATOM-NEXT:    fdivrs (%ecx) # sched: [34:17.00]
-; ATOM-NEXT:    fdivrl (%eax) # sched: [34:17.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retl # sched: [79:39.50]
-;
-; SLM-LABEL: test_fdivr:
-; SLM:       # %bb.0:
-; SLM-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [3:1.00]
-; SLM-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [3:1.00]
-; SLM-NEXT:    #APP
-; SLM-NEXT:    fdivr %st(0), %st(1) # sched: [19:17.00]
-; SLM-NEXT:    fdivr %st(2) # sched: [19:17.00]
-; SLM-NEXT:    fdivrs (%ecx) # sched: [22:17.00]
-; SLM-NEXT:    fdivrl (%eax) # sched: [22:17.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retl # sched: [4:1.00]
-;
-; SANDY-LABEL: test_fdivr:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; SANDY-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    fdivr %st(0), %st(1) # sched: [14:14.00]
-; SANDY-NEXT:    fdivr %st(2) # sched: [14:14.00]
-; SANDY-NEXT:    fdivrs (%ecx) # sched: [31:1.00]
-; SANDY-NEXT:    fdivrl (%eax) # sched: [31:1.00]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retl # sched: [6:1.00]
-;
-; HASWELL-LABEL: test_fdivr:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; HASWELL-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    fdivr %st(0), %st(1) # sched: [20:1.00]
-; HASWELL-NEXT:    fdivr %st(2) # sched: [24:1.00]
-; HASWELL-NEXT:    fdivrs (%ecx) # sched: [27:1.00]
-; HASWELL-NEXT:    fdivrl (%eax) # sched: [27:1.00]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retl # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_fdivr:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; BROADWELL-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    fdivr %st(0), %st(1) # sched: [20:1.00]
-; BROADWELL-NEXT:    fdivr %st(2) # sched: [15:1.00]
-; BROADWELL-NEXT:    fdivrs (%ecx) # sched: [26:1.00]
-; BROADWELL-NEXT:    fdivrl (%eax) # sched: [26:1.00]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retl # sched: [6:0.50]
-;
-; SKYLAKE-LABEL: test_fdivr:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; SKYLAKE-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    fdivr %st(0), %st(1) # sched: [20:1.00]
-; SKYLAKE-NEXT:    fdivr %st(2) # sched: [15:1.00]
-; SKYLAKE-NEXT:    fdivrs (%ecx) # sched: [27:1.00]
-; SKYLAKE-NEXT:    fdivrl (%eax) # sched: [27:1.00]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retl # sched: [6:0.50]
-;
-; SKX-LABEL: test_fdivr:
-; SKX:       # %bb.0:
-; SKX-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; SKX-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; SKX-NEXT:    #APP
-; SKX-NEXT:    fdivr %st(0), %st(1) # sched: [20:1.00]
-; SKX-NEXT:    fdivr %st(2) # sched: [15:1.00]
-; SKX-NEXT:    fdivrs (%ecx) # sched: [27:1.00]
-; SKX-NEXT:    fdivrl (%eax) # sched: [27:1.00]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retl # sched: [6:0.50]
-;
-; BDVER2-LABEL: test_fdivr:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; BDVER2-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    fdivr %st(0), %st(1) # sched: [9:9.50]
-; BDVER2-NEXT:    fdivr %st(2) # sched: [9:9.50]
-; BDVER2-NEXT:    fdivrs (%ecx) # sched: [14:9.50]
-; BDVER2-NEXT:    fdivrl (%eax) # sched: [14:9.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retl # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_fdivr:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:1.00]
-; BTVER2-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:1.00]
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    fdivr %st(0), %st(1) # sched: [19:19.00]
-; BTVER2-NEXT:    fdivr %st(2) # sched: [19:19.00]
-; BTVER2-NEXT:    fdivrs (%ecx) # sched: [24:19.00]
-; BTVER2-NEXT:    fdivrl (%eax) # sched: [24:19.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retl # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_fdivr:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [8:0.50]
-; ZNVER1-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [8:0.50]
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    fdivr %st(0), %st(1) # sched: [15:1.00]
-; ZNVER1-NEXT:    fdivr %st(2) # sched: [15:1.00]
-; ZNVER1-NEXT:    fdivrs (%ecx) # sched: [22:1.00]
-; ZNVER1-NEXT:    fdivrl (%eax) # sched: [22:1.00]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retl # sched: [1:0.50]
-  tail call void asm sideeffect "fdivr %st(0), %st(1) \0A\09 fdivr %st(2), %st(0) \0A\09 fdivrs $0 \0A\09 fdivrl $1", "*m,*m"(float *%a0, double *%a1) nounwind
-  ret void
-}
-
-define void @test_fdivrp_fidivr(i16 *%a0, i32 *%a1) optsize {
-; GENERIC-LABEL: test_fdivrp_fidivr:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; GENERIC-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    fdivrp %st(1)
-; GENERIC-NEXT:    fdivrp %st(2)
-; GENERIC-NEXT:    fidivrs (%ecx)
-; GENERIC-NEXT:    fidivrl (%eax)
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retl
-;
-; ATOM-LABEL: test_fdivrp_fidivr:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [1:1.00]
-; ATOM-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [1:1.00]
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    fdivrp %st(1) # sched: [34:17.00]
-; ATOM-NEXT:    fdivrp %st(2) # sched: [34:17.00]
-; ATOM-NEXT:    fidivrs (%ecx) # sched: [34:17.00]
-; ATOM-NEXT:    fidivrl (%eax) # sched: [34:17.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retl # sched: [79:39.50]
-;
-; SLM-LABEL: test_fdivrp_fidivr:
-; SLM:       # %bb.0:
-; SLM-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [3:1.00]
-; SLM-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [3:1.00]
-; SLM-NEXT:    #APP
-; SLM-NEXT:    fdivrp %st(1) # sched: [19:17.00]
-; SLM-NEXT:    fdivrp %st(2) # sched: [19:17.00]
-; SLM-NEXT:    fidivrs (%ecx) # sched: [22:17.00]
-; SLM-NEXT:    fidivrl (%eax) # sched: [22:17.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retl # sched: [4:1.00]
-;
-; SANDY-LABEL: test_fdivrp_fidivr:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; SANDY-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    fdivrp %st(1) # sched: [14:14.00]
-; SANDY-NEXT:    fdivrp %st(2) # sched: [14:14.00]
-; SANDY-NEXT:    fidivrs (%ecx) # sched: [34:1.00]
-; SANDY-NEXT:    fidivrl (%eax) # sched: [34:1.00]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retl # sched: [6:1.00]
-;
-; HASWELL-LABEL: test_fdivrp_fidivr:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; HASWELL-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    fdivrp %st(1) # sched: [20:1.00]
-; HASWELL-NEXT:    fdivrp %st(2) # sched: [20:1.00]
-; HASWELL-NEXT:    fidivrs (%ecx) # sched: [30:1.00]
-; HASWELL-NEXT:    fidivrl (%eax) # sched: [30:1.00]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retl # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_fdivrp_fidivr:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; BROADWELL-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    fdivrp %st(1) # sched: [20:1.00]
-; BROADWELL-NEXT:    fdivrp %st(2) # sched: [20:1.00]
-; BROADWELL-NEXT:    fidivrs (%ecx) # sched: [29:1.00]
-; BROADWELL-NEXT:    fidivrl (%eax) # sched: [29:1.00]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retl # sched: [6:0.50]
-;
-; SKYLAKE-LABEL: test_fdivrp_fidivr:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; SKYLAKE-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    fdivrp %st(1) # sched: [20:1.00]
-; SKYLAKE-NEXT:    fdivrp %st(2) # sched: [20:1.00]
-; SKYLAKE-NEXT:    fidivrs (%ecx) # sched: [30:1.00]
-; SKYLAKE-NEXT:    fidivrl (%eax) # sched: [30:1.00]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retl # sched: [6:0.50]
-;
-; SKX-LABEL: test_fdivrp_fidivr:
-; SKX:       # %bb.0:
-; SKX-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; SKX-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; SKX-NEXT:    #APP
-; SKX-NEXT:    fdivrp %st(1) # sched: [20:1.00]
-; SKX-NEXT:    fdivrp %st(2) # sched: [20:1.00]
-; SKX-NEXT:    fidivrs (%ecx) # sched: [30:1.00]
-; SKX-NEXT:    fidivrl (%eax) # sched: [30:1.00]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retl # sched: [6:0.50]
-;
-; BDVER2-LABEL: test_fdivrp_fidivr:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; BDVER2-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    fdivrp %st(1) # sched: [9:9.50]
-; BDVER2-NEXT:    fdivrp %st(2) # sched: [9:9.50]
-; BDVER2-NEXT:    fidivrs (%ecx) # sched: [14:9.50]
-; BDVER2-NEXT:    fidivrl (%eax) # sched: [14:9.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retl # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_fdivrp_fidivr:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:1.00]
-; BTVER2-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:1.00]
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    fdivrp %st(1) # sched: [19:19.00]
-; BTVER2-NEXT:    fdivrp %st(2) # sched: [19:19.00]
-; BTVER2-NEXT:    fidivrs (%ecx) # sched: [24:19.00]
-; BTVER2-NEXT:    fidivrl (%eax) # sched: [24:19.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retl # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_fdivrp_fidivr:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [8:0.50]
-; ZNVER1-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [8:0.50]
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    fdivrp %st(1) # sched: [15:1.00]
-; ZNVER1-NEXT:    fdivrp %st(2) # sched: [15:1.00]
-; ZNVER1-NEXT:    fidivrs (%ecx) # sched: [22:1.00]
-; ZNVER1-NEXT:    fidivrl (%eax) # sched: [22:1.00]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retl # sched: [1:0.50]
-  tail call void asm sideeffect "fdivrp \0A\09 fdivrp %st(2), %st(0) \0A\09 fidivrs $0 \0A\09 fidivrl $1", "*m,*m"(i16 *%a0, i32 *%a1) nounwind
-  ret void
-}
-
-define void @test_ffree() optsize {
-; GENERIC-LABEL: test_ffree:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    ffree %st(0)
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retl
-;
-; ATOM-LABEL: test_ffree:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    ffree %st(0) # sched: [1:0.50]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retl # sched: [79:39.50]
-;
-; SLM-LABEL: test_ffree:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    ffree %st(0) # sched: [100:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retl # sched: [4:1.00]
-;
-; SANDY-LABEL: test_ffree:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    ffree %st(0) # sched: [1:1.00]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retl # sched: [6:1.00]
-;
-; HASWELL-LABEL: test_ffree:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    ffree %st(0) # sched: [1:0.50]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retl # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_ffree:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    ffree %st(0) # sched: [100:0.25]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retl # sched: [6:0.50]
-;
-; SKYLAKE-LABEL: test_ffree:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    ffree %st(0) # sched: [100:0.25]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retl # sched: [6:0.50]
-;
-; SKX-LABEL: test_ffree:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    ffree %st(0) # sched: [100:0.25]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retl # sched: [6:0.50]
-;
-; BDVER2-LABEL: test_ffree:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    ffree %st(0) # sched: [100:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retl # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_ffree:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    ffree %st(0) # sched: [100:0.50]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retl # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_ffree:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    ffree %st(0) # sched: [11:1.00]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retl # sched: [1:0.50]
-  tail call void asm sideeffect "ffree %st(0)", ""() nounwind
-  ret void
-}
-
-define void @test_ficom(i16 *%a0, i32 *%a1) optsize {
-; GENERIC-LABEL: test_ficom:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; GENERIC-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    ficoms (%ecx)
-; GENERIC-NEXT:    ficoml (%eax)
-; GENERIC-NEXT:    ficomps (%ecx)
-; GENERIC-NEXT:    ficompl (%eax)
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retl
-;
-; ATOM-LABEL: test_ficom:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [1:1.00]
-; ATOM-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [1:1.00]
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    ficoms (%ecx) # sched: [5:5.00]
-; ATOM-NEXT:    ficoml (%eax) # sched: [5:5.00]
-; ATOM-NEXT:    ficomps (%ecx) # sched: [5:5.00]
-; ATOM-NEXT:    ficompl (%eax) # sched: [5:5.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retl # sched: [79:39.50]
-;
-; SLM-LABEL: test_ficom:
-; SLM:       # %bb.0:
-; SLM-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [3:1.00]
-; SLM-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [3:1.00]
-; SLM-NEXT:    #APP
-; SLM-NEXT:    ficoms (%ecx) # sched: [6:1.00]
-; SLM-NEXT:    ficoml (%eax) # sched: [6:1.00]
-; SLM-NEXT:    ficomps (%ecx) # sched: [6:1.00]
-; SLM-NEXT:    ficompl (%eax) # sched: [6:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retl # sched: [4:1.00]
-;
-; SANDY-LABEL: test_ficom:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; SANDY-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    ficoms (%ecx) # sched: [11:2.00]
-; SANDY-NEXT:    ficoml (%eax) # sched: [11:2.00]
-; SANDY-NEXT:    ficomps (%ecx) # sched: [11:2.00]
-; SANDY-NEXT:    ficompl (%eax) # sched: [11:2.00]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retl # sched: [6:1.00]
-;
-; HASWELL-LABEL: test_ficom:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; HASWELL-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    ficoms (%ecx) # sched: [11:2.00]
-; HASWELL-NEXT:    ficoml (%eax) # sched: [11:2.00]
-; HASWELL-NEXT:    ficomps (%ecx) # sched: [11:2.00]
-; HASWELL-NEXT:    ficompl (%eax) # sched: [11:2.00]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retl # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_ficom:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; BROADWELL-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    ficoms (%ecx) # sched: [10:2.00]
-; BROADWELL-NEXT:    ficoml (%eax) # sched: [10:2.00]
-; BROADWELL-NEXT:    ficomps (%ecx) # sched: [10:2.00]
-; BROADWELL-NEXT:    ficompl (%eax) # sched: [10:2.00]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retl # sched: [6:0.50]
-;
-; SKYLAKE-LABEL: test_ficom:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; SKYLAKE-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    ficoms (%ecx) # sched: [11:2.00]
-; SKYLAKE-NEXT:    ficoml (%eax) # sched: [11:2.00]
-; SKYLAKE-NEXT:    ficomps (%ecx) # sched: [11:2.00]
-; SKYLAKE-NEXT:    ficompl (%eax) # sched: [11:2.00]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retl # sched: [6:0.50]
-;
-; SKX-LABEL: test_ficom:
-; SKX:       # %bb.0:
-; SKX-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; SKX-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; SKX-NEXT:    #APP
-; SKX-NEXT:    ficoms (%ecx) # sched: [11:2.00]
-; SKX-NEXT:    ficoml (%eax) # sched: [11:2.00]
-; SKX-NEXT:    ficomps (%ecx) # sched: [11:2.00]
-; SKX-NEXT:    ficompl (%eax) # sched: [11:2.00]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retl # sched: [6:0.50]
-;
-; BDVER2-LABEL: test_ficom:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; BDVER2-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    ficoms (%ecx) # sched: [6:1.00]
-; BDVER2-NEXT:    ficoml (%eax) # sched: [6:1.00]
-; BDVER2-NEXT:    ficomps (%ecx) # sched: [6:1.00]
-; BDVER2-NEXT:    ficompl (%eax) # sched: [6:1.00]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retl # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_ficom:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:1.00]
-; BTVER2-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:1.00]
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    ficoms (%ecx) # sched: [8:1.00]
-; BTVER2-NEXT:    ficoml (%eax) # sched: [8:1.00]
-; BTVER2-NEXT:    ficomps (%ecx) # sched: [8:1.00]
-; BTVER2-NEXT:    ficompl (%eax) # sched: [8:1.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retl # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_ficom:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [8:0.50]
-; ZNVER1-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [8:0.50]
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    ficoms (%ecx) # sched: [12:1.50]
-; ZNVER1-NEXT:    ficoml (%eax) # sched: [12:1.50]
-; ZNVER1-NEXT:    ficomps (%ecx) # sched: [12:1.50]
-; ZNVER1-NEXT:    ficompl (%eax) # sched: [12:1.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retl # sched: [1:0.50]
-  tail call void asm sideeffect "ficoms $0 \0A\09 ficoml $1 \0A\09 ficomps $0 \0A\09 ficompl $1", "*m,*m"(i16 *%a0, i32 *%a1) nounwind
-  ret void
-}
-
-define void @test_fild(i16 *%a0, i32 *%a1, i64 *%a2) optsize {
-; GENERIC-LABEL: test_fild:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; GENERIC-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; GENERIC-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    filds (%edx)
-; GENERIC-NEXT:    fildl (%ecx)
-; GENERIC-NEXT:    fildll (%eax)
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retl
-;
-; ATOM-LABEL: test_fild:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [1:1.00]
-; ATOM-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [1:1.00]
-; ATOM-NEXT:    movl {{[0-9]+}}(%esp), %edx # sched: [1:1.00]
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    filds (%edx) # sched: [5:5.00]
-; ATOM-NEXT:    fildl (%ecx) # sched: [5:5.00]
-; ATOM-NEXT:    fildll (%eax) # sched: [5:5.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retl # sched: [79:39.50]
-;
-; SLM-LABEL: test_fild:
-; SLM:       # %bb.0:
-; SLM-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [3:1.00]
-; SLM-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [3:1.00]
-; SLM-NEXT:    movl {{[0-9]+}}(%esp), %edx # sched: [3:1.00]
-; SLM-NEXT:    #APP
-; SLM-NEXT:    filds (%edx) # sched: [3:1.00]
-; SLM-NEXT:    fildl (%ecx) # sched: [3:1.00]
-; SLM-NEXT:    fildll (%eax) # sched: [3:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retl # sched: [4:1.00]
-;
-; SANDY-LABEL: test_fild:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; SANDY-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; SANDY-NEXT:    movl {{[0-9]+}}(%esp), %edx # sched: [5:0.50]
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    filds (%edx) # sched: [10:1.00]
-; SANDY-NEXT:    fildl (%ecx) # sched: [10:1.00]
-; SANDY-NEXT:    fildll (%eax) # sched: [10:1.00]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retl # sched: [6:1.00]
-;
-; HASWELL-LABEL: test_fild:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; HASWELL-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; HASWELL-NEXT:    movl {{[0-9]+}}(%esp), %edx # sched: [5:0.50]
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    filds (%edx) # sched: [10:1.00]
-; HASWELL-NEXT:    fildl (%ecx) # sched: [10:1.00]
-; HASWELL-NEXT:    fildll (%eax) # sched: [10:1.00]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retl # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_fild:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; BROADWELL-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; BROADWELL-NEXT:    movl {{[0-9]+}}(%esp), %edx # sched: [5:0.50]
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    filds (%edx) # sched: [9:1.00]
-; BROADWELL-NEXT:    fildl (%ecx) # sched: [9:1.00]
-; BROADWELL-NEXT:    fildll (%eax) # sched: [9:1.00]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retl # sched: [6:0.50]
-;
-; SKYLAKE-LABEL: test_fild:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; SKYLAKE-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; SKYLAKE-NEXT:    movl {{[0-9]+}}(%esp), %edx # sched: [5:0.50]
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    filds (%edx) # sched: [10:1.00]
-; SKYLAKE-NEXT:    fildl (%ecx) # sched: [10:1.00]
-; SKYLAKE-NEXT:    fildll (%eax) # sched: [10:1.00]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retl # sched: [6:0.50]
-;
-; SKX-LABEL: test_fild:
-; SKX:       # %bb.0:
-; SKX-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; SKX-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; SKX-NEXT:    movl {{[0-9]+}}(%esp), %edx # sched: [5:0.50]
-; SKX-NEXT:    #APP
-; SKX-NEXT:    filds (%edx) # sched: [10:1.00]
-; SKX-NEXT:    fildl (%ecx) # sched: [10:1.00]
-; SKX-NEXT:    fildll (%eax) # sched: [10:1.00]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retl # sched: [6:0.50]
-;
-; BDVER2-LABEL: test_fild:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; BDVER2-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; BDVER2-NEXT:    movl {{[0-9]+}}(%esp), %edx # sched: [5:0.50]
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    filds (%edx) # sched: [5:0.50]
-; BDVER2-NEXT:    fildl (%ecx) # sched: [5:0.50]
-; BDVER2-NEXT:    fildll (%eax) # sched: [5:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retl # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_fild:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:1.00]
-; BTVER2-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:1.00]
-; BTVER2-NEXT:    movl {{[0-9]+}}(%esp), %edx # sched: [5:1.00]
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    filds (%edx) # sched: [5:1.00]
-; BTVER2-NEXT:    fildl (%ecx) # sched: [5:1.00]
-; BTVER2-NEXT:    fildll (%eax) # sched: [5:1.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retl # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_fild:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [8:0.50]
-; ZNVER1-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [8:0.50]
-; ZNVER1-NEXT:    movl {{[0-9]+}}(%esp), %edx # sched: [8:0.50]
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    filds (%edx) # sched: [11:1.00]
-; ZNVER1-NEXT:    fildl (%ecx) # sched: [11:1.00]
-; ZNVER1-NEXT:    fildll (%eax) # sched: [11:1.00]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retl # sched: [1:0.50]
-  tail call void asm sideeffect "filds $0 \0A\09 fildl $1 \0A\09 fildll $2", "*m,*m,*m"(i16 *%a0, i32 *%a1, i64 *%a2) nounwind
-  ret void
-}
-
-define void @test_fincstp() optsize {
-; GENERIC-LABEL: test_fincstp:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    fincstp
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retl
-;
-; ATOM-LABEL: test_fincstp:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    fincstp # sched: [1:0.50]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retl # sched: [79:39.50]
-;
-; SLM-LABEL: test_fincstp:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    fincstp # sched: [100:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retl # sched: [4:1.00]
-;
-; SANDY-LABEL: test_fincstp:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    fincstp # sched: [1:1.00]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retl # sched: [6:1.00]
-;
-; HASWELL-LABEL: test_fincstp:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    fincstp # sched: [1:0.50]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retl # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_fincstp:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    fincstp # sched: [1:0.50]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retl # sched: [6:0.50]
-;
-; SKYLAKE-LABEL: test_fincstp:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    fincstp # sched: [1:0.50]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retl # sched: [6:0.50]
-;
-; SKX-LABEL: test_fincstp:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    fincstp # sched: [1:0.50]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retl # sched: [6:0.50]
-;
-; BDVER2-LABEL: test_fincstp:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    fincstp # sched: [100:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retl # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_fincstp:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    fincstp # sched: [100:0.50]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retl # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_fincstp:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    fincstp # sched: [11:1.00]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retl # sched: [1:0.50]
-  tail call void asm sideeffect "fincstp", ""() nounwind
-  ret void
-}
-
-define void @test_finit() optsize {
-; GENERIC-LABEL: test_finit:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    wait
-; GENERIC-NEXT:    fninit
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retl
-;
-; ATOM-LABEL: test_finit:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    wait # sched: [1:0.50]
-; ATOM-NEXT:    fninit # sched: [63:31.50]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retl # sched: [79:39.50]
-;
-; SLM-LABEL: test_finit:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    wait # sched: [100:1.00]
-; SLM-NEXT:    fninit # sched: [100:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retl # sched: [4:1.00]
-;
-; SANDY-LABEL: test_finit:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    wait # sched: [100:0.33]
-; SANDY-NEXT:    fninit # sched: [5:1.33]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retl # sched: [6:1.00]
-;
-; HASWELL-LABEL: test_finit:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    wait # sched: [2:0.50]
-; HASWELL-NEXT:    fninit # sched: [75:6.00]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retl # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_finit:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    wait # sched: [2:0.50]
-; BROADWELL-NEXT:    fninit # sched: [75:6.00]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retl # sched: [6:0.50]
-;
-; SKYLAKE-LABEL: test_finit:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    wait # sched: [2:0.50]
-; SKYLAKE-NEXT:    fninit # sched: [75:6.00]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retl # sched: [6:0.50]
-;
-; SKX-LABEL: test_finit:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    wait # sched: [2:0.50]
-; SKX-NEXT:    fninit # sched: [75:6.00]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retl # sched: [6:0.50]
-;
-; BDVER2-LABEL: test_finit:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    wait # sched: [100:0.50]
-; BDVER2-NEXT:    fninit # sched: [100:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retl # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_finit:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    wait # sched: [100:0.50]
-; BTVER2-NEXT:    fninit # sched: [100:0.50]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retl # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_finit:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    wait # sched: [1:1.00]
-; ZNVER1-NEXT:    fninit # sched: [100:0.25]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retl # sched: [1:0.50]
-  tail call void asm sideeffect "finit", ""() nounwind
-  ret void
-}
-
-define void @test_fninit() optsize {
-; GENERIC-LABEL: test_fninit:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    fninit
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retl
-;
-; ATOM-LABEL: test_fninit:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    fninit # sched: [63:31.50]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retl # sched: [79:39.50]
-;
-; SLM-LABEL: test_fninit:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    fninit # sched: [100:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retl # sched: [4:1.00]
-;
-; SANDY-LABEL: test_fninit:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    fninit # sched: [5:1.33]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retl # sched: [6:1.00]
-;
-; HASWELL-LABEL: test_fninit:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    fninit # sched: [75:6.00]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retl # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_fninit:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    fninit # sched: [75:6.00]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retl # sched: [6:0.50]
-;
-; SKYLAKE-LABEL: test_fninit:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    fninit # sched: [75:6.00]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retl # sched: [6:0.50]
-;
-; SKX-LABEL: test_fninit:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    fninit # sched: [75:6.00]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retl # sched: [6:0.50]
-;
-; BDVER2-LABEL: test_fninit:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    fninit # sched: [100:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retl # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_fninit:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    fninit # sched: [100:0.50]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retl # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_fninit:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    fninit # sched: [100:0.25]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retl # sched: [1:0.50]
-  tail call void asm sideeffect "fninit", ""() nounwind
-  ret void
-}
-
-define void @test_fist_fistp_fisttp(i16* %a0, i32* %a1, i64 *%a2) optsize {
-; GENERIC-LABEL: test_fist_fistp_fisttp:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; GENERIC-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; GENERIC-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    fists (%edx)
-; GENERIC-NEXT:    fistl (%ecx)
-; GENERIC-NEXT:    fistps (%edx)
-; GENERIC-NEXT:    fistpl (%ecx)
-; GENERIC-NEXT:    fistpll (%eax)
-; GENERIC-NEXT:    fisttps (%edx)
-; GENERIC-NEXT:    fisttpl (%ecx)
-; GENERIC-NEXT:    fisttpll (%eax)
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retl
-;
-; ATOM-LABEL: test_fist_fistp_fisttp:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [1:1.00]
-; ATOM-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [1:1.00]
-; ATOM-NEXT:    movl {{[0-9]+}}(%esp), %edx # sched: [1:1.00]
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    fists (%edx) # sched: [6:3.00]
-; ATOM-NEXT:    fistl (%ecx) # sched: [6:3.00]
-; ATOM-NEXT:    fistps (%edx) # sched: [6:3.00]
-; ATOM-NEXT:    fistpl (%ecx) # sched: [6:3.00]
-; ATOM-NEXT:    fistpll (%eax) # sched: [6:3.00]
-; ATOM-NEXT:    fisttps (%edx) # sched: [2:1.00]
-; ATOM-NEXT:    fisttpl (%ecx) # sched: [2:1.00]
-; ATOM-NEXT:    fisttpll (%eax) # sched: [2:1.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retl # sched: [79:39.50]
-;
-; SLM-LABEL: test_fist_fistp_fisttp:
-; SLM:       # %bb.0:
-; SLM-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [3:1.00]
-; SLM-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [3:1.00]
-; SLM-NEXT:    movl {{[0-9]+}}(%esp), %edx # sched: [3:1.00]
-; SLM-NEXT:    #APP
-; SLM-NEXT:    fists (%edx) # sched: [1:1.00]
-; SLM-NEXT:    fistl (%ecx) # sched: [1:1.00]
-; SLM-NEXT:    fistps (%edx) # sched: [1:1.00]
-; SLM-NEXT:    fistpl (%ecx) # sched: [1:1.00]
-; SLM-NEXT:    fistpll (%eax) # sched: [1:1.00]
-; SLM-NEXT:    fisttps (%edx) # sched: [1:1.00]
-; SLM-NEXT:    fisttpl (%ecx) # sched: [1:1.00]
-; SLM-NEXT:    fisttpll (%eax) # sched: [1:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retl # sched: [4:1.00]
-;
-; SANDY-LABEL: test_fist_fistp_fisttp:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; SANDY-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; SANDY-NEXT:    movl {{[0-9]+}}(%esp), %edx # sched: [5:0.50]
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    fists (%edx) # sched: [9:1.00]
-; SANDY-NEXT:    fistl (%ecx) # sched: [9:1.00]
-; SANDY-NEXT:    fistps (%edx) # sched: [9:1.00]
-; SANDY-NEXT:    fistpl (%ecx) # sched: [9:1.00]
-; SANDY-NEXT:    fistpll (%eax) # sched: [9:1.00]
-; SANDY-NEXT:    fisttps (%edx) # sched: [5:1.00]
-; SANDY-NEXT:    fisttpl (%ecx) # sched: [5:1.00]
-; SANDY-NEXT:    fisttpll (%eax) # sched: [5:1.00]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retl # sched: [6:1.00]
-;
-; HASWELL-LABEL: test_fist_fistp_fisttp:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; HASWELL-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; HASWELL-NEXT:    movl {{[0-9]+}}(%esp), %edx # sched: [5:0.50]
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    fists (%edx) # sched: [4:1.00]
-; HASWELL-NEXT:    fistl (%ecx) # sched: [4:1.00]
-; HASWELL-NEXT:    fistps (%edx) # sched: [4:1.00]
-; HASWELL-NEXT:    fistpl (%ecx) # sched: [4:1.00]
-; HASWELL-NEXT:    fistpll (%eax) # sched: [4:1.00]
-; HASWELL-NEXT:    fisttps (%edx) # sched: [4:1.00]
-; HASWELL-NEXT:    fisttpl (%ecx) # sched: [4:1.00]
-; HASWELL-NEXT:    fisttpll (%eax) # sched: [4:1.00]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retl # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_fist_fistp_fisttp:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; BROADWELL-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; BROADWELL-NEXT:    movl {{[0-9]+}}(%esp), %edx # sched: [5:0.50]
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    fists (%edx) # sched: [4:1.00]
-; BROADWELL-NEXT:    fistl (%ecx) # sched: [4:1.00]
-; BROADWELL-NEXT:    fistps (%edx) # sched: [4:1.00]
-; BROADWELL-NEXT:    fistpl (%ecx) # sched: [4:1.00]
-; BROADWELL-NEXT:    fistpll (%eax) # sched: [4:1.00]
-; BROADWELL-NEXT:    fisttps (%edx) # sched: [4:1.00]
-; BROADWELL-NEXT:    fisttpl (%ecx) # sched: [4:1.00]
-; BROADWELL-NEXT:    fisttpll (%eax) # sched: [4:1.00]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retl # sched: [6:0.50]
-;
-; SKYLAKE-LABEL: test_fist_fistp_fisttp:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; SKYLAKE-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; SKYLAKE-NEXT:    movl {{[0-9]+}}(%esp), %edx # sched: [5:0.50]
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    fists (%edx) # sched: [4:1.00]
-; SKYLAKE-NEXT:    fistl (%ecx) # sched: [4:1.00]
-; SKYLAKE-NEXT:    fistps (%edx) # sched: [4:1.00]
-; SKYLAKE-NEXT:    fistpl (%ecx) # sched: [4:1.00]
-; SKYLAKE-NEXT:    fistpll (%eax) # sched: [4:1.00]
-; SKYLAKE-NEXT:    fisttps (%edx) # sched: [4:1.00]
-; SKYLAKE-NEXT:    fisttpl (%ecx) # sched: [4:1.00]
-; SKYLAKE-NEXT:    fisttpll (%eax) # sched: [4:1.00]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retl # sched: [6:0.50]
-;
-; SKX-LABEL: test_fist_fistp_fisttp:
-; SKX:       # %bb.0:
-; SKX-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; SKX-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; SKX-NEXT:    movl {{[0-9]+}}(%esp), %edx # sched: [5:0.50]
-; SKX-NEXT:    #APP
-; SKX-NEXT:    fists (%edx) # sched: [4:1.00]
-; SKX-NEXT:    fistl (%ecx) # sched: [4:1.00]
-; SKX-NEXT:    fistps (%edx) # sched: [4:1.00]
-; SKX-NEXT:    fistpl (%ecx) # sched: [4:1.00]
-; SKX-NEXT:    fistpll (%eax) # sched: [4:1.00]
-; SKX-NEXT:    fisttps (%edx) # sched: [4:1.00]
-; SKX-NEXT:    fisttpl (%ecx) # sched: [4:1.00]
-; SKX-NEXT:    fisttpll (%eax) # sched: [4:1.00]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retl # sched: [6:0.50]
-;
-; BDVER2-LABEL: test_fist_fistp_fisttp:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; BDVER2-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; BDVER2-NEXT:    movl {{[0-9]+}}(%esp), %edx # sched: [5:0.50]
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    fists (%edx) # sched: [1:1.00]
-; BDVER2-NEXT:    fistl (%ecx) # sched: [1:1.00]
-; BDVER2-NEXT:    fistps (%edx) # sched: [1:1.00]
-; BDVER2-NEXT:    fistpl (%ecx) # sched: [1:1.00]
-; BDVER2-NEXT:    fistpll (%eax) # sched: [1:1.00]
-; BDVER2-NEXT:    fisttps (%edx) # sched: [1:1.00]
-; BDVER2-NEXT:    fisttpl (%ecx) # sched: [1:1.00]
-; BDVER2-NEXT:    fisttpll (%eax) # sched: [1:1.00]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retl # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_fist_fistp_fisttp:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:1.00]
-; BTVER2-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:1.00]
-; BTVER2-NEXT:    movl {{[0-9]+}}(%esp), %edx # sched: [5:1.00]
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    fists (%edx) # sched: [1:1.00]
-; BTVER2-NEXT:    fistl (%ecx) # sched: [1:1.00]
-; BTVER2-NEXT:    fistps (%edx) # sched: [1:1.00]
-; BTVER2-NEXT:    fistpl (%ecx) # sched: [1:1.00]
-; BTVER2-NEXT:    fistpll (%eax) # sched: [1:1.00]
-; BTVER2-NEXT:    fisttps (%edx) # sched: [1:1.00]
-; BTVER2-NEXT:    fisttpl (%ecx) # sched: [1:1.00]
-; BTVER2-NEXT:    fisttpll (%eax) # sched: [1:1.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retl # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_fist_fistp_fisttp:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [8:0.50]
-; ZNVER1-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [8:0.50]
-; ZNVER1-NEXT:    movl {{[0-9]+}}(%esp), %edx # sched: [8:0.50]
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    fists (%edx) # sched: [12:0.50]
-; ZNVER1-NEXT:    fistl (%ecx) # sched: [12:0.50]
-; ZNVER1-NEXT:    fistps (%edx) # sched: [12:0.50]
-; ZNVER1-NEXT:    fistpl (%ecx) # sched: [12:0.50]
-; ZNVER1-NEXT:    fistpll (%eax) # sched: [12:0.50]
-; ZNVER1-NEXT:    fisttps (%edx) # sched: [12:0.50]
-; ZNVER1-NEXT:    fisttpl (%ecx) # sched: [12:0.50]
-; ZNVER1-NEXT:    fisttpll (%eax) # sched: [12:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retl # sched: [1:0.50]
-  tail call void asm sideeffect "fists $0 \0A\09 fistl $1 \0A\09 fistps $0 \0A\09 fistpl $1 \0A\09 fistpll $2 \0A\09 fisttps $0 \0A\09 fisttpl $1 \0A\09 fisttpll $2", "*m,*m,*m"(i16* %a0, i32* %a1, i64 *%a2) nounwind
-  ret void
-}
-
-define void @test_fld(i16* %a0, i32* %a1, i64 *%a2) optsize {
-; GENERIC-LABEL: test_fld:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; GENERIC-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; GENERIC-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    fld %st(0)
-; GENERIC-NEXT:    flds (%edx)
-; GENERIC-NEXT:    fldl (%ecx)
-; GENERIC-NEXT:    fldt (%eax)
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retl
-;
-; ATOM-LABEL: test_fld:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [1:1.00]
-; ATOM-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [1:1.00]
-; ATOM-NEXT:    movl {{[0-9]+}}(%esp), %edx # sched: [1:1.00]
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    fld %st(0) # sched: [1:1.00]
-; ATOM-NEXT:    flds (%edx) # sched: [1:1.00]
-; ATOM-NEXT:    fldl (%ecx) # sched: [1:1.00]
-; ATOM-NEXT:    fldt (%eax) # sched: [4:2.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retl # sched: [79:39.50]
-;
-; SLM-LABEL: test_fld:
-; SLM:       # %bb.0:
-; SLM-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [3:1.00]
-; SLM-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [3:1.00]
-; SLM-NEXT:    movl {{[0-9]+}}(%esp), %edx # sched: [3:1.00]
-; SLM-NEXT:    #APP
-; SLM-NEXT:    fld %st(0) # sched: [1:0.50]
-; SLM-NEXT:    flds (%edx) # sched: [3:1.00]
-; SLM-NEXT:    fldl (%ecx) # sched: [3:1.00]
-; SLM-NEXT:    fldt (%eax) # sched: [3:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retl # sched: [4:1.00]
-;
-; SANDY-LABEL: test_fld:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; SANDY-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; SANDY-NEXT:    movl {{[0-9]+}}(%esp), %edx # sched: [5:0.50]
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    fld %st(0) # sched: [1:1.00]
-; SANDY-NEXT:    flds (%edx) # sched: [9:1.00]
-; SANDY-NEXT:    fldl (%ecx) # sched: [9:1.00]
-; SANDY-NEXT:    fldt (%eax) # sched: [9:1.00]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retl # sched: [6:1.00]
-;
-; HASWELL-LABEL: test_fld:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; HASWELL-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; HASWELL-NEXT:    movl {{[0-9]+}}(%esp), %edx # sched: [5:0.50]
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    fld %st(0) # sched: [1:0.50]
-; HASWELL-NEXT:    flds (%edx) # sched: [7:0.50]
-; HASWELL-NEXT:    fldl (%ecx) # sched: [7:0.50]
-; HASWELL-NEXT:    fldt (%eax) # sched: [7:0.50]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retl # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_fld:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; BROADWELL-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; BROADWELL-NEXT:    movl {{[0-9]+}}(%esp), %edx # sched: [5:0.50]
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    fld %st(0) # sched: [1:0.25]
-; BROADWELL-NEXT:    flds (%edx) # sched: [6:0.50]
-; BROADWELL-NEXT:    fldl (%ecx) # sched: [6:0.50]
-; BROADWELL-NEXT:    fldt (%eax) # sched: [6:0.50]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retl # sched: [6:0.50]
-;
-; SKYLAKE-LABEL: test_fld:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; SKYLAKE-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; SKYLAKE-NEXT:    movl {{[0-9]+}}(%esp), %edx # sched: [5:0.50]
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    fld %st(0) # sched: [1:0.25]
-; SKYLAKE-NEXT:    flds (%edx) # sched: [7:0.50]
-; SKYLAKE-NEXT:    fldl (%ecx) # sched: [7:0.50]
-; SKYLAKE-NEXT:    fldt (%eax) # sched: [7:0.50]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retl # sched: [6:0.50]
-;
-; SKX-LABEL: test_fld:
-; SKX:       # %bb.0:
-; SKX-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; SKX-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; SKX-NEXT:    movl {{[0-9]+}}(%esp), %edx # sched: [5:0.50]
-; SKX-NEXT:    #APP
-; SKX-NEXT:    fld %st(0) # sched: [1:0.25]
-; SKX-NEXT:    flds (%edx) # sched: [7:0.50]
-; SKX-NEXT:    fldl (%ecx) # sched: [7:0.50]
-; SKX-NEXT:    fldt (%eax) # sched: [7:0.50]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retl # sched: [6:0.50]
-;
-; BDVER2-LABEL: test_fld:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; BDVER2-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; BDVER2-NEXT:    movl {{[0-9]+}}(%esp), %edx # sched: [5:0.50]
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    fld %st(0) # sched: [1:0.50]
-; BDVER2-NEXT:    flds (%edx) # sched: [5:0.50]
-; BDVER2-NEXT:    fldl (%ecx) # sched: [5:0.50]
-; BDVER2-NEXT:    fldt (%eax) # sched: [5:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retl # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_fld:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:1.00]
-; BTVER2-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:1.00]
-; BTVER2-NEXT:    movl {{[0-9]+}}(%esp), %edx # sched: [5:1.00]
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    fld %st(0) # sched: [1:0.50]
-; BTVER2-NEXT:    flds (%edx) # sched: [5:1.00]
-; BTVER2-NEXT:    fldl (%ecx) # sched: [5:1.00]
-; BTVER2-NEXT:    fldt (%eax) # sched: [5:1.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retl # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_fld:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [8:0.50]
-; ZNVER1-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [8:0.50]
-; ZNVER1-NEXT:    movl {{[0-9]+}}(%esp), %edx # sched: [8:0.50]
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    fld %st(0) # sched: [1:0.50]
-; ZNVER1-NEXT:    flds (%edx) # sched: [8:0.50]
-; ZNVER1-NEXT:    fldl (%ecx) # sched: [8:0.50]
-; ZNVER1-NEXT:    fldt (%eax) # sched: [1:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retl # sched: [1:0.50]
-  tail call void asm sideeffect "fld %st(0) \0A\09 flds $0 \0A\09 fldl $1 \0A\09 fldt $2", "*m,*m,*m"(i16* %a0, i32* %a1, i64 *%a2) nounwind
-  ret void
-}
-
-define void @test_fldcw_fldenv(i8* %a0) optsize {
-; GENERIC-LABEL: test_fldcw_fldenv:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    fldcw (%eax)
-; GENERIC-NEXT:    fldenv (%eax)
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retl
-;
-; ATOM-LABEL: test_fldcw_fldenv:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [1:1.00]
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    fldcw (%eax) # sched: [5:2.50]
-; ATOM-NEXT:    fldenv (%eax) # sched: [100:0.50]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retl # sched: [79:39.50]
-;
-; SLM-LABEL: test_fldcw_fldenv:
-; SLM:       # %bb.0:
-; SLM-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [3:1.00]
-; SLM-NEXT:    #APP
-; SLM-NEXT:    fldcw (%eax) # sched: [3:1.00]
-; SLM-NEXT:    fldenv (%eax) # sched: [100:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retl # sched: [4:1.00]
-;
-; SANDY-LABEL: test_fldcw_fldenv:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    fldcw (%eax) # sched: [8:2.00]
-; SANDY-NEXT:    fldenv (%eax) # sched: [100:0.33]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retl # sched: [6:1.00]
-;
-; HASWELL-LABEL: test_fldcw_fldenv:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    fldcw (%eax) # sched: [7:1.00]
-; HASWELL-NEXT:    fldenv (%eax) # sched: [61:14.00]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retl # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_fldcw_fldenv:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    fldcw (%eax) # sched: [7:1.00]
-; BROADWELL-NEXT:    fldenv (%eax) # sched: [60:14.00]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retl # sched: [6:0.50]
-;
-; SKYLAKE-LABEL: test_fldcw_fldenv:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    fldcw (%eax) # sched: [7:1.00]
-; SKYLAKE-NEXT:    fldenv (%eax) # sched: [62:14.00]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retl # sched: [6:0.50]
-;
-; SKX-LABEL: test_fldcw_fldenv:
-; SKX:       # %bb.0:
-; SKX-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; SKX-NEXT:    #APP
-; SKX-NEXT:    fldcw (%eax) # sched: [7:1.00]
-; SKX-NEXT:    fldenv (%eax) # sched: [62:14.00]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retl # sched: [6:0.50]
-;
-; BDVER2-LABEL: test_fldcw_fldenv:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    fldcw (%eax) # sched: [5:0.50]
-; BDVER2-NEXT:    fldenv (%eax) # sched: [100:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retl # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_fldcw_fldenv:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:1.00]
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    fldcw (%eax) # sched: [5:1.00]
-; BTVER2-NEXT:    fldenv (%eax) # sched: [100:0.50]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retl # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_fldcw_fldenv:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [8:0.50]
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    fldcw (%eax) # sched: [100:0.25]
-; ZNVER1-NEXT:    fldenv (%eax) # sched: [100:0.25]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retl # sched: [1:0.50]
-  tail call void asm sideeffect "fldcw $0 \0A\09 fldenv $0", "*m"(i8* %a0) nounwind
-  ret void
-}
-
-define void @test_fld1_fldl2e_fldl2t_fldlg2_fldln2_fldpi_fldz() optsize {
-; GENERIC-LABEL: test_fld1_fldl2e_fldl2t_fldlg2_fldln2_fldpi_fldz:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    fld1
-; GENERIC-NEXT:    fldl2e
-; GENERIC-NEXT:    fldl2t
-; GENERIC-NEXT:    fldlg2
-; GENERIC-NEXT:    fldln2
-; GENERIC-NEXT:    fldpi
-; GENERIC-NEXT:    fldz
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retl
-;
-; ATOM-LABEL: test_fld1_fldl2e_fldl2t_fldlg2_fldln2_fldpi_fldz:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    fld1 # sched: [6:3.00]
-; ATOM-NEXT:    fldl2e # sched: [10:5.00]
-; ATOM-NEXT:    fldl2t # sched: [10:5.00]
-; ATOM-NEXT:    fldlg2 # sched: [10:5.00]
-; ATOM-NEXT:    fldln2 # sched: [10:5.00]
-; ATOM-NEXT:    fldpi # sched: [10:5.00]
-; ATOM-NEXT:    fldz # sched: [1:0.50]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retl # sched: [79:39.50]
-;
-; SLM-LABEL: test_fld1_fldl2e_fldl2t_fldlg2_fldln2_fldpi_fldz:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    fld1 # sched: [1:0.50]
-; SLM-NEXT:    fldl2e # sched: [1:1.00]
-; SLM-NEXT:    fldl2t # sched: [1:1.00]
-; SLM-NEXT:    fldlg2 # sched: [1:1.00]
-; SLM-NEXT:    fldln2 # sched: [1:1.00]
-; SLM-NEXT:    fldpi # sched: [1:1.00]
-; SLM-NEXT:    fldz # sched: [1:0.50]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retl # sched: [4:1.00]
-;
-; SANDY-LABEL: test_fld1_fldl2e_fldl2t_fldlg2_fldln2_fldpi_fldz:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    fld1 # sched: [1:1.00]
-; SANDY-NEXT:    fldl2e # sched: [1:1.00]
-; SANDY-NEXT:    fldl2t # sched: [1:1.00]
-; SANDY-NEXT:    fldlg2 # sched: [1:1.00]
-; SANDY-NEXT:    fldln2 # sched: [1:1.00]
-; SANDY-NEXT:    fldpi # sched: [1:1.00]
-; SANDY-NEXT:    fldz # sched: [1:1.00]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retl # sched: [6:1.00]
-;
-; HASWELL-LABEL: test_fld1_fldl2e_fldl2t_fldlg2_fldln2_fldpi_fldz:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    fld1 # sched: [1:1.00]
-; HASWELL-NEXT:    fldl2e # sched: [1:1.00]
-; HASWELL-NEXT:    fldl2t # sched: [1:1.00]
-; HASWELL-NEXT:    fldlg2 # sched: [1:1.00]
-; HASWELL-NEXT:    fldln2 # sched: [1:1.00]
-; HASWELL-NEXT:    fldpi # sched: [1:1.00]
-; HASWELL-NEXT:    fldz # sched: [1:0.50]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retl # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_fld1_fldl2e_fldl2t_fldlg2_fldln2_fldpi_fldz:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    fld1 # sched: [1:1.00]
-; BROADWELL-NEXT:    fldl2e # sched: [1:1.00]
-; BROADWELL-NEXT:    fldl2t # sched: [1:1.00]
-; BROADWELL-NEXT:    fldlg2 # sched: [1:1.00]
-; BROADWELL-NEXT:    fldln2 # sched: [1:1.00]
-; BROADWELL-NEXT:    fldpi # sched: [1:1.00]
-; BROADWELL-NEXT:    fldz # sched: [1:0.50]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retl # sched: [6:0.50]
-;
-; SKYLAKE-LABEL: test_fld1_fldl2e_fldl2t_fldlg2_fldln2_fldpi_fldz:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    fld1 # sched: [1:1.00]
-; SKYLAKE-NEXT:    fldl2e # sched: [1:1.00]
-; SKYLAKE-NEXT:    fldl2t # sched: [1:1.00]
-; SKYLAKE-NEXT:    fldlg2 # sched: [1:1.00]
-; SKYLAKE-NEXT:    fldln2 # sched: [1:1.00]
-; SKYLAKE-NEXT:    fldpi # sched: [1:1.00]
-; SKYLAKE-NEXT:    fldz # sched: [1:0.50]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retl # sched: [6:0.50]
-;
-; SKX-LABEL: test_fld1_fldl2e_fldl2t_fldlg2_fldln2_fldpi_fldz:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    fld1 # sched: [1:1.00]
-; SKX-NEXT:    fldl2e # sched: [1:1.00]
-; SKX-NEXT:    fldl2t # sched: [1:1.00]
-; SKX-NEXT:    fldlg2 # sched: [1:1.00]
-; SKX-NEXT:    fldln2 # sched: [1:1.00]
-; SKX-NEXT:    fldpi # sched: [1:1.00]
-; SKX-NEXT:    fldz # sched: [1:0.50]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retl # sched: [6:0.50]
-;
-; BDVER2-LABEL: test_fld1_fldl2e_fldl2t_fldlg2_fldln2_fldpi_fldz:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    fld1 # sched: [3:1.00]
-; BDVER2-NEXT:    fldl2e # sched: [3:1.00]
-; BDVER2-NEXT:    fldl2t # sched: [3:1.00]
-; BDVER2-NEXT:    fldlg2 # sched: [3:1.00]
-; BDVER2-NEXT:    fldln2 # sched: [3:1.00]
-; BDVER2-NEXT:    fldpi # sched: [3:1.00]
-; BDVER2-NEXT:    fldz # sched: [3:1.00]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retl # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_fld1_fldl2e_fldl2t_fldlg2_fldln2_fldpi_fldz:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    fld1 # sched: [3:1.00]
-; BTVER2-NEXT:    fldl2e # sched: [3:1.00]
-; BTVER2-NEXT:    fldl2t # sched: [3:1.00]
-; BTVER2-NEXT:    fldlg2 # sched: [3:1.00]
-; BTVER2-NEXT:    fldln2 # sched: [3:1.00]
-; BTVER2-NEXT:    fldpi # sched: [3:1.00]
-; BTVER2-NEXT:    fldz # sched: [3:1.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retl # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_fld1_fldl2e_fldl2t_fldlg2_fldln2_fldpi_fldz:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    fld1 # sched: [11:1.00]
-; ZNVER1-NEXT:    fldl2e # sched: [11:1.00]
-; ZNVER1-NEXT:    fldl2t # sched: [11:1.00]
-; ZNVER1-NEXT:    fldlg2 # sched: [11:1.00]
-; ZNVER1-NEXT:    fldln2 # sched: [11:1.00]
-; ZNVER1-NEXT:    fldpi # sched: [11:1.00]
-; ZNVER1-NEXT:    fldz # sched: [8:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retl # sched: [1:0.50]
-  tail call void asm sideeffect "fld1 \0A\09 fldl2e \0A\09 fldl2t \0A\09 fldlg2 \0A\09 fldln2 \0A\09 fldpi \0A\09 fldz", ""() nounwind
-  ret void
-}
-
-define void @test_fmul(float *%a0, double *%a1) optsize {
-; GENERIC-LABEL: test_fmul:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; GENERIC-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    fmul %st(0), %st(1)
-; GENERIC-NEXT:    fmul %st(2)
-; GENERIC-NEXT:    fmuls (%ecx)
-; GENERIC-NEXT:    fmull (%eax)
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retl
-;
-; ATOM-LABEL: test_fmul:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [1:1.00]
-; ATOM-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [1:1.00]
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    fmul %st(0), %st(1) # sched: [4:4.00]
-; ATOM-NEXT:    fmul %st(2) # sched: [4:4.00]
-; ATOM-NEXT:    fmuls (%ecx) # sched: [4:4.00]
-; ATOM-NEXT:    fmull (%eax) # sched: [4:4.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retl # sched: [79:39.50]
-;
-; SLM-LABEL: test_fmul:
-; SLM:       # %bb.0:
-; SLM-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [3:1.00]
-; SLM-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [3:1.00]
-; SLM-NEXT:    #APP
-; SLM-NEXT:    fmul %st(0), %st(1) # sched: [5:2.00]
-; SLM-NEXT:    fmul %st(2) # sched: [5:2.00]
-; SLM-NEXT:    fmuls (%ecx) # sched: [8:2.00]
-; SLM-NEXT:    fmull (%eax) # sched: [8:2.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retl # sched: [4:1.00]
-;
-; SANDY-LABEL: test_fmul:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; SANDY-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    fmul %st(0), %st(1) # sched: [5:1.00]
-; SANDY-NEXT:    fmul %st(2) # sched: [5:1.00]
-; SANDY-NEXT:    fmuls (%ecx) # sched: [12:1.00]
-; SANDY-NEXT:    fmull (%eax) # sched: [12:1.00]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retl # sched: [6:1.00]
-;
-; HASWELL-LABEL: test_fmul:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; HASWELL-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    fmul %st(0), %st(1) # sched: [5:1.00]
-; HASWELL-NEXT:    fmul %st(2) # sched: [5:1.00]
-; HASWELL-NEXT:    fmuls (%ecx) # sched: [12:1.00]
-; HASWELL-NEXT:    fmull (%eax) # sched: [12:1.00]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retl # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_fmul:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; BROADWELL-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    fmul %st(0), %st(1) # sched: [5:1.00]
-; BROADWELL-NEXT:    fmul %st(2) # sched: [5:1.00]
-; BROADWELL-NEXT:    fmuls (%ecx) # sched: [11:1.00]
-; BROADWELL-NEXT:    fmull (%eax) # sched: [11:1.00]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retl # sched: [6:0.50]
-;
-; SKYLAKE-LABEL: test_fmul:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; SKYLAKE-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    fmul %st(0), %st(1) # sched: [4:1.00]
-; SKYLAKE-NEXT:    fmul %st(2) # sched: [4:1.00]
-; SKYLAKE-NEXT:    fmuls (%ecx) # sched: [11:1.00]
-; SKYLAKE-NEXT:    fmull (%eax) # sched: [11:1.00]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retl # sched: [6:0.50]
-;
-; SKX-LABEL: test_fmul:
-; SKX:       # %bb.0:
-; SKX-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; SKX-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; SKX-NEXT:    #APP
-; SKX-NEXT:    fmul %st(0), %st(1) # sched: [4:1.00]
-; SKX-NEXT:    fmul %st(2) # sched: [4:1.00]
-; SKX-NEXT:    fmuls (%ecx) # sched: [11:1.00]
-; SKX-NEXT:    fmull (%eax) # sched: [11:1.00]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retl # sched: [6:0.50]
-;
-; BDVER2-LABEL: test_fmul:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; BDVER2-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    fmul %st(0), %st(1) # sched: [5:1.00]
-; BDVER2-NEXT:    fmul %st(2) # sched: [5:1.00]
-; BDVER2-NEXT:    fmuls (%ecx) # sched: [10:1.00]
-; BDVER2-NEXT:    fmull (%eax) # sched: [10:1.00]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retl # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_fmul:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:1.00]
-; BTVER2-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:1.00]
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    fmul %st(0), %st(1) # sched: [2:1.00]
-; BTVER2-NEXT:    fmul %st(2) # sched: [2:1.00]
-; BTVER2-NEXT:    fmuls (%ecx) # sched: [7:1.00]
-; BTVER2-NEXT:    fmull (%eax) # sched: [7:1.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retl # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_fmul:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [8:0.50]
-; ZNVER1-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [8:0.50]
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    fmul %st(0), %st(1) # sched: [3:0.50]
-; ZNVER1-NEXT:    fmul %st(2) # sched: [3:0.50]
-; ZNVER1-NEXT:    fmuls (%ecx) # sched: [10:0.50]
-; ZNVER1-NEXT:    fmull (%eax) # sched: [10:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retl # sched: [1:0.50]
-  tail call void asm sideeffect "fmul %st(0), %st(1) \0A\09 fmul %st(2), %st(0) \0A\09 fmuls $0 \0A\09 fmull $1", "*m,*m"(float *%a0, double *%a1) nounwind
-  ret void
-}
-
-define void @test_fmulp_fimul(i16 *%a0, i32 *%a1) optsize {
-; GENERIC-LABEL: test_fmulp_fimul:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; GENERIC-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    fmulp %st(1)
-; GENERIC-NEXT:    fmulp %st(2)
-; GENERIC-NEXT:    fimuls (%ecx)
-; GENERIC-NEXT:    fimull (%eax)
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retl
-;
-; ATOM-LABEL: test_fmulp_fimul:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [1:1.00]
-; ATOM-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [1:1.00]
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    fmulp %st(1) # sched: [4:4.00]
-; ATOM-NEXT:    fmulp %st(2) # sched: [4:4.00]
-; ATOM-NEXT:    fimuls (%ecx) # sched: [4:4.00]
-; ATOM-NEXT:    fimull (%eax) # sched: [4:4.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retl # sched: [79:39.50]
-;
-; SLM-LABEL: test_fmulp_fimul:
-; SLM:       # %bb.0:
-; SLM-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [3:1.00]
-; SLM-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [3:1.00]
-; SLM-NEXT:    #APP
-; SLM-NEXT:    fmulp %st(1) # sched: [5:2.00]
-; SLM-NEXT:    fmulp %st(2) # sched: [5:2.00]
-; SLM-NEXT:    fimuls (%ecx) # sched: [8:2.00]
-; SLM-NEXT:    fimull (%eax) # sched: [8:2.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retl # sched: [4:1.00]
-;
-; SANDY-LABEL: test_fmulp_fimul:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; SANDY-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    fmulp %st(1) # sched: [5:1.00]
-; SANDY-NEXT:    fmulp %st(2) # sched: [5:1.00]
-; SANDY-NEXT:    fimuls (%ecx) # sched: [15:1.00]
-; SANDY-NEXT:    fimull (%eax) # sched: [15:1.00]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retl # sched: [6:1.00]
-;
-; HASWELL-LABEL: test_fmulp_fimul:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; HASWELL-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    fmulp %st(1) # sched: [5:1.00]
-; HASWELL-NEXT:    fmulp %st(2) # sched: [5:1.00]
-; HASWELL-NEXT:    fimuls (%ecx) # sched: [15:1.00]
-; HASWELL-NEXT:    fimull (%eax) # sched: [15:1.00]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retl # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_fmulp_fimul:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; BROADWELL-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    fmulp %st(1) # sched: [5:1.00]
-; BROADWELL-NEXT:    fmulp %st(2) # sched: [5:1.00]
-; BROADWELL-NEXT:    fimuls (%ecx) # sched: [14:1.00]
-; BROADWELL-NEXT:    fimull (%eax) # sched: [14:1.00]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retl # sched: [6:0.50]
-;
-; SKYLAKE-LABEL: test_fmulp_fimul:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; SKYLAKE-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    fmulp %st(1) # sched: [4:1.00]
-; SKYLAKE-NEXT:    fmulp %st(2) # sched: [4:1.00]
-; SKYLAKE-NEXT:    fimuls (%ecx) # sched: [14:1.00]
-; SKYLAKE-NEXT:    fimull (%eax) # sched: [14:1.00]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retl # sched: [6:0.50]
-;
-; SKX-LABEL: test_fmulp_fimul:
-; SKX:       # %bb.0:
-; SKX-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; SKX-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; SKX-NEXT:    #APP
-; SKX-NEXT:    fmulp %st(1) # sched: [4:1.00]
-; SKX-NEXT:    fmulp %st(2) # sched: [4:1.00]
-; SKX-NEXT:    fimuls (%ecx) # sched: [14:1.00]
-; SKX-NEXT:    fimull (%eax) # sched: [14:1.00]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retl # sched: [6:0.50]
-;
-; BDVER2-LABEL: test_fmulp_fimul:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; BDVER2-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    fmulp %st(1) # sched: [5:1.00]
-; BDVER2-NEXT:    fmulp %st(2) # sched: [5:1.00]
-; BDVER2-NEXT:    fimuls (%ecx) # sched: [10:1.00]
-; BDVER2-NEXT:    fimull (%eax) # sched: [10:1.00]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retl # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_fmulp_fimul:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:1.00]
-; BTVER2-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:1.00]
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    fmulp %st(1) # sched: [2:1.00]
-; BTVER2-NEXT:    fmulp %st(2) # sched: [2:1.00]
-; BTVER2-NEXT:    fimuls (%ecx) # sched: [7:1.00]
-; BTVER2-NEXT:    fimull (%eax) # sched: [7:1.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retl # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_fmulp_fimul:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [8:0.50]
-; ZNVER1-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [8:0.50]
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    fmulp %st(1) # sched: [3:0.50]
-; ZNVER1-NEXT:    fmulp %st(2) # sched: [3:0.50]
-; ZNVER1-NEXT:    fimuls (%ecx) # sched: [10:0.50]
-; ZNVER1-NEXT:    fimull (%eax) # sched: [10:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retl # sched: [1:0.50]
-  tail call void asm sideeffect "fmulp \0A\09 fmulp %st(2), %st(0) \0A\09 fimuls $0 \0A\09 fimull $1", "*m,*m"(i16 *%a0, i32 *%a1) nounwind
-  ret void
-}
-
-define void @test_fnop() optsize {
-; GENERIC-LABEL: test_fnop:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    fnop
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retl
-;
-; ATOM-LABEL: test_fnop:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    fnop # sched: [1:0.50]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retl # sched: [79:39.50]
-;
-; SLM-LABEL: test_fnop:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    fnop # sched: [1:0.50]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retl # sched: [4:1.00]
-;
-; SANDY-LABEL: test_fnop:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    fnop # sched: [1:1.00]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retl # sched: [6:1.00]
-;
-; HASWELL-LABEL: test_fnop:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    fnop # sched: [1:0.50]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retl # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_fnop:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    fnop # sched: [1:0.50]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retl # sched: [6:0.50]
-;
-; SKYLAKE-LABEL: test_fnop:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    fnop # sched: [1:0.50]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retl # sched: [6:0.50]
-;
-; SKX-LABEL: test_fnop:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    fnop # sched: [1:0.50]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retl # sched: [6:0.50]
-;
-; BDVER2-LABEL: test_fnop:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    fnop # sched: [1:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retl # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_fnop:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    fnop # sched: [1:0.50]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retl # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_fnop:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    fnop # sched: [1:1.00]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retl # sched: [1:0.50]
-  tail call void asm sideeffect "fnop", ""() nounwind
-  ret void
-}
-
-define void @test_fpatan() optsize {
-; GENERIC-LABEL: test_fpatan:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    fpatan
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retl
-;
-; ATOM-LABEL: test_fpatan:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    fpatan # sched: [183:91.50]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retl # sched: [79:39.50]
-;
-; SLM-LABEL: test_fpatan:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    fpatan # sched: [100:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retl # sched: [4:1.00]
-;
-; SANDY-LABEL: test_fpatan:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    fpatan # sched: [100:0.33]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retl # sched: [6:1.00]
-;
-; HASWELL-LABEL: test_fpatan:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    fpatan # sched: [100:0.25]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retl # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_fpatan:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    fpatan # sched: [100:0.25]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retl # sched: [6:0.50]
-;
-; SKYLAKE-LABEL: test_fpatan:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    fpatan # sched: [100:0.25]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retl # sched: [6:0.50]
-;
-; SKX-LABEL: test_fpatan:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    fpatan # sched: [100:0.25]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retl # sched: [6:0.50]
-;
-; BDVER2-LABEL: test_fpatan:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    fpatan # sched: [100:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retl # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_fpatan:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    fpatan # sched: [100:0.50]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retl # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_fpatan:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    fpatan # sched: [100:0.25]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retl # sched: [1:0.50]
-  tail call void asm sideeffect "fpatan", ""() nounwind
-  ret void
-}
-
-define void @test_fprem_fprem1() optsize {
-; GENERIC-LABEL: test_fprem_fprem1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    fprem
-; GENERIC-NEXT:    fprem1
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retl
-;
-; ATOM-LABEL: test_fprem_fprem1:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    fprem # sched: [55:27.50]
-; ATOM-NEXT:    fprem1 # sched: [71:35.50]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retl # sched: [79:39.50]
-;
-; SLM-LABEL: test_fprem_fprem1:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    fprem # sched: [100:1.00]
-; SLM-NEXT:    fprem1 # sched: [100:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retl # sched: [4:1.00]
-;
-; SANDY-LABEL: test_fprem_fprem1:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    fprem # sched: [100:0.33]
-; SANDY-NEXT:    fprem1 # sched: [100:0.33]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retl # sched: [6:1.00]
-;
-; HASWELL-LABEL: test_fprem_fprem1:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    fprem # sched: [19:7.00]
-; HASWELL-NEXT:    fprem1 # sched: [27:10.25]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retl # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_fprem_fprem1:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    fprem # sched: [100:0.25]
-; BROADWELL-NEXT:    fprem1 # sched: [100:0.25]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retl # sched: [6:0.50]
-;
-; SKYLAKE-LABEL: test_fprem_fprem1:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    fprem # sched: [100:0.25]
-; SKYLAKE-NEXT:    fprem1 # sched: [100:0.25]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retl # sched: [6:0.50]
-;
-; SKX-LABEL: test_fprem_fprem1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    fprem # sched: [100:0.25]
-; SKX-NEXT:    fprem1 # sched: [100:0.25]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retl # sched: [6:0.50]
-;
-; BDVER2-LABEL: test_fprem_fprem1:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    fprem # sched: [100:0.50]
-; BDVER2-NEXT:    fprem1 # sched: [100:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retl # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_fprem_fprem1:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    fprem # sched: [100:0.50]
-; BTVER2-NEXT:    fprem1 # sched: [100:0.50]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retl # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_fprem_fprem1:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    fprem # sched: [100:0.25]
-; ZNVER1-NEXT:    fprem1 # sched: [100:0.25]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retl # sched: [1:0.50]
-  tail call void asm sideeffect "fprem \0A\09 fprem1", ""() nounwind
-  ret void
-}
-
-define void @test_fptan() optsize {
-; GENERIC-LABEL: test_fptan:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    fptan
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retl
-;
-; ATOM-LABEL: test_fptan:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    fptan # sched: [168:84.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retl # sched: [79:39.50]
-;
-; SLM-LABEL: test_fptan:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    fptan # sched: [100:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retl # sched: [4:1.00]
-;
-; SANDY-LABEL: test_fptan:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    fptan # sched: [100:0.33]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retl # sched: [6:1.00]
-;
-; HASWELL-LABEL: test_fptan:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    fptan # sched: [100:0.25]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retl # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_fptan:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    fptan # sched: [100:0.25]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retl # sched: [6:0.50]
-;
-; SKYLAKE-LABEL: test_fptan:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    fptan # sched: [100:0.25]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retl # sched: [6:0.50]
-;
-; SKX-LABEL: test_fptan:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    fptan # sched: [100:0.25]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retl # sched: [6:0.50]
-;
-; BDVER2-LABEL: test_fptan:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    fptan # sched: [100:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retl # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_fptan:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    fptan # sched: [100:0.50]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retl # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_fptan:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    fptan # sched: [100:0.25]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retl # sched: [1:0.50]
-  tail call void asm sideeffect "fptan", ""() nounwind
-  ret void
-}
-
-define void @test_frndint() optsize {
-; GENERIC-LABEL: test_frndint:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    frndint
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retl
-;
-; ATOM-LABEL: test_frndint:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    frndint # sched: [46:23.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retl # sched: [79:39.50]
-;
-; SLM-LABEL: test_frndint:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    frndint # sched: [100:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retl # sched: [4:1.00]
-;
-; SANDY-LABEL: test_frndint:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    frndint # sched: [100:0.33]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retl # sched: [6:1.00]
-;
-; HASWELL-LABEL: test_frndint:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    frndint # sched: [11:4.25]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retl # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_frndint:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    frndint # sched: [100:0.25]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retl # sched: [6:0.50]
-;
-; SKYLAKE-LABEL: test_frndint:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    frndint # sched: [100:0.25]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retl # sched: [6:0.50]
-;
-; SKX-LABEL: test_frndint:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    frndint # sched: [100:0.25]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retl # sched: [6:0.50]
-;
-; BDVER2-LABEL: test_frndint:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    frndint # sched: [100:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retl # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_frndint:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    frndint # sched: [100:0.50]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retl # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_frndint:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    frndint # sched: [100:0.25]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retl # sched: [1:0.50]
-  tail call void asm sideeffect "frndint", ""() nounwind
-  ret void
-}
-
-define void @test_frstor(i8* %a0) optsize {
-; GENERIC-LABEL: test_frstor:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    frstor (%eax)
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retl
-;
-; ATOM-LABEL: test_frstor:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [1:1.00]
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    frstor (%eax) # sched: [100:0.50]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retl # sched: [79:39.50]
-;
-; SLM-LABEL: test_frstor:
-; SLM:       # %bb.0:
-; SLM-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [3:1.00]
-; SLM-NEXT:    #APP
-; SLM-NEXT:    frstor (%eax) # sched: [100:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retl # sched: [4:1.00]
-;
-; SANDY-LABEL: test_frstor:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    frstor (%eax) # sched: [100:0.33]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retl # sched: [6:1.00]
-;
-; HASWELL-LABEL: test_frstor:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    frstor (%eax) # sched: [1:22.50]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retl # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_frstor:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    frstor (%eax) # sched: [100:0.25]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retl # sched: [6:0.50]
-;
-; SKYLAKE-LABEL: test_frstor:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    frstor (%eax) # sched: [100:0.25]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retl # sched: [6:0.50]
-;
-; SKX-LABEL: test_frstor:
-; SKX:       # %bb.0:
-; SKX-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; SKX-NEXT:    #APP
-; SKX-NEXT:    frstor (%eax) # sched: [100:0.25]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retl # sched: [6:0.50]
-;
-; BDVER2-LABEL: test_frstor:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    frstor (%eax) # sched: [100:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retl # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_frstor:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:1.00]
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    frstor (%eax) # sched: [100:0.50]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retl # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_frstor:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [8:0.50]
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    frstor (%eax) # sched: [100:0.25]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retl # sched: [1:0.50]
-  tail call void asm sideeffect "frstor $0", "*m"(i8* %a0) nounwind
-  ret void
-}
-
-define void @test_fsave(i8* %a0) optsize {
-; GENERIC-LABEL: test_fsave:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    wait
-; GENERIC-NEXT:    fnsave (%eax)
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retl
-;
-; ATOM-LABEL: test_fsave:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [1:1.00]
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    wait # sched: [1:0.50]
-; ATOM-NEXT:    fnsave (%eax) # sched: [100:0.50]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retl # sched: [79:39.50]
-;
-; SLM-LABEL: test_fsave:
-; SLM:       # %bb.0:
-; SLM-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [3:1.00]
-; SLM-NEXT:    #APP
-; SLM-NEXT:    wait # sched: [100:1.00]
-; SLM-NEXT:    fnsave (%eax) # sched: [100:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retl # sched: [4:1.00]
-;
-; SANDY-LABEL: test_fsave:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    wait # sched: [100:0.33]
-; SANDY-NEXT:    fnsave (%eax) # sched: [100:0.33]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retl # sched: [6:1.00]
-;
-; HASWELL-LABEL: test_fsave:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    wait # sched: [2:0.50]
-; HASWELL-NEXT:    fnsave (%eax) # sched: [1:36.75]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retl # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_fsave:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    wait # sched: [2:0.50]
-; BROADWELL-NEXT:    fnsave (%eax) # sched: [100:0.25]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retl # sched: [6:0.50]
-;
-; SKYLAKE-LABEL: test_fsave:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    wait # sched: [2:0.50]
-; SKYLAKE-NEXT:    fnsave (%eax) # sched: [100:0.25]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retl # sched: [6:0.50]
-;
-; SKX-LABEL: test_fsave:
-; SKX:       # %bb.0:
-; SKX-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; SKX-NEXT:    #APP
-; SKX-NEXT:    wait # sched: [2:0.50]
-; SKX-NEXT:    fnsave (%eax) # sched: [100:0.25]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retl # sched: [6:0.50]
-;
-; BDVER2-LABEL: test_fsave:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    wait # sched: [100:0.50]
-; BDVER2-NEXT:    fnsave (%eax) # sched: [100:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retl # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_fsave:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:1.00]
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    wait # sched: [100:0.50]
-; BTVER2-NEXT:    fnsave (%eax) # sched: [100:0.50]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retl # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_fsave:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [8:0.50]
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    wait # sched: [1:1.00]
-; ZNVER1-NEXT:    fnsave (%eax) # sched: [100:0.25]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retl # sched: [1:0.50]
-  tail call void asm sideeffect "fsave $0", "*m"(i8* %a0) nounwind
-  ret void
-}
-
-define void @test_fnsave(i8* %a0) optsize {
-; GENERIC-LABEL: test_fnsave:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    fnsave (%eax)
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retl
-;
-; ATOM-LABEL: test_fnsave:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [1:1.00]
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    fnsave (%eax) # sched: [100:0.50]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retl # sched: [79:39.50]
-;
-; SLM-LABEL: test_fnsave:
-; SLM:       # %bb.0:
-; SLM-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [3:1.00]
-; SLM-NEXT:    #APP
-; SLM-NEXT:    fnsave (%eax) # sched: [100:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retl # sched: [4:1.00]
-;
-; SANDY-LABEL: test_fnsave:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    fnsave (%eax) # sched: [100:0.33]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retl # sched: [6:1.00]
-;
-; HASWELL-LABEL: test_fnsave:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    fnsave (%eax) # sched: [1:36.75]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retl # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_fnsave:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    fnsave (%eax) # sched: [100:0.25]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retl # sched: [6:0.50]
-;
-; SKYLAKE-LABEL: test_fnsave:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    fnsave (%eax) # sched: [100:0.25]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retl # sched: [6:0.50]
-;
-; SKX-LABEL: test_fnsave:
-; SKX:       # %bb.0:
-; SKX-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; SKX-NEXT:    #APP
-; SKX-NEXT:    fnsave (%eax) # sched: [100:0.25]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retl # sched: [6:0.50]
-;
-; BDVER2-LABEL: test_fnsave:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    fnsave (%eax) # sched: [100:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retl # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_fnsave:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:1.00]
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    fnsave (%eax) # sched: [100:0.50]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retl # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_fnsave:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [8:0.50]
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    fnsave (%eax) # sched: [100:0.25]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retl # sched: [1:0.50]
-  tail call void asm sideeffect "fnsave $0", "*m"(i8* %a0) nounwind
-  ret void
-}
-
-define void @test_fscale() optsize {
-; GENERIC-LABEL: test_fscale:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    fscale
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retl
-;
-; ATOM-LABEL: test_fscale:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    fscale # sched: [77:38.50]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retl # sched: [79:39.50]
-;
-; SLM-LABEL: test_fscale:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    fscale # sched: [100:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retl # sched: [4:1.00]
-;
-; SANDY-LABEL: test_fscale:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    fscale # sched: [100:0.33]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retl # sched: [6:1.00]
-;
-; HASWELL-LABEL: test_fscale:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    fscale # sched: [75:12.50]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retl # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_fscale:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    fscale # sched: [100:0.25]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retl # sched: [6:0.50]
-;
-; SKYLAKE-LABEL: test_fscale:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    fscale # sched: [100:0.25]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retl # sched: [6:0.50]
-;
-; SKX-LABEL: test_fscale:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    fscale # sched: [100:0.25]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retl # sched: [6:0.50]
-;
-; BDVER2-LABEL: test_fscale:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    fscale # sched: [100:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retl # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_fscale:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    fscale # sched: [100:0.50]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retl # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_fscale:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    fscale # sched: [100:0.25]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retl # sched: [1:0.50]
-  tail call void asm sideeffect "fscale", ""() nounwind
-  ret void
-}
-
-define void @test_fsin() optsize {
-; GENERIC-LABEL: test_fsin:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    fsin
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retl
-;
-; ATOM-LABEL: test_fsin:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    fsin # sched: [174:87.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retl # sched: [79:39.50]
-;
-; SLM-LABEL: test_fsin:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    fsin # sched: [100:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retl # sched: [4:1.00]
-;
-; SANDY-LABEL: test_fsin:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    fsin # sched: [100:0.33]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retl # sched: [6:1.00]
-;
-; HASWELL-LABEL: test_fsin:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    fsin # sched: [100:0.25]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retl # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_fsin:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    fsin # sched: [100:0.25]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retl # sched: [6:0.50]
-;
-; SKYLAKE-LABEL: test_fsin:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    fsin # sched: [100:0.25]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retl # sched: [6:0.50]
-;
-; SKX-LABEL: test_fsin:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    fsin # sched: [100:0.25]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retl # sched: [6:0.50]
-;
-; BDVER2-LABEL: test_fsin:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    fsin # sched: [100:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retl # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_fsin:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    fsin # sched: [100:0.50]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retl # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_fsin:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    fsin # sched: [100:0.25]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retl # sched: [1:0.50]
-  tail call void asm sideeffect "fsin", ""() nounwind
-  ret void
-}
-
-define void @test_fsincos() optsize {
-; GENERIC-LABEL: test_fsincos:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    fsincos
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retl
-;
-; ATOM-LABEL: test_fsincos:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    fsincos # sched: [174:87.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retl # sched: [79:39.50]
-;
-; SLM-LABEL: test_fsincos:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    fsincos # sched: [100:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retl # sched: [4:1.00]
-;
-; SANDY-LABEL: test_fsincos:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    fsincos # sched: [100:0.33]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retl # sched: [6:1.00]
-;
-; HASWELL-LABEL: test_fsincos:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    fsincos # sched: [100:0.25]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retl # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_fsincos:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    fsincos # sched: [100:0.25]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retl # sched: [6:0.50]
-;
-; SKYLAKE-LABEL: test_fsincos:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    fsincos # sched: [100:0.25]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retl # sched: [6:0.50]
-;
-; SKX-LABEL: test_fsincos:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    fsincos # sched: [100:0.25]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retl # sched: [6:0.50]
-;
-; BDVER2-LABEL: test_fsincos:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    fsincos # sched: [100:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retl # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_fsincos:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    fsincos # sched: [100:0.50]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retl # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_fsincos:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    fsincos # sched: [100:0.25]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retl # sched: [1:0.50]
-  tail call void asm sideeffect "fsincos", ""() nounwind
-  ret void
-}
-
-define void @test_fsqrt() optsize {
-; GENERIC-LABEL: test_fsqrt:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    fsqrt
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retl
-;
-; ATOM-LABEL: test_fsqrt:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    fsqrt # sched: [71:35.50]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retl # sched: [79:39.50]
-;
-; SLM-LABEL: test_fsqrt:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    fsqrt # sched: [40:40.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retl # sched: [4:1.00]
-;
-; SANDY-LABEL: test_fsqrt:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    fsqrt # sched: [24:24.00]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retl # sched: [6:1.00]
-;
-; HASWELL-LABEL: test_fsqrt:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    fsqrt # sched: [23:17.00]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retl # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_fsqrt:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    fsqrt # sched: [23:9.00]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retl # sched: [6:0.50]
-;
-; SKYLAKE-LABEL: test_fsqrt:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    fsqrt # sched: [21:7.00]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retl # sched: [6:0.50]
-;
-; SKX-LABEL: test_fsqrt:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    fsqrt # sched: [21:7.00]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retl # sched: [6:0.50]
-;
-; BDVER2-LABEL: test_fsqrt:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    fsqrt # sched: [1:17.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retl # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_fsqrt:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    fsqrt # sched: [35:35.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retl # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_fsqrt:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    fsqrt # sched: [20:20.00]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retl # sched: [1:0.50]
-  tail call void asm sideeffect "fsqrt", ""() nounwind
-  ret void
-}
-
-define void @test_fst_fstp(i16* %a0, i32* %a1, i64 *%a2) optsize {
-; GENERIC-LABEL: test_fst_fstp:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; GENERIC-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; GENERIC-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    fst %st(0)
-; GENERIC-NEXT:    fsts (%edx)
-; GENERIC-NEXT:    fstl (%ecx)
-; GENERIC-NEXT:    fstp %st(0)
-; GENERIC-NEXT:    fstpl (%edx)
-; GENERIC-NEXT:    fstpl (%ecx)
-; GENERIC-NEXT:    fstpt (%eax)
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retl
-;
-; ATOM-LABEL: test_fst_fstp:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [1:1.00]
-; ATOM-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [1:1.00]
-; ATOM-NEXT:    movl {{[0-9]+}}(%esp), %edx # sched: [1:1.00]
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    fst %st(0) # sched: [2:1.00]
-; ATOM-NEXT:    fsts (%edx) # sched: [2:1.00]
-; ATOM-NEXT:    fstl (%ecx) # sched: [2:1.00]
-; ATOM-NEXT:    fstp %st(0) # sched: [2:1.00]
-; ATOM-NEXT:    fstpl (%edx) # sched: [2:1.00]
-; ATOM-NEXT:    fstpl (%ecx) # sched: [2:1.00]
-; ATOM-NEXT:    fstpt (%eax) # sched: [5:2.50]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retl # sched: [79:39.50]
-;
-; SLM-LABEL: test_fst_fstp:
-; SLM:       # %bb.0:
-; SLM-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [3:1.00]
-; SLM-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [3:1.00]
-; SLM-NEXT:    movl {{[0-9]+}}(%esp), %edx # sched: [3:1.00]
-; SLM-NEXT:    #APP
-; SLM-NEXT:    fst %st(0) # sched: [1:0.50]
-; SLM-NEXT:    fsts (%edx) # sched: [1:1.00]
-; SLM-NEXT:    fstl (%ecx) # sched: [1:1.00]
-; SLM-NEXT:    fstp %st(0) # sched: [1:0.50]
-; SLM-NEXT:    fstpl (%edx) # sched: [1:1.00]
-; SLM-NEXT:    fstpl (%ecx) # sched: [1:1.00]
-; SLM-NEXT:    fstpt (%eax) # sched: [1:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retl # sched: [4:1.00]
-;
-; SANDY-LABEL: test_fst_fstp:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; SANDY-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; SANDY-NEXT:    movl {{[0-9]+}}(%esp), %edx # sched: [5:0.50]
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    fst %st(0) # sched: [1:1.00]
-; SANDY-NEXT:    fsts (%edx) # sched: [6:1.00]
-; SANDY-NEXT:    fstl (%ecx) # sched: [6:1.00]
-; SANDY-NEXT:    fstp %st(0) # sched: [1:1.00]
-; SANDY-NEXT:    fstpl (%edx) # sched: [6:1.00]
-; SANDY-NEXT:    fstpl (%ecx) # sched: [6:1.00]
-; SANDY-NEXT:    fstpt (%eax) # sched: [6:1.00]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retl # sched: [6:1.00]
-;
-; HASWELL-LABEL: test_fst_fstp:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; HASWELL-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; HASWELL-NEXT:    movl {{[0-9]+}}(%esp), %edx # sched: [5:0.50]
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    fst %st(0) # sched: [1:0.50]
-; HASWELL-NEXT:    fsts (%edx) # sched: [1:1.00]
-; HASWELL-NEXT:    fstl (%ecx) # sched: [1:1.00]
-; HASWELL-NEXT:    fstp %st(0) # sched: [1:0.50]
-; HASWELL-NEXT:    fstpl (%edx) # sched: [1:1.00]
-; HASWELL-NEXT:    fstpl (%ecx) # sched: [1:1.00]
-; HASWELL-NEXT:    fstpt (%eax) # sched: [1:1.00]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retl # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_fst_fstp:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; BROADWELL-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; BROADWELL-NEXT:    movl {{[0-9]+}}(%esp), %edx # sched: [5:0.50]
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    fst %st(0) # sched: [1:0.25]
-; BROADWELL-NEXT:    fsts (%edx) # sched: [1:1.00]
-; BROADWELL-NEXT:    fstl (%ecx) # sched: [1:1.00]
-; BROADWELL-NEXT:    fstp %st(0) # sched: [1:0.25]
-; BROADWELL-NEXT:    fstpl (%edx) # sched: [1:1.00]
-; BROADWELL-NEXT:    fstpl (%ecx) # sched: [1:1.00]
-; BROADWELL-NEXT:    fstpt (%eax) # sched: [1:1.00]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retl # sched: [6:0.50]
-;
-; SKYLAKE-LABEL: test_fst_fstp:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; SKYLAKE-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; SKYLAKE-NEXT:    movl {{[0-9]+}}(%esp), %edx # sched: [5:0.50]
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    fst %st(0) # sched: [1:0.25]
-; SKYLAKE-NEXT:    fsts (%edx) # sched: [1:1.00]
-; SKYLAKE-NEXT:    fstl (%ecx) # sched: [1:1.00]
-; SKYLAKE-NEXT:    fstp %st(0) # sched: [1:0.25]
-; SKYLAKE-NEXT:    fstpl (%edx) # sched: [1:1.00]
-; SKYLAKE-NEXT:    fstpl (%ecx) # sched: [1:1.00]
-; SKYLAKE-NEXT:    fstpt (%eax) # sched: [1:1.00]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retl # sched: [6:0.50]
-;
-; SKX-LABEL: test_fst_fstp:
-; SKX:       # %bb.0:
-; SKX-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; SKX-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; SKX-NEXT:    movl {{[0-9]+}}(%esp), %edx # sched: [5:0.50]
-; SKX-NEXT:    #APP
-; SKX-NEXT:    fst %st(0) # sched: [1:0.25]
-; SKX-NEXT:    fsts (%edx) # sched: [1:1.00]
-; SKX-NEXT:    fstl (%ecx) # sched: [1:1.00]
-; SKX-NEXT:    fstp %st(0) # sched: [1:0.25]
-; SKX-NEXT:    fstpl (%edx) # sched: [1:1.00]
-; SKX-NEXT:    fstpl (%ecx) # sched: [1:1.00]
-; SKX-NEXT:    fstpt (%eax) # sched: [1:1.00]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retl # sched: [6:0.50]
-;
-; BDVER2-LABEL: test_fst_fstp:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; BDVER2-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; BDVER2-NEXT:    movl {{[0-9]+}}(%esp), %edx # sched: [5:0.50]
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    fst %st(0) # sched: [1:0.50]
-; BDVER2-NEXT:    fsts (%edx) # sched: [1:1.00]
-; BDVER2-NEXT:    fstl (%ecx) # sched: [1:1.00]
-; BDVER2-NEXT:    fstp %st(0) # sched: [1:0.50]
-; BDVER2-NEXT:    fstpl (%edx) # sched: [1:1.00]
-; BDVER2-NEXT:    fstpl (%ecx) # sched: [1:1.00]
-; BDVER2-NEXT:    fstpt (%eax) # sched: [1:1.00]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retl # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_fst_fstp:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:1.00]
-; BTVER2-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:1.00]
-; BTVER2-NEXT:    movl {{[0-9]+}}(%esp), %edx # sched: [5:1.00]
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    fst %st(0) # sched: [1:0.50]
-; BTVER2-NEXT:    fsts (%edx) # sched: [1:1.00]
-; BTVER2-NEXT:    fstl (%ecx) # sched: [1:1.00]
-; BTVER2-NEXT:    fstp %st(0) # sched: [1:0.50]
-; BTVER2-NEXT:    fstpl (%edx) # sched: [1:1.00]
-; BTVER2-NEXT:    fstpl (%ecx) # sched: [1:1.00]
-; BTVER2-NEXT:    fstpt (%eax) # sched: [1:1.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retl # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_fst_fstp:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [8:0.50]
-; ZNVER1-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [8:0.50]
-; ZNVER1-NEXT:    movl {{[0-9]+}}(%esp), %edx # sched: [8:0.50]
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    fst %st(0) # sched: [5:0.50]
-; ZNVER1-NEXT:    fsts (%edx) # sched: [1:0.50]
-; ZNVER1-NEXT:    fstl (%ecx) # sched: [1:0.50]
-; ZNVER1-NEXT:    fstp %st(0) # sched: [5:0.50]
-; ZNVER1-NEXT:    fstpl (%edx) # sched: [1:0.50]
-; ZNVER1-NEXT:    fstpl (%ecx) # sched: [1:0.50]
-; ZNVER1-NEXT:    fstpt (%eax) # sched: [5:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retl # sched: [1:0.50]
-  tail call void asm sideeffect "fst %st(0) \0A\09 fsts $0 \0A\09 fstl $1 \0A\09 fstp %st(0) \0A\09 fstpl $0 \0A\09 fstpl $1 \0A\09 fstpt $2", "*m,*m,*m"(i16* %a0, i32* %a1, i64 *%a2) nounwind
-  ret void
-}
-
-define void @test_fstcw_fstenv_fstsw(i8* %a0) optsize {
-; GENERIC-LABEL: test_fstcw_fstenv_fstsw:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    wait
-; GENERIC-NEXT:    fnstcw (%eax)
-; GENERIC-NEXT:    wait
-; GENERIC-NEXT:    fnstenv (%eax)
-; GENERIC-NEXT:    wait
-; GENERIC-NEXT:    fnstsw (%eax)
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retl
-;
-; ATOM-LABEL: test_fstcw_fstenv_fstsw:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [1:1.00]
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    wait # sched: [1:0.50]
-; ATOM-NEXT:    fnstcw (%eax) # sched: [8:4.00]
-; ATOM-NEXT:    wait # sched: [1:0.50]
-; ATOM-NEXT:    fnstenv (%eax) # sched: [100:0.50]
-; ATOM-NEXT:    wait # sched: [1:0.50]
-; ATOM-NEXT:    fnstsw (%eax) # sched: [100:0.50]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retl # sched: [79:39.50]
-;
-; SLM-LABEL: test_fstcw_fstenv_fstsw:
-; SLM:       # %bb.0:
-; SLM-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [3:1.00]
-; SLM-NEXT:    #APP
-; SLM-NEXT:    wait # sched: [100:1.00]
-; SLM-NEXT:    fnstcw (%eax) # sched: [1:0.50]
-; SLM-NEXT:    wait # sched: [100:1.00]
-; SLM-NEXT:    fnstenv (%eax) # sched: [100:1.00]
-; SLM-NEXT:    wait # sched: [100:1.00]
-; SLM-NEXT:    fnstsw (%eax) # sched: [100:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retl # sched: [4:1.00]
-;
-; SANDY-LABEL: test_fstcw_fstenv_fstsw:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    wait # sched: [100:0.33]
-; SANDY-NEXT:    fnstcw (%eax) # sched: [7:1.00]
-; SANDY-NEXT:    wait # sched: [100:0.33]
-; SANDY-NEXT:    fnstenv (%eax) # sched: [100:0.33]
-; SANDY-NEXT:    wait # sched: [100:0.33]
-; SANDY-NEXT:    fnstsw (%eax) # sched: [7:1.00]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retl # sched: [6:1.00]
-;
-; HASWELL-LABEL: test_fstcw_fstenv_fstsw:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    wait # sched: [2:0.50]
-; HASWELL-NEXT:    fnstcw (%eax) # sched: [2:1.00]
-; HASWELL-NEXT:    wait # sched: [2:0.50]
-; HASWELL-NEXT:    fnstenv (%eax) # sched: [115:19.50]
-; HASWELL-NEXT:    wait # sched: [2:0.50]
-; HASWELL-NEXT:    fnstsw (%eax) # sched: [4:1.00]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retl # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_fstcw_fstenv_fstsw:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    wait # sched: [2:0.50]
-; BROADWELL-NEXT:    fnstcw (%eax) # sched: [2:1.00]
-; BROADWELL-NEXT:    wait # sched: [2:0.50]
-; BROADWELL-NEXT:    fnstenv (%eax) # sched: [115:19.50]
-; BROADWELL-NEXT:    wait # sched: [2:0.50]
-; BROADWELL-NEXT:    fnstsw (%eax) # sched: [4:1.00]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retl # sched: [6:0.50]
-;
-; SKYLAKE-LABEL: test_fstcw_fstenv_fstsw:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    wait # sched: [2:0.50]
-; SKYLAKE-NEXT:    fnstcw (%eax) # sched: [2:1.00]
-; SKYLAKE-NEXT:    wait # sched: [2:0.50]
-; SKYLAKE-NEXT:    fnstenv (%eax) # sched: [106:19.50]
-; SKYLAKE-NEXT:    wait # sched: [2:0.50]
-; SKYLAKE-NEXT:    fnstsw (%eax) # sched: [3:1.00]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retl # sched: [6:0.50]
-;
-; SKX-LABEL: test_fstcw_fstenv_fstsw:
-; SKX:       # %bb.0:
-; SKX-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; SKX-NEXT:    #APP
-; SKX-NEXT:    wait # sched: [2:0.50]
-; SKX-NEXT:    fnstcw (%eax) # sched: [2:1.00]
-; SKX-NEXT:    wait # sched: [2:0.50]
-; SKX-NEXT:    fnstenv (%eax) # sched: [106:19.50]
-; SKX-NEXT:    wait # sched: [2:0.50]
-; SKX-NEXT:    fnstsw (%eax) # sched: [3:1.00]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retl # sched: [6:0.50]
-;
-; BDVER2-LABEL: test_fstcw_fstenv_fstsw:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    wait # sched: [100:0.50]
-; BDVER2-NEXT:    fnstcw (%eax) # sched: [1:0.50]
-; BDVER2-NEXT:    wait # sched: [100:0.50]
-; BDVER2-NEXT:    fnstenv (%eax) # sched: [100:0.50]
-; BDVER2-NEXT:    wait # sched: [100:0.50]
-; BDVER2-NEXT:    fnstsw (%eax) # sched: [100:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retl # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_fstcw_fstenv_fstsw:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:1.00]
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    wait # sched: [100:0.50]
-; BTVER2-NEXT:    fnstcw (%eax) # sched: [1:0.50]
-; BTVER2-NEXT:    wait # sched: [100:0.50]
-; BTVER2-NEXT:    fnstenv (%eax) # sched: [100:0.50]
-; BTVER2-NEXT:    wait # sched: [100:0.50]
-; BTVER2-NEXT:    fnstsw (%eax) # sched: [100:0.50]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retl # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_fstcw_fstenv_fstsw:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [8:0.50]
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    wait # sched: [1:1.00]
-; ZNVER1-NEXT:    fnstcw (%eax) # sched: [100:0.25]
-; ZNVER1-NEXT:    wait # sched: [1:1.00]
-; ZNVER1-NEXT:    fnstenv (%eax) # sched: [100:0.25]
-; ZNVER1-NEXT:    wait # sched: [1:1.00]
-; ZNVER1-NEXT:    fnstsw (%eax) # sched: [100:0.25]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retl # sched: [1:0.50]
-  tail call void asm sideeffect "fstcw $0 \0A\09 fstenv $0 \0A\09 fstsw $0", "*m"(i8* %a0) nounwind
-  ret void
-}
-
-define void @test_fnstcw_fnstenv_fnstsw(i8* %a0) optsize {
-; GENERIC-LABEL: test_fnstcw_fnstenv_fnstsw:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    fnstcw (%eax)
-; GENERIC-NEXT:    fnstenv (%eax)
-; GENERIC-NEXT:    fnstsw (%eax)
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retl
-;
-; ATOM-LABEL: test_fnstcw_fnstenv_fnstsw:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [1:1.00]
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    fnstcw (%eax) # sched: [8:4.00]
-; ATOM-NEXT:    fnstenv (%eax) # sched: [100:0.50]
-; ATOM-NEXT:    fnstsw (%eax) # sched: [100:0.50]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retl # sched: [79:39.50]
-;
-; SLM-LABEL: test_fnstcw_fnstenv_fnstsw:
-; SLM:       # %bb.0:
-; SLM-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [3:1.00]
-; SLM-NEXT:    #APP
-; SLM-NEXT:    fnstcw (%eax) # sched: [1:0.50]
-; SLM-NEXT:    fnstenv (%eax) # sched: [100:1.00]
-; SLM-NEXT:    fnstsw (%eax) # sched: [100:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retl # sched: [4:1.00]
-;
-; SANDY-LABEL: test_fnstcw_fnstenv_fnstsw:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    fnstcw (%eax) # sched: [7:1.00]
-; SANDY-NEXT:    fnstenv (%eax) # sched: [100:0.33]
-; SANDY-NEXT:    fnstsw (%eax) # sched: [7:1.00]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retl # sched: [6:1.00]
-;
-; HASWELL-LABEL: test_fnstcw_fnstenv_fnstsw:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    fnstcw (%eax) # sched: [2:1.00]
-; HASWELL-NEXT:    fnstenv (%eax) # sched: [115:19.50]
-; HASWELL-NEXT:    fnstsw (%eax) # sched: [4:1.00]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retl # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_fnstcw_fnstenv_fnstsw:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    fnstcw (%eax) # sched: [2:1.00]
-; BROADWELL-NEXT:    fnstenv (%eax) # sched: [115:19.50]
-; BROADWELL-NEXT:    fnstsw (%eax) # sched: [4:1.00]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retl # sched: [6:0.50]
-;
-; SKYLAKE-LABEL: test_fnstcw_fnstenv_fnstsw:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    fnstcw (%eax) # sched: [2:1.00]
-; SKYLAKE-NEXT:    fnstenv (%eax) # sched: [106:19.50]
-; SKYLAKE-NEXT:    fnstsw (%eax) # sched: [3:1.00]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retl # sched: [6:0.50]
-;
-; SKX-LABEL: test_fnstcw_fnstenv_fnstsw:
-; SKX:       # %bb.0:
-; SKX-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; SKX-NEXT:    #APP
-; SKX-NEXT:    fnstcw (%eax) # sched: [2:1.00]
-; SKX-NEXT:    fnstenv (%eax) # sched: [106:19.50]
-; SKX-NEXT:    fnstsw (%eax) # sched: [3:1.00]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retl # sched: [6:0.50]
-;
-; BDVER2-LABEL: test_fnstcw_fnstenv_fnstsw:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    fnstcw (%eax) # sched: [1:0.50]
-; BDVER2-NEXT:    fnstenv (%eax) # sched: [100:0.50]
-; BDVER2-NEXT:    fnstsw (%eax) # sched: [100:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retl # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_fnstcw_fnstenv_fnstsw:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:1.00]
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    fnstcw (%eax) # sched: [1:0.50]
-; BTVER2-NEXT:    fnstenv (%eax) # sched: [100:0.50]
-; BTVER2-NEXT:    fnstsw (%eax) # sched: [100:0.50]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retl # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_fnstcw_fnstenv_fnstsw:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [8:0.50]
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    fnstcw (%eax) # sched: [100:0.25]
-; ZNVER1-NEXT:    fnstenv (%eax) # sched: [100:0.25]
-; ZNVER1-NEXT:    fnstsw (%eax) # sched: [100:0.25]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retl # sched: [1:0.50]
-  tail call void asm sideeffect "fnstcw $0 \0A\09 fnstenv $0 \0A\09 fnstsw $0", "*m"(i8* %a0) nounwind
-  ret void
-}
-
-define void @test_fsub(float *%a0, double *%a1) optsize {
-; GENERIC-LABEL: test_fsub:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; GENERIC-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    fsub %st(0), %st(1)
-; GENERIC-NEXT:    fsub %st(2)
-; GENERIC-NEXT:    fsubs (%ecx)
-; GENERIC-NEXT:    fsubl (%eax)
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retl
-;
-; ATOM-LABEL: test_fsub:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [1:1.00]
-; ATOM-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [1:1.00]
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    fsub %st(0), %st(1) # sched: [5:5.00]
-; ATOM-NEXT:    fsub %st(2) # sched: [5:5.00]
-; ATOM-NEXT:    fsubs (%ecx) # sched: [5:5.00]
-; ATOM-NEXT:    fsubl (%eax) # sched: [5:5.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retl # sched: [79:39.50]
-;
-; SLM-LABEL: test_fsub:
-; SLM:       # %bb.0:
-; SLM-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [3:1.00]
-; SLM-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [3:1.00]
-; SLM-NEXT:    #APP
-; SLM-NEXT:    fsub %st(0), %st(1) # sched: [3:1.00]
-; SLM-NEXT:    fsub %st(2) # sched: [3:1.00]
-; SLM-NEXT:    fsubs (%ecx) # sched: [6:1.00]
-; SLM-NEXT:    fsubl (%eax) # sched: [6:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retl # sched: [4:1.00]
-;
-; SANDY-LABEL: test_fsub:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; SANDY-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    fsub %st(0), %st(1) # sched: [3:1.00]
-; SANDY-NEXT:    fsub %st(2) # sched: [3:1.00]
-; SANDY-NEXT:    fsubs (%ecx) # sched: [10:1.00]
-; SANDY-NEXT:    fsubl (%eax) # sched: [10:1.00]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retl # sched: [6:1.00]
-;
-; HASWELL-LABEL: test_fsub:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; HASWELL-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    fsub %st(0), %st(1) # sched: [3:1.00]
-; HASWELL-NEXT:    fsub %st(2) # sched: [3:1.00]
-; HASWELL-NEXT:    fsubs (%ecx) # sched: [10:1.00]
-; HASWELL-NEXT:    fsubl (%eax) # sched: [10:1.00]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retl # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_fsub:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; BROADWELL-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    fsub %st(0), %st(1) # sched: [3:1.00]
-; BROADWELL-NEXT:    fsub %st(2) # sched: [3:1.00]
-; BROADWELL-NEXT:    fsubs (%ecx) # sched: [9:1.00]
-; BROADWELL-NEXT:    fsubl (%eax) # sched: [9:1.00]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retl # sched: [6:0.50]
-;
-; SKYLAKE-LABEL: test_fsub:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; SKYLAKE-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    fsub %st(0), %st(1) # sched: [3:1.00]
-; SKYLAKE-NEXT:    fsub %st(2) # sched: [3:1.00]
-; SKYLAKE-NEXT:    fsubs (%ecx) # sched: [10:1.00]
-; SKYLAKE-NEXT:    fsubl (%eax) # sched: [10:1.00]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retl # sched: [6:0.50]
-;
-; SKX-LABEL: test_fsub:
-; SKX:       # %bb.0:
-; SKX-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; SKX-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; SKX-NEXT:    #APP
-; SKX-NEXT:    fsub %st(0), %st(1) # sched: [3:1.00]
-; SKX-NEXT:    fsub %st(2) # sched: [3:1.00]
-; SKX-NEXT:    fsubs (%ecx) # sched: [10:1.00]
-; SKX-NEXT:    fsubl (%eax) # sched: [10:1.00]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retl # sched: [6:0.50]
-;
-; BDVER2-LABEL: test_fsub:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; BDVER2-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    fsub %st(0), %st(1) # sched: [5:1.00]
-; BDVER2-NEXT:    fsub %st(2) # sched: [5:1.00]
-; BDVER2-NEXT:    fsubs (%ecx) # sched: [10:1.00]
-; BDVER2-NEXT:    fsubl (%eax) # sched: [10:1.00]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retl # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_fsub:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:1.00]
-; BTVER2-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:1.00]
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    fsub %st(0), %st(1) # sched: [3:1.00]
-; BTVER2-NEXT:    fsub %st(2) # sched: [3:1.00]
-; BTVER2-NEXT:    fsubs (%ecx) # sched: [8:1.00]
-; BTVER2-NEXT:    fsubl (%eax) # sched: [8:1.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retl # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_fsub:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [8:0.50]
-; ZNVER1-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [8:0.50]
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    fsub %st(0), %st(1) # sched: [3:1.00]
-; ZNVER1-NEXT:    fsub %st(2) # sched: [3:1.00]
-; ZNVER1-NEXT:    fsubs (%ecx) # sched: [10:1.00]
-; ZNVER1-NEXT:    fsubl (%eax) # sched: [10:1.00]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retl # sched: [1:0.50]
-  tail call void asm sideeffect "fsub %st(0), %st(1) \0A\09 fsub %st(2), %st(0) \0A\09 fsubs $0 \0A\09 fsubl $1", "*m,*m"(float *%a0, double *%a1) nounwind
-  ret void
-}
-
-define void @test_fsubp_fisub(i16 *%a0, i32 *%a1) optsize {
-; GENERIC-LABEL: test_fsubp_fisub:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; GENERIC-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    fsubp %st(1)
-; GENERIC-NEXT:    fsubp %st(2)
-; GENERIC-NEXT:    fisubs (%ecx)
-; GENERIC-NEXT:    fisubl (%eax)
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retl
-;
-; ATOM-LABEL: test_fsubp_fisub:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [1:1.00]
-; ATOM-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [1:1.00]
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    fsubp %st(1) # sched: [5:5.00]
-; ATOM-NEXT:    fsubp %st(2) # sched: [5:5.00]
-; ATOM-NEXT:    fisubs (%ecx) # sched: [5:5.00]
-; ATOM-NEXT:    fisubl (%eax) # sched: [5:5.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retl # sched: [79:39.50]
-;
-; SLM-LABEL: test_fsubp_fisub:
-; SLM:       # %bb.0:
-; SLM-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [3:1.00]
-; SLM-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [3:1.00]
-; SLM-NEXT:    #APP
-; SLM-NEXT:    fsubp %st(1) # sched: [3:1.00]
-; SLM-NEXT:    fsubp %st(2) # sched: [3:1.00]
-; SLM-NEXT:    fisubs (%ecx) # sched: [6:1.00]
-; SLM-NEXT:    fisubl (%eax) # sched: [6:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retl # sched: [4:1.00]
-;
-; SANDY-LABEL: test_fsubp_fisub:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; SANDY-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    fsubp %st(1) # sched: [3:1.00]
-; SANDY-NEXT:    fsubp %st(2) # sched: [3:1.00]
-; SANDY-NEXT:    fisubs (%ecx) # sched: [13:2.00]
-; SANDY-NEXT:    fisubl (%eax) # sched: [13:2.00]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retl # sched: [6:1.00]
-;
-; HASWELL-LABEL: test_fsubp_fisub:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; HASWELL-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    fsubp %st(1) # sched: [3:1.00]
-; HASWELL-NEXT:    fsubp %st(2) # sched: [3:1.00]
-; HASWELL-NEXT:    fisubs (%ecx) # sched: [13:2.00]
-; HASWELL-NEXT:    fisubl (%eax) # sched: [13:2.00]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retl # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_fsubp_fisub:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; BROADWELL-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    fsubp %st(1) # sched: [3:1.00]
-; BROADWELL-NEXT:    fsubp %st(2) # sched: [3:1.00]
-; BROADWELL-NEXT:    fisubs (%ecx) # sched: [12:2.00]
-; BROADWELL-NEXT:    fisubl (%eax) # sched: [12:2.00]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retl # sched: [6:0.50]
-;
-; SKYLAKE-LABEL: test_fsubp_fisub:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; SKYLAKE-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    fsubp %st(1) # sched: [3:1.00]
-; SKYLAKE-NEXT:    fsubp %st(2) # sched: [3:1.00]
-; SKYLAKE-NEXT:    fisubs (%ecx) # sched: [13:2.00]
-; SKYLAKE-NEXT:    fisubl (%eax) # sched: [13:2.00]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retl # sched: [6:0.50]
-;
-; SKX-LABEL: test_fsubp_fisub:
-; SKX:       # %bb.0:
-; SKX-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; SKX-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; SKX-NEXT:    #APP
-; SKX-NEXT:    fsubp %st(1) # sched: [3:1.00]
-; SKX-NEXT:    fsubp %st(2) # sched: [3:1.00]
-; SKX-NEXT:    fisubs (%ecx) # sched: [13:2.00]
-; SKX-NEXT:    fisubl (%eax) # sched: [13:2.00]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retl # sched: [6:0.50]
-;
-; BDVER2-LABEL: test_fsubp_fisub:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; BDVER2-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    fsubp %st(1) # sched: [5:1.00]
-; BDVER2-NEXT:    fsubp %st(2) # sched: [5:1.00]
-; BDVER2-NEXT:    fisubs (%ecx) # sched: [10:1.00]
-; BDVER2-NEXT:    fisubl (%eax) # sched: [10:1.00]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retl # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_fsubp_fisub:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:1.00]
-; BTVER2-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:1.00]
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    fsubp %st(1) # sched: [3:1.00]
-; BTVER2-NEXT:    fsubp %st(2) # sched: [3:1.00]
-; BTVER2-NEXT:    fisubs (%ecx) # sched: [8:1.00]
-; BTVER2-NEXT:    fisubl (%eax) # sched: [8:1.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retl # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_fsubp_fisub:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [8:0.50]
-; ZNVER1-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [8:0.50]
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    fsubp %st(1) # sched: [3:1.00]
-; ZNVER1-NEXT:    fsubp %st(2) # sched: [3:1.00]
-; ZNVER1-NEXT:    fisubs (%ecx) # sched: [10:1.00]
-; ZNVER1-NEXT:    fisubl (%eax) # sched: [10:1.00]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retl # sched: [1:0.50]
-  tail call void asm sideeffect "fsubp \0A\09 fsubp %st(2), %st(0) \0A\09 fisubs $0 \0A\09 fisubl $1", "*m,*m"(i16 *%a0, i32 *%a1) nounwind
-  ret void
-}
-
-define void @test_fsubr(float *%a0, double *%a1) optsize {
-; GENERIC-LABEL: test_fsubr:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; GENERIC-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    fsubr %st(0), %st(1)
-; GENERIC-NEXT:    fsubr %st(2)
-; GENERIC-NEXT:    fsubrs (%ecx)
-; GENERIC-NEXT:    fsubrl (%eax)
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retl
-;
-; ATOM-LABEL: test_fsubr:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [1:1.00]
-; ATOM-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [1:1.00]
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    fsubr %st(0), %st(1) # sched: [5:5.00]
-; ATOM-NEXT:    fsubr %st(2) # sched: [5:5.00]
-; ATOM-NEXT:    fsubrs (%ecx) # sched: [5:5.00]
-; ATOM-NEXT:    fsubrl (%eax) # sched: [5:5.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retl # sched: [79:39.50]
-;
-; SLM-LABEL: test_fsubr:
-; SLM:       # %bb.0:
-; SLM-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [3:1.00]
-; SLM-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [3:1.00]
-; SLM-NEXT:    #APP
-; SLM-NEXT:    fsubr %st(0), %st(1) # sched: [3:1.00]
-; SLM-NEXT:    fsubr %st(2) # sched: [3:1.00]
-; SLM-NEXT:    fsubrs (%ecx) # sched: [6:1.00]
-; SLM-NEXT:    fsubrl (%eax) # sched: [6:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retl # sched: [4:1.00]
-;
-; SANDY-LABEL: test_fsubr:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; SANDY-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    fsubr %st(0), %st(1) # sched: [3:1.00]
-; SANDY-NEXT:    fsubr %st(2) # sched: [3:1.00]
-; SANDY-NEXT:    fsubrs (%ecx) # sched: [10:1.00]
-; SANDY-NEXT:    fsubrl (%eax) # sched: [10:1.00]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retl # sched: [6:1.00]
-;
-; HASWELL-LABEL: test_fsubr:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; HASWELL-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    fsubr %st(0), %st(1) # sched: [3:1.00]
-; HASWELL-NEXT:    fsubr %st(2) # sched: [3:1.00]
-; HASWELL-NEXT:    fsubrs (%ecx) # sched: [10:1.00]
-; HASWELL-NEXT:    fsubrl (%eax) # sched: [10:1.00]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retl # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_fsubr:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; BROADWELL-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    fsubr %st(0), %st(1) # sched: [3:1.00]
-; BROADWELL-NEXT:    fsubr %st(2) # sched: [3:1.00]
-; BROADWELL-NEXT:    fsubrs (%ecx) # sched: [9:1.00]
-; BROADWELL-NEXT:    fsubrl (%eax) # sched: [9:1.00]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retl # sched: [6:0.50]
-;
-; SKYLAKE-LABEL: test_fsubr:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; SKYLAKE-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    fsubr %st(0), %st(1) # sched: [3:1.00]
-; SKYLAKE-NEXT:    fsubr %st(2) # sched: [3:1.00]
-; SKYLAKE-NEXT:    fsubrs (%ecx) # sched: [10:1.00]
-; SKYLAKE-NEXT:    fsubrl (%eax) # sched: [10:1.00]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retl # sched: [6:0.50]
-;
-; SKX-LABEL: test_fsubr:
-; SKX:       # %bb.0:
-; SKX-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; SKX-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; SKX-NEXT:    #APP
-; SKX-NEXT:    fsubr %st(0), %st(1) # sched: [3:1.00]
-; SKX-NEXT:    fsubr %st(2) # sched: [3:1.00]
-; SKX-NEXT:    fsubrs (%ecx) # sched: [10:1.00]
-; SKX-NEXT:    fsubrl (%eax) # sched: [10:1.00]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retl # sched: [6:0.50]
-;
-; BDVER2-LABEL: test_fsubr:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; BDVER2-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    fsubr %st(0), %st(1) # sched: [5:1.00]
-; BDVER2-NEXT:    fsubr %st(2) # sched: [5:1.00]
-; BDVER2-NEXT:    fsubrs (%ecx) # sched: [10:1.00]
-; BDVER2-NEXT:    fsubrl (%eax) # sched: [10:1.00]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retl # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_fsubr:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:1.00]
-; BTVER2-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:1.00]
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    fsubr %st(0), %st(1) # sched: [3:1.00]
-; BTVER2-NEXT:    fsubr %st(2) # sched: [3:1.00]
-; BTVER2-NEXT:    fsubrs (%ecx) # sched: [8:1.00]
-; BTVER2-NEXT:    fsubrl (%eax) # sched: [8:1.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retl # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_fsubr:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [8:0.50]
-; ZNVER1-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [8:0.50]
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    fsubr %st(0), %st(1) # sched: [3:1.00]
-; ZNVER1-NEXT:    fsubr %st(2) # sched: [3:1.00]
-; ZNVER1-NEXT:    fsubrs (%ecx) # sched: [10:1.00]
-; ZNVER1-NEXT:    fsubrl (%eax) # sched: [10:1.00]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retl # sched: [1:0.50]
-  tail call void asm sideeffect "fsubr %st(0), %st(1) \0A\09 fsubr %st(2), %st(0) \0A\09 fsubrs $0 \0A\09 fsubrl $1", "*m,*m"(float *%a0, double *%a1) nounwind
-  ret void
-}
-
-define void @test_fsubrp_fisubr(i16 *%a0, i32 *%a1) optsize {
-; GENERIC-LABEL: test_fsubrp_fisubr:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; GENERIC-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    fsubrp %st(1)
-; GENERIC-NEXT:    fsubrp %st(2)
-; GENERIC-NEXT:    fisubrs (%ecx)
-; GENERIC-NEXT:    fisubrl (%eax)
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retl
-;
-; ATOM-LABEL: test_fsubrp_fisubr:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [1:1.00]
-; ATOM-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [1:1.00]
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    fsubrp %st(1) # sched: [5:5.00]
-; ATOM-NEXT:    fsubrp %st(2) # sched: [5:5.00]
-; ATOM-NEXT:    fisubrs (%ecx) # sched: [5:5.00]
-; ATOM-NEXT:    fisubrl (%eax) # sched: [5:5.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retl # sched: [79:39.50]
-;
-; SLM-LABEL: test_fsubrp_fisubr:
-; SLM:       # %bb.0:
-; SLM-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [3:1.00]
-; SLM-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [3:1.00]
-; SLM-NEXT:    #APP
-; SLM-NEXT:    fsubrp %st(1) # sched: [3:1.00]
-; SLM-NEXT:    fsubrp %st(2) # sched: [3:1.00]
-; SLM-NEXT:    fisubrs (%ecx) # sched: [6:1.00]
-; SLM-NEXT:    fisubrl (%eax) # sched: [6:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retl # sched: [4:1.00]
-;
-; SANDY-LABEL: test_fsubrp_fisubr:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; SANDY-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    fsubrp %st(1) # sched: [3:1.00]
-; SANDY-NEXT:    fsubrp %st(2) # sched: [3:1.00]
-; SANDY-NEXT:    fisubrs (%ecx) # sched: [13:2.00]
-; SANDY-NEXT:    fisubrl (%eax) # sched: [13:2.00]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retl # sched: [6:1.00]
-;
-; HASWELL-LABEL: test_fsubrp_fisubr:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; HASWELL-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    fsubrp %st(1) # sched: [3:1.00]
-; HASWELL-NEXT:    fsubrp %st(2) # sched: [3:1.00]
-; HASWELL-NEXT:    fisubrs (%ecx) # sched: [13:2.00]
-; HASWELL-NEXT:    fisubrl (%eax) # sched: [13:2.00]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retl # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_fsubrp_fisubr:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; BROADWELL-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    fsubrp %st(1) # sched: [3:1.00]
-; BROADWELL-NEXT:    fsubrp %st(2) # sched: [3:1.00]
-; BROADWELL-NEXT:    fisubrs (%ecx) # sched: [12:2.00]
-; BROADWELL-NEXT:    fisubrl (%eax) # sched: [12:2.00]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retl # sched: [6:0.50]
-;
-; SKYLAKE-LABEL: test_fsubrp_fisubr:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; SKYLAKE-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    fsubrp %st(1) # sched: [3:1.00]
-; SKYLAKE-NEXT:    fsubrp %st(2) # sched: [3:1.00]
-; SKYLAKE-NEXT:    fisubrs (%ecx) # sched: [13:2.00]
-; SKYLAKE-NEXT:    fisubrl (%eax) # sched: [13:2.00]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retl # sched: [6:0.50]
-;
-; SKX-LABEL: test_fsubrp_fisubr:
-; SKX:       # %bb.0:
-; SKX-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; SKX-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; SKX-NEXT:    #APP
-; SKX-NEXT:    fsubrp %st(1) # sched: [3:1.00]
-; SKX-NEXT:    fsubrp %st(2) # sched: [3:1.00]
-; SKX-NEXT:    fisubrs (%ecx) # sched: [13:2.00]
-; SKX-NEXT:    fisubrl (%eax) # sched: [13:2.00]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retl # sched: [6:0.50]
-;
-; BDVER2-LABEL: test_fsubrp_fisubr:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; BDVER2-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:0.50]
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    fsubrp %st(1) # sched: [5:1.00]
-; BDVER2-NEXT:    fsubrp %st(2) # sched: [5:1.00]
-; BDVER2-NEXT:    fisubrs (%ecx) # sched: [10:1.00]
-; BDVER2-NEXT:    fisubrl (%eax) # sched: [10:1.00]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retl # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_fsubrp_fisubr:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:1.00]
-; BTVER2-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [5:1.00]
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    fsubrp %st(1) # sched: [3:1.00]
-; BTVER2-NEXT:    fsubrp %st(2) # sched: [3:1.00]
-; BTVER2-NEXT:    fisubrs (%ecx) # sched: [8:1.00]
-; BTVER2-NEXT:    fisubrl (%eax) # sched: [8:1.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retl # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_fsubrp_fisubr:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [8:0.50]
-; ZNVER1-NEXT:    movl {{[0-9]+}}(%esp), %ecx # sched: [8:0.50]
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    fsubrp %st(1) # sched: [3:1.00]
-; ZNVER1-NEXT:    fsubrp %st(2) # sched: [3:1.00]
-; ZNVER1-NEXT:    fisubrs (%ecx) # sched: [10:1.00]
-; ZNVER1-NEXT:    fisubrl (%eax) # sched: [10:1.00]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retl # sched: [1:0.50]
-  tail call void asm sideeffect "fsubrp \0A\09 fsubrp %st(2), %st(0) \0A\09 fisubrs $0 \0A\09 fisubrl $1", "*m,*m"(i16 *%a0, i32 *%a1) nounwind
-  ret void
-}
-
-define void @test_ftst() optsize {
-; GENERIC-LABEL: test_ftst:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    ftst
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retl
-;
-; ATOM-LABEL: test_ftst:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    ftst # sched: [9:4.50]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retl # sched: [79:39.50]
-;
-; SLM-LABEL: test_ftst:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    ftst # sched: [3:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retl # sched: [4:1.00]
-;
-; SANDY-LABEL: test_ftst:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    ftst # sched: [3:1.00]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retl # sched: [6:1.00]
-;
-; HASWELL-LABEL: test_ftst:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    ftst # sched: [1:1.00]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retl # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_ftst:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    ftst # sched: [3:1.00]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retl # sched: [6:0.50]
-;
-; SKYLAKE-LABEL: test_ftst:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    ftst # sched: [2:1.00]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retl # sched: [6:0.50]
-;
-; SKX-LABEL: test_ftst:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    ftst # sched: [2:1.00]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retl # sched: [6:0.50]
-;
-; BDVER2-LABEL: test_ftst:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    ftst # sched: [1:1.00]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retl # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_ftst:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    ftst # sched: [3:1.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retl # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_ftst:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    ftst # sched: [1:1.00]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retl # sched: [1:0.50]
-  tail call void asm sideeffect "ftst", ""() nounwind
-  ret void
-}
-
-define void @test_fucom_fucomp_fucompp() optsize {
-; GENERIC-LABEL: test_fucom_fucomp_fucompp:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    fucom %st(1)
-; GENERIC-NEXT:    fucom %st(3)
-; GENERIC-NEXT:    fucomp %st(1)
-; GENERIC-NEXT:    fucomp %st(3)
-; GENERIC-NEXT:    fucompp
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retl
-;
-; ATOM-LABEL: test_fucom_fucomp_fucompp:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    fucom %st(1) # sched: [1:1.00]
-; ATOM-NEXT:    fucom %st(3) # sched: [1:1.00]
-; ATOM-NEXT:    fucomp %st(1) # sched: [1:1.00]
-; ATOM-NEXT:    fucomp %st(3) # sched: [1:1.00]
-; ATOM-NEXT:    fucompp # sched: [1:1.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retl # sched: [79:39.50]
-;
-; SLM-LABEL: test_fucom_fucomp_fucompp:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    fucom %st(1) # sched: [3:1.00]
-; SLM-NEXT:    fucom %st(3) # sched: [3:1.00]
-; SLM-NEXT:    fucomp %st(1) # sched: [3:1.00]
-; SLM-NEXT:    fucomp %st(3) # sched: [3:1.00]
-; SLM-NEXT:    fucompp # sched: [3:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retl # sched: [4:1.00]
-;
-; SANDY-LABEL: test_fucom_fucomp_fucompp:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    fucom %st(1) # sched: [1:1.00]
-; SANDY-NEXT:    fucom %st(3) # sched: [1:1.00]
-; SANDY-NEXT:    fucomp %st(1) # sched: [1:1.00]
-; SANDY-NEXT:    fucomp %st(3) # sched: [1:1.00]
-; SANDY-NEXT:    fucompp # sched: [3:1.00]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retl # sched: [6:1.00]
-;
-; HASWELL-LABEL: test_fucom_fucomp_fucompp:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    fucom %st(1) # sched: [1:1.00]
-; HASWELL-NEXT:    fucom %st(3) # sched: [1:1.00]
-; HASWELL-NEXT:    fucomp %st(1) # sched: [1:1.00]
-; HASWELL-NEXT:    fucomp %st(3) # sched: [1:1.00]
-; HASWELL-NEXT:    fucompp # sched: [1:0.50]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retl # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_fucom_fucomp_fucompp:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    fucom %st(1) # sched: [1:1.00]
-; BROADWELL-NEXT:    fucom %st(3) # sched: [1:1.00]
-; BROADWELL-NEXT:    fucomp %st(1) # sched: [1:1.00]
-; BROADWELL-NEXT:    fucomp %st(3) # sched: [1:1.00]
-; BROADWELL-NEXT:    fucompp # sched: [3:1.00]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retl # sched: [6:0.50]
-;
-; SKYLAKE-LABEL: test_fucom_fucomp_fucompp:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    fucom %st(1) # sched: [1:1.00]
-; SKYLAKE-NEXT:    fucom %st(3) # sched: [1:1.00]
-; SKYLAKE-NEXT:    fucomp %st(1) # sched: [1:1.00]
-; SKYLAKE-NEXT:    fucomp %st(3) # sched: [1:1.00]
-; SKYLAKE-NEXT:    fucompp # sched: [2:1.00]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retl # sched: [6:0.50]
-;
-; SKX-LABEL: test_fucom_fucomp_fucompp:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    fucom %st(1) # sched: [1:1.00]
-; SKX-NEXT:    fucom %st(3) # sched: [1:1.00]
-; SKX-NEXT:    fucomp %st(1) # sched: [1:1.00]
-; SKX-NEXT:    fucomp %st(3) # sched: [1:1.00]
-; SKX-NEXT:    fucompp # sched: [2:1.00]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retl # sched: [6:0.50]
-;
-; BDVER2-LABEL: test_fucom_fucomp_fucompp:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    fucom %st(1) # sched: [1:1.00]
-; BDVER2-NEXT:    fucom %st(3) # sched: [1:1.00]
-; BDVER2-NEXT:    fucomp %st(1) # sched: [1:1.00]
-; BDVER2-NEXT:    fucomp %st(3) # sched: [1:1.00]
-; BDVER2-NEXT:    fucompp # sched: [1:1.00]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retl # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_fucom_fucomp_fucompp:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    fucom %st(1) # sched: [3:1.00]
-; BTVER2-NEXT:    fucom %st(3) # sched: [3:1.00]
-; BTVER2-NEXT:    fucomp %st(1) # sched: [3:1.00]
-; BTVER2-NEXT:    fucomp %st(3) # sched: [3:1.00]
-; BTVER2-NEXT:    fucompp # sched: [3:1.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retl # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_fucom_fucomp_fucompp:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    fucom %st(1) # sched: [1:1.00]
-; ZNVER1-NEXT:    fucom %st(3) # sched: [1:1.00]
-; ZNVER1-NEXT:    fucomp %st(1) # sched: [1:1.00]
-; ZNVER1-NEXT:    fucomp %st(3) # sched: [1:1.00]
-; ZNVER1-NEXT:    fucompp # sched: [1:1.00]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retl # sched: [1:0.50]
-  tail call void asm sideeffect "fucom \0A\09 fucom %st(3) \0A\09 fucomp \0A\09 fucomp %st(3) \0A\09 fucompp", ""() nounwind
-  ret void
-}
-
-define void @test_fucomi_fucomip() optsize {
-; GENERIC-LABEL: test_fucomi_fucomip:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    fucomi %st(3)
-; GENERIC-NEXT:    fucompi %st(3)
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retl
-;
-; ATOM-LABEL: test_fucomi_fucomip:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    fucomi %st(3) # sched: [9:4.50]
-; ATOM-NEXT:    fucompi %st(3) # sched: [9:4.50]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retl # sched: [79:39.50]
-;
-; SLM-LABEL: test_fucomi_fucomip:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    fucomi %st(3) # sched: [3:1.00]
-; SLM-NEXT:    fucompi %st(3) # sched: [3:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retl # sched: [4:1.00]
-;
-; SANDY-LABEL: test_fucomi_fucomip:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    fucomi %st(3) # sched: [3:1.00]
-; SANDY-NEXT:    fucompi %st(3) # sched: [3:1.00]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retl # sched: [6:1.00]
-;
-; HASWELL-LABEL: test_fucomi_fucomip:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    fucomi %st(3) # sched: [1:0.50]
-; HASWELL-NEXT:    fucompi %st(3) # sched: [1:0.50]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retl # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_fucomi_fucomip:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    fucomi %st(3) # sched: [3:1.00]
-; BROADWELL-NEXT:    fucompi %st(3) # sched: [3:1.00]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retl # sched: [6:0.50]
-;
-; SKYLAKE-LABEL: test_fucomi_fucomip:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    fucomi %st(3) # sched: [2:1.00]
-; SKYLAKE-NEXT:    fucompi %st(3) # sched: [2:1.00]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retl # sched: [6:0.50]
-;
-; SKX-LABEL: test_fucomi_fucomip:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    fucomi %st(3) # sched: [2:1.00]
-; SKX-NEXT:    fucompi %st(3) # sched: [2:1.00]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retl # sched: [6:0.50]
-;
-; BDVER2-LABEL: test_fucomi_fucomip:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    fucomi %st(3) # sched: [1:1.00]
-; BDVER2-NEXT:    fucompi %st(3) # sched: [1:1.00]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retl # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_fucomi_fucomip:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    fucomi %st(3) # sched: [3:1.00]
-; BTVER2-NEXT:    fucompi %st(3) # sched: [3:1.00]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retl # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_fucomi_fucomip:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    fucomi %st(3) # sched: [9:0.50]
-; ZNVER1-NEXT:    fucompi %st(3) # sched: [9:0.50]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retl # sched: [1:0.50]
-  tail call void asm sideeffect "fucomi %st(3) \0A\09 fucomip %st(3)", ""() nounwind
-  ret void
-}
-
-define void @test_fwait() optsize {
-; GENERIC-LABEL: test_fwait:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    wait
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retl
-;
-; ATOM-LABEL: test_fwait:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    wait # sched: [1:0.50]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retl # sched: [79:39.50]
-;
-; SLM-LABEL: test_fwait:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    wait # sched: [100:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retl # sched: [4:1.00]
-;
-; SANDY-LABEL: test_fwait:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    wait # sched: [100:0.33]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retl # sched: [6:1.00]
-;
-; HASWELL-LABEL: test_fwait:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    wait # sched: [2:0.50]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retl # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_fwait:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    wait # sched: [2:0.50]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retl # sched: [6:0.50]
-;
-; SKYLAKE-LABEL: test_fwait:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    wait # sched: [2:0.50]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retl # sched: [6:0.50]
-;
-; SKX-LABEL: test_fwait:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    wait # sched: [2:0.50]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retl # sched: [6:0.50]
-;
-; BDVER2-LABEL: test_fwait:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    wait # sched: [100:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retl # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_fwait:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    wait # sched: [100:0.50]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retl # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_fwait:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    wait # sched: [1:1.00]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retl # sched: [1:0.50]
-  tail call void asm sideeffect "fwait", ""() nounwind
-  ret void
-}
-
-define void @test_fxam() optsize {
-; GENERIC-LABEL: test_fxam:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    fxam
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retl
-;
-; ATOM-LABEL: test_fxam:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    fxam # sched: [1:1.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retl # sched: [79:39.50]
-;
-; SLM-LABEL: test_fxam:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    fxam # sched: [100:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retl # sched: [4:1.00]
-;
-; SANDY-LABEL: test_fxam:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    fxam # sched: [100:0.33]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retl # sched: [6:1.00]
-;
-; HASWELL-LABEL: test_fxam:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    fxam # sched: [1:2.00]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retl # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_fxam:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    fxam # sched: [100:0.25]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retl # sched: [6:0.50]
-;
-; SKYLAKE-LABEL: test_fxam:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    fxam # sched: [100:0.25]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retl # sched: [6:0.50]
-;
-; SKX-LABEL: test_fxam:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    fxam # sched: [100:0.25]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retl # sched: [6:0.50]
-;
-; BDVER2-LABEL: test_fxam:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    fxam # sched: [100:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retl # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_fxam:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    fxam # sched: [100:0.50]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retl # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_fxam:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    fxam # sched: [1:1.00]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retl # sched: [1:0.50]
-  tail call void asm sideeffect "fxam", ""() nounwind
-  ret void
-}
-
-define void @test_fxch() optsize {
-; GENERIC-LABEL: test_fxch:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    fxch %st(1)
-; GENERIC-NEXT:    fxch %st(3)
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retl
-;
-; ATOM-LABEL: test_fxch:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    fxch %st(1) # sched: [1:1.00]
-; ATOM-NEXT:    fxch %st(3) # sched: [1:1.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retl # sched: [79:39.50]
-;
-; SLM-LABEL: test_fxch:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    fxch %st(1) # sched: [1:0.50]
-; SLM-NEXT:    fxch %st(3) # sched: [1:0.50]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retl # sched: [4:1.00]
-;
-; SANDY-LABEL: test_fxch:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    fxch %st(1) # sched: [1:0.33]
-; SANDY-NEXT:    fxch %st(3) # sched: [1:0.33]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retl # sched: [6:1.00]
-;
-; HASWELL-LABEL: test_fxch:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    fxch %st(1) # sched: [17:4.00]
-; HASWELL-NEXT:    fxch %st(3) # sched: [17:4.00]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retl # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_fxch:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    fxch %st(1) # sched: [14:4.00]
-; BROADWELL-NEXT:    fxch %st(3) # sched: [14:4.00]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retl # sched: [6:0.50]
-;
-; SKYLAKE-LABEL: test_fxch:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    fxch %st(1) # sched: [17:4.00]
-; SKYLAKE-NEXT:    fxch %st(3) # sched: [17:4.00]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retl # sched: [6:0.50]
-;
-; SKX-LABEL: test_fxch:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    fxch %st(1) # sched: [17:4.00]
-; SKX-NEXT:    fxch %st(3) # sched: [17:4.00]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retl # sched: [6:0.50]
-;
-; BDVER2-LABEL: test_fxch:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    fxch %st(1) # sched: [1:0.50]
-; BDVER2-NEXT:    fxch %st(3) # sched: [1:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retl # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_fxch:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    fxch %st(1) # sched: [1:0.50]
-; BTVER2-NEXT:    fxch %st(3) # sched: [1:0.50]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retl # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_fxch:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    fxch %st(1) # sched: [1:0.25]
-; ZNVER1-NEXT:    fxch %st(3) # sched: [1:0.25]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retl # sched: [1:0.50]
-  tail call void asm sideeffect "fxch \0A\09 fxch %st(3)", ""() nounwind
-  ret void
-}
-
-define void @test_fxrstor_fxsave(i8* %a0) optsize {
-; GENERIC-LABEL: test_fxrstor_fxsave:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    fxrstor (%eax)
-; GENERIC-NEXT:    fxsave (%eax)
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retl
-;
-; ATOM-LABEL: test_fxrstor_fxsave:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [1:1.00]
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    fxrstor (%eax) # sched: [141:70.50]
-; ATOM-NEXT:    fxsave (%eax) # sched: [140:70.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retl # sched: [79:39.50]
-;
-; SLM-LABEL: test_fxrstor_fxsave:
-; SLM:       # %bb.0:
-; SLM-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [3:1.00]
-; SLM-NEXT:    #APP
-; SLM-NEXT:    fxrstor (%eax) # sched: [100:1.00]
-; SLM-NEXT:    fxsave (%eax) # sched: [100:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retl # sched: [4:1.00]
-;
-; SANDY-LABEL: test_fxrstor_fxsave:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    fxrstor (%eax) # sched: [5:2.00]
-; SANDY-NEXT:    fxsave (%eax) # sched: [100:0.33]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retl # sched: [6:1.00]
-;
-; HASWELL-LABEL: test_fxrstor_fxsave:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    fxrstor (%eax) # sched: [64:16.50]
-; HASWELL-NEXT:    fxsave (%eax) # sched: [100:0.25]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retl # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_fxrstor_fxsave:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    fxrstor (%eax) # sched: [63:16.50]
-; BROADWELL-NEXT:    fxsave (%eax) # sched: [100:0.25]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retl # sched: [6:0.50]
-;
-; SKYLAKE-LABEL: test_fxrstor_fxsave:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    fxrstor (%eax) # sched: [63:16.50]
-; SKYLAKE-NEXT:    fxsave (%eax) # sched: [100:0.25]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retl # sched: [6:0.50]
-;
-; SKX-LABEL: test_fxrstor_fxsave:
-; SKX:       # %bb.0:
-; SKX-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; SKX-NEXT:    #APP
-; SKX-NEXT:    fxrstor (%eax) # sched: [63:16.50]
-; SKX-NEXT:    fxsave (%eax) # sched: [100:0.25]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retl # sched: [6:0.50]
-;
-; BDVER2-LABEL: test_fxrstor_fxsave:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:0.50]
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    fxrstor (%eax) # sched: [100:0.50]
-; BDVER2-NEXT:    fxsave (%eax) # sched: [100:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retl # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_fxrstor_fxsave:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [5:1.00]
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    fxrstor (%eax) # sched: [100:0.50]
-; BTVER2-NEXT:    fxsave (%eax) # sched: [100:0.50]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retl # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_fxrstor_fxsave:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    movl {{[0-9]+}}(%esp), %eax # sched: [8:0.50]
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    fxrstor (%eax) # sched: [100:0.25]
-; ZNVER1-NEXT:    fxsave (%eax) # sched: [100:0.25]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retl # sched: [1:0.50]
-  tail call void asm sideeffect "fxrstor $0 \0A\09 fxsave $0", "*m"(i8 *%a0) nounwind
-  ret void
-}
-
-define void @test_fxtract() optsize {
-; GENERIC-LABEL: test_fxtract:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    fxtract
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retl
-;
-; ATOM-LABEL: test_fxtract:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    fxtract # sched: [25:12.50]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retl # sched: [79:39.50]
-;
-; SLM-LABEL: test_fxtract:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    fxtract # sched: [100:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retl # sched: [4:1.00]
-;
-; SANDY-LABEL: test_fxtract:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    fxtract # sched: [100:0.33]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retl # sched: [6:1.00]
-;
-; HASWELL-LABEL: test_fxtract:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    fxtract # sched: [15:4.25]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retl # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_fxtract:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    fxtract # sched: [100:0.25]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retl # sched: [6:0.50]
-;
-; SKYLAKE-LABEL: test_fxtract:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    fxtract # sched: [100:0.25]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retl # sched: [6:0.50]
-;
-; SKX-LABEL: test_fxtract:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    fxtract # sched: [100:0.25]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retl # sched: [6:0.50]
-;
-; BDVER2-LABEL: test_fxtract:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    fxtract # sched: [100:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retl # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_fxtract:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    fxtract # sched: [100:0.50]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retl # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_fxtract:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    fxtract # sched: [100:0.25]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retl # sched: [1:0.50]
-  tail call void asm sideeffect "fxtract", ""() nounwind
-  ret void
-}
-
-define void @test_fyl2x() optsize {
-; GENERIC-LABEL: test_fyl2x:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    fyl2x
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retl
-;
-; ATOM-LABEL: test_fyl2x:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    fyl2x # sched: [146:73.00]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retl # sched: [79:39.50]
-;
-; SLM-LABEL: test_fyl2x:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    fyl2x # sched: [100:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retl # sched: [4:1.00]
-;
-; SANDY-LABEL: test_fyl2x:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    fyl2x # sched: [100:0.33]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retl # sched: [6:1.00]
-;
-; HASWELL-LABEL: test_fyl2x:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    fyl2x # sched: [100:0.25]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retl # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_fyl2x:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    fyl2x # sched: [100:0.25]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retl # sched: [6:0.50]
-;
-; SKYLAKE-LABEL: test_fyl2x:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    fyl2x # sched: [100:0.25]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retl # sched: [6:0.50]
-;
-; SKX-LABEL: test_fyl2x:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    fyl2x # sched: [100:0.25]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retl # sched: [6:0.50]
-;
-; BDVER2-LABEL: test_fyl2x:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    fyl2x # sched: [100:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retl # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_fyl2x:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    fyl2x # sched: [100:0.50]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retl # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_fyl2x:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    fyl2x # sched: [100:0.25]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retl # sched: [1:0.50]
-  tail call void asm sideeffect "fyl2x", ""() nounwind
-  ret void
-}
-
-define void @test_fyl2xp1() optsize {
-; GENERIC-LABEL: test_fyl2xp1:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    fyl2xp1
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retl
-;
-; ATOM-LABEL: test_fyl2xp1:
-; ATOM:       # %bb.0:
-; ATOM-NEXT:    #APP
-; ATOM-NEXT:    fyl2xp1 # sched: [147:73.50]
-; ATOM-NEXT:    #NO_APP
-; ATOM-NEXT:    retl # sched: [79:39.50]
-;
-; SLM-LABEL: test_fyl2xp1:
-; SLM:       # %bb.0:
-; SLM-NEXT:    #APP
-; SLM-NEXT:    fyl2xp1 # sched: [100:1.00]
-; SLM-NEXT:    #NO_APP
-; SLM-NEXT:    retl # sched: [4:1.00]
-;
-; SANDY-LABEL: test_fyl2xp1:
-; SANDY:       # %bb.0:
-; SANDY-NEXT:    #APP
-; SANDY-NEXT:    fyl2xp1 # sched: [100:0.33]
-; SANDY-NEXT:    #NO_APP
-; SANDY-NEXT:    retl # sched: [6:1.00]
-;
-; HASWELL-LABEL: test_fyl2xp1:
-; HASWELL:       # %bb.0:
-; HASWELL-NEXT:    #APP
-; HASWELL-NEXT:    fyl2xp1 # sched: [100:0.25]
-; HASWELL-NEXT:    #NO_APP
-; HASWELL-NEXT:    retl # sched: [7:1.00]
-;
-; BROADWELL-LABEL: test_fyl2xp1:
-; BROADWELL:       # %bb.0:
-; BROADWELL-NEXT:    #APP
-; BROADWELL-NEXT:    fyl2xp1 # sched: [100:0.25]
-; BROADWELL-NEXT:    #NO_APP
-; BROADWELL-NEXT:    retl # sched: [6:0.50]
-;
-; SKYLAKE-LABEL: test_fyl2xp1:
-; SKYLAKE:       # %bb.0:
-; SKYLAKE-NEXT:    #APP
-; SKYLAKE-NEXT:    fyl2xp1 # sched: [100:0.25]
-; SKYLAKE-NEXT:    #NO_APP
-; SKYLAKE-NEXT:    retl # sched: [6:0.50]
-;
-; SKX-LABEL: test_fyl2xp1:
-; SKX:       # %bb.0:
-; SKX-NEXT:    #APP
-; SKX-NEXT:    fyl2xp1 # sched: [100:0.25]
-; SKX-NEXT:    #NO_APP
-; SKX-NEXT:    retl # sched: [6:0.50]
-;
-; BDVER2-LABEL: test_fyl2xp1:
-; BDVER2:       # %bb.0:
-; BDVER2-NEXT:    #APP
-; BDVER2-NEXT:    fyl2xp1 # sched: [100:0.50]
-; BDVER2-NEXT:    #NO_APP
-; BDVER2-NEXT:    retl # sched: [5:1.00]
-;
-; BTVER2-LABEL: test_fyl2xp1:
-; BTVER2:       # %bb.0:
-; BTVER2-NEXT:    #APP
-; BTVER2-NEXT:    fyl2xp1 # sched: [100:0.50]
-; BTVER2-NEXT:    #NO_APP
-; BTVER2-NEXT:    retl # sched: [4:1.00]
-;
-; ZNVER1-LABEL: test_fyl2xp1:
-; ZNVER1:       # %bb.0:
-; ZNVER1-NEXT:    #APP
-; ZNVER1-NEXT:    fyl2xp1 # sched: [100:0.25]
-; ZNVER1-NEXT:    #NO_APP
-; ZNVER1-NEXT:    retl # sched: [1:0.50]
-  tail call void asm sideeffect "fyl2xp1", ""() nounwind
-  ret void
-}
diff --git a/test/CodeGen/X86/xaluo.ll b/test/CodeGen/X86/xaluo.ll
index 25aa45e1405608af378a00d71bb3381ddd223179..786c6178ec44ecb27c5723bda6597e6bad76962e 100644
--- a/test/CodeGen/X86/xaluo.ll
+++ b/test/CodeGen/X86/xaluo.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=x86_64-darwin-unknown < %s | FileCheck %s --check-prefix=SDAG
 ; RUN: llc -mtriple=x86_64-darwin-unknown -fast-isel -fast-isel-abort=1 < %s | FileCheck %s --check-prefix=FAST
-; RUN: llc -mtriple=x86_64-darwin-unknown -mcpu=knl < %s | FileCheck %s --check-prefix=KNL
+; RUN: llc -mtriple=x86_64-darwin-unknown -mcpu=knl < %s | FileCheck %s --check-prefix=SDAG
 
 ;
 ; Get the actual value of the overflow bit.
@@ -23,13 +23,6 @@ define zeroext i1 @saddoi8(i8 signext %v1, i8 signext %v2, i8* %res) {
 ; FAST-NEXT:    andb $1, %al
 ; FAST-NEXT:    movzbl %al, %eax
 ; FAST-NEXT:    retq
-;
-; KNL-LABEL: saddoi8:
-; KNL:       ## %bb.0:
-; KNL-NEXT:    addb %sil, %dil
-; KNL-NEXT:    seto %al
-; KNL-NEXT:    movb %dil, (%rdx)
-; KNL-NEXT:    retq
   %t = call {i8, i1} @llvm.sadd.with.overflow.i8(i8 %v1, i8 %v2)
   %val = extractvalue {i8, i1} %t, 0
   %obit = extractvalue {i8, i1} %t, 1
@@ -53,13 +46,6 @@ define zeroext i1 @saddoi16(i16 %v1, i16 %v2, i16* %res) {
 ; FAST-NEXT:    andb $1, %al
 ; FAST-NEXT:    movzbl %al, %eax
 ; FAST-NEXT:    retq
-;
-; KNL-LABEL: saddoi16:
-; KNL:       ## %bb.0:
-; KNL-NEXT:    addw %si, %di
-; KNL-NEXT:    seto %al
-; KNL-NEXT:    movw %di, (%rdx)
-; KNL-NEXT:    retq
   %t = call {i16, i1} @llvm.sadd.with.overflow.i16(i16 %v1, i16 %v2)
   %val = extractvalue {i16, i1} %t, 0
   %obit = extractvalue {i16, i1} %t, 1
@@ -83,13 +69,6 @@ define zeroext i1 @saddoi32(i32 %v1, i32 %v2, i32* %res) {
 ; FAST-NEXT:    andb $1, %al
 ; FAST-NEXT:    movzbl %al, %eax
 ; FAST-NEXT:    retq
-;
-; KNL-LABEL: saddoi32:
-; KNL:       ## %bb.0:
-; KNL-NEXT:    addl %esi, %edi
-; KNL-NEXT:    seto %al
-; KNL-NEXT:    movl %edi, (%rdx)
-; KNL-NEXT:    retq
   %t = call {i32, i1} @llvm.sadd.with.overflow.i32(i32 %v1, i32 %v2)
   %val = extractvalue {i32, i1} %t, 0
   %obit = extractvalue {i32, i1} %t, 1
@@ -113,13 +92,6 @@ define zeroext i1 @saddoi64(i64 %v1, i64 %v2, i64* %res) {
 ; FAST-NEXT:    andb $1, %al
 ; FAST-NEXT:    movzbl %al, %eax
 ; FAST-NEXT:    retq
-;
-; KNL-LABEL: saddoi64:
-; KNL:       ## %bb.0:
-; KNL-NEXT:    addq %rsi, %rdi
-; KNL-NEXT:    seto %al
-; KNL-NEXT:    movq %rdi, (%rdx)
-; KNL-NEXT:    retq
   %t = call {i64, i1} @llvm.sadd.with.overflow.i64(i64 %v1, i64 %v2)
   %val = extractvalue {i64, i1} %t, 0
   %obit = extractvalue {i64, i1} %t, 1
@@ -144,13 +116,6 @@ define zeroext i1 @saddoinci8(i8 %v1, i8* %res) {
 ; FAST-NEXT:    andb $1, %al
 ; FAST-NEXT:    movzbl %al, %eax
 ; FAST-NEXT:    retq
-;
-; KNL-LABEL: saddoinci8:
-; KNL:       ## %bb.0:
-; KNL-NEXT:    incb %dil
-; KNL-NEXT:    seto %al
-; KNL-NEXT:    movb %dil, (%rsi)
-; KNL-NEXT:    retq
   %t = call {i8, i1} @llvm.sadd.with.overflow.i8(i8 %v1, i8 1)
   %val = extractvalue {i8, i1} %t, 0
   %obit = extractvalue {i8, i1} %t, 1
@@ -174,13 +139,6 @@ define zeroext i1 @saddoinci16(i16 %v1, i16* %res) {
 ; FAST-NEXT:    andb $1, %al
 ; FAST-NEXT:    movzbl %al, %eax
 ; FAST-NEXT:    retq
-;
-; KNL-LABEL: saddoinci16:
-; KNL:       ## %bb.0:
-; KNL-NEXT:    incw %di
-; KNL-NEXT:    seto %al
-; KNL-NEXT:    movw %di, (%rsi)
-; KNL-NEXT:    retq
   %t = call {i16, i1} @llvm.sadd.with.overflow.i16(i16 %v1, i16 1)
   %val = extractvalue {i16, i1} %t, 0
   %obit = extractvalue {i16, i1} %t, 1
@@ -204,13 +162,6 @@ define zeroext i1 @saddoinci32(i32 %v1, i32* %res) {
 ; FAST-NEXT:    andb $1, %al
 ; FAST-NEXT:    movzbl %al, %eax
 ; FAST-NEXT:    retq
-;
-; KNL-LABEL: saddoinci32:
-; KNL:       ## %bb.0:
-; KNL-NEXT:    incl %edi
-; KNL-NEXT:    seto %al
-; KNL-NEXT:    movl %edi, (%rsi)
-; KNL-NEXT:    retq
   %t = call {i32, i1} @llvm.sadd.with.overflow.i32(i32 %v1, i32 1)
   %val = extractvalue {i32, i1} %t, 0
   %obit = extractvalue {i32, i1} %t, 1
@@ -234,13 +185,6 @@ define zeroext i1 @saddoinci64(i64 %v1, i64* %res) {
 ; FAST-NEXT:    andb $1, %al
 ; FAST-NEXT:    movzbl %al, %eax
 ; FAST-NEXT:    retq
-;
-; KNL-LABEL: saddoinci64:
-; KNL:       ## %bb.0:
-; KNL-NEXT:    incq %rdi
-; KNL-NEXT:    seto %al
-; KNL-NEXT:    movq %rdi, (%rsi)
-; KNL-NEXT:    retq
   %t = call {i64, i1} @llvm.sadd.with.overflow.i64(i64 %v1, i64 1)
   %val = extractvalue {i64, i1} %t, 0
   %obit = extractvalue {i64, i1} %t, 1
@@ -267,14 +211,6 @@ define zeroext i1 @saddoi64imm1(i64 %v1, i64* %res) {
 ; FAST-NEXT:    andb $1, %al
 ; FAST-NEXT:    movzbl %al, %eax
 ; FAST-NEXT:    retq
-;
-; KNL-LABEL: saddoi64imm1:
-; KNL:       ## %bb.0:
-; KNL-NEXT:    movl $2, %ecx
-; KNL-NEXT:    addq %rdi, %rcx
-; KNL-NEXT:    seto %al
-; KNL-NEXT:    movq %rcx, (%rsi)
-; KNL-NEXT:    retq
   %t = call {i64, i1} @llvm.sadd.with.overflow.i64(i64 2, i64 %v1)
   %val = extractvalue {i64, i1} %t, 0
   %obit = extractvalue {i64, i1} %t, 1
@@ -299,13 +235,6 @@ define zeroext i1 @saddoi64imm2(i64 %v1, i64* %res) {
 ; FAST-NEXT:    andb $1, %al
 ; FAST-NEXT:    movzbl %al, %eax
 ; FAST-NEXT:    retq
-;
-; KNL-LABEL: saddoi64imm2:
-; KNL:       ## %bb.0:
-; KNL-NEXT:    addq $-2147483648, %rdi ## imm = 0x80000000
-; KNL-NEXT:    seto %al
-; KNL-NEXT:    movq %rdi, (%rsi)
-; KNL-NEXT:    retq
   %t = call {i64, i1} @llvm.sadd.with.overflow.i64(i64 %v1, i64 -2147483648)
   %val = extractvalue {i64, i1} %t, 0
   %obit = extractvalue {i64, i1} %t, 1
@@ -331,14 +260,6 @@ define zeroext i1 @saddoi64imm3(i64 %v1, i64* %res) {
 ; FAST-NEXT:    andb $1, %cl
 ; FAST-NEXT:    movzbl %cl, %eax
 ; FAST-NEXT:    retq
-;
-; KNL-LABEL: saddoi64imm3:
-; KNL:       ## %bb.0:
-; KNL-NEXT:    movabsq $-21474836489, %rcx ## imm = 0xFFFFFFFAFFFFFFF7
-; KNL-NEXT:    addq %rdi, %rcx
-; KNL-NEXT:    seto %al
-; KNL-NEXT:    movq %rcx, (%rsi)
-; KNL-NEXT:    retq
   %t = call {i64, i1} @llvm.sadd.with.overflow.i64(i64 %v1, i64 -21474836489)
   %val = extractvalue {i64, i1} %t, 0
   %obit = extractvalue {i64, i1} %t, 1
@@ -362,13 +283,6 @@ define zeroext i1 @saddoi64imm4(i64 %v1, i64* %res) {
 ; FAST-NEXT:    andb $1, %al
 ; FAST-NEXT:    movzbl %al, %eax
 ; FAST-NEXT:    retq
-;
-; KNL-LABEL: saddoi64imm4:
-; KNL:       ## %bb.0:
-; KNL-NEXT:    addq $2147483647, %rdi ## imm = 0x7FFFFFFF
-; KNL-NEXT:    seto %al
-; KNL-NEXT:    movq %rdi, (%rsi)
-; KNL-NEXT:    retq
   %t = call {i64, i1} @llvm.sadd.with.overflow.i64(i64 %v1, i64 2147483647)
   %val = extractvalue {i64, i1} %t, 0
   %obit = extractvalue {i64, i1} %t, 1
@@ -394,14 +308,6 @@ define zeroext i1 @saddoi64imm5(i64 %v1, i64* %res) {
 ; FAST-NEXT:    andb $1, %cl
 ; FAST-NEXT:    movzbl %cl, %eax
 ; FAST-NEXT:    retq
-;
-; KNL-LABEL: saddoi64imm5:
-; KNL:       ## %bb.0:
-; KNL-NEXT:    movl $2147483648, %ecx ## imm = 0x80000000
-; KNL-NEXT:    addq %rdi, %rcx
-; KNL-NEXT:    seto %al
-; KNL-NEXT:    movq %rcx, (%rsi)
-; KNL-NEXT:    retq
   %t = call {i64, i1} @llvm.sadd.with.overflow.i64(i64 %v1, i64 2147483648)
   %val = extractvalue {i64, i1} %t, 0
   %obit = extractvalue {i64, i1} %t, 1
@@ -426,13 +332,6 @@ define zeroext i1 @uaddoi32(i32 %v1, i32 %v2, i32* %res) {
 ; FAST-NEXT:    andb $1, %al
 ; FAST-NEXT:    movzbl %al, %eax
 ; FAST-NEXT:    retq
-;
-; KNL-LABEL: uaddoi32:
-; KNL:       ## %bb.0:
-; KNL-NEXT:    addl %esi, %edi
-; KNL-NEXT:    setb %al
-; KNL-NEXT:    movl %edi, (%rdx)
-; KNL-NEXT:    retq
   %t = call {i32, i1} @llvm.uadd.with.overflow.i32(i32 %v1, i32 %v2)
   %val = extractvalue {i32, i1} %t, 0
   %obit = extractvalue {i32, i1} %t, 1
@@ -456,13 +355,6 @@ define zeroext i1 @uaddoi64(i64 %v1, i64 %v2, i64* %res) {
 ; FAST-NEXT:    andb $1, %al
 ; FAST-NEXT:    movzbl %al, %eax
 ; FAST-NEXT:    retq
-;
-; KNL-LABEL: uaddoi64:
-; KNL:       ## %bb.0:
-; KNL-NEXT:    addq %rsi, %rdi
-; KNL-NEXT:    setb %al
-; KNL-NEXT:    movq %rdi, (%rdx)
-; KNL-NEXT:    retq
   %t = call {i64, i1} @llvm.uadd.with.overflow.i64(i64 %v1, i64 %v2)
   %val = extractvalue {i64, i1} %t, 0
   %obit = extractvalue {i64, i1} %t, 1
@@ -474,8 +366,8 @@ define zeroext i1 @uaddoi64(i64 %v1, i64 %v2, i64* %res) {
 define zeroext i1 @uaddoinci8(i8 %v1, i8* %res) {
 ; SDAG-LABEL: uaddoinci8:
 ; SDAG:       ## %bb.0:
-; SDAG-NEXT:    addb $1, %dil
-; SDAG-NEXT:    setb %al
+; SDAG-NEXT:    incb %dil
+; SDAG-NEXT:    sete %al
 ; SDAG-NEXT:    movb %dil, (%rsi)
 ; SDAG-NEXT:    retq
 ;
@@ -487,13 +379,6 @@ define zeroext i1 @uaddoinci8(i8 %v1, i8* %res) {
 ; FAST-NEXT:    andb $1, %al
 ; FAST-NEXT:    movzbl %al, %eax
 ; FAST-NEXT:    retq
-;
-; KNL-LABEL: uaddoinci8:
-; KNL:       ## %bb.0:
-; KNL-NEXT:    addb $1, %dil
-; KNL-NEXT:    setb %al
-; KNL-NEXT:    movb %dil, (%rsi)
-; KNL-NEXT:    retq
   %t = call {i8, i1} @llvm.uadd.with.overflow.i8(i8 %v1, i8 1)
   %val = extractvalue {i8, i1} %t, 0
   %obit = extractvalue {i8, i1} %t, 1
@@ -504,8 +389,8 @@ define zeroext i1 @uaddoinci8(i8 %v1, i8* %res) {
 define zeroext i1 @uaddoinci16(i16 %v1, i16* %res) {
 ; SDAG-LABEL: uaddoinci16:
 ; SDAG:       ## %bb.0:
-; SDAG-NEXT:    addw $1, %di
-; SDAG-NEXT:    setb %al
+; SDAG-NEXT:    incw %di
+; SDAG-NEXT:    sete %al
 ; SDAG-NEXT:    movw %di, (%rsi)
 ; SDAG-NEXT:    retq
 ;
@@ -517,13 +402,6 @@ define zeroext i1 @uaddoinci16(i16 %v1, i16* %res) {
 ; FAST-NEXT:    andb $1, %al
 ; FAST-NEXT:    movzbl %al, %eax
 ; FAST-NEXT:    retq
-;
-; KNL-LABEL: uaddoinci16:
-; KNL:       ## %bb.0:
-; KNL-NEXT:    addw $1, %di
-; KNL-NEXT:    setb %al
-; KNL-NEXT:    movw %di, (%rsi)
-; KNL-NEXT:    retq
   %t = call {i16, i1} @llvm.uadd.with.overflow.i16(i16 %v1, i16 1)
   %val = extractvalue {i16, i1} %t, 0
   %obit = extractvalue {i16, i1} %t, 1
@@ -534,8 +412,8 @@ define zeroext i1 @uaddoinci16(i16 %v1, i16* %res) {
 define zeroext i1 @uaddoinci32(i32 %v1, i32* %res) {
 ; SDAG-LABEL: uaddoinci32:
 ; SDAG:       ## %bb.0:
-; SDAG-NEXT:    addl $1, %edi
-; SDAG-NEXT:    setb %al
+; SDAG-NEXT:    incl %edi
+; SDAG-NEXT:    sete %al
 ; SDAG-NEXT:    movl %edi, (%rsi)
 ; SDAG-NEXT:    retq
 ;
@@ -547,13 +425,6 @@ define zeroext i1 @uaddoinci32(i32 %v1, i32* %res) {
 ; FAST-NEXT:    andb $1, %al
 ; FAST-NEXT:    movzbl %al, %eax
 ; FAST-NEXT:    retq
-;
-; KNL-LABEL: uaddoinci32:
-; KNL:       ## %bb.0:
-; KNL-NEXT:    addl $1, %edi
-; KNL-NEXT:    setb %al
-; KNL-NEXT:    movl %edi, (%rsi)
-; KNL-NEXT:    retq
   %t = call {i32, i1} @llvm.uadd.with.overflow.i32(i32 %v1, i32 1)
   %val = extractvalue {i32, i1} %t, 0
   %obit = extractvalue {i32, i1} %t, 1
@@ -564,8 +435,8 @@ define zeroext i1 @uaddoinci32(i32 %v1, i32* %res) {
 define zeroext i1 @uaddoinci64(i64 %v1, i64* %res) {
 ; SDAG-LABEL: uaddoinci64:
 ; SDAG:       ## %bb.0:
-; SDAG-NEXT:    addq $1, %rdi
-; SDAG-NEXT:    setb %al
+; SDAG-NEXT:    incq %rdi
+; SDAG-NEXT:    sete %al
 ; SDAG-NEXT:    movq %rdi, (%rsi)
 ; SDAG-NEXT:    retq
 ;
@@ -577,13 +448,6 @@ define zeroext i1 @uaddoinci64(i64 %v1, i64* %res) {
 ; FAST-NEXT:    andb $1, %al
 ; FAST-NEXT:    movzbl %al, %eax
 ; FAST-NEXT:    retq
-;
-; KNL-LABEL: uaddoinci64:
-; KNL:       ## %bb.0:
-; KNL-NEXT:    addq $1, %rdi
-; KNL-NEXT:    setb %al
-; KNL-NEXT:    movq %rdi, (%rsi)
-; KNL-NEXT:    retq
   %t = call {i64, i1} @llvm.uadd.with.overflow.i64(i64 %v1, i64 1)
   %val = extractvalue {i64, i1} %t, 0
   %obit = extractvalue {i64, i1} %t, 1
@@ -608,13 +472,6 @@ define zeroext i1 @ssuboi32(i32 %v1, i32 %v2, i32* %res) {
 ; FAST-NEXT:    andb $1, %al
 ; FAST-NEXT:    movzbl %al, %eax
 ; FAST-NEXT:    retq
-;
-; KNL-LABEL: ssuboi32:
-; KNL:       ## %bb.0:
-; KNL-NEXT:    subl %esi, %edi
-; KNL-NEXT:    seto %al
-; KNL-NEXT:    movl %edi, (%rdx)
-; KNL-NEXT:    retq
   %t = call {i32, i1} @llvm.ssub.with.overflow.i32(i32 %v1, i32 %v2)
   %val = extractvalue {i32, i1} %t, 0
   %obit = extractvalue {i32, i1} %t, 1
@@ -638,13 +495,6 @@ define zeroext i1 @ssuboi64(i64 %v1, i64 %v2, i64* %res) {
 ; FAST-NEXT:    andb $1, %al
 ; FAST-NEXT:    movzbl %al, %eax
 ; FAST-NEXT:    retq
-;
-; KNL-LABEL: ssuboi64:
-; KNL:       ## %bb.0:
-; KNL-NEXT:    subq %rsi, %rdi
-; KNL-NEXT:    seto %al
-; KNL-NEXT:    movq %rdi, (%rdx)
-; KNL-NEXT:    retq
   %t = call {i64, i1} @llvm.ssub.with.overflow.i64(i64 %v1, i64 %v2)
   %val = extractvalue {i64, i1} %t, 0
   %obit = extractvalue {i64, i1} %t, 1
@@ -669,13 +519,6 @@ define zeroext i1 @usuboi32(i32 %v1, i32 %v2, i32* %res) {
 ; FAST-NEXT:    andb $1, %al
 ; FAST-NEXT:    movzbl %al, %eax
 ; FAST-NEXT:    retq
-;
-; KNL-LABEL: usuboi32:
-; KNL:       ## %bb.0:
-; KNL-NEXT:    subl %esi, %edi
-; KNL-NEXT:    setb %al
-; KNL-NEXT:    movl %edi, (%rdx)
-; KNL-NEXT:    retq
   %t = call {i32, i1} @llvm.usub.with.overflow.i32(i32 %v1, i32 %v2)
   %val = extractvalue {i32, i1} %t, 0
   %obit = extractvalue {i32, i1} %t, 1
@@ -699,13 +542,6 @@ define zeroext i1 @usuboi64(i64 %v1, i64 %v2, i64* %res) {
 ; FAST-NEXT:    andb $1, %al
 ; FAST-NEXT:    movzbl %al, %eax
 ; FAST-NEXT:    retq
-;
-; KNL-LABEL: usuboi64:
-; KNL:       ## %bb.0:
-; KNL-NEXT:    subq %rsi, %rdi
-; KNL-NEXT:    setb %al
-; KNL-NEXT:    movq %rdi, (%rdx)
-; KNL-NEXT:    retq
   %t = call {i64, i1} @llvm.usub.with.overflow.i64(i64 %v1, i64 %v2)
   %val = extractvalue {i64, i1} %t, 0
   %obit = extractvalue {i64, i1} %t, 1
@@ -732,14 +568,6 @@ define i32 @saddoselecti32(i32 %v1, i32 %v2) {
 ; FAST-NEXT:    addl %esi, %ecx
 ; FAST-NEXT:    cmovol %edi, %eax
 ; FAST-NEXT:    retq
-;
-; KNL-LABEL: saddoselecti32:
-; KNL:       ## %bb.0:
-; KNL-NEXT:    movl %esi, %eax
-; KNL-NEXT:    movl %edi, %ecx
-; KNL-NEXT:    addl %esi, %ecx
-; KNL-NEXT:    cmovol %edi, %eax
-; KNL-NEXT:    retq
   %t = call {i32, i1} @llvm.sadd.with.overflow.i32(i32 %v1, i32 %v2)
   %obit = extractvalue {i32, i1} %t, 1
   %ret = select i1 %obit, i32 %v1, i32 %v2
@@ -762,14 +590,6 @@ define i64 @saddoselecti64(i64 %v1, i64 %v2) {
 ; FAST-NEXT:    addq %rsi, %rcx
 ; FAST-NEXT:    cmovoq %rdi, %rax
 ; FAST-NEXT:    retq
-;
-; KNL-LABEL: saddoselecti64:
-; KNL:       ## %bb.0:
-; KNL-NEXT:    movq %rsi, %rax
-; KNL-NEXT:    movq %rdi, %rcx
-; KNL-NEXT:    addq %rsi, %rcx
-; KNL-NEXT:    cmovoq %rdi, %rax
-; KNL-NEXT:    retq
   %t = call {i64, i1} @llvm.sadd.with.overflow.i64(i64 %v1, i64 %v2)
   %obit = extractvalue {i64, i1} %t, 1
   %ret = select i1 %obit, i64 %v1, i64 %v2
@@ -792,14 +612,6 @@ define i32 @uaddoselecti32(i32 %v1, i32 %v2) {
 ; FAST-NEXT:    addl %esi, %ecx
 ; FAST-NEXT:    cmovbl %edi, %eax
 ; FAST-NEXT:    retq
-;
-; KNL-LABEL: uaddoselecti32:
-; KNL:       ## %bb.0:
-; KNL-NEXT:    movl %esi, %eax
-; KNL-NEXT:    movl %edi, %ecx
-; KNL-NEXT:    addl %esi, %ecx
-; KNL-NEXT:    cmovbl %edi, %eax
-; KNL-NEXT:    retq
   %t = call {i32, i1} @llvm.uadd.with.overflow.i32(i32 %v1, i32 %v2)
   %obit = extractvalue {i32, i1} %t, 1
   %ret = select i1 %obit, i32 %v1, i32 %v2
@@ -822,14 +634,6 @@ define i64 @uaddoselecti64(i64 %v1, i64 %v2) {
 ; FAST-NEXT:    addq %rsi, %rcx
 ; FAST-NEXT:    cmovbq %rdi, %rax
 ; FAST-NEXT:    retq
-;
-; KNL-LABEL: uaddoselecti64:
-; KNL:       ## %bb.0:
-; KNL-NEXT:    movq %rsi, %rax
-; KNL-NEXT:    movq %rdi, %rcx
-; KNL-NEXT:    addq %rsi, %rcx
-; KNL-NEXT:    cmovbq %rdi, %rax
-; KNL-NEXT:    retq
   %t = call {i64, i1} @llvm.uadd.with.overflow.i64(i64 %v1, i64 %v2)
   %obit = extractvalue {i64, i1} %t, 1
   %ret = select i1 %obit, i64 %v1, i64 %v2
@@ -850,13 +654,6 @@ define i32 @ssuboselecti32(i32 %v1, i32 %v2) {
 ; FAST-NEXT:    cmpl %esi, %edi
 ; FAST-NEXT:    cmovol %edi, %eax
 ; FAST-NEXT:    retq
-;
-; KNL-LABEL: ssuboselecti32:
-; KNL:       ## %bb.0:
-; KNL-NEXT:    movl %esi, %eax
-; KNL-NEXT:    cmpl %esi, %edi
-; KNL-NEXT:    cmovol %edi, %eax
-; KNL-NEXT:    retq
   %t = call {i32, i1} @llvm.ssub.with.overflow.i32(i32 %v1, i32 %v2)
   %obit = extractvalue {i32, i1} %t, 1
   %ret = select i1 %obit, i32 %v1, i32 %v2
@@ -877,13 +674,6 @@ define i64 @ssuboselecti64(i64 %v1, i64 %v2) {
 ; FAST-NEXT:    cmpq %rsi, %rdi
 ; FAST-NEXT:    cmovoq %rdi, %rax
 ; FAST-NEXT:    retq
-;
-; KNL-LABEL: ssuboselecti64:
-; KNL:       ## %bb.0:
-; KNL-NEXT:    movq %rsi, %rax
-; KNL-NEXT:    cmpq %rsi, %rdi
-; KNL-NEXT:    cmovoq %rdi, %rax
-; KNL-NEXT:    retq
   %t = call {i64, i1} @llvm.ssub.with.overflow.i64(i64 %v1, i64 %v2)
   %obit = extractvalue {i64, i1} %t, 1
   %ret = select i1 %obit, i64 %v1, i64 %v2
@@ -904,13 +694,6 @@ define i32 @usuboselecti32(i32 %v1, i32 %v2) {
 ; FAST-NEXT:    cmpl %esi, %edi
 ; FAST-NEXT:    cmovbl %edi, %eax
 ; FAST-NEXT:    retq
-;
-; KNL-LABEL: usuboselecti32:
-; KNL:       ## %bb.0:
-; KNL-NEXT:    movl %esi, %eax
-; KNL-NEXT:    cmpl %esi, %edi
-; KNL-NEXT:    cmovbl %edi, %eax
-; KNL-NEXT:    retq
   %t = call {i32, i1} @llvm.usub.with.overflow.i32(i32 %v1, i32 %v2)
   %obit = extractvalue {i32, i1} %t, 1
   %ret = select i1 %obit, i32 %v1, i32 %v2
@@ -931,13 +714,6 @@ define i64 @usuboselecti64(i64 %v1, i64 %v2) {
 ; FAST-NEXT:    cmpq %rsi, %rdi
 ; FAST-NEXT:    cmovbq %rdi, %rax
 ; FAST-NEXT:    retq
-;
-; KNL-LABEL: usuboselecti64:
-; KNL:       ## %bb.0:
-; KNL-NEXT:    movq %rsi, %rax
-; KNL-NEXT:    cmpq %rsi, %rdi
-; KNL-NEXT:    cmovbq %rdi, %rax
-; KNL-NEXT:    retq
   %t = call {i64, i1} @llvm.usub.with.overflow.i64(i64 %v1, i64 %v2)
   %obit = extractvalue {i64, i1} %t, 1
   %ret = select i1 %obit, i64 %v1, i64 %v2
@@ -973,17 +749,6 @@ define zeroext i1 @saddobri32(i32 %v1, i32 %v2) {
 ; FAST-NEXT:    andb $1, %al
 ; FAST-NEXT:    movzbl %al, %eax
 ; FAST-NEXT:    retq
-;
-; KNL-LABEL: saddobri32:
-; KNL:       ## %bb.0:
-; KNL-NEXT:    addl %esi, %edi
-; KNL-NEXT:    jo LBB31_1
-; KNL-NEXT:  ## %bb.2: ## %continue
-; KNL-NEXT:    movb $1, %al
-; KNL-NEXT:    retq
-; KNL-NEXT:  LBB31_1: ## %overflow
-; KNL-NEXT:    xorl %eax, %eax
-; KNL-NEXT:    retq
   %t = call {i32, i1} @llvm.sadd.with.overflow.i32(i32 %v1, i32 %v2)
   %val = extractvalue {i32, i1} %t, 0
   %obit = extractvalue {i32, i1} %t, 1
@@ -1022,17 +787,6 @@ define zeroext i1 @saddobri64(i64 %v1, i64 %v2) {
 ; FAST-NEXT:    andb $1, %al
 ; FAST-NEXT:    movzbl %al, %eax
 ; FAST-NEXT:    retq
-;
-; KNL-LABEL: saddobri64:
-; KNL:       ## %bb.0:
-; KNL-NEXT:    addq %rsi, %rdi
-; KNL-NEXT:    jo LBB32_1
-; KNL-NEXT:  ## %bb.2: ## %continue
-; KNL-NEXT:    movb $1, %al
-; KNL-NEXT:    retq
-; KNL-NEXT:  LBB32_1: ## %overflow
-; KNL-NEXT:    xorl %eax, %eax
-; KNL-NEXT:    retq
   %t = call {i64, i1} @llvm.sadd.with.overflow.i64(i64 %v1, i64 %v2)
   %val = extractvalue {i64, i1} %t, 0
   %obit = extractvalue {i64, i1} %t, 1
@@ -1071,17 +825,6 @@ define zeroext i1 @uaddobri32(i32 %v1, i32 %v2) {
 ; FAST-NEXT:    andb $1, %al
 ; FAST-NEXT:    movzbl %al, %eax
 ; FAST-NEXT:    retq
-;
-; KNL-LABEL: uaddobri32:
-; KNL:       ## %bb.0:
-; KNL-NEXT:    addl %esi, %edi
-; KNL-NEXT:    jb LBB33_1
-; KNL-NEXT:  ## %bb.2: ## %continue
-; KNL-NEXT:    movb $1, %al
-; KNL-NEXT:    retq
-; KNL-NEXT:  LBB33_1: ## %overflow
-; KNL-NEXT:    xorl %eax, %eax
-; KNL-NEXT:    retq
   %t = call {i32, i1} @llvm.uadd.with.overflow.i32(i32 %v1, i32 %v2)
   %val = extractvalue {i32, i1} %t, 0
   %obit = extractvalue {i32, i1} %t, 1
@@ -1120,17 +863,6 @@ define zeroext i1 @uaddobri64(i64 %v1, i64 %v2) {
 ; FAST-NEXT:    andb $1, %al
 ; FAST-NEXT:    movzbl %al, %eax
 ; FAST-NEXT:    retq
-;
-; KNL-LABEL: uaddobri64:
-; KNL:       ## %bb.0:
-; KNL-NEXT:    addq %rsi, %rdi
-; KNL-NEXT:    jb LBB34_1
-; KNL-NEXT:  ## %bb.2: ## %continue
-; KNL-NEXT:    movb $1, %al
-; KNL-NEXT:    retq
-; KNL-NEXT:  LBB34_1: ## %overflow
-; KNL-NEXT:    xorl %eax, %eax
-; KNL-NEXT:    retq
   %t = call {i64, i1} @llvm.uadd.with.overflow.i64(i64 %v1, i64 %v2)
   %val = extractvalue {i64, i1} %t, 0
   %obit = extractvalue {i64, i1} %t, 1
@@ -1169,17 +901,6 @@ define zeroext i1 @ssubobri32(i32 %v1, i32 %v2) {
 ; FAST-NEXT:    andb $1, %al
 ; FAST-NEXT:    movzbl %al, %eax
 ; FAST-NEXT:    retq
-;
-; KNL-LABEL: ssubobri32:
-; KNL:       ## %bb.0:
-; KNL-NEXT:    cmpl %esi, %edi
-; KNL-NEXT:    jo LBB35_1
-; KNL-NEXT:  ## %bb.2: ## %continue
-; KNL-NEXT:    movb $1, %al
-; KNL-NEXT:    retq
-; KNL-NEXT:  LBB35_1: ## %overflow
-; KNL-NEXT:    xorl %eax, %eax
-; KNL-NEXT:    retq
   %t = call {i32, i1} @llvm.ssub.with.overflow.i32(i32 %v1, i32 %v2)
   %val = extractvalue {i32, i1} %t, 0
   %obit = extractvalue {i32, i1} %t, 1
@@ -1218,17 +939,6 @@ define zeroext i1 @ssubobri64(i64 %v1, i64 %v2) {
 ; FAST-NEXT:    andb $1, %al
 ; FAST-NEXT:    movzbl %al, %eax
 ; FAST-NEXT:    retq
-;
-; KNL-LABEL: ssubobri64:
-; KNL:       ## %bb.0:
-; KNL-NEXT:    cmpq %rsi, %rdi
-; KNL-NEXT:    jo LBB36_1
-; KNL-NEXT:  ## %bb.2: ## %continue
-; KNL-NEXT:    movb $1, %al
-; KNL-NEXT:    retq
-; KNL-NEXT:  LBB36_1: ## %overflow
-; KNL-NEXT:    xorl %eax, %eax
-; KNL-NEXT:    retq
   %t = call {i64, i1} @llvm.ssub.with.overflow.i64(i64 %v1, i64 %v2)
   %val = extractvalue {i64, i1} %t, 0
   %obit = extractvalue {i64, i1} %t, 1
@@ -1267,17 +977,6 @@ define zeroext i1 @usubobri32(i32 %v1, i32 %v2) {
 ; FAST-NEXT:    andb $1, %al
 ; FAST-NEXT:    movzbl %al, %eax
 ; FAST-NEXT:    retq
-;
-; KNL-LABEL: usubobri32:
-; KNL:       ## %bb.0:
-; KNL-NEXT:    cmpl %esi, %edi
-; KNL-NEXT:    jb LBB37_1
-; KNL-NEXT:  ## %bb.2: ## %continue
-; KNL-NEXT:    movb $1, %al
-; KNL-NEXT:    retq
-; KNL-NEXT:  LBB37_1: ## %overflow
-; KNL-NEXT:    xorl %eax, %eax
-; KNL-NEXT:    retq
   %t = call {i32, i1} @llvm.usub.with.overflow.i32(i32 %v1, i32 %v2)
   %val = extractvalue {i32, i1} %t, 0
   %obit = extractvalue {i32, i1} %t, 1
@@ -1316,17 +1015,6 @@ define zeroext i1 @usubobri64(i64 %v1, i64 %v2) {
 ; FAST-NEXT:    andb $1, %al
 ; FAST-NEXT:    movzbl %al, %eax
 ; FAST-NEXT:    retq
-;
-; KNL-LABEL: usubobri64:
-; KNL:       ## %bb.0:
-; KNL-NEXT:    cmpq %rsi, %rdi
-; KNL-NEXT:    jb LBB38_1
-; KNL-NEXT:  ## %bb.2: ## %continue
-; KNL-NEXT:    movb $1, %al
-; KNL-NEXT:    retq
-; KNL-NEXT:  LBB38_1: ## %overflow
-; KNL-NEXT:    xorl %eax, %eax
-; KNL-NEXT:    retq
   %t = call {i64, i1} @llvm.usub.with.overflow.i64(i64 %v1, i64 %v2)
   %val = extractvalue {i64, i1} %t, 0
   %obit = extractvalue {i64, i1} %t, 1
@@ -1355,14 +1043,6 @@ define {i64, i1} @uaddoovf(i64 %a, i64 %b) {
 ; FAST-NEXT:    addq %rcx, %rax
 ; FAST-NEXT:    xorl %edx, %edx
 ; FAST-NEXT:    retq
-;
-; KNL-LABEL: uaddoovf:
-; KNL:       ## %bb.0:
-; KNL-NEXT:    movzbl %dil, %ecx
-; KNL-NEXT:    movzbl %sil, %eax
-; KNL-NEXT:    addq %rcx, %rax
-; KNL-NEXT:    xorl %edx, %edx
-; KNL-NEXT:    retq
   %1 = and i64 %a, 255
   %2 = and i64 %b, 255
   %t = call {i64, i1} @llvm.uadd.with.overflow.i64(i64 %1, i64 %2)
@@ -1383,13 +1063,6 @@ define {i64, i1} @usuboovf(i64 %a, i64 %b) {
 ; FAST-NEXT:    notq %rax
 ; FAST-NEXT:    xorl %edx, %edx
 ; FAST-NEXT:    retq
-;
-; KNL-LABEL: usuboovf:
-; KNL:       ## %bb.0:
-; KNL-NEXT:    movq %rsi, %rax
-; KNL-NEXT:    notq %rax
-; KNL-NEXT:    xorl %edx, %edx
-; KNL-NEXT:    retq
   %t0 = call {i64, i1} @llvm.usub.with.overflow.i64(i64 %a, i64 %a)
   %v0 = extractvalue {i64, i1} %t0, 0
   %o0 = extractvalue {i64, i1} %t0, 1
@@ -1405,6 +1078,60 @@ define {i64, i1} @usuboovf(i64 %a, i64 %b) {
   ret {i64, i1} %t
 }
 
+; Make sure we select an INC for both the data use and the flag use.
+define i32 @incovfselectstore(i32 %v1, i32 %v2, i32* %x) {
+; SDAG-LABEL: incovfselectstore:
+; SDAG:       ## %bb.0:
+; SDAG-NEXT:    movl %esi, %eax
+; SDAG-NEXT:    movl %edi, %ecx
+; SDAG-NEXT:    incl %ecx
+; SDAG-NEXT:    cmovol %edi, %eax
+; SDAG-NEXT:    movl %ecx, (%rdx)
+; SDAG-NEXT:    retq
+;
+; FAST-LABEL: incovfselectstore:
+; FAST:       ## %bb.0:
+; FAST-NEXT:    movl %esi, %eax
+; FAST-NEXT:    movl %edi, %ecx
+; FAST-NEXT:    incl %ecx
+; FAST-NEXT:    cmovol %edi, %eax
+; FAST-NEXT:    movl %ecx, (%rdx)
+; FAST-NEXT:    retq
+  %t = call {i32, i1} @llvm.sadd.with.overflow.i32(i32 %v1, i32 1)
+  %obit = extractvalue {i32, i1} %t, 1
+  %ret = select i1 %obit, i32 %v1, i32 %v2
+  %val = extractvalue {i32, i1} %t, 0
+  store i32 %val, i32* %x
+  ret i32 %ret
+}
+
+; Make sure we select a DEC for both the data use and the flag use.
+define i32 @decovfselectstore(i32 %v1, i32 %v2, i32* %x) {
+; SDAG-LABEL: decovfselectstore:
+; SDAG:       ## %bb.0:
+; SDAG-NEXT:    movl %esi, %eax
+; SDAG-NEXT:    movl %edi, %ecx
+; SDAG-NEXT:    decl %ecx
+; SDAG-NEXT:    cmovol %edi, %eax
+; SDAG-NEXT:    movl %ecx, (%rdx)
+; SDAG-NEXT:    retq
+;
+; FAST-LABEL: decovfselectstore:
+; FAST:       ## %bb.0:
+; FAST-NEXT:    movl %esi, %eax
+; FAST-NEXT:    movl %edi, %ecx
+; FAST-NEXT:    decl %ecx
+; FAST-NEXT:    cmovol %edi, %eax
+; FAST-NEXT:    movl %ecx, (%rdx)
+; FAST-NEXT:    retq
+  %t = call {i32, i1} @llvm.ssub.with.overflow.i32(i32 %v1, i32 1)
+  %obit = extractvalue {i32, i1} %t, 1
+  %ret = select i1 %obit, i32 %v1, i32 %v2
+  %val = extractvalue {i32, i1} %t, 0
+  store i32 %val, i32* %x
+  ret i32 %ret
+}
+
 declare {i8,  i1} @llvm.sadd.with.overflow.i8 (i8,  i8 ) nounwind readnone
 declare {i16, i1} @llvm.sadd.with.overflow.i16(i16, i16) nounwind readnone
 declare {i32, i1} @llvm.sadd.with.overflow.i32(i32, i32) nounwind readnone
diff --git a/test/CodeGen/X86/xmulo.ll b/test/CodeGen/X86/xmulo.ll
index 8d2e81fb54ad1996ccca9f2f8b4ebf27308196ff..86b15cce036f6efb01be4cf337091c74f1f043ba 100644
--- a/test/CodeGen/X86/xmulo.ll
+++ b/test/CodeGen/X86/xmulo.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=x86_64-darwin-unknown < %s | FileCheck %s --check-prefix=SDAG
-; RUN: llc -mtriple=x86_64-darwin-unknown -fast-isel -fast-isel-abort=1 < %s | FileCheck %s --check-prefix=FAST
-; RUN: llc -mtriple=x86_64-darwin-unknown -mcpu=knl < %s | FileCheck %s --check-prefix=KNL
+; RUN: llc -disable-peephole -mtriple=x86_64-darwin-unknown < %s | FileCheck %s --check-prefix=SDAG
+; RUN: llc -disable-peephole -mtriple=x86_64-darwin-unknown -fast-isel -fast-isel-abort=1 < %s | FileCheck %s --check-prefix=FAST
+; RUN: llc -disable-peephole -mtriple=x86_64-darwin-unknown -mcpu=knl < %s | FileCheck %s --check-prefix=SDAG --check-prefix=KNL
 
 define {i64, i1} @t1() nounwind {
 ; SDAG-LABEL: t1:
@@ -19,14 +19,6 @@ define {i64, i1} @t1() nounwind {
 ; FAST-NEXT:    mulq %rcx
 ; FAST-NEXT:    seto %dl
 ; FAST-NEXT:    retq
-;
-; KNL-LABEL: t1:
-; KNL:       ## %bb.0:
-; KNL-NEXT:    movl $8, %ecx
-; KNL-NEXT:    movl $9, %eax
-; KNL-NEXT:    mulq %rcx
-; KNL-NEXT:    seto %dl
-; KNL-NEXT:    retq
   %1 = call {i64, i1} @llvm.umul.with.overflow.i64(i64 9, i64 8)
   ret {i64, i1} %1
 }
@@ -47,14 +39,6 @@ define {i64, i1} @t2() nounwind {
 ; FAST-NEXT:    mulq %rcx
 ; FAST-NEXT:    seto %dl
 ; FAST-NEXT:    retq
-;
-; KNL-LABEL: t2:
-; KNL:       ## %bb.0:
-; KNL-NEXT:    xorl %ecx, %ecx
-; KNL-NEXT:    movl $9, %eax
-; KNL-NEXT:    mulq %rcx
-; KNL-NEXT:    seto %dl
-; KNL-NEXT:    retq
   %1 = call {i64, i1} @llvm.umul.with.overflow.i64(i64 9, i64 0)
   ret {i64, i1} %1
 }
@@ -75,14 +59,6 @@ define {i64, i1} @t3() nounwind {
 ; FAST-NEXT:    mulq %rcx
 ; FAST-NEXT:    seto %dl
 ; FAST-NEXT:    retq
-;
-; KNL-LABEL: t3:
-; KNL:       ## %bb.0:
-; KNL-NEXT:    movq $-1, %rcx
-; KNL-NEXT:    movl $9, %eax
-; KNL-NEXT:    mulq %rcx
-; KNL-NEXT:    seto %dl
-; KNL-NEXT:    retq
   %1 = call {i64, i1} @llvm.umul.with.overflow.i64(i64 9, i64 -1)
   ret {i64, i1} %1
 }
@@ -109,16 +85,6 @@ define zeroext i1 @smuloi8(i8 %v1, i8 %v2, i8* %res) {
 ; FAST-NEXT:    andb $1, %cl
 ; FAST-NEXT:    movzbl %cl, %eax
 ; FAST-NEXT:    retq
-;
-; KNL-LABEL: smuloi8:
-; KNL:       ## %bb.0:
-; KNL-NEXT:    movl %edi, %eax
-; KNL-NEXT:    ## kill: def $al killed $al killed $eax
-; KNL-NEXT:    imulb %sil
-; KNL-NEXT:    seto %cl
-; KNL-NEXT:    movb %al, (%rdx)
-; KNL-NEXT:    movl %ecx, %eax
-; KNL-NEXT:    retq
   %t = call {i8, i1} @llvm.smul.with.overflow.i8(i8 %v1, i8 %v2)
   %val = extractvalue {i8, i1} %t, 0
   %obit = extractvalue {i8, i1} %t, 1
@@ -142,13 +108,6 @@ define zeroext i1 @smuloi16(i16 %v1, i16 %v2, i16* %res) {
 ; FAST-NEXT:    andb $1, %al
 ; FAST-NEXT:    movzbl %al, %eax
 ; FAST-NEXT:    retq
-;
-; KNL-LABEL: smuloi16:
-; KNL:       ## %bb.0:
-; KNL-NEXT:    imulw %si, %di
-; KNL-NEXT:    seto %al
-; KNL-NEXT:    movw %di, (%rdx)
-; KNL-NEXT:    retq
   %t = call {i16, i1} @llvm.smul.with.overflow.i16(i16 %v1, i16 %v2)
   %val = extractvalue {i16, i1} %t, 0
   %obit = extractvalue {i16, i1} %t, 1
@@ -172,13 +131,6 @@ define zeroext i1 @smuloi32(i32 %v1, i32 %v2, i32* %res) {
 ; FAST-NEXT:    andb $1, %al
 ; FAST-NEXT:    movzbl %al, %eax
 ; FAST-NEXT:    retq
-;
-; KNL-LABEL: smuloi32:
-; KNL:       ## %bb.0:
-; KNL-NEXT:    imull %esi, %edi
-; KNL-NEXT:    seto %al
-; KNL-NEXT:    movl %edi, (%rdx)
-; KNL-NEXT:    retq
   %t = call {i32, i1} @llvm.smul.with.overflow.i32(i32 %v1, i32 %v2)
   %val = extractvalue {i32, i1} %t, 0
   %obit = extractvalue {i32, i1} %t, 1
@@ -202,13 +154,6 @@ define zeroext i1 @smuloi64(i64 %v1, i64 %v2, i64* %res) {
 ; FAST-NEXT:    andb $1, %al
 ; FAST-NEXT:    movzbl %al, %eax
 ; FAST-NEXT:    retq
-;
-; KNL-LABEL: smuloi64:
-; KNL:       ## %bb.0:
-; KNL-NEXT:    imulq %rsi, %rdi
-; KNL-NEXT:    seto %al
-; KNL-NEXT:    movq %rdi, (%rdx)
-; KNL-NEXT:    retq
   %t = call {i64, i1} @llvm.smul.with.overflow.i64(i64 %v1, i64 %v2)
   %val = extractvalue {i64, i1} %t, 0
   %obit = extractvalue {i64, i1} %t, 1
@@ -238,16 +183,6 @@ define zeroext i1 @umuloi8(i8 %v1, i8 %v2, i8* %res) {
 ; FAST-NEXT:    andb $1, %cl
 ; FAST-NEXT:    movzbl %cl, %eax
 ; FAST-NEXT:    retq
-;
-; KNL-LABEL: umuloi8:
-; KNL:       ## %bb.0:
-; KNL-NEXT:    movl %edi, %eax
-; KNL-NEXT:    ## kill: def $al killed $al killed $eax
-; KNL-NEXT:    mulb %sil
-; KNL-NEXT:    seto %cl
-; KNL-NEXT:    movb %al, (%rdx)
-; KNL-NEXT:    movl %ecx, %eax
-; KNL-NEXT:    retq
   %t = call {i8, i1} @llvm.umul.with.overflow.i8(i8 %v1, i8 %v2)
   %val = extractvalue {i8, i1} %t, 0
   %obit = extractvalue {i8, i1} %t, 1
@@ -278,17 +213,6 @@ define zeroext i1 @umuloi16(i16 %v1, i16 %v2, i16* %res) {
 ; FAST-NEXT:    andb $1, %dl
 ; FAST-NEXT:    movzbl %dl, %eax
 ; FAST-NEXT:    retq
-;
-; KNL-LABEL: umuloi16:
-; KNL:       ## %bb.0:
-; KNL-NEXT:    movq %rdx, %rcx
-; KNL-NEXT:    movl %edi, %eax
-; KNL-NEXT:    ## kill: def $ax killed $ax killed $eax
-; KNL-NEXT:    mulw %si
-; KNL-NEXT:    seto %dl
-; KNL-NEXT:    movw %ax, (%rcx)
-; KNL-NEXT:    movl %edx, %eax
-; KNL-NEXT:    retq
   %t = call {i16, i1} @llvm.umul.with.overflow.i16(i16 %v1, i16 %v2)
   %val = extractvalue {i16, i1} %t, 0
   %obit = extractvalue {i16, i1} %t, 1
@@ -317,16 +241,6 @@ define zeroext i1 @umuloi32(i32 %v1, i32 %v2, i32* %res) {
 ; FAST-NEXT:    andb $1, %dl
 ; FAST-NEXT:    movzbl %dl, %eax
 ; FAST-NEXT:    retq
-;
-; KNL-LABEL: umuloi32:
-; KNL:       ## %bb.0:
-; KNL-NEXT:    movq %rdx, %rcx
-; KNL-NEXT:    movl %edi, %eax
-; KNL-NEXT:    mull %esi
-; KNL-NEXT:    seto %dl
-; KNL-NEXT:    movl %eax, (%rcx)
-; KNL-NEXT:    movl %edx, %eax
-; KNL-NEXT:    retq
   %t = call {i32, i1} @llvm.umul.with.overflow.i32(i32 %v1, i32 %v2)
   %val = extractvalue {i32, i1} %t, 0
   %obit = extractvalue {i32, i1} %t, 1
@@ -355,16 +269,6 @@ define zeroext i1 @umuloi64(i64 %v1, i64 %v2, i64* %res) {
 ; FAST-NEXT:    andb $1, %dl
 ; FAST-NEXT:    movzbl %dl, %eax
 ; FAST-NEXT:    retq
-;
-; KNL-LABEL: umuloi64:
-; KNL:       ## %bb.0:
-; KNL-NEXT:    movq %rdx, %rcx
-; KNL-NEXT:    movq %rdi, %rax
-; KNL-NEXT:    mulq %rsi
-; KNL-NEXT:    seto %dl
-; KNL-NEXT:    movq %rax, (%rcx)
-; KNL-NEXT:    movl %edx, %eax
-; KNL-NEXT:    retq
   %t = call {i64, i1} @llvm.umul.with.overflow.i64(i64 %v1, i64 %v2)
   %val = extractvalue {i64, i1} %t, 0
   %obit = extractvalue {i64, i1} %t, 1
@@ -391,14 +295,6 @@ define i32 @smuloselecti32(i32 %v1, i32 %v2) {
 ; FAST-NEXT:    imull %esi, %ecx
 ; FAST-NEXT:    cmovol %edi, %eax
 ; FAST-NEXT:    retq
-;
-; KNL-LABEL: smuloselecti32:
-; KNL:       ## %bb.0:
-; KNL-NEXT:    movl %esi, %eax
-; KNL-NEXT:    movl %edi, %ecx
-; KNL-NEXT:    imull %esi, %ecx
-; KNL-NEXT:    cmovol %edi, %eax
-; KNL-NEXT:    retq
   %t = call {i32, i1} @llvm.smul.with.overflow.i32(i32 %v1, i32 %v2)
   %obit = extractvalue {i32, i1} %t, 1
   %ret = select i1 %obit, i32 %v1, i32 %v2
@@ -421,14 +317,6 @@ define i64 @smuloselecti64(i64 %v1, i64 %v2) {
 ; FAST-NEXT:    imulq %rsi, %rcx
 ; FAST-NEXT:    cmovoq %rdi, %rax
 ; FAST-NEXT:    retq
-;
-; KNL-LABEL: smuloselecti64:
-; KNL:       ## %bb.0:
-; KNL-NEXT:    movq %rsi, %rax
-; KNL-NEXT:    movq %rdi, %rcx
-; KNL-NEXT:    imulq %rsi, %rcx
-; KNL-NEXT:    cmovoq %rdi, %rax
-; KNL-NEXT:    retq
   %t = call {i64, i1} @llvm.smul.with.overflow.i64(i64 %v1, i64 %v2)
   %obit = extractvalue {i64, i1} %t, 1
   %ret = select i1 %obit, i64 %v1, i64 %v2
@@ -451,14 +339,6 @@ define i32 @umuloselecti32(i32 %v1, i32 %v2) {
 ; FAST-NEXT:    cmovol %edi, %esi
 ; FAST-NEXT:    movl %esi, %eax
 ; FAST-NEXT:    retq
-;
-; KNL-LABEL: umuloselecti32:
-; KNL:       ## %bb.0:
-; KNL-NEXT:    movl %edi, %eax
-; KNL-NEXT:    mull %esi
-; KNL-NEXT:    cmovol %edi, %esi
-; KNL-NEXT:    movl %esi, %eax
-; KNL-NEXT:    retq
   %t = call {i32, i1} @llvm.umul.with.overflow.i32(i32 %v1, i32 %v2)
   %obit = extractvalue {i32, i1} %t, 1
   %ret = select i1 %obit, i32 %v1, i32 %v2
@@ -481,14 +361,6 @@ define i64 @umuloselecti64(i64 %v1, i64 %v2) {
 ; FAST-NEXT:    cmovoq %rdi, %rsi
 ; FAST-NEXT:    movq %rsi, %rax
 ; FAST-NEXT:    retq
-;
-; KNL-LABEL: umuloselecti64:
-; KNL:       ## %bb.0:
-; KNL-NEXT:    movq %rdi, %rax
-; KNL-NEXT:    mulq %rsi
-; KNL-NEXT:    cmovoq %rdi, %rsi
-; KNL-NEXT:    movq %rsi, %rax
-; KNL-NEXT:    retq
   %t = call {i64, i1} @llvm.umul.with.overflow.i64(i64 %v1, i64 %v2)
   %obit = extractvalue {i64, i1} %t, 1
   %ret = select i1 %obit, i64 %v1, i64 %v2
@@ -498,43 +370,116 @@ define i64 @umuloselecti64(i64 %v1, i64 %v2) {
 ;
 ; Check the use of the overflow bit in combination with a branch instruction.
 ;
+define zeroext i1 @smulobri8(i8 %v1, i8 %v2) {
+; SDAG-LABEL: smulobri8:
+; SDAG:       ## %bb.0:
+; SDAG-NEXT:    movl %edi, %eax
+; SDAG-NEXT:    ## kill: def $al killed $al killed $eax
+; SDAG-NEXT:    imulb %sil
+; SDAG-NEXT:    jo LBB15_1
+; SDAG-NEXT:  ## %bb.2: ## %continue
+; SDAG-NEXT:    movb $1, %al
+; SDAG-NEXT:    retq
+; SDAG-NEXT:  LBB15_1: ## %overflow
+; SDAG-NEXT:    xorl %eax, %eax
+; SDAG-NEXT:    retq
+;
+; FAST-LABEL: smulobri8:
+; FAST:       ## %bb.0:
+; FAST-NEXT:    movl %edi, %eax
+; FAST-NEXT:    ## kill: def $al killed $al killed $eax
+; FAST-NEXT:    imulb %sil
+; FAST-NEXT:    seto %al
+; FAST-NEXT:    testb $1, %al
+; FAST-NEXT:    jne LBB15_1
+; FAST-NEXT:  ## %bb.2: ## %continue
+; FAST-NEXT:    movb $1, %al
+; FAST-NEXT:    andb $1, %al
+; FAST-NEXT:    movzbl %al, %eax
+; FAST-NEXT:    retq
+; FAST-NEXT:  LBB15_1: ## %overflow
+; FAST-NEXT:    xorl %eax, %eax
+; FAST-NEXT:    andb $1, %al
+; FAST-NEXT:    movzbl %al, %eax
+; FAST-NEXT:    retq
+  %t = call {i8, i1} @llvm.smul.with.overflow.i8(i8 %v1, i8 %v2)
+  %val = extractvalue {i8, i1} %t, 0
+  %obit = extractvalue {i8, i1} %t, 1
+  br i1 %obit, label %overflow, label %continue, !prof !0
+
+overflow:
+  ret i1 false
+
+continue:
+  ret i1 true
+}
+
+define zeroext i1 @smulobri16(i16 %v1, i16 %v2) {
+; SDAG-LABEL: smulobri16:
+; SDAG:       ## %bb.0:
+; SDAG-NEXT:    imulw %si, %di
+; SDAG-NEXT:    jo LBB16_1
+; SDAG-NEXT:  ## %bb.2: ## %continue
+; SDAG-NEXT:    movb $1, %al
+; SDAG-NEXT:    retq
+; SDAG-NEXT:  LBB16_1: ## %overflow
+; SDAG-NEXT:    xorl %eax, %eax
+; SDAG-NEXT:    retq
+;
+; FAST-LABEL: smulobri16:
+; FAST:       ## %bb.0:
+; FAST-NEXT:    imulw %si, %di
+; FAST-NEXT:    seto %al
+; FAST-NEXT:    testb $1, %al
+; FAST-NEXT:    jne LBB16_1
+; FAST-NEXT:  ## %bb.2: ## %continue
+; FAST-NEXT:    movb $1, %al
+; FAST-NEXT:    andb $1, %al
+; FAST-NEXT:    movzbl %al, %eax
+; FAST-NEXT:    retq
+; FAST-NEXT:  LBB16_1: ## %overflow
+; FAST-NEXT:    xorl %eax, %eax
+; FAST-NEXT:    andb $1, %al
+; FAST-NEXT:    movzbl %al, %eax
+; FAST-NEXT:    retq
+  %t = call {i16, i1} @llvm.smul.with.overflow.i16(i16 %v1, i16 %v2)
+  %val = extractvalue {i16, i1} %t, 0
+  %obit = extractvalue {i16, i1} %t, 1
+  br i1 %obit, label %overflow, label %continue, !prof !0
+
+overflow:
+  ret i1 false
+
+continue:
+  ret i1 true
+}
+
 define zeroext i1 @smulobri32(i32 %v1, i32 %v2) {
 ; SDAG-LABEL: smulobri32:
 ; SDAG:       ## %bb.0:
 ; SDAG-NEXT:    imull %esi, %edi
-; SDAG-NEXT:    jo LBB15_1
+; SDAG-NEXT:    jo LBB17_1
 ; SDAG-NEXT:  ## %bb.2: ## %continue
 ; SDAG-NEXT:    movb $1, %al
 ; SDAG-NEXT:    retq
-; SDAG-NEXT:  LBB15_1: ## %overflow
+; SDAG-NEXT:  LBB17_1: ## %overflow
 ; SDAG-NEXT:    xorl %eax, %eax
 ; SDAG-NEXT:    retq
 ;
 ; FAST-LABEL: smulobri32:
 ; FAST:       ## %bb.0:
 ; FAST-NEXT:    imull %esi, %edi
-; FAST-NEXT:    jo LBB15_1
+; FAST-NEXT:    jo LBB17_1
 ; FAST-NEXT:  ## %bb.2: ## %continue
 ; FAST-NEXT:    movb $1, %al
 ; FAST-NEXT:    andb $1, %al
 ; FAST-NEXT:    movzbl %al, %eax
 ; FAST-NEXT:    retq
-; FAST-NEXT:  LBB15_1: ## %overflow
+; FAST-NEXT:  LBB17_1: ## %overflow
 ; FAST-NEXT:    xorl %eax, %eax
 ; FAST-NEXT:    andb $1, %al
 ; FAST-NEXT:    movzbl %al, %eax
 ; FAST-NEXT:    retq
-;
-; KNL-LABEL: smulobri32:
-; KNL:       ## %bb.0:
-; KNL-NEXT:    imull %esi, %edi
-; KNL-NEXT:    jo LBB15_1
-; KNL-NEXT:  ## %bb.2: ## %continue
-; KNL-NEXT:    movb $1, %al
-; KNL-NEXT:    retq
-; KNL-NEXT:  LBB15_1: ## %overflow
-; KNL-NEXT:    xorl %eax, %eax
-; KNL-NEXT:    retq
   %t = call {i32, i1} @llvm.smul.with.overflow.i32(i32 %v1, i32 %v2)
   %val = extractvalue {i32, i1} %t, 0
   %obit = extractvalue {i32, i1} %t, 1
@@ -551,39 +496,28 @@ define zeroext i1 @smulobri64(i64 %v1, i64 %v2) {
 ; SDAG-LABEL: smulobri64:
 ; SDAG:       ## %bb.0:
 ; SDAG-NEXT:    imulq %rsi, %rdi
-; SDAG-NEXT:    jo LBB16_1
+; SDAG-NEXT:    jo LBB18_1
 ; SDAG-NEXT:  ## %bb.2: ## %continue
 ; SDAG-NEXT:    movb $1, %al
 ; SDAG-NEXT:    retq
-; SDAG-NEXT:  LBB16_1: ## %overflow
+; SDAG-NEXT:  LBB18_1: ## %overflow
 ; SDAG-NEXT:    xorl %eax, %eax
 ; SDAG-NEXT:    retq
 ;
 ; FAST-LABEL: smulobri64:
 ; FAST:       ## %bb.0:
 ; FAST-NEXT:    imulq %rsi, %rdi
-; FAST-NEXT:    jo LBB16_1
+; FAST-NEXT:    jo LBB18_1
 ; FAST-NEXT:  ## %bb.2: ## %continue
 ; FAST-NEXT:    movb $1, %al
 ; FAST-NEXT:    andb $1, %al
 ; FAST-NEXT:    movzbl %al, %eax
 ; FAST-NEXT:    retq
-; FAST-NEXT:  LBB16_1: ## %overflow
+; FAST-NEXT:  LBB18_1: ## %overflow
 ; FAST-NEXT:    xorl %eax, %eax
 ; FAST-NEXT:    andb $1, %al
 ; FAST-NEXT:    movzbl %al, %eax
 ; FAST-NEXT:    retq
-;
-; KNL-LABEL: smulobri64:
-; KNL:       ## %bb.0:
-; KNL-NEXT:    imulq %rsi, %rdi
-; KNL-NEXT:    jo LBB16_1
-; KNL-NEXT:  ## %bb.2: ## %continue
-; KNL-NEXT:    movb $1, %al
-; KNL-NEXT:    retq
-; KNL-NEXT:  LBB16_1: ## %overflow
-; KNL-NEXT:    xorl %eax, %eax
-; KNL-NEXT:    retq
   %t = call {i64, i1} @llvm.smul.with.overflow.i64(i64 %v1, i64 %v2)
   %val = extractvalue {i64, i1} %t, 0
   %obit = extractvalue {i64, i1} %t, 1
@@ -596,16 +530,104 @@ continue:
   ret i1 true
 }
 
+define zeroext i1 @umulobri8(i8 %v1, i8 %v2) {
+; SDAG-LABEL: umulobri8:
+; SDAG:       ## %bb.0:
+; SDAG-NEXT:    movl %edi, %eax
+; SDAG-NEXT:    ## kill: def $al killed $al killed $eax
+; SDAG-NEXT:    mulb %sil
+; SDAG-NEXT:    jo LBB19_1
+; SDAG-NEXT:  ## %bb.2: ## %continue
+; SDAG-NEXT:    movb $1, %al
+; SDAG-NEXT:    retq
+; SDAG-NEXT:  LBB19_1: ## %overflow
+; SDAG-NEXT:    xorl %eax, %eax
+; SDAG-NEXT:    retq
+;
+; FAST-LABEL: umulobri8:
+; FAST:       ## %bb.0:
+; FAST-NEXT:    movl %edi, %eax
+; FAST-NEXT:    ## kill: def $al killed $al killed $eax
+; FAST-NEXT:    mulb %sil
+; FAST-NEXT:    seto %al
+; FAST-NEXT:    testb $1, %al
+; FAST-NEXT:    jne LBB19_1
+; FAST-NEXT:  ## %bb.2: ## %continue
+; FAST-NEXT:    movb $1, %al
+; FAST-NEXT:    andb $1, %al
+; FAST-NEXT:    movzbl %al, %eax
+; FAST-NEXT:    retq
+; FAST-NEXT:  LBB19_1: ## %overflow
+; FAST-NEXT:    xorl %eax, %eax
+; FAST-NEXT:    andb $1, %al
+; FAST-NEXT:    movzbl %al, %eax
+; FAST-NEXT:    retq
+  %t = call {i8, i1} @llvm.umul.with.overflow.i8(i8 %v1, i8 %v2)
+  %val = extractvalue {i8, i1} %t, 0
+  %obit = extractvalue {i8, i1} %t, 1
+  br i1 %obit, label %overflow, label %continue, !prof !0
+
+overflow:
+  ret i1 false
+
+continue:
+  ret i1 true
+}
+
+define zeroext i1 @umulobri16(i16 %v1, i16 %v2) {
+; SDAG-LABEL: umulobri16:
+; SDAG:       ## %bb.0:
+; SDAG-NEXT:    movl %edi, %eax
+; SDAG-NEXT:    ## kill: def $ax killed $ax killed $eax
+; SDAG-NEXT:    mulw %si
+; SDAG-NEXT:    jo LBB20_1
+; SDAG-NEXT:  ## %bb.2: ## %continue
+; SDAG-NEXT:    movb $1, %al
+; SDAG-NEXT:    retq
+; SDAG-NEXT:  LBB20_1: ## %overflow
+; SDAG-NEXT:    xorl %eax, %eax
+; SDAG-NEXT:    retq
+;
+; FAST-LABEL: umulobri16:
+; FAST:       ## %bb.0:
+; FAST-NEXT:    movl %edi, %eax
+; FAST-NEXT:    ## kill: def $ax killed $ax killed $eax
+; FAST-NEXT:    mulw %si
+; FAST-NEXT:    seto %al
+; FAST-NEXT:    testb $1, %al
+; FAST-NEXT:    jne LBB20_1
+; FAST-NEXT:  ## %bb.2: ## %continue
+; FAST-NEXT:    movb $1, %al
+; FAST-NEXT:    andb $1, %al
+; FAST-NEXT:    movzbl %al, %eax
+; FAST-NEXT:    retq
+; FAST-NEXT:  LBB20_1: ## %overflow
+; FAST-NEXT:    xorl %eax, %eax
+; FAST-NEXT:    andb $1, %al
+; FAST-NEXT:    movzbl %al, %eax
+; FAST-NEXT:    retq
+  %t = call {i16, i1} @llvm.umul.with.overflow.i16(i16 %v1, i16 %v2)
+  %val = extractvalue {i16, i1} %t, 0
+  %obit = extractvalue {i16, i1} %t, 1
+  br i1 %obit, label %overflow, label %continue, !prof !0
+
+overflow:
+  ret i1 false
+
+continue:
+  ret i1 true
+}
+
 define zeroext i1 @umulobri32(i32 %v1, i32 %v2) {
 ; SDAG-LABEL: umulobri32:
 ; SDAG:       ## %bb.0:
 ; SDAG-NEXT:    movl %edi, %eax
 ; SDAG-NEXT:    mull %esi
-; SDAG-NEXT:    jo LBB17_1
+; SDAG-NEXT:    jo LBB21_1
 ; SDAG-NEXT:  ## %bb.2: ## %continue
 ; SDAG-NEXT:    movb $1, %al
 ; SDAG-NEXT:    retq
-; SDAG-NEXT:  LBB17_1: ## %overflow
+; SDAG-NEXT:  LBB21_1: ## %overflow
 ; SDAG-NEXT:    xorl %eax, %eax
 ; SDAG-NEXT:    retq
 ;
@@ -613,29 +635,17 @@ define zeroext i1 @umulobri32(i32 %v1, i32 %v2) {
 ; FAST:       ## %bb.0:
 ; FAST-NEXT:    movl %edi, %eax
 ; FAST-NEXT:    mull %esi
-; FAST-NEXT:    jo LBB17_1
+; FAST-NEXT:    jo LBB21_1
 ; FAST-NEXT:  ## %bb.2: ## %continue
 ; FAST-NEXT:    movb $1, %al
 ; FAST-NEXT:    andb $1, %al
 ; FAST-NEXT:    movzbl %al, %eax
 ; FAST-NEXT:    retq
-; FAST-NEXT:  LBB17_1: ## %overflow
+; FAST-NEXT:  LBB21_1: ## %overflow
 ; FAST-NEXT:    xorl %eax, %eax
 ; FAST-NEXT:    andb $1, %al
 ; FAST-NEXT:    movzbl %al, %eax
 ; FAST-NEXT:    retq
-;
-; KNL-LABEL: umulobri32:
-; KNL:       ## %bb.0:
-; KNL-NEXT:    movl %edi, %eax
-; KNL-NEXT:    mull %esi
-; KNL-NEXT:    jo LBB17_1
-; KNL-NEXT:  ## %bb.2: ## %continue
-; KNL-NEXT:    movb $1, %al
-; KNL-NEXT:    retq
-; KNL-NEXT:  LBB17_1: ## %overflow
-; KNL-NEXT:    xorl %eax, %eax
-; KNL-NEXT:    retq
   %t = call {i32, i1} @llvm.umul.with.overflow.i32(i32 %v1, i32 %v2)
   %val = extractvalue {i32, i1} %t, 0
   %obit = extractvalue {i32, i1} %t, 1
@@ -653,11 +663,11 @@ define zeroext i1 @umulobri64(i64 %v1, i64 %v2) {
 ; SDAG:       ## %bb.0:
 ; SDAG-NEXT:    movq %rdi, %rax
 ; SDAG-NEXT:    mulq %rsi
-; SDAG-NEXT:    jo LBB18_1
+; SDAG-NEXT:    jo LBB22_1
 ; SDAG-NEXT:  ## %bb.2: ## %continue
 ; SDAG-NEXT:    movb $1, %al
 ; SDAG-NEXT:    retq
-; SDAG-NEXT:  LBB18_1: ## %overflow
+; SDAG-NEXT:  LBB22_1: ## %overflow
 ; SDAG-NEXT:    xorl %eax, %eax
 ; SDAG-NEXT:    retq
 ;
@@ -665,29 +675,17 @@ define zeroext i1 @umulobri64(i64 %v1, i64 %v2) {
 ; FAST:       ## %bb.0:
 ; FAST-NEXT:    movq %rdi, %rax
 ; FAST-NEXT:    mulq %rsi
-; FAST-NEXT:    jo LBB18_1
+; FAST-NEXT:    jo LBB22_1
 ; FAST-NEXT:  ## %bb.2: ## %continue
 ; FAST-NEXT:    movb $1, %al
 ; FAST-NEXT:    andb $1, %al
 ; FAST-NEXT:    movzbl %al, %eax
 ; FAST-NEXT:    retq
-; FAST-NEXT:  LBB18_1: ## %overflow
+; FAST-NEXT:  LBB22_1: ## %overflow
 ; FAST-NEXT:    xorl %eax, %eax
 ; FAST-NEXT:    andb $1, %al
 ; FAST-NEXT:    movzbl %al, %eax
 ; FAST-NEXT:    retq
-;
-; KNL-LABEL: umulobri64:
-; KNL:       ## %bb.0:
-; KNL-NEXT:    movq %rdi, %rax
-; KNL-NEXT:    mulq %rsi
-; KNL-NEXT:    jo LBB18_1
-; KNL-NEXT:  ## %bb.2: ## %continue
-; KNL-NEXT:    movb $1, %al
-; KNL-NEXT:    retq
-; KNL-NEXT:  LBB18_1: ## %overflow
-; KNL-NEXT:    xorl %eax, %eax
-; KNL-NEXT:    retq
   %t = call {i64, i1} @llvm.umul.with.overflow.i64(i64 %v1, i64 %v2)
   %val = extractvalue {i64, i1} %t, 0
   %obit = extractvalue {i64, i1} %t, 1
@@ -718,21 +716,447 @@ define i1 @bug27873(i64 %c1, i1 %c2) {
 ; FAST-NEXT:    seto %al
 ; FAST-NEXT:    orb %sil, %al
 ; FAST-NEXT:    retq
-;
-; KNL-LABEL: bug27873:
-; KNL:       ## %bb.0:
-; KNL-NEXT:    movq %rdi, %rax
-; KNL-NEXT:    movl $160, %ecx
-; KNL-NEXT:    mulq %rcx
-; KNL-NEXT:    seto %al
-; KNL-NEXT:    orb %sil, %al
-; KNL-NEXT:    retq
   %mul = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 %c1, i64 160)
   %mul.overflow = extractvalue { i64, i1 } %mul, 1
   %x1 = or i1 %c2, %mul.overflow
   ret i1 %x1
 }
 
+define zeroext i1 @smuloi8_load(i8* %ptr1, i8 %v2, i8* %res) {
+; SDAG-LABEL: smuloi8_load:
+; SDAG:       ## %bb.0:
+; SDAG-NEXT:    movl %esi, %eax
+; SDAG-NEXT:    ## kill: def $al killed $al killed $eax
+; SDAG-NEXT:    imulb (%rdi)
+; SDAG-NEXT:    seto %cl
+; SDAG-NEXT:    movb %al, (%rdx)
+; SDAG-NEXT:    movl %ecx, %eax
+; SDAG-NEXT:    retq
+;
+; FAST-LABEL: smuloi8_load:
+; FAST:       ## %bb.0:
+; FAST-NEXT:    movb (%rdi), %al
+; FAST-NEXT:    imulb %sil
+; FAST-NEXT:    seto %cl
+; FAST-NEXT:    movb %al, (%rdx)
+; FAST-NEXT:    andb $1, %cl
+; FAST-NEXT:    movzbl %cl, %eax
+; FAST-NEXT:    retq
+  %v1 = load i8, i8* %ptr1
+  %t = call {i8, i1} @llvm.smul.with.overflow.i8(i8 %v1, i8 %v2)
+  %val = extractvalue {i8, i1} %t, 0
+  %obit = extractvalue {i8, i1} %t, 1
+  store i8 %val, i8* %res
+  ret i1 %obit
+}
+
+define zeroext i1 @smuloi8_load2(i8 %v1, i8* %ptr2, i8* %res) {
+; SDAG-LABEL: smuloi8_load2:
+; SDAG:       ## %bb.0:
+; SDAG-NEXT:    movl %edi, %eax
+; SDAG-NEXT:    ## kill: def $al killed $al killed $eax
+; SDAG-NEXT:    imulb (%rsi)
+; SDAG-NEXT:    seto %cl
+; SDAG-NEXT:    movb %al, (%rdx)
+; SDAG-NEXT:    movl %ecx, %eax
+; SDAG-NEXT:    retq
+;
+; FAST-LABEL: smuloi8_load2:
+; FAST:       ## %bb.0:
+; FAST-NEXT:    movl %edi, %eax
+; FAST-NEXT:    ## kill: def $al killed $al killed $eax
+; FAST-NEXT:    imulb (%rsi)
+; FAST-NEXT:    seto %cl
+; FAST-NEXT:    movb %al, (%rdx)
+; FAST-NEXT:    andb $1, %cl
+; FAST-NEXT:    movzbl %cl, %eax
+; FAST-NEXT:    retq
+  %v2 = load i8, i8* %ptr2
+  %t = call {i8, i1} @llvm.smul.with.overflow.i8(i8 %v1, i8 %v2)
+  %val = extractvalue {i8, i1} %t, 0
+  %obit = extractvalue {i8, i1} %t, 1
+  store i8 %val, i8* %res
+  ret i1 %obit
+}
+
+define zeroext i1 @smuloi16_load(i16* %ptr1, i16 %v2, i16* %res) {
+; SDAG-LABEL: smuloi16_load:
+; SDAG:       ## %bb.0:
+; SDAG-NEXT:    imulw (%rdi), %si
+; SDAG-NEXT:    seto %al
+; SDAG-NEXT:    movw %si, (%rdx)
+; SDAG-NEXT:    retq
+;
+; FAST-LABEL: smuloi16_load:
+; FAST:       ## %bb.0:
+; FAST-NEXT:    imulw (%rdi), %si
+; FAST-NEXT:    seto %al
+; FAST-NEXT:    movw %si, (%rdx)
+; FAST-NEXT:    andb $1, %al
+; FAST-NEXT:    movzbl %al, %eax
+; FAST-NEXT:    retq
+  %v1 = load i16, i16* %ptr1
+  %t = call {i16, i1} @llvm.smul.with.overflow.i16(i16 %v1, i16 %v2)
+  %val = extractvalue {i16, i1} %t, 0
+  %obit = extractvalue {i16, i1} %t, 1
+  store i16 %val, i16* %res
+  ret i1 %obit
+}
+
+define zeroext i1 @smuloi16_load2(i16 %v1, i16* %ptr2, i16* %res) {
+; SDAG-LABEL: smuloi16_load2:
+; SDAG:       ## %bb.0:
+; SDAG-NEXT:    imulw (%rsi), %di
+; SDAG-NEXT:    seto %al
+; SDAG-NEXT:    movw %di, (%rdx)
+; SDAG-NEXT:    retq
+;
+; FAST-LABEL: smuloi16_load2:
+; FAST:       ## %bb.0:
+; FAST-NEXT:    imulw (%rsi), %di
+; FAST-NEXT:    seto %al
+; FAST-NEXT:    movw %di, (%rdx)
+; FAST-NEXT:    andb $1, %al
+; FAST-NEXT:    movzbl %al, %eax
+; FAST-NEXT:    retq
+  %v2 = load i16, i16* %ptr2
+  %t = call {i16, i1} @llvm.smul.with.overflow.i16(i16 %v1, i16 %v2)
+  %val = extractvalue {i16, i1} %t, 0
+  %obit = extractvalue {i16, i1} %t, 1
+  store i16 %val, i16* %res
+  ret i1 %obit
+}
+
+define zeroext i1 @smuloi32_load(i32* %ptr1, i32 %v2, i32* %res) {
+; SDAG-LABEL: smuloi32_load:
+; SDAG:       ## %bb.0:
+; SDAG-NEXT:    imull (%rdi), %esi
+; SDAG-NEXT:    seto %al
+; SDAG-NEXT:    movl %esi, (%rdx)
+; SDAG-NEXT:    retq
+;
+; FAST-LABEL: smuloi32_load:
+; FAST:       ## %bb.0:
+; FAST-NEXT:    imull (%rdi), %esi
+; FAST-NEXT:    seto %al
+; FAST-NEXT:    movl %esi, (%rdx)
+; FAST-NEXT:    andb $1, %al
+; FAST-NEXT:    movzbl %al, %eax
+; FAST-NEXT:    retq
+  %v1 = load i32, i32* %ptr1
+  %t = call {i32, i1} @llvm.smul.with.overflow.i32(i32 %v1, i32 %v2)
+  %val = extractvalue {i32, i1} %t, 0
+  %obit = extractvalue {i32, i1} %t, 1
+  store i32 %val, i32* %res
+  ret i1 %obit
+}
+
+define zeroext i1 @smuloi32_load2(i32 %v1, i32* %ptr2, i32* %res) {
+; SDAG-LABEL: smuloi32_load2:
+; SDAG:       ## %bb.0:
+; SDAG-NEXT:    imull (%rsi), %edi
+; SDAG-NEXT:    seto %al
+; SDAG-NEXT:    movl %edi, (%rdx)
+; SDAG-NEXT:    retq
+;
+; FAST-LABEL: smuloi32_load2:
+; FAST:       ## %bb.0:
+; FAST-NEXT:    imull (%rsi), %edi
+; FAST-NEXT:    seto %al
+; FAST-NEXT:    movl %edi, (%rdx)
+; FAST-NEXT:    andb $1, %al
+; FAST-NEXT:    movzbl %al, %eax
+; FAST-NEXT:    retq
+  %v2 = load i32, i32* %ptr2
+  %t = call {i32, i1} @llvm.smul.with.overflow.i32(i32 %v1, i32 %v2)
+  %val = extractvalue {i32, i1} %t, 0
+  %obit = extractvalue {i32, i1} %t, 1
+  store i32 %val, i32* %res
+  ret i1 %obit
+}
+
+define zeroext i1 @smuloi64_load(i64* %ptr1, i64 %v2, i64* %res) {
+; SDAG-LABEL: smuloi64_load:
+; SDAG:       ## %bb.0:
+; SDAG-NEXT:    imulq (%rdi), %rsi
+; SDAG-NEXT:    seto %al
+; SDAG-NEXT:    movq %rsi, (%rdx)
+; SDAG-NEXT:    retq
+;
+; FAST-LABEL: smuloi64_load:
+; FAST:       ## %bb.0:
+; FAST-NEXT:    imulq (%rdi), %rsi
+; FAST-NEXT:    seto %al
+; FAST-NEXT:    movq %rsi, (%rdx)
+; FAST-NEXT:    andb $1, %al
+; FAST-NEXT:    movzbl %al, %eax
+; FAST-NEXT:    retq
+  %v1 = load i64, i64* %ptr1
+  %t = call {i64, i1} @llvm.smul.with.overflow.i64(i64 %v1, i64 %v2)
+  %val = extractvalue {i64, i1} %t, 0
+  %obit = extractvalue {i64, i1} %t, 1
+  store i64 %val, i64* %res
+  ret i1 %obit
+}
+
+define zeroext i1 @smuloi64_load2(i64 %v1, i64* %ptr2, i64* %res) {
+; SDAG-LABEL: smuloi64_load2:
+; SDAG:       ## %bb.0:
+; SDAG-NEXT:    imulq (%rsi), %rdi
+; SDAG-NEXT:    seto %al
+; SDAG-NEXT:    movq %rdi, (%rdx)
+; SDAG-NEXT:    retq
+;
+; FAST-LABEL: smuloi64_load2:
+; FAST:       ## %bb.0:
+; FAST-NEXT:    imulq (%rsi), %rdi
+; FAST-NEXT:    seto %al
+; FAST-NEXT:    movq %rdi, (%rdx)
+; FAST-NEXT:    andb $1, %al
+; FAST-NEXT:    movzbl %al, %eax
+; FAST-NEXT:    retq
+  %v2 = load i64, i64* %ptr2
+  %t = call {i64, i1} @llvm.smul.with.overflow.i64(i64 %v1, i64 %v2)
+  %val = extractvalue {i64, i1} %t, 0
+  %obit = extractvalue {i64, i1} %t, 1
+  store i64 %val, i64* %res
+  ret i1 %obit
+}
+
+define zeroext i1 @umuloi8_load(i8* %ptr1, i8 %v2, i8* %res) {
+; SDAG-LABEL: umuloi8_load:
+; SDAG:       ## %bb.0:
+; SDAG-NEXT:    movl %esi, %eax
+; SDAG-NEXT:    ## kill: def $al killed $al killed $eax
+; SDAG-NEXT:    mulb (%rdi)
+; SDAG-NEXT:    seto %cl
+; SDAG-NEXT:    movb %al, (%rdx)
+; SDAG-NEXT:    movl %ecx, %eax
+; SDAG-NEXT:    retq
+;
+; FAST-LABEL: umuloi8_load:
+; FAST:       ## %bb.0:
+; FAST-NEXT:    movb (%rdi), %al
+; FAST-NEXT:    mulb %sil
+; FAST-NEXT:    seto %cl
+; FAST-NEXT:    movb %al, (%rdx)
+; FAST-NEXT:    andb $1, %cl
+; FAST-NEXT:    movzbl %cl, %eax
+; FAST-NEXT:    retq
+  %v1 = load i8, i8* %ptr1
+  %t = call {i8, i1} @llvm.umul.with.overflow.i8(i8 %v1, i8 %v2)
+  %val = extractvalue {i8, i1} %t, 0
+  %obit = extractvalue {i8, i1} %t, 1
+  store i8 %val, i8* %res
+  ret i1 %obit
+}
+
+define zeroext i1 @umuloi8_load2(i8 %v1, i8* %ptr2, i8* %res) {
+; SDAG-LABEL: umuloi8_load2:
+; SDAG:       ## %bb.0:
+; SDAG-NEXT:    movl %edi, %eax
+; SDAG-NEXT:    ## kill: def $al killed $al killed $eax
+; SDAG-NEXT:    mulb (%rsi)
+; SDAG-NEXT:    seto %cl
+; SDAG-NEXT:    movb %al, (%rdx)
+; SDAG-NEXT:    movl %ecx, %eax
+; SDAG-NEXT:    retq
+;
+; FAST-LABEL: umuloi8_load2:
+; FAST:       ## %bb.0:
+; FAST-NEXT:    movl %edi, %eax
+; FAST-NEXT:    ## kill: def $al killed $al killed $eax
+; FAST-NEXT:    mulb (%rsi)
+; FAST-NEXT:    seto %cl
+; FAST-NEXT:    movb %al, (%rdx)
+; FAST-NEXT:    andb $1, %cl
+; FAST-NEXT:    movzbl %cl, %eax
+; FAST-NEXT:    retq
+  %v2 = load i8, i8* %ptr2
+  %t = call {i8, i1} @llvm.umul.with.overflow.i8(i8 %v1, i8 %v2)
+  %val = extractvalue {i8, i1} %t, 0
+  %obit = extractvalue {i8, i1} %t, 1
+  store i8 %val, i8* %res
+  ret i1 %obit
+}
+
+define zeroext i1 @umuloi16_load(i16* %ptr1, i16 %v2, i16* %res) {
+; SDAG-LABEL: umuloi16_load:
+; SDAG:       ## %bb.0:
+; SDAG-NEXT:    movq %rdx, %rcx
+; SDAG-NEXT:    movl %esi, %eax
+; SDAG-NEXT:    ## kill: def $ax killed $ax killed $eax
+; SDAG-NEXT:    mulw (%rdi)
+; SDAG-NEXT:    seto %dl
+; SDAG-NEXT:    movw %ax, (%rcx)
+; SDAG-NEXT:    movl %edx, %eax
+; SDAG-NEXT:    retq
+;
+; FAST-LABEL: umuloi16_load:
+; FAST:       ## %bb.0:
+; FAST-NEXT:    movq %rdx, %rcx
+; FAST-NEXT:    movzwl (%rdi), %eax
+; FAST-NEXT:    mulw %si
+; FAST-NEXT:    seto %dl
+; FAST-NEXT:    movw %ax, (%rcx)
+; FAST-NEXT:    andb $1, %dl
+; FAST-NEXT:    movzbl %dl, %eax
+; FAST-NEXT:    retq
+  %v1 = load i16, i16* %ptr1
+  %t = call {i16, i1} @llvm.umul.with.overflow.i16(i16 %v1, i16 %v2)
+  %val = extractvalue {i16, i1} %t, 0
+  %obit = extractvalue {i16, i1} %t, 1
+  store i16 %val, i16* %res
+  ret i1 %obit
+}
+
+define zeroext i1 @umuloi16_load2(i16 %v1, i16* %ptr2, i16* %res) {
+; SDAG-LABEL: umuloi16_load2:
+; SDAG:       ## %bb.0:
+; SDAG-NEXT:    movq %rdx, %rcx
+; SDAG-NEXT:    movl %edi, %eax
+; SDAG-NEXT:    ## kill: def $ax killed $ax killed $eax
+; SDAG-NEXT:    mulw (%rsi)
+; SDAG-NEXT:    seto %dl
+; SDAG-NEXT:    movw %ax, (%rcx)
+; SDAG-NEXT:    movl %edx, %eax
+; SDAG-NEXT:    retq
+;
+; FAST-LABEL: umuloi16_load2:
+; FAST:       ## %bb.0:
+; FAST-NEXT:    movq %rdx, %rcx
+; FAST-NEXT:    movl %edi, %eax
+; FAST-NEXT:    ## kill: def $ax killed $ax killed $eax
+; FAST-NEXT:    mulw (%rsi)
+; FAST-NEXT:    seto %dl
+; FAST-NEXT:    movw %ax, (%rcx)
+; FAST-NEXT:    andb $1, %dl
+; FAST-NEXT:    movzbl %dl, %eax
+; FAST-NEXT:    retq
+  %v2 = load i16, i16* %ptr2
+  %t = call {i16, i1} @llvm.umul.with.overflow.i16(i16 %v1, i16 %v2)
+  %val = extractvalue {i16, i1} %t, 0
+  %obit = extractvalue {i16, i1} %t, 1
+  store i16 %val, i16* %res
+  ret i1 %obit
+}
+
+define zeroext i1 @umuloi32_load(i32* %ptr1, i32 %v2, i32* %res) {
+; SDAG-LABEL: umuloi32_load:
+; SDAG:       ## %bb.0:
+; SDAG-NEXT:    movq %rdx, %rcx
+; SDAG-NEXT:    movl %esi, %eax
+; SDAG-NEXT:    mull (%rdi)
+; SDAG-NEXT:    seto %dl
+; SDAG-NEXT:    movl %eax, (%rcx)
+; SDAG-NEXT:    movl %edx, %eax
+; SDAG-NEXT:    retq
+;
+; FAST-LABEL: umuloi32_load:
+; FAST:       ## %bb.0:
+; FAST-NEXT:    movq %rdx, %rcx
+; FAST-NEXT:    movl (%rdi), %eax
+; FAST-NEXT:    mull %esi
+; FAST-NEXT:    seto %dl
+; FAST-NEXT:    movl %eax, (%rcx)
+; FAST-NEXT:    andb $1, %dl
+; FAST-NEXT:    movzbl %dl, %eax
+; FAST-NEXT:    retq
+  %v1 = load i32, i32* %ptr1
+  %t = call {i32, i1} @llvm.umul.with.overflow.i32(i32 %v1, i32 %v2)
+  %val = extractvalue {i32, i1} %t, 0
+  %obit = extractvalue {i32, i1} %t, 1
+  store i32 %val, i32* %res
+  ret i1 %obit
+}
+
+define zeroext i1 @umuloi32_load2(i32 %v1, i32* %ptr2, i32* %res) {
+; SDAG-LABEL: umuloi32_load2:
+; SDAG:       ## %bb.0:
+; SDAG-NEXT:    movq %rdx, %rcx
+; SDAG-NEXT:    movl %edi, %eax
+; SDAG-NEXT:    mull (%rsi)
+; SDAG-NEXT:    seto %dl
+; SDAG-NEXT:    movl %eax, (%rcx)
+; SDAG-NEXT:    movl %edx, %eax
+; SDAG-NEXT:    retq
+;
+; FAST-LABEL: umuloi32_load2:
+; FAST:       ## %bb.0:
+; FAST-NEXT:    movq %rdx, %rcx
+; FAST-NEXT:    movl %edi, %eax
+; FAST-NEXT:    mull (%rsi)
+; FAST-NEXT:    seto %dl
+; FAST-NEXT:    movl %eax, (%rcx)
+; FAST-NEXT:    andb $1, %dl
+; FAST-NEXT:    movzbl %dl, %eax
+; FAST-NEXT:    retq
+  %v2 = load i32, i32* %ptr2
+  %t = call {i32, i1} @llvm.umul.with.overflow.i32(i32 %v1, i32 %v2)
+  %val = extractvalue {i32, i1} %t, 0
+  %obit = extractvalue {i32, i1} %t, 1
+  store i32 %val, i32* %res
+  ret i1 %obit
+}
+
+define zeroext i1 @umuloi64_load(i64* %ptr1, i64 %v2, i64* %res) {
+; SDAG-LABEL: umuloi64_load:
+; SDAG:       ## %bb.0:
+; SDAG-NEXT:    movq %rdx, %rcx
+; SDAG-NEXT:    movq %rsi, %rax
+; SDAG-NEXT:    mulq (%rdi)
+; SDAG-NEXT:    seto %dl
+; SDAG-NEXT:    movq %rax, (%rcx)
+; SDAG-NEXT:    movl %edx, %eax
+; SDAG-NEXT:    retq
+;
+; FAST-LABEL: umuloi64_load:
+; FAST:       ## %bb.0:
+; FAST-NEXT:    movq %rdx, %rcx
+; FAST-NEXT:    movq (%rdi), %rax
+; FAST-NEXT:    mulq %rsi
+; FAST-NEXT:    seto %dl
+; FAST-NEXT:    movq %rax, (%rcx)
+; FAST-NEXT:    andb $1, %dl
+; FAST-NEXT:    movzbl %dl, %eax
+; FAST-NEXT:    retq
+  %v1 = load i64, i64* %ptr1
+  %t = call {i64, i1} @llvm.umul.with.overflow.i64(i64 %v1, i64 %v2)
+  %val = extractvalue {i64, i1} %t, 0
+  %obit = extractvalue {i64, i1} %t, 1
+  store i64 %val, i64* %res
+  ret i1 %obit
+}
+
+define zeroext i1 @umuloi64_load2(i64 %v1, i64* %ptr2, i64* %res) {
+; SDAG-LABEL: umuloi64_load2:
+; SDAG:       ## %bb.0:
+; SDAG-NEXT:    movq %rdx, %rcx
+; SDAG-NEXT:    movq %rdi, %rax
+; SDAG-NEXT:    mulq (%rsi)
+; SDAG-NEXT:    seto %dl
+; SDAG-NEXT:    movq %rax, (%rcx)
+; SDAG-NEXT:    movl %edx, %eax
+; SDAG-NEXT:    retq
+;
+; FAST-LABEL: umuloi64_load2:
+; FAST:       ## %bb.0:
+; FAST-NEXT:    movq %rdx, %rcx
+; FAST-NEXT:    movq %rdi, %rax
+; FAST-NEXT:    mulq (%rsi)
+; FAST-NEXT:    seto %dl
+; FAST-NEXT:    movq %rax, (%rcx)
+; FAST-NEXT:    andb $1, %dl
+; FAST-NEXT:    movzbl %dl, %eax
+; FAST-NEXT:    retq
+  %v2 = load i64, i64* %ptr2
+  %t = call {i64, i1} @llvm.umul.with.overflow.i64(i64 %v1, i64 %v2)
+  %val = extractvalue {i64, i1} %t, 0
+  %obit = extractvalue {i64, i1} %t, 1
+  store i64 %val, i64* %res
+  ret i1 %obit
+}
+
 declare {i8,  i1} @llvm.smul.with.overflow.i8 (i8,  i8 ) nounwind readnone
 declare {i16, i1} @llvm.smul.with.overflow.i16(i16, i16) nounwind readnone
 declare {i32, i1} @llvm.smul.with.overflow.i32(i32, i32) nounwind readnone
diff --git a/test/CodeGen/X86/xop-intrinsics-fast-isel.ll b/test/CodeGen/X86/xop-intrinsics-fast-isel.ll
index f31658319b03fad91189af36271b810abbc618ab..037c8f123aad8cb16e13e18e4dd0d58d87303332 100644
--- a/test/CodeGen/X86/xop-intrinsics-fast-isel.ll
+++ b/test/CodeGen/X86/xop-intrinsics-fast-isel.ll
@@ -388,11 +388,11 @@ define <2 x i64> @test_mm_rot_epi8(<2 x i64> %a0, <2 x i64> %a1) {
 ; ALL-NEXT:    ret{{[l|q]}}
   %arg0 = bitcast <2 x i64> %a0 to <16 x i8>
   %arg1 = bitcast <2 x i64> %a1 to <16 x i8>
-  %res = call <16 x i8> @llvm.x86.xop.vprotb(<16 x i8> %arg0, <16 x i8> %arg1)
+  %res = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %arg0, <16 x i8> %arg0, <16 x i8> %arg1)
   %bc = bitcast <16 x i8> %res to <2 x i64>
   ret <2 x i64> %bc
 }
-declare <16 x i8> @llvm.x86.xop.vprotb(<16 x i8>, <16 x i8>) nounwind readnone
+declare <16 x i8> @llvm.fshl.v16i8(<16 x i8>, <16 x i8>, <16 x i8>) nounwind readnone
 
 define <2 x i64> @test_mm_rot_epi16(<2 x i64> %a0, <2 x i64> %a1) {
 ; ALL-LABEL: test_mm_rot_epi16:
@@ -401,11 +401,11 @@ define <2 x i64> @test_mm_rot_epi16(<2 x i64> %a0, <2 x i64> %a1) {
 ; ALL-NEXT:    ret{{[l|q]}}
   %arg0 = bitcast <2 x i64> %a0 to <8 x i16>
   %arg1 = bitcast <2 x i64> %a1 to <8 x i16>
-  %res = call <8 x i16> @llvm.x86.xop.vprotw(<8 x i16> %arg0, <8 x i16> %arg1)
+  %res = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %arg0, <8 x i16> %arg0, <8 x i16> %arg1)
   %bc = bitcast <8 x i16> %res to <2 x i64>
   ret <2 x i64> %bc
 }
-declare <8 x i16> @llvm.x86.xop.vprotw(<8 x i16>, <8 x i16>) nounwind readnone
+declare <8 x i16> @llvm.fshl.v8i16(<8 x i16>, <8 x i16>, <8 x i16>) nounwind readnone
 
 define <2 x i64> @test_mm_rot_epi32(<2 x i64> %a0, <2 x i64> %a1) {
 ; ALL-LABEL: test_mm_rot_epi32:
@@ -414,21 +414,21 @@ define <2 x i64> @test_mm_rot_epi32(<2 x i64> %a0, <2 x i64> %a1) {
 ; ALL-NEXT:    ret{{[l|q]}}
   %arg0 = bitcast <2 x i64> %a0 to <4 x i32>
   %arg1 = bitcast <2 x i64> %a1 to <4 x i32>
-  %res = call <4 x i32> @llvm.x86.xop.vprotd(<4 x i32> %arg0, <4 x i32> %arg1)
+  %res = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %arg0, <4 x i32> %arg0, <4 x i32> %arg1)
   %bc = bitcast <4 x i32> %res to <2 x i64>
   ret <2 x i64> %bc
 }
-declare <4 x i32> @llvm.x86.xop.vprotd(<4 x i32>, <4 x i32>) nounwind readnone
+declare <4 x i32> @llvm.fshl.v4i32(<4 x i32>, <4 x i32>, <4 x i32>) nounwind readnone
 
 define <2 x i64> @test_mm_rot_epi64(<2 x i64> %a0, <2 x i64> %a1) {
 ; ALL-LABEL: test_mm_rot_epi64:
 ; ALL:       # %bb.0:
 ; ALL-NEXT:    vprotq %xmm1, %xmm0, %xmm0
 ; ALL-NEXT:    ret{{[l|q]}}
-  %res = call <2 x i64> @llvm.x86.xop.vprotq(<2 x i64> %a0, <2 x i64> %a1)
+  %res = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %a0, <2 x i64> %a0, <2 x i64> %a1)
   ret <2 x i64> %res
 }
-declare <2 x i64> @llvm.x86.xop.vprotq(<2 x i64>, <2 x i64>) nounwind readnone
+declare <2 x i64> @llvm.fshl.v2i64(<2 x i64>, <2 x i64>, <2 x i64>) nounwind readnone
 
 define <2 x i64> @test_mm_roti_epi8(<2 x i64> %a0) {
 ; ALL-LABEL: test_mm_roti_epi8:
@@ -436,45 +436,41 @@ define <2 x i64> @test_mm_roti_epi8(<2 x i64> %a0) {
 ; ALL-NEXT:    vprotb $1, %xmm0, %xmm0
 ; ALL-NEXT:    ret{{[l|q]}}
   %arg0 = bitcast <2 x i64> %a0 to <16 x i8>
-  %res = call <16 x i8> @llvm.x86.xop.vprotbi(<16 x i8> %arg0, i8 1)
+  %res = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %arg0, <16 x i8> %arg0, <16 x i8> <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1>)
   %bc = bitcast <16 x i8> %res to <2 x i64>
   ret <2 x i64> %bc
 }
-declare <16 x i8> @llvm.x86.xop.vprotbi(<16 x i8>, i8) nounwind readnone
 
 define <2 x i64> @test_mm_roti_epi16(<2 x i64> %a0) {
 ; ALL-LABEL: test_mm_roti_epi16:
 ; ALL:       # %bb.0:
-; ALL-NEXT:    vprotw $50, %xmm0, %xmm0
+; ALL-NEXT:    vprotw $2, %xmm0, %xmm0
 ; ALL-NEXT:    ret{{[l|q]}}
   %arg0 = bitcast <2 x i64> %a0 to <8 x i16>
-  %res = call <8 x i16> @llvm.x86.xop.vprotwi(<8 x i16> %arg0, i8 50)
+  %res = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %arg0, <8 x i16> %arg0, <8 x i16> <i16 50, i16 50, i16 50, i16 50, i16 50, i16 50, i16 50, i16 50>)
   %bc = bitcast <8 x i16> %res to <2 x i64>
   ret <2 x i64> %bc
 }
-declare <8 x i16> @llvm.x86.xop.vprotwi(<8 x i16>, i8) nounwind readnone
 
 define <2 x i64> @test_mm_roti_epi32(<2 x i64> %a0) {
 ; ALL-LABEL: test_mm_roti_epi32:
 ; ALL:       # %bb.0:
-; ALL-NEXT:    vprotd $226, %xmm0, %xmm0
+; ALL-NEXT:    vprotd $2, %xmm0, %xmm0
 ; ALL-NEXT:    ret{{[l|q]}}
   %arg0 = bitcast <2 x i64> %a0 to <4 x i32>
-  %res = call <4 x i32> @llvm.x86.xop.vprotdi(<4 x i32> %arg0, i8 -30)
+  %res = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %arg0, <4 x i32> %arg0, <4 x i32> <i32 -30, i32 -30, i32 -30, i32 -30>)
   %bc = bitcast <4 x i32> %res to <2 x i64>
   ret <2 x i64> %bc
 }
-declare <4 x i32> @llvm.x86.xop.vprotdi(<4 x i32>, i8) nounwind readnone
 
 define <2 x i64> @test_mm_roti_epi64(<2 x i64> %a0) {
 ; ALL-LABEL: test_mm_roti_epi64:
 ; ALL:       # %bb.0:
-; ALL-NEXT:    vprotq $100, %xmm0, %xmm0
+; ALL-NEXT:    vprotq $36, %xmm0, %xmm0
 ; ALL-NEXT:    ret{{[l|q]}}
-  %res = call <2 x i64> @llvm.x86.xop.vprotqi(<2 x i64> %a0, i8 100)
+  %res = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %a0, <2 x i64> %a0, <2 x i64> <i64 100, i64 100>)
   ret <2 x i64> %res
 }
-declare <2 x i64> @llvm.x86.xop.vprotqi(<2 x i64>, i8) nounwind readnone
 
 define <2 x i64> @test_mm_shl_epi8(<2 x i64> %a0, <2 x i64> %a1) {
 ; ALL-LABEL: test_mm_shl_epi8:
@@ -581,11 +577,11 @@ define <2 x i64> @test_mm_com_epu8(<2 x i64> %a0, <2 x i64> %a1) {
 ; ALL-NEXT:    ret{{[l|q]}}
   %arg0 = bitcast <2 x i64> %a0 to <16 x i8>
   %arg1 = bitcast <2 x i64> %a1 to <16 x i8>
-  %res = call <16 x i8> @llvm.x86.xop.vpcomub(<16 x i8> %arg0, <16 x i8> %arg1, i8 0)
+  %cmp = icmp ult <16 x i8> %arg0, %arg1
+  %res = sext <16 x i1> %cmp to <16 x i8>
   %bc = bitcast <16 x i8> %res to <2 x i64>
   ret <2 x i64> %bc
 }
-declare <16 x i8> @llvm.x86.xop.vpcomub(<16 x i8>, <16 x i8>, i8) nounwind readnone
 
 define <2 x i64> @test_mm_com_epu16(<2 x i64> %a0, <2 x i64> %a1) {
 ; ALL-LABEL: test_mm_com_epu16:
@@ -594,11 +590,11 @@ define <2 x i64> @test_mm_com_epu16(<2 x i64> %a0, <2 x i64> %a1) {
 ; ALL-NEXT:    ret{{[l|q]}}
   %arg0 = bitcast <2 x i64> %a0 to <8 x i16>
   %arg1 = bitcast <2 x i64> %a1 to <8 x i16>
-  %res = call <8 x i16> @llvm.x86.xop.vpcomuw(<8 x i16> %arg0, <8 x i16> %arg1, i8 0)
+  %cmp = icmp ult <8 x i16> %arg0, %arg1
+  %res = sext <8 x i1> %cmp to <8 x i16>
   %bc = bitcast <8 x i16> %res to <2 x i64>
   ret <2 x i64> %bc
 }
-declare <8 x i16> @llvm.x86.xop.vpcomuw(<8 x i16>, <8 x i16>, i8) nounwind readnone
 
 define <2 x i64> @test_mm_com_epu32(<2 x i64> %a0, <2 x i64> %a1) {
 ; ALL-LABEL: test_mm_com_epu32:
@@ -607,21 +603,21 @@ define <2 x i64> @test_mm_com_epu32(<2 x i64> %a0, <2 x i64> %a1) {
 ; ALL-NEXT:    ret{{[l|q]}}
   %arg0 = bitcast <2 x i64> %a0 to <4 x i32>
   %arg1 = bitcast <2 x i64> %a1 to <4 x i32>
-  %res = call <4 x i32> @llvm.x86.xop.vpcomud(<4 x i32> %arg0, <4 x i32> %arg1, i8 0)
+  %cmp = icmp ult <4 x i32> %arg0, %arg1
+  %res = sext <4 x i1> %cmp to <4 x i32>
   %bc = bitcast <4 x i32> %res to <2 x i64>
   ret <2 x i64> %bc
 }
-declare <4 x i32> @llvm.x86.xop.vpcomud(<4 x i32>, <4 x i32>, i8) nounwind readnone
 
 define <2 x i64> @test_mm_com_epu64(<2 x i64> %a0, <2 x i64> %a1) {
 ; ALL-LABEL: test_mm_com_epu64:
 ; ALL:       # %bb.0:
 ; ALL-NEXT:    vpcomltuq %xmm1, %xmm0, %xmm0
 ; ALL-NEXT:    ret{{[l|q]}}
-  %res = call <2 x i64> @llvm.x86.xop.vpcomuq(<2 x i64> %a0, <2 x i64> %a1, i8 0)
+  %cmp = icmp ult <2 x i64> %a0, %a1
+  %res = sext <2 x i1> %cmp to <2 x i64>
   ret <2 x i64> %res
 }
-declare <2 x i64> @llvm.x86.xop.vpcomuq(<2 x i64>, <2 x i64>, i8) nounwind readnone
 
 define <2 x i64> @test_mm_com_epi8(<2 x i64> %a0, <2 x i64> %a1) {
 ; ALL-LABEL: test_mm_com_epi8:
@@ -630,11 +626,11 @@ define <2 x i64> @test_mm_com_epi8(<2 x i64> %a0, <2 x i64> %a1) {
 ; ALL-NEXT:    ret{{[l|q]}}
   %arg0 = bitcast <2 x i64> %a0 to <16 x i8>
   %arg1 = bitcast <2 x i64> %a1 to <16 x i8>
-  %res = call <16 x i8> @llvm.x86.xop.vpcomb(<16 x i8> %arg0, <16 x i8> %arg1, i8 0)
+  %cmp = icmp slt <16 x i8> %arg0, %arg1
+  %res = sext <16 x i1> %cmp to <16 x i8>
   %bc = bitcast <16 x i8> %res to <2 x i64>
   ret <2 x i64> %bc
 }
-declare <16 x i8> @llvm.x86.xop.vpcomb(<16 x i8>, <16 x i8>, i8) nounwind readnone
 
 define <2 x i64> @test_mm_com_epi16(<2 x i64> %a0, <2 x i64> %a1) {
 ; ALL-LABEL: test_mm_com_epi16:
@@ -643,11 +639,11 @@ define <2 x i64> @test_mm_com_epi16(<2 x i64> %a0, <2 x i64> %a1) {
 ; ALL-NEXT:    ret{{[l|q]}}
   %arg0 = bitcast <2 x i64> %a0 to <8 x i16>
   %arg1 = bitcast <2 x i64> %a1 to <8 x i16>
-  %res = call <8 x i16> @llvm.x86.xop.vpcomw(<8 x i16> %arg0, <8 x i16> %arg1, i8 0)
+  %cmp = icmp slt <8 x i16> %arg0, %arg1
+  %res = sext <8 x i1> %cmp to <8 x i16>
   %bc = bitcast <8 x i16> %res to <2 x i64>
   ret <2 x i64> %bc
 }
-declare <8 x i16> @llvm.x86.xop.vpcomw(<8 x i16>, <8 x i16>, i8) nounwind readnone
 
 define <2 x i64> @test_mm_com_epi32(<2 x i64> %a0, <2 x i64> %a1) {
 ; ALL-LABEL: test_mm_com_epi32:
@@ -656,21 +652,21 @@ define <2 x i64> @test_mm_com_epi32(<2 x i64> %a0, <2 x i64> %a1) {
 ; ALL-NEXT:    ret{{[l|q]}}
   %arg0 = bitcast <2 x i64> %a0 to <4 x i32>
   %arg1 = bitcast <2 x i64> %a1 to <4 x i32>
-  %res = call <4 x i32> @llvm.x86.xop.vpcomd(<4 x i32> %arg0, <4 x i32> %arg1, i8 0)
+  %cmp = icmp slt <4 x i32> %arg0, %arg1
+  %res = sext <4 x i1> %cmp to <4 x i32>
   %bc = bitcast <4 x i32> %res to <2 x i64>
   ret <2 x i64> %bc
 }
-declare <4 x i32> @llvm.x86.xop.vpcomd(<4 x i32>, <4 x i32>, i8) nounwind readnone
 
 define <2 x i64> @test_mm_com_epi64(<2 x i64> %a0, <2 x i64> %a1) {
 ; ALL-LABEL: test_mm_com_epi64:
 ; ALL:       # %bb.0:
 ; ALL-NEXT:    vpcomltq %xmm1, %xmm0, %xmm0
 ; ALL-NEXT:    ret{{[l|q]}}
-  %res = call <2 x i64> @llvm.x86.xop.vpcomq(<2 x i64> %a0, <2 x i64> %a1, i8 0)
+  %cmp = icmp slt <2 x i64> %a0, %a1
+  %res = sext <2 x i1> %cmp to <2 x i64>
   ret <2 x i64> %res
 }
-declare <2 x i64> @llvm.x86.xop.vpcomq(<2 x i64>, <2 x i64>, i8) nounwind readnone
 
 define <2 x double> @test_mm_permute2_pd(<2 x double> %a0, <2 x double> %a1, <2 x i64> %a2) {
 ; ALL-LABEL: test_mm_permute2_pd:
@@ -773,20 +769,3 @@ define <4 x double> @test_mm256_frcz_pd(<4 x double> %a0) {
   ret <4 x double> %res
 }
 declare <4 x double> @llvm.x86.xop.vfrcz.pd.256(<4 x double>) nounwind readnone
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
diff --git a/test/CodeGen/X86/xop-intrinsics-x86_64-upgrade.ll b/test/CodeGen/X86/xop-intrinsics-x86_64-upgrade.ll
index c5493368ab1152f86ff05311ce8a386445aaf7a6..3d25e41787caba212e2785ff8cd7ab3f8b9a233b 100644
--- a/test/CodeGen/X86/xop-intrinsics-x86_64-upgrade.ll
+++ b/test/CodeGen/X86/xop-intrinsics-x86_64-upgrade.ll
@@ -129,7 +129,7 @@ declare <2 x i64> @llvm.x86.xop.vpcomeqq(<2 x i64>, <2 x i64>) nounwind readnone
 define <16 x i8> @test_int_x86_xop_vpcomequb(<16 x i8> %a0, <16 x i8> %a1) {
 ; CHECK-LABEL: test_int_x86_xop_vpcomequb:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vpcomequb %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    vpcomeqb %xmm1, %xmm0, %xmm0
 ; CHECK-NEXT:    retq
   %res = call <16 x i8> @llvm.x86.xop.vpcomequb(<16 x i8> %a0, <16 x i8> %a1) ;
   ret <16 x i8> %res
@@ -139,7 +139,7 @@ declare <16 x i8> @llvm.x86.xop.vpcomequb(<16 x i8>, <16 x i8>) nounwind readnon
 define <4 x i32> @test_int_x86_xop_vpcomequd(<4 x i32> %a0, <4 x i32> %a1) {
 ; CHECK-LABEL: test_int_x86_xop_vpcomequd:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vpcomequd %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    vpcomeqd %xmm1, %xmm0, %xmm0
 ; CHECK-NEXT:    retq
   %res = call <4 x i32> @llvm.x86.xop.vpcomequd(<4 x i32> %a0, <4 x i32> %a1) ;
   ret <4 x i32> %res
@@ -149,7 +149,7 @@ declare <4 x i32> @llvm.x86.xop.vpcomequd(<4 x i32>, <4 x i32>) nounwind readnon
 define <2 x i64> @test_int_x86_xop_vpcomequq(<2 x i64> %a0, <2 x i64> %a1) {
 ; CHECK-LABEL: test_int_x86_xop_vpcomequq:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vpcomequq %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    vpcomeqq %xmm1, %xmm0, %xmm0
 ; CHECK-NEXT:    retq
   %res = call <2 x i64> @llvm.x86.xop.vpcomequq(<2 x i64> %a0, <2 x i64> %a1) ;
   ret <2 x i64> %res
@@ -159,7 +159,7 @@ declare <2 x i64> @llvm.x86.xop.vpcomequq(<2 x i64>, <2 x i64>) nounwind readnon
 define <8 x i16> @test_int_x86_xop_vpcomequw(<8 x i16> %a0, <8 x i16> %a1) {
 ; CHECK-LABEL: test_int_x86_xop_vpcomequw:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vpcomequw %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    vpcomeqw %xmm1, %xmm0, %xmm0
 ; CHECK-NEXT:    retq
   %res = call <8 x i16> @llvm.x86.xop.vpcomequw(<8 x i16> %a0, <8 x i16> %a1) ;
   ret <8 x i16> %res
@@ -169,7 +169,7 @@ declare <8 x i16> @llvm.x86.xop.vpcomequw(<8 x i16>, <8 x i16>) nounwind readnon
 define <16 x i8> @test_int_x86_xop_vpcomfalseb(<16 x i8> %a0, <16 x i8> %a1) {
 ; CHECK-LABEL: test_int_x86_xop_vpcomfalseb:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vpcomfalseb %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    vxorps %xmm0, %xmm0, %xmm0
 ; CHECK-NEXT:    retq
   %res = call <16 x i8> @llvm.x86.xop.vpcomfalseb(<16 x i8> %a0, <16 x i8> %a1) ;
   ret <16 x i8> %res
@@ -179,7 +179,7 @@ declare <16 x i8> @llvm.x86.xop.vpcomfalseb(<16 x i8>, <16 x i8>) nounwind readn
 define <4 x i32> @test_int_x86_xop_vpcomfalsed(<4 x i32> %a0, <4 x i32> %a1) {
 ; CHECK-LABEL: test_int_x86_xop_vpcomfalsed:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vpcomfalsed %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    vxorps %xmm0, %xmm0, %xmm0
 ; CHECK-NEXT:    retq
   %res = call <4 x i32> @llvm.x86.xop.vpcomfalsed(<4 x i32> %a0, <4 x i32> %a1) ;
   ret <4 x i32> %res
@@ -189,7 +189,7 @@ declare <4 x i32> @llvm.x86.xop.vpcomfalsed(<4 x i32>, <4 x i32>) nounwind readn
 define <2 x i64> @test_int_x86_xop_vpcomfalseq(<2 x i64> %a0, <2 x i64> %a1) {
 ; CHECK-LABEL: test_int_x86_xop_vpcomfalseq:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vpcomfalseq %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    vxorps %xmm0, %xmm0, %xmm0
 ; CHECK-NEXT:    retq
   %res = call <2 x i64> @llvm.x86.xop.vpcomfalseq(<2 x i64> %a0, <2 x i64> %a1) ;
   ret <2 x i64> %res
@@ -199,7 +199,7 @@ declare <2 x i64> @llvm.x86.xop.vpcomfalseq(<2 x i64>, <2 x i64>) nounwind readn
 define <16 x i8> @test_int_x86_xop_vpcomfalseub(<16 x i8> %a0, <16 x i8> %a1) {
 ; CHECK-LABEL: test_int_x86_xop_vpcomfalseub:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vpcomfalseub %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    vxorps %xmm0, %xmm0, %xmm0
 ; CHECK-NEXT:    retq
   %res = call <16 x i8> @llvm.x86.xop.vpcomfalseub(<16 x i8> %a0, <16 x i8> %a1) ;
   ret <16 x i8> %res
@@ -209,7 +209,7 @@ declare <16 x i8> @llvm.x86.xop.vpcomfalseub(<16 x i8>, <16 x i8>) nounwind read
 define <4 x i32> @test_int_x86_xop_vpcomfalseud(<4 x i32> %a0, <4 x i32> %a1) {
 ; CHECK-LABEL: test_int_x86_xop_vpcomfalseud:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vpcomfalseud %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    vxorps %xmm0, %xmm0, %xmm0
 ; CHECK-NEXT:    retq
   %res = call <4 x i32> @llvm.x86.xop.vpcomfalseud(<4 x i32> %a0, <4 x i32> %a1) ;
   ret <4 x i32> %res
@@ -219,7 +219,7 @@ declare <4 x i32> @llvm.x86.xop.vpcomfalseud(<4 x i32>, <4 x i32>) nounwind read
 define <2 x i64> @test_int_x86_xop_vpcomfalseuq(<2 x i64> %a0, <2 x i64> %a1) {
 ; CHECK-LABEL: test_int_x86_xop_vpcomfalseuq:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vpcomfalseuq %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    vxorps %xmm0, %xmm0, %xmm0
 ; CHECK-NEXT:    retq
   %res = call <2 x i64> @llvm.x86.xop.vpcomfalseuq(<2 x i64> %a0, <2 x i64> %a1) ;
   ret <2 x i64> %res
@@ -229,7 +229,7 @@ declare <2 x i64> @llvm.x86.xop.vpcomfalseuq(<2 x i64>, <2 x i64>) nounwind read
 define <8 x i16> @test_int_x86_xop_vpcomfalseuw(<8 x i16> %a0, <8 x i16> %a1) {
 ; CHECK-LABEL: test_int_x86_xop_vpcomfalseuw:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vpcomfalseuw %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    vxorps %xmm0, %xmm0, %xmm0
 ; CHECK-NEXT:    retq
   %res = call <8 x i16> @llvm.x86.xop.vpcomfalseuw(<8 x i16> %a0, <8 x i16> %a1) ;
   ret <8 x i16> %res
@@ -239,7 +239,7 @@ declare <8 x i16> @llvm.x86.xop.vpcomfalseuw(<8 x i16>, <8 x i16>) nounwind read
 define <8 x i16> @test_int_x86_xop_vpcomfalsew(<8 x i16> %a0, <8 x i16> %a1) {
 ; CHECK-LABEL: test_int_x86_xop_vpcomfalsew:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vpcomfalsew %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    vxorps %xmm0, %xmm0, %xmm0
 ; CHECK-NEXT:    retq
   %res = call <8 x i16> @llvm.x86.xop.vpcomfalsew(<8 x i16> %a0, <8 x i16> %a1) ;
   ret <8 x i16> %res
@@ -599,7 +599,7 @@ declare <2 x i64> @llvm.x86.xop.vpcomneq(<2 x i64>, <2 x i64>) nounwind readnone
 define <16 x i8> @test_int_x86_xop_vpcomneub(<16 x i8> %a0, <16 x i8> %a1) {
 ; CHECK-LABEL: test_int_x86_xop_vpcomneub:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vpcomnequb %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    vpcomneqb %xmm1, %xmm0, %xmm0
 ; CHECK-NEXT:    retq
   %res = call <16 x i8> @llvm.x86.xop.vpcomneub(<16 x i8> %a0, <16 x i8> %a1) ;
   ret <16 x i8> %res
@@ -609,7 +609,7 @@ declare <16 x i8> @llvm.x86.xop.vpcomneub(<16 x i8>, <16 x i8>) nounwind readnon
 define <4 x i32> @test_int_x86_xop_vpcomneud(<4 x i32> %a0, <4 x i32> %a1) {
 ; CHECK-LABEL: test_int_x86_xop_vpcomneud:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vpcomnequd %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    vpcomneqd %xmm1, %xmm0, %xmm0
 ; CHECK-NEXT:    retq
   %res = call <4 x i32> @llvm.x86.xop.vpcomneud(<4 x i32> %a0, <4 x i32> %a1) ;
   ret <4 x i32> %res
@@ -619,7 +619,7 @@ declare <4 x i32> @llvm.x86.xop.vpcomneud(<4 x i32>, <4 x i32>) nounwind readnon
 define <2 x i64> @test_int_x86_xop_vpcomneuq(<2 x i64> %a0, <2 x i64> %a1) {
 ; CHECK-LABEL: test_int_x86_xop_vpcomneuq:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vpcomnequq %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    vpcomneqq %xmm1, %xmm0, %xmm0
 ; CHECK-NEXT:    retq
   %res = call <2 x i64> @llvm.x86.xop.vpcomneuq(<2 x i64> %a0, <2 x i64> %a1) ;
   ret <2 x i64> %res
@@ -629,7 +629,7 @@ declare <2 x i64> @llvm.x86.xop.vpcomneuq(<2 x i64>, <2 x i64>) nounwind readnon
 define <8 x i16> @test_int_x86_xop_vpcomneuw(<8 x i16> %a0, <8 x i16> %a1) {
 ; CHECK-LABEL: test_int_x86_xop_vpcomneuw:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vpcomnequw %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    vpcomneqw %xmm1, %xmm0, %xmm0
 ; CHECK-NEXT:    retq
   %res = call <8 x i16> @llvm.x86.xop.vpcomneuw(<8 x i16> %a0, <8 x i16> %a1) ;
   ret <8 x i16> %res
@@ -649,7 +649,7 @@ declare <8 x i16> @llvm.x86.xop.vpcomnew(<8 x i16>, <8 x i16>) nounwind readnone
 define <16 x i8> @test_int_x86_xop_vpcomtrueb(<16 x i8> %a0, <16 x i8> %a1) {
 ; CHECK-LABEL: test_int_x86_xop_vpcomtrueb:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vpcomtrueb %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
 ; CHECK-NEXT:    retq
   %res = call <16 x i8> @llvm.x86.xop.vpcomtrueb(<16 x i8> %a0, <16 x i8> %a1) ;
   ret <16 x i8> %res
@@ -659,7 +659,7 @@ declare <16 x i8> @llvm.x86.xop.vpcomtrueb(<16 x i8>, <16 x i8>) nounwind readno
 define <4 x i32> @test_int_x86_xop_vpcomtrued(<4 x i32> %a0, <4 x i32> %a1) {
 ; CHECK-LABEL: test_int_x86_xop_vpcomtrued:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vpcomtrued %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
 ; CHECK-NEXT:    retq
   %res = call <4 x i32> @llvm.x86.xop.vpcomtrued(<4 x i32> %a0, <4 x i32> %a1) ;
   ret <4 x i32> %res
@@ -669,7 +669,7 @@ declare <4 x i32> @llvm.x86.xop.vpcomtrued(<4 x i32>, <4 x i32>) nounwind readno
 define <2 x i64> @test_int_x86_xop_vpcomtrueq(<2 x i64> %a0, <2 x i64> %a1) {
 ; CHECK-LABEL: test_int_x86_xop_vpcomtrueq:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vpcomtrueq %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
 ; CHECK-NEXT:    retq
   %res = call <2 x i64> @llvm.x86.xop.vpcomtrueq(<2 x i64> %a0, <2 x i64> %a1) ;
   ret <2 x i64> %res
@@ -679,7 +679,7 @@ declare <2 x i64> @llvm.x86.xop.vpcomtrueq(<2 x i64>, <2 x i64>) nounwind readno
 define <16 x i8> @test_int_x86_xop_vpcomtrueub(<16 x i8> %a0, <16 x i8> %a1) {
 ; CHECK-LABEL: test_int_x86_xop_vpcomtrueub:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vpcomtrueub %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
 ; CHECK-NEXT:    retq
   %res = call <16 x i8> @llvm.x86.xop.vpcomtrueub(<16 x i8> %a0, <16 x i8> %a1) ;
   ret <16 x i8> %res
@@ -689,7 +689,7 @@ declare <16 x i8> @llvm.x86.xop.vpcomtrueub(<16 x i8>, <16 x i8>) nounwind readn
 define <4 x i32> @test_int_x86_xop_vpcomtrueud(<4 x i32> %a0, <4 x i32> %a1) {
 ; CHECK-LABEL: test_int_x86_xop_vpcomtrueud:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vpcomtrueud %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
 ; CHECK-NEXT:    retq
   %res = call <4 x i32> @llvm.x86.xop.vpcomtrueud(<4 x i32> %a0, <4 x i32> %a1) ;
   ret <4 x i32> %res
@@ -699,7 +699,7 @@ declare <4 x i32> @llvm.x86.xop.vpcomtrueud(<4 x i32>, <4 x i32>) nounwind readn
 define <2 x i64> @test_int_x86_xop_vpcomtrueuq(<2 x i64> %a0, <2 x i64> %a1) {
 ; CHECK-LABEL: test_int_x86_xop_vpcomtrueuq:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vpcomtrueuq %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
 ; CHECK-NEXT:    retq
   %res = call <2 x i64> @llvm.x86.xop.vpcomtrueuq(<2 x i64> %a0, <2 x i64> %a1) ;
   ret <2 x i64> %res
@@ -709,7 +709,7 @@ declare <2 x i64> @llvm.x86.xop.vpcomtrueuq(<2 x i64>, <2 x i64>) nounwind readn
 define <8 x i16> @test_int_x86_xop_vpcomtrueuw(<8 x i16> %a0, <8 x i16> %a1) {
 ; CHECK-LABEL: test_int_x86_xop_vpcomtrueuw:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vpcomtrueuw %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
 ; CHECK-NEXT:    retq
   %res = call <8 x i16> @llvm.x86.xop.vpcomtrueuw(<8 x i16> %a0, <8 x i16> %a1) ;
   ret <8 x i16> %res
@@ -719,13 +719,93 @@ declare <8 x i16> @llvm.x86.xop.vpcomtrueuw(<8 x i16>, <8 x i16>) nounwind readn
 define <8 x i16> @test_int_x86_xop_vpcomtruew(<8 x i16> %a0, <8 x i16> %a1) {
 ; CHECK-LABEL: test_int_x86_xop_vpcomtruew:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    vpcomtruew %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
 ; CHECK-NEXT:    retq
   %res = call <8 x i16> @llvm.x86.xop.vpcomtruew(<8 x i16> %a0, <8 x i16> %a1) ;
   ret <8 x i16> %res
 }
 declare <8 x i16> @llvm.x86.xop.vpcomtruew(<8 x i16>, <8 x i16>) nounwind readnone
 
+define <16 x i8> @test_int_x86_xop_vpcomb(<16 x i8> %a0, <16 x i8> %a1) {
+; CHECK-LABEL: test_int_x86_xop_vpcomb:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpcomltb %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    retq
+  %res = call <16 x i8> @llvm.x86.xop.vpcomb(<16 x i8> %a0, <16 x i8> %a1, i8 0) ;
+  ret <16 x i8> %res
+}
+declare <16 x i8> @llvm.x86.xop.vpcomb(<16 x i8>, <16 x i8>, i8) nounwind readnone
+
+define <8 x i16> @test_int_x86_xop_vpcomw(<8 x i16> %a0, <8 x i16> %a1) {
+; CHECK-LABEL: test_int_x86_xop_vpcomw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpcomltw %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    retq
+  %res = call <8 x i16> @llvm.x86.xop.vpcomw(<8 x i16> %a0, <8 x i16> %a1, i8 0) ;
+  ret <8 x i16> %res
+}
+declare <8 x i16> @llvm.x86.xop.vpcomw(<8 x i16>, <8 x i16>, i8) nounwind readnone
+
+define <4 x i32> @test_int_x86_xop_vpcomd(<4 x i32> %a0, <4 x i32> %a1) {
+; CHECK-LABEL: test_int_x86_xop_vpcomd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpcomltd %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    retq
+  %res = call <4 x i32> @llvm.x86.xop.vpcomd(<4 x i32> %a0, <4 x i32> %a1, i8 0) ;
+  ret <4 x i32> %res
+}
+declare <4 x i32> @llvm.x86.xop.vpcomd(<4 x i32>, <4 x i32>, i8) nounwind readnone
+
+define <2 x i64> @test_int_x86_xop_vpcomq(<2 x i64> %a0, <2 x i64> %a1) {
+; CHECK-LABEL: test_int_x86_xop_vpcomq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpcomltq %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    retq
+  %res = call <2 x i64> @llvm.x86.xop.vpcomq(<2 x i64> %a0, <2 x i64> %a1, i8 0) ;
+  ret <2 x i64> %res
+}
+declare <2 x i64> @llvm.x86.xop.vpcomq(<2 x i64>, <2 x i64>, i8) nounwind readnone
+
+define <16 x i8> @test_int_x86_xop_vpcomub(<16 x i8> %a0, <16 x i8> %a1) {
+; CHECK-LABEL: test_int_x86_xop_vpcomub:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpcomltub %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    retq
+  %res = call <16 x i8> @llvm.x86.xop.vpcomub(<16 x i8> %a0, <16 x i8> %a1, i8 0) ;
+  ret <16 x i8> %res
+}
+declare <16 x i8> @llvm.x86.xop.vpcomub(<16 x i8>, <16 x i8>, i8) nounwind readnone
+
+define <8 x i16> @test_int_x86_xop_vpcomuw(<8 x i16> %a0, <8 x i16> %a1) {
+; CHECK-LABEL: test_int_x86_xop_vpcomuw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpcomltuw %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    retq
+  %res = call <8 x i16> @llvm.x86.xop.vpcomuw(<8 x i16> %a0, <8 x i16> %a1, i8 0) ;
+  ret <8 x i16> %res
+}
+declare <8 x i16> @llvm.x86.xop.vpcomuw(<8 x i16>, <8 x i16>, i8) nounwind readnone
+
+define <4 x i32> @test_int_x86_xop_vpcomud(<4 x i32> %a0, <4 x i32> %a1) {
+; CHECK-LABEL: test_int_x86_xop_vpcomud:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpcomltud %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    retq
+  %res = call <4 x i32> @llvm.x86.xop.vpcomud(<4 x i32> %a0, <4 x i32> %a1, i8 0) ;
+  ret <4 x i32> %res
+}
+declare <4 x i32> @llvm.x86.xop.vpcomud(<4 x i32>, <4 x i32>, i8) nounwind readnone
+
+define <2 x i64> @test_int_x86_xop_vpcomuq(<2 x i64> %a0, <2 x i64> %a1) {
+; CHECK-LABEL: test_int_x86_xop_vpcomuq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpcomltuq %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    retq
+  %res = call <2 x i64> @llvm.x86.xop.vpcomuq(<2 x i64> %a0, <2 x i64> %a1, i8 0) ;
+  ret <2 x i64> %res
+}
+declare <2 x i64> @llvm.x86.xop.vpcomuq(<2 x i64>, <2 x i64>, i8) nounwind readnone
+
 define <2 x i64> @test_int_x86_xop_vpcmov(<2 x i64> %a0, <2 x i64> %a1, <2 x i64> %a2) {
 ; CHECK-LABEL: test_int_x86_xop_vpcmov:
 ; CHECK:       # %bb.0:
@@ -764,3 +844,82 @@ define <4 x i64> @test_int_x86_xop_vpcmov_256_rm(<4 x i64> %a0, <4 x i64> %a1, <
 }
 declare <4 x i64> @llvm.x86.xop.vpcmov.256(<4 x i64>, <4 x i64>, <4 x i64>) nounwind readnone
 
+define <16 x i8> @test_int_x86_xop_vprotb(<16 x i8> %a0, <16 x i8> %a1) {
+; CHECK-LABEL: test_int_x86_xop_vprotb:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vprotb %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    retq
+  %res = call <16 x i8> @llvm.x86.xop.vprotb(<16 x i8> %a0, <16 x i8> %a1) ;
+  ret <16 x i8> %res
+}
+declare <16 x i8> @llvm.x86.xop.vprotb(<16 x i8>, <16 x i8>) nounwind readnone
+
+define <4 x i32> @test_int_x86_xop_vprotd(<4 x i32> %a0, <4 x i32> %a1) {
+; CHECK-LABEL: test_int_x86_xop_vprotd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vprotd %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    retq
+  %res = call <4 x i32> @llvm.x86.xop.vprotd(<4 x i32> %a0, <4 x i32> %a1) ;
+  ret <4 x i32> %res
+}
+declare <4 x i32> @llvm.x86.xop.vprotd(<4 x i32>, <4 x i32>) nounwind readnone
+
+define <2 x i64> @test_int_x86_xop_vprotq(<2 x i64> %a0, <2 x i64> %a1) {
+; CHECK-LABEL: test_int_x86_xop_vprotq:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vprotq %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    retq
+  %res = call <2 x i64> @llvm.x86.xop.vprotq(<2 x i64> %a0, <2 x i64> %a1) ;
+  ret <2 x i64> %res
+}
+declare <2 x i64> @llvm.x86.xop.vprotq(<2 x i64>, <2 x i64>) nounwind readnone
+
+define <8 x i16> @test_int_x86_xop_vprotw(<8 x i16> %a0, <8 x i16> %a1) {
+; CHECK-LABEL: test_int_x86_xop_vprotw:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vprotw %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    retq
+  %res = call <8 x i16> @llvm.x86.xop.vprotw(<8 x i16> %a0, <8 x i16> %a1) ;
+  ret <8 x i16> %res
+}
+declare <8 x i16> @llvm.x86.xop.vprotw(<8 x i16>, <8 x i16>) nounwind readnone
+
+define <16 x i8> @test_int_x86_xop_vprotbi(<16 x i8> %a0) {
+; CHECK-LABEL: test_int_x86_xop_vprotbi:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vprotb $1, %xmm0, %xmm0
+; CHECK-NEXT:    retq
+  %res = call <16 x i8> @llvm.x86.xop.vprotbi(<16 x i8> %a0, i8 1) ;
+  ret <16 x i8> %res
+}
+declare <16 x i8> @llvm.x86.xop.vprotbi(<16 x i8>, i8) nounwind readnone
+
+define <4 x i32> @test_int_x86_xop_vprotdi(<4 x i32> %a0) {
+; CHECK-LABEL: test_int_x86_xop_vprotdi:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vprotd $30, %xmm0, %xmm0
+; CHECK-NEXT:    retq
+  %res = call <4 x i32> @llvm.x86.xop.vprotdi(<4 x i32> %a0, i8 -2) ;
+  ret <4 x i32> %res
+}
+declare <4 x i32> @llvm.x86.xop.vprotdi(<4 x i32>, i8) nounwind readnone
+
+define <2 x i64> @test_int_x86_xop_vprotqi(<2 x i64> %a0) {
+; CHECK-LABEL: test_int_x86_xop_vprotqi:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vprotq $3, %xmm0, %xmm0
+; CHECK-NEXT:    retq
+  %res = call <2 x i64> @llvm.x86.xop.vprotqi(<2 x i64> %a0, i8 3) ;
+  ret <2 x i64> %res
+}
+declare <2 x i64> @llvm.x86.xop.vprotqi(<2 x i64>, i8) nounwind readnone
+
+define <8 x i16> @test_int_x86_xop_vprotwi(<8 x i16> %a0) {
+; CHECK-LABEL: test_int_x86_xop_vprotwi:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vprotw $12, %xmm0, %xmm0
+; CHECK-NEXT:    retq
+  %res = call <8 x i16> @llvm.x86.xop.vprotwi(<8 x i16> %a0, i8 -4) ;
+  ret <8 x i16> %res
+}
+declare <8 x i16> @llvm.x86.xop.vprotwi(<8 x i16>, i8) nounwind readnone
diff --git a/test/CodeGen/X86/xop-intrinsics-x86_64.ll b/test/CodeGen/X86/xop-intrinsics-x86_64.ll
index d4c5420f20d08d9959fb05095efd57f5e14fba71..50e057895caa3b9d6e37a60fee93b0048fb12672 100644
--- a/test/CodeGen/X86/xop-intrinsics-x86_64.ll
+++ b/test/CodeGen/X86/xop-intrinsics-x86_64.ll
@@ -450,86 +450,6 @@ define <16 x i8> @test_int_x86_xop_vpperm_mr(<16 x i8> %a0, <16 x i8>* %a1, <16
 }
 declare <16 x i8> @llvm.x86.xop.vpperm(<16 x i8>, <16 x i8>, <16 x i8>) nounwind readnone
 
-define <16 x i8> @test_int_x86_xop_vprotb(<16 x i8> %a0, <16 x i8> %a1) {
-; CHECK-LABEL: test_int_x86_xop_vprotb:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vprotb %xmm1, %xmm0, %xmm0
-; CHECK-NEXT:    retq
-  %res = call <16 x i8> @llvm.x86.xop.vprotb(<16 x i8> %a0, <16 x i8> %a1) ;
-  ret <16 x i8> %res
-}
-declare <16 x i8> @llvm.x86.xop.vprotb(<16 x i8>, <16 x i8>) nounwind readnone
-
-define <4 x i32> @test_int_x86_xop_vprotd(<4 x i32> %a0, <4 x i32> %a1) {
-; CHECK-LABEL: test_int_x86_xop_vprotd:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vprotd %xmm1, %xmm0, %xmm0
-; CHECK-NEXT:    retq
-  %res = call <4 x i32> @llvm.x86.xop.vprotd(<4 x i32> %a0, <4 x i32> %a1) ;
-  ret <4 x i32> %res
-}
-declare <4 x i32> @llvm.x86.xop.vprotd(<4 x i32>, <4 x i32>) nounwind readnone
-
-define <2 x i64> @test_int_x86_xop_vprotq(<2 x i64> %a0, <2 x i64> %a1) {
-; CHECK-LABEL: test_int_x86_xop_vprotq:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vprotq %xmm1, %xmm0, %xmm0
-; CHECK-NEXT:    retq
-  %res = call <2 x i64> @llvm.x86.xop.vprotq(<2 x i64> %a0, <2 x i64> %a1) ;
-  ret <2 x i64> %res
-}
-declare <2 x i64> @llvm.x86.xop.vprotq(<2 x i64>, <2 x i64>) nounwind readnone
-
-define <8 x i16> @test_int_x86_xop_vprotw(<8 x i16> %a0, <8 x i16> %a1) {
-; CHECK-LABEL: test_int_x86_xop_vprotw:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vprotw %xmm1, %xmm0, %xmm0
-; CHECK-NEXT:    retq
-  %res = call <8 x i16> @llvm.x86.xop.vprotw(<8 x i16> %a0, <8 x i16> %a1) ;
-  ret <8 x i16> %res
-}
-declare <8 x i16> @llvm.x86.xop.vprotw(<8 x i16>, <8 x i16>) nounwind readnone
-
-define <16 x i8> @test_int_x86_xop_vprotbi(<16 x i8> %a0) {
-; CHECK-LABEL: test_int_x86_xop_vprotbi:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vprotb $1, %xmm0, %xmm0
-; CHECK-NEXT:    retq
-  %res = call <16 x i8> @llvm.x86.xop.vprotbi(<16 x i8> %a0, i8 1) ;
-  ret <16 x i8> %res
-}
-declare <16 x i8> @llvm.x86.xop.vprotbi(<16 x i8>, i8) nounwind readnone
-
-define <4 x i32> @test_int_x86_xop_vprotdi(<4 x i32> %a0) {
-; CHECK-LABEL: test_int_x86_xop_vprotdi:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vprotd $254, %xmm0, %xmm0
-; CHECK-NEXT:    retq
-  %res = call <4 x i32> @llvm.x86.xop.vprotdi(<4 x i32> %a0, i8 -2) ;
-  ret <4 x i32> %res
-}
-declare <4 x i32> @llvm.x86.xop.vprotdi(<4 x i32>, i8) nounwind readnone
-
-define <2 x i64> @test_int_x86_xop_vprotqi(<2 x i64> %a0) {
-; CHECK-LABEL: test_int_x86_xop_vprotqi:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vprotq $3, %xmm0, %xmm0
-; CHECK-NEXT:    retq
-  %res = call <2 x i64> @llvm.x86.xop.vprotqi(<2 x i64> %a0, i8 3) ;
-  ret <2 x i64> %res
-}
-declare <2 x i64> @llvm.x86.xop.vprotqi(<2 x i64>, i8) nounwind readnone
-
-define <8 x i16> @test_int_x86_xop_vprotwi(<8 x i16> %a0) {
-; CHECK-LABEL: test_int_x86_xop_vprotwi:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vprotw $252, %xmm0, %xmm0
-; CHECK-NEXT:    retq
-  %res = call <8 x i16> @llvm.x86.xop.vprotwi(<8 x i16> %a0, i8 -4) ;
-  ret <8 x i16> %res
-}
-declare <8 x i16> @llvm.x86.xop.vprotwi(<8 x i16>, i8) nounwind readnone
-
 define <16 x i8> @test_int_x86_xop_vpshab(<16 x i8> %a0, <16 x i8> %a1) {
 ; CHECK-LABEL: test_int_x86_xop_vpshab:
 ; CHECK:       # %bb.0:
@@ -743,84 +663,3 @@ define <8 x float> @test_int_x86_xop_vfrcz_ps_256_mem(<8 x float>* %a0) {
   ret <8 x float> %res
 }
 declare <8 x float> @llvm.x86.xop.vfrcz.ps.256(<8 x float>) nounwind readnone
-
-define <16 x i8> @test_int_x86_xop_vpcomb(<16 x i8> %a0, <16 x i8> %a1) {
-; CHECK-LABEL: test_int_x86_xop_vpcomb:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vpcomltb %xmm1, %xmm0, %xmm0
-; CHECK-NEXT:    retq
-  %res = call <16 x i8> @llvm.x86.xop.vpcomb(<16 x i8> %a0, <16 x i8> %a1, i8 0) ;
-  ret <16 x i8> %res
-}
-declare <16 x i8> @llvm.x86.xop.vpcomb(<16 x i8>, <16 x i8>, i8) nounwind readnone
-
-define <8 x i16> @test_int_x86_xop_vpcomw(<8 x i16> %a0, <8 x i16> %a1) {
-; CHECK-LABEL: test_int_x86_xop_vpcomw:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vpcomltw %xmm1, %xmm0, %xmm0
-; CHECK-NEXT:    retq
-  %res = call <8 x i16> @llvm.x86.xop.vpcomw(<8 x i16> %a0, <8 x i16> %a1, i8 0) ;
-  ret <8 x i16> %res
-}
-declare <8 x i16> @llvm.x86.xop.vpcomw(<8 x i16>, <8 x i16>, i8) nounwind readnone
-
-define <4 x i32> @test_int_x86_xop_vpcomd(<4 x i32> %a0, <4 x i32> %a1) {
-; CHECK-LABEL: test_int_x86_xop_vpcomd:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vpcomltd %xmm1, %xmm0, %xmm0
-; CHECK-NEXT:    retq
-  %res = call <4 x i32> @llvm.x86.xop.vpcomd(<4 x i32> %a0, <4 x i32> %a1, i8 0) ;
-  ret <4 x i32> %res
-}
-declare <4 x i32> @llvm.x86.xop.vpcomd(<4 x i32>, <4 x i32>, i8) nounwind readnone
-
-define <2 x i64> @test_int_x86_xop_vpcomq(<2 x i64> %a0, <2 x i64> %a1) {
-; CHECK-LABEL: test_int_x86_xop_vpcomq:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vpcomltq %xmm1, %xmm0, %xmm0
-; CHECK-NEXT:    retq
-  %res = call <2 x i64> @llvm.x86.xop.vpcomq(<2 x i64> %a0, <2 x i64> %a1, i8 0) ;
-  ret <2 x i64> %res
-}
-declare <2 x i64> @llvm.x86.xop.vpcomq(<2 x i64>, <2 x i64>, i8) nounwind readnone
-
-define <16 x i8> @test_int_x86_xop_vpcomub(<16 x i8> %a0, <16 x i8> %a1) {
-; CHECK-LABEL: test_int_x86_xop_vpcomub:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vpcomltub %xmm1, %xmm0, %xmm0
-; CHECK-NEXT:    retq
-  %res = call <16 x i8> @llvm.x86.xop.vpcomub(<16 x i8> %a0, <16 x i8> %a1, i8 0) ;
-  ret <16 x i8> %res
-}
-declare <16 x i8> @llvm.x86.xop.vpcomub(<16 x i8>, <16 x i8>, i8) nounwind readnone
-
-define <8 x i16> @test_int_x86_xop_vpcomuw(<8 x i16> %a0, <8 x i16> %a1) {
-; CHECK-LABEL: test_int_x86_xop_vpcomuw:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vpcomltuw %xmm1, %xmm0, %xmm0
-; CHECK-NEXT:    retq
-  %res = call <8 x i16> @llvm.x86.xop.vpcomuw(<8 x i16> %a0, <8 x i16> %a1, i8 0) ;
-  ret <8 x i16> %res
-}
-declare <8 x i16> @llvm.x86.xop.vpcomuw(<8 x i16>, <8 x i16>, i8) nounwind readnone
-
-define <4 x i32> @test_int_x86_xop_vpcomud(<4 x i32> %a0, <4 x i32> %a1) {
-; CHECK-LABEL: test_int_x86_xop_vpcomud:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vpcomltud %xmm1, %xmm0, %xmm0
-; CHECK-NEXT:    retq
-  %res = call <4 x i32> @llvm.x86.xop.vpcomud(<4 x i32> %a0, <4 x i32> %a1, i8 0) ;
-  ret <4 x i32> %res
-}
-declare <4 x i32> @llvm.x86.xop.vpcomud(<4 x i32>, <4 x i32>, i8) nounwind readnone
-
-define <2 x i64> @test_int_x86_xop_vpcomuq(<2 x i64> %a0, <2 x i64> %a1) {
-; CHECK-LABEL: test_int_x86_xop_vpcomuq:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vpcomltuq %xmm1, %xmm0, %xmm0
-; CHECK-NEXT:    retq
-  %res = call <2 x i64> @llvm.x86.xop.vpcomuq(<2 x i64> %a0, <2 x i64> %a1, i8 0) ;
-  ret <2 x i64> %res
-}
-declare <2 x i64> @llvm.x86.xop.vpcomuq(<2 x i64>, <2 x i64>, i8) nounwind readnone
-
diff --git a/test/CodeGen/X86/xop-schedule.ll b/test/CodeGen/X86/xop-schedule.ll
deleted file mode 100644
index ed621039d38c5d0f410e88c6ef987de9a3668ce1..0000000000000000000000000000000000000000
--- a/test/CodeGen/X86/xop-schedule.ll
+++ /dev/null
@@ -1,1818 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=x86-64 -mattr=+xop | FileCheck %s --check-prefix=GENERIC
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=bdver1 | FileCheck %s --check-prefix=BDVER --check-prefix=BDVER12 --check-prefix=BDVER1
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=bdver2 | FileCheck %s --check-prefix=BDVER --check-prefix=BDVER12 --check-prefix=BDVER2
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=bdver3 | FileCheck %s --check-prefix=BDVER --check-prefix=BDVER3
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -print-schedule -mcpu=bdver4 | FileCheck %s --check-prefix=BDVER --check-prefix=BDVER4
-
-define void @test_vfrczpd(<2 x double> %a0, <4 x double> %a1, <2 x double> *%a2, <4 x double> *%a3) {
-; GENERIC-LABEL: test_vfrczpd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    vfrczpd %xmm0, %xmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    vfrczpd %ymm1, %ymm1 # sched: [3:1.00]
-; GENERIC-NEXT:    vfrczpd (%rdi), %xmm0 # sched: [9:1.00]
-; GENERIC-NEXT:    vfrczpd (%rsi), %ymm1 # sched: [10:1.00]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    vzeroupper # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER12-LABEL: test_vfrczpd:
-; BDVER12:       # %bb.0:
-; BDVER12-NEXT:    #APP
-; BDVER12-NEXT:    vfrczpd %xmm0, %xmm0 # sched: [10:1.00]
-; BDVER12-NEXT:    vfrczpd %ymm1, %ymm1 # sched: [10:2.00]
-; BDVER12-NEXT:    vfrczpd (%rdi), %xmm0 # sched: [15:1.00]
-; BDVER12-NEXT:    vfrczpd (%rsi), %ymm1 # sched: [15:2.00]
-; BDVER12-NEXT:    #NO_APP
-; BDVER12-NEXT:    vzeroupper # sched: [46:4.00]
-; BDVER12-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER3-LABEL: test_vfrczpd:
-; BDVER3:       # %bb.0:
-; BDVER3-NEXT:    #APP
-; BDVER3-NEXT:    vfrczpd %xmm0, %xmm0
-; BDVER3-NEXT:    vfrczpd %ymm1, %ymm1
-; BDVER3-NEXT:    vfrczpd (%rdi), %xmm0
-; BDVER3-NEXT:    vfrczpd (%rsi), %ymm1
-; BDVER3-NEXT:    #NO_APP
-; BDVER3-NEXT:    vzeroupper
-; BDVER3-NEXT:    retq
-;
-; BDVER4-LABEL: test_vfrczpd:
-; BDVER4:       # %bb.0:
-; BDVER4-NEXT:    #APP
-; BDVER4-NEXT:    vfrczpd %xmm0, %xmm0
-; BDVER4-NEXT:    vfrczpd %ymm1, %ymm1
-; BDVER4-NEXT:    vfrczpd (%rdi), %xmm0
-; BDVER4-NEXT:    vfrczpd (%rsi), %ymm1
-; BDVER4-NEXT:    #NO_APP
-; BDVER4-NEXT:    vzeroupper
-; BDVER4-NEXT:    retq
-  call void asm sideeffect "vfrczpd $0, $0 \0a\09 vfrczpd $1, $1 \0a\09 vfrczpd $2, $0 \0a\09 vfrczpd $3, $1", "x,x,*m,*m"(<2 x double> %a0, <4 x double> %a1, <2 x double> *%a2, <4 x double> *%a3)
-  ret void
-}
-
-define void @test_vfrczps(<4 x float> %a0, <4 x double> %a1, <4 x float> *%a2, <4 x double> *%a3) {
-; GENERIC-LABEL: test_vfrczps:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    vfrczps %xmm0, %xmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    vfrczps %ymm1, %ymm1 # sched: [3:1.00]
-; GENERIC-NEXT:    vfrczps (%rdi), %xmm0 # sched: [9:1.00]
-; GENERIC-NEXT:    vfrczps (%rsi), %ymm1 # sched: [10:1.00]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    vzeroupper # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER12-LABEL: test_vfrczps:
-; BDVER12:       # %bb.0:
-; BDVER12-NEXT:    #APP
-; BDVER12-NEXT:    vfrczps %xmm0, %xmm0 # sched: [10:1.00]
-; BDVER12-NEXT:    vfrczps %ymm1, %ymm1 # sched: [10:2.00]
-; BDVER12-NEXT:    vfrczps (%rdi), %xmm0 # sched: [15:1.00]
-; BDVER12-NEXT:    vfrczps (%rsi), %ymm1 # sched: [15:2.00]
-; BDVER12-NEXT:    #NO_APP
-; BDVER12-NEXT:    vzeroupper # sched: [46:4.00]
-; BDVER12-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER3-LABEL: test_vfrczps:
-; BDVER3:       # %bb.0:
-; BDVER3-NEXT:    #APP
-; BDVER3-NEXT:    vfrczps %xmm0, %xmm0
-; BDVER3-NEXT:    vfrczps %ymm1, %ymm1
-; BDVER3-NEXT:    vfrczps (%rdi), %xmm0
-; BDVER3-NEXT:    vfrczps (%rsi), %ymm1
-; BDVER3-NEXT:    #NO_APP
-; BDVER3-NEXT:    vzeroupper
-; BDVER3-NEXT:    retq
-;
-; BDVER4-LABEL: test_vfrczps:
-; BDVER4:       # %bb.0:
-; BDVER4-NEXT:    #APP
-; BDVER4-NEXT:    vfrczps %xmm0, %xmm0
-; BDVER4-NEXT:    vfrczps %ymm1, %ymm1
-; BDVER4-NEXT:    vfrczps (%rdi), %xmm0
-; BDVER4-NEXT:    vfrczps (%rsi), %ymm1
-; BDVER4-NEXT:    #NO_APP
-; BDVER4-NEXT:    vzeroupper
-; BDVER4-NEXT:    retq
-  call void asm sideeffect "vfrczps $0, $0 \0a\09 vfrczps $1, $1 \0a\09 vfrczps $2, $0 \0a\09 vfrczps $3, $1", "x,x,*m,*m"(<4 x float> %a0, <4 x double> %a1, <4 x float> *%a2, <4 x double> *%a3)
-  ret void
-}
-
-define void @test_vfrczsd(<2 x double> %a0, <2 x double> *%a1) {
-; GENERIC-LABEL: test_vfrczsd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    vfrczsd %xmm0, %xmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    vfrczsd (%rdi), %xmm0 # sched: [9:1.00]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER12-LABEL: test_vfrczsd:
-; BDVER12:       # %bb.0:
-; BDVER12-NEXT:    #APP
-; BDVER12-NEXT:    vfrczsd %xmm0, %xmm0 # sched: [10:1.00]
-; BDVER12-NEXT:    vfrczsd (%rdi), %xmm0 # sched: [15:1.00]
-; BDVER12-NEXT:    #NO_APP
-; BDVER12-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER3-LABEL: test_vfrczsd:
-; BDVER3:       # %bb.0:
-; BDVER3-NEXT:    #APP
-; BDVER3-NEXT:    vfrczsd %xmm0, %xmm0
-; BDVER3-NEXT:    vfrczsd (%rdi), %xmm0
-; BDVER3-NEXT:    #NO_APP
-; BDVER3-NEXT:    retq
-;
-; BDVER4-LABEL: test_vfrczsd:
-; BDVER4:       # %bb.0:
-; BDVER4-NEXT:    #APP
-; BDVER4-NEXT:    vfrczsd %xmm0, %xmm0
-; BDVER4-NEXT:    vfrczsd (%rdi), %xmm0
-; BDVER4-NEXT:    #NO_APP
-; BDVER4-NEXT:    retq
-  call void asm sideeffect "vfrczsd $0, $0 \0a\09 vfrczsd $1, $0", "x,*m"(<2 x double> %a0, <2 x double> *%a1)
-  ret void
-}
-
-define void @test_vfrczss(<4 x float> %a0, <4 x double> *%a1) {
-; GENERIC-LABEL: test_vfrczss:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    vfrczss %xmm0, %xmm0 # sched: [3:1.00]
-; GENERIC-NEXT:    vfrczss (%rdi), %xmm0 # sched: [9:1.00]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER12-LABEL: test_vfrczss:
-; BDVER12:       # %bb.0:
-; BDVER12-NEXT:    #APP
-; BDVER12-NEXT:    vfrczss %xmm0, %xmm0 # sched: [10:1.00]
-; BDVER12-NEXT:    vfrczss (%rdi), %xmm0 # sched: [15:1.00]
-; BDVER12-NEXT:    #NO_APP
-; BDVER12-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER3-LABEL: test_vfrczss:
-; BDVER3:       # %bb.0:
-; BDVER3-NEXT:    #APP
-; BDVER3-NEXT:    vfrczss %xmm0, %xmm0
-; BDVER3-NEXT:    vfrczss (%rdi), %xmm0
-; BDVER3-NEXT:    #NO_APP
-; BDVER3-NEXT:    retq
-;
-; BDVER4-LABEL: test_vfrczss:
-; BDVER4:       # %bb.0:
-; BDVER4-NEXT:    #APP
-; BDVER4-NEXT:    vfrczss %xmm0, %xmm0
-; BDVER4-NEXT:    vfrczss (%rdi), %xmm0
-; BDVER4-NEXT:    #NO_APP
-; BDVER4-NEXT:    retq
-  call void asm sideeffect "vfrczss $0, $0 \0a\09 vfrczss $1, $0", "x,*m"(<4 x float> %a0, <4 x double> *%a1)
-  ret void
-}
-
-define void @test_vpcmov_128(<2 x i64> %a0, <2 x i64> %a1, <2 x i64> %a2, <2 x i64> *%a3) {
-; GENERIC-LABEL: test_vpcmov_128:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    vpcmov %xmm2, %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    vpcmov (%rdi), %xmm1, %xmm0, %xmm0 # sched: [7:0.50]
-; GENERIC-NEXT:    vpcmov %xmm2, (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER12-LABEL: test_vpcmov_128:
-; BDVER12:       # %bb.0:
-; BDVER12-NEXT:    #APP
-; BDVER12-NEXT:    vpcmov %xmm2, %xmm1, %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER12-NEXT:    vpcmov (%rdi), %xmm1, %xmm0, %xmm0 # sched: [7:0.50]
-; BDVER12-NEXT:    vpcmov %xmm2, (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; BDVER12-NEXT:    #NO_APP
-; BDVER12-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER3-LABEL: test_vpcmov_128:
-; BDVER3:       # %bb.0:
-; BDVER3-NEXT:    #APP
-; BDVER3-NEXT:    vpcmov %xmm2, %xmm1, %xmm0, %xmm0
-; BDVER3-NEXT:    vpcmov (%rdi), %xmm1, %xmm0, %xmm0
-; BDVER3-NEXT:    vpcmov %xmm2, (%rdi), %xmm0, %xmm0
-; BDVER3-NEXT:    #NO_APP
-; BDVER3-NEXT:    retq
-;
-; BDVER4-LABEL: test_vpcmov_128:
-; BDVER4:       # %bb.0:
-; BDVER4-NEXT:    #APP
-; BDVER4-NEXT:    vpcmov %xmm2, %xmm1, %xmm0, %xmm0
-; BDVER4-NEXT:    vpcmov (%rdi), %xmm1, %xmm0, %xmm0
-; BDVER4-NEXT:    vpcmov %xmm2, (%rdi), %xmm0, %xmm0
-; BDVER4-NEXT:    #NO_APP
-; BDVER4-NEXT:    retq
-  call void asm sideeffect "vpcmov $2, $1, $0, $0 \0a\09 vpcmov $3, $1, $0, $0 \0a\09 vpcmov $2, $3, $0, $0", "x,x,x,*m"(<2 x i64> %a0, <2 x i64> %a1, <2 x i64> %a2, <2 x i64> *%a3)
-  ret void
-}
-
-define void @test_vpcmov_256(<4 x i64> %a0, <4 x i64> %a1, <4 x i64> %a2, <4 x i64> *%a3) {
-; GENERIC-LABEL: test_vpcmov_256:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    vpcmov %ymm2, %ymm1, %ymm0, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vpcmov (%rdi), %ymm1, %ymm0, %ymm0 # sched: [8:1.00]
-; GENERIC-NEXT:    vpcmov %ymm2, (%rdi), %ymm0, %ymm0 # sched: [8:1.00]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    vzeroupper # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER12-LABEL: test_vpcmov_256:
-; BDVER12:       # %bb.0:
-; BDVER12-NEXT:    #APP
-; BDVER12-NEXT:    vpcmov %ymm2, %ymm1, %ymm0, %ymm0 # sched: [2:0.50]
-; BDVER12-NEXT:    vpcmov (%rdi), %ymm1, %ymm0, %ymm0 # sched: [7:1.00]
-; BDVER12-NEXT:    vpcmov %ymm2, (%rdi), %ymm0, %ymm0 # sched: [7:1.00]
-; BDVER12-NEXT:    #NO_APP
-; BDVER12-NEXT:    vzeroupper # sched: [46:4.00]
-; BDVER12-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER3-LABEL: test_vpcmov_256:
-; BDVER3:       # %bb.0:
-; BDVER3-NEXT:    #APP
-; BDVER3-NEXT:    vpcmov %ymm2, %ymm1, %ymm0, %ymm0
-; BDVER3-NEXT:    vpcmov (%rdi), %ymm1, %ymm0, %ymm0
-; BDVER3-NEXT:    vpcmov %ymm2, (%rdi), %ymm0, %ymm0
-; BDVER3-NEXT:    #NO_APP
-; BDVER3-NEXT:    vzeroupper
-; BDVER3-NEXT:    retq
-;
-; BDVER4-LABEL: test_vpcmov_256:
-; BDVER4:       # %bb.0:
-; BDVER4-NEXT:    #APP
-; BDVER4-NEXT:    vpcmov %ymm2, %ymm1, %ymm0, %ymm0
-; BDVER4-NEXT:    vpcmov (%rdi), %ymm1, %ymm0, %ymm0
-; BDVER4-NEXT:    vpcmov %ymm2, (%rdi), %ymm0, %ymm0
-; BDVER4-NEXT:    #NO_APP
-; BDVER4-NEXT:    vzeroupper
-; BDVER4-NEXT:    retq
-  call void asm sideeffect "vpcmov $2, $1, $0, $0 \0a\09 vpcmov $3, $1, $0, $0 \0a\09 vpcmov $2, $3, $0, $0", "x,x,x,*m"(<4 x i64> %a0, <4 x i64> %a1, <4 x i64> %a2, <4 x i64> *%a3)
-  ret void
-}
-
-define void @test_vpcom(<2 x i64> %a0, <2 x i64> %a1, <2 x i64> *%a2) {
-; GENERIC-LABEL: test_vpcom:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    vpcomb $3, %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    vpcomd $3, %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    vpcomq $3, %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    vpcomw $3, %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    vpcomb $3, (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; GENERIC-NEXT:    vpcomd $3, (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; GENERIC-NEXT:    vpcomq $3, (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; GENERIC-NEXT:    vpcomw $3, (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER12-LABEL: test_vpcom:
-; BDVER12:       # %bb.0:
-; BDVER12-NEXT:    #APP
-; BDVER12-NEXT:    vpcomb $3, %xmm1, %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER12-NEXT:    vpcomd $3, %xmm1, %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER12-NEXT:    vpcomq $3, %xmm1, %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER12-NEXT:    vpcomw $3, %xmm1, %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER12-NEXT:    vpcomb $3, (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; BDVER12-NEXT:    vpcomd $3, (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; BDVER12-NEXT:    vpcomq $3, (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; BDVER12-NEXT:    vpcomw $3, (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; BDVER12-NEXT:    #NO_APP
-; BDVER12-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER3-LABEL: test_vpcom:
-; BDVER3:       # %bb.0:
-; BDVER3-NEXT:    #APP
-; BDVER3-NEXT:    vpcomb $3, %xmm1, %xmm0, %xmm0
-; BDVER3-NEXT:    vpcomd $3, %xmm1, %xmm0, %xmm0
-; BDVER3-NEXT:    vpcomq $3, %xmm1, %xmm0, %xmm0
-; BDVER3-NEXT:    vpcomw $3, %xmm1, %xmm0, %xmm0
-; BDVER3-NEXT:    vpcomb $3, (%rdi), %xmm0, %xmm0
-; BDVER3-NEXT:    vpcomd $3, (%rdi), %xmm0, %xmm0
-; BDVER3-NEXT:    vpcomq $3, (%rdi), %xmm0, %xmm0
-; BDVER3-NEXT:    vpcomw $3, (%rdi), %xmm0, %xmm0
-; BDVER3-NEXT:    #NO_APP
-; BDVER3-NEXT:    retq
-;
-; BDVER4-LABEL: test_vpcom:
-; BDVER4:       # %bb.0:
-; BDVER4-NEXT:    #APP
-; BDVER4-NEXT:    vpcomb $3, %xmm1, %xmm0, %xmm0
-; BDVER4-NEXT:    vpcomd $3, %xmm1, %xmm0, %xmm0
-; BDVER4-NEXT:    vpcomq $3, %xmm1, %xmm0, %xmm0
-; BDVER4-NEXT:    vpcomw $3, %xmm1, %xmm0, %xmm0
-; BDVER4-NEXT:    vpcomb $3, (%rdi), %xmm0, %xmm0
-; BDVER4-NEXT:    vpcomd $3, (%rdi), %xmm0, %xmm0
-; BDVER4-NEXT:    vpcomq $3, (%rdi), %xmm0, %xmm0
-; BDVER4-NEXT:    vpcomw $3, (%rdi), %xmm0, %xmm0
-; BDVER4-NEXT:    #NO_APP
-; BDVER4-NEXT:    retq
-  call void asm sideeffect "vpcomb $3, $1, $0, $0 \0a\09 vpcomd $3, $1, $0, $0 \0a\09 vpcomq $3, $1, $0, $0 \0a\09 vpcomw $3, $1, $0, $0 \0a\09 vpcomb $3, $2, $0, $0 \0a\09 vpcomd $3, $2, $0, $0 \0a\09 vpcomq $3, $2, $0, $0 \0a\09 vpcomw $3, $2, $0, $0", "x,x,*m,i"(<2 x i64> %a0, <2 x i64> %a1, <2 x i64> *%a2, i8 3)
-  ret void
-}
-
-define void @test_vpcomu(<2 x i64> %a0, <2 x i64> %a1, <2 x i64> *%a2) {
-; GENERIC-LABEL: test_vpcomu:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    vpcomub $3, %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    vpcomud $3, %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    vpcomuq $3, %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    vpcomuw $3, %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    vpcomub $3, (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; GENERIC-NEXT:    vpcomud $3, (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; GENERIC-NEXT:    vpcomuq $3, (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; GENERIC-NEXT:    vpcomuw $3, (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER12-LABEL: test_vpcomu:
-; BDVER12:       # %bb.0:
-; BDVER12-NEXT:    #APP
-; BDVER12-NEXT:    vpcomub $3, %xmm1, %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER12-NEXT:    vpcomud $3, %xmm1, %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER12-NEXT:    vpcomuq $3, %xmm1, %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER12-NEXT:    vpcomuw $3, %xmm1, %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER12-NEXT:    vpcomub $3, (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; BDVER12-NEXT:    vpcomud $3, (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; BDVER12-NEXT:    vpcomuq $3, (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; BDVER12-NEXT:    vpcomuw $3, (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; BDVER12-NEXT:    #NO_APP
-; BDVER12-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER3-LABEL: test_vpcomu:
-; BDVER3:       # %bb.0:
-; BDVER3-NEXT:    #APP
-; BDVER3-NEXT:    vpcomub $3, %xmm1, %xmm0, %xmm0
-; BDVER3-NEXT:    vpcomud $3, %xmm1, %xmm0, %xmm0
-; BDVER3-NEXT:    vpcomuq $3, %xmm1, %xmm0, %xmm0
-; BDVER3-NEXT:    vpcomuw $3, %xmm1, %xmm0, %xmm0
-; BDVER3-NEXT:    vpcomub $3, (%rdi), %xmm0, %xmm0
-; BDVER3-NEXT:    vpcomud $3, (%rdi), %xmm0, %xmm0
-; BDVER3-NEXT:    vpcomuq $3, (%rdi), %xmm0, %xmm0
-; BDVER3-NEXT:    vpcomuw $3, (%rdi), %xmm0, %xmm0
-; BDVER3-NEXT:    #NO_APP
-; BDVER3-NEXT:    retq
-;
-; BDVER4-LABEL: test_vpcomu:
-; BDVER4:       # %bb.0:
-; BDVER4-NEXT:    #APP
-; BDVER4-NEXT:    vpcomub $3, %xmm1, %xmm0, %xmm0
-; BDVER4-NEXT:    vpcomud $3, %xmm1, %xmm0, %xmm0
-; BDVER4-NEXT:    vpcomuq $3, %xmm1, %xmm0, %xmm0
-; BDVER4-NEXT:    vpcomuw $3, %xmm1, %xmm0, %xmm0
-; BDVER4-NEXT:    vpcomub $3, (%rdi), %xmm0, %xmm0
-; BDVER4-NEXT:    vpcomud $3, (%rdi), %xmm0, %xmm0
-; BDVER4-NEXT:    vpcomuq $3, (%rdi), %xmm0, %xmm0
-; BDVER4-NEXT:    vpcomuw $3, (%rdi), %xmm0, %xmm0
-; BDVER4-NEXT:    #NO_APP
-; BDVER4-NEXT:    retq
-  call void asm sideeffect "vpcomub $3, $1, $0, $0 \0a\09 vpcomud $3, $1, $0, $0 \0a\09 vpcomuq $3, $1, $0, $0 \0a\09 vpcomuw $3, $1, $0, $0 \0a\09 vpcomub $3, $2, $0, $0 \0a\09 vpcomud $3, $2, $0, $0 \0a\09 vpcomuq $3, $2, $0, $0 \0a\09 vpcomuw $3, $2, $0, $0", "x,x,*m,i"(<2 x i64> %a0, <2 x i64> %a1, <2 x i64> *%a2, i8 3)
-  ret void
-}
-
-define void @test_vpermil2pd_128(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, <2 x double> *%a3) {
-; GENERIC-LABEL: test_vpermil2pd_128:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    vpermil2pd $3, %xmm2, %xmm1, %xmm0, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vpermil2pd $3, %xmm2, (%rdi), %xmm0, %xmm0 # sched: [7:1.00]
-; GENERIC-NEXT:    vpermil2pd $3, (%rdi), %xmm1, %xmm0, %xmm0 # sched: [7:1.00]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER12-LABEL: test_vpermil2pd_128:
-; BDVER12:       # %bb.0:
-; BDVER12-NEXT:    #APP
-; BDVER12-NEXT:    vpermil2pd $3, %xmm2, %xmm1, %xmm0, %xmm0 # sched: [3:2.00]
-; BDVER12-NEXT:    vpermil2pd $3, %xmm2, (%rdi), %xmm0, %xmm0 # sched: [8:2.00]
-; BDVER12-NEXT:    vpermil2pd $3, (%rdi), %xmm1, %xmm0, %xmm0 # sched: [8:2.00]
-; BDVER12-NEXT:    #NO_APP
-; BDVER12-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER3-LABEL: test_vpermil2pd_128:
-; BDVER3:       # %bb.0:
-; BDVER3-NEXT:    #APP
-; BDVER3-NEXT:    vpermil2pd $3, %xmm2, %xmm1, %xmm0, %xmm0
-; BDVER3-NEXT:    vpermil2pd $3, %xmm2, (%rdi), %xmm0, %xmm0
-; BDVER3-NEXT:    vpermil2pd $3, (%rdi), %xmm1, %xmm0, %xmm0
-; BDVER3-NEXT:    #NO_APP
-; BDVER3-NEXT:    retq
-;
-; BDVER4-LABEL: test_vpermil2pd_128:
-; BDVER4:       # %bb.0:
-; BDVER4-NEXT:    #APP
-; BDVER4-NEXT:    vpermil2pd $3, %xmm2, %xmm1, %xmm0, %xmm0
-; BDVER4-NEXT:    vpermil2pd $3, %xmm2, (%rdi), %xmm0, %xmm0
-; BDVER4-NEXT:    vpermil2pd $3, (%rdi), %xmm1, %xmm0, %xmm0
-; BDVER4-NEXT:    #NO_APP
-; BDVER4-NEXT:    retq
-  call void asm sideeffect "vpermil2pd $4, $2, $1, $0, $0 \0a\09 vpermil2pd $4, $2, $3, $0, $0 \0a\09 vpermil2pd $4, $3, $1, $0, $0", "x,x,x,*m,i"(<2 x double> %a0, <2 x double> %a1, <2 x double> %a2, <2 x double> *%a3, i8 3)
-  ret void
-}
-
-define void @test_vpermil2pd_256(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2, <4 x double> *%a3) {
-; GENERIC-LABEL: test_vpermil2pd_256:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    vpermil2pd $3, %ymm2, %ymm1, %ymm0, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vpermil2pd $3, %ymm2, (%rdi), %ymm0, %ymm0 # sched: [8:1.00]
-; GENERIC-NEXT:    vpermil2pd $3, (%rdi), %ymm1, %ymm0, %ymm0 # sched: [8:1.00]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    vzeroupper # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER12-LABEL: test_vpermil2pd_256:
-; BDVER12:       # %bb.0:
-; BDVER12-NEXT:    #APP
-; BDVER12-NEXT:    vpermil2pd $3, %ymm2, %ymm1, %ymm0, %ymm0 # sched: [3:3.00]
-; BDVER12-NEXT:    vpermil2pd $3, %ymm2, (%rdi), %ymm0, %ymm0 # sched: [8:3.00]
-; BDVER12-NEXT:    vpermil2pd $3, (%rdi), %ymm1, %ymm0, %ymm0 # sched: [8:3.00]
-; BDVER12-NEXT:    #NO_APP
-; BDVER12-NEXT:    vzeroupper # sched: [46:4.00]
-; BDVER12-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER3-LABEL: test_vpermil2pd_256:
-; BDVER3:       # %bb.0:
-; BDVER3-NEXT:    #APP
-; BDVER3-NEXT:    vpermil2pd $3, %ymm2, %ymm1, %ymm0, %ymm0
-; BDVER3-NEXT:    vpermil2pd $3, %ymm2, (%rdi), %ymm0, %ymm0
-; BDVER3-NEXT:    vpermil2pd $3, (%rdi), %ymm1, %ymm0, %ymm0
-; BDVER3-NEXT:    #NO_APP
-; BDVER3-NEXT:    vzeroupper
-; BDVER3-NEXT:    retq
-;
-; BDVER4-LABEL: test_vpermil2pd_256:
-; BDVER4:       # %bb.0:
-; BDVER4-NEXT:    #APP
-; BDVER4-NEXT:    vpermil2pd $3, %ymm2, %ymm1, %ymm0, %ymm0
-; BDVER4-NEXT:    vpermil2pd $3, %ymm2, (%rdi), %ymm0, %ymm0
-; BDVER4-NEXT:    vpermil2pd $3, (%rdi), %ymm1, %ymm0, %ymm0
-; BDVER4-NEXT:    #NO_APP
-; BDVER4-NEXT:    vzeroupper
-; BDVER4-NEXT:    retq
-  call void asm sideeffect "vpermil2pd $4, $2, $1, $0, $0 \0a\09 vpermil2pd $4, $2, $3, $0, $0 \0a\09 vpermil2pd $4, $3, $1, $0, $0", "x,x,x,*m,i"(<4 x double> %a0, <4 x double> %a1, <4 x double> %a2, <4 x double> *%a3, i8 3)
-  ret void
-}
-
-define void @test_vpermil2ps_128(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, <4 x float> *%a3) {
-; GENERIC-LABEL: test_vpermil2ps_128:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    vpermil2ps $3, %xmm2, %xmm1, %xmm0, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vpermil2ps $3, %xmm2, (%rdi), %xmm0, %xmm0 # sched: [7:1.00]
-; GENERIC-NEXT:    vpermil2ps $3, (%rdi), %xmm1, %xmm0, %xmm0 # sched: [7:1.00]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER12-LABEL: test_vpermil2ps_128:
-; BDVER12:       # %bb.0:
-; BDVER12-NEXT:    #APP
-; BDVER12-NEXT:    vpermil2ps $3, %xmm2, %xmm1, %xmm0, %xmm0 # sched: [3:2.00]
-; BDVER12-NEXT:    vpermil2ps $3, %xmm2, (%rdi), %xmm0, %xmm0 # sched: [8:2.00]
-; BDVER12-NEXT:    vpermil2ps $3, (%rdi), %xmm1, %xmm0, %xmm0 # sched: [8:2.00]
-; BDVER12-NEXT:    #NO_APP
-; BDVER12-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER3-LABEL: test_vpermil2ps_128:
-; BDVER3:       # %bb.0:
-; BDVER3-NEXT:    #APP
-; BDVER3-NEXT:    vpermil2ps $3, %xmm2, %xmm1, %xmm0, %xmm0
-; BDVER3-NEXT:    vpermil2ps $3, %xmm2, (%rdi), %xmm0, %xmm0
-; BDVER3-NEXT:    vpermil2ps $3, (%rdi), %xmm1, %xmm0, %xmm0
-; BDVER3-NEXT:    #NO_APP
-; BDVER3-NEXT:    retq
-;
-; BDVER4-LABEL: test_vpermil2ps_128:
-; BDVER4:       # %bb.0:
-; BDVER4-NEXT:    #APP
-; BDVER4-NEXT:    vpermil2ps $3, %xmm2, %xmm1, %xmm0, %xmm0
-; BDVER4-NEXT:    vpermil2ps $3, %xmm2, (%rdi), %xmm0, %xmm0
-; BDVER4-NEXT:    vpermil2ps $3, (%rdi), %xmm1, %xmm0, %xmm0
-; BDVER4-NEXT:    #NO_APP
-; BDVER4-NEXT:    retq
-  call void asm sideeffect "vpermil2ps $4, $2, $1, $0, $0 \0a\09 vpermil2ps $4, $2, $3, $0, $0 \0a\09 vpermil2ps $4, $3, $1, $0, $0", "x,x,x,*m,i"(<4 x float> %a0, <4 x float> %a1, <4 x float> %a2, <4 x float> *%a3, i8 3)
-  ret void
-}
-
-define void @test_vpermil2ps_256(<8 x float> %a0, <8 x float> %a1, <8 x float> %a2, <8 x float> *%a3) {
-; GENERIC-LABEL: test_vpermil2ps_256:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    vpermil2ps $3, %ymm2, %ymm1, %ymm0, %ymm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vpermil2ps $3, %ymm2, (%rdi), %ymm0, %ymm0 # sched: [8:1.00]
-; GENERIC-NEXT:    vpermil2ps $3, (%rdi), %ymm1, %ymm0, %ymm0 # sched: [8:1.00]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    vzeroupper # sched: [1:1.00]
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER12-LABEL: test_vpermil2ps_256:
-; BDVER12:       # %bb.0:
-; BDVER12-NEXT:    #APP
-; BDVER12-NEXT:    vpermil2ps $3, %ymm2, %ymm1, %ymm0, %ymm0 # sched: [3:3.00]
-; BDVER12-NEXT:    vpermil2ps $3, %ymm2, (%rdi), %ymm0, %ymm0 # sched: [8:3.00]
-; BDVER12-NEXT:    vpermil2ps $3, (%rdi), %ymm1, %ymm0, %ymm0 # sched: [8:3.00]
-; BDVER12-NEXT:    #NO_APP
-; BDVER12-NEXT:    vzeroupper # sched: [46:4.00]
-; BDVER12-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER3-LABEL: test_vpermil2ps_256:
-; BDVER3:       # %bb.0:
-; BDVER3-NEXT:    #APP
-; BDVER3-NEXT:    vpermil2ps $3, %ymm2, %ymm1, %ymm0, %ymm0
-; BDVER3-NEXT:    vpermil2ps $3, %ymm2, (%rdi), %ymm0, %ymm0
-; BDVER3-NEXT:    vpermil2ps $3, (%rdi), %ymm1, %ymm0, %ymm0
-; BDVER3-NEXT:    #NO_APP
-; BDVER3-NEXT:    vzeroupper
-; BDVER3-NEXT:    retq
-;
-; BDVER4-LABEL: test_vpermil2ps_256:
-; BDVER4:       # %bb.0:
-; BDVER4-NEXT:    #APP
-; BDVER4-NEXT:    vpermil2ps $3, %ymm2, %ymm1, %ymm0, %ymm0
-; BDVER4-NEXT:    vpermil2ps $3, %ymm2, (%rdi), %ymm0, %ymm0
-; BDVER4-NEXT:    vpermil2ps $3, (%rdi), %ymm1, %ymm0, %ymm0
-; BDVER4-NEXT:    #NO_APP
-; BDVER4-NEXT:    vzeroupper
-; BDVER4-NEXT:    retq
-  call void asm sideeffect "vpermil2ps $4, $2, $1, $0, $0 \0a\09 vpermil2ps $4, $2, $3, $0, $0 \0a\09 vpermil2ps $4, $3, $1, $0, $0", "x,x,x,*m,i"(<8 x float> %a0, <8 x float> %a1, <8 x float> %a2, <8 x float> *%a3, i8 3)
-  ret void
-}
-
-define void @test_vphaddbd(<2 x i64> %a0, <2 x i64> *%a1) {
-; GENERIC-LABEL: test_vphaddbd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    vphaddbd %xmm0, %xmm0 # sched: [3:1.50]
-; GENERIC-NEXT:    vphaddbd (%rdi), %xmm0 # sched: [9:1.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER12-LABEL: test_vphaddbd:
-; BDVER12:       # %bb.0:
-; BDVER12-NEXT:    #APP
-; BDVER12-NEXT:    vphaddbd %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER12-NEXT:    vphaddbd (%rdi), %xmm0 # sched: [7:0.50]
-; BDVER12-NEXT:    #NO_APP
-; BDVER12-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER3-LABEL: test_vphaddbd:
-; BDVER3:       # %bb.0:
-; BDVER3-NEXT:    #APP
-; BDVER3-NEXT:    vphaddbd %xmm0, %xmm0
-; BDVER3-NEXT:    vphaddbd (%rdi), %xmm0
-; BDVER3-NEXT:    #NO_APP
-; BDVER3-NEXT:    retq
-;
-; BDVER4-LABEL: test_vphaddbd:
-; BDVER4:       # %bb.0:
-; BDVER4-NEXT:    #APP
-; BDVER4-NEXT:    vphaddbd %xmm0, %xmm0
-; BDVER4-NEXT:    vphaddbd (%rdi), %xmm0
-; BDVER4-NEXT:    #NO_APP
-; BDVER4-NEXT:    retq
-  call void asm sideeffect "vphaddbd $0, $0 \0a\09 vphaddbd $1, $0", "x,*m"(<2 x i64> %a0, <2 x i64> *%a1)
-  ret void
-}
-
-define void @test_vphaddbq(<2 x i64> %a0, <2 x i64> *%a1) {
-; GENERIC-LABEL: test_vphaddbq:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    vphaddbq %xmm0, %xmm0 # sched: [3:1.50]
-; GENERIC-NEXT:    vphaddbq (%rdi), %xmm0 # sched: [9:1.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER12-LABEL: test_vphaddbq:
-; BDVER12:       # %bb.0:
-; BDVER12-NEXT:    #APP
-; BDVER12-NEXT:    vphaddbq %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER12-NEXT:    vphaddbq (%rdi), %xmm0 # sched: [7:0.50]
-; BDVER12-NEXT:    #NO_APP
-; BDVER12-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER3-LABEL: test_vphaddbq:
-; BDVER3:       # %bb.0:
-; BDVER3-NEXT:    #APP
-; BDVER3-NEXT:    vphaddbq %xmm0, %xmm0
-; BDVER3-NEXT:    vphaddbq (%rdi), %xmm0
-; BDVER3-NEXT:    #NO_APP
-; BDVER3-NEXT:    retq
-;
-; BDVER4-LABEL: test_vphaddbq:
-; BDVER4:       # %bb.0:
-; BDVER4-NEXT:    #APP
-; BDVER4-NEXT:    vphaddbq %xmm0, %xmm0
-; BDVER4-NEXT:    vphaddbq (%rdi), %xmm0
-; BDVER4-NEXT:    #NO_APP
-; BDVER4-NEXT:    retq
-  call void asm sideeffect "vphaddbq $0, $0 \0a\09 vphaddbq $1, $0", "x,*m"(<2 x i64> %a0, <2 x i64> *%a1)
-  ret void
-}
-
-define void @test_vphaddbw(<2 x i64> %a0, <2 x i64> *%a1) {
-; GENERIC-LABEL: test_vphaddbw:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    vphaddbw %xmm0, %xmm0 # sched: [3:1.50]
-; GENERIC-NEXT:    vphaddbw (%rdi), %xmm0 # sched: [9:1.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER12-LABEL: test_vphaddbw:
-; BDVER12:       # %bb.0:
-; BDVER12-NEXT:    #APP
-; BDVER12-NEXT:    vphaddbw %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER12-NEXT:    vphaddbw (%rdi), %xmm0 # sched: [7:0.50]
-; BDVER12-NEXT:    #NO_APP
-; BDVER12-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER3-LABEL: test_vphaddbw:
-; BDVER3:       # %bb.0:
-; BDVER3-NEXT:    #APP
-; BDVER3-NEXT:    vphaddbw %xmm0, %xmm0
-; BDVER3-NEXT:    vphaddbw (%rdi), %xmm0
-; BDVER3-NEXT:    #NO_APP
-; BDVER3-NEXT:    retq
-;
-; BDVER4-LABEL: test_vphaddbw:
-; BDVER4:       # %bb.0:
-; BDVER4-NEXT:    #APP
-; BDVER4-NEXT:    vphaddbw %xmm0, %xmm0
-; BDVER4-NEXT:    vphaddbw (%rdi), %xmm0
-; BDVER4-NEXT:    #NO_APP
-; BDVER4-NEXT:    retq
-  call void asm sideeffect "vphaddbw $0, $0 \0a\09 vphaddbw $1, $0", "x,*m"(<2 x i64> %a0, <2 x i64> *%a1)
-  ret void
-}
-
-define void @test_vphadddq(<2 x i64> %a0, <2 x i64> *%a1) {
-; GENERIC-LABEL: test_vphadddq:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    vphadddq %xmm0, %xmm0 # sched: [3:1.50]
-; GENERIC-NEXT:    vphadddq (%rdi), %xmm0 # sched: [9:1.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER12-LABEL: test_vphadddq:
-; BDVER12:       # %bb.0:
-; BDVER12-NEXT:    #APP
-; BDVER12-NEXT:    vphadddq %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER12-NEXT:    vphadddq (%rdi), %xmm0 # sched: [7:0.50]
-; BDVER12-NEXT:    #NO_APP
-; BDVER12-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER3-LABEL: test_vphadddq:
-; BDVER3:       # %bb.0:
-; BDVER3-NEXT:    #APP
-; BDVER3-NEXT:    vphadddq %xmm0, %xmm0
-; BDVER3-NEXT:    vphadddq (%rdi), %xmm0
-; BDVER3-NEXT:    #NO_APP
-; BDVER3-NEXT:    retq
-;
-; BDVER4-LABEL: test_vphadddq:
-; BDVER4:       # %bb.0:
-; BDVER4-NEXT:    #APP
-; BDVER4-NEXT:    vphadddq %xmm0, %xmm0
-; BDVER4-NEXT:    vphadddq (%rdi), %xmm0
-; BDVER4-NEXT:    #NO_APP
-; BDVER4-NEXT:    retq
-  call void asm sideeffect "vphadddq $0, $0 \0a\09 vphadddq $1, $0", "x,*m"(<2 x i64> %a0, <2 x i64> *%a1)
-  ret void
-}
-
-define void @test_vphaddubd(<2 x i64> %a0, <2 x i64> *%a1) {
-; GENERIC-LABEL: test_vphaddubd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    vphaddubd %xmm0, %xmm0 # sched: [3:1.50]
-; GENERIC-NEXT:    vphaddubd (%rdi), %xmm0 # sched: [9:1.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER12-LABEL: test_vphaddubd:
-; BDVER12:       # %bb.0:
-; BDVER12-NEXT:    #APP
-; BDVER12-NEXT:    vphaddubd %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER12-NEXT:    vphaddubd (%rdi), %xmm0 # sched: [7:0.50]
-; BDVER12-NEXT:    #NO_APP
-; BDVER12-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER3-LABEL: test_vphaddubd:
-; BDVER3:       # %bb.0:
-; BDVER3-NEXT:    #APP
-; BDVER3-NEXT:    vphaddubd %xmm0, %xmm0
-; BDVER3-NEXT:    vphaddubd (%rdi), %xmm0
-; BDVER3-NEXT:    #NO_APP
-; BDVER3-NEXT:    retq
-;
-; BDVER4-LABEL: test_vphaddubd:
-; BDVER4:       # %bb.0:
-; BDVER4-NEXT:    #APP
-; BDVER4-NEXT:    vphaddubd %xmm0, %xmm0
-; BDVER4-NEXT:    vphaddubd (%rdi), %xmm0
-; BDVER4-NEXT:    #NO_APP
-; BDVER4-NEXT:    retq
-  call void asm sideeffect "vphaddubd $0, $0 \0a\09 vphaddubd $1, $0", "x,*m"(<2 x i64> %a0, <2 x i64> *%a1)
-  ret void
-}
-
-define void @test_vphaddubq(<2 x i64> %a0, <2 x i64> *%a1) {
-; GENERIC-LABEL: test_vphaddubq:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    vphaddubq %xmm0, %xmm0 # sched: [3:1.50]
-; GENERIC-NEXT:    vphaddubq (%rdi), %xmm0 # sched: [9:1.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER12-LABEL: test_vphaddubq:
-; BDVER12:       # %bb.0:
-; BDVER12-NEXT:    #APP
-; BDVER12-NEXT:    vphaddubq %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER12-NEXT:    vphaddubq (%rdi), %xmm0 # sched: [7:0.50]
-; BDVER12-NEXT:    #NO_APP
-; BDVER12-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER3-LABEL: test_vphaddubq:
-; BDVER3:       # %bb.0:
-; BDVER3-NEXT:    #APP
-; BDVER3-NEXT:    vphaddubq %xmm0, %xmm0
-; BDVER3-NEXT:    vphaddubq (%rdi), %xmm0
-; BDVER3-NEXT:    #NO_APP
-; BDVER3-NEXT:    retq
-;
-; BDVER4-LABEL: test_vphaddubq:
-; BDVER4:       # %bb.0:
-; BDVER4-NEXT:    #APP
-; BDVER4-NEXT:    vphaddubq %xmm0, %xmm0
-; BDVER4-NEXT:    vphaddubq (%rdi), %xmm0
-; BDVER4-NEXT:    #NO_APP
-; BDVER4-NEXT:    retq
-  call void asm sideeffect "vphaddubq $0, $0 \0a\09 vphaddubq $1, $0", "x,*m"(<2 x i64> %a0, <2 x i64> *%a1)
-  ret void
-}
-
-define void @test_vphaddubw(<2 x i64> %a0, <2 x i64> *%a1) {
-; GENERIC-LABEL: test_vphaddubw:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    vphaddubw %xmm0, %xmm0 # sched: [3:1.50]
-; GENERIC-NEXT:    vphaddubw (%rdi), %xmm0 # sched: [9:1.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER12-LABEL: test_vphaddubw:
-; BDVER12:       # %bb.0:
-; BDVER12-NEXT:    #APP
-; BDVER12-NEXT:    vphaddubw %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER12-NEXT:    vphaddubw (%rdi), %xmm0 # sched: [7:0.50]
-; BDVER12-NEXT:    #NO_APP
-; BDVER12-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER3-LABEL: test_vphaddubw:
-; BDVER3:       # %bb.0:
-; BDVER3-NEXT:    #APP
-; BDVER3-NEXT:    vphaddubw %xmm0, %xmm0
-; BDVER3-NEXT:    vphaddubw (%rdi), %xmm0
-; BDVER3-NEXT:    #NO_APP
-; BDVER3-NEXT:    retq
-;
-; BDVER4-LABEL: test_vphaddubw:
-; BDVER4:       # %bb.0:
-; BDVER4-NEXT:    #APP
-; BDVER4-NEXT:    vphaddubw %xmm0, %xmm0
-; BDVER4-NEXT:    vphaddubw (%rdi), %xmm0
-; BDVER4-NEXT:    #NO_APP
-; BDVER4-NEXT:    retq
-  call void asm sideeffect "vphaddubw $0, $0 \0a\09 vphaddubw $1, $0", "x,*m"(<2 x i64> %a0, <2 x i64> *%a1)
-  ret void
-}
-
-define void @test_vphaddudq(<2 x i64> %a0, <2 x i64> *%a1) {
-; GENERIC-LABEL: test_vphaddudq:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    vphaddudq %xmm0, %xmm0 # sched: [3:1.50]
-; GENERIC-NEXT:    vphaddudq (%rdi), %xmm0 # sched: [9:1.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER12-LABEL: test_vphaddudq:
-; BDVER12:       # %bb.0:
-; BDVER12-NEXT:    #APP
-; BDVER12-NEXT:    vphaddudq %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER12-NEXT:    vphaddudq (%rdi), %xmm0 # sched: [7:0.50]
-; BDVER12-NEXT:    #NO_APP
-; BDVER12-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER3-LABEL: test_vphaddudq:
-; BDVER3:       # %bb.0:
-; BDVER3-NEXT:    #APP
-; BDVER3-NEXT:    vphaddudq %xmm0, %xmm0
-; BDVER3-NEXT:    vphaddudq (%rdi), %xmm0
-; BDVER3-NEXT:    #NO_APP
-; BDVER3-NEXT:    retq
-;
-; BDVER4-LABEL: test_vphaddudq:
-; BDVER4:       # %bb.0:
-; BDVER4-NEXT:    #APP
-; BDVER4-NEXT:    vphaddudq %xmm0, %xmm0
-; BDVER4-NEXT:    vphaddudq (%rdi), %xmm0
-; BDVER4-NEXT:    #NO_APP
-; BDVER4-NEXT:    retq
-  call void asm sideeffect "vphaddudq $0, $0 \0a\09 vphaddudq $1, $0", "x,*m"(<2 x i64> %a0, <2 x i64> *%a1)
-  ret void
-}
-
-define void @test_vphadduwd(<2 x i64> %a0, <2 x i64> *%a1) {
-; GENERIC-LABEL: test_vphadduwd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    vphadduwd %xmm0, %xmm0 # sched: [3:1.50]
-; GENERIC-NEXT:    vphadduwd (%rdi), %xmm0 # sched: [9:1.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER12-LABEL: test_vphadduwd:
-; BDVER12:       # %bb.0:
-; BDVER12-NEXT:    #APP
-; BDVER12-NEXT:    vphadduwd %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER12-NEXT:    vphadduwd (%rdi), %xmm0 # sched: [7:0.50]
-; BDVER12-NEXT:    #NO_APP
-; BDVER12-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER3-LABEL: test_vphadduwd:
-; BDVER3:       # %bb.0:
-; BDVER3-NEXT:    #APP
-; BDVER3-NEXT:    vphadduwd %xmm0, %xmm0
-; BDVER3-NEXT:    vphadduwd (%rdi), %xmm0
-; BDVER3-NEXT:    #NO_APP
-; BDVER3-NEXT:    retq
-;
-; BDVER4-LABEL: test_vphadduwd:
-; BDVER4:       # %bb.0:
-; BDVER4-NEXT:    #APP
-; BDVER4-NEXT:    vphadduwd %xmm0, %xmm0
-; BDVER4-NEXT:    vphadduwd (%rdi), %xmm0
-; BDVER4-NEXT:    #NO_APP
-; BDVER4-NEXT:    retq
-  call void asm sideeffect "vphadduwd $0, $0 \0a\09 vphadduwd $1, $0", "x,*m"(<2 x i64> %a0, <2 x i64> *%a1)
-  ret void
-}
-
-define void @test_vphadduwq(<2 x i64> %a0, <2 x i64> *%a1) {
-; GENERIC-LABEL: test_vphadduwq:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    vphadduwq %xmm0, %xmm0 # sched: [3:1.50]
-; GENERIC-NEXT:    vphadduwq (%rdi), %xmm0 # sched: [9:1.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER12-LABEL: test_vphadduwq:
-; BDVER12:       # %bb.0:
-; BDVER12-NEXT:    #APP
-; BDVER12-NEXT:    vphadduwq %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER12-NEXT:    vphadduwq (%rdi), %xmm0 # sched: [7:0.50]
-; BDVER12-NEXT:    #NO_APP
-; BDVER12-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER3-LABEL: test_vphadduwq:
-; BDVER3:       # %bb.0:
-; BDVER3-NEXT:    #APP
-; BDVER3-NEXT:    vphadduwq %xmm0, %xmm0
-; BDVER3-NEXT:    vphadduwq (%rdi), %xmm0
-; BDVER3-NEXT:    #NO_APP
-; BDVER3-NEXT:    retq
-;
-; BDVER4-LABEL: test_vphadduwq:
-; BDVER4:       # %bb.0:
-; BDVER4-NEXT:    #APP
-; BDVER4-NEXT:    vphadduwq %xmm0, %xmm0
-; BDVER4-NEXT:    vphadduwq (%rdi), %xmm0
-; BDVER4-NEXT:    #NO_APP
-; BDVER4-NEXT:    retq
-  call void asm sideeffect "vphadduwq $0, $0 \0a\09 vphadduwq $1, $0", "x,*m"(<2 x i64> %a0, <2 x i64> *%a1)
-  ret void
-}
-
-define void @test_vphaddwd(<2 x i64> %a0, <2 x i64> *%a1) {
-; GENERIC-LABEL: test_vphaddwd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    vphaddwd %xmm0, %xmm0 # sched: [3:1.50]
-; GENERIC-NEXT:    vphaddwd (%rdi), %xmm0 # sched: [9:1.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER12-LABEL: test_vphaddwd:
-; BDVER12:       # %bb.0:
-; BDVER12-NEXT:    #APP
-; BDVER12-NEXT:    vphaddwd %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER12-NEXT:    vphaddwd (%rdi), %xmm0 # sched: [7:0.50]
-; BDVER12-NEXT:    #NO_APP
-; BDVER12-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER3-LABEL: test_vphaddwd:
-; BDVER3:       # %bb.0:
-; BDVER3-NEXT:    #APP
-; BDVER3-NEXT:    vphaddwd %xmm0, %xmm0
-; BDVER3-NEXT:    vphaddwd (%rdi), %xmm0
-; BDVER3-NEXT:    #NO_APP
-; BDVER3-NEXT:    retq
-;
-; BDVER4-LABEL: test_vphaddwd:
-; BDVER4:       # %bb.0:
-; BDVER4-NEXT:    #APP
-; BDVER4-NEXT:    vphaddwd %xmm0, %xmm0
-; BDVER4-NEXT:    vphaddwd (%rdi), %xmm0
-; BDVER4-NEXT:    #NO_APP
-; BDVER4-NEXT:    retq
-  call void asm sideeffect "vphaddwd $0, $0 \0a\09 vphaddwd $1, $0", "x,*m"(<2 x i64> %a0, <2 x i64> *%a1)
-  ret void
-}
-
-define void @test_vphaddwq(<2 x i64> %a0, <2 x i64> *%a1) {
-; GENERIC-LABEL: test_vphaddwq:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    vphaddwq %xmm0, %xmm0 # sched: [3:1.50]
-; GENERIC-NEXT:    vphaddwq (%rdi), %xmm0 # sched: [9:1.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER12-LABEL: test_vphaddwq:
-; BDVER12:       # %bb.0:
-; BDVER12-NEXT:    #APP
-; BDVER12-NEXT:    vphaddwq %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER12-NEXT:    vphaddwq (%rdi), %xmm0 # sched: [7:0.50]
-; BDVER12-NEXT:    #NO_APP
-; BDVER12-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER3-LABEL: test_vphaddwq:
-; BDVER3:       # %bb.0:
-; BDVER3-NEXT:    #APP
-; BDVER3-NEXT:    vphaddwq %xmm0, %xmm0
-; BDVER3-NEXT:    vphaddwq (%rdi), %xmm0
-; BDVER3-NEXT:    #NO_APP
-; BDVER3-NEXT:    retq
-;
-; BDVER4-LABEL: test_vphaddwq:
-; BDVER4:       # %bb.0:
-; BDVER4-NEXT:    #APP
-; BDVER4-NEXT:    vphaddwq %xmm0, %xmm0
-; BDVER4-NEXT:    vphaddwq (%rdi), %xmm0
-; BDVER4-NEXT:    #NO_APP
-; BDVER4-NEXT:    retq
-  call void asm sideeffect "vphaddwq $0, $0 \0a\09 vphaddwq $1, $0", "x,*m"(<2 x i64> %a0, <2 x i64> *%a1)
-  ret void
-}
-
-define void @test_vphsubbw(<2 x i64> %a0, <2 x i64> *%a1) {
-; GENERIC-LABEL: test_vphsubbw:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    vphsubbw %xmm0, %xmm0 # sched: [3:1.50]
-; GENERIC-NEXT:    vphsubbw (%rdi), %xmm0 # sched: [9:1.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER12-LABEL: test_vphsubbw:
-; BDVER12:       # %bb.0:
-; BDVER12-NEXT:    #APP
-; BDVER12-NEXT:    vphsubbw %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER12-NEXT:    vphsubbw (%rdi), %xmm0 # sched: [7:0.50]
-; BDVER12-NEXT:    #NO_APP
-; BDVER12-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER3-LABEL: test_vphsubbw:
-; BDVER3:       # %bb.0:
-; BDVER3-NEXT:    #APP
-; BDVER3-NEXT:    vphsubbw %xmm0, %xmm0
-; BDVER3-NEXT:    vphsubbw (%rdi), %xmm0
-; BDVER3-NEXT:    #NO_APP
-; BDVER3-NEXT:    retq
-;
-; BDVER4-LABEL: test_vphsubbw:
-; BDVER4:       # %bb.0:
-; BDVER4-NEXT:    #APP
-; BDVER4-NEXT:    vphsubbw %xmm0, %xmm0
-; BDVER4-NEXT:    vphsubbw (%rdi), %xmm0
-; BDVER4-NEXT:    #NO_APP
-; BDVER4-NEXT:    retq
-  call void asm sideeffect "vphsubbw $0, $0 \0a\09 vphsubbw $1, $0", "x,*m"(<2 x i64> %a0, <2 x i64> *%a1)
-  ret void
-}
-
-define void @test_vphsubdq(<2 x i64> %a0, <2 x i64> *%a1) {
-; GENERIC-LABEL: test_vphsubdq:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    vphsubdq %xmm0, %xmm0 # sched: [3:1.50]
-; GENERIC-NEXT:    vphsubdq (%rdi), %xmm0 # sched: [9:1.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER12-LABEL: test_vphsubdq:
-; BDVER12:       # %bb.0:
-; BDVER12-NEXT:    #APP
-; BDVER12-NEXT:    vphsubdq %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER12-NEXT:    vphsubdq (%rdi), %xmm0 # sched: [7:0.50]
-; BDVER12-NEXT:    #NO_APP
-; BDVER12-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER3-LABEL: test_vphsubdq:
-; BDVER3:       # %bb.0:
-; BDVER3-NEXT:    #APP
-; BDVER3-NEXT:    vphsubdq %xmm0, %xmm0
-; BDVER3-NEXT:    vphsubdq (%rdi), %xmm0
-; BDVER3-NEXT:    #NO_APP
-; BDVER3-NEXT:    retq
-;
-; BDVER4-LABEL: test_vphsubdq:
-; BDVER4:       # %bb.0:
-; BDVER4-NEXT:    #APP
-; BDVER4-NEXT:    vphsubdq %xmm0, %xmm0
-; BDVER4-NEXT:    vphsubdq (%rdi), %xmm0
-; BDVER4-NEXT:    #NO_APP
-; BDVER4-NEXT:    retq
-  call void asm sideeffect "vphsubdq $0, $0 \0a\09 vphsubdq $1, $0", "x,*m"(<2 x i64> %a0, <2 x i64> *%a1)
-  ret void
-}
-
-define void @test_vphsubwd(<2 x i64> %a0, <2 x i64> *%a1) {
-; GENERIC-LABEL: test_vphsubwd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    vphsubwd %xmm0, %xmm0 # sched: [3:1.50]
-; GENERIC-NEXT:    vphsubwd (%rdi), %xmm0 # sched: [9:1.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER12-LABEL: test_vphsubwd:
-; BDVER12:       # %bb.0:
-; BDVER12-NEXT:    #APP
-; BDVER12-NEXT:    vphsubwd %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER12-NEXT:    vphsubwd (%rdi), %xmm0 # sched: [7:0.50]
-; BDVER12-NEXT:    #NO_APP
-; BDVER12-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER3-LABEL: test_vphsubwd:
-; BDVER3:       # %bb.0:
-; BDVER3-NEXT:    #APP
-; BDVER3-NEXT:    vphsubwd %xmm0, %xmm0
-; BDVER3-NEXT:    vphsubwd (%rdi), %xmm0
-; BDVER3-NEXT:    #NO_APP
-; BDVER3-NEXT:    retq
-;
-; BDVER4-LABEL: test_vphsubwd:
-; BDVER4:       # %bb.0:
-; BDVER4-NEXT:    #APP
-; BDVER4-NEXT:    vphsubwd %xmm0, %xmm0
-; BDVER4-NEXT:    vphsubwd (%rdi), %xmm0
-; BDVER4-NEXT:    #NO_APP
-; BDVER4-NEXT:    retq
-  call void asm sideeffect "vphsubwd $0, $0 \0a\09 vphsubwd $1, $0", "x,*m"(<2 x i64> %a0, <2 x i64> *%a1)
-  ret void
-}
-
-define void @test_vpmacsdd(<2 x i64> %a0, <2 x i64> %a1, <2 x i64> %a2, <2 x i64> *%a3) {
-; GENERIC-LABEL: test_vpmacsdd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    vpmacsdd %xmm2, %xmm1, %xmm0, %xmm0 # sched: [5:1.00]
-; GENERIC-NEXT:    vpmacsdd %xmm2, (%rdi), %xmm0, %xmm0 # sched: [11:1.00]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER12-LABEL: test_vpmacsdd:
-; BDVER12:       # %bb.0:
-; BDVER12-NEXT:    #APP
-; BDVER12-NEXT:    vpmacsdd %xmm2, %xmm1, %xmm0, %xmm0 # sched: [5:2.00]
-; BDVER12-NEXT:    vpmacsdd %xmm2, (%rdi), %xmm0, %xmm0 # sched: [10:2.00]
-; BDVER12-NEXT:    #NO_APP
-; BDVER12-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER3-LABEL: test_vpmacsdd:
-; BDVER3:       # %bb.0:
-; BDVER3-NEXT:    #APP
-; BDVER3-NEXT:    vpmacsdd %xmm2, %xmm1, %xmm0, %xmm0
-; BDVER3-NEXT:    vpmacsdd %xmm2, (%rdi), %xmm0, %xmm0
-; BDVER3-NEXT:    #NO_APP
-; BDVER3-NEXT:    retq
-;
-; BDVER4-LABEL: test_vpmacsdd:
-; BDVER4:       # %bb.0:
-; BDVER4-NEXT:    #APP
-; BDVER4-NEXT:    vpmacsdd %xmm2, %xmm1, %xmm0, %xmm0
-; BDVER4-NEXT:    vpmacsdd %xmm2, (%rdi), %xmm0, %xmm0
-; BDVER4-NEXT:    #NO_APP
-; BDVER4-NEXT:    retq
-  call void asm sideeffect "vpmacsdd $2, $1, $0, $0 \0a\09 vpmacsdd $2, $3, $0, $0", "x,x,x,*m"(<2 x i64> %a0, <2 x i64> %a1, <2 x i64> %a2, <2 x i64> *%a3)
-  ret void
-}
-
-define void @test_vpmacsdqh(<2 x i64> %a0, <2 x i64> %a1, <2 x i64> %a2, <2 x i64> *%a3) {
-; GENERIC-LABEL: test_vpmacsdqh:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    vpmacsdqh %xmm2, %xmm1, %xmm0, %xmm0 # sched: [5:1.00]
-; GENERIC-NEXT:    vpmacsdqh %xmm2, (%rdi), %xmm0, %xmm0 # sched: [11:1.00]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER12-LABEL: test_vpmacsdqh:
-; BDVER12:       # %bb.0:
-; BDVER12-NEXT:    #APP
-; BDVER12-NEXT:    vpmacsdqh %xmm2, %xmm1, %xmm0, %xmm0 # sched: [4:2.00]
-; BDVER12-NEXT:    vpmacsdqh %xmm2, (%rdi), %xmm0, %xmm0 # sched: [10:2.00]
-; BDVER12-NEXT:    #NO_APP
-; BDVER12-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER3-LABEL: test_vpmacsdqh:
-; BDVER3:       # %bb.0:
-; BDVER3-NEXT:    #APP
-; BDVER3-NEXT:    vpmacsdqh %xmm2, %xmm1, %xmm0, %xmm0
-; BDVER3-NEXT:    vpmacsdqh %xmm2, (%rdi), %xmm0, %xmm0
-; BDVER3-NEXT:    #NO_APP
-; BDVER3-NEXT:    retq
-;
-; BDVER4-LABEL: test_vpmacsdqh:
-; BDVER4:       # %bb.0:
-; BDVER4-NEXT:    #APP
-; BDVER4-NEXT:    vpmacsdqh %xmm2, %xmm1, %xmm0, %xmm0
-; BDVER4-NEXT:    vpmacsdqh %xmm2, (%rdi), %xmm0, %xmm0
-; BDVER4-NEXT:    #NO_APP
-; BDVER4-NEXT:    retq
-  call void asm sideeffect "vpmacsdqh $2, $1, $0, $0 \0a\09 vpmacsdqh $2, $3, $0, $0", "x,x,x,*m"(<2 x i64> %a0, <2 x i64> %a1, <2 x i64> %a2, <2 x i64> *%a3)
-  ret void
-}
-
-define void @test_vpmacsdql(<2 x i64> %a0, <2 x i64> %a1, <2 x i64> %a2, <2 x i64> *%a3) {
-; GENERIC-LABEL: test_vpmacsdql:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    vpmacsdql %xmm2, %xmm1, %xmm0, %xmm0 # sched: [5:1.00]
-; GENERIC-NEXT:    vpmacsdql %xmm2, (%rdi), %xmm0, %xmm0 # sched: [11:1.00]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER12-LABEL: test_vpmacsdql:
-; BDVER12:       # %bb.0:
-; BDVER12-NEXT:    #APP
-; BDVER12-NEXT:    vpmacsdql %xmm2, %xmm1, %xmm0, %xmm0 # sched: [4:2.00]
-; BDVER12-NEXT:    vpmacsdql %xmm2, (%rdi), %xmm0, %xmm0 # sched: [10:2.00]
-; BDVER12-NEXT:    #NO_APP
-; BDVER12-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER3-LABEL: test_vpmacsdql:
-; BDVER3:       # %bb.0:
-; BDVER3-NEXT:    #APP
-; BDVER3-NEXT:    vpmacsdql %xmm2, %xmm1, %xmm0, %xmm0
-; BDVER3-NEXT:    vpmacsdql %xmm2, (%rdi), %xmm0, %xmm0
-; BDVER3-NEXT:    #NO_APP
-; BDVER3-NEXT:    retq
-;
-; BDVER4-LABEL: test_vpmacsdql:
-; BDVER4:       # %bb.0:
-; BDVER4-NEXT:    #APP
-; BDVER4-NEXT:    vpmacsdql %xmm2, %xmm1, %xmm0, %xmm0
-; BDVER4-NEXT:    vpmacsdql %xmm2, (%rdi), %xmm0, %xmm0
-; BDVER4-NEXT:    #NO_APP
-; BDVER4-NEXT:    retq
-  call void asm sideeffect "vpmacsdql $2, $1, $0, $0 \0a\09 vpmacsdql $2, $3, $0, $0", "x,x,x,*m"(<2 x i64> %a0, <2 x i64> %a1, <2 x i64> %a2, <2 x i64> *%a3)
-  ret void
-}
-
-define void @test_vpmacssdd(<2 x i64> %a0, <2 x i64> %a1, <2 x i64> %a2, <2 x i64> *%a3) {
-; GENERIC-LABEL: test_vpmacssdd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    vpmacssdd %xmm2, %xmm1, %xmm0, %xmm0 # sched: [5:1.00]
-; GENERIC-NEXT:    vpmacssdd %xmm2, (%rdi), %xmm0, %xmm0 # sched: [11:1.00]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER12-LABEL: test_vpmacssdd:
-; BDVER12:       # %bb.0:
-; BDVER12-NEXT:    #APP
-; BDVER12-NEXT:    vpmacssdd %xmm2, %xmm1, %xmm0, %xmm0 # sched: [5:2.00]
-; BDVER12-NEXT:    vpmacssdd %xmm2, (%rdi), %xmm0, %xmm0 # sched: [10:2.00]
-; BDVER12-NEXT:    #NO_APP
-; BDVER12-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER3-LABEL: test_vpmacssdd:
-; BDVER3:       # %bb.0:
-; BDVER3-NEXT:    #APP
-; BDVER3-NEXT:    vpmacssdd %xmm2, %xmm1, %xmm0, %xmm0
-; BDVER3-NEXT:    vpmacssdd %xmm2, (%rdi), %xmm0, %xmm0
-; BDVER3-NEXT:    #NO_APP
-; BDVER3-NEXT:    retq
-;
-; BDVER4-LABEL: test_vpmacssdd:
-; BDVER4:       # %bb.0:
-; BDVER4-NEXT:    #APP
-; BDVER4-NEXT:    vpmacssdd %xmm2, %xmm1, %xmm0, %xmm0
-; BDVER4-NEXT:    vpmacssdd %xmm2, (%rdi), %xmm0, %xmm0
-; BDVER4-NEXT:    #NO_APP
-; BDVER4-NEXT:    retq
-  call void asm sideeffect "vpmacssdd $2, $1, $0, $0 \0a\09 vpmacssdd $2, $3, $0, $0", "x,x,x,*m"(<2 x i64> %a0, <2 x i64> %a1, <2 x i64> %a2, <2 x i64> *%a3)
-  ret void
-}
-
-define void @test_vpmacssdqh(<2 x i64> %a0, <2 x i64> %a1, <2 x i64> %a2, <2 x i64> *%a3) {
-; GENERIC-LABEL: test_vpmacssdqh:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    vpmacssdqh %xmm2, %xmm1, %xmm0, %xmm0 # sched: [5:1.00]
-; GENERIC-NEXT:    vpmacssdqh %xmm2, (%rdi), %xmm0, %xmm0 # sched: [11:1.00]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER12-LABEL: test_vpmacssdqh:
-; BDVER12:       # %bb.0:
-; BDVER12-NEXT:    #APP
-; BDVER12-NEXT:    vpmacssdqh %xmm2, %xmm1, %xmm0, %xmm0 # sched: [4:2.00]
-; BDVER12-NEXT:    vpmacssdqh %xmm2, (%rdi), %xmm0, %xmm0 # sched: [10:2.00]
-; BDVER12-NEXT:    #NO_APP
-; BDVER12-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER3-LABEL: test_vpmacssdqh:
-; BDVER3:       # %bb.0:
-; BDVER3-NEXT:    #APP
-; BDVER3-NEXT:    vpmacssdqh %xmm2, %xmm1, %xmm0, %xmm0
-; BDVER3-NEXT:    vpmacssdqh %xmm2, (%rdi), %xmm0, %xmm0
-; BDVER3-NEXT:    #NO_APP
-; BDVER3-NEXT:    retq
-;
-; BDVER4-LABEL: test_vpmacssdqh:
-; BDVER4:       # %bb.0:
-; BDVER4-NEXT:    #APP
-; BDVER4-NEXT:    vpmacssdqh %xmm2, %xmm1, %xmm0, %xmm0
-; BDVER4-NEXT:    vpmacssdqh %xmm2, (%rdi), %xmm0, %xmm0
-; BDVER4-NEXT:    #NO_APP
-; BDVER4-NEXT:    retq
-  call void asm sideeffect "vpmacssdqh $2, $1, $0, $0 \0a\09 vpmacssdqh $2, $3, $0, $0", "x,x,x,*m"(<2 x i64> %a0, <2 x i64> %a1, <2 x i64> %a2, <2 x i64> *%a3)
-  ret void
-}
-
-define void @test_vpmacssdql(<2 x i64> %a0, <2 x i64> %a1, <2 x i64> %a2, <2 x i64> *%a3) {
-; GENERIC-LABEL: test_vpmacssdql:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    vpmacssdql %xmm2, %xmm1, %xmm0, %xmm0 # sched: [5:1.00]
-; GENERIC-NEXT:    vpmacssdql %xmm2, (%rdi), %xmm0, %xmm0 # sched: [11:1.00]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER12-LABEL: test_vpmacssdql:
-; BDVER12:       # %bb.0:
-; BDVER12-NEXT:    #APP
-; BDVER12-NEXT:    vpmacssdql %xmm2, %xmm1, %xmm0, %xmm0 # sched: [4:2.00]
-; BDVER12-NEXT:    vpmacssdql %xmm2, (%rdi), %xmm0, %xmm0 # sched: [10:2.00]
-; BDVER12-NEXT:    #NO_APP
-; BDVER12-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER3-LABEL: test_vpmacssdql:
-; BDVER3:       # %bb.0:
-; BDVER3-NEXT:    #APP
-; BDVER3-NEXT:    vpmacssdql %xmm2, %xmm1, %xmm0, %xmm0
-; BDVER3-NEXT:    vpmacssdql %xmm2, (%rdi), %xmm0, %xmm0
-; BDVER3-NEXT:    #NO_APP
-; BDVER3-NEXT:    retq
-;
-; BDVER4-LABEL: test_vpmacssdql:
-; BDVER4:       # %bb.0:
-; BDVER4-NEXT:    #APP
-; BDVER4-NEXT:    vpmacssdql %xmm2, %xmm1, %xmm0, %xmm0
-; BDVER4-NEXT:    vpmacssdql %xmm2, (%rdi), %xmm0, %xmm0
-; BDVER4-NEXT:    #NO_APP
-; BDVER4-NEXT:    retq
-  call void asm sideeffect "vpmacssdql $2, $1, $0, $0 \0a\09 vpmacssdql $2, $3, $0, $0", "x,x,x,*m"(<2 x i64> %a0, <2 x i64> %a1, <2 x i64> %a2, <2 x i64> *%a3)
-  ret void
-}
-
-define void @test_vpmacsswd(<2 x i64> %a0, <2 x i64> %a1, <2 x i64> %a2, <2 x i64> *%a3) {
-; GENERIC-LABEL: test_vpmacsswd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    vpmacsswd %xmm2, %xmm1, %xmm0, %xmm0 # sched: [5:1.00]
-; GENERIC-NEXT:    vpmacsswd %xmm2, (%rdi), %xmm0, %xmm0 # sched: [11:1.00]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER12-LABEL: test_vpmacsswd:
-; BDVER12:       # %bb.0:
-; BDVER12-NEXT:    #APP
-; BDVER12-NEXT:    vpmacsswd %xmm2, %xmm1, %xmm0, %xmm0 # sched: [4:1.00]
-; BDVER12-NEXT:    vpmacsswd %xmm2, (%rdi), %xmm0, %xmm0 # sched: [9:1.00]
-; BDVER12-NEXT:    #NO_APP
-; BDVER12-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER3-LABEL: test_vpmacsswd:
-; BDVER3:       # %bb.0:
-; BDVER3-NEXT:    #APP
-; BDVER3-NEXT:    vpmacsswd %xmm2, %xmm1, %xmm0, %xmm0
-; BDVER3-NEXT:    vpmacsswd %xmm2, (%rdi), %xmm0, %xmm0
-; BDVER3-NEXT:    #NO_APP
-; BDVER3-NEXT:    retq
-;
-; BDVER4-LABEL: test_vpmacsswd:
-; BDVER4:       # %bb.0:
-; BDVER4-NEXT:    #APP
-; BDVER4-NEXT:    vpmacsswd %xmm2, %xmm1, %xmm0, %xmm0
-; BDVER4-NEXT:    vpmacsswd %xmm2, (%rdi), %xmm0, %xmm0
-; BDVER4-NEXT:    #NO_APP
-; BDVER4-NEXT:    retq
-  call void asm sideeffect "vpmacsswd $2, $1, $0, $0 \0a\09 vpmacsswd $2, $3, $0, $0", "x,x,x,*m"(<2 x i64> %a0, <2 x i64> %a1, <2 x i64> %a2, <2 x i64> *%a3)
-  ret void
-}
-
-define void @test_vpmacssww(<2 x i64> %a0, <2 x i64> %a1, <2 x i64> %a2, <2 x i64> *%a3) {
-; GENERIC-LABEL: test_vpmacssww:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    vpmacssww %xmm2, %xmm1, %xmm0, %xmm0 # sched: [5:1.00]
-; GENERIC-NEXT:    vpmacssww %xmm2, (%rdi), %xmm0, %xmm0 # sched: [11:1.00]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER12-LABEL: test_vpmacssww:
-; BDVER12:       # %bb.0:
-; BDVER12-NEXT:    #APP
-; BDVER12-NEXT:    vpmacssww %xmm2, %xmm1, %xmm0, %xmm0 # sched: [4:1.00]
-; BDVER12-NEXT:    vpmacssww %xmm2, (%rdi), %xmm0, %xmm0 # sched: [9:1.00]
-; BDVER12-NEXT:    #NO_APP
-; BDVER12-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER3-LABEL: test_vpmacssww:
-; BDVER3:       # %bb.0:
-; BDVER3-NEXT:    #APP
-; BDVER3-NEXT:    vpmacssww %xmm2, %xmm1, %xmm0, %xmm0
-; BDVER3-NEXT:    vpmacssww %xmm2, (%rdi), %xmm0, %xmm0
-; BDVER3-NEXT:    #NO_APP
-; BDVER3-NEXT:    retq
-;
-; BDVER4-LABEL: test_vpmacssww:
-; BDVER4:       # %bb.0:
-; BDVER4-NEXT:    #APP
-; BDVER4-NEXT:    vpmacssww %xmm2, %xmm1, %xmm0, %xmm0
-; BDVER4-NEXT:    vpmacssww %xmm2, (%rdi), %xmm0, %xmm0
-; BDVER4-NEXT:    #NO_APP
-; BDVER4-NEXT:    retq
-  call void asm sideeffect "vpmacssww $2, $1, $0, $0 \0a\09 vpmacssww $2, $3, $0, $0", "x,x,x,*m"(<2 x i64> %a0, <2 x i64> %a1, <2 x i64> %a2, <2 x i64> *%a3)
-  ret void
-}
-
-define void @test_vpmacswd(<2 x i64> %a0, <2 x i64> %a1, <2 x i64> %a2, <2 x i64> *%a3) {
-; GENERIC-LABEL: test_vpmacswd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    vpmacswd %xmm2, %xmm1, %xmm0, %xmm0 # sched: [5:1.00]
-; GENERIC-NEXT:    vpmacswd %xmm2, (%rdi), %xmm0, %xmm0 # sched: [11:1.00]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER12-LABEL: test_vpmacswd:
-; BDVER12:       # %bb.0:
-; BDVER12-NEXT:    #APP
-; BDVER12-NEXT:    vpmacswd %xmm2, %xmm1, %xmm0, %xmm0 # sched: [4:1.00]
-; BDVER12-NEXT:    vpmacswd %xmm2, (%rdi), %xmm0, %xmm0 # sched: [9:1.00]
-; BDVER12-NEXT:    #NO_APP
-; BDVER12-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER3-LABEL: test_vpmacswd:
-; BDVER3:       # %bb.0:
-; BDVER3-NEXT:    #APP
-; BDVER3-NEXT:    vpmacswd %xmm2, %xmm1, %xmm0, %xmm0
-; BDVER3-NEXT:    vpmacswd %xmm2, (%rdi), %xmm0, %xmm0
-; BDVER3-NEXT:    #NO_APP
-; BDVER3-NEXT:    retq
-;
-; BDVER4-LABEL: test_vpmacswd:
-; BDVER4:       # %bb.0:
-; BDVER4-NEXT:    #APP
-; BDVER4-NEXT:    vpmacswd %xmm2, %xmm1, %xmm0, %xmm0
-; BDVER4-NEXT:    vpmacswd %xmm2, (%rdi), %xmm0, %xmm0
-; BDVER4-NEXT:    #NO_APP
-; BDVER4-NEXT:    retq
-  call void asm sideeffect "vpmacswd $2, $1, $0, $0 \0a\09 vpmacswd $2, $3, $0, $0", "x,x,x,*m"(<2 x i64> %a0, <2 x i64> %a1, <2 x i64> %a2, <2 x i64> *%a3)
-  ret void
-}
-
-define void @test_vpmacsww(<2 x i64> %a0, <2 x i64> %a1, <2 x i64> %a2, <2 x i64> *%a3) {
-; GENERIC-LABEL: test_vpmacsww:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    vpmacsww %xmm2, %xmm1, %xmm0, %xmm0 # sched: [5:1.00]
-; GENERIC-NEXT:    vpmacsww %xmm2, (%rdi), %xmm0, %xmm0 # sched: [11:1.00]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER12-LABEL: test_vpmacsww:
-; BDVER12:       # %bb.0:
-; BDVER12-NEXT:    #APP
-; BDVER12-NEXT:    vpmacsww %xmm2, %xmm1, %xmm0, %xmm0 # sched: [4:1.00]
-; BDVER12-NEXT:    vpmacsww %xmm2, (%rdi), %xmm0, %xmm0 # sched: [9:1.00]
-; BDVER12-NEXT:    #NO_APP
-; BDVER12-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER3-LABEL: test_vpmacsww:
-; BDVER3:       # %bb.0:
-; BDVER3-NEXT:    #APP
-; BDVER3-NEXT:    vpmacsww %xmm2, %xmm1, %xmm0, %xmm0
-; BDVER3-NEXT:    vpmacsww %xmm2, (%rdi), %xmm0, %xmm0
-; BDVER3-NEXT:    #NO_APP
-; BDVER3-NEXT:    retq
-;
-; BDVER4-LABEL: test_vpmacsww:
-; BDVER4:       # %bb.0:
-; BDVER4-NEXT:    #APP
-; BDVER4-NEXT:    vpmacsww %xmm2, %xmm1, %xmm0, %xmm0
-; BDVER4-NEXT:    vpmacsww %xmm2, (%rdi), %xmm0, %xmm0
-; BDVER4-NEXT:    #NO_APP
-; BDVER4-NEXT:    retq
-  call void asm sideeffect "vpmacsww $2, $1, $0, $0 \0a\09 vpmacsww $2, $3, $0, $0", "x,x,x,*m"(<2 x i64> %a0, <2 x i64> %a1, <2 x i64> %a2, <2 x i64> *%a3)
-  ret void
-}
-
-define void @test_vpmadcsswd(<2 x i64> %a0, <2 x i64> %a1, <2 x i64> %a2, <2 x i64> *%a3) {
-; GENERIC-LABEL: test_vpmadcsswd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    vpmadcsswd %xmm2, %xmm1, %xmm0, %xmm0 # sched: [5:1.00]
-; GENERIC-NEXT:    vpmadcsswd %xmm2, (%rdi), %xmm0, %xmm0 # sched: [11:1.00]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER12-LABEL: test_vpmadcsswd:
-; BDVER12:       # %bb.0:
-; BDVER12-NEXT:    #APP
-; BDVER12-NEXT:    vpmadcsswd %xmm2, %xmm1, %xmm0, %xmm0 # sched: [4:1.00]
-; BDVER12-NEXT:    vpmadcsswd %xmm2, (%rdi), %xmm0, %xmm0 # sched: [9:1.00]
-; BDVER12-NEXT:    #NO_APP
-; BDVER12-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER3-LABEL: test_vpmadcsswd:
-; BDVER3:       # %bb.0:
-; BDVER3-NEXT:    #APP
-; BDVER3-NEXT:    vpmadcsswd %xmm2, %xmm1, %xmm0, %xmm0
-; BDVER3-NEXT:    vpmadcsswd %xmm2, (%rdi), %xmm0, %xmm0
-; BDVER3-NEXT:    #NO_APP
-; BDVER3-NEXT:    retq
-;
-; BDVER4-LABEL: test_vpmadcsswd:
-; BDVER4:       # %bb.0:
-; BDVER4-NEXT:    #APP
-; BDVER4-NEXT:    vpmadcsswd %xmm2, %xmm1, %xmm0, %xmm0
-; BDVER4-NEXT:    vpmadcsswd %xmm2, (%rdi), %xmm0, %xmm0
-; BDVER4-NEXT:    #NO_APP
-; BDVER4-NEXT:    retq
-  call void asm sideeffect "vpmadcsswd $2, $1, $0, $0 \0a\09 vpmadcsswd $2, $3, $0, $0", "x,x,x,*m"(<2 x i64> %a0, <2 x i64> %a1, <2 x i64> %a2, <2 x i64> *%a3)
-  ret void
-}
-
-define void @test_vpmadcswd(<2 x i64> %a0, <2 x i64> %a1, <2 x i64> %a2, <2 x i64> *%a3) {
-; GENERIC-LABEL: test_vpmadcswd:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    vpmadcswd %xmm2, %xmm1, %xmm0, %xmm0 # sched: [5:1.00]
-; GENERIC-NEXT:    vpmadcswd %xmm2, (%rdi), %xmm0, %xmm0 # sched: [11:1.00]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER12-LABEL: test_vpmadcswd:
-; BDVER12:       # %bb.0:
-; BDVER12-NEXT:    #APP
-; BDVER12-NEXT:    vpmadcswd %xmm2, %xmm1, %xmm0, %xmm0 # sched: [4:1.00]
-; BDVER12-NEXT:    vpmadcswd %xmm2, (%rdi), %xmm0, %xmm0 # sched: [9:1.00]
-; BDVER12-NEXT:    #NO_APP
-; BDVER12-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER3-LABEL: test_vpmadcswd:
-; BDVER3:       # %bb.0:
-; BDVER3-NEXT:    #APP
-; BDVER3-NEXT:    vpmadcswd %xmm2, %xmm1, %xmm0, %xmm0
-; BDVER3-NEXT:    vpmadcswd %xmm2, (%rdi), %xmm0, %xmm0
-; BDVER3-NEXT:    #NO_APP
-; BDVER3-NEXT:    retq
-;
-; BDVER4-LABEL: test_vpmadcswd:
-; BDVER4:       # %bb.0:
-; BDVER4-NEXT:    #APP
-; BDVER4-NEXT:    vpmadcswd %xmm2, %xmm1, %xmm0, %xmm0
-; BDVER4-NEXT:    vpmadcswd %xmm2, (%rdi), %xmm0, %xmm0
-; BDVER4-NEXT:    #NO_APP
-; BDVER4-NEXT:    retq
-  call void asm sideeffect "vpmadcswd $2, $1, $0, $0 \0a\09 vpmadcswd $2, $3, $0, $0", "x,x,x,*m"(<2 x i64> %a0, <2 x i64> %a1, <2 x i64> %a2, <2 x i64> *%a3)
-  ret void
-}
-
-define void @test_vpperm(<2 x i64> %a0, <2 x i64> %a1, <2 x i64> %a2, <2 x i64> *%a3) {
-; GENERIC-LABEL: test_vpperm:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    vpperm %xmm2, %xmm1, %xmm0, %xmm0 # sched: [1:0.50]
-; GENERIC-NEXT:    vpperm (%rdi), %xmm1, %xmm0, %xmm0 # sched: [7:0.50]
-; GENERIC-NEXT:    vpperm %xmm2, (%rdi), %xmm0, %xmm0 # sched: [7:0.50]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER12-LABEL: test_vpperm:
-; BDVER12:       # %bb.0:
-; BDVER12-NEXT:    #APP
-; BDVER12-NEXT:    vpperm %xmm2, %xmm1, %xmm0, %xmm0 # sched: [3:2.00]
-; BDVER12-NEXT:    vpperm (%rdi), %xmm1, %xmm0, %xmm0 # sched: [8:2.00]
-; BDVER12-NEXT:    vpperm %xmm2, (%rdi), %xmm0, %xmm0 # sched: [8:2.00]
-; BDVER12-NEXT:    #NO_APP
-; BDVER12-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER3-LABEL: test_vpperm:
-; BDVER3:       # %bb.0:
-; BDVER3-NEXT:    #APP
-; BDVER3-NEXT:    vpperm %xmm2, %xmm1, %xmm0, %xmm0
-; BDVER3-NEXT:    vpperm (%rdi), %xmm1, %xmm0, %xmm0
-; BDVER3-NEXT:    vpperm %xmm2, (%rdi), %xmm0, %xmm0
-; BDVER3-NEXT:    #NO_APP
-; BDVER3-NEXT:    retq
-;
-; BDVER4-LABEL: test_vpperm:
-; BDVER4:       # %bb.0:
-; BDVER4-NEXT:    #APP
-; BDVER4-NEXT:    vpperm %xmm2, %xmm1, %xmm0, %xmm0
-; BDVER4-NEXT:    vpperm (%rdi), %xmm1, %xmm0, %xmm0
-; BDVER4-NEXT:    vpperm %xmm2, (%rdi), %xmm0, %xmm0
-; BDVER4-NEXT:    #NO_APP
-; BDVER4-NEXT:    retq
-  call void asm sideeffect "vpperm $2, $1, $0, $0 \0A\09 vpperm $3, $1, $0, $0 \0A\09 vpperm $2, $3, $0, $0", "x,x,x,*m"(<2 x i64> %a0, <2 x i64> %a1, <2 x i64> %a2, <2 x i64> *%a3)
-  ret void
-}
-
-define void @test_vprot(<2 x i64> %a0, <2 x i64> %a1, <2 x i64> *%a2) {
-; GENERIC-LABEL: test_vprot:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    vprotb %xmm1, %xmm0, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vprotd %xmm1, %xmm0, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vprotq %xmm1, %xmm0, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vprotw %xmm1, %xmm0, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vprotb (%rdi), %xmm0, %xmm0 # sched: [7:1.00]
-; GENERIC-NEXT:    vprotd (%rdi), %xmm0, %xmm0 # sched: [7:1.00]
-; GENERIC-NEXT:    vprotq (%rdi), %xmm0, %xmm0 # sched: [7:1.00]
-; GENERIC-NEXT:    vprotw (%rdi), %xmm0, %xmm0 # sched: [7:1.00]
-; GENERIC-NEXT:    vprotb %xmm0, (%rdi), %xmm0 # sched: [7:1.00]
-; GENERIC-NEXT:    vprotd %xmm0, (%rdi), %xmm0 # sched: [7:1.00]
-; GENERIC-NEXT:    vprotq %xmm0, (%rdi), %xmm0 # sched: [7:1.00]
-; GENERIC-NEXT:    vprotw %xmm0, (%rdi), %xmm0 # sched: [7:1.00]
-; GENERIC-NEXT:    vprotb $7, %xmm0, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vprotd $7, %xmm0, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vprotq $7, %xmm0, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vprotw $7, %xmm0, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vprotb $7, (%rdi), %xmm0 # sched: [7:1.00]
-; GENERIC-NEXT:    vprotd $7, (%rdi), %xmm0 # sched: [7:1.00]
-; GENERIC-NEXT:    vprotq $7, (%rdi), %xmm0 # sched: [7:1.00]
-; GENERIC-NEXT:    vprotw $7, (%rdi), %xmm0 # sched: [7:1.00]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER12-LABEL: test_vprot:
-; BDVER12:       # %bb.0:
-; BDVER12-NEXT:    #APP
-; BDVER12-NEXT:    vprotb %xmm1, %xmm0, %xmm0 # sched: [3:0.50]
-; BDVER12-NEXT:    vprotd %xmm1, %xmm0, %xmm0 # sched: [3:0.50]
-; BDVER12-NEXT:    vprotq %xmm1, %xmm0, %xmm0 # sched: [3:0.50]
-; BDVER12-NEXT:    vprotw %xmm1, %xmm0, %xmm0 # sched: [3:0.50]
-; BDVER12-NEXT:    vprotb (%rdi), %xmm0, %xmm0 # sched: [8:0.50]
-; BDVER12-NEXT:    vprotd (%rdi), %xmm0, %xmm0 # sched: [8:0.50]
-; BDVER12-NEXT:    vprotq (%rdi), %xmm0, %xmm0 # sched: [8:0.50]
-; BDVER12-NEXT:    vprotw (%rdi), %xmm0, %xmm0 # sched: [8:0.50]
-; BDVER12-NEXT:    vprotb %xmm0, (%rdi), %xmm0 # sched: [8:0.50]
-; BDVER12-NEXT:    vprotd %xmm0, (%rdi), %xmm0 # sched: [8:0.50]
-; BDVER12-NEXT:    vprotq %xmm0, (%rdi), %xmm0 # sched: [8:0.50]
-; BDVER12-NEXT:    vprotw %xmm0, (%rdi), %xmm0 # sched: [8:0.50]
-; BDVER12-NEXT:    vprotb $7, %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER12-NEXT:    vprotd $7, %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER12-NEXT:    vprotq $7, %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER12-NEXT:    vprotw $7, %xmm0, %xmm0 # sched: [2:0.50]
-; BDVER12-NEXT:    vprotb $7, (%rdi), %xmm0 # sched: [7:0.50]
-; BDVER12-NEXT:    vprotd $7, (%rdi), %xmm0 # sched: [7:0.50]
-; BDVER12-NEXT:    vprotq $7, (%rdi), %xmm0 # sched: [7:0.50]
-; BDVER12-NEXT:    vprotw $7, (%rdi), %xmm0 # sched: [7:0.50]
-; BDVER12-NEXT:    #NO_APP
-; BDVER12-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER3-LABEL: test_vprot:
-; BDVER3:       # %bb.0:
-; BDVER3-NEXT:    #APP
-; BDVER3-NEXT:    vprotb %xmm1, %xmm0, %xmm0
-; BDVER3-NEXT:    vprotd %xmm1, %xmm0, %xmm0
-; BDVER3-NEXT:    vprotq %xmm1, %xmm0, %xmm0
-; BDVER3-NEXT:    vprotw %xmm1, %xmm0, %xmm0
-; BDVER3-NEXT:    vprotb (%rdi), %xmm0, %xmm0
-; BDVER3-NEXT:    vprotd (%rdi), %xmm0, %xmm0
-; BDVER3-NEXT:    vprotq (%rdi), %xmm0, %xmm0
-; BDVER3-NEXT:    vprotw (%rdi), %xmm0, %xmm0
-; BDVER3-NEXT:    vprotb %xmm0, (%rdi), %xmm0
-; BDVER3-NEXT:    vprotd %xmm0, (%rdi), %xmm0
-; BDVER3-NEXT:    vprotq %xmm0, (%rdi), %xmm0
-; BDVER3-NEXT:    vprotw %xmm0, (%rdi), %xmm0
-; BDVER3-NEXT:    vprotb $7, %xmm0, %xmm0
-; BDVER3-NEXT:    vprotd $7, %xmm0, %xmm0
-; BDVER3-NEXT:    vprotq $7, %xmm0, %xmm0
-; BDVER3-NEXT:    vprotw $7, %xmm0, %xmm0
-; BDVER3-NEXT:    vprotb $7, (%rdi), %xmm0
-; BDVER3-NEXT:    vprotd $7, (%rdi), %xmm0
-; BDVER3-NEXT:    vprotq $7, (%rdi), %xmm0
-; BDVER3-NEXT:    vprotw $7, (%rdi), %xmm0
-; BDVER3-NEXT:    #NO_APP
-; BDVER3-NEXT:    retq
-;
-; BDVER4-LABEL: test_vprot:
-; BDVER4:       # %bb.0:
-; BDVER4-NEXT:    #APP
-; BDVER4-NEXT:    vprotb %xmm1, %xmm0, %xmm0
-; BDVER4-NEXT:    vprotd %xmm1, %xmm0, %xmm0
-; BDVER4-NEXT:    vprotq %xmm1, %xmm0, %xmm0
-; BDVER4-NEXT:    vprotw %xmm1, %xmm0, %xmm0
-; BDVER4-NEXT:    vprotb (%rdi), %xmm0, %xmm0
-; BDVER4-NEXT:    vprotd (%rdi), %xmm0, %xmm0
-; BDVER4-NEXT:    vprotq (%rdi), %xmm0, %xmm0
-; BDVER4-NEXT:    vprotw (%rdi), %xmm0, %xmm0
-; BDVER4-NEXT:    vprotb %xmm0, (%rdi), %xmm0
-; BDVER4-NEXT:    vprotd %xmm0, (%rdi), %xmm0
-; BDVER4-NEXT:    vprotq %xmm0, (%rdi), %xmm0
-; BDVER4-NEXT:    vprotw %xmm0, (%rdi), %xmm0
-; BDVER4-NEXT:    vprotb $7, %xmm0, %xmm0
-; BDVER4-NEXT:    vprotd $7, %xmm0, %xmm0
-; BDVER4-NEXT:    vprotq $7, %xmm0, %xmm0
-; BDVER4-NEXT:    vprotw $7, %xmm0, %xmm0
-; BDVER4-NEXT:    vprotb $7, (%rdi), %xmm0
-; BDVER4-NEXT:    vprotd $7, (%rdi), %xmm0
-; BDVER4-NEXT:    vprotq $7, (%rdi), %xmm0
-; BDVER4-NEXT:    vprotw $7, (%rdi), %xmm0
-; BDVER4-NEXT:    #NO_APP
-; BDVER4-NEXT:    retq
-  call void asm sideeffect "vprotb $1, $0, $0 \0A\09 vprotd $1, $0, $0 \0A\09 vprotq $1, $0, $0 \0A\09 vprotw $1, $0, $0 \0A\09 vprotb $2, $0, $0 \0A\09 vprotd $2, $0, $0 \0A\09 vprotq $2, $0, $0 \0A\09 vprotw $2, $0, $0 \0A\09 vprotb $0, $2, $0 \0A\09 vprotd $0, $2, $0 \0A\09 vprotq $0, $2, $0 \0A\09 vprotw $0, $2, $0 \0A\09 vprotb $3, $0, $0 \0A\09 vprotd $3, $0, $0 \0A\09 vprotq $3, $0, $0 \0A\09 vprotw $3, $0, $0 \0A\09 vprotb $3, $2, $0 \0A\09 vprotd $3, $2, $0 \0A\09 vprotq $3, $2, $0 \0A\09 vprotw $3, $2, $0", "x,x,*m,i"(<2 x i64> %a0, <2 x i64> %a1, <2 x i64> *%a2, i8 7)
-  ret void
-}
-
-define void @test_vpsha(<2 x i64> %a0, <2 x i64> %a1, <2 x i64> *%a2) {
-; GENERIC-LABEL: test_vpsha:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    vpshab %xmm1, %xmm0, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vpshad %xmm1, %xmm0, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vpshaq %xmm1, %xmm0, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vpshaw %xmm1, %xmm0, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vpshab (%rdi), %xmm0, %xmm0 # sched: [7:1.00]
-; GENERIC-NEXT:    vpshad (%rdi), %xmm0, %xmm0 # sched: [7:1.00]
-; GENERIC-NEXT:    vpshaq (%rdi), %xmm0, %xmm0 # sched: [7:1.00]
-; GENERIC-NEXT:    vpshaw (%rdi), %xmm0, %xmm0 # sched: [7:1.00]
-; GENERIC-NEXT:    vpshab %xmm0, (%rdi), %xmm0 # sched: [7:1.00]
-; GENERIC-NEXT:    vpshad %xmm0, (%rdi), %xmm0 # sched: [7:1.00]
-; GENERIC-NEXT:    vpshaq %xmm0, (%rdi), %xmm0 # sched: [7:1.00]
-; GENERIC-NEXT:    vpshaw %xmm0, (%rdi), %xmm0 # sched: [7:1.00]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER12-LABEL: test_vpsha:
-; BDVER12:       # %bb.0:
-; BDVER12-NEXT:    #APP
-; BDVER12-NEXT:    vpshab %xmm1, %xmm0, %xmm0 # sched: [3:0.50]
-; BDVER12-NEXT:    vpshad %xmm1, %xmm0, %xmm0 # sched: [3:0.50]
-; BDVER12-NEXT:    vpshaq %xmm1, %xmm0, %xmm0 # sched: [3:0.50]
-; BDVER12-NEXT:    vpshaw %xmm1, %xmm0, %xmm0 # sched: [3:0.50]
-; BDVER12-NEXT:    vpshab (%rdi), %xmm0, %xmm0 # sched: [8:0.50]
-; BDVER12-NEXT:    vpshad (%rdi), %xmm0, %xmm0 # sched: [8:0.50]
-; BDVER12-NEXT:    vpshaq (%rdi), %xmm0, %xmm0 # sched: [8:0.50]
-; BDVER12-NEXT:    vpshaw (%rdi), %xmm0, %xmm0 # sched: [8:0.50]
-; BDVER12-NEXT:    vpshab %xmm0, (%rdi), %xmm0 # sched: [8:0.50]
-; BDVER12-NEXT:    vpshad %xmm0, (%rdi), %xmm0 # sched: [8:0.50]
-; BDVER12-NEXT:    vpshaq %xmm0, (%rdi), %xmm0 # sched: [8:0.50]
-; BDVER12-NEXT:    vpshaw %xmm0, (%rdi), %xmm0 # sched: [8:0.50]
-; BDVER12-NEXT:    #NO_APP
-; BDVER12-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER3-LABEL: test_vpsha:
-; BDVER3:       # %bb.0:
-; BDVER3-NEXT:    #APP
-; BDVER3-NEXT:    vpshab %xmm1, %xmm0, %xmm0
-; BDVER3-NEXT:    vpshad %xmm1, %xmm0, %xmm0
-; BDVER3-NEXT:    vpshaq %xmm1, %xmm0, %xmm0
-; BDVER3-NEXT:    vpshaw %xmm1, %xmm0, %xmm0
-; BDVER3-NEXT:    vpshab (%rdi), %xmm0, %xmm0
-; BDVER3-NEXT:    vpshad (%rdi), %xmm0, %xmm0
-; BDVER3-NEXT:    vpshaq (%rdi), %xmm0, %xmm0
-; BDVER3-NEXT:    vpshaw (%rdi), %xmm0, %xmm0
-; BDVER3-NEXT:    vpshab %xmm0, (%rdi), %xmm0
-; BDVER3-NEXT:    vpshad %xmm0, (%rdi), %xmm0
-; BDVER3-NEXT:    vpshaq %xmm0, (%rdi), %xmm0
-; BDVER3-NEXT:    vpshaw %xmm0, (%rdi), %xmm0
-; BDVER3-NEXT:    #NO_APP
-; BDVER3-NEXT:    retq
-;
-; BDVER4-LABEL: test_vpsha:
-; BDVER4:       # %bb.0:
-; BDVER4-NEXT:    #APP
-; BDVER4-NEXT:    vpshab %xmm1, %xmm0, %xmm0
-; BDVER4-NEXT:    vpshad %xmm1, %xmm0, %xmm0
-; BDVER4-NEXT:    vpshaq %xmm1, %xmm0, %xmm0
-; BDVER4-NEXT:    vpshaw %xmm1, %xmm0, %xmm0
-; BDVER4-NEXT:    vpshab (%rdi), %xmm0, %xmm0
-; BDVER4-NEXT:    vpshad (%rdi), %xmm0, %xmm0
-; BDVER4-NEXT:    vpshaq (%rdi), %xmm0, %xmm0
-; BDVER4-NEXT:    vpshaw (%rdi), %xmm0, %xmm0
-; BDVER4-NEXT:    vpshab %xmm0, (%rdi), %xmm0
-; BDVER4-NEXT:    vpshad %xmm0, (%rdi), %xmm0
-; BDVER4-NEXT:    vpshaq %xmm0, (%rdi), %xmm0
-; BDVER4-NEXT:    vpshaw %xmm0, (%rdi), %xmm0
-; BDVER4-NEXT:    #NO_APP
-; BDVER4-NEXT:    retq
-  call void asm sideeffect "vpshab $1, $0, $0 \0A\09 vpshad $1, $0, $0 \0A\09 vpshaq $1, $0, $0 \0A\09 vpshaw $1, $0, $0 \0A\09 vpshab $2, $0, $0 \0A\09 vpshad $2, $0, $0 \0A\09 vpshaq $2, $0, $0 \0A\09 vpshaw $2, $0, $0 \0A\09 vpshab $0, $2, $0 \0A\09 vpshad $0, $2, $0 \0A\09 vpshaq $0, $2, $0 \0A\09 vpshaw $0, $2, $0", "x,x,*m"(<2 x i64> %a0, <2 x i64> %a1, <2 x i64> *%a2)
-  ret void
-}
-
-define void @test_vpshl(<2 x i64> %a0, <2 x i64> %a1, <2 x i64> *%a2) {
-; GENERIC-LABEL: test_vpshl:
-; GENERIC:       # %bb.0:
-; GENERIC-NEXT:    #APP
-; GENERIC-NEXT:    vpshlb %xmm1, %xmm0, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vpshld %xmm1, %xmm0, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vpshlq %xmm1, %xmm0, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vpshlw %xmm1, %xmm0, %xmm0 # sched: [1:1.00]
-; GENERIC-NEXT:    vpshlb (%rdi), %xmm0, %xmm0 # sched: [7:1.00]
-; GENERIC-NEXT:    vpshld (%rdi), %xmm0, %xmm0 # sched: [7:1.00]
-; GENERIC-NEXT:    vpshlq (%rdi), %xmm0, %xmm0 # sched: [7:1.00]
-; GENERIC-NEXT:    vpshlw (%rdi), %xmm0, %xmm0 # sched: [7:1.00]
-; GENERIC-NEXT:    vpshlb %xmm0, (%rdi), %xmm0 # sched: [7:1.00]
-; GENERIC-NEXT:    vpshld %xmm0, (%rdi), %xmm0 # sched: [7:1.00]
-; GENERIC-NEXT:    vpshlq %xmm0, (%rdi), %xmm0 # sched: [7:1.00]
-; GENERIC-NEXT:    vpshlw %xmm0, (%rdi), %xmm0 # sched: [7:1.00]
-; GENERIC-NEXT:    #NO_APP
-; GENERIC-NEXT:    retq # sched: [1:1.00]
-;
-; BDVER12-LABEL: test_vpshl:
-; BDVER12:       # %bb.0:
-; BDVER12-NEXT:    #APP
-; BDVER12-NEXT:    vpshlb %xmm1, %xmm0, %xmm0 # sched: [3:0.50]
-; BDVER12-NEXT:    vpshld %xmm1, %xmm0, %xmm0 # sched: [3:0.50]
-; BDVER12-NEXT:    vpshlq %xmm1, %xmm0, %xmm0 # sched: [3:0.50]
-; BDVER12-NEXT:    vpshlw %xmm1, %xmm0, %xmm0 # sched: [3:0.50]
-; BDVER12-NEXT:    vpshlb (%rdi), %xmm0, %xmm0 # sched: [8:0.50]
-; BDVER12-NEXT:    vpshld (%rdi), %xmm0, %xmm0 # sched: [8:0.50]
-; BDVER12-NEXT:    vpshlq (%rdi), %xmm0, %xmm0 # sched: [8:0.50]
-; BDVER12-NEXT:    vpshlw (%rdi), %xmm0, %xmm0 # sched: [8:0.50]
-; BDVER12-NEXT:    vpshlb %xmm0, (%rdi), %xmm0 # sched: [8:0.50]
-; BDVER12-NEXT:    vpshld %xmm0, (%rdi), %xmm0 # sched: [8:0.50]
-; BDVER12-NEXT:    vpshlq %xmm0, (%rdi), %xmm0 # sched: [8:0.50]
-; BDVER12-NEXT:    vpshlw %xmm0, (%rdi), %xmm0 # sched: [8:0.50]
-; BDVER12-NEXT:    #NO_APP
-; BDVER12-NEXT:    retq # sched: [5:1.00]
-;
-; BDVER3-LABEL: test_vpshl:
-; BDVER3:       # %bb.0:
-; BDVER3-NEXT:    #APP
-; BDVER3-NEXT:    vpshlb %xmm1, %xmm0, %xmm0
-; BDVER3-NEXT:    vpshld %xmm1, %xmm0, %xmm0
-; BDVER3-NEXT:    vpshlq %xmm1, %xmm0, %xmm0
-; BDVER3-NEXT:    vpshlw %xmm1, %xmm0, %xmm0
-; BDVER3-NEXT:    vpshlb (%rdi), %xmm0, %xmm0
-; BDVER3-NEXT:    vpshld (%rdi), %xmm0, %xmm0
-; BDVER3-NEXT:    vpshlq (%rdi), %xmm0, %xmm0
-; BDVER3-NEXT:    vpshlw (%rdi), %xmm0, %xmm0
-; BDVER3-NEXT:    vpshlb %xmm0, (%rdi), %xmm0
-; BDVER3-NEXT:    vpshld %xmm0, (%rdi), %xmm0
-; BDVER3-NEXT:    vpshlq %xmm0, (%rdi), %xmm0
-; BDVER3-NEXT:    vpshlw %xmm0, (%rdi), %xmm0
-; BDVER3-NEXT:    #NO_APP
-; BDVER3-NEXT:    retq
-;
-; BDVER4-LABEL: test_vpshl:
-; BDVER4:       # %bb.0:
-; BDVER4-NEXT:    #APP
-; BDVER4-NEXT:    vpshlb %xmm1, %xmm0, %xmm0
-; BDVER4-NEXT:    vpshld %xmm1, %xmm0, %xmm0
-; BDVER4-NEXT:    vpshlq %xmm1, %xmm0, %xmm0
-; BDVER4-NEXT:    vpshlw %xmm1, %xmm0, %xmm0
-; BDVER4-NEXT:    vpshlb (%rdi), %xmm0, %xmm0
-; BDVER4-NEXT:    vpshld (%rdi), %xmm0, %xmm0
-; BDVER4-NEXT:    vpshlq (%rdi), %xmm0, %xmm0
-; BDVER4-NEXT:    vpshlw (%rdi), %xmm0, %xmm0
-; BDVER4-NEXT:    vpshlb %xmm0, (%rdi), %xmm0
-; BDVER4-NEXT:    vpshld %xmm0, (%rdi), %xmm0
-; BDVER4-NEXT:    vpshlq %xmm0, (%rdi), %xmm0
-; BDVER4-NEXT:    vpshlw %xmm0, (%rdi), %xmm0
-; BDVER4-NEXT:    #NO_APP
-; BDVER4-NEXT:    retq
-  call void asm sideeffect "vpshlb $1, $0, $0 \0A\09 vpshld $1, $0, $0 \0A\09 vpshlq $1, $0, $0 \0A\09 vpshlw $1, $0, $0 \0A\09 vpshlb $2, $0, $0 \0A\09 vpshld $2, $0, $0 \0A\09 vpshlq $2, $0, $0 \0A\09 vpshlw $2, $0, $0 \0A\09 vpshlb $0, $2, $0 \0A\09 vpshld $0, $2, $0 \0A\09 vpshlq $0, $2, $0 \0A\09 vpshlw $0, $2, $0", "x,x,*m"(<2 x i64> %a0, <2 x i64> %a1, <2 x i64> *%a2)
-  ret void
-}
diff --git a/test/CodeGen/X86/xor.ll b/test/CodeGen/X86/xor.ll
index 65e5e83d7275dcf395d9ad2adb3f231c4b400972..f86b5812ace88917ec63292f16f8bc1b9ee94902 100644
--- a/test/CodeGen/X86/xor.ll
+++ b/test/CodeGen/X86/xor.ll
@@ -420,26 +420,14 @@ define i32 @PR17487(i1 %tobool) {
 ; X64-LIN-LABEL: PR17487:
 ; X64-LIN:       # %bb.0:
 ; X64-LIN-NEXT:    movd %edi, %xmm0
-; X64-LIN-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,1,0,1]
-; X64-LIN-NEXT:    pandn {{.*}}(%rip), %xmm0
-; X64-LIN-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; X64-LIN-NEXT:    movq %xmm0, %rcx
-; X64-LIN-NEXT:    xorl %eax, %eax
-; X64-LIN-NEXT:    cmpq $1, %rcx
-; X64-LIN-NEXT:    setne %al
+; X64-LIN-NEXT:    pextrw $0, %xmm0, %eax
+; X64-LIN-NEXT:    andl $1, %eax
 ; X64-LIN-NEXT:    retq
 ;
 ; X64-WIN-LABEL: PR17487:
 ; X64-WIN:       # %bb.0:
+; X64-WIN-NEXT:    andb $1, %cl
 ; X64-WIN-NEXT:    movzbl %cl, %eax
-; X64-WIN-NEXT:    movd %eax, %xmm0
-; X64-WIN-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,1,0,1]
-; X64-WIN-NEXT:    pandn __xmm@{{.*}}(%rip), %xmm0
-; X64-WIN-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; X64-WIN-NEXT:    movq %xmm0, %rcx
-; X64-WIN-NEXT:    xorl %eax, %eax
-; X64-WIN-NEXT:    cmpq $1, %rcx
-; X64-WIN-NEXT:    setne %al
 ; X64-WIN-NEXT:    retq
   %tmp = insertelement <2 x i1> undef, i1 %tobool, i32 1
   %tmp1 = zext <2 x i1> %tmp to <2 x i64>
diff --git a/test/CodeGen/X86/zext-logicop-shift-load.ll b/test/CodeGen/X86/zext-logicop-shift-load.ll
index 319c177b30a6c9bd3abbb8b8f412831717e6d049..40d72243dea59921120b7fef6c08e059e89326b1 100644
--- a/test/CodeGen/X86/zext-logicop-shift-load.ll
+++ b/test/CodeGen/X86/zext-logicop-shift-load.ll
@@ -103,7 +103,6 @@ define i64 @test8(i8* %data) {
 ; CHECK-NEXT:    movzwl %ax, %eax
 ; CHECK-NEXT:    shrl $2, %eax
 ; CHECK-NEXT:    orl $60, %eax
-; CHECK-NEXT:    movl %eax, %eax
 ; CHECK-NEXT:    retq
 entry:
   %bf.load = load i8, i8* %data, align 4
diff --git a/test/CodeGen/XCore/epilogue_prologue.ll b/test/CodeGen/XCore/epilogue_prologue.ll
index 30e1bb9d86a67067e6428ea16558891eee5507f9..a7194419c6547cc591048b1b77786684d410a17a 100644
--- a/test/CodeGen/XCore/epilogue_prologue.ll
+++ b/test/CodeGen/XCore/epilogue_prologue.ll
@@ -1,5 +1,5 @@
 ; RUN: llc < %s -march=xcore | FileCheck %s
-; RUN: llc < %s -march=xcore -disable-fp-elim | FileCheck %s -check-prefix=CHECKFP
+; RUN: llc < %s -march=xcore -frame-pointer=all | FileCheck %s -check-prefix=CHECKFP
 
 ; When using SP for small frames, we don't need any scratch registers (SR).
 ; When using SP for large frames, we may need two scratch registers.
diff --git a/test/CodeGen/XCore/llvm-intrinsics.ll b/test/CodeGen/XCore/llvm-intrinsics.ll
index b7868d350b4a3977e2c06d4545e0eca2b2f9d894..cad565661141ab2a1184ae8b21afb51865c31df4 100644
--- a/test/CodeGen/XCore/llvm-intrinsics.ll
+++ b/test/CodeGen/XCore/llvm-intrinsics.ll
@@ -1,5 +1,5 @@
 ; RUN: llc < %s -march=xcore | FileCheck %s
-; RUN: llc < %s -march=xcore -disable-fp-elim | FileCheck %s -check-prefix=CHECKFP
+; RUN: llc < %s -march=xcore -frame-pointer=all | FileCheck %s -check-prefix=CHECKFP
 
 declare i8* @llvm.frameaddress(i32) nounwind readnone
 declare i8* @llvm.returnaddress(i32) nounwind
diff --git a/test/DebugInfo/AArch64/asan-stack-vars.mir b/test/DebugInfo/AArch64/asan-stack-vars.mir
index 428cef6272266b288a962c986d21ee212a72a3e7..4bbacc1ae0d4973a3ff585d7a29432308744d5fd 100644
--- a/test/DebugInfo/AArch64/asan-stack-vars.mir
+++ b/test/DebugInfo/AArch64/asan-stack-vars.mir
@@ -28,7 +28,7 @@
 # CHECK: "_cmd"
 # CHECK: DW_TAG_formal_parameter
 # CHECK-NEXT: DW_AT_location
-# CHECK-NEXT:   [0x{{0*}}, 0x{{.*}}):
+# CHECK-NEXT:   [0x{{.*}}, 0x{{.*}}):
 # CHECK-NOT:    DW_AT_
 # CHECK:        [0x{{.*}}, [[FN_END]]):
 # CHECK-NEXT: DW_AT_name {{.*}}"imageSize"
diff --git a/test/DebugInfo/AArch64/dwarfdump.ll b/test/DebugInfo/AArch64/dwarfdump.ll
index 6ce5f02344571b52fa8aad0f89521b895c4b458e..956b10d64ab4d6b182b94442979cd4d38e361862 100644
--- a/test/DebugInfo/AArch64/dwarfdump.ll
+++ b/test/DebugInfo/AArch64/dwarfdump.ll
@@ -14,9 +14,9 @@
 
 ; A couple of ABS64s similarly:
 
-; CHECK: DW_AT_low_pc [DW_FORM_addr] (0x0000000000000000)
+; CHECK: DW_AT_low_pc [DW_FORM_addr] (0x0000000000000000 ".text")
 ; CHECK-4: DW_AT_high_pc [DW_FORM_data4] (0x00000008)
-; CHECK-3: DW_AT_high_pc [DW_FORM_addr] (0x0000000000000008)
+; CHECK-3: DW_AT_high_pc [DW_FORM_addr] (0x0000000000000008 ".text")
 
 define i32 @main() nounwind !dbg !3 {
   ret i32 0, !dbg !8
diff --git a/test/DebugInfo/AArch64/frameindices.ll b/test/DebugInfo/AArch64/frameindices.ll
index 18cd54afbc4d26a295c3c8557bbad051e54677bf..a74e6bad30588b1ed139fccc341eaa7b86e57319 100644
--- a/test/DebugInfo/AArch64/frameindices.ll
+++ b/test/DebugInfo/AArch64/frameindices.ll
@@ -1,4 +1,4 @@
-; RUN: llc -disable-fp-elim -O0 -fast-isel -filetype=obj < %s | llvm-dwarfdump -v - | FileCheck %s
+; RUN: llc -frame-pointer=all -O0 -fast-isel -filetype=obj < %s | llvm-dwarfdump -v - | FileCheck %s
 ; Test that a variable with multiple entries in the MMI table makes it into the
 ; debug info.
 ;
diff --git a/test/DebugInfo/AArch64/prologue_end.ll b/test/DebugInfo/AArch64/prologue_end.ll
index 0187032e138f33121a5ea1bf9b0651cbf35ac93d..bafbcf752aa44e624abc0d04086f87d47927b4bd 100644
--- a/test/DebugInfo/AArch64/prologue_end.ll
+++ b/test/DebugInfo/AArch64/prologue_end.ll
@@ -1,4 +1,4 @@
-; RUN: llc -disable-fp-elim -O0 -fast-isel %s -mtriple aarch64-apple-darwin -o - | FileCheck %s
+; RUN: llc -frame-pointer=all -O0 -fast-isel %s -mtriple aarch64-apple-darwin -o - | FileCheck %s
 
 ; int func(void);
 ; void prologue_end_test() {
diff --git a/test/DebugInfo/AArch64/return-address-signing.ll b/test/DebugInfo/AArch64/return-address-signing.ll
new file mode 100644
index 0000000000000000000000000000000000000000..c679a9da65d6ee8cc4e02c60a3432e2e99bcf757
--- /dev/null
+++ b/test/DebugInfo/AArch64/return-address-signing.ll
@@ -0,0 +1,27 @@
+; RUN: llc -mtriple=aarch64-arm-none-eabi < %s -filetype=obj -o - \
+; RUN:    | llvm-dwarfdump -v - | FileCheck -check-prefix=CHECK %s
+
+;CHECK: CIE
+;CHECK: Augmentation:          "zR"
+define i32 @foo()  "sign-return-address"="all" {
+  ret i32 0
+}
+
+;CHECK: CIE
+;CHECK: Augmentation:          "zRB"
+
+define i32 @bar()  "sign-return-address"="all" "sign-return-address-key"="b_key" {
+  ret i32 0
+}
+
+;CHECK-NOT: CIE
+
+define i32 @baz()  "sign-return-address"="all" nounwind {
+  ret i32 0
+}
+
+;CHECK-NOT: CIE
+
+define i32 @qux()  "sign-return-address"="all" "sign-return-address-key"="b_key" nounwind {
+  ret i32 0
+}
diff --git a/test/DebugInfo/AArch64/tls-at-location.ll b/test/DebugInfo/AArch64/tls-at-location.ll
index 4fbbccf1e678d287e26dd291cb919c5cdf2695e6..e0cbe4123e2410dc1a7a937f6a7d984fc2f15723 100644
--- a/test/DebugInfo/AArch64/tls-at-location.ll
+++ b/test/DebugInfo/AArch64/tls-at-location.ll
@@ -21,7 +21,7 @@ attributes #0 = { noinline nounwind optnone }
 
 !0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression())
 !1 = distinct !DIGlobalVariable(name: "var", scope: !2, file: !3, line: 1, type: !6, isLocal: false, isDefinition: true)
-!2 = distinct !DICompileUnit(language: DW_LANG_C99, file: !3, producer: "clang version 7.0.0 (https://github.com/llvm-mirror/clang.git 43eac1f9d7d2c985831b485d9ccc807416d1cf29) (https://github.com/llvm-mirror/llvm.git d53cdbf4cc5414ea540174a036202c555ce8fc4b)", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, globals: !5)
+!2 = distinct !DICompileUnit(language: DW_LANG_C99, file: !3, producer: "clang version 7.0.0", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, globals: !5)
 !3 = !DIFile(filename: "tls-at-location.c", directory: "/home/lliu0/llvm/tls-at-location/DebugInfo/AArch64")
 !4 = !{}
 !5 = !{!0}
@@ -29,7 +29,7 @@ attributes #0 = { noinline nounwind optnone }
 !7 = !{i32 2, !"Dwarf Version", i32 4}
 !8 = !{i32 2, !"Debug Info Version", i32 3}
 !9 = !{i32 1, !"wchar_size", i32 4}
-!10 = !{!"clang version 7.0.0 (https://github.com/llvm-mirror/clang.git 43eac1f9d7d2c985831b485d9ccc807416d1cf29) (https://github.com/llvm-mirror/llvm.git d53cdbf4cc5414ea540174a036202c555ce8fc4b)"}
+!10 = !{!"clang version 7.0.0"}
 !11 = distinct !DISubprogram(name: "foo", scope: !3, file: !3, line: 3, type: !12, isLocal: false, isDefinition: true, scopeLine: 3, isOptimized: false, unit: !2)
 !12 = !DISubroutineType(types: !13)
 !13 = !{!6}
diff --git a/test/DebugInfo/ARM/PR26163.ll b/test/DebugInfo/ARM/PR26163.ll
index 3b3ec9c3f4f8612acb36ce84052579724f3034c1..5dac6ad83105d345a19d31472d9c8e957e6b938c 100644
--- a/test/DebugInfo/ARM/PR26163.ll
+++ b/test/DebugInfo/ARM/PR26163.ll
@@ -1,16 +1,17 @@
 ; RUN: llc -filetype=obj -o - < %s | llvm-dwarfdump -v -debug-info - | FileCheck %s
 ;
-; Checks that we're creating two ranges, one that terminates immediately
-; and one that spans the rest of the function. This isn't necessarily the
-; best thing to do here (and also not necessarily correct, since the first
-; one has a bit_piece), but it is what is currently being emitted, any
-; change here needs to be intentional, so the test is very specific.
+; Checks that we're omitting the first range, as it is empty, and that we're
+; emitting one that spans the rest of the function. In this case, the first
+; range, which we omit, describes 8 bytes of the variable using DW_OP_litX,
+; whereas the second one only describes 4 bytes, so clobbering the whole 8 byte
+; fragment with the 4 bytes fragment isn't necessarily best thing to do here,
+; but it is what is currently being emitted. Any change here needs to be
+; intentional, so the test is very specific.
 ;
 ; CHECK: DW_TAG_inlined_subroutine
 ; CHECK: DW_TAG_variable
 ; CHECK:   DW_AT_location [DW_FORM_sec_offset] ({{.*}}
-; CHECK:      [0x00000004, 0x00000004): DW_OP_lit0, DW_OP_stack_value, DW_OP_piece 0x8
-; CHECK:      [0x00000004, 0x00000014): DW_OP_lit0, DW_OP_stack_value, DW_OP_piece 0x4)
+; CHECK-NEXT: [0x00000004, 0x00000014): DW_OP_lit0, DW_OP_stack_value, DW_OP_piece 0x4)
 
 ; Created form the following test case (PR26163) with
 ; clang -cc1 -triple armv4t--freebsd11.0-gnueabi -emit-obj -debug-info-kind=standalone -O2 -x c test.c
@@ -69,7 +70,7 @@ entry:
 !llvm.module.flags = !{!22, !23, !24}
 !llvm.ident = !{!25}
 
-!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 3.9.0 (https://github.com/llvm-mirror/clang 89dda3855cda574f355e6defa1d77bdae5053994) (llvm/trunk 257891)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2)
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 3.9.0", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2)
 !1 = !DIFile(filename: "<stdin>", directory: "/home/ubuntu/bugs")
 !2 = !{}
 !4 = distinct !DISubprogram(name: "parse_config_file", scope: !5, file: !5, line: 22, type: !6, isLocal: false, isDefinition: true, scopeLine: 23, flags: DIFlagPrototyped, isOptimized: true, unit: !0, retainedNodes: !9)
@@ -93,7 +94,7 @@ entry:
 !22 = !{i32 2, !"Debug Info Version", i32 3}
 !23 = !{i32 1, !"wchar_size", i32 4}
 !24 = !{i32 1, !"min_enum_size", i32 4}
-!25 = !{!"clang version 3.9.0 (https://github.com/llvm-mirror/clang 89dda3855cda574f355e6defa1d77bdae5053994) (llvm/trunk 257891)"}
+!25 = !{!"clang version 3.9.0"}
 !26 = !DIExpression()
 !27 = !DILocation(line: 11, scope: !11, inlinedAt: !28)
 !28 = distinct !DILocation(line: 26, scope: !4)
diff --git a/test/DebugInfo/ARM/multiple-constant-uses-drops-dbgloc.ll b/test/DebugInfo/ARM/multiple-constant-uses-drops-dbgloc.ll
index a62bbdae3a5f9829e9c46fbacb41d9518273d4b5..a8e31b054357511d313d649019732334dad3fd67 100644
--- a/test/DebugInfo/ARM/multiple-constant-uses-drops-dbgloc.ll
+++ b/test/DebugInfo/ARM/multiple-constant-uses-drops-dbgloc.ll
@@ -45,7 +45,7 @@ attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"=
 !9 = !{i32 2, !"Debug Info Version", i32 3}
 !10 = !{i32 1, !"wchar_size", i32 4}
 !11 = !{i32 1, !"min_enum_size", i32 4}
-!12 = !{!"clang version 3.7.0 (http://llvm.org/git/clang.git 9b0abb9df531ef7928c8182120e1869affca17d5) (http://llvm.org/git/llvm.git b1e759524dd94f7ce1e24935daed8383927e96c1)"}
+!12 = !{!"clang version 3.7.0"}
 !13 = distinct !DISubprogram(name: "proc", scope: !3, file: !3, line: 4, type: !14, isLocal: false, isDefinition: true, scopeLine: 5, flags: DIFlagPrototyped, isOptimized: false, unit: !2, retainedNodes: !16)
 !14 = !DISubroutineType(types: !15)
 !15 = !{null}
diff --git a/test/DebugInfo/ARM/partial-subreg.ll b/test/DebugInfo/ARM/partial-subreg.ll
index a76725d9c2498dfe36e7820186613b2c7ea89582..1d9efc8aa4aba0cd053fb3443e21447418b79aa1 100644
--- a/test/DebugInfo/ARM/partial-subreg.ll
+++ b/test/DebugInfo/ARM/partial-subreg.ll
@@ -9,6 +9,7 @@
 ; CHECK:   DW_AT_name {{.*}}"subscript.get"
 ; CHECK:  DW_TAG_formal_parameter
 ; CHECK-NEXT: DW_AT_location [DW_FORM_sec_offset]	({{.*}}
+; CHECK-NEXT:  [0x{{.*}}, 0x{{.*}}): DW_OP_regx D16, DW_OP_piece 0x8, DW_OP_regx D17, DW_OP_piece 0x4
 ; CHECK-NEXT:  [0x{{.*}}, 0x{{.*}}): DW_OP_regx D16, DW_OP_piece 0x8, DW_OP_regx D17, DW_OP_piece 0x4, DW_OP_regx D16, DW_OP_piece 0x8, DW_OP_regx D17, DW_OP_piece 0x4
 
 source_filename = "simd.ll"
diff --git a/test/DebugInfo/ARM/prologue_end.ll b/test/DebugInfo/ARM/prologue_end.ll
index 7a4a0c7859cd7150ea346a0c383b6df651024653..2c4922d1a8a435f6e8a5b46a50dd66bf676b9cde 100644
--- a/test/DebugInfo/ARM/prologue_end.ll
+++ b/test/DebugInfo/ARM/prologue_end.ll
@@ -1,5 +1,5 @@
-; RUN: llc -disable-fp-elim -O0 %s -mtriple armv7-apple-darwin -o - | FileCheck %s
-; RUN: llc -disable-fp-elim -O0 %s -mtriple thumbv7-apple-darwin -o - | FileCheck %s
+; RUN: llc -frame-pointer=all -O0 %s -mtriple armv7-apple-darwin -o - | FileCheck %s
+; RUN: llc -frame-pointer=all -O0 %s -mtriple thumbv7-apple-darwin -o - | FileCheck %s
 
 ; int func(void);
 ; void prologue_end_test() {
diff --git a/test/DebugInfo/COFF/enum-co.ll b/test/DebugInfo/COFF/enum-co.ll
index 14c1955c46a56c4c91c25c9e8d0f214629f23dd1..edb3ee60166e95cb8275796e8dfee39e18b33da5 100644
--- a/test/DebugInfo/COFF/enum-co.ll
+++ b/test/DebugInfo/COFF/enum-co.ll
@@ -130,7 +130,7 @@ attributes #1 = { nounwind readnone speculatable }
 !llvm.module.flags = !{!25, !26, !27, !28}
 !llvm.ident = !{!29}
 
-!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !1, producer: "clang version 8.0.0 (https://github.com/llvm-mirror/clang.git 9884fc1d0881576784e9b50da9eb61a5eb427f1c) (https://github.com/llvm-mirror/llvm.git 33b1a96b81ba4e33cfc4a129ce43b5331e16936b)", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, nameTableKind: None)
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !1, producer: "clang version 8.0.0", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, nameTableKind: None)
 !1 = !DIFile(filename: "enum-co.cpp", directory: "D:\5Cupstream\5Cllvm\5Ctest\5CDebugInfo\5CCOFF", checksumkind: CSK_MD5, checksum: "2e53b90441669acca735bad28ed3a1ab")
 !2 = !{!3, !8, !13, !18}
 !3 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "Enum", file: !1, line: 4, baseType: !4, size: 32, elements: !5, identifier: ".?AW4Enum@@")
@@ -138,7 +138,7 @@ attributes #1 = { nounwind readnone speculatable }
 !5 = !{!6, !7}
 !6 = !DIEnumerator(name: "ON", value: 0)
 !7 = !DIEnumerator(name: "OFF", value: 1)
-!8 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "EnumClass", file: !1, line: 7, baseType: !4, size: 32, flags: DIFlagFixedEnum, elements: !9, identifier: ".?AW4EnumClass@@")
+!8 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "EnumClass", file: !1, line: 7, baseType: !4, size: 32, flags: DIFlagEnumClass, elements: !9, identifier: ".?AW4EnumClass@@")
 !9 = !{!10, !11, !12}
 !10 = !DIEnumerator(name: "RED", value: 0)
 !11 = !DIEnumerator(name: "BLUE", value: 1)
@@ -159,7 +159,7 @@ attributes #1 = { nounwind readnone speculatable }
 !26 = !{i32 2, !"Debug Info Version", i32 3}
 !27 = !{i32 1, !"wchar_size", i32 2}
 !28 = !{i32 7, !"PIC Level", i32 2}
-!29 = !{!"clang version 8.0.0 (https://github.com/llvm-mirror/clang.git 9884fc1d0881576784e9b50da9eb61a5eb427f1c) (https://github.com/llvm-mirror/llvm.git 33b1a96b81ba4e33cfc4a129ce43b5331e16936b)"}
+!29 = !{!"clang version 8.0.0"}
 !30 = distinct !DISubprogram(name: "Func_Enum", linkageName: "?Func_Enum@@YA?AW4Enum@@AEAW41@@Z", scope: !1, file: !1, line: 5, type: !31, isLocal: false, isDefinition: true, scopeLine: 5, flags: DIFlagPrototyped, isOptimized: false, unit: !0, retainedNodes: !17)
 !31 = !DISubroutineType(types: !32)
 !32 = !{!3, !33}
diff --git a/test/DebugInfo/COFF/fp-stack.ll b/test/DebugInfo/COFF/fp-stack.ll
index 2d57e5aa3f7b73cd9c7db8afcdf8c64add8ba1a9..f0d80b3e9b26a22cb93e7ea618e274dc90542136 100644
--- a/test/DebugInfo/COFF/fp-stack.ll
+++ b/test/DebugInfo/COFF/fp-stack.ll
@@ -10,14 +10,14 @@ entry:
   ret double %sub
 }
 
-; ASM:         .cv_def_range    Lfunc_begin0 Lfunc_end0, "A\021\200\000\000\000"
+; ASM:         .cv_def_range    Ltmp1 Lfunc_end0, "A\021\200\000\000\000"
 ; OBJ:    DefRangeRegisterSym {
 ; OBJ:      Register: ST0 (0x80)
 ; OBJ:      MayHaveNoName: 0
 ; OBJ:      LocalVariableAddrRange {
-; OBJ:        OffsetStart: .text+0x0
+; OBJ:        OffsetStart: .text+0x6
 ; OBJ:        ISectStart: 0x0
-; OBJ:        Range: 0x7
+; OBJ:        Range: 0x1
 ; OBJ:      }
 ; OBJ:    }
 
diff --git a/test/DebugInfo/COFF/fpo-stack-protect.ll b/test/DebugInfo/COFF/fpo-stack-protect.ll
index 643cebae969e3ad0a6398fae57e99ad355cc3f21..c9ea2c695b928422b2af850885863db33d26e05a 100644
--- a/test/DebugInfo/COFF/fpo-stack-protect.ll
+++ b/test/DebugInfo/COFF/fpo-stack-protect.ll
@@ -30,7 +30,7 @@
 ; CHECK:         addl    $20, %esp
 ; CHECK:         popl    %esi
 ; CHECK:         retl
-; CHECK: Ltmp2:
+; CHECK: Ltmp3:
 ; CHECK:         .cv_fpo_endproc
 
 ; ModuleID = 't.c'
diff --git a/test/DebugInfo/COFF/frameproc-flags.ll b/test/DebugInfo/COFF/frameproc-flags.ll
index b2a1ef97cfb5a9090c6e3327fc631568b98d74d4..b7c05928901c3256965734fdcb94822b7d2f1ed4 100644
--- a/test/DebugInfo/COFF/frameproc-flags.ll
+++ b/test/DebugInfo/COFF/frameproc-flags.ll
@@ -216,7 +216,7 @@ __try.cont:                                       ; preds = %entry, %__except.re
 define internal i32 @"?filt$0@0@seh@@"() #8 !dbg !74 {
 entry:
   %0 = tail call i8* @llvm.frameaddress(i32 1)
-  %1 = tail call i8* @llvm.x86.seh.recoverfp(i8* bitcast (void ()* @seh to i8*), i8* %0)
+  %1 = tail call i8* @llvm.eh.recoverfp(i8* bitcast (void ()* @seh to i8*), i8* %0)
   %2 = tail call i8* @llvm.localrecover(i8* bitcast (void ()* @seh to i8*), i8* %1, i32 0)
   %__exception_code = bitcast i8* %2 to i32*
   %3 = getelementptr inbounds i8, i8* %0, i32 -20, !dbg !76
@@ -233,7 +233,7 @@ entry:
 declare i8* @llvm.frameaddress(i32) #9
 
 ; Function Attrs: nounwind readnone
-declare i8* @llvm.x86.seh.recoverfp(i8*, i8*) #9
+declare i8* @llvm.eh.recoverfp(i8*, i8*) #9
 
 ; Function Attrs: nounwind readnone
 declare i8* @llvm.localrecover(i8*, i8*, i32) #9
diff --git a/test/DebugInfo/COFF/function-options.ll b/test/DebugInfo/COFF/function-options.ll
index 57d644724e54a73b91378ae31d7eae7a6cb287c9..98378f2ff696f3563a0921030661d8b622cc8c2f 100644
--- a/test/DebugInfo/COFF/function-options.ll
+++ b/test/DebugInfo/COFF/function-options.ll
@@ -473,14 +473,14 @@ attributes #1 = { nounwind readnone speculatable }
 !llvm.module.flags = !{!3, !4, !5, !6}
 !llvm.ident = !{!7}
 
-!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !1, producer: "clang version 8.0.0 (https://github.com/llvm-mirror/clang.git 9884fc1d0881576784e9b50da9eb61a5eb427f1c) (https://github.com/llvm-mirror/llvm.git 33b1a96b81ba4e33cfc4a129ce43b5331e16936b)", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, nameTableKind: None)
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !1, producer: "clang version 8.0.0", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, nameTableKind: None)
 !1 = !DIFile(filename: "function-options.cpp", directory: "\5Ctest\5CDebugInfo\5CCOFF", checksumkind: CSK_MD5, checksum: "e73e74ea0bd81174051f0a4746343e00")
 !2 = !{}
 !3 = !{i32 2, !"CodeView", i32 1}
 !4 = !{i32 2, !"Debug Info Version", i32 3}
 !5 = !{i32 1, !"wchar_size", i32 2}
 !6 = !{i32 7, !"PIC Level", i32 2}
-!7 = !{!"clang version 8.0.0 (https://github.com/llvm-mirror/clang.git 9884fc1d0881576784e9b50da9eb61a5eb427f1c) (https://github.com/llvm-mirror/llvm.git 33b1a96b81ba4e33cfc4a129ce43b5331e16936b)"}
+!7 = !{!"clang version 8.0.0"}
 !8 = distinct !DISubprogram(name: "Func_AClass", linkageName: "?Func_AClass@@YA?AVAClass@@AEAV1@@Z", scope: !9, file: !9, line: 6, type: !10, isLocal: false, isDefinition: true, scopeLine: 6, flags: DIFlagPrototyped, isOptimized: false, unit: !0, retainedNodes: !2)
 !9 = !DIFile(filename: "function-options.cpp", directory: "D:\5Cupstream\5Cllvm\5Ctest\5CDebugInfo\5CCOFF")
 !10 = !DISubroutineType(types: !11)
diff --git a/test/DebugInfo/COFF/global_visibility.ll b/test/DebugInfo/COFF/global_visibility.ll
new file mode 100644
index 0000000000000000000000000000000000000000..a93707f6cae00e1f4a2e92aa12412a1483741714
--- /dev/null
+++ b/test/DebugInfo/COFF/global_visibility.ll
@@ -0,0 +1,276 @@
+; RUN: llc < %s -filetype=obj | llvm-readobj - -codeview | FileCheck %s
+;
+; This test verifies global variables are emitted within the correct scope.
+;
+; -- global_visibility.cpp ----------------------------------------------------
+;  1	
+;  2	int global_int = 0;
+;  3	
+;  4	template <typename T> struct A {
+;  5	  static T comdat_int;
+;  6	  static T set(T value) { T r = comdat_int; comdat_int = value; return r; };
+;  7	};
+;  8	
+;  9	template <typename T> T A<T>::comdat_int = 42;
+; 10	
+; 11	void foo() {
+; 12	  static int local_int = 1;
+; 13	  {
+; 14	    static int nested_int = 2;
+; 15	    local_int = nested_int;
+; 16	  }
+; 17	  local_int = A<int>::set(42);
+; 18	}
+; 19	
+; 20	void bar() {
+; 21	  static int local_int = 3;
+; 22	  {
+; 23	    static int nested_int = 4;
+; 24	    local_int = nested_int;
+; 25	  }
+; 26	  local_int = A<unsigned>::set(42);
+; 27	}
+; 28	
+; -----------------------------------------------------------------------------
+;
+; $ clang -S -emit-llvm -g -gcodeview global_visibility.cpp
+;
+; NOTE: The scope for both DIGlobalVariable's named "nested_int" should refer
+;       to the appropriate DILexicalBlock, not a DISubprogram.
+;
+
+; CHECK: CodeViewDebugInfo [
+; CHECK:   Section: .debug$S (9)
+
+; CHECK:   Subsection [
+; CHECK:     SubSectionType: Symbols (0xF1)
+; CHECK:     GlobalProcIdSym {
+; CHECK:       Kind: S_GPROC32_ID (0x1147)
+; CHECK:       DisplayName: foo
+; CHECK:       LinkageName: ?foo@@YAXXZ
+; CHECK:     }
+; CHECK:     DataSym {
+; CHECK:       Kind: S_LDATA32 (0x110C)
+; CHECK:       DisplayName: local_int
+; CHECK:       LinkageName: ?local_int@?1??foo@@YAXXZ@4HA
+; CHECK:     }
+; CHECK:     BlockSym {
+; CHECK:       Kind: S_BLOCK32 (0x1103)
+; CHECK:     }
+; CHECK:     DataSym {
+; CHECK:       Kind: S_LDATA32 (0x110C)
+; CHECK:       DisplayName: nested_int
+; CHECK:       LinkageName: ?nested_int@?1??foo@@YAXXZ@4HA
+; CHECK:     }
+; CHECK:     ScopeEndSym {
+; CHECK:       Kind: S_END (0x6)
+; CHECK:     }
+; CHECK:     ProcEnd {
+; CHECK:       Kind: S_PROC_ID_END (0x114F)
+; CHECK:     }
+; CHECK:   ]
+; CHECK:   Subsection [
+; CHECK:     SubSectionType: Symbols (0xF1)
+; CHECK:     GlobalProcIdSym {
+; CHECK:       Kind: S_GPROC32_ID (0x1147)
+; CHECK:       DisplayName: bar
+; CHECK:       LinkageName: ?bar@@YAXXZ
+; CHECK:     }
+; CHECK:     DataSym {
+; CHECK:       Kind: S_LDATA32 (0x110C)
+; CHECK:       DisplayName: local_int
+; CHECK:       LinkageName: ?local_int@?1??bar@@YAXXZ@4HA
+; CHECK:     }
+; CHECK:     BlockSym {
+; CHECK:       Kind: S_BLOCK32 (0x1103)
+; CHECK:     }
+; CHECK:     DataSym {
+; CHECK:       Kind: S_LDATA32 (0x110C)
+; CHECK:       DisplayName: nested_int
+; CHECK:       LinkageName: ?nested_int@?1??bar@@YAXXZ@4HA
+; CHECK:     }
+; CHECK:     ScopeEndSym {
+; CHECK:       Kind: S_END (0x6)
+; CHECK:     }
+; CHECK:     ProcEnd {
+; CHECK:       Kind: S_PROC_ID_END (0x114F)
+; CHECK:     }
+; CHECK:   ]
+; CHECK:   Subsection [
+; CHECK:     SubSectionType: Symbols (0xF1)
+; CHECK:     GlobalData {
+; CHECK:       Kind: S_GDATA32 (0x110D)
+; CHECK:       DisplayName: global_int
+; CHECK:       LinkageName: ?global_int@@3HA
+; CHECK:     }
+; CHECK:   ]
+; CHECK: ]
+; CHECK: CodeViewDebugInfo [
+; CHECK:   Section: .debug$S (16)
+; CHECK:   Subsection [
+; CHECK:     SubSectionType: Symbols (0xF1)
+; CHECK:     GlobalData {
+; CHECK:       Kind: S_GDATA32 (0x110D)
+; CHECK:       DisplayName: comdat_int
+; CHECK:       LinkageName: ?comdat_int@?$A@H@@2HA
+; CHECK:     }
+; CHECK:   ]
+; CHECK: ]
+; CHECK: CodeViewDebugInfo [
+; CHECK:   Section: .debug$S (17)
+; CHECK:   Subsection [
+; CHECK:     SubSectionType: Symbols (0xF1)
+; CHECK:     GlobalData {
+; CHECK:       Kind: S_GDATA32 (0x110D)
+; CHECK:       DisplayName: comdat_int
+; CHECK:       LinkageName: ?comdat_int@?$A@I@@2IA
+; CHECK:     }
+; CHECK:   ]
+; CHECK: ]
+;
+
+; ModuleID = 'global_visibility.cpp'
+source_filename = "global_visibility.cpp"
+target datalayout = "e-m:w-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-pc-windows-msvc19.15.26730"
+
+$"?set@?$A@H@@SAHH@Z" = comdat any
+
+$"?set@?$A@I@@SAII@Z" = comdat any
+
+$"?comdat_int@?$A@H@@2HA" = comdat any
+
+$"?comdat_int@?$A@I@@2IA" = comdat any
+
+@"?global_int@@3HA" = dso_local global i32 0, align 4, !dbg !0
+@"?local_int@?1??foo@@YAXXZ@4HA" = internal global i32 1, align 4, !dbg !6
+@"?nested_int@?1??foo@@YAXXZ@4HA" = internal global i32 2, align 4, !dbg !12
+@"?local_int@?1??bar@@YAXXZ@4HA" = internal global i32 3, align 4, !dbg !14
+@"?nested_int@?1??bar@@YAXXZ@4HA" = internal global i32 4, align 4, !dbg !17
+@"?comdat_int@?$A@H@@2HA" = linkonce_odr dso_local global i32 42, comdat, align 4, !dbg !19
+@"?comdat_int@?$A@I@@2IA" = linkonce_odr dso_local global i32 42, comdat, align 4, !dbg !29
+
+; Function Attrs: noinline optnone uwtable
+define dso_local void @"?foo@@YAXXZ"() #0 !dbg !8 {
+entry:
+  %0 = load i32, i32* @"?nested_int@?1??foo@@YAXXZ@4HA", align 4, !dbg !45
+  store i32 %0, i32* @"?local_int@?1??foo@@YAXXZ@4HA", align 4, !dbg !45
+  %call = call i32 @"?set@?$A@H@@SAHH@Z"(i32 42), !dbg !47
+  store i32 %call, i32* @"?local_int@?1??foo@@YAXXZ@4HA", align 4, !dbg !47
+  ret void, !dbg !48
+}
+
+; Function Attrs: noinline nounwind optnone uwtable
+define linkonce_odr dso_local i32 @"?set@?$A@H@@SAHH@Z"(i32 %value) #1 comdat align 2 !dbg !49 {
+entry:
+  %value.addr = alloca i32, align 4
+  %r = alloca i32, align 4
+  store i32 %value, i32* %value.addr, align 4
+  call void @llvm.dbg.declare(metadata i32* %value.addr, metadata !50, metadata !DIExpression()), !dbg !51
+  call void @llvm.dbg.declare(metadata i32* %r, metadata !52, metadata !DIExpression()), !dbg !51
+  %0 = load i32, i32* @"?comdat_int@?$A@H@@2HA", align 4, !dbg !51
+  store i32 %0, i32* %r, align 4, !dbg !51
+  %1 = load i32, i32* %value.addr, align 4, !dbg !51
+  store i32 %1, i32* @"?comdat_int@?$A@H@@2HA", align 4, !dbg !51
+  %2 = load i32, i32* %r, align 4, !dbg !51
+  ret i32 %2, !dbg !51
+}
+
+; Function Attrs: noinline optnone uwtable
+define dso_local void @"?bar@@YAXXZ"() #0 !dbg !16 {
+entry:
+  %0 = load i32, i32* @"?nested_int@?1??bar@@YAXXZ@4HA", align 4, !dbg !53
+  store i32 %0, i32* @"?local_int@?1??bar@@YAXXZ@4HA", align 4, !dbg !53
+  %call = call i32 @"?set@?$A@I@@SAII@Z"(i32 42), !dbg !55
+  store i32 %call, i32* @"?local_int@?1??bar@@YAXXZ@4HA", align 4, !dbg !55
+  ret void, !dbg !56
+}
+
+; Function Attrs: noinline nounwind optnone uwtable
+define linkonce_odr dso_local i32 @"?set@?$A@I@@SAII@Z"(i32 %value) #1 comdat align 2 !dbg !57 {
+entry:
+  %value.addr = alloca i32, align 4
+  %r = alloca i32, align 4
+  store i32 %value, i32* %value.addr, align 4
+  call void @llvm.dbg.declare(metadata i32* %value.addr, metadata !58, metadata !DIExpression()), !dbg !59
+  call void @llvm.dbg.declare(metadata i32* %r, metadata !60, metadata !DIExpression()), !dbg !59
+  %0 = load i32, i32* @"?comdat_int@?$A@I@@2IA", align 4, !dbg !59
+  store i32 %0, i32* %r, align 4, !dbg !59
+  %1 = load i32, i32* %value.addr, align 4, !dbg !59
+  store i32 %1, i32* @"?comdat_int@?$A@I@@2IA", align 4, !dbg !59
+  %2 = load i32, i32* %r, align 4, !dbg !59
+  ret i32 %2, !dbg !59
+}
+
+; Function Attrs: nounwind readnone speculatable
+declare void @llvm.dbg.declare(metadata, metadata, metadata) #2
+
+attributes #0 = { noinline optnone uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { noinline nounwind optnone uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { nounwind readnone speculatable }
+
+!llvm.dbg.cu = !{!2}
+!llvm.module.flags = !{!40, !41, !42, !43}
+!llvm.ident = !{!44}
+
+!0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression())
+!1 = distinct !DIGlobalVariable(name: "global_int", linkageName: "?global_int@@3HA", scope: !2, file: !3, line: 2, type: !11, isLocal: false, isDefinition: true)
+!2 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !3, producer: "clang version 8.0.0 (trunk)", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, globals: !5, nameTableKind: None)
+!3 = !DIFile(filename: "global_visibility.cpp", directory: "C:\5Cpath\5Cto\5Cdirectory", checksumkind: CSK_MD5, checksum: "f59b9e5de12391471b1a61888cb68a3e")
+!4 = !{}
+!5 = !{!0, !6, !12, !14, !17, !19, !29}
+!6 = !DIGlobalVariableExpression(var: !7, expr: !DIExpression())
+!7 = distinct !DIGlobalVariable(name: "local_int", scope: !8, file: !3, line: 12, type: !11, isLocal: true, isDefinition: true)
+!8 = distinct !DISubprogram(name: "foo", linkageName: "?foo@@YAXXZ", scope: !3, file: !3, line: 11, type: !9, scopeLine: 11, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !2, retainedNodes: !4)
+!9 = !DISubroutineType(types: !10)
+!10 = !{null}
+!11 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!12 = !DIGlobalVariableExpression(var: !13, expr: !DIExpression())
+!13 = distinct !DIGlobalVariable(name: "nested_int", scope: !46, file: !3, line: 14, type: !11, isLocal: true, isDefinition: true)
+!14 = !DIGlobalVariableExpression(var: !15, expr: !DIExpression())
+!15 = distinct !DIGlobalVariable(name: "local_int", scope: !16, file: !3, line: 21, type: !11, isLocal: true, isDefinition: true)
+!16 = distinct !DISubprogram(name: "bar", linkageName: "?bar@@YAXXZ", scope: !3, file: !3, line: 20, type: !9, scopeLine: 20, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !2, retainedNodes: !4)
+!17 = !DIGlobalVariableExpression(var: !18, expr: !DIExpression())
+!18 = distinct !DIGlobalVariable(name: "nested_int", scope: !54, file: !3, line: 23, type: !11, isLocal: true, isDefinition: true)
+!19 = !DIGlobalVariableExpression(var: !20, expr: !DIExpression())
+!20 = distinct !DIGlobalVariable(name: "comdat_int", linkageName: "?comdat_int@?$A@H@@2HA", scope: !2, file: !3, line: 9, type: !11, isLocal: false, isDefinition: true, declaration: !21)
+!21 = !DIDerivedType(tag: DW_TAG_member, name: "comdat_int", scope: !22, file: !3, line: 5, baseType: !11, flags: DIFlagStaticMember)
+!22 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "A<int>", file: !3, line: 4, size: 8, flags: DIFlagTypePassByValue | DIFlagTrivial, elements: !23, templateParams: !27, identifier: ".?AU?$A@H@@")
+!23 = !{!21, !24}
+!24 = !DISubprogram(name: "set", linkageName: "?set@?$A@H@@SAHH@Z", scope: !22, file: !3, line: 6, type: !25, scopeLine: 6, flags: DIFlagPrototyped | DIFlagStaticMember, spFlags: 0)
+!25 = !DISubroutineType(types: !26)
+!26 = !{!11, !11}
+!27 = !{!28}
+!28 = !DITemplateTypeParameter(name: "T", type: !11)
+!29 = !DIGlobalVariableExpression(var: !30, expr: !DIExpression())
+!30 = distinct !DIGlobalVariable(name: "comdat_int", linkageName: "?comdat_int@?$A@I@@2IA", scope: !2, file: !3, line: 9, type: !31, isLocal: false, isDefinition: true, declaration: !32)
+!31 = !DIBasicType(name: "unsigned int", size: 32, encoding: DW_ATE_unsigned)
+!32 = !DIDerivedType(tag: DW_TAG_member, name: "comdat_int", scope: !33, file: !3, line: 5, baseType: !31, flags: DIFlagStaticMember)
+!33 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "A<unsigned int>", file: !3, line: 4, size: 8, flags: DIFlagTypePassByValue | DIFlagTrivial, elements: !34, templateParams: !38, identifier: ".?AU?$A@I@@")
+!34 = !{!32, !35}
+!35 = !DISubprogram(name: "set", linkageName: "?set@?$A@I@@SAII@Z", scope: !33, file: !3, line: 6, type: !36, scopeLine: 6, flags: DIFlagPrototyped | DIFlagStaticMember, spFlags: 0)
+!36 = !DISubroutineType(types: !37)
+!37 = !{!31, !31}
+!38 = !{!39}
+!39 = !DITemplateTypeParameter(name: "T", type: !31)
+!40 = !{i32 2, !"CodeView", i32 1}
+!41 = !{i32 2, !"Debug Info Version", i32 3}
+!42 = !{i32 1, !"wchar_size", i32 2}
+!43 = !{i32 7, !"PIC Level", i32 2}
+!44 = !{!"clang version 8.0.0 (trunk)"}
+!45 = !DILocation(line: 15, scope: !46)
+!46 = distinct !DILexicalBlock(scope: !8, file: !3, line: 13)
+!47 = !DILocation(line: 17, scope: !8)
+!48 = !DILocation(line: 18, scope: !8)
+!49 = distinct !DISubprogram(name: "set", linkageName: "?set@?$A@H@@SAHH@Z", scope: !22, file: !3, line: 6, type: !25, scopeLine: 6, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !2, declaration: !24, retainedNodes: !4)
+!50 = !DILocalVariable(name: "value", arg: 1, scope: !49, file: !3, line: 6, type: !11)
+!51 = !DILocation(line: 6, scope: !49)
+!52 = !DILocalVariable(name: "r", scope: !49, file: !3, line: 6, type: !11)
+!53 = !DILocation(line: 24, scope: !54)
+!54 = distinct !DILexicalBlock(scope: !16, file: !3, line: 22)
+!55 = !DILocation(line: 26, scope: !16)
+!56 = !DILocation(line: 27, scope: !16)
+!57 = distinct !DISubprogram(name: "set", linkageName: "?set@?$A@I@@SAII@Z", scope: !33, file: !3, line: 6, type: !36, scopeLine: 6, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !2, declaration: !35, retainedNodes: !4)
+!58 = !DILocalVariable(name: "value", arg: 1, scope: !57, file: !3, line: 6, type: !31)
+!59 = !DILocation(line: 6, scope: !57)
+!60 = !DILocalVariable(name: "r", scope: !57, file: !3, line: 6, type: !31)
diff --git a/test/DebugInfo/COFF/pieces.ll b/test/DebugInfo/COFF/pieces.ll
index 10a81e17d924834ad7932fdcfed90f142450ecff..e04255d4940ffc7622327dbbf4464c3b9185dd25 100644
--- a/test/DebugInfo/COFF/pieces.ll
+++ b/test/DebugInfo/COFF/pieces.ll
@@ -136,7 +136,7 @@
 ; ASM:        .asciz  "pad_right"             # Function name
 ; ASM:        .short  4414                    # Record kind: S_LOCAL
 ; ASM:        .asciz  "o"
-; ASM:        .cv_def_range    .Lfunc_begin1 .Ltmp8, "C\021\021\000\000\000\004\000\000\000"
+; ASM:        .cv_def_range    .Ltmp8 .Ltmp8, "C\021\021\000\000\000\004\000\000\000"
 
 ; OBJ-LABEL: GlobalProcIdSym {
 ; OBJ:         Kind: S_GPROC32_ID (0x1147)
@@ -159,7 +159,7 @@
 ; ASM:        .asciz  "pad_left"              # Function name
 ; ASM:        .short  4414                    # Record kind: S_LOCAL
 ; ASM:        .asciz  "o"
-; ASM:        .cv_def_range    .Lfunc_begin2 .Ltmp10, "C\021\021\000\000\000\000\000\000\000"
+; ASM:        .cv_def_range    .Ltmp10 .Ltmp10, "C\021\021\000\000\000\000\000\000\000"
 
 ; OBJ-LABEL: GlobalProcIdSym {
 ; OBJ:         Kind: S_GPROC32_ID (0x1147)
diff --git a/test/DebugInfo/COFF/register-variables.ll b/test/DebugInfo/COFF/register-variables.ll
index 24daa3c11ede9d0871a4f310f18bd22311dafa3e..626137199a98d58cd77916c0a5f91128d55c56c6 100644
--- a/test/DebugInfo/COFF/register-variables.ll
+++ b/test/DebugInfo/COFF/register-variables.ll
@@ -24,9 +24,9 @@
 ; ASM: f:                                      # @f
 ; ASM: .Lfunc_begin0:
 ; ASM: # %bb.0:                                 # %entry
+; ASM:         #DEBUG_VALUE: f:p <- $ecx
 ; ASM:         pushq   %rsi
 ; ASM:         subq    $32, %rsp
-; ASM:         #DEBUG_VALUE: f:p <- $ecx
 ; ASM:         movl    %ecx, %esi
 ; ASM: [[p_ecx_esi:\.Ltmp.*]]:
 ; ASM:         #DEBUG_VALUE: f:p <- $esi
diff --git a/test/DebugInfo/COFF/type-quals.ll b/test/DebugInfo/COFF/type-quals.ll
index c81daa81d0c0efecef8acca0e1a6aa0edeb349fb..fdf92bd40d768c5c071502eed82c4e07fdfc4d59 100644
--- a/test/DebugInfo/COFF/type-quals.ll
+++ b/test/DebugInfo/COFF/type-quals.ll
@@ -209,28 +209,42 @@
 ; CHECK:     FunctionType: void (int& __restrict) (0x1011)
 ; CHECK:     Name: g
 ; CHECK:   }
-; CHECK:   ArgList (0x1013) {
+; CHECK:   Modifier (0x1013) {
+; CHECK:     TypeLeafKind: LF_MODIFIER (0x1001)
+; CHECK:     ModifiedType: char (0x70)
+; CHECK:     Modifiers [ (0x1)
+; CHECK:       Const (0x1)
+; CHECK:     ]
+; CHECK:   }
+; CHECK:   Array (0x1014) {
+; CHECK:     TypeLeafKind: LF_ARRAY (0x1503)
+; CHECK:     ElementType: const char (0x1013)
+; CHECK:     IndexType: unsigned __int64 (0x23)
+; CHECK:     SizeOf: 4
+; CHECK:     Name:
+; CHECK:   }
+; CHECK:   ArgList (0x1015) {
 ; CHECK:     TypeLeafKind: LF_ARGLIST (0x1201)
 ; CHECK:     NumArgs: 0
 ; CHECK:     Arguments [
 ; CHECK:     ]
 ; CHECK:   }
-; CHECK:   Procedure (0x1014) {
+; CHECK:   Procedure (0x1016) {
 ; CHECK:     TypeLeafKind: LF_PROCEDURE (0x1008)
 ; CHECK:     ReturnType: void (0x3)
 ; CHECK:     CallingConvention: NearC (0x0)
 ; CHECK:     FunctionOptions [ (0x0)
 ; CHECK:     ]
 ; CHECK:     NumParameters: 0
-; CHECK:     ArgListType: () (0x1013)
+; CHECK:     ArgListType: () (0x1015)
 ; CHECK:   }
-; CHECK:   FuncId (0x1015) {
+; CHECK:   FuncId (0x1017) {
 ; CHECK:     TypeLeafKind: LF_FUNC_ID (0x1601)
 ; CHECK:     ParentScope: 0x0
-; CHECK:     FunctionType: void () (0x1014)
+; CHECK:     FunctionType: void () (0x1016)
 ; CHECK:     Name: h
 ; CHECK:   }
-; CHECK:   Struct (0x1016) {
+; CHECK:   Struct (0x1018) {
 ; CHECK:     TypeLeafKind: LF_STRUCTURE (0x1505)
 ; CHECK:     MemberCount: 0
 ; CHECK:     Properties [ (0x180)
@@ -243,9 +257,9 @@
 ; CHECK:     SizeOf: 0
 ; CHECK:     Name: h::Foo
 ; CHECK:   }
-; CHECK:   Pointer (0x1017) {
+; CHECK:   Pointer (0x1019) {
 ; CHECK:     TypeLeafKind: LF_POINTER (0x1002)
-; CHECK:     PointeeType: h::Foo (0x1016)
+; CHECK:     PointeeType: h::Foo (0x1018)
 ; CHECK:     PtrType: Near64 (0xC)
 ; CHECK:     PtrMode: Pointer (0x0)
 ; CHECK:     IsFlat: 0
@@ -255,26 +269,26 @@
 ; CHECK:     IsRestrict: 0
 ; CHECK:     SizeOf: 8
 ; CHECK:   }
-; CHECK:   ArgList (0x1018) {
+; CHECK:   ArgList (0x101A) {
 ; CHECK:     TypeLeafKind: LF_ARGLIST (0x1201)
 ; CHECK:     NumArgs: 1
 ; CHECK:     Arguments [
 ; CHECK:       ArgType: int (0x74)
 ; CHECK:     ]
 ; CHECK:   }
-; CHECK:   MemberFunction (0x1019) {
+; CHECK:   MemberFunction (0x101B) {
 ; CHECK:     TypeLeafKind: LF_MFUNCTION (0x1009)
 ; CHECK:     ReturnType: int (0x74)
-; CHECK:     ClassType: h::Foo (0x1016)
-; CHECK:     ThisType: h::Foo* const (0x1017)
+; CHECK:     ClassType: h::Foo (0x1018)
+; CHECK:     ThisType: h::Foo* const (0x1019)
 ; CHECK:     CallingConvention: NearC (0x0)
 ; CHECK:     FunctionOptions [ (0x0)
 ; CHECK:     ]
 ; CHECK:     NumParameters: 1
-; CHECK:     ArgListType: (int) (0x1018)
+; CHECK:     ArgListType: (int) (0x101A)
 ; CHECK:     ThisAdjustment: 0
 ; CHECK:   }
-; CHECK:   FieldList (0x101A) {
+; CHECK:   FieldList (0x101C) {
 ; CHECK:     TypeLeafKind: LF_FIELDLIST (0x1203)
 ; CHECK:     DataMember {
 ; CHECK:       TypeLeafKind: LF_MEMBER (0x150D)
@@ -286,24 +300,24 @@
 ; CHECK:     OneMethod {
 ; CHECK:       TypeLeafKind: LF_ONEMETHOD (0x1511)
 ; CHECK:       AccessSpecifier: Public (0x3)
-; CHECK:       Type: int h::Foo::(int) (0x1019)
+; CHECK:       Type: int h::Foo::(int) (0x101B)
 ; CHECK:       Name: func
 ; CHECK:     }
 ; CHECK:   }
-; CHECK:   Struct (0x101B) {
+; CHECK:   Struct (0x101D) {
 ; CHECK:     TypeLeafKind: LF_STRUCTURE (0x1505)
 ; CHECK:     MemberCount: 2
 ; CHECK:     Properties [ (0x100)
 ; CHECK:       Scoped (0x100)
 ; CHECK:     ]
-; CHECK:     FieldList: <field list> (0x101A)
+; CHECK:     FieldList: <field list> (0x101C)
 ; CHECK:     DerivedFrom: 0x0
 ; CHECK:     VShape: 0x0
 ; CHECK:     SizeOf: 4
 ; CHECK:     Name: h::Foo
 ; CHECK:   }
 
-; CHECK:   Pointer (0x101D) {
+; CHECK:   Pointer (0x101F) {
 ; CHECK:     TypeLeafKind: LF_POINTER (0x1002)
 ; CHECK:     PointeeType: int (0x74)
 ; CHECK:     PtrType: Near64 (0xC)
@@ -314,12 +328,12 @@
 ; CHECK:     IsUnaligned: 0
 ; CHECK:     IsRestrict: 1
 ; CHECK:     SizeOf: 4
-; CHECK:     ClassType: h::Foo (0x1016)
+; CHECK:     ClassType: h::Foo (0x1018)
 ; CHECK:     Representation: SingleInheritanceData (0x1)
 ; CHECK:   }
-; CHECK:   Pointer (0x101E) {
+; CHECK:   Pointer (0x1020) {
 ; CHECK:     TypeLeafKind: LF_POINTER (0x1002)
-; CHECK:     PointeeType: int h::Foo::(int) (0x1019)
+; CHECK:     PointeeType: int h::Foo::(int) (0x101B)
 ; CHECK:     PtrType: Near64 (0xC)
 ; CHECK:     PtrMode: PointerToMemberFunction (0x3)
 ; CHECK:     IsFlat: 0
@@ -328,18 +342,18 @@
 ; CHECK:     IsUnaligned: 0
 ; CHECK:     IsRestrict: 0
 ; CHECK:     SizeOf: 8
-; CHECK:     ClassType: h::Foo (0x1016)
+; CHECK:     ClassType: h::Foo (0x1018)
 ; CHECK:     Representation: SingleInheritanceFunction (0x5)
 ; CHECK:   }
-; CHECK:   MemberFuncId (0x101F) {
+; CHECK:   MemberFuncId (0x1021) {
 ; CHECK:     TypeLeafKind: LF_MFUNC_ID (0x1602)
-; CHECK:     ClassType: h::Foo (0x1016)
-; CHECK:     FunctionType: int h::Foo::(int) (0x1019)
+; CHECK:     ClassType: h::Foo (0x1018)
+; CHECK:     FunctionType: int h::Foo::(int) (0x101B)
 ; CHECK:     Name: func
 ; CHECK:   }
-; CHECK:   Pointer (0x1020) {
+; CHECK:   Pointer (0x1022) {
 ; CHECK:     TypeLeafKind: LF_POINTER (0x1002)
-; CHECK:     PointeeType: h::Foo (0x1016)
+; CHECK:     PointeeType: h::Foo (0x1018)
 ; CHECK:     PtrType: Near64 (0xC)
 ; CHECK:     PtrMode: Pointer (0x0)
 ; CHECK:     IsFlat: 0
@@ -349,20 +363,6 @@
 ; CHECK:     IsRestrict: 0
 ; CHECK:     SizeOf: 8
 ; CHECK:   }
-; CHECK:   Modifier (0x1021) {
-; CHECK:     TypeLeafKind: LF_MODIFIER (0x1001)
-; CHECK:     ModifiedType: char (0x70)
-; CHECK:     Modifiers [ (0x1)
-; CHECK:       Const (0x1)
-; CHECK:     ]
-; CHECK:   }
-; CHECK:   Array (0x1022) {
-; CHECK:     TypeLeafKind: LF_ARRAY (0x1503)
-; CHECK:     ElementType: const char (0x1021)
-; CHECK:     IndexType: unsigned __int64 (0x23)
-; CHECK:     SizeOf: 4
-; CHECK:     Name:
-; CHECK:   }
 ; CHECK: ]
 
 ; CHECK-LABEL: CodeViewDebugInfo [
diff --git a/test/DebugInfo/COFF/types-empty-member-fn.ll b/test/DebugInfo/COFF/types-empty-member-fn.ll
new file mode 100644
index 0000000000000000000000000000000000000000..87cba9d62099e8710e6d9adf8d0d4d028739ce53
--- /dev/null
+++ b/test/DebugInfo/COFF/types-empty-member-fn.ll
@@ -0,0 +1,72 @@
+; RUN: llc < %s -filetype=obj | llvm-readobj - -codeview | FileCheck %s
+
+; ModuleID = 'foo.3a1fbbbh-cgu.0'
+source_filename = "foo.3a1fbbbh-cgu.0"
+target datalayout = "e-m:w-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-pc-windows-msvc"
+
+; Rust source to regenerate:
+; $ cat foo.rs
+; pub struct Foo;
+; impl Foo {
+;     pub fn foo() {}
+; }
+; $ rustc foo.rs --crate-type lib -Cdebuginfo=1 --emit=llvm-ir
+
+; CHECK:      CodeViewTypes [
+; CHECK:        MemberFunction (0x1006) {
+; CHECK-NEXT:     TypeLeafKind: LF_MFUNCTION (0x1009)
+; CHECK-NEXT:     ReturnType: void (0x3)
+; CHECK-NEXT:     ClassType: foo::Foo (0x1000)
+; CHECK-NEXT:     ThisType: 0x0
+; CHECK-NEXT:     CallingConvention: NearC (0x0)
+; CHECK-NEXT:     FunctionOptions [ (0x0)
+; CHECK-NEXT:     ]
+; CHECK-NEXT:     NumParameters: 0
+; CHECK-NEXT:     ArgListType: () (0x1005)
+; CHECK-NEXT:     ThisAdjustment: 0
+; CHECK-NEXT:   }
+; CHECK-NEXT:   MemberFuncId (0x1007) {
+; CHECK-NEXT:     TypeLeafKind: LF_MFUNC_ID (0x1602)
+; CHECK-NEXT:     ClassType: foo::Foo (0x1000)
+; CHECK-NEXT:     FunctionType: void foo::Foo::() (0x1006)
+; CHECK-NEXT:     Name: foo
+; CHECK-NEXT:   }
+; CHECK:      CodeViewDebugInfo [
+; CHECK:        FunctionLineTable [
+; CHECK-NEXT:     LinkageName: _ZN3foo3Foo3foo17hc557c2121772885bE
+; CHECK-NEXT:     Flags: 0x0
+; CHECK-NEXT:     CodeSize: 0x1
+; CHECK-NEXT:     FilenameSegment [
+; CHECK-NEXT:       Filename: D:\rust\foo.rs (0x0)
+; CHECK-NEXT:       +0x0 [
+; CHECK-NEXT:         LineNumberStart: 3
+; CHECK-NEXT:         LineNumberEndDelta: 0
+; CHECK-NEXT:         IsStatement: No
+; CHECK-NEXT:       ]
+; CHECK-NEXT:     ]
+; CHECK-NEXT:   ]
+
+; foo::Foo::foo
+; Function Attrs: uwtable
+define void @_ZN3foo3Foo3foo17hc557c2121772885bE() unnamed_addr #0 !dbg !5 {
+start:
+  ret void, !dbg !10
+}
+
+attributes #0 = { uwtable "target-cpu"="x86-64" }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_Rust, file: !1, producer: "clang LLVM (rustc version 1.33.0-nightly (8b0f0156e 2019-01-22))", isOptimized: false, runtimeVersion: 0, emissionKind: LineTablesOnly, enums: !2)
+!1 = !DIFile(filename: "foo.rs", directory: "D:\5Crust")
+!2 = !{}
+!3 = !{i32 2, !"CodeView", i32 1}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = distinct !DISubprogram(name: "foo", linkageName: "_ZN3foo3Foo3foo17hc557c2121772885bE", scope: !6, file: !1, line: 3, type: !9, scopeLine: 3, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, templateParams: !2, retainedNodes: !2)
+!6 = !DICompositeType(tag: DW_TAG_structure_type, name: "Foo", scope: !8, file: !7, align: 8, elements: !2, templateParams: !2, identifier: "5105d9fe1a2a3c68518268151b672274")
+!7 = !DIFile(filename: "<unknown>", directory: "")
+!8 = !DINamespace(name: "foo", scope: null)
+!9 = !DISubroutineType(types: !2)
+!10 = !DILocation(line: 3, scope: !5)
diff --git a/test/DebugInfo/COFF/types-this-not-ptr.ll b/test/DebugInfo/COFF/types-this-not-ptr.ll
new file mode 100644
index 0000000000000000000000000000000000000000..ae47e6ffb789093290dd41da6099834738e765df
--- /dev/null
+++ b/test/DebugInfo/COFF/types-this-not-ptr.ll
@@ -0,0 +1,45 @@
+; RUN: llc -filetype=obj %s -o %t.obj
+; RUN: llvm-pdbutil dump -types %t.obj | FileCheck %s
+
+; Manually remove the "static" flag from the LLVM IR generated by compiling the
+; following C++ source:
+; struct Foo  {
+;   static void st_meth(int x, int y);
+; } f;
+
+; "this type" below should be <no type>.
+; CHECK: LF_MFUNCTION
+; CHECK-NEXT: return type = 0x0003 (void), # args = 2, param list = {{.*}}
+; CHECK-NEXT: class type = {{.*}}, this type = <no type>, this adjust = 0
+; CHECK-NEXT: calling conv = cdecl, options = None
+
+; ModuleID = 't.cpp'
+source_filename = "t.cpp"
+target datalayout = "e-m:w-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-pc-windows-msvc19.14.26433"
+
+%struct.Foo = type { i8 }
+
+@"?f@@3UFoo@@A" = dso_local global %struct.Foo zeroinitializer, align 1, !dbg !0
+
+!llvm.dbg.cu = !{!2}
+!llvm.module.flags = !{!12, !13, !14, !15}
+!llvm.ident = !{!16}
+
+!0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression())
+!1 = distinct !DIGlobalVariable(name: "f", linkageName: "?f@@3UFoo@@A", scope: !2, file: !3, line: 3, type: !6, isLocal: false, isDefinition: true)
+!2 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !3, producer: "clang version 8.0.0 ", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, globals: !5, nameTableKind: None)
+!3 = !DIFile(filename: "t.cpp", directory: "C:\5Csrc\5Cllvm-project\5Cbuild", checksumkind: CSK_MD5, checksum: "ac580c6cde5f3f394632dcaad04873a4")
+!4 = !{}
+!5 = !{!0}
+!6 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "Foo", file: !3, line: 1, size: 8, flags: DIFlagTypePassByValue | DIFlagTrivial, elements: !7, identifier: ".?AUFoo@@")
+!7 = !{!8}
+!8 = !DISubprogram(name: "st_meth", linkageName: "?st_meth@Foo@@SAXHH@Z", scope: !6, file: !3, line: 2, type: !9, scopeLine: 2, flags: DIFlagPrototyped, spFlags: 0)
+!9 = !DISubroutineType(types: !10)
+!10 = !{null, !11, !11}
+!11 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!12 = !{i32 2, !"CodeView", i32 1}
+!13 = !{i32 2, !"Debug Info Version", i32 3}
+!14 = !{i32 1, !"wchar_size", i32 2}
+!15 = !{i32 7, !"PIC Level", i32 2}
+!16 = !{!"clang version 8.0.0 "}
diff --git a/test/DebugInfo/Generic/2010-04-19-FramePtr.ll b/test/DebugInfo/Generic/2010-04-19-FramePtr.ll
index 11c3d689cab88c44d088cafadf68eec4cd29d5c5..77e4fa003865eb1609eac6c89164289150a16a19 100644
--- a/test/DebugInfo/Generic/2010-04-19-FramePtr.ll
+++ b/test/DebugInfo/Generic/2010-04-19-FramePtr.ll
@@ -1,6 +1,6 @@
 ; RUN: %llc_dwarf -debugger-tune=lldb -asm-verbose -O1 -o - < %s | FileCheck %s
 ; RUN: %llc_dwarf -debugger-tune=gdb -asm-verbose -O1 -o - < %s | FileCheck %s --check-prefix=DISABLE
-; RUN: %llc_dwarf -disable-fp-elim -debugger-tune=lldb -asm-verbose -O1 -o - < %s | FileCheck %s --check-prefix=DISABLE
+; RUN: %llc_dwarf -frame-pointer=all -debugger-tune=lldb -asm-verbose -O1 -o - < %s | FileCheck %s --check-prefix=DISABLE
 
 ; CHECK: DW_AT_APPLE_omit_frame_ptr
 ; DISABLE-NOT: DW_AT_APPLE_omit_frame_ptr
diff --git a/test/DebugInfo/Generic/debug-info-enum.ll b/test/DebugInfo/Generic/debug-info-enum.ll
index 5d887cd5cb8bc1e2ec40d8add715476280d8a867..85e22931a5fc1f3dda54aed71390197ca8974597 100644
--- a/test/DebugInfo/Generic/debug-info-enum.ll
+++ b/test/DebugInfo/Generic/debug-info-enum.ll
@@ -28,7 +28,7 @@
 !4 = !{!5, !10, !14, !19, !23, !28, !32, !37, !41}
 
 ; Test enumeration with a fixed "signed char" underlying type.
-!5 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "E0", file: !3, line: 2, baseType: !6, size: 8, flags: DIFlagFixedEnum, elements: !7, identifier: "_ZTS2E0")
+!5 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "E0", file: !3, line: 2, baseType: !6, size: 8, flags: DIFlagEnumClass, elements: !7, identifier: "_ZTS2E0")
 !6 = !DIBasicType(name: "signed char", size: 8, encoding: DW_ATE_signed_char)
 !7 = !{!8, !9}
 !8 = !DIEnumerator(name: "A0", value: -128)
@@ -46,7 +46,7 @@
 ; CHECK-NEXT:      DW_AT_const_value     (127)
 
 ; Test enumeration with a fixed "unsigned char" underlying type.
-!10 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "E1", file: !3, line: 12, baseType: !11, size: 8, flags: DIFlagFixedEnum, elements: !12, identifier: "_ZTS2E1")
+!10 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "E1", file: !3, line: 12, baseType: !11, size: 8, flags: DIFlagEnumClass, elements: !12, identifier: "_ZTS2E1")
 !11 = !DIBasicType(name: "unsigned char", size: 8, encoding: DW_ATE_unsigned_char)
 !12 = !{!13}
 !13 = !DIEnumerator(name: "A1", value: 255, isUnsigned: true)
@@ -60,7 +60,7 @@
 ; CHECK-NEXT:      DW_AT_const_value     (255)
 
 ; Test enumeration with a fixed "short" underlying type.
-!14 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "E2", file: !3, line: 18, baseType: !15, size: 16, flags: DIFlagFixedEnum, elements: !16, identifier: "_ZTS2E2")
+!14 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "E2", file: !3, line: 18, baseType: !15, size: 16, flags: DIFlagEnumClass, elements: !16, identifier: "_ZTS2E2")
 !15 = !DIBasicType(name: "short", size: 16, encoding: DW_ATE_signed)
 !16 = !{!17, !18}
 !17 = !DIEnumerator(name: "A2", value: -32768)
@@ -78,7 +78,7 @@
 ; CHECK-NEXT:      DW_AT_const_value     (32767)
 
 ; Test enumeration with a fixed "unsigned short" underlying type.
-!19 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "E3", file: !3, line: 28, baseType: !20, size: 16, flags: DIFlagFixedEnum, elements: !21, identifier: "_ZTS2E3")
+!19 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "E3", file: !3, line: 28, baseType: !20, size: 16, flags: DIFlagEnumClass, elements: !21, identifier: "_ZTS2E3")
 !20 = !DIBasicType(name: "unsigned short", size: 16, encoding: DW_ATE_unsigned)
 !21 = !{!22}
 !22 = !DIEnumerator(name: "A3", value: 65535, isUnsigned: true)
@@ -92,7 +92,7 @@
 ; CHECK-NEXT:      DW_AT_const_value     (65535)
 
 ; Test enumeration with a fixed "int" underlying type.
-!23 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "E4", file: !3, line: 34, baseType: !24, size: 32, flags: DIFlagFixedEnum, elements: !25, identifier: "_ZTS2E4")
+!23 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "E4", file: !3, line: 34, baseType: !24, size: 32, flags: DIFlagEnumClass, elements: !25, identifier: "_ZTS2E4")
 !24 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
 !25 = !{!26, !27}
 !26 = !DIEnumerator(name: "A4", value: -2147483648)
@@ -110,7 +110,7 @@
 ; CHECK-NEXT:      DW_AT_const_value     (2147483647)
 
 ; Test enumeration with a fixed "unsigend int" underlying type.
-!28 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "E5", file: !3, line: 41, baseType: !29, size: 32, flags: DIFlagFixedEnum, elements: !30, identifier: "_ZTS2E5")
+!28 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "E5", file: !3, line: 41, baseType: !29, size: 32, flags: DIFlagEnumClass, elements: !30, identifier: "_ZTS2E5")
 !29 = !DIBasicType(name: "unsigned int", size: 32, encoding: DW_ATE_unsigned)
 !30 = !{!31}
 !31 = !DIEnumerator(name: "A5", value: 4294967295, isUnsigned: true)
@@ -124,7 +124,7 @@
 ; CHECK-NEXT:      DW_AT_const_value     (4294967295)
 
 ; Test enumeration with a fixed "long long" underlying type.
-!32 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "E6", file: !3, line: 47, baseType: !33, size: 64, flags: DIFlagFixedEnum, elements: !34, identifier: "_ZTS2E6")
+!32 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "E6", file: !3, line: 47, baseType: !33, size: 64, flags: DIFlagEnumClass, elements: !34, identifier: "_ZTS2E6")
 !33 = !DIBasicType(name: "long long int", size: 64, encoding: DW_ATE_signed)
 !34 = !{!35, !36}
 !35 = !DIEnumerator(name: "A6", value: -9223372036854775808)
@@ -142,7 +142,7 @@
 ; CHECK-NEXT:      DW_AT_const_value     (9223372036854775807)
 
 ; Test enumeration with a fixed "unsigned long long" underlying type.
-!37 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "E7", file: !3, line: 57, baseType: !38, size: 64, flags: DIFlagFixedEnum, elements: !39, identifier: "_ZTS2E7")
+!37 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "E7", file: !3, line: 57, baseType: !38, size: 64, flags: DIFlagEnumClass, elements: !39, identifier: "_ZTS2E7")
 !38 = !DIBasicType(name: "long long unsigned int", size: 64, encoding: DW_ATE_unsigned)
 !39 = !{!40}
 !40 = !DIEnumerator(name: "A7", value: 18446744073709551615, isUnsigned: true)
@@ -168,11 +168,11 @@
 ; CHECK-NOT:       DW_AT_enum_class
 ; CHECK:           DW_AT_name      ("E8")
 
-; Test enumeration without a fixed underlying type, but with the DIFlagFixedEnum
+; Test enumeration without a fixed underlying type, but with the DIFlagEnumClass
 ; set. The DW_AT_enum_class attribute should be absent. This behaviour is
 ; intented to keep compatibilty with existing DWARF consumers, which may imply
 ; the type is present whenever DW_AT_enum_class is set.
-!63 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "E9", file: !3, line: 63, size: 32, flags: DIFlagFixedEnum,  elements: !64, identifier: "_ZTS2E9")
+!63 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "E9", file: !3, line: 63, size: 32, flags: DIFlagEnumClass,  elements: !64, identifier: "_ZTS2E9")
 !64 = !{!65, !66}
 !65 = !DIEnumerator(name: "A9", value: -128)
 !66 = !DIEnumerator(name: "B9", value: 127)
diff --git a/test/DebugInfo/Generic/debuginfofinder-forward-declaration.ll b/test/DebugInfo/Generic/debuginfofinder-forward-declaration.ll
index 771ac57f50a5a04a289e66c6a2ff0a27f238a2f9..675230d2140336540d82b88d189318317a0d2b9d 100644
--- a/test/DebugInfo/Generic/debuginfofinder-forward-declaration.ll
+++ b/test/DebugInfo/Generic/debuginfofinder-forward-declaration.ll
@@ -28,7 +28,7 @@ source_filename = "test/DebugInfo/Generic/debuginfofinder-forward-declaration.ll
 
 !0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression())
 !1 = !DIGlobalVariable(name: "y", scope: !2, file: !3, line: 7, type: !6, isLocal: false, isDefinition: true)
-!2 = distinct !DICompileUnit(language: DW_LANG_C99, file: !3, producer: "clang version 3.7.0 (http://llvm.org/git/clang.git 247b30a043eb8f39ea3708e7e995089da0a6b00f) (http://llvm.org/git/llvm.git 6ecc7365a89c771fd229bdd9ffcc178684ea1aa5)", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, retainedTypes: !4, globals: !5, imports: !4)
+!2 = distinct !DICompileUnit(language: DW_LANG_C99, file: !3, producer: "clang version 3.7.0", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, retainedTypes: !4, globals: !5, imports: !4)
 !3 = !DIFile(filename: "minimal.c", directory: "/tmp")
 !4 = !{}
 !5 = !{!0}
@@ -39,5 +39,5 @@ source_filename = "test/DebugInfo/Generic/debuginfofinder-forward-declaration.ll
 !10 = !DICompositeType(tag: DW_TAG_structure_type, name: "X", file: !3, line: 1, flags: DIFlagFwdDecl)
 !11 = !{i32 2, !"Dwarf Version", i32 4}
 !12 = !{i32 2, !"Debug Info Version", i32 3}
-!13 = !{!"clang version 3.7.0 (http://llvm.org/git/clang.git 247b30a043eb8f39ea3708e7e995089da0a6b00f) (http://llvm.org/git/llvm.git 6ecc7365a89c771fd229bdd9ffcc178684ea1aa5)"}
+!13 = !{!"clang version 3.7.0"}
 
diff --git a/test/DebugInfo/Generic/debuginfofinder-imported-global-variable.ll b/test/DebugInfo/Generic/debuginfofinder-imported-global-variable.ll
index 4ec06a29c4529ae36410fc1c90dc8d01e609aecb..270ab7ad47f876abe16442c30ad50a7c08ffc384 100644
--- a/test/DebugInfo/Generic/debuginfofinder-imported-global-variable.ll
+++ b/test/DebugInfo/Generic/debuginfofinder-imported-global-variable.ll
@@ -29,7 +29,7 @@
 !2 = !DINamespace(name: "s", scope: null)
 !3 = !DIFile(filename: "source.cpp", directory: "/somewhere")
 !4 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
-!5 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !3, producer: "clang version 7.0.99 (https://git.llvm.org/git/clang.git/ ec45e6c6530b8f9a1046d0a7efae467b3151783b) (https://git.llvm.org/git/llvm.git/ 9d2fcc2bf4a301d05d9b440a805847645637ab28)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !6, globals: !7, imports: !8)
+!5 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !3, producer: "clang version 7.0.99", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !6, globals: !7, imports: !8)
 !6 = !{}
 !7 = !{!0}
 !8 = !{!9}
@@ -38,4 +38,4 @@
 !11 = !{i32 2, !"Debug Info Version", i32 3}
 !12 = !{i32 1, !"wchar_size", i32 4}
 !13 = !{i32 7, !"PIC Level", i32 2}
-!14 = !{!"clang version 7.0.99 (https://git.llvm.org/git/clang.git/ ec45e6c6530b8f9a1046d0a7efae467b3151783b) (https://git.llvm.org/git/llvm.git/ 9d2fcc2bf4a301d05d9b440a805847645637ab28)"}
+!14 = !{!"clang version 7.0.99"}
diff --git a/test/DebugInfo/Generic/dwarf-public-names.ll b/test/DebugInfo/Generic/dwarf-public-names.ll
index 947b1f5e976cb82484f61c5d4a64947641500e66..5ba431abc17d2bae21af05469d5d6bdb250dd3f2 100644
--- a/test/DebugInfo/Generic/dwarf-public-names.ll
+++ b/test/DebugInfo/Generic/dwarf-public-names.ll
@@ -117,7 +117,7 @@ attributes #1 = { nounwind readnone }
 !17 = !DIGlobalVariableExpression(var: !18, expr: !DIExpression())
 !18 = !DIGlobalVariable(name: "global_namespace_variable", linkageName: "_ZN2ns25global_namespace_variableE", scope: !19, file: !3, line: 27, type: !6, isLocal: false, isDefinition: true)
 !19 = !DINamespace(name: "ns", scope: null)
-!20 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !3, producer: "clang version 3.3 (http://llvm.org/git/clang.git a09cd8103a6a719cb2628cdf0c91682250a17bd2) (http://llvm.org/git/llvm.git 47d03cec0afca0c01ae42b82916d1d731716cd20)", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !11, retainedTypes: !11, globals: !21, imports: !11) ; previously: invalid DW_TAG_base_type
+!20 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !3, producer: "clang version 3.3", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !11, retainedTypes: !11, globals: !21, imports: !11) ; previously: invalid DW_TAG_base_type
 !21 = !{!0, !15, !17}
 !22 = !{i32 1, !"Debug Info Version", i32 3}
 !23 = distinct !DISubprogram(name: "member_function", linkageName: "_ZN1C15member_functionEv", scope: null, file: !3, line: 9, type: !8, isLocal: false, isDefinition: true, scopeLine: 9, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, unit: !20, declaration: !7, retainedNodes: !11)
diff --git a/test/DebugInfo/Generic/incorrect-variable-debugloc1.ll b/test/DebugInfo/Generic/incorrect-variable-debugloc1.ll
index df7fb643e6edba899fd762bcf63f4e313d631e83..ce24b8af2cc678342b419ce8b645d91c3598b1cc 100644
--- a/test/DebugInfo/Generic/incorrect-variable-debugloc1.ll
+++ b/test/DebugInfo/Generic/incorrect-variable-debugloc1.ll
@@ -1,8 +1,4 @@
 ; REQUIRES: object-emission
-; This test is failing for powerpc64, because a location list for the
-; variable 'c' is not generated at all. Temporary marking this test as XFAIL
-; for powerpc, until PR21881 is fixed.
-; XFAIL: powerpc64
 
 ; RUN: %llc_dwarf -O2  -dwarf-version 2 -filetype=obj < %s | llvm-dwarfdump - | FileCheck %s  --check-prefix=DWARF23
 ; RUN: %llc_dwarf -O2  -dwarf-version 3 -filetype=obj < %s | llvm-dwarfdump - | FileCheck %s  --check-prefix=DWARF23
diff --git a/test/DebugInfo/Generic/pr40628.ll b/test/DebugInfo/Generic/pr40628.ll
new file mode 100644
index 0000000000000000000000000000000000000000..69f09f7eb5bf9c8e2780a8f2974f149a74cca9f7
--- /dev/null
+++ b/test/DebugInfo/Generic/pr40628.ll
@@ -0,0 +1,56 @@
+; RUN: opt -early-cse -S %s -o - | FileCheck %s
+
+; PR40628: The first load below is determined to be redundant by EarlyCSE.
+; During salvaging, the corresponding dbg.value could have a DW_OP_deref used
+; in its DIExpression to represent the redundant operation -- however LLVM
+; cannot currently determine that the subsequent store should terminate the
+; variables location range. A debugger would display zero for the "redundant"
+; variable after stepping onto the return instruction.
+
+; Test that the load being removed results in the corresponding dbg.value
+; being assigned the 'undef' value.
+
+; CHECK:      @foo
+; CHECK-NEXT: dbg.value(metadata i32 undef, metadata ![[DEADVAR:[0-9]+]],
+; CHECK-NEXT: load
+; CHECK-NEXT: dbg.value(metadata i32 %{{[0-9]+}}, metadata ![[LIVEVAR:[0-9]+]],
+; CHECK-NEXT: store
+; CHECK-NEXT: ret
+
+; CHECK:      ![[DEADVAR]] = !DILocalVariable(name: "redundant",
+; CHECK:      ![[LIVEVAR]] = !DILocalVariable(name: "loaded",
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define dso_local i32 @foo(i32*) !dbg !7 {
+  %2 = load i32, i32* %0, align 4, !dbg !23
+  call void @llvm.dbg.value(metadata i32 %2, metadata !16, metadata !DIExpression()), !dbg !23
+  %3 = load i32, i32* %0, align 4, !dbg !23
+  call void @llvm.dbg.value(metadata i32 %3, metadata !17, metadata !DIExpression()), !dbg !23
+  store i32 0, i32* %0, align 4, !dbg !23
+  ret i32 %3, !dbg !23
+}
+
+declare void @llvm.dbg.value(metadata, metadata, metadata)
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4, !5}
+!llvm.ident = !{!6}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, nameTableKind: None)
+!1 = !DIFile(filename: "pr40628.c", directory: ".")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{i32 1, !"wchar_size", i32 4}
+!6 = !{!"clang"}
+!7 = distinct !DISubprogram(name: "foo", scope: !1, file: !1, line: 2, type: !8, scopeLine: 3, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !12)
+!8 = !DISubroutineType(types: !9)
+!9 = !{!10, !11}
+!10 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!11 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !10, size: 64)
+!12 = !{!16, !17}
+!16 = !DILocalVariable(name: "redundant", scope: !7, file: !1, line: 4, type: !10)
+!17 = !DILocalVariable(name: "loaded", scope: !7, file: !1, line: 5, type: !10)
+!23 = !DILocation(line: 4, column: 7, scope: !7)
diff --git a/test/DebugInfo/Generic/string-offsets-form.ll b/test/DebugInfo/Generic/string-offsets-form.ll
index 31e5de45884a78a9a02dbb6775e2c5efdc4eb157..7aec82f12abd217e621ec74fa393076519bfa127 100644
--- a/test/DebugInfo/Generic/string-offsets-form.ll
+++ b/test/DebugInfo/Generic/string-offsets-form.ll
@@ -16,7 +16,7 @@
 ;
 ; CHECK:     .debug_info contents:
 ; CHECK-NOT: DW_FORM_strx2
-; CHECK:     DW_AT_name [DW_FORM_strx2] ( indexed (00000100) string =
+; CHECK:     DW_AT_name [DW_FORM_strx2] (indexed (00000100) string =
 
 ; ModuleID = 'enum.cpp'
 source_filename = "enum.cpp"
diff --git a/test/DebugInfo/Generic/varargs.ll b/test/DebugInfo/Generic/varargs.ll
index 96ad9c02ba9c0f68616f1f64cd3966dd27626ae8..c8193c05e944ead48524d7254ad32cb219e163d0 100644
--- a/test/DebugInfo/Generic/varargs.ll
+++ b/test/DebugInfo/Generic/varargs.ll
@@ -95,7 +95,7 @@ attributes #1 = { nounwind readnone }
 !23 = !DILocalVariable(name: "a", line: 16, scope: !14, file: !15, type: !4)
 !24 = !DILocation(line: 16, scope: !14)
 ; Manually modifed to avoid dependence on pointer size
-!25 = !DILocalVariable(name: "fptr", line: 18, scope: !14, file: !15, type: !16)
+!25 = !DILocalVariable(name: "fptr", line: 18, scope: !14, file: !15, type: !26)
 !26 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: !16)
 !27 = !DILocation(line: 18, scope: !14)
 !28 = !DILocation(line: 22, scope: !14)
diff --git a/test/DebugInfo/Inputs/dwarfdump-test.elf-x86-64 b/test/DebugInfo/Inputs/dwarfdump-test.elf-x86-64
index 455dd1ce4fc2c7551f1d39207fb5561286cd71a0..e3afa0f4288a1c087fc1926cc9a689afb0a00166 100755
Binary files a/test/DebugInfo/Inputs/dwarfdump-test.elf-x86-64 and b/test/DebugInfo/Inputs/dwarfdump-test.elf-x86-64 differ
diff --git a/test/DebugInfo/Inputs/dwarfdump-test.elf-x86-64.debuglink b/test/DebugInfo/Inputs/dwarfdump-test.elf-x86-64.debuglink
index 8c08037ea3717323779d6bf12602355d851fe717..9b513215f2bcf7bc2a33db6d967612d86e4c7945 100755
Binary files a/test/DebugInfo/Inputs/dwarfdump-test.elf-x86-64.debuglink and b/test/DebugInfo/Inputs/dwarfdump-test.elf-x86-64.debuglink differ
diff --git a/test/DebugInfo/Inputs/dwarfdump-test3-decl.h b/test/DebugInfo/Inputs/dwarfdump-test3-decl.h
index 4a79e959b0ceb8fa7e71cf9daff027260843125d..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 100644
--- a/test/DebugInfo/Inputs/dwarfdump-test3-decl.h
+++ b/test/DebugInfo/Inputs/dwarfdump-test3-decl.h
@@ -1,7 +0,0 @@
-#include "dwarfdump-test3-decl2.h"
-
-class C {
-  explicit C(bool a = false, bool b = false);
-};
-
-void do1() {}
diff --git a/test/DebugInfo/Inputs/dwarfdump-test3-decl2.h b/test/DebugInfo/Inputs/dwarfdump-test3-decl2.h
index 9c92d56fcf4edfb3fa11f5585f59e723c34f35ae..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 100644
--- a/test/DebugInfo/Inputs/dwarfdump-test3-decl2.h
+++ b/test/DebugInfo/Inputs/dwarfdump-test3-decl2.h
@@ -1 +0,0 @@
-void do2() { }
diff --git a/test/DebugInfo/Inputs/dwarfdump-test3.cc b/test/DebugInfo/Inputs/dwarfdump-test3.cc
index 7b4d7ea711802f3111549eb5f5637e5b6125ec38..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 100644
--- a/test/DebugInfo/Inputs/dwarfdump-test3.cc
+++ b/test/DebugInfo/Inputs/dwarfdump-test3.cc
@@ -1,12 +0,0 @@
-#include "dwarfdump-test3-decl.h"
-
-C::C(bool a, bool b) {}
-
-// Built with gcc 4.6.3
-// $ mkdir -p /tmp/dbginfo/include
-// $ mkdir -p /tmp/include
-// $ cp dwarfdump-test3.cc /tmp/dbginfo
-// $ cp dwarfdump-test3-decl.h /tmp/include
-// $ cp dwarfdump-test3-decl2.h /tmp/dbginfo/include
-// $ cd /tmp/dbginfo
-// $ gcc dwarfdump-test3.cc -g -I/tmp/include -Iinclude -fPIC -shared -o <output>
diff --git a/test/DebugInfo/Inputs/llvm-symbolizer-test.c b/test/DebugInfo/Inputs/llvm-symbolizer-test.c
index 4c40c00c67360e1dfd7541ef6324def25c18c8e0..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 100644
--- a/test/DebugInfo/Inputs/llvm-symbolizer-test.c
+++ b/test/DebugInfo/Inputs/llvm-symbolizer-test.c
@@ -1,18 +0,0 @@
-int f(int a, int b) {
-  return a + b;
-}
-
-int g(int a) {
-  return a + 1;
-}
-
-
-int main() {
-  return f(2, g(2));
-}
-
-// Built with Clang 3.3:
-// $ mkdir -p /tmp/dbginfo
-// $ cp llvm-symbolizer-test.c /tmp/dbginfo
-// $ cd /tmp/dbginfo
-// $ clang -g llvm-symbolizer-test.c -o <output>
diff --git a/test/DebugInfo/Inputs/llvm-symbolizer-test.elf-x86-64 b/test/DebugInfo/Inputs/llvm-symbolizer-test.elf-x86-64
index 99a448a7778997c1fbae59a6d4dd975d341fa3ac..c952cff13dea72375b2e86f449d193220d5a7a24 100755
Binary files a/test/DebugInfo/Inputs/llvm-symbolizer-test.elf-x86-64 and b/test/DebugInfo/Inputs/llvm-symbolizer-test.elf-x86-64 differ
diff --git a/test/DebugInfo/Inputs/loclists-dwp-b.ll b/test/DebugInfo/Inputs/loclists-dwp-b.ll
index 77081bd7c2834a015e5c172a541fd9d2c0b32cf9..0976bf9dde8742cae8b2be33f17cb00c5de92bff 100644
--- a/test/DebugInfo/Inputs/loclists-dwp-b.ll
+++ b/test/DebugInfo/Inputs/loclists-dwp-b.ll
@@ -13,13 +13,13 @@ declare void @llvm.dbg.value(metadata, metadata, metadata)
 !llvm.module.flags = !{!3, !4, !5}
 !llvm.ident = !{!6}
 
-!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !1, producer: "clang version 8.0.0 (https://git.llvm.org/git/clang.git/ 41055c6168135fe539801799e5c5636247cf0302) (https://git.llvm.org/git/llvm.git/ de0558be123ffbb5b5bd692c17dbd57a75fe684f)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, nameTableKind: None)
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !1, producer: "clang version 8.0.0", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, nameTableKind: None)
 !1 = !DIFile(filename: "b.cpp", directory: "/home/test/PRs/PR38990")
 !2 = !{}
 !3 = !{i32 2, !"Dwarf Version", i32 4}
 !4 = !{i32 2, !"Debug Info Version", i32 3}
 !5 = !{i32 1, !"wchar_size", i32 4}
-!6 = !{!"clang version 8.0.0 (https://git.llvm.org/git/clang.git/ 41055c6168135fe539801799e5c5636247cf0302) (https://git.llvm.org/git/llvm.git/ de0558be123ffbb5b5bd692c17dbd57a75fe684f)"}
+!6 = !{!"clang version 8.0.0"}
 !7 = distinct !DISubprogram(name: "b", linkageName: "_Z1bi", scope: !1, file: !1, line: 1, type: !8, isLocal: false, isDefinition: true, scopeLine: 1, flags: DIFlagPrototyped, isOptimized: true, unit: !0, retainedNodes: !11)
 !8 = !DISubroutineType(types: !9)
 !9 = !{null, !10}
diff --git a/test/DebugInfo/Inputs/symbolize-64bit-addr.c b/test/DebugInfo/Inputs/symbolize-64bit-addr.c
new file mode 100644
index 0000000000000000000000000000000000000000..137ff054ed13708f5b2c12be82e9c54e80865958
--- /dev/null
+++ b/test/DebugInfo/Inputs/symbolize-64bit-addr.c
@@ -0,0 +1,6 @@
+int _start() {
+    return 0;
+}
+
+// Compile with "clang --target=x86_64-pc-linux -c -g symbolize-64bit-addr.c".
+// Link with "ld.lld -Ttext=0xffffffff00000000 symbolize-64bit-addr.o -o symbolize-64bit-addr.elf.x86_64".
diff --git a/test/DebugInfo/Inputs/symbolize-64bit-addr.elf.x86_64 b/test/DebugInfo/Inputs/symbolize-64bit-addr.elf.x86_64
new file mode 100755
index 0000000000000000000000000000000000000000..1f99b21296bd41217c90ff432fabd9bd9609f401
Binary files /dev/null and b/test/DebugInfo/Inputs/symbolize-64bit-addr.elf.x86_64 differ
diff --git a/test/DebugInfo/Inputs/test-inline.o b/test/DebugInfo/Inputs/test-inline.o
index a650c91725d9db5504757fd91472e301e59bf2b1..93073a19c5225c00afd9a8c9292ae6ca16bd00ef 100644
Binary files a/test/DebugInfo/Inputs/test-inline.o and b/test/DebugInfo/Inputs/test-inline.o differ
diff --git a/test/DebugInfo/Inputs/test-parameters.o b/test/DebugInfo/Inputs/test-parameters.o
index 7f4b6702df2e76c0c55d422b425972714b824276..3b887c45640d1b9b4d8cedf7de94febeba8df505 100644
Binary files a/test/DebugInfo/Inputs/test-parameters.o and b/test/DebugInfo/Inputs/test-parameters.o differ
diff --git a/test/DebugInfo/MIR/AArch64/clobber-sp.mir b/test/DebugInfo/MIR/AArch64/clobber-sp.mir
index 4594065cc2969c8cd081b7d88376876214c90891..13c5935d0cf3e92c38982550ea6298a966a8aba9 100644
--- a/test/DebugInfo/MIR/AArch64/clobber-sp.mir
+++ b/test/DebugInfo/MIR/AArch64/clobber-sp.mir
@@ -4,7 +4,7 @@
 # CHECK: DW_TAG_formal_parameter
 # CHECK: DW_TAG_formal_parameter
 # CHECK-NEXT: DW_AT_location
-# CHECK-NEXT:   [0x0000000000000000, 0x0000000000000014): DW_OP_reg1 W1
+# CHECK-NEXT:   [0x0000000000000010, 0x0000000000000014): DW_OP_reg1 W1
 # CHECK-NEXT:   [0x0000000000000014, 0x0000000000000038): DW_OP_breg31 WSP+8
 # CHECK-NEXT: DW_AT_name {{.*}}"y"
 
diff --git a/test/DebugInfo/MIR/X86/debug-loc-0.mir b/test/DebugInfo/MIR/X86/debug-loc-0.mir
new file mode 100644
index 0000000000000000000000000000000000000000..3fb1e0ba71ae67aef3a959d47ecb2ba805df1f8d
--- /dev/null
+++ b/test/DebugInfo/MIR/X86/debug-loc-0.mir
@@ -0,0 +1,104 @@
+# Check that subsequent .loc 0 0 directives are collapsed.
+
+# RUN: llc -start-before=machine-cp -O2 -filetype=asm -mtriple=x86_64-apple-macosx10.9.0 -o - %s | FileCheck %s
+
+# CHECK: Ltmp0:
+# CHECK: .loc 1 0 0 prologue_end
+# CHECK-NOT: .loc 1 0 0
+# CHECK: .loc 1 37 1
+
+--- |
+  ; ModuleID = '<stdin>'
+  source_filename = "<stdin>"
+  target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+  target triple = "x86_64-apple-macosx10.9.0"
+
+  %swift.opaque = type opaque
+  %swift.metadata_response = type { %swift.type*, i64 }
+  %swift.type = type { i64 }
+
+  define hidden swiftcc void @"$S4main1fyyF"() !dbg !5 {
+  entry:
+    %s1.addr = alloca i8*, align 8
+    %0 = bitcast i8** %s1.addr to %swift.opaque**
+    store %swift.opaque* null, %swift.opaque** %0, align 8
+    %1 = call swiftcc %swift.metadata_response @"$S16resilient_struct4SizeVMa"(i64 0) #1, !dbg !10
+    %2 = extractvalue %swift.metadata_response %1, 0, !dbg !10
+    %3 = bitcast %swift.type* %2 to i8***, !dbg !10
+    %4 = getelementptr inbounds i8**, i8*** %3, i64 -1, !dbg !10
+    ret void, !dbg !12
+  }
+
+  declare swiftcc %swift.metadata_response @"$S16resilient_struct4SizeVMa"(i64)
+
+  ; Function Attrs: nounwind
+  declare void @llvm.stackprotector(i8*, i8**) #0
+
+  attributes #0 = { nounwind }
+  attributes #1 = { nounwind readnone }
+
+  !llvm.dbg.cu = !{!0}
+  !llvm.module.flags = !{!3, !4}
+
+  !0 = distinct !DICompileUnit(language: DW_LANG_Swift, file: !1, producer: "Swift version 4.2.1-dev effective-3.4.1 (LLVM 4ba03d9389, Clang 78aa734eee, Swift f4134ebef6)", isOptimized: false, runtimeVersion: 4, emissionKind: FullDebug, enums: !2, imports: !2)
+  !1 = !DIFile(filename: "struct_resilience.swift", directory: "/Users/jonas/swift")
+  !2 = !{}
+  !3 = !{i32 2, !"Dwarf Version", i32 4}
+  !4 = !{i32 2, !"Debug Info Version", i32 3}
+  !5 = distinct !DISubprogram(name: "f", linkageName: "$S4main1fyyF", scope: !0, file: !6, line: 23, type: !7, scopeLine: 23, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !2)
+  !6 = !DIFile(filename: "struct_resilience.swift", directory: "/Users/jonas/swift/swift/test/DebugInfo")
+  !7 = !DISubroutineType(types: !8)
+  !8 = !{!9}
+  !9 = !DICompositeType(tag: DW_TAG_structure_type, name: "$SytD", file: !6, elements: !2, runtimeLang: DW_LANG_Swift, identifier: "$SytD")
+  !10 = !DILocation(line: 0, scope: !11)
+  !11 = distinct !DILexicalBlock(scope: !5, file: !6, line: 23, column: 10)
+  !12 = !DILocation(line: 37, column: 1, scope: !11)
+
+...
+---
+name:            '$S4main1fyyF'
+alignment:       4
+exposesReturnsTwice: false
+legalized:       false
+regBankSelected: false
+selected:        false
+failedISel:      false
+tracksRegLiveness: true
+hasWinCFI:       false
+registers:       []
+liveins:         []
+frameInfo:
+  isFrameAddressTaken: false
+  isReturnAddressTaken: false
+  hasStackMap:     false
+  hasPatchPoint:   false
+  stackSize:       0
+  offsetAdjustment: 0
+  maxAlignment:    8
+  adjustsStack:    false
+  hasCalls:        true
+  stackProtector:  ''
+  maxCallFrameSize: 4294967295
+  cvBytesOfCalleeSavedRegisters: 0
+  hasOpaqueSPAdjustment: false
+  hasVAStart:      false
+  hasMustTailInVarArgFunc: false
+  localFrameSize:  0
+  savePoint:       ''
+  restorePoint:    ''
+fixedStack:      []
+stack:
+  - { id: 0, name: s1.addr, type: default, offset: 0, size: 8, alignment: 8,
+      stack-id: 0, callee-saved-register: '', callee-saved-restored: true,
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+constants:       []
+body:             |
+  bb.0.entry:
+    MOV64mi32 %stack.0.s1.addr, 1, $noreg, 0, $noreg, 0 :: (store 8 into %ir.0)
+    ADJCALLSTACKDOWN64 0, 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp, debug-location !10
+    dead $edi = MOV32r0 implicit-def dead $eflags, implicit-def $rdi, debug-location !10
+    CALL64pcrel32 @"$S16resilient_struct4SizeVMa", csr_64, implicit $rsp, implicit $ssp, implicit killed $rdi, implicit-def $rsp, implicit-def $ssp, implicit-def dead $rax, implicit-def dead $rdx, debug-location !10
+    ADJCALLSTACKUP64 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp, debug-location !10
+    RET 0, debug-location !12
+
+...
diff --git a/test/DebugInfo/MIR/X86/live-debug-values-restore.mir b/test/DebugInfo/MIR/X86/live-debug-values-restore.mir
new file mode 100644
index 0000000000000000000000000000000000000000..626328198b4fad8c06333c8f3721766f2fe340d4
--- /dev/null
+++ b/test/DebugInfo/MIR/X86/live-debug-values-restore.mir
@@ -0,0 +1,189 @@
+# RUN: llc -run-pass livedebugvalues -march=x86-64 -o - %s | FileCheck %s
+
+# Generated from the following source with:
+# clang -g -mllvm -stop-before=livedebugvalues -S -O2 test.c -o test.mir
+
+# #define FORCE_SPILL() \
+#   __asm volatile("" : : : \
+#                    "rax", "rbx", "rcx", "rdx", "rsi", "rdi", "rbp", "r8", \
+#                    "r9", "r10", "r11", "r12", "r13", "r14", "r15")
+# int f(int *p) {
+#   if (p) {
+#     FORCE_SPILL();
+#   }
+#   return *(p + 1);
+# }
+
+# Ascertain that the spill has been recognized and manifested in a DBG_VALUE.
+# CHECK: MOV64mr $rsp,{{.*-8.*}}killed{{.*}}$rdi :: (store 8 into %stack.0)
+# CHECK-NEXT: DBG_VALUE $rsp,{{.*}}![[MDIX:[0-9]+]],{{.*}}!DIExpression(DW_OP_constu, 8, DW_OP_minus)
+
+# Check for the restore.
+# CHECK: $rdi = MOV64rm $rsp,{{.*-8.*}}:: (load 8 from %stack.0)
+# CHECK-NEXT: DBG_VALUE $rdi,{{.*}}![[MDIX]], !DIExpression()
+
+--- |
+  define dso_local i32 @f(i32* readonly %p) local_unnamed_addr !dbg !7 {
+  entry:
+    call void @llvm.dbg.value(metadata i32* %p, metadata !13, metadata !DIExpression()), !dbg !14
+    %tobool = icmp eq i32* %p, null, !dbg !15
+    br i1 %tobool, label %if.end, label %if.then, !dbg !17
+  
+  if.then:                                          ; preds = %entry
+    tail call void asm sideeffect "", "~{rax},~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{rbp},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15},~{dirflag},~{fpsr},~{flags}"(), !dbg !18, !srcloc !20
+    br label %if.end, !dbg !21
+  
+  if.end:                                           ; preds = %entry, %if.then
+    %add.ptr = getelementptr inbounds i32, i32* %p, i64 1, !dbg !22
+    %0 = load i32, i32* %add.ptr, align 4, !dbg !23, !tbaa !24
+    ret i32 %0, !dbg !28
+  }
+  
+  declare void @llvm.dbg.value(metadata, metadata, metadata)
+  
+  !llvm.dbg.cu = !{!0}
+  !llvm.module.flags = !{!3, !4, !5}
+  !llvm.ident = !{!6}
+  
+  !0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 9.0.0 (https://git.llvm.org/git/clang.git/ 57a6ce7ac318de98e3e777e09cb9ed8282b5cc03) (https://git.llvm.org/git/llvm.git/ ff54a19e4912d7f15cb02798a7f2048441bff751)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, nameTableKind: None)
+  !1 = !DIFile(filename: "test2.c", directory: "/home/test")
+  !2 = !{}
+  !3 = !{i32 2, !"Dwarf Version", i32 4}
+  !4 = !{i32 2, !"Debug Info Version", i32 3}
+  !5 = !{i32 1, !"wchar_size", i32 4}
+  !6 = !{!"clang version 9.0.0 (https://git.llvm.org/git/clang.git/ 57a6ce7ac318de98e3e777e09cb9ed8282b5cc03) (https://git.llvm.org/git/llvm.git/ ff54a19e4912d7f15cb02798a7f2048441bff751)"}
+  !7 = distinct !DISubprogram(name: "f", scope: !1, file: !1, line: 5, type: !8, scopeLine: 5, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !12)
+  !8 = !DISubroutineType(types: !9)
+  !9 = !{!10, !11}
+  !10 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+  !11 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !10, size: 64)
+  !12 = !{!13}
+  !13 = !DILocalVariable(name: "p", arg: 1, scope: !7, file: !1, line: 5, type: !11)
+  !14 = !DILocation(line: 5, column: 12, scope: !7)
+  !15 = !DILocation(line: 6, column: 7, scope: !16)
+  !16 = distinct !DILexicalBlock(scope: !7, file: !1, line: 6, column: 7)
+  !17 = !DILocation(line: 6, column: 7, scope: !7)
+  !18 = !DILocation(line: 7, column: 5, scope: !19)
+  !19 = distinct !DILexicalBlock(scope: !16, file: !1, line: 6, column: 10)
+  !20 = !{i32 -2147471544}
+  !21 = !DILocation(line: 8, column: 3, scope: !19)
+  !22 = !DILocation(line: 9, column: 14, scope: !7)
+  !23 = !DILocation(line: 9, column: 10, scope: !7)
+  !24 = !{!25, !25, i64 0}
+  !25 = !{!"int", !26, i64 0}
+  !26 = !{!"omnipotent char", !27, i64 0}
+  !27 = !{!"Simple C/C++ TBAA"}
+  !28 = !DILocation(line: 9, column: 3, scope: !7)
+
+...
+---
+name:            f
+alignment:       4
+exposesReturnsTwice: false
+legalized:       false
+regBankSelected: false
+selected:        false
+failedISel:      false
+tracksRegLiveness: true
+hasWinCFI:       false
+registers:       []
+liveins:         
+  - { reg: '$rdi', virtual-reg: '' }
+frameInfo:       
+  isFrameAddressTaken: false
+  isReturnAddressTaken: false
+  hasStackMap:     false
+  hasPatchPoint:   false
+  stackSize:       48
+  offsetAdjustment: -48
+  maxAlignment:    8
+  adjustsStack:    false
+  hasCalls:        false
+  stackProtector:  ''
+  maxCallFrameSize: 0
+  cvBytesOfCalleeSavedRegisters: 48
+  hasOpaqueSPAdjustment: false
+  hasVAStart:      false
+  hasMustTailInVarArgFunc: false
+  localFrameSize:  0
+  savePoint:       ''
+  restorePoint:    ''
+fixedStack:      
+  - { id: 0, type: spill-slot, offset: -56, size: 8, alignment: 8, stack-id: 0, 
+      callee-saved-register: '$rbx', callee-saved-restored: true, debug-info-variable: '', 
+      debug-info-expression: '', debug-info-location: '' }
+  - { id: 1, type: spill-slot, offset: -48, size: 8, alignment: 16, stack-id: 0, 
+      callee-saved-register: '$r12', callee-saved-restored: true, debug-info-variable: '', 
+      debug-info-expression: '', debug-info-location: '' }
+  - { id: 2, type: spill-slot, offset: -40, size: 8, alignment: 8, stack-id: 0, 
+      callee-saved-register: '$r13', callee-saved-restored: true, debug-info-variable: '', 
+      debug-info-expression: '', debug-info-location: '' }
+  - { id: 3, type: spill-slot, offset: -32, size: 8, alignment: 16, stack-id: 0, 
+      callee-saved-register: '$r14', callee-saved-restored: true, debug-info-variable: '', 
+      debug-info-expression: '', debug-info-location: '' }
+  - { id: 4, type: spill-slot, offset: -24, size: 8, alignment: 8, stack-id: 0, 
+      callee-saved-register: '$r15', callee-saved-restored: true, debug-info-variable: '', 
+      debug-info-expression: '', debug-info-location: '' }
+  - { id: 5, type: spill-slot, offset: -16, size: 8, alignment: 16, stack-id: 0, 
+      callee-saved-register: '$rbp', callee-saved-restored: true, debug-info-variable: '', 
+      debug-info-expression: '', debug-info-location: '' }
+stack:           
+  - { id: 0, name: '', type: spill-slot, offset: -64, size: 8, alignment: 8, 
+      stack-id: 0, callee-saved-register: '', callee-saved-restored: true, 
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+constants:       []
+body:             |
+  bb.0.entry:
+    successors: %bb.2(0x30000000), %bb.1(0x50000000)
+    liveins: $rdi, $rbx, $r12, $r13, $r14, $r15, $rbp
+  
+    DBG_VALUE $rdi, $noreg, !13, !DIExpression(), debug-location !14
+    DBG_VALUE $rdi, $noreg, !13, !DIExpression(), debug-location !14
+    TEST64rr renamable $rdi, renamable $rdi, implicit-def $eflags, debug-location !15
+    JE_1 %bb.2, implicit $eflags, debug-location !17
+  
+  bb.1.if.then:
+    successors: %bb.2(0x80000000)
+    liveins: $rdi, $rbp, $r15, $r14, $r13, $r12, $rbx
+  
+    frame-setup PUSH64r killed $rbp, implicit-def $rsp, implicit $rsp
+    CFI_INSTRUCTION def_cfa_offset 16
+    frame-setup PUSH64r killed $r15, implicit-def $rsp, implicit $rsp
+    CFI_INSTRUCTION def_cfa_offset 24
+    frame-setup PUSH64r killed $r14, implicit-def $rsp, implicit $rsp
+    CFI_INSTRUCTION def_cfa_offset 32
+    frame-setup PUSH64r killed $r13, implicit-def $rsp, implicit $rsp
+    CFI_INSTRUCTION def_cfa_offset 40
+    frame-setup PUSH64r killed $r12, implicit-def $rsp, implicit $rsp
+    CFI_INSTRUCTION def_cfa_offset 48
+    frame-setup PUSH64r killed $rbx, implicit-def $rsp, implicit $rsp
+    CFI_INSTRUCTION def_cfa_offset 56
+    CFI_INSTRUCTION offset $rbx, -56
+    CFI_INSTRUCTION offset $r12, -48
+    CFI_INSTRUCTION offset $r13, -40
+    CFI_INSTRUCTION offset $r14, -32
+    CFI_INSTRUCTION offset $r15, -24
+    CFI_INSTRUCTION offset $rbp, -16
+    MOV64mr $rsp, 1, $noreg, -8, $noreg, killed renamable $rdi :: (store 8 into %stack.0)
+    INLINEASM &"", 1, 12, implicit-def dead early-clobber $rax, 12, implicit-def dead early-clobber $rbx, 12, implicit-def dead early-clobber $rcx, 12, implicit-def dead early-clobber $rdx, 12, implicit-def dead early-clobber $rsi, 12, implicit-def dead early-clobber $rdi, 12, implicit-def dead early-clobber $rbp, 12, implicit-def dead early-clobber $r8, 12, implicit-def dead early-clobber $r9, 12, implicit-def dead early-clobber $r10, 12, implicit-def dead early-clobber $r11, 12, implicit-def dead early-clobber $r12, 12, implicit-def dead early-clobber $r13, 12, implicit-def dead early-clobber $r14, 12, implicit-def dead early-clobber $r15, 12, implicit-def dead early-clobber $eflags, !20, debug-location !18
+    renamable $rdi = MOV64rm $rsp, 1, $noreg, -8, $noreg :: (load 8 from %stack.0)
+    $rbx = frame-destroy POP64r implicit-def $rsp, implicit $rsp
+    CFI_INSTRUCTION def_cfa_offset 48
+    $r12 = frame-destroy POP64r implicit-def $rsp, implicit $rsp
+    CFI_INSTRUCTION def_cfa_offset 40
+    $r13 = frame-destroy POP64r implicit-def $rsp, implicit $rsp
+    CFI_INSTRUCTION def_cfa_offset 32
+    $r14 = frame-destroy POP64r implicit-def $rsp, implicit $rsp
+    CFI_INSTRUCTION def_cfa_offset 24
+    $r15 = frame-destroy POP64r implicit-def $rsp, implicit $rsp
+    CFI_INSTRUCTION def_cfa_offset 16
+    $rbp = frame-destroy POP64r implicit-def $rsp, implicit $rsp
+    CFI_INSTRUCTION def_cfa_offset 8
+  
+  bb.2.if.end:
+    liveins: $rdi, $rbx, $r12, $r13, $r14, $r15, $rbp
+  
+    renamable $eax = MOV32rm killed renamable $rdi, 1, $noreg, 4, $noreg, debug-location !23 :: (load 4 from %ir.add.ptr, !tbaa !24)
+    RETQ $eax, debug-location !28
+
+...
diff --git a/test/DebugInfo/Mips/dwarfdump-tls.ll b/test/DebugInfo/Mips/dwarfdump-tls.ll
index 6aa429adb417b1d53d9dc558839e4425683d88fe..8d8af8c5124a9fe24a5b0e88aad889d7572caba7 100644
--- a/test/DebugInfo/Mips/dwarfdump-tls.ll
+++ b/test/DebugInfo/Mips/dwarfdump-tls.ll
@@ -1,12 +1,34 @@
-; RUN: llc -O0 -march=mips -mcpu=mips32r2 -filetype=obj -o=%t-32.o < %s
+; RUN: llc -O0 -march=mips -mcpu=mips32r2 -filetype=obj \
+; RUN:     -split-dwarf-file=foo.dwo -o=%t-32.o < %s
 ; RUN: llvm-dwarfdump %t-32.o 2>&1 | FileCheck %s
-; RUN: llc -O0 -march=mips64 -mcpu=mips64r2 -filetype=obj -o=%t-64.o < %s
+; RUN: llc -O0 -march=mips64 -mcpu=mips64r2 -filetype=obj \
+; RUN:     -split-dwarf-file=foo.dwo -o=%t-64.o < %s
 ; RUN: llvm-dwarfdump %t-64.o 2>&1 | FileCheck %s
 
+; RUN: llc -O0 -march=mips -mcpu=mips32r2 -filetype=asm \
+; RUN:     -split-dwarf-file=foo.dwo < %s | FileCheck -check-prefix=ASM32 %s
+; RUN: llc -O0 -march=mips64 -mcpu=mips64r2 -filetype=asm \
+; RUN:     -split-dwarf-file=foo.dwo < %s | FileCheck -check-prefix=ASM64 %s
+
 @x = thread_local global i32 5, align 4, !dbg !0
 
 ; CHECK-NOT: error: failed to compute relocation: R_MIPS_TLS_DTPREL
 
+; CHECK:      DW_AT_name      ("x")
+; CHECK-NEXT: DW_AT_type      (0x00000025 "int")
+; CHECK-NEXT: DW_AT_external  (true)
+; CHECK-NEXT: DW_AT_decl_file (0x01)
+; CHECK-NEXT: DW_AT_decl_line (1)
+; CHECK-NEXT: DW_AT_location  (DW_OP_GNU_const_index 0x0, {{DW_OP_GNU_push_tls_address|DW_OP_form_tls_address}})
+
+; ASM32:              .section        .debug_addr
+; ASM32-NEXT: $addr_table_base0:
+; ASM32-NEXT:         .4byte  x+32768
+
+; ASM64:              .section        .debug_addr
+; ASM64-NEXT: .Laddr_table_base0:
+; ASM64-NEXT:         .8byte  x+32768
+
 !llvm.dbg.cu = !{!2}
 !llvm.module.flags = !{!7, !8}
 
diff --git a/test/DebugInfo/Mips/eh_frame.ll b/test/DebugInfo/Mips/eh_frame.ll
index 4687443cb1cff294e18c43d1bae4bae0cce47385..122d0a7f6ab2abad2503ecab0885e48850c83151 100644
--- a/test/DebugInfo/Mips/eh_frame.ll
+++ b/test/DebugInfo/Mips/eh_frame.ll
@@ -1,9 +1,21 @@
-; RUN: llc -mtriple mips-unknown-linux-gnu -mattr=+micromips -O3 -filetype=obj -o - %s | llvm-readelf -r | FileCheck %s
-
-; CHECK: .rel.eh_frame
-; CHECK: DW.ref.__gxx_personality_v0
-; CHECK-NEXT: .text
-; CHECK-NEXT: .gcc_except_table
+; RUN: llc -mtriple mips-unknown-linux-gnu -mattr=+micromips -relocation-model=static -O3 -filetype=obj -o - %s | \
+; RUN:     llvm-readelf -r | FileCheck %s --check-prefix=CHECK-READELF
+; RUN: llc -mtriple mips-unknown-linux-gnu -mattr=+micromips -relocation-model=pic -O3 -filetype=obj -o - %s | \
+; RUN:     llvm-readelf -r | FileCheck %s --check-prefix=CHECK-READELF
+; RUN: llc -mtriple mips-unknown-linux-gnu -mattr=+micromips -relocation-model=static -O3 -filetype=obj -o - %s | \
+; RUN:     llvm-objdump -s -j .gcc_except_table - | FileCheck %s --check-prefix=CHECK-EXCEPT-TABLE-STATIC
+; RUN: llc -mtriple mips-unknown-linux-gnu -mattr=+micromips -relocation-model=pic -O3 -filetype=obj -o - %s | \
+; RUN:     llvm-objdump -s -j .gcc_except_table - | FileCheck %s --check-prefix=CHECK-EXCEPT-TABLE-PIC
+
+; CHECK-READELF: .rel.eh_frame
+; CHECK-READELF: DW.ref.__gxx_personality_v0
+; CHECK-READELF-NEXT: .text
+; CHECK-READELF-NEXT: .gcc_except_table
+
+; CHECK-EXCEPT-TABLE-STATIC: 0000 ff9b1501 0c011500 00150e23 01231e00  ...........#.#..
+; CHECK-EXCEPT-TABLE-STATIC: 0010 00010000 00000000
+; CHECK-EXCEPT-TABLE-PIC:    0000 ff9b1501 0c012d00 002d133f 013f2a00 ......-..-.?.?*.
+; CHECK-EXCEPT-TABLE-PIC:    0010 00010000 00000000                    ........
 
 @_ZTIi = external constant i8*
 
diff --git a/test/DebugInfo/Mips/prologue_end.ll b/test/DebugInfo/Mips/prologue_end.ll
index 6bb0448a122f80cd0ca0f57b19c3a3f7ca7b48eb..7886b9b0485f4aa5f290ec708c1069528fe1aaa3 100644
--- a/test/DebugInfo/Mips/prologue_end.ll
+++ b/test/DebugInfo/Mips/prologue_end.ll
@@ -1,7 +1,7 @@
 ; RUN: llc -O0 -mtriple mips-unknown-linux-gnu -relocation-model=static < %s | FileCheck %s -check-prefix=STATIC
-; RUN: llc -O0 -mtriple mips-unknown-linux-gnu -relocation-model=static -disable-fp-elim < %s | FileCheck %s -check-prefix=STATIC-FP
+; RUN: llc -O0 -mtriple mips-unknown-linux-gnu -relocation-model=static -frame-pointer=all < %s | FileCheck %s -check-prefix=STATIC-FP
 ; RUN: llc -O0 -mtriple mips-unknown-linux-gnu -relocation-model=pic < %s | FileCheck %s -check-prefix=PIC
-; RUN: llc -O0 -mtriple mips-unknown-linux-gnu -relocation-model=pic -disable-fp-elim < %s | FileCheck %s -check-prefix=PIC-FP
+; RUN: llc -O0 -mtriple mips-unknown-linux-gnu -relocation-model=pic -frame-pointer=all < %s | FileCheck %s -check-prefix=PIC-FP
 
 ; Generated using clang -O0 -emit-llvm -S -target mipsel-unknown-linux -g test.c -o test.ll
 ; test.c:
diff --git a/test/DebugInfo/NVPTX/cu-range-hole.ll b/test/DebugInfo/NVPTX/cu-range-hole.ll
index c8ea509396a943fb7ab74d58417f479936a92f1c..ccc5c1201fda2563a3760cd45ccc61082348e383 100644
--- a/test/DebugInfo/NVPTX/cu-range-hole.ll
+++ b/test/DebugInfo/NVPTX/cu-range-hole.ll
@@ -1,11 +1,12 @@
 ; RUN: llc < %s -mtriple=nvptx64-nvidia-cuda | FileCheck %s
 
-; CHECK: .target sm_{{[0-9]+}}//, debug
+; CHECK: .target sm_{{[0-9]+}}, debug
 
 ; CHECK: .visible .func  (.param .b32 func_retval0) b(
 ; CHECK: .param .b32 b_param_0
 ; CHECK: )
 ; CHECK: {
+; CHECK: .loc 1 1 0
 ; CHECK: Lfunc_begin0:
 ; CHECK: .loc 1 1 0
 ; CHECK: .loc 1 1 0
@@ -27,6 +28,7 @@
 ; CHECK: .param .b32 d_param_0
 ; CHECK: )
 ; CHECK: {
+; CHECK: .loc 1 3 0
 ; CHECK: Lfunc_begin2:
 ; CHECK: .loc 1 3 0
 ; CHECK: ret;
@@ -70,145 +72,145 @@ entry:
   ret i32 %add, !dbg !16
 }
 
-; CHECK: // .section .debug_abbrev
-; CHECK: // {
-; CHECK: // .b8 1                                // Abbreviation Code
-; CHECK: // .b8 17                               // DW_TAG_compile_unit
-; CHECK: // .b8 1                                // DW_CHILDREN_yes
-; CHECK: // .b8 37                               // DW_AT_producer
-; CHECK: // .b8 8                                // DW_FORM_string
-; CHECK: // .b8 19                               // DW_AT_language
-; CHECK: // .b8 5                                // DW_FORM_data2
-; CHECK: // .b8 3                                // DW_AT_name
-; CHECK: // .b8 8                                // DW_FORM_string
-; CHECK: // .b8 16                               // DW_AT_stmt_list
-; CHECK: // .b8 6                                // DW_FORM_data4
-; CHECK: // .b8 27                               // DW_AT_comp_dir
-; CHECK: // .b8 8                                // DW_FORM_string
-; CHECK: // .b8 17                               // DW_AT_low_pc
-; CHECK: // .b8 1                                // DW_FORM_addr
-; CHECK: // .b8 18                               // DW_AT_high_pc
-; CHECK: // .b8 1                                // DW_FORM_addr
-; CHECK: // .b8 0                                // EOM(1)
-; CHECK: // .b8 0                                // EOM(2)
-; CHECK: // .b8 2                                // Abbreviation Code
-; CHECK: // .b8 46                               // DW_TAG_subprogram
-; CHECK: // .b8 1                                // DW_CHILDREN_yes
-; CHECK: // .b8 17                               // DW_AT_low_pc
-; CHECK: // .b8 1                                // DW_FORM_addr
-; CHECK: // .b8 18                               // DW_AT_high_pc
-; CHECK: // .b8 1                                // DW_FORM_addr
-; CHECK: // .b8 64                               // DW_AT_frame_base
-; CHECK: // .b8 10                               // DW_FORM_block1
-; CHECK: // .b8 3                                // DW_AT_name
-; CHECK: // .b8 8                                // DW_FORM_string
-; CHECK: // .b8 58                               // DW_AT_decl_file
-; CHECK: // .b8 11                               // DW_FORM_data1
-; CHECK: // .b8 59                               // DW_AT_decl_line
-; CHECK: // .b8 11                               // DW_FORM_data1
-; CHECK: // .b8 39                               // DW_AT_prototyped
-; CHECK: // .b8 12                               // DW_FORM_flag
-; CHECK: // .b8 73                               // DW_AT_type
-; CHECK: // .b8 19                               // DW_FORM_ref4
-; CHECK: // .b8 63                               // DW_AT_external
-; CHECK: // .b8 12                               // DW_FORM_flag
-; CHECK: // .b8 0                                // EOM(1)
-; CHECK: // .b8 0                                // EOM(2)
-; CHECK: // .b8 3                                // Abbreviation Code
-; CHECK: // .b8 5                                // DW_TAG_formal_parameter
-; CHECK: // .b8 0                                // DW_CHILDREN_no
-; CHECK: // .b8 3                                // DW_AT_name
-; CHECK: // .b8 8                                // DW_FORM_string
-; CHECK: // .b8 58                               // DW_AT_decl_file
-; CHECK: // .b8 11                               // DW_FORM_data1
-; CHECK: // .b8 59                               // DW_AT_decl_line
-; CHECK: // .b8 11                               // DW_FORM_data1
-; CHECK: // .b8 73                               // DW_AT_type
-; CHECK: // .b8 19                               // DW_FORM_ref4
-; CHECK: // .b8 0                                // EOM(1)
-; CHECK: // .b8 0                                // EOM(2)
-; CHECK: // .b8 4                                // Abbreviation Code
-; CHECK: // .b8 36                               // DW_TAG_base_type
-; CHECK: // .b8 0                                // DW_CHILDREN_no
-; CHECK: // .b8 3                                // DW_AT_name
-; CHECK: // .b8 8                                // DW_FORM_string
-; CHECK: // .b8 62                               // DW_AT_encoding
-; CHECK: // .b8 11                               // DW_FORM_data1
-; CHECK: // .b8 11                               // DW_AT_byte_size
-; CHECK: // .b8 11                               // DW_FORM_data1
-; CHECK: // .b8 0                                // EOM(1)
-; CHECK: // .b8 0                                // EOM(2)
-; CHECK: // .b8 0                                // EOM(3)
-; CHECK: // }
-; CHECK: // .section .debug_info
-; CHECK: // {
-; CHECK: // .b32 183                             // Length of Unit
-; CHECK: // .b8 2                                // DWARF version number
-; CHECK: // .b8 0
-; CHECK: // .b32 .debug_abbrev                   // Offset Into Abbrev. Section
-; CHECK: // .b8 8                                // Address Size (in bytes)
-; CHECK: // .b8 1                                // Abbrev [1] 0xb:0xb0 DW_TAG_compile_unit
-; CHECK: // .b8 99,108,97,110,103,32,118,101,114,115,105,111,110,32,51,46,53,46,48,32,40,116,114,117,110,107,32,50,48,52,49,54,52,41,32,40,108,108,118,109 // DW_AT_producer
-; CHECK: // .b8 47,116,114,117,110,107,32,50,48,52,49,56,51,41
-; CHECK: // .b8 0
-; CHECK: // .b8 12                               // DW_AT_language
-; CHECK: // .b8 0
-; CHECK: // .b8 98,46,99                         // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b32 .debug_line                     // DW_AT_stmt_list
-; CHECK: // .b8 47,115,111,117,114,99,101        // DW_AT_comp_dir
-; CHECK: // .b8 0
-; CHECK: // .b64 Lfunc_begin0                    // DW_AT_low_pc
-; CHECK: // .b64 Lfunc_end2                      // DW_AT_high_pc
-; CHECK: // .b8 2                                // Abbrev [2] 0x65:0x27 DW_TAG_subprogram
-; CHECK: // .b64 Lfunc_begin0                    // DW_AT_low_pc
-; CHECK: // .b64 Lfunc_end0                      // DW_AT_high_pc
-; CHECK: // .b8 1                                // DW_AT_frame_base
-; CHECK: // .b8 156
-; CHECK: // .b8 98                               // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 1                                // DW_AT_decl_line
-; CHECK: // .b8 1                                // DW_AT_prototyped
-; CHECK: // .b32 179                             // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_external
-; CHECK: // .b8 3                                // Abbrev [3] 0x82:0x9 DW_TAG_formal_parameter
-; CHECK: // .b8 99                               // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 1                                // DW_AT_decl_line
-; CHECK: // .b32 179                             // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 2                                // Abbrev [2] 0x8c:0x27 DW_TAG_subprogram
-; CHECK: // .b64 Lfunc_begin2                    // DW_AT_low_pc
-; CHECK: // .b64 Lfunc_end2                      // DW_AT_high_pc
-; CHECK: // .b8 1                                // DW_AT_frame_base
-; CHECK: // .b8 156
-; CHECK: // .b8 100                              // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 3                                // DW_AT_decl_line
-; CHECK: // .b8 1                                // DW_AT_prototyped
-; CHECK: // .b32 179                             // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_external
-; CHECK: // .b8 3                                // Abbrev [3] 0xa9:0x9 DW_TAG_formal_parameter
-; CHECK: // .b8 101                              // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 3                                // DW_AT_decl_line
-; CHECK: // .b32 179                             // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 4                                // Abbrev [4] 0xb3:0x7 DW_TAG_base_type
-; CHECK: // .b8 105,110,116                      // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 5                                // DW_AT_encoding
-; CHECK: // .b8 4                                // DW_AT_byte_size
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // }
-; CHECK: // .section .debug_macinfo
-; CHECK: // {
-; CHECK: // .b8 0                                // End Of Macro List Mark
-; CHECK: // }
+; CHECK: .section .debug_abbrev
+; CHECK-NEXT: {
+; CHECK-NEXT: .b8 1                                // Abbreviation Code
+; CHECK-NEXT: .b8 17                               // DW_TAG_compile_unit
+; CHECK-NEXT: .b8 1                                // DW_CHILDREN_yes
+; CHECK-NEXT: .b8 37                               // DW_AT_producer
+; CHECK-NEXT: .b8 8                                // DW_FORM_string
+; CHECK-NEXT: .b8 19                               // DW_AT_language
+; CHECK-NEXT: .b8 5                                // DW_FORM_data2
+; CHECK-NEXT: .b8 3                                // DW_AT_name
+; CHECK-NEXT: .b8 8                                // DW_FORM_string
+; CHECK-NEXT: .b8 16                               // DW_AT_stmt_list
+; CHECK-NEXT: .b8 6                                // DW_FORM_data4
+; CHECK-NEXT: .b8 27                               // DW_AT_comp_dir
+; CHECK-NEXT: .b8 8                                // DW_FORM_string
+; CHECK-NEXT: .b8 17                               // DW_AT_low_pc
+; CHECK-NEXT: .b8 1                                // DW_FORM_addr
+; CHECK-NEXT: .b8 18                               // DW_AT_high_pc
+; CHECK-NEXT: .b8 1                                // DW_FORM_addr
+; CHECK-NEXT: .b8 0                                // EOM(1)
+; CHECK-NEXT: .b8 0                                // EOM(2)
+; CHECK-NEXT: .b8 2                                // Abbreviation Code
+; CHECK-NEXT: .b8 46                               // DW_TAG_subprogram
+; CHECK-NEXT: .b8 1                                // DW_CHILDREN_yes
+; CHECK-NEXT: .b8 17                               // DW_AT_low_pc
+; CHECK-NEXT: .b8 1                                // DW_FORM_addr
+; CHECK-NEXT: .b8 18                               // DW_AT_high_pc
+; CHECK-NEXT: .b8 1                                // DW_FORM_addr
+; CHECK-NEXT: .b8 64                               // DW_AT_frame_base
+; CHECK-NEXT: .b8 10                               // DW_FORM_block1
+; CHECK-NEXT: .b8 3                                // DW_AT_name
+; CHECK-NEXT: .b8 8                                // DW_FORM_string
+; CHECK-NEXT: .b8 58                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 11                               // DW_FORM_data1
+; CHECK-NEXT: .b8 59                               // DW_AT_decl_line
+; CHECK-NEXT: .b8 11                               // DW_FORM_data1
+; CHECK-NEXT: .b8 39                               // DW_AT_prototyped
+; CHECK-NEXT: .b8 12                               // DW_FORM_flag
+; CHECK-NEXT: .b8 73                               // DW_AT_type
+; CHECK-NEXT: .b8 19                               // DW_FORM_ref4
+; CHECK-NEXT: .b8 63                               // DW_AT_external
+; CHECK-NEXT: .b8 12                               // DW_FORM_flag
+; CHECK-NEXT: .b8 0                                // EOM(1)
+; CHECK-NEXT: .b8 0                                // EOM(2)
+; CHECK-NEXT: .b8 3                                // Abbreviation Code
+; CHECK-NEXT: .b8 5                                // DW_TAG_formal_parameter
+; CHECK-NEXT: .b8 0                                // DW_CHILDREN_no
+; CHECK-NEXT: .b8 3                                // DW_AT_name
+; CHECK-NEXT: .b8 8                                // DW_FORM_string
+; CHECK-NEXT: .b8 58                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 11                               // DW_FORM_data1
+; CHECK-NEXT: .b8 59                               // DW_AT_decl_line
+; CHECK-NEXT: .b8 11                               // DW_FORM_data1
+; CHECK-NEXT: .b8 73                               // DW_AT_type
+; CHECK-NEXT: .b8 19                               // DW_FORM_ref4
+; CHECK-NEXT: .b8 0                                // EOM(1)
+; CHECK-NEXT: .b8 0                                // EOM(2)
+; CHECK-NEXT: .b8 4                                // Abbreviation Code
+; CHECK-NEXT: .b8 36                               // DW_TAG_base_type
+; CHECK-NEXT: .b8 0                                // DW_CHILDREN_no
+; CHECK-NEXT: .b8 3                                // DW_AT_name
+; CHECK-NEXT: .b8 8                                // DW_FORM_string
+; CHECK-NEXT: .b8 62                               // DW_AT_encoding
+; CHECK-NEXT: .b8 11                               // DW_FORM_data1
+; CHECK-NEXT: .b8 11                               // DW_AT_byte_size
+; CHECK-NEXT: .b8 11                               // DW_FORM_data1
+; CHECK-NEXT: .b8 0                                // EOM(1)
+; CHECK-NEXT: .b8 0                                // EOM(2)
+; CHECK-NEXT: .b8 0                                // EOM(3)
+; CHECK-NEXT: }
+; CHECK-NEXT: .section .debug_info
+; CHECK-NEXT: {
+; CHECK-NEXT: .b32 183                             // Length of Unit
+; CHECK-NEXT: .b8 2                                // DWARF version number
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b32 .debug_abbrev                   // Offset Into Abbrev. Section
+; CHECK-NEXT: .b8 8                                // Address Size (in bytes)
+; CHECK-NEXT: .b8 1                                // Abbrev [1] 0xb:0xb0 DW_TAG_compile_unit
+; CHECK-NEXT: .b8 99,108,97,110,103,32,118,101,114,115,105,111,110,32,51,46,53,46,48,32,40,116,114,117,110,107,32,50,48,52,49,54,52,41,32,40,108,108,118,109 // DW_AT_producer
+; CHECK-NEXT: .b8 47,116,114,117,110,107,32,50,48,52,49,56,51,41
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 12                               // DW_AT_language
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 98,46,99                         // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b32 .debug_line                     // DW_AT_stmt_list
+; CHECK-NEXT: .b8 47,115,111,117,114,99,101        // DW_AT_comp_dir
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b64 Lfunc_begin0                    // DW_AT_low_pc
+; CHECK-NEXT: .b64 Lfunc_end2                      // DW_AT_high_pc
+; CHECK-NEXT: .b8 2                                // Abbrev [2] 0x65:0x27 DW_TAG_subprogram
+; CHECK-NEXT: .b64 Lfunc_begin0                    // DW_AT_low_pc
+; CHECK-NEXT: .b64 Lfunc_end0                      // DW_AT_high_pc
+; CHECK-NEXT: .b8 1                                // DW_AT_frame_base
+; CHECK-NEXT: .b8 156
+; CHECK-NEXT: .b8 98                               // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_line
+; CHECK-NEXT: .b8 1                                // DW_AT_prototyped
+; CHECK-NEXT: .b32 179                             // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_external
+; CHECK-NEXT: .b8 3                                // Abbrev [3] 0x82:0x9 DW_TAG_formal_parameter
+; CHECK-NEXT: .b8 99                               // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_line
+; CHECK-NEXT: .b32 179                             // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 2                                // Abbrev [2] 0x8c:0x27 DW_TAG_subprogram
+; CHECK-NEXT: .b64 Lfunc_begin2                    // DW_AT_low_pc
+; CHECK-NEXT: .b64 Lfunc_end2                      // DW_AT_high_pc
+; CHECK-NEXT: .b8 1                                // DW_AT_frame_base
+; CHECK-NEXT: .b8 156
+; CHECK-NEXT: .b8 100                              // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 3                                // DW_AT_decl_line
+; CHECK-NEXT: .b8 1                                // DW_AT_prototyped
+; CHECK-NEXT: .b32 179                             // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_external
+; CHECK-NEXT: .b8 3                                // Abbrev [3] 0xa9:0x9 DW_TAG_formal_parameter
+; CHECK-NEXT: .b8 101                              // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 3                                // DW_AT_decl_line
+; CHECK-NEXT: .b32 179                             // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 4                                // Abbrev [4] 0xb3:0x7 DW_TAG_base_type
+; CHECK-NEXT: .b8 105,110,116                      // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 5                                // DW_AT_encoding
+; CHECK-NEXT: .b8 4                                // DW_AT_byte_size
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: }
+; CHECK-NEXT: .section .debug_macinfo
+; CHECK-NEXT: {
+; CHECK-NEXT: .b8 0                                // End Of Macro List Mark
+; CHECK:      }
 
 attributes #0 = { nounwind uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 attributes #1 = { nounwind readnone }
diff --git a/test/DebugInfo/NVPTX/dbg-declare-alloca.ll b/test/DebugInfo/NVPTX/dbg-declare-alloca.ll
index 9a4beed23d7d8150636393b5114bb19fbdbab2b1..a6a9826d6c3436f826468d2b4cb230b28575b87e 100644
--- a/test/DebugInfo/NVPTX/dbg-declare-alloca.ll
+++ b/test/DebugInfo/NVPTX/dbg-declare-alloca.ll
@@ -1,6 +1,6 @@
 ; RUN: llc < %s -mtriple=nvptx64-nvidia-cuda | FileCheck %s
 
-; CHECK: .target sm_20//, debug
+; CHECK: .target sm_20, debug
 
 ; CHECK: .visible .func use_dbg_declare()
 ; CHECK: .local .align 8 .b8 __local_depot0[8];
@@ -23,169 +23,172 @@
 
 ; CHECK: .file 1 "test{{(/|\\\\)}}t.c"
 
-; CHECK: // .section .debug_abbrev
-; CHECK: // {
-; CHECK: // .b8 1                                // Abbreviation Code
-; CHECK: // .b8 17                               // DW_TAG_compile_unit
-; CHECK: // .b8 1                                // DW_CHILDREN_yes
-; CHECK: // .b8 37                               // DW_AT_producer
-; CHECK: // .b8 8                                // DW_FORM_string
-; CHECK: // .b8 19                               // DW_AT_language
-; CHECK: // .b8 5                                // DW_FORM_data2
-; CHECK: // .b8 3                                // DW_AT_name
-; CHECK: // .b8 8                                // DW_FORM_string
-; CHECK: // .b8 16                               // DW_AT_stmt_list
-; CHECK: // .b8 6                                // DW_FORM_data4
-; CHECK: // .b8 27                               // DW_AT_comp_dir
-; CHECK: // .b8 8                                // DW_FORM_string
-; CHECK: // .b8 17                               // DW_AT_low_pc
-; CHECK: // .b8 1                                // DW_FORM_addr
-; CHECK: // .b8 18                               // DW_AT_high_pc
-; CHECK: // .b8 1                                // DW_FORM_addr
-; CHECK: // .b8 0                                // EOM(1)
-; CHECK: // .b8 0                                // EOM(2)
-; CHECK: // .b8 2                                // Abbreviation Code
-; CHECK: // .b8 46                               // DW_TAG_subprogram
-; CHECK: // .b8 1                                // DW_CHILDREN_yes
-; CHECK: // .b8 17                               // DW_AT_low_pc
-; CHECK: // .b8 1                                // DW_FORM_addr
-; CHECK: // .b8 18                               // DW_AT_high_pc
-; CHECK: // .b8 1                                // DW_FORM_addr
-; CHECK: // .b8 64                               // DW_AT_frame_base
-; CHECK: // .b8 10                               // DW_FORM_block1
-; CHECK: // .b8 3                                // DW_AT_name
-; CHECK: // .b8 8                                // DW_FORM_string
-; CHECK: // .b8 58                               // DW_AT_decl_file
-; CHECK: // .b8 11                               // DW_FORM_data1
-; CHECK: // .b8 59                               // DW_AT_decl_line
-; CHECK: // .b8 11                               // DW_FORM_data1
-; CHECK: // .b8 39                               // DW_AT_prototyped
-; CHECK: // .b8 12                               // DW_FORM_flag
-; CHECK: // .b8 63                               // DW_AT_external
-; CHECK: // .b8 12                               // DW_FORM_flag
-; CHECK: // .b8 0                                // EOM(1)
-; CHECK: // .b8 0                                // EOM(2)
-; CHECK: // .b8 3                                // Abbreviation Code
-; CHECK: // .b8 52                               // DW_TAG_variable
-; CHECK: // .b8 0                                // DW_CHILDREN_no
-; CHECK: // .b8 2                                // DW_AT_location
-; CHECK: // .b8 10                               // DW_FORM_block1
-; CHECK: // .b8 3                                // DW_AT_name
-; CHECK: // .b8 8                                // DW_FORM_string
-; CHECK: // .b8 58                               // DW_AT_decl_file
-; CHECK: // .b8 11                               // DW_FORM_data1
-; CHECK: // .b8 59                               // DW_AT_decl_line
-; CHECK: // .b8 11                               // DW_FORM_data1
-; CHECK: // .b8 73                               // DW_AT_type
-; CHECK: // .b8 19                               // DW_FORM_ref4
-; CHECK: // .b8 0                                // EOM(1)
-; CHECK: // .b8 0                                // EOM(2)
-; CHECK: // .b8 4                                // Abbreviation Code
-; CHECK: // .b8 19                               // DW_TAG_structure_type
-; CHECK: // .b8 1                                // DW_CHILDREN_yes
-; CHECK: // .b8 3                                // DW_AT_name
-; CHECK: // .b8 8                                // DW_FORM_string
-; CHECK: // .b8 11                               // DW_AT_byte_size
-; CHECK: // .b8 11                               // DW_FORM_data1
-; CHECK: // .b8 58                               // DW_AT_decl_file
-; CHECK: // .b8 11                               // DW_FORM_data1
-; CHECK: // .b8 59                               // DW_AT_decl_line
-; CHECK: // .b8 11                               // DW_FORM_data1
-; CHECK: // .b8 0                                // EOM(1)
-; CHECK: // .b8 0                                // EOM(2)
-; CHECK: // .b8 5                                // Abbreviation Code
-; CHECK: // .b8 13                               // DW_TAG_member
-; CHECK: // .b8 0                                // DW_CHILDREN_no
-; CHECK: // .b8 3                                // DW_AT_name
-; CHECK: // .b8 8                                // DW_FORM_string
-; CHECK: // .b8 73                               // DW_AT_type
-; CHECK: // .b8 19                               // DW_FORM_ref4
-; CHECK: // .b8 58                               // DW_AT_decl_file
-; CHECK: // .b8 11                               // DW_FORM_data1
-; CHECK: // .b8 59                               // DW_AT_decl_line
-; CHECK: // .b8 11                               // DW_FORM_data1
-; CHECK: // .b8 56                               // DW_AT_data_member_location
-; CHECK: // .b8 10                               // DW_FORM_block1
-; CHECK: // .b8 0                                // EOM(1)
-; CHECK: // .b8 0                                // EOM(2)
-; CHECK: // .b8 6                                // Abbreviation Code
-; CHECK: // .b8 36                               // DW_TAG_base_type
-; CHECK: // .b8 0                                // DW_CHILDREN_no
-; CHECK: // .b8 3                                // DW_AT_name
-; CHECK: // .b8 8                                // DW_FORM_string
-; CHECK: // .b8 62                               // DW_AT_encoding
-; CHECK: // .b8 11                               // DW_FORM_data1
-; CHECK: // .b8 11                               // DW_AT_byte_size
-; CHECK: // .b8 11                               // DW_FORM_data1
-; CHECK: // .b8 0                                // EOM(1)
-; CHECK: // .b8 0                                // EOM(2)
-; CHECK: // .b8 0                                // EOM(3)
-; CHECK: // }
-; CHECK: // .section .debug_info
-; CHECK: // {
-; CHECK: // .b32 135                             // Length of Unit
-; CHECK: // .b8 2                                // DWARF version number
-; CHECK: // .b8 0
-; CHECK: // .b32 .debug_abbrev                   // Offset Into Abbrev. Section
-; CHECK: // .b8 8                                // Address Size (in bytes)
-; CHECK: // .b8 1                                // Abbrev [1] 0xb:0x80 DW_TAG_compile_unit
-; CHECK: // .b8 99,108,97,110,103                // DW_AT_producer
-; CHECK: // .b8 0
-; CHECK: // .b8 12                               // DW_AT_language
-; CHECK: // .b8 0
-; CHECK: // .b8 116,46,99                        // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b32 .debug_line                     // DW_AT_stmt_list
-; CHECK: // .b8 116,101,115,116                  // DW_AT_comp_dir
-; CHECK: // .b8 0
-; CHECK: // .b64 Lfunc_begin0                    // DW_AT_low_pc
-; CHECK: // .b64 Lfunc_end0                      // DW_AT_high_pc
-; CHECK: // .b8 2                                // Abbrev [2] 0x31:0x3d DW_TAG_subprogram
-; CHECK: // .b64 Lfunc_begin0                    // DW_AT_low_pc
-; CHECK: // .b64 Lfunc_end0                      // DW_AT_high_pc
-; CHECK: // .b8 1                                // DW_AT_frame_base
-; CHECK: // .b8 156
-; CHECK: // .b8 117,115,101,95,100,98,103,95,100,101,99,108,97,114,101 // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 3                                // DW_AT_decl_line
-; CHECK: // .b8 1                                // DW_AT_prototyped
-; CHECK: // .b8 1                                // DW_AT_external
-; CHECK: // .b8 3                                // Abbrev [3] 0x58:0x15 DW_TAG_variable
-; CHECK: // .b8 11                               // DW_AT_location
-; CHECK: // .b8 3
-; CHECK: // .b64 __local_depot0
-; CHECK: // .b8 35
-; CHECK: // .b8 0
-; CHECK: // .b8 111                              // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 4                                // DW_AT_decl_line
-; CHECK: // .b32 110                             // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 4                                // Abbrev [4] 0x6e:0x15 DW_TAG_structure_type
-; CHECK: // .b8 70,111,111                       // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 4                                // DW_AT_byte_size
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 1                                // DW_AT_decl_line
-; CHECK: // .b8 5                                // Abbrev [5] 0x76:0xc DW_TAG_member
-; CHECK: // .b8 120                              // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b32 131                             // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 1                                // DW_AT_decl_line
-; CHECK: // .b8 2                                // DW_AT_data_member_location
-; CHECK: // .b8 35
-; CHECK: // .b8 0
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 6                                // Abbrev [6] 0x83:0x7 DW_TAG_base_type
-; CHECK: // .b8 105,110,116                      // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 5                                // DW_AT_encoding
-; CHECK: // .b8 4                                // DW_AT_byte_size
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // }
+; CHECK: .section .debug_abbrev
+; CHECK-NEXT: {
+; CHECK-NEXT: .b8 1                                // Abbreviation Code
+; CHECK-NEXT: .b8 17                               // DW_TAG_compile_unit
+; CHECK-NEXT: .b8 1                                // DW_CHILDREN_yes
+; CHECK-NEXT: .b8 37                               // DW_AT_producer
+; CHECK-NEXT: .b8 8                                // DW_FORM_string
+; CHECK-NEXT: .b8 19                               // DW_AT_language
+; CHECK-NEXT: .b8 5                                // DW_FORM_data2
+; CHECK-NEXT: .b8 3                                // DW_AT_name
+; CHECK-NEXT: .b8 8                                // DW_FORM_string
+; CHECK-NEXT: .b8 16                               // DW_AT_stmt_list
+; CHECK-NEXT: .b8 6                                // DW_FORM_data4
+; CHECK-NEXT: .b8 27                               // DW_AT_comp_dir
+; CHECK-NEXT: .b8 8                                // DW_FORM_string
+; CHECK-NEXT: .b8 17                               // DW_AT_low_pc
+; CHECK-NEXT: .b8 1                                // DW_FORM_addr
+; CHECK-NEXT: .b8 18                               // DW_AT_high_pc
+; CHECK-NEXT: .b8 1                                // DW_FORM_addr
+; CHECK-NEXT: .b8 0                                // EOM(1)
+; CHECK-NEXT: .b8 0                                // EOM(2)
+; CHECK-NEXT: .b8 2                                // Abbreviation Code
+; CHECK-NEXT: .b8 46                               // DW_TAG_subprogram
+; CHECK-NEXT: .b8 1                                // DW_CHILDREN_yes
+; CHECK-NEXT: .b8 17                               // DW_AT_low_pc
+; CHECK-NEXT: .b8 1                                // DW_FORM_addr
+; CHECK-NEXT: .b8 18                               // DW_AT_high_pc
+; CHECK-NEXT: .b8 1                                // DW_FORM_addr
+; CHECK-NEXT: .b8 64                               // DW_AT_frame_base
+; CHECK-NEXT: .b8 10                               // DW_FORM_block1
+; CHECK-NEXT: .b8 3                                // DW_AT_name
+; CHECK-NEXT: .b8 8                                // DW_FORM_string
+; CHECK-NEXT: .b8 58                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 11                               // DW_FORM_data1
+; CHECK-NEXT: .b8 59                               // DW_AT_decl_line
+; CHECK-NEXT: .b8 11                               // DW_FORM_data1
+; CHECK-NEXT: .b8 39                               // DW_AT_prototyped
+; CHECK-NEXT: .b8 12                               // DW_FORM_flag
+; CHECK-NEXT: .b8 63                               // DW_AT_external
+; CHECK-NEXT: .b8 12                               // DW_FORM_flag
+; CHECK-NEXT: .b8 0                                // EOM(1)
+; CHECK-NEXT: .b8 0                                // EOM(2)
+; CHECK-NEXT: .b8 3                                // Abbreviation Code
+; CHECK-NEXT: .b8 52                               // DW_TAG_variable
+; CHECK-NEXT: .b8 0                                // DW_CHILDREN_no
+; CHECK-NEXT: .b8 51                               // DW_AT_address_class
+; CHECK-NEXT: .b8 11                               // DW_FORM_data1
+; CHECK-NEXT: .b8 2                                // DW_AT_location
+; CHECK-NEXT: .b8 10                               // DW_FORM_block1
+; CHECK-NEXT: .b8 3                                // DW_AT_name
+; CHECK-NEXT: .b8 8                                // DW_FORM_string
+; CHECK-NEXT: .b8 58                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 11                               // DW_FORM_data1
+; CHECK-NEXT: .b8 59                               // DW_AT_decl_line
+; CHECK-NEXT: .b8 11                               // DW_FORM_data1
+; CHECK-NEXT: .b8 73                               // DW_AT_type
+; CHECK-NEXT: .b8 19                               // DW_FORM_ref4
+; CHECK-NEXT: .b8 0                                // EOM(1)
+; CHECK-NEXT: .b8 0                                // EOM(2)
+; CHECK-NEXT: .b8 4                                // Abbreviation Code
+; CHECK-NEXT: .b8 19                               // DW_TAG_structure_type
+; CHECK-NEXT: .b8 1                                // DW_CHILDREN_yes
+; CHECK-NEXT: .b8 3                                // DW_AT_name
+; CHECK-NEXT: .b8 8                                // DW_FORM_string
+; CHECK-NEXT: .b8 11                               // DW_AT_byte_size
+; CHECK-NEXT: .b8 11                               // DW_FORM_data1
+; CHECK-NEXT: .b8 58                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 11                               // DW_FORM_data1
+; CHECK-NEXT: .b8 59                               // DW_AT_decl_line
+; CHECK-NEXT: .b8 11                               // DW_FORM_data1
+; CHECK-NEXT: .b8 0                                // EOM(1)
+; CHECK-NEXT: .b8 0                                // EOM(2)
+; CHECK-NEXT: .b8 5                                // Abbreviation Code
+; CHECK-NEXT: .b8 13                               // DW_TAG_member
+; CHECK-NEXT: .b8 0                                // DW_CHILDREN_no
+; CHECK-NEXT: .b8 3                                // DW_AT_name
+; CHECK-NEXT: .b8 8                                // DW_FORM_string
+; CHECK-NEXT: .b8 73                               // DW_AT_type
+; CHECK-NEXT: .b8 19                               // DW_FORM_ref4
+; CHECK-NEXT: .b8 58                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 11                               // DW_FORM_data1
+; CHECK-NEXT: .b8 59                               // DW_AT_decl_line
+; CHECK-NEXT: .b8 11                               // DW_FORM_data1
+; CHECK-NEXT: .b8 56                               // DW_AT_data_member_location
+; CHECK-NEXT: .b8 10                               // DW_FORM_block1
+; CHECK-NEXT: .b8 0                                // EOM(1)
+; CHECK-NEXT: .b8 0                                // EOM(2)
+; CHECK-NEXT: .b8 6                                // Abbreviation Code
+; CHECK-NEXT: .b8 36                               // DW_TAG_base_type
+; CHECK-NEXT: .b8 0                                // DW_CHILDREN_no
+; CHECK-NEXT: .b8 3                                // DW_AT_name
+; CHECK-NEXT: .b8 8                                // DW_FORM_string
+; CHECK-NEXT: .b8 62                               // DW_AT_encoding
+; CHECK-NEXT: .b8 11                               // DW_FORM_data1
+; CHECK-NEXT: .b8 11                               // DW_AT_byte_size
+; CHECK-NEXT: .b8 11                               // DW_FORM_data1
+; CHECK-NEXT: .b8 0                                // EOM(1)
+; CHECK-NEXT: .b8 0                                // EOM(2)
+; CHECK-NEXT: .b8 0                                // EOM(3)
+; CHECK-NEXT: }
+; CHECK-NEXT: .section .debug_info
+; CHECK-NEXT: {
+; CHECK-NEXT: .b32 136                             // Length of Unit
+; CHECK-NEXT: .b8 2                                // DWARF version number
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b32 .debug_abbrev                   // Offset Into Abbrev. Section
+; CHECK-NEXT: .b8 8                                // Address Size (in bytes)
+; CHECK-NEXT: .b8 1                                // Abbrev [1] 0xb:0x81 DW_TAG_compile_unit
+; CHECK-NEXT: .b8 99,108,97,110,103                // DW_AT_producer
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 12                               // DW_AT_language
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 116,46,99                        // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b32 .debug_line                     // DW_AT_stmt_list
+; CHECK-NEXT: .b8 116,101,115,116                  // DW_AT_comp_dir
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b64 Lfunc_begin0                    // DW_AT_low_pc
+; CHECK-NEXT: .b64 Lfunc_end0                      // DW_AT_high_pc
+; CHECK-NEXT: .b8 2                                // Abbrev [2] 0x31:0x3e DW_TAG_subprogram
+; CHECK-NEXT: .b64 Lfunc_begin0                    // DW_AT_low_pc
+; CHECK-NEXT: .b64 Lfunc_end0                      // DW_AT_high_pc
+; CHECK-NEXT: .b8 1                                // DW_AT_frame_base
+; CHECK-NEXT: .b8 156
+; CHECK-NEXT: .b8 117,115,101,95,100,98,103,95,100,101,99,108,97,114,101 // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 3                                // DW_AT_decl_line
+; CHECK-NEXT: .b8 1                                // DW_AT_prototyped
+; CHECK-NEXT: .b8 1                                // DW_AT_external
+; CHECK-NEXT: .b8 3                                // Abbrev [3] 0x58:0x16 DW_TAG_variable
+; CHECK-NEXT: .b8 6                                // DW_AT_address_class
+; CHECK-NEXT: .b8 11                               // DW_AT_location
+; CHECK-NEXT: .b8 3
+; CHECK-NEXT: .b64 __local_depot0
+; CHECK-NEXT: .b8 35
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 111                              // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 4                                // DW_AT_decl_line
+; CHECK-NEXT: .b32 111                             // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 4                                // Abbrev [4] 0x6f:0x15 DW_TAG_structure_type
+; CHECK-NEXT: .b8 70,111,111                       // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 4                                // DW_AT_byte_size
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_line
+; CHECK-NEXT: .b8 5                                // Abbrev [5] 0x77:0xc DW_TAG_member
+; CHECK-NEXT: .b8 120                              // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b32 132                             // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_line
+; CHECK-NEXT: .b8 2                                // DW_AT_data_member_location
+; CHECK-NEXT: .b8 35
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x84:0x7 DW_TAG_base_type
+; CHECK-NEXT: .b8 105,110,116                      // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 5                                // DW_AT_encoding
+; CHECK-NEXT: .b8 4                                // DW_AT_byte_size
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: }
 
 %struct.Foo = type { i32 }
 
diff --git a/test/DebugInfo/NVPTX/dbg-value-const-byref.ll b/test/DebugInfo/NVPTX/dbg-value-const-byref.ll
index b120a406454d0b1836a47ef7982c12e5995c80c0..9fda680aae57ce50507def9a3c8e7f4a4b2f44e9 100644
--- a/test/DebugInfo/NVPTX/dbg-value-const-byref.ll
+++ b/test/DebugInfo/NVPTX/dbg-value-const-byref.ll
@@ -17,10 +17,10 @@
 ; particularly variables that are described as constants and passed
 ; by reference.
 ;
-; CHECK: DEBUG_VALUE: foo:i <- [DW_OP_deref] $vrdepot
 ; CHECK: DEBUG_VALUE: foo:i <- 3
 ; CHECK: DEBUG_VALUE: foo:i <- 7
 ; CHECK: DEBUG_VALUE: foo:i <- %
+; CHECK: DEBUG_VALUE: foo:i <- [DW_OP_deref] $vrdepot
 
 ; Function Attrs: nounwind ssp uwtable
 define i32 @foo() #0 !dbg !4 {
diff --git a/test/DebugInfo/NVPTX/debug-addr-class.ll b/test/DebugInfo/NVPTX/debug-addr-class.ll
new file mode 100644
index 0000000000000000000000000000000000000000..3d8460d82475861fd41a038671076618632fbf5a
--- /dev/null
+++ b/test/DebugInfo/NVPTX/debug-addr-class.ll
@@ -0,0 +1,255 @@
+; RUN: llc -mtriple=nvptx64-nvidia-cuda < %s | FileCheck %s
+
+@GLOBAL = addrspace(1) externally_initialized global i32 0, align 4, !dbg !0
+@SHARED = addrspace(3) externally_initialized global i32 undef, align 4, !dbg !6
+
+define void @test(float, float*, float*, i32) !dbg !17 {
+  %5 = alloca float, align 4
+  %6 = alloca float*, align 8
+  %7 = alloca float*, align 8
+  %8 = alloca i32, align 4
+  store float %0, float* %5, align 4
+  call void @llvm.dbg.declare(metadata float* %5, metadata !22, metadata !DIExpression()), !dbg !23
+  store float* %1, float** %6, align 8
+  call void @llvm.dbg.declare(metadata float** %6, metadata !24, metadata !DIExpression()), !dbg !25
+  store float* %2, float** %7, align 8
+  call void @llvm.dbg.declare(metadata float** %7, metadata !26, metadata !DIExpression()), !dbg !27
+  store i32 %3, i32* %8, align 4
+  call void @llvm.dbg.declare(metadata i32* %8, metadata !28, metadata !DIExpression()), !dbg !29
+  %9 = load float, float* %5, align 4, !dbg !30
+  %10 = load float*, float** %6, align 8, !dbg !31
+  %11 = load i32, i32* %8, align 4, !dbg !32
+  %12 = sext i32 %11 to i64, !dbg !31
+  %13 = getelementptr inbounds float, float* %10, i64 %12, !dbg !31
+  %14 = load float, float* %13, align 4, !dbg !31
+  %15 = fmul contract float %9, %14, !dbg !33
+  %16 = load float*, float** %7, align 8, !dbg !34
+  %17 = load i32, i32* %8, align 4, !dbg !35
+  %18 = sext i32 %17 to i64, !dbg !34
+  %19 = getelementptr inbounds float, float* %16, i64 %18, !dbg !34
+  store float %15, float* %19, align 4, !dbg !36
+  store i32 0, i32* addrspacecast (i32 addrspace(1)* @GLOBAL to i32*), align 4, !dbg !37
+  store i32 0, i32* addrspacecast (i32 addrspace(3)* @SHARED to i32*), align 4, !dbg !38
+  ret void, !dbg !39
+}
+
+; Function Attrs: nounwind readnone speculatable
+declare void @llvm.dbg.declare(metadata, metadata, metadata)
+
+!llvm.dbg.cu = !{!2}
+!nvvm.annotations = !{!10}
+!llvm.module.flags = !{!11, !12, !13, !14, !15}
+!llvm.ident = !{!16}
+
+!0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression())
+!1 = distinct !DIGlobalVariable(name: "GLOBAL", scope: !2, file: !8, line: 3, type: !9, isLocal: false, isDefinition: true)
+!2 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !3, producer: "clang version 9.0.0 (trunk 351969) (llvm/trunk 351973)", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, globals: !5, nameTableKind: None)
+!3 = !DIFile(filename: "new.cc", directory: "/tmp")
+!4 = !{}
+!5 = !{!0, !6}
+!6 = !DIGlobalVariableExpression(var: !7, expr: !DIExpression(DW_OP_constu, 8, DW_OP_swap, DW_OP_xderef))
+!7 = distinct !DIGlobalVariable(name: "SHARED", scope: !2, file: !8, line: 4, type: !9, isLocal: false, isDefinition: true)
+!8 = !DIFile(filename: "test.cu", directory: "/tmp")
+!9 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!10 = !{void (float, float*, float*, i32)* @test, !"kernel", i32 1}
+!11 = !{i32 2, !"Dwarf Version", i32 2}
+!12 = !{i32 2, !"Debug Info Version", i32 3}
+!13 = !{i32 1, !"wchar_size", i32 4}
+!14 = !{i32 4, !"nvvm-reflect-ftz", i32 0}
+!15 = !{i32 7, !"PIC Level", i32 2}
+!16 = !{!"clang version 9.0.0 (trunk 351969) (llvm/trunk 351973)"}
+!17 = distinct !DISubprogram(name: "test", linkageName: "test", scope: !8, file: !8, line: 6, type: !18, scopeLine: 6, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !2, retainedNodes: !4)
+!18 = !DISubroutineType(types: !19)
+!19 = !{null, !20, !21, !21, !9}
+!20 = !DIBasicType(name: "float", size: 32, encoding: DW_ATE_float)
+!21 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !20, size: 64)
+!22 = !DILocalVariable(name: "a", arg: 1, scope: !17, file: !8, line: 6, type: !20)
+!23 = !DILocation(line: 6, column: 41, scope: !17)
+!24 = !DILocalVariable(name: "x", arg: 2, scope: !17, file: !8, line: 6, type: !21)
+!25 = !DILocation(line: 6, column: 51, scope: !17)
+!26 = !DILocalVariable(name: "y", arg: 3, scope: !17, file: !8, line: 6, type: !21)
+!27 = !DILocation(line: 6, column: 61, scope: !17)
+!28 = !DILocalVariable(name: "i", arg: 4, scope: !17, file: !8, line: 6, type: !9)
+!29 = !DILocation(line: 6, column: 68, scope: !17)
+!30 = !DILocation(line: 7, column: 10, scope: !17)
+!31 = !DILocation(line: 7, column: 14, scope: !17)
+!32 = !DILocation(line: 7, column: 16, scope: !17)
+!33 = !DILocation(line: 7, column: 12, scope: !17)
+!34 = !DILocation(line: 7, column: 3, scope: !17)
+!35 = !DILocation(line: 7, column: 5, scope: !17)
+!36 = !DILocation(line: 7, column: 8, scope: !17)
+!37 = !DILocation(line: 8, column: 10, scope: !17)
+!38 = !DILocation(line: 9, column: 10, scope: !17)
+!39 = !DILocation(line: 10, column: 1, scope: !17)
+
+; CHECK: .section .debug_abbrev
+; CHECK-NEXT: {
+; CHECK-NEXT: .b8 1                                   // Abbreviation Code
+; CHECK-NEXT: .b8 17                                  // DW_TAG_compile_unit
+; CHECK-NEXT: .b8 1                                   // DW_CHILDREN_yes
+; CHECK-NEXT: .b8 37                                  // DW_AT_producer
+; CHECK-NEXT: .b8 8                                   // DW_FORM_string
+; CHECK-NEXT: .b8 19                                  // DW_AT_language
+; CHECK-NEXT: .b8 5                                   // DW_FORM_data2
+; CHECK-NEXT: .b8 3                                   // DW_AT_name
+; CHECK-NEXT: .b8 8                                   // DW_FORM_string
+; CHECK-NEXT: .b8 16                                  // DW_AT_stmt_list
+; CHECK-NEXT: .b8 6                                   // DW_FORM_data4
+; CHECK-NEXT: .b8 27                                  // DW_AT_comp_dir
+; CHECK-NEXT: .b8 8                                   // DW_FORM_string
+; CHECK-NEXT: .b8 17                                  // DW_AT_low_pc
+; CHECK-NEXT: .b8 1                                   // DW_FORM_addr
+; CHECK-NEXT: .b8 18                                  // DW_AT_high_pc
+; CHECK-NEXT: .b8 1                                   // DW_FORM_addr
+; CHECK-NEXT: .b8 0                                   // EOM(1)
+; CHECK-NEXT: .b8 0                                   // EOM(2)
+; CHECK-NEXT: .b8 2                                   // Abbreviation Code
+; CHECK-NEXT: .b8 52                                  // DW_TAG_variable
+; CHECK-NEXT: .b8 0                                   // DW_CHILDREN_no
+; CHECK-NEXT: .b8 3                                   // DW_AT_name
+; CHECK-NEXT: .b8 8                                   // DW_FORM_string
+; CHECK-NEXT: .b8 73                                  // DW_AT_type
+; CHECK-NEXT: .b8 19                                  // DW_FORM_ref4
+; CHECK-NEXT: .b8 63                                  // DW_AT_external
+; CHECK-NEXT: .b8 12                                  // DW_FORM_flag
+; CHECK-NEXT: .b8 58                                  // DW_AT_decl_file
+; CHECK-NEXT: .b8 11                                  // DW_FORM_data1
+; CHECK-NEXT: .b8 59                                  // DW_AT_decl_line
+; CHECK-NEXT: .b8 11                                  // DW_FORM_data1
+; CHECK-NEXT: .b8 51                                  // DW_AT_address_class
+; CHECK-NEXT: .b8 11                                  // DW_FORM_data1
+; CHECK-NEXT: .b8 2                                   // DW_AT_location
+; CHECK-NEXT: .b8 10                                  // DW_FORM_block1
+; CHECK-NEXT: .b8 0                                   // EOM(1)
+; CHECK-NEXT: .b8 0                                   // EOM(2)
+; CHECK-NEXT: .b8 3                                   // Abbreviation Code
+; CHECK-NEXT: .b8 36                                  // DW_TAG_base_type
+; CHECK-NEXT: .b8 0                                   // DW_CHILDREN_no
+; CHECK-NEXT: .b8 3                                   // DW_AT_name
+; CHECK-NEXT: .b8 8                                   // DW_FORM_string
+; CHECK-NEXT: .b8 62                                  // DW_AT_encoding
+; CHECK-NEXT: .b8 11                                  // DW_FORM_data1
+; CHECK-NEXT: .b8 11                                  // DW_AT_byte_size
+; CHECK-NEXT: .b8 11                                  // DW_FORM_data1
+; CHECK-NEXT: .b8 0                                   // EOM(1)
+; CHECK-NEXT: .b8 0                                   // EOM(2)
+; CHECK-NEXT: .b8 4                                   // Abbreviation Code
+; CHECK-NEXT: .b8 46                                  // DW_TAG_subprogram
+; CHECK-NEXT: .b8 1                                   // DW_CHILDREN_yes
+; CHECK-NEXT: .b8 17                                  // DW_AT_low_pc
+; CHECK-NEXT: .b8 1                                   // DW_FORM_addr
+; CHECK-NEXT: .b8 18                                  // DW_AT_high_pc
+; CHECK-NEXT: .b8 1                                   // DW_FORM_addr
+; CHECK-NEXT: .b8 64                                  // DW_AT_frame_base
+; CHECK-NEXT: .b8 10                                  // DW_FORM_block1
+; CHECK-NEXT: .b8 135,64                              // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 8                                   // DW_FORM_string
+; CHECK-NEXT: .b8 3                                   // DW_AT_name
+; CHECK-NEXT: .b8 8                                   // DW_FORM_string
+; CHECK-NEXT: .b8 58                                  // DW_AT_decl_file
+; CHECK-NEXT: .b8 11                                  // DW_FORM_data1
+; CHECK-NEXT: .b8 59                                  // DW_AT_decl_line
+; CHECK-NEXT: .b8 11                                  // DW_FORM_data1
+; CHECK-NEXT: .b8 63                                  // DW_AT_external
+; CHECK-NEXT: .b8 12                                  // DW_FORM_flag
+; CHECK-NEXT: .b8 0                                   // EOM(1)
+; CHECK-NEXT: .b8 0                                   // EOM(2)
+; CHECK-NEXT: .b8 5                                   // Abbreviation Code
+; CHECK-NEXT: .b8 5                                   // DW_TAG_formal_parameter
+; CHECK-NEXT: .b8 0                                   // DW_CHILDREN_no
+; CHECK-NEXT: .b8 3                                   // DW_AT_name
+; CHECK-NEXT: .b8 8                                   // DW_FORM_string
+; CHECK-NEXT: .b8 58                                  // DW_AT_decl_file
+; CHECK-NEXT: .b8 11                                  // DW_FORM_data1
+; CHECK-NEXT: .b8 59                                  // DW_AT_decl_line
+; CHECK-NEXT: .b8 11                                  // DW_FORM_data1
+; CHECK-NEXT: .b8 73                                  // DW_AT_type
+; CHECK-NEXT: .b8 19                                  // DW_FORM_ref4
+; CHECK-NEXT: .b8 0                                   // EOM(1)
+; CHECK-NEXT: .b8 0                                   // EOM(2)
+; CHECK-NEXT: .b8 0                                   // EOM(3)
+; CHECK-NEXT: }
+; CHECK-NEXT: .section .debug_info
+; CHECK-NEXT: {
+; CHECK-NEXT: .b32 217                                // Length of Unit
+; CHECK-NEXT: .b8 2                                   // DWARF version number
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b32 .debug_abbrev                      // Offset Into Abbrev. Section
+; CHECK-NEXT: .b8 8                                   // Address Size (in bytes)
+; CHECK-NEXT: .b8 1                                   // Abbrev [1] 0xb:0xd2 DW_TAG_compile_unit
+; CHECK-NEXT: .b8 99,108,97,110,103,32,118,101,114,115,105,111,110,32,57,46,48,46,48,32,40,116,114,117,110,107,32,51,53,49,57,54,57,41,32,40,108,108,118,109 // DW_AT_producer
+; CHECK-NEXT: .b8 47,116,114,117,110,107,32,51,53,49,57,55,51,41
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 4                                   // DW_AT_language
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 110,101,119,46,99,99                // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b32 .debug_line                        // DW_AT_stmt_list
+; CHECK-NEXT: .b8 47,116,109,112                      // DW_AT_comp_dir
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b64 Lfunc_begin0                       // DW_AT_low_pc
+; CHECK-NEXT: .b64 Lfunc_end0                         // DW_AT_high_pc
+; CHECK-NEXT: .b8 2                                   // Abbrev [2] 0x65:0x1a DW_TAG_variable
+; CHECK-NEXT: .b8 71,76,79,66,65,76                   // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b32 127                                // DW_AT_type
+; CHECK-NEXT: .b8 1                                   // DW_AT_external
+; CHECK-NEXT: .b8 1                                   // DW_AT_decl_file
+; CHECK-NEXT: .b8 3                                   // DW_AT_decl_line
+; CHECK-NEXT: .b8 5                                   // DW_AT_address_class
+; CHECK-NEXT: .b8 9                                   // DW_AT_location
+; CHECK-NEXT: .b8 3
+; CHECK-NEXT: .b64 GLOBAL
+; CHECK-NEXT: .b8 3                                   // Abbrev [3] 0x7f:0x7 DW_TAG_base_type
+; CHECK-NEXT: .b8 105,110,116                         // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 5                                   // DW_AT_encoding
+; CHECK-NEXT: .b8 4                                   // DW_AT_byte_size
+; CHECK-NEXT: .b8 2                                   // Abbrev [2] 0x86:0x1a DW_TAG_variable
+; CHECK-NEXT: .b8 83,72,65,82,69,68                   // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b32 127                                // DW_AT_type
+; CHECK-NEXT: .b8 1                                   // DW_AT_external
+; CHECK-NEXT: .b8 1                                   // DW_AT_decl_file
+; CHECK-NEXT: .b8 4                                   // DW_AT_decl_line
+; CHECK-NEXT: .b8 8                                   // DW_AT_address_class
+; CHECK-NEXT: .b8 9                                   // DW_AT_location
+; CHECK-NEXT: .b8 3
+; CHECK-NEXT: .b64 SHARED
+; CHECK-NEXT: .b8 4                                   // Abbrev [4] 0xa0:0x33 DW_TAG_subprogram
+; CHECK-NEXT: .b64 Lfunc_begin0                       // DW_AT_low_pc
+; CHECK-NEXT: .b64 Lfunc_end0                         // DW_AT_high_pc
+; CHECK-NEXT: .b8 1                                   // DW_AT_frame_base
+; CHECK-NEXT: .b8 156
+; CHECK-NEXT: .b8 116,101,115,116                     // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 116,101,115,116                     // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 1                                   // DW_AT_decl_file
+; CHECK-NEXT: .b8 6                                   // DW_AT_decl_line
+; CHECK-NEXT: .b8 1                                   // DW_AT_external
+; CHECK-NEXT: .b8 5                                   // Abbrev [5] 0xc0:0x9 DW_TAG_formal_parameter
+; CHECK-NEXT: .b8 97                                  // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 1                                   // DW_AT_decl_file
+; CHECK-NEXT: .b8 6                                   // DW_AT_decl_line
+; CHECK-NEXT: .b32 211                                // DW_AT_type
+; CHECK-NEXT: .b8 5                                   // Abbrev [5] 0xc9:0x9 DW_TAG_formal_parameter
+; CHECK-NEXT: .b8 105                                 // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 1                                   // DW_AT_decl_file
+; CHECK-NEXT: .b8 6                                   // DW_AT_decl_line
+; CHECK-NEXT: .b32 127                                // DW_AT_type
+; CHECK-NEXT: .b8 0                                   // End Of Children Mark
+; CHECK-NEXT: .b8 3                                   // Abbrev [3] 0xd3:0x9 DW_TAG_base_type
+; CHECK-NEXT: .b8 102,108,111,97,116                  // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 4                                   // DW_AT_encoding
+; CHECK-NEXT: .b8 4                                   // DW_AT_byte_size
+; CHECK-NEXT: .b8 0                                   // End Of Children Mark
+; CHECK-NEXT: }
+; CHECK-NEXT: .section .debug_macinfo
+; CHECK-NEXT: {
+; CHECK-NEXT: .b8 0                                   // End Of Macro List Mark
+; CHECK:      }
+
diff --git a/test/DebugInfo/NVPTX/debug-empty.ll b/test/DebugInfo/NVPTX/debug-empty.ll
new file mode 100644
index 0000000000000000000000000000000000000000..f1ebf739d2fd39cf045ec62b89bbfe54a88c9b89
--- /dev/null
+++ b/test/DebugInfo/NVPTX/debug-empty.ll
@@ -0,0 +1,18 @@
+; RUN: llc < %s -mtriple=nvptx64-nvidia-cuda | FileCheck %s
+
+; CHECK: .target sm_{{[0-9]+$}}
+; CHECK-NOT: }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4, !5, !6, !7}
+!llvm.ident = !{!8}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !1, producer: "clang version 9.0.0 (trunk 351924) (llvm/trunk 351968)", isOptimized: false, runtimeVersion: 0, emissionKind: DebugDirectivesOnly, enums: !2, nameTableKind: None)
+!1 = !DIFile(filename: "new.cc", directory: "/test")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 2}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{i32 1, !"wchar_size", i32 4}
+!6 = !{i32 4, !"nvvm-reflect-ftz", i32 0}
+!7 = !{i32 7, !"PIC Level", i32 2}
+!8 = !{!"clang version 9.0.0 (trunk 351924) (llvm/trunk 351968)"}
diff --git a/test/DebugInfo/NVPTX/debug-file-loc.ll b/test/DebugInfo/NVPTX/debug-file-loc.ll
index a9ea67c338867f806a29ffc11200766fa57de031..631e1e2d2b2a6a6b901b7c83cd7ccd183471dfb8 100644
--- a/test/DebugInfo/NVPTX/debug-file-loc.ll
+++ b/test/DebugInfo/NVPTX/debug-file-loc.ll
@@ -8,7 +8,7 @@
 ;__device__ void bar() {}
 ;}
 
-; CHECK: .target sm_{{[0-9]+}}//, debug
+; CHECK: .target sm_{{[0-9]+}}, debug
 
 ; CHECK: .visible .func foo()
 ; CHECK: .loc [[FOO:[0-9]+]] 1 31
@@ -29,52 +29,52 @@ bb:
 
 ; CHECK-DAG: .file [[FOO]] "{{.*}}foo.h"
 ; CHECK-DAG: .file [[BAR]] "{{.*}}bar.cu"
-; CHECK: // .section .debug_abbrev
-; CHECK: // {
-; CHECK: // .b8 1                                // Abbreviation Code
-; CHECK: // .b8 17                               // DW_TAG_compile_unit
-; CHECK: // .b8 0                                // DW_CHILDREN_no
-; CHECK: // .b8 37                               // DW_AT_producer
-; CHECK: // .b8 8                                // DW_FORM_string
-; CHECK: // .b8 19                               // DW_AT_language
-; CHECK: // .b8 5                                // DW_FORM_data2
-; CHECK: // .b8 3                                // DW_AT_name
-; CHECK: // .b8 8                                // DW_FORM_string
-; CHECK: // .b8 16                               // DW_AT_stmt_list
-; CHECK: // .b8 6                                // DW_FORM_data4
-; CHECK: // .b8 27                               // DW_AT_comp_dir
-; CHECK: // .b8 8                                // DW_FORM_string
-; CHECK: // .b8 17                               // DW_AT_low_pc
-; CHECK: // .b8 1                                // DW_FORM_addr
-; CHECK: // .b8 18                               // DW_AT_high_pc
-; CHECK: // .b8 1                                // DW_FORM_addr
-; CHECK: // .b8 0                                // EOM(1)
-; CHECK: // .b8 0                                // EOM(2)
-; CHECK: // .b8 0                                // EOM(3)
-; CHECK: // }
-; CHECK: // .section .debug_info
-; CHECK: // {
-; CHECK: // .b32 50                              // Length of Unit
-; CHECK: // .b8 2                                // DWARF version number
-; CHECK: // .b8 0
-; CHECK: // .b32 .debug_abbrev                   // Offset Into Abbrev. Section
-; CHECK: // .b8 8                                // Address Size (in bytes)
-; CHECK: // .b8 1                                // Abbrev [1] 0xb:0x2b DW_TAG_compile_unit
-; CHECK: // .b8 0                                // DW_AT_producer
-; CHECK: // .b8 4                                // DW_AT_language
-; CHECK: // .b8 0
-; CHECK: // .b8 98,97,114,46,99,117              // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b32 .debug_line                     // DW_AT_stmt_list
-; CHECK: // .b8 47,115,111,117,114,99,101,47,100,105,114                // DW_AT_comp_dir
-; CHECK: // .b8 0
-; CHECK: // .b64 Lfunc_begin0                    // DW_AT_low_pc
-; CHECK: // .b64 Lfunc_end1                      // DW_AT_high_pc
-; CHECK: // }
-; CHECK: // .section .debug_macinfo
-; CHECK: // {
-; CHECK: // .b8 0                                // End Of Macro List Mark
-; CHECK: // }
+; CHECK: .section .debug_abbrev
+; CHECK-NEXT: {
+; CHECK-NEXT: .b8 1                                // Abbreviation Code
+; CHECK-NEXT: .b8 17                               // DW_TAG_compile_unit
+; CHECK-NEXT: .b8 0                                // DW_CHILDREN_no
+; CHECK-NEXT: .b8 37                               // DW_AT_producer
+; CHECK-NEXT: .b8 8                                // DW_FORM_string
+; CHECK-NEXT: .b8 19                               // DW_AT_language
+; CHECK-NEXT: .b8 5                                // DW_FORM_data2
+; CHECK-NEXT: .b8 3                                // DW_AT_name
+; CHECK-NEXT: .b8 8                                // DW_FORM_string
+; CHECK-NEXT: .b8 16                               // DW_AT_stmt_list
+; CHECK-NEXT: .b8 6                                // DW_FORM_data4
+; CHECK-NEXT: .b8 27                               // DW_AT_comp_dir
+; CHECK-NEXT: .b8 8                                // DW_FORM_string
+; CHECK-NEXT: .b8 17                               // DW_AT_low_pc
+; CHECK-NEXT: .b8 1                                // DW_FORM_addr
+; CHECK-NEXT: .b8 18                               // DW_AT_high_pc
+; CHECK-NEXT: .b8 1                                // DW_FORM_addr
+; CHECK-NEXT: .b8 0                                // EOM(1)
+; CHECK-NEXT: .b8 0                                // EOM(2)
+; CHECK-NEXT: .b8 0                                // EOM(3)
+; CHECK-NEXT: }
+; CHECK-NEXT: .section .debug_info
+; CHECK-NEXT: {
+; CHECK-NEXT: .b32 50                              // Length of Unit
+; CHECK-NEXT: .b8 2                                // DWARF version number
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b32 .debug_abbrev                   // Offset Into Abbrev. Section
+; CHECK-NEXT: .b8 8                                // Address Size (in bytes)
+; CHECK-NEXT: .b8 1                                // Abbrev [1] 0xb:0x2b DW_TAG_compile_unit
+; CHECK-NEXT: .b8 0                                // DW_AT_producer
+; CHECK-NEXT: .b8 4                                // DW_AT_language
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 98,97,114,46,99,117              // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b32 .debug_line                     // DW_AT_stmt_list
+; CHECK-NEXT: .b8 47,115,111,117,114,99,101,47,100,105,114                // DW_AT_comp_dir
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b64 Lfunc_begin0                    // DW_AT_low_pc
+; CHECK-NEXT: .b64 Lfunc_end1                      // DW_AT_high_pc
+; CHECK-NEXT: }
+; CHECK-NEXT: .section .debug_macinfo
+; CHECK-NEXT: {
+; CHECK-NEXT: .b8 0                                // End Of Macro List Mark
+; CHECK:      }
 
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!8, !9}
diff --git a/test/DebugInfo/NVPTX/debug-info.ll b/test/DebugInfo/NVPTX/debug-info.ll
index 47101b00ee79eebfb3c82385bfcbb5e5c3d70e01..04cd621a5b9f726653192361b4eb972948b9f527 100644
--- a/test/DebugInfo/NVPTX/debug-info.ll
+++ b/test/DebugInfo/NVPTX/debug-info.ll
@@ -9,7 +9,7 @@
 ;    res(a * x[i], y[i], &y[i]);
 ;}
 
-; CHECK: .target sm_{{[0-9]+}}//, debug
+; CHECK: .target sm_{{[0-9]+}}, debug
 
 ; CHECK: .visible .entry _Z5saxpyifPfS_(
 ; CHECK: .param .u32 {{.+}},
@@ -105,4696 +105,4698 @@ if.end:                                           ; preds = %if.then, %entry
 ; CHECK-DAG: .file [[DEBUG_INFO_CU]] "{{.*}}debug-info.cu"
 ; CHECK-DAG: .file [[BUILTUIN_VARS_H]] "{{.*}}clang/include{{/|\\\\}}__clang_cuda_builtin_vars.h"
 
-; CHECK: //	.section	.debug_abbrev
-; CHECK: //	{
-; CHECK: // .b8 1                                // Abbreviation Code
-; CHECK: // .b8 17                               // DW_TAG_compile_unit
-; CHECK: // .b8 1                                // DW_CHILDREN_yes
-; CHECK: // .b8 37                               // DW_AT_producer
-; CHECK: // .b8 8                                // DW_FORM_string
-; CHECK: // .b8 19                               // DW_AT_language
-; CHECK: // .b8 5                                // DW_FORM_data2
-; CHECK: // .b8 3                                // DW_AT_name
-; CHECK: // .b8 8                                // DW_FORM_string
-; CHECK: // .b8 16                               // DW_AT_stmt_list
-; CHECK: // .b8 6                                // DW_FORM_data4
-; CHECK: // .b8 27                               // DW_AT_comp_dir
-; CHECK: // .b8 8                                // DW_FORM_string
-; CHECK: // .b8 17                               // DW_AT_low_pc
-; CHECK: // .b8 1                                // DW_FORM_addr
-; CHECK: // .b8 18                               // DW_AT_high_pc
-; CHECK: // .b8 1                                // DW_FORM_addr
-; CHECK: // .b8 0                                // EOM(1)
-; CHECK: // .b8 0                                // EOM(2)
-; CHECK: // .b8 2                                // Abbreviation Code
-; CHECK: // .b8 57                               // DW_TAG_namespace
-; CHECK: // .b8 1                                // DW_CHILDREN_yes
-; CHECK: // .b8 3                                // DW_AT_name
-; CHECK: // .b8 8                                // DW_FORM_string
-; CHECK: // .b8 0                                // EOM(1)
-; CHECK: // .b8 0                                // EOM(2)
-; CHECK: // .b8 3                                // Abbreviation Code
-; CHECK: // .b8 8                                // DW_TAG_imported_declaration
-; CHECK: // .b8 0                                // DW_CHILDREN_no
-; CHECK: // .b8 58                               // DW_AT_decl_file
-; CHECK: // .b8 11                               // DW_FORM_data1
-; CHECK: // .b8 59                               // DW_AT_decl_line
-; CHECK: // .b8 11                               // DW_FORM_data1
-; CHECK: // .b8 24                               // DW_AT_import
-; CHECK: // .b8 19                               // DW_FORM_ref4
-; CHECK: // .b8 0                                // EOM(1)
-; CHECK: // .b8 0                                // EOM(2)
-; CHECK: // .b8 4                                // Abbreviation Code
-; CHECK: // .b8 8                                // DW_TAG_imported_declaration
-; CHECK: // .b8 0                                // DW_CHILDREN_no
-; CHECK: // .b8 58                               // DW_AT_decl_file
-; CHECK: // .b8 11                               // DW_FORM_data1
-; CHECK: // .b8 59                               // DW_AT_decl_line
-; CHECK: // .b8 5                                // DW_FORM_data2
-; CHECK: // .b8 24                               // DW_AT_import
-; CHECK: // .b8 19                               // DW_FORM_ref4
-; CHECK: // .b8 0                                // EOM(1)
-; CHECK: // .b8 0                                // EOM(2)
-; CHECK: // .b8 5                                // Abbreviation Code
-; CHECK: // .b8 46                               // DW_TAG_subprogram
-; CHECK: // .b8 1                                // DW_CHILDREN_yes
-; CHECK: // .b8 135,64                           // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 8                                // DW_FORM_string
-; CHECK: // .b8 3                                // DW_AT_name
-; CHECK: // .b8 8                                // DW_FORM_string
-; CHECK: // .b8 58                               // DW_AT_decl_file
-; CHECK: // .b8 11                               // DW_FORM_data1
-; CHECK: // .b8 59                               // DW_AT_decl_line
-; CHECK: // .b8 11                               // DW_FORM_data1
-; CHECK: // .b8 73                               // DW_AT_type
-; CHECK: // .b8 19                               // DW_FORM_ref4
-; CHECK: // .b8 60                               // DW_AT_declaration
-; CHECK: // .b8 12                               // DW_FORM_flag
-; CHECK: // .b8 0                                // EOM(1)
-; CHECK: // .b8 0                                // EOM(2)
-; CHECK: // .b8 6                                // Abbreviation Code
-; CHECK: // .b8 5                                // DW_TAG_formal_parameter
-; CHECK: // .b8 0                                // DW_CHILDREN_no
-; CHECK: // .b8 73                               // DW_AT_type
-; CHECK: // .b8 19                               // DW_FORM_ref4
-; CHECK: // .b8 0                                // EOM(1)
-; CHECK: // .b8 0                                // EOM(2)
-; CHECK: // .b8 7                                // Abbreviation Code
-; CHECK: // .b8 36                               // DW_TAG_base_type
-; CHECK: // .b8 0                                // DW_CHILDREN_no
-; CHECK: // .b8 3                                // DW_AT_name
-; CHECK: // .b8 8                                // DW_FORM_string
-; CHECK: // .b8 62                               // DW_AT_encoding
-; CHECK: // .b8 11                               // DW_FORM_data1
-; CHECK: // .b8 11                               // DW_AT_byte_size
-; CHECK: // .b8 11                               // DW_FORM_data1
-; CHECK: // .b8 0                                // EOM(1)
-; CHECK: // .b8 0                                // EOM(2)
-; CHECK: // .b8 8                                // Abbreviation Code
-; CHECK: // .b8 15                               // DW_TAG_pointer_type
-; CHECK: // .b8 0                                // DW_CHILDREN_no
-; CHECK: // .b8 73                               // DW_AT_type
-; CHECK: // .b8 19                               // DW_FORM_ref4
-; CHECK: // .b8 0                                // EOM(1)
-; CHECK: // .b8 0                                // EOM(2)
-; CHECK: // .b8 9                                // Abbreviation Code
-; CHECK: // .b8 38                               // DW_TAG_const_type
-; CHECK: // .b8 0                                // DW_CHILDREN_no
-; CHECK: // .b8 73                               // DW_AT_type
-; CHECK: // .b8 19                               // DW_FORM_ref4
-; CHECK: // .b8 0                                // EOM(1)
-; CHECK: // .b8 0                                // EOM(2)
-; CHECK: // .b8 10                               // Abbreviation Code
-; CHECK: // .b8 46                               // DW_TAG_subprogram
-; CHECK: // .b8 1                                // DW_CHILDREN_yes
-; CHECK: // .b8 3                                // DW_AT_name
-; CHECK: // .b8 8                                // DW_FORM_string
-; CHECK: // .b8 58                               // DW_AT_decl_file
-; CHECK: // .b8 11                               // DW_FORM_data1
-; CHECK: // .b8 59                               // DW_AT_decl_line
-; CHECK: // .b8 11                               // DW_FORM_data1
-; CHECK: // .b8 73                               // DW_AT_type
-; CHECK: // .b8 19                               // DW_FORM_ref4
-; CHECK: // .b8 60                               // DW_AT_declaration
-; CHECK: // .b8 12                               // DW_FORM_flag
-; CHECK: // .b8 63                               // DW_AT_external
-; CHECK: // .b8 12                               // DW_FORM_flag
-; CHECK: // .b8 0                                // EOM(1)
-; CHECK: // .b8 0                                // EOM(2)
-; CHECK: // .b8 11                               // Abbreviation Code
-; CHECK: // .b8 22                               // DW_TAG_typedef
-; CHECK: // .b8 0                                // DW_CHILDREN_no
-; CHECK: // .b8 73                               // DW_AT_type
-; CHECK: // .b8 19                               // DW_FORM_ref4
-; CHECK: // .b8 3                                // DW_AT_name
-; CHECK: // .b8 8                                // DW_FORM_string
-; CHECK: // .b8 58                               // DW_AT_decl_file
-; CHECK: // .b8 11                               // DW_FORM_data1
-; CHECK: // .b8 59                               // DW_AT_decl_line
-; CHECK: // .b8 11                               // DW_FORM_data1
-; CHECK: // .b8 0                                // EOM(1)
-; CHECK: // .b8 0                                // EOM(2)
-; CHECK: // .b8 12                               // Abbreviation Code
-; CHECK: // .b8 19                               // DW_TAG_structure_type
-; CHECK: // .b8 0                                // DW_CHILDREN_no
-; CHECK: // .b8 60                               // DW_AT_declaration
-; CHECK: // .b8 12                               // DW_FORM_flag
-; CHECK: // .b8 0                                // EOM(1)
-; CHECK: // .b8 0                                // EOM(2)
-; CHECK: // .b8 13                               // Abbreviation Code
-; CHECK: // .b8 19                               // DW_TAG_structure_type
-; CHECK: // .b8 1                                // DW_CHILDREN_yes
-; CHECK: // .b8 11                               // DW_AT_byte_size
-; CHECK: // .b8 11                               // DW_FORM_data1
-; CHECK: // .b8 58                               // DW_AT_decl_file
-; CHECK: // .b8 11                               // DW_FORM_data1
-; CHECK: // .b8 59                               // DW_AT_decl_line
-; CHECK: // .b8 11                               // DW_FORM_data1
-; CHECK: // .b8 0                                // EOM(1)
-; CHECK: // .b8 0                                // EOM(2)
-; CHECK: // .b8 14                               // Abbreviation Code
-; CHECK: // .b8 13                               // DW_TAG_member
-; CHECK: // .b8 0                                // DW_CHILDREN_no
-; CHECK: // .b8 3                                // DW_AT_name
-; CHECK: // .b8 8                                // DW_FORM_string
-; CHECK: // .b8 73                               // DW_AT_type
-; CHECK: // .b8 19                               // DW_FORM_ref4
-; CHECK: // .b8 58                               // DW_AT_decl_file
-; CHECK: // .b8 11                               // DW_FORM_data1
-; CHECK: // .b8 59                               // DW_AT_decl_line
-; CHECK: // .b8 11                               // DW_FORM_data1
-; CHECK: // .b8 56                               // DW_AT_data_member_location
-; CHECK: // .b8 10                               // DW_FORM_block1
-; CHECK: // .b8 0                                // EOM(1)
-; CHECK: // .b8 0                                // EOM(2)
-; CHECK: // .b8 15                               // Abbreviation Code
-; CHECK: // .b8 46                               // DW_TAG_subprogram
-; CHECK: // .b8 0                                // DW_CHILDREN_no
-; CHECK: // .b8 3                                // DW_AT_name
-; CHECK: // .b8 8                                // DW_FORM_string
-; CHECK: // .b8 58                               // DW_AT_decl_file
-; CHECK: // .b8 11                               // DW_FORM_data1
-; CHECK: // .b8 59                               // DW_AT_decl_line
-; CHECK: // .b8 5                                // DW_FORM_data2
-; CHECK: // .b8 60                               // DW_AT_declaration
-; CHECK: // .b8 12                               // DW_FORM_flag
-; CHECK: // .b8 63                               // DW_AT_external
-; CHECK: // .b8 12                               // DW_FORM_flag
-; CHECK: // .b8 135,1                            // DW_AT_noreturn
-; CHECK: // .b8 12                               // DW_FORM_flag
-; CHECK: // .b8 0                                // EOM(1)
-; CHECK: // .b8 0                                // EOM(2)
-; CHECK: // .b8 16                               // Abbreviation Code
-; CHECK: // .b8 46                               // DW_TAG_subprogram
-; CHECK: // .b8 1                                // DW_CHILDREN_yes
-; CHECK: // .b8 3                                // DW_AT_name
-; CHECK: // .b8 8                                // DW_FORM_string
-; CHECK: // .b8 58                               // DW_AT_decl_file
-; CHECK: // .b8 11                               // DW_FORM_data1
-; CHECK: // .b8 59                               // DW_AT_decl_line
-; CHECK: // .b8 5                                // DW_FORM_data2
-; CHECK: // .b8 73                               // DW_AT_type
-; CHECK: // .b8 19                               // DW_FORM_ref4
-; CHECK: // .b8 60                               // DW_AT_declaration
-; CHECK: // .b8 12                               // DW_FORM_flag
-; CHECK: // .b8 63                               // DW_AT_external
-; CHECK: // .b8 12                               // DW_FORM_flag
-; CHECK: // .b8 0                                // EOM(1)
-; CHECK: // .b8 0                                // EOM(2)
-; CHECK: // .b8 17                               // Abbreviation Code
-; CHECK: // .b8 21                               // DW_TAG_subroutine_type
-; CHECK: // .b8 0                                // DW_CHILDREN_no
-; CHECK: // .b8 0                                // EOM(1)
-; CHECK: // .b8 0                                // EOM(2)
-; CHECK: // .b8 18                               // Abbreviation Code
-; CHECK: // .b8 15                               // DW_TAG_pointer_type
-; CHECK: // .b8 0                                // DW_CHILDREN_no
-; CHECK: // .b8 0                                // EOM(1)
-; CHECK: // .b8 0                                // EOM(2)
-; CHECK: // .b8 19                               // Abbreviation Code
-; CHECK: // .b8 38                               // DW_TAG_const_type
-; CHECK: // .b8 0                                // DW_CHILDREN_no
-; CHECK: // .b8 0                                // EOM(1)
-; CHECK: // .b8 0                                // EOM(2)
-; CHECK: // .b8 20                               // Abbreviation Code
-; CHECK: // .b8 22                               // DW_TAG_typedef
-; CHECK: // .b8 0                                // DW_CHILDREN_no
-; CHECK: // .b8 73                               // DW_AT_type
-; CHECK: // .b8 19                               // DW_FORM_ref4
-; CHECK: // .b8 3                                // DW_AT_name
-; CHECK: // .b8 8                                // DW_FORM_string
-; CHECK: // .b8 58                               // DW_AT_decl_file
-; CHECK: // .b8 11                               // DW_FORM_data1
-; CHECK: // .b8 59                               // DW_AT_decl_line
-; CHECK: // .b8 5                                // DW_FORM_data2
-; CHECK: // .b8 0                                // EOM(1)
-; CHECK: // .b8 0                                // EOM(2)
-; CHECK: // .b8 21                               // Abbreviation Code
-; CHECK: // .b8 21                               // DW_TAG_subroutine_type
-; CHECK: // .b8 1                                // DW_CHILDREN_yes
-; CHECK: // .b8 73                               // DW_AT_type
-; CHECK: // .b8 19                               // DW_FORM_ref4
-; CHECK: // .b8 0                                // EOM(1)
-; CHECK: // .b8 0                                // EOM(2)
-; CHECK: // .b8 22                               // Abbreviation Code
-; CHECK: // .b8 46                               // DW_TAG_subprogram
-; CHECK: // .b8 1                                // DW_CHILDREN_yes
-; CHECK: // .b8 3                                // DW_AT_name
-; CHECK: // .b8 8                                // DW_FORM_string
-; CHECK: // .b8 58                               // DW_AT_decl_file
-; CHECK: // .b8 11                               // DW_FORM_data1
-; CHECK: // .b8 59                               // DW_AT_decl_line
-; CHECK: // .b8 5                                // DW_FORM_data2
-; CHECK: // .b8 60                               // DW_AT_declaration
-; CHECK: // .b8 12                               // DW_FORM_flag
-; CHECK: // .b8 63                               // DW_AT_external
-; CHECK: // .b8 12                               // DW_FORM_flag
-; CHECK: // .b8 135,1                            // DW_AT_noreturn
-; CHECK: // .b8 12                               // DW_FORM_flag
-; CHECK: // .b8 0                                // EOM(1)
-; CHECK: // .b8 0                                // EOM(2)
-; CHECK: // .b8 23                               // Abbreviation Code
-; CHECK: // .b8 46                               // DW_TAG_subprogram
-; CHECK: // .b8 1                                // DW_CHILDREN_yes
-; CHECK: // .b8 3                                // DW_AT_name
-; CHECK: // .b8 8                                // DW_FORM_string
-; CHECK: // .b8 58                               // DW_AT_decl_file
-; CHECK: // .b8 11                               // DW_FORM_data1
-; CHECK: // .b8 59                               // DW_AT_decl_line
-; CHECK: // .b8 5                                // DW_FORM_data2
-; CHECK: // .b8 60                               // DW_AT_declaration
-; CHECK: // .b8 12                               // DW_FORM_flag
-; CHECK: // .b8 63                               // DW_AT_external
-; CHECK: // .b8 12                               // DW_FORM_flag
-; CHECK: // .b8 0                                // EOM(1)
-; CHECK: // .b8 0                                // EOM(2)
-; CHECK: // .b8 24                               // Abbreviation Code
-; CHECK: // .b8 46                               // DW_TAG_subprogram
-; CHECK: // .b8 0                                // DW_CHILDREN_no
-; CHECK: // .b8 3                                // DW_AT_name
-; CHECK: // .b8 8                                // DW_FORM_string
-; CHECK: // .b8 58                               // DW_AT_decl_file
-; CHECK: // .b8 11                               // DW_FORM_data1
-; CHECK: // .b8 59                               // DW_AT_decl_line
-; CHECK: // .b8 5                                // DW_FORM_data2
-; CHECK: // .b8 73                               // DW_AT_type
-; CHECK: // .b8 19                               // DW_FORM_ref4
-; CHECK: // .b8 60                               // DW_AT_declaration
-; CHECK: // .b8 12                               // DW_FORM_flag
-; CHECK: // .b8 63                               // DW_AT_external
-; CHECK: // .b8 12                               // DW_FORM_flag
-; CHECK: // .b8 0                                // EOM(1)
-; CHECK: // .b8 0                                // EOM(2)
-; CHECK: // .b8 25                               // Abbreviation Code
-; CHECK: // .b8 46                               // DW_TAG_subprogram
-; CHECK: // .b8 1                                // DW_CHILDREN_yes
-; CHECK: // .b8 135,64                           // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 8                                // DW_FORM_string
-; CHECK: // .b8 3                                // DW_AT_name
-; CHECK: // .b8 8                                // DW_FORM_string
-; CHECK: // .b8 58                               // DW_AT_decl_file
-; CHECK: // .b8 11                               // DW_FORM_data1
-; CHECK: // .b8 59                               // DW_AT_decl_line
-; CHECK: // .b8 11                               // DW_FORM_data1
-; CHECK: // .b8 73                               // DW_AT_type
-; CHECK: // .b8 19                               // DW_FORM_ref4
-; CHECK: // .b8 60                               // DW_AT_declaration
-; CHECK: // .b8 12                               // DW_FORM_flag
-; CHECK: // .b8 63                               // DW_AT_external
-; CHECK: // .b8 12                               // DW_FORM_flag
-; CHECK: // .b8 0                                // EOM(1)
-; CHECK: // .b8 0                                // EOM(2)
-; CHECK: // .b8 26                               // Abbreviation Code
-; CHECK: // .b8 46                               // DW_TAG_subprogram
-; CHECK: // .b8 1                                // DW_CHILDREN_yes
-; CHECK: // .b8 135,64                           // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 8                                // DW_FORM_string
-; CHECK: // .b8 3                                // DW_AT_name
-; CHECK: // .b8 8                                // DW_FORM_string
-; CHECK: // .b8 58                               // DW_AT_decl_file
-; CHECK: // .b8 11                               // DW_FORM_data1
-; CHECK: // .b8 59                               // DW_AT_decl_line
-; CHECK: // .b8 5                                // DW_FORM_data2
-; CHECK: // .b8 73                               // DW_AT_type
-; CHECK: // .b8 19                               // DW_FORM_ref4
-; CHECK: // .b8 60                               // DW_AT_declaration
-; CHECK: // .b8 12                               // DW_FORM_flag
-; CHECK: // .b8 0                                // EOM(1)
-; CHECK: // .b8 0                                // EOM(2)
-; CHECK: // .b8 27                               // Abbreviation Code
-; CHECK: // .b8 19                               // DW_TAG_structure_type
-; CHECK: // .b8 1                                // DW_CHILDREN_yes
-; CHECK: // .b8 3                                // DW_AT_name
-; CHECK: // .b8 8                                // DW_FORM_string
-; CHECK: // .b8 11                               // DW_AT_byte_size
-; CHECK: // .b8 11                               // DW_FORM_data1
-; CHECK: // .b8 58                               // DW_AT_decl_file
-; CHECK: // .b8 11                               // DW_FORM_data1
-; CHECK: // .b8 59                               // DW_AT_decl_line
-; CHECK: // .b8 11                               // DW_FORM_data1
-; CHECK: // .b8 0                                // EOM(1)
-; CHECK: // .b8 0                                // EOM(2)
-; CHECK: // .b8 28                               // Abbreviation Code
-; CHECK: // .b8 46                               // DW_TAG_subprogram
-; CHECK: // .b8 0                                // DW_CHILDREN_no
-; CHECK: // .b8 135,64                           // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 8                                // DW_FORM_string
-; CHECK: // .b8 3                                // DW_AT_name
-; CHECK: // .b8 8                                // DW_FORM_string
-; CHECK: // .b8 58                               // DW_AT_decl_file
-; CHECK: // .b8 11                               // DW_FORM_data1
-; CHECK: // .b8 59                               // DW_AT_decl_line
-; CHECK: // .b8 11                               // DW_FORM_data1
-; CHECK: // .b8 73                               // DW_AT_type
-; CHECK: // .b8 19                               // DW_FORM_ref4
-; CHECK: // .b8 60                               // DW_AT_declaration
-; CHECK: // .b8 12                               // DW_FORM_flag
-; CHECK: // .b8 63                               // DW_AT_external
-; CHECK: // .b8 12                               // DW_FORM_flag
-; CHECK: // .b8 0                                // EOM(1)
-; CHECK: // .b8 0                                // EOM(2)
-; CHECK: // .b8 29                               // Abbreviation Code
-; CHECK: // .b8 5                                // DW_TAG_formal_parameter
-; CHECK: // .b8 0                                // DW_CHILDREN_no
-; CHECK: // .b8 73                               // DW_AT_type
-; CHECK: // .b8 19                               // DW_FORM_ref4
-; CHECK: // .b8 52                               // DW_AT_artificial
-; CHECK: // .b8 12                               // DW_FORM_flag
-; CHECK: // .b8 0                                // EOM(1)
-; CHECK: // .b8 0                                // EOM(2)
-; CHECK: // .b8 30                               // Abbreviation Code
-; CHECK: // .b8 46                               // DW_TAG_subprogram
-; CHECK: // .b8 1                                // DW_CHILDREN_yes
-; CHECK: // .b8 3                                // DW_AT_name
-; CHECK: // .b8 8                                // DW_FORM_string
-; CHECK: // .b8 58                               // DW_AT_decl_file
-; CHECK: // .b8 11                               // DW_FORM_data1
-; CHECK: // .b8 59                               // DW_AT_decl_line
-; CHECK: // .b8 11                               // DW_FORM_data1
-; CHECK: // .b8 60                               // DW_AT_declaration
-; CHECK: // .b8 12                               // DW_FORM_flag
-; CHECK: // .b8 63                               // DW_AT_external
-; CHECK: // .b8 12                               // DW_FORM_flag
-; CHECK: // .b8 50                               // DW_AT_accessibility
-; CHECK: // .b8 11                               // DW_FORM_data1
-; CHECK: // .b8 0                                // EOM(1)
-; CHECK: // .b8 0                                // EOM(2)
-; CHECK: // .b8 31                               // Abbreviation Code
-; CHECK: // .b8 46                               // DW_TAG_subprogram
-; CHECK: // .b8 1                                // DW_CHILDREN_yes
-; CHECK: // .b8 135,64                           // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 8                                // DW_FORM_string
-; CHECK: // .b8 3                                // DW_AT_name
-; CHECK: // .b8 8                                // DW_FORM_string
-; CHECK: // .b8 58                               // DW_AT_decl_file
-; CHECK: // .b8 11                               // DW_FORM_data1
-; CHECK: // .b8 59                               // DW_AT_decl_line
-; CHECK: // .b8 11                               // DW_FORM_data1
-; CHECK: // .b8 60                               // DW_AT_declaration
-; CHECK: // .b8 12                               // DW_FORM_flag
-; CHECK: // .b8 63                               // DW_AT_external
-; CHECK: // .b8 12                               // DW_FORM_flag
-; CHECK: // .b8 50                               // DW_AT_accessibility
-; CHECK: // .b8 11                               // DW_FORM_data1
-; CHECK: // .b8 0                                // EOM(1)
-; CHECK: // .b8 0                                // EOM(2)
-; CHECK: // .b8 32                               // Abbreviation Code
-; CHECK: // .b8 46                               // DW_TAG_subprogram
-; CHECK: // .b8 1                                // DW_CHILDREN_yes
-; CHECK: // .b8 135,64                           // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 8                                // DW_FORM_string
-; CHECK: // .b8 3                                // DW_AT_name
-; CHECK: // .b8 8                                // DW_FORM_string
-; CHECK: // .b8 58                               // DW_AT_decl_file
-; CHECK: // .b8 11                               // DW_FORM_data1
-; CHECK: // .b8 59                               // DW_AT_decl_line
-; CHECK: // .b8 11                               // DW_FORM_data1
-; CHECK: // .b8 73                               // DW_AT_type
-; CHECK: // .b8 19                               // DW_FORM_ref4
-; CHECK: // .b8 60                               // DW_AT_declaration
-; CHECK: // .b8 12                               // DW_FORM_flag
-; CHECK: // .b8 63                               // DW_AT_external
-; CHECK: // .b8 12                               // DW_FORM_flag
-; CHECK: // .b8 50                               // DW_AT_accessibility
-; CHECK: // .b8 11                               // DW_FORM_data1
-; CHECK: // .b8 0                                // EOM(1)
-; CHECK: // .b8 0                                // EOM(2)
-; CHECK: // .b8 33                               // Abbreviation Code
-; CHECK: // .b8 16                               // DW_TAG_reference_type
-; CHECK: // .b8 0                                // DW_CHILDREN_no
-; CHECK: // .b8 73                               // DW_AT_type
-; CHECK: // .b8 19                               // DW_FORM_ref4
-; CHECK: // .b8 0                                // EOM(1)
-; CHECK: // .b8 0                                // EOM(2)
-; CHECK: // .b8 34                               // Abbreviation Code
-; CHECK: // .b8 46                               // DW_TAG_subprogram
-; CHECK: // .b8 0                                // DW_CHILDREN_no
-; CHECK: // .b8 71                               // DW_AT_specification
-; CHECK: // .b8 19                               // DW_FORM_ref4
-; CHECK: // .b8 32                               // DW_AT_inline
-; CHECK: // .b8 11                               // DW_FORM_data1
-; CHECK: // .b8 0                                // EOM(1)
-; CHECK: // .b8 0                                // EOM(2)
-; CHECK: // .b8 35                               // Abbreviation Code
-; CHECK: // .b8 19                               // DW_TAG_structure_type
-; CHECK: // .b8 1                                // DW_CHILDREN_yes
-; CHECK: // .b8 3                                // DW_AT_name
-; CHECK: // .b8 8                                // DW_FORM_string
-; CHECK: // .b8 11                               // DW_AT_byte_size
-; CHECK: // .b8 11                               // DW_FORM_data1
-; CHECK: // .b8 58                               // DW_AT_decl_file
-; CHECK: // .b8 11                               // DW_FORM_data1
-; CHECK: // .b8 59                               // DW_AT_decl_line
-; CHECK: // .b8 5                                // DW_FORM_data2
-; CHECK: // .b8 0                                // EOM(1)
-; CHECK: // .b8 0                                // EOM(2)
-; CHECK: // .b8 36                               // Abbreviation Code
-; CHECK: // .b8 13                               // DW_TAG_member
-; CHECK: // .b8 0                                // DW_CHILDREN_no
-; CHECK: // .b8 3                                // DW_AT_name
-; CHECK: // .b8 8                                // DW_FORM_string
-; CHECK: // .b8 73                               // DW_AT_type
-; CHECK: // .b8 19                               // DW_FORM_ref4
-; CHECK: // .b8 58                               // DW_AT_decl_file
-; CHECK: // .b8 11                               // DW_FORM_data1
-; CHECK: // .b8 59                               // DW_AT_decl_line
-; CHECK: // .b8 5                                // DW_FORM_data2
-; CHECK: // .b8 56                               // DW_AT_data_member_location
-; CHECK: // .b8 10                               // DW_FORM_block1
-; CHECK: // .b8 0                                // EOM(1)
-; CHECK: // .b8 0                                // EOM(2)
-; CHECK: // .b8 37                               // Abbreviation Code
-; CHECK: // .b8 46                               // DW_TAG_subprogram
-; CHECK: // .b8 1                                // DW_CHILDREN_yes
-; CHECK: // .b8 135,64                           // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 8                                // DW_FORM_string
-; CHECK: // .b8 3                                // DW_AT_name
-; CHECK: // .b8 8                                // DW_FORM_string
-; CHECK: // .b8 58                               // DW_AT_decl_file
-; CHECK: // .b8 11                               // DW_FORM_data1
-; CHECK: // .b8 59                               // DW_AT_decl_line
-; CHECK: // .b8 5                                // DW_FORM_data2
-; CHECK: // .b8 73                               // DW_AT_type
-; CHECK: // .b8 19                               // DW_FORM_ref4
-; CHECK: // .b8 60                               // DW_AT_declaration
-; CHECK: // .b8 12                               // DW_FORM_flag
-; CHECK: // .b8 63                               // DW_AT_external
-; CHECK: // .b8 12                               // DW_FORM_flag
-; CHECK: // .b8 0                                // EOM(1)
-; CHECK: // .b8 0                                // EOM(2)
-; CHECK: // .b8 38                               // Abbreviation Code
-; CHECK: // .b8 46                               // DW_TAG_subprogram
-; CHECK: // .b8 1                                // DW_CHILDREN_yes
-; CHECK: // .b8 135,64                           // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 8                                // DW_FORM_string
-; CHECK: // .b8 3                                // DW_AT_name
-; CHECK: // .b8 8                                // DW_FORM_string
-; CHECK: // .b8 58                               // DW_AT_decl_file
-; CHECK: // .b8 11                               // DW_FORM_data1
-; CHECK: // .b8 59                               // DW_AT_decl_line
-; CHECK: // .b8 11                               // DW_FORM_data1
-; CHECK: // .b8 63                               // DW_AT_external
-; CHECK: // .b8 12                               // DW_FORM_flag
-; CHECK: // .b8 32                               // DW_AT_inline
-; CHECK: // .b8 11                               // DW_FORM_data1
-; CHECK: // .b8 0                                // EOM(1)
-; CHECK: // .b8 0                                // EOM(2)
-; CHECK: // .b8 39                               // Abbreviation Code
-; CHECK: // .b8 5                                // DW_TAG_formal_parameter
-; CHECK: // .b8 0                                // DW_CHILDREN_no
-; CHECK: // .b8 3                                // DW_AT_name
-; CHECK: // .b8 8                                // DW_FORM_string
-; CHECK: // .b8 58                               // DW_AT_decl_file
-; CHECK: // .b8 11                               // DW_FORM_data1
-; CHECK: // .b8 59                               // DW_AT_decl_line
-; CHECK: // .b8 11                               // DW_FORM_data1
-; CHECK: // .b8 73                               // DW_AT_type
-; CHECK: // .b8 19                               // DW_FORM_ref4
-; CHECK: // .b8 0                                // EOM(1)
-; CHECK: // .b8 0                                // EOM(2)
-; CHECK: // .b8 40                               // Abbreviation Code
-; CHECK: // .b8 46                               // DW_TAG_subprogram
-; CHECK: // .b8 1                                // DW_CHILDREN_yes
-; CHECK: // .b8 17                               // DW_AT_low_pc
-; CHECK: // .b8 1                                // DW_FORM_addr
-; CHECK: // .b8 18                               // DW_AT_high_pc
-; CHECK: // .b8 1                                // DW_FORM_addr
-; CHECK: // .b8 64                               // DW_AT_frame_base
-; CHECK: // .b8 10                               // DW_FORM_block1
-; CHECK: // .b8 135,64                           // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 8                                // DW_FORM_string
-; CHECK: // .b8 3                                // DW_AT_name
-; CHECK: // .b8 8                                // DW_FORM_string
-; CHECK: // .b8 58                               // DW_AT_decl_file
-; CHECK: // .b8 11                               // DW_FORM_data1
-; CHECK: // .b8 59                               // DW_AT_decl_line
-; CHECK: // .b8 11                               // DW_FORM_data1
-; CHECK: // .b8 63                               // DW_AT_external
-; CHECK: // .b8 12                               // DW_FORM_flag
-; CHECK: // .b8 0                                // EOM(1)
-; CHECK: // .b8 0                                // EOM(2)
-; CHECK: // .b8 41                               // Abbreviation Code
-; CHECK: // .b8 52                               // DW_TAG_variable
-; CHECK: // .b8 0                                // DW_CHILDREN_no
-; CHECK: // .b8 3                                // DW_AT_name
-; CHECK: // .b8 8                                // DW_FORM_string
-; CHECK: // .b8 58                               // DW_AT_decl_file
-; CHECK: // .b8 11                               // DW_FORM_data1
-; CHECK: // .b8 59                               // DW_AT_decl_line
-; CHECK: // .b8 11                               // DW_FORM_data1
-; CHECK: // .b8 73                               // DW_AT_type
-; CHECK: // .b8 19                               // DW_FORM_ref4
-; CHECK: // .b8 0                                // EOM(1)
-; CHECK: // .b8 0                                // EOM(2)
-; CHECK: // .b8 42                               // Abbreviation Code
-; CHECK: // .b8 29                               // DW_TAG_inlined_subroutine
-; CHECK: // .b8 0                                // DW_CHILDREN_no
-; CHECK: // .b8 49                               // DW_AT_abstract_origin
-; CHECK: // .b8 19                               // DW_FORM_ref4
-; CHECK: // .b8 17                               // DW_AT_low_pc
-; CHECK: // .b8 1                                // DW_FORM_addr
-; CHECK: // .b8 18                               // DW_AT_high_pc
-; CHECK: // .b8 1                                // DW_FORM_addr
-; CHECK: // .b8 88                               // DW_AT_call_file
-; CHECK: // .b8 11                               // DW_FORM_data1
-; CHECK: // .b8 89                               // DW_AT_call_line
-; CHECK: // .b8 11                               // DW_FORM_data1
-; CHECK: // .b8 0                                // EOM(1)
-; CHECK: // .b8 0                                // EOM(2)
-; CHECK: // .b8 43                               // Abbreviation Code
-; CHECK: // .b8 29                               // DW_TAG_inlined_subroutine
-; CHECK: // .b8 1                                // DW_CHILDREN_yes
-; CHECK: // .b8 49                               // DW_AT_abstract_origin
-; CHECK: // .b8 19                               // DW_FORM_ref4
-; CHECK: // .b8 17                               // DW_AT_low_pc
-; CHECK: // .b8 1                                // DW_FORM_addr
-; CHECK: // .b8 18                               // DW_AT_high_pc
-; CHECK: // .b8 1                                // DW_FORM_addr
-; CHECK: // .b8 88                               // DW_AT_call_file
-; CHECK: // .b8 11                               // DW_FORM_data1
-; CHECK: // .b8 89                               // DW_AT_call_line
-; CHECK: // .b8 11                               // DW_FORM_data1
-; CHECK: // .b8 0                                // EOM(1)
-; CHECK: // .b8 0                                // EOM(2)
-; CHECK: // .b8 44                               // Abbreviation Code
-; CHECK: // .b8 5                                // DW_TAG_formal_parameter
-; CHECK: // .b8 0                                // DW_CHILDREN_no
-; CHECK: // .b8 49                               // DW_AT_abstract_origin
-; CHECK: // .b8 19                               // DW_FORM_ref4
-; CHECK: // .b8 0                                // EOM(1)
-; CHECK: // .b8 0                                // EOM(2)
-; CHECK: // .b8 0                                // EOM(3)
-; CHECK: // }
-; CHECK: // .section .debug_info
-; CHECK: // {
-; CHECK: // .b32 10030                           // Length of Unit
-; CHECK: // .b8 2                                // DWARF version number
-; CHECK: // .b8 0
-; CHECK: // .b32 .debug_abbrev                   // Offset Into Abbrev. Section
-; CHECK: // .b8 8                                // Address Size (in bytes)
-; CHECK: // .b8 1                                // Abbrev [1] 0xb:0x2727 DW_TAG_compile_unit
-; CHECK: // .b8 0                                // DW_AT_producer
-; CHECK: // .b8 4                                // DW_AT_language
-; CHECK: // .b8 0
-; CHECK: // .b8 100,101,98,117,103,45,105,110,102,111,46,99,117 // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b32 .debug_line                     // DW_AT_stmt_list
-; CHECK: // .b8 47,115,111,109,101,47,100,105,114,101,99,116,111,114,121 // DW_AT_comp_dir
-; CHECK: // .b8 0
-; CHECK: // .b64 Lfunc_begin0                    // DW_AT_low_pc
-; CHECK: // .b64 Lfunc_end0                      // DW_AT_high_pc
-; CHECK: // .b8 2                                // Abbrev [2] 0x41:0x588 DW_TAG_namespace
-; CHECK: // .b8 115,116,100                      // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 3                                // Abbrev [3] 0x46:0x7 DW_TAG_imported_declaration
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 202                              // DW_AT_decl_line
-; CHECK: // .b32 1481                            // DW_AT_import
-; CHECK: // .b8 3                                // Abbrev [3] 0x4d:0x7 DW_TAG_imported_declaration
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 203                              // DW_AT_decl_line
-; CHECK: // .b32 1525                            // DW_AT_import
-; CHECK: // .b8 3                                // Abbrev [3] 0x54:0x7 DW_TAG_imported_declaration
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 204                              // DW_AT_decl_line
-; CHECK: // .b32 1563                            // DW_AT_import
-; CHECK: // .b8 3                                // Abbrev [3] 0x5b:0x7 DW_TAG_imported_declaration
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 205                              // DW_AT_decl_line
-; CHECK: // .b32 1594                            // DW_AT_import
-; CHECK: // .b8 3                                // Abbrev [3] 0x62:0x7 DW_TAG_imported_declaration
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 206                              // DW_AT_decl_line
-; CHECK: // .b32 1623                            // DW_AT_import
-; CHECK: // .b8 3                                // Abbrev [3] 0x69:0x7 DW_TAG_imported_declaration
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 207                              // DW_AT_decl_line
-; CHECK: // .b32 1654                            // DW_AT_import
-; CHECK: // .b8 3                                // Abbrev [3] 0x70:0x7 DW_TAG_imported_declaration
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 208                              // DW_AT_decl_line
-; CHECK: // .b32 1683                            // DW_AT_import
-; CHECK: // .b8 3                                // Abbrev [3] 0x77:0x7 DW_TAG_imported_declaration
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 209                              // DW_AT_decl_line
-; CHECK: // .b32 1720                            // DW_AT_import
-; CHECK: // .b8 3                                // Abbrev [3] 0x7e:0x7 DW_TAG_imported_declaration
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 210                              // DW_AT_decl_line
-; CHECK: // .b32 1751                            // DW_AT_import
-; CHECK: // .b8 3                                // Abbrev [3] 0x85:0x7 DW_TAG_imported_declaration
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 211                              // DW_AT_decl_line
-; CHECK: // .b32 1780                            // DW_AT_import
-; CHECK: // .b8 3                                // Abbrev [3] 0x8c:0x7 DW_TAG_imported_declaration
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 212                              // DW_AT_decl_line
-; CHECK: // .b32 1809                            // DW_AT_import
-; CHECK: // .b8 3                                // Abbrev [3] 0x93:0x7 DW_TAG_imported_declaration
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 213                              // DW_AT_decl_line
-; CHECK: // .b32 1852                            // DW_AT_import
-; CHECK: // .b8 3                                // Abbrev [3] 0x9a:0x7 DW_TAG_imported_declaration
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 214                              // DW_AT_decl_line
-; CHECK: // .b32 1879                            // DW_AT_import
-; CHECK: // .b8 3                                // Abbrev [3] 0xa1:0x7 DW_TAG_imported_declaration
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 215                              // DW_AT_decl_line
-; CHECK: // .b32 1908                            // DW_AT_import
-; CHECK: // .b8 3                                // Abbrev [3] 0xa8:0x7 DW_TAG_imported_declaration
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 216                              // DW_AT_decl_line
-; CHECK: // .b32 1935                            // DW_AT_import
-; CHECK: // .b8 3                                // Abbrev [3] 0xaf:0x7 DW_TAG_imported_declaration
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 217                              // DW_AT_decl_line
-; CHECK: // .b32 1964                            // DW_AT_import
-; CHECK: // .b8 3                                // Abbrev [3] 0xb6:0x7 DW_TAG_imported_declaration
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 218                              // DW_AT_decl_line
-; CHECK: // .b32 1991                            // DW_AT_import
-; CHECK: // .b8 3                                // Abbrev [3] 0xbd:0x7 DW_TAG_imported_declaration
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 219                              // DW_AT_decl_line
-; CHECK: // .b32 2020                            // DW_AT_import
-; CHECK: // .b8 3                                // Abbrev [3] 0xc4:0x7 DW_TAG_imported_declaration
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 220                              // DW_AT_decl_line
-; CHECK: // .b32 2051                            // DW_AT_import
-; CHECK: // .b8 3                                // Abbrev [3] 0xcb:0x7 DW_TAG_imported_declaration
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 221                              // DW_AT_decl_line
-; CHECK: // .b32 2080                            // DW_AT_import
-; CHECK: // .b8 3                                // Abbrev [3] 0xd2:0x7 DW_TAG_imported_declaration
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 222                              // DW_AT_decl_line
-; CHECK: // .b32 2115                            // DW_AT_import
-; CHECK: // .b8 3                                // Abbrev [3] 0xd9:0x7 DW_TAG_imported_declaration
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 223                              // DW_AT_decl_line
-; CHECK: // .b32 2146                            // DW_AT_import
-; CHECK: // .b8 3                                // Abbrev [3] 0xe0:0x7 DW_TAG_imported_declaration
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 224                              // DW_AT_decl_line
-; CHECK: // .b32 2185                            // DW_AT_import
-; CHECK: // .b8 3                                // Abbrev [3] 0xe7:0x7 DW_TAG_imported_declaration
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 225                              // DW_AT_decl_line
-; CHECK: // .b32 2220                            // DW_AT_import
-; CHECK: // .b8 3                                // Abbrev [3] 0xee:0x7 DW_TAG_imported_declaration
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 226                              // DW_AT_decl_line
-; CHECK: // .b32 2255                            // DW_AT_import
-; CHECK: // .b8 3                                // Abbrev [3] 0xf5:0x7 DW_TAG_imported_declaration
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 227                              // DW_AT_decl_line
-; CHECK: // .b32 2290                            // DW_AT_import
-; CHECK: // .b8 3                                // Abbrev [3] 0xfc:0x7 DW_TAG_imported_declaration
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 228                              // DW_AT_decl_line
-; CHECK: // .b32 2339                            // DW_AT_import
-; CHECK: // .b8 3                                // Abbrev [3] 0x103:0x7 DW_TAG_imported_declaration
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 229                              // DW_AT_decl_line
-; CHECK: // .b32 2382                            // DW_AT_import
-; CHECK: // .b8 3                                // Abbrev [3] 0x10a:0x7 DW_TAG_imported_declaration
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 230                              // DW_AT_decl_line
-; CHECK: // .b32 2419                            // DW_AT_import
-; CHECK: // .b8 3                                // Abbrev [3] 0x111:0x7 DW_TAG_imported_declaration
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 231                              // DW_AT_decl_line
-; CHECK: // .b32 2450                            // DW_AT_import
-; CHECK: // .b8 3                                // Abbrev [3] 0x118:0x7 DW_TAG_imported_declaration
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 232                              // DW_AT_decl_line
-; CHECK: // .b32 2495                            // DW_AT_import
-; CHECK: // .b8 3                                // Abbrev [3] 0x11f:0x7 DW_TAG_imported_declaration
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 233                              // DW_AT_decl_line
-; CHECK: // .b32 2540                            // DW_AT_import
-; CHECK: // .b8 3                                // Abbrev [3] 0x126:0x7 DW_TAG_imported_declaration
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 234                              // DW_AT_decl_line
-; CHECK: // .b32 2596                            // DW_AT_import
-; CHECK: // .b8 3                                // Abbrev [3] 0x12d:0x7 DW_TAG_imported_declaration
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 235                              // DW_AT_decl_line
-; CHECK: // .b32 2627                            // DW_AT_import
-; CHECK: // .b8 3                                // Abbrev [3] 0x134:0x7 DW_TAG_imported_declaration
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 236                              // DW_AT_decl_line
-; CHECK: // .b32 2666                            // DW_AT_import
-; CHECK: // .b8 3                                // Abbrev [3] 0x13b:0x7 DW_TAG_imported_declaration
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 237                              // DW_AT_decl_line
-; CHECK: // .b32 2716                            // DW_AT_import
-; CHECK: // .b8 3                                // Abbrev [3] 0x142:0x7 DW_TAG_imported_declaration
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 238                              // DW_AT_decl_line
-; CHECK: // .b32 2770                            // DW_AT_import
-; CHECK: // .b8 3                                // Abbrev [3] 0x149:0x7 DW_TAG_imported_declaration
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 239                              // DW_AT_decl_line
-; CHECK: // .b32 2801                            // DW_AT_import
-; CHECK: // .b8 3                                // Abbrev [3] 0x150:0x7 DW_TAG_imported_declaration
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 240                              // DW_AT_decl_line
-; CHECK: // .b32 2838                            // DW_AT_import
-; CHECK: // .b8 3                                // Abbrev [3] 0x157:0x7 DW_TAG_imported_declaration
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 241                              // DW_AT_decl_line
-; CHECK: // .b32 2888                            // DW_AT_import
-; CHECK: // .b8 3                                // Abbrev [3] 0x15e:0x7 DW_TAG_imported_declaration
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 242                              // DW_AT_decl_line
-; CHECK: // .b32 2929                            // DW_AT_import
-; CHECK: // .b8 3                                // Abbrev [3] 0x165:0x7 DW_TAG_imported_declaration
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 243                              // DW_AT_decl_line
-; CHECK: // .b32 2966                            // DW_AT_import
-; CHECK: // .b8 3                                // Abbrev [3] 0x16c:0x7 DW_TAG_imported_declaration
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 244                              // DW_AT_decl_line
-; CHECK: // .b32 2999                            // DW_AT_import
-; CHECK: // .b8 3                                // Abbrev [3] 0x173:0x7 DW_TAG_imported_declaration
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 245                              // DW_AT_decl_line
-; CHECK: // .b32 3030                            // DW_AT_import
-; CHECK: // .b8 3                                // Abbrev [3] 0x17a:0x7 DW_TAG_imported_declaration
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 246                              // DW_AT_decl_line
-; CHECK: // .b32 3063                            // DW_AT_import
-; CHECK: // .b8 3                                // Abbrev [3] 0x181:0x7 DW_TAG_imported_declaration
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 247                              // DW_AT_decl_line
-; CHECK: // .b32 3090                            // DW_AT_import
-; CHECK: // .b8 3                                // Abbrev [3] 0x188:0x7 DW_TAG_imported_declaration
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 248                              // DW_AT_decl_line
-; CHECK: // .b32 3121                            // DW_AT_import
-; CHECK: // .b8 3                                // Abbrev [3] 0x18f:0x7 DW_TAG_imported_declaration
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 249                              // DW_AT_decl_line
-; CHECK: // .b32 3152                            // DW_AT_import
-; CHECK: // .b8 3                                // Abbrev [3] 0x196:0x7 DW_TAG_imported_declaration
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 250                              // DW_AT_decl_line
-; CHECK: // .b32 3181                            // DW_AT_import
-; CHECK: // .b8 3                                // Abbrev [3] 0x19d:0x7 DW_TAG_imported_declaration
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 251                              // DW_AT_decl_line
-; CHECK: // .b32 3210                            // DW_AT_import
-; CHECK: // .b8 3                                // Abbrev [3] 0x1a4:0x7 DW_TAG_imported_declaration
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 252                              // DW_AT_decl_line
-; CHECK: // .b32 3241                            // DW_AT_import
-; CHECK: // .b8 3                                // Abbrev [3] 0x1ab:0x7 DW_TAG_imported_declaration
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 253                              // DW_AT_decl_line
-; CHECK: // .b32 3274                            // DW_AT_import
-; CHECK: // .b8 3                                // Abbrev [3] 0x1b2:0x7 DW_TAG_imported_declaration
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 254                              // DW_AT_decl_line
-; CHECK: // .b32 3309                            // DW_AT_import
-; CHECK: // .b8 3                                // Abbrev [3] 0x1b9:0x7 DW_TAG_imported_declaration
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 255                              // DW_AT_decl_line
-; CHECK: // .b32 3350                            // DW_AT_import
-; CHECK: // .b8 4                                // Abbrev [4] 0x1c0:0x8 DW_TAG_imported_declaration
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 0                                // DW_AT_decl_line
-; CHECK: // .b8 1
-; CHECK: // .b32 3407                            // DW_AT_import
-; CHECK: // .b8 4                                // Abbrev [4] 0x1c8:0x8 DW_TAG_imported_declaration
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 1                                // DW_AT_decl_line
-; CHECK: // .b8 1
-; CHECK: // .b32 3438                            // DW_AT_import
-; CHECK: // .b8 4                                // Abbrev [4] 0x1d0:0x8 DW_TAG_imported_declaration
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 2                                // DW_AT_decl_line
-; CHECK: // .b8 1
-; CHECK: // .b32 3477                            // DW_AT_import
-; CHECK: // .b8 4                                // Abbrev [4] 0x1d8:0x8 DW_TAG_imported_declaration
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 3                                // DW_AT_decl_line
-; CHECK: // .b8 1
-; CHECK: // .b32 3522                            // DW_AT_import
-; CHECK: // .b8 4                                // Abbrev [4] 0x1e0:0x8 DW_TAG_imported_declaration
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 4                                // DW_AT_decl_line
-; CHECK: // .b8 1
-; CHECK: // .b32 3555                            // DW_AT_import
-; CHECK: // .b8 4                                // Abbrev [4] 0x1e8:0x8 DW_TAG_imported_declaration
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 5                                // DW_AT_decl_line
-; CHECK: // .b8 1
-; CHECK: // .b32 3600                            // DW_AT_import
-; CHECK: // .b8 4                                // Abbrev [4] 0x1f0:0x8 DW_TAG_imported_declaration
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 6                                // DW_AT_decl_line
-; CHECK: // .b8 1
-; CHECK: // .b32 3646                            // DW_AT_import
-; CHECK: // .b8 4                                // Abbrev [4] 0x1f8:0x8 DW_TAG_imported_declaration
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 7                                // DW_AT_decl_line
-; CHECK: // .b8 1
-; CHECK: // .b32 3675                            // DW_AT_import
-; CHECK: // .b8 4                                // Abbrev [4] 0x200:0x8 DW_TAG_imported_declaration
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 8                                // DW_AT_decl_line
-; CHECK: // .b8 1
-; CHECK: // .b32 3706                            // DW_AT_import
-; CHECK: // .b8 4                                // Abbrev [4] 0x208:0x8 DW_TAG_imported_declaration
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 9                                // DW_AT_decl_line
-; CHECK: // .b8 1
-; CHECK: // .b32 3747                            // DW_AT_import
-; CHECK: // .b8 4                                // Abbrev [4] 0x210:0x8 DW_TAG_imported_declaration
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 10                               // DW_AT_decl_line
-; CHECK: // .b8 1
-; CHECK: // .b32 3786                            // DW_AT_import
-; CHECK: // .b8 4                                // Abbrev [4] 0x218:0x8 DW_TAG_imported_declaration
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 11                               // DW_AT_decl_line
-; CHECK: // .b8 1
-; CHECK: // .b32 3821                            // DW_AT_import
-; CHECK: // .b8 4                                // Abbrev [4] 0x220:0x8 DW_TAG_imported_declaration
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 12                               // DW_AT_decl_line
-; CHECK: // .b8 1
-; CHECK: // .b32 3848                            // DW_AT_import
-; CHECK: // .b8 4                                // Abbrev [4] 0x228:0x8 DW_TAG_imported_declaration
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 13                               // DW_AT_decl_line
-; CHECK: // .b8 1
-; CHECK: // .b32 3877                            // DW_AT_import
-; CHECK: // .b8 4                                // Abbrev [4] 0x230:0x8 DW_TAG_imported_declaration
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 14                               // DW_AT_decl_line
-; CHECK: // .b8 1
-; CHECK: // .b32 3906                            // DW_AT_import
-; CHECK: // .b8 4                                // Abbrev [4] 0x238:0x8 DW_TAG_imported_declaration
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 15                               // DW_AT_decl_line
-; CHECK: // .b8 1
-; CHECK: // .b32 3933                            // DW_AT_import
-; CHECK: // .b8 4                                // Abbrev [4] 0x240:0x8 DW_TAG_imported_declaration
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 16                               // DW_AT_decl_line
-; CHECK: // .b8 1
-; CHECK: // .b32 3962                            // DW_AT_import
-; CHECK: // .b8 4                                // Abbrev [4] 0x248:0x8 DW_TAG_imported_declaration
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 17                               // DW_AT_decl_line
-; CHECK: // .b8 1
-; CHECK: // .b32 3995                            // DW_AT_import
-; CHECK: // .b8 3                                // Abbrev [3] 0x250:0x7 DW_TAG_imported_declaration
-; CHECK: // .b8 3                                // DW_AT_decl_file
-; CHECK: // .b8 102                              // DW_AT_decl_line
-; CHECK: // .b32 4026                            // DW_AT_import
-; CHECK: // .b8 3                                // Abbrev [3] 0x257:0x7 DW_TAG_imported_declaration
-; CHECK: // .b8 3                                // DW_AT_decl_file
-; CHECK: // .b8 121                              // DW_AT_decl_line
-; CHECK: // .b32 4046                            // DW_AT_import
-; CHECK: // .b8 3                                // Abbrev [3] 0x25e:0x7 DW_TAG_imported_declaration
-; CHECK: // .b8 3                                // DW_AT_decl_file
-; CHECK: // .b8 140                              // DW_AT_decl_line
-; CHECK: // .b32 4066                            // DW_AT_import
-; CHECK: // .b8 3                                // Abbrev [3] 0x265:0x7 DW_TAG_imported_declaration
-; CHECK: // .b8 3                                // DW_AT_decl_file
-; CHECK: // .b8 159                              // DW_AT_decl_line
-; CHECK: // .b32 4086                            // DW_AT_import
-; CHECK: // .b8 3                                // Abbrev [3] 0x26c:0x7 DW_TAG_imported_declaration
-; CHECK: // .b8 3                                // DW_AT_decl_file
-; CHECK: // .b8 180                              // DW_AT_decl_line
-; CHECK: // .b32 4112                            // DW_AT_import
-; CHECK: // .b8 3                                // Abbrev [3] 0x273:0x7 DW_TAG_imported_declaration
-; CHECK: // .b8 3                                // DW_AT_decl_file
-; CHECK: // .b8 199                              // DW_AT_decl_line
-; CHECK: // .b32 4132                            // DW_AT_import
-; CHECK: // .b8 3                                // Abbrev [3] 0x27a:0x7 DW_TAG_imported_declaration
-; CHECK: // .b8 3                                // DW_AT_decl_file
-; CHECK: // .b8 218                              // DW_AT_decl_line
-; CHECK: // .b32 4151                            // DW_AT_import
-; CHECK: // .b8 3                                // Abbrev [3] 0x281:0x7 DW_TAG_imported_declaration
-; CHECK: // .b8 3                                // DW_AT_decl_file
-; CHECK: // .b8 237                              // DW_AT_decl_line
-; CHECK: // .b32 4171                            // DW_AT_import
-; CHECK: // .b8 4                                // Abbrev [4] 0x288:0x8 DW_TAG_imported_declaration
-; CHECK: // .b8 3                                // DW_AT_decl_file
-; CHECK: // .b8 0                                // DW_AT_decl_line
-; CHECK: // .b8 1
-; CHECK: // .b32 4190                            // DW_AT_import
-; CHECK: // .b8 4                                // Abbrev [4] 0x290:0x8 DW_TAG_imported_declaration
-; CHECK: // .b8 3                                // DW_AT_decl_file
-; CHECK: // .b8 19                               // DW_AT_decl_line
-; CHECK: // .b8 1
-; CHECK: // .b32 4210                            // DW_AT_import
-; CHECK: // .b8 4                                // Abbrev [4] 0x298:0x8 DW_TAG_imported_declaration
-; CHECK: // .b8 3                                // DW_AT_decl_file
-; CHECK: // .b8 38                               // DW_AT_decl_line
-; CHECK: // .b8 1
-; CHECK: // .b32 4231                            // DW_AT_import
-; CHECK: // .b8 4                                // Abbrev [4] 0x2a0:0x8 DW_TAG_imported_declaration
-; CHECK: // .b8 3                                // DW_AT_decl_file
-; CHECK: // .b8 59                               // DW_AT_decl_line
-; CHECK: // .b8 1
-; CHECK: // .b32 4256                            // DW_AT_import
-; CHECK: // .b8 4                                // Abbrev [4] 0x2a8:0x8 DW_TAG_imported_declaration
-; CHECK: // .b8 3                                // DW_AT_decl_file
-; CHECK: // .b8 78                               // DW_AT_decl_line
-; CHECK: // .b8 1
-; CHECK: // .b32 4282                            // DW_AT_import
-; CHECK: // .b8 4                                // Abbrev [4] 0x2b0:0x8 DW_TAG_imported_declaration
-; CHECK: // .b8 3                                // DW_AT_decl_file
-; CHECK: // .b8 97                               // DW_AT_decl_line
-; CHECK: // .b8 1
-; CHECK: // .b32 4308                            // DW_AT_import
-; CHECK: // .b8 4                                // Abbrev [4] 0x2b8:0x8 DW_TAG_imported_declaration
-; CHECK: // .b8 3                                // DW_AT_decl_file
-; CHECK: // .b8 116                              // DW_AT_decl_line
-; CHECK: // .b8 1
-; CHECK: // .b32 4327                            // DW_AT_import
-; CHECK: // .b8 4                                // Abbrev [4] 0x2c0:0x8 DW_TAG_imported_declaration
-; CHECK: // .b8 3                                // DW_AT_decl_file
-; CHECK: // .b8 135                              // DW_AT_decl_line
-; CHECK: // .b8 1
-; CHECK: // .b32 4348                            // DW_AT_import
-; CHECK: // .b8 4                                // Abbrev [4] 0x2c8:0x8 DW_TAG_imported_declaration
-; CHECK: // .b8 3                                // DW_AT_decl_file
-; CHECK: // .b8 147                              // DW_AT_decl_line
-; CHECK: // .b8 1
-; CHECK: // .b32 4378                            // DW_AT_import
-; CHECK: // .b8 4                                // Abbrev [4] 0x2d0:0x8 DW_TAG_imported_declaration
-; CHECK: // .b8 3                                // DW_AT_decl_file
-; CHECK: // .b8 184                              // DW_AT_decl_line
-; CHECK: // .b8 1
-; CHECK: // .b32 4402                            // DW_AT_import
-; CHECK: // .b8 4                                // Abbrev [4] 0x2d8:0x8 DW_TAG_imported_declaration
-; CHECK: // .b8 3                                // DW_AT_decl_file
-; CHECK: // .b8 203                              // DW_AT_decl_line
-; CHECK: // .b8 1
-; CHECK: // .b32 4421                            // DW_AT_import
-; CHECK: // .b8 4                                // Abbrev [4] 0x2e0:0x8 DW_TAG_imported_declaration
-; CHECK: // .b8 3                                // DW_AT_decl_file
-; CHECK: // .b8 222                              // DW_AT_decl_line
-; CHECK: // .b8 1
-; CHECK: // .b32 4441                            // DW_AT_import
-; CHECK: // .b8 4                                // Abbrev [4] 0x2e8:0x8 DW_TAG_imported_declaration
-; CHECK: // .b8 3                                // DW_AT_decl_file
-; CHECK: // .b8 241                              // DW_AT_decl_line
-; CHECK: // .b8 1
-; CHECK: // .b32 4461                            // DW_AT_import
-; CHECK: // .b8 4                                // Abbrev [4] 0x2f0:0x8 DW_TAG_imported_declaration
-; CHECK: // .b8 3                                // DW_AT_decl_file
-; CHECK: // .b8 4                                // DW_AT_decl_line
-; CHECK: // .b8 2
-; CHECK: // .b32 4480                            // DW_AT_import
-; CHECK: // .b8 3                                // Abbrev [3] 0x2f8:0x7 DW_TAG_imported_declaration
-; CHECK: // .b8 5                                // DW_AT_decl_file
-; CHECK: // .b8 118                              // DW_AT_decl_line
-; CHECK: // .b32 4500                            // DW_AT_import
-; CHECK: // .b8 3                                // Abbrev [3] 0x2ff:0x7 DW_TAG_imported_declaration
-; CHECK: // .b8 5                                // DW_AT_decl_file
-; CHECK: // .b8 119                              // DW_AT_decl_line
-; CHECK: // .b32 4515                            // DW_AT_import
-; CHECK: // .b8 3                                // Abbrev [3] 0x306:0x7 DW_TAG_imported_declaration
-; CHECK: // .b8 5                                // DW_AT_decl_file
-; CHECK: // .b8 121                              // DW_AT_decl_line
-; CHECK: // .b32 4563                            // DW_AT_import
-; CHECK: // .b8 3                                // Abbrev [3] 0x30d:0x7 DW_TAG_imported_declaration
-; CHECK: // .b8 5                                // DW_AT_decl_file
-; CHECK: // .b8 122                              // DW_AT_decl_line
-; CHECK: // .b32 4576                            // DW_AT_import
-; CHECK: // .b8 3                                // Abbrev [3] 0x314:0x7 DW_TAG_imported_declaration
-; CHECK: // .b8 5                                // DW_AT_decl_file
-; CHECK: // .b8 123                              // DW_AT_decl_line
-; CHECK: // .b32 4596                            // DW_AT_import
-; CHECK: // .b8 3                                // Abbrev [3] 0x31b:0x7 DW_TAG_imported_declaration
-; CHECK: // .b8 5                                // DW_AT_decl_file
-; CHECK: // .b8 129                              // DW_AT_decl_line
-; CHECK: // .b32 4625                            // DW_AT_import
-; CHECK: // .b8 3                                // Abbrev [3] 0x322:0x7 DW_TAG_imported_declaration
-; CHECK: // .b8 5                                // DW_AT_decl_file
-; CHECK: // .b8 130                              // DW_AT_decl_line
-; CHECK: // .b32 4645                            // DW_AT_import
-; CHECK: // .b8 3                                // Abbrev [3] 0x329:0x7 DW_TAG_imported_declaration
-; CHECK: // .b8 5                                // DW_AT_decl_file
-; CHECK: // .b8 131                              // DW_AT_decl_line
-; CHECK: // .b32 4666                            // DW_AT_import
-; CHECK: // .b8 3                                // Abbrev [3] 0x330:0x7 DW_TAG_imported_declaration
-; CHECK: // .b8 5                                // DW_AT_decl_file
-; CHECK: // .b8 132                              // DW_AT_decl_line
-; CHECK: // .b32 4687                            // DW_AT_import
-; CHECK: // .b8 3                                // Abbrev [3] 0x337:0x7 DW_TAG_imported_declaration
-; CHECK: // .b8 5                                // DW_AT_decl_file
-; CHECK: // .b8 133                              // DW_AT_decl_line
-; CHECK: // .b32 4815                            // DW_AT_import
-; CHECK: // .b8 3                                // Abbrev [3] 0x33e:0x7 DW_TAG_imported_declaration
-; CHECK: // .b8 5                                // DW_AT_decl_file
-; CHECK: // .b8 134                              // DW_AT_decl_line
-; CHECK: // .b32 4843                            // DW_AT_import
-; CHECK: // .b8 3                                // Abbrev [3] 0x345:0x7 DW_TAG_imported_declaration
-; CHECK: // .b8 5                                // DW_AT_decl_file
-; CHECK: // .b8 135                              // DW_AT_decl_line
-; CHECK: // .b32 4868                            // DW_AT_import
-; CHECK: // .b8 3                                // Abbrev [3] 0x34c:0x7 DW_TAG_imported_declaration
-; CHECK: // .b8 5                                // DW_AT_decl_file
-; CHECK: // .b8 136                              // DW_AT_decl_line
-; CHECK: // .b32 4886                            // DW_AT_import
-; CHECK: // .b8 3                                // Abbrev [3] 0x353:0x7 DW_TAG_imported_declaration
-; CHECK: // .b8 5                                // DW_AT_decl_file
-; CHECK: // .b8 137                              // DW_AT_decl_line
-; CHECK: // .b32 4903                            // DW_AT_import
-; CHECK: // .b8 3                                // Abbrev [3] 0x35a:0x7 DW_TAG_imported_declaration
-; CHECK: // .b8 5                                // DW_AT_decl_file
-; CHECK: // .b8 138                              // DW_AT_decl_line
-; CHECK: // .b32 4931                            // DW_AT_import
-; CHECK: // .b8 3                                // Abbrev [3] 0x361:0x7 DW_TAG_imported_declaration
-; CHECK: // .b8 5                                // DW_AT_decl_file
-; CHECK: // .b8 139                              // DW_AT_decl_line
-; CHECK: // .b32 4952                            // DW_AT_import
-; CHECK: // .b8 3                                // Abbrev [3] 0x368:0x7 DW_TAG_imported_declaration
-; CHECK: // .b8 5                                // DW_AT_decl_file
-; CHECK: // .b8 140                              // DW_AT_decl_line
-; CHECK: // .b32 4978                            // DW_AT_import
-; CHECK: // .b8 3                                // Abbrev [3] 0x36f:0x7 DW_TAG_imported_declaration
-; CHECK: // .b8 5                                // DW_AT_decl_file
-; CHECK: // .b8 142                              // DW_AT_decl_line
-; CHECK: // .b32 5001                            // DW_AT_import
-; CHECK: // .b8 3                                // Abbrev [3] 0x376:0x7 DW_TAG_imported_declaration
-; CHECK: // .b8 5                                // DW_AT_decl_file
-; CHECK: // .b8 143                              // DW_AT_decl_line
-; CHECK: // .b32 5028                            // DW_AT_import
-; CHECK: // .b8 3                                // Abbrev [3] 0x37d:0x7 DW_TAG_imported_declaration
-; CHECK: // .b8 5                                // DW_AT_decl_file
-; CHECK: // .b8 144                              // DW_AT_decl_line
-; CHECK: // .b32 5079                            // DW_AT_import
-; CHECK: // .b8 3                                // Abbrev [3] 0x384:0x7 DW_TAG_imported_declaration
-; CHECK: // .b8 5                                // DW_AT_decl_file
-; CHECK: // .b8 146                              // DW_AT_decl_line
-; CHECK: // .b32 5112                            // DW_AT_import
-; CHECK: // .b8 3                                // Abbrev [3] 0x38b:0x7 DW_TAG_imported_declaration
-; CHECK: // .b8 5                                // DW_AT_decl_file
-; CHECK: // .b8 152                              // DW_AT_decl_line
-; CHECK: // .b32 5145                            // DW_AT_import
-; CHECK: // .b8 3                                // Abbrev [3] 0x392:0x7 DW_TAG_imported_declaration
-; CHECK: // .b8 5                                // DW_AT_decl_file
-; CHECK: // .b8 153                              // DW_AT_decl_line
-; CHECK: // .b32 5160                            // DW_AT_import
-; CHECK: // .b8 3                                // Abbrev [3] 0x399:0x7 DW_TAG_imported_declaration
-; CHECK: // .b8 5                                // DW_AT_decl_file
-; CHECK: // .b8 154                              // DW_AT_decl_line
-; CHECK: // .b32 5189                            // DW_AT_import
-; CHECK: // .b8 3                                // Abbrev [3] 0x3a0:0x7 DW_TAG_imported_declaration
-; CHECK: // .b8 5                                // DW_AT_decl_file
-; CHECK: // .b8 155                              // DW_AT_decl_line
-; CHECK: // .b32 5223                            // DW_AT_import
-; CHECK: // .b8 3                                // Abbrev [3] 0x3a7:0x7 DW_TAG_imported_declaration
-; CHECK: // .b8 5                                // DW_AT_decl_file
-; CHECK: // .b8 156                              // DW_AT_decl_line
-; CHECK: // .b32 5255                            // DW_AT_import
-; CHECK: // .b8 3                                // Abbrev [3] 0x3ae:0x7 DW_TAG_imported_declaration
-; CHECK: // .b8 5                                // DW_AT_decl_file
-; CHECK: // .b8 157                              // DW_AT_decl_line
-; CHECK: // .b32 5287                            // DW_AT_import
-; CHECK: // .b8 3                                // Abbrev [3] 0x3b5:0x7 DW_TAG_imported_declaration
-; CHECK: // .b8 5                                // DW_AT_decl_file
-; CHECK: // .b8 158                              // DW_AT_decl_line
-; CHECK: // .b32 5320                            // DW_AT_import
-; CHECK: // .b8 3                                // Abbrev [3] 0x3bc:0x7 DW_TAG_imported_declaration
-; CHECK: // .b8 5                                // DW_AT_decl_file
-; CHECK: // .b8 160                              // DW_AT_decl_line
-; CHECK: // .b32 5343                            // DW_AT_import
-; CHECK: // .b8 3                                // Abbrev [3] 0x3c3:0x7 DW_TAG_imported_declaration
-; CHECK: // .b8 5                                // DW_AT_decl_file
-; CHECK: // .b8 161                              // DW_AT_decl_line
-; CHECK: // .b32 5388                            // DW_AT_import
-; CHECK: // .b8 3                                // Abbrev [3] 0x3ca:0x7 DW_TAG_imported_declaration
-; CHECK: // .b8 5                                // DW_AT_decl_file
-; CHECK: // .b8 241                              // DW_AT_decl_line
-; CHECK: // .b32 5536                            // DW_AT_import
-; CHECK: // .b8 3                                // Abbrev [3] 0x3d1:0x7 DW_TAG_imported_declaration
-; CHECK: // .b8 5                                // DW_AT_decl_file
-; CHECK: // .b8 243                              // DW_AT_decl_line
-; CHECK: // .b32 5585                            // DW_AT_import
-; CHECK: // .b8 3                                // Abbrev [3] 0x3d8:0x7 DW_TAG_imported_declaration
-; CHECK: // .b8 5                                // DW_AT_decl_file
-; CHECK: // .b8 245                              // DW_AT_decl_line
-; CHECK: // .b32 5604                            // DW_AT_import
-; CHECK: // .b8 3                                // Abbrev [3] 0x3df:0x7 DW_TAG_imported_declaration
-; CHECK: // .b8 5                                // DW_AT_decl_file
-; CHECK: // .b8 246                              // DW_AT_decl_line
-; CHECK: // .b32 5490                            // DW_AT_import
-; CHECK: // .b8 3                                // Abbrev [3] 0x3e6:0x7 DW_TAG_imported_declaration
-; CHECK: // .b8 5                                // DW_AT_decl_file
-; CHECK: // .b8 247                              // DW_AT_decl_line
-; CHECK: // .b32 5626                            // DW_AT_import
-; CHECK: // .b8 3                                // Abbrev [3] 0x3ed:0x7 DW_TAG_imported_declaration
-; CHECK: // .b8 5                                // DW_AT_decl_file
-; CHECK: // .b8 249                              // DW_AT_decl_line
-; CHECK: // .b32 5653                            // DW_AT_import
-; CHECK: // .b8 3                                // Abbrev [3] 0x3f4:0x7 DW_TAG_imported_declaration
-; CHECK: // .b8 5                                // DW_AT_decl_file
-; CHECK: // .b8 250                              // DW_AT_decl_line
-; CHECK: // .b32 5768                            // DW_AT_import
-; CHECK: // .b8 3                                // Abbrev [3] 0x3fb:0x7 DW_TAG_imported_declaration
-; CHECK: // .b8 5                                // DW_AT_decl_file
-; CHECK: // .b8 251                              // DW_AT_decl_line
-; CHECK: // .b32 5675                            // DW_AT_import
-; CHECK: // .b8 3                                // Abbrev [3] 0x402:0x7 DW_TAG_imported_declaration
-; CHECK: // .b8 5                                // DW_AT_decl_file
-; CHECK: // .b8 252                              // DW_AT_decl_line
-; CHECK: // .b32 5708                            // DW_AT_import
-; CHECK: // .b8 3                                // Abbrev [3] 0x409:0x7 DW_TAG_imported_declaration
-; CHECK: // .b8 5                                // DW_AT_decl_file
-; CHECK: // .b8 253                              // DW_AT_decl_line
-; CHECK: // .b32 5795                            // DW_AT_import
-; CHECK: // .b8 4                                // Abbrev [4] 0x410:0x8 DW_TAG_imported_declaration
-; CHECK: // .b8 10                               // DW_AT_decl_file
-; CHECK: // .b8 149                              // DW_AT_decl_line
-; CHECK: // .b8 1
-; CHECK: // .b32 5838                            // DW_AT_import
-; CHECK: // .b8 4                                // Abbrev [4] 0x418:0x8 DW_TAG_imported_declaration
-; CHECK: // .b8 10                               // DW_AT_decl_file
-; CHECK: // .b8 150                              // DW_AT_decl_line
-; CHECK: // .b8 1
-; CHECK: // .b32 5870                            // DW_AT_import
-; CHECK: // .b8 4                                // Abbrev [4] 0x420:0x8 DW_TAG_imported_declaration
-; CHECK: // .b8 10                               // DW_AT_decl_file
-; CHECK: // .b8 151                              // DW_AT_decl_line
-; CHECK: // .b8 1
-; CHECK: // .b32 5904                            // DW_AT_import
-; CHECK: // .b8 4                                // Abbrev [4] 0x428:0x8 DW_TAG_imported_declaration
-; CHECK: // .b8 10                               // DW_AT_decl_file
-; CHECK: // .b8 152                              // DW_AT_decl_line
-; CHECK: // .b8 1
-; CHECK: // .b32 5936                            // DW_AT_import
-; CHECK: // .b8 4                                // Abbrev [4] 0x430:0x8 DW_TAG_imported_declaration
-; CHECK: // .b8 10                               // DW_AT_decl_file
-; CHECK: // .b8 153                              // DW_AT_decl_line
-; CHECK: // .b8 1
-; CHECK: // .b32 5970                            // DW_AT_import
-; CHECK: // .b8 4                                // Abbrev [4] 0x438:0x8 DW_TAG_imported_declaration
-; CHECK: // .b8 10                               // DW_AT_decl_file
-; CHECK: // .b8 154                              // DW_AT_decl_line
-; CHECK: // .b8 1
-; CHECK: // .b32 6010                            // DW_AT_import
-; CHECK: // .b8 4                                // Abbrev [4] 0x440:0x8 DW_TAG_imported_declaration
-; CHECK: // .b8 10                               // DW_AT_decl_file
-; CHECK: // .b8 155                              // DW_AT_decl_line
-; CHECK: // .b8 1
-; CHECK: // .b32 6042                            // DW_AT_import
-; CHECK: // .b8 4                                // Abbrev [4] 0x448:0x8 DW_TAG_imported_declaration
-; CHECK: // .b8 10                               // DW_AT_decl_file
-; CHECK: // .b8 156                              // DW_AT_decl_line
-; CHECK: // .b8 1
-; CHECK: // .b32 6076                            // DW_AT_import
-; CHECK: // .b8 4                                // Abbrev [4] 0x450:0x8 DW_TAG_imported_declaration
-; CHECK: // .b8 10                               // DW_AT_decl_file
-; CHECK: // .b8 157                              // DW_AT_decl_line
-; CHECK: // .b8 1
-; CHECK: // .b32 6108                            // DW_AT_import
-; CHECK: // .b8 4                                // Abbrev [4] 0x458:0x8 DW_TAG_imported_declaration
-; CHECK: // .b8 10                               // DW_AT_decl_file
-; CHECK: // .b8 158                              // DW_AT_decl_line
-; CHECK: // .b8 1
-; CHECK: // .b32 6140                            // DW_AT_import
-; CHECK: // .b8 4                                // Abbrev [4] 0x460:0x8 DW_TAG_imported_declaration
-; CHECK: // .b8 10                               // DW_AT_decl_file
-; CHECK: // .b8 159                              // DW_AT_decl_line
-; CHECK: // .b8 1
-; CHECK: // .b32 6186                            // DW_AT_import
-; CHECK: // .b8 4                                // Abbrev [4] 0x468:0x8 DW_TAG_imported_declaration
-; CHECK: // .b8 10                               // DW_AT_decl_file
-; CHECK: // .b8 160                              // DW_AT_decl_line
-; CHECK: // .b8 1
-; CHECK: // .b32 6216                            // DW_AT_import
-; CHECK: // .b8 4                                // Abbrev [4] 0x470:0x8 DW_TAG_imported_declaration
-; CHECK: // .b8 10                               // DW_AT_decl_file
-; CHECK: // .b8 161                              // DW_AT_decl_line
-; CHECK: // .b8 1
-; CHECK: // .b32 6248                            // DW_AT_import
-; CHECK: // .b8 4                                // Abbrev [4] 0x478:0x8 DW_TAG_imported_declaration
-; CHECK: // .b8 10                               // DW_AT_decl_file
-; CHECK: // .b8 162                              // DW_AT_decl_line
-; CHECK: // .b8 1
-; CHECK: // .b32 6280                            // DW_AT_import
-; CHECK: // .b8 4                                // Abbrev [4] 0x480:0x8 DW_TAG_imported_declaration
-; CHECK: // .b8 10                               // DW_AT_decl_file
-; CHECK: // .b8 163                              // DW_AT_decl_line
-; CHECK: // .b8 1
-; CHECK: // .b32 6310                            // DW_AT_import
-; CHECK: // .b8 4                                // Abbrev [4] 0x488:0x8 DW_TAG_imported_declaration
-; CHECK: // .b8 10                               // DW_AT_decl_file
-; CHECK: // .b8 164                              // DW_AT_decl_line
-; CHECK: // .b8 1
-; CHECK: // .b32 6342                            // DW_AT_import
-; CHECK: // .b8 4                                // Abbrev [4] 0x490:0x8 DW_TAG_imported_declaration
-; CHECK: // .b8 10                               // DW_AT_decl_file
-; CHECK: // .b8 165                              // DW_AT_decl_line
-; CHECK: // .b8 1
-; CHECK: // .b32 6372                            // DW_AT_import
-; CHECK: // .b8 4                                // Abbrev [4] 0x498:0x8 DW_TAG_imported_declaration
-; CHECK: // .b8 10                               // DW_AT_decl_file
-; CHECK: // .b8 166                              // DW_AT_decl_line
-; CHECK: // .b8 1
-; CHECK: // .b32 6406                            // DW_AT_import
-; CHECK: // .b8 4                                // Abbrev [4] 0x4a0:0x8 DW_TAG_imported_declaration
-; CHECK: // .b8 10                               // DW_AT_decl_file
-; CHECK: // .b8 167                              // DW_AT_decl_line
-; CHECK: // .b8 1
-; CHECK: // .b32 6438                            // DW_AT_import
-; CHECK: // .b8 4                                // Abbrev [4] 0x4a8:0x8 DW_TAG_imported_declaration
-; CHECK: // .b8 10                               // DW_AT_decl_file
-; CHECK: // .b8 168                              // DW_AT_decl_line
-; CHECK: // .b8 1
-; CHECK: // .b32 6476                            // DW_AT_import
-; CHECK: // .b8 4                                // Abbrev [4] 0x4b0:0x8 DW_TAG_imported_declaration
-; CHECK: // .b8 10                               // DW_AT_decl_file
-; CHECK: // .b8 169                              // DW_AT_decl_line
-; CHECK: // .b8 1
-; CHECK: // .b32 6510                            // DW_AT_import
-; CHECK: // .b8 4                                // Abbrev [4] 0x4b8:0x8 DW_TAG_imported_declaration
-; CHECK: // .b8 10                               // DW_AT_decl_file
-; CHECK: // .b8 170                              // DW_AT_decl_line
-; CHECK: // .b8 1
-; CHECK: // .b32 6552                            // DW_AT_import
-; CHECK: // .b8 4                                // Abbrev [4] 0x4c0:0x8 DW_TAG_imported_declaration
-; CHECK: // .b8 10                               // DW_AT_decl_file
-; CHECK: // .b8 171                              // DW_AT_decl_line
-; CHECK: // .b8 1
-; CHECK: // .b32 6590                            // DW_AT_import
-; CHECK: // .b8 4                                // Abbrev [4] 0x4c8:0x8 DW_TAG_imported_declaration
-; CHECK: // .b8 10                               // DW_AT_decl_file
-; CHECK: // .b8 172                              // DW_AT_decl_line
-; CHECK: // .b8 1
-; CHECK: // .b32 6628                            // DW_AT_import
-; CHECK: // .b8 4                                // Abbrev [4] 0x4d0:0x8 DW_TAG_imported_declaration
-; CHECK: // .b8 10                               // DW_AT_decl_file
-; CHECK: // .b8 173                              // DW_AT_decl_line
-; CHECK: // .b8 1
-; CHECK: // .b32 6666                            // DW_AT_import
-; CHECK: // .b8 4                                // Abbrev [4] 0x4d8:0x8 DW_TAG_imported_declaration
-; CHECK: // .b8 10                               // DW_AT_decl_file
-; CHECK: // .b8 174                              // DW_AT_decl_line
-; CHECK: // .b8 1
-; CHECK: // .b32 6707                            // DW_AT_import
-; CHECK: // .b8 4                                // Abbrev [4] 0x4e0:0x8 DW_TAG_imported_declaration
-; CHECK: // .b8 10                               // DW_AT_decl_file
-; CHECK: // .b8 175                              // DW_AT_decl_line
-; CHECK: // .b8 1
-; CHECK: // .b32 6747                            // DW_AT_import
-; CHECK: // .b8 4                                // Abbrev [4] 0x4e8:0x8 DW_TAG_imported_declaration
-; CHECK: // .b8 10                               // DW_AT_decl_file
-; CHECK: // .b8 176                              // DW_AT_decl_line
-; CHECK: // .b8 1
-; CHECK: // .b32 6781                            // DW_AT_import
-; CHECK: // .b8 4                                // Abbrev [4] 0x4f0:0x8 DW_TAG_imported_declaration
-; CHECK: // .b8 10                               // DW_AT_decl_file
-; CHECK: // .b8 177                              // DW_AT_decl_line
-; CHECK: // .b8 1
-; CHECK: // .b32 6821                            // DW_AT_import
-; CHECK: // .b8 4                                // Abbrev [4] 0x4f8:0x8 DW_TAG_imported_declaration
-; CHECK: // .b8 10                               // DW_AT_decl_file
-; CHECK: // .b8 178                              // DW_AT_decl_line
-; CHECK: // .b8 1
-; CHECK: // .b32 6857                            // DW_AT_import
-; CHECK: // .b8 4                                // Abbrev [4] 0x500:0x8 DW_TAG_imported_declaration
-; CHECK: // .b8 10                               // DW_AT_decl_file
-; CHECK: // .b8 179                              // DW_AT_decl_line
-; CHECK: // .b8 1
-; CHECK: // .b32 6893                            // DW_AT_import
-; CHECK: // .b8 4                                // Abbrev [4] 0x508:0x8 DW_TAG_imported_declaration
-; CHECK: // .b8 10                               // DW_AT_decl_file
-; CHECK: // .b8 180                              // DW_AT_decl_line
-; CHECK: // .b8 1
-; CHECK: // .b32 6931                            // DW_AT_import
-; CHECK: // .b8 4                                // Abbrev [4] 0x510:0x8 DW_TAG_imported_declaration
-; CHECK: // .b8 10                               // DW_AT_decl_file
-; CHECK: // .b8 181                              // DW_AT_decl_line
-; CHECK: // .b8 1
-; CHECK: // .b32 6965                            // DW_AT_import
-; CHECK: // .b8 4                                // Abbrev [4] 0x518:0x8 DW_TAG_imported_declaration
-; CHECK: // .b8 10                               // DW_AT_decl_file
-; CHECK: // .b8 182                              // DW_AT_decl_line
-; CHECK: // .b8 1
-; CHECK: // .b32 6999                            // DW_AT_import
-; CHECK: // .b8 4                                // Abbrev [4] 0x520:0x8 DW_TAG_imported_declaration
-; CHECK: // .b8 10                               // DW_AT_decl_file
-; CHECK: // .b8 183                              // DW_AT_decl_line
-; CHECK: // .b8 1
-; CHECK: // .b32 7031                            // DW_AT_import
-; CHECK: // .b8 4                                // Abbrev [4] 0x528:0x8 DW_TAG_imported_declaration
-; CHECK: // .b8 10                               // DW_AT_decl_file
-; CHECK: // .b8 184                              // DW_AT_decl_line
-; CHECK: // .b8 1
-; CHECK: // .b32 7063                            // DW_AT_import
-; CHECK: // .b8 4                                // Abbrev [4] 0x530:0x8 DW_TAG_imported_declaration
-; CHECK: // .b8 10                               // DW_AT_decl_file
-; CHECK: // .b8 185                              // DW_AT_decl_line
-; CHECK: // .b8 1
-; CHECK: // .b32 7093                            // DW_AT_import
-; CHECK: // .b8 4                                // Abbrev [4] 0x538:0x8 DW_TAG_imported_declaration
-; CHECK: // .b8 10                               // DW_AT_decl_file
-; CHECK: // .b8 186                              // DW_AT_decl_line
-; CHECK: // .b8 1
-; CHECK: // .b32 7127                            // DW_AT_import
-; CHECK: // .b8 4                                // Abbrev [4] 0x540:0x8 DW_TAG_imported_declaration
-; CHECK: // .b8 10                               // DW_AT_decl_file
-; CHECK: // .b8 187                              // DW_AT_decl_line
-; CHECK: // .b8 1
-; CHECK: // .b32 7163                            // DW_AT_import
-; CHECK: // .b8 4                                // Abbrev [4] 0x548:0x8 DW_TAG_imported_declaration
-; CHECK: // .b8 10                               // DW_AT_decl_file
-; CHECK: // .b8 188                              // DW_AT_decl_line
-; CHECK: // .b8 1
-; CHECK: // .b32 7202                            // DW_AT_import
-; CHECK: // .b8 4                                // Abbrev [4] 0x550:0x8 DW_TAG_imported_declaration
-; CHECK: // .b8 10                               // DW_AT_decl_file
-; CHECK: // .b8 189                              // DW_AT_decl_line
-; CHECK: // .b8 1
-; CHECK: // .b32 7245                            // DW_AT_import
-; CHECK: // .b8 4                                // Abbrev [4] 0x558:0x8 DW_TAG_imported_declaration
-; CHECK: // .b8 10                               // DW_AT_decl_file
-; CHECK: // .b8 190                              // DW_AT_decl_line
-; CHECK: // .b8 1
-; CHECK: // .b32 7294                            // DW_AT_import
-; CHECK: // .b8 4                                // Abbrev [4] 0x560:0x8 DW_TAG_imported_declaration
-; CHECK: // .b8 10                               // DW_AT_decl_file
-; CHECK: // .b8 191                              // DW_AT_decl_line
-; CHECK: // .b8 1
-; CHECK: // .b32 7330                            // DW_AT_import
-; CHECK: // .b8 4                                // Abbrev [4] 0x568:0x8 DW_TAG_imported_declaration
-; CHECK: // .b8 10                               // DW_AT_decl_file
-; CHECK: // .b8 192                              // DW_AT_decl_line
-; CHECK: // .b8 1
-; CHECK: // .b32 7379                            // DW_AT_import
-; CHECK: // .b8 4                                // Abbrev [4] 0x570:0x8 DW_TAG_imported_declaration
-; CHECK: // .b8 10                               // DW_AT_decl_file
-; CHECK: // .b8 193                              // DW_AT_decl_line
-; CHECK: // .b8 1
-; CHECK: // .b32 7428                            // DW_AT_import
-; CHECK: // .b8 4                                // Abbrev [4] 0x578:0x8 DW_TAG_imported_declaration
-; CHECK: // .b8 10                               // DW_AT_decl_file
-; CHECK: // .b8 194                              // DW_AT_decl_line
-; CHECK: // .b8 1
-; CHECK: // .b32 7460                            // DW_AT_import
-; CHECK: // .b8 4                                // Abbrev [4] 0x580:0x8 DW_TAG_imported_declaration
-; CHECK: // .b8 10                               // DW_AT_decl_file
-; CHECK: // .b8 195                              // DW_AT_decl_line
-; CHECK: // .b8 1
-; CHECK: // .b32 7494                            // DW_AT_import
-; CHECK: // .b8 4                                // Abbrev [4] 0x588:0x8 DW_TAG_imported_declaration
-; CHECK: // .b8 10                               // DW_AT_decl_file
-; CHECK: // .b8 196                              // DW_AT_decl_line
-; CHECK: // .b8 1
-; CHECK: // .b32 7538                            // DW_AT_import
-; CHECK: // .b8 4                                // Abbrev [4] 0x590:0x8 DW_TAG_imported_declaration
-; CHECK: // .b8 10                               // DW_AT_decl_file
-; CHECK: // .b8 197                              // DW_AT_decl_line
-; CHECK: // .b8 1
-; CHECK: // .b32 7580                            // DW_AT_import
-; CHECK: // .b8 4                                // Abbrev [4] 0x598:0x8 DW_TAG_imported_declaration
-; CHECK: // .b8 10                               // DW_AT_decl_file
-; CHECK: // .b8 198                              // DW_AT_decl_line
-; CHECK: // .b8 1
-; CHECK: // .b32 7610                            // DW_AT_import
-; CHECK: // .b8 4                                // Abbrev [4] 0x5a0:0x8 DW_TAG_imported_declaration
-; CHECK: // .b8 10                               // DW_AT_decl_file
-; CHECK: // .b8 199                              // DW_AT_decl_line
-; CHECK: // .b8 1
-; CHECK: // .b32 7642                            // DW_AT_import
-; CHECK: // .b8 4                                // Abbrev [4] 0x5a8:0x8 DW_TAG_imported_declaration
-; CHECK: // .b8 10                               // DW_AT_decl_file
-; CHECK: // .b8 200                              // DW_AT_decl_line
-; CHECK: // .b8 1
-; CHECK: // .b32 7674                            // DW_AT_import
-; CHECK: // .b8 4                                // Abbrev [4] 0x5b0:0x8 DW_TAG_imported_declaration
-; CHECK: // .b8 10                               // DW_AT_decl_file
-; CHECK: // .b8 201                              // DW_AT_decl_line
-; CHECK: // .b8 1
-; CHECK: // .b32 7704                            // DW_AT_import
-; CHECK: // .b8 4                                // Abbrev [4] 0x5b8:0x8 DW_TAG_imported_declaration
-; CHECK: // .b8 10                               // DW_AT_decl_file
-; CHECK: // .b8 202                              // DW_AT_decl_line
-; CHECK: // .b8 1
-; CHECK: // .b32 7736                            // DW_AT_import
-; CHECK: // .b8 4                                // Abbrev [4] 0x5c0:0x8 DW_TAG_imported_declaration
-; CHECK: // .b8 10                               // DW_AT_decl_file
-; CHECK: // .b8 203                              // DW_AT_decl_line
-; CHECK: // .b8 1
-; CHECK: // .b32 7772                            // DW_AT_import
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 5                                // Abbrev [5] 0x5c9:0x1b DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,51,97,98,115,120        // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 97,98,115                        // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 44                               // DW_AT_decl_line
-; CHECK: // .b32 1508                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0x5de:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1508                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 7                                // Abbrev [7] 0x5e4:0x11 DW_TAG_base_type
-; CHECK: // .b8 108,111,110,103,32,108,111,110,103,32,105,110,116 // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 5                                // DW_AT_encoding
-; CHECK: // .b8 8                                // DW_AT_byte_size
-; CHECK: // .b8 5                                // Abbrev [5] 0x5f5:0x1d DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,52,97,99,111,115,102    // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 97,99,111,115                    // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 46                               // DW_AT_decl_line
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0x60c:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 7                                // Abbrev [7] 0x612:0x9 DW_TAG_base_type
-; CHECK: // .b8 102,108,111,97,116               // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 4                                // DW_AT_encoding
-; CHECK: // .b8 4                                // DW_AT_byte_size
-; CHECK: // .b8 5                                // Abbrev [5] 0x61b:0x1f DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,53,97,99,111,115,104,102 // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 97,99,111,115,104                // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 48                               // DW_AT_decl_line
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0x634:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 5                                // Abbrev [5] 0x63a:0x1d DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,52,97,115,105,110,102   // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 97,115,105,110                   // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 50                               // DW_AT_decl_line
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0x651:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 5                                // Abbrev [5] 0x657:0x1f DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,53,97,115,105,110,104,102 // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 97,115,105,110,104               // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 52                               // DW_AT_decl_line
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0x670:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 5                                // Abbrev [5] 0x676:0x1d DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,52,97,116,97,110,102    // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 97,116,97,110                    // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 56                               // DW_AT_decl_line
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0x68d:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 5                                // Abbrev [5] 0x693:0x25 DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,53,97,116,97,110,50,102,102 // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 97,116,97,110,50                 // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 54                               // DW_AT_decl_line
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0x6ad:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 6                                // Abbrev [6] 0x6b2:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 5                                // Abbrev [5] 0x6b8:0x1f DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,53,97,116,97,110,104,102 // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 97,116,97,110,104                // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 58                               // DW_AT_decl_line
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0x6d1:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 5                                // Abbrev [5] 0x6d7:0x1d DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,52,99,98,114,116,102    // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 99,98,114,116                    // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 60                               // DW_AT_decl_line
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0x6ee:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 5                                // Abbrev [5] 0x6f4:0x1d DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,52,99,101,105,108,102   // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 99,101,105,108                   // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 62                               // DW_AT_decl_line
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0x70b:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 5                                // Abbrev [5] 0x711:0x2b DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,56,99,111,112,121,115,105,103,110,102,102 // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 99,111,112,121,115,105,103,110   // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 64                               // DW_AT_decl_line
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0x731:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 6                                // Abbrev [6] 0x736:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 5                                // Abbrev [5] 0x73c:0x1b DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,51,99,111,115,102       // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 99,111,115                       // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 66                               // DW_AT_decl_line
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0x751:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 5                                // Abbrev [5] 0x757:0x1d DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,52,99,111,115,104,102   // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 99,111,115,104                   // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 68                               // DW_AT_decl_line
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0x76e:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 5                                // Abbrev [5] 0x774:0x1b DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,51,101,114,102,102      // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 101,114,102                      // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 72                               // DW_AT_decl_line
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0x789:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 5                                // Abbrev [5] 0x78f:0x1d DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,52,101,114,102,99,102   // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 101,114,102,99                   // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 70                               // DW_AT_decl_line
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0x7a6:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 5                                // Abbrev [5] 0x7ac:0x1b DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,51,101,120,112,102      // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 101,120,112                      // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 76                               // DW_AT_decl_line
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0x7c1:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 5                                // Abbrev [5] 0x7c7:0x1d DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,52,101,120,112,50,102   // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 101,120,112,50                   // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 74                               // DW_AT_decl_line
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0x7de:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 5                                // Abbrev [5] 0x7e4:0x1f DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,53,101,120,112,109,49,102 // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 101,120,112,109,49               // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 78                               // DW_AT_decl_line
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0x7fd:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 5                                // Abbrev [5] 0x803:0x1d DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,52,102,97,98,115,102    // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 102,97,98,115                    // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 80                               // DW_AT_decl_line
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0x81a:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 5                                // Abbrev [5] 0x820:0x23 DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,52,102,100,105,109,102,102 // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 102,100,105,109                  // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 82                               // DW_AT_decl_line
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0x838:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 6                                // Abbrev [6] 0x83d:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 5                                // Abbrev [5] 0x843:0x1f DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,53,102,108,111,111,114,102 // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 102,108,111,111,114              // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 84                               // DW_AT_decl_line
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0x85c:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 5                                // Abbrev [5] 0x862:0x27 DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,51,102,109,97,102,102,102 // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 102,109,97                       // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 86                               // DW_AT_decl_line
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0x879:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 6                                // Abbrev [6] 0x87e:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 6                                // Abbrev [6] 0x883:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 5                                // Abbrev [5] 0x889:0x23 DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,52,102,109,97,120,102,102 // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 102,109,97,120                   // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 88                               // DW_AT_decl_line
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0x8a1:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 6                                // Abbrev [6] 0x8a6:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 5                                // Abbrev [5] 0x8ac:0x23 DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,52,102,109,105,110,102,102 // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 102,109,105,110                  // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 90                               // DW_AT_decl_line
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0x8c4:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 6                                // Abbrev [6] 0x8c9:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 5                                // Abbrev [5] 0x8cf:0x23 DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,52,102,109,111,100,102,102 // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 102,109,111,100                  // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 92                               // DW_AT_decl_line
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0x8e7:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 6                                // Abbrev [6] 0x8ec:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 5                                // Abbrev [5] 0x8f2:0x2a DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,49,48,102,112,99,108,97,115,115,105,102,121,102 // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 102,112,99,108,97,115,115,105,102,121 // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 94                               // DW_AT_decl_line
-; CHECK: // .b32 2332                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0x916:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 7                                // Abbrev [7] 0x91c:0x7 DW_TAG_base_type
-; CHECK: // .b8 105,110,116                      // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 5                                // DW_AT_encoding
-; CHECK: // .b8 4                                // DW_AT_byte_size
-; CHECK: // .b8 5                                // Abbrev [5] 0x923:0x26 DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,53,102,114,101,120,112,102,80,105 // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 102,114,101,120,112              // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 96                               // DW_AT_decl_line
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0x93e:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 6                                // Abbrev [6] 0x943:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 2377                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 8                                // Abbrev [8] 0x949:0x5 DW_TAG_pointer_type
-; CHECK: // .b32 2332                            // DW_AT_type
-; CHECK: // .b8 5                                // Abbrev [5] 0x94e:0x25 DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,53,104,121,112,111,116,102,102 // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 104,121,112,111,116              // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 98                               // DW_AT_decl_line
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0x968:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 6                                // Abbrev [6] 0x96d:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 5                                // Abbrev [5] 0x973:0x1f DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,53,105,108,111,103,98,102 // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 105,108,111,103,98               // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 100                              // DW_AT_decl_line
-; CHECK: // .b32 2332                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0x98c:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 5                                // Abbrev [5] 0x992:0x25 DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,56,105,115,102,105,110,105,116,101,102 // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 105,115,102,105,110,105,116,101  // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 102                              // DW_AT_decl_line
-; CHECK: // .b32 2487                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0x9b1:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 7                                // Abbrev [7] 0x9b7:0x8 DW_TAG_base_type
-; CHECK: // .b8 98,111,111,108                   // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 2                                // DW_AT_encoding
-; CHECK: // .b8 1                                // DW_AT_byte_size
-; CHECK: // .b8 5                                // Abbrev [5] 0x9bf:0x2d DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,57,105,115,103,114,101,97,116,101,114,102,102 // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 105,115,103,114,101,97,116,101,114 // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 106                              // DW_AT_decl_line
-; CHECK: // .b32 2487                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0x9e1:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 6                                // Abbrev [6] 0x9e6:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 5                                // Abbrev [5] 0x9ec:0x38 DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,49,52,105,115,103,114,101,97,116,101,114,101,113,117,97,108,102,102 // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 105,115,103,114,101,97,116,101,114,101,113,117,97,108 // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 105                              // DW_AT_decl_line
-; CHECK: // .b32 2487                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0xa19:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 6                                // Abbrev [6] 0xa1e:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 5                                // Abbrev [5] 0xa24:0x1f DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,53,105,115,105,110,102,102 // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 105,115,105,110,102              // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 108                              // DW_AT_decl_line
-; CHECK: // .b32 2487                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0xa3d:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 5                                // Abbrev [5] 0xa43:0x27 DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,54,105,115,108,101,115,115,102,102 // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 105,115,108,101,115,115          // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 112                              // DW_AT_decl_line
-; CHECK: // .b32 2487                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0xa5f:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 6                                // Abbrev [6] 0xa64:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 5                                // Abbrev [5] 0xa6a:0x32 DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,49,49,105,115,108,101,115,115,101,113,117,97,108,102,102 // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 105,115,108,101,115,115,101,113,117,97,108 // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 111                              // DW_AT_decl_line
-; CHECK: // .b32 2487                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0xa91:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 6                                // Abbrev [6] 0xa96:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 5                                // Abbrev [5] 0xa9c:0x36 DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,49,51,105,115,108,101,115,115,103,114,101,97,116,101,114,102,102 // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 105,115,108,101,115,115,103,114,101,97,116,101,114 // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 114                              // DW_AT_decl_line
-; CHECK: // .b32 2487                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0xac7:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 6                                // Abbrev [6] 0xacc:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 5                                // Abbrev [5] 0xad2:0x1f DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,53,105,115,110,97,110,102 // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 105,115,110,97,110               // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 116                              // DW_AT_decl_line
-; CHECK: // .b32 2487                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0xaeb:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 5                                // Abbrev [5] 0xaf1:0x25 DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,56,105,115,110,111,114,109,97,108,102 // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 105,115,110,111,114,109,97,108   // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 118                              // DW_AT_decl_line
-; CHECK: // .b32 2487                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0xb10:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 5                                // Abbrev [5] 0xb16:0x32 DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,49,49,105,115,117,110,111,114,100,101,114,101,100,102,102 // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 105,115,117,110,111,114,100,101,114,101,100 // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 120                              // DW_AT_decl_line
-; CHECK: // .b32 2487                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0xb3d:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 6                                // Abbrev [6] 0xb42:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 5                                // Abbrev [5] 0xb48:0x1d DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,52,108,97,98,115,108    // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 108,97,98,115                    // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 121                              // DW_AT_decl_line
-; CHECK: // .b32 2917                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0xb5f:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 2917                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 7                                // Abbrev [7] 0xb65:0xc DW_TAG_base_type
-; CHECK: // .b8 108,111,110,103,32,105,110,116   // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 5                                // DW_AT_encoding
-; CHECK: // .b8 8                                // DW_AT_byte_size
-; CHECK: // .b8 5                                // Abbrev [5] 0xb71:0x25 DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,53,108,100,101,120,112,102,105 // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 108,100,101,120,112              // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 123                              // DW_AT_decl_line
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0xb8b:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 6                                // Abbrev [6] 0xb90:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 2332                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 5                                // Abbrev [5] 0xb96:0x21 DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,54,108,103,97,109,109,97,102 // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 108,103,97,109,109,97            // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 125                              // DW_AT_decl_line
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0xbb1:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 5                                // Abbrev [5] 0xbb7:0x1f DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,53,108,108,97,98,115,120 // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 108,108,97,98,115                // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 126                              // DW_AT_decl_line
-; CHECK: // .b32 1508                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0xbd0:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1508                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 5                                // Abbrev [5] 0xbd6:0x21 DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,54,108,108,114,105,110,116,102 // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 108,108,114,105,110,116          // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 128                              // DW_AT_decl_line
-; CHECK: // .b32 1508                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0xbf1:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 5                                // Abbrev [5] 0xbf7:0x1b DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,51,108,111,103,102      // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 108,111,103                      // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 138                              // DW_AT_decl_line
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0xc0c:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 5                                // Abbrev [5] 0xc12:0x1f DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,53,108,111,103,49,48,102 // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 108,111,103,49,48                // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 130                              // DW_AT_decl_line
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0xc2b:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 5                                // Abbrev [5] 0xc31:0x1f DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,53,108,111,103,49,112,102 // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 108,111,103,49,112               // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 132                              // DW_AT_decl_line
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0xc4a:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 5                                // Abbrev [5] 0xc50:0x1d DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,52,108,111,103,50,102   // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 108,111,103,50                   // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 134                              // DW_AT_decl_line
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0xc67:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 5                                // Abbrev [5] 0xc6d:0x1d DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,52,108,111,103,98,102   // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 108,111,103,98                   // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 136                              // DW_AT_decl_line
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0xc84:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 5                                // Abbrev [5] 0xc8a:0x1f DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,53,108,114,105,110,116,102 // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 108,114,105,110,116              // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 140                              // DW_AT_decl_line
-; CHECK: // .b32 2917                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0xca3:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 5                                // Abbrev [5] 0xca9:0x21 DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,54,108,114,111,117,110,100,102 // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 108,114,111,117,110,100          // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 142                              // DW_AT_decl_line
-; CHECK: // .b32 2917                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0xcc4:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 5                                // Abbrev [5] 0xcca:0x23 DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,55,108,108,114,111,117,110,100,102 // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 108,108,114,111,117,110,100      // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 143                              // DW_AT_decl_line
-; CHECK: // .b32 1508                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0xce7:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 5                                // Abbrev [5] 0xced:0x24 DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,52,109,111,100,102,102,80,102 // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 109,111,100,102                  // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 145                              // DW_AT_decl_line
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0xd06:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 6                                // Abbrev [6] 0xd0b:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 3345                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 8                                // Abbrev [8] 0xd11:0x5 DW_TAG_pointer_type
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 5                                // Abbrev [5] 0xd16:0x1d DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,51,110,97,110,80,75,99  // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 110,97,110                       // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 146                              // DW_AT_decl_line
-; CHECK: // .b32 3379                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0xd2d:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 3389                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 7                                // Abbrev [7] 0xd33:0xa DW_TAG_base_type
-; CHECK: // .b8 100,111,117,98,108,101           // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 4                                // DW_AT_encoding
-; CHECK: // .b8 8                                // DW_AT_byte_size
-; CHECK: // .b8 8                                // Abbrev [8] 0xd3d:0x5 DW_TAG_pointer_type
-; CHECK: // .b32 3394                            // DW_AT_type
-; CHECK: // .b8 9                                // Abbrev [9] 0xd42:0x5 DW_TAG_const_type
-; CHECK: // .b32 3399                            // DW_AT_type
-; CHECK: // .b8 7                                // Abbrev [7] 0xd47:0x8 DW_TAG_base_type
-; CHECK: // .b8 99,104,97,114                    // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 8                                // DW_AT_encoding
-; CHECK: // .b8 1                                // DW_AT_byte_size
-; CHECK: // .b8 5                                // Abbrev [5] 0xd4f:0x1f DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,52,110,97,110,102,80,75,99 // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 110,97,110,102                   // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 147                              // DW_AT_decl_line
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0xd68:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 3389                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 5                                // Abbrev [5] 0xd6e:0x27 DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,57,110,101,97,114,98,121,105,110,116,102 // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 110,101,97,114,98,121,105,110,116 // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 149                              // DW_AT_decl_line
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0xd8f:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 5                                // Abbrev [5] 0xd95:0x2d DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,57,110,101,120,116,97,102,116,101,114,102,102 // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 110,101,120,116,97,102,116,101,114 // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 151                              // DW_AT_decl_line
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0xdb7:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 6                                // Abbrev [6] 0xdbc:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 5                                // Abbrev [5] 0xdc2:0x21 DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,51,112,111,119,102,105  // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 112,111,119                      // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 155                              // DW_AT_decl_line
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0xdd8:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 6                                // Abbrev [6] 0xddd:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 2332                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 5                                // Abbrev [5] 0xde3:0x2d DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,57,114,101,109,97,105,110,100,101,114,102,102 // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 114,101,109,97,105,110,100,101,114 // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 157                              // DW_AT_decl_line
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0xe05:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 6                                // Abbrev [6] 0xe0a:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 5                                // Abbrev [5] 0xe10:0x2e DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,54,114,101,109,113,117,111,102,102,80,105 // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 114,101,109,113,117,111          // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 159                              // DW_AT_decl_line
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0xe2e:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 6                                // Abbrev [6] 0xe33:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 6                                // Abbrev [6] 0xe38:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 2377                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 5                                // Abbrev [5] 0xe3e:0x1d DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,52,114,105,110,116,102  // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 114,105,110,116                  // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 161                              // DW_AT_decl_line
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0xe55:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 5                                // Abbrev [5] 0xe5b:0x1f DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,53,114,111,117,110,100,102 // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 114,111,117,110,100              // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 163                              // DW_AT_decl_line
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0xe74:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 5                                // Abbrev [5] 0xe7a:0x29 DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,55,115,99,97,108,98,108,110,102,108 // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 115,99,97,108,98,108,110         // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 165                              // DW_AT_decl_line
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0xe98:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 6                                // Abbrev [6] 0xe9d:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 2917                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 5                                // Abbrev [5] 0xea3:0x27 DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,54,115,99,97,108,98,110,102,105 // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 115,99,97,108,98,110             // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 167                              // DW_AT_decl_line
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0xebf:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 6                                // Abbrev [6] 0xec4:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 2332                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 5                                // Abbrev [5] 0xeca:0x23 DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,55,115,105,103,110,98,105,116,102 // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 115,105,103,110,98,105,116       // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 169                              // DW_AT_decl_line
-; CHECK: // .b32 2487                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0xee7:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 5                                // Abbrev [5] 0xeed:0x1b DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,51,115,105,110,102      // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 115,105,110                      // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 171                              // DW_AT_decl_line
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0xf02:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 5                                // Abbrev [5] 0xf08:0x1d DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,52,115,105,110,104,102  // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 115,105,110,104                  // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 173                              // DW_AT_decl_line
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0xf1f:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 5                                // Abbrev [5] 0xf25:0x1d DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,52,115,113,114,116,102  // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 115,113,114,116                  // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 175                              // DW_AT_decl_line
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0xf3c:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 5                                // Abbrev [5] 0xf42:0x1b DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,51,116,97,110,102       // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 116,97,110                       // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 177                              // DW_AT_decl_line
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0xf57:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 5                                // Abbrev [5] 0xf5d:0x1d DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,52,116,97,110,104,102   // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 116,97,110,104                   // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 179                              // DW_AT_decl_line
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0xf74:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 5                                // Abbrev [5] 0xf7a:0x21 DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,54,116,103,97,109,109,97,102 // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 116,103,97,109,109,97            // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 181                              // DW_AT_decl_line
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0xf95:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 5                                // Abbrev [5] 0xf9b:0x1f DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,53,116,114,117,110,99,102 // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 116,114,117,110,99               // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 183                              // DW_AT_decl_line
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0xfb4:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 10                               // Abbrev [10] 0xfba:0x14 DW_TAG_subprogram
-; CHECK: // .b8 97,99,111,115                    // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 2                                // DW_AT_decl_file
-; CHECK: // .b8 54                               // DW_AT_decl_line
-; CHECK: // .b32 3379                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 1                                // DW_AT_external
-; CHECK: // .b8 6                                // Abbrev [6] 0xfc8:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 3379                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 10                               // Abbrev [10] 0xfce:0x14 DW_TAG_subprogram
-; CHECK: // .b8 97,115,105,110                   // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 2                                // DW_AT_decl_file
-; CHECK: // .b8 56                               // DW_AT_decl_line
-; CHECK: // .b32 3379                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 1                                // DW_AT_external
-; CHECK: // .b8 6                                // Abbrev [6] 0xfdc:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 3379                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 10                               // Abbrev [10] 0xfe2:0x14 DW_TAG_subprogram
-; CHECK: // .b8 97,116,97,110                    // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 2                                // DW_AT_decl_file
-; CHECK: // .b8 58                               // DW_AT_decl_line
-; CHECK: // .b32 3379                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 1                                // DW_AT_external
-; CHECK: // .b8 6                                // Abbrev [6] 0xff0:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 3379                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 10                               // Abbrev [10] 0xff6:0x1a DW_TAG_subprogram
-; CHECK: // .b8 97,116,97,110,50                 // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 2                                // DW_AT_decl_file
-; CHECK: // .b8 60                               // DW_AT_decl_line
-; CHECK: // .b32 3379                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 1                                // DW_AT_external
-; CHECK: // .b8 6                                // Abbrev [6] 0x1005:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 3379                            // DW_AT_type
-; CHECK: // .b8 6                                // Abbrev [6] 0x100a:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 3379                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 10                               // Abbrev [10] 0x1010:0x14 DW_TAG_subprogram
-; CHECK: // .b8 99,101,105,108                   // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 2                                // DW_AT_decl_file
-; CHECK: // .b8 178                              // DW_AT_decl_line
-; CHECK: // .b32 3379                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 1                                // DW_AT_external
-; CHECK: // .b8 6                                // Abbrev [6] 0x101e:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 3379                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 10                               // Abbrev [10] 0x1024:0x13 DW_TAG_subprogram
-; CHECK: // .b8 99,111,115                       // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 2                                // DW_AT_decl_file
-; CHECK: // .b8 63                               // DW_AT_decl_line
-; CHECK: // .b32 3379                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 1                                // DW_AT_external
-; CHECK: // .b8 6                                // Abbrev [6] 0x1031:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 3379                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 10                               // Abbrev [10] 0x1037:0x14 DW_TAG_subprogram
-; CHECK: // .b8 99,111,115,104                   // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 2                                // DW_AT_decl_file
-; CHECK: // .b8 72                               // DW_AT_decl_line
-; CHECK: // .b32 3379                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 1                                // DW_AT_external
-; CHECK: // .b8 6                                // Abbrev [6] 0x1045:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 3379                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 10                               // Abbrev [10] 0x104b:0x13 DW_TAG_subprogram
-; CHECK: // .b8 101,120,112                      // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 2                                // DW_AT_decl_file
-; CHECK: // .b8 100                              // DW_AT_decl_line
-; CHECK: // .b32 3379                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 1                                // DW_AT_external
-; CHECK: // .b8 6                                // Abbrev [6] 0x1058:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 3379                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 10                               // Abbrev [10] 0x105e:0x14 DW_TAG_subprogram
-; CHECK: // .b8 102,97,98,115                    // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 2                                // DW_AT_decl_file
-; CHECK: // .b8 181                              // DW_AT_decl_line
-; CHECK: // .b32 3379                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 1                                // DW_AT_external
-; CHECK: // .b8 6                                // Abbrev [6] 0x106c:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 3379                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 10                               // Abbrev [10] 0x1072:0x15 DW_TAG_subprogram
-; CHECK: // .b8 102,108,111,111,114              // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 2                                // DW_AT_decl_file
-; CHECK: // .b8 184                              // DW_AT_decl_line
-; CHECK: // .b32 3379                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 1                                // DW_AT_external
-; CHECK: // .b8 6                                // Abbrev [6] 0x1081:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 3379                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 10                               // Abbrev [10] 0x1087:0x19 DW_TAG_subprogram
-; CHECK: // .b8 102,109,111,100                  // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 2                                // DW_AT_decl_file
-; CHECK: // .b8 187                              // DW_AT_decl_line
-; CHECK: // .b32 3379                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 1                                // DW_AT_external
-; CHECK: // .b8 6                                // Abbrev [6] 0x1095:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 3379                            // DW_AT_type
-; CHECK: // .b8 6                                // Abbrev [6] 0x109a:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 3379                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 10                               // Abbrev [10] 0x10a0:0x1a DW_TAG_subprogram
-; CHECK: // .b8 102,114,101,120,112              // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 2                                // DW_AT_decl_file
-; CHECK: // .b8 103                              // DW_AT_decl_line
-; CHECK: // .b32 3379                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 1                                // DW_AT_external
-; CHECK: // .b8 6                                // Abbrev [6] 0x10af:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 3379                            // DW_AT_type
-; CHECK: // .b8 6                                // Abbrev [6] 0x10b4:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 2377                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 10                               // Abbrev [10] 0x10ba:0x1a DW_TAG_subprogram
-; CHECK: // .b8 108,100,101,120,112              // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 2                                // DW_AT_decl_file
-; CHECK: // .b8 106                              // DW_AT_decl_line
-; CHECK: // .b32 3379                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 1                                // DW_AT_external
-; CHECK: // .b8 6                                // Abbrev [6] 0x10c9:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 3379                            // DW_AT_type
-; CHECK: // .b8 6                                // Abbrev [6] 0x10ce:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 2332                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 10                               // Abbrev [10] 0x10d4:0x13 DW_TAG_subprogram
-; CHECK: // .b8 108,111,103                      // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 2                                // DW_AT_decl_file
-; CHECK: // .b8 109                              // DW_AT_decl_line
-; CHECK: // .b32 3379                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 1                                // DW_AT_external
-; CHECK: // .b8 6                                // Abbrev [6] 0x10e1:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 3379                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 10                               // Abbrev [10] 0x10e7:0x15 DW_TAG_subprogram
-; CHECK: // .b8 108,111,103,49,48                // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 2                                // DW_AT_decl_file
-; CHECK: // .b8 112                              // DW_AT_decl_line
-; CHECK: // .b32 3379                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 1                                // DW_AT_external
-; CHECK: // .b8 6                                // Abbrev [6] 0x10f6:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 3379                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 10                               // Abbrev [10] 0x10fc:0x19 DW_TAG_subprogram
-; CHECK: // .b8 109,111,100,102                  // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 2                                // DW_AT_decl_file
-; CHECK: // .b8 115                              // DW_AT_decl_line
-; CHECK: // .b32 3379                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 1                                // DW_AT_external
-; CHECK: // .b8 6                                // Abbrev [6] 0x110a:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 3379                            // DW_AT_type
-; CHECK: // .b8 6                                // Abbrev [6] 0x110f:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 4373                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 8                                // Abbrev [8] 0x1115:0x5 DW_TAG_pointer_type
-; CHECK: // .b32 3379                            // DW_AT_type
-; CHECK: // .b8 10                               // Abbrev [10] 0x111a:0x18 DW_TAG_subprogram
-; CHECK: // .b8 112,111,119                      // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 2                                // DW_AT_decl_file
-; CHECK: // .b8 153                              // DW_AT_decl_line
-; CHECK: // .b32 3379                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 1                                // DW_AT_external
-; CHECK: // .b8 6                                // Abbrev [6] 0x1127:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 3379                            // DW_AT_type
-; CHECK: // .b8 6                                // Abbrev [6] 0x112c:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 3379                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 10                               // Abbrev [10] 0x1132:0x13 DW_TAG_subprogram
-; CHECK: // .b8 115,105,110                      // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 2                                // DW_AT_decl_file
-; CHECK: // .b8 65                               // DW_AT_decl_line
-; CHECK: // .b32 3379                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 1                                // DW_AT_external
-; CHECK: // .b8 6                                // Abbrev [6] 0x113f:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 3379                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 10                               // Abbrev [10] 0x1145:0x14 DW_TAG_subprogram
-; CHECK: // .b8 115,105,110,104                  // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 2                                // DW_AT_decl_file
-; CHECK: // .b8 74                               // DW_AT_decl_line
-; CHECK: // .b32 3379                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 1                                // DW_AT_external
-; CHECK: // .b8 6                                // Abbrev [6] 0x1153:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 3379                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 10                               // Abbrev [10] 0x1159:0x14 DW_TAG_subprogram
-; CHECK: // .b8 115,113,114,116                  // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 2                                // DW_AT_decl_file
-; CHECK: // .b8 156                              // DW_AT_decl_line
-; CHECK: // .b32 3379                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 1                                // DW_AT_external
-; CHECK: // .b8 6                                // Abbrev [6] 0x1167:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 3379                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 10                               // Abbrev [10] 0x116d:0x13 DW_TAG_subprogram
-; CHECK: // .b8 116,97,110                       // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 2                                // DW_AT_decl_file
-; CHECK: // .b8 67                               // DW_AT_decl_line
-; CHECK: // .b32 3379                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 1                                // DW_AT_external
-; CHECK: // .b8 6                                // Abbrev [6] 0x117a:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 3379                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 10                               // Abbrev [10] 0x1180:0x14 DW_TAG_subprogram
-; CHECK: // .b8 116,97,110,104                   // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 2                                // DW_AT_decl_file
-; CHECK: // .b8 76                               // DW_AT_decl_line
-; CHECK: // .b32 3379                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 1                                // DW_AT_external
-; CHECK: // .b8 6                                // Abbrev [6] 0x118e:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 3379                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 11                               // Abbrev [11] 0x1194:0xd DW_TAG_typedef
-; CHECK: // .b32 4513                            // DW_AT_type
-; CHECK: // .b8 100,105,118,95,116               // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 4                                // DW_AT_decl_file
-; CHECK: // .b8 101                              // DW_AT_decl_line
-; CHECK: // .b8 12                               // Abbrev [12] 0x11a1:0x2 DW_TAG_structure_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 11                               // Abbrev [11] 0x11a3:0xe DW_TAG_typedef
-; CHECK: // .b32 4529                            // DW_AT_type
-; CHECK: // .b8 108,100,105,118,95,116           // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 4                                // DW_AT_decl_file
-; CHECK: // .b8 109                              // DW_AT_decl_line
-; CHECK: // .b8 13                               // Abbrev [13] 0x11b1:0x22 DW_TAG_structure_type
-; CHECK: // .b8 16                               // DW_AT_byte_size
-; CHECK: // .b8 4                                // DW_AT_decl_file
-; CHECK: // .b8 105                              // DW_AT_decl_line
-; CHECK: // .b8 14                               // Abbrev [14] 0x11b5:0xf DW_TAG_member
-; CHECK: // .b8 113,117,111,116                  // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b32 2917                            // DW_AT_type
-; CHECK: // .b8 4                                // DW_AT_decl_file
-; CHECK: // .b8 107                              // DW_AT_decl_line
-; CHECK: // .b8 2                                // DW_AT_data_member_location
-; CHECK: // .b8 35
-; CHECK: // .b8 0
-; CHECK: // .b8 14                               // Abbrev [14] 0x11c4:0xe DW_TAG_member
-; CHECK: // .b8 114,101,109                      // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b32 2917                            // DW_AT_type
-; CHECK: // .b8 4                                // DW_AT_decl_file
-; CHECK: // .b8 108                              // DW_AT_decl_line
-; CHECK: // .b8 2                                // DW_AT_data_member_location
-; CHECK: // .b8 35
-; CHECK: // .b8 8
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 15                               // Abbrev [15] 0x11d3:0xd DW_TAG_subprogram
-; CHECK: // .b8 97,98,111,114,116                // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 4                                // DW_AT_decl_file
-; CHECK: // .b8 3                                // DW_AT_decl_line
-; CHECK: // .b8 2
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 1                                // DW_AT_external
-; CHECK: // .b8 1                                // DW_AT_noreturn
-; CHECK: // .b8 16                               // Abbrev [16] 0x11e0:0x14 DW_TAG_subprogram
-; CHECK: // .b8 97,98,115                        // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 4                                // DW_AT_decl_file
-; CHECK: // .b8 7                                // DW_AT_decl_line
-; CHECK: // .b8 3
-; CHECK: // .b32 2332                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 1                                // DW_AT_external
-; CHECK: // .b8 6                                // Abbrev [6] 0x11ee:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 2332                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 16                               // Abbrev [16] 0x11f4:0x17 DW_TAG_subprogram
-; CHECK: // .b8 97,116,101,120,105,116           // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 4                                // DW_AT_decl_file
-; CHECK: // .b8 7                                // DW_AT_decl_line
-; CHECK: // .b8 2
-; CHECK: // .b32 2332                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 1                                // DW_AT_external
-; CHECK: // .b8 6                                // Abbrev [6] 0x1205:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 4619                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 8                                // Abbrev [8] 0x120b:0x5 DW_TAG_pointer_type
-; CHECK: // .b32 4624                            // DW_AT_type
-; CHECK: // .b8 17                               // Abbrev [17] 0x1210:0x1 DW_TAG_subroutine_type
-; CHECK: // .b8 10                               // Abbrev [10] 0x1211:0x14 DW_TAG_subprogram
-; CHECK: // .b8 97,116,111,102                   // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 6                                // DW_AT_decl_file
-; CHECK: // .b8 26                               // DW_AT_decl_line
-; CHECK: // .b32 3379                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 1                                // DW_AT_external
-; CHECK: // .b8 6                                // Abbrev [6] 0x121f:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 3389                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 16                               // Abbrev [16] 0x1225:0x15 DW_TAG_subprogram
-; CHECK: // .b8 97,116,111,105                   // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 4                                // DW_AT_decl_file
-; CHECK: // .b8 22                               // DW_AT_decl_line
-; CHECK: // .b8 1
-; CHECK: // .b32 2332                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 1                                // DW_AT_external
-; CHECK: // .b8 6                                // Abbrev [6] 0x1234:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 3389                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 16                               // Abbrev [16] 0x123a:0x15 DW_TAG_subprogram
-; CHECK: // .b8 97,116,111,108                   // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 4                                // DW_AT_decl_file
-; CHECK: // .b8 27                               // DW_AT_decl_line
-; CHECK: // .b8 1
-; CHECK: // .b32 2917                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 1                                // DW_AT_external
-; CHECK: // .b8 6                                // Abbrev [6] 0x1249:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 3389                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 10                               // Abbrev [10] 0x124f:0x2b DW_TAG_subprogram
-; CHECK: // .b8 98,115,101,97,114,99,104         // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 7                                // DW_AT_decl_file
-; CHECK: // .b8 20                               // DW_AT_decl_line
-; CHECK: // .b32 4730                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 1                                // DW_AT_external
-; CHECK: // .b8 6                                // Abbrev [6] 0x1260:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 4731                            // DW_AT_type
-; CHECK: // .b8 6                                // Abbrev [6] 0x1265:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 4731                            // DW_AT_type
-; CHECK: // .b8 6                                // Abbrev [6] 0x126a:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 4737                            // DW_AT_type
-; CHECK: // .b8 6                                // Abbrev [6] 0x126f:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 4737                            // DW_AT_type
-; CHECK: // .b8 6                                // Abbrev [6] 0x1274:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 4772                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 18                               // Abbrev [18] 0x127a:0x1 DW_TAG_pointer_type
-; CHECK: // .b8 8                                // Abbrev [8] 0x127b:0x5 DW_TAG_pointer_type
-; CHECK: // .b32 4736                            // DW_AT_type
-; CHECK: // .b8 19                               // Abbrev [19] 0x1280:0x1 DW_TAG_const_type
-; CHECK: // .b8 11                               // Abbrev [11] 0x1281:0xe DW_TAG_typedef
-; CHECK: // .b32 4751                            // DW_AT_type
-; CHECK: // .b8 115,105,122,101,95,116           // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 8                                // DW_AT_decl_file
-; CHECK: // .b8 62                               // DW_AT_decl_line
-; CHECK: // .b8 7                                // Abbrev [7] 0x128f:0x15 DW_TAG_base_type
-; CHECK: // .b8 108,111,110,103,32,117,110,115,105,103,110,101,100,32,105,110,116 // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 7                                // DW_AT_encoding
-; CHECK: // .b8 8                                // DW_AT_byte_size
-; CHECK: // .b8 20                               // Abbrev [20] 0x12a4:0x16 DW_TAG_typedef
-; CHECK: // .b32 4794                            // DW_AT_type
-; CHECK: // .b8 95,95,99,111,109,112,97,114,95,102,110,95,116 // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 4                                // DW_AT_decl_file
-; CHECK: // .b8 230                              // DW_AT_decl_line
-; CHECK: // .b8 2
-; CHECK: // .b8 8                                // Abbrev [8] 0x12ba:0x5 DW_TAG_pointer_type
-; CHECK: // .b32 4799                            // DW_AT_type
-; CHECK: // .b8 21                               // Abbrev [21] 0x12bf:0x10 DW_TAG_subroutine_type
-; CHECK: // .b32 2332                            // DW_AT_type
-; CHECK: // .b8 6                                // Abbrev [6] 0x12c4:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 4731                            // DW_AT_type
-; CHECK: // .b8 6                                // Abbrev [6] 0x12c9:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 4731                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 16                               // Abbrev [16] 0x12cf:0x1c DW_TAG_subprogram
-; CHECK: // .b8 99,97,108,108,111,99             // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 4                                // DW_AT_decl_file
-; CHECK: // .b8 212                              // DW_AT_decl_line
-; CHECK: // .b8 1
-; CHECK: // .b32 4730                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 1                                // DW_AT_external
-; CHECK: // .b8 6                                // Abbrev [6] 0x12e0:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 4737                            // DW_AT_type
-; CHECK: // .b8 6                                // Abbrev [6] 0x12e5:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 4737                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 16                               // Abbrev [16] 0x12eb:0x19 DW_TAG_subprogram
-; CHECK: // .b8 100,105,118                      // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 4                                // DW_AT_decl_file
-; CHECK: // .b8 21                               // DW_AT_decl_line
-; CHECK: // .b8 3
-; CHECK: // .b32 4500                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 1                                // DW_AT_external
-; CHECK: // .b8 6                                // Abbrev [6] 0x12f9:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 2332                            // DW_AT_type
-; CHECK: // .b8 6                                // Abbrev [6] 0x12fe:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 2332                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 22                               // Abbrev [22] 0x1304:0x12 DW_TAG_subprogram
-; CHECK: // .b8 101,120,105,116                  // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 4                                // DW_AT_decl_file
-; CHECK: // .b8 31                               // DW_AT_decl_line
-; CHECK: // .b8 2
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 1                                // DW_AT_external
-; CHECK: // .b8 1                                // DW_AT_noreturn
-; CHECK: // .b8 6                                // Abbrev [6] 0x1310:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 2332                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 23                               // Abbrev [23] 0x1316:0x11 DW_TAG_subprogram
-; CHECK: // .b8 102,114,101,101                  // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 4                                // DW_AT_decl_file
-; CHECK: // .b8 227                              // DW_AT_decl_line
-; CHECK: // .b8 1
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 1                                // DW_AT_external
-; CHECK: // .b8 6                                // Abbrev [6] 0x1321:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 4730                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 16                               // Abbrev [16] 0x1327:0x17 DW_TAG_subprogram
-; CHECK: // .b8 103,101,116,101,110,118          // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 4                                // DW_AT_decl_file
-; CHECK: // .b8 52                               // DW_AT_decl_line
-; CHECK: // .b8 2
-; CHECK: // .b32 4926                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 1                                // DW_AT_external
-; CHECK: // .b8 6                                // Abbrev [6] 0x1338:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 3389                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 8                                // Abbrev [8] 0x133e:0x5 DW_TAG_pointer_type
-; CHECK: // .b32 3399                            // DW_AT_type
-; CHECK: // .b8 16                               // Abbrev [16] 0x1343:0x15 DW_TAG_subprogram
-; CHECK: // .b8 108,97,98,115                    // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 4                                // DW_AT_decl_file
-; CHECK: // .b8 8                                // DW_AT_decl_line
-; CHECK: // .b8 3
-; CHECK: // .b32 2917                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 1                                // DW_AT_external
-; CHECK: // .b8 6                                // Abbrev [6] 0x1352:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 2917                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 16                               // Abbrev [16] 0x1358:0x1a DW_TAG_subprogram
-; CHECK: // .b8 108,100,105,118                  // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 4                                // DW_AT_decl_file
-; CHECK: // .b8 23                               // DW_AT_decl_line
-; CHECK: // .b8 3
-; CHECK: // .b32 4515                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 1                                // DW_AT_external
-; CHECK: // .b8 6                                // Abbrev [6] 0x1367:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 2917                            // DW_AT_type
-; CHECK: // .b8 6                                // Abbrev [6] 0x136c:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 2917                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 16                               // Abbrev [16] 0x1372:0x17 DW_TAG_subprogram
-; CHECK: // .b8 109,97,108,108,111,99            // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 4                                // DW_AT_decl_file
-; CHECK: // .b8 210                              // DW_AT_decl_line
-; CHECK: // .b8 1
-; CHECK: // .b32 4730                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 1                                // DW_AT_external
-; CHECK: // .b8 6                                // Abbrev [6] 0x1383:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 4737                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 16                               // Abbrev [16] 0x1389:0x1b DW_TAG_subprogram
-; CHECK: // .b8 109,98,108,101,110               // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 4                                // DW_AT_decl_file
-; CHECK: // .b8 95                               // DW_AT_decl_line
-; CHECK: // .b8 3
-; CHECK: // .b32 2332                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 1                                // DW_AT_external
-; CHECK: // .b8 6                                // Abbrev [6] 0x1399:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 3389                            // DW_AT_type
-; CHECK: // .b8 6                                // Abbrev [6] 0x139e:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 4737                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 16                               // Abbrev [16] 0x13a4:0x23 DW_TAG_subprogram
-; CHECK: // .b8 109,98,115,116,111,119,99,115    // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 4                                // DW_AT_decl_file
-; CHECK: // .b8 106                              // DW_AT_decl_line
-; CHECK: // .b8 3
-; CHECK: // .b32 4737                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 1                                // DW_AT_external
-; CHECK: // .b8 6                                // Abbrev [6] 0x13b7:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 5063                            // DW_AT_type
-; CHECK: // .b8 6                                // Abbrev [6] 0x13bc:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 3389                            // DW_AT_type
-; CHECK: // .b8 6                                // Abbrev [6] 0x13c1:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 4737                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 8                                // Abbrev [8] 0x13c7:0x5 DW_TAG_pointer_type
-; CHECK: // .b32 5068                            // DW_AT_type
-; CHECK: // .b8 7                                // Abbrev [7] 0x13cc:0xb DW_TAG_base_type
-; CHECK: // .b8 119,99,104,97,114,95,116         // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 5                                // DW_AT_encoding
-; CHECK: // .b8 4                                // DW_AT_byte_size
-; CHECK: // .b8 16                               // Abbrev [16] 0x13d7:0x21 DW_TAG_subprogram
-; CHECK: // .b8 109,98,116,111,119,99            // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 4                                // DW_AT_decl_file
-; CHECK: // .b8 98                               // DW_AT_decl_line
-; CHECK: // .b8 3
-; CHECK: // .b32 2332                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 1                                // DW_AT_external
-; CHECK: // .b8 6                                // Abbrev [6] 0x13e8:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 5063                            // DW_AT_type
-; CHECK: // .b8 6                                // Abbrev [6] 0x13ed:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 3389                            // DW_AT_type
-; CHECK: // .b8 6                                // Abbrev [6] 0x13f2:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 4737                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 23                               // Abbrev [23] 0x13f8:0x21 DW_TAG_subprogram
-; CHECK: // .b8 113,115,111,114,116              // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 4                                // DW_AT_decl_file
-; CHECK: // .b8 253                              // DW_AT_decl_line
-; CHECK: // .b8 2
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 1                                // DW_AT_external
-; CHECK: // .b8 6                                // Abbrev [6] 0x1404:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 4730                            // DW_AT_type
-; CHECK: // .b8 6                                // Abbrev [6] 0x1409:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 4737                            // DW_AT_type
-; CHECK: // .b8 6                                // Abbrev [6] 0x140e:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 4737                            // DW_AT_type
-; CHECK: // .b8 6                                // Abbrev [6] 0x1413:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 4772                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 24                               // Abbrev [24] 0x1419:0xf DW_TAG_subprogram
-; CHECK: // .b8 114,97,110,100                   // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 4                                // DW_AT_decl_file
-; CHECK: // .b8 118                              // DW_AT_decl_line
-; CHECK: // .b8 1
-; CHECK: // .b32 2332                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 1                                // DW_AT_external
-; CHECK: // .b8 16                               // Abbrev [16] 0x1428:0x1d DW_TAG_subprogram
-; CHECK: // .b8 114,101,97,108,108,111,99        // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 4                                // DW_AT_decl_file
-; CHECK: // .b8 224                              // DW_AT_decl_line
-; CHECK: // .b8 1
-; CHECK: // .b32 4730                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 1                                // DW_AT_external
-; CHECK: // .b8 6                                // Abbrev [6] 0x143a:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 4730                            // DW_AT_type
-; CHECK: // .b8 6                                // Abbrev [6] 0x143f:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 4737                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 23                               // Abbrev [23] 0x1445:0x12 DW_TAG_subprogram
-; CHECK: // .b8 115,114,97,110,100               // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 4                                // DW_AT_decl_file
-; CHECK: // .b8 120                              // DW_AT_decl_line
-; CHECK: // .b8 1
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 1                                // DW_AT_external
-; CHECK: // .b8 6                                // Abbrev [6] 0x1451:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 5207                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 7                                // Abbrev [7] 0x1457:0x10 DW_TAG_base_type
-; CHECK: // .b8 117,110,115,105,103,110,101,100,32,105,110,116 // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 7                                // DW_AT_encoding
-; CHECK: // .b8 4                                // DW_AT_byte_size
-; CHECK: // .b8 10                               // Abbrev [10] 0x1467:0x1b DW_TAG_subprogram
-; CHECK: // .b8 115,116,114,116,111,100          // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 4                                // DW_AT_decl_file
-; CHECK: // .b8 164                              // DW_AT_decl_line
-; CHECK: // .b32 3379                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 1                                // DW_AT_external
-; CHECK: // .b8 6                                // Abbrev [6] 0x1477:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 3389                            // DW_AT_type
-; CHECK: // .b8 6                                // Abbrev [6] 0x147c:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 5250                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 8                                // Abbrev [8] 0x1482:0x5 DW_TAG_pointer_type
-; CHECK: // .b32 4926                            // DW_AT_type
-; CHECK: // .b8 10                               // Abbrev [10] 0x1487:0x20 DW_TAG_subprogram
-; CHECK: // .b8 115,116,114,116,111,108          // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 4                                // DW_AT_decl_file
-; CHECK: // .b8 183                              // DW_AT_decl_line
-; CHECK: // .b32 2917                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 1                                // DW_AT_external
-; CHECK: // .b8 6                                // Abbrev [6] 0x1497:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 3389                            // DW_AT_type
-; CHECK: // .b8 6                                // Abbrev [6] 0x149c:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 5250                            // DW_AT_type
-; CHECK: // .b8 6                                // Abbrev [6] 0x14a1:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 2332                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 10                               // Abbrev [10] 0x14a7:0x21 DW_TAG_subprogram
-; CHECK: // .b8 115,116,114,116,111,117,108      // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 4                                // DW_AT_decl_file
-; CHECK: // .b8 187                              // DW_AT_decl_line
-; CHECK: // .b32 4751                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 1                                // DW_AT_external
-; CHECK: // .b8 6                                // Abbrev [6] 0x14b8:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 3389                            // DW_AT_type
-; CHECK: // .b8 6                                // Abbrev [6] 0x14bd:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 5250                            // DW_AT_type
-; CHECK: // .b8 6                                // Abbrev [6] 0x14c2:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 2332                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 16                               // Abbrev [16] 0x14c8:0x17 DW_TAG_subprogram
-; CHECK: // .b8 115,121,115,116,101,109          // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 4                                // DW_AT_decl_file
-; CHECK: // .b8 205                              // DW_AT_decl_line
-; CHECK: // .b8 2
-; CHECK: // .b32 2332                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 1                                // DW_AT_external
-; CHECK: // .b8 6                                // Abbrev [6] 0x14d9:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 3389                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 16                               // Abbrev [16] 0x14df:0x23 DW_TAG_subprogram
-; CHECK: // .b8 119,99,115,116,111,109,98,115    // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 4                                // DW_AT_decl_file
-; CHECK: // .b8 109                              // DW_AT_decl_line
-; CHECK: // .b8 3
-; CHECK: // .b32 4737                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 1                                // DW_AT_external
-; CHECK: // .b8 6                                // Abbrev [6] 0x14f2:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 4926                            // DW_AT_type
-; CHECK: // .b8 6                                // Abbrev [6] 0x14f7:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 5378                            // DW_AT_type
-; CHECK: // .b8 6                                // Abbrev [6] 0x14fc:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 4737                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 8                                // Abbrev [8] 0x1502:0x5 DW_TAG_pointer_type
-; CHECK: // .b32 5383                            // DW_AT_type
-; CHECK: // .b8 9                                // Abbrev [9] 0x1507:0x5 DW_TAG_const_type
-; CHECK: // .b32 5068                            // DW_AT_type
-; CHECK: // .b8 16                               // Abbrev [16] 0x150c:0x1c DW_TAG_subprogram
-; CHECK: // .b8 119,99,116,111,109,98            // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 4                                // DW_AT_decl_file
-; CHECK: // .b8 102                              // DW_AT_decl_line
-; CHECK: // .b8 3
-; CHECK: // .b32 2332                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 1                                // DW_AT_external
-; CHECK: // .b8 6                                // Abbrev [6] 0x151d:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 4926                            // DW_AT_type
-; CHECK: // .b8 6                                // Abbrev [6] 0x1522:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 5068                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 2                                // Abbrev [2] 0x1528:0x78 DW_TAG_namespace
-; CHECK: // .b8 95,95,103,110,117,95,99,120,120  // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 3                                // Abbrev [3] 0x1533:0x7 DW_TAG_imported_declaration
-; CHECK: // .b8 5                                // DW_AT_decl_file
-; CHECK: // .b8 201                              // DW_AT_decl_line
-; CHECK: // .b32 5536                            // DW_AT_import
-; CHECK: // .b8 3                                // Abbrev [3] 0x153a:0x7 DW_TAG_imported_declaration
-; CHECK: // .b8 5                                // DW_AT_decl_file
-; CHECK: // .b8 207                              // DW_AT_decl_line
-; CHECK: // .b32 5585                            // DW_AT_import
-; CHECK: // .b8 3                                // Abbrev [3] 0x1541:0x7 DW_TAG_imported_declaration
-; CHECK: // .b8 5                                // DW_AT_decl_file
-; CHECK: // .b8 211                              // DW_AT_decl_line
-; CHECK: // .b32 5604                            // DW_AT_import
-; CHECK: // .b8 3                                // Abbrev [3] 0x1548:0x7 DW_TAG_imported_declaration
-; CHECK: // .b8 5                                // DW_AT_decl_file
-; CHECK: // .b8 217                              // DW_AT_decl_line
-; CHECK: // .b32 5626                            // DW_AT_import
-; CHECK: // .b8 3                                // Abbrev [3] 0x154f:0x7 DW_TAG_imported_declaration
-; CHECK: // .b8 5                                // DW_AT_decl_file
-; CHECK: // .b8 228                              // DW_AT_decl_line
-; CHECK: // .b32 5653                            // DW_AT_import
-; CHECK: // .b8 3                                // Abbrev [3] 0x1556:0x7 DW_TAG_imported_declaration
-; CHECK: // .b8 5                                // DW_AT_decl_file
-; CHECK: // .b8 229                              // DW_AT_decl_line
-; CHECK: // .b32 5675                            // DW_AT_import
-; CHECK: // .b8 3                                // Abbrev [3] 0x155d:0x7 DW_TAG_imported_declaration
-; CHECK: // .b8 5                                // DW_AT_decl_file
-; CHECK: // .b8 230                              // DW_AT_decl_line
-; CHECK: // .b32 5708                            // DW_AT_import
-; CHECK: // .b8 3                                // Abbrev [3] 0x1564:0x7 DW_TAG_imported_declaration
-; CHECK: // .b8 5                                // DW_AT_decl_file
-; CHECK: // .b8 232                              // DW_AT_decl_line
-; CHECK: // .b32 5768                            // DW_AT_import
-; CHECK: // .b8 3                                // Abbrev [3] 0x156b:0x7 DW_TAG_imported_declaration
-; CHECK: // .b8 5                                // DW_AT_decl_file
-; CHECK: // .b8 233                              // DW_AT_decl_line
-; CHECK: // .b32 5795                            // DW_AT_import
-; CHECK: // .b8 25                               // Abbrev [25] 0x1572:0x2d DW_TAG_subprogram
-; CHECK: // .b8 95,90,78,57,95,95,103,110,117,95,99,120,120,51,100,105,118,69,120,120 // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 100,105,118                      // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 5                                // DW_AT_decl_file
-; CHECK: // .b8 214                              // DW_AT_decl_line
-; CHECK: // .b32 5536                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 1                                // DW_AT_external
-; CHECK: // .b8 6                                // Abbrev [6] 0x1594:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1508                            // DW_AT_type
-; CHECK: // .b8 6                                // Abbrev [6] 0x1599:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1508                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 11                               // Abbrev [11] 0x15a0:0xf DW_TAG_typedef
-; CHECK: // .b32 5551                            // DW_AT_type
-; CHECK: // .b8 108,108,100,105,118,95,116       // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 4                                // DW_AT_decl_file
-; CHECK: // .b8 121                              // DW_AT_decl_line
-; CHECK: // .b8 13                               // Abbrev [13] 0x15af:0x22 DW_TAG_structure_type
-; CHECK: // .b8 16                               // DW_AT_byte_size
-; CHECK: // .b8 4                                // DW_AT_decl_file
-; CHECK: // .b8 117                              // DW_AT_decl_line
-; CHECK: // .b8 14                               // Abbrev [14] 0x15b3:0xf DW_TAG_member
-; CHECK: // .b8 113,117,111,116                  // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b32 1508                            // DW_AT_type
-; CHECK: // .b8 4                                // DW_AT_decl_file
-; CHECK: // .b8 119                              // DW_AT_decl_line
-; CHECK: // .b8 2                                // DW_AT_data_member_location
-; CHECK: // .b8 35
-; CHECK: // .b8 0
-; CHECK: // .b8 14                               // Abbrev [14] 0x15c2:0xe DW_TAG_member
-; CHECK: // .b8 114,101,109                      // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b32 1508                            // DW_AT_type
-; CHECK: // .b8 4                                // DW_AT_decl_file
-; CHECK: // .b8 120                              // DW_AT_decl_line
-; CHECK: // .b8 2                                // DW_AT_data_member_location
-; CHECK: // .b8 35
-; CHECK: // .b8 8
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 22                               // Abbrev [22] 0x15d1:0x13 DW_TAG_subprogram
-; CHECK: // .b8 95,69,120,105,116                // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 4                                // DW_AT_decl_file
-; CHECK: // .b8 45                               // DW_AT_decl_line
-; CHECK: // .b8 2
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 1                                // DW_AT_external
-; CHECK: // .b8 1                                // DW_AT_noreturn
-; CHECK: // .b8 6                                // Abbrev [6] 0x15de:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 2332                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 16                               // Abbrev [16] 0x15e4:0x16 DW_TAG_subprogram
-; CHECK: // .b8 108,108,97,98,115                // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 4                                // DW_AT_decl_file
-; CHECK: // .b8 12                               // DW_AT_decl_line
-; CHECK: // .b8 3
-; CHECK: // .b32 1508                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 1                                // DW_AT_external
-; CHECK: // .b8 6                                // Abbrev [6] 0x15f4:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1508                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 16                               // Abbrev [16] 0x15fa:0x1b DW_TAG_subprogram
-; CHECK: // .b8 108,108,100,105,118              // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 4                                // DW_AT_decl_file
-; CHECK: // .b8 29                               // DW_AT_decl_line
-; CHECK: // .b8 3
-; CHECK: // .b32 5536                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 1                                // DW_AT_external
-; CHECK: // .b8 6                                // Abbrev [6] 0x160a:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1508                            // DW_AT_type
-; CHECK: // .b8 6                                // Abbrev [6] 0x160f:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1508                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 16                               // Abbrev [16] 0x1615:0x16 DW_TAG_subprogram
-; CHECK: // .b8 97,116,111,108,108               // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 4                                // DW_AT_decl_file
-; CHECK: // .b8 36                               // DW_AT_decl_line
-; CHECK: // .b8 1
-; CHECK: // .b32 1508                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 1                                // DW_AT_external
-; CHECK: // .b8 6                                // Abbrev [6] 0x1625:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 3389                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 10                               // Abbrev [10] 0x162b:0x21 DW_TAG_subprogram
-; CHECK: // .b8 115,116,114,116,111,108,108      // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 4                                // DW_AT_decl_file
-; CHECK: // .b8 209                              // DW_AT_decl_line
-; CHECK: // .b32 1508                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 1                                // DW_AT_external
-; CHECK: // .b8 6                                // Abbrev [6] 0x163c:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 3389                            // DW_AT_type
-; CHECK: // .b8 6                                // Abbrev [6] 0x1641:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 5250                            // DW_AT_type
-; CHECK: // .b8 6                                // Abbrev [6] 0x1646:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 2332                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 10                               // Abbrev [10] 0x164c:0x22 DW_TAG_subprogram
-; CHECK: // .b8 115,116,114,116,111,117,108,108  // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 4                                // DW_AT_decl_file
-; CHECK: // .b8 214                              // DW_AT_decl_line
-; CHECK: // .b32 5742                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 1                                // DW_AT_external
-; CHECK: // .b8 6                                // Abbrev [6] 0x165e:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 3389                            // DW_AT_type
-; CHECK: // .b8 6                                // Abbrev [6] 0x1663:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 5250                            // DW_AT_type
-; CHECK: // .b8 6                                // Abbrev [6] 0x1668:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 2332                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 7                                // Abbrev [7] 0x166e:0x1a DW_TAG_base_type
-; CHECK: // .b8 108,111,110,103,32,108,111,110,103,32,117,110,115,105,103,110,101,100,32,105,110,116 // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 7                                // DW_AT_encoding
-; CHECK: // .b8 8                                // DW_AT_byte_size
-; CHECK: // .b8 10                               // Abbrev [10] 0x1688:0x1b DW_TAG_subprogram
-; CHECK: // .b8 115,116,114,116,111,102          // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 4                                // DW_AT_decl_file
-; CHECK: // .b8 172                              // DW_AT_decl_line
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 1                                // DW_AT_external
-; CHECK: // .b8 6                                // Abbrev [6] 0x1698:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 3389                            // DW_AT_type
-; CHECK: // .b8 6                                // Abbrev [6] 0x169d:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 5250                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 10                               // Abbrev [10] 0x16a3:0x1c DW_TAG_subprogram
-; CHECK: // .b8 115,116,114,116,111,108,100      // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 4                                // DW_AT_decl_file
-; CHECK: // .b8 175                              // DW_AT_decl_line
-; CHECK: // .b32 5823                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 1                                // DW_AT_external
-; CHECK: // .b8 6                                // Abbrev [6] 0x16b4:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 3389                            // DW_AT_type
-; CHECK: // .b8 6                                // Abbrev [6] 0x16b9:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 5250                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 7                                // Abbrev [7] 0x16bf:0xf DW_TAG_base_type
-; CHECK: // .b8 108,111,110,103,32,100,111,117,98,108,101 // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 4                                // DW_AT_encoding
-; CHECK: // .b8 8                                // DW_AT_byte_size
-; CHECK: // .b8 26                               // Abbrev [26] 0x16ce:0x20 DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,53,97,99,111,115,102,102 // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 97,99,111,115,102                // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 9                                // DW_AT_decl_file
-; CHECK: // .b8 62                               // DW_AT_decl_line
-; CHECK: // .b8 5
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0x16e8:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 26                               // Abbrev [26] 0x16ee:0x22 DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,54,97,99,111,115,104,102,102 // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 97,99,111,115,104,102            // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 9                                // DW_AT_decl_file
-; CHECK: // .b8 90                               // DW_AT_decl_line
-; CHECK: // .b8 5
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0x170a:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 26                               // Abbrev [26] 0x1710:0x20 DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,53,97,115,105,110,102,102 // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 97,115,105,110,102               // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 9                                // DW_AT_decl_file
-; CHECK: // .b8 57                               // DW_AT_decl_line
-; CHECK: // .b8 5
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0x172a:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 26                               // Abbrev [26] 0x1730:0x22 DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,54,97,115,105,110,104,102,102 // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 97,115,105,110,104,102           // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 9                                // DW_AT_decl_file
-; CHECK: // .b8 95                               // DW_AT_decl_line
-; CHECK: // .b8 5
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0x174c:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 26                               // Abbrev [26] 0x1752:0x28 DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,54,97,116,97,110,50,102,102,102 // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 97,116,97,110,50,102             // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 9                                // DW_AT_decl_file
-; CHECK: // .b8 47                               // DW_AT_decl_line
-; CHECK: // .b8 5
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0x176f:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 6                                // Abbrev [6] 0x1774:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 26                               // Abbrev [26] 0x177a:0x20 DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,53,97,116,97,110,102,102 // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 97,116,97,110,102                // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 9                                // DW_AT_decl_file
-; CHECK: // .b8 52                               // DW_AT_decl_line
-; CHECK: // .b8 5
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0x1794:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 26                               // Abbrev [26] 0x179a:0x22 DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,54,97,116,97,110,104,102,102 // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 97,116,97,110,104,102            // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 9                                // DW_AT_decl_file
-; CHECK: // .b8 100                              // DW_AT_decl_line
-; CHECK: // .b8 5
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0x17b6:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 26                               // Abbrev [26] 0x17bc:0x20 DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,53,99,98,114,116,102,102 // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 99,98,114,116,102                // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 9                                // DW_AT_decl_file
-; CHECK: // .b8 150                              // DW_AT_decl_line
-; CHECK: // .b8 5
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0x17d6:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 26                               // Abbrev [26] 0x17dc:0x20 DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,53,99,101,105,108,102,102 // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 99,101,105,108,102               // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 11                               // DW_AT_decl_file
-; CHECK: // .b8 155                              // DW_AT_decl_line
-; CHECK: // .b8 2
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0x17f6:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 26                               // Abbrev [26] 0x17fc:0x2e DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,57,99,111,112,121,115,105,103,110,102,102,102 // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 99,111,112,121,115,105,103,110,102 // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 9                                // DW_AT_decl_file
-; CHECK: // .b8 165                              // DW_AT_decl_line
-; CHECK: // .b8 4
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0x181f:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 6                                // Abbrev [6] 0x1824:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 26                               // Abbrev [26] 0x182a:0x1e DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,52,99,111,115,102,102   // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 99,111,115,102                   // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 9                                // DW_AT_decl_file
-; CHECK: // .b8 219                              // DW_AT_decl_line
-; CHECK: // .b8 4
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0x1842:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 26                               // Abbrev [26] 0x1848:0x20 DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,53,99,111,115,104,102,102 // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 99,111,115,104,102               // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 9                                // DW_AT_decl_file
-; CHECK: // .b8 32                               // DW_AT_decl_line
-; CHECK: // .b8 5
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0x1862:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 26                               // Abbrev [26] 0x1868:0x20 DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,53,101,114,102,99,102,102 // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 101,114,102,99,102               // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 9                                // DW_AT_decl_file
-; CHECK: // .b8 210                              // DW_AT_decl_line
-; CHECK: // .b8 5
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0x1882:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 26                               // Abbrev [26] 0x1888:0x1e DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,52,101,114,102,102,102  // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 101,114,102,102                  // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 9                                // DW_AT_decl_file
-; CHECK: // .b8 200                              // DW_AT_decl_line
-; CHECK: // .b8 5
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0x18a0:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 26                               // Abbrev [26] 0x18a6:0x20 DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,53,101,120,112,50,102,102 // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 101,120,112,50,102               // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 11                               // DW_AT_decl_file
-; CHECK: // .b8 145                              // DW_AT_decl_line
-; CHECK: // .b8 2
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0x18c0:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 26                               // Abbrev [26] 0x18c6:0x1e DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,52,101,120,112,102,102  // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 101,120,112,102                  // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 9                                // DW_AT_decl_file
-; CHECK: // .b8 14                               // DW_AT_decl_line
-; CHECK: // .b8 5
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0x18de:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 26                               // Abbrev [26] 0x18e4:0x22 DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,54,101,120,112,109,49,102,102 // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 101,120,112,109,49,102           // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 9                                // DW_AT_decl_file
-; CHECK: // .b8 105                              // DW_AT_decl_line
-; CHECK: // .b8 5
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0x1900:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 26                               // Abbrev [26] 0x1906:0x20 DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,53,102,97,98,115,102,102 // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 102,97,98,115,102                // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 11                               // DW_AT_decl_file
-; CHECK: // .b8 95                               // DW_AT_decl_line
-; CHECK: // .b8 2
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0x1920:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 26                               // Abbrev [26] 0x1926:0x26 DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,53,102,100,105,109,102,102,102 // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 102,100,105,109,102              // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 9                                // DW_AT_decl_file
-; CHECK: // .b8 80                               // DW_AT_decl_line
-; CHECK: // .b8 6
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0x1941:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 6                                // Abbrev [6] 0x1946:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 26                               // Abbrev [26] 0x194c:0x22 DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,54,102,108,111,111,114,102,102 // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 102,108,111,111,114,102          // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 11                               // DW_AT_decl_file
-; CHECK: // .b8 85                               // DW_AT_decl_line
-; CHECK: // .b8 2
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0x1968:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 26                               // Abbrev [26] 0x196e:0x2a DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,52,102,109,97,102,102,102,102 // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 102,109,97,102                   // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 9                                // DW_AT_decl_file
-; CHECK: // .b8 32                               // DW_AT_decl_line
-; CHECK: // .b8 6
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0x1988:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 6                                // Abbrev [6] 0x198d:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 6                                // Abbrev [6] 0x1992:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 26                               // Abbrev [26] 0x1998:0x26 DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,53,102,109,97,120,102,102,102 // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 102,109,97,120,102               // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 11                               // DW_AT_decl_file
-; CHECK: // .b8 110                              // DW_AT_decl_line
-; CHECK: // .b8 2
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0x19b3:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 6                                // Abbrev [6] 0x19b8:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 26                               // Abbrev [26] 0x19be:0x26 DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,53,102,109,105,110,102,102,102 // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 102,109,105,110,102              // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 11                               // DW_AT_decl_file
-; CHECK: // .b8 105                              // DW_AT_decl_line
-; CHECK: // .b8 2
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0x19d9:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 6                                // Abbrev [6] 0x19de:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 26                               // Abbrev [26] 0x19e4:0x26 DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,53,102,109,111,100,102,102,102 // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 102,109,111,100,102              // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 9                                // DW_AT_decl_file
-; CHECK: // .b8 17                               // DW_AT_decl_line
-; CHECK: // .b8 6
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0x19ff:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 6                                // Abbrev [6] 0x1a04:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 26                               // Abbrev [26] 0x1a0a:0x29 DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,54,102,114,101,120,112,102,102,80,105 // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 102,114,101,120,112,102          // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 9                                // DW_AT_decl_file
-; CHECK: // .b8 7                                // DW_AT_decl_line
-; CHECK: // .b8 6
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0x1a28:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 6                                // Abbrev [6] 0x1a2d:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 2377                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 26                               // Abbrev [26] 0x1a33:0x28 DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,54,104,121,112,111,116,102,102,102 // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 104,121,112,111,116,102          // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 9                                // DW_AT_decl_file
-; CHECK: // .b8 110                              // DW_AT_decl_line
-; CHECK: // .b8 5
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0x1a50:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 6                                // Abbrev [6] 0x1a55:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 26                               // Abbrev [26] 0x1a5b:0x22 DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,54,105,108,111,103,98,102,102 // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 105,108,111,103,98,102           // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 9                                // DW_AT_decl_file
-; CHECK: // .b8 85                               // DW_AT_decl_line
-; CHECK: // .b8 6
-; CHECK: // .b32 2332                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0x1a77:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 26                               // Abbrev [26] 0x1a7d:0x28 DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,54,108,100,101,120,112,102,102,105 // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 108,100,101,120,112,102          // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 9                                // DW_AT_decl_file
-; CHECK: // .b8 240                              // DW_AT_decl_line
-; CHECK: // .b8 5
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0x1a9a:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 6                                // Abbrev [6] 0x1a9f:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 2332                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 26                               // Abbrev [26] 0x1aa5:0x24 DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,55,108,103,97,109,109,97,102,102 // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 108,103,97,109,109,97,102        // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 9                                // DW_AT_decl_file
-; CHECK: // .b8 235                              // DW_AT_decl_line
-; CHECK: // .b8 5
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0x1ac3:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 26                               // Abbrev [26] 0x1ac9:0x24 DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,55,108,108,114,105,110,116,102,102 // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 108,108,114,105,110,116,102      // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 9                                // DW_AT_decl_file
-; CHECK: // .b8 125                              // DW_AT_decl_line
-; CHECK: // .b8 4
-; CHECK: // .b32 1508                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0x1ae7:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 26                               // Abbrev [26] 0x1aed:0x26 DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,56,108,108,114,111,117,110,100,102,102 // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 108,108,114,111,117,110,100,102  // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 9                                // DW_AT_decl_file
-; CHECK: // .b8 66                               // DW_AT_decl_line
-; CHECK: // .b8 6
-; CHECK: // .b32 1508                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0x1b0d:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 26                               // Abbrev [26] 0x1b13:0x22 DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,54,108,111,103,49,48,102,102 // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 108,111,103,49,48,102            // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 9                                // DW_AT_decl_file
-; CHECK: // .b8 76                               // DW_AT_decl_line
-; CHECK: // .b8 5
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0x1b2f:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 26                               // Abbrev [26] 0x1b35:0x22 DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,54,108,111,103,49,112,102,102 // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 108,111,103,49,112,102           // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 9                                // DW_AT_decl_file
-; CHECK: // .b8 85                               // DW_AT_decl_line
-; CHECK: // .b8 5
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0x1b51:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 26                               // Abbrev [26] 0x1b57:0x20 DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,53,108,111,103,50,102,102 // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 108,111,103,50,102               // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 9                                // DW_AT_decl_file
-; CHECK: // .b8 5                                // DW_AT_decl_line
-; CHECK: // .b8 5
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0x1b71:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 26                               // Abbrev [26] 0x1b77:0x20 DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,53,108,111,103,98,102,102 // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 108,111,103,98,102               // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 9                                // DW_AT_decl_file
-; CHECK: // .b8 90                               // DW_AT_decl_line
-; CHECK: // .b8 6
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0x1b91:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 26                               // Abbrev [26] 0x1b97:0x1e DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,52,108,111,103,102,102  // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 108,111,103,102                  // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 9                                // DW_AT_decl_file
-; CHECK: // .b8 67                               // DW_AT_decl_line
-; CHECK: // .b8 5
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0x1baf:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 26                               // Abbrev [26] 0x1bb5:0x22 DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,54,108,114,105,110,116,102,102 // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 108,114,105,110,116,102          // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 9                                // DW_AT_decl_file
-; CHECK: // .b8 116                              // DW_AT_decl_line
-; CHECK: // .b8 4
-; CHECK: // .b32 2917                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0x1bd1:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 26                               // Abbrev [26] 0x1bd7:0x24 DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,55,108,114,111,117,110,100,102,102 // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 108,114,111,117,110,100,102      // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 9                                // DW_AT_decl_file
-; CHECK: // .b8 71                               // DW_AT_decl_line
-; CHECK: // .b8 6
-; CHECK: // .b32 2917                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0x1bf5:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 26                               // Abbrev [26] 0x1bfb:0x27 DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,53,109,111,100,102,102,102,80,102 // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 109,111,100,102,102              // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 9                                // DW_AT_decl_file
-; CHECK: // .b8 12                               // DW_AT_decl_line
-; CHECK: // .b8 6
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0x1c17:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 6                                // Abbrev [6] 0x1c1c:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 3345                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 26                               // Abbrev [26] 0x1c22:0x2b DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,49,48,110,101,97,114,98,121,105,110,116,102,102 // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 110,101,97,114,98,121,105,110,116,102 // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 9                                // DW_AT_decl_file
-; CHECK: // .b8 130                              // DW_AT_decl_line
-; CHECK: // .b8 4
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0x1c47:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 26                               // Abbrev [26] 0x1c4d:0x31 DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,49,48,110,101,120,116,97,102,116,101,114,102,102,102 // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 110,101,120,116,97,102,116,101,114,102 // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 9                                // DW_AT_decl_file
-; CHECK: // .b8 194                              // DW_AT_decl_line
-; CHECK: // .b8 4
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0x1c73:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 6                                // Abbrev [6] 0x1c78:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 26                               // Abbrev [26] 0x1c7e:0x24 DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,52,112,111,119,102,102,102 // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 112,111,119,102                  // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 9                                // DW_AT_decl_file
-; CHECK: // .b8 47                               // DW_AT_decl_line
-; CHECK: // .b8 6
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0x1c97:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 6                                // Abbrev [6] 0x1c9c:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 26                               // Abbrev [26] 0x1ca2:0x31 DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,49,48,114,101,109,97,105,110,100,101,114,102,102,102 // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 114,101,109,97,105,110,100,101,114,102 // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 9                                // DW_AT_decl_file
-; CHECK: // .b8 22                               // DW_AT_decl_line
-; CHECK: // .b8 6
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0x1cc8:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 6                                // Abbrev [6] 0x1ccd:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 26                               // Abbrev [26] 0x1cd3:0x31 DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,55,114,101,109,113,117,111,102,102,102,80,105 // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 114,101,109,113,117,111,102      // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 9                                // DW_AT_decl_file
-; CHECK: // .b8 27                               // DW_AT_decl_line
-; CHECK: // .b8 6
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0x1cf4:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 6                                // Abbrev [6] 0x1cf9:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 6                                // Abbrev [6] 0x1cfe:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 2377                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 26                               // Abbrev [26] 0x1d04:0x20 DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,53,114,105,110,116,102,102 // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 114,105,110,116,102              // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 9                                // DW_AT_decl_file
-; CHECK: // .b8 111                              // DW_AT_decl_line
-; CHECK: // .b8 4
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0x1d1e:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 26                               // Abbrev [26] 0x1d24:0x22 DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,54,114,111,117,110,100,102,102 // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 114,111,117,110,100,102          // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 9                                // DW_AT_decl_file
-; CHECK: // .b8 61                               // DW_AT_decl_line
-; CHECK: // .b8 6
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0x1d40:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 26                               // Abbrev [26] 0x1d46:0x2c DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,56,115,99,97,108,98,108,110,102,102,108 // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 115,99,97,108,98,108,110,102     // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 9                                // DW_AT_decl_file
-; CHECK: // .b8 250                              // DW_AT_decl_line
-; CHECK: // .b8 5
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0x1d67:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 6                                // Abbrev [6] 0x1d6c:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 2917                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 26                               // Abbrev [26] 0x1d72:0x2a DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,55,115,99,97,108,98,110,102,102,105 // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 115,99,97,108,98,110,102         // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 9                                // DW_AT_decl_file
-; CHECK: // .b8 245                              // DW_AT_decl_line
-; CHECK: // .b8 5
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0x1d91:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 6                                // Abbrev [6] 0x1d96:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 2332                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 26                               // Abbrev [26] 0x1d9c:0x1e DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,52,115,105,110,102,102  // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 115,105,110,102                  // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 9                                // DW_AT_decl_file
-; CHECK: // .b8 210                              // DW_AT_decl_line
-; CHECK: // .b8 4
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0x1db4:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 26                               // Abbrev [26] 0x1dba:0x20 DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,53,115,105,110,104,102,102 // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 115,105,110,104,102              // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 9                                // DW_AT_decl_file
-; CHECK: // .b8 37                               // DW_AT_decl_line
-; CHECK: // .b8 5
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0x1dd4:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 26                               // Abbrev [26] 0x1dda:0x20 DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,53,115,113,114,116,102,102 // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 115,113,114,116,102              // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 11                               // DW_AT_decl_file
-; CHECK: // .b8 139                              // DW_AT_decl_line
-; CHECK: // .b8 3
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0x1df4:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 26                               // Abbrev [26] 0x1dfa:0x1e DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,52,116,97,110,102,102   // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 116,97,110,102                   // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 9                                // DW_AT_decl_file
-; CHECK: // .b8 252                              // DW_AT_decl_line
-; CHECK: // .b8 4
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0x1e12:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 26                               // Abbrev [26] 0x1e18:0x20 DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,53,116,97,110,104,102,102 // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 116,97,110,104,102               // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 9                                // DW_AT_decl_file
-; CHECK: // .b8 42                               // DW_AT_decl_line
-; CHECK: // .b8 5
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0x1e32:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 26                               // Abbrev [26] 0x1e38:0x24 DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,55,116,103,97,109,109,97,102,102 // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 116,103,97,109,109,97,102        // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 9                                // DW_AT_decl_file
-; CHECK: // .b8 56                               // DW_AT_decl_line
-; CHECK: // .b8 6
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0x1e56:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 26                               // Abbrev [26] 0x1e5c:0x22 DW_TAG_subprogram
-; CHECK: // .b8 95,90,76,54,116,114,117,110,99,102,102 // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 116,114,117,110,99,102           // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 11                               // DW_AT_decl_file
-; CHECK: // .b8 150                              // DW_AT_decl_line
-; CHECK: // .b8 2
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 6                                // Abbrev [6] 0x1e78:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 27                               // Abbrev [27] 0x1e7e:0x22a DW_TAG_structure_type
-; CHECK: // .b8 95,95,99,117,100,97,95,98,117,105,108,116,105,110,95,98,108,111,99,107,73,100,120,95,116 // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 1                                // DW_AT_byte_size
-; CHECK: // .b8 13                               // DW_AT_decl_file
-; CHECK: // .b8 77                               // DW_AT_decl_line
-; CHECK: // .b8 28                               // Abbrev [28] 0x1e9c:0x4f DW_TAG_subprogram
-; CHECK: // .b8 95,90,78,50,53,95,95,99,117,100,97,95,98,117,105,108,116,105,110,95,98,108,111,99,107,73,100,120,95,116,49,55,95,95,102,101,116,99,104,95 // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 98,117,105,108,116,105,110,95,120,69,118
-; CHECK: // .b8 0
-; CHECK: // .b8 95,95,102,101,116,99,104,95,98,117,105,108,116,105,110,95,120 // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 13                               // DW_AT_decl_file
-; CHECK: // .b8 78                               // DW_AT_decl_line
-; CHECK: // .b32 5207                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 1                                // DW_AT_external
-; CHECK: // .b8 28                               // Abbrev [28] 0x1eeb:0x4f DW_TAG_subprogram
-; CHECK: // .b8 95,90,78,50,53,95,95,99,117,100,97,95,98,117,105,108,116,105,110,95,98,108,111,99,107,73,100,120,95,116,49,55,95,95,102,101,116,99,104,95 // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 98,117,105,108,116,105,110,95,121,69,118
-; CHECK: // .b8 0
-; CHECK: // .b8 95,95,102,101,116,99,104,95,98,117,105,108,116,105,110,95,121 // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 13                               // DW_AT_decl_file
-; CHECK: // .b8 79                               // DW_AT_decl_line
-; CHECK: // .b32 5207                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 1                                // DW_AT_external
-; CHECK: // .b8 28                               // Abbrev [28] 0x1f3a:0x4f DW_TAG_subprogram
-; CHECK: // .b8 95,90,78,50,53,95,95,99,117,100,97,95,98,117,105,108,116,105,110,95,98,108,111,99,107,73,100,120,95,116,49,55,95,95,102,101,116,99,104,95 // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 98,117,105,108,116,105,110,95,122,69,118
-; CHECK: // .b8 0
-; CHECK: // .b8 95,95,102,101,116,99,104,95,98,117,105,108,116,105,110,95,122 // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 13                               // DW_AT_decl_file
-; CHECK: // .b8 80                               // DW_AT_decl_line
-; CHECK: // .b32 5207                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 1                                // DW_AT_external
-; CHECK: // .b8 25                               // Abbrev [25] 0x1f89:0x49 DW_TAG_subprogram
-; CHECK: // .b8 95,90,78,75,50,53,95,95,99,117,100,97,95,98,117,105,108,116,105,110,95,98,108,111,99,107,73,100,120,95,116,99,118,53,117,105,110,116,51,69 // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 118
-; CHECK: // .b8 0
-; CHECK: // .b8 111,112,101,114,97,116,111,114,32,117,105,110,116,51 // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 13                               // DW_AT_decl_file
-; CHECK: // .b8 83                               // DW_AT_decl_line
-; CHECK: // .b32 8360                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 1                                // DW_AT_external
-; CHECK: // .b8 29                               // Abbrev [29] 0x1fcb:0x6 DW_TAG_formal_parameter
-; CHECK: // .b32 8407                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_artificial
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 30                               // Abbrev [30] 0x1fd2:0x27 DW_TAG_subprogram
-; CHECK: // .b8 95,95,99,117,100,97,95,98,117,105,108,116,105,110,95,98,108,111,99,107,73,100,120,95,116 // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 13                               // DW_AT_decl_file
-; CHECK: // .b8 85                               // DW_AT_decl_line
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 1                                // DW_AT_external
-; CHECK: // .b8 3                                // DW_AT_accessibility
-; CHECK:                                         // DW_ACCESS_private
-; CHECK: // .b8 29                               // Abbrev [29] 0x1ff2:0x6 DW_TAG_formal_parameter
-; CHECK: // .b32 8417                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_artificial
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 30                               // Abbrev [30] 0x1ff9:0x2c DW_TAG_subprogram
-; CHECK: // .b8 95,95,99,117,100,97,95,98,117,105,108,116,105,110,95,98,108,111,99,107,73,100,120,95,116 // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 13                               // DW_AT_decl_file
-; CHECK: // .b8 85                               // DW_AT_decl_line
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 1                                // DW_AT_external
-; CHECK: // .b8 3                                // DW_AT_accessibility
-; CHECK:                                         // DW_ACCESS_private
-; CHECK: // .b8 29                               // Abbrev [29] 0x2019:0x6 DW_TAG_formal_parameter
-; CHECK: // .b32 8417                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_artificial
-; CHECK: // .b8 6                                // Abbrev [6] 0x201f:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 8422                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 31                               // Abbrev [31] 0x2025:0x43 DW_TAG_subprogram
-; CHECK: // .b8 95,90,78,75,50,53,95,95,99,117,100,97,95,98,117,105,108,116,105,110,95,98,108,111,99,107,73,100,120,95,116,97,83,69,82,75,83,95 // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 111,112,101,114,97,116,111,114,61 // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 13                               // DW_AT_decl_file
-; CHECK: // .b8 85                               // DW_AT_decl_line
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 1                                // DW_AT_external
-; CHECK: // .b8 3                                // DW_AT_accessibility
-; CHECK:                                         // DW_ACCESS_private
-; CHECK: // .b8 29                               // Abbrev [29] 0x205c:0x6 DW_TAG_formal_parameter
-; CHECK: // .b32 8407                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_artificial
-; CHECK: // .b8 6                                // Abbrev [6] 0x2062:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 8422                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 32                               // Abbrev [32] 0x2068:0x3f DW_TAG_subprogram
-; CHECK: // .b8 95,90,78,75,50,53,95,95,99,117,100,97,95,98,117,105,108,116,105,110,95,98,108,111,99,107,73,100,120,95,116,97,100,69,118 // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 111,112,101,114,97,116,111,114,38 // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 13                               // DW_AT_decl_file
-; CHECK: // .b8 85                               // DW_AT_decl_line
-; CHECK: // .b32 8427                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 1                                // DW_AT_external
-; CHECK: // .b8 3                                // DW_AT_accessibility
-; CHECK:                                         // DW_ACCESS_private
-; CHECK: // .b8 29                               // Abbrev [29] 0x20a0:0x6 DW_TAG_formal_parameter
-; CHECK: // .b32 8407                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_artificial
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 27                               // Abbrev [27] 0x20a8:0x2f DW_TAG_structure_type
-; CHECK: // .b8 117,105,110,116,51               // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 12                               // DW_AT_byte_size
-; CHECK: // .b8 14                               // DW_AT_decl_file
-; CHECK: // .b8 190                              // DW_AT_decl_line
-; CHECK: // .b8 14                               // Abbrev [14] 0x20b2:0xc DW_TAG_member
-; CHECK: // .b8 120                              // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b32 5207                            // DW_AT_type
-; CHECK: // .b8 14                               // DW_AT_decl_file
-; CHECK: // .b8 192                              // DW_AT_decl_line
-; CHECK: // .b8 2                                // DW_AT_data_member_location
-; CHECK: // .b8 35
-; CHECK: // .b8 0
-; CHECK: // .b8 14                               // Abbrev [14] 0x20be:0xc DW_TAG_member
-; CHECK: // .b8 121                              // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b32 5207                            // DW_AT_type
-; CHECK: // .b8 14                               // DW_AT_decl_file
-; CHECK: // .b8 192                              // DW_AT_decl_line
-; CHECK: // .b8 2                                // DW_AT_data_member_location
-; CHECK: // .b8 35
-; CHECK: // .b8 4
-; CHECK: // .b8 14                               // Abbrev [14] 0x20ca:0xc DW_TAG_member
-; CHECK: // .b8 122                              // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b32 5207                            // DW_AT_type
-; CHECK: // .b8 14                               // DW_AT_decl_file
-; CHECK: // .b8 192                              // DW_AT_decl_line
-; CHECK: // .b8 2                                // DW_AT_data_member_location
-; CHECK: // .b8 35
-; CHECK: // .b8 8
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 8                                // Abbrev [8] 0x20d7:0x5 DW_TAG_pointer_type
-; CHECK: // .b32 8412                            // DW_AT_type
-; CHECK: // .b8 9                                // Abbrev [9] 0x20dc:0x5 DW_TAG_const_type
-; CHECK: // .b32 7806                            // DW_AT_type
-; CHECK: // .b8 8                                // Abbrev [8] 0x20e1:0x5 DW_TAG_pointer_type
-; CHECK: // .b32 7806                            // DW_AT_type
-; CHECK: // .b8 33                               // Abbrev [33] 0x20e6:0x5 DW_TAG_reference_type
-; CHECK: // .b32 8412                            // DW_AT_type
-; CHECK: // .b8 8                                // Abbrev [8] 0x20eb:0x5 DW_TAG_pointer_type
-; CHECK: // .b32 7806                            // DW_AT_type
-; CHECK: // .b8 34                               // Abbrev [34] 0x20f0:0x6 DW_TAG_subprogram
-; CHECK: // .b32 7836                            // DW_AT_specification
-; CHECK: // .b8 1                                // DW_AT_inline
-; CHECK: // .b8 27                               // Abbrev [27] 0x20f6:0x228 DW_TAG_structure_type
-; CHECK: // .b8 95,95,99,117,100,97,95,98,117,105,108,116,105,110,95,98,108,111,99,107,68,105,109,95,116 // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 1                                // DW_AT_byte_size
-; CHECK: // .b8 13                               // DW_AT_decl_file
-; CHECK: // .b8 88                               // DW_AT_decl_line
-; CHECK: // .b8 28                               // Abbrev [28] 0x2114:0x4f DW_TAG_subprogram
-; CHECK: // .b8 95,90,78,50,53,95,95,99,117,100,97,95,98,117,105,108,116,105,110,95,98,108,111,99,107,68,105,109,95,116,49,55,95,95,102,101,116,99,104,95 // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 98,117,105,108,116,105,110,95,120,69,118
-; CHECK: // .b8 0
-; CHECK: // .b8 95,95,102,101,116,99,104,95,98,117,105,108,116,105,110,95,120 // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 13                               // DW_AT_decl_file
-; CHECK: // .b8 89                               // DW_AT_decl_line
-; CHECK: // .b32 5207                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 1                                // DW_AT_external
-; CHECK: // .b8 28                               // Abbrev [28] 0x2163:0x4f DW_TAG_subprogram
-; CHECK: // .b8 95,90,78,50,53,95,95,99,117,100,97,95,98,117,105,108,116,105,110,95,98,108,111,99,107,68,105,109,95,116,49,55,95,95,102,101,116,99,104,95 // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 98,117,105,108,116,105,110,95,121,69,118
-; CHECK: // .b8 0
-; CHECK: // .b8 95,95,102,101,116,99,104,95,98,117,105,108,116,105,110,95,121 // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 13                               // DW_AT_decl_file
-; CHECK: // .b8 90                               // DW_AT_decl_line
-; CHECK: // .b32 5207                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 1                                // DW_AT_external
-; CHECK: // .b8 28                               // Abbrev [28] 0x21b2:0x4f DW_TAG_subprogram
-; CHECK: // .b8 95,90,78,50,53,95,95,99,117,100,97,95,98,117,105,108,116,105,110,95,98,108,111,99,107,68,105,109,95,116,49,55,95,95,102,101,116,99,104,95 // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 98,117,105,108,116,105,110,95,122,69,118
-; CHECK: // .b8 0
-; CHECK: // .b8 95,95,102,101,116,99,104,95,98,117,105,108,116,105,110,95,122 // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 13                               // DW_AT_decl_file
-; CHECK: // .b8 91                               // DW_AT_decl_line
-; CHECK: // .b32 5207                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 1                                // DW_AT_external
-; CHECK: // .b8 25                               // Abbrev [25] 0x2201:0x47 DW_TAG_subprogram
-; CHECK: // .b8 95,90,78,75,50,53,95,95,99,117,100,97,95,98,117,105,108,116,105,110,95,98,108,111,99,107,68,105,109,95,116,99,118,52,100,105,109,51,69,118 // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 111,112,101,114,97,116,111,114,32,100,105,109,51 // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 13                               // DW_AT_decl_file
-; CHECK: // .b8 94                               // DW_AT_decl_line
-; CHECK: // .b32 8990                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 1                                // DW_AT_external
-; CHECK: // .b8 29                               // Abbrev [29] 0x2241:0x6 DW_TAG_formal_parameter
-; CHECK: // .b32 9166                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_artificial
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 30                               // Abbrev [30] 0x2248:0x27 DW_TAG_subprogram
-; CHECK: // .b8 95,95,99,117,100,97,95,98,117,105,108,116,105,110,95,98,108,111,99,107,68,105,109,95,116 // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 13                               // DW_AT_decl_file
-; CHECK: // .b8 96                               // DW_AT_decl_line
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 1                                // DW_AT_external
-; CHECK: // .b8 3                                // DW_AT_accessibility
-; CHECK:                                         // DW_ACCESS_private
-; CHECK: // .b8 29                               // Abbrev [29] 0x2268:0x6 DW_TAG_formal_parameter
-; CHECK: // .b32 9176                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_artificial
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 30                               // Abbrev [30] 0x226f:0x2c DW_TAG_subprogram
-; CHECK: // .b8 95,95,99,117,100,97,95,98,117,105,108,116,105,110,95,98,108,111,99,107,68,105,109,95,116 // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 13                               // DW_AT_decl_file
-; CHECK: // .b8 96                               // DW_AT_decl_line
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 1                                // DW_AT_external
-; CHECK: // .b8 3                                // DW_AT_accessibility
-; CHECK:                                         // DW_ACCESS_private
-; CHECK: // .b8 29                               // Abbrev [29] 0x228f:0x6 DW_TAG_formal_parameter
-; CHECK: // .b32 9176                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_artificial
-; CHECK: // .b8 6                                // Abbrev [6] 0x2295:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 9181                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 31                               // Abbrev [31] 0x229b:0x43 DW_TAG_subprogram
-; CHECK: // .b8 95,90,78,75,50,53,95,95,99,117,100,97,95,98,117,105,108,116,105,110,95,98,108,111,99,107,68,105,109,95,116,97,83,69,82,75,83,95 // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 111,112,101,114,97,116,111,114,61 // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 13                               // DW_AT_decl_file
-; CHECK: // .b8 96                               // DW_AT_decl_line
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 1                                // DW_AT_external
-; CHECK: // .b8 3                                // DW_AT_accessibility
-; CHECK:                                         // DW_ACCESS_private
-; CHECK: // .b8 29                               // Abbrev [29] 0x22d2:0x6 DW_TAG_formal_parameter
-; CHECK: // .b32 9166                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_artificial
-; CHECK: // .b8 6                                // Abbrev [6] 0x22d8:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 9181                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 32                               // Abbrev [32] 0x22de:0x3f DW_TAG_subprogram
-; CHECK: // .b8 95,90,78,75,50,53,95,95,99,117,100,97,95,98,117,105,108,116,105,110,95,98,108,111,99,107,68,105,109,95,116,97,100,69,118 // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 111,112,101,114,97,116,111,114,38 // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 13                               // DW_AT_decl_file
-; CHECK: // .b8 96                               // DW_AT_decl_line
-; CHECK: // .b32 9186                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 1                                // DW_AT_external
-; CHECK: // .b8 3                                // DW_AT_accessibility
-; CHECK:                                         // DW_ACCESS_private
-; CHECK: // .b8 29                               // Abbrev [29] 0x2316:0x6 DW_TAG_formal_parameter
-; CHECK: // .b32 9166                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_artificial
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 35                               // Abbrev [35] 0x231e:0x9d DW_TAG_structure_type
-; CHECK: // .b8 100,105,109,51                   // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 12                               // DW_AT_byte_size
-; CHECK: // .b8 14                               // DW_AT_decl_file
-; CHECK: // .b8 161                              // DW_AT_decl_line
-; CHECK: // .b8 1
-; CHECK: // .b8 36                               // Abbrev [36] 0x2328:0xd DW_TAG_member
-; CHECK: // .b8 120                              // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b32 5207                            // DW_AT_type
-; CHECK: // .b8 14                               // DW_AT_decl_file
-; CHECK: // .b8 163                              // DW_AT_decl_line
-; CHECK: // .b8 1
-; CHECK: // .b8 2                                // DW_AT_data_member_location
-; CHECK: // .b8 35
-; CHECK: // .b8 0
-; CHECK: // .b8 36                               // Abbrev [36] 0x2335:0xd DW_TAG_member
-; CHECK: // .b8 121                              // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b32 5207                            // DW_AT_type
-; CHECK: // .b8 14                               // DW_AT_decl_file
-; CHECK: // .b8 163                              // DW_AT_decl_line
-; CHECK: // .b8 1
-; CHECK: // .b8 2                                // DW_AT_data_member_location
-; CHECK: // .b8 35
-; CHECK: // .b8 4
-; CHECK: // .b8 36                               // Abbrev [36] 0x2342:0xd DW_TAG_member
-; CHECK: // .b8 122                              // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b32 5207                            // DW_AT_type
-; CHECK: // .b8 14                               // DW_AT_decl_file
-; CHECK: // .b8 163                              // DW_AT_decl_line
-; CHECK: // .b8 1
-; CHECK: // .b8 2                                // DW_AT_data_member_location
-; CHECK: // .b8 35
-; CHECK: // .b8 8
-; CHECK: // .b8 23                               // Abbrev [23] 0x234f:0x21 DW_TAG_subprogram
-; CHECK: // .b8 100,105,109,51                   // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 14                               // DW_AT_decl_file
-; CHECK: // .b8 165                              // DW_AT_decl_line
-; CHECK: // .b8 1
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 1                                // DW_AT_external
-; CHECK: // .b8 29                               // Abbrev [29] 0x235a:0x6 DW_TAG_formal_parameter
-; CHECK: // .b32 9147                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_artificial
-; CHECK: // .b8 6                                // Abbrev [6] 0x2360:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 5207                            // DW_AT_type
-; CHECK: // .b8 6                                // Abbrev [6] 0x2365:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 5207                            // DW_AT_type
-; CHECK: // .b8 6                                // Abbrev [6] 0x236a:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 5207                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 23                               // Abbrev [23] 0x2370:0x17 DW_TAG_subprogram
-; CHECK: // .b8 100,105,109,51                   // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 14                               // DW_AT_decl_file
-; CHECK: // .b8 166                              // DW_AT_decl_line
-; CHECK: // .b8 1
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 1                                // DW_AT_external
-; CHECK: // .b8 29                               // Abbrev [29] 0x237b:0x6 DW_TAG_formal_parameter
-; CHECK: // .b32 9147                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_artificial
-; CHECK: // .b8 6                                // Abbrev [6] 0x2381:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 9152                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 37                               // Abbrev [37] 0x2387:0x33 DW_TAG_subprogram
-; CHECK: // .b8 95,90,78,52,100,105,109,51,99,118,53,117,105,110,116,51,69,118 // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 111,112,101,114,97,116,111,114,32,117,105,110,116,51 // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 14                               // DW_AT_decl_file
-; CHECK: // .b8 167                              // DW_AT_decl_line
-; CHECK: // .b8 1
-; CHECK: // .b32 9152                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 1                                // DW_AT_external
-; CHECK: // .b8 29                               // Abbrev [29] 0x23b3:0x6 DW_TAG_formal_parameter
-; CHECK: // .b32 9147                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_artificial
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 8                                // Abbrev [8] 0x23bb:0x5 DW_TAG_pointer_type
-; CHECK: // .b32 8990                            // DW_AT_type
-; CHECK: // .b8 20                               // Abbrev [20] 0x23c0:0xe DW_TAG_typedef
-; CHECK: // .b32 8360                            // DW_AT_type
-; CHECK: // .b8 117,105,110,116,51               // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 14                               // DW_AT_decl_file
-; CHECK: // .b8 127                              // DW_AT_decl_line
-; CHECK: // .b8 1
-; CHECK: // .b8 8                                // Abbrev [8] 0x23ce:0x5 DW_TAG_pointer_type
-; CHECK: // .b32 9171                            // DW_AT_type
-; CHECK: // .b8 9                                // Abbrev [9] 0x23d3:0x5 DW_TAG_const_type
-; CHECK: // .b32 8438                            // DW_AT_type
-; CHECK: // .b8 8                                // Abbrev [8] 0x23d8:0x5 DW_TAG_pointer_type
-; CHECK: // .b32 8438                            // DW_AT_type
-; CHECK: // .b8 33                               // Abbrev [33] 0x23dd:0x5 DW_TAG_reference_type
-; CHECK: // .b32 9171                            // DW_AT_type
-; CHECK: // .b8 8                                // Abbrev [8] 0x23e2:0x5 DW_TAG_pointer_type
-; CHECK: // .b32 8438                            // DW_AT_type
-; CHECK: // .b8 34                               // Abbrev [34] 0x23e7:0x6 DW_TAG_subprogram
-; CHECK: // .b32 8468                            // DW_AT_specification
-; CHECK: // .b8 1                                // DW_AT_inline
-; CHECK: // .b8 27                               // Abbrev [27] 0x23ed:0x233 DW_TAG_structure_type
-; CHECK: // .b8 95,95,99,117,100,97,95,98,117,105,108,116,105,110,95,116,104,114,101,97,100,73,100,120,95,116 // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 1                                // DW_AT_byte_size
-; CHECK: // .b8 13                               // DW_AT_decl_file
-; CHECK: // .b8 66                               // DW_AT_decl_line
-; CHECK: // .b8 28                               // Abbrev [28] 0x240c:0x50 DW_TAG_subprogram
-; CHECK: // .b8 95,90,78,50,54,95,95,99,117,100,97,95,98,117,105,108,116,105,110,95,116,104,114,101,97,100,73,100,120,95,116,49,55,95,95,102,101,116,99,104 // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 95,98,117,105,108,116,105,110,95,120,69,118
-; CHECK: // .b8 0
-; CHECK: // .b8 95,95,102,101,116,99,104,95,98,117,105,108,116,105,110,95,120 // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 13                               // DW_AT_decl_file
-; CHECK: // .b8 67                               // DW_AT_decl_line
-; CHECK: // .b32 5207                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 1                                // DW_AT_external
-; CHECK: // .b8 28                               // Abbrev [28] 0x245c:0x50 DW_TAG_subprogram
-; CHECK: // .b8 95,90,78,50,54,95,95,99,117,100,97,95,98,117,105,108,116,105,110,95,116,104,114,101,97,100,73,100,120,95,116,49,55,95,95,102,101,116,99,104 // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 95,98,117,105,108,116,105,110,95,121,69,118
-; CHECK: // .b8 0
-; CHECK: // .b8 95,95,102,101,116,99,104,95,98,117,105,108,116,105,110,95,121 // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 13                               // DW_AT_decl_file
-; CHECK: // .b8 68                               // DW_AT_decl_line
-; CHECK: // .b32 5207                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 1                                // DW_AT_external
-; CHECK: // .b8 28                               // Abbrev [28] 0x24ac:0x50 DW_TAG_subprogram
-; CHECK: // .b8 95,90,78,50,54,95,95,99,117,100,97,95,98,117,105,108,116,105,110,95,116,104,114,101,97,100,73,100,120,95,116,49,55,95,95,102,101,116,99,104 // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 95,98,117,105,108,116,105,110,95,122,69,118
-; CHECK: // .b8 0
-; CHECK: // .b8 95,95,102,101,116,99,104,95,98,117,105,108,116,105,110,95,122 // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 13                               // DW_AT_decl_file
-; CHECK: // .b8 69                               // DW_AT_decl_line
-; CHECK: // .b32 5207                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 1                                // DW_AT_external
-; CHECK: // .b8 25                               // Abbrev [25] 0x24fc:0x4a DW_TAG_subprogram
-; CHECK: // .b8 95,90,78,75,50,54,95,95,99,117,100,97,95,98,117,105,108,116,105,110,95,116,104,114,101,97,100,73,100,120,95,116,99,118,53,117,105,110,116,51 // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 69,118
-; CHECK: // .b8 0
-; CHECK: // .b8 111,112,101,114,97,116,111,114,32,117,105,110,116,51 // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 13                               // DW_AT_decl_file
-; CHECK: // .b8 72                               // DW_AT_decl_line
-; CHECK: // .b32 8360                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 1                                // DW_AT_external
-; CHECK: // .b8 29                               // Abbrev [29] 0x253f:0x6 DW_TAG_formal_parameter
-; CHECK: // .b32 9760                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_artificial
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 30                               // Abbrev [30] 0x2546:0x28 DW_TAG_subprogram
-; CHECK: // .b8 95,95,99,117,100,97,95,98,117,105,108,116,105,110,95,116,104,114,101,97,100,73,100,120,95,116 // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 13                               // DW_AT_decl_file
-; CHECK: // .b8 74                               // DW_AT_decl_line
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 1                                // DW_AT_external
-; CHECK: // .b8 3                                // DW_AT_accessibility
-; CHECK:                                         // DW_ACCESS_private
-; CHECK: // .b8 29                               // Abbrev [29] 0x2567:0x6 DW_TAG_formal_parameter
-; CHECK: // .b32 9770                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_artificial
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 30                               // Abbrev [30] 0x256e:0x2d DW_TAG_subprogram
-; CHECK: // .b8 95,95,99,117,100,97,95,98,117,105,108,116,105,110,95,116,104,114,101,97,100,73,100,120,95,116 // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 13                               // DW_AT_decl_file
-; CHECK: // .b8 74                               // DW_AT_decl_line
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 1                                // DW_AT_external
-; CHECK: // .b8 3                                // DW_AT_accessibility
-; CHECK:                                         // DW_ACCESS_private
-; CHECK: // .b8 29                               // Abbrev [29] 0x258f:0x6 DW_TAG_formal_parameter
-; CHECK: // .b32 9770                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_artificial
-; CHECK: // .b8 6                                // Abbrev [6] 0x2595:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 9775                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 31                               // Abbrev [31] 0x259b:0x44 DW_TAG_subprogram
-; CHECK: // .b8 95,90,78,75,50,54,95,95,99,117,100,97,95,98,117,105,108,116,105,110,95,116,104,114,101,97,100,73,100,120,95,116,97,83,69,82,75,83,95 // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 111,112,101,114,97,116,111,114,61 // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 13                               // DW_AT_decl_file
-; CHECK: // .b8 74                               // DW_AT_decl_line
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 1                                // DW_AT_external
-; CHECK: // .b8 3                                // DW_AT_accessibility
-; CHECK:                                         // DW_ACCESS_private
-; CHECK: // .b8 29                               // Abbrev [29] 0x25d3:0x6 DW_TAG_formal_parameter
-; CHECK: // .b32 9760                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_artificial
-; CHECK: // .b8 6                                // Abbrev [6] 0x25d9:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 9775                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 32                               // Abbrev [32] 0x25df:0x40 DW_TAG_subprogram
-; CHECK: // .b8 95,90,78,75,50,54,95,95,99,117,100,97,95,98,117,105,108,116,105,110,95,116,104,114,101,97,100,73,100,120,95,116,97,100,69,118 // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 111,112,101,114,97,116,111,114,38 // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 13                               // DW_AT_decl_file
-; CHECK: // .b8 74                               // DW_AT_decl_line
-; CHECK: // .b32 9780                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 1                                // DW_AT_external
-; CHECK: // .b8 3                                // DW_AT_accessibility
-; CHECK:                                         // DW_ACCESS_private
-; CHECK: // .b8 29                               // Abbrev [29] 0x2618:0x6 DW_TAG_formal_parameter
-; CHECK: // .b32 9760                            // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_artificial
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 8                                // Abbrev [8] 0x2620:0x5 DW_TAG_pointer_type
-; CHECK: // .b32 9765                            // DW_AT_type
-; CHECK: // .b8 9                                // Abbrev [9] 0x2625:0x5 DW_TAG_const_type
-; CHECK: // .b32 9197                            // DW_AT_type
-; CHECK: // .b8 8                                // Abbrev [8] 0x262a:0x5 DW_TAG_pointer_type
-; CHECK: // .b32 9197                            // DW_AT_type
-; CHECK: // .b8 33                               // Abbrev [33] 0x262f:0x5 DW_TAG_reference_type
-; CHECK: // .b32 9765                            // DW_AT_type
-; CHECK: // .b8 8                                // Abbrev [8] 0x2634:0x5 DW_TAG_pointer_type
-; CHECK: // .b32 9197                            // DW_AT_type
-; CHECK: // .b8 34                               // Abbrev [34] 0x2639:0x6 DW_TAG_subprogram
-; CHECK: // .b32 9228                            // DW_AT_specification
-; CHECK: // .b8 1                                // DW_AT_inline
-; CHECK: // .b8 38                               // Abbrev [38] 0x263f:0x32 DW_TAG_subprogram
-; CHECK: // .b8 95,90,51,114,101,115,102,102,80,102 // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 114,101,115                      // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 12                               // DW_AT_decl_file
-; CHECK: // .b8 3                                // DW_AT_decl_line
-; CHECK: // .b8 1                                // DW_AT_external
-; CHECK: // .b8 1                                // DW_AT_inline
-; CHECK: // .b8 39                               // Abbrev [39] 0x2653:0x9 DW_TAG_formal_parameter
-; CHECK: // .b8 120                              // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 12                               // DW_AT_decl_file
-; CHECK: // .b8 3                                // DW_AT_decl_line
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 39                               // Abbrev [39] 0x265c:0x9 DW_TAG_formal_parameter
-; CHECK: // .b8 121                              // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 12                               // DW_AT_decl_file
-; CHECK: // .b8 3                                // DW_AT_decl_line
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 39                               // Abbrev [39] 0x2665:0xb DW_TAG_formal_parameter
-; CHECK: // .b8 114,101,115                      // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 12                               // DW_AT_decl_file
-; CHECK: // .b8 3                                // DW_AT_decl_line
-; CHECK: // .b32 3345                            // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 40                               // Abbrev [40] 0x2671:0xc0 DW_TAG_subprogram
-; CHECK: // .b64 Lfunc_begin0                    // DW_AT_low_pc
-; CHECK: // .b64 Lfunc_end0                      // DW_AT_high_pc
-; CHECK: // .b8 1                                // DW_AT_frame_base
-; CHECK: // .b8 156
-; CHECK: // .b8 95,90,53,115,97,120,112,121,105,102,80,102,83,95 // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 115,97,120,112,121               // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 12                               // DW_AT_decl_file
-; CHECK: // .b8 5                                // DW_AT_decl_line
-; CHECK: // .b8 1                                // DW_AT_external
-; CHECK: // .b8 39                               // Abbrev [39] 0x269c:0x9 DW_TAG_formal_parameter
-; CHECK: // .b8 110                              // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 12                               // DW_AT_decl_file
-; CHECK: // .b8 5                                // DW_AT_decl_line
-; CHECK: // .b32 2332                            // DW_AT_type
-; CHECK: // .b8 39                               // Abbrev [39] 0x26a5:0x9 DW_TAG_formal_parameter
-; CHECK: // .b8 97                               // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 12                               // DW_AT_decl_file
-; CHECK: // .b8 5                                // DW_AT_decl_line
-; CHECK: // .b32 1554                            // DW_AT_type
-; CHECK: // .b8 39                               // Abbrev [39] 0x26ae:0x9 DW_TAG_formal_parameter
-; CHECK: // .b8 120                              // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 12                               // DW_AT_decl_file
-; CHECK: // .b8 5                                // DW_AT_decl_line
-; CHECK: // .b32 3345                            // DW_AT_type
-; CHECK: // .b8 39                               // Abbrev [39] 0x26b7:0x9 DW_TAG_formal_parameter
-; CHECK: // .b8 121                              // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 12                               // DW_AT_decl_file
-; CHECK: // .b8 5                                // DW_AT_decl_line
-; CHECK: // .b32 3345                            // DW_AT_type
-; CHECK: // .b8 41                               // Abbrev [41] 0x26c0:0x9 DW_TAG_variable
-; CHECK: // .b8 105                              // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 12                               // DW_AT_decl_file
-; CHECK: // .b8 6                                // DW_AT_decl_line
-; CHECK: // .b32 2332                            // DW_AT_type
-; CHECK: // .b8 42                               // Abbrev [42] 0x26c9:0x17 DW_TAG_inlined_subroutine
-; CHECK: // .b32 8432                            // DW_AT_abstract_origin
-; CHECK: // .b64 Ltmp0                           // DW_AT_low_pc
-; CHECK: // .b64 Ltmp1                           // DW_AT_high_pc
-; CHECK: // .b8 12                               // DW_AT_call_file
-; CHECK: // .b8 6                                // DW_AT_call_line
-; CHECK: // .b8 42                               // Abbrev [42] 0x26e0:0x17 DW_TAG_inlined_subroutine
-; CHECK: // .b32 9191                            // DW_AT_abstract_origin
-; CHECK: // .b64 Ltmp1                           // DW_AT_low_pc
-; CHECK: // .b64 Ltmp2                           // DW_AT_high_pc
-; CHECK: // .b8 12                               // DW_AT_call_file
-; CHECK: // .b8 6                                // DW_AT_call_line
-; CHECK: // .b8 42                               // Abbrev [42] 0x26f7:0x17 DW_TAG_inlined_subroutine
-; CHECK: // .b32 9785                            // DW_AT_abstract_origin
-; CHECK: // .b64 Ltmp2                           // DW_AT_low_pc
-; CHECK: // .b64 Ltmp3                           // DW_AT_high_pc
-; CHECK: // .b8 12                               // DW_AT_call_file
-; CHECK: // .b8 6                                // DW_AT_call_line
-; CHECK: // .b8 43                               // Abbrev [43] 0x270e:0x22 DW_TAG_inlined_subroutine
-; CHECK: // .b32 9791                            // DW_AT_abstract_origin
-; CHECK: // .b64 Ltmp8                           // DW_AT_low_pc
-; CHECK: // .b64 Ltmp9                           // DW_AT_high_pc
-; CHECK: // .b8 12                               // DW_AT_call_file
-; CHECK: // .b8 8                                // DW_AT_call_line
-; CHECK: // .b8 44                               // Abbrev [44] 0x2725:0x5 DW_TAG_formal_parameter
-; CHECK: // .b32 9820                            // DW_AT_abstract_origin
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // }
-; CHECK: // .section .debug_macinfo
-; CHECK: // {
-; CHECK: // .b8 0                                // End Of Macro List Mark
-; CHECK: // }
+; CHECK: .section .debug_abbrev
+; CHECK-NEXT: {
+; CHECK-NEXT: .b8 1                                // Abbreviation Code
+; CHECK-NEXT: .b8 17                               // DW_TAG_compile_unit
+; CHECK-NEXT: .b8 1                                // DW_CHILDREN_yes
+; CHECK-NEXT: .b8 37                               // DW_AT_producer
+; CHECK-NEXT: .b8 8                                // DW_FORM_string
+; CHECK-NEXT: .b8 19                               // DW_AT_language
+; CHECK-NEXT: .b8 5                                // DW_FORM_data2
+; CHECK-NEXT: .b8 3                                // DW_AT_name
+; CHECK-NEXT: .b8 8                                // DW_FORM_string
+; CHECK-NEXT: .b8 16                               // DW_AT_stmt_list
+; CHECK-NEXT: .b8 6                                // DW_FORM_data4
+; CHECK-NEXT: .b8 27                               // DW_AT_comp_dir
+; CHECK-NEXT: .b8 8                                // DW_FORM_string
+; CHECK-NEXT: .b8 17                               // DW_AT_low_pc
+; CHECK-NEXT: .b8 1                                // DW_FORM_addr
+; CHECK-NEXT: .b8 18                               // DW_AT_high_pc
+; CHECK-NEXT: .b8 1                                // DW_FORM_addr
+; CHECK-NEXT: .b8 0                                // EOM(1)
+; CHECK-NEXT: .b8 0                                // EOM(2)
+; CHECK-NEXT: .b8 2                                // Abbreviation Code
+; CHECK-NEXT: .b8 57                               // DW_TAG_namespace
+; CHECK-NEXT: .b8 1                                // DW_CHILDREN_yes
+; CHECK-NEXT: .b8 3                                // DW_AT_name
+; CHECK-NEXT: .b8 8                                // DW_FORM_string
+; CHECK-NEXT: .b8 0                                // EOM(1)
+; CHECK-NEXT: .b8 0                                // EOM(2)
+; CHECK-NEXT: .b8 3                                // Abbreviation Code
+; CHECK-NEXT: .b8 8                                // DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 0                                // DW_CHILDREN_no
+; CHECK-NEXT: .b8 58                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 11                               // DW_FORM_data1
+; CHECK-NEXT: .b8 59                               // DW_AT_decl_line
+; CHECK-NEXT: .b8 11                               // DW_FORM_data1
+; CHECK-NEXT: .b8 24                               // DW_AT_import
+; CHECK-NEXT: .b8 19                               // DW_FORM_ref4
+; CHECK-NEXT: .b8 0                                // EOM(1)
+; CHECK-NEXT: .b8 0                                // EOM(2)
+; CHECK-NEXT: .b8 4                                // Abbreviation Code
+; CHECK-NEXT: .b8 8                                // DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 0                                // DW_CHILDREN_no
+; CHECK-NEXT: .b8 58                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 11                               // DW_FORM_data1
+; CHECK-NEXT: .b8 59                               // DW_AT_decl_line
+; CHECK-NEXT: .b8 5                                // DW_FORM_data2
+; CHECK-NEXT: .b8 24                               // DW_AT_import
+; CHECK-NEXT: .b8 19                               // DW_FORM_ref4
+; CHECK-NEXT: .b8 0                                // EOM(1)
+; CHECK-NEXT: .b8 0                                // EOM(2)
+; CHECK-NEXT: .b8 5                                // Abbreviation Code
+; CHECK-NEXT: .b8 46                               // DW_TAG_subprogram
+; CHECK-NEXT: .b8 1                                // DW_CHILDREN_yes
+; CHECK-NEXT: .b8 135,64                           // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 8                                // DW_FORM_string
+; CHECK-NEXT: .b8 3                                // DW_AT_name
+; CHECK-NEXT: .b8 8                                // DW_FORM_string
+; CHECK-NEXT: .b8 58                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 11                               // DW_FORM_data1
+; CHECK-NEXT: .b8 59                               // DW_AT_decl_line
+; CHECK-NEXT: .b8 11                               // DW_FORM_data1
+; CHECK-NEXT: .b8 73                               // DW_AT_type
+; CHECK-NEXT: .b8 19                               // DW_FORM_ref4
+; CHECK-NEXT: .b8 60                               // DW_AT_declaration
+; CHECK-NEXT: .b8 12                               // DW_FORM_flag
+; CHECK-NEXT: .b8 0                                // EOM(1)
+; CHECK-NEXT: .b8 0                                // EOM(2)
+; CHECK-NEXT: .b8 6                                // Abbreviation Code
+; CHECK-NEXT: .b8 5                                // DW_TAG_formal_parameter
+; CHECK-NEXT: .b8 0                                // DW_CHILDREN_no
+; CHECK-NEXT: .b8 73                               // DW_AT_type
+; CHECK-NEXT: .b8 19                               // DW_FORM_ref4
+; CHECK-NEXT: .b8 0                                // EOM(1)
+; CHECK-NEXT: .b8 0                                // EOM(2)
+; CHECK-NEXT: .b8 7                                // Abbreviation Code
+; CHECK-NEXT: .b8 36                               // DW_TAG_base_type
+; CHECK-NEXT: .b8 0                                // DW_CHILDREN_no
+; CHECK-NEXT: .b8 3                                // DW_AT_name
+; CHECK-NEXT: .b8 8                                // DW_FORM_string
+; CHECK-NEXT: .b8 62                               // DW_AT_encoding
+; CHECK-NEXT: .b8 11                               // DW_FORM_data1
+; CHECK-NEXT: .b8 11                               // DW_AT_byte_size
+; CHECK-NEXT: .b8 11                               // DW_FORM_data1
+; CHECK-NEXT: .b8 0                                // EOM(1)
+; CHECK-NEXT: .b8 0                                // EOM(2)
+; CHECK-NEXT: .b8 8                                // Abbreviation Code
+; CHECK-NEXT: .b8 15                               // DW_TAG_pointer_type
+; CHECK-NEXT: .b8 0                                // DW_CHILDREN_no
+; CHECK-NEXT: .b8 73                               // DW_AT_type
+; CHECK-NEXT: .b8 19                               // DW_FORM_ref4
+; CHECK-NEXT: .b8 0                                // EOM(1)
+; CHECK-NEXT: .b8 0                                // EOM(2)
+; CHECK-NEXT: .b8 9                                // Abbreviation Code
+; CHECK-NEXT: .b8 38                               // DW_TAG_const_type
+; CHECK-NEXT: .b8 0                                // DW_CHILDREN_no
+; CHECK-NEXT: .b8 73                               // DW_AT_type
+; CHECK-NEXT: .b8 19                               // DW_FORM_ref4
+; CHECK-NEXT: .b8 0                                // EOM(1)
+; CHECK-NEXT: .b8 0                                // EOM(2)
+; CHECK-NEXT: .b8 10                               // Abbreviation Code
+; CHECK-NEXT: .b8 46                               // DW_TAG_subprogram
+; CHECK-NEXT: .b8 1                                // DW_CHILDREN_yes
+; CHECK-NEXT: .b8 3                                // DW_AT_name
+; CHECK-NEXT: .b8 8                                // DW_FORM_string
+; CHECK-NEXT: .b8 58                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 11                               // DW_FORM_data1
+; CHECK-NEXT: .b8 59                               // DW_AT_decl_line
+; CHECK-NEXT: .b8 11                               // DW_FORM_data1
+; CHECK-NEXT: .b8 73                               // DW_AT_type
+; CHECK-NEXT: .b8 19                               // DW_FORM_ref4
+; CHECK-NEXT: .b8 60                               // DW_AT_declaration
+; CHECK-NEXT: .b8 12                               // DW_FORM_flag
+; CHECK-NEXT: .b8 63                               // DW_AT_external
+; CHECK-NEXT: .b8 12                               // DW_FORM_flag
+; CHECK-NEXT: .b8 0                                // EOM(1)
+; CHECK-NEXT: .b8 0                                // EOM(2)
+; CHECK-NEXT: .b8 11                               // Abbreviation Code
+; CHECK-NEXT: .b8 22                               // DW_TAG_typedef
+; CHECK-NEXT: .b8 0                                // DW_CHILDREN_no
+; CHECK-NEXT: .b8 73                               // DW_AT_type
+; CHECK-NEXT: .b8 19                               // DW_FORM_ref4
+; CHECK-NEXT: .b8 3                                // DW_AT_name
+; CHECK-NEXT: .b8 8                                // DW_FORM_string
+; CHECK-NEXT: .b8 58                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 11                               // DW_FORM_data1
+; CHECK-NEXT: .b8 59                               // DW_AT_decl_line
+; CHECK-NEXT: .b8 11                               // DW_FORM_data1
+; CHECK-NEXT: .b8 0                                // EOM(1)
+; CHECK-NEXT: .b8 0                                // EOM(2)
+; CHECK-NEXT: .b8 12                               // Abbreviation Code
+; CHECK-NEXT: .b8 19                               // DW_TAG_structure_type
+; CHECK-NEXT: .b8 0                                // DW_CHILDREN_no
+; CHECK-NEXT: .b8 60                               // DW_AT_declaration
+; CHECK-NEXT: .b8 12                               // DW_FORM_flag
+; CHECK-NEXT: .b8 0                                // EOM(1)
+; CHECK-NEXT: .b8 0                                // EOM(2)
+; CHECK-NEXT: .b8 13                               // Abbreviation Code
+; CHECK-NEXT: .b8 19                               // DW_TAG_structure_type
+; CHECK-NEXT: .b8 1                                // DW_CHILDREN_yes
+; CHECK-NEXT: .b8 11                               // DW_AT_byte_size
+; CHECK-NEXT: .b8 11                               // DW_FORM_data1
+; CHECK-NEXT: .b8 58                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 11                               // DW_FORM_data1
+; CHECK-NEXT: .b8 59                               // DW_AT_decl_line
+; CHECK-NEXT: .b8 11                               // DW_FORM_data1
+; CHECK-NEXT: .b8 0                                // EOM(1)
+; CHECK-NEXT: .b8 0                                // EOM(2)
+; CHECK-NEXT: .b8 14                               // Abbreviation Code
+; CHECK-NEXT: .b8 13                               // DW_TAG_member
+; CHECK-NEXT: .b8 0                                // DW_CHILDREN_no
+; CHECK-NEXT: .b8 3                                // DW_AT_name
+; CHECK-NEXT: .b8 8                                // DW_FORM_string
+; CHECK-NEXT: .b8 73                               // DW_AT_type
+; CHECK-NEXT: .b8 19                               // DW_FORM_ref4
+; CHECK-NEXT: .b8 58                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 11                               // DW_FORM_data1
+; CHECK-NEXT: .b8 59                               // DW_AT_decl_line
+; CHECK-NEXT: .b8 11                               // DW_FORM_data1
+; CHECK-NEXT: .b8 56                               // DW_AT_data_member_location
+; CHECK-NEXT: .b8 10                               // DW_FORM_block1
+; CHECK-NEXT: .b8 0                                // EOM(1)
+; CHECK-NEXT: .b8 0                                // EOM(2)
+; CHECK-NEXT: .b8 15                               // Abbreviation Code
+; CHECK-NEXT: .b8 46                               // DW_TAG_subprogram
+; CHECK-NEXT: .b8 0                                // DW_CHILDREN_no
+; CHECK-NEXT: .b8 3                                // DW_AT_name
+; CHECK-NEXT: .b8 8                                // DW_FORM_string
+; CHECK-NEXT: .b8 58                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 11                               // DW_FORM_data1
+; CHECK-NEXT: .b8 59                               // DW_AT_decl_line
+; CHECK-NEXT: .b8 5                                // DW_FORM_data2
+; CHECK-NEXT: .b8 60                               // DW_AT_declaration
+; CHECK-NEXT: .b8 12                               // DW_FORM_flag
+; CHECK-NEXT: .b8 63                               // DW_AT_external
+; CHECK-NEXT: .b8 12                               // DW_FORM_flag
+; CHECK-NEXT: .b8 135,1                            // DW_AT_noreturn
+; CHECK-NEXT: .b8 12                               // DW_FORM_flag
+; CHECK-NEXT: .b8 0                                // EOM(1)
+; CHECK-NEXT: .b8 0                                // EOM(2)
+; CHECK-NEXT: .b8 16                               // Abbreviation Code
+; CHECK-NEXT: .b8 46                               // DW_TAG_subprogram
+; CHECK-NEXT: .b8 1                                // DW_CHILDREN_yes
+; CHECK-NEXT: .b8 3                                // DW_AT_name
+; CHECK-NEXT: .b8 8                                // DW_FORM_string
+; CHECK-NEXT: .b8 58                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 11                               // DW_FORM_data1
+; CHECK-NEXT: .b8 59                               // DW_AT_decl_line
+; CHECK-NEXT: .b8 5                                // DW_FORM_data2
+; CHECK-NEXT: .b8 73                               // DW_AT_type
+; CHECK-NEXT: .b8 19                               // DW_FORM_ref4
+; CHECK-NEXT: .b8 60                               // DW_AT_declaration
+; CHECK-NEXT: .b8 12                               // DW_FORM_flag
+; CHECK-NEXT: .b8 63                               // DW_AT_external
+; CHECK-NEXT: .b8 12                               // DW_FORM_flag
+; CHECK-NEXT: .b8 0                                // EOM(1)
+; CHECK-NEXT: .b8 0                                // EOM(2)
+; CHECK-NEXT: .b8 17                               // Abbreviation Code
+; CHECK-NEXT: .b8 21                               // DW_TAG_subroutine_type
+; CHECK-NEXT: .b8 0                                // DW_CHILDREN_no
+; CHECK-NEXT: .b8 0                                // EOM(1)
+; CHECK-NEXT: .b8 0                                // EOM(2)
+; CHECK-NEXT: .b8 18                               // Abbreviation Code
+; CHECK-NEXT: .b8 15                               // DW_TAG_pointer_type
+; CHECK-NEXT: .b8 0                                // DW_CHILDREN_no
+; CHECK-NEXT: .b8 0                                // EOM(1)
+; CHECK-NEXT: .b8 0                                // EOM(2)
+; CHECK-NEXT: .b8 19                               // Abbreviation Code
+; CHECK-NEXT: .b8 38                               // DW_TAG_const_type
+; CHECK-NEXT: .b8 0                                // DW_CHILDREN_no
+; CHECK-NEXT: .b8 0                                // EOM(1)
+; CHECK-NEXT: .b8 0                                // EOM(2)
+; CHECK-NEXT: .b8 20                               // Abbreviation Code
+; CHECK-NEXT: .b8 22                               // DW_TAG_typedef
+; CHECK-NEXT: .b8 0                                // DW_CHILDREN_no
+; CHECK-NEXT: .b8 73                               // DW_AT_type
+; CHECK-NEXT: .b8 19                               // DW_FORM_ref4
+; CHECK-NEXT: .b8 3                                // DW_AT_name
+; CHECK-NEXT: .b8 8                                // DW_FORM_string
+; CHECK-NEXT: .b8 58                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 11                               // DW_FORM_data1
+; CHECK-NEXT: .b8 59                               // DW_AT_decl_line
+; CHECK-NEXT: .b8 5                                // DW_FORM_data2
+; CHECK-NEXT: .b8 0                                // EOM(1)
+; CHECK-NEXT: .b8 0                                // EOM(2)
+; CHECK-NEXT: .b8 21                               // Abbreviation Code
+; CHECK-NEXT: .b8 21                               // DW_TAG_subroutine_type
+; CHECK-NEXT: .b8 1                                // DW_CHILDREN_yes
+; CHECK-NEXT: .b8 73                               // DW_AT_type
+; CHECK-NEXT: .b8 19                               // DW_FORM_ref4
+; CHECK-NEXT: .b8 0                                // EOM(1)
+; CHECK-NEXT: .b8 0                                // EOM(2)
+; CHECK-NEXT: .b8 22                               // Abbreviation Code
+; CHECK-NEXT: .b8 46                               // DW_TAG_subprogram
+; CHECK-NEXT: .b8 1                                // DW_CHILDREN_yes
+; CHECK-NEXT: .b8 3                                // DW_AT_name
+; CHECK-NEXT: .b8 8                                // DW_FORM_string
+; CHECK-NEXT: .b8 58                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 11                               // DW_FORM_data1
+; CHECK-NEXT: .b8 59                               // DW_AT_decl_line
+; CHECK-NEXT: .b8 5                                // DW_FORM_data2
+; CHECK-NEXT: .b8 60                               // DW_AT_declaration
+; CHECK-NEXT: .b8 12                               // DW_FORM_flag
+; CHECK-NEXT: .b8 63                               // DW_AT_external
+; CHECK-NEXT: .b8 12                               // DW_FORM_flag
+; CHECK-NEXT: .b8 135,1                            // DW_AT_noreturn
+; CHECK-NEXT: .b8 12                               // DW_FORM_flag
+; CHECK-NEXT: .b8 0                                // EOM(1)
+; CHECK-NEXT: .b8 0                                // EOM(2)
+; CHECK-NEXT: .b8 23                               // Abbreviation Code
+; CHECK-NEXT: .b8 46                               // DW_TAG_subprogram
+; CHECK-NEXT: .b8 1                                // DW_CHILDREN_yes
+; CHECK-NEXT: .b8 3                                // DW_AT_name
+; CHECK-NEXT: .b8 8                                // DW_FORM_string
+; CHECK-NEXT: .b8 58                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 11                               // DW_FORM_data1
+; CHECK-NEXT: .b8 59                               // DW_AT_decl_line
+; CHECK-NEXT: .b8 5                                // DW_FORM_data2
+; CHECK-NEXT: .b8 60                               // DW_AT_declaration
+; CHECK-NEXT: .b8 12                               // DW_FORM_flag
+; CHECK-NEXT: .b8 63                               // DW_AT_external
+; CHECK-NEXT: .b8 12                               // DW_FORM_flag
+; CHECK-NEXT: .b8 0                                // EOM(1)
+; CHECK-NEXT: .b8 0                                // EOM(2)
+; CHECK-NEXT: .b8 24                               // Abbreviation Code
+; CHECK-NEXT: .b8 46                               // DW_TAG_subprogram
+; CHECK-NEXT: .b8 0                                // DW_CHILDREN_no
+; CHECK-NEXT: .b8 3                                // DW_AT_name
+; CHECK-NEXT: .b8 8                                // DW_FORM_string
+; CHECK-NEXT: .b8 58                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 11                               // DW_FORM_data1
+; CHECK-NEXT: .b8 59                               // DW_AT_decl_line
+; CHECK-NEXT: .b8 5                                // DW_FORM_data2
+; CHECK-NEXT: .b8 73                               // DW_AT_type
+; CHECK-NEXT: .b8 19                               // DW_FORM_ref4
+; CHECK-NEXT: .b8 60                               // DW_AT_declaration
+; CHECK-NEXT: .b8 12                               // DW_FORM_flag
+; CHECK-NEXT: .b8 63                               // DW_AT_external
+; CHECK-NEXT: .b8 12                               // DW_FORM_flag
+; CHECK-NEXT: .b8 0                                // EOM(1)
+; CHECK-NEXT: .b8 0                                // EOM(2)
+; CHECK-NEXT: .b8 25                               // Abbreviation Code
+; CHECK-NEXT: .b8 46                               // DW_TAG_subprogram
+; CHECK-NEXT: .b8 1                                // DW_CHILDREN_yes
+; CHECK-NEXT: .b8 135,64                           // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 8                                // DW_FORM_string
+; CHECK-NEXT: .b8 3                                // DW_AT_name
+; CHECK-NEXT: .b8 8                                // DW_FORM_string
+; CHECK-NEXT: .b8 58                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 11                               // DW_FORM_data1
+; CHECK-NEXT: .b8 59                               // DW_AT_decl_line
+; CHECK-NEXT: .b8 11                               // DW_FORM_data1
+; CHECK-NEXT: .b8 73                               // DW_AT_type
+; CHECK-NEXT: .b8 19                               // DW_FORM_ref4
+; CHECK-NEXT: .b8 60                               // DW_AT_declaration
+; CHECK-NEXT: .b8 12                               // DW_FORM_flag
+; CHECK-NEXT: .b8 63                               // DW_AT_external
+; CHECK-NEXT: .b8 12                               // DW_FORM_flag
+; CHECK-NEXT: .b8 0                                // EOM(1)
+; CHECK-NEXT: .b8 0                                // EOM(2)
+; CHECK-NEXT: .b8 26                               // Abbreviation Code
+; CHECK-NEXT: .b8 46                               // DW_TAG_subprogram
+; CHECK-NEXT: .b8 1                                // DW_CHILDREN_yes
+; CHECK-NEXT: .b8 135,64                           // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 8                                // DW_FORM_string
+; CHECK-NEXT: .b8 3                                // DW_AT_name
+; CHECK-NEXT: .b8 8                                // DW_FORM_string
+; CHECK-NEXT: .b8 58                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 11                               // DW_FORM_data1
+; CHECK-NEXT: .b8 59                               // DW_AT_decl_line
+; CHECK-NEXT: .b8 5                                // DW_FORM_data2
+; CHECK-NEXT: .b8 73                               // DW_AT_type
+; CHECK-NEXT: .b8 19                               // DW_FORM_ref4
+; CHECK-NEXT: .b8 60                               // DW_AT_declaration
+; CHECK-NEXT: .b8 12                               // DW_FORM_flag
+; CHECK-NEXT: .b8 0                                // EOM(1)
+; CHECK-NEXT: .b8 0                                // EOM(2)
+; CHECK-NEXT: .b8 27                               // Abbreviation Code
+; CHECK-NEXT: .b8 19                               // DW_TAG_structure_type
+; CHECK-NEXT: .b8 1                                // DW_CHILDREN_yes
+; CHECK-NEXT: .b8 3                                // DW_AT_name
+; CHECK-NEXT: .b8 8                                // DW_FORM_string
+; CHECK-NEXT: .b8 11                               // DW_AT_byte_size
+; CHECK-NEXT: .b8 11                               // DW_FORM_data1
+; CHECK-NEXT: .b8 58                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 11                               // DW_FORM_data1
+; CHECK-NEXT: .b8 59                               // DW_AT_decl_line
+; CHECK-NEXT: .b8 11                               // DW_FORM_data1
+; CHECK-NEXT: .b8 0                                // EOM(1)
+; CHECK-NEXT: .b8 0                                // EOM(2)
+; CHECK-NEXT: .b8 28                               // Abbreviation Code
+; CHECK-NEXT: .b8 46                               // DW_TAG_subprogram
+; CHECK-NEXT: .b8 0                                // DW_CHILDREN_no
+; CHECK-NEXT: .b8 135,64                           // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 8                                // DW_FORM_string
+; CHECK-NEXT: .b8 3                                // DW_AT_name
+; CHECK-NEXT: .b8 8                                // DW_FORM_string
+; CHECK-NEXT: .b8 58                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 11                               // DW_FORM_data1
+; CHECK-NEXT: .b8 59                               // DW_AT_decl_line
+; CHECK-NEXT: .b8 11                               // DW_FORM_data1
+; CHECK-NEXT: .b8 73                               // DW_AT_type
+; CHECK-NEXT: .b8 19                               // DW_FORM_ref4
+; CHECK-NEXT: .b8 60                               // DW_AT_declaration
+; CHECK-NEXT: .b8 12                               // DW_FORM_flag
+; CHECK-NEXT: .b8 63                               // DW_AT_external
+; CHECK-NEXT: .b8 12                               // DW_FORM_flag
+; CHECK-NEXT: .b8 0                                // EOM(1)
+; CHECK-NEXT: .b8 0                                // EOM(2)
+; CHECK-NEXT: .b8 29                               // Abbreviation Code
+; CHECK-NEXT: .b8 5                                // DW_TAG_formal_parameter
+; CHECK-NEXT: .b8 0                                // DW_CHILDREN_no
+; CHECK-NEXT: .b8 73                               // DW_AT_type
+; CHECK-NEXT: .b8 19                               // DW_FORM_ref4
+; CHECK-NEXT: .b8 52                               // DW_AT_artificial
+; CHECK-NEXT: .b8 12                               // DW_FORM_flag
+; CHECK-NEXT: .b8 0                                // EOM(1)
+; CHECK-NEXT: .b8 0                                // EOM(2)
+; CHECK-NEXT: .b8 30                               // Abbreviation Code
+; CHECK-NEXT: .b8 46                               // DW_TAG_subprogram
+; CHECK-NEXT: .b8 1                                // DW_CHILDREN_yes
+; CHECK-NEXT: .b8 3                                // DW_AT_name
+; CHECK-NEXT: .b8 8                                // DW_FORM_string
+; CHECK-NEXT: .b8 58                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 11                               // DW_FORM_data1
+; CHECK-NEXT: .b8 59                               // DW_AT_decl_line
+; CHECK-NEXT: .b8 11                               // DW_FORM_data1
+; CHECK-NEXT: .b8 60                               // DW_AT_declaration
+; CHECK-NEXT: .b8 12                               // DW_FORM_flag
+; CHECK-NEXT: .b8 63                               // DW_AT_external
+; CHECK-NEXT: .b8 12                               // DW_FORM_flag
+; CHECK-NEXT: .b8 50                               // DW_AT_accessibility
+; CHECK-NEXT: .b8 11                               // DW_FORM_data1
+; CHECK-NEXT: .b8 0                                // EOM(1)
+; CHECK-NEXT: .b8 0                                // EOM(2)
+; CHECK-NEXT: .b8 31                               // Abbreviation Code
+; CHECK-NEXT: .b8 46                               // DW_TAG_subprogram
+; CHECK-NEXT: .b8 1                                // DW_CHILDREN_yes
+; CHECK-NEXT: .b8 135,64                           // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 8                                // DW_FORM_string
+; CHECK-NEXT: .b8 3                                // DW_AT_name
+; CHECK-NEXT: .b8 8                                // DW_FORM_string
+; CHECK-NEXT: .b8 58                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 11                               // DW_FORM_data1
+; CHECK-NEXT: .b8 59                               // DW_AT_decl_line
+; CHECK-NEXT: .b8 11                               // DW_FORM_data1
+; CHECK-NEXT: .b8 60                               // DW_AT_declaration
+; CHECK-NEXT: .b8 12                               // DW_FORM_flag
+; CHECK-NEXT: .b8 63                               // DW_AT_external
+; CHECK-NEXT: .b8 12                               // DW_FORM_flag
+; CHECK-NEXT: .b8 50                               // DW_AT_accessibility
+; CHECK-NEXT: .b8 11                               // DW_FORM_data1
+; CHECK-NEXT: .b8 0                                // EOM(1)
+; CHECK-NEXT: .b8 0                                // EOM(2)
+; CHECK-NEXT: .b8 32                               // Abbreviation Code
+; CHECK-NEXT: .b8 46                               // DW_TAG_subprogram
+; CHECK-NEXT: .b8 1                                // DW_CHILDREN_yes
+; CHECK-NEXT: .b8 135,64                           // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 8                                // DW_FORM_string
+; CHECK-NEXT: .b8 3                                // DW_AT_name
+; CHECK-NEXT: .b8 8                                // DW_FORM_string
+; CHECK-NEXT: .b8 58                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 11                               // DW_FORM_data1
+; CHECK-NEXT: .b8 59                               // DW_AT_decl_line
+; CHECK-NEXT: .b8 11                               // DW_FORM_data1
+; CHECK-NEXT: .b8 73                               // DW_AT_type
+; CHECK-NEXT: .b8 19                               // DW_FORM_ref4
+; CHECK-NEXT: .b8 60                               // DW_AT_declaration
+; CHECK-NEXT: .b8 12                               // DW_FORM_flag
+; CHECK-NEXT: .b8 63                               // DW_AT_external
+; CHECK-NEXT: .b8 12                               // DW_FORM_flag
+; CHECK-NEXT: .b8 50                               // DW_AT_accessibility
+; CHECK-NEXT: .b8 11                               // DW_FORM_data1
+; CHECK-NEXT: .b8 0                                // EOM(1)
+; CHECK-NEXT: .b8 0                                // EOM(2)
+; CHECK-NEXT: .b8 33                               // Abbreviation Code
+; CHECK-NEXT: .b8 16                               // DW_TAG_reference_type
+; CHECK-NEXT: .b8 0                                // DW_CHILDREN_no
+; CHECK-NEXT: .b8 73                               // DW_AT_type
+; CHECK-NEXT: .b8 19                               // DW_FORM_ref4
+; CHECK-NEXT: .b8 0                                // EOM(1)
+; CHECK-NEXT: .b8 0                                // EOM(2)
+; CHECK-NEXT: .b8 34                               // Abbreviation Code
+; CHECK-NEXT: .b8 46                               // DW_TAG_subprogram
+; CHECK-NEXT: .b8 0                                // DW_CHILDREN_no
+; CHECK-NEXT: .b8 71                               // DW_AT_specification
+; CHECK-NEXT: .b8 19                               // DW_FORM_ref4
+; CHECK-NEXT: .b8 32                               // DW_AT_inline
+; CHECK-NEXT: .b8 11                               // DW_FORM_data1
+; CHECK-NEXT: .b8 0                                // EOM(1)
+; CHECK-NEXT: .b8 0                                // EOM(2)
+; CHECK-NEXT: .b8 35                               // Abbreviation Code
+; CHECK-NEXT: .b8 19                               // DW_TAG_structure_type
+; CHECK-NEXT: .b8 1                                // DW_CHILDREN_yes
+; CHECK-NEXT: .b8 3                                // DW_AT_name
+; CHECK-NEXT: .b8 8                                // DW_FORM_string
+; CHECK-NEXT: .b8 11                               // DW_AT_byte_size
+; CHECK-NEXT: .b8 11                               // DW_FORM_data1
+; CHECK-NEXT: .b8 58                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 11                               // DW_FORM_data1
+; CHECK-NEXT: .b8 59                               // DW_AT_decl_line
+; CHECK-NEXT: .b8 5                                // DW_FORM_data2
+; CHECK-NEXT: .b8 0                                // EOM(1)
+; CHECK-NEXT: .b8 0                                // EOM(2)
+; CHECK-NEXT: .b8 36                               // Abbreviation Code
+; CHECK-NEXT: .b8 13                               // DW_TAG_member
+; CHECK-NEXT: .b8 0                                // DW_CHILDREN_no
+; CHECK-NEXT: .b8 3                                // DW_AT_name
+; CHECK-NEXT: .b8 8                                // DW_FORM_string
+; CHECK-NEXT: .b8 73                               // DW_AT_type
+; CHECK-NEXT: .b8 19                               // DW_FORM_ref4
+; CHECK-NEXT: .b8 58                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 11                               // DW_FORM_data1
+; CHECK-NEXT: .b8 59                               // DW_AT_decl_line
+; CHECK-NEXT: .b8 5                                // DW_FORM_data2
+; CHECK-NEXT: .b8 56                               // DW_AT_data_member_location
+; CHECK-NEXT: .b8 10                               // DW_FORM_block1
+; CHECK-NEXT: .b8 0                                // EOM(1)
+; CHECK-NEXT: .b8 0                                // EOM(2)
+; CHECK-NEXT: .b8 37                               // Abbreviation Code
+; CHECK-NEXT: .b8 46                               // DW_TAG_subprogram
+; CHECK-NEXT: .b8 1                                // DW_CHILDREN_yes
+; CHECK-NEXT: .b8 135,64                           // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 8                                // DW_FORM_string
+; CHECK-NEXT: .b8 3                                // DW_AT_name
+; CHECK-NEXT: .b8 8                                // DW_FORM_string
+; CHECK-NEXT: .b8 58                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 11                               // DW_FORM_data1
+; CHECK-NEXT: .b8 59                               // DW_AT_decl_line
+; CHECK-NEXT: .b8 5                                // DW_FORM_data2
+; CHECK-NEXT: .b8 73                               // DW_AT_type
+; CHECK-NEXT: .b8 19                               // DW_FORM_ref4
+; CHECK-NEXT: .b8 60                               // DW_AT_declaration
+; CHECK-NEXT: .b8 12                               // DW_FORM_flag
+; CHECK-NEXT: .b8 63                               // DW_AT_external
+; CHECK-NEXT: .b8 12                               // DW_FORM_flag
+; CHECK-NEXT: .b8 0                                // EOM(1)
+; CHECK-NEXT: .b8 0                                // EOM(2)
+; CHECK-NEXT: .b8 38                               // Abbreviation Code
+; CHECK-NEXT: .b8 46                               // DW_TAG_subprogram
+; CHECK-NEXT: .b8 1                                // DW_CHILDREN_yes
+; CHECK-NEXT: .b8 135,64                           // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 8                                // DW_FORM_string
+; CHECK-NEXT: .b8 3                                // DW_AT_name
+; CHECK-NEXT: .b8 8                                // DW_FORM_string
+; CHECK-NEXT: .b8 58                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 11                               // DW_FORM_data1
+; CHECK-NEXT: .b8 59                               // DW_AT_decl_line
+; CHECK-NEXT: .b8 11                               // DW_FORM_data1
+; CHECK-NEXT: .b8 63                               // DW_AT_external
+; CHECK-NEXT: .b8 12                               // DW_FORM_flag
+; CHECK-NEXT: .b8 32                               // DW_AT_inline
+; CHECK-NEXT: .b8 11                               // DW_FORM_data1
+; CHECK-NEXT: .b8 0                                // EOM(1)
+; CHECK-NEXT: .b8 0                                // EOM(2)
+; CHECK-NEXT: .b8 39                               // Abbreviation Code
+; CHECK-NEXT: .b8 5                                // DW_TAG_formal_parameter
+; CHECK-NEXT: .b8 0                                // DW_CHILDREN_no
+; CHECK-NEXT: .b8 3                                // DW_AT_name
+; CHECK-NEXT: .b8 8                                // DW_FORM_string
+; CHECK-NEXT: .b8 58                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 11                               // DW_FORM_data1
+; CHECK-NEXT: .b8 59                               // DW_AT_decl_line
+; CHECK-NEXT: .b8 11                               // DW_FORM_data1
+; CHECK-NEXT: .b8 73                               // DW_AT_type
+; CHECK-NEXT: .b8 19                               // DW_FORM_ref4
+; CHECK-NEXT: .b8 0                                // EOM(1)
+; CHECK-NEXT: .b8 0                                // EOM(2)
+; CHECK-NEXT: .b8 40                               // Abbreviation Code
+; CHECK-NEXT: .b8 46                               // DW_TAG_subprogram
+; CHECK-NEXT: .b8 1                                // DW_CHILDREN_yes
+; CHECK-NEXT: .b8 17                               // DW_AT_low_pc
+; CHECK-NEXT: .b8 1                                // DW_FORM_addr
+; CHECK-NEXT: .b8 18                               // DW_AT_high_pc
+; CHECK-NEXT: .b8 1                                // DW_FORM_addr
+; CHECK-NEXT: .b8 64                               // DW_AT_frame_base
+; CHECK-NEXT: .b8 10                               // DW_FORM_block1
+; CHECK-NEXT: .b8 135,64                           // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 8                                // DW_FORM_string
+; CHECK-NEXT: .b8 3                                // DW_AT_name
+; CHECK-NEXT: .b8 8                                // DW_FORM_string
+; CHECK-NEXT: .b8 58                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 11                               // DW_FORM_data1
+; CHECK-NEXT: .b8 59                               // DW_AT_decl_line
+; CHECK-NEXT: .b8 11                               // DW_FORM_data1
+; CHECK-NEXT: .b8 63                               // DW_AT_external
+; CHECK-NEXT: .b8 12                               // DW_FORM_flag
+; CHECK-NEXT: .b8 0                                // EOM(1)
+; CHECK-NEXT: .b8 0                                // EOM(2)
+; CHECK-NEXT: .b8 41                               // Abbreviation Code
+; CHECK-NEXT: .b8 52                               // DW_TAG_variable
+; CHECK-NEXT: .b8 0                                // DW_CHILDREN_no
+; CHECK-NEXT: .b8 3                                // DW_AT_name
+; CHECK-NEXT: .b8 8                                // DW_FORM_string
+; CHECK-NEXT: .b8 58                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 11                               // DW_FORM_data1
+; CHECK-NEXT: .b8 59                               // DW_AT_decl_line
+; CHECK-NEXT: .b8 11                               // DW_FORM_data1
+; CHECK-NEXT: .b8 73                               // DW_AT_type
+; CHECK-NEXT: .b8 19                               // DW_FORM_ref4
+; CHECK-NEXT: .b8 0                                // EOM(1)
+; CHECK-NEXT: .b8 0                                // EOM(2)
+; CHECK-NEXT: .b8 42                               // Abbreviation Code
+; CHECK-NEXT: .b8 29                               // DW_TAG_inlined_subroutine
+; CHECK-NEXT: .b8 0                                // DW_CHILDREN_no
+; CHECK-NEXT: .b8 49                               // DW_AT_abstract_origin
+; CHECK-NEXT: .b8 19                               // DW_FORM_ref4
+; CHECK-NEXT: .b8 17                               // DW_AT_low_pc
+; CHECK-NEXT: .b8 1                                // DW_FORM_addr
+; CHECK-NEXT: .b8 18                               // DW_AT_high_pc
+; CHECK-NEXT: .b8 1                                // DW_FORM_addr
+; CHECK-NEXT: .b8 88                               // DW_AT_call_file
+; CHECK-NEXT: .b8 11                               // DW_FORM_data1
+; CHECK-NEXT: .b8 89                               // DW_AT_call_line
+; CHECK-NEXT: .b8 11                               // DW_FORM_data1
+; CHECK-NEXT: .b8 0                                // EOM(1)
+; CHECK-NEXT: .b8 0                                // EOM(2)
+; CHECK-NEXT: .b8 43                               // Abbreviation Code
+; CHECK-NEXT: .b8 29                               // DW_TAG_inlined_subroutine
+; CHECK-NEXT: .b8 1                                // DW_CHILDREN_yes
+; CHECK-NEXT: .b8 49                               // DW_AT_abstract_origin
+; CHECK-NEXT: .b8 19                               // DW_FORM_ref4
+; CHECK-NEXT: .b8 17                               // DW_AT_low_pc
+; CHECK-NEXT: .b8 1                                // DW_FORM_addr
+; CHECK-NEXT: .b8 18                               // DW_AT_high_pc
+; CHECK-NEXT: .b8 1                                // DW_FORM_addr
+; CHECK-NEXT: .b8 88                               // DW_AT_call_file
+; CHECK-NEXT: .b8 11                               // DW_FORM_data1
+; CHECK-NEXT: .b8 89                               // DW_AT_call_line
+; CHECK-NEXT: .b8 11                               // DW_FORM_data1
+; CHECK-NEXT: .b8 0                                // EOM(1)
+; CHECK-NEXT: .b8 0                                // EOM(2)
+; CHECK-NEXT: .b8 44                               // Abbreviation Code
+; CHECK-NEXT: .b8 5                                // DW_TAG_formal_parameter
+; CHECK-NEXT: .b8 0                                // DW_CHILDREN_no
+; CHECK-NEXT: .b8 49                               // DW_AT_abstract_origin
+; CHECK-NEXT: .b8 19                               // DW_FORM_ref4
+; CHECK-NEXT: .b8 0                                // EOM(1)
+; CHECK-NEXT: .b8 0                                // EOM(2)
+; CHECK-NEXT: .b8 0                                // EOM(3)
+; CHECK-NEXT: }
+; CHECK-NEXT: .section .debug_info
+; CHECK-NEXT: {
+; CHECK-NEXT: .b32 10030                           // Length of Unit
+; CHECK-NEXT: .b8 2                                // DWARF version number
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b32 .debug_abbrev                   // Offset Into Abbrev. Section
+; CHECK-NEXT: .b8 8                                // Address Size (in bytes)
+; CHECK-NEXT: .b8 1                                // Abbrev [1] 0xb:0x2727 DW_TAG_compile_unit
+; CHECK-NEXT: .b8 0                                // DW_AT_producer
+; CHECK-NEXT: .b8 4                                // DW_AT_language
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 100,101,98,117,103,45,105,110,102,111,46,99,117 // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b32 .debug_line                     // DW_AT_stmt_list
+; CHECK-NEXT: .b8 47,115,111,109,101,47,100,105,114,101,99,116,111,114,121 // DW_AT_comp_dir
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b64 Lfunc_begin0                    // DW_AT_low_pc
+; CHECK-NEXT: .b64 Lfunc_end0                      // DW_AT_high_pc
+; CHECK-NEXT: .b8 2                                // Abbrev [2] 0x41:0x588 DW_TAG_namespace
+; CHECK-NEXT: .b8 115,116,100                      // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 3                                // Abbrev [3] 0x46:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 202                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 1481                            // DW_AT_import
+; CHECK-NEXT: .b8 3                                // Abbrev [3] 0x4d:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 203                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 1525                            // DW_AT_import
+; CHECK-NEXT: .b8 3                                // Abbrev [3] 0x54:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 204                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 1563                            // DW_AT_import
+; CHECK-NEXT: .b8 3                                // Abbrev [3] 0x5b:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 205                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 1594                            // DW_AT_import
+; CHECK-NEXT: .b8 3                                // Abbrev [3] 0x62:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 206                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 1623                            // DW_AT_import
+; CHECK-NEXT: .b8 3                                // Abbrev [3] 0x69:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 207                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 1654                            // DW_AT_import
+; CHECK-NEXT: .b8 3                                // Abbrev [3] 0x70:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 208                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 1683                            // DW_AT_import
+; CHECK-NEXT: .b8 3                                // Abbrev [3] 0x77:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 209                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 1720                            // DW_AT_import
+; CHECK-NEXT: .b8 3                                // Abbrev [3] 0x7e:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 210                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 1751                            // DW_AT_import
+; CHECK-NEXT: .b8 3                                // Abbrev [3] 0x85:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 211                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 1780                            // DW_AT_import
+; CHECK-NEXT: .b8 3                                // Abbrev [3] 0x8c:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 212                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 1809                            // DW_AT_import
+; CHECK-NEXT: .b8 3                                // Abbrev [3] 0x93:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 213                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 1852                            // DW_AT_import
+; CHECK-NEXT: .b8 3                                // Abbrev [3] 0x9a:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 214                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 1879                            // DW_AT_import
+; CHECK-NEXT: .b8 3                                // Abbrev [3] 0xa1:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 215                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 1908                            // DW_AT_import
+; CHECK-NEXT: .b8 3                                // Abbrev [3] 0xa8:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 216                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 1935                            // DW_AT_import
+; CHECK-NEXT: .b8 3                                // Abbrev [3] 0xaf:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 217                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 1964                            // DW_AT_import
+; CHECK-NEXT: .b8 3                                // Abbrev [3] 0xb6:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 218                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 1991                            // DW_AT_import
+; CHECK-NEXT: .b8 3                                // Abbrev [3] 0xbd:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 219                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 2020                            // DW_AT_import
+; CHECK-NEXT: .b8 3                                // Abbrev [3] 0xc4:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 220                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 2051                            // DW_AT_import
+; CHECK-NEXT: .b8 3                                // Abbrev [3] 0xcb:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 221                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 2080                            // DW_AT_import
+; CHECK-NEXT: .b8 3                                // Abbrev [3] 0xd2:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 222                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 2115                            // DW_AT_import
+; CHECK-NEXT: .b8 3                                // Abbrev [3] 0xd9:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 223                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 2146                            // DW_AT_import
+; CHECK-NEXT: .b8 3                                // Abbrev [3] 0xe0:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 224                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 2185                            // DW_AT_import
+; CHECK-NEXT: .b8 3                                // Abbrev [3] 0xe7:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 225                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 2220                            // DW_AT_import
+; CHECK-NEXT: .b8 3                                // Abbrev [3] 0xee:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 226                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 2255                            // DW_AT_import
+; CHECK-NEXT: .b8 3                                // Abbrev [3] 0xf5:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 227                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 2290                            // DW_AT_import
+; CHECK-NEXT: .b8 3                                // Abbrev [3] 0xfc:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 228                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 2339                            // DW_AT_import
+; CHECK-NEXT: .b8 3                                // Abbrev [3] 0x103:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 229                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 2382                            // DW_AT_import
+; CHECK-NEXT: .b8 3                                // Abbrev [3] 0x10a:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 230                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 2419                            // DW_AT_import
+; CHECK-NEXT: .b8 3                                // Abbrev [3] 0x111:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 231                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 2450                            // DW_AT_import
+; CHECK-NEXT: .b8 3                                // Abbrev [3] 0x118:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 232                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 2495                            // DW_AT_import
+; CHECK-NEXT: .b8 3                                // Abbrev [3] 0x11f:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 233                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 2540                            // DW_AT_import
+; CHECK-NEXT: .b8 3                                // Abbrev [3] 0x126:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 234                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 2596                            // DW_AT_import
+; CHECK-NEXT: .b8 3                                // Abbrev [3] 0x12d:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 235                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 2627                            // DW_AT_import
+; CHECK-NEXT: .b8 3                                // Abbrev [3] 0x134:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 236                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 2666                            // DW_AT_import
+; CHECK-NEXT: .b8 3                                // Abbrev [3] 0x13b:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 237                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 2716                            // DW_AT_import
+; CHECK-NEXT: .b8 3                                // Abbrev [3] 0x142:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 238                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 2770                            // DW_AT_import
+; CHECK-NEXT: .b8 3                                // Abbrev [3] 0x149:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 239                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 2801                            // DW_AT_import
+; CHECK-NEXT: .b8 3                                // Abbrev [3] 0x150:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 240                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 2838                            // DW_AT_import
+; CHECK-NEXT: .b8 3                                // Abbrev [3] 0x157:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 241                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 2888                            // DW_AT_import
+; CHECK-NEXT: .b8 3                                // Abbrev [3] 0x15e:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 242                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 2929                            // DW_AT_import
+; CHECK-NEXT: .b8 3                                // Abbrev [3] 0x165:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 243                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 2966                            // DW_AT_import
+; CHECK-NEXT: .b8 3                                // Abbrev [3] 0x16c:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 244                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 2999                            // DW_AT_import
+; CHECK-NEXT: .b8 3                                // Abbrev [3] 0x173:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 245                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 3030                            // DW_AT_import
+; CHECK-NEXT: .b8 3                                // Abbrev [3] 0x17a:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 246                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 3063                            // DW_AT_import
+; CHECK-NEXT: .b8 3                                // Abbrev [3] 0x181:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 247                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 3090                            // DW_AT_import
+; CHECK-NEXT: .b8 3                                // Abbrev [3] 0x188:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 248                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 3121                            // DW_AT_import
+; CHECK-NEXT: .b8 3                                // Abbrev [3] 0x18f:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 249                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 3152                            // DW_AT_import
+; CHECK-NEXT: .b8 3                                // Abbrev [3] 0x196:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 250                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 3181                            // DW_AT_import
+; CHECK-NEXT: .b8 3                                // Abbrev [3] 0x19d:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 251                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 3210                            // DW_AT_import
+; CHECK-NEXT: .b8 3                                // Abbrev [3] 0x1a4:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 252                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 3241                            // DW_AT_import
+; CHECK-NEXT: .b8 3                                // Abbrev [3] 0x1ab:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 253                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 3274                            // DW_AT_import
+; CHECK-NEXT: .b8 3                                // Abbrev [3] 0x1b2:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 254                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 3309                            // DW_AT_import
+; CHECK-NEXT: .b8 3                                // Abbrev [3] 0x1b9:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 255                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 3350                            // DW_AT_import
+; CHECK-NEXT: .b8 4                                // Abbrev [4] 0x1c0:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 0                                // DW_AT_decl_line
+; CHECK-NEXT: .b8 1
+; CHECK-NEXT: .b32 3407                            // DW_AT_import
+; CHECK-NEXT: .b8 4                                // Abbrev [4] 0x1c8:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_line
+; CHECK-NEXT: .b8 1
+; CHECK-NEXT: .b32 3438                            // DW_AT_import
+; CHECK-NEXT: .b8 4                                // Abbrev [4] 0x1d0:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 2                                // DW_AT_decl_line
+; CHECK-NEXT: .b8 1
+; CHECK-NEXT: .b32 3477                            // DW_AT_import
+; CHECK-NEXT: .b8 4                                // Abbrev [4] 0x1d8:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 3                                // DW_AT_decl_line
+; CHECK-NEXT: .b8 1
+; CHECK-NEXT: .b32 3522                            // DW_AT_import
+; CHECK-NEXT: .b8 4                                // Abbrev [4] 0x1e0:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 4                                // DW_AT_decl_line
+; CHECK-NEXT: .b8 1
+; CHECK-NEXT: .b32 3555                            // DW_AT_import
+; CHECK-NEXT: .b8 4                                // Abbrev [4] 0x1e8:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 5                                // DW_AT_decl_line
+; CHECK-NEXT: .b8 1
+; CHECK-NEXT: .b32 3600                            // DW_AT_import
+; CHECK-NEXT: .b8 4                                // Abbrev [4] 0x1f0:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 6                                // DW_AT_decl_line
+; CHECK-NEXT: .b8 1
+; CHECK-NEXT: .b32 3646                            // DW_AT_import
+; CHECK-NEXT: .b8 4                                // Abbrev [4] 0x1f8:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 7                                // DW_AT_decl_line
+; CHECK-NEXT: .b8 1
+; CHECK-NEXT: .b32 3675                            // DW_AT_import
+; CHECK-NEXT: .b8 4                                // Abbrev [4] 0x200:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 8                                // DW_AT_decl_line
+; CHECK-NEXT: .b8 1
+; CHECK-NEXT: .b32 3706                            // DW_AT_import
+; CHECK-NEXT: .b8 4                                // Abbrev [4] 0x208:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 9                                // DW_AT_decl_line
+; CHECK-NEXT: .b8 1
+; CHECK-NEXT: .b32 3747                            // DW_AT_import
+; CHECK-NEXT: .b8 4                                // Abbrev [4] 0x210:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 10                               // DW_AT_decl_line
+; CHECK-NEXT: .b8 1
+; CHECK-NEXT: .b32 3786                            // DW_AT_import
+; CHECK-NEXT: .b8 4                                // Abbrev [4] 0x218:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 11                               // DW_AT_decl_line
+; CHECK-NEXT: .b8 1
+; CHECK-NEXT: .b32 3821                            // DW_AT_import
+; CHECK-NEXT: .b8 4                                // Abbrev [4] 0x220:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 12                               // DW_AT_decl_line
+; CHECK-NEXT: .b8 1
+; CHECK-NEXT: .b32 3848                            // DW_AT_import
+; CHECK-NEXT: .b8 4                                // Abbrev [4] 0x228:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 13                               // DW_AT_decl_line
+; CHECK-NEXT: .b8 1
+; CHECK-NEXT: .b32 3877                            // DW_AT_import
+; CHECK-NEXT: .b8 4                                // Abbrev [4] 0x230:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 14                               // DW_AT_decl_line
+; CHECK-NEXT: .b8 1
+; CHECK-NEXT: .b32 3906                            // DW_AT_import
+; CHECK-NEXT: .b8 4                                // Abbrev [4] 0x238:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 15                               // DW_AT_decl_line
+; CHECK-NEXT: .b8 1
+; CHECK-NEXT: .b32 3933                            // DW_AT_import
+; CHECK-NEXT: .b8 4                                // Abbrev [4] 0x240:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 16                               // DW_AT_decl_line
+; CHECK-NEXT: .b8 1
+; CHECK-NEXT: .b32 3962                            // DW_AT_import
+; CHECK-NEXT: .b8 4                                // Abbrev [4] 0x248:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 17                               // DW_AT_decl_line
+; CHECK-NEXT: .b8 1
+; CHECK-NEXT: .b32 3995                            // DW_AT_import
+; CHECK-NEXT: .b8 3                                // Abbrev [3] 0x250:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 3                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 102                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 4026                            // DW_AT_import
+; CHECK-NEXT: .b8 3                                // Abbrev [3] 0x257:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 3                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 121                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 4046                            // DW_AT_import
+; CHECK-NEXT: .b8 3                                // Abbrev [3] 0x25e:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 3                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 140                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 4066                            // DW_AT_import
+; CHECK-NEXT: .b8 3                                // Abbrev [3] 0x265:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 3                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 159                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 4086                            // DW_AT_import
+; CHECK-NEXT: .b8 3                                // Abbrev [3] 0x26c:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 3                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 180                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 4112                            // DW_AT_import
+; CHECK-NEXT: .b8 3                                // Abbrev [3] 0x273:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 3                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 199                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 4132                            // DW_AT_import
+; CHECK-NEXT: .b8 3                                // Abbrev [3] 0x27a:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 3                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 218                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 4151                            // DW_AT_import
+; CHECK-NEXT: .b8 3                                // Abbrev [3] 0x281:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 3                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 237                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 4171                            // DW_AT_import
+; CHECK-NEXT: .b8 4                                // Abbrev [4] 0x288:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 3                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 0                                // DW_AT_decl_line
+; CHECK-NEXT: .b8 1
+; CHECK-NEXT: .b32 4190                            // DW_AT_import
+; CHECK-NEXT: .b8 4                                // Abbrev [4] 0x290:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 3                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 19                               // DW_AT_decl_line
+; CHECK-NEXT: .b8 1
+; CHECK-NEXT: .b32 4210                            // DW_AT_import
+; CHECK-NEXT: .b8 4                                // Abbrev [4] 0x298:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 3                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 38                               // DW_AT_decl_line
+; CHECK-NEXT: .b8 1
+; CHECK-NEXT: .b32 4231                            // DW_AT_import
+; CHECK-NEXT: .b8 4                                // Abbrev [4] 0x2a0:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 3                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 59                               // DW_AT_decl_line
+; CHECK-NEXT: .b8 1
+; CHECK-NEXT: .b32 4256                            // DW_AT_import
+; CHECK-NEXT: .b8 4                                // Abbrev [4] 0x2a8:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 3                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 78                               // DW_AT_decl_line
+; CHECK-NEXT: .b8 1
+; CHECK-NEXT: .b32 4282                            // DW_AT_import
+; CHECK-NEXT: .b8 4                                // Abbrev [4] 0x2b0:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 3                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 97                               // DW_AT_decl_line
+; CHECK-NEXT: .b8 1
+; CHECK-NEXT: .b32 4308                            // DW_AT_import
+; CHECK-NEXT: .b8 4                                // Abbrev [4] 0x2b8:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 3                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 116                              // DW_AT_decl_line
+; CHECK-NEXT: .b8 1
+; CHECK-NEXT: .b32 4327                            // DW_AT_import
+; CHECK-NEXT: .b8 4                                // Abbrev [4] 0x2c0:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 3                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 135                              // DW_AT_decl_line
+; CHECK-NEXT: .b8 1
+; CHECK-NEXT: .b32 4348                            // DW_AT_import
+; CHECK-NEXT: .b8 4                                // Abbrev [4] 0x2c8:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 3                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 147                              // DW_AT_decl_line
+; CHECK-NEXT: .b8 1
+; CHECK-NEXT: .b32 4378                            // DW_AT_import
+; CHECK-NEXT: .b8 4                                // Abbrev [4] 0x2d0:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 3                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 184                              // DW_AT_decl_line
+; CHECK-NEXT: .b8 1
+; CHECK-NEXT: .b32 4402                            // DW_AT_import
+; CHECK-NEXT: .b8 4                                // Abbrev [4] 0x2d8:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 3                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 203                              // DW_AT_decl_line
+; CHECK-NEXT: .b8 1
+; CHECK-NEXT: .b32 4421                            // DW_AT_import
+; CHECK-NEXT: .b8 4                                // Abbrev [4] 0x2e0:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 3                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 222                              // DW_AT_decl_line
+; CHECK-NEXT: .b8 1
+; CHECK-NEXT: .b32 4441                            // DW_AT_import
+; CHECK-NEXT: .b8 4                                // Abbrev [4] 0x2e8:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 3                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 241                              // DW_AT_decl_line
+; CHECK-NEXT: .b8 1
+; CHECK-NEXT: .b32 4461                            // DW_AT_import
+; CHECK-NEXT: .b8 4                                // Abbrev [4] 0x2f0:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 3                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 4                                // DW_AT_decl_line
+; CHECK-NEXT: .b8 2
+; CHECK-NEXT: .b32 4480                            // DW_AT_import
+; CHECK-NEXT: .b8 3                                // Abbrev [3] 0x2f8:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 5                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 118                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 4500                            // DW_AT_import
+; CHECK-NEXT: .b8 3                                // Abbrev [3] 0x2ff:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 5                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 119                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 4515                            // DW_AT_import
+; CHECK-NEXT: .b8 3                                // Abbrev [3] 0x306:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 5                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 121                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 4563                            // DW_AT_import
+; CHECK-NEXT: .b8 3                                // Abbrev [3] 0x30d:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 5                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 122                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 4576                            // DW_AT_import
+; CHECK-NEXT: .b8 3                                // Abbrev [3] 0x314:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 5                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 123                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 4596                            // DW_AT_import
+; CHECK-NEXT: .b8 3                                // Abbrev [3] 0x31b:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 5                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 129                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 4625                            // DW_AT_import
+; CHECK-NEXT: .b8 3                                // Abbrev [3] 0x322:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 5                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 130                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 4645                            // DW_AT_import
+; CHECK-NEXT: .b8 3                                // Abbrev [3] 0x329:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 5                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 131                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 4666                            // DW_AT_import
+; CHECK-NEXT: .b8 3                                // Abbrev [3] 0x330:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 5                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 132                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 4687                            // DW_AT_import
+; CHECK-NEXT: .b8 3                                // Abbrev [3] 0x337:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 5                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 133                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 4815                            // DW_AT_import
+; CHECK-NEXT: .b8 3                                // Abbrev [3] 0x33e:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 5                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 134                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 4843                            // DW_AT_import
+; CHECK-NEXT: .b8 3                                // Abbrev [3] 0x345:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 5                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 135                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 4868                            // DW_AT_import
+; CHECK-NEXT: .b8 3                                // Abbrev [3] 0x34c:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 5                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 136                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 4886                            // DW_AT_import
+; CHECK-NEXT: .b8 3                                // Abbrev [3] 0x353:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 5                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 137                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 4903                            // DW_AT_import
+; CHECK-NEXT: .b8 3                                // Abbrev [3] 0x35a:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 5                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 138                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 4931                            // DW_AT_import
+; CHECK-NEXT: .b8 3                                // Abbrev [3] 0x361:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 5                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 139                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 4952                            // DW_AT_import
+; CHECK-NEXT: .b8 3                                // Abbrev [3] 0x368:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 5                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 140                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 4978                            // DW_AT_import
+; CHECK-NEXT: .b8 3                                // Abbrev [3] 0x36f:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 5                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 142                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 5001                            // DW_AT_import
+; CHECK-NEXT: .b8 3                                // Abbrev [3] 0x376:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 5                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 143                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 5028                            // DW_AT_import
+; CHECK-NEXT: .b8 3                                // Abbrev [3] 0x37d:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 5                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 144                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 5079                            // DW_AT_import
+; CHECK-NEXT: .b8 3                                // Abbrev [3] 0x384:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 5                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 146                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 5112                            // DW_AT_import
+; CHECK-NEXT: .b8 3                                // Abbrev [3] 0x38b:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 5                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 152                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 5145                            // DW_AT_import
+; CHECK-NEXT: .b8 3                                // Abbrev [3] 0x392:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 5                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 153                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 5160                            // DW_AT_import
+; CHECK-NEXT: .b8 3                                // Abbrev [3] 0x399:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 5                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 154                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 5189                            // DW_AT_import
+; CHECK-NEXT: .b8 3                                // Abbrev [3] 0x3a0:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 5                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 155                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 5223                            // DW_AT_import
+; CHECK-NEXT: .b8 3                                // Abbrev [3] 0x3a7:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 5                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 156                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 5255                            // DW_AT_import
+; CHECK-NEXT: .b8 3                                // Abbrev [3] 0x3ae:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 5                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 157                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 5287                            // DW_AT_import
+; CHECK-NEXT: .b8 3                                // Abbrev [3] 0x3b5:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 5                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 158                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 5320                            // DW_AT_import
+; CHECK-NEXT: .b8 3                                // Abbrev [3] 0x3bc:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 5                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 160                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 5343                            // DW_AT_import
+; CHECK-NEXT: .b8 3                                // Abbrev [3] 0x3c3:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 5                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 161                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 5388                            // DW_AT_import
+; CHECK-NEXT: .b8 3                                // Abbrev [3] 0x3ca:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 5                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 241                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 5536                            // DW_AT_import
+; CHECK-NEXT: .b8 3                                // Abbrev [3] 0x3d1:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 5                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 243                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 5585                            // DW_AT_import
+; CHECK-NEXT: .b8 3                                // Abbrev [3] 0x3d8:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 5                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 245                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 5604                            // DW_AT_import
+; CHECK-NEXT: .b8 3                                // Abbrev [3] 0x3df:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 5                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 246                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 5490                            // DW_AT_import
+; CHECK-NEXT: .b8 3                                // Abbrev [3] 0x3e6:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 5                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 247                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 5626                            // DW_AT_import
+; CHECK-NEXT: .b8 3                                // Abbrev [3] 0x3ed:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 5                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 249                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 5653                            // DW_AT_import
+; CHECK-NEXT: .b8 3                                // Abbrev [3] 0x3f4:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 5                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 250                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 5768                            // DW_AT_import
+; CHECK-NEXT: .b8 3                                // Abbrev [3] 0x3fb:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 5                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 251                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 5675                            // DW_AT_import
+; CHECK-NEXT: .b8 3                                // Abbrev [3] 0x402:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 5                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 252                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 5708                            // DW_AT_import
+; CHECK-NEXT: .b8 3                                // Abbrev [3] 0x409:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 5                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 253                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 5795                            // DW_AT_import
+; CHECK-NEXT: .b8 4                                // Abbrev [4] 0x410:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 10                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 149                              // DW_AT_decl_line
+; CHECK-NEXT: .b8 1
+; CHECK-NEXT: .b32 5838                            // DW_AT_import
+; CHECK-NEXT: .b8 4                                // Abbrev [4] 0x418:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 10                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 150                              // DW_AT_decl_line
+; CHECK-NEXT: .b8 1
+; CHECK-NEXT: .b32 5870                            // DW_AT_import
+; CHECK-NEXT: .b8 4                                // Abbrev [4] 0x420:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 10                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 151                              // DW_AT_decl_line
+; CHECK-NEXT: .b8 1
+; CHECK-NEXT: .b32 5904                            // DW_AT_import
+; CHECK-NEXT: .b8 4                                // Abbrev [4] 0x428:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 10                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 152                              // DW_AT_decl_line
+; CHECK-NEXT: .b8 1
+; CHECK-NEXT: .b32 5936                            // DW_AT_import
+; CHECK-NEXT: .b8 4                                // Abbrev [4] 0x430:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 10                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 153                              // DW_AT_decl_line
+; CHECK-NEXT: .b8 1
+; CHECK-NEXT: .b32 5970                            // DW_AT_import
+; CHECK-NEXT: .b8 4                                // Abbrev [4] 0x438:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 10                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 154                              // DW_AT_decl_line
+; CHECK-NEXT: .b8 1
+; CHECK-NEXT: .b32 6010                            // DW_AT_import
+; CHECK-NEXT: .b8 4                                // Abbrev [4] 0x440:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 10                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 155                              // DW_AT_decl_line
+; CHECK-NEXT: .b8 1
+; CHECK-NEXT: .b32 6042                            // DW_AT_import
+; CHECK-NEXT: .b8 4                                // Abbrev [4] 0x448:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 10                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 156                              // DW_AT_decl_line
+; CHECK-NEXT: .b8 1
+; CHECK-NEXT: .b32 6076                            // DW_AT_import
+; CHECK-NEXT: .b8 4                                // Abbrev [4] 0x450:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 10                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 157                              // DW_AT_decl_line
+; CHECK-NEXT: .b8 1
+; CHECK-NEXT: .b32 6108                            // DW_AT_import
+; CHECK-NEXT: .b8 4                                // Abbrev [4] 0x458:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 10                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 158                              // DW_AT_decl_line
+; CHECK-NEXT: .b8 1
+; CHECK-NEXT: .b32 6140                            // DW_AT_import
+; CHECK-NEXT: .b8 4                                // Abbrev [4] 0x460:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 10                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 159                              // DW_AT_decl_line
+; CHECK-NEXT: .b8 1
+; CHECK-NEXT: .b32 6186                            // DW_AT_import
+; CHECK-NEXT: .b8 4                                // Abbrev [4] 0x468:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 10                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 160                              // DW_AT_decl_line
+; CHECK-NEXT: .b8 1
+; CHECK-NEXT: .b32 6216                            // DW_AT_import
+; CHECK-NEXT: .b8 4                                // Abbrev [4] 0x470:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 10                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 161                              // DW_AT_decl_line
+; CHECK-NEXT: .b8 1
+; CHECK-NEXT: .b32 6248                            // DW_AT_import
+; CHECK-NEXT: .b8 4                                // Abbrev [4] 0x478:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 10                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 162                              // DW_AT_decl_line
+; CHECK-NEXT: .b8 1
+; CHECK-NEXT: .b32 6280                            // DW_AT_import
+; CHECK-NEXT: .b8 4                                // Abbrev [4] 0x480:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 10                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 163                              // DW_AT_decl_line
+; CHECK-NEXT: .b8 1
+; CHECK-NEXT: .b32 6310                            // DW_AT_import
+; CHECK-NEXT: .b8 4                                // Abbrev [4] 0x488:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 10                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 164                              // DW_AT_decl_line
+; CHECK-NEXT: .b8 1
+; CHECK-NEXT: .b32 6342                            // DW_AT_import
+; CHECK-NEXT: .b8 4                                // Abbrev [4] 0x490:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 10                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 165                              // DW_AT_decl_line
+; CHECK-NEXT: .b8 1
+; CHECK-NEXT: .b32 6372                            // DW_AT_import
+; CHECK-NEXT: .b8 4                                // Abbrev [4] 0x498:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 10                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 166                              // DW_AT_decl_line
+; CHECK-NEXT: .b8 1
+; CHECK-NEXT: .b32 6406                            // DW_AT_import
+; CHECK-NEXT: .b8 4                                // Abbrev [4] 0x4a0:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 10                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 167                              // DW_AT_decl_line
+; CHECK-NEXT: .b8 1
+; CHECK-NEXT: .b32 6438                            // DW_AT_import
+; CHECK-NEXT: .b8 4                                // Abbrev [4] 0x4a8:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 10                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 168                              // DW_AT_decl_line
+; CHECK-NEXT: .b8 1
+; CHECK-NEXT: .b32 6476                            // DW_AT_import
+; CHECK-NEXT: .b8 4                                // Abbrev [4] 0x4b0:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 10                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 169                              // DW_AT_decl_line
+; CHECK-NEXT: .b8 1
+; CHECK-NEXT: .b32 6510                            // DW_AT_import
+; CHECK-NEXT: .b8 4                                // Abbrev [4] 0x4b8:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 10                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 170                              // DW_AT_decl_line
+; CHECK-NEXT: .b8 1
+; CHECK-NEXT: .b32 6552                            // DW_AT_import
+; CHECK-NEXT: .b8 4                                // Abbrev [4] 0x4c0:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 10                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 171                              // DW_AT_decl_line
+; CHECK-NEXT: .b8 1
+; CHECK-NEXT: .b32 6590                            // DW_AT_import
+; CHECK-NEXT: .b8 4                                // Abbrev [4] 0x4c8:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 10                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 172                              // DW_AT_decl_line
+; CHECK-NEXT: .b8 1
+; CHECK-NEXT: .b32 6628                            // DW_AT_import
+; CHECK-NEXT: .b8 4                                // Abbrev [4] 0x4d0:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 10                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 173                              // DW_AT_decl_line
+; CHECK-NEXT: .b8 1
+; CHECK-NEXT: .b32 6666                            // DW_AT_import
+; CHECK-NEXT: .b8 4                                // Abbrev [4] 0x4d8:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 10                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 174                              // DW_AT_decl_line
+; CHECK-NEXT: .b8 1
+; CHECK-NEXT: .b32 6707                            // DW_AT_import
+; CHECK-NEXT: .b8 4                                // Abbrev [4] 0x4e0:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 10                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 175                              // DW_AT_decl_line
+; CHECK-NEXT: .b8 1
+; CHECK-NEXT: .b32 6747                            // DW_AT_import
+; CHECK-NEXT: .b8 4                                // Abbrev [4] 0x4e8:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 10                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 176                              // DW_AT_decl_line
+; CHECK-NEXT: .b8 1
+; CHECK-NEXT: .b32 6781                            // DW_AT_import
+; CHECK-NEXT: .b8 4                                // Abbrev [4] 0x4f0:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 10                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 177                              // DW_AT_decl_line
+; CHECK-NEXT: .b8 1
+; CHECK-NEXT: .b32 6821                            // DW_AT_import
+; CHECK-NEXT: .b8 4                                // Abbrev [4] 0x4f8:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 10                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 178                              // DW_AT_decl_line
+; CHECK-NEXT: .b8 1
+; CHECK-NEXT: .b32 6857                            // DW_AT_import
+; CHECK-NEXT: .b8 4                                // Abbrev [4] 0x500:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 10                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 179                              // DW_AT_decl_line
+; CHECK-NEXT: .b8 1
+; CHECK-NEXT: .b32 6893                            // DW_AT_import
+; CHECK-NEXT: .b8 4                                // Abbrev [4] 0x508:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 10                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 180                              // DW_AT_decl_line
+; CHECK-NEXT: .b8 1
+; CHECK-NEXT: .b32 6931                            // DW_AT_import
+; CHECK-NEXT: .b8 4                                // Abbrev [4] 0x510:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 10                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 181                              // DW_AT_decl_line
+; CHECK-NEXT: .b8 1
+; CHECK-NEXT: .b32 6965                            // DW_AT_import
+; CHECK-NEXT: .b8 4                                // Abbrev [4] 0x518:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 10                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 182                              // DW_AT_decl_line
+; CHECK-NEXT: .b8 1
+; CHECK-NEXT: .b32 6999                            // DW_AT_import
+; CHECK-NEXT: .b8 4                                // Abbrev [4] 0x520:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 10                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 183                              // DW_AT_decl_line
+; CHECK-NEXT: .b8 1
+; CHECK-NEXT: .b32 7031                            // DW_AT_import
+; CHECK-NEXT: .b8 4                                // Abbrev [4] 0x528:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 10                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 184                              // DW_AT_decl_line
+; CHECK-NEXT: .b8 1
+; CHECK-NEXT: .b32 7063                            // DW_AT_import
+; CHECK-NEXT: .b8 4                                // Abbrev [4] 0x530:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 10                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 185                              // DW_AT_decl_line
+; CHECK-NEXT: .b8 1
+; CHECK-NEXT: .b32 7093                            // DW_AT_import
+; CHECK-NEXT: .b8 4                                // Abbrev [4] 0x538:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 10                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 186                              // DW_AT_decl_line
+; CHECK-NEXT: .b8 1
+; CHECK-NEXT: .b32 7127                            // DW_AT_import
+; CHECK-NEXT: .b8 4                                // Abbrev [4] 0x540:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 10                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 187                              // DW_AT_decl_line
+; CHECK-NEXT: .b8 1
+; CHECK-NEXT: .b32 7163                            // DW_AT_import
+; CHECK-NEXT: .b8 4                                // Abbrev [4] 0x548:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 10                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 188                              // DW_AT_decl_line
+; CHECK-NEXT: .b8 1
+; CHECK-NEXT: .b32 7202                            // DW_AT_import
+; CHECK-NEXT: .b8 4                                // Abbrev [4] 0x550:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 10                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 189                              // DW_AT_decl_line
+; CHECK-NEXT: .b8 1
+; CHECK-NEXT: .b32 7245                            // DW_AT_import
+; CHECK-NEXT: .b8 4                                // Abbrev [4] 0x558:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 10                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 190                              // DW_AT_decl_line
+; CHECK-NEXT: .b8 1
+; CHECK-NEXT: .b32 7294                            // DW_AT_import
+; CHECK-NEXT: .b8 4                                // Abbrev [4] 0x560:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 10                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 191                              // DW_AT_decl_line
+; CHECK-NEXT: .b8 1
+; CHECK-NEXT: .b32 7330                            // DW_AT_import
+; CHECK-NEXT: .b8 4                                // Abbrev [4] 0x568:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 10                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 192                              // DW_AT_decl_line
+; CHECK-NEXT: .b8 1
+; CHECK-NEXT: .b32 7379                            // DW_AT_import
+; CHECK-NEXT: .b8 4                                // Abbrev [4] 0x570:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 10                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 193                              // DW_AT_decl_line
+; CHECK-NEXT: .b8 1
+; CHECK-NEXT: .b32 7428                            // DW_AT_import
+; CHECK-NEXT: .b8 4                                // Abbrev [4] 0x578:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 10                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 194                              // DW_AT_decl_line
+; CHECK-NEXT: .b8 1
+; CHECK-NEXT: .b32 7460                            // DW_AT_import
+; CHECK-NEXT: .b8 4                                // Abbrev [4] 0x580:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 10                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 195                              // DW_AT_decl_line
+; CHECK-NEXT: .b8 1
+; CHECK-NEXT: .b32 7494                            // DW_AT_import
+; CHECK-NEXT: .b8 4                                // Abbrev [4] 0x588:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 10                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 196                              // DW_AT_decl_line
+; CHECK-NEXT: .b8 1
+; CHECK-NEXT: .b32 7538                            // DW_AT_import
+; CHECK-NEXT: .b8 4                                // Abbrev [4] 0x590:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 10                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 197                              // DW_AT_decl_line
+; CHECK-NEXT: .b8 1
+; CHECK-NEXT: .b32 7580                            // DW_AT_import
+; CHECK-NEXT: .b8 4                                // Abbrev [4] 0x598:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 10                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 198                              // DW_AT_decl_line
+; CHECK-NEXT: .b8 1
+; CHECK-NEXT: .b32 7610                            // DW_AT_import
+; CHECK-NEXT: .b8 4                                // Abbrev [4] 0x5a0:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 10                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 199                              // DW_AT_decl_line
+; CHECK-NEXT: .b8 1
+; CHECK-NEXT: .b32 7642                            // DW_AT_import
+; CHECK-NEXT: .b8 4                                // Abbrev [4] 0x5a8:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 10                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 200                              // DW_AT_decl_line
+; CHECK-NEXT: .b8 1
+; CHECK-NEXT: .b32 7674                            // DW_AT_import
+; CHECK-NEXT: .b8 4                                // Abbrev [4] 0x5b0:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 10                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 201                              // DW_AT_decl_line
+; CHECK-NEXT: .b8 1
+; CHECK-NEXT: .b32 7704                            // DW_AT_import
+; CHECK-NEXT: .b8 4                                // Abbrev [4] 0x5b8:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 10                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 202                              // DW_AT_decl_line
+; CHECK-NEXT: .b8 1
+; CHECK-NEXT: .b32 7736                            // DW_AT_import
+; CHECK-NEXT: .b8 4                                // Abbrev [4] 0x5c0:0x8 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 10                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 203                              // DW_AT_decl_line
+; CHECK-NEXT: .b8 1
+; CHECK-NEXT: .b32 7772                            // DW_AT_import
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 5                                // Abbrev [5] 0x5c9:0x1b DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,51,97,98,115,120        // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 97,98,115                        // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 44                               // DW_AT_decl_line
+; CHECK-NEXT: .b32 1508                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x5de:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1508                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 7                                // Abbrev [7] 0x5e4:0x11 DW_TAG_base_type
+; CHECK-NEXT: .b8 108,111,110,103,32,108,111,110,103,32,105,110,116 // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 5                                // DW_AT_encoding
+; CHECK-NEXT: .b8 8                                // DW_AT_byte_size
+; CHECK-NEXT: .b8 5                                // Abbrev [5] 0x5f5:0x1d DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,52,97,99,111,115,102    // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 97,99,111,115                    // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 46                               // DW_AT_decl_line
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x60c:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 7                                // Abbrev [7] 0x612:0x9 DW_TAG_base_type
+; CHECK-NEXT: .b8 102,108,111,97,116               // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 4                                // DW_AT_encoding
+; CHECK-NEXT: .b8 4                                // DW_AT_byte_size
+; CHECK-NEXT: .b8 5                                // Abbrev [5] 0x61b:0x1f DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,53,97,99,111,115,104,102 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 97,99,111,115,104                // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 48                               // DW_AT_decl_line
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x634:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 5                                // Abbrev [5] 0x63a:0x1d DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,52,97,115,105,110,102   // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 97,115,105,110                   // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 50                               // DW_AT_decl_line
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x651:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 5                                // Abbrev [5] 0x657:0x1f DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,53,97,115,105,110,104,102 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 97,115,105,110,104               // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 52                               // DW_AT_decl_line
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x670:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 5                                // Abbrev [5] 0x676:0x1d DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,52,97,116,97,110,102    // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 97,116,97,110                    // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 56                               // DW_AT_decl_line
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x68d:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 5                                // Abbrev [5] 0x693:0x25 DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,53,97,116,97,110,50,102,102 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 97,116,97,110,50                 // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 54                               // DW_AT_decl_line
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x6ad:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x6b2:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 5                                // Abbrev [5] 0x6b8:0x1f DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,53,97,116,97,110,104,102 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 97,116,97,110,104                // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 58                               // DW_AT_decl_line
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x6d1:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 5                                // Abbrev [5] 0x6d7:0x1d DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,52,99,98,114,116,102    // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 99,98,114,116                    // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 60                               // DW_AT_decl_line
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x6ee:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 5                                // Abbrev [5] 0x6f4:0x1d DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,52,99,101,105,108,102   // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 99,101,105,108                   // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 62                               // DW_AT_decl_line
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x70b:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 5                                // Abbrev [5] 0x711:0x2b DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,56,99,111,112,121,115,105,103,110,102,102 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 99,111,112,121,115,105,103,110   // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 64                               // DW_AT_decl_line
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x731:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x736:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 5                                // Abbrev [5] 0x73c:0x1b DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,51,99,111,115,102       // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 99,111,115                       // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 66                               // DW_AT_decl_line
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x751:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 5                                // Abbrev [5] 0x757:0x1d DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,52,99,111,115,104,102   // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 99,111,115,104                   // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 68                               // DW_AT_decl_line
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x76e:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 5                                // Abbrev [5] 0x774:0x1b DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,51,101,114,102,102      // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 101,114,102                      // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 72                               // DW_AT_decl_line
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x789:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 5                                // Abbrev [5] 0x78f:0x1d DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,52,101,114,102,99,102   // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 101,114,102,99                   // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 70                               // DW_AT_decl_line
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x7a6:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 5                                // Abbrev [5] 0x7ac:0x1b DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,51,101,120,112,102      // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 101,120,112                      // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 76                               // DW_AT_decl_line
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x7c1:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 5                                // Abbrev [5] 0x7c7:0x1d DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,52,101,120,112,50,102   // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 101,120,112,50                   // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 74                               // DW_AT_decl_line
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x7de:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 5                                // Abbrev [5] 0x7e4:0x1f DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,53,101,120,112,109,49,102 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 101,120,112,109,49               // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 78                               // DW_AT_decl_line
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x7fd:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 5                                // Abbrev [5] 0x803:0x1d DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,52,102,97,98,115,102    // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 102,97,98,115                    // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 80                               // DW_AT_decl_line
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x81a:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 5                                // Abbrev [5] 0x820:0x23 DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,52,102,100,105,109,102,102 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 102,100,105,109                  // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 82                               // DW_AT_decl_line
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x838:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x83d:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 5                                // Abbrev [5] 0x843:0x1f DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,53,102,108,111,111,114,102 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 102,108,111,111,114              // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 84                               // DW_AT_decl_line
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x85c:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 5                                // Abbrev [5] 0x862:0x27 DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,51,102,109,97,102,102,102 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 102,109,97                       // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 86                               // DW_AT_decl_line
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x879:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x87e:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x883:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 5                                // Abbrev [5] 0x889:0x23 DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,52,102,109,97,120,102,102 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 102,109,97,120                   // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 88                               // DW_AT_decl_line
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x8a1:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x8a6:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 5                                // Abbrev [5] 0x8ac:0x23 DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,52,102,109,105,110,102,102 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 102,109,105,110                  // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 90                               // DW_AT_decl_line
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x8c4:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x8c9:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 5                                // Abbrev [5] 0x8cf:0x23 DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,52,102,109,111,100,102,102 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 102,109,111,100                  // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 92                               // DW_AT_decl_line
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x8e7:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x8ec:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 5                                // Abbrev [5] 0x8f2:0x2a DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,49,48,102,112,99,108,97,115,115,105,102,121,102 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 102,112,99,108,97,115,115,105,102,121 // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 94                               // DW_AT_decl_line
+; CHECK-NEXT: .b32 2332                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x916:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 7                                // Abbrev [7] 0x91c:0x7 DW_TAG_base_type
+; CHECK-NEXT: .b8 105,110,116                      // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 5                                // DW_AT_encoding
+; CHECK-NEXT: .b8 4                                // DW_AT_byte_size
+; CHECK-NEXT: .b8 5                                // Abbrev [5] 0x923:0x26 DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,53,102,114,101,120,112,102,80,105 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 102,114,101,120,112              // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 96                               // DW_AT_decl_line
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x93e:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x943:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 2377                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 8                                // Abbrev [8] 0x949:0x5 DW_TAG_pointer_type
+; CHECK-NEXT: .b32 2332                            // DW_AT_type
+; CHECK-NEXT: .b8 5                                // Abbrev [5] 0x94e:0x25 DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,53,104,121,112,111,116,102,102 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 104,121,112,111,116              // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 98                               // DW_AT_decl_line
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x968:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x96d:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 5                                // Abbrev [5] 0x973:0x1f DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,53,105,108,111,103,98,102 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 105,108,111,103,98               // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 100                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 2332                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x98c:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 5                                // Abbrev [5] 0x992:0x25 DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,56,105,115,102,105,110,105,116,101,102 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 105,115,102,105,110,105,116,101  // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 102                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 2487                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x9b1:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 7                                // Abbrev [7] 0x9b7:0x8 DW_TAG_base_type
+; CHECK-NEXT: .b8 98,111,111,108                   // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 2                                // DW_AT_encoding
+; CHECK-NEXT: .b8 1                                // DW_AT_byte_size
+; CHECK-NEXT: .b8 5                                // Abbrev [5] 0x9bf:0x2d DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,57,105,115,103,114,101,97,116,101,114,102,102 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 105,115,103,114,101,97,116,101,114 // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 106                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 2487                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x9e1:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x9e6:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 5                                // Abbrev [5] 0x9ec:0x38 DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,49,52,105,115,103,114,101,97,116,101,114,101,113,117,97,108,102,102 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 105,115,103,114,101,97,116,101,114,101,113,117,97,108 // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 105                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 2487                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0xa19:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0xa1e:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 5                                // Abbrev [5] 0xa24:0x1f DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,53,105,115,105,110,102,102 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 105,115,105,110,102              // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 108                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 2487                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0xa3d:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 5                                // Abbrev [5] 0xa43:0x27 DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,54,105,115,108,101,115,115,102,102 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 105,115,108,101,115,115          // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 112                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 2487                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0xa5f:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0xa64:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 5                                // Abbrev [5] 0xa6a:0x32 DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,49,49,105,115,108,101,115,115,101,113,117,97,108,102,102 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 105,115,108,101,115,115,101,113,117,97,108 // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 111                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 2487                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0xa91:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0xa96:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 5                                // Abbrev [5] 0xa9c:0x36 DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,49,51,105,115,108,101,115,115,103,114,101,97,116,101,114,102,102 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 105,115,108,101,115,115,103,114,101,97,116,101,114 // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 114                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 2487                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0xac7:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0xacc:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 5                                // Abbrev [5] 0xad2:0x1f DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,53,105,115,110,97,110,102 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 105,115,110,97,110               // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 116                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 2487                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0xaeb:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 5                                // Abbrev [5] 0xaf1:0x25 DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,56,105,115,110,111,114,109,97,108,102 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 105,115,110,111,114,109,97,108   // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 118                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 2487                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0xb10:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 5                                // Abbrev [5] 0xb16:0x32 DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,49,49,105,115,117,110,111,114,100,101,114,101,100,102,102 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 105,115,117,110,111,114,100,101,114,101,100 // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 120                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 2487                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0xb3d:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0xb42:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 5                                // Abbrev [5] 0xb48:0x1d DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,52,108,97,98,115,108    // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 108,97,98,115                    // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 121                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 2917                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0xb5f:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 2917                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 7                                // Abbrev [7] 0xb65:0xc DW_TAG_base_type
+; CHECK-NEXT: .b8 108,111,110,103,32,105,110,116   // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 5                                // DW_AT_encoding
+; CHECK-NEXT: .b8 8                                // DW_AT_byte_size
+; CHECK-NEXT: .b8 5                                // Abbrev [5] 0xb71:0x25 DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,53,108,100,101,120,112,102,105 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 108,100,101,120,112              // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 123                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0xb8b:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0xb90:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 2332                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 5                                // Abbrev [5] 0xb96:0x21 DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,54,108,103,97,109,109,97,102 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 108,103,97,109,109,97            // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 125                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0xbb1:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 5                                // Abbrev [5] 0xbb7:0x1f DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,53,108,108,97,98,115,120 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 108,108,97,98,115                // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 126                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 1508                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0xbd0:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1508                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 5                                // Abbrev [5] 0xbd6:0x21 DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,54,108,108,114,105,110,116,102 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 108,108,114,105,110,116          // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 128                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 1508                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0xbf1:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 5                                // Abbrev [5] 0xbf7:0x1b DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,51,108,111,103,102      // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 108,111,103                      // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 138                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0xc0c:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 5                                // Abbrev [5] 0xc12:0x1f DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,53,108,111,103,49,48,102 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 108,111,103,49,48                // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 130                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0xc2b:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 5                                // Abbrev [5] 0xc31:0x1f DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,53,108,111,103,49,112,102 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 108,111,103,49,112               // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 132                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0xc4a:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 5                                // Abbrev [5] 0xc50:0x1d DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,52,108,111,103,50,102   // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 108,111,103,50                   // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 134                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0xc67:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 5                                // Abbrev [5] 0xc6d:0x1d DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,52,108,111,103,98,102   // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 108,111,103,98                   // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 136                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0xc84:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 5                                // Abbrev [5] 0xc8a:0x1f DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,53,108,114,105,110,116,102 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 108,114,105,110,116              // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 140                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 2917                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0xca3:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 5                                // Abbrev [5] 0xca9:0x21 DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,54,108,114,111,117,110,100,102 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 108,114,111,117,110,100          // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 142                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 2917                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0xcc4:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 5                                // Abbrev [5] 0xcca:0x23 DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,55,108,108,114,111,117,110,100,102 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 108,108,114,111,117,110,100      // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 143                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 1508                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0xce7:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 5                                // Abbrev [5] 0xced:0x24 DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,52,109,111,100,102,102,80,102 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 109,111,100,102                  // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 145                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0xd06:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0xd0b:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 3345                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 8                                // Abbrev [8] 0xd11:0x5 DW_TAG_pointer_type
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 5                                // Abbrev [5] 0xd16:0x1d DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,51,110,97,110,80,75,99  // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 110,97,110                       // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 146                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 3379                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0xd2d:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 3389                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 7                                // Abbrev [7] 0xd33:0xa DW_TAG_base_type
+; CHECK-NEXT: .b8 100,111,117,98,108,101           // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 4                                // DW_AT_encoding
+; CHECK-NEXT: .b8 8                                // DW_AT_byte_size
+; CHECK-NEXT: .b8 8                                // Abbrev [8] 0xd3d:0x5 DW_TAG_pointer_type
+; CHECK-NEXT: .b32 3394                            // DW_AT_type
+; CHECK-NEXT: .b8 9                                // Abbrev [9] 0xd42:0x5 DW_TAG_const_type
+; CHECK-NEXT: .b32 3399                            // DW_AT_type
+; CHECK-NEXT: .b8 7                                // Abbrev [7] 0xd47:0x8 DW_TAG_base_type
+; CHECK-NEXT: .b8 99,104,97,114                    // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 8                                // DW_AT_encoding
+; CHECK-NEXT: .b8 1                                // DW_AT_byte_size
+; CHECK-NEXT: .b8 5                                // Abbrev [5] 0xd4f:0x1f DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,52,110,97,110,102,80,75,99 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 110,97,110,102                   // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 147                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0xd68:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 3389                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 5                                // Abbrev [5] 0xd6e:0x27 DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,57,110,101,97,114,98,121,105,110,116,102 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 110,101,97,114,98,121,105,110,116 // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 149                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0xd8f:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 5                                // Abbrev [5] 0xd95:0x2d DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,57,110,101,120,116,97,102,116,101,114,102,102 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 110,101,120,116,97,102,116,101,114 // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 151                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0xdb7:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0xdbc:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 5                                // Abbrev [5] 0xdc2:0x21 DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,51,112,111,119,102,105  // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 112,111,119                      // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 155                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0xdd8:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0xddd:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 2332                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 5                                // Abbrev [5] 0xde3:0x2d DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,57,114,101,109,97,105,110,100,101,114,102,102 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 114,101,109,97,105,110,100,101,114 // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 157                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0xe05:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0xe0a:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 5                                // Abbrev [5] 0xe10:0x2e DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,54,114,101,109,113,117,111,102,102,80,105 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 114,101,109,113,117,111          // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 159                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0xe2e:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0xe33:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0xe38:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 2377                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 5                                // Abbrev [5] 0xe3e:0x1d DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,52,114,105,110,116,102  // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 114,105,110,116                  // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 161                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0xe55:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 5                                // Abbrev [5] 0xe5b:0x1f DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,53,114,111,117,110,100,102 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 114,111,117,110,100              // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 163                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0xe74:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 5                                // Abbrev [5] 0xe7a:0x29 DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,55,115,99,97,108,98,108,110,102,108 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 115,99,97,108,98,108,110         // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 165                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0xe98:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0xe9d:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 2917                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 5                                // Abbrev [5] 0xea3:0x27 DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,54,115,99,97,108,98,110,102,105 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 115,99,97,108,98,110             // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 167                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0xebf:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0xec4:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 2332                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 5                                // Abbrev [5] 0xeca:0x23 DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,55,115,105,103,110,98,105,116,102 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 115,105,103,110,98,105,116       // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 169                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 2487                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0xee7:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 5                                // Abbrev [5] 0xeed:0x1b DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,51,115,105,110,102      // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 115,105,110                      // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 171                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0xf02:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 5                                // Abbrev [5] 0xf08:0x1d DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,52,115,105,110,104,102  // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 115,105,110,104                  // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 173                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0xf1f:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 5                                // Abbrev [5] 0xf25:0x1d DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,52,115,113,114,116,102  // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 115,113,114,116                  // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 175                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0xf3c:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 5                                // Abbrev [5] 0xf42:0x1b DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,51,116,97,110,102       // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 116,97,110                       // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 177                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0xf57:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 5                                // Abbrev [5] 0xf5d:0x1d DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,52,116,97,110,104,102   // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 116,97,110,104                   // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 179                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0xf74:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 5                                // Abbrev [5] 0xf7a:0x21 DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,54,116,103,97,109,109,97,102 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 116,103,97,109,109,97            // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 181                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0xf95:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 5                                // Abbrev [5] 0xf9b:0x1f DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,53,116,114,117,110,99,102 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 116,114,117,110,99               // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 183                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0xfb4:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 10                               // Abbrev [10] 0xfba:0x14 DW_TAG_subprogram
+; CHECK-NEXT: .b8 97,99,111,115                    // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 2                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 54                               // DW_AT_decl_line
+; CHECK-NEXT: .b32 3379                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_external
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0xfc8:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 3379                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 10                               // Abbrev [10] 0xfce:0x14 DW_TAG_subprogram
+; CHECK-NEXT: .b8 97,115,105,110                   // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 2                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 56                               // DW_AT_decl_line
+; CHECK-NEXT: .b32 3379                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_external
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0xfdc:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 3379                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 10                               // Abbrev [10] 0xfe2:0x14 DW_TAG_subprogram
+; CHECK-NEXT: .b8 97,116,97,110                    // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 2                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 58                               // DW_AT_decl_line
+; CHECK-NEXT: .b32 3379                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_external
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0xff0:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 3379                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 10                               // Abbrev [10] 0xff6:0x1a DW_TAG_subprogram
+; CHECK-NEXT: .b8 97,116,97,110,50                 // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 2                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 60                               // DW_AT_decl_line
+; CHECK-NEXT: .b32 3379                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_external
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x1005:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 3379                            // DW_AT_type
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x100a:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 3379                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 10                               // Abbrev [10] 0x1010:0x14 DW_TAG_subprogram
+; CHECK-NEXT: .b8 99,101,105,108                   // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 2                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 178                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 3379                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_external
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x101e:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 3379                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 10                               // Abbrev [10] 0x1024:0x13 DW_TAG_subprogram
+; CHECK-NEXT: .b8 99,111,115                       // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 2                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 63                               // DW_AT_decl_line
+; CHECK-NEXT: .b32 3379                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_external
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x1031:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 3379                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 10                               // Abbrev [10] 0x1037:0x14 DW_TAG_subprogram
+; CHECK-NEXT: .b8 99,111,115,104                   // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 2                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 72                               // DW_AT_decl_line
+; CHECK-NEXT: .b32 3379                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_external
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x1045:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 3379                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 10                               // Abbrev [10] 0x104b:0x13 DW_TAG_subprogram
+; CHECK-NEXT: .b8 101,120,112                      // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 2                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 100                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 3379                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_external
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x1058:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 3379                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 10                               // Abbrev [10] 0x105e:0x14 DW_TAG_subprogram
+; CHECK-NEXT: .b8 102,97,98,115                    // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 2                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 181                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 3379                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_external
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x106c:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 3379                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 10                               // Abbrev [10] 0x1072:0x15 DW_TAG_subprogram
+; CHECK-NEXT: .b8 102,108,111,111,114              // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 2                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 184                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 3379                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_external
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x1081:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 3379                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 10                               // Abbrev [10] 0x1087:0x19 DW_TAG_subprogram
+; CHECK-NEXT: .b8 102,109,111,100                  // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 2                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 187                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 3379                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_external
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x1095:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 3379                            // DW_AT_type
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x109a:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 3379                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 10                               // Abbrev [10] 0x10a0:0x1a DW_TAG_subprogram
+; CHECK-NEXT: .b8 102,114,101,120,112              // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 2                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 103                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 3379                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_external
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x10af:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 3379                            // DW_AT_type
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x10b4:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 2377                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 10                               // Abbrev [10] 0x10ba:0x1a DW_TAG_subprogram
+; CHECK-NEXT: .b8 108,100,101,120,112              // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 2                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 106                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 3379                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_external
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x10c9:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 3379                            // DW_AT_type
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x10ce:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 2332                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 10                               // Abbrev [10] 0x10d4:0x13 DW_TAG_subprogram
+; CHECK-NEXT: .b8 108,111,103                      // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 2                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 109                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 3379                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_external
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x10e1:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 3379                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 10                               // Abbrev [10] 0x10e7:0x15 DW_TAG_subprogram
+; CHECK-NEXT: .b8 108,111,103,49,48                // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 2                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 112                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 3379                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_external
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x10f6:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 3379                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 10                               // Abbrev [10] 0x10fc:0x19 DW_TAG_subprogram
+; CHECK-NEXT: .b8 109,111,100,102                  // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 2                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 115                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 3379                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_external
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x110a:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 3379                            // DW_AT_type
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x110f:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 4373                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 8                                // Abbrev [8] 0x1115:0x5 DW_TAG_pointer_type
+; CHECK-NEXT: .b32 3379                            // DW_AT_type
+; CHECK-NEXT: .b8 10                               // Abbrev [10] 0x111a:0x18 DW_TAG_subprogram
+; CHECK-NEXT: .b8 112,111,119                      // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 2                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 153                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 3379                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_external
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x1127:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 3379                            // DW_AT_type
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x112c:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 3379                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 10                               // Abbrev [10] 0x1132:0x13 DW_TAG_subprogram
+; CHECK-NEXT: .b8 115,105,110                      // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 2                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 65                               // DW_AT_decl_line
+; CHECK-NEXT: .b32 3379                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_external
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x113f:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 3379                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 10                               // Abbrev [10] 0x1145:0x14 DW_TAG_subprogram
+; CHECK-NEXT: .b8 115,105,110,104                  // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 2                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 74                               // DW_AT_decl_line
+; CHECK-NEXT: .b32 3379                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_external
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x1153:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 3379                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 10                               // Abbrev [10] 0x1159:0x14 DW_TAG_subprogram
+; CHECK-NEXT: .b8 115,113,114,116                  // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 2                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 156                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 3379                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_external
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x1167:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 3379                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 10                               // Abbrev [10] 0x116d:0x13 DW_TAG_subprogram
+; CHECK-NEXT: .b8 116,97,110                       // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 2                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 67                               // DW_AT_decl_line
+; CHECK-NEXT: .b32 3379                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_external
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x117a:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 3379                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 10                               // Abbrev [10] 0x1180:0x14 DW_TAG_subprogram
+; CHECK-NEXT: .b8 116,97,110,104                   // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 2                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 76                               // DW_AT_decl_line
+; CHECK-NEXT: .b32 3379                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_external
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x118e:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 3379                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 11                               // Abbrev [11] 0x1194:0xd DW_TAG_typedef
+; CHECK-NEXT: .b32 4513                            // DW_AT_type
+; CHECK-NEXT: .b8 100,105,118,95,116               // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 4                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 101                              // DW_AT_decl_line
+; CHECK-NEXT: .b8 12                               // Abbrev [12] 0x11a1:0x2 DW_TAG_structure_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 11                               // Abbrev [11] 0x11a3:0xe DW_TAG_typedef
+; CHECK-NEXT: .b32 4529                            // DW_AT_type
+; CHECK-NEXT: .b8 108,100,105,118,95,116           // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 4                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 109                              // DW_AT_decl_line
+; CHECK-NEXT: .b8 13                               // Abbrev [13] 0x11b1:0x22 DW_TAG_structure_type
+; CHECK-NEXT: .b8 16                               // DW_AT_byte_size
+; CHECK-NEXT: .b8 4                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 105                              // DW_AT_decl_line
+; CHECK-NEXT: .b8 14                               // Abbrev [14] 0x11b5:0xf DW_TAG_member
+; CHECK-NEXT: .b8 113,117,111,116                  // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b32 2917                            // DW_AT_type
+; CHECK-NEXT: .b8 4                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 107                              // DW_AT_decl_line
+; CHECK-NEXT: .b8 2                                // DW_AT_data_member_location
+; CHECK-NEXT: .b8 35
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 14                               // Abbrev [14] 0x11c4:0xe DW_TAG_member
+; CHECK-NEXT: .b8 114,101,109                      // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b32 2917                            // DW_AT_type
+; CHECK-NEXT: .b8 4                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 108                              // DW_AT_decl_line
+; CHECK-NEXT: .b8 2                                // DW_AT_data_member_location
+; CHECK-NEXT: .b8 35
+; CHECK-NEXT: .b8 8
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 15                               // Abbrev [15] 0x11d3:0xd DW_TAG_subprogram
+; CHECK-NEXT: .b8 97,98,111,114,116                // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 4                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 3                                // DW_AT_decl_line
+; CHECK-NEXT: .b8 2
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_external
+; CHECK-NEXT: .b8 1                                // DW_AT_noreturn
+; CHECK-NEXT: .b8 16                               // Abbrev [16] 0x11e0:0x14 DW_TAG_subprogram
+; CHECK-NEXT: .b8 97,98,115                        // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 4                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 7                                // DW_AT_decl_line
+; CHECK-NEXT: .b8 3
+; CHECK-NEXT: .b32 2332                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_external
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x11ee:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 2332                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 16                               // Abbrev [16] 0x11f4:0x17 DW_TAG_subprogram
+; CHECK-NEXT: .b8 97,116,101,120,105,116           // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 4                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 7                                // DW_AT_decl_line
+; CHECK-NEXT: .b8 2
+; CHECK-NEXT: .b32 2332                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_external
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x1205:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 4619                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 8                                // Abbrev [8] 0x120b:0x5 DW_TAG_pointer_type
+; CHECK-NEXT: .b32 4624                            // DW_AT_type
+; CHECK-NEXT: .b8 17                               // Abbrev [17] 0x1210:0x1 DW_TAG_subroutine_type
+; CHECK-NEXT: .b8 10                               // Abbrev [10] 0x1211:0x14 DW_TAG_subprogram
+; CHECK-NEXT: .b8 97,116,111,102                   // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 6                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 26                               // DW_AT_decl_line
+; CHECK-NEXT: .b32 3379                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_external
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x121f:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 3389                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 16                               // Abbrev [16] 0x1225:0x15 DW_TAG_subprogram
+; CHECK-NEXT: .b8 97,116,111,105                   // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 4                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 22                               // DW_AT_decl_line
+; CHECK-NEXT: .b8 1
+; CHECK-NEXT: .b32 2332                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_external
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x1234:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 3389                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 16                               // Abbrev [16] 0x123a:0x15 DW_TAG_subprogram
+; CHECK-NEXT: .b8 97,116,111,108                   // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 4                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 27                               // DW_AT_decl_line
+; CHECK-NEXT: .b8 1
+; CHECK-NEXT: .b32 2917                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_external
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x1249:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 3389                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 10                               // Abbrev [10] 0x124f:0x2b DW_TAG_subprogram
+; CHECK-NEXT: .b8 98,115,101,97,114,99,104         // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 7                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 20                               // DW_AT_decl_line
+; CHECK-NEXT: .b32 4730                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_external
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x1260:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 4731                            // DW_AT_type
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x1265:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 4731                            // DW_AT_type
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x126a:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 4737                            // DW_AT_type
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x126f:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 4737                            // DW_AT_type
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x1274:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 4772                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 18                               // Abbrev [18] 0x127a:0x1 DW_TAG_pointer_type
+; CHECK-NEXT: .b8 8                                // Abbrev [8] 0x127b:0x5 DW_TAG_pointer_type
+; CHECK-NEXT: .b32 4736                            // DW_AT_type
+; CHECK-NEXT: .b8 19                               // Abbrev [19] 0x1280:0x1 DW_TAG_const_type
+; CHECK-NEXT: .b8 11                               // Abbrev [11] 0x1281:0xe DW_TAG_typedef
+; CHECK-NEXT: .b32 4751                            // DW_AT_type
+; CHECK-NEXT: .b8 115,105,122,101,95,116           // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 8                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 62                               // DW_AT_decl_line
+; CHECK-NEXT: .b8 7                                // Abbrev [7] 0x128f:0x15 DW_TAG_base_type
+; CHECK-NEXT: .b8 108,111,110,103,32,117,110,115,105,103,110,101,100,32,105,110,116 // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 7                                // DW_AT_encoding
+; CHECK-NEXT: .b8 8                                // DW_AT_byte_size
+; CHECK-NEXT: .b8 20                               // Abbrev [20] 0x12a4:0x16 DW_TAG_typedef
+; CHECK-NEXT: .b32 4794                            // DW_AT_type
+; CHECK-NEXT: .b8 95,95,99,111,109,112,97,114,95,102,110,95,116 // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 4                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 230                              // DW_AT_decl_line
+; CHECK-NEXT: .b8 2
+; CHECK-NEXT: .b8 8                                // Abbrev [8] 0x12ba:0x5 DW_TAG_pointer_type
+; CHECK-NEXT: .b32 4799                            // DW_AT_type
+; CHECK-NEXT: .b8 21                               // Abbrev [21] 0x12bf:0x10 DW_TAG_subroutine_type
+; CHECK-NEXT: .b32 2332                            // DW_AT_type
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x12c4:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 4731                            // DW_AT_type
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x12c9:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 4731                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 16                               // Abbrev [16] 0x12cf:0x1c DW_TAG_subprogram
+; CHECK-NEXT: .b8 99,97,108,108,111,99             // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 4                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 212                              // DW_AT_decl_line
+; CHECK-NEXT: .b8 1
+; CHECK-NEXT: .b32 4730                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_external
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x12e0:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 4737                            // DW_AT_type
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x12e5:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 4737                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 16                               // Abbrev [16] 0x12eb:0x19 DW_TAG_subprogram
+; CHECK-NEXT: .b8 100,105,118                      // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 4                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 21                               // DW_AT_decl_line
+; CHECK-NEXT: .b8 3
+; CHECK-NEXT: .b32 4500                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_external
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x12f9:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 2332                            // DW_AT_type
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x12fe:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 2332                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 22                               // Abbrev [22] 0x1304:0x12 DW_TAG_subprogram
+; CHECK-NEXT: .b8 101,120,105,116                  // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 4                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 31                               // DW_AT_decl_line
+; CHECK-NEXT: .b8 2
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_external
+; CHECK-NEXT: .b8 1                                // DW_AT_noreturn
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x1310:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 2332                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 23                               // Abbrev [23] 0x1316:0x11 DW_TAG_subprogram
+; CHECK-NEXT: .b8 102,114,101,101                  // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 4                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 227                              // DW_AT_decl_line
+; CHECK-NEXT: .b8 1
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_external
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x1321:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 4730                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 16                               // Abbrev [16] 0x1327:0x17 DW_TAG_subprogram
+; CHECK-NEXT: .b8 103,101,116,101,110,118          // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 4                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 52                               // DW_AT_decl_line
+; CHECK-NEXT: .b8 2
+; CHECK-NEXT: .b32 4926                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_external
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x1338:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 3389                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 8                                // Abbrev [8] 0x133e:0x5 DW_TAG_pointer_type
+; CHECK-NEXT: .b32 3399                            // DW_AT_type
+; CHECK-NEXT: .b8 16                               // Abbrev [16] 0x1343:0x15 DW_TAG_subprogram
+; CHECK-NEXT: .b8 108,97,98,115                    // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 4                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 8                                // DW_AT_decl_line
+; CHECK-NEXT: .b8 3
+; CHECK-NEXT: .b32 2917                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_external
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x1352:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 2917                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 16                               // Abbrev [16] 0x1358:0x1a DW_TAG_subprogram
+; CHECK-NEXT: .b8 108,100,105,118                  // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 4                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 23                               // DW_AT_decl_line
+; CHECK-NEXT: .b8 3
+; CHECK-NEXT: .b32 4515                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_external
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x1367:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 2917                            // DW_AT_type
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x136c:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 2917                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 16                               // Abbrev [16] 0x1372:0x17 DW_TAG_subprogram
+; CHECK-NEXT: .b8 109,97,108,108,111,99            // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 4                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 210                              // DW_AT_decl_line
+; CHECK-NEXT: .b8 1
+; CHECK-NEXT: .b32 4730                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_external
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x1383:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 4737                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 16                               // Abbrev [16] 0x1389:0x1b DW_TAG_subprogram
+; CHECK-NEXT: .b8 109,98,108,101,110               // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 4                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 95                               // DW_AT_decl_line
+; CHECK-NEXT: .b8 3
+; CHECK-NEXT: .b32 2332                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_external
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x1399:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 3389                            // DW_AT_type
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x139e:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 4737                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 16                               // Abbrev [16] 0x13a4:0x23 DW_TAG_subprogram
+; CHECK-NEXT: .b8 109,98,115,116,111,119,99,115    // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 4                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 106                              // DW_AT_decl_line
+; CHECK-NEXT: .b8 3
+; CHECK-NEXT: .b32 4737                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_external
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x13b7:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 5063                            // DW_AT_type
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x13bc:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 3389                            // DW_AT_type
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x13c1:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 4737                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 8                                // Abbrev [8] 0x13c7:0x5 DW_TAG_pointer_type
+; CHECK-NEXT: .b32 5068                            // DW_AT_type
+; CHECK-NEXT: .b8 7                                // Abbrev [7] 0x13cc:0xb DW_TAG_base_type
+; CHECK-NEXT: .b8 119,99,104,97,114,95,116         // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 5                                // DW_AT_encoding
+; CHECK-NEXT: .b8 4                                // DW_AT_byte_size
+; CHECK-NEXT: .b8 16                               // Abbrev [16] 0x13d7:0x21 DW_TAG_subprogram
+; CHECK-NEXT: .b8 109,98,116,111,119,99            // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 4                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 98                               // DW_AT_decl_line
+; CHECK-NEXT: .b8 3
+; CHECK-NEXT: .b32 2332                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_external
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x13e8:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 5063                            // DW_AT_type
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x13ed:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 3389                            // DW_AT_type
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x13f2:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 4737                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 23                               // Abbrev [23] 0x13f8:0x21 DW_TAG_subprogram
+; CHECK-NEXT: .b8 113,115,111,114,116              // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 4                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 253                              // DW_AT_decl_line
+; CHECK-NEXT: .b8 2
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_external
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x1404:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 4730                            // DW_AT_type
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x1409:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 4737                            // DW_AT_type
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x140e:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 4737                            // DW_AT_type
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x1413:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 4772                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 24                               // Abbrev [24] 0x1419:0xf DW_TAG_subprogram
+; CHECK-NEXT: .b8 114,97,110,100                   // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 4                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 118                              // DW_AT_decl_line
+; CHECK-NEXT: .b8 1
+; CHECK-NEXT: .b32 2332                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_external
+; CHECK-NEXT: .b8 16                               // Abbrev [16] 0x1428:0x1d DW_TAG_subprogram
+; CHECK-NEXT: .b8 114,101,97,108,108,111,99        // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 4                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 224                              // DW_AT_decl_line
+; CHECK-NEXT: .b8 1
+; CHECK-NEXT: .b32 4730                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_external
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x143a:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 4730                            // DW_AT_type
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x143f:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 4737                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 23                               // Abbrev [23] 0x1445:0x12 DW_TAG_subprogram
+; CHECK-NEXT: .b8 115,114,97,110,100               // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 4                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 120                              // DW_AT_decl_line
+; CHECK-NEXT: .b8 1
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_external
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x1451:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 5207                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 7                                // Abbrev [7] 0x1457:0x10 DW_TAG_base_type
+; CHECK-NEXT: .b8 117,110,115,105,103,110,101,100,32,105,110,116 // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 7                                // DW_AT_encoding
+; CHECK-NEXT: .b8 4                                // DW_AT_byte_size
+; CHECK-NEXT: .b8 10                               // Abbrev [10] 0x1467:0x1b DW_TAG_subprogram
+; CHECK-NEXT: .b8 115,116,114,116,111,100          // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 4                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 164                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 3379                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_external
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x1477:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 3389                            // DW_AT_type
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x147c:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 5250                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 8                                // Abbrev [8] 0x1482:0x5 DW_TAG_pointer_type
+; CHECK-NEXT: .b32 4926                            // DW_AT_type
+; CHECK-NEXT: .b8 10                               // Abbrev [10] 0x1487:0x20 DW_TAG_subprogram
+; CHECK-NEXT: .b8 115,116,114,116,111,108          // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 4                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 183                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 2917                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_external
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x1497:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 3389                            // DW_AT_type
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x149c:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 5250                            // DW_AT_type
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x14a1:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 2332                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 10                               // Abbrev [10] 0x14a7:0x21 DW_TAG_subprogram
+; CHECK-NEXT: .b8 115,116,114,116,111,117,108      // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 4                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 187                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 4751                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_external
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x14b8:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 3389                            // DW_AT_type
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x14bd:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 5250                            // DW_AT_type
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x14c2:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 2332                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 16                               // Abbrev [16] 0x14c8:0x17 DW_TAG_subprogram
+; CHECK-NEXT: .b8 115,121,115,116,101,109          // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 4                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 205                              // DW_AT_decl_line
+; CHECK-NEXT: .b8 2
+; CHECK-NEXT: .b32 2332                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_external
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x14d9:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 3389                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 16                               // Abbrev [16] 0x14df:0x23 DW_TAG_subprogram
+; CHECK-NEXT: .b8 119,99,115,116,111,109,98,115    // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 4                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 109                              // DW_AT_decl_line
+; CHECK-NEXT: .b8 3
+; CHECK-NEXT: .b32 4737                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_external
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x14f2:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 4926                            // DW_AT_type
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x14f7:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 5378                            // DW_AT_type
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x14fc:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 4737                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 8                                // Abbrev [8] 0x1502:0x5 DW_TAG_pointer_type
+; CHECK-NEXT: .b32 5383                            // DW_AT_type
+; CHECK-NEXT: .b8 9                                // Abbrev [9] 0x1507:0x5 DW_TAG_const_type
+; CHECK-NEXT: .b32 5068                            // DW_AT_type
+; CHECK-NEXT: .b8 16                               // Abbrev [16] 0x150c:0x1c DW_TAG_subprogram
+; CHECK-NEXT: .b8 119,99,116,111,109,98            // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 4                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 102                              // DW_AT_decl_line
+; CHECK-NEXT: .b8 3
+; CHECK-NEXT: .b32 2332                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_external
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x151d:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 4926                            // DW_AT_type
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x1522:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 5068                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 2                                // Abbrev [2] 0x1528:0x78 DW_TAG_namespace
+; CHECK-NEXT: .b8 95,95,103,110,117,95,99,120,120  // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 3                                // Abbrev [3] 0x1533:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 5                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 201                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 5536                            // DW_AT_import
+; CHECK-NEXT: .b8 3                                // Abbrev [3] 0x153a:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 5                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 207                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 5585                            // DW_AT_import
+; CHECK-NEXT: .b8 3                                // Abbrev [3] 0x1541:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 5                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 211                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 5604                            // DW_AT_import
+; CHECK-NEXT: .b8 3                                // Abbrev [3] 0x1548:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 5                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 217                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 5626                            // DW_AT_import
+; CHECK-NEXT: .b8 3                                // Abbrev [3] 0x154f:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 5                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 228                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 5653                            // DW_AT_import
+; CHECK-NEXT: .b8 3                                // Abbrev [3] 0x1556:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 5                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 229                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 5675                            // DW_AT_import
+; CHECK-NEXT: .b8 3                                // Abbrev [3] 0x155d:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 5                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 230                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 5708                            // DW_AT_import
+; CHECK-NEXT: .b8 3                                // Abbrev [3] 0x1564:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 5                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 232                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 5768                            // DW_AT_import
+; CHECK-NEXT: .b8 3                                // Abbrev [3] 0x156b:0x7 DW_TAG_imported_declaration
+; CHECK-NEXT: .b8 5                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 233                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 5795                            // DW_AT_import
+; CHECK-NEXT: .b8 25                               // Abbrev [25] 0x1572:0x2d DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,78,57,95,95,103,110,117,95,99,120,120,51,100,105,118,69,120,120 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 100,105,118                      // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 5                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 214                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 5536                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_external
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x1594:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1508                            // DW_AT_type
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x1599:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1508                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 11                               // Abbrev [11] 0x15a0:0xf DW_TAG_typedef
+; CHECK-NEXT: .b32 5551                            // DW_AT_type
+; CHECK-NEXT: .b8 108,108,100,105,118,95,116       // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 4                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 121                              // DW_AT_decl_line
+; CHECK-NEXT: .b8 13                               // Abbrev [13] 0x15af:0x22 DW_TAG_structure_type
+; CHECK-NEXT: .b8 16                               // DW_AT_byte_size
+; CHECK-NEXT: .b8 4                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 117                              // DW_AT_decl_line
+; CHECK-NEXT: .b8 14                               // Abbrev [14] 0x15b3:0xf DW_TAG_member
+; CHECK-NEXT: .b8 113,117,111,116                  // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b32 1508                            // DW_AT_type
+; CHECK-NEXT: .b8 4                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 119                              // DW_AT_decl_line
+; CHECK-NEXT: .b8 2                                // DW_AT_data_member_location
+; CHECK-NEXT: .b8 35
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 14                               // Abbrev [14] 0x15c2:0xe DW_TAG_member
+; CHECK-NEXT: .b8 114,101,109                      // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b32 1508                            // DW_AT_type
+; CHECK-NEXT: .b8 4                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 120                              // DW_AT_decl_line
+; CHECK-NEXT: .b8 2                                // DW_AT_data_member_location
+; CHECK-NEXT: .b8 35
+; CHECK-NEXT: .b8 8
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 22                               // Abbrev [22] 0x15d1:0x13 DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,69,120,105,116                // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 4                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 45                               // DW_AT_decl_line
+; CHECK-NEXT: .b8 2
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_external
+; CHECK-NEXT: .b8 1                                // DW_AT_noreturn
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x15de:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 2332                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 16                               // Abbrev [16] 0x15e4:0x16 DW_TAG_subprogram
+; CHECK-NEXT: .b8 108,108,97,98,115                // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 4                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 12                               // DW_AT_decl_line
+; CHECK-NEXT: .b8 3
+; CHECK-NEXT: .b32 1508                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_external
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x15f4:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1508                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 16                               // Abbrev [16] 0x15fa:0x1b DW_TAG_subprogram
+; CHECK-NEXT: .b8 108,108,100,105,118              // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 4                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 29                               // DW_AT_decl_line
+; CHECK-NEXT: .b8 3
+; CHECK-NEXT: .b32 5536                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_external
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x160a:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1508                            // DW_AT_type
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x160f:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1508                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 16                               // Abbrev [16] 0x1615:0x16 DW_TAG_subprogram
+; CHECK-NEXT: .b8 97,116,111,108,108               // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 4                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 36                               // DW_AT_decl_line
+; CHECK-NEXT: .b8 1
+; CHECK-NEXT: .b32 1508                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_external
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x1625:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 3389                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 10                               // Abbrev [10] 0x162b:0x21 DW_TAG_subprogram
+; CHECK-NEXT: .b8 115,116,114,116,111,108,108      // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 4                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 209                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 1508                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_external
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x163c:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 3389                            // DW_AT_type
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x1641:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 5250                            // DW_AT_type
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x1646:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 2332                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 10                               // Abbrev [10] 0x164c:0x22 DW_TAG_subprogram
+; CHECK-NEXT: .b8 115,116,114,116,111,117,108,108  // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 4                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 214                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 5742                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_external
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x165e:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 3389                            // DW_AT_type
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x1663:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 5250                            // DW_AT_type
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x1668:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 2332                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 7                                // Abbrev [7] 0x166e:0x1a DW_TAG_base_type
+; CHECK-NEXT: .b8 108,111,110,103,32,108,111,110,103,32,117,110,115,105,103,110,101,100,32,105,110,116 // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 7                                // DW_AT_encoding
+; CHECK-NEXT: .b8 8                                // DW_AT_byte_size
+; CHECK-NEXT: .b8 10                               // Abbrev [10] 0x1688:0x1b DW_TAG_subprogram
+; CHECK-NEXT: .b8 115,116,114,116,111,102          // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 4                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 172                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_external
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x1698:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 3389                            // DW_AT_type
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x169d:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 5250                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 10                               // Abbrev [10] 0x16a3:0x1c DW_TAG_subprogram
+; CHECK-NEXT: .b8 115,116,114,116,111,108,100      // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 4                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 175                              // DW_AT_decl_line
+; CHECK-NEXT: .b32 5823                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_external
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x16b4:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 3389                            // DW_AT_type
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x16b9:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 5250                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 7                                // Abbrev [7] 0x16bf:0xf DW_TAG_base_type
+; CHECK-NEXT: .b8 108,111,110,103,32,100,111,117,98,108,101 // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 4                                // DW_AT_encoding
+; CHECK-NEXT: .b8 8                                // DW_AT_byte_size
+; CHECK-NEXT: .b8 26                               // Abbrev [26] 0x16ce:0x20 DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,53,97,99,111,115,102,102 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 97,99,111,115,102                // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 9                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 62                               // DW_AT_decl_line
+; CHECK-NEXT: .b8 5
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x16e8:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 26                               // Abbrev [26] 0x16ee:0x22 DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,54,97,99,111,115,104,102,102 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 97,99,111,115,104,102            // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 9                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 90                               // DW_AT_decl_line
+; CHECK-NEXT: .b8 5
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x170a:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 26                               // Abbrev [26] 0x1710:0x20 DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,53,97,115,105,110,102,102 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 97,115,105,110,102               // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 9                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 57                               // DW_AT_decl_line
+; CHECK-NEXT: .b8 5
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x172a:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 26                               // Abbrev [26] 0x1730:0x22 DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,54,97,115,105,110,104,102,102 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 97,115,105,110,104,102           // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 9                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 95                               // DW_AT_decl_line
+; CHECK-NEXT: .b8 5
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x174c:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 26                               // Abbrev [26] 0x1752:0x28 DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,54,97,116,97,110,50,102,102,102 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 97,116,97,110,50,102             // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 9                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 47                               // DW_AT_decl_line
+; CHECK-NEXT: .b8 5
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x176f:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x1774:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 26                               // Abbrev [26] 0x177a:0x20 DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,53,97,116,97,110,102,102 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 97,116,97,110,102                // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 9                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 52                               // DW_AT_decl_line
+; CHECK-NEXT: .b8 5
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x1794:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 26                               // Abbrev [26] 0x179a:0x22 DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,54,97,116,97,110,104,102,102 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 97,116,97,110,104,102            // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 9                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 100                              // DW_AT_decl_line
+; CHECK-NEXT: .b8 5
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x17b6:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 26                               // Abbrev [26] 0x17bc:0x20 DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,53,99,98,114,116,102,102 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 99,98,114,116,102                // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 9                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 150                              // DW_AT_decl_line
+; CHECK-NEXT: .b8 5
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x17d6:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 26                               // Abbrev [26] 0x17dc:0x20 DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,53,99,101,105,108,102,102 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 99,101,105,108,102               // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 11                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 155                              // DW_AT_decl_line
+; CHECK-NEXT: .b8 2
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x17f6:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 26                               // Abbrev [26] 0x17fc:0x2e DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,57,99,111,112,121,115,105,103,110,102,102,102 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 99,111,112,121,115,105,103,110,102 // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 9                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 165                              // DW_AT_decl_line
+; CHECK-NEXT: .b8 4
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x181f:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x1824:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 26                               // Abbrev [26] 0x182a:0x1e DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,52,99,111,115,102,102   // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 99,111,115,102                   // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 9                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 219                              // DW_AT_decl_line
+; CHECK-NEXT: .b8 4
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x1842:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 26                               // Abbrev [26] 0x1848:0x20 DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,53,99,111,115,104,102,102 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 99,111,115,104,102               // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 9                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 32                               // DW_AT_decl_line
+; CHECK-NEXT: .b8 5
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x1862:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 26                               // Abbrev [26] 0x1868:0x20 DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,53,101,114,102,99,102,102 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 101,114,102,99,102               // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 9                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 210                              // DW_AT_decl_line
+; CHECK-NEXT: .b8 5
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x1882:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 26                               // Abbrev [26] 0x1888:0x1e DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,52,101,114,102,102,102  // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 101,114,102,102                  // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 9                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 200                              // DW_AT_decl_line
+; CHECK-NEXT: .b8 5
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x18a0:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 26                               // Abbrev [26] 0x18a6:0x20 DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,53,101,120,112,50,102,102 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 101,120,112,50,102               // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 11                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 145                              // DW_AT_decl_line
+; CHECK-NEXT: .b8 2
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x18c0:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 26                               // Abbrev [26] 0x18c6:0x1e DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,52,101,120,112,102,102  // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 101,120,112,102                  // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 9                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 14                               // DW_AT_decl_line
+; CHECK-NEXT: .b8 5
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x18de:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 26                               // Abbrev [26] 0x18e4:0x22 DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,54,101,120,112,109,49,102,102 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 101,120,112,109,49,102           // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 9                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 105                              // DW_AT_decl_line
+; CHECK-NEXT: .b8 5
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x1900:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 26                               // Abbrev [26] 0x1906:0x20 DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,53,102,97,98,115,102,102 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 102,97,98,115,102                // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 11                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 95                               // DW_AT_decl_line
+; CHECK-NEXT: .b8 2
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x1920:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 26                               // Abbrev [26] 0x1926:0x26 DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,53,102,100,105,109,102,102,102 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 102,100,105,109,102              // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 9                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 80                               // DW_AT_decl_line
+; CHECK-NEXT: .b8 6
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x1941:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x1946:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 26                               // Abbrev [26] 0x194c:0x22 DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,54,102,108,111,111,114,102,102 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 102,108,111,111,114,102          // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 11                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 85                               // DW_AT_decl_line
+; CHECK-NEXT: .b8 2
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x1968:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 26                               // Abbrev [26] 0x196e:0x2a DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,52,102,109,97,102,102,102,102 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 102,109,97,102                   // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 9                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 32                               // DW_AT_decl_line
+; CHECK-NEXT: .b8 6
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x1988:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x198d:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x1992:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 26                               // Abbrev [26] 0x1998:0x26 DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,53,102,109,97,120,102,102,102 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 102,109,97,120,102               // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 11                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 110                              // DW_AT_decl_line
+; CHECK-NEXT: .b8 2
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x19b3:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x19b8:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 26                               // Abbrev [26] 0x19be:0x26 DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,53,102,109,105,110,102,102,102 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 102,109,105,110,102              // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 11                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 105                              // DW_AT_decl_line
+; CHECK-NEXT: .b8 2
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x19d9:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x19de:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 26                               // Abbrev [26] 0x19e4:0x26 DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,53,102,109,111,100,102,102,102 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 102,109,111,100,102              // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 9                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 17                               // DW_AT_decl_line
+; CHECK-NEXT: .b8 6
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x19ff:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x1a04:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 26                               // Abbrev [26] 0x1a0a:0x29 DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,54,102,114,101,120,112,102,102,80,105 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 102,114,101,120,112,102          // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 9                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 7                                // DW_AT_decl_line
+; CHECK-NEXT: .b8 6
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x1a28:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x1a2d:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 2377                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 26                               // Abbrev [26] 0x1a33:0x28 DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,54,104,121,112,111,116,102,102,102 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 104,121,112,111,116,102          // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 9                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 110                              // DW_AT_decl_line
+; CHECK-NEXT: .b8 5
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x1a50:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x1a55:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 26                               // Abbrev [26] 0x1a5b:0x22 DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,54,105,108,111,103,98,102,102 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 105,108,111,103,98,102           // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 9                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 85                               // DW_AT_decl_line
+; CHECK-NEXT: .b8 6
+; CHECK-NEXT: .b32 2332                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x1a77:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 26                               // Abbrev [26] 0x1a7d:0x28 DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,54,108,100,101,120,112,102,102,105 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 108,100,101,120,112,102          // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 9                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 240                              // DW_AT_decl_line
+; CHECK-NEXT: .b8 5
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x1a9a:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x1a9f:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 2332                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 26                               // Abbrev [26] 0x1aa5:0x24 DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,55,108,103,97,109,109,97,102,102 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 108,103,97,109,109,97,102        // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 9                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 235                              // DW_AT_decl_line
+; CHECK-NEXT: .b8 5
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x1ac3:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 26                               // Abbrev [26] 0x1ac9:0x24 DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,55,108,108,114,105,110,116,102,102 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 108,108,114,105,110,116,102      // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 9                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 125                              // DW_AT_decl_line
+; CHECK-NEXT: .b8 4
+; CHECK-NEXT: .b32 1508                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x1ae7:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 26                               // Abbrev [26] 0x1aed:0x26 DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,56,108,108,114,111,117,110,100,102,102 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 108,108,114,111,117,110,100,102  // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 9                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 66                               // DW_AT_decl_line
+; CHECK-NEXT: .b8 6
+; CHECK-NEXT: .b32 1508                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x1b0d:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 26                               // Abbrev [26] 0x1b13:0x22 DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,54,108,111,103,49,48,102,102 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 108,111,103,49,48,102            // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 9                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 76                               // DW_AT_decl_line
+; CHECK-NEXT: .b8 5
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x1b2f:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 26                               // Abbrev [26] 0x1b35:0x22 DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,54,108,111,103,49,112,102,102 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 108,111,103,49,112,102           // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 9                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 85                               // DW_AT_decl_line
+; CHECK-NEXT: .b8 5
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x1b51:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 26                               // Abbrev [26] 0x1b57:0x20 DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,53,108,111,103,50,102,102 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 108,111,103,50,102               // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 9                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 5                                // DW_AT_decl_line
+; CHECK-NEXT: .b8 5
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x1b71:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 26                               // Abbrev [26] 0x1b77:0x20 DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,53,108,111,103,98,102,102 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 108,111,103,98,102               // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 9                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 90                               // DW_AT_decl_line
+; CHECK-NEXT: .b8 6
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x1b91:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 26                               // Abbrev [26] 0x1b97:0x1e DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,52,108,111,103,102,102  // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 108,111,103,102                  // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 9                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 67                               // DW_AT_decl_line
+; CHECK-NEXT: .b8 5
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x1baf:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 26                               // Abbrev [26] 0x1bb5:0x22 DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,54,108,114,105,110,116,102,102 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 108,114,105,110,116,102          // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 9                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 116                              // DW_AT_decl_line
+; CHECK-NEXT: .b8 4
+; CHECK-NEXT: .b32 2917                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x1bd1:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 26                               // Abbrev [26] 0x1bd7:0x24 DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,55,108,114,111,117,110,100,102,102 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 108,114,111,117,110,100,102      // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 9                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 71                               // DW_AT_decl_line
+; CHECK-NEXT: .b8 6
+; CHECK-NEXT: .b32 2917                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x1bf5:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 26                               // Abbrev [26] 0x1bfb:0x27 DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,53,109,111,100,102,102,102,80,102 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 109,111,100,102,102              // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 9                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 12                               // DW_AT_decl_line
+; CHECK-NEXT: .b8 6
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x1c17:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x1c1c:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 3345                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 26                               // Abbrev [26] 0x1c22:0x2b DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,49,48,110,101,97,114,98,121,105,110,116,102,102 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 110,101,97,114,98,121,105,110,116,102 // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 9                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 130                              // DW_AT_decl_line
+; CHECK-NEXT: .b8 4
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x1c47:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 26                               // Abbrev [26] 0x1c4d:0x31 DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,49,48,110,101,120,116,97,102,116,101,114,102,102,102 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 110,101,120,116,97,102,116,101,114,102 // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 9                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 194                              // DW_AT_decl_line
+; CHECK-NEXT: .b8 4
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x1c73:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x1c78:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 26                               // Abbrev [26] 0x1c7e:0x24 DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,52,112,111,119,102,102,102 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 112,111,119,102                  // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 9                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 47                               // DW_AT_decl_line
+; CHECK-NEXT: .b8 6
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x1c97:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x1c9c:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 26                               // Abbrev [26] 0x1ca2:0x31 DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,49,48,114,101,109,97,105,110,100,101,114,102,102,102 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 114,101,109,97,105,110,100,101,114,102 // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 9                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 22                               // DW_AT_decl_line
+; CHECK-NEXT: .b8 6
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x1cc8:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x1ccd:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 26                               // Abbrev [26] 0x1cd3:0x31 DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,55,114,101,109,113,117,111,102,102,102,80,105 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 114,101,109,113,117,111,102      // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 9                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 27                               // DW_AT_decl_line
+; CHECK-NEXT: .b8 6
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x1cf4:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x1cf9:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x1cfe:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 2377                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 26                               // Abbrev [26] 0x1d04:0x20 DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,53,114,105,110,116,102,102 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 114,105,110,116,102              // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 9                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 111                              // DW_AT_decl_line
+; CHECK-NEXT: .b8 4
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x1d1e:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 26                               // Abbrev [26] 0x1d24:0x22 DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,54,114,111,117,110,100,102,102 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 114,111,117,110,100,102          // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 9                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 61                               // DW_AT_decl_line
+; CHECK-NEXT: .b8 6
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x1d40:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 26                               // Abbrev [26] 0x1d46:0x2c DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,56,115,99,97,108,98,108,110,102,102,108 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 115,99,97,108,98,108,110,102     // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 9                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 250                              // DW_AT_decl_line
+; CHECK-NEXT: .b8 5
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x1d67:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x1d6c:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 2917                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 26                               // Abbrev [26] 0x1d72:0x2a DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,55,115,99,97,108,98,110,102,102,105 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 115,99,97,108,98,110,102         // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 9                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 245                              // DW_AT_decl_line
+; CHECK-NEXT: .b8 5
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x1d91:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x1d96:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 2332                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 26                               // Abbrev [26] 0x1d9c:0x1e DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,52,115,105,110,102,102  // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 115,105,110,102                  // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 9                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 210                              // DW_AT_decl_line
+; CHECK-NEXT: .b8 4
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x1db4:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 26                               // Abbrev [26] 0x1dba:0x20 DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,53,115,105,110,104,102,102 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 115,105,110,104,102              // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 9                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 37                               // DW_AT_decl_line
+; CHECK-NEXT: .b8 5
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x1dd4:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 26                               // Abbrev [26] 0x1dda:0x20 DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,53,115,113,114,116,102,102 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 115,113,114,116,102              // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 11                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 139                              // DW_AT_decl_line
+; CHECK-NEXT: .b8 3
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x1df4:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 26                               // Abbrev [26] 0x1dfa:0x1e DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,52,116,97,110,102,102   // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 116,97,110,102                   // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 9                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 252                              // DW_AT_decl_line
+; CHECK-NEXT: .b8 4
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x1e12:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 26                               // Abbrev [26] 0x1e18:0x20 DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,53,116,97,110,104,102,102 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 116,97,110,104,102               // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 9                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 42                               // DW_AT_decl_line
+; CHECK-NEXT: .b8 5
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x1e32:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 26                               // Abbrev [26] 0x1e38:0x24 DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,55,116,103,97,109,109,97,102,102 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 116,103,97,109,109,97,102        // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 9                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 56                               // DW_AT_decl_line
+; CHECK-NEXT: .b8 6
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x1e56:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 26                               // Abbrev [26] 0x1e5c:0x22 DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,76,54,116,114,117,110,99,102,102 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 116,114,117,110,99,102           // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 11                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 150                              // DW_AT_decl_line
+; CHECK-NEXT: .b8 2
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x1e78:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 27                               // Abbrev [27] 0x1e7e:0x22a DW_TAG_structure_type
+; CHECK-NEXT: .b8 95,95,99,117,100,97,95,98,117,105,108,116,105,110,95,98,108,111,99,107,73,100,120,95,116 // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 1                                // DW_AT_byte_size
+; CHECK-NEXT: .b8 13                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 77                               // DW_AT_decl_line
+; CHECK-NEXT: .b8 28                               // Abbrev [28] 0x1e9c:0x4f DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,78,50,53,95,95,99,117,100,97,95,98,117,105,108,116,105,110,95,98,108,111,99,107,73,100,120,95,116,49,55,95,95,102,101,116,99,104,95 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 98,117,105,108,116,105,110,95,120,69,118
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 95,95,102,101,116,99,104,95,98,117,105,108,116,105,110,95,120 // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 13                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 78                               // DW_AT_decl_line
+; CHECK-NEXT: .b32 5207                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_external
+; CHECK-NEXT: .b8 28                               // Abbrev [28] 0x1eeb:0x4f DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,78,50,53,95,95,99,117,100,97,95,98,117,105,108,116,105,110,95,98,108,111,99,107,73,100,120,95,116,49,55,95,95,102,101,116,99,104,95 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 98,117,105,108,116,105,110,95,121,69,118
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 95,95,102,101,116,99,104,95,98,117,105,108,116,105,110,95,121 // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 13                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 79                               // DW_AT_decl_line
+; CHECK-NEXT: .b32 5207                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_external
+; CHECK-NEXT: .b8 28                               // Abbrev [28] 0x1f3a:0x4f DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,78,50,53,95,95,99,117,100,97,95,98,117,105,108,116,105,110,95,98,108,111,99,107,73,100,120,95,116,49,55,95,95,102,101,116,99,104,95 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 98,117,105,108,116,105,110,95,122,69,118
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 95,95,102,101,116,99,104,95,98,117,105,108,116,105,110,95,122 // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 13                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 80                               // DW_AT_decl_line
+; CHECK-NEXT: .b32 5207                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_external
+; CHECK-NEXT: .b8 25                               // Abbrev [25] 0x1f89:0x49 DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,78,75,50,53,95,95,99,117,100,97,95,98,117,105,108,116,105,110,95,98,108,111,99,107,73,100,120,95,116,99,118,53,117,105,110,116,51,69 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 118
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 111,112,101,114,97,116,111,114,32,117,105,110,116,51 // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 13                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 83                               // DW_AT_decl_line
+; CHECK-NEXT: .b32 8360                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_external
+; CHECK-NEXT: .b8 29                               // Abbrev [29] 0x1fcb:0x6 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 8407                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_artificial
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 30                               // Abbrev [30] 0x1fd2:0x27 DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,95,99,117,100,97,95,98,117,105,108,116,105,110,95,98,108,111,99,107,73,100,120,95,116 // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 13                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 85                               // DW_AT_decl_line
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_external
+; CHECK-NEXT: .b8 3                                // DW_AT_accessibility
+; CHECK-NEXT:                                      // DW_ACCESS_private
+; CHECK-NEXT: .b8 29                               // Abbrev [29] 0x1ff2:0x6 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 8417                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_artificial
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 30                               // Abbrev [30] 0x1ff9:0x2c DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,95,99,117,100,97,95,98,117,105,108,116,105,110,95,98,108,111,99,107,73,100,120,95,116 // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 13                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 85                               // DW_AT_decl_line
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_external
+; CHECK-NEXT: .b8 3                                // DW_AT_accessibility
+; CHECK-NEXT:                                      // DW_ACCESS_private
+; CHECK-NEXT: .b8 29                               // Abbrev [29] 0x2019:0x6 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 8417                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_artificial
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x201f:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 8422                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 31                               // Abbrev [31] 0x2025:0x43 DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,78,75,50,53,95,95,99,117,100,97,95,98,117,105,108,116,105,110,95,98,108,111,99,107,73,100,120,95,116,97,83,69,82,75,83,95 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 111,112,101,114,97,116,111,114,61 // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 13                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 85                               // DW_AT_decl_line
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_external
+; CHECK-NEXT: .b8 3                                // DW_AT_accessibility
+; CHECK-NEXT:                                      // DW_ACCESS_private
+; CHECK-NEXT: .b8 29                               // Abbrev [29] 0x205c:0x6 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 8407                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_artificial
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x2062:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 8422                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 32                               // Abbrev [32] 0x2068:0x3f DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,78,75,50,53,95,95,99,117,100,97,95,98,117,105,108,116,105,110,95,98,108,111,99,107,73,100,120,95,116,97,100,69,118 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 111,112,101,114,97,116,111,114,38 // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 13                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 85                               // DW_AT_decl_line
+; CHECK-NEXT: .b32 8427                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_external
+; CHECK-NEXT: .b8 3                                // DW_AT_accessibility
+; CHECK-NEXT:                                      // DW_ACCESS_private
+; CHECK-NEXT: .b8 29                               // Abbrev [29] 0x20a0:0x6 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 8407                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_artificial
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 27                               // Abbrev [27] 0x20a8:0x2f DW_TAG_structure_type
+; CHECK-NEXT: .b8 117,105,110,116,51               // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 12                               // DW_AT_byte_size
+; CHECK-NEXT: .b8 14                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 190                              // DW_AT_decl_line
+; CHECK-NEXT: .b8 14                               // Abbrev [14] 0x20b2:0xc DW_TAG_member
+; CHECK-NEXT: .b8 120                              // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b32 5207                            // DW_AT_type
+; CHECK-NEXT: .b8 14                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 192                              // DW_AT_decl_line
+; CHECK-NEXT: .b8 2                                // DW_AT_data_member_location
+; CHECK-NEXT: .b8 35
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 14                               // Abbrev [14] 0x20be:0xc DW_TAG_member
+; CHECK-NEXT: .b8 121                              // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b32 5207                            // DW_AT_type
+; CHECK-NEXT: .b8 14                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 192                              // DW_AT_decl_line
+; CHECK-NEXT: .b8 2                                // DW_AT_data_member_location
+; CHECK-NEXT: .b8 35
+; CHECK-NEXT: .b8 4
+; CHECK-NEXT: .b8 14                               // Abbrev [14] 0x20ca:0xc DW_TAG_member
+; CHECK-NEXT: .b8 122                              // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b32 5207                            // DW_AT_type
+; CHECK-NEXT: .b8 14                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 192                              // DW_AT_decl_line
+; CHECK-NEXT: .b8 2                                // DW_AT_data_member_location
+; CHECK-NEXT: .b8 35
+; CHECK-NEXT: .b8 8
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 8                                // Abbrev [8] 0x20d7:0x5 DW_TAG_pointer_type
+; CHECK-NEXT: .b32 8412                            // DW_AT_type
+; CHECK-NEXT: .b8 9                                // Abbrev [9] 0x20dc:0x5 DW_TAG_const_type
+; CHECK-NEXT: .b32 7806                            // DW_AT_type
+; CHECK-NEXT: .b8 8                                // Abbrev [8] 0x20e1:0x5 DW_TAG_pointer_type
+; CHECK-NEXT: .b32 7806                            // DW_AT_type
+; CHECK-NEXT: .b8 33                               // Abbrev [33] 0x20e6:0x5 DW_TAG_reference_type
+; CHECK-NEXT: .b32 8412                            // DW_AT_type
+; CHECK-NEXT: .b8 8                                // Abbrev [8] 0x20eb:0x5 DW_TAG_pointer_type
+; CHECK-NEXT: .b32 7806                            // DW_AT_type
+; CHECK-NEXT: .b8 34                               // Abbrev [34] 0x20f0:0x6 DW_TAG_subprogram
+; CHECK-NEXT: .b32 7836                            // DW_AT_specification
+; CHECK-NEXT: .b8 1                                // DW_AT_inline
+; CHECK-NEXT: .b8 27                               // Abbrev [27] 0x20f6:0x228 DW_TAG_structure_type
+; CHECK-NEXT: .b8 95,95,99,117,100,97,95,98,117,105,108,116,105,110,95,98,108,111,99,107,68,105,109,95,116 // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 1                                // DW_AT_byte_size
+; CHECK-NEXT: .b8 13                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 88                               // DW_AT_decl_line
+; CHECK-NEXT: .b8 28                               // Abbrev [28] 0x2114:0x4f DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,78,50,53,95,95,99,117,100,97,95,98,117,105,108,116,105,110,95,98,108,111,99,107,68,105,109,95,116,49,55,95,95,102,101,116,99,104,95 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 98,117,105,108,116,105,110,95,120,69,118
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 95,95,102,101,116,99,104,95,98,117,105,108,116,105,110,95,120 // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 13                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 89                               // DW_AT_decl_line
+; CHECK-NEXT: .b32 5207                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_external
+; CHECK-NEXT: .b8 28                               // Abbrev [28] 0x2163:0x4f DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,78,50,53,95,95,99,117,100,97,95,98,117,105,108,116,105,110,95,98,108,111,99,107,68,105,109,95,116,49,55,95,95,102,101,116,99,104,95 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 98,117,105,108,116,105,110,95,121,69,118
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 95,95,102,101,116,99,104,95,98,117,105,108,116,105,110,95,121 // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 13                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 90                               // DW_AT_decl_line
+; CHECK-NEXT: .b32 5207                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_external
+; CHECK-NEXT: .b8 28                               // Abbrev [28] 0x21b2:0x4f DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,78,50,53,95,95,99,117,100,97,95,98,117,105,108,116,105,110,95,98,108,111,99,107,68,105,109,95,116,49,55,95,95,102,101,116,99,104,95 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 98,117,105,108,116,105,110,95,122,69,118
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 95,95,102,101,116,99,104,95,98,117,105,108,116,105,110,95,122 // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 13                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 91                               // DW_AT_decl_line
+; CHECK-NEXT: .b32 5207                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_external
+; CHECK-NEXT: .b8 25                               // Abbrev [25] 0x2201:0x47 DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,78,75,50,53,95,95,99,117,100,97,95,98,117,105,108,116,105,110,95,98,108,111,99,107,68,105,109,95,116,99,118,52,100,105,109,51,69,118 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 111,112,101,114,97,116,111,114,32,100,105,109,51 // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 13                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 94                               // DW_AT_decl_line
+; CHECK-NEXT: .b32 8990                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_external
+; CHECK-NEXT: .b8 29                               // Abbrev [29] 0x2241:0x6 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 9166                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_artificial
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 30                               // Abbrev [30] 0x2248:0x27 DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,95,99,117,100,97,95,98,117,105,108,116,105,110,95,98,108,111,99,107,68,105,109,95,116 // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 13                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 96                               // DW_AT_decl_line
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_external
+; CHECK-NEXT: .b8 3                                // DW_AT_accessibility
+; CHECK-NEXT:                                      // DW_ACCESS_private
+; CHECK-NEXT: .b8 29                               // Abbrev [29] 0x2268:0x6 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 9176                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_artificial
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 30                               // Abbrev [30] 0x226f:0x2c DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,95,99,117,100,97,95,98,117,105,108,116,105,110,95,98,108,111,99,107,68,105,109,95,116 // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 13                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 96                               // DW_AT_decl_line
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_external
+; CHECK-NEXT: .b8 3                                // DW_AT_accessibility
+; CHECK-NEXT:                                      // DW_ACCESS_private
+; CHECK-NEXT: .b8 29                               // Abbrev [29] 0x228f:0x6 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 9176                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_artificial
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x2295:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 9181                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 31                               // Abbrev [31] 0x229b:0x43 DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,78,75,50,53,95,95,99,117,100,97,95,98,117,105,108,116,105,110,95,98,108,111,99,107,68,105,109,95,116,97,83,69,82,75,83,95 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 111,112,101,114,97,116,111,114,61 // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 13                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 96                               // DW_AT_decl_line
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_external
+; CHECK-NEXT: .b8 3                                // DW_AT_accessibility
+; CHECK-NEXT:                                      // DW_ACCESS_private
+; CHECK-NEXT: .b8 29                               // Abbrev [29] 0x22d2:0x6 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 9166                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_artificial
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x22d8:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 9181                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 32                               // Abbrev [32] 0x22de:0x3f DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,78,75,50,53,95,95,99,117,100,97,95,98,117,105,108,116,105,110,95,98,108,111,99,107,68,105,109,95,116,97,100,69,118 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 111,112,101,114,97,116,111,114,38 // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 13                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 96                               // DW_AT_decl_line
+; CHECK-NEXT: .b32 9186                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_external
+; CHECK-NEXT: .b8 3                                // DW_AT_accessibility
+; CHECK-NEXT:                                      // DW_ACCESS_private
+; CHECK-NEXT: .b8 29                               // Abbrev [29] 0x2316:0x6 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 9166                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_artificial
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 35                               // Abbrev [35] 0x231e:0x9d DW_TAG_structure_type
+; CHECK-NEXT: .b8 100,105,109,51                   // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 12                               // DW_AT_byte_size
+; CHECK-NEXT: .b8 14                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 161                              // DW_AT_decl_line
+; CHECK-NEXT: .b8 1
+; CHECK-NEXT: .b8 36                               // Abbrev [36] 0x2328:0xd DW_TAG_member
+; CHECK-NEXT: .b8 120                              // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b32 5207                            // DW_AT_type
+; CHECK-NEXT: .b8 14                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 163                              // DW_AT_decl_line
+; CHECK-NEXT: .b8 1
+; CHECK-NEXT: .b8 2                                // DW_AT_data_member_location
+; CHECK-NEXT: .b8 35
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 36                               // Abbrev [36] 0x2335:0xd DW_TAG_member
+; CHECK-NEXT: .b8 121                              // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b32 5207                            // DW_AT_type
+; CHECK-NEXT: .b8 14                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 163                              // DW_AT_decl_line
+; CHECK-NEXT: .b8 1
+; CHECK-NEXT: .b8 2                                // DW_AT_data_member_location
+; CHECK-NEXT: .b8 35
+; CHECK-NEXT: .b8 4
+; CHECK-NEXT: .b8 36                               // Abbrev [36] 0x2342:0xd DW_TAG_member
+; CHECK-NEXT: .b8 122                              // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b32 5207                            // DW_AT_type
+; CHECK-NEXT: .b8 14                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 163                              // DW_AT_decl_line
+; CHECK-NEXT: .b8 1
+; CHECK-NEXT: .b8 2                                // DW_AT_data_member_location
+; CHECK-NEXT: .b8 35
+; CHECK-NEXT: .b8 8
+; CHECK-NEXT: .b8 23                               // Abbrev [23] 0x234f:0x21 DW_TAG_subprogram
+; CHECK-NEXT: .b8 100,105,109,51                   // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 14                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 165                              // DW_AT_decl_line
+; CHECK-NEXT: .b8 1
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_external
+; CHECK-NEXT: .b8 29                               // Abbrev [29] 0x235a:0x6 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 9147                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_artificial
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x2360:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 5207                            // DW_AT_type
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x2365:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 5207                            // DW_AT_type
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x236a:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 5207                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 23                               // Abbrev [23] 0x2370:0x17 DW_TAG_subprogram
+; CHECK-NEXT: .b8 100,105,109,51                   // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 14                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 166                              // DW_AT_decl_line
+; CHECK-NEXT: .b8 1
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_external
+; CHECK-NEXT: .b8 29                               // Abbrev [29] 0x237b:0x6 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 9147                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_artificial
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x2381:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 9152                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 37                               // Abbrev [37] 0x2387:0x33 DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,78,52,100,105,109,51,99,118,53,117,105,110,116,51,69,118 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 111,112,101,114,97,116,111,114,32,117,105,110,116,51 // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 14                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 167                              // DW_AT_decl_line
+; CHECK-NEXT: .b8 1
+; CHECK-NEXT: .b32 9152                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_external
+; CHECK-NEXT: .b8 29                               // Abbrev [29] 0x23b3:0x6 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 9147                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_artificial
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 8                                // Abbrev [8] 0x23bb:0x5 DW_TAG_pointer_type
+; CHECK-NEXT: .b32 8990                            // DW_AT_type
+; CHECK-NEXT: .b8 20                               // Abbrev [20] 0x23c0:0xe DW_TAG_typedef
+; CHECK-NEXT: .b32 8360                            // DW_AT_type
+; CHECK-NEXT: .b8 117,105,110,116,51               // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 14                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 127                              // DW_AT_decl_line
+; CHECK-NEXT: .b8 1
+; CHECK-NEXT: .b8 8                                // Abbrev [8] 0x23ce:0x5 DW_TAG_pointer_type
+; CHECK-NEXT: .b32 9171                            // DW_AT_type
+; CHECK-NEXT: .b8 9                                // Abbrev [9] 0x23d3:0x5 DW_TAG_const_type
+; CHECK-NEXT: .b32 8438                            // DW_AT_type
+; CHECK-NEXT: .b8 8                                // Abbrev [8] 0x23d8:0x5 DW_TAG_pointer_type
+; CHECK-NEXT: .b32 8438                            // DW_AT_type
+; CHECK-NEXT: .b8 33                               // Abbrev [33] 0x23dd:0x5 DW_TAG_reference_type
+; CHECK-NEXT: .b32 9171                            // DW_AT_type
+; CHECK-NEXT: .b8 8                                // Abbrev [8] 0x23e2:0x5 DW_TAG_pointer_type
+; CHECK-NEXT: .b32 8438                            // DW_AT_type
+; CHECK-NEXT: .b8 34                               // Abbrev [34] 0x23e7:0x6 DW_TAG_subprogram
+; CHECK-NEXT: .b32 8468                            // DW_AT_specification
+; CHECK-NEXT: .b8 1                                // DW_AT_inline
+; CHECK-NEXT: .b8 27                               // Abbrev [27] 0x23ed:0x233 DW_TAG_structure_type
+; CHECK-NEXT: .b8 95,95,99,117,100,97,95,98,117,105,108,116,105,110,95,116,104,114,101,97,100,73,100,120,95,116 // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 1                                // DW_AT_byte_size
+; CHECK-NEXT: .b8 13                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 66                               // DW_AT_decl_line
+; CHECK-NEXT: .b8 28                               // Abbrev [28] 0x240c:0x50 DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,78,50,54,95,95,99,117,100,97,95,98,117,105,108,116,105,110,95,116,104,114,101,97,100,73,100,120,95,116,49,55,95,95,102,101,116,99,104 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 95,98,117,105,108,116,105,110,95,120,69,118
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 95,95,102,101,116,99,104,95,98,117,105,108,116,105,110,95,120 // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 13                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 67                               // DW_AT_decl_line
+; CHECK-NEXT: .b32 5207                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_external
+; CHECK-NEXT: .b8 28                               // Abbrev [28] 0x245c:0x50 DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,78,50,54,95,95,99,117,100,97,95,98,117,105,108,116,105,110,95,116,104,114,101,97,100,73,100,120,95,116,49,55,95,95,102,101,116,99,104 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 95,98,117,105,108,116,105,110,95,121,69,118
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 95,95,102,101,116,99,104,95,98,117,105,108,116,105,110,95,121 // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 13                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 68                               // DW_AT_decl_line
+; CHECK-NEXT: .b32 5207                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_external
+; CHECK-NEXT: .b8 28                               // Abbrev [28] 0x24ac:0x50 DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,78,50,54,95,95,99,117,100,97,95,98,117,105,108,116,105,110,95,116,104,114,101,97,100,73,100,120,95,116,49,55,95,95,102,101,116,99,104 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 95,98,117,105,108,116,105,110,95,122,69,118
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 95,95,102,101,116,99,104,95,98,117,105,108,116,105,110,95,122 // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 13                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 69                               // DW_AT_decl_line
+; CHECK-NEXT: .b32 5207                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_external
+; CHECK-NEXT: .b8 25                               // Abbrev [25] 0x24fc:0x4a DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,78,75,50,54,95,95,99,117,100,97,95,98,117,105,108,116,105,110,95,116,104,114,101,97,100,73,100,120,95,116,99,118,53,117,105,110,116,51 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 69,118
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 111,112,101,114,97,116,111,114,32,117,105,110,116,51 // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 13                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 72                               // DW_AT_decl_line
+; CHECK-NEXT: .b32 8360                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_external
+; CHECK-NEXT: .b8 29                               // Abbrev [29] 0x253f:0x6 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 9760                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_artificial
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 30                               // Abbrev [30] 0x2546:0x28 DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,95,99,117,100,97,95,98,117,105,108,116,105,110,95,116,104,114,101,97,100,73,100,120,95,116 // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 13                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 74                               // DW_AT_decl_line
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_external
+; CHECK-NEXT: .b8 3                                // DW_AT_accessibility
+; CHECK-NEXT:                                      // DW_ACCESS_private
+; CHECK-NEXT: .b8 29                               // Abbrev [29] 0x2567:0x6 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 9770                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_artificial
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 30                               // Abbrev [30] 0x256e:0x2d DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,95,99,117,100,97,95,98,117,105,108,116,105,110,95,116,104,114,101,97,100,73,100,120,95,116 // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 13                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 74                               // DW_AT_decl_line
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_external
+; CHECK-NEXT: .b8 3                                // DW_AT_accessibility
+; CHECK-NEXT:                                      // DW_ACCESS_private
+; CHECK-NEXT: .b8 29                               // Abbrev [29] 0x258f:0x6 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 9770                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_artificial
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x2595:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 9775                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 31                               // Abbrev [31] 0x259b:0x44 DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,78,75,50,54,95,95,99,117,100,97,95,98,117,105,108,116,105,110,95,116,104,114,101,97,100,73,100,120,95,116,97,83,69,82,75,83,95 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 111,112,101,114,97,116,111,114,61 // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 13                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 74                               // DW_AT_decl_line
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_external
+; CHECK-NEXT: .b8 3                                // DW_AT_accessibility
+; CHECK-NEXT:                                      // DW_ACCESS_private
+; CHECK-NEXT: .b8 29                               // Abbrev [29] 0x25d3:0x6 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 9760                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_artificial
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x25d9:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 9775                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 32                               // Abbrev [32] 0x25df:0x40 DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,78,75,50,54,95,95,99,117,100,97,95,98,117,105,108,116,105,110,95,116,104,114,101,97,100,73,100,120,95,116,97,100,69,118 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 111,112,101,114,97,116,111,114,38 // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 13                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 74                               // DW_AT_decl_line
+; CHECK-NEXT: .b32 9780                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 1                                // DW_AT_external
+; CHECK-NEXT: .b8 3                                // DW_AT_accessibility
+; CHECK-NEXT:                                      // DW_ACCESS_private
+; CHECK-NEXT: .b8 29                               // Abbrev [29] 0x2618:0x6 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 9760                            // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_artificial
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 8                                // Abbrev [8] 0x2620:0x5 DW_TAG_pointer_type
+; CHECK-NEXT: .b32 9765                            // DW_AT_type
+; CHECK-NEXT: .b8 9                                // Abbrev [9] 0x2625:0x5 DW_TAG_const_type
+; CHECK-NEXT: .b32 9197                            // DW_AT_type
+; CHECK-NEXT: .b8 8                                // Abbrev [8] 0x262a:0x5 DW_TAG_pointer_type
+; CHECK-NEXT: .b32 9197                            // DW_AT_type
+; CHECK-NEXT: .b8 33                               // Abbrev [33] 0x262f:0x5 DW_TAG_reference_type
+; CHECK-NEXT: .b32 9765                            // DW_AT_type
+; CHECK-NEXT: .b8 8                                // Abbrev [8] 0x2634:0x5 DW_TAG_pointer_type
+; CHECK-NEXT: .b32 9197                            // DW_AT_type
+; CHECK-NEXT: .b8 34                               // Abbrev [34] 0x2639:0x6 DW_TAG_subprogram
+; CHECK-NEXT: .b32 9228                            // DW_AT_specification
+; CHECK-NEXT: .b8 1                                // DW_AT_inline
+; CHECK-NEXT: .b8 38                               // Abbrev [38] 0x263f:0x32 DW_TAG_subprogram
+; CHECK-NEXT: .b8 95,90,51,114,101,115,102,102,80,102 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 114,101,115                      // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 12                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 3                                // DW_AT_decl_line
+; CHECK-NEXT: .b8 1                                // DW_AT_external
+; CHECK-NEXT: .b8 1                                // DW_AT_inline
+; CHECK-NEXT: .b8 39                               // Abbrev [39] 0x2653:0x9 DW_TAG_formal_parameter
+; CHECK-NEXT: .b8 120                              // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 12                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 3                                // DW_AT_decl_line
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 39                               // Abbrev [39] 0x265c:0x9 DW_TAG_formal_parameter
+; CHECK-NEXT: .b8 121                              // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 12                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 3                                // DW_AT_decl_line
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 39                               // Abbrev [39] 0x2665:0xb DW_TAG_formal_parameter
+; CHECK-NEXT: .b8 114,101,115                      // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 12                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 3                                // DW_AT_decl_line
+; CHECK-NEXT: .b32 3345                            // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 40                               // Abbrev [40] 0x2671:0xc0 DW_TAG_subprogram
+; CHECK-NEXT: .b64 Lfunc_begin0                    // DW_AT_low_pc
+; CHECK-NEXT: .b64 Lfunc_end0                      // DW_AT_high_pc
+; CHECK-NEXT: .b8 1                                // DW_AT_frame_base
+; CHECK-NEXT: .b8 156
+; CHECK-NEXT: .b8 95,90,53,115,97,120,112,121,105,102,80,102,83,95 // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 115,97,120,112,121               // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 12                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 5                                // DW_AT_decl_line
+; CHECK-NEXT: .b8 1                                // DW_AT_external
+; CHECK-NEXT: .b8 39                               // Abbrev [39] 0x269c:0x9 DW_TAG_formal_parameter
+; CHECK-NEXT: .b8 110                              // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 12                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 5                                // DW_AT_decl_line
+; CHECK-NEXT: .b32 2332                            // DW_AT_type
+; CHECK-NEXT: .b8 39                               // Abbrev [39] 0x26a5:0x9 DW_TAG_formal_parameter
+; CHECK-NEXT: .b8 97                               // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 12                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 5                                // DW_AT_decl_line
+; CHECK-NEXT: .b32 1554                            // DW_AT_type
+; CHECK-NEXT: .b8 39                               // Abbrev [39] 0x26ae:0x9 DW_TAG_formal_parameter
+; CHECK-NEXT: .b8 120                              // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 12                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 5                                // DW_AT_decl_line
+; CHECK-NEXT: .b32 3345                            // DW_AT_type
+; CHECK-NEXT: .b8 39                               // Abbrev [39] 0x26b7:0x9 DW_TAG_formal_parameter
+; CHECK-NEXT: .b8 121                              // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 12                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 5                                // DW_AT_decl_line
+; CHECK-NEXT: .b32 3345                            // DW_AT_type
+; CHECK-NEXT: .b8 41                               // Abbrev [41] 0x26c0:0x9 DW_TAG_variable
+; CHECK-NEXT: .b8 105                              // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 12                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 6                                // DW_AT_decl_line
+; CHECK-NEXT: .b32 2332                            // DW_AT_type
+; CHECK-NEXT: .b8 42                               // Abbrev [42] 0x26c9:0x17 DW_TAG_inlined_subroutine
+; CHECK-NEXT: .b32 8432                            // DW_AT_abstract_origin
+; CHECK-NEXT: .b64 Ltmp0                           // DW_AT_low_pc
+; CHECK-NEXT: .b64 Ltmp1                           // DW_AT_high_pc
+; CHECK-NEXT: .b8 12                               // DW_AT_call_file
+; CHECK-NEXT: .b8 6                                // DW_AT_call_line
+; CHECK-NEXT: .b8 42                               // Abbrev [42] 0x26e0:0x17 DW_TAG_inlined_subroutine
+; CHECK-NEXT: .b32 9191                            // DW_AT_abstract_origin
+; CHECK-NEXT: .b64 Ltmp1                           // DW_AT_low_pc
+; CHECK-NEXT: .b64 Ltmp2                           // DW_AT_high_pc
+; CHECK-NEXT: .b8 12                               // DW_AT_call_file
+; CHECK-NEXT: .b8 6                                // DW_AT_call_line
+; CHECK-NEXT: .b8 42                               // Abbrev [42] 0x26f7:0x17 DW_TAG_inlined_subroutine
+; CHECK-NEXT: .b32 9785                            // DW_AT_abstract_origin
+; CHECK-NEXT: .b64 Ltmp2                           // DW_AT_low_pc
+; CHECK-NEXT: .b64 Ltmp3                           // DW_AT_high_pc
+; CHECK-NEXT: .b8 12                               // DW_AT_call_file
+; CHECK-NEXT: .b8 6                                // DW_AT_call_line
+; CHECK-NEXT: .b8 43                               // Abbrev [43] 0x270e:0x22 DW_TAG_inlined_subroutine
+; CHECK-NEXT: .b32 9791                            // DW_AT_abstract_origin
+; CHECK-NEXT: .b64 Ltmp10                          // DW_AT_low_pc
+; CHECK-NEXT: .b64 Ltmp11                          // DW_AT_high_pc
+; CHECK-NEXT: .b8 12                               // DW_AT_call_file
+; CHECK-NEXT: .b8 8                                // DW_AT_call_line
+; CHECK-NEXT: .b8 44                               // Abbrev [44] 0x2725:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 9811                            // DW_AT_abstract_origin
+; CHECK-NEXT: .b8 44                               // Abbrev [44] 0x272a:0x5 DW_TAG_formal_parameter
+; CHECK-NEXT: .b32 9820                            // DW_AT_abstract_origin
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: }
+; CHECK-NEXT: .section .debug_macinfo
+; CHECK-NEXT: {
+; CHECK-NEXT: .b8 0                                // End Of Macro List Mark
+; CHECK:      }
 
 ; Function Attrs: nounwind readnone
 declare i32 @llvm.nvvm.read.ptx.sreg.ctaid.x() #1
diff --git a/test/DebugInfo/NVPTX/debug-loc-offset.ll b/test/DebugInfo/NVPTX/debug-loc-offset.ll
index 91926517bbc6cd826f0b796bc5741171d779fad6..e1366b970ca6663d19a00415054f2a755b9119ae 100644
--- a/test/DebugInfo/NVPTX/debug-loc-offset.ll
+++ b/test/DebugInfo/NVPTX/debug-loc-offset.ll
@@ -1,6 +1,6 @@
 ; RUN: llc -mtriple=nvptx64-nvidia-cuda < %s | FileCheck %s
 
-; CHECK: .target sm_{{[0-9]+}}//, debug
+; CHECK: .target sm_{{[0-9]+}}, debug
 
 ; CHECK: .extern .func  (.param .b32 func_retval0) _ZN1A3fooEv
 ; CHECK: (
@@ -11,8 +11,9 @@
 
 ; CHECK: .visible .func  (.param .b32 func_retval0) _Z3bari(
 ; CHECK: {
-; CHECK: Lfunc_begin0:
 ; CHECK: .loc [[CU1:[0-9]+]] 1 0
+; CHECK: Lfunc_begin0:
+; CHECK: .loc [[CU1]] 1 0
 
 ; CHECK: //DEBUG_VALUE: bar:b <- {{[0-9]+}}
 ; CHECK: //DEBUG_VALUE: bar:b <- {{[0-9]+}}
@@ -39,8 +40,9 @@ declare void @llvm.dbg.value(metadata, metadata, metadata) #1
 
 ; CHECK: .visible .func _Z3baz1A(
 ; CHECK: {
-; CHECK: Lfunc_begin1:
 ; CHECK: .loc [[CU2:[0-9]+]] 6 0
+; CHECK: Lfunc_begin1:
+; CHECK: .loc [[CU2]] 6 0
 ; CHECK: //DEBUG_VALUE: baz:z <- {{[0-9]+}}
 ; CHECK: //DEBUG_VALUE: baz:z <- {{[0-9]+}}
 ; CHECK: .loc [[CU2]] 10 0
@@ -127,207 +129,207 @@ attributes #2 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "n
 !33 = !DILocation(line: 11, scope: !32)
 !34 = !DILocation(line: 12, scope: !14)
 
-; CHECK: // .section .debug_abbrev
-; CHECK: // {
-; CHECK: // .b8 1                                // Abbreviation Code
-; CHECK: // .b8 17                               // DW_TAG_compile_unit
-; CHECK: // .b8 1                                // DW_CHILDREN_yes
-; CHECK: // .b8 37                               // DW_AT_producer
-; CHECK: // .b8 8                                // DW_FORM_string
-; CHECK: // .b8 19                               // DW_AT_language
-; CHECK: // .b8 5                                // DW_FORM_data2
-; CHECK: // .b8 3                                // DW_AT_name
-; CHECK: // .b8 8                                // DW_FORM_string
-; CHECK: // .b8 16                               // DW_AT_stmt_list
-; CHECK: // .b8 6                                // DW_FORM_data4
-; CHECK: // .b8 27                               // DW_AT_comp_dir
-; CHECK: // .b8 8                                // DW_FORM_string
-; CHECK: // .b8 17                               // DW_AT_low_pc
-; CHECK: // .b8 1                                // DW_FORM_addr
-; CHECK: // .b8 18                               // DW_AT_high_pc
-; CHECK: // .b8 1                                // DW_FORM_addr
-; CHECK: // .b8 0                                // EOM(1)
-; CHECK: // .b8 0                                // EOM(2)
-; CHECK: // .b8 2                                // Abbreviation Code
-; CHECK: // .b8 19                               // DW_TAG_structure_type
-; CHECK: // .b8 0                                // DW_CHILDREN_no
-; CHECK: // .b8 3                                // DW_AT_name
-; CHECK: // .b8 8                                // DW_FORM_string
-; CHECK: // .b8 60                               // DW_AT_declaration
-; CHECK: // .b8 12                               // DW_FORM_flag
-; CHECK: // .b8 0                                // EOM(1)
-; CHECK: // .b8 0                                // EOM(2)
-; CHECK: // .b8 3                                // Abbreviation Code
-; CHECK: // .b8 46                               // DW_TAG_subprogram
-; CHECK: // .b8 1                                // DW_CHILDREN_yes
-; CHECK: // .b8 17                               // DW_AT_low_pc
-; CHECK: // .b8 1                                // DW_FORM_addr
-; CHECK: // .b8 18                               // DW_AT_high_pc
-; CHECK: // .b8 1                                // DW_FORM_addr
-; CHECK: // .b8 64                               // DW_AT_frame_base
-; CHECK: // .b8 10                               // DW_FORM_block1
-; CHECK: // .b8 135,64                           // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 8                                // DW_FORM_string
-; CHECK: // .b8 3                                // DW_AT_name
-; CHECK: // .b8 8                                // DW_FORM_string
-; CHECK: // .b8 58                               // DW_AT_decl_file
-; CHECK: // .b8 11                               // DW_FORM_data1
-; CHECK: // .b8 59                               // DW_AT_decl_line
-; CHECK: // .b8 11                               // DW_FORM_data1
-; CHECK: // .b8 63                               // DW_AT_external
-; CHECK: // .b8 12                               // DW_FORM_flag
-; CHECK: // .b8 0                                // EOM(1)
-; CHECK: // .b8 0                                // EOM(2)
-; CHECK: // .b8 4                                // Abbreviation Code
-; CHECK: // .b8 52                               // DW_TAG_variable
-; CHECK: // .b8 0                                // DW_CHILDREN_no
-; CHECK: // .b8 3                                // DW_AT_name
-; CHECK: // .b8 8                                // DW_FORM_string
-; CHECK: // .b8 58                               // DW_AT_decl_file
-; CHECK: // .b8 11                               // DW_FORM_data1
-; CHECK: // .b8 59                               // DW_AT_decl_line
-; CHECK: // .b8 11                               // DW_FORM_data1
-; CHECK: // .b8 73                               // DW_AT_type
-; CHECK: // .b8 16                               // DW_FORM_ref_addr
-; CHECK: // .b8 0                                // EOM(1)
-; CHECK: // .b8 0                                // EOM(2)
-; CHECK: // .b8 5                                // Abbreviation Code
-; CHECK: // .b8 46                               // DW_TAG_subprogram
-; CHECK: // .b8 1                                // DW_CHILDREN_yes
-; CHECK: // .b8 17                               // DW_AT_low_pc
-; CHECK: // .b8 1                                // DW_FORM_addr
-; CHECK: // .b8 18                               // DW_AT_high_pc
-; CHECK: // .b8 1                                // DW_FORM_addr
-; CHECK: // .b8 64                               // DW_AT_frame_base
-; CHECK: // .b8 10                               // DW_FORM_block1
-; CHECK: // .b8 135,64                           // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 8                                // DW_FORM_string
-; CHECK: // .b8 3                                // DW_AT_name
-; CHECK: // .b8 8                                // DW_FORM_string
-; CHECK: // .b8 58                               // DW_AT_decl_file
-; CHECK: // .b8 11                               // DW_FORM_data1
-; CHECK: // .b8 59                               // DW_AT_decl_line
-; CHECK: // .b8 11                               // DW_FORM_data1
-; CHECK: // .b8 73                               // DW_AT_type
-; CHECK: // .b8 19                               // DW_FORM_ref4
-; CHECK: // .b8 63                               // DW_AT_external
-; CHECK: // .b8 12                               // DW_FORM_flag
-; CHECK: // .b8 0                                // EOM(1)
-; CHECK: // .b8 0                                // EOM(2)
-; CHECK: // .b8 6                                // Abbreviation Code
-; CHECK: // .b8 5                                // DW_TAG_formal_parameter
-; CHECK: // .b8 0                                // DW_CHILDREN_no
-; CHECK: // .b8 3                                // DW_AT_name
-; CHECK: // .b8 8                                // DW_FORM_string
-; CHECK: // .b8 58                               // DW_AT_decl_file
-; CHECK: // .b8 11                               // DW_FORM_data1
-; CHECK: // .b8 59                               // DW_AT_decl_line
-; CHECK: // .b8 11                               // DW_FORM_data1
-; CHECK: // .b8 73                               // DW_AT_type
-; CHECK: // .b8 19                               // DW_FORM_ref4
-; CHECK: // .b8 0                                // EOM(1)
-; CHECK: // .b8 0                                // EOM(2)
-; CHECK: // .b8 7                                // Abbreviation Code
-; CHECK: // .b8 36                               // DW_TAG_base_type
-; CHECK: // .b8 0                                // DW_CHILDREN_no
-; CHECK: // .b8 3                                // DW_AT_name
-; CHECK: // .b8 8                                // DW_FORM_string
-; CHECK: // .b8 62                               // DW_AT_encoding
-; CHECK: // .b8 11                               // DW_FORM_data1
-; CHECK: // .b8 11                               // DW_AT_byte_size
-; CHECK: // .b8 11                               // DW_FORM_data1
-; CHECK: // .b8 0                                // EOM(1)
-; CHECK: // .b8 0                                // EOM(2)
-; CHECK: // .b8 0                                // EOM(3)
-; CHECK: // }
-; CHECK: // .section .debug_info
-; CHECK: // {
-; CHECK: // .b32 150                             // Length of Unit
-; CHECK: // .b8 2                                // DWARF version number
-; CHECK: // .b8 0
-; CHECK: // .b32 .debug_abbrev                   // Offset Into Abbrev. Section
-; CHECK: // .b8 8                                // Address Size (in bytes)
-; CHECK: // .b8 1                                // Abbrev [1] 0xb:0x8f DW_TAG_compile_unit
-; CHECK: // .b8 99,108,97,110,103,32,118,101,114,115,105,111,110,32,51,46,53,46,48,32,40,50,49,48,52,55,57,41 // DW_AT_producer
-; CHECK: // .b8 0
-; CHECK: // .b8 4                                // DW_AT_language
-; CHECK: // .b8 0
-; CHECK: // .b8 100,101,98,117,103,45,108,111,99,45,111,102,102,115,101,116,50,46,99,99 // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b32 .debug_line                     // DW_AT_stmt_list
-; CHECK: // .b8 47,108,108,118,109,95,99,109,97,107,101,95,103,99,99 // DW_AT_comp_dir
-; CHECK: // .b8 0
-; CHECK: // .b64 Lfunc_begin1                    // DW_AT_low_pc
-; CHECK: // .b64 Lfunc_end1                      // DW_AT_high_pc
-; CHECK: // .b8 2                                // Abbrev [2] 0x64:0x4 DW_TAG_structure_type
-; CHECK: // .b8 65                               // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 1                                // DW_AT_declaration
-; CHECK: // .b8 3                                // Abbrev [3] 0x68:0x31 DW_TAG_subprogram
-; CHECK: // .b64 Lfunc_begin1                    // DW_AT_low_pc
-; CHECK: // .b64 Lfunc_end1                      // DW_AT_high_pc
-; CHECK: // .b8 1                                // DW_AT_frame_base
-; CHECK: // .b8 156
-; CHECK: // .b8 95,90,51,98,97,122,49,65         // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 98,97,122                        // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 2                                // DW_AT_decl_file
-; CHECK: // .b8 6                                // DW_AT_decl_line
-; CHECK: // .b8 1                                // DW_AT_external
-; CHECK: // .b8 4                                // Abbrev [4] 0x8b:0xd DW_TAG_variable
-; CHECK: // .b8 122                              // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 2                                // DW_AT_decl_file
-; CHECK: // .b8 7                                // DW_AT_decl_line
-; CHECK: // .b64 .debug_info+302                 // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b32 152                             // Length of Unit
-; CHECK: // .b8 2                                // DWARF version number
-; CHECK: // .b8 0
-; CHECK: // .b32 .debug_abbrev                   // Offset Into Abbrev. Section
-; CHECK: // .b8 8                                // Address Size (in bytes)
-; CHECK: // .b8 1                                // Abbrev [1] 0xb:0x91 DW_TAG_compile_unit
-; CHECK: // .b8 99,108,97,110,103,32,118,101,114,115,105,111,110,32,51,46,53,46,48,32,40,50,49,48,52,55,57,41 // DW_AT_producer
-; CHECK: // .b8 0
-; CHECK: // .b8 4                                // DW_AT_language
-; CHECK: // .b8 0
-; CHECK: // .b8 100,101,98,117,103,45,108,111,99,45,111,102,102,115,101,116,49,46,99,99 // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b32 .debug_line                     // DW_AT_stmt_list
-; CHECK: // .b8 47,108,108,118,109,95,99,109,97,107,101,95,103,99,99 // DW_AT_comp_dir
-; CHECK: // .b8 0
-; CHECK: // .b64 Lfunc_begin0                    // DW_AT_low_pc
-; CHECK: // .b64 Lfunc_end0                      // DW_AT_high_pc
-; CHECK: // .b8 5                                // Abbrev [5] 0x64:0x30 DW_TAG_subprogram
-; CHECK: // .b64 Lfunc_begin0                    // DW_AT_low_pc
-; CHECK: // .b64 Lfunc_end0                      // DW_AT_high_pc
-; CHECK: // .b8 1                                // DW_AT_frame_base
-; CHECK: // .b8 156
-; CHECK: // .b8 95,90,51,98,97,114,105           // DW_AT_MIPS_linkage_name
-; CHECK: // .b8 0
-; CHECK: // .b8 98,97,114                        // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 1                                // DW_AT_decl_line
-; CHECK: // .b32 148                             // DW_AT_type
-; CHECK: // .b8 1                                // DW_AT_external
-; CHECK: // .b8 6                                // Abbrev [6] 0x8a:0x9 DW_TAG_formal_parameter
-; CHECK: // .b8 98                               // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 1                                // DW_AT_decl_file
-; CHECK: // .b8 1                                // DW_AT_decl_line
-; CHECK: // .b32 148                             // DW_AT_type
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // .b8 7                                // Abbrev [7] 0x94:0x7 DW_TAG_base_type
-; CHECK: // .b8 105,110,116                      // DW_AT_name
-; CHECK: // .b8 0
-; CHECK: // .b8 5                                // DW_AT_encoding
-; CHECK: // .b8 4                                // DW_AT_byte_size
-; CHECK: // .b8 0                                // End Of Children Mark
-; CHECK: // }
-; CHECK: // .section .debug_macinfo
-; CHECK: // {
-; CHECK: // .b8 0                                // End Of Macro List Mark
-; CHECK: // }
+; CHECK: .section .debug_abbrev
+; CHECK-NEXT: {
+; CHECK-NEXT: .b8 1                                // Abbreviation Code
+; CHECK-NEXT: .b8 17                               // DW_TAG_compile_unit
+; CHECK-NEXT: .b8 1                                // DW_CHILDREN_yes
+; CHECK-NEXT: .b8 37                               // DW_AT_producer
+; CHECK-NEXT: .b8 8                                // DW_FORM_string
+; CHECK-NEXT: .b8 19                               // DW_AT_language
+; CHECK-NEXT: .b8 5                                // DW_FORM_data2
+; CHECK-NEXT: .b8 3                                // DW_AT_name
+; CHECK-NEXT: .b8 8                                // DW_FORM_string
+; CHECK-NEXT: .b8 16                               // DW_AT_stmt_list
+; CHECK-NEXT: .b8 6                                // DW_FORM_data4
+; CHECK-NEXT: .b8 27                               // DW_AT_comp_dir
+; CHECK-NEXT: .b8 8                                // DW_FORM_string
+; CHECK-NEXT: .b8 17                               // DW_AT_low_pc
+; CHECK-NEXT: .b8 1                                // DW_FORM_addr
+; CHECK-NEXT: .b8 18                               // DW_AT_high_pc
+; CHECK-NEXT: .b8 1                                // DW_FORM_addr
+; CHECK-NEXT: .b8 0                                // EOM(1)
+; CHECK-NEXT: .b8 0                                // EOM(2)
+; CHECK-NEXT: .b8 2                                // Abbreviation Code
+; CHECK-NEXT: .b8 19                               // DW_TAG_structure_type
+; CHECK-NEXT: .b8 0                                // DW_CHILDREN_no
+; CHECK-NEXT: .b8 3                                // DW_AT_name
+; CHECK-NEXT: .b8 8                                // DW_FORM_string
+; CHECK-NEXT: .b8 60                               // DW_AT_declaration
+; CHECK-NEXT: .b8 12                               // DW_FORM_flag
+; CHECK-NEXT: .b8 0                                // EOM(1)
+; CHECK-NEXT: .b8 0                                // EOM(2)
+; CHECK-NEXT: .b8 3                                // Abbreviation Code
+; CHECK-NEXT: .b8 46                               // DW_TAG_subprogram
+; CHECK-NEXT: .b8 1                                // DW_CHILDREN_yes
+; CHECK-NEXT: .b8 17                               // DW_AT_low_pc
+; CHECK-NEXT: .b8 1                                // DW_FORM_addr
+; CHECK-NEXT: .b8 18                               // DW_AT_high_pc
+; CHECK-NEXT: .b8 1                                // DW_FORM_addr
+; CHECK-NEXT: .b8 64                               // DW_AT_frame_base
+; CHECK-NEXT: .b8 10                               // DW_FORM_block1
+; CHECK-NEXT: .b8 135,64                           // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 8                                // DW_FORM_string
+; CHECK-NEXT: .b8 3                                // DW_AT_name
+; CHECK-NEXT: .b8 8                                // DW_FORM_string
+; CHECK-NEXT: .b8 58                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 11                               // DW_FORM_data1
+; CHECK-NEXT: .b8 59                               // DW_AT_decl_line
+; CHECK-NEXT: .b8 11                               // DW_FORM_data1
+; CHECK-NEXT: .b8 63                               // DW_AT_external
+; CHECK-NEXT: .b8 12                               // DW_FORM_flag
+; CHECK-NEXT: .b8 0                                // EOM(1)
+; CHECK-NEXT: .b8 0                                // EOM(2)
+; CHECK-NEXT: .b8 4                                // Abbreviation Code
+; CHECK-NEXT: .b8 52                               // DW_TAG_variable
+; CHECK-NEXT: .b8 0                                // DW_CHILDREN_no
+; CHECK-NEXT: .b8 3                                // DW_AT_name
+; CHECK-NEXT: .b8 8                                // DW_FORM_string
+; CHECK-NEXT: .b8 58                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 11                               // DW_FORM_data1
+; CHECK-NEXT: .b8 59                               // DW_AT_decl_line
+; CHECK-NEXT: .b8 11                               // DW_FORM_data1
+; CHECK-NEXT: .b8 73                               // DW_AT_type
+; CHECK-NEXT: .b8 16                               // DW_FORM_ref_addr
+; CHECK-NEXT: .b8 0                                // EOM(1)
+; CHECK-NEXT: .b8 0                                // EOM(2)
+; CHECK-NEXT: .b8 5                                // Abbreviation Code
+; CHECK-NEXT: .b8 46                               // DW_TAG_subprogram
+; CHECK-NEXT: .b8 1                                // DW_CHILDREN_yes
+; CHECK-NEXT: .b8 17                               // DW_AT_low_pc
+; CHECK-NEXT: .b8 1                                // DW_FORM_addr
+; CHECK-NEXT: .b8 18                               // DW_AT_high_pc
+; CHECK-NEXT: .b8 1                                // DW_FORM_addr
+; CHECK-NEXT: .b8 64                               // DW_AT_frame_base
+; CHECK-NEXT: .b8 10                               // DW_FORM_block1
+; CHECK-NEXT: .b8 135,64                           // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 8                                // DW_FORM_string
+; CHECK-NEXT: .b8 3                                // DW_AT_name
+; CHECK-NEXT: .b8 8                                // DW_FORM_string
+; CHECK-NEXT: .b8 58                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 11                               // DW_FORM_data1
+; CHECK-NEXT: .b8 59                               // DW_AT_decl_line
+; CHECK-NEXT: .b8 11                               // DW_FORM_data1
+; CHECK-NEXT: .b8 73                               // DW_AT_type
+; CHECK-NEXT: .b8 19                               // DW_FORM_ref4
+; CHECK-NEXT: .b8 63                               // DW_AT_external
+; CHECK-NEXT: .b8 12                               // DW_FORM_flag
+; CHECK-NEXT: .b8 0                                // EOM(1)
+; CHECK-NEXT: .b8 0                                // EOM(2)
+; CHECK-NEXT: .b8 6                                // Abbreviation Code
+; CHECK-NEXT: .b8 5                                // DW_TAG_formal_parameter
+; CHECK-NEXT: .b8 0                                // DW_CHILDREN_no
+; CHECK-NEXT: .b8 3                                // DW_AT_name
+; CHECK-NEXT: .b8 8                                // DW_FORM_string
+; CHECK-NEXT: .b8 58                               // DW_AT_decl_file
+; CHECK-NEXT: .b8 11                               // DW_FORM_data1
+; CHECK-NEXT: .b8 59                               // DW_AT_decl_line
+; CHECK-NEXT: .b8 11                               // DW_FORM_data1
+; CHECK-NEXT: .b8 73                               // DW_AT_type
+; CHECK-NEXT: .b8 19                               // DW_FORM_ref4
+; CHECK-NEXT: .b8 0                                // EOM(1)
+; CHECK-NEXT: .b8 0                                // EOM(2)
+; CHECK-NEXT: .b8 7                                // Abbreviation Code
+; CHECK-NEXT: .b8 36                               // DW_TAG_base_type
+; CHECK-NEXT: .b8 0                                // DW_CHILDREN_no
+; CHECK-NEXT: .b8 3                                // DW_AT_name
+; CHECK-NEXT: .b8 8                                // DW_FORM_string
+; CHECK-NEXT: .b8 62                               // DW_AT_encoding
+; CHECK-NEXT: .b8 11                               // DW_FORM_data1
+; CHECK-NEXT: .b8 11                               // DW_AT_byte_size
+; CHECK-NEXT: .b8 11                               // DW_FORM_data1
+; CHECK-NEXT: .b8 0                                // EOM(1)
+; CHECK-NEXT: .b8 0                                // EOM(2)
+; CHECK-NEXT: .b8 0                                // EOM(3)
+; CHECK-NEXT: }
+; CHECK-NEXT: .section .debug_info
+; CHECK-NEXT: {
+; CHECK-NEXT: .b32 150                             // Length of Unit
+; CHECK-NEXT: .b8 2                                // DWARF version number
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b32 .debug_abbrev                   // Offset Into Abbrev. Section
+; CHECK-NEXT: .b8 8                                // Address Size (in bytes)
+; CHECK-NEXT: .b8 1                                // Abbrev [1] 0xb:0x8f DW_TAG_compile_unit
+; CHECK-NEXT: .b8 99,108,97,110,103,32,118,101,114,115,105,111,110,32,51,46,53,46,48,32,40,50,49,48,52,55,57,41 // DW_AT_producer
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 4                                // DW_AT_language
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 100,101,98,117,103,45,108,111,99,45,111,102,102,115,101,116,50,46,99,99 // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b32 .debug_line                     // DW_AT_stmt_list
+; CHECK-NEXT: .b8 47,108,108,118,109,95,99,109,97,107,101,95,103,99,99 // DW_AT_comp_dir
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b64 Lfunc_begin1                    // DW_AT_low_pc
+; CHECK-NEXT: .b64 Lfunc_end1                      // DW_AT_high_pc
+; CHECK-NEXT: .b8 2                                // Abbrev [2] 0x64:0x4 DW_TAG_structure_type
+; CHECK-NEXT: .b8 65                               // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 1                                // DW_AT_declaration
+; CHECK-NEXT: .b8 3                                // Abbrev [3] 0x68:0x31 DW_TAG_subprogram
+; CHECK-NEXT: .b64 Lfunc_begin1                    // DW_AT_low_pc
+; CHECK-NEXT: .b64 Lfunc_end1                      // DW_AT_high_pc
+; CHECK-NEXT: .b8 1                                // DW_AT_frame_base
+; CHECK-NEXT: .b8 156
+; CHECK-NEXT: .b8 95,90,51,98,97,122,49,65         // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 98,97,122                        // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 2                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 6                                // DW_AT_decl_line
+; CHECK-NEXT: .b8 1                                // DW_AT_external
+; CHECK-NEXT: .b8 4                                // Abbrev [4] 0x8b:0xd DW_TAG_variable
+; CHECK-NEXT: .b8 122                              // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 2                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 7                                // DW_AT_decl_line
+; CHECK-NEXT: .b64 .debug_info+302                 // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b32 152                             // Length of Unit
+; CHECK-NEXT: .b8 2                                // DWARF version number
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b32 .debug_abbrev                   // Offset Into Abbrev. Section
+; CHECK-NEXT: .b8 8                                // Address Size (in bytes)
+; CHECK-NEXT: .b8 1                                // Abbrev [1] 0xb:0x91 DW_TAG_compile_unit
+; CHECK-NEXT: .b8 99,108,97,110,103,32,118,101,114,115,105,111,110,32,51,46,53,46,48,32,40,50,49,48,52,55,57,41 // DW_AT_producer
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 4                                // DW_AT_language
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 100,101,98,117,103,45,108,111,99,45,111,102,102,115,101,116,49,46,99,99 // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b32 .debug_line                     // DW_AT_stmt_list
+; CHECK-NEXT: .b8 47,108,108,118,109,95,99,109,97,107,101,95,103,99,99 // DW_AT_comp_dir
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b64 Lfunc_begin0                    // DW_AT_low_pc
+; CHECK-NEXT: .b64 Lfunc_end0                      // DW_AT_high_pc
+; CHECK-NEXT: .b8 5                                // Abbrev [5] 0x64:0x30 DW_TAG_subprogram
+; CHECK-NEXT: .b64 Lfunc_begin0                    // DW_AT_low_pc
+; CHECK-NEXT: .b64 Lfunc_end0                      // DW_AT_high_pc
+; CHECK-NEXT: .b8 1                                // DW_AT_frame_base
+; CHECK-NEXT: .b8 156
+; CHECK-NEXT: .b8 95,90,51,98,97,114,105           // DW_AT_MIPS_linkage_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 98,97,114                        // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_line
+; CHECK-NEXT: .b32 148                             // DW_AT_type
+; CHECK-NEXT: .b8 1                                // DW_AT_external
+; CHECK-NEXT: .b8 6                                // Abbrev [6] 0x8a:0x9 DW_TAG_formal_parameter
+; CHECK-NEXT: .b8 98                               // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_file
+; CHECK-NEXT: .b8 1                                // DW_AT_decl_line
+; CHECK-NEXT: .b32 148                             // DW_AT_type
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: .b8 7                                // Abbrev [7] 0x94:0x7 DW_TAG_base_type
+; CHECK-NEXT: .b8 105,110,116                      // DW_AT_name
+; CHECK-NEXT: .b8 0
+; CHECK-NEXT: .b8 5                                // DW_AT_encoding
+; CHECK-NEXT: .b8 4                                // DW_AT_byte_size
+; CHECK-NEXT: .b8 0                                // End Of Children Mark
+; CHECK-NEXT: }
+; CHECK-NEXT: .section .debug_macinfo
+; CHECK-NEXT: {
+; CHECK-NEXT: .b8 0                                // End Of Macro List Mark
+; CHECK:      }
diff --git a/test/DebugInfo/Sparc/gnu-window-save.ll b/test/DebugInfo/Sparc/gnu-window-save.ll
index f13727767318788d905a766ef3514d1051272f22..fb9602bace775084b84585de3016ad1e0afe0eb7 100644
--- a/test/DebugInfo/Sparc/gnu-window-save.ll
+++ b/test/DebugInfo/Sparc/gnu-window-save.ll
@@ -55,7 +55,7 @@ attributes #1 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "n
 !llvm.module.flags = !{!9, !10}
 !llvm.ident = !{!11}
 
-!0 = distinct !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.5 (http://llvm.org/git/clang.git 6a0714fee07fb7c4e32d3972b4fe2ce2f5678cf4) (llvm/ 672e88e934757f76d5c5e5258be41e7615094844)", isOptimized: false, emissionKind: FullDebug, file: !1, enums: !2, retainedTypes: !2, globals: !2, imports: !2)
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, producer: "clang version 3.5", isOptimized: false, emissionKind: FullDebug, file: !1, enums: !2, retainedTypes: !2, globals: !2, imports: !2)
 !1 = !DIFile(filename: "hello.c", directory: "/home/venkatra/work/benchmarks/test/hello")
 !2 = !{}
 !4 = distinct !DISubprogram(name: "main", line: 3, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, unit: !0, scopeLine: 4, file: !1, scope: !5, type: !6, retainedNodes: !2)
@@ -65,6 +65,6 @@ attributes #1 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "n
 !8 = !DIBasicType(tag: DW_TAG_base_type, name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
 !9 = !{i32 2, !"Dwarf Version", i32 4}
 !10 = !{i32 1, !"Debug Info Version", i32 3}
-!11 = !{!"clang version 3.5 (http://llvm.org/git/clang.git 6a0714fee07fb7c4e32d3972b4fe2ce2f5678cf4) (llvm/ 672e88e934757f76d5c5e5258be41e7615094844)"}
+!11 = !{!"clang version 3.5"}
 !12 = !DILocation(line: 5, scope: !4)
 !13 = !DILocation(line: 6, scope: !4)
diff --git a/test/DebugInfo/Sparc/prologue_end.ll b/test/DebugInfo/Sparc/prologue_end.ll
index 78bd3def69cec67582603d2f67e07a9b38b33f43..91e6db29cfa66d3487eac8ee83e7f1534a0008ad 100644
--- a/test/DebugInfo/Sparc/prologue_end.ll
+++ b/test/DebugInfo/Sparc/prologue_end.ll
@@ -1,4 +1,4 @@
-; RUN: llc -disable-fp-elim -O0 %s -mtriple sparc -o - | FileCheck %s
+; RUN: llc -frame-pointer=all -O0 %s -mtriple sparc -o - | FileCheck %s
 
 ; int func(void);
 ; void prologue_end_test() {
diff --git a/test/DebugInfo/SystemZ/prologue_end.ll b/test/DebugInfo/SystemZ/prologue_end.ll
index 8251e82b2343f3551171907466499b43131c2a4b..c430b7905a8024ba75b69d86efed3637f99a6da0 100644
--- a/test/DebugInfo/SystemZ/prologue_end.ll
+++ b/test/DebugInfo/SystemZ/prologue_end.ll
@@ -1,4 +1,4 @@
-; RUN: llc -disable-fp-elim -O0 %s -mtriple s390x-linux-gnu -o - | FileCheck %s
+; RUN: llc -frame-pointer=all -O0 %s -mtriple s390x-linux-gnu -o - | FileCheck %s
 
 ; int func(void);
 ; void prologue_end_test() {
diff --git a/test/DebugInfo/SystemZ/variable-loc.ll b/test/DebugInfo/SystemZ/variable-loc.ll
index a281eb660d78b13913ff758fa2d9876f2a103828..6112a81643784fc89e9ebc019742c71695c64cb3 100644
--- a/test/DebugInfo/SystemZ/variable-loc.ll
+++ b/test/DebugInfo/SystemZ/variable-loc.ll
@@ -1,5 +1,5 @@
-; RUN: llc -mtriple=s390x-linux-gnu -disable-fp-elim < %s | FileCheck %s
-; RUN: llc -mtriple=s390x-linux-gnu -disable-fp-elim -filetype=obj < %s \
+; RUN: llc -mtriple=s390x-linux-gnu -frame-pointer=all < %s | FileCheck %s
+; RUN: llc -mtriple=s390x-linux-gnu -frame-pointer=all -filetype=obj < %s \
 ; RUN:     | llvm-dwarfdump -v -debug-info - | FileCheck --check-prefix=DEBUG %s
 ;
 ; This is a regression test making sure the location of variables is correct in
diff --git a/test/DebugInfo/SystemZ/variable-loc.s b/test/DebugInfo/SystemZ/variable-loc.s
index 6940b1be4d78da2463623516917eec0fc5327b72..8e2a59287f78cd3489391c2f99e785c1bd4db58e 100644
--- a/test/DebugInfo/SystemZ/variable-loc.s
+++ b/test/DebugInfo/SystemZ/variable-loc.s
@@ -11,8 +11,8 @@
 #
 # A couple of R_390_64s similarly:
 #
-# CHECK: DW_AT_low_pc [DW_FORM_addr] (0x0000000000000000)
-# CHECK: DW_AT_high_pc [DW_FORM_addr] (0x0000000000000050)
+# CHECK: DW_AT_low_pc [DW_FORM_addr] (0x0000000000000000 ".text")
+# CHECK: DW_AT_high_pc [DW_FORM_addr] (0x0000000000000050 ".text")
 
 
 	.file	"test/DebugInfo/SystemZ/variable-loc.ll"
diff --git a/test/DebugInfo/WebAssembly/dbg-value-live-interval.ll b/test/DebugInfo/WebAssembly/dbg-value-live-interval.ll
index 1754d2a7a21c7d7e17ebb1699bb3c03888acaf77..c43678e3c4f62ca95d7da571d83e02ef7c200676 100644
--- a/test/DebugInfo/WebAssembly/dbg-value-live-interval.ll
+++ b/test/DebugInfo/WebAssembly/dbg-value-live-interval.ll
@@ -4,7 +4,7 @@
 ; CHECK: bb.3.for.body.for.body_crit_edge:
 ; CHECK: [[REG:%[0-9]+]]:i32 = nsw ADD_I32 {{.*}} fib.c:7:7
 ; CHECK: DBG_VALUE [[REG]]:i32, $noreg, !"a", {{.*}} fib.c:5:13
-; CHECK: After WebAssembly Store Results:
+; CHECK: After WebAssembly Memory Intrinsic Results:
 
 ; ModuleID = 'fib.bc'
 source_filename = "fib.c"
diff --git a/test/DebugInfo/WebAssembly/dbg-value-move-clone.mir b/test/DebugInfo/WebAssembly/dbg-value-move-clone.mir
new file mode 100644
index 0000000000000000000000000000000000000000..1e10c136873c8d96d6c14a438708a02e92508103
--- /dev/null
+++ b/test/DebugInfo/WebAssembly/dbg-value-move-clone.mir
@@ -0,0 +1,65 @@
+# RUN: llc < %s -run-pass=wasm-reg-stackify -x=mir 2>&1 | FileCheck %s
+
+# CHECK: body:
+# CHECK: bb.0:
+# CHECK: %[[REG1:[0-9]+]]:i32 = CONST_I32 0,
+# CHECK-NEXT: DBG_VALUE %[[REG1]],
+# CHECK-NEXT: CALL_VOID @foo, %[[REG1]],
+# CHECK: bb.1:
+# CHECK: %[[REG2:[0-9]+]]:i32 = CONST_I32 0,
+# CHECK-NEXT: DBG_VALUE %[[REG2]],
+# CHECK-NEXT: CALL_VOID @foo, %[[REG2]],
+# CHECK: %[[REG3:[0-9]+]]:i32 = CONST_I32 0,
+# CHECK-NEXT: DBG_VALUE %[[REG3]],
+# CHECK-NEXT: CALL_VOID @foo, %[[REG3]],
+
+--- |
+  target datalayout = "e-m:e-p:32:32-i64:64-n32:64-S128"
+  target triple = "wasm32-unknown-unknown"
+
+  declare void @foo(i32)
+  declare i32 @bar()
+
+  define void @test(i64 %arg) {
+    unreachable
+  }
+
+  !llvm.dbg.cu = !{!0}
+  !llvm.module.flags = !{!4}
+  !0 = distinct !DICompileUnit(language: DW_LANG_Rust, file: !2, producer: "clang LLVM (rustc version 1.30.0-dev)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !1, globals: !1)
+  !1 = !{}
+  !2 = !DIFile(filename: "<unknown>", directory: "")
+  !3 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "&str", file: !2, size: 64, align: 32, elements: !{}, identifier: "111094d970b097647de579f9c509ef08")
+  !4 = !{i32 2, !"Debug Info Version", i32 3}
+  !5 = distinct !DILexicalBlock(scope: !6, file: !2, line: 357, column: 8)
+  !6 = distinct !DISubprogram(name: "testfoo", linkageName: "_testba", scope: !7, file: !2, line: 353, type: !8, isLocal: true, isDefinition: true, scopeLine: 353, flags: DIFlagPrototyped, isOptimized: true, unit: !0, templateParams: !1, retainedNodes: !9)
+  !7 = !DINamespace(name: "ptr", scope: null)
+  !8 = !DISubroutineType(types: !1)
+  !9 = !{!10}
+  !10 = !DILocalVariable(name: "val0", scope: !5, file: !2, line: 357, type: !3, align: 4)
+  !11 = !DILocalVariable(name: "val1", scope: !5, file: !2, line: 358, type: !3, align: 4)
+  !12 = !DILocalVariable(name: "val2", scope: !5, file: !2, line: 359, type: !3, align: 4)
+  !13 = !DILocation(line: 357, column: 12, scope: !5)
+  !14 = !DILocation(line: 358, column: 12, scope: !5)
+  !15 = !DILocation(line: 359, column: 12, scope: !5)
+
+---
+name: test
+liveins:
+  - { reg: '$arguments' }
+tracksRegLiveness: true
+body: |
+  bb.0:
+    successors: %bb.1
+    liveins: $arguments
+    %0:i64 = ARGUMENT_i64 0, implicit $arguments
+    %1:i32 = CONST_I32 0, implicit-def dead $arguments
+    DBG_VALUE %1:i32, $noreg, !10, !DIExpression(), debug-location !13; <unknown>:357:12 line no:357
+    CALL_VOID @foo, %1:i32, implicit-def dead $arguments, implicit $sp32, implicit $sp64
+
+  bb.1:
+    CALL_VOID @foo, %1:i32, implicit-def dead $arguments, implicit $sp32, implicit $sp64
+    CALL_VOID @foo, %1:i32, implicit-def dead $arguments, implicit $sp32, implicit $sp64
+    RETURN_VOID implicit-def dead $arguments
+
+...
diff --git a/test/DebugInfo/WebAssembly/dbg-value-move-reg-stackify.mir b/test/DebugInfo/WebAssembly/dbg-value-move-reg-stackify.mir
new file mode 100644
index 0000000000000000000000000000000000000000..d8ebd42dae086b6f3c9fe7bc73c66f7dff230f8a
--- /dev/null
+++ b/test/DebugInfo/WebAssembly/dbg-value-move-reg-stackify.mir
@@ -0,0 +1,60 @@
+# RUN: llc < %s -run-pass=wasm-reg-stackify -x=mir 2>&1 | FileCheck %s
+
+# CHECK: body:
+# CHECK: %1:i32 = I32_WRAP_I64 %0,
+# CHECK-NEXT: DBG_VALUE %1,
+# CHECK-NEXT: %1:i32 = CALL_I32 @bar,
+# CHECK-NEXT: DBG_VALUE %1,
+# CHECK-NEXT: %[[NEWREG:.*]]:i32 = CALL_I32 @bar,
+# CHECK-NEXT: DBG_VALUE %[[NEWREG]],
+# CHECK-NEXT: CALL_VOID @foo, %[[NEWREG]],
+
+--- |
+  target datalayout = "e-m:e-p:32:32-i64:64-n32:64-S128"
+  target triple = "wasm32-unknown-unknown"
+
+  declare void @foo(i32)
+  declare i32 @bar()
+
+  define void @test(i64 %arg) {
+    unreachable
+  }
+
+  !llvm.dbg.cu = !{!0}
+  !llvm.module.flags = !{!4}
+  !0 = distinct !DICompileUnit(language: DW_LANG_Rust, file: !2, producer: "clang LLVM (rustc version 1.30.0-dev)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !1, globals: !1)
+  !1 = !{}
+  !2 = !DIFile(filename: "<unknown>", directory: "")
+  !3 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "&str", file: !2, size: 64, align: 32, elements: !{}, identifier: "111094d970b097647de579f9c509ef08")
+  !4 = !{i32 2, !"Debug Info Version", i32 3}
+  !5 = distinct !DILexicalBlock(scope: !6, file: !2, line: 357, column: 8)
+  !6 = distinct !DISubprogram(name: "testfoo", linkageName: "_testba", scope: !7, file: !2, line: 353, type: !8, isLocal: true, isDefinition: true, scopeLine: 353, flags: DIFlagPrototyped, isOptimized: true, unit: !0, templateParams: !1, retainedNodes: !9)
+  !7 = !DINamespace(name: "ptr", scope: null)
+  !8 = !DISubroutineType(types: !1)
+  !9 = !{!10}
+  !10 = !DILocalVariable(name: "val0", scope: !5, file: !2, line: 357, type: !3, align: 4)
+  !11 = !DILocalVariable(name: "val1", scope: !5, file: !2, line: 358, type: !3, align: 4)
+  !12 = !DILocalVariable(name: "val2", scope: !5, file: !2, line: 359, type: !3, align: 4)
+  !13 = !DILocation(line: 357, column: 12, scope: !5)
+  !14 = !DILocation(line: 358, column: 12, scope: !5)
+  !15 = !DILocation(line: 359, column: 12, scope: !5)
+
+---
+name: test
+liveins:
+  - { reg: '$arguments' }
+tracksRegLiveness: true
+body: |
+  bb.0:
+    liveins: $arguments
+    %0:i64 = ARGUMENT_i64 0, implicit $arguments
+    %1:i32 = I32_WRAP_I64 %0:i64, implicit-def dead $arguments
+    DBG_VALUE %1:i32, $noreg, !10, !DIExpression(), debug-location !13; <unknown>:357:12 line no:357
+    %1:i32 = CALL_I32 @bar, implicit-def dead $arguments, implicit $sp32, implicit $sp64
+    DBG_VALUE %1:i32, $noreg, !11, !DIExpression(), debug-location !14; <unknown>:357:12 line no:357
+    %1:i32 = CALL_I32 @bar, implicit-def dead $arguments, implicit $sp32, implicit $sp64
+    DBG_VALUE %1:i32, $noreg, !12, !DIExpression(), debug-location !15; <unknown>:357:12 line no:357
+    CALL_VOID @foo, %1:i32, implicit-def dead $arguments, implicit $sp32, implicit $sp64
+    RETURN_VOID implicit-def dead $arguments
+
+...
diff --git a/test/DebugInfo/X86/PR26148.ll b/test/DebugInfo/X86/PR26148.ll
index 685f2d503717e60fe066efd35b86bd0cd8ceaaa5..c42b8e5145072c6a167b68a132e7e2e97333b0d2 100644
--- a/test/DebugInfo/X86/PR26148.ll
+++ b/test/DebugInfo/X86/PR26148.ll
@@ -16,11 +16,10 @@
 ; This is similar to the bug in test/DebugInfo/ARM/PR26163.ll, except that there is an
 ; extra non-overlapping range first. Thus, we make sure that the backend actually looks
 ; at all expressions when determining whether to merge ranges, not just the first one.
-; AS in 26163, we expect two ranges (as opposed to one), the first one being zero sized
+; AS in 26163, we only expect one range as the first one is zero sized.
 ;
 ;
-; CHECK: [0x0000000000000004, 0x0000000000000004): DW_OP_lit3, DW_OP_piece 0x4, DW_OP_reg5 RDI, DW_OP_piece 0x2
-; CHECK: [0x0000000000000004, 0x0000000000000014): DW_OP_lit3, DW_OP_piece 0x4, DW_OP_lit0, DW_OP_piece 0x4
+; CHECK: [0x0000000000000000, 0x000000000000000f): DW_OP_lit3, DW_OP_piece 0x4, DW_OP_lit0, DW_OP_piece 0x4
 
 source_filename = "test/DebugInfo/X86/PR26148.ll"
 target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
@@ -35,11 +34,8 @@ declare void @llvm.dbg.declare(metadata, metadata, metadata) #0
 
 ; Function Attrs: nounwind readnone
 declare void @llvm.dbg.value(metadata, metadata, metadata) #0
-; The attributes are here to force the zero-sized range not to be at the start of
-; the function, which has special interpretation in DWARF. The fact that this happens
-; at all is probably an LLVM bug.
 
-define void @fn1(i16 signext %p1) #1 !dbg !16 {
+define void @fn1(i16 signext %p1) !dbg !16 {
 entry:
   tail call void @llvm.dbg.value(metadata i16 %p1, metadata !20, metadata !23), !dbg !24
   tail call void @llvm.dbg.declare(metadata %struct.S0* undef, metadata !21, metadata !23), !dbg !25
@@ -60,7 +56,6 @@ entry:
 }
 
 attributes #0 = { nounwind readnone }
-attributes #1 = { "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" }
 
 !llvm.dbg.cu = !{!2}
 !llvm.module.flags = !{!12, !13, !14}
@@ -68,7 +63,7 @@ attributes #1 = { "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf
 
 !0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression())
 !1 = !DIGlobalVariable(name: "a", scope: !2, file: !3, line: 4, type: !6, isLocal: false, isDefinition: true)
-!2 = distinct !DICompileUnit(language: DW_LANG_C99, file: !3, producer: "clang version 3.9.0 (https://github.com/llvm-mirror/clang 8f258397c5afd7a708bd95770c718e81d08fb11a) (https://github.com/llvm-mirror/llvm 18481855bdfa1b4a424f81be8525db002671348d)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, globals: !5)
+!2 = distinct !DICompileUnit(language: DW_LANG_C99, file: !3, producer: "clang version 3.9.0", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, globals: !5)
 !3 = !DIFile(filename: "small.c", directory: "/Users/kfischer/Projects/clangbug")
 !4 = !{}
 !5 = !{!0}
@@ -81,7 +76,7 @@ attributes #1 = { "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf
 !12 = !{i32 2, !"Dwarf Version", i32 2}
 !13 = !{i32 2, !"Debug Info Version", i32 3}
 !14 = !{i32 1, !"PIC Level", i32 2}
-!15 = !{!"clang version 3.9.0 (https://github.com/llvm-mirror/clang 8f258397c5afd7a708bd95770c718e81d08fb11a) (https://github.com/llvm-mirror/llvm 18481855bdfa1b4a424f81be8525db002671348d)"}
+!15 = !{!"clang version 3.9.0"}
 !16 = distinct !DISubprogram(name: "fn1", scope: !3, file: !3, line: 5, type: !17, isLocal: false, isDefinition: true, scopeLine: 5, flags: DIFlagPrototyped, isOptimized: true, unit: !2, retainedNodes: !19)
 !17 = !DISubroutineType(types: !18)
 !18 = !{null, !9}
diff --git a/test/DebugInfo/X86/PR37234.ll b/test/DebugInfo/X86/PR37234.ll
index 51fae94423ecb037bef1cdc47a3b7df74ceeb021..6f7388755ee605c82143cac73ffaa351eb03968a 100644
--- a/test/DebugInfo/X86/PR37234.ll
+++ b/test/DebugInfo/X86/PR37234.ll
@@ -22,19 +22,18 @@
 ; CHECK:        #DEBUG_VALUE: main:aa <- 0
 ; CHECK: 	#DEBUG_VALUE: main:aa <- $[[REG:[0-9a-z]+]]
 ; CHECK: 	jmp	.LBB0_1
-; CHECK: .LBB0_3:
-; CHECK: 	#DEBUG_VALUE: main:aa <- $[[REG]]
-; CHECK: 	incl	%[[REG]]
-; CHECK: 	#DEBUG_VALUE: main:aa <- $[[REG]]
+; CHECK: .LBB0_2:
+; CHECK:        #DEBUG_VALUE: main:aa <- $[[REG]]
+; CHECK:        jne     .LBB0_1
+; CHECK: # %bb.{{.*}}:
+; CHECK:        #DEBUG_VALUE: main:aa <- $[[REG]]
+; CHECK:        incl    %[[REG]]
+; CHECK:        #DEBUG_VALUE: main:aa <- $[[REG]]
 ; CHECK: .LBB0_1:
 ; CHECK: 	#DEBUG_VALUE: main:aa <- $[[REG]]
-; CHECK: 	je	.LBB0_4
+; CHECK:        jne     .LBB0_2
 ; CHECK: # %bb.{{.*}}:
 ; CHECK: 	#DEBUG_VALUE: main:aa <- $[[REG]]
-; CHECK: 	jne	.LBB0_1
-; CHECK: 	jmp	.LBB0_3
-; CHECK: .LBB0_4:
-; CHECK: 	#DEBUG_VALUE: main:aa <- $[[REG]]
 ; CHECK: 	retq
 
 source_filename = "PR37234.cpp"
diff --git a/test/DebugInfo/X86/addr_comments.ll b/test/DebugInfo/X86/addr_comments.ll
new file mode 100644
index 0000000000000000000000000000000000000000..160f380bed6bb6305670fd02ace000dff0817476
--- /dev/null
+++ b/test/DebugInfo/X86/addr_comments.ll
@@ -0,0 +1,35 @@
+; RUN: llc %s -mtriple=i386-unknown-linux-gnu -filetype=asm -o - | FileCheck %s
+
+; CHECK:   .section .debug_addr
+; CHECK-NEXT:   .long   .Ldebug_addr_end0-.Ldebug_addr_start0 # Length of contribution
+; CHECK-NEXT: .Ldebug_addr_start0:
+; CHECK-NEXT:   .short  5 # DWARF version number
+; CHECK-NEXT:   .byte   4 # Address size
+; CHECK-NEXT:   .byte   0 # Segment selector size
+; CHECK-NEXT: .Laddr_table_base0:
+; CHECK-NEXT:   .long   .Lfunc_begin0
+; CHECK-NEXT: .Ldebug_addr_end0:
+ 
+; Function Attrs: noinline nounwind optnone uwtable
+define dso_local void @foo() #0 !dbg !7 {
+entry:
+  ret void, !dbg !10
+}
+
+attributes #0 = { noinline nounwind optnone uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4, !5}
+!llvm.ident = !{!6}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 8.0.0 (trunk 350004) (llvm/trunk 350008)", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, nameTableKind: None)
+!1 = !DIFile(filename: "foo.c", directory: "/usr/local/google/home/blaikie/dev/scratch")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 5}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{i32 1, !"wchar_size", i32 4}
+!6 = !{!"clang version 8.0.0 (trunk 350004) (llvm/trunk 350008)"}
+!7 = distinct !DISubprogram(name: "foo", scope: !1, file: !1, line: 1, type: !8, scopeLine: 1, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !2)
+!8 = !DISubroutineType(types: !9)
+!9 = !{null}
+!10 = !DILocation(line: 2, column: 1, scope: !7)
diff --git a/test/DebugInfo/X86/align_c11.ll b/test/DebugInfo/X86/align_c11.ll
index ab447f233cecda96bd82177a6d20a7ca9f6ebaf4..c488128da61daea32f8d5dcda1d7304b2c1fcec6 100644
--- a/test/DebugInfo/X86/align_c11.ll
+++ b/test/DebugInfo/X86/align_c11.ll
@@ -58,7 +58,7 @@ attributes #1 = { nounwind readnone }
 
 !0 = distinct !DIGlobalVariableExpression(var: !1, expr: !DIExpression())
 !1 = !DIGlobalVariable(name: "d", scope: !2, file: !3, line: 7, type: !6, isLocal: false, isDefinition: true, align: 16384)
-!2 = distinct !DICompileUnit(language: DW_LANG_C99, file: !3, producer: "clang version 4.0.0 (http://llvm.org/git/clang.git 9ce5220b821054019059c2ac4a9b132c7723832d) (http://llvm.org/git/llvm.git 9a6298be89ce0359b151c0a37af2776a12c69e85)", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, globals: !5)
+!2 = distinct !DICompileUnit(language: DW_LANG_C99, file: !3, producer: "clang version 4.0.0", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, globals: !5)
 !3 = !DIFile(filename: "test.c", directory: "/tmp")
 !4 = !{}
 !5 = !{!0}
@@ -72,7 +72,7 @@ attributes #1 = { nounwind readnone }
 !13 = !DISubrange(count: 2)
 !14 = !{i32 2, !"Dwarf Version", i32 4}
 !15 = !{i32 2, !"Debug Info Version", i32 3}
-!16 = !{!"clang version 4.0.0 (http://llvm.org/git/clang.git 9ce5220b821054019059c2ac4a9b132c7723832d) (http://llvm.org/git/llvm.git 9a6298be89ce0359b151c0a37af2776a12c69e85)"}
+!16 = !{!"clang version 4.0.0"}
 !17 = distinct !DISubprogram(name: "foo", scope: !3, file: !3, line: 8, type: !18, isLocal: false, isDefinition: true, scopeLine: 9, flags: DIFlagPrototyped, isOptimized: false, unit: !2, retainedNodes: !4)
 !18 = !DISubroutineType(types: !19)
 !19 = !{!20}
diff --git a/test/DebugInfo/X86/align_cpp11.ll b/test/DebugInfo/X86/align_cpp11.ll
index cc866fcacf168245d8671f6003cb7bb144906dba..c15714c5561b4f3d7f923cb10cdf0eac0bad26b6 100644
--- a/test/DebugInfo/X86/align_cpp11.ll
+++ b/test/DebugInfo/X86/align_cpp11.ll
@@ -120,7 +120,7 @@ attributes #1 = { nounwind readnone }
 
 !0 = distinct !DIGlobalVariableExpression(var: !1, expr: !DIExpression())
 !1 = !DIGlobalVariable(name: "c0", scope: !2, file: !6, line: 19, type: !19, isLocal: false, isDefinition: true)
-!2 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !3, producer: "clang version 4.0.0 (http://llvm.org/git/clang.git 9ce5220b821054019059c2ac4a9b132c7723832d) (http://llvm.org/git/llvm.git 9a6298be89ce0359b151c0a37af2776a12c69e85)", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, globals: !11)
+!2 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !3, producer: "clang version 4.0.0", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, globals: !11)
 !3 = !DIFile(filename: "test.cpp", directory: "/tmp")
 !4 = !{!5}
 !5 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "E", file: !6, line: 13, size: 32, align: 128, elements: !7, identifier: "_ZTS1E")
@@ -141,7 +141,7 @@ attributes #1 = { nounwind readnone }
 !20 = !{}
 !21 = !{i32 2, !"Dwarf Version", i32 4}
 !22 = !{i32 2, !"Debug Info Version", i32 3}
-!23 = !{!"clang version 4.0.0 (http://llvm.org/git/clang.git 9ce5220b821054019059c2ac4a9b132c7723832d) (http://llvm.org/git/llvm.git 9a6298be89ce0359b151c0a37af2776a12c69e85)"}
+!23 = !{!"clang version 4.0.0"}
 !24 = distinct !DISubprogram(name: "foo", linkageName: "_Z3foov", scope: !6, file: !6, line: 23, type: !25, isLocal: false, isDefinition: true, scopeLine: 24, flags: DIFlagPrototyped, isOptimized: false, unit: !2, retainedNodes: !20)
 !25 = !DISubroutineType(types: !26)
 !26 = !{null}
diff --git a/test/DebugInfo/X86/align_objc.ll b/test/DebugInfo/X86/align_objc.ll
index 99726bbd29a8b56c9097d46d1c34e15191bc93d4..1a69fd002a6b944152afa91ce6dda1a53198929f 100644
--- a/test/DebugInfo/X86/align_objc.ll
+++ b/test/DebugInfo/X86/align_objc.ll
@@ -70,7 +70,7 @@ attributes #1 = { nounwind readnone }
 
 !0 = distinct !DIGlobalVariableExpression(var: !1, expr: !DIExpression())
 !1 = !DIGlobalVariable(name: "s0", scope: !2, file: !3, line: 10, type: !6, isLocal: false, isDefinition: true)
-!2 = distinct !DICompileUnit(language: DW_LANG_ObjC, file: !3, producer: "clang version 4.0.0 (http://llvm.org/git/clang.git 9ce5220b821054019059c2ac4a9b132c7723832d) (http://llvm.org/git/llvm.git 9a6298be89ce0359b151c0a37af2776a12c69e85)", isOptimized: false, runtimeVersion: 1, emissionKind: FullDebug, enums: !4, globals: !5)
+!2 = distinct !DICompileUnit(language: DW_LANG_ObjC, file: !3, producer: "clang version 4.0.0", isOptimized: false, runtimeVersion: 1, emissionKind: FullDebug, enums: !4, globals: !5)
 !3 = !DIFile(filename: "test.m", directory: "/tmp")
 !4 = !{}
 !5 = !{!0}
@@ -81,7 +81,7 @@ attributes #1 = { nounwind readnone }
 !10 = !DIBasicType(name: "char", size: 8, encoding: DW_ATE_signed_char)
 !11 = !{i32 2, !"Dwarf Version", i32 4}
 !12 = !{i32 2, !"Debug Info Version", i32 3}
-!13 = !{!"clang version 4.0.0 (http://llvm.org/git/clang.git 9ce5220b821054019059c2ac4a9b132c7723832d) (http://llvm.org/git/llvm.git 9a6298be89ce0359b151c0a37af2776a12c69e85)"}
+!13 = !{!"clang version 4.0.0"}
 !14 = distinct !DISubprogram(name: "f", scope: !3, file: !3, line: 12, type: !15, isLocal: false, isDefinition: true, scopeLine: 12, isOptimized: false, unit: !2, retainedNodes: !4)
 !15 = !DISubroutineType(types: !16)
 !16 = !{null}
diff --git a/test/DebugInfo/X86/atomic-c11-dwarf-4.ll b/test/DebugInfo/X86/atomic-c11-dwarf-4.ll
index efbf1a2be5153b2e2884a0a5cd2b4ff7408bbb1d..737ff63420177766b2d3edb54245d598cb2218b0 100644
--- a/test/DebugInfo/X86/atomic-c11-dwarf-4.ll
+++ b/test/DebugInfo/X86/atomic-c11-dwarf-4.ll
@@ -24,7 +24,7 @@ target triple = "x86_64-unknown-linux-gnu"
 
 !0 = distinct !DIGlobalVariableExpression(var: !1, expr: !DIExpression())
 !1 = !DIGlobalVariable(name: "i", scope: !2, file: !3, line: 1, type: !6, isLocal: false, isDefinition: true)
-!2 = distinct !DICompileUnit(language: DW_LANG_C99, file: !3, producer: "clang version 4.0.0 (http://llvm.org/git/clang.git cd238117e3a8a57271a82d0bb03d6df6ad8f073e) (http://llvm.org/git/llvm.git 9fd063832c1541aad3907cd60ac344d36997905f)", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, globals: !5)
+!2 = distinct !DICompileUnit(language: DW_LANG_C99, file: !3, producer: "clang version 4.0.0", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, globals: !5)
 !3 = !DIFile(filename: "atomic.c", directory: "/tmp")
 !4 = !{}
 !5 = !{!0}
@@ -33,5 +33,5 @@ target triple = "x86_64-unknown-linux-gnu"
 !8 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
 !9 = !{i32 2, !"Dwarf Version", i32 4}
 !10 = !{i32 2, !"Debug Info Version", i32 3}
-!11 = !{!"clang version 4.0.0 (http://llvm.org/git/clang.git cd238117e3a8a57271a82d0bb03d6df6ad8f073e) (http://llvm.org/git/llvm.git 9fd063832c1541aad3907cd60ac344d36997905f)"}
+!11 = !{!"clang version 4.0.0"}
 
diff --git a/test/DebugInfo/X86/atomic-c11-dwarf-5.ll b/test/DebugInfo/X86/atomic-c11-dwarf-5.ll
index 859617681c896c10400e7853606f7d7bebf3029c..1fe755f0f77197f1cafe99617d9712fcda9f5611 100644
--- a/test/DebugInfo/X86/atomic-c11-dwarf-5.ll
+++ b/test/DebugInfo/X86/atomic-c11-dwarf-5.ll
@@ -25,7 +25,7 @@ target triple = "x86_64-unknown-linux-gnu"
 
 !0 = distinct !DIGlobalVariableExpression(var: !1, expr: !DIExpression())
 !1 = !DIGlobalVariable(name: "i", scope: !2, file: !3, line: 1, type: !6, isLocal: false, isDefinition: true)
-!2 = distinct !DICompileUnit(language: DW_LANG_C99, file: !3, producer: "clang version 4.0.0 (http://llvm.org/git/clang.git cd238117e3a8a57271a82d0bb03d6df6ad8f073e) (http://llvm.org/git/llvm.git 9fd063832c1541aad3907cd60ac344d36997905f)", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, globals: !5)
+!2 = distinct !DICompileUnit(language: DW_LANG_C99, file: !3, producer: "clang version 4.0.0", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, globals: !5)
 !3 = !DIFile(filename: "atomic.c", directory: "/tmp")
 !4 = !{}
 !5 = !{!0}
@@ -34,5 +34,5 @@ target triple = "x86_64-unknown-linux-gnu"
 !8 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
 !9 = !{i32 2, !"Dwarf Version", i32 5}
 !10 = !{i32 2, !"Debug Info Version", i32 3}
-!11 = !{!"clang version 4.0.0 (http://llvm.org/git/clang.git cd238117e3a8a57271a82d0bb03d6df6ad8f073e) (http://llvm.org/git/llvm.git 9fd063832c1541aad3907cd60ac344d36997905f)"}
+!11 = !{!"clang version 4.0.0"}
 
diff --git a/test/DebugInfo/X86/clone-module-2.ll b/test/DebugInfo/X86/clone-module-2.ll
index 23b1846dac0a9a8146c511ad400f4eaee06b9da5..320557c01a2e60963a4accc2d8021df24746f6f4 100644
--- a/test/DebugInfo/X86/clone-module-2.ll
+++ b/test/DebugInfo/X86/clone-module-2.ll
@@ -39,14 +39,14 @@ attributes #0 = { norecurse nounwind readnone ssp uwtable }
 !llvm.module.flags = !{!3, !4, !5, !6}
 !llvm.ident = !{!7}
 
-!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 7.0.0 (https://git.llvm.org/git/clang.git/ 195459d046e795f5952f7d2121e505eeb59c5574) (https://git.llvm.org/git/llvm.git/ e9dc5b5ade57869d1a443c568c6cf556ccf3b7af)", isOptimized: true, runtimeVersion: 0, emissionKind: LineTablesOnly, enums: !2)
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 7.0.0", isOptimized: true, runtimeVersion: 0, emissionKind: LineTablesOnly, enums: !2)
 !1 = !DIFile(filename: "test.c", directory: "/Volumes/Data/llvm/build/obj")
 !2 = !{}
 !3 = !{i32 2, !"Dwarf Version", i32 4}
 !4 = !{i32 2, !"Debug Info Version", i32 3}
 !5 = !{i32 1, !"wchar_size", i32 4}
 !6 = !{i32 7, !"PIC Level", i32 2}
-!7 = !{!"clang version 7.0.0 (https://git.llvm.org/git/clang.git/ 195459d046e795f5952f7d2121e505eeb59c5574) (https://git.llvm.org/git/llvm.git/ e9dc5b5ade57869d1a443c568c6cf556ccf3b7af)"}
+!7 = !{!"clang version 7.0.0"}
 !8 = !DILocation(line: 3, column: 72, scope: !9)
 !9 = distinct !DISubprogram(name: "eliminated", scope: !1, file: !1, line: 3, type: !10, isLocal: true, isDefinition: true, scopeLine: 3, flags: DIFlagPrototyped, isOptimized: true, unit: !0, retainedNodes: !2)
 !10 = !DISubroutineType(types: !2)
diff --git a/test/DebugInfo/X86/clone-module.ll b/test/DebugInfo/X86/clone-module.ll
index ae334be27251127c4a2a2275b15ce7d8bfff3306..8c7fd52a549d43ae533dc6a82bbd63b5fa8c5cb9 100644
--- a/test/DebugInfo/X86/clone-module.ll
+++ b/test/DebugInfo/X86/clone-module.ll
@@ -61,14 +61,14 @@ attributes #2 = { nounwind readnone speculatable }
 !llvm.module.flags = !{!3, !4, !5, !6}
 !llvm.ident = !{!7}
 
-!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 7.0.0 (https://git.llvm.org/git/clang.git/ 195459d046e795f5952f7d2121e505eeb59c5574) (https://git.llvm.org/git/llvm.git/ 69ec7d5667e9928db8435bfbee0da151c85a91c9)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2)
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 7.0.0", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2)
 !1 = !DIFile(filename: "clone-module.c", directory: "/somewhere")
 !2 = !{}
 !3 = !{i32 2, !"Dwarf Version", i32 4}
 !4 = !{i32 2, !"Debug Info Version", i32 3}
 !5 = !{i32 1, !"wchar_size", i32 4}
 !6 = !{i32 7, !"PIC Level", i32 2}
-!7 = !{!"clang version 7.0.0 (https://git.llvm.org/git/clang.git/ 195459d046e795f5952f7d2121e505eeb59c5574) (https://git.llvm.org/git/llvm.git/ 69ec7d5667e9928db8435bfbee0da151c85a91c9)"}
+!7 = !{!"clang version 7.0.0"}
 !8 = !DILocalVariable(name: "j", arg: 1, scope: !9, file: !1, line: 3, type: !12)
 !9 = distinct !DISubprogram(name: "inlined", scope: !1, file: !1, line: 3, type: !10, isLocal: false, isDefinition: true, scopeLine: 3, flags: DIFlagPrototyped, isOptimized: true, unit: !0, retainedNodes: !13)
 !10 = !DISubroutineType(types: !11)
diff --git a/test/DebugInfo/X86/dbg-value-arg-movement.ll b/test/DebugInfo/X86/dbg-value-arg-movement.ll
new file mode 100644
index 0000000000000000000000000000000000000000..b5692b80435ac209d3d4cbbd5e83cefe665062c8
--- /dev/null
+++ b/test/DebugInfo/X86/dbg-value-arg-movement.ll
@@ -0,0 +1,104 @@
+; RUN: llc -mtriple=x86_64-unknown-unknown -start-after=codegenprepare -stop-before=expand-isel-pseudos %s -o - | FileCheck %s
+
+; Test the movement of dbg.values of arguments. SelectionDAG tries to be
+; helpful and places DBG_VALUEs of Arguments at the start of functions.
+; Unfortunately, this doesn't necessarily make sense, as one can specify an
+; Argument IR Value as a variable location anywhere in the program.
+;
+; Distinguish cases where we want to hoist DBG_VALUEs, and those where we
+; don't, by whether the referred to variable is a parameter to the current
+; function. In the test below, 'xyzzy' is a parameter to an inlined function,
+; but should not be hoisted to the start of the function.
+;
+; Original test case, in which 'xyzzy' became unavailable because its DBG_VALUE
+; landed far from any uses, compiled "clang -O2 -g" with inlining,
+;
+;    int ext(void);
+;
+;    int
+;    foo(int xyzzy)
+;    {
+;      xyzzy = ext() * xyzzy;
+;      xyzzy += 1;
+;      ext();
+;      return xyzzy;
+;    }
+;
+;    int
+;    bar(int baz, int qux)
+;    {
+;      int fish;
+;      switch (qux) {
+;      case 12:        fish = 8;        break;
+;      case 848:       fish = 0;        break;
+;      case 99999:     fish = -1;       break;
+;      default:        fish = 12;
+;      }
+;      qux %= fish;
+;      qux += foo(baz);
+;      return qux;
+;    }
+
+; CHECK: [[BAZVAR:![0-9]+]] = !DILocalVariable(name: "baz",
+; CHECK: [[XYZVAR:![0-9]+]] = !DILocalVariable(name: "xyzzy",
+
+; Start of MIR function block,
+; CHECK-LABEL: body
+; Expect DBG_VALUE of physreg,
+; CHECK:       DBG_VALUE $edi, $noreg, [[BAZVAR]]
+; Expect DBG_VALUE of virtreg,
+; CHECK:       DBG_VALUE [[ARGREG:%[0-9]+]], $noreg, [[BAZVAR]]
+; Label for next block,
+; CHECK-LABEL: bb.1.next
+; Correctly place dbg.value in the 'next' block.
+; CHECK:       DBG_VALUE [[ARGREG]], $noreg, [[XYZVAR]]
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+declare i32 @ext()
+
+define dso_local i32 @bar(i32, i32) local_unnamed_addr !dbg !7 {
+  %3 = srem i32 %1, %0, !dbg !15
+  call void @llvm.dbg.value(metadata i32 %0, metadata !12, metadata !DIExpression()), !dbg !16
+  br label %next
+
+next:                                             ; preds = %2
+  call void @llvm.dbg.value(metadata i32 %0, metadata !17, metadata !DIExpression()), !dbg !22
+  %4 = tail call i32 @ext(), !dbg !24
+  %5 = mul nsw i32 %3, %4, !dbg !25
+  %6 = add i32 %5, %0, !dbg !25
+  ret i32 %6, !dbg !25
+}
+
+; Function Attrs: nounwind readnone speculatable
+declare void @llvm.dbg.value(metadata, metadata, metadata)
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4, !5}
+!llvm.ident = !{!6}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, globals: !2, nameTableKind: None)
+!1 = !DIFile(filename: "test.c", directory: ".")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{i32 1, !"wchar_size", i32 4}
+!6 = !{!"clang version 8.0.0"}
+!7 = distinct !DISubprogram(name: "bar", scope: !1, file: !1, line: 19, type: !8, scopeLine: 20, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !11)
+!8 = !DISubroutineType(types: !9)
+!9 = !{!10, !10, !10}
+!10 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!11 = !{!12}
+!12 = !DILocalVariable(name: "baz", arg: 1, scope: !7, file: !1, line: 19, type: !10)
+!15 = !DILocation(line: 35, column: 7, scope: !7)
+!16 = !DILocation(line: 19, column: 9, scope: !7)
+!17 = !DILocalVariable(name: "xyzzy", arg: 1, scope: !18, file: !1, line: 10, type: !10)
+!18 = distinct !DISubprogram(name: "foo", scope: !1, file: !1, line: 10, type: !19, scopeLine: 11, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !21)
+!19 = !DISubroutineType(types: !20)
+!20 = !{!10, !10}
+!21 = !{!17}
+!22 = !DILocation(line: 10, column: 9, scope: !18, inlinedAt: !23)
+!23 = distinct !DILocation(line: 36, column: 10, scope: !7)
+!24 = !DILocation(line: 12, column: 11, scope: !18, inlinedAt: !23)
+!25 = !DILocation(line: 37, column: 3, scope: !7)
diff --git a/test/DebugInfo/X86/dbg-value-frame-index-2.ll b/test/DebugInfo/X86/dbg-value-frame-index-2.ll
new file mode 100644
index 0000000000000000000000000000000000000000..3f810dba30320b647cf5b367b999848df1356014
--- /dev/null
+++ b/test/DebugInfo/X86/dbg-value-frame-index-2.ll
@@ -0,0 +1,79 @@
+; RUN: llc -start-after=codegenprepare -stop-before=expand-isel-pseudos < %s -o - | FileCheck %s
+
+; Test that stack frame dbg.values are lowered to DBG_VALUEs, in blocks that
+; are local to the alloca, and elsewhere. Differs from dbg-value-frame-index.ll
+; because this test does not result in the frame-index being in a vreg,
+; instead it's exclusively referred to by memory operands of instructions.
+;
+; Additionally test that we don't re-order with constant values -- both are
+; independent of the order the instructions get lowered, but should not
+; interleave.
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-unknown"
+
+declare void @dud()
+
+; CHECK: [[BARVAR:![0-9]+]] = !DILocalVariable(name: "bar",
+
+define i32 @foo() !dbg !6 {
+; CHECK-LABEL: body
+
+; CHECK:      DBG_VALUE 0, $noreg, [[BARVAR]]
+; CHECK-NEXT: MOV32mi %[[STACKLOC:[a-zA-Z0-9\.]+]], 1, $noreg
+; CHECK-NEXT: DBG_VALUE %[[STACKLOC]], $noreg, [[BARVAR]]
+
+  %p1 = alloca i32
+  call void @llvm.dbg.value(metadata i32 *null, metadata !17, metadata !DIExpression()), !dbg !18
+  store i32 0, i32 *%p1
+  call void @llvm.dbg.value(metadata i32 *%p1, metadata !17, metadata !DIExpression()), !dbg !18
+  br label %foo
+
+foo:
+
+; CHECK-LABEL: bb.1.foo
+; CHECK:      DBG_VALUE %[[STACKLOC]], $noreg, [[BARVAR]]
+
+  call void @dud()
+  call void @llvm.dbg.value(metadata i32 *%p1, metadata !17, metadata !DIExpression()), !dbg !18
+  br label %bar
+
+bar:
+
+; CHECK-LABEL: bb.2.bar
+; CHECK:      DBG_VALUE %[[STACKLOC]], $noreg, [[BARVAR]]
+; CHECK-NEXT: ADJCALLSTACKDOWN
+; CHECK-NEXT: CALL
+; CHECK-NEXT: ADJCALLSTACKUP
+; CHECK-NEXT: DBG_VALUE 0, $noreg, [[BARVAR]]
+  call void @llvm.dbg.value(metadata i32 *%p1, metadata !17, metadata !DIExpression()), !dbg !18
+  call void @dud()
+  call void @llvm.dbg.value(metadata i32 *null, metadata !17, metadata !DIExpression()), !dbg !18
+  %loaded = load i32, i32 *%p1
+  ret i32 %loaded, !dbg !19
+}
+
+; Function Attrs: nounwind readnone speculatable
+declare void @llvm.dbg.value(metadata, metadata, metadata) #6
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4}
+!llvm.ident = !{!5}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C, file: !1, producer: "", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, retainedTypes: !2, globals: !2)
+!1 = !DIFile(filename: "a.c", directory: "b")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{!""}
+!6 = distinct !DISubprogram(name: "foo", scope: !1, file: !1, line: 13, type: !7, isLocal: false, isDefinition: true, scopeLine: 14, isOptimized: false, unit: !0, retainedNodes: !2)
+!7 = !DISubroutineType(types: !8)
+!8 = !{!140}
+!140 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!15 = !{!16}
+!16 = !DISubrange(count: 5)
+!17 = !DILocalVariable(name: "bar", scope: !6, line: 13, type: !140)
+!18 = !DILocation(line: 13, column: 23, scope: !6)
+!19 = !DILocation(line: 15, column: 5, scope: !6)
+!20 = !DILocation(line: 16, column: 1, scope: !6)
+!21 = !DILocalVariable(name: "baz", scope: !6, line: 13, type: !140)
diff --git a/test/DebugInfo/X86/dbg-value-funcarg.ll b/test/DebugInfo/X86/dbg-value-funcarg.ll
new file mode 100644
index 0000000000000000000000000000000000000000..310d85c69b45ffd607a2b8cf8d30b8eeacc8675b
--- /dev/null
+++ b/test/DebugInfo/X86/dbg-value-funcarg.ll
@@ -0,0 +1,165 @@
+; RUN: llc -mtriple=x86_64-unknown-linux-gnu -start-after=codegenprepare -stop-before=expand-isel-pseudos -o - %s | FileCheck %s
+
+; Input to this test looked like this and was compiled using: clang -g -O1 -mllvm -stop-after=codegenprepare -S
+;
+;    extern void bar(int);
+;
+;    void foo_local(int t1a) {
+;      int local = 123;
+;      bar(local);
+;      local = t1a;
+;      bar(local);
+;    }
+;
+;    void foo_other_param(int t2a, int t2b) {
+;      bar(t2b);
+;      t2b = 123;
+;      bar(t2b);
+;      t2b = t2a;
+;      bar(t2b);
+;    }
+;
+;    void foo_same_param(int t3a) {
+;      bar(t3a);
+;      int tmp = t3a;
+;      t3a = 123;
+;      bar(t3a);
+;      t3a = tmp;
+;      bar(t3a);
+;    }
+;
+
+; Catch metadata references for involved variables.
+;
+; CHECK-DAG: ![[T1A:.*]] = !DILocalVariable(name: "t1a"
+; CHECK-DAG: ![[LOCAL:.*]] = !DILocalVariable(name: "local"
+; CHECK-DAG: ![[T2A:.*]] = !DILocalVariable(name: "t2a"
+; CHECK-DAG: ![[T2B:.*]] = !DILocalVariable(name: "t2b"
+; CHECK-DAG: ![[T3A:.*]] = !DILocalVariable(name: "t3a"
+; CHECK-DAG: ![[TMP:.*]] = !DILocalVariable(name: "tmp"
+
+
+define dso_local void @foo_local(i32 %t1a) local_unnamed_addr #0 !dbg !7 {
+; CHECK-LABEL: name:            foo_local
+; CHECK-NOT: DBG_VALUE
+; CHECK:      DBG_VALUE $edi, $noreg, ![[T1A]], !DIExpression(),
+; CHECK-NEXT: %0:gr32 = COPY $edi
+; CHECK-NEXT: DBG_VALUE %0, $noreg, ![[T1A]], !DIExpression(),
+; CHECK-NEXT: DBG_VALUE 123, $noreg, ![[LOCAL]], !DIExpression(),
+; CHECK-NOT: DBG_VALUE
+; CHECK:    CALL64pcrel32 @bar,
+; CHECK:    DBG_VALUE %0, $noreg, ![[LOCAL]], !DIExpression(),
+; CHECK:    DBG_VALUE $edi, $noreg, ![[T1A]], !DIExpression(),
+; CHECK-NOT: DBG_VALUE
+; CHECK:    TCRETURNdi64 @bar,
+entry:
+  call void @llvm.dbg.value(metadata i32 %t1a, metadata !12, metadata !DIExpression()), !dbg !14
+  call void @llvm.dbg.value(metadata i32 123, metadata !13, metadata !DIExpression()), !dbg !15
+  tail call void @bar(i32 123) #3, !dbg !16
+  call void @llvm.dbg.value(metadata i32 %t1a, metadata !13, metadata !DIExpression()), !dbg !15
+  tail call void @bar(i32 %t1a) #3, !dbg !17
+  ret void, !dbg !18
+}
+
+define dso_local void @foo_other_param(i32 %t2a, i32 %t2b) local_unnamed_addr #0 !dbg !19 {
+; CHECK-LABEL: name:            foo_other_param
+; CHECK: DBG_VALUE $edi, $noreg, ![[T2A]], !DIExpression(),
+; CHECK: DBG_VALUE $esi, $noreg, ![[T2B]], !DIExpression(),
+; CHECK: %1:gr32 = COPY $esi
+; CHECK: DBG_VALUE %1, $noreg, ![[T2B]], !DIExpression(),
+; CHECK: %0:gr32 = COPY $edi
+; CHECK: DBG_VALUE %0, $noreg, ![[T2A]], !DIExpression(),
+; CHECK: DBG_VALUE $edi, $noreg, ![[T2B]], !DIExpression(),
+; CHECK: CALL64pcrel32 @bar,
+; CHECK: DBG_VALUE 123, $noreg, ![[T2B]], !DIExpression(),
+; CHECK: CALL64pcrel32 @bar,
+; CHECK: DBG_VALUE %0, $noreg, ![[T2B]], !DIExpression(),
+; CHECK: DBG_VALUE $edi, $noreg, ![[T2A]], !DIExpression(),
+; CHECK: TCRETURNdi64 @bar,
+
+entry:
+  call void @llvm.dbg.value(metadata i32 %t2a, metadata !23, metadata !DIExpression()), !dbg !25
+  call void @llvm.dbg.value(metadata i32 %t2b, metadata !24, metadata !DIExpression()), !dbg !26
+  tail call void @bar(i32 %t2b) #3, !dbg !27
+  call void @llvm.dbg.value(metadata i32 123, metadata !24, metadata !DIExpression()), !dbg !26
+  tail call void @bar(i32 123) #3, !dbg !28
+  call void @llvm.dbg.value(metadata i32 %t2a, metadata !24, metadata !DIExpression()), !dbg !26
+  tail call void @bar(i32 %t2a) #3, !dbg !29
+  ret void, !dbg !30
+}
+
+define dso_local void @foo_same_param(i32 %t3a) local_unnamed_addr #0 !dbg !31 {
+; CHECK-LABEL: name:            foo_same_param
+; CHECK: DBG_VALUE $edi, $noreg, ![[T3A]], !DIExpression(),
+; CHECK: %0:gr32 = COPY $edi
+; CHECK: DBG_VALUE %0, $noreg, ![[T3A]], !DIExpression(),
+; CHECK: CALL64pcrel32 @bar,
+; CHECK: DBG_VALUE %0, $noreg, ![[TMP]], !DIExpression(),
+; CHECK: DBG_VALUE 123, $noreg, ![[T3A]], !DIExpression(),
+; CHECK: CALL64pcrel32 @bar,
+; CHECK: DBG_VALUE %0, $noreg, ![[T3A]], !DIExpression(),
+; CHECK: TCRETURNdi64 @bar,
+entry:
+  call void @llvm.dbg.value(metadata i32 %t3a, metadata !33, metadata !DIExpression()), !dbg !35
+  tail call void @bar(i32 %t3a) #3, !dbg !36
+  call void @llvm.dbg.value(metadata i32 %t3a, metadata !34, metadata !DIExpression()), !dbg !37
+  call void @llvm.dbg.value(metadata i32 123, metadata !33, metadata !DIExpression()), !dbg !35
+  tail call void @bar(i32 123) #3, !dbg !38
+  call void @llvm.dbg.value(metadata i32 %t3a, metadata !33, metadata !DIExpression()), !dbg !35
+  tail call void @bar(i32 %t3a) #3, !dbg !39
+  ret void, !dbg !40
+}
+
+declare void @llvm.dbg.value(metadata, metadata, metadata) #2
+
+declare dso_local void @bar(i32) local_unnamed_addr
+
+attributes #0 = { nounwind uwtable }
+attributes #2 = { nounwind readnone speculatable }
+attributes #3 = { nounwind }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4, !5}
+!llvm.ident = !{!6}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 9.0.0", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, nameTableKind: None)
+!1 = !DIFile(filename: "foo.c", directory: "")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{i32 1, !"wchar_size", i32 4}
+!6 = !{!"clang version 9.0.0"}
+!7 = distinct !DISubprogram(name: "foo_local", scope: !1, file: !1, line: 3, type: !8, scopeLine: 3, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !11)
+!8 = !DISubroutineType(types: !9)
+!9 = !{null, !10}
+!10 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!11 = !{!12, !13}
+!12 = !DILocalVariable(name: "t1a", arg: 1, scope: !7, file: !1, line: 3, type: !10)
+!13 = !DILocalVariable(name: "local", scope: !7, file: !1, line: 4, type: !10)
+!14 = !DILocation(line: 3, column: 20, scope: !7)
+!15 = !DILocation(line: 4, column: 7, scope: !7)
+!16 = !DILocation(line: 5, column: 3, scope: !7)
+!17 = !DILocation(line: 7, column: 3, scope: !7)
+!18 = !DILocation(line: 8, column: 1, scope: !7)
+!19 = distinct !DISubprogram(name: "foo_other_param", scope: !1, file: !1, line: 10, type: !20, scopeLine: 10, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !22)
+!20 = !DISubroutineType(types: !21)
+!21 = !{null, !10, !10}
+!22 = !{!23, !24}
+!23 = !DILocalVariable(name: "t2a", arg: 1, scope: !19, file: !1, line: 10, type: !10)
+!24 = !DILocalVariable(name: "t2b", arg: 2, scope: !19, file: !1, line: 10, type: !10)
+!25 = !DILocation(line: 10, column: 26, scope: !19)
+!26 = !DILocation(line: 10, column: 35, scope: !19)
+!27 = !DILocation(line: 11, column: 3, scope: !19)
+!28 = !DILocation(line: 13, column: 3, scope: !19)
+!29 = !DILocation(line: 15, column: 3, scope: !19)
+!30 = !DILocation(line: 16, column: 1, scope: !19)
+!31 = distinct !DISubprogram(name: "foo_same_param", scope: !1, file: !1, line: 18, type: !8, scopeLine: 18, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !32)
+!32 = !{!33, !34}
+!33 = !DILocalVariable(name: "t3a", arg: 1, scope: !31, file: !1, line: 18, type: !10)
+!34 = !DILocalVariable(name: "tmp", scope: !31, file: !1, line: 20, type: !10)
+!35 = !DILocation(line: 18, column: 25, scope: !31)
+!36 = !DILocation(line: 19, column: 3, scope: !31)
+!37 = !DILocation(line: 20, column: 7, scope: !31)
+!38 = !DILocation(line: 22, column: 3, scope: !31)
+!39 = !DILocation(line: 24, column: 3, scope: !31)
+!40 = !DILocation(line: 25, column: 1, scope: !31)
diff --git a/test/DebugInfo/X86/dbg-value-funcarg2.ll b/test/DebugInfo/X86/dbg-value-funcarg2.ll
new file mode 100644
index 0000000000000000000000000000000000000000..a55103580b6c319c743211e927d74913b6af68e4
--- /dev/null
+++ b/test/DebugInfo/X86/dbg-value-funcarg2.ll
@@ -0,0 +1,128 @@
+; RUN: llc -mtriple=x86_64-unknown-linux-gnu -start-after=codegenprepare -stop-before=expand-isel-pseudos -o - %s | FileCheck %s
+
+; Test case was generated from the following C code,
+; using: clang -g -O1 -S -emit-llvm s.c -o s.ll
+;
+; struct s { long long int i, j; };
+;
+; extern void bar(struct s, struct s, struct s);
+;
+; int f(struct s s1, struct s s2) {
+;   volatile struct s tmp = {0};
+;   bar(s1, s2, tmp);
+;   s1.i = s2.i;
+;   s1.j = s2.j;
+;   return s1.j + s1.j;
+; }
+
+; Catch metadata references for involved variables.
+;
+; CHECK-DAG: ![[S1:.*]] = !DILocalVariable(name: "s1"
+; CHECK-DAG: ![[S2:.*]] = !DILocalVariable(name: "s2"
+
+define dso_local i32 @f(i64 %s1.coerce0, i64 %s1.coerce1, i64 %s2.coerce0, i64 %s2.coerce1) local_unnamed_addr #0 !dbg !7 {
+; We expect DBG_VALUE instructions for the arguments at the entry.
+; CHECK-LABEL: name:            f
+; CHECK-NOT: DBG_VALUE
+; CHECK-DAG: DBG_VALUE $rdi, $noreg, ![[S1]], !DIExpression(DW_OP_LLVM_fragment, 0, 64)
+; CHECK-DAG: DBG_VALUE $rsi, $noreg, ![[S1]], !DIExpression(DW_OP_LLVM_fragment, 64, 64)
+; CHECK-DAG: DBG_VALUE $rdx, $noreg, ![[S2]], !DIExpression(DW_OP_LLVM_fragment, 0, 64)
+; CHECK-DAG: DBG_VALUE $rcx, $noreg, ![[S2]], !DIExpression(DW_OP_LLVM_fragment, 64, 64)
+; CHECK-NOT: DBG_VALUE
+
+; Then arguments are copied to virtual registers.
+; CHECK-NOT: DBG_VALUE
+; CHECK-DAG: %[[R1:.*]]:gr64 = COPY $rcx
+; CHECK-DAG: DBG_VALUE %[[R1]], $noreg, ![[S2]], !DIExpression(DW_OP_LLVM_fragment, 64, 64)
+; CHECK-DAG: %[[R2:.*]]:gr64 = COPY $rdx
+; CHECK-DAG: DBG_VALUE %[[R2]], $noreg, ![[S2]], !DIExpression(DW_OP_LLVM_fragment, 0, 64)
+; CHECK-DAG: %[[R3:.*]]:gr64 = COPY $rsi
+; CHECK-DAG: DBG_VALUE %[[R3]], $noreg, ![[S1]], !DIExpression(DW_OP_LLVM_fragment, 64, 64)
+; CHECK-DAG: %[[R4:.*]]:gr64 = COPY $rdi
+; CHECK-DAG: DBG_VALUE %[[R4]], $noreg, ![[S1]], !DIExpression(DW_OP_LLVM_fragment, 0, 64)
+; CHECK-NOT: DBG_VALUE
+
+; We have the call to bar.
+; CHECK:     ADJCALLSTACKDOWN
+; CHECK:     CALL64pcrel32 @bar
+
+; After the call we expect to find new DBG_VALUE instructions for "s1".
+; CHECK:     ADJCALLSTACKUP
+; CHECK-NOT: DBG_VALUE
+; CHECK-DAG: DBG_VALUE %[[R2]], $noreg, ![[S1]], !DIExpression(DW_OP_LLVM_fragment, 0, 64)
+; CHECK-DAG: DBG_VALUE %[[R1]], $noreg, ![[S1]], !DIExpression(DW_OP_LLVM_fragment, 64, 64)
+
+; And then no more DBG_VALUE instructions before the add.
+; CHECK-NOT: DBG_VALUE
+; CHECK:     ADD32rr
+
+entry:
+  %tmp.sroa.0 = alloca i64, align 8
+  %tmp.sroa.4 = alloca i64, align 8
+  call void @llvm.dbg.declare(metadata i64* %tmp.sroa.0, metadata !19, metadata !DIExpression(DW_OP_LLVM_fragment, 0, 64)), !dbg !21
+  call void @llvm.dbg.declare(metadata i64* %tmp.sroa.4, metadata !19, metadata !DIExpression(DW_OP_LLVM_fragment, 64, 64)), !dbg !21
+  call void @llvm.dbg.value(metadata i64 %s1.coerce0, metadata !17, metadata !DIExpression(DW_OP_LLVM_fragment, 0, 64)), !dbg !22
+  call void @llvm.dbg.value(metadata i64 %s1.coerce1, metadata !17, metadata !DIExpression(DW_OP_LLVM_fragment, 64, 64)), !dbg !22
+  call void @llvm.dbg.value(metadata i64 %s2.coerce0, metadata !18, metadata !DIExpression(DW_OP_LLVM_fragment, 0, 64)), !dbg !23
+  call void @llvm.dbg.value(metadata i64 %s2.coerce1, metadata !18, metadata !DIExpression(DW_OP_LLVM_fragment, 64, 64)), !dbg !23
+  %tmp.sroa.0.0..sroa_cast = bitcast i64* %tmp.sroa.0 to i8*, !dbg !24
+  call void @llvm.lifetime.start.p0i8(i64 8, i8* nonnull %tmp.sroa.0.0..sroa_cast), !dbg !24
+  %tmp.sroa.4.0..sroa_cast = bitcast i64* %tmp.sroa.4 to i8*, !dbg !24
+  call void @llvm.lifetime.start.p0i8(i64 8, i8* nonnull %tmp.sroa.4.0..sroa_cast), !dbg !24
+  store volatile i64 0, i64* %tmp.sroa.0, align 8, !dbg !21
+  store volatile i64 0, i64* %tmp.sroa.4, align 8, !dbg !21
+  tail call void @bar(i64 %s1.coerce0, i64 %s1.coerce1, i64 %s2.coerce0, i64 %s2.coerce1, i64 0, i64 0) #4, !dbg !25
+  call void @llvm.dbg.value(metadata i64 %s2.coerce0, metadata !17, metadata !DIExpression(DW_OP_LLVM_fragment, 0, 64)), !dbg !22
+  call void @llvm.dbg.value(metadata i64 %s2.coerce1, metadata !17, metadata !DIExpression(DW_OP_LLVM_fragment, 64, 64)), !dbg !22
+  %s2.coerce1.tr = trunc i64 %s2.coerce1 to i32, !dbg !26
+  %conv = shl i32 %s2.coerce1.tr, 1, !dbg !26
+  call void @llvm.lifetime.end.p0i8(i64 8, i8* nonnull %tmp.sroa.0.0..sroa_cast), !dbg !27
+  call void @llvm.lifetime.end.p0i8(i64 8, i8* nonnull %tmp.sroa.4.0..sroa_cast), !dbg !27
+  ret i32 %conv, !dbg !28
+}
+
+declare void @llvm.dbg.declare(metadata, metadata, metadata) #1
+declare void @llvm.dbg.value(metadata, metadata, metadata) #1
+declare void @llvm.lifetime.start.p0i8(i64, i8* nocapture) #2
+declare void @llvm.lifetime.end.p0i8(i64, i8* nocapture) #2
+
+declare dso_local void @bar(i64, i64, i64, i64, i64, i64) local_unnamed_addr
+
+attributes #0 = { nounwind uwtable }
+attributes #1 = { nounwind readnone speculatable }
+attributes #2 = { argmemonly nounwind }
+attributes #4 = { nounwind }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4, !5}
+!llvm.ident = !{!6}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "llvm-svn @ 353529", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, nameTableKind: None)
+!1 = !DIFile(filename: "s.c", directory: "")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{i32 1, !"wchar_size", i32 4}
+!6 = !{!"llvm-svn @ 353529"}
+!7 = distinct !DISubprogram(name: "f", scope: !1, file: !1, line: 5, type: !8, scopeLine: 5, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !16)
+!8 = !DISubroutineType(types: !9)
+!9 = !{!10, !11, !11}
+!10 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!11 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "s", file: !1, line: 1, size: 128, elements: !12)
+!12 = !{!13, !15}
+!13 = !DIDerivedType(tag: DW_TAG_member, name: "i", scope: !11, file: !1, line: 1, baseType: !14, size: 64)
+!14 = !DIBasicType(name: "long long int", size: 64, encoding: DW_ATE_signed)
+!15 = !DIDerivedType(tag: DW_TAG_member, name: "j", scope: !11, file: !1, line: 1, baseType: !14, size: 64, offset: 64)
+!16 = !{!17, !18, !19}
+!17 = !DILocalVariable(name: "s1", arg: 1, scope: !7, file: !1, line: 5, type: !11)
+!18 = !DILocalVariable(name: "s2", arg: 2, scope: !7, file: !1, line: 5, type: !11)
+!19 = !DILocalVariable(name: "tmp", scope: !7, file: !1, line: 6, type: !20)
+!20 = !DIDerivedType(tag: DW_TAG_volatile_type, baseType: !11)
+!21 = !DILocation(line: 6, column: 21, scope: !7)
+!22 = !DILocation(line: 5, column: 16, scope: !7)
+!23 = !DILocation(line: 5, column: 29, scope: !7)
+!24 = !DILocation(line: 6, column: 3, scope: !7)
+!25 = !DILocation(line: 7, column: 3, scope: !7)
+!26 = !DILocation(line: 10, column: 10, scope: !7)
+!27 = !DILocation(line: 11, column: 1, scope: !7)
+!28 = !DILocation(line: 10, column: 3, scope: !7)
diff --git a/test/DebugInfo/X86/debug-loc-asan.mir b/test/DebugInfo/X86/debug-loc-asan.mir
index e4a6057deefbe05a69dd0c072a86536fbf982535..901c6e28d767768b2fe7e9c1ef5c7aaab8677b22 100644
--- a/test/DebugInfo/X86/debug-loc-asan.mir
+++ b/test/DebugInfo/X86/debug-loc-asan.mir
@@ -27,7 +27,7 @@
 # We expect two location ranges for the variable.
 #
 # First, its address is stored in %rcx:
-# CHECK:      .quad .Lfunc_begin0-.Lfunc_begin0
+# CHECK:      .quad .Ltmp0-.Lfunc_begin0
 # CHECK-NEXT: .quad [[START_LABEL]]-.Lfunc_begin0
 # CHECK: DW_OP_breg2
 # DWARF:       DW_TAG_formal_parameter
diff --git a/test/DebugInfo/X86/debug-loc-offset.mir b/test/DebugInfo/X86/debug-loc-offset.mir
index c5f80d062970e02321981c11f485ed8f6a6cc130..728f9041b36f662de18a5c19a95e786f619a42be 100644
--- a/test/DebugInfo/X86/debug-loc-offset.mir
+++ b/test/DebugInfo/X86/debug-loc-offset.mir
@@ -32,7 +32,7 @@
 # Checking that we have two compile units with two sets of high/lo_pc.
 # CHECK: .debug_info contents
 # CHECK: DW_TAG_compile_unit
-# CHECK: DW_AT_low_pc {{.*}} (0x0000000000000020)
+# CHECK: DW_AT_low_pc {{.*}} (0x0000000000000020 ".text")
 # CHECK: DW_AT_high_pc
 #
 # CHECK: DW_TAG_subprogram
@@ -42,7 +42,7 @@
 # CHECK: DW_TAG_formal_parameter
 # CHECK-NOT: DW_TAG
 # CHECK:       DW_AT_location [DW_FORM_sec_offset]   ({{.*}}
-# CHECK-NEXT:    [0x00000020, 0x00000037): DW_OP_breg0 EAX+0, DW_OP_deref
+# CHECK-NEXT:    [0x00000029, 0x00000037): DW_OP_breg0 EAX+0, DW_OP_deref
 # CHECK-NEXT:    [0x00000037, 0x00000063): DW_OP_breg5 EBP-8, DW_OP_deref, DW_OP_deref
 # CHECK-NEXT:  DW_AT_name [DW_FORM_strp]{{.*}}"a"
 #
@@ -51,7 +51,7 @@
 # CHECK-NOT: DW_AT_location
 #
 # CHECK: DW_TAG_compile_unit
-# CHECK: DW_AT_low_pc {{.*}} (0x0000000000000000)
+# CHECK: DW_AT_low_pc {{.*}} (0x0000000000000000 ".text")
 # CHECK: DW_AT_high_pc
 #
 # CHECK: DW_TAG_subprogram
@@ -70,7 +70,7 @@
 # CHECK-NEXT:    [0x00000000, 0x0000000a): DW_OP_consts +0, DW_OP_stack_value
 # CHECK-NEXT:    [0x0000000a, 0x00000017): DW_OP_consts +1, DW_OP_stack_value
 # CHECK:       0x00000022:
-# CHECK-NEXT:    [0x00000000, 0x00000017): DW_OP_breg0 EAX+0, DW_OP_deref
+# CHECK-NEXT:    [0x00000009, 0x00000017): DW_OP_breg0 EAX+0, DW_OP_deref
 # CHECK-NEXT:    [0x00000017, 0x00000043): DW_OP_breg5 EBP-8, DW_OP_deref, DW_OP_deref
 --- |
   target triple = "i386-unknown-linux-gnu"
diff --git a/test/DebugInfo/X86/debug_addr.ll b/test/DebugInfo/X86/debug_addr.ll
index b50428a282cf856bce395eac4282cea6c799e5b6..e6dbe7d029d95759412eda0c650abac08a3fe1fc 100644
--- a/test/DebugInfo/X86/debug_addr.ll
+++ b/test/DebugInfo/X86/debug_addr.ll
@@ -32,7 +32,7 @@
 ; DWARF5-NOT: DW_TAG_{{.*}}
 ; DWARF5: DW_AT_GNU_dwo_name{{.*}}test.dwo
 ; DWARF5: DW_AT_addr_base{{.*}}0x00000008
-; DWARF5: DW_AT_low_pc [DW_FORM_addrx] ( indexed (00000000) address = 0x0000000000000000)
+; DWARF5: DW_AT_low_pc [DW_FORM_addrx] (indexed (00000000) address = 0x0000000000000000 ".text")
 ; DWARF5: .debug_addr contents:
 ; DWARF5-NEXT: 0x00000000: Addr Section: length = 0x0000000c, version = 0x0005, addr_size = 0x04, seg_size = 0x00
 ; DWARF5-NEXT: Addrs: [
diff --git a/test/DebugInfo/X86/dw_op_minus_direct.ll b/test/DebugInfo/X86/dw_op_minus_direct.ll
index bc4241c1cc1be81286d876f8f51470c7abcd45f0..efd640019017f8dff10303865f9ad66cb2762a1f 100644
--- a/test/DebugInfo/X86/dw_op_minus_direct.ll
+++ b/test/DebugInfo/X86/dw_op_minus_direct.ll
@@ -17,7 +17,7 @@
 
 ; CHECK: .debug_loc contents:
 ; CHECK: 0x00000000:
-; CHECK-NEXT:   [0x0000000000000000, 0x0000000000000004): DW_OP_breg0 RAX+0, DW_OP_constu 0xffffffff, DW_OP_and, DW_OP_lit1, DW_OP_minus, DW_OP_stack_value
+; CHECK-NEXT:   [0x0000000000000003, 0x0000000000000004): DW_OP_breg0 RAX+0, DW_OP_constu 0xffffffff, DW_OP_and, DW_OP_lit1, DW_OP_minus, DW_OP_stack_value
 ;        rax+0, constu 0xffffffff, and, constu 0x00000001, minus, stack-value
 
 source_filename = "minus.c"
diff --git a/test/DebugInfo/X86/dwarf-public-names.ll b/test/DebugInfo/X86/dwarf-public-names.ll
index a4df3316fdebd39095b9c295d9e16149ba903fe0..30b83df63c251cdcb9a20a05e624a01a5e8b5c3b 100644
--- a/test/DebugInfo/X86/dwarf-public-names.ll
+++ b/test/DebugInfo/X86/dwarf-public-names.ll
@@ -123,7 +123,7 @@ attributes #1 = { nounwind readnone }
 !17 = !DIGlobalVariableExpression(var: !18, expr: !DIExpression())
 !18 = !DIGlobalVariable(name: "global_namespace_variable", linkageName: "_ZN2ns25global_namespace_variableE", scope: !19, file: !3, line: 27, type: !6, isLocal: false, isDefinition: true)
 !19 = !DINamespace(name: "ns", scope: null)
-!20 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !3, producer: "clang version 3.3 (http://llvm.org/git/clang.git a09cd8103a6a719cb2628cdf0c91682250a17bd2) (http://llvm.org/git/llvm.git 47d03cec0afca0c01ae42b82916d1d731716cd20)", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !11, retainedTypes: !11, globals: !21, imports: !11) ; previously: invalid DW_TAG_base_type
+!20 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !3, producer: "clang version 3.3", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !11, retainedTypes: !11, globals: !21, imports: !11) ; previously: invalid DW_TAG_base_type
 !21 = !{!0, !15, !17}
 !22 = !{i32 1, !"Debug Info Version", i32 3}
 !23 = distinct !DISubprogram(name: "member_function", linkageName: "_ZN1C15member_functionEv", scope: null, file: !3, line: 9, type: !8, isLocal: false, isDefinition: true, scopeLine: 9, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, unit: !20, declaration: !7, retainedNodes: !11)
diff --git a/test/DebugInfo/X86/dwarfdump-debug-loclists.test b/test/DebugInfo/X86/dwarfdump-debug-loclists.test
index 669607fe557a36a3ed0465f0b076c90094474ca1..32f2482b5117c0676bd95c19eafca6d4aeab4656 100644
--- a/test/DebugInfo/X86/dwarfdump-debug-loclists.test
+++ b/test/DebugInfo/X86/dwarfdump-debug-loclists.test
@@ -9,7 +9,7 @@
 # CHECK-NEXT:    [0x0000000000000700, 0x0000000000000710): DW_OP_breg5 RDI+0
 
 # CHECK:      .debug_loclists contents:
-# CHECK-NEXT: 0x00000000: locations list header: length = 0x0000002f, version = 0x0005, addr_size = 0x08, seg_size = 0x00, offset_entry_count = 0x00000000
+# CHECK-NEXT: 0x00000000: locations list header: length = 0x0000002c, version = 0x0005, addr_size = 0x08, seg_size = 0x00, offset_entry_count = 0x00000000
 # CHECK-NEXT: 0x00000000:
 # CHECK-NEXT:   [0x0000000000000000, 0x0000000000000010): DW_OP_breg5 RDI+0
 # CHECK-NEXT:   [0x0000000000000530, 0x0000000000000540): DW_OP_breg6 RBP-8, DW_OP_deref
@@ -37,7 +37,7 @@
   .byte  4                       # DW_LLE_offset_pair
   .uleb128 0x0                   #   starting offset
   .uleb128 0x10                  #   ending offset
-  .short  2                      # Loc expr size
+  .byte  2                       # Loc expr size
   .byte  117                     # DW_OP_breg5
   .byte  0                       # 0
   
@@ -47,7 +47,7 @@
   .byte  4                       # DW_LLE_offset_pair
   .uleb128 0x30                  #   starting offset
   .uleb128 0x40                  #   ending offset
-  .short  3                      # Loc expr size
+  .byte  3                       # Loc expr size
   .byte  118                     # DW_OP_breg6
   .byte  120                     # -8
   .byte  6                       # DW_OP_deref
@@ -55,7 +55,7 @@
   .byte  8                       # DW_LLE_start_length
   .quad  0x700                   # Some address
   .uleb128 0x10                  #   length
-  .short  2                      # Loc expr size
+  .byte  2                       # Loc expr size
   .byte  117                     # DW_OP_breg5
   .byte  0                       # 0
   
diff --git a/test/DebugInfo/X86/dwarfdump-ranges-baseaddr.s b/test/DebugInfo/X86/dwarfdump-ranges-baseaddr.s
index 25ea30d737cc3ed87dfcc42ac807bc571f33d296..7d5a6f938bd4b7b86b2449c4608a35eb1658925e 100644
--- a/test/DebugInfo/X86/dwarfdump-ranges-baseaddr.s
+++ b/test/DebugInfo/X86/dwarfdump-ranges-baseaddr.s
@@ -4,7 +4,7 @@
 
 # CHECK: .debug_info contents:
 # CHECK: 0x0000000b: DW_TAG_compile_unit [1]
-# CHECK:             DW_AT_low_pc [DW_FORM_addr]       (0x0000000000000000)
+# CHECK:             DW_AT_low_pc [DW_FORM_addr]       (0x0000000000000000 ".text")
 # CHECK-NEXT:        DW_AT_ranges [DW_FORM_sec_offset] (0x00000000
 # CHECK-NEXT:    [0x0000000000000000, 0x0000000000000001) ".text"
 # CHECK-NEXT:    [0x0000000000000003, 0x0000000000000006) ".text"
diff --git a/test/DebugInfo/X86/dwarfdump-str-offsets-dwp.s b/test/DebugInfo/X86/dwarfdump-str-offsets-dwp.s
index fb095e3ebc25ccb063a85f77be719bc85b5fda3f..6620beaadd84bea5e42e9d1bba13f0eac90c368e 100644
--- a/test/DebugInfo/X86/dwarfdump-str-offsets-dwp.s
+++ b/test/DebugInfo/X86/dwarfdump-str-offsets-dwp.s
@@ -291,33 +291,33 @@ TU2_5_end:
 # CHECK:      Compile Unit
 # CHECK-NOT:  NULL
 # CHECK:      DW_TAG_compile_unit
-# CHECK-NEXT: DW_AT_producer [DW_FORM_strx] ( indexed (00000000) string = "Handmade DWARF producer")
-# CHECK-NEXT: DW_AT_name [DW_FORM_strx] ( indexed (00000001) string = "Compile_Unit_1")
-# CHECK-NEXT: DW_AT_name [DW_FORM_strx] ( indexed (00000002) string = "/home/test/CU1")
+# CHECK-NEXT: DW_AT_producer [DW_FORM_strx] (indexed (00000000) string = "Handmade DWARF producer")
+# CHECK-NEXT: DW_AT_name [DW_FORM_strx] (indexed (00000001) string = "Compile_Unit_1")
+# CHECK-NEXT: DW_AT_name [DW_FORM_strx] (indexed (00000002) string = "/home/test/CU1")
 # CHECK-NOT:  NULL
 
 # CHECK:      Compile Unit
 # CHECK-NOT:  NULL
 # CHECK:      DW_TAG_compile_unit
-# CHECK-NEXT: DW_AT_producer [DW_FORM_strx] ( indexed (00000000) string = "Handmade DWARF producer")
-# CHECK-NEXT: DW_AT_name [DW_FORM_strx] ( indexed (00000001) string = "Compile_Unit_2")
-# CHECK-NEXT: DW_AT_name [DW_FORM_strx] ( indexed (00000002) string = "/home/test/CU2")
+# CHECK-NEXT: DW_AT_producer [DW_FORM_strx] (indexed (00000000) string = "Handmade DWARF producer")
+# CHECK-NEXT: DW_AT_name [DW_FORM_strx] (indexed (00000001) string = "Compile_Unit_2")
+# CHECK-NEXT: DW_AT_name [DW_FORM_strx] (indexed (00000002) string = "/home/test/CU2")
 # 
 # CHECK:      Type Unit
 # CHECK-NOT:  NULL
 # CHECK:      DW_TAG_type_unit
-# CHECK-NEXT: DW_AT_name [DW_FORM_strx] ( indexed (00000003) string = "Type_Unit_1")
+# CHECK-NEXT: DW_AT_name [DW_FORM_strx] (indexed (00000003) string = "Type_Unit_1")
 # CHECK-NOT:  NULL
 # CHECK:      DW_TAG_structure_type
-# CHECK-NEXT: DW_AT_name [DW_FORM_strx] ( indexed (00000004) string = "MyStruct_1")
+# CHECK-NEXT: DW_AT_name [DW_FORM_strx] (indexed (00000004) string = "MyStruct_1")
 #
 # CHECK:      Type Unit
 # CHECK-NOT:  NULL
 # CHECK:      DW_TAG_type_unit
-# CHECK-NEXT: DW_AT_name [DW_FORM_strx] ( indexed (00000003) string = "Type_Unit_2")
+# CHECK-NEXT: DW_AT_name [DW_FORM_strx] (indexed (00000003) string = "Type_Unit_2")
 # CHECK-NOT:  NULL
 # CHECK:      DW_TAG_structure_type
-# CHECK-NEXT: DW_AT_name [DW_FORM_strx] ( indexed (00000004) string = "MyStruct_2")
+# CHECK-NEXT: DW_AT_name [DW_FORM_strx] (indexed (00000004) string = "MyStruct_2")
 
 # Verify the correct offets of the compile and type units contributions in the
 # index tables.
diff --git a/test/DebugInfo/X86/dwarfdump-str-offsets-macho.s b/test/DebugInfo/X86/dwarfdump-str-offsets-macho.s
index b2b5104862e7e97ce106c0e7c0c43ca03190f5ca..1332a94ec3c71d0f5eecb36e89bf0bf30aae4e45 100644
--- a/test/DebugInfo/X86/dwarfdump-str-offsets-macho.s
+++ b/test/DebugInfo/X86/dwarfdump-str-offsets-macho.s
@@ -216,37 +216,37 @@ TU_5_end:
 # COMMON:      .debug_info contents:
 # COMMON-NOT:  contents:     
 # COMMON:      DW_TAG_compile_unit
-# COMMON-NEXT: DW_AT_producer [DW_FORM_strx] ( indexed (00000000) string = "Handmade DWARF producer")
-# COMMON-NEXT: DW_AT_name [DW_FORM_strx] ( indexed (00000001) string = "Compile_Unit_1")
+# COMMON-NEXT: DW_AT_producer [DW_FORM_strx] (indexed (00000000) string = "Handmade DWARF producer")
+# COMMON-NEXT: DW_AT_name [DW_FORM_strx] (indexed (00000001) string = "Compile_Unit_1")
 # COMMON-NEXT: DW_AT_str_offsets_base [DW_FORM_sec_offset] (0x00000008)
-# COMMON-NEXT: DW_AT_comp_dir [DW_FORM_strx] ( indexed (00000002) string = "/home/test/CU1")
+# COMMON-NEXT: DW_AT_comp_dir [DW_FORM_strx] (indexed (00000002) string = "/home/test/CU1")
 # COMMON-NOT:  NULL
 # COMMON:      DW_TAG_subprogram
-# COMMON-NEXT: DW_AT_name [DW_FORM_strx1] ( indexed (00000003) string = "MyFunc")
+# COMMON-NEXT: DW_AT_name [DW_FORM_strx1] (indexed (00000003) string = "MyFunc")
 # COMMON-NOT:  NULL
 # COMMON:      DW_TAG_variable
-# COMMON-NEXT: DW_AT_name [DW_FORM_strx2] ( indexed (00000004) string = "MyVar1")
+# COMMON-NEXT: DW_AT_name [DW_FORM_strx2] (indexed (00000004) string = "MyVar1")
 # COMMON-NOT:  NULL
 # COMMON:      DW_TAG_variable
-# COMMON-NEXT: DW_AT_name [DW_FORM_strx3] ( indexed (00000005) string = "MyVar2")
+# COMMON-NEXT: DW_AT_name [DW_FORM_strx3] (indexed (00000005) string = "MyVar2")
 # COMMON-NOT:  NULL
 # COMMON:      DW_TAG_variable
-# COMMON-NEXT: DW_AT_name [DW_FORM_strx4] ( indexed (00000006) string = "MyVar3")
+# COMMON-NEXT: DW_AT_name [DW_FORM_strx4] (indexed (00000006) string = "MyVar3")
 # 
 # Second compile unit (b.cpp)
 # COMMON:      DW_TAG_compile_unit
-# COMMON-NEXT: DW_AT_producer [DW_FORM_strx] ( indexed (00000000) string = "Handmade DWARF producer")
-# COMMON-NEXT: DW_AT_name [DW_FORM_strx] ( indexed (00000001) string = "Compile_Unit_2")
+# COMMON-NEXT: DW_AT_producer [DW_FORM_strx] (indexed (00000000) string = "Handmade DWARF producer")
+# COMMON-NEXT: DW_AT_name [DW_FORM_strx] (indexed (00000001) string = "Compile_Unit_2")
 # COMMON-NEXT: DW_AT_str_offsets_base [DW_FORM_sec_offset] (0x00000038)
-# COMMON-NEXT: DW_AT_comp_dir [DW_FORM_strx] ( indexed (00000002) string = "/home/test/CU2")
+# COMMON-NEXT: DW_AT_comp_dir [DW_FORM_strx] (indexed (00000002) string = "/home/test/CU2")
 # 
 # The type unit
 # COMMON:      .debug_types contents:
 # COMMON:      DW_TAG_type_unit
-# COMMON-NEXT: DW_AT_name [DW_FORM_strx] ( indexed (00000000) string = "Type_Unit")
+# COMMON-NEXT: DW_AT_name [DW_FORM_strx] (indexed (00000000) string = "Type_Unit")
 # COMMON-NEXT: DW_AT_str_offsets_base [DW_FORM_sec_offset]       (0x00000058)
 # COMMON:      DW_TAG_structure_type
-# COMMON-NEXT: DW_AT_name [DW_FORM_strx] ( indexed (00000001) string = "MyStruct")
+# COMMON-NEXT: DW_AT_name [DW_FORM_strx] (indexed (00000001) string = "MyStruct")
 # 
 # The .debug_str_offsets section
 # COMMON:      .debug_str_offsets contents:
diff --git a/test/DebugInfo/X86/dwarfdump-str-offsets.s b/test/DebugInfo/X86/dwarfdump-str-offsets.s
index 2f4215a04ba94b8ec6a863f8615310341a0a33e4..230c660850692777a1eab1b53ed3451381625113 100644
--- a/test/DebugInfo/X86/dwarfdump-str-offsets.s
+++ b/test/DebugInfo/X86/dwarfdump-str-offsets.s
@@ -334,51 +334,51 @@ TU_split_5_end:
 # COMMON:      .debug_info contents:
 # COMMON-NOT:  contents:     
 # COMMON:      DW_TAG_compile_unit
-# COMMON-NEXT: DW_AT_producer [DW_FORM_strx] ( indexed (00000000) string = "Handmade DWARF producer")
-# COMMON-NEXT: DW_AT_name [DW_FORM_strx] ( indexed (00000001) string = "Compile_Unit_1")
+# COMMON-NEXT: DW_AT_producer [DW_FORM_strx] (indexed (00000000) string = "Handmade DWARF producer")
+# COMMON-NEXT: DW_AT_name [DW_FORM_strx] (indexed (00000001) string = "Compile_Unit_1")
 # COMMON-NEXT: DW_AT_str_offsets_base [DW_FORM_sec_offset] (0x00000008)
-# COMMON-NEXT: DW_AT_comp_dir [DW_FORM_strx] ( indexed (00000002) string = "/home/test/CU1")
+# COMMON-NEXT: DW_AT_comp_dir [DW_FORM_strx] (indexed (00000002) string = "/home/test/CU1")
 # COMMON-NOT:  NULL
 # COMMON:      DW_TAG_subprogram
-# COMMON-NEXT: DW_AT_name [DW_FORM_strx1] ( indexed (00000003) string = "MyFunc")
+# COMMON-NEXT: DW_AT_name [DW_FORM_strx1] (indexed (00000003) string = "MyFunc")
 # COMMON-NOT:  NULL
 # COMMON:      DW_TAG_variable
-# COMMON-NEXT: DW_AT_name [DW_FORM_strx2] ( indexed (00000004) string = "MyVar1")
+# COMMON-NEXT: DW_AT_name [DW_FORM_strx2] (indexed (00000004) string = "MyVar1")
 # COMMON-NOT:  NULL
 # COMMON:      DW_TAG_variable
-# COMMON-NEXT: DW_AT_name [DW_FORM_strx3] ( indexed (00000005) string = "MyVar2")
+# COMMON-NEXT: DW_AT_name [DW_FORM_strx3] (indexed (00000005) string = "MyVar2")
 # COMMON-NOT:  NULL
 # COMMON:      DW_TAG_variable
-# COMMON-NEXT: DW_AT_name [DW_FORM_strx4] ( indexed (00000006) string = "MyVar3")
+# COMMON-NEXT: DW_AT_name [DW_FORM_strx4] (indexed (00000006) string = "MyVar3")
 # 
 # Second compile unit (b.cpp)
 # COMMON:      DW_TAG_compile_unit
-# COMMON-NEXT: DW_AT_producer [DW_FORM_strx] ( indexed (00000000) string = "Handmade DWARF producer")
-# COMMON-NEXT: DW_AT_name [DW_FORM_strx] ( indexed (00000001) string = "Compile_Unit_2")
+# COMMON-NEXT: DW_AT_producer [DW_FORM_strx] (indexed (00000000) string = "Handmade DWARF producer")
+# COMMON-NEXT: DW_AT_name [DW_FORM_strx] (indexed (00000001) string = "Compile_Unit_2")
 # COMMON-NEXT: DW_AT_str_offsets_base [DW_FORM_sec_offset] (0x00000038)
-# COMMON-NEXT: DW_AT_comp_dir [DW_FORM_strx] ( indexed (00000002) string = "/home/test/CU2")
+# COMMON-NEXT: DW_AT_comp_dir [DW_FORM_strx] (indexed (00000002) string = "/home/test/CU2")
 # 
 # The split CU
 # SPLIT:       .debug_info.dwo contents:
 # SPLIT-NOT:   contents:
 # SPLIT:       DW_TAG_compile_unit
-# SPLIT-NEXT:  DW_AT_producer [DW_FORM_strx] ( indexed (00000000) string = "Handmade split DWARF producer")
-# SPLIT-NEXT:  DW_AT_name [DW_FORM_strx] ( indexed (00000001) string = "V5_split_compile_unit")
-# SPLIT-NEXT:  DW_AT_comp_dir [DW_FORM_strx] ( indexed (00000002) string = "/home/test/splitCU")
+# SPLIT-NEXT:  DW_AT_producer [DW_FORM_strx] (indexed (00000000) string = "Handmade split DWARF producer")
+# SPLIT-NEXT:  DW_AT_name [DW_FORM_strx] (indexed (00000001) string = "V5_split_compile_unit")
+# SPLIT-NEXT:  DW_AT_comp_dir [DW_FORM_strx] (indexed (00000002) string = "/home/test/splitCU")
 # 
 # The type unit
 # COMMON:      .debug_types contents:
 # COMMON:      DW_TAG_type_unit
-# COMMON-NEXT: DW_AT_name [DW_FORM_strx] ( indexed (00000000) string = "Type_Unit")
+# COMMON-NEXT: DW_AT_name [DW_FORM_strx] (indexed (00000000) string = "Type_Unit")
 # COMMON:      DW_TAG_structure_type
-# COMMON-NEXT: DW_AT_name [DW_FORM_strx] ( indexed (00000001) string = "MyStruct")
+# COMMON-NEXT: DW_AT_name [DW_FORM_strx] (indexed (00000001) string = "MyStruct")
 # 
 # The split type unit
 # SPLIT:       .debug_types.dwo contents:
 # SPLIT:       DW_TAG_type_unit
-# SPLIT-NEXT:  DW_AT_name [DW_FORM_strx] ( indexed (00000003) string = "V5_split_type_unit")
+# SPLIT-NEXT:  DW_AT_name [DW_FORM_strx] (indexed (00000003) string = "V5_split_type_unit")
 # SPLIT:       DW_TAG_structure_type
-# SPLIT-NEXT:  DW_AT_name [DW_FORM_strx] ( indexed (00000004) string = "V5_split_Mystruct")
+# SPLIT-NEXT:  DW_AT_name [DW_FORM_strx] (indexed (00000004) string = "V5_split_Mystruct")
 # 
 # The .debug_str_offsets section
 # COMMON:      .debug_str_offsets contents:
diff --git a/test/DebugInfo/X86/enum-class.ll b/test/DebugInfo/X86/enum-class.ll
index 92398c4202ca64ea00ef041ba0f3a555bf4189b0..19b594ee831d7908e9e10a6e72f0a274b978a589 100644
--- a/test/DebugInfo/X86/enum-class.ll
+++ b/test/DebugInfo/X86/enum-class.ll
@@ -13,13 +13,13 @@ source_filename = "test/DebugInfo/X86/enum-class.ll"
 !0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression())
 !1 = !DIGlobalVariable(name: "a", scope: null, file: !2, line: 4, type: !3, isLocal: false, isDefinition: true)
 !2 = !DIFile(filename: "foo.cpp", directory: "/Users/echristo/tmp")
-!3 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "A", file: !2, line: 1, baseType: !4, size: 32, flags: DIFlagFixedEnum, align: 32, elements: !5)
+!3 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "A", file: !2, line: 1, baseType: !4, size: 32, flags: DIFlagEnumClass, align: 32, elements: !5)
 !4 = !DIBasicType(name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
 !5 = !{!6}
 !6 = !DIEnumerator(name: "A1", value: 1)
 !7 = !DIGlobalVariableExpression(var: !8, expr: !DIExpression()) ; [ DW_TAG_enumerator ]
 !8 = !DIGlobalVariable(name: "b", scope: null, file: !2, line: 5, type: !9, isLocal: false, isDefinition: true)
-!9 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "B", file: !2, line: 2, baseType: !10, size: 64, flags: DIFlagFixedEnum, align: 64, elements: !11)
+!9 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "B", file: !2, line: 2, baseType: !10, size: 64, flags: DIFlagEnumClass, align: 64, elements: !11)
 !10 = !DIBasicType(name: "long unsigned int", size: 64, align: 64, encoding: DW_ATE_unsigned)
 !11 = !{!12}
 !12 = !DIEnumerator(name: "B1", value: 1) ; [ DW_TAG_enumerator ]
diff --git a/test/DebugInfo/X86/fission-cu.ll b/test/DebugInfo/X86/fission-cu.ll
index 95669b6dd598476dec73f454fdd462f9fb68058e..d5cca771435250cf4438b0880abae7637c5e40c4 100644
--- a/test/DebugInfo/X86/fission-cu.ll
+++ b/test/DebugInfo/X86/fission-cu.ll
@@ -65,23 +65,23 @@ source_filename = "test/DebugInfo/X86/fission-cu.ll"
 ; Check that the rest of the compile units have information.
 ; CHECK: .debug_info.dwo contents:
 ; CHECK: DW_TAG_compile_unit
-; CHECK: DW_AT_producer [DW_FORM_GNU_str_index] ( indexed (00000002) string = "clang version 3.3 (trunk 169021) (llvm/trunk 169020)")
+; CHECK: DW_AT_producer [DW_FORM_GNU_str_index] (indexed (00000002) string = "clang version 3.3 (trunk 169021) (llvm/trunk 169020)")
 ; CHECK: DW_AT_language [DW_FORM_data2]        (DW_LANG_C99)
-; CHECK: DW_AT_name [DW_FORM_GNU_str_index]    ( indexed (00000003) string = "baz.c")
-; CHECK: DW_AT_GNU_dwo_name [DW_FORM_GNU_str_index] ( indexed (00000004) string = "baz.dwo")
+; CHECK: DW_AT_name [DW_FORM_GNU_str_index]    (indexed (00000003) string = "baz.c")
+; CHECK: DW_AT_GNU_dwo_name [DW_FORM_GNU_str_index] (indexed (00000004) string = "baz.dwo")
 ; CHECK-NOT: DW_AT_low_pc
 ; CHECK-NOT: DW_AT_stmt_list
 ; CHECK-NOT: DW_AT_comp_dir
 ; CHECK: DW_AT_GNU_dwo_id [DW_FORM_data8]  (0x1f1f859683d49324)
 ; CHECK: DW_TAG_variable
-; CHECK: DW_AT_name [DW_FORM_GNU_str_index]     ( indexed (00000000) string = "a")
+; CHECK: DW_AT_name [DW_FORM_GNU_str_index]     (indexed (00000000) string = "a")
 ; CHECK: DW_AT_type [DW_FORM_ref4]       (cu + 0x{{[0-9a-f]*}} => {[[TYPE:0x[0-9a-f]*]]}
 ; CHECK: DW_AT_external [DW_FORM_flag_present]   (true)
 ; CHECK: DW_AT_decl_file [DW_FORM_data1] (0x01)
 ; CHECK: DW_AT_decl_line [DW_FORM_data1] (1)
 ; CHECK: DW_AT_location [DW_FORM_exprloc] (DW_OP_GNU_addr_index 0x0)
 ; CHECK: [[TYPE]]: DW_TAG_base_type
-; CHECK: DW_AT_name [DW_FORM_GNU_str_index]     ( indexed (00000001) string = "int")
+; CHECK: DW_AT_name [DW_FORM_GNU_str_index]     (indexed (00000001) string = "int")
 
 ; CHECK: .debug_str contents:
 ; CHECK: 0x00000000: "/usr/local/google/home/echristo/tmp"
diff --git a/test/DebugInfo/X86/generate-odr-hash.ll b/test/DebugInfo/X86/generate-odr-hash.ll
index f2776f37a8ab6df9262e69e400ff43d639acea2f..8a46f6db6d856a78ebc9bd0242e04b1418649d39 100644
--- a/test/DebugInfo/X86/generate-odr-hash.ll
+++ b/test/DebugInfo/X86/generate-odr-hash.ll
@@ -80,7 +80,7 @@
 ; CHECK-NOT: type_signature
 ; CHECK-LABEL: type_signature = 0x1d02f3be30cc5688
 ; CHECK: DW_TAG_structure_type
-; FISSION-NEXT: DW_AT_name {{.*}} ( indexed {{.*}} "bar"
+; FISSION-NEXT: DW_AT_name {{.*}} (indexed {{.*}} "bar"
 ; SINGLE-NEXT: DW_AT_name {{.*}} "bar"
 
 ; Check that we generate a hash for fluffy and the value.
diff --git a/test/DebugInfo/X86/gmlt-no-split-dwarf-inlining-empty.ll b/test/DebugInfo/X86/gmlt-no-split-dwarf-inlining-empty.ll
new file mode 100644
index 0000000000000000000000000000000000000000..e540492eb1d5936080074b46a6c6af1a87708638
--- /dev/null
+++ b/test/DebugInfo/X86/gmlt-no-split-dwarf-inlining-empty.ll
@@ -0,0 +1,30 @@
+; RUN: %llc_dwarf -split-dwarf-file=foo.dwo  %s -filetype=obj -o - | llvm-objdump -h - | FileCheck %s
+
+; Created from:
+;   void f1() {
+;   }
+; $ clang-tot gmlt-no-split-dwarf-inlining-empty.c -fno-split-dwarf-inlining -gmlt -gsplit-dwarf -c -emit-llvm -S
+
+; CHECK-NOT: .debug_{{.*}}.dwo
+
+target triple = "x86_64-pc-linux"
+
+define dso_local void @f1() !dbg !7 {
+entry:
+  ret void, !dbg !9
+}
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4, !5}
+!llvm.ident = !{!6}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 9.0.0 (trunk 353744) (llvm/trunk 353759)", isOptimized: false, runtimeVersion: 0, emissionKind: LineTablesOnly, enums: !2, splitDebugInlining: false, nameTableKind: GNU)
+!1 = !DIFile(filename: "gmlt-no-split-dwarf-inlining-empty.c", directory: "/usr/local/google/home/blaikie/dev/scratch")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{i32 1, !"wchar_size", i32 4}
+!6 = !{!"clang version 9.0.0 (trunk 353744) (llvm/trunk 353759)"}
+!7 = distinct !DISubprogram(name: "f1", scope: !1, file: !1, line: 1, type: !8, scopeLine: 1, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !2)
+!8 = !DISubroutineType(types: !2)
+!9 = !DILocation(line: 2, column: 1, scope: !7)
diff --git a/test/DebugInfo/X86/lazy-fission-comp-dir.ll b/test/DebugInfo/X86/lazy-fission-comp-dir.ll
new file mode 100644
index 0000000000000000000000000000000000000000..6055e7ecd3eb00672eafcd39d23b8e4f07b51105
--- /dev/null
+++ b/test/DebugInfo/X86/lazy-fission-comp-dir.ll
@@ -0,0 +1,32 @@
+; RUN: llc -split-dwarf-file=foo.dwo -O0 %s -mtriple=x86_64-unknown-linux-gnu -filetype=obj -o %t
+; RUN: llvm-dwarfdump -debug-info -debug-line %t | FileCheck %s
+
+; CHECK: .debug_info contents:
+; CHECK: DW_AT_comp_dir ("/usr/local/google/home/blaikie/dev/scratch")
+
+; CHECK: .debug_line contents:
+; CHECK: file_names[ 1]:
+; CHECK-NEXT:      name: "main.c"
+; CHECK-NEXT: dir_index: 0
+
+; Function Attrs: noinline nounwind optnone uwtable
+define dso_local i32 @main() !dbg !6 {
+entry:
+  ret i32 0, !dbg !10
+}
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4}
+!llvm.ident = !{!5}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 8.0.0 (trunk 349782) (llvm/trunk 349794)", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, nameTableKind: GNU)
+!1 = !DIFile(filename: "main.c", directory: "/usr/local/google/home/blaikie/dev/scratch")
+!2 = !{}
+!3 = !{i32 2, !"Debug Info Version", i32 3}
+!4 = !{i32 1, !"wchar_size", i32 4}
+!5 = !{!"clang version 8.0.0 (trunk 349782) (llvm/trunk 349794)"}
+!6 = distinct !DISubprogram(name: "main", scope: !1, file: !1, line: 1, type: !7, scopeLine: 1, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !2)
+!7 = !DISubroutineType(types: !8)
+!8 = !{!9}
+!9 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!10 = !DILocation(line: 2, column: 1, scope: !6)
diff --git a/test/DebugInfo/X86/loclists-dwp.ll b/test/DebugInfo/X86/loclists-dwp.ll
index a5ce92270d6da0b6082993026490e759e20ee7e7..cba81a8a952a1bb362b6b952beef5c091bc950f5 100644
--- a/test/DebugInfo/X86/loclists-dwp.ll
+++ b/test/DebugInfo/X86/loclists-dwp.ll
@@ -42,13 +42,13 @@ declare void @llvm.dbg.value(metadata, metadata, metadata)
 !llvm.module.flags = !{!3, !4, !5}
 !llvm.ident = !{!6}
 
-!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !1, producer: "clang version 8.0.0 (https://git.llvm.org/git/clang.git/ 41055c6168135fe539801799e5c5636247cf0302) (https://git.llvm.org/git/llvm.git/ de0558be123ffbb5b5bd692c17dbd57a75fe684f)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, nameTableKind: None)
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !1, producer: "clang version 8.0.0", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, nameTableKind: None)
 !1 = !DIFile(filename: "a.cpp", directory: "/home/test/PRs/PR38990")
 !2 = !{}
 !3 = !{i32 2, !"Dwarf Version", i32 4}
 !4 = !{i32 2, !"Debug Info Version", i32 3}
 !5 = !{i32 1, !"wchar_size", i32 4}
-!6 = !{!"clang version 8.0.0 (https://git.llvm.org/git/clang.git/ 41055c6168135fe539801799e5c5636247cf0302) (https://git.llvm.org/git/llvm.git/ de0558be123ffbb5b5bd692c17dbd57a75fe684f)"}
+!6 = !{!"clang version 8.0.0"}
 !7 = distinct !DISubprogram(name: "a", linkageName: "_Z1ai", scope: !1, file: !1, line: 2, type: !8, isLocal: false, isDefinition: true, scopeLine: 2, flags: DIFlagPrototyped, isOptimized: true, unit: !0, retainedNodes: !11)
 !8 = !DISubroutineType(types: !9)
 !9 = !{null, !10}
diff --git a/test/DebugInfo/X86/noreturn_c11.ll b/test/DebugInfo/X86/noreturn_c11.ll
index d044e585ff29c1b5fd7cc2780775f4cd5223ef38..8afbd9c2a0280bfd489ed9f03e9a50695a04777b 100644
--- a/test/DebugInfo/X86/noreturn_c11.ll
+++ b/test/DebugInfo/X86/noreturn_c11.ll
@@ -36,12 +36,12 @@ attributes #0 = { noreturn }
 !llvm.module.flags = !{!3, !4}
 !llvm.ident = !{!5}
 
-!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 4.0.0 (http://llvm.org/git/clang.git 08946d46f2add8cb241fdc09fc3731dd9dc5ecb5) (http://llvm.org/git/llvm.git d048aeecd34b8c336d1fd44e36c15b0b11c2ea4d)", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !2)
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 4.0.0", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !2)
 !1 = !DIFile(filename: "test.c", directory: "/home/del/test")
 !2 = !{}
 !3 = !{i32 2, !"Dwarf Version", i32 4}
 !4 = !{i32 2, !"Debug Info Version", i32 3}
-!5 = !{!"clang version 4.0.0 (http://llvm.org/git/clang.git 08946d46f2add8cb241fdc09fc3731dd9dc5ecb5) (http://llvm.org/git/llvm.git d048aeecd34b8c336d1fd44e36c15b0b11c2ea4d)"}
+!5 = !{!"clang version 4.0.0"}
 !6 = distinct !DISubprogram(name: "f", scope: !7, file: !7, line: 3, type: !8, isLocal: false, isDefinition: true, scopeLine: 4, flags: DIFlagNoReturn, isOptimized: false, unit: !0, retainedNodes: !2)
 !7 = !DIFile(filename: "./test.c", directory: "/home/del/test")
 !8 = !DISubroutineType(types: !9)
diff --git a/test/DebugInfo/X86/noreturn_cpp11.ll b/test/DebugInfo/X86/noreturn_cpp11.ll
index a4902e807d126ef4c7f20fe8fe317344886eb1b5..d3b6104f794373ad79e999ccafd682dbe2086cd8 100644
--- a/test/DebugInfo/X86/noreturn_cpp11.ll
+++ b/test/DebugInfo/X86/noreturn_cpp11.ll
@@ -42,12 +42,12 @@ attributes #0 = { noreturn }
 !llvm.module.flags = !{!3, !4}
 !llvm.ident = !{!5}
 
-!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !1, producer: "clang version 4.0.0 (http://llvm.org/git/clang.git 08946d46f2add8cb241fdc09fc3731dd9dc5ecb5) (http://llvm.org/git/llvm.git d048aeecd34b8c336d1fd44e36c15b0b11c2ea4d)", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !2)
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !1, producer: "clang version 4.0.0", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !2)
 !1 = !DIFile(filename: "test.cpp", directory: "/home/del/test")
 !2 = !{}
 !3 = !{i32 2, !"Dwarf Version", i32 4}
 !4 = !{i32 2, !"Debug Info Version", i32 3}
-!5 = !{!"clang version 4.0.0 (http://llvm.org/git/clang.git 08946d46f2add8cb241fdc09fc3731dd9dc5ecb5) (http://llvm.org/git/llvm.git d048aeecd34b8c336d1fd44e36c15b0b11c2ea4d)"}
+!5 = !{!"clang version 4.0.0"}
 !6 = distinct !DISubprogram(name: "f", linkageName: "_Z1fv", scope: !1, file: !1, line: 1, type: !7, isLocal: false, isDefinition: true, scopeLine: 1, flags: DIFlagPrototyped | DIFlagNoReturn, isOptimized: false, unit: !0, retainedNodes: !2)
 !7 = !DISubroutineType(types: !8)
 !8 = !{null}
diff --git a/test/DebugInfo/X86/noreturn_objc.ll b/test/DebugInfo/X86/noreturn_objc.ll
index 83c77c5deb152600f77a1e5893e2189cc6197ed2..1a63ab63b007001f5d445e926d47e7120885d258 100644
--- a/test/DebugInfo/X86/noreturn_objc.ll
+++ b/test/DebugInfo/X86/noreturn_objc.ll
@@ -38,12 +38,12 @@ attributes #0 = { noreturn }
 !llvm.module.flags = !{!3, !4}
 !llvm.ident = !{!5}
 
-!0 = distinct !DICompileUnit(language: DW_LANG_ObjC, file: !1, producer: "clang version 4.0.0 (http://llvm.org/git/clang.git 08946d46f2add8cb241fdc09fc3731dd9dc5ecb5) (http://llvm.org/git/llvm.git d048aeecd34b8c336d1fd44e36c15b0b11c2ea4d)", isOptimized: false, runtimeVersion: 1, emissionKind: FullDebug, enums: !2)
+!0 = distinct !DICompileUnit(language: DW_LANG_ObjC, file: !1, producer: "clang version 4.0.0", isOptimized: false, runtimeVersion: 1, emissionKind: FullDebug, enums: !2)
 !1 = !DIFile(filename: "test.m", directory: "/home/del/test/noreturn/objc")
 !2 = !{}
 !3 = !{i32 2, !"Dwarf Version", i32 4}
 !4 = !{i32 2, !"Debug Info Version", i32 3}
-!5 = !{!"clang version 4.0.0 (http://llvm.org/git/clang.git 08946d46f2add8cb241fdc09fc3731dd9dc5ecb5) (http://llvm.org/git/llvm.git d048aeecd34b8c336d1fd44e36c15b0b11c2ea4d)"}
+!5 = !{!"clang version 4.0.0"}
 !6 = distinct !DISubprogram(name: "f", scope: !7, file: !7, line: 2, type: !8, isLocal: false, isDefinition: true, scopeLine: 3, flags: DIFlagNoReturn, isOptimized: false, unit: !0, retainedNodes: !2)
 !7 = !DIFile(filename: "./test.m", directory: "/home/del/test/noreturn/objc")
 !8 = !DISubroutineType(types: !9)
diff --git a/test/DebugInfo/X86/pieces-3.ll b/test/DebugInfo/X86/pieces-3.ll
index 0f705f14f8f3a45e42b3555d1917b3b808795c46..e91bd262e79ccdfd8751f0d6f9fa52770d3d6fb1 100644
--- a/test/DebugInfo/X86/pieces-3.ll
+++ b/test/DebugInfo/X86/pieces-3.ll
@@ -18,12 +18,10 @@
 ; CHECK: DW_TAG_formal_parameter [3]
 ; CHECK-NEXT:   DW_AT_location [DW_FORM_data4]        (
 ; CHECK-NEXT:     [0x0000000000000000, 0x0000000000000007): DW_OP_reg5 RDI, DW_OP_piece 0x8, DW_OP_piece 0x4, DW_OP_reg4 RSI, DW_OP_piece 0x4
-; CHECK-NEXT:     [0x0000000000000007, 0x0000000000000007): DW_OP_reg5 RDI, DW_OP_piece 0x8, DW_OP_piece 0x4, DW_OP_reg0 RAX, DW_OP_piece 0x4)
 ; CHECK-NEXT:   DW_AT_name {{.*}}"outer"
 ; CHECK: DW_TAG_variable
-; CHECK-NEXT:   DW_AT_location [DW_FORM_data4]        (0x00000044
-; CHECK-NEXT:     [0x0000000000000007, 0x0000000000000007): DW_OP_reg0 RAX, DW_OP_piece 0x4)
-; CHECK-NEXT:   "i1"
+; CHECK-NEXT:   DW_AT_name {{.*}}"i1"
+; CHECK-NOT:    DW_AT_location
 
 ; ModuleID = '/Volumes/Data/llvm/test/DebugInfo/X86/sroasplit-2.ll'
 target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
diff --git a/test/DebugInfo/X86/pr40427.ll b/test/DebugInfo/X86/pr40427.ll
new file mode 100644
index 0000000000000000000000000000000000000000..8b3a4b2b366ed52358e0bac71de99516fe433a36
--- /dev/null
+++ b/test/DebugInfo/X86/pr40427.ll
@@ -0,0 +1,48 @@
+; RUN: llc -start-after=codegenprepare -stop-before=expand-isel-pseudos -o - < %s | FileCheck %s
+; Test for correct placement of DBG_VALUE, which in PR40427 is placed before
+; the load instruction it refers to. The circumstance replicated here is where
+; two instructions in a row, trunc and add, begin with no-op Copy{To,From}Reg
+; SDNodes that produce no instructions.
+; The DBG_VALUE instruction should come immediately after the load instruction
+; because the truncate is optimised out, and the DBG_VALUE should be placed
+; in front of the first instruction that occurs after the dbg.value.
+
+; CHECK: ![[DBGVAR:[0-9]+]] = !DILocalVariable(name: "bees",
+
+target triple = "x86_64-unknown-linux-gnu"
+
+define i16 @lolwat(i1 %spoons, i64 *%bees, i16 %yellow, i64 *%more) {
+entry:
+  br i1 %spoons, label %trueb, label %falseb
+trueb:
+  br label %block
+falseb:
+  br label %block
+block:
+; CHECK:      [[PHIREG:%[0-9]+]]:gr64 = PHI %6, %bb.2, %4, %bb.1
+; CHECK-NEXT: [[LOADR:%[0-9]+]]:gr16 = MOV16rm %0
+; CHECK-NEXT: DBG_VALUE [[LOADR]], $noreg, ![[DBGVAR]]
+; CHECK-NEXT: %{{[0-9]+}}:gr32 = IMPLICIT_DEF
+  %foo = phi i64 *[%bees, %trueb], [%more, %falseb]
+  %forks = bitcast i64 *%foo to i32 *
+  %ret = load i32, i32 *%forks, !dbg !6
+  %cast = trunc i32 %ret to i16, !dbg !6
+  call void @llvm.dbg.value(metadata i16 %cast, metadata !1, metadata !DIExpression()), !dbg !6
+  %orly2 = add i16 %yellow, 1
+  br label %bb1
+bb1:
+  %cheese = add i16 %orly2, %cast
+  ret i16 %cheese, !dbg !6
+}
+
+declare void @llvm.dbg.value(metadata, metadata, metadata)
+
+!llvm.module.flags = !{!4}
+!llvm.dbg.cu = !{!2}
+!1 = !DILocalVariable(name: "bees", scope: !5, type: null)
+!2 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !3, producer: "beards", isOptimized: true, runtimeVersion: 4, emissionKind: FullDebug)
+!3 = !DIFile(filename: "bees.cpp", directory: "")
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = distinct !DISubprogram(name: "nope", scope: !2, file: !3, line: 1, unit: !2)
+!6 = !DILocation(line: 0, scope: !5)
+!7 = !DILocalVariable(name: "flannel", scope: !5, type: null)
diff --git a/test/DebugInfo/X86/prolog-params.mir b/test/DebugInfo/X86/prolog-params.mir
new file mode 100644
index 0000000000000000000000000000000000000000..3c6761aa44b4cc62f3f671b8043e2f458cf8cc65
--- /dev/null
+++ b/test/DebugInfo/X86/prolog-params.mir
@@ -0,0 +1,156 @@
+# RUN: llc -mtriple=x86_64-unknown-linux-gnu -run-pass=prologepilog %s -o - | FileCheck %s
+
+# MIR test case produced by compiling:
+#
+# typedef struct { int a; long b; } S1;
+#
+# extern void bar(int *);
+#
+# void foo(int p1, S1 p2, int p3, int p4, int p5, int p6, int p7, int p8) {
+#   int local1 = p1;
+#   int local2 = p6;
+#   int arr[2];
+#   bar(arr);
+# }
+#
+# using the following commands (with some manual stripping of metadata):
+#
+# $ clang -O1 -g -S -emit-llvm
+# $ llc -O1 -stop-before=prologepilog
+
+--- |
+  target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+  target triple = "x86_64-unknown-linux-gnu"
+
+  ; Function Attrs: nounwind uwtable
+  define dso_local void @foo(i32 %p1, i32 %p2.coerce0, i64 %p2.coerce1, i32 %p3, i32 %p4, i32 %p5, i32 %p6, i32 %p7, i32 %p8) local_unnamed_addr #0 !dbg !7 {
+  entry:
+    %arr = alloca [2 x i32], align 4
+    call void @llvm.dbg.value(metadata i32 %p1, metadata !18, metadata !DIExpression()), !dbg !28
+    call void @llvm.dbg.value(metadata i32 %p2.coerce0, metadata !19, metadata !DIExpression(DW_OP_LLVM_fragment, 0, 32)), !dbg !28
+    call void @llvm.dbg.value(metadata i64 %p2.coerce1, metadata !19, metadata !DIExpression(DW_OP_LLVM_fragment, 64, 64)), !dbg !28
+    call void @llvm.dbg.value(metadata i32 %p3, metadata !20, metadata !DIExpression()), !dbg !28
+    call void @llvm.dbg.value(metadata i32 %p4, metadata !21, metadata !DIExpression()), !dbg !28
+    call void @llvm.dbg.value(metadata i32 %p5, metadata !22, metadata !DIExpression()), !dbg !28
+    call void @llvm.dbg.value(metadata i32 %p6, metadata !23, metadata !DIExpression()), !dbg !28
+    call void @llvm.dbg.value(metadata i32 %p7, metadata !24, metadata !DIExpression()), !dbg !28
+    call void @llvm.dbg.value(metadata i32 %p8, metadata !25, metadata !DIExpression()), !dbg !28
+    call void @llvm.dbg.value(metadata i32 %p1, metadata !26, metadata !DIExpression()), !dbg !28
+    call void @llvm.dbg.value(metadata i32 %p6, metadata !27, metadata !DIExpression()), !dbg !28
+    %arraydecay1 = bitcast [2 x i32]* %arr to i32*, !dbg !28
+    call void @bar(i32* nonnull %arraydecay1), !dbg !28
+    ret void, !dbg !28
+  }
+
+  ; Function Attrs: nounwind readnone speculatable
+  declare void @llvm.dbg.declare(metadata, metadata, metadata) #1
+
+  declare dso_local void @bar(i32*) local_unnamed_addr
+
+  ; Function Attrs: nounwind readnone speculatable
+  declare void @llvm.dbg.value(metadata, metadata, metadata) #1
+
+  attributes #0 = { nounwind uwtable }
+  attributes #1 = { nounwind readnone speculatable }
+
+  !llvm.dbg.cu = !{!0}
+  !llvm.module.flags = !{!3, !4, !5}
+  !llvm.ident = !{!6}
+
+  !0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 9.0.0", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, nameTableKind: None)
+  !1 = !DIFile(filename: "params.c", directory: "/")
+  !2 = !{}
+  !3 = !{i32 2, !"Dwarf Version", i32 4}
+  !4 = !{i32 2, !"Debug Info Version", i32 3}
+  !5 = !{i32 1, !"wchar_size", i32 4}
+  !6 = !{!"clang version 9.0.0"}
+  !7 = distinct !DISubprogram(name: "foo", scope: !1, file: !1, line: 5, type: !8, scopeLine: 5, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !17)
+  !8 = !DISubroutineType(types: !9)
+  !9 = !{null, !10, !11, !10, !10, !10, !10, !10, !10}
+  !10 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+  !11 = !DIDerivedType(tag: DW_TAG_typedef, name: "S1", file: !1, line: 1, baseType: !12)
+  !12 = distinct !DICompositeType(tag: DW_TAG_structure_type, file: !1, line: 1, size: 128, elements: !13)
+  !13 = !{!14, !15}
+  !14 = !DIDerivedType(tag: DW_TAG_member, name: "a", scope: !12, file: !1, line: 1, baseType: !10, size: 32)
+  !15 = !DIDerivedType(tag: DW_TAG_member, name: "b", scope: !12, file: !1, line: 1, baseType: !16, size: 64, offset: 64)
+  !16 = !DIBasicType(name: "long int", size: 64, encoding: DW_ATE_signed)
+  !17 = !{!18, !19, !20, !21, !22, !23, !24, !25, !26, !27}
+  !18 = !DILocalVariable(name: "p1", arg: 1, scope: !7, file: !1, line: 5, type: !10)
+  !19 = !DILocalVariable(name: "p2", arg: 2, scope: !7, file: !1, line: 5, type: !11)
+  !20 = !DILocalVariable(name: "p3", arg: 3, scope: !7, file: !1, line: 5, type: !10)
+  !21 = !DILocalVariable(name: "p4", arg: 4, scope: !7, file: !1, line: 5, type: !10)
+  !22 = !DILocalVariable(name: "p5", arg: 5, scope: !7, file: !1, line: 5, type: !10)
+  !23 = !DILocalVariable(name: "p6", arg: 6, scope: !7, file: !1, line: 5, type: !10)
+  !24 = !DILocalVariable(name: "p7", arg: 7, scope: !7, file: !1, line: 5, type: !10)
+  !25 = !DILocalVariable(name: "p8", arg: 8, scope: !7, file: !1, line: 5, type: !10)
+  !26 = !DILocalVariable(name: "local1", scope: !7, file: !1, line: 6, type: !10)
+  !27 = !DILocalVariable(name: "local2", scope: !7, file: !1, line: 7, type: !10)
+  !28 = !DILocation(line: 5, column: 21, scope: !7)
+
+...
+---
+name:            foo
+tracksRegLiveness: true
+fixedStack:
+  - { id: 0, type: default, offset: 16, size: 4, alignment: 16, stack-id: 0,
+      isImmutable: true, isAliased: false, callee-saved-register: '', callee-saved-restored: true }
+  - { id: 1, type: default, offset: 8, size: 4, alignment: 8, stack-id: 0,
+      isImmutable: true, isAliased: false, callee-saved-register: '', callee-saved-restored: true }
+  - { id: 2, type: default, offset: 0, size: 4, alignment: 16, stack-id: 0,
+      isImmutable: true, isAliased: false, callee-saved-register: '', callee-saved-restored: true }
+stack:
+  - { id: 0, name: arr, type: default, offset: 0, size: 8, alignment: 4,
+      stack-id: 0, callee-saved-register: '', callee-saved-restored: true }
+body:             |
+  bb.0.entry:
+    DBG_VALUE $edi, $noreg, !18, !DIExpression(), debug-location !28
+    DBG_VALUE $esi, $noreg, !19, !DIExpression(DW_OP_LLVM_fragment, 0, 32), debug-location !28
+    DBG_VALUE $rdx, $noreg, !19, !DIExpression(DW_OP_LLVM_fragment, 64, 64), debug-location !28
+    DBG_VALUE $ecx, $noreg, !20, !DIExpression(), debug-location !28
+    DBG_VALUE $r8d, $noreg, !21, !DIExpression(), debug-location !28
+    DBG_VALUE %fixed-stack.2, 0, !23, !DIExpression(), debug-location !28
+    DBG_VALUE $r9d, $noreg, !22, !DIExpression(), debug-location !28
+    DBG_VALUE $r9d, $noreg, !23, !DIExpression(), debug-location !28
+    DBG_VALUE %fixed-stack.1, 0, !24, !DIExpression(), debug-location !28
+    DBG_VALUE %fixed-stack.0, 0, !25, !DIExpression(), debug-location !28
+    DBG_VALUE $edi, $noreg, !26, !DIExpression(), debug-location !28
+    DBG_VALUE %fixed-stack.2, 0, !27, !DIExpression(), debug-location !28
+    ADJCALLSTACKDOWN64 0, 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp, debug-location !28
+    renamable $rdi = LEA64r %stack.0.arr, 1, $noreg, 0, $noreg
+    CALL64pcrel32 @bar, csr_64, implicit $rsp, implicit $ssp, implicit $rdi, implicit-def $rsp, implicit-def $ssp, debug-location !28
+    ADJCALLSTACKUP64 0, 0, implicit-def dead $rsp, implicit-def dead $eflags, implicit-def dead $ssp, implicit $rsp, implicit $ssp, debug-location !28
+    RET 0, debug-location !28
+
+...
+
+# CHECK: ![[P1:.*]] = !DILocalVariable(name: "p1"
+# CHECK: ![[P2:.*]] = !DILocalVariable(name: "p2"
+# CHECK: ![[P3:.*]] = !DILocalVariable(name: "p3"
+# CHECK: ![[P4:.*]] = !DILocalVariable(name: "p4"
+# CHECK: ![[P5:.*]] = !DILocalVariable(name: "p5"
+# CHECK: ![[P6:.*]] = !DILocalVariable(name: "p6"
+# CHECK: ![[P7:.*]] = !DILocalVariable(name: "p7"
+# CHECK: ![[P8:.*]] = !DILocalVariable(name: "p8"
+# CHECK: ![[LOCAL1:.*]] = !DILocalVariable(name: "local1"
+# CHECK: ![[LOCAL2:.*]] = !DILocalVariable(name: "local2"
+
+# Please note that an additional DBG_VALUE has been added for p6 to see that
+# FI- and register-described DBG_VALUEs for the same variable are not reordered.
+
+# Parameters passed in registers are emitted before the prologue code.
+# CHECK: bb.0.entry:
+# CHECK-NEXT: DBG_VALUE $edi, $noreg, ![[P1]], !DIExpression()
+# CHECK-NEXT: DBG_VALUE $esi, $noreg, ![[P2]], !DIExpression(DW_OP_LLVM_fragment, 0, 32)
+# CHECK-NEXT: DBG_VALUE $rdx, $noreg, ![[P2]], !DIExpression(DW_OP_LLVM_fragment, 64, 64)
+# CHECK-NEXT: DBG_VALUE $ecx, $noreg, ![[P3]], !DIExpression()
+# CHECK-NEXT: DBG_VALUE $r8d, $noreg, ![[P4]], !DIExpression()
+# CHECK-NEXT: DBG_VALUE $r9d, $noreg, ![[P5]], !DIExpression()
+# CHECK-NEXT: frame-setup PUSH64r
+
+# Locals and stack parameters are emitted after the prologue code.
+# CHECK: DBG_VALUE $rsp, 0, ![[P6]], !DIExpression(DW_OP_plus_uconst, 16)
+# CHECK-NEXT: DBG_VALUE $r9d, $noreg, ![[P6]], !DIExpression()
+# CHECK-NEXT: DBG_VALUE $rsp, 0, ![[P7]], !DIExpression(DW_OP_plus_uconst, 24)
+# CHECK-NEXT: DBG_VALUE $rsp, 0, ![[P8]], !DIExpression(DW_OP_plus_uconst, 32)
+# CHECK-NEXT: DBG_VALUE $edi, $noreg, ![[LOCAL1]], !DIExpression()
+# CHECK-NEXT: DBG_VALUE $rsp, 0, ![[LOCAL2]], !DIExpression(DW_OP_plus_uconst, 16)
diff --git a/test/DebugInfo/X86/prologue-stack.ll b/test/DebugInfo/X86/prologue-stack.ll
index ffce4033a8eac3cf1184db994c6f1ac52777e002..6072543861d35324dce36ca6281e5a6552fa3a8e 100644
--- a/test/DebugInfo/X86/prologue-stack.ll
+++ b/test/DebugInfo/X86/prologue-stack.ll
@@ -1,4 +1,4 @@
-; RUN: llc -fast-isel-sink-local-values -disable-fp-elim -O0 %s -mtriple x86_64-unknown-linux-gnu -o - | FileCheck %s
+; RUN: llc -fast-isel-sink-local-values -frame-pointer=all -O0 %s -mtriple x86_64-unknown-linux-gnu -o - | FileCheck %s
 
 ; int callme(int);
 ; int isel_line_test2() {
diff --git a/test/DebugInfo/X86/sdag-dangling-dbgvalue.ll b/test/DebugInfo/X86/sdag-dangling-dbgvalue.ll
index ca7cfb825716cd54dc6d21983a1c81cad2ef717b..34c97f7a4c07ad27dd72e17ce7f17d32a34a67b9 100644
--- a/test/DebugInfo/X86/sdag-dangling-dbgvalue.ll
+++ b/test/DebugInfo/X86/sdag-dangling-dbgvalue.ll
@@ -101,6 +101,7 @@ entry3:
 ; Verify that the def comes before the for bar4.
 define i32 @test4() local_unnamed_addr #0 !dbg !40 {
 ; CHECK-LABEL: bb.0.entry4
+; CHECK-NEXT:    DBG_VALUE $noreg, $noreg, ![[FOO4]], !DIExpression()
 ; CHECK-NEXT:    DBG_VALUE 0, $noreg, ![[FOO4]], !DIExpression()
 ; CHECK-NEXT:    [[REG4:%[0-9]+]]:gr64 =
 ; CHECK-NEXT:    DBG_VALUE [[REG4]], $noreg, ![[BAR4]], !DIExpression()
@@ -114,6 +115,7 @@ entry4:
 ; Verify that we do not get a DBG_VALUE that maps foo5 to @S here.
 define i32 @test5() local_unnamed_addr #0 !dbg !47 {
 ; CHECK-LABEL: bb.0.entry5:
+; CHECK-NEXT:    DBG_VALUE $noreg, $noreg, ![[FOO5]], !DIExpression()
 ; CHECK-NEXT:    DBG_VALUE 0, $noreg, ![[FOO5]], !DIExpression()
 ; CHECK-NEXT:    [[REG5:%[0-9]+]]:gr64 =
 ; CHECK-NEXT:    DBG_VALUE [[REG5]], $noreg, ![[BAR5]], !DIExpression()
diff --git a/test/DebugInfo/X86/sdag-dbgvalue-phi-use-2.ll b/test/DebugInfo/X86/sdag-dbgvalue-phi-use-2.ll
index 6c6a9597b5a65241c13b7c7bc9b60256416452f6..e68a69d62e8d4183f0c861d01a065c90848eddb3 100644
--- a/test/DebugInfo/X86/sdag-dbgvalue-phi-use-2.ll
+++ b/test/DebugInfo/X86/sdag-dbgvalue-phi-use-2.ll
@@ -44,15 +44,17 @@ for.body:                                         ; preds = %for.body.lr.ph, %fo
 ; CHECK-NEXT: [[REG4:%[0-9]+]]:gr32 = PHI
 ; CHECK-NEXT: DBG_VALUE [[REG3]], $noreg, !16
 ; CHECK-NEXT: DBG_VALUE 555, $noreg, !17
-; XXX: Shouldn't the following DBG_VALUE be placed after the add (ADD32rr).
+; CHECK-NEXT: [[ADDREG:%[0-9]+]]:gr32 = nuw nsw ADD32rr
 ; CHECK-NEXT: DBG_VALUE [[REG2]], $noreg, !17
-; CHECK-NEXT: ADD32rr
-; XXX: Shouldn't the following DBG_VALUE be placed after the mul (LEA etc).
+; CHECK:      [[MULREG:%[0-9]+]]:gr32 = LEA64_32r
 ; CHECK-NEXT: DBG_VALUE 777, $noreg, !17
-; CHECK:      INC32r
-; XXX: Shouldn't the following DBG_VALUE be placed after the icmp (the non-dead implicit def of $eflags)
-; CHECK:      DBG_VALUE [[REG4]]
-; CHECK-NEXT: implicit-def $eflags
+; XXX: The following DBG_VALUE should have stayed below the INC32r
+; CHECK-NEXT: DBG_VALUE [[MULREG]], $noreg, !16
+; CHECK-NEXT: [[INCREG:%[0-9]+]]:gr32 = nuw nsw INC32r
+; CHECK-NEXT: DBG_VALUE [[INCREG]], $noreg, !17
+; CHECK-NEXT: DBG_VALUE [[ADDREG]], $noreg, !15
+; CHECK-NEXT: implicit-def $eflags,
+; CHECK-NEXT: DBG_VALUE [[REG4]]
   %u.023 = phi i32 [ 0, %for.body.lr.ph ], [ %inc, %for.body ]
   %y.022 = phi i32 [ 13, %for.body.lr.ph ], [ %mul, %for.body ]
   %x.021 = phi i32 [ 9, %for.body.lr.ph ], [ %add, %for.body ]
diff --git a/test/DebugInfo/X86/sdag-dbgvalue-ssareg.ll b/test/DebugInfo/X86/sdag-dbgvalue-ssareg.ll
new file mode 100644
index 0000000000000000000000000000000000000000..dd9fac64c028b69f605e6fd9739c4f0cb811887c
--- /dev/null
+++ b/test/DebugInfo/X86/sdag-dbgvalue-ssareg.ll
@@ -0,0 +1,60 @@
+; RUN: llc -start-after=codegenprepare -stop-before expand-isel-pseudos -o - %s | FileCheck %s
+
+; Test that dbg.values of an SSA variable that's not used in a basic block,
+; is converted to a DBG_VALUE in that same basic block. We know that %1 is
+; live from the entry bb to the exit bb, is allocated a vreg because it's
+; used across basic blocks. We should be able to produce a DBG_VALUE that
+; refers to it in the nextbb block.
+
+source_filename = "debug.c"
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; Function Attrs: nounwind uwtable
+define dso_local i32 @main(i32 %arg0, i32 %arg1) local_unnamed_addr !dbg !11 {
+entry:
+  %0 = add i32 %arg0, 42, !dbg !26
+  %1 = add i32 %arg1, 101, !dbg !26
+  %cmp = icmp eq i32 %1, 0
+  br i1 %cmp, label %nextbb, label %exit, !dbg !26
+
+nextbb:
+; CHECK-LABEL: bb.{{.*}}.nextbb
+  %2 = mul i32 %0, %arg1, !dbg !26
+; CHECK: IMUL32rr
+  call void @llvm.dbg.value(metadata i32 %1, metadata !16, metadata !DIExpression()), !dbg !27
+; CHECK-NEXT: DBG_VALUE
+  br label %exit, !dbg !26
+
+; CHECK-LABEL: bb.{{.*}}.exit
+exit:
+  %3 = phi i32 [ 12, %entry ], [ %2, %nextbb ], !dbg !26
+  %4 = add i32 %1, %3, !dbg !26
+  ret i32 %4, !dbg !26
+}
+
+declare void @llvm.dbg.value(metadata, metadata, metadata)
+
+!llvm.dbg.cu = !{!2}
+!llvm.module.flags = !{!7, !8, !9}
+!llvm.ident = !{!10}
+
+!2 = distinct !DICompileUnit(language: DW_LANG_C99, file: !3, producer: "clang version 7.0.0 (x)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !4)
+!3 = !DIFile(filename: "debug.c", directory: "")
+!4 = !{}
+!6 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!7 = !{i32 2, !"Dwarf Version", i32 4}
+!8 = !{i32 2, !"Debug Info Version", i32 3}
+!9 = !{i32 1, !"wchar_size", i32 4}
+!10 = !{!"clang version 7.0.0 (x)"}
+!11 = distinct !DISubprogram(name: "main", scope: !3, file: !3, line: 3, type: !12, isLocal: false, isDefinition: true, scopeLine: 3, isOptimized: true, unit: !2, retainedNodes: !14)
+!12 = !DISubroutineType(types: !13)
+!13 = !{!6}
+!14 = !{!16}
+!16 = !DILocalVariable(name: "y", scope: !11, file: !3, line: 5, type: !6)
+!26 = !DILocation(line: 4, column: 7, scope: !11)
+!27 = !DILocation(line: 5, column: 7, scope: !11)
+!31 = !{!32, !32, i64 0}
+!32 = !{!"int", !33, i64 0}
+!33 = !{!"omnipotent char", !34, i64 0}
+!34 = !{!"Simple C/C++ TBAA"}
diff --git a/test/DebugInfo/X86/sdag-ir-salvage.ll b/test/DebugInfo/X86/sdag-ir-salvage.ll
new file mode 100644
index 0000000000000000000000000000000000000000..0bd5f18dd17ac7782e38b33da54886cd51bb0218
--- /dev/null
+++ b/test/DebugInfo/X86/sdag-ir-salvage.ll
@@ -0,0 +1,56 @@
+; RUN: llc -mtriple=x86_64-unknown-unknown -start-after=codegenprepare -stop-before expand-isel-pseudos %s -o -  | FileCheck %s
+
+; Test that the dbg.value for %baz, which doesn't exist in the 'next' bb,
+; can be salvaged back to the underlying argument vreg.
+
+; CHECK:       ![[AAAVAR:.*]] = !DILocalVariable(name: "aaa",
+; CHECK-LABEL: bb.1.next:
+; CHECK:       DBG_VALUE %{{[0-9]+}}, $noreg, ![[AAAVAR]]
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-linux-gnu"
+
+define i8 @f(i32* %foo) local_unnamed_addr !dbg !6 {
+entry:
+  %bar = getelementptr i32, i32* %foo, i32 4
+  %baz = bitcast i32* %bar to i8*
+  %quux = load i8, i8* %baz
+  br label %next
+
+next:                                             ; preds = %entry
+  tail call void @llvm.dbg.value(metadata i8* %baz, metadata !15, metadata !DIExpression()), !dbg !30
+  %xyzzy = add i8 %quux, 123
+  br label %fin
+
+fin:                                              ; preds = %next
+  %trains = getelementptr i32, i32* %foo, i32 3
+  %planes = bitcast i32* %trains to i8*
+  %cars = load i8, i8* %planes
+  %ret = add i8 %xyzzy, %cars
+  ret i8 %ret
+}
+
+; Function Attrs: nounwind readnone speculatable
+declare void @llvm.dbg.value(metadata, metadata, metadata) #0
+
+attributes #0 = { nounwind readnone speculatable }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!25, !26, !27, !28}
+!llvm.ident = !{!29}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2)
+!1 = !DIFile(filename: "test.c", directory: ".")
+!2 = !{}
+!6 = distinct !DISubprogram(name: "f", scope: !1, file: !1, line: 18, type: !7, scopeLine: 19, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !14)
+!7 = !DISubroutineType(types: !8)
+!8 = !{!13}
+!13 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_unsigned)
+!14 = !{!15}
+!15 = !DILocalVariable(name: "aaa", scope: !6, file: !1, line: 18, type: !13)
+!25 = !{i32 2, !"Dwarf Version", i32 4}
+!26 = !{i32 2, !"Debug Info Version", i32 3}
+!27 = !{i32 1, !"wchar_size", i32 4}
+!28 = !{i32 7, !"PIC Level", i32 2}
+!29 = !{!"clang"}
+!30 = !DILocation(line: 18, column: 14, scope: !6)
diff --git a/test/DebugInfo/X86/string-offsets-multiple-cus.ll b/test/DebugInfo/X86/string-offsets-multiple-cus.ll
index 7851dbfefd5556521dc1fb6456b0db5acf66ae7e..785bbec4481bcb50316c8e570f37ac6b5179613d 100644
--- a/test/DebugInfo/X86/string-offsets-multiple-cus.ll
+++ b/test/DebugInfo/X86/string-offsets-multiple-cus.ll
@@ -43,21 +43,21 @@
 ; TYPEUNITS-NOT:  NULL
 ; TYPEUNITS:      DW_TAG_enumerator
 ; TYPEUNITS-NOT:  {{DW_TAG|NULL}}
-; TYPEUNITS:      DW_AT_name [DW_FORM_strx1] ( indexed (00000005) string = "b")
+; TYPEUNITS:      DW_AT_name [DW_FORM_strx1] (indexed (00000005) string = "b")
 ; TYPEUNITS-NOT:  contents:
 ; TYPEUNITS:      DW_TAG_type_unit
 ; TYPEUNITS-NOT:  {{DW_TAG|NULL}}
 ; TYPEUNITS:      DW_AT_str_offsets_base [DW_FORM_sec_offset] (0x[[CU1_STROFF]])
 ; TYPEUNITS-NOT:  NULL
 ; TYPEUNITS:      DW_TAG_enumeration_type
-; TYPEUNITS:      DW_AT_name [DW_FORM_strx1] ( indexed (0000000d) string = "E2")
+; TYPEUNITS:      DW_AT_name [DW_FORM_strx1] (indexed (0000000d) string = "E2")
 ; TYPEUNITS-NOT:  contents:
 ; TYPEUNITS:      DW_TAG_type_unit
 ; TYPEUNITS-NOT:  {{DW_TAG|NULL}}
 ; TYPEUNITS:      DW_AT_str_offsets_base [DW_FORM_sec_offset] (0x[[CU1_STROFF]])
 ; TYPEUNITS-NOT:  NULL
 ; TYPEUNITS:      DW_TAG_enumeration_type
-; TYPEUNITS:      DW_AT_name [DW_FORM_strx1] ( indexed (00000013) string = "E3")
+; TYPEUNITS:      DW_AT_name [DW_FORM_strx1] (indexed (00000013) string = "E3")
 
 ; CU 1
 ; BOTH-NOT:    .contents:
@@ -74,7 +74,7 @@
 ; BOTH-NOT:    NULL
 ; BOTH:        DW_TAG_variable
 ; BOTH-NOT:    {{DW_TAG|NULL}}
-; BOTH:        DW_AT_name [DW_FORM_strx1] ( indexed (00000009) string = "glob2")
+; BOTH:        DW_AT_name [DW_FORM_strx1] (indexed (00000009) string = "glob2")
 ;
 ; CU 3
 ; BOTH-NOT:    contents:
@@ -84,7 +84,7 @@
 ; BOTH-NOT:    NULL
 ; BOTH:        DW_TAG_variable
 ; BOTH-NOT:    {{DW_TAG|NULL}}
-; BOTH:        DW_AT_name [DW_FORM_strx1] ( indexed (0000000f) string = "glob3")
+; BOTH:        DW_AT_name [DW_FORM_strx1] (indexed (0000000f) string = "glob3")
 ;
 ; Extract the offset of a string to verify that it is referenced in the string
 ; offsets section.
diff --git a/test/DebugInfo/X86/string-offsets-table-order.ll b/test/DebugInfo/X86/string-offsets-table-order.ll
index 1378b134659c76c936284bcb00b5552d3567eda3..e39f70c0e30990970f195cfb5cbc4e8bacd83be3 100644
--- a/test/DebugInfo/X86/string-offsets-table-order.ll
+++ b/test/DebugInfo/X86/string-offsets-table-order.ll
@@ -13,11 +13,11 @@
 
 ; CHECK: .debug_info contents:
 ; CHECK:   DW_TAG_compile_unit
-; CHECK:     DW_AT_comp_dir [DW_FORM_strx1] ( indexed (00000000) string = "X3")
+; CHECK:     DW_AT_comp_dir [DW_FORM_strx1] (indexed (00000000) string = "X3")
 ; CHECK:   DW_TAG_compile_unit
-; CHECK:     DW_AT_comp_dir [DW_FORM_strx1] ( indexed (00000001) string = "X2")
+; CHECK:     DW_AT_comp_dir [DW_FORM_strx1] (indexed (00000001) string = "X2")
 ; CHECK:   DW_TAG_compile_unit
-; CHECK:     DW_AT_comp_dir [DW_FORM_strx1] ( indexed (00000002) string = "X1")
+; CHECK:     DW_AT_comp_dir [DW_FORM_strx1] (indexed (00000002) string = "X1")
 ; CHECK: .debug_info.dwo contents:
 
 ; CHECK: .debug_str contents:
diff --git a/test/DebugInfo/X86/string-offsets-table.ll b/test/DebugInfo/X86/string-offsets-table.ll
index ae7797a3055625cb12952e1c7810f8250474d2de..9960fd833ed98ea5392285fe8a2313f4fd4bc73a 100644
--- a/test/DebugInfo/X86/string-offsets-table.ll
+++ b/test/DebugInfo/X86/string-offsets-table.ll
@@ -27,11 +27,11 @@
 ; MONOLITHIC-NOT:      contents:
 ; MONOLITHIC:          DW_TAG_compile_unit
 ; MONOLITHIC-NOT:      {{DW_TAG|NULL}}
-; MONOLITHIC:          DW_AT_producer [DW_FORM_strx1] ( indexed (00000000) string = "clang{{.*}}")
+; MONOLITHIC:          DW_AT_producer [DW_FORM_strx1] (indexed (00000000) string = "clang{{.*}}")
 ; MONOLITHIC-NOT:      {{DW_TAG|NULL}}
 ; MONOLITHIC:          DW_AT_str_offsets_base [DW_FORM_sec_offset] (0x00000008)
 ; MONOLITHIC-NOT:      {{DW_TAG|NULL}}
-; MONOLITHIC:          DW_AT_comp_dir [DW_FORM_strx1] ( indexed (00000002) string = "/home/{{.*}}")
+; MONOLITHIC:          DW_AT_comp_dir [DW_FORM_strx1] (indexed (00000002) string = "/home/{{.*}}")
 
 ; Extract the string offsets from the .debug_str section so we can check that 
 ; they are referenced correctly in the .debug_str_offsets section.
@@ -59,8 +59,8 @@
 ; SPLIT:      DW_TAG_compile_unit
 ; SPLIT-NOT:  {{DW_TAG|contents:}}
 ; SPLIT:      DW_AT_str_offsets_base [DW_FORM_sec_offset] (0x00000008)
-; SPLIT:      DW_AT_comp_dir [DW_FORM_strx1] ( indexed (00000000) string = "/home/test")
-; SPLIT:      DW_AT_GNU_dwo_name [DW_FORM_strx1] ( indexed (00000001) string = "foo.dwo")
+; SPLIT:      DW_AT_comp_dir [DW_FORM_strx1] (indexed (00000000) string = "/home/test")
+; SPLIT:      DW_AT_GNU_dwo_name [DW_FORM_strx1] (indexed (00000001) string = "foo.dwo")
 
 ; Check for the split CU in .debug_info.dwo.
 ; SPLIT:      .debug_info.dwo contents:
@@ -73,11 +73,11 @@
 ; SPLIT-NOT:  contents:
 ; SPLIT:      DW_TAG_enumerator
 ; SPLIT-NOT:  {{DW_TAG|NULL}}
-; SPLIT:      DW_AT_name [DW_FORM_strx1]    ( indexed (00000001) string = "a")
+; SPLIT:      DW_AT_name [DW_FORM_strx1]    (indexed (00000001) string = "a")
 ; SPLIT-NOT:  contents:
 ; SPLIT:      DW_TAG_enumerator
 ; SPLIT-NOT:  {{DW_TAG|NULL}}
-; SPLIT:      DW_AT_name [DW_FORM_strx1]    ( indexed (00000002) string = "b")
+; SPLIT:      DW_AT_name [DW_FORM_strx1]    (indexed (00000002) string = "b")
 ;
 ; Extract the string offsets referenced in the main file by the skeleton unit.
 ; SPLIT:      .debug_str contents:
diff --git a/test/DebugInfo/arange-overlap.test b/test/DebugInfo/arange-overlap.test
new file mode 100644
index 0000000000000000000000000000000000000000..c29402523ac39f4788cf40e6e70f41957bf4bfc1
--- /dev/null
+++ b/test/DebugInfo/arange-overlap.test
@@ -0,0 +1,4 @@
+RUN: llvm-symbolizer --no-demangle --obj=%p/Inputs/arange-overlap.elf-x86_64 0x714 | FileCheck %s
+
+CHECK: _ZN1S3bazEv
+CHECK-NEXT: {{.*}}arange-overlap.cc:6
diff --git a/test/DebugInfo/cross-cu-inlining.test b/test/DebugInfo/cross-cu-inlining.test
new file mode 100644
index 0000000000000000000000000000000000000000..b5cbffc616d80eea7c6e8a6683d73babe77fe07c
--- /dev/null
+++ b/test/DebugInfo/cross-cu-inlining.test
@@ -0,0 +1,8 @@
+RUN: llvm-symbolizer --obj=%p/Inputs/cross-cu-inlining.x86_64-macho.o 0x17 | FileCheck %s
+
+; func has been inlined into main by LTO. Check that the symbolizer is able
+; to resolve the cross-cu reference and retrieve func's name
+CHECK: func
+CHECK-NEXT: /tmp{{[/\\]}}cross-cu-inlining.c:16:3
+CHECK-NEXT: main
+CHECK-NEXT: /tmp{{[/\\]}}cross-cu-inlining.c:11:0
diff --git a/test/DebugInfo/debuglineinfo.test b/test/DebugInfo/debuglineinfo.test
index 720ba12ed7614b481e5d8054b8224ae1a8f11872..651bbe5a09552455593a8d9480f03822a8700245 100644
--- a/test/DebugInfo/debuglineinfo.test
+++ b/test/DebugInfo/debuglineinfo.test
@@ -1,50 +1,19 @@
 RUN: llvm-rtdyld -printline %p/Inputs/test-inline.o \
-RUN:   | FileCheck %s -check-prefix TEST_INLINE
+RUN:   | FileCheck %s
 RUN: llvm-rtdyld -printdebugline %p/Inputs/test-inline.o \
-RUN:   | FileCheck %s -check-prefix TEST_INLINE
+RUN:   | FileCheck %s
 RUN: llvm-rtdyld -printline %p/Inputs/test-parameters.o \
-RUN:   | FileCheck %s -check-prefix TEST_PARAMETERS
+RUN:   | FileCheck %s
 RUN: llvm-rtdyld -printdebugline %p/Inputs/test-parameters.o \
-RUN:   | FileCheck %s -check-prefix TEST_PARAMETERS
+RUN:   | FileCheck %s
 
 ; This test verifies that relocations are correctly applied to the
 ; .debug_line section and exercises DIContext::getLineInfoForAddressRange().
-; If relocations are not applied the first two functions will be reported as
-; both starting at address zero in the; line number table.
-TEST_INLINE:      Function: _Z15test_parametersPfPA2_dR11char_structPPitm, Size = 170
-TEST_INLINE-NEXT: Line info @ 0: test-inline.cpp, line:33
-TEST_INLINE-NEXT: Line info @ 35: test-inline.cpp, line:34
-TEST_INLINE-NEXT: Line info @ 165: test-inline.cpp, line:35
-TEST_INLINE-NEXT: Function: _Z3foov, Size = 3
-TEST_INLINE-NEXT: Line info @ 0: test-inline.cpp, line:28
-TEST_INLINE-NEXT: Line info @ 2: test-inline.cpp, line:29
-TEST_INLINE-NEXT: Function: main, Size = 146
-TEST_INLINE-NEXT: Line info @ 0: test-inline.cpp, line:39
-TEST_INLINE-NEXT: Line info @ 21: test-inline.cpp, line:41
-TEST_INLINE-NEXT: Line info @ 39: test-inline.cpp, line:42
-TEST_INLINE-NEXT: Line info @ 60: test-inline.cpp, line:44
-TEST_INLINE-NEXT: Line info @ 80: test-inline.cpp, line:48
-TEST_INLINE-NEXT: Line info @ 90: test-inline.cpp, line:45
-TEST_INLINE-NEXT: Line info @ 95: test-inline.cpp, line:46
-TEST_INLINE-NEXT: Line info @ 114: test-inline.cpp, line:48 
-TEST_INLINE-NEXT: Line info @ 141: test-inline.cpp, line:49
-
-; This test checks the case where all code is in a single section.
-TEST_PARAMETERS:      Function: _Z15test_parametersPfPA2_dR11char_structPPitm, Size = 170
-TEST_PARAMETERS-NEXT: Line info @ 0: test-parameters.cpp, line:33
-TEST_PARAMETERS-NEXT: Line info @ 35: test-parameters.cpp, line:34
-TEST_PARAMETERS-NEXT: Line info @ 165: test-parameters.cpp, line:35
-TEST_PARAMETERS-NEXT: Function: _Z3foov, Size = 3
-TEST_PARAMETERS-NEXT: Line info @ 0: test-parameters.cpp, line:28
-TEST_PARAMETERS-NEXT: Line info @ 2: test-parameters.cpp, line:29
-TEST_PARAMETERS-NEXT: Function: main, Size = 146
-TEST_PARAMETERS-NEXT: Line info @ 0: test-parameters.cpp, line:39
-TEST_PARAMETERS-NEXT: Line info @ 21: test-parameters.cpp, line:41
-TEST_PARAMETERS-NEXT: Line info @ 39: test-parameters.cpp, line:42
-TEST_PARAMETERS-NEXT: Line info @ 60: test-parameters.cpp, line:44
-TEST_PARAMETERS-NEXT: Line info @ 80: test-parameters.cpp, line:48
-TEST_PARAMETERS-NEXT: Line info @ 90: test-parameters.cpp, line:45
-TEST_PARAMETERS-NEXT: Line info @ 95: test-parameters.cpp, line:46
-TEST_PARAMETERS-NEXT: Line info @ 114: test-parameters.cpp, line:48 
-TEST_PARAMETERS-NEXT: Line info @ 141: test-parameters.cpp, line:49
 
+CHECK:      Function: _Z2f1v, Size = 6
+CHECK-NEXT:   Line info @ 0: test-inline.cpp, line:1
+CHECK-NEXT:   Line info @ 4: test-inline.cpp, line:1
+CHECK-NEXT: Function: _Z2f2v, Size = 11
+CHECK-NEXT:   Line info @ 0: test-inline.cpp, line:2
+CHECK-NEXT:   Line info @ 4: test-inline.cpp, line:2
+CHECK-NEXT:   Line info @ 9: test-inline.cpp, line:2
diff --git a/test/DebugInfo/gcc-local-mem-func.test b/test/DebugInfo/gcc-local-mem-func.test
new file mode 100644
index 0000000000000000000000000000000000000000..2eccd5a1299ccc519d46f97ff8b2a7967a22a499
--- /dev/null
+++ b/test/DebugInfo/gcc-local-mem-func.test
@@ -0,0 +1,5 @@
+RUN: llvm-symbolizer --no-demangle --obj=%p/Inputs/llvm-symbolizer-local-mem-func-gcc.elf-x86-64 0x61a | FileCheck %s
+
+CHECK-NOT:  local_mem_func
+CHECK:      _ZZ2f1vEN3foo14local_mem_funcEv
+CHECK-NEXT: {{.*}}local-mem-func.cpp:3:0
diff --git a/test/DebugInfo/high-pc-constant.test b/test/DebugInfo/high-pc-constant.test
new file mode 100644
index 0000000000000000000000000000000000000000..79007003e4c1a94f6ebae79868384afabaac2283
--- /dev/null
+++ b/test/DebugInfo/high-pc-constant.test
@@ -0,0 +1,10 @@
+RUN: llvm-symbolizer --inlining --obj=%p/Inputs/dwarfdump-inl-test.high_pc.elf-x86-64 0x568 | FileCheck %s
+
+CHECK:      inlined_h
+CHECK-NEXT: dwarfdump-inl-test.h:3
+CHECK-NEXT: inlined_g
+CHECK-NEXT: dwarfdump-inl-test.h:7
+CHECK-NEXT: inlined_f
+CHECK-NEXT: dwarfdump-inl-test.cc:3
+CHECK-NEXT: main
+CHECK-NEXT: dwarfdump-inl-test.cc:8
diff --git a/test/DebugInfo/llvm-symbolizer-split-dwarf-empty.test b/test/DebugInfo/llvm-symbolizer-split-dwarf-empty.test
deleted file mode 100644
index 994a58e73c08e07679a63a583cfe6a1ff64bc346..0000000000000000000000000000000000000000
--- a/test/DebugInfo/llvm-symbolizer-split-dwarf-empty.test
+++ /dev/null
@@ -1,10 +0,0 @@
-RUN: rm -rf %t && mkdir -p %t
-RUN: cd %t
-RUN: cp %p/Inputs/split-dwarf-empty.dwo %t
-RUN: echo "%p/Inputs/split-dwarf-empty.o 0xdeadbeef" > %t.input
-
-RUN: llvm-symbolizer --functions=linkage --inlining --demangle=false \
-RUN:    --default-arch=i386 < %t.input | FileCheck %s
-
-CHECK: ??
-CHECK: ??:0:0
diff --git a/test/DebugInfo/llvm-symbolizer-split-dwarf-no-skel-address.test b/test/DebugInfo/llvm-symbolizer-split-dwarf-no-skel-address.test
deleted file mode 100644
index b6c9ccc2d170c1f567d2c73d478e18dfee83d7f8..0000000000000000000000000000000000000000
--- a/test/DebugInfo/llvm-symbolizer-split-dwarf-no-skel-address.test
+++ /dev/null
@@ -1,19 +0,0 @@
-RUN: rm -rf %t && mkdir -p %t
-RUN: cd %t
-RUN: cp %p/Inputs/split-dwarf-no-skel-address.dwo %t
-RUN: echo "%p/Inputs/split-dwarf-no-skel-address.o 0x4" > %t.input
-
-RUN: llvm-symbolizer --functions=linkage --inlining --demangle=false \
-RUN:    --default-arch=i386 < %t.input | FileCheck %s
-
-Built from the following source:
-void f1();
-__attribute__((always_inline)) inline void f2() { f1(); }
-void f3() { f2(); }
-
-Compiled to assembly with clang, modified the skeleton CU to remove the
-high/low pc (& update the CU length field and abbrev to match) & then
-compile/objcopy to make the .o and .dwo.
-
-CHECK: _Z2f2v
-CHECK: test.cpp:2:51
diff --git a/test/DebugInfo/llvm-symbolizer-tu.test b/test/DebugInfo/llvm-symbolizer-tu.test
deleted file mode 100644
index c89a4f14d33cbfa0b3663c464d6457c58bf29a66..0000000000000000000000000000000000000000
--- a/test/DebugInfo/llvm-symbolizer-tu.test
+++ /dev/null
@@ -1,8 +0,0 @@
-RUN: echo "%p/Inputs/llvm-symbolizer-tu.elf-x86-64 0x0" \
-RUN:   | llvm-symbolizer | FileCheck %s
-
-Built from the following source:
-
-...
-
-CHECK: b.cpp:3:0
diff --git a/test/DebugInfo/llvm-symbolizer-zlib.test b/test/DebugInfo/llvm-symbolizer-zlib.test
deleted file mode 100644
index 2860d090eb8bbe3531de453a4b0eadcb5e4bdecf..0000000000000000000000000000000000000000
--- a/test/DebugInfo/llvm-symbolizer-zlib.test
+++ /dev/null
@@ -1,10 +0,0 @@
-REQUIRES: zlib
-
-RUN: echo "%p/Inputs/dwarfdump-test-zlib.elf-x86-64 0x4005e9" > %t1.input
-RUN: llvm-symbolizer < %t1.input | FileCheck %s
-
-RUN: echo "%p/Inputs/dwarfdump-test-zlibgnu.elf-x86-64 0x4005e9" > %t2.input
-RUN: llvm-symbolizer < %t2.input | FileCheck %s
-
-CHECK: main
-CHECK-NEXT: /tmp/dbginfo{{[/\\]}}dwarfdump-test-zlib.cc:16
diff --git a/test/DebugInfo/llvm-symbolizer.test b/test/DebugInfo/llvm-symbolizer.test
deleted file mode 100644
index a6a15491c205d00b254d2f22b6bab018edde49fe..0000000000000000000000000000000000000000
--- a/test/DebugInfo/llvm-symbolizer.test
+++ /dev/null
@@ -1,214 +0,0 @@
-RUN: rm -rf %t
-RUN: mkdir -p %t
-RUN: echo "%p/Inputs/dwarfdump-test.elf-x86-64 0x400559" > %t.input
-RUN: echo "%p/Inputs/dwarfdump-test.elf-x86-64.debuglink 0x400559" >> %t.input
-RUN: echo "%p/Inputs/dwarfdump-test.elf-x86-64 0x400436" >> %t.input
-RUN: echo "%p/Inputs/dwarfdump-test.elf-x86-64 0x400528" >> %t.input
-RUN: echo "%p/Inputs/dwarfdump-test.elf-x86-64 0x400586" >> %t.input
-RUN: echo "%p/Inputs/dwarfdump-test2.elf-x86-64 0x4004e8" >> %t.input
-RUN: echo "%p/Inputs/dwarfdump-test2.elf-x86-64 0x4004f4" >> %t.input
-RUN: echo "%p/Inputs/dwarfdump-test4.elf-x86-64 0x62c" >> %t.input
-RUN: echo "%p/Inputs/dwarfdump-inl-test.elf-x86-64 0x8dc" >> %t.input
-RUN: echo "%p/Inputs/dwarfdump-inl-test.elf-x86-64 0xa05" >> %t.input
-RUN: echo "%p/Inputs/dwarfdump-inl-test.elf-x86-64 0x987" >> %t.input
-RUN: echo "%p/Inputs/dwarfdump-inl-test.high_pc.elf-x86-64 0x568" >> %t.input
-RUN: cp "%p/Inputs/dwarfdump-test3.elf-x86-64-space" "%t/dwarfdump-test3.elf-x86-64 space"
-RUN: echo "\"%t/dwarfdump-test3.elf-x86-64 space\" 0x640" >> %t.input
-RUN: echo "\"%t/dwarfdump-test3.elf-x86-64 space\" 0x633" >> %t.input
-RUN: echo "\"%t/dwarfdump-test3.elf-x86-64 space\" 0x62d" >> %t.input
-RUN: echo "%p/Inputs/macho-universal 0x1f84" >> %t.input
-RUN: echo "%p/Inputs/macho-universal:i386 0x1f67" >> %t.input
-RUN: echo "%p/Inputs/macho-universal:x86_64 0x100000f05" >> %t.input
-RUN: echo "%p/Inputs/llvm-symbolizer-dwo-test 0x400514" >> %t.input
-RUN: echo "%p/Inputs/llvm-symbolizer-local-mem-func-gcc.elf-x86-64 0x61a" >> %t.input
-RUN: echo "%p/Inputs/fission-ranges.elf-x86_64 0x720" >> %t.input
-RUN: echo "%p/Inputs/arange-overlap.elf-x86_64 0x714" >> %t.input
-RUN: cp %p/Inputs/split-dwarf-test.dwo %t
-RUN: echo "%p/Inputs/split-dwarf-test 0x400504" >> %t.input
-RUN: echo "%p/Inputs/split-dwarf-test 0x4004f4" >> %t.input
-RUN: echo "%p/Inputs/cross-cu-inlining.x86_64-macho.o 0x17" >> %t.input
-RUN: cp %p/Inputs/split-dwarf-multiple-cu.dwo %t
-RUN: echo "%p/Inputs/split-dwarf-multiple-cu.o 0x4" >> %t.input
-RUN: cp %p/Inputs/split-dwarf-addr-object-relocation.dwo %t
-RUN: echo "%p/Inputs/split-dwarf-addr-object-relocation.o 0x14" >> %t.input
-RUN: cp %p/Inputs/split-dwarf-dwp.o %t
-RUN: cp %p/Inputs/split-dwarf-dwp.o.dwp %t
-RUN: echo "%t/split-dwarf-dwp.o 0x4" >> %t.input
-
-RUN: cd %t
-RUN: llvm-symbolizer --functions=linkage --inlining --demangle=false \
-RUN:    --default-arch=i386 < %t.input | FileCheck --check-prefix=CHECK --check-prefix=SPLIT --check-prefix=DWO %s
-
-Ensure we get the same results in the absence of gmlt-like data in the executable but the presence of a .dwo file
-
-RUN: echo "%p/Inputs/split-dwarf-test-nogmlt 0x400504" >> %t.input
-RUN: echo "%p/Inputs/split-dwarf-test-nogmlt 0x4004f4" >> %t.input
-RUN: llvm-symbolizer --functions=linkage --inlining --demangle=false \
-RUN:    --default-arch=i386 < %t.input | FileCheck --check-prefix=SPLIT --check-prefix=DWO %s
-
-Ensure we get gmlt like results in the absence of a .dwo file but the presence of gmlt-like data in the executable
-
-RUN: rm %t/split-dwarf-test.dwo
-RUN: echo "%p/Inputs/split-dwarf-test 0x400504" >> %t.input
-RUN: echo "%p/Inputs/split-dwarf-test 0x4004f4" >> %t.input
-RUN: llvm-symbolizer --functions=linkage --inlining --demangle=false \
-RUN:    --default-arch=i386 < %t.input | FileCheck --check-prefix=SPLIT --check-prefix=NODWO %s
-
-RUN: cp %p/Inputs/split-dwarf-dwp.o %t/split-dwarf-dwp-different-name.o
-RUN: echo "%t/split-dwarf-dwp-different-name.o 0x54" > %t.input
-RUN: llvm-symbolizer --functions=linkage --inlining --demangle=false \
-RUN:    --default-arch=i386 --dwp=%p/Inputs/split-dwarf-dwp.o.dwp < %t.input | FileCheck --check-prefix=DWP %s
-
-CHECK:       main
-CHECK-NEXT: /tmp/dbginfo{{[/\\]}}dwarfdump-test.cc:16
-
-CHECK:       main
-CHECK-NEXT: /tmp/dbginfo{{[/\\]}}dwarfdump-test.cc:16
-
-CHECK:      _start
-
-CHECK: _Z1fii
-CHECK-NEXT: /tmp/dbginfo{{[/\\]}}dwarfdump-test.cc:11
-
-CHECK: DummyClass
-CHECK-NEXT: /tmp/dbginfo{{[/\\]}}dwarfdump-test.cc:4
-
-CHECK: a
-CHECK-NEXT: /tmp/dbginfo{{[/\\]}}dwarfdump-test2-helper.cc:2
-
-CHECK: main
-CHECK-NEXT: /tmp/dbginfo{{[/\\]}}dwarfdump-test2-main.cc:4
-
-CHECK:      _Z1cv
-CHECK-NEXT: /tmp/dbginfo{{[/\\]}}dwarfdump-test4-part1.cc:2
-
-CHECK:      inlined_h
-CHECK-NEXT: dwarfdump-inl-test.h:2
-CHECK-NEXT: inlined_g
-CHECK-NEXT: dwarfdump-inl-test.h:7
-CHECK-NEXT: inlined_f
-CHECK-NEXT: dwarfdump-inl-test.cc:3
-CHECK-NEXT: main
-CHECK-NEXT: dwarfdump-inl-test.cc:8
-
-CHECK:      inlined_g
-CHECK-NEXT: dwarfdump-inl-test.h:7
-CHECK-NEXT: inlined_f
-CHECK-NEXT: dwarfdump-inl-test.cc:3
-CHECK-NEXT: main
-CHECK-NEXT: dwarfdump-inl-test.cc:8
-
-CHECK:      inlined_f
-CHECK-NEXT: dwarfdump-inl-test.cc:3
-CHECK-NEXT: main
-CHECK-NEXT: dwarfdump-inl-test.cc:8
-
-CHECK:      inlined_h
-CHECK-NEXT: dwarfdump-inl-test.h:3
-CHECK-NEXT: inlined_g
-CHECK-NEXT: dwarfdump-inl-test.h:7
-CHECK-NEXT: inlined_f
-CHECK-NEXT: dwarfdump-inl-test.cc:3
-CHECK-NEXT: main
-CHECK-NEXT: dwarfdump-inl-test.cc:8
-
-CHECK: C
-CHECK-NEXT: /tmp/dbginfo{{[/\\]}}dwarfdump-test3.cc:3
-
-CHECK: _Z3do1v
-CHECK-NEXT: /tmp/include{{[/\\]}}dwarfdump-test3-decl.h:7
-
-CHECK: _Z3do2v
-CHECK-NEXT: /tmp/dbginfo{{[/\\]}}include{{[/\\]}}dwarfdump-test3-decl2.h:1
-
-CHECK:      main
-CHECK:      _Z3inci
-CHECK:      _Z3inci
-
-CHECK: main
-CHECK-NEXT: llvm-symbolizer-dwo-test.cc:11
-
-CHECK-NOT: local_mem_func
-CHECK: _ZZ2f1vEN3foo14local_mem_funcEv
-CHECK-NEXT: {{.*}}local-mem-func.cpp:3:0
-
-CHECK: main
-CHECK-NEXT: {{.*}}fission-ranges.cc:6
-
-CHECK: _ZN1S3bazEv
-CHECK-NEXT: {{.*}}arange-overlap.cc:6
-
-DWO: _Z2f2v
-NODWO: {{^f2$}}
-SPLIT-NEXT: {{.*}}split-dwarf-test.cpp
-SPLIT-NEXT: main
-SPLIT-NEXT: {{.*}}split-dwarf-test.cpp
-
-SPLIT: _Z2f1v
-SPLIT-NEXT: {{.*}}split-dwarf-test.cpp
-
-; func has been inlined into main by LTO. Check that the symbolizer is able
-; to resolve the cross-cu reference and retrieve func's name
-CHECK: func
-CHECK-NEXT: /tmp{{[/\\]}}cross-cu-inlining.c:16:3
-CHECK-NEXT: main
-CHECK-NEXT: /tmp{{[/\\]}}cross-cu-inlining.c:11:0
-
-CHECK:      f2
-CHECK-NEXT: split-dwarf-multiple-cu.cpp:3:3
-CHECK-NEXT: f3
-CHECK-NEXT: split-dwarf-multiple-cu.cpp:6:0
-
-CHECK:      f2
-CHECK-NEXT: split-dwarf-addr-object-relocation.cpp:3:3
-CHECK-NEXT: f3
-CHECK-NEXT: split-dwarf-addr-object-relocation.cpp:6:0
-
-DWP:      f2
-DWP-NEXT: split-dwarf-dwp.cpp:3:3
-DWP-NEXT: main
-DWP-NEXT: split-dwarf-dwp.cpp:10:0
-
-RUN: echo "unexisting-file 0x1234" > %t.input2
-RUN: llvm-symbolizer < %t.input2 2>&1 | FileCheck %s --check-prefix=MISSING-FILE
-
-MISSING-FILE: LLVMSymbolizer: error reading file: {{[Nn]}}o such file or directory
-
-RUN: echo "%p/Inputs/macho-universal 0x1f84" > %t.input3
-RUN: llvm-symbolizer < %t.input3 | FileCheck %s --check-prefix=UNKNOWN-ARCH
-
-UNKNOWN-ARCH-NOT: main
-UNKNOWN-ARCH: ??
-UNKNOWN-ARCH-NOT: main
-
-RUN: echo "0x400559" > %t.input4
-RUN: echo "0x400436" >> %t.input4
-RUN: llvm-symbolizer --obj %p/Inputs/dwarfdump-test.elf-x86-64 < %t.input4 \
-RUN:   | FileCheck %s --check-prefix=BINARY
-
-BINARY:       main
-BINARY-NEXT: /tmp/dbginfo{{[/\\]}}dwarfdump-test.cc:16
-BINARY:      _start
-
-RUN: echo "0x400720" > %t.input5
-RUN: echo "0x4004a0" >> %t.input5
-RUN: echo "0x4006f0" >> %t.input5
-RUN: llvm-symbolizer --obj %p/Inputs/llvm-symbolizer-test.elf-x86-64 < %t.input5 \
-RUN:   | FileCheck %s --check-prefix=BINARY_C
-
-BINARY_C:       main
-BINARY_C-NEXT: /tmp/dbginfo{{[/\\]}}llvm-symbolizer-test.c:10
-BINARY_C:      _start
-BINARY_C:      {{g$}}
-
-RUN: echo "0x1f1" > %t.input6
-RUN: llvm-symbolizer --obj %p/Inputs/shared-object-stripped.elf-i386 < %t.input6 \
-RUN:   | FileCheck %s --check-prefix=STRIPPED
-
-STRIPPED:  global_func
-
-RUN: echo "%p/Inputs/dwarfdump-test4.elf-x86-64 0x62c" > %t.input7
-RUN: llvm-symbolizer --functions=short --demangle=false < %t.input7 \
-RUN:    | FileCheck %s --check-prefix=SHORT_FUNCTION_NAME
-
-SHORT_FUNCTION_NAME-NOT: _Z1cv
diff --git a/test/DebugInfo/symbolize-64bit-address.test b/test/DebugInfo/symbolize-64bit-address.test
new file mode 100644
index 0000000000000000000000000000000000000000..e834bab7c4e51b6301ec9f77ccd602d120ac8247
--- /dev/null
+++ b/test/DebugInfo/symbolize-64bit-address.test
@@ -0,0 +1,6 @@
+RUN: llvm-symbolizer 0xffffffff00000000 -a --obj=%p/Inputs/symbolize-64bit-addr.elf.x86_64 \
+RUN:   | FileCheck %s
+
+CHECK:      0xffffffff00000000
+CHECK-NEXT: _start
+CHECK-NEXT: symbolize-64bit-addr.c:1:0
diff --git a/test/DebugInfo/symbolize-absolute-path-in-debug-line.s b/test/DebugInfo/symbolize-absolute-path-in-debug-line.s
new file mode 100644
index 0000000000000000000000000000000000000000..4b4d26ed98c02c059d407034d74bb47b6e58f20f
--- /dev/null
+++ b/test/DebugInfo/symbolize-absolute-path-in-debug-line.s
@@ -0,0 +1,106 @@
+# Create .debug_line containing absolute path in filename. Show that the path is sensibly printed/found/etc.
+# REQUIRES: x86-registered-target
+
+# RUN: sed s!FILEPATH!%/s! %s > %t.s
+# RUN: llvm-mc --filetype=obj --triple=x86_64-pc-linux %t.s -o %t.o
+# RUN: llvm-symbolizer --obj=%t.o 0 | FileCheck %s -DPATH=%/s
+
+# CHECK: {{^}}[[PATH]]:1:0
+
+.type _start,@function
+_start:
+.Lfunc_begin0:
+    ret
+.Lfunc_end0:
+
+.section .debug_str,"MS",@progbits,1
+.Linfo_string1:
+    .asciz "test.c"
+.Linfo_string2:
+    .asciz "/some/dir"
+.Linfo_string3:
+    .asciz "_start"
+
+.section    .debug_abbrev,"",@progbits
+    .byte   1                       # Abbreviation Code
+    .byte   17                      # DW_TAG_compile_unit
+    .byte   1                       # DW_CHILDREN_yes
+    .byte   3                       # DW_AT_name
+    .byte   14                      # DW_FORM_strp
+    .byte   16                      # DW_AT_stmt_list
+    .byte   23                      # DW_FORM_sec_offset
+    .byte   27                      # DW_AT_comp_dir
+    .byte   14                      # DW_FORM_strp
+    .byte   17                      # DW_AT_low_pc
+    .byte   1                       # DW_FORM_addr
+    .byte   18                      # DW_AT_high_pc
+    .byte   6                       # DW_FORM_data4
+    .byte   0                       # EOM(1)
+    .byte   0                       # EOM(2)
+    .byte   2                       # Abbreviation Code
+    .byte   46                      # DW_TAG_subprogram
+    .byte   0                       # DW_CHILDREN_no
+    .byte   17                      # DW_AT_low_pc
+    .byte   1                       # DW_FORM_addr
+    .byte   18                      # DW_AT_high_pc
+    .byte   6                       # DW_FORM_data4
+    .byte   3                       # DW_AT_name
+    .byte   14                      # DW_FORM_strp
+    .byte   58                      # DW_AT_decl_file
+    .byte   11                      # DW_FORM_data1
+    .byte   59                      # DW_AT_decl_line
+    .byte   11                      # DW_FORM_data1
+    .byte   63                      # DW_AT_external
+    .byte   25                      # DW_FORM_flag_present
+    .byte   0                       # EOM(1)
+    .byte   0                       # EOM(2)
+    .byte   0                       # EOM(3)
+    .section    .debug_info,"",@progbits
+    .long   .Ldebug_info_end0-.Ldebug_info_start0 # Length of Unit
+.Ldebug_info_start0:
+    .short  4                       # DWARF version number
+    .long   .debug_abbrev           # Offset Into Abbrev. Section
+    .byte   8                       # Address Size (in bytes)
+    .byte   1                       # Abbrev [1] 0xb:0x35 DW_TAG_compile_unit
+    .long   .Linfo_string1          # DW_AT_name
+    .long   .Lline_table_start0     # DW_AT_stmt_list
+    .long   .Linfo_string2          # DW_AT_comp_dir
+    .quad   .Lfunc_begin0           # DW_AT_low_pc
+    .long   .Lfunc_end0-.Lfunc_begin0 # DW_AT_high_pc
+    .byte   2                       # Abbrev [2] 0x2a:0x15 DW_TAG_subprogram
+    .quad   .Lfunc_begin0           # DW_AT_low_pc
+    .long   .Lfunc_end0-.Lfunc_begin0 # DW_AT_high_pc
+    .long   .Linfo_string3          # DW_AT_name
+    .byte   1                       # DW_AT_decl_file
+    .byte   1                       # DW_AT_decl_line
+                                        # DW_AT_external
+    .byte   0                       # End Of Children Mark
+.Ldebug_info_end0:
+
+# Hand-written .debug_line to allow replacing in the absolute path
+# into the filename table at runtime.
+.section .debug_line,"",@progbits
+.Lline_table_start0:
+    .long .Ltable_end - .Ltable_start   # unit length
+.Ltable_start:
+    .short 4                            # version
+    .long .Lheader_end - .Lheader_start # header length
+.Lheader_start:
+    .byte 1                             # min instruction length
+    .byte 1                             # max ops per instruction
+    .byte 1                             # default is_stmt
+    .byte -5                            # line base
+    .byte 14                            # line range
+    .byte 13                            # opcode base
+    .byte 0, 1, 1, 1, 1, 0, 0, 0, 1, 0, 0, 1 # standard opcode lengths
+    .byte 0                             # directory table
+    .asciz "FILEPATH"                   # filename table
+    .byte 0, 0, 0
+    .byte 0
+.Lheader_end:
+    .byte 0, 9, 2                       # DW_LNE_set_address
+    .quad .Lfunc_begin0
+    .byte 1                             # DW_LNS_copy
+    .byte 33                            # +1 address, +1 line
+    .byte 0, 1, 1                       # DW_LNE_end_sequence
+.Ltable_end:
diff --git a/test/DebugInfo/symbolize-demangling.s b/test/DebugInfo/symbolize-demangling.s
new file mode 100644
index 0000000000000000000000000000000000000000..acd769a286ff4b8c114d21df028e7c32f279ff69
--- /dev/null
+++ b/test/DebugInfo/symbolize-demangling.s
@@ -0,0 +1,24 @@
+# REQUIRES: x86-registered-target
+
+# RUN: llvm-mc --filetype=obj --triple=x86_64-pc-linux %s -o %t.o -g
+
+# RUN: llvm-symbolizer --obj=%t.o 0 1 2 | FileCheck %s
+
+# CHECK:       f()
+# CHECK-NEXT:  symbolize-demangling.s:20
+# CHECK-EMPTY:
+# CHECK-NEXT:  {{^g$}}
+# CHECK-NEXT:  symbolize-demangling.s:22
+# CHECK-EMPTY:
+# CHECK-NEXT:  {{^baz$}}
+# CHECK-NEXT:  symbolize-demangling.s:24
+
+.type _Z1fv,@function
+.type g,@function
+.type baz,@function
+_Z1fv:
+  nop
+g:
+  nop
+baz:
+  nop
diff --git a/test/DebugInfo/symbolize-gnu-debuglink-fallback.test b/test/DebugInfo/symbolize-gnu-debuglink-fallback.test
new file mode 100644
index 0000000000000000000000000000000000000000..43d5a2c818fac39aabda4e6f52d475d3b2203565
--- /dev/null
+++ b/test/DebugInfo/symbolize-gnu-debuglink-fallback.test
@@ -0,0 +1,22 @@
+# REQUIRES: shell
+# Ensures that .debuglink can fallback to a separate location. This is normally
+# /usr/lib/debug (or /usr/libdata/debug for NetBSD), but can be configured on
+# the command line (mainly for testing).
+
+RUN: rm -rf %t/foo %t/bar
+RUN: mkdir -p %t/foo %t/bar/%t/foo
+
+RUN: cp %p/Inputs/dwarfdump-test.elf-x86-64.debuglink %t/foo
+
+RUN: llvm-symbolizer --obj=%t/foo/dwarfdump-test.elf-x86-64.debuglink 0x40113f \
+RUN:   --fallback-debug-path=%t/bar | FileCheck %s --check-prefix=UNKNOWN
+
+UNKNOWN:      main
+UNKNOWN-NEXT: ??:0:0
+
+RUN: cp %p/Inputs/dwarfdump-test.elf-x86-64 %t/bar/%t/foo
+RUN: llvm-symbolizer --obj=%t/foo/dwarfdump-test.elf-x86-64.debuglink 0x40113f \
+RUN:   --fallback-debug-path=%t/bar | FileCheck %s --check-prefix=FOUND
+
+FOUND:      main
+FOUND-NEXT: /tmp/dbginfo{{[/\\]}}dwarfdump-test.cc:16
diff --git a/test/DebugInfo/symbolize-gnu-debuglink-no-realpath.test b/test/DebugInfo/symbolize-gnu-debuglink-no-realpath.test
new file mode 100644
index 0000000000000000000000000000000000000000..5141ff6ce322cc273fce25cffeed51ef69141569
--- /dev/null
+++ b/test/DebugInfo/symbolize-gnu-debuglink-no-realpath.test
@@ -0,0 +1,17 @@
+# REQUIRES: shell
+# Ensure that no realpath assumptions are made about .gnu_debuglink paths.
+
+# Copy inputs to some other location with arbitrary names, with the original
+# filenames just being symlinks to the copies. Real files go in the "real" dir,
+# symlinks (with original names) go in "syms".
+RUN: rm -rf %t/real %t/syms
+RUN: mkdir -p %t/real %t/syms
+RUN: cp %p/Inputs/dwarfdump-test.elf-x86-64 %t/real/prog
+RUN: cp %p/Inputs/dwarfdump-test.elf-x86-64.debuglink %t/real/link
+RUN: ln -s %t/real/prog %t/syms/dwarfdump-test.elf-x86-64
+RUN: ln -s %t/real/link %t/syms/dwarfdump-test.elf-x86-64.debuglink
+
+RUN: llvm-symbolizer --obj=%t/syms/dwarfdump-test.elf-x86-64.debuglink 0x40113f | FileCheck %s
+
+CHECK:       main
+CHECK-NEXT: /tmp/dbginfo{{[/\\]}}dwarfdump-test.cc:16
diff --git a/test/DebugInfo/symbolize-gnu-debuglink.test b/test/DebugInfo/symbolize-gnu-debuglink.test
new file mode 100644
index 0000000000000000000000000000000000000000..93ffda8bf6ac8f9184460c9a8be4b7512557af3a
--- /dev/null
+++ b/test/DebugInfo/symbolize-gnu-debuglink.test
@@ -0,0 +1,4 @@
+RUN: llvm-symbolizer --obj=%p/Inputs/dwarfdump-test.elf-x86-64.debuglink 0x40113f | FileCheck %s
+
+CHECK:       main
+CHECK-NEXT: /tmp/dbginfo{{[/\\]}}dwarfdump-test.cc:16
diff --git a/test/DebugInfo/symbolize-inlined.test b/test/DebugInfo/symbolize-inlined.test
new file mode 100644
index 0000000000000000000000000000000000000000..2757721b81f8e479cc236d971de67d24f2b6f12f
--- /dev/null
+++ b/test/DebugInfo/symbolize-inlined.test
@@ -0,0 +1,22 @@
+RUN: llvm-symbolizer --inlining --obj=%p/Inputs/dwarfdump-inl-test.elf-x86-64 0x8dc 0xa05 0x987 | FileCheck %s
+
+CHECK:      inlined_h
+CHECK-NEXT: dwarfdump-inl-test.h:2
+CHECK-NEXT: inlined_g
+CHECK-NEXT: dwarfdump-inl-test.h:7
+CHECK-NEXT: inlined_f
+CHECK-NEXT: dwarfdump-inl-test.cc:3
+CHECK-NEXT: main
+CHECK-NEXT: dwarfdump-inl-test.cc:8
+
+CHECK:      inlined_g
+CHECK-NEXT: dwarfdump-inl-test.h:7
+CHECK-NEXT: inlined_f
+CHECK-NEXT: dwarfdump-inl-test.cc:3
+CHECK-NEXT: main
+CHECK-NEXT: dwarfdump-inl-test.cc:8
+
+CHECK:      inlined_f
+CHECK-NEXT: dwarfdump-inl-test.cc:3
+CHECK-NEXT: main
+CHECK-NEXT: dwarfdump-inl-test.cc:8
diff --git a/test/DebugInfo/symbolize-macho-universal-unknown-arch.test b/test/DebugInfo/symbolize-macho-universal-unknown-arch.test
new file mode 100644
index 0000000000000000000000000000000000000000..744a0d3e5492a091677e4c4a298acd732652fbf1
--- /dev/null
+++ b/test/DebugInfo/symbolize-macho-universal-unknown-arch.test
@@ -0,0 +1,5 @@
+RUN: llvm-symbolizer --obj=%p/Inputs/macho-universal 0x1f84 | FileCheck %s
+
+CHECK-NOT: main
+CHECK: ??
+CHECK-NOT: main
diff --git a/test/DebugInfo/symbolize-macho-universal.test b/test/DebugInfo/symbolize-macho-universal.test
new file mode 100644
index 0000000000000000000000000000000000000000..f775f68f93a714a6582f71f15701bcc775700f7e
--- /dev/null
+++ b/test/DebugInfo/symbolize-macho-universal.test
@@ -0,0 +1,7 @@
+RUN: llvm-symbolizer --no-demangle --default-arch=i386 --obj=%p/Inputs/macho-universal 0x1f84 | FileCheck %s --check-prefix=DEFAULT
+RUN: llvm-symbolizer --no-demangle --obj=%p/Inputs/macho-universal:i386 0x1f67 | FileCheck %s --check-prefix=I386
+RUN: llvm-symbolizer --no-demangle --obj=%p/Inputs/macho-universal:x86_64 0x100000f05 | FileCheck %s --check-prefix=X86-64
+
+DEFAULT: main
+I386:    _Z3inci
+X86-64:  _Z3inci
diff --git a/test/DebugInfo/symbolize-missing-file.test b/test/DebugInfo/symbolize-missing-file.test
new file mode 100644
index 0000000000000000000000000000000000000000..2762af4ff5dad549cdc576646292287b6bb69819
--- /dev/null
+++ b/test/DebugInfo/symbolize-missing-file.test
@@ -0,0 +1,3 @@
+RUN: llvm-symbolizer --obj=unexisting-file 0x1234 2>&1 | FileCheck %s
+
+CHECK: LLVMSymbolizer: error reading file: {{[Nn]}}o such file or directory
diff --git a/test/DebugInfo/symbolize-no-debug-str.test b/test/DebugInfo/symbolize-no-debug-str.test
new file mode 100644
index 0000000000000000000000000000000000000000..53386de3427337cc469dadbb0e603f45d4b7256f
--- /dev/null
+++ b/test/DebugInfo/symbolize-no-debug-str.test
@@ -0,0 +1,9 @@
+RUN: llvm-objcopy -R .debug_str %p/Inputs/llvm-symbolizer-test.elf-x86-64 %t.elf
+RUN: llvm-symbolizer --functions=linkage --obj=%t.elf 0x401140 | FileCheck %s --check-prefix=LINKAGE
+RUN: llvm-symbolizer --functions=short --obj=%t.elf 0x401140 | FileCheck %s --check-prefix=SHORT
+
+LINKAGE:      main
+LINKAGE-NEXT: {{^}}llvm-symbolizer-test.c:10:0
+
+SHORT:        ??
+SHORT-NEXT:   {{^}}llvm-symbolizer-test.c:10:0
diff --git a/test/DebugInfo/symbolize-shared-abbrev.s b/test/DebugInfo/symbolize-shared-abbrev.s
new file mode 100644
index 0000000000000000000000000000000000000000..972d0ae8ff8c1cb094fe214b7952f188651de3cf
--- /dev/null
+++ b/test/DebugInfo/symbolize-shared-abbrev.s
@@ -0,0 +1,162 @@
+# Show that multiple CUs can have a single common .debug_abbrev table. This can
+# occur due to e.g. LTO.
+
+# REQUIRES: x86-registered-target
+
+# RUN: llvm-mc --filetype=obj --triple=x86_64-pc-linux %s -o %t.o
+# RUN: llvm-symbolizer --obj=%t.o 0 16 --functions=short | FileCheck %s
+
+# CHECK:      foo
+# CHECK-NEXT: foo.c:1:0
+
+# CHECK:      bar
+# CHECK-NEXT: bar.c:2:0
+
+.global foo
+.type foo,@function
+foo:
+.Lfunc_begin0:
+    .file   1 "." "foo.c"
+    .loc    1 1 0
+    ret
+.Lfunc_end0:
+
+.global bar
+.p2align 4, 0x90
+.type bar,@function
+bar:
+.Lfunc_begin1:
+    .file   2 "." "bar.c"
+    .loc    2 2 0
+    ret
+.Lfunc_end1:
+
+    .section    .debug_str,"MS",@progbits,1
+.Linfo_string1:
+    .asciz  "foo.c"
+.Linfo_string2:
+    .asciz  "."
+.Linfo_string3:
+    .asciz  "foo"
+.Linfo_string4:
+    .asciz  "bar.c"
+.Linfo_string5:
+    .asciz  "bar"
+
+    # Regular .debug_abbrev section with CU and subprogram, but duplicated, with second
+    # half reordered slightly, to show that the correct abbrev is being referenced.
+    .section    .debug_abbrev,"",@progbits
+    .byte   1                       # Abbreviation Code
+    .byte   17                      # DW_TAG_compile_unit
+    .byte   1                       # DW_CHILDREN_yes
+    .byte   3                       # DW_AT_name
+    .byte   14                      # DW_FORM_strp
+    .byte   16                      # DW_AT_stmt_list
+    .byte   23                      # DW_FORM_sec_offset
+    .byte   27                      # DW_AT_comp_dir
+    .byte   14                      # DW_FORM_strp
+    .byte   17                      # DW_AT_low_pc
+    .byte   1                       # DW_FORM_addr
+    .byte   18                      # DW_AT_high_pc
+    .byte   6                       # DW_FORM_data4
+    .byte   0                       # EOM(1)
+    .byte   0                       # EOM(2)
+    .byte   2                       # Abbreviation Code
+    .byte   46                      # DW_TAG_subprogram
+    .byte   0                       # DW_CHILDREN_no
+    .byte   17                      # DW_AT_low_pc
+    .byte   1                       # DW_FORM_addr
+    .byte   18                      # DW_AT_high_pc
+    .byte   6                       # DW_FORM_data4
+    .byte   3                       # DW_AT_name
+    .byte   14                      # DW_FORM_strp
+    .byte   58                      # DW_AT_decl_file
+    .byte   11                      # DW_FORM_data1
+    .byte   59                      # DW_AT_decl_line
+    .byte   11                      # DW_FORM_data1
+    .byte   63                      # DW_AT_external
+    .byte   25                      # DW_FORM_flag_present
+    .byte   0                       # EOM(1)
+    .byte   0                       # EOM(2)
+    # Second-half starts here.
+    .byte   3                       # Abbreviation Code
+    .byte   17                      # DW_TAG_compile_unit
+    .byte   1                       # DW_CHILDREN_yes
+    .byte   16                      # DW_AT_stmt_list
+    .byte   23                      # DW_FORM_sec_offset
+    .byte   27                      # DW_AT_comp_dir
+    .byte   14                      # DW_FORM_strp
+    .byte   17                      # DW_AT_low_pc
+    .byte   1                       # DW_FORM_addr
+    .byte   18                      # DW_AT_high_pc
+    .byte   6                       # DW_FORM_data4
+    .byte   3                       # DW_AT_name
+    .byte   14                      # DW_FORM_strp
+    .byte   0                       # EOM(1)
+    .byte   0                       # EOM(2)
+    .byte   4                       # Abbreviation Code
+    .byte   46                      # DW_TAG_subprogram
+    .byte   0                       # DW_CHILDREN_no
+    .byte   17                      # DW_AT_low_pc
+    .byte   1                       # DW_FORM_addr
+    .byte   18                      # DW_AT_high_pc
+    .byte   6                       # DW_FORM_data4
+    .byte   58                      # DW_AT_decl_file
+    .byte   11                      # DW_FORM_data1
+    .byte   59                      # DW_AT_decl_line
+    .byte   11                      # DW_FORM_data1
+    .byte   63                      # DW_AT_external
+    .byte   25                      # DW_FORM_flag_present
+    .byte   3                       # DW_AT_name
+    .byte   14                      # DW_FORM_strp
+    .byte   0                       # EOM(1)
+    .byte   0                       # EOM(2)
+    .byte   0                       # EOM(3)
+
+    .section    .debug_info,"",@progbits
+    # First CU table.
+    .long   .Ldebug_info_end0-.Ldebug_info_start0 # Length of Unit
+.Ldebug_info_start0:
+    .short  4                       # DWARF version number
+    .long   .debug_abbrev           # Offset Into Abbrev. Section
+    .byte   8                       # Address Size (in bytes)
+    .byte   1                       # Abbrev [1] 0xb:0x35 DW_TAG_compile_unit
+    .long   .Linfo_string1          # DW_AT_name
+    .long   .Lline_table_start0     # DW_AT_stmt_list
+    .long   .Linfo_string2          # DW_AT_comp_dir
+    .quad   .Lfunc_begin0           # DW_AT_low_pc
+    .long   .Lfunc_end0-.Lfunc_begin0 # DW_AT_high_pc
+    .byte   2                       # Abbrev [2] 0x2a:0x15 DW_TAG_subprogram
+    .quad   .Lfunc_begin0           # DW_AT_low_pc
+    .long   .Lfunc_end0-.Lfunc_begin0 # DW_AT_high_pc
+    .long   .Linfo_string3          # DW_AT_name
+    .byte   1                       # DW_AT_decl_file
+    .byte   1                       # DW_AT_decl_line
+                                        # DW_AT_external
+    .byte   0                       # End Of Children Mark
+.Ldebug_info_end0:
+
+    # Second CU table.
+    .long   .Ldebug_info_end1-.Ldebug_info_start1 # Length of Unit
+.Ldebug_info_start1:
+    .short  4                       # DWARF version number
+    .long   .debug_abbrev           # Offset Into Abbrev. Section
+    .byte   8                       # Address Size (in bytes)
+    .byte   3                       # Abbrev [1] 0xb:0x35 DW_TAG_compile_unit
+    .long   .Lline_table_start0     # DW_AT_stmt_list
+    .long   .Linfo_string2          # DW_AT_comp_dir
+    .quad   .Lfunc_begin1           # DW_AT_low_pc
+    .long   .Lfunc_end1-.Lfunc_begin1 # DW_AT_high_pc
+    .long   .Linfo_string4          # DW_AT_name
+    .byte   4                       # Abbrev [2] 0x2a:0x15 DW_TAG_subprogram
+    .quad   .Lfunc_begin1           # DW_AT_low_pc
+    .long   .Lfunc_end1-.Lfunc_begin1 # DW_AT_high_pc
+    .byte   1                       # DW_AT_decl_file
+    .byte   1                       # DW_AT_decl_line
+                                        # DW_AT_external
+    .long   .Linfo_string5          # DW_AT_name
+    .byte   0                       # End Of Children Mark
+.Ldebug_info_end1:
+
+    .section    .debug_line,"",@progbits
+.Lline_table_start0:
diff --git a/test/DebugInfo/symbolize-split-dwarf-empty.test b/test/DebugInfo/symbolize-split-dwarf-empty.test
new file mode 100644
index 0000000000000000000000000000000000000000..090baf24e0e0864510cf665d38667e2de58e7c4a
--- /dev/null
+++ b/test/DebugInfo/symbolize-split-dwarf-empty.test
@@ -0,0 +1,9 @@
+cd and copy the DWO into the same directory so that it can be found by the
+symbolizer.
+RUN: rm -rf %t && mkdir -p %t
+RUN: cd %t
+RUN: cp %p/Inputs/split-dwarf-empty.dwo %t
+RUN: llvm-symbolizer --inlining --obj=%p/Inputs/split-dwarf-empty.o 0xdeadbeef | FileCheck %s
+
+CHECK: ??
+CHECK: ??:0:0
diff --git a/test/DebugInfo/symbolize-split-dwarf-no-skel-address.test b/test/DebugInfo/symbolize-split-dwarf-no-skel-address.test
new file mode 100644
index 0000000000000000000000000000000000000000..6c3f5ed35fd10d26a20923c4103f300a3daf8d1a
--- /dev/null
+++ b/test/DebugInfo/symbolize-split-dwarf-no-skel-address.test
@@ -0,0 +1,22 @@
+Copy the DWO and then cd into the output directory, so that the symbolizer would
+find the DWO if it attempted to.
+RUN: rm -rf %t && mkdir -p %t
+RUN: cd %t
+RUN: cp %p/Inputs/split-dwarf-no-skel-address.dwo %t
+RUN: llvm-symbolizer --inlining --no-demangle \
+RUN:    --obj=%p/Inputs/split-dwarf-no-skel-address.o 0x4 | FileCheck %s
+
+Built from the following source:
+void f1();
+__attribute__((always_inline)) inline void f2() { f1(); }
+void f3() { f2(); }
+
+Compiled to assembly with clang, modified the skeleton CU to remove the
+high/low pc (& update the CU length field and abbrev to match) & then
+compile/objcopy to make the .o and .dwo.
+
+Ensure that the f2 inlined frame is not included - it's inefficient to have to
+go and load all the debug info and search for the address ranges, so assume
+that a lack of ranges on the CU means the CU covers no addresses.
+
+CHECK-NOT: _Z2f2v
diff --git a/test/DebugInfo/symbolize-stripped-sections.test b/test/DebugInfo/symbolize-stripped-sections.test
new file mode 100644
index 0000000000000000000000000000000000000000..51cfd56792b1e22fe4895864e2a7fb3c7482b44d
--- /dev/null
+++ b/test/DebugInfo/symbolize-stripped-sections.test
@@ -0,0 +1,11 @@
+RUN: llvm-symbolizer --obj=%p/Inputs/llvm-symbolizer-test.elf-x86-64 0x401140 \
+RUN:   | FileCheck %s --check-prefix=UNSTRIPPED
+
+RUN: llvm-objcopy --strip-sections %p/Inputs/llvm-symbolizer-test.elf-x86-64 %t.elf
+RUN: llvm-symbolizer --obj=%t.elf 0x401140 | FileCheck %s --check-prefix=STRIPPED
+
+UNSTRIPPED:      main
+UNSTRIPPED-NEXT: llvm-symbolizer-test.c:10:0
+
+STRIPPED:        ??
+STRIPPED-NEXT:   ??:0:0
diff --git a/test/DebugInfo/symbolize-stripped.test b/test/DebugInfo/symbolize-stripped.test
new file mode 100644
index 0000000000000000000000000000000000000000..5c1131a556bb6afed2cc0fed0fc888ec797ed9ee
--- /dev/null
+++ b/test/DebugInfo/symbolize-stripped.test
@@ -0,0 +1,4 @@
+RUN: llvm-symbolizer 0x1f1 --obj %p/Inputs/shared-object-stripped.elf-i386 \
+RUN:   | FileCheck %s
+
+CHECK: global_func
diff --git a/test/DebugInfo/symbolize-tu.test b/test/DebugInfo/symbolize-tu.test
new file mode 100644
index 0000000000000000000000000000000000000000..7878a98cabe352c090ad96cc126d0ad885eca307
--- /dev/null
+++ b/test/DebugInfo/symbolize-tu.test
@@ -0,0 +1,7 @@
+RUN: llvm-symbolizer --obj=%p/Inputs/llvm-symbolizer-tu.elf-x86-64 0x0 | FileCheck %s
+
+Built from the following source:
+
+...
+
+CHECK: b.cpp:3:0
diff --git a/test/DebugInfo/symbolize-zlib.test b/test/DebugInfo/symbolize-zlib.test
new file mode 100644
index 0000000000000000000000000000000000000000..a02361da8733a14ba578b40d3a814c92f5b16eb1
--- /dev/null
+++ b/test/DebugInfo/symbolize-zlib.test
@@ -0,0 +1,7 @@
+REQUIRES: zlib
+
+RUN: llvm-symbolizer --obj=%p/Inputs/dwarfdump-test-zlib.elf-x86-64 0x4005e9 | FileCheck %s
+RUN: llvm-symbolizer --obj=%p/Inputs/dwarfdump-test-zlibgnu.elf-x86-64 0x4005e9 | FileCheck %s
+
+CHECK: main
+CHECK-NEXT: /tmp/dbginfo{{[/\\]}}dwarfdump-test-zlib.cc:16
diff --git a/test/DebugInfo/symbolize.test b/test/DebugInfo/symbolize.test
new file mode 100644
index 0000000000000000000000000000000000000000..8b43a11cd0ce1748e3edac4a419e9cdb2bec462b
--- /dev/null
+++ b/test/DebugInfo/symbolize.test
@@ -0,0 +1,29 @@
+RUN: echo "%p/Inputs/dwarfdump-test.elf-x86-64 0x40113f" > %t.input
+RUN: echo "%p/Inputs/dwarfdump-test.elf-x86-64 0x401020" >> %t.input
+RUN: echo "%p/Inputs/dwarfdump-test.elf-x86-64 0x40110e" >> %t.input
+RUN: echo "%p/Inputs/dwarfdump-test.elf-x86-64 0x401160" >> %t.input
+RUN: echo "%p/Inputs/dwarfdump-test2.elf-x86-64 0x4004e8" >> %t.input
+RUN: echo "%p/Inputs/dwarfdump-test2.elf-x86-64 0x4004f4" >> %t.input
+RUN: echo "%p/Inputs/dwarfdump-test4.elf-x86-64 0x62c" >> %t.input
+
+RUN: llvm-symbolizer < %t.input | FileCheck %s
+
+CHECK:       main
+CHECK-NEXT: /tmp/dbginfo{{[/\\]}}dwarfdump-test.cc:16
+
+CHECK:      _start
+
+CHECK:      f(int, int)
+CHECK-NEXT: /tmp/dbginfo{{[/\\]}}dwarfdump-test.cc:11
+
+CHECK:      DummyClass
+CHECK-NEXT: /tmp/dbginfo{{[/\\]}}dwarfdump-test.cc:4
+
+CHECK:      a
+CHECK-NEXT: /tmp/dbginfo{{[/\\]}}dwarfdump-test2-helper.cc:2
+
+CHECK:      main
+CHECK-NEXT: /tmp/dbginfo{{[/\\]}}dwarfdump-test2-main.cc:4
+
+CHECK:      c()
+CHECK-NEXT: /tmp/dbginfo{{[/\\]}}dwarfdump-test4-part1.cc:2
diff --git a/test/DebugInfo/unrolled-loop-remainder.ll b/test/DebugInfo/unrolled-loop-remainder.ll
index 96a25e28013eb481062680afa2abbc76bd3c6806..1256063b5c0db39d3ab8fdabda72142f35a77c28 100644
--- a/test/DebugInfo/unrolled-loop-remainder.ll
+++ b/test/DebugInfo/unrolled-loop-remainder.ll
@@ -68,7 +68,7 @@ attributes #0 = { norecurse nounwind readnone uwtable }
 
 !0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression())
 !1 = distinct !DIGlobalVariable(name: "b", scope: !2, file: !3, line: 2, type: !9, isLocal: false, isDefinition: true)
-!2 = distinct !DICompileUnit(language: DW_LANG_C99, file: !3, producer: "clang version 6.0.0 (http://llvm.org/git/clang.git 044091b728654e62444a7ea10e6efb489c705bed) (http://llvm.org/git/llvm.git 1c7b55afdb1c5791e0557d9e32e2dd07c7acb2b0)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, globals: !5)
+!2 = distinct !DICompileUnit(language: DW_LANG_C99, file: !3, producer: "clang version 6.0.0", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, globals: !5)
 !3 = !DIFile(filename: "loop.c", directory: "/work/projects/src/tests/unroll-debug-info")
 !4 = !{}
 !5 = !{!6, !0}
@@ -79,7 +79,7 @@ attributes #0 = { norecurse nounwind readnone uwtable }
 !10 = !{i32 2, !"Dwarf Version", i32 4}
 !11 = !{i32 2, !"Debug Info Version", i32 3}
 !12 = !{i32 1, !"wchar_size", i32 4}
-!13 = !{!"clang version 6.0.0 (http://llvm.org/git/clang.git 044091b728654e62444a7ea10e6efb489c705bed) (http://llvm.org/git/llvm.git 1c7b55afdb1c5791e0557d9e32e2dd07c7acb2b0)"}
+!13 = !{!"clang version 6.0.0"}
 !14 = distinct !DISubprogram(name: "c", scope: !3, file: !3, line: 3, type: !15, isLocal: false, isDefinition: true, scopeLine: 3, isOptimized: true, unit: !2, retainedNodes: !4)
 !15 = !DISubroutineType(types: !16)
 !16 = !{!9}
diff --git a/test/Demangle/ms-cxx17-noexcept.test b/test/Demangle/ms-cxx17-noexcept.test
new file mode 100644
index 0000000000000000000000000000000000000000..fc60d666d5207c81eb52b793572e8be114819261
--- /dev/null
+++ b/test/Demangle/ms-cxx17-noexcept.test
@@ -0,0 +1,25 @@
+; RUN: llvm-undname < %s | FileCheck %s
+
+; CHECK-NOT: Invalid mangled name
+
+?nochange@@YAXXZ
+; CHECK: void __cdecl nochange(void)
+
+?a@@YAXP6AHXZ@Z
+; CHECK: void __cdecl a(int (__cdecl *)(void))
+?a@@YAXP6AHX_E@Z
+; CHECK: void __cdecl a(int (__cdecl *)(void) noexcept)
+
+?b@@YAXP6AHXZ@Z
+; CHECK: void __cdecl b(int (__cdecl *)(void))
+
+?c@@YAXP6AHXZ@Z
+; CHECK: void __cdecl c(int (__cdecl *)(void))
+?c@@YAXP6AHX_E@Z
+; CHECK: void __cdecl c(int (__cdecl *)(void) noexcept)
+
+?ee@?$e@$$A6AXXZ@@EEAAXXZ
+; CHECK: private: virtual void __cdecl e<void __cdecl(void)>::ee(void)
+
+?ee@?$e@$$A6AXX_E@@EEAAXXZ
+; CHECK: private: virtual void __cdecl e<void __cdecl(void) noexcept>::ee(void)
diff --git a/test/FileCheck/check-empty.txt b/test/FileCheck/check-empty.txt
index 1caff2a37148bb7d9483f86fa81d7ecfe7629019..d2c592e501c8a7f631a9c488265cf51bab6af2e2 100644
--- a/test/FileCheck/check-empty.txt
+++ b/test/FileCheck/check-empty.txt
@@ -7,6 +7,6 @@
 ; NOFOO-NOT: foo
 
 ; EMPTY-ERR: FileCheck error: '-' is empty.
-; EMPTY-ERR-NEXT: FileCheck command line: {{.*}}FileCheck{{.*}}-check-prefix={{.*}}FOO {{.*}}check-empty.txt
+; EMPTY-ERR-NEXT: FileCheck command line: {{.*}}{{F|f}}ile{{C|c}}heck{{.*}}-check-prefix={{.*}}FOO {{.*}}check-empty.txt
 ; NO-EMPTY-ERR-NOT: FileCheck error: '-' is empty.
 ; NOT-FOUND: error: FOO: expected string not found in input
diff --git a/test/FileCheck/defines.txt b/test/FileCheck/defines.txt
index c788124dcbbad92de5d9ed364a4f6edd83e63458..f2628807155fa98fe223b9398a060ffd37954d30 100644
--- a/test/FileCheck/defines.txt
+++ b/test/FileCheck/defines.txt
@@ -3,16 +3,32 @@
 ;
 ; RUN: not FileCheck -DVALUE=10 -check-prefix NOT -input-file %s %s 2>&1 | FileCheck %s -check-prefix NOT-ERRMSG
 ; RUN: FileCheck -DVALUE=20 -check-prefix NOT -input-file %s %s
+; RUN: not FileCheck -DVALUE10 -input-file %s %s 2>&1 | FileCheck %s -check-prefix ERRCLIEQ1
+; RUN: not FileCheck -D -input-file %s %s 2>&1 | FileCheck %s -check-prefix ERRCLIEQ2
+; RUN: not FileCheck -D=10 -input-file %s %s 2>&1 | FileCheck %s -check-prefix ERRCLIVAR1
+; RUN: not FileCheck -D= -input-file %s %s 2>&1 | FileCheck %s -check-prefix ERRCLIVAR2
+; RUN: FileCheck -DVALUE= -check-prefix EMPTY -input-file %s %s 2>&1
 
 Value = 10
 ; CHECK: Value = [[VALUE]]
 ; NOT-NOT: Value = [[VALUE]]
 
-; ERRMSG: defines.txt:8:10: error: CHECK: expected string not found in input
+; ERRMSG: defines.txt:[[@LINE-3]]:10: error: CHECK: expected string not found in input
 ; ERRMSG: defines.txt:1:1: note: scanning from here
 ; ERRMSG: defines.txt:1:1: note: with variable "VALUE" equal to "20"
-; ERRMSG: defines.txt:7:1: note: possible intended match here
+; ERRMSG: defines.txt:[[@LINE-7]]:1: note: possible intended match here
 
-; NOT-ERRMSG: defines.txt:9:12: error: {{NOT}}-NOT: excluded string found in input
-; NOT-ERRMSG: defines.txt:7:1: note: found here
-; NOT-ERRMSG: defines.txt:7:1: note: with variable "VALUE" equal to "10"
\ No newline at end of file
+; NOT-ERRMSG: defines.txt:[[@LINE-7]]:12: error: {{NOT}}-NOT: excluded string found in input
+; NOT-ERRMSG: defines.txt:[[@LINE-10]]:1: note: found here
+; NOT-ERRMSG: defines.txt:[[@LINE-11]]:1: note: with variable "VALUE" equal to "10"
+
+; ERRCLIEQ1: Missing equal sign in command-line definition '-DVALUE10'
+
+; ERRCLIEQ2: {{F|f}}ile{{C|c}}heck{{[^:]*}}: for the -D option: requires a value!
+
+; ERRCLIVAR1: Missing pattern variable name in command-line definition '-D=10'
+
+; ERRCLIVAR2: Missing pattern variable name in command-line definition '-D='
+
+Empty value = @@
+; EMPTY: Empty value = @[[VALUE]]@
diff --git a/test/FileCheck/dump-input-annotations.txt b/test/FileCheck/dump-input-annotations.txt
index eaf1c941e672adbb659147a2f943873d7f4f90fd..b7bb8db5223cb021fff0f19c20d565893c4ff1c6 100644
--- a/test/FileCheck/dump-input-annotations.txt
+++ b/test/FileCheck/dump-input-annotations.txt
@@ -16,7 +16,12 @@
 ; RUN: echo 'CHECK: universe' >> %t.chk
 
 ; RUN: not FileCheck -dump-input=always -input-file %t.in %t.chk -v 2>&1 \
-; RUN: | FileCheck -strict-whitespace -match-full-lines -check-prefix=ALIGN %s
+; RUN: | FileCheck -strict-whitespace -match-full-lines -check-prefix=ALIGN \
+; RUN:             -implicit-check-not='remark:' %s
+
+; Verbose diagnostics are suppressed but not errors.
+; ALIGN:{{.*}}error:{{.*}}
+; ALIGN:{{.*}}possible intended match here{{.*}}
 
 ; ALIGN:Full input was:
 ; ALIGN-NEXT:<<<<<<
@@ -47,11 +52,18 @@
 ; RUN: echo 'CHECK: world' >> %t.chk
 
 ; RUN: not FileCheck -dump-input=always -input-file %t.in %t.chk 2>&1 \
-; RUN: | FileCheck -match-full-lines %s -check-prefix=CHK
+; RUN: | FileCheck -match-full-lines %s -check-prefix=CHK \
+; RUN:             -implicit-check-not='remark:'
 ; RUN: not FileCheck -dump-input=always -input-file %t.in %t.chk -v 2>&1 \
-; RUN: | FileCheck -match-full-lines %s -check-prefixes=CHK,CHK-V
+; RUN: | FileCheck -match-full-lines %s -check-prefixes=CHK,CHK-V \
+; RUN:             -implicit-check-not='remark:'
 ; RUN: not FileCheck -dump-input=always -input-file %t.in %t.chk -vv 2>&1 \
-; RUN: | FileCheck -match-full-lines %s -check-prefixes=CHK,CHK-V
+; RUN: | FileCheck -match-full-lines %s -check-prefixes=CHK,CHK-V \
+; RUN:             -implicit-check-not='remark:'
+
+; Verbose diagnostics are suppressed but not errors.
+; CHK: {{.*}}error:{{.*}}
+; CHK: {{.*}}possible intended match here{{.*}}
 
 ; CHK:        <<<<<<
 ; CHK-NEXT:              1: hello
@@ -77,11 +89,17 @@
 ; RUN: echo 'CHECK-COUNT-3: pete' > %t.chk
 
 ; RUN: not FileCheck -dump-input=always -input-file %t.in %t.chk 2>&1 \
-; RUN: | FileCheck -match-full-lines %s -check-prefixes=CNT,CNT-Q
+; RUN: | FileCheck -match-full-lines %s -check-prefixes=CNT,CNT-Q \
+; RUN:             -implicit-check-not='remark:'
 ; RUN: not FileCheck -dump-input=always -input-file %t.in %t.chk -v 2>&1 \
-; RUN: | FileCheck -match-full-lines %s -check-prefixes=CNT,CNT-V
+; RUN: | FileCheck -match-full-lines %s -check-prefixes=CNT,CNT-V \
+; RUN:             -implicit-check-not='remark:'
 ; RUN: not FileCheck -dump-input=always -input-file %t.in %t.chk -vv 2>&1 \
-; RUN: | FileCheck -match-full-lines %s -check-prefixes=CNT,CNT-V
+; RUN: | FileCheck -match-full-lines %s -check-prefixes=CNT,CNT-V \
+; RUN:             -implicit-check-not='remark:'
+
+; Verbose diagnostics are suppressed but not errors.
+; CNT: {{.*}}error:{{.*}}
 
 ; CNT:          <<<<<<
 ; CNT-NEXT:                1: pete
@@ -108,11 +126,17 @@
 ; RUN: echo 'CHECK-NEXT: world' >> %t.chk
 
 ; RUN: not FileCheck -dump-input=always -input-file %t.in %t.chk 2>&1 \
-; RUN: | FileCheck -match-full-lines %s -check-prefix=NXT
+; RUN: | FileCheck -match-full-lines %s -check-prefix=NXT \
+; RUN:             -implicit-check-not='remark:'
 ; RUN: not FileCheck -dump-input=always -input-file %t.in %t.chk -v 2>&1 \
-; RUN: | FileCheck -match-full-lines %s -check-prefixes=NXT,NXT-V
+; RUN: | FileCheck -match-full-lines %s -check-prefixes=NXT,NXT-V \
+; RUN:             -implicit-check-not='remark:'
 ; RUN: not FileCheck -dump-input=always -input-file %t.in %t.chk -vv 2>&1 \
-; RUN: | FileCheck -match-full-lines %s -check-prefixes=NXT,NXT-V,NXT-VV
+; RUN: | FileCheck -match-full-lines %s -check-prefixes=NXT,NXT-V,NXT-VV \
+; RUN:             -implicit-check-not='remark:'
+
+; Verbose diagnostics are suppressed but not errors.
+; NXT: {{.*}}error:{{.*}}
 
 ; NXT:        <<<<<<
 ; NXT-NEXT:            1: hello
@@ -155,11 +179,17 @@
 ; RUN: echo 'CHECK-SAME: again' >> %t.chk
 
 ; RUN: not FileCheck -dump-input=always -input-file %t.in %t.chk 2>&1 \
-; RUN: | FileCheck -match-full-lines %s -check-prefix=SAM
+; RUN: | FileCheck -match-full-lines %s -check-prefix=SAM \
+; RUN:             -implicit-check-not='remark:'
 ; RUN: not FileCheck -dump-input=always -input-file %t.in %t.chk -v 2>&1 \
-; RUN: | FileCheck -match-full-lines %s -check-prefixes=SAM,SAM-V
+; RUN: | FileCheck -match-full-lines %s -check-prefixes=SAM,SAM-V \
+; RUN:             -implicit-check-not='remark:'
 ; RUN: not FileCheck -dump-input=always -input-file %t.in %t.chk -vv 2>&1 \
-; RUN: | FileCheck -match-full-lines %s -check-prefixes=SAM,SAM-V,SAM-VV
+; RUN: | FileCheck -match-full-lines %s -check-prefixes=SAM,SAM-V,SAM-VV \
+; RUN:             -implicit-check-not='remark:'
+
+; Verbose diagnostics are suppressed but not errors.
+; SAM: {{.*}}error:{{.*}}
 
 ; SAM:        <<<<<<
 ; SAM-NEXT:            1: hello world!
@@ -174,7 +204,11 @@
 ; RUN: echo 'again' >> %t.in
 
 ; RUN: not FileCheck -dump-input=always -input-file %t.in %t.chk -v 2>&1 \
-; RUN: | FileCheck -match-full-lines %s -check-prefixes=SAM2
+; RUN: | FileCheck -match-full-lines %s -check-prefixes=SAM2 \
+; RUN:             -implicit-check-not='remark:'
+
+; Verbose diagnostics are suppressed but not errors.
+; SAM2: {{.*}}error:{{.*}}
 
 ; SAM2:      <<<<<<
 ; SAM2-NEXT:          1: hello world!
@@ -208,11 +242,17 @@
 ; RUN: echo 'CHECK-LABEL: label' >> %t.chk
 
 ; RUN: not FileCheck -dump-input=always -input-file %t.in %t.chk 2>&1 \
-; RUN: | FileCheck -match-full-lines %s -check-prefix=EMP
+; RUN: | FileCheck -match-full-lines %s -check-prefix=EMP \
+; RUN:             -implicit-check-not='remark:'
 ; RUN: not FileCheck -dump-input=always -input-file %t.in %t.chk -v 2>&1 \
-; RUN: | FileCheck -match-full-lines %s -check-prefixes=EMP,EMP-V
+; RUN: | FileCheck -match-full-lines %s -check-prefixes=EMP,EMP-V \
+; RUN:             -implicit-check-not='remark:'
 ; RUN: not FileCheck -dump-input=always -input-file %t.in %t.chk -vv 2>&1 \
-; RUN: | FileCheck -match-full-lines %s -check-prefixes=EMP,EMP-V,EMP-VV
+; RUN: | FileCheck -match-full-lines %s -check-prefixes=EMP,EMP-V,EMP-VV \
+; RUN:             -implicit-check-not='remark:'
+
+; Verbose diagnostics are suppressed but not errors.
+; EMP: {{.*}}error:{{.*}}
 
 ; EMP:        <<<<<<
 ; EMP-NEXT:            1: hello
@@ -236,11 +276,17 @@
 ; RUN: echo 'CHECK-EMPTY:' >> %t.chk
 
 ; RUN: not FileCheck -dump-input=always -input-file %t.in %t.chk 2>&1 \
-; RUN: | FileCheck -match-full-lines %s -check-prefix=EMP2
+; RUN: | FileCheck -match-full-lines %s -check-prefix=EMP2 \
+; RUN:             -implicit-check-not='remark:'
 ; RUN: not FileCheck -dump-input=always -input-file %t.in %t.chk -v 2>&1 \
-; RUN: | FileCheck -match-full-lines %s -check-prefixes=EMP2,EMP2-V
+; RUN: | FileCheck -match-full-lines %s -check-prefixes=EMP2,EMP2-V \
+; RUN:             -implicit-check-not='remark:'
 ; RUN: not FileCheck -dump-input=always -input-file %t.in %t.chk -vv 2>&1 \
-; RUN: | FileCheck -match-full-lines %s -check-prefixes=EMP2,EMP2-V,EMP2-VV
+; RUN: | FileCheck -match-full-lines %s -check-prefixes=EMP2,EMP2-V,EMP2-VV \
+; RUN:             -implicit-check-not='remark:'
+
+; Verbose diagnostics are suppressed but not errors.
+; EMP2: {{.*}}error:{{.*}}
 
 ; EMP2:        <<<<<<
 ; EMP2-NEXT:            1: hello
@@ -265,11 +311,17 @@
 ; RUN: echo 'CHECK-NOT: world' >> %t.chk
 
 ; RUN: not FileCheck -dump-input=always -input-file %t.in %t.chk 2>&1 \
-; RUN: | FileCheck -match-full-lines %s -check-prefix=NOT
+; RUN: | FileCheck -match-full-lines %s -check-prefix=NOT \
+; RUN:             -implicit-check-not='remark:'
 ; RUN: not FileCheck -dump-input=always -input-file %t.in %t.chk -v 2>&1 \
-; RUN: | FileCheck -match-full-lines %s -check-prefixes=NOT,NOT-V
+; RUN: | FileCheck -match-full-lines %s -check-prefixes=NOT,NOT-V \
+; RUN:             -implicit-check-not='remark:'
 ; RUN: not FileCheck -dump-input=always -input-file %t.in %t.chk -vv 2>&1 \
-; RUN: | FileCheck -match-full-lines %s -check-prefixes=NOT,NOT-V,NOT-VV
+; RUN: | FileCheck -match-full-lines %s -check-prefixes=NOT,NOT-V,NOT-VV \
+; RUN:             -implicit-check-not='remark:'
+
+; Verbose diagnostics are suppressed but not errors.
+; NOT: {{.*}}error:{{.*}}
 
 ; NOT:         <<<<<<
 ; NOT-NEXT:           1: hello
@@ -289,11 +341,17 @@
 ; RUN: echo 'CHECK: ain' >> %t.chk
 
 ; RUN: not FileCheck -dump-input=always -input-file %t.in %t.chk 2>&1 \
-; RUN: | FileCheck -match-full-lines %s -check-prefix=NOT2
+; RUN: | FileCheck -match-full-lines %s -check-prefix=NOT2 \
+; RUN:             -implicit-check-not='remark:'
 ; RUN: not FileCheck -dump-input=always -input-file %t.in %t.chk -v 2>&1 \
-; RUN: | FileCheck -match-full-lines %s -check-prefixes=NOT2,NOT2-V
+; RUN: | FileCheck -match-full-lines %s -check-prefixes=NOT2,NOT2-V \
+; RUN:             -implicit-check-not='remark:'
 ; RUN: not FileCheck -dump-input=always -input-file %t.in %t.chk -vv 2>&1 \
-; RUN: | FileCheck -match-full-lines %s -check-prefixes=NOT2,NOT2-V,NOT2-VV
+; RUN: | FileCheck -match-full-lines %s -check-prefixes=NOT2,NOT2-V,NOT2-VV \
+; RUN:             -implicit-check-not='remark:'
+
+; Verbose diagnostics are suppressed but not errors.
+; NOT2: {{.*}}error:{{.*}}
 
 ; NOT2:         <<<<<<
 ; NOT2-NEXT:             1: hello
@@ -330,11 +388,17 @@
 ; DAG-VV = -vv
 
 ; RUN: not FileCheck -dump-input=always -input-file %t.in %t.chk 2>&1 \
-; RUN: | FileCheck -match-full-lines %s -check-prefixes=DAG,DAG-Q
+; RUN: | FileCheck -match-full-lines %s -check-prefixes=DAG,DAG-Q \
+; RUN:             -implicit-check-not='remark:'
 ; RUN: not FileCheck -dump-input=always -input-file %t.in %t.chk -v 2>&1 \
-; RUN: | FileCheck -match-full-lines %s -check-prefixes=DAG,DAG-V,DAG-VQ
+; RUN: | FileCheck -match-full-lines %s -check-prefixes=DAG,DAG-V,DAG-VQ \
+; RUN:             -implicit-check-not='remark:'
 ; RUN: not FileCheck -dump-input=always -input-file %t.in %t.chk -vv 2>&1 \
-; RUN: | FileCheck -match-full-lines %s -check-prefixes=DAG,DAG-V,DAG-VV
+; RUN: | FileCheck -match-full-lines %s -check-prefixes=DAG,DAG-V,DAG-VV \
+; RUN:             -implicit-check-not='remark:'
+
+; Verbose diagnostics are suppressed but not errors.
+; DAG: {{.*}}error:{{.*}}
 
 ; DAG:         <<<<<<
 ; DAG-NEXT:             1: abc
@@ -371,11 +435,18 @@
 ; RUN: echo 'CHECK-LABEL: lab2' >> %t.chk
 
 ; RUN: not FileCheck -dump-input=always -input-file %t.in %t.chk 2>&1 \
-; RUN: | FileCheck -match-full-lines %s -check-prefixes=LAB
+; RUN: | FileCheck -match-full-lines %s -check-prefixes=LAB \
+; RUN:             -implicit-check-not='remark:'
 ; RUN: not FileCheck -dump-input=always -input-file %t.in %t.chk -v 2>&1 \
-; RUN: | FileCheck -match-full-lines %s -check-prefixes=LAB,LAB-V
+; RUN: | FileCheck -match-full-lines %s -check-prefixes=LAB,LAB-V \
+; RUN:             -implicit-check-not='remark:'
 ; RUN: not FileCheck -dump-input=always -input-file %t.in %t.chk -vv 2>&1 \
-; RUN: | FileCheck -match-full-lines %s -check-prefixes=LAB,LAB-V,LAB-VV
+; RUN: | FileCheck -match-full-lines %s -check-prefixes=LAB,LAB-V,LAB-VV \
+; RUN:             -implicit-check-not='remark:'
+
+; Verbose diagnostics are suppressed but not errors.
+; LAB: {{.*}}error:{{.*}}
+; LAB: {{.*}}possible intended match{{.*}}
 
 ; LAB:         <<<<<<
 ; LAB-NEXT:               1: lab0
@@ -390,5 +461,3 @@
 ; LAB-NEXT:    label:3'0     ~~~ error: no match found
 ; LAB-NEXT:    >>>>>>
 ; LAB-NOT:     {{.}}
-
-
diff --git a/test/FileCheck/dump-input-enable.txt b/test/FileCheck/dump-input-enable.txt
index 71e3d5dd9d6c3be739e9dffb24eb38b7e46d6f71..45592ddd2cd99a2f48c253cb979116905e1a60bf 100644
--- a/test/FileCheck/dump-input-enable.txt
+++ b/test/FileCheck/dump-input-enable.txt
@@ -1,14 +1,21 @@
-; RUN: echo ciao > %t.good
+;--------------------------------------------------
+; Create the check file, good input, and bad input.
+;
+; For both good and bad input, make sure the -v trace has at least one remark
+; so we can check how trace suppression is affected by -dump-input.
+;--------------------------------------------------
+
+; RUN: echo hello > %t.good
 ; RUN: echo world >> %t.good
 
 ; RUN: echo hello > %t.err
-; RUN: echo world >> %t.err
+; RUN: echo whirled >> %t.err
 
-; RUN: echo 'CHECK: ciao' > %t.check
+; RUN: echo 'CHECK: hello' > %t.check
 ; RUN: echo 'CHECK-NEXT: world' >> %t.check
 
 ;--------------------------------------------------
-; unknown value
+; Check -dump-input=<bad value>.
 ;--------------------------------------------------
 
 ; RUN: not FileCheck -input-file %t.good %t.check -check-prefix=CHECK \
@@ -19,10 +26,10 @@
 ; RUN: not FileCheck -dump-input=foobar 2>&1 \
 ; RUN: | FileCheck %s -match-full-lines -check-prefix=BADVAL
 
-BADVAL: FileCheck{{.*}}: for the -dump-input option: Cannot find option named 'foobar'!
+BADVAL: {{F|f}}ile{{C|c}}heck{{.*}}: for the -dump-input option: Cannot find option named 'foobar'!
 
 ;--------------------------------------------------
-; help
+; Check -dump-input=help.
 ;--------------------------------------------------
 
 ; Appended to normal command line.
@@ -38,91 +45,131 @@ HELP: try{{.*}}-color
 HELP-NOT: {{.}}
 
 ;--------------------------------------------------
-; never
+; Check -dump-input=never.
+;
+; Include the case without -v, which isn't covered elsewhere.
 ;--------------------------------------------------
 
 ; RUN: FileCheck -input-file %t.good %t.check -check-prefix=CHECK \
 ; RUN:           -match-full-lines -dump-input=never 2>&1 \
-; RUN: | FileCheck %s -match-full-lines -check-prefix=CHECK-NODUMP -allow-empty
+; RUN: | FileCheck %s -match-full-lines -allow-empty \
+; RUN:             -check-prefixes=NOTRACE,NODUMP
 
 ; RUN: not FileCheck -input-file %t.err %t.check -check-prefix=CHECK \
 ; RUN:               -match-full-lines -dump-input=never 2>&1 \
-; RUN: | FileCheck %s -match-full-lines -check-prefix=CHECK-NODUMP
+; RUN: | FileCheck %s -match-full-lines -check-prefixes=NOTRACE,ERR,NODUMP
+
+; RUN: FileCheck -input-file %t.good %t.check -check-prefix=CHECK \
+; RUN:           -match-full-lines -dump-input=never -v 2>&1 \
+; RUN: | FileCheck %s -match-full-lines -check-prefixes=TRACE,NODUMP
+
+; RUN: not FileCheck -input-file %t.err %t.check -check-prefix=CHECK \
+; RUN:               -match-full-lines -dump-input=never -v 2>&1 \
+; RUN: | FileCheck %s -match-full-lines -check-prefixes=TRACE,ERR,NODUMP
 
 ;--------------------------------------------------
-; default: never
+; Check no -dump-input, which defaults to never.
 ;--------------------------------------------------
 
 ; RUN: FileCheck -input-file %t.good %t.check -check-prefix=CHECK \
-; RUN:           -match-full-lines 2>&1 \
-; RUN: | FileCheck %s -match-full-lines -check-prefix=CHECK-NODUMP -allow-empty
+; RUN:           -match-full-lines -v 2>&1 \
+; RUN: | FileCheck %s -match-full-lines -check-prefixes=TRACE,NODUMP
 
 ; RUN: not FileCheck -input-file %t.err %t.check -check-prefix=CHECK \
-; RUN:               -match-full-lines 2>&1 \
-; RUN: | FileCheck %s -match-full-lines -check-prefix=CHECK-NODUMP
+; RUN:               -match-full-lines -v 2>&1 \
+; RUN: | FileCheck %s -match-full-lines -check-prefixes=TRACE,ERR,NODUMP
 
 ;--------------------------------------------------
-; fail
+; Check -dump-input=fail.
+;
+; Include the case without -v, which isn't covered elsewhere.
 ;--------------------------------------------------
 
 ; RUN: FileCheck -input-file %t.good %t.check -check-prefix=CHECK \
 ; RUN:           -match-full-lines -dump-input=fail 2>&1 \
-; RUN: | FileCheck %s -match-full-lines -check-prefix=CHECK-NODUMP -allow-empty
+; RUN: | FileCheck %s -match-full-lines -allow-empty \
+; RUN:             -check-prefixes=NOTRACE,NODUMP
 
 ; RUN: not FileCheck -input-file %t.err %t.check -check-prefix=CHECK \
 ; RUN:               -match-full-lines -dump-input=fail 2>&1 \
-; RUN: | FileCheck %s -match-full-lines -check-prefix=CHECK-ERR
+; RUN: | FileCheck %s -match-full-lines -check-prefixes=NOTRACE,ERR,DUMP-ERR
+
+; RUN: FileCheck -input-file %t.good %t.check -check-prefix=CHECK \
+; RUN:           -match-full-lines -dump-input=fail -v 2>&1 \
+; RUN: | FileCheck %s -match-full-lines -allow-empty \
+; RUN:             -check-prefixes=NOTRACE,NODUMP
+
+; RUN: not FileCheck -input-file %t.err %t.check -check-prefix=CHECK \
+; RUN:               -match-full-lines -dump-input=fail -v 2>&1 \
+; RUN: | FileCheck %s -match-full-lines \
+; RUN:                -check-prefixes=NOTRACE,ERR,DUMP-ERR,DUMP-ERR-V
 
 ;--------------------------------------------------
-; -dump-input-on-failure
+; Check -dump-input-on-failure.
 ;--------------------------------------------------
 
 ; RUN: FileCheck -input-file %t.good %t.check -check-prefix=CHECK \
-; RUN:           -match-full-lines -dump-input-on-failure 2>&1 \
-; RUN: | FileCheck %s -match-full-lines -check-prefix=CHECK-NODUMP -allow-empty
+; RUN:           -match-full-lines -dump-input-on-failure -v 2>&1 \
+; RUN: | FileCheck %s -match-full-lines -allow-empty \
+; RUN:             -check-prefixes=NOTRACE,NODUMP
 
 ; RUN: not FileCheck -input-file %t.err %t.check -check-prefix=CHECK \
-; RUN:               -match-full-lines -dump-input-on-failure 2>&1 \
-; RUN: | FileCheck %s -match-full-lines -check-prefix=CHECK-ERR
+; RUN:               -match-full-lines -dump-input-on-failure -v 2>&1 \
+; RUN: | FileCheck %s -match-full-lines \
+; RUN:                -check-prefixes=NOTRACE,ERR,DUMP-ERR,DUMP-ERR-V
 
 ; RUN: env FILECHECK_DUMP_INPUT_ON_FAILURE=1 \
 ; RUN: FileCheck -input-file %t.good %t.check -check-prefix=CHECK \
-; RUN:           -match-full-lines 2>&1 \
-; RUN: | FileCheck %s -match-full-lines -check-prefix=CHECK-NODUMP -allow-empty
+; RUN:           -match-full-lines -v 2>&1 \
+; RUN: | FileCheck %s -match-full-lines -allow-empty \
+; RUN:             -check-prefixes=NOTRACE,NODUMP
 
 ; RUN: env FILECHECK_DUMP_INPUT_ON_FAILURE=1 \
 ; RUN: not FileCheck -input-file %t.err %t.check -check-prefix=CHECK \
-; RUN:               -match-full-lines 2>&1 \
-; RUN: | FileCheck %s -match-full-lines -check-prefix=CHECK-ERR
+; RUN:               -match-full-lines -v 2>&1 \
+; RUN: | FileCheck %s -match-full-lines \
+; RUN:                -check-prefixes=NOTRACE,ERR,DUMP-ERR,DUMP-ERR-V
 
 ;--------------------------------------------------
-; always
+; Check -dump-input=always.
 ;--------------------------------------------------
 
 ; RUN: FileCheck -input-file %t.good %t.check -check-prefix=CHECK \
 ; RUN:           -match-full-lines -dump-input=always -v 2>&1 \
-; RUN: | FileCheck %s -match-full-lines -check-prefix=CHECK-GOOD
+; RUN: | FileCheck %s -match-full-lines -check-prefixes=NOTRACE,DUMP-OK
 
 ; RUN: not FileCheck -input-file %t.err %t.check -check-prefix=CHECK \
-; RUN:               -match-full-lines -dump-input=always 2>&1 \
-; RUN: | FileCheck %s -match-full-lines -check-prefix=CHECK-ERR
+; RUN:               -match-full-lines -dump-input=always -v 2>&1 \
+; RUN: | FileCheck %s -match-full-lines \
+; RUN:                -check-prefixes=NOTRACE,ERR,DUMP-ERR,DUMP-ERR-V
 
 ; END.
 
-; CHECK-GOOD: Full input was:
-; CHECK-GOOD-NEXT: <<<<<<
-; CHECK-GOOD-NEXT:          1: ciao
-; CHECK-GOOD-NEXT: check:1     ^~~~
-; CHECK-GOOD-NEXT:          2: world
-; CHECK-GOOD-NEXT: next:2      ^~~~~
-; CHECK-GOOD-NEXT: >>>>>>
-
-; CHECK-ERR: Full input was:
-; CHECK-ERR-NEXT: <<<<<<
-; CHECK-ERR-NEXT:          1: hello
-; CHECK-ERR-NEXT: check:1     X~~~~
-; CHECK-ERR-NEXT:          2: world
-; CHECK-ERR-NEXT: check:1     ~~~~~ error: no match found
-; CHECK-ERR-NEXT: >>>>>>
-
-; CHECK-NODUMP-NOT: <<<<<<
+;--------------------------------------------------
+; Check the output for all cases that actually process directives.
+;--------------------------------------------------
+
+; Trace is sometimes suppressed.
+; TRACE:       {{.*}}remark:{{.*}}
+; NOTRACE-NOT: remark:
+
+; Error diagnostics are never suppressed.
+; ERR: {{.*}}error:{{.*}}
+
+; NODUMP-NOT: <<<<<<
+
+; DUMP-OK:      Full input was:
+; DUMP-OK-NEXT: <<<<<<
+; DUMP-OK-NEXT:          1: hello
+; DUMP-OK-NEXT: check:1     ^~~~~
+; DUMP-OK-NEXT:          2: world
+; DUMP-OK-NEXT: next:2      ^~~~~
+; DUMP-OK-NEXT: >>>>>>
+
+; DUMP-ERR:        Full input was:
+; DUMP-ERR-NEXT:   <<<<<<
+; DUMP-ERR-NEXT:            1: hello
+; DUMP-ERR-V-NEXT: check:1     ^~~~~
+; DUMP-ERR-NEXT:            2: whirled
+; DUMP-ERR-NEXT:   next:2      X~~~~~~ error: no match found
+; DUMP-ERR-NEXT:   >>>>>>
diff --git a/test/FileCheck/lit.local.cfg b/test/FileCheck/lit.local.cfg
new file mode 100644
index 0000000000000000000000000000000000000000..307da456c3815056167d12632af8d3df3cbddd9a
--- /dev/null
+++ b/test/FileCheck/lit.local.cfg
@@ -0,0 +1,13 @@
+# Unset environment variables that the FileCheck tests
+# expect not to be set.
+file_check_expected_unset_vars = [
+  'FILECHECK_DUMP_INPUT_ON_FAILURE',
+  'FILECHECK_OPTS',
+]
+
+for env_var in file_check_expected_unset_vars:
+  if env_var in config.environment:
+    lit_config.note('Removing {} from environment for FileCheck tests'.format(
+      env_var)
+    )
+    config.environment.pop(env_var)
diff --git a/test/Instrumentation/AddressSanitizer/global_cstring_darwin.ll b/test/Instrumentation/AddressSanitizer/global_cstring_darwin.ll
index eb31c15b34488f05384823a57a3b616a345eed1b..0fc3205ba492c519b43131547d05f3344d2f9c3c 100644
--- a/test/Instrumentation/AddressSanitizer/global_cstring_darwin.ll
+++ b/test/Instrumentation/AddressSanitizer/global_cstring_darwin.ll
@@ -8,14 +8,14 @@ target triple = "x86_64-apple-macosx10.10.0"
 @.str.1 = private unnamed_addr constant [13 x i8] c"Hello world.\00", align 1
 @.str.2 = private unnamed_addr constant [4 x i8] c"%s\0A\00", align 1
 
-; CHECK: @.str.1 = internal unnamed_addr constant { [13 x i8], [51 x i8] } { [13 x i8] c"Hello world.\00", [51 x i8] zeroinitializer }, section "__TEXT,__asan_cstring,regular", align 32
-; CHECK: @.str.2 = internal unnamed_addr constant { [4 x i8], [60 x i8] } { [4 x i8] c"%s\0A\00", [60 x i8] zeroinitializer }, section "__TEXT,__asan_cstring,regular", align 32
+; CHECK: @.str.1 = internal constant { [13 x i8], [51 x i8] } { [13 x i8] c"Hello world.\00", [51 x i8] zeroinitializer }, section "__TEXT,__asan_cstring,regular", align 32
+; CHECK: @.str.2 = internal constant { [4 x i8], [60 x i8] } { [4 x i8] c"%s\0A\00", [60 x i8] zeroinitializer }, section "__TEXT,__asan_cstring,regular", align 32
 
 ; Shouldn't be put into special section:
 @.str.3 = private unnamed_addr constant [4 x i8] c"\00\01\02\03", align 1
 @.str.4 = private unnamed_addr global [7 x i8] c"Hello.\00", align 1
 @.str.5 = private unnamed_addr constant [8 x i8] c"Hello.\00\00", align 1
 
-; CHECK: @.str.3 = internal unnamed_addr constant { [4 x i8], [60 x i8] } { [4 x i8] c"\00\01\02\03", [60 x i8] zeroinitializer }, align 32
-; CHECK: @.str.4 = private unnamed_addr global { [7 x i8], [57 x i8] } { [7 x i8] c"Hello.\00", [57 x i8] zeroinitializer }, align 32
-; CHECK: @.str.5 = internal unnamed_addr constant { [8 x i8], [56 x i8] } { [8 x i8] c"Hello.\00\00", [56 x i8] zeroinitializer }, align 32
+; CHECK: @.str.3 = internal constant { [4 x i8], [60 x i8] } { [4 x i8] c"\00\01\02\03", [60 x i8] zeroinitializer }, align 32
+; CHECK: @.str.4 = private global { [7 x i8], [57 x i8] } { [7 x i8] c"Hello.\00", [57 x i8] zeroinitializer }, align 32
+; CHECK: @.str.5 = internal constant { [8 x i8], [56 x i8] } { [8 x i8] c"Hello.\00\00", [56 x i8] zeroinitializer }, align 32
diff --git a/test/Instrumentation/AddressSanitizer/global_metadata.ll b/test/Instrumentation/AddressSanitizer/global_metadata.ll
index ee42f1cef97af1568b15adef91267fae6b4531f9..ecee77209667e268aeb8c44d634e1b17516a11d4 100644
--- a/test/Instrumentation/AddressSanitizer/global_metadata.ll
+++ b/test/Instrumentation/AddressSanitizer/global_metadata.ll
@@ -15,7 +15,7 @@ target triple = "x86_64-unknown-linux-gnu"
 ; Check that globals were instrumented:
 
 ; CHECK: @global = global { i32, [60 x i8] } zeroinitializer, comdat, align 32
-; CHECK: @.str = internal unnamed_addr constant { [14 x i8], [50 x i8] } { [14 x i8] c"Hello, world!\00", [50 x i8] zeroinitializer }, comdat($".str${{[01-9a-f]+}}"), align 32
+; CHECK: @.str = internal constant { [14 x i8], [50 x i8] } { [14 x i8] c"Hello, world!\00", [50 x i8] zeroinitializer }, comdat($".str${{[01-9a-f]+}}"), align 32
 
 ; Check emitted location descriptions:
 ; CHECK: [[VARNAME:@___asan_gen_.[0-9]+]] = private unnamed_addr constant [7 x i8] c"global\00", align 1
diff --git a/test/Instrumentation/AddressSanitizer/global_metadata_array.ll b/test/Instrumentation/AddressSanitizer/global_metadata_array.ll
index 76f73778f0f169ac8cf6bb876b6baf2f0bb8e1ef..fbe0016140bafd530e42921792d41be9f5cc6cb4 100644
--- a/test/Instrumentation/AddressSanitizer/global_metadata_array.ll
+++ b/test/Instrumentation/AddressSanitizer/global_metadata_array.ll
@@ -15,7 +15,7 @@ target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
 ; Check that globals were instrumented:
 ; CHECK: @global = global { i32, [60 x i8] } zeroinitializer, align 32
-; CHECK: @.str = internal unnamed_addr constant { [14 x i8], [50 x i8] } { [14 x i8] c"Hello, world!\00", [50 x i8] zeroinitializer }{{.*}}, align 32
+; CHECK: @.str = internal constant { [14 x i8], [50 x i8] } { [14 x i8] c"Hello, world!\00", [50 x i8] zeroinitializer }{{.*}}, align 32
 
 ; Check emitted location descriptions:
 ; CHECK: [[VARNAME:@___asan_gen_.[0-9]+]] = private unnamed_addr constant [7 x i8] c"global\00", align 1
diff --git a/test/Instrumentation/AddressSanitizer/global_metadata_windows.ll b/test/Instrumentation/AddressSanitizer/global_metadata_windows.ll
index eeac65e1847301d716abbe711eb1eb73df14cd01..5602cb26e0049e520939716d1c88183c65eb9cc8 100644
--- a/test/Instrumentation/AddressSanitizer/global_metadata_windows.ll
+++ b/test/Instrumentation/AddressSanitizer/global_metadata_windows.ll
@@ -14,8 +14,8 @@ $mystr = comdat any
 ; CHECK: $dead_global = comdat noduplicates
 ; CHECK: $private_str = comdat noduplicates
 
-; CHECK: @dead_global = local_unnamed_addr global { i32, [60 x i8] } { i32 42, [60 x i8] zeroinitializer }, comdat, align 32
-; CHECK: @private_str = internal unnamed_addr constant { [8 x i8], [56 x i8] } { [8 x i8] c"private\00", [56 x i8] zeroinitializer }, comdat, align 32
+; CHECK: @dead_global = global { i32, [60 x i8] } { i32 42, [60 x i8] zeroinitializer }, comdat, align 32
+; CHECK: @private_str = internal constant { [8 x i8], [56 x i8] } { [8 x i8] c"private\00", [56 x i8] zeroinitializer }, comdat, align 32
 
 ; CHECK: @__asan_global_dead_global = private global { {{.*}} }, section ".ASAN$GL", comdat($dead_global), align 64
 ; CHECK: @__asan_global_private_str = private global { {{.*}} }, section ".ASAN$GL", comdat($private_str), align 64
diff --git a/test/Instrumentation/AddressSanitizer/instrument-no-return.ll b/test/Instrumentation/AddressSanitizer/instrument-no-return.ll
index 2e90bfc64b20c0f7e802f4dc559bdefe9701afee..22ee66301de25f5b0b1a4826aca6904596509726 100644
--- a/test/Instrumentation/AddressSanitizer/instrument-no-return.ll
+++ b/test/Instrumentation/AddressSanitizer/instrument-no-return.ll
@@ -1,37 +1,54 @@
-; RUN: opt < %s -asan -asan-module -S | FileCheck %s
+; RUN: opt < %s -asan -S | FileCheck %s
 ; AddressSanitizer must insert __asan_handle_no_return
-; before every noreturn call or invoke.
+; before noreturn calls that aren't inserted by sanitizers.
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 target triple = "x86_64-unknown-linux-gnu"
 
-declare void @MyNoReturnFunc(i32) noreturn
+declare void @NormalFunc()
+declare void @NoReturnFunc() noreturn
 
-define i32 @Call1(i8* nocapture %arg) uwtable sanitize_address {
-entry:
-  call void @MyNoReturnFunc(i32 1) noreturn  ; The call insn has noreturn attr.
-; CHECK:        @Call1
-; CHECK:        call void @__asan_handle_no_return
-; CHECK-NEXT:   call void @MyNoReturnFunc
-; CHECK-NEXT: unreachable
+; Instrument noreturn callsites (regardless of function)
+define i32 @Call1() sanitize_address {
+  call void @NormalFunc() noreturn
   unreachable
 }
+; CHECK-LABEL:  @Call1
+; CHECK-NEXT:   call void @__asan_handle_no_return
+; CHECK-NEXT:   call void @NormalFunc
 
-define i32 @Call2(i8* nocapture %arg) uwtable sanitize_address {
-entry:
-  call void @MyNoReturnFunc(i32 1)  ; No noreturn attribure on the call.
-; CHECK:        @Call2
-; CHECK:        call void @__asan_handle_no_return
-; CHECK-NEXT:   call void @MyNoReturnFunc
-; CHECK-NEXT: unreachable
+; Instrument calls to noreturn functions (regardless of callsite)
+define i32 @Call2() sanitize_address {
+  call void @NoReturnFunc()
+  unreachable
+}
+; CHECK-LABEL:  @Call2
+; CHECK-NEXT:   call void @__asan_handle_no_return
+; CHECK-NEXT:   call void @NoReturnFunc
+
+; Do *not* instrument callsites marked !nosanitize
+define i32 @Call3() sanitize_address {
+  call void @NoReturnFunc() noreturn, !nosanitize !{}
+  unreachable
+}
+; CHECK-LABEL:  @Call3
+; CHECK-NOT:    call void @__asan_handle_no_return
+; CHECK:        call void @NoReturnFunc
+
+; Do *not* instrument functions without ASan
+define i32 @Call4() {
+  call void @NoReturnFunc() noreturn
   unreachable
 }
+; CHECK-LABEL:  @Call4
+; CHECK-NOT:    call void @__asan_handle_no_return
+; CHECK:        call void @NoReturnFunc
 
 declare i32 @__gxx_personality_v0(...)
 
-define i64 @Invoke1(i8** %esc) nounwind uwtable ssp sanitize_address personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
+define i64 @Invoke1() nounwind uwtable ssp sanitize_address personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
 entry:
-  invoke void @MyNoReturnFunc(i32 1)
+  invoke void @NoReturnFunc()
           to label %invoke.cont unwind label %lpad
 
 invoke.cont:
@@ -42,8 +59,8 @@ lpad:
           filter [0 x i8*] zeroinitializer
   ret i64 1
 }
-; CHECK: @Invoke1
+; CHECK-LABEL:  @Invoke1
 ; CHECK:        call void @__asan_handle_no_return
-; CHECK-NEXT:   invoke void @MyNoReturnFunc
+; CHECK-NEXT:   invoke void @NoReturnFunc
 ; CHECK: ret i64 0
 ; CHECK: ret i64 1
diff --git a/test/Instrumentation/AddressSanitizer/localescape.ll b/test/Instrumentation/AddressSanitizer/localescape.ll
index d9daa8c96b13a85ae6fa8e1dc8e082dc44e9fd51..015b0e84ff16c80c1663d9a1145cd726a37868e3 100644
--- a/test/Instrumentation/AddressSanitizer/localescape.ll
+++ b/test/Instrumentation/AddressSanitizer/localescape.ll
@@ -6,7 +6,7 @@ target triple = "i686-pc-windows-msvc18.0.0"
 
 declare i32 @llvm.eh.typeid.for(i8*) #2
 declare i8* @llvm.frameaddress(i32)
-declare i8* @llvm.x86.seh.recoverfp(i8*, i8*)
+declare i8* @llvm.eh.recoverfp(i8*, i8*)
 declare i8* @llvm.localrecover(i8*, i8*, i32)
 declare void @llvm.localescape(...) #1
 
@@ -56,7 +56,7 @@ eh.resume:                                        ; preds = %lpad
 define internal i32 @"\01?filt$0@0@main@@"() #1 {
 entry:
   %0 = tail call i8* @llvm.frameaddress(i32 1)
-  %1 = tail call i8* @llvm.x86.seh.recoverfp(i8* bitcast (i32 ()* @main to i8*), i8* %0)
+  %1 = tail call i8* @llvm.eh.recoverfp(i8* bitcast (i32 ()* @main to i8*), i8* %0)
   %2 = tail call i8* @llvm.localrecover(i8* bitcast (i32 ()* @main to i8*), i8* %1, i32 0)
   %__exception_code = bitcast i8* %2 to i32*
   %3 = getelementptr inbounds i8, i8* %0, i32 -20
diff --git a/test/Instrumentation/AddressSanitizer/win-string-literal.ll b/test/Instrumentation/AddressSanitizer/win-string-literal.ll
index a86581c1e8e52b68f80b4aa228c3623082cc34ef..4d5126be87c6058fd6fd2064d06910759622913f 100644
--- a/test/Instrumentation/AddressSanitizer/win-string-literal.ll
+++ b/test/Instrumentation/AddressSanitizer/win-string-literal.ll
@@ -7,7 +7,7 @@
 ; CHECK: $"??_C@_04JIHMPGLA@asdf?$AA@" = comdat any
 
 ; CHECK: @"??_C@_04JIHMPGLA@asdf?$AA@" =
-; CHECK-SAME: linkonce_odr dso_local unnamed_addr constant { [5 x i8], [59 x i8] }
+; CHECK-SAME: linkonce_odr dso_local constant { [5 x i8], [59 x i8] }
 ; CHECK-SAME: { [5 x i8] c"asdf\00", [59 x i8] zeroinitializer }, comdat, align 32
 
 ; CHECK: @"__asan_global_??_C@_04JIHMPGLA@asdf?$AA@" =
@@ -15,7 +15,7 @@
 ; CHECK-SAME: { i64 ptrtoint ({ [5 x i8], [59 x i8] }* @"??_C@_04JIHMPGLA@asdf?$AA@" to i64),
 ; CHECK-SAME:   i64 5, i64 64, i64 ptrtoint ([17 x i8]* @___asan_gen_.1 to i64),
 ; CHECK-SAME:   i64 ptrtoint ([8 x i8]* @___asan_gen_ to i64), i64 0,
-; CHECK-SAME:   i64 ptrtoint ({ [6 x i8]*, i32, i32 }* @___asan_gen_.3 to i64), i64 -1 },
+; CHECK-SAME:   i64 ptrtoint ({ [6 x i8]*, i32, i32 }* @___asan_gen_.3 to i64), i64 0 },
 ; CHECK-SAME:   section ".ASAN$GL", comdat($"??_C@_04JIHMPGLA@asdf?$AA@"), align 64
 
 ; ModuleID = 't.cpp'
diff --git a/test/Instrumentation/HWAddressSanitizer/alloca.ll b/test/Instrumentation/HWAddressSanitizer/alloca.ll
index f13274171f0b0d8a460c57b0aec3bf60283a34fd..12175fa011fafa456190671418ac41146a796ac8 100644
--- a/test/Instrumentation/HWAddressSanitizer/alloca.ll
+++ b/test/Instrumentation/HWAddressSanitizer/alloca.ll
@@ -26,8 +26,7 @@ define void @test_alloca() sanitize_hwaddress {
 ; CHECK: %[[X_TAG2:[^ ]*]] = trunc i64 %[[X_TAG]] to i8
 ; CHECK: %[[E:[^ ]*]] = ptrtoint i32* %[[X]] to i64
 ; CHECK: %[[F:[^ ]*]] = lshr i64 %[[E]], 4
-; DYNAMIC-SHADOW: %[[F_DYN:[^ ]*]] = add i64 %[[F]], %.hwasan.shadow
-; DYNAMIC-SHADOW: %[[X_SHADOW:[^ ]*]] = inttoptr i64 %[[F_DYN]] to i8*
+; DYNAMIC-SHADOW: %[[X_SHADOW:[^ ]*]] = getelementptr i8, i8* %.hwasan.shadow, i64 %[[F]]
 ; ZERO-BASED-SHADOW: %[[X_SHADOW:[^ ]*]] = inttoptr i64 %[[F]] to i8*
 ; CHECK: call void @llvm.memset.p0i8.i64(i8* align 1 %[[X_SHADOW]], i8 %[[X_TAG2]], i64 1, i1 false)
 ; CHECK: call void @use32(i32* nonnull %[[X_HWASAN]])
@@ -36,8 +35,7 @@ define void @test_alloca() sanitize_hwaddress {
 ; UAR-TAGS: %[[X_TAG_UAR:[^ ]*]] = trunc i64 %[[BASE_TAG_COMPL]] to i8
 ; CHECK: %[[E2:[^ ]*]] = ptrtoint i32* %[[X]] to i64
 ; CHECK: %[[F2:[^ ]*]] = lshr i64 %[[E2]], 4
-; DYNAMIC-SHADOW: %[[F2_DYN:[^ ]*]] = add i64 %[[F2]], %.hwasan.shadow
-; DYNAMIC-SHADOW: %[[X_SHADOW2:[^ ]*]] = inttoptr i64 %[[F2_DYN]] to i8*
+; DYNAMIC-SHADOW: %[[X_SHADOW2:[^ ]*]] = getelementptr i8, i8* %.hwasan.shadow, i64 %[[F2]]
 ; ZERO-BASED-SHADOW: %[[X_SHADOW2:[^ ]*]] = inttoptr i64 %[[F2]] to i8*
 ; NO-UAR-TAGS: call void @llvm.memset.p0i8.i64(i8* align 1 %[[X_SHADOW2]], i8 0, i64 1, i1 false)
 ; UAR-TAGS: call void @llvm.memset.p0i8.i64(i8* align 1 %[[X_SHADOW2]], i8 %[[X_TAG_UAR]], i64 1, i1 false)
diff --git a/test/Instrumentation/HWAddressSanitizer/atomic.ll b/test/Instrumentation/HWAddressSanitizer/atomic.ll
index e6f7c2df23f528dd1eff5dbca725b6ff8e5fe255..8e8f34628f58e1390cc64e51b2dfb88f2428dc00 100644
--- a/test/Instrumentation/HWAddressSanitizer/atomic.ll
+++ b/test/Instrumentation/HWAddressSanitizer/atomic.ll
@@ -7,8 +7,8 @@ target triple = "aarch64--linux-android"
 
 define void @atomicrmw(i64* %ptr) sanitize_hwaddress {
 ; CHECK-LABEL: @atomicrmw(
-; CHECK: lshr i64 %[[A:[^ ]*]], 56
-; CHECK: call void asm sideeffect "brk #2323", "{x0}"(i64 %[[A]])
+; CHECK: [[PTRI8:%[^ ]*]] = bitcast i64* %ptr to i8*
+; CHECK: call void @llvm.hwasan.check.memaccess({{.*}}, i8* [[PTRI8]], i32 19)
 ; CHECK: atomicrmw add i64* %ptr, i64 1 seq_cst
 ; CHECK: ret void
 
@@ -19,8 +19,8 @@ entry:
 
 define void @cmpxchg(i64* %ptr, i64 %compare_to, i64 %new_value) sanitize_hwaddress {
 ; CHECK-LABEL: @cmpxchg(
-; CHECK: lshr i64 %[[A:[^ ]*]], 56
-; CHECK: call void asm sideeffect "brk #2323", "{x0}"(i64 %[[A]])
+; CHECK: [[PTRI8:%[^ ]*]] = bitcast i64* %ptr to i8*
+; CHECK: call void @llvm.hwasan.check.memaccess({{.*}}, i8* [[PTRI8]], i32 19)
 ; CHECK: cmpxchg i64* %ptr, i64 %compare_to, i64 %new_value seq_cst seq_cst
 ; CHECK: ret void
 
diff --git a/test/Instrumentation/HWAddressSanitizer/basic.ll b/test/Instrumentation/HWAddressSanitizer/basic.ll
index e02e5fc283b8406ee03851b48d20827caebb00a3..ec50ea788396a9e042c30b4310bec8e5df782d71 100644
--- a/test/Instrumentation/HWAddressSanitizer/basic.ll
+++ b/test/Instrumentation/HWAddressSanitizer/basic.ll
@@ -1,9 +1,9 @@
 ; Test basic address sanitizer instrumentation.
 ;
-; RUN: opt < %s -hwasan -hwasan-recover=0 -hwasan-with-ifunc=1 -hwasan-with-tls=0 -S | FileCheck %s --check-prefixes=CHECK,ABORT,DYNAMIC-SHADOW
-; RUN: opt < %s -hwasan -hwasan-recover=1 -hwasan-with-ifunc=1 -hwasan-with-tls=0 -S | FileCheck %s --check-prefixes=CHECK,RECOVER,DYNAMIC-SHADOW
-; RUN: opt < %s -hwasan -hwasan-recover=0 -hwasan-mapping-offset=0 -S | FileCheck %s --check-prefixes=CHECK,ABORT,ZERO-BASED-SHADOW
-; RUN: opt < %s -hwasan -hwasan-recover=1 -hwasan-mapping-offset=0 -S | FileCheck %s --check-prefixes=CHECK,RECOVER,ZERO-BASED-SHADOW
+; RUN: opt < %s -hwasan -hwasan-recover=0 -hwasan-with-ifunc=1 -hwasan-with-tls=0 -S | FileCheck %s --check-prefixes=CHECK,ABORT,ABORT-DYNAMIC-SHADOW
+; RUN: opt < %s -hwasan -hwasan-recover=1 -hwasan-with-ifunc=1 -hwasan-with-tls=0 -S | FileCheck %s --check-prefixes=CHECK,RECOVER,RECOVER-DYNAMIC-SHADOW
+; RUN: opt < %s -hwasan -hwasan-recover=0 -hwasan-mapping-offset=0 -S | FileCheck %s --check-prefixes=CHECK,ABORT,ABORT-ZERO-BASED-SHADOW
+; RUN: opt < %s -hwasan -hwasan-recover=1 -hwasan-mapping-offset=0 -S | FileCheck %s --check-prefixes=CHECK,RECOVER,RECOVER-ZERO-BASED-SHADOW
 
 ; CHECK: @llvm.global_ctors = appending global [1 x { i32, void ()*, i8* }] [{ i32, void ()*, i8* } { i32 0, void ()* @hwasan.module_ctor, i8* bitcast (void ()* @hwasan.module_ctor to i8*) }]
 ; CHECK: @__hwasan = private constant [0 x i8] zeroinitializer, section "__hwasan_frames", comdat($hwasan.module_ctor)
@@ -13,23 +13,23 @@ target triple = "aarch64--linux-android"
 
 define i8 @test_load8(i8* %a) sanitize_hwaddress {
 ; CHECK-LABEL: @test_load8(
-; CHECK: %[[A:[^ ]*]] = ptrtoint i8* %a to i64
-; CHECK: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
-; CHECK: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
-; CHECK: %[[C:[^ ]*]] = and i64 %[[A]], 72057594037927935
-; CHECK: %[[D:[^ ]*]] = lshr i64 %[[C]], 4
-; DYNAMIC-SHADOW: %[[D_DYN:[^ ]*]] = add i64 %[[D]], %.hwasan.shadow
-; DYNAMIC-SHADOW: %[[E:[^ ]*]] = inttoptr i64 %[[D_DYN]] to i8*
-; ZERO-BASED-SHADOW: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to i8*
-; CHECK: %[[MEMTAG:[^ ]*]] = load i8, i8* %[[E]]
-; CHECK: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
-; CHECK: br i1 %[[F]], label {{.*}}, label {{.*}}, !prof {{.*}}
-
-; ABORT: call void asm sideeffect "brk #2304", "{x0}"(i64 %[[A]])
-; ABORT: unreachable
+; RECOVER: %[[A:[^ ]*]] = ptrtoint i8* %a to i64
+; RECOVER: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
+; RECOVER: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
+; RECOVER: %[[C:[^ ]*]] = and i64 %[[A]], 72057594037927935
+; RECOVER: %[[D:[^ ]*]] = lshr i64 %[[C]], 4
+; RECOVER-DYNAMIC-SHADOW: %[[E:[^ ]*]] = getelementptr i8, i8* %.hwasan.shadow, i64 %4
+; RECOVER-ZERO-BASED-SHADOW: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to i8*
+; RECOVER: %[[MEMTAG:[^ ]*]] = load i8, i8* %[[E]]
+; RECOVER: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
+; RECOVER: br i1 %[[F]], label {{.*}}, label {{.*}}, !prof {{.*}}
+
 ; RECOVER: call void asm sideeffect "brk #2336", "{x0}"(i64 %[[A]])
 ; RECOVER: br label
 
+; ABORT-DYNAMIC-SHADOW: call void @llvm.hwasan.check.memaccess(i8* %.hwasan.shadow, i8* %a, i32 0)
+; ABORT-ZERO-BASED-SHADOW: call void @llvm.hwasan.check.memaccess(i8* null, i8* %a, i32 0)
+
 ; CHECK: %[[G:[^ ]*]] = load i8, i8* %a, align 4
 ; CHECK: ret i8 %[[G]]
 
@@ -40,23 +40,24 @@ entry:
 
 define i16 @test_load16(i16* %a) sanitize_hwaddress {
 ; CHECK-LABEL: @test_load16(
-; CHECK: %[[A:[^ ]*]] = ptrtoint i16* %a to i64
-; CHECK: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
-; CHECK: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
-; CHECK: %[[C:[^ ]*]] = and i64 %[[A]], 72057594037927935
-; CHECK: %[[D:[^ ]*]] = lshr i64 %[[C]], 4
-; DYNAMIC-SHADOW: %[[D_DYN:[^ ]*]] = add i64 %[[D]], %.hwasan.shadow
-; DYNAMIC-SHADOW: %[[E:[^ ]*]] = inttoptr i64 %[[D_DYN]] to i8*
-; ZERO-BASED-SHADOW: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to i8*
-; CHECK: %[[MEMTAG:[^ ]*]] = load i8, i8* %[[E]]
-; CHECK: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
-; CHECK: br i1 %[[F]], label {{.*}}, label {{.*}}, !prof {{.*}}
-
-; ABORT: call void asm sideeffect "brk #2305", "{x0}"(i64 %[[A]])
-; ABORT: unreachable
+; RECOVER: %[[A:[^ ]*]] = ptrtoint i16* %a to i64
+; RECOVER: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
+; RECOVER: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
+; RECOVER: %[[C:[^ ]*]] = and i64 %[[A]], 72057594037927935
+; RECOVER: %[[D:[^ ]*]] = lshr i64 %[[C]], 4
+; RECOVER-DYNAMIC-SHADOW: %[[E:[^ ]*]] = getelementptr i8, i8* %.hwasan.shadow, i64 %4
+; RECOVER-ZERO-BASED-SHADOW: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to i8*
+; RECOVER: %[[MEMTAG:[^ ]*]] = load i8, i8* %[[E]]
+; RECOVER: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
+; RECOVER: br i1 %[[F]], label {{.*}}, label {{.*}}, !prof {{.*}}
+
 ; RECOVER: call void asm sideeffect "brk #2337", "{x0}"(i64 %[[A]])
 ; RECOVER: br label
 
+; ABORT: %[[A:[^ ]*]] = bitcast i16* %a to i8*
+; ABORT-DYNAMIC-SHADOW: call void @llvm.hwasan.check.memaccess(i8* %.hwasan.shadow, i8* %[[A]], i32 1)
+; ABORT-ZERO-BASED-SHADOW: call void @llvm.hwasan.check.memaccess(i8* null, i8* %[[A]], i32 1)
+
 ; CHECK: %[[G:[^ ]*]] = load i16, i16* %a, align 4
 ; CHECK: ret i16 %[[G]]
 
@@ -67,23 +68,24 @@ entry:
 
 define i32 @test_load32(i32* %a) sanitize_hwaddress {
 ; CHECK-LABEL: @test_load32(
-; CHECK: %[[A:[^ ]*]] = ptrtoint i32* %a to i64
-; CHECK: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
-; CHECK: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
-; CHECK: %[[C:[^ ]*]] = and i64 %[[A]], 72057594037927935
-; CHECK: %[[D:[^ ]*]] = lshr i64 %[[C]], 4
-; DYNAMIC-SHADOW: %[[D_DYN:[^ ]*]] = add i64 %[[D]], %.hwasan.shadow
-; DYNAMIC-SHADOW: %[[E:[^ ]*]] = inttoptr i64 %[[D_DYN]] to i8*
-; ZERO-BASED-SHADOW: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to i8*
-; CHECK: %[[MEMTAG:[^ ]*]] = load i8, i8* %[[E]]
-; CHECK: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
-; CHECK: br i1 %[[F]], label {{.*}}, label {{.*}}, !prof {{.*}}
-
-; ABORT: call void asm sideeffect "brk #2306", "{x0}"(i64 %[[A]])
-; ABORT: unreachable
+; RECOVER: %[[A:[^ ]*]] = ptrtoint i32* %a to i64
+; RECOVER: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
+; RECOVER: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
+; RECOVER: %[[C:[^ ]*]] = and i64 %[[A]], 72057594037927935
+; RECOVER: %[[D:[^ ]*]] = lshr i64 %[[C]], 4
+; RECOVER-DYNAMIC-SHADOW: %[[E:[^ ]*]] = getelementptr i8, i8* %.hwasan.shadow, i64 %4
+; RECOVER-ZERO-BASED-SHADOW: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to i8*
+; RECOVER: %[[MEMTAG:[^ ]*]] = load i8, i8* %[[E]]
+; RECOVER: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
+; RECOVER: br i1 %[[F]], label {{.*}}, label {{.*}}, !prof {{.*}}
+
 ; RECOVER: call void asm sideeffect "brk #2338", "{x0}"(i64 %[[A]])
 ; RECOVER: br label
 
+; ABORT: %[[A:[^ ]*]] = bitcast i32* %a to i8*
+; ABORT-DYNAMIC-SHADOW: call void @llvm.hwasan.check.memaccess(i8* %.hwasan.shadow, i8* %[[A]], i32 2)
+; ABORT-ZERO-BASED-SHADOW: call void @llvm.hwasan.check.memaccess(i8* null, i8* %[[A]], i32 2)
+
 ; CHECK: %[[G:[^ ]*]] = load i32, i32* %a, align 4
 ; CHECK: ret i32 %[[G]]
 
@@ -94,23 +96,24 @@ entry:
 
 define i64 @test_load64(i64* %a) sanitize_hwaddress {
 ; CHECK-LABEL: @test_load64(
-; CHECK: %[[A:[^ ]*]] = ptrtoint i64* %a to i64
-; CHECK: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
-; CHECK: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
-; CHECK: %[[C:[^ ]*]] = and i64 %[[A]], 72057594037927935
-; CHECK: %[[D:[^ ]*]] = lshr i64 %[[C]], 4
-; DYNAMIC-SHADOW: %[[D_DYN:[^ ]*]] = add i64 %[[D]], %.hwasan.shadow
-; DYNAMIC-SHADOW: %[[E:[^ ]*]] = inttoptr i64 %[[D_DYN]] to i8*
-; ZERO-BASED-SHADOW: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to i8*
-; CHECK: %[[MEMTAG:[^ ]*]] = load i8, i8* %[[E]]
-; CHECK: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
-; CHECK: br i1 %[[F]], label {{.*}}, label {{.*}}, !prof {{.*}}
-
-; ABORT: call void asm sideeffect "brk #2307", "{x0}"(i64 %[[A]])
-; ABORT: unreachable
+; RECOVER: %[[A:[^ ]*]] = ptrtoint i64* %a to i64
+; RECOVER: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
+; RECOVER: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
+; RECOVER: %[[C:[^ ]*]] = and i64 %[[A]], 72057594037927935
+; RECOVER: %[[D:[^ ]*]] = lshr i64 %[[C]], 4
+; RECOVER-DYNAMIC-SHADOW: %[[E:[^ ]*]] = getelementptr i8, i8* %.hwasan.shadow, i64 %4
+; RECOVER-ZERO-BASED-SHADOW: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to i8*
+; RECOVER: %[[MEMTAG:[^ ]*]] = load i8, i8* %[[E]]
+; RECOVER: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
+; RECOVER: br i1 %[[F]], label {{.*}}, label {{.*}}, !prof {{.*}}
+
 ; RECOVER: call void asm sideeffect "brk #2339", "{x0}"(i64 %[[A]])
 ; RECOVER: br label
 
+; ABORT: %[[A:[^ ]*]] = bitcast i64* %a to i8*
+; ABORT-DYNAMIC-SHADOW: call void @llvm.hwasan.check.memaccess(i8* %.hwasan.shadow, i8* %[[A]], i32 3)
+; ABORT-ZERO-BASED-SHADOW: call void @llvm.hwasan.check.memaccess(i8* null, i8* %[[A]], i32 3)
+
 ; CHECK: %[[G:[^ ]*]] = load i64, i64* %a, align 8
 ; CHECK: ret i64 %[[G]]
 
@@ -121,23 +124,24 @@ entry:
 
 define i128 @test_load128(i128* %a) sanitize_hwaddress {
 ; CHECK-LABEL: @test_load128(
-; CHECK: %[[A:[^ ]*]] = ptrtoint i128* %a to i64
-; CHECK: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
-; CHECK: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
-; CHECK: %[[C:[^ ]*]] = and i64 %[[A]], 72057594037927935
-; CHECK: %[[D:[^ ]*]] = lshr i64 %[[C]], 4
-; DYNAMIC-SHADOW: %[[D_DYN:[^ ]*]] = add i64 %[[D]], %.hwasan.shadow
-; DYNAMIC-SHADOW: %[[E:[^ ]*]] = inttoptr i64 %[[D_DYN]] to i8*
-; ZERO-BASED-SHADOW: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to i8*
-; CHECK: %[[MEMTAG:[^ ]*]] = load i8, i8* %[[E]]
-; CHECK: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
-; CHECK: br i1 %[[F]], label {{.*}}, label {{.*}}, !prof {{.*}}
-
-; ABORT: call void asm sideeffect "brk #2308", "{x0}"(i64 %[[A]])
-; ABORT: unreachable
+; RECOVER: %[[A:[^ ]*]] = ptrtoint i128* %a to i64
+; RECOVER: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
+; RECOVER: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
+; RECOVER: %[[C:[^ ]*]] = and i64 %[[A]], 72057594037927935
+; RECOVER: %[[D:[^ ]*]] = lshr i64 %[[C]], 4
+; RECOVER-DYNAMIC-SHADOW: %[[E:[^ ]*]] = getelementptr i8, i8* %.hwasan.shadow, i64 %4
+; RECOVER-ZERO-BASED-SHADOW: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to i8*
+; RECOVER: %[[MEMTAG:[^ ]*]] = load i8, i8* %[[E]]
+; RECOVER: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
+; RECOVER: br i1 %[[F]], label {{.*}}, label {{.*}}, !prof {{.*}}
+
 ; RECOVER: call void asm sideeffect "brk #2340", "{x0}"(i64 %[[A]])
 ; RECOVER: br label
 
+; ABORT: %[[A:[^ ]*]] = bitcast i128* %a to i8*
+; ABORT-DYNAMIC-SHADOW: call void @llvm.hwasan.check.memaccess(i8* %.hwasan.shadow, i8* %[[A]], i32 4)
+; ABORT-ZERO-BASED-SHADOW: call void @llvm.hwasan.check.memaccess(i8* null, i8* %[[A]], i32 4)
+
 ; CHECK: %[[G:[^ ]*]] = load i128, i128* %a, align 16
 ; CHECK: ret i128 %[[G]]
 
@@ -161,23 +165,23 @@ entry:
 
 define void @test_store8(i8* %a, i8 %b) sanitize_hwaddress {
 ; CHECK-LABEL: @test_store8(
-; CHECK: %[[A:[^ ]*]] = ptrtoint i8* %a to i64
-; CHECK: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
-; CHECK: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
-; CHECK: %[[C:[^ ]*]] = and i64 %[[A]], 72057594037927935
-; CHECK: %[[D:[^ ]*]] = lshr i64 %[[C]], 4
-; DYNAMIC-SHADOW: %[[D_DYN:[^ ]*]] = add i64 %[[D]], %.hwasan.shadow
-; DYNAMIC-SHADOW: %[[E:[^ ]*]] = inttoptr i64 %[[D_DYN]] to i8*
-; ZERO-BASED-SHADOW: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to i8*
-; CHECK: %[[MEMTAG:[^ ]*]] = load i8, i8* %[[E]]
-; CHECK: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
-; CHECK: br i1 %[[F]], label {{.*}}, label {{.*}}, !prof {{.*}}
-
-; ABORT: call void asm sideeffect "brk #2320", "{x0}"(i64 %[[A]])
-; ABORT: unreachable
+; RECOVER: %[[A:[^ ]*]] = ptrtoint i8* %a to i64
+; RECOVER: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
+; RECOVER: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
+; RECOVER: %[[C:[^ ]*]] = and i64 %[[A]], 72057594037927935
+; RECOVER: %[[D:[^ ]*]] = lshr i64 %[[C]], 4
+; RECOVER-DYNAMIC-SHADOW: %[[E:[^ ]*]] = getelementptr i8, i8* %.hwasan.shadow, i64 %4
+; RECOVER-ZERO-BASED-SHADOW: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to i8*
+; RECOVER: %[[MEMTAG:[^ ]*]] = load i8, i8* %[[E]]
+; RECOVER: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
+; RECOVER: br i1 %[[F]], label {{.*}}, label {{.*}}, !prof {{.*}}
+
 ; RECOVER: call void asm sideeffect "brk #2352", "{x0}"(i64 %[[A]])
 ; RECOVER: br label
 
+; ABORT-DYNAMIC-SHADOW: call void @llvm.hwasan.check.memaccess(i8* %.hwasan.shadow, i8* %a, i32 16)
+; ABORT-ZERO-BASED-SHADOW: call void @llvm.hwasan.check.memaccess(i8* null, i8* %a, i32 16)
+
 ; CHECK: store i8 %b, i8* %a, align 4
 ; CHECK: ret void
 
@@ -188,23 +192,24 @@ entry:
 
 define void @test_store16(i16* %a, i16 %b) sanitize_hwaddress {
 ; CHECK-LABEL: @test_store16(
-; CHECK: %[[A:[^ ]*]] = ptrtoint i16* %a to i64
-; CHECK: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
-; CHECK: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
-; CHECK: %[[C:[^ ]*]] = and i64 %[[A]], 72057594037927935
-; CHECK: %[[D:[^ ]*]] = lshr i64 %[[C]], 4
-; DYNAMIC-SHADOW: %[[D_DYN:[^ ]*]] = add i64 %[[D]], %.hwasan.shadow
-; DYNAMIC-SHADOW: %[[E:[^ ]*]] = inttoptr i64 %[[D_DYN]] to i8*
-; ZERO-BASED-SHADOW: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to i8*
-; CHECK: %[[MEMTAG:[^ ]*]] = load i8, i8* %[[E]]
-; CHECK: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
-; CHECK: br i1 %[[F]], label {{.*}}, label {{.*}}, !prof {{.*}}
-
-; ABORT: call void asm sideeffect "brk #2321", "{x0}"(i64 %[[A]])
-; ABORT: unreachable
+; RECOVER: %[[A:[^ ]*]] = ptrtoint i16* %a to i64
+; RECOVER: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
+; RECOVER: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
+; RECOVER: %[[C:[^ ]*]] = and i64 %[[A]], 72057594037927935
+; RECOVER: %[[D:[^ ]*]] = lshr i64 %[[C]], 4
+; RECOVER-DYNAMIC-SHADOW: %[[E:[^ ]*]] = getelementptr i8, i8* %.hwasan.shadow, i64 %4
+; RECOVER-ZERO-BASED-SHADOW: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to i8*
+; RECOVER: %[[MEMTAG:[^ ]*]] = load i8, i8* %[[E]]
+; RECOVER: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
+; RECOVER: br i1 %[[F]], label {{.*}}, label {{.*}}, !prof {{.*}}
+
 ; RECOVER: call void asm sideeffect "brk #2353", "{x0}"(i64 %[[A]])
 ; RECOVER: br label
 
+; ABORT: %[[A:[^ ]*]] = bitcast i16* %a to i8*
+; ABORT-DYNAMIC-SHADOW: call void @llvm.hwasan.check.memaccess(i8* %.hwasan.shadow, i8* %[[A]], i32 17)
+; ABORT-ZERO-BASED-SHADOW: call void @llvm.hwasan.check.memaccess(i8* null, i8* %[[A]], i32 17)
+
 ; CHECK: store i16 %b, i16* %a, align 4
 ; CHECK: ret void
 
@@ -215,23 +220,24 @@ entry:
 
 define void @test_store32(i32* %a, i32 %b) sanitize_hwaddress {
 ; CHECK-LABEL: @test_store32(
-; CHECK: %[[A:[^ ]*]] = ptrtoint i32* %a to i64
-; CHECK: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
-; CHECK: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
-; CHECK: %[[C:[^ ]*]] = and i64 %[[A]], 72057594037927935
-; CHECK: %[[D:[^ ]*]] = lshr i64 %[[C]], 4
-; DYNAMIC-SHADOW: %[[D_DYN:[^ ]*]] = add i64 %[[D]], %.hwasan.shadow
-; DYNAMIC-SHADOW: %[[E:[^ ]*]] = inttoptr i64 %[[D_DYN]] to i8*
-; ZERO-BASED-SHADOW: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to i8*
-; CHECK: %[[MEMTAG:[^ ]*]] = load i8, i8* %[[E]]
-; CHECK: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
-; CHECK: br i1 %[[F]], label {{.*}}, label {{.*}}, !prof {{.*}}
-
-; ABORT: call void asm sideeffect "brk #2322", "{x0}"(i64 %[[A]])
-; ABORT: unreachable
+; RECOVER: %[[A:[^ ]*]] = ptrtoint i32* %a to i64
+; RECOVER: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
+; RECOVER: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
+; RECOVER: %[[C:[^ ]*]] = and i64 %[[A]], 72057594037927935
+; RECOVER: %[[D:[^ ]*]] = lshr i64 %[[C]], 4
+; RECOVER-DYNAMIC-SHADOW: %[[E:[^ ]*]] = getelementptr i8, i8* %.hwasan.shadow, i64 %4
+; RECOVER-ZERO-BASED-SHADOW: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to i8*
+; RECOVER: %[[MEMTAG:[^ ]*]] = load i8, i8* %[[E]]
+; RECOVER: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
+; RECOVER: br i1 %[[F]], label {{.*}}, label {{.*}}, !prof {{.*}}
+
 ; RECOVER: call void asm sideeffect "brk #2354", "{x0}"(i64 %[[A]])
 ; RECOVER: br label
 
+; ABORT: %[[A:[^ ]*]] = bitcast i32* %a to i8*
+; ABORT-DYNAMIC-SHADOW: call void @llvm.hwasan.check.memaccess(i8* %.hwasan.shadow, i8* %[[A]], i32 18)
+; ABORT-ZERO-BASED-SHADOW: call void @llvm.hwasan.check.memaccess(i8* null, i8* %[[A]], i32 18)
+
 ; CHECK: store i32 %b, i32* %a, align 4
 ; CHECK: ret void
 
@@ -242,23 +248,24 @@ entry:
 
 define void @test_store64(i64* %a, i64 %b) sanitize_hwaddress {
 ; CHECK-LABEL: @test_store64(
-; CHECK: %[[A:[^ ]*]] = ptrtoint i64* %a to i64
-; CHECK: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
-; CHECK: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
-; CHECK: %[[C:[^ ]*]] = and i64 %[[A]], 72057594037927935
-; CHECK: %[[D:[^ ]*]] = lshr i64 %[[C]], 4
-; DYNAMIC-SHADOW: %[[D_DYN:[^ ]*]] = add i64 %[[D]], %.hwasan.shadow
-; DYNAMIC-SHADOW: %[[E:[^ ]*]] = inttoptr i64 %[[D_DYN]] to i8*
-; ZERO-BASED-SHADOW: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to i8*
-; CHECK: %[[MEMTAG:[^ ]*]] = load i8, i8* %[[E]]
-; CHECK: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
-; CHECK: br i1 %[[F]], label {{.*}}, label {{.*}}, !prof {{.*}}
-
-; ABORT: call void asm sideeffect "brk #2323", "{x0}"(i64 %[[A]])
-; ABORT: unreachable
+; RECOVER: %[[A:[^ ]*]] = ptrtoint i64* %a to i64
+; RECOVER: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
+; RECOVER: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
+; RECOVER: %[[C:[^ ]*]] = and i64 %[[A]], 72057594037927935
+; RECOVER: %[[D:[^ ]*]] = lshr i64 %[[C]], 4
+; RECOVER-DYNAMIC-SHADOW: %[[E:[^ ]*]] = getelementptr i8, i8* %.hwasan.shadow, i64 %4
+; RECOVER-ZERO-BASED-SHADOW: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to i8*
+; RECOVER: %[[MEMTAG:[^ ]*]] = load i8, i8* %[[E]]
+; RECOVER: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
+; RECOVER: br i1 %[[F]], label {{.*}}, label {{.*}}, !prof {{.*}}
+
 ; RECOVER: call void asm sideeffect "brk #2355", "{x0}"(i64 %[[A]])
 ; RECOVER: br label
 
+; ABORT: %[[A:[^ ]*]] = bitcast i64* %a to i8*
+; ABORT-DYNAMIC-SHADOW: call void @llvm.hwasan.check.memaccess(i8* %.hwasan.shadow, i8* %[[A]], i32 19)
+; ABORT-ZERO-BASED-SHADOW: call void @llvm.hwasan.check.memaccess(i8* null, i8* %[[A]], i32 19)
+
 ; CHECK: store i64 %b, i64* %a, align 8
 ; CHECK: ret void
 
@@ -269,23 +276,24 @@ entry:
 
 define void @test_store128(i128* %a, i128 %b) sanitize_hwaddress {
 ; CHECK-LABEL: @test_store128(
-; CHECK: %[[A:[^ ]*]] = ptrtoint i128* %a to i64
-; CHECK: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
-; CHECK: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
-; CHECK: %[[C:[^ ]*]] = and i64 %[[A]], 72057594037927935
-; CHECK: %[[D:[^ ]*]] = lshr i64 %[[C]], 4
-; DYNAMIC-SHADOW: %[[D_DYN:[^ ]*]] = add i64 %[[D]], %.hwasan.shadow
-; DYNAMIC-SHADOW: %[[E:[^ ]*]] = inttoptr i64 %[[D_DYN]] to i8*
-; ZERO-BASED-SHADOW: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to i8*
-; CHECK: %[[MEMTAG:[^ ]*]] = load i8, i8* %[[E]]
-; CHECK: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
-; CHECK: br i1 %[[F]], label {{.*}}, label {{.*}}, !prof {{.*}}
-
-; ABORT: call void asm sideeffect "brk #2324", "{x0}"(i64 %[[A]])
-; ABORT: unreachable
+; RECOVER: %[[A:[^ ]*]] = ptrtoint i128* %a to i64
+; RECOVER: %[[B:[^ ]*]] = lshr i64 %[[A]], 56
+; RECOVER: %[[PTRTAG:[^ ]*]] = trunc i64 %[[B]] to i8
+; RECOVER: %[[C:[^ ]*]] = and i64 %[[A]], 72057594037927935
+; RECOVER: %[[D:[^ ]*]] = lshr i64 %[[C]], 4
+; RECOVER-DYNAMIC-SHADOW: %[[E:[^ ]*]] = getelementptr i8, i8* %.hwasan.shadow, i64 %4
+; RECOVER-ZERO-BASED-SHADOW: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to i8*
+; RECOVER: %[[MEMTAG:[^ ]*]] = load i8, i8* %[[E]]
+; RECOVER: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
+; RECOVER: br i1 %[[F]], label {{.*}}, label {{.*}}, !prof {{.*}}
+
 ; RECOVER: call void asm sideeffect "brk #2356", "{x0}"(i64 %[[A]])
 ; RECOVER: br label
 
+; ABORT: %[[A:[^ ]*]] = bitcast i128* %a to i8*
+; ABORT-DYNAMIC-SHADOW: call void @llvm.hwasan.check.memaccess(i8* %.hwasan.shadow, i8* %[[A]], i32 20)
+; ABORT-ZERO-BASED-SHADOW: call void @llvm.hwasan.check.memaccess(i8* null, i8* %[[A]], i32 20)
+
 ; CHECK: store i128 %b, i128* %a, align 16
 ; CHECK: ret void
 
diff --git a/test/Instrumentation/HWAddressSanitizer/kernel.ll b/test/Instrumentation/HWAddressSanitizer/kernel.ll
index ad3dba7f8962c8f1e144eb6ceaa9c35ca1666cbd..3f5891c1dddbaa822ff5d44c0b9eb90994169b62 100644
--- a/test/Instrumentation/HWAddressSanitizer/kernel.ll
+++ b/test/Instrumentation/HWAddressSanitizer/kernel.ll
@@ -1,11 +1,9 @@
 ; Test KHWASan instrumentation.
 ;
-; RUN: opt < %s -hwasan -hwasan-kernel=1 -S | FileCheck %s --allow-empty --check-prefixes=INIT
-; RUN: opt < %s -hwasan -hwasan-kernel=1 -S | FileCheck %s  --check-prefixes=CHECK,NOOFFSET,MATCH-ALL
-; RUN: opt < %s -hwasan -hwasan-kernel=1 -hwasan-mapping-offset=12345678 -S | FileCheck %s  --check-prefixes=CHECK,OFFSET,MATCH-ALL
-; RUN: opt < %s -hwasan -hwasan-kernel=1 -hwasan-recover=0 -S | FileCheck %s  --check-prefixes=CHECK,NOOFFSET,ABORT,MATCH-ALL
-; RUN: opt < %s -hwasan -hwasan-kernel=1 -hwasan-recover=1 -S | FileCheck %s  --check-prefixes=CHECK,NOOFFSET,RECOVER,MATCH-ALL
-; RUN: opt < %s -hwasan -hwasan-kernel=1 -hwasan-recover=1 -hwasan-match-all-tag=-1 -S | FileCheck %s  --check-prefixes=CHECK,NOOFFSET,RECOVER,NO-MATCH-ALL
+; RUN: opt < %s -hwasan -hwasan-kernel=1 -hwasan-recover=1 -S | FileCheck %s --allow-empty --check-prefixes=INIT
+; RUN: opt < %s -hwasan -hwasan-kernel=1 -hwasan-recover=1 -S | FileCheck %s  --check-prefixes=CHECK,NOOFFSET,MATCH-ALL
+; RUN: opt < %s -hwasan -hwasan-kernel=1 -hwasan-recover=1 -hwasan-mapping-offset=12345678 -S | FileCheck %s  --check-prefixes=CHECK,OFFSET,MATCH-ALL
+; RUN: opt < %s -hwasan -hwasan-kernel=1 -hwasan-recover=1 -hwasan-match-all-tag=-1 -S | FileCheck %s  --check-prefixes=CHECK,NOOFFSET,NO-MATCH-ALL
 
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 target triple = "aarch64--linux-android"
@@ -20,8 +18,7 @@ define i8 @test_load(i8* %a) sanitize_hwaddress {
 
 ; NOOFFSET: %[[E:[^ ]*]] = inttoptr i64 %[[D]] to i8*
 
-; OFFSET: %[[D1:[^ ]*]] = add i64 %[[D]], 12345678
-; OFFSET: %[[E:[^ ]*]] = inttoptr i64 %[[D1]] to i8*
+; OFFSET: %[[E:[^ ]*]] = getelementptr i8, i8* inttoptr (i64 12345678 to i8*), i64 %[[D]]
 
 ; CHECK: %[[MEMTAG:[^ ]*]] = load i8, i8* %[[E]]
 ; CHECK: %[[F:[^ ]*]] = icmp ne i8 %[[PTRTAG]], %[[MEMTAG]]
@@ -32,10 +29,8 @@ define i8 @test_load(i8* %a) sanitize_hwaddress {
 
 ; NO-MATCH-ALL: br i1 %[[F]], label {{.*}}, label {{.*}}, !prof {{.*}}
 
-; ABORT: call void asm sideeffect "brk #2304", "{x0}"(i64 %[[A]])
-; ABORT: unreachable
-; RECOVER: call void asm sideeffect "brk #2336", "{x0}"(i64 %[[A]])
-; RECOVER: br label
+; CHECK: call void asm sideeffect "brk #2336", "{x0}"(i64 %[[A]])
+; CHECK: br label
 
 ; CHECK: %[[G:[^ ]*]] = load i8, i8* %a, align 4
 ; CHECK: ret i8 %[[G]]
diff --git a/test/Instrumentation/HWAddressSanitizer/lazy-thread-init.ll b/test/Instrumentation/HWAddressSanitizer/lazy-thread-init.ll
new file mode 100644
index 0000000000000000000000000000000000000000..d5a2541eb7763505cdb66c51f62339ad707b5354
--- /dev/null
+++ b/test/Instrumentation/HWAddressSanitizer/lazy-thread-init.ll
@@ -0,0 +1,39 @@
+; RUN: opt -S -hwasan -hwasan-allow-ifunc < %s | FileCheck %s
+
+target triple = "aarch64--linux-android"
+
+declare i32 @bar([16 x i32]* %p)
+
+define void @alloca() sanitize_hwaddress "hwasan-abi"="interceptor" {
+  ; CHECK: alloca [16 x i32]
+  ; CHECK: [[A:%[^ ]*]] = call i8* @llvm.thread.pointer()
+  ; CHECK: [[B:%[^ ]*]] = getelementptr i8, i8* [[A]], i32 48
+  ; CHECK: [[C:%[^ ]*]] = bitcast i8* [[B]] to i64*
+  ; CHECK: [[LOAD:%[^ ]*]] = load i64, i64* [[C]]
+  ; CHECK: [[ICMP:%[^ ]*]] = icmp eq i64 [[LOAD]], 0
+  ; CHECK: br i1 [[ICMP]], label %[[INIT:[^,]*]], label %[[CONT:[^,]*]], !prof [[PROF:![0-9]+]]
+
+  ; CHECK: [[INIT]]:
+  ; CHECK: call void @__hwasan_thread_enter()
+  ; CHECK: [[RELOAD:%[^ ]*]] = load i64, i64* [[C]]
+  ; CHECK: br label %[[CONT]]
+
+  ; CHECK: [[CONT]]:
+  ; CHECK: phi i64 [ [[LOAD]], %0 ], [ [[RELOAD]], %[[INIT]] ]
+  ; CHECK: alloca i8
+
+  %p = alloca [16 x i32]
+  %size = call i32 @bar([16 x i32]* %p)
+  %q = alloca i8, i32 %size
+  ret void
+}
+
+define i32 @load(i32* %p) sanitize_hwaddress "hwasan-abi"="interceptor" {
+  ; CHECK: [[SHADOW:%[^ ]*]] = call i8* asm "", "=r,0"([0 x i8]* @__hwasan_shadow)
+  ; CHECK-NOT: icmp
+  ; CHECK: call void @llvm.hwasan.check.memaccess(i8* [[SHADOW]],
+  %v = load i32, i32* %p
+  ret i32 %v
+}
+
+; CHECK: [[PROF]] = !{!"branch_weights", i32 1, i32 100000}
diff --git a/test/Instrumentation/HWAddressSanitizer/mem-intrinsics.ll b/test/Instrumentation/HWAddressSanitizer/mem-intrinsics.ll
new file mode 100644
index 0000000000000000000000000000000000000000..b6a4a30ce566563649a8571cbacaeb537dbb6cc3
--- /dev/null
+++ b/test/Instrumentation/HWAddressSanitizer/mem-intrinsics.ll
@@ -0,0 +1,40 @@
+; RUN: opt -S -hwasan %s | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; Function Attrs: noinline nounwind optnone uwtable
+define dso_local i32 @main() sanitize_hwaddress {
+entry:
+  %retval = alloca i32, align 4
+  %Q = alloca [10 x i8], align 1
+  %P = alloca [10 x i8], align 1
+  store i32 0, i32* %retval, align 4
+  %arraydecay = getelementptr inbounds [10 x i8], [10 x i8]* %Q, i32 0, i32 0
+
+  call void @llvm.memset.p0i8.i64(i8* align 1 %arraydecay, i8 0, i64 10, i1 false)
+; CHECK: call i8* @__hwasan_memset
+
+  %arraydecay1 = getelementptr inbounds [10 x i8], [10 x i8]* %Q, i32 0, i32 0
+  %arraydecay2 = getelementptr inbounds [10 x i8], [10 x i8]* %Q, i32 0, i32 0
+  %add.ptr = getelementptr inbounds i8, i8* %arraydecay2, i64 5
+
+  call void @llvm.memmove.p0i8.p0i8.i64(i8* align 1 %arraydecay1, i8* align 1 %add.ptr, i64 5, i1 false)
+; CHECK: call i8* @__hwasan_memmove
+
+  %arraydecay3 = getelementptr inbounds [10 x i8], [10 x i8]* %P, i32 0, i32 0
+  %arraydecay4 = getelementptr inbounds [10 x i8], [10 x i8]* %Q, i32 0, i32 0
+
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 1 %arraydecay3, i8* align 1 %arraydecay4, i64 10, i1 false)
+; CHECK: call i8* @__hwasan_memcpy
+  ret i32 0
+}
+
+; Function Attrs: argmemonly nounwind
+declare void @llvm.memset.p0i8.i64(i8* nocapture writeonly, i8, i64, i1) #1
+
+; Function Attrs: argmemonly nounwind
+declare void @llvm.memmove.p0i8.p0i8.i64(i8* nocapture, i8* nocapture readonly, i64, i1) #1
+
+; Function Attrs: argmemonly nounwind
+declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture writeonly, i8* nocapture readonly, i64, i1) #1
diff --git a/test/Instrumentation/HWAddressSanitizer/prologue.ll b/test/Instrumentation/HWAddressSanitizer/prologue.ll
index 6b02b9863cd2e5e1f49be5fd7b40c593077f35be..1163dbe72debad7b8384bd019d9ce37d907ead87 100644
--- a/test/Instrumentation/HWAddressSanitizer/prologue.ll
+++ b/test/Instrumentation/HWAddressSanitizer/prologue.ll
@@ -1,15 +1,15 @@
 ; Test -hwasan-with-ifunc flag.
 ;
-; RUN: opt -hwasan -S < %s | \
-; RUN:     FileCheck %s --check-prefixes=CHECK,CHECK-TLS,CHECK-HISTORY
-; RUN: opt -hwasan -S -hwasan-with-ifunc=0 -hwasan-with-tls=1 -hwasan-record-stack-history=1 < %s | \
-; RUN:     FileCheck %s --check-prefixes=CHECK,CHECK-TLS,CHECK-HISTORY
-; RUN: opt -hwasan -S -hwasan-with-ifunc=0 -hwasan-with-tls=1 -hwasan-record-stack-history=0 < %s | \
-; RUN:     FileCheck %s --check-prefixes=CHECK,CHECK-TLS,CHECK-NOHISTORY
-; RUN: opt -hwasan -S -hwasan-with-ifunc=0 -hwasan-with-tls=0 < %s | \
+; RUN: opt -hwasan -hwasan-allow-ifunc -S < %s | \
+; RUN:     FileCheck %s --check-prefixes=CHECK,CHECK-NOGLOBAL,CHECK-TLS,CHECK-HISTORY
+; RUN: opt -hwasan -hwasan-allow-ifunc -S -hwasan-with-ifunc=0 -hwasan-with-tls=1 -hwasan-record-stack-history=1 < %s | \
+; RUN:     FileCheck %s --check-prefixes=CHECK,CHECK-NOGLOBAL,CHECK-TLS,CHECK-HISTORY
+; RUN: opt -hwasan -hwasan-allow-ifunc -S -hwasan-with-ifunc=0 -hwasan-with-tls=1 -hwasan-record-stack-history=0 < %s | \
+; RUN:     FileCheck %s --check-prefixes=CHECK,CHECK-NOGLOBAL,CHECK-IFUNC,CHECK-NOHISTORY
+; RUN: opt -hwasan -hwasan-allow-ifunc -S -hwasan-with-ifunc=0 -hwasan-with-tls=0 < %s | \
 ; RUN:     FileCheck %s --check-prefixes=CHECK,CHECK-GLOBAL,CHECK-NOHISTORY
-; RUN: opt -hwasan -S -hwasan-with-ifunc=1  -hwasan-with-tls=0 < %s | \
-; RUN:     FileCheck %s --check-prefixes=CHECK,CHECK-IFUNC,CHECK-NOHISTORY
+; RUN: opt -hwasan -hwasan-allow-ifunc -S -hwasan-with-ifunc=1  -hwasan-with-tls=0 < %s | \
+; RUN:     FileCheck %s --check-prefixes=CHECK,CHECk-NOGLOBAL,CHECK-IFUNC,CHECK-NOHISTORY
 
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 target triple = "aarch64--linux-android22"
@@ -23,17 +23,10 @@ define i32 @test_load(i32* %a) sanitize_hwaddress {
 ; CHECK-LABEL: @test_load
 ; CHECK: entry:
 
-; CHECK-IFUNC:   %[[A:[^ ]*]] = call i64 asm "", "=r,0"([0 x i8]* @__hwasan_shadow)
-; CHECK-IFUNC:   add i64 %{{.*}}, %[[A]]
+; CHECK-NOGLOBAL:   %[[A:[^ ]*]] = call i8* asm "", "=r,0"([0 x i8]* @__hwasan_shadow)
+; CHECK-NOGLOBAL:   @llvm.hwasan.check.memaccess(i8* %[[A]]
 
-; CHECK-GLOBAL: load i64, i64* @__hwasan_shadow_memory_dynamic_address
-
-; CHECK-TLS:   %[[A:[^ ]*]] = call i8* @llvm.thread.pointer()
-; CHECK-TLS:   %[[B:[^ ]*]] = getelementptr i8, i8* %[[A]], i32 64
-; CHECK-TLS:   %[[C:[^ ]*]] = bitcast i8* %[[B]] to i64*
-; CHECK-TLS:   %[[D:[^ ]*]] = load i64, i64* %[[C]]
-; CHECK-TLS:   %[[E:[^ ]*]] = or i64 %[[D]], 4294967295
-; CHECK-TLS:   = add i64 %[[E]], 1
+; CHECK-GLOBAL: load i8*, i8** @__hwasan_shadow_memory_dynamic_address
 
 ; "store i64" is only used to update stack history (this input IR intentionally does not use any i64)
 ; W/o any allocas, the history is not updated, even if it is enabled explicitly with -hwasan-record-stack-history=1
@@ -54,13 +47,13 @@ define void @test_alloca() sanitize_hwaddress {
 ; CHECK-LABEL: @test_alloca
 ; CHECK: entry:
 
-; CHECK-IFUNC:   %[[A:[^ ]*]] = call i64 asm "", "=r,0"([0 x i8]* @__hwasan_shadow)
-; CHECK-IFUNC:   add i64 %{{.*}}, %[[A]]
+; CHECK-IFUNC:   %[[A:[^ ]*]] = call i8* asm "", "=r,0"([0 x i8]* @__hwasan_shadow)
+; CHECK-IFUNC:   getelementptr i8, i8* %[[A]]
 
-; CHECK-GLOBAL: load i64, i64* @__hwasan_shadow_memory_dynamic_address
+; CHECK-GLOBAL: load i8*, i8** @__hwasan_shadow_memory_dynamic_address
 
 ; CHECK-TLS:   %[[A:[^ ]*]] = call i8* @llvm.thread.pointer()
-; CHECK-TLS:   %[[B:[^ ]*]] = getelementptr i8, i8* %[[A]], i32 64
+; CHECK-TLS:   %[[B:[^ ]*]] = getelementptr i8, i8* %[[A]], i32 48
 ; CHECK-TLS:   %[[C:[^ ]*]] = bitcast i8* %[[B]] to i64*
 ; CHECK-TLS:   %[[D:[^ ]*]] = load i64, i64* %[[C]]
 
diff --git a/test/Instrumentation/InstrProfiling/PR23499.ll b/test/Instrumentation/InstrProfiling/PR23499.ll
index 101fad19950e6c75a2992ad445581bbd7fceed6e..7f6bed689954119c4e96d63ecb3873fd0b53ec00 100644
--- a/test/Instrumentation/InstrProfiling/PR23499.ll
+++ b/test/Instrumentation/InstrProfiling/PR23499.ll
@@ -20,8 +20,8 @@ $_Z3barIvEvv = comdat any
 
 
 ; COFF-NOT: __profn__Z3barIvEvv
-; COFF: @__profc__Z3barIvEvv = linkonce_odr hidden global [1 x i64] zeroinitializer, section "{{.*}}prfc", comdat, align 8
-; COFF: @__profd__Z3barIvEvv = linkonce_odr hidden global { i64, i64, i64*, i8*, i8*, i32, [2 x i16] } { i64 4947693190065689389, i64 0, i64* getelementptr inbounds ([1 x i64], [1 x i64]* @__profc__Z3barIvEvv, i32 0, i32 0), i8*{{.*}}, i8* null, i32 1, [2 x i16] zeroinitializer }, section "{{.*}}prfd{{.*}}", comdat($__profc__Z3barIvEvv), align 8
+; COFF: @__profc__Z3barIvEvv = internal global [1 x i64] zeroinitializer, section "{{.*}}prfc$M", comdat($_Z3barIvEvv), align 8
+; COFF: @__profd__Z3barIvEvv = internal global { i64, i64, i64*, i8*, i8*, i32, [2 x i16] } { i64 4947693190065689389, i64 0, i64* getelementptr inbounds ([1 x i64], [1 x i64]* @__profc__Z3barIvEvv, i32 0, i32 0), i8*{{.*}}, i8* null, i32 1, [2 x i16] zeroinitializer }, section "{{.*}}prfd{{.*}}", comdat($_Z3barIvEvv), align 8
 
 
 declare void @llvm.instrprof.increment(i8*, i64, i32, i32) #1
diff --git a/test/Instrumentation/InstrProfiling/comdat.ll b/test/Instrumentation/InstrProfiling/comdat.ll
new file mode 100644
index 0000000000000000000000000000000000000000..c3bb7be47f6d2dc8578d803a6b2e7aea1967835b
--- /dev/null
+++ b/test/Instrumentation/InstrProfiling/comdat.ll
@@ -0,0 +1,38 @@
+; RUN: opt < %s -mtriple=x86_64-unknown-linux -instrprof -S | FileCheck %s --check-prefixes=ELF
+; RUN: opt < %s  -mtriple=x86_64-pc-win32-coff -instrprof -S | FileCheck %s --check-prefixes=COFF
+; RUN: opt < %s -mtriple=x86_64-unknown-linux -passes=instrprof -S | FileCheck %s --check-prefixes=ELF
+; RUN: opt < %s  -mtriple=x86_64-pc-win32-coff -passes=instrprof -S | FileCheck %s --check-prefixes=COFF
+
+; There are two main cases where comdats are necessary:
+; 1. standard inline functions (weak_odr / linkonce_odr)
+; 2. available externally functions (C99 inline / extern template / dllimport)
+; Check that we do the right thing for the two object formats with comdats, ELF
+; and COFF.
+
+declare void @llvm.instrprof.increment(i8*, i64, i32, i32)
+
+$foo_inline = comdat any
+
+@__profn_foo_inline = linkonce_odr hidden constant [10 x i8] c"foo_inline"
+
+; ELF: @__profc_foo_inline = linkonce_odr hidden global{{.*}}, section "__llvm_prf_cnts", comdat($__profv_foo_inline), align 8
+; ELF: @__profd_foo_inline = linkonce_odr hidden global{{.*}}, section "__llvm_prf_data", comdat($__profv_foo_inline), align 8
+; COFF: @__profc_foo_inline = internal global{{.*}}, section ".lprfc$M", comdat($foo_inline), align 8
+; COFF: @__profd_foo_inline = internal global{{.*}}, section ".lprfd$M", comdat($foo_inline), align 8
+define weak_odr void @foo_inline() comdat {
+  call void @llvm.instrprof.increment(i8* getelementptr inbounds ([10 x i8], [10 x i8]* @__profn_foo_inline, i32 0, i32 0), i64 0, i32 1, i32 0)
+  ret void
+}
+
+$foo_extern = comdat any
+
+@__profn_foo_extern = linkonce_odr hidden constant [10 x i8] c"foo_extern"
+
+; ELF: @__profc_foo_extern = linkonce_odr hidden global{{.*}}, section "__llvm_prf_cnts", comdat($__profv_foo_extern)
+; ELF: @__profd_foo_extern = linkonce_odr hidden global{{.*}}, section "__llvm_prf_data", comdat($__profv_foo_extern)
+; COFF: @__profc_foo_extern = linkonce_odr dso_local global{{.*}}, section ".lprfc$M", comdat, align 8
+; COFF: @__profd_foo_extern = internal global{{.*}}, section ".lprfd$M", comdat($__profc_foo_extern), align 8
+define available_externally void @foo_extern() {
+  call void @llvm.instrprof.increment(i8* getelementptr inbounds ([10 x i8], [10 x i8]* @__profn_foo_extern, i32 0, i32 0), i64 0, i32 1, i32 0)
+  ret void
+}
diff --git a/test/Instrumentation/InstrProfiling/linkage.ll b/test/Instrumentation/InstrProfiling/linkage.ll
index 97537579b17602e7ccb309be90a5424b0cc306d9..29ae5d6e90cb5fdd4519930c33477087366415cf 100644
--- a/test/Instrumentation/InstrProfiling/linkage.ll
+++ b/test/Instrumentation/InstrProfiling/linkage.ll
@@ -1,11 +1,11 @@
 ;; Check that runtime symbols get appropriate linkage.
 
-; RUN: opt < %s -mtriple=x86_64-apple-macosx10.10.0 -instrprof -S | FileCheck %s --check-prefixes=COMMON,MACHO
-; RUN: opt < %s -mtriple=x86_64-unknown-linux -instrprof -S | FileCheck %s --check-prefixes=COMMON,LINUX
-; RUN: opt < %s  -mtriple=x86_64-pc-win32-coff -instrprof -S | FileCheck %s --check-prefixes=COMMON,COFF
-; RUN: opt < %s -mtriple=x86_64-apple-macosx10.10.0 -passes=instrprof -S | FileCheck %s --check-prefixes=COMMON,MACHO
-; RUN: opt < %s -mtriple=x86_64-unknown-linux -passes=instrprof -S | FileCheck %s --check-prefixes=COMMON,LINUX
-; RUN: opt < %s  -mtriple=x86_64-pc-win32-coff -passes=instrprof -S | FileCheck %s --check-prefixes=COMMON,COFF
+; RUN: opt < %s -mtriple=x86_64-apple-macosx10.10.0 -instrprof -S | FileCheck %s --check-prefixes=POSIX,MACHO
+; RUN: opt < %s -mtriple=x86_64-unknown-linux -instrprof -S | FileCheck %s --check-prefixes=POSIX,LINUX
+; RUN: opt < %s  -mtriple=x86_64-pc-win32-coff -instrprof -S | FileCheck %s --check-prefixes=COFF
+; RUN: opt < %s -mtriple=x86_64-apple-macosx10.10.0 -passes=instrprof -S | FileCheck %s --check-prefixes=POSIX,MACHO
+; RUN: opt < %s -mtriple=x86_64-unknown-linux -passes=instrprof -S | FileCheck %s --check-prefixes=POSIX,LINUX
+; RUN: opt < %s  -mtriple=x86_64-pc-win32-coff -passes=instrprof -S | FileCheck %s --check-prefixes=COFF
 
 ; MACHO: @__llvm_profile_runtime = external global i32
 ; LINUX-NOT: @__llvm_profile_runtime = external global i32
@@ -16,31 +16,38 @@
 @__profn_foo_inline = linkonce_odr hidden constant [10 x i8] c"foo_inline"
 @__profn_foo_extern = linkonce_odr hidden constant [10 x i8] c"foo_extern"
 
-; COMMON: @__profc_foo = hidden global
-; COMMON: @__profd_foo = hidden global
+; POSIX: @__profc_foo = hidden global
+; POSIX: @__profd_foo = hidden global
+; COFF: @__profc_foo = internal global
+; COFF-NOT: comdat
+; COFF: @__profd_foo = internal global
 define void @foo() {
   call void @llvm.instrprof.increment(i8* getelementptr inbounds ([3 x i8], [3 x i8]* @__profn_foo, i32 0, i32 0), i64 0, i32 1, i32 0)
   ret void
 }
 
-; COMMON: @__profc_foo_weak = weak hidden global
-; COMMON: @__profd_foo_weak = weak hidden global
+; POSIX: @__profc_foo_weak = weak hidden global
+; POSIX: @__profd_foo_weak = weak hidden global
+; COFF: @__profc_foo_weak = internal global
+; COFF: @__profd_foo_weak = internal global
 define weak void @foo_weak() {
   call void @llvm.instrprof.increment(i8* getelementptr inbounds ([8 x i8], [8 x i8]* @__profn_foo_weak, i32 0, i32 0), i64 0, i32 1, i32 0)
   ret void
 }
 
-; COMMON: @"__profc_linkage.ll:foo_internal" = internal global
-; COMMON: @"__profd_linkage.ll:foo_internal" = internal global
+; POSIX: @"__profc_linkage.ll:foo_internal" = internal global
+; POSIX: @"__profd_linkage.ll:foo_internal" = internal global
+; COFF: @"__profc_linkage.ll:foo_internal" = internal global
+; COFF: @"__profd_linkage.ll:foo_internal" = internal global
 define internal void @foo_internal() {
   call void @llvm.instrprof.increment(i8* getelementptr inbounds ([23 x i8], [23 x i8]* @"__profn_linkage.ll:foo_internal", i32 0, i32 0), i64 0, i32 1, i32 0)
   ret void
 }
 
-; COMMON: @__profc_foo_inline = linkonce_odr hidden global
-; COFF-SAME: section ".lprfc", align 8
-; COMMON: @__profd_foo_inline = linkonce_odr hidden global
-; COFF-SAME: section ".lprfd", align 8
+; POSIX: @__profc_foo_inline = linkonce_odr hidden global
+; POSIX: @__profd_foo_inline = linkonce_odr hidden global
+; COFF: @__profc_foo_inline = internal global{{.*}} section ".lprfc$M", align 8
+; COFF: @__profd_foo_inline = internal global{{.*}} section ".lprfd$M", align 8
 define linkonce_odr void @foo_inline() {
   call void @llvm.instrprof.increment(i8* getelementptr inbounds ([10 x i8], [10 x i8]* @__profn_foo_inline, i32 0, i32 0), i64 0, i32 1, i32 0)
   ret void
@@ -50,8 +57,8 @@ define linkonce_odr void @foo_inline() {
 ; LINUX: @__profd_foo_extern = linkonce_odr hidden global {{.*}}section "__llvm_prf_data", comdat($__profv_foo_extern), align 8
 ; MACHO: @__profc_foo_extern = linkonce_odr hidden global
 ; MACHO: @__profd_foo_extern = linkonce_odr hidden global
-; COFF: @__profc_foo_extern = linkonce_odr hidden global {{.*}}section ".lprfc", comdat, align 8
-; COFF: @__profd_foo_extern = linkonce_odr hidden global {{.*}}section ".lprfd", comdat($__profc_foo_extern), align 8
+; COFF: @__profc_foo_extern = linkonce_odr dso_local global {{.*}}section ".lprfc$M", comdat, align 8
+; COFF: @__profd_foo_extern = internal global {{.*}}section ".lprfd$M", comdat($__profc_foo_extern), align 8
 define available_externally void @foo_extern() {
   call void @llvm.instrprof.increment(i8* getelementptr inbounds ([10 x i8], [10 x i8]* @__profn_foo_extern, i32 0, i32 0), i64 0, i32 1, i32 0)
   ret void
diff --git a/test/Instrumentation/InstrProfiling/platform.ll b/test/Instrumentation/InstrProfiling/platform.ll
index dbdde08b81524ef8d6cea9c06daee4fcae7aa984..4c54aea9aeccbcf40ab54ca65e9ee778f2e77d6d 100644
--- a/test/Instrumentation/InstrProfiling/platform.ll
+++ b/test/Instrumentation/InstrProfiling/platform.ll
@@ -20,13 +20,14 @@
 
 ; MACHO: @__profc_foo = hidden global [1 x i64] zeroinitializer, section "__DATA,__llvm_prf_cnts", align 8
 ; ELF: @__profc_foo = hidden global [1 x i64] zeroinitializer, section "__llvm_prf_cnts", align 8
-; WINDOWS: @__profc_foo = hidden global [1 x i64] zeroinitializer, section ".lprfc", align 8
+; WINDOWS: @__profc_foo = internal global [1 x i64] zeroinitializer, section ".lprfc$M", align 8
 
 ; MACHO: @__profd_foo = hidden {{.*}}, section "__DATA,__llvm_prf_data,regular,live_support", align 8
 ; ELF: @__profd_foo = hidden {{.*}}, section "__llvm_prf_data", align 8
-; WINDOWS: @__profd_foo = hidden {{.*}}, section ".lprfd", align 8
+; WINDOWS: @__profd_foo = internal global {{.*}}, section ".lprfd$M", align 8
 
-; ELF: @__llvm_prf_nm = private constant [{{.*}} x i8] c"{{.*}}", section "{{.*}}__llvm_prf_names"
+; ELF: @__llvm_prf_nm = private constant [{{.*}} x i8] c"{{.*}}", section "{{.*}}__llvm_prf_names", align 1
+; WINDOWS: @__llvm_prf_nm = private constant [{{.*}} x i8] c"{{.*}}", section "{{.*}}lprfn$M", align 1
 
 define void @foo() {
   call void @llvm.instrprof.increment(i8* getelementptr inbounds ([3 x i8], [3 x i8]* @__profn_foo, i32 0, i32 0), i64 0, i32 1, i32 0)
@@ -42,14 +43,11 @@ declare void @llvm.instrprof.increment(i8*, i64, i32, i32)
 ; LINUX-NOT: define internal void @__llvm_profile_register_functions
 ; FREEBSD-NOT: define internal void @__llvm_profile_register_functions
 ; PS4-NOT: define internal void @__llvm_profile_register_functions
+; WINDOWS-NOT: define internal void @__llvm_profile_register_functions
 
 ;; PR38340: When dynamic registration is used, we had a bug where we'd register
 ;; something that's not a __profd_* variable.
 
-; WINDOWS: define internal void @__llvm_profile_register_functions()
-; WINDOWS-NOT: __llvm_profile_runtime_user
-; WINDOWS: ret void
-
 ; SOLARIS: define internal void @__llvm_profile_register_functions
 ; SOLARIS-NOT: __llvm_profile_runtime_user
 ; SOLARIS: ret void
@@ -58,5 +56,5 @@ declare void @llvm.instrprof.increment(i8*, i64, i32, i32)
 ; LINUX-NOT: define internal void @__llvm_profile_init
 ; FREEBSD-NOT: define internal void @__llvm_profile_init
 ; PS4-NOT: define internal void @__llvm_profile_init
+; WINDOWS-NOT: define internal void @__llvm_profile_init
 ; SOLARIS: define internal void @__llvm_profile_init
-; WINDOWS: define internal void @__llvm_profile_init
diff --git a/test/Instrumentation/MemorySanitizer/AArch64/vararg.ll b/test/Instrumentation/MemorySanitizer/AArch64/vararg.ll
index 08387f8d1e92fa45ed5310a257372ec918fa1873..2f401b0eb8c68efc2b721ee993b6bfddbba9128e 100644
--- a/test/Instrumentation/MemorySanitizer/AArch64/vararg.ll
+++ b/test/Instrumentation/MemorySanitizer/AArch64/vararg.ll
@@ -1,3 +1,4 @@
+; RUN: opt < %s -S -passes=msan 2>&1 | FileCheck %s
 ; RUN: opt < %s -msan -S | FileCheck %s
 
 target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
diff --git a/test/Instrumentation/MemorySanitizer/Mips/vararg-mips64.ll b/test/Instrumentation/MemorySanitizer/Mips/vararg-mips64.ll
index 1463070ad853915268612553cd46576a69512c76..03d1fda5eda68918757a0f0b9b3c9f93b8d75b84 100644
--- a/test/Instrumentation/MemorySanitizer/Mips/vararg-mips64.ll
+++ b/test/Instrumentation/MemorySanitizer/Mips/vararg-mips64.ll
@@ -1,3 +1,4 @@
+; RUN: opt < %s -S -passes=msan 2>&1 | FileCheck %s
 ; RUN: opt < %s -msan -S | FileCheck %s
 
 target datalayout = "E-m:m-i8:8:32-i16:16:32-i64:64-n32:64-S128"
diff --git a/test/Instrumentation/MemorySanitizer/Mips/vararg-mips64el.ll b/test/Instrumentation/MemorySanitizer/Mips/vararg-mips64el.ll
index 9edcb3666f2190ed4db770181fde02ac37f5a85a..98b1d82110234898005b58e777858aa6c7e3e7c3 100644
--- a/test/Instrumentation/MemorySanitizer/Mips/vararg-mips64el.ll
+++ b/test/Instrumentation/MemorySanitizer/Mips/vararg-mips64el.ll
@@ -1,3 +1,4 @@
+; RUN: opt < %s -S -passes=msan 2>&1 | FileCheck %s
 ; RUN: opt < %s -msan -S | FileCheck %s
 
 target datalayout = "e-m:m-i8:8:32-i16:16:32-i64:64-n32:64-S128"
diff --git a/test/Instrumentation/MemorySanitizer/PowerPC/vararg-ppc64.ll b/test/Instrumentation/MemorySanitizer/PowerPC/vararg-ppc64.ll
index 5a2686900e9ca378f3e9994811d1e05245decb6c..9e74752bcd50900dc9984a68af17a5357a2f9ee0 100644
--- a/test/Instrumentation/MemorySanitizer/PowerPC/vararg-ppc64.ll
+++ b/test/Instrumentation/MemorySanitizer/PowerPC/vararg-ppc64.ll
@@ -1,3 +1,4 @@
+; RUN: opt < %s -S -passes=msan 2>&1 | FileCheck %s
 ; RUN: opt < %s -msan -S | FileCheck %s
 
 target datalayout = "E-m:e-i64:64-n32:64"
diff --git a/test/Instrumentation/MemorySanitizer/PowerPC/vararg-ppc64le.ll b/test/Instrumentation/MemorySanitizer/PowerPC/vararg-ppc64le.ll
index c139d62c4f28fc8198b3266078d7b07e0d185453..9d6c43657ed6557311320b3be0eab4af9f4c6021 100644
--- a/test/Instrumentation/MemorySanitizer/PowerPC/vararg-ppc64le.ll
+++ b/test/Instrumentation/MemorySanitizer/PowerPC/vararg-ppc64le.ll
@@ -1,3 +1,4 @@
+; RUN: opt < %s -S -passes=msan 2>&1 | FileCheck %s
 ; RUN: opt < %s -msan -S | FileCheck %s
 
 target datalayout = "e-m:e-i64:64-n32:64"
diff --git a/test/Instrumentation/MemorySanitizer/X86/vararg-too-large.ll b/test/Instrumentation/MemorySanitizer/X86/vararg-too-large.ll
index 800bd15f124bdbdaefdf36e6203176927b683efd..cd113a03059fc32883f5071a761d44f685c5db0c 100644
--- a/test/Instrumentation/MemorySanitizer/X86/vararg-too-large.ll
+++ b/test/Instrumentation/MemorySanitizer/X86/vararg-too-large.ll
@@ -1,3 +1,5 @@
+; RUN: opt < %s -msan-check-access-address=0 -S 2>&1 -passes=msan | FileCheck  \
+; RUN: %s
 ; RUN: opt < %s -msan -msan-check-access-address=0 -S 2>&1 | FileCheck %s
 
 ; Test that MSan doesn't generate code overflowing __msan_va_arg_tls when too many arguments are
diff --git a/test/Instrumentation/MemorySanitizer/X86/vararg.ll b/test/Instrumentation/MemorySanitizer/X86/vararg.ll
index 518c3dbf41d3689631e46438d2b162b4ef9a63b8..f384d8fbd905f01765217f916c532db8a62b1f0f 100644
--- a/test/Instrumentation/MemorySanitizer/X86/vararg.ll
+++ b/test/Instrumentation/MemorySanitizer/X86/vararg.ll
@@ -1,3 +1,4 @@
+; RUN: opt < %s -msan-check-access-address=0 -S -passes=msan 2>&1
 ; RUN: opt < %s -msan -msan-check-access-address=0 -S
 ; Test that code using va_start can be compiled on i386.
 
diff --git a/test/Instrumentation/MemorySanitizer/X86/vararg_call.ll b/test/Instrumentation/MemorySanitizer/X86/vararg_call.ll
index 3d036d7d8533f337d83060278edec049df18e0e1..eea84451fb59891e9d997608ce7d94833db8a30d 100644
--- a/test/Instrumentation/MemorySanitizer/X86/vararg_call.ll
+++ b/test/Instrumentation/MemorySanitizer/X86/vararg_call.ll
@@ -1,5 +1,13 @@
+; RUN: opt < %s -msan-check-access-address=0 -S -passes=msan 2>&1 | FileCheck  \
+; RUN: %s
 ; RUN: opt < %s -msan -msan-check-access-address=0 -S | FileCheck %s
+; RUN: opt < %s -msan-check-access-address=0 -msan-track-origins=1 -S          \
+; RUN: -passes=msan 2>&1 | FileCheck %s "--check-prefixes=CHECK,CHECK-ORIGIN"
+; RUN: opt < %s -msan-check-access-address=0 -S          \
+; RUN: -passes="msan<track-origins=1>" 2>&1 | FileCheck %s "--check-prefixes=CHECK,CHECK-ORIGIN"
 ; RUN: opt < %s -msan -msan-check-access-address=0 -msan-track-origins=1 -S | FileCheck %s --check-prefixes=CHECK,CHECK-ORIGIN
+; RUN: opt < %s -msan-check-access-address=0 -msan-track-origins=2 -S          \
+; RUN: -passes=msan 2>&1 | FileCheck %s "--check-prefixes=CHECK,CHECK-ORIGIN"
 ; RUN: opt < %s -msan -msan-check-access-address=0 -msan-track-origins=2 -S | FileCheck %s --check-prefixes=CHECK,CHECK-ORIGIN
 
 ; Test that shadow and origin are stored for variadic function params.
diff --git a/test/Instrumentation/MemorySanitizer/alloca.ll b/test/Instrumentation/MemorySanitizer/alloca.ll
index d1d3bc55a2bf8b21a7779f44220bfe39e77829b8..84b3efc707db456dee03bf0cc45f17d0fc35c39b 100644
--- a/test/Instrumentation/MemorySanitizer/alloca.ll
+++ b/test/Instrumentation/MemorySanitizer/alloca.ll
@@ -1,7 +1,19 @@
+; RUN: opt < %s -msan-check-access-address=0 -S -passes=msan 2>&1 | FileCheck  \
+; RUN: %s "--check-prefixes=CHECK,INLINE"
 ; RUN: opt < %s -msan -msan-check-access-address=0 -S | FileCheck %s --check-prefixes=CHECK,INLINE
+; RUN: opt < %s -msan-check-access-address=0 -msan-poison-stack-with-call=1 -S \
+; RUN: -passes=msan 2>&1 | FileCheck %s "--check-prefixes=CHECK,CALL"
 ; RUN: opt < %s -msan -msan-check-access-address=0 -msan-poison-stack-with-call=1 -S | FileCheck %s --check-prefixes=CHECK,CALL
+; RUN: opt < %s -msan-check-access-address=0 -msan-track-origins=1 -S          \
+; RUN: -passes=msan 2>&1 | FileCheck %s "--check-prefixes=CHECK,ORIGIN"
 ; RUN: opt < %s -msan -msan-check-access-address=0 -msan-track-origins=1 -S | FileCheck %s --check-prefixes=CHECK,ORIGIN
+; RUN: opt < %s -msan-check-access-address=0 -msan-track-origins=2 -S          \
+; RUN: -passes=msan 2>&1 | FileCheck %s "--check-prefixes=CHECK,ORIGIN"
 ; RUN: opt < %s -msan -msan-check-access-address=0 -msan-track-origins=2 -S | FileCheck %s --check-prefixes=CHECK,ORIGIN
+; RUN: opt < %s -S -passes="msan<kernel>" 2>&1 | FileCheck %s             \
+; RUN: "--check-prefixes=CHECK,KMSAN"
+; RUN: opt < %s -msan-kernel=1 -S -passes=msan 2>&1 | FileCheck %s             \
+; RUN: "--check-prefixes=CHECK,KMSAN"
 ; RUN: opt < %s -msan -msan-kernel=1 -S | FileCheck %s --check-prefixes=CHECK,KMSAN
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
diff --git a/test/Instrumentation/MemorySanitizer/array_types.ll b/test/Instrumentation/MemorySanitizer/array_types.ll
index e96716aaa97fe39a9e0b52b42fc4186687d1e8a2..d322354188daa8386169c399a1c7f6286b971598 100644
--- a/test/Instrumentation/MemorySanitizer/array_types.ll
+++ b/test/Instrumentation/MemorySanitizer/array_types.ll
@@ -1,4 +1,9 @@
+; RUN: opt < %s -msan-check-access-address=0 -S -passes=msan 2>&1 | FileCheck  \
+; RUN: %s
 ; RUN: opt < %s -msan -msan-check-access-address=0 -S | FileCheck %s
+; RUN: opt < %s -msan-check-access-address=0 -msan-track-origins=1 -S          \
+; RUN: -passes=msan 2>&1 | FileCheck -check-prefix=CHECK                       \
+; RUN: -check-prefix=CHECK-ORIGINS %s --allow-empty
 ; RUN: opt < %s -msan -msan-check-access-address=0 -msan-track-origins=1 -S | FileCheck -check-prefix=CHECK -check-prefix=CHECK-ORIGINS %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
diff --git a/test/Instrumentation/MemorySanitizer/atomics.ll b/test/Instrumentation/MemorySanitizer/atomics.ll
index 8033ed1e2d461269c62bbab8237cc059b52372b2..56f12a1fe6b91a4007fb3c0a3e37d3591c38450a 100644
--- a/test/Instrumentation/MemorySanitizer/atomics.ll
+++ b/test/Instrumentation/MemorySanitizer/atomics.ll
@@ -1,5 +1,11 @@
+; RUN: opt < %s -msan-check-access-address=0 -S -passes=msan 2>&1 | FileCheck  \
+; RUN: %s
 ; RUN: opt < %s -msan -msan-check-access-address=0 -S | FileCheck %s
+; RUN: opt < %s -msan-check-access-address=0 -msan-track-origins=1 -S          \
+; RUN: -passes=msan 2>&1 | FileCheck %s
 ; RUN: opt < %s -msan -msan-check-access-address=0 -msan-track-origins=1 -S | FileCheck %s
+; RUN: opt < %s -msan-check-access-address=0 -msan-track-origins=2 -S          \
+; RUN: -passes=msan 2>&1 | FileCheck %s
 ; RUN: opt < %s -msan -msan-check-access-address=0 -msan-track-origins=2 -S | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
diff --git a/test/Instrumentation/MemorySanitizer/byval-alignment.ll b/test/Instrumentation/MemorySanitizer/byval-alignment.ll
index df82a92f743f6fb39a93c9a74f71ca34088a6567..0d5da362f34c7cda71174c07fcad0cedc98c3987 100644
--- a/test/Instrumentation/MemorySanitizer/byval-alignment.ll
+++ b/test/Instrumentation/MemorySanitizer/byval-alignment.ll
@@ -1,5 +1,7 @@
 ; Test that copy alignment for byval arguments is limited by param-tls slot alignment.
 
+; RUN: opt < %s -msan-check-access-address=0 -S -passes=msan 2>&1 | FileCheck  \
+; RUN: %s
 ; RUN: opt < %s -msan -msan-check-access-address=0 -S | FileCheck %s
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
diff --git a/test/Instrumentation/MemorySanitizer/check-constant-shadow.ll b/test/Instrumentation/MemorySanitizer/check-constant-shadow.ll
index f1479440faf6f520578a83483c0b0c25136829cb..101107543a629f2aac207562d9b33bd82a87534c 100644
--- a/test/Instrumentation/MemorySanitizer/check-constant-shadow.ll
+++ b/test/Instrumentation/MemorySanitizer/check-constant-shadow.ll
@@ -1,3 +1,5 @@
+; RUN: opt < %s -msan-check-access-address=0 -msan-check-constant-shadow=1     \
+; RUN: -msan-track-origins=1 -S -passes=msan 2>&1 | FileCheck %s
 ; RUN: opt < %s -msan -msan-check-access-address=0 -msan-check-constant-shadow=1 -msan-track-origins=1 -S | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
diff --git a/test/Instrumentation/MemorySanitizer/check_access_address.ll b/test/Instrumentation/MemorySanitizer/check_access_address.ll
index 21bb41256061c45e3631a1ce1be5064fd23e2ced..8c245fed50c34becf5f9645004c3d0b3d36b6095 100644
--- a/test/Instrumentation/MemorySanitizer/check_access_address.ll
+++ b/test/Instrumentation/MemorySanitizer/check_access_address.ll
@@ -1,3 +1,5 @@
+; RUN: opt < %s -msan-check-access-address=1 -S -passes=msan 2>&1 | FileCheck  \
+; RUN: %s
 ; RUN: opt < %s -msan -msan-check-access-address=1 -S | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
diff --git a/test/Instrumentation/MemorySanitizer/csr.ll b/test/Instrumentation/MemorySanitizer/csr.ll
index a7664d45636846cc9c141a26ffe35ebce94fdfcb..f91888a43358adb53c8743930ff7b488fc8569bb 100644
--- a/test/Instrumentation/MemorySanitizer/csr.ll
+++ b/test/Instrumentation/MemorySanitizer/csr.ll
@@ -1,4 +1,8 @@
+; RUN: opt < %s -msan-check-access-address=0 -S -passes=msan 2>&1 | FileCheck  \
+; RUN: %s
 ; RUN: opt < %s -msan -msan-check-access-address=0 -S | FileCheck %s
+; RUN: opt < %s -msan-check-access-address=1 -S -passes=msan 2>&1 | FileCheck  \
+; RUN: %s --check-prefix=ADDR
 ; RUN: opt < %s -msan -msan-check-access-address=1 -S | FileCheck %s --check-prefix=ADDR
 ; REQUIRES: x86-registered-target
 
diff --git a/test/Instrumentation/MemorySanitizer/global_ctors_2to3.ll b/test/Instrumentation/MemorySanitizer/global_ctors_2to3.ll
index 8e0ab2683015b9f3ca32e611670e1b35e828683e..d841c6c05c9b768ae3db6c492ce74bb5d55485a7 100644
--- a/test/Instrumentation/MemorySanitizer/global_ctors_2to3.ll
+++ b/test/Instrumentation/MemorySanitizer/global_ctors_2to3.ll
@@ -1,4 +1,5 @@
 ; MSan converts 2-element global_ctors to 3-element when adding the new entry.
+; RUN: opt < %s -msan-with-comdat -S -passes=msan 2>&1 | FileCheck %s
 ; RUN: opt < %s -msan -msan-with-comdat -S | FileCheck %s
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
diff --git a/test/Instrumentation/MemorySanitizer/instrumentation-with-call-threshold.ll b/test/Instrumentation/MemorySanitizer/instrumentation-with-call-threshold.ll
index 855125a88de8c980c84482ad409a8ccdf2dff359..424a5e05ffef4cfab9026be2bdcdd297bcca0223 100644
--- a/test/Instrumentation/MemorySanitizer/instrumentation-with-call-threshold.ll
+++ b/test/Instrumentation/MemorySanitizer/instrumentation-with-call-threshold.ll
@@ -2,8 +2,19 @@
 ; Test that in with-calls mode there are no calls to __msan_chain_origin - they
 ; are done from __msan_maybe_store_origin_*.
 
+; RUN: opt < %s -msan-check-access-address=0                                   \
+; RUN: -msan-instrumentation-with-call-threshold=0 -S -passes=msan 2>&1 |      \
+; RUN: FileCheck %s
 ; RUN: opt < %s -msan -msan-check-access-address=0 -msan-instrumentation-with-call-threshold=0 -S | FileCheck %s
+; RUN: opt < %s -msan-check-access-address=0                                   \
+; RUN: -msan-instrumentation-with-call-threshold=0 -msan-track-origins=1 -S    \
+; RUN: -passes=msan 2>&1 | FileCheck -check-prefix=CHECK                       \
+; RUN: -check-prefix=CHECK-ORIGINS %s
 ; RUN: opt < %s -msan -msan-check-access-address=0 -msan-instrumentation-with-call-threshold=0 -msan-track-origins=1 -S | FileCheck -check-prefix=CHECK -check-prefix=CHECK-ORIGINS %s
+; RUN: opt < %s -msan-check-access-address=0                                   \
+; RUN: -msan-instrumentation-with-call-threshold=0 -msan-track-origins=2 -S    \
+; RUN: -passes=msan 2>&1 | FileCheck -check-prefix=CHECK                       \
+; RUN: -check-prefix=CHECK-ORIGINS %s
 ; RUN: opt < %s -msan -msan-check-access-address=0 -msan-instrumentation-with-call-threshold=0 -msan-track-origins=2 -S | FileCheck -check-prefix=CHECK -check-prefix=CHECK-ORIGINS %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
diff --git a/test/Instrumentation/MemorySanitizer/manual-shadow.ll b/test/Instrumentation/MemorySanitizer/manual-shadow.ll
index f4664cba37e9ceae48a5a9b51226e5cbebea0ca6..ab19f3d9b78fd78ddef4b77463ea121c2632c3da 100644
--- a/test/Instrumentation/MemorySanitizer/manual-shadow.ll
+++ b/test/Instrumentation/MemorySanitizer/manual-shadow.ll
@@ -1,9 +1,20 @@
 ; Test that the msan layout customization options work as expected
 ;
+; RUN: opt < %s -msan-shadow-base 3735928559 -S -passes=msan 2>&1 | FileCheck  \
+; RUN: --check-prefix=CHECK-BASE %s
 ; RUN: opt < %s -msan -msan-shadow-base 3735928559 -S | FileCheck --check-prefix=CHECK-BASE %s
+; RUN: opt < %s -msan-shadow-base 3735928559 -msan-and-mask 4294901760 -S      \
+; RUN: -passes=msan 2>&1 | FileCheck --check-prefix=CHECK-AND %s
 ; RUN: opt < %s -msan -msan-shadow-base 3735928559 -msan-and-mask 4294901760 -S | FileCheck --check-prefix=CHECK-AND %s
+; RUN: opt < %s -msan-shadow-base 3735928559 -msan-xor-mask 48879 -S           \
+; RUN: -passes=msan 2>&1 | FileCheck --check-prefix=CHECK-XOR %s
 ; RUN: opt < %s -msan -msan-shadow-base 3735928559 -msan-xor-mask 48879 -S | FileCheck --check-prefix=CHECK-XOR %s
+; RUN: opt < %s -msan-shadow-base 3735928559 -msan-xor-mask 48879              \
+; RUN: -msan-and-mask 4294901760 -S -passes=msan 2>&1 | FileCheck              \
+; RUN: --check-prefix=CHECK-XOR-AND %s
 ; RUN: opt < %s -msan -msan-shadow-base 3735928559 -msan-xor-mask 48879 -msan-and-mask 4294901760 -S | FileCheck --check-prefix=CHECK-XOR-AND %s
+; RUN: opt < %s -msan-track-origins 1 -msan-origin-base 1777777 -S -passes=msan\
+; RUN: 2>&1 | FileCheck --check-prefix=CHECK-ORIGIN-BASE %s
 ; RUN: opt < %s -msan -msan-track-origins 1 -msan-origin-base 1777777 -S | FileCheck --check-prefix=CHECK-ORIGIN-BASE %s
 
 target triple = "x86_64-unknown-linux-gnu"
diff --git a/test/Instrumentation/MemorySanitizer/masked-store-load.ll b/test/Instrumentation/MemorySanitizer/masked-store-load.ll
index 9391c9f8f26060aa00dfb85b1b369dfc66bc172c..226f66515b64d46bcf2cc94498615b4188ba7752 100644
--- a/test/Instrumentation/MemorySanitizer/masked-store-load.ll
+++ b/test/Instrumentation/MemorySanitizer/masked-store-load.ll
@@ -1,5 +1,11 @@
+; RUN: opt < %s -msan-check-access-address=0 -S -passes=msan 2>&1 | FileCheck  \
+; RUN: %s
 ; RUN: opt < %s -msan -msan-check-access-address=0 -S | FileCheck %s
+; RUN: opt < %s -msan-check-access-address=0 -msan-track-origins=1 -S          \
+; RUN: -passes=msan 2>&1 | FileCheck %s "--check-prefixes=CHECK,CHECK-ORIGIN"
 ; RUN: opt < %s -msan -msan-check-access-address=0 -msan-track-origins=1 -S | FileCheck %s --check-prefixes=CHECK,CHECK-ORIGIN
+; RUN: opt < %s -msan-check-access-address=1 -S -passes=msan 2>&1 | FileCheck  \
+; RUN: %s --check-prefix=ADDR
 ; RUN: opt < %s -msan -msan-check-access-address=1 -S | FileCheck %s --check-prefix=ADDR
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
diff --git a/test/Instrumentation/MemorySanitizer/missing_origin.ll b/test/Instrumentation/MemorySanitizer/missing_origin.ll
index 0a18914d42a713913362de9c198e7d4aeb9dc951..f529098627a4b06e1930f483bdcc2db8a42a9353 100644
--- a/test/Instrumentation/MemorySanitizer/missing_origin.ll
+++ b/test/Instrumentation/MemorySanitizer/missing_origin.ll
@@ -1,3 +1,5 @@
+; RUN: opt < %s -msan-check-access-address=0 -msan-track-origins=1 -S          \
+; RUN: -passes=msan 2>&1 | FileCheck %s
 ; RUN: opt < %s -msan -msan-check-access-address=0 -msan-track-origins=1 -S | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
diff --git a/test/Instrumentation/MemorySanitizer/msan_asm_conservative.ll b/test/Instrumentation/MemorySanitizer/msan_asm_conservative.ll
index bc6c44c2cb2a4d39d5e0fb70d35f5ca3e44d4fb2..c338d049206c1eb7aeed7e2981ddd6d0e7942741 100644
--- a/test/Instrumentation/MemorySanitizer/msan_asm_conservative.ll
+++ b/test/Instrumentation/MemorySanitizer/msan_asm_conservative.ll
@@ -1,5 +1,11 @@
 ; Test for handling of asm constraints in MSan instrumentation.
+; RUN: opt < %s -msan-kernel=1 -msan-check-access-address=0                    \
+; RUN: -msan-handle-asm-conservative=0 -S -passes=msan 2>&1 | FileCheck        \
+; RUN: "-check-prefixes=CHECK,CHECK-NONCONS" %s
 ; RUN: opt < %s -msan -msan-kernel=1 -msan-check-access-address=0 -msan-handle-asm-conservative=0 -S | FileCheck -check-prefixes=CHECK,CHECK-NONCONS %s
+; RUN: opt < %s -msan-kernel=1 -msan-check-access-address=0                    \
+; RUN: -msan-handle-asm-conservative=1 -S -passes=msan 2>&1 | FileCheck        \
+; RUN: "-check-prefixes=CHECK,CHECK-CONS" %s
 ; RUN: opt < %s -msan -msan-kernel=1 -msan-check-access-address=0 -msan-handle-asm-conservative=1 -S | FileCheck -check-prefixes=CHECK,CHECK-CONS %s
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
@@ -179,8 +185,6 @@ entry:
 }
 
 ; CHECK-LABEL: @f_2i_2o_mem
-; CHECK-CONS: call void @__msan_instrument_asm_load({{.*}}@is1{{.*}}, i64 4)
-; CHECK-CONS: call void @__msan_instrument_asm_load({{.*}}@is2{{.*}}, i64 4)
 ; CHECK-CONS: call void @__msan_instrument_asm_store({{.*}}@id1{{.*}}, i64 4)
 ; CHECK-CONS: call void @__msan_instrument_asm_store({{.*}}@id2{{.*}}, i64 4)
 ; CHECK: call void asm "", "=*m,=*m,*m,*m,~{dirflag},~{fpsr},~{flags}"(i32* @id1, i32* @id2, i32* @is1, i32* @is2)
@@ -198,7 +202,6 @@ entry:
 
 ; CHECK-LABEL: @f_1i_1o_memreg
 ; CHECK: [[IS1_F7:%.*]] = load i32, i32* @is1, align 4
-; CHECK-CONS: call void @__msan_instrument_asm_load({{.*}}@is1{{.*}}, i64 4)
 ; CHECK-CONS: call void @__msan_instrument_asm_store({{.*}}@id1{{.*}}, i64 4)
 ; CHECK: call void @__msan_warning
 ; CHECK: call i32 asm "", "=r,=*m,r,*m,~{dirflag},~{fpsr},~{flags}"(i32* @id1, i32 [[IS1_F7]], i32* @is1)
@@ -257,9 +260,6 @@ entry:
 }
 
 ; CHECK-LABEL: @f_3i_3o_complex_mem
-; CHECK-CONS: call void @__msan_instrument_asm_load({{.*}}@pair1{{.*}}, i64 8)
-; CHECK-CONS: call void @__msan_instrument_asm_load(i8* @c1, i64 1)
-; CHECK-CONS: call void @__msan_instrument_asm_load({{.*}}@memcpy_s1{{.*}}, i64 8)
 ; CHECK-CONS: call void @__msan_instrument_asm_store({{.*}}@pair2{{.*}}, i64 8)
 ; CHECK-CONS: call void @__msan_instrument_asm_store({{.*}}@c2{{.*}}, i64 1)
 ; CHECK-CONS: call void @__msan_instrument_asm_store({{.*}}@memcpy_d1{{.*}}, i64 8)
diff --git a/test/Instrumentation/MemorySanitizer/msan_basic.ll b/test/Instrumentation/MemorySanitizer/msan_basic.ll
index 78d31dc3434f2442544679f1a70617937d4efba7..569c2320c5c914cb86f2bac6bc48a40bc4842952 100644
--- a/test/Instrumentation/MemorySanitizer/msan_basic.ll
+++ b/test/Instrumentation/MemorySanitizer/msan_basic.ll
@@ -1,4 +1,9 @@
+; RUN: opt < %s -msan-check-access-address=0 -S -passes=msan 2>&1 | FileCheck  \
+; RUN: -allow-deprecated-dag-overlap %s
 ; RUN: opt < %s -msan -msan-check-access-address=0 -S | FileCheck -allow-deprecated-dag-overlap %s
+; RUN: opt < %s -msan-check-access-address=0 -msan-track-origins=1 -S          \
+; RUN: -passes=msan 2>&1 | FileCheck -allow-deprecated-dag-overlap             \
+; RUN: -check-prefix=CHECK -check-prefix=CHECK-ORIGINS %s
 ; RUN: opt < %s -msan -msan-check-access-address=0 -msan-track-origins=1 -S | FileCheck -allow-deprecated-dag-overlap -check-prefix=CHECK -check-prefix=CHECK-ORIGINS %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
diff --git a/test/Instrumentation/MemorySanitizer/msan_kernel_basic.ll b/test/Instrumentation/MemorySanitizer/msan_kernel_basic.ll
index 28bbf3cd7088ab2a16aeee2b9eff7ae502b75012..680a672d12abacb67ddbc303c42226e6b8ed5740 100644
--- a/test/Instrumentation/MemorySanitizer/msan_kernel_basic.ll
+++ b/test/Instrumentation/MemorySanitizer/msan_kernel_basic.ll
@@ -1,4 +1,6 @@
 ; KMSAN instrumentation tests
+; RUN: opt < %s -msan-kernel=1 -S -passes=msan 2>&1 | FileCheck %s             \
+; RUN: -check-prefixes=CHECK
 ; RUN: opt < %s -msan -msan-kernel=1 -S | FileCheck %s -check-prefixes=CHECK
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
diff --git a/test/Instrumentation/MemorySanitizer/msan_llvm_is_constant.ll b/test/Instrumentation/MemorySanitizer/msan_llvm_is_constant.ll
new file mode 100644
index 0000000000000000000000000000000000000000..4f316be23579879285b00d6173eebcea67fd8aaa
--- /dev/null
+++ b/test/Instrumentation/MemorySanitizer/msan_llvm_is_constant.ll
@@ -0,0 +1,24 @@
+; Make sure MSan doesn't insert shadow checks for @llvm.is.constant.* arguments.
+
+; RUN: opt < %s -msan-kernel=1 -S -passes=msan 2>&1 | FileCheck                \
+; RUN: -check-prefixes=CHECK %s
+; RUN: opt < %s -msan -msan-kernel=1 -S | FileCheck -check-prefixes=CHECK %s
+; RUN: opt < %s -S -passes=msan 2>&1 | FileCheck -check-prefixes=CHECK %s
+; RUN: opt < %s -msan -S | FileCheck -check-prefixes=CHECK %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; Function Attrs: nounwind readnone uwtable
+define dso_local i32 @bar(i32 %v) local_unnamed_addr sanitize_memory {
+entry:
+  %0 = tail call i1 @llvm.is.constant.i32(i32 %v)
+  %1 = zext i1 %0 to i32
+  ret i32 %1
+}
+
+; CHECK-LABEL: bar
+; CHECK-NOT: call void @__msan_warning
+
+; Function Attrs: nounwind readnone
+declare i1 @llvm.is.constant.i32(i32)
diff --git a/test/Instrumentation/MemorySanitizer/msan_x86_bts_asm.ll b/test/Instrumentation/MemorySanitizer/msan_x86_bts_asm.ll
index c9e6f1afe21d0be9716bd302cb65380e0ef81ca8..1c0072f5cead3d7af76f2dfafe41c793ba2273e3 100644
--- a/test/Instrumentation/MemorySanitizer/msan_x86_bts_asm.ll
+++ b/test/Instrumentation/MemorySanitizer/msan_x86_bts_asm.ll
@@ -1,5 +1,11 @@
 ; Test for the conservative assembly handling mode used by KMSAN.
+; RUN: opt < %s -msan-kernel=1 -msan-check-access-address=0                    \
+; RUN: -msan-handle-asm-conservative=0 -S -passes=msan 2>&1 | FileCheck        \
+; RUN: "-check-prefixes=CHECK,CHECK-NONCONS" %s
 ; RUN: opt < %s -msan -msan-kernel=1 -msan-check-access-address=0 -msan-handle-asm-conservative=0 -S | FileCheck -check-prefixes=CHECK,CHECK-NONCONS %s
+; RUN: opt < %s -msan-kernel=1 -msan-check-access-address=0                    \
+; RUN: -msan-handle-asm-conservative=1 -S -passes=msan 2>&1 | FileCheck        \
+; RUN: "-check-prefixes=CHECK,CHECK-CONS" %s
 ; RUN: opt < %s -msan -msan-kernel=1 -msan-check-access-address=0 -msan-handle-asm-conservative=1 -S | FileCheck -check-prefixes=CHECK,CHECK-CONS %s
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
diff --git a/test/Instrumentation/MemorySanitizer/msan_x86intrinsics.ll b/test/Instrumentation/MemorySanitizer/msan_x86intrinsics.ll
index c4ec7fa291991a9d0758f10cb6f1dbfd1f13e429..c5bdc2aee5dcc77aedb0171ade357a98467109e8 100644
--- a/test/Instrumentation/MemorySanitizer/msan_x86intrinsics.ll
+++ b/test/Instrumentation/MemorySanitizer/msan_x86intrinsics.ll
@@ -1,4 +1,9 @@
+; RUN: opt < %s -msan-check-access-address=0 -S -passes=msan 2>&1 | FileCheck  \
+; RUN: %s
 ; RUN: opt < %s -msan -msan-check-access-address=0 -S | FileCheck %s
+; RUN: opt < %s -msan-check-access-address=0 -msan-track-origins=1 -S          \
+; RUN: -passes=msan 2>&1 | FileCheck -check-prefix=CHECK                       \
+; RUN: -check-prefix=CHECK-ORIGINS %s
 ; RUN: opt < %s -msan -msan-check-access-address=0 -msan-track-origins=1 -S | FileCheck -check-prefix=CHECK -check-prefix=CHECK-ORIGINS %s
 ; REQUIRES: x86-registered-target
 
@@ -46,14 +51,14 @@ declare <16 x i8> @llvm.x86.sse3.ldu.dq(i8* %p) nounwind
 ; Check that shadow is OR'ed, and origin is Select'ed
 ; And no shadow checks!
 
-define <8 x i16> @Paddsw128(<8 x i16> %a, <8 x i16> %b) nounwind uwtable sanitize_memory {
-  %call = call <8 x i16> @llvm.x86.sse2.padds.w(<8 x i16> %a, <8 x i16> %b)
+define <8 x i16> @Pmulhuw128(<8 x i16> %a, <8 x i16> %b) nounwind uwtable sanitize_memory {
+  %call = call <8 x i16> @llvm.x86.sse2.pmulhu.w(<8 x i16> %a, <8 x i16> %b)
   ret <8 x i16> %call
 }
 
-declare <8 x i16> @llvm.x86.sse2.padds.w(<8 x i16> %a, <8 x i16> %b) nounwind
+declare <8 x i16> @llvm.x86.sse2.pmulhu.w(<8 x i16> %a, <8 x i16> %b) nounwind
 
-; CHECK-LABEL: @Paddsw128
+; CHECK-LABEL: @Pmulhuw128
 ; CHECK-NEXT: load <8 x i16>, <8 x i16>* {{.*}} @__msan_param_tls
 ; CHECK-ORIGINS: load i32, i32* {{.*}} @__msan_param_origin_tls
 ; CHECK-NEXT: load <8 x i16>, <8 x i16>* {{.*}} @__msan_param_tls
@@ -62,7 +67,7 @@ declare <8 x i16> @llvm.x86.sse2.padds.w(<8 x i16> %a, <8 x i16> %b) nounwind
 ; CHECK-ORIGINS: = bitcast <8 x i16> {{.*}} to i128
 ; CHECK-ORIGINS-NEXT: = icmp ne i128 {{.*}}, 0
 ; CHECK-ORIGINS-NEXT: = select i1 {{.*}}, i32 {{.*}}, i32
-; CHECK-NEXT: call <8 x i16> @llvm.x86.sse2.padds.w
+; CHECK-NEXT: call <8 x i16> @llvm.x86.sse2.pmulhu.w
 ; CHECK-NEXT: store <8 x i16> {{.*}} @__msan_retval_tls
 ; CHECK-ORIGINS: store i32 {{.*}} @__msan_retval_origin_tls
 ; CHECK-NEXT: ret <8 x i16>
diff --git a/test/Instrumentation/MemorySanitizer/mul_by_constant.ll b/test/Instrumentation/MemorySanitizer/mul_by_constant.ll
index 7736d94717fe93abc897a9ce5df77febe1c67b11..959f0195dc50f6ec887aa1a95228860d10f05419 100644
--- a/test/Instrumentation/MemorySanitizer/mul_by_constant.ll
+++ b/test/Instrumentation/MemorySanitizer/mul_by_constant.ll
@@ -1,3 +1,5 @@
+; RUN: opt < %s -msan-check-access-address=0 -S -passes=msan 2>&1 | FileCheck  \
+; RUN: %s
 ; RUN: opt < %s -msan -msan-check-access-address=0 -S | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
diff --git a/test/Instrumentation/MemorySanitizer/nosanitize.ll b/test/Instrumentation/MemorySanitizer/nosanitize.ll
index 082a6f580e57a9c2a9d913c2a6ae733b2284e587..d68be085180feb97e99c8f4634229969f4ea7c37 100644
--- a/test/Instrumentation/MemorySanitizer/nosanitize.ll
+++ b/test/Instrumentation/MemorySanitizer/nosanitize.ll
@@ -1,5 +1,7 @@
 ; Verify that calls with !nosanitize are not instrumented by MSan.
+; RUN: opt < %s -S -passes=msan 2>&1 | FileCheck %s
 ; RUN: opt < %s -msan -S | FileCheck %s
+; RUN: opt < %s -msan-track-origins=1 -S -passes=msan 2>&1 | FileCheck %s
 ; RUN: opt < %s -msan -msan-track-origins=1 -S | FileCheck %s
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
diff --git a/test/Instrumentation/MemorySanitizer/origin-alignment.ll b/test/Instrumentation/MemorySanitizer/origin-alignment.ll
index abd8dd0e533fa6b44034dab1cc590902fff09e80..c2afe794c8cd0f7bd3cb9540144b3dc1bdcef88f 100644
--- a/test/Instrumentation/MemorySanitizer/origin-alignment.ll
+++ b/test/Instrumentation/MemorySanitizer/origin-alignment.ll
@@ -1,4 +1,10 @@
+; RUN: opt < %s -msan-check-access-address=0 -msan-track-origins=1 -S          \
+; RUN: -passes=msan 2>&1 | FileCheck -check-prefix=CHECK                       \
+; RUN: -check-prefix=CHECK-ORIGINS1 %s
 ; RUN: opt < %s -msan -msan-check-access-address=0 -msan-track-origins=1 -S | FileCheck -check-prefix=CHECK -check-prefix=CHECK-ORIGINS1 %s
+; RUN: opt < %s -msan-check-access-address=0 -msan-track-origins=2 -S          \
+; RUN: -passes=msan 2>&1 | FileCheck -check-prefix=CHECK                       \
+; RUN: -check-prefix=CHECK-ORIGINS2 %s
 ; RUN: opt < %s -msan -msan-check-access-address=0 -msan-track-origins=2 -S | FileCheck -check-prefix=CHECK -check-prefix=CHECK-ORIGINS2 %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
diff --git a/test/Instrumentation/MemorySanitizer/origin-array.ll b/test/Instrumentation/MemorySanitizer/origin-array.ll
index d9936ff8e78ab6efb3befd9069bf73ecfd515170..0a647872c0f6774fd6d4aab7bc8c5c5c507fe679 100644
--- a/test/Instrumentation/MemorySanitizer/origin-array.ll
+++ b/test/Instrumentation/MemorySanitizer/origin-array.ll
@@ -1,3 +1,5 @@
+; RUN: opt < %s -msan-check-access-address=0 -msan-track-origins=2 -S          \
+; RUN: -passes=msan 2>&1 | FileCheck %s
 ; RUN: opt < %s -msan -msan-check-access-address=0 -msan-track-origins=2 -S | FileCheck %s
 
 target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
diff --git a/test/Instrumentation/MemorySanitizer/pr32842.ll b/test/Instrumentation/MemorySanitizer/pr32842.ll
index 5d74c9a193bf5b547fb8291f167994eee1c775dd..c342f16bd37aab565a65d206b50345a942b58a7e 100644
--- a/test/Instrumentation/MemorySanitizer/pr32842.ll
+++ b/test/Instrumentation/MemorySanitizer/pr32842.ll
@@ -1,5 +1,6 @@
 ; Regression test for https://bugs.llvm.org/show_bug.cgi?id=32842
 ;
+; RUN: opt < %s -S -passes=msan 2>&1 | FileCheck %s
 ; RUN: opt < %s -msan -S | FileCheck %s
 ;target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
diff --git a/test/Instrumentation/MemorySanitizer/return_from_main.ll b/test/Instrumentation/MemorySanitizer/return_from_main.ll
index 82e2d13bc09bb9261879b67ab09d31930d2b8882..0e5d3400de38a8bdadceddddca2a71062662d0f1 100644
--- a/test/Instrumentation/MemorySanitizer/return_from_main.ll
+++ b/test/Instrumentation/MemorySanitizer/return_from_main.ll
@@ -1,3 +1,5 @@
+; RUN: opt < %s -msan-check-access-address=0 -S -passes=msan 2>&1 | FileCheck  \
+; RUN: %s
 ; RUN: opt < %s -msan -msan-check-access-address=0 -S | FileCheck %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
diff --git a/test/Instrumentation/MemorySanitizer/store-long-origin.ll b/test/Instrumentation/MemorySanitizer/store-long-origin.ll
index 23ba4a18e6bc065727fc2ee1ccb1660ac43bf007..f8973fc814ac10faf7dab796ed72e7b12f1bf473 100644
--- a/test/Instrumentation/MemorySanitizer/store-long-origin.ll
+++ b/test/Instrumentation/MemorySanitizer/store-long-origin.ll
@@ -1,3 +1,5 @@
+; RUN: opt < %s -msan-check-access-address=0 -msan-track-origins=1 -S          \
+; RUN: -passes=msan 2>&1 | FileCheck %s
 ; RUN: opt < %s -msan -msan-check-access-address=0 -msan-track-origins=1 -S | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
diff --git a/test/Instrumentation/MemorySanitizer/store-origin.ll b/test/Instrumentation/MemorySanitizer/store-origin.ll
index 70722c63f2959c140facea53c862f51ad08cd544..9a0c01936e7c57edb7076a6956739de03f7633b5 100644
--- a/test/Instrumentation/MemorySanitizer/store-origin.ll
+++ b/test/Instrumentation/MemorySanitizer/store-origin.ll
@@ -1,5 +1,13 @@
+; RUN: opt < %s -msan-check-access-address=0 -msan-track-origins=1 -S          \
+; RUN: -passes=msan 2>&1 | FileCheck                                           \
+; RUN: "-check-prefixes=CHECK,CHECK-MSAN,CHECK-ORIGINS1" %s
 ; RUN: opt < %s -msan -msan-check-access-address=0 -msan-track-origins=1 -S | FileCheck -check-prefixes=CHECK,CHECK-MSAN,CHECK-ORIGINS1 %s
+; RUN: opt < %s -msan-check-access-address=0 -msan-track-origins=2 -S          \
+; RUN: -passes=msan 2>&1 | FileCheck                                           \
+; RUN: "-check-prefixes=CHECK,CHECK-MSAN,CHECK-ORIGINS2" %s
 ; RUN: opt < %s -msan -msan-check-access-address=0 -msan-track-origins=2 -S | FileCheck -check-prefixes=CHECK,CHECK-MSAN,CHECK-ORIGINS2 %s
+; RUN: opt < %s -msan-kernel=1 -msan-check-access-address=0 -S -passes=msan    \
+; RUN: 2>&1 | FileCheck "-check-prefixes=CHECK,CHECK-KMSAN,CHECK-ORIGINS2" %s
 ; RUN: opt < %s -msan -msan-kernel=1 -msan-check-access-address=0 -S | FileCheck -check-prefixes=CHECK,CHECK-KMSAN,CHECK-ORIGINS2 %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
diff --git a/test/Instrumentation/MemorySanitizer/str-nobuiltin.ll b/test/Instrumentation/MemorySanitizer/str-nobuiltin.ll
index 0aa196782cadc3e1f7924124d8a8205b41ec34bf..96223cd08183d0d98e75069ed17e059284aced49 100644
--- a/test/Instrumentation/MemorySanitizer/str-nobuiltin.ll
+++ b/test/Instrumentation/MemorySanitizer/str-nobuiltin.ll
@@ -1,5 +1,6 @@
 ; Test marking string functions as nobuiltin in memory sanitizer.
 ;
+; RUN: opt < %s -S -passes=msan 2>&1 | FileCheck %s
 ; RUN: opt < %s -msan -S | FileCheck %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 target triple = "x86_64-unknown-linux-gnu"
diff --git a/test/Instrumentation/MemorySanitizer/unreachable.ll b/test/Instrumentation/MemorySanitizer/unreachable.ll
index ac5aea9a077146ef3e9934f3b4d226e0174a1ebf..eff7371e1231d38b1a76224cbf79cbe3b08277b3 100644
--- a/test/Instrumentation/MemorySanitizer/unreachable.ll
+++ b/test/Instrumentation/MemorySanitizer/unreachable.ll
@@ -1,3 +1,4 @@
+; RUN: opt < %s -S -passes=msan 2>&1 | FileCheck %s
 ; RUN: opt < %s -msan -S | FileCheck %s
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
diff --git a/test/Instrumentation/MemorySanitizer/unsized_type.ll b/test/Instrumentation/MemorySanitizer/unsized_type.ll
index 94ae92d3354a48412c20905046c237bd38579457..4bffe808d3a231522eef62b0e3674dcc9d355c7a 100644
--- a/test/Instrumentation/MemorySanitizer/unsized_type.ll
+++ b/test/Instrumentation/MemorySanitizer/unsized_type.ll
@@ -1,5 +1,6 @@
 ; Check that unsized token types used by coroutine intrinsics do not cause
 ; assertion failures.
+; RUN: opt < %s -S 2>&1 -passes=msan | FileCheck %s
 ; RUN: opt < %s -msan -S 2>&1 | FileCheck %s
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
diff --git a/test/Instrumentation/MemorySanitizer/vector_arith.ll b/test/Instrumentation/MemorySanitizer/vector_arith.ll
index 6652fdff89b045f1ad36e65e4de5d7d115cff100..4b213d17ee4220a247d8cbfa4c735017052166a2 100644
--- a/test/Instrumentation/MemorySanitizer/vector_arith.ll
+++ b/test/Instrumentation/MemorySanitizer/vector_arith.ll
@@ -1,3 +1,5 @@
+; RUN: opt < %s -msan-check-access-address=0 -S -passes=msan 2>&1 | FileCheck  \
+; RUN: %s
 ; RUN: opt < %s -msan -msan-check-access-address=0 -S | FileCheck %s
 ; REQUIRES: x86-registered-target
 
diff --git a/test/Instrumentation/MemorySanitizer/vector_cmp.ll b/test/Instrumentation/MemorySanitizer/vector_cmp.ll
index 910b1351330a344fa0a6d19fdde50ae8d4e2be33..6031dddccc4c7dd81abdaf667a4840a04c71c060 100644
--- a/test/Instrumentation/MemorySanitizer/vector_cmp.ll
+++ b/test/Instrumentation/MemorySanitizer/vector_cmp.ll
@@ -1,3 +1,5 @@
+; RUN: opt < %s -msan-check-access-address=0 -S -passes=msan 2>&1 | FileCheck  \
+; RUN: %s
 ; RUN: opt < %s -msan -msan-check-access-address=0 -S | FileCheck %s
 ; REQUIRES: x86-registered-target
 
diff --git a/test/Instrumentation/MemorySanitizer/vector_cvt.ll b/test/Instrumentation/MemorySanitizer/vector_cvt.ll
index b70ef7d6254e3a50e9303eea05b338f97bc3b531..abbb0a8c98142dadd9146761364869edef17b2d0 100644
--- a/test/Instrumentation/MemorySanitizer/vector_cvt.ll
+++ b/test/Instrumentation/MemorySanitizer/vector_cvt.ll
@@ -1,3 +1,5 @@
+; RUN: opt < %s -msan-check-access-address=0 -S -passes=msan 2>&1 | FileCheck  \
+; RUN: %s
 ; RUN: opt < %s -msan -msan-check-access-address=0 -S | FileCheck %s
 ; REQUIRES: x86-registered-target
 
diff --git a/test/Instrumentation/MemorySanitizer/vector_pack.ll b/test/Instrumentation/MemorySanitizer/vector_pack.ll
index 574e7b890034b0f5828dde0378e5718979716a32..50d9303f4e9a2697236af5b3d488f3875d52d234 100644
--- a/test/Instrumentation/MemorySanitizer/vector_pack.ll
+++ b/test/Instrumentation/MemorySanitizer/vector_pack.ll
@@ -1,3 +1,5 @@
+; RUN: opt < %s -msan-check-access-address=0 -S -passes=msan 2>&1 | FileCheck  \
+; RUN: %s
 ; RUN: opt < %s -msan -msan-check-access-address=0 -S | FileCheck %s
 ; REQUIRES: x86-registered-target
 
diff --git a/test/Instrumentation/MemorySanitizer/vector_shift.ll b/test/Instrumentation/MemorySanitizer/vector_shift.ll
index c605c97bba174799296c8b3d1fa064e901974299..9f74869c656adacd5e0be8449c7c71b4c16fbf55 100644
--- a/test/Instrumentation/MemorySanitizer/vector_shift.ll
+++ b/test/Instrumentation/MemorySanitizer/vector_shift.ll
@@ -1,3 +1,5 @@
+; RUN: opt < %s -msan-check-access-address=0 -S -passes=msan 2>&1 | FileCheck  \
+; RUN: %s
 ; RUN: opt < %s -msan -msan-check-access-address=0 -S | FileCheck %s
 ; REQUIRES: x86-registered-target
 
diff --git a/test/Instrumentation/MemorySanitizer/with-call-type-size.ll b/test/Instrumentation/MemorySanitizer/with-call-type-size.ll
index 2a3cbf70ea58b74a554e8ef083279bef2bbbb7eb..da5e75b9d2bfe4994b3318599484d61a4e77c54b 100644
--- a/test/Instrumentation/MemorySanitizer/with-call-type-size.ll
+++ b/test/Instrumentation/MemorySanitizer/with-call-type-size.ll
@@ -1,3 +1,5 @@
+; RUN: opt < %s -msan-instrumentation-with-call-threshold=0 -S -passes=msan    \
+; RUN: 2>&1 | FileCheck %s
 ; RUN: opt < %s -msan -msan-instrumentation-with-call-threshold=0 -S | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
@@ -82,4 +84,4 @@ define <4 x i32> @test65(<4 x i32> %vec, i65 %idx, i32 %x) sanitize_memory {
 ; CHECK-NOT:     call void @__msan_maybe_warning_
 ; CHECK:         icmp ne i65 %{{.*}}, 0
 ; CHECK-NOT:     call void @__msan_maybe_warning_
-; CHECK:         ret <4 x i32>
\ No newline at end of file
+; CHECK:         ret <4 x i32>
diff --git a/test/Instrumentation/SanitizerCoverage/backedge-pruning.ll b/test/Instrumentation/SanitizerCoverage/backedge-pruning.ll
new file mode 100644
index 0000000000000000000000000000000000000000..103198311279b79e10cf25879715cb27b3836c28
--- /dev/null
+++ b/test/Instrumentation/SanitizerCoverage/backedge-pruning.ll
@@ -0,0 +1,32 @@
+; Test -sanitizer-coverage-trace-compares=1 and how it prunes backedge compares.
+; RUN: opt < %s -sancov -sanitizer-coverage-level=1 -sanitizer-coverage-trace-compares=1  -sanitizer-coverage-prune-blocks=1 -S | FileCheck %s --check-prefix=PRUNE
+; RUN: opt < %s -sancov -sanitizer-coverage-level=1 -sanitizer-coverage-trace-compares=1  -sanitizer-coverage-prune-blocks=0 -S | FileCheck %s --check-prefix=NOPRUNE
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
+target triple = "x86_64-unknown-linux-gnu"
+
+define dso_local void @foo(i32* nocapture readnone %a, i32 %n) local_unnamed_addr {
+entry:
+  br label %do.body
+
+do.body:
+  %i.0 = phi i32 [ 0, %entry ], [ %inc, %do.body ]
+  tail call void (...) @bar()
+  %inc = add nuw nsw i32 %i.0, 1
+  %cmp = icmp slt i32 %inc, %n
+;PRUNE-LABEL: foo
+;PRUNE-NOT: __sanitizer_cov_trace_cmp4
+;PRUNE: ret void
+
+;NOPRUNE-LABEL: foo
+;NOPRUNE: call void @__sanitizer_cov_trace_cmp4
+;NOPRUNE-NEXT: icmp
+;NOPRUNE: ret void
+
+  br i1 %cmp, label %do.body, label %do.end
+
+do.end:
+  ret void
+}
+
+declare dso_local void @bar(...) local_unnamed_addr
diff --git a/test/Instrumentation/SanitizerCoverage/coff-used-ctor.ll b/test/Instrumentation/SanitizerCoverage/coff-used-ctor.ll
new file mode 100644
index 0000000000000000000000000000000000000000..fd12eed8e3669e6eaa6467e2ef8ee09f61ebf7dd
--- /dev/null
+++ b/test/Instrumentation/SanitizerCoverage/coff-used-ctor.ll
@@ -0,0 +1,11 @@
+; Checks that sancov.module_ctor is marked used.
+; RUN: opt < %s -sancov -sanitizer-coverage-level=1 -sanitizer-coverage-inline-8bit-counters=1 -sanitizer-coverage-pc-table=1 -S | FileCheck %s
+target datalayout = "e-m:w-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-pc-windows-msvc19.14.26433"
+
+define void @foo() {
+entry:
+  ret void
+}
+
+; CHECK: @llvm.used = appending global {{.*}} @sancov.module_ctor
\ No newline at end of file
diff --git a/test/Instrumentation/SanitizerCoverage/interposable-symbol-nocomdat.ll b/test/Instrumentation/SanitizerCoverage/interposable-symbol-nocomdat.ll
new file mode 100644
index 0000000000000000000000000000000000000000..c79a2fb5fff0d5708f41aa6b3d310e555cb5fc7c
--- /dev/null
+++ b/test/Instrumentation/SanitizerCoverage/interposable-symbol-nocomdat.ll
@@ -0,0 +1,37 @@
+; Test that interposable symbols do not get put in comdats.
+; RUN: opt < %s -sancov -sanitizer-coverage-level=3 -sanitizer-coverage-trace-pc-guard -mtriple x86_64-linux-gnu -S | FileCheck %s
+; RUN: opt < %s -sancov -sanitizer-coverage-level=3 -sanitizer-coverage-trace-pc-guard -mtriple x86_64-windows-msvc -S | FileCheck %s
+
+define void @Vanilla() {
+entry:
+  ret void
+}
+
+define linkonce void @LinkOnce() {
+entry:
+  ret void
+}
+
+define weak void @Weak() {
+entry:
+  ret void
+}
+
+declare extern_weak void @ExternWeak()
+
+define linkonce_odr void @LinkOnceOdr() {
+entry:
+  ret void
+}
+
+define weak_odr void @WeakOdr() {
+entry:
+  ret void
+}
+
+; CHECK: define void @Vanilla() comdat {
+; CHECK: define linkonce void @LinkOnce() {
+; CHECK: define weak void @Weak() {
+; CHECK: declare extern_weak void @ExternWeak()
+; CHECK: define linkonce_odr void @LinkOnceOdr() comdat {
+; CHECK: define weak_odr void @WeakOdr() comdat {
diff --git a/test/Instrumentation/SanitizerCoverage/seh.ll b/test/Instrumentation/SanitizerCoverage/seh.ll
index f432573af64a6692696cddab3862706aaa5c4f40..94d1a2e9acdfb904be36136d4993a54c555783be 100644
--- a/test/Instrumentation/SanitizerCoverage/seh.ll
+++ b/test/Instrumentation/SanitizerCoverage/seh.ll
@@ -7,7 +7,7 @@ target triple = "i686-pc-windows-msvc18.0.0"
 
 declare i32 @llvm.eh.typeid.for(i8*) #2
 declare i8* @llvm.frameaddress(i32)
-declare i8* @llvm.x86.seh.recoverfp(i8*, i8*)
+declare i8* @llvm.eh.recoverfp(i8*, i8*)
 declare i8* @llvm.localrecover(i8*, i8*, i32)
 declare void @llvm.localescape(...) #1
 
@@ -55,7 +55,7 @@ eh.resume:                                        ; preds = %lpad
 define internal i32 @"\01?filt$0@0@main@@"() #1 {
 entry:
   %0 = tail call i8* @llvm.frameaddress(i32 1)
-  %1 = tail call i8* @llvm.x86.seh.recoverfp(i8* bitcast (i32 ()* @main to i8*), i8* %0)
+  %1 = tail call i8* @llvm.eh.recoverfp(i8* bitcast (i32 ()* @main to i8*), i8* %0)
   %2 = tail call i8* @llvm.localrecover(i8* bitcast (i32 ()* @main to i8*), i8* %1, i32 0)
   %__exception_code = bitcast i8* %2 to i32*
   %3 = getelementptr inbounds i8, i8* %0, i32 -20
diff --git a/test/Instrumentation/SanitizerCoverage/stack-depth-variable-declared-by-user.ll b/test/Instrumentation/SanitizerCoverage/stack-depth-variable-declared-by-user.ll
new file mode 100644
index 0000000000000000000000000000000000000000..1ad96f82a694a2622d1ad0ea78ce72893465962c
--- /dev/null
+++ b/test/Instrumentation/SanitizerCoverage/stack-depth-variable-declared-by-user.ll
@@ -0,0 +1,12 @@
+; Ensure that we terminate with a useful error message (instead of crash) if the
+; user declares `__sancov_lowest_stack` with an unexpected type.
+; RUN: not opt < %s -sancov -sanitizer-coverage-level=1 \
+; RUN:         -sanitizer-coverage-stack-depth -S 2>&1 | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; Wrong type: i32 instead of expected i64
+@__sancov_lowest_stack = thread_local global i32 0
+
+; CHECK: error: '__sancov_lowest_stack' should not be declared by the user
diff --git a/test/Instrumentation/ThreadSanitizer/tsan_basic.ll b/test/Instrumentation/ThreadSanitizer/tsan_basic.ll
index 69d4117399b754e281a1f4f9c54c3636f21db02e..8b85d7b8bddc7552ad714398702fba6af554d2cf 100644
--- a/test/Instrumentation/ThreadSanitizer/tsan_basic.ll
+++ b/test/Instrumentation/ThreadSanitizer/tsan_basic.ll
@@ -1,4 +1,5 @@
 ; RUN: opt < %s -tsan -S | FileCheck %s
+; RUN: opt < %s -passes=tsan -S | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
 target triple = "x86_64-unknown-linux-gnu"
@@ -77,6 +78,5 @@ define void @SwiftErrorCall(i8** swifterror) sanitize_thread {
   call void @SwiftError(i8** %0)
   ret void
 }
-
 ; CHECK: define internal void @tsan.module_ctor()
 ; CHECK: call void @__tsan_init()
diff --git a/test/LTO/Resolution/X86/export-jumptable.ll b/test/LTO/Resolution/X86/export-jumptable.ll
index d6cce854c1b066a568ebe88f1dd1831b58c47e46..8ced9d962923525d32abd7f0a2abfada286161e3 100644
--- a/test/LTO/Resolution/X86/export-jumptable.ll
+++ b/test/LTO/Resolution/X86/export-jumptable.ll
@@ -2,7 +2,7 @@
 ; the full LTO object file; any such functions will be referenced by the jump
 ; table.
 
-; RUN: opt -thinlto-bc -o %t %s
+; RUN: opt -thinlto-bc -thinlto-split-lto-unit -o %t %s
 ; RUN: llvm-lto2 run -o %t2 -r %t,f1,p -r %t,f2,p -r %t,_start,px %t -save-temps
 ; RUN: llvm-dis %t2.1.2.internalize.bc -o - | FileCheck %s
 
diff --git a/test/LTO/Resolution/X86/local-def-dllimport.ll b/test/LTO/Resolution/X86/local-def-dllimport.ll
new file mode 100644
index 0000000000000000000000000000000000000000..c97e4b7e14c1479e33a0cb99c47360104cade80a
--- /dev/null
+++ b/test/LTO/Resolution/X86/local-def-dllimport.ll
@@ -0,0 +1,32 @@
+; RUN: opt -thinlto-bc -thinlto-split-lto-unit -o %t0.bc %s
+; RUN: llvm-lto2 run -r %t0.bc,__imp_f,l \
+; RUN:               -r %t0.bc,g,p \
+; RUN:               -r %t0.bc,g,l \
+; RUN:               -r %t0.bc,e,l \
+; RUN:               -r %t0.bc,main,x \
+; RUN:               -save-temps -o %t1 %t0.bc
+; RUN: llvm-dis %t1.1.3.import.bc -o - | FileCheck %s
+source_filename = "test.cpp"
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+$g = comdat any
+@g = global i8 42, comdat, !type !0
+
+; CHECK: define
+; CHECK-NOT: dllimport
+; CHECK-SAME: @f
+define available_externally dllimport i8* @f() {
+  ret i8* @g
+}
+
+define i8* @e() {
+  ret i8* @g
+}
+
+define i32 @main() {
+  %1 = call i8* @f()
+  %2 = ptrtoint i8* %1 to i32
+  ret i32 %2
+}
+!0 = !{i32 0, !"typeid"}
diff --git a/test/LTO/Resolution/X86/lowertypetests.ll b/test/LTO/Resolution/X86/lowertypetests.ll
index b87452c8e7aaf4c3c96d2946fb0f0b5b3a808608..e5be4b98cbf4edd56211ef9177035ae83b67b0a5 100644
--- a/test/LTO/Resolution/X86/lowertypetests.ll
+++ b/test/LTO/Resolution/X86/lowertypetests.ll
@@ -1,4 +1,4 @@
-; RUN: opt -thinlto-bc -o %t %s
+; RUN: opt -thinlto-bc -thinlto-split-lto-unit -o %t %s
 ; RUN: llvm-lto2 run -r %t,f,plx -r %t,g_alias,plx -r %t,foo,lx -r %t,foo,plx -r %t,bar,lx -r %t,bar,plx -o %t1 %t
 ; RUN: llvm-nm %t1.0 | FileCheck --check-prefix=MERGED %s
 ; RUN: llvm-nm %t1.1 | FileCheck %s
diff --git a/test/LTO/Resolution/X86/lto-unit-check.ll b/test/LTO/Resolution/X86/lto-unit-check.ll
new file mode 100644
index 0000000000000000000000000000000000000000..1736a5b216aeddb9c4c72f13eb7653b1fd7d8445
--- /dev/null
+++ b/test/LTO/Resolution/X86/lto-unit-check.ll
@@ -0,0 +1,55 @@
+; Test to ensure that the Enable Split LTO Unit flag is set properly in the
+; summary, and that we correctly silently handle linking bitcode files with
+; different values of this flag.
+
+; Linking bitcode both with EnableSplitLTOUnit set should work
+; RUN: opt -thinlto-bc -thinlto-split-lto-unit -o %t1 %s
+; RUN: llvm-bcanalyzer -dump %t1 | FileCheck %s --check-prefix=SPLITLTOUNIT
+; RUN: llvm-dis -o - %t1 | FileCheck %s --check-prefix=ENABLESPLITFLAG
+; RUN: opt -thinlto-bc -thinlto-split-lto-unit -o %t2 %s
+; RUN: llvm-bcanalyzer -dump %t2 | FileCheck %s --check-prefix=SPLITLTOUNIT
+; RUN: llvm-dis -o - %t2 | FileCheck %s --check-prefix=ENABLESPLITFLAG
+; RUN: llvm-lto2 run -o %t3 %t1 %t2
+
+; Linking bitcode both without EnableSplitLTOUnit set should work
+; RUN: opt -thinlto-bc -thinlto-split-lto-unit=false -o %t1 %s
+; RUN: llvm-bcanalyzer -dump %t1 | FileCheck %s --check-prefix=NOSPLITLTOUNIT
+; RUN: llvm-dis -o - %t1 | FileCheck %s --check-prefix=NOENABLESPLITFLAG
+; RUN: opt -thinlto-bc -thinlto-split-lto-unit=false -o %t2 %s
+; RUN: llvm-bcanalyzer -dump %t2 | FileCheck %s --check-prefix=NOSPLITLTOUNIT
+; RUN: llvm-dis -o - %t2 | FileCheck %s --check-prefix=NOENABLESPLITFLAG
+; RUN: llvm-lto2 run -o %t3 %t1 %t2
+
+; Linking bitcode with different values of EnableSplitLTOUnit should succeed
+; (silently skipping any optimizations like whole program devirt that rely
+; on all modules being split).
+; RUN: opt -thinlto-bc -thinlto-split-lto-unit -o %t1 %s
+; RUN: llvm-bcanalyzer -dump %t1 | FileCheck %s --check-prefix=SPLITLTOUNIT
+; RUN: llvm-dis -o - %t1 | FileCheck %s --check-prefix=ENABLESPLITFLAG
+; RUN: opt -thinlto-bc -thinlto-split-lto-unit=false -o %t2 %s
+; RUN: llvm-bcanalyzer -dump %t2 | FileCheck %s --check-prefix=NOSPLITLTOUNIT
+; RUN: llvm-dis -o - %t2 | FileCheck %s --check-prefix=NOENABLESPLITFLAG
+; RUN: llvm-lto2 run -o %t3 %t1 %t2
+
+; Linking bitcode with different values of EnableSplitLTOUnit (reverse order)
+; should succeed (silently skipping any optimizations like whole program devirt
+; that rely on all modules being split).
+; RUN: opt -thinlto-bc -thinlto-split-lto-unit=false -o %t1 %s
+; RUN: llvm-bcanalyzer -dump %t1 | FileCheck %s --check-prefix=NOSPLITLTOUNIT
+; RUN: llvm-dis -o - %t1 | FileCheck %s --check-prefix=NOENABLESPLITFLAG
+; RUN: opt -thinlto-bc -thinlto-split-lto-unit -o %t2 %s
+; RUN: llvm-bcanalyzer -dump %t2 | FileCheck %s --check-prefix=SPLITLTOUNIT
+; RUN: llvm-dis -o - %t2 | FileCheck %s --check-prefix=ENABLESPLITFLAG
+; RUN: llvm-lto2 run -o %t3 %t1 %t2
+
+; The flag should be set when splitting is disabled (for backwards compatibility
+; with older bitcode where it was always enabled).
+; SPLITLTOUNIT: <FLAGS op0=8/>
+; NOSPLITLTOUNIT: <FLAGS op0=0/>
+
+; Check that the corresponding module flag is set when expected.
+; ENABLESPLITFLAG: !{i32 1, !"EnableSplitLTOUnit", i32 1}
+; NOENABLESPLITFLAG-NOT: !{i32 1, !"EnableSplitLTOUnit", i32 1}
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
diff --git a/test/Linker/Inputs/metadata-source-a.ll b/test/Linker/Inputs/metadata-source-a.ll
index e828770e048a6d74ac0b56276386ffa9a195fce0..21a2a840836af5b1264c4c1d233cdafe5a72dce8 100644
--- a/test/Linker/Inputs/metadata-source-a.ll
+++ b/test/Linker/Inputs/metadata-source-a.ll
@@ -11,7 +11,7 @@ target triple = "x86_64-unknown-linux-gnu"
 
 !0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression())
 !1 = distinct !DIGlobalVariable(name: "a", scope: !2, file: !3, line: 1, type: !6, isLocal: false, isDefinition: true)
-!2 = distinct !DICompileUnit(language: DW_LANG_C99, file: !3, producer: "clang version 7.0.0 (https://git.llvm.org/git/clang.git/ c12b573f9ac61655cce52628b34235f58edaf984) (https://scott.linder@llvm.org/git/llvm.git 90c4822e8541eb07891cd03e614c530c30f8aa12)", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, globals: !5)
+!2 = distinct !DICompileUnit(language: DW_LANG_C99, file: !3, producer: "clang version 7.0.0", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, globals: !5)
 !3 = !DIFile(filename: "a.c", directory: "/home/slinder1/test/link", source: "int a;\0A")
 !4 = !{}
 !5 = !{!0}
@@ -19,4 +19,4 @@ target triple = "x86_64-unknown-linux-gnu"
 !7 = !{i32 2, !"Dwarf Version", i32 4}
 !8 = !{i32 2, !"Debug Info Version", i32 3}
 !9 = !{i32 1, !"wchar_size", i32 4}
-!10 = !{!"clang version 7.0.0 (https://git.llvm.org/git/clang.git/ c12b573f9ac61655cce52628b34235f58edaf984) (https://scott.linder@llvm.org/git/llvm.git 90c4822e8541eb07891cd03e614c530c30f8aa12)"}
+!10 = !{!"clang version 7.0.0"}
diff --git a/test/Linker/Inputs/metadata-source-b.ll b/test/Linker/Inputs/metadata-source-b.ll
index 2bda92e71b26b77f740e1047a41e002b6f885cbb..074704b11fec09c638f07d04fefdff2aca4bf2bd 100644
--- a/test/Linker/Inputs/metadata-source-b.ll
+++ b/test/Linker/Inputs/metadata-source-b.ll
@@ -11,7 +11,7 @@ target triple = "x86_64-unknown-linux-gnu"
 
 !0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression())
 !1 = distinct !DIGlobalVariable(name: "b", scope: !2, file: !3, line: 1, type: !6, isLocal: false, isDefinition: true)
-!2 = distinct !DICompileUnit(language: DW_LANG_C99, file: !3, producer: "clang version 7.0.0 (https://git.llvm.org/git/clang.git/ c12b573f9ac61655cce52628b34235f58edaf984) (https://scott.linder@llvm.org/git/llvm.git 90c4822e8541eb07891cd03e614c530c30f8aa12)", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, globals: !5)
+!2 = distinct !DICompileUnit(language: DW_LANG_C99, file: !3, producer: "clang version 7.0.0", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, globals: !5)
 !3 = !DIFile(filename: "b.c", directory: "/home/slinder1/test/link", source: "int b;\0A")
 !4 = !{}
 !5 = !{!0}
@@ -19,4 +19,4 @@ target triple = "x86_64-unknown-linux-gnu"
 !7 = !{i32 2, !"Dwarf Version", i32 4}
 !8 = !{i32 2, !"Debug Info Version", i32 3}
 !9 = !{i32 1, !"wchar_size", i32 4}
-!10 = !{!"clang version 7.0.0 (https://git.llvm.org/git/clang.git/ c12b573f9ac61655cce52628b34235f58edaf984) (https://scott.linder@llvm.org/git/llvm.git 90c4822e8541eb07891cd03e614c530c30f8aa12)"}
+!10 = !{!"clang version 7.0.0"}
diff --git a/test/Linker/Inputs/type-unique-inheritance-a.ll b/test/Linker/Inputs/type-unique-inheritance-a.ll
index 094ea6454c13fe448d4d1d812907fc237b29bed4..4c8502623cbdd3fa9c3f4d67df313ee6e1268101 100644
--- a/test/Linker/Inputs/type-unique-inheritance-a.ll
+++ b/test/Linker/Inputs/type-unique-inheritance-a.ll
@@ -66,7 +66,7 @@ attributes #1 = { nounwind readnone }
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!19, !25}
 
-!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.4 (http://llvm.org/git/clang.git f54e02f969d02d640103db73efc30c45439fceab) (http://llvm.org/git/llvm.git 284353b55896cb1babfaa7add7c0a363245342d2)", isOptimized: false, emissionKind: FullDebug, file: !1, enums: !2, retainedTypes: !3, globals: !2, imports: !2)
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.4", isOptimized: false, emissionKind: FullDebug, file: !1, enums: !2, retainedTypes: !3, globals: !2, imports: !2)
 !1 = !DIFile(filename: "foo.cpp", directory: "/Users/mren/c_testing/type_unique_air/inher")
 !2 = !{}
 !3 = !{!4, !8}
diff --git a/test/Linker/Inputs/type-unique-inheritance-b.ll b/test/Linker/Inputs/type-unique-inheritance-b.ll
index 15f6cb29a3dda46022e9b7a04969b5b9ac4feb46..1684a2ebcbc72d6599ca781bb076f9dc9bf684d3 100644
--- a/test/Linker/Inputs/type-unique-inheritance-b.ll
+++ b/test/Linker/Inputs/type-unique-inheritance-b.ll
@@ -40,7 +40,7 @@ attributes #3 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "n
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!27, !38}
 
-!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.4 (http://llvm.org/git/clang.git f54e02f969d02d640103db73efc30c45439fceab) (http://llvm.org/git/llvm.git 284353b55896cb1babfaa7add7c0a363245342d2)", isOptimized: false, emissionKind: FullDebug, file: !1, enums: !2, retainedTypes: !3, globals: !2, imports: !2)
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.4", isOptimized: false, emissionKind: FullDebug, file: !1, enums: !2, retainedTypes: !3, globals: !2, imports: !2)
 !1 = !DIFile(filename: "bar.cpp", directory: "/Users/mren/c_testing/type_unique_air/inher")
 !2 = !{}
 !3 = !{!4, !11, !15}
diff --git a/test/Linker/Inputs/type-unique-simple2-a.ll b/test/Linker/Inputs/type-unique-simple2-a.ll
index ae0f931bebd3184371fd2de67f568b5c64f7b19d..35f6ea50a8f73c08f8e81960c1d22f53de10901a 100644
--- a/test/Linker/Inputs/type-unique-simple2-a.ll
+++ b/test/Linker/Inputs/type-unique-simple2-a.ll
@@ -63,7 +63,7 @@ attributes #1 = { nounwind readnone }
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!16, !22}
 
-!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.4 (http://llvm.org/git/clang.git 8a3f9e46cb988d2c664395b21910091e3730ae82) (http://llvm.org/git/llvm.git 4699e9549358bc77824a59114548eecc3f7c523c)", isOptimized: false, emissionKind: FullDebug, file: !1, enums: !2, retainedTypes: !3, globals: !2, imports: !2)
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.4", isOptimized: false, emissionKind: FullDebug, file: !1, enums: !2, retainedTypes: !3, globals: !2, imports: !2)
 !1 = !DIFile(filename: "foo.cpp", directory: ".")
 !2 = !{}
 !3 = !{!4}
diff --git a/test/Linker/Inputs/type-unique-simple2-b.ll b/test/Linker/Inputs/type-unique-simple2-b.ll
index 04e139d107e84cc08d989ef832301526a60b45fb..e81466efc6ed60f9d72d175011c8d68a369df923 100644
--- a/test/Linker/Inputs/type-unique-simple2-b.ll
+++ b/test/Linker/Inputs/type-unique-simple2-b.ll
@@ -36,7 +36,7 @@ attributes #3 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "n
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!19, !28}
 
-!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.4 (http://llvm.org/git/clang.git 8a3f9e46cb988d2c664395b21910091e3730ae82) (http://llvm.org/git/llvm.git 4699e9549358bc77824a59114548eecc3f7c523c)", isOptimized: false, emissionKind: FullDebug, file: !1, enums: !2, retainedTypes: !3, globals: !2, imports: !2)
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.4", isOptimized: false, emissionKind: FullDebug, file: !1, enums: !2, retainedTypes: !3, globals: !2, imports: !2)
 !1 = !DIFile(filename: "bar.cpp", directory: ".")
 !2 = !{}
 !3 = !{!4}
diff --git a/test/Linker/type-unique-simple-a.ll b/test/Linker/type-unique-simple-a.ll
index c53d4fe4cc805c7a99365a23da7c93256f136f30..c6e6c2c3a4832d7513486cc8a1bc7e06593db8a4 100644
--- a/test/Linker/type-unique-simple-a.ll
+++ b/test/Linker/type-unique-simple-a.ll
@@ -68,7 +68,7 @@ attributes #1 = { nounwind readnone }
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!14, !20}
 
-!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.4 (http://llvm.org/git/clang.git c23b1db6268c8e7ce64026d57d1510c1aac200a0) (http://llvm.org/git/llvm.git 09b98fe3978eddefc2145adc1056cf21580ce945)", isOptimized: false, emissionKind: FullDebug, file: !1, enums: !2, retainedTypes: !3, globals: !2, imports: !2)
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.4", isOptimized: false, emissionKind: FullDebug, file: !1, enums: !2, retainedTypes: !3, globals: !2, imports: !2)
 !1 = !DIFile(filename: "foo.cpp", directory: "/Users/mren/c_testing/type_unique_air/simple")
 !2 = !{}
 !3 = !{!4}
diff --git a/test/Linker/type-unique-simple-b.ll b/test/Linker/type-unique-simple-b.ll
index 8f9f7e153ef53924b088b21cfd85ff51181e14b0..85dcfcaad7f944aae5abc1929f6c86f3a32abde1 100644
--- a/test/Linker/type-unique-simple-b.ll
+++ b/test/Linker/type-unique-simple-b.ll
@@ -38,7 +38,7 @@ attributes #3 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "n
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!17, !26}
 
-!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.4 (http://llvm.org/git/clang.git c23b1db6268c8e7ce64026d57d1510c1aac200a0) (http://llvm.org/git/llvm.git 09b98fe3978eddefc2145adc1056cf21580ce945)", isOptimized: false, emissionKind: FullDebug, file: !1, enums: !2, retainedTypes: !3, globals: !2, imports: !2)
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.4", isOptimized: false, emissionKind: FullDebug, file: !1, enums: !2, retainedTypes: !3, globals: !2, imports: !2)
 !1 = !DIFile(filename: "bar.cpp", directory: "/Users/mren/c_testing/type_unique_air/simple")
 !2 = !{}
 !3 = !{!4}
diff --git a/test/MC/AArch64/SVE/directive-arch.s b/test/MC/AArch64/SVE/directive-arch.s
new file mode 100644
index 0000000000000000000000000000000000000000..99ddf4e7bf9f6426d3589ee7f7384ab0834d95cc
--- /dev/null
+++ b/test/MC/AArch64/SVE/directive-arch.s
@@ -0,0 +1,6 @@
+// RUN: llvm-mc -triple=aarch64 < %s | FileCheck %s
+
+.arch armv8-a+sve
+
+ptrue   p0.b, pow2
+// CHECK: ptrue   p0.b, pow2
diff --git a/test/MC/AArch64/armv8.5a-predctrl-error.s b/test/MC/AArch64/armv8.5a-predres-error.s
similarity index 92%
rename from test/MC/AArch64/armv8.5a-predctrl-error.s
rename to test/MC/AArch64/armv8.5a-predres-error.s
index 0fd49b02bee07cdb6dd40b085dc19b8fac31aa27..295252d0f6783f5da1c8f078e2f51ac3420f8557 100644
--- a/test/MC/AArch64/armv8.5a-predctrl-error.s
+++ b/test/MC/AArch64/armv8.5a-predres-error.s
@@ -1,4 +1,4 @@
-// RUN: not llvm-mc -triple aarch64-none-linux-gnu -show-encoding -mattr=+predctrl < %s 2>&1| FileCheck %s
+// RUN: not llvm-mc -triple aarch64-none-linux-gnu -show-encoding -mattr=+predres < %s 2>&1| FileCheck %s
 
 cfp rctx
 dvp rctx
diff --git a/test/MC/AArch64/armv8.5a-predctrl.s b/test/MC/AArch64/armv8.5a-predres.s
similarity index 69%
rename from test/MC/AArch64/armv8.5a-predctrl.s
rename to test/MC/AArch64/armv8.5a-predres.s
index af7dda76862d810659e908361d4b2c14c95a7269..4bab34769ac9d86613d5772d7db44ff22277ba1f 100644
--- a/test/MC/AArch64/armv8.5a-predctrl.s
+++ b/test/MC/AArch64/armv8.5a-predres.s
@@ -1,6 +1,6 @@
-// RUN:     llvm-mc -triple aarch64-none-linux-gnu -show-encoding -mattr=+predctrl < %s      | FileCheck %s
+// RUN:     llvm-mc -triple aarch64-none-linux-gnu -show-encoding -mattr=+predres < %s      | FileCheck %s
 // RUN:     llvm-mc -triple aarch64-none-linux-gnu -show-encoding -mattr=+v8.5a    < %s      | FileCheck %s
-// RUN: not llvm-mc -triple aarch64-none-linux-gnu -show-encoding -mattr=-predctrl < %s 2>&1 | FileCheck %s --check-prefix=NOPREDCTRL
+// RUN: not llvm-mc -triple aarch64-none-linux-gnu -show-encoding -mattr=-predres < %s 2>&1 | FileCheck %s --check-prefix=NOPREDCTRL
 
 cfp rctx, x0
 dvp rctx, x1
@@ -10,9 +10,9 @@ cpp rctx, x2
 // CHECK: dvp rctx, x1      // encoding: [0xa1,0x73,0x0b,0xd5]
 // CHECK: cpp rctx, x2      // encoding: [0xe2,0x73,0x0b,0xd5]
 
-// NOPREDCTRL: CFPRCTX requires predctrl
+// NOPREDCTRL: CFPRCTX requires predres
 // NOPREDCTRL-NEXT: cfp
-// NOPREDCTRL: DVPRCTX requires predctrl
+// NOPREDCTRL: DVPRCTX requires predres
 // NOPREDCTRL-NEXT: dvp
-// NOPREDCTRL: CPPRCTX requires predctrl
+// NOPREDCTRL: CPPRCTX requires predres
 // NOPREDCTRL-NEXT: cpp
diff --git a/test/MC/AArch64/armv8.5a-specctrl.s b/test/MC/AArch64/armv8.5a-sb.s
similarity index 59%
rename from test/MC/AArch64/armv8.5a-specctrl.s
rename to test/MC/AArch64/armv8.5a-sb.s
index 279a396250c383e8ba41495180a48a105ac757e8..7d88aed428bd6f2845a65089f8f63764203c37b8 100644
--- a/test/MC/AArch64/armv8.5a-specctrl.s
+++ b/test/MC/AArch64/armv8.5a-sb.s
@@ -1,11 +1,11 @@
-// RUN:     llvm-mc -triple aarch64-none-linux-gnu -show-encoding -mattr=+specctrl < %s      | FileCheck %s
-// RUN:     llvm-mc -triple aarch64-none-linux-gnu -show-encoding -mattr=+v8.5a    < %s      | FileCheck %s
-// RUN: not llvm-mc -triple aarch64-none-linux-gnu -show-encoding -mattr=-specctrl < %s 2>&1 | FileCheck %s --check-prefix=NOSB
+// RUN:     llvm-mc -triple aarch64-none-linux-gnu -show-encoding -mattr=+sb -o - %s      | FileCheck %s
+// RUN:     llvm-mc -triple aarch64-none-linux-gnu -show-encoding -mattr=+v8.5a -o - %s      | FileCheck %s
+// RUN: not llvm-mc -triple aarch64-none-linux-gnu -show-encoding -mattr=-sb -o - %s 2>&1 | FileCheck %s --check-prefix=NOSB
 
 // Flag manipulation
 sb
 
 // CHECK: sb // encoding: [0xff,0x30,0x03,0xd5]
 
-// NOSB: instruction requires: specctrl
+// NOSB: instruction requires: sb
 // NOSB-NEXT: sb
diff --git a/test/MC/AArch64/atomic-acquire-comment.s b/test/MC/AArch64/atomic-acquire-comment.s
new file mode 100644
index 0000000000000000000000000000000000000000..62aadddf083721ba9d20bca2010f765d7b67cbee
--- /dev/null
+++ b/test/MC/AArch64/atomic-acquire-comment.s
@@ -0,0 +1,190 @@
+; RUN: llvm-mc -triple aarch64-apple-ios -mattr=+lse %s | FileCheck %s
+
+    ; CHECK: ldaddab w3, wzr, [x5] ; acquire semantics dropped since destination is zero
+    ; CHECK: ldaddah w5, wzr, [x0] ; acquire semantics dropped since destination is zero
+    ; CHECK: ldadda w7, wzr, [x5] ; acquire semantics dropped since destination is zero
+    ; CHECK: ldadda x9, xzr, [sp] ; acquire semantics dropped since destination is zero
+    ldaddab w3, wzr, [x5]
+    ldaddah w5, wzr, [x0]
+    ldadda w7, wzr, [x5]
+    ldadda x9, xzr, [sp]
+
+    ; CHECK: ldaddalb w3, wzr, [x5] ; acquire semantics dropped since destination is zero
+    ; CHECK: ldaddalh w5, wzr, [x0] ; acquire semantics dropped since destination is zero
+    ; CHECK: ldaddal w7, wzr, [x5] ; acquire semantics dropped since destination is zero
+    ; CHECK: ldaddal x9, xzr, [sp] ; acquire semantics dropped since destination is zero
+    ldaddalb w3, wzr, [x5]
+    ldaddalh w5, wzr, [x0]
+    ldaddal w7, wzr, [x5]
+    ldaddal x9, xzr, [sp]
+
+    ; CHECK: ldclrab w3, wzr, [x5] ; acquire semantics dropped since destination is zero
+    ; CHECK: ldclrah w5, wzr, [x0] ; acquire semantics dropped since destination is zero
+    ; CHECK: ldclra w7, wzr, [x5] ; acquire semantics dropped since destination is zero
+    ; CHECK: ldclra x9, xzr, [sp] ; acquire semantics dropped since destination is zero
+    ldclrab w3, wzr, [x5]
+    ldclrah w5, wzr, [x0]
+    ldclra w7, wzr, [x5]
+    ldclra x9, xzr, [sp]
+
+    ; CHECK: ldclralb w3, wzr, [x5] ; acquire semantics dropped since destination is zero
+    ; CHECK: ldclralh w5, wzr, [x0] ; acquire semantics dropped since destination is zero
+    ; CHECK: ldclral w7, wzr, [x5] ; acquire semantics dropped since destination is zero
+    ; CHECK: ldclral x9, xzr, [sp] ; acquire semantics dropped since destination is zero
+    ldclralb w3, wzr, [x5]
+    ldclralh w5, wzr, [x0]
+    ldclral w7, wzr, [x5]
+    ldclral x9, xzr, [sp]
+
+    ; CHECK: ldeorab w3, wzr, [x5] ; acquire semantics dropped since destination is zero
+    ; CHECK: ldeorah w5, wzr, [x0] ; acquire semantics dropped since destination is zero
+    ; CHECK: ldeora w7, wzr, [x5] ; acquire semantics dropped since destination is zero
+    ; CHECK: ldeora x9, xzr, [sp] ; acquire semantics dropped since destination is zero
+    ldeorab w3, wzr, [x5]
+    ldeorah w5, wzr, [x0]
+    ldeora w7, wzr, [x5]
+    ldeora x9, xzr, [sp]
+
+    ; CHECK: ldeoralb w3, wzr, [x5] ; acquire semantics dropped since destination is zero
+    ; CHECK: ldeoralh w5, wzr, [x0] ; acquire semantics dropped since destination is zero
+    ; CHECK: ldeoral w7, wzr, [x5] ; acquire semantics dropped since destination is zero
+    ; CHECK: ldeoral x9, xzr, [sp] ; acquire semantics dropped since destination is zero
+    ldeoralb w3, wzr, [x5]
+    ldeoralh w5, wzr, [x0]
+    ldeoral w7, wzr, [x5]
+    ldeoral x9, xzr, [sp]
+
+    ; CHECK: ldsetab w3, wzr, [x5] ; acquire semantics dropped since destination is zero
+    ; CHECK: ldsetah w5, wzr, [x0] ; acquire semantics dropped since destination is zero
+    ; CHECK: ldseta w7, wzr, [x5] ; acquire semantics dropped since destination is zero
+    ; CHECK: ldseta x9, xzr, [sp] ; acquire semantics dropped since destination is zero
+    ldsetab w3, wzr, [x5]
+    ldsetah w5, wzr, [x0]
+    ldseta w7, wzr, [x5]
+    ldseta x9, xzr, [sp]
+
+    ; CHECK: ldsetalb w3, wzr, [x5] ; acquire semantics dropped since destination is zero
+    ; CHECK: ldsetalh w5, wzr, [x0] ; acquire semantics dropped since destination is zero
+    ; CHECK: ldsetal w7, wzr, [x5] ; acquire semantics dropped since destination is zero
+    ; CHECK: ldsetal x9, xzr, [sp] ; acquire semantics dropped since destination is zero
+    ldsetalb w3, wzr, [x5]
+    ldsetalh w5, wzr, [x0]
+    ldsetal w7, wzr, [x5]
+    ldsetal x9, xzr, [sp]
+
+    ; CHECK: ldsmaxab w3, wzr, [x5] ; acquire semantics dropped since destination is zero
+    ; CHECK: ldsmaxah w5, wzr, [x0] ; acquire semantics dropped since destination is zero
+    ; CHECK: ldsmaxa w7, wzr, [x5] ; acquire semantics dropped since destination is zero
+    ; CHECK: ldsmaxa x9, xzr, [sp] ; acquire semantics dropped since destination is zero
+    ldsmaxab w3, wzr, [x5]
+    ldsmaxah w5, wzr, [x0]
+    ldsmaxa w7, wzr, [x5]
+    ldsmaxa x9, xzr, [sp]
+
+    ; CHECK: ldsmaxalb w3, wzr, [x5] ; acquire semantics dropped since destination is zero
+    ; CHECK: ldsmaxalh w5, wzr, [x0] ; acquire semantics dropped since destination is zero
+    ; CHECK: ldsmaxal w7, wzr, [x5] ; acquire semantics dropped since destination is zero
+    ; CHECK: ldsmaxal x9, xzr, [sp] ; acquire semantics dropped since destination is zero
+    ldsmaxalb w3, wzr, [x5]
+    ldsmaxalh w5, wzr, [x0]
+    ldsmaxal w7, wzr, [x5]
+    ldsmaxal x9, xzr, [sp]
+
+    ; CHECK: ldsminab w3, wzr, [x5] ; acquire semantics dropped since destination is zero
+    ; CHECK: ldsminah w5, wzr, [x0] ; acquire semantics dropped since destination is zero
+    ; CHECK: ldsmina w7, wzr, [x5] ; acquire semantics dropped since destination is zero
+    ; CHECK: ldsmina x9, xzr, [sp] ; acquire semantics dropped since destination is zero
+    ldsminab w3, wzr, [x5]
+    ldsminah w5, wzr, [x0]
+    ldsmina w7, wzr, [x5]
+    ldsmina x9, xzr, [sp]
+
+    ; CHECK: ldsminalb w3, wzr, [x5] ; acquire semantics dropped since destination is zero
+    ; CHECK: ldsminalh w5, wzr, [x0] ; acquire semantics dropped since destination is zero
+    ; CHECK: ldsminal w7, wzr, [x5] ; acquire semantics dropped since destination is zero
+    ; CHECK: ldsminal x9, xzr, [sp] ; acquire semantics dropped since destination is zero
+    ldsminalb w3, wzr, [x5]
+    ldsminalh w5, wzr, [x0]
+    ldsminal w7, wzr, [x5]
+    ldsminal x9, xzr, [sp]
+
+    ; CHECK: ldumaxab w3, wzr, [x5] ; acquire semantics dropped since destination is zero
+    ; CHECK: ldumaxah w5, wzr, [x0] ; acquire semantics dropped since destination is zero
+    ; CHECK: ldumaxa w7, wzr, [x5] ; acquire semantics dropped since destination is zero
+    ; CHECK: ldumaxa x9, xzr, [sp] ; acquire semantics dropped since destination is zero
+    ldumaxab w3, wzr, [x5]
+    ldumaxah w5, wzr, [x0]
+    ldumaxa w7, wzr, [x5]
+    ldumaxa x9, xzr, [sp]
+
+    ; CHECK: ldumaxalb w3, wzr, [x5] ; acquire semantics dropped since destination is zero
+    ; CHECK: ldumaxalh w5, wzr, [x0] ; acquire semantics dropped since destination is zero
+    ; CHECK: ldumaxal w7, wzr, [x5] ; acquire semantics dropped since destination is zero
+    ; CHECK: ldumaxal x9, xzr, [sp] ; acquire semantics dropped since destination is zero
+    ldumaxalb w3, wzr, [x5]
+    ldumaxalh w5, wzr, [x0]
+    ldumaxal w7, wzr, [x5]
+    ldumaxal x9, xzr, [sp]
+
+    ; CHECK: lduminab w3, wzr, [x5] ; acquire semantics dropped since destination is zero
+    ; CHECK: lduminah w5, wzr, [x0] ; acquire semantics dropped since destination is zero
+    ; CHECK: ldumina w7, wzr, [x5] ; acquire semantics dropped since destination is zero
+    ; CHECK: ldumina x9, xzr, [sp] ; acquire semantics dropped since destination is zero
+    lduminab w3, wzr, [x5]
+    lduminah w5, wzr, [x0]
+    ldumina w7, wzr, [x5]
+    ldumina x9, xzr, [sp]
+
+    ; CHECK: lduminalb w3, wzr, [x5] ; acquire semantics dropped since destination is zero
+    ; CHECK: lduminalh w5, wzr, [x0] ; acquire semantics dropped since destination is zero
+    ; CHECK: lduminal w7, wzr, [x5] ; acquire semantics dropped since destination is zero
+    ; CHECK: lduminal x9, xzr, [sp] ; acquire semantics dropped since destination is zero
+    lduminalb w3, wzr, [x5]
+    lduminalh w5, wzr, [x0]
+    lduminal w7, wzr, [x5]
+    lduminal x9, xzr, [sp]
+
+    ; CHECK: swpab w3, wzr, [x5] ; acquire semantics dropped since destination is zero
+    ; CHECK: swpah w5, wzr, [x0] ; acquire semantics dropped since destination is zero
+    ; CHECK: swpa w7, wzr, [x5] ; acquire semantics dropped since destination is zero
+    ; CHECK: swpa x9, xzr, [sp] ; acquire semantics dropped since destination is zero
+    swpab w3, wzr, [x5]
+    swpah w5, wzr, [x0]
+    swpa w7, wzr, [x5]
+    swpa x9, xzr, [sp]
+
+    ; CHECK: swpalb w3, wzr, [x5] ; acquire semantics dropped since destination is zero
+    ; CHECK: swpalh w5, wzr, [x0] ; acquire semantics dropped since destination is zero
+    ; CHECK: swpal w7, wzr, [x5] ; acquire semantics dropped since destination is zero
+    ; CHECK: swpal x9, xzr, [sp] ; acquire semantics dropped since destination is zero
+    swpalb w3, wzr, [x5]
+    swpalh w5, wzr, [x0]
+    swpal w7, wzr, [x5]
+    swpal x9, xzr, [sp]
+
+    ; CHECK: ldaddal xzr, x3, [x0]{{$}}
+    ; CHECK: ldeora wzr, w7, [x5]{{$}}
+    ; CHECK: ldsminb w5, w9, [sp]{{$}}
+    ldaddal xzr, x3, [x0]
+    ldeora wzr, w7, [x5]
+    ldsminb w5, w9, [sp]
+
+    ; CAS instructions aren't affected.
+
+    ; CHECK: casab w3, wzr, [x5]{{$}}
+    ; CHECK: casah w5, wzr, [x0]{{$}}
+    ; CHECK: casa w7, wzr, [x5]{{$}}
+    ; CHECK: casa x9, xzr, [sp]{{$}}
+    casab w3, wzr, [x5]
+    casah w5, wzr, [x0]
+    casa w7, wzr, [x5]
+    casa x9, xzr, [sp]
+
+    ; CHECK: casalb w3, wzr, [x5]{{$}}
+    ; CHECK: casalh w5, wzr, [x0]{{$}}
+    ; CHECK: casal w7, wzr, [x5]{{$}}
+    ; CHECK: casal x9, xzr, [sp]{{$}}
+    casalb w3, wzr, [x5]
+    casalh w5, wzr, [x0]
+    casal w7, wzr, [x5]
+    casal x9, xzr, [sp]
diff --git a/test/MC/AArch64/cold.s b/test/MC/AArch64/cold.s
new file mode 100644
index 0000000000000000000000000000000000000000..44cb52fd7946869734305c2e73dd2e8174240e40
--- /dev/null
+++ b/test/MC/AArch64/cold.s
@@ -0,0 +1,13 @@
+// RUN: llvm-mc -triple aarch64-apple-ios -o - -filetype=obj %s | \
+// RUN:   llvm-readobj -symbols | FileCheck %s -check-prefix=READOBJ
+
+// READOBJ-LABEL: Name: cold_func
+// READOBJ-NEXT: Type: Section
+// READOBJ-NEXT: Section: __text
+// READOBJ-NEXT: RefType: UndefinedNonLazy (0x0)
+// READOBJ-NEXT: Flags [ (0x400)
+
+  .text
+  .cold cold_func
+cold_func:
+  ret
diff --git a/test/MC/AArch64/directive-arch_extension-nosimd.s b/test/MC/AArch64/directive-arch_extension-nosimd.s
new file mode 100644
index 0000000000000000000000000000000000000000..cbc03d05b342b630e2f03371343bffaa44ff1083
--- /dev/null
+++ b/test/MC/AArch64/directive-arch_extension-nosimd.s
@@ -0,0 +1,6 @@
+// RUN: not llvm-mc -triple aarch64 -filetype asm -o - %s 2>&1 | FileCheck %s
+
+	.arch_extension nosimd
+
+	add v0.8b, v0.8b, v0.8b
+// CHECK: error: instruction requires: neon
diff --git a/test/MC/AArch64/directive-arch_extension-simd.s b/test/MC/AArch64/directive-arch_extension-simd.s
new file mode 100644
index 0000000000000000000000000000000000000000..04ebeee863443126d3ec2d13192ca9e0ba3c06bd
--- /dev/null
+++ b/test/MC/AArch64/directive-arch_extension-simd.s
@@ -0,0 +1,6 @@
+// RUN: llvm-mc -triple aarch64 -mattr=-simd -filetype asm -o - %s | FileCheck %s
+
+	.arch_extension simd
+
+	add v0.8b, v0.8b, v0.8b
+// CHECK: add v0.8b, v0.8b, v0.8b
diff --git a/test/MC/AArch64/fixup-absolute-signed.s b/test/MC/AArch64/fixup-absolute-signed.s
new file mode 100644
index 0000000000000000000000000000000000000000..3dd93c9128aa83472faa4afe17be9e352221bf0d
--- /dev/null
+++ b/test/MC/AArch64/fixup-absolute-signed.s
@@ -0,0 +1,44 @@
+// RUN: llvm-mc -triple aarch64--none-eabi -filetype obj < %s -o - | llvm-objdump -d - | FileCheck %s
+
+onepart_before = 12345
+twopart_before = -12345678
+threepart_before = -1234567890
+
+// CHECK: movn     x0, #0, lsl #32
+// CHECK: movn     x0, #0, lsl #32
+movz x0, #:abs_g2_s:threepart_before
+movz x0, #:abs_g2_s:threepart_after
+
+// CHECK: movk    x0, #65535, lsl #32
+// CHECK: movk    x0, #65535, lsl #32
+movk x0, #:abs_g2_nc:threepart_before
+movk x0, #:abs_g2_nc:threepart_after
+
+// CHECK: mov     x0, #-12320769
+// CHECK: mov     x0, #-12320769
+movz x0, #:abs_g1_s:twopart_before
+movz x0, #:abs_g1_s:twopart_after
+
+// CHECK: movk    x0, #46697, lsl #16
+// CHECK: movk    x0, #46697, lsl #16
+movk x0, #:abs_g1_nc:threepart_before
+movk x0, #:abs_g1_nc:threepart_after
+
+// CHECK: mov     x0, #12345
+// CHECK: mov     x0, #12345
+movz x0, #:abs_g0_s:onepart_before
+movz x0, #:abs_g0_s:onepart_after
+
+// CHECK: movk    x0, #64814
+// CHECK: movk    x0, #64814
+movk x0, #:abs_g0_nc:threepart_before
+movk x0, #:abs_g0_nc:threepart_after
+
+// CHECK: mov     x0, #12345
+// CHECK: mov     x0, #12345
+movn x0, #:abs_g0_s:onepart_before
+movn x0, #:abs_g0_s:onepart_after
+
+onepart_after = 12345
+twopart_after = -12345678
+threepart_after = -1234567890
diff --git a/test/MC/AArch64/fixup-absolute.s b/test/MC/AArch64/fixup-absolute.s
new file mode 100644
index 0000000000000000000000000000000000000000..ed4b3b0d4c95b5ea6c3cfd335cb354b3fe66c984
--- /dev/null
+++ b/test/MC/AArch64/fixup-absolute.s
@@ -0,0 +1,40 @@
+// RUN: llvm-mc -triple aarch64--none-eabi -filetype obj < %s -o - | llvm-objdump -d - | FileCheck %s
+
+onepart_before = 0x1234
+twopart_before = 0x12345678
+threepart_before = 0x1234567890AB
+fourpart_before = 0x1234567890ABCDEF
+
+// CHECK: mov     x0, #1311673391471656960
+// CHECK: mov     x0, #1311673391471656960
+movz x0, #:abs_g3:fourpart_before
+movz x0, #:abs_g3:fourpart_after
+// CHECK: mov     x0, #20014547599360
+// CHECK: mov     x0, #20014547599360
+movz x0, #:abs_g2:threepart_before
+movz x0, #:abs_g2:threepart_after
+// CHECK: movk    x0, #22136, lsl #32
+// CHECK: movk    x0, #22136, lsl #32
+movk x0, #:abs_g2_nc:fourpart_before
+movk x0, #:abs_g2_nc:fourpart_after
+// CHECK: mov     x0, #305397760
+// CHECK: mov     x0, #305397760
+movz x0, #:abs_g1:twopart_before
+movz x0, #:abs_g1:twopart_after
+// CHECK: movk    x0, #37035, lsl #16
+// CHECK: movk    x0, #37035, lsl #16
+movk x0, #:abs_g1_nc:fourpart_before
+movk x0, #:abs_g1_nc:fourpart_after
+// CHECK: mov     x0, #4660
+// CHECK: mov     x0, #4660
+movz x0, #:abs_g0:onepart_before
+movz x0, #:abs_g0:onepart_after
+// CHECK: movk    x0, #52719
+// CHECK: movk    x0, #52719
+movk x0, #:abs_g0_nc:fourpart_before
+movk x0, #:abs_g0_nc:fourpart_after
+
+onepart_after = 0x1234
+twopart_after = 0x12345678
+threepart_after = 0x1234567890AB
+fourpart_after = 0x1234567890ABCDEF
diff --git a/test/MC/AArch64/fixup-out-of-range.s b/test/MC/AArch64/fixup-out-of-range.s
index 3acffe202f03c75c63f35ee1b25ad50a4d12ed9f..500c1d1d1f9afe9aa97fa70945a6423ce2861d11 100644
--- a/test/MC/AArch64/fixup-out-of-range.s
+++ b/test/MC/AArch64/fixup-out-of-range.s
@@ -55,6 +55,22 @@
 // CHECK: :[[@LINE+1]]:{{[0-9]+}}: error: fixup not sufficiently aligned
   b unaligned
 
+// CHECK: :[[@LINE+1]]:{{[0-9]+}}: error: fixup value out of range
+  movz x0, #:abs_g0:value1
+
+// CHECK: :[[@LINE+1]]:{{[0-9]+}}: error: fixup value out of range
+  movz x0, #:abs_g1:value2
+
+// CHECK: :[[@LINE+1]]:{{[0-9]+}}: error: fixup value out of range
+  movz x0, #:abs_g0_s:value1
+
+// CHECK: :[[@LINE+1]]:{{[0-9]+}}: error: fixup value out of range
+  movz x0, #:abs_g1_s:value2
+
+// CHECK: :[[@LINE+1]]:{{[0-9]+}}: error: relocation for a thread-local variable points to an absolute symbol
+  movz x0, #:tprel_g0:value1
+
+
   .byte 0
 unaligned:
   .byte 0
@@ -63,3 +79,5 @@ unaligned:
   .balign 8
 distant:
   .word 0
+value1 = 0x12345678
+value2 = 0x123456789
diff --git a/test/MC/AMDGPU/hsa-v3.s b/test/MC/AMDGPU/hsa-v3.s
index 6adcdb8447854abcd95c58dc1128f912ac03c58d..515da8803aab86dfb3ffbf392b1ccc05bc143029 100644
--- a/test/MC/AMDGPU/hsa-v3.s
+++ b/test/MC/AMDGPU/hsa-v3.s
@@ -1,5 +1,5 @@
-// RUN: llvm-mc -mattr=+code-object-v3 -triple amdgcn-amd-amdhsa -mcpu=gfx900 -mattr=+xnack < %s | FileCheck --check-prefix=ASM %s
-// RUN: llvm-mc -mattr=+code-object-v3 -triple amdgcn-amd-amdhsa -mcpu=gfx900 -mattr=+xnack -filetype=obj < %s > %t
+// RUN: llvm-mc -mattr=+code-object-v3 -triple amdgcn-amd-amdhsa -mcpu=gfx904 -mattr=+xnack < %s | FileCheck --check-prefix=ASM %s
+// RUN: llvm-mc -mattr=+code-object-v3 -triple amdgcn-amd-amdhsa -mcpu=gfx904 -mattr=+xnack -filetype=obj < %s > %t
 // RUN: llvm-readobj -elf-output-style=GNU -sections -symbols -relocations %t | FileCheck --check-prefix=READOBJ %s
 // RUN: llvm-objdump -s -j .rodata %t | FileCheck --check-prefix=OBJDUMP %s
 
@@ -44,8 +44,8 @@
 .text
 // ASM: .text
 
-.amdgcn_target "amdgcn-amd-amdhsa--gfx900+xnack"
-// ASM: .amdgcn_target "amdgcn-amd-amdhsa--gfx900+xnack"
+.amdgcn_target "amdgcn-amd-amdhsa--gfx904+xnack"
+// ASM: .amdgcn_target "amdgcn-amd-amdhsa--gfx904+xnack"
 
 .p2align 8
 .type minimal,@function
@@ -187,6 +187,12 @@ special_sgpr:
 .byte .amdgcn.gfx_generation_number
 // ASM: .byte 9
 
+.byte .amdgcn.gfx_generation_minor
+// ASM: .byte 0
+
+.byte .amdgcn.gfx_generation_stepping
+// ASM: .byte 4
+
 .byte .amdgcn.next_free_vgpr
 // ASM: .byte 0
 .byte .amdgcn.next_free_sgpr
diff --git a/test/MC/AMDGPU/lds_direct-err.s b/test/MC/AMDGPU/lds_direct-err.s
new file mode 100644
index 0000000000000000000000000000000000000000..25a5cda6a54f8c2075d975fd503a62021d230ece
--- /dev/null
+++ b/test/MC/AMDGPU/lds_direct-err.s
@@ -0,0 +1,59 @@
+// RUN: not llvm-mc -arch=amdgcn -mcpu=gfx900 %s 2>&1 | FileCheck %s --check-prefix=NOGFX9
+
+//---------------------------------------------------------------------------//
+// lds_direct may be used only with vector ALU instructions
+//---------------------------------------------------------------------------//
+
+s_and_b32 s2, lds_direct, s1
+// NOGFX9: error
+
+//---------------------------------------------------------------------------//
+// lds_direct may not be used with V_{LSHL,LSHR,ASHL}REV opcodes
+//---------------------------------------------------------------------------//
+
+v_ashrrev_i16 v0, lds_direct, v0
+// NOGFX9: error
+
+v_ashrrev_i32 v0, lds_direct, v0
+// NOGFX9: error
+
+v_lshlrev_b16 v0, lds_direct, v0
+// NOGFX9: error
+
+v_lshlrev_b32 v0, lds_direct, v0
+// NOGFX9: error
+
+v_lshrrev_b16 v0, lds_direct, v0
+// NOGFX9: error
+
+v_lshrrev_b32 v0, lds_direct, v0
+// NOGFX9: error
+
+v_pk_ashrrev_i16 v0, lds_direct, v0
+// NOGFX9: error
+
+v_pk_lshlrev_b16 v0, lds_direct, v0
+// NOGFX9: error
+
+v_pk_lshrrev_b16 v0, lds_direct, v0
+// NOGFX9: error
+
+//---------------------------------------------------------------------------//
+// lds_direct cannot be used with 64-bit and larger operands
+//---------------------------------------------------------------------------//
+
+v_add_f64 v[0:1], lds_direct, v[0:1]
+// NOGFX9: error
+
+//---------------------------------------------------------------------------//
+// Only SRC0 may specify lds_direct
+//---------------------------------------------------------------------------//
+
+v_add_i32 v0, v0, lds_direct
+// NOGFX9: error
+
+v_add_i32 lds_direct, v0, v0
+// NOGFX9: error
+
+v_fma_f32 v0, v0, v0, lds_direct
+// NOGFX9: error
diff --git a/test/MC/AMDGPU/lds_direct.s b/test/MC/AMDGPU/lds_direct.s
new file mode 100644
index 0000000000000000000000000000000000000000..33f631e79b02e16ba9935747aa0b68985aeb8f7a
--- /dev/null
+++ b/test/MC/AMDGPU/lds_direct.s
@@ -0,0 +1,116 @@
+// RUN: llvm-mc -arch=amdgcn -mcpu=gfx900 -show-encoding %s | FileCheck %s --check-prefix=GFX9
+
+//---------------------------------------------------------------------------//
+// VOP1/3
+//---------------------------------------------------------------------------//
+
+v_mov_b32 v0, src_lds_direct
+// GFX9: v_mov_b32_e32 v0, src_lds_direct ; encoding: [0xfe,0x02,0x00,0x7e]
+
+v_mov_b32_e64 v0, src_lds_direct
+// GFX9: v_mov_b32_e64 v0, src_lds_direct ; encoding: [0x00,0x00,0x41,0xd1,0xfe,0x00,0x00,0x00]
+
+v_cvt_f64_i32 v[0:1], src_lds_direct
+// GFX9: v_cvt_f64_i32_e32 v[0:1], src_lds_direct ; encoding: [0xfe,0x08,0x00,0x7e]
+
+v_cvt_f64_i32_e64 v[0:1], src_lds_direct
+// GFX9: v_cvt_f64_i32_e64 v[0:1], src_lds_direct ; encoding: [0x00,0x00,0x44,0xd1,0xfe,0x00,0x00,0x00]
+
+v_mov_fed_b32 v0, src_lds_direct
+// GFX9: v_mov_fed_b32_e32 v0, src_lds_direct ; encoding: [0xfe,0x12,0x00,0x7e]
+
+v_mov_fed_b32_e64 v0, src_lds_direct
+// GFX9: v_mov_fed_b32_e64 v0, src_lds_direct ; encoding: [0x00,0x00,0x49,0xd1,0xfe,0x00,0x00,0x00]
+
+v_fract_f32 v0, src_lds_direct
+// GFX9: v_fract_f32_e32 v0, src_lds_direct ; encoding: [0xfe,0x36,0x00,0x7e]
+
+v_fract_f32_e64 v0, src_lds_direct
+// GFX9: v_fract_f32_e64 v0, src_lds_direct ; encoding: [0x00,0x00,0x5b,0xd1,0xfe,0x00,0x00,0x00]
+
+v_cvt_f16_u16 v0, src_lds_direct
+// GFX9: v_cvt_f16_u16_e32 v0, src_lds_direct ; encoding: [0xfe,0x72,0x00,0x7e]
+
+//---------------------------------------------------------------------------//
+// VOP2/3
+//---------------------------------------------------------------------------//
+
+v_cndmask_b32 v0, src_lds_direct, v0, vcc
+// GFX9: v_cndmask_b32_e32 v0, src_lds_direct, v0, vcc ; encoding: [0xfe,0x00,0x00,0x00]
+
+v_cndmask_b32_e64 v0, src_lds_direct, v0, s[0:1]
+// GFX9: v_cndmask_b32_e64 v0, src_lds_direct, v0, s[0:1] ; encoding: [0x00,0x00,0x00,0xd1,0xfe,0x00,0x02,0x00]
+
+v_add_f32 v0, src_lds_direct, v0
+// GFX9: v_add_f32_e32 v0, src_lds_direct, v0 ; encoding: [0xfe,0x00,0x00,0x02]
+
+v_add_f32_e64 v0, src_lds_direct, v0
+// GFX9: v_add_f32_e64 v0, src_lds_direct, v0 ; encoding: [0x00,0x00,0x01,0xd1,0xfe,0x00,0x02,0x00]
+
+v_mul_i32_i24 v0, src_lds_direct, v0
+// GFX9: v_mul_i32_i24_e32 v0, src_lds_direct, v0 ; encoding: [0xfe,0x00,0x00,0x0c]
+
+v_add_co_u32 v0, vcc, src_lds_direct, v0
+// GFX9: v_add_co_u32_e32 v0, vcc, src_lds_direct, v0 ; encoding: [0xfe,0x00,0x00,0x32]
+
+//---------------------------------------------------------------------------//
+// VOP3
+//---------------------------------------------------------------------------//
+
+v_add_co_u32_e64 v0, s[0:1], src_lds_direct, v0
+// GFX9: v_add_co_u32_e64 v0, s[0:1], src_lds_direct, v0 ; encoding: [0x00,0x00,0x19,0xd1,0xfe,0x00,0x02,0x00]
+
+v_madmk_f16 v0, src_lds_direct, 0x1121, v0
+// GFX9: v_madmk_f16 v0, src_lds_direct, 0x1121, v0 ; encoding: [0xfe,0x00,0x00,0x48,0x21,0x11,0x00,0x00]
+
+v_madak_f16 v0, src_lds_direct, v0, 0x1121
+// GFX9: v_madak_f16 v0, src_lds_direct, v0, 0x1121 ; encoding: [0xfe,0x00,0x00,0x4a,0x21,0x11,0x00,0x00]
+
+v_mad_f32 v0, src_lds_direct, v0, v0
+// GFX9: v_mad_f32 v0, src_lds_direct, v0, v0 ; encoding: [0x00,0x00,0xc1,0xd1,0xfe,0x00,0x02,0x04]
+
+v_fma_f32 v0, src_lds_direct, v0, v0
+// GFX9: v_fma_f32 v0, src_lds_direct, v0, v0 ; encoding: [0x00,0x00,0xcb,0xd1,0xfe,0x00,0x02,0x04]
+
+v_min3_i16 v0, src_lds_direct, v0, v0
+// GFX9: v_min3_i16 v0, src_lds_direct, v0, v0 ; encoding: [0x00,0x00,0xf5,0xd1,0xfe,0x00,0x02,0x04]
+
+v_max3_f16 v0, src_lds_direct, v0, v0
+// GFX9: v_max3_f16 v0, src_lds_direct, v0, v0 ; encoding: [0x00,0x00,0xf7,0xd1,0xfe,0x00,0x02,0x04]
+
+//---------------------------------------------------------------------------//
+// VOP3P
+//---------------------------------------------------------------------------//
+
+v_pk_mad_i16 v0, src_lds_direct, v0, v0
+// GFX9: v_pk_mad_i16 v0, src_lds_direct, v0, v0 ; encoding: [0x00,0x40,0x80,0xd3,0xfe,0x00,0x02,0x1c]
+
+v_pk_add_i16 v0, src_lds_direct, v0
+// GFX9: v_pk_add_i16 v0, src_lds_direct, v0 ; encoding: [0x00,0x00,0x82,0xd3,0xfe,0x00,0x02,0x18]
+
+//---------------------------------------------------------------------------//
+// VOPC
+//---------------------------------------------------------------------------//
+
+v_cmp_lt_f16 vcc, src_lds_direct, v0
+// GFX9: v_cmp_lt_f16_e32 vcc, src_lds_direct, v0 ; encoding: [0xfe,0x00,0x42,0x7c]
+
+v_cmp_eq_f32 vcc, src_lds_direct, v0
+// GFX9: v_cmp_eq_f32_e32 vcc, src_lds_direct, v0 ; encoding: [0xfe,0x00,0x84,0x7c]
+
+v_cmpx_neq_f32 vcc, src_lds_direct, v0
+// GFX9: v_cmpx_neq_f32_e32 vcc, src_lds_direct, v0 ; encoding: [0xfe,0x00,0xba,0x7c]
+
+//---------------------------------------------------------------------------//
+// lds_direct alias
+//---------------------------------------------------------------------------//
+
+v_cmp_lt_f16 vcc, lds_direct, v0
+// GFX9: v_cmp_lt_f16_e32 vcc, src_lds_direct, v0 ; encoding: [0xfe,0x00,0x42,0x7c]
+
+//---------------------------------------------------------------------------//
+// FIXME: enable lds_direct for the following opcodes and add tests
+//---------------------------------------------------------------------------//
+
+//v_readfirstlane_b32 s0, src_lds_direct
+//v_readlane_b32 s0, src_lds_direct, s0
diff --git a/test/MC/AMDGPU/literals.s b/test/MC/AMDGPU/literals.s
index c18da5dd8ffec09b7229740132719084942bb915..486fe823a219abad7c3e10ab3ab6e7bea57096ad 100644
--- a/test/MC/AMDGPU/literals.s
+++ b/test/MC/AMDGPU/literals.s
@@ -429,7 +429,7 @@ v_and_b32_e32 v0, 0xffffffffffffffff, v1
 v_trunc_f32_e32 v0, 0x3fc45f306dc9c882
 
 // NOSICI: error: invalid operand for instruction
-// VI: v_fract_f64_e32 v[0:1], 0.15915494 ; encoding: [0xf8,0x64,0x00,0x7e]
+// VI: v_fract_f64_e32 v[0:1], 0.15915494309189532 ; encoding: [0xf8,0x64,0x00,0x7e]
 v_fract_f64_e32 v[0:1], 0x3fc45f306dc9c882
 
 // SICI: v_trunc_f32_e32 v0, 0x3e22f983 ; encoding: [0xff,0x42,0x00,0x7e,0x83,0xf9,0x22,0x3e]
@@ -445,7 +445,7 @@ v_fract_f64_e32 v[0:1], 0x3e22f983
 v_trunc_f32_e64 v0, 0x3fc45f306dc9c882
 
 // NOSICI: error: invalid operand for instruction
-// VI: v_fract_f64_e64 v[0:1], 0.15915494 ; encoding: [0x00,0x00,0x72,0xd1,0xf8,0x00,0x00,0x00]
+// VI: v_fract_f64_e64 v[0:1], 0.15915494309189532 ; encoding: [0x00,0x00,0x72,0xd1,0xf8,0x00,0x00,0x00]
 v_fract_f64_e64 v[0:1], 0x3fc45f306dc9c882
 
 // NOSICI: error: invalid operand for instruction
@@ -457,7 +457,7 @@ v_trunc_f32_e64 v0, 0x3e22f983
 v_fract_f64_e64 v[0:1], 0x3e22f983
 
 // NOSICI: error: invalid operand for instruction
-// VI: s_mov_b64 s[0:1], 0.15915494 ; encoding: [0xf8,0x01,0x80,0xbe]
+// VI: s_mov_b64 s[0:1], 0.15915494309189532 ; encoding: [0xf8,0x01,0x80,0xbe]
 s_mov_b64_e32 s[0:1], 0.159154943091895317852646485335
 
 // SICI: v_and_b32_e32 v0, 0x3e22f983, v1 ; encoding: [0xff,0x02,0x00,0x36,0x83,0xf9,0x22,0x3e]
@@ -469,7 +469,7 @@ v_and_b32_e32 v0, 0.159154943091895317852646485335, v1
 v_and_b32_e64 v0, 0.159154943091895317852646485335, v1
 
 // SICI: v_fract_f64_e32 v[0:1], 0x3fc45f30 ; encoding: [0xff,0x7c,0x00,0x7e,0x30,0x5f,0xc4,0x3f]
-// VI: v_fract_f64_e32 v[0:1], 0.15915494 ; encoding: [0xf8,0x64,0x00,0x7e]
+// VI: v_fract_f64_e32 v[0:1], 0.15915494309189532 ; encoding: [0xf8,0x64,0x00,0x7e]
 v_fract_f64 v[0:1], 0.159154943091895317852646485335
 
 // SICI: v_trunc_f32_e32 v0, 0x3e22f983 ; encoding: [0xff,0x42,0x00,0x7e,0x83,0xf9,0x22,0x3e]
diff --git a/test/MC/AMDGPU/reloc.s b/test/MC/AMDGPU/reloc.s
index 06c44599acc59c914f231c2a50b95acd9367203f..0a349da49a3c9d4119843b54e4cd648c60308618 100644
--- a/test/MC/AMDGPU/reloc.s
+++ b/test/MC/AMDGPU/reloc.s
@@ -3,14 +3,14 @@
 // CHECK: Relocations [
 // CHECK: .rel.text {
 // CHECK: R_AMDGPU_ABS32_LO SCRATCH_RSRC_DWORD0 0x0
-// CHECK: R_AMDGPU_ABS32_HI SCRATCH_RSRC_DWORD1 0x0
+// CHECK: R_AMDGPU_ABS32_LO SCRATCH_RSRC_DWORD1 0x0
 // CHECK: R_AMDGPU_GOTPCREL global_var0 0x0
 // CHECK: R_AMDGPU_GOTPCREL32_LO global_var1 0x0
 // CHECK: R_AMDGPU_GOTPCREL32_HI global_var2 0x0
 // CHECK: R_AMDGPU_REL32_LO global_var3 0x0
 // CHECK: R_AMDGPU_REL32_HI global_var4 0x0
 // CHECK: R_AMDGPU_ABS32_LO SCRATCH_RSRC_DWORD0 0x0
-// CHECK: R_AMDGPU_ABS32_HI SCRATCH_RSRC_DWORD1 0x0
+// CHECK: R_AMDGPU_ABS32_LO SCRATCH_RSRC_DWORD1 0x0
 // CHECK: R_AMDGPU_GOTPCREL global_var0 0x0
 // CHECK: R_AMDGPU_GOTPCREL32_LO global_var1 0x0
 // CHECK: R_AMDGPU_GOTPCREL32_HI global_var2 0x0
diff --git a/test/MC/AMDGPU/sop2-err.s b/test/MC/AMDGPU/sop2-err.s
index 5115489a7f2765c7c3b34c67545164f43df423de..128a3d7b33ceb566a8ae6ddcbd0c1e680ad17477 100644
--- a/test/MC/AMDGPU/sop2-err.s
+++ b/test/MC/AMDGPU/sop2-err.s
@@ -5,3 +5,9 @@ s_cbranch_g_fork 100, s[6:7]
 
 s_cbranch_g_fork s[6:7], 100
 // GCN: error: invalid operand for instruction
+
+s_and_b32 s2, 0x12345678, 0x12345679
+// GCN: error: only one literal operand is allowed
+
+s_and_b64 s[2:3], 0x12345678, 0x12345679
+// GCN: error: only one literal operand is allowed
diff --git a/test/MC/AMDGPU/sop2.s b/test/MC/AMDGPU/sop2.s
index 7bcbb9720574f34e1870ca66ace0abade16021df..8102b0d7f331162d9427eda20bac81dae58327f5 100644
--- a/test/MC/AMDGPU/sop2.s
+++ b/test/MC/AMDGPU/sop2.s
@@ -50,6 +50,14 @@ s_and_b32 s2, s4, s6
 // SICI: s_and_b32 s2, s4, s6 ; encoding: [0x04,0x06,0x02,0x87]
 // GFX89: s_and_b32 s2, s4, s6 ; encoding: [0x04,0x06,0x02,0x86]
 
+s_and_b32 s2, 1234, 1234
+// SICI: s_and_b32 s2, 0x4d2, 0x4d2 ; encoding: [0xff,0xff,0x02,0x87,0xd2,0x04,0x00,0x00]
+// GFX89: s_and_b32 s2, 0x4d2, 0x4d2 ; encoding: [0xff,0xff,0x02,0x86,0xd2,0x04,0x00,0x00]
+
+s_and_b32 s2, 0xFFFF0000, -65536
+// SICI: s_and_b32 s2, 0xffff0000, 0xffff0000 ; encoding: [0xff,0xff,0x02,0x87,0x00,0x00,0xff,0xff]
+// GFX89: s_and_b32 s2, 0xffff0000, 0xffff0000 ; encoding: [0xff,0xff,0x02,0x86,0x00,0x00,0xff,0xff]
+
 s_and_b64 s[2:3], s[4:5], s[6:7]
 // SICI: s_and_b64 s[2:3], s[4:5], s[6:7] ; encoding: [0x04,0x06,0x82,0x87]
 // GFX89: s_and_b64 s[2:3], s[4:5], s[6:7] ; encoding: [0x04,0x06,0x82,0x86]
@@ -134,6 +142,10 @@ s_ashr_i64 s[2:3], s[4:5], s6
 // SICI: s_ashr_i64 s[2:3], s[4:5], s6 ; encoding: [0x04,0x06,0x82,0x91]
 // GFX89: s_ashr_i64 s[2:3], s[4:5], s6 ; encoding: [0x04,0x06,0x82,0x90]
 
+s_ashr_i64 s[2:3], -65536, 0xFFFF0000
+// SICI: s_ashr_i64 s[2:3], 0xffff0000, 0xffff0000 ; encoding: [0xff,0xff,0x82,0x91,0x00,0x00,0xff,0xff]
+// GFX89: s_ashr_i64 s[2:3], 0xffff0000, 0xffff0000 ; encoding: [0xff,0xff,0x82,0x90,0x00,0x00,0xff,0xff]
+
 s_bfm_b32 s2, s4, s6
 // SICI: s_bfm_b32 s2, s4, s6 ; encoding: [0x04,0x06,0x02,0x92]
 // GFX89: s_bfm_b32 s2, s4, s6 ; encoding: [0x04,0x06,0x02,0x91]
diff --git a/test/MC/AMDGPU/sopc-err.s b/test/MC/AMDGPU/sopc-err.s
index b6d954271d3483899e99914ce28466072ac42a54..10b25445376b1e79e34895338cd7f4460ecdbffb 100644
--- a/test/MC/AMDGPU/sopc-err.s
+++ b/test/MC/AMDGPU/sopc-err.s
@@ -8,3 +8,9 @@ s_set_gpr_idx_on s0, 16
 
 s_set_gpr_idx_on s0, -1
 // GCN: error: invalid operand for instruction
+
+s_cmp_eq_i32 0x12345678, 0x12345679
+// GCN: error: only one literal operand is allowed
+
+s_cmp_eq_u64 0x12345678, 0x12345679
+// GCN: error: only one literal operand is allowed
diff --git a/test/MC/AMDGPU/sopc.s b/test/MC/AMDGPU/sopc.s
index dc922b65a6e9f373c47ece614b04ef30cb1286f4..c7b337d75a533fa971417d55e8f52fc808831101 100644
--- a/test/MC/AMDGPU/sopc.s
+++ b/test/MC/AMDGPU/sopc.s
@@ -9,6 +9,12 @@
 s_cmp_eq_i32 s1, s2
 // GCN: s_cmp_eq_i32 s1, s2 ; encoding: [0x01,0x02,0x00,0xbf]
 
+s_cmp_eq_i32 0xabcd1234, 0xabcd1234
+// GCN: s_cmp_eq_i32 0xabcd1234, 0xabcd1234 ; encoding: [0xff,0xff,0x00,0xbf,0x34,0x12,0xcd,0xab]
+
+s_cmp_eq_i32 0xFFFF0000, -65536
+// GCN: s_cmp_eq_i32 0xffff0000, 0xffff0000 ; encoding: [0xff,0xff,0x00,0xbf,0x00,0x00,0xff,0xff]
+
 s_cmp_lg_i32 s1, s2
 // GCN: s_cmp_lg_i32 s1, s2 ; encoding: [0x01,0x02,0x01,0xbf]
 
diff --git a/test/MC/ARM/armv8.5a-sb-error-thumb.s b/test/MC/ARM/armv8.5a-sb-error-thumb.s
new file mode 100644
index 0000000000000000000000000000000000000000..5f88bf6c06a34326820763e40864a2572aca854a
--- /dev/null
+++ b/test/MC/ARM/armv8.5a-sb-error-thumb.s
@@ -0,0 +1,6 @@
+// RUN: not llvm-mc -triple thumbv8 -show-encoding -mattr=+sb < %s 2>&1 | FileCheck %s
+
+it eq
+sbeq
+
+// CHECK: instruction 'sb' is not predicable, but condition code specified
diff --git a/test/MC/ARM/armv8.5a-sb-error.s b/test/MC/ARM/armv8.5a-sb-error.s
new file mode 100644
index 0000000000000000000000000000000000000000..917b742e8aa0190883d9e66e938a9c5a19456892
--- /dev/null
+++ b/test/MC/ARM/armv8.5a-sb-error.s
@@ -0,0 +1,5 @@
+// RUN: not llvm-mc -triple armv8 -show-encoding -mattr=+sb < %s 2>&1 | FileCheck %s
+
+sbeq
+
+// CHECK: instruction 'sb' is not predicable
diff --git a/test/MC/ARM/armv8.5a-sb.s b/test/MC/ARM/armv8.5a-sb.s
new file mode 100644
index 0000000000000000000000000000000000000000..f0c9ee4de20591c49d806920161a9c60fa24039d
--- /dev/null
+++ b/test/MC/ARM/armv8.5a-sb.s
@@ -0,0 +1,15 @@
+// RUN:     llvm-mc -triple armv8   -show-encoding -mattr=+sb < %s      | FileCheck %s
+// RUN:     llvm-mc -triple armv8   -show-encoding -mattr=+v8.5a    < %s      | FileCheck %s
+// RUN: not llvm-mc -triple armv8   -show-encoding -mattr=-sb < %s 2>&1 | FileCheck %s --check-prefix=NOSB
+// RUN:     llvm-mc -triple thumbv8 -show-encoding -mattr=+sb < %s      | FileCheck %s --check-prefix=THUMB
+// RUN:     llvm-mc -triple thumbv8 -show-encoding -mattr=+v8.5a    < %s      | FileCheck %s --check-prefix=THUMB
+// RUN: not llvm-mc -triple thumbv8 -show-encoding -mattr=-sb < %s 2>&1 | FileCheck %s --check-prefix=NOSB
+
+// Flag manipulation
+sb
+
+// CHECK: sb    @ encoding: [0x70,0xf0,0x7f,0xf5]
+// THUMB: sb    @ encoding: [0xbf,0xf3,0x70,0x8f]
+
+// NOSB: instruction requires: sb
+// NOSB-NEXT: sb
diff --git a/test/MC/ARM/armv8.5a-specctrl-error-thumb.s b/test/MC/ARM/armv8.5a-specctrl-error-thumb.s
deleted file mode 100644
index 359aec960147a401a7b0d1afd9ba9cb8e42b13c9..0000000000000000000000000000000000000000
--- a/test/MC/ARM/armv8.5a-specctrl-error-thumb.s
+++ /dev/null
@@ -1,6 +0,0 @@
-// RUN: not llvm-mc -triple thumbv8 -show-encoding -mattr=+specctrl < %s 2>&1 | FileCheck %s
-
-it eq
-sbeq
-
-// CHECK: instruction 'sb' is not predicable, but condition code specified
diff --git a/test/MC/ARM/armv8.5a-specctrl-error.s b/test/MC/ARM/armv8.5a-specctrl-error.s
deleted file mode 100644
index 5a018df6b0ff2133b7feecc6f3a0dc1d49dc3048..0000000000000000000000000000000000000000
--- a/test/MC/ARM/armv8.5a-specctrl-error.s
+++ /dev/null
@@ -1,5 +0,0 @@
-// RUN: not llvm-mc -triple armv8 -show-encoding -mattr=+specctrl < %s 2>&1 | FileCheck %s
-
-sbeq
-
-// CHECK: instruction 'sb' is not predicable
diff --git a/test/MC/ARM/armv8.5a-specctrl.s b/test/MC/ARM/armv8.5a-specctrl.s
deleted file mode 100644
index 2d799e680ed8bee21d3d3d31633983d29761ac36..0000000000000000000000000000000000000000
--- a/test/MC/ARM/armv8.5a-specctrl.s
+++ /dev/null
@@ -1,15 +0,0 @@
-// RUN:     llvm-mc -triple armv8   -show-encoding -mattr=+specctrl < %s      | FileCheck %s
-// RUN:     llvm-mc -triple armv8   -show-encoding -mattr=+v8.5a    < %s      | FileCheck %s
-// RUN: not llvm-mc -triple armv8   -show-encoding -mattr=-specctrl < %s 2>&1 | FileCheck %s --check-prefix=NOSB
-// RUN:     llvm-mc -triple thumbv8 -show-encoding -mattr=+specctrl < %s      | FileCheck %s --check-prefix=THUMB
-// RUN:     llvm-mc -triple thumbv8 -show-encoding -mattr=+v8.5a    < %s      | FileCheck %s --check-prefix=THUMB
-// RUN: not llvm-mc -triple thumbv8 -show-encoding -mattr=-specctrl < %s 2>&1 | FileCheck %s --check-prefix=NOSB
-
-// Flag manipulation
-sb
-
-// CHECK: sb    @ encoding: [0x70,0xf0,0x7f,0xf5]
-// THUMB: sb    @ encoding: [0xbf,0xf3,0x70,0x8f]
-
-// NOSB: instruction requires: specctrl
-// NOSB-NEXT: sb
diff --git a/test/MC/AVR/inst-cbr.s b/test/MC/AVR/inst-cbr.s
index 012683e66dd1b7ba3511385e7dda07a011e9a002..f85ae87dff0ba5df4c98e2e59262d99013e67ce9 100644
--- a/test/MC/AVR/inst-cbr.s
+++ b/test/MC/AVR/inst-cbr.s
@@ -8,7 +8,7 @@ foo:
   cbr r20, 173
   cbr r31, 0
 
-; CHECK: cbr r17, 208                 ; encoding: [0x1f,0x72]
-; CHECK: cbr r24, 190                 ; encoding: [0x81,0x74]
-; CHECK: cbr r20, 173                 ; encoding: [0x42,0x75]
-; CHECK: cbr r31, 0                   ; encoding: [0xff,0x7f]
+; CHECK: andi r17, -209              ; encoding: [0x1f,0x72]
+; CHECK: andi r24, -191              ; encoding: [0x81,0x74]
+; CHECK: andi r20, -174              ; encoding: [0x42,0x75]
+; CHECK: andi r31, -1                ; encoding: [0xff,0x7f]
diff --git a/test/MC/AsmParser/debug-only-comments.s b/test/MC/AsmParser/debug-only-comments.s
new file mode 100644
index 0000000000000000000000000000000000000000..0330653498b412fc9e31861b0f0c1aa03a2318bd
--- /dev/null
+++ b/test/MC/AsmParser/debug-only-comments.s
@@ -0,0 +1,6 @@
+        # RUN: llvm-mc -triple i386-linux-gnu -g -dwarf-version 4 < %s | FileCheck %s
+        # RUN: llvm-mc -triple i386-linux-gnu -g -dwarf-version 5 < %s | FileCheck %s
+        # CHECK: .section .debug_info
+        # CHECK: .section .debug_info
+        # CHECK-NOT: .section
+        # CHECK: .ascii "<stdin>"
diff --git a/test/MC/AsmParser/ifdef.s b/test/MC/AsmParser/ifdef.s
index 98bff6525b2b62f240ca08e1fe8d00b9a652490c..55fb666fe638002dd18a5da6ed65860d5b1ece0b 100644
--- a/test/MC/AsmParser/ifdef.s
+++ b/test/MC/AsmParser/ifdef.s
@@ -27,3 +27,17 @@ defined:
 .else
 	.byte 1
 .endif
+
+# .ifdef directive does not count as a use, so ensure redefinition is permitted
+.set var, 1
+.ifdef var
+	.set var, 2
+.else
+	.set var, 3
+.endif
+# CHECK: .byte 2
+.byte var
+
+.set var, 4
+# CHECK: .byte 4
+.byte var
diff --git a/test/MC/AsmParser/ifndef.s b/test/MC/AsmParser/ifndef.s
index d9c9c5457a700b769ed0b7f2ee36d1ed09370bc6..c20c8c22b773ba832978cdc008f0ab535420de94 100644
--- a/test/MC/AsmParser/ifndef.s
+++ b/test/MC/AsmParser/ifndef.s
@@ -27,3 +27,17 @@ defined:
 .else
 	.byte 0
 .endif
+
+# .ifndef directive does not count as a use, so ensure redefinition is permitted
+.set var, 1
+.ifndef var
+	.set var, 2
+.else
+	.set var, 3
+.endif
+# CHECK: .byte 3
+.byte var
+
+.set var, 4
+# CHECK: .byte 4
+.byte var
diff --git a/test/MC/AsmParser/negativ_altmacro_expression.s b/test/MC/AsmParser/negativ_altmacro_expression.s
index fabf46c47ef1f8e8d203c0a09c32d2b2bfac8d21..be8c66eefcc9f140f0912f053cc06ff8aa2e904b 100644
--- a/test/MC/AsmParser/negativ_altmacro_expression.s
+++ b/test/MC/AsmParser/negativ_altmacro_expression.s
@@ -6,7 +6,7 @@
 # .noaltmacro returns the format into a regular macro handling.
 # The default mode is ".noaltmacro" as first test checks.
 
-# CHECK:  error: unknown token in expression
+# CHECK:  error: expected immediate expression
 # CHECK-NEXT: addl $%(1%4), %eax
 .macro inner_percent arg
     addl $\arg, %eax
@@ -18,12 +18,12 @@ inner_percent %(1%4)
 .noaltmacro
 
 # CHECK: multi_args_macro %(1+4-5) 1 %2+1
-# CHECK: error: unknown token in expression
+# CHECK: error: expected immediate expression
 # CHECK-NEXT: addl $%(1+4-5), %eax
 
 
 # CHECK: multi_args_macro %(1+4-5),1,%4%10
-# CHECK: error: unknown token in expression
+# CHECK: error: expected immediate expression
 # CHECK-NEXT: addl $%(1+4-5), %eax
 .macro multi_args_macro arg1 arg2 arg3
   label\arg1\arg2\arg3:
diff --git a/test/MC/BPF/insn-unit-32.s b/test/MC/BPF/insn-unit-32.s
index 95e087914db2b64b91a689c42805ceb34eeb669a..9d09bd29135e1534e801dfe7c32901c1290fa682 100644
--- a/test/MC/BPF/insn-unit-32.s
+++ b/test/MC/BPF/insn-unit-32.s
@@ -54,3 +54,50 @@
 // CHECK: b4 09 00 00 01 00 00 00      w9 = 1
 // CHECK: b4 09 00 00 ff ff ff ff      w9 = -1
 // CHECK: c4 0a 00 00 40 00 00 00      w10 s>>= 64
+
+  if w0 == w1 goto Llabel0   // BPF_JEQ  | BPF_X
+  if w3 != w4 goto Llabel0   // BPF_JNE  | BPF_X
+// CHECK: 1e 10 0b 00 00 00 00 00 	if w0 == w1 goto +11
+// CHECK: 5e 43 0a 00 00 00 00 00 	if w3 != w4 goto +10
+
+  if w1 > w2 goto Llabel0    // BPF_JGT  | BPF_X
+  if w2 >= w3 goto Llabel0   // BPF_JGE  | BPF_X
+  if w4 s> w5 goto Llabel0   // BPF_JSGT | BPF_X
+  if w5 s>= w6 goto Llabel0  // BPF_JSGE | BPF_X
+// CHECK: 2e 21 09 00 00 00 00 00 	if w1 > w2 goto +9
+// CHECK: 3e 32 08 00 00 00 00 00 	if w2 >= w3 goto +8
+// CHECK: 6e 54 07 00 00 00 00 00 	if w4 s> w5 goto +7
+// CHECK: 7e 65 06 00 00 00 00 00 	if w5 s>= w6 goto +6
+
+  if w6 < w7 goto Llabel0    // BPF_JLT  | BPF_X
+  if w7 <= w8 goto Llabel0   // BPF_JLE  | BPF_X
+  if w8 s< w9 goto Llabel0   // BPF_JSLT | BPF_X
+  if w9 s<= w10 goto Llabel0 // BPF_JSLE | BPF_X
+// CHECK: ae 76 05 00 00 00 00 00 	if w6 < w7 goto +5
+// CHECK: be 87 04 00 00 00 00 00 	if w7 <= w8 goto +4
+// CHECK: ce 98 03 00 00 00 00 00 	if w8 s< w9 goto +3
+// CHECK: de a9 02 00 00 00 00 00 	if w9 s<= w10 goto +2
+
+  if w0 == 0 goto Llabel0           // BPF_JEQ  | BPF_K
+  if w3 != -1 goto Llabel0          // BPF_JNE  | BPF_K
+// CHECK: 16 00 01 00 00 00 00 00 	if w0 == 0 goto +1
+// CHECK: 56 03 00 00 ff ff ff ff 	if w3 != -1 goto +0
+
+Llabel0 :
+  if w1 > 64 goto Llabel0           // BPF_JGT  | BPF_K
+  if w2 >= 0xffffffff goto Llabel0  // BPF_JGE  | BPF_K
+  if w4 s> 0xffffffff goto Llabel0  // BPF_JSGT | BPF_K
+  if w5 s>= 0x7fffffff goto Llabel0 // BPF_JSGE | BPF_K
+// CHECK: 26 01 ff ff 40 00 00 00 	if w1 > 64 goto -1
+// CHECK: 36 02 fe ff ff ff ff ff 	if w2 >= -1 goto -2
+// CHECK: 66 04 fd ff ff ff ff ff 	if w4 s> -1 goto -3
+// CHECK: 76 05 fc ff ff ff ff 7f 	if w5 s>= 2147483647 goto -4
+
+  if w6 < 0xff goto Llabel0         // BPF_JLT  | BPF_K
+  if w7 <= 0xffff goto Llabel0      // BPF_JLE  | BPF_K
+  if w8 s< 0 goto Llabel0           // BPF_JSLT | BPF_K
+  if w9 s<= -1 goto Llabel0         // BPF_JSLE | BPF_K
+// CHECK: a6 06 fb ff ff 00 00 00 	if w6 < 255 goto -5
+// CHECK: b6 07 fa ff ff ff 00 00 	if w7 <= 65535 goto -6
+// CHECK: c6 08 f9 ff 00 00 00 00 	if w8 s< 0 goto -7
+// CHECK: d6 09 f8 ff ff ff ff ff 	if w9 s<= -1 goto -8
diff --git a/test/MC/COFF/assoc-private.s b/test/MC/COFF/assoc-private.s
index 30e49bc1d167db3a7b30f5b878fb36707168e1d7..b85b73bc2340b6836e404de97047e4287d3341c3 100644
--- a/test/MC/COFF/assoc-private.s
+++ b/test/MC/COFF/assoc-private.s
@@ -5,7 +5,7 @@
 # it anyway.
 
 # CHECK: Sections:
-# CHECK: Idx Name          Size      Address          Type
+# CHECK: Idx Name          Size     VMA              Type
 # CHECK:   3 .rdata        00000004 0000000000000000 DATA
 # CHECK:   4 .CRT$XCU      00000008 0000000000000000 DATA
 # CHECK: SYMBOL TABLE:
diff --git a/test/MC/COFF/bigobj.py b/test/MC/COFF/bigobj.py
index 2d610738b9da87e87b13dc3297198c8e1266a3d9..7908fb3364f621064e1037dcfd71331b065775e4 100644
--- a/test/MC/COFF/bigobj.py
+++ b/test/MC/COFF/bigobj.py
@@ -1,5 +1,7 @@
 # RUN: python %s | llvm-mc -filetype=obj -triple i686-pc-win32 - | llvm-readobj -h | FileCheck %s
 
+from __future__ import print_function
+
 # This test checks that the COFF object emitter can produce objects with
 # more than 65279 sections.
 
diff --git a/test/MC/COFF/cross-section-relative.s b/test/MC/COFF/cross-section-relative.s
index 6f172922e9b8ed9d86b89cf67503a459a337cd49..a9b7e6dbd5de5cb176b93420a64ebbd0ce9464a5 100644
--- a/test/MC/COFF/cross-section-relative.s
+++ b/test/MC/COFF/cross-section-relative.s
@@ -87,21 +87,25 @@ t6:
 // READOBJ-NEXT:      Offset: 0x0
 // READOBJ-NEXT:      Type: IMAGE_REL_AMD64_REL32 (4)
 // READOBJ-NEXT:      Symbol: g3
+// READOBJ-NEXT:      SymbolIndex: 12
 // READOBJ-NEXT:    }
 // READOBJ-NEXT:    Relocation {
 // READOBJ-NEXT:      Offset: 0x4
 // READOBJ-NEXT:      Type: IMAGE_REL_AMD64_ADDR32NB (3)
 // READOBJ-NEXT:      Symbol: g3
+// READOBJ-NEXT:      SymbolIndex: 12
 // READOBJ-NEXT:    }
 // READOBJ-NEXT:    Relocation {
 // READOBJ-NEXT:      Offset: 0x20
 // READOBJ-NEXT:      Type: IMAGE_REL_AMD64_REL32 (4)
 // READOBJ-NEXT:      Symbol: g3
+// READOBJ-NEXT:      SymbolIndex: 12
 // READOBJ-NEXT:    }
 // READOBJ-NEXT:    Relocation {
 // READOBJ-NEXT:      Offset: 0x28
 // READOBJ-NEXT:      Type: IMAGE_REL_AMD64_REL32 (4)
 // READOBJ-NEXT:      Symbol: foobar
+// READOBJ-NEXT:      SymbolIndex: 20
 // READOBJ-NEXT:    }
 // READOBJ-NEXT:  }
 // READOBJ-NEXT:]
diff --git a/test/MC/COFF/file.s b/test/MC/COFF/file.s
index a18a1f476acaa0ff444ba85b01f24c8733647494..dbe3b024f6c84680301791cfafedff713bae058e 100644
--- a/test/MC/COFF/file.s
+++ b/test/MC/COFF/file.s
@@ -1,6 +1,11 @@
 // RUN: llvm-mc -triple i686-windows -filetype obj %s -o - | llvm-objdump -t - \
 // RUN:   | FileCheck %s
 
+// Round trip through .s output to exercise MCAsmStreamer.
+// RUN: llvm-mc -triple i686-windows %s -o - \
+// RUN:   | llvm-mc -triple i686-windows - -filetype=obj -o - | llvm-objdump -t - \
+// RUN:   | FileCheck %s
+
 // RUN: llvm-mc -triple i686-windows -filetype obj %s -o - \
 // RUN:	  | llvm-readobj -symbols | FileCheck %s -check-prefix CHECK-SCN
 
diff --git a/test/MC/Disassembler/AArch64/armv8.5a-predctrl.txt b/test/MC/Disassembler/AArch64/armv8.5a-predres.txt
similarity index 62%
rename from test/MC/Disassembler/AArch64/armv8.5a-predctrl.txt
rename to test/MC/Disassembler/AArch64/armv8.5a-predres.txt
index ecfdeec86f115265cfaf415d759c572d98df3837..5d4e0731c854f5751fa4bfa9664c510f28a022ab 100644
--- a/test/MC/Disassembler/AArch64/armv8.5a-predctrl.txt
+++ b/test/MC/Disassembler/AArch64/armv8.5a-predres.txt
@@ -1,6 +1,6 @@
-# RUN: llvm-mc -triple=aarch64 -mattr=+predctrl -disassemble < %s      | FileCheck %s
+# RUN: llvm-mc -triple=aarch64 -mattr=+predres -disassemble < %s      | FileCheck %s
 # RUN: llvm-mc -triple=aarch64 -mattr=+v8.5a    -disassemble < %s      | FileCheck %s
-# RUN: llvm-mc -triple=aarch64 -mattr=-predctrl -disassemble < %s 2>&1 | FileCheck %s --check-prefix=NOSB
+# RUN: llvm-mc -triple=aarch64 -mattr=-predres -disassemble < %s 2>&1 | FileCheck %s --check-prefix=NOSB
 
 [0x80 0x73 0x0b 0xd5]
 [0xa1 0x73 0x0b 0xd5]
diff --git a/test/MC/Disassembler/AArch64/armv8.5a-sb.txt b/test/MC/Disassembler/AArch64/armv8.5a-sb.txt
new file mode 100644
index 0000000000000000000000000000000000000000..25f32d45d625d3aec831afea1d2052599353905d
--- /dev/null
+++ b/test/MC/Disassembler/AArch64/armv8.5a-sb.txt
@@ -0,0 +1,9 @@
+# RUN: llvm-mc -triple=aarch64-none-linux-gnu -mattr=+sb -disassemble < %s      | FileCheck %s
+# RUN: llvm-mc -triple=aarch64-none-linux-gnu -mattr=+v8.5a    -disassemble < %s      | FileCheck %s
+# RUN: llvm-mc -triple=aarch64-none-linux-gnu -mattr=-sb -disassemble < %s 2>&1 | FileCheck %s --check-prefix=NOSB
+
+# New reg
+0xff 0x30 0x03 0xd5
+
+# CHECK: sb
+# NOSB:  msr S0_3_C3_C0_7, xzr
diff --git a/test/MC/Disassembler/AArch64/armv8.5a-specctrl.txt b/test/MC/Disassembler/AArch64/armv8.5a-specctrl.txt
deleted file mode 100644
index d2174ae1dc11d617193b9397f66c3f7f62e6283b..0000000000000000000000000000000000000000
--- a/test/MC/Disassembler/AArch64/armv8.5a-specctrl.txt
+++ /dev/null
@@ -1,9 +0,0 @@
-# RUN: llvm-mc -triple=aarch64-none-linux-gnu -mattr=+specctrl -disassemble < %s      | FileCheck %s
-# RUN: llvm-mc -triple=aarch64-none-linux-gnu -mattr=+v8.5a    -disassemble < %s      | FileCheck %s
-# RUN: llvm-mc -triple=aarch64-none-linux-gnu -mattr=-specctrl -disassemble < %s 2>&1 | FileCheck %s --check-prefix=NOSB
-
-# New reg
-0xff 0x30 0x03 0xd5
-
-# CHECK: sb
-# NOSB:  msr S0_3_C3_C0_7, xzr
diff --git a/test/MC/Disassembler/AMDGPU/lds_direct_gfx9.txt b/test/MC/Disassembler/AMDGPU/lds_direct_gfx9.txt
new file mode 100644
index 0000000000000000000000000000000000000000..3480d6f2a6837ec279078a6c0be1183541f3e24a
--- /dev/null
+++ b/test/MC/Disassembler/AMDGPU/lds_direct_gfx9.txt
@@ -0,0 +1,19 @@
+# RUN: llvm-mc -arch=amdgcn -mcpu=gfx900 -disassemble -show-encoding < %s | FileCheck %s --check-prefix=GFX9
+
+# GFX9: v_mov_b32_e32 v0, src_lds_direct ; encoding: [0xfe,0x02,0x00,0x7e]
+0xfe,0x02,0x00,0x7e
+
+# GFX9: v_mov_b32_e64 v0, src_lds_direct ; encoding: [0x00,0x00,0x41,0xd1,0xfe,0x00,0x00,0x00]
+0x00,0x00,0x41,0xd1,0xfe,0x00,0x00,0x00
+
+# GFX9: v_add_f32_e32 v0, src_lds_direct, v0 ; encoding: [0xfe,0x00,0x00,0x02]
+0xfe,0x00,0x00,0x02
+
+# GFX9: v_pk_mad_i16 v0, src_lds_direct, v0, v0 ; encoding: [0x00,0x40,0x80,0xd3,0xfe,0x00,0x02,0x1c]
+0x00,0x40,0x80,0xd3,0xfe,0x00,0x02,0x1c
+
+# GFX9: v_pk_mul_lo_u16 v0, src_lds_direct, v0 ; encoding: [0x00,0x00,0x81,0xd3,0xfe,0x00,0x02,0x18]
+0x00,0x00,0x81,0xd3,0xfe,0x00,0x02,0x18
+
+# GFX9: v_cmpx_le_i32_e32 vcc, src_lds_direct, v0 ; encoding: [0xfe,0x00,0xa6,0x7d]
+0xfe,0x00,0xa6,0x7d
diff --git a/test/MC/Disassembler/AMDGPU/literal_vi.txt b/test/MC/Disassembler/AMDGPU/literal_vi.txt
new file mode 100644
index 0000000000000000000000000000000000000000..05483ccd0008d9b08f5f77ecbf46c662a89ea7f8
--- /dev/null
+++ b/test/MC/Disassembler/AMDGPU/literal_vi.txt
@@ -0,0 +1,7 @@
+# RUN: llvm-mc -arch=amdgcn -mcpu=tonga -disassemble -show-encoding %s | FileCheck -check-prefix=VI %s
+
+# VI: v_fract_f64_e32 v[0:1], 0.15915494309189532 ; encoding: [0xf8,0x64,0x00,0x7e]
+0xf8,0x64,0x00,0x7e
+
+# VI: v_trunc_f32_e32 v0, 0.15915494 ; encoding: [0xf8,0x38,0x00,0x7e]
+0xf8,0x38,0x00,0x7e
diff --git a/test/MC/Disassembler/ARM/armv8.5a-sb-thumb.txt b/test/MC/Disassembler/ARM/armv8.5a-sb-thumb.txt
new file mode 100644
index 0000000000000000000000000000000000000000..f2389f1c905953b9a07411c9c1338fd3b15a75e1
--- /dev/null
+++ b/test/MC/Disassembler/ARM/armv8.5a-sb-thumb.txt
@@ -0,0 +1,9 @@
+# RUN: llvm-mc -triple=thumbv8 -mattr=+sb -disassemble < %s      | FileCheck %s
+# RUN: llvm-mc -triple=thumbv8 -mattr=+v8.5a    -disassemble < %s      | FileCheck %s
+# RUN: llvm-mc -triple=thumbv8 -mattr=-sb -disassemble < %s 2>&1 | FileCheck %s --check-prefix=NOSB
+
+0xbf 0xf3 0x70 0x8f
+
+# CHECK: sb
+# NOSB: invalid instruction encoding
+# NOSB-NEXT: 0xbf 0xf3 0x70 0x8f
diff --git a/test/MC/Disassembler/ARM/armv8.5a-specctrl.txt b/test/MC/Disassembler/ARM/armv8.5a-sb.txt
similarity index 50%
rename from test/MC/Disassembler/ARM/armv8.5a-specctrl.txt
rename to test/MC/Disassembler/ARM/armv8.5a-sb.txt
index f9d8b5397ad7f08a388bf5d89354f5d0cef1f672..40c9478effc757f1d4781c2bd63aa99a0548c12b 100644
--- a/test/MC/Disassembler/ARM/armv8.5a-specctrl.txt
+++ b/test/MC/Disassembler/ARM/armv8.5a-sb.txt
@@ -1,6 +1,6 @@
-# RUN: llvm-mc -triple=armv8 -mattr=+specctrl -disassemble < %s      | FileCheck %s
+# RUN: llvm-mc -triple=armv8 -mattr=+sb -disassemble < %s      | FileCheck %s
 # RUN: llvm-mc -triple=armv8 -mattr=+v8.5a    -disassemble < %s      | FileCheck %s
-# RUN: llvm-mc -triple=armv8 -mattr=-specctrl -disassemble < %s 2>&1 | FileCheck %s --check-prefix=NOSB
+# RUN: llvm-mc -triple=armv8 -mattr=-sb -disassemble < %s 2>&1 | FileCheck %s --check-prefix=NOSB
 
 0x70 0xf0 0x7f 0xf5
 
diff --git a/test/MC/Disassembler/ARM/armv8.5a-specctrl-thumb.txt b/test/MC/Disassembler/ARM/armv8.5a-specctrl-thumb.txt
deleted file mode 100644
index 5703408b13027ecdaf6c7d96039e052ba4bf4db4..0000000000000000000000000000000000000000
--- a/test/MC/Disassembler/ARM/armv8.5a-specctrl-thumb.txt
+++ /dev/null
@@ -1,9 +0,0 @@
-# RUN: llvm-mc -triple=thumbv8 -mattr=+specctrl -disassemble < %s      | FileCheck %s
-# RUN: llvm-mc -triple=thumbv8 -mattr=+v8.5a    -disassemble < %s      | FileCheck %s
-# RUN: llvm-mc -triple=thumbv8 -mattr=-specctrl -disassemble < %s 2>&1 | FileCheck %s --check-prefix=NOSB
-
-0xbf 0xf3 0x70 0x8f
-
-# CHECK: sb
-# NOSB: invalid instruction encoding
-# NOSB-NEXT: 0xbf 0xf3 0x70 0x8f
diff --git a/test/MC/Disassembler/MSP430/msp430.txt b/test/MC/Disassembler/MSP430/msp430.txt
index c7d6ff576da8bea860f92640c67128b1f5eb7f7a..8e06f9de6afd038e943b3a56a0a8abdc0892da07 100644
--- a/test/MC/Disassembler/MSP430/msp430.txt
+++ b/test/MC/Disassembler/MSP430/msp430.txt
@@ -19,9 +19,10 @@
 0x1f 0x40 0x2a 0x00           # CHECK: mov 42, r15
 0xb0 0x12 0x81 0x01           # CHECK: call #385
 0x97 0x12 0x06 0x00           # CHECK: call 6(r7)
-0xa7 0xb2 0x02 0x00           # CHECK: bit #34, 2(r7)
+0xa7 0xb2 0x02 0x00           # CHECK: bit #4, 2(r7)
 0xa9 0x57 0x08 0x00           # CHECK: add @r7, 8(r9)
 0xb7 0xe7 0xfe 0xff           # CHECK: xor @r7+, -2(r7)
 
 0xbf 0x40 0x2a 0x00 0x0c 0x00 # CHECK: mov #42, 12(r15)
+0xb7 0xb0 0x22 0x00 0x02 0x00 # CHECK: bit #34, 2(r7)
 0x9a 0xb9 0x10 0x00 0x08 0x00 # CHECK: bit 16(r9), 8(r10)
diff --git a/test/MC/Disassembler/MSP430/unknown.txt b/test/MC/Disassembler/MSP430/unknown.txt
new file mode 100644
index 0000000000000000000000000000000000000000..d7e680b98aea5cd954d8248c9af4ad2d90b60ea9
--- /dev/null
+++ b/test/MC/Disassembler/MSP430/unknown.txt
@@ -0,0 +1,13 @@
+# RUN: not llvm-mc -disassemble -triple=msp430 %s 2>&1 | FileCheck %s
+
+# This should not decode as 'and.b @r15+, (0)r1' [0xf1,0xff,0x00,0x00]
+[0xf1 0xff]
+# CHECK: warning: invalid instruction encoding
+
+# This should not decode as 'add 6(r7), 6(r5)' [0x95 0x57 0x06 0x00 0x06 0x00]
+[0x95 0x57 0x06 0x00]
+# CHECK: warning: invalid instruction encoding
+
+# This should not decode as 'call 6(r7)' [0x97 0x12 0x06 0x00]
+[0x97 0x12]
+# CHECK: warning: invalid instruction encoding
diff --git a/test/MC/Disassembler/WebAssembly/wasm-error.txt b/test/MC/Disassembler/WebAssembly/wasm-error.txt
new file mode 100644
index 0000000000000000000000000000000000000000..6468a4f0c1897dbc9cab848e24870096c5fd6b33
--- /dev/null
+++ b/test/MC/Disassembler/WebAssembly/wasm-error.txt
@@ -0,0 +1,9 @@
+# RUN: llvm-mc --disassemble %s -triple=wasm32-unknown-unknown | FileCheck %s
+
+# CHECK: .text
+
+# CHECK: block invalid_type
+0x02 0x00
+
+# CHECK: br 16 # Invalid depth argument!
+0x0C 0x10
diff --git a/test/MC/Disassembler/WebAssembly/wasm.txt b/test/MC/Disassembler/WebAssembly/wasm.txt
index fd21db9ad471564664ccd7a3a058c37e07e8ba52..8a119fb6b0f4f005611ba924ee72edc405d21e87 100644
--- a/test/MC/Disassembler/WebAssembly/wasm.txt
+++ b/test/MC/Disassembler/WebAssembly/wasm.txt
@@ -14,25 +14,26 @@
 # CHECK: i64.load32_u 16:p2align=1
 0x35 0x01 0x10
 
-# CHECK: block
-# 3
-# FIXME: WebAssemblyInstPrinter does not currently print block number.
-0x02 0x03
+# CHECK: block f64
+0x02 0x7C
 
 # CHECK: call_indirect
 # $0=, 128, 0
 # FIXME: WebAssemblyInstPrinter does not print immediates.
 0x11 0x80 0x01 0x00
 
-# CHECK: get_local 128
+# CHECK: call 0
+# CHECK-NOT: except_ref.call 0
+0x10 0x00
+
+# CHECK: local.get 128
 0x20 0x80 0x01
 
 # Prefix byte example:
-# CHECK: i64.trunc_u:sat/f64
+# CHECK: i64.trunc_sat_f64_u
 0xFC 0x07
 
-# v128.const is arbitrarily disassembled as v16i8
-# CHECK: v128.const 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
+# CHECK: v128.const 50462976, 117835012, 185207048, 252579084
 0xFD 0x02 0x00 0x01 0x02 0x03 0x04 0x05 0x06 0x07 0x08 0x09 0x0A 0x0B 0x0C 0x0D 0x0E 0x0F
 
 # CHECK: v8x16.shuffle 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
@@ -46,3 +47,11 @@
 # CHECK: i64x2.any_true
 # CHECK-NOT: i64.div_u
 0xFD 0x85 0x81 0x80 0x80 0x80 0x80 0x00
+
+# Check br_table, which has its own operand type.
+# CHECK: br_table {0, 1, 2}
+0x0E 0x02 0x00 0x01 0x02
+
+# This can mean end_block/end_loop/end_if/end_function/end_try..
+# CHECK: end
+0x0B
diff --git a/test/MC/Disassembler/X86/fp-stack.txt b/test/MC/Disassembler/X86/fp-stack.txt
index 8c4ad47eb8732a0ccefe8e52fd4ef87d71168725..1b1687b2a204b6a04476c1eab0ae69fe446a0926 100644
--- a/test/MC/Disassembler/X86/fp-stack.txt
+++ b/test/MC/Disassembler/X86/fp-stack.txt
@@ -1,52 +1,52 @@
 # RUN: llvm-mc --disassemble %s -triple=x86_64 | FileCheck %s
 # RUN: llvm-mc --disassemble %s -triple=i686-apple-darwin9 | FileCheck %s
 
-# CHECK: fadd %st(0)
+# CHECK: fadd %st(0), %st
 0xd8,0xc0
 
-# CHECK: fadd %st(1)
+# CHECK: fadd %st(1), %st
 0xd8,0xc1
 
-# CHECK: fadd %st(2)
+# CHECK: fadd %st(2), %st
 0xd8,0xc2
 
-# CHECK: fadd %st(3)
+# CHECK: fadd %st(3), %st
 0xd8,0xc3
 
-# CHECK: fadd %st(4)
+# CHECK: fadd %st(4), %st
 0xd8,0xc4
 
-# CHECK: fadd %st(5)
+# CHECK: fadd %st(5), %st
 0xd8,0xc5
 
-# CHECK: fadd %st(6)
+# CHECK: fadd %st(6), %st
 0xd8,0xc6
 
-# CHECK: fadd %st(7)
+# CHECK: fadd %st(7), %st
 0xd8,0xc7
 
-# CHECK: fmul %st(0)
+# CHECK: fmul %st(0), %st
 0xd8,0xc8
 
-# CHECK: fmul %st(1)
+# CHECK: fmul %st(1), %st
 0xd8,0xc9
 
-# CHECK: fmul %st(2)
+# CHECK: fmul %st(2), %st
 0xd8,0xca
 
-# CHECK: fmul %st(3)
+# CHECK: fmul %st(3), %st
 0xd8,0xcb
 
-# CHECK: fmul %st(4)
+# CHECK: fmul %st(4), %st
 0xd8,0xcc
 
-# CHECK: fmul %st(5)
+# CHECK: fmul %st(5), %st
 0xd8,0xcd
 
-# CHECK: fmul %st(6)
+# CHECK: fmul %st(6), %st
 0xd8,0xce
 
-# CHECK: fmul %st(7)
+# CHECK: fmul %st(7), %st
 0xd8,0xcf
 
 # CHECK: fcom %st(0)
@@ -97,100 +97,100 @@
 # CHECK: fcomp %st(7)
 0xd8,0xdf
 
-# CHECK: fsub %st(0)
+# CHECK: fsub %st(0), %st
 0xd8,0xe0
 
-# CHECK: fsub %st(1)
+# CHECK: fsub %st(1), %st
 0xd8,0xe1
 
-# CHECK: fsub %st(2)
+# CHECK: fsub %st(2), %st
 0xd8,0xe2
 
-# CHECK: fsub %st(3)
+# CHECK: fsub %st(3), %st
 0xd8,0xe3
 
-# CHECK: fsub %st(4)
+# CHECK: fsub %st(4), %st
 0xd8,0xe4
 
-# CHECK: fsub %st(5)
+# CHECK: fsub %st(5), %st
 0xd8,0xe5
 
-# CHECK: fsub %st(6)
+# CHECK: fsub %st(6), %st
 0xd8,0xe6
 
-# CHECK: fsub %st(7)
+# CHECK: fsub %st(7), %st
 0xd8,0xe7
 
-# CHECK: fsubr %st(0)
+# CHECK: fsubr %st(0), %st
 0xd8,0xe8
 
-# CHECK: fsubr %st(1)
+# CHECK: fsubr %st(1), %st
 0xd8,0xe9
 
-# CHECK: fsubr %st(2)
+# CHECK: fsubr %st(2), %st
 0xd8,0xea
 
-# CHECK: fsubr %st(3)
+# CHECK: fsubr %st(3), %st
 0xd8,0xeb
 
-# CHECK: fsubr %st(4)
+# CHECK: fsubr %st(4), %st
 0xd8,0xec
 
-# CHECK: fsubr %st(5)
+# CHECK: fsubr %st(5), %st
 0xd8,0xed
 
-# CHECK: fsubr %st(6)
+# CHECK: fsubr %st(6), %st
 0xd8,0xee
 
-# CHECK: fsubr %st(7)
+# CHECK: fsubr %st(7), %st
 0xd8,0xef
 
-# CHECK: fdiv %st(0)
+# CHECK: fdiv %st(0), %st
 0xd8,0xf0
 
-# CHECK: fdiv %st(1)
+# CHECK: fdiv %st(1), %st
 0xd8,0xf1
 
-# CHECK: fdiv %st(2)
+# CHECK: fdiv %st(2), %st
 0xd8,0xf2
 
-# CHECK: fdiv %st(3)
+# CHECK: fdiv %st(3), %st
 0xd8,0xf3
 
-# CHECK: fdiv %st(4)
+# CHECK: fdiv %st(4), %st
 0xd8,0xf4
 
-# CHECK: fdiv %st(5)
+# CHECK: fdiv %st(5), %st
 0xd8,0xf5
 
-# CHECK: fdiv %st(6)
+# CHECK: fdiv %st(6), %st
 0xd8,0xf6
 
-# CHECK: fdiv %st(7)
+# CHECK: fdiv %st(7), %st
 0xd8,0xf7
 
-# CHECK: fdivr %st(0)
+# CHECK: fdivr %st(0), %st
 0xd8,0xf8
 
-# CHECK: fdivr %st(1)
+# CHECK: fdivr %st(1), %st
 0xd8,0xf9
 
-# CHECK: fdivr %st(2)
+# CHECK: fdivr %st(2), %st
 0xd8,0xfa
 
-# CHECK: fdivr %st(3)
+# CHECK: fdivr %st(3), %st
 0xd8,0xfb
 
-# CHECK: fdivr %st(4)
+# CHECK: fdivr %st(4), %st
 0xd8,0xfc
 
-# CHECK: fdivr %st(5)
+# CHECK: fdivr %st(5), %st
 0xd8,0xfd
 
-# CHECK: fdivr %st(6)
+# CHECK: fdivr %st(6), %st
 0xd8,0xfe
 
-# CHECK: fdivr %st(7)
+# CHECK: fdivr %st(7), %st
 0xd8,0xff
 
 # CHECK: fld %st(0)
@@ -325,199 +325,199 @@
 # CHECK: fcos
 0xd9,0xff
 
-# CHECK: fcmovb %st(0), %st(0)
+# CHECK: fcmovb %st(0), %st
 0xda,0xc0
 
-# CHECK: fcmovb %st(1), %st(0)
+# CHECK: fcmovb %st(1), %st
 0xda,0xc1
 
-# CHECK: fcmovb %st(2), %st(0)
+# CHECK: fcmovb %st(2), %st
 0xda,0xc2
 
-# CHECK: fcmovb %st(3), %st(0)
+# CHECK: fcmovb %st(3), %st
 0xda,0xc3
 
-# CHECK: fcmovb %st(4), %st(0)
+# CHECK: fcmovb %st(4), %st
 0xda,0xc4
 
-# CHECK: fcmovb %st(5), %st(0)
+# CHECK: fcmovb %st(5), %st
 0xda,0xc5
 
-# CHECK: fcmovb %st(6), %st(0)
+# CHECK: fcmovb %st(6), %st
 0xda,0xc6
 
-# CHECK: fcmovb %st(7), %st(0)
+# CHECK: fcmovb %st(7), %st
 0xda,0xc7
 
-# CHECK: fcmove %st(0), %st(0)
+# CHECK: fcmove %st(0), %st
 0xda,0xc8
 
-# CHECK: fcmove %st(1), %st(0)
+# CHECK: fcmove %st(1), %st
 0xda,0xc9
 
-# CHECK: fcmove %st(2), %st(0)
+# CHECK: fcmove %st(2), %st
 0xda,0xca
 
-# CHECK: fcmove %st(3), %st(0)
+# CHECK: fcmove %st(3), %st
 0xda,0xcb
 
-# CHECK: fcmove %st(4), %st(0)
+# CHECK: fcmove %st(4), %st
 0xda,0xcc
 
-# CHECK: fcmove %st(5), %st(0)
+# CHECK: fcmove %st(5), %st
 0xda,0xcd
 
-# CHECK: fcmove %st(6), %st(0)
+# CHECK: fcmove %st(6), %st
 0xda,0xce
 
-# CHECK: fcmove %st(7), %st(0)
+# CHECK: fcmove %st(7), %st
 0xda,0xcf
 
-# CHECK: fcmovbe %st(0), %st(0)
+# CHECK: fcmovbe %st(0), %st
 0xda,0xd0
 
-# CHECK: fcmovbe %st(1), %st(0)
+# CHECK: fcmovbe %st(1), %st
 0xda,0xd1
 
-# CHECK: fcmovbe %st(2), %st(0)
+# CHECK: fcmovbe %st(2), %st
 0xda,0xd2
 
-# CHECK: fcmovbe %st(3), %st(0)
+# CHECK: fcmovbe %st(3), %st
 0xda,0xd3
 
-# CHECK: fcmovbe %st(4), %st(0)
+# CHECK: fcmovbe %st(4), %st
 0xda,0xd4
 
-# CHECK: fcmovbe %st(5), %st(0)
+# CHECK: fcmovbe %st(5), %st
 0xda,0xd5
 
-# CHECK: fcmovbe %st(6), %st(0)
+# CHECK: fcmovbe %st(6), %st
 0xda,0xd6
 
-# CHECK: fcmovbe %st(7), %st(0)
+# CHECK: fcmovbe %st(7), %st
 0xda,0xd7
 
-# CHECK: fcmovu %st(0), %st(0)
+# CHECK: fcmovu %st(0), %st
 0xda,0xd8
 
-# CHECK: fcmovu %st(1), %st(0)
+# CHECK: fcmovu %st(1), %st
 0xda,0xd9
 
-# CHECK: fcmovu %st(2), %st(0)
+# CHECK: fcmovu %st(2), %st
 0xda,0xda
 
-# CHECK: fcmovu %st(3), %st(0)
+# CHECK: fcmovu %st(3), %st
 0xda,0xdb
 
-# CHECK: fcmovu %st(4), %st(0)
+# CHECK: fcmovu %st(4), %st
 0xda,0xdc
 
-# CHECK: fcmovu %st(5), %st(0)
+# CHECK: fcmovu %st(5), %st
 0xda,0xdd
 
-# CHECK: fcmovu %st(6), %st(0)
+# CHECK: fcmovu %st(6), %st
 0xda,0xde
 
-# CHECK: fcmovu %st(7), %st(0)
+# CHECK: fcmovu %st(7), %st
 0xda,0xdf
 
 # CHECK: fucompp
 0xda,0xe9
 
-# CHECK: fcmovnb %st(0), %st(0)
+# CHECK: fcmovnb %st(0), %st
 0xdb,0xc0
 
-# CHECK: fcmovnb %st(1), %st(0)
+# CHECK: fcmovnb %st(1), %st
 0xdb,0xc1
 
-# CHECK: fcmovnb %st(2), %st(0)
+# CHECK: fcmovnb %st(2), %st
 0xdb,0xc2
 
-# CHECK: fcmovnb %st(3), %st(0)
+# CHECK: fcmovnb %st(3), %st
 0xdb,0xc3
 
-# CHECK: fcmovnb %st(4), %st(0)
+# CHECK: fcmovnb %st(4), %st
 0xdb,0xc4
 
-# CHECK: fcmovnb %st(5), %st(0)
+# CHECK: fcmovnb %st(5), %st
 0xdb,0xc5
 
-# CHECK: fcmovnb %st(6), %st(0)
+# CHECK: fcmovnb %st(6), %st
 0xdb,0xc6
 
-# CHECK: fcmovnb %st(7), %st(0)
+# CHECK: fcmovnb %st(7), %st
 0xdb,0xc7
 
-# CHECK: fcmovne %st(0), %st(0)
+# CHECK: fcmovne %st(0), %st
 0xdb,0xc8
 
-# CHECK: fcmovne %st(1), %st(0)
+# CHECK: fcmovne %st(1), %st
 0xdb,0xc9
 
-# CHECK: fcmovne %st(2), %st(0)
+# CHECK: fcmovne %st(2), %st
 0xdb,0xca
 
-# CHECK: fcmovne %st(3), %st(0)
+# CHECK: fcmovne %st(3), %st
 0xdb,0xcb
 
-# CHECK: fcmovne %st(4), %st(0)
+# CHECK: fcmovne %st(4), %st
 0xdb,0xcc
 
-# CHECK: fcmovne %st(5), %st(0)
+# CHECK: fcmovne %st(5), %st
 0xdb,0xcd
 
-# CHECK: fcmovne %st(6), %st(0)
+# CHECK: fcmovne %st(6), %st
 0xdb,0xce
 
-# CHECK: fcmovne %st(7), %st(0)
+# CHECK: fcmovne %st(7), %st
 0xdb,0xcf
 
-# CHECK: fcmovnbe %st(0), %st(0)
+# CHECK: fcmovnbe %st(0), %st
 0xdb,0xd0
 
-# CHECK: fcmovnbe %st(1), %st(0)
+# CHECK: fcmovnbe %st(1), %st
 0xdb,0xd1
 
-# CHECK: fcmovnbe %st(2), %st(0)
+# CHECK: fcmovnbe %st(2), %st
 0xdb,0xd2
 
-# CHECK: fcmovnbe %st(3), %st(0)
+# CHECK: fcmovnbe %st(3), %st
 0xdb,0xd3
 
-# CHECK: fcmovnbe %st(4), %st(0)
+# CHECK: fcmovnbe %st(4), %st
 0xdb,0xd4
 
-# CHECK: fcmovnbe %st(5), %st(0)
+# CHECK: fcmovnbe %st(5), %st
 0xdb,0xd5
 
-# CHECK: fcmovnbe %st(6), %st(0)
+# CHECK: fcmovnbe %st(6), %st
 0xdb,0xd6
 
-# CHECK: fcmovnbe %st(7), %st(0)
+# CHECK: fcmovnbe %st(7), %st
 0xdb,0xd7
 
-# CHECK: fcmovnu %st(0), %st(0)
+# CHECK: fcmovnu %st(0), %st
 0xdb,0xd8
 
-# CHECK: fcmovnu %st(1), %st(0)
+# CHECK: fcmovnu %st(1), %st
 0xdb,0xd9
 
-# CHECK: fcmovnu %st(2), %st(0)
+# CHECK: fcmovnu %st(2), %st
 0xdb,0xda
 
-# CHECK: fcmovnu %st(3), %st(0)
+# CHECK: fcmovnu %st(3), %st
 0xdb,0xdb
 
-# CHECK: fcmovnu %st(4), %st(0)
+# CHECK: fcmovnu %st(4), %st
 0xdb,0xdc
 
-# CHECK: fcmovnu %st(5), %st(0)
+# CHECK: fcmovnu %st(5), %st
 0xdb,0xdd
 
-# CHECK: fcmovnu %st(6), %st(0)
+# CHECK: fcmovnu %st(6), %st
 0xdb,0xde
 
-# CHECK: fcmovnu %st(7), %st(0)
+# CHECK: fcmovnu %st(7), %st
 0xdb,0xdf
 
 # CHECK: fnclex
@@ -574,148 +574,148 @@
 # CHECK: fcomi %st(7)
 0xdb,0xf7
 
-# CHECK: fadd %st(0), %st(0)
+# CHECK: fadd %st, %st(0)
 0xdc,0xc0
 
-# CHECK: fadd %st(0), %st(1)
+# CHECK: fadd %st, %st(1)
 0xdc,0xc1
 
-# CHECK: fadd %st(0), %st(2)
+# CHECK: fadd %st, %st(2)
 0xdc,0xc2
 
-# CHECK: fadd %st(0), %st(3)
+# CHECK: fadd %st, %st(3)
 0xdc,0xc3
 
-# CHECK: fadd %st(0), %st(4)
+# CHECK: fadd %st, %st(4)
 0xdc,0xc4
 
-# CHECK: fadd %st(0), %st(5)
+# CHECK: fadd %st, %st(5)
 0xdc,0xc5
 
-# CHECK: fadd %st(0), %st(6)
+# CHECK: fadd %st, %st(6)
 0xdc,0xc6
 
-# CHECK: fadd %st(0), %st(7)
+# CHECK: fadd %st, %st(7)
 0xdc,0xc7
 
-# CHECK: fmul %st(0), %st(0)
+# CHECK: fmul %st, %st(0)
 0xdc,0xc8
 
-# CHECK: fmul %st(0), %st(1)
+# CHECK: fmul %st, %st(1)
 0xdc,0xc9
 
-# CHECK: fmul %st(0), %st(2)
+# CHECK: fmul %st, %st(2)
 0xdc,0xca
 
-# CHECK: fmul %st(0), %st(3)
+# CHECK: fmul %st, %st(3)
 0xdc,0xcb
 
-# CHECK: fmul %st(0), %st(4)
+# CHECK: fmul %st, %st(4)
 0xdc,0xcc
 
-# CHECK: fmul %st(0), %st(5)
+# CHECK: fmul %st, %st(5)
 0xdc,0xcd
 
-# CHECK: fmul %st(0), %st(6)
+# CHECK: fmul %st, %st(6)
 0xdc,0xce
 
-# CHECK: fmul %st(0), %st(7)
+# CHECK: fmul %st, %st(7)
 0xdc,0xcf
 
-# CHECK: fsub %st(0), %st(0)
+# CHECK: fsub %st, %st(0)
 0xdc,0xe0
 
-# CHECK: fsub %st(0), %st(1)
+# CHECK: fsub %st, %st(1)
 0xdc,0xe1
 
-# CHECK: fsub %st(0), %st(2)
+# CHECK: fsub %st, %st(2)
 0xdc,0xe2
 
-# CHECK: fsub %st(0), %st(3)
+# CHECK: fsub %st, %st(3)
 0xdc,0xe3
 
-# CHECK: fsub %st(0), %st(4)
+# CHECK: fsub %st, %st(4)
 0xdc,0xe4
 
-# CHECK: fsub %st(0), %st(5)
+# CHECK: fsub %st, %st(5)
 0xdc,0xe5
 
-# CHECK: fsub %st(0), %st(6)
+# CHECK: fsub %st, %st(6)
 0xdc,0xe6
 
-# CHECK: fsub %st(0), %st(7)
+# CHECK: fsub %st, %st(7)
 0xdc,0xe7
 
-# CHECK: fsubr %st(0), %st(0)
+# CHECK: fsubr %st, %st(0)
 0xdc,0xe8
 
-# CHECK: fsubr %st(0), %st(1)
+# CHECK: fsubr %st, %st(1)
 0xdc,0xe9
 
-# CHECK: fsubr %st(0), %st(2)
+# CHECK: fsubr %st, %st(2)
 0xdc,0xea
 
-# CHECK: fsubr %st(0), %st(3)
+# CHECK: fsubr %st, %st(3)
 0xdc,0xeb
 
-# CHECK: fsubr %st(0), %st(4)
+# CHECK: fsubr %st, %st(4)
 0xdc,0xec
 
-# CHECK: fsubr %st(0), %st(5)
+# CHECK: fsubr %st, %st(5)
 0xdc,0xed
 
-# CHECK: fsubr %st(0), %st(6)
+# CHECK: fsubr %st, %st(6)
 0xdc,0xee
 
-# CHECK: fsubr %st(0), %st(7)
+# CHECK: fsubr %st, %st(7)
 0xdc,0xef
 
-# CHECK: fdiv %st(0), %st(0)
+# CHECK: fdiv %st, %st(0)
 0xdc,0xf0
 
-# CHECK: fdiv %st(0), %st(1)
+# CHECK: fdiv %st, %st(1)
 0xdc,0xf1
 
-# CHECK: fdiv %st(0), %st(2)
+# CHECK: fdiv %st, %st(2)
 0xdc,0xf2
 
-# CHECK: fdiv %st(0), %st(3)
+# CHECK: fdiv %st, %st(3)
 0xdc,0xf3
 
-# CHECK: fdiv %st(0), %st(4)
+# CHECK: fdiv %st, %st(4)
 0xdc,0xf4
 
-# CHECK: fdiv %st(0), %st(5)
+# CHECK: fdiv %st, %st(5)
 0xdc,0xf5
 
-# CHECK: fdiv %st(0), %st(6)
+# CHECK: fdiv %st, %st(6)
 0xdc,0xf6
 
-# CHECK: fdiv %st(0), %st(7)
+# CHECK: fdiv %st, %st(7)
 0xdc,0xf7
 
-# CHECK: fdivr %st(0), %st(0)
+# CHECK: fdivr %st, %st(0)
 0xdc,0xf8
 
-# CHECK: fdivr %st(0), %st(1)
+# CHECK: fdivr %st, %st(1)
 0xdc,0xf9
 
-# CHECK: fdivr %st(0), %st(2)
+# CHECK: fdivr %st, %st(2)
 0xdc,0xfa
 
-# CHECK: fdivr %st(0), %st(3)
+# CHECK: fdivr %st, %st(3)
 0xdc,0xfb
 
-# CHECK: fdivr %st(0), %st(4)
+# CHECK: fdivr %st, %st(4)
 0xdc,0xfc
 
-# CHECK: fdivr %st(0), %st(5)
+# CHECK: fdivr %st, %st(5)
 0xdc,0xfd
 
-# CHECK: fdivr %st(0), %st(6)
+# CHECK: fdivr %st, %st(6)
 0xdc,0xfe
 
-# CHECK: fdivr %st(0), %st(7)
+# CHECK: fdivr %st, %st(7)
 0xdc,0xff
 
 # CHECK: ffree %st(0)
@@ -838,151 +838,151 @@
 # CHECK: fucomp %st(7)
 0xdd,0xef
 
-# CHECK: faddp %st(0)
+# CHECK: faddp %st, %st(0)
 0xde,0xc0
 
-# CHECK: faddp %st(1)
+# CHECK: faddp %st, %st(1)
 0xde,0xc1
 
-# CHECK: faddp %st(2)
+# CHECK: faddp %st, %st(2)
 0xde,0xc2
 
-# CHECK: faddp %st(3)
+# CHECK: faddp %st, %st(3)
 0xde,0xc3
 
-# CHECK: faddp %st(4)
+# CHECK: faddp %st, %st(4)
 0xde,0xc4
 
-# CHECK: faddp %st(5)
+# CHECK: faddp %st, %st(5)
 0xde,0xc5
 
-# CHECK: faddp %st(6)
+# CHECK: faddp %st, %st(6)
 0xde,0xc6
 
-# CHECK: faddp %st(7)
+# CHECK: faddp %st, %st(7)
 0xde,0xc7
 
-# CHECK: fmulp %st(0)
+# CHECK: fmulp %st, %st(0)
 0xde,0xc8
 
-# CHECK: fmulp %st(1)
+# CHECK: fmulp %st, %st(1)
 0xde,0xc9
 
-# CHECK: fmulp %st(2)
+# CHECK: fmulp %st, %st(2)
 0xde,0xca
 
-# CHECK: fmulp %st(3)
+# CHECK: fmulp %st, %st(3)
 0xde,0xcb
 
-# CHECK: fmulp %st(4)
+# CHECK: fmulp %st, %st(4)
 0xde,0xcc
 
-# CHECK: fmulp %st(5)
+# CHECK: fmulp %st, %st(5)
 0xde,0xcd
 
-# CHECK: fmulp %st(6)
+# CHECK: fmulp %st, %st(6)
 0xde,0xce
 
-# CHECK: fmulp %st(7)
+# CHECK: fmulp %st, %st(7)
 0xde,0xcf
 
 # CHECK: fcompp
 0xde,0xd9
 
-# CHECK: fsubp %st(0)
+# CHECK: fsubp %st, %st(0)
 0xde,0xe0
 
-# CHECK: fsubp %st(1)
+# CHECK: fsubp %st, %st(1)
 0xde,0xe1
 
-# CHECK: fsubp %st(2)
+# CHECK: fsubp %st, %st(2)
 0xde,0xe2
 
-# CHECK: fsubp %st(3)
+# CHECK: fsubp %st, %st(3)
 0xde,0xe3
 
-# CHECK: fsubp %st(4)
+# CHECK: fsubp %st, %st(4)
 0xde,0xe4
 
-# CHECK: fsubp %st(5)
+# CHECK: fsubp %st, %st(5)
 0xde,0xe5
 
-# CHECK: fsubp %st(6)
+# CHECK: fsubp %st, %st(6)
 0xde,0xe6
 
-# CHECK: fsubp %st(7)
+# CHECK: fsubp %st, %st(7)
 0xde,0xe7
 
-# CHECK: fsubrp %st(0)
+# CHECK: fsubrp %st, %st(0)
 0xde,0xe8
 
-# CHECK: fsubrp %st(1)
+# CHECK: fsubrp %st, %st(1)
 0xde,0xe9
 
-# CHECK: fsubrp %st(2)
+# CHECK: fsubrp %st, %st(2)
 0xde,0xea
 
-# CHECK: fsubrp %st(3)
+# CHECK: fsubrp %st, %st(3)
 0xde,0xeb
 
-# CHECK: fsubrp %st(4)
+# CHECK: fsubrp %st, %st(4)
 0xde,0xec
 
-# CHECK: fsubrp %st(5)
+# CHECK: fsubrp %st, %st(5)
 0xde,0xed
 
-# CHECK: fsubrp %st(6)
+# CHECK: fsubrp %st, %st(6)
 0xde,0xee
 
-# CHECK: fsubrp %st(7)
+# CHECK: fsubrp %st, %st(7)
 0xde,0xef
 
-# CHECK: fdivp %st(0)
+# CHECK: fdivp %st, %st(0)
 0xde,0xf0
 
-# CHECK: fdivp %st(1)
+# CHECK: fdivp %st, %st(1)
 0xde,0xf1
 
-# CHECK: fdivp %st(2)
+# CHECK: fdivp %st, %st(2)
 0xde,0xf2
 
-# CHECK: fdivp %st(3)
+# CHECK: fdivp %st, %st(3)
 0xde,0xf3
 
-# CHECK: fdivp %st(4)
+# CHECK: fdivp %st, %st(4)
 0xde,0xf4
 
-# CHECK: fdivp %st(5)
+# CHECK: fdivp %st, %st(5)
 0xde,0xf5
 
-# CHECK: fdivp %st(6)
+# CHECK: fdivp %st, %st(6)
 0xde,0xf6
 
-# CHECK: fdivp %st(7)
+# CHECK: fdivp %st, %st(7)
 0xde,0xf7
 
-# CHECK: fdivrp %st(0)
+# CHECK: fdivrp %st, %st(0)
 0xde,0xf8
 
-# CHECK: fdivrp %st(1)
+# CHECK: fdivrp %st, %st(1)
 0xde,0xf9
 
-# CHECK: fdivrp %st(2)
+# CHECK: fdivrp %st, %st(2)
 0xde,0xfa
 
-# CHECK: fdivrp %st(3)
+# CHECK: fdivrp %st, %st(3)
 0xde,0xfb
 
-# CHECK: fdivrp %st(4)
+# CHECK: fdivrp %st, %st(4)
 0xde,0xfc
 
-# CHECK: fdivrp %st(5)
+# CHECK: fdivrp %st, %st(5)
 0xde,0xfd
 
-# CHECK: fdivrp %st(6)
+# CHECK: fdivrp %st, %st(6)
 0xde,0xfe
 
-# CHECK: fdivrp %st(7)
+# CHECK: fdivrp %st, %st(7)
 0xde,0xff
 
 # CHECK: ffreep %st(0)
diff --git a/test/MC/Disassembler/X86/x86-16.txt b/test/MC/Disassembler/X86/x86-16.txt
index 43cd09516c3b1640e2c4745a678bd616ae8beba1..286aa88489cb48b08cc501f79861eb44689713ab 100644
--- a/test/MC/Disassembler/X86/x86-16.txt
+++ b/test/MC/Disassembler/X86/x86-16.txt
@@ -759,10 +759,10 @@
 # CHECK: strl %eax
 0x66 0x0f 0x00 0xc8
 
-# CHECK: fsubp %st(1)
+# CHECK: fsubp %st, %st(1)
 0xde 0xe1
 
-# CHECK: fsubp %st(2)
+# CHECK: fsubp %st, %st(2)
 0xde 0xe2
 
 # CHECKX: nop
diff --git a/test/MC/ELF/ARM/clang-section.s b/test/MC/ELF/ARM/clang-section.s
index 73bae69aa26bd35cca1bdde3b693d48359ed23d9..0018cfe13d17890c7c3ef58f28fb9abc1fc4bfe8 100644
--- a/test/MC/ELF/ARM/clang-section.s
+++ b/test/MC/ELF/ARM/clang-section.s
@@ -220,7 +220,7 @@ p:
 	.size	p, 4
 
 
-	.ident	"clang version 5.0.0 (http://llvm.org/git/clang.git 254242a3ad440307fb451093a429c71ea9a8c888) (http://llvm.org/git/llvm.git 3c8daefbe3d1672ac1dae775b211f881f0063038)"
+	.ident	"clang version 5.0.0"
 	.section	".note.GNU-stack","",%progbits
 	.eabi_attribute	30, 1	@ Tag_ABI_optimization_goals
 
diff --git a/test/MC/ELF/cfi-b-key-frame.s b/test/MC/ELF/cfi-b-key-frame.s
new file mode 100644
index 0000000000000000000000000000000000000000..95d514674d64a277f1bfdcf8e1a22defa5cb0984
--- /dev/null
+++ b/test/MC/ELF/cfi-b-key-frame.s
@@ -0,0 +1,7 @@
+// REQUIRES: aarch64-registered-target
+// RUN: llvm-mc -filetype=obj -triple aarch64-arm-none-eabi %s -o - | llvm-dwarfdump - -v | FileCheck %s
+#CHECK:   Augmentation:          "zRB"
+f1:
+        .cfi_startproc
+        .cfi_b_key_frame
+        .cfi_endproc
diff --git a/test/MC/ELF/invalid-symver.s b/test/MC/ELF/invalid-symver.s
index 3c4f8c084b94b6da55ff1cc918fb0677ef353d49..d9f97b102b579b1cd89bef20630a7be9f36bbec9 100644
--- a/test/MC/ELF/invalid-symver.s
+++ b/test/MC/ELF/invalid-symver.s
@@ -1,7 +1,7 @@
 // RUN: not llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o %t 2> %t.out
 // RUN: FileCheck --input-file=%t.out %s
 
-// CHECK: A @@ version cannot be undefined
+// CHECK: error: versioned symbol foo@@bar must be defined
 
         .symver undefined, foo@@bar
         .long undefined
diff --git a/test/MC/ELF/multiple-different-symver.s b/test/MC/ELF/multiple-different-symver.s
index d7706f89f41982ad209974f23bb0a25d869e7e06..c34626c08173c200eae33d688b4dd3dcae06941e 100644
--- a/test/MC/ELF/multiple-different-symver.s
+++ b/test/MC/ELF/multiple-different-symver.s
@@ -1,6 +1,6 @@
 // RUN: not llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o %t 2>&1 | FileCheck %s
 
-// CHECK: Multiple symbol versions defined for foo
+// CHECK: error: multiple symbol versions defined for foo
 
 .symver foo, foo@1
 .symver foo, foo@2
diff --git a/test/MC/MSP430/addrmode.s b/test/MC/MSP430/addrmode.s
index 46051c00fed7a1367d7e6585c50059611778ddb4..c787e683aeab8d2ce80b47c0abe50222a6b721b3 100644
--- a/test/MC/MSP430/addrmode.s
+++ b/test/MC/MSP430/addrmode.s
@@ -21,11 +21,13 @@ foo:
   mov #42, 12(r15)
   mov #42, &disp
   mov disp, disp+2
+  mov r7, @r15
 
 ; CHECK: mov #42, r15          ; encoding: [0x3f,0x40,0x2a,0x00]
 ; CHECK: mov #42, 12(r15)      ; encoding: [0xbf,0x40,0x2a,0x00,0x0c,0x00]
 ; CHECK: mov #42, &disp        ; encoding: [0xb2,0x40,0x2a,0x00,A,A]
 ; CHECK: mov disp, disp+2      ; encoding: [0x90,0x40,A,A,B,B]
+; CHECK: mov r7, 0(r15)        ; encoding: [0x8f,0x47,0x00,0x00]
 
   add r7, r8
   add 6(r7), r8
@@ -93,6 +95,8 @@ foo:
 
   call r7
   call 6(r7)
+  call @r7
+  call @r7+
   call disp+6(r7)
   call &disp
   call disp
@@ -100,11 +104,61 @@ foo:
 
 ; CHECK: call r7               ; encoding: [0x87,0x12]
 ; CHECK: call 6(r7)            ; encoding: [0x97,0x12,0x06,0x00]
+; CHECK: call @r7              ; encoding: [0xa7,0x12]
+; CHECK: call @r7+             ; encoding: [0xb7,0x12]
 ; CHECK: call disp+6(r7)       ; encoding: [0x97,0x12,A,A]
 ; CHECK: call &disp            ; encoding: [0x92,0x12,A,A]
 ; CHECK: call disp             ; encoding: [0x90,0x12,A,A]
 ; CHECK: call #disp            ; encoding: [0xb0,0x12,A,A]
 
+  rra r7      ; CHECK: rra r7                ; encoding: [0x07,0x11]
+  rra 2(r7)   ; CHECK: rra 2(r7)             ; encoding: [0x17,0x11,0x02,0x00]
+  rra @r7     ; CHECK: rra @r7               ; encoding: [0x27,0x11]
+  rra @r7+    ; CHECK: rra @r7+              ; encoding: [0x37,0x11]
+
+  rrc r7      ; CHECK: rrc r7                ; encoding: [0x07,0x10]
+  rrc 2(r7)   ; CHECK: rrc 2(r7)             ; encoding: [0x17,0x10,0x02,0x00]
+  rrc @r7     ; CHECK: rrc @r7               ; encoding: [0x27,0x10]
+  rrc @r7+    ; CHECK: rrc @r7+              ; encoding: [0x37,0x10]
+
+  swpb r7     ; CHECK: swpb r7               ; encoding: [0x87,0x10]
+  swpb 2(r7)  ; CHECK: swpb 2(r7)            ; encoding: [0x97,0x10,0x02,0x00]
+  swpb @r7    ; CHECK: swpb @r7              ; encoding: [0xa7,0x10]
+  swpb @r7+   ; CHECK: swpb @r7+             ; encoding: [0xb7,0x10]
+
+  sxt r7      ; CHECK: sxt r7                ; encoding: [0x87,0x11]
+  sxt 2(r7)   ; CHECK: sxt 2(r7)             ; encoding: [0x97,0x11,0x02,0x00]
+  sxt @r7     ; CHECK: sxt @r7               ; encoding: [0xa7,0x11]
+  sxt @r7+    ; CHECK: sxt @r7+              ; encoding: [0xb7,0x11]
+
+  cmp r5, r7        ; CHECK: cmp r5, r7        ; encoding: [0x07,0x95]
+  cmp 2(r5), r7     ; CHECK: cmp 2(r5), r7     ; encoding: [0x17,0x95,0x02,0x00]
+  cmp #-1, r7       ; CHECK: cmp #-1, r7       ; encoding: [0x37,0x93]
+  cmp #42, r7       ; CHECK: cmp #42, r7       ; encoding: [0x37,0x90,0x2a,0x00]
+  cmp @r5, r7       ; CHECK: cmp @r5, r7       ; encoding: [0x27,0x95]
+  cmp @r5+, r7      ; CHECK: cmp @r5+, r7      ; encoding: [0x37,0x95]
+
+  cmp r5, 2(r7)     ; CHECK: cmp r5, 2(r7)     ; encoding: [0x87,0x95,0x02,0x00]
+  cmp 2(r7), 2(r7)  ; CHECK: cmp 2(r7), 2(r7)  ; encoding: [0x97,0x97,0x02,0x00,0x02,0x00]
+  cmp #-1, 2(r7)    ; CHECK: cmp #-1, 2(r7)    ; encoding: [0xb7,0x93,0x02,0x00]
+  cmp #42, 2(r7)    ; CHECK: cmp #42, 2(r7)    ; encoding: [0xb7,0x90,0x2a,0x00,0x02,0x00]
+  cmp @r5, 2(r7)    ; CHECK: cmp @r5, 2(r7)    ; encoding: [0xa7,0x95,0x02,0x00]
+  cmp @r5+, 2(r7)   ; CHECK: cmp @r5+, 2(r7)   ; encoding: [0xb7,0x95,0x02,0x00]
+
+  bit r5, r7        ; CHECK: bit r5, r7        ; encoding: [0x07,0xb5]
+  bit 2(r5), r7     ; CHECK: bit 2(r5), r7     ; encoding: [0x17,0xb5,0x02,0x00]
+  bit #-1, r7       ; CHECK: bit #-1, r7       ; encoding: [0x37,0xb3]
+  bit #42, r7       ; CHECK: bit #42, r7       ; encoding: [0x37,0xb0,0x2a,0x00]
+  bit @r5, r7       ; CHECK: bit @r5, r7       ; encoding: [0x27,0xb5]
+  bit @r5+, r7      ; CHECK: bit @r5+, r7      ; encoding: [0x37,0xb5]
+
+  bit r5, 2(r7)     ; CHECK: bit r5, 2(r7)     ; encoding: [0x87,0xb5,0x02,0x00]
+  bit 2(r7), 2(r7)  ; CHECK: bit 2(r7), 2(r7)  ; encoding: [0x97,0xb7,0x02,0x00,0x02,0x00]
+  bit #-1, 2(r7)    ; CHECK: bit #-1, 2(r7)    ; encoding: [0xb7,0xb3,0x02,0x00]
+  bit #42, 2(r7)    ; CHECK: bit #42, 2(r7)    ; encoding: [0xb7,0xb0,0x2a,0x00,0x02,0x00]
+  bit @r5, 2(r7)    ; CHECK: bit @r5, 2(r7)    ; encoding: [0xa7,0xb5,0x02,0x00]
+  bit @r5+, 2(r7)   ; CHECK: bit @r5+, 2(r7)   ; encoding: [0xb7,0xb5,0x02,0x00]
+
 disp:
   .word 0xcafe
   .word 0xbabe
diff --git a/test/MC/MSP430/const.s b/test/MC/MSP430/const.s
index f5cca109a50bebfa25db7fcea61c8165a658d606..dfaf32fbc777f065a3372a7e834359985bd17727 100644
--- a/test/MC/MSP430/const.s
+++ b/test/MC/MSP430/const.s
@@ -1,10 +1,13 @@
 ; RUN: llvm-mc -triple msp430 -show-encoding < %s | FileCheck %s
-  mov #4, r15 ; CHECK: mov #4, r15 ; encoding: [0x2f,0x42]
-  mov #8, r15 ; CHECK: mov #8, r15 ; encoding: [0x3f,0x42]
-  mov #0, r15 ; CHECK: clr r15     ; encoding: [0x0f,0x43]
-  mov #1, r15 ; CHECK: mov #1, r15 ; encoding: [0x1f,0x43]
-  mov #2, r15 ; CHECK: mov #2, r15 ; encoding: [0x2f,0x43]
-  mov #-1, r7 ; CHECK: mov #-1, r7 ; encoding: [0x37,0x43]
 
-  push #8     ; CHECK: push #8     ; encoding: [0x32,0x12]
-  push #42    ; CHECK: push #42    ; encoding: [0x30,0x12,0x2a,0x00]
+  mov #4, r15   ; CHECK: mov #4, r15   ; encoding: [0x2f,0x42]
+  mov #8, r15   ; CHECK: mov #8, r15   ; encoding: [0x3f,0x42]
+  mov #0, r15   ; CHECK: clr r15       ; encoding: [0x0f,0x43]
+  mov #1, r15   ; CHECK: mov #1, r15   ; encoding: [0x1f,0x43]
+  mov #2, r15   ; CHECK: mov #2, r15   ; encoding: [0x2f,0x43]
+  mov #-1, r7   ; CHECK: mov #-1, r7   ; encoding: [0x37,0x43]
+
+  push #8       ; CHECK: push #8       ; encoding: [0x32,0x12]
+  push #42      ; CHECK: push #42      ; encoding: [0x30,0x12,0x2a,0x00]
+
+  bit #1, 0(r7) ; CHECK: bit #1, 0(r7) ; encoding: [0x97,0xb3,0x00,0x00]
diff --git a/test/MC/MSP430/invalid.s b/test/MC/MSP430/invalid.s
index 2815b520dd5556f0921d2d3e2f3532d83bb96c51..ce686bd83f1edb01b67ff56e835de1e5c72e9616 100644
--- a/test/MC/MSP430/invalid.s
+++ b/test/MC/MSP430/invalid.s
@@ -4,7 +4,6 @@ foo:
   mov    r7        ; CHECK: :[[@LINE]]:3: error: too few operands for instruction
 
   ;; invalid destination addressing modes
-  mov    r7, @r15  ; CHECK: :[[@LINE]]:14: error: invalid operand for instruction
   mov    r7, @r15+ ; CHECK: :[[@LINE]]:14: error: invalid operand for instruction
   mov    r7, #0    ; CHECK: :[[@LINE]]:14: error: invalid operand for instruction
   mov    r7, #123  ; CHECK: :[[@LINE]]:14: error: invalid operand for instruction
diff --git a/test/MC/MSP430/msp430-separator.s b/test/MC/MSP430/msp430-separator.s
new file mode 100644
index 0000000000000000000000000000000000000000..498e86e1df3b256a8d894b1cf318e84cbe520bb8
--- /dev/null
+++ b/test/MC/MSP430/msp430-separator.s
@@ -0,0 +1,15 @@
+; RUN: llvm-mc -triple msp430 < %s | FileCheck %s
+
+; MSP430 supports multiple assembly statements on the same line
+; separated by a '{' character.
+
+; Check that the '{' is recognized as a line separator and
+; multiple statements correctly parsed.
+
+_foo:
+; CHECK:      foo
+; CHECK:      add r10, r11
+; CHECK-NEXT: call r11
+; CHECK-NEXT: mov r11, 2(r1)
+add r10, r11 { call r11 { mov r11, 2(r1)
+ret
diff --git a/test/MC/MachO/file.s b/test/MC/MachO/file.s
index 3ddfb2efe2246803e8a9a489ec5f31fccd3c5963..eddbb599d97a27ed9455d93073d73a4aab5767ae 100644
--- a/test/MC/MachO/file.s
+++ b/test/MC/MachO/file.s
@@ -1,5 +1,8 @@
 // RUN: llvm-mc -triple i386-apple-darwin9 %s -filetype=obj -o - | llvm-readobj -s -section-data | FileCheck %s
 
+// This number-less file directive is ignored on MachO.
+        .file "bar/baz.s"
+
         .file	1 "dir/foo"
         nop
 
diff --git a/test/MC/Mips/cpsetup.s b/test/MC/Mips/cpsetup.s
index 907e4fe6a889c5b5a1040cc081c11f72acfbd1e4..c963df09ce6ec7ad69ab22919b0f16e585e3ffba 100644
--- a/test/MC/Mips/cpsetup.s
+++ b/test/MC/Mips/cpsetup.s
@@ -1,5 +1,5 @@
 # RUN: llvm-mc -triple mips-unknown-linux -target-abi o32 -filetype=obj -o - %s | \
-# RUN:   llvm-objdump -d -r - | FileCheck -check-prefixes=ALL,O32 %s
+# RUN:   llvm-objdump -d -r -z - | FileCheck -check-prefixes=ALL,O32 %s
 
 # RUN: llvm-mc -triple mips-unknown-linux -target-abi o32 %s | \
 # RUN:   FileCheck -check-prefixes=ALL,ASM,ASM-O32 %s
@@ -7,14 +7,14 @@
 # FIXME: Now we check .cpsetup expansion for `-mno-shared` case only.
 #        We also need to implement/check the `-mshared` case.
 # RUN: llvm-mc -triple mips64-unknown-linux -target-abi n32 -filetype=obj -o - %s | \
-# RUN:   llvm-objdump -d -r - | \
+# RUN:   llvm-objdump -d -r -z - | \
 # RUN:   FileCheck -check-prefixes=ALL,NXX,N32 %s
 
 # RUN: llvm-mc -triple mips64-unknown-linux -target-abi n32 %s | \
 # RUN:   FileCheck -check-prefixes=ALL,ASM,ASM-N32 %s
 
 # RUN: llvm-mc -triple mips64-unknown-linux %s -filetype=obj -o - | \
-# RUN:   llvm-objdump -d -r - | \
+# RUN:   llvm-objdump -d -r -z - | \
 # RUN:   FileCheck -check-prefixes=ALL,NXX,N64 %s
 
 # RUN: llvm-mc -triple mips64-unknown-linux %s | \
diff --git a/test/MC/Mips/micromips-branch-fixup.s b/test/MC/Mips/micromips-branch-fixup.s
index 98b48421b05e78952c26b04e14240a6c94b9de51..3ad625fa4c13dc8effed26022a4f771e635397a1 100644
--- a/test/MC/Mips/micromips-branch-fixup.s
+++ b/test/MC/Mips/micromips-branch-fixup.s
@@ -24,29 +24,23 @@
 # CHECK-FIXUP:                     value: bar, kind: fixup_MICROMIPS_PC16_S1
 # CHECK-FIXUP: nop             # encoding: [0x00,0x0c]
 # CHECK-FIXUP: beq $3, $4, bar # encoding: [0x83'A',0x94'A',0x00,0x00]
-# CHECK-FIXUP:                 #   fixup A - offset: 0,
-# CHECK-FIXUP:                     value: bar, kind: fixup_MICROMIPS_PC16_S1
-# CHECK-FIXUP: nop             # encoding: [0x00,0x00,0x00,0x00]
+# CHECK-FIXUP:                 #   fixup A - offset: 0, value: bar, kind: fixup_MICROMIPS_PC16_S1
+# CHECK-FIXUP: nop             # encoding: [0x00,0x0c]
 # CHECK-FIXUP: bne $3, $4, bar # encoding: [0x83'A',0xb4'A',0x00,0x00]
-# CHECK-FIXUP:                 #   fixup A - offset: 0,
-# CHECK-FIXUP:                     value: bar, kind: fixup_MICROMIPS_PC16_S1
-# CHECK-FIXUP: nop             # encoding: [0x00,0x00,0x00,0x00]
+# CHECK-FIXUP:                 #   fixup A - offset: 0, value: bar, kind: fixup_MICROMIPS_PC16_S1
+# CHECK-FIXUP: nop             # encoding: [0x00,0x0c]
 # CHECK-FIXUP: bgez    $4, bar # encoding: [0x44'A',0x40'A',0x00,0x00]
-# CHECK-FIXUP:                 #   fixup A - offset: 0,
-# CHECK-FIXUP:                     value: bar, kind: fixup_MICROMIPS_PC16_S1
-# CHECK-FIXUP: nop             # encoding: [0x00,0x00,0x00,0x00]
+# CHECK-FIXUP-NEXT:                 #   fixup A - offset: 0, value: bar, kind: fixup_MICROMIPS_PC16_S1
+# CHECK-FIXUP-NEXT: nop             # encoding: [0x00,0x0c]
 # CHECK-FIXUP: bgtz    $4, bar # encoding: [0xc4'A',0x40'A',0x00,0x00]
-# CHECK-FIXUP:                 #   fixup A - offset: 0,
-# CHECK-FIXUP:                     value: bar, kind: fixup_MICROMIPS_PC16_S1
-# CHECK-FIXUP: nop             # encoding: [0x00,0x00,0x00,0x00]
+# CHECK-FIXUP-NEXT:                 #   fixup A - offset: 0, value: bar, kind: fixup_MICROMIPS_PC16_S1
+# CHECK-FIXUP-NEXT: nop             # encoding: [0x00,0x0c]
 # CHECK-FIXUP: blez    $4, bar # encoding: [0x84'A',0x40'A',0x00,0x00]
-# CHECK-FIXUP:                 #   fixup A - offset: 0,
-# CHECK-FIXUP:                     value: bar, kind: fixup_MICROMIPS_PC16_S1
-# CHECK-FIXUP: nop             # encoding: [0x00,0x00,0x00,0x00]
+# CHECK-FIXUP-NEXT:                 #   fixup A - offset: 0, value: bar, kind: fixup_MICROMIPS_PC16_S1
+# CHECK-FIXUP-NEXT: nop             # encoding: [0x00,0x0c]
 # CHECK-FIXUP: bltz    $4, bar # encoding: [0x04'A',0x40'A',0x00,0x00]
-# CHECK-FIXUP:                 #   fixup A - offset: 0,
-# CHECK-FIXUP:                     value: bar, kind: fixup_MICROMIPS_PC16_S1
-# CHECK-FIXUP: nop             # encoding: [0x00,0x00,0x00,0x00]
+# CHECK-FIXUP-NEXT:                 #   fixup A - offset: 0, value: bar, kind: fixup_MICROMIPS_PC16_S1
+# CHECK-FIXUP-NEXT: nop             # encoding: [0x00,0x0c]
 # CHECK-FIXUP: bgezal  $4, bar # encoding: [0x64'A',0x40'A',0x00,0x00]
 # CHECK-FIXUP:                 #   fixup A - offset: 0,
 # CHECK-FIXUP:                     value: bar, kind: fixup_MICROMIPS_PC16_S1
diff --git a/test/MC/Mips/micromips-branch-instructions.s b/test/MC/Mips/micromips-branch-instructions.s
index e85b92521ea25d78d7034f1dbe0ce7897b01b74e..d82d5ad971f5e4891dae360f39a91623410691fe 100644
--- a/test/MC/Mips/micromips-branch-instructions.s
+++ b/test/MC/Mips/micromips-branch-instructions.s
@@ -12,23 +12,23 @@
 # CHECK-EL: b   1332             # encoding: [0x00,0x94,0x9a,0x02]
 # CHECK-EL: nop                  # encoding: [0x00,0x0c]
 # CHECK-EL: beq $9, $6, 1332     # encoding: [0xc9,0x94,0x9a,0x02]
-# CHECK-EL: nop                  # encoding: [0x00,0x00,0x00,0x00]
+# CHECK-EL-NEXT: nop                  # encoding: [0x00,0x0c]
 # CHECK-EL: bgez $6, 1332        # encoding: [0x46,0x40,0x9a,0x02]
-# CHECK-EL: nop                  # encoding: [0x00,0x00,0x00,0x00]
+# CHECK-EL-NEXT: nop                  # encoding: [0x00,0x0c]
 # CHECK-EL: bgezal $6, 1332      # encoding: [0x66,0x40,0x9a,0x02]
 # CHECK-EL: nop                  # encoding: [0x00,0x00,0x00,0x00]
 # CHECK-EL: bltzal $6, 1332      # encoding: [0x26,0x40,0x9a,0x02]
 # CHECK-EL: nop                  # encoding: [0x00,0x00,0x00,0x00]
 # CHECK-EL: bgtz $6, 1332        # encoding: [0xc6,0x40,0x9a,0x02]
-# CHECK-EL: nop                  # encoding: [0x00,0x00,0x00,0x00]
+# CHECK-EL-NEXT: nop                  # encoding: [0x00,0x0c]
 # CHECK-EL: blez $6, 1332        # encoding: [0x86,0x40,0x9a,0x02]
-# CHECK-EL: nop                  # encoding: [0x00,0x00,0x00,0x00]
+# CHECK-EL-NEXT: nop                  # encoding: [0x00,0x0c]
 # CHECK-EL: bne $9, $6, 1332     # encoding: [0xc9,0xb4,0x9a,0x02]
-# CHECK-EL: nop                  # encoding: [0x00,0x00,0x00,0x00]
+# CHECK-EL-NEXT: nop                  # encoding: [0x00,0x0c]
 # CHECK-EL: bal 1332             # encoding: [0x60,0x40,0x9a,0x02]
 # CHECK-EL: nop                  # encoding: [0x00,0x00,0x00,0x00]
 # CHECK-EL: bltz $6, 1332        # encoding: [0x06,0x40,0x9a,0x02]
-# CHECK-EL: nop                  # encoding: [0x00,0x00,0x00,0x00]
+# CHECK-EL-NEXT: nop                  # encoding: [0x00,0x0c]
 # CHECK-EL: bgezals $6, 1332     # encoding: [0x66,0x42,0x9a,0x02]
 # CHECK-EL: nop                  # encoding: [0x00,0x0c]
 # CHECK-EL: bltzals $6, 1332     # encoding: [0x26,0x42,0x9a,0x02]
@@ -39,23 +39,23 @@
 # CHECK-EB: b   1332             # encoding: [0x94,0x00,0x02,0x9a]
 # CHECK-EB: nop                  # encoding: [0x0c,0x00]
 # CHECK-EB: beq $9, $6, 1332     # encoding: [0x94,0xc9,0x02,0x9a]
-# CHECK-EB: nop                  # encoding: [0x00,0x00,0x00,0x00]
+# CHECK-EB-NEXT: nop                  # encoding: [0x0c,0x00]
 # CHECK-EB: bgez $6, 1332        # encoding: [0x40,0x46,0x02,0x9a]
-# CHECK-EB: nop                  # encoding: [0x00,0x00,0x00,0x00]
+# CHECK-EB-NEXT: nop                  # encoding: [0x0c,0x00]
 # CHECK-EB: bgezal $6, 1332      # encoding: [0x40,0x66,0x02,0x9a]
 # CHECK-EB: nop                  # encoding: [0x00,0x00,0x00,0x00]
 # CHECK-EB: bltzal $6, 1332      # encoding: [0x40,0x26,0x02,0x9a]
 # CHECK-EB: nop                  # encoding: [0x00,0x00,0x00,0x00]
 # CHECK-EB: bgtz $6, 1332        # encoding: [0x40,0xc6,0x02,0x9a]
-# CHECK-EB: nop                  # encoding: [0x00,0x00,0x00,0x00]
+# CHECK-EB-NEXT: nop                  # encoding: [0x0c,0x00]
 # CHECK-EB: blez $6, 1332        # encoding: [0x40,0x86,0x02,0x9a]
-# CHECK-EB: nop                  # encoding: [0x00,0x00,0x00,0x00]
+# CHECK-EB-NEXT: nop                  # encoding: [0x0c,0x00]
 # CHECK-EB: bne $9, $6, 1332     # encoding: [0xb4,0xc9,0x02,0x9a]
-# CHECK-EB: nop                  # encoding: [0x00,0x00,0x00,0x00]
+# CHECK-EB-NEXT: nop                  # encoding: [0x0c,0x00]
 # CHECK-EB: bal 1332             # encoding: [0x40,0x60,0x02,0x9a]
 # CHECK-EB: nop                  # encoding: [0x00,0x00,0x00,0x00]
 # CHECK-EB: bltz $6, 1332        # encoding: [0x40,0x06,0x02,0x9a]
-# CHECK-EB: nop                  # encoding: [0x00,0x00,0x00,0x00]
+# CHECK-EB-NEXT: nop                  # encoding: [0x0c,0x00]
 # CHECK-EB: bgezals $6, 1332     # encoding: [0x42,0x66,0x02,0x9a]
 # CHECK-EB: nop                  # encoding: [0x0c,0x00]
 # CHECK-EB: bltzals $6, 1332     # encoding: [0x42,0x26,0x02,0x9a]
diff --git a/test/MC/Mips/micromips-el-fixup-data.s b/test/MC/Mips/micromips-el-fixup-data.s
index 0f26c39e75c8b9ec5a855994529fc2cfac6a30e6..0b9a02f529997e601880978517781c62926e9d9e 100644
--- a/test/MC/Mips/micromips-el-fixup-data.s
+++ b/test/MC/Mips/micromips-el-fixup-data.s
@@ -16,7 +16,7 @@ main:
     addiu   $sp, $sp, -16
     bnez    $9, lab1
 
-# CHECK:    09 b4 04 00    bnez $9, 12
+# CHECK:    09 b4 03 00    bnez $9, 10
 
     addu    $zero, $zero, $zero
 lab1:
diff --git a/test/MC/Mips/micromips-jump-instructions.s b/test/MC/Mips/micromips-jump-instructions.s
index 3147a3f256d4d80d44ab7bb0ed218d2f1d49772b..f8d4974de6cd3aa78dbe76fe1050ad1a490d373d 100644
--- a/test/MC/Mips/micromips-jump-instructions.s
+++ b/test/MC/Mips/micromips-jump-instructions.s
@@ -10,7 +10,7 @@
 # Little endian
 #------------------------------------------------------------------------------
 # CHECK-EL: j 1328      # encoding: [0x00,0xd4,0x98,0x02]
-# CHECK-EL: nop         # encoding: [0x00,0x00,0x00,0x00]
+# CHECK-EL: nop         # encoding: [0x00,0x0c]
 # CHECK-EL: jal 1328    # encoding: [0x00,0xf4,0x98,0x02]
 # CHECK-EL: nop         # encoding: [0x00,0x00,0x00,0x00]
 # CHECK-EL: jalr $ra, $6 # encoding: [0xe6,0x03,0x3c,0x0f]
@@ -33,7 +33,7 @@
 # Big endian
 #------------------------------------------------------------------------------
 # CHECK-EB: j 1328      # encoding: [0xd4,0x00,0x02,0x98]
-# CHECK-EB: nop         # encoding: [0x00,0x00,0x00,0x00]
+# CHECK-EB: nop         # encoding: [0x0c,0x00]
 # CHECK-EB: jal 1328    # encoding: [0xf4,0x00,0x02,0x98]
 # CHECK-EB: nop         # encoding: [0x00,0x00,0x00,0x00]
 # CHECK-EB: jalr $ra, $6 # encoding: [0x03,0xe6,0x0f,0x3c]
diff --git a/test/MC/Mips/micromips-tailr.s b/test/MC/Mips/micromips-tailr.s
index 0c21a7bf2107da57b86ad5a9e046d7db62f42906..a11ac61eac2d24d10ae707bbbf35e24cd79533dc 100644
--- a/test/MC/Mips/micromips-tailr.s
+++ b/test/MC/Mips/micromips-tailr.s
@@ -11,9 +11,8 @@
 # CHECK-FIXUP:   addiu $2, $zero, 1332
 # CHECK-FIXUP:         # encoding: [0x40,0x30,0x34,0x05]
 # CHECK-FIXUP:   j foo # encoding: [A,0xd4'A',A,0b000000AA]
-# CHECK-FIXUP:         #   fixup A - offset: 0,
-# CHECK-FIXUP:             value: foo, kind: fixup_MICROMIPS_26_S1
-# CHECK-FIXUP:   nop   # encoding: [0x00,0x00,0x00,0x00]
+# CHECK-FIXUP-NEXT:         #   fixup A - offset: 0, value: foo, kind: fixup_MICROMIPS_26_S1
+# CHECK-FIXUP-NEXT:   nop   # encoding: [0x00,0x0c]
 #------------------------------------------------------------------------------
 # Check that the appropriate relocations were created.
 #------------------------------------------------------------------------------
diff --git a/test/MC/Mips/nacl-mask.s b/test/MC/Mips/nacl-mask.s
index 604b5c83721e99490fcd77834ffdf8df4ee4afb8..e7eba3769ac05580b6892de507504b580e92d719 100644
--- a/test/MC/Mips/nacl-mask.s
+++ b/test/MC/Mips/nacl-mask.s
@@ -1,5 +1,5 @@
 # RUN: llvm-mc -filetype=obj -triple=mipsel-unknown-nacl %s \
-# RUN:   | llvm-objdump -disassemble -no-show-raw-insn - | FileCheck %s
+# RUN:   | llvm-objdump -disassemble -z -no-show-raw-insn - | FileCheck %s
 
 # This test tests that address-masking sandboxing is added when given assembly
 # input.
diff --git a/test/MC/Mips/unsupported-relocation.s b/test/MC/Mips/unsupported-relocation.s
index 151a559671fbadda983b6305fa2546a93dae0d26..258b48399873133ceb1d0eb1fd3ae708a856db06 100644
--- a/test/MC/Mips/unsupported-relocation.s
+++ b/test/MC/Mips/unsupported-relocation.s
@@ -1,5 +1,5 @@
-# RUN: not llvm-mc -triple mips-unknown-linux -filetype=obj %s 2>%t
-# RUN: FileCheck %s < %t
+# RUN: not llvm-mc -triple mips-unknown-linux -filetype=obj -o %t %s 2>&1 \
+# RUN:  |  FileCheck %s
 
 # Check that we emit an error for unsupported relocations instead of crashing.
 
@@ -8,6 +8,8 @@
         .data
 foo:
         .byte   x
+# CHECK: :[[@LINE-1]]:17: error: MIPS does not support one byte relocations
         .byte   x+1
-
-# CHECK: LLVM ERROR: MIPS does not support one byte relocations
+# CHECK: :[[@LINE-1]]:17: error: MIPS does not support one byte relocations
+        .quad   x-foo
+# CHECK: :[[@LINE-1]]:17: error: MIPS does not support 64-bit PC-relative relocations
diff --git a/test/MC/RISCV/align.s b/test/MC/RISCV/align.s
new file mode 100644
index 0000000000000000000000000000000000000000..e62af93155587f557e0f92884d371544451b6d30
--- /dev/null
+++ b/test/MC/RISCV/align.s
@@ -0,0 +1,105 @@
+# The file testing Nop insertion with R_RISCV_ALIGN for relaxation.
+
+# Relaxation enabled:
+# RUN: llvm-mc -filetype=obj -triple riscv32 -mattr=+relax < %s \
+# RUN:     | llvm-objdump -d -riscv-no-aliases - \
+# RUN:     | FileCheck -check-prefix=RELAX-INST %s
+# RUN: llvm-mc -filetype=obj -triple riscv32 -mattr=+relax < %s \
+# RUN:     | llvm-readobj -r | FileCheck -check-prefix=RELAX-RELOC %s
+
+# Relaxation disabled:
+# RUN: llvm-mc -filetype=obj -triple riscv32 -mattr=-relax < %s \
+# RUN:     | llvm-objdump -d -riscv-no-aliases - \
+# RUN:     | FileCheck -check-prefix=NORELAX-INST %s
+# RUN: llvm-mc -filetype=obj -triple riscv32 -mattr=-relax < %s \
+# RUN:     | llvm-readobj -r | FileCheck -check-prefix=NORELAX-RELOC %s
+
+# Relaxation enabled with C extension:
+# RUN: llvm-mc -filetype=obj -triple riscv32 -mattr=+c,+relax < %s \
+# RUN:     | llvm-objdump -d -riscv-no-aliases - \
+# RUN:     | FileCheck -check-prefix=C-EXT-RELAX-INST %s
+# RUN: llvm-mc -filetype=obj -triple riscv32 -mattr=+c,+relax < %s \
+# RUN:     | llvm-readobj -r | FileCheck -check-prefix=C-EXT-RELAX-RELOC %s
+
+# Relaxation disabled with C extension:
+# RUN: llvm-mc -filetype=obj -triple riscv32 -mattr=+c,-relax < %s \
+# RUN:     | llvm-objdump -d -riscv-no-aliases - \
+# RUN:     | FileCheck -check-prefix=C-EXT-NORELAX-INST %s
+# RUN: llvm-mc -filetype=obj -triple riscv32 -mattr=+c,-relax < %s \
+# RUN:     | llvm-readobj -r | FileCheck -check-prefix=C-EXT-NORELAX-RELOC %s
+
+# We need to insert N-MinNopSize bytes NOPs and R_RISCV_ALIGN relocation
+# type for .align N directive when linker relaxation enabled.
+# Linker could satisfy alignment by removing NOPs after linker relaxation.
+
+# The first R_RISCV_ALIGN come from
+# MCELFStreamer::InitSections() EmitCodeAlignment(4).
+# C-EXT-RELAX-RELOC: R_RISCV_ALIGN - 0x2
+# C-EXT-RELAX-INST:  c.nop
+test:
+	.p2align 2
+# C-EXT-RELAX-RELOC: R_RISCV_ALIGN - 0x2
+# C-EXT-RELAX-INST:  c.nop
+	bne     zero, a0, .LBB0_2
+	mv	a0, zero
+	.p2align 3
+# RELAX-RELOC: R_RISCV_ALIGN - 0x4
+# RELAX-INST:  addi    zero, zero, 0
+# C-EXT-RELAX-RELOC: R_RISCV_ALIGN - 0x6
+# C-EXT-RELAX-INST:  addi    zero, zero, 0
+# C-EXT-RELAX-INST:  c.nop
+# C-EXT-NORELAX-INST: addi    zero, zero, 0
+	add	a0, a0, a1
+	.align 4
+.LBB0_2:
+# RELAX-RELOC: R_RISCV_ALIGN - 0xC
+# RELAX-INST:  addi    zero, zero, 0
+# RELAX-INST:  addi    zero, zero, 0
+# RELAX-INST:  addi    zero, zero, 0
+# NORELAX-INST: addi    zero, zero, 0
+# C-EXT-RELAX-RELOC: R_RISCV_ALIGN - 0xE
+# C-EXT-RELAX-INST:  addi    zero, zero, 0
+# C-EXT-RELAX-INST:  addi    zero, zero, 0
+# C-EXT-RELAX-INST:  addi    zero, zero, 0
+# C-EXT-RELAX-INST:  c.nop
+# C-EXT-INST: addi    zero, zero, 0
+# C-EXT-INST: c.nop
+	add	a0, a0, a1
+	.p2align 3
+.constant_pool:
+.long	3126770193
+# RELAX-RELOC: R_RISCV_ALIGN - 0x4
+# RELAX-INST:  addi    zero, zero, 0
+# NORELAX-INST: addi    zero, zero, 0
+# C-EXT-RELAX-RELOC: R_RISCV_ALIGN - 0x6
+# C-EXT-RELAX-INST:  addi    zero, zero, 0
+# C-EXT-RELAX-INST:  c.nop
+# C-EXT-INST: addi    zero, zero, 0
+# C-EXT-INST: c.nop
+	add	a0, a0, a1
+# Alignment directive with specific padding value 0x01.
+# We will not emit R_RISCV_ALIGN in this case.
+# The behavior is the same as GNU assembler.
+	.p2align 4, 1
+# RELAX-RELOC-NOT: R_RISCV_ALIGN - 0xC
+# RELAX-INST:  01 01
+# RELAX-INST:  01 01
+# C-EXT-RELAX-RELOC-NOT: R_RISCV_ALIGN - 0xE
+# C-EXT-RELAX-INST:  01 01
+# C-EXT-INST:  01 01
+	ret
+# NORELAX-RELOC-NOT: R_RISCV
+# C-EXT-NORELAX-RELOC-NOT: R_RISCV
+# We only need to insert R_RISCV_ALIGN for code section
+# when the linker relaxation enabled.
+        .data
+	.p2align        3
+# RELAX-RELOC-NOT: R_RISCV_ALIGN
+# C-EXT-RELAX-RELOC-NOT: R_RISCV_ALIGN
+data1:
+	.word 7
+	.p2align        4
+# RELAX-RELOC-NOT: R_RISCV_ALIGN
+# C-EXT-RELAX-RELOC-NOT: R_RISCV_ALIGN
+data2:
+	.word 9
diff --git a/test/Assembler/empty-string.s b/test/MC/RISCV/empty-string.s
similarity index 100%
rename from test/Assembler/empty-string.s
rename to test/MC/RISCV/empty-string.s
diff --git a/test/MC/RISCV/fixups.s b/test/MC/RISCV/fixups.s
index 0f5432dd1176bdae4a61025310c52524847ca781..f0377debabb9ef25927b590fa0933a66f5e99db5 100644
--- a/test/MC/RISCV/fixups.s
+++ b/test/MC/RISCV/fixups.s
@@ -24,15 +24,26 @@ sw a0, %lo(val)(t1)
 # CHECK-FIXUP: fixup A - offset: 0, value: %lo(val), kind: fixup_riscv_lo12_s
 # CHECK-INSTR: sw a0, 1656(t1)
 
+1:
+auipc t1, %pcrel_hi(.LBB0)
+# CHECK-FIXUP: fixup A - offset: 0, value: %pcrel_hi(.LBB0), kind: fixup_riscv_pcrel_hi20
+# CHECK-INSTR: auipc t1, 0
+addi t1, t1, %pcrel_lo(1b)
+# CHECK-FIXUP: fixup A - offset: 0, value: %pcrel_lo(.Ltmp0), kind: fixup_riscv_pcrel_lo12_i
+# CHECK-INSTR: addi t1, t1, -16
+sw t1, %pcrel_lo(1b)(t1)
+# CHECK-FIXUP: fixup A - offset: 0, value: %pcrel_lo(.Ltmp0), kind: fixup_riscv_pcrel_lo12_s
+# CHECK-INSTR: sw t1, -16(t1)
+
 jal zero, .LBB0
 # CHECK-FIXUP: fixup A - offset: 0, value: .LBB0, kind: fixup_riscv_jal
-# CHECK-INSTR: jal zero, -16
+# CHECK-INSTR: jal zero, -28
 jal zero, .LBB2
 # CHECK-FIXUP: fixup A - offset: 0, value: .LBB2, kind: fixup_riscv_jal
 # CHECK-INSTR: jal zero, 330996
 beq a0, a1, .LBB0
 # CHECK-FIXUP: fixup A - offset: 0, value: .LBB0, kind: fixup_riscv_branch
-# CHECK-INSTR: beq a0, a1, -24
+# CHECK-INSTR: beq a0, a1, -36
 blt a0, a1, .LBB1
 # CHECK-FIXUP: fixup A - offset: 0, value: .LBB1, kind: fixup_riscv_branch
 # CHECK-INSTR: blt a0, a1, 1108
diff --git a/test/MC/RISCV/linker-relaxation.s b/test/MC/RISCV/linker-relaxation.s
index 0184d174e9017ad1fccc5c5fef65420a903671df..104811f98d51a6e66d4159b68f155aebc82d6664 100644
--- a/test/MC/RISCV/linker-relaxation.s
+++ b/test/MC/RISCV/linker-relaxation.s
@@ -18,9 +18,58 @@ call foo
 # NORELAX-RELOC: R_RISCV_CALL foo 0x0
 # NORELAX-RELOC-NOT: R_RISCV_RELAX
 # RELAX-RELOC: R_RISCV_CALL foo 0x0
-# RELAX-RELOC: R_RISCV_RELAX foo 0x0
+# RELAX-RELOC: R_RISCV_RELAX - 0x0
 # RELAX-FIXUP: fixup A - offset: 0, value: foo, kind: fixup_riscv_call
-# RELAX-FIXUP: fixup B - offset: 0, value: foo, kind: fixup_riscv_relax
+# RELAX-FIXUP: fixup B - offset: 0, value: 0, kind: fixup_riscv_relax
 beq s1, s1, .L1
 # RELAX-RELOC: R_RISCV_BRANCH .L1 0x0
 # RELAX-FIXUP: fixup A - offset: 0, value: .L1, kind: fixup_riscv_branch
+# RELAX-FIXUP-NOT: fixup B - offset: 0, value: 0, kind: fixup_riscv_relax
+
+lui t1, %hi(foo)
+# NORELAX-RELOC: R_RISCV_HI20 foo 0x0
+# NORELAX-RELOC-NOT: R_RISCV_RELAX
+# RELOC: R_RISCV_HI20 foo 0x0
+# RELAX-RELOC: R_RISCV_RELAX - 0x0
+# RELAX-FIXUP: fixup A - offset: 0, value: %hi(foo), kind: fixup_riscv_hi20
+# RELAX-FIXUP: fixup B - offset: 0, value: 0, kind: fixup_riscv_relax
+
+addi t1, t1, %lo(foo)
+# NORELAX-RELOC: R_RISCV_LO12_I foo 0x0
+# NORELAX-RELOC-NOT: R_RISCV_RELAX
+# RELOC: R_RISCV_LO12_I foo 0x0
+# RELAX-RELOC: R_RISCV_RELAX - 0x0
+# RELAX-FIXUP: fixup A - offset: 0, value: %lo(foo), kind: fixup_riscv_lo12_i
+# RELAX-FIXUP: fixup B - offset: 0, value: 0, kind: fixup_riscv_relax
+
+sb t1, %lo(foo)(a2)
+# NORELAX-RELOC: R_RISCV_LO12_S foo 0x0
+# NORELAX-RELOC-NOT: R_RISCV_RELAX
+# RELOC: R_RISCV_LO12_S foo 0x0
+# RELAX-RELOC: R_RISCV_RELAX - 0x0
+# RELAX-FIXUP: fixup A - offset: 0, value: %lo(foo), kind: fixup_riscv_lo12_s
+# RELAX-FIXUP: fixup B - offset: 0, value: 0, kind: fixup_riscv_relax
+
+auipc t1, %pcrel_hi(foo)
+# NORELAX-RELOC: R_RISCV_PCREL_HI20 foo 0x0
+# NORELAX-RELOC-NOT: R_RISCV_RELAX
+# RELOC: R_RISCV_PCREL_HI20 foo 0x0
+# RELAX-RELOC: R_RISCV_RELAX - 0x0
+# RELAX-FIXUP: fixup A - offset: 0, value: %pcrel_hi(foo), kind: fixup_riscv_pcrel_hi20
+# RELAX-FIXUP: fixup B - offset: 0, value: 0, kind: fixup_riscv_relax
+
+addi t1, t1, %pcrel_lo(foo)
+# NORELAX-RELOC: R_RISCV_PCREL_LO12_I foo 0x0
+# NORELAX-RELOC-NOT: R_RISCV_RELAX
+# RELOC: R_RISCV_PCREL_LO12_I foo 0x0
+# RELAX-RELOC: R_RISCV_RELAX - 0x0
+# RELAX-FIXUP: fixup A - offset: 0, value: %pcrel_lo(foo), kind: fixup_riscv_pcrel_lo12_i
+# RELAX-FIXUP: fixup B - offset: 0, value: 0, kind: fixup_riscv_relax
+
+sb t1, %pcrel_lo(foo)(a2)
+# NORELAX-RELOC: R_RISCV_PCREL_LO12_S foo 0x0
+# NORELAX-RELOC-NOT: R_RISCV_RELAX
+# RELOC: R_RISCV_PCREL_LO12_S foo 0x0
+# RELAX-RELOC: R_RISCV_RELAX - 0x0
+# RELAX-FIXUP: fixup A - offset: 0, value: %pcrel_lo(foo), kind: fixup_riscv_pcrel_lo12_s
+# RELAX-FIXUP: fixup B - offset: 0, value: 0, kind: fixup_riscv_relax
diff --git a/test/MC/RISCV/option-pushpop.s b/test/MC/RISCV/option-pushpop.s
index 9e3ae4156a44ab3907904719241b81c5ea69e081..230664ec40204242da5d2601a80049727964176c 100644
--- a/test/MC/RISCV/option-pushpop.s
+++ b/test/MC/RISCV/option-pushpop.s
@@ -21,7 +21,7 @@
 
 # CHECK-INST: call foo
 # CHECK-RELOC: R_RISCV_CALL foo 0x0
-# CHECK-RELOC-NOT: R_RISCV_RELAX foo 0x0
+# CHECK-RELOC-NOT: R_RISCV_RELAX - 0x0
 call foo
 
 # CHECK-INST: addi s0, sp, 1020
@@ -36,7 +36,7 @@ addi s0, sp, 1020
 # CHECK-INST: .option relax
 # CHECK-INST: call bar
 # CHECK-RELOC-NEXT: R_RISCV_CALL bar 0x0
-# CHECK-RELOC-NEXT: R_RISCV_RELAX bar 0x0
+# CHECK-RELOC-NEXT: R_RISCV_RELAX - 0x0
 call bar
 
 .option push    # Push relax=true, rvc=false
@@ -58,14 +58,14 @@ addi s0, sp, 1020
 
 # CHECK-INST: call bar
 # CHECK-RELOC-NEXT: R_RISCV_CALL bar 0x0
-# CHECK-RELOC-NEXT: R_RISCV_RELAX bar 0x0
+# CHECK-RELOC-NEXT: R_RISCV_RELAX - 0x0
 call bar
 
 .option pop     # Pop relax=false, rvc=false
 # CHECK-INST: .option pop
 # CHECK-INST: call baz
 # CHECK-RELOC: R_RISCV_CALL baz 0x0
-# CHECK-RELOC-NOT: R_RISCV_RELAX baz 0x0
+# CHECK-RELOC-NOT: R_RISCV_RELAX - 0x0
 call baz
 
 # CHECK-INST: addi s0, sp, 1020
diff --git a/test/MC/RISCV/option-relax.s b/test/MC/RISCV/option-relax.s
index 82ec24b710c127c5ee50a1b1e4bd282f9565e3b9..d5768941af0c4216e39046cb603e4d1709417b63 100644
--- a/test/MC/RISCV/option-relax.s
+++ b/test/MC/RISCV/option-relax.s
@@ -21,7 +21,7 @@
 
 # CHECK-INST: call foo
 # CHECK-RELOC: R_RISCV_CALL foo 0x0
-# CHECK-RELOC-NOT: R_RISCV_RELAX foo 0x0
+# CHECK-RELOC-NOT: R_RISCV_RELAX - 0x0
 call foo
 
 # CHECK-RELOC-NEXT: R_RISCV_ADD64
@@ -38,7 +38,7 @@ beq s1, s1, .L1
 
 # CHECK-INST: call bar
 # CHECK-RELOC-NEXT: R_RISCV_CALL bar 0x0
-# CHECK-RELOC-NEXT: R_RISCV_RELAX bar 0x0
+# CHECK-RELOC-NEXT: R_RISCV_RELAX - 0x0
 call bar
 
 # CHECK-RELOC-NEXT: R_RISCV_ADD64
@@ -54,7 +54,7 @@ beq s1, s1, .L1
 
 # CHECK-INST: call baz
 # CHECK-RELOC-NEXT: R_RISCV_CALL baz 0x0
-# CHECK-RELOC-NOT: R_RISCV_RELAX baz 0x0
+# CHECK-RELOC-NOT: R_RISCV_RELAX - 0x0
 call baz
 
 # CHECK-RELOC-NEXT: R_RISCV_ADD64
diff --git a/test/MC/RISCV/pcrel-lo12-invalid.s b/test/MC/RISCV/pcrel-lo12-invalid.s
new file mode 100644
index 0000000000000000000000000000000000000000..7cf2494ad8e92312cb87a019a18d287c76c40142
--- /dev/null
+++ b/test/MC/RISCV/pcrel-lo12-invalid.s
@@ -0,0 +1,5 @@
+# RUN: not llvm-mc -triple riscv32 -mattr=-relax -filetype obj < %s -o /dev/null 2>&1 | FileCheck %s
+# RUN: not llvm-mc -triple riscv32 -mattr=+relax -filetype obj < %s -o /dev/null 2>&1 | FileCheck %s
+
+1:
+  addi a0, a0, %pcrel_lo(1b) # CHECK: :[[@LINE]]:3: error: could not find corresponding %pcrel_hi
diff --git a/test/MC/RISCV/relocations.s b/test/MC/RISCV/relocations.s
index b68b11bf195f40389a6c3a8fdb8d948c6c207d8b..a879c9a54adb902cc7a081e085788bf784a0b9e7 100644
--- a/test/MC/RISCV/relocations.s
+++ b/test/MC/RISCV/relocations.s
@@ -44,6 +44,7 @@ sb t1, %lo(foo+4)(a2)
 # INSTR: sb t1, %lo(foo+4)(a2)
 # FIXUP: fixup A - offset: 0, value: %lo(foo+4), kind: fixup_riscv_lo12_s
 
+.L0:
 auipc t1, %pcrel_hi(foo)
 # RELOC: R_RISCV_PCREL_HI20 foo 0x0
 # INSTR: auipc t1, %pcrel_hi(foo)
@@ -54,25 +55,15 @@ auipc t1, %pcrel_hi(foo+4)
 # INSTR: auipc t1, %pcrel_hi(foo+4)
 # FIXUP: fixup A - offset: 0, value: %pcrel_hi(foo+4), kind: fixup_riscv_pcrel_hi20
 
-addi t1, t1, %pcrel_lo(foo)
-# RELOC: R_RISCV_PCREL_LO12_I foo 0x0
-# INSTR: addi t1, t1, %pcrel_lo(foo)
-# FIXUP: fixup A - offset: 0, value: %pcrel_lo(foo), kind: fixup_riscv_pcrel_lo12_i
+addi t1, t1, %pcrel_lo(.L0)
+# RELOC: R_RISCV_PCREL_LO12_I .L0 0x0
+# INSTR: addi t1, t1, %pcrel_lo(.L0)
+# FIXUP: fixup A - offset: 0, value: %pcrel_lo(.L0), kind: fixup_riscv_pcrel_lo12_i
 
-addi t1, t1, %pcrel_lo(foo+4)
-# RELOC: R_RISCV_PCREL_LO12_I foo 0x4
-# INSTR: addi t1, t1, %pcrel_lo(foo+4)
-# FIXUP: fixup A - offset: 0, value: %pcrel_lo(foo+4), kind: fixup_riscv_pcrel_lo12_i
-
-sb t1, %pcrel_lo(foo)(a2)
-# RELOC: R_RISCV_PCREL_LO12_S foo 0x0
-# INSTR: sb t1, %pcrel_lo(foo)(a2)
-# FIXUP: fixup A - offset: 0, value: %pcrel_lo(foo), kind: fixup_riscv_pcrel_lo12_s
-
-sb t1, %pcrel_lo(foo+4)(a2)
-# RELOC: R_RISCV_PCREL_LO12_S foo 0x4
-# INSTR: sb t1, %pcrel_lo(foo+4)(a2)
-# FIXUP: fixup A - offset: 0, value: %pcrel_lo(foo+4), kind: fixup_riscv_pcrel_lo12_s
+sb t1, %pcrel_lo(.L0)(a2)
+# RELOC: R_RISCV_PCREL_LO12_S .L0 0x0
+# INSTR: sb t1, %pcrel_lo(.L0)(a2)
+# FIXUP: fixup A - offset: 0, value: %pcrel_lo(.L0), kind: fixup_riscv_pcrel_lo12_s
 
 jal zero, foo
 # RELOC: R_RISCV_JAL
diff --git a/test/MC/RISCV/rv32i-aliases-valid.s b/test/MC/RISCV/rv32i-aliases-valid.s
index 04baa9e488be339bc3dbf67b65e972c8473784a2..f9866722cc16422e43eed9c4d567df27eeadfe31 100644
--- a/test/MC/RISCV/rv32i-aliases-valid.s
+++ b/test/MC/RISCV/rv32i-aliases-valid.s
@@ -3,10 +3,10 @@
 # RUN: llvm-mc %s -triple=riscv32 \
 # RUN:     | FileCheck -check-prefixes=CHECK-EXPAND,CHECK-ALIAS %s
 # RUN: llvm-mc -filetype=obj -triple riscv32 < %s \
-# RUN:     | llvm-objdump -riscv-no-aliases -d - \
-# RUN:     | FileCheck -check-prefixes=CHECK-EXPAND,CHECK-INST %s
+# RUN:     | llvm-objdump -riscv-no-aliases -d -r - \
+# RUN:     | FileCheck -check-prefixes=CHECK-OBJ-NOALIAS,CHECK-EXPAND,CHECK-INST %s
 # RUN: llvm-mc -filetype=obj -triple riscv32 < %s \
-# RUN:     | llvm-objdump -d - \
+# RUN:     | llvm-objdump -d -r - \
 # RUN:     | FileCheck -check-prefixes=CHECK-EXPAND,CHECK-ALIAS %s
 
 # The following check prefixes are used in this test:
@@ -69,6 +69,32 @@ li x12, 0x80000000
 # CHECK-EXPAND: addi a2, zero, -1
 li x12, 0xFFFFFFFF
 
+# CHECK-EXPAND: addi a0, zero, 1110
+li a0, %lo(0x123456)
+# CHECK-OBJ-NOALIAS: addi a0, zero, 0
+# CHECK-OBJ: R_RISCV_PCREL_LO12
+li a0, %pcrel_lo(0x123456)
+
+# CHECK-OBJ-NOALIAS: addi a0, zero, 0
+# CHECK-OBJ: R_RISCV_LO12
+li a0, %lo(foo)
+# CHECK-OBJ-NOALIAS: addi a0, zero, 0
+# CHECK-OBJ: R_RISCV_PCREL_LO12
+li a0, %pcrel_lo(foo)
+
+.equ CONST, 0x123456
+# CHECK-EXPAND: lui a0, 291
+# CHECK-EXPAND: addi a0, a0, 1110
+li a0, CONST
+# CHECK-EXPAND: lui a0, 291
+# CHECK-EXPAND: addi a0, a0, 1111
+li a0, CONST+1
+
+.equ CONST, 0x654321
+# CHECK-EXPAND: lui a0, 1620
+# CHECK-EXPAND: addi a0, a0, 801
+li a0, CONST
+
 # CHECK-INST: csrrs t4, instreth, zero
 # CHECK-ALIAS: rdinstreth t4
 rdinstreth x29
diff --git a/test/MC/RISCV/rv32i-valid.s b/test/MC/RISCV/rv32i-valid.s
index 0df53e661b90961374efd49c8bffa0c9d5b3f429..85a841b805017bad2931212d8223a30ce8f2f384 100644
--- a/test/MC/RISCV/rv32i-valid.s
+++ b/test/MC/RISCV/rv32i-valid.s
@@ -9,6 +9,8 @@
 # RUN:     | llvm-objdump -riscv-no-aliases -d -r - \
 # RUN:     | FileCheck -check-prefixes=CHECK-OBJ,CHECK-ASM-AND-OBJ %s
 
+.equ CONST, 30
+
 # CHECK-ASM-AND-OBJ: lui a0, 2
 # CHECK-ASM: encoding: [0x37,0x25,0x00,0x00]
 lui a0, 2
@@ -35,6 +37,12 @@ lui gp, 0
 # CHECK-OBJ: lui a0, 0
 # CHECK-OBJ: R_RISCV_HI20 foo
 lui a0, %hi(foo)
+# CHECK-ASM-AND-OBJ: lui a0, 30
+# CHECK-ASM: encoding: [0x37,0xe5,0x01,0x00]
+lui a0, CONST
+# CHECK-ASM-AND-OBJ: lui a0, 31
+# CHECK-ASM: encoding: [0x37,0xf5,0x01,0x00]
+lui a0, CONST+1
 
 # CHECK-ASM-AND-OBJ: auipc a0, 2
 # CHECK-ASM: encoding: [0x17,0x25,0x00,0x00]
@@ -53,6 +61,9 @@ auipc gp, 0
 # CHECK-OBJ: auipc a0, 0
 # CHECK-OBJ: R_RISCV_PCREL_HI20 foo
 auipc a0, %pcrel_hi(foo)
+# CHECK-ASM-AND-OBJ: auipc a0, 30
+# CHECK-ASM: encoding: [0x17,0xe5,0x01,0x00]
+auipc a0, CONST
 
 # CHECK-ASM-AND-OBJ: jal a2, 1048574
 # CHECK-ASM: encoding: [0x6f,0xf6,0xff,0x7f]
@@ -70,6 +81,9 @@ jal a0, foo
 # CHECK-OBJ: jal a0, 0
 # CHECK-OBJ: R_RISCV_JAL a0
 jal a0, a0
+# CHECK-ASM-AND-OBJ: jal a0, 30
+# CHECK-ASM: encoding: [0x6f,0x05,0xe0,0x01]
+jal a0, CONST
 
 # CHECK-ASM-AND-OBJ: jalr a0, a1, -2048
 # CHECK-ASM: encoding: [0x67,0x85,0x05,0x80]
@@ -83,6 +97,9 @@ jalr t2, t1, 2047
 # CHECK-ASM-AND-OBJ: jalr sp, zero, 256
 # CHECK-ASM: encoding: [0x67,0x01,0x00,0x10]
 jalr sp, zero, 256
+# CHECK-ASM-AND-OBJ: jalr a1, a2, 30
+# CHECK-ASM: encoding: [0xe7,0x05,0xe6,0x01]
+jalr a1, a2, CONST
 
 # CHECK-ASM-AND-OBJ: beq s1, s1, 102
 # CHECK-ASM: encoding: [0x63,0x83,0x94,0x06]
@@ -102,6 +119,9 @@ bltu zero, zero, 0
 # CHECK-ASM-AND-OBJ: bgeu s8, sp, 512
 # CHECK-ASM: encoding: [0x63,0x70,0x2c,0x20]
 bgeu s8, sp, 512
+# CHECK-ASM-AND-OBJ: bgeu t0, t1, 30
+# CHECK-ASM: encoding: [0x63,0xff,0x62,0x00]
+bgeu t0, t1, CONST
 
 # CHECK-ASM-AND-OBJ: lb s3, 4(ra)
 # CHECK-ASM: encoding: [0x83,0x89,0x40,0x00]
@@ -131,6 +151,9 @@ lbu s5, %lo(foo)(s6)
 # CHECK-OBJ: lhu t3, 0(t3)
 # CHECK-OBJ: R_RISCV_PCREL_LO12
 lhu t3, %pcrel_lo(foo)(t3)
+# CHECK-ASM-AND-OBJ: lb t0, 30(t1)
+# CHECK-ASM: encoding: [0x83,0x02,0xe3,0x01]
+lb t0, CONST(t1)
 
 # CHECK-ASM-AND-OBJ: sb a0, 2047(a2)
 # CHECK-ASM: encoding: [0xa3,0x0f,0xa6,0x7e]
@@ -144,6 +167,9 @@ sh t3, %lo(2048)(t5)
 # CHECK-ASM-AND-OBJ: sw ra, 999(zero)
 # CHECK-ASM: encoding: [0xa3,0x23,0x10,0x3e]
 sw ra, 999(zero)
+# CHECK-ASM-AND-OBJ: sw a0, 30(t0)
+# CHECK-ASM: encoding: [0x23,0xaf,0xa2,0x00]
+sw a0, CONST(t0)
 
 # CHECK-ASM-AND-OBJ: addi ra, sp, 2
 # CHECK-ASM: encoding: [0x93,0x00,0x21,0x00]
@@ -153,6 +179,9 @@ addi ra, sp, 2
 # CHECK-OBJ: addi ra, sp, 0
 # CHECK-OBJ: R_RISCV_LO12
 addi ra, sp, %lo(foo)
+# CHECK-ASM-AND-OBJ: addi ra, sp, 30
+# CHECK-ASM: encoding: [0x93,0x00,0xe1,0x01]
+addi ra, sp, CONST
 # CHECK-ASM-AND-OBJ: slti a0, a2, -20
 # CHECK-ASM: encoding: [0x13,0x25,0xc6,0xfe]
 slti a0, a2, -20
@@ -184,6 +213,9 @@ srli a0, a4, 0
 # CHECK-ASM-AND-OBJ: srai a2, sp, 15
 # CHECK-ASM: encoding: [0x13,0x56,0xf1,0x40]
 srai a2, sp, 15
+# CHECK-ASM-AND-OBJ: slli t3, t3, 30
+# CHECK-ASM: encoding: [0x13,0x1e,0xee,0x01]
+slli t3, t3, CONST
 
 # CHECK-ASM-AND-OBJ: add ra, zero, zero
 # CHECK-ASM: encoding: [0xb3,0x00,0x00,0x00]
@@ -252,6 +284,8 @@ ebreak
 # CHECK-ASM: encoding: [0x73,0x10,0x00,0xc0]
 unimp
 
+.equ CONST, 16
+
 # CHECK-ASM-AND-OBJ: csrrw t0, 4095, t1
 # CHECK-ASM: encoding: [0xf3,0x12,0xf3,0xff]
 csrrw t0, 0xfff, t1
diff --git a/test/MC/RISCV/rv64i-aliases-valid.s b/test/MC/RISCV/rv64i-aliases-valid.s
index 64bb36eaad6fd1918ef92edfeae1cf5fdeb20c01..a705525a5a6927aeece901efaf3c12ab50384702 100644
--- a/test/MC/RISCV/rv64i-aliases-valid.s
+++ b/test/MC/RISCV/rv64i-aliases-valid.s
@@ -4,7 +4,7 @@
 # RUN:     | FileCheck -check-prefixes=CHECK-EXPAND,CHECK-ALIAS %s
 # RUN: llvm-mc -filetype=obj -triple riscv64 < %s \
 # RUN:     | llvm-objdump -riscv-no-aliases -d - \
-# RUN:     | FileCheck -check-prefixes=CHECK-EXPAND,CHECK-INST %s
+# RUN:     | FileCheck -check-prefixes=CHECK-OBJ-NOALIAS,CHECK-EXPAND,CHECK-INST %s
 # RUN: llvm-mc -filetype=obj -triple riscv64 < %s \
 # RUN:     | llvm-objdump -d - \
 # RUN:     | FileCheck -check-prefixes=CHECK-EXPAND,CHECK-ALIAS %s
@@ -105,6 +105,29 @@ li t4, 0x123456789abcdef0
 # CHECK-EXPAND: addi t5, zero, -1
 li t5, 0xFFFFFFFFFFFFFFFF
 
+# CHECK-EXPAND: addi a0, zero, 1110
+li a0, %lo(0x123456)
+# CHECK-OBJ-NOALIAS: addi a0, zero, 0
+# CHECK-OBJ: R_RISCV_PCREL_LO12
+li a0, %pcrel_lo(0x123456)
+
+# CHECK-OBJ-NOALIAS: addi a0, zero, 0
+# CHECK-OBJ: R_RISCV_LO12
+li a0, %lo(foo)
+# CHECK-OBJ-NOALIAS: addi a0, zero, 0
+# CHECK-OBJ: R_RISCV_PCREL_LO12
+li a0, %pcrel_lo(foo)
+
+.equ CONST, 0x123456
+# CHECK-EXPAND: lui a0, 291
+# CHECK-EXPAND: addiw a0, a0, 1110
+li a0, CONST
+
+.equ CONST, 0x654321
+# CHECK-EXPAND: lui a0, 1620
+# CHECK-EXPAND: addiw a0, a0, 801
+li a0, CONST
+
 # CHECK-INST: subw t6, zero, ra
 # CHECK-ALIAS: negw t6, ra
 negw x31, x1
diff --git a/test/MC/RISCV/rv64i-valid.s b/test/MC/RISCV/rv64i-valid.s
index 337b996e4470c49c32dd35bb04d25ed88e394ed7..dab502c92b9e58e93a329dc3ef497c6ac9e728d7 100644
--- a/test/MC/RISCV/rv64i-valid.s
+++ b/test/MC/RISCV/rv64i-valid.s
@@ -4,6 +4,8 @@
 # RUN:     | llvm-objdump -riscv-no-aliases -d -r - \
 # RUN:     | FileCheck -check-prefixes=CHECK-OBJ,CHECK-ASM-AND-OBJ %s
 
+.equ CONST, 31
+
 # CHECK-ASM-AND-OBJ: lwu zero, 4(ra)
 # CHECK-ASM: encoding: [0x03,0xe0,0x40,0x00]
 lwu x0, 4(x1)
@@ -49,6 +51,9 @@ srli x24, x25, 0
 # CHECK-ASM-AND-OBJ: srai s10, s11, 31
 # CHECK-ASM: encoding: [0x13,0xdd,0xfd,0x41]
 srai x26, x27, 31
+# CHECK-ASM-AND-OBJ: srai s10, s11, 31
+# CHECK-ASM: encoding: [0x13,0xdd,0xfd,0x41]
+srai x26, x27, CONST
 
 # CHECK-ASM-AND-OBJ: addiw t3, t4, -2048
 # CHECK-ASM: encoding: [0x1b,0x8e,0x0e,0x80]
@@ -75,6 +80,9 @@ sraiw s0, s1, 0
 # CHECK-ASM-AND-OBJ: sraiw a0, a1, 31
 # CHECK-ASM: encoding: [0x1b,0xd5,0xf5,0x41]
 sraiw a0, a1, 31
+# CHECK-ASM-AND-OBJ: sraiw a0, a1, 31
+# CHECK-ASM: encoding: [0x1b,0xd5,0xf5,0x41]
+sraiw a0, a1, CONST
 
 # CHECK-ASM-AND-OBJ: addw a2, a3, a4
 # CHECK-ASM: encoding: [0x3b,0x86,0xe6,0x00]
diff --git a/test/MC/WebAssembly/annotations.s b/test/MC/WebAssembly/annotations.s
new file mode 100644
index 0000000000000000000000000000000000000000..e049279a2793edf59fd137191bd14d0fa007dce3
--- /dev/null
+++ b/test/MC/WebAssembly/annotations.s
@@ -0,0 +1,71 @@
+# RUN: llvm-mc -triple=wasm32-unknown-unknown -mattr=+exception-handling < %s | FileCheck %s
+
+# Tests if block/loop/try/catch/end/branch/rethrow instructions are correctly
+# printed with their annotations.
+
+  .text
+  .section .text.test_annotation,"",@
+  .type    test_annotation,@function
+test_annotation:
+  .functype   test_annotation () -> ()
+  .eventtype  __cpp_exception i32
+  try
+  br        0
+  catch
+  block
+  br_if     0
+  loop
+  br_if     1
+  end_loop
+  end_block
+  try
+  rethrow
+  catch
+  block
+  try
+  br        0
+  catch
+  local.set 0
+  block    i32
+  local.get 0
+  br_on_exn 0, __cpp_exception@EVENT
+  rethrow
+  end_block
+  end_try
+  end_block
+  rethrow
+  end_try
+  end_try
+  end_function
+
+
+# CHECK:      test_annotation:
+# CHECK:        try
+# CHECK-NEXT:   br        0               # 0: down to label0
+# CHECK-NEXT:   catch                     # catch0:
+# CHECK-NEXT:   block
+# CHECK-NEXT:   br_if     0               # 0: down to label1
+# CHECK-NEXT:   loop                      # label2:
+# CHECK-NEXT:   br_if     1               # 1: down to label1
+# CHECK-NEXT:   end_loop
+# CHECK-NEXT:   end_block                 # label1:
+# CHECK-NEXT:   try
+# CHECK-NEXT:   rethrow                   # down to catch1
+# CHECK-NEXT:   catch                     # catch1:
+# CHECK-NEXT:   block
+# CHECK-NEXT:   try
+# CHECK-NEXT:   br        0               # 0: down to label5
+# CHECK-NEXT:   catch                     # catch2:
+# CHECK-NEXT:   local.set 0
+# CHECK-NEXT:   block    i32
+# CHECK-NEXT:   local.get 0
+# CHECK-NEXT:   br_on_exn 0, __cpp_exception@EVENT # 0: down to label6
+# CHECK-NEXT:   rethrow                   # to caller
+# CHECK-NEXT:   end_block                 # label6:
+# CHECK-NEXT:   end_try                   # label5:
+# CHECK-NEXT:   end_block                 # label4:
+# CHECK-NEXT:   rethrow                   # to caller
+# CHECK-NEXT:   end_try                   # label3:
+# CHECK-NEXT:   end_try                   # label0:
+# CHECK-NEXT:   end_function
+
diff --git a/test/MC/WebAssembly/array-fill.ll b/test/MC/WebAssembly/array-fill.ll
index 7af4e740dc8ab769dbd5293034e4532f4d7ce176..7f3d8bda346587b7ccfa56f5ec6fa946e10efc89 100644
--- a/test/MC/WebAssembly/array-fill.ll
+++ b/test/MC/WebAssembly/array-fill.ll
@@ -22,6 +22,6 @@ target triple = "wasm32-unknown-unknown"
 ; CHECK-NEXT:     SegmentInfo:    
 ; CHECK-NEXT:       - Index:           0
 ; CHECK-NEXT:         Name:            .data
-; CHECK-NEXT:         Alignment:       1
+; CHECK-NEXT:         Alignment:       0
 ; CHECK-NEXT:         Flags:           [ ]
 ; CHECK-NEXT: ...
diff --git a/test/MC/WebAssembly/assembler-binary.ll b/test/MC/WebAssembly/assembler-binary.ll
index 3667a522ada51643dce866ecfd9c7c5943ab5a10..35bd3a88646cbf0118fdb616bb4f842f565e4a3c 100644
--- a/test/MC/WebAssembly/assembler-binary.ll
+++ b/test/MC/WebAssembly/assembler-binary.ll
@@ -28,8 +28,6 @@ entry:
 ; ASM-NEXT: 	.functype	foo (i32) -> ()
 ; ASM-NEXT: 	call	bar@FUNCTION
 ; ASM-NEXT: 	end_function
-; ASM-NEXT: .Lfunc_end0:
-; ASM-NEXT: 	.size	foo, .Lfunc_end0-foo
 ; ASM:       	.functype	bar () -> ()
 
 
@@ -57,7 +55,7 @@ entry:
 ; CHECK-NEXT:         Field:           __indirect_function_table
 ; CHECK-NEXT:         Kind:            TABLE
 ; CHECK-NEXT:         Table:
-; CHECK-NEXT:           ElemType:        ANYFUNC
+; CHECK-NEXT:           ElemType:        FUNCREF
 ; CHECK-NEXT:           Limits:
 ; CHECK-NEXT:             Initial:         0x00000000
 ; CHECK-NEXT:       - Module:          env
@@ -68,7 +66,7 @@ entry:
 ; CHECK-NEXT:     FunctionTypes:   [ 0 ]
 ; CHECK-NEXT:   - Type:            CODE
 ; CHECK-NEXT:     Relocations:
-; CHECK-NEXT:       - Type:            R_WEBASSEMBLY_FUNCTION_INDEX_LEB
+; CHECK-NEXT:       - Type:            R_WASM_FUNCTION_INDEX_LEB
 ; CHECK-NEXT:         Index:           1
 ; CHECK-NEXT:         Offset:          0x00000004
 ; CHECK-NEXT:     Functions:
@@ -77,7 +75,7 @@ entry:
 ; CHECK-NEXT:         Body:            1080808080000B
 ; CHECK-NEXT:   - Type:            CUSTOM
 ; CHECK-NEXT:     Name:            linking
-; CHECK-NEXT:     Version:         1
+; CHECK-NEXT:     Version:         2
 ; CHECK-NEXT:     SymbolTable:
 ; CHECK-NEXT:       - Index:           0
 ; CHECK-NEXT:         Kind:            FUNCTION
diff --git a/test/MC/WebAssembly/basic-assembly-errors.s b/test/MC/WebAssembly/basic-assembly-errors.s
new file mode 100644
index 0000000000000000000000000000000000000000..eead082e93f7a222834e96b2a2f3959400a95d48
--- /dev/null
+++ b/test/MC/WebAssembly/basic-assembly-errors.s
@@ -0,0 +1,20 @@
+# RUN: not llvm-mc -triple=wasm32-unknown-unknown -mattr=+simd128,+nontrapping-fptoint,+exception-handling < %s 2>&1 | FileCheck %s
+
+# CHECK: End of block construct with no start: end_try
+    end_try
+test0:
+    .functype   test0 () -> ()
+# CHECK: Block construct type mismatch, expected: end_function, instead got: end_loop
+    end_loop
+    block
+# CHECK: Block construct type mismatch, expected: end_block, instead got: end_if
+    end_if
+    try
+    loop
+# CHECK: Block construct type mismatch, expected: end_loop, instead got: end_function
+# CHECK: error: Unmatched block construct(s) at function end: loop
+# CHECK: error: Unmatched block construct(s) at function end: try
+# CHECK: error: Unmatched block construct(s) at function end: block
+# CHECK: error: Unmatched block construct(s) at function end: function
+    end_function
+
diff --git a/test/MC/WebAssembly/basic-assembly.s b/test/MC/WebAssembly/basic-assembly.s
index 6d95db482917262d71f4f00c333390a37fa8b2c5..ab2925891520623c66f2f7e4f65716699a18463e 100644
--- a/test/MC/WebAssembly/basic-assembly.s
+++ b/test/MC/WebAssembly/basic-assembly.s
@@ -1,34 +1,31 @@
-# RUN: llvm-mc -triple=wasm32-unknown-unknown -mattr=+simd128,+nontrapping-fptoint,+exception-handling < %s | FileCheck %s
-# this one is just here to see if it converts to .o without errors, but doesn't check any output:
-# RUN: llvm-mc -triple=wasm32-unknown-unknown -filetype=obj -mattr=+simd128,+nontrapping-fptoint,+exception-handling < %s
+# RUN: llvm-mc -triple=wasm32-unknown-unknown -mattr=+unimplemented-simd128,+nontrapping-fptoint,+exception-handling < %s | FileCheck %s
+# Check that it converts to .o without errors, but don't check any output:
+# RUN: llvm-mc -triple=wasm32-unknown-unknown -filetype=obj -mattr=+unimplemented-simd128,+nontrapping-fptoint,+exception-handling -o %t.o < %s
 
-    .text
-    .section .text.main,"",@
-    .type    test0,@function
 test0:
     # Test all types:
     .functype   test0 (i32, i64) -> (i32)
     .eventtype  __cpp_exception i32
     .local      f32, f64, v128, v128
     # Explicit getlocal/setlocal:
-    get_local   2
-    set_local   2
+    local.get   2
+    local.set   2
     # Immediates:
     i32.const   -1
     f64.const   0x1.999999999999ap1
     v128.const  0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
     v128.const  0, 1, 2, 3, 4, 5, 6, 7
     # Indirect addressing:
-    get_local   0
+    local.get   0
     f64.store   0
     # Loops, conditionals, binary ops, calls etc:
-    block
+    block       i32
     i32.const   1
-    get_local   0
+    local.get   0
     i32.ge_s
     br_if       0        # 0: down to label0
 .LBB0_1:
-    loop             # label1:
+    loop        i32      # label1:
     call        something1@FUNCTION
     i64.const   1234
     i32.call    something2@FUNCTION
@@ -36,46 +33,56 @@ test0:
     call_indirect 0
     i32.const   1
     i32.add
-    tee_local   0
-    get_local   0
+    local.tee   0
+    local.get   0
     i32.lt_s
     br_if       0        # 0: up to label1
 .LBB0_2:
     end_loop
-    end_block                       # label0:
-    get_local   4
-    get_local   5
-    block
-    block
-    block
-    block
+    end_block            # label0:
+    local.get   4
+    local.get   5
+    block       void
+    block       i64
+    block       f32
+    block       f64
     br_table {0, 1, 2}   # 2 entries, default
     end_block            # first entry jumps here.
-    i32.const 1
+    i32.const   1
     br 2
     end_block            # second entry jumps here.
-    i32.const 2
+    i32.const   2
     br 1
     end_block            # default jumps here.
-    i32.const 3
+    i32.const   3
     end_block            # "switch" exit.
+    if          # void
+    if          i32
+    end_if
+    else
+    end_if
     f32x4.add
     # Test correct parsing of instructions with / and : in them:
     # TODO: enable once instruction has been added.
-    #i32x4.trunc_s/f32x4:sat
-    i32.trunc_s/f32
-    try
+    #i32x4.trunc_sat_f32x4_s
+    i32.trunc_f32_s
+    try         except_ref
 .LBB0_3:
-    i32.catch   0
+    catch
+    local.set 0
+    block       i32
+    local.get 0
+    br_on_exn 0, __cpp_exception@EVENT
+    rethrow
 .LBB0_4:
-    catch_all
-.LBB0_5:
+    end_block
     end_try
-    #i32.trunc_s:sat/f32
-    get_global  __stack_pointer@GLOBAL
+    i32.const 0
+    throw 0
+.LBB0_5:
+    #i32.trunc_sat_f32_s
+    global.get  __stack_pointer@GLOBAL
     end_function
-.Lfunc_end0:
-	.size	test0, .Lfunc_end0-test0
     .globaltype	__stack_pointer, i32
 
 # CHECK:           .text
@@ -83,21 +90,21 @@ test0:
 # CHECK-NEXT:      .functype test0 (i32, i64) -> (i32)
 # CHECK-NEXT:      .eventtype  __cpp_exception i32
 # CHECK-NEXT:      .local      f32, f64
-# CHECK-NEXT:      get_local   2
-# CHECK-NEXT:      set_local   2
+# CHECK-NEXT:      local.get   2
+# CHECK-NEXT:      local.set   2
 # CHECK-NEXT:      i32.const   -1
 # CHECK-NEXT:      f64.const   0x1.999999999999ap1
 # CHECK-NEXT:      v128.const  0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
 # CHECK-NEXT:      v128.const  0, 1, 2, 3, 4, 5, 6, 7
-# CHECK-NEXT:      get_local   0
+# CHECK-NEXT:      local.get   0
 # CHECK-NEXT:      f64.store   0:p2align=0
-# CHECK-NEXT:      block
+# CHECK-NEXT:      block       i32
 # CHECK-NEXT:      i32.const   1
-# CHECK-NEXT:      get_local   0
+# CHECK-NEXT:      local.get   0
 # CHECK-NEXT:      i32.ge_s
 # CHECK-NEXT:      br_if 0            # 0: down to label0
 # CHECK-NEXT:  .LBB0_1:
-# CHECK-NEXT:      loop                    # label1:
+# CHECK-NEXT:      loop        i32         # label1:
 # CHECK-NEXT:      call        something1@FUNCTION
 # CHECK-NEXT:      i64.const   1234
 # CHECK-NEXT:      i32.call    something2@FUNCTION
@@ -105,19 +112,19 @@ test0:
 # CHECK-NEXT:      call_indirect 0
 # CHECK-NEXT:      i32.const   1
 # CHECK-NEXT:      i32.add
-# CHECK-NEXT:      tee_local   0
-# CHECK-NEXT:      get_local   0
+# CHECK-NEXT:      local.tee   0
+# CHECK-NEXT:      local.get   0
 # CHECK-NEXT:      i32.lt_s
 # CHECK-NEXT:      br_if 0            # 0: up to label1
 # CHECK-NEXT:  .LBB0_2:
 # CHECK-NEXT:      end_loop
 # CHECK-NEXT:      end_block                       # label0:
-# CHECK-NEXT:      get_local   4
-# CHECK-NEXT:      get_local   5
-# CHECK-NEXT:      block
-# CHECK-NEXT:      block
-# CHECK-NEXT:      block
+# CHECK-NEXT:      local.get   4
+# CHECK-NEXT:      local.get   5
 # CHECK-NEXT:      block
+# CHECK-NEXT:      block       i64
+# CHECK-NEXT:      block       f32
+# CHECK-NEXT:      block       f64
 # CHECK-NEXT:      br_table {0, 1, 2}  # 1: down to label4
 # CHECK-NEXT:                          # 2: down to label3
 # CHECK-NEXT:      end_block           # label5:
@@ -129,16 +136,28 @@ test0:
 # CHECK-NEXT:      end_block           # label3:
 # CHECK-NEXT:      i32.const 3
 # CHECK-NEXT:      end_block           # label2:
+# CHECK-NEXT:      if
+# CHECK-NEXT:      if          i32
+# CHECK-NEXT:      end_if
+# CHECK-NEXT:      else
+# CHECK-NEXT:      end_if
 # CHECK-NEXT:      f32x4.add
-# CHECK-NEXT:      i32.trunc_s/f32
-# CHECK-NEXT:      try
+# CHECK-NEXT:      i32.trunc_f32_s
+# CHECK-NEXT:      try         except_ref
 # CHECK-NEXT:  .LBB0_3:
-# CHECK-NEXT:      i32.catch   0
+# CHECK-NEXT:      catch
+# CHECK-NEXT:      local.set 0
+# CHECK-NEXT:      block       i32
+# CHECK-NEXT:      local.get 0
+# CHECK-NEXT:      br_on_exn 0, __cpp_exception@EVENT
+# CHECK-NEXT:      rethrow
 # CHECK-NEXT:  .LBB0_4:
-# CHECK-NEXT:      catch_all
-# CHECK-NEXT:  .LBB0_5:
+# CHECK-NEXT:      end_block
 # CHECK-NEXT:      end_try
-# CHECK-NEXT:      get_global  __stack_pointer@GLOBAL
+# CHECK-NEXT:      i32.const 0
+# CHECK-NEXT:      throw 0
+# CHECK-NEXT:  .LBB0_5:
+# CHECK-NEXT:      global.get  __stack_pointer@GLOBAL
 # CHECK-NEXT:      end_function
 
 # CHECK:           .globaltype	__stack_pointer, i32
diff --git a/test/MC/WebAssembly/bss.ll b/test/MC/WebAssembly/bss.ll
index 2c0cdf289ec44e091dc66779ce1d7be21444945e..1ab3ec1339bad648057eb6aa82f37cc364d6510c 100644
--- a/test/MC/WebAssembly/bss.ll
+++ b/test/MC/WebAssembly/bss.ll
@@ -64,18 +64,18 @@ target triple = "wasm32-unknown-unknown"
 ; CHECK-NEXT:     SegmentInfo:    
 ; CHECK-NEXT:       - Index:           0
 ; CHECK-NEXT:         Name:            .bss.g0
-; CHECK-NEXT:         Alignment:       4
+; CHECK-NEXT:         Alignment:       2
 ; CHECK-NEXT:         Flags:           [ ]
 ; CHECK-NEXT:       - Index:           1
 ; CHECK-NEXT:         Name:            .bss.g1
-; CHECK-NEXT:         Alignment:       4
+; CHECK-NEXT:         Alignment:       2
 ; CHECK-NEXT:         Flags:           [ ]
 ; CHECK-NEXT:       - Index:           2
 ; CHECK-NEXT:         Name:            .bss.foo
-; CHECK-NEXT:         Alignment:       1
+; CHECK-NEXT:         Alignment:       0
 ; CHECK-NEXT:         Flags:           [ ]
 ; CHECK-NEXT:       - Index:           3
 ; CHECK-NEXT:         Name:            .bss.bar
-; CHECK-NEXT:         Alignment:       1
+; CHECK-NEXT:         Alignment:       0
 ; CHECK-NEXT:         Flags:           [ ]
 ; CHECK-NEXT: ...
diff --git a/test/MC/WebAssembly/comdat.ll b/test/MC/WebAssembly/comdat.ll
index 7bba78ba4ef1b2af78de85194fd7c0e1de45787b..9b61930cc7e0df921d021d8017b749ced01ce0b8 100644
--- a/test/MC/WebAssembly/comdat.ll
+++ b/test/MC/WebAssembly/comdat.ll
@@ -41,7 +41,7 @@ define linkonce_odr i32 @sharedFn() #1 comdat($sharedComdat) {
 ; CHECK-NEXT:         Field:           __indirect_function_table
 ; CHECK-NEXT:         Kind:            TABLE
 ; CHECK-NEXT:         Table:           
-; CHECK-NEXT:           ElemType:        ANYFUNC
+; CHECK-NEXT:           ElemType:        FUNCREF
 ; CHECK-NEXT:           Limits:          
 ; CHECK-NEXT:             Initial:         0x00000000
 ; CHECK-NEXT:       - Module:          env
@@ -52,7 +52,7 @@ define linkonce_odr i32 @sharedFn() #1 comdat($sharedComdat) {
 ; CHECK-NEXT:     FunctionTypes:   [ 0, 0, 0 ]
 ; CHECK-NEXT:  - Type:            CODE
 ; CHECK-NEXT:    Relocations:
-; CHECK-NEXT:      - Type:            R_WEBASSEMBLY_FUNCTION_INDEX_LEB
+; CHECK-NEXT:      - Type:            R_WASM_FUNCTION_INDEX_LEB
 ; CHECK-NEXT:        Index:           1
 ; CHECK-NEXT:        Offset:          0x00000004
 ; CHECK-NEXT:    Functions:
@@ -75,7 +75,7 @@ define linkonce_odr i32 @sharedFn() #1 comdat($sharedComdat) {
 ; CHECK-NEXT:        Content:         '616263'
 ; CHECK-NEXT:  - Type:            CUSTOM
 ; CHECK-NEXT:    Name:            linking
-; CHECK-NEXT:    Version:         1
+; CHECK-NEXT:    Version:         2
 ; CHECK-NEXT:    SymbolTable:
 ; CHECK-NEXT:       - Index:           0
 ; CHECK-NEXT:         Kind:            FUNCTION
@@ -106,7 +106,7 @@ define linkonce_odr i32 @sharedFn() #1 comdat($sharedComdat) {
 ; CHECK-NEXT:    SegmentInfo:
 ; CHECK-NEXT:      - Index:           0
 ; CHECK-NEXT:        Name:            .rodata.constantData
-; CHECK-NEXT:        Alignment:       1
+; CHECK-NEXT:        Alignment:       0
 ; CHECK-NEXT:        Flags:           [  ]
 ; CHECK-NEXT:    Comdats:
 ; CHECK-NEXT:        Name:            basicInlineFn
diff --git a/test/MC/WebAssembly/custom-sections.ll b/test/MC/WebAssembly/custom-sections.ll
index 8ee04e0b0449dc6513ecbec51bd186320636f68c..f662d55352bbdb8f6ea22f93d2beb20154fba898 100644
--- a/test/MC/WebAssembly/custom-sections.ll
+++ b/test/MC/WebAssembly/custom-sections.ll
@@ -9,6 +9,9 @@ target triple = "wasm32-unknown-unknown"
 !2 = !{ !"green", !"qux" }
 !wasm.custom_sections = !{ !0, !1, !2 }
 
+!3 = !{ !"clang version 123"}
+!llvm.ident = !{!3}
+
 ; CHECK:  Section {
 ; CHECK:    Type: CUSTOM (0x0)
 ; CHECK:    Size: 3
@@ -21,3 +24,9 @@ target triple = "wasm32-unknown-unknown"
 ; CHECK:    Offset: 85
 ; CHECK:    Name: green
 ; CHECK:  }
+; CHECK:  Section {
+; CHECK:    Type: CUSTOM (0x0)
+; CHECK:    Size: 25
+; CHECK:    Offset: 118
+; CHECK:    Name: producers
+; CHECK:  }
diff --git a/test/MC/WebAssembly/debug-info.ll b/test/MC/WebAssembly/debug-info.ll
index 05e33927917100fe7f8f054aa97fa1f65407aa92..fe4a4cf698319773543f31baaf27101ced305ff0 100644
--- a/test/MC/WebAssembly/debug-info.ll
+++ b/test/MC/WebAssembly/debug-info.ll
@@ -124,77 +124,112 @@
 ; CHECK-NEXT:    Offset: 991
 ; CHECK-NEXT:    Name: reloc..debug_line
 ; CHECK-NEXT:  }
+; CHECK-NEXT:  Section {
+; CHECK-NEXT:    Type: CUSTOM (0x0)
+; CHECK-NEXT:    Size: 77
+; CHECK-NEXT:    Offset: 1021
+; CHECK-NEXT:    Name: producers
+; CHECK-NEXT:  }
 ; CHECK-NEXT:]
 ; CHECK-NEXT:Relocations [
 ; CHECK-NEXT:  Section (6) DATA {
-; CHECK-NEXT:    0x6 R_WEBASSEMBLY_MEMORY_ADDR_I32 myextern 0
-; CHECK-NEXT:    0xF R_WEBASSEMBLY_TABLE_INDEX_I32 f2
+; CHECK-NEXT:    0x6 R_WASM_MEMORY_ADDR_I32 myextern 0
+; CHECK-NEXT:    0xF R_WASM_TABLE_INDEX_I32 f2
 ; CHECK-NEXT:  }
 ; CHECK-NEXT:  Section (9) .debug_info {
-; CHECK-NEXT:    0x6 R_WEBASSEMBLY_SECTION_OFFSET_I32 .debug_abbrev 0
-; CHECK-NEXT:    0xC R_WEBASSEMBLY_SECTION_OFFSET_I32 .debug_str 0
-; CHECK-NEXT:    0x12 R_WEBASSEMBLY_SECTION_OFFSET_I32 .debug_str 55
-; CHECK-NEXT:    0x16 R_WEBASSEMBLY_SECTION_OFFSET_I32 .debug_line 0
-; CHECK-NEXT:    0x1A R_WEBASSEMBLY_SECTION_OFFSET_I32 .debug_str 62
-; CHECK-NEXT:    0x1E R_WEBASSEMBLY_FUNCTION_OFFSET_I32 f2 0
-; CHECK-NEXT:    0x27 R_WEBASSEMBLY_SECTION_OFFSET_I32 .debug_str 105
-; CHECK-NEXT:    0x33 R_WEBASSEMBLY_MEMORY_ADDR_I32 foo 0
-; CHECK-NEXT:    0x3D R_WEBASSEMBLY_SECTION_OFFSET_I32 .debug_str 109
-; CHECK-NEXT:    0x44 R_WEBASSEMBLY_SECTION_OFFSET_I32 .debug_str 113
-; CHECK-NEXT:    0x50 R_WEBASSEMBLY_MEMORY_ADDR_I32 ptr2 0
-; CHECK-NEXT:    0x5B R_WEBASSEMBLY_FUNCTION_OFFSET_I32 f2 0
-; CHECK-NEXT:    0x63 R_WEBASSEMBLY_SECTION_OFFSET_I32 .debug_str 118
+; CHECK-NEXT:    0x6 R_WASM_SECTION_OFFSET_I32 .debug_abbrev 0
+; CHECK-NEXT:    0xC R_WASM_SECTION_OFFSET_I32 .debug_str 0
+; CHECK-NEXT:    0x12 R_WASM_SECTION_OFFSET_I32 .debug_str 55
+; CHECK-NEXT:    0x16 R_WASM_SECTION_OFFSET_I32 .debug_line 0
+; CHECK-NEXT:    0x1A R_WASM_SECTION_OFFSET_I32 .debug_str 62
+; CHECK-NEXT:    0x1E R_WASM_FUNCTION_OFFSET_I32 f2 0
+; CHECK-NEXT:    0x27 R_WASM_SECTION_OFFSET_I32 .debug_str 105
+; CHECK-NEXT:    0x33 R_WASM_MEMORY_ADDR_I32 foo 0
+; CHECK-NEXT:    0x3D R_WASM_SECTION_OFFSET_I32 .debug_str 109
+; CHECK-NEXT:    0x44 R_WASM_SECTION_OFFSET_I32 .debug_str 113
+; CHECK-NEXT:    0x50 R_WASM_MEMORY_ADDR_I32 ptr2 0
+; CHECK-NEXT:    0x5B R_WASM_FUNCTION_OFFSET_I32 f2 0
+; CHECK-NEXT:    0x63 R_WASM_SECTION_OFFSET_I32 .debug_str 118
 ; CHECK-NEXT:  }
 ; CHECK-NEXT:  Section (11) .debug_pubnames {
-; CHECK-NEXT:    0x6 R_WEBASSEMBLY_SECTION_OFFSET_I32 .debug_info 0
+; CHECK-NEXT:    0x6 R_WASM_SECTION_OFFSET_I32 .debug_info 0
 ; CHECK-NEXT:  }
 ; CHECK-NEXT:  Section (12) .debug_pubtypes {
-; CHECK-NEXT:    0x6 R_WEBASSEMBLY_SECTION_OFFSET_I32 .debug_info 0
+; CHECK-NEXT:    0x6 R_WASM_SECTION_OFFSET_I32 .debug_info 0
 ; CHECK-NEXT:  }
 ; CHECK-NEXT:  Section (13) .debug_line {
-; CHECK-NEXT:    0x2B R_WEBASSEMBLY_FUNCTION_OFFSET_I32 f2 0
+; CHECK-NEXT:    0x2B R_WASM_FUNCTION_OFFSET_I32 f2 0
 ; CHECK-NEXT:  }
 ; CHECK-NEXT:]
 ; CHECK-NEXT:Symbols [
 ; CHECK-NEXT:  Symbol {
 ; CHECK-NEXT:    Name: f2
 ; CHECK-NEXT:    Type: FUNCTION (0x0)
-; CHECK-NEXT:    Flags: 0x4
+; CHECK-NEXT:    Flags [ (0x4)
+; CHECK-NEXT:      VISIBILITY_HIDDEN (0x4)
+; CHECK-NEXT:    ]
+; CHECK-NEXT:    ElementIndex: 0x0
 ; CHECK-NEXT:  }
 ; CHECK-NEXT:  Symbol {
 ; CHECK-NEXT:    Name: foo
 ; CHECK-NEXT:    Type: DATA (0x1)
-; CHECK-NEXT:    Flags: 0x4
+; CHECK-NEXT:    Flags [ (0x4)
+; CHECK-NEXT:      VISIBILITY_HIDDEN (0x4)
+; CHECK-NEXT:    ]
+; CHECK-NEXT:    Offset: 0x0
+; CHECK-NEXT:    Segment: 0x0
+; CHECK-NEXT:    Size: 0x4
 ; CHECK-NEXT:  }
 ; CHECK-NEXT:  Symbol {
 ; CHECK-NEXT:    Name: myextern
 ; CHECK-NEXT:    Type: DATA (0x1)
-; CHECK-NEXT:    Flags: 0x10
+; CHECK-NEXT:    Flags [ (0x10)
+; CHECK-NEXT:      UNDEFINED (0x10)
+; CHECK-NEXT:    ]
+; CHECK-NEXT:    ImportName:
+; CHECK-NEXT:    ImportModule:
 ; CHECK-NEXT:  }
 ; CHECK-NEXT:  Symbol {
 ; CHECK-NEXT:    Name: ptr2
 ; CHECK-NEXT:    Type: DATA (0x1)
-; CHECK-NEXT:    Flags: 0x4
+; CHECK-NEXT:    Flags [ (0x4)
+; CHECK-NEXT:      VISIBILITY_HIDDEN (0x4)
+; CHECK-NEXT:    ]
+; CHECK-NEXT:    Offset: 0x0
+; CHECK-NEXT:    Segment: 0x1
+; CHECK-NEXT:    Size: 0x4
 ; CHECK-NEXT:  }
 ; CHECK-NEXT:  Symbol {
 ; CHECK-NEXT:    Name: .debug_str
 ; CHECK-NEXT:    Type: SECTION (0x3)
-; CHECK-NEXT:    Flags: 0x2
+; CHECK-NEXT:    Flags [ (0x2)
+; CHECK-NEXT:      BINDING_LOCAL (0x2)
+; CHECK-NEXT:    ]
+; CHECK-NEXT:    ElementIndex: 0x6
 ; CHECK-NEXT:  }
 ; CHECK-NEXT:  Symbol {
 ; CHECK-NEXT:    Name: .debug_abbrev
 ; CHECK-NEXT:    Type: SECTION (0x3)
-; CHECK-NEXT:    Flags: 0x2
+; CHECK-NEXT:    Flags [ (0x2)
+; CHECK-NEXT:      BINDING_LOCAL (0x2)
+; CHECK-NEXT:    ]
+; CHECK-NEXT:    ElementIndex: 0x7
 ; CHECK-NEXT:  }
 ; CHECK-NEXT:  Symbol {
 ; CHECK-NEXT:    Name: .debug_info
 ; CHECK-NEXT:    Type: SECTION (0x3)
-; CHECK-NEXT:    Flags: 0x2
+; CHECK-NEXT:    Flags [ (0x2)
+; CHECK-NEXT:      BINDING_LOCAL (0x2)
+; CHECK-NEXT:    ]
+; CHECK-NEXT:    ElementIndex: 0x8
 ; CHECK-NEXT:  }
 ; CHECK-NEXT:  Symbol {
 ; CHECK-NEXT:    Name: .debug_line
 ; CHECK-NEXT:    Type: SECTION (0x3)
-; CHECK-NEXT:    Flags: 0x2
+; CHECK-NEXT:    Flags [ (0x2)
+; CHECK-NEXT:      BINDING_LOCAL (0x2)
+; CHECK-NEXT:    ]
+; CHECK-NEXT:    ElementIndex: 0xC
 ; CHECK-NEXT:  }
 ; CHECK-NEXT:]
 
diff --git a/test/MC/WebAssembly/event-section.ll b/test/MC/WebAssembly/event-section.ll
index 213817007f0287fac2ce1f768c36ebda8dab09f5..b42ddd20570cab2cd61bce26d4a560ed2366cb7e 100644
--- a/test/MC/WebAssembly/event-section.ll
+++ b/test/MC/WebAssembly/event-section.ll
@@ -35,16 +35,16 @@ define i32 @test_throw1(i8* %p) {
 
 ; CHECK-NEXT:   - Type:            CODE
 ; CHECK-NEXT:     Relocations:
-; CHECK-NEXT:       - Type:            R_WEBASSEMBLY_EVENT_INDEX_LEB
+; CHECK-NEXT:       - Type:            R_WASM_EVENT_INDEX_LEB
 ; CHECK-NEXT:         Index:           1
 ; CHECK-NEXT:         Offset:          0x00000006
-; CHECK-NEXT:       - Type:            R_WEBASSEMBLY_EVENT_INDEX_LEB
+; CHECK-NEXT:       - Type:            R_WASM_EVENT_INDEX_LEB
 ; CHECK-NEXT:         Index:           1
 ; CHECK-NEXT:         Offset:          0x00000011
 
 ; CHECK:        - Type:            CUSTOM
 ; CHECK-NEXT:     Name:            linking
-; CHECK-NEXT:     Version:         1
+; CHECK-NEXT:     Version:         2
 ; CHECK-NEXT:     SymbolTable:
 
 ; CHECK:            - Index:           1
diff --git a/test/MC/WebAssembly/explicit-sections.ll b/test/MC/WebAssembly/explicit-sections.ll
index e069cee6ff4ffa5a05e90f52d46b9773ee42adaf..ab2bb163fff4145024eadd90676e69f5d4dc343a 100644
--- a/test/MC/WebAssembly/explicit-sections.ll
+++ b/test/MC/WebAssembly/explicit-sections.ll
@@ -60,14 +60,14 @@ target triple = "wasm32-unknown-unknown"
 ; CHECK-NEXT:     SegmentInfo:    
 ; CHECK-NEXT:       - Index:           0
 ; CHECK-NEXT:         Name:            .data.global0
-; CHECK-NEXT:         Alignment:       8
+; CHECK-NEXT:         Alignment:       3
 ; CHECK-NEXT:         Flags:           [ ]
 ; CHECK-NEXT:       - Index:           1
 ; CHECK-NEXT:         Name:            .sec1
-; CHECK-NEXT:         Alignment:       8
+; CHECK-NEXT:         Alignment:       3
 ; CHECK-NEXT:         Flags:           [ ]
 ; CHECK-NEXT:       - Index:           2
 ; CHECK-NEXT:         Name:            .sec2
-; CHECK-NEXT:         Alignment:       8
+; CHECK-NEXT:         Alignment:       3
 ; CHECK-NEXT:         Flags:           [ ]
 ; CHECK-NEXT: ...
diff --git a/test/MC/WebAssembly/external-data.ll b/test/MC/WebAssembly/external-data.ll
index 5c92475e214ce42fd59069271f7086e517618645..4f172a146a665c6b8ce8a46af8d3b88b8ba3d231 100644
--- a/test/MC/WebAssembly/external-data.ll
+++ b/test/MC/WebAssembly/external-data.ll
@@ -12,7 +12,7 @@ target triple = "wasm32-unknown-unknown"
 
 ; CHECK:        - Type:            DATA
 ; CHECK-NEXT:     Relocations:
-; CHECK-NEXT:       - Type:            R_WEBASSEMBLY_MEMORY_ADDR_I32
+; CHECK-NEXT:       - Type:            R_WASM_MEMORY_ADDR_I32
 ; CHECK-NEXT:         Index:           2
 ; CHECK-NEXT:         Offset:          0x00000013
 ; CHECK-NEXT:     Segments:
diff --git a/test/MC/WebAssembly/external-func-address.ll b/test/MC/WebAssembly/external-func-address.ll
index 60ec23a1a8ed3d04722413f4b15a025c79fb03ac..95e679ab65d1cd6136bd67d2f0869cdae605a77f 100644
--- a/test/MC/WebAssembly/external-func-address.ll
+++ b/test/MC/WebAssembly/external-func-address.ll
@@ -8,7 +8,7 @@ target triple = "wasm32-unknown-unknown"
 declare void @f0(i32) #0
 @ptr_to_f0 = hidden global void (i32)* @f0, align 4
 
-attributes #0 = { "wasm-import-module"="somewhere" }
+attributes #0 = { "wasm-import-module"="somewhere" "wasm-import-name"="something" }
 
 declare void @f1(i32) #1
 @ptr_to_f1 = hidden global void (i32)* @f1, align 4
@@ -47,7 +47,7 @@ define void @call(i32) {
 ; CHECK-NEXT:         Kind:            FUNCTION
 ; CHECK-NEXT:         SigIndex:        1
 ; CHECK:            - Module:          somewhere
-; CHECK-NEXT:         Field:           f0
+; CHECK-NEXT:         Field:           something
 ; CHECK:            - Module:          env
 ; CHECK-NEXT:         Field:           f1
 ; CHECK-NEXT:         Kind:            FUNCTION
@@ -60,6 +60,6 @@ define void @call(i32) {
 ; CHECK-NEXT:         Functions:       [ 1, 2 ]
 ; CHECK:        - Type:            DATA
 ; CHECK-NEXT:     Relocations:
-; CHECK-NEXT:       - Type:            R_WEBASSEMBLY_TABLE_INDEX_I32
+; CHECK-NEXT:       - Type:            R_WASM_TABLE_INDEX_I32
 ; CHECK-NEXT:         Index:           3
 ; CHECK-NEXT:         Offset:          0x00000006
diff --git a/test/MC/WebAssembly/func-address.ll b/test/MC/WebAssembly/func-address.ll
index f9128e6eecded8a3c605005df45a5bceeb2c71ef..e7bea2808f398f7824ba077a6d0e4ad67e4d849a 100644
--- a/test/MC/WebAssembly/func-address.ll
+++ b/test/MC/WebAssembly/func-address.ll
@@ -30,22 +30,22 @@ entry:
 ; CHECK:      Relocations [
 ; CHECK-NEXT:   Section (5) CODE {
 ; CHECK-NEXT:     Relocation {
-; CHECK-NEXT:       Type: R_WEBASSEMBLY_FUNCTION_INDEX_LEB (0)
+; CHECK-NEXT:       Type: R_WASM_FUNCTION_INDEX_LEB (0)
 ; CHECK-NEXT:       Offset: 0x4
 ; CHECK-NEXT:       Symbol: import1
 ; CHECK-NEXT:     }
 ; CHECK-NEXT:     Relocation {
-; CHECK-NEXT:       Type: R_WEBASSEMBLY_FUNCTION_INDEX_LEB (0)
+; CHECK-NEXT:       Type: R_WASM_FUNCTION_INDEX_LEB (0)
 ; CHECK-NEXT:       Offset: 0xB
 ; CHECK-NEXT:       Symbol: import2
 ; CHECK-NEXT:     }
 ; CHECK-NEXT:     Relocation {
-; CHECK-NEXT:       Type: R_WEBASSEMBLY_GLOBAL_INDEX_LEB (7)
+; CHECK-NEXT:       Type: R_WASM_GLOBAL_INDEX_LEB (7)
 ; CHECK-NEXT:       Offset: 0x15
 ; CHECK-NEXT:       Symbol: __stack_pointer
 ; CHECK-NEXT:     }
 ; CHECK-NEXT:     Relocation {
-; CHECK-NEXT:       Type: R_WEBASSEMBLY_TABLE_INDEX_SLEB (1)
+; CHECK-NEXT:       Type: R_WASM_TABLE_INDEX_SLEB (1)
 ; CHECK-NEXT:       Offset: 0x1E
 ; CHECK-NEXT:       Symbol: import3
 ; CHECK-NEXT:     }
diff --git a/test/MC/WebAssembly/function-alias.ll b/test/MC/WebAssembly/function-alias.ll
new file mode 100644
index 0000000000000000000000000000000000000000..0a3c53989bc5b0b8bb5b706d97b392be15425242
--- /dev/null
+++ b/test/MC/WebAssembly/function-alias.ll
@@ -0,0 +1,35 @@
+; RUN: llc -filetype=obj %s -o - | llvm-readobj -symbols | FileCheck %s
+
+target datalayout = "e-m:e-p:32:32-i64:64-n32:64-S128"
+target triple = "wasm32-unknown-unknown-wasm"
+
+@foo = alias i8, bitcast (i8* ()* @func to i8*)
+@bar = alias i8* (), i8* ()* @func
+
+define i8* @func() {
+  ret i8* @foo;
+}
+
+; CHECK:      Symbols [
+; CHECK-NEXT:   Symbol {
+; CHECK-NEXT:     Name: func
+; CHECK-NEXT:     Type: FUNCTION (0x0)
+; CHECK-NEXT:     Flags [ (0x0)
+; CHECK-NEXT:     ]
+; CHECK-NEXT:     ElementIndex: 0x0
+; CHECK-NEXT:   }
+; CHECK-NEXT:   Symbol {
+; CHECK-NEXT:     Name: foo
+; CHECK-NEXT:     Type: FUNCTION (0x0)
+; CHECK-NEXT:     Flags [ (0x0)
+; CHECK-NEXT:     ]
+; CHECK-NEXT:     ElementIndex: 0x0
+; CHECK-NEXT:   }
+; CHECK-NEXT:   Symbol {
+; CHECK-NEXT:     Name: bar
+; CHECK-NEXT:     Type: FUNCTION (0x0)
+; CHECK-NEXT:     Flags [ (0x0)
+; CHECK-NEXT:     ]
+; CHECK-NEXT:     ElementIndex: 0x0
+; CHECK-NEXT:   }
+; CHECK-NEXT: ]
diff --git a/test/MC/WebAssembly/function-sections.ll b/test/MC/WebAssembly/function-sections.ll
index 8f1e29adccf4f443dacfa5ed0bfbf368e1269733..b9b9c28376eb4fcb5494b66a12c6d6f23a8242f9 100644
--- a/test/MC/WebAssembly/function-sections.ll
+++ b/test/MC/WebAssembly/function-sections.ll
@@ -14,7 +14,7 @@ entry:
 
 ; CHECK:        - Type:            CUSTOM
 ; CHECK-NEXT:     Name:            linking
-; CHECK-NEXT:     Version:         1
+; CHECK-NEXT:     Version:         2
 ; CHECK-NEXT:     SymbolTable:     
 ; CHECK-NEXT:       - Index:           0
 ; CHECK-NEXT:         Kind:            FUNCTION
diff --git a/test/MC/WebAssembly/global-ctor-dtor.ll b/test/MC/WebAssembly/global-ctor-dtor.ll
index 7464d66c6971e5db9e6391b938c0d690cbf3ccf0..89d519383fb9f2621bcdab91b4c1a3cf1e1e6aa4 100644
--- a/test/MC/WebAssembly/global-ctor-dtor.ll
+++ b/test/MC/WebAssembly/global-ctor-dtor.ll
@@ -30,7 +30,7 @@ declare void @func3()
 ; CHECK-NEXT:         Field:           __indirect_function_table
 ; CHECK-NEXT:         Kind:            TABLE
 ; CHECK-NEXT:         Table:           
-; CHECK-NEXT:           ElemType:        ANYFUNC
+; CHECK-NEXT:           ElemType:        FUNCREF
 ; CHECK-NEXT:           Limits:          
 ; CHECK-NEXT:             Initial:         0x00000002
 ; CHECK-NEXT:       - Module:          env
@@ -63,28 +63,28 @@ declare void @func3()
 ; CHECK-NEXT:         Functions:       [ 5, 7 ]
 ; CHECK-NEXT:   - Type:            CODE
 ; CHECK-NEXT:     Relocations:     
-; CHECK-NEXT:       - Type:            R_WEBASSEMBLY_FUNCTION_INDEX_LEB
+; CHECK-NEXT:       - Type:            R_WASM_FUNCTION_INDEX_LEB
 ; CHECK-NEXT:         Index:           1
 ; CHECK-NEXT:         Offset:          0x00000004
-; CHECK-NEXT:       - Type:            R_WEBASSEMBLY_TABLE_INDEX_SLEB
+; CHECK-NEXT:       - Type:            R_WASM_TABLE_INDEX_SLEB
 ; CHECK-NEXT:         Index:           0
 ; CHECK-NEXT:         Offset:          0x0000000F
-; CHECK-NEXT:       - Type:            R_WEBASSEMBLY_MEMORY_ADDR_SLEB
+; CHECK-NEXT:       - Type:            R_WASM_MEMORY_ADDR_SLEB
 ; CHECK-NEXT:         Index:           3
 ; CHECK-NEXT:         Offset:          0x00000017
-; CHECK-NEXT:       - Type:            R_WEBASSEMBLY_FUNCTION_INDEX_LEB
+; CHECK-NEXT:       - Type:            R_WASM_FUNCTION_INDEX_LEB
 ; CHECK-NEXT:         Index:           4
 ; CHECK-NEXT:         Offset:          0x0000001D
-; CHECK-NEXT:       - Type:            R_WEBASSEMBLY_FUNCTION_INDEX_LEB
+; CHECK-NEXT:       - Type:            R_WASM_FUNCTION_INDEX_LEB
 ; CHECK-NEXT:         Index:           6
 ; CHECK-NEXT:         Offset:          0x0000002C
-; CHECK-NEXT:       - Type:            R_WEBASSEMBLY_TABLE_INDEX_SLEB
+; CHECK-NEXT:       - Type:            R_WASM_TABLE_INDEX_SLEB
 ; CHECK-NEXT:         Index:           5
 ; CHECK-NEXT:         Offset:          0x00000037
-; CHECK-NEXT:       - Type:            R_WEBASSEMBLY_MEMORY_ADDR_SLEB
+; CHECK-NEXT:       - Type:            R_WASM_MEMORY_ADDR_SLEB
 ; CHECK-NEXT:         Index:           3
 ; CHECK-NEXT:         Offset:          0x0000003F
-; CHECK-NEXT:       - Type:            R_WEBASSEMBLY_FUNCTION_INDEX_LEB
+; CHECK-NEXT:       - Type:            R_WASM_FUNCTION_INDEX_LEB
 ; CHECK-NEXT:         Index:           4
 ; CHECK-NEXT:         Offset:          0x00000045
 ; CHECK-NEXT:     Functions:       
@@ -110,7 +110,7 @@ declare void @func3()
 ; CHECK-NEXT:         Content:         '01040000'
 ; CHECK-NEXT:   - Type:            CUSTOM
 ; CHECK-NEXT:     Name:            linking
-; CHECK-NEXT:     Version:         1
+; CHECK-NEXT:     Version:         2
 ; CHECK-NEXT:     SymbolTable:
 ; CHECK-NEXT:       - Index:           0
 ; CHECK-NEXT:         Kind:            FUNCTION
@@ -170,7 +170,7 @@ declare void @func3()
 ; CHECK-NEXT:     SegmentInfo:
 ; CHECK-NEXT:       - Index:           0
 ; CHECK-NEXT:         Name:            .data.global1
-; CHECK-NEXT:         Alignment:       8
+; CHECK-NEXT:         Alignment:       3
 ; CHECK-NEXT:         Flags:           [ ]
 ; CHECK-NEXT:     InitFunctions:     
 ; CHECK-NEXT:       - Priority: 42
diff --git a/test/MC/WebAssembly/import-module.ll b/test/MC/WebAssembly/import-module.ll
new file mode 100644
index 0000000000000000000000000000000000000000..461d5c20ae9b78eef797464b1efcec96c0469a71
--- /dev/null
+++ b/test/MC/WebAssembly/import-module.ll
@@ -0,0 +1,31 @@
+; RUN: llc -filetype=obj %s -o - | obj2yaml | FileCheck %s
+
+target triple = "wasm32-unknown-unknown"
+
+define void @test() {
+  call void @foo()
+  call void @plain()
+  ret void
+}
+
+declare void @foo() #0
+declare void @plain()
+
+attributes #0 = { "wasm-import-module"="bar" "wasm-import-name"="qux" }
+
+; CHECK:        - Type:            IMPORT
+; CHECK-NEXT:     Imports:         
+; CHECK:            - Module:          bar
+; CHECK-NEXT:         Field:           qux
+; CHECK-NEXT:         Kind:            FUNCTION
+
+; CHECK:            - Module:          env
+; CHECK-NEXT:         Field:           plain
+; CHECK-NEXT:         Kind:            FUNCTION
+
+; CHECK:        - Type:            CUSTOM
+; CHECK:              Name:            foo
+; CHECK-NEXT:         Flags:           [ UNDEFINED ]
+
+; CHECK:              Name:            plain
+; CHECK-NEXT:         Flags:           [ UNDEFINED ]
diff --git a/test/MC/WebAssembly/no-dead-strip.ll b/test/MC/WebAssembly/no-dead-strip.ll
new file mode 100644
index 0000000000000000000000000000000000000000..1810c49c7c131a44cbb86bc22caa15cf33a0c090
--- /dev/null
+++ b/test/MC/WebAssembly/no-dead-strip.ll
@@ -0,0 +1,21 @@
+; RUN: llc -filetype=obj -wasm-keep-registers %s -o - | llvm-readobj --symbols | FileCheck %s
+
+target triple = "wasm32-unknown-unknown"
+
+@llvm.used = appending global [1 x i8*] [i8* bitcast (i32 ()* @foo to i8*)], section "llvm.metadata"
+
+define i32 @foo() {
+entry:
+    ret i32 0
+}
+
+; CHECK:      Symbols [
+; CHECK-NEXT:   Symbol {
+; CHECK-NEXT:     Name: foo
+; CHECK-NEXT:     Type: FUNCTION (0x0)
+; CHECK-NEXT:     Flags [ (0x20)
+; CHECK-NEXT:       EXPORTED (0x20)
+; CHECK-NEXT:     ]
+; CHECK-NEXT:     ElementIndex: 0x0
+; CHECK-NEXT:   }
+; CHECK-NEXT: ]
diff --git a/test/MC/WebAssembly/null-output.s b/test/MC/WebAssembly/null-output.s
new file mode 100644
index 0000000000000000000000000000000000000000..a25d095e0cbef84595872d8c965bf5d97f9dce65
--- /dev/null
+++ b/test/MC/WebAssembly/null-output.s
@@ -0,0 +1,10 @@
+# RUN: llvm-mc -triple=wasm32-unknown-unknown -filetype=obj -o /dev/null < %s
+
+    .text
+    .section .text.main,"",@
+    .type    main,@function
+main:
+    .functype   main (i32, i32) -> (i32)
+    end_function
+.Lfunc_end0:
+    .size main, .Lfunc_end0-main
diff --git a/test/MC/WebAssembly/objdump.s b/test/MC/WebAssembly/objdump.s
new file mode 100644
index 0000000000000000000000000000000000000000..f1cedc7db86e9049906f5663582ae15c2831065a
--- /dev/null
+++ b/test/MC/WebAssembly/objdump.s
@@ -0,0 +1,26 @@
+# RUN: llvm-mc -triple=wasm32-unknown-unknown -filetype=obj -o %t.o -mattr=+simd128,+nontrapping-fptoint,+exception-handling < %s
+# RUN: llvm-objdump -triple=wasm32-unknown-unknown -disassemble %t.o | FileCheck %s
+
+test0:
+    .functype   test0 (i32, i64) -> (i32)
+    .local      f32, f64, v128, v128
+    local.get   2
+    end_function
+
+test1:
+    .functype   test1 (i32, i64) -> (i32)
+    .local      i32, i64, except_ref
+    local.get   3
+    end_function
+
+
+# CHECK-LABEL: CODE:
+# CHECK:             # 2 functions in section.
+# CHECK-LABEL: test0:
+# CHECK-NEXT:        .local  f32, f64, v128, v128
+# CHECK-NEXT:       9:       20 02  local.get	2
+# CHECK-NEXT:       b:       0b     end
+# CHECK-LABEL: test1:
+# CHECK-NEXT:        .local  i32, i64, except_ref
+# CHECK-NEXT:      14:       20 03  local.get	3
+# CHECK-NEXT:      16:       0b     end
diff --git a/test/MC/WebAssembly/reloc-code.ll b/test/MC/WebAssembly/reloc-code.ll
index d5080a81bcabcabc27c2e4dbc556c096bd09f9d8..4180d5f5a73d4b9f87f32b9ef9ee1320ff363f67 100644
--- a/test/MC/WebAssembly/reloc-code.ll
+++ b/test/MC/WebAssembly/reloc-code.ll
@@ -26,34 +26,34 @@ entry:
 ; CHECK: Relocations [
 ; CHECK-NEXT:   Section (4) CODE {
 ; CHECK-NEXT:     Relocation {
-; CHECK-NEXT:       Type: R_WEBASSEMBLY_MEMORY_ADDR_LEB (3)
+; CHECK-NEXT:       Type: R_WASM_MEMORY_ADDR_LEB (3)
 ; CHECK-NEXT:       Offset: 0x9
 ; CHECK-NEXT:       Symbol: b
 ; CHECK-NEXT:       Addend: 0
 ; CHECK-NEXT:     }
 ; CHECK-NEXT:     Relocation {
-; CHECK-NEXT:       Type: R_WEBASSEMBLY_MEMORY_ADDR_LEB (3)
+; CHECK-NEXT:       Type: R_WASM_MEMORY_ADDR_LEB (3)
 ; CHECK-NEXT:       Offset: 0x14
 ; CHECK-NEXT:       Symbol: a
 ; CHECK-NEXT:       Addend: 0
 ; CHECK-NEXT:     }
 ; CHECK-NEXT:     Relocation {
-; CHECK-NEXT:       Type: R_WEBASSEMBLY_TYPE_INDEX_LEB (6)
+; CHECK-NEXT:       Type: R_WASM_TYPE_INDEX_LEB (6)
 ; CHECK-NEXT:       Offset: 0x1A
 ; CHECK-NEXT:       Index: 0x1
 ; CHECK-NEXT:     }
 ; CHECK-NEXT:     Relocation {
-; CHECK-NEXT:       Type: R_WEBASSEMBLY_TYPE_INDEX_LEB (6)
+; CHECK-NEXT:       Type: R_WASM_TYPE_INDEX_LEB (6)
 ; CHECK-NEXT:       Offset: 0x24
 ; CHECK-NEXT:       Index: 0x0
 ; CHECK-NEXT:     }
 ; CHECK-NEXT:     Relocation {
-; CHECK-NEXT:       Type: R_WEBASSEMBLY_FUNCTION_INDEX_LEB (0)
+; CHECK-NEXT:       Type: R_WASM_FUNCTION_INDEX_LEB (0)
 ; CHECK-NEXT:       Offset: 0x2D
 ; CHECK-NEXT:       Symbol: c
 ; CHECK-NEXT:     }
 ; CHECK-NEXT:     Relocation {
-; CHECK-NEXT:       Type: R_WEBASSEMBLY_FUNCTION_INDEX_LEB (0)
+; CHECK-NEXT:       Type: R_WASM_FUNCTION_INDEX_LEB (0)
 ; CHECK-NEXT:       Offset: 0x34
 ; CHECK-NEXT:       Symbol: d
 ; CHECK-NEXT:     }
diff --git a/test/MC/WebAssembly/reloc-data.ll b/test/MC/WebAssembly/reloc-data.ll
index 5c0a3c9bde4c7da3b91ee7a2974315bdc7da562e..2c4b206fe9f55527031dbe835db61e6f8ecc1405 100644
--- a/test/MC/WebAssembly/reloc-data.ll
+++ b/test/MC/WebAssembly/reloc-data.ll
@@ -15,31 +15,31 @@ target triple = "wasm32-unknown-unknown"
 ; CHECK:      Relocations [
 ; CHECK-NEXT:   Section (2) DATA {
 ; CHECK-NEXT:     Relocation {
-; CHECK-NEXT:       Type: R_WEBASSEMBLY_MEMORY_ADDR_I32 (5)
+; CHECK-NEXT:       Type: R_WASM_MEMORY_ADDR_I32 (5)
 ; CHECK-NEXT:       Offset: 0x13
 ; CHECK-NEXT:       Symbol: foo
 ; CHECK-NEXT:       Addend: 8
 ; CHECK-NEXT:     }
 ; CHECK-NEXT:     Relocation {
-; CHECK-NEXT:       Type: R_WEBASSEMBLY_MEMORY_ADDR_I32 (5)
+; CHECK-NEXT:       Type: R_WASM_MEMORY_ADDR_I32 (5)
 ; CHECK-NEXT:       Offset: 0x1C
 ; CHECK-NEXT:       Symbol: bar
 ; CHECK-NEXT:       Addend: -16
 ; CHECK-NEXT:     }
 ; CHECK-NEXT:     Relocation {
-; CHECK-NEXT:       Type: R_WEBASSEMBLY_MEMORY_ADDR_I32 (5)
+; CHECK-NEXT:       Type: R_WASM_MEMORY_ADDR_I32 (5)
 ; CHECK-NEXT:       Offset: 0x25
 ; CHECK-NEXT:       Symbol: foo
 ; CHECK-NEXT:       Addend: 0
 ; CHECK-NEXT:     }
 ; CHECK-NEXT:     Relocation {
-; CHECK-NEXT:       Type: R_WEBASSEMBLY_MEMORY_ADDR_I32 (5)
+; CHECK-NEXT:       Type: R_WASM_MEMORY_ADDR_I32 (5)
 ; CHECK-NEXT:       Offset: 0x29
 ; CHECK-NEXT:       Symbol: foo
 ; CHECK-NEXT:       Addend: 0
 ; CHECK-NEXT:     }
 ; CHECK-NEXT:     Relocation {
-; CHECK-NEXT:       Type: R_WEBASSEMBLY_MEMORY_ADDR_I32 (5)
+; CHECK-NEXT:       Type: R_WASM_MEMORY_ADDR_I32 (5)
 ; CHECK-NEXT:       Offset: 0x2D
 ; CHECK-NEXT:       Symbol: foo
 ; CHECK-NEXT:       Addend: 0
diff --git a/test/MC/WebAssembly/simd-encodings.s b/test/MC/WebAssembly/simd-encodings.s
index abb033ca2150604bb62e0694eb5e934c7e563d75..0752f34e1d2bb1a1d06d11f3341a4d6fa10dc46f 100644
--- a/test/MC/WebAssembly/simd-encodings.s
+++ b/test/MC/WebAssembly/simd-encodings.s
@@ -1,4 +1,7 @@
-# RUN: llvm-mc -show-encoding -triple=wasm32-unkown-unknown -mattr=+sign-ext,+simd128 < %s | FileCheck %s
+# RUN: llvm-mc -show-encoding -triple=wasm32-unkown-unknown -mattr=+unimplemented-simd128 < %s | FileCheck %s
+
+main:
+    .functype main () -> ()
 
     # CHECK: v128.load 48:p2align=0 # encoding: [0xfd,0x00,0x00,0x30]
     v128.load 48
@@ -424,28 +427,28 @@
     # CHECK: f64x2.max # encoding: [0xfd,0xaa,0x01]
     f64x2.max
 
-    # CHECK: i32x4.trunc_sat_s/f32x4 # encoding: [0xfd,0xab,0x01]
-    i32x4.trunc_sat_s/f32x4
+    # CHECK: i32x4.trunc_sat_f32x4_s # encoding: [0xfd,0xab,0x01]
+    i32x4.trunc_sat_f32x4_s
 
-    # CHECK: i32x4.trunc_sat_u/f32x4 # encoding: [0xfd,0xac,0x01]
-    i32x4.trunc_sat_u/f32x4
+    # CHECK: i32x4.trunc_sat_f32x4_u # encoding: [0xfd,0xac,0x01]
+    i32x4.trunc_sat_f32x4_u
 
-    # CHECK: i64x2.trunc_sat_s/f64x2 # encoding: [0xfd,0xad,0x01]
-    i64x2.trunc_sat_s/f64x2
+    # CHECK: i64x2.trunc_sat_f64x2_s # encoding: [0xfd,0xad,0x01]
+    i64x2.trunc_sat_f64x2_s
 
-    # CHECK: i64x2.trunc_sat_u/f64x2 # encoding: [0xfd,0xae,0x01]
-    i64x2.trunc_sat_u/f64x2
+    # CHECK: i64x2.trunc_sat_f64x2_u # encoding: [0xfd,0xae,0x01]
+    i64x2.trunc_sat_f64x2_u
 
-    # CHECK: f32x4.convert_s/i32x4 # encoding: [0xfd,0xaf,0x01]
-    f32x4.convert_s/i32x4
+    # CHECK: f32x4.convert_i32x4_s # encoding: [0xfd,0xaf,0x01]
+    f32x4.convert_i32x4_s
 
-    # CHECK: f32x4.convert_u/i32x4 # encoding: [0xfd,0xb0,0x01]
-    f32x4.convert_u/i32x4
+    # CHECK: f32x4.convert_i32x4_u # encoding: [0xfd,0xb0,0x01]
+    f32x4.convert_i32x4_u
 
-    # CHECK: f64x2.convert_s/i64x2 # encoding: [0xfd,0xb1,0x01]
-    f64x2.convert_s/i64x2
+    # CHECK: f64x2.convert_i64x2_s # encoding: [0xfd,0xb1,0x01]
+    f64x2.convert_i64x2_s
 
-    # CHECK: f64x2.convert_u/i64x2 # encoding: [0xfd,0xb2,0x01]
-    f64x2.convert_u/i64x2
+    # CHECK: f64x2.convert_i64x2_u # encoding: [0xfd,0xb2,0x01]
+    f64x2.convert_i64x2_u
 
     end_function
diff --git a/test/MC/WebAssembly/stack-ptr.ll b/test/MC/WebAssembly/stack-ptr.ll
index 9770003fb8e1e1bfb3db6ab73725ae98dc581ccb..147f04ffec158786bc6aba046f615bacca89ccc3 100644
--- a/test/MC/WebAssembly/stack-ptr.ll
+++ b/test/MC/WebAssembly/stack-ptr.ll
@@ -19,5 +19,5 @@ entry:
 ; CHECK:         GlobalMutable:   true
 ; CHECK:   - Type:            CODE
 ; CHECK:     Relocations:
-; CHECK:       - Type:            R_WEBASSEMBLY_GLOBAL_INDEX_LEB
+; CHECK:       - Type:            R_WASM_GLOBAL_INDEX_LEB
 ; CHECK:         Index:           0
diff --git a/test/MC/WebAssembly/types.ll b/test/MC/WebAssembly/types.ll
index b8012420a8f782c95383ba56592b2462dcf83b1a..6eeeef2f5311d6f1614d6547c32a04d755c7a20a 100644
--- a/test/MC/WebAssembly/types.ll
+++ b/test/MC/WebAssembly/types.ll
@@ -1,4 +1,4 @@
-; RUN: llc -wasm-enable-unimplemented-simd -mattr=+sign-ext,+simd128 -filetype=obj %s -o - | obj2yaml | FileCheck %s
+; RUN: llc -mattr=+unimplemented-simd128 -filetype=obj %s -o - | obj2yaml | FileCheck %s
 
 target triple = "wasm32-unknown-unknown"
 
diff --git a/test/MC/WebAssembly/unnamed-data.ll b/test/MC/WebAssembly/unnamed-data.ll
index 47949026ca819f3ea2e42a3498b3ed43b8524fe3..48d61d78c6feb9e5be8eca47d6377c558a31be23 100644
--- a/test/MC/WebAssembly/unnamed-data.ll
+++ b/test/MC/WebAssembly/unnamed-data.ll
@@ -11,10 +11,10 @@ target triple = "wasm32-unknown-unknown"
 
 ; CHECK:        - Type:            DATA
 ; CHECK-NEXT:     Relocations:     
-; CHECK-NEXT:       - Type:            R_WEBASSEMBLY_MEMORY_ADDR_I32
+; CHECK-NEXT:       - Type:            R_WASM_MEMORY_ADDR_I32
 ; CHECK-NEXT:         Index:           0
 ; CHECK-NEXT:         Offset:          0x0000001C
-; CHECK-NEXT:       - Type:            R_WEBASSEMBLY_MEMORY_ADDR_I32
+; CHECK-NEXT:       - Type:            R_WASM_MEMORY_ADDR_I32
 ; CHECK-NEXT:         Index:           1
 ; CHECK-NEXT:         Offset:          0x00000025
 ; CHECK-NEXT:     Segments:        
@@ -44,7 +44,7 @@ target triple = "wasm32-unknown-unknown"
 ; CHECK-NEXT:         Content:         '06000000'
 ; CHECK-NEXT:   - Type:            CUSTOM
 ; CHECK-NEXT:     Name:            linking
-; CHECK-NEXT:     Version:         1
+; CHECK-NEXT:     Version:         2
 ; CHECK-NEXT:     SymbolTable:      
 ; CHECK-NEXT:       - Index:           0
 ; CHECK-NEXT:         Kind:            DATA
@@ -73,18 +73,18 @@ target triple = "wasm32-unknown-unknown"
 ; CHECK-NEXT:     SegmentInfo:    
 ; CHECK-NEXT:       - Index:       0
 ; CHECK-NEXT:         Name:        .rodata..L.str1
-; CHECK-NEXT:         Alignment:   1
+; CHECK-NEXT:         Alignment:   0
 ; CHECK-NEXT:         Flags:       [ ]
 ; CHECK-NEXT:       - Index:       1
 ; CHECK-NEXT:         Name:        .rodata..L.str2
-; CHECK-NEXT:         Alignment:   1
+; CHECK-NEXT:         Alignment:   0
 ; CHECK-NEXT:         Flags:       [ ]
 ; CHECK-NEXT:       - Index:       2
 ; CHECK-NEXT:         Name:        .data.a
-; CHECK-NEXT:         Alignment:   8
+; CHECK-NEXT:         Alignment:   3
 ; CHECK-NEXT:         Flags:       [ ]
 ; CHECK-NEXT:       - Index:       3
 ; CHECK-NEXT:         Name:        .data.b
-; CHECK-NEXT:         Alignment:   8
+; CHECK-NEXT:         Alignment:   3
 ; CHECK-NEXT:         Flags:       [ ]
 ; CHECK_NEXT:   ...
diff --git a/test/MC/WebAssembly/weak-alias.ll b/test/MC/WebAssembly/weak-alias.ll
index bf636953ea4ebdb4f08216d91acab292c7d79ec7..c02ca889222f178a7396bf95624e989b12f3d19e 100644
--- a/test/MC/WebAssembly/weak-alias.ll
+++ b/test/MC/WebAssembly/weak-alias.ll
@@ -62,7 +62,7 @@ entry:
 ; CHECK-NEXT:         Field:           __indirect_function_table
 ; CHECK-NEXT:         Kind:            TABLE
 ; CHECK-NEXT:         Table:
-; CHECK-NEXT:           ElemType:        ANYFUNC
+; CHECK-NEXT:           ElemType:        FUNCREF
 ; CHECK-NEXT:           Limits:
 ; CHECK-NEXT:             Initial:         0x00000001
 ; CHECK-NEXT:   - Type:            FUNCTION
@@ -75,22 +75,22 @@ entry:
 ; CHECK-NEXT:         Functions:       [ 0 ]
 ; CHECK-NEXT:   - Type:            CODE
 ; CHECK-NEXT:     Relocations:
-; CHECK-NEXT:       - Type:            R_WEBASSEMBLY_FUNCTION_INDEX_LEB
+; CHECK-NEXT:       - Type:            R_WASM_FUNCTION_INDEX_LEB
 ; CHECK-NEXT:         Index:           0
 ; CHECK-NEXT:         Offset:          0x00000009
-; CHECK-NEXT:       - Type:            R_WEBASSEMBLY_FUNCTION_INDEX_LEB
+; CHECK-NEXT:       - Type:            R_WASM_FUNCTION_INDEX_LEB
 ; CHECK-NEXT:         Index:           3
 ; CHECK-NEXT:         Offset:          0x00000012
-; CHECK-NEXT:       - Type:            R_WEBASSEMBLY_MEMORY_ADDR_LEB
+; CHECK-NEXT:       - Type:            R_WASM_MEMORY_ADDR_LEB
 ; CHECK-NEXT:         Index:           5
 ; CHECK-NEXT:         Offset:          0x0000001E
-; CHECK-NEXT:       - Type:            R_WEBASSEMBLY_TYPE_INDEX_LEB
+; CHECK-NEXT:       - Type:            R_WASM_TYPE_INDEX_LEB
 ; CHECK-NEXT:         Index:           0
 ; CHECK-NEXT:         Offset:          0x00000024
-; CHECK-NEXT:       - Type:            R_WEBASSEMBLY_MEMORY_ADDR_LEB
+; CHECK-NEXT:       - Type:            R_WASM_MEMORY_ADDR_LEB
 ; CHECK-NEXT:         Index:           7
 ; CHECK-NEXT:         Offset:          0x00000031
-; CHECK-NEXT:       - Type:            R_WEBASSEMBLY_TYPE_INDEX_LEB
+; CHECK-NEXT:       - Type:            R_WASM_TYPE_INDEX_LEB
 ; CHECK-NEXT:         Index:           0
 ; CHECK-NEXT:         Offset:          0x00000037
 ; CHECK-NEXT:     Functions:
@@ -111,10 +111,10 @@ entry:
 ; CHECK-NEXT:         Body:            410028029080808000118080808000000B
 ; CHECK-NEXT:   - Type:            DATA
 ; CHECK-NEXT:     Relocations:
-; CHECK-NEXT:       - Type:            R_WEBASSEMBLY_TABLE_INDEX_I32
+; CHECK-NEXT:       - Type:            R_WASM_TABLE_INDEX_I32
 ; CHECK-NEXT:         Index:           0
 ; CHECK-NEXT:         Offset:          0x0000000F
-; CHECK-NEXT:       - Type:            R_WEBASSEMBLY_TABLE_INDEX_I32
+; CHECK-NEXT:       - Type:            R_WASM_TABLE_INDEX_I32
 ; CHECK-NEXT:         Index:           3
 ; CHECK-NEXT:         Offset:          0x00000018
 ; CHECK-NEXT:     Segments:
@@ -138,7 +138,7 @@ entry:
 ; CHECK-NEXT:         Content:         '01000000'
 ; CHECK-NEXT:   - Type:            CUSTOM
 ; CHECK-NEXT:     Name:            linking
-; CHECK-NEXT:     Version:         1
+; CHECK-NEXT:     Version:         2
 ; CHECK-NEXT:     SymbolTable:
 ; CHECK-NEXT:       - Index:           0
 ; CHECK-NEXT:         Kind:            FUNCTION
@@ -197,26 +197,26 @@ entry:
 ; CHECK-NEXT:     SegmentInfo:
 ; CHECK-NEXT:       - Index:           0
 ; CHECK-NEXT:         Name:            .data.bar
-; CHECK-NEXT:         Alignment:       8
+; CHECK-NEXT:         Alignment:       3
 ; CHECK-NEXT:         Flags:           [ ]
 ; CHECK-NEXT:       - Index:           1
 ; CHECK-NEXT:         Name:            .data.direct_address
-; CHECK-NEXT:         Alignment:       8
+; CHECK-NEXT:         Alignment:       3
 ; CHECK-NEXT:         Flags:           [ ]
 ; CHECK-NEXT:       - Index:           2
 ; CHECK-NEXT:         Name:            .data.alias_address
-; CHECK-NEXT:         Alignment:       8
+; CHECK-NEXT:         Alignment:       3
 ; CHECK-NEXT:         Flags:           [ ]
 ; CHECK-NEXT: ...
 
 ; CHECK-SYMS: SYMBOL TABLE:
-; CHECK-SYMS-NEXT: 00000000 g     F CODE	.hidden foo
-; CHECK-SYMS-NEXT: 00000001 g     F CODE	.hidden call_direct
-; CHECK-SYMS-NEXT: 00000002 g     F CODE	.hidden call_alias
-; CHECK-SYMS-NEXT: 00000000 gw    F CODE	.hidden foo_alias
-; CHECK-SYMS-NEXT: 00000003 g     F CODE	.hidden call_direct_ptr
+; CHECK-SYMS-NEXT: 00000001 g     F CODE	.hidden foo
+; CHECK-SYMS-NEXT: 00000006 g     F CODE	.hidden call_direct
+; CHECK-SYMS-NEXT: 0000000f g     F CODE	.hidden call_alias
+; CHECK-SYMS-NEXT: 00000001 gw    F CODE	.hidden foo_alias
+; CHECK-SYMS-NEXT: 00000018 g     F CODE	.hidden call_direct_ptr
 ; CHECK-SYMS-NEXT: 00000008 g     O DATA	direct_address
-; CHECK-SYMS-NEXT: 00000004 g     F CODE	.hidden call_alias_ptr
+; CHECK-SYMS-NEXT: 0000002b g     F CODE	.hidden call_alias_ptr
 ; CHECK-SYMS-NEXT: 00000010 g     O DATA	alias_address
 ; CHECK-SYMS-NEXT: 00000000 g     O DATA	bar
 ; CHECK-SYMS-NEXT: 00000000 gw    O DATA	.hidden bar_alias
diff --git a/test/MC/X86/AVX512F_SCALAR-32.s b/test/MC/X86/AVX512F_SCALAR-32.s
new file mode 100644
index 0000000000000000000000000000000000000000..2028f1d9f4d8d1e0fcf072493cbe8f1e4d3aa7bd
--- /dev/null
+++ b/test/MC/X86/AVX512F_SCALAR-32.s
@@ -0,0 +1,7094 @@
+// RUN: llvm-mc -triple i386-unknown-unknown -mcpu=skx --show-encoding %s | FileCheck %s
+
+// FIXME some of these tests use VEX encodings because we have no way to force
+// an EVEX encoding. gas has an {evex} prefix that can force the mode, but
+// we don't have that.
+
+// CHECK: vaddsd -485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x58,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vaddsd -485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vaddsd 485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x58,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vaddsd 485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vaddsd -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0x58,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vaddsd -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vaddsd 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0x58,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vaddsd 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vaddsd -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x8a,0x58,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vaddsd -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vaddsd 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x8a,0x58,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vaddsd 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vaddsd 485498096(%edx), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x58,0x8a,0xf0,0x1c,0xf0,0x1c]
+vaddsd 485498096(%edx), %xmm1, %xmm1
+
+// CHECK: vaddsd 485498096(%edx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0x58,0x8a,0xf0,0x1c,0xf0,0x1c]
+vaddsd 485498096(%edx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vaddsd 485498096(%edx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x8a,0x58,0x8a,0xf0,0x1c,0xf0,0x1c]
+vaddsd 485498096(%edx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vaddsd 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x58,0x0d,0xf0,0x1c,0xf0,0x1c]
+vaddsd 485498096, %xmm1, %xmm1
+
+// CHECK: vaddsd 485498096, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0x58,0x0d,0xf0,0x1c,0xf0,0x1c]
+vaddsd 485498096, %xmm1, %xmm1 {%k2}
+
+// CHECK: vaddsd 485498096, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x8a,0x58,0x0d,0xf0,0x1c,0xf0,0x1c]
+vaddsd 485498096, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vaddsd 512(%edx,%eax), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x58,0x8c,0x02,0x00,0x02,0x00,0x00]
+vaddsd 512(%edx,%eax), %xmm1, %xmm1
+
+// CHECK: vaddsd 512(%edx,%eax), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0x58,0x4c,0x02,0x40]
+vaddsd 512(%edx,%eax), %xmm1, %xmm1 {%k2}
+
+// CHECK: vaddsd 512(%edx,%eax), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x8a,0x58,0x4c,0x02,0x40]
+vaddsd 512(%edx,%eax), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vaddsd (%edx), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x58,0x0a]
+vaddsd (%edx), %xmm1, %xmm1
+
+// CHECK: vaddsd (%edx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0x58,0x0a]
+vaddsd (%edx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vaddsd (%edx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x8a,0x58,0x0a]
+vaddsd (%edx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vaddsd {rd-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0xf7,0x38,0x58,0xc9]
+vaddsd {rd-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vaddsd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x3a,0x58,0xc9]
+vaddsd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vaddsd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0xba,0x58,0xc9]
+vaddsd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vaddsd {rn-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0xf7,0x18,0x58,0xc9]
+vaddsd {rn-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vaddsd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x1a,0x58,0xc9]
+vaddsd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vaddsd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x9a,0x58,0xc9]
+vaddsd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vaddsd {ru-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0xf7,0x58,0x58,0xc9]
+vaddsd {ru-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vaddsd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x5a,0x58,0xc9]
+vaddsd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vaddsd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0xda,0x58,0xc9]
+vaddsd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vaddsd {rz-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0xf7,0x78,0x58,0xc9]
+vaddsd {rz-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vaddsd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x7a,0x58,0xc9]
+vaddsd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vaddsd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0xfa,0x58,0xc9]
+vaddsd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vaddsd %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x58,0xc9]
+vaddsd %xmm1, %xmm1, %xmm1
+
+// CHECK: vaddsd %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0x58,0xc9]
+vaddsd %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vaddsd %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x8a,0x58,0xc9]
+vaddsd %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vaddss 256(%edx,%eax), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x58,0x8c,0x02,0x00,0x01,0x00,0x00]
+vaddss 256(%edx,%eax), %xmm1, %xmm1
+
+// CHECK: vaddss 256(%edx,%eax), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0x58,0x4c,0x02,0x40]
+vaddss 256(%edx,%eax), %xmm1, %xmm1 {%k2}
+
+// CHECK: vaddss 256(%edx,%eax), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x8a,0x58,0x4c,0x02,0x40]
+vaddss 256(%edx,%eax), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vaddss -485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x58,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vaddss -485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vaddss 485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x58,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vaddss 485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vaddss -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0x58,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vaddss -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vaddss 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0x58,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vaddss 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vaddss -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x8a,0x58,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vaddss -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vaddss 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x8a,0x58,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vaddss 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vaddss 485498096(%edx), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x58,0x8a,0xf0,0x1c,0xf0,0x1c]
+vaddss 485498096(%edx), %xmm1, %xmm1
+
+// CHECK: vaddss 485498096(%edx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0x58,0x8a,0xf0,0x1c,0xf0,0x1c]
+vaddss 485498096(%edx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vaddss 485498096(%edx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x8a,0x58,0x8a,0xf0,0x1c,0xf0,0x1c]
+vaddss 485498096(%edx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vaddss 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x58,0x0d,0xf0,0x1c,0xf0,0x1c]
+vaddss 485498096, %xmm1, %xmm1
+
+// CHECK: vaddss 485498096, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0x58,0x0d,0xf0,0x1c,0xf0,0x1c]
+vaddss 485498096, %xmm1, %xmm1 {%k2}
+
+// CHECK: vaddss 485498096, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x8a,0x58,0x0d,0xf0,0x1c,0xf0,0x1c]
+vaddss 485498096, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vaddss (%edx), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x58,0x0a]
+vaddss (%edx), %xmm1, %xmm1
+
+// CHECK: vaddss (%edx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0x58,0x0a]
+vaddss (%edx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vaddss (%edx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x8a,0x58,0x0a]
+vaddss (%edx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vaddss {rd-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0x76,0x38,0x58,0xc9]
+vaddss {rd-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vaddss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x3a,0x58,0xc9]
+vaddss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vaddss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0xba,0x58,0xc9]
+vaddss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vaddss {rn-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0x76,0x18,0x58,0xc9]
+vaddss {rn-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vaddss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x1a,0x58,0xc9]
+vaddss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vaddss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x9a,0x58,0xc9]
+vaddss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vaddss {ru-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0x76,0x58,0x58,0xc9]
+vaddss {ru-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vaddss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x5a,0x58,0xc9]
+vaddss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vaddss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0xda,0x58,0xc9]
+vaddss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vaddss {rz-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0x76,0x78,0x58,0xc9]
+vaddss {rz-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vaddss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x7a,0x58,0xc9]
+vaddss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vaddss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0xfa,0x58,0xc9]
+vaddss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vaddss %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x58,0xc9]
+vaddss %xmm1, %xmm1, %xmm1
+
+// CHECK: vaddss %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0x58,0xc9]
+vaddss %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vaddss %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x8a,0x58,0xc9]
+vaddss %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vcmpeqsd -485498096(%edx,%eax,4), %xmm1, %k2
+// CHECK: encoding: [0x62,0xf1,0xf7,0x08,0xc2,0x94,0x82,0x10,0xe3,0x0f,0xe3,0x00]
+vcmpeqsd -485498096(%edx,%eax,4), %xmm1, %k2
+
+// CHECK: vcmpeqsd 485498096(%edx,%eax,4), %xmm1, %k2
+// CHECK: encoding: [0x62,0xf1,0xf7,0x08,0xc2,0x94,0x82,0xf0,0x1c,0xf0,0x1c,0x00]
+vcmpeqsd 485498096(%edx,%eax,4), %xmm1, %k2
+
+// CHECK: vcmpeqsd -485498096(%edx,%eax,4), %xmm1, %k2 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0xc2,0x94,0x82,0x10,0xe3,0x0f,0xe3,0x00]
+vcmpeqsd -485498096(%edx,%eax,4), %xmm1, %k2 {%k2}
+
+// CHECK: vcmpeqsd 485498096(%edx,%eax,4), %xmm1, %k2 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0xc2,0x94,0x82,0xf0,0x1c,0xf0,0x1c,0x00]
+vcmpeqsd 485498096(%edx,%eax,4), %xmm1, %k2 {%k2}
+
+// CHECK: vcmpeqsd 485498096(%edx), %xmm1, %k2
+// CHECK: encoding: [0x62,0xf1,0xf7,0x08,0xc2,0x92,0xf0,0x1c,0xf0,0x1c,0x00]
+vcmpeqsd 485498096(%edx), %xmm1, %k2
+
+// CHECK: vcmpeqsd 485498096(%edx), %xmm1, %k2 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0xc2,0x92,0xf0,0x1c,0xf0,0x1c,0x00]
+vcmpeqsd 485498096(%edx), %xmm1, %k2 {%k2}
+
+// CHECK: vcmpeqsd 485498096, %xmm1, %k2
+// CHECK: encoding: [0x62,0xf1,0xf7,0x08,0xc2,0x15,0xf0,0x1c,0xf0,0x1c,0x00]
+vcmpeqsd 485498096, %xmm1, %k2
+
+// CHECK: vcmpeqsd 485498096, %xmm1, %k2 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0xc2,0x15,0xf0,0x1c,0xf0,0x1c,0x00]
+vcmpeqsd 485498096, %xmm1, %k2 {%k2}
+
+// CHECK: vcmpeqsd 512(%edx,%eax), %xmm1, %k2
+// CHECK: encoding: [0x62,0xf1,0xf7,0x08,0xc2,0x54,0x02,0x40,0x00]
+vcmpeqsd 512(%edx,%eax), %xmm1, %k2
+
+// CHECK: vcmpeqsd 512(%edx,%eax), %xmm1, %k2 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0xc2,0x54,0x02,0x40,0x00]
+vcmpeqsd 512(%edx,%eax), %xmm1, %k2 {%k2}
+
+// CHECK: vcmpeqsd (%edx), %xmm1, %k2
+// CHECK: encoding: [0x62,0xf1,0xf7,0x08,0xc2,0x12,0x00]
+vcmpeqsd (%edx), %xmm1, %k2
+
+// CHECK: vcmpeqsd (%edx), %xmm1, %k2 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0xc2,0x12,0x00]
+vcmpeqsd (%edx), %xmm1, %k2 {%k2}
+
+// CHECK: vcmpeqsd {sae}, %xmm1, %xmm1, %k2
+// CHECK: encoding: [0x62,0xf1,0xf7,0x18,0xc2,0xd1,0x00]
+vcmpeqsd {sae}, %xmm1, %xmm1, %k2
+
+// CHECK: vcmpeqsd {sae}, %xmm1, %xmm1, %k2 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x1a,0xc2,0xd1,0x00]
+vcmpeqsd {sae}, %xmm1, %xmm1, %k2 {%k2}
+
+// CHECK: vcmpeqsd %xmm1, %xmm1, %k2
+// CHECK: encoding: [0x62,0xf1,0xf7,0x08,0xc2,0xd1,0x00]
+vcmpeqsd %xmm1, %xmm1, %k2
+
+// CHECK: vcmpeqsd %xmm1, %xmm1, %k2 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0xc2,0xd1,0x00]
+vcmpeqsd %xmm1, %xmm1, %k2 {%k2}
+
+// CHECK: vcmpeqss 256(%edx,%eax), %xmm1, %k2
+// CHECK: encoding: [0x62,0xf1,0x76,0x08,0xc2,0x54,0x02,0x40,0x00]
+vcmpeqss 256(%edx,%eax), %xmm1, %k2
+
+// CHECK: vcmpeqss 256(%edx,%eax), %xmm1, %k2 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0xc2,0x54,0x02,0x40,0x00]
+vcmpeqss 256(%edx,%eax), %xmm1, %k2 {%k2}
+
+// CHECK: vcmpeqss -485498096(%edx,%eax,4), %xmm1, %k2
+// CHECK: encoding: [0x62,0xf1,0x76,0x08,0xc2,0x94,0x82,0x10,0xe3,0x0f,0xe3,0x00]
+vcmpeqss -485498096(%edx,%eax,4), %xmm1, %k2
+
+// CHECK: vcmpeqss 485498096(%edx,%eax,4), %xmm1, %k2
+// CHECK: encoding: [0x62,0xf1,0x76,0x08,0xc2,0x94,0x82,0xf0,0x1c,0xf0,0x1c,0x00]
+vcmpeqss 485498096(%edx,%eax,4), %xmm1, %k2
+
+// CHECK: vcmpeqss -485498096(%edx,%eax,4), %xmm1, %k2 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0xc2,0x94,0x82,0x10,0xe3,0x0f,0xe3,0x00]
+vcmpeqss -485498096(%edx,%eax,4), %xmm1, %k2 {%k2}
+
+// CHECK: vcmpeqss 485498096(%edx,%eax,4), %xmm1, %k2 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0xc2,0x94,0x82,0xf0,0x1c,0xf0,0x1c,0x00]
+vcmpeqss 485498096(%edx,%eax,4), %xmm1, %k2 {%k2}
+
+// CHECK: vcmpeqss 485498096(%edx), %xmm1, %k2
+// CHECK: encoding: [0x62,0xf1,0x76,0x08,0xc2,0x92,0xf0,0x1c,0xf0,0x1c,0x00]
+vcmpeqss 485498096(%edx), %xmm1, %k2
+
+// CHECK: vcmpeqss 485498096(%edx), %xmm1, %k2 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0xc2,0x92,0xf0,0x1c,0xf0,0x1c,0x00]
+vcmpeqss 485498096(%edx), %xmm1, %k2 {%k2}
+
+// CHECK: vcmpeqss 485498096, %xmm1, %k2
+// CHECK: encoding: [0x62,0xf1,0x76,0x08,0xc2,0x15,0xf0,0x1c,0xf0,0x1c,0x00]
+vcmpeqss 485498096, %xmm1, %k2
+
+// CHECK: vcmpeqss 485498096, %xmm1, %k2 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0xc2,0x15,0xf0,0x1c,0xf0,0x1c,0x00]
+vcmpeqss 485498096, %xmm1, %k2 {%k2}
+
+// CHECK: vcmpeqss (%edx), %xmm1, %k2
+// CHECK: encoding: [0x62,0xf1,0x76,0x08,0xc2,0x12,0x00]
+vcmpeqss (%edx), %xmm1, %k2
+
+// CHECK: vcmpeqss (%edx), %xmm1, %k2 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0xc2,0x12,0x00]
+vcmpeqss (%edx), %xmm1, %k2 {%k2}
+
+// CHECK: vcmpeqss {sae}, %xmm1, %xmm1, %k2
+// CHECK: encoding: [0x62,0xf1,0x76,0x18,0xc2,0xd1,0x00]
+vcmpeqss {sae}, %xmm1, %xmm1, %k2
+
+// CHECK: vcmpeqss {sae}, %xmm1, %xmm1, %k2 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x1a,0xc2,0xd1,0x00]
+vcmpeqss {sae}, %xmm1, %xmm1, %k2 {%k2}
+
+// CHECK: vcmpeqss %xmm1, %xmm1, %k2
+// CHECK: encoding: [0x62,0xf1,0x76,0x08,0xc2,0xd1,0x00]
+vcmpeqss %xmm1, %xmm1, %k2
+
+// CHECK: vcmpeqss %xmm1, %xmm1, %k2 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0xc2,0xd1,0x00]
+vcmpeqss %xmm1, %xmm1, %k2 {%k2}
+
+// CHECK: vcomisd -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0xc5,0xf9,0x2f,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vcomisd -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: vcomisd 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0xc5,0xf9,0x2f,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vcomisd 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: vcomisd 485498096(%edx), %xmm1
+// CHECK: encoding: [0xc5,0xf9,0x2f,0x8a,0xf0,0x1c,0xf0,0x1c]
+vcomisd 485498096(%edx), %xmm1
+
+// CHECK: vcomisd 485498096, %xmm1
+// CHECK: encoding: [0xc5,0xf9,0x2f,0x0d,0xf0,0x1c,0xf0,0x1c]
+vcomisd 485498096, %xmm1
+
+// CHECK: vcomisd 512(%edx,%eax), %xmm1
+// CHECK: encoding: [0xc5,0xf9,0x2f,0x8c,0x02,0x00,0x02,0x00,0x00]
+vcomisd 512(%edx,%eax), %xmm1
+
+// CHECK: vcomisd (%edx), %xmm1
+// CHECK: encoding: [0xc5,0xf9,0x2f,0x0a]
+vcomisd (%edx), %xmm1
+
+// CHECK: vcomisd {sae}, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0xfd,0x18,0x2f,0xc9]
+vcomisd {sae}, %xmm1, %xmm1
+
+// CHECK: vcomisd %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf9,0x2f,0xc9]
+vcomisd %xmm1, %xmm1
+
+// CHECK: vcomiss 256(%edx,%eax), %xmm1
+// CHECK: encoding: [0xc5,0xf8,0x2f,0x8c,0x02,0x00,0x01,0x00,0x00]
+vcomiss 256(%edx,%eax), %xmm1
+
+// CHECK: vcomiss -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0xc5,0xf8,0x2f,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vcomiss -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: vcomiss 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0xc5,0xf8,0x2f,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vcomiss 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: vcomiss 485498096(%edx), %xmm1
+// CHECK: encoding: [0xc5,0xf8,0x2f,0x8a,0xf0,0x1c,0xf0,0x1c]
+vcomiss 485498096(%edx), %xmm1
+
+// CHECK: vcomiss 485498096, %xmm1
+// CHECK: encoding: [0xc5,0xf8,0x2f,0x0d,0xf0,0x1c,0xf0,0x1c]
+vcomiss 485498096, %xmm1
+
+// CHECK: vcomiss (%edx), %xmm1
+// CHECK: encoding: [0xc5,0xf8,0x2f,0x0a]
+vcomiss (%edx), %xmm1
+
+// CHECK: vcomiss {sae}, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0x7c,0x18,0x2f,0xc9]
+vcomiss {sae}, %xmm1, %xmm1
+
+// CHECK: vcomiss %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf8,0x2f,0xc9]
+vcomiss %xmm1, %xmm1
+
+// CHECK: vcvtsd2ss -485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x5a,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vcvtsd2ss -485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vcvtsd2ss 485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x5a,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vcvtsd2ss 485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vcvtsd2ss -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0x5a,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vcvtsd2ss -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vcvtsd2ss 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0x5a,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vcvtsd2ss 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vcvtsd2ss -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x8a,0x5a,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vcvtsd2ss -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vcvtsd2ss 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x8a,0x5a,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vcvtsd2ss 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vcvtsd2ss 485498096(%edx), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x5a,0x8a,0xf0,0x1c,0xf0,0x1c]
+vcvtsd2ss 485498096(%edx), %xmm1, %xmm1
+
+// CHECK: vcvtsd2ss 485498096(%edx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0x5a,0x8a,0xf0,0x1c,0xf0,0x1c]
+vcvtsd2ss 485498096(%edx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vcvtsd2ss 485498096(%edx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x8a,0x5a,0x8a,0xf0,0x1c,0xf0,0x1c]
+vcvtsd2ss 485498096(%edx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vcvtsd2ss 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x5a,0x0d,0xf0,0x1c,0xf0,0x1c]
+vcvtsd2ss 485498096, %xmm1, %xmm1
+
+// CHECK: vcvtsd2ss 485498096, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0x5a,0x0d,0xf0,0x1c,0xf0,0x1c]
+vcvtsd2ss 485498096, %xmm1, %xmm1 {%k2}
+
+// CHECK: vcvtsd2ss 485498096, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x8a,0x5a,0x0d,0xf0,0x1c,0xf0,0x1c]
+vcvtsd2ss 485498096, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vcvtsd2ss 512(%edx,%eax), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x5a,0x8c,0x02,0x00,0x02,0x00,0x00]
+vcvtsd2ss 512(%edx,%eax), %xmm1, %xmm1
+
+// CHECK: vcvtsd2ss 512(%edx,%eax), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0x5a,0x4c,0x02,0x40]
+vcvtsd2ss 512(%edx,%eax), %xmm1, %xmm1 {%k2}
+
+// CHECK: vcvtsd2ss 512(%edx,%eax), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x8a,0x5a,0x4c,0x02,0x40]
+vcvtsd2ss 512(%edx,%eax), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vcvtsd2ss (%edx), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x5a,0x0a]
+vcvtsd2ss (%edx), %xmm1, %xmm1
+
+// CHECK: vcvtsd2ss (%edx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0x5a,0x0a]
+vcvtsd2ss (%edx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vcvtsd2ss (%edx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x8a,0x5a,0x0a]
+vcvtsd2ss (%edx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vcvtsd2ss {rd-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0xf7,0x38,0x5a,0xc9]
+vcvtsd2ss {rd-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vcvtsd2ss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x3a,0x5a,0xc9]
+vcvtsd2ss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vcvtsd2ss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0xba,0x5a,0xc9]
+vcvtsd2ss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vcvtsd2ss {rn-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0xf7,0x18,0x5a,0xc9]
+vcvtsd2ss {rn-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vcvtsd2ss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x1a,0x5a,0xc9]
+vcvtsd2ss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vcvtsd2ss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x9a,0x5a,0xc9]
+vcvtsd2ss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vcvtsd2ss {ru-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0xf7,0x58,0x5a,0xc9]
+vcvtsd2ss {ru-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vcvtsd2ss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x5a,0x5a,0xc9]
+vcvtsd2ss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vcvtsd2ss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0xda,0x5a,0xc9]
+vcvtsd2ss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vcvtsd2ss {rz-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0xf7,0x78,0x5a,0xc9]
+vcvtsd2ss {rz-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vcvtsd2ss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x7a,0x5a,0xc9]
+vcvtsd2ss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vcvtsd2ss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0xfa,0x5a,0xc9]
+vcvtsd2ss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vcvtsd2ss %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x5a,0xc9]
+vcvtsd2ss %xmm1, %xmm1, %xmm1
+
+// CHECK: vcvtsd2ss %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0x5a,0xc9]
+vcvtsd2ss %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vcvtsd2ss %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x8a,0x5a,0xc9]
+vcvtsd2ss %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vcvtsi2sdl 256(%edx,%eax), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x2a,0x8c,0x02,0x00,0x01,0x00,0x00]
+vcvtsi2sdl 256(%edx,%eax), %xmm1, %xmm1
+
+// CHECK: vcvtsi2sdl -485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x2a,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vcvtsi2sdl -485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vcvtsi2sdl 485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x2a,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vcvtsi2sdl 485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vcvtsi2sdl 485498096(%edx), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x2a,0x8a,0xf0,0x1c,0xf0,0x1c]
+vcvtsi2sdl 485498096(%edx), %xmm1, %xmm1
+
+// CHECK: vcvtsi2sdl 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x2a,0x0d,0xf0,0x1c,0xf0,0x1c]
+vcvtsi2sdl 485498096, %xmm1, %xmm1
+
+// CHECK: vcvtsi2sdl (%edx), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x2a,0x0a]
+vcvtsi2sdl (%edx), %xmm1, %xmm1
+
+// CHECK: vcvtsi2ssl 256(%edx,%eax), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x2a,0x8c,0x02,0x00,0x01,0x00,0x00]
+vcvtsi2ssl 256(%edx,%eax), %xmm1, %xmm1
+
+// CHECK: vcvtsi2ssl -485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x2a,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vcvtsi2ssl -485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vcvtsi2ssl 485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x2a,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vcvtsi2ssl 485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vcvtsi2ssl 485498096(%edx), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x2a,0x8a,0xf0,0x1c,0xf0,0x1c]
+vcvtsi2ssl 485498096(%edx), %xmm1, %xmm1
+
+// CHECK: vcvtsi2ssl 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x2a,0x0d,0xf0,0x1c,0xf0,0x1c]
+vcvtsi2ssl 485498096, %xmm1, %xmm1
+
+// CHECK: vcvtsi2ssl (%edx), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x2a,0x0a]
+vcvtsi2ssl (%edx), %xmm1, %xmm1
+
+// CHECK: vcvtss2sd 256(%edx,%eax), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x5a,0x8c,0x02,0x00,0x01,0x00,0x00]
+vcvtss2sd 256(%edx,%eax), %xmm1, %xmm1
+
+// CHECK: vcvtss2sd 256(%edx,%eax), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0x5a,0x4c,0x02,0x40]
+vcvtss2sd 256(%edx,%eax), %xmm1, %xmm1 {%k2}
+
+// CHECK: vcvtss2sd 256(%edx,%eax), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x8a,0x5a,0x4c,0x02,0x40]
+vcvtss2sd 256(%edx,%eax), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vcvtss2sd -485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x5a,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vcvtss2sd -485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vcvtss2sd 485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x5a,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vcvtss2sd 485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vcvtss2sd -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0x5a,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vcvtss2sd -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vcvtss2sd 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0x5a,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vcvtss2sd 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vcvtss2sd -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x8a,0x5a,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vcvtss2sd -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vcvtss2sd 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x8a,0x5a,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vcvtss2sd 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vcvtss2sd 485498096(%edx), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x5a,0x8a,0xf0,0x1c,0xf0,0x1c]
+vcvtss2sd 485498096(%edx), %xmm1, %xmm1
+
+// CHECK: vcvtss2sd 485498096(%edx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0x5a,0x8a,0xf0,0x1c,0xf0,0x1c]
+vcvtss2sd 485498096(%edx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vcvtss2sd 485498096(%edx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x8a,0x5a,0x8a,0xf0,0x1c,0xf0,0x1c]
+vcvtss2sd 485498096(%edx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vcvtss2sd 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x5a,0x0d,0xf0,0x1c,0xf0,0x1c]
+vcvtss2sd 485498096, %xmm1, %xmm1
+
+// CHECK: vcvtss2sd 485498096, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0x5a,0x0d,0xf0,0x1c,0xf0,0x1c]
+vcvtss2sd 485498096, %xmm1, %xmm1 {%k2}
+
+// CHECK: vcvtss2sd 485498096, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x8a,0x5a,0x0d,0xf0,0x1c,0xf0,0x1c]
+vcvtss2sd 485498096, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vcvtss2sd (%edx), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x5a,0x0a]
+vcvtss2sd (%edx), %xmm1, %xmm1
+
+// CHECK: vcvtss2sd (%edx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0x5a,0x0a]
+vcvtss2sd (%edx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vcvtss2sd (%edx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x8a,0x5a,0x0a]
+vcvtss2sd (%edx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vcvtss2sd {sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0x76,0x18,0x5a,0xc9]
+vcvtss2sd {sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vcvtss2sd {sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x1a,0x5a,0xc9]
+vcvtss2sd {sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vcvtss2sd {sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x9a,0x5a,0xc9]
+vcvtss2sd {sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vcvtss2sd %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x5a,0xc9]
+vcvtss2sd %xmm1, %xmm1, %xmm1
+
+// CHECK: vcvtss2sd %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0x5a,0xc9]
+vcvtss2sd %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vcvtss2sd %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x8a,0x5a,0xc9]
+vcvtss2sd %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vcvtusi2sdl 256(%edx,%eax), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0x77,0x08,0x7b,0x4c,0x02,0x40]
+vcvtusi2sdl 256(%edx,%eax), %xmm1, %xmm1
+
+// CHECK: vcvtusi2sdl -485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0x77,0x08,0x7b,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vcvtusi2sdl -485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vcvtusi2sdl 485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0x77,0x08,0x7b,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vcvtusi2sdl 485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vcvtusi2sdl 485498096(%edx), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0x77,0x08,0x7b,0x8a,0xf0,0x1c,0xf0,0x1c]
+vcvtusi2sdl 485498096(%edx), %xmm1, %xmm1
+
+// CHECK: vcvtusi2sdl 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0x77,0x08,0x7b,0x0d,0xf0,0x1c,0xf0,0x1c]
+vcvtusi2sdl 485498096, %xmm1, %xmm1
+
+// CHECK: vcvtusi2sdl (%edx), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0x77,0x08,0x7b,0x0a]
+vcvtusi2sdl (%edx), %xmm1, %xmm1
+
+// CHECK: vcvtusi2ssl 256(%edx,%eax), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0x76,0x08,0x7b,0x4c,0x02,0x40]
+vcvtusi2ssl 256(%edx,%eax), %xmm1, %xmm1
+
+// CHECK: vcvtusi2ssl -485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0x76,0x08,0x7b,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vcvtusi2ssl -485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vcvtusi2ssl 485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0x76,0x08,0x7b,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vcvtusi2ssl 485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vcvtusi2ssl 485498096(%edx), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0x76,0x08,0x7b,0x8a,0xf0,0x1c,0xf0,0x1c]
+vcvtusi2ssl 485498096(%edx), %xmm1, %xmm1
+
+// CHECK: vcvtusi2ssl 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0x76,0x08,0x7b,0x0d,0xf0,0x1c,0xf0,0x1c]
+vcvtusi2ssl 485498096, %xmm1, %xmm1
+
+// CHECK: vcvtusi2ssl (%edx), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0x76,0x08,0x7b,0x0a]
+vcvtusi2ssl (%edx), %xmm1, %xmm1
+
+// CHECK: vdivsd -485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x5e,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vdivsd -485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vdivsd 485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x5e,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vdivsd 485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vdivsd -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0x5e,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vdivsd -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vdivsd 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0x5e,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vdivsd 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vdivsd -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x8a,0x5e,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vdivsd -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vdivsd 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x8a,0x5e,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vdivsd 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vdivsd 485498096(%edx), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x5e,0x8a,0xf0,0x1c,0xf0,0x1c]
+vdivsd 485498096(%edx), %xmm1, %xmm1
+
+// CHECK: vdivsd 485498096(%edx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0x5e,0x8a,0xf0,0x1c,0xf0,0x1c]
+vdivsd 485498096(%edx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vdivsd 485498096(%edx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x8a,0x5e,0x8a,0xf0,0x1c,0xf0,0x1c]
+vdivsd 485498096(%edx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vdivsd 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x5e,0x0d,0xf0,0x1c,0xf0,0x1c]
+vdivsd 485498096, %xmm1, %xmm1
+
+// CHECK: vdivsd 485498096, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0x5e,0x0d,0xf0,0x1c,0xf0,0x1c]
+vdivsd 485498096, %xmm1, %xmm1 {%k2}
+
+// CHECK: vdivsd 485498096, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x8a,0x5e,0x0d,0xf0,0x1c,0xf0,0x1c]
+vdivsd 485498096, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vdivsd 512(%edx,%eax), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x5e,0x8c,0x02,0x00,0x02,0x00,0x00]
+vdivsd 512(%edx,%eax), %xmm1, %xmm1
+
+// CHECK: vdivsd 512(%edx,%eax), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0x5e,0x4c,0x02,0x40]
+vdivsd 512(%edx,%eax), %xmm1, %xmm1 {%k2}
+
+// CHECK: vdivsd 512(%edx,%eax), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x8a,0x5e,0x4c,0x02,0x40]
+vdivsd 512(%edx,%eax), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vdivsd (%edx), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x5e,0x0a]
+vdivsd (%edx), %xmm1, %xmm1
+
+// CHECK: vdivsd (%edx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0x5e,0x0a]
+vdivsd (%edx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vdivsd (%edx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x8a,0x5e,0x0a]
+vdivsd (%edx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vdivsd {rd-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0xf7,0x38,0x5e,0xc9]
+vdivsd {rd-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vdivsd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x3a,0x5e,0xc9]
+vdivsd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vdivsd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0xba,0x5e,0xc9]
+vdivsd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vdivsd {rn-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0xf7,0x18,0x5e,0xc9]
+vdivsd {rn-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vdivsd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x1a,0x5e,0xc9]
+vdivsd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vdivsd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x9a,0x5e,0xc9]
+vdivsd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vdivsd {ru-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0xf7,0x58,0x5e,0xc9]
+vdivsd {ru-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vdivsd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x5a,0x5e,0xc9]
+vdivsd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vdivsd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0xda,0x5e,0xc9]
+vdivsd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vdivsd {rz-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0xf7,0x78,0x5e,0xc9]
+vdivsd {rz-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vdivsd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x7a,0x5e,0xc9]
+vdivsd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vdivsd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0xfa,0x5e,0xc9]
+vdivsd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vdivsd %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x5e,0xc9]
+vdivsd %xmm1, %xmm1, %xmm1
+
+// CHECK: vdivsd %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0x5e,0xc9]
+vdivsd %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vdivsd %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x8a,0x5e,0xc9]
+vdivsd %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vdivss 256(%edx,%eax), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x5e,0x8c,0x02,0x00,0x01,0x00,0x00]
+vdivss 256(%edx,%eax), %xmm1, %xmm1
+
+// CHECK: vdivss 256(%edx,%eax), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0x5e,0x4c,0x02,0x40]
+vdivss 256(%edx,%eax), %xmm1, %xmm1 {%k2}
+
+// CHECK: vdivss 256(%edx,%eax), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x8a,0x5e,0x4c,0x02,0x40]
+vdivss 256(%edx,%eax), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vdivss -485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x5e,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vdivss -485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vdivss 485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x5e,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vdivss 485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vdivss -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0x5e,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vdivss -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vdivss 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0x5e,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vdivss 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vdivss -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x8a,0x5e,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vdivss -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vdivss 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x8a,0x5e,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vdivss 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vdivss 485498096(%edx), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x5e,0x8a,0xf0,0x1c,0xf0,0x1c]
+vdivss 485498096(%edx), %xmm1, %xmm1
+
+// CHECK: vdivss 485498096(%edx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0x5e,0x8a,0xf0,0x1c,0xf0,0x1c]
+vdivss 485498096(%edx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vdivss 485498096(%edx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x8a,0x5e,0x8a,0xf0,0x1c,0xf0,0x1c]
+vdivss 485498096(%edx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vdivss 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x5e,0x0d,0xf0,0x1c,0xf0,0x1c]
+vdivss 485498096, %xmm1, %xmm1
+
+// CHECK: vdivss 485498096, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0x5e,0x0d,0xf0,0x1c,0xf0,0x1c]
+vdivss 485498096, %xmm1, %xmm1 {%k2}
+
+// CHECK: vdivss 485498096, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x8a,0x5e,0x0d,0xf0,0x1c,0xf0,0x1c]
+vdivss 485498096, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vdivss (%edx), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x5e,0x0a]
+vdivss (%edx), %xmm1, %xmm1
+
+// CHECK: vdivss (%edx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0x5e,0x0a]
+vdivss (%edx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vdivss (%edx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x8a,0x5e,0x0a]
+vdivss (%edx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vdivss {rd-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0x76,0x38,0x5e,0xc9]
+vdivss {rd-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vdivss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x3a,0x5e,0xc9]
+vdivss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vdivss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0xba,0x5e,0xc9]
+vdivss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vdivss {rn-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0x76,0x18,0x5e,0xc9]
+vdivss {rn-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vdivss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x1a,0x5e,0xc9]
+vdivss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vdivss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x9a,0x5e,0xc9]
+vdivss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vdivss {ru-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0x76,0x58,0x5e,0xc9]
+vdivss {ru-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vdivss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x5a,0x5e,0xc9]
+vdivss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vdivss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0xda,0x5e,0xc9]
+vdivss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vdivss {rz-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0x76,0x78,0x5e,0xc9]
+vdivss {rz-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vdivss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x7a,0x5e,0xc9]
+vdivss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vdivss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0xfa,0x5e,0xc9]
+vdivss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vdivss %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x5e,0xc9]
+vdivss %xmm1, %xmm1, %xmm1
+
+// CHECK: vdivss %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0x5e,0xc9]
+vdivss %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vdivss %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x8a,0x5e,0xc9]
+vdivss %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfixupimmsd $0, -485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf3,0xf5,0x08,0x55,0x8c,0x82,0x10,0xe3,0x0f,0xe3,0x00]
+vfixupimmsd $0, -485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vfixupimmsd $0, 485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf3,0xf5,0x08,0x55,0x8c,0x82,0xf0,0x1c,0xf0,0x1c,0x00]
+vfixupimmsd $0, 485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vfixupimmsd $0, -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf3,0xf5,0x0a,0x55,0x8c,0x82,0x10,0xe3,0x0f,0xe3,0x00]
+vfixupimmsd $0, -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfixupimmsd $0, 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf3,0xf5,0x0a,0x55,0x8c,0x82,0xf0,0x1c,0xf0,0x1c,0x00]
+vfixupimmsd $0, 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfixupimmsd $0, -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf3,0xf5,0x8a,0x55,0x8c,0x82,0x10,0xe3,0x0f,0xe3,0x00]
+vfixupimmsd $0, -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfixupimmsd $0, 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf3,0xf5,0x8a,0x55,0x8c,0x82,0xf0,0x1c,0xf0,0x1c,0x00]
+vfixupimmsd $0, 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfixupimmsd $0, 485498096(%edx), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf3,0xf5,0x08,0x55,0x8a,0xf0,0x1c,0xf0,0x1c,0x00]
+vfixupimmsd $0, 485498096(%edx), %xmm1, %xmm1
+
+// CHECK: vfixupimmsd $0, 485498096(%edx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf3,0xf5,0x0a,0x55,0x8a,0xf0,0x1c,0xf0,0x1c,0x00]
+vfixupimmsd $0, 485498096(%edx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfixupimmsd $0, 485498096(%edx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf3,0xf5,0x8a,0x55,0x8a,0xf0,0x1c,0xf0,0x1c,0x00]
+vfixupimmsd $0, 485498096(%edx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfixupimmsd $0, 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf3,0xf5,0x08,0x55,0x0d,0xf0,0x1c,0xf0,0x1c,0x00]
+vfixupimmsd $0, 485498096, %xmm1, %xmm1
+
+// CHECK: vfixupimmsd $0, 485498096, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf3,0xf5,0x0a,0x55,0x0d,0xf0,0x1c,0xf0,0x1c,0x00]
+vfixupimmsd $0, 485498096, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfixupimmsd $0, 485498096, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf3,0xf5,0x8a,0x55,0x0d,0xf0,0x1c,0xf0,0x1c,0x00]
+vfixupimmsd $0, 485498096, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfixupimmsd $0, 512(%edx,%eax), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf3,0xf5,0x08,0x55,0x4c,0x02,0x40,0x00]
+vfixupimmsd $0, 512(%edx,%eax), %xmm1, %xmm1
+
+// CHECK: vfixupimmsd $0, 512(%edx,%eax), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf3,0xf5,0x0a,0x55,0x4c,0x02,0x40,0x00]
+vfixupimmsd $0, 512(%edx,%eax), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfixupimmsd $0, 512(%edx,%eax), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf3,0xf5,0x8a,0x55,0x4c,0x02,0x40,0x00]
+vfixupimmsd $0, 512(%edx,%eax), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfixupimmsd $0, (%edx), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf3,0xf5,0x08,0x55,0x0a,0x00]
+vfixupimmsd $0, (%edx), %xmm1, %xmm1
+
+// CHECK: vfixupimmsd $0, (%edx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf3,0xf5,0x0a,0x55,0x0a,0x00]
+vfixupimmsd $0, (%edx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfixupimmsd $0, (%edx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf3,0xf5,0x8a,0x55,0x0a,0x00]
+vfixupimmsd $0, (%edx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfixupimmsd $0, {sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf3,0xf5,0x18,0x55,0xc9,0x00]
+vfixupimmsd $0, {sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfixupimmsd $0, {sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf3,0xf5,0x1a,0x55,0xc9,0x00]
+vfixupimmsd $0, {sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfixupimmsd $0, {sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf3,0xf5,0x9a,0x55,0xc9,0x00]
+vfixupimmsd $0, {sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfixupimmsd $0, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf3,0xf5,0x08,0x55,0xc9,0x00]
+vfixupimmsd $0, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfixupimmsd $0, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf3,0xf5,0x0a,0x55,0xc9,0x00]
+vfixupimmsd $0, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfixupimmsd $0, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf3,0xf5,0x8a,0x55,0xc9,0x00]
+vfixupimmsd $0, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfixupimmss $0, 256(%edx,%eax), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf3,0x75,0x08,0x55,0x4c,0x02,0x40,0x00]
+vfixupimmss $0, 256(%edx,%eax), %xmm1, %xmm1
+
+// CHECK: vfixupimmss $0, 256(%edx,%eax), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf3,0x75,0x0a,0x55,0x4c,0x02,0x40,0x00]
+vfixupimmss $0, 256(%edx,%eax), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfixupimmss $0, 256(%edx,%eax), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf3,0x75,0x8a,0x55,0x4c,0x02,0x40,0x00]
+vfixupimmss $0, 256(%edx,%eax), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfixupimmss $0, -485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf3,0x75,0x08,0x55,0x8c,0x82,0x10,0xe3,0x0f,0xe3,0x00]
+vfixupimmss $0, -485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vfixupimmss $0, 485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf3,0x75,0x08,0x55,0x8c,0x82,0xf0,0x1c,0xf0,0x1c,0x00]
+vfixupimmss $0, 485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vfixupimmss $0, -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf3,0x75,0x0a,0x55,0x8c,0x82,0x10,0xe3,0x0f,0xe3,0x00]
+vfixupimmss $0, -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfixupimmss $0, 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf3,0x75,0x0a,0x55,0x8c,0x82,0xf0,0x1c,0xf0,0x1c,0x00]
+vfixupimmss $0, 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfixupimmss $0, -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf3,0x75,0x8a,0x55,0x8c,0x82,0x10,0xe3,0x0f,0xe3,0x00]
+vfixupimmss $0, -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfixupimmss $0, 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf3,0x75,0x8a,0x55,0x8c,0x82,0xf0,0x1c,0xf0,0x1c,0x00]
+vfixupimmss $0, 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfixupimmss $0, 485498096(%edx), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf3,0x75,0x08,0x55,0x8a,0xf0,0x1c,0xf0,0x1c,0x00]
+vfixupimmss $0, 485498096(%edx), %xmm1, %xmm1
+
+// CHECK: vfixupimmss $0, 485498096(%edx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf3,0x75,0x0a,0x55,0x8a,0xf0,0x1c,0xf0,0x1c,0x00]
+vfixupimmss $0, 485498096(%edx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfixupimmss $0, 485498096(%edx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf3,0x75,0x8a,0x55,0x8a,0xf0,0x1c,0xf0,0x1c,0x00]
+vfixupimmss $0, 485498096(%edx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfixupimmss $0, 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf3,0x75,0x08,0x55,0x0d,0xf0,0x1c,0xf0,0x1c,0x00]
+vfixupimmss $0, 485498096, %xmm1, %xmm1
+
+// CHECK: vfixupimmss $0, 485498096, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf3,0x75,0x0a,0x55,0x0d,0xf0,0x1c,0xf0,0x1c,0x00]
+vfixupimmss $0, 485498096, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfixupimmss $0, 485498096, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf3,0x75,0x8a,0x55,0x0d,0xf0,0x1c,0xf0,0x1c,0x00]
+vfixupimmss $0, 485498096, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfixupimmss $0, (%edx), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf3,0x75,0x08,0x55,0x0a,0x00]
+vfixupimmss $0, (%edx), %xmm1, %xmm1
+
+// CHECK: vfixupimmss $0, (%edx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf3,0x75,0x0a,0x55,0x0a,0x00]
+vfixupimmss $0, (%edx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfixupimmss $0, (%edx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf3,0x75,0x8a,0x55,0x0a,0x00]
+vfixupimmss $0, (%edx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfixupimmss $0, {sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf3,0x75,0x18,0x55,0xc9,0x00]
+vfixupimmss $0, {sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfixupimmss $0, {sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf3,0x75,0x1a,0x55,0xc9,0x00]
+vfixupimmss $0, {sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfixupimmss $0, {sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf3,0x75,0x9a,0x55,0xc9,0x00]
+vfixupimmss $0, {sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfixupimmss $0, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf3,0x75,0x08,0x55,0xc9,0x00]
+vfixupimmss $0, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfixupimmss $0, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf3,0x75,0x0a,0x55,0xc9,0x00]
+vfixupimmss $0, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfixupimmss $0, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf3,0x75,0x8a,0x55,0xc9,0x00]
+vfixupimmss $0, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd132sd -485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0x99,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vfmadd132sd -485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vfmadd132sd 485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0x99,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vfmadd132sd 485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vfmadd132sd -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0x99,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vfmadd132sd -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd132sd 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0x99,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vfmadd132sd 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd132sd -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0x99,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vfmadd132sd -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd132sd 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0x99,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vfmadd132sd 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd132sd 485498096(%edx), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0x99,0x8a,0xf0,0x1c,0xf0,0x1c]
+vfmadd132sd 485498096(%edx), %xmm1, %xmm1
+
+// CHECK: vfmadd132sd 485498096(%edx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0x99,0x8a,0xf0,0x1c,0xf0,0x1c]
+vfmadd132sd 485498096(%edx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd132sd 485498096(%edx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0x99,0x8a,0xf0,0x1c,0xf0,0x1c]
+vfmadd132sd 485498096(%edx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd132sd 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0x99,0x0d,0xf0,0x1c,0xf0,0x1c]
+vfmadd132sd 485498096, %xmm1, %xmm1
+
+// CHECK: vfmadd132sd 485498096, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0x99,0x0d,0xf0,0x1c,0xf0,0x1c]
+vfmadd132sd 485498096, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd132sd 485498096, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0x99,0x0d,0xf0,0x1c,0xf0,0x1c]
+vfmadd132sd 485498096, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd132sd 512(%edx,%eax), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0x99,0x8c,0x02,0x00,0x02,0x00,0x00]
+vfmadd132sd 512(%edx,%eax), %xmm1, %xmm1
+
+// CHECK: vfmadd132sd 512(%edx,%eax), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0x99,0x4c,0x02,0x40]
+vfmadd132sd 512(%edx,%eax), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd132sd 512(%edx,%eax), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0x99,0x4c,0x02,0x40]
+vfmadd132sd 512(%edx,%eax), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd132sd (%edx), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0x99,0x0a]
+vfmadd132sd (%edx), %xmm1, %xmm1
+
+// CHECK: vfmadd132sd (%edx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0x99,0x0a]
+vfmadd132sd (%edx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd132sd (%edx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0x99,0x0a]
+vfmadd132sd (%edx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd132sd {rd-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x38,0x99,0xc9]
+vfmadd132sd {rd-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfmadd132sd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x3a,0x99,0xc9]
+vfmadd132sd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd132sd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0xba,0x99,0xc9]
+vfmadd132sd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd132sd {rn-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x18,0x99,0xc9]
+vfmadd132sd {rn-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfmadd132sd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x1a,0x99,0xc9]
+vfmadd132sd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd132sd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x9a,0x99,0xc9]
+vfmadd132sd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd132sd {ru-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x58,0x99,0xc9]
+vfmadd132sd {ru-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfmadd132sd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x5a,0x99,0xc9]
+vfmadd132sd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd132sd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0xda,0x99,0xc9]
+vfmadd132sd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd132sd {rz-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x78,0x99,0xc9]
+vfmadd132sd {rz-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfmadd132sd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x7a,0x99,0xc9]
+vfmadd132sd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd132sd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0xfa,0x99,0xc9]
+vfmadd132sd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd132sd %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0x99,0xc9]
+vfmadd132sd %xmm1, %xmm1, %xmm1
+
+// CHECK: vfmadd132sd %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0x99,0xc9]
+vfmadd132sd %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd132sd %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0x99,0xc9]
+vfmadd132sd %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd132ss 256(%edx,%eax), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0x99,0x8c,0x02,0x00,0x01,0x00,0x00]
+vfmadd132ss 256(%edx,%eax), %xmm1, %xmm1
+
+// CHECK: vfmadd132ss 256(%edx,%eax), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0x99,0x4c,0x02,0x40]
+vfmadd132ss 256(%edx,%eax), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd132ss 256(%edx,%eax), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0x99,0x4c,0x02,0x40]
+vfmadd132ss 256(%edx,%eax), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd132ss -485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0x99,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vfmadd132ss -485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vfmadd132ss 485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0x99,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vfmadd132ss 485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vfmadd132ss -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0x99,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vfmadd132ss -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd132ss 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0x99,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vfmadd132ss 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd132ss -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0x99,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vfmadd132ss -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd132ss 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0x99,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vfmadd132ss 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd132ss 485498096(%edx), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0x99,0x8a,0xf0,0x1c,0xf0,0x1c]
+vfmadd132ss 485498096(%edx), %xmm1, %xmm1
+
+// CHECK: vfmadd132ss 485498096(%edx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0x99,0x8a,0xf0,0x1c,0xf0,0x1c]
+vfmadd132ss 485498096(%edx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd132ss 485498096(%edx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0x99,0x8a,0xf0,0x1c,0xf0,0x1c]
+vfmadd132ss 485498096(%edx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd132ss 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0x99,0x0d,0xf0,0x1c,0xf0,0x1c]
+vfmadd132ss 485498096, %xmm1, %xmm1
+
+// CHECK: vfmadd132ss 485498096, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0x99,0x0d,0xf0,0x1c,0xf0,0x1c]
+vfmadd132ss 485498096, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd132ss 485498096, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0x99,0x0d,0xf0,0x1c,0xf0,0x1c]
+vfmadd132ss 485498096, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd132ss (%edx), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0x99,0x0a]
+vfmadd132ss (%edx), %xmm1, %xmm1
+
+// CHECK: vfmadd132ss (%edx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0x99,0x0a]
+vfmadd132ss (%edx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd132ss (%edx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0x99,0x0a]
+vfmadd132ss (%edx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd132ss {rd-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x38,0x99,0xc9]
+vfmadd132ss {rd-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfmadd132ss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x3a,0x99,0xc9]
+vfmadd132ss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd132ss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0xba,0x99,0xc9]
+vfmadd132ss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd132ss {rn-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x18,0x99,0xc9]
+vfmadd132ss {rn-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfmadd132ss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x1a,0x99,0xc9]
+vfmadd132ss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd132ss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x9a,0x99,0xc9]
+vfmadd132ss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd132ss {ru-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x58,0x99,0xc9]
+vfmadd132ss {ru-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfmadd132ss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x5a,0x99,0xc9]
+vfmadd132ss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd132ss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0xda,0x99,0xc9]
+vfmadd132ss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd132ss {rz-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x78,0x99,0xc9]
+vfmadd132ss {rz-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfmadd132ss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x7a,0x99,0xc9]
+vfmadd132ss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd132ss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0xfa,0x99,0xc9]
+vfmadd132ss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd132ss %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0x99,0xc9]
+vfmadd132ss %xmm1, %xmm1, %xmm1
+
+// CHECK: vfmadd132ss %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0x99,0xc9]
+vfmadd132ss %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd132ss %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0x99,0xc9]
+vfmadd132ss %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd213sd -485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0xa9,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vfmadd213sd -485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vfmadd213sd 485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0xa9,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vfmadd213sd 485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vfmadd213sd -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0xa9,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vfmadd213sd -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd213sd 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0xa9,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vfmadd213sd 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd213sd -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0xa9,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vfmadd213sd -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd213sd 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0xa9,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vfmadd213sd 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd213sd 485498096(%edx), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0xa9,0x8a,0xf0,0x1c,0xf0,0x1c]
+vfmadd213sd 485498096(%edx), %xmm1, %xmm1
+
+// CHECK: vfmadd213sd 485498096(%edx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0xa9,0x8a,0xf0,0x1c,0xf0,0x1c]
+vfmadd213sd 485498096(%edx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd213sd 485498096(%edx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0xa9,0x8a,0xf0,0x1c,0xf0,0x1c]
+vfmadd213sd 485498096(%edx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd213sd 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0xa9,0x0d,0xf0,0x1c,0xf0,0x1c]
+vfmadd213sd 485498096, %xmm1, %xmm1
+
+// CHECK: vfmadd213sd 485498096, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0xa9,0x0d,0xf0,0x1c,0xf0,0x1c]
+vfmadd213sd 485498096, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd213sd 485498096, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0xa9,0x0d,0xf0,0x1c,0xf0,0x1c]
+vfmadd213sd 485498096, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd213sd 512(%edx,%eax), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0xa9,0x8c,0x02,0x00,0x02,0x00,0x00]
+vfmadd213sd 512(%edx,%eax), %xmm1, %xmm1
+
+// CHECK: vfmadd213sd 512(%edx,%eax), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0xa9,0x4c,0x02,0x40]
+vfmadd213sd 512(%edx,%eax), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd213sd 512(%edx,%eax), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0xa9,0x4c,0x02,0x40]
+vfmadd213sd 512(%edx,%eax), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd213sd (%edx), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0xa9,0x0a]
+vfmadd213sd (%edx), %xmm1, %xmm1
+
+// CHECK: vfmadd213sd (%edx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0xa9,0x0a]
+vfmadd213sd (%edx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd213sd (%edx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0xa9,0x0a]
+vfmadd213sd (%edx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd213sd {rd-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x38,0xa9,0xc9]
+vfmadd213sd {rd-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfmadd213sd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x3a,0xa9,0xc9]
+vfmadd213sd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd213sd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0xba,0xa9,0xc9]
+vfmadd213sd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd213sd {rn-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x18,0xa9,0xc9]
+vfmadd213sd {rn-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfmadd213sd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x1a,0xa9,0xc9]
+vfmadd213sd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd213sd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x9a,0xa9,0xc9]
+vfmadd213sd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd213sd {ru-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x58,0xa9,0xc9]
+vfmadd213sd {ru-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfmadd213sd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x5a,0xa9,0xc9]
+vfmadd213sd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd213sd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0xda,0xa9,0xc9]
+vfmadd213sd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd213sd {rz-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x78,0xa9,0xc9]
+vfmadd213sd {rz-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfmadd213sd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x7a,0xa9,0xc9]
+vfmadd213sd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd213sd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0xfa,0xa9,0xc9]
+vfmadd213sd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd213sd %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0xa9,0xc9]
+vfmadd213sd %xmm1, %xmm1, %xmm1
+
+// CHECK: vfmadd213sd %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0xa9,0xc9]
+vfmadd213sd %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd213sd %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0xa9,0xc9]
+vfmadd213sd %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd213ss 256(%edx,%eax), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0xa9,0x8c,0x02,0x00,0x01,0x00,0x00]
+vfmadd213ss 256(%edx,%eax), %xmm1, %xmm1
+
+// CHECK: vfmadd213ss 256(%edx,%eax), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0xa9,0x4c,0x02,0x40]
+vfmadd213ss 256(%edx,%eax), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd213ss 256(%edx,%eax), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0xa9,0x4c,0x02,0x40]
+vfmadd213ss 256(%edx,%eax), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd213ss -485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0xa9,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vfmadd213ss -485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vfmadd213ss 485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0xa9,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vfmadd213ss 485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vfmadd213ss -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0xa9,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vfmadd213ss -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd213ss 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0xa9,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vfmadd213ss 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd213ss -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0xa9,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vfmadd213ss -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd213ss 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0xa9,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vfmadd213ss 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd213ss 485498096(%edx), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0xa9,0x8a,0xf0,0x1c,0xf0,0x1c]
+vfmadd213ss 485498096(%edx), %xmm1, %xmm1
+
+// CHECK: vfmadd213ss 485498096(%edx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0xa9,0x8a,0xf0,0x1c,0xf0,0x1c]
+vfmadd213ss 485498096(%edx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd213ss 485498096(%edx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0xa9,0x8a,0xf0,0x1c,0xf0,0x1c]
+vfmadd213ss 485498096(%edx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd213ss 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0xa9,0x0d,0xf0,0x1c,0xf0,0x1c]
+vfmadd213ss 485498096, %xmm1, %xmm1
+
+// CHECK: vfmadd213ss 485498096, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0xa9,0x0d,0xf0,0x1c,0xf0,0x1c]
+vfmadd213ss 485498096, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd213ss 485498096, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0xa9,0x0d,0xf0,0x1c,0xf0,0x1c]
+vfmadd213ss 485498096, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd213ss (%edx), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0xa9,0x0a]
+vfmadd213ss (%edx), %xmm1, %xmm1
+
+// CHECK: vfmadd213ss (%edx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0xa9,0x0a]
+vfmadd213ss (%edx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd213ss (%edx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0xa9,0x0a]
+vfmadd213ss (%edx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd213ss {rd-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x38,0xa9,0xc9]
+vfmadd213ss {rd-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfmadd213ss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x3a,0xa9,0xc9]
+vfmadd213ss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd213ss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0xba,0xa9,0xc9]
+vfmadd213ss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd213ss {rn-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x18,0xa9,0xc9]
+vfmadd213ss {rn-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfmadd213ss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x1a,0xa9,0xc9]
+vfmadd213ss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd213ss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x9a,0xa9,0xc9]
+vfmadd213ss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd213ss {ru-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x58,0xa9,0xc9]
+vfmadd213ss {ru-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfmadd213ss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x5a,0xa9,0xc9]
+vfmadd213ss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd213ss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0xda,0xa9,0xc9]
+vfmadd213ss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd213ss {rz-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x78,0xa9,0xc9]
+vfmadd213ss {rz-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfmadd213ss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x7a,0xa9,0xc9]
+vfmadd213ss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd213ss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0xfa,0xa9,0xc9]
+vfmadd213ss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd213ss %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0xa9,0xc9]
+vfmadd213ss %xmm1, %xmm1, %xmm1
+
+// CHECK: vfmadd213ss %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0xa9,0xc9]
+vfmadd213ss %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd213ss %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0xa9,0xc9]
+vfmadd213ss %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd231sd -485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0xb9,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vfmadd231sd -485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vfmadd231sd 485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0xb9,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vfmadd231sd 485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vfmadd231sd -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0xb9,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vfmadd231sd -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd231sd 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0xb9,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vfmadd231sd 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd231sd -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0xb9,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vfmadd231sd -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd231sd 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0xb9,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vfmadd231sd 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd231sd 485498096(%edx), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0xb9,0x8a,0xf0,0x1c,0xf0,0x1c]
+vfmadd231sd 485498096(%edx), %xmm1, %xmm1
+
+// CHECK: vfmadd231sd 485498096(%edx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0xb9,0x8a,0xf0,0x1c,0xf0,0x1c]
+vfmadd231sd 485498096(%edx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd231sd 485498096(%edx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0xb9,0x8a,0xf0,0x1c,0xf0,0x1c]
+vfmadd231sd 485498096(%edx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd231sd 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0xb9,0x0d,0xf0,0x1c,0xf0,0x1c]
+vfmadd231sd 485498096, %xmm1, %xmm1
+
+// CHECK: vfmadd231sd 485498096, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0xb9,0x0d,0xf0,0x1c,0xf0,0x1c]
+vfmadd231sd 485498096, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd231sd 485498096, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0xb9,0x0d,0xf0,0x1c,0xf0,0x1c]
+vfmadd231sd 485498096, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd231sd 512(%edx,%eax), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0xb9,0x8c,0x02,0x00,0x02,0x00,0x00]
+vfmadd231sd 512(%edx,%eax), %xmm1, %xmm1
+
+// CHECK: vfmadd231sd 512(%edx,%eax), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0xb9,0x4c,0x02,0x40]
+vfmadd231sd 512(%edx,%eax), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd231sd 512(%edx,%eax), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0xb9,0x4c,0x02,0x40]
+vfmadd231sd 512(%edx,%eax), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd231sd (%edx), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0xb9,0x0a]
+vfmadd231sd (%edx), %xmm1, %xmm1
+
+// CHECK: vfmadd231sd (%edx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0xb9,0x0a]
+vfmadd231sd (%edx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd231sd (%edx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0xb9,0x0a]
+vfmadd231sd (%edx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd231sd {rd-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x38,0xb9,0xc9]
+vfmadd231sd {rd-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfmadd231sd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x3a,0xb9,0xc9]
+vfmadd231sd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd231sd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0xba,0xb9,0xc9]
+vfmadd231sd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd231sd {rn-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x18,0xb9,0xc9]
+vfmadd231sd {rn-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfmadd231sd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x1a,0xb9,0xc9]
+vfmadd231sd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd231sd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x9a,0xb9,0xc9]
+vfmadd231sd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd231sd {ru-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x58,0xb9,0xc9]
+vfmadd231sd {ru-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfmadd231sd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x5a,0xb9,0xc9]
+vfmadd231sd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd231sd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0xda,0xb9,0xc9]
+vfmadd231sd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd231sd {rz-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x78,0xb9,0xc9]
+vfmadd231sd {rz-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfmadd231sd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x7a,0xb9,0xc9]
+vfmadd231sd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd231sd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0xfa,0xb9,0xc9]
+vfmadd231sd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd231sd %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0xb9,0xc9]
+vfmadd231sd %xmm1, %xmm1, %xmm1
+
+// CHECK: vfmadd231sd %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0xb9,0xc9]
+vfmadd231sd %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd231sd %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0xb9,0xc9]
+vfmadd231sd %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd231ss 256(%edx,%eax), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0xb9,0x8c,0x02,0x00,0x01,0x00,0x00]
+vfmadd231ss 256(%edx,%eax), %xmm1, %xmm1
+
+// CHECK: vfmadd231ss 256(%edx,%eax), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0xb9,0x4c,0x02,0x40]
+vfmadd231ss 256(%edx,%eax), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd231ss 256(%edx,%eax), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0xb9,0x4c,0x02,0x40]
+vfmadd231ss 256(%edx,%eax), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd231ss -485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0xb9,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vfmadd231ss -485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vfmadd231ss 485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0xb9,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vfmadd231ss 485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vfmadd231ss -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0xb9,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vfmadd231ss -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd231ss 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0xb9,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vfmadd231ss 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd231ss -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0xb9,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vfmadd231ss -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd231ss 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0xb9,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vfmadd231ss 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd231ss 485498096(%edx), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0xb9,0x8a,0xf0,0x1c,0xf0,0x1c]
+vfmadd231ss 485498096(%edx), %xmm1, %xmm1
+
+// CHECK: vfmadd231ss 485498096(%edx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0xb9,0x8a,0xf0,0x1c,0xf0,0x1c]
+vfmadd231ss 485498096(%edx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd231ss 485498096(%edx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0xb9,0x8a,0xf0,0x1c,0xf0,0x1c]
+vfmadd231ss 485498096(%edx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd231ss 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0xb9,0x0d,0xf0,0x1c,0xf0,0x1c]
+vfmadd231ss 485498096, %xmm1, %xmm1
+
+// CHECK: vfmadd231ss 485498096, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0xb9,0x0d,0xf0,0x1c,0xf0,0x1c]
+vfmadd231ss 485498096, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd231ss 485498096, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0xb9,0x0d,0xf0,0x1c,0xf0,0x1c]
+vfmadd231ss 485498096, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd231ss (%edx), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0xb9,0x0a]
+vfmadd231ss (%edx), %xmm1, %xmm1
+
+// CHECK: vfmadd231ss (%edx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0xb9,0x0a]
+vfmadd231ss (%edx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd231ss (%edx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0xb9,0x0a]
+vfmadd231ss (%edx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd231ss {rd-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x38,0xb9,0xc9]
+vfmadd231ss {rd-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfmadd231ss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x3a,0xb9,0xc9]
+vfmadd231ss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd231ss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0xba,0xb9,0xc9]
+vfmadd231ss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd231ss {rn-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x18,0xb9,0xc9]
+vfmadd231ss {rn-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfmadd231ss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x1a,0xb9,0xc9]
+vfmadd231ss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd231ss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x9a,0xb9,0xc9]
+vfmadd231ss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd231ss {ru-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x58,0xb9,0xc9]
+vfmadd231ss {ru-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfmadd231ss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x5a,0xb9,0xc9]
+vfmadd231ss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd231ss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0xda,0xb9,0xc9]
+vfmadd231ss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd231ss {rz-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x78,0xb9,0xc9]
+vfmadd231ss {rz-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfmadd231ss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x7a,0xb9,0xc9]
+vfmadd231ss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd231ss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0xfa,0xb9,0xc9]
+vfmadd231ss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd231ss %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0xb9,0xc9]
+vfmadd231ss %xmm1, %xmm1, %xmm1
+
+// CHECK: vfmadd231ss %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0xb9,0xc9]
+vfmadd231ss %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd231ss %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0xb9,0xc9]
+vfmadd231ss %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub132sd -485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0x9b,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vfmsub132sd -485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vfmsub132sd 485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0x9b,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vfmsub132sd 485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vfmsub132sd -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0x9b,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vfmsub132sd -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub132sd 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0x9b,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vfmsub132sd 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub132sd -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0x9b,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vfmsub132sd -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub132sd 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0x9b,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vfmsub132sd 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub132sd 485498096(%edx), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0x9b,0x8a,0xf0,0x1c,0xf0,0x1c]
+vfmsub132sd 485498096(%edx), %xmm1, %xmm1
+
+// CHECK: vfmsub132sd 485498096(%edx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0x9b,0x8a,0xf0,0x1c,0xf0,0x1c]
+vfmsub132sd 485498096(%edx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub132sd 485498096(%edx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0x9b,0x8a,0xf0,0x1c,0xf0,0x1c]
+vfmsub132sd 485498096(%edx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub132sd 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0x9b,0x0d,0xf0,0x1c,0xf0,0x1c]
+vfmsub132sd 485498096, %xmm1, %xmm1
+
+// CHECK: vfmsub132sd 485498096, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0x9b,0x0d,0xf0,0x1c,0xf0,0x1c]
+vfmsub132sd 485498096, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub132sd 485498096, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0x9b,0x0d,0xf0,0x1c,0xf0,0x1c]
+vfmsub132sd 485498096, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub132sd 512(%edx,%eax), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0x9b,0x8c,0x02,0x00,0x02,0x00,0x00]
+vfmsub132sd 512(%edx,%eax), %xmm1, %xmm1
+
+// CHECK: vfmsub132sd 512(%edx,%eax), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0x9b,0x4c,0x02,0x40]
+vfmsub132sd 512(%edx,%eax), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub132sd 512(%edx,%eax), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0x9b,0x4c,0x02,0x40]
+vfmsub132sd 512(%edx,%eax), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub132sd (%edx), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0x9b,0x0a]
+vfmsub132sd (%edx), %xmm1, %xmm1
+
+// CHECK: vfmsub132sd (%edx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0x9b,0x0a]
+vfmsub132sd (%edx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub132sd (%edx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0x9b,0x0a]
+vfmsub132sd (%edx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub132sd {rd-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x38,0x9b,0xc9]
+vfmsub132sd {rd-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfmsub132sd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x3a,0x9b,0xc9]
+vfmsub132sd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub132sd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0xba,0x9b,0xc9]
+vfmsub132sd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub132sd {rn-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x18,0x9b,0xc9]
+vfmsub132sd {rn-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfmsub132sd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x1a,0x9b,0xc9]
+vfmsub132sd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub132sd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x9a,0x9b,0xc9]
+vfmsub132sd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub132sd {ru-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x58,0x9b,0xc9]
+vfmsub132sd {ru-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfmsub132sd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x5a,0x9b,0xc9]
+vfmsub132sd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub132sd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0xda,0x9b,0xc9]
+vfmsub132sd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub132sd {rz-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x78,0x9b,0xc9]
+vfmsub132sd {rz-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfmsub132sd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x7a,0x9b,0xc9]
+vfmsub132sd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub132sd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0xfa,0x9b,0xc9]
+vfmsub132sd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub132sd %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0x9b,0xc9]
+vfmsub132sd %xmm1, %xmm1, %xmm1
+
+// CHECK: vfmsub132sd %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0x9b,0xc9]
+vfmsub132sd %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub132sd %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0x9b,0xc9]
+vfmsub132sd %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub132ss 256(%edx,%eax), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0x9b,0x8c,0x02,0x00,0x01,0x00,0x00]
+vfmsub132ss 256(%edx,%eax), %xmm1, %xmm1
+
+// CHECK: vfmsub132ss 256(%edx,%eax), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0x9b,0x4c,0x02,0x40]
+vfmsub132ss 256(%edx,%eax), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub132ss 256(%edx,%eax), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0x9b,0x4c,0x02,0x40]
+vfmsub132ss 256(%edx,%eax), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub132ss -485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0x9b,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vfmsub132ss -485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vfmsub132ss 485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0x9b,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vfmsub132ss 485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vfmsub132ss -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0x9b,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vfmsub132ss -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub132ss 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0x9b,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vfmsub132ss 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub132ss -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0x9b,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vfmsub132ss -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub132ss 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0x9b,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vfmsub132ss 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub132ss 485498096(%edx), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0x9b,0x8a,0xf0,0x1c,0xf0,0x1c]
+vfmsub132ss 485498096(%edx), %xmm1, %xmm1
+
+// CHECK: vfmsub132ss 485498096(%edx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0x9b,0x8a,0xf0,0x1c,0xf0,0x1c]
+vfmsub132ss 485498096(%edx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub132ss 485498096(%edx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0x9b,0x8a,0xf0,0x1c,0xf0,0x1c]
+vfmsub132ss 485498096(%edx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub132ss 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0x9b,0x0d,0xf0,0x1c,0xf0,0x1c]
+vfmsub132ss 485498096, %xmm1, %xmm1
+
+// CHECK: vfmsub132ss 485498096, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0x9b,0x0d,0xf0,0x1c,0xf0,0x1c]
+vfmsub132ss 485498096, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub132ss 485498096, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0x9b,0x0d,0xf0,0x1c,0xf0,0x1c]
+vfmsub132ss 485498096, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub132ss (%edx), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0x9b,0x0a]
+vfmsub132ss (%edx), %xmm1, %xmm1
+
+// CHECK: vfmsub132ss (%edx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0x9b,0x0a]
+vfmsub132ss (%edx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub132ss (%edx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0x9b,0x0a]
+vfmsub132ss (%edx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub132ss {rd-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x38,0x9b,0xc9]
+vfmsub132ss {rd-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfmsub132ss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x3a,0x9b,0xc9]
+vfmsub132ss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub132ss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0xba,0x9b,0xc9]
+vfmsub132ss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub132ss {rn-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x18,0x9b,0xc9]
+vfmsub132ss {rn-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfmsub132ss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x1a,0x9b,0xc9]
+vfmsub132ss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub132ss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x9a,0x9b,0xc9]
+vfmsub132ss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub132ss {ru-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x58,0x9b,0xc9]
+vfmsub132ss {ru-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfmsub132ss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x5a,0x9b,0xc9]
+vfmsub132ss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub132ss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0xda,0x9b,0xc9]
+vfmsub132ss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub132ss {rz-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x78,0x9b,0xc9]
+vfmsub132ss {rz-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfmsub132ss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x7a,0x9b,0xc9]
+vfmsub132ss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub132ss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0xfa,0x9b,0xc9]
+vfmsub132ss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub132ss %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0x9b,0xc9]
+vfmsub132ss %xmm1, %xmm1, %xmm1
+
+// CHECK: vfmsub132ss %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0x9b,0xc9]
+vfmsub132ss %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub132ss %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0x9b,0xc9]
+vfmsub132ss %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub213sd -485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0xab,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vfmsub213sd -485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vfmsub213sd 485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0xab,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vfmsub213sd 485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vfmsub213sd -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0xab,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vfmsub213sd -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub213sd 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0xab,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vfmsub213sd 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub213sd -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0xab,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vfmsub213sd -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub213sd 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0xab,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vfmsub213sd 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub213sd 485498096(%edx), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0xab,0x8a,0xf0,0x1c,0xf0,0x1c]
+vfmsub213sd 485498096(%edx), %xmm1, %xmm1
+
+// CHECK: vfmsub213sd 485498096(%edx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0xab,0x8a,0xf0,0x1c,0xf0,0x1c]
+vfmsub213sd 485498096(%edx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub213sd 485498096(%edx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0xab,0x8a,0xf0,0x1c,0xf0,0x1c]
+vfmsub213sd 485498096(%edx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub213sd 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0xab,0x0d,0xf0,0x1c,0xf0,0x1c]
+vfmsub213sd 485498096, %xmm1, %xmm1
+
+// CHECK: vfmsub213sd 485498096, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0xab,0x0d,0xf0,0x1c,0xf0,0x1c]
+vfmsub213sd 485498096, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub213sd 485498096, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0xab,0x0d,0xf0,0x1c,0xf0,0x1c]
+vfmsub213sd 485498096, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub213sd 512(%edx,%eax), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0xab,0x8c,0x02,0x00,0x02,0x00,0x00]
+vfmsub213sd 512(%edx,%eax), %xmm1, %xmm1
+
+// CHECK: vfmsub213sd 512(%edx,%eax), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0xab,0x4c,0x02,0x40]
+vfmsub213sd 512(%edx,%eax), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub213sd 512(%edx,%eax), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0xab,0x4c,0x02,0x40]
+vfmsub213sd 512(%edx,%eax), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub213sd (%edx), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0xab,0x0a]
+vfmsub213sd (%edx), %xmm1, %xmm1
+
+// CHECK: vfmsub213sd (%edx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0xab,0x0a]
+vfmsub213sd (%edx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub213sd (%edx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0xab,0x0a]
+vfmsub213sd (%edx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub213sd {rd-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x38,0xab,0xc9]
+vfmsub213sd {rd-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfmsub213sd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x3a,0xab,0xc9]
+vfmsub213sd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub213sd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0xba,0xab,0xc9]
+vfmsub213sd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub213sd {rn-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x18,0xab,0xc9]
+vfmsub213sd {rn-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfmsub213sd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x1a,0xab,0xc9]
+vfmsub213sd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub213sd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x9a,0xab,0xc9]
+vfmsub213sd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub213sd {ru-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x58,0xab,0xc9]
+vfmsub213sd {ru-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfmsub213sd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x5a,0xab,0xc9]
+vfmsub213sd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub213sd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0xda,0xab,0xc9]
+vfmsub213sd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub213sd {rz-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x78,0xab,0xc9]
+vfmsub213sd {rz-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfmsub213sd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x7a,0xab,0xc9]
+vfmsub213sd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub213sd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0xfa,0xab,0xc9]
+vfmsub213sd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub213sd %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0xab,0xc9]
+vfmsub213sd %xmm1, %xmm1, %xmm1
+
+// CHECK: vfmsub213sd %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0xab,0xc9]
+vfmsub213sd %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub213sd %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0xab,0xc9]
+vfmsub213sd %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub213ss 256(%edx,%eax), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0xab,0x8c,0x02,0x00,0x01,0x00,0x00]
+vfmsub213ss 256(%edx,%eax), %xmm1, %xmm1
+
+// CHECK: vfmsub213ss 256(%edx,%eax), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0xab,0x4c,0x02,0x40]
+vfmsub213ss 256(%edx,%eax), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub213ss 256(%edx,%eax), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0xab,0x4c,0x02,0x40]
+vfmsub213ss 256(%edx,%eax), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub213ss -485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0xab,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vfmsub213ss -485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vfmsub213ss 485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0xab,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vfmsub213ss 485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vfmsub213ss -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0xab,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vfmsub213ss -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub213ss 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0xab,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vfmsub213ss 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub213ss -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0xab,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vfmsub213ss -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub213ss 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0xab,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vfmsub213ss 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub213ss 485498096(%edx), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0xab,0x8a,0xf0,0x1c,0xf0,0x1c]
+vfmsub213ss 485498096(%edx), %xmm1, %xmm1
+
+// CHECK: vfmsub213ss 485498096(%edx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0xab,0x8a,0xf0,0x1c,0xf0,0x1c]
+vfmsub213ss 485498096(%edx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub213ss 485498096(%edx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0xab,0x8a,0xf0,0x1c,0xf0,0x1c]
+vfmsub213ss 485498096(%edx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub213ss 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0xab,0x0d,0xf0,0x1c,0xf0,0x1c]
+vfmsub213ss 485498096, %xmm1, %xmm1
+
+// CHECK: vfmsub213ss 485498096, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0xab,0x0d,0xf0,0x1c,0xf0,0x1c]
+vfmsub213ss 485498096, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub213ss 485498096, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0xab,0x0d,0xf0,0x1c,0xf0,0x1c]
+vfmsub213ss 485498096, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub213ss (%edx), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0xab,0x0a]
+vfmsub213ss (%edx), %xmm1, %xmm1
+
+// CHECK: vfmsub213ss (%edx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0xab,0x0a]
+vfmsub213ss (%edx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub213ss (%edx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0xab,0x0a]
+vfmsub213ss (%edx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub213ss {rd-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x38,0xab,0xc9]
+vfmsub213ss {rd-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfmsub213ss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x3a,0xab,0xc9]
+vfmsub213ss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub213ss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0xba,0xab,0xc9]
+vfmsub213ss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub213ss {rn-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x18,0xab,0xc9]
+vfmsub213ss {rn-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfmsub213ss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x1a,0xab,0xc9]
+vfmsub213ss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub213ss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x9a,0xab,0xc9]
+vfmsub213ss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub213ss {ru-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x58,0xab,0xc9]
+vfmsub213ss {ru-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfmsub213ss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x5a,0xab,0xc9]
+vfmsub213ss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub213ss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0xda,0xab,0xc9]
+vfmsub213ss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub213ss {rz-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x78,0xab,0xc9]
+vfmsub213ss {rz-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfmsub213ss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x7a,0xab,0xc9]
+vfmsub213ss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub213ss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0xfa,0xab,0xc9]
+vfmsub213ss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub213ss %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0xab,0xc9]
+vfmsub213ss %xmm1, %xmm1, %xmm1
+
+// CHECK: vfmsub213ss %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0xab,0xc9]
+vfmsub213ss %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub213ss %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0xab,0xc9]
+vfmsub213ss %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub231sd -485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0xbb,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vfmsub231sd -485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vfmsub231sd 485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0xbb,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vfmsub231sd 485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vfmsub231sd -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0xbb,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vfmsub231sd -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub231sd 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0xbb,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vfmsub231sd 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub231sd -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0xbb,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vfmsub231sd -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub231sd 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0xbb,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vfmsub231sd 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub231sd 485498096(%edx), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0xbb,0x8a,0xf0,0x1c,0xf0,0x1c]
+vfmsub231sd 485498096(%edx), %xmm1, %xmm1
+
+// CHECK: vfmsub231sd 485498096(%edx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0xbb,0x8a,0xf0,0x1c,0xf0,0x1c]
+vfmsub231sd 485498096(%edx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub231sd 485498096(%edx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0xbb,0x8a,0xf0,0x1c,0xf0,0x1c]
+vfmsub231sd 485498096(%edx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub231sd 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0xbb,0x0d,0xf0,0x1c,0xf0,0x1c]
+vfmsub231sd 485498096, %xmm1, %xmm1
+
+// CHECK: vfmsub231sd 485498096, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0xbb,0x0d,0xf0,0x1c,0xf0,0x1c]
+vfmsub231sd 485498096, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub231sd 485498096, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0xbb,0x0d,0xf0,0x1c,0xf0,0x1c]
+vfmsub231sd 485498096, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub231sd 512(%edx,%eax), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0xbb,0x8c,0x02,0x00,0x02,0x00,0x00]
+vfmsub231sd 512(%edx,%eax), %xmm1, %xmm1
+
+// CHECK: vfmsub231sd 512(%edx,%eax), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0xbb,0x4c,0x02,0x40]
+vfmsub231sd 512(%edx,%eax), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub231sd 512(%edx,%eax), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0xbb,0x4c,0x02,0x40]
+vfmsub231sd 512(%edx,%eax), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub231sd (%edx), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0xbb,0x0a]
+vfmsub231sd (%edx), %xmm1, %xmm1
+
+// CHECK: vfmsub231sd (%edx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0xbb,0x0a]
+vfmsub231sd (%edx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub231sd (%edx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0xbb,0x0a]
+vfmsub231sd (%edx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub231sd {rd-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x38,0xbb,0xc9]
+vfmsub231sd {rd-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfmsub231sd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x3a,0xbb,0xc9]
+vfmsub231sd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub231sd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0xba,0xbb,0xc9]
+vfmsub231sd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub231sd {rn-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x18,0xbb,0xc9]
+vfmsub231sd {rn-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfmsub231sd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x1a,0xbb,0xc9]
+vfmsub231sd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub231sd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x9a,0xbb,0xc9]
+vfmsub231sd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub231sd {ru-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x58,0xbb,0xc9]
+vfmsub231sd {ru-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfmsub231sd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x5a,0xbb,0xc9]
+vfmsub231sd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub231sd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0xda,0xbb,0xc9]
+vfmsub231sd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub231sd {rz-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x78,0xbb,0xc9]
+vfmsub231sd {rz-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfmsub231sd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x7a,0xbb,0xc9]
+vfmsub231sd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub231sd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0xfa,0xbb,0xc9]
+vfmsub231sd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub231sd %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0xbb,0xc9]
+vfmsub231sd %xmm1, %xmm1, %xmm1
+
+// CHECK: vfmsub231sd %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0xbb,0xc9]
+vfmsub231sd %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub231sd %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0xbb,0xc9]
+vfmsub231sd %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub231ss 256(%edx,%eax), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0xbb,0x8c,0x02,0x00,0x01,0x00,0x00]
+vfmsub231ss 256(%edx,%eax), %xmm1, %xmm1
+
+// CHECK: vfmsub231ss 256(%edx,%eax), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0xbb,0x4c,0x02,0x40]
+vfmsub231ss 256(%edx,%eax), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub231ss 256(%edx,%eax), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0xbb,0x4c,0x02,0x40]
+vfmsub231ss 256(%edx,%eax), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub231ss -485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0xbb,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vfmsub231ss -485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vfmsub231ss 485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0xbb,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vfmsub231ss 485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vfmsub231ss -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0xbb,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vfmsub231ss -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub231ss 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0xbb,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vfmsub231ss 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub231ss -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0xbb,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vfmsub231ss -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub231ss 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0xbb,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vfmsub231ss 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub231ss 485498096(%edx), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0xbb,0x8a,0xf0,0x1c,0xf0,0x1c]
+vfmsub231ss 485498096(%edx), %xmm1, %xmm1
+
+// CHECK: vfmsub231ss 485498096(%edx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0xbb,0x8a,0xf0,0x1c,0xf0,0x1c]
+vfmsub231ss 485498096(%edx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub231ss 485498096(%edx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0xbb,0x8a,0xf0,0x1c,0xf0,0x1c]
+vfmsub231ss 485498096(%edx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub231ss 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0xbb,0x0d,0xf0,0x1c,0xf0,0x1c]
+vfmsub231ss 485498096, %xmm1, %xmm1
+
+// CHECK: vfmsub231ss 485498096, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0xbb,0x0d,0xf0,0x1c,0xf0,0x1c]
+vfmsub231ss 485498096, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub231ss 485498096, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0xbb,0x0d,0xf0,0x1c,0xf0,0x1c]
+vfmsub231ss 485498096, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub231ss (%edx), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0xbb,0x0a]
+vfmsub231ss (%edx), %xmm1, %xmm1
+
+// CHECK: vfmsub231ss (%edx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0xbb,0x0a]
+vfmsub231ss (%edx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub231ss (%edx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0xbb,0x0a]
+vfmsub231ss (%edx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub231ss {rd-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x38,0xbb,0xc9]
+vfmsub231ss {rd-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfmsub231ss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x3a,0xbb,0xc9]
+vfmsub231ss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub231ss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0xba,0xbb,0xc9]
+vfmsub231ss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub231ss {rn-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x18,0xbb,0xc9]
+vfmsub231ss {rn-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfmsub231ss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x1a,0xbb,0xc9]
+vfmsub231ss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub231ss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x9a,0xbb,0xc9]
+vfmsub231ss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub231ss {ru-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x58,0xbb,0xc9]
+vfmsub231ss {ru-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfmsub231ss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x5a,0xbb,0xc9]
+vfmsub231ss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub231ss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0xda,0xbb,0xc9]
+vfmsub231ss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub231ss {rz-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x78,0xbb,0xc9]
+vfmsub231ss {rz-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfmsub231ss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x7a,0xbb,0xc9]
+vfmsub231ss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub231ss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0xfa,0xbb,0xc9]
+vfmsub231ss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub231ss %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0xbb,0xc9]
+vfmsub231ss %xmm1, %xmm1, %xmm1
+
+// CHECK: vfmsub231ss %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0xbb,0xc9]
+vfmsub231ss %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub231ss %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0xbb,0xc9]
+vfmsub231ss %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd132sd -485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0x9d,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vfnmadd132sd -485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vfnmadd132sd 485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0x9d,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vfnmadd132sd 485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vfnmadd132sd -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0x9d,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vfnmadd132sd -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd132sd 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0x9d,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vfnmadd132sd 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd132sd -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0x9d,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vfnmadd132sd -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd132sd 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0x9d,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vfnmadd132sd 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd132sd 485498096(%edx), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0x9d,0x8a,0xf0,0x1c,0xf0,0x1c]
+vfnmadd132sd 485498096(%edx), %xmm1, %xmm1
+
+// CHECK: vfnmadd132sd 485498096(%edx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0x9d,0x8a,0xf0,0x1c,0xf0,0x1c]
+vfnmadd132sd 485498096(%edx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd132sd 485498096(%edx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0x9d,0x8a,0xf0,0x1c,0xf0,0x1c]
+vfnmadd132sd 485498096(%edx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd132sd 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0x9d,0x0d,0xf0,0x1c,0xf0,0x1c]
+vfnmadd132sd 485498096, %xmm1, %xmm1
+
+// CHECK: vfnmadd132sd 485498096, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0x9d,0x0d,0xf0,0x1c,0xf0,0x1c]
+vfnmadd132sd 485498096, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd132sd 485498096, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0x9d,0x0d,0xf0,0x1c,0xf0,0x1c]
+vfnmadd132sd 485498096, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd132sd 512(%edx,%eax), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0x9d,0x8c,0x02,0x00,0x02,0x00,0x00]
+vfnmadd132sd 512(%edx,%eax), %xmm1, %xmm1
+
+// CHECK: vfnmadd132sd 512(%edx,%eax), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0x9d,0x4c,0x02,0x40]
+vfnmadd132sd 512(%edx,%eax), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd132sd 512(%edx,%eax), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0x9d,0x4c,0x02,0x40]
+vfnmadd132sd 512(%edx,%eax), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd132sd (%edx), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0x9d,0x0a]
+vfnmadd132sd (%edx), %xmm1, %xmm1
+
+// CHECK: vfnmadd132sd (%edx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0x9d,0x0a]
+vfnmadd132sd (%edx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd132sd (%edx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0x9d,0x0a]
+vfnmadd132sd (%edx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd132sd {rd-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x38,0x9d,0xc9]
+vfnmadd132sd {rd-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfnmadd132sd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x3a,0x9d,0xc9]
+vfnmadd132sd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd132sd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0xba,0x9d,0xc9]
+vfnmadd132sd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd132sd {rn-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x18,0x9d,0xc9]
+vfnmadd132sd {rn-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfnmadd132sd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x1a,0x9d,0xc9]
+vfnmadd132sd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd132sd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x9a,0x9d,0xc9]
+vfnmadd132sd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd132sd {ru-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x58,0x9d,0xc9]
+vfnmadd132sd {ru-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfnmadd132sd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x5a,0x9d,0xc9]
+vfnmadd132sd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd132sd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0xda,0x9d,0xc9]
+vfnmadd132sd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd132sd {rz-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x78,0x9d,0xc9]
+vfnmadd132sd {rz-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfnmadd132sd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x7a,0x9d,0xc9]
+vfnmadd132sd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd132sd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0xfa,0x9d,0xc9]
+vfnmadd132sd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd132sd %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0x9d,0xc9]
+vfnmadd132sd %xmm1, %xmm1, %xmm1
+
+// CHECK: vfnmadd132sd %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0x9d,0xc9]
+vfnmadd132sd %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd132sd %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0x9d,0xc9]
+vfnmadd132sd %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd132ss 256(%edx,%eax), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0x9d,0x8c,0x02,0x00,0x01,0x00,0x00]
+vfnmadd132ss 256(%edx,%eax), %xmm1, %xmm1
+
+// CHECK: vfnmadd132ss 256(%edx,%eax), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0x9d,0x4c,0x02,0x40]
+vfnmadd132ss 256(%edx,%eax), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd132ss 256(%edx,%eax), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0x9d,0x4c,0x02,0x40]
+vfnmadd132ss 256(%edx,%eax), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd132ss -485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0x9d,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vfnmadd132ss -485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vfnmadd132ss 485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0x9d,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vfnmadd132ss 485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vfnmadd132ss -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0x9d,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vfnmadd132ss -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd132ss 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0x9d,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vfnmadd132ss 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd132ss -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0x9d,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vfnmadd132ss -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd132ss 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0x9d,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vfnmadd132ss 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd132ss 485498096(%edx), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0x9d,0x8a,0xf0,0x1c,0xf0,0x1c]
+vfnmadd132ss 485498096(%edx), %xmm1, %xmm1
+
+// CHECK: vfnmadd132ss 485498096(%edx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0x9d,0x8a,0xf0,0x1c,0xf0,0x1c]
+vfnmadd132ss 485498096(%edx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd132ss 485498096(%edx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0x9d,0x8a,0xf0,0x1c,0xf0,0x1c]
+vfnmadd132ss 485498096(%edx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd132ss 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0x9d,0x0d,0xf0,0x1c,0xf0,0x1c]
+vfnmadd132ss 485498096, %xmm1, %xmm1
+
+// CHECK: vfnmadd132ss 485498096, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0x9d,0x0d,0xf0,0x1c,0xf0,0x1c]
+vfnmadd132ss 485498096, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd132ss 485498096, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0x9d,0x0d,0xf0,0x1c,0xf0,0x1c]
+vfnmadd132ss 485498096, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd132ss (%edx), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0x9d,0x0a]
+vfnmadd132ss (%edx), %xmm1, %xmm1
+
+// CHECK: vfnmadd132ss (%edx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0x9d,0x0a]
+vfnmadd132ss (%edx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd132ss (%edx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0x9d,0x0a]
+vfnmadd132ss (%edx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd132ss {rd-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x38,0x9d,0xc9]
+vfnmadd132ss {rd-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfnmadd132ss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x3a,0x9d,0xc9]
+vfnmadd132ss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd132ss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0xba,0x9d,0xc9]
+vfnmadd132ss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd132ss {rn-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x18,0x9d,0xc9]
+vfnmadd132ss {rn-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfnmadd132ss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x1a,0x9d,0xc9]
+vfnmadd132ss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd132ss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x9a,0x9d,0xc9]
+vfnmadd132ss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd132ss {ru-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x58,0x9d,0xc9]
+vfnmadd132ss {ru-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfnmadd132ss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x5a,0x9d,0xc9]
+vfnmadd132ss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd132ss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0xda,0x9d,0xc9]
+vfnmadd132ss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd132ss {rz-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x78,0x9d,0xc9]
+vfnmadd132ss {rz-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfnmadd132ss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x7a,0x9d,0xc9]
+vfnmadd132ss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd132ss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0xfa,0x9d,0xc9]
+vfnmadd132ss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd132ss %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0x9d,0xc9]
+vfnmadd132ss %xmm1, %xmm1, %xmm1
+
+// CHECK: vfnmadd132ss %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0x9d,0xc9]
+vfnmadd132ss %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd132ss %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0x9d,0xc9]
+vfnmadd132ss %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd213sd -485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0xad,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vfnmadd213sd -485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vfnmadd213sd 485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0xad,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vfnmadd213sd 485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vfnmadd213sd -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0xad,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vfnmadd213sd -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd213sd 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0xad,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vfnmadd213sd 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd213sd -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0xad,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vfnmadd213sd -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd213sd 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0xad,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vfnmadd213sd 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd213sd 485498096(%edx), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0xad,0x8a,0xf0,0x1c,0xf0,0x1c]
+vfnmadd213sd 485498096(%edx), %xmm1, %xmm1
+
+// CHECK: vfnmadd213sd 485498096(%edx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0xad,0x8a,0xf0,0x1c,0xf0,0x1c]
+vfnmadd213sd 485498096(%edx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd213sd 485498096(%edx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0xad,0x8a,0xf0,0x1c,0xf0,0x1c]
+vfnmadd213sd 485498096(%edx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd213sd 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0xad,0x0d,0xf0,0x1c,0xf0,0x1c]
+vfnmadd213sd 485498096, %xmm1, %xmm1
+
+// CHECK: vfnmadd213sd 485498096, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0xad,0x0d,0xf0,0x1c,0xf0,0x1c]
+vfnmadd213sd 485498096, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd213sd 485498096, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0xad,0x0d,0xf0,0x1c,0xf0,0x1c]
+vfnmadd213sd 485498096, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd213sd 512(%edx,%eax), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0xad,0x8c,0x02,0x00,0x02,0x00,0x00]
+vfnmadd213sd 512(%edx,%eax), %xmm1, %xmm1
+
+// CHECK: vfnmadd213sd 512(%edx,%eax), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0xad,0x4c,0x02,0x40]
+vfnmadd213sd 512(%edx,%eax), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd213sd 512(%edx,%eax), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0xad,0x4c,0x02,0x40]
+vfnmadd213sd 512(%edx,%eax), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd213sd (%edx), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0xad,0x0a]
+vfnmadd213sd (%edx), %xmm1, %xmm1
+
+// CHECK: vfnmadd213sd (%edx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0xad,0x0a]
+vfnmadd213sd (%edx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd213sd (%edx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0xad,0x0a]
+vfnmadd213sd (%edx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd213sd {rd-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x38,0xad,0xc9]
+vfnmadd213sd {rd-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfnmadd213sd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x3a,0xad,0xc9]
+vfnmadd213sd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd213sd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0xba,0xad,0xc9]
+vfnmadd213sd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd213sd {rn-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x18,0xad,0xc9]
+vfnmadd213sd {rn-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfnmadd213sd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x1a,0xad,0xc9]
+vfnmadd213sd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd213sd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x9a,0xad,0xc9]
+vfnmadd213sd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd213sd {ru-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x58,0xad,0xc9]
+vfnmadd213sd {ru-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfnmadd213sd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x5a,0xad,0xc9]
+vfnmadd213sd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd213sd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0xda,0xad,0xc9]
+vfnmadd213sd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd213sd {rz-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x78,0xad,0xc9]
+vfnmadd213sd {rz-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfnmadd213sd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x7a,0xad,0xc9]
+vfnmadd213sd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd213sd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0xfa,0xad,0xc9]
+vfnmadd213sd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd213sd %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0xad,0xc9]
+vfnmadd213sd %xmm1, %xmm1, %xmm1
+
+// CHECK: vfnmadd213sd %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0xad,0xc9]
+vfnmadd213sd %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd213sd %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0xad,0xc9]
+vfnmadd213sd %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd213ss 256(%edx,%eax), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0xad,0x8c,0x02,0x00,0x01,0x00,0x00]
+vfnmadd213ss 256(%edx,%eax), %xmm1, %xmm1
+
+// CHECK: vfnmadd213ss 256(%edx,%eax), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0xad,0x4c,0x02,0x40]
+vfnmadd213ss 256(%edx,%eax), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd213ss 256(%edx,%eax), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0xad,0x4c,0x02,0x40]
+vfnmadd213ss 256(%edx,%eax), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd213ss -485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0xad,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vfnmadd213ss -485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vfnmadd213ss 485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0xad,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vfnmadd213ss 485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vfnmadd213ss -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0xad,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vfnmadd213ss -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd213ss 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0xad,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vfnmadd213ss 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd213ss -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0xad,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vfnmadd213ss -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd213ss 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0xad,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vfnmadd213ss 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd213ss 485498096(%edx), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0xad,0x8a,0xf0,0x1c,0xf0,0x1c]
+vfnmadd213ss 485498096(%edx), %xmm1, %xmm1
+
+// CHECK: vfnmadd213ss 485498096(%edx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0xad,0x8a,0xf0,0x1c,0xf0,0x1c]
+vfnmadd213ss 485498096(%edx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd213ss 485498096(%edx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0xad,0x8a,0xf0,0x1c,0xf0,0x1c]
+vfnmadd213ss 485498096(%edx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd213ss 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0xad,0x0d,0xf0,0x1c,0xf0,0x1c]
+vfnmadd213ss 485498096, %xmm1, %xmm1
+
+// CHECK: vfnmadd213ss 485498096, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0xad,0x0d,0xf0,0x1c,0xf0,0x1c]
+vfnmadd213ss 485498096, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd213ss 485498096, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0xad,0x0d,0xf0,0x1c,0xf0,0x1c]
+vfnmadd213ss 485498096, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd213ss (%edx), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0xad,0x0a]
+vfnmadd213ss (%edx), %xmm1, %xmm1
+
+// CHECK: vfnmadd213ss (%edx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0xad,0x0a]
+vfnmadd213ss (%edx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd213ss (%edx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0xad,0x0a]
+vfnmadd213ss (%edx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd213ss {rd-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x38,0xad,0xc9]
+vfnmadd213ss {rd-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfnmadd213ss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x3a,0xad,0xc9]
+vfnmadd213ss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd213ss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0xba,0xad,0xc9]
+vfnmadd213ss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd213ss {rn-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x18,0xad,0xc9]
+vfnmadd213ss {rn-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfnmadd213ss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x1a,0xad,0xc9]
+vfnmadd213ss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd213ss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x9a,0xad,0xc9]
+vfnmadd213ss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd213ss {ru-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x58,0xad,0xc9]
+vfnmadd213ss {ru-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfnmadd213ss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x5a,0xad,0xc9]
+vfnmadd213ss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd213ss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0xda,0xad,0xc9]
+vfnmadd213ss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd213ss {rz-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x78,0xad,0xc9]
+vfnmadd213ss {rz-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfnmadd213ss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x7a,0xad,0xc9]
+vfnmadd213ss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd213ss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0xfa,0xad,0xc9]
+vfnmadd213ss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd213ss %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0xad,0xc9]
+vfnmadd213ss %xmm1, %xmm1, %xmm1
+
+// CHECK: vfnmadd213ss %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0xad,0xc9]
+vfnmadd213ss %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd213ss %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0xad,0xc9]
+vfnmadd213ss %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd231sd -485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0xbd,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vfnmadd231sd -485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vfnmadd231sd 485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0xbd,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vfnmadd231sd 485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vfnmadd231sd -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0xbd,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vfnmadd231sd -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd231sd 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0xbd,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vfnmadd231sd 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd231sd -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0xbd,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vfnmadd231sd -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd231sd 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0xbd,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vfnmadd231sd 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd231sd 485498096(%edx), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0xbd,0x8a,0xf0,0x1c,0xf0,0x1c]
+vfnmadd231sd 485498096(%edx), %xmm1, %xmm1
+
+// CHECK: vfnmadd231sd 485498096(%edx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0xbd,0x8a,0xf0,0x1c,0xf0,0x1c]
+vfnmadd231sd 485498096(%edx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd231sd 485498096(%edx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0xbd,0x8a,0xf0,0x1c,0xf0,0x1c]
+vfnmadd231sd 485498096(%edx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd231sd 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0xbd,0x0d,0xf0,0x1c,0xf0,0x1c]
+vfnmadd231sd 485498096, %xmm1, %xmm1
+
+// CHECK: vfnmadd231sd 485498096, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0xbd,0x0d,0xf0,0x1c,0xf0,0x1c]
+vfnmadd231sd 485498096, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd231sd 485498096, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0xbd,0x0d,0xf0,0x1c,0xf0,0x1c]
+vfnmadd231sd 485498096, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd231sd 512(%edx,%eax), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0xbd,0x8c,0x02,0x00,0x02,0x00,0x00]
+vfnmadd231sd 512(%edx,%eax), %xmm1, %xmm1
+
+// CHECK: vfnmadd231sd 512(%edx,%eax), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0xbd,0x4c,0x02,0x40]
+vfnmadd231sd 512(%edx,%eax), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd231sd 512(%edx,%eax), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0xbd,0x4c,0x02,0x40]
+vfnmadd231sd 512(%edx,%eax), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd231sd (%edx), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0xbd,0x0a]
+vfnmadd231sd (%edx), %xmm1, %xmm1
+
+// CHECK: vfnmadd231sd (%edx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0xbd,0x0a]
+vfnmadd231sd (%edx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd231sd (%edx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0xbd,0x0a]
+vfnmadd231sd (%edx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd231sd {rd-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x38,0xbd,0xc9]
+vfnmadd231sd {rd-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfnmadd231sd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x3a,0xbd,0xc9]
+vfnmadd231sd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd231sd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0xba,0xbd,0xc9]
+vfnmadd231sd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd231sd {rn-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x18,0xbd,0xc9]
+vfnmadd231sd {rn-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfnmadd231sd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x1a,0xbd,0xc9]
+vfnmadd231sd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd231sd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x9a,0xbd,0xc9]
+vfnmadd231sd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd231sd {ru-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x58,0xbd,0xc9]
+vfnmadd231sd {ru-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfnmadd231sd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x5a,0xbd,0xc9]
+vfnmadd231sd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd231sd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0xda,0xbd,0xc9]
+vfnmadd231sd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd231sd {rz-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x78,0xbd,0xc9]
+vfnmadd231sd {rz-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfnmadd231sd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x7a,0xbd,0xc9]
+vfnmadd231sd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd231sd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0xfa,0xbd,0xc9]
+vfnmadd231sd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd231sd %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0xbd,0xc9]
+vfnmadd231sd %xmm1, %xmm1, %xmm1
+
+// CHECK: vfnmadd231sd %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0xbd,0xc9]
+vfnmadd231sd %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd231sd %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0xbd,0xc9]
+vfnmadd231sd %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd231ss 256(%edx,%eax), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0xbd,0x8c,0x02,0x00,0x01,0x00,0x00]
+vfnmadd231ss 256(%edx,%eax), %xmm1, %xmm1
+
+// CHECK: vfnmadd231ss 256(%edx,%eax), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0xbd,0x4c,0x02,0x40]
+vfnmadd231ss 256(%edx,%eax), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd231ss 256(%edx,%eax), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0xbd,0x4c,0x02,0x40]
+vfnmadd231ss 256(%edx,%eax), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd231ss -485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0xbd,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vfnmadd231ss -485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vfnmadd231ss 485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0xbd,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vfnmadd231ss 485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vfnmadd231ss -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0xbd,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vfnmadd231ss -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd231ss 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0xbd,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vfnmadd231ss 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd231ss -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0xbd,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vfnmadd231ss -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd231ss 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0xbd,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vfnmadd231ss 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd231ss 485498096(%edx), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0xbd,0x8a,0xf0,0x1c,0xf0,0x1c]
+vfnmadd231ss 485498096(%edx), %xmm1, %xmm1
+
+// CHECK: vfnmadd231ss 485498096(%edx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0xbd,0x8a,0xf0,0x1c,0xf0,0x1c]
+vfnmadd231ss 485498096(%edx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd231ss 485498096(%edx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0xbd,0x8a,0xf0,0x1c,0xf0,0x1c]
+vfnmadd231ss 485498096(%edx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd231ss 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0xbd,0x0d,0xf0,0x1c,0xf0,0x1c]
+vfnmadd231ss 485498096, %xmm1, %xmm1
+
+// CHECK: vfnmadd231ss 485498096, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0xbd,0x0d,0xf0,0x1c,0xf0,0x1c]
+vfnmadd231ss 485498096, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd231ss 485498096, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0xbd,0x0d,0xf0,0x1c,0xf0,0x1c]
+vfnmadd231ss 485498096, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd231ss (%edx), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0xbd,0x0a]
+vfnmadd231ss (%edx), %xmm1, %xmm1
+
+// CHECK: vfnmadd231ss (%edx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0xbd,0x0a]
+vfnmadd231ss (%edx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd231ss (%edx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0xbd,0x0a]
+vfnmadd231ss (%edx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd231ss {rd-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x38,0xbd,0xc9]
+vfnmadd231ss {rd-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfnmadd231ss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x3a,0xbd,0xc9]
+vfnmadd231ss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd231ss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0xba,0xbd,0xc9]
+vfnmadd231ss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd231ss {rn-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x18,0xbd,0xc9]
+vfnmadd231ss {rn-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfnmadd231ss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x1a,0xbd,0xc9]
+vfnmadd231ss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd231ss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x9a,0xbd,0xc9]
+vfnmadd231ss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd231ss {ru-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x58,0xbd,0xc9]
+vfnmadd231ss {ru-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfnmadd231ss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x5a,0xbd,0xc9]
+vfnmadd231ss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd231ss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0xda,0xbd,0xc9]
+vfnmadd231ss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd231ss {rz-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x78,0xbd,0xc9]
+vfnmadd231ss {rz-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfnmadd231ss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x7a,0xbd,0xc9]
+vfnmadd231ss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd231ss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0xfa,0xbd,0xc9]
+vfnmadd231ss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd231ss %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0xbd,0xc9]
+vfnmadd231ss %xmm1, %xmm1, %xmm1
+
+// CHECK: vfnmadd231ss %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0xbd,0xc9]
+vfnmadd231ss %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd231ss %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0xbd,0xc9]
+vfnmadd231ss %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmsub132sd -485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0x9f,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vfnmsub132sd -485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vfnmsub132sd 485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0x9f,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vfnmsub132sd 485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vfnmsub132sd -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0x9f,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vfnmsub132sd -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmsub132sd 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0x9f,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vfnmsub132sd 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmsub132sd -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0x9f,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vfnmsub132sd -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmsub132sd 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0x9f,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vfnmsub132sd 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmsub132sd 485498096(%edx), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0x9f,0x8a,0xf0,0x1c,0xf0,0x1c]
+vfnmsub132sd 485498096(%edx), %xmm1, %xmm1
+
+// CHECK: vfnmsub132sd 485498096(%edx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0x9f,0x8a,0xf0,0x1c,0xf0,0x1c]
+vfnmsub132sd 485498096(%edx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmsub132sd 485498096(%edx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0x9f,0x8a,0xf0,0x1c,0xf0,0x1c]
+vfnmsub132sd 485498096(%edx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmsub132sd 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0x9f,0x0d,0xf0,0x1c,0xf0,0x1c]
+vfnmsub132sd 485498096, %xmm1, %xmm1
+
+// CHECK: vfnmsub132sd 485498096, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0x9f,0x0d,0xf0,0x1c,0xf0,0x1c]
+vfnmsub132sd 485498096, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmsub132sd 485498096, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0x9f,0x0d,0xf0,0x1c,0xf0,0x1c]
+vfnmsub132sd 485498096, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmsub132sd 512(%edx,%eax), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0x9f,0x8c,0x02,0x00,0x02,0x00,0x00]
+vfnmsub132sd 512(%edx,%eax), %xmm1, %xmm1
+
+// CHECK: vfnmsub132sd 512(%edx,%eax), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0x9f,0x4c,0x02,0x40]
+vfnmsub132sd 512(%edx,%eax), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmsub132sd 512(%edx,%eax), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0x9f,0x4c,0x02,0x40]
+vfnmsub132sd 512(%edx,%eax), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmsub132sd (%edx), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0x9f,0x0a]
+vfnmsub132sd (%edx), %xmm1, %xmm1
+
+// CHECK: vfnmsub132sd (%edx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0x9f,0x0a]
+vfnmsub132sd (%edx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmsub132sd (%edx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0x9f,0x0a]
+vfnmsub132sd (%edx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmsub132sd {rd-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x38,0x9f,0xc9]
+vfnmsub132sd {rd-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfnmsub132sd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x3a,0x9f,0xc9]
+vfnmsub132sd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmsub132sd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0xba,0x9f,0xc9]
+vfnmsub132sd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmsub132sd {rn-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x18,0x9f,0xc9]
+vfnmsub132sd {rn-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfnmsub132sd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x1a,0x9f,0xc9]
+vfnmsub132sd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmsub132sd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x9a,0x9f,0xc9]
+vfnmsub132sd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmsub132sd {ru-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x58,0x9f,0xc9]
+vfnmsub132sd {ru-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfnmsub132sd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x5a,0x9f,0xc9]
+vfnmsub132sd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmsub132sd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0xda,0x9f,0xc9]
+vfnmsub132sd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmsub132sd {rz-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x78,0x9f,0xc9]
+vfnmsub132sd {rz-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfnmsub132sd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x7a,0x9f,0xc9]
+vfnmsub132sd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmsub132sd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0xfa,0x9f,0xc9]
+vfnmsub132sd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmsub132sd %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0x9f,0xc9]
+vfnmsub132sd %xmm1, %xmm1, %xmm1
+
+// CHECK: vfnmsub132sd %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0x9f,0xc9]
+vfnmsub132sd %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmsub132sd %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0x9f,0xc9]
+vfnmsub132sd %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmsub132ss 256(%edx,%eax), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0x9f,0x8c,0x02,0x00,0x01,0x00,0x00]
+vfnmsub132ss 256(%edx,%eax), %xmm1, %xmm1
+
+// CHECK: vfnmsub132ss 256(%edx,%eax), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0x9f,0x4c,0x02,0x40]
+vfnmsub132ss 256(%edx,%eax), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmsub132ss 256(%edx,%eax), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0x9f,0x4c,0x02,0x40]
+vfnmsub132ss 256(%edx,%eax), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmsub132ss -485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0x9f,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vfnmsub132ss -485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vfnmsub132ss 485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0x9f,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vfnmsub132ss 485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vfnmsub132ss -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0x9f,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vfnmsub132ss -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmsub132ss 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0x9f,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vfnmsub132ss 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmsub132ss -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0x9f,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vfnmsub132ss -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmsub132ss 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0x9f,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vfnmsub132ss 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmsub132ss 485498096(%edx), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0x9f,0x8a,0xf0,0x1c,0xf0,0x1c]
+vfnmsub132ss 485498096(%edx), %xmm1, %xmm1
+
+// CHECK: vfnmsub132ss 485498096(%edx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0x9f,0x8a,0xf0,0x1c,0xf0,0x1c]
+vfnmsub132ss 485498096(%edx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmsub132ss 485498096(%edx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0x9f,0x8a,0xf0,0x1c,0xf0,0x1c]
+vfnmsub132ss 485498096(%edx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmsub132ss 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0x9f,0x0d,0xf0,0x1c,0xf0,0x1c]
+vfnmsub132ss 485498096, %xmm1, %xmm1
+
+// CHECK: vfnmsub132ss 485498096, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0x9f,0x0d,0xf0,0x1c,0xf0,0x1c]
+vfnmsub132ss 485498096, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmsub132ss 485498096, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0x9f,0x0d,0xf0,0x1c,0xf0,0x1c]
+vfnmsub132ss 485498096, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmsub132ss (%edx), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0x9f,0x0a]
+vfnmsub132ss (%edx), %xmm1, %xmm1
+
+// CHECK: vfnmsub132ss (%edx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0x9f,0x0a]
+vfnmsub132ss (%edx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmsub132ss (%edx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0x9f,0x0a]
+vfnmsub132ss (%edx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmsub132ss {rd-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x38,0x9f,0xc9]
+vfnmsub132ss {rd-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfnmsub132ss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x3a,0x9f,0xc9]
+vfnmsub132ss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmsub132ss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0xba,0x9f,0xc9]
+vfnmsub132ss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmsub132ss {rn-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x18,0x9f,0xc9]
+vfnmsub132ss {rn-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfnmsub132ss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x1a,0x9f,0xc9]
+vfnmsub132ss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmsub132ss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x9a,0x9f,0xc9]
+vfnmsub132ss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmsub132ss {ru-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x58,0x9f,0xc9]
+vfnmsub132ss {ru-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfnmsub132ss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x5a,0x9f,0xc9]
+vfnmsub132ss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmsub132ss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0xda,0x9f,0xc9]
+vfnmsub132ss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmsub132ss {rz-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x78,0x9f,0xc9]
+vfnmsub132ss {rz-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfnmsub132ss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x7a,0x9f,0xc9]
+vfnmsub132ss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmsub132ss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0xfa,0x9f,0xc9]
+vfnmsub132ss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmsub132ss %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0x9f,0xc9]
+vfnmsub132ss %xmm1, %xmm1, %xmm1
+
+// CHECK: vfnmsub132ss %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0x9f,0xc9]
+vfnmsub132ss %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmsub132ss %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0x9f,0xc9]
+vfnmsub132ss %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmsub213sd -485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0xaf,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vfnmsub213sd -485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vfnmsub213sd 485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0xaf,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vfnmsub213sd 485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vfnmsub213sd -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0xaf,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vfnmsub213sd -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmsub213sd 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0xaf,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vfnmsub213sd 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmsub213sd -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0xaf,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vfnmsub213sd -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmsub213sd 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0xaf,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vfnmsub213sd 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmsub213sd 485498096(%edx), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0xaf,0x8a,0xf0,0x1c,0xf0,0x1c]
+vfnmsub213sd 485498096(%edx), %xmm1, %xmm1
+
+// CHECK: vfnmsub213sd 485498096(%edx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0xaf,0x8a,0xf0,0x1c,0xf0,0x1c]
+vfnmsub213sd 485498096(%edx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmsub213sd 485498096(%edx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0xaf,0x8a,0xf0,0x1c,0xf0,0x1c]
+vfnmsub213sd 485498096(%edx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmsub213sd 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0xaf,0x0d,0xf0,0x1c,0xf0,0x1c]
+vfnmsub213sd 485498096, %xmm1, %xmm1
+
+// CHECK: vfnmsub213sd 485498096, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0xaf,0x0d,0xf0,0x1c,0xf0,0x1c]
+vfnmsub213sd 485498096, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmsub213sd 485498096, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0xaf,0x0d,0xf0,0x1c,0xf0,0x1c]
+vfnmsub213sd 485498096, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmsub213sd 512(%edx,%eax), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0xaf,0x8c,0x02,0x00,0x02,0x00,0x00]
+vfnmsub213sd 512(%edx,%eax), %xmm1, %xmm1
+
+// CHECK: vfnmsub213sd 512(%edx,%eax), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0xaf,0x4c,0x02,0x40]
+vfnmsub213sd 512(%edx,%eax), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmsub213sd 512(%edx,%eax), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0xaf,0x4c,0x02,0x40]
+vfnmsub213sd 512(%edx,%eax), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmsub213sd (%edx), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0xaf,0x0a]
+vfnmsub213sd (%edx), %xmm1, %xmm1
+
+// CHECK: vfnmsub213sd (%edx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0xaf,0x0a]
+vfnmsub213sd (%edx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmsub213sd (%edx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0xaf,0x0a]
+vfnmsub213sd (%edx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmsub213sd {rd-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x38,0xaf,0xc9]
+vfnmsub213sd {rd-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfnmsub213sd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x3a,0xaf,0xc9]
+vfnmsub213sd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmsub213sd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0xba,0xaf,0xc9]
+vfnmsub213sd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmsub213sd {rn-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x18,0xaf,0xc9]
+vfnmsub213sd {rn-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfnmsub213sd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x1a,0xaf,0xc9]
+vfnmsub213sd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmsub213sd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x9a,0xaf,0xc9]
+vfnmsub213sd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmsub213sd {ru-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x58,0xaf,0xc9]
+vfnmsub213sd {ru-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfnmsub213sd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x5a,0xaf,0xc9]
+vfnmsub213sd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmsub213sd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0xda,0xaf,0xc9]
+vfnmsub213sd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmsub213sd {rz-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x78,0xaf,0xc9]
+vfnmsub213sd {rz-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfnmsub213sd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x7a,0xaf,0xc9]
+vfnmsub213sd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmsub213sd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0xfa,0xaf,0xc9]
+vfnmsub213sd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmsub213sd %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0xaf,0xc9]
+vfnmsub213sd %xmm1, %xmm1, %xmm1
+
+// CHECK: vfnmsub213sd %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0xaf,0xc9]
+vfnmsub213sd %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmsub213sd %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0xaf,0xc9]
+vfnmsub213sd %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmsub213ss 256(%edx,%eax), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0xaf,0x8c,0x02,0x00,0x01,0x00,0x00]
+vfnmsub213ss 256(%edx,%eax), %xmm1, %xmm1
+
+// CHECK: vfnmsub213ss 256(%edx,%eax), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0xaf,0x4c,0x02,0x40]
+vfnmsub213ss 256(%edx,%eax), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmsub213ss 256(%edx,%eax), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0xaf,0x4c,0x02,0x40]
+vfnmsub213ss 256(%edx,%eax), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmsub213ss -485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0xaf,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vfnmsub213ss -485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vfnmsub213ss 485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0xaf,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vfnmsub213ss 485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vfnmsub213ss -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0xaf,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vfnmsub213ss -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmsub213ss 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0xaf,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vfnmsub213ss 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmsub213ss -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0xaf,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vfnmsub213ss -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmsub213ss 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0xaf,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vfnmsub213ss 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmsub213ss 485498096(%edx), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0xaf,0x8a,0xf0,0x1c,0xf0,0x1c]
+vfnmsub213ss 485498096(%edx), %xmm1, %xmm1
+
+// CHECK: vfnmsub213ss 485498096(%edx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0xaf,0x8a,0xf0,0x1c,0xf0,0x1c]
+vfnmsub213ss 485498096(%edx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmsub213ss 485498096(%edx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0xaf,0x8a,0xf0,0x1c,0xf0,0x1c]
+vfnmsub213ss 485498096(%edx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmsub213ss 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0xaf,0x0d,0xf0,0x1c,0xf0,0x1c]
+vfnmsub213ss 485498096, %xmm1, %xmm1
+
+// CHECK: vfnmsub213ss 485498096, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0xaf,0x0d,0xf0,0x1c,0xf0,0x1c]
+vfnmsub213ss 485498096, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmsub213ss 485498096, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0xaf,0x0d,0xf0,0x1c,0xf0,0x1c]
+vfnmsub213ss 485498096, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmsub213ss (%edx), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0xaf,0x0a]
+vfnmsub213ss (%edx), %xmm1, %xmm1
+
+// CHECK: vfnmsub213ss (%edx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0xaf,0x0a]
+vfnmsub213ss (%edx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmsub213ss (%edx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0xaf,0x0a]
+vfnmsub213ss (%edx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmsub213ss {rd-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x38,0xaf,0xc9]
+vfnmsub213ss {rd-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfnmsub213ss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x3a,0xaf,0xc9]
+vfnmsub213ss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmsub213ss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0xba,0xaf,0xc9]
+vfnmsub213ss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmsub213ss {rn-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x18,0xaf,0xc9]
+vfnmsub213ss {rn-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfnmsub213ss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x1a,0xaf,0xc9]
+vfnmsub213ss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmsub213ss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x9a,0xaf,0xc9]
+vfnmsub213ss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmsub213ss {ru-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x58,0xaf,0xc9]
+vfnmsub213ss {ru-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfnmsub213ss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x5a,0xaf,0xc9]
+vfnmsub213ss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmsub213ss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0xda,0xaf,0xc9]
+vfnmsub213ss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmsub213ss {rz-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x78,0xaf,0xc9]
+vfnmsub213ss {rz-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfnmsub213ss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x7a,0xaf,0xc9]
+vfnmsub213ss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmsub213ss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0xfa,0xaf,0xc9]
+vfnmsub213ss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmsub213ss %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0xaf,0xc9]
+vfnmsub213ss %xmm1, %xmm1, %xmm1
+
+// CHECK: vfnmsub213ss %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0xaf,0xc9]
+vfnmsub213ss %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmsub213ss %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0xaf,0xc9]
+vfnmsub213ss %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmsub231sd -485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0xbf,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vfnmsub231sd -485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vfnmsub231sd 485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0xbf,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vfnmsub231sd 485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vfnmsub231sd -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0xbf,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vfnmsub231sd -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmsub231sd 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0xbf,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vfnmsub231sd 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmsub231sd 485498096(%edx), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0xbf,0x8a,0xf0,0x1c,0xf0,0x1c]
+vfnmsub231sd 485498096(%edx), %xmm1, %xmm1
+
+// CHECK: vfnmsub231sd 485498096(%edx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0xbf,0x8a,0xf0,0x1c,0xf0,0x1c]
+vfnmsub231sd 485498096(%edx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmsub231sd 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0xbf,0x0d,0xf0,0x1c,0xf0,0x1c]
+vfnmsub231sd 485498096, %xmm1, %xmm1
+
+// CHECK: vfnmsub231sd 485498096, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0xbf,0x0d,0xf0,0x1c,0xf0,0x1c]
+vfnmsub231sd 485498096, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmsub231sd 485498096, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0xb
+vfnmsub231sd 485498096, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmsub231sd 512(%edx,%eax), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0xbf,0x8c,0x02,0x00,0x02,0x00,0x00]
+vfnmsub231sd 512(%edx,%eax), %xmm1, %xmm1
+
+// CHECK: vfnmsub231sd 512(%edx,%eax), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0xbf,0x4c,0x02,0x40]
+vfnmsub231sd 512(%edx,%eax), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmsub231sd (%edx), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0xbf,0x0a]
+vfnmsub231sd (%edx), %xmm1, %xmm1
+
+// CHECK: vfnmsub231sd (%edx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0xbf,0x0a]
+vfnmsub231sd (%edx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmsub231sd {rd-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x38,0xbf,0xc9]
+vfnmsub231sd {rd-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfnmsub231sd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x3a,0xbf,0xc9]
+vfnmsub231sd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmsub231sd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0xba,0xbf,0xc9]
+vfnmsub231sd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmsub231sd {rn-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x18,0xbf,0xc9]
+vfnmsub231sd {rn-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfnmsub231sd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x1a,0xbf,0xc9]
+vfnmsub231sd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmsub231sd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x9a,0xbf,0xc9]
+vfnmsub231sd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmsub231sd {ru-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x58,0xbf,0xc9]
+vfnmsub231sd {ru-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfnmsub231sd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x5a,0xbf,0xc9]
+vfnmsub231sd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmsub231sd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0xda,0xbf,0xc9]
+vfnmsub231sd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmsub231sd {rz-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x78,0xbf,0xc9]
+vfnmsub231sd {rz-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfnmsub231sd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x7a,0xbf,0xc9]
+vfnmsub231sd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmsub231sd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0xfa,0xbf,0xc9]
+vfnmsub231sd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmsub231sd %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0xbf,0xc9]
+vfnmsub231sd %xmm1, %xmm1, %xmm1
+
+// CHECK: vfnmsub231sd %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0xbf,0xc9]
+vfnmsub231sd %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmsub231sd %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0xbf,0xc9]
+vfnmsub231sd %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmsub231ss 256(%edx,%eax), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0xbf,0x8c,0x02,0x00,0x01,0x00,0x00]
+vfnmsub231ss 256(%edx,%eax), %xmm1, %xmm1
+
+// CHECK: vfnmsub231ss 256(%edx,%eax), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0xbf,0x4c,0x02,0x40]
+vfnmsub231ss 256(%edx,%eax), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmsub231ss 256(%edx,%eax), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0xbf,0x4c,0x02,0x40]
+vfnmsub231ss 256(%edx,%eax), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmsub231ss -485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0xbf,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vfnmsub231ss -485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vfnmsub231ss 485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0xbf,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vfnmsub231ss 485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vfnmsub231ss -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0xbf,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vfnmsub231ss -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmsub231ss 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0xbf,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vfnmsub231ss 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmsub231ss -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0xbf,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vfnmsub231ss -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmsub231ss 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0xbf,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vfnmsub231ss 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmsub231ss 485498096(%edx), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0xbf,0x8a,0xf0,0x1c,0xf0,0x1c]
+vfnmsub231ss 485498096(%edx), %xmm1, %xmm1
+
+// CHECK: vfnmsub231ss 485498096(%edx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0xbf,0x8a,0xf0,0x1c,0xf0,0x1c]
+vfnmsub231ss 485498096(%edx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmsub231ss 485498096(%edx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0xbf,0x8a,0xf0,0x1c,0xf0,0x1c]
+vfnmsub231ss 485498096(%edx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmsub231ss 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0xbf,0x0d,0xf0,0x1c,0xf0,0x1c]
+vfnmsub231ss 485498096, %xmm1, %xmm1
+
+// CHECK: vfnmsub231ss 485498096, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0xbf,0x0d,0xf0,0x1c,0xf0,0x1c]
+vfnmsub231ss 485498096, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmsub231ss 485498096, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0xbf,0x0d,0xf0,0x1c,0xf0,0x1c]
+vfnmsub231ss 485498096, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmsub231ss (%edx), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0xbf,0x0a]
+vfnmsub231ss (%edx), %xmm1, %xmm1
+
+// CHECK: vfnmsub231ss (%edx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0xbf,0x0a]
+vfnmsub231ss (%edx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmsub231ss (%edx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0xbf,0x0a]
+vfnmsub231ss (%edx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmsub231ss {rd-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x38,0xbf,0xc9]
+vfnmsub231ss {rd-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfnmsub231ss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x3a,0xbf,0xc9]
+vfnmsub231ss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmsub231ss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0xba,0xbf,0xc9]
+vfnmsub231ss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmsub231ss {rn-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x18,0xbf,0xc9]
+vfnmsub231ss {rn-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfnmsub231ss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x1a,0xbf,0xc9]
+vfnmsub231ss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmsub231ss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x9a,0xbf,0xc9]
+vfnmsub231ss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmsub231ss {ru-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x58,0xbf,0xc9]
+vfnmsub231ss {ru-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfnmsub231ss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x5a,0xbf,0xc9]
+vfnmsub231ss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmsub231ss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0xda,0xbf,0xc9]
+vfnmsub231ss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmsub231ss {rz-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x78,0xbf,0xc9]
+vfnmsub231ss {rz-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfnmsub231ss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x7a,0xbf,0xc9]
+vfnmsub231ss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmsub231ss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0xfa,0xbf,0xc9]
+vfnmsub231ss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmsub231ss %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0xbf,0xc9]
+vfnmsub231ss %xmm1, %xmm1, %xmm1
+
+// CHECK: vfnmsub231ss %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0xbf,0xc9]
+vfnmsub231ss %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmsub231ss %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0xbf,0xc9]
+vfnmsub231ss %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vgetexpsd -485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x08,0x43,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vgetexpsd -485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vgetexpsd 485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x08,0x43,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vgetexpsd 485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vgetexpsd -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0x43,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vgetexpsd -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vgetexpsd 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0x43,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vgetexpsd 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vgetexpsd -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0x43,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vgetexpsd -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vgetexpsd 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0x43,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vgetexpsd 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vgetexpsd 485498096(%edx), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x08,0x43,0x8a,0xf0,0x1c,0xf0,0x1c]
+vgetexpsd 485498096(%edx), %xmm1, %xmm1
+
+// CHECK: vgetexpsd 485498096(%edx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0x43,0x8a,0xf0,0x1c,0xf0,0x1c]
+vgetexpsd 485498096(%edx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vgetexpsd 485498096(%edx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0x43,0x8a,0xf0,0x1c,0xf0,0x1c]
+vgetexpsd 485498096(%edx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vgetexpsd 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x08,0x43,0x0d,0xf0,0x1c,0xf0,0x1c]
+vgetexpsd 485498096, %xmm1, %xmm1
+
+// CHECK: vgetexpsd 485498096, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0x43,0x0d,0xf0,0x1c,0xf0,0x1c]
+vgetexpsd 485498096, %xmm1, %xmm1 {%k2}
+
+// CHECK: vgetexpsd 485498096, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0x43,0x0d,0xf0,0x1c,0xf0,0x1c]
+vgetexpsd 485498096, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vgetexpsd 512(%edx,%eax), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x08,0x43,0x4c,0x02,0x40]
+vgetexpsd 512(%edx,%eax), %xmm1, %xmm1
+
+// CHECK: vgetexpsd 512(%edx,%eax), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0x43,0x4c,0x02,0x40]
+vgetexpsd 512(%edx,%eax), %xmm1, %xmm1 {%k2}
+
+// CHECK: vgetexpsd 512(%edx,%eax), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0x43,0x4c,0x02,0x40]
+vgetexpsd 512(%edx,%eax), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vgetexpsd (%edx), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x08,0x43,0x0a]
+vgetexpsd (%edx), %xmm1, %xmm1
+
+// CHECK: vgetexpsd (%edx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0x43,0x0a]
+vgetexpsd (%edx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vgetexpsd (%edx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0x43,0x0a]
+vgetexpsd (%edx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vgetexpsd {sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x18,0x43,0xc9]
+vgetexpsd {sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vgetexpsd {sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x1a,0x43,0xc9]
+vgetexpsd {sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vgetexpsd {sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x9a,0x43,0xc9]
+vgetexpsd {sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vgetexpsd %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x08,0x43,0xc9]
+vgetexpsd %xmm1, %xmm1, %xmm1
+
+// CHECK: vgetexpsd %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0x43,0xc9]
+vgetexpsd %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vgetexpsd %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0x43,0xc9]
+vgetexpsd %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vgetexpss 256(%edx,%eax), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x08,0x43,0x4c,0x02,0x40]
+vgetexpss 256(%edx,%eax), %xmm1, %xmm1
+
+// CHECK: vgetexpss 256(%edx,%eax), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0x43,0x4c,0x02,0x40]
+vgetexpss 256(%edx,%eax), %xmm1, %xmm1 {%k2}
+
+// CHECK: vgetexpss 256(%edx,%eax), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0x43,0x4c,0x02,0x40]
+vgetexpss 256(%edx,%eax), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vgetexpss -485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x08,0x43,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vgetexpss -485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vgetexpss 485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x08,0x43,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vgetexpss 485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vgetexpss -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0x43,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vgetexpss -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vgetexpss 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0x43,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vgetexpss 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vgetexpss -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0x43,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vgetexpss -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vgetexpss 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0x43,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vgetexpss 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vgetexpss 485498096(%edx), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x08,0x43,0x8a,0xf0,0x1c,0xf0,0x1c]
+vgetexpss 485498096(%edx), %xmm1, %xmm1
+
+// CHECK: vgetexpss 485498096(%edx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0x43,0x8a,0xf0,0x1c,0xf0,0x1c]
+vgetexpss 485498096(%edx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vgetexpss 485498096(%edx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0x43,0x8a,0xf0,0x1c,0xf0,0x1c]
+vgetexpss 485498096(%edx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vgetexpss 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x08,0x43,0x0d,0xf0,0x1c,0xf0,0x1c]
+vgetexpss 485498096, %xmm1, %xmm1
+
+// CHECK: vgetexpss 485498096, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0x43,0x0d,0xf0,0x1c,0xf0,0x1c]
+vgetexpss 485498096, %xmm1, %xmm1 {%k2}
+
+// CHECK: vgetexpss 485498096, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0x43,0x0d,0xf0,0x1c,0xf0,0x1c]
+vgetexpss 485498096, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vgetexpss (%edx), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x08,0x43,0x0a]
+vgetexpss (%edx), %xmm1, %xmm1
+
+// CHECK: vgetexpss (%edx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0x43,0x0a]
+vgetexpss (%edx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vgetexpss (%edx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0x43,0x0a]
+vgetexpss (%edx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vgetexpss {sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x18,0x43,0xc9]
+vgetexpss {sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vgetexpss {sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x1a,0x43,0xc9]
+vgetexpss {sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vgetexpss {sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x9a,0x43,0xc9]
+vgetexpss {sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vgetexpss %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x08,0x43,0xc9]
+vgetexpss %xmm1, %xmm1, %xmm1
+
+// CHECK: vgetexpss %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0x43,0xc9]
+vgetexpss %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vgetexpss %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0x43,0xc9]
+vgetexpss %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vgetmantsd $0, -485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf3,0xf5,0x08,0x27,0x8c,0x82,0x10,0xe3,0x0f,0xe3,0x00]
+vgetmantsd $0, -485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vgetmantsd $0, 485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf3,0xf5,0x08,0x27,0x8c,0x82,0xf0,0x1c,0xf0,0x1c,0x00]
+vgetmantsd $0, 485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vgetmantsd $0, -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf3,0xf5,0x0a,0x27,0x8c,0x82,0x10,0xe3,0x0f,0xe3,0x00]
+vgetmantsd $0, -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vgetmantsd $0, 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf3,0xf5,0x0a,0x27,0x8c,0x82,0xf0,0x1c,0xf0,0x1c,0x00]
+vgetmantsd $0, 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vgetmantsd $0, -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf3,0xf5,0x8a,0x27,0x8c,0x82,0x10,0xe3,0x0f,0xe3,0x00]
+vgetmantsd $0, -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vgetmantsd $0, 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf3,0xf5,0x8a,0x27,0x8c,0x82,0xf0,0x1c,0xf0,0x1c,0x00]
+vgetmantsd $0, 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vgetmantsd $0, 485498096(%edx), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf3,0xf5,0x08,0x27,0x8a,0xf0,0x1c,0xf0,0x1c,0x00]
+vgetmantsd $0, 485498096(%edx), %xmm1, %xmm1
+
+// CHECK: vgetmantsd $0, 485498096(%edx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf3,0xf5,0x0a,0x27,0x8a,0xf0,0x1c,0xf0,0x1c,0x00]
+vgetmantsd $0, 485498096(%edx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vgetmantsd $0, 485498096(%edx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf3,0xf5,0x8a,0x27,0x8a,0xf0,0x1c,0xf0,0x1c,0x00]
+vgetmantsd $0, 485498096(%edx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vgetmantsd $0, 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf3,0xf5,0x08,0x27,0x0d,0xf0,0x1c,0xf0,0x1c,0x00]
+vgetmantsd $0, 485498096, %xmm1, %xmm1
+
+// CHECK: vgetmantsd $0, 485498096, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf3,0xf5,0x0a,0x27,0x0d,0xf0,0x1c,0xf0,0x1c,0x00]
+vgetmantsd $0, 485498096, %xmm1, %xmm1 {%k2}
+
+// CHECK: vgetmantsd $0, 485498096, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf3,0xf5,0x8a,0x27,0x0d,0xf0,0x1c,0xf0,0x1c,0x00]
+vgetmantsd $0, 485498096, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vgetmantsd $0, 512(%edx,%eax), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf3,0xf5,0x08,0x27,0x4c,0x02,0x40,0x00]
+vgetmantsd $0, 512(%edx,%eax), %xmm1, %xmm1
+
+// CHECK: vgetmantsd $0, 512(%edx,%eax), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf3,0xf5,0x0a,0x27,0x4c,0x02,0x40,0x00]
+vgetmantsd $0, 512(%edx,%eax), %xmm1, %xmm1 {%k2}
+
+// CHECK: vgetmantsd $0, 512(%edx,%eax), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf3,0xf5,0x8a,0x27,0x4c,0x02,0x40,0x00]
+vgetmantsd $0, 512(%edx,%eax), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vgetmantsd $0, (%edx), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf3,0xf5,0x08,0x27,0x0a,0x00]
+vgetmantsd $0, (%edx), %xmm1, %xmm1
+
+// CHECK: vgetmantsd $0, (%edx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf3,0xf5,0x0a,0x27,0x0a,0x00]
+vgetmantsd $0, (%edx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vgetmantsd $0, (%edx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf3,0xf5,0x8a,0x27,0x0a,0x00]
+vgetmantsd $0, (%edx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vgetmantsd $0, {sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf3,0xf5,0x18,0x27,0xc9,0x00]
+vgetmantsd $0, {sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vgetmantsd $0, {sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf3,0xf5,0x1a,0x27,0xc9,0x00]
+vgetmantsd $0, {sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vgetmantsd $0, {sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf3,0xf5,0x9a,0x27,0xc9,0x00]
+vgetmantsd $0, {sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vgetmantsd $0, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf3,0xf5,0x08,0x27,0xc9,0x00]
+vgetmantsd $0, %xmm1, %xmm1, %xmm1
+
+// CHECK: vgetmantsd $0, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf3,0xf5,0x0a,0x27,0xc9,0x00]
+vgetmantsd $0, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vgetmantsd $0, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf3,0xf5,0x8a,0x27,0xc9,0x00]
+vgetmantsd $0, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vgetmantss $0, 256(%edx,%eax), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf3,0x75,0x08,0x27,0x4c,0x02,0x40,0x00]
+vgetmantss $0, 256(%edx,%eax), %xmm1, %xmm1
+
+// CHECK: vgetmantss $0, 256(%edx,%eax), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf3,0x75,0x0a,0x27,0x4c,0x02,0x40,0x00]
+vgetmantss $0, 256(%edx,%eax), %xmm1, %xmm1 {%k2}
+
+// CHECK: vgetmantss $0, 256(%edx,%eax), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf3,0x75,0x8a,0x27,0x4c,0x02,0x40,0x00]
+vgetmantss $0, 256(%edx,%eax), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vgetmantss $0, -485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf3,0x75,0x08,0x27,0x8c,0x82,0x10,0xe3,0x0f,0xe3,0x00]
+vgetmantss $0, -485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vgetmantss $0, 485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf3,0x75,0x08,0x27,0x8c,0x82,0xf0,0x1c,0xf0,0x1c,0x00]
+vgetmantss $0, 485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vgetmantss $0, -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf3,0x75,0x0a,0x27,0x8c,0x82,0x10,0xe3,0x0f,0xe3,0x00]
+vgetmantss $0, -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vgetmantss $0, 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf3,0x75,0x0a,0x27,0x8c,0x82,0xf0,0x1c,0xf0,0x1c,0x00]
+vgetmantss $0, 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vgetmantss $0, -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf3,0x75,0x8a,0x27,0x8c,0x82,0x10,0xe3,0x0f,0xe3,0x00]
+vgetmantss $0, -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vgetmantss $0, 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf3,0x75,0x8a,0x27,0x8c,0x82,0xf0,0x1c,0xf0,0x1c,0x00]
+vgetmantss $0, 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vgetmantss $0, 485498096(%edx), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf3,0x75,0x08,0x27,0x8a,0xf0,0x1c,0xf0,0x1c,0x00]
+vgetmantss $0, 485498096(%edx), %xmm1, %xmm1
+
+// CHECK: vgetmantss $0, 485498096(%edx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf3,0x75,0x0a,0x27,0x8a,0xf0,0x1c,0xf0,0x1c,0x00]
+vgetmantss $0, 485498096(%edx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vgetmantss $0, 485498096(%edx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf3,0x75,0x8a,0x27,0x8a,0xf0,0x1c,0xf0,0x1c,0x00]
+vgetmantss $0, 485498096(%edx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vgetmantss $0, 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf3,0x75,0x08,0x27,0x0d,0xf0,0x1c,0xf0,0x1c,0x00]
+vgetmantss $0, 485498096, %xmm1, %xmm1
+
+// CHECK: vgetmantss $0, 485498096, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf3,0x75,0x0a,0x27,0x0d,0xf0,0x1c,0xf0,0x1c,0x00]
+vgetmantss $0, 485498096, %xmm1, %xmm1 {%k2}
+
+// CHECK: vgetmantss $0, 485498096, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf3,0x75,0x8a,0x27,0x0d,0xf0,0x1c,0xf0,0x1c,0x00]
+vgetmantss $0, 485498096, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vgetmantss $0, (%edx), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf3,0x75,0x08,0x27,0x0a,0x00]
+vgetmantss $0, (%edx), %xmm1, %xmm1
+
+// CHECK: vgetmantss $0, (%edx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf3,0x75,0x0a,0x27,0x0a,0x00]
+vgetmantss $0, (%edx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vgetmantss $0, (%edx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf3,0x75,0x8a,0x27,0x0a,0x00]
+vgetmantss $0, (%edx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vgetmantss $0, {sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf3,0x75,0x18,0x27,0xc9,0x00]
+vgetmantss $0, {sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vgetmantss $0, {sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf3,0x75,0x1a,0x27,0xc9,0x00]
+vgetmantss $0, {sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vgetmantss $0, {sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf3,0x75,0x9a,0x27,0xc9,0x00]
+vgetmantss $0, {sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vgetmantss $0, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf3,0x75,0x08,0x27,0xc9,0x00]
+vgetmantss $0, %xmm1, %xmm1, %xmm1
+
+// CHECK: vgetmantss $0, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf3,0x75,0x0a,0x27,0xc9,0x00]
+vgetmantss $0, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vgetmantss $0, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf3,0x75,0x8a,0x27,0xc9,0x00]
+vgetmantss $0, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vmaxsd -485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x5f,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vmaxsd -485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vmaxsd 485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x5f,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vmaxsd 485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vmaxsd -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0x5f,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vmaxsd -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vmaxsd 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0x5f,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vmaxsd 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vmaxsd -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x8a,0x5f,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vmaxsd -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vmaxsd 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x8a,0x5f,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vmaxsd 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vmaxsd 485498096(%edx), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x5f,0x8a,0xf0,0x1c,0xf0,0x1c]
+vmaxsd 485498096(%edx), %xmm1, %xmm1
+
+// CHECK: vmaxsd 485498096(%edx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0x5f,0x8a,0xf0,0x1c,0xf0,0x1c]
+vmaxsd 485498096(%edx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vmaxsd 485498096(%edx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x8a,0x5f,0x8a,0xf0,0x1c,0xf0,0x1c]
+vmaxsd 485498096(%edx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vmaxsd 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x5f,0x0d,0xf0,0x1c,0xf0,0x1c]
+vmaxsd 485498096, %xmm1, %xmm1
+
+// CHECK: vmaxsd 485498096, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0x5f,0x0d,0xf0,0x1c,0xf0,0x1c]
+vmaxsd 485498096, %xmm1, %xmm1 {%k2}
+
+// CHECK: vmaxsd 485498096, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x8a,0x5f,0x0d,0xf0,0x1c,0xf0,0x1c]
+vmaxsd 485498096, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vmaxsd 512(%edx,%eax), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x5f,0x8c,0x02,0x00,0x02,0x00,0x00]
+vmaxsd 512(%edx,%eax), %xmm1, %xmm1
+
+// CHECK: vmaxsd 512(%edx,%eax), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0x5f,0x4c,0x02,0x40]
+vmaxsd 512(%edx,%eax), %xmm1, %xmm1 {%k2}
+
+// CHECK: vmaxsd 512(%edx,%eax), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x8a,0x5f,0x4c,0x02,0x40]
+vmaxsd 512(%edx,%eax), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vmaxsd (%edx), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x5f,0x0a]
+vmaxsd (%edx), %xmm1, %xmm1
+
+// CHECK: vmaxsd (%edx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0x5f,0x0a]
+vmaxsd (%edx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vmaxsd (%edx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x8a,0x5f,0x0a]
+vmaxsd (%edx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vmaxsd {sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0xf7,0x18,0x5f,0xc9]
+vmaxsd {sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vmaxsd {sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x1a,0x5f,0xc9]
+vmaxsd {sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vmaxsd {sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x9a,0x5f,0xc9]
+vmaxsd {sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vmaxsd %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x5f,0xc9]
+vmaxsd %xmm1, %xmm1, %xmm1
+
+// CHECK: vmaxsd %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0x5f,0xc9]
+vmaxsd %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vmaxsd %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x8a,0x5f,0xc9]
+vmaxsd %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vmaxss 256(%edx,%eax), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x5f,0x8c,0x02,0x00,0x01,0x00,0x00]
+vmaxss 256(%edx,%eax), %xmm1, %xmm1
+
+// CHECK: vmaxss 256(%edx,%eax), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0x5f,0x4c,0x02,0x40]
+vmaxss 256(%edx,%eax), %xmm1, %xmm1 {%k2}
+
+// CHECK: vmaxss 256(%edx,%eax), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x8a,0x5f,0x4c,0x02,0x40]
+vmaxss 256(%edx,%eax), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vmaxss -485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x5f,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vmaxss -485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vmaxss 485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x5f,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vmaxss 485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vmaxss -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0x5f,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vmaxss -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vmaxss 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0x5f,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vmaxss 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vmaxss -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x8a,0x5f,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vmaxss -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vmaxss 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x8a,0x5f,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vmaxss 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vmaxss 485498096(%edx), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x5f,0x8a,0xf0,0x1c,0xf0,0x1c]
+vmaxss 485498096(%edx), %xmm1, %xmm1
+
+// CHECK: vmaxss 485498096(%edx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0x5f,0x8a,0xf0,0x1c,0xf0,0x1c]
+vmaxss 485498096(%edx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vmaxss 485498096(%edx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x8a,0x5f,0x8a,0xf0,0x1c,0xf0,0x1c]
+vmaxss 485498096(%edx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vmaxss 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x5f,0x0d,0xf0,0x1c,0xf0,0x1c]
+vmaxss 485498096, %xmm1, %xmm1
+
+// CHECK: vmaxss 485498096, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0x5f,0x0d,0xf0,0x1c,0xf0,0x1c]
+vmaxss 485498096, %xmm1, %xmm1 {%k2}
+
+// CHECK: vmaxss 485498096, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x8a,0x5f,0x0d,0xf0,0x1c,0xf0,0x1c]
+vmaxss 485498096, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vmaxss (%edx), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x5f,0x0a]
+vmaxss (%edx), %xmm1, %xmm1
+
+// CHECK: vmaxss (%edx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0x5f,0x0a]
+vmaxss (%edx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vmaxss (%edx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x8a,0x5f,0x0a]
+vmaxss (%edx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vmaxss {sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0x76,0x18,0x5f,0xc9]
+vmaxss {sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vmaxss {sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x1a,0x5f,0xc9]
+vmaxss {sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vmaxss {sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x9a,0x5f,0xc9]
+vmaxss {sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vmaxss %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x5f,0xc9]
+vmaxss %xmm1, %xmm1, %xmm1
+
+// CHECK: vmaxss %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0x5f,0xc9]
+vmaxss %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vmaxss %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x8a,0x5f,0xc9]
+vmaxss %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vminsd -485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x5d,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vminsd -485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vminsd 485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x5d,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vminsd 485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vminsd -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0x5d,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vminsd -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vminsd 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0x5d,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vminsd 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vminsd -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x8a,0x5d,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vminsd -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vminsd 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x8a,0x5d,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vminsd 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vminsd 485498096(%edx), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x5d,0x8a,0xf0,0x1c,0xf0,0x1c]
+vminsd 485498096(%edx), %xmm1, %xmm1
+
+// CHECK: vminsd 485498096(%edx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0x5d,0x8a,0xf0,0x1c,0xf0,0x1c]
+vminsd 485498096(%edx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vminsd 485498096(%edx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x8a,0x5d,0x8a,0xf0,0x1c,0xf0,0x1c]
+vminsd 485498096(%edx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vminsd 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x5d,0x0d,0xf0,0x1c,0xf0,0x1c]
+vminsd 485498096, %xmm1, %xmm1
+
+// CHECK: vminsd 485498096, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0x5d,0x0d,0xf0,0x1c,0xf0,0x1c]
+vminsd 485498096, %xmm1, %xmm1 {%k2}
+
+// CHECK: vminsd 485498096, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x8a,0x5d,0x0d,0xf0,0x1c,0xf0,0x1c]
+vminsd 485498096, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vminsd 512(%edx,%eax), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x5d,0x8c,0x02,0x00,0x02,0x00,0x00]
+vminsd 512(%edx,%eax), %xmm1, %xmm1
+
+// CHECK: vminsd 512(%edx,%eax), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0x5d,0x4c,0x02,0x40]
+vminsd 512(%edx,%eax), %xmm1, %xmm1 {%k2}
+
+// CHECK: vminsd 512(%edx,%eax), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x8a,0x5d,0x4c,0x02,0x40]
+vminsd 512(%edx,%eax), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vminsd (%edx), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x5d,0x0a]
+vminsd (%edx), %xmm1, %xmm1
+
+// CHECK: vminsd (%edx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0x5d,0x0a]
+vminsd (%edx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vminsd (%edx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x8a,0x5d,0x0a]
+vminsd (%edx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vminsd {sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0xf7,0x18,0x5d,0xc9]
+vminsd {sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vminsd {sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x1a,0x5d,0xc9]
+vminsd {sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vminsd {sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x9a,0x5d,0xc9]
+vminsd {sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vminsd %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x5d,0xc9]
+vminsd %xmm1, %xmm1, %xmm1
+
+// CHECK: vminsd %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0x5d,0xc9]
+vminsd %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vminsd %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x8a,0x5d,0xc9]
+vminsd %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vminss 256(%edx,%eax), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x5d,0x8c,0x02,0x00,0x01,0x00,0x00]
+vminss 256(%edx,%eax), %xmm1, %xmm1
+
+// CHECK: vminss 256(%edx,%eax), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0x5d,0x4c,0x02,0x40]
+vminss 256(%edx,%eax), %xmm1, %xmm1 {%k2}
+
+// CHECK: vminss 256(%edx,%eax), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x8a,0x5d,0x4c,0x02,0x40]
+vminss 256(%edx,%eax), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vminss -485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x5d,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vminss -485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vminss 485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x5d,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vminss 485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vminss -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0x5d,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vminss -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vminss 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0x5d,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vminss 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vminss -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x8a,0x5d,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vminss -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vminss 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x8a,0x5d,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vminss 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vminss 485498096(%edx), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x5d,0x8a,0xf0,0x1c,0xf0,0x1c]
+vminss 485498096(%edx), %xmm1, %xmm1
+
+// CHECK: vminss 485498096(%edx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0x5d,0x8a,0xf0,0x1c,0xf0,0x1c]
+vminss 485498096(%edx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vminss 485498096(%edx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x8a,0x5d,0x8a,0xf0,0x1c,0xf0,0x1c]
+vminss 485498096(%edx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vminss 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x5d,0x0d,0xf0,0x1c,0xf0,0x1c]
+vminss 485498096, %xmm1, %xmm1
+
+// CHECK: vminss 485498096, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0x5d,0x0d,0xf0,0x1c,0xf0,0x1c]
+vminss 485498096, %xmm1, %xmm1 {%k2}
+
+// CHECK: vminss 485498096, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x8a,0x5d,0x0d,0xf0,0x1c,0xf0,0x1c]
+vminss 485498096, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vminss (%edx), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x5d,0x0a]
+vminss (%edx), %xmm1, %xmm1
+
+// CHECK: vminss (%edx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0x5d,0x0a]
+vminss (%edx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vminss (%edx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x8a,0x5d,0x0a]
+vminss (%edx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vminss {sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0x76,0x18,0x5d,0xc9]
+vminss {sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vminss {sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x1a,0x5d,0xc9]
+vminss {sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vminss {sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x9a,0x5d,0xc9]
+vminss {sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vminss %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x5d,0xc9]
+vminss %xmm1, %xmm1, %xmm1
+
+// CHECK: vminss %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0x5d,0xc9]
+vminss %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vminss %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x8a,0x5d,0xc9]
+vminss %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vmovsd -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0xc5,0xfb,0x10,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vmovsd -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: vmovsd 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0xc5,0xfb,0x10,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vmovsd 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: vmovsd -485498096(%edx,%eax,4), %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xff,0x0a,0x10,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vmovsd -485498096(%edx,%eax,4), %xmm1 {%k2}
+
+// CHECK: vmovsd 485498096(%edx,%eax,4), %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xff,0x0a,0x10,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vmovsd 485498096(%edx,%eax,4), %xmm1 {%k2}
+
+// CHECK: vmovsd -485498096(%edx,%eax,4), %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xff,0x8a,0x10,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vmovsd -485498096(%edx,%eax,4), %xmm1 {%k2} {z}
+
+// CHECK: vmovsd 485498096(%edx,%eax,4), %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xff,0x8a,0x10,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vmovsd 485498096(%edx,%eax,4), %xmm1 {%k2} {z}
+
+// CHECK: vmovsd 485498096(%edx), %xmm1
+// CHECK: encoding: [0xc5,0xfb,0x10,0x8a,0xf0,0x1c,0xf0,0x1c]
+vmovsd 485498096(%edx), %xmm1
+
+// CHECK: vmovsd 485498096(%edx), %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xff,0x0a,0x10,0x8a,0xf0,0x1c,0xf0,0x1c]
+vmovsd 485498096(%edx), %xmm1 {%k2}
+
+// CHECK: vmovsd 485498096(%edx), %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xff,0x8a,0x10,0x8a,0xf0,0x1c,0xf0,0x1c]
+vmovsd 485498096(%edx), %xmm1 {%k2} {z}
+
+// CHECK: vmovsd 485498096, %xmm1
+// CHECK: encoding: [0xc5,0xfb,0x10,0x0d,0xf0,0x1c,0xf0,0x1c]
+vmovsd 485498096, %xmm1
+
+// CHECK: vmovsd 485498096, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xff,0x0a,0x10,0x0d,0xf0,0x1c,0xf0,0x1c]
+vmovsd 485498096, %xmm1 {%k2}
+
+// CHECK: vmovsd 485498096, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xff,0x8a,0x10,0x0d,0xf0,0x1c,0xf0,0x1c]
+vmovsd 485498096, %xmm1 {%k2} {z}
+
+// CHECK: vmovsd 512(%edx,%eax), %xmm1
+// CHECK: encoding: [0xc5,0xfb,0x10,0x8c,0x02,0x00,0x02,0x00,0x00]
+vmovsd 512(%edx,%eax), %xmm1
+
+// CHECK: vmovsd 512(%edx,%eax), %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xff,0x0a,0x10,0x4c,0x02,0x40]
+vmovsd 512(%edx,%eax), %xmm1 {%k2}
+
+// CHECK: vmovsd 512(%edx,%eax), %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xff,0x8a,0x10,0x4c,0x02,0x40]
+vmovsd 512(%edx,%eax), %xmm1 {%k2} {z}
+
+// CHECK: vmovsd (%edx), %xmm1
+// CHECK: encoding: [0xc5,0xfb,0x10,0x0a]
+vmovsd (%edx), %xmm1
+
+// CHECK: vmovsd (%edx), %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xff,0x0a,0x10,0x0a]
+vmovsd (%edx), %xmm1 {%k2}
+
+// CHECK: vmovsd (%edx), %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xff,0x8a,0x10,0x0a]
+vmovsd (%edx), %xmm1 {%k2} {z}
+
+// CHECK: vmovsd %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x11,0xc9]
+vmovsd.s %xmm1, %xmm1, %xmm1
+
+// CHECK: vmovsd %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0x11,0xc9]
+vmovsd.s %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vmovsd %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x8a,0x11,0xc9]
+vmovsd.s %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vmovsd %xmm1, -485498096(%edx,%eax,4)
+// CHECK: encoding: [0xc5,0xfb,0x11,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vmovsd %xmm1, -485498096(%edx,%eax,4)
+
+// CHECK: vmovsd %xmm1, 485498096(%edx,%eax,4)
+// CHECK: encoding: [0xc5,0xfb,0x11,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vmovsd %xmm1, 485498096(%edx,%eax,4)
+
+// CHECK: vmovsd %xmm1, -485498096(%edx,%eax,4) {%k2}
+// CHECK: encoding: [0x62,0xf1,0xff,0x0a,0x11,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vmovsd %xmm1, -485498096(%edx,%eax,4) {%k2}
+
+// CHECK: vmovsd %xmm1, 485498096(%edx,%eax,4) {%k2}
+// CHECK: encoding: [0x62,0xf1,0xff,0x0a,0x11,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vmovsd %xmm1, 485498096(%edx,%eax,4) {%k2}
+
+// CHECK: vmovsd %xmm1, 485498096(%edx)
+// CHECK: encoding: [0xc5,0xfb,0x11,0x8a,0xf0,0x1c,0xf0,0x1c]
+vmovsd %xmm1, 485498096(%edx)
+
+// CHECK: vmovsd %xmm1, 485498096(%edx) {%k2}
+// CHECK: encoding: [0x62,0xf1,0xff,0x0a,0x11,0x8a,0xf0,0x1c,0xf0,0x1c]
+vmovsd %xmm1, 485498096(%edx) {%k2}
+
+// CHECK: vmovsd %xmm1, 485498096
+// CHECK: encoding: [0xc5,0xfb,0x11,0x0d,0xf0,0x1c,0xf0,0x1c]
+vmovsd %xmm1, 485498096
+
+// CHECK: vmovsd %xmm1, 485498096 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xff,0x0a,0x11,0x0d,0xf0,0x1c,0xf0,0x1c]
+vmovsd %xmm1, 485498096 {%k2}
+
+// CHECK: vmovsd %xmm1, 512(%edx,%eax)
+// CHECK: encoding: [0xc5,0xfb,0x11,0x8c,0x02,0x00,0x02,0x00,0x00]
+vmovsd %xmm1, 512(%edx,%eax)
+
+// CHECK: vmovsd %xmm1, 512(%edx,%eax) {%k2}
+// CHECK: encoding: [0x62,0xf1,0xff,0x0a,0x11,0x4c,0x02,0x40]
+vmovsd %xmm1, 512(%edx,%eax) {%k2}
+
+// CHECK: vmovsd %xmm1, (%edx)
+// CHECK: encoding: [0xc5,0xfb,0x11,0x0a]
+vmovsd %xmm1, (%edx)
+
+// CHECK: vmovsd %xmm1, (%edx) {%k2}
+// CHECK: encoding: [0x62,0xf1,0xff,0x0a,0x11,0x0a]
+vmovsd %xmm1, (%edx) {%k2}
+
+// CHECK: vmovsd %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x10,0xc9]
+vmovsd %xmm1, %xmm1, %xmm1
+
+// CHECK: vmovsd %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0x10,0xc9]
+vmovsd %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vmovsd %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x8a,0x10,0xc9]
+vmovsd %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vmovss 256(%edx,%eax), %xmm1
+// CHECK: encoding: [0xc5,0xfa,0x10,0x8c,0x02,0x00,0x01,0x00,0x00]
+vmovss 256(%edx,%eax), %xmm1
+
+// CHECK: vmovss 256(%edx,%eax), %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x7e,0x0a,0x10,0x4c,0x02,0x40]
+vmovss 256(%edx,%eax), %xmm1 {%k2}
+
+// CHECK: vmovss 256(%edx,%eax), %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x7e,0x8a,0x10,0x4c,0x02,0x40]
+vmovss 256(%edx,%eax), %xmm1 {%k2} {z}
+
+// CHECK: vmovss -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0xc5,0xfa,0x10,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vmovss -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: vmovss 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0xc5,0xfa,0x10,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vmovss 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: vmovss -485498096(%edx,%eax,4), %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x7e,0x0a,0x10,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vmovss -485498096(%edx,%eax,4), %xmm1 {%k2}
+
+// CHECK: vmovss 485498096(%edx,%eax,4), %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x7e,0x0a,0x10,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vmovss 485498096(%edx,%eax,4), %xmm1 {%k2}
+
+// CHECK: vmovss -485498096(%edx,%eax,4), %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x7e,0x8a,0x10,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vmovss -485498096(%edx,%eax,4), %xmm1 {%k2} {z}
+
+// CHECK: vmovss 485498096(%edx,%eax,4), %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x7e,0x8a,0x10,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vmovss 485498096(%edx,%eax,4), %xmm1 {%k2} {z}
+
+// CHECK: vmovss 485498096(%edx), %xmm1
+// CHECK: encoding: [0xc5,0xfa,0x10,0x8a,0xf0,0x1c,0xf0,0x1c]
+vmovss 485498096(%edx), %xmm1
+
+// CHECK: vmovss 485498096(%edx), %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x7e,0x0a,0x10,0x8a,0xf0,0x1c,0xf0,0x1c]
+vmovss 485498096(%edx), %xmm1 {%k2}
+
+// CHECK: vmovss 485498096(%edx), %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x7e,0x8a,0x10,0x8a,0xf0,0x1c,0xf0,0x1c]
+vmovss 485498096(%edx), %xmm1 {%k2} {z}
+
+// CHECK: vmovss 485498096, %xmm1
+// CHECK: encoding: [0xc5,0xfa,0x10,0x0d,0xf0,0x1c,0xf0,0x1c]
+vmovss 485498096, %xmm1
+
+// CHECK: vmovss 485498096, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x7e,0x0a,0x10,0x0d,0xf0,0x1c,0xf0,0x1c]
+vmovss 485498096, %xmm1 {%k2}
+
+// CHECK: vmovss 485498096, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x7e,0x8a,0x10,0x0d,0xf0,0x1c,0xf0,0x1c]
+vmovss 485498096, %xmm1 {%k2} {z}
+
+// CHECK: vmovss (%edx), %xmm1
+// CHECK: encoding: [0xc5,0xfa,0x10,0x0a]
+vmovss (%edx), %xmm1
+
+// CHECK: vmovss (%edx), %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x7e,0x0a,0x10,0x0a]
+vmovss (%edx), %xmm1 {%k2}
+
+// CHECK: vmovss (%edx), %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x7e,0x8a,0x10,0x0a]
+vmovss (%edx), %xmm1 {%k2} {z}
+
+// CHECK: vmovss %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x11,0xc9]
+vmovss.s %xmm1, %xmm1, %xmm1
+
+// CHECK: vmovss %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0x11,0xc9]
+vmovss.s %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vmovss %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x8a,0x11,0xc9]
+vmovss.s %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vmovss %xmm1, 256(%edx,%eax)
+// CHECK: encoding: [0xc5,0xfa,0x11,0x8c,0x02,0x00,0x01,0x00,0x00]
+vmovss %xmm1, 256(%edx,%eax)
+
+// CHECK: vmovss %xmm1, 256(%edx,%eax) {%k2}
+// CHECK: encoding: [0x62,0xf1,0x7e,0x0a,0x11,0x4c,0x02,0x40]
+vmovss %xmm1, 256(%edx,%eax) {%k2}
+
+// CHECK: vmovss %xmm1, -485498096(%edx,%eax,4)
+// CHECK: encoding: [0xc5,0xfa,0x11,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vmovss %xmm1, -485498096(%edx,%eax,4)
+
+// CHECK: vmovss %xmm1, 485498096(%edx,%eax,4)
+// CHECK: encoding: [0xc5,0xfa,0x11,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vmovss %xmm1, 485498096(%edx,%eax,4)
+
+// CHECK: vmovss %xmm1, -485498096(%edx,%eax,4) {%k2}
+// CHECK: encoding: [0x62,0xf1,0x7e,0x0a,0x11,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vmovss %xmm1, -485498096(%edx,%eax,4) {%k2}
+
+// CHECK: vmovss %xmm1, 485498096(%edx,%eax,4) {%k2}
+// CHECK: encoding: [0x62,0xf1,0x7e,0x0a,0x11,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vmovss %xmm1, 485498096(%edx,%eax,4) {%k2}
+
+// CHECK: vmovss %xmm1, 485498096(%edx)
+// CHECK: encoding: [0xc5,0xfa,0x11,0x8a,0xf0,0x1c,0xf0,0x1c]
+vmovss %xmm1, 485498096(%edx)
+
+// CHECK: vmovss %xmm1, 485498096(%edx) {%k2}
+// CHECK: encoding: [0x62,0xf1,0x7e,0x0a,0x11,0x8a,0xf0,0x1c,0xf0,0x1c]
+vmovss %xmm1, 485498096(%edx) {%k2}
+
+// CHECK: vmovss %xmm1, 485498096
+// CHECK: encoding: [0xc5,0xfa,0x11,0x0d,0xf0,0x1c,0xf0,0x1c]
+vmovss %xmm1, 485498096
+
+// CHECK: vmovss %xmm1, 485498096 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x7e,0x0a,0x11,0x0d,0xf0,0x1c,0xf0,0x1c]
+vmovss %xmm1, 485498096 {%k2}
+
+// CHECK: vmovss %xmm1, (%edx)
+// CHECK: encoding: [0xc5,0xfa,0x11,0x0a]
+vmovss %xmm1, (%edx)
+
+// CHECK: vmovss %xmm1, (%edx) {%k2}
+// CHECK: encoding: [0x62,0xf1,0x7e,0x0a,0x11,0x0a]
+vmovss %xmm1, (%edx) {%k2}
+
+// CHECK: vmovss %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x10,0xc9]
+vmovss %xmm1, %xmm1, %xmm1
+
+// CHECK: vmovss %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0x10,0xc9]
+vmovss %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vmovss %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x8a,0x10,0xc9]
+vmovss %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vmulsd -485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x59,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vmulsd -485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vmulsd 485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x59,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vmulsd 485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vmulsd -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0x59,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vmulsd -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vmulsd 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0x59,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vmulsd 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vmulsd -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x8a,0x59,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vmulsd -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vmulsd 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x8a,0x59,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vmulsd 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vmulsd 485498096(%edx), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x59,0x8a,0xf0,0x1c,0xf0,0x1c]
+vmulsd 485498096(%edx), %xmm1, %xmm1
+
+// CHECK: vmulsd 485498096(%edx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0x59,0x8a,0xf0,0x1c,0xf0,0x1c]
+vmulsd 485498096(%edx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vmulsd 485498096(%edx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x8a,0x59,0x8a,0xf0,0x1c,0xf0,0x1c]
+vmulsd 485498096(%edx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vmulsd 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x59,0x0d,0xf0,0x1c,0xf0,0x1c]
+vmulsd 485498096, %xmm1, %xmm1
+
+// CHECK: vmulsd 485498096, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0x59,0x0d,0xf0,0x1c,0xf0,0x1c]
+vmulsd 485498096, %xmm1, %xmm1 {%k2}
+
+// CHECK: vmulsd 485498096, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x8a,0x59,0x0d,0xf0,0x1c,0xf0,0x1c]
+vmulsd 485498096, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vmulsd 512(%edx,%eax), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x59,0x8c,0x02,0x00,0x02,0x00,0x00]
+vmulsd 512(%edx,%eax), %xmm1, %xmm1
+
+// CHECK: vmulsd 512(%edx,%eax), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0x59,0x4c,0x02,0x40]
+vmulsd 512(%edx,%eax), %xmm1, %xmm1 {%k2}
+
+// CHECK: vmulsd 512(%edx,%eax), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x8a,0x59,0x4c,0x02,0x40]
+vmulsd 512(%edx,%eax), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vmulsd (%edx), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x59,0x0a]
+vmulsd (%edx), %xmm1, %xmm1
+
+// CHECK: vmulsd (%edx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0x59,0x0a]
+vmulsd (%edx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vmulsd (%edx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x8a,0x59,0x0a]
+vmulsd (%edx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vmulsd {rd-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0xf7,0x38,0x59,0xc9]
+vmulsd {rd-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vmulsd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x3a,0x59,0xc9]
+vmulsd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vmulsd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0xba,0x59,0xc9]
+vmulsd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vmulsd {rn-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0xf7,0x18,0x59,0xc9]
+vmulsd {rn-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vmulsd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x1a,0x59,0xc9]
+vmulsd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vmulsd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x9a,0x59,0xc9]
+vmulsd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vmulsd {ru-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0xf7,0x58,0x59,0xc9]
+vmulsd {ru-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vmulsd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x5a,0x59,0xc9]
+vmulsd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vmulsd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0xda,0x59,0xc9]
+vmulsd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vmulsd {rz-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0xf7,0x78,0x59,0xc9]
+vmulsd {rz-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vmulsd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x7a,0x59,0xc9]
+vmulsd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vmulsd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0xfa,0x59,0xc9]
+vmulsd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vmulsd %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x59,0xc9]
+vmulsd %xmm1, %xmm1, %xmm1
+
+// CHECK: vmulsd %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0x59,0xc9]
+vmulsd %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vmulsd %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x8a,0x59,0xc9]
+vmulsd %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vmulss 256(%edx,%eax), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x59,0x8c,0x02,0x00,0x01,0x00,0x00]
+vmulss 256(%edx,%eax), %xmm1, %xmm1
+
+// CHECK: vmulss 256(%edx,%eax), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0x59,0x4c,0x02,0x40]
+vmulss 256(%edx,%eax), %xmm1, %xmm1 {%k2}
+
+// CHECK: vmulss 256(%edx,%eax), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x8a,0x59,0x4c,0x02,0x40]
+vmulss 256(%edx,%eax), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vmulss -485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x59,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vmulss -485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vmulss 485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x59,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vmulss 485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vmulss -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0x59,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vmulss -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vmulss 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0x59,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vmulss 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vmulss -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x8a,0x59,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vmulss -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vmulss 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x8a,0x59,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vmulss 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vmulss 485498096(%edx), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x59,0x8a,0xf0,0x1c,0xf0,0x1c]
+vmulss 485498096(%edx), %xmm1, %xmm1
+
+// CHECK: vmulss 485498096(%edx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0x59,0x8a,0xf0,0x1c,0xf0,0x1c]
+vmulss 485498096(%edx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vmulss 485498096(%edx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x8a,0x59,0x8a,0xf0,0x1c,0xf0,0x1c]
+vmulss 485498096(%edx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vmulss 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x59,0x0d,0xf0,0x1c,0xf0,0x1c]
+vmulss 485498096, %xmm1, %xmm1
+
+// CHECK: vmulss 485498096, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0x59,0x0d,0xf0,0x1c,0xf0,0x1c]
+vmulss 485498096, %xmm1, %xmm1 {%k2}
+
+// CHECK: vmulss 485498096, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x8a,0x59,0x0d,0xf0,0x1c,0xf0,0x1c]
+vmulss 485498096, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vmulss (%edx), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x59,0x0a]
+vmulss (%edx), %xmm1, %xmm1
+
+// CHECK: vmulss (%edx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0x59,0x0a]
+vmulss (%edx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vmulss (%edx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x8a,0x59,0x0a]
+vmulss (%edx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vmulss {rd-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0x76,0x38,0x59,0xc9]
+vmulss {rd-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vmulss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x3a,0x59,0xc9]
+vmulss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vmulss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0xba,0x59,0xc9]
+vmulss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vmulss {rn-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0x76,0x18,0x59,0xc9]
+vmulss {rn-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vmulss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x1a,0x59,0xc9]
+vmulss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vmulss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x9a,0x59,0xc9]
+vmulss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vmulss {ru-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0x76,0x58,0x59,0xc9]
+vmulss {ru-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vmulss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x5a,0x59,0xc9]
+vmulss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vmulss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0xda,0x59,0xc9]
+vmulss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vmulss {rz-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0x76,0x78,0x59,0xc9]
+vmulss {rz-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vmulss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x7a,0x59,0xc9]
+vmulss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vmulss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0xfa,0x59,0xc9]
+vmulss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vmulss %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x59,0xc9]
+vmulss %xmm1, %xmm1, %xmm1
+
+// CHECK: vmulss %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0x59,0xc9]
+vmulss %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vmulss %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x8a,0x59,0xc9]
+vmulss %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vrcp14sd -485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x08,0x4d,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vrcp14sd -485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vrcp14sd 485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x08,0x4d,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vrcp14sd 485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vrcp14sd -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0x4d,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vrcp14sd -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vrcp14sd 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0x4d,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vrcp14sd 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vrcp14sd -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0x4d,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vrcp14sd -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vrcp14sd 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0x4d,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vrcp14sd 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vrcp14sd 485498096(%edx), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x08,0x4d,0x8a,0xf0,0x1c,0xf0,0x1c]
+vrcp14sd 485498096(%edx), %xmm1, %xmm1
+
+// CHECK: vrcp14sd 485498096(%edx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0x4d,0x8a,0xf0,0x1c,0xf0,0x1c]
+vrcp14sd 485498096(%edx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vrcp14sd 485498096(%edx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0x4d,0x8a,0xf0,0x1c,0xf0,0x1c]
+vrcp14sd 485498096(%edx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vrcp14sd 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x08,0x4d,0x0d,0xf0,0x1c,0xf0,0x1c]
+vrcp14sd 485498096, %xmm1, %xmm1
+
+// CHECK: vrcp14sd 485498096, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0x4d,0x0d,0xf0,0x1c,0xf0,0x1c]
+vrcp14sd 485498096, %xmm1, %xmm1 {%k2}
+
+// CHECK: vrcp14sd 485498096, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0x4d,0x0d,0xf0,0x1c,0xf0,0x1c]
+vrcp14sd 485498096, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vrcp14sd 512(%edx,%eax), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x08,0x4d,0x4c,0x02,0x40]
+vrcp14sd 512(%edx,%eax), %xmm1, %xmm1
+
+// CHECK: vrcp14sd 512(%edx,%eax), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0x4d,0x4c,0x02,0x40]
+vrcp14sd 512(%edx,%eax), %xmm1, %xmm1 {%k2}
+
+// CHECK: vrcp14sd 512(%edx,%eax), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0x4d,0x4c,0x02,0x40]
+vrcp14sd 512(%edx,%eax), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vrcp14sd (%edx), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x08,0x4d,0x0a]
+vrcp14sd (%edx), %xmm1, %xmm1
+
+// CHECK: vrcp14sd (%edx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0x4d,0x0a]
+vrcp14sd (%edx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vrcp14sd (%edx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0x4d,0x0a]
+vrcp14sd (%edx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vrcp14sd %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x08,0x4d,0xc9]
+vrcp14sd %xmm1, %xmm1, %xmm1
+
+// CHECK: vrcp14sd %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0x4d,0xc9]
+vrcp14sd %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vrcp14sd %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0x4d,0xc9]
+vrcp14sd %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vrcp14ss 256(%edx,%eax), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x08,0x4d,0x4c,0x02,0x40]
+vrcp14ss 256(%edx,%eax), %xmm1, %xmm1
+
+// CHECK: vrcp14ss 256(%edx,%eax), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0x4d,0x4c,0x02,0x40]
+vrcp14ss 256(%edx,%eax), %xmm1, %xmm1 {%k2}
+
+// CHECK: vrcp14ss 256(%edx,%eax), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0x4d,0x4c,0x02,0x40]
+vrcp14ss 256(%edx,%eax), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vrcp14ss -485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x08,0x4d,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vrcp14ss -485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vrcp14ss 485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x08,0x4d,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vrcp14ss 485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vrcp14ss -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0x4d,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vrcp14ss -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vrcp14ss 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0x4d,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vrcp14ss 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vrcp14ss -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0x4d,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vrcp14ss -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vrcp14ss 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0x4d,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vrcp14ss 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vrcp14ss 485498096(%edx), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x08,0x4d,0x8a,0xf0,0x1c,0xf0,0x1c]
+vrcp14ss 485498096(%edx), %xmm1, %xmm1
+
+// CHECK: vrcp14ss 485498096(%edx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0x4d,0x8a,0xf0,0x1c,0xf0,0x1c]
+vrcp14ss 485498096(%edx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vrcp14ss 485498096(%edx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0x4d,0x8a,0xf0,0x1c,0xf0,0x1c]
+vrcp14ss 485498096(%edx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vrcp14ss 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x08,0x4d,0x0d,0xf0,0x1c,0xf0,0x1c]
+vrcp14ss 485498096, %xmm1, %xmm1
+
+// CHECK: vrcp14ss 485498096, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0x4d,0x0d,0xf0,0x1c,0xf0,0x1c]
+vrcp14ss 485498096, %xmm1, %xmm1 {%k2}
+
+// CHECK: vrcp14ss 485498096, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0x4d,0x0d,0xf0,0x1c,0xf0,0x1c]
+vrcp14ss 485498096, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vrcp14ss (%edx), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x08,0x4d,0x0a]
+vrcp14ss (%edx), %xmm1, %xmm1
+
+// CHECK: vrcp14ss (%edx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0x4d,0x0a]
+vrcp14ss (%edx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vrcp14ss (%edx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0x4d,0x0a]
+vrcp14ss (%edx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vrcp14ss %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x08,0x4d,0xc9]
+vrcp14ss %xmm1, %xmm1, %xmm1
+
+// CHECK: vrcp14ss %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0x4d,0xc9]
+vrcp14ss %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vrcp14ss %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0x4d,0xc9]
+vrcp14ss %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vrndscalesd $0, -485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf3,0xf5,0x08,0x0b,0x8c,0x82,0x10,0xe3,0x0f,0xe3,0x00]
+vrndscalesd $0, -485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vrndscalesd $0, 485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf3,0xf5,0x08,0x0b,0x8c,0x82,0xf0,0x1c,0xf0,0x1c,0x00]
+vrndscalesd $0, 485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vrndscalesd $0, -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf3,0xf5,0x0a,0x0b,0x8c,0x82,0x10,0xe3,0x0f,0xe3,0x00]
+vrndscalesd $0, -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vrndscalesd $0, 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf3,0xf5,0x0a,0x0b,0x8c,0x82,0xf0,0x1c,0xf0,0x1c,0x00]
+vrndscalesd $0, 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vrndscalesd $0, -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf3,0xf5,0x8a,0x0b,0x8c,0x82,0x10,0xe3,0x0f,0xe3,0x00]
+vrndscalesd $0, -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vrndscalesd $0, 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf3,0xf5,0x8a,0x0b,0x8c,0x82,0xf0,0x1c,0xf0,0x1c,0x00]
+vrndscalesd $0, 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vrndscalesd $0, 485498096(%edx), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf3,0xf5,0x08,0x0b,0x8a,0xf0,0x1c,0xf0,0x1c,0x00]
+vrndscalesd $0, 485498096(%edx), %xmm1, %xmm1
+
+// CHECK: vrndscalesd $0, 485498096(%edx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf3,0xf5,0x0a,0x0b,0x8a,0xf0,0x1c,0xf0,0x1c,0x00]
+vrndscalesd $0, 485498096(%edx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vrndscalesd $0, 485498096(%edx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf3,0xf5,0x8a,0x0b,0x8a,0xf0,0x1c,0xf0,0x1c,0x00]
+vrndscalesd $0, 485498096(%edx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vrndscalesd $0, 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf3,0xf5,0x08,0x0b,0x0d,0xf0,0x1c,0xf0,0x1c,0x00]
+vrndscalesd $0, 485498096, %xmm1, %xmm1
+
+// CHECK: vrndscalesd $0, 485498096, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf3,0xf5,0x0a,0x0b,0x0d,0xf0,0x1c,0xf0,0x1c,0x00]
+vrndscalesd $0, 485498096, %xmm1, %xmm1 {%k2}
+
+// CHECK: vrndscalesd $0, 485498096, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf3,0xf5,0x8a,0x0b,0x0d,0xf0,0x1c,0xf0,0x1c,0x00]
+vrndscalesd $0, 485498096, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vrndscalesd $0, 512(%edx,%eax), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf3,0xf5,0x08,0x0b,0x4c,0x02,0x40,0x00]
+vrndscalesd $0, 512(%edx,%eax), %xmm1, %xmm1
+
+// CHECK: vrndscalesd $0, 512(%edx,%eax), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf3,0xf5,0x0a,0x0b,0x4c,0x02,0x40,0x00]
+vrndscalesd $0, 512(%edx,%eax), %xmm1, %xmm1 {%k2}
+
+// CHECK: vrndscalesd $0, 512(%edx,%eax), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf3,0xf5,0x8a,0x0b,0x4c,0x02,0x40,0x00]
+vrndscalesd $0, 512(%edx,%eax), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vrndscalesd $0, (%edx), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf3,0xf5,0x08,0x0b,0x0a,0x00]
+vrndscalesd $0, (%edx), %xmm1, %xmm1
+
+// CHECK: vrndscalesd $0, (%edx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf3,0xf5,0x0a,0x0b,0x0a,0x00]
+vrndscalesd $0, (%edx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vrndscalesd $0, (%edx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf3,0xf5,0x8a,0x0b,0x0a,0x00]
+vrndscalesd $0, (%edx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vrndscalesd $0, {sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf3,0xf5,0x18,0x0b,0xc9,0x00]
+vrndscalesd $0, {sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vrndscalesd $0, {sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf3,0xf5,0x1a,0x0b,0xc9,0x00]
+vrndscalesd $0, {sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vrndscalesd $0, {sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf3,0xf5,0x9a,0x0b,0xc9,0x00]
+vrndscalesd $0, {sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vrndscalesd $0, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf3,0xf5,0x08,0x0b,0xc9,0x00]
+vrndscalesd $0, %xmm1, %xmm1, %xmm1
+
+// CHECK: vrndscalesd $0, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf3,0xf5,0x0a,0x0b,0xc9,0x00]
+vrndscalesd $0, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vrndscalesd $0, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf3,0xf5,0x8a,0x0b,0xc9,0x00]
+vrndscalesd $0, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vrndscaless $0, 256(%edx,%eax), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf3,0x75,0x08,0x0a,0x4c,0x02,0x40,0x00]
+vrndscaless $0, 256(%edx,%eax), %xmm1, %xmm1
+
+// CHECK: vrndscaless $0, 256(%edx,%eax), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf3,0x75,0x0a,0x0a,0x4c,0x02,0x40,0x00]
+vrndscaless $0, 256(%edx,%eax), %xmm1, %xmm1 {%k2}
+
+// CHECK: vrndscaless $0, 256(%edx,%eax), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf3,0x75,0x8a,0x0a,0x4c,0x02,0x40,0x00]
+vrndscaless $0, 256(%edx,%eax), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vrndscaless $0, -485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf3,0x75,0x08,0x0a,0x8c,0x82,0x10,0xe3,0x0f,0xe3,0x00]
+vrndscaless $0, -485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vrndscaless $0, 485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf3,0x75,0x08,0x0a,0x8c,0x82,0xf0,0x1c,0xf0,0x1c,0x00]
+vrndscaless $0, 485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vrndscaless $0, -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf3,0x75,0x0a,0x0a,0x8c,0x82,0x10,0xe3,0x0f,0xe3,0x00]
+vrndscaless $0, -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vrndscaless $0, 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf3,0x75,0x0a,0x0a,0x8c,0x82,0xf0,0x1c,0xf0,0x1c,0x00]
+vrndscaless $0, 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vrndscaless $0, -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf3,0x75,0x8a,0x0a,0x8c,0x82,0x10,0xe3,0x0f,0xe3,0x00]
+vrndscaless $0, -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vrndscaless $0, 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf3,0x75,0x8a,0x0a,0x8c,0x82,0xf0,0x1c,0xf0,0x1c,0x00]
+vrndscaless $0, 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vrndscaless $0, 485498096(%edx), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf3,0x75,0x08,0x0a,0x8a,0xf0,0x1c,0xf0,0x1c,0x00]
+vrndscaless $0, 485498096(%edx), %xmm1, %xmm1
+
+// CHECK: vrndscaless $0, 485498096(%edx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf3,0x75,0x0a,0x0a,0x8a,0xf0,0x1c,0xf0,0x1c,0x00]
+vrndscaless $0, 485498096(%edx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vrndscaless $0, 485498096(%edx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf3,0x75,0x8a,0x0a,0x8a,0xf0,0x1c,0xf0,0x1c,0x00]
+vrndscaless $0, 485498096(%edx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vrndscaless $0, 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf3,0x75,0x08,0x0a,0x0d,0xf0,0x1c,0xf0,0x1c,0x00]
+vrndscaless $0, 485498096, %xmm1, %xmm1
+
+// CHECK: vrndscaless $0, 485498096, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf3,0x75,0x0a,0x0a,0x0d,0xf0,0x1c,0xf0,0x1c,0x00]
+vrndscaless $0, 485498096, %xmm1, %xmm1 {%k2}
+
+// CHECK: vrndscaless $0, 485498096, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf3,0x75,0x8a,0x0a,0x0d,0xf0,0x1c,0xf0,0x1c,0x00]
+vrndscaless $0, 485498096, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vrndscaless $0, (%edx), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf3,0x75,0x08,0x0a,0x0a,0x00]
+vrndscaless $0, (%edx), %xmm1, %xmm1
+
+// CHECK: vrndscaless $0, (%edx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf3,0x75,0x0a,0x0a,0x0a,0x00]
+vrndscaless $0, (%edx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vrndscaless $0, (%edx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf3,0x75,0x8a,0x0a,0x0a,0x00]
+vrndscaless $0, (%edx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vrndscaless $0, {sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf3,0x75,0x18,0x0a,0xc9,0x00]
+vrndscaless $0, {sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vrndscaless $0, {sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf3,0x75,0x1a,0x0a,0xc9,0x00]
+vrndscaless $0, {sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vrndscaless $0, {sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf3,0x75,0x9a,0x0a,0xc9,0x00]
+vrndscaless $0, {sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vrndscaless $0, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf3,0x75,0x08,0x0a,0xc9,0x00]
+vrndscaless $0, %xmm1, %xmm1, %xmm1
+
+// CHECK: vrndscaless $0, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf3,0x75,0x0a,0x0a,0xc9,0x00]
+vrndscaless $0, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vrndscaless $0, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf3,0x75,0x8a,0x0a,0xc9,0x00]
+vrndscaless $0, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vrsqrt14sd -485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x08,0x4f,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vrsqrt14sd -485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vrsqrt14sd 485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x08,0x4f,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vrsqrt14sd 485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vrsqrt14sd -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0x4f,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vrsqrt14sd -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vrsqrt14sd 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0x4f,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vrsqrt14sd 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vrsqrt14sd -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0x4f,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vrsqrt14sd -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vrsqrt14sd 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0x4f,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vrsqrt14sd 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vrsqrt14sd 485498096(%edx), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x08,0x4f,0x8a,0xf0,0x1c,0xf0,0x1c]
+vrsqrt14sd 485498096(%edx), %xmm1, %xmm1
+
+// CHECK: vrsqrt14sd 485498096(%edx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0x4f,0x8a,0xf0,0x1c,0xf0,0x1c]
+vrsqrt14sd 485498096(%edx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vrsqrt14sd 485498096(%edx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0x4f,0x8a,0xf0,0x1c,0xf0,0x1c]
+vrsqrt14sd 485498096(%edx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vrsqrt14sd 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x08,0x4f,0x0d,0xf0,0x1c,0xf0,0x1c]
+vrsqrt14sd 485498096, %xmm1, %xmm1
+
+// CHECK: vrsqrt14sd 485498096, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0x4f,0x0d,0xf0,0x1c,0xf0,0x1c]
+vrsqrt14sd 485498096, %xmm1, %xmm1 {%k2}
+
+// CHECK: vrsqrt14sd 485498096, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0x4f,0x0d,0xf0,0x1c,0xf0,0x1c]
+vrsqrt14sd 485498096, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vrsqrt14sd 512(%edx,%eax), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x08,0x4f,0x4c,0x02,0x40]
+vrsqrt14sd 512(%edx,%eax), %xmm1, %xmm1
+
+// CHECK: vrsqrt14sd 512(%edx,%eax), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0x4f,0x4c,0x02,0x40]
+vrsqrt14sd 512(%edx,%eax), %xmm1, %xmm1 {%k2}
+
+// CHECK: vrsqrt14sd 512(%edx,%eax), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0x4f,0x4c,0x02,0x40]
+vrsqrt14sd 512(%edx,%eax), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vrsqrt14sd (%edx), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x08,0x4f,0x0a]
+vrsqrt14sd (%edx), %xmm1, %xmm1
+
+// CHECK: vrsqrt14sd (%edx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0x4f,0x0a]
+vrsqrt14sd (%edx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vrsqrt14sd (%edx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0x4f,0x0a]
+vrsqrt14sd (%edx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vrsqrt14sd %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x08,0x4f,0xc9]
+vrsqrt14sd %xmm1, %xmm1, %xmm1
+
+// CHECK: vrsqrt14sd %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0x4f,0xc9]
+vrsqrt14sd %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vrsqrt14sd %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0x4f,0xc9]
+vrsqrt14sd %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vrsqrt14ss 256(%edx,%eax), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x08,0x4f,0x4c,0x02,0x40]
+vrsqrt14ss 256(%edx,%eax), %xmm1, %xmm1
+
+// CHECK: vrsqrt14ss 256(%edx,%eax), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0x4f,0x4c,0x02,0x40]
+vrsqrt14ss 256(%edx,%eax), %xmm1, %xmm1 {%k2}
+
+// CHECK: vrsqrt14ss 256(%edx,%eax), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0x4f,0x4c,0x02,0x40]
+vrsqrt14ss 256(%edx,%eax), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vrsqrt14ss -485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x08,0x4f,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vrsqrt14ss -485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vrsqrt14ss 485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x08,0x4f,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vrsqrt14ss 485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vrsqrt14ss -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0x4f,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vrsqrt14ss -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vrsqrt14ss 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0x4f,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vrsqrt14ss 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vrsqrt14ss -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0x4f,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vrsqrt14ss -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vrsqrt14ss 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0x4f,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vrsqrt14ss 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vrsqrt14ss 485498096(%edx), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x08,0x4f,0x8a,0xf0,0x1c,0xf0,0x1c]
+vrsqrt14ss 485498096(%edx), %xmm1, %xmm1
+
+// CHECK: vrsqrt14ss 485498096(%edx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0x4f,0x8a,0xf0,0x1c,0xf0,0x1c]
+vrsqrt14ss 485498096(%edx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vrsqrt14ss 485498096(%edx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0x4f,0x8a,0xf0,0x1c,0xf0,0x1c]
+vrsqrt14ss 485498096(%edx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vrsqrt14ss 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x08,0x4f,0x0d,0xf0,0x1c,0xf0,0x1c]
+vrsqrt14ss 485498096, %xmm1, %xmm1
+
+// CHECK: vrsqrt14ss 485498096, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0x4f,0x0d,0xf0,0x1c,0xf0,0x1c]
+vrsqrt14ss 485498096, %xmm1, %xmm1 {%k2}
+
+// CHECK: vrsqrt14ss 485498096, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0x4f,0x0d,0xf0,0x1c,0xf0,0x1c]
+vrsqrt14ss 485498096, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vrsqrt14ss (%edx), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x08,0x4f,0x0a]
+vrsqrt14ss (%edx), %xmm1, %xmm1
+
+// CHECK: vrsqrt14ss (%edx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0x4f,0x0a]
+vrsqrt14ss (%edx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vrsqrt14ss (%edx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0x4f,0x0a]
+vrsqrt14ss (%edx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vrsqrt14ss %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x08,0x4f,0xc9]
+vrsqrt14ss %xmm1, %xmm1, %xmm1
+
+// CHECK: vrsqrt14ss %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0x4f,0xc9]
+vrsqrt14ss %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vrsqrt14ss %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0x4f,0xc9]
+vrsqrt14ss %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vscalefsd -485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x08,0x2d,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vscalefsd -485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vscalefsd 485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x08,0x2d,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vscalefsd 485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vscalefsd -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0x2d,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vscalefsd -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vscalefsd 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0x2d,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vscalefsd 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vscalefsd -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0x2d,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vscalefsd -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vscalefsd 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0x2d,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vscalefsd 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vscalefsd 485498096(%edx), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x08,0x2d,0x8a,0xf0,0x1c,0xf0,0x1c]
+vscalefsd 485498096(%edx), %xmm1, %xmm1
+
+// CHECK: vscalefsd 485498096(%edx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0x2d,0x8a,0xf0,0x1c,0xf0,0x1c]
+vscalefsd 485498096(%edx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vscalefsd 485498096(%edx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0x2d,0x8a,0xf0,0x1c,0xf0,0x1c]
+vscalefsd 485498096(%edx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vscalefsd 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x08,0x2d,0x0d,0xf0,0x1c,0xf0,0x1c]
+vscalefsd 485498096, %xmm1, %xmm1
+
+// CHECK: vscalefsd 485498096, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0x2d,0x0d,0xf0,0x1c,0xf0,0x1c]
+vscalefsd 485498096, %xmm1, %xmm1 {%k2}
+
+// CHECK: vscalefsd 485498096, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0x2d,0x0d,0xf0,0x1c,0xf0,0x1c]
+vscalefsd 485498096, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vscalefsd 512(%edx,%eax), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x08,0x2d,0x4c,0x02,0x40]
+vscalefsd 512(%edx,%eax), %xmm1, %xmm1
+
+// CHECK: vscalefsd 512(%edx,%eax), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0x2d,0x4c,0x02,0x40]
+vscalefsd 512(%edx,%eax), %xmm1, %xmm1 {%k2}
+
+// CHECK: vscalefsd 512(%edx,%eax), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0x2d,0x4c,0x02,0x40]
+vscalefsd 512(%edx,%eax), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vscalefsd (%edx), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x08,0x2d,0x0a]
+vscalefsd (%edx), %xmm1, %xmm1
+
+// CHECK: vscalefsd (%edx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0x2d,0x0a]
+vscalefsd (%edx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vscalefsd (%edx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0x2d,0x0a]
+vscalefsd (%edx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vscalefsd {rd-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x38,0x2d,0xc9]
+vscalefsd {rd-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vscalefsd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x3a,0x2d,0xc9]
+vscalefsd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vscalefsd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0xba,0x2d,0xc9]
+vscalefsd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vscalefsd {rn-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x18,0x2d,0xc9]
+vscalefsd {rn-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vscalefsd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x1a,0x2d,0xc9]
+vscalefsd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vscalefsd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x9a,0x2d,0xc9]
+vscalefsd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vscalefsd {ru-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x58,0x2d,0xc9]
+vscalefsd {ru-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vscalefsd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x5a,0x2d,0xc9]
+vscalefsd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vscalefsd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0xda,0x2d,0xc9]
+vscalefsd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vscalefsd {rz-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x78,0x2d,0xc9]
+vscalefsd {rz-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vscalefsd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x7a,0x2d,0xc9]
+vscalefsd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vscalefsd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0xfa,0x2d,0xc9]
+vscalefsd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vscalefsd %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x08,0x2d,0xc9]
+vscalefsd %xmm1, %xmm1, %xmm1
+
+// CHECK: vscalefsd %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0x2d,0xc9]
+vscalefsd %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vscalefsd %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0x2d,0xc9]
+vscalefsd %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vscalefss 256(%edx,%eax), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x08,0x2d,0x4c,0x02,0x40]
+vscalefss 256(%edx,%eax), %xmm1, %xmm1
+
+// CHECK: vscalefss 256(%edx,%eax), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0x2d,0x4c,0x02,0x40]
+vscalefss 256(%edx,%eax), %xmm1, %xmm1 {%k2}
+
+// CHECK: vscalefss 256(%edx,%eax), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0x2d,0x4c,0x02,0x40]
+vscalefss 256(%edx,%eax), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vscalefss -485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x08,0x2d,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vscalefss -485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vscalefss 485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x08,0x2d,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vscalefss 485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vscalefss -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0x2d,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vscalefss -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vscalefss 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0x2d,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vscalefss 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vscalefss -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0x2d,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vscalefss -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vscalefss 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0x2d,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vscalefss 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vscalefss 485498096(%edx), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x08,0x2d,0x8a,0xf0,0x1c,0xf0,0x1c]
+vscalefss 485498096(%edx), %xmm1, %xmm1
+
+// CHECK: vscalefss 485498096(%edx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0x2d,0x8a,0xf0,0x1c,0xf0,0x1c]
+vscalefss 485498096(%edx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vscalefss 485498096(%edx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0x2d,0x8a,0xf0,0x1c,0xf0,0x1c]
+vscalefss 485498096(%edx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vscalefss 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x08,0x2d,0x0d,0xf0,0x1c,0xf0,0x1c]
+vscalefss 485498096, %xmm1, %xmm1
+
+// CHECK: vscalefss 485498096, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0x2d,0x0d,0xf0,0x1c,0xf0,0x1c]
+vscalefss 485498096, %xmm1, %xmm1 {%k2}
+
+// CHECK: vscalefss 485498096, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0x2d,0x0d,0xf0,0x1c,0xf0,0x1c]
+vscalefss 485498096, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vscalefss (%edx), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x08,0x2d,0x0a]
+vscalefss (%edx), %xmm1, %xmm1
+
+// CHECK: vscalefss (%edx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0x2d,0x0a]
+vscalefss (%edx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vscalefss (%edx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0x2d,0x0a]
+vscalefss (%edx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vscalefss {rd-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x38,0x2d,0xc9]
+vscalefss {rd-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vscalefss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x3a,0x2d,0xc9]
+vscalefss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vscalefss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0xba,0x2d,0xc9]
+vscalefss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vscalefss {rn-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x18,0x2d,0xc9]
+vscalefss {rn-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vscalefss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x1a,0x2d,0xc9]
+vscalefss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vscalefss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x9a,0x2d,0xc9]
+vscalefss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vscalefss {ru-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x58,0x2d,0xc9]
+vscalefss {ru-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vscalefss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x5a,0x2d,0xc9]
+vscalefss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vscalefss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0xda,0x2d,0xc9]
+vscalefss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vscalefss {rz-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x78,0x2d,0xc9]
+vscalefss {rz-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vscalefss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x7a,0x2d,0xc9]
+vscalefss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vscalefss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0xfa,0x2d,0xc9]
+vscalefss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vscalefss %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x08,0x2d,0xc9]
+vscalefss %xmm1, %xmm1, %xmm1
+
+// CHECK: vscalefss %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0x2d,0xc9]
+vscalefss %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vscalefss %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0x2d,0xc9]
+vscalefss %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vsqrtsd -485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x51,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vsqrtsd -485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vsqrtsd 485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x51,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vsqrtsd 485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vsqrtsd -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0x51,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vsqrtsd -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vsqrtsd 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0x51,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vsqrtsd 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vsqrtsd -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x8a,0x51,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vsqrtsd -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vsqrtsd 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x8a,0x51,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vsqrtsd 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vsqrtsd 485498096(%edx), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x51,0x8a,0xf0,0x1c,0xf0,0x1c]
+vsqrtsd 485498096(%edx), %xmm1, %xmm1
+
+// CHECK: vsqrtsd 485498096(%edx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0x51,0x8a,0xf0,0x1c,0xf0,0x1c]
+vsqrtsd 485498096(%edx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vsqrtsd 485498096(%edx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x8a,0x51,0x8a,0xf0,0x1c,0xf0,0x1c]
+vsqrtsd 485498096(%edx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vsqrtsd 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x51,0x0d,0xf0,0x1c,0xf0,0x1c]
+vsqrtsd 485498096, %xmm1, %xmm1
+
+// CHECK: vsqrtsd 485498096, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0x51,0x0d,0xf0,0x1c,0xf0,0x1c]
+vsqrtsd 485498096, %xmm1, %xmm1 {%k2}
+
+// CHECK: vsqrtsd 485498096, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x8a,0x51,0x0d,0xf0,0x1c,0xf0,0x1c]
+vsqrtsd 485498096, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vsqrtsd 512(%edx,%eax), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x51,0x8c,0x02,0x00,0x02,0x00,0x00]
+vsqrtsd 512(%edx,%eax), %xmm1, %xmm1
+
+// CHECK: vsqrtsd 512(%edx,%eax), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0x51,0x4c,0x02,0x40]
+vsqrtsd 512(%edx,%eax), %xmm1, %xmm1 {%k2}
+
+// CHECK: vsqrtsd 512(%edx,%eax), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x8a,0x51,0x4c,0x02,0x40]
+vsqrtsd 512(%edx,%eax), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vsqrtsd (%edx), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x51,0x0a]
+vsqrtsd (%edx), %xmm1, %xmm1
+
+// CHECK: vsqrtsd (%edx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0x51,0x0a]
+vsqrtsd (%edx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vsqrtsd (%edx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x8a,0x51,0x0a]
+vsqrtsd (%edx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vsqrtsd {rd-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0xf7,0x38,0x51,0xc9]
+vsqrtsd {rd-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vsqrtsd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x3a,0x51,0xc9]
+vsqrtsd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vsqrtsd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0xba,0x51,0xc9]
+vsqrtsd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vsqrtsd {rn-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0xf7,0x18,0x51,0xc9]
+vsqrtsd {rn-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vsqrtsd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x1a,0x51,0xc9]
+vsqrtsd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vsqrtsd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x9a,0x51,0xc9]
+vsqrtsd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vsqrtsd {ru-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0xf7,0x58,0x51,0xc9]
+vsqrtsd {ru-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vsqrtsd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x5a,0x51,0xc9]
+vsqrtsd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vsqrtsd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0xda,0x51,0xc9]
+vsqrtsd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vsqrtsd {rz-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0xf7,0x78,0x51,0xc9]
+vsqrtsd {rz-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vsqrtsd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x7a,0x51,0xc9]
+vsqrtsd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vsqrtsd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0xfa,0x51,0xc9]
+vsqrtsd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vsqrtsd %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x51,0xc9]
+vsqrtsd %xmm1, %xmm1, %xmm1
+
+// CHECK: vsqrtsd %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0x51,0xc9]
+vsqrtsd %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vsqrtsd %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x8a,0x51,0xc9]
+vsqrtsd %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vsqrtss 256(%edx,%eax), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x51,0x8c,0x02,0x00,0x01,0x00,0x00]
+vsqrtss 256(%edx,%eax), %xmm1, %xmm1
+
+// CHECK: vsqrtss 256(%edx,%eax), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0x51,0x4c,0x02,0x40]
+vsqrtss 256(%edx,%eax), %xmm1, %xmm1 {%k2}
+
+// CHECK: vsqrtss 256(%edx,%eax), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x8a,0x51,0x4c,0x02,0x40]
+vsqrtss 256(%edx,%eax), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vsqrtss -485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x51,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vsqrtss -485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vsqrtss 485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x51,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vsqrtss 485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vsqrtss -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0x51,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vsqrtss -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vsqrtss 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0x51,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vsqrtss 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vsqrtss -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x8a,0x51,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vsqrtss -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vsqrtss 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x8a,0x51,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vsqrtss 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vsqrtss 485498096(%edx), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x51,0x8a,0xf0,0x1c,0xf0,0x1c]
+vsqrtss 485498096(%edx), %xmm1, %xmm1
+
+// CHECK: vsqrtss 485498096(%edx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0x51,0x8a,0xf0,0x1c,0xf0,0x1c]
+vsqrtss 485498096(%edx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vsqrtss 485498096(%edx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x8a,0x51,0x8a,0xf0,0x1c,0xf0,0x1c]
+vsqrtss 485498096(%edx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vsqrtss 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x51,0x0d,0xf0,0x1c,0xf0,0x1c]
+vsqrtss 485498096, %xmm1, %xmm1
+
+// CHECK: vsqrtss 485498096, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0x51,0x0d,0xf0,0x1c,0xf0,0x1c]
+vsqrtss 485498096, %xmm1, %xmm1 {%k2}
+
+// CHECK: vsqrtss 485498096, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x8a,0x51,0x0d,0xf0,0x1c,0xf0,0x1c]
+vsqrtss 485498096, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vsqrtss (%edx), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x51,0x0a]
+vsqrtss (%edx), %xmm1, %xmm1
+
+// CHECK: vsqrtss (%edx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0x51,0x0a]
+vsqrtss (%edx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vsqrtss (%edx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x8a,0x51,0x0a]
+vsqrtss (%edx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vsqrtss {rd-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0x76,0x38,0x51,0xc9]
+vsqrtss {rd-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vsqrtss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x3a,0x51,0xc9]
+vsqrtss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vsqrtss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0xba,0x51,0xc9]
+vsqrtss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vsqrtss {rn-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0x76,0x18,0x51,0xc9]
+vsqrtss {rn-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vsqrtss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x1a,0x51,0xc9]
+vsqrtss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vsqrtss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x9a,0x51,0xc9]
+vsqrtss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vsqrtss {ru-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0x76,0x58,0x51,0xc9]
+vsqrtss {ru-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vsqrtss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x5a,0x51,0xc9]
+vsqrtss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vsqrtss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0xda,0x51,0xc9]
+vsqrtss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vsqrtss {rz-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0x76,0x78,0x51,0xc9]
+vsqrtss {rz-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vsqrtss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x7a,0x51,0xc9]
+vsqrtss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vsqrtss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0xfa,0x51,0xc9]
+vsqrtss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vsqrtss %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x51,0xc9]
+vsqrtss %xmm1, %xmm1, %xmm1
+
+// CHECK: vsqrtss %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0x51,0xc9]
+vsqrtss %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vsqrtss %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x8a,0x51,0xc9]
+vsqrtss %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vsubsd -485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x5c,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vsubsd -485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vsubsd 485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x5c,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vsubsd 485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vsubsd -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0x5c,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vsubsd -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vsubsd 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0x5c,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vsubsd 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vsubsd -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x8a,0x5c,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vsubsd -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vsubsd 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x8a,0x5c,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vsubsd 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vsubsd 485498096(%edx), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x5c,0x8a,0xf0,0x1c,0xf0,0x1c]
+vsubsd 485498096(%edx), %xmm1, %xmm1
+
+// CHECK: vsubsd 485498096(%edx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0x5c,0x8a,0xf0,0x1c,0xf0,0x1c]
+vsubsd 485498096(%edx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vsubsd 485498096(%edx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x8a,0x5c,0x8a,0xf0,0x1c,0xf0,0x1c]
+vsubsd 485498096(%edx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vsubsd 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x5c,0x0d,0xf0,0x1c,0xf0,0x1c]
+vsubsd 485498096, %xmm1, %xmm1
+
+// CHECK: vsubsd 485498096, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0x5c,0x0d,0xf0,0x1c,0xf0,0x1c]
+vsubsd 485498096, %xmm1, %xmm1 {%k2}
+
+// CHECK: vsubsd 485498096, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x8a,0x5c,0x0d,0xf0,0x1c,0xf0,0x1c]
+vsubsd 485498096, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vsubsd 512(%edx,%eax), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x5c,0x8c,0x02,0x00,0x02,0x00,0x00]
+vsubsd 512(%edx,%eax), %xmm1, %xmm1
+
+// CHECK: vsubsd 512(%edx,%eax), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0x5c,0x4c,0x02,0x40]
+vsubsd 512(%edx,%eax), %xmm1, %xmm1 {%k2}
+
+// CHECK: vsubsd 512(%edx,%eax), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x8a,0x5c,0x4c,0x02,0x40]
+vsubsd 512(%edx,%eax), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vsubsd (%edx), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x5c,0x0a]
+vsubsd (%edx), %xmm1, %xmm1
+
+// CHECK: vsubsd (%edx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0x5c,0x0a]
+vsubsd (%edx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vsubsd (%edx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x8a,0x5c,0x0a]
+vsubsd (%edx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vsubsd {rd-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0xf7,0x38,0x5c,0xc9]
+vsubsd {rd-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vsubsd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x3a,0x5c,0xc9]
+vsubsd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vsubsd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0xba,0x5c,0xc9]
+vsubsd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vsubsd {rn-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0xf7,0x18,0x5c,0xc9]
+vsubsd {rn-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vsubsd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x1a,0x5c,0xc9]
+vsubsd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vsubsd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x9a,0x5c,0xc9]
+vsubsd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vsubsd {ru-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0xf7,0x58,0x5c,0xc9]
+vsubsd {ru-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vsubsd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x5a,0x5c,0xc9]
+vsubsd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vsubsd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0xda,0x5c,0xc9]
+vsubsd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vsubsd {rz-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0xf7,0x78,0x5c,0xc9]
+vsubsd {rz-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vsubsd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x7a,0x5c,0xc9]
+vsubsd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vsubsd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0xfa,0x5c,0xc9]
+vsubsd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vsubsd %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x5c,0xc9]
+vsubsd %xmm1, %xmm1, %xmm1
+
+// CHECK: vsubsd %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0x5c,0xc9]
+vsubsd %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vsubsd %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x8a,0x5c,0xc9]
+vsubsd %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vsubss 256(%edx,%eax), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x5c,0x8c,0x02,0x00,0x01,0x00,0x00]
+vsubss 256(%edx,%eax), %xmm1, %xmm1
+
+// CHECK: vsubss 256(%edx,%eax), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0x5c,0x4c,0x02,0x40]
+vsubss 256(%edx,%eax), %xmm1, %xmm1 {%k2}
+
+// CHECK: vsubss 256(%edx,%eax), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x8a,0x5c,0x4c,0x02,0x40]
+vsubss 256(%edx,%eax), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vsubss -485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x5c,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vsubss -485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vsubss 485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x5c,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vsubss 485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vsubss -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0x5c,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vsubss -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vsubss 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0x5c,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vsubss 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vsubss -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x8a,0x5c,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vsubss -485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vsubss 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x8a,0x5c,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vsubss 485498096(%edx,%eax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vsubss 485498096(%edx), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x5c,0x8a,0xf0,0x1c,0xf0,0x1c]
+vsubss 485498096(%edx), %xmm1, %xmm1
+
+// CHECK: vsubss 485498096(%edx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0x5c,0x8a,0xf0,0x1c,0xf0,0x1c]
+vsubss 485498096(%edx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vsubss 485498096(%edx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x8a,0x5c,0x8a,0xf0,0x1c,0xf0,0x1c]
+vsubss 485498096(%edx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vsubss 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x5c,0x0d,0xf0,0x1c,0xf0,0x1c]
+vsubss 485498096, %xmm1, %xmm1
+
+// CHECK: vsubss 485498096, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0x5c,0x0d,0xf0,0x1c,0xf0,0x1c]
+vsubss 485498096, %xmm1, %xmm1 {%k2}
+
+// CHECK: vsubss 485498096, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x8a,0x5c,0x0d,0xf0,0x1c,0xf0,0x1c]
+vsubss 485498096, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vsubss (%edx), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x5c,0x0a]
+vsubss (%edx), %xmm1, %xmm1
+
+// CHECK: vsubss (%edx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0x5c,0x0a]
+vsubss (%edx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vsubss (%edx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x8a,0x5c,0x0a]
+vsubss (%edx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vsubss {rd-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0x76,0x38,0x5c,0xc9]
+vsubss {rd-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vsubss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x3a,0x5c,0xc9]
+vsubss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vsubss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0xba,0x5c,0xc9]
+vsubss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vsubss {rn-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0x76,0x18,0x5c,0xc9]
+vsubss {rn-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vsubss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x1a,0x5c,0xc9]
+vsubss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vsubss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x9a,0x5c,0xc9]
+vsubss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vsubss {ru-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0x76,0x58,0x5c,0xc9]
+vsubss {ru-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vsubss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x5a,0x5c,0xc9]
+vsubss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vsubss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0xda,0x5c,0xc9]
+vsubss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vsubss {rz-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0x76,0x78,0x5c,0xc9]
+vsubss {rz-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vsubss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x7a,0x5c,0xc9]
+vsubss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vsubss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0xfa,0x5c,0xc9]
+vsubss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vsubss %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x5c,0xc9]
+vsubss %xmm1, %xmm1, %xmm1
+
+// CHECK: vsubss %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0x5c,0xc9]
+vsubss %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vsubss %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x8a,0x5c,0xc9]
+vsubss %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vucomisd -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0xc5,0xf9,0x2e,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vucomisd -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: vucomisd 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0xc5,0xf9,0x2e,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vucomisd 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: vucomisd 485498096(%edx), %xmm1
+// CHECK: encoding: [0xc5,0xf9,0x2e,0x8a,0xf0,0x1c,0xf0,0x1c]
+vucomisd 485498096(%edx), %xmm1
+
+// CHECK: vucomisd 485498096, %xmm1
+// CHECK: encoding: [0xc5,0xf9,0x2e,0x0d,0xf0,0x1c,0xf0,0x1c]
+vucomisd 485498096, %xmm1
+
+// CHECK: vucomisd 512(%edx,%eax), %xmm1
+// CHECK: encoding: [0xc5,0xf9,0x2e,0x8c,0x02,0x00,0x02,0x00,0x00]
+vucomisd 512(%edx,%eax), %xmm1
+
+// CHECK: vucomisd (%edx), %xmm1
+// CHECK: encoding: [0xc5,0xf9,0x2e,0x0a]
+vucomisd (%edx), %xmm1
+
+// CHECK: vucomisd {sae}, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0xfd,0x18,0x2e,0xc9]
+vucomisd {sae}, %xmm1, %xmm1
+
+// CHECK: vucomisd %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf9,0x2e,0xc9]
+vucomisd %xmm1, %xmm1
+
+// CHECK: vucomiss 256(%edx,%eax), %xmm1
+// CHECK: encoding: [0xc5,0xf8,0x2e,0x8c,0x02,0x00,0x01,0x00,0x00]
+vucomiss 256(%edx,%eax), %xmm1
+
+// CHECK: vucomiss -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0xc5,0xf8,0x2e,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vucomiss -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: vucomiss 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0xc5,0xf8,0x2e,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vucomiss 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: vucomiss 485498096(%edx), %xmm1
+// CHECK: encoding: [0xc5,0xf8,0x2e,0x8a,0xf0,0x1c,0xf0,0x1c]
+vucomiss 485498096(%edx), %xmm1
+
+// CHECK: vucomiss 485498096, %xmm1
+// CHECK: encoding: [0xc5,0xf8,0x2e,0x0d,0xf0,0x1c,0xf0,0x1c]
+vucomiss 485498096, %xmm1
+
+// CHECK: vucomiss (%edx), %xmm1
+// CHECK: encoding: [0xc5,0xf8,0x2e,0x0a]
+vucomiss (%edx), %xmm1
+
+// CHECK: vucomiss {sae}, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0x7c,0x18,0x2e,0xc9]
+vucomiss {sae}, %xmm1, %xmm1
+
+// CHECK: vucomiss %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf8,0x2e,0xc9]
+vucomiss %xmm1, %xmm1
+
diff --git a/test/MC/X86/AVX512F_SCALAR-64.s b/test/MC/X86/AVX512F_SCALAR-64.s
new file mode 100644
index 0000000000000000000000000000000000000000..b0b1fe5eb2105f92206ee0070f55f14a404f7d38
--- /dev/null
+++ b/test/MC/X86/AVX512F_SCALAR-64.s
@@ -0,0 +1,15298 @@
+// RUN: llvm-mc -triple x86_64-unknown-unknown -mcpu=skx --show-encoding %s | FileCheck %s
+
+// CHECK: vaddsd 485498096, %xmm15, %xmm15
+// CHECK: encoding: [0xc5,0x03,0x58,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vaddsd 485498096, %xmm15, %xmm15
+
+// CHECK: vaddsd 485498096, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x71,0x87,0x0a,0x58,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vaddsd 485498096, %xmm15, %xmm15 {%k2}
+
+// CHECK: vaddsd 485498096, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x71,0x87,0x8a,0x58,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vaddsd 485498096, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vaddsd 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x58,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vaddsd 485498096, %xmm1, %xmm1
+
+// CHECK: vaddsd 485498096, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0x58,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vaddsd 485498096, %xmm1, %xmm1 {%k2}
+
+// CHECK: vaddsd 485498096, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x8a,0x58,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vaddsd 485498096, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vaddsd 512(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0xc5,0x03,0x58,0xbc,0x82,0x00,0x02,0x00,0x00]
+vaddsd 512(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vaddsd -512(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0xc5,0x03,0x58,0xbc,0x82,0x00,0xfe,0xff,0xff]
+vaddsd -512(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vaddsd 512(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x71,0x87,0x0a,0x58,0x7c,0x82,0x40]
+vaddsd 512(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+
+// CHECK: vaddsd -512(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x71,0x87,0x0a,0x58,0x7c,0x82,0xc0]
+vaddsd -512(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+
+// CHECK: vaddsd 512(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x71,0x87,0x8a,0x58,0x7c,0x82,0x40]
+vaddsd 512(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vaddsd -512(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x71,0x87,0x8a,0x58,0x7c,0x82,0xc0]
+vaddsd -512(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vaddsd 512(%rdx,%rax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x58,0x8c,0x82,0x00,0x02,0x00,0x00]
+vaddsd 512(%rdx,%rax,4), %xmm1, %xmm1
+
+// CHECK: vaddsd -512(%rdx,%rax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x58,0x8c,0x82,0x00,0xfe,0xff,0xff]
+vaddsd -512(%rdx,%rax,4), %xmm1, %xmm1
+
+// CHECK: vaddsd 512(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0x58,0x4c,0x82,0x40]
+vaddsd 512(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vaddsd -512(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0x58,0x4c,0x82,0xc0]
+vaddsd -512(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vaddsd 512(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x8a,0x58,0x4c,0x82,0x40]
+vaddsd 512(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vaddsd -512(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x8a,0x58,0x4c,0x82,0xc0]
+vaddsd -512(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vaddsd 512(%rdx,%rax), %xmm15, %xmm15
+// CHECK: encoding: [0xc5,0x03,0x58,0xbc,0x02,0x00,0x02,0x00,0x00]
+vaddsd 512(%rdx,%rax), %xmm15, %xmm15
+
+// CHECK: vaddsd 512(%rdx,%rax), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x71,0x87,0x0a,0x58,0x7c,0x02,0x40]
+vaddsd 512(%rdx,%rax), %xmm15, %xmm15 {%k2}
+
+// CHECK: vaddsd 512(%rdx,%rax), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x71,0x87,0x8a,0x58,0x7c,0x02,0x40]
+vaddsd 512(%rdx,%rax), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vaddsd 512(%rdx,%rax), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x58,0x8c,0x02,0x00,0x02,0x00,0x00]
+vaddsd 512(%rdx,%rax), %xmm1, %xmm1
+
+// CHECK: vaddsd 512(%rdx,%rax), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0x58,0x4c,0x02,0x40]
+vaddsd 512(%rdx,%rax), %xmm1, %xmm1 {%k2}
+
+// CHECK: vaddsd 512(%rdx,%rax), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x8a,0x58,0x4c,0x02,0x40]
+vaddsd 512(%rdx,%rax), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vaddsd 512(%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0xc5,0x03,0x58,0xba,0x00,0x02,0x00,0x00]
+vaddsd 512(%rdx), %xmm15, %xmm15
+
+// CHECK: vaddsd 512(%rdx), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x71,0x87,0x0a,0x58,0x7a,0x40]
+vaddsd 512(%rdx), %xmm15, %xmm15 {%k2}
+
+// CHECK: vaddsd 512(%rdx), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x71,0x87,0x8a,0x58,0x7a,0x40]
+vaddsd 512(%rdx), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vaddsd 512(%rdx), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x58,0x8a,0x00,0x02,0x00,0x00]
+vaddsd 512(%rdx), %xmm1, %xmm1
+
+// CHECK: vaddsd 512(%rdx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0x58,0x4a,0x40]
+vaddsd 512(%rdx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vaddsd 512(%rdx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x8a,0x58,0x4a,0x40]
+vaddsd 512(%rdx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vaddsd {rd-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x51,0x87,0x38,0x58,0xff]
+vaddsd {rd-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vaddsd {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x51,0x87,0x3a,0x58,0xff]
+vaddsd {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vaddsd {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x51,0x87,0xba,0x58,0xff]
+vaddsd {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vaddsd {rd-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0xf7,0x38,0x58,0xc9]
+vaddsd {rd-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vaddsd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x3a,0x58,0xc9]
+vaddsd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vaddsd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0xba,0x58,0xc9]
+vaddsd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vaddsd (%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0xc5,0x03,0x58,0x3a]
+vaddsd (%rdx), %xmm15, %xmm15
+
+// CHECK: vaddsd (%rdx), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x71,0x87,0x0a,0x58,0x3a]
+vaddsd (%rdx), %xmm15, %xmm15 {%k2}
+
+// CHECK: vaddsd (%rdx), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x71,0x87,0x8a,0x58,0x3a]
+vaddsd (%rdx), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vaddsd (%rdx), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x58,0x0a]
+vaddsd (%rdx), %xmm1, %xmm1
+
+// CHECK: vaddsd (%rdx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0x58,0x0a]
+vaddsd (%rdx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vaddsd (%rdx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x8a,0x58,0x0a]
+vaddsd (%rdx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vaddsd {rn-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x51,0x87,0x18,0x58,0xff]
+vaddsd {rn-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vaddsd {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x51,0x87,0x1a,0x58,0xff]
+vaddsd {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vaddsd {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x51,0x87,0x9a,0x58,0xff]
+vaddsd {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vaddsd {rn-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0xf7,0x18,0x58,0xc9]
+vaddsd {rn-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vaddsd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x1a,0x58,0xc9]
+vaddsd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vaddsd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x9a,0x58,0xc9]
+vaddsd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vaddsd {ru-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x51,0x87,0x58,0x58,0xff]
+vaddsd {ru-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vaddsd {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x51,0x87,0x5a,0x58,0xff]
+vaddsd {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vaddsd {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x51,0x87,0xda,0x58,0xff]
+vaddsd {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vaddsd {ru-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0xf7,0x58,0x58,0xc9]
+vaddsd {ru-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vaddsd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x5a,0x58,0xc9]
+vaddsd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vaddsd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0xda,0x58,0xc9]
+vaddsd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vaddsd {rz-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x51,0x87,0x78,0x58,0xff]
+vaddsd {rz-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vaddsd {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x51,0x87,0x7a,0x58,0xff]
+vaddsd {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vaddsd {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x51,0x87,0xfa,0x58,0xff]
+vaddsd {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vaddsd {rz-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0xf7,0x78,0x58,0xc9]
+vaddsd {rz-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vaddsd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x7a,0x58,0xc9]
+vaddsd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vaddsd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0xfa,0x58,0xc9]
+vaddsd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vaddsd %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x41,0x03,0x58,0xff]
+vaddsd %xmm15, %xmm15, %xmm15
+
+// CHECK: vaddsd %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x51,0x87,0x0a,0x58,0xff]
+vaddsd %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vaddsd %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x51,0x87,0x8a,0x58,0xff]
+vaddsd %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vaddsd %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x58,0xc9]
+vaddsd %xmm1, %xmm1, %xmm1
+
+// CHECK: vaddsd %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0x58,0xc9]
+vaddsd %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vaddsd %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x8a,0x58,0xc9]
+vaddsd %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vaddss 256(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0xc5,0x02,0x58,0xbc,0x82,0x00,0x01,0x00,0x00]
+vaddss 256(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vaddss -256(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0xc5,0x02,0x58,0xbc,0x82,0x00,0xff,0xff,0xff]
+vaddss -256(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vaddss 256(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x71,0x06,0x0a,0x58,0x7c,0x82,0x40]
+vaddss 256(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+
+// CHECK: vaddss -256(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x71,0x06,0x0a,0x58,0x7c,0x82,0xc0]
+vaddss -256(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+
+// CHECK: vaddss 256(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x71,0x06,0x8a,0x58,0x7c,0x82,0x40]
+vaddss 256(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vaddss -256(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x71,0x06,0x8a,0x58,0x7c,0x82,0xc0]
+vaddss -256(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vaddss 256(%rdx,%rax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x58,0x8c,0x82,0x00,0x01,0x00,0x00]
+vaddss 256(%rdx,%rax,4), %xmm1, %xmm1
+
+// CHECK: vaddss -256(%rdx,%rax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x58,0x8c,0x82,0x00,0xff,0xff,0xff]
+vaddss -256(%rdx,%rax,4), %xmm1, %xmm1
+
+// CHECK: vaddss 256(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0x58,0x4c,0x82,0x40]
+vaddss 256(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vaddss -256(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0x58,0x4c,0x82,0xc0]
+vaddss -256(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vaddss 256(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x8a,0x58,0x4c,0x82,0x40]
+vaddss 256(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vaddss -256(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x8a,0x58,0x4c,0x82,0xc0]
+vaddss -256(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vaddss 256(%rdx,%rax), %xmm15, %xmm15
+// CHECK: encoding: [0xc5,0x02,0x58,0xbc,0x02,0x00,0x01,0x00,0x00]
+vaddss 256(%rdx,%rax), %xmm15, %xmm15
+
+// CHECK: vaddss 256(%rdx,%rax), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x71,0x06,0x0a,0x58,0x7c,0x02,0x40]
+vaddss 256(%rdx,%rax), %xmm15, %xmm15 {%k2}
+
+// CHECK: vaddss 256(%rdx,%rax), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x71,0x06,0x8a,0x58,0x7c,0x02,0x40]
+vaddss 256(%rdx,%rax), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vaddss 256(%rdx,%rax), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x58,0x8c,0x02,0x00,0x01,0x00,0x00]
+vaddss 256(%rdx,%rax), %xmm1, %xmm1
+
+// CHECK: vaddss 256(%rdx,%rax), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0x58,0x4c,0x02,0x40]
+vaddss 256(%rdx,%rax), %xmm1, %xmm1 {%k2}
+
+// CHECK: vaddss 256(%rdx,%rax), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x8a,0x58,0x4c,0x02,0x40]
+vaddss 256(%rdx,%rax), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vaddss 256(%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0xc5,0x02,0x58,0xba,0x00,0x01,0x00,0x00]
+vaddss 256(%rdx), %xmm15, %xmm15
+
+// CHECK: vaddss 256(%rdx), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x71,0x06,0x0a,0x58,0x7a,0x40]
+vaddss 256(%rdx), %xmm15, %xmm15 {%k2}
+
+// CHECK: vaddss 256(%rdx), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x71,0x06,0x8a,0x58,0x7a,0x40]
+vaddss 256(%rdx), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vaddss 256(%rdx), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x58,0x8a,0x00,0x01,0x00,0x00]
+vaddss 256(%rdx), %xmm1, %xmm1
+
+// CHECK: vaddss 256(%rdx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0x58,0x4a,0x40]
+vaddss 256(%rdx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vaddss 256(%rdx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x8a,0x58,0x4a,0x40]
+vaddss 256(%rdx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vaddss 485498096, %xmm15, %xmm15
+// CHECK: encoding: [0xc5,0x02,0x58,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vaddss 485498096, %xmm15, %xmm15
+
+// CHECK: vaddss 485498096, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x71,0x06,0x0a,0x58,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vaddss 485498096, %xmm15, %xmm15 {%k2}
+
+// CHECK: vaddss 485498096, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x71,0x06,0x8a,0x58,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vaddss 485498096, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vaddss 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x58,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vaddss 485498096, %xmm1, %xmm1
+
+// CHECK: vaddss 485498096, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0x58,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vaddss 485498096, %xmm1, %xmm1 {%k2}
+
+// CHECK: vaddss 485498096, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x8a,0x58,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vaddss 485498096, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vaddss {rd-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x51,0x06,0x38,0x58,0xff]
+vaddss {rd-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vaddss {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x51,0x06,0x3a,0x58,0xff]
+vaddss {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vaddss {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x51,0x06,0xba,0x58,0xff]
+vaddss {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vaddss {rd-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0x76,0x38,0x58,0xc9]
+vaddss {rd-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vaddss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x3a,0x58,0xc9]
+vaddss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vaddss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0xba,0x58,0xc9]
+vaddss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vaddss (%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0xc5,0x02,0x58,0x3a]
+vaddss (%rdx), %xmm15, %xmm15
+
+// CHECK: vaddss (%rdx), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x71,0x06,0x0a,0x58,0x3a]
+vaddss (%rdx), %xmm15, %xmm15 {%k2}
+
+// CHECK: vaddss (%rdx), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x71,0x06,0x8a,0x58,0x3a]
+vaddss (%rdx), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vaddss (%rdx), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x58,0x0a]
+vaddss (%rdx), %xmm1, %xmm1
+
+// CHECK: vaddss (%rdx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0x58,0x0a]
+vaddss (%rdx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vaddss (%rdx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x8a,0x58,0x0a]
+vaddss (%rdx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vaddss {rn-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x51,0x06,0x18,0x58,0xff]
+vaddss {rn-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vaddss {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x51,0x06,0x1a,0x58,0xff]
+vaddss {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vaddss {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x51,0x06,0x9a,0x58,0xff]
+vaddss {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vaddss {rn-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0x76,0x18,0x58,0xc9]
+vaddss {rn-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vaddss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x1a,0x58,0xc9]
+vaddss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vaddss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x9a,0x58,0xc9]
+vaddss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vaddss {ru-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x51,0x06,0x58,0x58,0xff]
+vaddss {ru-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vaddss {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x51,0x06,0x5a,0x58,0xff]
+vaddss {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vaddss {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x51,0x06,0xda,0x58,0xff]
+vaddss {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vaddss {ru-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0x76,0x58,0x58,0xc9]
+vaddss {ru-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vaddss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x5a,0x58,0xc9]
+vaddss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vaddss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0xda,0x58,0xc9]
+vaddss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vaddss {rz-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x51,0x06,0x78,0x58,0xff]
+vaddss {rz-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vaddss {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x51,0x06,0x7a,0x58,0xff]
+vaddss {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vaddss {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x51,0x06,0xfa,0x58,0xff]
+vaddss {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vaddss {rz-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0x76,0x78,0x58,0xc9]
+vaddss {rz-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vaddss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x7a,0x58,0xc9]
+vaddss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vaddss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0xfa,0x58,0xc9]
+vaddss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vaddss %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x41,0x02,0x58,0xff]
+vaddss %xmm15, %xmm15, %xmm15
+
+// CHECK: vaddss %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x51,0x06,0x0a,0x58,0xff]
+vaddss %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vaddss %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x51,0x06,0x8a,0x58,0xff]
+vaddss %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vaddss %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x58,0xc9]
+vaddss %xmm1, %xmm1, %xmm1
+
+// CHECK: vaddss %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0x58,0xc9]
+vaddss %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vaddss %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x8a,0x58,0xc9]
+vaddss %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vcmpeqsd 485498096, %xmm15, %k2
+// CHECK: encoding: [0x62,0xf1,0x87,0x08,0xc2,0x14,0x25,0xf0,0x1c,0xf0,0x1c,0x00]
+vcmpeqsd 485498096, %xmm15, %k2
+
+// CHECK: vcmpeqsd 485498096, %xmm15, %k2 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x87,0x0a,0xc2,0x14,0x25,0xf0,0x1c,0xf0,0x1c,0x00]
+vcmpeqsd 485498096, %xmm15, %k2 {%k2}
+
+// CHECK: vcmpeqsd 485498096, %xmm1, %k2
+// CHECK: encoding: [0x62,0xf1,0xf7,0x08,0xc2,0x14,0x25,0xf0,0x1c,0xf0,0x1c,0x00]
+vcmpeqsd 485498096, %xmm1, %k2
+
+// CHECK: vcmpeqsd 485498096, %xmm1, %k2 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0xc2,0x14,0x25,0xf0,0x1c,0xf0,0x1c,0x00]
+vcmpeqsd 485498096, %xmm1, %k2 {%k2}
+
+// CHECK: vcmpeqsd 512(%rdx,%rax,4), %xmm15, %k2
+// CHECK: encoding: [0x62,0xf1,0x87,0x08,0xc2,0x54,0x82,0x40,0x00]
+vcmpeqsd 512(%rdx,%rax,4), %xmm15, %k2
+
+// CHECK: vcmpeqsd -512(%rdx,%rax,4), %xmm15, %k2
+// CHECK: encoding: [0x62,0xf1,0x87,0x08,0xc2,0x54,0x82,0xc0,0x00]
+vcmpeqsd -512(%rdx,%rax,4), %xmm15, %k2
+
+// CHECK: vcmpeqsd 512(%rdx,%rax,4), %xmm15, %k2 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x87,0x0a,0xc2,0x54,0x82,0x40,0x00]
+vcmpeqsd 512(%rdx,%rax,4), %xmm15, %k2 {%k2}
+
+// CHECK: vcmpeqsd -512(%rdx,%rax,4), %xmm15, %k2 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x87,0x0a,0xc2,0x54,0x82,0xc0,0x00]
+vcmpeqsd -512(%rdx,%rax,4), %xmm15, %k2 {%k2}
+
+// CHECK: vcmpeqsd 512(%rdx,%rax,4), %xmm1, %k2
+// CHECK: encoding: [0x62,0xf1,0xf7,0x08,0xc2,0x54,0x82,0x40,0x00]
+vcmpeqsd 512(%rdx,%rax,4), %xmm1, %k2
+
+// CHECK: vcmpeqsd -512(%rdx,%rax,4), %xmm1, %k2
+// CHECK: encoding: [0x62,0xf1,0xf7,0x08,0xc2,0x54,0x82,0xc0,0x00]
+vcmpeqsd -512(%rdx,%rax,4), %xmm1, %k2
+
+// CHECK: vcmpeqsd 512(%rdx,%rax,4), %xmm1, %k2 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0xc2,0x54,0x82,0x40,0x00]
+vcmpeqsd 512(%rdx,%rax,4), %xmm1, %k2 {%k2}
+
+// CHECK: vcmpeqsd -512(%rdx,%rax,4), %xmm1, %k2 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0xc2,0x54,0x82,0xc0,0x00]
+vcmpeqsd -512(%rdx,%rax,4), %xmm1, %k2 {%k2}
+
+// CHECK: vcmpeqsd 512(%rdx,%rax), %xmm15, %k2
+// CHECK: encoding: [0x62,0xf1,0x87,0x08,0xc2,0x54,0x02,0x40,0x00]
+vcmpeqsd 512(%rdx,%rax), %xmm15, %k2
+
+// CHECK: vcmpeqsd 512(%rdx,%rax), %xmm15, %k2 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x87,0x0a,0xc2,0x54,0x02,0x40,0x00]
+vcmpeqsd 512(%rdx,%rax), %xmm15, %k2 {%k2}
+
+// CHECK: vcmpeqsd 512(%rdx,%rax), %xmm1, %k2
+// CHECK: encoding: [0x62,0xf1,0xf7,0x08,0xc2,0x54,0x02,0x40,0x00]
+vcmpeqsd 512(%rdx,%rax), %xmm1, %k2
+
+// CHECK: vcmpeqsd 512(%rdx,%rax), %xmm1, %k2 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0xc2,0x54,0x02,0x40,0x00]
+vcmpeqsd 512(%rdx,%rax), %xmm1, %k2 {%k2}
+
+// CHECK: vcmpeqsd 512(%rdx), %xmm15, %k2
+// CHECK: encoding: [0x62,0xf1,0x87,0x08,0xc2,0x52,0x40,0x00]
+vcmpeqsd 512(%rdx), %xmm15, %k2
+
+// CHECK: vcmpeqsd 512(%rdx), %xmm15, %k2 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x87,0x0a,0xc2,0x52,0x40,0x00]
+vcmpeqsd 512(%rdx), %xmm15, %k2 {%k2}
+
+// CHECK: vcmpeqsd 512(%rdx), %xmm1, %k2
+// CHECK: encoding: [0x62,0xf1,0xf7,0x08,0xc2,0x52,0x40,0x00]
+vcmpeqsd 512(%rdx), %xmm1, %k2
+
+// CHECK: vcmpeqsd 512(%rdx), %xmm1, %k2 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0xc2,0x52,0x40,0x00]
+vcmpeqsd 512(%rdx), %xmm1, %k2 {%k2}
+
+// CHECK: vcmpeqsd (%rdx), %xmm15, %k2
+// CHECK: encoding: [0x62,0xf1,0x87,0x08,0xc2,0x12,0x00]
+vcmpeqsd (%rdx), %xmm15, %k2
+
+// CHECK: vcmpeqsd (%rdx), %xmm15, %k2 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x87,0x0a,0xc2,0x12,0x00]
+vcmpeqsd (%rdx), %xmm15, %k2 {%k2}
+
+// CHECK: vcmpeqsd (%rdx), %xmm1, %k2
+// CHECK: encoding: [0x62,0xf1,0xf7,0x08,0xc2,0x12,0x00]
+vcmpeqsd (%rdx), %xmm1, %k2
+
+// CHECK: vcmpeqsd (%rdx), %xmm1, %k2 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0xc2,0x12,0x00]
+vcmpeqsd (%rdx), %xmm1, %k2 {%k2}
+
+// CHECK: vcmpeqsd {sae}, %xmm15, %xmm15, %k2
+// CHECK: encoding: [0x62,0xd1,0x87,0x18,0xc2,0xd7,0x00]
+vcmpeqsd {sae}, %xmm15, %xmm15, %k2
+
+// CHECK: vcmpeqsd {sae}, %xmm15, %xmm15, %k2 {%k2}
+// CHECK: encoding: [0x62,0xd1,0x87,0x1a,0xc2,0xd7,0x00]
+vcmpeqsd {sae}, %xmm15, %xmm15, %k2 {%k2}
+
+// CHECK: vcmpeqsd {sae}, %xmm1, %xmm1, %k2
+// CHECK: encoding: [0x62,0xf1,0xf7,0x18,0xc2,0xd1,0x00]
+vcmpeqsd {sae}, %xmm1, %xmm1, %k2
+
+// CHECK: vcmpeqsd {sae}, %xmm1, %xmm1, %k2 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x1a,0xc2,0xd1,0x00]
+vcmpeqsd {sae}, %xmm1, %xmm1, %k2 {%k2}
+
+// CHECK: vcmpeqsd %xmm15, %xmm15, %k2
+// CHECK: encoding: [0x62,0xd1,0x87,0x08,0xc2,0xd7,0x00]
+vcmpeqsd %xmm15, %xmm15, %k2
+
+// CHECK: vcmpeqsd %xmm15, %xmm15, %k2 {%k2}
+// CHECK: encoding: [0x62,0xd1,0x87,0x0a,0xc2,0xd7,0x00]
+vcmpeqsd %xmm15, %xmm15, %k2 {%k2}
+
+// CHECK: vcmpeqsd %xmm1, %xmm1, %k2
+// CHECK: encoding: [0x62,0xf1,0xf7,0x08,0xc2,0xd1,0x00]
+vcmpeqsd %xmm1, %xmm1, %k2
+
+// CHECK: vcmpeqsd %xmm1, %xmm1, %k2 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0xc2,0xd1,0x00]
+vcmpeqsd %xmm1, %xmm1, %k2 {%k2}
+
+// CHECK: vcmpeqss 256(%rdx,%rax,4), %xmm15, %k2
+// CHECK: encoding: [0x62,0xf1,0x06,0x08,0xc2,0x54,0x82,0x40,0x00]
+vcmpeqss 256(%rdx,%rax,4), %xmm15, %k2
+
+// CHECK: vcmpeqss -256(%rdx,%rax,4), %xmm15, %k2
+// CHECK: encoding: [0x62,0xf1,0x06,0x08,0xc2,0x54,0x82,0xc0,0x00]
+vcmpeqss -256(%rdx,%rax,4), %xmm15, %k2
+
+// CHECK: vcmpeqss 256(%rdx,%rax,4), %xmm15, %k2 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x06,0x0a,0xc2,0x54,0x82,0x40,0x00]
+vcmpeqss 256(%rdx,%rax,4), %xmm15, %k2 {%k2}
+
+// CHECK: vcmpeqss -256(%rdx,%rax,4), %xmm15, %k2 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x06,0x0a,0xc2,0x54,0x82,0xc0,0x00]
+vcmpeqss -256(%rdx,%rax,4), %xmm15, %k2 {%k2}
+
+// CHECK: vcmpeqss 256(%rdx,%rax,4), %xmm1, %k2
+// CHECK: encoding: [0x62,0xf1,0x76,0x08,0xc2,0x54,0x82,0x40,0x00]
+vcmpeqss 256(%rdx,%rax,4), %xmm1, %k2
+
+// CHECK: vcmpeqss -256(%rdx,%rax,4), %xmm1, %k2
+// CHECK: encoding: [0x62,0xf1,0x76,0x08,0xc2,0x54,0x82,0xc0,0x00]
+vcmpeqss -256(%rdx,%rax,4), %xmm1, %k2
+
+// CHECK: vcmpeqss 256(%rdx,%rax,4), %xmm1, %k2 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0xc2,0x54,0x82,0x40,0x00]
+vcmpeqss 256(%rdx,%rax,4), %xmm1, %k2 {%k2}
+
+// CHECK: vcmpeqss -256(%rdx,%rax,4), %xmm1, %k2 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0xc2,0x54,0x82,0xc0,0x00]
+vcmpeqss -256(%rdx,%rax,4), %xmm1, %k2 {%k2}
+
+// CHECK: vcmpeqss 256(%rdx,%rax), %xmm15, %k2
+// CHECK: encoding: [0x62,0xf1,0x06,0x08,0xc2,0x54,0x02,0x40,0x00]
+vcmpeqss 256(%rdx,%rax), %xmm15, %k2
+
+// CHECK: vcmpeqss 256(%rdx,%rax), %xmm15, %k2 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x06,0x0a,0xc2,0x54,0x02,0x40,0x00]
+vcmpeqss 256(%rdx,%rax), %xmm15, %k2 {%k2}
+
+// CHECK: vcmpeqss 256(%rdx,%rax), %xmm1, %k2
+// CHECK: encoding: [0x62,0xf1,0x76,0x08,0xc2,0x54,0x02,0x40,0x00]
+vcmpeqss 256(%rdx,%rax), %xmm1, %k2
+
+// CHECK: vcmpeqss 256(%rdx,%rax), %xmm1, %k2 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0xc2,0x54,0x02,0x40,0x00]
+vcmpeqss 256(%rdx,%rax), %xmm1, %k2 {%k2}
+
+// CHECK: vcmpeqss 256(%rdx), %xmm15, %k2
+// CHECK: encoding: [0x62,0xf1,0x06,0x08,0xc2,0x52,0x40,0x00]
+vcmpeqss 256(%rdx), %xmm15, %k2
+
+// CHECK: vcmpeqss 256(%rdx), %xmm15, %k2 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x06,0x0a,0xc2,0x52,0x40,0x00]
+vcmpeqss 256(%rdx), %xmm15, %k2 {%k2}
+
+// CHECK: vcmpeqss 256(%rdx), %xmm1, %k2
+// CHECK: encoding: [0x62,0xf1,0x76,0x08,0xc2,0x52,0x40,0x00]
+vcmpeqss 256(%rdx), %xmm1, %k2
+
+// CHECK: vcmpeqss 256(%rdx), %xmm1, %k2 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0xc2,0x52,0x40,0x00]
+vcmpeqss 256(%rdx), %xmm1, %k2 {%k2}
+
+// CHECK: vcmpeqss 485498096, %xmm15, %k2
+// CHECK: encoding: [0x62,0xf1,0x06,0x08,0xc2,0x14,0x25,0xf0,0x1c,0xf0,0x1c,0x00]
+vcmpeqss 485498096, %xmm15, %k2
+
+// CHECK: vcmpeqss 485498096, %xmm15, %k2 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x06,0x0a,0xc2,0x14,0x25,0xf0,0x1c,0xf0,0x1c,0x00]
+vcmpeqss 485498096, %xmm15, %k2 {%k2}
+
+// CHECK: vcmpeqss 485498096, %xmm1, %k2
+// CHECK: encoding: [0x62,0xf1,0x76,0x08,0xc2,0x14,0x25,0xf0,0x1c,0xf0,0x1c,0x00]
+vcmpeqss 485498096, %xmm1, %k2
+
+// CHECK: vcmpeqss 485498096, %xmm1, %k2 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0xc2,0x14,0x25,0xf0,0x1c,0xf0,0x1c,0x00]
+vcmpeqss 485498096, %xmm1, %k2 {%k2}
+
+// CHECK: vcmpeqss (%rdx), %xmm15, %k2
+// CHECK: encoding: [0x62,0xf1,0x06,0x08,0xc2,0x12,0x00]
+vcmpeqss (%rdx), %xmm15, %k2
+
+// CHECK: vcmpeqss (%rdx), %xmm15, %k2 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x06,0x0a,0xc2,0x12,0x00]
+vcmpeqss (%rdx), %xmm15, %k2 {%k2}
+
+// CHECK: vcmpeqss (%rdx), %xmm1, %k2
+// CHECK: encoding: [0x62,0xf1,0x76,0x08,0xc2,0x12,0x00]
+vcmpeqss (%rdx), %xmm1, %k2
+
+// CHECK: vcmpeqss (%rdx), %xmm1, %k2 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0xc2,0x12,0x00]
+vcmpeqss (%rdx), %xmm1, %k2 {%k2}
+
+// CHECK: vcmpeqss {sae}, %xmm15, %xmm15, %k2
+// CHECK: encoding: [0x62,0xd1,0x06,0x18,0xc2,0xd7,0x00]
+vcmpeqss {sae}, %xmm15, %xmm15, %k2
+
+// CHECK: vcmpeqss {sae}, %xmm15, %xmm15, %k2 {%k2}
+// CHECK: encoding: [0x62,0xd1,0x06,0x1a,0xc2,0xd7,0x00]
+vcmpeqss {sae}, %xmm15, %xmm15, %k2 {%k2}
+
+// CHECK: vcmpeqss {sae}, %xmm1, %xmm1, %k2
+// CHECK: encoding: [0x62,0xf1,0x76,0x18,0xc2,0xd1,0x00]
+vcmpeqss {sae}, %xmm1, %xmm1, %k2
+
+// CHECK: vcmpeqss {sae}, %xmm1, %xmm1, %k2 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x1a,0xc2,0xd1,0x00]
+vcmpeqss {sae}, %xmm1, %xmm1, %k2 {%k2}
+
+// CHECK: vcmpeqss %xmm15, %xmm15, %k2
+// CHECK: encoding: [0x62,0xd1,0x06,0x08,0xc2,0xd7,0x00]
+vcmpeqss %xmm15, %xmm15, %k2
+
+// CHECK: vcmpeqss %xmm15, %xmm15, %k2 {%k2}
+// CHECK: encoding: [0x62,0xd1,0x06,0x0a,0xc2,0xd7,0x00]
+vcmpeqss %xmm15, %xmm15, %k2 {%k2}
+
+// CHECK: vcmpeqss %xmm1, %xmm1, %k2
+// CHECK: encoding: [0x62,0xf1,0x76,0x08,0xc2,0xd1,0x00]
+vcmpeqss %xmm1, %xmm1, %k2
+
+// CHECK: vcmpeqss %xmm1, %xmm1, %k2 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0xc2,0xd1,0x00]
+vcmpeqss %xmm1, %xmm1, %k2 {%k2}
+
+// CHECK: vcomisd 485498096, %xmm15
+// CHECK: encoding: [0xc5,0x79,0x2f,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vcomisd 485498096, %xmm15
+
+// CHECK: vcomisd 485498096, %xmm1
+// CHECK: encoding: [0xc5,0xf9,0x2f,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vcomisd 485498096, %xmm1
+
+// CHECK: vcomisd 512(%rdx,%rax,4), %xmm15
+// CHECK: encoding: [0xc5,0x79,0x2f,0xbc,0x82,0x00,0x02,0x00,0x00]
+vcomisd 512(%rdx,%rax,4), %xmm15
+
+// CHECK: vcomisd -512(%rdx,%rax,4), %xmm15
+// CHECK: encoding: [0xc5,0x79,0x2f,0xbc,0x82,0x00,0xfe,0xff,0xff]
+vcomisd -512(%rdx,%rax,4), %xmm15
+
+// CHECK: vcomisd 512(%rdx,%rax,4), %xmm1
+// CHECK: encoding: [0xc5,0xf9,0x2f,0x8c,0x82,0x00,0x02,0x00,0x00]
+vcomisd 512(%rdx,%rax,4), %xmm1
+
+// CHECK: vcomisd -512(%rdx,%rax,4), %xmm1
+// CHECK: encoding: [0xc5,0xf9,0x2f,0x8c,0x82,0x00,0xfe,0xff,0xff]
+vcomisd -512(%rdx,%rax,4), %xmm1
+
+// CHECK: vcomisd 512(%rdx,%rax), %xmm15
+// CHECK: encoding: [0xc5,0x79,0x2f,0xbc,0x02,0x00,0x02,0x00,0x00]
+vcomisd 512(%rdx,%rax), %xmm15
+
+// CHECK: vcomisd 512(%rdx,%rax), %xmm1
+// CHECK: encoding: [0xc5,0xf9,0x2f,0x8c,0x02,0x00,0x02,0x00,0x00]
+vcomisd 512(%rdx,%rax), %xmm1
+
+// CHECK: vcomisd 512(%rdx), %xmm15
+// CHECK: encoding: [0xc5,0x79,0x2f,0xba,0x00,0x02,0x00,0x00]
+vcomisd 512(%rdx), %xmm15
+
+// CHECK: vcomisd 512(%rdx), %xmm1
+// CHECK: encoding: [0xc5,0xf9,0x2f,0x8a,0x00,0x02,0x00,0x00]
+vcomisd 512(%rdx), %xmm1
+
+// CHECK: vcomisd (%rdx), %xmm15
+// CHECK: encoding: [0xc5,0x79,0x2f,0x3a]
+vcomisd (%rdx), %xmm15
+
+// CHECK: vcomisd (%rdx), %xmm1
+// CHECK: encoding: [0xc5,0xf9,0x2f,0x0a]
+vcomisd (%rdx), %xmm1
+
+// CHECK: vcomisd {sae}, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x51,0xfd,0x18,0x2f,0xff]
+vcomisd {sae}, %xmm15, %xmm15
+
+// CHECK: vcomisd {sae}, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0xfd,0x18,0x2f,0xc9]
+vcomisd {sae}, %xmm1, %xmm1
+
+// CHECK: vcomisd %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x41,0x79,0x2f,0xff]
+vcomisd %xmm15, %xmm15
+
+// CHECK: vcomisd %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf9,0x2f,0xc9]
+vcomisd %xmm1, %xmm1
+
+// CHECK: vcomiss 256(%rdx,%rax,4), %xmm15
+// CHECK: encoding: [0xc5,0x78,0x2f,0xbc,0x82,0x00,0x01,0x00,0x00]
+vcomiss 256(%rdx,%rax,4), %xmm15
+
+// CHECK: vcomiss -256(%rdx,%rax,4), %xmm15
+// CHECK: encoding: [0xc5,0x78,0x2f,0xbc,0x82,0x00,0xff,0xff,0xff]
+vcomiss -256(%rdx,%rax,4), %xmm15
+
+// CHECK: vcomiss 256(%rdx,%rax,4), %xmm1
+// CHECK: encoding: [0xc5,0xf8,0x2f,0x8c,0x82,0x00,0x01,0x00,0x00]
+vcomiss 256(%rdx,%rax,4), %xmm1
+
+// CHECK: vcomiss -256(%rdx,%rax,4), %xmm1
+// CHECK: encoding: [0xc5,0xf8,0x2f,0x8c,0x82,0x00,0xff,0xff,0xff]
+vcomiss -256(%rdx,%rax,4), %xmm1
+
+// CHECK: vcomiss 256(%rdx,%rax), %xmm15
+// CHECK: encoding: [0xc5,0x78,0x2f,0xbc,0x02,0x00,0x01,0x00,0x00]
+vcomiss 256(%rdx,%rax), %xmm15
+
+// CHECK: vcomiss 256(%rdx,%rax), %xmm1
+// CHECK: encoding: [0xc5,0xf8,0x2f,0x8c,0x02,0x00,0x01,0x00,0x00]
+vcomiss 256(%rdx,%rax), %xmm1
+
+// CHECK: vcomiss 256(%rdx), %xmm15
+// CHECK: encoding: [0xc5,0x78,0x2f,0xba,0x00,0x01,0x00,0x00]
+vcomiss 256(%rdx), %xmm15
+
+// CHECK: vcomiss 256(%rdx), %xmm1
+// CHECK: encoding: [0xc5,0xf8,0x2f,0x8a,0x00,0x01,0x00,0x00]
+vcomiss 256(%rdx), %xmm1
+
+// CHECK: vcomiss 485498096, %xmm15
+// CHECK: encoding: [0xc5,0x78,0x2f,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vcomiss 485498096, %xmm15
+
+// CHECK: vcomiss 485498096, %xmm1
+// CHECK: encoding: [0xc5,0xf8,0x2f,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vcomiss 485498096, %xmm1
+
+// CHECK: vcomiss (%rdx), %xmm15
+// CHECK: encoding: [0xc5,0x78,0x2f,0x3a]
+vcomiss (%rdx), %xmm15
+
+// CHECK: vcomiss (%rdx), %xmm1
+// CHECK: encoding: [0xc5,0xf8,0x2f,0x0a]
+vcomiss (%rdx), %xmm1
+
+// CHECK: vcomiss {sae}, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x51,0x7c,0x18,0x2f,0xff]
+vcomiss {sae}, %xmm15, %xmm15
+
+// CHECK: vcomiss {sae}, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0x7c,0x18,0x2f,0xc9]
+vcomiss {sae}, %xmm1, %xmm1
+
+// CHECK: vcomiss %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x41,0x78,0x2f,0xff]
+vcomiss %xmm15, %xmm15
+
+// CHECK: vcomiss %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf8,0x2f,0xc9]
+vcomiss %xmm1, %xmm1
+
+// CHECK: vcvtsd2si 485498096, %r13d
+// CHECK: encoding: [0xc5,0x7b,0x2d,0x2c,0x25,0xf0,0x1c,0xf0,0x1c]
+vcvtsd2si 485498096, %r13d
+
+// CHECK: vcvtsd2si 485498096, %r15
+// CHECK: encoding: [0xc4,0x61,0xfb,0x2d,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vcvtsd2si 485498096, %r15
+
+// CHECK: vcvtsd2si 512(%rdx), %r13d
+// CHECK: encoding: [0xc5,0x7b,0x2d,0xaa,0x00,0x02,0x00,0x00]
+vcvtsd2si 512(%rdx), %r13d
+
+// CHECK: vcvtsd2si 512(%rdx), %r15
+// CHECK: encoding: [0xc4,0x61,0xfb,0x2d,0xba,0x00,0x02,0x00,0x00]
+vcvtsd2si 512(%rdx), %r15
+
+// CHECK: vcvtsd2si 512(%rdx,%rax,4), %r13d
+// CHECK: encoding: [0xc5,0x7b,0x2d,0xac,0x82,0x00,0x02,0x00,0x00]
+vcvtsd2si 512(%rdx,%rax,4), %r13d
+
+// CHECK: vcvtsd2si -512(%rdx,%rax,4), %r13d
+// CHECK: encoding: [0xc5,0x7b,0x2d,0xac,0x82,0x00,0xfe,0xff,0xff]
+vcvtsd2si -512(%rdx,%rax,4), %r13d
+
+// CHECK: vcvtsd2si 512(%rdx,%rax,4), %r15
+// CHECK: encoding: [0xc4,0x61,0xfb,0x2d,0xbc,0x82,0x00,0x02,0x00,0x00]
+vcvtsd2si 512(%rdx,%rax,4), %r15
+
+// CHECK: vcvtsd2si -512(%rdx,%rax,4), %r15
+// CHECK: encoding: [0xc4,0x61,0xfb,0x2d,0xbc,0x82,0x00,0xfe,0xff,0xff]
+vcvtsd2si -512(%rdx,%rax,4), %r15
+
+// CHECK: vcvtsd2si 512(%rdx,%rax), %r13d
+// CHECK: encoding: [0xc5,0x7b,0x2d,0xac,0x02,0x00,0x02,0x00,0x00]
+vcvtsd2si 512(%rdx,%rax), %r13d
+
+// CHECK: vcvtsd2si 512(%rdx,%rax), %r15
+// CHECK: encoding: [0xc4,0x61,0xfb,0x2d,0xbc,0x02,0x00,0x02,0x00,0x00]
+vcvtsd2si 512(%rdx,%rax), %r15
+
+// CHECK: vcvtsd2si {rd-sae}, %xmm15, %r13d
+// CHECK: encoding: [0x62,0x51,0x7f,0x38,0x2d,0xef]
+vcvtsd2si {rd-sae}, %xmm15, %r13d
+
+// CHECK: vcvtsd2si {rd-sae}, %xmm15, %r15
+// CHECK: encoding: [0x62,0x51,0xff,0x38,0x2d,0xff]
+vcvtsd2si {rd-sae}, %xmm15, %r15
+
+// CHECK: vcvtsd2si {rd-sae}, %xmm1, %r13d
+// CHECK: encoding: [0x62,0x71,0x7f,0x38,0x2d,0xe9]
+vcvtsd2si {rd-sae}, %xmm1, %r13d
+
+// CHECK: vcvtsd2si {rd-sae}, %xmm1, %r15
+// CHECK: encoding: [0x62,0x71,0xff,0x38,0x2d,0xf9]
+vcvtsd2si {rd-sae}, %xmm1, %r15
+
+// CHECK: vcvtsd2si (%rdx), %r13d
+// CHECK: encoding: [0xc5,0x7b,0x2d,0x2a]
+vcvtsd2si (%rdx), %r13d
+
+// CHECK: vcvtsd2si (%rdx), %r15
+// CHECK: encoding: [0xc4,0x61,0xfb,0x2d,0x3a]
+vcvtsd2si (%rdx), %r15
+
+// CHECK: vcvtsd2si {rn-sae}, %xmm15, %r13d
+// CHECK: encoding: [0x62,0x51,0x7f,0x18,0x2d,0xef]
+vcvtsd2si {rn-sae}, %xmm15, %r13d
+
+// CHECK: vcvtsd2si {rn-sae}, %xmm15, %r15
+// CHECK: encoding: [0x62,0x51,0xff,0x18,0x2d,0xff]
+vcvtsd2si {rn-sae}, %xmm15, %r15
+
+// CHECK: vcvtsd2si {rn-sae}, %xmm1, %r13d
+// CHECK: encoding: [0x62,0x71,0x7f,0x18,0x2d,0xe9]
+vcvtsd2si {rn-sae}, %xmm1, %r13d
+
+// CHECK: vcvtsd2si {rn-sae}, %xmm1, %r15
+// CHECK: encoding: [0x62,0x71,0xff,0x18,0x2d,0xf9]
+vcvtsd2si {rn-sae}, %xmm1, %r15
+
+// CHECK: vcvtsd2si {ru-sae}, %xmm15, %r13d
+// CHECK: encoding: [0x62,0x51,0x7f,0x58,0x2d,0xef]
+vcvtsd2si {ru-sae}, %xmm15, %r13d
+
+// CHECK: vcvtsd2si {ru-sae}, %xmm15, %r15
+// CHECK: encoding: [0x62,0x51,0xff,0x58,0x2d,0xff]
+vcvtsd2si {ru-sae}, %xmm15, %r15
+
+// CHECK: vcvtsd2si {ru-sae}, %xmm1, %r13d
+// CHECK: encoding: [0x62,0x71,0x7f,0x58,0x2d,0xe9]
+vcvtsd2si {ru-sae}, %xmm1, %r13d
+
+// CHECK: vcvtsd2si {ru-sae}, %xmm1, %r15
+// CHECK: encoding: [0x62,0x71,0xff,0x58,0x2d,0xf9]
+vcvtsd2si {ru-sae}, %xmm1, %r15
+
+// CHECK: vcvtsd2si {rz-sae}, %xmm15, %r13d
+// CHECK: encoding: [0x62,0x51,0x7f,0x78,0x2d,0xef]
+vcvtsd2si {rz-sae}, %xmm15, %r13d
+
+// CHECK: vcvtsd2si {rz-sae}, %xmm15, %r15
+// CHECK: encoding: [0x62,0x51,0xff,0x78,0x2d,0xff]
+vcvtsd2si {rz-sae}, %xmm15, %r15
+
+// CHECK: vcvtsd2si {rz-sae}, %xmm1, %r13d
+// CHECK: encoding: [0x62,0x71,0x7f,0x78,0x2d,0xe9]
+vcvtsd2si {rz-sae}, %xmm1, %r13d
+
+// CHECK: vcvtsd2si {rz-sae}, %xmm1, %r15
+// CHECK: encoding: [0x62,0x71,0xff,0x78,0x2d,0xf9]
+vcvtsd2si {rz-sae}, %xmm1, %r15
+
+// CHECK: vcvtsd2si %xmm15, %r13d
+// CHECK: encoding: [0xc4,0x41,0x7b,0x2d,0xef]
+vcvtsd2si %xmm15, %r13d
+
+// CHECK: vcvtsd2si %xmm15, %r15
+// CHECK: encoding: [0xc4,0x41,0xfb,0x2d,0xff]
+vcvtsd2si %xmm15, %r15
+
+// CHECK: vcvtsd2si %xmm1, %r13d
+// CHECK: encoding: [0xc5,0x7b,0x2d,0xe9]
+vcvtsd2si %xmm1, %r13d
+
+// CHECK: vcvtsd2si %xmm1, %r15
+// CHECK: encoding: [0xc4,0x61,0xfb,0x2d,0xf9]
+vcvtsd2si %xmm1, %r15
+
+// CHECK: vcvtsd2ss 485498096, %xmm15, %xmm15
+// CHECK: encoding: [0xc5,0x03,0x5a,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vcvtsd2ss 485498096, %xmm15, %xmm15
+
+// CHECK: vcvtsd2ss 485498096, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x71,0x87,0x0a,0x5a,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vcvtsd2ss 485498096, %xmm15, %xmm15 {%k2}
+
+// CHECK: vcvtsd2ss 485498096, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x71,0x87,0x8a,0x5a,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vcvtsd2ss 485498096, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vcvtsd2ss 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x5a,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vcvtsd2ss 485498096, %xmm1, %xmm1
+
+// CHECK: vcvtsd2ss 485498096, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0x5a,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vcvtsd2ss 485498096, %xmm1, %xmm1 {%k2}
+
+// CHECK: vcvtsd2ss 485498096, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x8a,0x5a,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vcvtsd2ss 485498096, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vcvtsd2ss 512(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0xc5,0x03,0x5a,0xbc,0x82,0x00,0x02,0x00,0x00]
+vcvtsd2ss 512(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vcvtsd2ss -512(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0xc5,0x03,0x5a,0xbc,0x82,0x00,0xfe,0xff,0xff]
+vcvtsd2ss -512(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vcvtsd2ss 512(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x71,0x87,0x0a,0x5a,0x7c,0x82,0x40]
+vcvtsd2ss 512(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+
+// CHECK: vcvtsd2ss -512(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x71,0x87,0x0a,0x5a,0x7c,0x82,0xc0]
+vcvtsd2ss -512(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+
+// CHECK: vcvtsd2ss 512(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x71,0x87,0x8a,0x5a,0x7c,0x82,0x40]
+vcvtsd2ss 512(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vcvtsd2ss -512(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x71,0x87,0x8a,0x5a,0x7c,0x82,0xc0]
+vcvtsd2ss -512(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vcvtsd2ss 512(%rdx,%rax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x5a,0x8c,0x82,0x00,0x02,0x00,0x00]
+vcvtsd2ss 512(%rdx,%rax,4), %xmm1, %xmm1
+
+// CHECK: vcvtsd2ss -512(%rdx,%rax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x5a,0x8c,0x82,0x00,0xfe,0xff,0xff]
+vcvtsd2ss -512(%rdx,%rax,4), %xmm1, %xmm1
+
+// CHECK: vcvtsd2ss 512(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0x5a,0x4c,0x82,0x40]
+vcvtsd2ss 512(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vcvtsd2ss -512(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0x5a,0x4c,0x82,0xc0]
+vcvtsd2ss -512(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vcvtsd2ss 512(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x8a,0x5a,0x4c,0x82,0x40]
+vcvtsd2ss 512(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vcvtsd2ss -512(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x8a,0x5a,0x4c,0x82,0xc0]
+vcvtsd2ss -512(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vcvtsd2ss 512(%rdx,%rax), %xmm15, %xmm15
+// CHECK: encoding: [0xc5,0x03,0x5a,0xbc,0x02,0x00,0x02,0x00,0x00]
+vcvtsd2ss 512(%rdx,%rax), %xmm15, %xmm15
+
+// CHECK: vcvtsd2ss 512(%rdx,%rax), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x71,0x87,0x0a,0x5a,0x7c,0x02,0x40]
+vcvtsd2ss 512(%rdx,%rax), %xmm15, %xmm15 {%k2}
+
+// CHECK: vcvtsd2ss 512(%rdx,%rax), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x71,0x87,0x8a,0x5a,0x7c,0x02,0x40]
+vcvtsd2ss 512(%rdx,%rax), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vcvtsd2ss 512(%rdx,%rax), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x5a,0x8c,0x02,0x00,0x02,0x00,0x00]
+vcvtsd2ss 512(%rdx,%rax), %xmm1, %xmm1
+
+// CHECK: vcvtsd2ss 512(%rdx,%rax), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0x5a,0x4c,0x02,0x40]
+vcvtsd2ss 512(%rdx,%rax), %xmm1, %xmm1 {%k2}
+
+// CHECK: vcvtsd2ss 512(%rdx,%rax), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x8a,0x5a,0x4c,0x02,0x40]
+vcvtsd2ss 512(%rdx,%rax), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vcvtsd2ss 512(%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0xc5,0x03,0x5a,0xba,0x00,0x02,0x00,0x00]
+vcvtsd2ss 512(%rdx), %xmm15, %xmm15
+
+// CHECK: vcvtsd2ss 512(%rdx), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x71,0x87,0x0a,0x5a,0x7a,0x40]
+vcvtsd2ss 512(%rdx), %xmm15, %xmm15 {%k2}
+
+// CHECK: vcvtsd2ss 512(%rdx), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x71,0x87,0x8a,0x5a,0x7a,0x40]
+vcvtsd2ss 512(%rdx), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vcvtsd2ss 512(%rdx), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x5a,0x8a,0x00,0x02,0x00,0x00]
+vcvtsd2ss 512(%rdx), %xmm1, %xmm1
+
+// CHECK: vcvtsd2ss 512(%rdx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0x5a,0x4a,0x40]
+vcvtsd2ss 512(%rdx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vcvtsd2ss 512(%rdx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x8a,0x5a,0x4a,0x40]
+vcvtsd2ss 512(%rdx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vcvtsd2ss {rd-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x51,0x87,0x38,0x5a,0xff]
+vcvtsd2ss {rd-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vcvtsd2ss {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x51,0x87,0x3a,0x5a,0xff]
+vcvtsd2ss {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vcvtsd2ss {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x51,0x87,0xba,0x5a,0xff]
+vcvtsd2ss {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vcvtsd2ss {rd-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0xf7,0x38,0x5a,0xc9]
+vcvtsd2ss {rd-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vcvtsd2ss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x3a,0x5a,0xc9]
+vcvtsd2ss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vcvtsd2ss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0xba,0x5a,0xc9]
+vcvtsd2ss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vcvtsd2ss (%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0xc5,0x03,0x5a,0x3a]
+vcvtsd2ss (%rdx), %xmm15, %xmm15
+
+// CHECK: vcvtsd2ss (%rdx), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x71,0x87,0x0a,0x5a,0x3a]
+vcvtsd2ss (%rdx), %xmm15, %xmm15 {%k2}
+
+// CHECK: vcvtsd2ss (%rdx), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x71,0x87,0x8a,0x5a,0x3a]
+vcvtsd2ss (%rdx), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vcvtsd2ss (%rdx), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x5a,0x0a]
+vcvtsd2ss (%rdx), %xmm1, %xmm1
+
+// CHECK: vcvtsd2ss (%rdx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0x5a,0x0a]
+vcvtsd2ss (%rdx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vcvtsd2ss (%rdx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x8a,0x5a,0x0a]
+vcvtsd2ss (%rdx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vcvtsd2ss {rn-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x51,0x87,0x18,0x5a,0xff]
+vcvtsd2ss {rn-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vcvtsd2ss {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x51,0x87,0x1a,0x5a,0xff]
+vcvtsd2ss {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vcvtsd2ss {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x51,0x87,0x9a,0x5a,0xff]
+vcvtsd2ss {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vcvtsd2ss {rn-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0xf7,0x18,0x5a,0xc9]
+vcvtsd2ss {rn-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vcvtsd2ss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x1a,0x5a,0xc9]
+vcvtsd2ss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vcvtsd2ss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x9a,0x5a,0xc9]
+vcvtsd2ss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vcvtsd2ss {ru-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x51,0x87,0x58,0x5a,0xff]
+vcvtsd2ss {ru-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vcvtsd2ss {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x51,0x87,0x5a,0x5a,0xff]
+vcvtsd2ss {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vcvtsd2ss {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x51,0x87,0xda,0x5a,0xff]
+vcvtsd2ss {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vcvtsd2ss {ru-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0xf7,0x58,0x5a,0xc9]
+vcvtsd2ss {ru-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vcvtsd2ss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x5a,0x5a,0xc9]
+vcvtsd2ss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vcvtsd2ss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0xda,0x5a,0xc9]
+vcvtsd2ss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vcvtsd2ss {rz-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x51,0x87,0x78,0x5a,0xff]
+vcvtsd2ss {rz-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vcvtsd2ss {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x51,0x87,0x7a,0x5a,0xff]
+vcvtsd2ss {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vcvtsd2ss {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x51,0x87,0xfa,0x5a,0xff]
+vcvtsd2ss {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vcvtsd2ss {rz-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0xf7,0x78,0x5a,0xc9]
+vcvtsd2ss {rz-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vcvtsd2ss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x7a,0x5a,0xc9]
+vcvtsd2ss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vcvtsd2ss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0xfa,0x5a,0xc9]
+vcvtsd2ss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vcvtsd2ss %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x41,0x03,0x5a,0xff]
+vcvtsd2ss %xmm15, %xmm15, %xmm15
+
+// CHECK: vcvtsd2ss %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x51,0x87,0x0a,0x5a,0xff]
+vcvtsd2ss %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vcvtsd2ss %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x51,0x87,0x8a,0x5a,0xff]
+vcvtsd2ss %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vcvtsd2ss %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x5a,0xc9]
+vcvtsd2ss %xmm1, %xmm1, %xmm1
+
+// CHECK: vcvtsd2ss %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0x5a,0xc9]
+vcvtsd2ss %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vcvtsd2ss %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x8a,0x5a,0xc9]
+vcvtsd2ss %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vcvtsd2usi 485498096, %r13d
+// CHECK: encoding: [0x62,0x71,0x7f,0x08,0x79,0x2c,0x25,0xf0,0x1c,0xf0,0x1c]
+vcvtsd2usi 485498096, %r13d
+
+// CHECK: vcvtsd2usi 485498096, %r15
+// CHECK: encoding: [0x62,0x71,0xff,0x08,0x79,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vcvtsd2usi 485498096, %r15
+
+// CHECK: vcvtsd2usi 512(%rdx), %r13d
+// CHECK: encoding: [0x62,0x71,0x7f,0x08,0x79,0x6a,0x40]
+vcvtsd2usi 512(%rdx), %r13d
+
+// CHECK: vcvtsd2usi 512(%rdx), %r15
+// CHECK: encoding: [0x62,0x71,0xff,0x08,0x79,0x7a,0x40]
+vcvtsd2usi 512(%rdx), %r15
+
+// CHECK: vcvtsd2usi 512(%rdx,%rax,4), %r13d
+// CHECK: encoding: [0x62,0x71,0x7f,0x08,0x79,0x6c,0x82,0x40]
+vcvtsd2usi 512(%rdx,%rax,4), %r13d
+
+// CHECK: vcvtsd2usi -512(%rdx,%rax,4), %r13d
+// CHECK: encoding: [0x62,0x71,0x7f,0x08,0x79,0x6c,0x82,0xc0]
+vcvtsd2usi -512(%rdx,%rax,4), %r13d
+
+// CHECK: vcvtsd2usi 512(%rdx,%rax,4), %r15
+// CHECK: encoding: [0x62,0x71,0xff,0x08,0x79,0x7c,0x82,0x40]
+vcvtsd2usi 512(%rdx,%rax,4), %r15
+
+// CHECK: vcvtsd2usi -512(%rdx,%rax,4), %r15
+// CHECK: encoding: [0x62,0x71,0xff,0x08,0x79,0x7c,0x82,0xc0]
+vcvtsd2usi -512(%rdx,%rax,4), %r15
+
+// CHECK: vcvtsd2usi 512(%rdx,%rax), %r13d
+// CHECK: encoding: [0x62,0x71,0x7f,0x08,0x79,0x6c,0x02,0x40]
+vcvtsd2usi 512(%rdx,%rax), %r13d
+
+// CHECK: vcvtsd2usi 512(%rdx,%rax), %r15
+// CHECK: encoding: [0x62,0x71,0xff,0x08,0x79,0x7c,0x02,0x40]
+vcvtsd2usi 512(%rdx,%rax), %r15
+
+// CHECK: vcvtsd2usi {rd-sae}, %xmm15, %r13d
+// CHECK: encoding: [0x62,0x51,0x7f,0x38,0x79,0xef]
+vcvtsd2usi {rd-sae}, %xmm15, %r13d
+
+// CHECK: vcvtsd2usi {rd-sae}, %xmm15, %r15
+// CHECK: encoding: [0x62,0x51,0xff,0x38,0x79,0xff]
+vcvtsd2usi {rd-sae}, %xmm15, %r15
+
+// CHECK: vcvtsd2usi {rd-sae}, %xmm1, %r13d
+// CHECK: encoding: [0x62,0x71,0x7f,0x38,0x79,0xe9]
+vcvtsd2usi {rd-sae}, %xmm1, %r13d
+
+// CHECK: vcvtsd2usi {rd-sae}, %xmm1, %r15
+// CHECK: encoding: [0x62,0x71,0xff,0x38,0x79,0xf9]
+vcvtsd2usi {rd-sae}, %xmm1, %r15
+
+// CHECK: vcvtsd2usi (%rdx), %r13d
+// CHECK: encoding: [0x62,0x71,0x7f,0x08,0x79,0x2a]
+vcvtsd2usi (%rdx), %r13d
+
+// CHECK: vcvtsd2usi (%rdx), %r15
+// CHECK: encoding: [0x62,0x71,0xff,0x08,0x79,0x3a]
+vcvtsd2usi (%rdx), %r15
+
+// CHECK: vcvtsd2usi {rn-sae}, %xmm15, %r13d
+// CHECK: encoding: [0x62,0x51,0x7f,0x18,0x79,0xef]
+vcvtsd2usi {rn-sae}, %xmm15, %r13d
+
+// CHECK: vcvtsd2usi {rn-sae}, %xmm15, %r15
+// CHECK: encoding: [0x62,0x51,0xff,0x18,0x79,0xff]
+vcvtsd2usi {rn-sae}, %xmm15, %r15
+
+// CHECK: vcvtsd2usi {rn-sae}, %xmm1, %r13d
+// CHECK: encoding: [0x62,0x71,0x7f,0x18,0x79,0xe9]
+vcvtsd2usi {rn-sae}, %xmm1, %r13d
+
+// CHECK: vcvtsd2usi {rn-sae}, %xmm1, %r15
+// CHECK: encoding: [0x62,0x71,0xff,0x18,0x79,0xf9]
+vcvtsd2usi {rn-sae}, %xmm1, %r15
+
+// CHECK: vcvtsd2usi {ru-sae}, %xmm15, %r13d
+// CHECK: encoding: [0x62,0x51,0x7f,0x58,0x79,0xef]
+vcvtsd2usi {ru-sae}, %xmm15, %r13d
+
+// CHECK: vcvtsd2usi {ru-sae}, %xmm15, %r15
+// CHECK: encoding: [0x62,0x51,0xff,0x58,0x79,0xff]
+vcvtsd2usi {ru-sae}, %xmm15, %r15
+
+// CHECK: vcvtsd2usi {ru-sae}, %xmm1, %r13d
+// CHECK: encoding: [0x62,0x71,0x7f,0x58,0x79,0xe9]
+vcvtsd2usi {ru-sae}, %xmm1, %r13d
+
+// CHECK: vcvtsd2usi {ru-sae}, %xmm1, %r15
+// CHECK: encoding: [0x62,0x71,0xff,0x58,0x79,0xf9]
+vcvtsd2usi {ru-sae}, %xmm1, %r15
+
+// CHECK: vcvtsd2usi {rz-sae}, %xmm15, %r13d
+// CHECK: encoding: [0x62,0x51,0x7f,0x78,0x79,0xef]
+vcvtsd2usi {rz-sae}, %xmm15, %r13d
+
+// CHECK: vcvtsd2usi {rz-sae}, %xmm15, %r15
+// CHECK: encoding: [0x62,0x51,0xff,0x78,0x79,0xff]
+vcvtsd2usi {rz-sae}, %xmm15, %r15
+
+// CHECK: vcvtsd2usi {rz-sae}, %xmm1, %r13d
+// CHECK: encoding: [0x62,0x71,0x7f,0x78,0x79,0xe9]
+vcvtsd2usi {rz-sae}, %xmm1, %r13d
+
+// CHECK: vcvtsd2usi {rz-sae}, %xmm1, %r15
+// CHECK: encoding: [0x62,0x71,0xff,0x78,0x79,0xf9]
+vcvtsd2usi {rz-sae}, %xmm1, %r15
+
+// CHECK: vcvtsd2usi %xmm15, %r13d
+// CHECK: encoding: [0x62,0x51,0x7f,0x08,0x79,0xef]
+vcvtsd2usi %xmm15, %r13d
+
+// CHECK: vcvtsd2usi %xmm15, %r15
+// CHECK: encoding: [0x62,0x51,0xff,0x08,0x79,0xff]
+vcvtsd2usi %xmm15, %r15
+
+// CHECK: vcvtsd2usi %xmm1, %r13d
+// CHECK: encoding: [0x62,0x71,0x7f,0x08,0x79,0xe9]
+vcvtsd2usi %xmm1, %r13d
+
+// CHECK: vcvtsd2usi %xmm1, %r15
+// CHECK: encoding: [0x62,0x71,0xff,0x08,0x79,0xf9]
+vcvtsd2usi %xmm1, %r15
+
+// CHECK: vcvtsi2sdl 256(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0xc5,0x03,0x2a,0xbc,0x82,0x00,0x01,0x00,0x00]
+vcvtsi2sdl 256(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vcvtsi2sdl -256(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0xc5,0x03,0x2a,0xbc,0x82,0x00,0xff,0xff,0xff]
+vcvtsi2sdl -256(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vcvtsi2sdl 256(%rdx,%rax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x2a,0x8c,0x82,0x00,0x01,0x00,0x00]
+vcvtsi2sdl 256(%rdx,%rax,4), %xmm1, %xmm1
+
+// CHECK: vcvtsi2sdl -256(%rdx,%rax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x2a,0x8c,0x82,0x00,0xff,0xff,0xff]
+vcvtsi2sdl -256(%rdx,%rax,4), %xmm1, %xmm1
+
+// CHECK: vcvtsi2sdl 256(%rdx,%rax), %xmm15, %xmm15
+// CHECK: encoding: [0xc5,0x03,0x2a,0xbc,0x02,0x00,0x01,0x00,0x00]
+vcvtsi2sdl 256(%rdx,%rax), %xmm15, %xmm15
+
+// CHECK: vcvtsi2sdl 256(%rdx,%rax), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x2a,0x8c,0x02,0x00,0x01,0x00,0x00]
+vcvtsi2sdl 256(%rdx,%rax), %xmm1, %xmm1
+
+// CHECK: vcvtsi2sdl 256(%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0xc5,0x03,0x2a,0xba,0x00,0x01,0x00,0x00]
+vcvtsi2sdl 256(%rdx), %xmm15, %xmm15
+
+// CHECK: vcvtsi2sdl 256(%rdx), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x2a,0x8a,0x00,0x01,0x00,0x00]
+vcvtsi2sdl 256(%rdx), %xmm1, %xmm1
+
+// CHECK: vcvtsi2sdl 485498096, %xmm15, %xmm15
+// CHECK: encoding: [0xc5,0x03,0x2a,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vcvtsi2sdl 485498096, %xmm15, %xmm15
+
+// CHECK: vcvtsi2sdl 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x2a,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vcvtsi2sdl 485498096, %xmm1, %xmm1
+
+// CHECK: vcvtsi2sdl %r13d, %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x41,0x03,0x2a,0xfd]
+vcvtsi2sdl %r13d, %xmm15, %xmm15
+
+// CHECK: vcvtsi2sdl %r13d, %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xc1,0x73,0x2a,0xcd]
+vcvtsi2sdl %r13d, %xmm1, %xmm1
+
+// CHECK: vcvtsi2sdl (%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0xc5,0x03,0x2a,0x3a]
+vcvtsi2sdl (%rdx), %xmm15, %xmm15
+
+// CHECK: vcvtsi2sdl (%rdx), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x2a,0x0a]
+vcvtsi2sdl (%rdx), %xmm1, %xmm1
+
+// CHECK: vcvtsi2sdq 485498096, %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x61,0x83,0x2a,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vcvtsi2sdq 485498096, %xmm15, %xmm15
+
+// CHECK: vcvtsi2sdq 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe1,0xf3,0x2a,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vcvtsi2sdq 485498096, %xmm1, %xmm1
+
+// CHECK: vcvtsi2sdq 512(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x61,0x83,0x2a,0xbc,0x82,0x00,0x02,0x00,0x00]
+vcvtsi2sdq 512(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vcvtsi2sdq -512(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x61,0x83,0x2a,0xbc,0x82,0x00,0xfe,0xff,0xff]
+vcvtsi2sdq -512(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vcvtsi2sdq 512(%rdx,%rax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe1,0xf3,0x2a,0x8c,0x82,0x00,0x02,0x00,0x00]
+vcvtsi2sdq 512(%rdx,%rax,4), %xmm1, %xmm1
+
+// CHECK: vcvtsi2sdq -512(%rdx,%rax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe1,0xf3,0x2a,0x8c,0x82,0x00,0xfe,0xff,0xff]
+vcvtsi2sdq -512(%rdx,%rax,4), %xmm1, %xmm1
+
+// CHECK: vcvtsi2sdq 512(%rdx,%rax), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x61,0x83,0x2a,0xbc,0x02,0x00,0x02,0x00,0x00]
+vcvtsi2sdq 512(%rdx,%rax), %xmm15, %xmm15
+
+// CHECK: vcvtsi2sdq 512(%rdx,%rax), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe1,0xf3,0x2a,0x8c,0x02,0x00,0x02,0x00,0x00]
+vcvtsi2sdq 512(%rdx,%rax), %xmm1, %xmm1
+
+// CHECK: vcvtsi2sdq 512(%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x61,0x83,0x2a,0xba,0x00,0x02,0x00,0x00]
+vcvtsi2sdq 512(%rdx), %xmm15, %xmm15
+
+// CHECK: vcvtsi2sdq 512(%rdx), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe1,0xf3,0x2a,0x8a,0x00,0x02,0x00,0x00]
+vcvtsi2sdq 512(%rdx), %xmm1, %xmm1
+
+// CHECK: vcvtsi2sdq %r15, {rd-sae}, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x51,0x87,0x38,0x2a,0xff]
+vcvtsi2sdq %r15, {rd-sae}, %xmm15, %xmm15
+
+// CHECK: vcvtsi2sdq %r15, {rd-sae}, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xd1,0xf7,0x38,0x2a,0xcf]
+vcvtsi2sdq %r15, {rd-sae}, %xmm1, %xmm1
+
+// CHECK: vcvtsi2sdq %r15, {rn-sae}, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x51,0x87,0x18,0x2a,0xff]
+vcvtsi2sdq %r15, {rn-sae}, %xmm15, %xmm15
+
+// CHECK: vcvtsi2sdq %r15, {rn-sae}, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xd1,0xf7,0x18,0x2a,0xcf]
+vcvtsi2sdq %r15, {rn-sae}, %xmm1, %xmm1
+
+// CHECK: vcvtsi2sdq %r15, {ru-sae}, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x51,0x87,0x58,0x2a,0xff]
+vcvtsi2sdq %r15, {ru-sae}, %xmm15, %xmm15
+
+// CHECK: vcvtsi2sdq %r15, {ru-sae}, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xd1,0xf7,0x58,0x2a,0xcf]
+vcvtsi2sdq %r15, {ru-sae}, %xmm1, %xmm1
+
+// CHECK: vcvtsi2sdq %r15, {rz-sae}, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x51,0x87,0x78,0x2a,0xff]
+vcvtsi2sdq %r15, {rz-sae}, %xmm15, %xmm15
+
+// CHECK: vcvtsi2sdq %r15, {rz-sae}, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xd1,0xf7,0x78,0x2a,0xcf]
+vcvtsi2sdq %r15, {rz-sae}, %xmm1, %xmm1
+
+// CHECK: vcvtsi2sdq %r15, %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x41,0x83,0x2a,0xff]
+vcvtsi2sdq %r15, %xmm15, %xmm15
+
+// CHECK: vcvtsi2sdq %r15, %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xc1,0xf3,0x2a,0xcf]
+vcvtsi2sdq %r15, %xmm1, %xmm1
+
+// CHECK: vcvtsi2sdq (%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x61,0x83,0x2a,0x3a]
+vcvtsi2sdq (%rdx), %xmm15, %xmm15
+
+// CHECK: vcvtsi2sdq (%rdx), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe1,0xf3,0x2a,0x0a]
+vcvtsi2sdq (%rdx), %xmm1, %xmm1
+
+// CHECK: vcvtsi2ssl 256(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0xc5,0x02,0x2a,0xbc,0x82,0x00,0x01,0x00,0x00]
+vcvtsi2ssl 256(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vcvtsi2ssl -256(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0xc5,0x02,0x2a,0xbc,0x82,0x00,0xff,0xff,0xff]
+vcvtsi2ssl -256(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vcvtsi2ssl 256(%rdx,%rax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x2a,0x8c,0x82,0x00,0x01,0x00,0x00]
+vcvtsi2ssl 256(%rdx,%rax,4), %xmm1, %xmm1
+
+// CHECK: vcvtsi2ssl -256(%rdx,%rax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x2a,0x8c,0x82,0x00,0xff,0xff,0xff]
+vcvtsi2ssl -256(%rdx,%rax,4), %xmm1, %xmm1
+
+// CHECK: vcvtsi2ssl 256(%rdx,%rax), %xmm15, %xmm15
+// CHECK: encoding: [0xc5,0x02,0x2a,0xbc,0x02,0x00,0x01,0x00,0x00]
+vcvtsi2ssl 256(%rdx,%rax), %xmm15, %xmm15
+
+// CHECK: vcvtsi2ssl 256(%rdx,%rax), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x2a,0x8c,0x02,0x00,0x01,0x00,0x00]
+vcvtsi2ssl 256(%rdx,%rax), %xmm1, %xmm1
+
+// CHECK: vcvtsi2ssl 256(%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0xc5,0x02,0x2a,0xba,0x00,0x01,0x00,0x00]
+vcvtsi2ssl 256(%rdx), %xmm15, %xmm15
+
+// CHECK: vcvtsi2ssl 256(%rdx), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x2a,0x8a,0x00,0x01,0x00,0x00]
+vcvtsi2ssl 256(%rdx), %xmm1, %xmm1
+
+// CHECK: vcvtsi2ssl 485498096, %xmm15, %xmm15
+// CHECK: encoding: [0xc5,0x02,0x2a,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vcvtsi2ssl 485498096, %xmm15, %xmm15
+
+// CHECK: vcvtsi2ssl 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x2a,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vcvtsi2ssl 485498096, %xmm1, %xmm1
+
+// CHECK: vcvtsi2ssl %r13d, {rd-sae}, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x51,0x06,0x38,0x2a,0xfd]
+vcvtsi2ssl %r13d, {rd-sae}, %xmm15, %xmm15
+
+// CHECK: vcvtsi2ssl %r13d, {rd-sae}, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xd1,0x76,0x38,0x2a,0xcd]
+vcvtsi2ssl %r13d, {rd-sae}, %xmm1, %xmm1
+
+// CHECK: vcvtsi2ssl %r13d, {rn-sae}, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x51,0x06,0x18,0x2a,0xfd]
+vcvtsi2ssl %r13d, {rn-sae}, %xmm15, %xmm15
+
+// CHECK: vcvtsi2ssl %r13d, {rn-sae}, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xd1,0x76,0x18,0x2a,0xcd]
+vcvtsi2ssl %r13d, {rn-sae}, %xmm1, %xmm1
+
+// CHECK: vcvtsi2ssl %r13d, {ru-sae}, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x51,0x06,0x58,0x2a,0xfd]
+vcvtsi2ssl %r13d, {ru-sae}, %xmm15, %xmm15
+
+// CHECK: vcvtsi2ssl %r13d, {ru-sae}, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xd1,0x76,0x58,0x2a,0xcd]
+vcvtsi2ssl %r13d, {ru-sae}, %xmm1, %xmm1
+
+// CHECK: vcvtsi2ssl %r13d, {rz-sae}, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x51,0x06,0x78,0x2a,0xfd]
+vcvtsi2ssl %r13d, {rz-sae}, %xmm15, %xmm15
+
+// CHECK: vcvtsi2ssl %r13d, {rz-sae}, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xd1,0x76,0x78,0x2a,0xcd]
+vcvtsi2ssl %r13d, {rz-sae}, %xmm1, %xmm1
+
+// CHECK: vcvtsi2ssl %r13d, %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x41,0x02,0x2a,0xfd]
+vcvtsi2ssl %r13d, %xmm15, %xmm15
+
+// CHECK: vcvtsi2ssl %r13d, %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xc1,0x72,0x2a,0xcd]
+vcvtsi2ssl %r13d, %xmm1, %xmm1
+
+// CHECK: vcvtsi2ssl (%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0xc5,0x02,0x2a,0x3a]
+vcvtsi2ssl (%rdx), %xmm15, %xmm15
+
+// CHECK: vcvtsi2ssl (%rdx), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x2a,0x0a]
+vcvtsi2ssl (%rdx), %xmm1, %xmm1
+
+// CHECK: vcvtsi2ssq 485498096, %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x61,0x82,0x2a,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vcvtsi2ssq 485498096, %xmm15, %xmm15
+
+// CHECK: vcvtsi2ssq 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe1,0xf2,0x2a,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vcvtsi2ssq 485498096, %xmm1, %xmm1
+
+// CHECK: vcvtsi2ssq 512(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x61,0x82,0x2a,0xbc,0x82,0x00,0x02,0x00,0x00]
+vcvtsi2ssq 512(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vcvtsi2ssq -512(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x61,0x82,0x2a,0xbc,0x82,0x00,0xfe,0xff,0xff]
+vcvtsi2ssq -512(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vcvtsi2ssq 512(%rdx,%rax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe1,0xf2,0x2a,0x8c,0x82,0x00,0x02,0x00,0x00]
+vcvtsi2ssq 512(%rdx,%rax,4), %xmm1, %xmm1
+
+// CHECK: vcvtsi2ssq -512(%rdx,%rax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe1,0xf2,0x2a,0x8c,0x82,0x00,0xfe,0xff,0xff]
+vcvtsi2ssq -512(%rdx,%rax,4), %xmm1, %xmm1
+
+// CHECK: vcvtsi2ssq 512(%rdx,%rax), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x61,0x82,0x2a,0xbc,0x02,0x00,0x02,0x00,0x00]
+vcvtsi2ssq 512(%rdx,%rax), %xmm15, %xmm15
+
+// CHECK: vcvtsi2ssq 512(%rdx,%rax), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe1,0xf2,0x2a,0x8c,0x02,0x00,0x02,0x00,0x00]
+vcvtsi2ssq 512(%rdx,%rax), %xmm1, %xmm1
+
+// CHECK: vcvtsi2ssq 512(%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x61,0x82,0x2a,0xba,0x00,0x02,0x00,0x00]
+vcvtsi2ssq 512(%rdx), %xmm15, %xmm15
+
+// CHECK: vcvtsi2ssq 512(%rdx), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe1,0xf2,0x2a,0x8a,0x00,0x02,0x00,0x00]
+vcvtsi2ssq 512(%rdx), %xmm1, %xmm1
+
+// CHECK: vcvtsi2ssq %r15, {rd-sae}, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x51,0x86,0x38,0x2a,0xff]
+vcvtsi2ssq %r15, {rd-sae}, %xmm15, %xmm15
+
+// CHECK: vcvtsi2ssq %r15, {rd-sae}, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xd1,0xf6,0x38,0x2a,0xcf]
+vcvtsi2ssq %r15, {rd-sae}, %xmm1, %xmm1
+
+// CHECK: vcvtsi2ssq %r15, {rn-sae}, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x51,0x86,0x18,0x2a,0xff]
+vcvtsi2ssq %r15, {rn-sae}, %xmm15, %xmm15
+
+// CHECK: vcvtsi2ssq %r15, {rn-sae}, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xd1,0xf6,0x18,0x2a,0xcf]
+vcvtsi2ssq %r15, {rn-sae}, %xmm1, %xmm1
+
+// CHECK: vcvtsi2ssq %r15, {ru-sae}, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x51,0x86,0x58,0x2a,0xff]
+vcvtsi2ssq %r15, {ru-sae}, %xmm15, %xmm15
+
+// CHECK: vcvtsi2ssq %r15, {ru-sae}, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xd1,0xf6,0x58,0x2a,0xcf]
+vcvtsi2ssq %r15, {ru-sae}, %xmm1, %xmm1
+
+// CHECK: vcvtsi2ssq %r15, {rz-sae}, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x51,0x86,0x78,0x2a,0xff]
+vcvtsi2ssq %r15, {rz-sae}, %xmm15, %xmm15
+
+// CHECK: vcvtsi2ssq %r15, {rz-sae}, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xd1,0xf6,0x78,0x2a,0xcf]
+vcvtsi2ssq %r15, {rz-sae}, %xmm1, %xmm1
+
+// CHECK: vcvtsi2ssq %r15, %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x41,0x82,0x2a,0xff]
+vcvtsi2ssq %r15, %xmm15, %xmm15
+
+// CHECK: vcvtsi2ssq %r15, %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xc1,0xf2,0x2a,0xcf]
+vcvtsi2ssq %r15, %xmm1, %xmm1
+
+// CHECK: vcvtsi2ssq (%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x61,0x82,0x2a,0x3a]
+vcvtsi2ssq (%rdx), %xmm15, %xmm15
+
+// CHECK: vcvtsi2ssq (%rdx), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe1,0xf2,0x2a,0x0a]
+vcvtsi2ssq (%rdx), %xmm1, %xmm1
+
+// CHECK: vcvtss2sd 256(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0xc5,0x02,0x5a,0xbc,0x82,0x00,0x01,0x00,0x00]
+vcvtss2sd 256(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vcvtss2sd -256(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0xc5,0x02,0x5a,0xbc,0x82,0x00,0xff,0xff,0xff]
+vcvtss2sd -256(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vcvtss2sd 256(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x71,0x06,0x0a,0x5a,0x7c,0x82,0x40]
+vcvtss2sd 256(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+
+// CHECK: vcvtss2sd -256(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x71,0x06,0x0a,0x5a,0x7c,0x82,0xc0]
+vcvtss2sd -256(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+
+// CHECK: vcvtss2sd 256(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x71,0x06,0x8a,0x5a,0x7c,0x82,0x40]
+vcvtss2sd 256(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vcvtss2sd -256(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x71,0x06,0x8a,0x5a,0x7c,0x82,0xc0]
+vcvtss2sd -256(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vcvtss2sd 256(%rdx,%rax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x5a,0x8c,0x82,0x00,0x01,0x00,0x00]
+vcvtss2sd 256(%rdx,%rax,4), %xmm1, %xmm1
+
+// CHECK: vcvtss2sd -256(%rdx,%rax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x5a,0x8c,0x82,0x00,0xff,0xff,0xff]
+vcvtss2sd -256(%rdx,%rax,4), %xmm1, %xmm1
+
+// CHECK: vcvtss2sd 256(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0x5a,0x4c,0x82,0x40]
+vcvtss2sd 256(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vcvtss2sd -256(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0x5a,0x4c,0x82,0xc0]
+vcvtss2sd -256(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vcvtss2sd 256(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x8a,0x5a,0x4c,0x82,0x40]
+vcvtss2sd 256(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vcvtss2sd -256(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x8a,0x5a,0x4c,0x82,0xc0]
+vcvtss2sd -256(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vcvtss2sd 256(%rdx,%rax), %xmm15, %xmm15
+// CHECK: encoding: [0xc5,0x02,0x5a,0xbc,0x02,0x00,0x01,0x00,0x00]
+vcvtss2sd 256(%rdx,%rax), %xmm15, %xmm15
+
+// CHECK: vcvtss2sd 256(%rdx,%rax), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x71,0x06,0x0a,0x5a,0x7c,0x02,0x40]
+vcvtss2sd 256(%rdx,%rax), %xmm15, %xmm15 {%k2}
+
+// CHECK: vcvtss2sd 256(%rdx,%rax), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x71,0x06,0x8a,0x5a,0x7c,0x02,0x40]
+vcvtss2sd 256(%rdx,%rax), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vcvtss2sd 256(%rdx,%rax), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x5a,0x8c,0x02,0x00,0x01,0x00,0x00]
+vcvtss2sd 256(%rdx,%rax), %xmm1, %xmm1
+
+// CHECK: vcvtss2sd 256(%rdx,%rax), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0x5a,0x4c,0x02,0x40]
+vcvtss2sd 256(%rdx,%rax), %xmm1, %xmm1 {%k2}
+
+// CHECK: vcvtss2sd 256(%rdx,%rax), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x8a,0x5a,0x4c,0x02,0x40]
+vcvtss2sd 256(%rdx,%rax), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vcvtss2sd 256(%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0xc5,0x02,0x5a,0xba,0x00,0x01,0x00,0x00]
+vcvtss2sd 256(%rdx), %xmm15, %xmm15
+
+// CHECK: vcvtss2sd 256(%rdx), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x71,0x06,0x0a,0x5a,0x7a,0x40]
+vcvtss2sd 256(%rdx), %xmm15, %xmm15 {%k2}
+
+// CHECK: vcvtss2sd 256(%rdx), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x71,0x06,0x8a,0x5a,0x7a,0x40]
+vcvtss2sd 256(%rdx), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vcvtss2sd 256(%rdx), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x5a,0x8a,0x00,0x01,0x00,0x00]
+vcvtss2sd 256(%rdx), %xmm1, %xmm1
+
+// CHECK: vcvtss2sd 256(%rdx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0x5a,0x4a,0x40]
+vcvtss2sd 256(%rdx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vcvtss2sd 256(%rdx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x8a,0x5a,0x4a,0x40]
+vcvtss2sd 256(%rdx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vcvtss2sd 485498096, %xmm15, %xmm15
+// CHECK: encoding: [0xc5,0x02,0x5a,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vcvtss2sd 485498096, %xmm15, %xmm15
+
+// CHECK: vcvtss2sd 485498096, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x71,0x06,0x0a,0x5a,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vcvtss2sd 485498096, %xmm15, %xmm15 {%k2}
+
+// CHECK: vcvtss2sd 485498096, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x71,0x06,0x8a,0x5a,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vcvtss2sd 485498096, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vcvtss2sd 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x5a,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vcvtss2sd 485498096, %xmm1, %xmm1
+
+// CHECK: vcvtss2sd 485498096, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0x5a,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vcvtss2sd 485498096, %xmm1, %xmm1 {%k2}
+
+// CHECK: vcvtss2sd 485498096, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x8a,0x5a,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vcvtss2sd 485498096, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vcvtss2sd (%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0xc5,0x02,0x5a,0x3a]
+vcvtss2sd (%rdx), %xmm15, %xmm15
+
+// CHECK: vcvtss2sd (%rdx), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x71,0x06,0x0a,0x5a,0x3a]
+vcvtss2sd (%rdx), %xmm15, %xmm15 {%k2}
+
+// CHECK: vcvtss2sd (%rdx), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x71,0x06,0x8a,0x5a,0x3a]
+vcvtss2sd (%rdx), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vcvtss2sd (%rdx), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x5a,0x0a]
+vcvtss2sd (%rdx), %xmm1, %xmm1
+
+// CHECK: vcvtss2sd (%rdx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0x5a,0x0a]
+vcvtss2sd (%rdx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vcvtss2sd (%rdx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x8a,0x5a,0x0a]
+vcvtss2sd (%rdx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vcvtss2sd {sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x51,0x06,0x18,0x5a,0xff]
+vcvtss2sd {sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vcvtss2sd {sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x51,0x06,0x1a,0x5a,0xff]
+vcvtss2sd {sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vcvtss2sd {sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x51,0x06,0x9a,0x5a,0xff]
+vcvtss2sd {sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vcvtss2sd {sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0x76,0x18,0x5a,0xc9]
+vcvtss2sd {sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vcvtss2sd {sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x1a,0x5a,0xc9]
+vcvtss2sd {sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vcvtss2sd {sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x9a,0x5a,0xc9]
+vcvtss2sd {sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vcvtss2sd %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x41,0x02,0x5a,0xff]
+vcvtss2sd %xmm15, %xmm15, %xmm15
+
+// CHECK: vcvtss2sd %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x51,0x06,0x0a,0x5a,0xff]
+vcvtss2sd %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vcvtss2sd %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x51,0x06,0x8a,0x5a,0xff]
+vcvtss2sd %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vcvtss2sd %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x5a,0xc9]
+vcvtss2sd %xmm1, %xmm1, %xmm1
+
+// CHECK: vcvtss2sd %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0x5a,0xc9]
+vcvtss2sd %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vcvtss2sd %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x8a,0x5a,0xc9]
+vcvtss2sd %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vcvtss2si 256(%rdx), %r13d
+// CHECK: encoding: [0xc5,0x7a,0x2d,0xaa,0x00,0x01,0x00,0x00]
+vcvtss2si 256(%rdx), %r13d
+
+// CHECK: vcvtss2si 256(%rdx), %r15
+// CHECK: encoding: [0xc4,0x61,0xfa,0x2d,0xba,0x00,0x01,0x00,0x00]
+vcvtss2si 256(%rdx), %r15
+
+// CHECK: vcvtss2si 256(%rdx,%rax,4), %r13d
+// CHECK: encoding: [0xc5,0x7a,0x2d,0xac,0x82,0x00,0x01,0x00,0x00]
+vcvtss2si 256(%rdx,%rax,4), %r13d
+
+// CHECK: vcvtss2si -256(%rdx,%rax,4), %r13d
+// CHECK: encoding: [0xc5,0x7a,0x2d,0xac,0x82,0x00,0xff,0xff,0xff]
+vcvtss2si -256(%rdx,%rax,4), %r13d
+
+// CHECK: vcvtss2si 256(%rdx,%rax,4), %r15
+// CHECK: encoding: [0xc4,0x61,0xfa,0x2d,0xbc,0x82,0x00,0x01,0x00,0x00]
+vcvtss2si 256(%rdx,%rax,4), %r15
+
+// CHECK: vcvtss2si -256(%rdx,%rax,4), %r15
+// CHECK: encoding: [0xc4,0x61,0xfa,0x2d,0xbc,0x82,0x00,0xff,0xff,0xff]
+vcvtss2si -256(%rdx,%rax,4), %r15
+
+// CHECK: vcvtss2si 256(%rdx,%rax), %r13d
+// CHECK: encoding: [0xc5,0x7a,0x2d,0xac,0x02,0x00,0x01,0x00,0x00]
+vcvtss2si 256(%rdx,%rax), %r13d
+
+// CHECK: vcvtss2si 256(%rdx,%rax), %r15
+// CHECK: encoding: [0xc4,0x61,0xfa,0x2d,0xbc,0x02,0x00,0x01,0x00,0x00]
+vcvtss2si 256(%rdx,%rax), %r15
+
+// CHECK: vcvtss2si 485498096, %r13d
+// CHECK: encoding: [0xc5,0x7a,0x2d,0x2c,0x25,0xf0,0x1c,0xf0,0x1c]
+vcvtss2si 485498096, %r13d
+
+// CHECK: vcvtss2si 485498096, %r15
+// CHECK: encoding: [0xc4,0x61,0xfa,0x2d,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vcvtss2si 485498096, %r15
+
+// CHECK: vcvtss2si {rd-sae}, %xmm15, %r13d
+// CHECK: encoding: [0x62,0x51,0x7e,0x38,0x2d,0xef]
+vcvtss2si {rd-sae}, %xmm15, %r13d
+
+// CHECK: vcvtss2si {rd-sae}, %xmm15, %r15
+// CHECK: encoding: [0x62,0x51,0xfe,0x38,0x2d,0xff]
+vcvtss2si {rd-sae}, %xmm15, %r15
+
+// CHECK: vcvtss2si {rd-sae}, %xmm1, %r13d
+// CHECK: encoding: [0x62,0x71,0x7e,0x38,0x2d,0xe9]
+vcvtss2si {rd-sae}, %xmm1, %r13d
+
+// CHECK: vcvtss2si {rd-sae}, %xmm1, %r15
+// CHECK: encoding: [0x62,0x71,0xfe,0x38,0x2d,0xf9]
+vcvtss2si {rd-sae}, %xmm1, %r15
+
+// CHECK: vcvtss2si (%rdx), %r13d
+// CHECK: encoding: [0xc5,0x7a,0x2d,0x2a]
+vcvtss2si (%rdx), %r13d
+
+// CHECK: vcvtss2si (%rdx), %r15
+// CHECK: encoding: [0xc4,0x61,0xfa,0x2d,0x3a]
+vcvtss2si (%rdx), %r15
+
+// CHECK: vcvtss2si {rn-sae}, %xmm15, %r13d
+// CHECK: encoding: [0x62,0x51,0x7e,0x18,0x2d,0xef]
+vcvtss2si {rn-sae}, %xmm15, %r13d
+
+// CHECK: vcvtss2si {rn-sae}, %xmm15, %r15
+// CHECK: encoding: [0x62,0x51,0xfe,0x18,0x2d,0xff]
+vcvtss2si {rn-sae}, %xmm15, %r15
+
+// CHECK: vcvtss2si {rn-sae}, %xmm1, %r13d
+// CHECK: encoding: [0x62,0x71,0x7e,0x18,0x2d,0xe9]
+vcvtss2si {rn-sae}, %xmm1, %r13d
+
+// CHECK: vcvtss2si {rn-sae}, %xmm1, %r15
+// CHECK: encoding: [0x62,0x71,0xfe,0x18,0x2d,0xf9]
+vcvtss2si {rn-sae}, %xmm1, %r15
+
+// CHECK: vcvtss2si {ru-sae}, %xmm15, %r13d
+// CHECK: encoding: [0x62,0x51,0x7e,0x58,0x2d,0xef]
+vcvtss2si {ru-sae}, %xmm15, %r13d
+
+// CHECK: vcvtss2si {ru-sae}, %xmm15, %r15
+// CHECK: encoding: [0x62,0x51,0xfe,0x58,0x2d,0xff]
+vcvtss2si {ru-sae}, %xmm15, %r15
+
+// CHECK: vcvtss2si {ru-sae}, %xmm1, %r13d
+// CHECK: encoding: [0x62,0x71,0x7e,0x58,0x2d,0xe9]
+vcvtss2si {ru-sae}, %xmm1, %r13d
+
+// CHECK: vcvtss2si {ru-sae}, %xmm1, %r15
+// CHECK: encoding: [0x62,0x71,0xfe,0x58,0x2d,0xf9]
+vcvtss2si {ru-sae}, %xmm1, %r15
+
+// CHECK: vcvtss2si {rz-sae}, %xmm15, %r13d
+// CHECK: encoding: [0x62,0x51,0x7e,0x78,0x2d,0xef]
+vcvtss2si {rz-sae}, %xmm15, %r13d
+
+// CHECK: vcvtss2si {rz-sae}, %xmm15, %r15
+// CHECK: encoding: [0x62,0x51,0xfe,0x78,0x2d,0xff]
+vcvtss2si {rz-sae}, %xmm15, %r15
+
+// CHECK: vcvtss2si {rz-sae}, %xmm1, %r13d
+// CHECK: encoding: [0x62,0x71,0x7e,0x78,0x2d,0xe9]
+vcvtss2si {rz-sae}, %xmm1, %r13d
+
+// CHECK: vcvtss2si {rz-sae}, %xmm1, %r15
+// CHECK: encoding: [0x62,0x71,0xfe,0x78,0x2d,0xf9]
+vcvtss2si {rz-sae}, %xmm1, %r15
+
+// CHECK: vcvtss2si %xmm15, %r13d
+// CHECK: encoding: [0xc4,0x41,0x7a,0x2d,0xef]
+vcvtss2si %xmm15, %r13d
+
+// CHECK: vcvtss2si %xmm15, %r15
+// CHECK: encoding: [0xc4,0x41,0xfa,0x2d,0xff]
+vcvtss2si %xmm15, %r15
+
+// CHECK: vcvtss2si %xmm1, %r13d
+// CHECK: encoding: [0xc5,0x7a,0x2d,0xe9]
+vcvtss2si %xmm1, %r13d
+
+// CHECK: vcvtss2si %xmm1, %r15
+// CHECK: encoding: [0xc4,0x61,0xfa,0x2d,0xf9]
+vcvtss2si %xmm1, %r15
+
+// CHECK: vcvtss2usi 256(%rdx), %r13d
+// CHECK: encoding: [0x62,0x71,0x7e,0x08,0x79,0x6a,0x40]
+vcvtss2usi 256(%rdx), %r13d
+
+// CHECK: vcvtss2usi 256(%rdx), %r15
+// CHECK: encoding: [0x62,0x71,0xfe,0x08,0x79,0x7a,0x40]
+vcvtss2usi 256(%rdx), %r15
+
+// CHECK: vcvtss2usi 256(%rdx,%rax,4), %r13d
+// CHECK: encoding: [0x62,0x71,0x7e,0x08,0x79,0x6c,0x82,0x40]
+vcvtss2usi 256(%rdx,%rax,4), %r13d
+
+// CHECK: vcvtss2usi -256(%rdx,%rax,4), %r13d
+// CHECK: encoding: [0x62,0x71,0x7e,0x08,0x79,0x6c,0x82,0xc0]
+vcvtss2usi -256(%rdx,%rax,4), %r13d
+
+// CHECK: vcvtss2usi 256(%rdx,%rax,4), %r15
+// CHECK: encoding: [0x62,0x71,0xfe,0x08,0x79,0x7c,0x82,0x40]
+vcvtss2usi 256(%rdx,%rax,4), %r15
+
+// CHECK: vcvtss2usi -256(%rdx,%rax,4), %r15
+// CHECK: encoding: [0x62,0x71,0xfe,0x08,0x79,0x7c,0x82,0xc0]
+vcvtss2usi -256(%rdx,%rax,4), %r15
+
+// CHECK: vcvtss2usi 256(%rdx,%rax), %r13d
+// CHECK: encoding: [0x62,0x71,0x7e,0x08,0x79,0x6c,0x02,0x40]
+vcvtss2usi 256(%rdx,%rax), %r13d
+
+// CHECK: vcvtss2usi 256(%rdx,%rax), %r15
+// CHECK: encoding: [0x62,0x71,0xfe,0x08,0x79,0x7c,0x02,0x40]
+vcvtss2usi 256(%rdx,%rax), %r15
+
+// CHECK: vcvtss2usi 485498096, %r13d
+// CHECK: encoding: [0x62,0x71,0x7e,0x08,0x79,0x2c,0x25,0xf0,0x1c,0xf0,0x1c]
+vcvtss2usi 485498096, %r13d
+
+// CHECK: vcvtss2usi 485498096, %r15
+// CHECK: encoding: [0x62,0x71,0xfe,0x08,0x79,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vcvtss2usi 485498096, %r15
+
+// CHECK: vcvtss2usi {rd-sae}, %xmm15, %r13d
+// CHECK: encoding: [0x62,0x51,0x7e,0x38,0x79,0xef]
+vcvtss2usi {rd-sae}, %xmm15, %r13d
+
+// CHECK: vcvtss2usi {rd-sae}, %xmm15, %r15
+// CHECK: encoding: [0x62,0x51,0xfe,0x38,0x79,0xff]
+vcvtss2usi {rd-sae}, %xmm15, %r15
+
+// CHECK: vcvtss2usi {rd-sae}, %xmm1, %r13d
+// CHECK: encoding: [0x62,0x71,0x7e,0x38,0x79,0xe9]
+vcvtss2usi {rd-sae}, %xmm1, %r13d
+
+// CHECK: vcvtss2usi {rd-sae}, %xmm1, %r15
+// CHECK: encoding: [0x62,0x71,0xfe,0x38,0x79,0xf9]
+vcvtss2usi {rd-sae}, %xmm1, %r15
+
+// CHECK: vcvtss2usi (%rdx), %r13d
+// CHECK: encoding: [0x62,0x71,0x7e,0x08,0x79,0x2a]
+vcvtss2usi (%rdx), %r13d
+
+// CHECK: vcvtss2usi (%rdx), %r15
+// CHECK: encoding: [0x62,0x71,0xfe,0x08,0x79,0x3a]
+vcvtss2usi (%rdx), %r15
+
+// CHECK: vcvtss2usi {rn-sae}, %xmm15, %r13d
+// CHECK: encoding: [0x62,0x51,0x7e,0x18,0x79,0xef]
+vcvtss2usi {rn-sae}, %xmm15, %r13d
+
+// CHECK: vcvtss2usi {rn-sae}, %xmm15, %r15
+// CHECK: encoding: [0x62,0x51,0xfe,0x18,0x79,0xff]
+vcvtss2usi {rn-sae}, %xmm15, %r15
+
+// CHECK: vcvtss2usi {rn-sae}, %xmm1, %r13d
+// CHECK: encoding: [0x62,0x71,0x7e,0x18,0x79,0xe9]
+vcvtss2usi {rn-sae}, %xmm1, %r13d
+
+// CHECK: vcvtss2usi {rn-sae}, %xmm1, %r15
+// CHECK: encoding: [0x62,0x71,0xfe,0x18,0x79,0xf9]
+vcvtss2usi {rn-sae}, %xmm1, %r15
+
+// CHECK: vcvtss2usi {ru-sae}, %xmm15, %r13d
+// CHECK: encoding: [0x62,0x51,0x7e,0x58,0x79,0xef]
+vcvtss2usi {ru-sae}, %xmm15, %r13d
+
+// CHECK: vcvtss2usi {ru-sae}, %xmm15, %r15
+// CHECK: encoding: [0x62,0x51,0xfe,0x58,0x79,0xff]
+vcvtss2usi {ru-sae}, %xmm15, %r15
+
+// CHECK: vcvtss2usi {ru-sae}, %xmm1, %r13d
+// CHECK: encoding: [0x62,0x71,0x7e,0x58,0x79,0xe9]
+vcvtss2usi {ru-sae}, %xmm1, %r13d
+
+// CHECK: vcvtss2usi {ru-sae}, %xmm1, %r15
+// CHECK: encoding: [0x62,0x71,0xfe,0x58,0x79,0xf9]
+vcvtss2usi {ru-sae}, %xmm1, %r15
+
+// CHECK: vcvtss2usi {rz-sae}, %xmm15, %r13d
+// CHECK: encoding: [0x62,0x51,0x7e,0x78,0x79,0xef]
+vcvtss2usi {rz-sae}, %xmm15, %r13d
+
+// CHECK: vcvtss2usi {rz-sae}, %xmm15, %r15
+// CHECK: encoding: [0x62,0x51,0xfe,0x78,0x79,0xff]
+vcvtss2usi {rz-sae}, %xmm15, %r15
+
+// CHECK: vcvtss2usi {rz-sae}, %xmm1, %r13d
+// CHECK: encoding: [0x62,0x71,0x7e,0x78,0x79,0xe9]
+vcvtss2usi {rz-sae}, %xmm1, %r13d
+
+// CHECK: vcvtss2usi {rz-sae}, %xmm1, %r15
+// CHECK: encoding: [0x62,0x71,0xfe,0x78,0x79,0xf9]
+vcvtss2usi {rz-sae}, %xmm1, %r15
+
+// CHECK: vcvtss2usi %xmm15, %r13d
+// CHECK: encoding: [0x62,0x51,0x7e,0x08,0x79,0xef]
+vcvtss2usi %xmm15, %r13d
+
+// CHECK: vcvtss2usi %xmm15, %r15
+// CHECK: encoding: [0x62,0x51,0xfe,0x08,0x79,0xff]
+vcvtss2usi %xmm15, %r15
+
+// CHECK: vcvtss2usi %xmm1, %r13d
+// CHECK: encoding: [0x62,0x71,0x7e,0x08,0x79,0xe9]
+vcvtss2usi %xmm1, %r13d
+
+// CHECK: vcvtss2usi %xmm1, %r15
+// CHECK: encoding: [0x62,0x71,0xfe,0x08,0x79,0xf9]
+vcvtss2usi %xmm1, %r15
+
+// CHECK: vcvttsd2si 485498096, %r13d
+// CHECK: encoding: [0xc5,0x7b,0x2c,0x2c,0x25,0xf0,0x1c,0xf0,0x1c]
+vcvttsd2si 485498096, %r13d
+
+// CHECK: vcvttsd2si 485498096, %r15
+// CHECK: encoding: [0xc4,0x61,0xfb,0x2c,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vcvttsd2si 485498096, %r15
+
+// CHECK: vcvttsd2si 512(%rdx), %r13d
+// CHECK: encoding: [0xc5,0x7b,0x2c,0xaa,0x00,0x02,0x00,0x00]
+vcvttsd2si 512(%rdx), %r13d
+
+// CHECK: vcvttsd2si 512(%rdx), %r15
+// CHECK: encoding: [0xc4,0x61,0xfb,0x2c,0xba,0x00,0x02,0x00,0x00]
+vcvttsd2si 512(%rdx), %r15
+
+// CHECK: vcvttsd2si 512(%rdx,%rax,4), %r13d
+// CHECK: encoding: [0xc5,0x7b,0x2c,0xac,0x82,0x00,0x02,0x00,0x00]
+vcvttsd2si 512(%rdx,%rax,4), %r13d
+
+// CHECK: vcvttsd2si -512(%rdx,%rax,4), %r13d
+// CHECK: encoding: [0xc5,0x7b,0x2c,0xac,0x82,0x00,0xfe,0xff,0xff]
+vcvttsd2si -512(%rdx,%rax,4), %r13d
+
+// CHECK: vcvttsd2si 512(%rdx,%rax,4), %r15
+// CHECK: encoding: [0xc4,0x61,0xfb,0x2c,0xbc,0x82,0x00,0x02,0x00,0x00]
+vcvttsd2si 512(%rdx,%rax,4), %r15
+
+// CHECK: vcvttsd2si -512(%rdx,%rax,4), %r15
+// CHECK: encoding: [0xc4,0x61,0xfb,0x2c,0xbc,0x82,0x00,0xfe,0xff,0xff]
+vcvttsd2si -512(%rdx,%rax,4), %r15
+
+// CHECK: vcvttsd2si 512(%rdx,%rax), %r13d
+// CHECK: encoding: [0xc5,0x7b,0x2c,0xac,0x02,0x00,0x02,0x00,0x00]
+vcvttsd2si 512(%rdx,%rax), %r13d
+
+// CHECK: vcvttsd2si 512(%rdx,%rax), %r15
+// CHECK: encoding: [0xc4,0x61,0xfb,0x2c,0xbc,0x02,0x00,0x02,0x00,0x00]
+vcvttsd2si 512(%rdx,%rax), %r15
+
+// CHECK: vcvttsd2si (%rdx), %r13d
+// CHECK: encoding: [0xc5,0x7b,0x2c,0x2a]
+vcvttsd2si (%rdx), %r13d
+
+// CHECK: vcvttsd2si (%rdx), %r15
+// CHECK: encoding: [0xc4,0x61,0xfb,0x2c,0x3a]
+vcvttsd2si (%rdx), %r15
+
+// CHECK: vcvttsd2si {sae}, %xmm15, %r13d
+// CHECK: encoding: [0x62,0x51,0x7f,0x18,0x2c,0xef]
+vcvttsd2si {sae}, %xmm15, %r13d
+
+// CHECK: vcvttsd2si {sae}, %xmm15, %r15
+// CHECK: encoding: [0x62,0x51,0xff,0x18,0x2c,0xff]
+vcvttsd2si {sae}, %xmm15, %r15
+
+// CHECK: vcvttsd2si {sae}, %xmm1, %r13d
+// CHECK: encoding: [0x62,0x71,0x7f,0x18,0x2c,0xe9]
+vcvttsd2si {sae}, %xmm1, %r13d
+
+// CHECK: vcvttsd2si {sae}, %xmm1, %r15
+// CHECK: encoding: [0x62,0x71,0xff,0x18,0x2c,0xf9]
+vcvttsd2si {sae}, %xmm1, %r15
+
+// CHECK: vcvttsd2si %xmm15, %r13d
+// CHECK: encoding: [0xc4,0x41,0x7b,0x2c,0xef]
+vcvttsd2si %xmm15, %r13d
+
+// CHECK: vcvttsd2si %xmm15, %r15
+// CHECK: encoding: [0xc4,0x41,0xfb,0x2c,0xff]
+vcvttsd2si %xmm15, %r15
+
+// CHECK: vcvttsd2si %xmm1, %r13d
+// CHECK: encoding: [0xc5,0x7b,0x2c,0xe9]
+vcvttsd2si %xmm1, %r13d
+
+// CHECK: vcvttsd2si %xmm1, %r15
+// CHECK: encoding: [0xc4,0x61,0xfb,0x2c,0xf9]
+vcvttsd2si %xmm1, %r15
+
+// CHECK: vcvttsd2usi 485498096, %r13d
+// CHECK: encoding: [0x62,0x71,0x7f,0x08,0x78,0x2c,0x25,0xf0,0x1c,0xf0,0x1c]
+vcvttsd2usi 485498096, %r13d
+
+// CHECK: vcvttsd2usi 485498096, %r15
+// CHECK: encoding: [0x62,0x71,0xff,0x08,0x78,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vcvttsd2usi 485498096, %r15
+
+// CHECK: vcvttsd2usi 512(%rdx), %r13d
+// CHECK: encoding: [0x62,0x71,0x7f,0x08,0x78,0x6a,0x40]
+vcvttsd2usi 512(%rdx), %r13d
+
+// CHECK: vcvttsd2usi 512(%rdx), %r15
+// CHECK: encoding: [0x62,0x71,0xff,0x08,0x78,0x7a,0x40]
+vcvttsd2usi 512(%rdx), %r15
+
+// CHECK: vcvttsd2usi 512(%rdx,%rax,4), %r13d
+// CHECK: encoding: [0x62,0x71,0x7f,0x08,0x78,0x6c,0x82,0x40]
+vcvttsd2usi 512(%rdx,%rax,4), %r13d
+
+// CHECK: vcvttsd2usi -512(%rdx,%rax,4), %r13d
+// CHECK: encoding: [0x62,0x71,0x7f,0x08,0x78,0x6c,0x82,0xc0]
+vcvttsd2usi -512(%rdx,%rax,4), %r13d
+
+// CHECK: vcvttsd2usi 512(%rdx,%rax,4), %r15
+// CHECK: encoding: [0x62,0x71,0xff,0x08,0x78,0x7c,0x82,0x40]
+vcvttsd2usi 512(%rdx,%rax,4), %r15
+
+// CHECK: vcvttsd2usi -512(%rdx,%rax,4), %r15
+// CHECK: encoding: [0x62,0x71,0xff,0x08,0x78,0x7c,0x82,0xc0]
+vcvttsd2usi -512(%rdx,%rax,4), %r15
+
+// CHECK: vcvttsd2usi 512(%rdx,%rax), %r13d
+// CHECK: encoding: [0x62,0x71,0x7f,0x08,0x78,0x6c,0x02,0x40]
+vcvttsd2usi 512(%rdx,%rax), %r13d
+
+// CHECK: vcvttsd2usi 512(%rdx,%rax), %r15
+// CHECK: encoding: [0x62,0x71,0xff,0x08,0x78,0x7c,0x02,0x40]
+vcvttsd2usi 512(%rdx,%rax), %r15
+
+// CHECK: vcvttsd2usi (%rdx), %r13d
+// CHECK: encoding: [0x62,0x71,0x7f,0x08,0x78,0x2a]
+vcvttsd2usi (%rdx), %r13d
+
+// CHECK: vcvttsd2usi (%rdx), %r15
+// CHECK: encoding: [0x62,0x71,0xff,0x08,0x78,0x3a]
+vcvttsd2usi (%rdx), %r15
+
+// CHECK: vcvttsd2usi {sae}, %xmm15, %r13d
+// CHECK: encoding: [0x62,0x51,0x7f,0x18,0x78,0xef]
+vcvttsd2usi {sae}, %xmm15, %r13d
+
+// CHECK: vcvttsd2usi {sae}, %xmm15, %r15
+// CHECK: encoding: [0x62,0x51,0xff,0x18,0x78,0xff]
+vcvttsd2usi {sae}, %xmm15, %r15
+
+// CHECK: vcvttsd2usi {sae}, %xmm1, %r13d
+// CHECK: encoding: [0x62,0x71,0x7f,0x18,0x78,0xe9]
+vcvttsd2usi {sae}, %xmm1, %r13d
+
+// CHECK: vcvttsd2usi {sae}, %xmm1, %r15
+// CHECK: encoding: [0x62,0x71,0xff,0x18,0x78,0xf9]
+vcvttsd2usi {sae}, %xmm1, %r15
+
+// CHECK: vcvttsd2usi %xmm15, %r13d
+// CHECK: encoding: [0x62,0x51,0x7f,0x08,0x78,0xef]
+vcvttsd2usi %xmm15, %r13d
+
+// CHECK: vcvttsd2usi %xmm15, %r15
+// CHECK: encoding: [0x62,0x51,0xff,0x08,0x78,0xff]
+vcvttsd2usi %xmm15, %r15
+
+// CHECK: vcvttsd2usi %xmm1, %r13d
+// CHECK: encoding: [0x62,0x71,0x7f,0x08,0x78,0xe9]
+vcvttsd2usi %xmm1, %r13d
+
+// CHECK: vcvttsd2usi %xmm1, %r15
+// CHECK: encoding: [0x62,0x71,0xff,0x08,0x78,0xf9]
+vcvttsd2usi %xmm1, %r15
+
+// CHECK: vcvttss2si 256(%rdx), %r13d
+// CHECK: encoding: [0xc5,0x7a,0x2c,0xaa,0x00,0x01,0x00,0x00]
+vcvttss2si 256(%rdx), %r13d
+
+// CHECK: vcvttss2si 256(%rdx), %r15
+// CHECK: encoding: [0xc4,0x61,0xfa,0x2c,0xba,0x00,0x01,0x00,0x00]
+vcvttss2si 256(%rdx), %r15
+
+// CHECK: vcvttss2si 256(%rdx,%rax,4), %r13d
+// CHECK: encoding: [0xc5,0x7a,0x2c,0xac,0x82,0x00,0x01,0x00,0x00]
+vcvttss2si 256(%rdx,%rax,4), %r13d
+
+// CHECK: vcvttss2si -256(%rdx,%rax,4), %r13d
+// CHECK: encoding: [0xc5,0x7a,0x2c,0xac,0x82,0x00,0xff,0xff,0xff]
+vcvttss2si -256(%rdx,%rax,4), %r13d
+
+// CHECK: vcvttss2si 256(%rdx,%rax,4), %r15
+// CHECK: encoding: [0xc4,0x61,0xfa,0x2c,0xbc,0x82,0x00,0x01,0x00,0x00]
+vcvttss2si 256(%rdx,%rax,4), %r15
+
+// CHECK: vcvttss2si -256(%rdx,%rax,4), %r15
+// CHECK: encoding: [0xc4,0x61,0xfa,0x2c,0xbc,0x82,0x00,0xff,0xff,0xff]
+vcvttss2si -256(%rdx,%rax,4), %r15
+
+// CHECK: vcvttss2si 256(%rdx,%rax), %r13d
+// CHECK: encoding: [0xc5,0x7a,0x2c,0xac,0x02,0x00,0x01,0x00,0x00]
+vcvttss2si 256(%rdx,%rax), %r13d
+
+// CHECK: vcvttss2si 256(%rdx,%rax), %r15
+// CHECK: encoding: [0xc4,0x61,0xfa,0x2c,0xbc,0x02,0x00,0x01,0x00,0x00]
+vcvttss2si 256(%rdx,%rax), %r15
+
+// CHECK: vcvttss2si 485498096, %r13d
+// CHECK: encoding: [0xc5,0x7a,0x2c,0x2c,0x25,0xf0,0x1c,0xf0,0x1c]
+vcvttss2si 485498096, %r13d
+
+// CHECK: vcvttss2si 485498096, %r15
+// CHECK: encoding: [0xc4,0x61,0xfa,0x2c,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vcvttss2si 485498096, %r15
+
+// CHECK: vcvttss2si (%rdx), %r13d
+// CHECK: encoding: [0xc5,0x7a,0x2c,0x2a]
+vcvttss2si (%rdx), %r13d
+
+// CHECK: vcvttss2si (%rdx), %r15
+// CHECK: encoding: [0xc4,0x61,0xfa,0x2c,0x3a]
+vcvttss2si (%rdx), %r15
+
+// CHECK: vcvttss2si {sae}, %xmm15, %r13d
+// CHECK: encoding: [0x62,0x51,0x7e,0x18,0x2c,0xef]
+vcvttss2si {sae}, %xmm15, %r13d
+
+// CHECK: vcvttss2si {sae}, %xmm15, %r15
+// CHECK: encoding: [0x62,0x51,0xfe,0x18,0x2c,0xff]
+vcvttss2si {sae}, %xmm15, %r15
+
+// CHECK: vcvttss2si {sae}, %xmm1, %r13d
+// CHECK: encoding: [0x62,0x71,0x7e,0x18,0x2c,0xe9]
+vcvttss2si {sae}, %xmm1, %r13d
+
+// CHECK: vcvttss2si {sae}, %xmm1, %r15
+// CHECK: encoding: [0x62,0x71,0xfe,0x18,0x2c,0xf9]
+vcvttss2si {sae}, %xmm1, %r15
+
+// CHECK: vcvttss2si %xmm15, %r13d
+// CHECK: encoding: [0xc4,0x41,0x7a,0x2c,0xef]
+vcvttss2si %xmm15, %r13d
+
+// CHECK: vcvttss2si %xmm15, %r15
+// CHECK: encoding: [0xc4,0x41,0xfa,0x2c,0xff]
+vcvttss2si %xmm15, %r15
+
+// CHECK: vcvttss2si %xmm1, %r13d
+// CHECK: encoding: [0xc5,0x7a,0x2c,0xe9]
+vcvttss2si %xmm1, %r13d
+
+// CHECK: vcvttss2si %xmm1, %r15
+// CHECK: encoding: [0xc4,0x61,0xfa,0x2c,0xf9]
+vcvttss2si %xmm1, %r15
+
+// CHECK: vcvttss2usi 256(%rdx), %r13d
+// CHECK: encoding: [0x62,0x71,0x7e,0x08,0x78,0x6a,0x40]
+vcvttss2usi 256(%rdx), %r13d
+
+// CHECK: vcvttss2usi 256(%rdx), %r15
+// CHECK: encoding: [0x62,0x71,0xfe,0x08,0x78,0x7a,0x40]
+vcvttss2usi 256(%rdx), %r15
+
+// CHECK: vcvttss2usi 256(%rdx,%rax,4), %r13d
+// CHECK: encoding: [0x62,0x71,0x7e,0x08,0x78,0x6c,0x82,0x40]
+vcvttss2usi 256(%rdx,%rax,4), %r13d
+
+// CHECK: vcvttss2usi -256(%rdx,%rax,4), %r13d
+// CHECK: encoding: [0x62,0x71,0x7e,0x08,0x78,0x6c,0x82,0xc0]
+vcvttss2usi -256(%rdx,%rax,4), %r13d
+
+// CHECK: vcvttss2usi 256(%rdx,%rax,4), %r15
+// CHECK: encoding: [0x62,0x71,0xfe,0x08,0x78,0x7c,0x82,0x40]
+vcvttss2usi 256(%rdx,%rax,4), %r15
+
+// CHECK: vcvttss2usi -256(%rdx,%rax,4), %r15
+// CHECK: encoding: [0x62,0x71,0xfe,0x08,0x78,0x7c,0x82,0xc0]
+vcvttss2usi -256(%rdx,%rax,4), %r15
+
+// CHECK: vcvttss2usi 256(%rdx,%rax), %r13d
+// CHECK: encoding: [0x62,0x71,0x7e,0x08,0x78,0x6c,0x02,0x40]
+vcvttss2usi 256(%rdx,%rax), %r13d
+
+// CHECK: vcvttss2usi 256(%rdx,%rax), %r15
+// CHECK: encoding: [0x62,0x71,0xfe,0x08,0x78,0x7c,0x02,0x40]
+vcvttss2usi 256(%rdx,%rax), %r15
+
+// CHECK: vcvttss2usi 485498096, %r13d
+// CHECK: encoding: [0x62,0x71,0x7e,0x08,0x78,0x2c,0x25,0xf0,0x1c,0xf0,0x1c]
+vcvttss2usi 485498096, %r13d
+
+// CHECK: vcvttss2usi 485498096, %r15
+// CHECK: encoding: [0x62,0x71,0xfe,0x08,0x78,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vcvttss2usi 485498096, %r15
+
+// CHECK: vcvttss2usi (%rdx), %r13d
+// CHECK: encoding: [0x62,0x71,0x7e,0x08,0x78,0x2a]
+vcvttss2usi (%rdx), %r13d
+
+// CHECK: vcvttss2usi (%rdx), %r15
+// CHECK: encoding: [0x62,0x71,0xfe,0x08,0x78,0x3a]
+vcvttss2usi (%rdx), %r15
+
+// CHECK: vcvttss2usi {sae}, %xmm15, %r13d
+// CHECK: encoding: [0x62,0x51,0x7e,0x18,0x78,0xef]
+vcvttss2usi {sae}, %xmm15, %r13d
+
+// CHECK: vcvttss2usi {sae}, %xmm15, %r15
+// CHECK: encoding: [0x62,0x51,0xfe,0x18,0x78,0xff]
+vcvttss2usi {sae}, %xmm15, %r15
+
+// CHECK: vcvttss2usi {sae}, %xmm1, %r13d
+// CHECK: encoding: [0x62,0x71,0x7e,0x18,0x78,0xe9]
+vcvttss2usi {sae}, %xmm1, %r13d
+
+// CHECK: vcvttss2usi {sae}, %xmm1, %r15
+// CHECK: encoding: [0x62,0x71,0xfe,0x18,0x78,0xf9]
+vcvttss2usi {sae}, %xmm1, %r15
+
+// CHECK: vcvttss2usi %xmm15, %r13d
+// CHECK: encoding: [0x62,0x51,0x7e,0x08,0x78,0xef]
+vcvttss2usi %xmm15, %r13d
+
+// CHECK: vcvttss2usi %xmm15, %r15
+// CHECK: encoding: [0x62,0x51,0xfe,0x08,0x78,0xff]
+vcvttss2usi %xmm15, %r15
+
+// CHECK: vcvttss2usi %xmm1, %r13d
+// CHECK: encoding: [0x62,0x71,0x7e,0x08,0x78,0xe9]
+vcvttss2usi %xmm1, %r13d
+
+// CHECK: vcvttss2usi %xmm1, %r15
+// CHECK: encoding: [0x62,0x71,0xfe,0x08,0x78,0xf9]
+vcvttss2usi %xmm1, %r15
+
+// CHECK: vcvtusi2sdl 256(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x71,0x07,0x08,0x7b,0x7c,0x82,0x40]
+vcvtusi2sdl 256(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vcvtusi2sdl -256(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x71,0x07,0x08,0x7b,0x7c,0x82,0xc0]
+vcvtusi2sdl -256(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vcvtusi2sdl 256(%rdx,%rax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0x77,0x08,0x7b,0x4c,0x82,0x40]
+vcvtusi2sdl 256(%rdx,%rax,4), %xmm1, %xmm1
+
+// CHECK: vcvtusi2sdl -256(%rdx,%rax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0x77,0x08,0x7b,0x4c,0x82,0xc0]
+vcvtusi2sdl -256(%rdx,%rax,4), %xmm1, %xmm1
+
+// CHECK: vcvtusi2sdl 256(%rdx,%rax), %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x71,0x07,0x08,0x7b,0x7c,0x02,0x40]
+vcvtusi2sdl 256(%rdx,%rax), %xmm15, %xmm15
+
+// CHECK: vcvtusi2sdl 256(%rdx,%rax), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0x77,0x08,0x7b,0x4c,0x02,0x40]
+vcvtusi2sdl 256(%rdx,%rax), %xmm1, %xmm1
+
+// CHECK: vcvtusi2sdl 256(%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x71,0x07,0x08,0x7b,0x7a,0x40]
+vcvtusi2sdl 256(%rdx), %xmm15, %xmm15
+
+// CHECK: vcvtusi2sdl 256(%rdx), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0x77,0x08,0x7b,0x4a,0x40]
+vcvtusi2sdl 256(%rdx), %xmm1, %xmm1
+
+// CHECK: vcvtusi2sdl 485498096, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x71,0x07,0x08,0x7b,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vcvtusi2sdl 485498096, %xmm15, %xmm15
+
+// CHECK: vcvtusi2sdl 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0x77,0x08,0x7b,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vcvtusi2sdl 485498096, %xmm1, %xmm1
+
+// CHECK: vcvtusi2sdl %r13d, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x51,0x07,0x08,0x7b,0xfd]
+vcvtusi2sdl %r13d, %xmm15, %xmm15
+
+// CHECK: vcvtusi2sdl %r13d, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xd1,0x77,0x08,0x7b,0xcd]
+vcvtusi2sdl %r13d, %xmm1, %xmm1
+
+// CHECK: vcvtusi2sdl (%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x71,0x07,0x08,0x7b,0x3a]
+vcvtusi2sdl (%rdx), %xmm15, %xmm15
+
+// CHECK: vcvtusi2sdl (%rdx), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0x77,0x08,0x7b,0x0a]
+vcvtusi2sdl (%rdx), %xmm1, %xmm1
+
+// CHECK: vcvtusi2sdq 485498096, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x71,0x87,0x08,0x7b,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vcvtusi2sdq 485498096, %xmm15, %xmm15
+
+// CHECK: vcvtusi2sdq 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0xf7,0x08,0x7b,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vcvtusi2sdq 485498096, %xmm1, %xmm1
+
+// CHECK: vcvtusi2sdq 512(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x71,0x87,0x08,0x7b,0x7c,0x82,0x40]
+vcvtusi2sdq 512(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vcvtusi2sdq -512(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x71,0x87,0x08,0x7b,0x7c,0x82,0xc0]
+vcvtusi2sdq -512(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vcvtusi2sdq 512(%rdx,%rax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0xf7,0x08,0x7b,0x4c,0x82,0x40]
+vcvtusi2sdq 512(%rdx,%rax,4), %xmm1, %xmm1
+
+// CHECK: vcvtusi2sdq -512(%rdx,%rax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0xf7,0x08,0x7b,0x4c,0x82,0xc0]
+vcvtusi2sdq -512(%rdx,%rax,4), %xmm1, %xmm1
+
+// CHECK: vcvtusi2sdq 512(%rdx,%rax), %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x71,0x87,0x08,0x7b,0x7c,0x02,0x40]
+vcvtusi2sdq 512(%rdx,%rax), %xmm15, %xmm15
+
+// CHECK: vcvtusi2sdq 512(%rdx,%rax), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0xf7,0x08,0x7b,0x4c,0x02,0x40]
+vcvtusi2sdq 512(%rdx,%rax), %xmm1, %xmm1
+
+// CHECK: vcvtusi2sdq 512(%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x71,0x87,0x08,0x7b,0x7a,0x40]
+vcvtusi2sdq 512(%rdx), %xmm15, %xmm15
+
+// CHECK: vcvtusi2sdq 512(%rdx), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0xf7,0x08,0x7b,0x4a,0x40]
+vcvtusi2sdq 512(%rdx), %xmm1, %xmm1
+
+// CHECK: vcvtusi2sdq %r15, {rd-sae}, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x51,0x87,0x38,0x7b,0xff]
+vcvtusi2sdq %r15, {rd-sae}, %xmm15, %xmm15
+
+// CHECK: vcvtusi2sdq %r15, {rd-sae}, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xd1,0xf7,0x38,0x7b,0xcf]
+vcvtusi2sdq %r15, {rd-sae}, %xmm1, %xmm1
+
+// CHECK: vcvtusi2sdq %r15, {rn-sae}, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x51,0x87,0x18,0x7b,0xff]
+vcvtusi2sdq %r15, {rn-sae}, %xmm15, %xmm15
+
+// CHECK: vcvtusi2sdq %r15, {rn-sae}, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xd1,0xf7,0x18,0x7b,0xcf]
+vcvtusi2sdq %r15, {rn-sae}, %xmm1, %xmm1
+
+// CHECK: vcvtusi2sdq %r15, {ru-sae}, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x51,0x87,0x58,0x7b,0xff]
+vcvtusi2sdq %r15, {ru-sae}, %xmm15, %xmm15
+
+// CHECK: vcvtusi2sdq %r15, {ru-sae}, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xd1,0xf7,0x58,0x7b,0xcf]
+vcvtusi2sdq %r15, {ru-sae}, %xmm1, %xmm1
+
+// CHECK: vcvtusi2sdq %r15, {rz-sae}, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x51,0x87,0x78,0x7b,0xff]
+vcvtusi2sdq %r15, {rz-sae}, %xmm15, %xmm15
+
+// CHECK: vcvtusi2sdq %r15, {rz-sae}, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xd1,0xf7,0x78,0x7b,0xcf]
+vcvtusi2sdq %r15, {rz-sae}, %xmm1, %xmm1
+
+// CHECK: vcvtusi2sdq %r15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x51,0x87,0x08,0x7b,0xff]
+vcvtusi2sdq %r15, %xmm15, %xmm15
+
+// CHECK: vcvtusi2sdq %r15, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xd1,0xf7,0x08,0x7b,0xcf]
+vcvtusi2sdq %r15, %xmm1, %xmm1
+
+// CHECK: vcvtusi2sdq (%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x71,0x87,0x08,0x7b,0x3a]
+vcvtusi2sdq (%rdx), %xmm15, %xmm15
+
+// CHECK: vcvtusi2sdq (%rdx), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0xf7,0x08,0x7b,0x0a]
+vcvtusi2sdq (%rdx), %xmm1, %xmm1
+
+// CHECK: vcvtusi2ssl 256(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x71,0x06,0x08,0x7b,0x7c,0x82,0x40]
+vcvtusi2ssl 256(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vcvtusi2ssl -256(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x71,0x06,0x08,0x7b,0x7c,0x82,0xc0]
+vcvtusi2ssl -256(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vcvtusi2ssl 256(%rdx,%rax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0x76,0x08,0x7b,0x4c,0x82,0x40]
+vcvtusi2ssl 256(%rdx,%rax,4), %xmm1, %xmm1
+
+// CHECK: vcvtusi2ssl -256(%rdx,%rax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0x76,0x08,0x7b,0x4c,0x82,0xc0]
+vcvtusi2ssl -256(%rdx,%rax,4), %xmm1, %xmm1
+
+// CHECK: vcvtusi2ssl 256(%rdx,%rax), %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x71,0x06,0x08,0x7b,0x7c,0x02,0x40]
+vcvtusi2ssl 256(%rdx,%rax), %xmm15, %xmm15
+
+// CHECK: vcvtusi2ssl 256(%rdx,%rax), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0x76,0x08,0x7b,0x4c,0x02,0x40]
+vcvtusi2ssl 256(%rdx,%rax), %xmm1, %xmm1
+
+// CHECK: vcvtusi2ssl 256(%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x71,0x06,0x08,0x7b,0x7a,0x40]
+vcvtusi2ssl 256(%rdx), %xmm15, %xmm15
+
+// CHECK: vcvtusi2ssl 256(%rdx), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0x76,0x08,0x7b,0x4a,0x40]
+vcvtusi2ssl 256(%rdx), %xmm1, %xmm1
+
+// CHECK: vcvtusi2ssl 485498096, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x71,0x06,0x08,0x7b,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vcvtusi2ssl 485498096, %xmm15, %xmm15
+
+// CHECK: vcvtusi2ssl 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0x76,0x08,0x7b,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vcvtusi2ssl 485498096, %xmm1, %xmm1
+
+// CHECK: vcvtusi2ssl %r13d, {rd-sae}, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x51,0x06,0x38,0x7b,0xfd]
+vcvtusi2ssl %r13d, {rd-sae}, %xmm15, %xmm15
+
+// CHECK: vcvtusi2ssl %r13d, {rd-sae}, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xd1,0x76,0x38,0x7b,0xcd]
+vcvtusi2ssl %r13d, {rd-sae}, %xmm1, %xmm1
+
+// CHECK: vcvtusi2ssl %r13d, {rn-sae}, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x51,0x06,0x18,0x7b,0xfd]
+vcvtusi2ssl %r13d, {rn-sae}, %xmm15, %xmm15
+
+// CHECK: vcvtusi2ssl %r13d, {rn-sae}, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xd1,0x76,0x18,0x7b,0xcd]
+vcvtusi2ssl %r13d, {rn-sae}, %xmm1, %xmm1
+
+// CHECK: vcvtusi2ssl %r13d, {ru-sae}, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x51,0x06,0x58,0x7b,0xfd]
+vcvtusi2ssl %r13d, {ru-sae}, %xmm15, %xmm15
+
+// CHECK: vcvtusi2ssl %r13d, {ru-sae}, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xd1,0x76,0x58,0x7b,0xcd]
+vcvtusi2ssl %r13d, {ru-sae}, %xmm1, %xmm1
+
+// CHECK: vcvtusi2ssl %r13d, {rz-sae}, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x51,0x06,0x78,0x7b,0xfd]
+vcvtusi2ssl %r13d, {rz-sae}, %xmm15, %xmm15
+
+// CHECK: vcvtusi2ssl %r13d, {rz-sae}, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xd1,0x76,0x78,0x7b,0xcd]
+vcvtusi2ssl %r13d, {rz-sae}, %xmm1, %xmm1
+
+// CHECK: vcvtusi2ssl %r13d, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x51,0x06,0x08,0x7b,0xfd]
+vcvtusi2ssl %r13d, %xmm15, %xmm15
+
+// CHECK: vcvtusi2ssl %r13d, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xd1,0x76,0x08,0x7b,0xcd]
+vcvtusi2ssl %r13d, %xmm1, %xmm1
+
+// CHECK: vcvtusi2ssl (%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x71,0x06,0x08,0x7b,0x3a]
+vcvtusi2ssl (%rdx), %xmm15, %xmm15
+
+// CHECK: vcvtusi2ssl (%rdx), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0x76,0x08,0x7b,0x0a]
+vcvtusi2ssl (%rdx), %xmm1, %xmm1
+
+// CHECK: vcvtusi2ssq 485498096, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x71,0x86,0x08,0x7b,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vcvtusi2ssq 485498096, %xmm15, %xmm15
+
+// CHECK: vcvtusi2ssq 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0xf6,0x08,0x7b,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vcvtusi2ssq 485498096, %xmm1, %xmm1
+
+// CHECK: vcvtusi2ssq 512(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x71,0x86,0x08,0x7b,0x7c,0x82,0x40]
+vcvtusi2ssq 512(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vcvtusi2ssq -512(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x71,0x86,0x08,0x7b,0x7c,0x82,0xc0]
+vcvtusi2ssq -512(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vcvtusi2ssq 512(%rdx,%rax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0xf6,0x08,0x7b,0x4c,0x82,0x40]
+vcvtusi2ssq 512(%rdx,%rax,4), %xmm1, %xmm1
+
+// CHECK: vcvtusi2ssq -512(%rdx,%rax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0xf6,0x08,0x7b,0x4c,0x82,0xc0]
+vcvtusi2ssq -512(%rdx,%rax,4), %xmm1, %xmm1
+
+// CHECK: vcvtusi2ssq 512(%rdx,%rax), %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x71,0x86,0x08,0x7b,0x7c,0x02,0x40]
+vcvtusi2ssq 512(%rdx,%rax), %xmm15, %xmm15
+
+// CHECK: vcvtusi2ssq 512(%rdx,%rax), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0xf6,0x08,0x7b,0x4c,0x02,0x40]
+vcvtusi2ssq 512(%rdx,%rax), %xmm1, %xmm1
+
+// CHECK: vcvtusi2ssq 512(%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x71,0x86,0x08,0x7b,0x7a,0x40]
+vcvtusi2ssq 512(%rdx), %xmm15, %xmm15
+
+// CHECK: vcvtusi2ssq 512(%rdx), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0xf6,0x08,0x7b,0x4a,0x40]
+vcvtusi2ssq 512(%rdx), %xmm1, %xmm1
+
+// CHECK: vcvtusi2ssq %r15, {rd-sae}, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x51,0x86,0x38,0x7b,0xff]
+vcvtusi2ssq %r15, {rd-sae}, %xmm15, %xmm15
+
+// CHECK: vcvtusi2ssq %r15, {rd-sae}, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xd1,0xf6,0x38,0x7b,0xcf]
+vcvtusi2ssq %r15, {rd-sae}, %xmm1, %xmm1
+
+// CHECK: vcvtusi2ssq %r15, {rn-sae}, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x51,0x86,0x18,0x7b,0xff]
+vcvtusi2ssq %r15, {rn-sae}, %xmm15, %xmm15
+
+// CHECK: vcvtusi2ssq %r15, {rn-sae}, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xd1,0xf6,0x18,0x7b,0xcf]
+vcvtusi2ssq %r15, {rn-sae}, %xmm1, %xmm1
+
+// CHECK: vcvtusi2ssq %r15, {ru-sae}, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x51,0x86,0x58,0x7b,0xff]
+vcvtusi2ssq %r15, {ru-sae}, %xmm15, %xmm15
+
+// CHECK: vcvtusi2ssq %r15, {ru-sae}, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xd1,0xf6,0x58,0x7b,0xcf]
+vcvtusi2ssq %r15, {ru-sae}, %xmm1, %xmm1
+
+// CHECK: vcvtusi2ssq %r15, {rz-sae}, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x51,0x86,0x78,0x7b,0xff]
+vcvtusi2ssq %r15, {rz-sae}, %xmm15, %xmm15
+
+// CHECK: vcvtusi2ssq %r15, {rz-sae}, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xd1,0xf6,0x78,0x7b,0xcf]
+vcvtusi2ssq %r15, {rz-sae}, %xmm1, %xmm1
+
+// CHECK: vcvtusi2ssq %r15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x51,0x86,0x08,0x7b,0xff]
+vcvtusi2ssq %r15, %xmm15, %xmm15
+
+// CHECK: vcvtusi2ssq %r15, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xd1,0xf6,0x08,0x7b,0xcf]
+vcvtusi2ssq %r15, %xmm1, %xmm1
+
+// CHECK: vcvtusi2ssq (%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x71,0x86,0x08,0x7b,0x3a]
+vcvtusi2ssq (%rdx), %xmm15, %xmm15
+
+// CHECK: vcvtusi2ssq (%rdx), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0xf6,0x08,0x7b,0x0a]
+vcvtusi2ssq (%rdx), %xmm1, %xmm1
+
+// CHECK: vdivsd 485498096, %xmm15, %xmm15
+// CHECK: encoding: [0xc5,0x03,0x5e,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vdivsd 485498096, %xmm15, %xmm15
+
+// CHECK: vdivsd 485498096, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x71,0x87,0x0a,0x5e,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vdivsd 485498096, %xmm15, %xmm15 {%k2}
+
+// CHECK: vdivsd 485498096, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x71,0x87,0x8a,0x5e,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vdivsd 485498096, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vdivsd 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x5e,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vdivsd 485498096, %xmm1, %xmm1
+
+// CHECK: vdivsd 485498096, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0x5e,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vdivsd 485498096, %xmm1, %xmm1 {%k2}
+
+// CHECK: vdivsd 485498096, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x8a,0x5e,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vdivsd 485498096, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vdivsd 512(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0xc5,0x03,0x5e,0xbc,0x82,0x00,0x02,0x00,0x00]
+vdivsd 512(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vdivsd -512(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0xc5,0x03,0x5e,0xbc,0x82,0x00,0xfe,0xff,0xff]
+vdivsd -512(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vdivsd 512(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x71,0x87,0x0a,0x5e,0x7c,0x82,0x40]
+vdivsd 512(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+
+// CHECK: vdivsd -512(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x71,0x87,0x0a,0x5e,0x7c,0x82,0xc0]
+vdivsd -512(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+
+// CHECK: vdivsd 512(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x71,0x87,0x8a,0x5e,0x7c,0x82,0x40]
+vdivsd 512(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vdivsd -512(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x71,0x87,0x8a,0x5e,0x7c,0x82,0xc0]
+vdivsd -512(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vdivsd 512(%rdx,%rax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x5e,0x8c,0x82,0x00,0x02,0x00,0x00]
+vdivsd 512(%rdx,%rax,4), %xmm1, %xmm1
+
+// CHECK: vdivsd -512(%rdx,%rax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x5e,0x8c,0x82,0x00,0xfe,0xff,0xff]
+vdivsd -512(%rdx,%rax,4), %xmm1, %xmm1
+
+// CHECK: vdivsd 512(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0x5e,0x4c,0x82,0x40]
+vdivsd 512(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vdivsd -512(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0x5e,0x4c,0x82,0xc0]
+vdivsd -512(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vdivsd 512(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x8a,0x5e,0x4c,0x82,0x40]
+vdivsd 512(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vdivsd -512(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x8a,0x5e,0x4c,0x82,0xc0]
+vdivsd -512(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vdivsd 512(%rdx,%rax), %xmm15, %xmm15
+// CHECK: encoding: [0xc5,0x03,0x5e,0xbc,0x02,0x00,0x02,0x00,0x00]
+vdivsd 512(%rdx,%rax), %xmm15, %xmm15
+
+// CHECK: vdivsd 512(%rdx,%rax), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x71,0x87,0x0a,0x5e,0x7c,0x02,0x40]
+vdivsd 512(%rdx,%rax), %xmm15, %xmm15 {%k2}
+
+// CHECK: vdivsd 512(%rdx,%rax), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x71,0x87,0x8a,0x5e,0x7c,0x02,0x40]
+vdivsd 512(%rdx,%rax), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vdivsd 512(%rdx,%rax), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x5e,0x8c,0x02,0x00,0x02,0x00,0x00]
+vdivsd 512(%rdx,%rax), %xmm1, %xmm1
+
+// CHECK: vdivsd 512(%rdx,%rax), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0x5e,0x4c,0x02,0x40]
+vdivsd 512(%rdx,%rax), %xmm1, %xmm1 {%k2}
+
+// CHECK: vdivsd 512(%rdx,%rax), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x8a,0x5e,0x4c,0x02,0x40]
+vdivsd 512(%rdx,%rax), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vdivsd 512(%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0xc5,0x03,0x5e,0xba,0x00,0x02,0x00,0x00]
+vdivsd 512(%rdx), %xmm15, %xmm15
+
+// CHECK: vdivsd 512(%rdx), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x71,0x87,0x0a,0x5e,0x7a,0x40]
+vdivsd 512(%rdx), %xmm15, %xmm15 {%k2}
+
+// CHECK: vdivsd 512(%rdx), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x71,0x87,0x8a,0x5e,0x7a,0x40]
+vdivsd 512(%rdx), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vdivsd 512(%rdx), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x5e,0x8a,0x00,0x02,0x00,0x00]
+vdivsd 512(%rdx), %xmm1, %xmm1
+
+// CHECK: vdivsd 512(%rdx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0x5e,0x4a,0x40]
+vdivsd 512(%rdx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vdivsd 512(%rdx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x8a,0x5e,0x4a,0x40]
+vdivsd 512(%rdx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vdivsd {rd-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x51,0x87,0x38,0x5e,0xff]
+vdivsd {rd-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vdivsd {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x51,0x87,0x3a,0x5e,0xff]
+vdivsd {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vdivsd {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x51,0x87,0xba,0x5e,0xff]
+vdivsd {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vdivsd {rd-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0xf7,0x38,0x5e,0xc9]
+vdivsd {rd-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vdivsd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x3a,0x5e,0xc9]
+vdivsd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vdivsd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0xba,0x5e,0xc9]
+vdivsd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vdivsd (%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0xc5,0x03,0x5e,0x3a]
+vdivsd (%rdx), %xmm15, %xmm15
+
+// CHECK: vdivsd (%rdx), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x71,0x87,0x0a,0x5e,0x3a]
+vdivsd (%rdx), %xmm15, %xmm15 {%k2}
+
+// CHECK: vdivsd (%rdx), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x71,0x87,0x8a,0x5e,0x3a]
+vdivsd (%rdx), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vdivsd (%rdx), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x5e,0x0a]
+vdivsd (%rdx), %xmm1, %xmm1
+
+// CHECK: vdivsd (%rdx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0x5e,0x0a]
+vdivsd (%rdx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vdivsd (%rdx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x8a,0x5e,0x0a]
+vdivsd (%rdx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vdivsd {rn-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x51,0x87,0x18,0x5e,0xff]
+vdivsd {rn-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vdivsd {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x51,0x87,0x1a,0x5e,0xff]
+vdivsd {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vdivsd {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x51,0x87,0x9a,0x5e,0xff]
+vdivsd {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vdivsd {rn-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0xf7,0x18,0x5e,0xc9]
+vdivsd {rn-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vdivsd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x1a,0x5e,0xc9]
+vdivsd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vdivsd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x9a,0x5e,0xc9]
+vdivsd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vdivsd {ru-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x51,0x87,0x58,0x5e,0xff]
+vdivsd {ru-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vdivsd {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x51,0x87,0x5a,0x5e,0xff]
+vdivsd {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vdivsd {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x51,0x87,0xda,0x5e,0xff]
+vdivsd {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vdivsd {ru-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0xf7,0x58,0x5e,0xc9]
+vdivsd {ru-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vdivsd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x5a,0x5e,0xc9]
+vdivsd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vdivsd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0xda,0x5e,0xc9]
+vdivsd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vdivsd {rz-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x51,0x87,0x78,0x5e,0xff]
+vdivsd {rz-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vdivsd {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x51,0x87,0x7a,0x5e,0xff]
+vdivsd {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vdivsd {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x51,0x87,0xfa,0x5e,0xff]
+vdivsd {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vdivsd {rz-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0xf7,0x78,0x5e,0xc9]
+vdivsd {rz-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vdivsd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x7a,0x5e,0xc9]
+vdivsd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vdivsd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0xfa,0x5e,0xc9]
+vdivsd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vdivsd %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x41,0x03,0x5e,0xff]
+vdivsd %xmm15, %xmm15, %xmm15
+
+// CHECK: vdivsd %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x51,0x87,0x0a,0x5e,0xff]
+vdivsd %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vdivsd %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x51,0x87,0x8a,0x5e,0xff]
+vdivsd %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vdivsd %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x5e,0xc9]
+vdivsd %xmm1, %xmm1, %xmm1
+
+// CHECK: vdivsd %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0x5e,0xc9]
+vdivsd %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vdivsd %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x8a,0x5e,0xc9]
+vdivsd %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vdivss 256(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0xc5,0x02,0x5e,0xbc,0x82,0x00,0x01,0x00,0x00]
+vdivss 256(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vdivss -256(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0xc5,0x02,0x5e,0xbc,0x82,0x00,0xff,0xff,0xff]
+vdivss -256(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vdivss 256(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x71,0x06,0x0a,0x5e,0x7c,0x82,0x40]
+vdivss 256(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+
+// CHECK: vdivss -256(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x71,0x06,0x0a,0x5e,0x7c,0x82,0xc0]
+vdivss -256(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+
+// CHECK: vdivss 256(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x71,0x06,0x8a,0x5e,0x7c,0x82,0x40]
+vdivss 256(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vdivss -256(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x71,0x06,0x8a,0x5e,0x7c,0x82,0xc0]
+vdivss -256(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vdivss 256(%rdx,%rax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x5e,0x8c,0x82,0x00,0x01,0x00,0x00]
+vdivss 256(%rdx,%rax,4), %xmm1, %xmm1
+
+// CHECK: vdivss -256(%rdx,%rax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x5e,0x8c,0x82,0x00,0xff,0xff,0xff]
+vdivss -256(%rdx,%rax,4), %xmm1, %xmm1
+
+// CHECK: vdivss 256(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0x5e,0x4c,0x82,0x40]
+vdivss 256(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vdivss -256(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0x5e,0x4c,0x82,0xc0]
+vdivss -256(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vdivss 256(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x8a,0x5e,0x4c,0x82,0x40]
+vdivss 256(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vdivss -256(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x8a,0x5e,0x4c,0x82,0xc0]
+vdivss -256(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vdivss 256(%rdx,%rax), %xmm15, %xmm15
+// CHECK: encoding: [0xc5,0x02,0x5e,0xbc,0x02,0x00,0x01,0x00,0x00]
+vdivss 256(%rdx,%rax), %xmm15, %xmm15
+
+// CHECK: vdivss 256(%rdx,%rax), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x71,0x06,0x0a,0x5e,0x7c,0x02,0x40]
+vdivss 256(%rdx,%rax), %xmm15, %xmm15 {%k2}
+
+// CHECK: vdivss 256(%rdx,%rax), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x71,0x06,0x8a,0x5e,0x7c,0x02,0x40]
+vdivss 256(%rdx,%rax), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vdivss 256(%rdx,%rax), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x5e,0x8c,0x02,0x00,0x01,0x00,0x00]
+vdivss 256(%rdx,%rax), %xmm1, %xmm1
+
+// CHECK: vdivss 256(%rdx,%rax), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0x5e,0x4c,0x02,0x40]
+vdivss 256(%rdx,%rax), %xmm1, %xmm1 {%k2}
+
+// CHECK: vdivss 256(%rdx,%rax), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x8a,0x5e,0x4c,0x02,0x40]
+vdivss 256(%rdx,%rax), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vdivss 256(%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0xc5,0x02,0x5e,0xba,0x00,0x01,0x00,0x00]
+vdivss 256(%rdx), %xmm15, %xmm15
+
+// CHECK: vdivss 256(%rdx), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x71,0x06,0x0a,0x5e,0x7a,0x40]
+vdivss 256(%rdx), %xmm15, %xmm15 {%k2}
+
+// CHECK: vdivss 256(%rdx), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x71,0x06,0x8a,0x5e,0x7a,0x40]
+vdivss 256(%rdx), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vdivss 256(%rdx), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x5e,0x8a,0x00,0x01,0x00,0x00]
+vdivss 256(%rdx), %xmm1, %xmm1
+
+// CHECK: vdivss 256(%rdx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0x5e,0x4a,0x40]
+vdivss 256(%rdx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vdivss 256(%rdx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x8a,0x5e,0x4a,0x40]
+vdivss 256(%rdx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vdivss 485498096, %xmm15, %xmm15
+// CHECK: encoding: [0xc5,0x02,0x5e,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vdivss 485498096, %xmm15, %xmm15
+
+// CHECK: vdivss 485498096, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x71,0x06,0x0a,0x5e,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vdivss 485498096, %xmm15, %xmm15 {%k2}
+
+// CHECK: vdivss 485498096, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x71,0x06,0x8a,0x5e,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vdivss 485498096, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vdivss 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x5e,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vdivss 485498096, %xmm1, %xmm1
+
+// CHECK: vdivss 485498096, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0x5e,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vdivss 485498096, %xmm1, %xmm1 {%k2}
+
+// CHECK: vdivss 485498096, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x8a,0x5e,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vdivss 485498096, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vdivss {rd-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x51,0x06,0x38,0x5e,0xff]
+vdivss {rd-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vdivss {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x51,0x06,0x3a,0x5e,0xff]
+vdivss {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vdivss {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x51,0x06,0xba,0x5e,0xff]
+vdivss {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vdivss {rd-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0x76,0x38,0x5e,0xc9]
+vdivss {rd-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vdivss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x3a,0x5e,0xc9]
+vdivss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vdivss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0xba,0x5e,0xc9]
+vdivss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vdivss (%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0xc5,0x02,0x5e,0x3a]
+vdivss (%rdx), %xmm15, %xmm15
+
+// CHECK: vdivss (%rdx), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x71,0x06,0x0a,0x5e,0x3a]
+vdivss (%rdx), %xmm15, %xmm15 {%k2}
+
+// CHECK: vdivss (%rdx), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x71,0x06,0x8a,0x5e,0x3a]
+vdivss (%rdx), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vdivss (%rdx), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x5e,0x0a]
+vdivss (%rdx), %xmm1, %xmm1
+
+// CHECK: vdivss (%rdx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0x5e,0x0a]
+vdivss (%rdx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vdivss (%rdx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x8a,0x5e,0x0a]
+vdivss (%rdx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vdivss {rn-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x51,0x06,0x18,0x5e,0xff]
+vdivss {rn-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vdivss {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x51,0x06,0x1a,0x5e,0xff]
+vdivss {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vdivss {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x51,0x06,0x9a,0x5e,0xff]
+vdivss {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vdivss {rn-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0x76,0x18,0x5e,0xc9]
+vdivss {rn-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vdivss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x1a,0x5e,0xc9]
+vdivss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vdivss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x9a,0x5e,0xc9]
+vdivss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vdivss {ru-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x51,0x06,0x58,0x5e,0xff]
+vdivss {ru-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vdivss {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x51,0x06,0x5a,0x5e,0xff]
+vdivss {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vdivss {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x51,0x06,0xda,0x5e,0xff]
+vdivss {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vdivss {ru-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0x76,0x58,0x5e,0xc9]
+vdivss {ru-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vdivss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x5a,0x5e,0xc9]
+vdivss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vdivss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0xda,0x5e,0xc9]
+vdivss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vdivss {rz-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x51,0x06,0x78,0x5e,0xff]
+vdivss {rz-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vdivss {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x51,0x06,0x7a,0x5e,0xff]
+vdivss {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vdivss {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x51,0x06,0xfa,0x5e,0xff]
+vdivss {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vdivss {rz-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0x76,0x78,0x5e,0xc9]
+vdivss {rz-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vdivss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x7a,0x5e,0xc9]
+vdivss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vdivss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0xfa,0x5e,0xc9]
+vdivss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vdivss %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x41,0x02,0x5e,0xff]
+vdivss %xmm15, %xmm15, %xmm15
+
+// CHECK: vdivss %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x51,0x06,0x0a,0x5e,0xff]
+vdivss %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vdivss %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x51,0x06,0x8a,0x5e,0xff]
+vdivss %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vdivss %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x5e,0xc9]
+vdivss %xmm1, %xmm1, %xmm1
+
+// CHECK: vdivss %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0x5e,0xc9]
+vdivss %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vdivss %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x8a,0x5e,0xc9]
+vdivss %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfixupimmsd $0, 485498096, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x73,0x85,0x08,0x55,0x3c,0x25,0xf0,0x1c,0xf0,0x1c,0x00]
+vfixupimmsd $0, 485498096, %xmm15, %xmm15
+
+// CHECK: vfixupimmsd $0, 485498096, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x73,0x85,0x0a,0x55,0x3c,0x25,0xf0,0x1c,0xf0,0x1c,0x00]
+vfixupimmsd $0, 485498096, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfixupimmsd $0, 485498096, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x73,0x85,0x8a,0x55,0x3c,0x25,0xf0,0x1c,0xf0,0x1c,0x00]
+vfixupimmsd $0, 485498096, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfixupimmsd $0, 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf3,0xf5,0x08,0x55,0x0c,0x25,0xf0,0x1c,0xf0,0x1c,0x00]
+vfixupimmsd $0, 485498096, %xmm1, %xmm1
+
+// CHECK: vfixupimmsd $0, 485498096, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf3,0xf5,0x0a,0x55,0x0c,0x25,0xf0,0x1c,0xf0,0x1c,0x00]
+vfixupimmsd $0, 485498096, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfixupimmsd $0, 485498096, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf3,0xf5,0x8a,0x55,0x0c,0x25,0xf0,0x1c,0xf0,0x1c,0x00]
+vfixupimmsd $0, 485498096, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfixupimmsd $0, 512(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x73,0x85,0x08,0x55,0x7c,0x82,0x40,0x00]
+vfixupimmsd $0, 512(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vfixupimmsd $0, -512(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x73,0x85,0x08,0x55,0x7c,0x82,0xc0,0x00]
+vfixupimmsd $0, -512(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vfixupimmsd $0, 512(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x73,0x85,0x0a,0x55,0x7c,0x82,0x40,0x00]
+vfixupimmsd $0, 512(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfixupimmsd $0, -512(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x73,0x85,0x0a,0x55,0x7c,0x82,0xc0,0x00]
+vfixupimmsd $0, -512(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfixupimmsd $0, 512(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x73,0x85,0x8a,0x55,0x7c,0x82,0x40,0x00]
+vfixupimmsd $0, 512(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfixupimmsd $0, -512(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x73,0x85,0x8a,0x55,0x7c,0x82,0xc0,0x00]
+vfixupimmsd $0, -512(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfixupimmsd $0, 512(%rdx,%rax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf3,0xf5,0x08,0x55,0x4c,0x82,0x40,0x00]
+vfixupimmsd $0, 512(%rdx,%rax,4), %xmm1, %xmm1
+
+// CHECK: vfixupimmsd $0, -512(%rdx,%rax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf3,0xf5,0x08,0x55,0x4c,0x82,0xc0,0x00]
+vfixupimmsd $0, -512(%rdx,%rax,4), %xmm1, %xmm1
+
+// CHECK: vfixupimmsd $0, 512(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf3,0xf5,0x0a,0x55,0x4c,0x82,0x40,0x00]
+vfixupimmsd $0, 512(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfixupimmsd $0, -512(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf3,0xf5,0x0a,0x55,0x4c,0x82,0xc0,0x00]
+vfixupimmsd $0, -512(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfixupimmsd $0, 512(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf3,0xf5,0x8a,0x55,0x4c,0x82,0x40,0x00]
+vfixupimmsd $0, 512(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfixupimmsd $0, -512(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf3,0xf5,0x8a,0x55,0x4c,0x82,0xc0,0x00]
+vfixupimmsd $0, -512(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfixupimmsd $0, 512(%rdx,%rax), %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x73,0x85,0x08,0x55,0x7c,0x02,0x40,0x00]
+vfixupimmsd $0, 512(%rdx,%rax), %xmm15, %xmm15
+
+// CHECK: vfixupimmsd $0, 512(%rdx,%rax), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x73,0x85,0x0a,0x55,0x7c,0x02,0x40,0x00]
+vfixupimmsd $0, 512(%rdx,%rax), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfixupimmsd $0, 512(%rdx,%rax), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x73,0x85,0x8a,0x55,0x7c,0x02,0x40,0x00]
+vfixupimmsd $0, 512(%rdx,%rax), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfixupimmsd $0, 512(%rdx,%rax), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf3,0xf5,0x08,0x55,0x4c,0x02,0x40,0x00]
+vfixupimmsd $0, 512(%rdx,%rax), %xmm1, %xmm1
+
+// CHECK: vfixupimmsd $0, 512(%rdx,%rax), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf3,0xf5,0x0a,0x55,0x4c,0x02,0x40,0x00]
+vfixupimmsd $0, 512(%rdx,%rax), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfixupimmsd $0, 512(%rdx,%rax), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf3,0xf5,0x8a,0x55,0x4c,0x02,0x40,0x00]
+vfixupimmsd $0, 512(%rdx,%rax), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfixupimmsd $0, 512(%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x73,0x85,0x08,0x55,0x7a,0x40,0x00]
+vfixupimmsd $0, 512(%rdx), %xmm15, %xmm15
+
+// CHECK: vfixupimmsd $0, 512(%rdx), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x73,0x85,0x0a,0x55,0x7a,0x40,0x00]
+vfixupimmsd $0, 512(%rdx), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfixupimmsd $0, 512(%rdx), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x73,0x85,0x8a,0x55,0x7a,0x40,0x00]
+vfixupimmsd $0, 512(%rdx), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfixupimmsd $0, 512(%rdx), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf3,0xf5,0x08,0x55,0x4a,0x40,0x00]
+vfixupimmsd $0, 512(%rdx), %xmm1, %xmm1
+
+// CHECK: vfixupimmsd $0, 512(%rdx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf3,0xf5,0x0a,0x55,0x4a,0x40,0x00]
+vfixupimmsd $0, 512(%rdx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfixupimmsd $0, 512(%rdx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf3,0xf5,0x8a,0x55,0x4a,0x40,0x00]
+vfixupimmsd $0, 512(%rdx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfixupimmsd $0, (%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x73,0x85,0x08,0x55,0x3a,0x00]
+vfixupimmsd $0, (%rdx), %xmm15, %xmm15
+
+// CHECK: vfixupimmsd $0, (%rdx), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x73,0x85,0x0a,0x55,0x3a,0x00]
+vfixupimmsd $0, (%rdx), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfixupimmsd $0, (%rdx), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x73,0x85,0x8a,0x55,0x3a,0x00]
+vfixupimmsd $0, (%rdx), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfixupimmsd $0, (%rdx), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf3,0xf5,0x08,0x55,0x0a,0x00]
+vfixupimmsd $0, (%rdx), %xmm1, %xmm1
+
+// CHECK: vfixupimmsd $0, (%rdx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf3,0xf5,0x0a,0x55,0x0a,0x00]
+vfixupimmsd $0, (%rdx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfixupimmsd $0, (%rdx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf3,0xf5,0x8a,0x55,0x0a,0x00]
+vfixupimmsd $0, (%rdx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfixupimmsd $0, {sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x53,0x85,0x18,0x55,0xff,0x00]
+vfixupimmsd $0, {sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vfixupimmsd $0, {sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x53,0x85,0x1a,0x55,0xff,0x00]
+vfixupimmsd $0, {sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfixupimmsd $0, {sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x53,0x85,0x9a,0x55,0xff,0x00]
+vfixupimmsd $0, {sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfixupimmsd $0, {sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf3,0xf5,0x18,0x55,0xc9,0x00]
+vfixupimmsd $0, {sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfixupimmsd $0, {sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf3,0xf5,0x1a,0x55,0xc9,0x00]
+vfixupimmsd $0, {sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfixupimmsd $0, {sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf3,0xf5,0x9a,0x55,0xc9,0x00]
+vfixupimmsd $0, {sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfixupimmsd $0, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x53,0x85,0x08,0x55,0xff,0x00]
+vfixupimmsd $0, %xmm15, %xmm15, %xmm15
+
+// CHECK: vfixupimmsd $0, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x53,0x85,0x0a,0x55,0xff,0x00]
+vfixupimmsd $0, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfixupimmsd $0, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x53,0x85,0x8a,0x55,0xff,0x00]
+vfixupimmsd $0, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfixupimmsd $0, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf3,0xf5,0x08,0x55,0xc9,0x00]
+vfixupimmsd $0, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfixupimmsd $0, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf3,0xf5,0x0a,0x55,0xc9,0x00]
+vfixupimmsd $0, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfixupimmsd $0, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf3,0xf5,0x8a,0x55,0xc9,0x00]
+vfixupimmsd $0, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfixupimmss $0, 256(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x73,0x05,0x08,0x55,0x7c,0x82,0x40,0x00]
+vfixupimmss $0, 256(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vfixupimmss $0, -256(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x73,0x05,0x08,0x55,0x7c,0x82,0xc0,0x00]
+vfixupimmss $0, -256(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vfixupimmss $0, 256(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x73,0x05,0x0a,0x55,0x7c,0x82,0x40,0x00]
+vfixupimmss $0, 256(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfixupimmss $0, -256(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x73,0x05,0x0a,0x55,0x7c,0x82,0xc0,0x00]
+vfixupimmss $0, -256(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfixupimmss $0, 256(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x73,0x05,0x8a,0x55,0x7c,0x82,0x40,0x00]
+vfixupimmss $0, 256(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfixupimmss $0, -256(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x73,0x05,0x8a,0x55,0x7c,0x82,0xc0,0x00]
+vfixupimmss $0, -256(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfixupimmss $0, 256(%rdx,%rax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf3,0x75,0x08,0x55,0x4c,0x82,0x40,0x00]
+vfixupimmss $0, 256(%rdx,%rax,4), %xmm1, %xmm1
+
+// CHECK: vfixupimmss $0, -256(%rdx,%rax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf3,0x75,0x08,0x55,0x4c,0x82,0xc0,0x00]
+vfixupimmss $0, -256(%rdx,%rax,4), %xmm1, %xmm1
+
+// CHECK: vfixupimmss $0, 256(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf3,0x75,0x0a,0x55,0x4c,0x82,0x40,0x00]
+vfixupimmss $0, 256(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfixupimmss $0, -256(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf3,0x75,0x0a,0x55,0x4c,0x82,0xc0,0x00]
+vfixupimmss $0, -256(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfixupimmss $0, 256(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf3,0x75,0x8a,0x55,0x4c,0x82,0x40,0x00]
+vfixupimmss $0, 256(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfixupimmss $0, -256(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf3,0x75,0x8a,0x55,0x4c,0x82,0xc0,0x00]
+vfixupimmss $0, -256(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfixupimmss $0, 256(%rdx,%rax), %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x73,0x05,0x08,0x55,0x7c,0x02,0x40,0x00]
+vfixupimmss $0, 256(%rdx,%rax), %xmm15, %xmm15
+
+// CHECK: vfixupimmss $0, 256(%rdx,%rax), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x73,0x05,0x0a,0x55,0x7c,0x02,0x40,0x00]
+vfixupimmss $0, 256(%rdx,%rax), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfixupimmss $0, 256(%rdx,%rax), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x73,0x05,0x8a,0x55,0x7c,0x02,0x40,0x00]
+vfixupimmss $0, 256(%rdx,%rax), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfixupimmss $0, 256(%rdx,%rax), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf3,0x75,0x08,0x55,0x4c,0x02,0x40,0x00]
+vfixupimmss $0, 256(%rdx,%rax), %xmm1, %xmm1
+
+// CHECK: vfixupimmss $0, 256(%rdx,%rax), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf3,0x75,0x0a,0x55,0x4c,0x02,0x40,0x00]
+vfixupimmss $0, 256(%rdx,%rax), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfixupimmss $0, 256(%rdx,%rax), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf3,0x75,0x8a,0x55,0x4c,0x02,0x40,0x00]
+vfixupimmss $0, 256(%rdx,%rax), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfixupimmss $0, 256(%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x73,0x05,0x08,0x55,0x7a,0x40,0x00]
+vfixupimmss $0, 256(%rdx), %xmm15, %xmm15
+
+// CHECK: vfixupimmss $0, 256(%rdx), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x73,0x05,0x0a,0x55,0x7a,0x40,0x00]
+vfixupimmss $0, 256(%rdx), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfixupimmss $0, 256(%rdx), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x73,0x05,0x8a,0x55,0x7a,0x40,0x00]
+vfixupimmss $0, 256(%rdx), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfixupimmss $0, 256(%rdx), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf3,0x75,0x08,0x55,0x4a,0x40,0x00]
+vfixupimmss $0, 256(%rdx), %xmm1, %xmm1
+
+// CHECK: vfixupimmss $0, 256(%rdx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf3,0x75,0x0a,0x55,0x4a,0x40,0x00]
+vfixupimmss $0, 256(%rdx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfixupimmss $0, 256(%rdx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf3,0x75,0x8a,0x55,0x4a,0x40,0x00]
+vfixupimmss $0, 256(%rdx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfixupimmss $0, 485498096, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x73,0x05,0x08,0x55,0x3c,0x25,0xf0,0x1c,0xf0,0x1c,0x00]
+vfixupimmss $0, 485498096, %xmm15, %xmm15
+
+// CHECK: vfixupimmss $0, 485498096, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x73,0x05,0x0a,0x55,0x3c,0x25,0xf0,0x1c,0xf0,0x1c,0x00]
+vfixupimmss $0, 485498096, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfixupimmss $0, 485498096, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x73,0x05,0x8a,0x55,0x3c,0x25,0xf0,0x1c,0xf0,0x1c,0x00]
+vfixupimmss $0, 485498096, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfixupimmss $0, 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf3,0x75,0x08,0x55,0x0c,0x25,0xf0,0x1c,0xf0,0x1c,0x00]
+vfixupimmss $0, 485498096, %xmm1, %xmm1
+
+// CHECK: vfixupimmss $0, 485498096, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf3,0x75,0x0a,0x55,0x0c,0x25,0xf0,0x1c,0xf0,0x1c,0x00]
+vfixupimmss $0, 485498096, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfixupimmss $0, 485498096, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf3,0x75,0x8a,0x55,0x0c,0x25,0xf0,0x1c,0xf0,0x1c,0x00]
+vfixupimmss $0, 485498096, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfixupimmss $0, (%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x73,0x05,0x08,0x55,0x3a,0x00]
+vfixupimmss $0, (%rdx), %xmm15, %xmm15
+
+// CHECK: vfixupimmss $0, (%rdx), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x73,0x05,0x0a,0x55,0x3a,0x00]
+vfixupimmss $0, (%rdx), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfixupimmss $0, (%rdx), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x73,0x05,0x8a,0x55,0x3a,0x00]
+vfixupimmss $0, (%rdx), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfixupimmss $0, (%rdx), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf3,0x75,0x08,0x55,0x0a,0x00]
+vfixupimmss $0, (%rdx), %xmm1, %xmm1
+
+// CHECK: vfixupimmss $0, (%rdx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf3,0x75,0x0a,0x55,0x0a,0x00]
+vfixupimmss $0, (%rdx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfixupimmss $0, (%rdx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf3,0x75,0x8a,0x55,0x0a,0x00]
+vfixupimmss $0, (%rdx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfixupimmss $0, {sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x53,0x05,0x18,0x55,0xff,0x00]
+vfixupimmss $0, {sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vfixupimmss $0, {sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x53,0x05,0x1a,0x55,0xff,0x00]
+vfixupimmss $0, {sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfixupimmss $0, {sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x53,0x05,0x9a,0x55,0xff,0x00]
+vfixupimmss $0, {sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfixupimmss $0, {sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf3,0x75,0x18,0x55,0xc9,0x00]
+vfixupimmss $0, {sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfixupimmss $0, {sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf3,0x75,0x1a,0x55,0xc9,0x00]
+vfixupimmss $0, {sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfixupimmss $0, {sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf3,0x75,0x9a,0x55,0xc9,0x00]
+vfixupimmss $0, {sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfixupimmss $0, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x53,0x05,0x08,0x55,0xff,0x00]
+vfixupimmss $0, %xmm15, %xmm15, %xmm15
+
+// CHECK: vfixupimmss $0, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x53,0x05,0x0a,0x55,0xff,0x00]
+vfixupimmss $0, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfixupimmss $0, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x53,0x05,0x8a,0x55,0xff,0x00]
+vfixupimmss $0, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfixupimmss $0, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf3,0x75,0x08,0x55,0xc9,0x00]
+vfixupimmss $0, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfixupimmss $0, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf3,0x75,0x0a,0x55,0xc9,0x00]
+vfixupimmss $0, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfixupimmss $0, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf3,0x75,0x8a,0x55,0xc9,0x00]
+vfixupimmss $0, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd132sd 485498096, %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x81,0x99,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfmadd132sd 485498096, %xmm15, %xmm15
+
+// CHECK: vfmadd132sd 485498096, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x85,0x0a,0x99,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfmadd132sd 485498096, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmadd132sd 485498096, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x85,0x8a,0x99,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfmadd132sd 485498096, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmadd132sd 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0x99,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfmadd132sd 485498096, %xmm1, %xmm1
+
+// CHECK: vfmadd132sd 485498096, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0x99,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfmadd132sd 485498096, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd132sd 485498096, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0x99,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfmadd132sd 485498096, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd132sd 512(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x81,0x99,0xbc,0x82,0x00,0x02,0x00,0x00]
+vfmadd132sd 512(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vfmadd132sd -512(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x81,0x99,0xbc,0x82,0x00,0xfe,0xff,0xff]
+vfmadd132sd -512(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vfmadd132sd 512(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x85,0x0a,0x99,0x7c,0x82,0x40]
+vfmadd132sd 512(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmadd132sd -512(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x85,0x0a,0x99,0x7c,0x82,0xc0]
+vfmadd132sd -512(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmadd132sd 512(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x85,0x8a,0x99,0x7c,0x82,0x40]
+vfmadd132sd 512(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmadd132sd -512(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x85,0x8a,0x99,0x7c,0x82,0xc0]
+vfmadd132sd -512(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmadd132sd 512(%rdx,%rax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0x99,0x8c,0x82,0x00,0x02,0x00,0x00]
+vfmadd132sd 512(%rdx,%rax,4), %xmm1, %xmm1
+
+// CHECK: vfmadd132sd -512(%rdx,%rax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0x99,0x8c,0x82,0x00,0xfe,0xff,0xff]
+vfmadd132sd -512(%rdx,%rax,4), %xmm1, %xmm1
+
+// CHECK: vfmadd132sd 512(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0x99,0x4c,0x82,0x40]
+vfmadd132sd 512(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd132sd -512(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0x99,0x4c,0x82,0xc0]
+vfmadd132sd -512(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd132sd 512(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0x99,0x4c,0x82,0x40]
+vfmadd132sd 512(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd132sd -512(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0x99,0x4c,0x82,0xc0]
+vfmadd132sd -512(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd132sd 512(%rdx,%rax), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x81,0x99,0xbc,0x02,0x00,0x02,0x00,0x00]
+vfmadd132sd 512(%rdx,%rax), %xmm15, %xmm15
+
+// CHECK: vfmadd132sd 512(%rdx,%rax), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x85,0x0a,0x99,0x7c,0x02,0x40]
+vfmadd132sd 512(%rdx,%rax), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmadd132sd 512(%rdx,%rax), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x85,0x8a,0x99,0x7c,0x02,0x40]
+vfmadd132sd 512(%rdx,%rax), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmadd132sd 512(%rdx,%rax), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0x99,0x8c,0x02,0x00,0x02,0x00,0x00]
+vfmadd132sd 512(%rdx,%rax), %xmm1, %xmm1
+
+// CHECK: vfmadd132sd 512(%rdx,%rax), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0x99,0x4c,0x02,0x40]
+vfmadd132sd 512(%rdx,%rax), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd132sd 512(%rdx,%rax), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0x99,0x4c,0x02,0x40]
+vfmadd132sd 512(%rdx,%rax), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd132sd 512(%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x81,0x99,0xba,0x00,0x02,0x00,0x00]
+vfmadd132sd 512(%rdx), %xmm15, %xmm15
+
+// CHECK: vfmadd132sd 512(%rdx), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x85,0x0a,0x99,0x7a,0x40]
+vfmadd132sd 512(%rdx), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmadd132sd 512(%rdx), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x85,0x8a,0x99,0x7a,0x40]
+vfmadd132sd 512(%rdx), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmadd132sd 512(%rdx), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0x99,0x8a,0x00,0x02,0x00,0x00]
+vfmadd132sd 512(%rdx), %xmm1, %xmm1
+
+// CHECK: vfmadd132sd 512(%rdx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0x99,0x4a,0x40]
+vfmadd132sd 512(%rdx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd132sd 512(%rdx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0x99,0x4a,0x40]
+vfmadd132sd 512(%rdx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd132sd {rd-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x52,0x85,0x38,0x99,0xff]
+vfmadd132sd {rd-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vfmadd132sd {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x85,0x3a,0x99,0xff]
+vfmadd132sd {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmadd132sd {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x85,0xba,0x99,0xff]
+vfmadd132sd {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmadd132sd {rd-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x38,0x99,0xc9]
+vfmadd132sd {rd-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfmadd132sd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x3a,0x99,0xc9]
+vfmadd132sd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd132sd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0xba,0x99,0xc9]
+vfmadd132sd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd132sd (%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x81,0x99,0x3a]
+vfmadd132sd (%rdx), %xmm15, %xmm15
+
+// CHECK: vfmadd132sd (%rdx), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x85,0x0a,0x99,0x3a]
+vfmadd132sd (%rdx), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmadd132sd (%rdx), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x85,0x8a,0x99,0x3a]
+vfmadd132sd (%rdx), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmadd132sd (%rdx), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0x99,0x0a]
+vfmadd132sd (%rdx), %xmm1, %xmm1
+
+// CHECK: vfmadd132sd (%rdx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0x99,0x0a]
+vfmadd132sd (%rdx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd132sd (%rdx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0x99,0x0a]
+vfmadd132sd (%rdx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd132sd {rn-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x52,0x85,0x18,0x99,0xff]
+vfmadd132sd {rn-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vfmadd132sd {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x85,0x1a,0x99,0xff]
+vfmadd132sd {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmadd132sd {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x85,0x9a,0x99,0xff]
+vfmadd132sd {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmadd132sd {rn-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x18,0x99,0xc9]
+vfmadd132sd {rn-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfmadd132sd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x1a,0x99,0xc9]
+vfmadd132sd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd132sd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x9a,0x99,0xc9]
+vfmadd132sd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd132sd {ru-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x52,0x85,0x58,0x99,0xff]
+vfmadd132sd {ru-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vfmadd132sd {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x85,0x5a,0x99,0xff]
+vfmadd132sd {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmadd132sd {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x85,0xda,0x99,0xff]
+vfmadd132sd {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmadd132sd {ru-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x58,0x99,0xc9]
+vfmadd132sd {ru-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfmadd132sd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x5a,0x99,0xc9]
+vfmadd132sd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd132sd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0xda,0x99,0xc9]
+vfmadd132sd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd132sd {rz-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x52,0x85,0x78,0x99,0xff]
+vfmadd132sd {rz-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vfmadd132sd {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x85,0x7a,0x99,0xff]
+vfmadd132sd {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmadd132sd {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x85,0xfa,0x99,0xff]
+vfmadd132sd {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmadd132sd {rz-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x78,0x99,0xc9]
+vfmadd132sd {rz-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfmadd132sd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x7a,0x99,0xc9]
+vfmadd132sd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd132sd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0xfa,0x99,0xc9]
+vfmadd132sd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd132sd %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x42,0x81,0x99,0xff]
+vfmadd132sd %xmm15, %xmm15, %xmm15
+
+// CHECK: vfmadd132sd %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x85,0x0a,0x99,0xff]
+vfmadd132sd %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmadd132sd %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x85,0x8a,0x99,0xff]
+vfmadd132sd %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmadd132sd %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0x99,0xc9]
+vfmadd132sd %xmm1, %xmm1, %xmm1
+
+// CHECK: vfmadd132sd %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0x99,0xc9]
+vfmadd132sd %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd132sd %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0x99,0xc9]
+vfmadd132sd %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd132ss 256(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x01,0x99,0xbc,0x82,0x00,0x01,0x00,0x00]
+vfmadd132ss 256(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vfmadd132ss -256(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x01,0x99,0xbc,0x82,0x00,0xff,0xff,0xff]
+vfmadd132ss -256(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vfmadd132ss 256(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x05,0x0a,0x99,0x7c,0x82,0x40]
+vfmadd132ss 256(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmadd132ss -256(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x05,0x0a,0x99,0x7c,0x82,0xc0]
+vfmadd132ss -256(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmadd132ss 256(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x05,0x8a,0x99,0x7c,0x82,0x40]
+vfmadd132ss 256(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmadd132ss -256(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x05,0x8a,0x99,0x7c,0x82,0xc0]
+vfmadd132ss -256(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmadd132ss 256(%rdx,%rax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0x99,0x8c,0x82,0x00,0x01,0x00,0x00]
+vfmadd132ss 256(%rdx,%rax,4), %xmm1, %xmm1
+
+// CHECK: vfmadd132ss -256(%rdx,%rax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0x99,0x8c,0x82,0x00,0xff,0xff,0xff]
+vfmadd132ss -256(%rdx,%rax,4), %xmm1, %xmm1
+
+// CHECK: vfmadd132ss 256(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0x99,0x4c,0x82,0x40]
+vfmadd132ss 256(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd132ss -256(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0x99,0x4c,0x82,0xc0]
+vfmadd132ss -256(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd132ss 256(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0x99,0x4c,0x82,0x40]
+vfmadd132ss 256(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd132ss -256(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0x99,0x4c,0x82,0xc0]
+vfmadd132ss -256(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd132ss 256(%rdx,%rax), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x01,0x99,0xbc,0x02,0x00,0x01,0x00,0x00]
+vfmadd132ss 256(%rdx,%rax), %xmm15, %xmm15
+
+// CHECK: vfmadd132ss 256(%rdx,%rax), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x05,0x0a,0x99,0x7c,0x02,0x40]
+vfmadd132ss 256(%rdx,%rax), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmadd132ss 256(%rdx,%rax), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x05,0x8a,0x99,0x7c,0x02,0x40]
+vfmadd132ss 256(%rdx,%rax), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmadd132ss 256(%rdx,%rax), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0x99,0x8c,0x02,0x00,0x01,0x00,0x00]
+vfmadd132ss 256(%rdx,%rax), %xmm1, %xmm1
+
+// CHECK: vfmadd132ss 256(%rdx,%rax), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0x99,0x4c,0x02,0x40]
+vfmadd132ss 256(%rdx,%rax), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd132ss 256(%rdx,%rax), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0x99,0x4c,0x02,0x40]
+vfmadd132ss 256(%rdx,%rax), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd132ss 256(%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x01,0x99,0xba,0x00,0x01,0x00,0x00]
+vfmadd132ss 256(%rdx), %xmm15, %xmm15
+
+// CHECK: vfmadd132ss 256(%rdx), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x05,0x0a,0x99,0x7a,0x40]
+vfmadd132ss 256(%rdx), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmadd132ss 256(%rdx), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x05,0x8a,0x99,0x7a,0x40]
+vfmadd132ss 256(%rdx), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmadd132ss 256(%rdx), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0x99,0x8a,0x00,0x01,0x00,0x00]
+vfmadd132ss 256(%rdx), %xmm1, %xmm1
+
+// CHECK: vfmadd132ss 256(%rdx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0x99,0x4a,0x40]
+vfmadd132ss 256(%rdx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd132ss 256(%rdx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0x99,0x4a,0x40]
+vfmadd132ss 256(%rdx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd132ss 485498096, %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x01,0x99,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfmadd132ss 485498096, %xmm15, %xmm15
+
+// CHECK: vfmadd132ss 485498096, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x05,0x0a,0x99,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfmadd132ss 485498096, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmadd132ss 485498096, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x05,0x8a,0x99,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfmadd132ss 485498096, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmadd132ss 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0x99,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfmadd132ss 485498096, %xmm1, %xmm1
+
+// CHECK: vfmadd132ss 485498096, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0x99,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfmadd132ss 485498096, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd132ss 485498096, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0x99,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfmadd132ss 485498096, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd132ss {rd-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x52,0x05,0x38,0x99,0xff]
+vfmadd132ss {rd-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vfmadd132ss {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x05,0x3a,0x99,0xff]
+vfmadd132ss {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmadd132ss {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x05,0xba,0x99,0xff]
+vfmadd132ss {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmadd132ss {rd-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x38,0x99,0xc9]
+vfmadd132ss {rd-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfmadd132ss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x3a,0x99,0xc9]
+vfmadd132ss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd132ss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0xba,0x99,0xc9]
+vfmadd132ss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd132ss (%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x01,0x99,0x3a]
+vfmadd132ss (%rdx), %xmm15, %xmm15
+
+// CHECK: vfmadd132ss (%rdx), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x05,0x0a,0x99,0x3a]
+vfmadd132ss (%rdx), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmadd132ss (%rdx), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x05,0x8a,0x99,0x3a]
+vfmadd132ss (%rdx), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmadd132ss (%rdx), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0x99,0x0a]
+vfmadd132ss (%rdx), %xmm1, %xmm1
+
+// CHECK: vfmadd132ss (%rdx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0x99,0x0a]
+vfmadd132ss (%rdx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd132ss (%rdx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0x99,0x0a]
+vfmadd132ss (%rdx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd132ss {rn-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x52,0x05,0x18,0x99,0xff]
+vfmadd132ss {rn-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vfmadd132ss {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x05,0x1a,0x99,0xff]
+vfmadd132ss {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmadd132ss {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x05,0x9a,0x99,0xff]
+vfmadd132ss {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmadd132ss {rn-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x18,0x99,0xc9]
+vfmadd132ss {rn-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfmadd132ss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x1a,0x99,0xc9]
+vfmadd132ss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd132ss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x9a,0x99,0xc9]
+vfmadd132ss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd132ss {ru-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x52,0x05,0x58,0x99,0xff]
+vfmadd132ss {ru-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vfmadd132ss {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x05,0x5a,0x99,0xff]
+vfmadd132ss {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmadd132ss {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x05,0xda,0x99,0xff]
+vfmadd132ss {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmadd132ss {ru-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x58,0x99,0xc9]
+vfmadd132ss {ru-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfmadd132ss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x5a,0x99,0xc9]
+vfmadd132ss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd132ss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0xda,0x99,0xc9]
+vfmadd132ss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd132ss {rz-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x52,0x05,0x78,0x99,0xff]
+vfmadd132ss {rz-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vfmadd132ss {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x05,0x7a,0x99,0xff]
+vfmadd132ss {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmadd132ss {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x05,0xfa,0x99,0xff]
+vfmadd132ss {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmadd132ss {rz-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x78,0x99,0xc9]
+vfmadd132ss {rz-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfmadd132ss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x7a,0x99,0xc9]
+vfmadd132ss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd132ss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0xfa,0x99,0xc9]
+vfmadd132ss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd132ss %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x42,0x01,0x99,0xff]
+vfmadd132ss %xmm15, %xmm15, %xmm15
+
+// CHECK: vfmadd132ss %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x05,0x0a,0x99,0xff]
+vfmadd132ss %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmadd132ss %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x05,0x8a,0x99,0xff]
+vfmadd132ss %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmadd132ss %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0x99,0xc9]
+vfmadd132ss %xmm1, %xmm1, %xmm1
+
+// CHECK: vfmadd132ss %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0x99,0xc9]
+vfmadd132ss %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd132ss %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0x99,0xc9]
+vfmadd132ss %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd213sd 485498096, %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x81,0xa9,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfmadd213sd 485498096, %xmm15, %xmm15
+
+// CHECK: vfmadd213sd 485498096, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x85,0x0a,0xa9,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfmadd213sd 485498096, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmadd213sd 485498096, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x85,0x8a,0xa9,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfmadd213sd 485498096, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmadd213sd 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0xa9,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfmadd213sd 485498096, %xmm1, %xmm1
+
+// CHECK: vfmadd213sd 485498096, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0xa9,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfmadd213sd 485498096, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd213sd 485498096, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0xa9,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfmadd213sd 485498096, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd213sd 512(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x81,0xa9,0xbc,0x82,0x00,0x02,0x00,0x00]
+vfmadd213sd 512(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vfmadd213sd -512(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x81,0xa9,0xbc,0x82,0x00,0xfe,0xff,0xff]
+vfmadd213sd -512(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vfmadd213sd 512(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x85,0x0a,0xa9,0x7c,0x82,0x40]
+vfmadd213sd 512(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmadd213sd -512(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x85,0x0a,0xa9,0x7c,0x82,0xc0]
+vfmadd213sd -512(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmadd213sd 512(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x85,0x8a,0xa9,0x7c,0x82,0x40]
+vfmadd213sd 512(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmadd213sd -512(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x85,0x8a,0xa9,0x7c,0x82,0xc0]
+vfmadd213sd -512(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmadd213sd 512(%rdx,%rax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0xa9,0x8c,0x82,0x00,0x02,0x00,0x00]
+vfmadd213sd 512(%rdx,%rax,4), %xmm1, %xmm1
+
+// CHECK: vfmadd213sd -512(%rdx,%rax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0xa9,0x8c,0x82,0x00,0xfe,0xff,0xff]
+vfmadd213sd -512(%rdx,%rax,4), %xmm1, %xmm1
+
+// CHECK: vfmadd213sd 512(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0xa9,0x4c,0x82,0x40]
+vfmadd213sd 512(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd213sd -512(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0xa9,0x4c,0x82,0xc0]
+vfmadd213sd -512(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd213sd 512(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0xa9,0x4c,0x82,0x40]
+vfmadd213sd 512(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd213sd -512(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0xa9,0x4c,0x82,0xc0]
+vfmadd213sd -512(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd213sd 512(%rdx,%rax), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x81,0xa9,0xbc,0x02,0x00,0x02,0x00,0x00]
+vfmadd213sd 512(%rdx,%rax), %xmm15, %xmm15
+
+// CHECK: vfmadd213sd 512(%rdx,%rax), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x85,0x0a,0xa9,0x7c,0x02,0x40]
+vfmadd213sd 512(%rdx,%rax), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmadd213sd 512(%rdx,%rax), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x85,0x8a,0xa9,0x7c,0x02,0x40]
+vfmadd213sd 512(%rdx,%rax), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmadd213sd 512(%rdx,%rax), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0xa9,0x8c,0x02,0x00,0x02,0x00,0x00]
+vfmadd213sd 512(%rdx,%rax), %xmm1, %xmm1
+
+// CHECK: vfmadd213sd 512(%rdx,%rax), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0xa9,0x4c,0x02,0x40]
+vfmadd213sd 512(%rdx,%rax), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd213sd 512(%rdx,%rax), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0xa9,0x4c,0x02,0x40]
+vfmadd213sd 512(%rdx,%rax), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd213sd 512(%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x81,0xa9,0xba,0x00,0x02,0x00,0x00]
+vfmadd213sd 512(%rdx), %xmm15, %xmm15
+
+// CHECK: vfmadd213sd 512(%rdx), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x85,0x0a,0xa9,0x7a,0x40]
+vfmadd213sd 512(%rdx), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmadd213sd 512(%rdx), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x85,0x8a,0xa9,0x7a,0x40]
+vfmadd213sd 512(%rdx), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmadd213sd 512(%rdx), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0xa9,0x8a,0x00,0x02,0x00,0x00]
+vfmadd213sd 512(%rdx), %xmm1, %xmm1
+
+// CHECK: vfmadd213sd 512(%rdx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0xa9,0x4a,0x40]
+vfmadd213sd 512(%rdx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd213sd 512(%rdx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0xa9,0x4a,0x40]
+vfmadd213sd 512(%rdx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd213sd {rd-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x52,0x85,0x38,0xa9,0xff]
+vfmadd213sd {rd-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vfmadd213sd {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x85,0x3a,0xa9,0xff]
+vfmadd213sd {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmadd213sd {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x85,0xba,0xa9,0xff]
+vfmadd213sd {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmadd213sd {rd-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x38,0xa9,0xc9]
+vfmadd213sd {rd-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfmadd213sd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x3a,0xa9,0xc9]
+vfmadd213sd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd213sd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0xba,0xa9,0xc9]
+vfmadd213sd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd213sd (%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x81,0xa9,0x3a]
+vfmadd213sd (%rdx), %xmm15, %xmm15
+
+// CHECK: vfmadd213sd (%rdx), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x85,0x0a,0xa9,0x3a]
+vfmadd213sd (%rdx), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmadd213sd (%rdx), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x85,0x8a,0xa9,0x3a]
+vfmadd213sd (%rdx), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmadd213sd (%rdx), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0xa9,0x0a]
+vfmadd213sd (%rdx), %xmm1, %xmm1
+
+// CHECK: vfmadd213sd (%rdx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0xa9,0x0a]
+vfmadd213sd (%rdx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd213sd (%rdx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0xa9,0x0a]
+vfmadd213sd (%rdx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd213sd {rn-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x52,0x85,0x18,0xa9,0xff]
+vfmadd213sd {rn-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vfmadd213sd {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x85,0x1a,0xa9,0xff]
+vfmadd213sd {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmadd213sd {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x85,0x9a,0xa9,0xff]
+vfmadd213sd {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmadd213sd {rn-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x18,0xa9,0xc9]
+vfmadd213sd {rn-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfmadd213sd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x1a,0xa9,0xc9]
+vfmadd213sd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd213sd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x9a,0xa9,0xc9]
+vfmadd213sd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd213sd {ru-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x52,0x85,0x58,0xa9,0xff]
+vfmadd213sd {ru-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vfmadd213sd {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x85,0x5a,0xa9,0xff]
+vfmadd213sd {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmadd213sd {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x85,0xda,0xa9,0xff]
+vfmadd213sd {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmadd213sd {ru-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x58,0xa9,0xc9]
+vfmadd213sd {ru-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfmadd213sd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x5a,0xa9,0xc9]
+vfmadd213sd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd213sd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0xda,0xa9,0xc9]
+vfmadd213sd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd213sd {rz-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x52,0x85,0x78,0xa9,0xff]
+vfmadd213sd {rz-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vfmadd213sd {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x85,0x7a,0xa9,0xff]
+vfmadd213sd {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmadd213sd {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x85,0xfa,0xa9,0xff]
+vfmadd213sd {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmadd213sd {rz-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x78,0xa9,0xc9]
+vfmadd213sd {rz-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfmadd213sd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x7a,0xa9,0xc9]
+vfmadd213sd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd213sd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0xfa,0xa9,0xc9]
+vfmadd213sd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd213sd %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x42,0x81,0xa9,0xff]
+vfmadd213sd %xmm15, %xmm15, %xmm15
+
+// CHECK: vfmadd213sd %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x85,0x0a,0xa9,0xff]
+vfmadd213sd %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmadd213sd %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x85,0x8a,0xa9,0xff]
+vfmadd213sd %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmadd213sd %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0xa9,0xc9]
+vfmadd213sd %xmm1, %xmm1, %xmm1
+
+// CHECK: vfmadd213sd %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0xa9,0xc9]
+vfmadd213sd %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd213sd %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0xa9,0xc9]
+vfmadd213sd %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd213ss 256(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x01,0xa9,0xbc,0x82,0x00,0x01,0x00,0x00]
+vfmadd213ss 256(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vfmadd213ss -256(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x01,0xa9,0xbc,0x82,0x00,0xff,0xff,0xff]
+vfmadd213ss -256(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vfmadd213ss 256(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x05,0x0a,0xa9,0x7c,0x82,0x40]
+vfmadd213ss 256(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmadd213ss -256(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x05,0x0a,0xa9,0x7c,0x82,0xc0]
+vfmadd213ss -256(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmadd213ss 256(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x05,0x8a,0xa9,0x7c,0x82,0x40]
+vfmadd213ss 256(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmadd213ss -256(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x05,0x8a,0xa9,0x7c,0x82,0xc0]
+vfmadd213ss -256(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmadd213ss 256(%rdx,%rax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0xa9,0x8c,0x82,0x00,0x01,0x00,0x00]
+vfmadd213ss 256(%rdx,%rax,4), %xmm1, %xmm1
+
+// CHECK: vfmadd213ss -256(%rdx,%rax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0xa9,0x8c,0x82,0x00,0xff,0xff,0xff]
+vfmadd213ss -256(%rdx,%rax,4), %xmm1, %xmm1
+
+// CHECK: vfmadd213ss 256(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0xa9,0x4c,0x82,0x40]
+vfmadd213ss 256(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd213ss -256(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0xa9,0x4c,0x82,0xc0]
+vfmadd213ss -256(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd213ss 256(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0xa9,0x4c,0x82,0x40]
+vfmadd213ss 256(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd213ss -256(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0xa9,0x4c,0x82,0xc0]
+vfmadd213ss -256(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd213ss 256(%rdx,%rax), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x01,0xa9,0xbc,0x02,0x00,0x01,0x00,0x00]
+vfmadd213ss 256(%rdx,%rax), %xmm15, %xmm15
+
+// CHECK: vfmadd213ss 256(%rdx,%rax), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x05,0x0a,0xa9,0x7c,0x02,0x40]
+vfmadd213ss 256(%rdx,%rax), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmadd213ss 256(%rdx,%rax), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x05,0x8a,0xa9,0x7c,0x02,0x40]
+vfmadd213ss 256(%rdx,%rax), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmadd213ss 256(%rdx,%rax), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0xa9,0x8c,0x02,0x00,0x01,0x00,0x00]
+vfmadd213ss 256(%rdx,%rax), %xmm1, %xmm1
+
+// CHECK: vfmadd213ss 256(%rdx,%rax), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0xa9,0x4c,0x02,0x40]
+vfmadd213ss 256(%rdx,%rax), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd213ss 256(%rdx,%rax), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0xa9,0x4c,0x02,0x40]
+vfmadd213ss 256(%rdx,%rax), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd213ss 256(%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x01,0xa9,0xba,0x00,0x01,0x00,0x00]
+vfmadd213ss 256(%rdx), %xmm15, %xmm15
+
+// CHECK: vfmadd213ss 256(%rdx), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x05,0x0a,0xa9,0x7a,0x40]
+vfmadd213ss 256(%rdx), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmadd213ss 256(%rdx), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x05,0x8a,0xa9,0x7a,0x40]
+vfmadd213ss 256(%rdx), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmadd213ss 256(%rdx), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0xa9,0x8a,0x00,0x01,0x00,0x00]
+vfmadd213ss 256(%rdx), %xmm1, %xmm1
+
+// CHECK: vfmadd213ss 256(%rdx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0xa9,0x4a,0x40]
+vfmadd213ss 256(%rdx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd213ss 256(%rdx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0xa9,0x4a,0x40]
+vfmadd213ss 256(%rdx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd213ss 485498096, %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x01,0xa9,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfmadd213ss 485498096, %xmm15, %xmm15
+
+// CHECK: vfmadd213ss 485498096, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x05,0x0a,0xa9,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfmadd213ss 485498096, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmadd213ss 485498096, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x05,0x8a,0xa9,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfmadd213ss 485498096, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmadd213ss 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0xa9,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfmadd213ss 485498096, %xmm1, %xmm1
+
+// CHECK: vfmadd213ss 485498096, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0xa9,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfmadd213ss 485498096, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd213ss 485498096, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0xa9,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfmadd213ss 485498096, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd213ss {rd-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x52,0x05,0x38,0xa9,0xff]
+vfmadd213ss {rd-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vfmadd213ss {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x05,0x3a,0xa9,0xff]
+vfmadd213ss {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmadd213ss {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x05,0xba,0xa9,0xff]
+vfmadd213ss {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmadd213ss {rd-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x38,0xa9,0xc9]
+vfmadd213ss {rd-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfmadd213ss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x3a,0xa9,0xc9]
+vfmadd213ss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd213ss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0xba,0xa9,0xc9]
+vfmadd213ss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd213ss (%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x01,0xa9,0x3a]
+vfmadd213ss (%rdx), %xmm15, %xmm15
+
+// CHECK: vfmadd213ss (%rdx), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x05,0x0a,0xa9,0x3a]
+vfmadd213ss (%rdx), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmadd213ss (%rdx), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x05,0x8a,0xa9,0x3a]
+vfmadd213ss (%rdx), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmadd213ss (%rdx), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0xa9,0x0a]
+vfmadd213ss (%rdx), %xmm1, %xmm1
+
+// CHECK: vfmadd213ss (%rdx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0xa9,0x0a]
+vfmadd213ss (%rdx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd213ss (%rdx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0xa9,0x0a]
+vfmadd213ss (%rdx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd213ss {rn-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x52,0x05,0x18,0xa9,0xff]
+vfmadd213ss {rn-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vfmadd213ss {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x05,0x1a,0xa9,0xff]
+vfmadd213ss {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmadd213ss {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x05,0x9a,0xa9,0xff]
+vfmadd213ss {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmadd213ss {rn-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x18,0xa9,0xc9]
+vfmadd213ss {rn-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfmadd213ss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x1a,0xa9,0xc9]
+vfmadd213ss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd213ss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x9a,0xa9,0xc9]
+vfmadd213ss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd213ss {ru-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x52,0x05,0x58,0xa9,0xff]
+vfmadd213ss {ru-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vfmadd213ss {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x05,0x5a,0xa9,0xff]
+vfmadd213ss {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmadd213ss {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x05,0xda,0xa9,0xff]
+vfmadd213ss {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmadd213ss {ru-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x58,0xa9,0xc9]
+vfmadd213ss {ru-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfmadd213ss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x5a,0xa9,0xc9]
+vfmadd213ss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd213ss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0xda,0xa9,0xc9]
+vfmadd213ss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd213ss {rz-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x52,0x05,0x78,0xa9,0xff]
+vfmadd213ss {rz-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vfmadd213ss {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x05,0x7a,0xa9,0xff]
+vfmadd213ss {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmadd213ss {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x05,0xfa,0xa9,0xff]
+vfmadd213ss {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmadd213ss {rz-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x78,0xa9,0xc9]
+vfmadd213ss {rz-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfmadd213ss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x7a,0xa9,0xc9]
+vfmadd213ss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd213ss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0xfa,0xa9,0xc9]
+vfmadd213ss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd213ss %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x42,0x01,0xa9,0xff]
+vfmadd213ss %xmm15, %xmm15, %xmm15
+
+// CHECK: vfmadd213ss %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x05,0x0a,0xa9,0xff]
+vfmadd213ss %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmadd213ss %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x05,0x8a,0xa9,0xff]
+vfmadd213ss %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmadd213ss %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0xa9,0xc9]
+vfmadd213ss %xmm1, %xmm1, %xmm1
+
+// CHECK: vfmadd213ss %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0xa9,0xc9]
+vfmadd213ss %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd213ss %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0xa9,0xc9]
+vfmadd213ss %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd231sd 485498096, %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x81,0xb9,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfmadd231sd 485498096, %xmm15, %xmm15
+
+// CHECK: vfmadd231sd 485498096, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x85,0x0a,0xb9,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfmadd231sd 485498096, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmadd231sd 485498096, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x85,0x8a,0xb9,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfmadd231sd 485498096, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmadd231sd 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0xb9,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfmadd231sd 485498096, %xmm1, %xmm1
+
+// CHECK: vfmadd231sd 485498096, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0xb9,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfmadd231sd 485498096, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd231sd 485498096, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0xb9,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfmadd231sd 485498096, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd231sd 512(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x81,0xb9,0xbc,0x82,0x00,0x02,0x00,0x00]
+vfmadd231sd 512(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vfmadd231sd -512(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x81,0xb9,0xbc,0x82,0x00,0xfe,0xff,0xff]
+vfmadd231sd -512(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vfmadd231sd 512(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x85,0x0a,0xb9,0x7c,0x82,0x40]
+vfmadd231sd 512(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmadd231sd -512(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x85,0x0a,0xb9,0x7c,0x82,0xc0]
+vfmadd231sd -512(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmadd231sd 512(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x85,0x8a,0xb9,0x7c,0x82,0x40]
+vfmadd231sd 512(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmadd231sd -512(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x85,0x8a,0xb9,0x7c,0x82,0xc0]
+vfmadd231sd -512(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmadd231sd 512(%rdx,%rax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0xb9,0x8c,0x82,0x00,0x02,0x00,0x00]
+vfmadd231sd 512(%rdx,%rax,4), %xmm1, %xmm1
+
+// CHECK: vfmadd231sd -512(%rdx,%rax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0xb9,0x8c,0x82,0x00,0xfe,0xff,0xff]
+vfmadd231sd -512(%rdx,%rax,4), %xmm1, %xmm1
+
+// CHECK: vfmadd231sd 512(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0xb9,0x4c,0x82,0x40]
+vfmadd231sd 512(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd231sd -512(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0xb9,0x4c,0x82,0xc0]
+vfmadd231sd -512(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd231sd 512(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0xb9,0x4c,0x82,0x40]
+vfmadd231sd 512(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd231sd -512(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0xb9,0x4c,0x82,0xc0]
+vfmadd231sd -512(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd231sd 512(%rdx,%rax), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x81,0xb9,0xbc,0x02,0x00,0x02,0x00,0x00]
+vfmadd231sd 512(%rdx,%rax), %xmm15, %xmm15
+
+// CHECK: vfmadd231sd 512(%rdx,%rax), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x85,0x0a,0xb9,0x7c,0x02,0x40]
+vfmadd231sd 512(%rdx,%rax), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmadd231sd 512(%rdx,%rax), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x85,0x8a,0xb9,0x7c,0x02,0x40]
+vfmadd231sd 512(%rdx,%rax), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmadd231sd 512(%rdx,%rax), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0xb9,0x8c,0x02,0x00,0x02,0x00,0x00]
+vfmadd231sd 512(%rdx,%rax), %xmm1, %xmm1
+
+// CHECK: vfmadd231sd 512(%rdx,%rax), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0xb9,0x4c,0x02,0x40]
+vfmadd231sd 512(%rdx,%rax), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd231sd 512(%rdx,%rax), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0xb9,0x4c,0x02,0x40]
+vfmadd231sd 512(%rdx,%rax), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd231sd 512(%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x81,0xb9,0xba,0x00,0x02,0x00,0x00]
+vfmadd231sd 512(%rdx), %xmm15, %xmm15
+
+// CHECK: vfmadd231sd 512(%rdx), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x85,0x0a,0xb9,0x7a,0x40]
+vfmadd231sd 512(%rdx), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmadd231sd 512(%rdx), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x85,0x8a,0xb9,0x7a,0x40]
+vfmadd231sd 512(%rdx), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmadd231sd 512(%rdx), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0xb9,0x8a,0x00,0x02,0x00,0x00]
+vfmadd231sd 512(%rdx), %xmm1, %xmm1
+
+// CHECK: vfmadd231sd 512(%rdx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0xb9,0x4a,0x40]
+vfmadd231sd 512(%rdx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd231sd 512(%rdx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0xb9,0x4a,0x40]
+vfmadd231sd 512(%rdx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd231sd {rd-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x52,0x85,0x38,0xb9,0xff]
+vfmadd231sd {rd-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vfmadd231sd {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x85,0x3a,0xb9,0xff]
+vfmadd231sd {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmadd231sd {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x85,0xba,0xb9,0xff]
+vfmadd231sd {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmadd231sd {rd-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x38,0xb9,0xc9]
+vfmadd231sd {rd-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfmadd231sd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x3a,0xb9,0xc9]
+vfmadd231sd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd231sd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0xba,0xb9,0xc9]
+vfmadd231sd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd231sd (%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x81,0xb9,0x3a]
+vfmadd231sd (%rdx), %xmm15, %xmm15
+
+// CHECK: vfmadd231sd (%rdx), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x85,0x0a,0xb9,0x3a]
+vfmadd231sd (%rdx), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmadd231sd (%rdx), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x85,0x8a,0xb9,0x3a]
+vfmadd231sd (%rdx), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmadd231sd (%rdx), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0xb9,0x0a]
+vfmadd231sd (%rdx), %xmm1, %xmm1
+
+// CHECK: vfmadd231sd (%rdx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0xb9,0x0a]
+vfmadd231sd (%rdx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd231sd (%rdx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0xb9,0x0a]
+vfmadd231sd (%rdx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd231sd {rn-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x52,0x85,0x18,0xb9,0xff]
+vfmadd231sd {rn-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vfmadd231sd {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x85,0x1a,0xb9,0xff]
+vfmadd231sd {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmadd231sd {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x85,0x9a,0xb9,0xff]
+vfmadd231sd {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmadd231sd {rn-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x18,0xb9,0xc9]
+vfmadd231sd {rn-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfmadd231sd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x1a,0xb9,0xc9]
+vfmadd231sd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd231sd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x9a,0xb9,0xc9]
+vfmadd231sd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd231sd {ru-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x52,0x85,0x58,0xb9,0xff]
+vfmadd231sd {ru-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vfmadd231sd {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x85,0x5a,0xb9,0xff]
+vfmadd231sd {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmadd231sd {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x85,0xda,0xb9,0xff]
+vfmadd231sd {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmadd231sd {ru-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x58,0xb9,0xc9]
+vfmadd231sd {ru-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfmadd231sd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x5a,0xb9,0xc9]
+vfmadd231sd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd231sd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0xda,0xb9,0xc9]
+vfmadd231sd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd231sd {rz-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x52,0x85,0x78,0xb9,0xff]
+vfmadd231sd {rz-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vfmadd231sd {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x85,0x7a,0xb9,0xff]
+vfmadd231sd {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmadd231sd {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x85,0xfa,0xb9,0xff]
+vfmadd231sd {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmadd231sd {rz-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x78,0xb9,0xc9]
+vfmadd231sd {rz-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfmadd231sd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x7a,0xb9,0xc9]
+vfmadd231sd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd231sd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0xfa,0xb9,0xc9]
+vfmadd231sd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd231sd %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x42,0x81,0xb9,0xff]
+vfmadd231sd %xmm15, %xmm15, %xmm15
+
+// CHECK: vfmadd231sd %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x85,0x0a,0xb9,0xff]
+vfmadd231sd %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmadd231sd %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x85,0x8a,0xb9,0xff]
+vfmadd231sd %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmadd231sd %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0xb9,0xc9]
+vfmadd231sd %xmm1, %xmm1, %xmm1
+
+// CHECK: vfmadd231sd %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0xb9,0xc9]
+vfmadd231sd %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd231sd %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0xb9,0xc9]
+vfmadd231sd %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd231ss 256(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x01,0xb9,0xbc,0x82,0x00,0x01,0x00,0x00]
+vfmadd231ss 256(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vfmadd231ss -256(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x01,0xb9,0xbc,0x82,0x00,0xff,0xff,0xff]
+vfmadd231ss -256(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vfmadd231ss 256(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x05,0x0a,0xb9,0x7c,0x82,0x40]
+vfmadd231ss 256(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmadd231ss -256(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x05,0x0a,0xb9,0x7c,0x82,0xc0]
+vfmadd231ss -256(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmadd231ss 256(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x05,0x8a,0xb9,0x7c,0x82,0x40]
+vfmadd231ss 256(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmadd231ss -256(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x05,0x8a,0xb9,0x7c,0x82,0xc0]
+vfmadd231ss -256(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmadd231ss 256(%rdx,%rax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0xb9,0x8c,0x82,0x00,0x01,0x00,0x00]
+vfmadd231ss 256(%rdx,%rax,4), %xmm1, %xmm1
+
+// CHECK: vfmadd231ss -256(%rdx,%rax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0xb9,0x8c,0x82,0x00,0xff,0xff,0xff]
+vfmadd231ss -256(%rdx,%rax,4), %xmm1, %xmm1
+
+// CHECK: vfmadd231ss 256(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0xb9,0x4c,0x82,0x40]
+vfmadd231ss 256(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd231ss -256(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0xb9,0x4c,0x82,0xc0]
+vfmadd231ss -256(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd231ss 256(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0xb9,0x4c,0x82,0x40]
+vfmadd231ss 256(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd231ss -256(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0xb9,0x4c,0x82,0xc0]
+vfmadd231ss -256(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd231ss 256(%rdx,%rax), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x01,0xb9,0xbc,0x02,0x00,0x01,0x00,0x00]
+vfmadd231ss 256(%rdx,%rax), %xmm15, %xmm15
+
+// CHECK: vfmadd231ss 256(%rdx,%rax), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x05,0x0a,0xb9,0x7c,0x02,0x40]
+vfmadd231ss 256(%rdx,%rax), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmadd231ss 256(%rdx,%rax), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x05,0x8a,0xb9,0x7c,0x02,0x40]
+vfmadd231ss 256(%rdx,%rax), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmadd231ss 256(%rdx,%rax), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0xb9,0x8c,0x02,0x00,0x01,0x00,0x00]
+vfmadd231ss 256(%rdx,%rax), %xmm1, %xmm1
+
+// CHECK: vfmadd231ss 256(%rdx,%rax), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0xb9,0x4c,0x02,0x40]
+vfmadd231ss 256(%rdx,%rax), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd231ss 256(%rdx,%rax), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0xb9,0x4c,0x02,0x40]
+vfmadd231ss 256(%rdx,%rax), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd231ss 256(%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x01,0xb9,0xba,0x00,0x01,0x00,0x00]
+vfmadd231ss 256(%rdx), %xmm15, %xmm15
+
+// CHECK: vfmadd231ss 256(%rdx), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x05,0x0a,0xb9,0x7a,0x40]
+vfmadd231ss 256(%rdx), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmadd231ss 256(%rdx), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x05,0x8a,0xb9,0x7a,0x40]
+vfmadd231ss 256(%rdx), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmadd231ss 256(%rdx), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0xb9,0x8a,0x00,0x01,0x00,0x00]
+vfmadd231ss 256(%rdx), %xmm1, %xmm1
+
+// CHECK: vfmadd231ss 256(%rdx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0xb9,0x4a,0x40]
+vfmadd231ss 256(%rdx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd231ss 256(%rdx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0xb9,0x4a,0x40]
+vfmadd231ss 256(%rdx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd231ss 485498096, %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x01,0xb9,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfmadd231ss 485498096, %xmm15, %xmm15
+
+// CHECK: vfmadd231ss 485498096, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x05,0x0a,0xb9,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfmadd231ss 485498096, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmadd231ss 485498096, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x05,0x8a,0xb9,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfmadd231ss 485498096, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmadd231ss 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0xb9,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfmadd231ss 485498096, %xmm1, %xmm1
+
+// CHECK: vfmadd231ss 485498096, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0xb9,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfmadd231ss 485498096, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd231ss 485498096, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0xb9,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfmadd231ss 485498096, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd231ss {rd-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x52,0x05,0x38,0xb9,0xff]
+vfmadd231ss {rd-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vfmadd231ss {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x05,0x3a,0xb9,0xff]
+vfmadd231ss {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmadd231ss {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x05,0xba,0xb9,0xff]
+vfmadd231ss {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmadd231ss {rd-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x38,0xb9,0xc9]
+vfmadd231ss {rd-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfmadd231ss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x3a,0xb9,0xc9]
+vfmadd231ss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd231ss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0xba,0xb9,0xc9]
+vfmadd231ss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd231ss (%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x01,0xb9,0x3a]
+vfmadd231ss (%rdx), %xmm15, %xmm15
+
+// CHECK: vfmadd231ss (%rdx), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x05,0x0a,0xb9,0x3a]
+vfmadd231ss (%rdx), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmadd231ss (%rdx), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x05,0x8a,0xb9,0x3a]
+vfmadd231ss (%rdx), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmadd231ss (%rdx), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0xb9,0x0a]
+vfmadd231ss (%rdx), %xmm1, %xmm1
+
+// CHECK: vfmadd231ss (%rdx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0xb9,0x0a]
+vfmadd231ss (%rdx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd231ss (%rdx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0xb9,0x0a]
+vfmadd231ss (%rdx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd231ss {rn-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x52,0x05,0x18,0xb9,0xff]
+vfmadd231ss {rn-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vfmadd231ss {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x05,0x1a,0xb9,0xff]
+vfmadd231ss {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmadd231ss {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x05,0x9a,0xb9,0xff]
+vfmadd231ss {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmadd231ss {rn-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x18,0xb9,0xc9]
+vfmadd231ss {rn-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfmadd231ss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x1a,0xb9,0xc9]
+vfmadd231ss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd231ss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x9a,0xb9,0xc9]
+vfmadd231ss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd231ss {ru-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x52,0x05,0x58,0xb9,0xff]
+vfmadd231ss {ru-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vfmadd231ss {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x05,0x5a,0xb9,0xff]
+vfmadd231ss {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmadd231ss {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x05,0xda,0xb9,0xff]
+vfmadd231ss {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmadd231ss {ru-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x58,0xb9,0xc9]
+vfmadd231ss {ru-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfmadd231ss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x5a,0xb9,0xc9]
+vfmadd231ss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd231ss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0xda,0xb9,0xc9]
+vfmadd231ss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd231ss {rz-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x52,0x05,0x78,0xb9,0xff]
+vfmadd231ss {rz-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vfmadd231ss {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x05,0x7a,0xb9,0xff]
+vfmadd231ss {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmadd231ss {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x05,0xfa,0xb9,0xff]
+vfmadd231ss {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmadd231ss {rz-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x78,0xb9,0xc9]
+vfmadd231ss {rz-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfmadd231ss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x7a,0xb9,0xc9]
+vfmadd231ss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd231ss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0xfa,0xb9,0xc9]
+vfmadd231ss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmadd231ss %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x42,0x01,0xb9,0xff]
+vfmadd231ss %xmm15, %xmm15, %xmm15
+
+// CHECK: vfmadd231ss %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x05,0x0a,0xb9,0xff]
+vfmadd231ss %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmadd231ss %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x05,0x8a,0xb9,0xff]
+vfmadd231ss %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmadd231ss %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0xb9,0xc9]
+vfmadd231ss %xmm1, %xmm1, %xmm1
+
+// CHECK: vfmadd231ss %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0xb9,0xc9]
+vfmadd231ss %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmadd231ss %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0xb9,0xc9]
+vfmadd231ss %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub132sd 485498096, %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x81,0x9b,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfmsub132sd 485498096, %xmm15, %xmm15
+
+// CHECK: vfmsub132sd 485498096, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x85,0x0a,0x9b,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfmsub132sd 485498096, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmsub132sd 485498096, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x85,0x8a,0x9b,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfmsub132sd 485498096, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmsub132sd 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0x9b,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfmsub132sd 485498096, %xmm1, %xmm1
+
+// CHECK: vfmsub132sd 485498096, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0x9b,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfmsub132sd 485498096, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub132sd 485498096, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0x9b,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfmsub132sd 485498096, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub132sd 512(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x81,0x9b,0xbc,0x82,0x00,0x02,0x00,0x00]
+vfmsub132sd 512(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vfmsub132sd -512(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x81,0x9b,0xbc,0x82,0x00,0xfe,0xff,0xff]
+vfmsub132sd -512(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vfmsub132sd 512(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x85,0x0a,0x9b,0x7c,0x82,0x40]
+vfmsub132sd 512(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmsub132sd -512(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x85,0x0a,0x9b,0x7c,0x82,0xc0]
+vfmsub132sd -512(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmsub132sd 512(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x85,0x8a,0x9b,0x7c,0x82,0x40]
+vfmsub132sd 512(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmsub132sd -512(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x85,0x8a,0x9b,0x7c,0x82,0xc0]
+vfmsub132sd -512(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmsub132sd 512(%rdx,%rax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0x9b,0x8c,0x82,0x00,0x02,0x00,0x00]
+vfmsub132sd 512(%rdx,%rax,4), %xmm1, %xmm1
+
+// CHECK: vfmsub132sd -512(%rdx,%rax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0x9b,0x8c,0x82,0x00,0xfe,0xff,0xff]
+vfmsub132sd -512(%rdx,%rax,4), %xmm1, %xmm1
+
+// CHECK: vfmsub132sd 512(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0x9b,0x4c,0x82,0x40]
+vfmsub132sd 512(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub132sd -512(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0x9b,0x4c,0x82,0xc0]
+vfmsub132sd -512(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub132sd 512(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0x9b,0x4c,0x82,0x40]
+vfmsub132sd 512(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub132sd -512(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0x9b,0x4c,0x82,0xc0]
+vfmsub132sd -512(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub132sd 512(%rdx,%rax), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x81,0x9b,0xbc,0x02,0x00,0x02,0x00,0x00]
+vfmsub132sd 512(%rdx,%rax), %xmm15, %xmm15
+
+// CHECK: vfmsub132sd 512(%rdx,%rax), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x85,0x0a,0x9b,0x7c,0x02,0x40]
+vfmsub132sd 512(%rdx,%rax), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmsub132sd 512(%rdx,%rax), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x85,0x8a,0x9b,0x7c,0x02,0x40]
+vfmsub132sd 512(%rdx,%rax), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmsub132sd 512(%rdx,%rax), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0x9b,0x8c,0x02,0x00,0x02,0x00,0x00]
+vfmsub132sd 512(%rdx,%rax), %xmm1, %xmm1
+
+// CHECK: vfmsub132sd 512(%rdx,%rax), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0x9b,0x4c,0x02,0x40]
+vfmsub132sd 512(%rdx,%rax), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub132sd 512(%rdx,%rax), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0x9b,0x4c,0x02,0x40]
+vfmsub132sd 512(%rdx,%rax), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub132sd 512(%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x81,0x9b,0xba,0x00,0x02,0x00,0x00]
+vfmsub132sd 512(%rdx), %xmm15, %xmm15
+
+// CHECK: vfmsub132sd 512(%rdx), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x85,0x0a,0x9b,0x7a,0x40]
+vfmsub132sd 512(%rdx), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmsub132sd 512(%rdx), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x85,0x8a,0x9b,0x7a,0x40]
+vfmsub132sd 512(%rdx), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmsub132sd 512(%rdx), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0x9b,0x8a,0x00,0x02,0x00,0x00]
+vfmsub132sd 512(%rdx), %xmm1, %xmm1
+
+// CHECK: vfmsub132sd 512(%rdx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0x9b,0x4a,0x40]
+vfmsub132sd 512(%rdx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub132sd 512(%rdx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0x9b,0x4a,0x40]
+vfmsub132sd 512(%rdx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub132sd {rd-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x52,0x85,0x38,0x9b,0xff]
+vfmsub132sd {rd-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vfmsub132sd {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x85,0x3a,0x9b,0xff]
+vfmsub132sd {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmsub132sd {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x85,0xba,0x9b,0xff]
+vfmsub132sd {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmsub132sd {rd-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x38,0x9b,0xc9]
+vfmsub132sd {rd-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfmsub132sd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x3a,0x9b,0xc9]
+vfmsub132sd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub132sd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0xba,0x9b,0xc9]
+vfmsub132sd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub132sd (%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x81,0x9b,0x3a]
+vfmsub132sd (%rdx), %xmm15, %xmm15
+
+// CHECK: vfmsub132sd (%rdx), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x85,0x0a,0x9b,0x3a]
+vfmsub132sd (%rdx), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmsub132sd (%rdx), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x85,0x8a,0x9b,0x3a]
+vfmsub132sd (%rdx), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmsub132sd (%rdx), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0x9b,0x0a]
+vfmsub132sd (%rdx), %xmm1, %xmm1
+
+// CHECK: vfmsub132sd (%rdx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0x9b,0x0a]
+vfmsub132sd (%rdx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub132sd (%rdx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0x9b,0x0a]
+vfmsub132sd (%rdx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub132sd {rn-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x52,0x85,0x18,0x9b,0xff]
+vfmsub132sd {rn-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vfmsub132sd {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x85,0x1a,0x9b,0xff]
+vfmsub132sd {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmsub132sd {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x85,0x9a,0x9b,0xff]
+vfmsub132sd {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmsub132sd {rn-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x18,0x9b,0xc9]
+vfmsub132sd {rn-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfmsub132sd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x1a,0x9b,0xc9]
+vfmsub132sd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub132sd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x9a,0x9b,0xc9]
+vfmsub132sd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub132sd {ru-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x52,0x85,0x58,0x9b,0xff]
+vfmsub132sd {ru-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vfmsub132sd {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x85,0x5a,0x9b,0xff]
+vfmsub132sd {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmsub132sd {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x85,0xda,0x9b,0xff]
+vfmsub132sd {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmsub132sd {ru-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x58,0x9b,0xc9]
+vfmsub132sd {ru-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfmsub132sd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x5a,0x9b,0xc9]
+vfmsub132sd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub132sd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0xda,0x9b,0xc9]
+vfmsub132sd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub132sd {rz-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x52,0x85,0x78,0x9b,0xff]
+vfmsub132sd {rz-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vfmsub132sd {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x85,0x7a,0x9b,0xff]
+vfmsub132sd {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmsub132sd {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x85,0xfa,0x9b,0xff]
+vfmsub132sd {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmsub132sd {rz-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x78,0x9b,0xc9]
+vfmsub132sd {rz-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfmsub132sd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x7a,0x9b,0xc9]
+vfmsub132sd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub132sd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0xfa,0x9b,0xc9]
+vfmsub132sd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub132sd %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x42,0x81,0x9b,0xff]
+vfmsub132sd %xmm15, %xmm15, %xmm15
+
+// CHECK: vfmsub132sd %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x85,0x0a,0x9b,0xff]
+vfmsub132sd %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmsub132sd %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x85,0x8a,0x9b,0xff]
+vfmsub132sd %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmsub132sd %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0x9b,0xc9]
+vfmsub132sd %xmm1, %xmm1, %xmm1
+
+// CHECK: vfmsub132sd %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0x9b,0xc9]
+vfmsub132sd %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub132sd %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0x9b,0xc9]
+vfmsub132sd %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub132ss 256(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x01,0x9b,0xbc,0x82,0x00,0x01,0x00,0x00]
+vfmsub132ss 256(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vfmsub132ss -256(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x01,0x9b,0xbc,0x82,0x00,0xff,0xff,0xff]
+vfmsub132ss -256(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vfmsub132ss 256(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x05,0x0a,0x9b,0x7c,0x82,0x40]
+vfmsub132ss 256(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmsub132ss -256(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x05,0x0a,0x9b,0x7c,0x82,0xc0]
+vfmsub132ss -256(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmsub132ss 256(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x05,0x8a,0x9b,0x7c,0x82,0x40]
+vfmsub132ss 256(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmsub132ss -256(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x05,0x8a,0x9b,0x7c,0x82,0xc0]
+vfmsub132ss -256(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmsub132ss 256(%rdx,%rax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0x9b,0x8c,0x82,0x00,0x01,0x00,0x00]
+vfmsub132ss 256(%rdx,%rax,4), %xmm1, %xmm1
+
+// CHECK: vfmsub132ss -256(%rdx,%rax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0x9b,0x8c,0x82,0x00,0xff,0xff,0xff]
+vfmsub132ss -256(%rdx,%rax,4), %xmm1, %xmm1
+
+// CHECK: vfmsub132ss 256(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0x9b,0x4c,0x82,0x40]
+vfmsub132ss 256(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub132ss -256(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0x9b,0x4c,0x82,0xc0]
+vfmsub132ss -256(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub132ss 256(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0x9b,0x4c,0x82,0x40]
+vfmsub132ss 256(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub132ss -256(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0x9b,0x4c,0x82,0xc0]
+vfmsub132ss -256(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub132ss 256(%rdx,%rax), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x01,0x9b,0xbc,0x02,0x00,0x01,0x00,0x00]
+vfmsub132ss 256(%rdx,%rax), %xmm15, %xmm15
+
+// CHECK: vfmsub132ss 256(%rdx,%rax), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x05,0x0a,0x9b,0x7c,0x02,0x40]
+vfmsub132ss 256(%rdx,%rax), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmsub132ss 256(%rdx,%rax), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x05,0x8a,0x9b,0x7c,0x02,0x40]
+vfmsub132ss 256(%rdx,%rax), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmsub132ss 256(%rdx,%rax), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0x9b,0x8c,0x02,0x00,0x01,0x00,0x00]
+vfmsub132ss 256(%rdx,%rax), %xmm1, %xmm1
+
+// CHECK: vfmsub132ss 256(%rdx,%rax), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0x9b,0x4c,0x02,0x40]
+vfmsub132ss 256(%rdx,%rax), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub132ss 256(%rdx,%rax), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0x9b,0x4c,0x02,0x40]
+vfmsub132ss 256(%rdx,%rax), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub132ss 256(%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x01,0x9b,0xba,0x00,0x01,0x00,0x00]
+vfmsub132ss 256(%rdx), %xmm15, %xmm15
+
+// CHECK: vfmsub132ss 256(%rdx), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x05,0x0a,0x9b,0x7a,0x40]
+vfmsub132ss 256(%rdx), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmsub132ss 256(%rdx), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x05,0x8a,0x9b,0x7a,0x40]
+vfmsub132ss 256(%rdx), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmsub132ss 256(%rdx), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0x9b,0x8a,0x00,0x01,0x00,0x00]
+vfmsub132ss 256(%rdx), %xmm1, %xmm1
+
+// CHECK: vfmsub132ss 256(%rdx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0x9b,0x4a,0x40]
+vfmsub132ss 256(%rdx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub132ss 256(%rdx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0x9b,0x4a,0x40]
+vfmsub132ss 256(%rdx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub132ss 485498096, %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x01,0x9b,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfmsub132ss 485498096, %xmm15, %xmm15
+
+// CHECK: vfmsub132ss 485498096, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x05,0x0a,0x9b,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfmsub132ss 485498096, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmsub132ss 485498096, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x05,0x8a,0x9b,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfmsub132ss 485498096, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmsub132ss 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0x9b,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfmsub132ss 485498096, %xmm1, %xmm1
+
+// CHECK: vfmsub132ss 485498096, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0x9b,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfmsub132ss 485498096, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub132ss 485498096, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0x9b,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfmsub132ss 485498096, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub132ss {rd-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x52,0x05,0x38,0x9b,0xff]
+vfmsub132ss {rd-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vfmsub132ss {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x05,0x3a,0x9b,0xff]
+vfmsub132ss {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmsub132ss {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x05,0xba,0x9b,0xff]
+vfmsub132ss {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmsub132ss {rd-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x38,0x9b,0xc9]
+vfmsub132ss {rd-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfmsub132ss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x3a,0x9b,0xc9]
+vfmsub132ss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub132ss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0xba,0x9b,0xc9]
+vfmsub132ss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub132ss (%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x01,0x9b,0x3a]
+vfmsub132ss (%rdx), %xmm15, %xmm15
+
+// CHECK: vfmsub132ss (%rdx), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x05,0x0a,0x9b,0x3a]
+vfmsub132ss (%rdx), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmsub132ss (%rdx), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x05,0x8a,0x9b,0x3a]
+vfmsub132ss (%rdx), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmsub132ss (%rdx), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0x9b,0x0a]
+vfmsub132ss (%rdx), %xmm1, %xmm1
+
+// CHECK: vfmsub132ss (%rdx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0x9b,0x0a]
+vfmsub132ss (%rdx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub132ss (%rdx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0x9b,0x0a]
+vfmsub132ss (%rdx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub132ss {rn-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x52,0x05,0x18,0x9b,0xff]
+vfmsub132ss {rn-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vfmsub132ss {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x05,0x1a,0x9b,0xff]
+vfmsub132ss {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmsub132ss {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x05,0x9a,0x9b,0xff]
+vfmsub132ss {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmsub132ss {rn-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x18,0x9b,0xc9]
+vfmsub132ss {rn-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfmsub132ss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x1a,0x9b,0xc9]
+vfmsub132ss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub132ss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x9a,0x9b,0xc9]
+vfmsub132ss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub132ss {ru-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x52,0x05,0x58,0x9b,0xff]
+vfmsub132ss {ru-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vfmsub132ss {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x05,0x5a,0x9b,0xff]
+vfmsub132ss {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmsub132ss {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x05,0xda,0x9b,0xff]
+vfmsub132ss {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmsub132ss {ru-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x58,0x9b,0xc9]
+vfmsub132ss {ru-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfmsub132ss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x5a,0x9b,0xc9]
+vfmsub132ss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub132ss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0xda,0x9b,0xc9]
+vfmsub132ss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub132ss {rz-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x52,0x05,0x78,0x9b,0xff]
+vfmsub132ss {rz-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vfmsub132ss {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x05,0x7a,0x9b,0xff]
+vfmsub132ss {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmsub132ss {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x05,0xfa,0x9b,0xff]
+vfmsub132ss {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmsub132ss {rz-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x78,0x9b,0xc9]
+vfmsub132ss {rz-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfmsub132ss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x7a,0x9b,0xc9]
+vfmsub132ss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub132ss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0xfa,0x9b,0xc9]
+vfmsub132ss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub132ss %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x42,0x01,0x9b,0xff]
+vfmsub132ss %xmm15, %xmm15, %xmm15
+
+// CHECK: vfmsub132ss %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x05,0x0a,0x9b,0xff]
+vfmsub132ss %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmsub132ss %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x05,0x8a,0x9b,0xff]
+vfmsub132ss %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmsub132ss %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0x9b,0xc9]
+vfmsub132ss %xmm1, %xmm1, %xmm1
+
+// CHECK: vfmsub132ss %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0x9b,0xc9]
+vfmsub132ss %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub132ss %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0x9b,0xc9]
+vfmsub132ss %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub213sd 485498096, %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x81,0xab,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfmsub213sd 485498096, %xmm15, %xmm15
+
+// CHECK: vfmsub213sd 485498096, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x85,0x0a,0xab,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfmsub213sd 485498096, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmsub213sd 485498096, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x85,0x8a,0xab,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfmsub213sd 485498096, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmsub213sd 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0xab,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfmsub213sd 485498096, %xmm1, %xmm1
+
+// CHECK: vfmsub213sd 485498096, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0xab,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfmsub213sd 485498096, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub213sd 485498096, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0xab,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfmsub213sd 485498096, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub213sd 512(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x81,0xab,0xbc,0x82,0x00,0x02,0x00,0x00]
+vfmsub213sd 512(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vfmsub213sd -512(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x81,0xab,0xbc,0x82,0x00,0xfe,0xff,0xff]
+vfmsub213sd -512(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vfmsub213sd 512(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x85,0x0a,0xab,0x7c,0x82,0x40]
+vfmsub213sd 512(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmsub213sd -512(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x85,0x0a,0xab,0x7c,0x82,0xc0]
+vfmsub213sd -512(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmsub213sd 512(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x85,0x8a,0xab,0x7c,0x82,0x40]
+vfmsub213sd 512(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmsub213sd -512(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x85,0x8a,0xab,0x7c,0x82,0xc0]
+vfmsub213sd -512(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmsub213sd 512(%rdx,%rax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0xab,0x8c,0x82,0x00,0x02,0x00,0x00]
+vfmsub213sd 512(%rdx,%rax,4), %xmm1, %xmm1
+
+// CHECK: vfmsub213sd -512(%rdx,%rax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0xab,0x8c,0x82,0x00,0xfe,0xff,0xff]
+vfmsub213sd -512(%rdx,%rax,4), %xmm1, %xmm1
+
+// CHECK: vfmsub213sd 512(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0xab,0x4c,0x82,0x40]
+vfmsub213sd 512(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub213sd -512(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0xab,0x4c,0x82,0xc0]
+vfmsub213sd -512(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub213sd 512(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0xab,0x4c,0x82,0x40]
+vfmsub213sd 512(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub213sd -512(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0xab,0x4c,0x82,0xc0]
+vfmsub213sd -512(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub213sd 512(%rdx,%rax), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x81,0xab,0xbc,0x02,0x00,0x02,0x00,0x00]
+vfmsub213sd 512(%rdx,%rax), %xmm15, %xmm15
+
+// CHECK: vfmsub213sd 512(%rdx,%rax), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x85,0x0a,0xab,0x7c,0x02,0x40]
+vfmsub213sd 512(%rdx,%rax), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmsub213sd 512(%rdx,%rax), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x85,0x8a,0xab,0x7c,0x02,0x40]
+vfmsub213sd 512(%rdx,%rax), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmsub213sd 512(%rdx,%rax), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0xab,0x8c,0x02,0x00,0x02,0x00,0x00]
+vfmsub213sd 512(%rdx,%rax), %xmm1, %xmm1
+
+// CHECK: vfmsub213sd 512(%rdx,%rax), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0xab,0x4c,0x02,0x40]
+vfmsub213sd 512(%rdx,%rax), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub213sd 512(%rdx,%rax), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0xab,0x4c,0x02,0x40]
+vfmsub213sd 512(%rdx,%rax), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub213sd 512(%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x81,0xab,0xba,0x00,0x02,0x00,0x00]
+vfmsub213sd 512(%rdx), %xmm15, %xmm15
+
+// CHECK: vfmsub213sd 512(%rdx), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x85,0x0a,0xab,0x7a,0x40]
+vfmsub213sd 512(%rdx), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmsub213sd 512(%rdx), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x85,0x8a,0xab,0x7a,0x40]
+vfmsub213sd 512(%rdx), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmsub213sd 512(%rdx), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0xab,0x8a,0x00,0x02,0x00,0x00]
+vfmsub213sd 512(%rdx), %xmm1, %xmm1
+
+// CHECK: vfmsub213sd 512(%rdx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0xab,0x4a,0x40]
+vfmsub213sd 512(%rdx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub213sd 512(%rdx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0xab,0x4a,0x40]
+vfmsub213sd 512(%rdx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub213sd {rd-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x52,0x85,0x38,0xab,0xff]
+vfmsub213sd {rd-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vfmsub213sd {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x85,0x3a,0xab,0xff]
+vfmsub213sd {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmsub213sd {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x85,0xba,0xab,0xff]
+vfmsub213sd {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmsub213sd {rd-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x38,0xab,0xc9]
+vfmsub213sd {rd-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfmsub213sd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x3a,0xab,0xc9]
+vfmsub213sd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub213sd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0xba,0xab,0xc9]
+vfmsub213sd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub213sd (%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x81,0xab,0x3a]
+vfmsub213sd (%rdx), %xmm15, %xmm15
+
+// CHECK: vfmsub213sd (%rdx), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x85,0x0a,0xab,0x3a]
+vfmsub213sd (%rdx), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmsub213sd (%rdx), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x85,0x8a,0xab,0x3a]
+vfmsub213sd (%rdx), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmsub213sd (%rdx), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0xab,0x0a]
+vfmsub213sd (%rdx), %xmm1, %xmm1
+
+// CHECK: vfmsub213sd (%rdx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0xab,0x0a]
+vfmsub213sd (%rdx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub213sd (%rdx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0xab,0x0a]
+vfmsub213sd (%rdx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub213sd {rn-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x52,0x85,0x18,0xab,0xff]
+vfmsub213sd {rn-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vfmsub213sd {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x85,0x1a,0xab,0xff]
+vfmsub213sd {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmsub213sd {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x85,0x9a,0xab,0xff]
+vfmsub213sd {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmsub213sd {rn-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x18,0xab,0xc9]
+vfmsub213sd {rn-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfmsub213sd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x1a,0xab,0xc9]
+vfmsub213sd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub213sd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x9a,0xab,0xc9]
+vfmsub213sd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub213sd {ru-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x52,0x85,0x58,0xab,0xff]
+vfmsub213sd {ru-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vfmsub213sd {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x85,0x5a,0xab,0xff]
+vfmsub213sd {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmsub213sd {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x85,0xda,0xab,0xff]
+vfmsub213sd {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmsub213sd {ru-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x58,0xab,0xc9]
+vfmsub213sd {ru-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfmsub213sd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x5a,0xab,0xc9]
+vfmsub213sd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub213sd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0xda,0xab,0xc9]
+vfmsub213sd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub213sd {rz-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x52,0x85,0x78,0xab,0xff]
+vfmsub213sd {rz-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vfmsub213sd {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x85,0x7a,0xab,0xff]
+vfmsub213sd {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmsub213sd {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x85,0xfa,0xab,0xff]
+vfmsub213sd {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmsub213sd {rz-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x78,0xab,0xc9]
+vfmsub213sd {rz-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfmsub213sd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x7a,0xab,0xc9]
+vfmsub213sd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub213sd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0xfa,0xab,0xc9]
+vfmsub213sd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub213sd %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x42,0x81,0xab,0xff]
+vfmsub213sd %xmm15, %xmm15, %xmm15
+
+// CHECK: vfmsub213sd %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x85,0x0a,0xab,0xff]
+vfmsub213sd %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmsub213sd %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x85,0x8a,0xab,0xff]
+vfmsub213sd %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmsub213sd %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0xab,0xc9]
+vfmsub213sd %xmm1, %xmm1, %xmm1
+
+// CHECK: vfmsub213sd %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0xab,0xc9]
+vfmsub213sd %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub213sd %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0xab,0xc9]
+vfmsub213sd %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub213ss 256(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x01,0xab,0xbc,0x82,0x00,0x01,0x00,0x00]
+vfmsub213ss 256(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vfmsub213ss -256(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x01,0xab,0xbc,0x82,0x00,0xff,0xff,0xff]
+vfmsub213ss -256(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vfmsub213ss 256(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x05,0x0a,0xab,0x7c,0x82,0x40]
+vfmsub213ss 256(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmsub213ss -256(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x05,0x0a,0xab,0x7c,0x82,0xc0]
+vfmsub213ss -256(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmsub213ss 256(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x05,0x8a,0xab,0x7c,0x82,0x40]
+vfmsub213ss 256(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmsub213ss -256(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x05,0x8a,0xab,0x7c,0x82,0xc0]
+vfmsub213ss -256(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmsub213ss 256(%rdx,%rax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0xab,0x8c,0x82,0x00,0x01,0x00,0x00]
+vfmsub213ss 256(%rdx,%rax,4), %xmm1, %xmm1
+
+// CHECK: vfmsub213ss -256(%rdx,%rax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0xab,0x8c,0x82,0x00,0xff,0xff,0xff]
+vfmsub213ss -256(%rdx,%rax,4), %xmm1, %xmm1
+
+// CHECK: vfmsub213ss 256(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0xab,0x4c,0x82,0x40]
+vfmsub213ss 256(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub213ss -256(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0xab,0x4c,0x82,0xc0]
+vfmsub213ss -256(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub213ss 256(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0xab,0x4c,0x82,0x40]
+vfmsub213ss 256(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub213ss -256(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0xab,0x4c,0x82,0xc0]
+vfmsub213ss -256(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub213ss 256(%rdx,%rax), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x01,0xab,0xbc,0x02,0x00,0x01,0x00,0x00]
+vfmsub213ss 256(%rdx,%rax), %xmm15, %xmm15
+
+// CHECK: vfmsub213ss 256(%rdx,%rax), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x05,0x0a,0xab,0x7c,0x02,0x40]
+vfmsub213ss 256(%rdx,%rax), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmsub213ss 256(%rdx,%rax), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x05,0x8a,0xab,0x7c,0x02,0x40]
+vfmsub213ss 256(%rdx,%rax), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmsub213ss 256(%rdx,%rax), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0xab,0x8c,0x02,0x00,0x01,0x00,0x00]
+vfmsub213ss 256(%rdx,%rax), %xmm1, %xmm1
+
+// CHECK: vfmsub213ss 256(%rdx,%rax), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0xab,0x4c,0x02,0x40]
+vfmsub213ss 256(%rdx,%rax), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub213ss 256(%rdx,%rax), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0xab,0x4c,0x02,0x40]
+vfmsub213ss 256(%rdx,%rax), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub213ss 256(%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x01,0xab,0xba,0x00,0x01,0x00,0x00]
+vfmsub213ss 256(%rdx), %xmm15, %xmm15
+
+// CHECK: vfmsub213ss 256(%rdx), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x05,0x0a,0xab,0x7a,0x40]
+vfmsub213ss 256(%rdx), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmsub213ss 256(%rdx), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x05,0x8a,0xab,0x7a,0x40]
+vfmsub213ss 256(%rdx), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmsub213ss 256(%rdx), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0xab,0x8a,0x00,0x01,0x00,0x00]
+vfmsub213ss 256(%rdx), %xmm1, %xmm1
+
+// CHECK: vfmsub213ss 256(%rdx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0xab,0x4a,0x40]
+vfmsub213ss 256(%rdx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub213ss 256(%rdx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0xab,0x4a,0x40]
+vfmsub213ss 256(%rdx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub213ss 485498096, %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x01,0xab,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfmsub213ss 485498096, %xmm15, %xmm15
+
+// CHECK: vfmsub213ss 485498096, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x05,0x0a,0xab,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfmsub213ss 485498096, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmsub213ss 485498096, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x05,0x8a,0xab,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfmsub213ss 485498096, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmsub213ss 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0xab,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfmsub213ss 485498096, %xmm1, %xmm1
+
+// CHECK: vfmsub213ss 485498096, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0xab,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfmsub213ss 485498096, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub213ss 485498096, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0xab,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfmsub213ss 485498096, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub213ss {rd-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x52,0x05,0x38,0xab,0xff]
+vfmsub213ss {rd-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vfmsub213ss {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x05,0x3a,0xab,0xff]
+vfmsub213ss {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmsub213ss {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x05,0xba,0xab,0xff]
+vfmsub213ss {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmsub213ss {rd-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x38,0xab,0xc9]
+vfmsub213ss {rd-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfmsub213ss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x3a,0xab,0xc9]
+vfmsub213ss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub213ss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0xba,0xab,0xc9]
+vfmsub213ss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub213ss (%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x01,0xab,0x3a]
+vfmsub213ss (%rdx), %xmm15, %xmm15
+
+// CHECK: vfmsub213ss (%rdx), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x05,0x0a,0xab,0x3a]
+vfmsub213ss (%rdx), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmsub213ss (%rdx), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x05,0x8a,0xab,0x3a]
+vfmsub213ss (%rdx), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmsub213ss (%rdx), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0xab,0x0a]
+vfmsub213ss (%rdx), %xmm1, %xmm1
+
+// CHECK: vfmsub213ss (%rdx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0xab,0x0a]
+vfmsub213ss (%rdx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub213ss (%rdx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0xab,0x0a]
+vfmsub213ss (%rdx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub213ss {rn-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x52,0x05,0x18,0xab,0xff]
+vfmsub213ss {rn-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vfmsub213ss {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x05,0x1a,0xab,0xff]
+vfmsub213ss {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmsub213ss {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x05,0x9a,0xab,0xff]
+vfmsub213ss {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmsub213ss {rn-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x18,0xab,0xc9]
+vfmsub213ss {rn-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfmsub213ss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x1a,0xab,0xc9]
+vfmsub213ss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub213ss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x9a,0xab,0xc9]
+vfmsub213ss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub213ss {ru-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x52,0x05,0x58,0xab,0xff]
+vfmsub213ss {ru-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vfmsub213ss {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x05,0x5a,0xab,0xff]
+vfmsub213ss {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmsub213ss {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x05,0xda,0xab,0xff]
+vfmsub213ss {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmsub213ss {ru-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x58,0xab,0xc9]
+vfmsub213ss {ru-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfmsub213ss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x5a,0xab,0xc9]
+vfmsub213ss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub213ss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0xda,0xab,0xc9]
+vfmsub213ss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub213ss {rz-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x52,0x05,0x78,0xab,0xff]
+vfmsub213ss {rz-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vfmsub213ss {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x05,0x7a,0xab,0xff]
+vfmsub213ss {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmsub213ss {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x05,0xfa,0xab,0xff]
+vfmsub213ss {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmsub213ss {rz-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x78,0xab,0xc9]
+vfmsub213ss {rz-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfmsub213ss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x7a,0xab,0xc9]
+vfmsub213ss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub213ss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0xfa,0xab,0xc9]
+vfmsub213ss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub213ss %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x42,0x01,0xab,0xff]
+vfmsub213ss %xmm15, %xmm15, %xmm15
+
+// CHECK: vfmsub213ss %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x05,0x0a,0xab,0xff]
+vfmsub213ss %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmsub213ss %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x05,0x8a,0xab,0xff]
+vfmsub213ss %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmsub213ss %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0xab,0xc9]
+vfmsub213ss %xmm1, %xmm1, %xmm1
+
+// CHECK: vfmsub213ss %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0xab,0xc9]
+vfmsub213ss %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub213ss %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0xab,0xc9]
+vfmsub213ss %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub231sd 485498096, %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x81,0xbb,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfmsub231sd 485498096, %xmm15, %xmm15
+
+// CHECK: vfmsub231sd 485498096, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x85,0x0a,0xbb,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfmsub231sd 485498096, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmsub231sd 485498096, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x85,0x8a,0xbb,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfmsub231sd 485498096, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmsub231sd 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0xbb,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfmsub231sd 485498096, %xmm1, %xmm1
+
+// CHECK: vfmsub231sd 485498096, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0xbb,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfmsub231sd 485498096, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub231sd 485498096, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0xbb,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfmsub231sd 485498096, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub231sd 512(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x81,0xbb,0xbc,0x82,0x00,0x02,0x00,0x00]
+vfmsub231sd 512(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vfmsub231sd -512(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x81,0xbb,0xbc,0x82,0x00,0xfe,0xff,0xff]
+vfmsub231sd -512(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vfmsub231sd 512(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x85,0x0a,0xbb,0x7c,0x82,0x40]
+vfmsub231sd 512(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmsub231sd -512(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x85,0x0a,0xbb,0x7c,0x82,0xc0]
+vfmsub231sd -512(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmsub231sd 512(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x85,0x8a,0xbb,0x7c,0x82,0x40]
+vfmsub231sd 512(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmsub231sd -512(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x85,0x8a,0xbb,0x7c,0x82,0xc0]
+vfmsub231sd -512(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmsub231sd 512(%rdx,%rax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0xbb,0x8c,0x82,0x00,0x02,0x00,0x00]
+vfmsub231sd 512(%rdx,%rax,4), %xmm1, %xmm1
+
+// CHECK: vfmsub231sd -512(%rdx,%rax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0xbb,0x8c,0x82,0x00,0xfe,0xff,0xff]
+vfmsub231sd -512(%rdx,%rax,4), %xmm1, %xmm1
+
+// CHECK: vfmsub231sd 512(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0xbb,0x4c,0x82,0x40]
+vfmsub231sd 512(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub231sd -512(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0xbb,0x4c,0x82,0xc0]
+vfmsub231sd -512(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub231sd 512(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0xbb,0x4c,0x82,0x40]
+vfmsub231sd 512(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub231sd -512(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0xbb,0x4c,0x82,0xc0]
+vfmsub231sd -512(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub231sd 512(%rdx,%rax), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x81,0xbb,0xbc,0x02,0x00,0x02,0x00,0x00]
+vfmsub231sd 512(%rdx,%rax), %xmm15, %xmm15
+
+// CHECK: vfmsub231sd 512(%rdx,%rax), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x85,0x0a,0xbb,0x7c,0x02,0x40]
+vfmsub231sd 512(%rdx,%rax), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmsub231sd 512(%rdx,%rax), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x85,0x8a,0xbb,0x7c,0x02,0x40]
+vfmsub231sd 512(%rdx,%rax), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmsub231sd 512(%rdx,%rax), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0xbb,0x8c,0x02,0x00,0x02,0x00,0x00]
+vfmsub231sd 512(%rdx,%rax), %xmm1, %xmm1
+
+// CHECK: vfmsub231sd 512(%rdx,%rax), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0xbb,0x4c,0x02,0x40]
+vfmsub231sd 512(%rdx,%rax), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub231sd 512(%rdx,%rax), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0xbb,0x4c,0x02,0x40]
+vfmsub231sd 512(%rdx,%rax), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub231sd 512(%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x81,0xbb,0xba,0x00,0x02,0x00,0x00]
+vfmsub231sd 512(%rdx), %xmm15, %xmm15
+
+// CHECK: vfmsub231sd 512(%rdx), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x85,0x0a,0xbb,0x7a,0x40]
+vfmsub231sd 512(%rdx), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmsub231sd 512(%rdx), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x85,0x8a,0xbb,0x7a,0x40]
+vfmsub231sd 512(%rdx), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmsub231sd 512(%rdx), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0xbb,0x8a,0x00,0x02,0x00,0x00]
+vfmsub231sd 512(%rdx), %xmm1, %xmm1
+
+// CHECK: vfmsub231sd 512(%rdx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0xbb,0x4a,0x40]
+vfmsub231sd 512(%rdx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub231sd 512(%rdx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0xbb,0x4a,0x40]
+vfmsub231sd 512(%rdx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub231sd {rd-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x52,0x85,0x38,0xbb,0xff]
+vfmsub231sd {rd-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vfmsub231sd {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x85,0x3a,0xbb,0xff]
+vfmsub231sd {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmsub231sd {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x85,0xba,0xbb,0xff]
+vfmsub231sd {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmsub231sd {rd-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x38,0xbb,0xc9]
+vfmsub231sd {rd-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfmsub231sd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x3a,0xbb,0xc9]
+vfmsub231sd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub231sd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0xba,0xbb,0xc9]
+vfmsub231sd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub231sd (%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x81,0xbb,0x3a]
+vfmsub231sd (%rdx), %xmm15, %xmm15
+
+// CHECK: vfmsub231sd (%rdx), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x85,0x0a,0xbb,0x3a]
+vfmsub231sd (%rdx), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmsub231sd (%rdx), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x85,0x8a,0xbb,0x3a]
+vfmsub231sd (%rdx), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmsub231sd (%rdx), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0xbb,0x0a]
+vfmsub231sd (%rdx), %xmm1, %xmm1
+
+// CHECK: vfmsub231sd (%rdx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0xbb,0x0a]
+vfmsub231sd (%rdx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub231sd (%rdx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0xbb,0x0a]
+vfmsub231sd (%rdx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub231sd {rn-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x52,0x85,0x18,0xbb,0xff]
+vfmsub231sd {rn-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vfmsub231sd {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x85,0x1a,0xbb,0xff]
+vfmsub231sd {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmsub231sd {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x85,0x9a,0xbb,0xff]
+vfmsub231sd {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmsub231sd {rn-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x18,0xbb,0xc9]
+vfmsub231sd {rn-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfmsub231sd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x1a,0xbb,0xc9]
+vfmsub231sd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub231sd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x9a,0xbb,0xc9]
+vfmsub231sd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub231sd {ru-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x52,0x85,0x58,0xbb,0xff]
+vfmsub231sd {ru-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vfmsub231sd {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x85,0x5a,0xbb,0xff]
+vfmsub231sd {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmsub231sd {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x85,0xda,0xbb,0xff]
+vfmsub231sd {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmsub231sd {ru-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x58,0xbb,0xc9]
+vfmsub231sd {ru-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfmsub231sd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x5a,0xbb,0xc9]
+vfmsub231sd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub231sd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0xda,0xbb,0xc9]
+vfmsub231sd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub231sd {rz-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x52,0x85,0x78,0xbb,0xff]
+vfmsub231sd {rz-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vfmsub231sd {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x85,0x7a,0xbb,0xff]
+vfmsub231sd {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmsub231sd {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x85,0xfa,0xbb,0xff]
+vfmsub231sd {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmsub231sd {rz-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x78,0xbb,0xc9]
+vfmsub231sd {rz-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfmsub231sd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x7a,0xbb,0xc9]
+vfmsub231sd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub231sd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0xfa,0xbb,0xc9]
+vfmsub231sd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub231sd %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x42,0x81,0xbb,0xff]
+vfmsub231sd %xmm15, %xmm15, %xmm15
+
+// CHECK: vfmsub231sd %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x85,0x0a,0xbb,0xff]
+vfmsub231sd %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmsub231sd %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x85,0x8a,0xbb,0xff]
+vfmsub231sd %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmsub231sd %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0xbb,0xc9]
+vfmsub231sd %xmm1, %xmm1, %xmm1
+
+// CHECK: vfmsub231sd %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0xbb,0xc9]
+vfmsub231sd %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub231sd %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0xbb,0xc9]
+vfmsub231sd %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub231ss 256(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x01,0xbb,0xbc,0x82,0x00,0x01,0x00,0x00]
+vfmsub231ss 256(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vfmsub231ss -256(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x01,0xbb,0xbc,0x82,0x00,0xff,0xff,0xff]
+vfmsub231ss -256(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vfmsub231ss 256(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x05,0x0a,0xbb,0x7c,0x82,0x40]
+vfmsub231ss 256(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmsub231ss -256(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x05,0x0a,0xbb,0x7c,0x82,0xc0]
+vfmsub231ss -256(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmsub231ss 256(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x05,0x8a,0xbb,0x7c,0x82,0x40]
+vfmsub231ss 256(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmsub231ss -256(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x05,0x8a,0xbb,0x7c,0x82,0xc0]
+vfmsub231ss -256(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmsub231ss 256(%rdx,%rax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0xbb,0x8c,0x82,0x00,0x01,0x00,0x00]
+vfmsub231ss 256(%rdx,%rax,4), %xmm1, %xmm1
+
+// CHECK: vfmsub231ss -256(%rdx,%rax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0xbb,0x8c,0x82,0x00,0xff,0xff,0xff]
+vfmsub231ss -256(%rdx,%rax,4), %xmm1, %xmm1
+
+// CHECK: vfmsub231ss 256(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0xbb,0x4c,0x82,0x40]
+vfmsub231ss 256(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub231ss -256(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0xbb,0x4c,0x82,0xc0]
+vfmsub231ss -256(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub231ss 256(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0xbb,0x4c,0x82,0x40]
+vfmsub231ss 256(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub231ss -256(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0xbb,0x4c,0x82,0xc0]
+vfmsub231ss -256(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub231ss 256(%rdx,%rax), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x01,0xbb,0xbc,0x02,0x00,0x01,0x00,0x00]
+vfmsub231ss 256(%rdx,%rax), %xmm15, %xmm15
+
+// CHECK: vfmsub231ss 256(%rdx,%rax), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x05,0x0a,0xbb,0x7c,0x02,0x40]
+vfmsub231ss 256(%rdx,%rax), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmsub231ss 256(%rdx,%rax), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x05,0x8a,0xbb,0x7c,0x02,0x40]
+vfmsub231ss 256(%rdx,%rax), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmsub231ss 256(%rdx,%rax), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0xbb,0x8c,0x02,0x00,0x01,0x00,0x00]
+vfmsub231ss 256(%rdx,%rax), %xmm1, %xmm1
+
+// CHECK: vfmsub231ss 256(%rdx,%rax), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0xbb,0x4c,0x02,0x40]
+vfmsub231ss 256(%rdx,%rax), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub231ss 256(%rdx,%rax), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0xbb,0x4c,0x02,0x40]
+vfmsub231ss 256(%rdx,%rax), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub231ss 256(%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x01,0xbb,0xba,0x00,0x01,0x00,0x00]
+vfmsub231ss 256(%rdx), %xmm15, %xmm15
+
+// CHECK: vfmsub231ss 256(%rdx), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x05,0x0a,0xbb,0x7a,0x40]
+vfmsub231ss 256(%rdx), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmsub231ss 256(%rdx), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x05,0x8a,0xbb,0x7a,0x40]
+vfmsub231ss 256(%rdx), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmsub231ss 256(%rdx), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0xbb,0x8a,0x00,0x01,0x00,0x00]
+vfmsub231ss 256(%rdx), %xmm1, %xmm1
+
+// CHECK: vfmsub231ss 256(%rdx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0xbb,0x4a,0x40]
+vfmsub231ss 256(%rdx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub231ss 256(%rdx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0xbb,0x4a,0x40]
+vfmsub231ss 256(%rdx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub231ss 485498096, %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x01,0xbb,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfmsub231ss 485498096, %xmm15, %xmm15
+
+// CHECK: vfmsub231ss 485498096, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x05,0x0a,0xbb,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfmsub231ss 485498096, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmsub231ss 485498096, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x05,0x8a,0xbb,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfmsub231ss 485498096, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmsub231ss 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0xbb,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfmsub231ss 485498096, %xmm1, %xmm1
+
+// CHECK: vfmsub231ss 485498096, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0xbb,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfmsub231ss 485498096, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub231ss 485498096, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0xbb,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfmsub231ss 485498096, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub231ss {rd-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x52,0x05,0x38,0xbb,0xff]
+vfmsub231ss {rd-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vfmsub231ss {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x05,0x3a,0xbb,0xff]
+vfmsub231ss {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmsub231ss {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x05,0xba,0xbb,0xff]
+vfmsub231ss {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmsub231ss {rd-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x38,0xbb,0xc9]
+vfmsub231ss {rd-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfmsub231ss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x3a,0xbb,0xc9]
+vfmsub231ss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub231ss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0xba,0xbb,0xc9]
+vfmsub231ss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub231ss (%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x01,0xbb,0x3a]
+vfmsub231ss (%rdx), %xmm15, %xmm15
+
+// CHECK: vfmsub231ss (%rdx), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x05,0x0a,0xbb,0x3a]
+vfmsub231ss (%rdx), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmsub231ss (%rdx), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x05,0x8a,0xbb,0x3a]
+vfmsub231ss (%rdx), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmsub231ss (%rdx), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0xbb,0x0a]
+vfmsub231ss (%rdx), %xmm1, %xmm1
+
+// CHECK: vfmsub231ss (%rdx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0xbb,0x0a]
+vfmsub231ss (%rdx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub231ss (%rdx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0xbb,0x0a]
+vfmsub231ss (%rdx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub231ss {rn-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x52,0x05,0x18,0xbb,0xff]
+vfmsub231ss {rn-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vfmsub231ss {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x05,0x1a,0xbb,0xff]
+vfmsub231ss {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmsub231ss {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x05,0x9a,0xbb,0xff]
+vfmsub231ss {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmsub231ss {rn-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x18,0xbb,0xc9]
+vfmsub231ss {rn-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfmsub231ss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x1a,0xbb,0xc9]
+vfmsub231ss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub231ss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x9a,0xbb,0xc9]
+vfmsub231ss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub231ss {ru-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x52,0x05,0x58,0xbb,0xff]
+vfmsub231ss {ru-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vfmsub231ss {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x05,0x5a,0xbb,0xff]
+vfmsub231ss {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmsub231ss {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x05,0xda,0xbb,0xff]
+vfmsub231ss {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmsub231ss {ru-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x58,0xbb,0xc9]
+vfmsub231ss {ru-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfmsub231ss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x5a,0xbb,0xc9]
+vfmsub231ss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub231ss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0xda,0xbb,0xc9]
+vfmsub231ss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub231ss {rz-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x52,0x05,0x78,0xbb,0xff]
+vfmsub231ss {rz-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vfmsub231ss {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x05,0x7a,0xbb,0xff]
+vfmsub231ss {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmsub231ss {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x05,0xfa,0xbb,0xff]
+vfmsub231ss {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmsub231ss {rz-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x78,0xbb,0xc9]
+vfmsub231ss {rz-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfmsub231ss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x7a,0xbb,0xc9]
+vfmsub231ss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub231ss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0xfa,0xbb,0xc9]
+vfmsub231ss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfmsub231ss %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x42,0x01,0xbb,0xff]
+vfmsub231ss %xmm15, %xmm15, %xmm15
+
+// CHECK: vfmsub231ss %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x05,0x0a,0xbb,0xff]
+vfmsub231ss %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfmsub231ss %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x05,0x8a,0xbb,0xff]
+vfmsub231ss %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfmsub231ss %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0xbb,0xc9]
+vfmsub231ss %xmm1, %xmm1, %xmm1
+
+// CHECK: vfmsub231ss %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0xbb,0xc9]
+vfmsub231ss %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfmsub231ss %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0xbb,0xc9]
+vfmsub231ss %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd132sd 485498096, %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x81,0x9d,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfnmadd132sd 485498096, %xmm15, %xmm15
+
+// CHECK: vfnmadd132sd 485498096, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x85,0x0a,0x9d,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfnmadd132sd 485498096, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfnmadd132sd 485498096, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x85,0x8a,0x9d,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfnmadd132sd 485498096, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfnmadd132sd 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0x9d,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfnmadd132sd 485498096, %xmm1, %xmm1
+
+// CHECK: vfnmadd132sd 485498096, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0x9d,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfnmadd132sd 485498096, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd132sd 485498096, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0x9d,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfnmadd132sd 485498096, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd132sd 512(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x81,0x9d,0xbc,0x82,0x00,0x02,0x00,0x00]
+vfnmadd132sd 512(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vfnmadd132sd -512(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x81,0x9d,0xbc,0x82,0x00,0xfe,0xff,0xff]
+vfnmadd132sd -512(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vfnmadd132sd 512(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x85,0x0a,0x9d,0x7c,0x82,0x40]
+vfnmadd132sd 512(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfnmadd132sd -512(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x85,0x0a,0x9d,0x7c,0x82,0xc0]
+vfnmadd132sd -512(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfnmadd132sd 512(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x85,0x8a,0x9d,0x7c,0x82,0x40]
+vfnmadd132sd 512(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfnmadd132sd -512(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x85,0x8a,0x9d,0x7c,0x82,0xc0]
+vfnmadd132sd -512(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfnmadd132sd 512(%rdx,%rax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0x9d,0x8c,0x82,0x00,0x02,0x00,0x00]
+vfnmadd132sd 512(%rdx,%rax,4), %xmm1, %xmm1
+
+// CHECK: vfnmadd132sd -512(%rdx,%rax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0x9d,0x8c,0x82,0x00,0xfe,0xff,0xff]
+vfnmadd132sd -512(%rdx,%rax,4), %xmm1, %xmm1
+
+// CHECK: vfnmadd132sd 512(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0x9d,0x4c,0x82,0x40]
+vfnmadd132sd 512(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd132sd -512(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0x9d,0x4c,0x82,0xc0]
+vfnmadd132sd -512(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd132sd 512(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0x9d,0x4c,0x82,0x40]
+vfnmadd132sd 512(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd132sd -512(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0x9d,0x4c,0x82,0xc0]
+vfnmadd132sd -512(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd132sd 512(%rdx,%rax), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x81,0x9d,0xbc,0x02,0x00,0x02,0x00,0x00]
+vfnmadd132sd 512(%rdx,%rax), %xmm15, %xmm15
+
+// CHECK: vfnmadd132sd 512(%rdx,%rax), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x85,0x0a,0x9d,0x7c,0x02,0x40]
+vfnmadd132sd 512(%rdx,%rax), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfnmadd132sd 512(%rdx,%rax), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x85,0x8a,0x9d,0x7c,0x02,0x40]
+vfnmadd132sd 512(%rdx,%rax), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfnmadd132sd 512(%rdx,%rax), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0x9d,0x8c,0x02,0x00,0x02,0x00,0x00]
+vfnmadd132sd 512(%rdx,%rax), %xmm1, %xmm1
+
+// CHECK: vfnmadd132sd 512(%rdx,%rax), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0x9d,0x4c,0x02,0x40]
+vfnmadd132sd 512(%rdx,%rax), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd132sd 512(%rdx,%rax), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0x9d,0x4c,0x02,0x40]
+vfnmadd132sd 512(%rdx,%rax), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd132sd 512(%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x81,0x9d,0xba,0x00,0x02,0x00,0x00]
+vfnmadd132sd 512(%rdx), %xmm15, %xmm15
+
+// CHECK: vfnmadd132sd 512(%rdx), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x85,0x0a,0x9d,0x7a,0x40]
+vfnmadd132sd 512(%rdx), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfnmadd132sd 512(%rdx), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x85,0x8a,0x9d,0x7a,0x40]
+vfnmadd132sd 512(%rdx), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfnmadd132sd 512(%rdx), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0x9d,0x8a,0x00,0x02,0x00,0x00]
+vfnmadd132sd 512(%rdx), %xmm1, %xmm1
+
+// CHECK: vfnmadd132sd 512(%rdx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0x9d,0x4a,0x40]
+vfnmadd132sd 512(%rdx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd132sd 512(%rdx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0x9d,0x4a,0x40]
+vfnmadd132sd 512(%rdx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd132sd {rd-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x52,0x85,0x38,0x9d,0xff]
+vfnmadd132sd {rd-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vfnmadd132sd {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x85,0x3a,0x9d,0xff]
+vfnmadd132sd {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfnmadd132sd {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x85,0xba,0x9d,0xff]
+vfnmadd132sd {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfnmadd132sd {rd-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x38,0x9d,0xc9]
+vfnmadd132sd {rd-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfnmadd132sd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x3a,0x9d,0xc9]
+vfnmadd132sd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd132sd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0xba,0x9d,0xc9]
+vfnmadd132sd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd132sd (%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x81,0x9d,0x3a]
+vfnmadd132sd (%rdx), %xmm15, %xmm15
+
+// CHECK: vfnmadd132sd (%rdx), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x85,0x0a,0x9d,0x3a]
+vfnmadd132sd (%rdx), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfnmadd132sd (%rdx), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x85,0x8a,0x9d,0x3a]
+vfnmadd132sd (%rdx), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfnmadd132sd (%rdx), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0x9d,0x0a]
+vfnmadd132sd (%rdx), %xmm1, %xmm1
+
+// CHECK: vfnmadd132sd (%rdx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0x9d,0x0a]
+vfnmadd132sd (%rdx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd132sd (%rdx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0x9d,0x0a]
+vfnmadd132sd (%rdx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd132sd {rn-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x52,0x85,0x18,0x9d,0xff]
+vfnmadd132sd {rn-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vfnmadd132sd {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x85,0x1a,0x9d,0xff]
+vfnmadd132sd {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfnmadd132sd {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x85,0x9a,0x9d,0xff]
+vfnmadd132sd {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfnmadd132sd {rn-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x18,0x9d,0xc9]
+vfnmadd132sd {rn-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfnmadd132sd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x1a,0x9d,0xc9]
+vfnmadd132sd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd132sd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x9a,0x9d,0xc9]
+vfnmadd132sd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd132sd {ru-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x52,0x85,0x58,0x9d,0xff]
+vfnmadd132sd {ru-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vfnmadd132sd {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x85,0x5a,0x9d,0xff]
+vfnmadd132sd {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfnmadd132sd {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x85,0xda,0x9d,0xff]
+vfnmadd132sd {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfnmadd132sd {ru-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x58,0x9d,0xc9]
+vfnmadd132sd {ru-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfnmadd132sd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x5a,0x9d,0xc9]
+vfnmadd132sd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd132sd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0xda,0x9d,0xc9]
+vfnmadd132sd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd132sd {rz-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x52,0x85,0x78,0x9d,0xff]
+vfnmadd132sd {rz-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vfnmadd132sd {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x85,0x7a,0x9d,0xff]
+vfnmadd132sd {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfnmadd132sd {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x85,0xfa,0x9d,0xff]
+vfnmadd132sd {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfnmadd132sd {rz-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x78,0x9d,0xc9]
+vfnmadd132sd {rz-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfnmadd132sd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x7a,0x9d,0xc9]
+vfnmadd132sd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd132sd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0xfa,0x9d,0xc9]
+vfnmadd132sd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd132sd %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x42,0x81,0x9d,0xff]
+vfnmadd132sd %xmm15, %xmm15, %xmm15
+
+// CHECK: vfnmadd132sd %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x85,0x0a,0x9d,0xff]
+vfnmadd132sd %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfnmadd132sd %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x85,0x8a,0x9d,0xff]
+vfnmadd132sd %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfnmadd132sd %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0x9d,0xc9]
+vfnmadd132sd %xmm1, %xmm1, %xmm1
+
+// CHECK: vfnmadd132sd %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0x9d,0xc9]
+vfnmadd132sd %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd132sd %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0x9d,0xc9]
+vfnmadd132sd %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd132ss 256(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x01,0x9d,0xbc,0x82,0x00,0x01,0x00,0x00]
+vfnmadd132ss 256(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vfnmadd132ss -256(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x01,0x9d,0xbc,0x82,0x00,0xff,0xff,0xff]
+vfnmadd132ss -256(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vfnmadd132ss 256(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x05,0x0a,0x9d,0x7c,0x82,0x40]
+vfnmadd132ss 256(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfnmadd132ss -256(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x05,0x0a,0x9d,0x7c,0x82,0xc0]
+vfnmadd132ss -256(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfnmadd132ss 256(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x05,0x8a,0x9d,0x7c,0x82,0x40]
+vfnmadd132ss 256(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfnmadd132ss -256(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x05,0x8a,0x9d,0x7c,0x82,0xc0]
+vfnmadd132ss -256(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfnmadd132ss 256(%rdx,%rax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0x9d,0x8c,0x82,0x00,0x01,0x00,0x00]
+vfnmadd132ss 256(%rdx,%rax,4), %xmm1, %xmm1
+
+// CHECK: vfnmadd132ss -256(%rdx,%rax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0x9d,0x8c,0x82,0x00,0xff,0xff,0xff]
+vfnmadd132ss -256(%rdx,%rax,4), %xmm1, %xmm1
+
+// CHECK: vfnmadd132ss 256(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0x9d,0x4c,0x82,0x40]
+vfnmadd132ss 256(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd132ss -256(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0x9d,0x4c,0x82,0xc0]
+vfnmadd132ss -256(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd132ss 256(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0x9d,0x4c,0x82,0x40]
+vfnmadd132ss 256(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd132ss -256(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0x9d,0x4c,0x82,0xc0]
+vfnmadd132ss -256(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd132ss 256(%rdx,%rax), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x01,0x9d,0xbc,0x02,0x00,0x01,0x00,0x00]
+vfnmadd132ss 256(%rdx,%rax), %xmm15, %xmm15
+
+// CHECK: vfnmadd132ss 256(%rdx,%rax), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x05,0x0a,0x9d,0x7c,0x02,0x40]
+vfnmadd132ss 256(%rdx,%rax), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfnmadd132ss 256(%rdx,%rax), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x05,0x8a,0x9d,0x7c,0x02,0x40]
+vfnmadd132ss 256(%rdx,%rax), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfnmadd132ss 256(%rdx,%rax), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0x9d,0x8c,0x02,0x00,0x01,0x00,0x00]
+vfnmadd132ss 256(%rdx,%rax), %xmm1, %xmm1
+
+// CHECK: vfnmadd132ss 256(%rdx,%rax), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0x9d,0x4c,0x02,0x40]
+vfnmadd132ss 256(%rdx,%rax), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd132ss 256(%rdx,%rax), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0x9d,0x4c,0x02,0x40]
+vfnmadd132ss 256(%rdx,%rax), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd132ss 256(%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x01,0x9d,0xba,0x00,0x01,0x00,0x00]
+vfnmadd132ss 256(%rdx), %xmm15, %xmm15
+
+// CHECK: vfnmadd132ss 256(%rdx), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x05,0x0a,0x9d,0x7a,0x40]
+vfnmadd132ss 256(%rdx), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfnmadd132ss 256(%rdx), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x05,0x8a,0x9d,0x7a,0x40]
+vfnmadd132ss 256(%rdx), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfnmadd132ss 256(%rdx), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0x9d,0x8a,0x00,0x01,0x00,0x00]
+vfnmadd132ss 256(%rdx), %xmm1, %xmm1
+
+// CHECK: vfnmadd132ss 256(%rdx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0x9d,0x4a,0x40]
+vfnmadd132ss 256(%rdx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd132ss 256(%rdx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0x9d,0x4a,0x40]
+vfnmadd132ss 256(%rdx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd132ss 485498096, %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x01,0x9d,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfnmadd132ss 485498096, %xmm15, %xmm15
+
+// CHECK: vfnmadd132ss 485498096, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x05,0x0a,0x9d,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfnmadd132ss 485498096, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfnmadd132ss 485498096, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x05,0x8a,0x9d,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfnmadd132ss 485498096, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfnmadd132ss 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0x9d,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfnmadd132ss 485498096, %xmm1, %xmm1
+
+// CHECK: vfnmadd132ss 485498096, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0x9d,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfnmadd132ss 485498096, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd132ss 485498096, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0x9d,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfnmadd132ss 485498096, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd132ss {rd-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x52,0x05,0x38,0x9d,0xff]
+vfnmadd132ss {rd-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vfnmadd132ss {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x05,0x3a,0x9d,0xff]
+vfnmadd132ss {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfnmadd132ss {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x05,0xba,0x9d,0xff]
+vfnmadd132ss {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfnmadd132ss {rd-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x38,0x9d,0xc9]
+vfnmadd132ss {rd-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfnmadd132ss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x3a,0x9d,0xc9]
+vfnmadd132ss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd132ss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0xba,0x9d,0xc9]
+vfnmadd132ss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd132ss (%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x01,0x9d,0x3a]
+vfnmadd132ss (%rdx), %xmm15, %xmm15
+
+// CHECK: vfnmadd132ss (%rdx), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x05,0x0a,0x9d,0x3a]
+vfnmadd132ss (%rdx), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfnmadd132ss (%rdx), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x05,0x8a,0x9d,0x3a]
+vfnmadd132ss (%rdx), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfnmadd132ss (%rdx), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0x9d,0x0a]
+vfnmadd132ss (%rdx), %xmm1, %xmm1
+
+// CHECK: vfnmadd132ss (%rdx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0x9d,0x0a]
+vfnmadd132ss (%rdx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd132ss (%rdx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0x9d,0x0a]
+vfnmadd132ss (%rdx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd132ss {rn-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x52,0x05,0x18,0x9d,0xff]
+vfnmadd132ss {rn-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vfnmadd132ss {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x05,0x1a,0x9d,0xff]
+vfnmadd132ss {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfnmadd132ss {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x05,0x9a,0x9d,0xff]
+vfnmadd132ss {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfnmadd132ss {rn-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x18,0x9d,0xc9]
+vfnmadd132ss {rn-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfnmadd132ss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x1a,0x9d,0xc9]
+vfnmadd132ss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd132ss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x9a,0x9d,0xc9]
+vfnmadd132ss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd132ss {ru-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x52,0x05,0x58,0x9d,0xff]
+vfnmadd132ss {ru-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vfnmadd132ss {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x05,0x5a,0x9d,0xff]
+vfnmadd132ss {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfnmadd132ss {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x05,0xda,0x9d,0xff]
+vfnmadd132ss {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfnmadd132ss {ru-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x58,0x9d,0xc9]
+vfnmadd132ss {ru-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfnmadd132ss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x5a,0x9d,0xc9]
+vfnmadd132ss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd132ss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0xda,0x9d,0xc9]
+vfnmadd132ss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd132ss {rz-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x52,0x05,0x78,0x9d,0xff]
+vfnmadd132ss {rz-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vfnmadd132ss {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x05,0x7a,0x9d,0xff]
+vfnmadd132ss {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfnmadd132ss {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x05,0xfa,0x9d,0xff]
+vfnmadd132ss {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfnmadd132ss {rz-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x78,0x9d,0xc9]
+vfnmadd132ss {rz-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfnmadd132ss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x7a,0x9d,0xc9]
+vfnmadd132ss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd132ss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0xfa,0x9d,0xc9]
+vfnmadd132ss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd132ss %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x42,0x01,0x9d,0xff]
+vfnmadd132ss %xmm15, %xmm15, %xmm15
+
+// CHECK: vfnmadd132ss %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x05,0x0a,0x9d,0xff]
+vfnmadd132ss %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfnmadd132ss %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x05,0x8a,0x9d,0xff]
+vfnmadd132ss %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfnmadd132ss %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0x9d,0xc9]
+vfnmadd132ss %xmm1, %xmm1, %xmm1
+
+// CHECK: vfnmadd132ss %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0x9d,0xc9]
+vfnmadd132ss %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd132ss %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0x9d,0xc9]
+vfnmadd132ss %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd213sd 485498096, %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x81,0xad,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfnmadd213sd 485498096, %xmm15, %xmm15
+
+// CHECK: vfnmadd213sd 485498096, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x85,0x0a,0xad,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfnmadd213sd 485498096, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfnmadd213sd 485498096, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x85,0x8a,0xad,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfnmadd213sd 485498096, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfnmadd213sd 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0xad,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfnmadd213sd 485498096, %xmm1, %xmm1
+
+// CHECK: vfnmadd213sd 485498096, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0xad,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfnmadd213sd 485498096, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd213sd 485498096, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0xad,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfnmadd213sd 485498096, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd213sd 512(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x81,0xad,0xbc,0x82,0x00,0x02,0x00,0x00]
+vfnmadd213sd 512(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vfnmadd213sd -512(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x81,0xad,0xbc,0x82,0x00,0xfe,0xff,0xff]
+vfnmadd213sd -512(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vfnmadd213sd 512(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x85,0x0a,0xad,0x7c,0x82,0x40]
+vfnmadd213sd 512(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfnmadd213sd -512(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x85,0x0a,0xad,0x7c,0x82,0xc0]
+vfnmadd213sd -512(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfnmadd213sd 512(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x85,0x8a,0xad,0x7c,0x82,0x40]
+vfnmadd213sd 512(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfnmadd213sd -512(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x85,0x8a,0xad,0x7c,0x82,0xc0]
+vfnmadd213sd -512(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfnmadd213sd 512(%rdx,%rax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0xad,0x8c,0x82,0x00,0x02,0x00,0x00]
+vfnmadd213sd 512(%rdx,%rax,4), %xmm1, %xmm1
+
+// CHECK: vfnmadd213sd -512(%rdx,%rax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0xad,0x8c,0x82,0x00,0xfe,0xff,0xff]
+vfnmadd213sd -512(%rdx,%rax,4), %xmm1, %xmm1
+
+// CHECK: vfnmadd213sd 512(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0xad,0x4c,0x82,0x40]
+vfnmadd213sd 512(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd213sd -512(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0xad,0x4c,0x82,0xc0]
+vfnmadd213sd -512(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd213sd 512(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0xad,0x4c,0x82,0x40]
+vfnmadd213sd 512(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd213sd -512(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0xad,0x4c,0x82,0xc0]
+vfnmadd213sd -512(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd213sd 512(%rdx,%rax), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x81,0xad,0xbc,0x02,0x00,0x02,0x00,0x00]
+vfnmadd213sd 512(%rdx,%rax), %xmm15, %xmm15
+
+// CHECK: vfnmadd213sd 512(%rdx,%rax), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x85,0x0a,0xad,0x7c,0x02,0x40]
+vfnmadd213sd 512(%rdx,%rax), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfnmadd213sd 512(%rdx,%rax), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x85,0x8a,0xad,0x7c,0x02,0x40]
+vfnmadd213sd 512(%rdx,%rax), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfnmadd213sd 512(%rdx,%rax), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0xad,0x8c,0x02,0x00,0x02,0x00,0x00]
+vfnmadd213sd 512(%rdx,%rax), %xmm1, %xmm1
+
+// CHECK: vfnmadd213sd 512(%rdx,%rax), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0xad,0x4c,0x02,0x40]
+vfnmadd213sd 512(%rdx,%rax), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd213sd 512(%rdx,%rax), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0xad,0x4c,0x02,0x40]
+vfnmadd213sd 512(%rdx,%rax), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd213sd 512(%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x81,0xad,0xba,0x00,0x02,0x00,0x00]
+vfnmadd213sd 512(%rdx), %xmm15, %xmm15
+
+// CHECK: vfnmadd213sd 512(%rdx), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x85,0x0a,0xad,0x7a,0x40]
+vfnmadd213sd 512(%rdx), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfnmadd213sd 512(%rdx), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x85,0x8a,0xad,0x7a,0x40]
+vfnmadd213sd 512(%rdx), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfnmadd213sd 512(%rdx), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0xad,0x8a,0x00,0x02,0x00,0x00]
+vfnmadd213sd 512(%rdx), %xmm1, %xmm1
+
+// CHECK: vfnmadd213sd 512(%rdx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0xad,0x4a,0x40]
+vfnmadd213sd 512(%rdx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd213sd 512(%rdx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0xad,0x4a,0x40]
+vfnmadd213sd 512(%rdx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd213sd {rd-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x52,0x85,0x38,0xad,0xff]
+vfnmadd213sd {rd-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vfnmadd213sd {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x85,0x3a,0xad,0xff]
+vfnmadd213sd {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfnmadd213sd {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x85,0xba,0xad,0xff]
+vfnmadd213sd {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfnmadd213sd {rd-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x38,0xad,0xc9]
+vfnmadd213sd {rd-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfnmadd213sd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x3a,0xad,0xc9]
+vfnmadd213sd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd213sd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0xba,0xad,0xc9]
+vfnmadd213sd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd213sd (%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x81,0xad,0x3a]
+vfnmadd213sd (%rdx), %xmm15, %xmm15
+
+// CHECK: vfnmadd213sd (%rdx), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x85,0x0a,0xad,0x3a]
+vfnmadd213sd (%rdx), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfnmadd213sd (%rdx), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x85,0x8a,0xad,0x3a]
+vfnmadd213sd (%rdx), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfnmadd213sd (%rdx), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0xad,0x0a]
+vfnmadd213sd (%rdx), %xmm1, %xmm1
+
+// CHECK: vfnmadd213sd (%rdx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0xad,0x0a]
+vfnmadd213sd (%rdx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd213sd (%rdx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0xad,0x0a]
+vfnmadd213sd (%rdx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd213sd {rn-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x52,0x85,0x18,0xad,0xff]
+vfnmadd213sd {rn-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vfnmadd213sd {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x85,0x1a,0xad,0xff]
+vfnmadd213sd {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfnmadd213sd {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x85,0x9a,0xad,0xff]
+vfnmadd213sd {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfnmadd213sd {rn-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x18,0xad,0xc9]
+vfnmadd213sd {rn-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfnmadd213sd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x1a,0xad,0xc9]
+vfnmadd213sd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd213sd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x9a,0xad,0xc9]
+vfnmadd213sd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd213sd {ru-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x52,0x85,0x58,0xad,0xff]
+vfnmadd213sd {ru-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vfnmadd213sd {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x85,0x5a,0xad,0xff]
+vfnmadd213sd {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfnmadd213sd {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x85,0xda,0xad,0xff]
+vfnmadd213sd {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfnmadd213sd {ru-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x58,0xad,0xc9]
+vfnmadd213sd {ru-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfnmadd213sd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x5a,0xad,0xc9]
+vfnmadd213sd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd213sd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0xda,0xad,0xc9]
+vfnmadd213sd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd213sd {rz-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x52,0x85,0x78,0xad,0xff]
+vfnmadd213sd {rz-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vfnmadd213sd {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x85,0x7a,0xad,0xff]
+vfnmadd213sd {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfnmadd213sd {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x85,0xfa,0xad,0xff]
+vfnmadd213sd {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfnmadd213sd {rz-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x78,0xad,0xc9]
+vfnmadd213sd {rz-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfnmadd213sd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x7a,0xad,0xc9]
+vfnmadd213sd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd213sd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0xfa,0xad,0xc9]
+vfnmadd213sd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd213sd %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x42,0x81,0xad,0xff]
+vfnmadd213sd %xmm15, %xmm15, %xmm15
+
+// CHECK: vfnmadd213sd %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x85,0x0a,0xad,0xff]
+vfnmadd213sd %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfnmadd213sd %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x85,0x8a,0xad,0xff]
+vfnmadd213sd %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfnmadd213sd %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0xad,0xc9]
+vfnmadd213sd %xmm1, %xmm1, %xmm1
+
+// CHECK: vfnmadd213sd %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0xad,0xc9]
+vfnmadd213sd %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd213sd %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0xad,0xc9]
+vfnmadd213sd %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd213ss 256(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x01,0xad,0xbc,0x82,0x00,0x01,0x00,0x00]
+vfnmadd213ss 256(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vfnmadd213ss -256(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x01,0xad,0xbc,0x82,0x00,0xff,0xff,0xff]
+vfnmadd213ss -256(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vfnmadd213ss 256(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x05,0x0a,0xad,0x7c,0x82,0x40]
+vfnmadd213ss 256(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfnmadd213ss -256(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x05,0x0a,0xad,0x7c,0x82,0xc0]
+vfnmadd213ss -256(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfnmadd213ss 256(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x05,0x8a,0xad,0x7c,0x82,0x40]
+vfnmadd213ss 256(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfnmadd213ss -256(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x05,0x8a,0xad,0x7c,0x82,0xc0]
+vfnmadd213ss -256(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfnmadd213ss 256(%rdx,%rax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0xad,0x8c,0x82,0x00,0x01,0x00,0x00]
+vfnmadd213ss 256(%rdx,%rax,4), %xmm1, %xmm1
+
+// CHECK: vfnmadd213ss -256(%rdx,%rax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0xad,0x8c,0x82,0x00,0xff,0xff,0xff]
+vfnmadd213ss -256(%rdx,%rax,4), %xmm1, %xmm1
+
+// CHECK: vfnmadd213ss 256(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0xad,0x4c,0x82,0x40]
+vfnmadd213ss 256(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd213ss -256(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0xad,0x4c,0x82,0xc0]
+vfnmadd213ss -256(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd213ss 256(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0xad,0x4c,0x82,0x40]
+vfnmadd213ss 256(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd213ss -256(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0xad,0x4c,0x82,0xc0]
+vfnmadd213ss -256(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd213ss 256(%rdx,%rax), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x01,0xad,0xbc,0x02,0x00,0x01,0x00,0x00]
+vfnmadd213ss 256(%rdx,%rax), %xmm15, %xmm15
+
+// CHECK: vfnmadd213ss 256(%rdx,%rax), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x05,0x0a,0xad,0x7c,0x02,0x40]
+vfnmadd213ss 256(%rdx,%rax), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfnmadd213ss 256(%rdx,%rax), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x05,0x8a,0xad,0x7c,0x02,0x40]
+vfnmadd213ss 256(%rdx,%rax), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfnmadd213ss 256(%rdx,%rax), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0xad,0x8c,0x02,0x00,0x01,0x00,0x00]
+vfnmadd213ss 256(%rdx,%rax), %xmm1, %xmm1
+
+// CHECK: vfnmadd213ss 256(%rdx,%rax), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0xad,0x4c,0x02,0x40]
+vfnmadd213ss 256(%rdx,%rax), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd213ss 256(%rdx,%rax), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0xad,0x4c,0x02,0x40]
+vfnmadd213ss 256(%rdx,%rax), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd213ss 256(%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x01,0xad,0xba,0x00,0x01,0x00,0x00]
+vfnmadd213ss 256(%rdx), %xmm15, %xmm15
+
+// CHECK: vfnmadd213ss 256(%rdx), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x05,0x0a,0xad,0x7a,0x40]
+vfnmadd213ss 256(%rdx), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfnmadd213ss 256(%rdx), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x05,0x8a,0xad,0x7a,0x40]
+vfnmadd213ss 256(%rdx), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfnmadd213ss 256(%rdx), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0xad,0x8a,0x00,0x01,0x00,0x00]
+vfnmadd213ss 256(%rdx), %xmm1, %xmm1
+
+// CHECK: vfnmadd213ss 256(%rdx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0xad,0x4a,0x40]
+vfnmadd213ss 256(%rdx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd213ss 256(%rdx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0xad,0x4a,0x40]
+vfnmadd213ss 256(%rdx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd213ss 485498096, %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x01,0xad,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfnmadd213ss 485498096, %xmm15, %xmm15
+
+// CHECK: vfnmadd213ss 485498096, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x05,0x0a,0xad,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfnmadd213ss 485498096, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfnmadd213ss 485498096, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x05,0x8a,0xad,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfnmadd213ss 485498096, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfnmadd213ss 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0xad,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfnmadd213ss 485498096, %xmm1, %xmm1
+
+// CHECK: vfnmadd213ss 485498096, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0xad,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfnmadd213ss 485498096, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd213ss 485498096, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0xad,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfnmadd213ss 485498096, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd213ss {rd-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x52,0x05,0x38,0xad,0xff]
+vfnmadd213ss {rd-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vfnmadd213ss {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x05,0x3a,0xad,0xff]
+vfnmadd213ss {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfnmadd213ss {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x05,0xba,0xad,0xff]
+vfnmadd213ss {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfnmadd213ss {rd-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x38,0xad,0xc9]
+vfnmadd213ss {rd-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfnmadd213ss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x3a,0xad,0xc9]
+vfnmadd213ss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd213ss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0xba,0xad,0xc9]
+vfnmadd213ss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd213ss (%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x01,0xad,0x3a]
+vfnmadd213ss (%rdx), %xmm15, %xmm15
+
+// CHECK: vfnmadd213ss (%rdx), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x05,0x0a,0xad,0x3a]
+vfnmadd213ss (%rdx), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfnmadd213ss (%rdx), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x05,0x8a,0xad,0x3a]
+vfnmadd213ss (%rdx), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfnmadd213ss (%rdx), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0xad,0x0a]
+vfnmadd213ss (%rdx), %xmm1, %xmm1
+
+// CHECK: vfnmadd213ss (%rdx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0xad,0x0a]
+vfnmadd213ss (%rdx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd213ss (%rdx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0xad,0x0a]
+vfnmadd213ss (%rdx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd213ss {rn-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x52,0x05,0x18,0xad,0xff]
+vfnmadd213ss {rn-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vfnmadd213ss {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x05,0x1a,0xad,0xff]
+vfnmadd213ss {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfnmadd213ss {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x05,0x9a,0xad,0xff]
+vfnmadd213ss {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfnmadd213ss {rn-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x18,0xad,0xc9]
+vfnmadd213ss {rn-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfnmadd213ss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x1a,0xad,0xc9]
+vfnmadd213ss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd213ss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x9a,0xad,0xc9]
+vfnmadd213ss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd213ss {ru-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x52,0x05,0x58,0xad,0xff]
+vfnmadd213ss {ru-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vfnmadd213ss {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x05,0x5a,0xad,0xff]
+vfnmadd213ss {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfnmadd213ss {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x05,0xda,0xad,0xff]
+vfnmadd213ss {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfnmadd213ss {ru-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x58,0xad,0xc9]
+vfnmadd213ss {ru-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfnmadd213ss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x5a,0xad,0xc9]
+vfnmadd213ss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd213ss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0xda,0xad,0xc9]
+vfnmadd213ss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd213ss {rz-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x52,0x05,0x78,0xad,0xff]
+vfnmadd213ss {rz-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vfnmadd213ss {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x05,0x7a,0xad,0xff]
+vfnmadd213ss {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfnmadd213ss {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x05,0xfa,0xad,0xff]
+vfnmadd213ss {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfnmadd213ss {rz-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x78,0xad,0xc9]
+vfnmadd213ss {rz-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfnmadd213ss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x7a,0xad,0xc9]
+vfnmadd213ss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd213ss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0xfa,0xad,0xc9]
+vfnmadd213ss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd213ss %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x42,0x01,0xad,0xff]
+vfnmadd213ss %xmm15, %xmm15, %xmm15
+
+// CHECK: vfnmadd213ss %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x05,0x0a,0xad,0xff]
+vfnmadd213ss %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfnmadd213ss %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x05,0x8a,0xad,0xff]
+vfnmadd213ss %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfnmadd213ss %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0xad,0xc9]
+vfnmadd213ss %xmm1, %xmm1, %xmm1
+
+// CHECK: vfnmadd213ss %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0xad,0xc9]
+vfnmadd213ss %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd213ss %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0xad,0xc9]
+vfnmadd213ss %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd231sd 485498096, %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x81,0xbd,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfnmadd231sd 485498096, %xmm15, %xmm15
+
+// CHECK: vfnmadd231sd 485498096, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x85,0x0a,0xbd,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfnmadd231sd 485498096, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfnmadd231sd 485498096, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x85,0x8a,0xbd,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfnmadd231sd 485498096, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfnmadd231sd 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0xbd,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfnmadd231sd 485498096, %xmm1, %xmm1
+
+// CHECK: vfnmadd231sd 485498096, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0xbd,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfnmadd231sd 485498096, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd231sd 485498096, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0xbd,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfnmadd231sd 485498096, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd231sd 512(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x81,0xbd,0xbc,0x82,0x00,0x02,0x00,0x00]
+vfnmadd231sd 512(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vfnmadd231sd -512(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x81,0xbd,0xbc,0x82,0x00,0xfe,0xff,0xff]
+vfnmadd231sd -512(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vfnmadd231sd 512(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x85,0x0a,0xbd,0x7c,0x82,0x40]
+vfnmadd231sd 512(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfnmadd231sd -512(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x85,0x0a,0xbd,0x7c,0x82,0xc0]
+vfnmadd231sd -512(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfnmadd231sd 512(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x85,0x8a,0xbd,0x7c,0x82,0x40]
+vfnmadd231sd 512(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfnmadd231sd -512(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x85,0x8a,0xbd,0x7c,0x82,0xc0]
+vfnmadd231sd -512(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfnmadd231sd 512(%rdx,%rax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0xbd,0x8c,0x82,0x00,0x02,0x00,0x00]
+vfnmadd231sd 512(%rdx,%rax,4), %xmm1, %xmm1
+
+// CHECK: vfnmadd231sd -512(%rdx,%rax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0xbd,0x8c,0x82,0x00,0xfe,0xff,0xff]
+vfnmadd231sd -512(%rdx,%rax,4), %xmm1, %xmm1
+
+// CHECK: vfnmadd231sd 512(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0xbd,0x4c,0x82,0x40]
+vfnmadd231sd 512(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd231sd -512(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0xbd,0x4c,0x82,0xc0]
+vfnmadd231sd -512(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd231sd 512(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0xbd,0x4c,0x82,0x40]
+vfnmadd231sd 512(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd231sd -512(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0xbd,0x4c,0x82,0xc0]
+vfnmadd231sd -512(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd231sd 512(%rdx,%rax), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x81,0xbd,0xbc,0x02,0x00,0x02,0x00,0x00]
+vfnmadd231sd 512(%rdx,%rax), %xmm15, %xmm15
+
+// CHECK: vfnmadd231sd 512(%rdx,%rax), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x85,0x0a,0xbd,0x7c,0x02,0x40]
+vfnmadd231sd 512(%rdx,%rax), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfnmadd231sd 512(%rdx,%rax), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x85,0x8a,0xbd,0x7c,0x02,0x40]
+vfnmadd231sd 512(%rdx,%rax), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfnmadd231sd 512(%rdx,%rax), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0xbd,0x8c,0x02,0x00,0x02,0x00,0x00]
+vfnmadd231sd 512(%rdx,%rax), %xmm1, %xmm1
+
+// CHECK: vfnmadd231sd 512(%rdx,%rax), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0xbd,0x4c,0x02,0x40]
+vfnmadd231sd 512(%rdx,%rax), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd231sd 512(%rdx,%rax), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0xbd,0x4c,0x02,0x40]
+vfnmadd231sd 512(%rdx,%rax), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd231sd 512(%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x81,0xbd,0xba,0x00,0x02,0x00,0x00]
+vfnmadd231sd 512(%rdx), %xmm15, %xmm15
+
+// CHECK: vfnmadd231sd 512(%rdx), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x85,0x0a,0xbd,0x7a,0x40]
+vfnmadd231sd 512(%rdx), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfnmadd231sd 512(%rdx), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x85,0x8a,0xbd,0x7a,0x40]
+vfnmadd231sd 512(%rdx), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfnmadd231sd 512(%rdx), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0xbd,0x8a,0x00,0x02,0x00,0x00]
+vfnmadd231sd 512(%rdx), %xmm1, %xmm1
+
+// CHECK: vfnmadd231sd 512(%rdx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0xbd,0x4a,0x40]
+vfnmadd231sd 512(%rdx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd231sd 512(%rdx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0xbd,0x4a,0x40]
+vfnmadd231sd 512(%rdx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd231sd {rd-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x52,0x85,0x38,0xbd,0xff]
+vfnmadd231sd {rd-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vfnmadd231sd {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x85,0x3a,0xbd,0xff]
+vfnmadd231sd {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfnmadd231sd {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x85,0xba,0xbd,0xff]
+vfnmadd231sd {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfnmadd231sd {rd-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x38,0xbd,0xc9]
+vfnmadd231sd {rd-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfnmadd231sd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x3a,0xbd,0xc9]
+vfnmadd231sd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd231sd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0xba,0xbd,0xc9]
+vfnmadd231sd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd231sd (%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x81,0xbd,0x3a]
+vfnmadd231sd (%rdx), %xmm15, %xmm15
+
+// CHECK: vfnmadd231sd (%rdx), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x85,0x0a,0xbd,0x3a]
+vfnmadd231sd (%rdx), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfnmadd231sd (%rdx), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x85,0x8a,0xbd,0x3a]
+vfnmadd231sd (%rdx), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfnmadd231sd (%rdx), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0xbd,0x0a]
+vfnmadd231sd (%rdx), %xmm1, %xmm1
+
+// CHECK: vfnmadd231sd (%rdx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0xbd,0x0a]
+vfnmadd231sd (%rdx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd231sd (%rdx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0xbd,0x0a]
+vfnmadd231sd (%rdx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd231sd {rn-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x52,0x85,0x18,0xbd,0xff]
+vfnmadd231sd {rn-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vfnmadd231sd {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x85,0x1a,0xbd,0xff]
+vfnmadd231sd {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfnmadd231sd {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x85,0x9a,0xbd,0xff]
+vfnmadd231sd {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfnmadd231sd {rn-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x18,0xbd,0xc9]
+vfnmadd231sd {rn-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfnmadd231sd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x1a,0xbd,0xc9]
+vfnmadd231sd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd231sd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x9a,0xbd,0xc9]
+vfnmadd231sd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd231sd {ru-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x52,0x85,0x58,0xbd,0xff]
+vfnmadd231sd {ru-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vfnmadd231sd {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x85,0x5a,0xbd,0xff]
+vfnmadd231sd {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfnmadd231sd {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x85,0xda,0xbd,0xff]
+vfnmadd231sd {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfnmadd231sd {ru-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x58,0xbd,0xc9]
+vfnmadd231sd {ru-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfnmadd231sd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x5a,0xbd,0xc9]
+vfnmadd231sd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd231sd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0xda,0xbd,0xc9]
+vfnmadd231sd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd231sd {rz-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x52,0x85,0x78,0xbd,0xff]
+vfnmadd231sd {rz-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vfnmadd231sd {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x85,0x7a,0xbd,0xff]
+vfnmadd231sd {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfnmadd231sd {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x85,0xfa,0xbd,0xff]
+vfnmadd231sd {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfnmadd231sd {rz-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x78,0xbd,0xc9]
+vfnmadd231sd {rz-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfnmadd231sd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x7a,0xbd,0xc9]
+vfnmadd231sd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd231sd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0xfa,0xbd,0xc9]
+vfnmadd231sd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd231sd %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x42,0x81,0xbd,0xff]
+vfnmadd231sd %xmm15, %xmm15, %xmm15
+
+// CHECK: vfnmadd231sd %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x85,0x0a,0xbd,0xff]
+vfnmadd231sd %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfnmadd231sd %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x85,0x8a,0xbd,0xff]
+vfnmadd231sd %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfnmadd231sd %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0xbd,0xc9]
+vfnmadd231sd %xmm1, %xmm1, %xmm1
+
+// CHECK: vfnmadd231sd %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0xbd,0xc9]
+vfnmadd231sd %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd231sd %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0xbd,0xc9]
+vfnmadd231sd %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd231ss 256(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x01,0xbd,0xbc,0x82,0x00,0x01,0x00,0x00]
+vfnmadd231ss 256(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vfnmadd231ss -256(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x01,0xbd,0xbc,0x82,0x00,0xff,0xff,0xff]
+vfnmadd231ss -256(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vfnmadd231ss 256(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x05,0x0a,0xbd,0x7c,0x82,0x40]
+vfnmadd231ss 256(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfnmadd231ss -256(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x05,0x0a,0xbd,0x7c,0x82,0xc0]
+vfnmadd231ss -256(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfnmadd231ss 256(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x05,0x8a,0xbd,0x7c,0x82,0x40]
+vfnmadd231ss 256(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfnmadd231ss -256(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x05,0x8a,0xbd,0x7c,0x82,0xc0]
+vfnmadd231ss -256(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfnmadd231ss 256(%rdx,%rax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0xbd,0x8c,0x82,0x00,0x01,0x00,0x00]
+vfnmadd231ss 256(%rdx,%rax,4), %xmm1, %xmm1
+
+// CHECK: vfnmadd231ss -256(%rdx,%rax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0xbd,0x8c,0x82,0x00,0xff,0xff,0xff]
+vfnmadd231ss -256(%rdx,%rax,4), %xmm1, %xmm1
+
+// CHECK: vfnmadd231ss 256(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0xbd,0x4c,0x82,0x40]
+vfnmadd231ss 256(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd231ss -256(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0xbd,0x4c,0x82,0xc0]
+vfnmadd231ss -256(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd231ss 256(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0xbd,0x4c,0x82,0x40]
+vfnmadd231ss 256(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd231ss -256(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0xbd,0x4c,0x82,0xc0]
+vfnmadd231ss -256(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd231ss 256(%rdx,%rax), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x01,0xbd,0xbc,0x02,0x00,0x01,0x00,0x00]
+vfnmadd231ss 256(%rdx,%rax), %xmm15, %xmm15
+
+// CHECK: vfnmadd231ss 256(%rdx,%rax), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x05,0x0a,0xbd,0x7c,0x02,0x40]
+vfnmadd231ss 256(%rdx,%rax), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfnmadd231ss 256(%rdx,%rax), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x05,0x8a,0xbd,0x7c,0x02,0x40]
+vfnmadd231ss 256(%rdx,%rax), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfnmadd231ss 256(%rdx,%rax), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0xbd,0x8c,0x02,0x00,0x01,0x00,0x00]
+vfnmadd231ss 256(%rdx,%rax), %xmm1, %xmm1
+
+// CHECK: vfnmadd231ss 256(%rdx,%rax), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0xbd,0x4c,0x02,0x40]
+vfnmadd231ss 256(%rdx,%rax), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd231ss 256(%rdx,%rax), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0xbd,0x4c,0x02,0x40]
+vfnmadd231ss 256(%rdx,%rax), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd231ss 256(%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x01,0xbd,0xba,0x00,0x01,0x00,0x00]
+vfnmadd231ss 256(%rdx), %xmm15, %xmm15
+
+// CHECK: vfnmadd231ss 256(%rdx), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x05,0x0a,0xbd,0x7a,0x40]
+vfnmadd231ss 256(%rdx), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfnmadd231ss 256(%rdx), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x05,0x8a,0xbd,0x7a,0x40]
+vfnmadd231ss 256(%rdx), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfnmadd231ss 256(%rdx), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0xbd,0x8a,0x00,0x01,0x00,0x00]
+vfnmadd231ss 256(%rdx), %xmm1, %xmm1
+
+// CHECK: vfnmadd231ss 256(%rdx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0xbd,0x4a,0x40]
+vfnmadd231ss 256(%rdx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd231ss 256(%rdx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0xbd,0x4a,0x40]
+vfnmadd231ss 256(%rdx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd231ss 485498096, %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x01,0xbd,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfnmadd231ss 485498096, %xmm15, %xmm15
+
+// CHECK: vfnmadd231ss 485498096, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x05,0x0a,0xbd,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfnmadd231ss 485498096, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfnmadd231ss 485498096, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x05,0x8a,0xbd,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfnmadd231ss 485498096, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfnmadd231ss 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0xbd,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfnmadd231ss 485498096, %xmm1, %xmm1
+
+// CHECK: vfnmadd231ss 485498096, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0xbd,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfnmadd231ss 485498096, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd231ss 485498096, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0xbd,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfnmadd231ss 485498096, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd231ss {rd-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x52,0x05,0x38,0xbd,0xff]
+vfnmadd231ss {rd-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vfnmadd231ss {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x05,0x3a,0xbd,0xff]
+vfnmadd231ss {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfnmadd231ss {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x05,0xba,0xbd,0xff]
+vfnmadd231ss {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfnmadd231ss {rd-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x38,0xbd,0xc9]
+vfnmadd231ss {rd-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfnmadd231ss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x3a,0xbd,0xc9]
+vfnmadd231ss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd231ss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0xba,0xbd,0xc9]
+vfnmadd231ss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd231ss (%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x01,0xbd,0x3a]
+vfnmadd231ss (%rdx), %xmm15, %xmm15
+
+// CHECK: vfnmadd231ss (%rdx), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x05,0x0a,0xbd,0x3a]
+vfnmadd231ss (%rdx), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfnmadd231ss (%rdx), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x05,0x8a,0xbd,0x3a]
+vfnmadd231ss (%rdx), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfnmadd231ss (%rdx), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0xbd,0x0a]
+vfnmadd231ss (%rdx), %xmm1, %xmm1
+
+// CHECK: vfnmadd231ss (%rdx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0xbd,0x0a]
+vfnmadd231ss (%rdx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd231ss (%rdx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0xbd,0x0a]
+vfnmadd231ss (%rdx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd231ss {rn-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x52,0x05,0x18,0xbd,0xff]
+vfnmadd231ss {rn-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vfnmadd231ss {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x05,0x1a,0xbd,0xff]
+vfnmadd231ss {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfnmadd231ss {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x05,0x9a,0xbd,0xff]
+vfnmadd231ss {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfnmadd231ss {rn-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x18,0xbd,0xc9]
+vfnmadd231ss {rn-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfnmadd231ss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x1a,0xbd,0xc9]
+vfnmadd231ss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd231ss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x9a,0xbd,0xc9]
+vfnmadd231ss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd231ss {ru-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x52,0x05,0x58,0xbd,0xff]
+vfnmadd231ss {ru-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vfnmadd231ss {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x05,0x5a,0xbd,0xff]
+vfnmadd231ss {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfnmadd231ss {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x05,0xda,0xbd,0xff]
+vfnmadd231ss {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfnmadd231ss {ru-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x58,0xbd,0xc9]
+vfnmadd231ss {ru-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfnmadd231ss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x5a,0xbd,0xc9]
+vfnmadd231ss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd231ss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0xda,0xbd,0xc9]
+vfnmadd231ss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd231ss {rz-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x52,0x05,0x78,0xbd,0xff]
+vfnmadd231ss {rz-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vfnmadd231ss {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x05,0x7a,0xbd,0xff]
+vfnmadd231ss {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfnmadd231ss {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x05,0xfa,0xbd,0xff]
+vfnmadd231ss {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfnmadd231ss {rz-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x78,0xbd,0xc9]
+vfnmadd231ss {rz-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfnmadd231ss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x7a,0xbd,0xc9]
+vfnmadd231ss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd231ss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0xfa,0xbd,0xc9]
+vfnmadd231ss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmadd231ss %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x42,0x01,0xbd,0xff]
+vfnmadd231ss %xmm15, %xmm15, %xmm15
+
+// CHECK: vfnmadd231ss %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x05,0x0a,0xbd,0xff]
+vfnmadd231ss %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfnmadd231ss %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x05,0x8a,0xbd,0xff]
+vfnmadd231ss %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfnmadd231ss %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0xbd,0xc9]
+vfnmadd231ss %xmm1, %xmm1, %xmm1
+
+// CHECK: vfnmadd231ss %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0xbd,0xc9]
+vfnmadd231ss %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmadd231ss %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0xbd,0xc9]
+vfnmadd231ss %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmsub132sd 485498096, %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x81,0x9f,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfnmsub132sd 485498096, %xmm15, %xmm15
+
+// CHECK: vfnmsub132sd 485498096, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x85,0x0a,0x9f,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfnmsub132sd 485498096, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfnmsub132sd 485498096, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x85,0x8a,0x9f,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfnmsub132sd 485498096, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfnmsub132sd 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0x9f,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfnmsub132sd 485498096, %xmm1, %xmm1
+
+// CHECK: vfnmsub132sd 485498096, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0x9f,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfnmsub132sd 485498096, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmsub132sd 485498096, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0x9f,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfnmsub132sd 485498096, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmsub132sd 512(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x81,0x9f,0xbc,0x82,0x00,0x02,0x00,0x00]
+vfnmsub132sd 512(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vfnmsub132sd -512(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x81,0x9f,0xbc,0x82,0x00,0xfe,0xff,0xff]
+vfnmsub132sd -512(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vfnmsub132sd 512(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x85,0x0a,0x9f,0x7c,0x82,0x40]
+vfnmsub132sd 512(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfnmsub132sd -512(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x85,0x0a,0x9f,0x7c,0x82,0xc0]
+vfnmsub132sd -512(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfnmsub132sd 512(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x85,0x8a,0x9f,0x7c,0x82,0x40]
+vfnmsub132sd 512(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfnmsub132sd -512(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x85,0x8a,0x9f,0x7c,0x82,0xc0]
+vfnmsub132sd -512(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfnmsub132sd 512(%rdx,%rax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0x9f,0x8c,0x82,0x00,0x02,0x00,0x00]
+vfnmsub132sd 512(%rdx,%rax,4), %xmm1, %xmm1
+
+// CHECK: vfnmsub132sd -512(%rdx,%rax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0x9f,0x8c,0x82,0x00,0xfe,0xff,0xff]
+vfnmsub132sd -512(%rdx,%rax,4), %xmm1, %xmm1
+
+// CHECK: vfnmsub132sd 512(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0x9f,0x4c,0x82,0x40]
+vfnmsub132sd 512(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmsub132sd -512(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0x9f,0x4c,0x82,0xc0]
+vfnmsub132sd -512(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmsub132sd 512(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0x9f,0x4c,0x82,0x40]
+vfnmsub132sd 512(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmsub132sd -512(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0x9f,0x4c,0x82,0xc0]
+vfnmsub132sd -512(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmsub132sd 512(%rdx,%rax), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x81,0x9f,0xbc,0x02,0x00,0x02,0x00,0x00]
+vfnmsub132sd 512(%rdx,%rax), %xmm15, %xmm15
+
+// CHECK: vfnmsub132sd 512(%rdx,%rax), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x85,0x0a,0x9f,0x7c,0x02,0x40]
+vfnmsub132sd 512(%rdx,%rax), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfnmsub132sd 512(%rdx,%rax), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x85,0x8a,0x9f,0x7c,0x02,0x40]
+vfnmsub132sd 512(%rdx,%rax), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfnmsub132sd 512(%rdx,%rax), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0x9f,0x8c,0x02,0x00,0x02,0x00,0x00]
+vfnmsub132sd 512(%rdx,%rax), %xmm1, %xmm1
+
+// CHECK: vfnmsub132sd 512(%rdx,%rax), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0x9f,0x4c,0x02,0x40]
+vfnmsub132sd 512(%rdx,%rax), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmsub132sd 512(%rdx,%rax), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0x9f,0x4c,0x02,0x40]
+vfnmsub132sd 512(%rdx,%rax), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmsub132sd 512(%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x81,0x9f,0xba,0x00,0x02,0x00,0x00]
+vfnmsub132sd 512(%rdx), %xmm15, %xmm15
+
+// CHECK: vfnmsub132sd 512(%rdx), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x85,0x0a,0x9f,0x7a,0x40]
+vfnmsub132sd 512(%rdx), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfnmsub132sd 512(%rdx), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x85,0x8a,0x9f,0x7a,0x40]
+vfnmsub132sd 512(%rdx), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfnmsub132sd 512(%rdx), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0x9f,0x8a,0x00,0x02,0x00,0x00]
+vfnmsub132sd 512(%rdx), %xmm1, %xmm1
+
+// CHECK: vfnmsub132sd 512(%rdx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0x9f,0x4a,0x40]
+vfnmsub132sd 512(%rdx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmsub132sd 512(%rdx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0x9f,0x4a,0x40]
+vfnmsub132sd 512(%rdx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmsub132sd {rd-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x52,0x85,0x38,0x9f,0xff]
+vfnmsub132sd {rd-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vfnmsub132sd {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x85,0x3a,0x9f,0xff]
+vfnmsub132sd {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfnmsub132sd {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x85,0xba,0x9f,0xff]
+vfnmsub132sd {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfnmsub132sd {rd-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x38,0x9f,0xc9]
+vfnmsub132sd {rd-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfnmsub132sd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x3a,0x9f,0xc9]
+vfnmsub132sd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmsub132sd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0xba,0x9f,0xc9]
+vfnmsub132sd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmsub132sd (%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x81,0x9f,0x3a]
+vfnmsub132sd (%rdx), %xmm15, %xmm15
+
+// CHECK: vfnmsub132sd (%rdx), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x85,0x0a,0x9f,0x3a]
+vfnmsub132sd (%rdx), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfnmsub132sd (%rdx), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x85,0x8a,0x9f,0x3a]
+vfnmsub132sd (%rdx), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfnmsub132sd (%rdx), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0x9f,0x0a]
+vfnmsub132sd (%rdx), %xmm1, %xmm1
+
+// CHECK: vfnmsub132sd (%rdx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0x9f,0x0a]
+vfnmsub132sd (%rdx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmsub132sd (%rdx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0x9f,0x0a]
+vfnmsub132sd (%rdx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmsub132sd {rn-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x52,0x85,0x18,0x9f,0xff]
+vfnmsub132sd {rn-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vfnmsub132sd {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x85,0x1a,0x9f,0xff]
+vfnmsub132sd {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfnmsub132sd {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x85,0x9a,0x9f,0xff]
+vfnmsub132sd {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfnmsub132sd {rn-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x18,0x9f,0xc9]
+vfnmsub132sd {rn-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfnmsub132sd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x1a,0x9f,0xc9]
+vfnmsub132sd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmsub132sd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x9a,0x9f,0xc9]
+vfnmsub132sd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmsub132sd {ru-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x52,0x85,0x58,0x9f,0xff]
+vfnmsub132sd {ru-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vfnmsub132sd {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x85,0x5a,0x9f,0xff]
+vfnmsub132sd {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfnmsub132sd {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x85,0xda,0x9f,0xff]
+vfnmsub132sd {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfnmsub132sd {ru-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x58,0x9f,0xc9]
+vfnmsub132sd {ru-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfnmsub132sd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x5a,0x9f,0xc9]
+vfnmsub132sd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmsub132sd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0xda,0x9f,0xc9]
+vfnmsub132sd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmsub132sd {rz-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x52,0x85,0x78,0x9f,0xff]
+vfnmsub132sd {rz-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vfnmsub132sd {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x85,0x7a,0x9f,0xff]
+vfnmsub132sd {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfnmsub132sd {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x85,0xfa,0x9f,0xff]
+vfnmsub132sd {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfnmsub132sd {rz-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x78,0x9f,0xc9]
+vfnmsub132sd {rz-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfnmsub132sd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x7a,0x9f,0xc9]
+vfnmsub132sd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmsub132sd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0xfa,0x9f,0xc9]
+vfnmsub132sd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmsub132sd %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x42,0x81,0x9f,0xff]
+vfnmsub132sd %xmm15, %xmm15, %xmm15
+
+// CHECK: vfnmsub132sd %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x85,0x0a,0x9f,0xff]
+vfnmsub132sd %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfnmsub132sd %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x85,0x8a,0x9f,0xff]
+vfnmsub132sd %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfnmsub132sd %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0x9f,0xc9]
+vfnmsub132sd %xmm1, %xmm1, %xmm1
+
+// CHECK: vfnmsub132sd %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0x9f,0xc9]
+vfnmsub132sd %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmsub132sd %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0x9f,0xc9]
+vfnmsub132sd %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmsub132ss 256(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x01,0x9f,0xbc,0x82,0x00,0x01,0x00,0x00]
+vfnmsub132ss 256(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vfnmsub132ss -256(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x01,0x9f,0xbc,0x82,0x00,0xff,0xff,0xff]
+vfnmsub132ss -256(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vfnmsub132ss 256(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x05,0x0a,0x9f,0x7c,0x82,0x40]
+vfnmsub132ss 256(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfnmsub132ss -256(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x05,0x0a,0x9f,0x7c,0x82,0xc0]
+vfnmsub132ss -256(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfnmsub132ss 256(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x05,0x8a,0x9f,0x7c,0x82,0x40]
+vfnmsub132ss 256(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfnmsub132ss -256(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x05,0x8a,0x9f,0x7c,0x82,0xc0]
+vfnmsub132ss -256(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfnmsub132ss 256(%rdx,%rax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0x9f,0x8c,0x82,0x00,0x01,0x00,0x00]
+vfnmsub132ss 256(%rdx,%rax,4), %xmm1, %xmm1
+
+// CHECK: vfnmsub132ss -256(%rdx,%rax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0x9f,0x8c,0x82,0x00,0xff,0xff,0xff]
+vfnmsub132ss -256(%rdx,%rax,4), %xmm1, %xmm1
+
+// CHECK: vfnmsub132ss 256(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0x9f,0x4c,0x82,0x40]
+vfnmsub132ss 256(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmsub132ss -256(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0x9f,0x4c,0x82,0xc0]
+vfnmsub132ss -256(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmsub132ss 256(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0x9f,0x4c,0x82,0x40]
+vfnmsub132ss 256(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmsub132ss -256(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0x9f,0x4c,0x82,0xc0]
+vfnmsub132ss -256(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmsub132ss 256(%rdx,%rax), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x01,0x9f,0xbc,0x02,0x00,0x01,0x00,0x00]
+vfnmsub132ss 256(%rdx,%rax), %xmm15, %xmm15
+
+// CHECK: vfnmsub132ss 256(%rdx,%rax), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x05,0x0a,0x9f,0x7c,0x02,0x40]
+vfnmsub132ss 256(%rdx,%rax), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfnmsub132ss 256(%rdx,%rax), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x05,0x8a,0x9f,0x7c,0x02,0x40]
+vfnmsub132ss 256(%rdx,%rax), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfnmsub132ss 256(%rdx,%rax), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0x9f,0x8c,0x02,0x00,0x01,0x00,0x00]
+vfnmsub132ss 256(%rdx,%rax), %xmm1, %xmm1
+
+// CHECK: vfnmsub132ss 256(%rdx,%rax), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0x9f,0x4c,0x02,0x40]
+vfnmsub132ss 256(%rdx,%rax), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmsub132ss 256(%rdx,%rax), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0x9f,0x4c,0x02,0x40]
+vfnmsub132ss 256(%rdx,%rax), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmsub132ss 256(%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x01,0x9f,0xba,0x00,0x01,0x00,0x00]
+vfnmsub132ss 256(%rdx), %xmm15, %xmm15
+
+// CHECK: vfnmsub132ss 256(%rdx), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x05,0x0a,0x9f,0x7a,0x40]
+vfnmsub132ss 256(%rdx), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfnmsub132ss 256(%rdx), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x05,0x8a,0x9f,0x7a,0x40]
+vfnmsub132ss 256(%rdx), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfnmsub132ss 256(%rdx), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0x9f,0x8a,0x00,0x01,0x00,0x00]
+vfnmsub132ss 256(%rdx), %xmm1, %xmm1
+
+// CHECK: vfnmsub132ss 256(%rdx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0x9f,0x4a,0x40]
+vfnmsub132ss 256(%rdx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmsub132ss 256(%rdx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0x9f,0x4a,0x40]
+vfnmsub132ss 256(%rdx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmsub132ss 485498096, %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x01,0x9f,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfnmsub132ss 485498096, %xmm15, %xmm15
+
+// CHECK: vfnmsub132ss 485498096, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x05,0x0a,0x9f,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfnmsub132ss 485498096, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfnmsub132ss 485498096, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x05,0x8a,0x9f,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfnmsub132ss 485498096, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfnmsub132ss 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0x9f,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfnmsub132ss 485498096, %xmm1, %xmm1
+
+// CHECK: vfnmsub132ss 485498096, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0x9f,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfnmsub132ss 485498096, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmsub132ss 485498096, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0x9f,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfnmsub132ss 485498096, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmsub132ss {rd-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x52,0x05,0x38,0x9f,0xff]
+vfnmsub132ss {rd-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vfnmsub132ss {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x05,0x3a,0x9f,0xff]
+vfnmsub132ss {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfnmsub132ss {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x05,0xba,0x9f,0xff]
+vfnmsub132ss {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfnmsub132ss {rd-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x38,0x9f,0xc9]
+vfnmsub132ss {rd-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfnmsub132ss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x3a,0x9f,0xc9]
+vfnmsub132ss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmsub132ss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0xba,0x9f,0xc9]
+vfnmsub132ss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmsub132ss (%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x01,0x9f,0x3a]
+vfnmsub132ss (%rdx), %xmm15, %xmm15
+
+// CHECK: vfnmsub132ss (%rdx), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x05,0x0a,0x9f,0x3a]
+vfnmsub132ss (%rdx), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfnmsub132ss (%rdx), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x05,0x8a,0x9f,0x3a]
+vfnmsub132ss (%rdx), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfnmsub132ss (%rdx), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0x9f,0x0a]
+vfnmsub132ss (%rdx), %xmm1, %xmm1
+
+// CHECK: vfnmsub132ss (%rdx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0x9f,0x0a]
+vfnmsub132ss (%rdx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmsub132ss (%rdx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0x9f,0x0a]
+vfnmsub132ss (%rdx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmsub132ss {rn-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x52,0x05,0x18,0x9f,0xff]
+vfnmsub132ss {rn-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vfnmsub132ss {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x05,0x1a,0x9f,0xff]
+vfnmsub132ss {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfnmsub132ss {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x05,0x9a,0x9f,0xff]
+vfnmsub132ss {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfnmsub132ss {rn-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x18,0x9f,0xc9]
+vfnmsub132ss {rn-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfnmsub132ss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x1a,0x9f,0xc9]
+vfnmsub132ss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmsub132ss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x9a,0x9f,0xc9]
+vfnmsub132ss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmsub132ss {ru-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x52,0x05,0x58,0x9f,0xff]
+vfnmsub132ss {ru-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vfnmsub132ss {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x05,0x5a,0x9f,0xff]
+vfnmsub132ss {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfnmsub132ss {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x05,0xda,0x9f,0xff]
+vfnmsub132ss {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfnmsub132ss {ru-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x58,0x9f,0xc9]
+vfnmsub132ss {ru-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfnmsub132ss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x5a,0x9f,0xc9]
+vfnmsub132ss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmsub132ss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0xda,0x9f,0xc9]
+vfnmsub132ss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmsub132ss {rz-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x52,0x05,0x78,0x9f,0xff]
+vfnmsub132ss {rz-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vfnmsub132ss {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x05,0x7a,0x9f,0xff]
+vfnmsub132ss {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfnmsub132ss {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x05,0xfa,0x9f,0xff]
+vfnmsub132ss {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfnmsub132ss {rz-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x78,0x9f,0xc9]
+vfnmsub132ss {rz-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfnmsub132ss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x7a,0x9f,0xc9]
+vfnmsub132ss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmsub132ss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0xfa,0x9f,0xc9]
+vfnmsub132ss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmsub132ss %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x42,0x01,0x9f,0xff]
+vfnmsub132ss %xmm15, %xmm15, %xmm15
+
+// CHECK: vfnmsub132ss %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x05,0x0a,0x9f,0xff]
+vfnmsub132ss %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfnmsub132ss %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x05,0x8a,0x9f,0xff]
+vfnmsub132ss %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfnmsub132ss %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0x9f,0xc9]
+vfnmsub132ss %xmm1, %xmm1, %xmm1
+
+// CHECK: vfnmsub132ss %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0x9f,0xc9]
+vfnmsub132ss %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmsub132ss %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0x9f,0xc9]
+vfnmsub132ss %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmsub213sd 485498096, %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x81,0xaf,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfnmsub213sd 485498096, %xmm15, %xmm15
+
+// CHECK: vfnmsub213sd 485498096, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x85,0x0a,0xaf,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfnmsub213sd 485498096, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfnmsub213sd 485498096, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x85,0x8a,0xaf,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfnmsub213sd 485498096, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfnmsub213sd 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0xaf,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfnmsub213sd 485498096, %xmm1, %xmm1
+
+// CHECK: vfnmsub213sd 485498096, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0xaf,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfnmsub213sd 485498096, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmsub213sd 485498096, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0xaf,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfnmsub213sd 485498096, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmsub213sd 512(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x81,0xaf,0xbc,0x82,0x00,0x02,0x00,0x00]
+vfnmsub213sd 512(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vfnmsub213sd -512(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x81,0xaf,0xbc,0x82,0x00,0xfe,0xff,0xff]
+vfnmsub213sd -512(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vfnmsub213sd 512(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x85,0x0a,0xaf,0x7c,0x82,0x40]
+vfnmsub213sd 512(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfnmsub213sd -512(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x85,0x0a,0xaf,0x7c,0x82,0xc0]
+vfnmsub213sd -512(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfnmsub213sd 512(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x85,0x8a,0xaf,0x7c,0x82,0x40]
+vfnmsub213sd 512(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfnmsub213sd -512(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x85,0x8a,0xaf,0x7c,0x82,0xc0]
+vfnmsub213sd -512(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfnmsub213sd 512(%rdx,%rax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0xaf,0x8c,0x82,0x00,0x02,0x00,0x00]
+vfnmsub213sd 512(%rdx,%rax,4), %xmm1, %xmm1
+
+// CHECK: vfnmsub213sd -512(%rdx,%rax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0xaf,0x8c,0x82,0x00,0xfe,0xff,0xff]
+vfnmsub213sd -512(%rdx,%rax,4), %xmm1, %xmm1
+
+// CHECK: vfnmsub213sd 512(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0xaf,0x4c,0x82,0x40]
+vfnmsub213sd 512(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmsub213sd -512(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0xaf,0x4c,0x82,0xc0]
+vfnmsub213sd -512(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmsub213sd 512(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0xaf,0x4c,0x82,0x40]
+vfnmsub213sd 512(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmsub213sd -512(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0xaf,0x4c,0x82,0xc0]
+vfnmsub213sd -512(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmsub213sd 512(%rdx,%rax), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x81,0xaf,0xbc,0x02,0x00,0x02,0x00,0x00]
+vfnmsub213sd 512(%rdx,%rax), %xmm15, %xmm15
+
+// CHECK: vfnmsub213sd 512(%rdx,%rax), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x85,0x0a,0xaf,0x7c,0x02,0x40]
+vfnmsub213sd 512(%rdx,%rax), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfnmsub213sd 512(%rdx,%rax), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x85,0x8a,0xaf,0x7c,0x02,0x40]
+vfnmsub213sd 512(%rdx,%rax), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfnmsub213sd 512(%rdx,%rax), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0xaf,0x8c,0x02,0x00,0x02,0x00,0x00]
+vfnmsub213sd 512(%rdx,%rax), %xmm1, %xmm1
+
+// CHECK: vfnmsub213sd 512(%rdx,%rax), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0xaf,0x4c,0x02,0x40]
+vfnmsub213sd 512(%rdx,%rax), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmsub213sd 512(%rdx,%rax), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0xaf,0x4c,0x02,0x40]
+vfnmsub213sd 512(%rdx,%rax), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmsub213sd 512(%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x81,0xaf,0xba,0x00,0x02,0x00,0x00]
+vfnmsub213sd 512(%rdx), %xmm15, %xmm15
+
+// CHECK: vfnmsub213sd 512(%rdx), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x85,0x0a,0xaf,0x7a,0x40]
+vfnmsub213sd 512(%rdx), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfnmsub213sd 512(%rdx), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x85,0x8a,0xaf,0x7a,0x40]
+vfnmsub213sd 512(%rdx), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfnmsub213sd 512(%rdx), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0xaf,0x8a,0x00,0x02,0x00,0x00]
+vfnmsub213sd 512(%rdx), %xmm1, %xmm1
+
+// CHECK: vfnmsub213sd 512(%rdx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0xaf,0x4a,0x40]
+vfnmsub213sd 512(%rdx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmsub213sd 512(%rdx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0xaf,0x4a,0x40]
+vfnmsub213sd 512(%rdx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmsub213sd {rd-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x52,0x85,0x38,0xaf,0xff]
+vfnmsub213sd {rd-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vfnmsub213sd {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x85,0x3a,0xaf,0xff]
+vfnmsub213sd {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfnmsub213sd {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x85,0xba,0xaf,0xff]
+vfnmsub213sd {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfnmsub213sd {rd-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x38,0xaf,0xc9]
+vfnmsub213sd {rd-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfnmsub213sd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x3a,0xaf,0xc9]
+vfnmsub213sd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmsub213sd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0xba,0xaf,0xc9]
+vfnmsub213sd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmsub213sd (%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x81,0xaf,0x3a]
+vfnmsub213sd (%rdx), %xmm15, %xmm15
+
+// CHECK: vfnmsub213sd (%rdx), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x85,0x0a,0xaf,0x3a]
+vfnmsub213sd (%rdx), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfnmsub213sd (%rdx), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x85,0x8a,0xaf,0x3a]
+vfnmsub213sd (%rdx), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfnmsub213sd (%rdx), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0xaf,0x0a]
+vfnmsub213sd (%rdx), %xmm1, %xmm1
+
+// CHECK: vfnmsub213sd (%rdx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0xaf,0x0a]
+vfnmsub213sd (%rdx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmsub213sd (%rdx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0xaf,0x0a]
+vfnmsub213sd (%rdx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmsub213sd {rn-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x52,0x85,0x18,0xaf,0xff]
+vfnmsub213sd {rn-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vfnmsub213sd {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x85,0x1a,0xaf,0xff]
+vfnmsub213sd {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfnmsub213sd {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x85,0x9a,0xaf,0xff]
+vfnmsub213sd {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfnmsub213sd {rn-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x18,0xaf,0xc9]
+vfnmsub213sd {rn-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfnmsub213sd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x1a,0xaf,0xc9]
+vfnmsub213sd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmsub213sd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x9a,0xaf,0xc9]
+vfnmsub213sd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmsub213sd {ru-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x52,0x85,0x58,0xaf,0xff]
+vfnmsub213sd {ru-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vfnmsub213sd {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x85,0x5a,0xaf,0xff]
+vfnmsub213sd {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfnmsub213sd {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x85,0xda,0xaf,0xff]
+vfnmsub213sd {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfnmsub213sd {ru-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x58,0xaf,0xc9]
+vfnmsub213sd {ru-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfnmsub213sd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x5a,0xaf,0xc9]
+vfnmsub213sd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmsub213sd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0xda,0xaf,0xc9]
+vfnmsub213sd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmsub213sd {rz-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x52,0x85,0x78,0xaf,0xff]
+vfnmsub213sd {rz-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vfnmsub213sd {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x85,0x7a,0xaf,0xff]
+vfnmsub213sd {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfnmsub213sd {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x85,0xfa,0xaf,0xff]
+vfnmsub213sd {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfnmsub213sd {rz-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x78,0xaf,0xc9]
+vfnmsub213sd {rz-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfnmsub213sd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x7a,0xaf,0xc9]
+vfnmsub213sd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmsub213sd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0xfa,0xaf,0xc9]
+vfnmsub213sd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmsub213sd %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x42,0x81,0xaf,0xff]
+vfnmsub213sd %xmm15, %xmm15, %xmm15
+
+// CHECK: vfnmsub213sd %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x85,0x0a,0xaf,0xff]
+vfnmsub213sd %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfnmsub213sd %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x85,0x8a,0xaf,0xff]
+vfnmsub213sd %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfnmsub213sd %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0xf1,0xaf,0xc9]
+vfnmsub213sd %xmm1, %xmm1, %xmm1
+
+// CHECK: vfnmsub213sd %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0xaf,0xc9]
+vfnmsub213sd %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmsub213sd %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0xaf,0xc9]
+vfnmsub213sd %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmsub213ss 256(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x01,0xaf,0xbc,0x82,0x00,0x01,0x00,0x00]
+vfnmsub213ss 256(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vfnmsub213ss -256(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x01,0xaf,0xbc,0x82,0x00,0xff,0xff,0xff]
+vfnmsub213ss -256(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vfnmsub213ss 256(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x05,0x0a,0xaf,0x7c,0x82,0x40]
+vfnmsub213ss 256(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfnmsub213ss -256(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x05,0x0a,0xaf,0x7c,0x82,0xc0]
+vfnmsub213ss -256(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfnmsub213ss 256(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x05,0x8a,0xaf,0x7c,0x82,0x40]
+vfnmsub213ss 256(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfnmsub213ss -256(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x05,0x8a,0xaf,0x7c,0x82,0xc0]
+vfnmsub213ss -256(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfnmsub213ss 256(%rdx,%rax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0xaf,0x8c,0x82,0x00,0x01,0x00,0x00]
+vfnmsub213ss 256(%rdx,%rax,4), %xmm1, %xmm1
+
+// CHECK: vfnmsub213ss -256(%rdx,%rax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0xaf,0x8c,0x82,0x00,0xff,0xff,0xff]
+vfnmsub213ss -256(%rdx,%rax,4), %xmm1, %xmm1
+
+// CHECK: vfnmsub213ss 256(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0xaf,0x4c,0x82,0x40]
+vfnmsub213ss 256(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmsub213ss -256(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0xaf,0x4c,0x82,0xc0]
+vfnmsub213ss -256(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmsub213ss 256(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0xaf,0x4c,0x82,0x40]
+vfnmsub213ss 256(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmsub213ss -256(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0xaf,0x4c,0x82,0xc0]
+vfnmsub213ss -256(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmsub213ss 256(%rdx,%rax), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x01,0xaf,0xbc,0x02,0x00,0x01,0x00,0x00]
+vfnmsub213ss 256(%rdx,%rax), %xmm15, %xmm15
+
+// CHECK: vfnmsub213ss 256(%rdx,%rax), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x05,0x0a,0xaf,0x7c,0x02,0x40]
+vfnmsub213ss 256(%rdx,%rax), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfnmsub213ss 256(%rdx,%rax), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x05,0x8a,0xaf,0x7c,0x02,0x40]
+vfnmsub213ss 256(%rdx,%rax), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfnmsub213ss 256(%rdx,%rax), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0xaf,0x8c,0x02,0x00,0x01,0x00,0x00]
+vfnmsub213ss 256(%rdx,%rax), %xmm1, %xmm1
+
+// CHECK: vfnmsub213ss 256(%rdx,%rax), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0xaf,0x4c,0x02,0x40]
+vfnmsub213ss 256(%rdx,%rax), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmsub213ss 256(%rdx,%rax), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0xaf,0x4c,0x02,0x40]
+vfnmsub213ss 256(%rdx,%rax), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmsub213ss 256(%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x01,0xaf,0xba,0x00,0x01,0x00,0x00]
+vfnmsub213ss 256(%rdx), %xmm15, %xmm15
+
+// CHECK: vfnmsub213ss 256(%rdx), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x05,0x0a,0xaf,0x7a,0x40]
+vfnmsub213ss 256(%rdx), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfnmsub213ss 256(%rdx), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x05,0x8a,0xaf,0x7a,0x40]
+vfnmsub213ss 256(%rdx), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfnmsub213ss 256(%rdx), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0xaf,0x8a,0x00,0x01,0x00,0x00]
+vfnmsub213ss 256(%rdx), %xmm1, %xmm1
+
+// CHECK: vfnmsub213ss 256(%rdx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0xaf,0x4a,0x40]
+vfnmsub213ss 256(%rdx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmsub213ss 256(%rdx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0xaf,0x4a,0x40]
+vfnmsub213ss 256(%rdx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmsub213ss 485498096, %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x01,0xaf,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfnmsub213ss 485498096, %xmm15, %xmm15
+
+// CHECK: vfnmsub213ss 485498096, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x05,0x0a,0xaf,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfnmsub213ss 485498096, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfnmsub213ss 485498096, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x05,0x8a,0xaf,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfnmsub213ss 485498096, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfnmsub213ss 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0xaf,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfnmsub213ss 485498096, %xmm1, %xmm1
+
+// CHECK: vfnmsub213ss 485498096, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0xaf,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfnmsub213ss 485498096, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmsub213ss 485498096, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0xaf,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfnmsub213ss 485498096, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmsub213ss {rd-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x52,0x05,0x38,0xaf,0xff]
+vfnmsub213ss {rd-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vfnmsub213ss {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x05,0x3a,0xaf,0xff]
+vfnmsub213ss {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfnmsub213ss {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x05,0xba,0xaf,0xff]
+vfnmsub213ss {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfnmsub213ss {rd-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x38,0xaf,0xc9]
+vfnmsub213ss {rd-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfnmsub213ss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x3a,0xaf,0xc9]
+vfnmsub213ss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmsub213ss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0xba,0xaf,0xc9]
+vfnmsub213ss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmsub213ss (%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x01,0xaf,0x3a]
+vfnmsub213ss (%rdx), %xmm15, %xmm15
+
+// CHECK: vfnmsub213ss (%rdx), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x05,0x0a,0xaf,0x3a]
+vfnmsub213ss (%rdx), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfnmsub213ss (%rdx), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x05,0x8a,0xaf,0x3a]
+vfnmsub213ss (%rdx), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfnmsub213ss (%rdx), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0xaf,0x0a]
+vfnmsub213ss (%rdx), %xmm1, %xmm1
+
+// CHECK: vfnmsub213ss (%rdx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0xaf,0x0a]
+vfnmsub213ss (%rdx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmsub213ss (%rdx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0xaf,0x0a]
+vfnmsub213ss (%rdx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmsub213ss {rn-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x52,0x05,0x18,0xaf,0xff]
+vfnmsub213ss {rn-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vfnmsub213ss {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x05,0x1a,0xaf,0xff]
+vfnmsub213ss {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfnmsub213ss {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x05,0x9a,0xaf,0xff]
+vfnmsub213ss {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfnmsub213ss {rn-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x18,0xaf,0xc9]
+vfnmsub213ss {rn-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfnmsub213ss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x1a,0xaf,0xc9]
+vfnmsub213ss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmsub213ss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x9a,0xaf,0xc9]
+vfnmsub213ss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmsub213ss {ru-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x52,0x05,0x58,0xaf,0xff]
+vfnmsub213ss {ru-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vfnmsub213ss {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x05,0x5a,0xaf,0xff]
+vfnmsub213ss {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfnmsub213ss {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x05,0xda,0xaf,0xff]
+vfnmsub213ss {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfnmsub213ss {ru-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x58,0xaf,0xc9]
+vfnmsub213ss {ru-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfnmsub213ss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x5a,0xaf,0xc9]
+vfnmsub213ss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmsub213ss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0xda,0xaf,0xc9]
+vfnmsub213ss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmsub213ss {rz-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x52,0x05,0x78,0xaf,0xff]
+vfnmsub213ss {rz-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vfnmsub213ss {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x05,0x7a,0xaf,0xff]
+vfnmsub213ss {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfnmsub213ss {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x05,0xfa,0xaf,0xff]
+vfnmsub213ss {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfnmsub213ss {rz-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x78,0xaf,0xc9]
+vfnmsub213ss {rz-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfnmsub213ss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x7a,0xaf,0xc9]
+vfnmsub213ss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmsub213ss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0xfa,0xaf,0xc9]
+vfnmsub213ss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmsub213ss %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x42,0x01,0xaf,0xff]
+vfnmsub213ss %xmm15, %xmm15, %xmm15
+
+// CHECK: vfnmsub213ss %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x05,0x0a,0xaf,0xff]
+vfnmsub213ss %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfnmsub213ss %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x05,0x8a,0xaf,0xff]
+vfnmsub213ss %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfnmsub213ss %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0xaf,0xc9]
+vfnmsub213ss %xmm1, %xmm1, %xmm1
+
+// CHECK: vfnmsub213ss %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0xaf,0xc9]
+vfnmsub213ss %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmsub213ss %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0xaf,0xc9]
+vfnmsub213ss %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmsub231sd {rd-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x52,0x85,0x38,0xbf,0xff]
+vfnmsub231sd {rd-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vfnmsub231sd {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x85,0x3a,0xbf,0xff]
+vfnmsub231sd {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfnmsub231sd {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x85,0xba,0xbf,0xff]
+vfnmsub231sd {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfnmsub231sd {rd-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x38,0xbf,0xc9]
+vfnmsub231sd {rd-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfnmsub231sd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x3a,0xbf,0xc9]
+vfnmsub231sd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmsub231sd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0xba,0xbf,0xc9]
+vfnmsub231sd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmsub231sd {rn-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x52,0x85,0x18,0xbf,0xff]
+vfnmsub231sd {rn-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vfnmsub231sd {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x85,0x1a,0xbf,0xff]
+vfnmsub231sd {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfnmsub231sd {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x85,0x9a,0xbf,0xff]
+vfnmsub231sd {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfnmsub231sd {rn-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x18,0xbf,0xc9]
+vfnmsub231sd {rn-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfnmsub231sd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x1a,0xbf,0xc9]
+vfnmsub231sd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmsub231sd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x9a,0xbf,0xc9]
+vfnmsub231sd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmsub231sd {ru-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x52,0x85,0x58,0xbf,0xff]
+vfnmsub231sd {ru-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vfnmsub231sd {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x85,0x5a,0xbf,0xff]
+vfnmsub231sd {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfnmsub231sd {ru-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x58,0xbf,0xc9]
+vfnmsub231sd {ru-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfnmsub231sd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x5a,0xbf,0xc9]
+vfnmsub231sd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmsub231ss 256(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x01,0xbf,0xbc,0x82,0x00,0x01,0x00,0x00]
+vfnmsub231ss 256(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vfnmsub231ss -256(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x01,0xbf,0xbc,0x82,0x00,0xff,0xff,0xff]
+vfnmsub231ss -256(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vfnmsub231ss 256(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x05,0x0a,0xbf,0x7c,0x82,0x40]
+vfnmsub231ss 256(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfnmsub231ss -256(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x05,0x0a,0xbf,0x7c,0x82,0xc0]
+vfnmsub231ss -256(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfnmsub231ss 256(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x05,0x8a,0xbf,0x7c,0x82,0x40]
+vfnmsub231ss 256(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfnmsub231ss -256(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x05,0x8a,0xbf,0x7c,0x82,0xc0]
+vfnmsub231ss -256(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfnmsub231ss 256(%rdx,%rax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0xbf,0x8c,0x82,0x00,0x01,0x00,0x00]
+vfnmsub231ss 256(%rdx,%rax,4), %xmm1, %xmm1
+
+// CHECK: vfnmsub231ss -256(%rdx,%rax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0xbf,0x8c,0x82,0x00,0xff,0xff,0xff]
+vfnmsub231ss -256(%rdx,%rax,4), %xmm1, %xmm1
+
+// CHECK: vfnmsub231ss 256(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0xbf,0x4c,0x82,0x40]
+vfnmsub231ss 256(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmsub231ss -256(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0xbf,0x4c,0x82,0xc0]
+vfnmsub231ss -256(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmsub231ss 256(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0xbf,0x4c,0x82,0x40]
+vfnmsub231ss 256(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmsub231ss -256(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0xbf,0x4c,0x82,0xc0]
+vfnmsub231ss -256(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmsub231ss 256(%rdx,%rax), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x01,0xbf,0xbc,0x02,0x00,0x01,0x00,0x00]
+vfnmsub231ss 256(%rdx,%rax), %xmm15, %xmm15
+
+// CHECK: vfnmsub231ss 256(%rdx,%rax), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x05,0x0a,0xbf,0x7c,0x02,0x40]
+vfnmsub231ss 256(%rdx,%rax), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfnmsub231ss 256(%rdx,%rax), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x05,0x8a,0xbf,0x7c,0x02,0x40]
+vfnmsub231ss 256(%rdx,%rax), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfnmsub231ss 256(%rdx,%rax), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0xbf,0x8c,0x02,0x00,0x01,0x00,0x00]
+vfnmsub231ss 256(%rdx,%rax), %xmm1, %xmm1
+
+// CHECK: vfnmsub231ss 256(%rdx,%rax), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0xbf,0x4c,0x02,0x40]
+vfnmsub231ss 256(%rdx,%rax), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmsub231ss 256(%rdx,%rax), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0xbf,0x4c,0x02,0x40]
+vfnmsub231ss 256(%rdx,%rax), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmsub231ss 256(%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x01,0xbf,0xba,0x00,0x01,0x00,0x00]
+vfnmsub231ss 256(%rdx), %xmm15, %xmm15
+
+// CHECK: vfnmsub231ss 256(%rdx), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x05,0x0a,0xbf,0x7a,0x40]
+vfnmsub231ss 256(%rdx), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfnmsub231ss 256(%rdx), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x05,0x8a,0xbf,0x7a,0x40]
+vfnmsub231ss 256(%rdx), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfnmsub231ss 256(%rdx), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0xbf,0x8a,0x00,0x01,0x00,0x00]
+vfnmsub231ss 256(%rdx), %xmm1, %xmm1
+
+// CHECK: vfnmsub231ss 256(%rdx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0xbf,0x4a,0x40]
+vfnmsub231ss 256(%rdx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmsub231ss 256(%rdx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0xbf,0x4a,0x40]
+vfnmsub231ss 256(%rdx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmsub231ss 485498096, %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x01,0xbf,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfnmsub231ss 485498096, %xmm15, %xmm15
+
+// CHECK: vfnmsub231ss 485498096, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x05,0x0a,0xbf,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfnmsub231ss 485498096, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfnmsub231ss 485498096, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x05,0x8a,0xbf,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfnmsub231ss 485498096, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfnmsub231ss 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0xbf,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfnmsub231ss 485498096, %xmm1, %xmm1
+
+// CHECK: vfnmsub231ss 485498096, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0xbf,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfnmsub231ss 485498096, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmsub231ss 485498096, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0xbf,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfnmsub231ss 485498096, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmsub231ss {rd-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x52,0x05,0x38,0xbf,0xff]
+vfnmsub231ss {rd-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vfnmsub231ss {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x05,0x3a,0xbf,0xff]
+vfnmsub231ss {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfnmsub231ss {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x05,0xba,0xbf,0xff]
+vfnmsub231ss {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfnmsub231ss {rd-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x38,0xbf,0xc9]
+vfnmsub231ss {rd-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfnmsub231ss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x3a,0xbf,0xc9]
+vfnmsub231ss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmsub231ss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0xba,0xbf,0xc9]
+vfnmsub231ss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmsub231ss (%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x62,0x01,0xbf,0x3a]
+vfnmsub231ss (%rdx), %xmm15, %xmm15
+
+// CHECK: vfnmsub231ss (%rdx), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x05,0x0a,0xbf,0x3a]
+vfnmsub231ss (%rdx), %xmm15, %xmm15 {%k2}
+
+// CHECK: vfnmsub231ss (%rdx), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x05,0x8a,0xbf,0x3a]
+vfnmsub231ss (%rdx), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfnmsub231ss (%rdx), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0xbf,0x0a]
+vfnmsub231ss (%rdx), %xmm1, %xmm1
+
+// CHECK: vfnmsub231ss (%rdx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0xbf,0x0a]
+vfnmsub231ss (%rdx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmsub231ss (%rdx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0xbf,0x0a]
+vfnmsub231ss (%rdx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmsub231ss {rn-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x52,0x05,0x18,0xbf,0xff]
+vfnmsub231ss {rn-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vfnmsub231ss {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x05,0x1a,0xbf,0xff]
+vfnmsub231ss {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfnmsub231ss {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x05,0x9a,0xbf,0xff]
+vfnmsub231ss {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfnmsub231ss {rn-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x18,0xbf,0xc9]
+vfnmsub231ss {rn-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfnmsub231ss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x1a,0xbf,0xc9]
+vfnmsub231ss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmsub231ss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x9a,0xbf,0xc9]
+vfnmsub231ss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmsub231ss {ru-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x52,0x05,0x58,0xbf,0xff]
+vfnmsub231ss {ru-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vfnmsub231ss {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x05,0x5a,0xbf,0xff]
+vfnmsub231ss {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfnmsub231ss {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x05,0xda,0xbf,0xff]
+vfnmsub231ss {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfnmsub231ss {ru-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x58,0xbf,0xc9]
+vfnmsub231ss {ru-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfnmsub231ss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x5a,0xbf,0xc9]
+vfnmsub231ss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmsub231ss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0xda,0xbf,0xc9]
+vfnmsub231ss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmsub231ss {rz-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x52,0x05,0x78,0xbf,0xff]
+vfnmsub231ss {rz-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vfnmsub231ss {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x05,0x7a,0xbf,0xff]
+vfnmsub231ss {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfnmsub231ss {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x05,0xfa,0xbf,0xff]
+vfnmsub231ss {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfnmsub231ss {rz-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x78,0xbf,0xc9]
+vfnmsub231ss {rz-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vfnmsub231ss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x7a,0xbf,0xc9]
+vfnmsub231ss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmsub231ss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0xfa,0xbf,0xc9]
+vfnmsub231ss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vfnmsub231ss %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x42,0x01,0xbf,0xff]
+vfnmsub231ss %xmm15, %xmm15, %xmm15
+
+// CHECK: vfnmsub231ss %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x05,0x0a,0xbf,0xff]
+vfnmsub231ss %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vfnmsub231ss %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x05,0x8a,0xbf,0xff]
+vfnmsub231ss %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vfnmsub231ss %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe2,0x71,0xbf,0xc9]
+vfnmsub231ss %xmm1, %xmm1, %xmm1
+
+// CHECK: vfnmsub231ss %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0xbf,0xc9]
+vfnmsub231ss %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vfnmsub231ss %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0xbf,0xc9]
+vfnmsub231ss %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vgetexpsd 485498096, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x72,0x85,0x08,0x43,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vgetexpsd 485498096, %xmm15, %xmm15
+
+// CHECK: vgetexpsd 485498096, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x85,0x0a,0x43,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vgetexpsd 485498096, %xmm15, %xmm15 {%k2}
+
+// CHECK: vgetexpsd 485498096, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x85,0x8a,0x43,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vgetexpsd 485498096, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vgetexpsd 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x08,0x43,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vgetexpsd 485498096, %xmm1, %xmm1
+
+// CHECK: vgetexpsd 485498096, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0x43,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vgetexpsd 485498096, %xmm1, %xmm1 {%k2}
+
+// CHECK: vgetexpsd 485498096, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0x43,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vgetexpsd 485498096, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vgetexpsd 512(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x72,0x85,0x08,0x43,0x7c,0x82,0x40]
+vgetexpsd 512(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vgetexpsd -512(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x72,0x85,0x08,0x43,0x7c,0x82,0xc0]
+vgetexpsd -512(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vgetexpsd 512(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x85,0x0a,0x43,0x7c,0x82,0x40]
+vgetexpsd 512(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+
+// CHECK: vgetexpsd -512(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x85,0x0a,0x43,0x7c,0x82,0xc0]
+vgetexpsd -512(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+
+// CHECK: vgetexpsd 512(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x85,0x8a,0x43,0x7c,0x82,0x40]
+vgetexpsd 512(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vgetexpsd -512(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x85,0x8a,0x43,0x7c,0x82,0xc0]
+vgetexpsd -512(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vgetexpsd 512(%rdx,%rax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x08,0x43,0x4c,0x82,0x40]
+vgetexpsd 512(%rdx,%rax,4), %xmm1, %xmm1
+
+// CHECK: vgetexpsd -512(%rdx,%rax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x08,0x43,0x4c,0x82,0xc0]
+vgetexpsd -512(%rdx,%rax,4), %xmm1, %xmm1
+
+// CHECK: vgetexpsd 512(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0x43,0x4c,0x82,0x40]
+vgetexpsd 512(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vgetexpsd -512(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0x43,0x4c,0x82,0xc0]
+vgetexpsd -512(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vgetexpsd 512(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0x43,0x4c,0x82,0x40]
+vgetexpsd 512(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vgetexpsd -512(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0x43,0x4c,0x82,0xc0]
+vgetexpsd -512(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vgetexpsd 512(%rdx,%rax), %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x72,0x85,0x08,0x43,0x7c,0x02,0x40]
+vgetexpsd 512(%rdx,%rax), %xmm15, %xmm15
+
+// CHECK: vgetexpsd 512(%rdx,%rax), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x85,0x0a,0x43,0x7c,0x02,0x40]
+vgetexpsd 512(%rdx,%rax), %xmm15, %xmm15 {%k2}
+
+// CHECK: vgetexpsd 512(%rdx,%rax), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x85,0x8a,0x43,0x7c,0x02,0x40]
+vgetexpsd 512(%rdx,%rax), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vgetexpsd 512(%rdx,%rax), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x08,0x43,0x4c,0x02,0x40]
+vgetexpsd 512(%rdx,%rax), %xmm1, %xmm1
+
+// CHECK: vgetexpsd 512(%rdx,%rax), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0x43,0x4c,0x02,0x40]
+vgetexpsd 512(%rdx,%rax), %xmm1, %xmm1 {%k2}
+
+// CHECK: vgetexpsd 512(%rdx,%rax), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0x43,0x4c,0x02,0x40]
+vgetexpsd 512(%rdx,%rax), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vgetexpsd 512(%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x72,0x85,0x08,0x43,0x7a,0x40]
+vgetexpsd 512(%rdx), %xmm15, %xmm15
+
+// CHECK: vgetexpsd 512(%rdx), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x85,0x0a,0x43,0x7a,0x40]
+vgetexpsd 512(%rdx), %xmm15, %xmm15 {%k2}
+
+// CHECK: vgetexpsd 512(%rdx), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x85,0x8a,0x43,0x7a,0x40]
+vgetexpsd 512(%rdx), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vgetexpsd 512(%rdx), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x08,0x43,0x4a,0x40]
+vgetexpsd 512(%rdx), %xmm1, %xmm1
+
+// CHECK: vgetexpsd 512(%rdx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0x43,0x4a,0x40]
+vgetexpsd 512(%rdx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vgetexpsd 512(%rdx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0x43,0x4a,0x40]
+vgetexpsd 512(%rdx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vgetexpsd (%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x72,0x85,0x08,0x43,0x3a]
+vgetexpsd (%rdx), %xmm15, %xmm15
+
+// CHECK: vgetexpsd (%rdx), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x85,0x0a,0x43,0x3a]
+vgetexpsd (%rdx), %xmm15, %xmm15 {%k2}
+
+// CHECK: vgetexpsd (%rdx), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x85,0x8a,0x43,0x3a]
+vgetexpsd (%rdx), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vgetexpsd (%rdx), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x08,0x43,0x0a]
+vgetexpsd (%rdx), %xmm1, %xmm1
+
+// CHECK: vgetexpsd (%rdx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0x43,0x0a]
+vgetexpsd (%rdx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vgetexpsd (%rdx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0x43,0x0a]
+vgetexpsd (%rdx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vgetexpsd {sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x52,0x85,0x18,0x43,0xff]
+vgetexpsd {sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vgetexpsd {sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x85,0x1a,0x43,0xff]
+vgetexpsd {sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vgetexpsd {sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x85,0x9a,0x43,0xff]
+vgetexpsd {sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vgetexpsd {sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x18,0x43,0xc9]
+vgetexpsd {sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vgetexpsd {sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x1a,0x43,0xc9]
+vgetexpsd {sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vgetexpsd {sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x9a,0x43,0xc9]
+vgetexpsd {sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vgetexpsd %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x52,0x85,0x08,0x43,0xff]
+vgetexpsd %xmm15, %xmm15, %xmm15
+
+// CHECK: vgetexpsd %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x85,0x0a,0x43,0xff]
+vgetexpsd %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vgetexpsd %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x85,0x8a,0x43,0xff]
+vgetexpsd %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vgetexpsd %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x08,0x43,0xc9]
+vgetexpsd %xmm1, %xmm1, %xmm1
+
+// CHECK: vgetexpsd %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0x43,0xc9]
+vgetexpsd %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vgetexpsd %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0x43,0xc9]
+vgetexpsd %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vgetexpss 256(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x72,0x05,0x08,0x43,0x7c,0x82,0x40]
+vgetexpss 256(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vgetexpss -256(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x72,0x05,0x08,0x43,0x7c,0x82,0xc0]
+vgetexpss -256(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vgetexpss 256(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x05,0x0a,0x43,0x7c,0x82,0x40]
+vgetexpss 256(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+
+// CHECK: vgetexpss -256(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x05,0x0a,0x43,0x7c,0x82,0xc0]
+vgetexpss -256(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+
+// CHECK: vgetexpss 256(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x05,0x8a,0x43,0x7c,0x82,0x40]
+vgetexpss 256(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vgetexpss -256(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x05,0x8a,0x43,0x7c,0x82,0xc0]
+vgetexpss -256(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vgetexpss 256(%rdx,%rax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x08,0x43,0x4c,0x82,0x40]
+vgetexpss 256(%rdx,%rax,4), %xmm1, %xmm1
+
+// CHECK: vgetexpss -256(%rdx,%rax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x08,0x43,0x4c,0x82,0xc0]
+vgetexpss -256(%rdx,%rax,4), %xmm1, %xmm1
+
+// CHECK: vgetexpss 256(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0x43,0x4c,0x82,0x40]
+vgetexpss 256(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vgetexpss -256(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0x43,0x4c,0x82,0xc0]
+vgetexpss -256(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vgetexpss 256(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0x43,0x4c,0x82,0x40]
+vgetexpss 256(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vgetexpss -256(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0x43,0x4c,0x82,0xc0]
+vgetexpss -256(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vgetexpss 256(%rdx,%rax), %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x72,0x05,0x08,0x43,0x7c,0x02,0x40]
+vgetexpss 256(%rdx,%rax), %xmm15, %xmm15
+
+// CHECK: vgetexpss 256(%rdx,%rax), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x05,0x0a,0x43,0x7c,0x02,0x40]
+vgetexpss 256(%rdx,%rax), %xmm15, %xmm15 {%k2}
+
+// CHECK: vgetexpss 256(%rdx,%rax), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x05,0x8a,0x43,0x7c,0x02,0x40]
+vgetexpss 256(%rdx,%rax), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vgetexpss 256(%rdx,%rax), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x08,0x43,0x4c,0x02,0x40]
+vgetexpss 256(%rdx,%rax), %xmm1, %xmm1
+
+// CHECK: vgetexpss 256(%rdx,%rax), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0x43,0x4c,0x02,0x40]
+vgetexpss 256(%rdx,%rax), %xmm1, %xmm1 {%k2}
+
+// CHECK: vgetexpss 256(%rdx,%rax), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0x43,0x4c,0x02,0x40]
+vgetexpss 256(%rdx,%rax), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vgetexpss 256(%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x72,0x05,0x08,0x43,0x7a,0x40]
+vgetexpss 256(%rdx), %xmm15, %xmm15
+
+// CHECK: vgetexpss 256(%rdx), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x05,0x0a,0x43,0x7a,0x40]
+vgetexpss 256(%rdx), %xmm15, %xmm15 {%k2}
+
+// CHECK: vgetexpss 256(%rdx), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x05,0x8a,0x43,0x7a,0x40]
+vgetexpss 256(%rdx), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vgetexpss 256(%rdx), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x08,0x43,0x4a,0x40]
+vgetexpss 256(%rdx), %xmm1, %xmm1
+
+// CHECK: vgetexpss 256(%rdx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0x43,0x4a,0x40]
+vgetexpss 256(%rdx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vgetexpss 256(%rdx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0x43,0x4a,0x40]
+vgetexpss 256(%rdx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vgetexpss 485498096, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x72,0x05,0x08,0x43,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vgetexpss 485498096, %xmm15, %xmm15
+
+// CHECK: vgetexpss 485498096, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x05,0x0a,0x43,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vgetexpss 485498096, %xmm15, %xmm15 {%k2}
+
+// CHECK: vgetexpss 485498096, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x05,0x8a,0x43,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vgetexpss 485498096, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vgetexpss 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x08,0x43,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vgetexpss 485498096, %xmm1, %xmm1
+
+// CHECK: vgetexpss 485498096, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0x43,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vgetexpss 485498096, %xmm1, %xmm1 {%k2}
+
+// CHECK: vgetexpss 485498096, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0x43,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vgetexpss 485498096, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vgetexpss (%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x72,0x05,0x08,0x43,0x3a]
+vgetexpss (%rdx), %xmm15, %xmm15
+
+// CHECK: vgetexpss (%rdx), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x05,0x0a,0x43,0x3a]
+vgetexpss (%rdx), %xmm15, %xmm15 {%k2}
+
+// CHECK: vgetexpss (%rdx), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x05,0x8a,0x43,0x3a]
+vgetexpss (%rdx), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vgetexpss (%rdx), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x08,0x43,0x0a]
+vgetexpss (%rdx), %xmm1, %xmm1
+
+// CHECK: vgetexpss (%rdx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0x43,0x0a]
+vgetexpss (%rdx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vgetexpss (%rdx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0x43,0x0a]
+vgetexpss (%rdx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vgetexpss {sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x52,0x05,0x18,0x43,0xff]
+vgetexpss {sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vgetexpss {sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x05,0x1a,0x43,0xff]
+vgetexpss {sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vgetexpss {sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x05,0x9a,0x43,0xff]
+vgetexpss {sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vgetexpss {sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x18,0x43,0xc9]
+vgetexpss {sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vgetexpss {sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x1a,0x43,0xc9]
+vgetexpss {sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vgetexpss {sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x9a,0x43,0xc9]
+vgetexpss {sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vgetexpss %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x52,0x05,0x08,0x43,0xff]
+vgetexpss %xmm15, %xmm15, %xmm15
+
+// CHECK: vgetexpss %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x05,0x0a,0x43,0xff]
+vgetexpss %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vgetexpss %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x05,0x8a,0x43,0xff]
+vgetexpss %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vgetexpss %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x08,0x43,0xc9]
+vgetexpss %xmm1, %xmm1, %xmm1
+
+// CHECK: vgetexpss %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0x43,0xc9]
+vgetexpss %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vgetexpss %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0x43,0xc9]
+vgetexpss %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vgetmantsd $0, 485498096, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x73,0x85,0x08,0x27,0x3c,0x25,0xf0,0x1c,0xf0,0x1c,0x00]
+vgetmantsd $0, 485498096, %xmm15, %xmm15
+
+// CHECK: vgetmantsd $0, 485498096, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x73,0x85,0x0a,0x27,0x3c,0x25,0xf0,0x1c,0xf0,0x1c,0x00]
+vgetmantsd $0, 485498096, %xmm15, %xmm15 {%k2}
+
+// CHECK: vgetmantsd $0, 485498096, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x73,0x85,0x8a,0x27,0x3c,0x25,0xf0,0x1c,0xf0,0x1c,0x00]
+vgetmantsd $0, 485498096, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vgetmantsd $0, 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf3,0xf5,0x08,0x27,0x0c,0x25,0xf0,0x1c,0xf0,0x1c,0x00]
+vgetmantsd $0, 485498096, %xmm1, %xmm1
+
+// CHECK: vgetmantsd $0, 485498096, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf3,0xf5,0x0a,0x27,0x0c,0x25,0xf0,0x1c,0xf0,0x1c,0x00]
+vgetmantsd $0, 485498096, %xmm1, %xmm1 {%k2}
+
+// CHECK: vgetmantsd $0, 485498096, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf3,0xf5,0x8a,0x27,0x0c,0x25,0xf0,0x1c,0xf0,0x1c,0x00]
+vgetmantsd $0, 485498096, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vgetmantsd $0, 512(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x73,0x85,0x08,0x27,0x7c,0x82,0x40,0x00]
+vgetmantsd $0, 512(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vgetmantsd $0, -512(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x73,0x85,0x08,0x27,0x7c,0x82,0xc0,0x00]
+vgetmantsd $0, -512(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vgetmantsd $0, 512(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x73,0x85,0x0a,0x27,0x7c,0x82,0x40,0x00]
+vgetmantsd $0, 512(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+
+// CHECK: vgetmantsd $0, -512(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x73,0x85,0x0a,0x27,0x7c,0x82,0xc0,0x00]
+vgetmantsd $0, -512(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+
+// CHECK: vgetmantsd $0, 512(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x73,0x85,0x8a,0x27,0x7c,0x82,0x40,0x00]
+vgetmantsd $0, 512(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vgetmantsd $0, -512(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x73,0x85,0x8a,0x27,0x7c,0x82,0xc0,0x00]
+vgetmantsd $0, -512(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vgetmantsd $0, 512(%rdx,%rax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf3,0xf5,0x08,0x27,0x4c,0x82,0x40,0x00]
+vgetmantsd $0, 512(%rdx,%rax,4), %xmm1, %xmm1
+
+// CHECK: vgetmantsd $0, -512(%rdx,%rax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf3,0xf5,0x08,0x27,0x4c,0x82,0xc0,0x00]
+vgetmantsd $0, -512(%rdx,%rax,4), %xmm1, %xmm1
+
+// CHECK: vgetmantsd $0, 512(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf3,0xf5,0x0a,0x27,0x4c,0x82,0x40,0x00]
+vgetmantsd $0, 512(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vgetmantsd $0, -512(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf3,0xf5,0x0a,0x27,0x4c,0x82,0xc0,0x00]
+vgetmantsd $0, -512(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vgetmantsd $0, 512(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf3,0xf5,0x8a,0x27,0x4c,0x82,0x40,0x00]
+vgetmantsd $0, 512(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vgetmantsd $0, -512(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf3,0xf5,0x8a,0x27,0x4c,0x82,0xc0,0x00]
+vgetmantsd $0, -512(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vgetmantsd $0, 512(%rdx,%rax), %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x73,0x85,0x08,0x27,0x7c,0x02,0x40,0x00]
+vgetmantsd $0, 512(%rdx,%rax), %xmm15, %xmm15
+
+// CHECK: vgetmantsd $0, 512(%rdx,%rax), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x73,0x85,0x0a,0x27,0x7c,0x02,0x40,0x00]
+vgetmantsd $0, 512(%rdx,%rax), %xmm15, %xmm15 {%k2}
+
+// CHECK: vgetmantsd $0, 512(%rdx,%rax), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x73,0x85,0x8a,0x27,0x7c,0x02,0x40,0x00]
+vgetmantsd $0, 512(%rdx,%rax), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vgetmantsd $0, 512(%rdx,%rax), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf3,0xf5,0x08,0x27,0x4c,0x02,0x40,0x00]
+vgetmantsd $0, 512(%rdx,%rax), %xmm1, %xmm1
+
+// CHECK: vgetmantsd $0, 512(%rdx,%rax), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf3,0xf5,0x0a,0x27,0x4c,0x02,0x40,0x00]
+vgetmantsd $0, 512(%rdx,%rax), %xmm1, %xmm1 {%k2}
+
+// CHECK: vgetmantsd $0, 512(%rdx,%rax), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf3,0xf5,0x8a,0x27,0x4c,0x02,0x40,0x00]
+vgetmantsd $0, 512(%rdx,%rax), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vgetmantsd $0, 512(%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x73,0x85,0x08,0x27,0x7a,0x40,0x00]
+vgetmantsd $0, 512(%rdx), %xmm15, %xmm15
+
+// CHECK: vgetmantsd $0, 512(%rdx), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x73,0x85,0x0a,0x27,0x7a,0x40,0x00]
+vgetmantsd $0, 512(%rdx), %xmm15, %xmm15 {%k2}
+
+// CHECK: vgetmantsd $0, 512(%rdx), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x73,0x85,0x8a,0x27,0x7a,0x40,0x00]
+vgetmantsd $0, 512(%rdx), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vgetmantsd $0, 512(%rdx), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf3,0xf5,0x08,0x27,0x4a,0x40,0x00]
+vgetmantsd $0, 512(%rdx), %xmm1, %xmm1
+
+// CHECK: vgetmantsd $0, 512(%rdx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf3,0xf5,0x0a,0x27,0x4a,0x40,0x00]
+vgetmantsd $0, 512(%rdx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vgetmantsd $0, 512(%rdx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf3,0xf5,0x8a,0x27,0x4a,0x40,0x00]
+vgetmantsd $0, 512(%rdx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vgetmantsd $0, (%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x73,0x85,0x08,0x27,0x3a,0x00]
+vgetmantsd $0, (%rdx), %xmm15, %xmm15
+
+// CHECK: vgetmantsd $0, (%rdx), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x73,0x85,0x0a,0x27,0x3a,0x00]
+vgetmantsd $0, (%rdx), %xmm15, %xmm15 {%k2}
+
+// CHECK: vgetmantsd $0, (%rdx), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x73,0x85,0x8a,0x27,0x3a,0x00]
+vgetmantsd $0, (%rdx), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vgetmantsd $0, (%rdx), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf3,0xf5,0x08,0x27,0x0a,0x00]
+vgetmantsd $0, (%rdx), %xmm1, %xmm1
+
+// CHECK: vgetmantsd $0, (%rdx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf3,0xf5,0x0a,0x27,0x0a,0x00]
+vgetmantsd $0, (%rdx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vgetmantsd $0, (%rdx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf3,0xf5,0x8a,0x27,0x0a,0x00]
+vgetmantsd $0, (%rdx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vgetmantsd $0, {sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x53,0x85,0x18,0x27,0xff,0x00]
+vgetmantsd $0, {sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vgetmantsd $0, {sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x53,0x85,0x1a,0x27,0xff,0x00]
+vgetmantsd $0, {sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vgetmantsd $0, {sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x53,0x85,0x9a,0x27,0xff,0x00]
+vgetmantsd $0, {sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vgetmantsd $0, {sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf3,0xf5,0x18,0x27,0xc9,0x00]
+vgetmantsd $0, {sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vgetmantsd $0, {sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf3,0xf5,0x1a,0x27,0xc9,0x00]
+vgetmantsd $0, {sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vgetmantsd $0, {sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf3,0xf5,0x9a,0x27,0xc9,0x00]
+vgetmantsd $0, {sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vgetmantsd $0, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x53,0x85,0x08,0x27,0xff,0x00]
+vgetmantsd $0, %xmm15, %xmm15, %xmm15
+
+// CHECK: vgetmantsd $0, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x53,0x85,0x0a,0x27,0xff,0x00]
+vgetmantsd $0, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vgetmantsd $0, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x53,0x85,0x8a,0x27,0xff,0x00]
+vgetmantsd $0, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vgetmantsd $0, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf3,0xf5,0x08,0x27,0xc9,0x00]
+vgetmantsd $0, %xmm1, %xmm1, %xmm1
+
+// CHECK: vgetmantsd $0, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf3,0xf5,0x0a,0x27,0xc9,0x00]
+vgetmantsd $0, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vgetmantsd $0, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf3,0xf5,0x8a,0x27,0xc9,0x00]
+vgetmantsd $0, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vgetmantss $0, 256(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x73,0x05,0x08,0x27,0x7c,0x82,0x40,0x00]
+vgetmantss $0, 256(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vgetmantss $0, -256(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x73,0x05,0x08,0x27,0x7c,0x82,0xc0,0x00]
+vgetmantss $0, -256(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vgetmantss $0, 256(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x73,0x05,0x0a,0x27,0x7c,0x82,0x40,0x00]
+vgetmantss $0, 256(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+
+// CHECK: vgetmantss $0, -256(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x73,0x05,0x0a,0x27,0x7c,0x82,0xc0,0x00]
+vgetmantss $0, -256(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+
+// CHECK: vgetmantss $0, 256(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x73,0x05,0x8a,0x27,0x7c,0x82,0x40,0x00]
+vgetmantss $0, 256(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vgetmantss $0, -256(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x73,0x05,0x8a,0x27,0x7c,0x82,0xc0,0x00]
+vgetmantss $0, -256(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vgetmantss $0, 256(%rdx,%rax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf3,0x75,0x08,0x27,0x4c,0x82,0x40,0x00]
+vgetmantss $0, 256(%rdx,%rax,4), %xmm1, %xmm1
+
+// CHECK: vgetmantss $0, -256(%rdx,%rax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf3,0x75,0x08,0x27,0x4c,0x82,0xc0,0x00]
+vgetmantss $0, -256(%rdx,%rax,4), %xmm1, %xmm1
+
+// CHECK: vgetmantss $0, 256(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf3,0x75,0x0a,0x27,0x4c,0x82,0x40,0x00]
+vgetmantss $0, 256(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vgetmantss $0, -256(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf3,0x75,0x0a,0x27,0x4c,0x82,0xc0,0x00]
+vgetmantss $0, -256(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vgetmantss $0, 256(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf3,0x75,0x8a,0x27,0x4c,0x82,0x40,0x00]
+vgetmantss $0, 256(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vgetmantss $0, -256(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf3,0x75,0x8a,0x27,0x4c,0x82,0xc0,0x00]
+vgetmantss $0, -256(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vgetmantss $0, 256(%rdx,%rax), %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x73,0x05,0x08,0x27,0x7c,0x02,0x40,0x00]
+vgetmantss $0, 256(%rdx,%rax), %xmm15, %xmm15
+
+// CHECK: vgetmantss $0, 256(%rdx,%rax), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x73,0x05,0x0a,0x27,0x7c,0x02,0x40,0x00]
+vgetmantss $0, 256(%rdx,%rax), %xmm15, %xmm15 {%k2}
+
+// CHECK: vgetmantss $0, 256(%rdx,%rax), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x73,0x05,0x8a,0x27,0x7c,0x02,0x40,0x00]
+vgetmantss $0, 256(%rdx,%rax), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vgetmantss $0, 256(%rdx,%rax), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf3,0x75,0x08,0x27,0x4c,0x02,0x40,0x00]
+vgetmantss $0, 256(%rdx,%rax), %xmm1, %xmm1
+
+// CHECK: vgetmantss $0, 256(%rdx,%rax), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf3,0x75,0x0a,0x27,0x4c,0x02,0x40,0x00]
+vgetmantss $0, 256(%rdx,%rax), %xmm1, %xmm1 {%k2}
+
+// CHECK: vgetmantss $0, 256(%rdx,%rax), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf3,0x75,0x8a,0x27,0x4c,0x02,0x40,0x00]
+vgetmantss $0, 256(%rdx,%rax), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vgetmantss $0, 256(%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x73,0x05,0x08,0x27,0x7a,0x40,0x00]
+vgetmantss $0, 256(%rdx), %xmm15, %xmm15
+
+// CHECK: vgetmantss $0, 256(%rdx), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x73,0x05,0x0a,0x27,0x7a,0x40,0x00]
+vgetmantss $0, 256(%rdx), %xmm15, %xmm15 {%k2}
+
+// CHECK: vgetmantss $0, 256(%rdx), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x73,0x05,0x8a,0x27,0x7a,0x40,0x00]
+vgetmantss $0, 256(%rdx), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vgetmantss $0, 256(%rdx), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf3,0x75,0x08,0x27,0x4a,0x40,0x00]
+vgetmantss $0, 256(%rdx), %xmm1, %xmm1
+
+// CHECK: vgetmantss $0, 256(%rdx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf3,0x75,0x0a,0x27,0x4a,0x40,0x00]
+vgetmantss $0, 256(%rdx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vgetmantss $0, 256(%rdx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf3,0x75,0x8a,0x27,0x4a,0x40,0x00]
+vgetmantss $0, 256(%rdx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vgetmantss $0, 485498096, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x73,0x05,0x08,0x27,0x3c,0x25,0xf0,0x1c,0xf0,0x1c,0x00]
+vgetmantss $0, 485498096, %xmm15, %xmm15
+
+// CHECK: vgetmantss $0, 485498096, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x73,0x05,0x0a,0x27,0x3c,0x25,0xf0,0x1c,0xf0,0x1c,0x00]
+vgetmantss $0, 485498096, %xmm15, %xmm15 {%k2}
+
+// CHECK: vgetmantss $0, 485498096, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x73,0x05,0x8a,0x27,0x3c,0x25,0xf0,0x1c,0xf0,0x1c,0x00]
+vgetmantss $0, 485498096, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vgetmantss $0, 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf3,0x75,0x08,0x27,0x0c,0x25,0xf0,0x1c,0xf0,0x1c,0x00]
+vgetmantss $0, 485498096, %xmm1, %xmm1
+
+// CHECK: vgetmantss $0, 485498096, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf3,0x75,0x0a,0x27,0x0c,0x25,0xf0,0x1c,0xf0,0x1c,0x00]
+vgetmantss $0, 485498096, %xmm1, %xmm1 {%k2}
+
+// CHECK: vgetmantss $0, 485498096, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf3,0x75,0x8a,0x27,0x0c,0x25,0xf0,0x1c,0xf0,0x1c,0x00]
+vgetmantss $0, 485498096, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vgetmantss $0, (%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x73,0x05,0x08,0x27,0x3a,0x00]
+vgetmantss $0, (%rdx), %xmm15, %xmm15
+
+// CHECK: vgetmantss $0, (%rdx), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x73,0x05,0x0a,0x27,0x3a,0x00]
+vgetmantss $0, (%rdx), %xmm15, %xmm15 {%k2}
+
+// CHECK: vgetmantss $0, (%rdx), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x73,0x05,0x8a,0x27,0x3a,0x00]
+vgetmantss $0, (%rdx), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vgetmantss $0, (%rdx), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf3,0x75,0x08,0x27,0x0a,0x00]
+vgetmantss $0, (%rdx), %xmm1, %xmm1
+
+// CHECK: vgetmantss $0, (%rdx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf3,0x75,0x0a,0x27,0x0a,0x00]
+vgetmantss $0, (%rdx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vgetmantss $0, (%rdx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf3,0x75,0x8a,0x27,0x0a,0x00]
+vgetmantss $0, (%rdx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vgetmantss $0, {sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x53,0x05,0x18,0x27,0xff,0x00]
+vgetmantss $0, {sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vgetmantss $0, {sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x53,0x05,0x1a,0x27,0xff,0x00]
+vgetmantss $0, {sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vgetmantss $0, {sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x53,0x05,0x9a,0x27,0xff,0x00]
+vgetmantss $0, {sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vgetmantss $0, {sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf3,0x75,0x18,0x27,0xc9,0x00]
+vgetmantss $0, {sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vgetmantss $0, {sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf3,0x75,0x1a,0x27,0xc9,0x00]
+vgetmantss $0, {sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vgetmantss $0, {sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf3,0x75,0x9a,0x27,0xc9,0x00]
+vgetmantss $0, {sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vgetmantss $0, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x53,0x05,0x08,0x27,0xff,0x00]
+vgetmantss $0, %xmm15, %xmm15, %xmm15
+
+// CHECK: vgetmantss $0, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x53,0x05,0x0a,0x27,0xff,0x00]
+vgetmantss $0, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vgetmantss $0, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x53,0x05,0x8a,0x27,0xff,0x00]
+vgetmantss $0, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vgetmantss $0, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf3,0x75,0x08,0x27,0xc9,0x00]
+vgetmantss $0, %xmm1, %xmm1, %xmm1
+
+// CHECK: vgetmantss $0, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf3,0x75,0x0a,0x27,0xc9,0x00]
+vgetmantss $0, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vgetmantss $0, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf3,0x75,0x8a,0x27,0xc9,0x00]
+vgetmantss $0, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vmaxsd 485498096, %xmm15, %xmm15
+// CHECK: encoding: [0xc5,0x03,0x5f,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vmaxsd 485498096, %xmm15, %xmm15
+
+// CHECK: vmaxsd 485498096, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x71,0x87,0x0a,0x5f,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vmaxsd 485498096, %xmm15, %xmm15 {%k2}
+
+// CHECK: vmaxsd 485498096, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x71,0x87,0x8a,0x5f,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vmaxsd 485498096, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vmaxsd 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x5f,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vmaxsd 485498096, %xmm1, %xmm1
+
+// CHECK: vmaxsd 485498096, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0x5f,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vmaxsd 485498096, %xmm1, %xmm1 {%k2}
+
+// CHECK: vmaxsd 485498096, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x8a,0x5f,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vmaxsd 485498096, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vmaxsd 512(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0xc5,0x03,0x5f,0xbc,0x82,0x00,0x02,0x00,0x00]
+vmaxsd 512(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vmaxsd -512(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0xc5,0x03,0x5f,0xbc,0x82,0x00,0xfe,0xff,0xff]
+vmaxsd -512(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vmaxsd 512(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x71,0x87,0x0a,0x5f,0x7c,0x82,0x40]
+vmaxsd 512(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+
+// CHECK: vmaxsd -512(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x71,0x87,0x0a,0x5f,0x7c,0x82,0xc0]
+vmaxsd -512(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+
+// CHECK: vmaxsd 512(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x71,0x87,0x8a,0x5f,0x7c,0x82,0x40]
+vmaxsd 512(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vmaxsd -512(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x71,0x87,0x8a,0x5f,0x7c,0x82,0xc0]
+vmaxsd -512(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vmaxsd 512(%rdx,%rax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x5f,0x8c,0x82,0x00,0x02,0x00,0x00]
+vmaxsd 512(%rdx,%rax,4), %xmm1, %xmm1
+
+// CHECK: vmaxsd -512(%rdx,%rax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x5f,0x8c,0x82,0x00,0xfe,0xff,0xff]
+vmaxsd -512(%rdx,%rax,4), %xmm1, %xmm1
+
+// CHECK: vmaxsd 512(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0x5f,0x4c,0x82,0x40]
+vmaxsd 512(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vmaxsd -512(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0x5f,0x4c,0x82,0xc0]
+vmaxsd -512(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vmaxsd 512(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x8a,0x5f,0x4c,0x82,0x40]
+vmaxsd 512(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vmaxsd -512(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x8a,0x5f,0x4c,0x82,0xc0]
+vmaxsd -512(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vmaxsd 512(%rdx,%rax), %xmm15, %xmm15
+// CHECK: encoding: [0xc5,0x03,0x5f,0xbc,0x02,0x00,0x02,0x00,0x00]
+vmaxsd 512(%rdx,%rax), %xmm15, %xmm15
+
+// CHECK: vmaxsd 512(%rdx,%rax), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x71,0x87,0x0a,0x5f,0x7c,0x02,0x40]
+vmaxsd 512(%rdx,%rax), %xmm15, %xmm15 {%k2}
+
+// CHECK: vmaxsd 512(%rdx,%rax), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x71,0x87,0x8a,0x5f,0x7c,0x02,0x40]
+vmaxsd 512(%rdx,%rax), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vmaxsd 512(%rdx,%rax), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x5f,0x8c,0x02,0x00,0x02,0x00,0x00]
+vmaxsd 512(%rdx,%rax), %xmm1, %xmm1
+
+// CHECK: vmaxsd 512(%rdx,%rax), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0x5f,0x4c,0x02,0x40]
+vmaxsd 512(%rdx,%rax), %xmm1, %xmm1 {%k2}
+
+// CHECK: vmaxsd 512(%rdx,%rax), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x8a,0x5f,0x4c,0x02,0x40]
+vmaxsd 512(%rdx,%rax), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vmaxsd 512(%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0xc5,0x03,0x5f,0xba,0x00,0x02,0x00,0x00]
+vmaxsd 512(%rdx), %xmm15, %xmm15
+
+// CHECK: vmaxsd 512(%rdx), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x71,0x87,0x0a,0x5f,0x7a,0x40]
+vmaxsd 512(%rdx), %xmm15, %xmm15 {%k2}
+
+// CHECK: vmaxsd 512(%rdx), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x71,0x87,0x8a,0x5f,0x7a,0x40]
+vmaxsd 512(%rdx), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vmaxsd 512(%rdx), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x5f,0x8a,0x00,0x02,0x00,0x00]
+vmaxsd 512(%rdx), %xmm1, %xmm1
+
+// CHECK: vmaxsd 512(%rdx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0x5f,0x4a,0x40]
+vmaxsd 512(%rdx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vmaxsd 512(%rdx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x8a,0x5f,0x4a,0x40]
+vmaxsd 512(%rdx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vmaxsd (%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0xc5,0x03,0x5f,0x3a]
+vmaxsd (%rdx), %xmm15, %xmm15
+
+// CHECK: vmaxsd (%rdx), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x71,0x87,0x0a,0x5f,0x3a]
+vmaxsd (%rdx), %xmm15, %xmm15 {%k2}
+
+// CHECK: vmaxsd (%rdx), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x71,0x87,0x8a,0x5f,0x3a]
+vmaxsd (%rdx), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vmaxsd (%rdx), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x5f,0x0a]
+vmaxsd (%rdx), %xmm1, %xmm1
+
+// CHECK: vmaxsd (%rdx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0x5f,0x0a]
+vmaxsd (%rdx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vmaxsd (%rdx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x8a,0x5f,0x0a]
+vmaxsd (%rdx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vmaxsd {sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x51,0x87,0x18,0x5f,0xff]
+vmaxsd {sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vmaxsd {sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x51,0x87,0x1a,0x5f,0xff]
+vmaxsd {sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vmaxsd {sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x51,0x87,0x9a,0x5f,0xff]
+vmaxsd {sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vmaxsd {sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0xf7,0x18,0x5f,0xc9]
+vmaxsd {sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vmaxsd {sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x1a,0x5f,0xc9]
+vmaxsd {sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vmaxsd {sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x9a,0x5f,0xc9]
+vmaxsd {sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vmaxsd %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x41,0x03,0x5f,0xff]
+vmaxsd %xmm15, %xmm15, %xmm15
+
+// CHECK: vmaxsd %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x51,0x87,0x0a,0x5f,0xff]
+vmaxsd %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vmaxsd %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x51,0x87,0x8a,0x5f,0xff]
+vmaxsd %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vmaxsd %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x5f,0xc9]
+vmaxsd %xmm1, %xmm1, %xmm1
+
+// CHECK: vmaxsd %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0x5f,0xc9]
+vmaxsd %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vmaxsd %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x8a,0x5f,0xc9]
+vmaxsd %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vmaxss 256(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0xc5,0x02,0x5f,0xbc,0x82,0x00,0x01,0x00,0x00]
+vmaxss 256(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vmaxss -256(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0xc5,0x02,0x5f,0xbc,0x82,0x00,0xff,0xff,0xff]
+vmaxss -256(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vmaxss 256(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x71,0x06,0x0a,0x5f,0x7c,0x82,0x40]
+vmaxss 256(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+
+// CHECK: vmaxss -256(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x71,0x06,0x0a,0x5f,0x7c,0x82,0xc0]
+vmaxss -256(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+
+// CHECK: vmaxss 256(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x71,0x06,0x8a,0x5f,0x7c,0x82,0x40]
+vmaxss 256(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vmaxss -256(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x71,0x06,0x8a,0x5f,0x7c,0x82,0xc0]
+vmaxss -256(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vmaxss 256(%rdx,%rax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x5f,0x8c,0x82,0x00,0x01,0x00,0x00]
+vmaxss 256(%rdx,%rax,4), %xmm1, %xmm1
+
+// CHECK: vmaxss -256(%rdx,%rax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x5f,0x8c,0x82,0x00,0xff,0xff,0xff]
+vmaxss -256(%rdx,%rax,4), %xmm1, %xmm1
+
+// CHECK: vmaxss 256(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0x5f,0x4c,0x82,0x40]
+vmaxss 256(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vmaxss -256(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0x5f,0x4c,0x82,0xc0]
+vmaxss -256(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vmaxss 256(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x8a,0x5f,0x4c,0x82,0x40]
+vmaxss 256(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vmaxss -256(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x8a,0x5f,0x4c,0x82,0xc0]
+vmaxss -256(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vmaxss 256(%rdx,%rax), %xmm15, %xmm15
+// CHECK: encoding: [0xc5,0x02,0x5f,0xbc,0x02,0x00,0x01,0x00,0x00]
+vmaxss 256(%rdx,%rax), %xmm15, %xmm15
+
+// CHECK: vmaxss 256(%rdx,%rax), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x71,0x06,0x0a,0x5f,0x7c,0x02,0x40]
+vmaxss 256(%rdx,%rax), %xmm15, %xmm15 {%k2}
+
+// CHECK: vmaxss 256(%rdx,%rax), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x71,0x06,0x8a,0x5f,0x7c,0x02,0x40]
+vmaxss 256(%rdx,%rax), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vmaxss 256(%rdx,%rax), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x5f,0x8c,0x02,0x00,0x01,0x00,0x00]
+vmaxss 256(%rdx,%rax), %xmm1, %xmm1
+
+// CHECK: vmaxss 256(%rdx,%rax), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0x5f,0x4c,0x02,0x40]
+vmaxss 256(%rdx,%rax), %xmm1, %xmm1 {%k2}
+
+// CHECK: vmaxss 256(%rdx,%rax), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x8a,0x5f,0x4c,0x02,0x40]
+vmaxss 256(%rdx,%rax), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vmaxss 256(%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0xc5,0x02,0x5f,0xba,0x00,0x01,0x00,0x00]
+vmaxss 256(%rdx), %xmm15, %xmm15
+
+// CHECK: vmaxss 256(%rdx), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x71,0x06,0x0a,0x5f,0x7a,0x40]
+vmaxss 256(%rdx), %xmm15, %xmm15 {%k2}
+
+// CHECK: vmaxss 256(%rdx), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x71,0x06,0x8a,0x5f,0x7a,0x40]
+vmaxss 256(%rdx), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vmaxss 256(%rdx), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x5f,0x8a,0x00,0x01,0x00,0x00]
+vmaxss 256(%rdx), %xmm1, %xmm1
+
+// CHECK: vmaxss 256(%rdx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0x5f,0x4a,0x40]
+vmaxss 256(%rdx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vmaxss 256(%rdx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x8a,0x5f,0x4a,0x40]
+vmaxss 256(%rdx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vmaxss 485498096, %xmm15, %xmm15
+// CHECK: encoding: [0xc5,0x02,0x5f,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vmaxss 485498096, %xmm15, %xmm15
+
+// CHECK: vmaxss 485498096, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x71,0x06,0x0a,0x5f,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vmaxss 485498096, %xmm15, %xmm15 {%k2}
+
+// CHECK: vmaxss 485498096, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x71,0x06,0x8a,0x5f,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vmaxss 485498096, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vmaxss 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x5f,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vmaxss 485498096, %xmm1, %xmm1
+
+// CHECK: vmaxss 485498096, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0x5f,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vmaxss 485498096, %xmm1, %xmm1 {%k2}
+
+// CHECK: vmaxss 485498096, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x8a,0x5f,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vmaxss 485498096, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vmaxss (%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0xc5,0x02,0x5f,0x3a]
+vmaxss (%rdx), %xmm15, %xmm15
+
+// CHECK: vmaxss (%rdx), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x71,0x06,0x0a,0x5f,0x3a]
+vmaxss (%rdx), %xmm15, %xmm15 {%k2}
+
+// CHECK: vmaxss (%rdx), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x71,0x06,0x8a,0x5f,0x3a]
+vmaxss (%rdx), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vmaxss (%rdx), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x5f,0x0a]
+vmaxss (%rdx), %xmm1, %xmm1
+
+// CHECK: vmaxss (%rdx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0x5f,0x0a]
+vmaxss (%rdx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vmaxss (%rdx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x8a,0x5f,0x0a]
+vmaxss (%rdx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vmaxss {sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x51,0x06,0x18,0x5f,0xff]
+vmaxss {sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vmaxss {sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x51,0x06,0x1a,0x5f,0xff]
+vmaxss {sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vmaxss {sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x51,0x06,0x9a,0x5f,0xff]
+vmaxss {sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vmaxss {sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0x76,0x18,0x5f,0xc9]
+vmaxss {sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vmaxss {sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x1a,0x5f,0xc9]
+vmaxss {sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vmaxss {sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x9a,0x5f,0xc9]
+vmaxss {sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vmaxss %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x41,0x02,0x5f,0xff]
+vmaxss %xmm15, %xmm15, %xmm15
+
+// CHECK: vmaxss %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x51,0x06,0x0a,0x5f,0xff]
+vmaxss %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vmaxss %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x51,0x06,0x8a,0x5f,0xff]
+vmaxss %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vmaxss %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x5f,0xc9]
+vmaxss %xmm1, %xmm1, %xmm1
+
+// CHECK: vmaxss %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0x5f,0xc9]
+vmaxss %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vmaxss %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x8a,0x5f,0xc9]
+vmaxss %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vminsd 485498096, %xmm15, %xmm15
+// CHECK: encoding: [0xc5,0x03,0x5d,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vminsd 485498096, %xmm15, %xmm15
+
+// CHECK: vminsd 485498096, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x71,0x87,0x0a,0x5d,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vminsd 485498096, %xmm15, %xmm15 {%k2}
+
+// CHECK: vminsd 485498096, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x71,0x87,0x8a,0x5d,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vminsd 485498096, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vminsd 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x5d,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vminsd 485498096, %xmm1, %xmm1
+
+// CHECK: vminsd 485498096, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0x5d,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vminsd 485498096, %xmm1, %xmm1 {%k2}
+
+// CHECK: vminsd 485498096, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x8a,0x5d,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vminsd 485498096, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vminsd 512(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0xc5,0x03,0x5d,0xbc,0x82,0x00,0x02,0x00,0x00]
+vminsd 512(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vminsd -512(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0xc5,0x03,0x5d,0xbc,0x82,0x00,0xfe,0xff,0xff]
+vminsd -512(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vminsd 512(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x71,0x87,0x0a,0x5d,0x7c,0x82,0x40]
+vminsd 512(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+
+// CHECK: vminsd -512(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x71,0x87,0x0a,0x5d,0x7c,0x82,0xc0]
+vminsd -512(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+
+// CHECK: vminsd 512(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x71,0x87,0x8a,0x5d,0x7c,0x82,0x40]
+vminsd 512(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vminsd -512(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x71,0x87,0x8a,0x5d,0x7c,0x82,0xc0]
+vminsd -512(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vminsd 512(%rdx,%rax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x5d,0x8c,0x82,0x00,0x02,0x00,0x00]
+vminsd 512(%rdx,%rax,4), %xmm1, %xmm1
+
+// CHECK: vminsd -512(%rdx,%rax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x5d,0x8c,0x82,0x00,0xfe,0xff,0xff]
+vminsd -512(%rdx,%rax,4), %xmm1, %xmm1
+
+// CHECK: vminsd 512(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0x5d,0x4c,0x82,0x40]
+vminsd 512(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vminsd -512(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0x5d,0x4c,0x82,0xc0]
+vminsd -512(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vminsd 512(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x8a,0x5d,0x4c,0x82,0x40]
+vminsd 512(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vminsd -512(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x8a,0x5d,0x4c,0x82,0xc0]
+vminsd -512(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vminsd 512(%rdx,%rax), %xmm15, %xmm15
+// CHECK: encoding: [0xc5,0x03,0x5d,0xbc,0x02,0x00,0x02,0x00,0x00]
+vminsd 512(%rdx,%rax), %xmm15, %xmm15
+
+// CHECK: vminsd 512(%rdx,%rax), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x71,0x87,0x0a,0x5d,0x7c,0x02,0x40]
+vminsd 512(%rdx,%rax), %xmm15, %xmm15 {%k2}
+
+// CHECK: vminsd 512(%rdx,%rax), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x71,0x87,0x8a,0x5d,0x7c,0x02,0x40]
+vminsd 512(%rdx,%rax), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vminsd 512(%rdx,%rax), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x5d,0x8c,0x02,0x00,0x02,0x00,0x00]
+vminsd 512(%rdx,%rax), %xmm1, %xmm1
+
+// CHECK: vminsd 512(%rdx,%rax), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0x5d,0x4c,0x02,0x40]
+vminsd 512(%rdx,%rax), %xmm1, %xmm1 {%k2}
+
+// CHECK: vminsd 512(%rdx,%rax), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x8a,0x5d,0x4c,0x02,0x40]
+vminsd 512(%rdx,%rax), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vminsd 512(%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0xc5,0x03,0x5d,0xba,0x00,0x02,0x00,0x00]
+vminsd 512(%rdx), %xmm15, %xmm15
+
+// CHECK: vminsd 512(%rdx), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x71,0x87,0x0a,0x5d,0x7a,0x40]
+vminsd 512(%rdx), %xmm15, %xmm15 {%k2}
+
+// CHECK: vminsd 512(%rdx), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x71,0x87,0x8a,0x5d,0x7a,0x40]
+vminsd 512(%rdx), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vminsd 512(%rdx), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x5d,0x8a,0x00,0x02,0x00,0x00]
+vminsd 512(%rdx), %xmm1, %xmm1
+
+// CHECK: vminsd 512(%rdx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0x5d,0x4a,0x40]
+vminsd 512(%rdx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vminsd 512(%rdx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x8a,0x5d,0x4a,0x40]
+vminsd 512(%rdx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vminsd (%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0xc5,0x03,0x5d,0x3a]
+vminsd (%rdx), %xmm15, %xmm15
+
+// CHECK: vminsd (%rdx), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x71,0x87,0x0a,0x5d,0x3a]
+vminsd (%rdx), %xmm15, %xmm15 {%k2}
+
+// CHECK: vminsd (%rdx), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x71,0x87,0x8a,0x5d,0x3a]
+vminsd (%rdx), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vminsd (%rdx), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x5d,0x0a]
+vminsd (%rdx), %xmm1, %xmm1
+
+// CHECK: vminsd (%rdx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0x5d,0x0a]
+vminsd (%rdx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vminsd (%rdx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x8a,0x5d,0x0a]
+vminsd (%rdx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vminsd {sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x51,0x87,0x18,0x5d,0xff]
+vminsd {sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vminsd {sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x51,0x87,0x1a,0x5d,0xff]
+vminsd {sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vminsd {sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x51,0x87,0x9a,0x5d,0xff]
+vminsd {sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vminsd {sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0xf7,0x18,0x5d,0xc9]
+vminsd {sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vminsd {sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x1a,0x5d,0xc9]
+vminsd {sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vminsd {sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x9a,0x5d,0xc9]
+vminsd {sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vminsd %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x41,0x03,0x5d,0xff]
+vminsd %xmm15, %xmm15, %xmm15
+
+// CHECK: vminsd %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x51,0x87,0x0a,0x5d,0xff]
+vminsd %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vminsd %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x51,0x87,0x8a,0x5d,0xff]
+vminsd %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vminsd %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x5d,0xc9]
+vminsd %xmm1, %xmm1, %xmm1
+
+// CHECK: vminsd %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0x5d,0xc9]
+vminsd %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vminsd %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x8a,0x5d,0xc9]
+vminsd %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vminss 256(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0xc5,0x02,0x5d,0xbc,0x82,0x00,0x01,0x00,0x00]
+vminss 256(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vminss -256(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0xc5,0x02,0x5d,0xbc,0x82,0x00,0xff,0xff,0xff]
+vminss -256(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vminss 256(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x71,0x06,0x0a,0x5d,0x7c,0x82,0x40]
+vminss 256(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+
+// CHECK: vminss -256(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x71,0x06,0x0a,0x5d,0x7c,0x82,0xc0]
+vminss -256(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+
+// CHECK: vminss 256(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x71,0x06,0x8a,0x5d,0x7c,0x82,0x40]
+vminss 256(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vminss -256(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x71,0x06,0x8a,0x5d,0x7c,0x82,0xc0]
+vminss -256(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vminss 256(%rdx,%rax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x5d,0x8c,0x82,0x00,0x01,0x00,0x00]
+vminss 256(%rdx,%rax,4), %xmm1, %xmm1
+
+// CHECK: vminss -256(%rdx,%rax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x5d,0x8c,0x82,0x00,0xff,0xff,0xff]
+vminss -256(%rdx,%rax,4), %xmm1, %xmm1
+
+// CHECK: vminss 256(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0x5d,0x4c,0x82,0x40]
+vminss 256(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vminss -256(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0x5d,0x4c,0x82,0xc0]
+vminss -256(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vminss 256(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x8a,0x5d,0x4c,0x82,0x40]
+vminss 256(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vminss -256(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x8a,0x5d,0x4c,0x82,0xc0]
+vminss -256(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vminss 256(%rdx,%rax), %xmm15, %xmm15
+// CHECK: encoding: [0xc5,0x02,0x5d,0xbc,0x02,0x00,0x01,0x00,0x00]
+vminss 256(%rdx,%rax), %xmm15, %xmm15
+
+// CHECK: vminss 256(%rdx,%rax), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x71,0x06,0x0a,0x5d,0x7c,0x02,0x40]
+vminss 256(%rdx,%rax), %xmm15, %xmm15 {%k2}
+
+// CHECK: vminss 256(%rdx,%rax), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x71,0x06,0x8a,0x5d,0x7c,0x02,0x40]
+vminss 256(%rdx,%rax), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vminss 256(%rdx,%rax), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x5d,0x8c,0x02,0x00,0x01,0x00,0x00]
+vminss 256(%rdx,%rax), %xmm1, %xmm1
+
+// CHECK: vminss 256(%rdx,%rax), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0x5d,0x4c,0x02,0x40]
+vminss 256(%rdx,%rax), %xmm1, %xmm1 {%k2}
+
+// CHECK: vminss 256(%rdx,%rax), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x8a,0x5d,0x4c,0x02,0x40]
+vminss 256(%rdx,%rax), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vminss 256(%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0xc5,0x02,0x5d,0xba,0x00,0x01,0x00,0x00]
+vminss 256(%rdx), %xmm15, %xmm15
+
+// CHECK: vminss 256(%rdx), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x71,0x06,0x0a,0x5d,0x7a,0x40]
+vminss 256(%rdx), %xmm15, %xmm15 {%k2}
+
+// CHECK: vminss 256(%rdx), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x71,0x06,0x8a,0x5d,0x7a,0x40]
+vminss 256(%rdx), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vminss 256(%rdx), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x5d,0x8a,0x00,0x01,0x00,0x00]
+vminss 256(%rdx), %xmm1, %xmm1
+
+// CHECK: vminss 256(%rdx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0x5d,0x4a,0x40]
+vminss 256(%rdx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vminss 256(%rdx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x8a,0x5d,0x4a,0x40]
+vminss 256(%rdx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vminss 485498096, %xmm15, %xmm15
+// CHECK: encoding: [0xc5,0x02,0x5d,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vminss 485498096, %xmm15, %xmm15
+
+// CHECK: vminss 485498096, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x71,0x06,0x0a,0x5d,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vminss 485498096, %xmm15, %xmm15 {%k2}
+
+// CHECK: vminss 485498096, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x71,0x06,0x8a,0x5d,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vminss 485498096, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vminss 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x5d,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vminss 485498096, %xmm1, %xmm1
+
+// CHECK: vminss 485498096, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0x5d,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vminss 485498096, %xmm1, %xmm1 {%k2}
+
+// CHECK: vminss 485498096, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x8a,0x5d,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vminss 485498096, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vminss (%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0xc5,0x02,0x5d,0x3a]
+vminss (%rdx), %xmm15, %xmm15
+
+// CHECK: vminss (%rdx), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x71,0x06,0x0a,0x5d,0x3a]
+vminss (%rdx), %xmm15, %xmm15 {%k2}
+
+// CHECK: vminss (%rdx), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x71,0x06,0x8a,0x5d,0x3a]
+vminss (%rdx), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vminss (%rdx), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x5d,0x0a]
+vminss (%rdx), %xmm1, %xmm1
+
+// CHECK: vminss (%rdx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0x5d,0x0a]
+vminss (%rdx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vminss (%rdx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x8a,0x5d,0x0a]
+vminss (%rdx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vminss {sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x51,0x06,0x18,0x5d,0xff]
+vminss {sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vminss {sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x51,0x06,0x1a,0x5d,0xff]
+vminss {sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vminss {sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x51,0x06,0x9a,0x5d,0xff]
+vminss {sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vminss {sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0x76,0x18,0x5d,0xc9]
+vminss {sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vminss {sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x1a,0x5d,0xc9]
+vminss {sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vminss {sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x9a,0x5d,0xc9]
+vminss {sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vminss %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x41,0x02,0x5d,0xff]
+vminss %xmm15, %xmm15, %xmm15
+
+// CHECK: vminss %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x51,0x06,0x0a,0x5d,0xff]
+vminss %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vminss %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x51,0x06,0x8a,0x5d,0xff]
+vminss %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vminss %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x5d,0xc9]
+vminss %xmm1, %xmm1, %xmm1
+
+// CHECK: vminss %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0x5d,0xc9]
+vminss %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vminss %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x8a,0x5d,0xc9]
+vminss %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vmovsd 485498096, %xmm15
+// CHECK: encoding: [0xc5,0x7b,0x10,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vmovsd 485498096, %xmm15
+
+// CHECK: vmovsd 485498096, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x71,0xff,0x0a,0x10,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vmovsd 485498096, %xmm15 {%k2}
+
+// CHECK: vmovsd 485498096, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x71,0xff,0x8a,0x10,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vmovsd 485498096, %xmm15 {%k2} {z}
+
+// CHECK: vmovsd 485498096, %xmm1
+// CHECK: encoding: [0xc5,0xfb,0x10,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vmovsd 485498096, %xmm1
+
+// CHECK: vmovsd 485498096, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xff,0x0a,0x10,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vmovsd 485498096, %xmm1 {%k2}
+
+// CHECK: vmovsd 485498096, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xff,0x8a,0x10,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vmovsd 485498096, %xmm1 {%k2} {z}
+
+// CHECK: vmovsd 512(%rdx,%rax,4), %xmm15
+// CHECK: encoding: [0xc5,0x7b,0x10,0xbc,0x82,0x00,0x02,0x00,0x00]
+vmovsd 512(%rdx,%rax,4), %xmm15
+
+// CHECK: vmovsd -512(%rdx,%rax,4), %xmm15
+// CHECK: encoding: [0xc5,0x7b,0x10,0xbc,0x82,0x00,0xfe,0xff,0xff]
+vmovsd -512(%rdx,%rax,4), %xmm15
+
+// CHECK: vmovsd 512(%rdx,%rax,4), %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x71,0xff,0x0a,0x10,0x7c,0x82,0x40]
+vmovsd 512(%rdx,%rax,4), %xmm15 {%k2}
+
+// CHECK: vmovsd -512(%rdx,%rax,4), %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x71,0xff,0x0a,0x10,0x7c,0x82,0xc0]
+vmovsd -512(%rdx,%rax,4), %xmm15 {%k2}
+
+// CHECK: vmovsd 512(%rdx,%rax,4), %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x71,0xff,0x8a,0x10,0x7c,0x82,0x40]
+vmovsd 512(%rdx,%rax,4), %xmm15 {%k2} {z}
+
+// CHECK: vmovsd -512(%rdx,%rax,4), %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x71,0xff,0x8a,0x10,0x7c,0x82,0xc0]
+vmovsd -512(%rdx,%rax,4), %xmm15 {%k2} {z}
+
+// CHECK: vmovsd 512(%rdx,%rax,4), %xmm1
+// CHECK: encoding: [0xc5,0xfb,0x10,0x8c,0x82,0x00,0x02,0x00,0x00]
+vmovsd 512(%rdx,%rax,4), %xmm1
+
+// CHECK: vmovsd -512(%rdx,%rax,4), %xmm1
+// CHECK: encoding: [0xc5,0xfb,0x10,0x8c,0x82,0x00,0xfe,0xff,0xff]
+vmovsd -512(%rdx,%rax,4), %xmm1
+
+// CHECK: vmovsd 512(%rdx,%rax,4), %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xff,0x0a,0x10,0x4c,0x82,0x40]
+vmovsd 512(%rdx,%rax,4), %xmm1 {%k2}
+
+// CHECK: vmovsd -512(%rdx,%rax,4), %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xff,0x0a,0x10,0x4c,0x82,0xc0]
+vmovsd -512(%rdx,%rax,4), %xmm1 {%k2}
+
+// CHECK: vmovsd 512(%rdx,%rax,4), %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xff,0x8a,0x10,0x4c,0x82,0x40]
+vmovsd 512(%rdx,%rax,4), %xmm1 {%k2} {z}
+
+// CHECK: vmovsd -512(%rdx,%rax,4), %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xff,0x8a,0x10,0x4c,0x82,0xc0]
+vmovsd -512(%rdx,%rax,4), %xmm1 {%k2} {z}
+
+// CHECK: vmovsd 512(%rdx,%rax), %xmm15
+// CHECK: encoding: [0xc5,0x7b,0x10,0xbc,0x02,0x00,0x02,0x00,0x00]
+vmovsd 512(%rdx,%rax), %xmm15
+
+// CHECK: vmovsd 512(%rdx,%rax), %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x71,0xff,0x0a,0x10,0x7c,0x02,0x40]
+vmovsd 512(%rdx,%rax), %xmm15 {%k2}
+
+// CHECK: vmovsd 512(%rdx,%rax), %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x71,0xff,0x8a,0x10,0x7c,0x02,0x40]
+vmovsd 512(%rdx,%rax), %xmm15 {%k2} {z}
+
+// CHECK: vmovsd 512(%rdx,%rax), %xmm1
+// CHECK: encoding: [0xc5,0xfb,0x10,0x8c,0x02,0x00,0x02,0x00,0x00]
+vmovsd 512(%rdx,%rax), %xmm1
+
+// CHECK: vmovsd 512(%rdx,%rax), %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xff,0x0a,0x10,0x4c,0x02,0x40]
+vmovsd 512(%rdx,%rax), %xmm1 {%k2}
+
+// CHECK: vmovsd 512(%rdx,%rax), %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xff,0x8a,0x10,0x4c,0x02,0x40]
+vmovsd 512(%rdx,%rax), %xmm1 {%k2} {z}
+
+// CHECK: vmovsd 512(%rdx), %xmm15
+// CHECK: encoding: [0xc5,0x7b,0x10,0xba,0x00,0x02,0x00,0x00]
+vmovsd 512(%rdx), %xmm15
+
+// CHECK: vmovsd 512(%rdx), %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x71,0xff,0x0a,0x10,0x7a,0x40]
+vmovsd 512(%rdx), %xmm15 {%k2}
+
+// CHECK: vmovsd 512(%rdx), %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x71,0xff,0x8a,0x10,0x7a,0x40]
+vmovsd 512(%rdx), %xmm15 {%k2} {z}
+
+// CHECK: vmovsd 512(%rdx), %xmm1
+// CHECK: encoding: [0xc5,0xfb,0x10,0x8a,0x00,0x02,0x00,0x00]
+vmovsd 512(%rdx), %xmm1
+
+// CHECK: vmovsd 512(%rdx), %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xff,0x0a,0x10,0x4a,0x40]
+vmovsd 512(%rdx), %xmm1 {%k2}
+
+// CHECK: vmovsd 512(%rdx), %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xff,0x8a,0x10,0x4a,0x40]
+vmovsd 512(%rdx), %xmm1 {%k2} {z}
+
+// CHECK: vmovsd (%rdx), %xmm15
+// CHECK: encoding: [0xc5,0x7b,0x10,0x3a]
+vmovsd (%rdx), %xmm15
+
+// CHECK: vmovsd (%rdx), %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x71,0xff,0x0a,0x10,0x3a]
+vmovsd (%rdx), %xmm15 {%k2}
+
+// CHECK: vmovsd (%rdx), %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x71,0xff,0x8a,0x10,0x3a]
+vmovsd (%rdx), %xmm15 {%k2} {z}
+
+// CHECK: vmovsd (%rdx), %xmm1
+// CHECK: encoding: [0xc5,0xfb,0x10,0x0a]
+vmovsd (%rdx), %xmm1
+
+// CHECK: vmovsd (%rdx), %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xff,0x0a,0x10,0x0a]
+vmovsd (%rdx), %xmm1 {%k2}
+
+// CHECK: vmovsd (%rdx), %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xff,0x8a,0x10,0x0a]
+vmovsd (%rdx), %xmm1 {%k2} {z}
+
+// CHECK: vmovsd %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x41,0x03,0x11,0xff]
+vmovsd.s %xmm15, %xmm15, %xmm15
+
+// CHECK: vmovsd %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x51,0x87,0x0a,0x11,0xff]
+vmovsd.s %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vmovsd %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x51,0x87,0x8a,0x11,0xff]
+vmovsd.s %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vmovsd %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x11,0xc9]
+vmovsd.s %xmm1, %xmm1, %xmm1
+
+// CHECK: vmovsd %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0x11,0xc9]
+vmovsd.s %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vmovsd %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x8a,0x11,0xc9]
+vmovsd.s %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vmovsd %xmm1, 485498096
+// CHECK: encoding: [0xc5,0xfb,0x11,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vmovsd %xmm1, 485498096
+
+// CHECK: vmovsd %xmm1, 485498096 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xff,0x0a,0x11,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vmovsd %xmm1, 485498096 {%k2}
+
+// CHECK: vmovsd %xmm1, 512(%rdx)
+// CHECK: encoding: [0xc5,0xfb,0x11,0x8a,0x00,0x02,0x00,0x00]
+vmovsd %xmm1, 512(%rdx)
+
+// CHECK: vmovsd %xmm1, 512(%rdx) {%k2}
+// CHECK: encoding: [0x62,0xf1,0xff,0x0a,0x11,0x4a,0x40]
+vmovsd %xmm1, 512(%rdx) {%k2}
+
+// CHECK: vmovsd %xmm1, 512(%rdx,%rax,4)
+// CHECK: encoding: [0xc5,0xfb,0x11,0x8c,0x82,0x00,0x02,0x00,0x00]
+vmovsd %xmm1, 512(%rdx,%rax,4)
+
+// CHECK: vmovsd %xmm1, -512(%rdx,%rax,4)
+// CHECK: encoding: [0xc5,0xfb,0x11,0x8c,0x82,0x00,0xfe,0xff,0xff]
+vmovsd %xmm1, -512(%rdx,%rax,4)
+
+// CHECK: vmovsd %xmm1, 512(%rdx,%rax,4) {%k2}
+// CHECK: encoding: [0x62,0xf1,0xff,0x0a,0x11,0x4c,0x82,0x40]
+vmovsd %xmm1, 512(%rdx,%rax,4) {%k2}
+
+// CHECK: vmovsd %xmm1, -512(%rdx,%rax,4) {%k2}
+// CHECK: encoding: [0x62,0xf1,0xff,0x0a,0x11,0x4c,0x82,0xc0]
+vmovsd %xmm1, -512(%rdx,%rax,4) {%k2}
+
+// CHECK: vmovsd %xmm1, 512(%rdx,%rax)
+// CHECK: encoding: [0xc5,0xfb,0x11,0x8c,0x02,0x00,0x02,0x00,0x00]
+vmovsd %xmm1, 512(%rdx,%rax)
+
+// CHECK: vmovsd %xmm1, 512(%rdx,%rax) {%k2}
+// CHECK: encoding: [0x62,0xf1,0xff,0x0a,0x11,0x4c,0x02,0x40]
+vmovsd %xmm1, 512(%rdx,%rax) {%k2}
+
+// CHECK: vmovsd %xmm15, 485498096
+// CHECK: encoding: [0xc5,0x7b,0x11,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vmovsd %xmm15, 485498096
+
+// CHECK: vmovsd %xmm15, 485498096 {%k2}
+// CHECK: encoding: [0x62,0x71,0xff,0x0a,0x11,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vmovsd %xmm15, 485498096 {%k2}
+
+// CHECK: vmovsd %xmm15, 512(%rdx)
+// CHECK: encoding: [0xc5,0x7b,0x11,0xba,0x00,0x02,0x00,0x00]
+vmovsd %xmm15, 512(%rdx)
+
+// CHECK: vmovsd %xmm15, 512(%rdx) {%k2}
+// CHECK: encoding: [0x62,0x71,0xff,0x0a,0x11,0x7a,0x40]
+vmovsd %xmm15, 512(%rdx) {%k2}
+
+// CHECK: vmovsd %xmm15, 512(%rdx,%rax,4)
+// CHECK: encoding: [0xc5,0x7b,0x11,0xbc,0x82,0x00,0x02,0x00,0x00]
+vmovsd %xmm15, 512(%rdx,%rax,4)
+
+// CHECK: vmovsd %xmm15, -512(%rdx,%rax,4)
+// CHECK: encoding: [0xc5,0x7b,0x11,0xbc,0x82,0x00,0xfe,0xff,0xff]
+vmovsd %xmm15, -512(%rdx,%rax,4)
+
+// CHECK: vmovsd %xmm15, 512(%rdx,%rax,4) {%k2}
+// CHECK: encoding: [0x62,0x71,0xff,0x0a,0x11,0x7c,0x82,0x40]
+vmovsd %xmm15, 512(%rdx,%rax,4) {%k2}
+
+// CHECK: vmovsd %xmm15, -512(%rdx,%rax,4) {%k2}
+// CHECK: encoding: [0x62,0x71,0xff,0x0a,0x11,0x7c,0x82,0xc0]
+vmovsd %xmm15, -512(%rdx,%rax,4) {%k2}
+
+// CHECK: vmovsd %xmm15, 512(%rdx,%rax)
+// CHECK: encoding: [0xc5,0x7b,0x11,0xbc,0x02,0x00,0x02,0x00,0x00]
+vmovsd %xmm15, 512(%rdx,%rax)
+
+// CHECK: vmovsd %xmm15, 512(%rdx,%rax) {%k2}
+// CHECK: encoding: [0x62,0x71,0xff,0x0a,0x11,0x7c,0x02,0x40]
+vmovsd %xmm15, 512(%rdx,%rax) {%k2}
+
+// CHECK: vmovsd %xmm15, (%rdx)
+// CHECK: encoding: [0xc5,0x7b,0x11,0x3a]
+vmovsd %xmm15, (%rdx)
+
+// CHECK: vmovsd %xmm15, (%rdx) {%k2}
+// CHECK: encoding: [0x62,0x71,0xff,0x0a,0x11,0x3a]
+vmovsd %xmm15, (%rdx) {%k2}
+
+// CHECK: vmovsd %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x41,0x03,0x10,0xff]
+vmovsd %xmm15, %xmm15, %xmm15
+
+// CHECK: vmovsd %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x51,0x87,0x0a,0x10,0xff]
+vmovsd %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vmovsd %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x51,0x87,0x8a,0x10,0xff]
+vmovsd %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vmovsd %xmm1, (%rdx)
+// CHECK: encoding: [0xc5,0xfb,0x11,0x0a]
+vmovsd %xmm1, (%rdx)
+
+// CHECK: vmovsd %xmm1, (%rdx) {%k2}
+// CHECK: encoding: [0x62,0xf1,0xff,0x0a,0x11,0x0a]
+vmovsd %xmm1, (%rdx) {%k2}
+
+// CHECK: vmovsd %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x10,0xc9]
+vmovsd %xmm1, %xmm1, %xmm1
+
+// CHECK: vmovsd %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0x10,0xc9]
+vmovsd %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vmovsd %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x8a,0x10,0xc9]
+vmovsd %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vmovss 256(%rdx,%rax,4), %xmm15
+// CHECK: encoding: [0xc5,0x7a,0x10,0xbc,0x82,0x00,0x01,0x00,0x00]
+vmovss 256(%rdx,%rax,4), %xmm15
+
+// CHECK: vmovss -256(%rdx,%rax,4), %xmm15
+// CHECK: encoding: [0xc5,0x7a,0x10,0xbc,0x82,0x00,0xff,0xff,0xff]
+vmovss -256(%rdx,%rax,4), %xmm15
+
+// CHECK: vmovss 256(%rdx,%rax,4), %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x71,0x7e,0x0a,0x10,0x7c,0x82,0x40]
+vmovss 256(%rdx,%rax,4), %xmm15 {%k2}
+
+// CHECK: vmovss -256(%rdx,%rax,4), %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x71,0x7e,0x0a,0x10,0x7c,0x82,0xc0]
+vmovss -256(%rdx,%rax,4), %xmm15 {%k2}
+
+// CHECK: vmovss 256(%rdx,%rax,4), %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x71,0x7e,0x8a,0x10,0x7c,0x82,0x40]
+vmovss 256(%rdx,%rax,4), %xmm15 {%k2} {z}
+
+// CHECK: vmovss -256(%rdx,%rax,4), %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x71,0x7e,0x8a,0x10,0x7c,0x82,0xc0]
+vmovss -256(%rdx,%rax,4), %xmm15 {%k2} {z}
+
+// CHECK: vmovss 256(%rdx,%rax,4), %xmm1
+// CHECK: encoding: [0xc5,0xfa,0x10,0x8c,0x82,0x00,0x01,0x00,0x00]
+vmovss 256(%rdx,%rax,4), %xmm1
+
+// CHECK: vmovss -256(%rdx,%rax,4), %xmm1
+// CHECK: encoding: [0xc5,0xfa,0x10,0x8c,0x82,0x00,0xff,0xff,0xff]
+vmovss -256(%rdx,%rax,4), %xmm1
+
+// CHECK: vmovss 256(%rdx,%rax,4), %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x7e,0x0a,0x10,0x4c,0x82,0x40]
+vmovss 256(%rdx,%rax,4), %xmm1 {%k2}
+
+// CHECK: vmovss -256(%rdx,%rax,4), %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x7e,0x0a,0x10,0x4c,0x82,0xc0]
+vmovss -256(%rdx,%rax,4), %xmm1 {%k2}
+
+// CHECK: vmovss 256(%rdx,%rax,4), %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x7e,0x8a,0x10,0x4c,0x82,0x40]
+vmovss 256(%rdx,%rax,4), %xmm1 {%k2} {z}
+
+// CHECK: vmovss -256(%rdx,%rax,4), %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x7e,0x8a,0x10,0x4c,0x82,0xc0]
+vmovss -256(%rdx,%rax,4), %xmm1 {%k2} {z}
+
+// CHECK: vmovss 256(%rdx,%rax), %xmm15
+// CHECK: encoding: [0xc5,0x7a,0x10,0xbc,0x02,0x00,0x01,0x00,0x00]
+vmovss 256(%rdx,%rax), %xmm15
+
+// CHECK: vmovss 256(%rdx,%rax), %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x71,0x7e,0x0a,0x10,0x7c,0x02,0x40]
+vmovss 256(%rdx,%rax), %xmm15 {%k2}
+
+// CHECK: vmovss 256(%rdx,%rax), %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x71,0x7e,0x8a,0x10,0x7c,0x02,0x40]
+vmovss 256(%rdx,%rax), %xmm15 {%k2} {z}
+
+// CHECK: vmovss 256(%rdx,%rax), %xmm1
+// CHECK: encoding: [0xc5,0xfa,0x10,0x8c,0x02,0x00,0x01,0x00,0x00]
+vmovss 256(%rdx,%rax), %xmm1
+
+// CHECK: vmovss 256(%rdx,%rax), %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x7e,0x0a,0x10,0x4c,0x02,0x40]
+vmovss 256(%rdx,%rax), %xmm1 {%k2}
+
+// CHECK: vmovss 256(%rdx,%rax), %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x7e,0x8a,0x10,0x4c,0x02,0x40]
+vmovss 256(%rdx,%rax), %xmm1 {%k2} {z}
+
+// CHECK: vmovss 256(%rdx), %xmm15
+// CHECK: encoding: [0xc5,0x7a,0x10,0xba,0x00,0x01,0x00,0x00]
+vmovss 256(%rdx), %xmm15
+
+// CHECK: vmovss 256(%rdx), %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x71,0x7e,0x0a,0x10,0x7a,0x40]
+vmovss 256(%rdx), %xmm15 {%k2}
+
+// CHECK: vmovss 256(%rdx), %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x71,0x7e,0x8a,0x10,0x7a,0x40]
+vmovss 256(%rdx), %xmm15 {%k2} {z}
+
+// CHECK: vmovss 256(%rdx), %xmm1
+// CHECK: encoding: [0xc5,0xfa,0x10,0x8a,0x00,0x01,0x00,0x00]
+vmovss 256(%rdx), %xmm1
+
+// CHECK: vmovss 256(%rdx), %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x7e,0x0a,0x10,0x4a,0x40]
+vmovss 256(%rdx), %xmm1 {%k2}
+
+// CHECK: vmovss 256(%rdx), %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x7e,0x8a,0x10,0x4a,0x40]
+vmovss 256(%rdx), %xmm1 {%k2} {z}
+
+// CHECK: vmovss 485498096, %xmm15
+// CHECK: encoding: [0xc5,0x7a,0x10,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vmovss 485498096, %xmm15
+
+// CHECK: vmovss 485498096, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x71,0x7e,0x0a,0x10,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vmovss 485498096, %xmm15 {%k2}
+
+// CHECK: vmovss 485498096, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x71,0x7e,0x8a,0x10,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vmovss 485498096, %xmm15 {%k2} {z}
+
+// CHECK: vmovss 485498096, %xmm1
+// CHECK: encoding: [0xc5,0xfa,0x10,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vmovss 485498096, %xmm1
+
+// CHECK: vmovss 485498096, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x7e,0x0a,0x10,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vmovss 485498096, %xmm1 {%k2}
+
+// CHECK: vmovss 485498096, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x7e,0x8a,0x10,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vmovss 485498096, %xmm1 {%k2} {z}
+
+// CHECK: vmovss (%rdx), %xmm15
+// CHECK: encoding: [0xc5,0x7a,0x10,0x3a]
+vmovss (%rdx), %xmm15
+
+// CHECK: vmovss (%rdx), %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x71,0x7e,0x0a,0x10,0x3a]
+vmovss (%rdx), %xmm15 {%k2}
+
+// CHECK: vmovss (%rdx), %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x71,0x7e,0x8a,0x10,0x3a]
+vmovss (%rdx), %xmm15 {%k2} {z}
+
+// CHECK: vmovss (%rdx), %xmm1
+// CHECK: encoding: [0xc5,0xfa,0x10,0x0a]
+vmovss (%rdx), %xmm1
+
+// CHECK: vmovss (%rdx), %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x7e,0x0a,0x10,0x0a]
+vmovss (%rdx), %xmm1 {%k2}
+
+// CHECK: vmovss (%rdx), %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x7e,0x8a,0x10,0x0a]
+vmovss (%rdx), %xmm1 {%k2} {z}
+
+// CHECK: vmovss %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x41,0x02,0x11,0xff]
+vmovss.s %xmm15, %xmm15, %xmm15
+
+// CHECK: vmovss %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x51,0x06,0x0a,0x11,0xff]
+vmovss.s %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vmovss %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x51,0x06,0x8a,0x11,0xff]
+vmovss.s %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vmovss %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x11,0xc9]
+vmovss.s %xmm1, %xmm1, %xmm1
+
+// CHECK: vmovss %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0x11,0xc9]
+vmovss.s %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vmovss %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x8a,0x11,0xc9]
+vmovss.s %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vmovss %xmm1, 256(%rdx)
+// CHECK: encoding: [0xc5,0xfa,0x11,0x8a,0x00,0x01,0x00,0x00]
+vmovss %xmm1, 256(%rdx)
+
+// CHECK: vmovss %xmm1, 256(%rdx) {%k2}
+// CHECK: encoding: [0x62,0xf1,0x7e,0x0a,0x11,0x4a,0x40]
+vmovss %xmm1, 256(%rdx) {%k2}
+
+// CHECK: vmovss %xmm1, 256(%rdx,%rax,4)
+// CHECK: encoding: [0xc5,0xfa,0x11,0x8c,0x82,0x00,0x01,0x00,0x00]
+vmovss %xmm1, 256(%rdx,%rax,4)
+
+// CHECK: vmovss %xmm1, -256(%rdx,%rax,4)
+// CHECK: encoding: [0xc5,0xfa,0x11,0x8c,0x82,0x00,0xff,0xff,0xff]
+vmovss %xmm1, -256(%rdx,%rax,4)
+
+// CHECK: vmovss %xmm1, 256(%rdx,%rax,4) {%k2}
+// CHECK: encoding: [0x62,0xf1,0x7e,0x0a,0x11,0x4c,0x82,0x40]
+vmovss %xmm1, 256(%rdx,%rax,4) {%k2}
+
+// CHECK: vmovss %xmm1, -256(%rdx,%rax,4) {%k2}
+// CHECK: encoding: [0x62,0xf1,0x7e,0x0a,0x11,0x4c,0x82,0xc0]
+vmovss %xmm1, -256(%rdx,%rax,4) {%k2}
+
+// CHECK: vmovss %xmm1, 256(%rdx,%rax)
+// CHECK: encoding: [0xc5,0xfa,0x11,0x8c,0x02,0x00,0x01,0x00,0x00]
+vmovss %xmm1, 256(%rdx,%rax)
+
+// CHECK: vmovss %xmm1, 256(%rdx,%rax) {%k2}
+// CHECK: encoding: [0x62,0xf1,0x7e,0x0a,0x11,0x4c,0x02,0x40]
+vmovss %xmm1, 256(%rdx,%rax) {%k2}
+
+// CHECK: vmovss %xmm1, 485498096
+// CHECK: encoding: [0xc5,0xfa,0x11,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vmovss %xmm1, 485498096
+
+// CHECK: vmovss %xmm1, 485498096 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x7e,0x0a,0x11,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vmovss %xmm1, 485498096 {%k2}
+
+// CHECK: vmovss %xmm15, 256(%rdx)
+// CHECK: encoding: [0xc5,0x7a,0x11,0xba,0x00,0x01,0x00,0x00]
+vmovss %xmm15, 256(%rdx)
+
+// CHECK: vmovss %xmm15, 256(%rdx) {%k2}
+// CHECK: encoding: [0x62,0x71,0x7e,0x0a,0x11,0x7a,0x40]
+vmovss %xmm15, 256(%rdx) {%k2}
+
+// CHECK: vmovss %xmm15, 256(%rdx,%rax,4)
+// CHECK: encoding: [0xc5,0x7a,0x11,0xbc,0x82,0x00,0x01,0x00,0x00]
+vmovss %xmm15, 256(%rdx,%rax,4)
+
+// CHECK: vmovss %xmm15, -256(%rdx,%rax,4)
+// CHECK: encoding: [0xc5,0x7a,0x11,0xbc,0x82,0x00,0xff,0xff,0xff]
+vmovss %xmm15, -256(%rdx,%rax,4)
+
+// CHECK: vmovss %xmm15, 256(%rdx,%rax,4) {%k2}
+// CHECK: encoding: [0x62,0x71,0x7e,0x0a,0x11,0x7c,0x82,0x40]
+vmovss %xmm15, 256(%rdx,%rax,4) {%k2}
+
+// CHECK: vmovss %xmm15, -256(%rdx,%rax,4) {%k2}
+// CHECK: encoding: [0x62,0x71,0x7e,0x0a,0x11,0x7c,0x82,0xc0]
+vmovss %xmm15, -256(%rdx,%rax,4) {%k2}
+
+// CHECK: vmovss %xmm15, 256(%rdx,%rax)
+// CHECK: encoding: [0xc5,0x7a,0x11,0xbc,0x02,0x00,0x01,0x00,0x00]
+vmovss %xmm15, 256(%rdx,%rax)
+
+// CHECK: vmovss %xmm15, 256(%rdx,%rax) {%k2}
+// CHECK: encoding: [0x62,0x71,0x7e,0x0a,0x11,0x7c,0x02,0x40]
+vmovss %xmm15, 256(%rdx,%rax) {%k2}
+
+// CHECK: vmovss %xmm15, 485498096
+// CHECK: encoding: [0xc5,0x7a,0x11,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vmovss %xmm15, 485498096
+
+// CHECK: vmovss %xmm15, 485498096 {%k2}
+// CHECK: encoding: [0x62,0x71,0x7e,0x0a,0x11,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vmovss %xmm15, 485498096 {%k2}
+
+// CHECK: vmovss %xmm15, (%rdx)
+// CHECK: encoding: [0xc5,0x7a,0x11,0x3a]
+vmovss %xmm15, (%rdx)
+
+// CHECK: vmovss %xmm15, (%rdx) {%k2}
+// CHECK: encoding: [0x62,0x71,0x7e,0x0a,0x11,0x3a]
+vmovss %xmm15, (%rdx) {%k2}
+
+// CHECK: vmovss %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x41,0x02,0x10,0xff]
+vmovss %xmm15, %xmm15, %xmm15
+
+// CHECK: vmovss %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x51,0x06,0x0a,0x10,0xff]
+vmovss %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vmovss %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x51,0x06,0x8a,0x10,0xff]
+vmovss %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vmovss %xmm1, (%rdx)
+// CHECK: encoding: [0xc5,0xfa,0x11,0x0a]
+vmovss %xmm1, (%rdx)
+
+// CHECK: vmovss %xmm1, (%rdx) {%k2}
+// CHECK: encoding: [0x62,0xf1,0x7e,0x0a,0x11,0x0a]
+vmovss %xmm1, (%rdx) {%k2}
+
+// CHECK: vmovss %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x10,0xc9]
+vmovss %xmm1, %xmm1, %xmm1
+
+// CHECK: vmovss %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0x10,0xc9]
+vmovss %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vmovss %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x8a,0x10,0xc9]
+vmovss %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vmulsd 485498096, %xmm15, %xmm15
+// CHECK: encoding: [0xc5,0x03,0x59,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vmulsd 485498096, %xmm15, %xmm15
+
+// CHECK: vmulsd 485498096, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x71,0x87,0x0a,0x59,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vmulsd 485498096, %xmm15, %xmm15 {%k2}
+
+// CHECK: vmulsd 485498096, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x71,0x87,0x8a,0x59,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vmulsd 485498096, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vmulsd 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x59,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vmulsd 485498096, %xmm1, %xmm1
+
+// CHECK: vmulsd 485498096, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0x59,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vmulsd 485498096, %xmm1, %xmm1 {%k2}
+
+// CHECK: vmulsd 485498096, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x8a,0x59,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vmulsd 485498096, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vmulsd 512(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0xc5,0x03,0x59,0xbc,0x82,0x00,0x02,0x00,0x00]
+vmulsd 512(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vmulsd -512(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0xc5,0x03,0x59,0xbc,0x82,0x00,0xfe,0xff,0xff]
+vmulsd -512(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vmulsd 512(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x71,0x87,0x0a,0x59,0x7c,0x82,0x40]
+vmulsd 512(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+
+// CHECK: vmulsd -512(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x71,0x87,0x0a,0x59,0x7c,0x82,0xc0]
+vmulsd -512(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+
+// CHECK: vmulsd 512(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x71,0x87,0x8a,0x59,0x7c,0x82,0x40]
+vmulsd 512(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vmulsd -512(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x71,0x87,0x8a,0x59,0x7c,0x82,0xc0]
+vmulsd -512(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vmulsd 512(%rdx,%rax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x59,0x8c,0x82,0x00,0x02,0x00,0x00]
+vmulsd 512(%rdx,%rax,4), %xmm1, %xmm1
+
+// CHECK: vmulsd -512(%rdx,%rax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x59,0x8c,0x82,0x00,0xfe,0xff,0xff]
+vmulsd -512(%rdx,%rax,4), %xmm1, %xmm1
+
+// CHECK: vmulsd 512(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0x59,0x4c,0x82,0x40]
+vmulsd 512(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vmulsd -512(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0x59,0x4c,0x82,0xc0]
+vmulsd -512(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vmulsd 512(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x8a,0x59,0x4c,0x82,0x40]
+vmulsd 512(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vmulsd -512(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x8a,0x59,0x4c,0x82,0xc0]
+vmulsd -512(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vmulsd 512(%rdx,%rax), %xmm15, %xmm15
+// CHECK: encoding: [0xc5,0x03,0x59,0xbc,0x02,0x00,0x02,0x00,0x00]
+vmulsd 512(%rdx,%rax), %xmm15, %xmm15
+
+// CHECK: vmulsd 512(%rdx,%rax), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x71,0x87,0x0a,0x59,0x7c,0x02,0x40]
+vmulsd 512(%rdx,%rax), %xmm15, %xmm15 {%k2}
+
+// CHECK: vmulsd 512(%rdx,%rax), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x71,0x87,0x8a,0x59,0x7c,0x02,0x40]
+vmulsd 512(%rdx,%rax), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vmulsd 512(%rdx,%rax), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x59,0x8c,0x02,0x00,0x02,0x00,0x00]
+vmulsd 512(%rdx,%rax), %xmm1, %xmm1
+
+// CHECK: vmulsd 512(%rdx,%rax), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0x59,0x4c,0x02,0x40]
+vmulsd 512(%rdx,%rax), %xmm1, %xmm1 {%k2}
+
+// CHECK: vmulsd 512(%rdx,%rax), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x8a,0x59,0x4c,0x02,0x40]
+vmulsd 512(%rdx,%rax), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vmulsd 512(%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0xc5,0x03,0x59,0xba,0x00,0x02,0x00,0x00]
+vmulsd 512(%rdx), %xmm15, %xmm15
+
+// CHECK: vmulsd 512(%rdx), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x71,0x87,0x0a,0x59,0x7a,0x40]
+vmulsd 512(%rdx), %xmm15, %xmm15 {%k2}
+
+// CHECK: vmulsd 512(%rdx), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x71,0x87,0x8a,0x59,0x7a,0x40]
+vmulsd 512(%rdx), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vmulsd 512(%rdx), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x59,0x8a,0x00,0x02,0x00,0x00]
+vmulsd 512(%rdx), %xmm1, %xmm1
+
+// CHECK: vmulsd 512(%rdx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0x59,0x4a,0x40]
+vmulsd 512(%rdx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vmulsd 512(%rdx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x8a,0x59,0x4a,0x40]
+vmulsd 512(%rdx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vmulsd {rd-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x51,0x87,0x38,0x59,0xff]
+vmulsd {rd-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vmulsd {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x51,0x87,0x3a,0x59,0xff]
+vmulsd {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vmulsd {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x51,0x87,0xba,0x59,0xff]
+vmulsd {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vmulsd {rd-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0xf7,0x38,0x59,0xc9]
+vmulsd {rd-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vmulsd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x3a,0x59,0xc9]
+vmulsd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vmulsd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0xba,0x59,0xc9]
+vmulsd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vmulsd (%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0xc5,0x03,0x59,0x3a]
+vmulsd (%rdx), %xmm15, %xmm15
+
+// CHECK: vmulsd (%rdx), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x71,0x87,0x0a,0x59,0x3a]
+vmulsd (%rdx), %xmm15, %xmm15 {%k2}
+
+// CHECK: vmulsd (%rdx), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x71,0x87,0x8a,0x59,0x3a]
+vmulsd (%rdx), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vmulsd (%rdx), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x59,0x0a]
+vmulsd (%rdx), %xmm1, %xmm1
+
+// CHECK: vmulsd (%rdx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0x59,0x0a]
+vmulsd (%rdx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vmulsd (%rdx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x8a,0x59,0x0a]
+vmulsd (%rdx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vmulsd {rn-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x51,0x87,0x18,0x59,0xff]
+vmulsd {rn-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vmulsd {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x51,0x87,0x1a,0x59,0xff]
+vmulsd {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vmulsd {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x51,0x87,0x9a,0x59,0xff]
+vmulsd {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vmulsd {rn-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0xf7,0x18,0x59,0xc9]
+vmulsd {rn-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vmulsd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x1a,0x59,0xc9]
+vmulsd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vmulsd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x9a,0x59,0xc9]
+vmulsd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vmulsd {ru-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x51,0x87,0x58,0x59,0xff]
+vmulsd {ru-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vmulsd {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x51,0x87,0x5a,0x59,0xff]
+vmulsd {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vmulsd {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x51,0x87,0xda,0x59,0xff]
+vmulsd {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vmulsd {ru-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0xf7,0x58,0x59,0xc9]
+vmulsd {ru-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vmulsd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x5a,0x59,0xc9]
+vmulsd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vmulsd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0xda,0x59,0xc9]
+vmulsd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vmulsd {rz-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x51,0x87,0x78,0x59,0xff]
+vmulsd {rz-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vmulsd {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x51,0x87,0x7a,0x59,0xff]
+vmulsd {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vmulsd {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x51,0x87,0xfa,0x59,0xff]
+vmulsd {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vmulsd {rz-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0xf7,0x78,0x59,0xc9]
+vmulsd {rz-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vmulsd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x7a,0x59,0xc9]
+vmulsd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vmulsd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0xfa,0x59,0xc9]
+vmulsd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vmulsd %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x41,0x03,0x59,0xff]
+vmulsd %xmm15, %xmm15, %xmm15
+
+// CHECK: vmulsd %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x51,0x87,0x0a,0x59,0xff]
+vmulsd %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vmulsd %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x51,0x87,0x8a,0x59,0xff]
+vmulsd %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vmulsd %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x59,0xc9]
+vmulsd %xmm1, %xmm1, %xmm1
+
+// CHECK: vmulsd %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0x59,0xc9]
+vmulsd %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vmulsd %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x8a,0x59,0xc9]
+vmulsd %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vmulss 256(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0xc5,0x02,0x59,0xbc,0x82,0x00,0x01,0x00,0x00]
+vmulss 256(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vmulss -256(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0xc5,0x02,0x59,0xbc,0x82,0x00,0xff,0xff,0xff]
+vmulss -256(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vmulss 256(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x71,0x06,0x0a,0x59,0x7c,0x82,0x40]
+vmulss 256(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+
+// CHECK: vmulss -256(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x71,0x06,0x0a,0x59,0x7c,0x82,0xc0]
+vmulss -256(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+
+// CHECK: vmulss 256(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x71,0x06,0x8a,0x59,0x7c,0x82,0x40]
+vmulss 256(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vmulss -256(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x71,0x06,0x8a,0x59,0x7c,0x82,0xc0]
+vmulss -256(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vmulss 256(%rdx,%rax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x59,0x8c,0x82,0x00,0x01,0x00,0x00]
+vmulss 256(%rdx,%rax,4), %xmm1, %xmm1
+
+// CHECK: vmulss -256(%rdx,%rax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x59,0x8c,0x82,0x00,0xff,0xff,0xff]
+vmulss -256(%rdx,%rax,4), %xmm1, %xmm1
+
+// CHECK: vmulss 256(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0x59,0x4c,0x82,0x40]
+vmulss 256(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vmulss -256(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0x59,0x4c,0x82,0xc0]
+vmulss -256(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vmulss 256(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x8a,0x59,0x4c,0x82,0x40]
+vmulss 256(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vmulss -256(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x8a,0x59,0x4c,0x82,0xc0]
+vmulss -256(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vmulss 256(%rdx,%rax), %xmm15, %xmm15
+// CHECK: encoding: [0xc5,0x02,0x59,0xbc,0x02,0x00,0x01,0x00,0x00]
+vmulss 256(%rdx,%rax), %xmm15, %xmm15
+
+// CHECK: vmulss 256(%rdx,%rax), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x71,0x06,0x0a,0x59,0x7c,0x02,0x40]
+vmulss 256(%rdx,%rax), %xmm15, %xmm15 {%k2}
+
+// CHECK: vmulss 256(%rdx,%rax), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x71,0x06,0x8a,0x59,0x7c,0x02,0x40]
+vmulss 256(%rdx,%rax), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vmulss 256(%rdx,%rax), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x59,0x8c,0x02,0x00,0x01,0x00,0x00]
+vmulss 256(%rdx,%rax), %xmm1, %xmm1
+
+// CHECK: vmulss 256(%rdx,%rax), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0x59,0x4c,0x02,0x40]
+vmulss 256(%rdx,%rax), %xmm1, %xmm1 {%k2}
+
+// CHECK: vmulss 256(%rdx,%rax), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x8a,0x59,0x4c,0x02,0x40]
+vmulss 256(%rdx,%rax), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vmulss 256(%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0xc5,0x02,0x59,0xba,0x00,0x01,0x00,0x00]
+vmulss 256(%rdx), %xmm15, %xmm15
+
+// CHECK: vmulss 256(%rdx), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x71,0x06,0x0a,0x59,0x7a,0x40]
+vmulss 256(%rdx), %xmm15, %xmm15 {%k2}
+
+// CHECK: vmulss 256(%rdx), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x71,0x06,0x8a,0x59,0x7a,0x40]
+vmulss 256(%rdx), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vmulss 256(%rdx), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x59,0x8a,0x00,0x01,0x00,0x00]
+vmulss 256(%rdx), %xmm1, %xmm1
+
+// CHECK: vmulss 256(%rdx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0x59,0x4a,0x40]
+vmulss 256(%rdx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vmulss 256(%rdx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x8a,0x59,0x4a,0x40]
+vmulss 256(%rdx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vmulss 485498096, %xmm15, %xmm15
+// CHECK: encoding: [0xc5,0x02,0x59,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vmulss 485498096, %xmm15, %xmm15
+
+// CHECK: vmulss 485498096, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x71,0x06,0x0a,0x59,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vmulss 485498096, %xmm15, %xmm15 {%k2}
+
+// CHECK: vmulss 485498096, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x71,0x06,0x8a,0x59,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vmulss 485498096, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vmulss 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x59,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vmulss 485498096, %xmm1, %xmm1
+
+// CHECK: vmulss 485498096, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0x59,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vmulss 485498096, %xmm1, %xmm1 {%k2}
+
+// CHECK: vmulss 485498096, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x8a,0x59,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vmulss 485498096, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vmulss {rd-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x51,0x06,0x38,0x59,0xff]
+vmulss {rd-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vmulss {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x51,0x06,0x3a,0x59,0xff]
+vmulss {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vmulss {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x51,0x06,0xba,0x59,0xff]
+vmulss {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vmulss {rd-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0x76,0x38,0x59,0xc9]
+vmulss {rd-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vmulss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x3a,0x59,0xc9]
+vmulss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vmulss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0xba,0x59,0xc9]
+vmulss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vmulss (%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0xc5,0x02,0x59,0x3a]
+vmulss (%rdx), %xmm15, %xmm15
+
+// CHECK: vmulss (%rdx), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x71,0x06,0x0a,0x59,0x3a]
+vmulss (%rdx), %xmm15, %xmm15 {%k2}
+
+// CHECK: vmulss (%rdx), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x71,0x06,0x8a,0x59,0x3a]
+vmulss (%rdx), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vmulss (%rdx), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x59,0x0a]
+vmulss (%rdx), %xmm1, %xmm1
+
+// CHECK: vmulss (%rdx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0x59,0x0a]
+vmulss (%rdx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vmulss (%rdx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x8a,0x59,0x0a]
+vmulss (%rdx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vmulss {rn-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x51,0x06,0x18,0x59,0xff]
+vmulss {rn-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vmulss {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x51,0x06,0x1a,0x59,0xff]
+vmulss {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vmulss {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x51,0x06,0x9a,0x59,0xff]
+vmulss {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vmulss {rn-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0x76,0x18,0x59,0xc9]
+vmulss {rn-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vmulss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x1a,0x59,0xc9]
+vmulss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vmulss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x9a,0x59,0xc9]
+vmulss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vmulss {ru-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x51,0x06,0x58,0x59,0xff]
+vmulss {ru-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vmulss {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x51,0x06,0x5a,0x59,0xff]
+vmulss {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vmulss {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x51,0x06,0xda,0x59,0xff]
+vmulss {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vmulss {ru-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0x76,0x58,0x59,0xc9]
+vmulss {ru-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vmulss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x5a,0x59,0xc9]
+vmulss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vmulss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0xda,0x59,0xc9]
+vmulss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vmulss {rz-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x51,0x06,0x78,0x59,0xff]
+vmulss {rz-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vmulss {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x51,0x06,0x7a,0x59,0xff]
+vmulss {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vmulss {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x51,0x06,0xfa,0x59,0xff]
+vmulss {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vmulss {rz-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0x76,0x78,0x59,0xc9]
+vmulss {rz-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vmulss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x7a,0x59,0xc9]
+vmulss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vmulss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0xfa,0x59,0xc9]
+vmulss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vmulss %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x41,0x02,0x59,0xff]
+vmulss %xmm15, %xmm15, %xmm15
+
+// CHECK: vmulss %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x51,0x06,0x0a,0x59,0xff]
+vmulss %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vmulss %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x51,0x06,0x8a,0x59,0xff]
+vmulss %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vmulss %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x59,0xc9]
+vmulss %xmm1, %xmm1, %xmm1
+
+// CHECK: vmulss %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0x59,0xc9]
+vmulss %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vmulss %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x8a,0x59,0xc9]
+vmulss %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vrcp14sd 485498096, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x72,0x85,0x08,0x4d,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vrcp14sd 485498096, %xmm15, %xmm15
+
+// CHECK: vrcp14sd 485498096, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x85,0x0a,0x4d,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vrcp14sd 485498096, %xmm15, %xmm15 {%k2}
+
+// CHECK: vrcp14sd 485498096, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x85,0x8a,0x4d,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vrcp14sd 485498096, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vrcp14sd 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x08,0x4d,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vrcp14sd 485498096, %xmm1, %xmm1
+
+// CHECK: vrcp14sd 485498096, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0x4d,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vrcp14sd 485498096, %xmm1, %xmm1 {%k2}
+
+// CHECK: vrcp14sd 485498096, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0x4d,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vrcp14sd 485498096, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vrcp14sd 512(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x72,0x85,0x08,0x4d,0x7c,0x82,0x40]
+vrcp14sd 512(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vrcp14sd -512(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x72,0x85,0x08,0x4d,0x7c,0x82,0xc0]
+vrcp14sd -512(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vrcp14sd 512(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x85,0x0a,0x4d,0x7c,0x82,0x40]
+vrcp14sd 512(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+
+// CHECK: vrcp14sd -512(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x85,0x0a,0x4d,0x7c,0x82,0xc0]
+vrcp14sd -512(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+
+// CHECK: vrcp14sd 512(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x85,0x8a,0x4d,0x7c,0x82,0x40]
+vrcp14sd 512(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vrcp14sd -512(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x85,0x8a,0x4d,0x7c,0x82,0xc0]
+vrcp14sd -512(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vrcp14sd 512(%rdx,%rax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x08,0x4d,0x4c,0x82,0x40]
+vrcp14sd 512(%rdx,%rax,4), %xmm1, %xmm1
+
+// CHECK: vrcp14sd -512(%rdx,%rax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x08,0x4d,0x4c,0x82,0xc0]
+vrcp14sd -512(%rdx,%rax,4), %xmm1, %xmm1
+
+// CHECK: vrcp14sd 512(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0x4d,0x4c,0x82,0x40]
+vrcp14sd 512(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vrcp14sd -512(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0x4d,0x4c,0x82,0xc0]
+vrcp14sd -512(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vrcp14sd 512(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0x4d,0x4c,0x82,0x40]
+vrcp14sd 512(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vrcp14sd -512(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0x4d,0x4c,0x82,0xc0]
+vrcp14sd -512(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vrcp14sd 512(%rdx,%rax), %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x72,0x85,0x08,0x4d,0x7c,0x02,0x40]
+vrcp14sd 512(%rdx,%rax), %xmm15, %xmm15
+
+// CHECK: vrcp14sd 512(%rdx,%rax), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x85,0x0a,0x4d,0x7c,0x02,0x40]
+vrcp14sd 512(%rdx,%rax), %xmm15, %xmm15 {%k2}
+
+// CHECK: vrcp14sd 512(%rdx,%rax), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x85,0x8a,0x4d,0x7c,0x02,0x40]
+vrcp14sd 512(%rdx,%rax), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vrcp14sd 512(%rdx,%rax), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x08,0x4d,0x4c,0x02,0x40]
+vrcp14sd 512(%rdx,%rax), %xmm1, %xmm1
+
+// CHECK: vrcp14sd 512(%rdx,%rax), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0x4d,0x4c,0x02,0x40]
+vrcp14sd 512(%rdx,%rax), %xmm1, %xmm1 {%k2}
+
+// CHECK: vrcp14sd 512(%rdx,%rax), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0x4d,0x4c,0x02,0x40]
+vrcp14sd 512(%rdx,%rax), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vrcp14sd 512(%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x72,0x85,0x08,0x4d,0x7a,0x40]
+vrcp14sd 512(%rdx), %xmm15, %xmm15
+
+// CHECK: vrcp14sd 512(%rdx), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x85,0x0a,0x4d,0x7a,0x40]
+vrcp14sd 512(%rdx), %xmm15, %xmm15 {%k2}
+
+// CHECK: vrcp14sd 512(%rdx), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x85,0x8a,0x4d,0x7a,0x40]
+vrcp14sd 512(%rdx), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vrcp14sd 512(%rdx), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x08,0x4d,0x4a,0x40]
+vrcp14sd 512(%rdx), %xmm1, %xmm1
+
+// CHECK: vrcp14sd 512(%rdx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0x4d,0x4a,0x40]
+vrcp14sd 512(%rdx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vrcp14sd 512(%rdx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0x4d,0x4a,0x40]
+vrcp14sd 512(%rdx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vrcp14sd (%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x72,0x85,0x08,0x4d,0x3a]
+vrcp14sd (%rdx), %xmm15, %xmm15
+
+// CHECK: vrcp14sd (%rdx), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x85,0x0a,0x4d,0x3a]
+vrcp14sd (%rdx), %xmm15, %xmm15 {%k2}
+
+// CHECK: vrcp14sd (%rdx), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x85,0x8a,0x4d,0x3a]
+vrcp14sd (%rdx), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vrcp14sd (%rdx), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x08,0x4d,0x0a]
+vrcp14sd (%rdx), %xmm1, %xmm1
+
+// CHECK: vrcp14sd (%rdx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0x4d,0x0a]
+vrcp14sd (%rdx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vrcp14sd (%rdx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0x4d,0x0a]
+vrcp14sd (%rdx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vrcp14sd %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x52,0x85,0x08,0x4d,0xff]
+vrcp14sd %xmm15, %xmm15, %xmm15
+
+// CHECK: vrcp14sd %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x85,0x0a,0x4d,0xff]
+vrcp14sd %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vrcp14sd %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x85,0x8a,0x4d,0xff]
+vrcp14sd %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vrcp14sd %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x08,0x4d,0xc9]
+vrcp14sd %xmm1, %xmm1, %xmm1
+
+// CHECK: vrcp14sd %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0x4d,0xc9]
+vrcp14sd %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vrcp14sd %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0x4d,0xc9]
+vrcp14sd %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vrcp14ss 256(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x72,0x05,0x08,0x4d,0x7c,0x82,0x40]
+vrcp14ss 256(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vrcp14ss -256(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x72,0x05,0x08,0x4d,0x7c,0x82,0xc0]
+vrcp14ss -256(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vrcp14ss 256(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x05,0x0a,0x4d,0x7c,0x82,0x40]
+vrcp14ss 256(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+
+// CHECK: vrcp14ss -256(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x05,0x0a,0x4d,0x7c,0x82,0xc0]
+vrcp14ss -256(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+
+// CHECK: vrcp14ss 256(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x05,0x8a,0x4d,0x7c,0x82,0x40]
+vrcp14ss 256(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vrcp14ss -256(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x05,0x8a,0x4d,0x7c,0x82,0xc0]
+vrcp14ss -256(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vrcp14ss 256(%rdx,%rax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x08,0x4d,0x4c,0x82,0x40]
+vrcp14ss 256(%rdx,%rax,4), %xmm1, %xmm1
+
+// CHECK: vrcp14ss -256(%rdx,%rax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x08,0x4d,0x4c,0x82,0xc0]
+vrcp14ss -256(%rdx,%rax,4), %xmm1, %xmm1
+
+// CHECK: vrcp14ss 256(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0x4d,0x4c,0x82,0x40]
+vrcp14ss 256(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vrcp14ss -256(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0x4d,0x4c,0x82,0xc0]
+vrcp14ss -256(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vrcp14ss 256(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0x4d,0x4c,0x82,0x40]
+vrcp14ss 256(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vrcp14ss -256(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0x4d,0x4c,0x82,0xc0]
+vrcp14ss -256(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vrcp14ss 256(%rdx,%rax), %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x72,0x05,0x08,0x4d,0x7c,0x02,0x40]
+vrcp14ss 256(%rdx,%rax), %xmm15, %xmm15
+
+// CHECK: vrcp14ss 256(%rdx,%rax), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x05,0x0a,0x4d,0x7c,0x02,0x40]
+vrcp14ss 256(%rdx,%rax), %xmm15, %xmm15 {%k2}
+
+// CHECK: vrcp14ss 256(%rdx,%rax), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x05,0x8a,0x4d,0x7c,0x02,0x40]
+vrcp14ss 256(%rdx,%rax), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vrcp14ss 256(%rdx,%rax), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x08,0x4d,0x4c,0x02,0x40]
+vrcp14ss 256(%rdx,%rax), %xmm1, %xmm1
+
+// CHECK: vrcp14ss 256(%rdx,%rax), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0x4d,0x4c,0x02,0x40]
+vrcp14ss 256(%rdx,%rax), %xmm1, %xmm1 {%k2}
+
+// CHECK: vrcp14ss 256(%rdx,%rax), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0x4d,0x4c,0x02,0x40]
+vrcp14ss 256(%rdx,%rax), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vrcp14ss 256(%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x72,0x05,0x08,0x4d,0x7a,0x40]
+vrcp14ss 256(%rdx), %xmm15, %xmm15
+
+// CHECK: vrcp14ss 256(%rdx), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x05,0x0a,0x4d,0x7a,0x40]
+vrcp14ss 256(%rdx), %xmm15, %xmm15 {%k2}
+
+// CHECK: vrcp14ss 256(%rdx), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x05,0x8a,0x4d,0x7a,0x40]
+vrcp14ss 256(%rdx), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vrcp14ss 256(%rdx), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x08,0x4d,0x4a,0x40]
+vrcp14ss 256(%rdx), %xmm1, %xmm1
+
+// CHECK: vrcp14ss 256(%rdx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0x4d,0x4a,0x40]
+vrcp14ss 256(%rdx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vrcp14ss 256(%rdx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0x4d,0x4a,0x40]
+vrcp14ss 256(%rdx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vrcp14ss 485498096, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x72,0x05,0x08,0x4d,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vrcp14ss 485498096, %xmm15, %xmm15
+
+// CHECK: vrcp14ss 485498096, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x05,0x0a,0x4d,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vrcp14ss 485498096, %xmm15, %xmm15 {%k2}
+
+// CHECK: vrcp14ss 485498096, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x05,0x8a,0x4d,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vrcp14ss 485498096, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vrcp14ss 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x08,0x4d,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vrcp14ss 485498096, %xmm1, %xmm1
+
+// CHECK: vrcp14ss 485498096, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0x4d,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vrcp14ss 485498096, %xmm1, %xmm1 {%k2}
+
+// CHECK: vrcp14ss 485498096, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0x4d,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vrcp14ss 485498096, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vrcp14ss (%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x72,0x05,0x08,0x4d,0x3a]
+vrcp14ss (%rdx), %xmm15, %xmm15
+
+// CHECK: vrcp14ss (%rdx), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x05,0x0a,0x4d,0x3a]
+vrcp14ss (%rdx), %xmm15, %xmm15 {%k2}
+
+// CHECK: vrcp14ss (%rdx), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x05,0x8a,0x4d,0x3a]
+vrcp14ss (%rdx), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vrcp14ss (%rdx), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x08,0x4d,0x0a]
+vrcp14ss (%rdx), %xmm1, %xmm1
+
+// CHECK: vrcp14ss (%rdx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0x4d,0x0a]
+vrcp14ss (%rdx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vrcp14ss (%rdx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0x4d,0x0a]
+vrcp14ss (%rdx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vrcp14ss %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x52,0x05,0x08,0x4d,0xff]
+vrcp14ss %xmm15, %xmm15, %xmm15
+
+// CHECK: vrcp14ss %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x05,0x0a,0x4d,0xff]
+vrcp14ss %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vrcp14ss %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x05,0x8a,0x4d,0xff]
+vrcp14ss %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vrcp14ss %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x08,0x4d,0xc9]
+vrcp14ss %xmm1, %xmm1, %xmm1
+
+// CHECK: vrcp14ss %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0x4d,0xc9]
+vrcp14ss %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vrcp14ss %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0x4d,0xc9]
+vrcp14ss %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vrndscalesd $0, 485498096, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x73,0x85,0x08,0x0b,0x3c,0x25,0xf0,0x1c,0xf0,0x1c,0x00]
+vrndscalesd $0, 485498096, %xmm15, %xmm15
+
+// CHECK: vrndscalesd $0, 485498096, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x73,0x85,0x0a,0x0b,0x3c,0x25,0xf0,0x1c,0xf0,0x1c,0x00]
+vrndscalesd $0, 485498096, %xmm15, %xmm15 {%k2}
+
+// CHECK: vrndscalesd $0, 485498096, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x73,0x85,0x8a,0x0b,0x3c,0x25,0xf0,0x1c,0xf0,0x1c,0x00]
+vrndscalesd $0, 485498096, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vrndscalesd $0, 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf3,0xf5,0x08,0x0b,0x0c,0x25,0xf0,0x1c,0xf0,0x1c,0x00]
+vrndscalesd $0, 485498096, %xmm1, %xmm1
+
+// CHECK: vrndscalesd $0, 485498096, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf3,0xf5,0x0a,0x0b,0x0c,0x25,0xf0,0x1c,0xf0,0x1c,0x00]
+vrndscalesd $0, 485498096, %xmm1, %xmm1 {%k2}
+
+// CHECK: vrndscalesd $0, 485498096, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf3,0xf5,0x8a,0x0b,0x0c,0x25,0xf0,0x1c,0xf0,0x1c,0x00]
+vrndscalesd $0, 485498096, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vrndscalesd $0, 512(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x73,0x85,0x08,0x0b,0x7c,0x82,0x40,0x00]
+vrndscalesd $0, 512(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vrndscalesd $0, -512(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x73,0x85,0x08,0x0b,0x7c,0x82,0xc0,0x00]
+vrndscalesd $0, -512(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vrndscalesd $0, 512(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x73,0x85,0x0a,0x0b,0x7c,0x82,0x40,0x00]
+vrndscalesd $0, 512(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+
+// CHECK: vrndscalesd $0, -512(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x73,0x85,0x0a,0x0b,0x7c,0x82,0xc0,0x00]
+vrndscalesd $0, -512(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+
+// CHECK: vrndscalesd $0, 512(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x73,0x85,0x8a,0x0b,0x7c,0x82,0x40,0x00]
+vrndscalesd $0, 512(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vrndscalesd $0, -512(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x73,0x85,0x8a,0x0b,0x7c,0x82,0xc0,0x00]
+vrndscalesd $0, -512(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vrndscalesd $0, 512(%rdx,%rax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf3,0xf5,0x08,0x0b,0x4c,0x82,0x40,0x00]
+vrndscalesd $0, 512(%rdx,%rax,4), %xmm1, %xmm1
+
+// CHECK: vrndscalesd $0, -512(%rdx,%rax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf3,0xf5,0x08,0x0b,0x4c,0x82,0xc0,0x00]
+vrndscalesd $0, -512(%rdx,%rax,4), %xmm1, %xmm1
+
+// CHECK: vrndscalesd $0, 512(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf3,0xf5,0x0a,0x0b,0x4c,0x82,0x40,0x00]
+vrndscalesd $0, 512(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vrndscalesd $0, -512(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf3,0xf5,0x0a,0x0b,0x4c,0x82,0xc0,0x00]
+vrndscalesd $0, -512(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vrndscalesd $0, 512(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf3,0xf5,0x8a,0x0b,0x4c,0x82,0x40,0x00]
+vrndscalesd $0, 512(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vrndscalesd $0, -512(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf3,0xf5,0x8a,0x0b,0x4c,0x82,0xc0,0x00]
+vrndscalesd $0, -512(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vrndscalesd $0, 512(%rdx,%rax), %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x73,0x85,0x08,0x0b,0x7c,0x02,0x40,0x00]
+vrndscalesd $0, 512(%rdx,%rax), %xmm15, %xmm15
+
+// CHECK: vrndscalesd $0, 512(%rdx,%rax), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x73,0x85,0x0a,0x0b,0x7c,0x02,0x40,0x00]
+vrndscalesd $0, 512(%rdx,%rax), %xmm15, %xmm15 {%k2}
+
+// CHECK: vrndscalesd $0, 512(%rdx,%rax), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x73,0x85,0x8a,0x0b,0x7c,0x02,0x40,0x00]
+vrndscalesd $0, 512(%rdx,%rax), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vrndscalesd $0, 512(%rdx,%rax), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf3,0xf5,0x08,0x0b,0x4c,0x02,0x40,0x00]
+vrndscalesd $0, 512(%rdx,%rax), %xmm1, %xmm1
+
+// CHECK: vrndscalesd $0, 512(%rdx,%rax), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf3,0xf5,0x0a,0x0b,0x4c,0x02,0x40,0x00]
+vrndscalesd $0, 512(%rdx,%rax), %xmm1, %xmm1 {%k2}
+
+// CHECK: vrndscalesd $0, 512(%rdx,%rax), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf3,0xf5,0x8a,0x0b,0x4c,0x02,0x40,0x00]
+vrndscalesd $0, 512(%rdx,%rax), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vrndscalesd $0, 512(%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x73,0x85,0x08,0x0b,0x7a,0x40,0x00]
+vrndscalesd $0, 512(%rdx), %xmm15, %xmm15
+
+// CHECK: vrndscalesd $0, 512(%rdx), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x73,0x85,0x0a,0x0b,0x7a,0x40,0x00]
+vrndscalesd $0, 512(%rdx), %xmm15, %xmm15 {%k2}
+
+// CHECK: vrndscalesd $0, 512(%rdx), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x73,0x85,0x8a,0x0b,0x7a,0x40,0x00]
+vrndscalesd $0, 512(%rdx), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vrndscalesd $0, 512(%rdx), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf3,0xf5,0x08,0x0b,0x4a,0x40,0x00]
+vrndscalesd $0, 512(%rdx), %xmm1, %xmm1
+
+// CHECK: vrndscalesd $0, 512(%rdx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf3,0xf5,0x0a,0x0b,0x4a,0x40,0x00]
+vrndscalesd $0, 512(%rdx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vrndscalesd $0, 512(%rdx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf3,0xf5,0x8a,0x0b,0x4a,0x40,0x00]
+vrndscalesd $0, 512(%rdx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vrndscalesd $0, (%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x73,0x85,0x08,0x0b,0x3a,0x00]
+vrndscalesd $0, (%rdx), %xmm15, %xmm15
+
+// CHECK: vrndscalesd $0, (%rdx), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x73,0x85,0x0a,0x0b,0x3a,0x00]
+vrndscalesd $0, (%rdx), %xmm15, %xmm15 {%k2}
+
+// CHECK: vrndscalesd $0, (%rdx), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x73,0x85,0x8a,0x0b,0x3a,0x00]
+vrndscalesd $0, (%rdx), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vrndscalesd $0, (%rdx), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf3,0xf5,0x08,0x0b,0x0a,0x00]
+vrndscalesd $0, (%rdx), %xmm1, %xmm1
+
+// CHECK: vrndscalesd $0, (%rdx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf3,0xf5,0x0a,0x0b,0x0a,0x00]
+vrndscalesd $0, (%rdx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vrndscalesd $0, (%rdx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf3,0xf5,0x8a,0x0b,0x0a,0x00]
+vrndscalesd $0, (%rdx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vrndscalesd $0, {sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x53,0x85,0x18,0x0b,0xff,0x00]
+vrndscalesd $0, {sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vrndscalesd $0, {sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x53,0x85,0x1a,0x0b,0xff,0x00]
+vrndscalesd $0, {sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vrndscalesd $0, {sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x53,0x85,0x9a,0x0b,0xff,0x00]
+vrndscalesd $0, {sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vrndscalesd $0, {sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf3,0xf5,0x18,0x0b,0xc9,0x00]
+vrndscalesd $0, {sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vrndscalesd $0, {sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf3,0xf5,0x1a,0x0b,0xc9,0x00]
+vrndscalesd $0, {sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vrndscalesd $0, {sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf3,0xf5,0x9a,0x0b,0xc9,0x00]
+vrndscalesd $0, {sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vrndscalesd $0, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x53,0x85,0x08,0x0b,0xff,0x00]
+vrndscalesd $0, %xmm15, %xmm15, %xmm15
+
+// CHECK: vrndscalesd $0, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x53,0x85,0x0a,0x0b,0xff,0x00]
+vrndscalesd $0, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vrndscalesd $0, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x53,0x85,0x8a,0x0b,0xff,0x00]
+vrndscalesd $0, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vrndscalesd $0, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf3,0xf5,0x08,0x0b,0xc9,0x00]
+vrndscalesd $0, %xmm1, %xmm1, %xmm1
+
+// CHECK: vrndscalesd $0, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf3,0xf5,0x0a,0x0b,0xc9,0x00]
+vrndscalesd $0, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vrndscalesd $0, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf3,0xf5,0x8a,0x0b,0xc9,0x00]
+vrndscalesd $0, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vrndscaless $0, 256(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x73,0x05,0x08,0x0a,0x7c,0x82,0x40,0x00]
+vrndscaless $0, 256(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vrndscaless $0, -256(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x73,0x05,0x08,0x0a,0x7c,0x82,0xc0,0x00]
+vrndscaless $0, -256(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vrndscaless $0, 256(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x73,0x05,0x0a,0x0a,0x7c,0x82,0x40,0x00]
+vrndscaless $0, 256(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+
+// CHECK: vrndscaless $0, -256(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x73,0x05,0x0a,0x0a,0x7c,0x82,0xc0,0x00]
+vrndscaless $0, -256(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+
+// CHECK: vrndscaless $0, 256(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x73,0x05,0x8a,0x0a,0x7c,0x82,0x40,0x00]
+vrndscaless $0, 256(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vrndscaless $0, -256(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x73,0x05,0x8a,0x0a,0x7c,0x82,0xc0,0x00]
+vrndscaless $0, -256(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vrndscaless $0, 256(%rdx,%rax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf3,0x75,0x08,0x0a,0x4c,0x82,0x40,0x00]
+vrndscaless $0, 256(%rdx,%rax,4), %xmm1, %xmm1
+
+// CHECK: vrndscaless $0, -256(%rdx,%rax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf3,0x75,0x08,0x0a,0x4c,0x82,0xc0,0x00]
+vrndscaless $0, -256(%rdx,%rax,4), %xmm1, %xmm1
+
+// CHECK: vrndscaless $0, 256(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf3,0x75,0x0a,0x0a,0x4c,0x82,0x40,0x00]
+vrndscaless $0, 256(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vrndscaless $0, -256(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf3,0x75,0x0a,0x0a,0x4c,0x82,0xc0,0x00]
+vrndscaless $0, -256(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vrndscaless $0, 256(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf3,0x75,0x8a,0x0a,0x4c,0x82,0x40,0x00]
+vrndscaless $0, 256(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vrndscaless $0, -256(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf3,0x75,0x8a,0x0a,0x4c,0x82,0xc0,0x00]
+vrndscaless $0, -256(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vrndscaless $0, 256(%rdx,%rax), %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x73,0x05,0x08,0x0a,0x7c,0x02,0x40,0x00]
+vrndscaless $0, 256(%rdx,%rax), %xmm15, %xmm15
+
+// CHECK: vrndscaless $0, 256(%rdx,%rax), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x73,0x05,0x0a,0x0a,0x7c,0x02,0x40,0x00]
+vrndscaless $0, 256(%rdx,%rax), %xmm15, %xmm15 {%k2}
+
+// CHECK: vrndscaless $0, 256(%rdx,%rax), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x73,0x05,0x8a,0x0a,0x7c,0x02,0x40,0x00]
+vrndscaless $0, 256(%rdx,%rax), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vrndscaless $0, 256(%rdx,%rax), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf3,0x75,0x08,0x0a,0x4c,0x02,0x40,0x00]
+vrndscaless $0, 256(%rdx,%rax), %xmm1, %xmm1
+
+// CHECK: vrndscaless $0, 256(%rdx,%rax), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf3,0x75,0x0a,0x0a,0x4c,0x02,0x40,0x00]
+vrndscaless $0, 256(%rdx,%rax), %xmm1, %xmm1 {%k2}
+
+// CHECK: vrndscaless $0, 256(%rdx,%rax), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf3,0x75,0x8a,0x0a,0x4c,0x02,0x40,0x00]
+vrndscaless $0, 256(%rdx,%rax), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vrndscaless $0, 256(%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x73,0x05,0x08,0x0a,0x7a,0x40,0x00]
+vrndscaless $0, 256(%rdx), %xmm15, %xmm15
+
+// CHECK: vrndscaless $0, 256(%rdx), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x73,0x05,0x0a,0x0a,0x7a,0x40,0x00]
+vrndscaless $0, 256(%rdx), %xmm15, %xmm15 {%k2}
+
+// CHECK: vrndscaless $0, 256(%rdx), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x73,0x05,0x8a,0x0a,0x7a,0x40,0x00]
+vrndscaless $0, 256(%rdx), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vrndscaless $0, 256(%rdx), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf3,0x75,0x08,0x0a,0x4a,0x40,0x00]
+vrndscaless $0, 256(%rdx), %xmm1, %xmm1
+
+// CHECK: vrndscaless $0, 256(%rdx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf3,0x75,0x0a,0x0a,0x4a,0x40,0x00]
+vrndscaless $0, 256(%rdx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vrndscaless $0, 256(%rdx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf3,0x75,0x8a,0x0a,0x4a,0x40,0x00]
+vrndscaless $0, 256(%rdx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vrndscaless $0, 485498096, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x73,0x05,0x08,0x0a,0x3c,0x25,0xf0,0x1c,0xf0,0x1c,0x00]
+vrndscaless $0, 485498096, %xmm15, %xmm15
+
+// CHECK: vrndscaless $0, 485498096, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x73,0x05,0x0a,0x0a,0x3c,0x25,0xf0,0x1c,0xf0,0x1c,0x00]
+vrndscaless $0, 485498096, %xmm15, %xmm15 {%k2}
+
+// CHECK: vrndscaless $0, 485498096, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x73,0x05,0x8a,0x0a,0x3c,0x25,0xf0,0x1c,0xf0,0x1c,0x00]
+vrndscaless $0, 485498096, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vrndscaless $0, 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf3,0x75,0x08,0x0a,0x0c,0x25,0xf0,0x1c,0xf0,0x1c,0x00]
+vrndscaless $0, 485498096, %xmm1, %xmm1
+
+// CHECK: vrndscaless $0, 485498096, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf3,0x75,0x0a,0x0a,0x0c,0x25,0xf0,0x1c,0xf0,0x1c,0x00]
+vrndscaless $0, 485498096, %xmm1, %xmm1 {%k2}
+
+// CHECK: vrndscaless $0, 485498096, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf3,0x75,0x8a,0x0a,0x0c,0x25,0xf0,0x1c,0xf0,0x1c,0x00]
+vrndscaless $0, 485498096, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vrndscaless $0, (%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x73,0x05,0x08,0x0a,0x3a,0x00]
+vrndscaless $0, (%rdx), %xmm15, %xmm15
+
+// CHECK: vrndscaless $0, (%rdx), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x73,0x05,0x0a,0x0a,0x3a,0x00]
+vrndscaless $0, (%rdx), %xmm15, %xmm15 {%k2}
+
+// CHECK: vrndscaless $0, (%rdx), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x73,0x05,0x8a,0x0a,0x3a,0x00]
+vrndscaless $0, (%rdx), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vrndscaless $0, (%rdx), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf3,0x75,0x08,0x0a,0x0a,0x00]
+vrndscaless $0, (%rdx), %xmm1, %xmm1
+
+// CHECK: vrndscaless $0, (%rdx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf3,0x75,0x0a,0x0a,0x0a,0x00]
+vrndscaless $0, (%rdx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vrndscaless $0, (%rdx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf3,0x75,0x8a,0x0a,0x0a,0x00]
+vrndscaless $0, (%rdx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vrndscaless $0, {sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x53,0x05,0x18,0x0a,0xff,0x00]
+vrndscaless $0, {sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vrndscaless $0, {sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x53,0x05,0x1a,0x0a,0xff,0x00]
+vrndscaless $0, {sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vrndscaless $0, {sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x53,0x05,0x9a,0x0a,0xff,0x00]
+vrndscaless $0, {sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vrndscaless $0, {sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf3,0x75,0x18,0x0a,0xc9,0x00]
+vrndscaless $0, {sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vrndscaless $0, {sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf3,0x75,0x1a,0x0a,0xc9,0x00]
+vrndscaless $0, {sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vrndscaless $0, {sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf3,0x75,0x9a,0x0a,0xc9,0x00]
+vrndscaless $0, {sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vrndscaless $0, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x53,0x05,0x08,0x0a,0xff,0x00]
+vrndscaless $0, %xmm15, %xmm15, %xmm15
+
+// CHECK: vrndscaless $0, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x53,0x05,0x0a,0x0a,0xff,0x00]
+vrndscaless $0, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vrndscaless $0, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x53,0x05,0x8a,0x0a,0xff,0x00]
+vrndscaless $0, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vrndscaless $0, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf3,0x75,0x08,0x0a,0xc9,0x00]
+vrndscaless $0, %xmm1, %xmm1, %xmm1
+
+// CHECK: vrndscaless $0, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf3,0x75,0x0a,0x0a,0xc9,0x00]
+vrndscaless $0, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vrndscaless $0, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf3,0x75,0x8a,0x0a,0xc9,0x00]
+vrndscaless $0, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vrsqrt14sd 485498096, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x72,0x85,0x08,0x4f,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vrsqrt14sd 485498096, %xmm15, %xmm15
+
+// CHECK: vrsqrt14sd 485498096, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x85,0x0a,0x4f,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vrsqrt14sd 485498096, %xmm15, %xmm15 {%k2}
+
+// CHECK: vrsqrt14sd 485498096, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x85,0x8a,0x4f,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vrsqrt14sd 485498096, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vrsqrt14sd 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x08,0x4f,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vrsqrt14sd 485498096, %xmm1, %xmm1
+
+// CHECK: vrsqrt14sd 485498096, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0x4f,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vrsqrt14sd 485498096, %xmm1, %xmm1 {%k2}
+
+// CHECK: vrsqrt14sd 485498096, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0x4f,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vrsqrt14sd 485498096, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vrsqrt14sd 512(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x72,0x85,0x08,0x4f,0x7c,0x82,0x40]
+vrsqrt14sd 512(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vrsqrt14sd -512(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x72,0x85,0x08,0x4f,0x7c,0x82,0xc0]
+vrsqrt14sd -512(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vrsqrt14sd 512(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x85,0x0a,0x4f,0x7c,0x82,0x40]
+vrsqrt14sd 512(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+
+// CHECK: vrsqrt14sd -512(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x85,0x0a,0x4f,0x7c,0x82,0xc0]
+vrsqrt14sd -512(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+
+// CHECK: vrsqrt14sd 512(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x85,0x8a,0x4f,0x7c,0x82,0x40]
+vrsqrt14sd 512(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vrsqrt14sd -512(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x85,0x8a,0x4f,0x7c,0x82,0xc0]
+vrsqrt14sd -512(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vrsqrt14sd 512(%rdx,%rax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x08,0x4f,0x4c,0x82,0x40]
+vrsqrt14sd 512(%rdx,%rax,4), %xmm1, %xmm1
+
+// CHECK: vrsqrt14sd -512(%rdx,%rax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x08,0x4f,0x4c,0x82,0xc0]
+vrsqrt14sd -512(%rdx,%rax,4), %xmm1, %xmm1
+
+// CHECK: vrsqrt14sd 512(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0x4f,0x4c,0x82,0x40]
+vrsqrt14sd 512(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vrsqrt14sd -512(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0x4f,0x4c,0x82,0xc0]
+vrsqrt14sd -512(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vrsqrt14sd 512(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0x4f,0x4c,0x82,0x40]
+vrsqrt14sd 512(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vrsqrt14sd -512(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0x4f,0x4c,0x82,0xc0]
+vrsqrt14sd -512(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vrsqrt14sd 512(%rdx,%rax), %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x72,0x85,0x08,0x4f,0x7c,0x02,0x40]
+vrsqrt14sd 512(%rdx,%rax), %xmm15, %xmm15
+
+// CHECK: vrsqrt14sd 512(%rdx,%rax), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x85,0x0a,0x4f,0x7c,0x02,0x40]
+vrsqrt14sd 512(%rdx,%rax), %xmm15, %xmm15 {%k2}
+
+// CHECK: vrsqrt14sd 512(%rdx,%rax), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x85,0x8a,0x4f,0x7c,0x02,0x40]
+vrsqrt14sd 512(%rdx,%rax), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vrsqrt14sd 512(%rdx,%rax), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x08,0x4f,0x4c,0x02,0x40]
+vrsqrt14sd 512(%rdx,%rax), %xmm1, %xmm1
+
+// CHECK: vrsqrt14sd 512(%rdx,%rax), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0x4f,0x4c,0x02,0x40]
+vrsqrt14sd 512(%rdx,%rax), %xmm1, %xmm1 {%k2}
+
+// CHECK: vrsqrt14sd 512(%rdx,%rax), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0x4f,0x4c,0x02,0x40]
+vrsqrt14sd 512(%rdx,%rax), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vrsqrt14sd 512(%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x72,0x85,0x08,0x4f,0x7a,0x40]
+vrsqrt14sd 512(%rdx), %xmm15, %xmm15
+
+// CHECK: vrsqrt14sd 512(%rdx), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x85,0x0a,0x4f,0x7a,0x40]
+vrsqrt14sd 512(%rdx), %xmm15, %xmm15 {%k2}
+
+// CHECK: vrsqrt14sd 512(%rdx), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x85,0x8a,0x4f,0x7a,0x40]
+vrsqrt14sd 512(%rdx), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vrsqrt14sd 512(%rdx), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x08,0x4f,0x4a,0x40]
+vrsqrt14sd 512(%rdx), %xmm1, %xmm1
+
+// CHECK: vrsqrt14sd 512(%rdx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0x4f,0x4a,0x40]
+vrsqrt14sd 512(%rdx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vrsqrt14sd 512(%rdx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0x4f,0x4a,0x40]
+vrsqrt14sd 512(%rdx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vrsqrt14sd (%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x72,0x85,0x08,0x4f,0x3a]
+vrsqrt14sd (%rdx), %xmm15, %xmm15
+
+// CHECK: vrsqrt14sd (%rdx), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x85,0x0a,0x4f,0x3a]
+vrsqrt14sd (%rdx), %xmm15, %xmm15 {%k2}
+
+// CHECK: vrsqrt14sd (%rdx), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x85,0x8a,0x4f,0x3a]
+vrsqrt14sd (%rdx), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vrsqrt14sd (%rdx), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x08,0x4f,0x0a]
+vrsqrt14sd (%rdx), %xmm1, %xmm1
+
+// CHECK: vrsqrt14sd (%rdx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0x4f,0x0a]
+vrsqrt14sd (%rdx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vrsqrt14sd (%rdx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0x4f,0x0a]
+vrsqrt14sd (%rdx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vrsqrt14sd %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x52,0x85,0x08,0x4f,0xff]
+vrsqrt14sd %xmm15, %xmm15, %xmm15
+
+// CHECK: vrsqrt14sd %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x85,0x0a,0x4f,0xff]
+vrsqrt14sd %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vrsqrt14sd %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x85,0x8a,0x4f,0xff]
+vrsqrt14sd %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vrsqrt14sd %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x08,0x4f,0xc9]
+vrsqrt14sd %xmm1, %xmm1, %xmm1
+
+// CHECK: vrsqrt14sd %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0x4f,0xc9]
+vrsqrt14sd %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vrsqrt14sd %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0x4f,0xc9]
+vrsqrt14sd %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vrsqrt14ss 256(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x72,0x05,0x08,0x4f,0x7c,0x82,0x40]
+vrsqrt14ss 256(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vrsqrt14ss -256(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x72,0x05,0x08,0x4f,0x7c,0x82,0xc0]
+vrsqrt14ss -256(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vrsqrt14ss 256(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x05,0x0a,0x4f,0x7c,0x82,0x40]
+vrsqrt14ss 256(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+
+// CHECK: vrsqrt14ss -256(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x05,0x0a,0x4f,0x7c,0x82,0xc0]
+vrsqrt14ss -256(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+
+// CHECK: vrsqrt14ss 256(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x05,0x8a,0x4f,0x7c,0x82,0x40]
+vrsqrt14ss 256(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vrsqrt14ss -256(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x05,0x8a,0x4f,0x7c,0x82,0xc0]
+vrsqrt14ss -256(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vrsqrt14ss 256(%rdx,%rax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x08,0x4f,0x4c,0x82,0x40]
+vrsqrt14ss 256(%rdx,%rax,4), %xmm1, %xmm1
+
+// CHECK: vrsqrt14ss -256(%rdx,%rax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x08,0x4f,0x4c,0x82,0xc0]
+vrsqrt14ss -256(%rdx,%rax,4), %xmm1, %xmm1
+
+// CHECK: vrsqrt14ss 256(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0x4f,0x4c,0x82,0x40]
+vrsqrt14ss 256(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vrsqrt14ss -256(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0x4f,0x4c,0x82,0xc0]
+vrsqrt14ss -256(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vrsqrt14ss 256(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0x4f,0x4c,0x82,0x40]
+vrsqrt14ss 256(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vrsqrt14ss -256(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0x4f,0x4c,0x82,0xc0]
+vrsqrt14ss -256(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vrsqrt14ss 256(%rdx,%rax), %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x72,0x05,0x08,0x4f,0x7c,0x02,0x40]
+vrsqrt14ss 256(%rdx,%rax), %xmm15, %xmm15
+
+// CHECK: vrsqrt14ss 256(%rdx,%rax), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x05,0x0a,0x4f,0x7c,0x02,0x40]
+vrsqrt14ss 256(%rdx,%rax), %xmm15, %xmm15 {%k2}
+
+// CHECK: vrsqrt14ss 256(%rdx,%rax), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x05,0x8a,0x4f,0x7c,0x02,0x40]
+vrsqrt14ss 256(%rdx,%rax), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vrsqrt14ss 256(%rdx,%rax), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x08,0x4f,0x4c,0x02,0x40]
+vrsqrt14ss 256(%rdx,%rax), %xmm1, %xmm1
+
+// CHECK: vrsqrt14ss 256(%rdx,%rax), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0x4f,0x4c,0x02,0x40]
+vrsqrt14ss 256(%rdx,%rax), %xmm1, %xmm1 {%k2}
+
+// CHECK: vrsqrt14ss 256(%rdx,%rax), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0x4f,0x4c,0x02,0x40]
+vrsqrt14ss 256(%rdx,%rax), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vrsqrt14ss 256(%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x72,0x05,0x08,0x4f,0x7a,0x40]
+vrsqrt14ss 256(%rdx), %xmm15, %xmm15
+
+// CHECK: vrsqrt14ss 256(%rdx), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x05,0x0a,0x4f,0x7a,0x40]
+vrsqrt14ss 256(%rdx), %xmm15, %xmm15 {%k2}
+
+// CHECK: vrsqrt14ss 256(%rdx), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x05,0x8a,0x4f,0x7a,0x40]
+vrsqrt14ss 256(%rdx), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vrsqrt14ss 256(%rdx), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x08,0x4f,0x4a,0x40]
+vrsqrt14ss 256(%rdx), %xmm1, %xmm1
+
+// CHECK: vrsqrt14ss 256(%rdx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0x4f,0x4a,0x40]
+vrsqrt14ss 256(%rdx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vrsqrt14ss 256(%rdx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0x4f,0x4a,0x40]
+vrsqrt14ss 256(%rdx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vrsqrt14ss 485498096, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x72,0x05,0x08,0x4f,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vrsqrt14ss 485498096, %xmm15, %xmm15
+
+// CHECK: vrsqrt14ss 485498096, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x05,0x0a,0x4f,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vrsqrt14ss 485498096, %xmm15, %xmm15 {%k2}
+
+// CHECK: vrsqrt14ss 485498096, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x05,0x8a,0x4f,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vrsqrt14ss 485498096, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vrsqrt14ss 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x08,0x4f,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vrsqrt14ss 485498096, %xmm1, %xmm1
+
+// CHECK: vrsqrt14ss 485498096, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0x4f,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vrsqrt14ss 485498096, %xmm1, %xmm1 {%k2}
+
+// CHECK: vrsqrt14ss 485498096, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0x4f,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vrsqrt14ss 485498096, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vrsqrt14ss (%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x72,0x05,0x08,0x4f,0x3a]
+vrsqrt14ss (%rdx), %xmm15, %xmm15
+
+// CHECK: vrsqrt14ss (%rdx), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x05,0x0a,0x4f,0x3a]
+vrsqrt14ss (%rdx), %xmm15, %xmm15 {%k2}
+
+// CHECK: vrsqrt14ss (%rdx), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x05,0x8a,0x4f,0x3a]
+vrsqrt14ss (%rdx), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vrsqrt14ss (%rdx), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x08,0x4f,0x0a]
+vrsqrt14ss (%rdx), %xmm1, %xmm1
+
+// CHECK: vrsqrt14ss (%rdx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0x4f,0x0a]
+vrsqrt14ss (%rdx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vrsqrt14ss (%rdx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0x4f,0x0a]
+vrsqrt14ss (%rdx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vrsqrt14ss %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x52,0x05,0x08,0x4f,0xff]
+vrsqrt14ss %xmm15, %xmm15, %xmm15
+
+// CHECK: vrsqrt14ss %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x05,0x0a,0x4f,0xff]
+vrsqrt14ss %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vrsqrt14ss %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x05,0x8a,0x4f,0xff]
+vrsqrt14ss %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vrsqrt14ss %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x08,0x4f,0xc9]
+vrsqrt14ss %xmm1, %xmm1, %xmm1
+
+// CHECK: vrsqrt14ss %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0x4f,0xc9]
+vrsqrt14ss %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vrsqrt14ss %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0x4f,0xc9]
+vrsqrt14ss %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vscalefsd 485498096, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x72,0x85,0x08,0x2d,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vscalefsd 485498096, %xmm15, %xmm15
+
+// CHECK: vscalefsd 485498096, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x85,0x0a,0x2d,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vscalefsd 485498096, %xmm15, %xmm15 {%k2}
+
+// CHECK: vscalefsd 485498096, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x85,0x8a,0x2d,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vscalefsd 485498096, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vscalefsd 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x08,0x2d,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vscalefsd 485498096, %xmm1, %xmm1
+
+// CHECK: vscalefsd 485498096, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0x2d,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vscalefsd 485498096, %xmm1, %xmm1 {%k2}
+
+// CHECK: vscalefsd 485498096, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0x2d,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vscalefsd 485498096, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vscalefsd 512(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x72,0x85,0x08,0x2d,0x7c,0x82,0x40]
+vscalefsd 512(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vscalefsd -512(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x72,0x85,0x08,0x2d,0x7c,0x82,0xc0]
+vscalefsd -512(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vscalefsd 512(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x85,0x0a,0x2d,0x7c,0x82,0x40]
+vscalefsd 512(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+
+// CHECK: vscalefsd -512(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x85,0x0a,0x2d,0x7c,0x82,0xc0]
+vscalefsd -512(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+
+// CHECK: vscalefsd 512(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x85,0x8a,0x2d,0x7c,0x82,0x40]
+vscalefsd 512(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vscalefsd -512(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x85,0x8a,0x2d,0x7c,0x82,0xc0]
+vscalefsd -512(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vscalefsd 512(%rdx,%rax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x08,0x2d,0x4c,0x82,0x40]
+vscalefsd 512(%rdx,%rax,4), %xmm1, %xmm1
+
+// CHECK: vscalefsd -512(%rdx,%rax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x08,0x2d,0x4c,0x82,0xc0]
+vscalefsd -512(%rdx,%rax,4), %xmm1, %xmm1
+
+// CHECK: vscalefsd 512(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0x2d,0x4c,0x82,0x40]
+vscalefsd 512(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vscalefsd -512(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0x2d,0x4c,0x82,0xc0]
+vscalefsd -512(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vscalefsd 512(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0x2d,0x4c,0x82,0x40]
+vscalefsd 512(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vscalefsd -512(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0x2d,0x4c,0x82,0xc0]
+vscalefsd -512(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vscalefsd 512(%rdx,%rax), %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x72,0x85,0x08,0x2d,0x7c,0x02,0x40]
+vscalefsd 512(%rdx,%rax), %xmm15, %xmm15
+
+// CHECK: vscalefsd 512(%rdx,%rax), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x85,0x0a,0x2d,0x7c,0x02,0x40]
+vscalefsd 512(%rdx,%rax), %xmm15, %xmm15 {%k2}
+
+// CHECK: vscalefsd 512(%rdx,%rax), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x85,0x8a,0x2d,0x7c,0x02,0x40]
+vscalefsd 512(%rdx,%rax), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vscalefsd 512(%rdx,%rax), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x08,0x2d,0x4c,0x02,0x40]
+vscalefsd 512(%rdx,%rax), %xmm1, %xmm1
+
+// CHECK: vscalefsd 512(%rdx,%rax), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0x2d,0x4c,0x02,0x40]
+vscalefsd 512(%rdx,%rax), %xmm1, %xmm1 {%k2}
+
+// CHECK: vscalefsd 512(%rdx,%rax), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0x2d,0x4c,0x02,0x40]
+vscalefsd 512(%rdx,%rax), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vscalefsd 512(%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x72,0x85,0x08,0x2d,0x7a,0x40]
+vscalefsd 512(%rdx), %xmm15, %xmm15
+
+// CHECK: vscalefsd 512(%rdx), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x85,0x0a,0x2d,0x7a,0x40]
+vscalefsd 512(%rdx), %xmm15, %xmm15 {%k2}
+
+// CHECK: vscalefsd 512(%rdx), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x85,0x8a,0x2d,0x7a,0x40]
+vscalefsd 512(%rdx), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vscalefsd 512(%rdx), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x08,0x2d,0x4a,0x40]
+vscalefsd 512(%rdx), %xmm1, %xmm1
+
+// CHECK: vscalefsd 512(%rdx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0x2d,0x4a,0x40]
+vscalefsd 512(%rdx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vscalefsd 512(%rdx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0x2d,0x4a,0x40]
+vscalefsd 512(%rdx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vscalefsd {rd-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x52,0x85,0x38,0x2d,0xff]
+vscalefsd {rd-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vscalefsd {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x85,0x3a,0x2d,0xff]
+vscalefsd {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vscalefsd {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x85,0xba,0x2d,0xff]
+vscalefsd {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vscalefsd {rd-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x38,0x2d,0xc9]
+vscalefsd {rd-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vscalefsd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x3a,0x2d,0xc9]
+vscalefsd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vscalefsd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0xba,0x2d,0xc9]
+vscalefsd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vscalefsd (%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x72,0x85,0x08,0x2d,0x3a]
+vscalefsd (%rdx), %xmm15, %xmm15
+
+// CHECK: vscalefsd (%rdx), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x85,0x0a,0x2d,0x3a]
+vscalefsd (%rdx), %xmm15, %xmm15 {%k2}
+
+// CHECK: vscalefsd (%rdx), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x85,0x8a,0x2d,0x3a]
+vscalefsd (%rdx), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vscalefsd (%rdx), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x08,0x2d,0x0a]
+vscalefsd (%rdx), %xmm1, %xmm1
+
+// CHECK: vscalefsd (%rdx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0x2d,0x0a]
+vscalefsd (%rdx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vscalefsd (%rdx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0x2d,0x0a]
+vscalefsd (%rdx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vscalefsd {rn-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x52,0x85,0x18,0x2d,0xff]
+vscalefsd {rn-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vscalefsd {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x85,0x1a,0x2d,0xff]
+vscalefsd {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vscalefsd {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x85,0x9a,0x2d,0xff]
+vscalefsd {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vscalefsd {rn-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x18,0x2d,0xc9]
+vscalefsd {rn-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vscalefsd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x1a,0x2d,0xc9]
+vscalefsd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vscalefsd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x9a,0x2d,0xc9]
+vscalefsd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vscalefsd {ru-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x52,0x85,0x58,0x2d,0xff]
+vscalefsd {ru-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vscalefsd {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x85,0x5a,0x2d,0xff]
+vscalefsd {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vscalefsd {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x85,0xda,0x2d,0xff]
+vscalefsd {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vscalefsd {ru-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x58,0x2d,0xc9]
+vscalefsd {ru-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vscalefsd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x5a,0x2d,0xc9]
+vscalefsd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vscalefsd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0xda,0x2d,0xc9]
+vscalefsd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vscalefsd {rz-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x52,0x85,0x78,0x2d,0xff]
+vscalefsd {rz-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vscalefsd {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x85,0x7a,0x2d,0xff]
+vscalefsd {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vscalefsd {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x85,0xfa,0x2d,0xff]
+vscalefsd {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vscalefsd {rz-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x78,0x2d,0xc9]
+vscalefsd {rz-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vscalefsd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x7a,0x2d,0xc9]
+vscalefsd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vscalefsd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0xfa,0x2d,0xc9]
+vscalefsd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vscalefsd %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x52,0x85,0x08,0x2d,0xff]
+vscalefsd %xmm15, %xmm15, %xmm15
+
+// CHECK: vscalefsd %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x85,0x0a,0x2d,0xff]
+vscalefsd %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vscalefsd %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x85,0x8a,0x2d,0xff]
+vscalefsd %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vscalefsd %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0xf5,0x08,0x2d,0xc9]
+vscalefsd %xmm1, %xmm1, %xmm1
+
+// CHECK: vscalefsd %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x0a,0x2d,0xc9]
+vscalefsd %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vscalefsd %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0xf5,0x8a,0x2d,0xc9]
+vscalefsd %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vscalefss 256(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x72,0x05,0x08,0x2d,0x7c,0x82,0x40]
+vscalefss 256(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vscalefss -256(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x72,0x05,0x08,0x2d,0x7c,0x82,0xc0]
+vscalefss -256(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vscalefss 256(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x05,0x0a,0x2d,0x7c,0x82,0x40]
+vscalefss 256(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+
+// CHECK: vscalefss -256(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x05,0x0a,0x2d,0x7c,0x82,0xc0]
+vscalefss -256(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+
+// CHECK: vscalefss 256(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x05,0x8a,0x2d,0x7c,0x82,0x40]
+vscalefss 256(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vscalefss -256(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x05,0x8a,0x2d,0x7c,0x82,0xc0]
+vscalefss -256(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vscalefss 256(%rdx,%rax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x08,0x2d,0x4c,0x82,0x40]
+vscalefss 256(%rdx,%rax,4), %xmm1, %xmm1
+
+// CHECK: vscalefss -256(%rdx,%rax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x08,0x2d,0x4c,0x82,0xc0]
+vscalefss -256(%rdx,%rax,4), %xmm1, %xmm1
+
+// CHECK: vscalefss 256(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0x2d,0x4c,0x82,0x40]
+vscalefss 256(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vscalefss -256(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0x2d,0x4c,0x82,0xc0]
+vscalefss -256(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vscalefss 256(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0x2d,0x4c,0x82,0x40]
+vscalefss 256(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vscalefss -256(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0x2d,0x4c,0x82,0xc0]
+vscalefss -256(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vscalefss 256(%rdx,%rax), %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x72,0x05,0x08,0x2d,0x7c,0x02,0x40]
+vscalefss 256(%rdx,%rax), %xmm15, %xmm15
+
+// CHECK: vscalefss 256(%rdx,%rax), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x05,0x0a,0x2d,0x7c,0x02,0x40]
+vscalefss 256(%rdx,%rax), %xmm15, %xmm15 {%k2}
+
+// CHECK: vscalefss 256(%rdx,%rax), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x05,0x8a,0x2d,0x7c,0x02,0x40]
+vscalefss 256(%rdx,%rax), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vscalefss 256(%rdx,%rax), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x08,0x2d,0x4c,0x02,0x40]
+vscalefss 256(%rdx,%rax), %xmm1, %xmm1
+
+// CHECK: vscalefss 256(%rdx,%rax), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0x2d,0x4c,0x02,0x40]
+vscalefss 256(%rdx,%rax), %xmm1, %xmm1 {%k2}
+
+// CHECK: vscalefss 256(%rdx,%rax), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0x2d,0x4c,0x02,0x40]
+vscalefss 256(%rdx,%rax), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vscalefss 256(%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x72,0x05,0x08,0x2d,0x7a,0x40]
+vscalefss 256(%rdx), %xmm15, %xmm15
+
+// CHECK: vscalefss 256(%rdx), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x05,0x0a,0x2d,0x7a,0x40]
+vscalefss 256(%rdx), %xmm15, %xmm15 {%k2}
+
+// CHECK: vscalefss 256(%rdx), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x05,0x8a,0x2d,0x7a,0x40]
+vscalefss 256(%rdx), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vscalefss 256(%rdx), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x08,0x2d,0x4a,0x40]
+vscalefss 256(%rdx), %xmm1, %xmm1
+
+// CHECK: vscalefss 256(%rdx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0x2d,0x4a,0x40]
+vscalefss 256(%rdx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vscalefss 256(%rdx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0x2d,0x4a,0x40]
+vscalefss 256(%rdx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vscalefss 485498096, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x72,0x05,0x08,0x2d,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vscalefss 485498096, %xmm15, %xmm15
+
+// CHECK: vscalefss 485498096, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x05,0x0a,0x2d,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vscalefss 485498096, %xmm15, %xmm15 {%k2}
+
+// CHECK: vscalefss 485498096, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x05,0x8a,0x2d,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vscalefss 485498096, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vscalefss 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x08,0x2d,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vscalefss 485498096, %xmm1, %xmm1
+
+// CHECK: vscalefss 485498096, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0x2d,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vscalefss 485498096, %xmm1, %xmm1 {%k2}
+
+// CHECK: vscalefss 485498096, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0x2d,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vscalefss 485498096, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vscalefss {rd-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x52,0x05,0x38,0x2d,0xff]
+vscalefss {rd-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vscalefss {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x05,0x3a,0x2d,0xff]
+vscalefss {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vscalefss {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x05,0xba,0x2d,0xff]
+vscalefss {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vscalefss {rd-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x38,0x2d,0xc9]
+vscalefss {rd-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vscalefss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x3a,0x2d,0xc9]
+vscalefss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vscalefss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0xba,0x2d,0xc9]
+vscalefss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vscalefss (%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x72,0x05,0x08,0x2d,0x3a]
+vscalefss (%rdx), %xmm15, %xmm15
+
+// CHECK: vscalefss (%rdx), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x72,0x05,0x0a,0x2d,0x3a]
+vscalefss (%rdx), %xmm15, %xmm15 {%k2}
+
+// CHECK: vscalefss (%rdx), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x72,0x05,0x8a,0x2d,0x3a]
+vscalefss (%rdx), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vscalefss (%rdx), %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x08,0x2d,0x0a]
+vscalefss (%rdx), %xmm1, %xmm1
+
+// CHECK: vscalefss (%rdx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0x2d,0x0a]
+vscalefss (%rdx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vscalefss (%rdx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0x2d,0x0a]
+vscalefss (%rdx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vscalefss {rn-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x52,0x05,0x18,0x2d,0xff]
+vscalefss {rn-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vscalefss {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x05,0x1a,0x2d,0xff]
+vscalefss {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vscalefss {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x05,0x9a,0x2d,0xff]
+vscalefss {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vscalefss {rn-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x18,0x2d,0xc9]
+vscalefss {rn-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vscalefss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x1a,0x2d,0xc9]
+vscalefss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vscalefss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x9a,0x2d,0xc9]
+vscalefss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vscalefss {ru-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x52,0x05,0x58,0x2d,0xff]
+vscalefss {ru-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vscalefss {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x05,0x5a,0x2d,0xff]
+vscalefss {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vscalefss {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x05,0xda,0x2d,0xff]
+vscalefss {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vscalefss {ru-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x58,0x2d,0xc9]
+vscalefss {ru-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vscalefss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x5a,0x2d,0xc9]
+vscalefss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vscalefss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0xda,0x2d,0xc9]
+vscalefss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vscalefss {rz-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x52,0x05,0x78,0x2d,0xff]
+vscalefss {rz-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vscalefss {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x05,0x7a,0x2d,0xff]
+vscalefss {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vscalefss {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x05,0xfa,0x2d,0xff]
+vscalefss {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vscalefss {rz-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x78,0x2d,0xc9]
+vscalefss {rz-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vscalefss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x7a,0x2d,0xc9]
+vscalefss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vscalefss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0xfa,0x2d,0xc9]
+vscalefss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vscalefss %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x52,0x05,0x08,0x2d,0xff]
+vscalefss %xmm15, %xmm15, %xmm15
+
+// CHECK: vscalefss %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x52,0x05,0x0a,0x2d,0xff]
+vscalefss %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vscalefss %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x52,0x05,0x8a,0x2d,0xff]
+vscalefss %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vscalefss %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf2,0x75,0x08,0x2d,0xc9]
+vscalefss %xmm1, %xmm1, %xmm1
+
+// CHECK: vscalefss %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf2,0x75,0x0a,0x2d,0xc9]
+vscalefss %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vscalefss %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf2,0x75,0x8a,0x2d,0xc9]
+vscalefss %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vsqrtsd 485498096, %xmm15, %xmm15
+// CHECK: encoding: [0xc5,0x03,0x51,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vsqrtsd 485498096, %xmm15, %xmm15
+
+// CHECK: vsqrtsd 485498096, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x71,0x87,0x0a,0x51,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vsqrtsd 485498096, %xmm15, %xmm15 {%k2}
+
+// CHECK: vsqrtsd 485498096, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x71,0x87,0x8a,0x51,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vsqrtsd 485498096, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vsqrtsd 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x51,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vsqrtsd 485498096, %xmm1, %xmm1
+
+// CHECK: vsqrtsd 485498096, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0x51,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vsqrtsd 485498096, %xmm1, %xmm1 {%k2}
+
+// CHECK: vsqrtsd 485498096, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x8a,0x51,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vsqrtsd 485498096, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vsqrtsd 512(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0xc5,0x03,0x51,0xbc,0x82,0x00,0x02,0x00,0x00]
+vsqrtsd 512(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vsqrtsd -512(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0xc5,0x03,0x51,0xbc,0x82,0x00,0xfe,0xff,0xff]
+vsqrtsd -512(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vsqrtsd 512(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x71,0x87,0x0a,0x51,0x7c,0x82,0x40]
+vsqrtsd 512(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+
+// CHECK: vsqrtsd -512(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x71,0x87,0x0a,0x51,0x7c,0x82,0xc0]
+vsqrtsd -512(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+
+// CHECK: vsqrtsd 512(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x71,0x87,0x8a,0x51,0x7c,0x82,0x40]
+vsqrtsd 512(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vsqrtsd -512(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x71,0x87,0x8a,0x51,0x7c,0x82,0xc0]
+vsqrtsd -512(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vsqrtsd 512(%rdx,%rax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x51,0x8c,0x82,0x00,0x02,0x00,0x00]
+vsqrtsd 512(%rdx,%rax,4), %xmm1, %xmm1
+
+// CHECK: vsqrtsd -512(%rdx,%rax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x51,0x8c,0x82,0x00,0xfe,0xff,0xff]
+vsqrtsd -512(%rdx,%rax,4), %xmm1, %xmm1
+
+// CHECK: vsqrtsd 512(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0x51,0x4c,0x82,0x40]
+vsqrtsd 512(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vsqrtsd -512(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0x51,0x4c,0x82,0xc0]
+vsqrtsd -512(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vsqrtsd 512(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x8a,0x51,0x4c,0x82,0x40]
+vsqrtsd 512(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vsqrtsd -512(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x8a,0x51,0x4c,0x82,0xc0]
+vsqrtsd -512(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vsqrtsd 512(%rdx,%rax), %xmm15, %xmm15
+// CHECK: encoding: [0xc5,0x03,0x51,0xbc,0x02,0x00,0x02,0x00,0x00]
+vsqrtsd 512(%rdx,%rax), %xmm15, %xmm15
+
+// CHECK: vsqrtsd 512(%rdx,%rax), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x71,0x87,0x0a,0x51,0x7c,0x02,0x40]
+vsqrtsd 512(%rdx,%rax), %xmm15, %xmm15 {%k2}
+
+// CHECK: vsqrtsd 512(%rdx,%rax), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x71,0x87,0x8a,0x51,0x7c,0x02,0x40]
+vsqrtsd 512(%rdx,%rax), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vsqrtsd 512(%rdx,%rax), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x51,0x8c,0x02,0x00,0x02,0x00,0x00]
+vsqrtsd 512(%rdx,%rax), %xmm1, %xmm1
+
+// CHECK: vsqrtsd 512(%rdx,%rax), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0x51,0x4c,0x02,0x40]
+vsqrtsd 512(%rdx,%rax), %xmm1, %xmm1 {%k2}
+
+// CHECK: vsqrtsd 512(%rdx,%rax), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x8a,0x51,0x4c,0x02,0x40]
+vsqrtsd 512(%rdx,%rax), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vsqrtsd 512(%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0xc5,0x03,0x51,0xba,0x00,0x02,0x00,0x00]
+vsqrtsd 512(%rdx), %xmm15, %xmm15
+
+// CHECK: vsqrtsd 512(%rdx), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x71,0x87,0x0a,0x51,0x7a,0x40]
+vsqrtsd 512(%rdx), %xmm15, %xmm15 {%k2}
+
+// CHECK: vsqrtsd 512(%rdx), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x71,0x87,0x8a,0x51,0x7a,0x40]
+vsqrtsd 512(%rdx), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vsqrtsd 512(%rdx), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x51,0x8a,0x00,0x02,0x00,0x00]
+vsqrtsd 512(%rdx), %xmm1, %xmm1
+
+// CHECK: vsqrtsd 512(%rdx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0x51,0x4a,0x40]
+vsqrtsd 512(%rdx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vsqrtsd 512(%rdx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x8a,0x51,0x4a,0x40]
+vsqrtsd 512(%rdx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vsqrtsd {rd-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x51,0x87,0x38,0x51,0xff]
+vsqrtsd {rd-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vsqrtsd {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x51,0x87,0x3a,0x51,0xff]
+vsqrtsd {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vsqrtsd {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x51,0x87,0xba,0x51,0xff]
+vsqrtsd {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vsqrtsd {rd-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0xf7,0x38,0x51,0xc9]
+vsqrtsd {rd-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vsqrtsd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x3a,0x51,0xc9]
+vsqrtsd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vsqrtsd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0xba,0x51,0xc9]
+vsqrtsd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vsqrtsd (%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0xc5,0x03,0x51,0x3a]
+vsqrtsd (%rdx), %xmm15, %xmm15
+
+// CHECK: vsqrtsd (%rdx), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x71,0x87,0x0a,0x51,0x3a]
+vsqrtsd (%rdx), %xmm15, %xmm15 {%k2}
+
+// CHECK: vsqrtsd (%rdx), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x71,0x87,0x8a,0x51,0x3a]
+vsqrtsd (%rdx), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vsqrtsd (%rdx), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x51,0x0a]
+vsqrtsd (%rdx), %xmm1, %xmm1
+
+// CHECK: vsqrtsd (%rdx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0x51,0x0a]
+vsqrtsd (%rdx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vsqrtsd (%rdx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x8a,0x51,0x0a]
+vsqrtsd (%rdx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vsqrtsd {rn-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x51,0x87,0x18,0x51,0xff]
+vsqrtsd {rn-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vsqrtsd {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x51,0x87,0x1a,0x51,0xff]
+vsqrtsd {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vsqrtsd {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x51,0x87,0x9a,0x51,0xff]
+vsqrtsd {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vsqrtsd {rn-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0xf7,0x18,0x51,0xc9]
+vsqrtsd {rn-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vsqrtsd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x1a,0x51,0xc9]
+vsqrtsd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vsqrtsd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x9a,0x51,0xc9]
+vsqrtsd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vsqrtsd {ru-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x51,0x87,0x58,0x51,0xff]
+vsqrtsd {ru-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vsqrtsd {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x51,0x87,0x5a,0x51,0xff]
+vsqrtsd {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vsqrtsd {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x51,0x87,0xda,0x51,0xff]
+vsqrtsd {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vsqrtsd {ru-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0xf7,0x58,0x51,0xc9]
+vsqrtsd {ru-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vsqrtsd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x5a,0x51,0xc9]
+vsqrtsd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vsqrtsd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0xda,0x51,0xc9]
+vsqrtsd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vsqrtsd {rz-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x51,0x87,0x78,0x51,0xff]
+vsqrtsd {rz-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vsqrtsd {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x51,0x87,0x7a,0x51,0xff]
+vsqrtsd {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vsqrtsd {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x51,0x87,0xfa,0x51,0xff]
+vsqrtsd {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vsqrtsd {rz-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0xf7,0x78,0x51,0xc9]
+vsqrtsd {rz-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vsqrtsd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x7a,0x51,0xc9]
+vsqrtsd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vsqrtsd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0xfa,0x51,0xc9]
+vsqrtsd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vsqrtsd %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x41,0x03,0x51,0xff]
+vsqrtsd %xmm15, %xmm15, %xmm15
+
+// CHECK: vsqrtsd %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x51,0x87,0x0a,0x51,0xff]
+vsqrtsd %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vsqrtsd %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x51,0x87,0x8a,0x51,0xff]
+vsqrtsd %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vsqrtsd %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x51,0xc9]
+vsqrtsd %xmm1, %xmm1, %xmm1
+
+// CHECK: vsqrtsd %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0x51,0xc9]
+vsqrtsd %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vsqrtsd %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x8a,0x51,0xc9]
+vsqrtsd %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vsqrtss 256(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0xc5,0x02,0x51,0xbc,0x82,0x00,0x01,0x00,0x00]
+vsqrtss 256(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vsqrtss -256(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0xc5,0x02,0x51,0xbc,0x82,0x00,0xff,0xff,0xff]
+vsqrtss -256(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vsqrtss 256(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x71,0x06,0x0a,0x51,0x7c,0x82,0x40]
+vsqrtss 256(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+
+// CHECK: vsqrtss -256(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x71,0x06,0x0a,0x51,0x7c,0x82,0xc0]
+vsqrtss -256(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+
+// CHECK: vsqrtss 256(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x71,0x06,0x8a,0x51,0x7c,0x82,0x40]
+vsqrtss 256(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vsqrtss -256(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x71,0x06,0x8a,0x51,0x7c,0x82,0xc0]
+vsqrtss -256(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vsqrtss 256(%rdx,%rax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x51,0x8c,0x82,0x00,0x01,0x00,0x00]
+vsqrtss 256(%rdx,%rax,4), %xmm1, %xmm1
+
+// CHECK: vsqrtss -256(%rdx,%rax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x51,0x8c,0x82,0x00,0xff,0xff,0xff]
+vsqrtss -256(%rdx,%rax,4), %xmm1, %xmm1
+
+// CHECK: vsqrtss 256(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0x51,0x4c,0x82,0x40]
+vsqrtss 256(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vsqrtss -256(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0x51,0x4c,0x82,0xc0]
+vsqrtss -256(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vsqrtss 256(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x8a,0x51,0x4c,0x82,0x40]
+vsqrtss 256(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vsqrtss -256(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x8a,0x51,0x4c,0x82,0xc0]
+vsqrtss -256(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vsqrtss 256(%rdx,%rax), %xmm15, %xmm15
+// CHECK: encoding: [0xc5,0x02,0x51,0xbc,0x02,0x00,0x01,0x00,0x00]
+vsqrtss 256(%rdx,%rax), %xmm15, %xmm15
+
+// CHECK: vsqrtss 256(%rdx,%rax), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x71,0x06,0x0a,0x51,0x7c,0x02,0x40]
+vsqrtss 256(%rdx,%rax), %xmm15, %xmm15 {%k2}
+
+// CHECK: vsqrtss 256(%rdx,%rax), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x71,0x06,0x8a,0x51,0x7c,0x02,0x40]
+vsqrtss 256(%rdx,%rax), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vsqrtss 256(%rdx,%rax), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x51,0x8c,0x02,0x00,0x01,0x00,0x00]
+vsqrtss 256(%rdx,%rax), %xmm1, %xmm1
+
+// CHECK: vsqrtss 256(%rdx,%rax), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0x51,0x4c,0x02,0x40]
+vsqrtss 256(%rdx,%rax), %xmm1, %xmm1 {%k2}
+
+// CHECK: vsqrtss 256(%rdx,%rax), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x8a,0x51,0x4c,0x02,0x40]
+vsqrtss 256(%rdx,%rax), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vsqrtss 256(%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0xc5,0x02,0x51,0xba,0x00,0x01,0x00,0x00]
+vsqrtss 256(%rdx), %xmm15, %xmm15
+
+// CHECK: vsqrtss 256(%rdx), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x71,0x06,0x0a,0x51,0x7a,0x40]
+vsqrtss 256(%rdx), %xmm15, %xmm15 {%k2}
+
+// CHECK: vsqrtss 256(%rdx), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x71,0x06,0x8a,0x51,0x7a,0x40]
+vsqrtss 256(%rdx), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vsqrtss 256(%rdx), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x51,0x8a,0x00,0x01,0x00,0x00]
+vsqrtss 256(%rdx), %xmm1, %xmm1
+
+// CHECK: vsqrtss 256(%rdx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0x51,0x4a,0x40]
+vsqrtss 256(%rdx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vsqrtss 256(%rdx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x8a,0x51,0x4a,0x40]
+vsqrtss 256(%rdx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vsqrtss 485498096, %xmm15, %xmm15
+// CHECK: encoding: [0xc5,0x02,0x51,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vsqrtss 485498096, %xmm15, %xmm15
+
+// CHECK: vsqrtss 485498096, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x71,0x06,0x0a,0x51,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vsqrtss 485498096, %xmm15, %xmm15 {%k2}
+
+// CHECK: vsqrtss 485498096, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x71,0x06,0x8a,0x51,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vsqrtss 485498096, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vsqrtss 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x51,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vsqrtss 485498096, %xmm1, %xmm1
+
+// CHECK: vsqrtss 485498096, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0x51,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vsqrtss 485498096, %xmm1, %xmm1 {%k2}
+
+// CHECK: vsqrtss 485498096, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x8a,0x51,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vsqrtss 485498096, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vsqrtss {rd-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x51,0x06,0x38,0x51,0xff]
+vsqrtss {rd-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vsqrtss {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x51,0x06,0x3a,0x51,0xff]
+vsqrtss {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vsqrtss {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x51,0x06,0xba,0x51,0xff]
+vsqrtss {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vsqrtss {rd-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0x76,0x38,0x51,0xc9]
+vsqrtss {rd-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vsqrtss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x3a,0x51,0xc9]
+vsqrtss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vsqrtss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0xba,0x51,0xc9]
+vsqrtss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vsqrtss (%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0xc5,0x02,0x51,0x3a]
+vsqrtss (%rdx), %xmm15, %xmm15
+
+// CHECK: vsqrtss (%rdx), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x71,0x06,0x0a,0x51,0x3a]
+vsqrtss (%rdx), %xmm15, %xmm15 {%k2}
+
+// CHECK: vsqrtss (%rdx), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x71,0x06,0x8a,0x51,0x3a]
+vsqrtss (%rdx), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vsqrtss (%rdx), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x51,0x0a]
+vsqrtss (%rdx), %xmm1, %xmm1
+
+// CHECK: vsqrtss (%rdx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0x51,0x0a]
+vsqrtss (%rdx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vsqrtss (%rdx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x8a,0x51,0x0a]
+vsqrtss (%rdx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vsqrtss {rn-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x51,0x06,0x18,0x51,0xff]
+vsqrtss {rn-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vsqrtss {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x51,0x06,0x1a,0x51,0xff]
+vsqrtss {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vsqrtss {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x51,0x06,0x9a,0x51,0xff]
+vsqrtss {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vsqrtss {rn-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0x76,0x18,0x51,0xc9]
+vsqrtss {rn-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vsqrtss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x1a,0x51,0xc9]
+vsqrtss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vsqrtss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x9a,0x51,0xc9]
+vsqrtss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vsqrtss {ru-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x51,0x06,0x58,0x51,0xff]
+vsqrtss {ru-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vsqrtss {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x51,0x06,0x5a,0x51,0xff]
+vsqrtss {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vsqrtss {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x51,0x06,0xda,0x51,0xff]
+vsqrtss {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vsqrtss {ru-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0x76,0x58,0x51,0xc9]
+vsqrtss {ru-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vsqrtss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x5a,0x51,0xc9]
+vsqrtss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vsqrtss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0xda,0x51,0xc9]
+vsqrtss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vsqrtss {rz-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x51,0x06,0x78,0x51,0xff]
+vsqrtss {rz-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vsqrtss {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x51,0x06,0x7a,0x51,0xff]
+vsqrtss {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vsqrtss {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x51,0x06,0xfa,0x51,0xff]
+vsqrtss {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vsqrtss {rz-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0x76,0x78,0x51,0xc9]
+vsqrtss {rz-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vsqrtss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x7a,0x51,0xc9]
+vsqrtss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vsqrtss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0xfa,0x51,0xc9]
+vsqrtss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vsqrtss %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x41,0x02,0x51,0xff]
+vsqrtss %xmm15, %xmm15, %xmm15
+
+// CHECK: vsqrtss %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x51,0x06,0x0a,0x51,0xff]
+vsqrtss %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vsqrtss %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x51,0x06,0x8a,0x51,0xff]
+vsqrtss %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vsqrtss %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x51,0xc9]
+vsqrtss %xmm1, %xmm1, %xmm1
+
+// CHECK: vsqrtss %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0x51,0xc9]
+vsqrtss %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vsqrtss %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x8a,0x51,0xc9]
+vsqrtss %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vsubsd 485498096, %xmm15, %xmm15
+// CHECK: encoding: [0xc5,0x03,0x5c,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vsubsd 485498096, %xmm15, %xmm15
+
+// CHECK: vsubsd 485498096, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x71,0x87,0x0a,0x5c,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vsubsd 485498096, %xmm15, %xmm15 {%k2}
+
+// CHECK: vsubsd 485498096, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x71,0x87,0x8a,0x5c,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vsubsd 485498096, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vsubsd 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x5c,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vsubsd 485498096, %xmm1, %xmm1
+
+// CHECK: vsubsd 485498096, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0x5c,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vsubsd 485498096, %xmm1, %xmm1 {%k2}
+
+// CHECK: vsubsd 485498096, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x8a,0x5c,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vsubsd 485498096, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vsubsd 512(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0xc5,0x03,0x5c,0xbc,0x82,0x00,0x02,0x00,0x00]
+vsubsd 512(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vsubsd -512(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0xc5,0x03,0x5c,0xbc,0x82,0x00,0xfe,0xff,0xff]
+vsubsd -512(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vsubsd 512(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x71,0x87,0x0a,0x5c,0x7c,0x82,0x40]
+vsubsd 512(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+
+// CHECK: vsubsd -512(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x71,0x87,0x0a,0x5c,0x7c,0x82,0xc0]
+vsubsd -512(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+
+// CHECK: vsubsd 512(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x71,0x87,0x8a,0x5c,0x7c,0x82,0x40]
+vsubsd 512(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vsubsd -512(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x71,0x87,0x8a,0x5c,0x7c,0x82,0xc0]
+vsubsd -512(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vsubsd 512(%rdx,%rax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x5c,0x8c,0x82,0x00,0x02,0x00,0x00]
+vsubsd 512(%rdx,%rax,4), %xmm1, %xmm1
+
+// CHECK: vsubsd -512(%rdx,%rax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x5c,0x8c,0x82,0x00,0xfe,0xff,0xff]
+vsubsd -512(%rdx,%rax,4), %xmm1, %xmm1
+
+// CHECK: vsubsd 512(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0x5c,0x4c,0x82,0x40]
+vsubsd 512(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vsubsd -512(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0x5c,0x4c,0x82,0xc0]
+vsubsd -512(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vsubsd 512(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x8a,0x5c,0x4c,0x82,0x40]
+vsubsd 512(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vsubsd -512(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x8a,0x5c,0x4c,0x82,0xc0]
+vsubsd -512(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vsubsd 512(%rdx,%rax), %xmm15, %xmm15
+// CHECK: encoding: [0xc5,0x03,0x5c,0xbc,0x02,0x00,0x02,0x00,0x00]
+vsubsd 512(%rdx,%rax), %xmm15, %xmm15
+
+// CHECK: vsubsd 512(%rdx,%rax), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x71,0x87,0x0a,0x5c,0x7c,0x02,0x40]
+vsubsd 512(%rdx,%rax), %xmm15, %xmm15 {%k2}
+
+// CHECK: vsubsd 512(%rdx,%rax), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x71,0x87,0x8a,0x5c,0x7c,0x02,0x40]
+vsubsd 512(%rdx,%rax), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vsubsd 512(%rdx,%rax), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x5c,0x8c,0x02,0x00,0x02,0x00,0x00]
+vsubsd 512(%rdx,%rax), %xmm1, %xmm1
+
+// CHECK: vsubsd 512(%rdx,%rax), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0x5c,0x4c,0x02,0x40]
+vsubsd 512(%rdx,%rax), %xmm1, %xmm1 {%k2}
+
+// CHECK: vsubsd 512(%rdx,%rax), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x8a,0x5c,0x4c,0x02,0x40]
+vsubsd 512(%rdx,%rax), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vsubsd 512(%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0xc5,0x03,0x5c,0xba,0x00,0x02,0x00,0x00]
+vsubsd 512(%rdx), %xmm15, %xmm15
+
+// CHECK: vsubsd 512(%rdx), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x71,0x87,0x0a,0x5c,0x7a,0x40]
+vsubsd 512(%rdx), %xmm15, %xmm15 {%k2}
+
+// CHECK: vsubsd 512(%rdx), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x71,0x87,0x8a,0x5c,0x7a,0x40]
+vsubsd 512(%rdx), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vsubsd 512(%rdx), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x5c,0x8a,0x00,0x02,0x00,0x00]
+vsubsd 512(%rdx), %xmm1, %xmm1
+
+// CHECK: vsubsd 512(%rdx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0x5c,0x4a,0x40]
+vsubsd 512(%rdx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vsubsd 512(%rdx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x8a,0x5c,0x4a,0x40]
+vsubsd 512(%rdx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vsubsd {rd-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x51,0x87,0x38,0x5c,0xff]
+vsubsd {rd-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vsubsd {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x51,0x87,0x3a,0x5c,0xff]
+vsubsd {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vsubsd {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x51,0x87,0xba,0x5c,0xff]
+vsubsd {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vsubsd {rd-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0xf7,0x38,0x5c,0xc9]
+vsubsd {rd-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vsubsd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x3a,0x5c,0xc9]
+vsubsd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vsubsd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0xba,0x5c,0xc9]
+vsubsd {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vsubsd (%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0xc5,0x03,0x5c,0x3a]
+vsubsd (%rdx), %xmm15, %xmm15
+
+// CHECK: vsubsd (%rdx), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x71,0x87,0x0a,0x5c,0x3a]
+vsubsd (%rdx), %xmm15, %xmm15 {%k2}
+
+// CHECK: vsubsd (%rdx), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x71,0x87,0x8a,0x5c,0x3a]
+vsubsd (%rdx), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vsubsd (%rdx), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x5c,0x0a]
+vsubsd (%rdx), %xmm1, %xmm1
+
+// CHECK: vsubsd (%rdx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0x5c,0x0a]
+vsubsd (%rdx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vsubsd (%rdx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x8a,0x5c,0x0a]
+vsubsd (%rdx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vsubsd {rn-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x51,0x87,0x18,0x5c,0xff]
+vsubsd {rn-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vsubsd {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x51,0x87,0x1a,0x5c,0xff]
+vsubsd {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vsubsd {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x51,0x87,0x9a,0x5c,0xff]
+vsubsd {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vsubsd {rn-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0xf7,0x18,0x5c,0xc9]
+vsubsd {rn-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vsubsd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x1a,0x5c,0xc9]
+vsubsd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vsubsd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x9a,0x5c,0xc9]
+vsubsd {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vsubsd {ru-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x51,0x87,0x58,0x5c,0xff]
+vsubsd {ru-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vsubsd {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x51,0x87,0x5a,0x5c,0xff]
+vsubsd {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vsubsd {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x51,0x87,0xda,0x5c,0xff]
+vsubsd {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vsubsd {ru-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0xf7,0x58,0x5c,0xc9]
+vsubsd {ru-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vsubsd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x5a,0x5c,0xc9]
+vsubsd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vsubsd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0xda,0x5c,0xc9]
+vsubsd {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vsubsd {rz-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x51,0x87,0x78,0x5c,0xff]
+vsubsd {rz-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vsubsd {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x51,0x87,0x7a,0x5c,0xff]
+vsubsd {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vsubsd {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x51,0x87,0xfa,0x5c,0xff]
+vsubsd {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vsubsd {rz-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0xf7,0x78,0x5c,0xc9]
+vsubsd {rz-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vsubsd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x7a,0x5c,0xc9]
+vsubsd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vsubsd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0xfa,0x5c,0xc9]
+vsubsd {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vsubsd %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x41,0x03,0x5c,0xff]
+vsubsd %xmm15, %xmm15, %xmm15
+
+// CHECK: vsubsd %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x51,0x87,0x0a,0x5c,0xff]
+vsubsd %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vsubsd %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x51,0x87,0x8a,0x5c,0xff]
+vsubsd %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vsubsd %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf3,0x5c,0xc9]
+vsubsd %xmm1, %xmm1, %xmm1
+
+// CHECK: vsubsd %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x0a,0x5c,0xc9]
+vsubsd %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vsubsd %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0xf7,0x8a,0x5c,0xc9]
+vsubsd %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vsubss 256(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0xc5,0x02,0x5c,0xbc,0x82,0x00,0x01,0x00,0x00]
+vsubss 256(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vsubss -256(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0xc5,0x02,0x5c,0xbc,0x82,0x00,0xff,0xff,0xff]
+vsubss -256(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vsubss 256(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x71,0x06,0x0a,0x5c,0x7c,0x82,0x40]
+vsubss 256(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+
+// CHECK: vsubss -256(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x71,0x06,0x0a,0x5c,0x7c,0x82,0xc0]
+vsubss -256(%rdx,%rax,4), %xmm15, %xmm15 {%k2}
+
+// CHECK: vsubss 256(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x71,0x06,0x8a,0x5c,0x7c,0x82,0x40]
+vsubss 256(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vsubss -256(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x71,0x06,0x8a,0x5c,0x7c,0x82,0xc0]
+vsubss -256(%rdx,%rax,4), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vsubss 256(%rdx,%rax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x5c,0x8c,0x82,0x00,0x01,0x00,0x00]
+vsubss 256(%rdx,%rax,4), %xmm1, %xmm1
+
+// CHECK: vsubss -256(%rdx,%rax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x5c,0x8c,0x82,0x00,0xff,0xff,0xff]
+vsubss -256(%rdx,%rax,4), %xmm1, %xmm1
+
+// CHECK: vsubss 256(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0x5c,0x4c,0x82,0x40]
+vsubss 256(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vsubss -256(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0x5c,0x4c,0x82,0xc0]
+vsubss -256(%rdx,%rax,4), %xmm1, %xmm1 {%k2}
+
+// CHECK: vsubss 256(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x8a,0x5c,0x4c,0x82,0x40]
+vsubss 256(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vsubss -256(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x8a,0x5c,0x4c,0x82,0xc0]
+vsubss -256(%rdx,%rax,4), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vsubss 256(%rdx,%rax), %xmm15, %xmm15
+// CHECK: encoding: [0xc5,0x02,0x5c,0xbc,0x02,0x00,0x01,0x00,0x00]
+vsubss 256(%rdx,%rax), %xmm15, %xmm15
+
+// CHECK: vsubss 256(%rdx,%rax), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x71,0x06,0x0a,0x5c,0x7c,0x02,0x40]
+vsubss 256(%rdx,%rax), %xmm15, %xmm15 {%k2}
+
+// CHECK: vsubss 256(%rdx,%rax), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x71,0x06,0x8a,0x5c,0x7c,0x02,0x40]
+vsubss 256(%rdx,%rax), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vsubss 256(%rdx,%rax), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x5c,0x8c,0x02,0x00,0x01,0x00,0x00]
+vsubss 256(%rdx,%rax), %xmm1, %xmm1
+
+// CHECK: vsubss 256(%rdx,%rax), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0x5c,0x4c,0x02,0x40]
+vsubss 256(%rdx,%rax), %xmm1, %xmm1 {%k2}
+
+// CHECK: vsubss 256(%rdx,%rax), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x8a,0x5c,0x4c,0x02,0x40]
+vsubss 256(%rdx,%rax), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vsubss 256(%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0xc5,0x02,0x5c,0xba,0x00,0x01,0x00,0x00]
+vsubss 256(%rdx), %xmm15, %xmm15
+
+// CHECK: vsubss 256(%rdx), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x71,0x06,0x0a,0x5c,0x7a,0x40]
+vsubss 256(%rdx), %xmm15, %xmm15 {%k2}
+
+// CHECK: vsubss 256(%rdx), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x71,0x06,0x8a,0x5c,0x7a,0x40]
+vsubss 256(%rdx), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vsubss 256(%rdx), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x5c,0x8a,0x00,0x01,0x00,0x00]
+vsubss 256(%rdx), %xmm1, %xmm1
+
+// CHECK: vsubss 256(%rdx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0x5c,0x4a,0x40]
+vsubss 256(%rdx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vsubss 256(%rdx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x8a,0x5c,0x4a,0x40]
+vsubss 256(%rdx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vsubss 485498096, %xmm15, %xmm15
+// CHECK: encoding: [0xc5,0x02,0x5c,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vsubss 485498096, %xmm15, %xmm15
+
+// CHECK: vsubss 485498096, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x71,0x06,0x0a,0x5c,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vsubss 485498096, %xmm15, %xmm15 {%k2}
+
+// CHECK: vsubss 485498096, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x71,0x06,0x8a,0x5c,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vsubss 485498096, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vsubss 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x5c,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vsubss 485498096, %xmm1, %xmm1
+
+// CHECK: vsubss 485498096, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0x5c,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vsubss 485498096, %xmm1, %xmm1 {%k2}
+
+// CHECK: vsubss 485498096, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x8a,0x5c,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vsubss 485498096, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vsubss {rd-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x51,0x06,0x38,0x5c,0xff]
+vsubss {rd-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vsubss {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x51,0x06,0x3a,0x5c,0xff]
+vsubss {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vsubss {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x51,0x06,0xba,0x5c,0xff]
+vsubss {rd-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vsubss {rd-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0x76,0x38,0x5c,0xc9]
+vsubss {rd-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vsubss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x3a,0x5c,0xc9]
+vsubss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vsubss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0xba,0x5c,0xc9]
+vsubss {rd-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vsubss (%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0xc5,0x02,0x5c,0x3a]
+vsubss (%rdx), %xmm15, %xmm15
+
+// CHECK: vsubss (%rdx), %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x71,0x06,0x0a,0x5c,0x3a]
+vsubss (%rdx), %xmm15, %xmm15 {%k2}
+
+// CHECK: vsubss (%rdx), %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x71,0x06,0x8a,0x5c,0x3a]
+vsubss (%rdx), %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vsubss (%rdx), %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x5c,0x0a]
+vsubss (%rdx), %xmm1, %xmm1
+
+// CHECK: vsubss (%rdx), %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0x5c,0x0a]
+vsubss (%rdx), %xmm1, %xmm1 {%k2}
+
+// CHECK: vsubss (%rdx), %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x8a,0x5c,0x0a]
+vsubss (%rdx), %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vsubss {rn-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x51,0x06,0x18,0x5c,0xff]
+vsubss {rn-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vsubss {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x51,0x06,0x1a,0x5c,0xff]
+vsubss {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vsubss {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x51,0x06,0x9a,0x5c,0xff]
+vsubss {rn-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vsubss {rn-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0x76,0x18,0x5c,0xc9]
+vsubss {rn-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vsubss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x1a,0x5c,0xc9]
+vsubss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vsubss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x9a,0x5c,0xc9]
+vsubss {rn-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vsubss {ru-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x51,0x06,0x58,0x5c,0xff]
+vsubss {ru-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vsubss {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x51,0x06,0x5a,0x5c,0xff]
+vsubss {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vsubss {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x51,0x06,0xda,0x5c,0xff]
+vsubss {ru-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vsubss {ru-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0x76,0x58,0x5c,0xc9]
+vsubss {ru-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vsubss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x5a,0x5c,0xc9]
+vsubss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vsubss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0xda,0x5c,0xc9]
+vsubss {ru-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vsubss {rz-sae}, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x51,0x06,0x78,0x5c,0xff]
+vsubss {rz-sae}, %xmm15, %xmm15, %xmm15
+
+// CHECK: vsubss {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x51,0x06,0x7a,0x5c,0xff]
+vsubss {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vsubss {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x51,0x06,0xfa,0x5c,0xff]
+vsubss {rz-sae}, %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vsubss {rz-sae}, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0x76,0x78,0x5c,0xc9]
+vsubss {rz-sae}, %xmm1, %xmm1, %xmm1
+
+// CHECK: vsubss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x7a,0x5c,0xc9]
+vsubss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vsubss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0xfa,0x5c,0xc9]
+vsubss {rz-sae}, %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vsubss %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x41,0x02,0x5c,0xff]
+vsubss %xmm15, %xmm15, %xmm15
+
+// CHECK: vsubss %xmm15, %xmm15, %xmm15 {%k2}
+// CHECK: encoding: [0x62,0x51,0x06,0x0a,0x5c,0xff]
+vsubss %xmm15, %xmm15, %xmm15 {%k2}
+
+// CHECK: vsubss %xmm15, %xmm15, %xmm15 {%k2} {z}
+// CHECK: encoding: [0x62,0x51,0x06,0x8a,0x5c,0xff]
+vsubss %xmm15, %xmm15, %xmm15 {%k2} {z}
+
+// CHECK: vsubss %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf2,0x5c,0xc9]
+vsubss %xmm1, %xmm1, %xmm1
+
+// CHECK: vsubss %xmm1, %xmm1, %xmm1 {%k2}
+// CHECK: encoding: [0x62,0xf1,0x76,0x0a,0x5c,0xc9]
+vsubss %xmm1, %xmm1, %xmm1 {%k2}
+
+// CHECK: vsubss %xmm1, %xmm1, %xmm1 {%k2} {z}
+// CHECK: encoding: [0x62,0xf1,0x76,0x8a,0x5c,0xc9]
+vsubss %xmm1, %xmm1, %xmm1 {%k2} {z}
+
+// CHECK: vucomisd 485498096, %xmm15
+// CHECK: encoding: [0xc5,0x79,0x2e,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vucomisd 485498096, %xmm15
+
+// CHECK: vucomisd 485498096, %xmm1
+// CHECK: encoding: [0xc5,0xf9,0x2e,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vucomisd 485498096, %xmm1
+
+// CHECK: vucomisd 512(%rdx,%rax,4), %xmm15
+// CHECK: encoding: [0xc5,0x79,0x2e,0xbc,0x82,0x00,0x02,0x00,0x00]
+vucomisd 512(%rdx,%rax,4), %xmm15
+
+// CHECK: vucomisd -512(%rdx,%rax,4), %xmm15
+// CHECK: encoding: [0xc5,0x79,0x2e,0xbc,0x82,0x00,0xfe,0xff,0xff]
+vucomisd -512(%rdx,%rax,4), %xmm15
+
+// CHECK: vucomisd 512(%rdx,%rax,4), %xmm1
+// CHECK: encoding: [0xc5,0xf9,0x2e,0x8c,0x82,0x00,0x02,0x00,0x00]
+vucomisd 512(%rdx,%rax,4), %xmm1
+
+// CHECK: vucomisd -512(%rdx,%rax,4), %xmm1
+// CHECK: encoding: [0xc5,0xf9,0x2e,0x8c,0x82,0x00,0xfe,0xff,0xff]
+vucomisd -512(%rdx,%rax,4), %xmm1
+
+// CHECK: vucomisd 512(%rdx,%rax), %xmm15
+// CHECK: encoding: [0xc5,0x79,0x2e,0xbc,0x02,0x00,0x02,0x00,0x00]
+vucomisd 512(%rdx,%rax), %xmm15
+
+// CHECK: vucomisd 512(%rdx,%rax), %xmm1
+// CHECK: encoding: [0xc5,0xf9,0x2e,0x8c,0x02,0x00,0x02,0x00,0x00]
+vucomisd 512(%rdx,%rax), %xmm1
+
+// CHECK: vucomisd 512(%rdx), %xmm15
+// CHECK: encoding: [0xc5,0x79,0x2e,0xba,0x00,0x02,0x00,0x00]
+vucomisd 512(%rdx), %xmm15
+
+// CHECK: vucomisd 512(%rdx), %xmm1
+// CHECK: encoding: [0xc5,0xf9,0x2e,0x8a,0x00,0x02,0x00,0x00]
+vucomisd 512(%rdx), %xmm1
+
+// CHECK: vucomisd (%rdx), %xmm15
+// CHECK: encoding: [0xc5,0x79,0x2e,0x3a]
+vucomisd (%rdx), %xmm15
+
+// CHECK: vucomisd (%rdx), %xmm1
+// CHECK: encoding: [0xc5,0xf9,0x2e,0x0a]
+vucomisd (%rdx), %xmm1
+
+// CHECK: vucomisd {sae}, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x51,0xfd,0x18,0x2e,0xff]
+vucomisd {sae}, %xmm15, %xmm15
+
+// CHECK: vucomisd {sae}, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0xfd,0x18,0x2e,0xc9]
+vucomisd {sae}, %xmm1, %xmm1
+
+// CHECK: vucomisd %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x41,0x79,0x2e,0xff]
+vucomisd %xmm15, %xmm15
+
+// CHECK: vucomisd %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf9,0x2e,0xc9]
+vucomisd %xmm1, %xmm1
+
+// CHECK: vucomiss 256(%rdx,%rax,4), %xmm15
+// CHECK: encoding: [0xc5,0x78,0x2e,0xbc,0x82,0x00,0x01,0x00,0x00]
+vucomiss 256(%rdx,%rax,4), %xmm15
+
+// CHECK: vucomiss -256(%rdx,%rax,4), %xmm15
+// CHECK: encoding: [0xc5,0x78,0x2e,0xbc,0x82,0x00,0xff,0xff,0xff]
+vucomiss -256(%rdx,%rax,4), %xmm15
+
+// CHECK: vucomiss 256(%rdx,%rax,4), %xmm1
+// CHECK: encoding: [0xc5,0xf8,0x2e,0x8c,0x82,0x00,0x01,0x00,0x00]
+vucomiss 256(%rdx,%rax,4), %xmm1
+
+// CHECK: vucomiss -256(%rdx,%rax,4), %xmm1
+// CHECK: encoding: [0xc5,0xf8,0x2e,0x8c,0x82,0x00,0xff,0xff,0xff]
+vucomiss -256(%rdx,%rax,4), %xmm1
+
+// CHECK: vucomiss 256(%rdx,%rax), %xmm15
+// CHECK: encoding: [0xc5,0x78,0x2e,0xbc,0x02,0x00,0x01,0x00,0x00]
+vucomiss 256(%rdx,%rax), %xmm15
+
+// CHECK: vucomiss 256(%rdx,%rax), %xmm1
+// CHECK: encoding: [0xc5,0xf8,0x2e,0x8c,0x02,0x00,0x01,0x00,0x00]
+vucomiss 256(%rdx,%rax), %xmm1
+
+// CHECK: vucomiss 256(%rdx), %xmm15
+// CHECK: encoding: [0xc5,0x78,0x2e,0xba,0x00,0x01,0x00,0x00]
+vucomiss 256(%rdx), %xmm15
+
+// CHECK: vucomiss 256(%rdx), %xmm1
+// CHECK: encoding: [0xc5,0xf8,0x2e,0x8a,0x00,0x01,0x00,0x00]
+vucomiss 256(%rdx), %xmm1
+
+// CHECK: vucomiss 485498096, %xmm15
+// CHECK: encoding: [0xc5,0x78,0x2e,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vucomiss 485498096, %xmm15
+
+// CHECK: vucomiss 485498096, %xmm1
+// CHECK: encoding: [0xc5,0xf8,0x2e,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vucomiss 485498096, %xmm1
+
+// CHECK: vucomiss (%rdx), %xmm15
+// CHECK: encoding: [0xc5,0x78,0x2e,0x3a]
+vucomiss (%rdx), %xmm15
+
+// CHECK: vucomiss (%rdx), %xmm1
+// CHECK: encoding: [0xc5,0xf8,0x2e,0x0a]
+vucomiss (%rdx), %xmm1
+
+// CHECK: vucomiss {sae}, %xmm15, %xmm15
+// CHECK: encoding: [0x62,0x51,0x7c,0x18,0x2e,0xff]
+vucomiss {sae}, %xmm15, %xmm15
+
+// CHECK: vucomiss {sae}, %xmm1, %xmm1
+// CHECK: encoding: [0x62,0xf1,0x7c,0x18,0x2e,0xc9]
+vucomiss {sae}, %xmm1, %xmm1
+
+// CHECK: vucomiss %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x41,0x78,0x2e,0xff]
+vucomiss %xmm15, %xmm15
+
+// CHECK: vucomiss %xmm1, %xmm1
+// CHECK: encoding: [0xc5,0xf8,0x2e,0xc9]
+vucomiss %xmm1, %xmm1
+
diff --git a/test/MC/X86/LWP-32.s b/test/MC/X86/LWP-32.s
new file mode 100644
index 0000000000000000000000000000000000000000..d91d16cc95f5d2938e53d4f50e8603ad56d36bd1
--- /dev/null
+++ b/test/MC/X86/LWP-32.s
@@ -0,0 +1,65 @@
+// RUN: llvm-mc -triple i386-unknown-unknown --show-encoding %s | FileCheck %s
+
+// CHECK: llwpcb %eax
+// CHECK: encoding: [0x8f,0xe9,0x78,0x12,0xc0]
+llwpcb %eax
+
+// CHECK: lwpins $0, -485498096(%edx,%eax,4), %edx
+// CHECK: encoding: [0x8f,0xea,0x68,0x12,0x84,0x82,0x10,0xe3,0x0f,0xe3,0x00,0x00,0x00,0x00]
+lwpins $0, -485498096(%edx,%eax,4), %edx
+
+// CHECK: lwpins $0, 485498096(%edx,%eax,4), %edx
+// CHECK: encoding: [0x8f,0xea,0x68,0x12,0x84,0x82,0xf0,0x1c,0xf0,0x1c,0x00,0x00,0x00,0x00]
+lwpins $0, 485498096(%edx,%eax,4), %edx
+
+// CHECK: lwpins $0, 485498096(%edx), %edx
+// CHECK: encoding: [0x8f,0xea,0x68,0x12,0x82,0xf0,0x1c,0xf0,0x1c,0x00,0x00,0x00,0x00]
+lwpins $0, 485498096(%edx), %edx
+
+// CHECK: lwpins $0, 485498096, %edx
+// CHECK: encoding: [0x8f,0xea,0x68,0x12,0x05,0xf0,0x1c,0xf0,0x1c,0x00,0x00,0x00,0x00]
+lwpins $0, 485498096, %edx
+
+// CHECK: lwpins $0, 64(%edx,%eax), %edx
+// CHECK: encoding: [0x8f,0xea,0x68,0x12,0x44,0x02,0x40,0x00,0x00,0x00,0x00]
+lwpins $0, 64(%edx,%eax), %edx
+
+// CHECK: lwpins $0, %eax, %edx
+// CHECK: encoding: [0x8f,0xea,0x68,0x12,0xc0,0x00,0x00,0x00,0x00]
+lwpins $0, %eax, %edx
+
+// CHECK: lwpins $0, (%edx), %edx
+// CHECK: encoding: [0x8f,0xea,0x68,0x12,0x02,0x00,0x00,0x00,0x00]
+lwpins $0, (%edx), %edx
+
+// CHECK: lwpval $0, -485498096(%edx,%eax,4), %edx
+// CHECK: encoding: [0x8f,0xea,0x68,0x12,0x8c,0x82,0x10,0xe3,0x0f,0xe3,0x00,0x00,0x00,0x00]
+lwpval $0, -485498096(%edx,%eax,4), %edx
+
+// CHECK: lwpval $0, 485498096(%edx,%eax,4), %edx
+// CHECK: encoding: [0x8f,0xea,0x68,0x12,0x8c,0x82,0xf0,0x1c,0xf0,0x1c,0x00,0x00,0x00,0x00]
+lwpval $0, 485498096(%edx,%eax,4), %edx
+
+// CHECK: lwpval $0, 485498096(%edx), %edx
+// CHECK: encoding: [0x8f,0xea,0x68,0x12,0x8a,0xf0,0x1c,0xf0,0x1c,0x00,0x00,0x00,0x00]
+lwpval $0, 485498096(%edx), %edx
+
+// CHECK: lwpval $0, 485498096, %edx
+// CHECK: encoding: [0x8f,0xea,0x68,0x12,0x0d,0xf0,0x1c,0xf0,0x1c,0x00,0x00,0x00,0x00]
+lwpval $0, 485498096, %edx
+
+// CHECK: lwpval $0, 64(%edx,%eax), %edx
+// CHECK: encoding: [0x8f,0xea,0x68,0x12,0x4c,0x02,0x40,0x00,0x00,0x00,0x00]
+lwpval $0, 64(%edx,%eax), %edx
+
+// CHECK: lwpval $0, %eax, %edx
+// CHECK: encoding: [0x8f,0xea,0x68,0x12,0xc8,0x00,0x00,0x00,0x00]
+lwpval $0, %eax, %edx
+
+// CHECK: lwpval $0, (%edx), %edx
+// CHECK: encoding: [0x8f,0xea,0x68,0x12,0x0a,0x00,0x00,0x00,0x00]
+lwpval $0, (%edx), %edx
+
+// CHECK: slwpcb %eax
+// CHECK: encoding: [0x8f,0xe9,0x78,0x12,0xc8]
+slwpcb %eax
diff --git a/test/MC/X86/LWP-64.s b/test/MC/X86/LWP-64.s
new file mode 100644
index 0000000000000000000000000000000000000000..e1971c976db7b683864ef48ff9dada5f9d8d9d4c
--- /dev/null
+++ b/test/MC/X86/LWP-64.s
@@ -0,0 +1,129 @@
+// RUN: llvm-mc -triple x86_64-unknown-unknown --show-encoding %s | FileCheck %s
+
+// CHECK: llwpcb %r13d
+// CHECK: encoding: [0x8f,0xc9,0x78,0x12,0xc5]
+llwpcb %r13d
+
+// CHECK: llwpcb %r13
+// CHECK: encoding: [0x8f,0xc9,0xf8,0x12,0xc5]
+llwpcb %r13
+
+// CHECK: lwpins $0, 485498096, %r15d
+// CHECK: encoding: [0x8f,0xea,0x00,0x12,0x04,0x25,0xf0,0x1c,0xf0,0x1c,0x00,0x00,0x00,0x00]
+lwpins $0, 485498096, %r15d
+
+// CHECK: lwpins $0, 485498096, %r15
+// CHECK: encoding: [0x8f,0xea,0x80,0x12,0x04,0x25,0xf0,0x1c,0xf0,0x1c,0x00,0x00,0x00,0x00]
+lwpins $0, 485498096, %r15
+
+// CHECK: lwpins $0, 64(%rdx), %r15d
+// CHECK: encoding: [0x8f,0xea,0x00,0x12,0x42,0x40,0x00,0x00,0x00,0x00]
+lwpins $0, 64(%rdx), %r15d
+
+// CHECK: lwpins $0, 64(%rdx), %r15
+// CHECK: encoding: [0x8f,0xea,0x80,0x12,0x42,0x40,0x00,0x00,0x00,0x00]
+lwpins $0, 64(%rdx), %r15
+
+// CHECK: lwpins $0, 64(%rdx,%rax,4), %r15d
+// CHECK: encoding: [0x8f,0xea,0x00,0x12,0x44,0x82,0x40,0x00,0x00,0x00,0x00]
+lwpins $0, 64(%rdx,%rax,4), %r15d
+
+// CHECK: lwpins $0, 64(%rdx,%rax,4), %r15
+// CHECK: encoding: [0x8f,0xea,0x80,0x12,0x44,0x82,0x40,0x00,0x00,0x00,0x00]
+lwpins $0, 64(%rdx,%rax,4), %r15
+
+// CHECK: lwpins $0, -64(%rdx,%rax,4), %r15d
+// CHECK: encoding: [0x8f,0xea,0x00,0x12,0x44,0x82,0xc0,0x00,0x00,0x00,0x00]
+lwpins $0, -64(%rdx,%rax,4), %r15d
+
+// CHECK: lwpins $0, -64(%rdx,%rax,4), %r15
+// CHECK: encoding: [0x8f,0xea,0x80,0x12,0x44,0x82,0xc0,0x00,0x00,0x00,0x00]
+lwpins $0, -64(%rdx,%rax,4), %r15
+
+// CHECK: lwpins $0, 64(%rdx,%rax), %r15d
+// CHECK: encoding: [0x8f,0xea,0x00,0x12,0x44,0x02,0x40,0x00,0x00,0x00,0x00]
+lwpins $0, 64(%rdx,%rax), %r15d
+
+// CHECK: lwpins $0, 64(%rdx,%rax), %r15
+// CHECK: encoding: [0x8f,0xea,0x80,0x12,0x44,0x02,0x40,0x00,0x00,0x00,0x00]
+lwpins $0, 64(%rdx,%rax), %r15
+
+// CHECK: lwpins $0, %r13d, %r15d
+// CHECK: encoding: [0x8f,0xca,0x00,0x12,0xc5,0x00,0x00,0x00,0x00]
+lwpins $0, %r13d, %r15d
+
+// CHECK: lwpins $0, %r13d, %r15
+// CHECK: encoding: [0x8f,0xca,0x80,0x12,0xc5,0x00,0x00,0x00,0x00]
+lwpins $0, %r13d, %r15
+
+// CHECK: lwpins $0, (%rdx), %r15d
+// CHECK: encoding: [0x8f,0xea,0x00,0x12,0x02,0x00,0x00,0x00,0x00]
+lwpins $0, (%rdx), %r15d
+
+// CHECK: lwpins $0, (%rdx), %r15
+// CHECK: encoding: [0x8f,0xea,0x80,0x12,0x02,0x00,0x00,0x00,0x00]
+lwpins $0, (%rdx), %r15
+
+// CHECK: lwpval $0, 485498096, %r15d
+// CHECK: encoding: [0x8f,0xea,0x00,0x12,0x0c,0x25,0xf0,0x1c,0xf0,0x1c,0x00,0x00,0x00,0x00]
+lwpval $0, 485498096, %r15d
+
+// CHECK: lwpval $0, 485498096, %r15
+// CHECK: encoding: [0x8f,0xea,0x80,0x12,0x0c,0x25,0xf0,0x1c,0xf0,0x1c,0x00,0x00,0x00,0x00]
+lwpval $0, 485498096, %r15
+
+// CHECK: lwpval $0, 64(%rdx), %r15d
+// CHECK: encoding: [0x8f,0xea,0x00,0x12,0x4a,0x40,0x00,0x00,0x00,0x00]
+lwpval $0, 64(%rdx), %r15d
+
+// CHECK: lwpval $0, 64(%rdx), %r15
+// CHECK: encoding: [0x8f,0xea,0x80,0x12,0x4a,0x40,0x00,0x00,0x00,0x00]
+lwpval $0, 64(%rdx), %r15
+
+// CHECK: lwpval $0, 64(%rdx,%rax,4), %r15d
+// CHECK: encoding: [0x8f,0xea,0x00,0x12,0x4c,0x82,0x40,0x00,0x00,0x00,0x00]
+lwpval $0, 64(%rdx,%rax,4), %r15d
+
+// CHECK: lwpval $0, 64(%rdx,%rax,4), %r15
+// CHECK: encoding: [0x8f,0xea,0x80,0x12,0x4c,0x82,0x40,0x00,0x00,0x00,0x00]
+lwpval $0, 64(%rdx,%rax,4), %r15
+
+// CHECK: lwpval $0, -64(%rdx,%rax,4), %r15d
+// CHECK: encoding: [0x8f,0xea,0x00,0x12,0x4c,0x82,0xc0,0x00,0x00,0x00,0x00]
+lwpval $0, -64(%rdx,%rax,4), %r15d
+
+// CHECK: lwpval $0, -64(%rdx,%rax,4), %r15
+// CHECK: encoding: [0x8f,0xea,0x80,0x12,0x4c,0x82,0xc0,0x00,0x00,0x00,0x00]
+lwpval $0, -64(%rdx,%rax,4), %r15
+
+// CHECK: lwpval $0, 64(%rdx,%rax), %r15d
+// CHECK: encoding: [0x8f,0xea,0x00,0x12,0x4c,0x02,0x40,0x00,0x00,0x00,0x00]
+lwpval $0, 64(%rdx,%rax), %r15d
+
+// CHECK: lwpval $0, 64(%rdx,%rax), %r15
+// CHECK: encoding: [0x8f,0xea,0x80,0x12,0x4c,0x02,0x40,0x00,0x00,0x00,0x00]
+lwpval $0, 64(%rdx,%rax), %r15
+
+// CHECK: lwpval $0, %r13d, %r15d
+// CHECK: encoding: [0x8f,0xca,0x00,0x12,0xcd,0x00,0x00,0x00,0x00]
+lwpval $0, %r13d, %r15d
+
+// CHECK: lwpval $0, %r13d, %r15
+// CHECK: encoding: [0x8f,0xca,0x80,0x12,0xcd,0x00,0x00,0x00,0x00]
+lwpval $0, %r13d, %r15
+
+// CHECK: lwpval $0, (%rdx), %r15d
+// CHECK: encoding: [0x8f,0xea,0x00,0x12,0x0a,0x00,0x00,0x00,0x00]
+lwpval $0, (%rdx), %r15d
+
+// CHECK: lwpval $0, (%rdx), %r15
+// CHECK: encoding: [0x8f,0xea,0x80,0x12,0x0a,0x00,0x00,0x00,0x00]
+lwpval $0, (%rdx), %r15
+
+// CHECK: slwpcb %r13d
+// CHECK: encoding: [0x8f,0xc9,0x78,0x12,0xcd]
+slwpcb %r13d
+
+// CHECK: slwpcb %r13
+// CHECK: encoding: [0x8f,0xc9,0xf8,0x12,0xcd]
+slwpcb %r13
diff --git a/test/MC/X86/MMX-32.s b/test/MC/X86/MMX-32.s
index 25fa370fd6b94e46c22c5ff8ddb6a9771a5a86cd..eb1a04f610291dfd1adf320da2d0d641187a6ec0 100755
--- a/test/MC/X86/MMX-32.s
+++ b/test/MC/X86/MMX-32.s
@@ -272,35 +272,7 @@ paddd (%edx), %mm4
 // CHECK: encoding: [0x0f,0xfe,0xe4]        
 paddd %mm4, %mm4 
 
-// CHECK: paddq -485498096(%edx,%eax,4), %mm4 
-// CHECK: encoding: [0x0f,0xd4,0xa4,0x82,0x10,0xe3,0x0f,0xe3]        
-paddq -485498096(%edx,%eax,4), %mm4 
-
-// CHECK: paddq 485498096(%edx,%eax,4), %mm4 
-// CHECK: encoding: [0x0f,0xd4,0xa4,0x82,0xf0,0x1c,0xf0,0x1c]        
-paddq 485498096(%edx,%eax,4), %mm4 
-
-// CHECK: paddq 485498096(%edx), %mm4 
-// CHECK: encoding: [0x0f,0xd4,0xa2,0xf0,0x1c,0xf0,0x1c]        
-paddq 485498096(%edx), %mm4 
-
-// CHECK: paddq 485498096, %mm4 
-// CHECK: encoding: [0x0f,0xd4,0x25,0xf0,0x1c,0xf0,0x1c]        
-paddq 485498096, %mm4 
-
-// CHECK: paddq 64(%edx,%eax), %mm4 
-// CHECK: encoding: [0x0f,0xd4,0x64,0x02,0x40]        
-paddq 64(%edx,%eax), %mm4 
-
-// CHECK: paddq (%edx), %mm4 
-// CHECK: encoding: [0x0f,0xd4,0x22]        
-paddq (%edx), %mm4 
-
-// CHECK: paddq %mm4, %mm4 
-// CHECK: encoding: [0x0f,0xd4,0xe4]        
-paddq %mm4, %mm4 
-
-// CHECK: paddsb -485498096(%edx,%eax,4), %mm4 
+// CHECK: paddsb -485498096(%edx,%eax,4), %mm4
 // CHECK: encoding: [0x0f,0xec,0xa4,0x82,0x10,0xe3,0x0f,0xe3]        
 paddsb -485498096(%edx,%eax,4), %mm4 
 
@@ -720,30 +692,6 @@ pcmpgtw (%edx), %mm4
 // CHECK: encoding: [0x0f,0x65,0xe4]        
 pcmpgtw %mm4, %mm4 
 
-// CHECK: pinsrw $0, -485498096(%edx,%eax,4), %mm4 
-// CHECK: encoding: [0x0f,0xc4,0xa4,0x82,0x10,0xe3,0x0f,0xe3,0x00]       
-pinsrw $0, -485498096(%edx,%eax,4), %mm4 
-
-// CHECK: pinsrw $0, 485498096(%edx,%eax,4), %mm4 
-// CHECK: encoding: [0x0f,0xc4,0xa4,0x82,0xf0,0x1c,0xf0,0x1c,0x00]       
-pinsrw $0, 485498096(%edx,%eax,4), %mm4 
-
-// CHECK: pinsrw $0, 485498096(%edx), %mm4 
-// CHECK: encoding: [0x0f,0xc4,0xa2,0xf0,0x1c,0xf0,0x1c,0x00]       
-pinsrw $0, 485498096(%edx), %mm4 
-
-// CHECK: pinsrw $0, 485498096, %mm4 
-// CHECK: encoding: [0x0f,0xc4,0x25,0xf0,0x1c,0xf0,0x1c,0x00]       
-pinsrw $0, 485498096, %mm4 
-
-// CHECK: pinsrw $0, 64(%edx,%eax), %mm4 
-// CHECK: encoding: [0x0f,0xc4,0x64,0x02,0x40,0x00]       
-pinsrw $0, 64(%edx,%eax), %mm4 
-
-// CHECK: pinsrw $0, (%edx), %mm4 
-// CHECK: encoding: [0x0f,0xc4,0x22,0x00]       
-pinsrw $0, (%edx), %mm4 
-
 // CHECK: pmaddwd -485498096(%edx,%eax,4), %mm4 
 // CHECK: encoding: [0x0f,0xf5,0xa4,0x82,0x10,0xe3,0x0f,0xe3]        
 pmaddwd -485498096(%edx,%eax,4), %mm4 
@@ -1364,34 +1312,6 @@ psubd (%edx), %mm4
 // CHECK: encoding: [0x0f,0xfa,0xe4]        
 psubd %mm4, %mm4 
 
-// CHECK: psubq -485498096(%edx,%eax,4), %mm4 
-// CHECK: encoding: [0x0f,0xfb,0xa4,0x82,0x10,0xe3,0x0f,0xe3]        
-psubq -485498096(%edx,%eax,4), %mm4 
-
-// CHECK: psubq 485498096(%edx,%eax,4), %mm4 
-// CHECK: encoding: [0x0f,0xfb,0xa4,0x82,0xf0,0x1c,0xf0,0x1c]        
-psubq 485498096(%edx,%eax,4), %mm4 
-
-// CHECK: psubq 485498096(%edx), %mm4 
-// CHECK: encoding: [0x0f,0xfb,0xa2,0xf0,0x1c,0xf0,0x1c]        
-psubq 485498096(%edx), %mm4 
-
-// CHECK: psubq 485498096, %mm4 
-// CHECK: encoding: [0x0f,0xfb,0x25,0xf0,0x1c,0xf0,0x1c]        
-psubq 485498096, %mm4 
-
-// CHECK: psubq 64(%edx,%eax), %mm4 
-// CHECK: encoding: [0x0f,0xfb,0x64,0x02,0x40]        
-psubq 64(%edx,%eax), %mm4 
-
-// CHECK: psubq (%edx), %mm4 
-// CHECK: encoding: [0x0f,0xfb,0x22]        
-psubq (%edx), %mm4 
-
-// CHECK: psubq %mm4, %mm4 
-// CHECK: encoding: [0x0f,0xfb,0xe4]        
-psubq %mm4, %mm4 
-
 // CHECK: psubsb -485498096(%edx,%eax,4), %mm4 
 // CHECK: encoding: [0x0f,0xe8,0xa4,0x82,0x10,0xe3,0x0f,0xe3]        
 psubsb -485498096(%edx,%eax,4), %mm4 
diff --git a/test/MC/X86/MMX-64.s b/test/MC/X86/MMX-64.s
index 4a148f216d5543eb621779354f553ea2b4a642a4..889acc4830e136f4f4e61fe98d5b06ea8a06728a 100644
--- a/test/MC/X86/MMX-64.s
+++ b/test/MC/X86/MMX-64.s
@@ -288,34 +288,6 @@ paddd %mm4, %mm4
 // CHECK: encoding: [0x0f,0xfe,0x22]        
 paddd (%rdx), %mm4 
 
-// CHECK: paddq 485498096, %mm4 
-// CHECK: encoding: [0x0f,0xd4,0x24,0x25,0xf0,0x1c,0xf0,0x1c]        
-paddq 485498096, %mm4 
-
-// CHECK: paddq 64(%rdx), %mm4 
-// CHECK: encoding: [0x0f,0xd4,0x62,0x40]        
-paddq 64(%rdx), %mm4 
-
-// CHECK: paddq 64(%rdx,%rax,4), %mm4 
-// CHECK: encoding: [0x0f,0xd4,0x64,0x82,0x40]        
-paddq 64(%rdx,%rax,4), %mm4 
-
-// CHECK: paddq -64(%rdx,%rax,4), %mm4 
-// CHECK: encoding: [0x0f,0xd4,0x64,0x82,0xc0]        
-paddq -64(%rdx,%rax,4), %mm4 
-
-// CHECK: paddq 64(%rdx,%rax), %mm4 
-// CHECK: encoding: [0x0f,0xd4,0x64,0x02,0x40]        
-paddq 64(%rdx,%rax), %mm4 
-
-// CHECK: paddq %mm4, %mm4 
-// CHECK: encoding: [0x0f,0xd4,0xe4]        
-paddq %mm4, %mm4 
-
-// CHECK: paddq (%rdx), %mm4 
-// CHECK: encoding: [0x0f,0xd4,0x22]        
-paddq (%rdx), %mm4 
-
 // CHECK: paddsb 485498096, %mm4 
 // CHECK: encoding: [0x0f,0xec,0x24,0x25,0xf0,0x1c,0xf0,0x1c]        
 paddsb 485498096, %mm4 
@@ -736,39 +708,7 @@ pcmpgtw %mm4, %mm4
 // CHECK: encoding: [0x0f,0x65,0x22]        
 pcmpgtw (%rdx), %mm4 
 
-// CHECK: pextrw $0, %mm4, %r13d 
-// CHECK: encoding: [0x44,0x0f,0xc5,0xec,0x00]       
-pextrw $0, %mm4, %r13d 
-
-// CHECK: pinsrw $0, 485498096, %mm4 
-// CHECK: encoding: [0x0f,0xc4,0x24,0x25,0xf0,0x1c,0xf0,0x1c,0x00]       
-pinsrw $0, 485498096, %mm4 
-
-// CHECK: pinsrw $0, 64(%rdx), %mm4 
-// CHECK: encoding: [0x0f,0xc4,0x62,0x40,0x00]       
-pinsrw $0, 64(%rdx), %mm4 
-
-// CHECK: pinsrw $0, 64(%rdx,%rax,4), %mm4 
-// CHECK: encoding: [0x0f,0xc4,0x64,0x82,0x40,0x00]       
-pinsrw $0, 64(%rdx,%rax,4), %mm4 
-
-// CHECK: pinsrw $0, -64(%rdx,%rax,4), %mm4 
-// CHECK: encoding: [0x0f,0xc4,0x64,0x82,0xc0,0x00]       
-pinsrw $0, -64(%rdx,%rax,4), %mm4 
-
-// CHECK: pinsrw $0, 64(%rdx,%rax), %mm4 
-// CHECK: encoding: [0x0f,0xc4,0x64,0x02,0x40,0x00]       
-pinsrw $0, 64(%rdx,%rax), %mm4 
-
-// CHECK: pinsrw $0, %r13d, %mm4 
-// CHECK: encoding: [0x41,0x0f,0xc4,0xe5,0x00]       
-pinsrw $0, %r13d, %mm4 
-
-// CHECK: pinsrw $0, (%rdx), %mm4 
-// CHECK: encoding: [0x0f,0xc4,0x22,0x00]       
-pinsrw $0, (%rdx), %mm4 
-
-// CHECK: pmaddwd 485498096, %mm4 
+// CHECK: pmaddwd 485498096, %mm4
 // CHECK: encoding: [0x0f,0xf5,0x24,0x25,0xf0,0x1c,0xf0,0x1c]        
 pmaddwd 485498096, %mm4 
 
@@ -1388,34 +1328,6 @@ psubd %mm4, %mm4
 // CHECK: encoding: [0x0f,0xfa,0x22]        
 psubd (%rdx), %mm4 
 
-// CHECK: psubq 485498096, %mm4 
-// CHECK: encoding: [0x0f,0xfb,0x24,0x25,0xf0,0x1c,0xf0,0x1c]        
-psubq 485498096, %mm4 
-
-// CHECK: psubq 64(%rdx), %mm4 
-// CHECK: encoding: [0x0f,0xfb,0x62,0x40]        
-psubq 64(%rdx), %mm4 
-
-// CHECK: psubq 64(%rdx,%rax,4), %mm4 
-// CHECK: encoding: [0x0f,0xfb,0x64,0x82,0x40]        
-psubq 64(%rdx,%rax,4), %mm4 
-
-// CHECK: psubq -64(%rdx,%rax,4), %mm4 
-// CHECK: encoding: [0x0f,0xfb,0x64,0x82,0xc0]        
-psubq -64(%rdx,%rax,4), %mm4 
-
-// CHECK: psubq 64(%rdx,%rax), %mm4 
-// CHECK: encoding: [0x0f,0xfb,0x64,0x02,0x40]        
-psubq 64(%rdx,%rax), %mm4 
-
-// CHECK: psubq %mm4, %mm4 
-// CHECK: encoding: [0x0f,0xfb,0xe4]        
-psubq %mm4, %mm4 
-
-// CHECK: psubq (%rdx), %mm4 
-// CHECK: encoding: [0x0f,0xfb,0x22]        
-psubq (%rdx), %mm4 
-
 // CHECK: psubsb 485498096, %mm4 
 // CHECK: encoding: [0x0f,0xe8,0x24,0x25,0xf0,0x1c,0xf0,0x1c]        
 psubsb 485498096, %mm4 
diff --git a/test/MC/X86/PPRO-32.s b/test/MC/X86/PPRO-32.s
index bbd933e58af093095e618d64069dce5b65c5a2fd..6deea6de9c449ba2d97bff99f3125de3c37dc9b2 100644
--- a/test/MC/X86/PPRO-32.s
+++ b/test/MC/X86/PPRO-32.s
@@ -64,37 +64,37 @@ cmovpl %eax, %eax
 // CHECK: encoding: [0x0f,0x48,0xc0]        
 cmovsl %eax, %eax 
 
-// CHECK: fcmovbe %st(4), %st(0) 
+// CHECK: fcmovbe %st(4), %st 
 // CHECK: encoding: [0xda,0xd4]        
-fcmovbe %st(4), %st(0) 
+fcmovbe %st(4), %st 
 
-// CHECK: fcmovb %st(4), %st(0) 
+// CHECK: fcmovb %st(4), %st 
 // CHECK: encoding: [0xda,0xc4]        
-fcmovb %st(4), %st(0) 
+fcmovb %st(4), %st 
 
-// CHECK: fcmove %st(4), %st(0) 
+// CHECK: fcmove %st(4), %st 
 // CHECK: encoding: [0xda,0xcc]        
-fcmove %st(4), %st(0) 
+fcmove %st(4), %st 
 
-// CHECK: fcmovnbe %st(4), %st(0) 
+// CHECK: fcmovnbe %st(4), %st 
 // CHECK: encoding: [0xdb,0xd4]        
-fcmovnbe %st(4), %st(0) 
+fcmovnbe %st(4), %st 
 
-// CHECK: fcmovnb %st(4), %st(0) 
+// CHECK: fcmovnb %st(4), %st 
 // CHECK: encoding: [0xdb,0xc4]        
-fcmovnb %st(4), %st(0) 
+fcmovnb %st(4), %st 
 
-// CHECK: fcmovne %st(4), %st(0) 
+// CHECK: fcmovne %st(4), %st 
 // CHECK: encoding: [0xdb,0xcc]        
-fcmovne %st(4), %st(0) 
+fcmovne %st(4), %st 
 
-// CHECK: fcmovnu %st(4), %st(0) 
+// CHECK: fcmovnu %st(4), %st 
 // CHECK: encoding: [0xdb,0xdc]        
-fcmovnu %st(4), %st(0) 
+fcmovnu %st(4), %st 
 
-// CHECK: fcmovu %st(4), %st(0) 
+// CHECK: fcmovu %st(4), %st 
 // CHECK: encoding: [0xda,0xdc]        
-fcmovu %st(4), %st(0) 
+fcmovu %st(4), %st 
 
 // CHECK: fcomi %st(4) 
 // CHECK: encoding: [0xdb,0xf4]         
diff --git a/test/MC/X86/PPRO-64.s b/test/MC/X86/PPRO-64.s
index a41d4a0f93cc982ce531fb6a65e2a38de599eace..8004772e8f3defb30ea0f9864eb6fe1d9a51789a 100644
--- a/test/MC/X86/PPRO-64.s
+++ b/test/MC/X86/PPRO-64.s
@@ -64,37 +64,37 @@ cmovpl %r13d, %r13d
 // CHECK: encoding: [0x45,0x0f,0x48,0xed]        
 cmovsl %r13d, %r13d 
 
-// CHECK: fcmovbe %st(4), %st(0) 
+// CHECK: fcmovbe %st(4), %st 
 // CHECK: encoding: [0xda,0xd4]        
-fcmovbe %st(4), %st(0) 
+fcmovbe %st(4), %st 
 
-// CHECK: fcmovb %st(4), %st(0) 
+// CHECK: fcmovb %st(4), %st 
 // CHECK: encoding: [0xda,0xc4]        
-fcmovb %st(4), %st(0) 
+fcmovb %st(4), %st 
 
-// CHECK: fcmove %st(4), %st(0) 
+// CHECK: fcmove %st(4), %st 
 // CHECK: encoding: [0xda,0xcc]        
-fcmove %st(4), %st(0) 
+fcmove %st(4), %st 
 
-// CHECK: fcmovnbe %st(4), %st(0) 
+// CHECK: fcmovnbe %st(4), %st 
 // CHECK: encoding: [0xdb,0xd4]        
-fcmovnbe %st(4), %st(0) 
+fcmovnbe %st(4), %st 
 
-// CHECK: fcmovnb %st(4), %st(0) 
+// CHECK: fcmovnb %st(4), %st 
 // CHECK: encoding: [0xdb,0xc4]        
-fcmovnb %st(4), %st(0) 
+fcmovnb %st(4), %st 
 
-// CHECK: fcmovne %st(4), %st(0) 
+// CHECK: fcmovne %st(4), %st 
 // CHECK: encoding: [0xdb,0xcc]        
-fcmovne %st(4), %st(0) 
+fcmovne %st(4), %st 
 
-// CHECK: fcmovnu %st(4), %st(0) 
+// CHECK: fcmovnu %st(4), %st 
 // CHECK: encoding: [0xdb,0xdc]        
-fcmovnu %st(4), %st(0) 
+fcmovnu %st(4), %st 
 
-// CHECK: fcmovu %st(4), %st(0) 
+// CHECK: fcmovu %st(4), %st 
 // CHECK: encoding: [0xda,0xdc]        
-fcmovu %st(4), %st(0) 
+fcmovu %st(4), %st 
 
 // CHECK: fcomi %st(4) 
 // CHECK: encoding: [0xdb,0xf4]         
diff --git a/test/MC/X86/SSE-32.s b/test/MC/X86/SSE-32.s
new file mode 100644
index 0000000000000000000000000000000000000000..98d252b5e9ed7b71f831088f0962652da7e905be
--- /dev/null
+++ b/test/MC/X86/SSE-32.s
@@ -0,0 +1,1246 @@
+// RUN: llvm-mc -triple i386-unknown-unknown --show-encoding %s | FileCheck %s
+
+// CHECK: addps -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x0f,0x58,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+addps -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: addps 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x0f,0x58,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+addps 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: addps 485498096(%edx), %xmm1
+// CHECK: encoding: [0x0f,0x58,0x8a,0xf0,0x1c,0xf0,0x1c]
+addps 485498096(%edx), %xmm1
+
+// CHECK: addps 485498096, %xmm1
+// CHECK: encoding: [0x0f,0x58,0x0d,0xf0,0x1c,0xf0,0x1c]
+addps 485498096, %xmm1
+
+// CHECK: addps 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x0f,0x58,0x4c,0x02,0x40]
+addps 64(%edx,%eax), %xmm1
+
+// CHECK: addps (%edx), %xmm1
+// CHECK: encoding: [0x0f,0x58,0x0a]
+addps (%edx), %xmm1
+
+// CHECK: addps %xmm1, %xmm1
+// CHECK: encoding: [0x0f,0x58,0xc9]
+addps %xmm1, %xmm1
+
+// CHECK: addss -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x58,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+addss -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: addss 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x58,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+addss 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: addss 485498096(%edx), %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x58,0x8a,0xf0,0x1c,0xf0,0x1c]
+addss 485498096(%edx), %xmm1
+
+// CHECK: addss 485498096, %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x58,0x0d,0xf0,0x1c,0xf0,0x1c]
+addss 485498096, %xmm1
+
+// CHECK: addss 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x58,0x4c,0x02,0x40]
+addss 64(%edx,%eax), %xmm1
+
+// CHECK: addss (%edx), %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x58,0x0a]
+addss (%edx), %xmm1
+
+// CHECK: addss %xmm1, %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x58,0xc9]
+addss %xmm1, %xmm1
+
+// CHECK: andnps -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x0f,0x55,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+andnps -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: andnps 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x0f,0x55,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+andnps 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: andnps 485498096(%edx), %xmm1
+// CHECK: encoding: [0x0f,0x55,0x8a,0xf0,0x1c,0xf0,0x1c]
+andnps 485498096(%edx), %xmm1
+
+// CHECK: andnps 485498096, %xmm1
+// CHECK: encoding: [0x0f,0x55,0x0d,0xf0,0x1c,0xf0,0x1c]
+andnps 485498096, %xmm1
+
+// CHECK: andnps 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x0f,0x55,0x4c,0x02,0x40]
+andnps 64(%edx,%eax), %xmm1
+
+// CHECK: andnps (%edx), %xmm1
+// CHECK: encoding: [0x0f,0x55,0x0a]
+andnps (%edx), %xmm1
+
+// CHECK: andnps %xmm1, %xmm1
+// CHECK: encoding: [0x0f,0x55,0xc9]
+andnps %xmm1, %xmm1
+
+// CHECK: andps -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x0f,0x54,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+andps -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: andps 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x0f,0x54,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+andps 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: andps 485498096(%edx), %xmm1
+// CHECK: encoding: [0x0f,0x54,0x8a,0xf0,0x1c,0xf0,0x1c]
+andps 485498096(%edx), %xmm1
+
+// CHECK: andps 485498096, %xmm1
+// CHECK: encoding: [0x0f,0x54,0x0d,0xf0,0x1c,0xf0,0x1c]
+andps 485498096, %xmm1
+
+// CHECK: andps 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x0f,0x54,0x4c,0x02,0x40]
+andps 64(%edx,%eax), %xmm1
+
+// CHECK: andps (%edx), %xmm1
+// CHECK: encoding: [0x0f,0x54,0x0a]
+andps (%edx), %xmm1
+
+// CHECK: andps %xmm1, %xmm1
+// CHECK: encoding: [0x0f,0x54,0xc9]
+andps %xmm1, %xmm1
+
+// CHECK: cmpeqps -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x0f,0xc2,0x8c,0x82,0x10,0xe3,0x0f,0xe3,0x00]
+cmpeqps -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: cmpeqps 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x0f,0xc2,0x8c,0x82,0xf0,0x1c,0xf0,0x1c,0x00]
+cmpeqps 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: cmpeqps 485498096(%edx), %xmm1
+// CHECK: encoding: [0x0f,0xc2,0x8a,0xf0,0x1c,0xf0,0x1c,0x00]
+cmpeqps 485498096(%edx), %xmm1
+
+// CHECK: cmpeqps 485498096, %xmm1
+// CHECK: encoding: [0x0f,0xc2,0x0d,0xf0,0x1c,0xf0,0x1c,0x00]
+cmpeqps 485498096, %xmm1
+
+// CHECK: cmpeqps 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x0f,0xc2,0x4c,0x02,0x40,0x00]
+cmpeqps 64(%edx,%eax), %xmm1
+
+// CHECK: cmpeqps (%edx), %xmm1
+// CHECK: encoding: [0x0f,0xc2,0x0a,0x00]
+cmpeqps (%edx), %xmm1
+
+// CHECK: cmpeqps %xmm1, %xmm1
+// CHECK: encoding: [0x0f,0xc2,0xc9,0x00]
+cmpeqps %xmm1, %xmm1
+
+// CHECK: cmpeqss -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0xf3,0x0f,0xc2,0x8c,0x82,0x10,0xe3,0x0f,0xe3,0x00]
+cmpeqss -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: cmpeqss 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0xf3,0x0f,0xc2,0x8c,0x82,0xf0,0x1c,0xf0,0x1c,0x00]
+cmpeqss 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: cmpeqss 485498096(%edx), %xmm1
+// CHECK: encoding: [0xf3,0x0f,0xc2,0x8a,0xf0,0x1c,0xf0,0x1c,0x00]
+cmpeqss 485498096(%edx), %xmm1
+
+// CHECK: cmpeqss 485498096, %xmm1
+// CHECK: encoding: [0xf3,0x0f,0xc2,0x0d,0xf0,0x1c,0xf0,0x1c,0x00]
+cmpeqss 485498096, %xmm1
+
+// CHECK: cmpeqss 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0xf3,0x0f,0xc2,0x4c,0x02,0x40,0x00]
+cmpeqss 64(%edx,%eax), %xmm1
+
+// CHECK: cmpeqss (%edx), %xmm1
+// CHECK: encoding: [0xf3,0x0f,0xc2,0x0a,0x00]
+cmpeqss (%edx), %xmm1
+
+// CHECK: cmpeqss %xmm1, %xmm1
+// CHECK: encoding: [0xf3,0x0f,0xc2,0xc9,0x00]
+cmpeqss %xmm1, %xmm1
+
+// CHECK: comiss -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x0f,0x2f,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+comiss -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: comiss 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x0f,0x2f,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+comiss 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: comiss 485498096(%edx), %xmm1
+// CHECK: encoding: [0x0f,0x2f,0x8a,0xf0,0x1c,0xf0,0x1c]
+comiss 485498096(%edx), %xmm1
+
+// CHECK: comiss 485498096, %xmm1
+// CHECK: encoding: [0x0f,0x2f,0x0d,0xf0,0x1c,0xf0,0x1c]
+comiss 485498096, %xmm1
+
+// CHECK: comiss 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x0f,0x2f,0x4c,0x02,0x40]
+comiss 64(%edx,%eax), %xmm1
+
+// CHECK: comiss (%edx), %xmm1
+// CHECK: encoding: [0x0f,0x2f,0x0a]
+comiss (%edx), %xmm1
+
+// CHECK: comiss %xmm1, %xmm1
+// CHECK: encoding: [0x0f,0x2f,0xc9]
+comiss %xmm1, %xmm1
+
+// CHECK: cvtpi2ps -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x0f,0x2a,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+cvtpi2ps -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: cvtpi2ps 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x0f,0x2a,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+cvtpi2ps 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: cvtpi2ps 485498096(%edx), %xmm1
+// CHECK: encoding: [0x0f,0x2a,0x8a,0xf0,0x1c,0xf0,0x1c]
+cvtpi2ps 485498096(%edx), %xmm1
+
+// CHECK: cvtpi2ps 485498096, %xmm1
+// CHECK: encoding: [0x0f,0x2a,0x0d,0xf0,0x1c,0xf0,0x1c]
+cvtpi2ps 485498096, %xmm1
+
+// CHECK: cvtpi2ps 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x0f,0x2a,0x4c,0x02,0x40]
+cvtpi2ps 64(%edx,%eax), %xmm1
+
+// CHECK: cvtpi2ps (%edx), %xmm1
+// CHECK: encoding: [0x0f,0x2a,0x0a]
+cvtpi2ps (%edx), %xmm1
+
+// CHECK: cvtpi2ps %mm4, %xmm1
+// CHECK: encoding: [0x0f,0x2a,0xcc]
+cvtpi2ps %mm4, %xmm1
+
+// CHECK: cvtps2pi -485498096(%edx,%eax,4), %mm4
+// CHECK: encoding: [0x0f,0x2d,0xa4,0x82,0x10,0xe3,0x0f,0xe3]
+cvtps2pi -485498096(%edx,%eax,4), %mm4
+
+// CHECK: cvtps2pi 485498096(%edx,%eax,4), %mm4
+// CHECK: encoding: [0x0f,0x2d,0xa4,0x82,0xf0,0x1c,0xf0,0x1c]
+cvtps2pi 485498096(%edx,%eax,4), %mm4
+
+// CHECK: cvtps2pi 485498096(%edx), %mm4
+// CHECK: encoding: [0x0f,0x2d,0xa2,0xf0,0x1c,0xf0,0x1c]
+cvtps2pi 485498096(%edx), %mm4
+
+// CHECK: cvtps2pi 485498096, %mm4
+// CHECK: encoding: [0x0f,0x2d,0x25,0xf0,0x1c,0xf0,0x1c]
+cvtps2pi 485498096, %mm4
+
+// CHECK: cvtps2pi 64(%edx,%eax), %mm4
+// CHECK: encoding: [0x0f,0x2d,0x64,0x02,0x40]
+cvtps2pi 64(%edx,%eax), %mm4
+
+// CHECK: cvtps2pi (%edx), %mm4
+// CHECK: encoding: [0x0f,0x2d,0x22]
+cvtps2pi (%edx), %mm4
+
+// CHECK: cvtps2pi %xmm1, %mm4
+// CHECK: encoding: [0x0f,0x2d,0xe1]
+cvtps2pi %xmm1, %mm4
+
+// CHECK: cvtsi2ssl -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x2a,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+cvtsi2ssl -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: cvtsi2ssl 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x2a,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+cvtsi2ssl 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: cvtsi2ssl 485498096(%edx), %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x2a,0x8a,0xf0,0x1c,0xf0,0x1c]
+cvtsi2ssl 485498096(%edx), %xmm1
+
+// CHECK: cvtsi2ssl 485498096, %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x2a,0x0d,0xf0,0x1c,0xf0,0x1c]
+cvtsi2ssl 485498096, %xmm1
+
+// CHECK: cvtsi2ssl 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x2a,0x4c,0x02,0x40]
+cvtsi2ssl 64(%edx,%eax), %xmm1
+
+// CHECK: cvtsi2ssl (%edx), %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x2a,0x0a]
+cvtsi2ssl (%edx), %xmm1
+
+// CHECK: cvttps2pi -485498096(%edx,%eax,4), %mm4
+// CHECK: encoding: [0x0f,0x2c,0xa4,0x82,0x10,0xe3,0x0f,0xe3]
+cvttps2pi -485498096(%edx,%eax,4), %mm4
+
+// CHECK: cvttps2pi 485498096(%edx,%eax,4), %mm4
+// CHECK: encoding: [0x0f,0x2c,0xa4,0x82,0xf0,0x1c,0xf0,0x1c]
+cvttps2pi 485498096(%edx,%eax,4), %mm4
+
+// CHECK: cvttps2pi 485498096(%edx), %mm4
+// CHECK: encoding: [0x0f,0x2c,0xa2,0xf0,0x1c,0xf0,0x1c]
+cvttps2pi 485498096(%edx), %mm4
+
+// CHECK: cvttps2pi 485498096, %mm4
+// CHECK: encoding: [0x0f,0x2c,0x25,0xf0,0x1c,0xf0,0x1c]
+cvttps2pi 485498096, %mm4
+
+// CHECK: cvttps2pi 64(%edx,%eax), %mm4
+// CHECK: encoding: [0x0f,0x2c,0x64,0x02,0x40]
+cvttps2pi 64(%edx,%eax), %mm4
+
+// CHECK: cvttps2pi (%edx), %mm4
+// CHECK: encoding: [0x0f,0x2c,0x22]
+cvttps2pi (%edx), %mm4
+
+// CHECK: cvttps2pi %xmm1, %mm4
+// CHECK: encoding: [0x0f,0x2c,0xe1]
+cvttps2pi %xmm1, %mm4
+
+// CHECK: divps -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x0f,0x5e,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+divps -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: divps 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x0f,0x5e,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+divps 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: divps 485498096(%edx), %xmm1
+// CHECK: encoding: [0x0f,0x5e,0x8a,0xf0,0x1c,0xf0,0x1c]
+divps 485498096(%edx), %xmm1
+
+// CHECK: divps 485498096, %xmm1
+// CHECK: encoding: [0x0f,0x5e,0x0d,0xf0,0x1c,0xf0,0x1c]
+divps 485498096, %xmm1
+
+// CHECK: divps 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x0f,0x5e,0x4c,0x02,0x40]
+divps 64(%edx,%eax), %xmm1
+
+// CHECK: divps (%edx), %xmm1
+// CHECK: encoding: [0x0f,0x5e,0x0a]
+divps (%edx), %xmm1
+
+// CHECK: divps %xmm1, %xmm1
+// CHECK: encoding: [0x0f,0x5e,0xc9]
+divps %xmm1, %xmm1
+
+// CHECK: divss -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x5e,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+divss -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: divss 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x5e,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+divss 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: divss 485498096(%edx), %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x5e,0x8a,0xf0,0x1c,0xf0,0x1c]
+divss 485498096(%edx), %xmm1
+
+// CHECK: divss 485498096, %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x5e,0x0d,0xf0,0x1c,0xf0,0x1c]
+divss 485498096, %xmm1
+
+// CHECK: divss 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x5e,0x4c,0x02,0x40]
+divss 64(%edx,%eax), %xmm1
+
+// CHECK: divss (%edx), %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x5e,0x0a]
+divss (%edx), %xmm1
+
+// CHECK: divss %xmm1, %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x5e,0xc9]
+divss %xmm1, %xmm1
+
+// CHECK: maxps -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x0f,0x5f,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+maxps -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: maxps 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x0f,0x5f,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+maxps 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: maxps 485498096(%edx), %xmm1
+// CHECK: encoding: [0x0f,0x5f,0x8a,0xf0,0x1c,0xf0,0x1c]
+maxps 485498096(%edx), %xmm1
+
+// CHECK: maxps 485498096, %xmm1
+// CHECK: encoding: [0x0f,0x5f,0x0d,0xf0,0x1c,0xf0,0x1c]
+maxps 485498096, %xmm1
+
+// CHECK: maxps 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x0f,0x5f,0x4c,0x02,0x40]
+maxps 64(%edx,%eax), %xmm1
+
+// CHECK: maxps (%edx), %xmm1
+// CHECK: encoding: [0x0f,0x5f,0x0a]
+maxps (%edx), %xmm1
+
+// CHECK: maxps %xmm1, %xmm1
+// CHECK: encoding: [0x0f,0x5f,0xc9]
+maxps %xmm1, %xmm1
+
+// CHECK: maxss -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x5f,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+maxss -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: maxss 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x5f,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+maxss 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: maxss 485498096(%edx), %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x5f,0x8a,0xf0,0x1c,0xf0,0x1c]
+maxss 485498096(%edx), %xmm1
+
+// CHECK: maxss 485498096, %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x5f,0x0d,0xf0,0x1c,0xf0,0x1c]
+maxss 485498096, %xmm1
+
+// CHECK: maxss 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x5f,0x4c,0x02,0x40]
+maxss 64(%edx,%eax), %xmm1
+
+// CHECK: maxss (%edx), %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x5f,0x0a]
+maxss (%edx), %xmm1
+
+// CHECK: maxss %xmm1, %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x5f,0xc9]
+maxss %xmm1, %xmm1
+
+// CHECK: minps -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x0f,0x5d,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+minps -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: minps 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x0f,0x5d,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+minps 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: minps 485498096(%edx), %xmm1
+// CHECK: encoding: [0x0f,0x5d,0x8a,0xf0,0x1c,0xf0,0x1c]
+minps 485498096(%edx), %xmm1
+
+// CHECK: minps 485498096, %xmm1
+// CHECK: encoding: [0x0f,0x5d,0x0d,0xf0,0x1c,0xf0,0x1c]
+minps 485498096, %xmm1
+
+// CHECK: minps 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x0f,0x5d,0x4c,0x02,0x40]
+minps 64(%edx,%eax), %xmm1
+
+// CHECK: minps (%edx), %xmm1
+// CHECK: encoding: [0x0f,0x5d,0x0a]
+minps (%edx), %xmm1
+
+// CHECK: minps %xmm1, %xmm1
+// CHECK: encoding: [0x0f,0x5d,0xc9]
+minps %xmm1, %xmm1
+
+// CHECK: minss -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x5d,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+minss -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: minss 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x5d,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+minss 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: minss 485498096(%edx), %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x5d,0x8a,0xf0,0x1c,0xf0,0x1c]
+minss 485498096(%edx), %xmm1
+
+// CHECK: minss 485498096, %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x5d,0x0d,0xf0,0x1c,0xf0,0x1c]
+minss 485498096, %xmm1
+
+// CHECK: minss 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x5d,0x4c,0x02,0x40]
+minss 64(%edx,%eax), %xmm1
+
+// CHECK: minss (%edx), %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x5d,0x0a]
+minss (%edx), %xmm1
+
+// CHECK: minss %xmm1, %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x5d,0xc9]
+minss %xmm1, %xmm1
+
+// CHECK: movaps -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x0f,0x28,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+movaps -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: movaps 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x0f,0x28,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+movaps 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: movaps 485498096(%edx), %xmm1
+// CHECK: encoding: [0x0f,0x28,0x8a,0xf0,0x1c,0xf0,0x1c]
+movaps 485498096(%edx), %xmm1
+
+// CHECK: movaps 485498096, %xmm1
+// CHECK: encoding: [0x0f,0x28,0x0d,0xf0,0x1c,0xf0,0x1c]
+movaps 485498096, %xmm1
+
+// CHECK: movaps 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x0f,0x28,0x4c,0x02,0x40]
+movaps 64(%edx,%eax), %xmm1
+
+// CHECK: movaps (%edx), %xmm1
+// CHECK: encoding: [0x0f,0x28,0x0a]
+movaps (%edx), %xmm1
+
+// CHECK: movaps %xmm1, 485498096
+// CHECK: encoding: [0x0f,0x29,0x0d,0xf0,0x1c,0xf0,0x1c]
+movaps %xmm1, 485498096
+
+// CHECK: movaps %xmm1, 485498096(%edx)
+// CHECK: encoding: [0x0f,0x29,0x8a,0xf0,0x1c,0xf0,0x1c]
+movaps %xmm1, 485498096(%edx)
+
+// CHECK: movaps %xmm1, -485498096(%edx,%eax,4)
+// CHECK: encoding: [0x0f,0x29,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+movaps %xmm1, -485498096(%edx,%eax,4)
+
+// CHECK: movaps %xmm1, 485498096(%edx,%eax,4)
+// CHECK: encoding: [0x0f,0x29,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+movaps %xmm1, 485498096(%edx,%eax,4)
+
+// CHECK: movaps %xmm1, 64(%edx,%eax)
+// CHECK: encoding: [0x0f,0x29,0x4c,0x02,0x40]
+movaps %xmm1, 64(%edx,%eax)
+
+// CHECK: movaps %xmm1, (%edx)
+// CHECK: encoding: [0x0f,0x29,0x0a]
+movaps %xmm1, (%edx)
+
+// CHECK: movaps %xmm1, %xmm1
+// CHECK: encoding: [0x0f,0x28,0xc9]
+movaps %xmm1, %xmm1
+
+// CHECK: movhlps %xmm1, %xmm1
+// CHECK: encoding: [0x0f,0x12,0xc9]
+movhlps %xmm1, %xmm1
+
+// CHECK: movhps -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x0f,0x16,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+movhps -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: movhps 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x0f,0x16,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+movhps 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: movhps 485498096(%edx), %xmm1
+// CHECK: encoding: [0x0f,0x16,0x8a,0xf0,0x1c,0xf0,0x1c]
+movhps 485498096(%edx), %xmm1
+
+// CHECK: movhps 485498096, %xmm1
+// CHECK: encoding: [0x0f,0x16,0x0d,0xf0,0x1c,0xf0,0x1c]
+movhps 485498096, %xmm1
+
+// CHECK: movhps 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x0f,0x16,0x4c,0x02,0x40]
+movhps 64(%edx,%eax), %xmm1
+
+// CHECK: movhps (%edx), %xmm1
+// CHECK: encoding: [0x0f,0x16,0x0a]
+movhps (%edx), %xmm1
+
+// CHECK: movhps %xmm1, 485498096
+// CHECK: encoding: [0x0f,0x17,0x0d,0xf0,0x1c,0xf0,0x1c]
+movhps %xmm1, 485498096
+
+// CHECK: movhps %xmm1, 485498096(%edx)
+// CHECK: encoding: [0x0f,0x17,0x8a,0xf0,0x1c,0xf0,0x1c]
+movhps %xmm1, 485498096(%edx)
+
+// CHECK: movhps %xmm1, -485498096(%edx,%eax,4)
+// CHECK: encoding: [0x0f,0x17,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+movhps %xmm1, -485498096(%edx,%eax,4)
+
+// CHECK: movhps %xmm1, 485498096(%edx,%eax,4)
+// CHECK: encoding: [0x0f,0x17,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+movhps %xmm1, 485498096(%edx,%eax,4)
+
+// CHECK: movhps %xmm1, 64(%edx,%eax)
+// CHECK: encoding: [0x0f,0x17,0x4c,0x02,0x40]
+movhps %xmm1, 64(%edx,%eax)
+
+// CHECK: movhps %xmm1, (%edx)
+// CHECK: encoding: [0x0f,0x17,0x0a]
+movhps %xmm1, (%edx)
+
+// CHECK: movlhps %xmm1, %xmm1
+// CHECK: encoding: [0x0f,0x16,0xc9]
+movlhps %xmm1, %xmm1
+
+// CHECK: movlps -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x0f,0x12,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+movlps -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: movlps 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x0f,0x12,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+movlps 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: movlps 485498096(%edx), %xmm1
+// CHECK: encoding: [0x0f,0x12,0x8a,0xf0,0x1c,0xf0,0x1c]
+movlps 485498096(%edx), %xmm1
+
+// CHECK: movlps 485498096, %xmm1
+// CHECK: encoding: [0x0f,0x12,0x0d,0xf0,0x1c,0xf0,0x1c]
+movlps 485498096, %xmm1
+
+// CHECK: movlps 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x0f,0x12,0x4c,0x02,0x40]
+movlps 64(%edx,%eax), %xmm1
+
+// CHECK: movlps (%edx), %xmm1
+// CHECK: encoding: [0x0f,0x12,0x0a]
+movlps (%edx), %xmm1
+
+// CHECK: movlps %xmm1, 485498096
+// CHECK: encoding: [0x0f,0x13,0x0d,0xf0,0x1c,0xf0,0x1c]
+movlps %xmm1, 485498096
+
+// CHECK: movlps %xmm1, 485498096(%edx)
+// CHECK: encoding: [0x0f,0x13,0x8a,0xf0,0x1c,0xf0,0x1c]
+movlps %xmm1, 485498096(%edx)
+
+// CHECK: movlps %xmm1, -485498096(%edx,%eax,4)
+// CHECK: encoding: [0x0f,0x13,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+movlps %xmm1, -485498096(%edx,%eax,4)
+
+// CHECK: movlps %xmm1, 485498096(%edx,%eax,4)
+// CHECK: encoding: [0x0f,0x13,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+movlps %xmm1, 485498096(%edx,%eax,4)
+
+// CHECK: movlps %xmm1, 64(%edx,%eax)
+// CHECK: encoding: [0x0f,0x13,0x4c,0x02,0x40]
+movlps %xmm1, 64(%edx,%eax)
+
+// CHECK: movlps %xmm1, (%edx)
+// CHECK: encoding: [0x0f,0x13,0x0a]
+movlps %xmm1, (%edx)
+
+// CHECK: movntps %xmm1, 485498096
+// CHECK: encoding: [0x0f,0x2b,0x0d,0xf0,0x1c,0xf0,0x1c]
+movntps %xmm1, 485498096
+
+// CHECK: movntps %xmm1, 485498096(%edx)
+// CHECK: encoding: [0x0f,0x2b,0x8a,0xf0,0x1c,0xf0,0x1c]
+movntps %xmm1, 485498096(%edx)
+
+// CHECK: movntps %xmm1, -485498096(%edx,%eax,4)
+// CHECK: encoding: [0x0f,0x2b,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+movntps %xmm1, -485498096(%edx,%eax,4)
+
+// CHECK: movntps %xmm1, 485498096(%edx,%eax,4)
+// CHECK: encoding: [0x0f,0x2b,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+movntps %xmm1, 485498096(%edx,%eax,4)
+
+// CHECK: movntps %xmm1, 64(%edx,%eax)
+// CHECK: encoding: [0x0f,0x2b,0x4c,0x02,0x40]
+movntps %xmm1, 64(%edx,%eax)
+
+// CHECK: movntps %xmm1, (%edx)
+// CHECK: encoding: [0x0f,0x2b,0x0a]
+movntps %xmm1, (%edx)
+
+// CHECK: movss -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x10,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+movss -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: movss 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x10,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+movss 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: movss 485498096(%edx), %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x10,0x8a,0xf0,0x1c,0xf0,0x1c]
+movss 485498096(%edx), %xmm1
+
+// CHECK: movss 485498096, %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x10,0x0d,0xf0,0x1c,0xf0,0x1c]
+movss 485498096, %xmm1
+
+// CHECK: movss 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x10,0x4c,0x02,0x40]
+movss 64(%edx,%eax), %xmm1
+
+// CHECK: movss (%edx), %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x10,0x0a]
+movss (%edx), %xmm1
+
+// CHECK: movss %xmm1, 485498096
+// CHECK: encoding: [0xf3,0x0f,0x11,0x0d,0xf0,0x1c,0xf0,0x1c]
+movss %xmm1, 485498096
+
+// CHECK: movss %xmm1, 485498096(%edx)
+// CHECK: encoding: [0xf3,0x0f,0x11,0x8a,0xf0,0x1c,0xf0,0x1c]
+movss %xmm1, 485498096(%edx)
+
+// CHECK: movss %xmm1, -485498096(%edx,%eax,4)
+// CHECK: encoding: [0xf3,0x0f,0x11,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+movss %xmm1, -485498096(%edx,%eax,4)
+
+// CHECK: movss %xmm1, 485498096(%edx,%eax,4)
+// CHECK: encoding: [0xf3,0x0f,0x11,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+movss %xmm1, 485498096(%edx,%eax,4)
+
+// CHECK: movss %xmm1, 64(%edx,%eax)
+// CHECK: encoding: [0xf3,0x0f,0x11,0x4c,0x02,0x40]
+movss %xmm1, 64(%edx,%eax)
+
+// CHECK: movss %xmm1, (%edx)
+// CHECK: encoding: [0xf3,0x0f,0x11,0x0a]
+movss %xmm1, (%edx)
+
+// CHECK: movss %xmm1, %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x10,0xc9]
+movss %xmm1, %xmm1
+
+// CHECK: movups -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x0f,0x10,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+movups -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: movups 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x0f,0x10,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+movups 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: movups 485498096(%edx), %xmm1
+// CHECK: encoding: [0x0f,0x10,0x8a,0xf0,0x1c,0xf0,0x1c]
+movups 485498096(%edx), %xmm1
+
+// CHECK: movups 485498096, %xmm1
+// CHECK: encoding: [0x0f,0x10,0x0d,0xf0,0x1c,0xf0,0x1c]
+movups 485498096, %xmm1
+
+// CHECK: movups 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x0f,0x10,0x4c,0x02,0x40]
+movups 64(%edx,%eax), %xmm1
+
+// CHECK: movups (%edx), %xmm1
+// CHECK: encoding: [0x0f,0x10,0x0a]
+movups (%edx), %xmm1
+
+// CHECK: movups %xmm1, 485498096
+// CHECK: encoding: [0x0f,0x11,0x0d,0xf0,0x1c,0xf0,0x1c]
+movups %xmm1, 485498096
+
+// CHECK: movups %xmm1, 485498096(%edx)
+// CHECK: encoding: [0x0f,0x11,0x8a,0xf0,0x1c,0xf0,0x1c]
+movups %xmm1, 485498096(%edx)
+
+// CHECK: movups %xmm1, -485498096(%edx,%eax,4)
+// CHECK: encoding: [0x0f,0x11,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+movups %xmm1, -485498096(%edx,%eax,4)
+
+// CHECK: movups %xmm1, 485498096(%edx,%eax,4)
+// CHECK: encoding: [0x0f,0x11,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+movups %xmm1, 485498096(%edx,%eax,4)
+
+// CHECK: movups %xmm1, 64(%edx,%eax)
+// CHECK: encoding: [0x0f,0x11,0x4c,0x02,0x40]
+movups %xmm1, 64(%edx,%eax)
+
+// CHECK: movups %xmm1, (%edx)
+// CHECK: encoding: [0x0f,0x11,0x0a]
+movups %xmm1, (%edx)
+
+// CHECK: movups %xmm1, %xmm1
+// CHECK: encoding: [0x0f,0x10,0xc9]
+movups %xmm1, %xmm1
+
+// CHECK: mulps -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x0f,0x59,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+mulps -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: mulps 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x0f,0x59,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+mulps 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: mulps 485498096(%edx), %xmm1
+// CHECK: encoding: [0x0f,0x59,0x8a,0xf0,0x1c,0xf0,0x1c]
+mulps 485498096(%edx), %xmm1
+
+// CHECK: mulps 485498096, %xmm1
+// CHECK: encoding: [0x0f,0x59,0x0d,0xf0,0x1c,0xf0,0x1c]
+mulps 485498096, %xmm1
+
+// CHECK: mulps 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x0f,0x59,0x4c,0x02,0x40]
+mulps 64(%edx,%eax), %xmm1
+
+// CHECK: mulps (%edx), %xmm1
+// CHECK: encoding: [0x0f,0x59,0x0a]
+mulps (%edx), %xmm1
+
+// CHECK: mulps %xmm1, %xmm1
+// CHECK: encoding: [0x0f,0x59,0xc9]
+mulps %xmm1, %xmm1
+
+// CHECK: mulss -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x59,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+mulss -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: mulss 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x59,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+mulss 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: mulss 485498096(%edx), %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x59,0x8a,0xf0,0x1c,0xf0,0x1c]
+mulss 485498096(%edx), %xmm1
+
+// CHECK: mulss 485498096, %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x59,0x0d,0xf0,0x1c,0xf0,0x1c]
+mulss 485498096, %xmm1
+
+// CHECK: mulss 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x59,0x4c,0x02,0x40]
+mulss 64(%edx,%eax), %xmm1
+
+// CHECK: mulss (%edx), %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x59,0x0a]
+mulss (%edx), %xmm1
+
+// CHECK: mulss %xmm1, %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x59,0xc9]
+mulss %xmm1, %xmm1
+
+// CHECK: orps -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x0f,0x56,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+orps -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: orps 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x0f,0x56,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+orps 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: orps 485498096(%edx), %xmm1
+// CHECK: encoding: [0x0f,0x56,0x8a,0xf0,0x1c,0xf0,0x1c]
+orps 485498096(%edx), %xmm1
+
+// CHECK: orps 485498096, %xmm1
+// CHECK: encoding: [0x0f,0x56,0x0d,0xf0,0x1c,0xf0,0x1c]
+orps 485498096, %xmm1
+
+// CHECK: orps 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x0f,0x56,0x4c,0x02,0x40]
+orps 64(%edx,%eax), %xmm1
+
+// CHECK: orps (%edx), %xmm1
+// CHECK: encoding: [0x0f,0x56,0x0a]
+orps (%edx), %xmm1
+
+// CHECK: orps %xmm1, %xmm1
+// CHECK: encoding: [0x0f,0x56,0xc9]
+orps %xmm1, %xmm1
+
+// CHECK: pextrw $0, %mm4, %eax
+// CHECK: encoding: [0x0f,0xc5,0xc4,0x00]
+pextrw $0, %mm4, %eax
+
+// CHECK: pinsrw $0, -485498096(%edx,%eax,4), %mm4
+// CHECK: encoding: [0x0f,0xc4,0xa4,0x82,0x10,0xe3,0x0f,0xe3,0x00]
+pinsrw $0, -485498096(%edx,%eax,4), %mm4
+
+// CHECK: pinsrw $0, 485498096(%edx,%eax,4), %mm4
+// CHECK: encoding: [0x0f,0xc4,0xa4,0x82,0xf0,0x1c,0xf0,0x1c,0x00]
+pinsrw $0, 485498096(%edx,%eax,4), %mm4
+
+// CHECK: pinsrw $0, 485498096(%edx), %mm4
+// CHECK: encoding: [0x0f,0xc4,0xa2,0xf0,0x1c,0xf0,0x1c,0x00]
+pinsrw $0, 485498096(%edx), %mm4
+
+// CHECK: pinsrw $0, 485498096, %mm4
+// CHECK: encoding: [0x0f,0xc4,0x25,0xf0,0x1c,0xf0,0x1c,0x00]
+pinsrw $0, 485498096, %mm4
+
+// CHECK: pinsrw $0, 64(%edx,%eax), %mm4
+// CHECK: encoding: [0x0f,0xc4,0x64,0x02,0x40,0x00]
+pinsrw $0, 64(%edx,%eax), %mm4
+
+// CHECK: pinsrw $0, (%edx), %mm4
+// CHECK: encoding: [0x0f,0xc4,0x22,0x00]
+pinsrw $0, (%edx), %mm4
+
+// CHECK: pinsrw $0, %eax, %mm4
+// CHECK: encoding: [0x0f,0xc4,0xe0,0x00]
+pinsrw $0, %eax, %mm4
+
+// CHECK: pmovmskb %mm4, %eax
+// CHECK: encoding: [0x0f,0xd7,0xc4]
+pmovmskb %mm4, %eax
+
+// CHECK: rcpps -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x0f,0x53,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+rcpps -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: rcpps 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x0f,0x53,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+rcpps 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: rcpps 485498096(%edx), %xmm1
+// CHECK: encoding: [0x0f,0x53,0x8a,0xf0,0x1c,0xf0,0x1c]
+rcpps 485498096(%edx), %xmm1
+
+// CHECK: rcpps 485498096, %xmm1
+// CHECK: encoding: [0x0f,0x53,0x0d,0xf0,0x1c,0xf0,0x1c]
+rcpps 485498096, %xmm1
+
+// CHECK: rcpps 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x0f,0x53,0x4c,0x02,0x40]
+rcpps 64(%edx,%eax), %xmm1
+
+// CHECK: rcpps (%edx), %xmm1
+// CHECK: encoding: [0x0f,0x53,0x0a]
+rcpps (%edx), %xmm1
+
+// CHECK: rcpps %xmm1, %xmm1
+// CHECK: encoding: [0x0f,0x53,0xc9]
+rcpps %xmm1, %xmm1
+
+// CHECK: rcpss -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x53,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+rcpss -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: rcpss 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x53,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+rcpss 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: rcpss 485498096(%edx), %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x53,0x8a,0xf0,0x1c,0xf0,0x1c]
+rcpss 485498096(%edx), %xmm1
+
+// CHECK: rcpss 485498096, %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x53,0x0d,0xf0,0x1c,0xf0,0x1c]
+rcpss 485498096, %xmm1
+
+// CHECK: rcpss 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x53,0x4c,0x02,0x40]
+rcpss 64(%edx,%eax), %xmm1
+
+// CHECK: rcpss (%edx), %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x53,0x0a]
+rcpss (%edx), %xmm1
+
+// CHECK: rcpss %xmm1, %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x53,0xc9]
+rcpss %xmm1, %xmm1
+
+// CHECK: rsqrtps -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x0f,0x52,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+rsqrtps -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: rsqrtps 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x0f,0x52,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+rsqrtps 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: rsqrtps 485498096(%edx), %xmm1
+// CHECK: encoding: [0x0f,0x52,0x8a,0xf0,0x1c,0xf0,0x1c]
+rsqrtps 485498096(%edx), %xmm1
+
+// CHECK: rsqrtps 485498096, %xmm1
+// CHECK: encoding: [0x0f,0x52,0x0d,0xf0,0x1c,0xf0,0x1c]
+rsqrtps 485498096, %xmm1
+
+// CHECK: rsqrtps 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x0f,0x52,0x4c,0x02,0x40]
+rsqrtps 64(%edx,%eax), %xmm1
+
+// CHECK: rsqrtps (%edx), %xmm1
+// CHECK: encoding: [0x0f,0x52,0x0a]
+rsqrtps (%edx), %xmm1
+
+// CHECK: rsqrtps %xmm1, %xmm1
+// CHECK: encoding: [0x0f,0x52,0xc9]
+rsqrtps %xmm1, %xmm1
+
+// CHECK: rsqrtss -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x52,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+rsqrtss -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: rsqrtss 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x52,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+rsqrtss 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: rsqrtss 485498096(%edx), %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x52,0x8a,0xf0,0x1c,0xf0,0x1c]
+rsqrtss 485498096(%edx), %xmm1
+
+// CHECK: rsqrtss 485498096, %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x52,0x0d,0xf0,0x1c,0xf0,0x1c]
+rsqrtss 485498096, %xmm1
+
+// CHECK: rsqrtss 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x52,0x4c,0x02,0x40]
+rsqrtss 64(%edx,%eax), %xmm1
+
+// CHECK: rsqrtss (%edx), %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x52,0x0a]
+rsqrtss (%edx), %xmm1
+
+// CHECK: rsqrtss %xmm1, %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x52,0xc9]
+rsqrtss %xmm1, %xmm1
+
+// CHECK: sfence
+// CHECK: encoding: [0x0f,0xae,0xf8]
+sfence
+
+// CHECK: shufps $0, -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x0f,0xc6,0x8c,0x82,0x10,0xe3,0x0f,0xe3,0x00]
+shufps $0, -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: shufps $0, 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x0f,0xc6,0x8c,0x82,0xf0,0x1c,0xf0,0x1c,0x00]
+shufps $0, 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: shufps $0, 485498096(%edx), %xmm1
+// CHECK: encoding: [0x0f,0xc6,0x8a,0xf0,0x1c,0xf0,0x1c,0x00]
+shufps $0, 485498096(%edx), %xmm1
+
+// CHECK: shufps $0, 485498096, %xmm1
+// CHECK: encoding: [0x0f,0xc6,0x0d,0xf0,0x1c,0xf0,0x1c,0x00]
+shufps $0, 485498096, %xmm1
+
+// CHECK: shufps $0, 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x0f,0xc6,0x4c,0x02,0x40,0x00]
+shufps $0, 64(%edx,%eax), %xmm1
+
+// CHECK: shufps $0, (%edx), %xmm1
+// CHECK: encoding: [0x0f,0xc6,0x0a,0x00]
+shufps $0, (%edx), %xmm1
+
+// CHECK: shufps $0, %xmm1, %xmm1
+// CHECK: encoding: [0x0f,0xc6,0xc9,0x00]
+shufps $0, %xmm1, %xmm1
+
+// CHECK: sqrtps -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x0f,0x51,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+sqrtps -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: sqrtps 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x0f,0x51,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+sqrtps 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: sqrtps 485498096(%edx), %xmm1
+// CHECK: encoding: [0x0f,0x51,0x8a,0xf0,0x1c,0xf0,0x1c]
+sqrtps 485498096(%edx), %xmm1
+
+// CHECK: sqrtps 485498096, %xmm1
+// CHECK: encoding: [0x0f,0x51,0x0d,0xf0,0x1c,0xf0,0x1c]
+sqrtps 485498096, %xmm1
+
+// CHECK: sqrtps 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x0f,0x51,0x4c,0x02,0x40]
+sqrtps 64(%edx,%eax), %xmm1
+
+// CHECK: sqrtps (%edx), %xmm1
+// CHECK: encoding: [0x0f,0x51,0x0a]
+sqrtps (%edx), %xmm1
+
+// CHECK: sqrtps %xmm1, %xmm1
+// CHECK: encoding: [0x0f,0x51,0xc9]
+sqrtps %xmm1, %xmm1
+
+// CHECK: sqrtss -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x51,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+sqrtss -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: sqrtss 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x51,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+sqrtss 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: sqrtss 485498096(%edx), %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x51,0x8a,0xf0,0x1c,0xf0,0x1c]
+sqrtss 485498096(%edx), %xmm1
+
+// CHECK: sqrtss 485498096, %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x51,0x0d,0xf0,0x1c,0xf0,0x1c]
+sqrtss 485498096, %xmm1
+
+// CHECK: sqrtss 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x51,0x4c,0x02,0x40]
+sqrtss 64(%edx,%eax), %xmm1
+
+// CHECK: sqrtss (%edx), %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x51,0x0a]
+sqrtss (%edx), %xmm1
+
+// CHECK: sqrtss %xmm1, %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x51,0xc9]
+sqrtss %xmm1, %xmm1
+
+// CHECK: subps -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x0f,0x5c,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+subps -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: subps 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x0f,0x5c,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+subps 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: subps 485498096(%edx), %xmm1
+// CHECK: encoding: [0x0f,0x5c,0x8a,0xf0,0x1c,0xf0,0x1c]
+subps 485498096(%edx), %xmm1
+
+// CHECK: subps 485498096, %xmm1
+// CHECK: encoding: [0x0f,0x5c,0x0d,0xf0,0x1c,0xf0,0x1c]
+subps 485498096, %xmm1
+
+// CHECK: subps 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x0f,0x5c,0x4c,0x02,0x40]
+subps 64(%edx,%eax), %xmm1
+
+// CHECK: subps (%edx), %xmm1
+// CHECK: encoding: [0x0f,0x5c,0x0a]
+subps (%edx), %xmm1
+
+// CHECK: subps %xmm1, %xmm1
+// CHECK: encoding: [0x0f,0x5c,0xc9]
+subps %xmm1, %xmm1
+
+// CHECK: subss -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x5c,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+subss -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: subss 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x5c,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+subss 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: subss 485498096(%edx), %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x5c,0x8a,0xf0,0x1c,0xf0,0x1c]
+subss 485498096(%edx), %xmm1
+
+// CHECK: subss 485498096, %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x5c,0x0d,0xf0,0x1c,0xf0,0x1c]
+subss 485498096, %xmm1
+
+// CHECK: subss 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x5c,0x4c,0x02,0x40]
+subss 64(%edx,%eax), %xmm1
+
+// CHECK: subss (%edx), %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x5c,0x0a]
+subss (%edx), %xmm1
+
+// CHECK: subss %xmm1, %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x5c,0xc9]
+subss %xmm1, %xmm1
+
+// CHECK: ucomiss -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x0f,0x2e,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+ucomiss -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: ucomiss 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x0f,0x2e,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+ucomiss 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: ucomiss 485498096(%edx), %xmm1
+// CHECK: encoding: [0x0f,0x2e,0x8a,0xf0,0x1c,0xf0,0x1c]
+ucomiss 485498096(%edx), %xmm1
+
+// CHECK: ucomiss 485498096, %xmm1
+// CHECK: encoding: [0x0f,0x2e,0x0d,0xf0,0x1c,0xf0,0x1c]
+ucomiss 485498096, %xmm1
+
+// CHECK: ucomiss 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x0f,0x2e,0x4c,0x02,0x40]
+ucomiss 64(%edx,%eax), %xmm1
+
+// CHECK: ucomiss (%edx), %xmm1
+// CHECK: encoding: [0x0f,0x2e,0x0a]
+ucomiss (%edx), %xmm1
+
+// CHECK: ucomiss %xmm1, %xmm1
+// CHECK: encoding: [0x0f,0x2e,0xc9]
+ucomiss %xmm1, %xmm1
+
+// CHECK: unpckhps -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x0f,0x15,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+unpckhps -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: unpckhps 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x0f,0x15,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+unpckhps 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: unpckhps 485498096(%edx), %xmm1
+// CHECK: encoding: [0x0f,0x15,0x8a,0xf0,0x1c,0xf0,0x1c]
+unpckhps 485498096(%edx), %xmm1
+
+// CHECK: unpckhps 485498096, %xmm1
+// CHECK: encoding: [0x0f,0x15,0x0d,0xf0,0x1c,0xf0,0x1c]
+unpckhps 485498096, %xmm1
+
+// CHECK: unpckhps 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x0f,0x15,0x4c,0x02,0x40]
+unpckhps 64(%edx,%eax), %xmm1
+
+// CHECK: unpckhps (%edx), %xmm1
+// CHECK: encoding: [0x0f,0x15,0x0a]
+unpckhps (%edx), %xmm1
+
+// CHECK: unpckhps %xmm1, %xmm1
+// CHECK: encoding: [0x0f,0x15,0xc9]
+unpckhps %xmm1, %xmm1
+
+// CHECK: unpcklps -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x0f,0x14,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+unpcklps -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: unpcklps 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x0f,0x14,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+unpcklps 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: unpcklps 485498096(%edx), %xmm1
+// CHECK: encoding: [0x0f,0x14,0x8a,0xf0,0x1c,0xf0,0x1c]
+unpcklps 485498096(%edx), %xmm1
+
+// CHECK: unpcklps 485498096, %xmm1
+// CHECK: encoding: [0x0f,0x14,0x0d,0xf0,0x1c,0xf0,0x1c]
+unpcklps 485498096, %xmm1
+
+// CHECK: unpcklps 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x0f,0x14,0x4c,0x02,0x40]
+unpcklps 64(%edx,%eax), %xmm1
+
+// CHECK: unpcklps (%edx), %xmm1
+// CHECK: encoding: [0x0f,0x14,0x0a]
+unpcklps (%edx), %xmm1
+
+// CHECK: unpcklps %xmm1, %xmm1
+// CHECK: encoding: [0x0f,0x14,0xc9]
+unpcklps %xmm1, %xmm1
+
+// CHECK: xorps -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x0f,0x57,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+xorps -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: xorps 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x0f,0x57,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+xorps 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: xorps 485498096(%edx), %xmm1
+// CHECK: encoding: [0x0f,0x57,0x8a,0xf0,0x1c,0xf0,0x1c]
+xorps 485498096(%edx), %xmm1
+
+// CHECK: xorps 485498096, %xmm1
+// CHECK: encoding: [0x0f,0x57,0x0d,0xf0,0x1c,0xf0,0x1c]
+xorps 485498096, %xmm1
+
+// CHECK: xorps 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x0f,0x57,0x4c,0x02,0x40]
+xorps 64(%edx,%eax), %xmm1
+
+// CHECK: xorps (%edx), %xmm1
+// CHECK: encoding: [0x0f,0x57,0x0a]
+xorps (%edx), %xmm1
+
+// CHECK: xorps %xmm1, %xmm1
+// CHECK: encoding: [0x0f,0x57,0xc9]
+xorps %xmm1, %xmm1
+
diff --git a/test/MC/X86/SSE-64.s b/test/MC/X86/SSE-64.s
new file mode 100644
index 0000000000000000000000000000000000000000..af0a717977b011756452a915cfbab06ac3caa5a7
--- /dev/null
+++ b/test/MC/X86/SSE-64.s
@@ -0,0 +1,1398 @@
+// RUN: llvm-mc -triple x86_64-unknown-unknown --show-encoding %s | FileCheck %s
+
+// CHECK: addps 485498096, %xmm6
+// CHECK: encoding: [0x0f,0x58,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+addps 485498096, %xmm6
+
+// CHECK: addps -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x0f,0x58,0x74,0x82,0xc0]
+addps -64(%rdx,%rax,4), %xmm6
+
+// CHECK: addps 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x0f,0x58,0x74,0x82,0x40]
+addps 64(%rdx,%rax,4), %xmm6
+
+// CHECK: addps 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x0f,0x58,0x74,0x02,0x40]
+addps 64(%rdx,%rax), %xmm6
+
+// CHECK: addps 64(%rdx), %xmm6
+// CHECK: encoding: [0x0f,0x58,0x72,0x40]
+addps 64(%rdx), %xmm6
+
+// CHECK: addps (%rdx), %xmm6
+// CHECK: encoding: [0x0f,0x58,0x32]
+addps (%rdx), %xmm6
+
+// CHECK: addps %xmm6, %xmm6
+// CHECK: encoding: [0x0f,0x58,0xf6]
+addps %xmm6, %xmm6
+
+// CHECK: addss 485498096, %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x58,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+addss 485498096, %xmm6
+
+// CHECK: addss -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x58,0x74,0x82,0xc0]
+addss -64(%rdx,%rax,4), %xmm6
+
+// CHECK: addss 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x58,0x74,0x82,0x40]
+addss 64(%rdx,%rax,4), %xmm6
+
+// CHECK: addss 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x58,0x74,0x02,0x40]
+addss 64(%rdx,%rax), %xmm6
+
+// CHECK: addss 64(%rdx), %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x58,0x72,0x40]
+addss 64(%rdx), %xmm6
+
+// CHECK: addss (%rdx), %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x58,0x32]
+addss (%rdx), %xmm6
+
+// CHECK: addss %xmm6, %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x58,0xf6]
+addss %xmm6, %xmm6
+
+// CHECK: andnps 485498096, %xmm6
+// CHECK: encoding: [0x0f,0x55,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+andnps 485498096, %xmm6
+
+// CHECK: andnps -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x0f,0x55,0x74,0x82,0xc0]
+andnps -64(%rdx,%rax,4), %xmm6
+
+// CHECK: andnps 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x0f,0x55,0x74,0x82,0x40]
+andnps 64(%rdx,%rax,4), %xmm6
+
+// CHECK: andnps 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x0f,0x55,0x74,0x02,0x40]
+andnps 64(%rdx,%rax), %xmm6
+
+// CHECK: andnps 64(%rdx), %xmm6
+// CHECK: encoding: [0x0f,0x55,0x72,0x40]
+andnps 64(%rdx), %xmm6
+
+// CHECK: andnps (%rdx), %xmm6
+// CHECK: encoding: [0x0f,0x55,0x32]
+andnps (%rdx), %xmm6
+
+// CHECK: andnps %xmm6, %xmm6
+// CHECK: encoding: [0x0f,0x55,0xf6]
+andnps %xmm6, %xmm6
+
+// CHECK: andps 485498096, %xmm6
+// CHECK: encoding: [0x0f,0x54,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+andps 485498096, %xmm6
+
+// CHECK: andps -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x0f,0x54,0x74,0x82,0xc0]
+andps -64(%rdx,%rax,4), %xmm6
+
+// CHECK: andps 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x0f,0x54,0x74,0x82,0x40]
+andps 64(%rdx,%rax,4), %xmm6
+
+// CHECK: andps 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x0f,0x54,0x74,0x02,0x40]
+andps 64(%rdx,%rax), %xmm6
+
+// CHECK: andps 64(%rdx), %xmm6
+// CHECK: encoding: [0x0f,0x54,0x72,0x40]
+andps 64(%rdx), %xmm6
+
+// CHECK: andps (%rdx), %xmm6
+// CHECK: encoding: [0x0f,0x54,0x32]
+andps (%rdx), %xmm6
+
+// CHECK: andps %xmm6, %xmm6
+// CHECK: encoding: [0x0f,0x54,0xf6]
+andps %xmm6, %xmm6
+
+// CHECK: cmpeqps 485498096, %xmm6
+// CHECK: encoding: [0x0f,0xc2,0x34,0x25,0xf0,0x1c,0xf0,0x1c,0x00]
+cmpeqps 485498096, %xmm6
+
+// CHECK: cmpeqps -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x0f,0xc2,0x74,0x82,0xc0,0x00]
+cmpeqps -64(%rdx,%rax,4), %xmm6
+
+// CHECK: cmpeqps 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x0f,0xc2,0x74,0x82,0x40,0x00]
+cmpeqps 64(%rdx,%rax,4), %xmm6
+
+// CHECK: cmpeqps 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x0f,0xc2,0x74,0x02,0x40,0x00]
+cmpeqps 64(%rdx,%rax), %xmm6
+
+// CHECK: cmpeqps 64(%rdx), %xmm6
+// CHECK: encoding: [0x0f,0xc2,0x72,0x40,0x00]
+cmpeqps 64(%rdx), %xmm6
+
+// CHECK: cmpeqps (%rdx), %xmm6
+// CHECK: encoding: [0x0f,0xc2,0x32,0x00]
+cmpeqps (%rdx), %xmm6
+
+// CHECK: cmpeqps %xmm6, %xmm6
+// CHECK: encoding: [0x0f,0xc2,0xf6,0x00]
+cmpeqps %xmm6, %xmm6
+
+// CHECK: cmpeqss 485498096, %xmm6
+// CHECK: encoding: [0xf3,0x0f,0xc2,0x34,0x25,0xf0,0x1c,0xf0,0x1c,0x00]
+cmpeqss 485498096, %xmm6
+
+// CHECK: cmpeqss -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0xf3,0x0f,0xc2,0x74,0x82,0xc0,0x00]
+cmpeqss -64(%rdx,%rax,4), %xmm6
+
+// CHECK: cmpeqss 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0xf3,0x0f,0xc2,0x74,0x82,0x40,0x00]
+cmpeqss 64(%rdx,%rax,4), %xmm6
+
+// CHECK: cmpeqss 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0xf3,0x0f,0xc2,0x74,0x02,0x40,0x00]
+cmpeqss 64(%rdx,%rax), %xmm6
+
+// CHECK: cmpeqss 64(%rdx), %xmm6
+// CHECK: encoding: [0xf3,0x0f,0xc2,0x72,0x40,0x00]
+cmpeqss 64(%rdx), %xmm6
+
+// CHECK: cmpeqss (%rdx), %xmm6
+// CHECK: encoding: [0xf3,0x0f,0xc2,0x32,0x00]
+cmpeqss (%rdx), %xmm6
+
+// CHECK: cmpeqss %xmm6, %xmm6
+// CHECK: encoding: [0xf3,0x0f,0xc2,0xf6,0x00]
+cmpeqss %xmm6, %xmm6
+
+// CHECK: comiss 485498096, %xmm6
+// CHECK: encoding: [0x0f,0x2f,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+comiss 485498096, %xmm6
+
+// CHECK: comiss -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x0f,0x2f,0x74,0x82,0xc0]
+comiss -64(%rdx,%rax,4), %xmm6
+
+// CHECK: comiss 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x0f,0x2f,0x74,0x82,0x40]
+comiss 64(%rdx,%rax,4), %xmm6
+
+// CHECK: comiss 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x0f,0x2f,0x74,0x02,0x40]
+comiss 64(%rdx,%rax), %xmm6
+
+// CHECK: comiss 64(%rdx), %xmm6
+// CHECK: encoding: [0x0f,0x2f,0x72,0x40]
+comiss 64(%rdx), %xmm6
+
+// CHECK: comiss (%rdx), %xmm6
+// CHECK: encoding: [0x0f,0x2f,0x32]
+comiss (%rdx), %xmm6
+
+// CHECK: comiss %xmm6, %xmm6
+// CHECK: encoding: [0x0f,0x2f,0xf6]
+comiss %xmm6, %xmm6
+
+// CHECK: cvtpi2ps 485498096, %xmm6
+// CHECK: encoding: [0x0f,0x2a,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+cvtpi2ps 485498096, %xmm6
+
+// CHECK: cvtpi2ps -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x0f,0x2a,0x74,0x82,0xc0]
+cvtpi2ps -64(%rdx,%rax,4), %xmm6
+
+// CHECK: cvtpi2ps 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x0f,0x2a,0x74,0x82,0x40]
+cvtpi2ps 64(%rdx,%rax,4), %xmm6
+
+// CHECK: cvtpi2ps 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x0f,0x2a,0x74,0x02,0x40]
+cvtpi2ps 64(%rdx,%rax), %xmm6
+
+// CHECK: cvtpi2ps 64(%rdx), %xmm6
+// CHECK: encoding: [0x0f,0x2a,0x72,0x40]
+cvtpi2ps 64(%rdx), %xmm6
+
+// CHECK: cvtpi2ps %mm4, %xmm6
+// CHECK: encoding: [0x0f,0x2a,0xf4]
+cvtpi2ps %mm4, %xmm6
+
+// CHECK: cvtpi2ps (%rdx), %xmm6
+// CHECK: encoding: [0x0f,0x2a,0x32]
+cvtpi2ps (%rdx), %xmm6
+
+// CHECK: cvtps2pi 485498096, %mm4
+// CHECK: encoding: [0x0f,0x2d,0x24,0x25,0xf0,0x1c,0xf0,0x1c]
+cvtps2pi 485498096, %mm4
+
+// CHECK: cvtps2pi 64(%rdx), %mm4
+// CHECK: encoding: [0x0f,0x2d,0x62,0x40]
+cvtps2pi 64(%rdx), %mm4
+
+// CHECK: cvtps2pi -64(%rdx,%rax,4), %mm4
+// CHECK: encoding: [0x0f,0x2d,0x64,0x82,0xc0]
+cvtps2pi -64(%rdx,%rax,4), %mm4
+
+// CHECK: cvtps2pi 64(%rdx,%rax,4), %mm4
+// CHECK: encoding: [0x0f,0x2d,0x64,0x82,0x40]
+cvtps2pi 64(%rdx,%rax,4), %mm4
+
+// CHECK: cvtps2pi 64(%rdx,%rax), %mm4
+// CHECK: encoding: [0x0f,0x2d,0x64,0x02,0x40]
+cvtps2pi 64(%rdx,%rax), %mm4
+
+// CHECK: cvtps2pi (%rdx), %mm4
+// CHECK: encoding: [0x0f,0x2d,0x22]
+cvtps2pi (%rdx), %mm4
+
+// CHECK: cvtps2pi %xmm6, %mm4
+// CHECK: encoding: [0x0f,0x2d,0xe6]
+cvtps2pi %xmm6, %mm4
+
+// CHECK: cvtsi2ssl 485498096, %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x2a,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+cvtsi2ssl 485498096, %xmm6
+
+// CHECK: cvtsi2ssl -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x2a,0x74,0x82,0xc0]
+cvtsi2ssl -64(%rdx,%rax,4), %xmm6
+
+// CHECK: cvtsi2ssl 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x2a,0x74,0x82,0x40]
+cvtsi2ssl 64(%rdx,%rax,4), %xmm6
+
+// CHECK: cvtsi2ssl 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x2a,0x74,0x02,0x40]
+cvtsi2ssl 64(%rdx,%rax), %xmm6
+
+// CHECK: cvtsi2ssl 64(%rdx), %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x2a,0x72,0x40]
+cvtsi2ssl 64(%rdx), %xmm6
+
+// CHECK: cvtsi2ssl %r13d, %xmm6
+// CHECK: encoding: [0xf3,0x41,0x0f,0x2a,0xf5]
+cvtsi2ssl %r13d, %xmm6
+
+// CHECK: cvtsi2ssl (%rdx), %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x2a,0x32]
+cvtsi2ssl (%rdx), %xmm6
+
+// CHECK: cvtsi2ssq 485498096, %xmm6
+// CHECK: encoding: [0xf3,0x48,0x0f,0x2a,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+cvtsi2ssq 485498096, %xmm6
+
+// CHECK: cvtsi2ssq -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0xf3,0x48,0x0f,0x2a,0x74,0x82,0xc0]
+cvtsi2ssq -64(%rdx,%rax,4), %xmm6
+
+// CHECK: cvtsi2ssq 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0xf3,0x48,0x0f,0x2a,0x74,0x82,0x40]
+cvtsi2ssq 64(%rdx,%rax,4), %xmm6
+
+// CHECK: cvtsi2ssq 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0xf3,0x48,0x0f,0x2a,0x74,0x02,0x40]
+cvtsi2ssq 64(%rdx,%rax), %xmm6
+
+// CHECK: cvtsi2ssq 64(%rdx), %xmm6
+// CHECK: encoding: [0xf3,0x48,0x0f,0x2a,0x72,0x40]
+cvtsi2ssq 64(%rdx), %xmm6
+
+// CHECK: cvtsi2ssq %r15, %xmm6
+// CHECK: encoding: [0xf3,0x49,0x0f,0x2a,0xf7]
+cvtsi2ssq %r15, %xmm6
+
+// CHECK: cvtsi2ssq (%rdx), %xmm6
+// CHECK: encoding: [0xf3,0x48,0x0f,0x2a,0x32]
+cvtsi2ssq (%rdx), %xmm6
+
+// CHECK: cvtss2si 485498096, %r13d
+// CHECK: encoding: [0xf3,0x44,0x0f,0x2d,0x2c,0x25,0xf0,0x1c,0xf0,0x1c]
+cvtss2si 485498096, %r13d
+
+// CHECK: cvtss2si 485498096, %r15
+// CHECK: encoding: [0xf3,0x4c,0x0f,0x2d,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+cvtss2si 485498096, %r15
+
+// CHECK: cvtss2si 64(%rdx), %r13d
+// CHECK: encoding: [0xf3,0x44,0x0f,0x2d,0x6a,0x40]
+cvtss2si 64(%rdx), %r13d
+
+// CHECK: cvtss2si 64(%rdx), %r15
+// CHECK: encoding: [0xf3,0x4c,0x0f,0x2d,0x7a,0x40]
+cvtss2si 64(%rdx), %r15
+
+// CHECK: cvtss2si -64(%rdx,%rax,4), %r13d
+// CHECK: encoding: [0xf3,0x44,0x0f,0x2d,0x6c,0x82,0xc0]
+cvtss2si -64(%rdx,%rax,4), %r13d
+
+// CHECK: cvtss2si 64(%rdx,%rax,4), %r13d
+// CHECK: encoding: [0xf3,0x44,0x0f,0x2d,0x6c,0x82,0x40]
+cvtss2si 64(%rdx,%rax,4), %r13d
+
+// CHECK: cvtss2si -64(%rdx,%rax,4), %r15
+// CHECK: encoding: [0xf3,0x4c,0x0f,0x2d,0x7c,0x82,0xc0]
+cvtss2si -64(%rdx,%rax,4), %r15
+
+// CHECK: cvtss2si 64(%rdx,%rax,4), %r15
+// CHECK: encoding: [0xf3,0x4c,0x0f,0x2d,0x7c,0x82,0x40]
+cvtss2si 64(%rdx,%rax,4), %r15
+
+// CHECK: cvtss2si 64(%rdx,%rax), %r13d
+// CHECK: encoding: [0xf3,0x44,0x0f,0x2d,0x6c,0x02,0x40]
+cvtss2si 64(%rdx,%rax), %r13d
+
+// CHECK: cvtss2si 64(%rdx,%rax), %r15
+// CHECK: encoding: [0xf3,0x4c,0x0f,0x2d,0x7c,0x02,0x40]
+cvtss2si 64(%rdx,%rax), %r15
+
+// CHECK: cvtss2si (%rdx), %r13d
+// CHECK: encoding: [0xf3,0x44,0x0f,0x2d,0x2a]
+cvtss2si (%rdx), %r13d
+
+// CHECK: cvtss2si (%rdx), %r15
+// CHECK: encoding: [0xf3,0x4c,0x0f,0x2d,0x3a]
+cvtss2si (%rdx), %r15
+
+// CHECK: cvtss2si %xmm6, %r13d
+// CHECK: encoding: [0xf3,0x44,0x0f,0x2d,0xee]
+cvtss2si %xmm6, %r13d
+
+// CHECK: cvtss2si %xmm6, %r15
+// CHECK: encoding: [0xf3,0x4c,0x0f,0x2d,0xfe]
+cvtss2si %xmm6, %r15
+
+// CHECK: cvttps2pi 485498096, %mm4
+// CHECK: encoding: [0x0f,0x2c,0x24,0x25,0xf0,0x1c,0xf0,0x1c]
+cvttps2pi 485498096, %mm4
+
+// CHECK: cvttps2pi 64(%rdx), %mm4
+// CHECK: encoding: [0x0f,0x2c,0x62,0x40]
+cvttps2pi 64(%rdx), %mm4
+
+// CHECK: cvttps2pi -64(%rdx,%rax,4), %mm4
+// CHECK: encoding: [0x0f,0x2c,0x64,0x82,0xc0]
+cvttps2pi -64(%rdx,%rax,4), %mm4
+
+// CHECK: cvttps2pi 64(%rdx,%rax,4), %mm4
+// CHECK: encoding: [0x0f,0x2c,0x64,0x82,0x40]
+cvttps2pi 64(%rdx,%rax,4), %mm4
+
+// CHECK: cvttps2pi 64(%rdx,%rax), %mm4
+// CHECK: encoding: [0x0f,0x2c,0x64,0x02,0x40]
+cvttps2pi 64(%rdx,%rax), %mm4
+
+// CHECK: cvttps2pi (%rdx), %mm4
+// CHECK: encoding: [0x0f,0x2c,0x22]
+cvttps2pi (%rdx), %mm4
+
+// CHECK: cvttps2pi %xmm6, %mm4
+// CHECK: encoding: [0x0f,0x2c,0xe6]
+cvttps2pi %xmm6, %mm4
+
+// CHECK: cvttss2si 485498096, %r13d
+// CHECK: encoding: [0xf3,0x44,0x0f,0x2c,0x2c,0x25,0xf0,0x1c,0xf0,0x1c]
+cvttss2si 485498096, %r13d
+
+// CHECK: cvttss2si 485498096, %r15
+// CHECK: encoding: [0xf3,0x4c,0x0f,0x2c,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+cvttss2si 485498096, %r15
+
+// CHECK: cvttss2si 64(%rdx), %r13d
+// CHECK: encoding: [0xf3,0x44,0x0f,0x2c,0x6a,0x40]
+cvttss2si 64(%rdx), %r13d
+
+// CHECK: cvttss2si 64(%rdx), %r15
+// CHECK: encoding: [0xf3,0x4c,0x0f,0x2c,0x7a,0x40]
+cvttss2si 64(%rdx), %r15
+
+// CHECK: cvttss2si -64(%rdx,%rax,4), %r13d
+// CHECK: encoding: [0xf3,0x44,0x0f,0x2c,0x6c,0x82,0xc0]
+cvttss2si -64(%rdx,%rax,4), %r13d
+
+// CHECK: cvttss2si 64(%rdx,%rax,4), %r13d
+// CHECK: encoding: [0xf3,0x44,0x0f,0x2c,0x6c,0x82,0x40]
+cvttss2si 64(%rdx,%rax,4), %r13d
+
+// CHECK: cvttss2si -64(%rdx,%rax,4), %r15
+// CHECK: encoding: [0xf3,0x4c,0x0f,0x2c,0x7c,0x82,0xc0]
+cvttss2si -64(%rdx,%rax,4), %r15
+
+// CHECK: cvttss2si 64(%rdx,%rax,4), %r15
+// CHECK: encoding: [0xf3,0x4c,0x0f,0x2c,0x7c,0x82,0x40]
+cvttss2si 64(%rdx,%rax,4), %r15
+
+// CHECK: cvttss2si 64(%rdx,%rax), %r13d
+// CHECK: encoding: [0xf3,0x44,0x0f,0x2c,0x6c,0x02,0x40]
+cvttss2si 64(%rdx,%rax), %r13d
+
+// CHECK: cvttss2si 64(%rdx,%rax), %r15
+// CHECK: encoding: [0xf3,0x4c,0x0f,0x2c,0x7c,0x02,0x40]
+cvttss2si 64(%rdx,%rax), %r15
+
+// CHECK: cvttss2si (%rdx), %r13d
+// CHECK: encoding: [0xf3,0x44,0x0f,0x2c,0x2a]
+cvttss2si (%rdx), %r13d
+
+// CHECK: cvttss2si (%rdx), %r15
+// CHECK: encoding: [0xf3,0x4c,0x0f,0x2c,0x3a]
+cvttss2si (%rdx), %r15
+
+// CHECK: cvttss2si %xmm6, %r13d
+// CHECK: encoding: [0xf3,0x44,0x0f,0x2c,0xee]
+cvttss2si %xmm6, %r13d
+
+// CHECK: cvttss2si %xmm6, %r15
+// CHECK: encoding: [0xf3,0x4c,0x0f,0x2c,0xfe]
+cvttss2si %xmm6, %r15
+
+// CHECK: divps 485498096, %xmm6
+// CHECK: encoding: [0x0f,0x5e,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+divps 485498096, %xmm6
+
+// CHECK: divps -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x0f,0x5e,0x74,0x82,0xc0]
+divps -64(%rdx,%rax,4), %xmm6
+
+// CHECK: divps 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x0f,0x5e,0x74,0x82,0x40]
+divps 64(%rdx,%rax,4), %xmm6
+
+// CHECK: divps 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x0f,0x5e,0x74,0x02,0x40]
+divps 64(%rdx,%rax), %xmm6
+
+// CHECK: divps 64(%rdx), %xmm6
+// CHECK: encoding: [0x0f,0x5e,0x72,0x40]
+divps 64(%rdx), %xmm6
+
+// CHECK: divps (%rdx), %xmm6
+// CHECK: encoding: [0x0f,0x5e,0x32]
+divps (%rdx), %xmm6
+
+// CHECK: divps %xmm6, %xmm6
+// CHECK: encoding: [0x0f,0x5e,0xf6]
+divps %xmm6, %xmm6
+
+// CHECK: divss 485498096, %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x5e,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+divss 485498096, %xmm6
+
+// CHECK: divss -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x5e,0x74,0x82,0xc0]
+divss -64(%rdx,%rax,4), %xmm6
+
+// CHECK: divss 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x5e,0x74,0x82,0x40]
+divss 64(%rdx,%rax,4), %xmm6
+
+// CHECK: divss 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x5e,0x74,0x02,0x40]
+divss 64(%rdx,%rax), %xmm6
+
+// CHECK: divss 64(%rdx), %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x5e,0x72,0x40]
+divss 64(%rdx), %xmm6
+
+// CHECK: divss (%rdx), %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x5e,0x32]
+divss (%rdx), %xmm6
+
+// CHECK: divss %xmm6, %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x5e,0xf6]
+divss %xmm6, %xmm6
+
+// CHECK: maxps 485498096, %xmm6
+// CHECK: encoding: [0x0f,0x5f,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+maxps 485498096, %xmm6
+
+// CHECK: maxps -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x0f,0x5f,0x74,0x82,0xc0]
+maxps -64(%rdx,%rax,4), %xmm6
+
+// CHECK: maxps 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x0f,0x5f,0x74,0x82,0x40]
+maxps 64(%rdx,%rax,4), %xmm6
+
+// CHECK: maxps 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x0f,0x5f,0x74,0x02,0x40]
+maxps 64(%rdx,%rax), %xmm6
+
+// CHECK: maxps 64(%rdx), %xmm6
+// CHECK: encoding: [0x0f,0x5f,0x72,0x40]
+maxps 64(%rdx), %xmm6
+
+// CHECK: maxps (%rdx), %xmm6
+// CHECK: encoding: [0x0f,0x5f,0x32]
+maxps (%rdx), %xmm6
+
+// CHECK: maxps %xmm6, %xmm6
+// CHECK: encoding: [0x0f,0x5f,0xf6]
+maxps %xmm6, %xmm6
+
+// CHECK: maxss 485498096, %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x5f,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+maxss 485498096, %xmm6
+
+// CHECK: maxss -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x5f,0x74,0x82,0xc0]
+maxss -64(%rdx,%rax,4), %xmm6
+
+// CHECK: maxss 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x5f,0x74,0x82,0x40]
+maxss 64(%rdx,%rax,4), %xmm6
+
+// CHECK: maxss 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x5f,0x74,0x02,0x40]
+maxss 64(%rdx,%rax), %xmm6
+
+// CHECK: maxss 64(%rdx), %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x5f,0x72,0x40]
+maxss 64(%rdx), %xmm6
+
+// CHECK: maxss (%rdx), %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x5f,0x32]
+maxss (%rdx), %xmm6
+
+// CHECK: maxss %xmm6, %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x5f,0xf6]
+maxss %xmm6, %xmm6
+
+// CHECK: minps 485498096, %xmm6
+// CHECK: encoding: [0x0f,0x5d,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+minps 485498096, %xmm6
+
+// CHECK: minps -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x0f,0x5d,0x74,0x82,0xc0]
+minps -64(%rdx,%rax,4), %xmm6
+
+// CHECK: minps 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x0f,0x5d,0x74,0x82,0x40]
+minps 64(%rdx,%rax,4), %xmm6
+
+// CHECK: minps 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x0f,0x5d,0x74,0x02,0x40]
+minps 64(%rdx,%rax), %xmm6
+
+// CHECK: minps 64(%rdx), %xmm6
+// CHECK: encoding: [0x0f,0x5d,0x72,0x40]
+minps 64(%rdx), %xmm6
+
+// CHECK: minps (%rdx), %xmm6
+// CHECK: encoding: [0x0f,0x5d,0x32]
+minps (%rdx), %xmm6
+
+// CHECK: minps %xmm6, %xmm6
+// CHECK: encoding: [0x0f,0x5d,0xf6]
+minps %xmm6, %xmm6
+
+// CHECK: minss 485498096, %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x5d,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+minss 485498096, %xmm6
+
+// CHECK: minss -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x5d,0x74,0x82,0xc0]
+minss -64(%rdx,%rax,4), %xmm6
+
+// CHECK: minss 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x5d,0x74,0x82,0x40]
+minss 64(%rdx,%rax,4), %xmm6
+
+// CHECK: minss 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x5d,0x74,0x02,0x40]
+minss 64(%rdx,%rax), %xmm6
+
+// CHECK: minss 64(%rdx), %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x5d,0x72,0x40]
+minss 64(%rdx), %xmm6
+
+// CHECK: minss (%rdx), %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x5d,0x32]
+minss (%rdx), %xmm6
+
+// CHECK: minss %xmm6, %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x5d,0xf6]
+minss %xmm6, %xmm6
+
+// CHECK: movaps 485498096, %xmm6
+// CHECK: encoding: [0x0f,0x28,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+movaps 485498096, %xmm6
+
+// CHECK: movaps -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x0f,0x28,0x74,0x82,0xc0]
+movaps -64(%rdx,%rax,4), %xmm6
+
+// CHECK: movaps 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x0f,0x28,0x74,0x82,0x40]
+movaps 64(%rdx,%rax,4), %xmm6
+
+// CHECK: movaps 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x0f,0x28,0x74,0x02,0x40]
+movaps 64(%rdx,%rax), %xmm6
+
+// CHECK: movaps 64(%rdx), %xmm6
+// CHECK: encoding: [0x0f,0x28,0x72,0x40]
+movaps 64(%rdx), %xmm6
+
+// CHECK: movaps (%rdx), %xmm6
+// CHECK: encoding: [0x0f,0x28,0x32]
+movaps (%rdx), %xmm6
+
+// CHECK: movaps %xmm6, 485498096
+// CHECK: encoding: [0x0f,0x29,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+movaps %xmm6, 485498096
+
+// CHECK: movaps %xmm6, 64(%rdx)
+// CHECK: encoding: [0x0f,0x29,0x72,0x40]
+movaps %xmm6, 64(%rdx)
+
+// CHECK: movaps %xmm6, 64(%rdx,%rax)
+// CHECK: encoding: [0x0f,0x29,0x74,0x02,0x40]
+movaps %xmm6, 64(%rdx,%rax)
+
+// CHECK: movaps %xmm6, -64(%rdx,%rax,4)
+// CHECK: encoding: [0x0f,0x29,0x74,0x82,0xc0]
+movaps %xmm6, -64(%rdx,%rax,4)
+
+// CHECK: movaps %xmm6, 64(%rdx,%rax,4)
+// CHECK: encoding: [0x0f,0x29,0x74,0x82,0x40]
+movaps %xmm6, 64(%rdx,%rax,4)
+
+// CHECK: movaps %xmm6, (%rdx)
+// CHECK: encoding: [0x0f,0x29,0x32]
+movaps %xmm6, (%rdx)
+
+// CHECK: movaps %xmm6, %xmm6
+// CHECK: encoding: [0x0f,0x28,0xf6]
+movaps %xmm6, %xmm6
+
+// CHECK: movhlps %xmm6, %xmm6
+// CHECK: encoding: [0x0f,0x12,0xf6]
+movhlps %xmm6, %xmm6
+
+// CHECK: movhps 485498096, %xmm6
+// CHECK: encoding: [0x0f,0x16,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+movhps 485498096, %xmm6
+
+// CHECK: movhps -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x0f,0x16,0x74,0x82,0xc0]
+movhps -64(%rdx,%rax,4), %xmm6
+
+// CHECK: movhps 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x0f,0x16,0x74,0x82,0x40]
+movhps 64(%rdx,%rax,4), %xmm6
+
+// CHECK: movhps 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x0f,0x16,0x74,0x02,0x40]
+movhps 64(%rdx,%rax), %xmm6
+
+// CHECK: movhps 64(%rdx), %xmm6
+// CHECK: encoding: [0x0f,0x16,0x72,0x40]
+movhps 64(%rdx), %xmm6
+
+// CHECK: movhps (%rdx), %xmm6
+// CHECK: encoding: [0x0f,0x16,0x32]
+movhps (%rdx), %xmm6
+
+// CHECK: movhps %xmm6, 485498096
+// CHECK: encoding: [0x0f,0x17,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+movhps %xmm6, 485498096
+
+// CHECK: movhps %xmm6, 64(%rdx)
+// CHECK: encoding: [0x0f,0x17,0x72,0x40]
+movhps %xmm6, 64(%rdx)
+
+// CHECK: movhps %xmm6, 64(%rdx,%rax)
+// CHECK: encoding: [0x0f,0x17,0x74,0x02,0x40]
+movhps %xmm6, 64(%rdx,%rax)
+
+// CHECK: movhps %xmm6, -64(%rdx,%rax,4)
+// CHECK: encoding: [0x0f,0x17,0x74,0x82,0xc0]
+movhps %xmm6, -64(%rdx,%rax,4)
+
+// CHECK: movhps %xmm6, 64(%rdx,%rax,4)
+// CHECK: encoding: [0x0f,0x17,0x74,0x82,0x40]
+movhps %xmm6, 64(%rdx,%rax,4)
+
+// CHECK: movhps %xmm6, (%rdx)
+// CHECK: encoding: [0x0f,0x17,0x32]
+movhps %xmm6, (%rdx)
+
+// CHECK: movlhps %xmm6, %xmm6
+// CHECK: encoding: [0x0f,0x16,0xf6]
+movlhps %xmm6, %xmm6
+
+// CHECK: movlps 485498096, %xmm6
+// CHECK: encoding: [0x0f,0x12,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+movlps 485498096, %xmm6
+
+// CHECK: movlps -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x0f,0x12,0x74,0x82,0xc0]
+movlps -64(%rdx,%rax,4), %xmm6
+
+// CHECK: movlps 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x0f,0x12,0x74,0x82,0x40]
+movlps 64(%rdx,%rax,4), %xmm6
+
+// CHECK: movlps 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x0f,0x12,0x74,0x02,0x40]
+movlps 64(%rdx,%rax), %xmm6
+
+// CHECK: movlps 64(%rdx), %xmm6
+// CHECK: encoding: [0x0f,0x12,0x72,0x40]
+movlps 64(%rdx), %xmm6
+
+// CHECK: movlps (%rdx), %xmm6
+// CHECK: encoding: [0x0f,0x12,0x32]
+movlps (%rdx), %xmm6
+
+// CHECK: movlps %xmm6, 485498096
+// CHECK: encoding: [0x0f,0x13,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+movlps %xmm6, 485498096
+
+// CHECK: movlps %xmm6, 64(%rdx)
+// CHECK: encoding: [0x0f,0x13,0x72,0x40]
+movlps %xmm6, 64(%rdx)
+
+// CHECK: movlps %xmm6, 64(%rdx,%rax)
+// CHECK: encoding: [0x0f,0x13,0x74,0x02,0x40]
+movlps %xmm6, 64(%rdx,%rax)
+
+// CHECK: movlps %xmm6, -64(%rdx,%rax,4)
+// CHECK: encoding: [0x0f,0x13,0x74,0x82,0xc0]
+movlps %xmm6, -64(%rdx,%rax,4)
+
+// CHECK: movlps %xmm6, 64(%rdx,%rax,4)
+// CHECK: encoding: [0x0f,0x13,0x74,0x82,0x40]
+movlps %xmm6, 64(%rdx,%rax,4)
+
+// CHECK: movlps %xmm6, (%rdx)
+// CHECK: encoding: [0x0f,0x13,0x32]
+movlps %xmm6, (%rdx)
+
+// CHECK: movmskps %xmm6, %r13d
+// CHECK: encoding: [0x44,0x0f,0x50,0xee]
+movmskps %xmm6, %r13d
+
+// CHECK: movntps %xmm6, 485498096
+// CHECK: encoding: [0x0f,0x2b,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+movntps %xmm6, 485498096
+
+// CHECK: movntps %xmm6, 64(%rdx)
+// CHECK: encoding: [0x0f,0x2b,0x72,0x40]
+movntps %xmm6, 64(%rdx)
+
+// CHECK: movntps %xmm6, 64(%rdx,%rax)
+// CHECK: encoding: [0x0f,0x2b,0x74,0x02,0x40]
+movntps %xmm6, 64(%rdx,%rax)
+
+// CHECK: movntps %xmm6, -64(%rdx,%rax,4)
+// CHECK: encoding: [0x0f,0x2b,0x74,0x82,0xc0]
+movntps %xmm6, -64(%rdx,%rax,4)
+
+// CHECK: movntps %xmm6, 64(%rdx,%rax,4)
+// CHECK: encoding: [0x0f,0x2b,0x74,0x82,0x40]
+movntps %xmm6, 64(%rdx,%rax,4)
+
+// CHECK: movntps %xmm6, (%rdx)
+// CHECK: encoding: [0x0f,0x2b,0x32]
+movntps %xmm6, (%rdx)
+
+// CHECK: movss 485498096, %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x10,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+movss 485498096, %xmm6
+
+// CHECK: movss -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x10,0x74,0x82,0xc0]
+movss -64(%rdx,%rax,4), %xmm6
+
+// CHECK: movss 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x10,0x74,0x82,0x40]
+movss 64(%rdx,%rax,4), %xmm6
+
+// CHECK: movss 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x10,0x74,0x02,0x40]
+movss 64(%rdx,%rax), %xmm6
+
+// CHECK: movss 64(%rdx), %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x10,0x72,0x40]
+movss 64(%rdx), %xmm6
+
+// CHECK: movss (%rdx), %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x10,0x32]
+movss (%rdx), %xmm6
+
+// CHECK: movss %xmm6, 485498096
+// CHECK: encoding: [0xf3,0x0f,0x11,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+movss %xmm6, 485498096
+
+// CHECK: movss %xmm6, 64(%rdx)
+// CHECK: encoding: [0xf3,0x0f,0x11,0x72,0x40]
+movss %xmm6, 64(%rdx)
+
+// CHECK: movss %xmm6, 64(%rdx,%rax)
+// CHECK: encoding: [0xf3,0x0f,0x11,0x74,0x02,0x40]
+movss %xmm6, 64(%rdx,%rax)
+
+// CHECK: movss %xmm6, -64(%rdx,%rax,4)
+// CHECK: encoding: [0xf3,0x0f,0x11,0x74,0x82,0xc0]
+movss %xmm6, -64(%rdx,%rax,4)
+
+// CHECK: movss %xmm6, 64(%rdx,%rax,4)
+// CHECK: encoding: [0xf3,0x0f,0x11,0x74,0x82,0x40]
+movss %xmm6, 64(%rdx,%rax,4)
+
+// CHECK: movss %xmm6, (%rdx)
+// CHECK: encoding: [0xf3,0x0f,0x11,0x32]
+movss %xmm6, (%rdx)
+
+// CHECK: movss %xmm6, %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x10,0xf6]
+movss %xmm6, %xmm6
+
+// CHECK: movups 485498096, %xmm6
+// CHECK: encoding: [0x0f,0x10,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+movups 485498096, %xmm6
+
+// CHECK: movups -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x0f,0x10,0x74,0x82,0xc0]
+movups -64(%rdx,%rax,4), %xmm6
+
+// CHECK: movups 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x0f,0x10,0x74,0x82,0x40]
+movups 64(%rdx,%rax,4), %xmm6
+
+// CHECK: movups 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x0f,0x10,0x74,0x02,0x40]
+movups 64(%rdx,%rax), %xmm6
+
+// CHECK: movups 64(%rdx), %xmm6
+// CHECK: encoding: [0x0f,0x10,0x72,0x40]
+movups 64(%rdx), %xmm6
+
+// CHECK: movups (%rdx), %xmm6
+// CHECK: encoding: [0x0f,0x10,0x32]
+movups (%rdx), %xmm6
+
+// CHECK: movups %xmm6, 485498096
+// CHECK: encoding: [0x0f,0x11,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+movups %xmm6, 485498096
+
+// CHECK: movups %xmm6, 64(%rdx)
+// CHECK: encoding: [0x0f,0x11,0x72,0x40]
+movups %xmm6, 64(%rdx)
+
+// CHECK: movups %xmm6, 64(%rdx,%rax)
+// CHECK: encoding: [0x0f,0x11,0x74,0x02,0x40]
+movups %xmm6, 64(%rdx,%rax)
+
+// CHECK: movups %xmm6, -64(%rdx,%rax,4)
+// CHECK: encoding: [0x0f,0x11,0x74,0x82,0xc0]
+movups %xmm6, -64(%rdx,%rax,4)
+
+// CHECK: movups %xmm6, 64(%rdx,%rax,4)
+// CHECK: encoding: [0x0f,0x11,0x74,0x82,0x40]
+movups %xmm6, 64(%rdx,%rax,4)
+
+// CHECK: movups %xmm6, (%rdx)
+// CHECK: encoding: [0x0f,0x11,0x32]
+movups %xmm6, (%rdx)
+
+// CHECK: movups %xmm6, %xmm6
+// CHECK: encoding: [0x0f,0x10,0xf6]
+movups %xmm6, %xmm6
+
+// CHECK: mulps 485498096, %xmm6
+// CHECK: encoding: [0x0f,0x59,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+mulps 485498096, %xmm6
+
+// CHECK: mulps -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x0f,0x59,0x74,0x82,0xc0]
+mulps -64(%rdx,%rax,4), %xmm6
+
+// CHECK: mulps 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x0f,0x59,0x74,0x82,0x40]
+mulps 64(%rdx,%rax,4), %xmm6
+
+// CHECK: mulps 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x0f,0x59,0x74,0x02,0x40]
+mulps 64(%rdx,%rax), %xmm6
+
+// CHECK: mulps 64(%rdx), %xmm6
+// CHECK: encoding: [0x0f,0x59,0x72,0x40]
+mulps 64(%rdx), %xmm6
+
+// CHECK: mulps (%rdx), %xmm6
+// CHECK: encoding: [0x0f,0x59,0x32]
+mulps (%rdx), %xmm6
+
+// CHECK: mulps %xmm6, %xmm6
+// CHECK: encoding: [0x0f,0x59,0xf6]
+mulps %xmm6, %xmm6
+
+// CHECK: mulss 485498096, %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x59,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+mulss 485498096, %xmm6
+
+// CHECK: mulss -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x59,0x74,0x82,0xc0]
+mulss -64(%rdx,%rax,4), %xmm6
+
+// CHECK: mulss 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x59,0x74,0x82,0x40]
+mulss 64(%rdx,%rax,4), %xmm6
+
+// CHECK: mulss 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x59,0x74,0x02,0x40]
+mulss 64(%rdx,%rax), %xmm6
+
+// CHECK: mulss 64(%rdx), %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x59,0x72,0x40]
+mulss 64(%rdx), %xmm6
+
+// CHECK: mulss (%rdx), %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x59,0x32]
+mulss (%rdx), %xmm6
+
+// CHECK: mulss %xmm6, %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x59,0xf6]
+mulss %xmm6, %xmm6
+
+// CHECK: orps 485498096, %xmm6
+// CHECK: encoding: [0x0f,0x56,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+orps 485498096, %xmm6
+
+// CHECK: orps -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x0f,0x56,0x74,0x82,0xc0]
+orps -64(%rdx,%rax,4), %xmm6
+
+// CHECK: orps 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x0f,0x56,0x74,0x82,0x40]
+orps 64(%rdx,%rax,4), %xmm6
+
+// CHECK: orps 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x0f,0x56,0x74,0x02,0x40]
+orps 64(%rdx,%rax), %xmm6
+
+// CHECK: orps 64(%rdx), %xmm6
+// CHECK: encoding: [0x0f,0x56,0x72,0x40]
+orps 64(%rdx), %xmm6
+
+// CHECK: orps (%rdx), %xmm6
+// CHECK: encoding: [0x0f,0x56,0x32]
+orps (%rdx), %xmm6
+
+// CHECK: orps %xmm6, %xmm6
+// CHECK: encoding: [0x0f,0x56,0xf6]
+orps %xmm6, %xmm6
+
+// CHECK: pextrw $0, %mm4, %r13d
+// CHECK: encoding: [0x44,0x0f,0xc5,0xec,0x00]
+pextrw $0, %mm4, %r13d
+
+// CHECK: pinsrw $0, -485498096(%rdx,%rax,4), %mm4
+// CHECK: encoding: [0x0f,0xc4,0xa4,0x82,0x10,0xe3,0x0f,0xe3,0x00]
+pinsrw $0, -485498096(%rdx,%rax,4), %mm4
+
+// CHECK: pinsrw $0, 485498096(%rdx,%rax,4), %mm4
+// CHECK: encoding: [0x0f,0xc4,0xa4,0x82,0xf0,0x1c,0xf0,0x1c,0x00]
+pinsrw $0, 485498096(%rdx,%rax,4), %mm4
+
+// CHECK: pinsrw $0, 485498096(%rdx), %mm4
+// CHECK: encoding: [0x0f,0xc4,0xa2,0xf0,0x1c,0xf0,0x1c,0x00]
+pinsrw $0, 485498096(%rdx), %mm4
+
+// CHECK: pinsrw $0, 485498096, %mm4
+// CHECK: encoding: [0x0f,0xc4,0x24,0x25,0xf0,0x1c,0xf0,0x1c,0x00]
+pinsrw $0, 485498096, %mm4
+
+// CHECK: pinsrw $0, 64(%rdx,%rax), %mm4
+// CHECK: encoding: [0x0f,0xc4,0x64,0x02,0x40,0x00]
+pinsrw $0, 64(%rdx,%rax), %mm4
+
+// CHECK: pinsrw $0, (%rdx), %mm4
+// CHECK: encoding: [0x0f,0xc4,0x22,0x00]
+pinsrw $0, (%rdx), %mm4
+
+// CHECK: pinsrw $0, %r13d, %mm4
+// CHECK: encoding: [0x41,0x0f,0xc4,0xe5,0x00]
+pinsrw $0, %r13d, %mm4
+
+// CHECK: pmovmskb %mm4, %r13d
+// CHECK: encoding: [0x44,0x0f,0xd7,0xec]
+pmovmskb %mm4, %r13d
+
+// CHECK: pmovmskb %mm4, %r13d
+// CHECK: encoding: [0x44,0x0f,0xd7,0xec]
+pmovmskb %mm4, %r13
+
+// CHECK: rcpps 485498096, %xmm6
+// CHECK: encoding: [0x0f,0x53,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+rcpps 485498096, %xmm6
+
+// CHECK: rcpps -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x0f,0x53,0x74,0x82,0xc0]
+rcpps -64(%rdx,%rax,4), %xmm6
+
+// CHECK: rcpps 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x0f,0x53,0x74,0x82,0x40]
+rcpps 64(%rdx,%rax,4), %xmm6
+
+// CHECK: rcpps 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x0f,0x53,0x74,0x02,0x40]
+rcpps 64(%rdx,%rax), %xmm6
+
+// CHECK: rcpps 64(%rdx), %xmm6
+// CHECK: encoding: [0x0f,0x53,0x72,0x40]
+rcpps 64(%rdx), %xmm6
+
+// CHECK: rcpps (%rdx), %xmm6
+// CHECK: encoding: [0x0f,0x53,0x32]
+rcpps (%rdx), %xmm6
+
+// CHECK: rcpps %xmm6, %xmm6
+// CHECK: encoding: [0x0f,0x53,0xf6]
+rcpps %xmm6, %xmm6
+
+// CHECK: rcpss 485498096, %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x53,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+rcpss 485498096, %xmm6
+
+// CHECK: rcpss -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x53,0x74,0x82,0xc0]
+rcpss -64(%rdx,%rax,4), %xmm6
+
+// CHECK: rcpss 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x53,0x74,0x82,0x40]
+rcpss 64(%rdx,%rax,4), %xmm6
+
+// CHECK: rcpss 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x53,0x74,0x02,0x40]
+rcpss 64(%rdx,%rax), %xmm6
+
+// CHECK: rcpss 64(%rdx), %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x53,0x72,0x40]
+rcpss 64(%rdx), %xmm6
+
+// CHECK: rcpss (%rdx), %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x53,0x32]
+rcpss (%rdx), %xmm6
+
+// CHECK: rcpss %xmm6, %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x53,0xf6]
+rcpss %xmm6, %xmm6
+
+// CHECK: rsqrtps 485498096, %xmm6
+// CHECK: encoding: [0x0f,0x52,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+rsqrtps 485498096, %xmm6
+
+// CHECK: rsqrtps -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x0f,0x52,0x74,0x82,0xc0]
+rsqrtps -64(%rdx,%rax,4), %xmm6
+
+// CHECK: rsqrtps 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x0f,0x52,0x74,0x82,0x40]
+rsqrtps 64(%rdx,%rax,4), %xmm6
+
+// CHECK: rsqrtps 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x0f,0x52,0x74,0x02,0x40]
+rsqrtps 64(%rdx,%rax), %xmm6
+
+// CHECK: rsqrtps 64(%rdx), %xmm6
+// CHECK: encoding: [0x0f,0x52,0x72,0x40]
+rsqrtps 64(%rdx), %xmm6
+
+// CHECK: rsqrtps (%rdx), %xmm6
+// CHECK: encoding: [0x0f,0x52,0x32]
+rsqrtps (%rdx), %xmm6
+
+// CHECK: rsqrtps %xmm6, %xmm6
+// CHECK: encoding: [0x0f,0x52,0xf6]
+rsqrtps %xmm6, %xmm6
+
+// CHECK: rsqrtss 485498096, %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x52,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+rsqrtss 485498096, %xmm6
+
+// CHECK: rsqrtss -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x52,0x74,0x82,0xc0]
+rsqrtss -64(%rdx,%rax,4), %xmm6
+
+// CHECK: rsqrtss 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x52,0x74,0x82,0x40]
+rsqrtss 64(%rdx,%rax,4), %xmm6
+
+// CHECK: rsqrtss 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x52,0x74,0x02,0x40]
+rsqrtss 64(%rdx,%rax), %xmm6
+
+// CHECK: rsqrtss 64(%rdx), %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x52,0x72,0x40]
+rsqrtss 64(%rdx), %xmm6
+
+// CHECK: rsqrtss (%rdx), %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x52,0x32]
+rsqrtss (%rdx), %xmm6
+
+// CHECK: rsqrtss %xmm6, %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x52,0xf6]
+rsqrtss %xmm6, %xmm6
+
+// CHECK: sfence
+// CHECK: encoding: [0x0f,0xae,0xf8]
+sfence
+
+// CHECK: shufps $0, 485498096, %xmm6
+// CHECK: encoding: [0x0f,0xc6,0x34,0x25,0xf0,0x1c,0xf0,0x1c,0x00]
+shufps $0, 485498096, %xmm6
+
+// CHECK: shufps $0, -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x0f,0xc6,0x74,0x82,0xc0,0x00]
+shufps $0, -64(%rdx,%rax,4), %xmm6
+
+// CHECK: shufps $0, 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x0f,0xc6,0x74,0x82,0x40,0x00]
+shufps $0, 64(%rdx,%rax,4), %xmm6
+
+// CHECK: shufps $0, 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x0f,0xc6,0x74,0x02,0x40,0x00]
+shufps $0, 64(%rdx,%rax), %xmm6
+
+// CHECK: shufps $0, 64(%rdx), %xmm6
+// CHECK: encoding: [0x0f,0xc6,0x72,0x40,0x00]
+shufps $0, 64(%rdx), %xmm6
+
+// CHECK: shufps $0, (%rdx), %xmm6
+// CHECK: encoding: [0x0f,0xc6,0x32,0x00]
+shufps $0, (%rdx), %xmm6
+
+// CHECK: shufps $0, %xmm6, %xmm6
+// CHECK: encoding: [0x0f,0xc6,0xf6,0x00]
+shufps $0, %xmm6, %xmm6
+
+// CHECK: sqrtps 485498096, %xmm6
+// CHECK: encoding: [0x0f,0x51,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+sqrtps 485498096, %xmm6
+
+// CHECK: sqrtps -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x0f,0x51,0x74,0x82,0xc0]
+sqrtps -64(%rdx,%rax,4), %xmm6
+
+// CHECK: sqrtps 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x0f,0x51,0x74,0x82,0x40]
+sqrtps 64(%rdx,%rax,4), %xmm6
+
+// CHECK: sqrtps 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x0f,0x51,0x74,0x02,0x40]
+sqrtps 64(%rdx,%rax), %xmm6
+
+// CHECK: sqrtps 64(%rdx), %xmm6
+// CHECK: encoding: [0x0f,0x51,0x72,0x40]
+sqrtps 64(%rdx), %xmm6
+
+// CHECK: sqrtps (%rdx), %xmm6
+// CHECK: encoding: [0x0f,0x51,0x32]
+sqrtps (%rdx), %xmm6
+
+// CHECK: sqrtps %xmm6, %xmm6
+// CHECK: encoding: [0x0f,0x51,0xf6]
+sqrtps %xmm6, %xmm6
+
+// CHECK: sqrtss 485498096, %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x51,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+sqrtss 485498096, %xmm6
+
+// CHECK: sqrtss -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x51,0x74,0x82,0xc0]
+sqrtss -64(%rdx,%rax,4), %xmm6
+
+// CHECK: sqrtss 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x51,0x74,0x82,0x40]
+sqrtss 64(%rdx,%rax,4), %xmm6
+
+// CHECK: sqrtss 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x51,0x74,0x02,0x40]
+sqrtss 64(%rdx,%rax), %xmm6
+
+// CHECK: sqrtss 64(%rdx), %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x51,0x72,0x40]
+sqrtss 64(%rdx), %xmm6
+
+// CHECK: sqrtss (%rdx), %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x51,0x32]
+sqrtss (%rdx), %xmm6
+
+// CHECK: sqrtss %xmm6, %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x51,0xf6]
+sqrtss %xmm6, %xmm6
+
+// CHECK: subps 485498096, %xmm6
+// CHECK: encoding: [0x0f,0x5c,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+subps 485498096, %xmm6
+
+// CHECK: subps -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x0f,0x5c,0x74,0x82,0xc0]
+subps -64(%rdx,%rax,4), %xmm6
+
+// CHECK: subps 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x0f,0x5c,0x74,0x82,0x40]
+subps 64(%rdx,%rax,4), %xmm6
+
+// CHECK: subps 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x0f,0x5c,0x74,0x02,0x40]
+subps 64(%rdx,%rax), %xmm6
+
+// CHECK: subps 64(%rdx), %xmm6
+// CHECK: encoding: [0x0f,0x5c,0x72,0x40]
+subps 64(%rdx), %xmm6
+
+// CHECK: subps (%rdx), %xmm6
+// CHECK: encoding: [0x0f,0x5c,0x32]
+subps (%rdx), %xmm6
+
+// CHECK: subps %xmm6, %xmm6
+// CHECK: encoding: [0x0f,0x5c,0xf6]
+subps %xmm6, %xmm6
+
+// CHECK: subss 485498096, %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x5c,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+subss 485498096, %xmm6
+
+// CHECK: subss -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x5c,0x74,0x82,0xc0]
+subss -64(%rdx,%rax,4), %xmm6
+
+// CHECK: subss 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x5c,0x74,0x82,0x40]
+subss 64(%rdx,%rax,4), %xmm6
+
+// CHECK: subss 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x5c,0x74,0x02,0x40]
+subss 64(%rdx,%rax), %xmm6
+
+// CHECK: subss 64(%rdx), %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x5c,0x72,0x40]
+subss 64(%rdx), %xmm6
+
+// CHECK: subss (%rdx), %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x5c,0x32]
+subss (%rdx), %xmm6
+
+// CHECK: subss %xmm6, %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x5c,0xf6]
+subss %xmm6, %xmm6
+
+// CHECK: ucomiss 485498096, %xmm6
+// CHECK: encoding: [0x0f,0x2e,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+ucomiss 485498096, %xmm6
+
+// CHECK: ucomiss -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x0f,0x2e,0x74,0x82,0xc0]
+ucomiss -64(%rdx,%rax,4), %xmm6
+
+// CHECK: ucomiss 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x0f,0x2e,0x74,0x82,0x40]
+ucomiss 64(%rdx,%rax,4), %xmm6
+
+// CHECK: ucomiss 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x0f,0x2e,0x74,0x02,0x40]
+ucomiss 64(%rdx,%rax), %xmm6
+
+// CHECK: ucomiss 64(%rdx), %xmm6
+// CHECK: encoding: [0x0f,0x2e,0x72,0x40]
+ucomiss 64(%rdx), %xmm6
+
+// CHECK: ucomiss (%rdx), %xmm6
+// CHECK: encoding: [0x0f,0x2e,0x32]
+ucomiss (%rdx), %xmm6
+
+// CHECK: ucomiss %xmm6, %xmm6
+// CHECK: encoding: [0x0f,0x2e,0xf6]
+ucomiss %xmm6, %xmm6
+
+// CHECK: unpckhps 485498096, %xmm6
+// CHECK: encoding: [0x0f,0x15,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+unpckhps 485498096, %xmm6
+
+// CHECK: unpckhps -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x0f,0x15,0x74,0x82,0xc0]
+unpckhps -64(%rdx,%rax,4), %xmm6
+
+// CHECK: unpckhps 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x0f,0x15,0x74,0x82,0x40]
+unpckhps 64(%rdx,%rax,4), %xmm6
+
+// CHECK: unpckhps 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x0f,0x15,0x74,0x02,0x40]
+unpckhps 64(%rdx,%rax), %xmm6
+
+// CHECK: unpckhps 64(%rdx), %xmm6
+// CHECK: encoding: [0x0f,0x15,0x72,0x40]
+unpckhps 64(%rdx), %xmm6
+
+// CHECK: unpckhps (%rdx), %xmm6
+// CHECK: encoding: [0x0f,0x15,0x32]
+unpckhps (%rdx), %xmm6
+
+// CHECK: unpckhps %xmm6, %xmm6
+// CHECK: encoding: [0x0f,0x15,0xf6]
+unpckhps %xmm6, %xmm6
+
+// CHECK: unpcklps 485498096, %xmm6
+// CHECK: encoding: [0x0f,0x14,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+unpcklps 485498096, %xmm6
+
+// CHECK: unpcklps -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x0f,0x14,0x74,0x82,0xc0]
+unpcklps -64(%rdx,%rax,4), %xmm6
+
+// CHECK: unpcklps 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x0f,0x14,0x74,0x82,0x40]
+unpcklps 64(%rdx,%rax,4), %xmm6
+
+// CHECK: unpcklps 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x0f,0x14,0x74,0x02,0x40]
+unpcklps 64(%rdx,%rax), %xmm6
+
+// CHECK: unpcklps 64(%rdx), %xmm6
+// CHECK: encoding: [0x0f,0x14,0x72,0x40]
+unpcklps 64(%rdx), %xmm6
+
+// CHECK: unpcklps (%rdx), %xmm6
+// CHECK: encoding: [0x0f,0x14,0x32]
+unpcklps (%rdx), %xmm6
+
+// CHECK: unpcklps %xmm6, %xmm6
+// CHECK: encoding: [0x0f,0x14,0xf6]
+unpcklps %xmm6, %xmm6
+
+// CHECK: xorps 485498096, %xmm6
+// CHECK: encoding: [0x0f,0x57,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+xorps 485498096, %xmm6
+
+// CHECK: xorps -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x0f,0x57,0x74,0x82,0xc0]
+xorps -64(%rdx,%rax,4), %xmm6
+
+// CHECK: xorps 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x0f,0x57,0x74,0x82,0x40]
+xorps 64(%rdx,%rax,4), %xmm6
+
+// CHECK: xorps 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x0f,0x57,0x74,0x02,0x40]
+xorps 64(%rdx,%rax), %xmm6
+
+// CHECK: xorps 64(%rdx), %xmm6
+// CHECK: encoding: [0x0f,0x57,0x72,0x40]
+xorps 64(%rdx), %xmm6
+
+// CHECK: xorps (%rdx), %xmm6
+// CHECK: encoding: [0x0f,0x57,0x32]
+xorps (%rdx), %xmm6
+
+// CHECK: xorps %xmm6, %xmm6
+// CHECK: encoding: [0x0f,0x57,0xf6]
+xorps %xmm6, %xmm6
+
diff --git a/test/MC/X86/SSE2-32.s b/test/MC/X86/SSE2-32.s
new file mode 100644
index 0000000000000000000000000000000000000000..5c2ccefbf3340ed4f0f37a416a154833e4a60e7a
--- /dev/null
+++ b/test/MC/X86/SSE2-32.s
@@ -0,0 +1,3446 @@
+// RUN: llvm-mc -triple i386-unknown-unknown --show-encoding %s | FileCheck %s
+
+// CHECK: addpd -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x58,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+addpd -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: addpd 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x58,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+addpd 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: addpd 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x58,0x8a,0xf0,0x1c,0xf0,0x1c]
+addpd 485498096(%edx), %xmm1
+
+// CHECK: addpd 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x58,0x0d,0xf0,0x1c,0xf0,0x1c]
+addpd 485498096, %xmm1
+
+// CHECK: addpd 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x58,0x4c,0x02,0x40]
+addpd 64(%edx,%eax), %xmm1
+
+// CHECK: addpd (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x58,0x0a]
+addpd (%edx), %xmm1
+
+// CHECK: addpd %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x58,0xc9]
+addpd %xmm1, %xmm1
+
+// CHECK: addsd -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0xf2,0x0f,0x58,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+addsd -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: addsd 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0xf2,0x0f,0x58,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+addsd 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: addsd 485498096(%edx), %xmm1
+// CHECK: encoding: [0xf2,0x0f,0x58,0x8a,0xf0,0x1c,0xf0,0x1c]
+addsd 485498096(%edx), %xmm1
+
+// CHECK: addsd 485498096, %xmm1
+// CHECK: encoding: [0xf2,0x0f,0x58,0x0d,0xf0,0x1c,0xf0,0x1c]
+addsd 485498096, %xmm1
+
+// CHECK: addsd 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0xf2,0x0f,0x58,0x4c,0x02,0x40]
+addsd 64(%edx,%eax), %xmm1
+
+// CHECK: addsd (%edx), %xmm1
+// CHECK: encoding: [0xf2,0x0f,0x58,0x0a]
+addsd (%edx), %xmm1
+
+// CHECK: addsd %xmm1, %xmm1
+// CHECK: encoding: [0xf2,0x0f,0x58,0xc9]
+addsd %xmm1, %xmm1
+
+// CHECK: andnpd -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x55,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+andnpd -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: andnpd 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x55,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+andnpd 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: andnpd 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x55,0x8a,0xf0,0x1c,0xf0,0x1c]
+andnpd 485498096(%edx), %xmm1
+
+// CHECK: andnpd 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x55,0x0d,0xf0,0x1c,0xf0,0x1c]
+andnpd 485498096, %xmm1
+
+// CHECK: andnpd 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x55,0x4c,0x02,0x40]
+andnpd 64(%edx,%eax), %xmm1
+
+// CHECK: andnpd (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x55,0x0a]
+andnpd (%edx), %xmm1
+
+// CHECK: andnpd %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x55,0xc9]
+andnpd %xmm1, %xmm1
+
+// CHECK: andpd -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x54,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+andpd -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: andpd 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x54,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+andpd 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: andpd 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x54,0x8a,0xf0,0x1c,0xf0,0x1c]
+andpd 485498096(%edx), %xmm1
+
+// CHECK: andpd 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x54,0x0d,0xf0,0x1c,0xf0,0x1c]
+andpd 485498096, %xmm1
+
+// CHECK: andpd 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x54,0x4c,0x02,0x40]
+andpd 64(%edx,%eax), %xmm1
+
+// CHECK: andpd (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x54,0x0a]
+andpd (%edx), %xmm1
+
+// CHECK: andpd %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x54,0xc9]
+andpd %xmm1, %xmm1
+
+// CHECK: cmpeqpd -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xc2,0x8c,0x82,0x10,0xe3,0x0f,0xe3,0x00]
+cmpeqpd -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: cmpeqpd 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xc2,0x8c,0x82,0xf0,0x1c,0xf0,0x1c,0x00]
+cmpeqpd 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: cmpeqpd 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xc2,0x8a,0xf0,0x1c,0xf0,0x1c,0x00]
+cmpeqpd 485498096(%edx), %xmm1
+
+// CHECK: cmpeqpd 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0xc2,0x0d,0xf0,0x1c,0xf0,0x1c,0x00]
+cmpeqpd 485498096, %xmm1
+
+// CHECK: cmpeqpd 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xc2,0x4c,0x02,0x40,0x00]
+cmpeqpd 64(%edx,%eax), %xmm1
+
+// CHECK: cmpeqpd (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xc2,0x0a,0x00]
+cmpeqpd (%edx), %xmm1
+
+// CHECK: cmpeqpd %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0xc2,0xc9,0x00]
+cmpeqpd %xmm1, %xmm1
+
+// CHECK: cmpeqsd -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0xf2,0x0f,0xc2,0x8c,0x82,0x10,0xe3,0x0f,0xe3,0x00]
+cmpeqsd -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: cmpeqsd 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0xf2,0x0f,0xc2,0x8c,0x82,0xf0,0x1c,0xf0,0x1c,0x00]
+cmpeqsd 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: cmpeqsd 485498096(%edx), %xmm1
+// CHECK: encoding: [0xf2,0x0f,0xc2,0x8a,0xf0,0x1c,0xf0,0x1c,0x00]
+cmpeqsd 485498096(%edx), %xmm1
+
+// CHECK: cmpeqsd 485498096, %xmm1
+// CHECK: encoding: [0xf2,0x0f,0xc2,0x0d,0xf0,0x1c,0xf0,0x1c,0x00]
+cmpeqsd 485498096, %xmm1
+
+// CHECK: cmpeqsd 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0xf2,0x0f,0xc2,0x4c,0x02,0x40,0x00]
+cmpeqsd 64(%edx,%eax), %xmm1
+
+// CHECK: cmpeqsd (%edx), %xmm1
+// CHECK: encoding: [0xf2,0x0f,0xc2,0x0a,0x00]
+cmpeqsd (%edx), %xmm1
+
+// CHECK: cmpeqsd %xmm1, %xmm1
+// CHECK: encoding: [0xf2,0x0f,0xc2,0xc9,0x00]
+cmpeqsd %xmm1, %xmm1
+
+// CHECK: comisd -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x2f,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+comisd -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: comisd 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x2f,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+comisd 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: comisd 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x2f,0x8a,0xf0,0x1c,0xf0,0x1c]
+comisd 485498096(%edx), %xmm1
+
+// CHECK: comisd 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x2f,0x0d,0xf0,0x1c,0xf0,0x1c]
+comisd 485498096, %xmm1
+
+// CHECK: comisd 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x2f,0x4c,0x02,0x40]
+comisd 64(%edx,%eax), %xmm1
+
+// CHECK: comisd (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x2f,0x0a]
+comisd (%edx), %xmm1
+
+// CHECK: comisd %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x2f,0xc9]
+comisd %xmm1, %xmm1
+
+// CHECK: cvtdq2pd -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0xf3,0x0f,0xe6,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+cvtdq2pd -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: cvtdq2pd 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0xf3,0x0f,0xe6,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+cvtdq2pd 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: cvtdq2pd 485498096(%edx), %xmm1
+// CHECK: encoding: [0xf3,0x0f,0xe6,0x8a,0xf0,0x1c,0xf0,0x1c]
+cvtdq2pd 485498096(%edx), %xmm1
+
+// CHECK: cvtdq2pd 485498096, %xmm1
+// CHECK: encoding: [0xf3,0x0f,0xe6,0x0d,0xf0,0x1c,0xf0,0x1c]
+cvtdq2pd 485498096, %xmm1
+
+// CHECK: cvtdq2pd 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0xf3,0x0f,0xe6,0x4c,0x02,0x40]
+cvtdq2pd 64(%edx,%eax), %xmm1
+
+// CHECK: cvtdq2pd (%edx), %xmm1
+// CHECK: encoding: [0xf3,0x0f,0xe6,0x0a]
+cvtdq2pd (%edx), %xmm1
+
+// CHECK: cvtdq2pd %xmm1, %xmm1
+// CHECK: encoding: [0xf3,0x0f,0xe6,0xc9]
+cvtdq2pd %xmm1, %xmm1
+
+// CHECK: cvtdq2ps -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x0f,0x5b,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+cvtdq2ps -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: cvtdq2ps 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x0f,0x5b,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+cvtdq2ps 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: cvtdq2ps 485498096(%edx), %xmm1
+// CHECK: encoding: [0x0f,0x5b,0x8a,0xf0,0x1c,0xf0,0x1c]
+cvtdq2ps 485498096(%edx), %xmm1
+
+// CHECK: cvtdq2ps 485498096, %xmm1
+// CHECK: encoding: [0x0f,0x5b,0x0d,0xf0,0x1c,0xf0,0x1c]
+cvtdq2ps 485498096, %xmm1
+
+// CHECK: cvtdq2ps 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x0f,0x5b,0x4c,0x02,0x40]
+cvtdq2ps 64(%edx,%eax), %xmm1
+
+// CHECK: cvtdq2ps (%edx), %xmm1
+// CHECK: encoding: [0x0f,0x5b,0x0a]
+cvtdq2ps (%edx), %xmm1
+
+// CHECK: cvtdq2ps %xmm1, %xmm1
+// CHECK: encoding: [0x0f,0x5b,0xc9]
+cvtdq2ps %xmm1, %xmm1
+
+// CHECK: cvtpd2dq -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0xf2,0x0f,0xe6,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+cvtpd2dq -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: cvtpd2dq 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0xf2,0x0f,0xe6,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+cvtpd2dq 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: cvtpd2dq 485498096(%edx), %xmm1
+// CHECK: encoding: [0xf2,0x0f,0xe6,0x8a,0xf0,0x1c,0xf0,0x1c]
+cvtpd2dq 485498096(%edx), %xmm1
+
+// CHECK: cvtpd2dq 485498096, %xmm1
+// CHECK: encoding: [0xf2,0x0f,0xe6,0x0d,0xf0,0x1c,0xf0,0x1c]
+cvtpd2dq 485498096, %xmm1
+
+// CHECK: cvtpd2dq 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0xf2,0x0f,0xe6,0x4c,0x02,0x40]
+cvtpd2dq 64(%edx,%eax), %xmm1
+
+// CHECK: cvtpd2dq (%edx), %xmm1
+// CHECK: encoding: [0xf2,0x0f,0xe6,0x0a]
+cvtpd2dq (%edx), %xmm1
+
+// CHECK: cvtpd2dq %xmm1, %xmm1
+// CHECK: encoding: [0xf2,0x0f,0xe6,0xc9]
+cvtpd2dq %xmm1, %xmm1
+
+// CHECK: cvtpd2pi -485498096(%edx,%eax,4), %mm4
+// CHECK: encoding: [0x66,0x0f,0x2d,0xa4,0x82,0x10,0xe3,0x0f,0xe3]
+cvtpd2pi -485498096(%edx,%eax,4), %mm4
+
+// CHECK: cvtpd2pi 485498096(%edx,%eax,4), %mm4
+// CHECK: encoding: [0x66,0x0f,0x2d,0xa4,0x82,0xf0,0x1c,0xf0,0x1c]
+cvtpd2pi 485498096(%edx,%eax,4), %mm4
+
+// CHECK: cvtpd2pi 485498096(%edx), %mm4
+// CHECK: encoding: [0x66,0x0f,0x2d,0xa2,0xf0,0x1c,0xf0,0x1c]
+cvtpd2pi 485498096(%edx), %mm4
+
+// CHECK: cvtpd2pi 485498096, %mm4
+// CHECK: encoding: [0x66,0x0f,0x2d,0x25,0xf0,0x1c,0xf0,0x1c]
+cvtpd2pi 485498096, %mm4
+
+// CHECK: cvtpd2pi 64(%edx,%eax), %mm4
+// CHECK: encoding: [0x66,0x0f,0x2d,0x64,0x02,0x40]
+cvtpd2pi 64(%edx,%eax), %mm4
+
+// CHECK: cvtpd2pi (%edx), %mm4
+// CHECK: encoding: [0x66,0x0f,0x2d,0x22]
+cvtpd2pi (%edx), %mm4
+
+// CHECK: cvtpd2pi %xmm1, %mm4
+// CHECK: encoding: [0x66,0x0f,0x2d,0xe1]
+cvtpd2pi %xmm1, %mm4
+
+// CHECK: cvtpd2ps -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x5a,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+cvtpd2ps -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: cvtpd2ps 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x5a,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+cvtpd2ps 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: cvtpd2ps 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x5a,0x8a,0xf0,0x1c,0xf0,0x1c]
+cvtpd2ps 485498096(%edx), %xmm1
+
+// CHECK: cvtpd2ps 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x5a,0x0d,0xf0,0x1c,0xf0,0x1c]
+cvtpd2ps 485498096, %xmm1
+
+// CHECK: cvtpd2ps 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x5a,0x4c,0x02,0x40]
+cvtpd2ps 64(%edx,%eax), %xmm1
+
+// CHECK: cvtpd2ps (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x5a,0x0a]
+cvtpd2ps (%edx), %xmm1
+
+// CHECK: cvtpd2ps %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x5a,0xc9]
+cvtpd2ps %xmm1, %xmm1
+
+// CHECK: cvtpi2pd -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x2a,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+cvtpi2pd -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: cvtpi2pd 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x2a,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+cvtpi2pd 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: cvtpi2pd 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x2a,0x8a,0xf0,0x1c,0xf0,0x1c]
+cvtpi2pd 485498096(%edx), %xmm1
+
+// CHECK: cvtpi2pd 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x2a,0x0d,0xf0,0x1c,0xf0,0x1c]
+cvtpi2pd 485498096, %xmm1
+
+// CHECK: cvtpi2pd 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x2a,0x4c,0x02,0x40]
+cvtpi2pd 64(%edx,%eax), %xmm1
+
+// CHECK: cvtpi2pd (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x2a,0x0a]
+cvtpi2pd (%edx), %xmm1
+
+// CHECK: cvtpi2pd %mm4, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x2a,0xcc]
+cvtpi2pd %mm4, %xmm1
+
+// CHECK: cvtps2dq -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x5b,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+cvtps2dq -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: cvtps2dq 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x5b,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+cvtps2dq 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: cvtps2dq 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x5b,0x8a,0xf0,0x1c,0xf0,0x1c]
+cvtps2dq 485498096(%edx), %xmm1
+
+// CHECK: cvtps2dq 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x5b,0x0d,0xf0,0x1c,0xf0,0x1c]
+cvtps2dq 485498096, %xmm1
+
+// CHECK: cvtps2dq 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x5b,0x4c,0x02,0x40]
+cvtps2dq 64(%edx,%eax), %xmm1
+
+// CHECK: cvtps2dq (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x5b,0x0a]
+cvtps2dq (%edx), %xmm1
+
+// CHECK: cvtps2dq %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x5b,0xc9]
+cvtps2dq %xmm1, %xmm1
+
+// CHECK: cvtps2pd -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x0f,0x5a,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+cvtps2pd -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: cvtps2pd 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x0f,0x5a,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+cvtps2pd 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: cvtps2pd 485498096(%edx), %xmm1
+// CHECK: encoding: [0x0f,0x5a,0x8a,0xf0,0x1c,0xf0,0x1c]
+cvtps2pd 485498096(%edx), %xmm1
+
+// CHECK: cvtps2pd 485498096, %xmm1
+// CHECK: encoding: [0x0f,0x5a,0x0d,0xf0,0x1c,0xf0,0x1c]
+cvtps2pd 485498096, %xmm1
+
+// CHECK: cvtps2pd 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x0f,0x5a,0x4c,0x02,0x40]
+cvtps2pd 64(%edx,%eax), %xmm1
+
+// CHECK: cvtps2pd (%edx), %xmm1
+// CHECK: encoding: [0x0f,0x5a,0x0a]
+cvtps2pd (%edx), %xmm1
+
+// CHECK: cvtps2pd %xmm1, %xmm1
+// CHECK: encoding: [0x0f,0x5a,0xc9]
+cvtps2pd %xmm1, %xmm1
+
+// CHECK: cvtsd2ss -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0xf2,0x0f,0x5a,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+cvtsd2ss -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: cvtsd2ss 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0xf2,0x0f,0x5a,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+cvtsd2ss 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: cvtsd2ss 485498096(%edx), %xmm1
+// CHECK: encoding: [0xf2,0x0f,0x5a,0x8a,0xf0,0x1c,0xf0,0x1c]
+cvtsd2ss 485498096(%edx), %xmm1
+
+// CHECK: cvtsd2ss 485498096, %xmm1
+// CHECK: encoding: [0xf2,0x0f,0x5a,0x0d,0xf0,0x1c,0xf0,0x1c]
+cvtsd2ss 485498096, %xmm1
+
+// CHECK: cvtsd2ss 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0xf2,0x0f,0x5a,0x4c,0x02,0x40]
+cvtsd2ss 64(%edx,%eax), %xmm1
+
+// CHECK: cvtsd2ss (%edx), %xmm1
+// CHECK: encoding: [0xf2,0x0f,0x5a,0x0a]
+cvtsd2ss (%edx), %xmm1
+
+// CHECK: cvtsd2ss %xmm1, %xmm1
+// CHECK: encoding: [0xf2,0x0f,0x5a,0xc9]
+cvtsd2ss %xmm1, %xmm1
+
+// CHECK: cvtsi2sdl -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0xf2,0x0f,0x2a,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+cvtsi2sdl -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: cvtsi2sdl 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0xf2,0x0f,0x2a,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+cvtsi2sdl 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: cvtsi2sdl 485498096(%edx), %xmm1
+// CHECK: encoding: [0xf2,0x0f,0x2a,0x8a,0xf0,0x1c,0xf0,0x1c]
+cvtsi2sdl 485498096(%edx), %xmm1
+
+// CHECK: cvtsi2sdl 485498096, %xmm1
+// CHECK: encoding: [0xf2,0x0f,0x2a,0x0d,0xf0,0x1c,0xf0,0x1c]
+cvtsi2sdl 485498096, %xmm1
+
+// CHECK: cvtsi2sdl 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0xf2,0x0f,0x2a,0x4c,0x02,0x40]
+cvtsi2sdl 64(%edx,%eax), %xmm1
+
+// CHECK: cvtsi2sdl (%edx), %xmm1
+// CHECK: encoding: [0xf2,0x0f,0x2a,0x0a]
+cvtsi2sdl (%edx), %xmm1
+
+// CHECK: cvtss2sd -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x5a,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+cvtss2sd -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: cvtss2sd 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x5a,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+cvtss2sd 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: cvtss2sd 485498096(%edx), %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x5a,0x8a,0xf0,0x1c,0xf0,0x1c]
+cvtss2sd 485498096(%edx), %xmm1
+
+// CHECK: cvtss2sd 485498096, %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x5a,0x0d,0xf0,0x1c,0xf0,0x1c]
+cvtss2sd 485498096, %xmm1
+
+// CHECK: cvtss2sd 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x5a,0x4c,0x02,0x40]
+cvtss2sd 64(%edx,%eax), %xmm1
+
+// CHECK: cvtss2sd (%edx), %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x5a,0x0a]
+cvtss2sd (%edx), %xmm1
+
+// CHECK: cvtss2sd %xmm1, %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x5a,0xc9]
+cvtss2sd %xmm1, %xmm1
+
+// CHECK: cvttpd2dq -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xe6,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+cvttpd2dq -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: cvttpd2dq 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xe6,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+cvttpd2dq 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: cvttpd2dq 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xe6,0x8a,0xf0,0x1c,0xf0,0x1c]
+cvttpd2dq 485498096(%edx), %xmm1
+
+// CHECK: cvttpd2dq 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0xe6,0x0d,0xf0,0x1c,0xf0,0x1c]
+cvttpd2dq 485498096, %xmm1
+
+// CHECK: cvttpd2dq 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xe6,0x4c,0x02,0x40]
+cvttpd2dq 64(%edx,%eax), %xmm1
+
+// CHECK: cvttpd2dq (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xe6,0x0a]
+cvttpd2dq (%edx), %xmm1
+
+// CHECK: cvttpd2dq %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0xe6,0xc9]
+cvttpd2dq %xmm1, %xmm1
+
+// CHECK: cvttpd2pi -485498096(%edx,%eax,4), %mm4
+// CHECK: encoding: [0x66,0x0f,0x2c,0xa4,0x82,0x10,0xe3,0x0f,0xe3]
+cvttpd2pi -485498096(%edx,%eax,4), %mm4
+
+// CHECK: cvttpd2pi 485498096(%edx,%eax,4), %mm4
+// CHECK: encoding: [0x66,0x0f,0x2c,0xa4,0x82,0xf0,0x1c,0xf0,0x1c]
+cvttpd2pi 485498096(%edx,%eax,4), %mm4
+
+// CHECK: cvttpd2pi 485498096(%edx), %mm4
+// CHECK: encoding: [0x66,0x0f,0x2c,0xa2,0xf0,0x1c,0xf0,0x1c]
+cvttpd2pi 485498096(%edx), %mm4
+
+// CHECK: cvttpd2pi 485498096, %mm4
+// CHECK: encoding: [0x66,0x0f,0x2c,0x25,0xf0,0x1c,0xf0,0x1c]
+cvttpd2pi 485498096, %mm4
+
+// CHECK: cvttpd2pi 64(%edx,%eax), %mm4
+// CHECK: encoding: [0x66,0x0f,0x2c,0x64,0x02,0x40]
+cvttpd2pi 64(%edx,%eax), %mm4
+
+// CHECK: cvttpd2pi (%edx), %mm4
+// CHECK: encoding: [0x66,0x0f,0x2c,0x22]
+cvttpd2pi (%edx), %mm4
+
+// CHECK: cvttpd2pi %xmm1, %mm4
+// CHECK: encoding: [0x66,0x0f,0x2c,0xe1]
+cvttpd2pi %xmm1, %mm4
+
+// CHECK: cvttps2dq -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x5b,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+cvttps2dq -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: cvttps2dq 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x5b,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+cvttps2dq 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: cvttps2dq 485498096(%edx), %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x5b,0x8a,0xf0,0x1c,0xf0,0x1c]
+cvttps2dq 485498096(%edx), %xmm1
+
+// CHECK: cvttps2dq 485498096, %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x5b,0x0d,0xf0,0x1c,0xf0,0x1c]
+cvttps2dq 485498096, %xmm1
+
+// CHECK: cvttps2dq 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x5b,0x4c,0x02,0x40]
+cvttps2dq 64(%edx,%eax), %xmm1
+
+// CHECK: cvttps2dq (%edx), %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x5b,0x0a]
+cvttps2dq (%edx), %xmm1
+
+// CHECK: cvttps2dq %xmm1, %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x5b,0xc9]
+cvttps2dq %xmm1, %xmm1
+
+// CHECK: divpd -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x5e,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+divpd -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: divpd 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x5e,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+divpd 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: divpd 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x5e,0x8a,0xf0,0x1c,0xf0,0x1c]
+divpd 485498096(%edx), %xmm1
+
+// CHECK: divpd 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x5e,0x0d,0xf0,0x1c,0xf0,0x1c]
+divpd 485498096, %xmm1
+
+// CHECK: divpd 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x5e,0x4c,0x02,0x40]
+divpd 64(%edx,%eax), %xmm1
+
+// CHECK: divpd (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x5e,0x0a]
+divpd (%edx), %xmm1
+
+// CHECK: divpd %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x5e,0xc9]
+divpd %xmm1, %xmm1
+
+// CHECK: divsd -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0xf2,0x0f,0x5e,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+divsd -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: divsd 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0xf2,0x0f,0x5e,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+divsd 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: divsd 485498096(%edx), %xmm1
+// CHECK: encoding: [0xf2,0x0f,0x5e,0x8a,0xf0,0x1c,0xf0,0x1c]
+divsd 485498096(%edx), %xmm1
+
+// CHECK: divsd 485498096, %xmm1
+// CHECK: encoding: [0xf2,0x0f,0x5e,0x0d,0xf0,0x1c,0xf0,0x1c]
+divsd 485498096, %xmm1
+
+// CHECK: divsd 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0xf2,0x0f,0x5e,0x4c,0x02,0x40]
+divsd 64(%edx,%eax), %xmm1
+
+// CHECK: divsd (%edx), %xmm1
+// CHECK: encoding: [0xf2,0x0f,0x5e,0x0a]
+divsd (%edx), %xmm1
+
+// CHECK: divsd %xmm1, %xmm1
+// CHECK: encoding: [0xf2,0x0f,0x5e,0xc9]
+divsd %xmm1, %xmm1
+
+// CHECK: lfence
+// CHECK: encoding: [0x0f,0xae,0xe8]
+lfence
+
+// CHECK: maskmovdqu %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0xf7,0xc9]
+maskmovdqu %xmm1, %xmm1
+
+// CHECK: maxpd -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x5f,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+maxpd -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: maxpd 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x5f,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+maxpd 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: maxpd 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x5f,0x8a,0xf0,0x1c,0xf0,0x1c]
+maxpd 485498096(%edx), %xmm1
+
+// CHECK: maxpd 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x5f,0x0d,0xf0,0x1c,0xf0,0x1c]
+maxpd 485498096, %xmm1
+
+// CHECK: maxpd 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x5f,0x4c,0x02,0x40]
+maxpd 64(%edx,%eax), %xmm1
+
+// CHECK: maxpd (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x5f,0x0a]
+maxpd (%edx), %xmm1
+
+// CHECK: maxpd %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x5f,0xc9]
+maxpd %xmm1, %xmm1
+
+// CHECK: maxsd -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0xf2,0x0f,0x5f,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+maxsd -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: maxsd 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0xf2,0x0f,0x5f,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+maxsd 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: maxsd 485498096(%edx), %xmm1
+// CHECK: encoding: [0xf2,0x0f,0x5f,0x8a,0xf0,0x1c,0xf0,0x1c]
+maxsd 485498096(%edx), %xmm1
+
+// CHECK: maxsd 485498096, %xmm1
+// CHECK: encoding: [0xf2,0x0f,0x5f,0x0d,0xf0,0x1c,0xf0,0x1c]
+maxsd 485498096, %xmm1
+
+// CHECK: maxsd 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0xf2,0x0f,0x5f,0x4c,0x02,0x40]
+maxsd 64(%edx,%eax), %xmm1
+
+// CHECK: maxsd (%edx), %xmm1
+// CHECK: encoding: [0xf2,0x0f,0x5f,0x0a]
+maxsd (%edx), %xmm1
+
+// CHECK: maxsd %xmm1, %xmm1
+// CHECK: encoding: [0xf2,0x0f,0x5f,0xc9]
+maxsd %xmm1, %xmm1
+
+// CHECK: mfence
+// CHECK: encoding: [0x0f,0xae,0xf0]
+mfence
+
+// CHECK: minpd -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x5d,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+minpd -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: minpd 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x5d,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+minpd 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: minpd 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x5d,0x8a,0xf0,0x1c,0xf0,0x1c]
+minpd 485498096(%edx), %xmm1
+
+// CHECK: minpd 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x5d,0x0d,0xf0,0x1c,0xf0,0x1c]
+minpd 485498096, %xmm1
+
+// CHECK: minpd 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x5d,0x4c,0x02,0x40]
+minpd 64(%edx,%eax), %xmm1
+
+// CHECK: minpd (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x5d,0x0a]
+minpd (%edx), %xmm1
+
+// CHECK: minpd %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x5d,0xc9]
+minpd %xmm1, %xmm1
+
+// CHECK: minsd -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0xf2,0x0f,0x5d,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+minsd -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: minsd 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0xf2,0x0f,0x5d,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+minsd 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: minsd 485498096(%edx), %xmm1
+// CHECK: encoding: [0xf2,0x0f,0x5d,0x8a,0xf0,0x1c,0xf0,0x1c]
+minsd 485498096(%edx), %xmm1
+
+// CHECK: minsd 485498096, %xmm1
+// CHECK: encoding: [0xf2,0x0f,0x5d,0x0d,0xf0,0x1c,0xf0,0x1c]
+minsd 485498096, %xmm1
+
+// CHECK: minsd 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0xf2,0x0f,0x5d,0x4c,0x02,0x40]
+minsd 64(%edx,%eax), %xmm1
+
+// CHECK: minsd (%edx), %xmm1
+// CHECK: encoding: [0xf2,0x0f,0x5d,0x0a]
+minsd (%edx), %xmm1
+
+// CHECK: minsd %xmm1, %xmm1
+// CHECK: encoding: [0xf2,0x0f,0x5d,0xc9]
+minsd %xmm1, %xmm1
+
+// CHECK: movapd -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x28,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+movapd -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: movapd 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x28,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+movapd 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: movapd 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x28,0x8a,0xf0,0x1c,0xf0,0x1c]
+movapd 485498096(%edx), %xmm1
+
+// CHECK: movapd 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x28,0x0d,0xf0,0x1c,0xf0,0x1c]
+movapd 485498096, %xmm1
+
+// CHECK: movapd 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x28,0x4c,0x02,0x40]
+movapd 64(%edx,%eax), %xmm1
+
+// CHECK: movapd (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x28,0x0a]
+movapd (%edx), %xmm1
+
+// CHECK: movapd %xmm1, 485498096
+// CHECK: encoding: [0x66,0x0f,0x29,0x0d,0xf0,0x1c,0xf0,0x1c]
+movapd %xmm1, 485498096
+
+// CHECK: movapd %xmm1, 485498096(%edx)
+// CHECK: encoding: [0x66,0x0f,0x29,0x8a,0xf0,0x1c,0xf0,0x1c]
+movapd %xmm1, 485498096(%edx)
+
+// CHECK: movapd %xmm1, -485498096(%edx,%eax,4)
+// CHECK: encoding: [0x66,0x0f,0x29,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+movapd %xmm1, -485498096(%edx,%eax,4)
+
+// CHECK: movapd %xmm1, 485498096(%edx,%eax,4)
+// CHECK: encoding: [0x66,0x0f,0x29,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+movapd %xmm1, 485498096(%edx,%eax,4)
+
+// CHECK: movapd %xmm1, 64(%edx,%eax)
+// CHECK: encoding: [0x66,0x0f,0x29,0x4c,0x02,0x40]
+movapd %xmm1, 64(%edx,%eax)
+
+// CHECK: movapd %xmm1, (%edx)
+// CHECK: encoding: [0x66,0x0f,0x29,0x0a]
+movapd %xmm1, (%edx)
+
+// CHECK: movapd %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x28,0xc9]
+movapd %xmm1, %xmm1
+
+// CHECK: movd -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x6e,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+movd -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: movd 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x6e,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+movd 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: movd 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x6e,0x8a,0xf0,0x1c,0xf0,0x1c]
+movd 485498096(%edx), %xmm1
+
+// CHECK: movd 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x6e,0x0d,0xf0,0x1c,0xf0,0x1c]
+movd 485498096, %xmm1
+
+// CHECK: movd 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x6e,0x4c,0x02,0x40]
+movd 64(%edx,%eax), %xmm1
+
+// CHECK: movd (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x6e,0x0a]
+movd (%edx), %xmm1
+
+// CHECK: movdq2q %xmm1, %mm4
+// CHECK: encoding: [0xf2,0x0f,0xd6,0xe1]
+movdq2q %xmm1, %mm4
+
+// CHECK: movdqa -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x6f,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+movdqa -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: movdqa 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x6f,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+movdqa 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: movdqa 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x6f,0x8a,0xf0,0x1c,0xf0,0x1c]
+movdqa 485498096(%edx), %xmm1
+
+// CHECK: movdqa 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x6f,0x0d,0xf0,0x1c,0xf0,0x1c]
+movdqa 485498096, %xmm1
+
+// CHECK: movdqa 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x6f,0x4c,0x02,0x40]
+movdqa 64(%edx,%eax), %xmm1
+
+// CHECK: movdqa (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x6f,0x0a]
+movdqa (%edx), %xmm1
+
+// CHECK: movdqa %xmm1, 485498096
+// CHECK: encoding: [0x66,0x0f,0x7f,0x0d,0xf0,0x1c,0xf0,0x1c]
+movdqa %xmm1, 485498096
+
+// CHECK: movdqa %xmm1, 485498096(%edx)
+// CHECK: encoding: [0x66,0x0f,0x7f,0x8a,0xf0,0x1c,0xf0,0x1c]
+movdqa %xmm1, 485498096(%edx)
+
+// CHECK: movdqa %xmm1, -485498096(%edx,%eax,4)
+// CHECK: encoding: [0x66,0x0f,0x7f,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+movdqa %xmm1, -485498096(%edx,%eax,4)
+
+// CHECK: movdqa %xmm1, 485498096(%edx,%eax,4)
+// CHECK: encoding: [0x66,0x0f,0x7f,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+movdqa %xmm1, 485498096(%edx,%eax,4)
+
+// CHECK: movdqa %xmm1, 64(%edx,%eax)
+// CHECK: encoding: [0x66,0x0f,0x7f,0x4c,0x02,0x40]
+movdqa %xmm1, 64(%edx,%eax)
+
+// CHECK: movdqa %xmm1, (%edx)
+// CHECK: encoding: [0x66,0x0f,0x7f,0x0a]
+movdqa %xmm1, (%edx)
+
+// CHECK: movdqa %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x6f,0xc9]
+movdqa %xmm1, %xmm1
+
+// CHECK: movdqu -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x6f,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+movdqu -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: movdqu 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x6f,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+movdqu 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: movdqu 485498096(%edx), %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x6f,0x8a,0xf0,0x1c,0xf0,0x1c]
+movdqu 485498096(%edx), %xmm1
+
+// CHECK: movdqu 485498096, %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x6f,0x0d,0xf0,0x1c,0xf0,0x1c]
+movdqu 485498096, %xmm1
+
+// CHECK: movdqu 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x6f,0x4c,0x02,0x40]
+movdqu 64(%edx,%eax), %xmm1
+
+// CHECK: movdqu (%edx), %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x6f,0x0a]
+movdqu (%edx), %xmm1
+
+// CHECK: movdqu %xmm1, 485498096
+// CHECK: encoding: [0xf3,0x0f,0x7f,0x0d,0xf0,0x1c,0xf0,0x1c]
+movdqu %xmm1, 485498096
+
+// CHECK: movdqu %xmm1, 485498096(%edx)
+// CHECK: encoding: [0xf3,0x0f,0x7f,0x8a,0xf0,0x1c,0xf0,0x1c]
+movdqu %xmm1, 485498096(%edx)
+
+// CHECK: movdqu %xmm1, -485498096(%edx,%eax,4)
+// CHECK: encoding: [0xf3,0x0f,0x7f,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+movdqu %xmm1, -485498096(%edx,%eax,4)
+
+// CHECK: movdqu %xmm1, 485498096(%edx,%eax,4)
+// CHECK: encoding: [0xf3,0x0f,0x7f,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+movdqu %xmm1, 485498096(%edx,%eax,4)
+
+// CHECK: movdqu %xmm1, 64(%edx,%eax)
+// CHECK: encoding: [0xf3,0x0f,0x7f,0x4c,0x02,0x40]
+movdqu %xmm1, 64(%edx,%eax)
+
+// CHECK: movdqu %xmm1, (%edx)
+// CHECK: encoding: [0xf3,0x0f,0x7f,0x0a]
+movdqu %xmm1, (%edx)
+
+// CHECK: movdqu %xmm1, %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x6f,0xc9]
+movdqu %xmm1, %xmm1
+
+// CHECK: movd %xmm1, 485498096
+// CHECK: encoding: [0x66,0x0f,0x7e,0x0d,0xf0,0x1c,0xf0,0x1c]
+movd %xmm1, 485498096
+
+// CHECK: movd %xmm1, 485498096(%edx)
+// CHECK: encoding: [0x66,0x0f,0x7e,0x8a,0xf0,0x1c,0xf0,0x1c]
+movd %xmm1, 485498096(%edx)
+
+// CHECK: movd %xmm1, -485498096(%edx,%eax,4)
+// CHECK: encoding: [0x66,0x0f,0x7e,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+movd %xmm1, -485498096(%edx,%eax,4)
+
+// CHECK: movd %xmm1, 485498096(%edx,%eax,4)
+// CHECK: encoding: [0x66,0x0f,0x7e,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+movd %xmm1, 485498096(%edx,%eax,4)
+
+// CHECK: movd %xmm1, 64(%edx,%eax)
+// CHECK: encoding: [0x66,0x0f,0x7e,0x4c,0x02,0x40]
+movd %xmm1, 64(%edx,%eax)
+
+// CHECK: movd %xmm1, (%edx)
+// CHECK: encoding: [0x66,0x0f,0x7e,0x0a]
+movd %xmm1, (%edx)
+
+// CHECK: movhpd -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x16,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+movhpd -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: movhpd 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x16,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+movhpd 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: movhpd 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x16,0x8a,0xf0,0x1c,0xf0,0x1c]
+movhpd 485498096(%edx), %xmm1
+
+// CHECK: movhpd 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x16,0x0d,0xf0,0x1c,0xf0,0x1c]
+movhpd 485498096, %xmm1
+
+// CHECK: movhpd 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x16,0x4c,0x02,0x40]
+movhpd 64(%edx,%eax), %xmm1
+
+// CHECK: movhpd (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x16,0x0a]
+movhpd (%edx), %xmm1
+
+// CHECK: movhpd %xmm1, 485498096
+// CHECK: encoding: [0x66,0x0f,0x17,0x0d,0xf0,0x1c,0xf0,0x1c]
+movhpd %xmm1, 485498096
+
+// CHECK: movhpd %xmm1, 485498096(%edx)
+// CHECK: encoding: [0x66,0x0f,0x17,0x8a,0xf0,0x1c,0xf0,0x1c]
+movhpd %xmm1, 485498096(%edx)
+
+// CHECK: movhpd %xmm1, -485498096(%edx,%eax,4)
+// CHECK: encoding: [0x66,0x0f,0x17,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+movhpd %xmm1, -485498096(%edx,%eax,4)
+
+// CHECK: movhpd %xmm1, 485498096(%edx,%eax,4)
+// CHECK: encoding: [0x66,0x0f,0x17,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+movhpd %xmm1, 485498096(%edx,%eax,4)
+
+// CHECK: movhpd %xmm1, 64(%edx,%eax)
+// CHECK: encoding: [0x66,0x0f,0x17,0x4c,0x02,0x40]
+movhpd %xmm1, 64(%edx,%eax)
+
+// CHECK: movhpd %xmm1, (%edx)
+// CHECK: encoding: [0x66,0x0f,0x17,0x0a]
+movhpd %xmm1, (%edx)
+
+// CHECK: movlpd -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x12,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+movlpd -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: movlpd 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x12,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+movlpd 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: movlpd 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x12,0x8a,0xf0,0x1c,0xf0,0x1c]
+movlpd 485498096(%edx), %xmm1
+
+// CHECK: movlpd 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x12,0x0d,0xf0,0x1c,0xf0,0x1c]
+movlpd 485498096, %xmm1
+
+// CHECK: movlpd 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x12,0x4c,0x02,0x40]
+movlpd 64(%edx,%eax), %xmm1
+
+// CHECK: movlpd (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x12,0x0a]
+movlpd (%edx), %xmm1
+
+// CHECK: movlpd %xmm1, 485498096
+// CHECK: encoding: [0x66,0x0f,0x13,0x0d,0xf0,0x1c,0xf0,0x1c]
+movlpd %xmm1, 485498096
+
+// CHECK: movlpd %xmm1, 485498096(%edx)
+// CHECK: encoding: [0x66,0x0f,0x13,0x8a,0xf0,0x1c,0xf0,0x1c]
+movlpd %xmm1, 485498096(%edx)
+
+// CHECK: movlpd %xmm1, -485498096(%edx,%eax,4)
+// CHECK: encoding: [0x66,0x0f,0x13,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+movlpd %xmm1, -485498096(%edx,%eax,4)
+
+// CHECK: movlpd %xmm1, 485498096(%edx,%eax,4)
+// CHECK: encoding: [0x66,0x0f,0x13,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+movlpd %xmm1, 485498096(%edx,%eax,4)
+
+// CHECK: movlpd %xmm1, 64(%edx,%eax)
+// CHECK: encoding: [0x66,0x0f,0x13,0x4c,0x02,0x40]
+movlpd %xmm1, 64(%edx,%eax)
+
+// CHECK: movlpd %xmm1, (%edx)
+// CHECK: encoding: [0x66,0x0f,0x13,0x0a]
+movlpd %xmm1, (%edx)
+
+// CHECK: movntdq %xmm1, 485498096
+// CHECK: encoding: [0x66,0x0f,0xe7,0x0d,0xf0,0x1c,0xf0,0x1c]
+movntdq %xmm1, 485498096
+
+// CHECK: movntdq %xmm1, 485498096(%edx)
+// CHECK: encoding: [0x66,0x0f,0xe7,0x8a,0xf0,0x1c,0xf0,0x1c]
+movntdq %xmm1, 485498096(%edx)
+
+// CHECK: movntdq %xmm1, -485498096(%edx,%eax,4)
+// CHECK: encoding: [0x66,0x0f,0xe7,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+movntdq %xmm1, -485498096(%edx,%eax,4)
+
+// CHECK: movntdq %xmm1, 485498096(%edx,%eax,4)
+// CHECK: encoding: [0x66,0x0f,0xe7,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+movntdq %xmm1, 485498096(%edx,%eax,4)
+
+// CHECK: movntdq %xmm1, 64(%edx,%eax)
+// CHECK: encoding: [0x66,0x0f,0xe7,0x4c,0x02,0x40]
+movntdq %xmm1, 64(%edx,%eax)
+
+// CHECK: movntdq %xmm1, (%edx)
+// CHECK: encoding: [0x66,0x0f,0xe7,0x0a]
+movntdq %xmm1, (%edx)
+
+// CHECK: movntpd %xmm1, 485498096
+// CHECK: encoding: [0x66,0x0f,0x2b,0x0d,0xf0,0x1c,0xf0,0x1c]
+movntpd %xmm1, 485498096
+
+// CHECK: movntpd %xmm1, 485498096(%edx)
+// CHECK: encoding: [0x66,0x0f,0x2b,0x8a,0xf0,0x1c,0xf0,0x1c]
+movntpd %xmm1, 485498096(%edx)
+
+// CHECK: movntpd %xmm1, -485498096(%edx,%eax,4)
+// CHECK: encoding: [0x66,0x0f,0x2b,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+movntpd %xmm1, -485498096(%edx,%eax,4)
+
+// CHECK: movntpd %xmm1, 485498096(%edx,%eax,4)
+// CHECK: encoding: [0x66,0x0f,0x2b,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+movntpd %xmm1, 485498096(%edx,%eax,4)
+
+// CHECK: movntpd %xmm1, 64(%edx,%eax)
+// CHECK: encoding: [0x66,0x0f,0x2b,0x4c,0x02,0x40]
+movntpd %xmm1, 64(%edx,%eax)
+
+// CHECK: movntpd %xmm1, (%edx)
+// CHECK: encoding: [0x66,0x0f,0x2b,0x0a]
+movntpd %xmm1, (%edx)
+
+// CHECK: movq2dq %mm4, %xmm1
+// CHECK: encoding: [0xf3,0x0f,0xd6,0xcc]
+movq2dq %mm4, %xmm1
+
+// CHECK: movq -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x7e,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+movq -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: movq 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x7e,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+movq 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: movq 485498096(%edx), %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x7e,0x8a,0xf0,0x1c,0xf0,0x1c]
+movq 485498096(%edx), %xmm1
+
+// CHECK: movq 485498096, %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x7e,0x0d,0xf0,0x1c,0xf0,0x1c]
+movq 485498096, %xmm1
+
+// CHECK: movq 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x7e,0x4c,0x02,0x40]
+movq 64(%edx,%eax), %xmm1
+
+// CHECK: movq (%edx), %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x7e,0x0a]
+movq (%edx), %xmm1
+
+// CHECK: movq %xmm1, 485498096
+// CHECK: encoding: [0x66,0x0f,0xd6,0x0d,0xf0,0x1c,0xf0,0x1c]
+movq %xmm1, 485498096
+
+// CHECK: movq %xmm1, 485498096(%edx)
+// CHECK: encoding: [0x66,0x0f,0xd6,0x8a,0xf0,0x1c,0xf0,0x1c]
+movq %xmm1, 485498096(%edx)
+
+// CHECK: movq %xmm1, -485498096(%edx,%eax,4)
+// CHECK: encoding: [0x66,0x0f,0xd6,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+movq %xmm1, -485498096(%edx,%eax,4)
+
+// CHECK: movq %xmm1, 485498096(%edx,%eax,4)
+// CHECK: encoding: [0x66,0x0f,0xd6,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+movq %xmm1, 485498096(%edx,%eax,4)
+
+// CHECK: movq %xmm1, 64(%edx,%eax)
+// CHECK: encoding: [0x66,0x0f,0xd6,0x4c,0x02,0x40]
+movq %xmm1, 64(%edx,%eax)
+
+// CHECK: movq %xmm1, (%edx)
+// CHECK: encoding: [0x66,0x0f,0xd6,0x0a]
+movq %xmm1, (%edx)
+
+// CHECK: movq %xmm1, %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x7e,0xc9]
+movq %xmm1, %xmm1
+
+// CHECK: movsd -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0xf2,0x0f,0x10,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+movsd -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: movsd 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0xf2,0x0f,0x10,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+movsd 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: movsd 485498096(%edx), %xmm1
+// CHECK: encoding: [0xf2,0x0f,0x10,0x8a,0xf0,0x1c,0xf0,0x1c]
+movsd 485498096(%edx), %xmm1
+
+// CHECK: movsd 485498096, %xmm1
+// CHECK: encoding: [0xf2,0x0f,0x10,0x0d,0xf0,0x1c,0xf0,0x1c]
+movsd 485498096, %xmm1
+
+// CHECK: movsd 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0xf2,0x0f,0x10,0x4c,0x02,0x40]
+movsd 64(%edx,%eax), %xmm1
+
+// CHECK: movsd (%edx), %xmm1
+// CHECK: encoding: [0xf2,0x0f,0x10,0x0a]
+movsd (%edx), %xmm1
+
+// CHECK: movsd %xmm1, 485498096
+// CHECK: encoding: [0xf2,0x0f,0x11,0x0d,0xf0,0x1c,0xf0,0x1c]
+movsd %xmm1, 485498096
+
+// CHECK: movsd %xmm1, 485498096(%edx)
+// CHECK: encoding: [0xf2,0x0f,0x11,0x8a,0xf0,0x1c,0xf0,0x1c]
+movsd %xmm1, 485498096(%edx)
+
+// CHECK: movsd %xmm1, -485498096(%edx,%eax,4)
+// CHECK: encoding: [0xf2,0x0f,0x11,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+movsd %xmm1, -485498096(%edx,%eax,4)
+
+// CHECK: movsd %xmm1, 485498096(%edx,%eax,4)
+// CHECK: encoding: [0xf2,0x0f,0x11,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+movsd %xmm1, 485498096(%edx,%eax,4)
+
+// CHECK: movsd %xmm1, 64(%edx,%eax)
+// CHECK: encoding: [0xf2,0x0f,0x11,0x4c,0x02,0x40]
+movsd %xmm1, 64(%edx,%eax)
+
+// CHECK: movsd %xmm1, (%edx)
+// CHECK: encoding: [0xf2,0x0f,0x11,0x0a]
+movsd %xmm1, (%edx)
+
+// CHECK: movsd %xmm1, %xmm1
+// CHECK: encoding: [0xf2,0x0f,0x10,0xc9]
+movsd %xmm1, %xmm1
+
+// CHECK: movupd -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x10,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+movupd -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: movupd 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x10,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+movupd 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: movupd 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x10,0x8a,0xf0,0x1c,0xf0,0x1c]
+movupd 485498096(%edx), %xmm1
+
+// CHECK: movupd 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x10,0x0d,0xf0,0x1c,0xf0,0x1c]
+movupd 485498096, %xmm1
+
+// CHECK: movupd 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x10,0x4c,0x02,0x40]
+movupd 64(%edx,%eax), %xmm1
+
+// CHECK: movupd (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x10,0x0a]
+movupd (%edx), %xmm1
+
+// CHECK: movupd %xmm1, 485498096
+// CHECK: encoding: [0x66,0x0f,0x11,0x0d,0xf0,0x1c,0xf0,0x1c]
+movupd %xmm1, 485498096
+
+// CHECK: movupd %xmm1, 485498096(%edx)
+// CHECK: encoding: [0x66,0x0f,0x11,0x8a,0xf0,0x1c,0xf0,0x1c]
+movupd %xmm1, 485498096(%edx)
+
+// CHECK: movupd %xmm1, -485498096(%edx,%eax,4)
+// CHECK: encoding: [0x66,0x0f,0x11,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+movupd %xmm1, -485498096(%edx,%eax,4)
+
+// CHECK: movupd %xmm1, 485498096(%edx,%eax,4)
+// CHECK: encoding: [0x66,0x0f,0x11,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+movupd %xmm1, 485498096(%edx,%eax,4)
+
+// CHECK: movupd %xmm1, 64(%edx,%eax)
+// CHECK: encoding: [0x66,0x0f,0x11,0x4c,0x02,0x40]
+movupd %xmm1, 64(%edx,%eax)
+
+// CHECK: movupd %xmm1, (%edx)
+// CHECK: encoding: [0x66,0x0f,0x11,0x0a]
+movupd %xmm1, (%edx)
+
+// CHECK: movupd %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x10,0xc9]
+movupd %xmm1, %xmm1
+
+// CHECK: mulpd -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x59,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+mulpd -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: mulpd 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x59,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+mulpd 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: mulpd 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x59,0x8a,0xf0,0x1c,0xf0,0x1c]
+mulpd 485498096(%edx), %xmm1
+
+// CHECK: mulpd 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x59,0x0d,0xf0,0x1c,0xf0,0x1c]
+mulpd 485498096, %xmm1
+
+// CHECK: mulpd 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x59,0x4c,0x02,0x40]
+mulpd 64(%edx,%eax), %xmm1
+
+// CHECK: mulpd (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x59,0x0a]
+mulpd (%edx), %xmm1
+
+// CHECK: mulpd %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x59,0xc9]
+mulpd %xmm1, %xmm1
+
+// CHECK: mulsd -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0xf2,0x0f,0x59,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+mulsd -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: mulsd 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0xf2,0x0f,0x59,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+mulsd 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: mulsd 485498096(%edx), %xmm1
+// CHECK: encoding: [0xf2,0x0f,0x59,0x8a,0xf0,0x1c,0xf0,0x1c]
+mulsd 485498096(%edx), %xmm1
+
+// CHECK: mulsd 485498096, %xmm1
+// CHECK: encoding: [0xf2,0x0f,0x59,0x0d,0xf0,0x1c,0xf0,0x1c]
+mulsd 485498096, %xmm1
+
+// CHECK: mulsd 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0xf2,0x0f,0x59,0x4c,0x02,0x40]
+mulsd 64(%edx,%eax), %xmm1
+
+// CHECK: mulsd (%edx), %xmm1
+// CHECK: encoding: [0xf2,0x0f,0x59,0x0a]
+mulsd (%edx), %xmm1
+
+// CHECK: mulsd %xmm1, %xmm1
+// CHECK: encoding: [0xf2,0x0f,0x59,0xc9]
+mulsd %xmm1, %xmm1
+
+// CHECK: orpd -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x56,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+orpd -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: orpd 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x56,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+orpd 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: orpd 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x56,0x8a,0xf0,0x1c,0xf0,0x1c]
+orpd 485498096(%edx), %xmm1
+
+// CHECK: orpd 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x56,0x0d,0xf0,0x1c,0xf0,0x1c]
+orpd 485498096, %xmm1
+
+// CHECK: orpd 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x56,0x4c,0x02,0x40]
+orpd 64(%edx,%eax), %xmm1
+
+// CHECK: orpd (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x56,0x0a]
+orpd (%edx), %xmm1
+
+// CHECK: orpd %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x56,0xc9]
+orpd %xmm1, %xmm1
+
+// CHECK: packssdw -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x6b,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+packssdw -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: packssdw 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x6b,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+packssdw 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: packssdw 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x6b,0x8a,0xf0,0x1c,0xf0,0x1c]
+packssdw 485498096(%edx), %xmm1
+
+// CHECK: packssdw 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x6b,0x0d,0xf0,0x1c,0xf0,0x1c]
+packssdw 485498096, %xmm1
+
+// CHECK: packssdw 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x6b,0x4c,0x02,0x40]
+packssdw 64(%edx,%eax), %xmm1
+
+// CHECK: packssdw (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x6b,0x0a]
+packssdw (%edx), %xmm1
+
+// CHECK: packssdw %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x6b,0xc9]
+packssdw %xmm1, %xmm1
+
+// CHECK: packsswb -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x63,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+packsswb -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: packsswb 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x63,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+packsswb 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: packsswb 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x63,0x8a,0xf0,0x1c,0xf0,0x1c]
+packsswb 485498096(%edx), %xmm1
+
+// CHECK: packsswb 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x63,0x0d,0xf0,0x1c,0xf0,0x1c]
+packsswb 485498096, %xmm1
+
+// CHECK: packsswb 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x63,0x4c,0x02,0x40]
+packsswb 64(%edx,%eax), %xmm1
+
+// CHECK: packsswb (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x63,0x0a]
+packsswb (%edx), %xmm1
+
+// CHECK: packsswb %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x63,0xc9]
+packsswb %xmm1, %xmm1
+
+// CHECK: packuswb -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x67,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+packuswb -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: packuswb 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x67,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+packuswb 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: packuswb 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x67,0x8a,0xf0,0x1c,0xf0,0x1c]
+packuswb 485498096(%edx), %xmm1
+
+// CHECK: packuswb 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x67,0x0d,0xf0,0x1c,0xf0,0x1c]
+packuswb 485498096, %xmm1
+
+// CHECK: packuswb 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x67,0x4c,0x02,0x40]
+packuswb 64(%edx,%eax), %xmm1
+
+// CHECK: packuswb (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x67,0x0a]
+packuswb (%edx), %xmm1
+
+// CHECK: packuswb %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x67,0xc9]
+packuswb %xmm1, %xmm1
+
+// CHECK: paddb -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xfc,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+paddb -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: paddb 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xfc,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+paddb 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: paddb 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xfc,0x8a,0xf0,0x1c,0xf0,0x1c]
+paddb 485498096(%edx), %xmm1
+
+// CHECK: paddb 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0xfc,0x0d,0xf0,0x1c,0xf0,0x1c]
+paddb 485498096, %xmm1
+
+// CHECK: paddb 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xfc,0x4c,0x02,0x40]
+paddb 64(%edx,%eax), %xmm1
+
+// CHECK: paddb (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xfc,0x0a]
+paddb (%edx), %xmm1
+
+// CHECK: paddb %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0xfc,0xc9]
+paddb %xmm1, %xmm1
+
+// CHECK: paddd -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xfe,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+paddd -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: paddd 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xfe,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+paddd 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: paddd 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xfe,0x8a,0xf0,0x1c,0xf0,0x1c]
+paddd 485498096(%edx), %xmm1
+
+// CHECK: paddd 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0xfe,0x0d,0xf0,0x1c,0xf0,0x1c]
+paddd 485498096, %xmm1
+
+// CHECK: paddd 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xfe,0x4c,0x02,0x40]
+paddd 64(%edx,%eax), %xmm1
+
+// CHECK: paddd (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xfe,0x0a]
+paddd (%edx), %xmm1
+
+// CHECK: paddd %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0xfe,0xc9]
+paddd %xmm1, %xmm1
+
+// CHECK: paddq -485498096(%edx,%eax,4), %mm4
+// CHECK: encoding: [0x0f,0xd4,0xa4,0x82,0x10,0xe3,0x0f,0xe3]
+paddq -485498096(%edx,%eax,4), %mm4
+
+// CHECK: paddq 485498096(%edx,%eax,4), %mm4
+// CHECK: encoding: [0x0f,0xd4,0xa4,0x82,0xf0,0x1c,0xf0,0x1c]
+paddq 485498096(%edx,%eax,4), %mm4
+
+// CHECK: paddq -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xd4,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+paddq -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: paddq 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xd4,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+paddq 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: paddq 485498096(%edx), %mm4
+// CHECK: encoding: [0x0f,0xd4,0xa2,0xf0,0x1c,0xf0,0x1c]
+paddq 485498096(%edx), %mm4
+
+// CHECK: paddq 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xd4,0x8a,0xf0,0x1c,0xf0,0x1c]
+paddq 485498096(%edx), %xmm1
+
+// CHECK: paddq 485498096, %mm4
+// CHECK: encoding: [0x0f,0xd4,0x25,0xf0,0x1c,0xf0,0x1c]
+paddq 485498096, %mm4
+
+// CHECK: paddq 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0xd4,0x0d,0xf0,0x1c,0xf0,0x1c]
+paddq 485498096, %xmm1
+
+// CHECK: paddq 64(%edx,%eax), %mm4
+// CHECK: encoding: [0x0f,0xd4,0x64,0x02,0x40]
+paddq 64(%edx,%eax), %mm4
+
+// CHECK: paddq 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xd4,0x4c,0x02,0x40]
+paddq 64(%edx,%eax), %xmm1
+
+// CHECK: paddq (%edx), %mm4
+// CHECK: encoding: [0x0f,0xd4,0x22]
+paddq (%edx), %mm4
+
+// CHECK: paddq (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xd4,0x0a]
+paddq (%edx), %xmm1
+
+// CHECK: paddq %mm4, %mm4
+// CHECK: encoding: [0x0f,0xd4,0xe4]
+paddq %mm4, %mm4
+
+// CHECK: paddq %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0xd4,0xc9]
+paddq %xmm1, %xmm1
+
+// CHECK: paddsb -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xec,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+paddsb -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: paddsb 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xec,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+paddsb 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: paddsb 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xec,0x8a,0xf0,0x1c,0xf0,0x1c]
+paddsb 485498096(%edx), %xmm1
+
+// CHECK: paddsb 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0xec,0x0d,0xf0,0x1c,0xf0,0x1c]
+paddsb 485498096, %xmm1
+
+// CHECK: paddsb 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xec,0x4c,0x02,0x40]
+paddsb 64(%edx,%eax), %xmm1
+
+// CHECK: paddsb (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xec,0x0a]
+paddsb (%edx), %xmm1
+
+// CHECK: paddsb %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0xec,0xc9]
+paddsb %xmm1, %xmm1
+
+// CHECK: paddsw -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xed,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+paddsw -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: paddsw 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xed,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+paddsw 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: paddsw 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xed,0x8a,0xf0,0x1c,0xf0,0x1c]
+paddsw 485498096(%edx), %xmm1
+
+// CHECK: paddsw 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0xed,0x0d,0xf0,0x1c,0xf0,0x1c]
+paddsw 485498096, %xmm1
+
+// CHECK: paddsw 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xed,0x4c,0x02,0x40]
+paddsw 64(%edx,%eax), %xmm1
+
+// CHECK: paddsw (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xed,0x0a]
+paddsw (%edx), %xmm1
+
+// CHECK: paddsw %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0xed,0xc9]
+paddsw %xmm1, %xmm1
+
+// CHECK: paddusb -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xdc,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+paddusb -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: paddusb 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xdc,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+paddusb 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: paddusb 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xdc,0x8a,0xf0,0x1c,0xf0,0x1c]
+paddusb 485498096(%edx), %xmm1
+
+// CHECK: paddusb 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0xdc,0x0d,0xf0,0x1c,0xf0,0x1c]
+paddusb 485498096, %xmm1
+
+// CHECK: paddusb 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xdc,0x4c,0x02,0x40]
+paddusb 64(%edx,%eax), %xmm1
+
+// CHECK: paddusb (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xdc,0x0a]
+paddusb (%edx), %xmm1
+
+// CHECK: paddusb %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0xdc,0xc9]
+paddusb %xmm1, %xmm1
+
+// CHECK: paddusw -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xdd,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+paddusw -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: paddusw 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xdd,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+paddusw 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: paddusw 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xdd,0x8a,0xf0,0x1c,0xf0,0x1c]
+paddusw 485498096(%edx), %xmm1
+
+// CHECK: paddusw 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0xdd,0x0d,0xf0,0x1c,0xf0,0x1c]
+paddusw 485498096, %xmm1
+
+// CHECK: paddusw 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xdd,0x4c,0x02,0x40]
+paddusw 64(%edx,%eax), %xmm1
+
+// CHECK: paddusw (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xdd,0x0a]
+paddusw (%edx), %xmm1
+
+// CHECK: paddusw %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0xdd,0xc9]
+paddusw %xmm1, %xmm1
+
+// CHECK: paddw -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xfd,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+paddw -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: paddw 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xfd,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+paddw 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: paddw 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xfd,0x8a,0xf0,0x1c,0xf0,0x1c]
+paddw 485498096(%edx), %xmm1
+
+// CHECK: paddw 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0xfd,0x0d,0xf0,0x1c,0xf0,0x1c]
+paddw 485498096, %xmm1
+
+// CHECK: paddw 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xfd,0x4c,0x02,0x40]
+paddw 64(%edx,%eax), %xmm1
+
+// CHECK: paddw (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xfd,0x0a]
+paddw (%edx), %xmm1
+
+// CHECK: paddw %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0xfd,0xc9]
+paddw %xmm1, %xmm1
+
+// CHECK: pand -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xdb,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+pand -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pand 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xdb,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+pand 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pand 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xdb,0x8a,0xf0,0x1c,0xf0,0x1c]
+pand 485498096(%edx), %xmm1
+
+// CHECK: pand 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0xdb,0x0d,0xf0,0x1c,0xf0,0x1c]
+pand 485498096, %xmm1
+
+// CHECK: pand 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xdb,0x4c,0x02,0x40]
+pand 64(%edx,%eax), %xmm1
+
+// CHECK: pand (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xdb,0x0a]
+pand (%edx), %xmm1
+
+// CHECK: pandn -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xdf,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+pandn -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pandn 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xdf,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+pandn 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pandn 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xdf,0x8a,0xf0,0x1c,0xf0,0x1c]
+pandn 485498096(%edx), %xmm1
+
+// CHECK: pandn 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0xdf,0x0d,0xf0,0x1c,0xf0,0x1c]
+pandn 485498096, %xmm1
+
+// CHECK: pandn 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xdf,0x4c,0x02,0x40]
+pandn 64(%edx,%eax), %xmm1
+
+// CHECK: pandn (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xdf,0x0a]
+pandn (%edx), %xmm1
+
+// CHECK: pandn %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0xdf,0xc9]
+pandn %xmm1, %xmm1
+
+// CHECK: pand %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0xdb,0xc9]
+pand %xmm1, %xmm1
+
+// CHECK: pavgb -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xe0,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+pavgb -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pavgb 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xe0,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+pavgb 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pavgb 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xe0,0x8a,0xf0,0x1c,0xf0,0x1c]
+pavgb 485498096(%edx), %xmm1
+
+// CHECK: pavgb 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0xe0,0x0d,0xf0,0x1c,0xf0,0x1c]
+pavgb 485498096, %xmm1
+
+// CHECK: pavgb 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xe0,0x4c,0x02,0x40]
+pavgb 64(%edx,%eax), %xmm1
+
+// CHECK: pavgb (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xe0,0x0a]
+pavgb (%edx), %xmm1
+
+// CHECK: pavgb %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0xe0,0xc9]
+pavgb %xmm1, %xmm1
+
+// CHECK: pavgw -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xe3,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+pavgw -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pavgw 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xe3,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+pavgw 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pavgw 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xe3,0x8a,0xf0,0x1c,0xf0,0x1c]
+pavgw 485498096(%edx), %xmm1
+
+// CHECK: pavgw 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0xe3,0x0d,0xf0,0x1c,0xf0,0x1c]
+pavgw 485498096, %xmm1
+
+// CHECK: pavgw 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xe3,0x4c,0x02,0x40]
+pavgw 64(%edx,%eax), %xmm1
+
+// CHECK: pavgw (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xe3,0x0a]
+pavgw (%edx), %xmm1
+
+// CHECK: pavgw %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0xe3,0xc9]
+pavgw %xmm1, %xmm1
+
+// CHECK: pcmpeqb -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x74,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+pcmpeqb -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pcmpeqb 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x74,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+pcmpeqb 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pcmpeqb 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x74,0x8a,0xf0,0x1c,0xf0,0x1c]
+pcmpeqb 485498096(%edx), %xmm1
+
+// CHECK: pcmpeqb 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x74,0x0d,0xf0,0x1c,0xf0,0x1c]
+pcmpeqb 485498096, %xmm1
+
+// CHECK: pcmpeqb 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x74,0x4c,0x02,0x40]
+pcmpeqb 64(%edx,%eax), %xmm1
+
+// CHECK: pcmpeqb (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x74,0x0a]
+pcmpeqb (%edx), %xmm1
+
+// CHECK: pcmpeqb %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x74,0xc9]
+pcmpeqb %xmm1, %xmm1
+
+// CHECK: pcmpeqd -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x76,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+pcmpeqd -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pcmpeqd 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x76,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+pcmpeqd 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pcmpeqd 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x76,0x8a,0xf0,0x1c,0xf0,0x1c]
+pcmpeqd 485498096(%edx), %xmm1
+
+// CHECK: pcmpeqd 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x76,0x0d,0xf0,0x1c,0xf0,0x1c]
+pcmpeqd 485498096, %xmm1
+
+// CHECK: pcmpeqd 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x76,0x4c,0x02,0x40]
+pcmpeqd 64(%edx,%eax), %xmm1
+
+// CHECK: pcmpeqd (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x76,0x0a]
+pcmpeqd (%edx), %xmm1
+
+// CHECK: pcmpeqd %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x76,0xc9]
+pcmpeqd %xmm1, %xmm1
+
+// CHECK: pcmpeqw -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x75,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+pcmpeqw -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pcmpeqw 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x75,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+pcmpeqw 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pcmpeqw 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x75,0x8a,0xf0,0x1c,0xf0,0x1c]
+pcmpeqw 485498096(%edx), %xmm1
+
+// CHECK: pcmpeqw 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x75,0x0d,0xf0,0x1c,0xf0,0x1c]
+pcmpeqw 485498096, %xmm1
+
+// CHECK: pcmpeqw 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x75,0x4c,0x02,0x40]
+pcmpeqw 64(%edx,%eax), %xmm1
+
+// CHECK: pcmpeqw (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x75,0x0a]
+pcmpeqw (%edx), %xmm1
+
+// CHECK: pcmpeqw %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x75,0xc9]
+pcmpeqw %xmm1, %xmm1
+
+// CHECK: pcmpgtb -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x64,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+pcmpgtb -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pcmpgtb 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x64,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+pcmpgtb 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pcmpgtb 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x64,0x8a,0xf0,0x1c,0xf0,0x1c]
+pcmpgtb 485498096(%edx), %xmm1
+
+// CHECK: pcmpgtb 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x64,0x0d,0xf0,0x1c,0xf0,0x1c]
+pcmpgtb 485498096, %xmm1
+
+// CHECK: pcmpgtb 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x64,0x4c,0x02,0x40]
+pcmpgtb 64(%edx,%eax), %xmm1
+
+// CHECK: pcmpgtb (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x64,0x0a]
+pcmpgtb (%edx), %xmm1
+
+// CHECK: pcmpgtb %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x64,0xc9]
+pcmpgtb %xmm1, %xmm1
+
+// CHECK: pcmpgtd -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x66,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+pcmpgtd -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pcmpgtd 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x66,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+pcmpgtd 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pcmpgtd 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x66,0x8a,0xf0,0x1c,0xf0,0x1c]
+pcmpgtd 485498096(%edx), %xmm1
+
+// CHECK: pcmpgtd 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x66,0x0d,0xf0,0x1c,0xf0,0x1c]
+pcmpgtd 485498096, %xmm1
+
+// CHECK: pcmpgtd 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x66,0x4c,0x02,0x40]
+pcmpgtd 64(%edx,%eax), %xmm1
+
+// CHECK: pcmpgtd (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x66,0x0a]
+pcmpgtd (%edx), %xmm1
+
+// CHECK: pcmpgtd %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x66,0xc9]
+pcmpgtd %xmm1, %xmm1
+
+// CHECK: pcmpgtw -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x65,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+pcmpgtw -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pcmpgtw 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x65,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+pcmpgtw 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pcmpgtw 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x65,0x8a,0xf0,0x1c,0xf0,0x1c]
+pcmpgtw 485498096(%edx), %xmm1
+
+// CHECK: pcmpgtw 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x65,0x0d,0xf0,0x1c,0xf0,0x1c]
+pcmpgtw 485498096, %xmm1
+
+// CHECK: pcmpgtw 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x65,0x4c,0x02,0x40]
+pcmpgtw 64(%edx,%eax), %xmm1
+
+// CHECK: pcmpgtw (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x65,0x0a]
+pcmpgtw (%edx), %xmm1
+
+// CHECK: pcmpgtw %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x65,0xc9]
+pcmpgtw %xmm1, %xmm1
+
+// CHECK: pextrw $0, %xmm1, %eax
+// CHECK: encoding: [0x66,0x0f,0xc5,0xc1,0x00]
+pextrw $0, %xmm1, %eax
+
+// CHECK: pinsrw $0, -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xc4,0x8c,0x82,0x10,0xe3,0x0f,0xe3,0x00]
+pinsrw $0, -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pinsrw $0, 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xc4,0x8c,0x82,0xf0,0x1c,0xf0,0x1c,0x00]
+pinsrw $0, 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pinsrw $0, 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xc4,0x8a,0xf0,0x1c,0xf0,0x1c,0x00]
+pinsrw $0, 485498096(%edx), %xmm1
+
+// CHECK: pinsrw $0, 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0xc4,0x0d,0xf0,0x1c,0xf0,0x1c,0x00]
+pinsrw $0, 485498096, %xmm1
+
+// CHECK: pinsrw $0, 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xc4,0x4c,0x02,0x40,0x00]
+pinsrw $0, 64(%edx,%eax), %xmm1
+
+// CHECK: pinsrw $0, (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xc4,0x0a,0x00]
+pinsrw $0, (%edx), %xmm1
+
+// CHECK: pmaddwd -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xf5,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+pmaddwd -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pmaddwd 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xf5,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+pmaddwd 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pmaddwd 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xf5,0x8a,0xf0,0x1c,0xf0,0x1c]
+pmaddwd 485498096(%edx), %xmm1
+
+// CHECK: pmaddwd 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0xf5,0x0d,0xf0,0x1c,0xf0,0x1c]
+pmaddwd 485498096, %xmm1
+
+// CHECK: pmaddwd 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xf5,0x4c,0x02,0x40]
+pmaddwd 64(%edx,%eax), %xmm1
+
+// CHECK: pmaddwd (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xf5,0x0a]
+pmaddwd (%edx), %xmm1
+
+// CHECK: pmaddwd %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0xf5,0xc9]
+pmaddwd %xmm1, %xmm1
+
+// CHECK: pmaxsw -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xee,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+pmaxsw -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pmaxsw 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xee,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+pmaxsw 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pmaxsw 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xee,0x8a,0xf0,0x1c,0xf0,0x1c]
+pmaxsw 485498096(%edx), %xmm1
+
+// CHECK: pmaxsw 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0xee,0x0d,0xf0,0x1c,0xf0,0x1c]
+pmaxsw 485498096, %xmm1
+
+// CHECK: pmaxsw 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xee,0x4c,0x02,0x40]
+pmaxsw 64(%edx,%eax), %xmm1
+
+// CHECK: pmaxsw (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xee,0x0a]
+pmaxsw (%edx), %xmm1
+
+// CHECK: pmaxsw %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0xee,0xc9]
+pmaxsw %xmm1, %xmm1
+
+// CHECK: pmaxub -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xde,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+pmaxub -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pmaxub 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xde,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+pmaxub 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pmaxub 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xde,0x8a,0xf0,0x1c,0xf0,0x1c]
+pmaxub 485498096(%edx), %xmm1
+
+// CHECK: pmaxub 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0xde,0x0d,0xf0,0x1c,0xf0,0x1c]
+pmaxub 485498096, %xmm1
+
+// CHECK: pmaxub 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xde,0x4c,0x02,0x40]
+pmaxub 64(%edx,%eax), %xmm1
+
+// CHECK: pmaxub (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xde,0x0a]
+pmaxub (%edx), %xmm1
+
+// CHECK: pmaxub %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0xde,0xc9]
+pmaxub %xmm1, %xmm1
+
+// CHECK: pminsw -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xea,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+pminsw -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pminsw 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xea,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+pminsw 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pminsw 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xea,0x8a,0xf0,0x1c,0xf0,0x1c]
+pminsw 485498096(%edx), %xmm1
+
+// CHECK: pminsw 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0xea,0x0d,0xf0,0x1c,0xf0,0x1c]
+pminsw 485498096, %xmm1
+
+// CHECK: pminsw 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xea,0x4c,0x02,0x40]
+pminsw 64(%edx,%eax), %xmm1
+
+// CHECK: pminsw (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xea,0x0a]
+pminsw (%edx), %xmm1
+
+// CHECK: pminsw %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0xea,0xc9]
+pminsw %xmm1, %xmm1
+
+// CHECK: pminub -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xda,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+pminub -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pminub 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xda,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+pminub 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pminub 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xda,0x8a,0xf0,0x1c,0xf0,0x1c]
+pminub 485498096(%edx), %xmm1
+
+// CHECK: pminub 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0xda,0x0d,0xf0,0x1c,0xf0,0x1c]
+pminub 485498096, %xmm1
+
+// CHECK: pminub 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xda,0x4c,0x02,0x40]
+pminub 64(%edx,%eax), %xmm1
+
+// CHECK: pminub (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xda,0x0a]
+pminub (%edx), %xmm1
+
+// CHECK: pminub %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0xda,0xc9]
+pminub %xmm1, %xmm1
+
+// CHECK: pmulhuw -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xe4,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+pmulhuw -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pmulhuw 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xe4,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+pmulhuw 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pmulhuw 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xe4,0x8a,0xf0,0x1c,0xf0,0x1c]
+pmulhuw 485498096(%edx), %xmm1
+
+// CHECK: pmulhuw 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0xe4,0x0d,0xf0,0x1c,0xf0,0x1c]
+pmulhuw 485498096, %xmm1
+
+// CHECK: pmulhuw 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xe4,0x4c,0x02,0x40]
+pmulhuw 64(%edx,%eax), %xmm1
+
+// CHECK: pmulhuw (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xe4,0x0a]
+pmulhuw (%edx), %xmm1
+
+// CHECK: pmulhuw %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0xe4,0xc9]
+pmulhuw %xmm1, %xmm1
+
+// CHECK: pmulhw -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xe5,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+pmulhw -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pmulhw 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xe5,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+pmulhw 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pmulhw 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xe5,0x8a,0xf0,0x1c,0xf0,0x1c]
+pmulhw 485498096(%edx), %xmm1
+
+// CHECK: pmulhw 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0xe5,0x0d,0xf0,0x1c,0xf0,0x1c]
+pmulhw 485498096, %xmm1
+
+// CHECK: pmulhw 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xe5,0x4c,0x02,0x40]
+pmulhw 64(%edx,%eax), %xmm1
+
+// CHECK: pmulhw (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xe5,0x0a]
+pmulhw (%edx), %xmm1
+
+// CHECK: pmulhw %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0xe5,0xc9]
+pmulhw %xmm1, %xmm1
+
+// CHECK: pmullw -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xd5,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+pmullw -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pmullw 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xd5,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+pmullw 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pmullw 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xd5,0x8a,0xf0,0x1c,0xf0,0x1c]
+pmullw 485498096(%edx), %xmm1
+
+// CHECK: pmullw 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0xd5,0x0d,0xf0,0x1c,0xf0,0x1c]
+pmullw 485498096, %xmm1
+
+// CHECK: pmullw 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xd5,0x4c,0x02,0x40]
+pmullw 64(%edx,%eax), %xmm1
+
+// CHECK: pmullw (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xd5,0x0a]
+pmullw (%edx), %xmm1
+
+// CHECK: pmullw %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0xd5,0xc9]
+pmullw %xmm1, %xmm1
+
+// CHECK: pmuludq -485498096(%edx,%eax,4), %mm4
+// CHECK: encoding: [0x0f,0xf4,0xa4,0x82,0x10,0xe3,0x0f,0xe3]
+pmuludq -485498096(%edx,%eax,4), %mm4
+
+// CHECK: pmuludq 485498096(%edx,%eax,4), %mm4
+// CHECK: encoding: [0x0f,0xf4,0xa4,0x82,0xf0,0x1c,0xf0,0x1c]
+pmuludq 485498096(%edx,%eax,4), %mm4
+
+// CHECK: pmuludq -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xf4,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+pmuludq -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pmuludq 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xf4,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+pmuludq 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pmuludq 485498096(%edx), %mm4
+// CHECK: encoding: [0x0f,0xf4,0xa2,0xf0,0x1c,0xf0,0x1c]
+pmuludq 485498096(%edx), %mm4
+
+// CHECK: pmuludq 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xf4,0x8a,0xf0,0x1c,0xf0,0x1c]
+pmuludq 485498096(%edx), %xmm1
+
+// CHECK: pmuludq 485498096, %mm4
+// CHECK: encoding: [0x0f,0xf4,0x25,0xf0,0x1c,0xf0,0x1c]
+pmuludq 485498096, %mm4
+
+// CHECK: pmuludq 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0xf4,0x0d,0xf0,0x1c,0xf0,0x1c]
+pmuludq 485498096, %xmm1
+
+// CHECK: pmuludq 64(%edx,%eax), %mm4
+// CHECK: encoding: [0x0f,0xf4,0x64,0x02,0x40]
+pmuludq 64(%edx,%eax), %mm4
+
+// CHECK: pmuludq 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xf4,0x4c,0x02,0x40]
+pmuludq 64(%edx,%eax), %xmm1
+
+// CHECK: pmuludq (%edx), %mm4
+// CHECK: encoding: [0x0f,0xf4,0x22]
+pmuludq (%edx), %mm4
+
+// CHECK: pmuludq (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xf4,0x0a]
+pmuludq (%edx), %xmm1
+
+// CHECK: pmuludq %mm4, %mm4
+// CHECK: encoding: [0x0f,0xf4,0xe4]
+pmuludq %mm4, %mm4
+
+// CHECK: pmuludq %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0xf4,0xc9]
+pmuludq %xmm1, %xmm1
+
+// CHECK: por -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xeb,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+por -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: por 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xeb,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+por 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: por 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xeb,0x8a,0xf0,0x1c,0xf0,0x1c]
+por 485498096(%edx), %xmm1
+
+// CHECK: por 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0xeb,0x0d,0xf0,0x1c,0xf0,0x1c]
+por 485498096, %xmm1
+
+// CHECK: por 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xeb,0x4c,0x02,0x40]
+por 64(%edx,%eax), %xmm1
+
+// CHECK: por (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xeb,0x0a]
+por (%edx), %xmm1
+
+// CHECK: por %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0xeb,0xc9]
+por %xmm1, %xmm1
+
+// CHECK: psadbw -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xf6,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+psadbw -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: psadbw 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xf6,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+psadbw 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: psadbw 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xf6,0x8a,0xf0,0x1c,0xf0,0x1c]
+psadbw 485498096(%edx), %xmm1
+
+// CHECK: psadbw 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0xf6,0x0d,0xf0,0x1c,0xf0,0x1c]
+psadbw 485498096, %xmm1
+
+// CHECK: psadbw 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xf6,0x4c,0x02,0x40]
+psadbw 64(%edx,%eax), %xmm1
+
+// CHECK: psadbw (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xf6,0x0a]
+psadbw (%edx), %xmm1
+
+// CHECK: psadbw %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0xf6,0xc9]
+psadbw %xmm1, %xmm1
+
+// CHECK: pshufd $0, -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x70,0x8c,0x82,0x10,0xe3,0x0f,0xe3,0x00]
+pshufd $0, -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pshufd $0, 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x70,0x8c,0x82,0xf0,0x1c,0xf0,0x1c,0x00]
+pshufd $0, 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pshufd $0, 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x70,0x8a,0xf0,0x1c,0xf0,0x1c,0x00]
+pshufd $0, 485498096(%edx), %xmm1
+
+// CHECK: pshufd $0, 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x70,0x0d,0xf0,0x1c,0xf0,0x1c,0x00]
+pshufd $0, 485498096, %xmm1
+
+// CHECK: pshufd $0, 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x70,0x4c,0x02,0x40,0x00]
+pshufd $0, 64(%edx,%eax), %xmm1
+
+// CHECK: pshufd $0, (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x70,0x0a,0x00]
+pshufd $0, (%edx), %xmm1
+
+// CHECK: pshufd $0, %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x70,0xc9,0x00]
+pshufd $0, %xmm1, %xmm1
+
+// CHECK: pshufhw $0, -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x70,0x8c,0x82,0x10,0xe3,0x0f,0xe3,0x00]
+pshufhw $0, -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pshufhw $0, 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x70,0x8c,0x82,0xf0,0x1c,0xf0,0x1c,0x00]
+pshufhw $0, 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pshufhw $0, 485498096(%edx), %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x70,0x8a,0xf0,0x1c,0xf0,0x1c,0x00]
+pshufhw $0, 485498096(%edx), %xmm1
+
+// CHECK: pshufhw $0, 485498096, %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x70,0x0d,0xf0,0x1c,0xf0,0x1c,0x00]
+pshufhw $0, 485498096, %xmm1
+
+// CHECK: pshufhw $0, 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x70,0x4c,0x02,0x40,0x00]
+pshufhw $0, 64(%edx,%eax), %xmm1
+
+// CHECK: pshufhw $0, (%edx), %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x70,0x0a,0x00]
+pshufhw $0, (%edx), %xmm1
+
+// CHECK: pshufhw $0, %xmm1, %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x70,0xc9,0x00]
+pshufhw $0, %xmm1, %xmm1
+
+// CHECK: pshuflw $0, -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0xf2,0x0f,0x70,0x8c,0x82,0x10,0xe3,0x0f,0xe3,0x00]
+pshuflw $0, -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pshuflw $0, 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0xf2,0x0f,0x70,0x8c,0x82,0xf0,0x1c,0xf0,0x1c,0x00]
+pshuflw $0, 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pshuflw $0, 485498096(%edx), %xmm1
+// CHECK: encoding: [0xf2,0x0f,0x70,0x8a,0xf0,0x1c,0xf0,0x1c,0x00]
+pshuflw $0, 485498096(%edx), %xmm1
+
+// CHECK: pshuflw $0, 485498096, %xmm1
+// CHECK: encoding: [0xf2,0x0f,0x70,0x0d,0xf0,0x1c,0xf0,0x1c,0x00]
+pshuflw $0, 485498096, %xmm1
+
+// CHECK: pshuflw $0, 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0xf2,0x0f,0x70,0x4c,0x02,0x40,0x00]
+pshuflw $0, 64(%edx,%eax), %xmm1
+
+// CHECK: pshuflw $0, (%edx), %xmm1
+// CHECK: encoding: [0xf2,0x0f,0x70,0x0a,0x00]
+pshuflw $0, (%edx), %xmm1
+
+// CHECK: pshuflw $0, %xmm1, %xmm1
+// CHECK: encoding: [0xf2,0x0f,0x70,0xc9,0x00]
+pshuflw $0, %xmm1, %xmm1
+
+// CHECK: pslld $0, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x72,0xf1,0x00]
+pslld $0, %xmm1
+
+// CHECK: pslld -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xf2,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+pslld -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pslld 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xf2,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+pslld 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pslld 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xf2,0x8a,0xf0,0x1c,0xf0,0x1c]
+pslld 485498096(%edx), %xmm1
+
+// CHECK: pslld 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0xf2,0x0d,0xf0,0x1c,0xf0,0x1c]
+pslld 485498096, %xmm1
+
+// CHECK: pslld 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xf2,0x4c,0x02,0x40]
+pslld 64(%edx,%eax), %xmm1
+
+// CHECK: pslld (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xf2,0x0a]
+pslld (%edx), %xmm1
+
+// CHECK: pslldq $0, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x73,0xf9,0x00]
+pslldq $0, %xmm1
+
+// CHECK: pslld %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0xf2,0xc9]
+pslld %xmm1, %xmm1
+
+// CHECK: psllq $0, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x73,0xf1,0x00]
+psllq $0, %xmm1
+
+// CHECK: psllq -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xf3,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+psllq -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: psllq 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xf3,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+psllq 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: psllq 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xf3,0x8a,0xf0,0x1c,0xf0,0x1c]
+psllq 485498096(%edx), %xmm1
+
+// CHECK: psllq 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0xf3,0x0d,0xf0,0x1c,0xf0,0x1c]
+psllq 485498096, %xmm1
+
+// CHECK: psllq 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xf3,0x4c,0x02,0x40]
+psllq 64(%edx,%eax), %xmm1
+
+// CHECK: psllq (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xf3,0x0a]
+psllq (%edx), %xmm1
+
+// CHECK: psllq %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0xf3,0xc9]
+psllq %xmm1, %xmm1
+
+// CHECK: psllw $0, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x71,0xf1,0x00]
+psllw $0, %xmm1
+
+// CHECK: psllw -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xf1,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+psllw -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: psllw 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xf1,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+psllw 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: psllw 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xf1,0x8a,0xf0,0x1c,0xf0,0x1c]
+psllw 485498096(%edx), %xmm1
+
+// CHECK: psllw 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0xf1,0x0d,0xf0,0x1c,0xf0,0x1c]
+psllw 485498096, %xmm1
+
+// CHECK: psllw 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xf1,0x4c,0x02,0x40]
+psllw 64(%edx,%eax), %xmm1
+
+// CHECK: psllw (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xf1,0x0a]
+psllw (%edx), %xmm1
+
+// CHECK: psllw %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0xf1,0xc9]
+psllw %xmm1, %xmm1
+
+// CHECK: psrad $0, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x72,0xe1,0x00]
+psrad $0, %xmm1
+
+// CHECK: psrad -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xe2,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+psrad -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: psrad 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xe2,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+psrad 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: psrad 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xe2,0x8a,0xf0,0x1c,0xf0,0x1c]
+psrad 485498096(%edx), %xmm1
+
+// CHECK: psrad 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0xe2,0x0d,0xf0,0x1c,0xf0,0x1c]
+psrad 485498096, %xmm1
+
+// CHECK: psrad 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xe2,0x4c,0x02,0x40]
+psrad 64(%edx,%eax), %xmm1
+
+// CHECK: psrad (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xe2,0x0a]
+psrad (%edx), %xmm1
+
+// CHECK: psrad %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0xe2,0xc9]
+psrad %xmm1, %xmm1
+
+// CHECK: psraw $0, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x71,0xe1,0x00]
+psraw $0, %xmm1
+
+// CHECK: psraw -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xe1,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+psraw -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: psraw 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xe1,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+psraw 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: psraw 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xe1,0x8a,0xf0,0x1c,0xf0,0x1c]
+psraw 485498096(%edx), %xmm1
+
+// CHECK: psraw 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0xe1,0x0d,0xf0,0x1c,0xf0,0x1c]
+psraw 485498096, %xmm1
+
+// CHECK: psraw 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xe1,0x4c,0x02,0x40]
+psraw 64(%edx,%eax), %xmm1
+
+// CHECK: psraw (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xe1,0x0a]
+psraw (%edx), %xmm1
+
+// CHECK: psraw %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0xe1,0xc9]
+psraw %xmm1, %xmm1
+
+// CHECK: psrld $0, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x72,0xd1,0x00]
+psrld $0, %xmm1
+
+// CHECK: psrld -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xd2,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+psrld -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: psrld 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xd2,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+psrld 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: psrld 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xd2,0x8a,0xf0,0x1c,0xf0,0x1c]
+psrld 485498096(%edx), %xmm1
+
+// CHECK: psrld 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0xd2,0x0d,0xf0,0x1c,0xf0,0x1c]
+psrld 485498096, %xmm1
+
+// CHECK: psrld 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xd2,0x4c,0x02,0x40]
+psrld 64(%edx,%eax), %xmm1
+
+// CHECK: psrld (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xd2,0x0a]
+psrld (%edx), %xmm1
+
+// CHECK: psrldq $0, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x73,0xd9,0x00]
+psrldq $0, %xmm1
+
+// CHECK: psrld %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0xd2,0xc9]
+psrld %xmm1, %xmm1
+
+// CHECK: psrlq $0, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x73,0xd1,0x00]
+psrlq $0, %xmm1
+
+// CHECK: psrlq -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xd3,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+psrlq -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: psrlq 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xd3,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+psrlq 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: psrlq 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xd3,0x8a,0xf0,0x1c,0xf0,0x1c]
+psrlq 485498096(%edx), %xmm1
+
+// CHECK: psrlq 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0xd3,0x0d,0xf0,0x1c,0xf0,0x1c]
+psrlq 485498096, %xmm1
+
+// CHECK: psrlq 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xd3,0x4c,0x02,0x40]
+psrlq 64(%edx,%eax), %xmm1
+
+// CHECK: psrlq (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xd3,0x0a]
+psrlq (%edx), %xmm1
+
+// CHECK: psrlq %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0xd3,0xc9]
+psrlq %xmm1, %xmm1
+
+// CHECK: psrlw $0, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x71,0xd1,0x00]
+psrlw $0, %xmm1
+
+// CHECK: psrlw -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xd1,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+psrlw -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: psrlw 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xd1,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+psrlw 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: psrlw 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xd1,0x8a,0xf0,0x1c,0xf0,0x1c]
+psrlw 485498096(%edx), %xmm1
+
+// CHECK: psrlw 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0xd1,0x0d,0xf0,0x1c,0xf0,0x1c]
+psrlw 485498096, %xmm1
+
+// CHECK: psrlw 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xd1,0x4c,0x02,0x40]
+psrlw 64(%edx,%eax), %xmm1
+
+// CHECK: psrlw (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xd1,0x0a]
+psrlw (%edx), %xmm1
+
+// CHECK: psrlw %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0xd1,0xc9]
+psrlw %xmm1, %xmm1
+
+// CHECK: psubb -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xf8,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+psubb -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: psubb 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xf8,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+psubb 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: psubb 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xf8,0x8a,0xf0,0x1c,0xf0,0x1c]
+psubb 485498096(%edx), %xmm1
+
+// CHECK: psubb 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0xf8,0x0d,0xf0,0x1c,0xf0,0x1c]
+psubb 485498096, %xmm1
+
+// CHECK: psubb 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xf8,0x4c,0x02,0x40]
+psubb 64(%edx,%eax), %xmm1
+
+// CHECK: psubb (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xf8,0x0a]
+psubb (%edx), %xmm1
+
+// CHECK: psubb %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0xf8,0xc9]
+psubb %xmm1, %xmm1
+
+// CHECK: psubd -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xfa,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+psubd -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: psubd 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xfa,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+psubd 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: psubd 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xfa,0x8a,0xf0,0x1c,0xf0,0x1c]
+psubd 485498096(%edx), %xmm1
+
+// CHECK: psubd 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0xfa,0x0d,0xf0,0x1c,0xf0,0x1c]
+psubd 485498096, %xmm1
+
+// CHECK: psubd 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xfa,0x4c,0x02,0x40]
+psubd 64(%edx,%eax), %xmm1
+
+// CHECK: psubd (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xfa,0x0a]
+psubd (%edx), %xmm1
+
+// CHECK: psubd %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0xfa,0xc9]
+psubd %xmm1, %xmm1
+
+// CHECK: psubq -485498096(%edx,%eax,4), %mm4
+// CHECK: encoding: [0x0f,0xfb,0xa4,0x82,0x10,0xe3,0x0f,0xe3]
+psubq -485498096(%edx,%eax,4), %mm4
+
+// CHECK: psubq 485498096(%edx,%eax,4), %mm4
+// CHECK: encoding: [0x0f,0xfb,0xa4,0x82,0xf0,0x1c,0xf0,0x1c]
+psubq 485498096(%edx,%eax,4), %mm4
+
+// CHECK: psubq -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xfb,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+psubq -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: psubq 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xfb,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+psubq 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: psubq 485498096(%edx), %mm4
+// CHECK: encoding: [0x0f,0xfb,0xa2,0xf0,0x1c,0xf0,0x1c]
+psubq 485498096(%edx), %mm4
+
+// CHECK: psubq 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xfb,0x8a,0xf0,0x1c,0xf0,0x1c]
+psubq 485498096(%edx), %xmm1
+
+// CHECK: psubq 485498096, %mm4
+// CHECK: encoding: [0x0f,0xfb,0x25,0xf0,0x1c,0xf0,0x1c]
+psubq 485498096, %mm4
+
+// CHECK: psubq 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0xfb,0x0d,0xf0,0x1c,0xf0,0x1c]
+psubq 485498096, %xmm1
+
+// CHECK: psubq 64(%edx,%eax), %mm4
+// CHECK: encoding: [0x0f,0xfb,0x64,0x02,0x40]
+psubq 64(%edx,%eax), %mm4
+
+// CHECK: psubq 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xfb,0x4c,0x02,0x40]
+psubq 64(%edx,%eax), %xmm1
+
+// CHECK: psubq (%edx), %mm4
+// CHECK: encoding: [0x0f,0xfb,0x22]
+psubq (%edx), %mm4
+
+// CHECK: psubq (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xfb,0x0a]
+psubq (%edx), %xmm1
+
+// CHECK: psubq %mm4, %mm4
+// CHECK: encoding: [0x0f,0xfb,0xe4]
+psubq %mm4, %mm4
+
+// CHECK: psubq %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0xfb,0xc9]
+psubq %xmm1, %xmm1
+
+// CHECK: psubsb -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xe8,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+psubsb -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: psubsb 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xe8,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+psubsb 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: psubsb 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xe8,0x8a,0xf0,0x1c,0xf0,0x1c]
+psubsb 485498096(%edx), %xmm1
+
+// CHECK: psubsb 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0xe8,0x0d,0xf0,0x1c,0xf0,0x1c]
+psubsb 485498096, %xmm1
+
+// CHECK: psubsb 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xe8,0x4c,0x02,0x40]
+psubsb 64(%edx,%eax), %xmm1
+
+// CHECK: psubsb (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xe8,0x0a]
+psubsb (%edx), %xmm1
+
+// CHECK: psubsb %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0xe8,0xc9]
+psubsb %xmm1, %xmm1
+
+// CHECK: psubsw -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xe9,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+psubsw -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: psubsw 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xe9,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+psubsw 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: psubsw 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xe9,0x8a,0xf0,0x1c,0xf0,0x1c]
+psubsw 485498096(%edx), %xmm1
+
+// CHECK: psubsw 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0xe9,0x0d,0xf0,0x1c,0xf0,0x1c]
+psubsw 485498096, %xmm1
+
+// CHECK: psubsw 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xe9,0x4c,0x02,0x40]
+psubsw 64(%edx,%eax), %xmm1
+
+// CHECK: psubsw (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xe9,0x0a]
+psubsw (%edx), %xmm1
+
+// CHECK: psubsw %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0xe9,0xc9]
+psubsw %xmm1, %xmm1
+
+// CHECK: psubusb -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xd8,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+psubusb -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: psubusb 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xd8,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+psubusb 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: psubusb 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xd8,0x8a,0xf0,0x1c,0xf0,0x1c]
+psubusb 485498096(%edx), %xmm1
+
+// CHECK: psubusb 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0xd8,0x0d,0xf0,0x1c,0xf0,0x1c]
+psubusb 485498096, %xmm1
+
+// CHECK: psubusb 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xd8,0x4c,0x02,0x40]
+psubusb 64(%edx,%eax), %xmm1
+
+// CHECK: psubusb (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xd8,0x0a]
+psubusb (%edx), %xmm1
+
+// CHECK: psubusb %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0xd8,0xc9]
+psubusb %xmm1, %xmm1
+
+// CHECK: psubusw -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xd9,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+psubusw -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: psubusw 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xd9,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+psubusw 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: psubusw 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xd9,0x8a,0xf0,0x1c,0xf0,0x1c]
+psubusw 485498096(%edx), %xmm1
+
+// CHECK: psubusw 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0xd9,0x0d,0xf0,0x1c,0xf0,0x1c]
+psubusw 485498096, %xmm1
+
+// CHECK: psubusw 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xd9,0x4c,0x02,0x40]
+psubusw 64(%edx,%eax), %xmm1
+
+// CHECK: psubusw (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xd9,0x0a]
+psubusw (%edx), %xmm1
+
+// CHECK: psubusw %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0xd9,0xc9]
+psubusw %xmm1, %xmm1
+
+// CHECK: psubw -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xf9,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+psubw -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: psubw 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xf9,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+psubw 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: psubw 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xf9,0x8a,0xf0,0x1c,0xf0,0x1c]
+psubw 485498096(%edx), %xmm1
+
+// CHECK: psubw 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0xf9,0x0d,0xf0,0x1c,0xf0,0x1c]
+psubw 485498096, %xmm1
+
+// CHECK: psubw 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xf9,0x4c,0x02,0x40]
+psubw 64(%edx,%eax), %xmm1
+
+// CHECK: psubw (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xf9,0x0a]
+psubw (%edx), %xmm1
+
+// CHECK: psubw %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0xf9,0xc9]
+psubw %xmm1, %xmm1
+
+// CHECK: punpckhbw -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x68,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+punpckhbw -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: punpckhbw 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x68,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+punpckhbw 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: punpckhbw 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x68,0x8a,0xf0,0x1c,0xf0,0x1c]
+punpckhbw 485498096(%edx), %xmm1
+
+// CHECK: punpckhbw 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x68,0x0d,0xf0,0x1c,0xf0,0x1c]
+punpckhbw 485498096, %xmm1
+
+// CHECK: punpckhbw 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x68,0x4c,0x02,0x40]
+punpckhbw 64(%edx,%eax), %xmm1
+
+// CHECK: punpckhbw (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x68,0x0a]
+punpckhbw (%edx), %xmm1
+
+// CHECK: punpckhbw %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x68,0xc9]
+punpckhbw %xmm1, %xmm1
+
+// CHECK: punpckhdq -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x6a,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+punpckhdq -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: punpckhdq 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x6a,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+punpckhdq 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: punpckhdq 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x6a,0x8a,0xf0,0x1c,0xf0,0x1c]
+punpckhdq 485498096(%edx), %xmm1
+
+// CHECK: punpckhdq 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x6a,0x0d,0xf0,0x1c,0xf0,0x1c]
+punpckhdq 485498096, %xmm1
+
+// CHECK: punpckhdq 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x6a,0x4c,0x02,0x40]
+punpckhdq 64(%edx,%eax), %xmm1
+
+// CHECK: punpckhdq (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x6a,0x0a]
+punpckhdq (%edx), %xmm1
+
+// CHECK: punpckhdq %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x6a,0xc9]
+punpckhdq %xmm1, %xmm1
+
+// CHECK: punpckhqdq -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x6d,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+punpckhqdq -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: punpckhqdq 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x6d,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+punpckhqdq 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: punpckhqdq 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x6d,0x8a,0xf0,0x1c,0xf0,0x1c]
+punpckhqdq 485498096(%edx), %xmm1
+
+// CHECK: punpckhqdq 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x6d,0x0d,0xf0,0x1c,0xf0,0x1c]
+punpckhqdq 485498096, %xmm1
+
+// CHECK: punpckhqdq 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x6d,0x4c,0x02,0x40]
+punpckhqdq 64(%edx,%eax), %xmm1
+
+// CHECK: punpckhqdq (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x6d,0x0a]
+punpckhqdq (%edx), %xmm1
+
+// CHECK: punpckhqdq %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x6d,0xc9]
+punpckhqdq %xmm1, %xmm1
+
+// CHECK: punpckhwd -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x69,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+punpckhwd -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: punpckhwd 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x69,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+punpckhwd 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: punpckhwd 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x69,0x8a,0xf0,0x1c,0xf0,0x1c]
+punpckhwd 485498096(%edx), %xmm1
+
+// CHECK: punpckhwd 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x69,0x0d,0xf0,0x1c,0xf0,0x1c]
+punpckhwd 485498096, %xmm1
+
+// CHECK: punpckhwd 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x69,0x4c,0x02,0x40]
+punpckhwd 64(%edx,%eax), %xmm1
+
+// CHECK: punpckhwd (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x69,0x0a]
+punpckhwd (%edx), %xmm1
+
+// CHECK: punpckhwd %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x69,0xc9]
+punpckhwd %xmm1, %xmm1
+
+// CHECK: punpcklbw -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x60,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+punpcklbw -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: punpcklbw 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x60,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+punpcklbw 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: punpcklbw 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x60,0x8a,0xf0,0x1c,0xf0,0x1c]
+punpcklbw 485498096(%edx), %xmm1
+
+// CHECK: punpcklbw 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x60,0x0d,0xf0,0x1c,0xf0,0x1c]
+punpcklbw 485498096, %xmm1
+
+// CHECK: punpcklbw 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x60,0x4c,0x02,0x40]
+punpcklbw 64(%edx,%eax), %xmm1
+
+// CHECK: punpcklbw (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x60,0x0a]
+punpcklbw (%edx), %xmm1
+
+// CHECK: punpcklbw %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x60,0xc9]
+punpcklbw %xmm1, %xmm1
+
+// CHECK: punpckldq -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x62,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+punpckldq -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: punpckldq 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x62,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+punpckldq 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: punpckldq 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x62,0x8a,0xf0,0x1c,0xf0,0x1c]
+punpckldq 485498096(%edx), %xmm1
+
+// CHECK: punpckldq 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x62,0x0d,0xf0,0x1c,0xf0,0x1c]
+punpckldq 485498096, %xmm1
+
+// CHECK: punpckldq 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x62,0x4c,0x02,0x40]
+punpckldq 64(%edx,%eax), %xmm1
+
+// CHECK: punpckldq (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x62,0x0a]
+punpckldq (%edx), %xmm1
+
+// CHECK: punpckldq %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x62,0xc9]
+punpckldq %xmm1, %xmm1
+
+// CHECK: punpcklqdq -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x6c,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+punpcklqdq -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: punpcklqdq 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x6c,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+punpcklqdq 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: punpcklqdq 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x6c,0x8a,0xf0,0x1c,0xf0,0x1c]
+punpcklqdq 485498096(%edx), %xmm1
+
+// CHECK: punpcklqdq 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x6c,0x0d,0xf0,0x1c,0xf0,0x1c]
+punpcklqdq 485498096, %xmm1
+
+// CHECK: punpcklqdq 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x6c,0x4c,0x02,0x40]
+punpcklqdq 64(%edx,%eax), %xmm1
+
+// CHECK: punpcklqdq (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x6c,0x0a]
+punpcklqdq (%edx), %xmm1
+
+// CHECK: punpcklqdq %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x6c,0xc9]
+punpcklqdq %xmm1, %xmm1
+
+// CHECK: punpcklwd -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x61,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+punpcklwd -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: punpcklwd 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x61,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+punpcklwd 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: punpcklwd 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x61,0x8a,0xf0,0x1c,0xf0,0x1c]
+punpcklwd 485498096(%edx), %xmm1
+
+// CHECK: punpcklwd 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x61,0x0d,0xf0,0x1c,0xf0,0x1c]
+punpcklwd 485498096, %xmm1
+
+// CHECK: punpcklwd 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x61,0x4c,0x02,0x40]
+punpcklwd 64(%edx,%eax), %xmm1
+
+// CHECK: punpcklwd (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x61,0x0a]
+punpcklwd (%edx), %xmm1
+
+// CHECK: punpcklwd %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x61,0xc9]
+punpcklwd %xmm1, %xmm1
+
+// CHECK: pxor -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xef,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+pxor -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pxor 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xef,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+pxor 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pxor 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xef,0x8a,0xf0,0x1c,0xf0,0x1c]
+pxor 485498096(%edx), %xmm1
+
+// CHECK: pxor 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0xef,0x0d,0xf0,0x1c,0xf0,0x1c]
+pxor 485498096, %xmm1
+
+// CHECK: pxor 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xef,0x4c,0x02,0x40]
+pxor 64(%edx,%eax), %xmm1
+
+// CHECK: pxor (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xef,0x0a]
+pxor (%edx), %xmm1
+
+// CHECK: pxor %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0xef,0xc9]
+pxor %xmm1, %xmm1
+
+// CHECK: shufpd $0, -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xc6,0x8c,0x82,0x10,0xe3,0x0f,0xe3,0x00]
+shufpd $0, -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: shufpd $0, 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xc6,0x8c,0x82,0xf0,0x1c,0xf0,0x1c,0x00]
+shufpd $0, 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: shufpd $0, 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xc6,0x8a,0xf0,0x1c,0xf0,0x1c,0x00]
+shufpd $0, 485498096(%edx), %xmm1
+
+// CHECK: shufpd $0, 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0xc6,0x0d,0xf0,0x1c,0xf0,0x1c,0x00]
+shufpd $0, 485498096, %xmm1
+
+// CHECK: shufpd $0, 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xc6,0x4c,0x02,0x40,0x00]
+shufpd $0, 64(%edx,%eax), %xmm1
+
+// CHECK: shufpd $0, (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xc6,0x0a,0x00]
+shufpd $0, (%edx), %xmm1
+
+// CHECK: shufpd $0, %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0xc6,0xc9,0x00]
+shufpd $0, %xmm1, %xmm1
+
+// CHECK: sqrtpd -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x51,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+sqrtpd -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: sqrtpd 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x51,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+sqrtpd 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: sqrtpd 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x51,0x8a,0xf0,0x1c,0xf0,0x1c]
+sqrtpd 485498096(%edx), %xmm1
+
+// CHECK: sqrtpd 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x51,0x0d,0xf0,0x1c,0xf0,0x1c]
+sqrtpd 485498096, %xmm1
+
+// CHECK: sqrtpd 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x51,0x4c,0x02,0x40]
+sqrtpd 64(%edx,%eax), %xmm1
+
+// CHECK: sqrtpd (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x51,0x0a]
+sqrtpd (%edx), %xmm1
+
+// CHECK: sqrtpd %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x51,0xc9]
+sqrtpd %xmm1, %xmm1
+
+// CHECK: sqrtsd -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0xf2,0x0f,0x51,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+sqrtsd -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: sqrtsd 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0xf2,0x0f,0x51,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+sqrtsd 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: sqrtsd 485498096(%edx), %xmm1
+// CHECK: encoding: [0xf2,0x0f,0x51,0x8a,0xf0,0x1c,0xf0,0x1c]
+sqrtsd 485498096(%edx), %xmm1
+
+// CHECK: sqrtsd 485498096, %xmm1
+// CHECK: encoding: [0xf2,0x0f,0x51,0x0d,0xf0,0x1c,0xf0,0x1c]
+sqrtsd 485498096, %xmm1
+
+// CHECK: sqrtsd 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0xf2,0x0f,0x51,0x4c,0x02,0x40]
+sqrtsd 64(%edx,%eax), %xmm1
+
+// CHECK: sqrtsd (%edx), %xmm1
+// CHECK: encoding: [0xf2,0x0f,0x51,0x0a]
+sqrtsd (%edx), %xmm1
+
+// CHECK: sqrtsd %xmm1, %xmm1
+// CHECK: encoding: [0xf2,0x0f,0x51,0xc9]
+sqrtsd %xmm1, %xmm1
+
+// CHECK: subpd -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x5c,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+subpd -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: subpd 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x5c,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+subpd 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: subpd 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x5c,0x8a,0xf0,0x1c,0xf0,0x1c]
+subpd 485498096(%edx), %xmm1
+
+// CHECK: subpd 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x5c,0x0d,0xf0,0x1c,0xf0,0x1c]
+subpd 485498096, %xmm1
+
+// CHECK: subpd 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x5c,0x4c,0x02,0x40]
+subpd 64(%edx,%eax), %xmm1
+
+// CHECK: subpd (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x5c,0x0a]
+subpd (%edx), %xmm1
+
+// CHECK: subpd %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x5c,0xc9]
+subpd %xmm1, %xmm1
+
+// CHECK: subsd -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0xf2,0x0f,0x5c,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+subsd -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: subsd 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0xf2,0x0f,0x5c,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+subsd 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: subsd 485498096(%edx), %xmm1
+// CHECK: encoding: [0xf2,0x0f,0x5c,0x8a,0xf0,0x1c,0xf0,0x1c]
+subsd 485498096(%edx), %xmm1
+
+// CHECK: subsd 485498096, %xmm1
+// CHECK: encoding: [0xf2,0x0f,0x5c,0x0d,0xf0,0x1c,0xf0,0x1c]
+subsd 485498096, %xmm1
+
+// CHECK: subsd 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0xf2,0x0f,0x5c,0x4c,0x02,0x40]
+subsd 64(%edx,%eax), %xmm1
+
+// CHECK: subsd (%edx), %xmm1
+// CHECK: encoding: [0xf2,0x0f,0x5c,0x0a]
+subsd (%edx), %xmm1
+
+// CHECK: subsd %xmm1, %xmm1
+// CHECK: encoding: [0xf2,0x0f,0x5c,0xc9]
+subsd %xmm1, %xmm1
+
+// CHECK: ucomisd -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x2e,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+ucomisd -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: ucomisd 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x2e,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+ucomisd 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: ucomisd 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x2e,0x8a,0xf0,0x1c,0xf0,0x1c]
+ucomisd 485498096(%edx), %xmm1
+
+// CHECK: ucomisd 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x2e,0x0d,0xf0,0x1c,0xf0,0x1c]
+ucomisd 485498096, %xmm1
+
+// CHECK: ucomisd 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x2e,0x4c,0x02,0x40]
+ucomisd 64(%edx,%eax), %xmm1
+
+// CHECK: ucomisd (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x2e,0x0a]
+ucomisd (%edx), %xmm1
+
+// CHECK: ucomisd %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x2e,0xc9]
+ucomisd %xmm1, %xmm1
+
+// CHECK: unpckhpd -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x15,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+unpckhpd -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: unpckhpd 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x15,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+unpckhpd 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: unpckhpd 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x15,0x8a,0xf0,0x1c,0xf0,0x1c]
+unpckhpd 485498096(%edx), %xmm1
+
+// CHECK: unpckhpd 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x15,0x0d,0xf0,0x1c,0xf0,0x1c]
+unpckhpd 485498096, %xmm1
+
+// CHECK: unpckhpd 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x15,0x4c,0x02,0x40]
+unpckhpd 64(%edx,%eax), %xmm1
+
+// CHECK: unpckhpd (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x15,0x0a]
+unpckhpd (%edx), %xmm1
+
+// CHECK: unpckhpd %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x15,0xc9]
+unpckhpd %xmm1, %xmm1
+
+// CHECK: unpcklpd -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x14,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+unpcklpd -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: unpcklpd 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x14,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+unpcklpd 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: unpcklpd 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x14,0x8a,0xf0,0x1c,0xf0,0x1c]
+unpcklpd 485498096(%edx), %xmm1
+
+// CHECK: unpcklpd 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x14,0x0d,0xf0,0x1c,0xf0,0x1c]
+unpcklpd 485498096, %xmm1
+
+// CHECK: unpcklpd 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x14,0x4c,0x02,0x40]
+unpcklpd 64(%edx,%eax), %xmm1
+
+// CHECK: unpcklpd (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x14,0x0a]
+unpcklpd (%edx), %xmm1
+
+// CHECK: unpcklpd %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x14,0xc9]
+unpcklpd %xmm1, %xmm1
+
+// CHECK: xorpd -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x57,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+xorpd -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: xorpd 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x57,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+xorpd 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: xorpd 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x57,0x8a,0xf0,0x1c,0xf0,0x1c]
+xorpd 485498096(%edx), %xmm1
+
+// CHECK: xorpd 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x57,0x0d,0xf0,0x1c,0xf0,0x1c]
+xorpd 485498096, %xmm1
+
+// CHECK: xorpd 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x57,0x4c,0x02,0x40]
+xorpd 64(%edx,%eax), %xmm1
+
+// CHECK: xorpd (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x57,0x0a]
+xorpd (%edx), %xmm1
+
+// CHECK: xorpd %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x57,0xc9]
+xorpd %xmm1, %xmm1
+
diff --git a/test/MC/X86/SSE2-64.s b/test/MC/X86/SSE2-64.s
new file mode 100644
index 0000000000000000000000000000000000000000..fdc1514261d16ae043fc4b03bcee9daa4956e656
--- /dev/null
+++ b/test/MC/X86/SSE2-64.s
@@ -0,0 +1,3642 @@
+// RUN: llvm-mc -triple x86_64-unknown-unknown --show-encoding %s | FileCheck %s
+
+// CHECK: addpd 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x58,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+addpd 485498096, %xmm6
+
+// CHECK: addpd -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x58,0x74,0x82,0xc0]
+addpd -64(%rdx,%rax,4), %xmm6
+
+// CHECK: addpd 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x58,0x74,0x82,0x40]
+addpd 64(%rdx,%rax,4), %xmm6
+
+// CHECK: addpd 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x58,0x74,0x02,0x40]
+addpd 64(%rdx,%rax), %xmm6
+
+// CHECK: addpd 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x58,0x72,0x40]
+addpd 64(%rdx), %xmm6
+
+// CHECK: addpd (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x58,0x32]
+addpd (%rdx), %xmm6
+
+// CHECK: addpd %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x58,0xf6]
+addpd %xmm6, %xmm6
+
+// CHECK: addsd 485498096, %xmm6
+// CHECK: encoding: [0xf2,0x0f,0x58,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+addsd 485498096, %xmm6
+
+// CHECK: addsd -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0xf2,0x0f,0x58,0x74,0x82,0xc0]
+addsd -64(%rdx,%rax,4), %xmm6
+
+// CHECK: addsd 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0xf2,0x0f,0x58,0x74,0x82,0x40]
+addsd 64(%rdx,%rax,4), %xmm6
+
+// CHECK: addsd 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0xf2,0x0f,0x58,0x74,0x02,0x40]
+addsd 64(%rdx,%rax), %xmm6
+
+// CHECK: addsd 64(%rdx), %xmm6
+// CHECK: encoding: [0xf2,0x0f,0x58,0x72,0x40]
+addsd 64(%rdx), %xmm6
+
+// CHECK: addsd (%rdx), %xmm6
+// CHECK: encoding: [0xf2,0x0f,0x58,0x32]
+addsd (%rdx), %xmm6
+
+// CHECK: addsd %xmm6, %xmm6
+// CHECK: encoding: [0xf2,0x0f,0x58,0xf6]
+addsd %xmm6, %xmm6
+
+// CHECK: andnpd 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x55,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+andnpd 485498096, %xmm6
+
+// CHECK: andnpd -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x55,0x74,0x82,0xc0]
+andnpd -64(%rdx,%rax,4), %xmm6
+
+// CHECK: andnpd 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x55,0x74,0x82,0x40]
+andnpd 64(%rdx,%rax,4), %xmm6
+
+// CHECK: andnpd 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x55,0x74,0x02,0x40]
+andnpd 64(%rdx,%rax), %xmm6
+
+// CHECK: andnpd 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x55,0x72,0x40]
+andnpd 64(%rdx), %xmm6
+
+// CHECK: andnpd (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x55,0x32]
+andnpd (%rdx), %xmm6
+
+// CHECK: andnpd %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x55,0xf6]
+andnpd %xmm6, %xmm6
+
+// CHECK: andpd 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x54,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+andpd 485498096, %xmm6
+
+// CHECK: andpd -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x54,0x74,0x82,0xc0]
+andpd -64(%rdx,%rax,4), %xmm6
+
+// CHECK: andpd 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x54,0x74,0x82,0x40]
+andpd 64(%rdx,%rax,4), %xmm6
+
+// CHECK: andpd 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x54,0x74,0x02,0x40]
+andpd 64(%rdx,%rax), %xmm6
+
+// CHECK: andpd 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x54,0x72,0x40]
+andpd 64(%rdx), %xmm6
+
+// CHECK: andpd (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x54,0x32]
+andpd (%rdx), %xmm6
+
+// CHECK: andpd %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x54,0xf6]
+andpd %xmm6, %xmm6
+
+// CHECK: cmpeqpd 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0xc2,0x34,0x25,0xf0,0x1c,0xf0,0x1c,0x00]
+cmpeqpd 485498096, %xmm6
+
+// CHECK: cmpeqpd -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xc2,0x74,0x82,0xc0,0x00]
+cmpeqpd -64(%rdx,%rax,4), %xmm6
+
+// CHECK: cmpeqpd 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xc2,0x74,0x82,0x40,0x00]
+cmpeqpd 64(%rdx,%rax,4), %xmm6
+
+// CHECK: cmpeqpd 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xc2,0x74,0x02,0x40,0x00]
+cmpeqpd 64(%rdx,%rax), %xmm6
+
+// CHECK: cmpeqpd 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xc2,0x72,0x40,0x00]
+cmpeqpd 64(%rdx), %xmm6
+
+// CHECK: cmpeqpd (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xc2,0x32,0x00]
+cmpeqpd (%rdx), %xmm6
+
+// CHECK: cmpeqpd %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0xc2,0xf6,0x00]
+cmpeqpd %xmm6, %xmm6
+
+// CHECK: cmpeqsd 485498096, %xmm6
+// CHECK: encoding: [0xf2,0x0f,0xc2,0x34,0x25,0xf0,0x1c,0xf0,0x1c,0x00]
+cmpeqsd 485498096, %xmm6
+
+// CHECK: cmpeqsd -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0xf2,0x0f,0xc2,0x74,0x82,0xc0,0x00]
+cmpeqsd -64(%rdx,%rax,4), %xmm6
+
+// CHECK: cmpeqsd 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0xf2,0x0f,0xc2,0x74,0x82,0x40,0x00]
+cmpeqsd 64(%rdx,%rax,4), %xmm6
+
+// CHECK: cmpeqsd 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0xf2,0x0f,0xc2,0x74,0x02,0x40,0x00]
+cmpeqsd 64(%rdx,%rax), %xmm6
+
+// CHECK: cmpeqsd 64(%rdx), %xmm6
+// CHECK: encoding: [0xf2,0x0f,0xc2,0x72,0x40,0x00]
+cmpeqsd 64(%rdx), %xmm6
+
+// CHECK: cmpeqsd (%rdx), %xmm6
+// CHECK: encoding: [0xf2,0x0f,0xc2,0x32,0x00]
+cmpeqsd (%rdx), %xmm6
+
+// CHECK: cmpeqsd %xmm6, %xmm6
+// CHECK: encoding: [0xf2,0x0f,0xc2,0xf6,0x00]
+cmpeqsd %xmm6, %xmm6
+
+// CHECK: comisd 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x2f,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+comisd 485498096, %xmm6
+
+// CHECK: comisd -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x2f,0x74,0x82,0xc0]
+comisd -64(%rdx,%rax,4), %xmm6
+
+// CHECK: comisd 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x2f,0x74,0x82,0x40]
+comisd 64(%rdx,%rax,4), %xmm6
+
+// CHECK: comisd 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x2f,0x74,0x02,0x40]
+comisd 64(%rdx,%rax), %xmm6
+
+// CHECK: comisd 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x2f,0x72,0x40]
+comisd 64(%rdx), %xmm6
+
+// CHECK: comisd (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x2f,0x32]
+comisd (%rdx), %xmm6
+
+// CHECK: comisd %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x2f,0xf6]
+comisd %xmm6, %xmm6
+
+// CHECK: cvtdq2pd 485498096, %xmm6
+// CHECK: encoding: [0xf3,0x0f,0xe6,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+cvtdq2pd 485498096, %xmm6
+
+// CHECK: cvtdq2pd -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0xf3,0x0f,0xe6,0x74,0x82,0xc0]
+cvtdq2pd -64(%rdx,%rax,4), %xmm6
+
+// CHECK: cvtdq2pd 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0xf3,0x0f,0xe6,0x74,0x82,0x40]
+cvtdq2pd 64(%rdx,%rax,4), %xmm6
+
+// CHECK: cvtdq2pd 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0xf3,0x0f,0xe6,0x74,0x02,0x40]
+cvtdq2pd 64(%rdx,%rax), %xmm6
+
+// CHECK: cvtdq2pd 64(%rdx), %xmm6
+// CHECK: encoding: [0xf3,0x0f,0xe6,0x72,0x40]
+cvtdq2pd 64(%rdx), %xmm6
+
+// CHECK: cvtdq2pd (%rdx), %xmm6
+// CHECK: encoding: [0xf3,0x0f,0xe6,0x32]
+cvtdq2pd (%rdx), %xmm6
+
+// CHECK: cvtdq2pd %xmm6, %xmm6
+// CHECK: encoding: [0xf3,0x0f,0xe6,0xf6]
+cvtdq2pd %xmm6, %xmm6
+
+// CHECK: cvtdq2ps 485498096, %xmm6
+// CHECK: encoding: [0x0f,0x5b,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+cvtdq2ps 485498096, %xmm6
+
+// CHECK: cvtdq2ps -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x0f,0x5b,0x74,0x82,0xc0]
+cvtdq2ps -64(%rdx,%rax,4), %xmm6
+
+// CHECK: cvtdq2ps 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x0f,0x5b,0x74,0x82,0x40]
+cvtdq2ps 64(%rdx,%rax,4), %xmm6
+
+// CHECK: cvtdq2ps 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x0f,0x5b,0x74,0x02,0x40]
+cvtdq2ps 64(%rdx,%rax), %xmm6
+
+// CHECK: cvtdq2ps 64(%rdx), %xmm6
+// CHECK: encoding: [0x0f,0x5b,0x72,0x40]
+cvtdq2ps 64(%rdx), %xmm6
+
+// CHECK: cvtdq2ps (%rdx), %xmm6
+// CHECK: encoding: [0x0f,0x5b,0x32]
+cvtdq2ps (%rdx), %xmm6
+
+// CHECK: cvtdq2ps %xmm6, %xmm6
+// CHECK: encoding: [0x0f,0x5b,0xf6]
+cvtdq2ps %xmm6, %xmm6
+
+// CHECK: cvtpd2dq 485498096, %xmm6
+// CHECK: encoding: [0xf2,0x0f,0xe6,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+cvtpd2dq 485498096, %xmm6
+
+// CHECK: cvtpd2dq -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0xf2,0x0f,0xe6,0x74,0x82,0xc0]
+cvtpd2dq -64(%rdx,%rax,4), %xmm6
+
+// CHECK: cvtpd2dq 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0xf2,0x0f,0xe6,0x74,0x82,0x40]
+cvtpd2dq 64(%rdx,%rax,4), %xmm6
+
+// CHECK: cvtpd2dq 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0xf2,0x0f,0xe6,0x74,0x02,0x40]
+cvtpd2dq 64(%rdx,%rax), %xmm6
+
+// CHECK: cvtpd2dq 64(%rdx), %xmm6
+// CHECK: encoding: [0xf2,0x0f,0xe6,0x72,0x40]
+cvtpd2dq 64(%rdx), %xmm6
+
+// CHECK: cvtpd2dq (%rdx), %xmm6
+// CHECK: encoding: [0xf2,0x0f,0xe6,0x32]
+cvtpd2dq (%rdx), %xmm6
+
+// CHECK: cvtpd2dq %xmm6, %xmm6
+// CHECK: encoding: [0xf2,0x0f,0xe6,0xf6]
+cvtpd2dq %xmm6, %xmm6
+
+// CHECK: cvtpd2pi 485498096, %mm4
+// CHECK: encoding: [0x66,0x0f,0x2d,0x24,0x25,0xf0,0x1c,0xf0,0x1c]
+cvtpd2pi 485498096, %mm4
+
+// CHECK: cvtpd2pi 64(%rdx), %mm4
+// CHECK: encoding: [0x66,0x0f,0x2d,0x62,0x40]
+cvtpd2pi 64(%rdx), %mm4
+
+// CHECK: cvtpd2pi -64(%rdx,%rax,4), %mm4
+// CHECK: encoding: [0x66,0x0f,0x2d,0x64,0x82,0xc0]
+cvtpd2pi -64(%rdx,%rax,4), %mm4
+
+// CHECK: cvtpd2pi 64(%rdx,%rax,4), %mm4
+// CHECK: encoding: [0x66,0x0f,0x2d,0x64,0x82,0x40]
+cvtpd2pi 64(%rdx,%rax,4), %mm4
+
+// CHECK: cvtpd2pi 64(%rdx,%rax), %mm4
+// CHECK: encoding: [0x66,0x0f,0x2d,0x64,0x02,0x40]
+cvtpd2pi 64(%rdx,%rax), %mm4
+
+// CHECK: cvtpd2pi (%rdx), %mm4
+// CHECK: encoding: [0x66,0x0f,0x2d,0x22]
+cvtpd2pi (%rdx), %mm4
+
+// CHECK: cvtpd2pi %xmm6, %mm4
+// CHECK: encoding: [0x66,0x0f,0x2d,0xe6]
+cvtpd2pi %xmm6, %mm4
+
+// CHECK: cvtpd2ps 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x5a,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+cvtpd2ps 485498096, %xmm6
+
+// CHECK: cvtpd2ps -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x5a,0x74,0x82,0xc0]
+cvtpd2ps -64(%rdx,%rax,4), %xmm6
+
+// CHECK: cvtpd2ps 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x5a,0x74,0x82,0x40]
+cvtpd2ps 64(%rdx,%rax,4), %xmm6
+
+// CHECK: cvtpd2ps 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x5a,0x74,0x02,0x40]
+cvtpd2ps 64(%rdx,%rax), %xmm6
+
+// CHECK: cvtpd2ps 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x5a,0x72,0x40]
+cvtpd2ps 64(%rdx), %xmm6
+
+// CHECK: cvtpd2ps (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x5a,0x32]
+cvtpd2ps (%rdx), %xmm6
+
+// CHECK: cvtpd2ps %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x5a,0xf6]
+cvtpd2ps %xmm6, %xmm6
+
+// CHECK: cvtpi2pd 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x2a,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+cvtpi2pd 485498096, %xmm6
+
+// CHECK: cvtpi2pd -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x2a,0x74,0x82,0xc0]
+cvtpi2pd -64(%rdx,%rax,4), %xmm6
+
+// CHECK: cvtpi2pd 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x2a,0x74,0x82,0x40]
+cvtpi2pd 64(%rdx,%rax,4), %xmm6
+
+// CHECK: cvtpi2pd 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x2a,0x74,0x02,0x40]
+cvtpi2pd 64(%rdx,%rax), %xmm6
+
+// CHECK: cvtpi2pd 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x2a,0x72,0x40]
+cvtpi2pd 64(%rdx), %xmm6
+
+// CHECK: cvtpi2pd %mm4, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x2a,0xf4]
+cvtpi2pd %mm4, %xmm6
+
+// CHECK: cvtpi2pd (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x2a,0x32]
+cvtpi2pd (%rdx), %xmm6
+
+// CHECK: cvtps2dq 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x5b,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+cvtps2dq 485498096, %xmm6
+
+// CHECK: cvtps2dq -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x5b,0x74,0x82,0xc0]
+cvtps2dq -64(%rdx,%rax,4), %xmm6
+
+// CHECK: cvtps2dq 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x5b,0x74,0x82,0x40]
+cvtps2dq 64(%rdx,%rax,4), %xmm6
+
+// CHECK: cvtps2dq 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x5b,0x74,0x02,0x40]
+cvtps2dq 64(%rdx,%rax), %xmm6
+
+// CHECK: cvtps2dq 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x5b,0x72,0x40]
+cvtps2dq 64(%rdx), %xmm6
+
+// CHECK: cvtps2dq (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x5b,0x32]
+cvtps2dq (%rdx), %xmm6
+
+// CHECK: cvtps2dq %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x5b,0xf6]
+cvtps2dq %xmm6, %xmm6
+
+// CHECK: cvtps2pd 485498096, %xmm6
+// CHECK: encoding: [0x0f,0x5a,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+cvtps2pd 485498096, %xmm6
+
+// CHECK: cvtps2pd -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x0f,0x5a,0x74,0x82,0xc0]
+cvtps2pd -64(%rdx,%rax,4), %xmm6
+
+// CHECK: cvtps2pd 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x0f,0x5a,0x74,0x82,0x40]
+cvtps2pd 64(%rdx,%rax,4), %xmm6
+
+// CHECK: cvtps2pd 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x0f,0x5a,0x74,0x02,0x40]
+cvtps2pd 64(%rdx,%rax), %xmm6
+
+// CHECK: cvtps2pd 64(%rdx), %xmm6
+// CHECK: encoding: [0x0f,0x5a,0x72,0x40]
+cvtps2pd 64(%rdx), %xmm6
+
+// CHECK: cvtps2pd (%rdx), %xmm6
+// CHECK: encoding: [0x0f,0x5a,0x32]
+cvtps2pd (%rdx), %xmm6
+
+// CHECK: cvtps2pd %xmm6, %xmm6
+// CHECK: encoding: [0x0f,0x5a,0xf6]
+cvtps2pd %xmm6, %xmm6
+
+// CHECK: cvtsd2si 485498096, %r13d
+// CHECK: encoding: [0xf2,0x44,0x0f,0x2d,0x2c,0x25,0xf0,0x1c,0xf0,0x1c]
+cvtsd2si 485498096, %r13d
+
+// CHECK: cvtsd2si 485498096, %r15
+// CHECK: encoding: [0xf2,0x4c,0x0f,0x2d,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+cvtsd2si 485498096, %r15
+
+// CHECK: cvtsd2si 64(%rdx), %r13d
+// CHECK: encoding: [0xf2,0x44,0x0f,0x2d,0x6a,0x40]
+cvtsd2si 64(%rdx), %r13d
+
+// CHECK: cvtsd2si 64(%rdx), %r15
+// CHECK: encoding: [0xf2,0x4c,0x0f,0x2d,0x7a,0x40]
+cvtsd2si 64(%rdx), %r15
+
+// CHECK: cvtsd2si -64(%rdx,%rax,4), %r13d
+// CHECK: encoding: [0xf2,0x44,0x0f,0x2d,0x6c,0x82,0xc0]
+cvtsd2si -64(%rdx,%rax,4), %r13d
+
+// CHECK: cvtsd2si 64(%rdx,%rax,4), %r13d
+// CHECK: encoding: [0xf2,0x44,0x0f,0x2d,0x6c,0x82,0x40]
+cvtsd2si 64(%rdx,%rax,4), %r13d
+
+// CHECK: cvtsd2si -64(%rdx,%rax,4), %r15
+// CHECK: encoding: [0xf2,0x4c,0x0f,0x2d,0x7c,0x82,0xc0]
+cvtsd2si -64(%rdx,%rax,4), %r15
+
+// CHECK: cvtsd2si 64(%rdx,%rax,4), %r15
+// CHECK: encoding: [0xf2,0x4c,0x0f,0x2d,0x7c,0x82,0x40]
+cvtsd2si 64(%rdx,%rax,4), %r15
+
+// CHECK: cvtsd2si 64(%rdx,%rax), %r13d
+// CHECK: encoding: [0xf2,0x44,0x0f,0x2d,0x6c,0x02,0x40]
+cvtsd2si 64(%rdx,%rax), %r13d
+
+// CHECK: cvtsd2si 64(%rdx,%rax), %r15
+// CHECK: encoding: [0xf2,0x4c,0x0f,0x2d,0x7c,0x02,0x40]
+cvtsd2si 64(%rdx,%rax), %r15
+
+// CHECK: cvtsd2si (%rdx), %r13d
+// CHECK: encoding: [0xf2,0x44,0x0f,0x2d,0x2a]
+cvtsd2si (%rdx), %r13d
+
+// CHECK: cvtsd2si (%rdx), %r15
+// CHECK: encoding: [0xf2,0x4c,0x0f,0x2d,0x3a]
+cvtsd2si (%rdx), %r15
+
+// CHECK: cvtsd2si %xmm6, %r13d
+// CHECK: encoding: [0xf2,0x44,0x0f,0x2d,0xee]
+cvtsd2si %xmm6, %r13d
+
+// CHECK: cvtsd2si %xmm6, %r15
+// CHECK: encoding: [0xf2,0x4c,0x0f,0x2d,0xfe]
+cvtsd2si %xmm6, %r15
+
+// CHECK: cvtsd2ss 485498096, %xmm6
+// CHECK: encoding: [0xf2,0x0f,0x5a,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+cvtsd2ss 485498096, %xmm6
+
+// CHECK: cvtsd2ss -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0xf2,0x0f,0x5a,0x74,0x82,0xc0]
+cvtsd2ss -64(%rdx,%rax,4), %xmm6
+
+// CHECK: cvtsd2ss 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0xf2,0x0f,0x5a,0x74,0x82,0x40]
+cvtsd2ss 64(%rdx,%rax,4), %xmm6
+
+// CHECK: cvtsd2ss 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0xf2,0x0f,0x5a,0x74,0x02,0x40]
+cvtsd2ss 64(%rdx,%rax), %xmm6
+
+// CHECK: cvtsd2ss 64(%rdx), %xmm6
+// CHECK: encoding: [0xf2,0x0f,0x5a,0x72,0x40]
+cvtsd2ss 64(%rdx), %xmm6
+
+// CHECK: cvtsd2ss (%rdx), %xmm6
+// CHECK: encoding: [0xf2,0x0f,0x5a,0x32]
+cvtsd2ss (%rdx), %xmm6
+
+// CHECK: cvtsd2ss %xmm6, %xmm6
+// CHECK: encoding: [0xf2,0x0f,0x5a,0xf6]
+cvtsd2ss %xmm6, %xmm6
+
+// CHECK: cvtsi2sdl 485498096, %xmm6
+// CHECK: encoding: [0xf2,0x0f,0x2a,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+cvtsi2sdl 485498096, %xmm6
+
+// CHECK: cvtsi2sdl -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0xf2,0x0f,0x2a,0x74,0x82,0xc0]
+cvtsi2sdl -64(%rdx,%rax,4), %xmm6
+
+// CHECK: cvtsi2sdl 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0xf2,0x0f,0x2a,0x74,0x82,0x40]
+cvtsi2sdl 64(%rdx,%rax,4), %xmm6
+
+// CHECK: cvtsi2sdl 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0xf2,0x0f,0x2a,0x74,0x02,0x40]
+cvtsi2sdl 64(%rdx,%rax), %xmm6
+
+// CHECK: cvtsi2sdl 64(%rdx), %xmm6
+// CHECK: encoding: [0xf2,0x0f,0x2a,0x72,0x40]
+cvtsi2sdl 64(%rdx), %xmm6
+
+// CHECK: cvtsi2sdl %r13d, %xmm6
+// CHECK: encoding: [0xf2,0x41,0x0f,0x2a,0xf5]
+cvtsi2sdl %r13d, %xmm6
+
+// CHECK: cvtsi2sdl (%rdx), %xmm6
+// CHECK: encoding: [0xf2,0x0f,0x2a,0x32]
+cvtsi2sdl (%rdx), %xmm6
+
+// CHECK: cvtsi2sdq 485498096, %xmm6
+// CHECK: encoding: [0xf2,0x48,0x0f,0x2a,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+cvtsi2sdq 485498096, %xmm6
+
+// CHECK: cvtsi2sdq -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0xf2,0x48,0x0f,0x2a,0x74,0x82,0xc0]
+cvtsi2sdq -64(%rdx,%rax,4), %xmm6
+
+// CHECK: cvtsi2sdq 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0xf2,0x48,0x0f,0x2a,0x74,0x82,0x40]
+cvtsi2sdq 64(%rdx,%rax,4), %xmm6
+
+// CHECK: cvtsi2sdq 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0xf2,0x48,0x0f,0x2a,0x74,0x02,0x40]
+cvtsi2sdq 64(%rdx,%rax), %xmm6
+
+// CHECK: cvtsi2sdq 64(%rdx), %xmm6
+// CHECK: encoding: [0xf2,0x48,0x0f,0x2a,0x72,0x40]
+cvtsi2sdq 64(%rdx), %xmm6
+
+// CHECK: cvtsi2sdq %r15, %xmm6
+// CHECK: encoding: [0xf2,0x49,0x0f,0x2a,0xf7]
+cvtsi2sdq %r15, %xmm6
+
+// CHECK: cvtsi2sdq (%rdx), %xmm6
+// CHECK: encoding: [0xf2,0x48,0x0f,0x2a,0x32]
+cvtsi2sdq (%rdx), %xmm6
+
+// CHECK: cvtss2sd 485498096, %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x5a,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+cvtss2sd 485498096, %xmm6
+
+// CHECK: cvtss2sd -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x5a,0x74,0x82,0xc0]
+cvtss2sd -64(%rdx,%rax,4), %xmm6
+
+// CHECK: cvtss2sd 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x5a,0x74,0x82,0x40]
+cvtss2sd 64(%rdx,%rax,4), %xmm6
+
+// CHECK: cvtss2sd 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x5a,0x74,0x02,0x40]
+cvtss2sd 64(%rdx,%rax), %xmm6
+
+// CHECK: cvtss2sd 64(%rdx), %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x5a,0x72,0x40]
+cvtss2sd 64(%rdx), %xmm6
+
+// CHECK: cvtss2sd (%rdx), %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x5a,0x32]
+cvtss2sd (%rdx), %xmm6
+
+// CHECK: cvtss2sd %xmm6, %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x5a,0xf6]
+cvtss2sd %xmm6, %xmm6
+
+// CHECK: cvttpd2dq 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0xe6,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+cvttpd2dq 485498096, %xmm6
+
+// CHECK: cvttpd2dq -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xe6,0x74,0x82,0xc0]
+cvttpd2dq -64(%rdx,%rax,4), %xmm6
+
+// CHECK: cvttpd2dq 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xe6,0x74,0x82,0x40]
+cvttpd2dq 64(%rdx,%rax,4), %xmm6
+
+// CHECK: cvttpd2dq 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xe6,0x74,0x02,0x40]
+cvttpd2dq 64(%rdx,%rax), %xmm6
+
+// CHECK: cvttpd2dq 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xe6,0x72,0x40]
+cvttpd2dq 64(%rdx), %xmm6
+
+// CHECK: cvttpd2dq (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xe6,0x32]
+cvttpd2dq (%rdx), %xmm6
+
+// CHECK: cvttpd2dq %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0xe6,0xf6]
+cvttpd2dq %xmm6, %xmm6
+
+// CHECK: cvttpd2pi 485498096, %mm4
+// CHECK: encoding: [0x66,0x0f,0x2c,0x24,0x25,0xf0,0x1c,0xf0,0x1c]
+cvttpd2pi 485498096, %mm4
+
+// CHECK: cvttpd2pi 64(%rdx), %mm4
+// CHECK: encoding: [0x66,0x0f,0x2c,0x62,0x40]
+cvttpd2pi 64(%rdx), %mm4
+
+// CHECK: cvttpd2pi -64(%rdx,%rax,4), %mm4
+// CHECK: encoding: [0x66,0x0f,0x2c,0x64,0x82,0xc0]
+cvttpd2pi -64(%rdx,%rax,4), %mm4
+
+// CHECK: cvttpd2pi 64(%rdx,%rax,4), %mm4
+// CHECK: encoding: [0x66,0x0f,0x2c,0x64,0x82,0x40]
+cvttpd2pi 64(%rdx,%rax,4), %mm4
+
+// CHECK: cvttpd2pi 64(%rdx,%rax), %mm4
+// CHECK: encoding: [0x66,0x0f,0x2c,0x64,0x02,0x40]
+cvttpd2pi 64(%rdx,%rax), %mm4
+
+// CHECK: cvttpd2pi (%rdx), %mm4
+// CHECK: encoding: [0x66,0x0f,0x2c,0x22]
+cvttpd2pi (%rdx), %mm4
+
+// CHECK: cvttpd2pi %xmm6, %mm4
+// CHECK: encoding: [0x66,0x0f,0x2c,0xe6]
+cvttpd2pi %xmm6, %mm4
+
+// CHECK: cvttps2dq 485498096, %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x5b,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+cvttps2dq 485498096, %xmm6
+
+// CHECK: cvttps2dq -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x5b,0x74,0x82,0xc0]
+cvttps2dq -64(%rdx,%rax,4), %xmm6
+
+// CHECK: cvttps2dq 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x5b,0x74,0x82,0x40]
+cvttps2dq 64(%rdx,%rax,4), %xmm6
+
+// CHECK: cvttps2dq 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x5b,0x74,0x02,0x40]
+cvttps2dq 64(%rdx,%rax), %xmm6
+
+// CHECK: cvttps2dq 64(%rdx), %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x5b,0x72,0x40]
+cvttps2dq 64(%rdx), %xmm6
+
+// CHECK: cvttps2dq (%rdx), %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x5b,0x32]
+cvttps2dq (%rdx), %xmm6
+
+// CHECK: cvttps2dq %xmm6, %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x5b,0xf6]
+cvttps2dq %xmm6, %xmm6
+
+// CHECK: cvttsd2si 485498096, %r13d
+// CHECK: encoding: [0xf2,0x44,0x0f,0x2c,0x2c,0x25,0xf0,0x1c,0xf0,0x1c]
+cvttsd2si 485498096, %r13d
+
+// CHECK: cvttsd2si 485498096, %r15
+// CHECK: encoding: [0xf2,0x4c,0x0f,0x2c,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+cvttsd2si 485498096, %r15
+
+// CHECK: cvttsd2si 64(%rdx), %r13d
+// CHECK: encoding: [0xf2,0x44,0x0f,0x2c,0x6a,0x40]
+cvttsd2si 64(%rdx), %r13d
+
+// CHECK: cvttsd2si 64(%rdx), %r15
+// CHECK: encoding: [0xf2,0x4c,0x0f,0x2c,0x7a,0x40]
+cvttsd2si 64(%rdx), %r15
+
+// CHECK: cvttsd2si -64(%rdx,%rax,4), %r13d
+// CHECK: encoding: [0xf2,0x44,0x0f,0x2c,0x6c,0x82,0xc0]
+cvttsd2si -64(%rdx,%rax,4), %r13d
+
+// CHECK: cvttsd2si 64(%rdx,%rax,4), %r13d
+// CHECK: encoding: [0xf2,0x44,0x0f,0x2c,0x6c,0x82,0x40]
+cvttsd2si 64(%rdx,%rax,4), %r13d
+
+// CHECK: cvttsd2si -64(%rdx,%rax,4), %r15
+// CHECK: encoding: [0xf2,0x4c,0x0f,0x2c,0x7c,0x82,0xc0]
+cvttsd2si -64(%rdx,%rax,4), %r15
+
+// CHECK: cvttsd2si 64(%rdx,%rax,4), %r15
+// CHECK: encoding: [0xf2,0x4c,0x0f,0x2c,0x7c,0x82,0x40]
+cvttsd2si 64(%rdx,%rax,4), %r15
+
+// CHECK: cvttsd2si 64(%rdx,%rax), %r13d
+// CHECK: encoding: [0xf2,0x44,0x0f,0x2c,0x6c,0x02,0x40]
+cvttsd2si 64(%rdx,%rax), %r13d
+
+// CHECK: cvttsd2si 64(%rdx,%rax), %r15
+// CHECK: encoding: [0xf2,0x4c,0x0f,0x2c,0x7c,0x02,0x40]
+cvttsd2si 64(%rdx,%rax), %r15
+
+// CHECK: cvttsd2si (%rdx), %r13d
+// CHECK: encoding: [0xf2,0x44,0x0f,0x2c,0x2a]
+cvttsd2si (%rdx), %r13d
+
+// CHECK: cvttsd2si (%rdx), %r15
+// CHECK: encoding: [0xf2,0x4c,0x0f,0x2c,0x3a]
+cvttsd2si (%rdx), %r15
+
+// CHECK: cvttsd2si %xmm6, %r13d
+// CHECK: encoding: [0xf2,0x44,0x0f,0x2c,0xee]
+cvttsd2si %xmm6, %r13d
+
+// CHECK: cvttsd2si %xmm6, %r15
+// CHECK: encoding: [0xf2,0x4c,0x0f,0x2c,0xfe]
+cvttsd2si %xmm6, %r15
+
+// CHECK: divpd 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x5e,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+divpd 485498096, %xmm6
+
+// CHECK: divpd -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x5e,0x74,0x82,0xc0]
+divpd -64(%rdx,%rax,4), %xmm6
+
+// CHECK: divpd 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x5e,0x74,0x82,0x40]
+divpd 64(%rdx,%rax,4), %xmm6
+
+// CHECK: divpd 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x5e,0x74,0x02,0x40]
+divpd 64(%rdx,%rax), %xmm6
+
+// CHECK: divpd 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x5e,0x72,0x40]
+divpd 64(%rdx), %xmm6
+
+// CHECK: divpd (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x5e,0x32]
+divpd (%rdx), %xmm6
+
+// CHECK: divpd %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x5e,0xf6]
+divpd %xmm6, %xmm6
+
+// CHECK: divsd 485498096, %xmm6
+// CHECK: encoding: [0xf2,0x0f,0x5e,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+divsd 485498096, %xmm6
+
+// CHECK: divsd -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0xf2,0x0f,0x5e,0x74,0x82,0xc0]
+divsd -64(%rdx,%rax,4), %xmm6
+
+// CHECK: divsd 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0xf2,0x0f,0x5e,0x74,0x82,0x40]
+divsd 64(%rdx,%rax,4), %xmm6
+
+// CHECK: divsd 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0xf2,0x0f,0x5e,0x74,0x02,0x40]
+divsd 64(%rdx,%rax), %xmm6
+
+// CHECK: divsd 64(%rdx), %xmm6
+// CHECK: encoding: [0xf2,0x0f,0x5e,0x72,0x40]
+divsd 64(%rdx), %xmm6
+
+// CHECK: divsd (%rdx), %xmm6
+// CHECK: encoding: [0xf2,0x0f,0x5e,0x32]
+divsd (%rdx), %xmm6
+
+// CHECK: divsd %xmm6, %xmm6
+// CHECK: encoding: [0xf2,0x0f,0x5e,0xf6]
+divsd %xmm6, %xmm6
+
+// CHECK: lfence
+// CHECK: encoding: [0x0f,0xae,0xe8]
+lfence
+
+// CHECK: maskmovdqu %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0xf7,0xf6]
+maskmovdqu %xmm6, %xmm6
+
+// CHECK: maxpd 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x5f,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+maxpd 485498096, %xmm6
+
+// CHECK: maxpd -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x5f,0x74,0x82,0xc0]
+maxpd -64(%rdx,%rax,4), %xmm6
+
+// CHECK: maxpd 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x5f,0x74,0x82,0x40]
+maxpd 64(%rdx,%rax,4), %xmm6
+
+// CHECK: maxpd 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x5f,0x74,0x02,0x40]
+maxpd 64(%rdx,%rax), %xmm6
+
+// CHECK: maxpd 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x5f,0x72,0x40]
+maxpd 64(%rdx), %xmm6
+
+// CHECK: maxpd (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x5f,0x32]
+maxpd (%rdx), %xmm6
+
+// CHECK: maxpd %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x5f,0xf6]
+maxpd %xmm6, %xmm6
+
+// CHECK: maxsd 485498096, %xmm6
+// CHECK: encoding: [0xf2,0x0f,0x5f,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+maxsd 485498096, %xmm6
+
+// CHECK: maxsd -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0xf2,0x0f,0x5f,0x74,0x82,0xc0]
+maxsd -64(%rdx,%rax,4), %xmm6
+
+// CHECK: maxsd 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0xf2,0x0f,0x5f,0x74,0x82,0x40]
+maxsd 64(%rdx,%rax,4), %xmm6
+
+// CHECK: maxsd 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0xf2,0x0f,0x5f,0x74,0x02,0x40]
+maxsd 64(%rdx,%rax), %xmm6
+
+// CHECK: maxsd 64(%rdx), %xmm6
+// CHECK: encoding: [0xf2,0x0f,0x5f,0x72,0x40]
+maxsd 64(%rdx), %xmm6
+
+// CHECK: maxsd (%rdx), %xmm6
+// CHECK: encoding: [0xf2,0x0f,0x5f,0x32]
+maxsd (%rdx), %xmm6
+
+// CHECK: maxsd %xmm6, %xmm6
+// CHECK: encoding: [0xf2,0x0f,0x5f,0xf6]
+maxsd %xmm6, %xmm6
+
+// CHECK: mfence
+// CHECK: encoding: [0x0f,0xae,0xf0]
+mfence
+
+// CHECK: minpd 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x5d,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+minpd 485498096, %xmm6
+
+// CHECK: minpd -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x5d,0x74,0x82,0xc0]
+minpd -64(%rdx,%rax,4), %xmm6
+
+// CHECK: minpd 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x5d,0x74,0x82,0x40]
+minpd 64(%rdx,%rax,4), %xmm6
+
+// CHECK: minpd 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x5d,0x74,0x02,0x40]
+minpd 64(%rdx,%rax), %xmm6
+
+// CHECK: minpd 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x5d,0x72,0x40]
+minpd 64(%rdx), %xmm6
+
+// CHECK: minpd (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x5d,0x32]
+minpd (%rdx), %xmm6
+
+// CHECK: minpd %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x5d,0xf6]
+minpd %xmm6, %xmm6
+
+// CHECK: minsd 485498096, %xmm6
+// CHECK: encoding: [0xf2,0x0f,0x5d,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+minsd 485498096, %xmm6
+
+// CHECK: minsd -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0xf2,0x0f,0x5d,0x74,0x82,0xc0]
+minsd -64(%rdx,%rax,4), %xmm6
+
+// CHECK: minsd 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0xf2,0x0f,0x5d,0x74,0x82,0x40]
+minsd 64(%rdx,%rax,4), %xmm6
+
+// CHECK: minsd 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0xf2,0x0f,0x5d,0x74,0x02,0x40]
+minsd 64(%rdx,%rax), %xmm6
+
+// CHECK: minsd 64(%rdx), %xmm6
+// CHECK: encoding: [0xf2,0x0f,0x5d,0x72,0x40]
+minsd 64(%rdx), %xmm6
+
+// CHECK: minsd (%rdx), %xmm6
+// CHECK: encoding: [0xf2,0x0f,0x5d,0x32]
+minsd (%rdx), %xmm6
+
+// CHECK: minsd %xmm6, %xmm6
+// CHECK: encoding: [0xf2,0x0f,0x5d,0xf6]
+minsd %xmm6, %xmm6
+
+// CHECK: movapd 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x28,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+movapd 485498096, %xmm6
+
+// CHECK: movapd -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x28,0x74,0x82,0xc0]
+movapd -64(%rdx,%rax,4), %xmm6
+
+// CHECK: movapd 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x28,0x74,0x82,0x40]
+movapd 64(%rdx,%rax,4), %xmm6
+
+// CHECK: movapd 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x28,0x74,0x02,0x40]
+movapd 64(%rdx,%rax), %xmm6
+
+// CHECK: movapd 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x28,0x72,0x40]
+movapd 64(%rdx), %xmm6
+
+// CHECK: movapd (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x28,0x32]
+movapd (%rdx), %xmm6
+
+// CHECK: movapd %xmm6, 485498096
+// CHECK: encoding: [0x66,0x0f,0x29,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+movapd %xmm6, 485498096
+
+// CHECK: movapd %xmm6, 64(%rdx)
+// CHECK: encoding: [0x66,0x0f,0x29,0x72,0x40]
+movapd %xmm6, 64(%rdx)
+
+// CHECK: movapd %xmm6, 64(%rdx,%rax)
+// CHECK: encoding: [0x66,0x0f,0x29,0x74,0x02,0x40]
+movapd %xmm6, 64(%rdx,%rax)
+
+// CHECK: movapd %xmm6, -64(%rdx,%rax,4)
+// CHECK: encoding: [0x66,0x0f,0x29,0x74,0x82,0xc0]
+movapd %xmm6, -64(%rdx,%rax,4)
+
+// CHECK: movapd %xmm6, 64(%rdx,%rax,4)
+// CHECK: encoding: [0x66,0x0f,0x29,0x74,0x82,0x40]
+movapd %xmm6, 64(%rdx,%rax,4)
+
+// CHECK: movapd %xmm6, (%rdx)
+// CHECK: encoding: [0x66,0x0f,0x29,0x32]
+movapd %xmm6, (%rdx)
+
+// CHECK: movapd %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x28,0xf6]
+movapd %xmm6, %xmm6
+
+// CHECK: movd 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x6e,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+movd 485498096, %xmm6
+
+// CHECK: movd -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x6e,0x74,0x82,0xc0]
+movd -64(%rdx,%rax,4), %xmm6
+
+// CHECK: movd 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x6e,0x74,0x82,0x40]
+movd 64(%rdx,%rax,4), %xmm6
+
+// CHECK: movd 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x6e,0x74,0x02,0x40]
+movd 64(%rdx,%rax), %xmm6
+
+// CHECK: movd 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x6e,0x72,0x40]
+movd 64(%rdx), %xmm6
+
+// CHECK: movdq2q %xmm6, %mm4
+// CHECK: encoding: [0xf2,0x0f,0xd6,0xe6]
+movdq2q %xmm6, %mm4
+
+// CHECK: movdqa 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x6f,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+movdqa 485498096, %xmm6
+
+// CHECK: movdqa -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x6f,0x74,0x82,0xc0]
+movdqa -64(%rdx,%rax,4), %xmm6
+
+// CHECK: movdqa 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x6f,0x74,0x82,0x40]
+movdqa 64(%rdx,%rax,4), %xmm6
+
+// CHECK: movdqa 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x6f,0x74,0x02,0x40]
+movdqa 64(%rdx,%rax), %xmm6
+
+// CHECK: movdqa 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x6f,0x72,0x40]
+movdqa 64(%rdx), %xmm6
+
+// CHECK: movdqa (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x6f,0x32]
+movdqa (%rdx), %xmm6
+
+// CHECK: movdqa %xmm6, 485498096
+// CHECK: encoding: [0x66,0x0f,0x7f,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+movdqa %xmm6, 485498096
+
+// CHECK: movdqa %xmm6, 64(%rdx)
+// CHECK: encoding: [0x66,0x0f,0x7f,0x72,0x40]
+movdqa %xmm6, 64(%rdx)
+
+// CHECK: movdqa %xmm6, 64(%rdx,%rax)
+// CHECK: encoding: [0x66,0x0f,0x7f,0x74,0x02,0x40]
+movdqa %xmm6, 64(%rdx,%rax)
+
+// CHECK: movdqa %xmm6, -64(%rdx,%rax,4)
+// CHECK: encoding: [0x66,0x0f,0x7f,0x74,0x82,0xc0]
+movdqa %xmm6, -64(%rdx,%rax,4)
+
+// CHECK: movdqa %xmm6, 64(%rdx,%rax,4)
+// CHECK: encoding: [0x66,0x0f,0x7f,0x74,0x82,0x40]
+movdqa %xmm6, 64(%rdx,%rax,4)
+
+// CHECK: movdqa %xmm6, (%rdx)
+// CHECK: encoding: [0x66,0x0f,0x7f,0x32]
+movdqa %xmm6, (%rdx)
+
+// CHECK: movdqa %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x6f,0xf6]
+movdqa %xmm6, %xmm6
+
+// CHECK: movdqu 485498096, %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x6f,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+movdqu 485498096, %xmm6
+
+// CHECK: movdqu -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x6f,0x74,0x82,0xc0]
+movdqu -64(%rdx,%rax,4), %xmm6
+
+// CHECK: movdqu 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x6f,0x74,0x82,0x40]
+movdqu 64(%rdx,%rax,4), %xmm6
+
+// CHECK: movdqu 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x6f,0x74,0x02,0x40]
+movdqu 64(%rdx,%rax), %xmm6
+
+// CHECK: movdqu 64(%rdx), %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x6f,0x72,0x40]
+movdqu 64(%rdx), %xmm6
+
+// CHECK: movdqu (%rdx), %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x6f,0x32]
+movdqu (%rdx), %xmm6
+
+// CHECK: movdqu %xmm6, 485498096
+// CHECK: encoding: [0xf3,0x0f,0x7f,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+movdqu %xmm6, 485498096
+
+// CHECK: movdqu %xmm6, 64(%rdx)
+// CHECK: encoding: [0xf3,0x0f,0x7f,0x72,0x40]
+movdqu %xmm6, 64(%rdx)
+
+// CHECK: movdqu %xmm6, 64(%rdx,%rax)
+// CHECK: encoding: [0xf3,0x0f,0x7f,0x74,0x02,0x40]
+movdqu %xmm6, 64(%rdx,%rax)
+
+// CHECK: movdqu %xmm6, -64(%rdx,%rax,4)
+// CHECK: encoding: [0xf3,0x0f,0x7f,0x74,0x82,0xc0]
+movdqu %xmm6, -64(%rdx,%rax,4)
+
+// CHECK: movdqu %xmm6, 64(%rdx,%rax,4)
+// CHECK: encoding: [0xf3,0x0f,0x7f,0x74,0x82,0x40]
+movdqu %xmm6, 64(%rdx,%rax,4)
+
+// CHECK: movdqu %xmm6, (%rdx)
+// CHECK: encoding: [0xf3,0x0f,0x7f,0x32]
+movdqu %xmm6, (%rdx)
+
+// CHECK: movdqu %xmm6, %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x6f,0xf6]
+movdqu %xmm6, %xmm6
+
+// CHECK: movd %r13d, %xmm6
+// CHECK: encoding: [0x66,0x41,0x0f,0x6e,0xf5]
+movd %r13d, %xmm6
+
+// CHECK: movd (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x6e,0x32]
+movd (%rdx), %xmm6
+
+// CHECK: movd %xmm6, 485498096
+// CHECK: encoding: [0x66,0x0f,0x7e,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+movd %xmm6, 485498096
+
+// CHECK: movd %xmm6, 64(%rdx)
+// CHECK: encoding: [0x66,0x0f,0x7e,0x72,0x40]
+movd %xmm6, 64(%rdx)
+
+// CHECK: movd %xmm6, 64(%rdx,%rax)
+// CHECK: encoding: [0x66,0x0f,0x7e,0x74,0x02,0x40]
+movd %xmm6, 64(%rdx,%rax)
+
+// CHECK: movd %xmm6, -64(%rdx,%rax,4)
+// CHECK: encoding: [0x66,0x0f,0x7e,0x74,0x82,0xc0]
+movd %xmm6, -64(%rdx,%rax,4)
+
+// CHECK: movd %xmm6, 64(%rdx,%rax,4)
+// CHECK: encoding: [0x66,0x0f,0x7e,0x74,0x82,0x40]
+movd %xmm6, 64(%rdx,%rax,4)
+
+// CHECK: movd %xmm6, %r13d
+// CHECK: encoding: [0x66,0x41,0x0f,0x7e,0xf5]
+movd %xmm6, %r13d
+
+// CHECK: movd %xmm6, (%rdx)
+// CHECK: encoding: [0x66,0x0f,0x7e,0x32]
+movd %xmm6, (%rdx)
+
+// CHECK: movhpd 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x16,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+movhpd 485498096, %xmm6
+
+// CHECK: movhpd -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x16,0x74,0x82,0xc0]
+movhpd -64(%rdx,%rax,4), %xmm6
+
+// CHECK: movhpd 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x16,0x74,0x82,0x40]
+movhpd 64(%rdx,%rax,4), %xmm6
+
+// CHECK: movhpd 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x16,0x74,0x02,0x40]
+movhpd 64(%rdx,%rax), %xmm6
+
+// CHECK: movhpd 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x16,0x72,0x40]
+movhpd 64(%rdx), %xmm6
+
+// CHECK: movhpd (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x16,0x32]
+movhpd (%rdx), %xmm6
+
+// CHECK: movhpd %xmm6, 485498096
+// CHECK: encoding: [0x66,0x0f,0x17,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+movhpd %xmm6, 485498096
+
+// CHECK: movhpd %xmm6, 64(%rdx)
+// CHECK: encoding: [0x66,0x0f,0x17,0x72,0x40]
+movhpd %xmm6, 64(%rdx)
+
+// CHECK: movhpd %xmm6, 64(%rdx,%rax)
+// CHECK: encoding: [0x66,0x0f,0x17,0x74,0x02,0x40]
+movhpd %xmm6, 64(%rdx,%rax)
+
+// CHECK: movhpd %xmm6, -64(%rdx,%rax,4)
+// CHECK: encoding: [0x66,0x0f,0x17,0x74,0x82,0xc0]
+movhpd %xmm6, -64(%rdx,%rax,4)
+
+// CHECK: movhpd %xmm6, 64(%rdx,%rax,4)
+// CHECK: encoding: [0x66,0x0f,0x17,0x74,0x82,0x40]
+movhpd %xmm6, 64(%rdx,%rax,4)
+
+// CHECK: movhpd %xmm6, (%rdx)
+// CHECK: encoding: [0x66,0x0f,0x17,0x32]
+movhpd %xmm6, (%rdx)
+
+// CHECK: movlpd 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x12,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+movlpd 485498096, %xmm6
+
+// CHECK: movlpd -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x12,0x74,0x82,0xc0]
+movlpd -64(%rdx,%rax,4), %xmm6
+
+// CHECK: movlpd 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x12,0x74,0x82,0x40]
+movlpd 64(%rdx,%rax,4), %xmm6
+
+// CHECK: movlpd 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x12,0x74,0x02,0x40]
+movlpd 64(%rdx,%rax), %xmm6
+
+// CHECK: movlpd 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x12,0x72,0x40]
+movlpd 64(%rdx), %xmm6
+
+// CHECK: movlpd (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x12,0x32]
+movlpd (%rdx), %xmm6
+
+// CHECK: movlpd %xmm6, 485498096
+// CHECK: encoding: [0x66,0x0f,0x13,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+movlpd %xmm6, 485498096
+
+// CHECK: movlpd %xmm6, 64(%rdx)
+// CHECK: encoding: [0x66,0x0f,0x13,0x72,0x40]
+movlpd %xmm6, 64(%rdx)
+
+// CHECK: movlpd %xmm6, 64(%rdx,%rax)
+// CHECK: encoding: [0x66,0x0f,0x13,0x74,0x02,0x40]
+movlpd %xmm6, 64(%rdx,%rax)
+
+// CHECK: movlpd %xmm6, -64(%rdx,%rax,4)
+// CHECK: encoding: [0x66,0x0f,0x13,0x74,0x82,0xc0]
+movlpd %xmm6, -64(%rdx,%rax,4)
+
+// CHECK: movlpd %xmm6, 64(%rdx,%rax,4)
+// CHECK: encoding: [0x66,0x0f,0x13,0x74,0x82,0x40]
+movlpd %xmm6, 64(%rdx,%rax,4)
+
+// CHECK: movlpd %xmm6, (%rdx)
+// CHECK: encoding: [0x66,0x0f,0x13,0x32]
+movlpd %xmm6, (%rdx)
+
+// CHECK: movmskpd %xmm6, %r13d
+// CHECK: encoding: [0x66,0x44,0x0f,0x50,0xee]
+movmskpd %xmm6, %r13d
+
+// CHECK: movntdq %xmm6, 485498096
+// CHECK: encoding: [0x66,0x0f,0xe7,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+movntdq %xmm6, 485498096
+
+// CHECK: movntdq %xmm6, 64(%rdx)
+// CHECK: encoding: [0x66,0x0f,0xe7,0x72,0x40]
+movntdq %xmm6, 64(%rdx)
+
+// CHECK: movntdq %xmm6, 64(%rdx,%rax)
+// CHECK: encoding: [0x66,0x0f,0xe7,0x74,0x02,0x40]
+movntdq %xmm6, 64(%rdx,%rax)
+
+// CHECK: movntdq %xmm6, -64(%rdx,%rax,4)
+// CHECK: encoding: [0x66,0x0f,0xe7,0x74,0x82,0xc0]
+movntdq %xmm6, -64(%rdx,%rax,4)
+
+// CHECK: movntdq %xmm6, 64(%rdx,%rax,4)
+// CHECK: encoding: [0x66,0x0f,0xe7,0x74,0x82,0x40]
+movntdq %xmm6, 64(%rdx,%rax,4)
+
+// CHECK: movntdq %xmm6, (%rdx)
+// CHECK: encoding: [0x66,0x0f,0xe7,0x32]
+movntdq %xmm6, (%rdx)
+
+// CHECK: movntil %r13d, 485498096
+// CHECK: encoding: [0x44,0x0f,0xc3,0x2c,0x25,0xf0,0x1c,0xf0,0x1c]
+movntil %r13d, 485498096
+
+// CHECK: movntil %r13d, 64(%rdx)
+// CHECK: encoding: [0x44,0x0f,0xc3,0x6a,0x40]
+movntil %r13d, 64(%rdx)
+
+// CHECK: movntil %r13d, 64(%rdx,%rax)
+// CHECK: encoding: [0x44,0x0f,0xc3,0x6c,0x02,0x40]
+movntil %r13d, 64(%rdx,%rax)
+
+// CHECK: movntil %r13d, -64(%rdx,%rax,4)
+// CHECK: encoding: [0x44,0x0f,0xc3,0x6c,0x82,0xc0]
+movntil %r13d, -64(%rdx,%rax,4)
+
+// CHECK: movntil %r13d, 64(%rdx,%rax,4)
+// CHECK: encoding: [0x44,0x0f,0xc3,0x6c,0x82,0x40]
+movntil %r13d, 64(%rdx,%rax,4)
+
+// CHECK: movntil %r13d, (%rdx)
+// CHECK: encoding: [0x44,0x0f,0xc3,0x2a]
+movntil %r13d, (%rdx)
+
+// CHECK: movntpd %xmm6, 485498096
+// CHECK: encoding: [0x66,0x0f,0x2b,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+movntpd %xmm6, 485498096
+
+// CHECK: movntpd %xmm6, 64(%rdx)
+// CHECK: encoding: [0x66,0x0f,0x2b,0x72,0x40]
+movntpd %xmm6, 64(%rdx)
+
+// CHECK: movntpd %xmm6, 64(%rdx,%rax)
+// CHECK: encoding: [0x66,0x0f,0x2b,0x74,0x02,0x40]
+movntpd %xmm6, 64(%rdx,%rax)
+
+// CHECK: movntpd %xmm6, -64(%rdx,%rax,4)
+// CHECK: encoding: [0x66,0x0f,0x2b,0x74,0x82,0xc0]
+movntpd %xmm6, -64(%rdx,%rax,4)
+
+// CHECK: movntpd %xmm6, 64(%rdx,%rax,4)
+// CHECK: encoding: [0x66,0x0f,0x2b,0x74,0x82,0x40]
+movntpd %xmm6, 64(%rdx,%rax,4)
+
+// CHECK: movntpd %xmm6, (%rdx)
+// CHECK: encoding: [0x66,0x0f,0x2b,0x32]
+movntpd %xmm6, (%rdx)
+
+// CHECK: movq2dq %mm4, %xmm6
+// CHECK: encoding: [0xf3,0x0f,0xd6,0xf4]
+movq2dq %mm4, %xmm6
+
+// CHECK: movq 485498096, %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x7e,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+movq 485498096, %xmm6
+
+// CHECK: movq -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x7e,0x74,0x82,0xc0]
+movq -64(%rdx,%rax,4), %xmm6
+
+// CHECK: movq 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x7e,0x74,0x82,0x40]
+movq 64(%rdx,%rax,4), %xmm6
+
+// CHECK: movq 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x7e,0x74,0x02,0x40]
+movq 64(%rdx,%rax), %xmm6
+
+// CHECK: movq 64(%rdx), %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x7e,0x72,0x40]
+movq 64(%rdx), %xmm6
+
+// CHECK: movq %r15, %xmm6
+// CHECK: encoding: [0x66,0x49,0x0f,0x6e,0xf7]
+movq %r15, %xmm6
+
+// CHECK: movq (%rdx), %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x7e,0x32]
+movq (%rdx), %xmm6
+
+// CHECK: movq %xmm6, 485498096
+// CHECK: encoding: [0x66,0x0f,0xd6,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+movq %xmm6, 485498096
+
+// CHECK: movq %xmm6, 64(%rdx)
+// CHECK: encoding: [0x66,0x0f,0xd6,0x72,0x40]
+movq %xmm6, 64(%rdx)
+
+// CHECK: movq %xmm6, 64(%rdx,%rax)
+// CHECK: encoding: [0x66,0x0f,0xd6,0x74,0x02,0x40]
+movq %xmm6, 64(%rdx,%rax)
+
+// CHECK: movq %xmm6, -64(%rdx,%rax,4)
+// CHECK: encoding: [0x66,0x0f,0xd6,0x74,0x82,0xc0]
+movq %xmm6, -64(%rdx,%rax,4)
+
+// CHECK: movq %xmm6, 64(%rdx,%rax,4)
+// CHECK: encoding: [0x66,0x0f,0xd6,0x74,0x82,0x40]
+movq %xmm6, 64(%rdx,%rax,4)
+
+// CHECK: movq %xmm6, %r15
+// CHECK: encoding: [0x66,0x49,0x0f,0x7e,0xf7]
+movq %xmm6, %r15
+
+// CHECK: movq %xmm6, (%rdx)
+// CHECK: encoding: [0x66,0x0f,0xd6,0x32]
+movq %xmm6, (%rdx)
+
+// CHECK: movq %xmm6, %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x7e,0xf6]
+movq %xmm6, %xmm6
+
+// CHECK: movsd 485498096, %xmm6
+// CHECK: encoding: [0xf2,0x0f,0x10,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+movsd 485498096, %xmm6
+
+// CHECK: movsd -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0xf2,0x0f,0x10,0x74,0x82,0xc0]
+movsd -64(%rdx,%rax,4), %xmm6
+
+// CHECK: movsd 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0xf2,0x0f,0x10,0x74,0x82,0x40]
+movsd 64(%rdx,%rax,4), %xmm6
+
+// CHECK: movsd 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0xf2,0x0f,0x10,0x74,0x02,0x40]
+movsd 64(%rdx,%rax), %xmm6
+
+// CHECK: movsd 64(%rdx), %xmm6
+// CHECK: encoding: [0xf2,0x0f,0x10,0x72,0x40]
+movsd 64(%rdx), %xmm6
+
+// CHECK: movsd (%rdx), %xmm6
+// CHECK: encoding: [0xf2,0x0f,0x10,0x32]
+movsd (%rdx), %xmm6
+
+// CHECK: movsd %xmm6, 485498096
+// CHECK: encoding: [0xf2,0x0f,0x11,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+movsd %xmm6, 485498096
+
+// CHECK: movsd %xmm6, 64(%rdx)
+// CHECK: encoding: [0xf2,0x0f,0x11,0x72,0x40]
+movsd %xmm6, 64(%rdx)
+
+// CHECK: movsd %xmm6, 64(%rdx,%rax)
+// CHECK: encoding: [0xf2,0x0f,0x11,0x74,0x02,0x40]
+movsd %xmm6, 64(%rdx,%rax)
+
+// CHECK: movsd %xmm6, -64(%rdx,%rax,4)
+// CHECK: encoding: [0xf2,0x0f,0x11,0x74,0x82,0xc0]
+movsd %xmm6, -64(%rdx,%rax,4)
+
+// CHECK: movsd %xmm6, 64(%rdx,%rax,4)
+// CHECK: encoding: [0xf2,0x0f,0x11,0x74,0x82,0x40]
+movsd %xmm6, 64(%rdx,%rax,4)
+
+// CHECK: movsd %xmm6, (%rdx)
+// CHECK: encoding: [0xf2,0x0f,0x11,0x32]
+movsd %xmm6, (%rdx)
+
+// CHECK: movsd %xmm6, %xmm6
+// CHECK: encoding: [0xf2,0x0f,0x10,0xf6]
+movsd %xmm6, %xmm6
+
+// CHECK: movupd 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x10,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+movupd 485498096, %xmm6
+
+// CHECK: movupd -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x10,0x74,0x82,0xc0]
+movupd -64(%rdx,%rax,4), %xmm6
+
+// CHECK: movupd 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x10,0x74,0x82,0x40]
+movupd 64(%rdx,%rax,4), %xmm6
+
+// CHECK: movupd 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x10,0x74,0x02,0x40]
+movupd 64(%rdx,%rax), %xmm6
+
+// CHECK: movupd 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x10,0x72,0x40]
+movupd 64(%rdx), %xmm6
+
+// CHECK: movupd (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x10,0x32]
+movupd (%rdx), %xmm6
+
+// CHECK: movupd %xmm6, 485498096
+// CHECK: encoding: [0x66,0x0f,0x11,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+movupd %xmm6, 485498096
+
+// CHECK: movupd %xmm6, 64(%rdx)
+// CHECK: encoding: [0x66,0x0f,0x11,0x72,0x40]
+movupd %xmm6, 64(%rdx)
+
+// CHECK: movupd %xmm6, 64(%rdx,%rax)
+// CHECK: encoding: [0x66,0x0f,0x11,0x74,0x02,0x40]
+movupd %xmm6, 64(%rdx,%rax)
+
+// CHECK: movupd %xmm6, -64(%rdx,%rax,4)
+// CHECK: encoding: [0x66,0x0f,0x11,0x74,0x82,0xc0]
+movupd %xmm6, -64(%rdx,%rax,4)
+
+// CHECK: movupd %xmm6, 64(%rdx,%rax,4)
+// CHECK: encoding: [0x66,0x0f,0x11,0x74,0x82,0x40]
+movupd %xmm6, 64(%rdx,%rax,4)
+
+// CHECK: movupd %xmm6, (%rdx)
+// CHECK: encoding: [0x66,0x0f,0x11,0x32]
+movupd %xmm6, (%rdx)
+
+// CHECK: movupd %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x10,0xf6]
+movupd %xmm6, %xmm6
+
+// CHECK: mulpd 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x59,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+mulpd 485498096, %xmm6
+
+// CHECK: mulpd -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x59,0x74,0x82,0xc0]
+mulpd -64(%rdx,%rax,4), %xmm6
+
+// CHECK: mulpd 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x59,0x74,0x82,0x40]
+mulpd 64(%rdx,%rax,4), %xmm6
+
+// CHECK: mulpd 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x59,0x74,0x02,0x40]
+mulpd 64(%rdx,%rax), %xmm6
+
+// CHECK: mulpd 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x59,0x72,0x40]
+mulpd 64(%rdx), %xmm6
+
+// CHECK: mulpd (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x59,0x32]
+mulpd (%rdx), %xmm6
+
+// CHECK: mulpd %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x59,0xf6]
+mulpd %xmm6, %xmm6
+
+// CHECK: mulsd 485498096, %xmm6
+// CHECK: encoding: [0xf2,0x0f,0x59,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+mulsd 485498096, %xmm6
+
+// CHECK: mulsd -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0xf2,0x0f,0x59,0x74,0x82,0xc0]
+mulsd -64(%rdx,%rax,4), %xmm6
+
+// CHECK: mulsd 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0xf2,0x0f,0x59,0x74,0x82,0x40]
+mulsd 64(%rdx,%rax,4), %xmm6
+
+// CHECK: mulsd 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0xf2,0x0f,0x59,0x74,0x02,0x40]
+mulsd 64(%rdx,%rax), %xmm6
+
+// CHECK: mulsd 64(%rdx), %xmm6
+// CHECK: encoding: [0xf2,0x0f,0x59,0x72,0x40]
+mulsd 64(%rdx), %xmm6
+
+// CHECK: mulsd (%rdx), %xmm6
+// CHECK: encoding: [0xf2,0x0f,0x59,0x32]
+mulsd (%rdx), %xmm6
+
+// CHECK: mulsd %xmm6, %xmm6
+// CHECK: encoding: [0xf2,0x0f,0x59,0xf6]
+mulsd %xmm6, %xmm6
+
+// CHECK: orpd 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x56,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+orpd 485498096, %xmm6
+
+// CHECK: orpd -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x56,0x74,0x82,0xc0]
+orpd -64(%rdx,%rax,4), %xmm6
+
+// CHECK: orpd 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x56,0x74,0x82,0x40]
+orpd 64(%rdx,%rax,4), %xmm6
+
+// CHECK: orpd 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x56,0x74,0x02,0x40]
+orpd 64(%rdx,%rax), %xmm6
+
+// CHECK: orpd 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x56,0x72,0x40]
+orpd 64(%rdx), %xmm6
+
+// CHECK: orpd (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x56,0x32]
+orpd (%rdx), %xmm6
+
+// CHECK: orpd %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x56,0xf6]
+orpd %xmm6, %xmm6
+
+// CHECK: packssdw 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x6b,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+packssdw 485498096, %xmm6
+
+// CHECK: packssdw -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x6b,0x74,0x82,0xc0]
+packssdw -64(%rdx,%rax,4), %xmm6
+
+// CHECK: packssdw 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x6b,0x74,0x82,0x40]
+packssdw 64(%rdx,%rax,4), %xmm6
+
+// CHECK: packssdw 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x6b,0x74,0x02,0x40]
+packssdw 64(%rdx,%rax), %xmm6
+
+// CHECK: packssdw 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x6b,0x72,0x40]
+packssdw 64(%rdx), %xmm6
+
+// CHECK: packssdw (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x6b,0x32]
+packssdw (%rdx), %xmm6
+
+// CHECK: packssdw %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x6b,0xf6]
+packssdw %xmm6, %xmm6
+
+// CHECK: packsswb 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x63,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+packsswb 485498096, %xmm6
+
+// CHECK: packsswb -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x63,0x74,0x82,0xc0]
+packsswb -64(%rdx,%rax,4), %xmm6
+
+// CHECK: packsswb 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x63,0x74,0x82,0x40]
+packsswb 64(%rdx,%rax,4), %xmm6
+
+// CHECK: packsswb 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x63,0x74,0x02,0x40]
+packsswb 64(%rdx,%rax), %xmm6
+
+// CHECK: packsswb 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x63,0x72,0x40]
+packsswb 64(%rdx), %xmm6
+
+// CHECK: packsswb (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x63,0x32]
+packsswb (%rdx), %xmm6
+
+// CHECK: packsswb %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x63,0xf6]
+packsswb %xmm6, %xmm6
+
+// CHECK: packuswb 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x67,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+packuswb 485498096, %xmm6
+
+// CHECK: packuswb -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x67,0x74,0x82,0xc0]
+packuswb -64(%rdx,%rax,4), %xmm6
+
+// CHECK: packuswb 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x67,0x74,0x82,0x40]
+packuswb 64(%rdx,%rax,4), %xmm6
+
+// CHECK: packuswb 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x67,0x74,0x02,0x40]
+packuswb 64(%rdx,%rax), %xmm6
+
+// CHECK: packuswb 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x67,0x72,0x40]
+packuswb 64(%rdx), %xmm6
+
+// CHECK: packuswb (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x67,0x32]
+packuswb (%rdx), %xmm6
+
+// CHECK: packuswb %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x67,0xf6]
+packuswb %xmm6, %xmm6
+
+// CHECK: paddb 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0xfc,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+paddb 485498096, %xmm6
+
+// CHECK: paddb -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xfc,0x74,0x82,0xc0]
+paddb -64(%rdx,%rax,4), %xmm6
+
+// CHECK: paddb 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xfc,0x74,0x82,0x40]
+paddb 64(%rdx,%rax,4), %xmm6
+
+// CHECK: paddb 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xfc,0x74,0x02,0x40]
+paddb 64(%rdx,%rax), %xmm6
+
+// CHECK: paddb 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xfc,0x72,0x40]
+paddb 64(%rdx), %xmm6
+
+// CHECK: paddb (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xfc,0x32]
+paddb (%rdx), %xmm6
+
+// CHECK: paddb %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0xfc,0xf6]
+paddb %xmm6, %xmm6
+
+// CHECK: paddd 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0xfe,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+paddd 485498096, %xmm6
+
+// CHECK: paddd -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xfe,0x74,0x82,0xc0]
+paddd -64(%rdx,%rax,4), %xmm6
+
+// CHECK: paddd 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xfe,0x74,0x82,0x40]
+paddd 64(%rdx,%rax,4), %xmm6
+
+// CHECK: paddd 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xfe,0x74,0x02,0x40]
+paddd 64(%rdx,%rax), %xmm6
+
+// CHECK: paddd 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xfe,0x72,0x40]
+paddd 64(%rdx), %xmm6
+
+// CHECK: paddd (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xfe,0x32]
+paddd (%rdx), %xmm6
+
+// CHECK: paddd %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0xfe,0xf6]
+paddd %xmm6, %xmm6
+
+// CHECK: paddq 485498096, %mm4
+// CHECK: encoding: [0x0f,0xd4,0x24,0x25,0xf0,0x1c,0xf0,0x1c]
+paddq 485498096, %mm4
+
+// CHECK: paddq 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0xd4,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+paddq 485498096, %xmm6
+
+// CHECK: paddq 64(%rdx), %mm4
+// CHECK: encoding: [0x0f,0xd4,0x62,0x40]
+paddq 64(%rdx), %mm4
+
+// CHECK: paddq -64(%rdx,%rax,4), %mm4
+// CHECK: encoding: [0x0f,0xd4,0x64,0x82,0xc0]
+paddq -64(%rdx,%rax,4), %mm4
+
+// CHECK: paddq 64(%rdx,%rax,4), %mm4
+// CHECK: encoding: [0x0f,0xd4,0x64,0x82,0x40]
+paddq 64(%rdx,%rax,4), %mm4
+
+// CHECK: paddq -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xd4,0x74,0x82,0xc0]
+paddq -64(%rdx,%rax,4), %xmm6
+
+// CHECK: paddq 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xd4,0x74,0x82,0x40]
+paddq 64(%rdx,%rax,4), %xmm6
+
+// CHECK: paddq 64(%rdx,%rax), %mm4
+// CHECK: encoding: [0x0f,0xd4,0x64,0x02,0x40]
+paddq 64(%rdx,%rax), %mm4
+
+// CHECK: paddq 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xd4,0x74,0x02,0x40]
+paddq 64(%rdx,%rax), %xmm6
+
+// CHECK: paddq 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xd4,0x72,0x40]
+paddq 64(%rdx), %xmm6
+
+// CHECK: paddq %mm4, %mm4
+// CHECK: encoding: [0x0f,0xd4,0xe4]
+paddq %mm4, %mm4
+
+// CHECK: paddq (%rdx), %mm4
+// CHECK: encoding: [0x0f,0xd4,0x22]
+paddq (%rdx), %mm4
+
+// CHECK: paddq (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xd4,0x32]
+paddq (%rdx), %xmm6
+
+// CHECK: paddq %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0xd4,0xf6]
+paddq %xmm6, %xmm6
+
+// CHECK: paddsb 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0xec,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+paddsb 485498096, %xmm6
+
+// CHECK: paddsb -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xec,0x74,0x82,0xc0]
+paddsb -64(%rdx,%rax,4), %xmm6
+
+// CHECK: paddsb 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xec,0x74,0x82,0x40]
+paddsb 64(%rdx,%rax,4), %xmm6
+
+// CHECK: paddsb 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xec,0x74,0x02,0x40]
+paddsb 64(%rdx,%rax), %xmm6
+
+// CHECK: paddsb 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xec,0x72,0x40]
+paddsb 64(%rdx), %xmm6
+
+// CHECK: paddsb (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xec,0x32]
+paddsb (%rdx), %xmm6
+
+// CHECK: paddsb %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0xec,0xf6]
+paddsb %xmm6, %xmm6
+
+// CHECK: paddsw 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0xed,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+paddsw 485498096, %xmm6
+
+// CHECK: paddsw -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xed,0x74,0x82,0xc0]
+paddsw -64(%rdx,%rax,4), %xmm6
+
+// CHECK: paddsw 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xed,0x74,0x82,0x40]
+paddsw 64(%rdx,%rax,4), %xmm6
+
+// CHECK: paddsw 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xed,0x74,0x02,0x40]
+paddsw 64(%rdx,%rax), %xmm6
+
+// CHECK: paddsw 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xed,0x72,0x40]
+paddsw 64(%rdx), %xmm6
+
+// CHECK: paddsw (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xed,0x32]
+paddsw (%rdx), %xmm6
+
+// CHECK: paddsw %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0xed,0xf6]
+paddsw %xmm6, %xmm6
+
+// CHECK: paddusb 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0xdc,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+paddusb 485498096, %xmm6
+
+// CHECK: paddusb -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xdc,0x74,0x82,0xc0]
+paddusb -64(%rdx,%rax,4), %xmm6
+
+// CHECK: paddusb 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xdc,0x74,0x82,0x40]
+paddusb 64(%rdx,%rax,4), %xmm6
+
+// CHECK: paddusb 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xdc,0x74,0x02,0x40]
+paddusb 64(%rdx,%rax), %xmm6
+
+// CHECK: paddusb 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xdc,0x72,0x40]
+paddusb 64(%rdx), %xmm6
+
+// CHECK: paddusb (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xdc,0x32]
+paddusb (%rdx), %xmm6
+
+// CHECK: paddusb %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0xdc,0xf6]
+paddusb %xmm6, %xmm6
+
+// CHECK: paddusw 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0xdd,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+paddusw 485498096, %xmm6
+
+// CHECK: paddusw -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xdd,0x74,0x82,0xc0]
+paddusw -64(%rdx,%rax,4), %xmm6
+
+// CHECK: paddusw 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xdd,0x74,0x82,0x40]
+paddusw 64(%rdx,%rax,4), %xmm6
+
+// CHECK: paddusw 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xdd,0x74,0x02,0x40]
+paddusw 64(%rdx,%rax), %xmm6
+
+// CHECK: paddusw 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xdd,0x72,0x40]
+paddusw 64(%rdx), %xmm6
+
+// CHECK: paddusw (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xdd,0x32]
+paddusw (%rdx), %xmm6
+
+// CHECK: paddusw %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0xdd,0xf6]
+paddusw %xmm6, %xmm6
+
+// CHECK: paddw 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0xfd,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+paddw 485498096, %xmm6
+
+// CHECK: paddw -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xfd,0x74,0x82,0xc0]
+paddw -64(%rdx,%rax,4), %xmm6
+
+// CHECK: paddw 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xfd,0x74,0x82,0x40]
+paddw 64(%rdx,%rax,4), %xmm6
+
+// CHECK: paddw 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xfd,0x74,0x02,0x40]
+paddw 64(%rdx,%rax), %xmm6
+
+// CHECK: paddw 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xfd,0x72,0x40]
+paddw 64(%rdx), %xmm6
+
+// CHECK: paddw (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xfd,0x32]
+paddw (%rdx), %xmm6
+
+// CHECK: paddw %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0xfd,0xf6]
+paddw %xmm6, %xmm6
+
+// CHECK: pand 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0xdb,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+pand 485498096, %xmm6
+
+// CHECK: pand -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xdb,0x74,0x82,0xc0]
+pand -64(%rdx,%rax,4), %xmm6
+
+// CHECK: pand 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xdb,0x74,0x82,0x40]
+pand 64(%rdx,%rax,4), %xmm6
+
+// CHECK: pand 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xdb,0x74,0x02,0x40]
+pand 64(%rdx,%rax), %xmm6
+
+// CHECK: pand 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xdb,0x72,0x40]
+pand 64(%rdx), %xmm6
+
+// CHECK: pandn 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0xdf,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+pandn 485498096, %xmm6
+
+// CHECK: pandn -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xdf,0x74,0x82,0xc0]
+pandn -64(%rdx,%rax,4), %xmm6
+
+// CHECK: pandn 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xdf,0x74,0x82,0x40]
+pandn 64(%rdx,%rax,4), %xmm6
+
+// CHECK: pandn 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xdf,0x74,0x02,0x40]
+pandn 64(%rdx,%rax), %xmm6
+
+// CHECK: pandn 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xdf,0x72,0x40]
+pandn 64(%rdx), %xmm6
+
+// CHECK: pandn (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xdf,0x32]
+pandn (%rdx), %xmm6
+
+// CHECK: pandn %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0xdf,0xf6]
+pandn %xmm6, %xmm6
+
+// CHECK: pand (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xdb,0x32]
+pand (%rdx), %xmm6
+
+// CHECK: pand %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0xdb,0xf6]
+pand %xmm6, %xmm6
+
+// CHECK: pavgb 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0xe0,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+pavgb 485498096, %xmm6
+
+// CHECK: pavgb -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xe0,0x74,0x82,0xc0]
+pavgb -64(%rdx,%rax,4), %xmm6
+
+// CHECK: pavgb 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xe0,0x74,0x82,0x40]
+pavgb 64(%rdx,%rax,4), %xmm6
+
+// CHECK: pavgb 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xe0,0x74,0x02,0x40]
+pavgb 64(%rdx,%rax), %xmm6
+
+// CHECK: pavgb 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xe0,0x72,0x40]
+pavgb 64(%rdx), %xmm6
+
+// CHECK: pavgb (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xe0,0x32]
+pavgb (%rdx), %xmm6
+
+// CHECK: pavgb %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0xe0,0xf6]
+pavgb %xmm6, %xmm6
+
+// CHECK: pavgw 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0xe3,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+pavgw 485498096, %xmm6
+
+// CHECK: pavgw -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xe3,0x74,0x82,0xc0]
+pavgw -64(%rdx,%rax,4), %xmm6
+
+// CHECK: pavgw 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xe3,0x74,0x82,0x40]
+pavgw 64(%rdx,%rax,4), %xmm6
+
+// CHECK: pavgw 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xe3,0x74,0x02,0x40]
+pavgw 64(%rdx,%rax), %xmm6
+
+// CHECK: pavgw 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xe3,0x72,0x40]
+pavgw 64(%rdx), %xmm6
+
+// CHECK: pavgw (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xe3,0x32]
+pavgw (%rdx), %xmm6
+
+// CHECK: pavgw %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0xe3,0xf6]
+pavgw %xmm6, %xmm6
+
+// CHECK: pcmpeqb 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x74,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+pcmpeqb 485498096, %xmm6
+
+// CHECK: pcmpeqb -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x74,0x74,0x82,0xc0]
+pcmpeqb -64(%rdx,%rax,4), %xmm6
+
+// CHECK: pcmpeqb 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x74,0x74,0x82,0x40]
+pcmpeqb 64(%rdx,%rax,4), %xmm6
+
+// CHECK: pcmpeqb 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x74,0x74,0x02,0x40]
+pcmpeqb 64(%rdx,%rax), %xmm6
+
+// CHECK: pcmpeqb 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x74,0x72,0x40]
+pcmpeqb 64(%rdx), %xmm6
+
+// CHECK: pcmpeqb (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x74,0x32]
+pcmpeqb (%rdx), %xmm6
+
+// CHECK: pcmpeqb %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x74,0xf6]
+pcmpeqb %xmm6, %xmm6
+
+// CHECK: pcmpeqd 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x76,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+pcmpeqd 485498096, %xmm6
+
+// CHECK: pcmpeqd -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x76,0x74,0x82,0xc0]
+pcmpeqd -64(%rdx,%rax,4), %xmm6
+
+// CHECK: pcmpeqd 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x76,0x74,0x82,0x40]
+pcmpeqd 64(%rdx,%rax,4), %xmm6
+
+// CHECK: pcmpeqd 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x76,0x74,0x02,0x40]
+pcmpeqd 64(%rdx,%rax), %xmm6
+
+// CHECK: pcmpeqd 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x76,0x72,0x40]
+pcmpeqd 64(%rdx), %xmm6
+
+// CHECK: pcmpeqd (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x76,0x32]
+pcmpeqd (%rdx), %xmm6
+
+// CHECK: pcmpeqd %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x76,0xf6]
+pcmpeqd %xmm6, %xmm6
+
+// CHECK: pcmpeqw 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x75,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+pcmpeqw 485498096, %xmm6
+
+// CHECK: pcmpeqw -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x75,0x74,0x82,0xc0]
+pcmpeqw -64(%rdx,%rax,4), %xmm6
+
+// CHECK: pcmpeqw 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x75,0x74,0x82,0x40]
+pcmpeqw 64(%rdx,%rax,4), %xmm6
+
+// CHECK: pcmpeqw 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x75,0x74,0x02,0x40]
+pcmpeqw 64(%rdx,%rax), %xmm6
+
+// CHECK: pcmpeqw 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x75,0x72,0x40]
+pcmpeqw 64(%rdx), %xmm6
+
+// CHECK: pcmpeqw (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x75,0x32]
+pcmpeqw (%rdx), %xmm6
+
+// CHECK: pcmpeqw %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x75,0xf6]
+pcmpeqw %xmm6, %xmm6
+
+// CHECK: pcmpgtb 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x64,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+pcmpgtb 485498096, %xmm6
+
+// CHECK: pcmpgtb -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x64,0x74,0x82,0xc0]
+pcmpgtb -64(%rdx,%rax,4), %xmm6
+
+// CHECK: pcmpgtb 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x64,0x74,0x82,0x40]
+pcmpgtb 64(%rdx,%rax,4), %xmm6
+
+// CHECK: pcmpgtb 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x64,0x74,0x02,0x40]
+pcmpgtb 64(%rdx,%rax), %xmm6
+
+// CHECK: pcmpgtb 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x64,0x72,0x40]
+pcmpgtb 64(%rdx), %xmm6
+
+// CHECK: pcmpgtb (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x64,0x32]
+pcmpgtb (%rdx), %xmm6
+
+// CHECK: pcmpgtb %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x64,0xf6]
+pcmpgtb %xmm6, %xmm6
+
+// CHECK: pcmpgtd 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x66,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+pcmpgtd 485498096, %xmm6
+
+// CHECK: pcmpgtd -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x66,0x74,0x82,0xc0]
+pcmpgtd -64(%rdx,%rax,4), %xmm6
+
+// CHECK: pcmpgtd 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x66,0x74,0x82,0x40]
+pcmpgtd 64(%rdx,%rax,4), %xmm6
+
+// CHECK: pcmpgtd 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x66,0x74,0x02,0x40]
+pcmpgtd 64(%rdx,%rax), %xmm6
+
+// CHECK: pcmpgtd 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x66,0x72,0x40]
+pcmpgtd 64(%rdx), %xmm6
+
+// CHECK: pcmpgtd (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x66,0x32]
+pcmpgtd (%rdx), %xmm6
+
+// CHECK: pcmpgtd %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x66,0xf6]
+pcmpgtd %xmm6, %xmm6
+
+// CHECK: pcmpgtw 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x65,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+pcmpgtw 485498096, %xmm6
+
+// CHECK: pcmpgtw -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x65,0x74,0x82,0xc0]
+pcmpgtw -64(%rdx,%rax,4), %xmm6
+
+// CHECK: pcmpgtw 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x65,0x74,0x82,0x40]
+pcmpgtw 64(%rdx,%rax,4), %xmm6
+
+// CHECK: pcmpgtw 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x65,0x74,0x02,0x40]
+pcmpgtw 64(%rdx,%rax), %xmm6
+
+// CHECK: pcmpgtw 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x65,0x72,0x40]
+pcmpgtw 64(%rdx), %xmm6
+
+// CHECK: pcmpgtw (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x65,0x32]
+pcmpgtw (%rdx), %xmm6
+
+// CHECK: pcmpgtw %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x65,0xf6]
+pcmpgtw %xmm6, %xmm6
+
+// CHECK: pextrw $0, %xmm6, %r13d
+// CHECK: encoding: [0x66,0x44,0x0f,0xc5,0xee,0x00]
+pextrw $0, %xmm6, %r13d
+
+// CHECK: pinsrw $0, 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0xc4,0x34,0x25,0xf0,0x1c,0xf0,0x1c,0x00]
+pinsrw $0, 485498096, %xmm6
+
+// CHECK: pinsrw $0, -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xc4,0x74,0x82,0xc0,0x00]
+pinsrw $0, -64(%rdx,%rax,4), %xmm6
+
+// CHECK: pinsrw $0, 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xc4,0x74,0x82,0x40,0x00]
+pinsrw $0, 64(%rdx,%rax,4), %xmm6
+
+// CHECK: pinsrw $0, 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xc4,0x74,0x02,0x40,0x00]
+pinsrw $0, 64(%rdx,%rax), %xmm6
+
+// CHECK: pinsrw $0, 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xc4,0x72,0x40,0x00]
+pinsrw $0, 64(%rdx), %xmm6
+
+// CHECK: pinsrw $0, %r13d, %xmm6
+// CHECK: encoding: [0x66,0x41,0x0f,0xc4,0xf5,0x00]
+pinsrw $0, %r13d, %xmm6
+
+// CHECK: pinsrw $0, (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xc4,0x32,0x00]
+pinsrw $0, (%rdx), %xmm6
+
+// CHECK: pmaddwd 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0xf5,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+pmaddwd 485498096, %xmm6
+
+// CHECK: pmaddwd -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xf5,0x74,0x82,0xc0]
+pmaddwd -64(%rdx,%rax,4), %xmm6
+
+// CHECK: pmaddwd 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xf5,0x74,0x82,0x40]
+pmaddwd 64(%rdx,%rax,4), %xmm6
+
+// CHECK: pmaddwd 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xf5,0x74,0x02,0x40]
+pmaddwd 64(%rdx,%rax), %xmm6
+
+// CHECK: pmaddwd 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xf5,0x72,0x40]
+pmaddwd 64(%rdx), %xmm6
+
+// CHECK: pmaddwd (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xf5,0x32]
+pmaddwd (%rdx), %xmm6
+
+// CHECK: pmaddwd %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0xf5,0xf6]
+pmaddwd %xmm6, %xmm6
+
+// CHECK: pmaxsw 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0xee,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+pmaxsw 485498096, %xmm6
+
+// CHECK: pmaxsw -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xee,0x74,0x82,0xc0]
+pmaxsw -64(%rdx,%rax,4), %xmm6
+
+// CHECK: pmaxsw 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xee,0x74,0x82,0x40]
+pmaxsw 64(%rdx,%rax,4), %xmm6
+
+// CHECK: pmaxsw 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xee,0x74,0x02,0x40]
+pmaxsw 64(%rdx,%rax), %xmm6
+
+// CHECK: pmaxsw 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xee,0x72,0x40]
+pmaxsw 64(%rdx), %xmm6
+
+// CHECK: pmaxsw (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xee,0x32]
+pmaxsw (%rdx), %xmm6
+
+// CHECK: pmaxsw %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0xee,0xf6]
+pmaxsw %xmm6, %xmm6
+
+// CHECK: pmaxub 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0xde,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+pmaxub 485498096, %xmm6
+
+// CHECK: pmaxub -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xde,0x74,0x82,0xc0]
+pmaxub -64(%rdx,%rax,4), %xmm6
+
+// CHECK: pmaxub 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xde,0x74,0x82,0x40]
+pmaxub 64(%rdx,%rax,4), %xmm6
+
+// CHECK: pmaxub 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xde,0x74,0x02,0x40]
+pmaxub 64(%rdx,%rax), %xmm6
+
+// CHECK: pmaxub 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xde,0x72,0x40]
+pmaxub 64(%rdx), %xmm6
+
+// CHECK: pmaxub (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xde,0x32]
+pmaxub (%rdx), %xmm6
+
+// CHECK: pmaxub %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0xde,0xf6]
+pmaxub %xmm6, %xmm6
+
+// CHECK: pminsw 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0xea,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+pminsw 485498096, %xmm6
+
+// CHECK: pminsw -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xea,0x74,0x82,0xc0]
+pminsw -64(%rdx,%rax,4), %xmm6
+
+// CHECK: pminsw 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xea,0x74,0x82,0x40]
+pminsw 64(%rdx,%rax,4), %xmm6
+
+// CHECK: pminsw 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xea,0x74,0x02,0x40]
+pminsw 64(%rdx,%rax), %xmm6
+
+// CHECK: pminsw 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xea,0x72,0x40]
+pminsw 64(%rdx), %xmm6
+
+// CHECK: pminsw (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xea,0x32]
+pminsw (%rdx), %xmm6
+
+// CHECK: pminsw %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0xea,0xf6]
+pminsw %xmm6, %xmm6
+
+// CHECK: pminub 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0xda,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+pminub 485498096, %xmm6
+
+// CHECK: pminub -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xda,0x74,0x82,0xc0]
+pminub -64(%rdx,%rax,4), %xmm6
+
+// CHECK: pminub 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xda,0x74,0x82,0x40]
+pminub 64(%rdx,%rax,4), %xmm6
+
+// CHECK: pminub 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xda,0x74,0x02,0x40]
+pminub 64(%rdx,%rax), %xmm6
+
+// CHECK: pminub 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xda,0x72,0x40]
+pminub 64(%rdx), %xmm6
+
+// CHECK: pminub (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xda,0x32]
+pminub (%rdx), %xmm6
+
+// CHECK: pminub %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0xda,0xf6]
+pminub %xmm6, %xmm6
+
+// CHECK: pmovmskb %xmm6, %r13d
+// CHECK: encoding: [0x66,0x44,0x0f,0xd7,0xee]
+pmovmskb %xmm6, %r13d
+
+// CHECK: pmulhuw 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0xe4,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+pmulhuw 485498096, %xmm6
+
+// CHECK: pmulhuw -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xe4,0x74,0x82,0xc0]
+pmulhuw -64(%rdx,%rax,4), %xmm6
+
+// CHECK: pmulhuw 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xe4,0x74,0x82,0x40]
+pmulhuw 64(%rdx,%rax,4), %xmm6
+
+// CHECK: pmulhuw 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xe4,0x74,0x02,0x40]
+pmulhuw 64(%rdx,%rax), %xmm6
+
+// CHECK: pmulhuw 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xe4,0x72,0x40]
+pmulhuw 64(%rdx), %xmm6
+
+// CHECK: pmulhuw (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xe4,0x32]
+pmulhuw (%rdx), %xmm6
+
+// CHECK: pmulhuw %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0xe4,0xf6]
+pmulhuw %xmm6, %xmm6
+
+// CHECK: pmulhw 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0xe5,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+pmulhw 485498096, %xmm6
+
+// CHECK: pmulhw -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xe5,0x74,0x82,0xc0]
+pmulhw -64(%rdx,%rax,4), %xmm6
+
+// CHECK: pmulhw 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xe5,0x74,0x82,0x40]
+pmulhw 64(%rdx,%rax,4), %xmm6
+
+// CHECK: pmulhw 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xe5,0x74,0x02,0x40]
+pmulhw 64(%rdx,%rax), %xmm6
+
+// CHECK: pmulhw 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xe5,0x72,0x40]
+pmulhw 64(%rdx), %xmm6
+
+// CHECK: pmulhw (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xe5,0x32]
+pmulhw (%rdx), %xmm6
+
+// CHECK: pmulhw %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0xe5,0xf6]
+pmulhw %xmm6, %xmm6
+
+// CHECK: pmullw 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0xd5,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+pmullw 485498096, %xmm6
+
+// CHECK: pmullw -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xd5,0x74,0x82,0xc0]
+pmullw -64(%rdx,%rax,4), %xmm6
+
+// CHECK: pmullw 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xd5,0x74,0x82,0x40]
+pmullw 64(%rdx,%rax,4), %xmm6
+
+// CHECK: pmullw 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xd5,0x74,0x02,0x40]
+pmullw 64(%rdx,%rax), %xmm6
+
+// CHECK: pmullw 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xd5,0x72,0x40]
+pmullw 64(%rdx), %xmm6
+
+// CHECK: pmullw (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xd5,0x32]
+pmullw (%rdx), %xmm6
+
+// CHECK: pmullw %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0xd5,0xf6]
+pmullw %xmm6, %xmm6
+
+// CHECK: pmuludq 485498096, %mm4
+// CHECK: encoding: [0x0f,0xf4,0x24,0x25,0xf0,0x1c,0xf0,0x1c]
+pmuludq 485498096, %mm4
+
+// CHECK: pmuludq 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0xf4,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+pmuludq 485498096, %xmm6
+
+// CHECK: pmuludq 64(%rdx), %mm4
+// CHECK: encoding: [0x0f,0xf4,0x62,0x40]
+pmuludq 64(%rdx), %mm4
+
+// CHECK: pmuludq -64(%rdx,%rax,4), %mm4
+// CHECK: encoding: [0x0f,0xf4,0x64,0x82,0xc0]
+pmuludq -64(%rdx,%rax,4), %mm4
+
+// CHECK: pmuludq 64(%rdx,%rax,4), %mm4
+// CHECK: encoding: [0x0f,0xf4,0x64,0x82,0x40]
+pmuludq 64(%rdx,%rax,4), %mm4
+
+// CHECK: pmuludq -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xf4,0x74,0x82,0xc0]
+pmuludq -64(%rdx,%rax,4), %xmm6
+
+// CHECK: pmuludq 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xf4,0x74,0x82,0x40]
+pmuludq 64(%rdx,%rax,4), %xmm6
+
+// CHECK: pmuludq 64(%rdx,%rax), %mm4
+// CHECK: encoding: [0x0f,0xf4,0x64,0x02,0x40]
+pmuludq 64(%rdx,%rax), %mm4
+
+// CHECK: pmuludq 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xf4,0x74,0x02,0x40]
+pmuludq 64(%rdx,%rax), %xmm6
+
+// CHECK: pmuludq 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xf4,0x72,0x40]
+pmuludq 64(%rdx), %xmm6
+
+// CHECK: pmuludq %mm4, %mm4
+// CHECK: encoding: [0x0f,0xf4,0xe4]
+pmuludq %mm4, %mm4
+
+// CHECK: pmuludq (%rdx), %mm4
+// CHECK: encoding: [0x0f,0xf4,0x22]
+pmuludq (%rdx), %mm4
+
+// CHECK: pmuludq (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xf4,0x32]
+pmuludq (%rdx), %xmm6
+
+// CHECK: pmuludq %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0xf4,0xf6]
+pmuludq %xmm6, %xmm6
+
+// CHECK: por 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0xeb,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+por 485498096, %xmm6
+
+// CHECK: por -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xeb,0x74,0x82,0xc0]
+por -64(%rdx,%rax,4), %xmm6
+
+// CHECK: por 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xeb,0x74,0x82,0x40]
+por 64(%rdx,%rax,4), %xmm6
+
+// CHECK: por 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xeb,0x74,0x02,0x40]
+por 64(%rdx,%rax), %xmm6
+
+// CHECK: por 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xeb,0x72,0x40]
+por 64(%rdx), %xmm6
+
+// CHECK: por (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xeb,0x32]
+por (%rdx), %xmm6
+
+// CHECK: por %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0xeb,0xf6]
+por %xmm6, %xmm6
+
+// CHECK: psadbw 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0xf6,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+psadbw 485498096, %xmm6
+
+// CHECK: psadbw -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xf6,0x74,0x82,0xc0]
+psadbw -64(%rdx,%rax,4), %xmm6
+
+// CHECK: psadbw 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xf6,0x74,0x82,0x40]
+psadbw 64(%rdx,%rax,4), %xmm6
+
+// CHECK: psadbw 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xf6,0x74,0x02,0x40]
+psadbw 64(%rdx,%rax), %xmm6
+
+// CHECK: psadbw 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xf6,0x72,0x40]
+psadbw 64(%rdx), %xmm6
+
+// CHECK: psadbw (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xf6,0x32]
+psadbw (%rdx), %xmm6
+
+// CHECK: psadbw %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0xf6,0xf6]
+psadbw %xmm6, %xmm6
+
+// CHECK: pshufd $0, 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x70,0x34,0x25,0xf0,0x1c,0xf0,0x1c,0x00]
+pshufd $0, 485498096, %xmm6
+
+// CHECK: pshufd $0, -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x70,0x74,0x82,0xc0,0x00]
+pshufd $0, -64(%rdx,%rax,4), %xmm6
+
+// CHECK: pshufd $0, 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x70,0x74,0x82,0x40,0x00]
+pshufd $0, 64(%rdx,%rax,4), %xmm6
+
+// CHECK: pshufd $0, 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x70,0x74,0x02,0x40,0x00]
+pshufd $0, 64(%rdx,%rax), %xmm6
+
+// CHECK: pshufd $0, 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x70,0x72,0x40,0x00]
+pshufd $0, 64(%rdx), %xmm6
+
+// CHECK: pshufd $0, (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x70,0x32,0x00]
+pshufd $0, (%rdx), %xmm6
+
+// CHECK: pshufd $0, %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x70,0xf6,0x00]
+pshufd $0, %xmm6, %xmm6
+
+// CHECK: pshufhw $0, 485498096, %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x70,0x34,0x25,0xf0,0x1c,0xf0,0x1c,0x00]
+pshufhw $0, 485498096, %xmm6
+
+// CHECK: pshufhw $0, -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x70,0x74,0x82,0xc0,0x00]
+pshufhw $0, -64(%rdx,%rax,4), %xmm6
+
+// CHECK: pshufhw $0, 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x70,0x74,0x82,0x40,0x00]
+pshufhw $0, 64(%rdx,%rax,4), %xmm6
+
+// CHECK: pshufhw $0, 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x70,0x74,0x02,0x40,0x00]
+pshufhw $0, 64(%rdx,%rax), %xmm6
+
+// CHECK: pshufhw $0, 64(%rdx), %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x70,0x72,0x40,0x00]
+pshufhw $0, 64(%rdx), %xmm6
+
+// CHECK: pshufhw $0, (%rdx), %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x70,0x32,0x00]
+pshufhw $0, (%rdx), %xmm6
+
+// CHECK: pshufhw $0, %xmm6, %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x70,0xf6,0x00]
+pshufhw $0, %xmm6, %xmm6
+
+// CHECK: pshuflw $0, 485498096, %xmm6
+// CHECK: encoding: [0xf2,0x0f,0x70,0x34,0x25,0xf0,0x1c,0xf0,0x1c,0x00]
+pshuflw $0, 485498096, %xmm6
+
+// CHECK: pshuflw $0, -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0xf2,0x0f,0x70,0x74,0x82,0xc0,0x00]
+pshuflw $0, -64(%rdx,%rax,4), %xmm6
+
+// CHECK: pshuflw $0, 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0xf2,0x0f,0x70,0x74,0x82,0x40,0x00]
+pshuflw $0, 64(%rdx,%rax,4), %xmm6
+
+// CHECK: pshuflw $0, 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0xf2,0x0f,0x70,0x74,0x02,0x40,0x00]
+pshuflw $0, 64(%rdx,%rax), %xmm6
+
+// CHECK: pshuflw $0, 64(%rdx), %xmm6
+// CHECK: encoding: [0xf2,0x0f,0x70,0x72,0x40,0x00]
+pshuflw $0, 64(%rdx), %xmm6
+
+// CHECK: pshuflw $0, (%rdx), %xmm6
+// CHECK: encoding: [0xf2,0x0f,0x70,0x32,0x00]
+pshuflw $0, (%rdx), %xmm6
+
+// CHECK: pshuflw $0, %xmm6, %xmm6
+// CHECK: encoding: [0xf2,0x0f,0x70,0xf6,0x00]
+pshuflw $0, %xmm6, %xmm6
+
+// CHECK: pslld $0, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x72,0xf6,0x00]
+pslld $0, %xmm6
+
+// CHECK: pslld 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0xf2,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+pslld 485498096, %xmm6
+
+// CHECK: pslld -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xf2,0x74,0x82,0xc0]
+pslld -64(%rdx,%rax,4), %xmm6
+
+// CHECK: pslld 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xf2,0x74,0x82,0x40]
+pslld 64(%rdx,%rax,4), %xmm6
+
+// CHECK: pslld 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xf2,0x74,0x02,0x40]
+pslld 64(%rdx,%rax), %xmm6
+
+// CHECK: pslld 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xf2,0x72,0x40]
+pslld 64(%rdx), %xmm6
+
+// CHECK: pslldq $0, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x73,0xfe,0x00]
+pslldq $0, %xmm6
+
+// CHECK: pslld (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xf2,0x32]
+pslld (%rdx), %xmm6
+
+// CHECK: pslld %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0xf2,0xf6]
+pslld %xmm6, %xmm6
+
+// CHECK: psllq $0, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x73,0xf6,0x00]
+psllq $0, %xmm6
+
+// CHECK: psllq 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0xf3,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+psllq 485498096, %xmm6
+
+// CHECK: psllq -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xf3,0x74,0x82,0xc0]
+psllq -64(%rdx,%rax,4), %xmm6
+
+// CHECK: psllq 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xf3,0x74,0x82,0x40]
+psllq 64(%rdx,%rax,4), %xmm6
+
+// CHECK: psllq 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xf3,0x74,0x02,0x40]
+psllq 64(%rdx,%rax), %xmm6
+
+// CHECK: psllq 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xf3,0x72,0x40]
+psllq 64(%rdx), %xmm6
+
+// CHECK: psllq (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xf3,0x32]
+psllq (%rdx), %xmm6
+
+// CHECK: psllq %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0xf3,0xf6]
+psllq %xmm6, %xmm6
+
+// CHECK: psllw $0, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x71,0xf6,0x00]
+psllw $0, %xmm6
+
+// CHECK: psllw 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0xf1,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+psllw 485498096, %xmm6
+
+// CHECK: psllw -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xf1,0x74,0x82,0xc0]
+psllw -64(%rdx,%rax,4), %xmm6
+
+// CHECK: psllw 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xf1,0x74,0x82,0x40]
+psllw 64(%rdx,%rax,4), %xmm6
+
+// CHECK: psllw 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xf1,0x74,0x02,0x40]
+psllw 64(%rdx,%rax), %xmm6
+
+// CHECK: psllw 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xf1,0x72,0x40]
+psllw 64(%rdx), %xmm6
+
+// CHECK: psllw (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xf1,0x32]
+psllw (%rdx), %xmm6
+
+// CHECK: psllw %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0xf1,0xf6]
+psllw %xmm6, %xmm6
+
+// CHECK: psrad $0, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x72,0xe6,0x00]
+psrad $0, %xmm6
+
+// CHECK: psrad 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0xe2,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+psrad 485498096, %xmm6
+
+// CHECK: psrad -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xe2,0x74,0x82,0xc0]
+psrad -64(%rdx,%rax,4), %xmm6
+
+// CHECK: psrad 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xe2,0x74,0x82,0x40]
+psrad 64(%rdx,%rax,4), %xmm6
+
+// CHECK: psrad 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xe2,0x74,0x02,0x40]
+psrad 64(%rdx,%rax), %xmm6
+
+// CHECK: psrad 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xe2,0x72,0x40]
+psrad 64(%rdx), %xmm6
+
+// CHECK: psrad (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xe2,0x32]
+psrad (%rdx), %xmm6
+
+// CHECK: psrad %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0xe2,0xf6]
+psrad %xmm6, %xmm6
+
+// CHECK: psraw $0, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x71,0xe6,0x00]
+psraw $0, %xmm6
+
+// CHECK: psraw 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0xe1,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+psraw 485498096, %xmm6
+
+// CHECK: psraw -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xe1,0x74,0x82,0xc0]
+psraw -64(%rdx,%rax,4), %xmm6
+
+// CHECK: psraw 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xe1,0x74,0x82,0x40]
+psraw 64(%rdx,%rax,4), %xmm6
+
+// CHECK: psraw 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xe1,0x74,0x02,0x40]
+psraw 64(%rdx,%rax), %xmm6
+
+// CHECK: psraw 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xe1,0x72,0x40]
+psraw 64(%rdx), %xmm6
+
+// CHECK: psraw (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xe1,0x32]
+psraw (%rdx), %xmm6
+
+// CHECK: psraw %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0xe1,0xf6]
+psraw %xmm6, %xmm6
+
+// CHECK: psrld $0, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x72,0xd6,0x00]
+psrld $0, %xmm6
+
+// CHECK: psrld 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0xd2,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+psrld 485498096, %xmm6
+
+// CHECK: psrld -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xd2,0x74,0x82,0xc0]
+psrld -64(%rdx,%rax,4), %xmm6
+
+// CHECK: psrld 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xd2,0x74,0x82,0x40]
+psrld 64(%rdx,%rax,4), %xmm6
+
+// CHECK: psrld 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xd2,0x74,0x02,0x40]
+psrld 64(%rdx,%rax), %xmm6
+
+// CHECK: psrld 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xd2,0x72,0x40]
+psrld 64(%rdx), %xmm6
+
+// CHECK: psrldq $0, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x73,0xde,0x00]
+psrldq $0, %xmm6
+
+// CHECK: psrld (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xd2,0x32]
+psrld (%rdx), %xmm6
+
+// CHECK: psrld %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0xd2,0xf6]
+psrld %xmm6, %xmm6
+
+// CHECK: psrlq $0, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x73,0xd6,0x00]
+psrlq $0, %xmm6
+
+// CHECK: psrlq 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0xd3,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+psrlq 485498096, %xmm6
+
+// CHECK: psrlq -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xd3,0x74,0x82,0xc0]
+psrlq -64(%rdx,%rax,4), %xmm6
+
+// CHECK: psrlq 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xd3,0x74,0x82,0x40]
+psrlq 64(%rdx,%rax,4), %xmm6
+
+// CHECK: psrlq 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xd3,0x74,0x02,0x40]
+psrlq 64(%rdx,%rax), %xmm6
+
+// CHECK: psrlq 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xd3,0x72,0x40]
+psrlq 64(%rdx), %xmm6
+
+// CHECK: psrlq (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xd3,0x32]
+psrlq (%rdx), %xmm6
+
+// CHECK: psrlq %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0xd3,0xf6]
+psrlq %xmm6, %xmm6
+
+// CHECK: psrlw $0, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x71,0xd6,0x00]
+psrlw $0, %xmm6
+
+// CHECK: psrlw 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0xd1,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+psrlw 485498096, %xmm6
+
+// CHECK: psrlw -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xd1,0x74,0x82,0xc0]
+psrlw -64(%rdx,%rax,4), %xmm6
+
+// CHECK: psrlw 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xd1,0x74,0x82,0x40]
+psrlw 64(%rdx,%rax,4), %xmm6
+
+// CHECK: psrlw 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xd1,0x74,0x02,0x40]
+psrlw 64(%rdx,%rax), %xmm6
+
+// CHECK: psrlw 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xd1,0x72,0x40]
+psrlw 64(%rdx), %xmm6
+
+// CHECK: psrlw (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xd1,0x32]
+psrlw (%rdx), %xmm6
+
+// CHECK: psrlw %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0xd1,0xf6]
+psrlw %xmm6, %xmm6
+
+// CHECK: psubb 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0xf8,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+psubb 485498096, %xmm6
+
+// CHECK: psubb -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xf8,0x74,0x82,0xc0]
+psubb -64(%rdx,%rax,4), %xmm6
+
+// CHECK: psubb 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xf8,0x74,0x82,0x40]
+psubb 64(%rdx,%rax,4), %xmm6
+
+// CHECK: psubb 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xf8,0x74,0x02,0x40]
+psubb 64(%rdx,%rax), %xmm6
+
+// CHECK: psubb 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xf8,0x72,0x40]
+psubb 64(%rdx), %xmm6
+
+// CHECK: psubb (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xf8,0x32]
+psubb (%rdx), %xmm6
+
+// CHECK: psubb %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0xf8,0xf6]
+psubb %xmm6, %xmm6
+
+// CHECK: psubd 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0xfa,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+psubd 485498096, %xmm6
+
+// CHECK: psubd -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xfa,0x74,0x82,0xc0]
+psubd -64(%rdx,%rax,4), %xmm6
+
+// CHECK: psubd 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xfa,0x74,0x82,0x40]
+psubd 64(%rdx,%rax,4), %xmm6
+
+// CHECK: psubd 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xfa,0x74,0x02,0x40]
+psubd 64(%rdx,%rax), %xmm6
+
+// CHECK: psubd 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xfa,0x72,0x40]
+psubd 64(%rdx), %xmm6
+
+// CHECK: psubd (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xfa,0x32]
+psubd (%rdx), %xmm6
+
+// CHECK: psubd %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0xfa,0xf6]
+psubd %xmm6, %xmm6
+
+// CHECK: psubq 485498096, %mm4
+// CHECK: encoding: [0x0f,0xfb,0x24,0x25,0xf0,0x1c,0xf0,0x1c]
+psubq 485498096, %mm4
+
+// CHECK: psubq 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0xfb,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+psubq 485498096, %xmm6
+
+// CHECK: psubq 64(%rdx), %mm4
+// CHECK: encoding: [0x0f,0xfb,0x62,0x40]
+psubq 64(%rdx), %mm4
+
+// CHECK: psubq -64(%rdx,%rax,4), %mm4
+// CHECK: encoding: [0x0f,0xfb,0x64,0x82,0xc0]
+psubq -64(%rdx,%rax,4), %mm4
+
+// CHECK: psubq 64(%rdx,%rax,4), %mm4
+// CHECK: encoding: [0x0f,0xfb,0x64,0x82,0x40]
+psubq 64(%rdx,%rax,4), %mm4
+
+// CHECK: psubq -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xfb,0x74,0x82,0xc0]
+psubq -64(%rdx,%rax,4), %xmm6
+
+// CHECK: psubq 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xfb,0x74,0x82,0x40]
+psubq 64(%rdx,%rax,4), %xmm6
+
+// CHECK: psubq 64(%rdx,%rax), %mm4
+// CHECK: encoding: [0x0f,0xfb,0x64,0x02,0x40]
+psubq 64(%rdx,%rax), %mm4
+
+// CHECK: psubq 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xfb,0x74,0x02,0x40]
+psubq 64(%rdx,%rax), %xmm6
+
+// CHECK: psubq 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xfb,0x72,0x40]
+psubq 64(%rdx), %xmm6
+
+// CHECK: psubq %mm4, %mm4
+// CHECK: encoding: [0x0f,0xfb,0xe4]
+psubq %mm4, %mm4
+
+// CHECK: psubq (%rdx), %mm4
+// CHECK: encoding: [0x0f,0xfb,0x22]
+psubq (%rdx), %mm4
+
+// CHECK: psubq (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xfb,0x32]
+psubq (%rdx), %xmm6
+
+// CHECK: psubq %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0xfb,0xf6]
+psubq %xmm6, %xmm6
+
+// CHECK: psubsb 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0xe8,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+psubsb 485498096, %xmm6
+
+// CHECK: psubsb -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xe8,0x74,0x82,0xc0]
+psubsb -64(%rdx,%rax,4), %xmm6
+
+// CHECK: psubsb 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xe8,0x74,0x82,0x40]
+psubsb 64(%rdx,%rax,4), %xmm6
+
+// CHECK: psubsb 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xe8,0x74,0x02,0x40]
+psubsb 64(%rdx,%rax), %xmm6
+
+// CHECK: psubsb 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xe8,0x72,0x40]
+psubsb 64(%rdx), %xmm6
+
+// CHECK: psubsb (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xe8,0x32]
+psubsb (%rdx), %xmm6
+
+// CHECK: psubsb %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0xe8,0xf6]
+psubsb %xmm6, %xmm6
+
+// CHECK: psubsw 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0xe9,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+psubsw 485498096, %xmm6
+
+// CHECK: psubsw -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xe9,0x74,0x82,0xc0]
+psubsw -64(%rdx,%rax,4), %xmm6
+
+// CHECK: psubsw 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xe9,0x74,0x82,0x40]
+psubsw 64(%rdx,%rax,4), %xmm6
+
+// CHECK: psubsw 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xe9,0x74,0x02,0x40]
+psubsw 64(%rdx,%rax), %xmm6
+
+// CHECK: psubsw 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xe9,0x72,0x40]
+psubsw 64(%rdx), %xmm6
+
+// CHECK: psubsw (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xe9,0x32]
+psubsw (%rdx), %xmm6
+
+// CHECK: psubsw %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0xe9,0xf6]
+psubsw %xmm6, %xmm6
+
+// CHECK: psubusb 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0xd8,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+psubusb 485498096, %xmm6
+
+// CHECK: psubusb -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xd8,0x74,0x82,0xc0]
+psubusb -64(%rdx,%rax,4), %xmm6
+
+// CHECK: psubusb 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xd8,0x74,0x82,0x40]
+psubusb 64(%rdx,%rax,4), %xmm6
+
+// CHECK: psubusb 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xd8,0x74,0x02,0x40]
+psubusb 64(%rdx,%rax), %xmm6
+
+// CHECK: psubusb 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xd8,0x72,0x40]
+psubusb 64(%rdx), %xmm6
+
+// CHECK: psubusb (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xd8,0x32]
+psubusb (%rdx), %xmm6
+
+// CHECK: psubusb %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0xd8,0xf6]
+psubusb %xmm6, %xmm6
+
+// CHECK: psubusw 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0xd9,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+psubusw 485498096, %xmm6
+
+// CHECK: psubusw -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xd9,0x74,0x82,0xc0]
+psubusw -64(%rdx,%rax,4), %xmm6
+
+// CHECK: psubusw 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xd9,0x74,0x82,0x40]
+psubusw 64(%rdx,%rax,4), %xmm6
+
+// CHECK: psubusw 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xd9,0x74,0x02,0x40]
+psubusw 64(%rdx,%rax), %xmm6
+
+// CHECK: psubusw 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xd9,0x72,0x40]
+psubusw 64(%rdx), %xmm6
+
+// CHECK: psubusw (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xd9,0x32]
+psubusw (%rdx), %xmm6
+
+// CHECK: psubusw %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0xd9,0xf6]
+psubusw %xmm6, %xmm6
+
+// CHECK: psubw 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0xf9,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+psubw 485498096, %xmm6
+
+// CHECK: psubw -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xf9,0x74,0x82,0xc0]
+psubw -64(%rdx,%rax,4), %xmm6
+
+// CHECK: psubw 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xf9,0x74,0x82,0x40]
+psubw 64(%rdx,%rax,4), %xmm6
+
+// CHECK: psubw 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xf9,0x74,0x02,0x40]
+psubw 64(%rdx,%rax), %xmm6
+
+// CHECK: psubw 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xf9,0x72,0x40]
+psubw 64(%rdx), %xmm6
+
+// CHECK: psubw (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xf9,0x32]
+psubw (%rdx), %xmm6
+
+// CHECK: psubw %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0xf9,0xf6]
+psubw %xmm6, %xmm6
+
+// CHECK: punpckhbw 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x68,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+punpckhbw 485498096, %xmm6
+
+// CHECK: punpckhbw -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x68,0x74,0x82,0xc0]
+punpckhbw -64(%rdx,%rax,4), %xmm6
+
+// CHECK: punpckhbw 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x68,0x74,0x82,0x40]
+punpckhbw 64(%rdx,%rax,4), %xmm6
+
+// CHECK: punpckhbw 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x68,0x74,0x02,0x40]
+punpckhbw 64(%rdx,%rax), %xmm6
+
+// CHECK: punpckhbw 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x68,0x72,0x40]
+punpckhbw 64(%rdx), %xmm6
+
+// CHECK: punpckhbw (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x68,0x32]
+punpckhbw (%rdx), %xmm6
+
+// CHECK: punpckhbw %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x68,0xf6]
+punpckhbw %xmm6, %xmm6
+
+// CHECK: punpckhdq 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x6a,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+punpckhdq 485498096, %xmm6
+
+// CHECK: punpckhdq -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x6a,0x74,0x82,0xc0]
+punpckhdq -64(%rdx,%rax,4), %xmm6
+
+// CHECK: punpckhdq 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x6a,0x74,0x82,0x40]
+punpckhdq 64(%rdx,%rax,4), %xmm6
+
+// CHECK: punpckhdq 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x6a,0x74,0x02,0x40]
+punpckhdq 64(%rdx,%rax), %xmm6
+
+// CHECK: punpckhdq 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x6a,0x72,0x40]
+punpckhdq 64(%rdx), %xmm6
+
+// CHECK: punpckhdq (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x6a,0x32]
+punpckhdq (%rdx), %xmm6
+
+// CHECK: punpckhdq %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x6a,0xf6]
+punpckhdq %xmm6, %xmm6
+
+// CHECK: punpckhqdq 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x6d,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+punpckhqdq 485498096, %xmm6
+
+// CHECK: punpckhqdq -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x6d,0x74,0x82,0xc0]
+punpckhqdq -64(%rdx,%rax,4), %xmm6
+
+// CHECK: punpckhqdq 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x6d,0x74,0x82,0x40]
+punpckhqdq 64(%rdx,%rax,4), %xmm6
+
+// CHECK: punpckhqdq 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x6d,0x74,0x02,0x40]
+punpckhqdq 64(%rdx,%rax), %xmm6
+
+// CHECK: punpckhqdq 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x6d,0x72,0x40]
+punpckhqdq 64(%rdx), %xmm6
+
+// CHECK: punpckhqdq (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x6d,0x32]
+punpckhqdq (%rdx), %xmm6
+
+// CHECK: punpckhqdq %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x6d,0xf6]
+punpckhqdq %xmm6, %xmm6
+
+// CHECK: punpckhwd 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x69,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+punpckhwd 485498096, %xmm6
+
+// CHECK: punpckhwd -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x69,0x74,0x82,0xc0]
+punpckhwd -64(%rdx,%rax,4), %xmm6
+
+// CHECK: punpckhwd 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x69,0x74,0x82,0x40]
+punpckhwd 64(%rdx,%rax,4), %xmm6
+
+// CHECK: punpckhwd 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x69,0x74,0x02,0x40]
+punpckhwd 64(%rdx,%rax), %xmm6
+
+// CHECK: punpckhwd 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x69,0x72,0x40]
+punpckhwd 64(%rdx), %xmm6
+
+// CHECK: punpckhwd (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x69,0x32]
+punpckhwd (%rdx), %xmm6
+
+// CHECK: punpckhwd %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x69,0xf6]
+punpckhwd %xmm6, %xmm6
+
+// CHECK: punpcklbw 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x60,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+punpcklbw 485498096, %xmm6
+
+// CHECK: punpcklbw -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x60,0x74,0x82,0xc0]
+punpcklbw -64(%rdx,%rax,4), %xmm6
+
+// CHECK: punpcklbw 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x60,0x74,0x82,0x40]
+punpcklbw 64(%rdx,%rax,4), %xmm6
+
+// CHECK: punpcklbw 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x60,0x74,0x02,0x40]
+punpcklbw 64(%rdx,%rax), %xmm6
+
+// CHECK: punpcklbw 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x60,0x72,0x40]
+punpcklbw 64(%rdx), %xmm6
+
+// CHECK: punpcklbw (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x60,0x32]
+punpcklbw (%rdx), %xmm6
+
+// CHECK: punpcklbw %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x60,0xf6]
+punpcklbw %xmm6, %xmm6
+
+// CHECK: punpckldq 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x62,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+punpckldq 485498096, %xmm6
+
+// CHECK: punpckldq -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x62,0x74,0x82,0xc0]
+punpckldq -64(%rdx,%rax,4), %xmm6
+
+// CHECK: punpckldq 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x62,0x74,0x82,0x40]
+punpckldq 64(%rdx,%rax,4), %xmm6
+
+// CHECK: punpckldq 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x62,0x74,0x02,0x40]
+punpckldq 64(%rdx,%rax), %xmm6
+
+// CHECK: punpckldq 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x62,0x72,0x40]
+punpckldq 64(%rdx), %xmm6
+
+// CHECK: punpckldq (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x62,0x32]
+punpckldq (%rdx), %xmm6
+
+// CHECK: punpckldq %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x62,0xf6]
+punpckldq %xmm6, %xmm6
+
+// CHECK: punpcklqdq 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x6c,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+punpcklqdq 485498096, %xmm6
+
+// CHECK: punpcklqdq -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x6c,0x74,0x82,0xc0]
+punpcklqdq -64(%rdx,%rax,4), %xmm6
+
+// CHECK: punpcklqdq 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x6c,0x74,0x82,0x40]
+punpcklqdq 64(%rdx,%rax,4), %xmm6
+
+// CHECK: punpcklqdq 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x6c,0x74,0x02,0x40]
+punpcklqdq 64(%rdx,%rax), %xmm6
+
+// CHECK: punpcklqdq 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x6c,0x72,0x40]
+punpcklqdq 64(%rdx), %xmm6
+
+// CHECK: punpcklqdq (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x6c,0x32]
+punpcklqdq (%rdx), %xmm6
+
+// CHECK: punpcklqdq %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x6c,0xf6]
+punpcklqdq %xmm6, %xmm6
+
+// CHECK: punpcklwd 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x61,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+punpcklwd 485498096, %xmm6
+
+// CHECK: punpcklwd -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x61,0x74,0x82,0xc0]
+punpcklwd -64(%rdx,%rax,4), %xmm6
+
+// CHECK: punpcklwd 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x61,0x74,0x82,0x40]
+punpcklwd 64(%rdx,%rax,4), %xmm6
+
+// CHECK: punpcklwd 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x61,0x74,0x02,0x40]
+punpcklwd 64(%rdx,%rax), %xmm6
+
+// CHECK: punpcklwd 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x61,0x72,0x40]
+punpcklwd 64(%rdx), %xmm6
+
+// CHECK: punpcklwd (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x61,0x32]
+punpcklwd (%rdx), %xmm6
+
+// CHECK: punpcklwd %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x61,0xf6]
+punpcklwd %xmm6, %xmm6
+
+// CHECK: pxor 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0xef,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+pxor 485498096, %xmm6
+
+// CHECK: pxor -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xef,0x74,0x82,0xc0]
+pxor -64(%rdx,%rax,4), %xmm6
+
+// CHECK: pxor 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xef,0x74,0x82,0x40]
+pxor 64(%rdx,%rax,4), %xmm6
+
+// CHECK: pxor 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xef,0x74,0x02,0x40]
+pxor 64(%rdx,%rax), %xmm6
+
+// CHECK: pxor 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xef,0x72,0x40]
+pxor 64(%rdx), %xmm6
+
+// CHECK: pxor (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xef,0x32]
+pxor (%rdx), %xmm6
+
+// CHECK: pxor %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0xef,0xf6]
+pxor %xmm6, %xmm6
+
+// CHECK: shufpd $0, 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0xc6,0x34,0x25,0xf0,0x1c,0xf0,0x1c,0x00]
+shufpd $0, 485498096, %xmm6
+
+// CHECK: shufpd $0, -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xc6,0x74,0x82,0xc0,0x00]
+shufpd $0, -64(%rdx,%rax,4), %xmm6
+
+// CHECK: shufpd $0, 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xc6,0x74,0x82,0x40,0x00]
+shufpd $0, 64(%rdx,%rax,4), %xmm6
+
+// CHECK: shufpd $0, 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xc6,0x74,0x02,0x40,0x00]
+shufpd $0, 64(%rdx,%rax), %xmm6
+
+// CHECK: shufpd $0, 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xc6,0x72,0x40,0x00]
+shufpd $0, 64(%rdx), %xmm6
+
+// CHECK: shufpd $0, (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xc6,0x32,0x00]
+shufpd $0, (%rdx), %xmm6
+
+// CHECK: shufpd $0, %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0xc6,0xf6,0x00]
+shufpd $0, %xmm6, %xmm6
+
+// CHECK: sqrtpd 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x51,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+sqrtpd 485498096, %xmm6
+
+// CHECK: sqrtpd -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x51,0x74,0x82,0xc0]
+sqrtpd -64(%rdx,%rax,4), %xmm6
+
+// CHECK: sqrtpd 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x51,0x74,0x82,0x40]
+sqrtpd 64(%rdx,%rax,4), %xmm6
+
+// CHECK: sqrtpd 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x51,0x74,0x02,0x40]
+sqrtpd 64(%rdx,%rax), %xmm6
+
+// CHECK: sqrtpd 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x51,0x72,0x40]
+sqrtpd 64(%rdx), %xmm6
+
+// CHECK: sqrtpd (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x51,0x32]
+sqrtpd (%rdx), %xmm6
+
+// CHECK: sqrtpd %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x51,0xf6]
+sqrtpd %xmm6, %xmm6
+
+// CHECK: sqrtsd 485498096, %xmm6
+// CHECK: encoding: [0xf2,0x0f,0x51,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+sqrtsd 485498096, %xmm6
+
+// CHECK: sqrtsd -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0xf2,0x0f,0x51,0x74,0x82,0xc0]
+sqrtsd -64(%rdx,%rax,4), %xmm6
+
+// CHECK: sqrtsd 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0xf2,0x0f,0x51,0x74,0x82,0x40]
+sqrtsd 64(%rdx,%rax,4), %xmm6
+
+// CHECK: sqrtsd 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0xf2,0x0f,0x51,0x74,0x02,0x40]
+sqrtsd 64(%rdx,%rax), %xmm6
+
+// CHECK: sqrtsd 64(%rdx), %xmm6
+// CHECK: encoding: [0xf2,0x0f,0x51,0x72,0x40]
+sqrtsd 64(%rdx), %xmm6
+
+// CHECK: sqrtsd (%rdx), %xmm6
+// CHECK: encoding: [0xf2,0x0f,0x51,0x32]
+sqrtsd (%rdx), %xmm6
+
+// CHECK: sqrtsd %xmm6, %xmm6
+// CHECK: encoding: [0xf2,0x0f,0x51,0xf6]
+sqrtsd %xmm6, %xmm6
+
+// CHECK: subpd 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x5c,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+subpd 485498096, %xmm6
+
+// CHECK: subpd -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x5c,0x74,0x82,0xc0]
+subpd -64(%rdx,%rax,4), %xmm6
+
+// CHECK: subpd 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x5c,0x74,0x82,0x40]
+subpd 64(%rdx,%rax,4), %xmm6
+
+// CHECK: subpd 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x5c,0x74,0x02,0x40]
+subpd 64(%rdx,%rax), %xmm6
+
+// CHECK: subpd 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x5c,0x72,0x40]
+subpd 64(%rdx), %xmm6
+
+// CHECK: subpd (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x5c,0x32]
+subpd (%rdx), %xmm6
+
+// CHECK: subpd %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x5c,0xf6]
+subpd %xmm6, %xmm6
+
+// CHECK: subsd 485498096, %xmm6
+// CHECK: encoding: [0xf2,0x0f,0x5c,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+subsd 485498096, %xmm6
+
+// CHECK: subsd -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0xf2,0x0f,0x5c,0x74,0x82,0xc0]
+subsd -64(%rdx,%rax,4), %xmm6
+
+// CHECK: subsd 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0xf2,0x0f,0x5c,0x74,0x82,0x40]
+subsd 64(%rdx,%rax,4), %xmm6
+
+// CHECK: subsd 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0xf2,0x0f,0x5c,0x74,0x02,0x40]
+subsd 64(%rdx,%rax), %xmm6
+
+// CHECK: subsd 64(%rdx), %xmm6
+// CHECK: encoding: [0xf2,0x0f,0x5c,0x72,0x40]
+subsd 64(%rdx), %xmm6
+
+// CHECK: subsd (%rdx), %xmm6
+// CHECK: encoding: [0xf2,0x0f,0x5c,0x32]
+subsd (%rdx), %xmm6
+
+// CHECK: subsd %xmm6, %xmm6
+// CHECK: encoding: [0xf2,0x0f,0x5c,0xf6]
+subsd %xmm6, %xmm6
+
+// CHECK: ucomisd 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x2e,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+ucomisd 485498096, %xmm6
+
+// CHECK: ucomisd -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x2e,0x74,0x82,0xc0]
+ucomisd -64(%rdx,%rax,4), %xmm6
+
+// CHECK: ucomisd 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x2e,0x74,0x82,0x40]
+ucomisd 64(%rdx,%rax,4), %xmm6
+
+// CHECK: ucomisd 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x2e,0x74,0x02,0x40]
+ucomisd 64(%rdx,%rax), %xmm6
+
+// CHECK: ucomisd 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x2e,0x72,0x40]
+ucomisd 64(%rdx), %xmm6
+
+// CHECK: ucomisd (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x2e,0x32]
+ucomisd (%rdx), %xmm6
+
+// CHECK: ucomisd %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x2e,0xf6]
+ucomisd %xmm6, %xmm6
+
+// CHECK: unpckhpd 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x15,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+unpckhpd 485498096, %xmm6
+
+// CHECK: unpckhpd -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x15,0x74,0x82,0xc0]
+unpckhpd -64(%rdx,%rax,4), %xmm6
+
+// CHECK: unpckhpd 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x15,0x74,0x82,0x40]
+unpckhpd 64(%rdx,%rax,4), %xmm6
+
+// CHECK: unpckhpd 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x15,0x74,0x02,0x40]
+unpckhpd 64(%rdx,%rax), %xmm6
+
+// CHECK: unpckhpd 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x15,0x72,0x40]
+unpckhpd 64(%rdx), %xmm6
+
+// CHECK: unpckhpd (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x15,0x32]
+unpckhpd (%rdx), %xmm6
+
+// CHECK: unpckhpd %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x15,0xf6]
+unpckhpd %xmm6, %xmm6
+
+// CHECK: unpcklpd 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x14,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+unpcklpd 485498096, %xmm6
+
+// CHECK: unpcklpd -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x14,0x74,0x82,0xc0]
+unpcklpd -64(%rdx,%rax,4), %xmm6
+
+// CHECK: unpcklpd 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x14,0x74,0x82,0x40]
+unpcklpd 64(%rdx,%rax,4), %xmm6
+
+// CHECK: unpcklpd 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x14,0x74,0x02,0x40]
+unpcklpd 64(%rdx,%rax), %xmm6
+
+// CHECK: unpcklpd 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x14,0x72,0x40]
+unpcklpd 64(%rdx), %xmm6
+
+// CHECK: unpcklpd (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x14,0x32]
+unpcklpd (%rdx), %xmm6
+
+// CHECK: unpcklpd %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x14,0xf6]
+unpcklpd %xmm6, %xmm6
+
+// CHECK: xorpd 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x57,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+xorpd 485498096, %xmm6
+
+// CHECK: xorpd -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x57,0x74,0x82,0xc0]
+xorpd -64(%rdx,%rax,4), %xmm6
+
+// CHECK: xorpd 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x57,0x74,0x82,0x40]
+xorpd 64(%rdx,%rax,4), %xmm6
+
+// CHECK: xorpd 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x57,0x74,0x02,0x40]
+xorpd 64(%rdx,%rax), %xmm6
+
+// CHECK: xorpd 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x57,0x72,0x40]
+xorpd 64(%rdx), %xmm6
+
+// CHECK: xorpd (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x57,0x32]
+xorpd (%rdx), %xmm6
+
+// CHECK: xorpd %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x57,0xf6]
+xorpd %xmm6, %xmm6
+
diff --git a/test/MC/X86/SSE3-32.s b/test/MC/X86/SSE3-32.s
new file mode 100644
index 0000000000000000000000000000000000000000..2fe639978b85f267e3ffb682bfc3266a70bc42ad
--- /dev/null
+++ b/test/MC/X86/SSE3-32.s
@@ -0,0 +1,358 @@
+// RUN: llvm-mc -triple i386-unknown-unknown --show-encoding %s | FileCheck %s
+
+// CHECK: addsubpd -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xd0,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+addsubpd -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: addsubpd 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xd0,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+addsubpd 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: addsubpd 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xd0,0x8a,0xf0,0x1c,0xf0,0x1c]
+addsubpd 485498096(%edx), %xmm1
+
+// CHECK: addsubpd 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0xd0,0x0d,0xf0,0x1c,0xf0,0x1c]
+addsubpd 485498096, %xmm1
+
+// CHECK: addsubpd 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xd0,0x4c,0x02,0x40]
+addsubpd 64(%edx,%eax), %xmm1
+
+// CHECK: addsubpd (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0xd0,0x0a]
+addsubpd (%edx), %xmm1
+
+// CHECK: addsubpd %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0xd0,0xc9]
+addsubpd %xmm1, %xmm1
+
+// CHECK: addsubps -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0xf2,0x0f,0xd0,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+addsubps -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: addsubps 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0xf2,0x0f,0xd0,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+addsubps 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: addsubps 485498096(%edx), %xmm1
+// CHECK: encoding: [0xf2,0x0f,0xd0,0x8a,0xf0,0x1c,0xf0,0x1c]
+addsubps 485498096(%edx), %xmm1
+
+// CHECK: addsubps 485498096, %xmm1
+// CHECK: encoding: [0xf2,0x0f,0xd0,0x0d,0xf0,0x1c,0xf0,0x1c]
+addsubps 485498096, %xmm1
+
+// CHECK: addsubps 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0xf2,0x0f,0xd0,0x4c,0x02,0x40]
+addsubps 64(%edx,%eax), %xmm1
+
+// CHECK: addsubps (%edx), %xmm1
+// CHECK: encoding: [0xf2,0x0f,0xd0,0x0a]
+addsubps (%edx), %xmm1
+
+// CHECK: addsubps %xmm1, %xmm1
+// CHECK: encoding: [0xf2,0x0f,0xd0,0xc9]
+addsubps %xmm1, %xmm1
+
+// CHECK: fisttpl -485498096(%edx,%eax,4)
+// CHECK: encoding: [0xdb,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+fisttpl -485498096(%edx,%eax,4)
+
+// CHECK: fisttpl 485498096(%edx,%eax,4)
+// CHECK: encoding: [0xdb,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+fisttpl 485498096(%edx,%eax,4)
+
+// CHECK: fisttpl 485498096(%edx)
+// CHECK: encoding: [0xdb,0x8a,0xf0,0x1c,0xf0,0x1c]
+fisttpl 485498096(%edx)
+
+// CHECK: fisttpl 485498096
+// CHECK: encoding: [0xdb,0x0d,0xf0,0x1c,0xf0,0x1c]
+fisttpl 485498096
+
+// CHECK: fisttpl 64(%edx,%eax)
+// CHECK: encoding: [0xdb,0x4c,0x02,0x40]
+fisttpl 64(%edx,%eax)
+
+// CHECK: fisttpl (%edx)
+// CHECK: encoding: [0xdb,0x0a]
+fisttpl (%edx)
+
+// CHECK: fisttpll -485498096(%edx,%eax,4)
+// CHECK: encoding: [0xdd,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+fisttpll -485498096(%edx,%eax,4)
+
+// CHECK: fisttpll 485498096(%edx,%eax,4)
+// CHECK: encoding: [0xdd,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+fisttpll 485498096(%edx,%eax,4)
+
+// CHECK: fisttpll 485498096(%edx)
+// CHECK: encoding: [0xdd,0x8a,0xf0,0x1c,0xf0,0x1c]
+fisttpll 485498096(%edx)
+
+// CHECK: fisttpll 485498096
+// CHECK: encoding: [0xdd,0x0d,0xf0,0x1c,0xf0,0x1c]
+fisttpll 485498096
+
+// CHECK: fisttpll 64(%edx,%eax)
+// CHECK: encoding: [0xdd,0x4c,0x02,0x40]
+fisttpll 64(%edx,%eax)
+
+// CHECK: fisttpll (%edx)
+// CHECK: encoding: [0xdd,0x0a]
+fisttpll (%edx)
+
+// CHECK: fisttps -485498096(%edx,%eax,4)
+// CHECK: encoding: [0xdf,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+fisttps -485498096(%edx,%eax,4)
+
+// CHECK: fisttps 485498096(%edx,%eax,4)
+// CHECK: encoding: [0xdf,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+fisttps 485498096(%edx,%eax,4)
+
+// CHECK: fisttps 485498096(%edx)
+// CHECK: encoding: [0xdf,0x8a,0xf0,0x1c,0xf0,0x1c]
+fisttps 485498096(%edx)
+
+// CHECK: fisttps 485498096
+// CHECK: encoding: [0xdf,0x0d,0xf0,0x1c,0xf0,0x1c]
+fisttps 485498096
+
+// CHECK: fisttps 64(%edx,%eax)
+// CHECK: encoding: [0xdf,0x4c,0x02,0x40]
+fisttps 64(%edx,%eax)
+
+// CHECK: fisttps (%edx)
+// CHECK: encoding: [0xdf,0x0a]
+fisttps (%edx)
+
+// CHECK: haddpd -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x7c,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+haddpd -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: haddpd 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x7c,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+haddpd 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: haddpd 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x7c,0x8a,0xf0,0x1c,0xf0,0x1c]
+haddpd 485498096(%edx), %xmm1
+
+// CHECK: haddpd 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x7c,0x0d,0xf0,0x1c,0xf0,0x1c]
+haddpd 485498096, %xmm1
+
+// CHECK: haddpd 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x7c,0x4c,0x02,0x40]
+haddpd 64(%edx,%eax), %xmm1
+
+// CHECK: haddpd (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x7c,0x0a]
+haddpd (%edx), %xmm1
+
+// CHECK: haddpd %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x7c,0xc9]
+haddpd %xmm1, %xmm1
+
+// CHECK: haddps -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0xf2,0x0f,0x7c,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+haddps -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: haddps 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0xf2,0x0f,0x7c,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+haddps 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: haddps 485498096(%edx), %xmm1
+// CHECK: encoding: [0xf2,0x0f,0x7c,0x8a,0xf0,0x1c,0xf0,0x1c]
+haddps 485498096(%edx), %xmm1
+
+// CHECK: haddps 485498096, %xmm1
+// CHECK: encoding: [0xf2,0x0f,0x7c,0x0d,0xf0,0x1c,0xf0,0x1c]
+haddps 485498096, %xmm1
+
+// CHECK: haddps 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0xf2,0x0f,0x7c,0x4c,0x02,0x40]
+haddps 64(%edx,%eax), %xmm1
+
+// CHECK: haddps (%edx), %xmm1
+// CHECK: encoding: [0xf2,0x0f,0x7c,0x0a]
+haddps (%edx), %xmm1
+
+// CHECK: haddps %xmm1, %xmm1
+// CHECK: encoding: [0xf2,0x0f,0x7c,0xc9]
+haddps %xmm1, %xmm1
+
+// CHECK: hsubpd -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x7d,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+hsubpd -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: hsubpd 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x7d,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+hsubpd 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: hsubpd 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x7d,0x8a,0xf0,0x1c,0xf0,0x1c]
+hsubpd 485498096(%edx), %xmm1
+
+// CHECK: hsubpd 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x7d,0x0d,0xf0,0x1c,0xf0,0x1c]
+hsubpd 485498096, %xmm1
+
+// CHECK: hsubpd 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x7d,0x4c,0x02,0x40]
+hsubpd 64(%edx,%eax), %xmm1
+
+// CHECK: hsubpd (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x7d,0x0a]
+hsubpd (%edx), %xmm1
+
+// CHECK: hsubpd %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x7d,0xc9]
+hsubpd %xmm1, %xmm1
+
+// CHECK: hsubps -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0xf2,0x0f,0x7d,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+hsubps -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: hsubps 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0xf2,0x0f,0x7d,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+hsubps 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: hsubps 485498096(%edx), %xmm1
+// CHECK: encoding: [0xf2,0x0f,0x7d,0x8a,0xf0,0x1c,0xf0,0x1c]
+hsubps 485498096(%edx), %xmm1
+
+// CHECK: hsubps 485498096, %xmm1
+// CHECK: encoding: [0xf2,0x0f,0x7d,0x0d,0xf0,0x1c,0xf0,0x1c]
+hsubps 485498096, %xmm1
+
+// CHECK: hsubps 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0xf2,0x0f,0x7d,0x4c,0x02,0x40]
+hsubps 64(%edx,%eax), %xmm1
+
+// CHECK: hsubps (%edx), %xmm1
+// CHECK: encoding: [0xf2,0x0f,0x7d,0x0a]
+hsubps (%edx), %xmm1
+
+// CHECK: hsubps %xmm1, %xmm1
+// CHECK: encoding: [0xf2,0x0f,0x7d,0xc9]
+hsubps %xmm1, %xmm1
+
+// CHECK: lddqu -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0xf2,0x0f,0xf0,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+lddqu -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: lddqu 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0xf2,0x0f,0xf0,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+lddqu 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: lddqu 485498096(%edx), %xmm1
+// CHECK: encoding: [0xf2,0x0f,0xf0,0x8a,0xf0,0x1c,0xf0,0x1c]
+lddqu 485498096(%edx), %xmm1
+
+// CHECK: lddqu 485498096, %xmm1
+// CHECK: encoding: [0xf2,0x0f,0xf0,0x0d,0xf0,0x1c,0xf0,0x1c]
+lddqu 485498096, %xmm1
+
+// CHECK: lddqu 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0xf2,0x0f,0xf0,0x4c,0x02,0x40]
+lddqu 64(%edx,%eax), %xmm1
+
+// CHECK: lddqu (%edx), %xmm1
+// CHECK: encoding: [0xf2,0x0f,0xf0,0x0a]
+lddqu (%edx), %xmm1
+
+// CHECK: monitor
+// CHECK: encoding: [0x0f,0x01,0xc8]
+monitor
+
+// CHECK: movddup -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0xf2,0x0f,0x12,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+movddup -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: movddup 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0xf2,0x0f,0x12,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+movddup 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: movddup 485498096(%edx), %xmm1
+// CHECK: encoding: [0xf2,0x0f,0x12,0x8a,0xf0,0x1c,0xf0,0x1c]
+movddup 485498096(%edx), %xmm1
+
+// CHECK: movddup 485498096, %xmm1
+// CHECK: encoding: [0xf2,0x0f,0x12,0x0d,0xf0,0x1c,0xf0,0x1c]
+movddup 485498096, %xmm1
+
+// CHECK: movddup 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0xf2,0x0f,0x12,0x4c,0x02,0x40]
+movddup 64(%edx,%eax), %xmm1
+
+// CHECK: movddup (%edx), %xmm1
+// CHECK: encoding: [0xf2,0x0f,0x12,0x0a]
+movddup (%edx), %xmm1
+
+// CHECK: movddup %xmm1, %xmm1
+// CHECK: encoding: [0xf2,0x0f,0x12,0xc9]
+movddup %xmm1, %xmm1
+
+// CHECK: movshdup -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x16,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+movshdup -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: movshdup 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x16,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+movshdup 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: movshdup 485498096(%edx), %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x16,0x8a,0xf0,0x1c,0xf0,0x1c]
+movshdup 485498096(%edx), %xmm1
+
+// CHECK: movshdup 485498096, %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x16,0x0d,0xf0,0x1c,0xf0,0x1c]
+movshdup 485498096, %xmm1
+
+// CHECK: movshdup 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x16,0x4c,0x02,0x40]
+movshdup 64(%edx,%eax), %xmm1
+
+// CHECK: movshdup (%edx), %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x16,0x0a]
+movshdup (%edx), %xmm1
+
+// CHECK: movshdup %xmm1, %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x16,0xc9]
+movshdup %xmm1, %xmm1
+
+// CHECK: movsldup -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x12,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+movsldup -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: movsldup 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x12,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+movsldup 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: movsldup 485498096(%edx), %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x12,0x8a,0xf0,0x1c,0xf0,0x1c]
+movsldup 485498096(%edx), %xmm1
+
+// CHECK: movsldup 485498096, %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x12,0x0d,0xf0,0x1c,0xf0,0x1c]
+movsldup 485498096, %xmm1
+
+// CHECK: movsldup 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x12,0x4c,0x02,0x40]
+movsldup 64(%edx,%eax), %xmm1
+
+// CHECK: movsldup (%edx), %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x12,0x0a]
+movsldup (%edx), %xmm1
+
+// CHECK: movsldup %xmm1, %xmm1
+// CHECK: encoding: [0xf3,0x0f,0x12,0xc9]
+movsldup %xmm1, %xmm1
+
+// CHECK: mwait
+// CHECK: encoding: [0x0f,0x01,0xc9]
+mwait
+
diff --git a/test/MC/X86/SSE3-64.s b/test/MC/X86/SSE3-64.s
new file mode 100644
index 0000000000000000000000000000000000000000..60e2dba60e7c10c79ffbb520220df523d4904ea9
--- /dev/null
+++ b/test/MC/X86/SSE3-64.s
@@ -0,0 +1,358 @@
+// RUN: llvm-mc -triple x86_64-unknown-unknown --show-encoding %s | FileCheck %s
+
+// CHECK: addsubpd 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0xd0,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+addsubpd 485498096, %xmm6
+
+// CHECK: addsubpd -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xd0,0x74,0x82,0xc0]
+addsubpd -64(%rdx,%rax,4), %xmm6
+
+// CHECK: addsubpd 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xd0,0x74,0x82,0x40]
+addsubpd 64(%rdx,%rax,4), %xmm6
+
+// CHECK: addsubpd 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xd0,0x74,0x02,0x40]
+addsubpd 64(%rdx,%rax), %xmm6
+
+// CHECK: addsubpd 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xd0,0x72,0x40]
+addsubpd 64(%rdx), %xmm6
+
+// CHECK: addsubpd (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0xd0,0x32]
+addsubpd (%rdx), %xmm6
+
+// CHECK: addsubpd %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0xd0,0xf6]
+addsubpd %xmm6, %xmm6
+
+// CHECK: addsubps 485498096, %xmm6
+// CHECK: encoding: [0xf2,0x0f,0xd0,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+addsubps 485498096, %xmm6
+
+// CHECK: addsubps -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0xf2,0x0f,0xd0,0x74,0x82,0xc0]
+addsubps -64(%rdx,%rax,4), %xmm6
+
+// CHECK: addsubps 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0xf2,0x0f,0xd0,0x74,0x82,0x40]
+addsubps 64(%rdx,%rax,4), %xmm6
+
+// CHECK: addsubps 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0xf2,0x0f,0xd0,0x74,0x02,0x40]
+addsubps 64(%rdx,%rax), %xmm6
+
+// CHECK: addsubps 64(%rdx), %xmm6
+// CHECK: encoding: [0xf2,0x0f,0xd0,0x72,0x40]
+addsubps 64(%rdx), %xmm6
+
+// CHECK: addsubps (%rdx), %xmm6
+// CHECK: encoding: [0xf2,0x0f,0xd0,0x32]
+addsubps (%rdx), %xmm6
+
+// CHECK: addsubps %xmm6, %xmm6
+// CHECK: encoding: [0xf2,0x0f,0xd0,0xf6]
+addsubps %xmm6, %xmm6
+
+// CHECK: fisttpl 485498096
+// CHECK: encoding: [0xdb,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+fisttpl 485498096
+
+// CHECK: fisttpl 64(%rdx)
+// CHECK: encoding: [0xdb,0x4a,0x40]
+fisttpl 64(%rdx)
+
+// CHECK: fisttpl -64(%rdx,%rax,4)
+// CHECK: encoding: [0xdb,0x4c,0x82,0xc0]
+fisttpl -64(%rdx,%rax,4)
+
+// CHECK: fisttpl 64(%rdx,%rax,4)
+// CHECK: encoding: [0xdb,0x4c,0x82,0x40]
+fisttpl 64(%rdx,%rax,4)
+
+// CHECK: fisttpl 64(%rdx,%rax)
+// CHECK: encoding: [0xdb,0x4c,0x02,0x40]
+fisttpl 64(%rdx,%rax)
+
+// CHECK: fisttpll 485498096
+// CHECK: encoding: [0xdd,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+fisttpll 485498096
+
+// CHECK: fisttpll 64(%rdx)
+// CHECK: encoding: [0xdd,0x4a,0x40]
+fisttpll 64(%rdx)
+
+// CHECK: fisttpll -64(%rdx,%rax,4)
+// CHECK: encoding: [0xdd,0x4c,0x82,0xc0]
+fisttpll -64(%rdx,%rax,4)
+
+// CHECK: fisttpll 64(%rdx,%rax,4)
+// CHECK: encoding: [0xdd,0x4c,0x82,0x40]
+fisttpll 64(%rdx,%rax,4)
+
+// CHECK: fisttpll 64(%rdx,%rax)
+// CHECK: encoding: [0xdd,0x4c,0x02,0x40]
+fisttpll 64(%rdx,%rax)
+
+// CHECK: fisttpll (%rdx)
+// CHECK: encoding: [0xdd,0x0a]
+fisttpll (%rdx)
+
+// CHECK: fisttpl (%rdx)
+// CHECK: encoding: [0xdb,0x0a]
+fisttpl (%rdx)
+
+// CHECK: fisttps 485498096
+// CHECK: encoding: [0xdf,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+fisttps 485498096
+
+// CHECK: fisttps 64(%rdx)
+// CHECK: encoding: [0xdf,0x4a,0x40]
+fisttps 64(%rdx)
+
+// CHECK: fisttps -64(%rdx,%rax,4)
+// CHECK: encoding: [0xdf,0x4c,0x82,0xc0]
+fisttps -64(%rdx,%rax,4)
+
+// CHECK: fisttps 64(%rdx,%rax,4)
+// CHECK: encoding: [0xdf,0x4c,0x82,0x40]
+fisttps 64(%rdx,%rax,4)
+
+// CHECK: fisttps 64(%rdx,%rax)
+// CHECK: encoding: [0xdf,0x4c,0x02,0x40]
+fisttps 64(%rdx,%rax)
+
+// CHECK: fisttps (%rdx)
+// CHECK: encoding: [0xdf,0x0a]
+fisttps (%rdx)
+
+// CHECK: haddpd 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x7c,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+haddpd 485498096, %xmm6
+
+// CHECK: haddpd -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x7c,0x74,0x82,0xc0]
+haddpd -64(%rdx,%rax,4), %xmm6
+
+// CHECK: haddpd 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x7c,0x74,0x82,0x40]
+haddpd 64(%rdx,%rax,4), %xmm6
+
+// CHECK: haddpd 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x7c,0x74,0x02,0x40]
+haddpd 64(%rdx,%rax), %xmm6
+
+// CHECK: haddpd 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x7c,0x72,0x40]
+haddpd 64(%rdx), %xmm6
+
+// CHECK: haddpd (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x7c,0x32]
+haddpd (%rdx), %xmm6
+
+// CHECK: haddpd %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x7c,0xf6]
+haddpd %xmm6, %xmm6
+
+// CHECK: haddps 485498096, %xmm6
+// CHECK: encoding: [0xf2,0x0f,0x7c,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+haddps 485498096, %xmm6
+
+// CHECK: haddps -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0xf2,0x0f,0x7c,0x74,0x82,0xc0]
+haddps -64(%rdx,%rax,4), %xmm6
+
+// CHECK: haddps 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0xf2,0x0f,0x7c,0x74,0x82,0x40]
+haddps 64(%rdx,%rax,4), %xmm6
+
+// CHECK: haddps 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0xf2,0x0f,0x7c,0x74,0x02,0x40]
+haddps 64(%rdx,%rax), %xmm6
+
+// CHECK: haddps 64(%rdx), %xmm6
+// CHECK: encoding: [0xf2,0x0f,0x7c,0x72,0x40]
+haddps 64(%rdx), %xmm6
+
+// CHECK: haddps (%rdx), %xmm6
+// CHECK: encoding: [0xf2,0x0f,0x7c,0x32]
+haddps (%rdx), %xmm6
+
+// CHECK: haddps %xmm6, %xmm6
+// CHECK: encoding: [0xf2,0x0f,0x7c,0xf6]
+haddps %xmm6, %xmm6
+
+// CHECK: hsubpd 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x7d,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+hsubpd 485498096, %xmm6
+
+// CHECK: hsubpd -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x7d,0x74,0x82,0xc0]
+hsubpd -64(%rdx,%rax,4), %xmm6
+
+// CHECK: hsubpd 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x7d,0x74,0x82,0x40]
+hsubpd 64(%rdx,%rax,4), %xmm6
+
+// CHECK: hsubpd 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x7d,0x74,0x02,0x40]
+hsubpd 64(%rdx,%rax), %xmm6
+
+// CHECK: hsubpd 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x7d,0x72,0x40]
+hsubpd 64(%rdx), %xmm6
+
+// CHECK: hsubpd (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x7d,0x32]
+hsubpd (%rdx), %xmm6
+
+// CHECK: hsubpd %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x7d,0xf6]
+hsubpd %xmm6, %xmm6
+
+// CHECK: hsubps 485498096, %xmm6
+// CHECK: encoding: [0xf2,0x0f,0x7d,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+hsubps 485498096, %xmm6
+
+// CHECK: hsubps -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0xf2,0x0f,0x7d,0x74,0x82,0xc0]
+hsubps -64(%rdx,%rax,4), %xmm6
+
+// CHECK: hsubps 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0xf2,0x0f,0x7d,0x74,0x82,0x40]
+hsubps 64(%rdx,%rax,4), %xmm6
+
+// CHECK: hsubps 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0xf2,0x0f,0x7d,0x74,0x02,0x40]
+hsubps 64(%rdx,%rax), %xmm6
+
+// CHECK: hsubps 64(%rdx), %xmm6
+// CHECK: encoding: [0xf2,0x0f,0x7d,0x72,0x40]
+hsubps 64(%rdx), %xmm6
+
+// CHECK: hsubps (%rdx), %xmm6
+// CHECK: encoding: [0xf2,0x0f,0x7d,0x32]
+hsubps (%rdx), %xmm6
+
+// CHECK: hsubps %xmm6, %xmm6
+// CHECK: encoding: [0xf2,0x0f,0x7d,0xf6]
+hsubps %xmm6, %xmm6
+
+// CHECK: lddqu 485498096, %xmm6
+// CHECK: encoding: [0xf2,0x0f,0xf0,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+lddqu 485498096, %xmm6
+
+// CHECK: lddqu -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0xf2,0x0f,0xf0,0x74,0x82,0xc0]
+lddqu -64(%rdx,%rax,4), %xmm6
+
+// CHECK: lddqu 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0xf2,0x0f,0xf0,0x74,0x82,0x40]
+lddqu 64(%rdx,%rax,4), %xmm6
+
+// CHECK: lddqu 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0xf2,0x0f,0xf0,0x74,0x02,0x40]
+lddqu 64(%rdx,%rax), %xmm6
+
+// CHECK: lddqu 64(%rdx), %xmm6
+// CHECK: encoding: [0xf2,0x0f,0xf0,0x72,0x40]
+lddqu 64(%rdx), %xmm6
+
+// CHECK: lddqu (%rdx), %xmm6
+// CHECK: encoding: [0xf2,0x0f,0xf0,0x32]
+lddqu (%rdx), %xmm6
+
+// CHECK: monitor
+// CHECK: encoding: [0x0f,0x01,0xc8]
+monitor
+
+// CHECK: movddup 485498096, %xmm6
+// CHECK: encoding: [0xf2,0x0f,0x12,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+movddup 485498096, %xmm6
+
+// CHECK: movddup -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0xf2,0x0f,0x12,0x74,0x82,0xc0]
+movddup -64(%rdx,%rax,4), %xmm6
+
+// CHECK: movddup 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0xf2,0x0f,0x12,0x74,0x82,0x40]
+movddup 64(%rdx,%rax,4), %xmm6
+
+// CHECK: movddup 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0xf2,0x0f,0x12,0x74,0x02,0x40]
+movddup 64(%rdx,%rax), %xmm6
+
+// CHECK: movddup 64(%rdx), %xmm6
+// CHECK: encoding: [0xf2,0x0f,0x12,0x72,0x40]
+movddup 64(%rdx), %xmm6
+
+// CHECK: movddup (%rdx), %xmm6
+// CHECK: encoding: [0xf2,0x0f,0x12,0x32]
+movddup (%rdx), %xmm6
+
+// CHECK: movddup %xmm6, %xmm6
+// CHECK: encoding: [0xf2,0x0f,0x12,0xf6]
+movddup %xmm6, %xmm6
+
+// CHECK: movshdup 485498096, %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x16,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+movshdup 485498096, %xmm6
+
+// CHECK: movshdup -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x16,0x74,0x82,0xc0]
+movshdup -64(%rdx,%rax,4), %xmm6
+
+// CHECK: movshdup 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x16,0x74,0x82,0x40]
+movshdup 64(%rdx,%rax,4), %xmm6
+
+// CHECK: movshdup 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x16,0x74,0x02,0x40]
+movshdup 64(%rdx,%rax), %xmm6
+
+// CHECK: movshdup 64(%rdx), %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x16,0x72,0x40]
+movshdup 64(%rdx), %xmm6
+
+// CHECK: movshdup (%rdx), %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x16,0x32]
+movshdup (%rdx), %xmm6
+
+// CHECK: movshdup %xmm6, %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x16,0xf6]
+movshdup %xmm6, %xmm6
+
+// CHECK: movsldup 485498096, %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x12,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+movsldup 485498096, %xmm6
+
+// CHECK: movsldup -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x12,0x74,0x82,0xc0]
+movsldup -64(%rdx,%rax,4), %xmm6
+
+// CHECK: movsldup 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x12,0x74,0x82,0x40]
+movsldup 64(%rdx,%rax,4), %xmm6
+
+// CHECK: movsldup 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x12,0x74,0x02,0x40]
+movsldup 64(%rdx,%rax), %xmm6
+
+// CHECK: movsldup 64(%rdx), %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x12,0x72,0x40]
+movsldup 64(%rdx), %xmm6
+
+// CHECK: movsldup (%rdx), %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x12,0x32]
+movsldup (%rdx), %xmm6
+
+// CHECK: movsldup %xmm6, %xmm6
+// CHECK: encoding: [0xf3,0x0f,0x12,0xf6]
+movsldup %xmm6, %xmm6
+
+// CHECK: mwait
+// CHECK: encoding: [0x0f,0x01,0xc9]
+mwait
+
diff --git a/test/MC/X86/SSE41-32.s b/test/MC/X86/SSE41-32.s
new file mode 100644
index 0000000000000000000000000000000000000000..14ab62b20df7f341112ff0249a1275e67fb171e7
--- /dev/null
+++ b/test/MC/X86/SSE41-32.s
@@ -0,0 +1,1290 @@
+// RUN: llvm-mc -triple i386-unknown-unknown --show-encoding %s | FileCheck %s
+
+// CHECK: blendpd $0, -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x3a,0x0d,0x8c,0x82,0x10,0xe3,0x0f,0xe3,0x00]
+blendpd $0, -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: blendpd $0, 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x3a,0x0d,0x8c,0x82,0xf0,0x1c,0xf0,0x1c,0x00]
+blendpd $0, 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: blendpd $0, 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x3a,0x0d,0x8a,0xf0,0x1c,0xf0,0x1c,0x00]
+blendpd $0, 485498096(%edx), %xmm1
+
+// CHECK: blendpd $0, 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x3a,0x0d,0x0d,0xf0,0x1c,0xf0,0x1c,0x00]
+blendpd $0, 485498096, %xmm1
+
+// CHECK: blendpd $0, 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x3a,0x0d,0x4c,0x02,0x40,0x00]
+blendpd $0, 64(%edx,%eax), %xmm1
+
+// CHECK: blendpd $0, (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x3a,0x0d,0x0a,0x00]
+blendpd $0, (%edx), %xmm1
+
+// CHECK: blendpd $0, %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x3a,0x0d,0xc9,0x00]
+blendpd $0, %xmm1, %xmm1
+
+// CHECK: blendps $0, -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x3a,0x0c,0x8c,0x82,0x10,0xe3,0x0f,0xe3,0x00]
+blendps $0, -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: blendps $0, 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x3a,0x0c,0x8c,0x82,0xf0,0x1c,0xf0,0x1c,0x00]
+blendps $0, 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: blendps $0, 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x3a,0x0c,0x8a,0xf0,0x1c,0xf0,0x1c,0x00]
+blendps $0, 485498096(%edx), %xmm1
+
+// CHECK: blendps $0, 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x3a,0x0c,0x0d,0xf0,0x1c,0xf0,0x1c,0x00]
+blendps $0, 485498096, %xmm1
+
+// CHECK: blendps $0, 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x3a,0x0c,0x4c,0x02,0x40,0x00]
+blendps $0, 64(%edx,%eax), %xmm1
+
+// CHECK: blendps $0, (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x3a,0x0c,0x0a,0x00]
+blendps $0, (%edx), %xmm1
+
+// CHECK: blendps $0, %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x3a,0x0c,0xc9,0x00]
+blendps $0, %xmm1, %xmm1
+
+// CHECK: blendvpd %xmm0, -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x15,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+blendvpd %xmm0, -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: blendvpd %xmm0, 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x15,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+blendvpd %xmm0, 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: blendvpd %xmm0, 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x15,0x8a,0xf0,0x1c,0xf0,0x1c]
+blendvpd %xmm0, 485498096(%edx), %xmm1
+
+// CHECK: blendvpd %xmm0, 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x15,0x0d,0xf0,0x1c,0xf0,0x1c]
+blendvpd %xmm0, 485498096, %xmm1
+
+// CHECK: blendvpd %xmm0, 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x15,0x4c,0x02,0x40]
+blendvpd %xmm0, 64(%edx,%eax), %xmm1
+
+// CHECK: blendvpd %xmm0, (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x15,0x0a]
+blendvpd %xmm0, (%edx), %xmm1
+
+// CHECK: blendvpd %xmm0, %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x15,0xc9]
+blendvpd %xmm0, %xmm1, %xmm1
+
+// CHECK: blendvps %xmm0, -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x14,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+blendvps %xmm0, -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: blendvps %xmm0, 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x14,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+blendvps %xmm0, 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: blendvps %xmm0, 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x14,0x8a,0xf0,0x1c,0xf0,0x1c]
+blendvps %xmm0, 485498096(%edx), %xmm1
+
+// CHECK: blendvps %xmm0, 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x14,0x0d,0xf0,0x1c,0xf0,0x1c]
+blendvps %xmm0, 485498096, %xmm1
+
+// CHECK: blendvps %xmm0, 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x14,0x4c,0x02,0x40]
+blendvps %xmm0, 64(%edx,%eax), %xmm1
+
+// CHECK: blendvps %xmm0, (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x14,0x0a]
+blendvps %xmm0, (%edx), %xmm1
+
+// CHECK: blendvps %xmm0, %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x14,0xc9]
+blendvps %xmm0, %xmm1, %xmm1
+
+// CHECK: dppd $0, -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x3a,0x41,0x8c,0x82,0x10,0xe3,0x0f,0xe3,0x00]
+dppd $0, -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: dppd $0, 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x3a,0x41,0x8c,0x82,0xf0,0x1c,0xf0,0x1c,0x00]
+dppd $0, 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: dppd $0, 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x3a,0x41,0x8a,0xf0,0x1c,0xf0,0x1c,0x00]
+dppd $0, 485498096(%edx), %xmm1
+
+// CHECK: dppd $0, 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x3a,0x41,0x0d,0xf0,0x1c,0xf0,0x1c,0x00]
+dppd $0, 485498096, %xmm1
+
+// CHECK: dppd $0, 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x3a,0x41,0x4c,0x02,0x40,0x00]
+dppd $0, 64(%edx,%eax), %xmm1
+
+// CHECK: dppd $0, (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x3a,0x41,0x0a,0x00]
+dppd $0, (%edx), %xmm1
+
+// CHECK: dppd $0, %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x3a,0x41,0xc9,0x00]
+dppd $0, %xmm1, %xmm1
+
+// CHECK: dpps $0, -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x3a,0x40,0x8c,0x82,0x10,0xe3,0x0f,0xe3,0x00]
+dpps $0, -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: dpps $0, 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x3a,0x40,0x8c,0x82,0xf0,0x1c,0xf0,0x1c,0x00]
+dpps $0, 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: dpps $0, 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x3a,0x40,0x8a,0xf0,0x1c,0xf0,0x1c,0x00]
+dpps $0, 485498096(%edx), %xmm1
+
+// CHECK: dpps $0, 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x3a,0x40,0x0d,0xf0,0x1c,0xf0,0x1c,0x00]
+dpps $0, 485498096, %xmm1
+
+// CHECK: dpps $0, 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x3a,0x40,0x4c,0x02,0x40,0x00]
+dpps $0, 64(%edx,%eax), %xmm1
+
+// CHECK: dpps $0, (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x3a,0x40,0x0a,0x00]
+dpps $0, (%edx), %xmm1
+
+// CHECK: dpps $0, %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x3a,0x40,0xc9,0x00]
+dpps $0, %xmm1, %xmm1
+
+// CHECK: extractps $0, %xmm1, -485498096(%edx,%eax,4)
+// CHECK: encoding: [0x66,0x0f,0x3a,0x17,0x8c,0x82,0x10,0xe3,0x0f,0xe3,0x00]
+extractps $0, %xmm1, -485498096(%edx,%eax,4)
+
+// CHECK: extractps $0, %xmm1, 485498096(%edx,%eax,4)
+// CHECK: encoding: [0x66,0x0f,0x3a,0x17,0x8c,0x82,0xf0,0x1c,0xf0,0x1c,0x00]
+extractps $0, %xmm1, 485498096(%edx,%eax,4)
+
+// CHECK: extractps $0, %xmm1, 485498096(%edx)
+// CHECK: encoding: [0x66,0x0f,0x3a,0x17,0x8a,0xf0,0x1c,0xf0,0x1c,0x00]
+extractps $0, %xmm1, 485498096(%edx)
+
+// CHECK: extractps $0, %xmm1, 485498096
+// CHECK: encoding: [0x66,0x0f,0x3a,0x17,0x0d,0xf0,0x1c,0xf0,0x1c,0x00]
+extractps $0, %xmm1, 485498096
+
+// CHECK: extractps $0, %xmm1, 64(%edx,%eax)
+// CHECK: encoding: [0x66,0x0f,0x3a,0x17,0x4c,0x02,0x40,0x00]
+extractps $0, %xmm1, 64(%edx,%eax)
+
+// CHECK: extractps $0, %xmm1, (%edx)
+// CHECK: encoding: [0x66,0x0f,0x3a,0x17,0x0a,0x00]
+extractps $0, %xmm1, (%edx)
+
+// CHECK: insertps $0, -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x3a,0x21,0x8c,0x82,0x10,0xe3,0x0f,0xe3,0x00]
+insertps $0, -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: insertps $0, 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x3a,0x21,0x8c,0x82,0xf0,0x1c,0xf0,0x1c,0x00]
+insertps $0, 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: insertps $0, 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x3a,0x21,0x8a,0xf0,0x1c,0xf0,0x1c,0x00]
+insertps $0, 485498096(%edx), %xmm1
+
+// CHECK: insertps $0, 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x3a,0x21,0x0d,0xf0,0x1c,0xf0,0x1c,0x00]
+insertps $0, 485498096, %xmm1
+
+// CHECK: insertps $0, 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x3a,0x21,0x4c,0x02,0x40,0x00]
+insertps $0, 64(%edx,%eax), %xmm1
+
+// CHECK: insertps $0, (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x3a,0x21,0x0a,0x00]
+insertps $0, (%edx), %xmm1
+
+// CHECK: insertps $0, %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x3a,0x21,0xc9,0x00]
+insertps $0, %xmm1, %xmm1
+
+// CHECK: movntdqa -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x2a,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+movntdqa -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: movntdqa 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x2a,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+movntdqa 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: movntdqa 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x2a,0x8a,0xf0,0x1c,0xf0,0x1c]
+movntdqa 485498096(%edx), %xmm1
+
+// CHECK: movntdqa 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x2a,0x0d,0xf0,0x1c,0xf0,0x1c]
+movntdqa 485498096, %xmm1
+
+// CHECK: movntdqa 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x2a,0x4c,0x02,0x40]
+movntdqa 64(%edx,%eax), %xmm1
+
+// CHECK: movntdqa (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x2a,0x0a]
+movntdqa (%edx), %xmm1
+
+// CHECK: mpsadbw $0, -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x3a,0x42,0x8c,0x82,0x10,0xe3,0x0f,0xe3,0x00]
+mpsadbw $0, -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: mpsadbw $0, 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x3a,0x42,0x8c,0x82,0xf0,0x1c,0xf0,0x1c,0x00]
+mpsadbw $0, 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: mpsadbw $0, 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x3a,0x42,0x8a,0xf0,0x1c,0xf0,0x1c,0x00]
+mpsadbw $0, 485498096(%edx), %xmm1
+
+// CHECK: mpsadbw $0, 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x3a,0x42,0x0d,0xf0,0x1c,0xf0,0x1c,0x00]
+mpsadbw $0, 485498096, %xmm1
+
+// CHECK: mpsadbw $0, 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x3a,0x42,0x4c,0x02,0x40,0x00]
+mpsadbw $0, 64(%edx,%eax), %xmm1
+
+// CHECK: mpsadbw $0, (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x3a,0x42,0x0a,0x00]
+mpsadbw $0, (%edx), %xmm1
+
+// CHECK: mpsadbw $0, %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x3a,0x42,0xc9,0x00]
+mpsadbw $0, %xmm1, %xmm1
+
+// CHECK: packusdw -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x2b,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+packusdw -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: packusdw 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x2b,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+packusdw 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: packusdw 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x2b,0x8a,0xf0,0x1c,0xf0,0x1c]
+packusdw 485498096(%edx), %xmm1
+
+// CHECK: packusdw 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x2b,0x0d,0xf0,0x1c,0xf0,0x1c]
+packusdw 485498096, %xmm1
+
+// CHECK: packusdw 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x2b,0x4c,0x02,0x40]
+packusdw 64(%edx,%eax), %xmm1
+
+// CHECK: packusdw (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x2b,0x0a]
+packusdw (%edx), %xmm1
+
+// CHECK: packusdw %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x2b,0xc9]
+packusdw %xmm1, %xmm1
+
+// CHECK: pblendvb %xmm0, -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x10,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+pblendvb %xmm0, -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pblendvb %xmm0, 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x10,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+pblendvb %xmm0, 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pblendvb %xmm0, 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x10,0x8a,0xf0,0x1c,0xf0,0x1c]
+pblendvb %xmm0, 485498096(%edx), %xmm1
+
+// CHECK: pblendvb %xmm0, 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x10,0x0d,0xf0,0x1c,0xf0,0x1c]
+pblendvb %xmm0, 485498096, %xmm1
+
+// CHECK: pblendvb %xmm0, 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x10,0x4c,0x02,0x40]
+pblendvb %xmm0, 64(%edx,%eax), %xmm1
+
+// CHECK: pblendvb %xmm0, (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x10,0x0a]
+pblendvb %xmm0, (%edx), %xmm1
+
+// CHECK: pblendvb %xmm0, %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x10,0xc9]
+pblendvb %xmm0, %xmm1, %xmm1
+
+// CHECK: pblendw $0, -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x3a,0x0e,0x8c,0x82,0x10,0xe3,0x0f,0xe3,0x00]
+pblendw $0, -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pblendw $0, 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x3a,0x0e,0x8c,0x82,0xf0,0x1c,0xf0,0x1c,0x00]
+pblendw $0, 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pblendw $0, 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x3a,0x0e,0x8a,0xf0,0x1c,0xf0,0x1c,0x00]
+pblendw $0, 485498096(%edx), %xmm1
+
+// CHECK: pblendw $0, 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x3a,0x0e,0x0d,0xf0,0x1c,0xf0,0x1c,0x00]
+pblendw $0, 485498096, %xmm1
+
+// CHECK: pblendw $0, 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x3a,0x0e,0x4c,0x02,0x40,0x00]
+pblendw $0, 64(%edx,%eax), %xmm1
+
+// CHECK: pblendw $0, (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x3a,0x0e,0x0a,0x00]
+pblendw $0, (%edx), %xmm1
+
+// CHECK: pblendw $0, %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x3a,0x0e,0xc9,0x00]
+pblendw $0, %xmm1, %xmm1
+
+// CHECK: pcmpeqq -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x29,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+pcmpeqq -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pcmpeqq 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x29,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+pcmpeqq 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pcmpeqq 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x29,0x8a,0xf0,0x1c,0xf0,0x1c]
+pcmpeqq 485498096(%edx), %xmm1
+
+// CHECK: pcmpeqq 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x29,0x0d,0xf0,0x1c,0xf0,0x1c]
+pcmpeqq 485498096, %xmm1
+
+// CHECK: pcmpeqq 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x29,0x4c,0x02,0x40]
+pcmpeqq 64(%edx,%eax), %xmm1
+
+// CHECK: pcmpeqq (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x29,0x0a]
+pcmpeqq (%edx), %xmm1
+
+// CHECK: pcmpeqq %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x29,0xc9]
+pcmpeqq %xmm1, %xmm1
+
+// CHECK: pextrb $0, %xmm1, -485498096(%edx,%eax,4)
+// CHECK: encoding: [0x66,0x0f,0x3a,0x14,0x8c,0x82,0x10,0xe3,0x0f,0xe3,0x00]
+pextrb $0, %xmm1, -485498096(%edx,%eax,4)
+
+// CHECK: pextrb $0, %xmm1, 485498096(%edx,%eax,4)
+// CHECK: encoding: [0x66,0x0f,0x3a,0x14,0x8c,0x82,0xf0,0x1c,0xf0,0x1c,0x00]
+pextrb $0, %xmm1, 485498096(%edx,%eax,4)
+
+// CHECK: pextrb $0, %xmm1, 485498096(%edx)
+// CHECK: encoding: [0x66,0x0f,0x3a,0x14,0x8a,0xf0,0x1c,0xf0,0x1c,0x00]
+pextrb $0, %xmm1, 485498096(%edx)
+
+// CHECK: pextrb $0, %xmm1, 485498096
+// CHECK: encoding: [0x66,0x0f,0x3a,0x14,0x0d,0xf0,0x1c,0xf0,0x1c,0x00]
+pextrb $0, %xmm1, 485498096
+
+// CHECK: pextrb $0, %xmm1, 64(%edx,%eax)
+// CHECK: encoding: [0x66,0x0f,0x3a,0x14,0x4c,0x02,0x40,0x00]
+pextrb $0, %xmm1, 64(%edx,%eax)
+
+// CHECK: pextrb $0, %xmm1, (%edx)
+// CHECK: encoding: [0x66,0x0f,0x3a,0x14,0x0a,0x00]
+pextrb $0, %xmm1, (%edx)
+
+// CHECK: pextrd $0, %xmm1, -485498096(%edx,%eax,4)
+// CHECK: encoding: [0x66,0x0f,0x3a,0x16,0x8c,0x82,0x10,0xe3,0x0f,0xe3,0x00]
+pextrd $0, %xmm1, -485498096(%edx,%eax,4)
+
+// CHECK: pextrd $0, %xmm1, 485498096(%edx,%eax,4)
+// CHECK: encoding: [0x66,0x0f,0x3a,0x16,0x8c,0x82,0xf0,0x1c,0xf0,0x1c,0x00]
+pextrd $0, %xmm1, 485498096(%edx,%eax,4)
+
+// CHECK: pextrd $0, %xmm1, 485498096(%edx)
+// CHECK: encoding: [0x66,0x0f,0x3a,0x16,0x8a,0xf0,0x1c,0xf0,0x1c,0x00]
+pextrd $0, %xmm1, 485498096(%edx)
+
+// CHECK: pextrd $0, %xmm1, 485498096
+// CHECK: encoding: [0x66,0x0f,0x3a,0x16,0x0d,0xf0,0x1c,0xf0,0x1c,0x00]
+pextrd $0, %xmm1, 485498096
+
+// CHECK: pextrd $0, %xmm1, 64(%edx,%eax)
+// CHECK: encoding: [0x66,0x0f,0x3a,0x16,0x4c,0x02,0x40,0x00]
+pextrd $0, %xmm1, 64(%edx,%eax)
+
+// CHECK: pextrd $0, %xmm1, (%edx)
+// CHECK: encoding: [0x66,0x0f,0x3a,0x16,0x0a,0x00]
+pextrd $0, %xmm1, (%edx)
+
+// CHECK: pextrw $0, %xmm1, -485498096(%edx,%eax,4)
+// CHECK: encoding: [0x66,0x0f,0x3a,0x15,0x8c,0x82,0x10,0xe3,0x0f,0xe3,0x00]
+pextrw $0, %xmm1, -485498096(%edx,%eax,4)
+
+// CHECK: pextrw $0, %xmm1, 485498096(%edx,%eax,4)
+// CHECK: encoding: [0x66,0x0f,0x3a,0x15,0x8c,0x82,0xf0,0x1c,0xf0,0x1c,0x00]
+pextrw $0, %xmm1, 485498096(%edx,%eax,4)
+
+// CHECK: pextrw $0, %xmm1, 485498096(%edx)
+// CHECK: encoding: [0x66,0x0f,0x3a,0x15,0x8a,0xf0,0x1c,0xf0,0x1c,0x00]
+pextrw $0, %xmm1, 485498096(%edx)
+
+// CHECK: pextrw $0, %xmm1, 485498096
+// CHECK: encoding: [0x66,0x0f,0x3a,0x15,0x0d,0xf0,0x1c,0xf0,0x1c,0x00]
+pextrw $0, %xmm1, 485498096
+
+// CHECK: pextrw $0, %xmm1, 64(%edx,%eax)
+// CHECK: encoding: [0x66,0x0f,0x3a,0x15,0x4c,0x02,0x40,0x00]
+pextrw $0, %xmm1, 64(%edx,%eax)
+
+// CHECK: pextrw $0, %xmm1, (%edx)
+// CHECK: encoding: [0x66,0x0f,0x3a,0x15,0x0a,0x00]
+pextrw $0, %xmm1, (%edx)
+
+// CHECK: phminposuw -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x41,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+phminposuw -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: phminposuw 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x41,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+phminposuw 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: phminposuw 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x41,0x8a,0xf0,0x1c,0xf0,0x1c]
+phminposuw 485498096(%edx), %xmm1
+
+// CHECK: phminposuw 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x41,0x0d,0xf0,0x1c,0xf0,0x1c]
+phminposuw 485498096, %xmm1
+
+// CHECK: phminposuw 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x41,0x4c,0x02,0x40]
+phminposuw 64(%edx,%eax), %xmm1
+
+// CHECK: phminposuw (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x41,0x0a]
+phminposuw (%edx), %xmm1
+
+// CHECK: phminposuw %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x41,0xc9]
+phminposuw %xmm1, %xmm1
+
+// CHECK: pinsrb $0, -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x3a,0x20,0x8c,0x82,0x10,0xe3,0x0f,0xe3,0x00]
+pinsrb $0, -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pinsrb $0, 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x3a,0x20,0x8c,0x82,0xf0,0x1c,0xf0,0x1c,0x00]
+pinsrb $0, 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pinsrb $0, 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x3a,0x20,0x8a,0xf0,0x1c,0xf0,0x1c,0x00]
+pinsrb $0, 485498096(%edx), %xmm1
+
+// CHECK: pinsrb $0, 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x3a,0x20,0x0d,0xf0,0x1c,0xf0,0x1c,0x00]
+pinsrb $0, 485498096, %xmm1
+
+// CHECK: pinsrb $0, 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x3a,0x20,0x4c,0x02,0x40,0x00]
+pinsrb $0, 64(%edx,%eax), %xmm1
+
+// CHECK: pinsrb $0, (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x3a,0x20,0x0a,0x00]
+pinsrb $0, (%edx), %xmm1
+
+// CHECK: pinsrd $0, -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x3a,0x22,0x8c,0x82,0x10,0xe3,0x0f,0xe3,0x00]
+pinsrd $0, -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pinsrd $0, 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x3a,0x22,0x8c,0x82,0xf0,0x1c,0xf0,0x1c,0x00]
+pinsrd $0, 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pinsrd $0, 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x3a,0x22,0x8a,0xf0,0x1c,0xf0,0x1c,0x00]
+pinsrd $0, 485498096(%edx), %xmm1
+
+// CHECK: pinsrd $0, 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x3a,0x22,0x0d,0xf0,0x1c,0xf0,0x1c,0x00]
+pinsrd $0, 485498096, %xmm1
+
+// CHECK: pinsrd $0, 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x3a,0x22,0x4c,0x02,0x40,0x00]
+pinsrd $0, 64(%edx,%eax), %xmm1
+
+// CHECK: pinsrd $0, (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x3a,0x22,0x0a,0x00]
+pinsrd $0, (%edx), %xmm1
+
+// CHECK: pmaxsb -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x3c,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+pmaxsb -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pmaxsb 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x3c,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+pmaxsb 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pmaxsb 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x3c,0x8a,0xf0,0x1c,0xf0,0x1c]
+pmaxsb 485498096(%edx), %xmm1
+
+// CHECK: pmaxsb 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x3c,0x0d,0xf0,0x1c,0xf0,0x1c]
+pmaxsb 485498096, %xmm1
+
+// CHECK: pmaxsb 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x3c,0x4c,0x02,0x40]
+pmaxsb 64(%edx,%eax), %xmm1
+
+// CHECK: pmaxsb (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x3c,0x0a]
+pmaxsb (%edx), %xmm1
+
+// CHECK: pmaxsb %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x3c,0xc9]
+pmaxsb %xmm1, %xmm1
+
+// CHECK: pmaxsd -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x3d,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+pmaxsd -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pmaxsd 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x3d,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+pmaxsd 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pmaxsd 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x3d,0x8a,0xf0,0x1c,0xf0,0x1c]
+pmaxsd 485498096(%edx), %xmm1
+
+// CHECK: pmaxsd 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x3d,0x0d,0xf0,0x1c,0xf0,0x1c]
+pmaxsd 485498096, %xmm1
+
+// CHECK: pmaxsd 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x3d,0x4c,0x02,0x40]
+pmaxsd 64(%edx,%eax), %xmm1
+
+// CHECK: pmaxsd (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x3d,0x0a]
+pmaxsd (%edx), %xmm1
+
+// CHECK: pmaxsd %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x3d,0xc9]
+pmaxsd %xmm1, %xmm1
+
+// CHECK: pmaxud -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x3f,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+pmaxud -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pmaxud 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x3f,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+pmaxud 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pmaxud 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x3f,0x8a,0xf0,0x1c,0xf0,0x1c]
+pmaxud 485498096(%edx), %xmm1
+
+// CHECK: pmaxud 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x3f,0x0d,0xf0,0x1c,0xf0,0x1c]
+pmaxud 485498096, %xmm1
+
+// CHECK: pmaxud 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x3f,0x4c,0x02,0x40]
+pmaxud 64(%edx,%eax), %xmm1
+
+// CHECK: pmaxud (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x3f,0x0a]
+pmaxud (%edx), %xmm1
+
+// CHECK: pmaxud %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x3f,0xc9]
+pmaxud %xmm1, %xmm1
+
+// CHECK: pmaxuw -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x3e,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+pmaxuw -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pmaxuw 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x3e,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+pmaxuw 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pmaxuw 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x3e,0x8a,0xf0,0x1c,0xf0,0x1c]
+pmaxuw 485498096(%edx), %xmm1
+
+// CHECK: pmaxuw 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x3e,0x0d,0xf0,0x1c,0xf0,0x1c]
+pmaxuw 485498096, %xmm1
+
+// CHECK: pmaxuw 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x3e,0x4c,0x02,0x40]
+pmaxuw 64(%edx,%eax), %xmm1
+
+// CHECK: pmaxuw (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x3e,0x0a]
+pmaxuw (%edx), %xmm1
+
+// CHECK: pmaxuw %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x3e,0xc9]
+pmaxuw %xmm1, %xmm1
+
+// CHECK: pminsb -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x38,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+pminsb -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pminsb 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x38,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+pminsb 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pminsb 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x38,0x8a,0xf0,0x1c,0xf0,0x1c]
+pminsb 485498096(%edx), %xmm1
+
+// CHECK: pminsb 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x38,0x0d,0xf0,0x1c,0xf0,0x1c]
+pminsb 485498096, %xmm1
+
+// CHECK: pminsb 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x38,0x4c,0x02,0x40]
+pminsb 64(%edx,%eax), %xmm1
+
+// CHECK: pminsb (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x38,0x0a]
+pminsb (%edx), %xmm1
+
+// CHECK: pminsb %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x38,0xc9]
+pminsb %xmm1, %xmm1
+
+// CHECK: pminsd -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x39,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+pminsd -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pminsd 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x39,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+pminsd 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pminsd 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x39,0x8a,0xf0,0x1c,0xf0,0x1c]
+pminsd 485498096(%edx), %xmm1
+
+// CHECK: pminsd 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x39,0x0d,0xf0,0x1c,0xf0,0x1c]
+pminsd 485498096, %xmm1
+
+// CHECK: pminsd 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x39,0x4c,0x02,0x40]
+pminsd 64(%edx,%eax), %xmm1
+
+// CHECK: pminsd (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x39,0x0a]
+pminsd (%edx), %xmm1
+
+// CHECK: pminsd %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x39,0xc9]
+pminsd %xmm1, %xmm1
+
+// CHECK: pminud -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x3b,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+pminud -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pminud 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x3b,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+pminud 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pminud 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x3b,0x8a,0xf0,0x1c,0xf0,0x1c]
+pminud 485498096(%edx), %xmm1
+
+// CHECK: pminud 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x3b,0x0d,0xf0,0x1c,0xf0,0x1c]
+pminud 485498096, %xmm1
+
+// CHECK: pminud 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x3b,0x4c,0x02,0x40]
+pminud 64(%edx,%eax), %xmm1
+
+// CHECK: pminud (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x3b,0x0a]
+pminud (%edx), %xmm1
+
+// CHECK: pminud %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x3b,0xc9]
+pminud %xmm1, %xmm1
+
+// CHECK: pminuw -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x3a,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+pminuw -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pminuw 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x3a,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+pminuw 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pminuw 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x3a,0x8a,0xf0,0x1c,0xf0,0x1c]
+pminuw 485498096(%edx), %xmm1
+
+// CHECK: pminuw 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x3a,0x0d,0xf0,0x1c,0xf0,0x1c]
+pminuw 485498096, %xmm1
+
+// CHECK: pminuw 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x3a,0x4c,0x02,0x40]
+pminuw 64(%edx,%eax), %xmm1
+
+// CHECK: pminuw (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x3a,0x0a]
+pminuw (%edx), %xmm1
+
+// CHECK: pminuw %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x3a,0xc9]
+pminuw %xmm1, %xmm1
+
+// CHECK: pmovsxbd -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x21,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+pmovsxbd -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pmovsxbd 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x21,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+pmovsxbd 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pmovsxbd 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x21,0x8a,0xf0,0x1c,0xf0,0x1c]
+pmovsxbd 485498096(%edx), %xmm1
+
+// CHECK: pmovsxbd 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x21,0x0d,0xf0,0x1c,0xf0,0x1c]
+pmovsxbd 485498096, %xmm1
+
+// CHECK: pmovsxbd 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x21,0x4c,0x02,0x40]
+pmovsxbd 64(%edx,%eax), %xmm1
+
+// CHECK: pmovsxbd (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x21,0x0a]
+pmovsxbd (%edx), %xmm1
+
+// CHECK: pmovsxbd %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x21,0xc9]
+pmovsxbd %xmm1, %xmm1
+
+// CHECK: pmovsxbq -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x22,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+pmovsxbq -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pmovsxbq 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x22,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+pmovsxbq 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pmovsxbq 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x22,0x8a,0xf0,0x1c,0xf0,0x1c]
+pmovsxbq 485498096(%edx), %xmm1
+
+// CHECK: pmovsxbq 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x22,0x0d,0xf0,0x1c,0xf0,0x1c]
+pmovsxbq 485498096, %xmm1
+
+// CHECK: pmovsxbq 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x22,0x4c,0x02,0x40]
+pmovsxbq 64(%edx,%eax), %xmm1
+
+// CHECK: pmovsxbq (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x22,0x0a]
+pmovsxbq (%edx), %xmm1
+
+// CHECK: pmovsxbq %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x22,0xc9]
+pmovsxbq %xmm1, %xmm1
+
+// CHECK: pmovsxbw -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x20,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+pmovsxbw -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pmovsxbw 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x20,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+pmovsxbw 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pmovsxbw 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x20,0x8a,0xf0,0x1c,0xf0,0x1c]
+pmovsxbw 485498096(%edx), %xmm1
+
+// CHECK: pmovsxbw 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x20,0x0d,0xf0,0x1c,0xf0,0x1c]
+pmovsxbw 485498096, %xmm1
+
+// CHECK: pmovsxbw 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x20,0x4c,0x02,0x40]
+pmovsxbw 64(%edx,%eax), %xmm1
+
+// CHECK: pmovsxbw (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x20,0x0a]
+pmovsxbw (%edx), %xmm1
+
+// CHECK: pmovsxbw %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x20,0xc9]
+pmovsxbw %xmm1, %xmm1
+
+// CHECK: pmovsxdq -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x25,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+pmovsxdq -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pmovsxdq 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x25,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+pmovsxdq 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pmovsxdq 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x25,0x8a,0xf0,0x1c,0xf0,0x1c]
+pmovsxdq 485498096(%edx), %xmm1
+
+// CHECK: pmovsxdq 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x25,0x0d,0xf0,0x1c,0xf0,0x1c]
+pmovsxdq 485498096, %xmm1
+
+// CHECK: pmovsxdq 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x25,0x4c,0x02,0x40]
+pmovsxdq 64(%edx,%eax), %xmm1
+
+// CHECK: pmovsxdq (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x25,0x0a]
+pmovsxdq (%edx), %xmm1
+
+// CHECK: pmovsxdq %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x25,0xc9]
+pmovsxdq %xmm1, %xmm1
+
+// CHECK: pmovsxwd -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x23,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+pmovsxwd -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pmovsxwd 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x23,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+pmovsxwd 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pmovsxwd 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x23,0x8a,0xf0,0x1c,0xf0,0x1c]
+pmovsxwd 485498096(%edx), %xmm1
+
+// CHECK: pmovsxwd 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x23,0x0d,0xf0,0x1c,0xf0,0x1c]
+pmovsxwd 485498096, %xmm1
+
+// CHECK: pmovsxwd 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x23,0x4c,0x02,0x40]
+pmovsxwd 64(%edx,%eax), %xmm1
+
+// CHECK: pmovsxwd (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x23,0x0a]
+pmovsxwd (%edx), %xmm1
+
+// CHECK: pmovsxwd %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x23,0xc9]
+pmovsxwd %xmm1, %xmm1
+
+// CHECK: pmovsxwq -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x24,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+pmovsxwq -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pmovsxwq 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x24,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+pmovsxwq 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pmovsxwq 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x24,0x8a,0xf0,0x1c,0xf0,0x1c]
+pmovsxwq 485498096(%edx), %xmm1
+
+// CHECK: pmovsxwq 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x24,0x0d,0xf0,0x1c,0xf0,0x1c]
+pmovsxwq 485498096, %xmm1
+
+// CHECK: pmovsxwq 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x24,0x4c,0x02,0x40]
+pmovsxwq 64(%edx,%eax), %xmm1
+
+// CHECK: pmovsxwq (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x24,0x0a]
+pmovsxwq (%edx), %xmm1
+
+// CHECK: pmovsxwq %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x24,0xc9]
+pmovsxwq %xmm1, %xmm1
+
+// CHECK: pmovzxbd -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x31,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+pmovzxbd -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pmovzxbd 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x31,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+pmovzxbd 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pmovzxbd 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x31,0x8a,0xf0,0x1c,0xf0,0x1c]
+pmovzxbd 485498096(%edx), %xmm1
+
+// CHECK: pmovzxbd 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x31,0x0d,0xf0,0x1c,0xf0,0x1c]
+pmovzxbd 485498096, %xmm1
+
+// CHECK: pmovzxbd 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x31,0x4c,0x02,0x40]
+pmovzxbd 64(%edx,%eax), %xmm1
+
+// CHECK: pmovzxbd (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x31,0x0a]
+pmovzxbd (%edx), %xmm1
+
+// CHECK: pmovzxbd %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x31,0xc9]
+pmovzxbd %xmm1, %xmm1
+
+// CHECK: pmovzxbq -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x32,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+pmovzxbq -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pmovzxbq 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x32,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+pmovzxbq 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pmovzxbq 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x32,0x8a,0xf0,0x1c,0xf0,0x1c]
+pmovzxbq 485498096(%edx), %xmm1
+
+// CHECK: pmovzxbq 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x32,0x0d,0xf0,0x1c,0xf0,0x1c]
+pmovzxbq 485498096, %xmm1
+
+// CHECK: pmovzxbq 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x32,0x4c,0x02,0x40]
+pmovzxbq 64(%edx,%eax), %xmm1
+
+// CHECK: pmovzxbq (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x32,0x0a]
+pmovzxbq (%edx), %xmm1
+
+// CHECK: pmovzxbq %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x32,0xc9]
+pmovzxbq %xmm1, %xmm1
+
+// CHECK: pmovzxbw -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x30,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+pmovzxbw -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pmovzxbw 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x30,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+pmovzxbw 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pmovzxbw 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x30,0x8a,0xf0,0x1c,0xf0,0x1c]
+pmovzxbw 485498096(%edx), %xmm1
+
+// CHECK: pmovzxbw 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x30,0x0d,0xf0,0x1c,0xf0,0x1c]
+pmovzxbw 485498096, %xmm1
+
+// CHECK: pmovzxbw 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x30,0x4c,0x02,0x40]
+pmovzxbw 64(%edx,%eax), %xmm1
+
+// CHECK: pmovzxbw (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x30,0x0a]
+pmovzxbw (%edx), %xmm1
+
+// CHECK: pmovzxbw %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x30,0xc9]
+pmovzxbw %xmm1, %xmm1
+
+// CHECK: pmovzxdq -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x35,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+pmovzxdq -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pmovzxdq 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x35,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+pmovzxdq 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pmovzxdq 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x35,0x8a,0xf0,0x1c,0xf0,0x1c]
+pmovzxdq 485498096(%edx), %xmm1
+
+// CHECK: pmovzxdq 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x35,0x0d,0xf0,0x1c,0xf0,0x1c]
+pmovzxdq 485498096, %xmm1
+
+// CHECK: pmovzxdq 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x35,0x4c,0x02,0x40]
+pmovzxdq 64(%edx,%eax), %xmm1
+
+// CHECK: pmovzxdq (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x35,0x0a]
+pmovzxdq (%edx), %xmm1
+
+// CHECK: pmovzxdq %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x35,0xc9]
+pmovzxdq %xmm1, %xmm1
+
+// CHECK: pmovzxwd -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x33,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+pmovzxwd -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pmovzxwd 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x33,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+pmovzxwd 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pmovzxwd 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x33,0x8a,0xf0,0x1c,0xf0,0x1c]
+pmovzxwd 485498096(%edx), %xmm1
+
+// CHECK: pmovzxwd 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x33,0x0d,0xf0,0x1c,0xf0,0x1c]
+pmovzxwd 485498096, %xmm1
+
+// CHECK: pmovzxwd 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x33,0x4c,0x02,0x40]
+pmovzxwd 64(%edx,%eax), %xmm1
+
+// CHECK: pmovzxwd (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x33,0x0a]
+pmovzxwd (%edx), %xmm1
+
+// CHECK: pmovzxwd %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x33,0xc9]
+pmovzxwd %xmm1, %xmm1
+
+// CHECK: pmovzxwq -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x34,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+pmovzxwq -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pmovzxwq 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x34,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+pmovzxwq 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pmovzxwq 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x34,0x8a,0xf0,0x1c,0xf0,0x1c]
+pmovzxwq 485498096(%edx), %xmm1
+
+// CHECK: pmovzxwq 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x34,0x0d,0xf0,0x1c,0xf0,0x1c]
+pmovzxwq 485498096, %xmm1
+
+// CHECK: pmovzxwq 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x34,0x4c,0x02,0x40]
+pmovzxwq 64(%edx,%eax), %xmm1
+
+// CHECK: pmovzxwq (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x34,0x0a]
+pmovzxwq (%edx), %xmm1
+
+// CHECK: pmovzxwq %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x34,0xc9]
+pmovzxwq %xmm1, %xmm1
+
+// CHECK: pmuldq -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x28,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+pmuldq -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pmuldq 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x28,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+pmuldq 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pmuldq 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x28,0x8a,0xf0,0x1c,0xf0,0x1c]
+pmuldq 485498096(%edx), %xmm1
+
+// CHECK: pmuldq 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x28,0x0d,0xf0,0x1c,0xf0,0x1c]
+pmuldq 485498096, %xmm1
+
+// CHECK: pmuldq 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x28,0x4c,0x02,0x40]
+pmuldq 64(%edx,%eax), %xmm1
+
+// CHECK: pmuldq (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x28,0x0a]
+pmuldq (%edx), %xmm1
+
+// CHECK: pmuldq %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x28,0xc9]
+pmuldq %xmm1, %xmm1
+
+// CHECK: pmulld -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x40,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+pmulld -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pmulld 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x40,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+pmulld 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pmulld 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x40,0x8a,0xf0,0x1c,0xf0,0x1c]
+pmulld 485498096(%edx), %xmm1
+
+// CHECK: pmulld 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x40,0x0d,0xf0,0x1c,0xf0,0x1c]
+pmulld 485498096, %xmm1
+
+// CHECK: pmulld 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x40,0x4c,0x02,0x40]
+pmulld 64(%edx,%eax), %xmm1
+
+// CHECK: pmulld (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x40,0x0a]
+pmulld (%edx), %xmm1
+
+// CHECK: pmulld %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x40,0xc9]
+pmulld %xmm1, %xmm1
+
+// CHECK: ptest -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x17,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+ptest -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: ptest 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x17,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+ptest 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: ptest 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x17,0x8a,0xf0,0x1c,0xf0,0x1c]
+ptest 485498096(%edx), %xmm1
+
+// CHECK: ptest 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x17,0x0d,0xf0,0x1c,0xf0,0x1c]
+ptest 485498096, %xmm1
+
+// CHECK: ptest 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x17,0x4c,0x02,0x40]
+ptest 64(%edx,%eax), %xmm1
+
+// CHECK: ptest (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x17,0x0a]
+ptest (%edx), %xmm1
+
+// CHECK: ptest %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x17,0xc9]
+ptest %xmm1, %xmm1
+
+// CHECK: roundpd $0, -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x3a,0x09,0x8c,0x82,0x10,0xe3,0x0f,0xe3,0x00]
+roundpd $0, -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: roundpd $0, 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x3a,0x09,0x8c,0x82,0xf0,0x1c,0xf0,0x1c,0x00]
+roundpd $0, 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: roundpd $0, 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x3a,0x09,0x8a,0xf0,0x1c,0xf0,0x1c,0x00]
+roundpd $0, 485498096(%edx), %xmm1
+
+// CHECK: roundpd $0, 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x3a,0x09,0x0d,0xf0,0x1c,0xf0,0x1c,0x00]
+roundpd $0, 485498096, %xmm1
+
+// CHECK: roundpd $0, 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x3a,0x09,0x4c,0x02,0x40,0x00]
+roundpd $0, 64(%edx,%eax), %xmm1
+
+// CHECK: roundpd $0, (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x3a,0x09,0x0a,0x00]
+roundpd $0, (%edx), %xmm1
+
+// CHECK: roundpd $0, %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x3a,0x09,0xc9,0x00]
+roundpd $0, %xmm1, %xmm1
+
+// CHECK: roundps $0, -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x3a,0x08,0x8c,0x82,0x10,0xe3,0x0f,0xe3,0x00]
+roundps $0, -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: roundps $0, 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x3a,0x08,0x8c,0x82,0xf0,0x1c,0xf0,0x1c,0x00]
+roundps $0, 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: roundps $0, 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x3a,0x08,0x8a,0xf0,0x1c,0xf0,0x1c,0x00]
+roundps $0, 485498096(%edx), %xmm1
+
+// CHECK: roundps $0, 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x3a,0x08,0x0d,0xf0,0x1c,0xf0,0x1c,0x00]
+roundps $0, 485498096, %xmm1
+
+// CHECK: roundps $0, 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x3a,0x08,0x4c,0x02,0x40,0x00]
+roundps $0, 64(%edx,%eax), %xmm1
+
+// CHECK: roundps $0, (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x3a,0x08,0x0a,0x00]
+roundps $0, (%edx), %xmm1
+
+// CHECK: roundps $0, %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x3a,0x08,0xc9,0x00]
+roundps $0, %xmm1, %xmm1
+
+// CHECK: roundsd $0, -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x3a,0x0b,0x8c,0x82,0x10,0xe3,0x0f,0xe3,0x00]
+roundsd $0, -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: roundsd $0, 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x3a,0x0b,0x8c,0x82,0xf0,0x1c,0xf0,0x1c,0x00]
+roundsd $0, 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: roundsd $0, 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x3a,0x0b,0x8a,0xf0,0x1c,0xf0,0x1c,0x00]
+roundsd $0, 485498096(%edx), %xmm1
+
+// CHECK: roundsd $0, 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x3a,0x0b,0x0d,0xf0,0x1c,0xf0,0x1c,0x00]
+roundsd $0, 485498096, %xmm1
+
+// CHECK: roundsd $0, 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x3a,0x0b,0x4c,0x02,0x40,0x00]
+roundsd $0, 64(%edx,%eax), %xmm1
+
+// CHECK: roundsd $0, (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x3a,0x0b,0x0a,0x00]
+roundsd $0, (%edx), %xmm1
+
+// CHECK: roundsd $0, %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x3a,0x0b,0xc9,0x00]
+roundsd $0, %xmm1, %xmm1
+
+// CHECK: roundss $0, -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x3a,0x0a,0x8c,0x82,0x10,0xe3,0x0f,0xe3,0x00]
+roundss $0, -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: roundss $0, 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x3a,0x0a,0x8c,0x82,0xf0,0x1c,0xf0,0x1c,0x00]
+roundss $0, 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: roundss $0, 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x3a,0x0a,0x8a,0xf0,0x1c,0xf0,0x1c,0x00]
+roundss $0, 485498096(%edx), %xmm1
+
+// CHECK: roundss $0, 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x3a,0x0a,0x0d,0xf0,0x1c,0xf0,0x1c,0x00]
+roundss $0, 485498096, %xmm1
+
+// CHECK: roundss $0, 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x3a,0x0a,0x4c,0x02,0x40,0x00]
+roundss $0, 64(%edx,%eax), %xmm1
+
+// CHECK: roundss $0, (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x3a,0x0a,0x0a,0x00]
+roundss $0, (%edx), %xmm1
+
+// CHECK: roundss $0, %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x3a,0x0a,0xc9,0x00]
+roundss $0, %xmm1, %xmm1
+
diff --git a/test/MC/X86/SSE41-64.s b/test/MC/X86/SSE41-64.s
new file mode 100644
index 0000000000000000000000000000000000000000..92e796cddd8911bf62c0be2e25fc9a4384cc073a
--- /dev/null
+++ b/test/MC/X86/SSE41-64.s
@@ -0,0 +1,1370 @@
+// RUN: llvm-mc -triple x86_64-unknown-unknown --show-encoding %s | FileCheck %s
+
+// CHECK: blendpd $0, 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x3a,0x0d,0x34,0x25,0xf0,0x1c,0xf0,0x1c,0x00]
+blendpd $0, 485498096, %xmm6
+
+// CHECK: blendpd $0, 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x3a,0x0d,0x74,0x82,0x40,0x00]
+blendpd $0, 64(%rdx,%rax,4), %xmm6
+
+// CHECK: blendpd $0, -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x3a,0x0d,0x74,0x82,0xc0,0x00]
+blendpd $0, -64(%rdx,%rax,4), %xmm6
+
+// CHECK: blendpd $0, 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x3a,0x0d,0x74,0x02,0x40,0x00]
+blendpd $0, 64(%rdx,%rax), %xmm6
+
+// CHECK: blendpd $0, 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x3a,0x0d,0x72,0x40,0x00]
+blendpd $0, 64(%rdx), %xmm6
+
+// CHECK: blendpd $0, (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x3a,0x0d,0x32,0x00]
+blendpd $0, (%rdx), %xmm6
+
+// CHECK: blendpd $0, %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x3a,0x0d,0xf6,0x00]
+blendpd $0, %xmm6, %xmm6
+
+// CHECK: blendps $0, 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x3a,0x0c,0x34,0x25,0xf0,0x1c,0xf0,0x1c,0x00]
+blendps $0, 485498096, %xmm6
+
+// CHECK: blendps $0, 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x3a,0x0c,0x74,0x82,0x40,0x00]
+blendps $0, 64(%rdx,%rax,4), %xmm6
+
+// CHECK: blendps $0, -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x3a,0x0c,0x74,0x82,0xc0,0x00]
+blendps $0, -64(%rdx,%rax,4), %xmm6
+
+// CHECK: blendps $0, 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x3a,0x0c,0x74,0x02,0x40,0x00]
+blendps $0, 64(%rdx,%rax), %xmm6
+
+// CHECK: blendps $0, 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x3a,0x0c,0x72,0x40,0x00]
+blendps $0, 64(%rdx), %xmm6
+
+// CHECK: blendps $0, (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x3a,0x0c,0x32,0x00]
+blendps $0, (%rdx), %xmm6
+
+// CHECK: blendps $0, %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x3a,0x0c,0xf6,0x00]
+blendps $0, %xmm6, %xmm6
+
+// CHECK: blendvpd %xmm0, 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x15,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+blendvpd %xmm0, 485498096, %xmm6
+
+// CHECK: blendvpd %xmm0, 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x15,0x74,0x82,0x40]
+blendvpd %xmm0, 64(%rdx,%rax,4), %xmm6
+
+// CHECK: blendvpd %xmm0, -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x15,0x74,0x82,0xc0]
+blendvpd %xmm0, -64(%rdx,%rax,4), %xmm6
+
+// CHECK: blendvpd %xmm0, 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x15,0x74,0x02,0x40]
+blendvpd %xmm0, 64(%rdx,%rax), %xmm6
+
+// CHECK: blendvpd %xmm0, 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x15,0x72,0x40]
+blendvpd %xmm0, 64(%rdx), %xmm6
+
+// CHECK: blendvpd %xmm0, (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x15,0x32]
+blendvpd %xmm0, (%rdx), %xmm6
+
+// CHECK: blendvpd %xmm0, %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x15,0xf6]
+blendvpd %xmm0, %xmm6, %xmm6
+
+// CHECK: blendvps %xmm0, 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x14,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+blendvps %xmm0, 485498096, %xmm6
+
+// CHECK: blendvps %xmm0, 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x14,0x74,0x82,0x40]
+blendvps %xmm0, 64(%rdx,%rax,4), %xmm6
+
+// CHECK: blendvps %xmm0, -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x14,0x74,0x82,0xc0]
+blendvps %xmm0, -64(%rdx,%rax,4), %xmm6
+
+// CHECK: blendvps %xmm0, 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x14,0x74,0x02,0x40]
+blendvps %xmm0, 64(%rdx,%rax), %xmm6
+
+// CHECK: blendvps %xmm0, 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x14,0x72,0x40]
+blendvps %xmm0, 64(%rdx), %xmm6
+
+// CHECK: blendvps %xmm0, (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x14,0x32]
+blendvps %xmm0, (%rdx), %xmm6
+
+// CHECK: blendvps %xmm0, %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x14,0xf6]
+blendvps %xmm0, %xmm6, %xmm6
+
+// CHECK: dppd $0, 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x3a,0x41,0x34,0x25,0xf0,0x1c,0xf0,0x1c,0x00]
+dppd $0, 485498096, %xmm6
+
+// CHECK: dppd $0, 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x3a,0x41,0x74,0x82,0x40,0x00]
+dppd $0, 64(%rdx,%rax,4), %xmm6
+
+// CHECK: dppd $0, -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x3a,0x41,0x74,0x82,0xc0,0x00]
+dppd $0, -64(%rdx,%rax,4), %xmm6
+
+// CHECK: dppd $0, 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x3a,0x41,0x74,0x02,0x40,0x00]
+dppd $0, 64(%rdx,%rax), %xmm6
+
+// CHECK: dppd $0, 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x3a,0x41,0x72,0x40,0x00]
+dppd $0, 64(%rdx), %xmm6
+
+// CHECK: dppd $0, (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x3a,0x41,0x32,0x00]
+dppd $0, (%rdx), %xmm6
+
+// CHECK: dppd $0, %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x3a,0x41,0xf6,0x00]
+dppd $0, %xmm6, %xmm6
+
+// CHECK: dpps $0, 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x3a,0x40,0x34,0x25,0xf0,0x1c,0xf0,0x1c,0x00]
+dpps $0, 485498096, %xmm6
+
+// CHECK: dpps $0, 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x3a,0x40,0x74,0x82,0x40,0x00]
+dpps $0, 64(%rdx,%rax,4), %xmm6
+
+// CHECK: dpps $0, -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x3a,0x40,0x74,0x82,0xc0,0x00]
+dpps $0, -64(%rdx,%rax,4), %xmm6
+
+// CHECK: dpps $0, 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x3a,0x40,0x74,0x02,0x40,0x00]
+dpps $0, 64(%rdx,%rax), %xmm6
+
+// CHECK: dpps $0, 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x3a,0x40,0x72,0x40,0x00]
+dpps $0, 64(%rdx), %xmm6
+
+// CHECK: dpps $0, (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x3a,0x40,0x32,0x00]
+dpps $0, (%rdx), %xmm6
+
+// CHECK: dpps $0, %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x3a,0x40,0xf6,0x00]
+dpps $0, %xmm6, %xmm6
+
+// CHECK: extractps $0, %xmm6, 485498096
+// CHECK: encoding: [0x66,0x0f,0x3a,0x17,0x34,0x25,0xf0,0x1c,0xf0,0x1c,0x00]
+extractps $0, %xmm6, 485498096
+
+// CHECK: extractps $0, %xmm6, 64(%rdx)
+// CHECK: encoding: [0x66,0x0f,0x3a,0x17,0x72,0x40,0x00]
+extractps $0, %xmm6, 64(%rdx)
+
+// CHECK: extractps $0, %xmm6, 64(%rdx,%rax,4)
+// CHECK: encoding: [0x66,0x0f,0x3a,0x17,0x74,0x82,0x40,0x00]
+extractps $0, %xmm6, 64(%rdx,%rax,4)
+
+// CHECK: extractps $0, %xmm6, -64(%rdx,%rax,4)
+// CHECK: encoding: [0x66,0x0f,0x3a,0x17,0x74,0x82,0xc0,0x00]
+extractps $0, %xmm6, -64(%rdx,%rax,4)
+
+// CHECK: extractps $0, %xmm6, 64(%rdx,%rax)
+// CHECK: encoding: [0x66,0x0f,0x3a,0x17,0x74,0x02,0x40,0x00]
+extractps $0, %xmm6, 64(%rdx,%rax)
+
+// CHECK: extractps $0, %xmm6, %r13d
+// CHECK: encoding: [0x66,0x41,0x0f,0x3a,0x17,0xf5,0x00]
+extractps $0, %xmm6, %r13d
+
+// CHECK: extractps $0, %xmm6, (%rdx)
+// CHECK: encoding: [0x66,0x0f,0x3a,0x17,0x32,0x00]
+extractps $0, %xmm6, (%rdx)
+
+// CHECK: insertps $0, 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x3a,0x21,0x34,0x25,0xf0,0x1c,0xf0,0x1c,0x00]
+insertps $0, 485498096, %xmm6
+
+// CHECK: insertps $0, 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x3a,0x21,0x74,0x82,0x40,0x00]
+insertps $0, 64(%rdx,%rax,4), %xmm6
+
+// CHECK: insertps $0, -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x3a,0x21,0x74,0x82,0xc0,0x00]
+insertps $0, -64(%rdx,%rax,4), %xmm6
+
+// CHECK: insertps $0, 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x3a,0x21,0x74,0x02,0x40,0x00]
+insertps $0, 64(%rdx,%rax), %xmm6
+
+// CHECK: insertps $0, 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x3a,0x21,0x72,0x40,0x00]
+insertps $0, 64(%rdx), %xmm6
+
+// CHECK: insertps $0, (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x3a,0x21,0x32,0x00]
+insertps $0, (%rdx), %xmm6
+
+// CHECK: insertps $0, %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x3a,0x21,0xf6,0x00]
+insertps $0, %xmm6, %xmm6
+
+// CHECK: movntdqa 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x2a,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+movntdqa 485498096, %xmm6
+
+// CHECK: movntdqa 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x2a,0x74,0x82,0x40]
+movntdqa 64(%rdx,%rax,4), %xmm6
+
+// CHECK: movntdqa -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x2a,0x74,0x82,0xc0]
+movntdqa -64(%rdx,%rax,4), %xmm6
+
+// CHECK: movntdqa 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x2a,0x74,0x02,0x40]
+movntdqa 64(%rdx,%rax), %xmm6
+
+// CHECK: movntdqa 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x2a,0x72,0x40]
+movntdqa 64(%rdx), %xmm6
+
+// CHECK: movntdqa (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x2a,0x32]
+movntdqa (%rdx), %xmm6
+
+// CHECK: mpsadbw $0, 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x3a,0x42,0x34,0x25,0xf0,0x1c,0xf0,0x1c,0x00]
+mpsadbw $0, 485498096, %xmm6
+
+// CHECK: mpsadbw $0, 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x3a,0x42,0x74,0x82,0x40,0x00]
+mpsadbw $0, 64(%rdx,%rax,4), %xmm6
+
+// CHECK: mpsadbw $0, -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x3a,0x42,0x74,0x82,0xc0,0x00]
+mpsadbw $0, -64(%rdx,%rax,4), %xmm6
+
+// CHECK: mpsadbw $0, 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x3a,0x42,0x74,0x02,0x40,0x00]
+mpsadbw $0, 64(%rdx,%rax), %xmm6
+
+// CHECK: mpsadbw $0, 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x3a,0x42,0x72,0x40,0x00]
+mpsadbw $0, 64(%rdx), %xmm6
+
+// CHECK: mpsadbw $0, (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x3a,0x42,0x32,0x00]
+mpsadbw $0, (%rdx), %xmm6
+
+// CHECK: mpsadbw $0, %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x3a,0x42,0xf6,0x00]
+mpsadbw $0, %xmm6, %xmm6
+
+// CHECK: packusdw 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x2b,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+packusdw 485498096, %xmm6
+
+// CHECK: packusdw 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x2b,0x74,0x82,0x40]
+packusdw 64(%rdx,%rax,4), %xmm6
+
+// CHECK: packusdw -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x2b,0x74,0x82,0xc0]
+packusdw -64(%rdx,%rax,4), %xmm6
+
+// CHECK: packusdw 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x2b,0x74,0x02,0x40]
+packusdw 64(%rdx,%rax), %xmm6
+
+// CHECK: packusdw 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x2b,0x72,0x40]
+packusdw 64(%rdx), %xmm6
+
+// CHECK: packusdw (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x2b,0x32]
+packusdw (%rdx), %xmm6
+
+// CHECK: packusdw %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x2b,0xf6]
+packusdw %xmm6, %xmm6
+
+// CHECK: pblendvb %xmm0, 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x10,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+pblendvb %xmm0, 485498096, %xmm6
+
+// CHECK: pblendvb %xmm0, 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x10,0x74,0x82,0x40]
+pblendvb %xmm0, 64(%rdx,%rax,4), %xmm6
+
+// CHECK: pblendvb %xmm0, -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x10,0x74,0x82,0xc0]
+pblendvb %xmm0, -64(%rdx,%rax,4), %xmm6
+
+// CHECK: pblendvb %xmm0, 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x10,0x74,0x02,0x40]
+pblendvb %xmm0, 64(%rdx,%rax), %xmm6
+
+// CHECK: pblendvb %xmm0, 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x10,0x72,0x40]
+pblendvb %xmm0, 64(%rdx), %xmm6
+
+// CHECK: pblendvb %xmm0, (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x10,0x32]
+pblendvb %xmm0, (%rdx), %xmm6
+
+// CHECK: pblendvb %xmm0, %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x10,0xf6]
+pblendvb %xmm0, %xmm6, %xmm6
+
+// CHECK: pblendw $0, 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x3a,0x0e,0x34,0x25,0xf0,0x1c,0xf0,0x1c,0x00]
+pblendw $0, 485498096, %xmm6
+
+// CHECK: pblendw $0, 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x3a,0x0e,0x74,0x82,0x40,0x00]
+pblendw $0, 64(%rdx,%rax,4), %xmm6
+
+// CHECK: pblendw $0, -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x3a,0x0e,0x74,0x82,0xc0,0x00]
+pblendw $0, -64(%rdx,%rax,4), %xmm6
+
+// CHECK: pblendw $0, 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x3a,0x0e,0x74,0x02,0x40,0x00]
+pblendw $0, 64(%rdx,%rax), %xmm6
+
+// CHECK: pblendw $0, 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x3a,0x0e,0x72,0x40,0x00]
+pblendw $0, 64(%rdx), %xmm6
+
+// CHECK: pblendw $0, (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x3a,0x0e,0x32,0x00]
+pblendw $0, (%rdx), %xmm6
+
+// CHECK: pblendw $0, %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x3a,0x0e,0xf6,0x00]
+pblendw $0, %xmm6, %xmm6
+
+// CHECK: pcmpeqq 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x29,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+pcmpeqq 485498096, %xmm6
+
+// CHECK: pcmpeqq 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x29,0x74,0x82,0x40]
+pcmpeqq 64(%rdx,%rax,4), %xmm6
+
+// CHECK: pcmpeqq -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x29,0x74,0x82,0xc0]
+pcmpeqq -64(%rdx,%rax,4), %xmm6
+
+// CHECK: pcmpeqq 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x29,0x74,0x02,0x40]
+pcmpeqq 64(%rdx,%rax), %xmm6
+
+// CHECK: pcmpeqq 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x29,0x72,0x40]
+pcmpeqq 64(%rdx), %xmm6
+
+// CHECK: pcmpeqq (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x29,0x32]
+pcmpeqq (%rdx), %xmm6
+
+// CHECK: pcmpeqq %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x29,0xf6]
+pcmpeqq %xmm6, %xmm6
+
+// CHECK: pextrb $0, %xmm6, 485498096
+// CHECK: encoding: [0x66,0x0f,0x3a,0x14,0x34,0x25,0xf0,0x1c,0xf0,0x1c,0x00]
+pextrb $0, %xmm6, 485498096
+
+// CHECK: pextrb $0, %xmm6, 64(%rdx)
+// CHECK: encoding: [0x66,0x0f,0x3a,0x14,0x72,0x40,0x00]
+pextrb $0, %xmm6, 64(%rdx)
+
+// CHECK: pextrb $0, %xmm6, 64(%rdx,%rax,4)
+// CHECK: encoding: [0x66,0x0f,0x3a,0x14,0x74,0x82,0x40,0x00]
+pextrb $0, %xmm6, 64(%rdx,%rax,4)
+
+// CHECK: pextrb $0, %xmm6, -64(%rdx,%rax,4)
+// CHECK: encoding: [0x66,0x0f,0x3a,0x14,0x74,0x82,0xc0,0x00]
+pextrb $0, %xmm6, -64(%rdx,%rax,4)
+
+// CHECK: pextrb $0, %xmm6, 64(%rdx,%rax)
+// CHECK: encoding: [0x66,0x0f,0x3a,0x14,0x74,0x02,0x40,0x00]
+pextrb $0, %xmm6, 64(%rdx,%rax)
+
+// CHECK: pextrb $0, %xmm6, %r13d
+// CHECK: encoding: [0x66,0x41,0x0f,0x3a,0x14,0xf5,0x00]
+pextrb $0, %xmm6, %r13d
+
+// CHECK: pextrb $0, %xmm6, (%rdx)
+// CHECK: encoding: [0x66,0x0f,0x3a,0x14,0x32,0x00]
+pextrb $0, %xmm6, (%rdx)
+
+// CHECK: pextrd $0, %xmm6, 485498096
+// CHECK: encoding: [0x66,0x0f,0x3a,0x16,0x34,0x25,0xf0,0x1c,0xf0,0x1c,0x00]
+pextrd $0, %xmm6, 485498096
+
+// CHECK: pextrd $0, %xmm6, 64(%rdx)
+// CHECK: encoding: [0x66,0x0f,0x3a,0x16,0x72,0x40,0x00]
+pextrd $0, %xmm6, 64(%rdx)
+
+// CHECK: pextrd $0, %xmm6, 64(%rdx,%rax,4)
+// CHECK: encoding: [0x66,0x0f,0x3a,0x16,0x74,0x82,0x40,0x00]
+pextrd $0, %xmm6, 64(%rdx,%rax,4)
+
+// CHECK: pextrd $0, %xmm6, -64(%rdx,%rax,4)
+// CHECK: encoding: [0x66,0x0f,0x3a,0x16,0x74,0x82,0xc0,0x00]
+pextrd $0, %xmm6, -64(%rdx,%rax,4)
+
+// CHECK: pextrd $0, %xmm6, 64(%rdx,%rax)
+// CHECK: encoding: [0x66,0x0f,0x3a,0x16,0x74,0x02,0x40,0x00]
+pextrd $0, %xmm6, 64(%rdx,%rax)
+
+// CHECK: pextrd $0, %xmm6, %r13d
+// CHECK: encoding: [0x66,0x41,0x0f,0x3a,0x16,0xf5,0x00]
+pextrd $0, %xmm6, %r13d
+
+// CHECK: pextrd $0, %xmm6, (%rdx)
+// CHECK: encoding: [0x66,0x0f,0x3a,0x16,0x32,0x00]
+pextrd $0, %xmm6, (%rdx)
+
+// CHECK: pextrq $0, %xmm6, 485498096
+// CHECK: encoding: [0x66,0x48,0x0f,0x3a,0x16,0x34,0x25,0xf0,0x1c,0xf0,0x1c,0x00]
+pextrq $0, %xmm6, 485498096
+
+// CHECK: pextrq $0, %xmm6, 64(%rdx)
+// CHECK: encoding: [0x66,0x48,0x0f,0x3a,0x16,0x72,0x40,0x00]
+pextrq $0, %xmm6, 64(%rdx)
+
+// CHECK: pextrq $0, %xmm6, 64(%rdx,%rax,4)
+// CHECK: encoding: [0x66,0x48,0x0f,0x3a,0x16,0x74,0x82,0x40,0x00]
+pextrq $0, %xmm6, 64(%rdx,%rax,4)
+
+// CHECK: pextrq $0, %xmm6, -64(%rdx,%rax,4)
+// CHECK: encoding: [0x66,0x48,0x0f,0x3a,0x16,0x74,0x82,0xc0,0x00]
+pextrq $0, %xmm6, -64(%rdx,%rax,4)
+
+// CHECK: pextrq $0, %xmm6, 64(%rdx,%rax)
+// CHECK: encoding: [0x66,0x48,0x0f,0x3a,0x16,0x74,0x02,0x40,0x00]
+pextrq $0, %xmm6, 64(%rdx,%rax)
+
+// CHECK: pextrq $0, %xmm6, %r15
+// CHECK: encoding: [0x66,0x49,0x0f,0x3a,0x16,0xf7,0x00]
+pextrq $0, %xmm6, %r15
+
+// CHECK: pextrq $0, %xmm6, (%rdx)
+// CHECK: encoding: [0x66,0x48,0x0f,0x3a,0x16,0x32,0x00]
+pextrq $0, %xmm6, (%rdx)
+
+// CHECK: pextrw $0, %xmm6, 485498096
+// CHECK: encoding: [0x66,0x0f,0x3a,0x15,0x34,0x25,0xf0,0x1c,0xf0,0x1c,0x00]
+pextrw $0, %xmm6, 485498096
+
+// CHECK: pextrw $0, %xmm6, 64(%rdx)
+// CHECK: encoding: [0x66,0x0f,0x3a,0x15,0x72,0x40,0x00]
+pextrw $0, %xmm6, 64(%rdx)
+
+// CHECK: pextrw $0, %xmm6, 64(%rdx,%rax,4)
+// CHECK: encoding: [0x66,0x0f,0x3a,0x15,0x74,0x82,0x40,0x00]
+pextrw $0, %xmm6, 64(%rdx,%rax,4)
+
+// CHECK: pextrw $0, %xmm6, -64(%rdx,%rax,4)
+// CHECK: encoding: [0x66,0x0f,0x3a,0x15,0x74,0x82,0xc0,0x00]
+pextrw $0, %xmm6, -64(%rdx,%rax,4)
+
+// CHECK: pextrw $0, %xmm6, 64(%rdx,%rax)
+// CHECK: encoding: [0x66,0x0f,0x3a,0x15,0x74,0x02,0x40,0x00]
+pextrw $0, %xmm6, 64(%rdx,%rax)
+
+// CHECK: pextrw $0, %xmm6, %r13d
+// CHECK: encoding: [0x66,0x44,0x0f,0xc5,0xee,0x00]
+pextrw $0, %xmm6, %r13d
+
+// CHECK: pextrw $0, %xmm6, (%rdx)
+// CHECK: encoding: [0x66,0x0f,0x3a,0x15,0x32,0x00]
+pextrw $0, %xmm6, (%rdx)
+
+// CHECK: phminposuw 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x41,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+phminposuw 485498096, %xmm6
+
+// CHECK: phminposuw 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x41,0x74,0x82,0x40]
+phminposuw 64(%rdx,%rax,4), %xmm6
+
+// CHECK: phminposuw -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x41,0x74,0x82,0xc0]
+phminposuw -64(%rdx,%rax,4), %xmm6
+
+// CHECK: phminposuw 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x41,0x74,0x02,0x40]
+phminposuw 64(%rdx,%rax), %xmm6
+
+// CHECK: phminposuw 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x41,0x72,0x40]
+phminposuw 64(%rdx), %xmm6
+
+// CHECK: phminposuw (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x41,0x32]
+phminposuw (%rdx), %xmm6
+
+// CHECK: phminposuw %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x41,0xf6]
+phminposuw %xmm6, %xmm6
+
+// CHECK: pinsrb $0, 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x3a,0x20,0x34,0x25,0xf0,0x1c,0xf0,0x1c,0x00]
+pinsrb $0, 485498096, %xmm6
+
+// CHECK: pinsrb $0, 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x3a,0x20,0x74,0x82,0x40,0x00]
+pinsrb $0, 64(%rdx,%rax,4), %xmm6
+
+// CHECK: pinsrb $0, -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x3a,0x20,0x74,0x82,0xc0,0x00]
+pinsrb $0, -64(%rdx,%rax,4), %xmm6
+
+// CHECK: pinsrb $0, 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x3a,0x20,0x74,0x02,0x40,0x00]
+pinsrb $0, 64(%rdx,%rax), %xmm6
+
+// CHECK: pinsrb $0, 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x3a,0x20,0x72,0x40,0x00]
+pinsrb $0, 64(%rdx), %xmm6
+
+// CHECK: pinsrb $0, %r13d, %xmm6
+// CHECK: encoding: [0x66,0x41,0x0f,0x3a,0x20,0xf5,0x00]
+pinsrb $0, %r13d, %xmm6
+
+// CHECK: pinsrb $0, (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x3a,0x20,0x32,0x00]
+pinsrb $0, (%rdx), %xmm6
+
+// CHECK: pinsrd $0, 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x3a,0x22,0x34,0x25,0xf0,0x1c,0xf0,0x1c,0x00]
+pinsrd $0, 485498096, %xmm6
+
+// CHECK: pinsrd $0, 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x3a,0x22,0x74,0x82,0x40,0x00]
+pinsrd $0, 64(%rdx,%rax,4), %xmm6
+
+// CHECK: pinsrd $0, -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x3a,0x22,0x74,0x82,0xc0,0x00]
+pinsrd $0, -64(%rdx,%rax,4), %xmm6
+
+// CHECK: pinsrd $0, 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x3a,0x22,0x74,0x02,0x40,0x00]
+pinsrd $0, 64(%rdx,%rax), %xmm6
+
+// CHECK: pinsrd $0, 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x3a,0x22,0x72,0x40,0x00]
+pinsrd $0, 64(%rdx), %xmm6
+
+// CHECK: pinsrd $0, %r13d, %xmm6
+// CHECK: encoding: [0x66,0x41,0x0f,0x3a,0x22,0xf5,0x00]
+pinsrd $0, %r13d, %xmm6
+
+// CHECK: pinsrd $0, (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x3a,0x22,0x32,0x00]
+pinsrd $0, (%rdx), %xmm6
+
+// CHECK: pinsrq $0, 485498096, %xmm6
+// CHECK: encoding: [0x66,0x48,0x0f,0x3a,0x22,0x34,0x25,0xf0,0x1c,0xf0,0x1c,0x00]
+pinsrq $0, 485498096, %xmm6
+
+// CHECK: pinsrq $0, 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x48,0x0f,0x3a,0x22,0x74,0x82,0x40,0x00]
+pinsrq $0, 64(%rdx,%rax,4), %xmm6
+
+// CHECK: pinsrq $0, -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x48,0x0f,0x3a,0x22,0x74,0x82,0xc0,0x00]
+pinsrq $0, -64(%rdx,%rax,4), %xmm6
+
+// CHECK: pinsrq $0, 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x48,0x0f,0x3a,0x22,0x74,0x02,0x40,0x00]
+pinsrq $0, 64(%rdx,%rax), %xmm6
+
+// CHECK: pinsrq $0, 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x48,0x0f,0x3a,0x22,0x72,0x40,0x00]
+pinsrq $0, 64(%rdx), %xmm6
+
+// CHECK: pinsrq $0, %r15, %xmm6
+// CHECK: encoding: [0x66,0x49,0x0f,0x3a,0x22,0xf7,0x00]
+pinsrq $0, %r15, %xmm6
+
+// CHECK: pinsrq $0, (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x48,0x0f,0x3a,0x22,0x32,0x00]
+pinsrq $0, (%rdx), %xmm6
+
+// CHECK: pmaxsb 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x3c,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+pmaxsb 485498096, %xmm6
+
+// CHECK: pmaxsb 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x3c,0x74,0x82,0x40]
+pmaxsb 64(%rdx,%rax,4), %xmm6
+
+// CHECK: pmaxsb -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x3c,0x74,0x82,0xc0]
+pmaxsb -64(%rdx,%rax,4), %xmm6
+
+// CHECK: pmaxsb 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x3c,0x74,0x02,0x40]
+pmaxsb 64(%rdx,%rax), %xmm6
+
+// CHECK: pmaxsb 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x3c,0x72,0x40]
+pmaxsb 64(%rdx), %xmm6
+
+// CHECK: pmaxsb (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x3c,0x32]
+pmaxsb (%rdx), %xmm6
+
+// CHECK: pmaxsb %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x3c,0xf6]
+pmaxsb %xmm6, %xmm6
+
+// CHECK: pmaxsd 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x3d,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+pmaxsd 485498096, %xmm6
+
+// CHECK: pmaxsd 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x3d,0x74,0x82,0x40]
+pmaxsd 64(%rdx,%rax,4), %xmm6
+
+// CHECK: pmaxsd -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x3d,0x74,0x82,0xc0]
+pmaxsd -64(%rdx,%rax,4), %xmm6
+
+// CHECK: pmaxsd 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x3d,0x74,0x02,0x40]
+pmaxsd 64(%rdx,%rax), %xmm6
+
+// CHECK: pmaxsd 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x3d,0x72,0x40]
+pmaxsd 64(%rdx), %xmm6
+
+// CHECK: pmaxsd (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x3d,0x32]
+pmaxsd (%rdx), %xmm6
+
+// CHECK: pmaxsd %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x3d,0xf6]
+pmaxsd %xmm6, %xmm6
+
+// CHECK: pmaxud 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x3f,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+pmaxud 485498096, %xmm6
+
+// CHECK: pmaxud 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x3f,0x74,0x82,0x40]
+pmaxud 64(%rdx,%rax,4), %xmm6
+
+// CHECK: pmaxud -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x3f,0x74,0x82,0xc0]
+pmaxud -64(%rdx,%rax,4), %xmm6
+
+// CHECK: pmaxud 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x3f,0x74,0x02,0x40]
+pmaxud 64(%rdx,%rax), %xmm6
+
+// CHECK: pmaxud 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x3f,0x72,0x40]
+pmaxud 64(%rdx), %xmm6
+
+// CHECK: pmaxud (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x3f,0x32]
+pmaxud (%rdx), %xmm6
+
+// CHECK: pmaxud %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x3f,0xf6]
+pmaxud %xmm6, %xmm6
+
+// CHECK: pmaxuw 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x3e,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+pmaxuw 485498096, %xmm6
+
+// CHECK: pmaxuw 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x3e,0x74,0x82,0x40]
+pmaxuw 64(%rdx,%rax,4), %xmm6
+
+// CHECK: pmaxuw -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x3e,0x74,0x82,0xc0]
+pmaxuw -64(%rdx,%rax,4), %xmm6
+
+// CHECK: pmaxuw 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x3e,0x74,0x02,0x40]
+pmaxuw 64(%rdx,%rax), %xmm6
+
+// CHECK: pmaxuw 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x3e,0x72,0x40]
+pmaxuw 64(%rdx), %xmm6
+
+// CHECK: pmaxuw (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x3e,0x32]
+pmaxuw (%rdx), %xmm6
+
+// CHECK: pmaxuw %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x3e,0xf6]
+pmaxuw %xmm6, %xmm6
+
+// CHECK: pminsb 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x38,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+pminsb 485498096, %xmm6
+
+// CHECK: pminsb 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x38,0x74,0x82,0x40]
+pminsb 64(%rdx,%rax,4), %xmm6
+
+// CHECK: pminsb -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x38,0x74,0x82,0xc0]
+pminsb -64(%rdx,%rax,4), %xmm6
+
+// CHECK: pminsb 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x38,0x74,0x02,0x40]
+pminsb 64(%rdx,%rax), %xmm6
+
+// CHECK: pminsb 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x38,0x72,0x40]
+pminsb 64(%rdx), %xmm6
+
+// CHECK: pminsb (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x38,0x32]
+pminsb (%rdx), %xmm6
+
+// CHECK: pminsb %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x38,0xf6]
+pminsb %xmm6, %xmm6
+
+// CHECK: pminsd 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x39,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+pminsd 485498096, %xmm6
+
+// CHECK: pminsd 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x39,0x74,0x82,0x40]
+pminsd 64(%rdx,%rax,4), %xmm6
+
+// CHECK: pminsd -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x39,0x74,0x82,0xc0]
+pminsd -64(%rdx,%rax,4), %xmm6
+
+// CHECK: pminsd 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x39,0x74,0x02,0x40]
+pminsd 64(%rdx,%rax), %xmm6
+
+// CHECK: pminsd 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x39,0x72,0x40]
+pminsd 64(%rdx), %xmm6
+
+// CHECK: pminsd (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x39,0x32]
+pminsd (%rdx), %xmm6
+
+// CHECK: pminsd %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x39,0xf6]
+pminsd %xmm6, %xmm6
+
+// CHECK: pminud 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x3b,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+pminud 485498096, %xmm6
+
+// CHECK: pminud 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x3b,0x74,0x82,0x40]
+pminud 64(%rdx,%rax,4), %xmm6
+
+// CHECK: pminud -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x3b,0x74,0x82,0xc0]
+pminud -64(%rdx,%rax,4), %xmm6
+
+// CHECK: pminud 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x3b,0x74,0x02,0x40]
+pminud 64(%rdx,%rax), %xmm6
+
+// CHECK: pminud 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x3b,0x72,0x40]
+pminud 64(%rdx), %xmm6
+
+// CHECK: pminud (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x3b,0x32]
+pminud (%rdx), %xmm6
+
+// CHECK: pminud %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x3b,0xf6]
+pminud %xmm6, %xmm6
+
+// CHECK: pminuw 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x3a,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+pminuw 485498096, %xmm6
+
+// CHECK: pminuw 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x3a,0x74,0x82,0x40]
+pminuw 64(%rdx,%rax,4), %xmm6
+
+// CHECK: pminuw -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x3a,0x74,0x82,0xc0]
+pminuw -64(%rdx,%rax,4), %xmm6
+
+// CHECK: pminuw 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x3a,0x74,0x02,0x40]
+pminuw 64(%rdx,%rax), %xmm6
+
+// CHECK: pminuw 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x3a,0x72,0x40]
+pminuw 64(%rdx), %xmm6
+
+// CHECK: pminuw (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x3a,0x32]
+pminuw (%rdx), %xmm6
+
+// CHECK: pminuw %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x3a,0xf6]
+pminuw %xmm6, %xmm6
+
+// CHECK: pmovsxbd 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x21,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+pmovsxbd 485498096, %xmm6
+
+// CHECK: pmovsxbd 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x21,0x74,0x82,0x40]
+pmovsxbd 64(%rdx,%rax,4), %xmm6
+
+// CHECK: pmovsxbd -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x21,0x74,0x82,0xc0]
+pmovsxbd -64(%rdx,%rax,4), %xmm6
+
+// CHECK: pmovsxbd 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x21,0x74,0x02,0x40]
+pmovsxbd 64(%rdx,%rax), %xmm6
+
+// CHECK: pmovsxbd 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x21,0x72,0x40]
+pmovsxbd 64(%rdx), %xmm6
+
+// CHECK: pmovsxbd (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x21,0x32]
+pmovsxbd (%rdx), %xmm6
+
+// CHECK: pmovsxbd %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x21,0xf6]
+pmovsxbd %xmm6, %xmm6
+
+// CHECK: pmovsxbq 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x22,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+pmovsxbq 485498096, %xmm6
+
+// CHECK: pmovsxbq 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x22,0x74,0x82,0x40]
+pmovsxbq 64(%rdx,%rax,4), %xmm6
+
+// CHECK: pmovsxbq -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x22,0x74,0x82,0xc0]
+pmovsxbq -64(%rdx,%rax,4), %xmm6
+
+// CHECK: pmovsxbq 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x22,0x74,0x02,0x40]
+pmovsxbq 64(%rdx,%rax), %xmm6
+
+// CHECK: pmovsxbq 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x22,0x72,0x40]
+pmovsxbq 64(%rdx), %xmm6
+
+// CHECK: pmovsxbq (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x22,0x32]
+pmovsxbq (%rdx), %xmm6
+
+// CHECK: pmovsxbq %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x22,0xf6]
+pmovsxbq %xmm6, %xmm6
+
+// CHECK: pmovsxbw 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x20,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+pmovsxbw 485498096, %xmm6
+
+// CHECK: pmovsxbw 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x20,0x74,0x82,0x40]
+pmovsxbw 64(%rdx,%rax,4), %xmm6
+
+// CHECK: pmovsxbw -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x20,0x74,0x82,0xc0]
+pmovsxbw -64(%rdx,%rax,4), %xmm6
+
+// CHECK: pmovsxbw 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x20,0x74,0x02,0x40]
+pmovsxbw 64(%rdx,%rax), %xmm6
+
+// CHECK: pmovsxbw 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x20,0x72,0x40]
+pmovsxbw 64(%rdx), %xmm6
+
+// CHECK: pmovsxbw (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x20,0x32]
+pmovsxbw (%rdx), %xmm6
+
+// CHECK: pmovsxbw %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x20,0xf6]
+pmovsxbw %xmm6, %xmm6
+
+// CHECK: pmovsxdq 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x25,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+pmovsxdq 485498096, %xmm6
+
+// CHECK: pmovsxdq 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x25,0x74,0x82,0x40]
+pmovsxdq 64(%rdx,%rax,4), %xmm6
+
+// CHECK: pmovsxdq -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x25,0x74,0x82,0xc0]
+pmovsxdq -64(%rdx,%rax,4), %xmm6
+
+// CHECK: pmovsxdq 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x25,0x74,0x02,0x40]
+pmovsxdq 64(%rdx,%rax), %xmm6
+
+// CHECK: pmovsxdq 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x25,0x72,0x40]
+pmovsxdq 64(%rdx), %xmm6
+
+// CHECK: pmovsxdq (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x25,0x32]
+pmovsxdq (%rdx), %xmm6
+
+// CHECK: pmovsxdq %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x25,0xf6]
+pmovsxdq %xmm6, %xmm6
+
+// CHECK: pmovsxwd 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x23,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+pmovsxwd 485498096, %xmm6
+
+// CHECK: pmovsxwd 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x23,0x74,0x82,0x40]
+pmovsxwd 64(%rdx,%rax,4), %xmm6
+
+// CHECK: pmovsxwd -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x23,0x74,0x82,0xc0]
+pmovsxwd -64(%rdx,%rax,4), %xmm6
+
+// CHECK: pmovsxwd 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x23,0x74,0x02,0x40]
+pmovsxwd 64(%rdx,%rax), %xmm6
+
+// CHECK: pmovsxwd 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x23,0x72,0x40]
+pmovsxwd 64(%rdx), %xmm6
+
+// CHECK: pmovsxwd (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x23,0x32]
+pmovsxwd (%rdx), %xmm6
+
+// CHECK: pmovsxwd %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x23,0xf6]
+pmovsxwd %xmm6, %xmm6
+
+// CHECK: pmovsxwq 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x24,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+pmovsxwq 485498096, %xmm6
+
+// CHECK: pmovsxwq 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x24,0x74,0x82,0x40]
+pmovsxwq 64(%rdx,%rax,4), %xmm6
+
+// CHECK: pmovsxwq -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x24,0x74,0x82,0xc0]
+pmovsxwq -64(%rdx,%rax,4), %xmm6
+
+// CHECK: pmovsxwq 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x24,0x74,0x02,0x40]
+pmovsxwq 64(%rdx,%rax), %xmm6
+
+// CHECK: pmovsxwq 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x24,0x72,0x40]
+pmovsxwq 64(%rdx), %xmm6
+
+// CHECK: pmovsxwq (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x24,0x32]
+pmovsxwq (%rdx), %xmm6
+
+// CHECK: pmovsxwq %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x24,0xf6]
+pmovsxwq %xmm6, %xmm6
+
+// CHECK: pmovzxbd 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x31,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+pmovzxbd 485498096, %xmm6
+
+// CHECK: pmovzxbd 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x31,0x74,0x82,0x40]
+pmovzxbd 64(%rdx,%rax,4), %xmm6
+
+// CHECK: pmovzxbd -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x31,0x74,0x82,0xc0]
+pmovzxbd -64(%rdx,%rax,4), %xmm6
+
+// CHECK: pmovzxbd 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x31,0x74,0x02,0x40]
+pmovzxbd 64(%rdx,%rax), %xmm6
+
+// CHECK: pmovzxbd 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x31,0x72,0x40]
+pmovzxbd 64(%rdx), %xmm6
+
+// CHECK: pmovzxbd (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x31,0x32]
+pmovzxbd (%rdx), %xmm6
+
+// CHECK: pmovzxbd %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x31,0xf6]
+pmovzxbd %xmm6, %xmm6
+
+// CHECK: pmovzxbq 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x32,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+pmovzxbq 485498096, %xmm6
+
+// CHECK: pmovzxbq 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x32,0x74,0x82,0x40]
+pmovzxbq 64(%rdx,%rax,4), %xmm6
+
+// CHECK: pmovzxbq -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x32,0x74,0x82,0xc0]
+pmovzxbq -64(%rdx,%rax,4), %xmm6
+
+// CHECK: pmovzxbq 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x32,0x74,0x02,0x40]
+pmovzxbq 64(%rdx,%rax), %xmm6
+
+// CHECK: pmovzxbq 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x32,0x72,0x40]
+pmovzxbq 64(%rdx), %xmm6
+
+// CHECK: pmovzxbq (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x32,0x32]
+pmovzxbq (%rdx), %xmm6
+
+// CHECK: pmovzxbq %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x32,0xf6]
+pmovzxbq %xmm6, %xmm6
+
+// CHECK: pmovzxbw 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x30,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+pmovzxbw 485498096, %xmm6
+
+// CHECK: pmovzxbw 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x30,0x74,0x82,0x40]
+pmovzxbw 64(%rdx,%rax,4), %xmm6
+
+// CHECK: pmovzxbw -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x30,0x74,0x82,0xc0]
+pmovzxbw -64(%rdx,%rax,4), %xmm6
+
+// CHECK: pmovzxbw 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x30,0x74,0x02,0x40]
+pmovzxbw 64(%rdx,%rax), %xmm6
+
+// CHECK: pmovzxbw 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x30,0x72,0x40]
+pmovzxbw 64(%rdx), %xmm6
+
+// CHECK: pmovzxbw (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x30,0x32]
+pmovzxbw (%rdx), %xmm6
+
+// CHECK: pmovzxbw %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x30,0xf6]
+pmovzxbw %xmm6, %xmm6
+
+// CHECK: pmovzxdq 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x35,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+pmovzxdq 485498096, %xmm6
+
+// CHECK: pmovzxdq 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x35,0x74,0x82,0x40]
+pmovzxdq 64(%rdx,%rax,4), %xmm6
+
+// CHECK: pmovzxdq -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x35,0x74,0x82,0xc0]
+pmovzxdq -64(%rdx,%rax,4), %xmm6
+
+// CHECK: pmovzxdq 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x35,0x74,0x02,0x40]
+pmovzxdq 64(%rdx,%rax), %xmm6
+
+// CHECK: pmovzxdq 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x35,0x72,0x40]
+pmovzxdq 64(%rdx), %xmm6
+
+// CHECK: pmovzxdq (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x35,0x32]
+pmovzxdq (%rdx), %xmm6
+
+// CHECK: pmovzxdq %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x35,0xf6]
+pmovzxdq %xmm6, %xmm6
+
+// CHECK: pmovzxwd 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x33,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+pmovzxwd 485498096, %xmm6
+
+// CHECK: pmovzxwd 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x33,0x74,0x82,0x40]
+pmovzxwd 64(%rdx,%rax,4), %xmm6
+
+// CHECK: pmovzxwd -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x33,0x74,0x82,0xc0]
+pmovzxwd -64(%rdx,%rax,4), %xmm6
+
+// CHECK: pmovzxwd 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x33,0x74,0x02,0x40]
+pmovzxwd 64(%rdx,%rax), %xmm6
+
+// CHECK: pmovzxwd 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x33,0x72,0x40]
+pmovzxwd 64(%rdx), %xmm6
+
+// CHECK: pmovzxwd (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x33,0x32]
+pmovzxwd (%rdx), %xmm6
+
+// CHECK: pmovzxwd %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x33,0xf6]
+pmovzxwd %xmm6, %xmm6
+
+// CHECK: pmovzxwq 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x34,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+pmovzxwq 485498096, %xmm6
+
+// CHECK: pmovzxwq 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x34,0x74,0x82,0x40]
+pmovzxwq 64(%rdx,%rax,4), %xmm6
+
+// CHECK: pmovzxwq -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x34,0x74,0x82,0xc0]
+pmovzxwq -64(%rdx,%rax,4), %xmm6
+
+// CHECK: pmovzxwq 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x34,0x74,0x02,0x40]
+pmovzxwq 64(%rdx,%rax), %xmm6
+
+// CHECK: pmovzxwq 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x34,0x72,0x40]
+pmovzxwq 64(%rdx), %xmm6
+
+// CHECK: pmovzxwq (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x34,0x32]
+pmovzxwq (%rdx), %xmm6
+
+// CHECK: pmovzxwq %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x34,0xf6]
+pmovzxwq %xmm6, %xmm6
+
+// CHECK: pmuldq 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x28,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+pmuldq 485498096, %xmm6
+
+// CHECK: pmuldq 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x28,0x74,0x82,0x40]
+pmuldq 64(%rdx,%rax,4), %xmm6
+
+// CHECK: pmuldq -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x28,0x74,0x82,0xc0]
+pmuldq -64(%rdx,%rax,4), %xmm6
+
+// CHECK: pmuldq 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x28,0x74,0x02,0x40]
+pmuldq 64(%rdx,%rax), %xmm6
+
+// CHECK: pmuldq 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x28,0x72,0x40]
+pmuldq 64(%rdx), %xmm6
+
+// CHECK: pmuldq (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x28,0x32]
+pmuldq (%rdx), %xmm6
+
+// CHECK: pmuldq %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x28,0xf6]
+pmuldq %xmm6, %xmm6
+
+// CHECK: pmulld 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x40,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+pmulld 485498096, %xmm6
+
+// CHECK: pmulld 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x40,0x74,0x82,0x40]
+pmulld 64(%rdx,%rax,4), %xmm6
+
+// CHECK: pmulld -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x40,0x74,0x82,0xc0]
+pmulld -64(%rdx,%rax,4), %xmm6
+
+// CHECK: pmulld 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x40,0x74,0x02,0x40]
+pmulld 64(%rdx,%rax), %xmm6
+
+// CHECK: pmulld 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x40,0x72,0x40]
+pmulld 64(%rdx), %xmm6
+
+// CHECK: pmulld (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x40,0x32]
+pmulld (%rdx), %xmm6
+
+// CHECK: pmulld %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x40,0xf6]
+pmulld %xmm6, %xmm6
+
+// CHECK: ptest 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x17,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+ptest 485498096, %xmm6
+
+// CHECK: ptest 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x17,0x74,0x82,0x40]
+ptest 64(%rdx,%rax,4), %xmm6
+
+// CHECK: ptest -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x17,0x74,0x82,0xc0]
+ptest -64(%rdx,%rax,4), %xmm6
+
+// CHECK: ptest 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x17,0x74,0x02,0x40]
+ptest 64(%rdx,%rax), %xmm6
+
+// CHECK: ptest 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x17,0x72,0x40]
+ptest 64(%rdx), %xmm6
+
+// CHECK: ptest (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x17,0x32]
+ptest (%rdx), %xmm6
+
+// CHECK: ptest %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x17,0xf6]
+ptest %xmm6, %xmm6
+
+// CHECK: roundpd $0, 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x3a,0x09,0x34,0x25,0xf0,0x1c,0xf0,0x1c,0x00]
+roundpd $0, 485498096, %xmm6
+
+// CHECK: roundpd $0, 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x3a,0x09,0x74,0x82,0x40,0x00]
+roundpd $0, 64(%rdx,%rax,4), %xmm6
+
+// CHECK: roundpd $0, -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x3a,0x09,0x74,0x82,0xc0,0x00]
+roundpd $0, -64(%rdx,%rax,4), %xmm6
+
+// CHECK: roundpd $0, 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x3a,0x09,0x74,0x02,0x40,0x00]
+roundpd $0, 64(%rdx,%rax), %xmm6
+
+// CHECK: roundpd $0, 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x3a,0x09,0x72,0x40,0x00]
+roundpd $0, 64(%rdx), %xmm6
+
+// CHECK: roundpd $0, (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x3a,0x09,0x32,0x00]
+roundpd $0, (%rdx), %xmm6
+
+// CHECK: roundpd $0, %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x3a,0x09,0xf6,0x00]
+roundpd $0, %xmm6, %xmm6
+
+// CHECK: roundps $0, 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x3a,0x08,0x34,0x25,0xf0,0x1c,0xf0,0x1c,0x00]
+roundps $0, 485498096, %xmm6
+
+// CHECK: roundps $0, 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x3a,0x08,0x74,0x82,0x40,0x00]
+roundps $0, 64(%rdx,%rax,4), %xmm6
+
+// CHECK: roundps $0, -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x3a,0x08,0x74,0x82,0xc0,0x00]
+roundps $0, -64(%rdx,%rax,4), %xmm6
+
+// CHECK: roundps $0, 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x3a,0x08,0x74,0x02,0x40,0x00]
+roundps $0, 64(%rdx,%rax), %xmm6
+
+// CHECK: roundps $0, 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x3a,0x08,0x72,0x40,0x00]
+roundps $0, 64(%rdx), %xmm6
+
+// CHECK: roundps $0, (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x3a,0x08,0x32,0x00]
+roundps $0, (%rdx), %xmm6
+
+// CHECK: roundps $0, %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x3a,0x08,0xf6,0x00]
+roundps $0, %xmm6, %xmm6
+
+// CHECK: roundsd $0, 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x3a,0x0b,0x34,0x25,0xf0,0x1c,0xf0,0x1c,0x00]
+roundsd $0, 485498096, %xmm6
+
+// CHECK: roundsd $0, 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x3a,0x0b,0x74,0x82,0x40,0x00]
+roundsd $0, 64(%rdx,%rax,4), %xmm6
+
+// CHECK: roundsd $0, -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x3a,0x0b,0x74,0x82,0xc0,0x00]
+roundsd $0, -64(%rdx,%rax,4), %xmm6
+
+// CHECK: roundsd $0, 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x3a,0x0b,0x74,0x02,0x40,0x00]
+roundsd $0, 64(%rdx,%rax), %xmm6
+
+// CHECK: roundsd $0, 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x3a,0x0b,0x72,0x40,0x00]
+roundsd $0, 64(%rdx), %xmm6
+
+// CHECK: roundsd $0, (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x3a,0x0b,0x32,0x00]
+roundsd $0, (%rdx), %xmm6
+
+// CHECK: roundsd $0, %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x3a,0x0b,0xf6,0x00]
+roundsd $0, %xmm6, %xmm6
+
+// CHECK: roundss $0, 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x3a,0x0a,0x34,0x25,0xf0,0x1c,0xf0,0x1c,0x00]
+roundss $0, 485498096, %xmm6
+
+// CHECK: roundss $0, 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x3a,0x0a,0x74,0x82,0x40,0x00]
+roundss $0, 64(%rdx,%rax,4), %xmm6
+
+// CHECK: roundss $0, -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x3a,0x0a,0x74,0x82,0xc0,0x00]
+roundss $0, -64(%rdx,%rax,4), %xmm6
+
+// CHECK: roundss $0, 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x3a,0x0a,0x74,0x02,0x40,0x00]
+roundss $0, 64(%rdx,%rax), %xmm6
+
+// CHECK: roundss $0, 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x3a,0x0a,0x72,0x40,0x00]
+roundss $0, 64(%rdx), %xmm6
+
+// CHECK: roundss $0, (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x3a,0x0a,0x32,0x00]
+roundss $0, (%rdx), %xmm6
+
+// CHECK: roundss $0, %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x3a,0x0a,0xf6,0x00]
+roundss $0, %xmm6, %xmm6
+
diff --git a/test/MC/X86/SSE42-32.s b/test/MC/X86/SSE42-32.s
new file mode 100644
index 0000000000000000000000000000000000000000..0c4d8c640a5fff4447044719d35b7149ac6d9090
--- /dev/null
+++ b/test/MC/X86/SSE42-32.s
@@ -0,0 +1,226 @@
+// RUN: llvm-mc -triple i386-unknown-unknown --show-encoding %s | FileCheck %s
+
+// CHECK: crc32b 485498096, %ebx
+// CHECK: encoding: [0xf2,0x0f,0x38,0xf0,0x1d,0xf0,0x1c,0xf0,0x1c]
+crc32b 485498096, %ebx
+
+// CHECK: crc32b 64(%edx), %ebx
+// CHECK: encoding: [0xf2,0x0f,0x38,0xf0,0x5a,0x40]
+crc32b 64(%edx), %ebx
+
+// CHECK: crc32b -64(%edx,%eax,4), %ebx
+// CHECK: encoding: [0xf2,0x0f,0x38,0xf0,0x5c,0x82,0xc0]
+crc32b -64(%edx,%eax,4), %ebx
+
+// CHECK: crc32b 64(%edx,%eax,4), %ebx
+// CHECK: encoding: [0xf2,0x0f,0x38,0xf0,0x5c,0x82,0x40]
+crc32b 64(%edx,%eax,4), %ebx
+
+// CHECK: crc32b 64(%edx,%eax), %ebx
+// CHECK: encoding: [0xf2,0x0f,0x38,0xf0,0x5c,0x02,0x40]
+crc32b 64(%edx,%eax), %ebx
+
+// CHECK: crc32b %al, %ebx
+// CHECK: encoding: [0xf2,0x0f,0x38,0xf0,0xd8]
+crc32b %al, %ebx
+
+// CHECK: crc32b (%edx), %ebx
+// CHECK: encoding: [0xf2,0x0f,0x38,0xf0,0x1a]
+crc32b (%edx), %ebx
+
+// CHECK: crc32w 485498096, %ebx
+// CHECK: encoding: [0x66,0xf2,0x0f,0x38,0xf1,0x1d,0xf0,0x1c,0xf0,0x1c]
+crc32w 485498096, %ebx
+
+// CHECK: crc32w 64(%edx), %ebx
+// CHECK: encoding: [0x66,0xf2,0x0f,0x38,0xf1,0x5a,0x40]
+crc32w 64(%edx), %ebx
+
+// CHECK: crc32w -64(%edx,%eax,4), %ebx
+// CHECK: encoding: [0x66,0xf2,0x0f,0x38,0xf1,0x5c,0x82,0xc0]
+crc32w -64(%edx,%eax,4), %ebx
+
+// CHECK: crc32w 64(%edx,%eax,4), %ebx
+// CHECK: encoding: [0x66,0xf2,0x0f,0x38,0xf1,0x5c,0x82,0x40]
+crc32w 64(%edx,%eax,4), %ebx
+
+// CHECK: crc32w 64(%edx,%eax), %ebx
+// CHECK: encoding: [0x66,0xf2,0x0f,0x38,0xf1,0x5c,0x02,0x40]
+crc32w 64(%edx,%eax), %ebx
+
+// CHECK: crc32w %ax, %ebx
+// CHECK: encoding: [0x66,0xf2,0x0f,0x38,0xf1,0xd8]
+crc32w %ax, %ebx
+
+// CHECK: crc32w (%edx), %ebx
+// CHECK: encoding: [0x66,0xf2,0x0f,0x38,0xf1,0x1a]
+crc32w (%edx), %ebx
+
+// CHECK: crc32l 485498096, %ebx
+// CHECK: encoding: [0xf2,0x0f,0x38,0xf1,0x1d,0xf0,0x1c,0xf0,0x1c]
+crc32l 485498096, %ebx
+
+// CHECK: crc32l 64(%edx), %ebx
+// CHECK: encoding: [0xf2,0x0f,0x38,0xf1,0x5a,0x40]
+crc32l 64(%edx), %ebx
+
+// CHECK: crc32l -64(%edx,%eax,4), %ebx
+// CHECK: encoding: [0xf2,0x0f,0x38,0xf1,0x5c,0x82,0xc0]
+crc32l -64(%edx,%eax,4), %ebx
+
+// CHECK: crc32l 64(%edx,%eax,4), %ebx
+// CHECK: encoding: [0xf2,0x0f,0x38,0xf1,0x5c,0x82,0x40]
+crc32l 64(%edx,%eax,4), %ebx
+
+// CHECK: crc32l 64(%edx,%eax), %ebx
+// CHECK: encoding: [0xf2,0x0f,0x38,0xf1,0x5c,0x02,0x40]
+crc32l 64(%edx,%eax), %ebx
+
+// CHECK: crc32l %eax, %ebx
+// CHECK: encoding: [0xf2,0x0f,0x38,0xf1,0xd8]
+crc32l %eax, %ebx
+
+// CHECK: crc32l (%edx), %ebx
+// CHECK: encoding: [0xf2,0x0f,0x38,0xf1,0x1a]
+crc32l (%edx), %ebx
+
+// CHECK: pcmpestri $0, -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x3a,0x61,0x8c,0x82,0x10,0xe3,0x0f,0xe3,0x00]
+pcmpestri $0, -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pcmpestri $0, 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x3a,0x61,0x8c,0x82,0xf0,0x1c,0xf0,0x1c,0x00]
+pcmpestri $0, 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pcmpestri $0, 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x3a,0x61,0x8a,0xf0,0x1c,0xf0,0x1c,0x00]
+pcmpestri $0, 485498096(%edx), %xmm1
+
+// CHECK: pcmpestri $0, 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x3a,0x61,0x0d,0xf0,0x1c,0xf0,0x1c,0x00]
+pcmpestri $0, 485498096, %xmm1
+
+// CHECK: pcmpestri $0, 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x3a,0x61,0x4c,0x02,0x40,0x00]
+pcmpestri $0, 64(%edx,%eax), %xmm1
+
+// CHECK: pcmpestri $0, (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x3a,0x61,0x0a,0x00]
+pcmpestri $0, (%edx), %xmm1
+
+// CHECK: pcmpestri $0, %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x3a,0x61,0xc9,0x00]
+pcmpestri $0, %xmm1, %xmm1
+
+// CHECK: pcmpestrm $0, -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x3a,0x60,0x8c,0x82,0x10,0xe3,0x0f,0xe3,0x00]
+pcmpestrm $0, -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pcmpestrm $0, 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x3a,0x60,0x8c,0x82,0xf0,0x1c,0xf0,0x1c,0x00]
+pcmpestrm $0, 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pcmpestrm $0, 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x3a,0x60,0x8a,0xf0,0x1c,0xf0,0x1c,0x00]
+pcmpestrm $0, 485498096(%edx), %xmm1
+
+// CHECK: pcmpestrm $0, 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x3a,0x60,0x0d,0xf0,0x1c,0xf0,0x1c,0x00]
+pcmpestrm $0, 485498096, %xmm1
+
+// CHECK: pcmpestrm $0, 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x3a,0x60,0x4c,0x02,0x40,0x00]
+pcmpestrm $0, 64(%edx,%eax), %xmm1
+
+// CHECK: pcmpestrm $0, (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x3a,0x60,0x0a,0x00]
+pcmpestrm $0, (%edx), %xmm1
+
+// CHECK: pcmpestrm $0, %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x3a,0x60,0xc9,0x00]
+pcmpestrm $0, %xmm1, %xmm1
+
+// CHECK: pcmpgtq -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x37,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+pcmpgtq -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pcmpgtq 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x37,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+pcmpgtq 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pcmpgtq 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x37,0x8a,0xf0,0x1c,0xf0,0x1c]
+pcmpgtq 485498096(%edx), %xmm1
+
+// CHECK: pcmpgtq 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x37,0x0d,0xf0,0x1c,0xf0,0x1c]
+pcmpgtq 485498096, %xmm1
+
+// CHECK: pcmpgtq 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x37,0x4c,0x02,0x40]
+pcmpgtq 64(%edx,%eax), %xmm1
+
+// CHECK: pcmpgtq (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x37,0x0a]
+pcmpgtq (%edx), %xmm1
+
+// CHECK: pcmpgtq %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x37,0xc9]
+pcmpgtq %xmm1, %xmm1
+
+// CHECK: pcmpistri $0, -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x3a,0x63,0x8c,0x82,0x10,0xe3,0x0f,0xe3,0x00]
+pcmpistri $0, -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pcmpistri $0, 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x3a,0x63,0x8c,0x82,0xf0,0x1c,0xf0,0x1c,0x00]
+pcmpistri $0, 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pcmpistri $0, 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x3a,0x63,0x8a,0xf0,0x1c,0xf0,0x1c,0x00]
+pcmpistri $0, 485498096(%edx), %xmm1
+
+// CHECK: pcmpistri $0, 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x3a,0x63,0x0d,0xf0,0x1c,0xf0,0x1c,0x00]
+pcmpistri $0, 485498096, %xmm1
+
+// CHECK: pcmpistri $0, 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x3a,0x63,0x4c,0x02,0x40,0x00]
+pcmpistri $0, 64(%edx,%eax), %xmm1
+
+// CHECK: pcmpistri $0, (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x3a,0x63,0x0a,0x00]
+pcmpistri $0, (%edx), %xmm1
+
+// CHECK: pcmpistri $0, %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x3a,0x63,0xc9,0x00]
+pcmpistri $0, %xmm1, %xmm1
+
+// CHECK: pcmpistrm $0, -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x3a,0x62,0x8c,0x82,0x10,0xe3,0x0f,0xe3,0x00]
+pcmpistrm $0, -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pcmpistrm $0, 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x3a,0x62,0x8c,0x82,0xf0,0x1c,0xf0,0x1c,0x00]
+pcmpistrm $0, 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pcmpistrm $0, 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x3a,0x62,0x8a,0xf0,0x1c,0xf0,0x1c,0x00]
+pcmpistrm $0, 485498096(%edx), %xmm1
+
+// CHECK: pcmpistrm $0, 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x3a,0x62,0x0d,0xf0,0x1c,0xf0,0x1c,0x00]
+pcmpistrm $0, 485498096, %xmm1
+
+// CHECK: pcmpistrm $0, 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x3a,0x62,0x4c,0x02,0x40,0x00]
+pcmpistrm $0, 64(%edx,%eax), %xmm1
+
+// CHECK: pcmpistrm $0, (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x3a,0x62,0x0a,0x00]
+pcmpistrm $0, (%edx), %xmm1
+
+// CHECK: pcmpistrm $0, %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x3a,0x62,0xc9,0x00]
+pcmpistrm $0, %xmm1, %xmm1
+
diff --git a/test/MC/X86/SSE42-64.s b/test/MC/X86/SSE42-64.s
new file mode 100644
index 0000000000000000000000000000000000000000..bde7d361563658fb3eccfc961c6e56f9a94cc34c
--- /dev/null
+++ b/test/MC/X86/SSE42-64.s
@@ -0,0 +1,254 @@
+// RUN: llvm-mc -triple x86_64-unknown-unknown --show-encoding %s | FileCheck %s
+
+// CHECK: crc32b 485498096, %r13d
+// CHECK: encoding: [0xf2,0x44,0x0f,0x38,0xf0,0x2c,0x25,0xf0,0x1c,0xf0,0x1c]
+crc32b 485498096, %r13d
+
+// CHECK: crc32b 64(%rdx), %r13d
+// CHECK: encoding: [0xf2,0x44,0x0f,0x38,0xf0,0x6a,0x40]
+crc32b 64(%rdx), %r13d
+
+// CHECK: crc32b -64(%rdx,%rax,4), %r13d
+// CHECK: encoding: [0xf2,0x44,0x0f,0x38,0xf0,0x6c,0x82,0xc0]
+crc32b -64(%rdx,%rax,4), %r13d
+
+// CHECK: crc32b 64(%rdx,%rax,4), %r13d
+// CHECK: encoding: [0xf2,0x44,0x0f,0x38,0xf0,0x6c,0x82,0x40]
+crc32b 64(%rdx,%rax,4), %r13d
+
+// CHECK: crc32b 64(%rdx,%rax), %r13d
+// CHECK: encoding: [0xf2,0x44,0x0f,0x38,0xf0,0x6c,0x02,0x40]
+crc32b 64(%rdx,%rax), %r13d
+
+// CHECK: crc32b %r14b, %r13d
+// CHECK: encoding: [0xf2,0x45,0x0f,0x38,0xf0,0xee]
+crc32b %r14b, %r13d
+
+// CHECK: crc32b (%rdx), %r13d
+// CHECK: encoding: [0xf2,0x44,0x0f,0x38,0xf0,0x2a]
+crc32b (%rdx), %r13d
+
+// CHECK: crc32w 485498096, %r13d
+// CHECK: encoding: [0x66,0xf2,0x44,0x0f,0x38,0xf1,0x2c,0x25,0xf0,0x1c,0xf0,0x1c]
+crc32w 485498096, %r13d
+
+// CHECK: crc32w 64(%rdx), %r13d
+// CHECK: encoding: [0x66,0xf2,0x44,0x0f,0x38,0xf1,0x6a,0x40]
+crc32w 64(%rdx), %r13d
+
+// CHECK: crc32w -64(%rdx,%rax,4), %r13d
+// CHECK: encoding: [0x66,0xf2,0x44,0x0f,0x38,0xf1,0x6c,0x82,0xc0]
+crc32w -64(%rdx,%rax,4), %r13d
+
+// CHECK: crc32w 64(%rdx,%rax,4), %r13d
+// CHECK: encoding: [0x66,0xf2,0x44,0x0f,0x38,0xf1,0x6c,0x82,0x40]
+crc32w 64(%rdx,%rax,4), %r13d
+
+// CHECK: crc32w 64(%rdx,%rax), %r13d
+// CHECK: encoding: [0x66,0xf2,0x44,0x0f,0x38,0xf1,0x6c,0x02,0x40]
+crc32w 64(%rdx,%rax), %r13d
+
+// CHECK: crc32w %r14w, %r13d
+// CHECK: encoding: [0x66,0xf2,0x45,0x0f,0x38,0xf1,0xee]
+crc32w %r14w, %r13d
+
+// CHECK: crc32w (%rdx), %r13d
+// CHECK: encoding: [0x66,0xf2,0x44,0x0f,0x38,0xf1,0x2a]
+crc32w (%rdx), %r13d
+
+// CHECK: crc32l 485498096, %r13d
+// CHECK: encoding: [0xf2,0x44,0x0f,0x38,0xf1,0x2c,0x25,0xf0,0x1c,0xf0,0x1c]
+crc32l 485498096, %r13d
+
+// CHECK: crc32l 64(%rdx), %r13d
+// CHECK: encoding: [0xf2,0x44,0x0f,0x38,0xf1,0x6a,0x40]
+crc32l 64(%rdx), %r13d
+
+// CHECK: crc32l -64(%rdx,%rax,4), %r13d
+// CHECK: encoding: [0xf2,0x44,0x0f,0x38,0xf1,0x6c,0x82,0xc0]
+crc32l -64(%rdx,%rax,4), %r13d
+
+// CHECK: crc32l 64(%rdx,%rax,4), %r13d
+// CHECK: encoding: [0xf2,0x44,0x0f,0x38,0xf1,0x6c,0x82,0x40]
+crc32l 64(%rdx,%rax,4), %r13d
+
+// CHECK: crc32l 64(%rdx,%rax), %r13d
+// CHECK: encoding: [0xf2,0x44,0x0f,0x38,0xf1,0x6c,0x02,0x40]
+crc32l 64(%rdx,%rax), %r13d
+
+// CHECK: crc32l %r14d, %r13d
+// CHECK: encoding: [0xf2,0x45,0x0f,0x38,0xf1,0xee]
+crc32l %r14d, %r13d
+
+// CHECK: crc32l (%rdx), %r13d
+// CHECK: encoding: [0xf2,0x44,0x0f,0x38,0xf1,0x2a]
+crc32l (%rdx), %r13d
+
+// CHECK: crc32q 485498096, %r13
+// CHECK: encoding: [0xf2,0x4c,0x0f,0x38,0xf1,0x2c,0x25,0xf0,0x1c,0xf0,0x1c]
+crc32q 485498096, %r13
+
+// CHECK: crc32q 64(%rdx), %r13
+// CHECK: encoding: [0xf2,0x4c,0x0f,0x38,0xf1,0x6a,0x40]
+crc32q 64(%rdx), %r13
+
+// CHECK: crc32q -64(%rdx,%rax,4), %r13
+// CHECK: encoding: [0xf2,0x4c,0x0f,0x38,0xf1,0x6c,0x82,0xc0]
+crc32q -64(%rdx,%rax,4), %r13
+
+// CHECK: crc32q 64(%rdx,%rax,4), %r13
+// CHECK: encoding: [0xf2,0x4c,0x0f,0x38,0xf1,0x6c,0x82,0x40]
+crc32q 64(%rdx,%rax,4), %r13
+
+// CHECK: crc32q 64(%rdx,%rax), %r13
+// CHECK: encoding: [0xf2,0x4c,0x0f,0x38,0xf1,0x6c,0x02,0x40]
+crc32q 64(%rdx,%rax), %r13
+
+// CHECK: crc32q %r14, %r13
+// CHECK: encoding: [0xf2,0x4d,0x0f,0x38,0xf1,0xee]
+crc32q %r14, %r13
+
+// CHECK: crc32q (%rdx), %r13
+// CHECK: encoding: [0xf2,0x4c,0x0f,0x38,0xf1,0x2a]
+crc32q (%rdx), %r13
+
+// CHECK: pcmpestri $0, 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x3a,0x61,0x34,0x25,0xf0,0x1c,0xf0,0x1c,0x00]
+pcmpestri $0, 485498096, %xmm6
+
+// CHECK: pcmpestri $0, -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x3a,0x61,0x74,0x82,0xc0,0x00]
+pcmpestri $0, -64(%rdx,%rax,4), %xmm6
+
+// CHECK: pcmpestri $0, 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x3a,0x61,0x74,0x82,0x40,0x00]
+pcmpestri $0, 64(%rdx,%rax,4), %xmm6
+
+// CHECK: pcmpestri $0, 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x3a,0x61,0x74,0x02,0x40,0x00]
+pcmpestri $0, 64(%rdx,%rax), %xmm6
+
+// CHECK: pcmpestri $0, 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x3a,0x61,0x72,0x40,0x00]
+pcmpestri $0, 64(%rdx), %xmm6
+
+// CHECK: pcmpestri $0, (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x3a,0x61,0x32,0x00]
+pcmpestri $0, (%rdx), %xmm6
+
+// CHECK: pcmpestri $0, %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x3a,0x61,0xf6,0x00]
+pcmpestri $0, %xmm6, %xmm6
+
+// CHECK: pcmpestrm $0, 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x3a,0x60,0x34,0x25,0xf0,0x1c,0xf0,0x1c,0x00]
+pcmpestrm $0, 485498096, %xmm6
+
+// CHECK: pcmpestrm $0, -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x3a,0x60,0x74,0x82,0xc0,0x00]
+pcmpestrm $0, -64(%rdx,%rax,4), %xmm6
+
+// CHECK: pcmpestrm $0, 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x3a,0x60,0x74,0x82,0x40,0x00]
+pcmpestrm $0, 64(%rdx,%rax,4), %xmm6
+
+// CHECK: pcmpestrm $0, 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x3a,0x60,0x74,0x02,0x40,0x00]
+pcmpestrm $0, 64(%rdx,%rax), %xmm6
+
+// CHECK: pcmpestrm $0, 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x3a,0x60,0x72,0x40,0x00]
+pcmpestrm $0, 64(%rdx), %xmm6
+
+// CHECK: pcmpestrm $0, (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x3a,0x60,0x32,0x00]
+pcmpestrm $0, (%rdx), %xmm6
+
+// CHECK: pcmpestrm $0, %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x3a,0x60,0xf6,0x00]
+pcmpestrm $0, %xmm6, %xmm6
+
+// CHECK: pcmpgtq 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x37,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+pcmpgtq 485498096, %xmm6
+
+// CHECK: pcmpgtq -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x37,0x74,0x82,0xc0]
+pcmpgtq -64(%rdx,%rax,4), %xmm6
+
+// CHECK: pcmpgtq 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x37,0x74,0x82,0x40]
+pcmpgtq 64(%rdx,%rax,4), %xmm6
+
+// CHECK: pcmpgtq 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x37,0x74,0x02,0x40]
+pcmpgtq 64(%rdx,%rax), %xmm6
+
+// CHECK: pcmpgtq 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x37,0x72,0x40]
+pcmpgtq 64(%rdx), %xmm6
+
+// CHECK: pcmpgtq (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x37,0x32]
+pcmpgtq (%rdx), %xmm6
+
+// CHECK: pcmpgtq %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x37,0xf6]
+pcmpgtq %xmm6, %xmm6
+
+// CHECK: pcmpistri $0, 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x3a,0x63,0x34,0x25,0xf0,0x1c,0xf0,0x1c,0x00]
+pcmpistri $0, 485498096, %xmm6
+
+// CHECK: pcmpistri $0, -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x3a,0x63,0x74,0x82,0xc0,0x00]
+pcmpistri $0, -64(%rdx,%rax,4), %xmm6
+
+// CHECK: pcmpistri $0, 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x3a,0x63,0x74,0x82,0x40,0x00]
+pcmpistri $0, 64(%rdx,%rax,4), %xmm6
+
+// CHECK: pcmpistri $0, 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x3a,0x63,0x74,0x02,0x40,0x00]
+pcmpistri $0, 64(%rdx,%rax), %xmm6
+
+// CHECK: pcmpistri $0, 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x3a,0x63,0x72,0x40,0x00]
+pcmpistri $0, 64(%rdx), %xmm6
+
+// CHECK: pcmpistri $0, (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x3a,0x63,0x32,0x00]
+pcmpistri $0, (%rdx), %xmm6
+
+// CHECK: pcmpistri $0, %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x3a,0x63,0xf6,0x00]
+pcmpistri $0, %xmm6, %xmm6
+
+// CHECK: pcmpistrm $0, 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x3a,0x62,0x34,0x25,0xf0,0x1c,0xf0,0x1c,0x00]
+pcmpistrm $0, 485498096, %xmm6
+
+// CHECK: pcmpistrm $0, -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x3a,0x62,0x74,0x82,0xc0,0x00]
+pcmpistrm $0, -64(%rdx,%rax,4), %xmm6
+
+// CHECK: pcmpistrm $0, 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x3a,0x62,0x74,0x82,0x40,0x00]
+pcmpistrm $0, 64(%rdx,%rax,4), %xmm6
+
+// CHECK: pcmpistrm $0, 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x3a,0x62,0x74,0x02,0x40,0x00]
+pcmpistrm $0, 64(%rdx,%rax), %xmm6
+
+// CHECK: pcmpistrm $0, 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x3a,0x62,0x72,0x40,0x00]
+pcmpistrm $0, 64(%rdx), %xmm6
+
+// CHECK: pcmpistrm $0, (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x3a,0x62,0x32,0x00]
+pcmpistrm $0, (%rdx), %xmm6
+
+// CHECK: pcmpistrm $0, %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x3a,0x62,0xf6,0x00]
+pcmpistrm $0, %xmm6, %xmm6
+
diff --git a/test/MC/X86/SSE4a-32.s b/test/MC/X86/SSE4a-32.s
new file mode 100644
index 0000000000000000000000000000000000000000..034221a8007bd4c3fa23df99a3893b254ae85764
--- /dev/null
+++ b/test/MC/X86/SSE4a-32.s
@@ -0,0 +1,65 @@
+// RUN: llvm-mc -triple i386-unknown-unknown --show-encoding %s | FileCheck %s
+
+// CHECK: extrq $0, $0, %xmm0
+// CHECK: encoding: [0x66,0x0f,0x78,0xc0,0x00,0x00]
+extrq $0, $0, %xmm0
+
+// CHECK: extrq %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x79,0xc9]
+extrq %xmm1, %xmm1
+
+// CHECK: insertq $0, $0, %xmm1, %xmm1
+// CHECK: encoding: [0xf2,0x0f,0x78,0xc9,0x00,0x00]
+insertq $0, $0, %xmm1, %xmm1
+
+// CHECK: insertq %xmm1, %xmm1
+// CHECK: encoding: [0xf2,0x0f,0x79,0xc9]
+insertq %xmm1, %xmm1
+
+// CHECK: movntsd %xmm1, -485498096(%edx,%eax,4)
+// CHECK: encoding: [0xf2,0x0f,0x2b,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+movntsd %xmm1, -485498096(%edx,%eax,4)
+
+// CHECK: movntsd %xmm1, 485498096(%edx,%eax,4)
+// CHECK: encoding: [0xf2,0x0f,0x2b,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+movntsd %xmm1, 485498096(%edx,%eax,4)
+
+// CHECK: movntsd %xmm1, 485498096(%edx)
+// CHECK: encoding: [0xf2,0x0f,0x2b,0x8a,0xf0,0x1c,0xf0,0x1c]
+movntsd %xmm1, 485498096(%edx)
+
+// CHECK: movntsd %xmm1, 485498096
+// CHECK: encoding: [0xf2,0x0f,0x2b,0x0d,0xf0,0x1c,0xf0,0x1c]
+movntsd %xmm1, 485498096
+
+// CHECK: movntsd %xmm1, 64(%edx,%eax)
+// CHECK: encoding: [0xf2,0x0f,0x2b,0x4c,0x02,0x40]
+movntsd %xmm1, 64(%edx,%eax)
+
+// CHECK: movntsd %xmm1, (%edx)
+// CHECK: encoding: [0xf2,0x0f,0x2b,0x0a]
+movntsd %xmm1, (%edx)
+
+// CHECK: movntss %xmm1, -485498096(%edx,%eax,4)
+// CHECK: encoding: [0xf3,0x0f,0x2b,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+movntss %xmm1, -485498096(%edx,%eax,4)
+
+// CHECK: movntss %xmm1, 485498096(%edx,%eax,4)
+// CHECK: encoding: [0xf3,0x0f,0x2b,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+movntss %xmm1, 485498096(%edx,%eax,4)
+
+// CHECK: movntss %xmm1, 485498096(%edx)
+// CHECK: encoding: [0xf3,0x0f,0x2b,0x8a,0xf0,0x1c,0xf0,0x1c]
+movntss %xmm1, 485498096(%edx)
+
+// CHECK: movntss %xmm1, 485498096
+// CHECK: encoding: [0xf3,0x0f,0x2b,0x0d,0xf0,0x1c,0xf0,0x1c]
+movntss %xmm1, 485498096
+
+// CHECK: movntss %xmm1, 64(%edx,%eax)
+// CHECK: encoding: [0xf3,0x0f,0x2b,0x4c,0x02,0x40]
+movntss %xmm1, 64(%edx,%eax)
+
+// CHECK: movntss %xmm1, (%edx)
+// CHECK: encoding: [0xf3,0x0f,0x2b,0x0a]
+movntss %xmm1, (%edx)
diff --git a/test/MC/X86/SSE4a-64.s b/test/MC/X86/SSE4a-64.s
new file mode 100644
index 0000000000000000000000000000000000000000..eb21404ebeba4c4ce3abc2313e54afd4e69ba5b2
--- /dev/null
+++ b/test/MC/X86/SSE4a-64.s
@@ -0,0 +1,77 @@
+// RUN: llvm-mc -triple x86_64-unknown-unknown --show-encoding %s | FileCheck %s
+
+// CHECK: extrq $0, $0, %xmm8
+// CHECK: encoding: [0x66,0x41,0x0f,0x78,0xc0,0x00,0x00]
+extrq $0, $0, %xmm8
+
+// CHECK: extrq %xmm8, %xmm8
+// CHECK: encoding: [0x66,0x45,0x0f,0x79,0xc0]
+extrq %xmm8, %xmm8
+
+// CHECK: insertq $0, $0, %xmm14, %xmm14
+// CHECK: encoding: [0xf2,0x45,0x0f,0x78,0xf6,0x00,0x00]
+insertq $0, $0, %xmm14, %xmm14
+
+// CHECK: insertq %xmm14, %xmm14
+// CHECK: encoding: [0xf2,0x45,0x0f,0x79,0xf6]
+insertq %xmm14, %xmm14
+
+// CHECK: movntsd %xmm6, 485498096
+// CHECK: encoding: [0xf2,0x0f,0x2b,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+movntsd %xmm6, 485498096
+
+// CHECK: movntsd %xmm14, 485498096
+// CHECK: encoding: [0xf2,0x44,0x0f,0x2b,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+movntsd %xmm14, 485498096
+
+// CHECK: movntsd %xmm14, 64(%rdx)
+// CHECK: encoding: [0xf2,0x44,0x0f,0x2b,0x72,0x40]
+movntsd %xmm14, 64(%rdx)
+
+// CHECK: movntsd %xmm14, 64(%rdx,%rax,4)
+// CHECK: encoding: [0xf2,0x44,0x0f,0x2b,0x74,0x82,0x40]
+movntsd %xmm14, 64(%rdx,%rax,4)
+
+// CHECK: movntsd %xmm14, -64(%rdx,%rax,4)
+// CHECK: encoding: [0xf2,0x44,0x0f,0x2b,0x74,0x82,0xc0]
+movntsd %xmm14, -64(%rdx,%rax,4)
+
+// CHECK: movntsd %xmm14, 64(%rdx,%rax)
+// CHECK: encoding: [0xf2,0x44,0x0f,0x2b,0x74,0x02,0x40]
+movntsd %xmm14, 64(%rdx,%rax)
+
+// CHECK: movntsd %xmm14, (%rdx)
+// CHECK: encoding: [0xf2,0x44,0x0f,0x2b,0x32]
+movntsd %xmm14, (%rdx)
+
+// CHECK: movntss %xmm6, 485498096
+// CHECK: encoding: [0xf3,0x0f,0x2b,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+movntss %xmm6, 485498096
+
+// CHECK: movntss %xmm14, 485498096
+// CHECK: encoding: [0xf3,0x44,0x0f,0x2b,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+movntss %xmm14, 485498096
+
+// CHECK: movntss %xmm14, 485498096
+// CHECK: encoding: [0xf3,0x44,0x0f,0x2b,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+movntss %xmm14, 485498096
+
+// CHECK: movntss %xmm14, 64(%rdx)
+// CHECK: encoding: [0xf3,0x44,0x0f,0x2b,0x72,0x40]
+movntss %xmm14, 64(%rdx)
+
+// CHECK: movntss %xmm14, 64(%rdx,%rax,4)
+// CHECK: encoding: [0xf3,0x44,0x0f,0x2b,0x74,0x82,0x40]
+movntss %xmm14, 64(%rdx,%rax,4)
+
+// CHECK: movntss %xmm14, -64(%rdx,%rax,4)
+// CHECK: encoding: [0xf3,0x44,0x0f,0x2b,0x74,0x82,0xc0]
+movntss %xmm14, -64(%rdx,%rax,4)
+
+// CHECK: movntss %xmm14, 64(%rdx,%rax)
+// CHECK: encoding: [0xf3,0x44,0x0f,0x2b,0x74,0x02,0x40]
+movntss %xmm14, 64(%rdx,%rax)
+
+// CHECK: movntss %xmm14, (%rdx)
+// CHECK: encoding: [0xf3,0x44,0x0f,0x2b,0x32]
+movntss %xmm14, (%rdx)
diff --git a/test/MC/X86/SSEMXCSR-32.s b/test/MC/X86/SSEMXCSR-32.s
new file mode 100644
index 0000000000000000000000000000000000000000..7d6efed3cdd2bbd07586c5b01e08e213a9aa3cdd
--- /dev/null
+++ b/test/MC/X86/SSEMXCSR-32.s
@@ -0,0 +1,50 @@
+// RUN: llvm-mc -triple i386-unknown-unknown --show-encoding %s | FileCheck %s
+
+// CHECK: ldmxcsr -485498096(%edx,%eax,4)
+// CHECK: encoding: [0x0f,0xae,0x94,0x82,0x10,0xe3,0x0f,0xe3]
+ldmxcsr -485498096(%edx,%eax,4)
+
+// CHECK: ldmxcsr 485498096(%edx,%eax,4)
+// CHECK: encoding: [0x0f,0xae,0x94,0x82,0xf0,0x1c,0xf0,0x1c]
+ldmxcsr 485498096(%edx,%eax,4)
+
+// CHECK: ldmxcsr 485498096(%edx)
+// CHECK: encoding: [0x0f,0xae,0x92,0xf0,0x1c,0xf0,0x1c]
+ldmxcsr 485498096(%edx)
+
+// CHECK: ldmxcsr 485498096
+// CHECK: encoding: [0x0f,0xae,0x15,0xf0,0x1c,0xf0,0x1c]
+ldmxcsr 485498096
+
+// CHECK: ldmxcsr 64(%edx,%eax)
+// CHECK: encoding: [0x0f,0xae,0x54,0x02,0x40]
+ldmxcsr 64(%edx,%eax)
+
+// CHECK: ldmxcsr (%edx)
+// CHECK: encoding: [0x0f,0xae,0x12]
+ldmxcsr (%edx)
+
+// CHECK: stmxcsr -485498096(%edx,%eax,4)
+// CHECK: encoding: [0x0f,0xae,0x9c,0x82,0x10,0xe3,0x0f,0xe3]
+stmxcsr -485498096(%edx,%eax,4)
+
+// CHECK: stmxcsr 485498096(%edx,%eax,4)
+// CHECK: encoding: [0x0f,0xae,0x9c,0x82,0xf0,0x1c,0xf0,0x1c]
+stmxcsr 485498096(%edx,%eax,4)
+
+// CHECK: stmxcsr 485498096(%edx)
+// CHECK: encoding: [0x0f,0xae,0x9a,0xf0,0x1c,0xf0,0x1c]
+stmxcsr 485498096(%edx)
+
+// CHECK: stmxcsr 485498096
+// CHECK: encoding: [0x0f,0xae,0x1d,0xf0,0x1c,0xf0,0x1c]
+stmxcsr 485498096
+
+// CHECK: stmxcsr 64(%edx,%eax)
+// CHECK: encoding: [0x0f,0xae,0x5c,0x02,0x40]
+stmxcsr 64(%edx,%eax)
+
+// CHECK: stmxcsr (%edx)
+// CHECK: encoding: [0x0f,0xae,0x1a]
+stmxcsr (%edx)
+
diff --git a/test/MC/X86/SSEMXCSR-64.s b/test/MC/X86/SSEMXCSR-64.s
new file mode 100644
index 0000000000000000000000000000000000000000..ba9648e413f81d865562f420b76b947fb1944614
--- /dev/null
+++ b/test/MC/X86/SSEMXCSR-64.s
@@ -0,0 +1,50 @@
+// RUN: llvm-mc -triple x86_64-unknown-unknown --show-encoding %s | FileCheck %s
+
+// CHECK: ldmxcsr 485498096
+// CHECK: encoding: [0x0f,0xae,0x14,0x25,0xf0,0x1c,0xf0,0x1c]
+ldmxcsr 485498096
+
+// CHECK: ldmxcsr 64(%rdx)
+// CHECK: encoding: [0x0f,0xae,0x52,0x40]
+ldmxcsr 64(%rdx)
+
+// CHECK: ldmxcsr -64(%rdx,%rax,4)
+// CHECK: encoding: [0x0f,0xae,0x54,0x82,0xc0]
+ldmxcsr -64(%rdx,%rax,4)
+
+// CHECK: ldmxcsr 64(%rdx,%rax,4)
+// CHECK: encoding: [0x0f,0xae,0x54,0x82,0x40]
+ldmxcsr 64(%rdx,%rax,4)
+
+// CHECK: ldmxcsr 64(%rdx,%rax)
+// CHECK: encoding: [0x0f,0xae,0x54,0x02,0x40]
+ldmxcsr 64(%rdx,%rax)
+
+// CHECK: ldmxcsr (%rdx)
+// CHECK: encoding: [0x0f,0xae,0x12]
+ldmxcsr (%rdx)
+
+// CHECK: stmxcsr 485498096
+// CHECK: encoding: [0x0f,0xae,0x1c,0x25,0xf0,0x1c,0xf0,0x1c]
+stmxcsr 485498096
+
+// CHECK: stmxcsr 64(%rdx)
+// CHECK: encoding: [0x0f,0xae,0x5a,0x40]
+stmxcsr 64(%rdx)
+
+// CHECK: stmxcsr -64(%rdx,%rax,4)
+// CHECK: encoding: [0x0f,0xae,0x5c,0x82,0xc0]
+stmxcsr -64(%rdx,%rax,4)
+
+// CHECK: stmxcsr 64(%rdx,%rax,4)
+// CHECK: encoding: [0x0f,0xae,0x5c,0x82,0x40]
+stmxcsr 64(%rdx,%rax,4)
+
+// CHECK: stmxcsr 64(%rdx,%rax)
+// CHECK: encoding: [0x0f,0xae,0x5c,0x02,0x40]
+stmxcsr 64(%rdx,%rax)
+
+// CHECK: stmxcsr (%rdx)
+// CHECK: encoding: [0x0f,0xae,0x1a]
+stmxcsr (%rdx)
+
diff --git a/test/MC/X86/SSE_PREFETCH-32.s b/test/MC/X86/SSE_PREFETCH-32.s
new file mode 100644
index 0000000000000000000000000000000000000000..256a63df08ab6c0786b7ee94a1d263d40bc045f6
--- /dev/null
+++ b/test/MC/X86/SSE_PREFETCH-32.s
@@ -0,0 +1,98 @@
+// RUN: llvm-mc -triple i386-unknown-unknown --show-encoding %s | FileCheck %s
+
+// CHECK: prefetchnta -485498096(%edx,%eax,4)
+// CHECK: encoding: [0x0f,0x18,0x84,0x82,0x10,0xe3,0x0f,0xe3]
+prefetchnta -485498096(%edx,%eax,4)
+
+// CHECK: prefetchnta 485498096(%edx,%eax,4)
+// CHECK: encoding: [0x0f,0x18,0x84,0x82,0xf0,0x1c,0xf0,0x1c]
+prefetchnta 485498096(%edx,%eax,4)
+
+// CHECK: prefetchnta 485498096(%edx)
+// CHECK: encoding: [0x0f,0x18,0x82,0xf0,0x1c,0xf0,0x1c]
+prefetchnta 485498096(%edx)
+
+// CHECK: prefetchnta 485498096
+// CHECK: encoding: [0x0f,0x18,0x05,0xf0,0x1c,0xf0,0x1c]
+prefetchnta 485498096
+
+// CHECK: prefetchnta 64(%edx,%eax)
+// CHECK: encoding: [0x0f,0x18,0x44,0x02,0x40]
+prefetchnta 64(%edx,%eax)
+
+// CHECK: prefetchnta (%edx)
+// CHECK: encoding: [0x0f,0x18,0x02]
+prefetchnta (%edx)
+
+// CHECK: prefetcht0 -485498096(%edx,%eax,4)
+// CHECK: encoding: [0x0f,0x18,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+prefetcht0 -485498096(%edx,%eax,4)
+
+// CHECK: prefetcht0 485498096(%edx,%eax,4)
+// CHECK: encoding: [0x0f,0x18,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+prefetcht0 485498096(%edx,%eax,4)
+
+// CHECK: prefetcht0 485498096(%edx)
+// CHECK: encoding: [0x0f,0x18,0x8a,0xf0,0x1c,0xf0,0x1c]
+prefetcht0 485498096(%edx)
+
+// CHECK: prefetcht0 485498096
+// CHECK: encoding: [0x0f,0x18,0x0d,0xf0,0x1c,0xf0,0x1c]
+prefetcht0 485498096
+
+// CHECK: prefetcht0 64(%edx,%eax)
+// CHECK: encoding: [0x0f,0x18,0x4c,0x02,0x40]
+prefetcht0 64(%edx,%eax)
+
+// CHECK: prefetcht0 (%edx)
+// CHECK: encoding: [0x0f,0x18,0x0a]
+prefetcht0 (%edx)
+
+// CHECK: prefetcht1 -485498096(%edx,%eax,4)
+// CHECK: encoding: [0x0f,0x18,0x94,0x82,0x10,0xe3,0x0f,0xe3]
+prefetcht1 -485498096(%edx,%eax,4)
+
+// CHECK: prefetcht1 485498096(%edx,%eax,4)
+// CHECK: encoding: [0x0f,0x18,0x94,0x82,0xf0,0x1c,0xf0,0x1c]
+prefetcht1 485498096(%edx,%eax,4)
+
+// CHECK: prefetcht1 485498096(%edx)
+// CHECK: encoding: [0x0f,0x18,0x92,0xf0,0x1c,0xf0,0x1c]
+prefetcht1 485498096(%edx)
+
+// CHECK: prefetcht1 485498096
+// CHECK: encoding: [0x0f,0x18,0x15,0xf0,0x1c,0xf0,0x1c]
+prefetcht1 485498096
+
+// CHECK: prefetcht1 64(%edx,%eax)
+// CHECK: encoding: [0x0f,0x18,0x54,0x02,0x40]
+prefetcht1 64(%edx,%eax)
+
+// CHECK: prefetcht1 (%edx)
+// CHECK: encoding: [0x0f,0x18,0x12]
+prefetcht1 (%edx)
+
+// CHECK: prefetcht2 -485498096(%edx,%eax,4)
+// CHECK: encoding: [0x0f,0x18,0x9c,0x82,0x10,0xe3,0x0f,0xe3]
+prefetcht2 -485498096(%edx,%eax,4)
+
+// CHECK: prefetcht2 485498096(%edx,%eax,4)
+// CHECK: encoding: [0x0f,0x18,0x9c,0x82,0xf0,0x1c,0xf0,0x1c]
+prefetcht2 485498096(%edx,%eax,4)
+
+// CHECK: prefetcht2 485498096(%edx)
+// CHECK: encoding: [0x0f,0x18,0x9a,0xf0,0x1c,0xf0,0x1c]
+prefetcht2 485498096(%edx)
+
+// CHECK: prefetcht2 485498096
+// CHECK: encoding: [0x0f,0x18,0x1d,0xf0,0x1c,0xf0,0x1c]
+prefetcht2 485498096
+
+// CHECK: prefetcht2 64(%edx,%eax)
+// CHECK: encoding: [0x0f,0x18,0x5c,0x02,0x40]
+prefetcht2 64(%edx,%eax)
+
+// CHECK: prefetcht2 (%edx)
+// CHECK: encoding: [0x0f,0x18,0x1a]
+prefetcht2 (%edx)
+
diff --git a/test/MC/X86/SSE_PREFETCH-64.s b/test/MC/X86/SSE_PREFETCH-64.s
new file mode 100644
index 0000000000000000000000000000000000000000..8b0ad764a39c6c1494c5492374a27cc05bf060e2
--- /dev/null
+++ b/test/MC/X86/SSE_PREFETCH-64.s
@@ -0,0 +1,98 @@
+// RUN: llvm-mc -triple x86_64-unknown-unknown --show-encoding %s | FileCheck %s
+
+// CHECK: prefetchnta 485498096
+// CHECK: encoding: [0x0f,0x18,0x04,0x25,0xf0,0x1c,0xf0,0x1c]
+prefetchnta 485498096
+
+// CHECK: prefetchnta 64(%rdx)
+// CHECK: encoding: [0x0f,0x18,0x42,0x40]
+prefetchnta 64(%rdx)
+
+// CHECK: prefetchnta -64(%rdx,%rax,4)
+// CHECK: encoding: [0x0f,0x18,0x44,0x82,0xc0]
+prefetchnta -64(%rdx,%rax,4)
+
+// CHECK: prefetchnta 64(%rdx,%rax,4)
+// CHECK: encoding: [0x0f,0x18,0x44,0x82,0x40]
+prefetchnta 64(%rdx,%rax,4)
+
+// CHECK: prefetchnta 64(%rdx,%rax)
+// CHECK: encoding: [0x0f,0x18,0x44,0x02,0x40]
+prefetchnta 64(%rdx,%rax)
+
+// CHECK: prefetchnta (%rdx)
+// CHECK: encoding: [0x0f,0x18,0x02]
+prefetchnta (%rdx)
+
+// CHECK: prefetcht0 485498096
+// CHECK: encoding: [0x0f,0x18,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+prefetcht0 485498096
+
+// CHECK: prefetcht0 64(%rdx)
+// CHECK: encoding: [0x0f,0x18,0x4a,0x40]
+prefetcht0 64(%rdx)
+
+// CHECK: prefetcht0 -64(%rdx,%rax,4)
+// CHECK: encoding: [0x0f,0x18,0x4c,0x82,0xc0]
+prefetcht0 -64(%rdx,%rax,4)
+
+// CHECK: prefetcht0 64(%rdx,%rax,4)
+// CHECK: encoding: [0x0f,0x18,0x4c,0x82,0x40]
+prefetcht0 64(%rdx,%rax,4)
+
+// CHECK: prefetcht0 64(%rdx,%rax)
+// CHECK: encoding: [0x0f,0x18,0x4c,0x02,0x40]
+prefetcht0 64(%rdx,%rax)
+
+// CHECK: prefetcht0 (%rdx)
+// CHECK: encoding: [0x0f,0x18,0x0a]
+prefetcht0 (%rdx)
+
+// CHECK: prefetcht1 485498096
+// CHECK: encoding: [0x0f,0x18,0x14,0x25,0xf0,0x1c,0xf0,0x1c]
+prefetcht1 485498096
+
+// CHECK: prefetcht1 64(%rdx)
+// CHECK: encoding: [0x0f,0x18,0x52,0x40]
+prefetcht1 64(%rdx)
+
+// CHECK: prefetcht1 -64(%rdx,%rax,4)
+// CHECK: encoding: [0x0f,0x18,0x54,0x82,0xc0]
+prefetcht1 -64(%rdx,%rax,4)
+
+// CHECK: prefetcht1 64(%rdx,%rax,4)
+// CHECK: encoding: [0x0f,0x18,0x54,0x82,0x40]
+prefetcht1 64(%rdx,%rax,4)
+
+// CHECK: prefetcht1 64(%rdx,%rax)
+// CHECK: encoding: [0x0f,0x18,0x54,0x02,0x40]
+prefetcht1 64(%rdx,%rax)
+
+// CHECK: prefetcht1 (%rdx)
+// CHECK: encoding: [0x0f,0x18,0x12]
+prefetcht1 (%rdx)
+
+// CHECK: prefetcht2 485498096
+// CHECK: encoding: [0x0f,0x18,0x1c,0x25,0xf0,0x1c,0xf0,0x1c]
+prefetcht2 485498096
+
+// CHECK: prefetcht2 64(%rdx)
+// CHECK: encoding: [0x0f,0x18,0x5a,0x40]
+prefetcht2 64(%rdx)
+
+// CHECK: prefetcht2 -64(%rdx,%rax,4)
+// CHECK: encoding: [0x0f,0x18,0x5c,0x82,0xc0]
+prefetcht2 -64(%rdx,%rax,4)
+
+// CHECK: prefetcht2 64(%rdx,%rax,4)
+// CHECK: encoding: [0x0f,0x18,0x5c,0x82,0x40]
+prefetcht2 64(%rdx,%rax,4)
+
+// CHECK: prefetcht2 64(%rdx,%rax)
+// CHECK: encoding: [0x0f,0x18,0x5c,0x02,0x40]
+prefetcht2 64(%rdx,%rax)
+
+// CHECK: prefetcht2 (%rdx)
+// CHECK: encoding: [0x0f,0x18,0x1a]
+prefetcht2 (%rdx)
+
diff --git a/test/MC/X86/SSSE3-32.s b/test/MC/X86/SSSE3-32.s
new file mode 100644
index 0000000000000000000000000000000000000000..9462688258d347f7af7a41516af9ab848f7c49cd
--- /dev/null
+++ b/test/MC/X86/SSSE3-32.s
@@ -0,0 +1,898 @@
+// RUN: llvm-mc -triple i386-unknown-unknown --show-encoding %s | FileCheck %s
+
+// CHECK: pabsb -485498096(%edx,%eax,4), %mm4
+// CHECK: encoding: [0x0f,0x38,0x1c,0xa4,0x82,0x10,0xe3,0x0f,0xe3]
+pabsb -485498096(%edx,%eax,4), %mm4
+
+// CHECK: pabsb 485498096(%edx,%eax,4), %mm4
+// CHECK: encoding: [0x0f,0x38,0x1c,0xa4,0x82,0xf0,0x1c,0xf0,0x1c]
+pabsb 485498096(%edx,%eax,4), %mm4
+
+// CHECK: pabsb -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x1c,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+pabsb -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pabsb 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x1c,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+pabsb 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pabsb 485498096(%edx), %mm4
+// CHECK: encoding: [0x0f,0x38,0x1c,0xa2,0xf0,0x1c,0xf0,0x1c]
+pabsb 485498096(%edx), %mm4
+
+// CHECK: pabsb 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x1c,0x8a,0xf0,0x1c,0xf0,0x1c]
+pabsb 485498096(%edx), %xmm1
+
+// CHECK: pabsb 485498096, %mm4
+// CHECK: encoding: [0x0f,0x38,0x1c,0x25,0xf0,0x1c,0xf0,0x1c]
+pabsb 485498096, %mm4
+
+// CHECK: pabsb 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x1c,0x0d,0xf0,0x1c,0xf0,0x1c]
+pabsb 485498096, %xmm1
+
+// CHECK: pabsb 64(%edx,%eax), %mm4
+// CHECK: encoding: [0x0f,0x38,0x1c,0x64,0x02,0x40]
+pabsb 64(%edx,%eax), %mm4
+
+// CHECK: pabsb 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x1c,0x4c,0x02,0x40]
+pabsb 64(%edx,%eax), %xmm1
+
+// CHECK: pabsb (%edx), %mm4
+// CHECK: encoding: [0x0f,0x38,0x1c,0x22]
+pabsb (%edx), %mm4
+
+// CHECK: pabsb (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x1c,0x0a]
+pabsb (%edx), %xmm1
+
+// CHECK: pabsb %mm4, %mm4
+// CHECK: encoding: [0x0f,0x38,0x1c,0xe4]
+pabsb %mm4, %mm4
+
+// CHECK: pabsb %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x1c,0xc9]
+pabsb %xmm1, %xmm1
+
+// CHECK: pabsd -485498096(%edx,%eax,4), %mm4
+// CHECK: encoding: [0x0f,0x38,0x1e,0xa4,0x82,0x10,0xe3,0x0f,0xe3]
+pabsd -485498096(%edx,%eax,4), %mm4
+
+// CHECK: pabsd 485498096(%edx,%eax,4), %mm4
+// CHECK: encoding: [0x0f,0x38,0x1e,0xa4,0x82,0xf0,0x1c,0xf0,0x1c]
+pabsd 485498096(%edx,%eax,4), %mm4
+
+// CHECK: pabsd -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x1e,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+pabsd -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pabsd 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x1e,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+pabsd 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pabsd 485498096(%edx), %mm4
+// CHECK: encoding: [0x0f,0x38,0x1e,0xa2,0xf0,0x1c,0xf0,0x1c]
+pabsd 485498096(%edx), %mm4
+
+// CHECK: pabsd 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x1e,0x8a,0xf0,0x1c,0xf0,0x1c]
+pabsd 485498096(%edx), %xmm1
+
+// CHECK: pabsd 485498096, %mm4
+// CHECK: encoding: [0x0f,0x38,0x1e,0x25,0xf0,0x1c,0xf0,0x1c]
+pabsd 485498096, %mm4
+
+// CHECK: pabsd 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x1e,0x0d,0xf0,0x1c,0xf0,0x1c]
+pabsd 485498096, %xmm1
+
+// CHECK: pabsd 64(%edx,%eax), %mm4
+// CHECK: encoding: [0x0f,0x38,0x1e,0x64,0x02,0x40]
+pabsd 64(%edx,%eax), %mm4
+
+// CHECK: pabsd 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x1e,0x4c,0x02,0x40]
+pabsd 64(%edx,%eax), %xmm1
+
+// CHECK: pabsd (%edx), %mm4
+// CHECK: encoding: [0x0f,0x38,0x1e,0x22]
+pabsd (%edx), %mm4
+
+// CHECK: pabsd (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x1e,0x0a]
+pabsd (%edx), %xmm1
+
+// CHECK: pabsd %mm4, %mm4
+// CHECK: encoding: [0x0f,0x38,0x1e,0xe4]
+pabsd %mm4, %mm4
+
+// CHECK: pabsd %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x1e,0xc9]
+pabsd %xmm1, %xmm1
+
+// CHECK: pabsw -485498096(%edx,%eax,4), %mm4
+// CHECK: encoding: [0x0f,0x38,0x1d,0xa4,0x82,0x10,0xe3,0x0f,0xe3]
+pabsw -485498096(%edx,%eax,4), %mm4
+
+// CHECK: pabsw 485498096(%edx,%eax,4), %mm4
+// CHECK: encoding: [0x0f,0x38,0x1d,0xa4,0x82,0xf0,0x1c,0xf0,0x1c]
+pabsw 485498096(%edx,%eax,4), %mm4
+
+// CHECK: pabsw -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x1d,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+pabsw -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pabsw 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x1d,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+pabsw 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pabsw 485498096(%edx), %mm4
+// CHECK: encoding: [0x0f,0x38,0x1d,0xa2,0xf0,0x1c,0xf0,0x1c]
+pabsw 485498096(%edx), %mm4
+
+// CHECK: pabsw 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x1d,0x8a,0xf0,0x1c,0xf0,0x1c]
+pabsw 485498096(%edx), %xmm1
+
+// CHECK: pabsw 485498096, %mm4
+// CHECK: encoding: [0x0f,0x38,0x1d,0x25,0xf0,0x1c,0xf0,0x1c]
+pabsw 485498096, %mm4
+
+// CHECK: pabsw 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x1d,0x0d,0xf0,0x1c,0xf0,0x1c]
+pabsw 485498096, %xmm1
+
+// CHECK: pabsw 64(%edx,%eax), %mm4
+// CHECK: encoding: [0x0f,0x38,0x1d,0x64,0x02,0x40]
+pabsw 64(%edx,%eax), %mm4
+
+// CHECK: pabsw 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x1d,0x4c,0x02,0x40]
+pabsw 64(%edx,%eax), %xmm1
+
+// CHECK: pabsw (%edx), %mm4
+// CHECK: encoding: [0x0f,0x38,0x1d,0x22]
+pabsw (%edx), %mm4
+
+// CHECK: pabsw (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x1d,0x0a]
+pabsw (%edx), %xmm1
+
+// CHECK: pabsw %mm4, %mm4
+// CHECK: encoding: [0x0f,0x38,0x1d,0xe4]
+pabsw %mm4, %mm4
+
+// CHECK: pabsw %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x1d,0xc9]
+pabsw %xmm1, %xmm1
+
+// CHECK: palignr $0, -485498096(%edx,%eax,4), %mm4
+// CHECK: encoding: [0x0f,0x3a,0x0f,0xa4,0x82,0x10,0xe3,0x0f,0xe3,0x00]
+palignr $0, -485498096(%edx,%eax,4), %mm4
+
+// CHECK: palignr $0, 485498096(%edx,%eax,4), %mm4
+// CHECK: encoding: [0x0f,0x3a,0x0f,0xa4,0x82,0xf0,0x1c,0xf0,0x1c,0x00]
+palignr $0, 485498096(%edx,%eax,4), %mm4
+
+// CHECK: palignr $0, -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x3a,0x0f,0x8c,0x82,0x10,0xe3,0x0f,0xe3,0x00]
+palignr $0, -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: palignr $0, 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x3a,0x0f,0x8c,0x82,0xf0,0x1c,0xf0,0x1c,0x00]
+palignr $0, 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: palignr $0, 485498096(%edx), %mm4
+// CHECK: encoding: [0x0f,0x3a,0x0f,0xa2,0xf0,0x1c,0xf0,0x1c,0x00]
+palignr $0, 485498096(%edx), %mm4
+
+// CHECK: palignr $0, 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x3a,0x0f,0x8a,0xf0,0x1c,0xf0,0x1c,0x00]
+palignr $0, 485498096(%edx), %xmm1
+
+// CHECK: palignr $0, 485498096, %mm4
+// CHECK: encoding: [0x0f,0x3a,0x0f,0x25,0xf0,0x1c,0xf0,0x1c,0x00]
+palignr $0, 485498096, %mm4
+
+// CHECK: palignr $0, 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x3a,0x0f,0x0d,0xf0,0x1c,0xf0,0x1c,0x00]
+palignr $0, 485498096, %xmm1
+
+// CHECK: palignr $0, 64(%edx,%eax), %mm4
+// CHECK: encoding: [0x0f,0x3a,0x0f,0x64,0x02,0x40,0x00]
+palignr $0, 64(%edx,%eax), %mm4
+
+// CHECK: palignr $0, 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x3a,0x0f,0x4c,0x02,0x40,0x00]
+palignr $0, 64(%edx,%eax), %xmm1
+
+// CHECK: palignr $0, (%edx), %mm4
+// CHECK: encoding: [0x0f,0x3a,0x0f,0x22,0x00]
+palignr $0, (%edx), %mm4
+
+// CHECK: palignr $0, (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x3a,0x0f,0x0a,0x00]
+palignr $0, (%edx), %xmm1
+
+// CHECK: palignr $0, %mm4, %mm4
+// CHECK: encoding: [0x0f,0x3a,0x0f,0xe4,0x00]
+palignr $0, %mm4, %mm4
+
+// CHECK: palignr $0, %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x3a,0x0f,0xc9,0x00]
+palignr $0, %xmm1, %xmm1
+
+// CHECK: phaddd -485498096(%edx,%eax,4), %mm4
+// CHECK: encoding: [0x0f,0x38,0x02,0xa4,0x82,0x10,0xe3,0x0f,0xe3]
+phaddd -485498096(%edx,%eax,4), %mm4
+
+// CHECK: phaddd 485498096(%edx,%eax,4), %mm4
+// CHECK: encoding: [0x0f,0x38,0x02,0xa4,0x82,0xf0,0x1c,0xf0,0x1c]
+phaddd 485498096(%edx,%eax,4), %mm4
+
+// CHECK: phaddd -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x02,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+phaddd -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: phaddd 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x02,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+phaddd 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: phaddd 485498096(%edx), %mm4
+// CHECK: encoding: [0x0f,0x38,0x02,0xa2,0xf0,0x1c,0xf0,0x1c]
+phaddd 485498096(%edx), %mm4
+
+// CHECK: phaddd 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x02,0x8a,0xf0,0x1c,0xf0,0x1c]
+phaddd 485498096(%edx), %xmm1
+
+// CHECK: phaddd 485498096, %mm4
+// CHECK: encoding: [0x0f,0x38,0x02,0x25,0xf0,0x1c,0xf0,0x1c]
+phaddd 485498096, %mm4
+
+// CHECK: phaddd 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x02,0x0d,0xf0,0x1c,0xf0,0x1c]
+phaddd 485498096, %xmm1
+
+// CHECK: phaddd 64(%edx,%eax), %mm4
+// CHECK: encoding: [0x0f,0x38,0x02,0x64,0x02,0x40]
+phaddd 64(%edx,%eax), %mm4
+
+// CHECK: phaddd 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x02,0x4c,0x02,0x40]
+phaddd 64(%edx,%eax), %xmm1
+
+// CHECK: phaddd (%edx), %mm4
+// CHECK: encoding: [0x0f,0x38,0x02,0x22]
+phaddd (%edx), %mm4
+
+// CHECK: phaddd (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x02,0x0a]
+phaddd (%edx), %xmm1
+
+// CHECK: phaddd %mm4, %mm4
+// CHECK: encoding: [0x0f,0x38,0x02,0xe4]
+phaddd %mm4, %mm4
+
+// CHECK: phaddd %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x02,0xc9]
+phaddd %xmm1, %xmm1
+
+// CHECK: phaddsw -485498096(%edx,%eax,4), %mm4
+// CHECK: encoding: [0x0f,0x38,0x03,0xa4,0x82,0x10,0xe3,0x0f,0xe3]
+phaddsw -485498096(%edx,%eax,4), %mm4
+
+// CHECK: phaddsw 485498096(%edx,%eax,4), %mm4
+// CHECK: encoding: [0x0f,0x38,0x03,0xa4,0x82,0xf0,0x1c,0xf0,0x1c]
+phaddsw 485498096(%edx,%eax,4), %mm4
+
+// CHECK: phaddsw -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x03,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+phaddsw -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: phaddsw 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x03,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+phaddsw 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: phaddsw 485498096(%edx), %mm4
+// CHECK: encoding: [0x0f,0x38,0x03,0xa2,0xf0,0x1c,0xf0,0x1c]
+phaddsw 485498096(%edx), %mm4
+
+// CHECK: phaddsw 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x03,0x8a,0xf0,0x1c,0xf0,0x1c]
+phaddsw 485498096(%edx), %xmm1
+
+// CHECK: phaddsw 485498096, %mm4
+// CHECK: encoding: [0x0f,0x38,0x03,0x25,0xf0,0x1c,0xf0,0x1c]
+phaddsw 485498096, %mm4
+
+// CHECK: phaddsw 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x03,0x0d,0xf0,0x1c,0xf0,0x1c]
+phaddsw 485498096, %xmm1
+
+// CHECK: phaddsw 64(%edx,%eax), %mm4
+// CHECK: encoding: [0x0f,0x38,0x03,0x64,0x02,0x40]
+phaddsw 64(%edx,%eax), %mm4
+
+// CHECK: phaddsw 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x03,0x4c,0x02,0x40]
+phaddsw 64(%edx,%eax), %xmm1
+
+// CHECK: phaddsw (%edx), %mm4
+// CHECK: encoding: [0x0f,0x38,0x03,0x22]
+phaddsw (%edx), %mm4
+
+// CHECK: phaddsw (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x03,0x0a]
+phaddsw (%edx), %xmm1
+
+// CHECK: phaddsw %mm4, %mm4
+// CHECK: encoding: [0x0f,0x38,0x03,0xe4]
+phaddsw %mm4, %mm4
+
+// CHECK: phaddsw %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x03,0xc9]
+phaddsw %xmm1, %xmm1
+
+// CHECK: phaddw -485498096(%edx,%eax,4), %mm4
+// CHECK: encoding: [0x0f,0x38,0x01,0xa4,0x82,0x10,0xe3,0x0f,0xe3]
+phaddw -485498096(%edx,%eax,4), %mm4
+
+// CHECK: phaddw 485498096(%edx,%eax,4), %mm4
+// CHECK: encoding: [0x0f,0x38,0x01,0xa4,0x82,0xf0,0x1c,0xf0,0x1c]
+phaddw 485498096(%edx,%eax,4), %mm4
+
+// CHECK: phaddw -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x01,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+phaddw -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: phaddw 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x01,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+phaddw 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: phaddw 485498096(%edx), %mm4
+// CHECK: encoding: [0x0f,0x38,0x01,0xa2,0xf0,0x1c,0xf0,0x1c]
+phaddw 485498096(%edx), %mm4
+
+// CHECK: phaddw 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x01,0x8a,0xf0,0x1c,0xf0,0x1c]
+phaddw 485498096(%edx), %xmm1
+
+// CHECK: phaddw 485498096, %mm4
+// CHECK: encoding: [0x0f,0x38,0x01,0x25,0xf0,0x1c,0xf0,0x1c]
+phaddw 485498096, %mm4
+
+// CHECK: phaddw 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x01,0x0d,0xf0,0x1c,0xf0,0x1c]
+phaddw 485498096, %xmm1
+
+// CHECK: phaddw 64(%edx,%eax), %mm4
+// CHECK: encoding: [0x0f,0x38,0x01,0x64,0x02,0x40]
+phaddw 64(%edx,%eax), %mm4
+
+// CHECK: phaddw 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x01,0x4c,0x02,0x40]
+phaddw 64(%edx,%eax), %xmm1
+
+// CHECK: phaddw (%edx), %mm4
+// CHECK: encoding: [0x0f,0x38,0x01,0x22]
+phaddw (%edx), %mm4
+
+// CHECK: phaddw (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x01,0x0a]
+phaddw (%edx), %xmm1
+
+// CHECK: phaddw %mm4, %mm4
+// CHECK: encoding: [0x0f,0x38,0x01,0xe4]
+phaddw %mm4, %mm4
+
+// CHECK: phaddw %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x01,0xc9]
+phaddw %xmm1, %xmm1
+
+// CHECK: phsubd -485498096(%edx,%eax,4), %mm4
+// CHECK: encoding: [0x0f,0x38,0x06,0xa4,0x82,0x10,0xe3,0x0f,0xe3]
+phsubd -485498096(%edx,%eax,4), %mm4
+
+// CHECK: phsubd 485498096(%edx,%eax,4), %mm4
+// CHECK: encoding: [0x0f,0x38,0x06,0xa4,0x82,0xf0,0x1c,0xf0,0x1c]
+phsubd 485498096(%edx,%eax,4), %mm4
+
+// CHECK: phsubd -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x06,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+phsubd -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: phsubd 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x06,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+phsubd 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: phsubd 485498096(%edx), %mm4
+// CHECK: encoding: [0x0f,0x38,0x06,0xa2,0xf0,0x1c,0xf0,0x1c]
+phsubd 485498096(%edx), %mm4
+
+// CHECK: phsubd 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x06,0x8a,0xf0,0x1c,0xf0,0x1c]
+phsubd 485498096(%edx), %xmm1
+
+// CHECK: phsubd 485498096, %mm4
+// CHECK: encoding: [0x0f,0x38,0x06,0x25,0xf0,0x1c,0xf0,0x1c]
+phsubd 485498096, %mm4
+
+// CHECK: phsubd 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x06,0x0d,0xf0,0x1c,0xf0,0x1c]
+phsubd 485498096, %xmm1
+
+// CHECK: phsubd 64(%edx,%eax), %mm4
+// CHECK: encoding: [0x0f,0x38,0x06,0x64,0x02,0x40]
+phsubd 64(%edx,%eax), %mm4
+
+// CHECK: phsubd 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x06,0x4c,0x02,0x40]
+phsubd 64(%edx,%eax), %xmm1
+
+// CHECK: phsubd (%edx), %mm4
+// CHECK: encoding: [0x0f,0x38,0x06,0x22]
+phsubd (%edx), %mm4
+
+// CHECK: phsubd (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x06,0x0a]
+phsubd (%edx), %xmm1
+
+// CHECK: phsubd %mm4, %mm4
+// CHECK: encoding: [0x0f,0x38,0x06,0xe4]
+phsubd %mm4, %mm4
+
+// CHECK: phsubd %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x06,0xc9]
+phsubd %xmm1, %xmm1
+
+// CHECK: phsubsw -485498096(%edx,%eax,4), %mm4
+// CHECK: encoding: [0x0f,0x38,0x07,0xa4,0x82,0x10,0xe3,0x0f,0xe3]
+phsubsw -485498096(%edx,%eax,4), %mm4
+
+// CHECK: phsubsw 485498096(%edx,%eax,4), %mm4
+// CHECK: encoding: [0x0f,0x38,0x07,0xa4,0x82,0xf0,0x1c,0xf0,0x1c]
+phsubsw 485498096(%edx,%eax,4), %mm4
+
+// CHECK: phsubsw -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x07,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+phsubsw -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: phsubsw 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x07,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+phsubsw 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: phsubsw 485498096(%edx), %mm4
+// CHECK: encoding: [0x0f,0x38,0x07,0xa2,0xf0,0x1c,0xf0,0x1c]
+phsubsw 485498096(%edx), %mm4
+
+// CHECK: phsubsw 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x07,0x8a,0xf0,0x1c,0xf0,0x1c]
+phsubsw 485498096(%edx), %xmm1
+
+// CHECK: phsubsw 485498096, %mm4
+// CHECK: encoding: [0x0f,0x38,0x07,0x25,0xf0,0x1c,0xf0,0x1c]
+phsubsw 485498096, %mm4
+
+// CHECK: phsubsw 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x07,0x0d,0xf0,0x1c,0xf0,0x1c]
+phsubsw 485498096, %xmm1
+
+// CHECK: phsubsw 64(%edx,%eax), %mm4
+// CHECK: encoding: [0x0f,0x38,0x07,0x64,0x02,0x40]
+phsubsw 64(%edx,%eax), %mm4
+
+// CHECK: phsubsw 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x07,0x4c,0x02,0x40]
+phsubsw 64(%edx,%eax), %xmm1
+
+// CHECK: phsubsw (%edx), %mm4
+// CHECK: encoding: [0x0f,0x38,0x07,0x22]
+phsubsw (%edx), %mm4
+
+// CHECK: phsubsw (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x07,0x0a]
+phsubsw (%edx), %xmm1
+
+// CHECK: phsubsw %mm4, %mm4
+// CHECK: encoding: [0x0f,0x38,0x07,0xe4]
+phsubsw %mm4, %mm4
+
+// CHECK: phsubsw %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x07,0xc9]
+phsubsw %xmm1, %xmm1
+
+// CHECK: phsubw -485498096(%edx,%eax,4), %mm4
+// CHECK: encoding: [0x0f,0x38,0x05,0xa4,0x82,0x10,0xe3,0x0f,0xe3]
+phsubw -485498096(%edx,%eax,4), %mm4
+
+// CHECK: phsubw 485498096(%edx,%eax,4), %mm4
+// CHECK: encoding: [0x0f,0x38,0x05,0xa4,0x82,0xf0,0x1c,0xf0,0x1c]
+phsubw 485498096(%edx,%eax,4), %mm4
+
+// CHECK: phsubw -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x05,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+phsubw -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: phsubw 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x05,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+phsubw 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: phsubw 485498096(%edx), %mm4
+// CHECK: encoding: [0x0f,0x38,0x05,0xa2,0xf0,0x1c,0xf0,0x1c]
+phsubw 485498096(%edx), %mm4
+
+// CHECK: phsubw 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x05,0x8a,0xf0,0x1c,0xf0,0x1c]
+phsubw 485498096(%edx), %xmm1
+
+// CHECK: phsubw 485498096, %mm4
+// CHECK: encoding: [0x0f,0x38,0x05,0x25,0xf0,0x1c,0xf0,0x1c]
+phsubw 485498096, %mm4
+
+// CHECK: phsubw 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x05,0x0d,0xf0,0x1c,0xf0,0x1c]
+phsubw 485498096, %xmm1
+
+// CHECK: phsubw 64(%edx,%eax), %mm4
+// CHECK: encoding: [0x0f,0x38,0x05,0x64,0x02,0x40]
+phsubw 64(%edx,%eax), %mm4
+
+// CHECK: phsubw 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x05,0x4c,0x02,0x40]
+phsubw 64(%edx,%eax), %xmm1
+
+// CHECK: phsubw (%edx), %mm4
+// CHECK: encoding: [0x0f,0x38,0x05,0x22]
+phsubw (%edx), %mm4
+
+// CHECK: phsubw (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x05,0x0a]
+phsubw (%edx), %xmm1
+
+// CHECK: phsubw %mm4, %mm4
+// CHECK: encoding: [0x0f,0x38,0x05,0xe4]
+phsubw %mm4, %mm4
+
+// CHECK: phsubw %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x05,0xc9]
+phsubw %xmm1, %xmm1
+
+// CHECK: pmaddubsw -485498096(%edx,%eax,4), %mm4
+// CHECK: encoding: [0x0f,0x38,0x04,0xa4,0x82,0x10,0xe3,0x0f,0xe3]
+pmaddubsw -485498096(%edx,%eax,4), %mm4
+
+// CHECK: pmaddubsw 485498096(%edx,%eax,4), %mm4
+// CHECK: encoding: [0x0f,0x38,0x04,0xa4,0x82,0xf0,0x1c,0xf0,0x1c]
+pmaddubsw 485498096(%edx,%eax,4), %mm4
+
+// CHECK: pmaddubsw -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x04,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+pmaddubsw -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pmaddubsw 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x04,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+pmaddubsw 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pmaddubsw 485498096(%edx), %mm4
+// CHECK: encoding: [0x0f,0x38,0x04,0xa2,0xf0,0x1c,0xf0,0x1c]
+pmaddubsw 485498096(%edx), %mm4
+
+// CHECK: pmaddubsw 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x04,0x8a,0xf0,0x1c,0xf0,0x1c]
+pmaddubsw 485498096(%edx), %xmm1
+
+// CHECK: pmaddubsw 485498096, %mm4
+// CHECK: encoding: [0x0f,0x38,0x04,0x25,0xf0,0x1c,0xf0,0x1c]
+pmaddubsw 485498096, %mm4
+
+// CHECK: pmaddubsw 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x04,0x0d,0xf0,0x1c,0xf0,0x1c]
+pmaddubsw 485498096, %xmm1
+
+// CHECK: pmaddubsw 64(%edx,%eax), %mm4
+// CHECK: encoding: [0x0f,0x38,0x04,0x64,0x02,0x40]
+pmaddubsw 64(%edx,%eax), %mm4
+
+// CHECK: pmaddubsw 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x04,0x4c,0x02,0x40]
+pmaddubsw 64(%edx,%eax), %xmm1
+
+// CHECK: pmaddubsw (%edx), %mm4
+// CHECK: encoding: [0x0f,0x38,0x04,0x22]
+pmaddubsw (%edx), %mm4
+
+// CHECK: pmaddubsw (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x04,0x0a]
+pmaddubsw (%edx), %xmm1
+
+// CHECK: pmaddubsw %mm4, %mm4
+// CHECK: encoding: [0x0f,0x38,0x04,0xe4]
+pmaddubsw %mm4, %mm4
+
+// CHECK: pmaddubsw %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x04,0xc9]
+pmaddubsw %xmm1, %xmm1
+
+// CHECK: pmulhrsw -485498096(%edx,%eax,4), %mm4
+// CHECK: encoding: [0x0f,0x38,0x0b,0xa4,0x82,0x10,0xe3,0x0f,0xe3]
+pmulhrsw -485498096(%edx,%eax,4), %mm4
+
+// CHECK: pmulhrsw 485498096(%edx,%eax,4), %mm4
+// CHECK: encoding: [0x0f,0x38,0x0b,0xa4,0x82,0xf0,0x1c,0xf0,0x1c]
+pmulhrsw 485498096(%edx,%eax,4), %mm4
+
+// CHECK: pmulhrsw -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x0b,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+pmulhrsw -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pmulhrsw 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x0b,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+pmulhrsw 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pmulhrsw 485498096(%edx), %mm4
+// CHECK: encoding: [0x0f,0x38,0x0b,0xa2,0xf0,0x1c,0xf0,0x1c]
+pmulhrsw 485498096(%edx), %mm4
+
+// CHECK: pmulhrsw 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x0b,0x8a,0xf0,0x1c,0xf0,0x1c]
+pmulhrsw 485498096(%edx), %xmm1
+
+// CHECK: pmulhrsw 485498096, %mm4
+// CHECK: encoding: [0x0f,0x38,0x0b,0x25,0xf0,0x1c,0xf0,0x1c]
+pmulhrsw 485498096, %mm4
+
+// CHECK: pmulhrsw 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x0b,0x0d,0xf0,0x1c,0xf0,0x1c]
+pmulhrsw 485498096, %xmm1
+
+// CHECK: pmulhrsw 64(%edx,%eax), %mm4
+// CHECK: encoding: [0x0f,0x38,0x0b,0x64,0x02,0x40]
+pmulhrsw 64(%edx,%eax), %mm4
+
+// CHECK: pmulhrsw 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x0b,0x4c,0x02,0x40]
+pmulhrsw 64(%edx,%eax), %xmm1
+
+// CHECK: pmulhrsw (%edx), %mm4
+// CHECK: encoding: [0x0f,0x38,0x0b,0x22]
+pmulhrsw (%edx), %mm4
+
+// CHECK: pmulhrsw (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x0b,0x0a]
+pmulhrsw (%edx), %xmm1
+
+// CHECK: pmulhrsw %mm4, %mm4
+// CHECK: encoding: [0x0f,0x38,0x0b,0xe4]
+pmulhrsw %mm4, %mm4
+
+// CHECK: pmulhrsw %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x0b,0xc9]
+pmulhrsw %xmm1, %xmm1
+
+// CHECK: pshufb -485498096(%edx,%eax,4), %mm4
+// CHECK: encoding: [0x0f,0x38,0x00,0xa4,0x82,0x10,0xe3,0x0f,0xe3]
+pshufb -485498096(%edx,%eax,4), %mm4
+
+// CHECK: pshufb 485498096(%edx,%eax,4), %mm4
+// CHECK: encoding: [0x0f,0x38,0x00,0xa4,0x82,0xf0,0x1c,0xf0,0x1c]
+pshufb 485498096(%edx,%eax,4), %mm4
+
+// CHECK: pshufb -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x00,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+pshufb -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pshufb 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x00,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+pshufb 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: pshufb 485498096(%edx), %mm4
+// CHECK: encoding: [0x0f,0x38,0x00,0xa2,0xf0,0x1c,0xf0,0x1c]
+pshufb 485498096(%edx), %mm4
+
+// CHECK: pshufb 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x00,0x8a,0xf0,0x1c,0xf0,0x1c]
+pshufb 485498096(%edx), %xmm1
+
+// CHECK: pshufb 485498096, %mm4
+// CHECK: encoding: [0x0f,0x38,0x00,0x25,0xf0,0x1c,0xf0,0x1c]
+pshufb 485498096, %mm4
+
+// CHECK: pshufb 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x00,0x0d,0xf0,0x1c,0xf0,0x1c]
+pshufb 485498096, %xmm1
+
+// CHECK: pshufb 64(%edx,%eax), %mm4
+// CHECK: encoding: [0x0f,0x38,0x00,0x64,0x02,0x40]
+pshufb 64(%edx,%eax), %mm4
+
+// CHECK: pshufb 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x00,0x4c,0x02,0x40]
+pshufb 64(%edx,%eax), %xmm1
+
+// CHECK: pshufb (%edx), %mm4
+// CHECK: encoding: [0x0f,0x38,0x00,0x22]
+pshufb (%edx), %mm4
+
+// CHECK: pshufb (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x00,0x0a]
+pshufb (%edx), %xmm1
+
+// CHECK: pshufb %mm4, %mm4
+// CHECK: encoding: [0x0f,0x38,0x00,0xe4]
+pshufb %mm4, %mm4
+
+// CHECK: pshufb %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x00,0xc9]
+pshufb %xmm1, %xmm1
+
+// CHECK: psignb -485498096(%edx,%eax,4), %mm4
+// CHECK: encoding: [0x0f,0x38,0x08,0xa4,0x82,0x10,0xe3,0x0f,0xe3]
+psignb -485498096(%edx,%eax,4), %mm4
+
+// CHECK: psignb 485498096(%edx,%eax,4), %mm4
+// CHECK: encoding: [0x0f,0x38,0x08,0xa4,0x82,0xf0,0x1c,0xf0,0x1c]
+psignb 485498096(%edx,%eax,4), %mm4
+
+// CHECK: psignb -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x08,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+psignb -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: psignb 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x08,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+psignb 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: psignb 485498096(%edx), %mm4
+// CHECK: encoding: [0x0f,0x38,0x08,0xa2,0xf0,0x1c,0xf0,0x1c]
+psignb 485498096(%edx), %mm4
+
+// CHECK: psignb 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x08,0x8a,0xf0,0x1c,0xf0,0x1c]
+psignb 485498096(%edx), %xmm1
+
+// CHECK: psignb 485498096, %mm4
+// CHECK: encoding: [0x0f,0x38,0x08,0x25,0xf0,0x1c,0xf0,0x1c]
+psignb 485498096, %mm4
+
+// CHECK: psignb 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x08,0x0d,0xf0,0x1c,0xf0,0x1c]
+psignb 485498096, %xmm1
+
+// CHECK: psignb 64(%edx,%eax), %mm4
+// CHECK: encoding: [0x0f,0x38,0x08,0x64,0x02,0x40]
+psignb 64(%edx,%eax), %mm4
+
+// CHECK: psignb 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x08,0x4c,0x02,0x40]
+psignb 64(%edx,%eax), %xmm1
+
+// CHECK: psignb (%edx), %mm4
+// CHECK: encoding: [0x0f,0x38,0x08,0x22]
+psignb (%edx), %mm4
+
+// CHECK: psignb (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x08,0x0a]
+psignb (%edx), %xmm1
+
+// CHECK: psignb %mm4, %mm4
+// CHECK: encoding: [0x0f,0x38,0x08,0xe4]
+psignb %mm4, %mm4
+
+// CHECK: psignb %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x08,0xc9]
+psignb %xmm1, %xmm1
+
+// CHECK: psignd -485498096(%edx,%eax,4), %mm4
+// CHECK: encoding: [0x0f,0x38,0x0a,0xa4,0x82,0x10,0xe3,0x0f,0xe3]
+psignd -485498096(%edx,%eax,4), %mm4
+
+// CHECK: psignd 485498096(%edx,%eax,4), %mm4
+// CHECK: encoding: [0x0f,0x38,0x0a,0xa4,0x82,0xf0,0x1c,0xf0,0x1c]
+psignd 485498096(%edx,%eax,4), %mm4
+
+// CHECK: psignd -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x0a,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+psignd -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: psignd 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x0a,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+psignd 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: psignd 485498096(%edx), %mm4
+// CHECK: encoding: [0x0f,0x38,0x0a,0xa2,0xf0,0x1c,0xf0,0x1c]
+psignd 485498096(%edx), %mm4
+
+// CHECK: psignd 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x0a,0x8a,0xf0,0x1c,0xf0,0x1c]
+psignd 485498096(%edx), %xmm1
+
+// CHECK: psignd 485498096, %mm4
+// CHECK: encoding: [0x0f,0x38,0x0a,0x25,0xf0,0x1c,0xf0,0x1c]
+psignd 485498096, %mm4
+
+// CHECK: psignd 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x0a,0x0d,0xf0,0x1c,0xf0,0x1c]
+psignd 485498096, %xmm1
+
+// CHECK: psignd 64(%edx,%eax), %mm4
+// CHECK: encoding: [0x0f,0x38,0x0a,0x64,0x02,0x40]
+psignd 64(%edx,%eax), %mm4
+
+// CHECK: psignd 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x0a,0x4c,0x02,0x40]
+psignd 64(%edx,%eax), %xmm1
+
+// CHECK: psignd (%edx), %mm4
+// CHECK: encoding: [0x0f,0x38,0x0a,0x22]
+psignd (%edx), %mm4
+
+// CHECK: psignd (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x0a,0x0a]
+psignd (%edx), %xmm1
+
+// CHECK: psignd %mm4, %mm4
+// CHECK: encoding: [0x0f,0x38,0x0a,0xe4]
+psignd %mm4, %mm4
+
+// CHECK: psignd %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x0a,0xc9]
+psignd %xmm1, %xmm1
+
+// CHECK: psignw -485498096(%edx,%eax,4), %mm4
+// CHECK: encoding: [0x0f,0x38,0x09,0xa4,0x82,0x10,0xe3,0x0f,0xe3]
+psignw -485498096(%edx,%eax,4), %mm4
+
+// CHECK: psignw 485498096(%edx,%eax,4), %mm4
+// CHECK: encoding: [0x0f,0x38,0x09,0xa4,0x82,0xf0,0x1c,0xf0,0x1c]
+psignw 485498096(%edx,%eax,4), %mm4
+
+// CHECK: psignw -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x09,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+psignw -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: psignw 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x09,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+psignw 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: psignw 485498096(%edx), %mm4
+// CHECK: encoding: [0x0f,0x38,0x09,0xa2,0xf0,0x1c,0xf0,0x1c]
+psignw 485498096(%edx), %mm4
+
+// CHECK: psignw 485498096(%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x09,0x8a,0xf0,0x1c,0xf0,0x1c]
+psignw 485498096(%edx), %xmm1
+
+// CHECK: psignw 485498096, %mm4
+// CHECK: encoding: [0x0f,0x38,0x09,0x25,0xf0,0x1c,0xf0,0x1c]
+psignw 485498096, %mm4
+
+// CHECK: psignw 485498096, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x09,0x0d,0xf0,0x1c,0xf0,0x1c]
+psignw 485498096, %xmm1
+
+// CHECK: psignw 64(%edx,%eax), %mm4
+// CHECK: encoding: [0x0f,0x38,0x09,0x64,0x02,0x40]
+psignw 64(%edx,%eax), %mm4
+
+// CHECK: psignw 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x09,0x4c,0x02,0x40]
+psignw 64(%edx,%eax), %xmm1
+
+// CHECK: psignw (%edx), %mm4
+// CHECK: encoding: [0x0f,0x38,0x09,0x22]
+psignw (%edx), %mm4
+
+// CHECK: psignw (%edx), %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x09,0x0a]
+psignw (%edx), %xmm1
+
+// CHECK: psignw %mm4, %mm4
+// CHECK: encoding: [0x0f,0x38,0x09,0xe4]
+psignw %mm4, %mm4
+
+// CHECK: psignw %xmm1, %xmm1
+// CHECK: encoding: [0x66,0x0f,0x38,0x09,0xc9]
+psignw %xmm1, %xmm1
+
diff --git a/test/MC/X86/SSSE3-64.s b/test/MC/X86/SSSE3-64.s
new file mode 100644
index 0000000000000000000000000000000000000000..79b29b7f92a4c098a4d8bb88b1fedc3a1ab34458
--- /dev/null
+++ b/test/MC/X86/SSSE3-64.s
@@ -0,0 +1,898 @@
+// RUN: llvm-mc -triple x86_64-unknown-unknown --show-encoding %s | FileCheck %s
+
+// CHECK: pabsb 485498096, %mm4
+// CHECK: encoding: [0x0f,0x38,0x1c,0x24,0x25,0xf0,0x1c,0xf0,0x1c]
+pabsb 485498096, %mm4
+
+// CHECK: pabsb 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x1c,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+pabsb 485498096, %xmm6
+
+// CHECK: pabsb 64(%rdx), %mm4
+// CHECK: encoding: [0x0f,0x38,0x1c,0x62,0x40]
+pabsb 64(%rdx), %mm4
+
+// CHECK: pabsb -64(%rdx,%rax,4), %mm4
+// CHECK: encoding: [0x0f,0x38,0x1c,0x64,0x82,0xc0]
+pabsb -64(%rdx,%rax,4), %mm4
+
+// CHECK: pabsb 64(%rdx,%rax,4), %mm4
+// CHECK: encoding: [0x0f,0x38,0x1c,0x64,0x82,0x40]
+pabsb 64(%rdx,%rax,4), %mm4
+
+// CHECK: pabsb -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x1c,0x74,0x82,0xc0]
+pabsb -64(%rdx,%rax,4), %xmm6
+
+// CHECK: pabsb 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x1c,0x74,0x82,0x40]
+pabsb 64(%rdx,%rax,4), %xmm6
+
+// CHECK: pabsb 64(%rdx,%rax), %mm4
+// CHECK: encoding: [0x0f,0x38,0x1c,0x64,0x02,0x40]
+pabsb 64(%rdx,%rax), %mm4
+
+// CHECK: pabsb 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x1c,0x74,0x02,0x40]
+pabsb 64(%rdx,%rax), %xmm6
+
+// CHECK: pabsb 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x1c,0x72,0x40]
+pabsb 64(%rdx), %xmm6
+
+// CHECK: pabsb %mm4, %mm4
+// CHECK: encoding: [0x0f,0x38,0x1c,0xe4]
+pabsb %mm4, %mm4
+
+// CHECK: pabsb (%rdx), %mm4
+// CHECK: encoding: [0x0f,0x38,0x1c,0x22]
+pabsb (%rdx), %mm4
+
+// CHECK: pabsb (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x1c,0x32]
+pabsb (%rdx), %xmm6
+
+// CHECK: pabsb %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x1c,0xf6]
+pabsb %xmm6, %xmm6
+
+// CHECK: pabsd 485498096, %mm4
+// CHECK: encoding: [0x0f,0x38,0x1e,0x24,0x25,0xf0,0x1c,0xf0,0x1c]
+pabsd 485498096, %mm4
+
+// CHECK: pabsd 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x1e,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+pabsd 485498096, %xmm6
+
+// CHECK: pabsd 64(%rdx), %mm4
+// CHECK: encoding: [0x0f,0x38,0x1e,0x62,0x40]
+pabsd 64(%rdx), %mm4
+
+// CHECK: pabsd -64(%rdx,%rax,4), %mm4
+// CHECK: encoding: [0x0f,0x38,0x1e,0x64,0x82,0xc0]
+pabsd -64(%rdx,%rax,4), %mm4
+
+// CHECK: pabsd 64(%rdx,%rax,4), %mm4
+// CHECK: encoding: [0x0f,0x38,0x1e,0x64,0x82,0x40]
+pabsd 64(%rdx,%rax,4), %mm4
+
+// CHECK: pabsd -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x1e,0x74,0x82,0xc0]
+pabsd -64(%rdx,%rax,4), %xmm6
+
+// CHECK: pabsd 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x1e,0x74,0x82,0x40]
+pabsd 64(%rdx,%rax,4), %xmm6
+
+// CHECK: pabsd 64(%rdx,%rax), %mm4
+// CHECK: encoding: [0x0f,0x38,0x1e,0x64,0x02,0x40]
+pabsd 64(%rdx,%rax), %mm4
+
+// CHECK: pabsd 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x1e,0x74,0x02,0x40]
+pabsd 64(%rdx,%rax), %xmm6
+
+// CHECK: pabsd 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x1e,0x72,0x40]
+pabsd 64(%rdx), %xmm6
+
+// CHECK: pabsd %mm4, %mm4
+// CHECK: encoding: [0x0f,0x38,0x1e,0xe4]
+pabsd %mm4, %mm4
+
+// CHECK: pabsd (%rdx), %mm4
+// CHECK: encoding: [0x0f,0x38,0x1e,0x22]
+pabsd (%rdx), %mm4
+
+// CHECK: pabsd (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x1e,0x32]
+pabsd (%rdx), %xmm6
+
+// CHECK: pabsd %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x1e,0xf6]
+pabsd %xmm6, %xmm6
+
+// CHECK: pabsw 485498096, %mm4
+// CHECK: encoding: [0x0f,0x38,0x1d,0x24,0x25,0xf0,0x1c,0xf0,0x1c]
+pabsw 485498096, %mm4
+
+// CHECK: pabsw 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x1d,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+pabsw 485498096, %xmm6
+
+// CHECK: pabsw 64(%rdx), %mm4
+// CHECK: encoding: [0x0f,0x38,0x1d,0x62,0x40]
+pabsw 64(%rdx), %mm4
+
+// CHECK: pabsw -64(%rdx,%rax,4), %mm4
+// CHECK: encoding: [0x0f,0x38,0x1d,0x64,0x82,0xc0]
+pabsw -64(%rdx,%rax,4), %mm4
+
+// CHECK: pabsw 64(%rdx,%rax,4), %mm4
+// CHECK: encoding: [0x0f,0x38,0x1d,0x64,0x82,0x40]
+pabsw 64(%rdx,%rax,4), %mm4
+
+// CHECK: pabsw -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x1d,0x74,0x82,0xc0]
+pabsw -64(%rdx,%rax,4), %xmm6
+
+// CHECK: pabsw 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x1d,0x74,0x82,0x40]
+pabsw 64(%rdx,%rax,4), %xmm6
+
+// CHECK: pabsw 64(%rdx,%rax), %mm4
+// CHECK: encoding: [0x0f,0x38,0x1d,0x64,0x02,0x40]
+pabsw 64(%rdx,%rax), %mm4
+
+// CHECK: pabsw 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x1d,0x74,0x02,0x40]
+pabsw 64(%rdx,%rax), %xmm6
+
+// CHECK: pabsw 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x1d,0x72,0x40]
+pabsw 64(%rdx), %xmm6
+
+// CHECK: pabsw %mm4, %mm4
+// CHECK: encoding: [0x0f,0x38,0x1d,0xe4]
+pabsw %mm4, %mm4
+
+// CHECK: pabsw (%rdx), %mm4
+// CHECK: encoding: [0x0f,0x38,0x1d,0x22]
+pabsw (%rdx), %mm4
+
+// CHECK: pabsw (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x1d,0x32]
+pabsw (%rdx), %xmm6
+
+// CHECK: pabsw %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x1d,0xf6]
+pabsw %xmm6, %xmm6
+
+// CHECK: palignr $0, 485498096, %mm4
+// CHECK: encoding: [0x0f,0x3a,0x0f,0x24,0x25,0xf0,0x1c,0xf0,0x1c,0x00]
+palignr $0, 485498096, %mm4
+
+// CHECK: palignr $0, 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x3a,0x0f,0x34,0x25,0xf0,0x1c,0xf0,0x1c,0x00]
+palignr $0, 485498096, %xmm6
+
+// CHECK: palignr $0, 64(%rdx), %mm4
+// CHECK: encoding: [0x0f,0x3a,0x0f,0x62,0x40,0x00]
+palignr $0, 64(%rdx), %mm4
+
+// CHECK: palignr $0, -64(%rdx,%rax,4), %mm4
+// CHECK: encoding: [0x0f,0x3a,0x0f,0x64,0x82,0xc0,0x00]
+palignr $0, -64(%rdx,%rax,4), %mm4
+
+// CHECK: palignr $0, 64(%rdx,%rax,4), %mm4
+// CHECK: encoding: [0x0f,0x3a,0x0f,0x64,0x82,0x40,0x00]
+palignr $0, 64(%rdx,%rax,4), %mm4
+
+// CHECK: palignr $0, -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x3a,0x0f,0x74,0x82,0xc0,0x00]
+palignr $0, -64(%rdx,%rax,4), %xmm6
+
+// CHECK: palignr $0, 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x3a,0x0f,0x74,0x82,0x40,0x00]
+palignr $0, 64(%rdx,%rax,4), %xmm6
+
+// CHECK: palignr $0, 64(%rdx,%rax), %mm4
+// CHECK: encoding: [0x0f,0x3a,0x0f,0x64,0x02,0x40,0x00]
+palignr $0, 64(%rdx,%rax), %mm4
+
+// CHECK: palignr $0, 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x3a,0x0f,0x74,0x02,0x40,0x00]
+palignr $0, 64(%rdx,%rax), %xmm6
+
+// CHECK: palignr $0, 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x3a,0x0f,0x72,0x40,0x00]
+palignr $0, 64(%rdx), %xmm6
+
+// CHECK: palignr $0, %mm4, %mm4
+// CHECK: encoding: [0x0f,0x3a,0x0f,0xe4,0x00]
+palignr $0, %mm4, %mm4
+
+// CHECK: palignr $0, (%rdx), %mm4
+// CHECK: encoding: [0x0f,0x3a,0x0f,0x22,0x00]
+palignr $0, (%rdx), %mm4
+
+// CHECK: palignr $0, (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x3a,0x0f,0x32,0x00]
+palignr $0, (%rdx), %xmm6
+
+// CHECK: palignr $0, %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x3a,0x0f,0xf6,0x00]
+palignr $0, %xmm6, %xmm6
+
+// CHECK: phaddd 485498096, %mm4
+// CHECK: encoding: [0x0f,0x38,0x02,0x24,0x25,0xf0,0x1c,0xf0,0x1c]
+phaddd 485498096, %mm4
+
+// CHECK: phaddd 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x02,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+phaddd 485498096, %xmm6
+
+// CHECK: phaddd 64(%rdx), %mm4
+// CHECK: encoding: [0x0f,0x38,0x02,0x62,0x40]
+phaddd 64(%rdx), %mm4
+
+// CHECK: phaddd -64(%rdx,%rax,4), %mm4
+// CHECK: encoding: [0x0f,0x38,0x02,0x64,0x82,0xc0]
+phaddd -64(%rdx,%rax,4), %mm4
+
+// CHECK: phaddd 64(%rdx,%rax,4), %mm4
+// CHECK: encoding: [0x0f,0x38,0x02,0x64,0x82,0x40]
+phaddd 64(%rdx,%rax,4), %mm4
+
+// CHECK: phaddd -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x02,0x74,0x82,0xc0]
+phaddd -64(%rdx,%rax,4), %xmm6
+
+// CHECK: phaddd 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x02,0x74,0x82,0x40]
+phaddd 64(%rdx,%rax,4), %xmm6
+
+// CHECK: phaddd 64(%rdx,%rax), %mm4
+// CHECK: encoding: [0x0f,0x38,0x02,0x64,0x02,0x40]
+phaddd 64(%rdx,%rax), %mm4
+
+// CHECK: phaddd 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x02,0x74,0x02,0x40]
+phaddd 64(%rdx,%rax), %xmm6
+
+// CHECK: phaddd 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x02,0x72,0x40]
+phaddd 64(%rdx), %xmm6
+
+// CHECK: phaddd %mm4, %mm4
+// CHECK: encoding: [0x0f,0x38,0x02,0xe4]
+phaddd %mm4, %mm4
+
+// CHECK: phaddd (%rdx), %mm4
+// CHECK: encoding: [0x0f,0x38,0x02,0x22]
+phaddd (%rdx), %mm4
+
+// CHECK: phaddd (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x02,0x32]
+phaddd (%rdx), %xmm6
+
+// CHECK: phaddd %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x02,0xf6]
+phaddd %xmm6, %xmm6
+
+// CHECK: phaddsw 485498096, %mm4
+// CHECK: encoding: [0x0f,0x38,0x03,0x24,0x25,0xf0,0x1c,0xf0,0x1c]
+phaddsw 485498096, %mm4
+
+// CHECK: phaddsw 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x03,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+phaddsw 485498096, %xmm6
+
+// CHECK: phaddsw 64(%rdx), %mm4
+// CHECK: encoding: [0x0f,0x38,0x03,0x62,0x40]
+phaddsw 64(%rdx), %mm4
+
+// CHECK: phaddsw -64(%rdx,%rax,4), %mm4
+// CHECK: encoding: [0x0f,0x38,0x03,0x64,0x82,0xc0]
+phaddsw -64(%rdx,%rax,4), %mm4
+
+// CHECK: phaddsw 64(%rdx,%rax,4), %mm4
+// CHECK: encoding: [0x0f,0x38,0x03,0x64,0x82,0x40]
+phaddsw 64(%rdx,%rax,4), %mm4
+
+// CHECK: phaddsw -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x03,0x74,0x82,0xc0]
+phaddsw -64(%rdx,%rax,4), %xmm6
+
+// CHECK: phaddsw 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x03,0x74,0x82,0x40]
+phaddsw 64(%rdx,%rax,4), %xmm6
+
+// CHECK: phaddsw 64(%rdx,%rax), %mm4
+// CHECK: encoding: [0x0f,0x38,0x03,0x64,0x02,0x40]
+phaddsw 64(%rdx,%rax), %mm4
+
+// CHECK: phaddsw 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x03,0x74,0x02,0x40]
+phaddsw 64(%rdx,%rax), %xmm6
+
+// CHECK: phaddsw 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x03,0x72,0x40]
+phaddsw 64(%rdx), %xmm6
+
+// CHECK: phaddsw %mm4, %mm4
+// CHECK: encoding: [0x0f,0x38,0x03,0xe4]
+phaddsw %mm4, %mm4
+
+// CHECK: phaddsw (%rdx), %mm4
+// CHECK: encoding: [0x0f,0x38,0x03,0x22]
+phaddsw (%rdx), %mm4
+
+// CHECK: phaddsw (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x03,0x32]
+phaddsw (%rdx), %xmm6
+
+// CHECK: phaddsw %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x03,0xf6]
+phaddsw %xmm6, %xmm6
+
+// CHECK: phaddw 485498096, %mm4
+// CHECK: encoding: [0x0f,0x38,0x01,0x24,0x25,0xf0,0x1c,0xf0,0x1c]
+phaddw 485498096, %mm4
+
+// CHECK: phaddw 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x01,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+phaddw 485498096, %xmm6
+
+// CHECK: phaddw 64(%rdx), %mm4
+// CHECK: encoding: [0x0f,0x38,0x01,0x62,0x40]
+phaddw 64(%rdx), %mm4
+
+// CHECK: phaddw -64(%rdx,%rax,4), %mm4
+// CHECK: encoding: [0x0f,0x38,0x01,0x64,0x82,0xc0]
+phaddw -64(%rdx,%rax,4), %mm4
+
+// CHECK: phaddw 64(%rdx,%rax,4), %mm4
+// CHECK: encoding: [0x0f,0x38,0x01,0x64,0x82,0x40]
+phaddw 64(%rdx,%rax,4), %mm4
+
+// CHECK: phaddw -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x01,0x74,0x82,0xc0]
+phaddw -64(%rdx,%rax,4), %xmm6
+
+// CHECK: phaddw 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x01,0x74,0x82,0x40]
+phaddw 64(%rdx,%rax,4), %xmm6
+
+// CHECK: phaddw 64(%rdx,%rax), %mm4
+// CHECK: encoding: [0x0f,0x38,0x01,0x64,0x02,0x40]
+phaddw 64(%rdx,%rax), %mm4
+
+// CHECK: phaddw 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x01,0x74,0x02,0x40]
+phaddw 64(%rdx,%rax), %xmm6
+
+// CHECK: phaddw 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x01,0x72,0x40]
+phaddw 64(%rdx), %xmm6
+
+// CHECK: phaddw %mm4, %mm4
+// CHECK: encoding: [0x0f,0x38,0x01,0xe4]
+phaddw %mm4, %mm4
+
+// CHECK: phaddw (%rdx), %mm4
+// CHECK: encoding: [0x0f,0x38,0x01,0x22]
+phaddw (%rdx), %mm4
+
+// CHECK: phaddw (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x01,0x32]
+phaddw (%rdx), %xmm6
+
+// CHECK: phaddw %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x01,0xf6]
+phaddw %xmm6, %xmm6
+
+// CHECK: phsubd 485498096, %mm4
+// CHECK: encoding: [0x0f,0x38,0x06,0x24,0x25,0xf0,0x1c,0xf0,0x1c]
+phsubd 485498096, %mm4
+
+// CHECK: phsubd 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x06,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+phsubd 485498096, %xmm6
+
+// CHECK: phsubd 64(%rdx), %mm4
+// CHECK: encoding: [0x0f,0x38,0x06,0x62,0x40]
+phsubd 64(%rdx), %mm4
+
+// CHECK: phsubd -64(%rdx,%rax,4), %mm4
+// CHECK: encoding: [0x0f,0x38,0x06,0x64,0x82,0xc0]
+phsubd -64(%rdx,%rax,4), %mm4
+
+// CHECK: phsubd 64(%rdx,%rax,4), %mm4
+// CHECK: encoding: [0x0f,0x38,0x06,0x64,0x82,0x40]
+phsubd 64(%rdx,%rax,4), %mm4
+
+// CHECK: phsubd -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x06,0x74,0x82,0xc0]
+phsubd -64(%rdx,%rax,4), %xmm6
+
+// CHECK: phsubd 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x06,0x74,0x82,0x40]
+phsubd 64(%rdx,%rax,4), %xmm6
+
+// CHECK: phsubd 64(%rdx,%rax), %mm4
+// CHECK: encoding: [0x0f,0x38,0x06,0x64,0x02,0x40]
+phsubd 64(%rdx,%rax), %mm4
+
+// CHECK: phsubd 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x06,0x74,0x02,0x40]
+phsubd 64(%rdx,%rax), %xmm6
+
+// CHECK: phsubd 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x06,0x72,0x40]
+phsubd 64(%rdx), %xmm6
+
+// CHECK: phsubd %mm4, %mm4
+// CHECK: encoding: [0x0f,0x38,0x06,0xe4]
+phsubd %mm4, %mm4
+
+// CHECK: phsubd (%rdx), %mm4
+// CHECK: encoding: [0x0f,0x38,0x06,0x22]
+phsubd (%rdx), %mm4
+
+// CHECK: phsubd (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x06,0x32]
+phsubd (%rdx), %xmm6
+
+// CHECK: phsubd %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x06,0xf6]
+phsubd %xmm6, %xmm6
+
+// CHECK: phsubsw 485498096, %mm4
+// CHECK: encoding: [0x0f,0x38,0x07,0x24,0x25,0xf0,0x1c,0xf0,0x1c]
+phsubsw 485498096, %mm4
+
+// CHECK: phsubsw 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x07,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+phsubsw 485498096, %xmm6
+
+// CHECK: phsubsw 64(%rdx), %mm4
+// CHECK: encoding: [0x0f,0x38,0x07,0x62,0x40]
+phsubsw 64(%rdx), %mm4
+
+// CHECK: phsubsw -64(%rdx,%rax,4), %mm4
+// CHECK: encoding: [0x0f,0x38,0x07,0x64,0x82,0xc0]
+phsubsw -64(%rdx,%rax,4), %mm4
+
+// CHECK: phsubsw 64(%rdx,%rax,4), %mm4
+// CHECK: encoding: [0x0f,0x38,0x07,0x64,0x82,0x40]
+phsubsw 64(%rdx,%rax,4), %mm4
+
+// CHECK: phsubsw -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x07,0x74,0x82,0xc0]
+phsubsw -64(%rdx,%rax,4), %xmm6
+
+// CHECK: phsubsw 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x07,0x74,0x82,0x40]
+phsubsw 64(%rdx,%rax,4), %xmm6
+
+// CHECK: phsubsw 64(%rdx,%rax), %mm4
+// CHECK: encoding: [0x0f,0x38,0x07,0x64,0x02,0x40]
+phsubsw 64(%rdx,%rax), %mm4
+
+// CHECK: phsubsw 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x07,0x74,0x02,0x40]
+phsubsw 64(%rdx,%rax), %xmm6
+
+// CHECK: phsubsw 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x07,0x72,0x40]
+phsubsw 64(%rdx), %xmm6
+
+// CHECK: phsubsw %mm4, %mm4
+// CHECK: encoding: [0x0f,0x38,0x07,0xe4]
+phsubsw %mm4, %mm4
+
+// CHECK: phsubsw (%rdx), %mm4
+// CHECK: encoding: [0x0f,0x38,0x07,0x22]
+phsubsw (%rdx), %mm4
+
+// CHECK: phsubsw (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x07,0x32]
+phsubsw (%rdx), %xmm6
+
+// CHECK: phsubsw %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x07,0xf6]
+phsubsw %xmm6, %xmm6
+
+// CHECK: phsubw 485498096, %mm4
+// CHECK: encoding: [0x0f,0x38,0x05,0x24,0x25,0xf0,0x1c,0xf0,0x1c]
+phsubw 485498096, %mm4
+
+// CHECK: phsubw 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x05,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+phsubw 485498096, %xmm6
+
+// CHECK: phsubw 64(%rdx), %mm4
+// CHECK: encoding: [0x0f,0x38,0x05,0x62,0x40]
+phsubw 64(%rdx), %mm4
+
+// CHECK: phsubw -64(%rdx,%rax,4), %mm4
+// CHECK: encoding: [0x0f,0x38,0x05,0x64,0x82,0xc0]
+phsubw -64(%rdx,%rax,4), %mm4
+
+// CHECK: phsubw 64(%rdx,%rax,4), %mm4
+// CHECK: encoding: [0x0f,0x38,0x05,0x64,0x82,0x40]
+phsubw 64(%rdx,%rax,4), %mm4
+
+// CHECK: phsubw -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x05,0x74,0x82,0xc0]
+phsubw -64(%rdx,%rax,4), %xmm6
+
+// CHECK: phsubw 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x05,0x74,0x82,0x40]
+phsubw 64(%rdx,%rax,4), %xmm6
+
+// CHECK: phsubw 64(%rdx,%rax), %mm4
+// CHECK: encoding: [0x0f,0x38,0x05,0x64,0x02,0x40]
+phsubw 64(%rdx,%rax), %mm4
+
+// CHECK: phsubw 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x05,0x74,0x02,0x40]
+phsubw 64(%rdx,%rax), %xmm6
+
+// CHECK: phsubw 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x05,0x72,0x40]
+phsubw 64(%rdx), %xmm6
+
+// CHECK: phsubw %mm4, %mm4
+// CHECK: encoding: [0x0f,0x38,0x05,0xe4]
+phsubw %mm4, %mm4
+
+// CHECK: phsubw (%rdx), %mm4
+// CHECK: encoding: [0x0f,0x38,0x05,0x22]
+phsubw (%rdx), %mm4
+
+// CHECK: phsubw (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x05,0x32]
+phsubw (%rdx), %xmm6
+
+// CHECK: phsubw %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x05,0xf6]
+phsubw %xmm6, %xmm6
+
+// CHECK: pmaddubsw 485498096, %mm4
+// CHECK: encoding: [0x0f,0x38,0x04,0x24,0x25,0xf0,0x1c,0xf0,0x1c]
+pmaddubsw 485498096, %mm4
+
+// CHECK: pmaddubsw 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x04,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+pmaddubsw 485498096, %xmm6
+
+// CHECK: pmaddubsw 64(%rdx), %mm4
+// CHECK: encoding: [0x0f,0x38,0x04,0x62,0x40]
+pmaddubsw 64(%rdx), %mm4
+
+// CHECK: pmaddubsw -64(%rdx,%rax,4), %mm4
+// CHECK: encoding: [0x0f,0x38,0x04,0x64,0x82,0xc0]
+pmaddubsw -64(%rdx,%rax,4), %mm4
+
+// CHECK: pmaddubsw 64(%rdx,%rax,4), %mm4
+// CHECK: encoding: [0x0f,0x38,0x04,0x64,0x82,0x40]
+pmaddubsw 64(%rdx,%rax,4), %mm4
+
+// CHECK: pmaddubsw -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x04,0x74,0x82,0xc0]
+pmaddubsw -64(%rdx,%rax,4), %xmm6
+
+// CHECK: pmaddubsw 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x04,0x74,0x82,0x40]
+pmaddubsw 64(%rdx,%rax,4), %xmm6
+
+// CHECK: pmaddubsw 64(%rdx,%rax), %mm4
+// CHECK: encoding: [0x0f,0x38,0x04,0x64,0x02,0x40]
+pmaddubsw 64(%rdx,%rax), %mm4
+
+// CHECK: pmaddubsw 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x04,0x74,0x02,0x40]
+pmaddubsw 64(%rdx,%rax), %xmm6
+
+// CHECK: pmaddubsw 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x04,0x72,0x40]
+pmaddubsw 64(%rdx), %xmm6
+
+// CHECK: pmaddubsw %mm4, %mm4
+// CHECK: encoding: [0x0f,0x38,0x04,0xe4]
+pmaddubsw %mm4, %mm4
+
+// CHECK: pmaddubsw (%rdx), %mm4
+// CHECK: encoding: [0x0f,0x38,0x04,0x22]
+pmaddubsw (%rdx), %mm4
+
+// CHECK: pmaddubsw (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x04,0x32]
+pmaddubsw (%rdx), %xmm6
+
+// CHECK: pmaddubsw %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x04,0xf6]
+pmaddubsw %xmm6, %xmm6
+
+// CHECK: pmulhrsw 485498096, %mm4
+// CHECK: encoding: [0x0f,0x38,0x0b,0x24,0x25,0xf0,0x1c,0xf0,0x1c]
+pmulhrsw 485498096, %mm4
+
+// CHECK: pmulhrsw 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x0b,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+pmulhrsw 485498096, %xmm6
+
+// CHECK: pmulhrsw 64(%rdx), %mm4
+// CHECK: encoding: [0x0f,0x38,0x0b,0x62,0x40]
+pmulhrsw 64(%rdx), %mm4
+
+// CHECK: pmulhrsw -64(%rdx,%rax,4), %mm4
+// CHECK: encoding: [0x0f,0x38,0x0b,0x64,0x82,0xc0]
+pmulhrsw -64(%rdx,%rax,4), %mm4
+
+// CHECK: pmulhrsw 64(%rdx,%rax,4), %mm4
+// CHECK: encoding: [0x0f,0x38,0x0b,0x64,0x82,0x40]
+pmulhrsw 64(%rdx,%rax,4), %mm4
+
+// CHECK: pmulhrsw -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x0b,0x74,0x82,0xc0]
+pmulhrsw -64(%rdx,%rax,4), %xmm6
+
+// CHECK: pmulhrsw 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x0b,0x74,0x82,0x40]
+pmulhrsw 64(%rdx,%rax,4), %xmm6
+
+// CHECK: pmulhrsw 64(%rdx,%rax), %mm4
+// CHECK: encoding: [0x0f,0x38,0x0b,0x64,0x02,0x40]
+pmulhrsw 64(%rdx,%rax), %mm4
+
+// CHECK: pmulhrsw 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x0b,0x74,0x02,0x40]
+pmulhrsw 64(%rdx,%rax), %xmm6
+
+// CHECK: pmulhrsw 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x0b,0x72,0x40]
+pmulhrsw 64(%rdx), %xmm6
+
+// CHECK: pmulhrsw %mm4, %mm4
+// CHECK: encoding: [0x0f,0x38,0x0b,0xe4]
+pmulhrsw %mm4, %mm4
+
+// CHECK: pmulhrsw (%rdx), %mm4
+// CHECK: encoding: [0x0f,0x38,0x0b,0x22]
+pmulhrsw (%rdx), %mm4
+
+// CHECK: pmulhrsw (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x0b,0x32]
+pmulhrsw (%rdx), %xmm6
+
+// CHECK: pmulhrsw %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x0b,0xf6]
+pmulhrsw %xmm6, %xmm6
+
+// CHECK: pshufb 485498096, %mm4
+// CHECK: encoding: [0x0f,0x38,0x00,0x24,0x25,0xf0,0x1c,0xf0,0x1c]
+pshufb 485498096, %mm4
+
+// CHECK: pshufb 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x00,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+pshufb 485498096, %xmm6
+
+// CHECK: pshufb 64(%rdx), %mm4
+// CHECK: encoding: [0x0f,0x38,0x00,0x62,0x40]
+pshufb 64(%rdx), %mm4
+
+// CHECK: pshufb -64(%rdx,%rax,4), %mm4
+// CHECK: encoding: [0x0f,0x38,0x00,0x64,0x82,0xc0]
+pshufb -64(%rdx,%rax,4), %mm4
+
+// CHECK: pshufb 64(%rdx,%rax,4), %mm4
+// CHECK: encoding: [0x0f,0x38,0x00,0x64,0x82,0x40]
+pshufb 64(%rdx,%rax,4), %mm4
+
+// CHECK: pshufb -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x00,0x74,0x82,0xc0]
+pshufb -64(%rdx,%rax,4), %xmm6
+
+// CHECK: pshufb 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x00,0x74,0x82,0x40]
+pshufb 64(%rdx,%rax,4), %xmm6
+
+// CHECK: pshufb 64(%rdx,%rax), %mm4
+// CHECK: encoding: [0x0f,0x38,0x00,0x64,0x02,0x40]
+pshufb 64(%rdx,%rax), %mm4
+
+// CHECK: pshufb 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x00,0x74,0x02,0x40]
+pshufb 64(%rdx,%rax), %xmm6
+
+// CHECK: pshufb 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x00,0x72,0x40]
+pshufb 64(%rdx), %xmm6
+
+// CHECK: pshufb %mm4, %mm4
+// CHECK: encoding: [0x0f,0x38,0x00,0xe4]
+pshufb %mm4, %mm4
+
+// CHECK: pshufb (%rdx), %mm4
+// CHECK: encoding: [0x0f,0x38,0x00,0x22]
+pshufb (%rdx), %mm4
+
+// CHECK: pshufb (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x00,0x32]
+pshufb (%rdx), %xmm6
+
+// CHECK: pshufb %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x00,0xf6]
+pshufb %xmm6, %xmm6
+
+// CHECK: psignb 485498096, %mm4
+// CHECK: encoding: [0x0f,0x38,0x08,0x24,0x25,0xf0,0x1c,0xf0,0x1c]
+psignb 485498096, %mm4
+
+// CHECK: psignb 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x08,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+psignb 485498096, %xmm6
+
+// CHECK: psignb 64(%rdx), %mm4
+// CHECK: encoding: [0x0f,0x38,0x08,0x62,0x40]
+psignb 64(%rdx), %mm4
+
+// CHECK: psignb -64(%rdx,%rax,4), %mm4
+// CHECK: encoding: [0x0f,0x38,0x08,0x64,0x82,0xc0]
+psignb -64(%rdx,%rax,4), %mm4
+
+// CHECK: psignb 64(%rdx,%rax,4), %mm4
+// CHECK: encoding: [0x0f,0x38,0x08,0x64,0x82,0x40]
+psignb 64(%rdx,%rax,4), %mm4
+
+// CHECK: psignb -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x08,0x74,0x82,0xc0]
+psignb -64(%rdx,%rax,4), %xmm6
+
+// CHECK: psignb 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x08,0x74,0x82,0x40]
+psignb 64(%rdx,%rax,4), %xmm6
+
+// CHECK: psignb 64(%rdx,%rax), %mm4
+// CHECK: encoding: [0x0f,0x38,0x08,0x64,0x02,0x40]
+psignb 64(%rdx,%rax), %mm4
+
+// CHECK: psignb 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x08,0x74,0x02,0x40]
+psignb 64(%rdx,%rax), %xmm6
+
+// CHECK: psignb 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x08,0x72,0x40]
+psignb 64(%rdx), %xmm6
+
+// CHECK: psignb %mm4, %mm4
+// CHECK: encoding: [0x0f,0x38,0x08,0xe4]
+psignb %mm4, %mm4
+
+// CHECK: psignb (%rdx), %mm4
+// CHECK: encoding: [0x0f,0x38,0x08,0x22]
+psignb (%rdx), %mm4
+
+// CHECK: psignb (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x08,0x32]
+psignb (%rdx), %xmm6
+
+// CHECK: psignb %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x08,0xf6]
+psignb %xmm6, %xmm6
+
+// CHECK: psignd 485498096, %mm4
+// CHECK: encoding: [0x0f,0x38,0x0a,0x24,0x25,0xf0,0x1c,0xf0,0x1c]
+psignd 485498096, %mm4
+
+// CHECK: psignd 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x0a,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+psignd 485498096, %xmm6
+
+// CHECK: psignd 64(%rdx), %mm4
+// CHECK: encoding: [0x0f,0x38,0x0a,0x62,0x40]
+psignd 64(%rdx), %mm4
+
+// CHECK: psignd -64(%rdx,%rax,4), %mm4
+// CHECK: encoding: [0x0f,0x38,0x0a,0x64,0x82,0xc0]
+psignd -64(%rdx,%rax,4), %mm4
+
+// CHECK: psignd 64(%rdx,%rax,4), %mm4
+// CHECK: encoding: [0x0f,0x38,0x0a,0x64,0x82,0x40]
+psignd 64(%rdx,%rax,4), %mm4
+
+// CHECK: psignd -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x0a,0x74,0x82,0xc0]
+psignd -64(%rdx,%rax,4), %xmm6
+
+// CHECK: psignd 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x0a,0x74,0x82,0x40]
+psignd 64(%rdx,%rax,4), %xmm6
+
+// CHECK: psignd 64(%rdx,%rax), %mm4
+// CHECK: encoding: [0x0f,0x38,0x0a,0x64,0x02,0x40]
+psignd 64(%rdx,%rax), %mm4
+
+// CHECK: psignd 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x0a,0x74,0x02,0x40]
+psignd 64(%rdx,%rax), %xmm6
+
+// CHECK: psignd 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x0a,0x72,0x40]
+psignd 64(%rdx), %xmm6
+
+// CHECK: psignd %mm4, %mm4
+// CHECK: encoding: [0x0f,0x38,0x0a,0xe4]
+psignd %mm4, %mm4
+
+// CHECK: psignd (%rdx), %mm4
+// CHECK: encoding: [0x0f,0x38,0x0a,0x22]
+psignd (%rdx), %mm4
+
+// CHECK: psignd (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x0a,0x32]
+psignd (%rdx), %xmm6
+
+// CHECK: psignd %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x0a,0xf6]
+psignd %xmm6, %xmm6
+
+// CHECK: psignw 485498096, %mm4
+// CHECK: encoding: [0x0f,0x38,0x09,0x24,0x25,0xf0,0x1c,0xf0,0x1c]
+psignw 485498096, %mm4
+
+// CHECK: psignw 485498096, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x09,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+psignw 485498096, %xmm6
+
+// CHECK: psignw 64(%rdx), %mm4
+// CHECK: encoding: [0x0f,0x38,0x09,0x62,0x40]
+psignw 64(%rdx), %mm4
+
+// CHECK: psignw -64(%rdx,%rax,4), %mm4
+// CHECK: encoding: [0x0f,0x38,0x09,0x64,0x82,0xc0]
+psignw -64(%rdx,%rax,4), %mm4
+
+// CHECK: psignw 64(%rdx,%rax,4), %mm4
+// CHECK: encoding: [0x0f,0x38,0x09,0x64,0x82,0x40]
+psignw 64(%rdx,%rax,4), %mm4
+
+// CHECK: psignw -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x09,0x74,0x82,0xc0]
+psignw -64(%rdx,%rax,4), %xmm6
+
+// CHECK: psignw 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x09,0x74,0x82,0x40]
+psignw 64(%rdx,%rax,4), %xmm6
+
+// CHECK: psignw 64(%rdx,%rax), %mm4
+// CHECK: encoding: [0x0f,0x38,0x09,0x64,0x02,0x40]
+psignw 64(%rdx,%rax), %mm4
+
+// CHECK: psignw 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x09,0x74,0x02,0x40]
+psignw 64(%rdx,%rax), %xmm6
+
+// CHECK: psignw 64(%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x09,0x72,0x40]
+psignw 64(%rdx), %xmm6
+
+// CHECK: psignw %mm4, %mm4
+// CHECK: encoding: [0x0f,0x38,0x09,0xe4]
+psignw %mm4, %mm4
+
+// CHECK: psignw (%rdx), %mm4
+// CHECK: encoding: [0x0f,0x38,0x09,0x22]
+psignw (%rdx), %mm4
+
+// CHECK: psignw (%rdx), %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x09,0x32]
+psignw (%rdx), %xmm6
+
+// CHECK: psignw %xmm6, %xmm6
+// CHECK: encoding: [0x66,0x0f,0x38,0x09,0xf6]
+psignw %xmm6, %xmm6
+
diff --git a/test/MC/X86/X87-32.s b/test/MC/X86/X87-32.s
index 967763de93e057e0f1e6b017a32a9825f9383987..d414ab65b7dc4563e1ed4eedf28236a245546ad2 100755
--- a/test/MC/X86/X87-32.s
+++ b/test/MC/X86/X87-32.s
@@ -31,7 +31,7 @@ faddl 64(%edx,%eax)
 // CHECK: encoding: [0xdc,0x02]        
 faddl (%edx) 
 
-// CHECK: faddp %st(4) 
+// CHECK: faddp %st, %st(4) 
 // CHECK: encoding: [0xde,0xc4]        
 faddp %st(4) 
 
@@ -59,11 +59,11 @@ fadds 64(%edx,%eax)
 // CHECK: encoding: [0xd8,0x02]        
 fadds (%edx) 
 
-// CHECK: fadd %st(0), %st(4) 
+// CHECK: fadd %st, %st(4) 
 // CHECK: encoding: [0xdc,0xc4]       
-fadd %st(0), %st(4) 
+fadd %st, %st(4) 
 
-// CHECK: fadd %st(4) 
+// CHECK: fadd %st(4), %st
 // CHECK: encoding: [0xd8,0xc4]        
 fadd %st(4) 
 
@@ -259,7 +259,7 @@ fdivl 64(%edx,%eax)
 // CHECK: encoding: [0xdc,0x32]        
 fdivl (%edx) 
 
-// CHECK: fdivp %st(4) 
+// CHECK: fdivp %st, %st(4) 
 // CHECK: encoding: [0xde,0xf4]        
 fdivp %st(4) 
 
@@ -287,7 +287,7 @@ fdivrl 64(%edx,%eax)
 // CHECK: encoding: [0xdc,0x3a]        
 fdivrl (%edx) 
 
-// CHECK: fdivrp %st(4) 
+// CHECK: fdivrp %st, %st(4) 
 // CHECK: encoding: [0xde,0xfc]        
 fdivrp %st(4) 
 
@@ -315,11 +315,11 @@ fdivrs 64(%edx,%eax)
 // CHECK: encoding: [0xd8,0x3a]        
 fdivrs (%edx) 
 
-// CHECK: fdivr %st(0), %st(4) 
+// CHECK: fdivr %st, %st(4) 
 // CHECK: encoding: [0xdc,0xfc]       
-fdivr %st(0), %st(4) 
+fdivr %st, %st(4) 
 
-// CHECK: fdivr %st(4) 
+// CHECK: fdivr %st(4), %st
 // CHECK: encoding: [0xd8,0xfc]        
 fdivr %st(4) 
 
@@ -347,11 +347,11 @@ fdivs 64(%edx,%eax)
 // CHECK: encoding: [0xd8,0x32]        
 fdivs (%edx) 
 
-// CHECK: fdiv %st(0), %st(4) 
+// CHECK: fdiv %st, %st(4) 
 // CHECK: encoding: [0xdc,0xf4]       
-fdiv %st(0), %st(4) 
+fdiv %st, %st(4) 
 
-// CHECK: fdiv %st(4) 
+// CHECK: fdiv %st(4), %st
 // CHECK: encoding: [0xd8,0xf4]        
 fdiv %st(4) 
 
@@ -1119,7 +1119,7 @@ fmull 64(%edx,%eax)
 // CHECK: encoding: [0xdc,0x0a]        
 fmull (%edx) 
 
-// CHECK: fmulp %st(4) 
+// CHECK: fmulp %st, %st(4) 
 // CHECK: encoding: [0xde,0xcc]        
 fmulp %st(4) 
 
@@ -1147,11 +1147,11 @@ fmuls 64(%edx,%eax)
 // CHECK: encoding: [0xd8,0x0a]        
 fmuls (%edx) 
 
-// CHECK: fmul %st(0), %st(4) 
+// CHECK: fmul %st, %st(4) 
 // CHECK: encoding: [0xdc,0xcc]       
-fmul %st(0), %st(4) 
+fmul %st, %st(4) 
 
-// CHECK: fmul %st(4) 
+// CHECK: fmul %st(4), %st
 // CHECK: encoding: [0xd8,0xcc]        
 fmul %st(4) 
 
@@ -1479,7 +1479,7 @@ fsubl 64(%edx,%eax)
 // CHECK: encoding: [0xdc,0x22]        
 fsubl (%edx) 
 
-// CHECK: fsubp %st(4) 
+// CHECK: fsubp %st, %st(4) 
 // CHECK: encoding: [0xde,0xe4]        
 fsubp %st(4) 
 
@@ -1507,7 +1507,7 @@ fsubrl 64(%edx,%eax)
 // CHECK: encoding: [0xdc,0x2a]        
 fsubrl (%edx) 
 
-// CHECK: fsubrp %st(4) 
+// CHECK: fsubrp %st, %st(4) 
 // CHECK: encoding: [0xde,0xec]        
 fsubrp %st(4) 
 
@@ -1535,11 +1535,11 @@ fsubrs 64(%edx,%eax)
 // CHECK: encoding: [0xd8,0x2a]        
 fsubrs (%edx) 
 
-// CHECK: fsubr %st(0), %st(4) 
+// CHECK: fsubr %st, %st(4) 
 // CHECK: encoding: [0xdc,0xec]       
-fsubr %st(0), %st(4) 
+fsubr %st, %st(4) 
 
-// CHECK: fsubr %st(4) 
+// CHECK: fsubr %st(4), %st
 // CHECK: encoding: [0xd8,0xec]        
 fsubr %st(4) 
 
@@ -1567,11 +1567,11 @@ fsubs 64(%edx,%eax)
 // CHECK: encoding: [0xd8,0x22]        
 fsubs (%edx) 
 
-// CHECK: fsub %st(0), %st(4) 
+// CHECK: fsub %st, %st(4) 
 // CHECK: encoding: [0xdc,0xe4]       
-fsub %st(0), %st(4) 
+fsub %st, %st(4) 
 
-// CHECK: fsub %st(4) 
+// CHECK: fsub %st(4), %st
 // CHECK: encoding: [0xd8,0xe4]        
 fsub %st(4) 
 
diff --git a/test/MC/X86/X87-64.s b/test/MC/X86/X87-64.s
index a3b76b4e4b0ad47e4372c1eaa9b91f79dffb7c8d..1103f326ea047a7bdff3d5e6bbf0c8e10ac192dd 100755
--- a/test/MC/X86/X87-64.s
+++ b/test/MC/X86/X87-64.s
@@ -31,7 +31,7 @@ faddl 64(%rdx,%rax)
 // CHECK: encoding: [0xdc,0x02]        
 faddl (%rdx) 
 
-// CHECK: faddp %st(4) 
+// CHECK: faddp %st, %st(4) 
 // CHECK: encoding: [0xde,0xc4]        
 faddp %st(4) 
 
@@ -59,11 +59,11 @@ fadds 64(%rdx,%rax)
 // CHECK: encoding: [0xd8,0x02]        
 fadds (%rdx) 
 
-// CHECK: fadd %st(0), %st(4) 
+// CHECK: fadd %st, %st(4) 
 // CHECK: encoding: [0xdc,0xc4]       
-fadd %st(0), %st(4) 
+fadd %st, %st(4) 
 
-// CHECK: fadd %st(4) 
+// CHECK: fadd %st(4), %st
 // CHECK: encoding: [0xd8,0xc4]        
 fadd %st(4) 
 
@@ -259,7 +259,7 @@ fdivl 64(%rdx,%rax)
 // CHECK: encoding: [0xdc,0x32]        
 fdivl (%rdx) 
 
-// CHECK: fdivp %st(4) 
+// CHECK: fdivp %st, %st(4) 
 // CHECK: encoding: [0xde,0xf4]        
 fdivp %st(4) 
 
@@ -287,7 +287,7 @@ fdivrl 64(%rdx,%rax)
 // CHECK: encoding: [0xdc,0x3a]        
 fdivrl (%rdx) 
 
-// CHECK: fdivrp %st(4) 
+// CHECK: fdivrp %st, %st(4) 
 // CHECK: encoding: [0xde,0xfc]        
 fdivrp %st(4) 
 
@@ -315,11 +315,11 @@ fdivrs 64(%rdx,%rax)
 // CHECK: encoding: [0xd8,0x3a]        
 fdivrs (%rdx) 
 
-// CHECK: fdivr %st(0), %st(4) 
+// CHECK: fdivr %st, %st(4) 
 // CHECK: encoding: [0xdc,0xfc]       
-fdivr %st(0), %st(4) 
+fdivr %st, %st(4) 
 
-// CHECK: fdivr %st(4) 
+// CHECK: fdivr %st(4), %st
 // CHECK: encoding: [0xd8,0xfc]        
 fdivr %st(4) 
 
@@ -347,11 +347,11 @@ fdivs 64(%rdx,%rax)
 // CHECK: encoding: [0xd8,0x32]        
 fdivs (%rdx) 
 
-// CHECK: fdiv %st(0), %st(4) 
+// CHECK: fdiv %st, %st(4) 
 // CHECK: encoding: [0xdc,0xf4]       
-fdiv %st(0), %st(4) 
+fdiv %st, %st(4) 
 
-// CHECK: fdiv %st(4) 
+// CHECK: fdiv %st(4), %st
 // CHECK: encoding: [0xd8,0xf4]        
 fdiv %st(4) 
 
@@ -1119,7 +1119,7 @@ fmull 64(%rdx,%rax)
 // CHECK: encoding: [0xdc,0x0a]        
 fmull (%rdx) 
 
-// CHECK: fmulp %st(4) 
+// CHECK: fmulp %st, %st(4) 
 // CHECK: encoding: [0xde,0xcc]        
 fmulp %st(4) 
 
@@ -1147,9 +1147,9 @@ fmuls 64(%rdx,%rax)
 // CHECK: encoding: [0xd8,0x0a]        
 fmuls (%rdx) 
 
-// CHECK: fmul %st(0), %st(4) 
+// CHECK: fmul %st, %st(4) 
 // CHECK: encoding: [0xdc,0xcc]       
-fmul %st(0), %st(4) 
+fmul %st, %st(4) 
 
 // CHECK: fmul %st(4) 
 // CHECK: encoding: [0xd8,0xcc]        
@@ -1479,7 +1479,7 @@ fsubl 64(%rdx,%rax)
 // CHECK: encoding: [0xdc,0x22]        
 fsubl (%rdx) 
 
-// CHECK: fsubp %st(4) 
+// CHECK: fsubp %st, %st(4) 
 // CHECK: encoding: [0xde,0xe4]        
 fsubp %st(4) 
 
@@ -1507,7 +1507,7 @@ fsubrl 64(%rdx,%rax)
 // CHECK: encoding: [0xdc,0x2a]        
 fsubrl (%rdx) 
 
-// CHECK: fsubrp %st(4) 
+// CHECK: fsubrp %st, %st(4) 
 // CHECK: encoding: [0xde,0xec]        
 fsubrp %st(4) 
 
@@ -1535,11 +1535,11 @@ fsubrs 64(%rdx,%rax)
 // CHECK: encoding: [0xd8,0x2a]        
 fsubrs (%rdx) 
 
-// CHECK: fsubr %st(0), %st(4) 
+// CHECK: fsubr %st, %st(4) 
 // CHECK: encoding: [0xdc,0xec]       
-fsubr %st(0), %st(4) 
+fsubr %st, %st(4) 
 
-// CHECK: fsubr %st(4) 
+// CHECK: fsubr %st(4), %st
 // CHECK: encoding: [0xd8,0xec]        
 fsubr %st(4) 
 
@@ -1567,11 +1567,11 @@ fsubs 64(%rdx,%rax)
 // CHECK: encoding: [0xd8,0x22]        
 fsubs (%rdx) 
 
-// CHECK: fsub %st(0), %st(4) 
+// CHECK: fsub %st, %st(4) 
 // CHECK: encoding: [0xdc,0xe4]       
-fsub %st(0), %st(4) 
+fsub %st, %st(4) 
 
-// CHECK: fsub %st(4) 
+// CHECK: fsub %st(4), %st
 // CHECK: encoding: [0xd8,0xe4]        
 fsub %st(4) 
 
diff --git a/test/MC/X86/XOP-32.s b/test/MC/X86/XOP-32.s
new file mode 100644
index 0000000000000000000000000000000000000000..669b3f49febb0e94e4d6fda9da3e5f712458bee6
--- /dev/null
+++ b/test/MC/X86/XOP-32.s
@@ -0,0 +1,2250 @@
+// RUN: llvm-mc -triple i386-unknown-unknown --show-encoding %s | FileCheck %s
+
+// CHECK: vfrczpd -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0x81,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vfrczpd -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: vfrczpd 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0x81,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vfrczpd 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: vfrczpd -485498096(%edx,%eax,4), %ymm4
+// CHECK: encoding: [0x8f,0xe9,0x7c,0x81,0xa4,0x82,0x10,0xe3,0x0f,0xe3]
+vfrczpd -485498096(%edx,%eax,4), %ymm4
+
+// CHECK: vfrczpd 485498096(%edx,%eax,4), %ymm4
+// CHECK: encoding: [0x8f,0xe9,0x7c,0x81,0xa4,0x82,0xf0,0x1c,0xf0,0x1c]
+vfrczpd 485498096(%edx,%eax,4), %ymm4
+
+// CHECK: vfrczpd 485498096(%edx), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0x81,0x8a,0xf0,0x1c,0xf0,0x1c]
+vfrczpd 485498096(%edx), %xmm1
+
+// CHECK: vfrczpd 485498096(%edx), %ymm4
+// CHECK: encoding: [0x8f,0xe9,0x7c,0x81,0xa2,0xf0,0x1c,0xf0,0x1c]
+vfrczpd 485498096(%edx), %ymm4
+
+// CHECK: vfrczpd 485498096, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0x81,0x0d,0xf0,0x1c,0xf0,0x1c]
+vfrczpd 485498096, %xmm1
+
+// CHECK: vfrczpd 485498096, %ymm4
+// CHECK: encoding: [0x8f,0xe9,0x7c,0x81,0x25,0xf0,0x1c,0xf0,0x1c]
+vfrczpd 485498096, %ymm4
+
+// CHECK: vfrczpd 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0x81,0x4c,0x02,0x40]
+vfrczpd 64(%edx,%eax), %xmm1
+
+// CHECK: vfrczpd 64(%edx,%eax), %ymm4
+// CHECK: encoding: [0x8f,0xe9,0x7c,0x81,0x64,0x02,0x40]
+vfrczpd 64(%edx,%eax), %ymm4
+
+// CHECK: vfrczpd (%edx), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0x81,0x0a]
+vfrczpd (%edx), %xmm1
+
+// CHECK: vfrczpd (%edx), %ymm4
+// CHECK: encoding: [0x8f,0xe9,0x7c,0x81,0x22]
+vfrczpd (%edx), %ymm4
+
+// CHECK: vfrczpd %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0x81,0xc9]
+vfrczpd %xmm1, %xmm1
+
+// CHECK: vfrczpd %ymm4, %ymm4
+// CHECK: encoding: [0x8f,0xe9,0x7c,0x81,0xe4]
+vfrczpd %ymm4, %ymm4
+
+// CHECK: vfrczps -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0x80,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vfrczps -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: vfrczps 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0x80,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vfrczps 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: vfrczps -485498096(%edx,%eax,4), %ymm4
+// CHECK: encoding: [0x8f,0xe9,0x7c,0x80,0xa4,0x82,0x10,0xe3,0x0f,0xe3]
+vfrczps -485498096(%edx,%eax,4), %ymm4
+
+// CHECK: vfrczps 485498096(%edx,%eax,4), %ymm4
+// CHECK: encoding: [0x8f,0xe9,0x7c,0x80,0xa4,0x82,0xf0,0x1c,0xf0,0x1c]
+vfrczps 485498096(%edx,%eax,4), %ymm4
+
+// CHECK: vfrczps 485498096(%edx), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0x80,0x8a,0xf0,0x1c,0xf0,0x1c]
+vfrczps 485498096(%edx), %xmm1
+
+// CHECK: vfrczps 485498096(%edx), %ymm4
+// CHECK: encoding: [0x8f,0xe9,0x7c,0x80,0xa2,0xf0,0x1c,0xf0,0x1c]
+vfrczps 485498096(%edx), %ymm4
+
+// CHECK: vfrczps 485498096, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0x80,0x0d,0xf0,0x1c,0xf0,0x1c]
+vfrczps 485498096, %xmm1
+
+// CHECK: vfrczps 485498096, %ymm4
+// CHECK: encoding: [0x8f,0xe9,0x7c,0x80,0x25,0xf0,0x1c,0xf0,0x1c]
+vfrczps 485498096, %ymm4
+
+// CHECK: vfrczps 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0x80,0x4c,0x02,0x40]
+vfrczps 64(%edx,%eax), %xmm1
+
+// CHECK: vfrczps 64(%edx,%eax), %ymm4
+// CHECK: encoding: [0x8f,0xe9,0x7c,0x80,0x64,0x02,0x40]
+vfrczps 64(%edx,%eax), %ymm4
+
+// CHECK: vfrczps (%edx), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0x80,0x0a]
+vfrczps (%edx), %xmm1
+
+// CHECK: vfrczps (%edx), %ymm4
+// CHECK: encoding: [0x8f,0xe9,0x7c,0x80,0x22]
+vfrczps (%edx), %ymm4
+
+// CHECK: vfrczps %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0x80,0xc9]
+vfrczps %xmm1, %xmm1
+
+// CHECK: vfrczps %ymm4, %ymm4
+// CHECK: encoding: [0x8f,0xe9,0x7c,0x80,0xe4]
+vfrczps %ymm4, %ymm4
+
+// CHECK: vfrczsd -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0x83,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vfrczsd -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: vfrczsd 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0x83,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vfrczsd 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: vfrczsd 485498096(%edx), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0x83,0x8a,0xf0,0x1c,0xf0,0x1c]
+vfrczsd 485498096(%edx), %xmm1
+
+// CHECK: vfrczsd 485498096, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0x83,0x0d,0xf0,0x1c,0xf0,0x1c]
+vfrczsd 485498096, %xmm1
+
+// CHECK: vfrczsd 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0x83,0x4c,0x02,0x40]
+vfrczsd 64(%edx,%eax), %xmm1
+
+// CHECK: vfrczsd (%edx), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0x83,0x0a]
+vfrczsd (%edx), %xmm1
+
+// CHECK: vfrczsd %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0x83,0xc9]
+vfrczsd %xmm1, %xmm1
+
+// CHECK: vfrczss -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0x82,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vfrczss -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: vfrczss 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0x82,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vfrczss 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: vfrczss 485498096(%edx), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0x82,0x8a,0xf0,0x1c,0xf0,0x1c]
+vfrczss 485498096(%edx), %xmm1
+
+// CHECK: vfrczss 485498096, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0x82,0x0d,0xf0,0x1c,0xf0,0x1c]
+vfrczss 485498096, %xmm1
+
+// CHECK: vfrczss 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0x82,0x4c,0x02,0x40]
+vfrczss 64(%edx,%eax), %xmm1
+
+// CHECK: vfrczss (%edx), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0x82,0x0a]
+vfrczss (%edx), %xmm1
+
+// CHECK: vfrczss %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0x82,0xc9]
+vfrczss %xmm1, %xmm1
+
+// CHECK: vpcmov -485498096(%edx,%eax,4), %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0xf0,0xa2,0x8c,0x82,0x10,0xe3,0x0f,0xe3,0x10]
+vpcmov -485498096(%edx,%eax,4), %xmm1, %xmm1, %xmm1
+
+// CHECK: vpcmov 485498096(%edx,%eax,4), %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0xf0,0xa2,0x8c,0x82,0xf0,0x1c,0xf0,0x1c,0x10]
+vpcmov 485498096(%edx,%eax,4), %xmm1, %xmm1, %xmm1
+
+// CHECK: vpcmov -485498096(%edx,%eax,4), %ymm4, %ymm4, %ymm4
+// CHECK: encoding: [0x8f,0xe8,0xdc,0xa2,0xa4,0x82,0x10,0xe3,0x0f,0xe3,0x40]
+vpcmov -485498096(%edx,%eax,4), %ymm4, %ymm4, %ymm4
+
+// CHECK: vpcmov 485498096(%edx,%eax,4), %ymm4, %ymm4, %ymm4
+// CHECK: encoding: [0x8f,0xe8,0xdc,0xa2,0xa4,0x82,0xf0,0x1c,0xf0,0x1c,0x40]
+vpcmov 485498096(%edx,%eax,4), %ymm4, %ymm4, %ymm4
+
+// CHECK: vpcmov 485498096(%edx), %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0xf0,0xa2,0x8a,0xf0,0x1c,0xf0,0x1c,0x10]
+vpcmov 485498096(%edx), %xmm1, %xmm1, %xmm1
+
+// CHECK: vpcmov 485498096(%edx), %ymm4, %ymm4, %ymm4
+// CHECK: encoding: [0x8f,0xe8,0xdc,0xa2,0xa2,0xf0,0x1c,0xf0,0x1c,0x40]
+vpcmov 485498096(%edx), %ymm4, %ymm4, %ymm4
+
+// CHECK: vpcmov 485498096, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0xf0,0xa2,0x0d,0xf0,0x1c,0xf0,0x1c,0x10]
+vpcmov 485498096, %xmm1, %xmm1, %xmm1
+
+// CHECK: vpcmov 485498096, %ymm4, %ymm4, %ymm4
+// CHECK: encoding: [0x8f,0xe8,0xdc,0xa2,0x25,0xf0,0x1c,0xf0,0x1c,0x40]
+vpcmov 485498096, %ymm4, %ymm4, %ymm4
+
+// CHECK: vpcmov 64(%edx,%eax), %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0xf0,0xa2,0x4c,0x02,0x40,0x10]
+vpcmov 64(%edx,%eax), %xmm1, %xmm1, %xmm1
+
+// CHECK: vpcmov 64(%edx,%eax), %ymm4, %ymm4, %ymm4
+// CHECK: encoding: [0x8f,0xe8,0xdc,0xa2,0x64,0x02,0x40,0x40]
+vpcmov 64(%edx,%eax), %ymm4, %ymm4, %ymm4
+
+// CHECK: vpcmov (%edx), %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0xf0,0xa2,0x0a,0x10]
+vpcmov (%edx), %xmm1, %xmm1, %xmm1
+
+// CHECK: vpcmov (%edx), %ymm4, %ymm4, %ymm4
+// CHECK: encoding: [0x8f,0xe8,0xdc,0xa2,0x22,0x40]
+vpcmov (%edx), %ymm4, %ymm4, %ymm4
+
+// CHECK: vpcmov %xmm1, -485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0xa2,0x8c,0x82,0x10,0xe3,0x0f,0xe3,0x10]
+vpcmov %xmm1, -485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vpcmov %xmm1, 485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0xa2,0x8c,0x82,0xf0,0x1c,0xf0,0x1c,0x10]
+vpcmov %xmm1, 485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vpcmov %xmm1, 485498096(%edx), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0xa2,0x8a,0xf0,0x1c,0xf0,0x1c,0x10]
+vpcmov %xmm1, 485498096(%edx), %xmm1, %xmm1
+
+// CHECK: vpcmov %xmm1, 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0xa2,0x0d,0xf0,0x1c,0xf0,0x1c,0x10]
+vpcmov %xmm1, 485498096, %xmm1, %xmm1
+
+// CHECK: vpcmov %xmm1, 64(%edx,%eax), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0xa2,0x4c,0x02,0x40,0x10]
+vpcmov %xmm1, 64(%edx,%eax), %xmm1, %xmm1
+
+// CHECK: vpcmov %xmm1, (%edx), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0xa2,0x0a,0x10]
+vpcmov %xmm1, (%edx), %xmm1, %xmm1
+
+// CHECK: vpcmov %xmm1, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0xa2,0xc9,0x10]
+vpcmov %xmm1, %xmm1, %xmm1, %xmm1
+
+// CHECK: vpcmov %ymm4, -485498096(%edx,%eax,4), %ymm4, %ymm4
+// CHECK: encoding: [0x8f,0xe8,0x5c,0xa2,0xa4,0x82,0x10,0xe3,0x0f,0xe3,0x40]
+vpcmov %ymm4, -485498096(%edx,%eax,4), %ymm4, %ymm4
+
+// CHECK: vpcmov %ymm4, 485498096(%edx,%eax,4), %ymm4, %ymm4
+// CHECK: encoding: [0x8f,0xe8,0x5c,0xa2,0xa4,0x82,0xf0,0x1c,0xf0,0x1c,0x40]
+vpcmov %ymm4, 485498096(%edx,%eax,4), %ymm4, %ymm4
+
+// CHECK: vpcmov %ymm4, 485498096(%edx), %ymm4, %ymm4
+// CHECK: encoding: [0x8f,0xe8,0x5c,0xa2,0xa2,0xf0,0x1c,0xf0,0x1c,0x40]
+vpcmov %ymm4, 485498096(%edx), %ymm4, %ymm4
+
+// CHECK: vpcmov %ymm4, 485498096, %ymm4, %ymm4
+// CHECK: encoding: [0x8f,0xe8,0x5c,0xa2,0x25,0xf0,0x1c,0xf0,0x1c,0x40]
+vpcmov %ymm4, 485498096, %ymm4, %ymm4
+
+// CHECK: vpcmov %ymm4, 64(%edx,%eax), %ymm4, %ymm4
+// CHECK: encoding: [0x8f,0xe8,0x5c,0xa2,0x64,0x02,0x40,0x40]
+vpcmov %ymm4, 64(%edx,%eax), %ymm4, %ymm4
+
+// CHECK: vpcmov %ymm4, (%edx), %ymm4, %ymm4
+// CHECK: encoding: [0x8f,0xe8,0x5c,0xa2,0x22,0x40]
+vpcmov %ymm4, (%edx), %ymm4, %ymm4
+
+// CHECK: vpcmov %ymm4, %ymm4, %ymm4, %ymm4
+// CHECK: encoding: [0x8f,0xe8,0x5c,0xa2,0xe4,0x40]
+vpcmov %ymm4, %ymm4, %ymm4, %ymm4
+
+// CHECK: vpcomltb -485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0xcc,0x8c,0x82,0x10,0xe3,0x0f,0xe3,0x00]
+vpcomltb -485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vpcomltb 485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0xcc,0x8c,0x82,0xf0,0x1c,0xf0,0x1c,0x00]
+vpcomltb 485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vpcomltb 485498096(%edx), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0xcc,0x8a,0xf0,0x1c,0xf0,0x1c,0x00]
+vpcomltb 485498096(%edx), %xmm1, %xmm1
+
+// CHECK: vpcomltb 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0xcc,0x0d,0xf0,0x1c,0xf0,0x1c,0x00]
+vpcomltb 485498096, %xmm1, %xmm1
+
+// CHECK: vpcomltb 64(%edx,%eax), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0xcc,0x4c,0x02,0x40,0x00]
+vpcomltb 64(%edx,%eax), %xmm1, %xmm1
+
+// CHECK: vpcomltb (%edx), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0xcc,0x0a,0x00]
+vpcomltb (%edx), %xmm1, %xmm1
+
+// CHECK: vpcomltb %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0xcc,0xc9,0x00]
+vpcomltb %xmm1, %xmm1, %xmm1
+
+// CHECK: vpcomltd -485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0xce,0x8c,0x82,0x10,0xe3,0x0f,0xe3,0x00]
+vpcomltd -485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vpcomltd 485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0xce,0x8c,0x82,0xf0,0x1c,0xf0,0x1c,0x00]
+vpcomltd 485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vpcomltd 485498096(%edx), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0xce,0x8a,0xf0,0x1c,0xf0,0x1c,0x00]
+vpcomltd 485498096(%edx), %xmm1, %xmm1
+
+// CHECK: vpcomltd 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0xce,0x0d,0xf0,0x1c,0xf0,0x1c,0x00]
+vpcomltd 485498096, %xmm1, %xmm1
+
+// CHECK: vpcomltd 64(%edx,%eax), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0xce,0x4c,0x02,0x40,0x00]
+vpcomltd 64(%edx,%eax), %xmm1, %xmm1
+
+// CHECK: vpcomltd (%edx), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0xce,0x0a,0x00]
+vpcomltd (%edx), %xmm1, %xmm1
+
+// CHECK: vpcomltd %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0xce,0xc9,0x00]
+vpcomltd %xmm1, %xmm1, %xmm1
+
+// CHECK: vpcomltq -485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0xcf,0x8c,0x82,0x10,0xe3,0x0f,0xe3,0x00]
+vpcomltq -485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vpcomltq 485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0xcf,0x8c,0x82,0xf0,0x1c,0xf0,0x1c,0x00]
+vpcomltq 485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vpcomltq 485498096(%edx), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0xcf,0x8a,0xf0,0x1c,0xf0,0x1c,0x00]
+vpcomltq 485498096(%edx), %xmm1, %xmm1
+
+// CHECK: vpcomltq 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0xcf,0x0d,0xf0,0x1c,0xf0,0x1c,0x00]
+vpcomltq 485498096, %xmm1, %xmm1
+
+// CHECK: vpcomltq 64(%edx,%eax), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0xcf,0x4c,0x02,0x40,0x00]
+vpcomltq 64(%edx,%eax), %xmm1, %xmm1
+
+// CHECK: vpcomltq (%edx), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0xcf,0x0a,0x00]
+vpcomltq (%edx), %xmm1, %xmm1
+
+// CHECK: vpcomltq %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0xcf,0xc9,0x00]
+vpcomltq %xmm1, %xmm1, %xmm1
+
+// CHECK: vpcomltub -485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0xec,0x8c,0x82,0x10,0xe3,0x0f,0xe3,0x00]
+vpcomltub -485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vpcomltub 485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0xec,0x8c,0x82,0xf0,0x1c,0xf0,0x1c,0x00]
+vpcomltub 485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vpcomltub 485498096(%edx), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0xec,0x8a,0xf0,0x1c,0xf0,0x1c,0x00]
+vpcomltub 485498096(%edx), %xmm1, %xmm1
+
+// CHECK: vpcomltub 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0xec,0x0d,0xf0,0x1c,0xf0,0x1c,0x00]
+vpcomltub 485498096, %xmm1, %xmm1
+
+// CHECK: vpcomltub 64(%edx,%eax), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0xec,0x4c,0x02,0x40,0x00]
+vpcomltub 64(%edx,%eax), %xmm1, %xmm1
+
+// CHECK: vpcomltub (%edx), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0xec,0x0a,0x00]
+vpcomltub (%edx), %xmm1, %xmm1
+
+// CHECK: vpcomltub %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0xec,0xc9,0x00]
+vpcomltub %xmm1, %xmm1, %xmm1
+
+// CHECK: vpcomltud -485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0xee,0x8c,0x82,0x10,0xe3,0x0f,0xe3,0x00]
+vpcomltud -485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vpcomltud 485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0xee,0x8c,0x82,0xf0,0x1c,0xf0,0x1c,0x00]
+vpcomltud 485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vpcomltud 485498096(%edx), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0xee,0x8a,0xf0,0x1c,0xf0,0x1c,0x00]
+vpcomltud 485498096(%edx), %xmm1, %xmm1
+
+// CHECK: vpcomltud 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0xee,0x0d,0xf0,0x1c,0xf0,0x1c,0x00]
+vpcomltud 485498096, %xmm1, %xmm1
+
+// CHECK: vpcomltud 64(%edx,%eax), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0xee,0x4c,0x02,0x40,0x00]
+vpcomltud 64(%edx,%eax), %xmm1, %xmm1
+
+// CHECK: vpcomltud (%edx), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0xee,0x0a,0x00]
+vpcomltud (%edx), %xmm1, %xmm1
+
+// CHECK: vpcomltud %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0xee,0xc9,0x00]
+vpcomltud %xmm1, %xmm1, %xmm1
+
+// CHECK: vpcomltuq -485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0xef,0x8c,0x82,0x10,0xe3,0x0f,0xe3,0x00]
+vpcomltuq -485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vpcomltuq 485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0xef,0x8c,0x82,0xf0,0x1c,0xf0,0x1c,0x00]
+vpcomltuq 485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vpcomltuq 485498096(%edx), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0xef,0x8a,0xf0,0x1c,0xf0,0x1c,0x00]
+vpcomltuq 485498096(%edx), %xmm1, %xmm1
+
+// CHECK: vpcomltuq 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0xef,0x0d,0xf0,0x1c,0xf0,0x1c,0x00]
+vpcomltuq 485498096, %xmm1, %xmm1
+
+// CHECK: vpcomltuq 64(%edx,%eax), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0xef,0x4c,0x02,0x40,0x00]
+vpcomltuq 64(%edx,%eax), %xmm1, %xmm1
+
+// CHECK: vpcomltuq (%edx), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0xef,0x0a,0x00]
+vpcomltuq (%edx), %xmm1, %xmm1
+
+// CHECK: vpcomltuq %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0xef,0xc9,0x00]
+vpcomltuq %xmm1, %xmm1, %xmm1
+
+// CHECK: vpcomltuw -485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0xed,0x8c,0x82,0x10,0xe3,0x0f,0xe3,0x00]
+vpcomltuw -485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vpcomltuw 485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0xed,0x8c,0x82,0xf0,0x1c,0xf0,0x1c,0x00]
+vpcomltuw 485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vpcomltuw 485498096(%edx), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0xed,0x8a,0xf0,0x1c,0xf0,0x1c,0x00]
+vpcomltuw 485498096(%edx), %xmm1, %xmm1
+
+// CHECK: vpcomltuw 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0xed,0x0d,0xf0,0x1c,0xf0,0x1c,0x00]
+vpcomltuw 485498096, %xmm1, %xmm1
+
+// CHECK: vpcomltuw 64(%edx,%eax), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0xed,0x4c,0x02,0x40,0x00]
+vpcomltuw 64(%edx,%eax), %xmm1, %xmm1
+
+// CHECK: vpcomltuw (%edx), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0xed,0x0a,0x00]
+vpcomltuw (%edx), %xmm1, %xmm1
+
+// CHECK: vpcomltuw %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0xed,0xc9,0x00]
+vpcomltuw %xmm1, %xmm1, %xmm1
+
+// CHECK: vpcomltw -485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0xcd,0x8c,0x82,0x10,0xe3,0x0f,0xe3,0x00]
+vpcomltw -485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vpcomltw 485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0xcd,0x8c,0x82,0xf0,0x1c,0xf0,0x1c,0x00]
+vpcomltw 485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vpcomltw 485498096(%edx), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0xcd,0x8a,0xf0,0x1c,0xf0,0x1c,0x00]
+vpcomltw 485498096(%edx), %xmm1, %xmm1
+
+// CHECK: vpcomltw 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0xcd,0x0d,0xf0,0x1c,0xf0,0x1c,0x00]
+vpcomltw 485498096, %xmm1, %xmm1
+
+// CHECK: vpcomltw 64(%edx,%eax), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0xcd,0x4c,0x02,0x40,0x00]
+vpcomltw 64(%edx,%eax), %xmm1, %xmm1
+
+// CHECK: vpcomltw (%edx), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0xcd,0x0a,0x00]
+vpcomltw (%edx), %xmm1, %xmm1
+
+// CHECK: vpcomltw %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0xcd,0xc9,0x00]
+vpcomltw %xmm1, %xmm1, %xmm1
+
+// CHECK: vpermil2pd $0, -485498096(%edx,%eax,4), %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe3,0xf1,0x49,0x8c,0x82,0x10,0xe3,0x0f,0xe3,0x10]
+vpermil2pd $0, -485498096(%edx,%eax,4), %xmm1, %xmm1, %xmm1
+
+// CHECK: vpermil2pd $0, 485498096(%edx,%eax,4), %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe3,0xf1,0x49,0x8c,0x82,0xf0,0x1c,0xf0,0x1c,0x10]
+vpermil2pd $0, 485498096(%edx,%eax,4), %xmm1, %xmm1, %xmm1
+
+// CHECK: vpermil2pd $0, -485498096(%edx,%eax,4), %ymm4, %ymm4, %ymm4
+// CHECK: encoding: [0xc4,0xe3,0xdd,0x49,0xa4,0x82,0x10,0xe3,0x0f,0xe3,0x40]
+vpermil2pd $0, -485498096(%edx,%eax,4), %ymm4, %ymm4, %ymm4
+
+// CHECK: vpermil2pd $0, 485498096(%edx,%eax,4), %ymm4, %ymm4, %ymm4
+// CHECK: encoding: [0xc4,0xe3,0xdd,0x49,0xa4,0x82,0xf0,0x1c,0xf0,0x1c,0x40]
+vpermil2pd $0, 485498096(%edx,%eax,4), %ymm4, %ymm4, %ymm4
+
+// CHECK: vpermil2pd $0, 485498096(%edx), %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe3,0xf1,0x49,0x8a,0xf0,0x1c,0xf0,0x1c,0x10]
+vpermil2pd $0, 485498096(%edx), %xmm1, %xmm1, %xmm1
+
+// CHECK: vpermil2pd $0, 485498096(%edx), %ymm4, %ymm4, %ymm4
+// CHECK: encoding: [0xc4,0xe3,0xdd,0x49,0xa2,0xf0,0x1c,0xf0,0x1c,0x40]
+vpermil2pd $0, 485498096(%edx), %ymm4, %ymm4, %ymm4
+
+// CHECK: vpermil2pd $0, 485498096, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe3,0xf1,0x49,0x0d,0xf0,0x1c,0xf0,0x1c,0x10]
+vpermil2pd $0, 485498096, %xmm1, %xmm1, %xmm1
+
+// CHECK: vpermil2pd $0, 485498096, %ymm4, %ymm4, %ymm4
+// CHECK: encoding: [0xc4,0xe3,0xdd,0x49,0x25,0xf0,0x1c,0xf0,0x1c,0x40]
+vpermil2pd $0, 485498096, %ymm4, %ymm4, %ymm4
+
+// CHECK: vpermil2pd $0, 64(%edx,%eax), %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe3,0xf1,0x49,0x4c,0x02,0x40,0x10]
+vpermil2pd $0, 64(%edx,%eax), %xmm1, %xmm1, %xmm1
+
+// CHECK: vpermil2pd $0, 64(%edx,%eax), %ymm4, %ymm4, %ymm4
+// CHECK: encoding: [0xc4,0xe3,0xdd,0x49,0x64,0x02,0x40,0x40]
+vpermil2pd $0, 64(%edx,%eax), %ymm4, %ymm4, %ymm4
+
+// CHECK: vpermil2pd $0, (%edx), %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe3,0xf1,0x49,0x0a,0x10]
+vpermil2pd $0, (%edx), %xmm1, %xmm1, %xmm1
+
+// CHECK: vpermil2pd $0, (%edx), %ymm4, %ymm4, %ymm4
+// CHECK: encoding: [0xc4,0xe3,0xdd,0x49,0x22,0x40]
+vpermil2pd $0, (%edx), %ymm4, %ymm4, %ymm4
+
+// CHECK: vpermil2pd $0, %xmm1, -485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe3,0x71,0x49,0x8c,0x82,0x10,0xe3,0x0f,0xe3,0x10]
+vpermil2pd $0, %xmm1, -485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vpermil2pd $0, %xmm1, 485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe3,0x71,0x49,0x8c,0x82,0xf0,0x1c,0xf0,0x1c,0x10]
+vpermil2pd $0, %xmm1, 485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vpermil2pd $0, %xmm1, 485498096(%edx), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe3,0x71,0x49,0x8a,0xf0,0x1c,0xf0,0x1c,0x10]
+vpermil2pd $0, %xmm1, 485498096(%edx), %xmm1, %xmm1
+
+// CHECK: vpermil2pd $0, %xmm1, 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe3,0x71,0x49,0x0d,0xf0,0x1c,0xf0,0x1c,0x10]
+vpermil2pd $0, %xmm1, 485498096, %xmm1, %xmm1
+
+// CHECK: vpermil2pd $0, %xmm1, 64(%edx,%eax), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe3,0x71,0x49,0x4c,0x02,0x40,0x10]
+vpermil2pd $0, %xmm1, 64(%edx,%eax), %xmm1, %xmm1
+
+// CHECK: vpermil2pd $0, %xmm1, (%edx), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe3,0x71,0x49,0x0a,0x10]
+vpermil2pd $0, %xmm1, (%edx), %xmm1, %xmm1
+
+// CHECK: vpermil2pd $0, %xmm1, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe3,0x71,0x49,0xc9,0x10]
+vpermil2pd $0, %xmm1, %xmm1, %xmm1, %xmm1
+
+// CHECK: vpermil2pd $0, %ymm4, -485498096(%edx,%eax,4), %ymm4, %ymm4
+// CHECK: encoding: [0xc4,0xe3,0x5d,0x49,0xa4,0x82,0x10,0xe3,0x0f,0xe3,0x40]
+vpermil2pd $0, %ymm4, -485498096(%edx,%eax,4), %ymm4, %ymm4
+
+// CHECK: vpermil2pd $0, %ymm4, 485498096(%edx,%eax,4), %ymm4, %ymm4
+// CHECK: encoding: [0xc4,0xe3,0x5d,0x49,0xa4,0x82,0xf0,0x1c,0xf0,0x1c,0x40]
+vpermil2pd $0, %ymm4, 485498096(%edx,%eax,4), %ymm4, %ymm4
+
+// CHECK: vpermil2pd $0, %ymm4, 485498096(%edx), %ymm4, %ymm4
+// CHECK: encoding: [0xc4,0xe3,0x5d,0x49,0xa2,0xf0,0x1c,0xf0,0x1c,0x40]
+vpermil2pd $0, %ymm4, 485498096(%edx), %ymm4, %ymm4
+
+// CHECK: vpermil2pd $0, %ymm4, 485498096, %ymm4, %ymm4
+// CHECK: encoding: [0xc4,0xe3,0x5d,0x49,0x25,0xf0,0x1c,0xf0,0x1c,0x40]
+vpermil2pd $0, %ymm4, 485498096, %ymm4, %ymm4
+
+// CHECK: vpermil2pd $0, %ymm4, 64(%edx,%eax), %ymm4, %ymm4
+// CHECK: encoding: [0xc4,0xe3,0x5d,0x49,0x64,0x02,0x40,0x40]
+vpermil2pd $0, %ymm4, 64(%edx,%eax), %ymm4, %ymm4
+
+// CHECK: vpermil2pd $0, %ymm4, (%edx), %ymm4, %ymm4
+// CHECK: encoding: [0xc4,0xe3,0x5d,0x49,0x22,0x40]
+vpermil2pd $0, %ymm4, (%edx), %ymm4, %ymm4
+
+// CHECK: vpermil2pd $0, %ymm4, %ymm4, %ymm4, %ymm4
+// CHECK: encoding: [0xc4,0xe3,0x5d,0x49,0xe4,0x40]
+vpermil2pd $0, %ymm4, %ymm4, %ymm4, %ymm4
+
+// CHECK: vpermil2ps $0, -485498096(%edx,%eax,4), %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe3,0xf1,0x48,0x8c,0x82,0x10,0xe3,0x0f,0xe3,0x10]
+vpermil2ps $0, -485498096(%edx,%eax,4), %xmm1, %xmm1, %xmm1
+
+// CHECK: vpermil2ps $0, 485498096(%edx,%eax,4), %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe3,0xf1,0x48,0x8c,0x82,0xf0,0x1c,0xf0,0x1c,0x10]
+vpermil2ps $0, 485498096(%edx,%eax,4), %xmm1, %xmm1, %xmm1
+
+// CHECK: vpermil2ps $0, -485498096(%edx,%eax,4), %ymm4, %ymm4, %ymm4
+// CHECK: encoding: [0xc4,0xe3,0xdd,0x48,0xa4,0x82,0x10,0xe3,0x0f,0xe3,0x40]
+vpermil2ps $0, -485498096(%edx,%eax,4), %ymm4, %ymm4, %ymm4
+
+// CHECK: vpermil2ps $0, 485498096(%edx,%eax,4), %ymm4, %ymm4, %ymm4
+// CHECK: encoding: [0xc4,0xe3,0xdd,0x48,0xa4,0x82,0xf0,0x1c,0xf0,0x1c,0x40]
+vpermil2ps $0, 485498096(%edx,%eax,4), %ymm4, %ymm4, %ymm4
+
+// CHECK: vpermil2ps $0, 485498096(%edx), %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe3,0xf1,0x48,0x8a,0xf0,0x1c,0xf0,0x1c,0x10]
+vpermil2ps $0, 485498096(%edx), %xmm1, %xmm1, %xmm1
+
+// CHECK: vpermil2ps $0, 485498096(%edx), %ymm4, %ymm4, %ymm4
+// CHECK: encoding: [0xc4,0xe3,0xdd,0x48,0xa2,0xf0,0x1c,0xf0,0x1c,0x40]
+vpermil2ps $0, 485498096(%edx), %ymm4, %ymm4, %ymm4
+
+// CHECK: vpermil2ps $0, 485498096, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe3,0xf1,0x48,0x0d,0xf0,0x1c,0xf0,0x1c,0x10]
+vpermil2ps $0, 485498096, %xmm1, %xmm1, %xmm1
+
+// CHECK: vpermil2ps $0, 485498096, %ymm4, %ymm4, %ymm4
+// CHECK: encoding: [0xc4,0xe3,0xdd,0x48,0x25,0xf0,0x1c,0xf0,0x1c,0x40]
+vpermil2ps $0, 485498096, %ymm4, %ymm4, %ymm4
+
+// CHECK: vpermil2ps $0, 64(%edx,%eax), %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe3,0xf1,0x48,0x4c,0x02,0x40,0x10]
+vpermil2ps $0, 64(%edx,%eax), %xmm1, %xmm1, %xmm1
+
+// CHECK: vpermil2ps $0, 64(%edx,%eax), %ymm4, %ymm4, %ymm4
+// CHECK: encoding: [0xc4,0xe3,0xdd,0x48,0x64,0x02,0x40,0x40]
+vpermil2ps $0, 64(%edx,%eax), %ymm4, %ymm4, %ymm4
+
+// CHECK: vpermil2ps $0, (%edx), %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe3,0xf1,0x48,0x0a,0x10]
+vpermil2ps $0, (%edx), %xmm1, %xmm1, %xmm1
+
+// CHECK: vpermil2ps $0, (%edx), %ymm4, %ymm4, %ymm4
+// CHECK: encoding: [0xc4,0xe3,0xdd,0x48,0x22,0x40]
+vpermil2ps $0, (%edx), %ymm4, %ymm4, %ymm4
+
+// CHECK: vpermil2ps $0, %xmm1, -485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe3,0x71,0x48,0x8c,0x82,0x10,0xe3,0x0f,0xe3,0x10]
+vpermil2ps $0, %xmm1, -485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vpermil2ps $0, %xmm1, 485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe3,0x71,0x48,0x8c,0x82,0xf0,0x1c,0xf0,0x1c,0x10]
+vpermil2ps $0, %xmm1, 485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vpermil2ps $0, %xmm1, 485498096(%edx), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe3,0x71,0x48,0x8a,0xf0,0x1c,0xf0,0x1c,0x10]
+vpermil2ps $0, %xmm1, 485498096(%edx), %xmm1, %xmm1
+
+// CHECK: vpermil2ps $0, %xmm1, 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe3,0x71,0x48,0x0d,0xf0,0x1c,0xf0,0x1c,0x10]
+vpermil2ps $0, %xmm1, 485498096, %xmm1, %xmm1
+
+// CHECK: vpermil2ps $0, %xmm1, 64(%edx,%eax), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe3,0x71,0x48,0x4c,0x02,0x40,0x10]
+vpermil2ps $0, %xmm1, 64(%edx,%eax), %xmm1, %xmm1
+
+// CHECK: vpermil2ps $0, %xmm1, (%edx), %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe3,0x71,0x48,0x0a,0x10]
+vpermil2ps $0, %xmm1, (%edx), %xmm1, %xmm1
+
+// CHECK: vpermil2ps $0, %xmm1, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0xc4,0xe3,0x71,0x48,0xc9,0x10]
+vpermil2ps $0, %xmm1, %xmm1, %xmm1, %xmm1
+
+// CHECK: vpermil2ps $0, %ymm4, -485498096(%edx,%eax,4), %ymm4, %ymm4
+// CHECK: encoding: [0xc4,0xe3,0x5d,0x48,0xa4,0x82,0x10,0xe3,0x0f,0xe3,0x40]
+vpermil2ps $0, %ymm4, -485498096(%edx,%eax,4), %ymm4, %ymm4
+
+// CHECK: vpermil2ps $0, %ymm4, 485498096(%edx,%eax,4), %ymm4, %ymm4
+// CHECK: encoding: [0xc4,0xe3,0x5d,0x48,0xa4,0x82,0xf0,0x1c,0xf0,0x1c,0x40]
+vpermil2ps $0, %ymm4, 485498096(%edx,%eax,4), %ymm4, %ymm4
+
+// CHECK: vpermil2ps $0, %ymm4, 485498096(%edx), %ymm4, %ymm4
+// CHECK: encoding: [0xc4,0xe3,0x5d,0x48,0xa2,0xf0,0x1c,0xf0,0x1c,0x40]
+vpermil2ps $0, %ymm4, 485498096(%edx), %ymm4, %ymm4
+
+// CHECK: vpermil2ps $0, %ymm4, 485498096, %ymm4, %ymm4
+// CHECK: encoding: [0xc4,0xe3,0x5d,0x48,0x25,0xf0,0x1c,0xf0,0x1c,0x40]
+vpermil2ps $0, %ymm4, 485498096, %ymm4, %ymm4
+
+// CHECK: vpermil2ps $0, %ymm4, 64(%edx,%eax), %ymm4, %ymm4
+// CHECK: encoding: [0xc4,0xe3,0x5d,0x48,0x64,0x02,0x40,0x40]
+vpermil2ps $0, %ymm4, 64(%edx,%eax), %ymm4, %ymm4
+
+// CHECK: vpermil2ps $0, %ymm4, (%edx), %ymm4, %ymm4
+// CHECK: encoding: [0xc4,0xe3,0x5d,0x48,0x22,0x40]
+vpermil2ps $0, %ymm4, (%edx), %ymm4, %ymm4
+
+// CHECK: vpermil2ps $0, %ymm4, %ymm4, %ymm4, %ymm4
+// CHECK: encoding: [0xc4,0xe3,0x5d,0x48,0xe4,0x40]
+vpermil2ps $0, %ymm4, %ymm4, %ymm4, %ymm4
+
+// CHECK: vphaddbd -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0xc2,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vphaddbd -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: vphaddbd 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0xc2,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vphaddbd 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: vphaddbd 485498096(%edx), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0xc2,0x8a,0xf0,0x1c,0xf0,0x1c]
+vphaddbd 485498096(%edx), %xmm1
+
+// CHECK: vphaddbd 485498096, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0xc2,0x0d,0xf0,0x1c,0xf0,0x1c]
+vphaddbd 485498096, %xmm1
+
+// CHECK: vphaddbd 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0xc2,0x4c,0x02,0x40]
+vphaddbd 64(%edx,%eax), %xmm1
+
+// CHECK: vphaddbd (%edx), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0xc2,0x0a]
+vphaddbd (%edx), %xmm1
+
+// CHECK: vphaddbd %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0xc2,0xc9]
+vphaddbd %xmm1, %xmm1
+
+// CHECK: vphaddbq -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0xc3,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vphaddbq -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: vphaddbq 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0xc3,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vphaddbq 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: vphaddbq 485498096(%edx), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0xc3,0x8a,0xf0,0x1c,0xf0,0x1c]
+vphaddbq 485498096(%edx), %xmm1
+
+// CHECK: vphaddbq 485498096, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0xc3,0x0d,0xf0,0x1c,0xf0,0x1c]
+vphaddbq 485498096, %xmm1
+
+// CHECK: vphaddbq 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0xc3,0x4c,0x02,0x40]
+vphaddbq 64(%edx,%eax), %xmm1
+
+// CHECK: vphaddbq (%edx), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0xc3,0x0a]
+vphaddbq (%edx), %xmm1
+
+// CHECK: vphaddbq %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0xc3,0xc9]
+vphaddbq %xmm1, %xmm1
+
+// CHECK: vphaddbw -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0xc1,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vphaddbw -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: vphaddbw 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0xc1,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vphaddbw 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: vphaddbw 485498096(%edx), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0xc1,0x8a,0xf0,0x1c,0xf0,0x1c]
+vphaddbw 485498096(%edx), %xmm1
+
+// CHECK: vphaddbw 485498096, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0xc1,0x0d,0xf0,0x1c,0xf0,0x1c]
+vphaddbw 485498096, %xmm1
+
+// CHECK: vphaddbw 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0xc1,0x4c,0x02,0x40]
+vphaddbw 64(%edx,%eax), %xmm1
+
+// CHECK: vphaddbw (%edx), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0xc1,0x0a]
+vphaddbw (%edx), %xmm1
+
+// CHECK: vphaddbw %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0xc1,0xc9]
+vphaddbw %xmm1, %xmm1
+
+// CHECK: vphadddq -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0xcb,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vphadddq -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: vphadddq 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0xcb,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vphadddq 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: vphadddq 485498096(%edx), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0xcb,0x8a,0xf0,0x1c,0xf0,0x1c]
+vphadddq 485498096(%edx), %xmm1
+
+// CHECK: vphadddq 485498096, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0xcb,0x0d,0xf0,0x1c,0xf0,0x1c]
+vphadddq 485498096, %xmm1
+
+// CHECK: vphadddq 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0xcb,0x4c,0x02,0x40]
+vphadddq 64(%edx,%eax), %xmm1
+
+// CHECK: vphadddq (%edx), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0xcb,0x0a]
+vphadddq (%edx), %xmm1
+
+// CHECK: vphadddq %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0xcb,0xc9]
+vphadddq %xmm1, %xmm1
+
+// CHECK: vphaddubd -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0xd2,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vphaddubd -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: vphaddubd 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0xd2,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vphaddubd 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: vphaddubd 485498096(%edx), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0xd2,0x8a,0xf0,0x1c,0xf0,0x1c]
+vphaddubd 485498096(%edx), %xmm1
+
+// CHECK: vphaddubd 485498096, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0xd2,0x0d,0xf0,0x1c,0xf0,0x1c]
+vphaddubd 485498096, %xmm1
+
+// CHECK: vphaddubd 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0xd2,0x4c,0x02,0x40]
+vphaddubd 64(%edx,%eax), %xmm1
+
+// CHECK: vphaddubd (%edx), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0xd2,0x0a]
+vphaddubd (%edx), %xmm1
+
+// CHECK: vphaddubd %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0xd2,0xc9]
+vphaddubd %xmm1, %xmm1
+
+// CHECK: vphaddubq -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0xd3,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vphaddubq -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: vphaddubq 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0xd3,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vphaddubq 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: vphaddubq 485498096(%edx), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0xd3,0x8a,0xf0,0x1c,0xf0,0x1c]
+vphaddubq 485498096(%edx), %xmm1
+
+// CHECK: vphaddubq 485498096, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0xd3,0x0d,0xf0,0x1c,0xf0,0x1c]
+vphaddubq 485498096, %xmm1
+
+// CHECK: vphaddubq 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0xd3,0x4c,0x02,0x40]
+vphaddubq 64(%edx,%eax), %xmm1
+
+// CHECK: vphaddubq (%edx), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0xd3,0x0a]
+vphaddubq (%edx), %xmm1
+
+// CHECK: vphaddubq %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0xd3,0xc9]
+vphaddubq %xmm1, %xmm1
+
+// CHECK: vphaddubw -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0xd1,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vphaddubw -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: vphaddubw 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0xd1,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vphaddubw 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: vphaddubw 485498096(%edx), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0xd1,0x8a,0xf0,0x1c,0xf0,0x1c]
+vphaddubw 485498096(%edx), %xmm1
+
+// CHECK: vphaddubw 485498096, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0xd1,0x0d,0xf0,0x1c,0xf0,0x1c]
+vphaddubw 485498096, %xmm1
+
+// CHECK: vphaddubw 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0xd1,0x4c,0x02,0x40]
+vphaddubw 64(%edx,%eax), %xmm1
+
+// CHECK: vphaddubw (%edx), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0xd1,0x0a]
+vphaddubw (%edx), %xmm1
+
+// CHECK: vphaddubw %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0xd1,0xc9]
+vphaddubw %xmm1, %xmm1
+
+// CHECK: vphaddudq -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0xdb,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vphaddudq -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: vphaddudq 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0xdb,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vphaddudq 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: vphaddudq 485498096(%edx), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0xdb,0x8a,0xf0,0x1c,0xf0,0x1c]
+vphaddudq 485498096(%edx), %xmm1
+
+// CHECK: vphaddudq 485498096, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0xdb,0x0d,0xf0,0x1c,0xf0,0x1c]
+vphaddudq 485498096, %xmm1
+
+// CHECK: vphaddudq 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0xdb,0x4c,0x02,0x40]
+vphaddudq 64(%edx,%eax), %xmm1
+
+// CHECK: vphaddudq (%edx), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0xdb,0x0a]
+vphaddudq (%edx), %xmm1
+
+// CHECK: vphaddudq %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0xdb,0xc9]
+vphaddudq %xmm1, %xmm1
+
+// CHECK: vphadduwd -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0xd6,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vphadduwd -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: vphadduwd 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0xd6,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vphadduwd 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: vphadduwd 485498096(%edx), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0xd6,0x8a,0xf0,0x1c,0xf0,0x1c]
+vphadduwd 485498096(%edx), %xmm1
+
+// CHECK: vphadduwd 485498096, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0xd6,0x0d,0xf0,0x1c,0xf0,0x1c]
+vphadduwd 485498096, %xmm1
+
+// CHECK: vphadduwd 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0xd6,0x4c,0x02,0x40]
+vphadduwd 64(%edx,%eax), %xmm1
+
+// CHECK: vphadduwd (%edx), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0xd6,0x0a]
+vphadduwd (%edx), %xmm1
+
+// CHECK: vphadduwd %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0xd6,0xc9]
+vphadduwd %xmm1, %xmm1
+
+// CHECK: vphadduwq -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0xd7,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vphadduwq -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: vphadduwq 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0xd7,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vphadduwq 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: vphadduwq 485498096(%edx), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0xd7,0x8a,0xf0,0x1c,0xf0,0x1c]
+vphadduwq 485498096(%edx), %xmm1
+
+// CHECK: vphadduwq 485498096, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0xd7,0x0d,0xf0,0x1c,0xf0,0x1c]
+vphadduwq 485498096, %xmm1
+
+// CHECK: vphadduwq 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0xd7,0x4c,0x02,0x40]
+vphadduwq 64(%edx,%eax), %xmm1
+
+// CHECK: vphadduwq (%edx), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0xd7,0x0a]
+vphadduwq (%edx), %xmm1
+
+// CHECK: vphadduwq %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0xd7,0xc9]
+vphadduwq %xmm1, %xmm1
+
+// CHECK: vphaddwd -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0xc6,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vphaddwd -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: vphaddwd 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0xc6,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vphaddwd 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: vphaddwd 485498096(%edx), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0xc6,0x8a,0xf0,0x1c,0xf0,0x1c]
+vphaddwd 485498096(%edx), %xmm1
+
+// CHECK: vphaddwd 485498096, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0xc6,0x0d,0xf0,0x1c,0xf0,0x1c]
+vphaddwd 485498096, %xmm1
+
+// CHECK: vphaddwd 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0xc6,0x4c,0x02,0x40]
+vphaddwd 64(%edx,%eax), %xmm1
+
+// CHECK: vphaddwd (%edx), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0xc6,0x0a]
+vphaddwd (%edx), %xmm1
+
+// CHECK: vphaddwd %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0xc6,0xc9]
+vphaddwd %xmm1, %xmm1
+
+// CHECK: vphaddwq -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0xc7,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vphaddwq -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: vphaddwq 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0xc7,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vphaddwq 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: vphaddwq 485498096(%edx), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0xc7,0x8a,0xf0,0x1c,0xf0,0x1c]
+vphaddwq 485498096(%edx), %xmm1
+
+// CHECK: vphaddwq 485498096, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0xc7,0x0d,0xf0,0x1c,0xf0,0x1c]
+vphaddwq 485498096, %xmm1
+
+// CHECK: vphaddwq 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0xc7,0x4c,0x02,0x40]
+vphaddwq 64(%edx,%eax), %xmm1
+
+// CHECK: vphaddwq (%edx), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0xc7,0x0a]
+vphaddwq (%edx), %xmm1
+
+// CHECK: vphaddwq %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0xc7,0xc9]
+vphaddwq %xmm1, %xmm1
+
+// CHECK: vphsubbw -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0xe1,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vphsubbw -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: vphsubbw 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0xe1,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vphsubbw 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: vphsubbw 485498096(%edx), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0xe1,0x8a,0xf0,0x1c,0xf0,0x1c]
+vphsubbw 485498096(%edx), %xmm1
+
+// CHECK: vphsubbw 485498096, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0xe1,0x0d,0xf0,0x1c,0xf0,0x1c]
+vphsubbw 485498096, %xmm1
+
+// CHECK: vphsubbw 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0xe1,0x4c,0x02,0x40]
+vphsubbw 64(%edx,%eax), %xmm1
+
+// CHECK: vphsubbw (%edx), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0xe1,0x0a]
+vphsubbw (%edx), %xmm1
+
+// CHECK: vphsubbw %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0xe1,0xc9]
+vphsubbw %xmm1, %xmm1
+
+// CHECK: vphsubdq -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0xe3,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vphsubdq -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: vphsubdq 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0xe3,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vphsubdq 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: vphsubdq 485498096(%edx), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0xe3,0x8a,0xf0,0x1c,0xf0,0x1c]
+vphsubdq 485498096(%edx), %xmm1
+
+// CHECK: vphsubdq 485498096, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0xe3,0x0d,0xf0,0x1c,0xf0,0x1c]
+vphsubdq 485498096, %xmm1
+
+// CHECK: vphsubdq 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0xe3,0x4c,0x02,0x40]
+vphsubdq 64(%edx,%eax), %xmm1
+
+// CHECK: vphsubdq (%edx), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0xe3,0x0a]
+vphsubdq (%edx), %xmm1
+
+// CHECK: vphsubdq %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0xe3,0xc9]
+vphsubdq %xmm1, %xmm1
+
+// CHECK: vphsubwd -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0xe2,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vphsubwd -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: vphsubwd 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0xe2,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vphsubwd 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: vphsubwd 485498096(%edx), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0xe2,0x8a,0xf0,0x1c,0xf0,0x1c]
+vphsubwd 485498096(%edx), %xmm1
+
+// CHECK: vphsubwd 485498096, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0xe2,0x0d,0xf0,0x1c,0xf0,0x1c]
+vphsubwd 485498096, %xmm1
+
+// CHECK: vphsubwd 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0xe2,0x4c,0x02,0x40]
+vphsubwd 64(%edx,%eax), %xmm1
+
+// CHECK: vphsubwd (%edx), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0xe2,0x0a]
+vphsubwd (%edx), %xmm1
+
+// CHECK: vphsubwd %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x78,0xe2,0xc9]
+vphsubwd %xmm1, %xmm1
+
+// CHECK: vpmacsdd %xmm1, -485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0x9e,0x8c,0x82,0x10,0xe3,0x0f,0xe3,0x10]
+vpmacsdd %xmm1, -485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vpmacsdd %xmm1, 485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0x9e,0x8c,0x82,0xf0,0x1c,0xf0,0x1c,0x10]
+vpmacsdd %xmm1, 485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vpmacsdd %xmm1, 485498096(%edx), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0x9e,0x8a,0xf0,0x1c,0xf0,0x1c,0x10]
+vpmacsdd %xmm1, 485498096(%edx), %xmm1, %xmm1
+
+// CHECK: vpmacsdd %xmm1, 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0x9e,0x0d,0xf0,0x1c,0xf0,0x1c,0x10]
+vpmacsdd %xmm1, 485498096, %xmm1, %xmm1
+
+// CHECK: vpmacsdd %xmm1, 64(%edx,%eax), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0x9e,0x4c,0x02,0x40,0x10]
+vpmacsdd %xmm1, 64(%edx,%eax), %xmm1, %xmm1
+
+// CHECK: vpmacsdd %xmm1, (%edx), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0x9e,0x0a,0x10]
+vpmacsdd %xmm1, (%edx), %xmm1, %xmm1
+
+// CHECK: vpmacsdd %xmm1, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0x9e,0xc9,0x10]
+vpmacsdd %xmm1, %xmm1, %xmm1, %xmm1
+
+// CHECK: vpmacsdqh %xmm1, -485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0x9f,0x8c,0x82,0x10,0xe3,0x0f,0xe3,0x10]
+vpmacsdqh %xmm1, -485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vpmacsdqh %xmm1, 485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0x9f,0x8c,0x82,0xf0,0x1c,0xf0,0x1c,0x10]
+vpmacsdqh %xmm1, 485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vpmacsdqh %xmm1, 485498096(%edx), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0x9f,0x8a,0xf0,0x1c,0xf0,0x1c,0x10]
+vpmacsdqh %xmm1, 485498096(%edx), %xmm1, %xmm1
+
+// CHECK: vpmacsdqh %xmm1, 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0x9f,0x0d,0xf0,0x1c,0xf0,0x1c,0x10]
+vpmacsdqh %xmm1, 485498096, %xmm1, %xmm1
+
+// CHECK: vpmacsdqh %xmm1, 64(%edx,%eax), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0x9f,0x4c,0x02,0x40,0x10]
+vpmacsdqh %xmm1, 64(%edx,%eax), %xmm1, %xmm1
+
+// CHECK: vpmacsdqh %xmm1, (%edx), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0x9f,0x0a,0x10]
+vpmacsdqh %xmm1, (%edx), %xmm1, %xmm1
+
+// CHECK: vpmacsdqh %xmm1, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0x9f,0xc9,0x10]
+vpmacsdqh %xmm1, %xmm1, %xmm1, %xmm1
+
+// CHECK: vpmacsdql %xmm1, -485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0x97,0x8c,0x82,0x10,0xe3,0x0f,0xe3,0x10]
+vpmacsdql %xmm1, -485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vpmacsdql %xmm1, 485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0x97,0x8c,0x82,0xf0,0x1c,0xf0,0x1c,0x10]
+vpmacsdql %xmm1, 485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vpmacsdql %xmm1, 485498096(%edx), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0x97,0x8a,0xf0,0x1c,0xf0,0x1c,0x10]
+vpmacsdql %xmm1, 485498096(%edx), %xmm1, %xmm1
+
+// CHECK: vpmacsdql %xmm1, 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0x97,0x0d,0xf0,0x1c,0xf0,0x1c,0x10]
+vpmacsdql %xmm1, 485498096, %xmm1, %xmm1
+
+// CHECK: vpmacsdql %xmm1, 64(%edx,%eax), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0x97,0x4c,0x02,0x40,0x10]
+vpmacsdql %xmm1, 64(%edx,%eax), %xmm1, %xmm1
+
+// CHECK: vpmacsdql %xmm1, (%edx), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0x97,0x0a,0x10]
+vpmacsdql %xmm1, (%edx), %xmm1, %xmm1
+
+// CHECK: vpmacsdql %xmm1, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0x97,0xc9,0x10]
+vpmacsdql %xmm1, %xmm1, %xmm1, %xmm1
+
+// CHECK: vpmacssdd %xmm1, -485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0x8e,0x8c,0x82,0x10,0xe3,0x0f,0xe3,0x10]
+vpmacssdd %xmm1, -485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vpmacssdd %xmm1, 485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0x8e,0x8c,0x82,0xf0,0x1c,0xf0,0x1c,0x10]
+vpmacssdd %xmm1, 485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vpmacssdd %xmm1, 485498096(%edx), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0x8e,0x8a,0xf0,0x1c,0xf0,0x1c,0x10]
+vpmacssdd %xmm1, 485498096(%edx), %xmm1, %xmm1
+
+// CHECK: vpmacssdd %xmm1, 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0x8e,0x0d,0xf0,0x1c,0xf0,0x1c,0x10]
+vpmacssdd %xmm1, 485498096, %xmm1, %xmm1
+
+// CHECK: vpmacssdd %xmm1, 64(%edx,%eax), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0x8e,0x4c,0x02,0x40,0x10]
+vpmacssdd %xmm1, 64(%edx,%eax), %xmm1, %xmm1
+
+// CHECK: vpmacssdd %xmm1, (%edx), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0x8e,0x0a,0x10]
+vpmacssdd %xmm1, (%edx), %xmm1, %xmm1
+
+// CHECK: vpmacssdd %xmm1, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0x8e,0xc9,0x10]
+vpmacssdd %xmm1, %xmm1, %xmm1, %xmm1
+
+// CHECK: vpmacssdqh %xmm1, -485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0x8f,0x8c,0x82,0x10,0xe3,0x0f,0xe3,0x10]
+vpmacssdqh %xmm1, -485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vpmacssdqh %xmm1, 485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0x8f,0x8c,0x82,0xf0,0x1c,0xf0,0x1c,0x10]
+vpmacssdqh %xmm1, 485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vpmacssdqh %xmm1, 485498096(%edx), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0x8f,0x8a,0xf0,0x1c,0xf0,0x1c,0x10]
+vpmacssdqh %xmm1, 485498096(%edx), %xmm1, %xmm1
+
+// CHECK: vpmacssdqh %xmm1, 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0x8f,0x0d,0xf0,0x1c,0xf0,0x1c,0x10]
+vpmacssdqh %xmm1, 485498096, %xmm1, %xmm1
+
+// CHECK: vpmacssdqh %xmm1, 64(%edx,%eax), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0x8f,0x4c,0x02,0x40,0x10]
+vpmacssdqh %xmm1, 64(%edx,%eax), %xmm1, %xmm1
+
+// CHECK: vpmacssdqh %xmm1, (%edx), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0x8f,0x0a,0x10]
+vpmacssdqh %xmm1, (%edx), %xmm1, %xmm1
+
+// CHECK: vpmacssdqh %xmm1, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0x8f,0xc9,0x10]
+vpmacssdqh %xmm1, %xmm1, %xmm1, %xmm1
+
+// CHECK: vpmacssdql %xmm1, -485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0x87,0x8c,0x82,0x10,0xe3,0x0f,0xe3,0x10]
+vpmacssdql %xmm1, -485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vpmacssdql %xmm1, 485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0x87,0x8c,0x82,0xf0,0x1c,0xf0,0x1c,0x10]
+vpmacssdql %xmm1, 485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vpmacssdql %xmm1, 485498096(%edx), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0x87,0x8a,0xf0,0x1c,0xf0,0x1c,0x10]
+vpmacssdql %xmm1, 485498096(%edx), %xmm1, %xmm1
+
+// CHECK: vpmacssdql %xmm1, 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0x87,0x0d,0xf0,0x1c,0xf0,0x1c,0x10]
+vpmacssdql %xmm1, 485498096, %xmm1, %xmm1
+
+// CHECK: vpmacssdql %xmm1, 64(%edx,%eax), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0x87,0x4c,0x02,0x40,0x10]
+vpmacssdql %xmm1, 64(%edx,%eax), %xmm1, %xmm1
+
+// CHECK: vpmacssdql %xmm1, (%edx), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0x87,0x0a,0x10]
+vpmacssdql %xmm1, (%edx), %xmm1, %xmm1
+
+// CHECK: vpmacssdql %xmm1, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0x87,0xc9,0x10]
+vpmacssdql %xmm1, %xmm1, %xmm1, %xmm1
+
+// CHECK: vpmacsswd %xmm1, -485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0x86,0x8c,0x82,0x10,0xe3,0x0f,0xe3,0x10]
+vpmacsswd %xmm1, -485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vpmacsswd %xmm1, 485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0x86,0x8c,0x82,0xf0,0x1c,0xf0,0x1c,0x10]
+vpmacsswd %xmm1, 485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vpmacsswd %xmm1, 485498096(%edx), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0x86,0x8a,0xf0,0x1c,0xf0,0x1c,0x10]
+vpmacsswd %xmm1, 485498096(%edx), %xmm1, %xmm1
+
+// CHECK: vpmacsswd %xmm1, 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0x86,0x0d,0xf0,0x1c,0xf0,0x1c,0x10]
+vpmacsswd %xmm1, 485498096, %xmm1, %xmm1
+
+// CHECK: vpmacsswd %xmm1, 64(%edx,%eax), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0x86,0x4c,0x02,0x40,0x10]
+vpmacsswd %xmm1, 64(%edx,%eax), %xmm1, %xmm1
+
+// CHECK: vpmacsswd %xmm1, (%edx), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0x86,0x0a,0x10]
+vpmacsswd %xmm1, (%edx), %xmm1, %xmm1
+
+// CHECK: vpmacsswd %xmm1, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0x86,0xc9,0x10]
+vpmacsswd %xmm1, %xmm1, %xmm1, %xmm1
+
+// CHECK: vpmacssww %xmm1, -485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0x85,0x8c,0x82,0x10,0xe3,0x0f,0xe3,0x10]
+vpmacssww %xmm1, -485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vpmacssww %xmm1, 485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0x85,0x8c,0x82,0xf0,0x1c,0xf0,0x1c,0x10]
+vpmacssww %xmm1, 485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vpmacssww %xmm1, 485498096(%edx), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0x85,0x8a,0xf0,0x1c,0xf0,0x1c,0x10]
+vpmacssww %xmm1, 485498096(%edx), %xmm1, %xmm1
+
+// CHECK: vpmacssww %xmm1, 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0x85,0x0d,0xf0,0x1c,0xf0,0x1c,0x10]
+vpmacssww %xmm1, 485498096, %xmm1, %xmm1
+
+// CHECK: vpmacssww %xmm1, 64(%edx,%eax), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0x85,0x4c,0x02,0x40,0x10]
+vpmacssww %xmm1, 64(%edx,%eax), %xmm1, %xmm1
+
+// CHECK: vpmacssww %xmm1, (%edx), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0x85,0x0a,0x10]
+vpmacssww %xmm1, (%edx), %xmm1, %xmm1
+
+// CHECK: vpmacssww %xmm1, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0x85,0xc9,0x10]
+vpmacssww %xmm1, %xmm1, %xmm1, %xmm1
+
+// CHECK: vpmacswd %xmm1, -485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0x96,0x8c,0x82,0x10,0xe3,0x0f,0xe3,0x10]
+vpmacswd %xmm1, -485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vpmacswd %xmm1, 485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0x96,0x8c,0x82,0xf0,0x1c,0xf0,0x1c,0x10]
+vpmacswd %xmm1, 485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vpmacswd %xmm1, 485498096(%edx), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0x96,0x8a,0xf0,0x1c,0xf0,0x1c,0x10]
+vpmacswd %xmm1, 485498096(%edx), %xmm1, %xmm1
+
+// CHECK: vpmacswd %xmm1, 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0x96,0x0d,0xf0,0x1c,0xf0,0x1c,0x10]
+vpmacswd %xmm1, 485498096, %xmm1, %xmm1
+
+// CHECK: vpmacswd %xmm1, 64(%edx,%eax), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0x96,0x4c,0x02,0x40,0x10]
+vpmacswd %xmm1, 64(%edx,%eax), %xmm1, %xmm1
+
+// CHECK: vpmacswd %xmm1, (%edx), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0x96,0x0a,0x10]
+vpmacswd %xmm1, (%edx), %xmm1, %xmm1
+
+// CHECK: vpmacswd %xmm1, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0x96,0xc9,0x10]
+vpmacswd %xmm1, %xmm1, %xmm1, %xmm1
+
+// CHECK: vpmacsww %xmm1, -485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0x95,0x8c,0x82,0x10,0xe3,0x0f,0xe3,0x10]
+vpmacsww %xmm1, -485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vpmacsww %xmm1, 485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0x95,0x8c,0x82,0xf0,0x1c,0xf0,0x1c,0x10]
+vpmacsww %xmm1, 485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vpmacsww %xmm1, 485498096(%edx), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0x95,0x8a,0xf0,0x1c,0xf0,0x1c,0x10]
+vpmacsww %xmm1, 485498096(%edx), %xmm1, %xmm1
+
+// CHECK: vpmacsww %xmm1, 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0x95,0x0d,0xf0,0x1c,0xf0,0x1c,0x10]
+vpmacsww %xmm1, 485498096, %xmm1, %xmm1
+
+// CHECK: vpmacsww %xmm1, 64(%edx,%eax), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0x95,0x4c,0x02,0x40,0x10]
+vpmacsww %xmm1, 64(%edx,%eax), %xmm1, %xmm1
+
+// CHECK: vpmacsww %xmm1, (%edx), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0x95,0x0a,0x10]
+vpmacsww %xmm1, (%edx), %xmm1, %xmm1
+
+// CHECK: vpmacsww %xmm1, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0x95,0xc9,0x10]
+vpmacsww %xmm1, %xmm1, %xmm1, %xmm1
+
+// CHECK: vpmadcsswd %xmm1, -485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0xa6,0x8c,0x82,0x10,0xe3,0x0f,0xe3,0x10]
+vpmadcsswd %xmm1, -485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vpmadcsswd %xmm1, 485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0xa6,0x8c,0x82,0xf0,0x1c,0xf0,0x1c,0x10]
+vpmadcsswd %xmm1, 485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vpmadcsswd %xmm1, 485498096(%edx), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0xa6,0x8a,0xf0,0x1c,0xf0,0x1c,0x10]
+vpmadcsswd %xmm1, 485498096(%edx), %xmm1, %xmm1
+
+// CHECK: vpmadcsswd %xmm1, 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0xa6,0x0d,0xf0,0x1c,0xf0,0x1c,0x10]
+vpmadcsswd %xmm1, 485498096, %xmm1, %xmm1
+
+// CHECK: vpmadcsswd %xmm1, 64(%edx,%eax), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0xa6,0x4c,0x02,0x40,0x10]
+vpmadcsswd %xmm1, 64(%edx,%eax), %xmm1, %xmm1
+
+// CHECK: vpmadcsswd %xmm1, (%edx), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0xa6,0x0a,0x10]
+vpmadcsswd %xmm1, (%edx), %xmm1, %xmm1
+
+// CHECK: vpmadcsswd %xmm1, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0xa6,0xc9,0x10]
+vpmadcsswd %xmm1, %xmm1, %xmm1, %xmm1
+
+// CHECK: vpmadcswd %xmm1, -485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0xb6,0x8c,0x82,0x10,0xe3,0x0f,0xe3,0x10]
+vpmadcswd %xmm1, -485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vpmadcswd %xmm1, 485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0xb6,0x8c,0x82,0xf0,0x1c,0xf0,0x1c,0x10]
+vpmadcswd %xmm1, 485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vpmadcswd %xmm1, 485498096(%edx), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0xb6,0x8a,0xf0,0x1c,0xf0,0x1c,0x10]
+vpmadcswd %xmm1, 485498096(%edx), %xmm1, %xmm1
+
+// CHECK: vpmadcswd %xmm1, 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0xb6,0x0d,0xf0,0x1c,0xf0,0x1c,0x10]
+vpmadcswd %xmm1, 485498096, %xmm1, %xmm1
+
+// CHECK: vpmadcswd %xmm1, 64(%edx,%eax), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0xb6,0x4c,0x02,0x40,0x10]
+vpmadcswd %xmm1, 64(%edx,%eax), %xmm1, %xmm1
+
+// CHECK: vpmadcswd %xmm1, (%edx), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0xb6,0x0a,0x10]
+vpmadcswd %xmm1, (%edx), %xmm1, %xmm1
+
+// CHECK: vpmadcswd %xmm1, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0xb6,0xc9,0x10]
+vpmadcswd %xmm1, %xmm1, %xmm1, %xmm1
+
+// CHECK: vpperm -485498096(%edx,%eax,4), %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0xf0,0xa3,0x8c,0x82,0x10,0xe3,0x0f,0xe3,0x10]
+vpperm -485498096(%edx,%eax,4), %xmm1, %xmm1, %xmm1
+
+// CHECK: vpperm 485498096(%edx,%eax,4), %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0xf0,0xa3,0x8c,0x82,0xf0,0x1c,0xf0,0x1c,0x10]
+vpperm 485498096(%edx,%eax,4), %xmm1, %xmm1, %xmm1
+
+// CHECK: vpperm 485498096(%edx), %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0xf0,0xa3,0x8a,0xf0,0x1c,0xf0,0x1c,0x10]
+vpperm 485498096(%edx), %xmm1, %xmm1, %xmm1
+
+// CHECK: vpperm 485498096, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0xf0,0xa3,0x0d,0xf0,0x1c,0xf0,0x1c,0x10]
+vpperm 485498096, %xmm1, %xmm1, %xmm1
+
+// CHECK: vpperm 64(%edx,%eax), %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0xf0,0xa3,0x4c,0x02,0x40,0x10]
+vpperm 64(%edx,%eax), %xmm1, %xmm1, %xmm1
+
+// CHECK: vpperm (%edx), %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0xf0,0xa3,0x0a,0x10]
+vpperm (%edx), %xmm1, %xmm1, %xmm1
+
+// CHECK: vpperm %xmm1, -485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0xa3,0x8c,0x82,0x10,0xe3,0x0f,0xe3,0x10]
+vpperm %xmm1, -485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vpperm %xmm1, 485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0xa3,0x8c,0x82,0xf0,0x1c,0xf0,0x1c,0x10]
+vpperm %xmm1, 485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vpperm %xmm1, 485498096(%edx), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0xa3,0x8a,0xf0,0x1c,0xf0,0x1c,0x10]
+vpperm %xmm1, 485498096(%edx), %xmm1, %xmm1
+
+// CHECK: vpperm %xmm1, 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0xa3,0x0d,0xf0,0x1c,0xf0,0x1c,0x10]
+vpperm %xmm1, 485498096, %xmm1, %xmm1
+
+// CHECK: vpperm %xmm1, 64(%edx,%eax), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0xa3,0x4c,0x02,0x40,0x10]
+vpperm %xmm1, 64(%edx,%eax), %xmm1, %xmm1
+
+// CHECK: vpperm %xmm1, (%edx), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0xa3,0x0a,0x10]
+vpperm %xmm1, (%edx), %xmm1, %xmm1
+
+// CHECK: vpperm %xmm1, %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x70,0xa3,0xc9,0x10]
+vpperm %xmm1, %xmm1, %xmm1, %xmm1
+
+// CHECK: vprotb $0, -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x78,0xc0,0x8c,0x82,0x10,0xe3,0x0f,0xe3,0x00]
+vprotb $0, -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: vprotb $0, 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x78,0xc0,0x8c,0x82,0xf0,0x1c,0xf0,0x1c,0x00]
+vprotb $0, 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: vprotb $0, 485498096(%edx), %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x78,0xc0,0x8a,0xf0,0x1c,0xf0,0x1c,0x00]
+vprotb $0, 485498096(%edx), %xmm1
+
+// CHECK: vprotb $0, 485498096, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x78,0xc0,0x0d,0xf0,0x1c,0xf0,0x1c,0x00]
+vprotb $0, 485498096, %xmm1
+
+// CHECK: vprotb $0, 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x78,0xc0,0x4c,0x02,0x40,0x00]
+vprotb $0, 64(%edx,%eax), %xmm1
+
+// CHECK: vprotb $0, (%edx), %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x78,0xc0,0x0a,0x00]
+vprotb $0, (%edx), %xmm1
+
+// CHECK: vprotb $0, %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x78,0xc0,0xc9,0x00]
+vprotb $0, %xmm1, %xmm1
+
+// CHECK: vprotb -485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0xf0,0x90,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vprotb -485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vprotb 485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0xf0,0x90,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vprotb 485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vprotb 485498096(%edx), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0xf0,0x90,0x8a,0xf0,0x1c,0xf0,0x1c]
+vprotb 485498096(%edx), %xmm1, %xmm1
+
+// CHECK: vprotb 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0xf0,0x90,0x0d,0xf0,0x1c,0xf0,0x1c]
+vprotb 485498096, %xmm1, %xmm1
+
+// CHECK: vprotb 64(%edx,%eax), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0xf0,0x90,0x4c,0x02,0x40]
+vprotb 64(%edx,%eax), %xmm1, %xmm1
+
+// CHECK: vprotb (%edx), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0xf0,0x90,0x0a]
+vprotb (%edx), %xmm1, %xmm1
+
+// CHECK: vprotb %xmm1, -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x70,0x90,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vprotb %xmm1, -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: vprotb %xmm1, 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x70,0x90,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vprotb %xmm1, 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: vprotb %xmm1, 485498096(%edx), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x70,0x90,0x8a,0xf0,0x1c,0xf0,0x1c]
+vprotb %xmm1, 485498096(%edx), %xmm1
+
+// CHECK: vprotb %xmm1, 485498096, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x70,0x90,0x0d,0xf0,0x1c,0xf0,0x1c]
+vprotb %xmm1, 485498096, %xmm1
+
+// CHECK: vprotb %xmm1, 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x70,0x90,0x4c,0x02,0x40]
+vprotb %xmm1, 64(%edx,%eax), %xmm1
+
+// CHECK: vprotb %xmm1, (%edx), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x70,0x90,0x0a]
+vprotb %xmm1, (%edx), %xmm1
+
+// CHECK: vprotb %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x70,0x90,0xc9]
+vprotb %xmm1, %xmm1, %xmm1
+
+// CHECK: vprotd $0, -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x78,0xc2,0x8c,0x82,0x10,0xe3,0x0f,0xe3,0x00]
+vprotd $0, -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: vprotd $0, 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x78,0xc2,0x8c,0x82,0xf0,0x1c,0xf0,0x1c,0x00]
+vprotd $0, 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: vprotd $0, 485498096(%edx), %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x78,0xc2,0x8a,0xf0,0x1c,0xf0,0x1c,0x00]
+vprotd $0, 485498096(%edx), %xmm1
+
+// CHECK: vprotd $0, 485498096, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x78,0xc2,0x0d,0xf0,0x1c,0xf0,0x1c,0x00]
+vprotd $0, 485498096, %xmm1
+
+// CHECK: vprotd $0, 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x78,0xc2,0x4c,0x02,0x40,0x00]
+vprotd $0, 64(%edx,%eax), %xmm1
+
+// CHECK: vprotd $0, (%edx), %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x78,0xc2,0x0a,0x00]
+vprotd $0, (%edx), %xmm1
+
+// CHECK: vprotd $0, %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x78,0xc2,0xc9,0x00]
+vprotd $0, %xmm1, %xmm1
+
+// CHECK: vprotd -485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0xf0,0x92,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vprotd -485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vprotd 485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0xf0,0x92,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vprotd 485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vprotd 485498096(%edx), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0xf0,0x92,0x8a,0xf0,0x1c,0xf0,0x1c]
+vprotd 485498096(%edx), %xmm1, %xmm1
+
+// CHECK: vprotd 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0xf0,0x92,0x0d,0xf0,0x1c,0xf0,0x1c]
+vprotd 485498096, %xmm1, %xmm1
+
+// CHECK: vprotd 64(%edx,%eax), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0xf0,0x92,0x4c,0x02,0x40]
+vprotd 64(%edx,%eax), %xmm1, %xmm1
+
+// CHECK: vprotd (%edx), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0xf0,0x92,0x0a]
+vprotd (%edx), %xmm1, %xmm1
+
+// CHECK: vprotd %xmm1, -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x70,0x92,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vprotd %xmm1, -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: vprotd %xmm1, 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x70,0x92,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vprotd %xmm1, 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: vprotd %xmm1, 485498096(%edx), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x70,0x92,0x8a,0xf0,0x1c,0xf0,0x1c]
+vprotd %xmm1, 485498096(%edx), %xmm1
+
+// CHECK: vprotd %xmm1, 485498096, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x70,0x92,0x0d,0xf0,0x1c,0xf0,0x1c]
+vprotd %xmm1, 485498096, %xmm1
+
+// CHECK: vprotd %xmm1, 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x70,0x92,0x4c,0x02,0x40]
+vprotd %xmm1, 64(%edx,%eax), %xmm1
+
+// CHECK: vprotd %xmm1, (%edx), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x70,0x92,0x0a]
+vprotd %xmm1, (%edx), %xmm1
+
+// CHECK: vprotd %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x70,0x92,0xc9]
+vprotd %xmm1, %xmm1, %xmm1
+
+// CHECK: vprotq $0, -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x78,0xc3,0x8c,0x82,0x10,0xe3,0x0f,0xe3,0x00]
+vprotq $0, -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: vprotq $0, 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x78,0xc3,0x8c,0x82,0xf0,0x1c,0xf0,0x1c,0x00]
+vprotq $0, 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: vprotq $0, 485498096(%edx), %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x78,0xc3,0x8a,0xf0,0x1c,0xf0,0x1c,0x00]
+vprotq $0, 485498096(%edx), %xmm1
+
+// CHECK: vprotq $0, 485498096, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x78,0xc3,0x0d,0xf0,0x1c,0xf0,0x1c,0x00]
+vprotq $0, 485498096, %xmm1
+
+// CHECK: vprotq $0, 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x78,0xc3,0x4c,0x02,0x40,0x00]
+vprotq $0, 64(%edx,%eax), %xmm1
+
+// CHECK: vprotq $0, (%edx), %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x78,0xc3,0x0a,0x00]
+vprotq $0, (%edx), %xmm1
+
+// CHECK: vprotq $0, %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x78,0xc3,0xc9,0x00]
+vprotq $0, %xmm1, %xmm1
+
+// CHECK: vprotq -485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0xf0,0x93,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vprotq -485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vprotq 485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0xf0,0x93,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vprotq 485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vprotq 485498096(%edx), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0xf0,0x93,0x8a,0xf0,0x1c,0xf0,0x1c]
+vprotq 485498096(%edx), %xmm1, %xmm1
+
+// CHECK: vprotq 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0xf0,0x93,0x0d,0xf0,0x1c,0xf0,0x1c]
+vprotq 485498096, %xmm1, %xmm1
+
+// CHECK: vprotq 64(%edx,%eax), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0xf0,0x93,0x4c,0x02,0x40]
+vprotq 64(%edx,%eax), %xmm1, %xmm1
+
+// CHECK: vprotq (%edx), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0xf0,0x93,0x0a]
+vprotq (%edx), %xmm1, %xmm1
+
+// CHECK: vprotq %xmm1, -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x70,0x93,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vprotq %xmm1, -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: vprotq %xmm1, 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x70,0x93,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vprotq %xmm1, 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: vprotq %xmm1, 485498096(%edx), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x70,0x93,0x8a,0xf0,0x1c,0xf0,0x1c]
+vprotq %xmm1, 485498096(%edx), %xmm1
+
+// CHECK: vprotq %xmm1, 485498096, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x70,0x93,0x0d,0xf0,0x1c,0xf0,0x1c]
+vprotq %xmm1, 485498096, %xmm1
+
+// CHECK: vprotq %xmm1, 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x70,0x93,0x4c,0x02,0x40]
+vprotq %xmm1, 64(%edx,%eax), %xmm1
+
+// CHECK: vprotq %xmm1, (%edx), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x70,0x93,0x0a]
+vprotq %xmm1, (%edx), %xmm1
+
+// CHECK: vprotq %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x70,0x93,0xc9]
+vprotq %xmm1, %xmm1, %xmm1
+
+// CHECK: vprotw $0, -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x78,0xc1,0x8c,0x82,0x10,0xe3,0x0f,0xe3,0x00]
+vprotw $0, -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: vprotw $0, 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x78,0xc1,0x8c,0x82,0xf0,0x1c,0xf0,0x1c,0x00]
+vprotw $0, 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: vprotw $0, 485498096(%edx), %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x78,0xc1,0x8a,0xf0,0x1c,0xf0,0x1c,0x00]
+vprotw $0, 485498096(%edx), %xmm1
+
+// CHECK: vprotw $0, 485498096, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x78,0xc1,0x0d,0xf0,0x1c,0xf0,0x1c,0x00]
+vprotw $0, 485498096, %xmm1
+
+// CHECK: vprotw $0, 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x78,0xc1,0x4c,0x02,0x40,0x00]
+vprotw $0, 64(%edx,%eax), %xmm1
+
+// CHECK: vprotw $0, (%edx), %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x78,0xc1,0x0a,0x00]
+vprotw $0, (%edx), %xmm1
+
+// CHECK: vprotw $0, %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe8,0x78,0xc1,0xc9,0x00]
+vprotw $0, %xmm1, %xmm1
+
+// CHECK: vprotw -485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0xf0,0x91,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vprotw -485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vprotw 485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0xf0,0x91,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vprotw 485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vprotw 485498096(%edx), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0xf0,0x91,0x8a,0xf0,0x1c,0xf0,0x1c]
+vprotw 485498096(%edx), %xmm1, %xmm1
+
+// CHECK: vprotw 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0xf0,0x91,0x0d,0xf0,0x1c,0xf0,0x1c]
+vprotw 485498096, %xmm1, %xmm1
+
+// CHECK: vprotw 64(%edx,%eax), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0xf0,0x91,0x4c,0x02,0x40]
+vprotw 64(%edx,%eax), %xmm1, %xmm1
+
+// CHECK: vprotw (%edx), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0xf0,0x91,0x0a]
+vprotw (%edx), %xmm1, %xmm1
+
+// CHECK: vprotw %xmm1, -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x70,0x91,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vprotw %xmm1, -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: vprotw %xmm1, 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x70,0x91,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vprotw %xmm1, 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: vprotw %xmm1, 485498096(%edx), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x70,0x91,0x8a,0xf0,0x1c,0xf0,0x1c]
+vprotw %xmm1, 485498096(%edx), %xmm1
+
+// CHECK: vprotw %xmm1, 485498096, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x70,0x91,0x0d,0xf0,0x1c,0xf0,0x1c]
+vprotw %xmm1, 485498096, %xmm1
+
+// CHECK: vprotw %xmm1, 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x70,0x91,0x4c,0x02,0x40]
+vprotw %xmm1, 64(%edx,%eax), %xmm1
+
+// CHECK: vprotw %xmm1, (%edx), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x70,0x91,0x0a]
+vprotw %xmm1, (%edx), %xmm1
+
+// CHECK: vprotw %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x70,0x91,0xc9]
+vprotw %xmm1, %xmm1, %xmm1
+
+// CHECK: vpshab -485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0xf0,0x98,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vpshab -485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vpshab 485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0xf0,0x98,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vpshab 485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vpshab 485498096(%edx), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0xf0,0x98,0x8a,0xf0,0x1c,0xf0,0x1c]
+vpshab 485498096(%edx), %xmm1, %xmm1
+
+// CHECK: vpshab 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0xf0,0x98,0x0d,0xf0,0x1c,0xf0,0x1c]
+vpshab 485498096, %xmm1, %xmm1
+
+// CHECK: vpshab 64(%edx,%eax), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0xf0,0x98,0x4c,0x02,0x40]
+vpshab 64(%edx,%eax), %xmm1, %xmm1
+
+// CHECK: vpshab (%edx), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0xf0,0x98,0x0a]
+vpshab (%edx), %xmm1, %xmm1
+
+// CHECK: vpshab %xmm1, -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x70,0x98,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vpshab %xmm1, -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: vpshab %xmm1, 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x70,0x98,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vpshab %xmm1, 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: vpshab %xmm1, 485498096(%edx), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x70,0x98,0x8a,0xf0,0x1c,0xf0,0x1c]
+vpshab %xmm1, 485498096(%edx), %xmm1
+
+// CHECK: vpshab %xmm1, 485498096, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x70,0x98,0x0d,0xf0,0x1c,0xf0,0x1c]
+vpshab %xmm1, 485498096, %xmm1
+
+// CHECK: vpshab %xmm1, 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x70,0x98,0x4c,0x02,0x40]
+vpshab %xmm1, 64(%edx,%eax), %xmm1
+
+// CHECK: vpshab %xmm1, (%edx), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x70,0x98,0x0a]
+vpshab %xmm1, (%edx), %xmm1
+
+// CHECK: vpshab %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x70,0x98,0xc9]
+vpshab %xmm1, %xmm1, %xmm1
+
+// CHECK: vpshad -485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0xf0,0x9a,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vpshad -485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vpshad 485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0xf0,0x9a,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vpshad 485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vpshad 485498096(%edx), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0xf0,0x9a,0x8a,0xf0,0x1c,0xf0,0x1c]
+vpshad 485498096(%edx), %xmm1, %xmm1
+
+// CHECK: vpshad 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0xf0,0x9a,0x0d,0xf0,0x1c,0xf0,0x1c]
+vpshad 485498096, %xmm1, %xmm1
+
+// CHECK: vpshad 64(%edx,%eax), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0xf0,0x9a,0x4c,0x02,0x40]
+vpshad 64(%edx,%eax), %xmm1, %xmm1
+
+// CHECK: vpshad (%edx), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0xf0,0x9a,0x0a]
+vpshad (%edx), %xmm1, %xmm1
+
+// CHECK: vpshad %xmm1, -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x70,0x9a,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vpshad %xmm1, -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: vpshad %xmm1, 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x70,0x9a,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vpshad %xmm1, 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: vpshad %xmm1, 485498096(%edx), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x70,0x9a,0x8a,0xf0,0x1c,0xf0,0x1c]
+vpshad %xmm1, 485498096(%edx), %xmm1
+
+// CHECK: vpshad %xmm1, 485498096, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x70,0x9a,0x0d,0xf0,0x1c,0xf0,0x1c]
+vpshad %xmm1, 485498096, %xmm1
+
+// CHECK: vpshad %xmm1, 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x70,0x9a,0x4c,0x02,0x40]
+vpshad %xmm1, 64(%edx,%eax), %xmm1
+
+// CHECK: vpshad %xmm1, (%edx), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x70,0x9a,0x0a]
+vpshad %xmm1, (%edx), %xmm1
+
+// CHECK: vpshad %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x70,0x9a,0xc9]
+vpshad %xmm1, %xmm1, %xmm1
+
+// CHECK: vpshaq -485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0xf0,0x9b,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vpshaq -485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vpshaq 485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0xf0,0x9b,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vpshaq 485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vpshaq 485498096(%edx), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0xf0,0x9b,0x8a,0xf0,0x1c,0xf0,0x1c]
+vpshaq 485498096(%edx), %xmm1, %xmm1
+
+// CHECK: vpshaq 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0xf0,0x9b,0x0d,0xf0,0x1c,0xf0,0x1c]
+vpshaq 485498096, %xmm1, %xmm1
+
+// CHECK: vpshaq 64(%edx,%eax), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0xf0,0x9b,0x4c,0x02,0x40]
+vpshaq 64(%edx,%eax), %xmm1, %xmm1
+
+// CHECK: vpshaq (%edx), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0xf0,0x9b,0x0a]
+vpshaq (%edx), %xmm1, %xmm1
+
+// CHECK: vpshaq %xmm1, -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x70,0x9b,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vpshaq %xmm1, -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: vpshaq %xmm1, 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x70,0x9b,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vpshaq %xmm1, 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: vpshaq %xmm1, 485498096(%edx), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x70,0x9b,0x8a,0xf0,0x1c,0xf0,0x1c]
+vpshaq %xmm1, 485498096(%edx), %xmm1
+
+// CHECK: vpshaq %xmm1, 485498096, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x70,0x9b,0x0d,0xf0,0x1c,0xf0,0x1c]
+vpshaq %xmm1, 485498096, %xmm1
+
+// CHECK: vpshaq %xmm1, 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x70,0x9b,0x4c,0x02,0x40]
+vpshaq %xmm1, 64(%edx,%eax), %xmm1
+
+// CHECK: vpshaq %xmm1, (%edx), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x70,0x9b,0x0a]
+vpshaq %xmm1, (%edx), %xmm1
+
+// CHECK: vpshaq %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x70,0x9b,0xc9]
+vpshaq %xmm1, %xmm1, %xmm1
+
+// CHECK: vpshaw -485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0xf0,0x99,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vpshaw -485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vpshaw 485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0xf0,0x99,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vpshaw 485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vpshaw 485498096(%edx), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0xf0,0x99,0x8a,0xf0,0x1c,0xf0,0x1c]
+vpshaw 485498096(%edx), %xmm1, %xmm1
+
+// CHECK: vpshaw 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0xf0,0x99,0x0d,0xf0,0x1c,0xf0,0x1c]
+vpshaw 485498096, %xmm1, %xmm1
+
+// CHECK: vpshaw 64(%edx,%eax), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0xf0,0x99,0x4c,0x02,0x40]
+vpshaw 64(%edx,%eax), %xmm1, %xmm1
+
+// CHECK: vpshaw (%edx), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0xf0,0x99,0x0a]
+vpshaw (%edx), %xmm1, %xmm1
+
+// CHECK: vpshaw %xmm1, -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x70,0x99,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vpshaw %xmm1, -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: vpshaw %xmm1, 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x70,0x99,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vpshaw %xmm1, 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: vpshaw %xmm1, 485498096(%edx), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x70,0x99,0x8a,0xf0,0x1c,0xf0,0x1c]
+vpshaw %xmm1, 485498096(%edx), %xmm1
+
+// CHECK: vpshaw %xmm1, 485498096, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x70,0x99,0x0d,0xf0,0x1c,0xf0,0x1c]
+vpshaw %xmm1, 485498096, %xmm1
+
+// CHECK: vpshaw %xmm1, 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x70,0x99,0x4c,0x02,0x40]
+vpshaw %xmm1, 64(%edx,%eax), %xmm1
+
+// CHECK: vpshaw %xmm1, (%edx), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x70,0x99,0x0a]
+vpshaw %xmm1, (%edx), %xmm1
+
+// CHECK: vpshaw %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x70,0x99,0xc9]
+vpshaw %xmm1, %xmm1, %xmm1
+
+// CHECK: vpshlb -485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0xf0,0x94,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vpshlb -485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vpshlb 485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0xf0,0x94,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vpshlb 485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vpshlb 485498096(%edx), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0xf0,0x94,0x8a,0xf0,0x1c,0xf0,0x1c]
+vpshlb 485498096(%edx), %xmm1, %xmm1
+
+// CHECK: vpshlb 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0xf0,0x94,0x0d,0xf0,0x1c,0xf0,0x1c]
+vpshlb 485498096, %xmm1, %xmm1
+
+// CHECK: vpshlb 64(%edx,%eax), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0xf0,0x94,0x4c,0x02,0x40]
+vpshlb 64(%edx,%eax), %xmm1, %xmm1
+
+// CHECK: vpshlb (%edx), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0xf0,0x94,0x0a]
+vpshlb (%edx), %xmm1, %xmm1
+
+// CHECK: vpshlb %xmm1, -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x70,0x94,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vpshlb %xmm1, -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: vpshlb %xmm1, 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x70,0x94,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vpshlb %xmm1, 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: vpshlb %xmm1, 485498096(%edx), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x70,0x94,0x8a,0xf0,0x1c,0xf0,0x1c]
+vpshlb %xmm1, 485498096(%edx), %xmm1
+
+// CHECK: vpshlb %xmm1, 485498096, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x70,0x94,0x0d,0xf0,0x1c,0xf0,0x1c]
+vpshlb %xmm1, 485498096, %xmm1
+
+// CHECK: vpshlb %xmm1, 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x70,0x94,0x4c,0x02,0x40]
+vpshlb %xmm1, 64(%edx,%eax), %xmm1
+
+// CHECK: vpshlb %xmm1, (%edx), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x70,0x94,0x0a]
+vpshlb %xmm1, (%edx), %xmm1
+
+// CHECK: vpshlb %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x70,0x94,0xc9]
+vpshlb %xmm1, %xmm1, %xmm1
+
+// CHECK: vpshld -485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0xf0,0x96,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vpshld -485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vpshld 485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0xf0,0x96,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vpshld 485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vpshld 485498096(%edx), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0xf0,0x96,0x8a,0xf0,0x1c,0xf0,0x1c]
+vpshld 485498096(%edx), %xmm1, %xmm1
+
+// CHECK: vpshld 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0xf0,0x96,0x0d,0xf0,0x1c,0xf0,0x1c]
+vpshld 485498096, %xmm1, %xmm1
+
+// CHECK: vpshld 64(%edx,%eax), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0xf0,0x96,0x4c,0x02,0x40]
+vpshld 64(%edx,%eax), %xmm1, %xmm1
+
+// CHECK: vpshld (%edx), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0xf0,0x96,0x0a]
+vpshld (%edx), %xmm1, %xmm1
+
+// CHECK: vpshld %xmm1, -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x70,0x96,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vpshld %xmm1, -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: vpshld %xmm1, 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x70,0x96,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vpshld %xmm1, 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: vpshld %xmm1, 485498096(%edx), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x70,0x96,0x8a,0xf0,0x1c,0xf0,0x1c]
+vpshld %xmm1, 485498096(%edx), %xmm1
+
+// CHECK: vpshld %xmm1, 485498096, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x70,0x96,0x0d,0xf0,0x1c,0xf0,0x1c]
+vpshld %xmm1, 485498096, %xmm1
+
+// CHECK: vpshld %xmm1, 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x70,0x96,0x4c,0x02,0x40]
+vpshld %xmm1, 64(%edx,%eax), %xmm1
+
+// CHECK: vpshld %xmm1, (%edx), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x70,0x96,0x0a]
+vpshld %xmm1, (%edx), %xmm1
+
+// CHECK: vpshld %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x70,0x96,0xc9]
+vpshld %xmm1, %xmm1, %xmm1
+
+// CHECK: vpshlq -485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0xf0,0x97,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vpshlq -485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vpshlq 485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0xf0,0x97,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vpshlq 485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vpshlq 485498096(%edx), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0xf0,0x97,0x8a,0xf0,0x1c,0xf0,0x1c]
+vpshlq 485498096(%edx), %xmm1, %xmm1
+
+// CHECK: vpshlq 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0xf0,0x97,0x0d,0xf0,0x1c,0xf0,0x1c]
+vpshlq 485498096, %xmm1, %xmm1
+
+// CHECK: vpshlq 64(%edx,%eax), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0xf0,0x97,0x4c,0x02,0x40]
+vpshlq 64(%edx,%eax), %xmm1, %xmm1
+
+// CHECK: vpshlq (%edx), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0xf0,0x97,0x0a]
+vpshlq (%edx), %xmm1, %xmm1
+
+// CHECK: vpshlq %xmm1, -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x70,0x97,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vpshlq %xmm1, -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: vpshlq %xmm1, 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x70,0x97,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vpshlq %xmm1, 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: vpshlq %xmm1, 485498096(%edx), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x70,0x97,0x8a,0xf0,0x1c,0xf0,0x1c]
+vpshlq %xmm1, 485498096(%edx), %xmm1
+
+// CHECK: vpshlq %xmm1, 485498096, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x70,0x97,0x0d,0xf0,0x1c,0xf0,0x1c]
+vpshlq %xmm1, 485498096, %xmm1
+
+// CHECK: vpshlq %xmm1, 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x70,0x97,0x4c,0x02,0x40]
+vpshlq %xmm1, 64(%edx,%eax), %xmm1
+
+// CHECK: vpshlq %xmm1, (%edx), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x70,0x97,0x0a]
+vpshlq %xmm1, (%edx), %xmm1
+
+// CHECK: vpshlq %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x70,0x97,0xc9]
+vpshlq %xmm1, %xmm1, %xmm1
+
+// CHECK: vpshlw -485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0xf0,0x95,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vpshlw -485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vpshlw 485498096(%edx,%eax,4), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0xf0,0x95,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vpshlw 485498096(%edx,%eax,4), %xmm1, %xmm1
+
+// CHECK: vpshlw 485498096(%edx), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0xf0,0x95,0x8a,0xf0,0x1c,0xf0,0x1c]
+vpshlw 485498096(%edx), %xmm1, %xmm1
+
+// CHECK: vpshlw 485498096, %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0xf0,0x95,0x0d,0xf0,0x1c,0xf0,0x1c]
+vpshlw 485498096, %xmm1, %xmm1
+
+// CHECK: vpshlw 64(%edx,%eax), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0xf0,0x95,0x4c,0x02,0x40]
+vpshlw 64(%edx,%eax), %xmm1, %xmm1
+
+// CHECK: vpshlw (%edx), %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0xf0,0x95,0x0a]
+vpshlw (%edx), %xmm1, %xmm1
+
+// CHECK: vpshlw %xmm1, -485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x70,0x95,0x8c,0x82,0x10,0xe3,0x0f,0xe3]
+vpshlw %xmm1, -485498096(%edx,%eax,4), %xmm1
+
+// CHECK: vpshlw %xmm1, 485498096(%edx,%eax,4), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x70,0x95,0x8c,0x82,0xf0,0x1c,0xf0,0x1c]
+vpshlw %xmm1, 485498096(%edx,%eax,4), %xmm1
+
+// CHECK: vpshlw %xmm1, 485498096(%edx), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x70,0x95,0x8a,0xf0,0x1c,0xf0,0x1c]
+vpshlw %xmm1, 485498096(%edx), %xmm1
+
+// CHECK: vpshlw %xmm1, 485498096, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x70,0x95,0x0d,0xf0,0x1c,0xf0,0x1c]
+vpshlw %xmm1, 485498096, %xmm1
+
+// CHECK: vpshlw %xmm1, 64(%edx,%eax), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x70,0x95,0x4c,0x02,0x40]
+vpshlw %xmm1, 64(%edx,%eax), %xmm1
+
+// CHECK: vpshlw %xmm1, (%edx), %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x70,0x95,0x0a]
+vpshlw %xmm1, (%edx), %xmm1
+
+// CHECK: vpshlw %xmm1, %xmm1, %xmm1
+// CHECK: encoding: [0x8f,0xe9,0x70,0x95,0xc9]
+vpshlw %xmm1, %xmm1, %xmm1
+
diff --git a/test/MC/X86/XOP-64.s b/test/MC/X86/XOP-64.s
new file mode 100644
index 0000000000000000000000000000000000000000..bdf108df5199c99ba32efa66b81f33ee5bb9c073
--- /dev/null
+++ b/test/MC/X86/XOP-64.s
@@ -0,0 +1,4502 @@
+// RUN: llvm-mc -triple x86_64-unknown-unknown --show-encoding %s | FileCheck %s
+
+// CHECK: vfrczpd 485498096, %xmm15
+// CHECK: encoding: [0x8f,0x69,0x78,0x81,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfrczpd 485498096, %xmm15
+
+// CHECK: vfrczpd 485498096, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0x81,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+vfrczpd 485498096, %xmm6
+
+// CHECK: vfrczpd 485498096, %ymm7
+// CHECK: encoding: [0x8f,0xe9,0x7c,0x81,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfrczpd 485498096, %ymm7
+
+// CHECK: vfrczpd 485498096, %ymm9
+// CHECK: encoding: [0x8f,0x69,0x7c,0x81,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfrczpd 485498096, %ymm9
+
+// CHECK: vfrczpd 64(%rdx,%rax,4), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x78,0x81,0x7c,0x82,0x40]
+vfrczpd 64(%rdx,%rax,4), %xmm15
+
+// CHECK: vfrczpd -64(%rdx,%rax,4), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x78,0x81,0x7c,0x82,0xc0]
+vfrczpd -64(%rdx,%rax,4), %xmm15
+
+// CHECK: vfrczpd 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0x81,0x74,0x82,0x40]
+vfrczpd 64(%rdx,%rax,4), %xmm6
+
+// CHECK: vfrczpd -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0x81,0x74,0x82,0xc0]
+vfrczpd -64(%rdx,%rax,4), %xmm6
+
+// CHECK: vfrczpd 64(%rdx,%rax,4), %ymm7
+// CHECK: encoding: [0x8f,0xe9,0x7c,0x81,0x7c,0x82,0x40]
+vfrczpd 64(%rdx,%rax,4), %ymm7
+
+// CHECK: vfrczpd -64(%rdx,%rax,4), %ymm7
+// CHECK: encoding: [0x8f,0xe9,0x7c,0x81,0x7c,0x82,0xc0]
+vfrczpd -64(%rdx,%rax,4), %ymm7
+
+// CHECK: vfrczpd 64(%rdx,%rax,4), %ymm9
+// CHECK: encoding: [0x8f,0x69,0x7c,0x81,0x4c,0x82,0x40]
+vfrczpd 64(%rdx,%rax,4), %ymm9
+
+// CHECK: vfrczpd -64(%rdx,%rax,4), %ymm9
+// CHECK: encoding: [0x8f,0x69,0x7c,0x81,0x4c,0x82,0xc0]
+vfrczpd -64(%rdx,%rax,4), %ymm9
+
+// CHECK: vfrczpd 64(%rdx,%rax), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x78,0x81,0x7c,0x02,0x40]
+vfrczpd 64(%rdx,%rax), %xmm15
+
+// CHECK: vfrczpd 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0x81,0x74,0x02,0x40]
+vfrczpd 64(%rdx,%rax), %xmm6
+
+// CHECK: vfrczpd 64(%rdx,%rax), %ymm7
+// CHECK: encoding: [0x8f,0xe9,0x7c,0x81,0x7c,0x02,0x40]
+vfrczpd 64(%rdx,%rax), %ymm7
+
+// CHECK: vfrczpd 64(%rdx,%rax), %ymm9
+// CHECK: encoding: [0x8f,0x69,0x7c,0x81,0x4c,0x02,0x40]
+vfrczpd 64(%rdx,%rax), %ymm9
+
+// CHECK: vfrczpd 64(%rdx), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x78,0x81,0x7a,0x40]
+vfrczpd 64(%rdx), %xmm15
+
+// CHECK: vfrczpd 64(%rdx), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0x81,0x72,0x40]
+vfrczpd 64(%rdx), %xmm6
+
+// CHECK: vfrczpd 64(%rdx), %ymm7
+// CHECK: encoding: [0x8f,0xe9,0x7c,0x81,0x7a,0x40]
+vfrczpd 64(%rdx), %ymm7
+
+// CHECK: vfrczpd 64(%rdx), %ymm9
+// CHECK: encoding: [0x8f,0x69,0x7c,0x81,0x4a,0x40]
+vfrczpd 64(%rdx), %ymm9
+
+// CHECK: vfrczpd (%rdx), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x78,0x81,0x3a]
+vfrczpd (%rdx), %xmm15
+
+// CHECK: vfrczpd (%rdx), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0x81,0x32]
+vfrczpd (%rdx), %xmm6
+
+// CHECK: vfrczpd (%rdx), %ymm7
+// CHECK: encoding: [0x8f,0xe9,0x7c,0x81,0x3a]
+vfrczpd (%rdx), %ymm7
+
+// CHECK: vfrczpd (%rdx), %ymm9
+// CHECK: encoding: [0x8f,0x69,0x7c,0x81,0x0a]
+vfrczpd (%rdx), %ymm9
+
+// CHECK: vfrczpd %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x49,0x78,0x81,0xff]
+vfrczpd %xmm15, %xmm15
+
+// CHECK: vfrczpd %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0x81,0xf6]
+vfrczpd %xmm6, %xmm6
+
+// CHECK: vfrczpd %ymm7, %ymm7
+// CHECK: encoding: [0x8f,0xe9,0x7c,0x81,0xff]
+vfrczpd %ymm7, %ymm7
+
+// CHECK: vfrczpd %ymm9, %ymm9
+// CHECK: encoding: [0x8f,0x49,0x7c,0x81,0xc9]
+vfrczpd %ymm9, %ymm9
+
+// CHECK: vfrczps 485498096, %xmm15
+// CHECK: encoding: [0x8f,0x69,0x78,0x80,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfrczps 485498096, %xmm15
+
+// CHECK: vfrczps 485498096, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0x80,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+vfrczps 485498096, %xmm6
+
+// CHECK: vfrczps 485498096, %ymm7
+// CHECK: encoding: [0x8f,0xe9,0x7c,0x80,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfrczps 485498096, %ymm7
+
+// CHECK: vfrczps 485498096, %ymm9
+// CHECK: encoding: [0x8f,0x69,0x7c,0x80,0x0c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfrczps 485498096, %ymm9
+
+// CHECK: vfrczps 64(%rdx,%rax,4), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x78,0x80,0x7c,0x82,0x40]
+vfrczps 64(%rdx,%rax,4), %xmm15
+
+// CHECK: vfrczps -64(%rdx,%rax,4), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x78,0x80,0x7c,0x82,0xc0]
+vfrczps -64(%rdx,%rax,4), %xmm15
+
+// CHECK: vfrczps 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0x80,0x74,0x82,0x40]
+vfrczps 64(%rdx,%rax,4), %xmm6
+
+// CHECK: vfrczps -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0x80,0x74,0x82,0xc0]
+vfrczps -64(%rdx,%rax,4), %xmm6
+
+// CHECK: vfrczps 64(%rdx,%rax,4), %ymm7
+// CHECK: encoding: [0x8f,0xe9,0x7c,0x80,0x7c,0x82,0x40]
+vfrczps 64(%rdx,%rax,4), %ymm7
+
+// CHECK: vfrczps -64(%rdx,%rax,4), %ymm7
+// CHECK: encoding: [0x8f,0xe9,0x7c,0x80,0x7c,0x82,0xc0]
+vfrczps -64(%rdx,%rax,4), %ymm7
+
+// CHECK: vfrczps 64(%rdx,%rax,4), %ymm9
+// CHECK: encoding: [0x8f,0x69,0x7c,0x80,0x4c,0x82,0x40]
+vfrczps 64(%rdx,%rax,4), %ymm9
+
+// CHECK: vfrczps -64(%rdx,%rax,4), %ymm9
+// CHECK: encoding: [0x8f,0x69,0x7c,0x80,0x4c,0x82,0xc0]
+vfrczps -64(%rdx,%rax,4), %ymm9
+
+// CHECK: vfrczps 64(%rdx,%rax), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x78,0x80,0x7c,0x02,0x40]
+vfrczps 64(%rdx,%rax), %xmm15
+
+// CHECK: vfrczps 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0x80,0x74,0x02,0x40]
+vfrczps 64(%rdx,%rax), %xmm6
+
+// CHECK: vfrczps 64(%rdx,%rax), %ymm7
+// CHECK: encoding: [0x8f,0xe9,0x7c,0x80,0x7c,0x02,0x40]
+vfrczps 64(%rdx,%rax), %ymm7
+
+// CHECK: vfrczps 64(%rdx,%rax), %ymm9
+// CHECK: encoding: [0x8f,0x69,0x7c,0x80,0x4c,0x02,0x40]
+vfrczps 64(%rdx,%rax), %ymm9
+
+// CHECK: vfrczps 64(%rdx), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x78,0x80,0x7a,0x40]
+vfrczps 64(%rdx), %xmm15
+
+// CHECK: vfrczps 64(%rdx), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0x80,0x72,0x40]
+vfrczps 64(%rdx), %xmm6
+
+// CHECK: vfrczps 64(%rdx), %ymm7
+// CHECK: encoding: [0x8f,0xe9,0x7c,0x80,0x7a,0x40]
+vfrczps 64(%rdx), %ymm7
+
+// CHECK: vfrczps 64(%rdx), %ymm9
+// CHECK: encoding: [0x8f,0x69,0x7c,0x80,0x4a,0x40]
+vfrczps 64(%rdx), %ymm9
+
+// CHECK: vfrczps (%rdx), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x78,0x80,0x3a]
+vfrczps (%rdx), %xmm15
+
+// CHECK: vfrczps (%rdx), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0x80,0x32]
+vfrczps (%rdx), %xmm6
+
+// CHECK: vfrczps (%rdx), %ymm7
+// CHECK: encoding: [0x8f,0xe9,0x7c,0x80,0x3a]
+vfrczps (%rdx), %ymm7
+
+// CHECK: vfrczps (%rdx), %ymm9
+// CHECK: encoding: [0x8f,0x69,0x7c,0x80,0x0a]
+vfrczps (%rdx), %ymm9
+
+// CHECK: vfrczps %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x49,0x78,0x80,0xff]
+vfrczps %xmm15, %xmm15
+
+// CHECK: vfrczps %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0x80,0xf6]
+vfrczps %xmm6, %xmm6
+
+// CHECK: vfrczps %ymm7, %ymm7
+// CHECK: encoding: [0x8f,0xe9,0x7c,0x80,0xff]
+vfrczps %ymm7, %ymm7
+
+// CHECK: vfrczps %ymm9, %ymm9
+// CHECK: encoding: [0x8f,0x49,0x7c,0x80,0xc9]
+vfrczps %ymm9, %ymm9
+
+// CHECK: vfrczsd 485498096, %xmm15
+// CHECK: encoding: [0x8f,0x69,0x78,0x83,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfrczsd 485498096, %xmm15
+
+// CHECK: vfrczsd 485498096, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0x83,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+vfrczsd 485498096, %xmm6
+
+// CHECK: vfrczsd 64(%rdx,%rax,4), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x78,0x83,0x7c,0x82,0x40]
+vfrczsd 64(%rdx,%rax,4), %xmm15
+
+// CHECK: vfrczsd -64(%rdx,%rax,4), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x78,0x83,0x7c,0x82,0xc0]
+vfrczsd -64(%rdx,%rax,4), %xmm15
+
+// CHECK: vfrczsd 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0x83,0x74,0x82,0x40]
+vfrczsd 64(%rdx,%rax,4), %xmm6
+
+// CHECK: vfrczsd -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0x83,0x74,0x82,0xc0]
+vfrczsd -64(%rdx,%rax,4), %xmm6
+
+// CHECK: vfrczsd 64(%rdx,%rax), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x78,0x83,0x7c,0x02,0x40]
+vfrczsd 64(%rdx,%rax), %xmm15
+
+// CHECK: vfrczsd 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0x83,0x74,0x02,0x40]
+vfrczsd 64(%rdx,%rax), %xmm6
+
+// CHECK: vfrczsd 64(%rdx), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x78,0x83,0x7a,0x40]
+vfrczsd 64(%rdx), %xmm15
+
+// CHECK: vfrczsd 64(%rdx), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0x83,0x72,0x40]
+vfrczsd 64(%rdx), %xmm6
+
+// CHECK: vfrczsd (%rdx), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x78,0x83,0x3a]
+vfrczsd (%rdx), %xmm15
+
+// CHECK: vfrczsd (%rdx), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0x83,0x32]
+vfrczsd (%rdx), %xmm6
+
+// CHECK: vfrczsd %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x49,0x78,0x83,0xff]
+vfrczsd %xmm15, %xmm15
+
+// CHECK: vfrczsd %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0x83,0xf6]
+vfrczsd %xmm6, %xmm6
+
+// CHECK: vfrczss 485498096, %xmm15
+// CHECK: encoding: [0x8f,0x69,0x78,0x82,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vfrczss 485498096, %xmm15
+
+// CHECK: vfrczss 485498096, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0x82,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+vfrczss 485498096, %xmm6
+
+// CHECK: vfrczss 64(%rdx,%rax,4), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x78,0x82,0x7c,0x82,0x40]
+vfrczss 64(%rdx,%rax,4), %xmm15
+
+// CHECK: vfrczss -64(%rdx,%rax,4), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x78,0x82,0x7c,0x82,0xc0]
+vfrczss -64(%rdx,%rax,4), %xmm15
+
+// CHECK: vfrczss 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0x82,0x74,0x82,0x40]
+vfrczss 64(%rdx,%rax,4), %xmm6
+
+// CHECK: vfrczss -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0x82,0x74,0x82,0xc0]
+vfrczss -64(%rdx,%rax,4), %xmm6
+
+// CHECK: vfrczss 64(%rdx,%rax), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x78,0x82,0x7c,0x02,0x40]
+vfrczss 64(%rdx,%rax), %xmm15
+
+// CHECK: vfrczss 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0x82,0x74,0x02,0x40]
+vfrczss 64(%rdx,%rax), %xmm6
+
+// CHECK: vfrczss 64(%rdx), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x78,0x82,0x7a,0x40]
+vfrczss 64(%rdx), %xmm15
+
+// CHECK: vfrczss 64(%rdx), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0x82,0x72,0x40]
+vfrczss 64(%rdx), %xmm6
+
+// CHECK: vfrczss (%rdx), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x78,0x82,0x3a]
+vfrczss (%rdx), %xmm15
+
+// CHECK: vfrczss (%rdx), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0x82,0x32]
+vfrczss (%rdx), %xmm6
+
+// CHECK: vfrczss %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x49,0x78,0x82,0xff]
+vfrczss %xmm15, %xmm15
+
+// CHECK: vfrczss %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0x82,0xf6]
+vfrczss %xmm6, %xmm6
+
+// CHECK: vpcmov 485498096, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x80,0xa2,0x3c,0x25,0xf0,0x1c,0xf0,0x1c,0xf0]
+vpcmov 485498096, %xmm15, %xmm15, %xmm15
+
+// CHECK: vpcmov 485498096, %xmm6, %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0xc8,0xa2,0x34,0x25,0xf0,0x1c,0xf0,0x1c,0x60]
+vpcmov 485498096, %xmm6, %xmm6, %xmm6
+
+// CHECK: vpcmov 485498096, %ymm7, %ymm7, %ymm7
+// CHECK: encoding: [0x8f,0xe8,0xc4,0xa2,0x3c,0x25,0xf0,0x1c,0xf0,0x1c,0x70]
+vpcmov 485498096, %ymm7, %ymm7, %ymm7
+
+// CHECK: vpcmov 485498096, %ymm9, %ymm9, %ymm9
+// CHECK: encoding: [0x8f,0x68,0xb4,0xa2,0x0c,0x25,0xf0,0x1c,0xf0,0x1c,0x90]
+vpcmov 485498096, %ymm9, %ymm9, %ymm9
+
+// CHECK: vpcmov 64(%rdx,%rax,4), %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x80,0xa2,0x7c,0x82,0x40,0xf0]
+vpcmov 64(%rdx,%rax,4), %xmm15, %xmm15, %xmm15
+
+// CHECK: vpcmov -64(%rdx,%rax,4), %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x80,0xa2,0x7c,0x82,0xc0,0xf0]
+vpcmov -64(%rdx,%rax,4), %xmm15, %xmm15, %xmm15
+
+// CHECK: vpcmov 64(%rdx,%rax,4), %xmm6, %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0xc8,0xa2,0x74,0x82,0x40,0x60]
+vpcmov 64(%rdx,%rax,4), %xmm6, %xmm6, %xmm6
+
+// CHECK: vpcmov -64(%rdx,%rax,4), %xmm6, %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0xc8,0xa2,0x74,0x82,0xc0,0x60]
+vpcmov -64(%rdx,%rax,4), %xmm6, %xmm6, %xmm6
+
+// CHECK: vpcmov 64(%rdx,%rax,4), %ymm7, %ymm7, %ymm7
+// CHECK: encoding: [0x8f,0xe8,0xc4,0xa2,0x7c,0x82,0x40,0x70]
+vpcmov 64(%rdx,%rax,4), %ymm7, %ymm7, %ymm7
+
+// CHECK: vpcmov -64(%rdx,%rax,4), %ymm7, %ymm7, %ymm7
+// CHECK: encoding: [0x8f,0xe8,0xc4,0xa2,0x7c,0x82,0xc0,0x70]
+vpcmov -64(%rdx,%rax,4), %ymm7, %ymm7, %ymm7
+
+// CHECK: vpcmov 64(%rdx,%rax,4), %ymm9, %ymm9, %ymm9
+// CHECK: encoding: [0x8f,0x68,0xb4,0xa2,0x4c,0x82,0x40,0x90]
+vpcmov 64(%rdx,%rax,4), %ymm9, %ymm9, %ymm9
+
+// CHECK: vpcmov -64(%rdx,%rax,4), %ymm9, %ymm9, %ymm9
+// CHECK: encoding: [0x8f,0x68,0xb4,0xa2,0x4c,0x82,0xc0,0x90]
+vpcmov -64(%rdx,%rax,4), %ymm9, %ymm9, %ymm9
+
+// CHECK: vpcmov 64(%rdx,%rax), %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x80,0xa2,0x7c,0x02,0x40,0xf0]
+vpcmov 64(%rdx,%rax), %xmm15, %xmm15, %xmm15
+
+// CHECK: vpcmov 64(%rdx,%rax), %xmm6, %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0xc8,0xa2,0x74,0x02,0x40,0x60]
+vpcmov 64(%rdx,%rax), %xmm6, %xmm6, %xmm6
+
+// CHECK: vpcmov 64(%rdx,%rax), %ymm7, %ymm7, %ymm7
+// CHECK: encoding: [0x8f,0xe8,0xc4,0xa2,0x7c,0x02,0x40,0x70]
+vpcmov 64(%rdx,%rax), %ymm7, %ymm7, %ymm7
+
+// CHECK: vpcmov 64(%rdx,%rax), %ymm9, %ymm9, %ymm9
+// CHECK: encoding: [0x8f,0x68,0xb4,0xa2,0x4c,0x02,0x40,0x90]
+vpcmov 64(%rdx,%rax), %ymm9, %ymm9, %ymm9
+
+// CHECK: vpcmov 64(%rdx), %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x80,0xa2,0x7a,0x40,0xf0]
+vpcmov 64(%rdx), %xmm15, %xmm15, %xmm15
+
+// CHECK: vpcmov 64(%rdx), %xmm6, %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0xc8,0xa2,0x72,0x40,0x60]
+vpcmov 64(%rdx), %xmm6, %xmm6, %xmm6
+
+// CHECK: vpcmov 64(%rdx), %ymm7, %ymm7, %ymm7
+// CHECK: encoding: [0x8f,0xe8,0xc4,0xa2,0x7a,0x40,0x70]
+vpcmov 64(%rdx), %ymm7, %ymm7, %ymm7
+
+// CHECK: vpcmov 64(%rdx), %ymm9, %ymm9, %ymm9
+// CHECK: encoding: [0x8f,0x68,0xb4,0xa2,0x4a,0x40,0x90]
+vpcmov 64(%rdx), %ymm9, %ymm9, %ymm9
+
+// CHECK: vpcmov (%rdx), %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x80,0xa2,0x3a,0xf0]
+vpcmov (%rdx), %xmm15, %xmm15, %xmm15
+
+// CHECK: vpcmov (%rdx), %xmm6, %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0xc8,0xa2,0x32,0x60]
+vpcmov (%rdx), %xmm6, %xmm6, %xmm6
+
+// CHECK: vpcmov (%rdx), %ymm7, %ymm7, %ymm7
+// CHECK: encoding: [0x8f,0xe8,0xc4,0xa2,0x3a,0x70]
+vpcmov (%rdx), %ymm7, %ymm7, %ymm7
+
+// CHECK: vpcmov (%rdx), %ymm9, %ymm9, %ymm9
+// CHECK: encoding: [0x8f,0x68,0xb4,0xa2,0x0a,0x90]
+vpcmov (%rdx), %ymm9, %ymm9, %ymm9
+
+// CHECK: vpcmov %xmm15, 485498096, %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0xa2,0x3c,0x25,0xf0,0x1c,0xf0,0x1c,0xf0]
+vpcmov %xmm15, 485498096, %xmm15, %xmm15
+
+// CHECK: vpcmov %xmm15, 64(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0xa2,0x7c,0x82,0x40,0xf0]
+vpcmov %xmm15, 64(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vpcmov %xmm15, -64(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0xa2,0x7c,0x82,0xc0,0xf0]
+vpcmov %xmm15, -64(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vpcmov %xmm15, 64(%rdx,%rax), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0xa2,0x7c,0x02,0x40,0xf0]
+vpcmov %xmm15, 64(%rdx,%rax), %xmm15, %xmm15
+
+// CHECK: vpcmov %xmm15, 64(%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0xa2,0x7a,0x40,0xf0]
+vpcmov %xmm15, 64(%rdx), %xmm15, %xmm15
+
+// CHECK: vpcmov %xmm15, (%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0xa2,0x3a,0xf0]
+vpcmov %xmm15, (%rdx), %xmm15, %xmm15
+
+// CHECK: vpcmov %xmm15, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x48,0x00,0xa2,0xff,0xf0]
+vpcmov %xmm15, %xmm15, %xmm15, %xmm15
+
+// CHECK: vpcmov %xmm6, 485498096, %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0xa2,0x34,0x25,0xf0,0x1c,0xf0,0x1c,0x60]
+vpcmov %xmm6, 485498096, %xmm6, %xmm6
+
+// CHECK: vpcmov %xmm6, 64(%rdx,%rax,4), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0xa2,0x74,0x82,0x40,0x60]
+vpcmov %xmm6, 64(%rdx,%rax,4), %xmm6, %xmm6
+
+// CHECK: vpcmov %xmm6, -64(%rdx,%rax,4), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0xa2,0x74,0x82,0xc0,0x60]
+vpcmov %xmm6, -64(%rdx,%rax,4), %xmm6, %xmm6
+
+// CHECK: vpcmov %xmm6, 64(%rdx,%rax), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0xa2,0x74,0x02,0x40,0x60]
+vpcmov %xmm6, 64(%rdx,%rax), %xmm6, %xmm6
+
+// CHECK: vpcmov %xmm6, 64(%rdx), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0xa2,0x72,0x40,0x60]
+vpcmov %xmm6, 64(%rdx), %xmm6, %xmm6
+
+// CHECK: vpcmov %xmm6, (%rdx), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0xa2,0x32,0x60]
+vpcmov %xmm6, (%rdx), %xmm6, %xmm6
+
+// CHECK: vpcmov %xmm6, %xmm6, %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0xa2,0xf6,0x60]
+vpcmov %xmm6, %xmm6, %xmm6, %xmm6
+
+// CHECK: vpcmov %ymm7, 485498096, %ymm7, %ymm7
+// CHECK: encoding: [0x8f,0xe8,0x44,0xa2,0x3c,0x25,0xf0,0x1c,0xf0,0x1c,0x70]
+vpcmov %ymm7, 485498096, %ymm7, %ymm7
+
+// CHECK: vpcmov %ymm7, 64(%rdx,%rax,4), %ymm7, %ymm7
+// CHECK: encoding: [0x8f,0xe8,0x44,0xa2,0x7c,0x82,0x40,0x70]
+vpcmov %ymm7, 64(%rdx,%rax,4), %ymm7, %ymm7
+
+// CHECK: vpcmov %ymm7, -64(%rdx,%rax,4), %ymm7, %ymm7
+// CHECK: encoding: [0x8f,0xe8,0x44,0xa2,0x7c,0x82,0xc0,0x70]
+vpcmov %ymm7, -64(%rdx,%rax,4), %ymm7, %ymm7
+
+// CHECK: vpcmov %ymm7, 64(%rdx,%rax), %ymm7, %ymm7
+// CHECK: encoding: [0x8f,0xe8,0x44,0xa2,0x7c,0x02,0x40,0x70]
+vpcmov %ymm7, 64(%rdx,%rax), %ymm7, %ymm7
+
+// CHECK: vpcmov %ymm7, 64(%rdx), %ymm7, %ymm7
+// CHECK: encoding: [0x8f,0xe8,0x44,0xa2,0x7a,0x40,0x70]
+vpcmov %ymm7, 64(%rdx), %ymm7, %ymm7
+
+// CHECK: vpcmov %ymm7, (%rdx), %ymm7, %ymm7
+// CHECK: encoding: [0x8f,0xe8,0x44,0xa2,0x3a,0x70]
+vpcmov %ymm7, (%rdx), %ymm7, %ymm7
+
+// CHECK: vpcmov %ymm7, %ymm7, %ymm7, %ymm7
+// CHECK: encoding: [0x8f,0xe8,0x44,0xa2,0xff,0x70]
+vpcmov %ymm7, %ymm7, %ymm7, %ymm7
+
+// CHECK: vpcmov %ymm9, 485498096, %ymm9, %ymm9
+// CHECK: encoding: [0x8f,0x68,0x34,0xa2,0x0c,0x25,0xf0,0x1c,0xf0,0x1c,0x90]
+vpcmov %ymm9, 485498096, %ymm9, %ymm9
+
+// CHECK: vpcmov %ymm9, 64(%rdx,%rax,4), %ymm9, %ymm9
+// CHECK: encoding: [0x8f,0x68,0x34,0xa2,0x4c,0x82,0x40,0x90]
+vpcmov %ymm9, 64(%rdx,%rax,4), %ymm9, %ymm9
+
+// CHECK: vpcmov %ymm9, -64(%rdx,%rax,4), %ymm9, %ymm9
+// CHECK: encoding: [0x8f,0x68,0x34,0xa2,0x4c,0x82,0xc0,0x90]
+vpcmov %ymm9, -64(%rdx,%rax,4), %ymm9, %ymm9
+
+// CHECK: vpcmov %ymm9, 64(%rdx,%rax), %ymm9, %ymm9
+// CHECK: encoding: [0x8f,0x68,0x34,0xa2,0x4c,0x02,0x40,0x90]
+vpcmov %ymm9, 64(%rdx,%rax), %ymm9, %ymm9
+
+// CHECK: vpcmov %ymm9, 64(%rdx), %ymm9, %ymm9
+// CHECK: encoding: [0x8f,0x68,0x34,0xa2,0x4a,0x40,0x90]
+vpcmov %ymm9, 64(%rdx), %ymm9, %ymm9
+
+// CHECK: vpcmov %ymm9, (%rdx), %ymm9, %ymm9
+// CHECK: encoding: [0x8f,0x68,0x34,0xa2,0x0a,0x90]
+vpcmov %ymm9, (%rdx), %ymm9, %ymm9
+
+// CHECK: vpcmov %ymm9, %ymm9, %ymm9, %ymm9
+// CHECK: encoding: [0x8f,0x48,0x34,0xa2,0xc9,0x90]
+vpcmov %ymm9, %ymm9, %ymm9, %ymm9
+
+// CHECK: vpcomltb 485498096, %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0xcc,0x3c,0x25,0xf0,0x1c,0xf0,0x1c,0x00]
+vpcomltb 485498096, %xmm15, %xmm15
+
+// CHECK: vpcomltb 485498096, %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0xcc,0x34,0x25,0xf0,0x1c,0xf0,0x1c,0x00]
+vpcomltb 485498096, %xmm6, %xmm6
+
+// CHECK: vpcomltb 64(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0xcc,0x7c,0x82,0x40,0x00]
+vpcomltb 64(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vpcomltb -64(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0xcc,0x7c,0x82,0xc0,0x00]
+vpcomltb -64(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vpcomltb 64(%rdx,%rax,4), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0xcc,0x74,0x82,0x40,0x00]
+vpcomltb 64(%rdx,%rax,4), %xmm6, %xmm6
+
+// CHECK: vpcomltb -64(%rdx,%rax,4), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0xcc,0x74,0x82,0xc0,0x00]
+vpcomltb -64(%rdx,%rax,4), %xmm6, %xmm6
+
+// CHECK: vpcomltb 64(%rdx,%rax), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0xcc,0x7c,0x02,0x40,0x00]
+vpcomltb 64(%rdx,%rax), %xmm15, %xmm15
+
+// CHECK: vpcomltb 64(%rdx,%rax), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0xcc,0x74,0x02,0x40,0x00]
+vpcomltb 64(%rdx,%rax), %xmm6, %xmm6
+
+// CHECK: vpcomltb 64(%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0xcc,0x7a,0x40,0x00]
+vpcomltb 64(%rdx), %xmm15, %xmm15
+
+// CHECK: vpcomltb 64(%rdx), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0xcc,0x72,0x40,0x00]
+vpcomltb 64(%rdx), %xmm6, %xmm6
+
+// CHECK: vpcomltb (%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0xcc,0x3a,0x00]
+vpcomltb (%rdx), %xmm15, %xmm15
+
+// CHECK: vpcomltb (%rdx), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0xcc,0x32,0x00]
+vpcomltb (%rdx), %xmm6, %xmm6
+
+// CHECK: vpcomltb %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x48,0x00,0xcc,0xff,0x00]
+vpcomltb %xmm15, %xmm15, %xmm15
+
+// CHECK: vpcomltb %xmm6, %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0xcc,0xf6,0x00]
+vpcomltb %xmm6, %xmm6, %xmm6
+
+// CHECK: vpcomltd 485498096, %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0xce,0x3c,0x25,0xf0,0x1c,0xf0,0x1c,0x00]
+vpcomltd 485498096, %xmm15, %xmm15
+
+// CHECK: vpcomltd 485498096, %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0xce,0x34,0x25,0xf0,0x1c,0xf0,0x1c,0x00]
+vpcomltd 485498096, %xmm6, %xmm6
+
+// CHECK: vpcomltd 64(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0xce,0x7c,0x82,0x40,0x00]
+vpcomltd 64(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vpcomltd -64(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0xce,0x7c,0x82,0xc0,0x00]
+vpcomltd -64(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vpcomltd 64(%rdx,%rax,4), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0xce,0x74,0x82,0x40,0x00]
+vpcomltd 64(%rdx,%rax,4), %xmm6, %xmm6
+
+// CHECK: vpcomltd -64(%rdx,%rax,4), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0xce,0x74,0x82,0xc0,0x00]
+vpcomltd -64(%rdx,%rax,4), %xmm6, %xmm6
+
+// CHECK: vpcomltd 64(%rdx,%rax), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0xce,0x7c,0x02,0x40,0x00]
+vpcomltd 64(%rdx,%rax), %xmm15, %xmm15
+
+// CHECK: vpcomltd 64(%rdx,%rax), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0xce,0x74,0x02,0x40,0x00]
+vpcomltd 64(%rdx,%rax), %xmm6, %xmm6
+
+// CHECK: vpcomltd 64(%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0xce,0x7a,0x40,0x00]
+vpcomltd 64(%rdx), %xmm15, %xmm15
+
+// CHECK: vpcomltd 64(%rdx), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0xce,0x72,0x40,0x00]
+vpcomltd 64(%rdx), %xmm6, %xmm6
+
+// CHECK: vpcomltd (%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0xce,0x3a,0x00]
+vpcomltd (%rdx), %xmm15, %xmm15
+
+// CHECK: vpcomltd (%rdx), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0xce,0x32,0x00]
+vpcomltd (%rdx), %xmm6, %xmm6
+
+// CHECK: vpcomltd %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x48,0x00,0xce,0xff,0x00]
+vpcomltd %xmm15, %xmm15, %xmm15
+
+// CHECK: vpcomltd %xmm6, %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0xce,0xf6,0x00]
+vpcomltd %xmm6, %xmm6, %xmm6
+
+// CHECK: vpcomltq 485498096, %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0xcf,0x3c,0x25,0xf0,0x1c,0xf0,0x1c,0x00]
+vpcomltq 485498096, %xmm15, %xmm15
+
+// CHECK: vpcomltq 485498096, %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0xcf,0x34,0x25,0xf0,0x1c,0xf0,0x1c,0x00]
+vpcomltq 485498096, %xmm6, %xmm6
+
+// CHECK: vpcomltq 64(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0xcf,0x7c,0x82,0x40,0x00]
+vpcomltq 64(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vpcomltq -64(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0xcf,0x7c,0x82,0xc0,0x00]
+vpcomltq -64(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vpcomltq 64(%rdx,%rax,4), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0xcf,0x74,0x82,0x40,0x00]
+vpcomltq 64(%rdx,%rax,4), %xmm6, %xmm6
+
+// CHECK: vpcomltq -64(%rdx,%rax,4), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0xcf,0x74,0x82,0xc0,0x00]
+vpcomltq -64(%rdx,%rax,4), %xmm6, %xmm6
+
+// CHECK: vpcomltq 64(%rdx,%rax), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0xcf,0x7c,0x02,0x40,0x00]
+vpcomltq 64(%rdx,%rax), %xmm15, %xmm15
+
+// CHECK: vpcomltq 64(%rdx,%rax), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0xcf,0x74,0x02,0x40,0x00]
+vpcomltq 64(%rdx,%rax), %xmm6, %xmm6
+
+// CHECK: vpcomltq 64(%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0xcf,0x7a,0x40,0x00]
+vpcomltq 64(%rdx), %xmm15, %xmm15
+
+// CHECK: vpcomltq 64(%rdx), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0xcf,0x72,0x40,0x00]
+vpcomltq 64(%rdx), %xmm6, %xmm6
+
+// CHECK: vpcomltq (%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0xcf,0x3a,0x00]
+vpcomltq (%rdx), %xmm15, %xmm15
+
+// CHECK: vpcomltq (%rdx), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0xcf,0x32,0x00]
+vpcomltq (%rdx), %xmm6, %xmm6
+
+// CHECK: vpcomltq %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x48,0x00,0xcf,0xff,0x00]
+vpcomltq %xmm15, %xmm15, %xmm15
+
+// CHECK: vpcomltq %xmm6, %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0xcf,0xf6,0x00]
+vpcomltq %xmm6, %xmm6, %xmm6
+
+// CHECK: vpcomltub 485498096, %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0xec,0x3c,0x25,0xf0,0x1c,0xf0,0x1c,0x00]
+vpcomltub 485498096, %xmm15, %xmm15
+
+// CHECK: vpcomltub 485498096, %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0xec,0x34,0x25,0xf0,0x1c,0xf0,0x1c,0x00]
+vpcomltub 485498096, %xmm6, %xmm6
+
+// CHECK: vpcomltub 64(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0xec,0x7c,0x82,0x40,0x00]
+vpcomltub 64(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vpcomltub -64(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0xec,0x7c,0x82,0xc0,0x00]
+vpcomltub -64(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vpcomltub 64(%rdx,%rax,4), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0xec,0x74,0x82,0x40,0x00]
+vpcomltub 64(%rdx,%rax,4), %xmm6, %xmm6
+
+// CHECK: vpcomltub -64(%rdx,%rax,4), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0xec,0x74,0x82,0xc0,0x00]
+vpcomltub -64(%rdx,%rax,4), %xmm6, %xmm6
+
+// CHECK: vpcomltub 64(%rdx,%rax), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0xec,0x7c,0x02,0x40,0x00]
+vpcomltub 64(%rdx,%rax), %xmm15, %xmm15
+
+// CHECK: vpcomltub 64(%rdx,%rax), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0xec,0x74,0x02,0x40,0x00]
+vpcomltub 64(%rdx,%rax), %xmm6, %xmm6
+
+// CHECK: vpcomltub 64(%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0xec,0x7a,0x40,0x00]
+vpcomltub 64(%rdx), %xmm15, %xmm15
+
+// CHECK: vpcomltub 64(%rdx), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0xec,0x72,0x40,0x00]
+vpcomltub 64(%rdx), %xmm6, %xmm6
+
+// CHECK: vpcomltub (%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0xec,0x3a,0x00]
+vpcomltub (%rdx), %xmm15, %xmm15
+
+// CHECK: vpcomltub (%rdx), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0xec,0x32,0x00]
+vpcomltub (%rdx), %xmm6, %xmm6
+
+// CHECK: vpcomltub %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x48,0x00,0xec,0xff,0x00]
+vpcomltub %xmm15, %xmm15, %xmm15
+
+// CHECK: vpcomltub %xmm6, %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0xec,0xf6,0x00]
+vpcomltub %xmm6, %xmm6, %xmm6
+
+// CHECK: vpcomltud 485498096, %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0xee,0x3c,0x25,0xf0,0x1c,0xf0,0x1c,0x00]
+vpcomltud 485498096, %xmm15, %xmm15
+
+// CHECK: vpcomltud 485498096, %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0xee,0x34,0x25,0xf0,0x1c,0xf0,0x1c,0x00]
+vpcomltud 485498096, %xmm6, %xmm6
+
+// CHECK: vpcomltud 64(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0xee,0x7c,0x82,0x40,0x00]
+vpcomltud 64(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vpcomltud -64(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0xee,0x7c,0x82,0xc0,0x00]
+vpcomltud -64(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vpcomltud 64(%rdx,%rax,4), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0xee,0x74,0x82,0x40,0x00]
+vpcomltud 64(%rdx,%rax,4), %xmm6, %xmm6
+
+// CHECK: vpcomltud -64(%rdx,%rax,4), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0xee,0x74,0x82,0xc0,0x00]
+vpcomltud -64(%rdx,%rax,4), %xmm6, %xmm6
+
+// CHECK: vpcomltud 64(%rdx,%rax), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0xee,0x7c,0x02,0x40,0x00]
+vpcomltud 64(%rdx,%rax), %xmm15, %xmm15
+
+// CHECK: vpcomltud 64(%rdx,%rax), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0xee,0x74,0x02,0x40,0x00]
+vpcomltud 64(%rdx,%rax), %xmm6, %xmm6
+
+// CHECK: vpcomltud 64(%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0xee,0x7a,0x40,0x00]
+vpcomltud 64(%rdx), %xmm15, %xmm15
+
+// CHECK: vpcomltud 64(%rdx), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0xee,0x72,0x40,0x00]
+vpcomltud 64(%rdx), %xmm6, %xmm6
+
+// CHECK: vpcomltud (%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0xee,0x3a,0x00]
+vpcomltud (%rdx), %xmm15, %xmm15
+
+// CHECK: vpcomltud (%rdx), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0xee,0x32,0x00]
+vpcomltud (%rdx), %xmm6, %xmm6
+
+// CHECK: vpcomltud %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x48,0x00,0xee,0xff,0x00]
+vpcomltud %xmm15, %xmm15, %xmm15
+
+// CHECK: vpcomltud %xmm6, %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0xee,0xf6,0x00]
+vpcomltud %xmm6, %xmm6, %xmm6
+
+// CHECK: vpcomltuq 485498096, %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0xef,0x3c,0x25,0xf0,0x1c,0xf0,0x1c,0x00]
+vpcomltuq 485498096, %xmm15, %xmm15
+
+// CHECK: vpcomltuq 485498096, %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0xef,0x34,0x25,0xf0,0x1c,0xf0,0x1c,0x00]
+vpcomltuq 485498096, %xmm6, %xmm6
+
+// CHECK: vpcomltuq 64(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0xef,0x7c,0x82,0x40,0x00]
+vpcomltuq 64(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vpcomltuq -64(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0xef,0x7c,0x82,0xc0,0x00]
+vpcomltuq -64(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vpcomltuq 64(%rdx,%rax,4), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0xef,0x74,0x82,0x40,0x00]
+vpcomltuq 64(%rdx,%rax,4), %xmm6, %xmm6
+
+// CHECK: vpcomltuq -64(%rdx,%rax,4), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0xef,0x74,0x82,0xc0,0x00]
+vpcomltuq -64(%rdx,%rax,4), %xmm6, %xmm6
+
+// CHECK: vpcomltuq 64(%rdx,%rax), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0xef,0x7c,0x02,0x40,0x00]
+vpcomltuq 64(%rdx,%rax), %xmm15, %xmm15
+
+// CHECK: vpcomltuq 64(%rdx,%rax), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0xef,0x74,0x02,0x40,0x00]
+vpcomltuq 64(%rdx,%rax), %xmm6, %xmm6
+
+// CHECK: vpcomltuq 64(%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0xef,0x7a,0x40,0x00]
+vpcomltuq 64(%rdx), %xmm15, %xmm15
+
+// CHECK: vpcomltuq 64(%rdx), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0xef,0x72,0x40,0x00]
+vpcomltuq 64(%rdx), %xmm6, %xmm6
+
+// CHECK: vpcomltuq (%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0xef,0x3a,0x00]
+vpcomltuq (%rdx), %xmm15, %xmm15
+
+// CHECK: vpcomltuq (%rdx), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0xef,0x32,0x00]
+vpcomltuq (%rdx), %xmm6, %xmm6
+
+// CHECK: vpcomltuq %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x48,0x00,0xef,0xff,0x00]
+vpcomltuq %xmm15, %xmm15, %xmm15
+
+// CHECK: vpcomltuq %xmm6, %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0xef,0xf6,0x00]
+vpcomltuq %xmm6, %xmm6, %xmm6
+
+// CHECK: vpcomltuw 485498096, %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0xed,0x3c,0x25,0xf0,0x1c,0xf0,0x1c,0x00]
+vpcomltuw 485498096, %xmm15, %xmm15
+
+// CHECK: vpcomltuw 485498096, %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0xed,0x34,0x25,0xf0,0x1c,0xf0,0x1c,0x00]
+vpcomltuw 485498096, %xmm6, %xmm6
+
+// CHECK: vpcomltuw 64(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0xed,0x7c,0x82,0x40,0x00]
+vpcomltuw 64(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vpcomltuw -64(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0xed,0x7c,0x82,0xc0,0x00]
+vpcomltuw -64(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vpcomltuw 64(%rdx,%rax,4), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0xed,0x74,0x82,0x40,0x00]
+vpcomltuw 64(%rdx,%rax,4), %xmm6, %xmm6
+
+// CHECK: vpcomltuw -64(%rdx,%rax,4), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0xed,0x74,0x82,0xc0,0x00]
+vpcomltuw -64(%rdx,%rax,4), %xmm6, %xmm6
+
+// CHECK: vpcomltuw 64(%rdx,%rax), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0xed,0x7c,0x02,0x40,0x00]
+vpcomltuw 64(%rdx,%rax), %xmm15, %xmm15
+
+// CHECK: vpcomltuw 64(%rdx,%rax), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0xed,0x74,0x02,0x40,0x00]
+vpcomltuw 64(%rdx,%rax), %xmm6, %xmm6
+
+// CHECK: vpcomltuw 64(%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0xed,0x7a,0x40,0x00]
+vpcomltuw 64(%rdx), %xmm15, %xmm15
+
+// CHECK: vpcomltuw 64(%rdx), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0xed,0x72,0x40,0x00]
+vpcomltuw 64(%rdx), %xmm6, %xmm6
+
+// CHECK: vpcomltuw (%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0xed,0x3a,0x00]
+vpcomltuw (%rdx), %xmm15, %xmm15
+
+// CHECK: vpcomltuw (%rdx), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0xed,0x32,0x00]
+vpcomltuw (%rdx), %xmm6, %xmm6
+
+// CHECK: vpcomltuw %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x48,0x00,0xed,0xff,0x00]
+vpcomltuw %xmm15, %xmm15, %xmm15
+
+// CHECK: vpcomltuw %xmm6, %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0xed,0xf6,0x00]
+vpcomltuw %xmm6, %xmm6, %xmm6
+
+// CHECK: vpcomltw 485498096, %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0xcd,0x3c,0x25,0xf0,0x1c,0xf0,0x1c,0x00]
+vpcomltw 485498096, %xmm15, %xmm15
+
+// CHECK: vpcomltw 485498096, %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0xcd,0x34,0x25,0xf0,0x1c,0xf0,0x1c,0x00]
+vpcomltw 485498096, %xmm6, %xmm6
+
+// CHECK: vpcomltw 64(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0xcd,0x7c,0x82,0x40,0x00]
+vpcomltw 64(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vpcomltw -64(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0xcd,0x7c,0x82,0xc0,0x00]
+vpcomltw -64(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vpcomltw 64(%rdx,%rax,4), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0xcd,0x74,0x82,0x40,0x00]
+vpcomltw 64(%rdx,%rax,4), %xmm6, %xmm6
+
+// CHECK: vpcomltw -64(%rdx,%rax,4), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0xcd,0x74,0x82,0xc0,0x00]
+vpcomltw -64(%rdx,%rax,4), %xmm6, %xmm6
+
+// CHECK: vpcomltw 64(%rdx,%rax), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0xcd,0x7c,0x02,0x40,0x00]
+vpcomltw 64(%rdx,%rax), %xmm15, %xmm15
+
+// CHECK: vpcomltw 64(%rdx,%rax), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0xcd,0x74,0x02,0x40,0x00]
+vpcomltw 64(%rdx,%rax), %xmm6, %xmm6
+
+// CHECK: vpcomltw 64(%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0xcd,0x7a,0x40,0x00]
+vpcomltw 64(%rdx), %xmm15, %xmm15
+
+// CHECK: vpcomltw 64(%rdx), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0xcd,0x72,0x40,0x00]
+vpcomltw 64(%rdx), %xmm6, %xmm6
+
+// CHECK: vpcomltw (%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0xcd,0x3a,0x00]
+vpcomltw (%rdx), %xmm15, %xmm15
+
+// CHECK: vpcomltw (%rdx), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0xcd,0x32,0x00]
+vpcomltw (%rdx), %xmm6, %xmm6
+
+// CHECK: vpcomltw %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x48,0x00,0xcd,0xff,0x00]
+vpcomltw %xmm15, %xmm15, %xmm15
+
+// CHECK: vpcomltw %xmm6, %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0xcd,0xf6,0x00]
+vpcomltw %xmm6, %xmm6, %xmm6
+
+// CHECK: vpermil2pd $0, 485498096, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x63,0x81,0x49,0x3c,0x25,0xf0,0x1c,0xf0,0x1c,0xf0]
+vpermil2pd $0, 485498096, %xmm15, %xmm15, %xmm15
+
+// CHECK: vpermil2pd $0, 485498096, %xmm6, %xmm6, %xmm6
+// CHECK: encoding: [0xc4,0xe3,0xc9,0x49,0x34,0x25,0xf0,0x1c,0xf0,0x1c,0x60]
+vpermil2pd $0, 485498096, %xmm6, %xmm6, %xmm6
+
+// CHECK: vpermil2pd $0, 485498096, %ymm7, %ymm7, %ymm7
+// CHECK: encoding: [0xc4,0xe3,0xc5,0x49,0x3c,0x25,0xf0,0x1c,0xf0,0x1c,0x70]
+vpermil2pd $0, 485498096, %ymm7, %ymm7, %ymm7
+
+// CHECK: vpermil2pd $0, 485498096, %ymm9, %ymm9, %ymm9
+// CHECK: encoding: [0xc4,0x63,0xb5,0x49,0x0c,0x25,0xf0,0x1c,0xf0,0x1c,0x90]
+vpermil2pd $0, 485498096, %ymm9, %ymm9, %ymm9
+
+// CHECK: vpermil2pd $0, 64(%rdx,%rax,4), %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x63,0x81,0x49,0x7c,0x82,0x40,0xf0]
+vpermil2pd $0, 64(%rdx,%rax,4), %xmm15, %xmm15, %xmm15
+
+// CHECK: vpermil2pd $0, -64(%rdx,%rax,4), %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x63,0x81,0x49,0x7c,0x82,0xc0,0xf0]
+vpermil2pd $0, -64(%rdx,%rax,4), %xmm15, %xmm15, %xmm15
+
+// CHECK: vpermil2pd $0, 64(%rdx,%rax,4), %xmm6, %xmm6, %xmm6
+// CHECK: encoding: [0xc4,0xe3,0xc9,0x49,0x74,0x82,0x40,0x60]
+vpermil2pd $0, 64(%rdx,%rax,4), %xmm6, %xmm6, %xmm6
+
+// CHECK: vpermil2pd $0, -64(%rdx,%rax,4), %xmm6, %xmm6, %xmm6
+// CHECK: encoding: [0xc4,0xe3,0xc9,0x49,0x74,0x82,0xc0,0x60]
+vpermil2pd $0, -64(%rdx,%rax,4), %xmm6, %xmm6, %xmm6
+
+// CHECK: vpermil2pd $0, 64(%rdx,%rax,4), %ymm7, %ymm7, %ymm7
+// CHECK: encoding: [0xc4,0xe3,0xc5,0x49,0x7c,0x82,0x40,0x70]
+vpermil2pd $0, 64(%rdx,%rax,4), %ymm7, %ymm7, %ymm7
+
+// CHECK: vpermil2pd $0, -64(%rdx,%rax,4), %ymm7, %ymm7, %ymm7
+// CHECK: encoding: [0xc4,0xe3,0xc5,0x49,0x7c,0x82,0xc0,0x70]
+vpermil2pd $0, -64(%rdx,%rax,4), %ymm7, %ymm7, %ymm7
+
+// CHECK: vpermil2pd $0, 64(%rdx,%rax,4), %ymm9, %ymm9, %ymm9
+// CHECK: encoding: [0xc4,0x63,0xb5,0x49,0x4c,0x82,0x40,0x90]
+vpermil2pd $0, 64(%rdx,%rax,4), %ymm9, %ymm9, %ymm9
+
+// CHECK: vpermil2pd $0, -64(%rdx,%rax,4), %ymm9, %ymm9, %ymm9
+// CHECK: encoding: [0xc4,0x63,0xb5,0x49,0x4c,0x82,0xc0,0x90]
+vpermil2pd $0, -64(%rdx,%rax,4), %ymm9, %ymm9, %ymm9
+
+// CHECK: vpermil2pd $0, 64(%rdx,%rax), %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x63,0x81,0x49,0x7c,0x02,0x40,0xf0]
+vpermil2pd $0, 64(%rdx,%rax), %xmm15, %xmm15, %xmm15
+
+// CHECK: vpermil2pd $0, 64(%rdx,%rax), %xmm6, %xmm6, %xmm6
+// CHECK: encoding: [0xc4,0xe3,0xc9,0x49,0x74,0x02,0x40,0x60]
+vpermil2pd $0, 64(%rdx,%rax), %xmm6, %xmm6, %xmm6
+
+// CHECK: vpermil2pd $0, 64(%rdx,%rax), %ymm7, %ymm7, %ymm7
+// CHECK: encoding: [0xc4,0xe3,0xc5,0x49,0x7c,0x02,0x40,0x70]
+vpermil2pd $0, 64(%rdx,%rax), %ymm7, %ymm7, %ymm7
+
+// CHECK: vpermil2pd $0, 64(%rdx,%rax), %ymm9, %ymm9, %ymm9
+// CHECK: encoding: [0xc4,0x63,0xb5,0x49,0x4c,0x02,0x40,0x90]
+vpermil2pd $0, 64(%rdx,%rax), %ymm9, %ymm9, %ymm9
+
+// CHECK: vpermil2pd $0, 64(%rdx), %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x63,0x81,0x49,0x7a,0x40,0xf0]
+vpermil2pd $0, 64(%rdx), %xmm15, %xmm15, %xmm15
+
+// CHECK: vpermil2pd $0, 64(%rdx), %xmm6, %xmm6, %xmm6
+// CHECK: encoding: [0xc4,0xe3,0xc9,0x49,0x72,0x40,0x60]
+vpermil2pd $0, 64(%rdx), %xmm6, %xmm6, %xmm6
+
+// CHECK: vpermil2pd $0, 64(%rdx), %ymm7, %ymm7, %ymm7
+// CHECK: encoding: [0xc4,0xe3,0xc5,0x49,0x7a,0x40,0x70]
+vpermil2pd $0, 64(%rdx), %ymm7, %ymm7, %ymm7
+
+// CHECK: vpermil2pd $0, 64(%rdx), %ymm9, %ymm9, %ymm9
+// CHECK: encoding: [0xc4,0x63,0xb5,0x49,0x4a,0x40,0x90]
+vpermil2pd $0, 64(%rdx), %ymm9, %ymm9, %ymm9
+
+// CHECK: vpermil2pd $0, (%rdx), %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x63,0x81,0x49,0x3a,0xf0]
+vpermil2pd $0, (%rdx), %xmm15, %xmm15, %xmm15
+
+// CHECK: vpermil2pd $0, (%rdx), %xmm6, %xmm6, %xmm6
+// CHECK: encoding: [0xc4,0xe3,0xc9,0x49,0x32,0x60]
+vpermil2pd $0, (%rdx), %xmm6, %xmm6, %xmm6
+
+// CHECK: vpermil2pd $0, (%rdx), %ymm7, %ymm7, %ymm7
+// CHECK: encoding: [0xc4,0xe3,0xc5,0x49,0x3a,0x70]
+vpermil2pd $0, (%rdx), %ymm7, %ymm7, %ymm7
+
+// CHECK: vpermil2pd $0, (%rdx), %ymm9, %ymm9, %ymm9
+// CHECK: encoding: [0xc4,0x63,0xb5,0x49,0x0a,0x90]
+vpermil2pd $0, (%rdx), %ymm9, %ymm9, %ymm9
+
+// CHECK: vpermil2pd $0, %xmm15, 485498096, %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x63,0x01,0x49,0x3c,0x25,0xf0,0x1c,0xf0,0x1c,0xf0]
+vpermil2pd $0, %xmm15, 485498096, %xmm15, %xmm15
+
+// CHECK: vpermil2pd $0, %xmm15, 64(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x63,0x01,0x49,0x7c,0x82,0x40,0xf0]
+vpermil2pd $0, %xmm15, 64(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vpermil2pd $0, %xmm15, -64(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x63,0x01,0x49,0x7c,0x82,0xc0,0xf0]
+vpermil2pd $0, %xmm15, -64(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vpermil2pd $0, %xmm15, 64(%rdx,%rax), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x63,0x01,0x49,0x7c,0x02,0x40,0xf0]
+vpermil2pd $0, %xmm15, 64(%rdx,%rax), %xmm15, %xmm15
+
+// CHECK: vpermil2pd $0, %xmm15, 64(%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x63,0x01,0x49,0x7a,0x40,0xf0]
+vpermil2pd $0, %xmm15, 64(%rdx), %xmm15, %xmm15
+
+// CHECK: vpermil2pd $0, %xmm15, (%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x63,0x01,0x49,0x3a,0xf0]
+vpermil2pd $0, %xmm15, (%rdx), %xmm15, %xmm15
+
+// CHECK: vpermil2pd $0, %xmm15, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x43,0x01,0x49,0xff,0xf0]
+vpermil2pd $0, %xmm15, %xmm15, %xmm15, %xmm15
+
+// CHECK: vpermil2pd $0, %xmm6, 485498096, %xmm6, %xmm6
+// CHECK: encoding: [0xc4,0xe3,0x49,0x49,0x34,0x25,0xf0,0x1c,0xf0,0x1c,0x60]
+vpermil2pd $0, %xmm6, 485498096, %xmm6, %xmm6
+
+// CHECK: vpermil2pd $0, %xmm6, 64(%rdx,%rax,4), %xmm6, %xmm6
+// CHECK: encoding: [0xc4,0xe3,0x49,0x49,0x74,0x82,0x40,0x60]
+vpermil2pd $0, %xmm6, 64(%rdx,%rax,4), %xmm6, %xmm6
+
+// CHECK: vpermil2pd $0, %xmm6, -64(%rdx,%rax,4), %xmm6, %xmm6
+// CHECK: encoding: [0xc4,0xe3,0x49,0x49,0x74,0x82,0xc0,0x60]
+vpermil2pd $0, %xmm6, -64(%rdx,%rax,4), %xmm6, %xmm6
+
+// CHECK: vpermil2pd $0, %xmm6, 64(%rdx,%rax), %xmm6, %xmm6
+// CHECK: encoding: [0xc4,0xe3,0x49,0x49,0x74,0x02,0x40,0x60]
+vpermil2pd $0, %xmm6, 64(%rdx,%rax), %xmm6, %xmm6
+
+// CHECK: vpermil2pd $0, %xmm6, 64(%rdx), %xmm6, %xmm6
+// CHECK: encoding: [0xc4,0xe3,0x49,0x49,0x72,0x40,0x60]
+vpermil2pd $0, %xmm6, 64(%rdx), %xmm6, %xmm6
+
+// CHECK: vpermil2pd $0, %xmm6, (%rdx), %xmm6, %xmm6
+// CHECK: encoding: [0xc4,0xe3,0x49,0x49,0x32,0x60]
+vpermil2pd $0, %xmm6, (%rdx), %xmm6, %xmm6
+
+// CHECK: vpermil2pd $0, %xmm6, %xmm6, %xmm6, %xmm6
+// CHECK: encoding: [0xc4,0xe3,0x49,0x49,0xf6,0x60]
+vpermil2pd $0, %xmm6, %xmm6, %xmm6, %xmm6
+
+// CHECK: vpermil2pd $0, %ymm7, 485498096, %ymm7, %ymm7
+// CHECK: encoding: [0xc4,0xe3,0x45,0x49,0x3c,0x25,0xf0,0x1c,0xf0,0x1c,0x70]
+vpermil2pd $0, %ymm7, 485498096, %ymm7, %ymm7
+
+// CHECK: vpermil2pd $0, %ymm7, 64(%rdx,%rax,4), %ymm7, %ymm7
+// CHECK: encoding: [0xc4,0xe3,0x45,0x49,0x7c,0x82,0x40,0x70]
+vpermil2pd $0, %ymm7, 64(%rdx,%rax,4), %ymm7, %ymm7
+
+// CHECK: vpermil2pd $0, %ymm7, -64(%rdx,%rax,4), %ymm7, %ymm7
+// CHECK: encoding: [0xc4,0xe3,0x45,0x49,0x7c,0x82,0xc0,0x70]
+vpermil2pd $0, %ymm7, -64(%rdx,%rax,4), %ymm7, %ymm7
+
+// CHECK: vpermil2pd $0, %ymm7, 64(%rdx,%rax), %ymm7, %ymm7
+// CHECK: encoding: [0xc4,0xe3,0x45,0x49,0x7c,0x02,0x40,0x70]
+vpermil2pd $0, %ymm7, 64(%rdx,%rax), %ymm7, %ymm7
+
+// CHECK: vpermil2pd $0, %ymm7, 64(%rdx), %ymm7, %ymm7
+// CHECK: encoding: [0xc4,0xe3,0x45,0x49,0x7a,0x40,0x70]
+vpermil2pd $0, %ymm7, 64(%rdx), %ymm7, %ymm7
+
+// CHECK: vpermil2pd $0, %ymm7, (%rdx), %ymm7, %ymm7
+// CHECK: encoding: [0xc4,0xe3,0x45,0x49,0x3a,0x70]
+vpermil2pd $0, %ymm7, (%rdx), %ymm7, %ymm7
+
+// CHECK: vpermil2pd $0, %ymm7, %ymm7, %ymm7, %ymm7
+// CHECK: encoding: [0xc4,0xe3,0x45,0x49,0xff,0x70]
+vpermil2pd $0, %ymm7, %ymm7, %ymm7, %ymm7
+
+// CHECK: vpermil2pd $0, %ymm9, 485498096, %ymm9, %ymm9
+// CHECK: encoding: [0xc4,0x63,0x35,0x49,0x0c,0x25,0xf0,0x1c,0xf0,0x1c,0x90]
+vpermil2pd $0, %ymm9, 485498096, %ymm9, %ymm9
+
+// CHECK: vpermil2pd $0, %ymm9, 64(%rdx,%rax,4), %ymm9, %ymm9
+// CHECK: encoding: [0xc4,0x63,0x35,0x49,0x4c,0x82,0x40,0x90]
+vpermil2pd $0, %ymm9, 64(%rdx,%rax,4), %ymm9, %ymm9
+
+// CHECK: vpermil2pd $0, %ymm9, -64(%rdx,%rax,4), %ymm9, %ymm9
+// CHECK: encoding: [0xc4,0x63,0x35,0x49,0x4c,0x82,0xc0,0x90]
+vpermil2pd $0, %ymm9, -64(%rdx,%rax,4), %ymm9, %ymm9
+
+// CHECK: vpermil2pd $0, %ymm9, 64(%rdx,%rax), %ymm9, %ymm9
+// CHECK: encoding: [0xc4,0x63,0x35,0x49,0x4c,0x02,0x40,0x90]
+vpermil2pd $0, %ymm9, 64(%rdx,%rax), %ymm9, %ymm9
+
+// CHECK: vpermil2pd $0, %ymm9, 64(%rdx), %ymm9, %ymm9
+// CHECK: encoding: [0xc4,0x63,0x35,0x49,0x4a,0x40,0x90]
+vpermil2pd $0, %ymm9, 64(%rdx), %ymm9, %ymm9
+
+// CHECK: vpermil2pd $0, %ymm9, (%rdx), %ymm9, %ymm9
+// CHECK: encoding: [0xc4,0x63,0x35,0x49,0x0a,0x90]
+vpermil2pd $0, %ymm9, (%rdx), %ymm9, %ymm9
+
+// CHECK: vpermil2pd $0, %ymm9, %ymm9, %ymm9, %ymm9
+// CHECK: encoding: [0xc4,0x43,0x35,0x49,0xc9,0x90]
+vpermil2pd $0, %ymm9, %ymm9, %ymm9, %ymm9
+
+// CHECK: vpermil2ps $0, 485498096, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x63,0x81,0x48,0x3c,0x25,0xf0,0x1c,0xf0,0x1c,0xf0]
+vpermil2ps $0, 485498096, %xmm15, %xmm15, %xmm15
+
+// CHECK: vpermil2ps $0, 485498096, %xmm6, %xmm6, %xmm6
+// CHECK: encoding: [0xc4,0xe3,0xc9,0x48,0x34,0x25,0xf0,0x1c,0xf0,0x1c,0x60]
+vpermil2ps $0, 485498096, %xmm6, %xmm6, %xmm6
+
+// CHECK: vpermil2ps $0, 485498096, %ymm7, %ymm7, %ymm7
+// CHECK: encoding: [0xc4,0xe3,0xc5,0x48,0x3c,0x25,0xf0,0x1c,0xf0,0x1c,0x70]
+vpermil2ps $0, 485498096, %ymm7, %ymm7, %ymm7
+
+// CHECK: vpermil2ps $0, 485498096, %ymm9, %ymm9, %ymm9
+// CHECK: encoding: [0xc4,0x63,0xb5,0x48,0x0c,0x25,0xf0,0x1c,0xf0,0x1c,0x90]
+vpermil2ps $0, 485498096, %ymm9, %ymm9, %ymm9
+
+// CHECK: vpermil2ps $0, 64(%rdx,%rax,4), %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x63,0x81,0x48,0x7c,0x82,0x40,0xf0]
+vpermil2ps $0, 64(%rdx,%rax,4), %xmm15, %xmm15, %xmm15
+
+// CHECK: vpermil2ps $0, -64(%rdx,%rax,4), %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x63,0x81,0x48,0x7c,0x82,0xc0,0xf0]
+vpermil2ps $0, -64(%rdx,%rax,4), %xmm15, %xmm15, %xmm15
+
+// CHECK: vpermil2ps $0, 64(%rdx,%rax,4), %xmm6, %xmm6, %xmm6
+// CHECK: encoding: [0xc4,0xe3,0xc9,0x48,0x74,0x82,0x40,0x60]
+vpermil2ps $0, 64(%rdx,%rax,4), %xmm6, %xmm6, %xmm6
+
+// CHECK: vpermil2ps $0, -64(%rdx,%rax,4), %xmm6, %xmm6, %xmm6
+// CHECK: encoding: [0xc4,0xe3,0xc9,0x48,0x74,0x82,0xc0,0x60]
+vpermil2ps $0, -64(%rdx,%rax,4), %xmm6, %xmm6, %xmm6
+
+// CHECK: vpermil2ps $0, 64(%rdx,%rax,4), %ymm7, %ymm7, %ymm7
+// CHECK: encoding: [0xc4,0xe3,0xc5,0x48,0x7c,0x82,0x40,0x70]
+vpermil2ps $0, 64(%rdx,%rax,4), %ymm7, %ymm7, %ymm7
+
+// CHECK: vpermil2ps $0, -64(%rdx,%rax,4), %ymm7, %ymm7, %ymm7
+// CHECK: encoding: [0xc4,0xe3,0xc5,0x48,0x7c,0x82,0xc0,0x70]
+vpermil2ps $0, -64(%rdx,%rax,4), %ymm7, %ymm7, %ymm7
+
+// CHECK: vpermil2ps $0, 64(%rdx,%rax,4), %ymm9, %ymm9, %ymm9
+// CHECK: encoding: [0xc4,0x63,0xb5,0x48,0x4c,0x82,0x40,0x90]
+vpermil2ps $0, 64(%rdx,%rax,4), %ymm9, %ymm9, %ymm9
+
+// CHECK: vpermil2ps $0, -64(%rdx,%rax,4), %ymm9, %ymm9, %ymm9
+// CHECK: encoding: [0xc4,0x63,0xb5,0x48,0x4c,0x82,0xc0,0x90]
+vpermil2ps $0, -64(%rdx,%rax,4), %ymm9, %ymm9, %ymm9
+
+// CHECK: vpermil2ps $0, 64(%rdx,%rax), %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x63,0x81,0x48,0x7c,0x02,0x40,0xf0]
+vpermil2ps $0, 64(%rdx,%rax), %xmm15, %xmm15, %xmm15
+
+// CHECK: vpermil2ps $0, 64(%rdx,%rax), %xmm6, %xmm6, %xmm6
+// CHECK: encoding: [0xc4,0xe3,0xc9,0x48,0x74,0x02,0x40,0x60]
+vpermil2ps $0, 64(%rdx,%rax), %xmm6, %xmm6, %xmm6
+
+// CHECK: vpermil2ps $0, 64(%rdx,%rax), %ymm7, %ymm7, %ymm7
+// CHECK: encoding: [0xc4,0xe3,0xc5,0x48,0x7c,0x02,0x40,0x70]
+vpermil2ps $0, 64(%rdx,%rax), %ymm7, %ymm7, %ymm7
+
+// CHECK: vpermil2ps $0, 64(%rdx,%rax), %ymm9, %ymm9, %ymm9
+// CHECK: encoding: [0xc4,0x63,0xb5,0x48,0x4c,0x02,0x40,0x90]
+vpermil2ps $0, 64(%rdx,%rax), %ymm9, %ymm9, %ymm9
+
+// CHECK: vpermil2ps $0, 64(%rdx), %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x63,0x81,0x48,0x7a,0x40,0xf0]
+vpermil2ps $0, 64(%rdx), %xmm15, %xmm15, %xmm15
+
+// CHECK: vpermil2ps $0, 64(%rdx), %xmm6, %xmm6, %xmm6
+// CHECK: encoding: [0xc4,0xe3,0xc9,0x48,0x72,0x40,0x60]
+vpermil2ps $0, 64(%rdx), %xmm6, %xmm6, %xmm6
+
+// CHECK: vpermil2ps $0, 64(%rdx), %ymm7, %ymm7, %ymm7
+// CHECK: encoding: [0xc4,0xe3,0xc5,0x48,0x7a,0x40,0x70]
+vpermil2ps $0, 64(%rdx), %ymm7, %ymm7, %ymm7
+
+// CHECK: vpermil2ps $0, 64(%rdx), %ymm9, %ymm9, %ymm9
+// CHECK: encoding: [0xc4,0x63,0xb5,0x48,0x4a,0x40,0x90]
+vpermil2ps $0, 64(%rdx), %ymm9, %ymm9, %ymm9
+
+// CHECK: vpermil2ps $0, (%rdx), %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x63,0x81,0x48,0x3a,0xf0]
+vpermil2ps $0, (%rdx), %xmm15, %xmm15, %xmm15
+
+// CHECK: vpermil2ps $0, (%rdx), %xmm6, %xmm6, %xmm6
+// CHECK: encoding: [0xc4,0xe3,0xc9,0x48,0x32,0x60]
+vpermil2ps $0, (%rdx), %xmm6, %xmm6, %xmm6
+
+// CHECK: vpermil2ps $0, (%rdx), %ymm7, %ymm7, %ymm7
+// CHECK: encoding: [0xc4,0xe3,0xc5,0x48,0x3a,0x70]
+vpermil2ps $0, (%rdx), %ymm7, %ymm7, %ymm7
+
+// CHECK: vpermil2ps $0, (%rdx), %ymm9, %ymm9, %ymm9
+// CHECK: encoding: [0xc4,0x63,0xb5,0x48,0x0a,0x90]
+vpermil2ps $0, (%rdx), %ymm9, %ymm9, %ymm9
+
+// CHECK: vpermil2ps $0, %xmm15, 485498096, %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x63,0x01,0x48,0x3c,0x25,0xf0,0x1c,0xf0,0x1c,0xf0]
+vpermil2ps $0, %xmm15, 485498096, %xmm15, %xmm15
+
+// CHECK: vpermil2ps $0, %xmm15, 64(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x63,0x01,0x48,0x7c,0x82,0x40,0xf0]
+vpermil2ps $0, %xmm15, 64(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vpermil2ps $0, %xmm15, -64(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x63,0x01,0x48,0x7c,0x82,0xc0,0xf0]
+vpermil2ps $0, %xmm15, -64(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vpermil2ps $0, %xmm15, 64(%rdx,%rax), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x63,0x01,0x48,0x7c,0x02,0x40,0xf0]
+vpermil2ps $0, %xmm15, 64(%rdx,%rax), %xmm15, %xmm15
+
+// CHECK: vpermil2ps $0, %xmm15, 64(%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x63,0x01,0x48,0x7a,0x40,0xf0]
+vpermil2ps $0, %xmm15, 64(%rdx), %xmm15, %xmm15
+
+// CHECK: vpermil2ps $0, %xmm15, (%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x63,0x01,0x48,0x3a,0xf0]
+vpermil2ps $0, %xmm15, (%rdx), %xmm15, %xmm15
+
+// CHECK: vpermil2ps $0, %xmm15, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0xc4,0x43,0x01,0x48,0xff,0xf0]
+vpermil2ps $0, %xmm15, %xmm15, %xmm15, %xmm15
+
+// CHECK: vpermil2ps $0, %xmm6, 485498096, %xmm6, %xmm6
+// CHECK: encoding: [0xc4,0xe3,0x49,0x48,0x34,0x25,0xf0,0x1c,0xf0,0x1c,0x60]
+vpermil2ps $0, %xmm6, 485498096, %xmm6, %xmm6
+
+// CHECK: vpermil2ps $0, %xmm6, 64(%rdx,%rax,4), %xmm6, %xmm6
+// CHECK: encoding: [0xc4,0xe3,0x49,0x48,0x74,0x82,0x40,0x60]
+vpermil2ps $0, %xmm6, 64(%rdx,%rax,4), %xmm6, %xmm6
+
+// CHECK: vpermil2ps $0, %xmm6, -64(%rdx,%rax,4), %xmm6, %xmm6
+// CHECK: encoding: [0xc4,0xe3,0x49,0x48,0x74,0x82,0xc0,0x60]
+vpermil2ps $0, %xmm6, -64(%rdx,%rax,4), %xmm6, %xmm6
+
+// CHECK: vpermil2ps $0, %xmm6, 64(%rdx,%rax), %xmm6, %xmm6
+// CHECK: encoding: [0xc4,0xe3,0x49,0x48,0x74,0x02,0x40,0x60]
+vpermil2ps $0, %xmm6, 64(%rdx,%rax), %xmm6, %xmm6
+
+// CHECK: vpermil2ps $0, %xmm6, 64(%rdx), %xmm6, %xmm6
+// CHECK: encoding: [0xc4,0xe3,0x49,0x48,0x72,0x40,0x60]
+vpermil2ps $0, %xmm6, 64(%rdx), %xmm6, %xmm6
+
+// CHECK: vpermil2ps $0, %xmm6, (%rdx), %xmm6, %xmm6
+// CHECK: encoding: [0xc4,0xe3,0x49,0x48,0x32,0x60]
+vpermil2ps $0, %xmm6, (%rdx), %xmm6, %xmm6
+
+// CHECK: vpermil2ps $0, %xmm6, %xmm6, %xmm6, %xmm6
+// CHECK: encoding: [0xc4,0xe3,0x49,0x48,0xf6,0x60]
+vpermil2ps $0, %xmm6, %xmm6, %xmm6, %xmm6
+
+// CHECK: vpermil2ps $0, %ymm7, 485498096, %ymm7, %ymm7
+// CHECK: encoding: [0xc4,0xe3,0x45,0x48,0x3c,0x25,0xf0,0x1c,0xf0,0x1c,0x70]
+vpermil2ps $0, %ymm7, 485498096, %ymm7, %ymm7
+
+// CHECK: vpermil2ps $0, %ymm7, 64(%rdx,%rax,4), %ymm7, %ymm7
+// CHECK: encoding: [0xc4,0xe3,0x45,0x48,0x7c,0x82,0x40,0x70]
+vpermil2ps $0, %ymm7, 64(%rdx,%rax,4), %ymm7, %ymm7
+
+// CHECK: vpermil2ps $0, %ymm7, -64(%rdx,%rax,4), %ymm7, %ymm7
+// CHECK: encoding: [0xc4,0xe3,0x45,0x48,0x7c,0x82,0xc0,0x70]
+vpermil2ps $0, %ymm7, -64(%rdx,%rax,4), %ymm7, %ymm7
+
+// CHECK: vpermil2ps $0, %ymm7, 64(%rdx,%rax), %ymm7, %ymm7
+// CHECK: encoding: [0xc4,0xe3,0x45,0x48,0x7c,0x02,0x40,0x70]
+vpermil2ps $0, %ymm7, 64(%rdx,%rax), %ymm7, %ymm7
+
+// CHECK: vpermil2ps $0, %ymm7, 64(%rdx), %ymm7, %ymm7
+// CHECK: encoding: [0xc4,0xe3,0x45,0x48,0x7a,0x40,0x70]
+vpermil2ps $0, %ymm7, 64(%rdx), %ymm7, %ymm7
+
+// CHECK: vpermil2ps $0, %ymm7, (%rdx), %ymm7, %ymm7
+// CHECK: encoding: [0xc4,0xe3,0x45,0x48,0x3a,0x70]
+vpermil2ps $0, %ymm7, (%rdx), %ymm7, %ymm7
+
+// CHECK: vpermil2ps $0, %ymm7, %ymm7, %ymm7, %ymm7
+// CHECK: encoding: [0xc4,0xe3,0x45,0x48,0xff,0x70]
+vpermil2ps $0, %ymm7, %ymm7, %ymm7, %ymm7
+
+// CHECK: vpermil2ps $0, %ymm9, 485498096, %ymm9, %ymm9
+// CHECK: encoding: [0xc4,0x63,0x35,0x48,0x0c,0x25,0xf0,0x1c,0xf0,0x1c,0x90]
+vpermil2ps $0, %ymm9, 485498096, %ymm9, %ymm9
+
+// CHECK: vpermil2ps $0, %ymm9, 64(%rdx,%rax,4), %ymm9, %ymm9
+// CHECK: encoding: [0xc4,0x63,0x35,0x48,0x4c,0x82,0x40,0x90]
+vpermil2ps $0, %ymm9, 64(%rdx,%rax,4), %ymm9, %ymm9
+
+// CHECK: vpermil2ps $0, %ymm9, -64(%rdx,%rax,4), %ymm9, %ymm9
+// CHECK: encoding: [0xc4,0x63,0x35,0x48,0x4c,0x82,0xc0,0x90]
+vpermil2ps $0, %ymm9, -64(%rdx,%rax,4), %ymm9, %ymm9
+
+// CHECK: vpermil2ps $0, %ymm9, 64(%rdx,%rax), %ymm9, %ymm9
+// CHECK: encoding: [0xc4,0x63,0x35,0x48,0x4c,0x02,0x40,0x90]
+vpermil2ps $0, %ymm9, 64(%rdx,%rax), %ymm9, %ymm9
+
+// CHECK: vpermil2ps $0, %ymm9, 64(%rdx), %ymm9, %ymm9
+// CHECK: encoding: [0xc4,0x63,0x35,0x48,0x4a,0x40,0x90]
+vpermil2ps $0, %ymm9, 64(%rdx), %ymm9, %ymm9
+
+// CHECK: vpermil2ps $0, %ymm9, (%rdx), %ymm9, %ymm9
+// CHECK: encoding: [0xc4,0x63,0x35,0x48,0x0a,0x90]
+vpermil2ps $0, %ymm9, (%rdx), %ymm9, %ymm9
+
+// CHECK: vpermil2ps $0, %ymm9, %ymm9, %ymm9, %ymm9
+// CHECK: encoding: [0xc4,0x43,0x35,0x48,0xc9,0x90]
+vpermil2ps $0, %ymm9, %ymm9, %ymm9, %ymm9
+
+// CHECK: vphaddbd 485498096, %xmm15
+// CHECK: encoding: [0x8f,0x69,0x78,0xc2,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vphaddbd 485498096, %xmm15
+
+// CHECK: vphaddbd 485498096, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0xc2,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+vphaddbd 485498096, %xmm6
+
+// CHECK: vphaddbd 64(%rdx,%rax,4), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x78,0xc2,0x7c,0x82,0x40]
+vphaddbd 64(%rdx,%rax,4), %xmm15
+
+// CHECK: vphaddbd -64(%rdx,%rax,4), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x78,0xc2,0x7c,0x82,0xc0]
+vphaddbd -64(%rdx,%rax,4), %xmm15
+
+// CHECK: vphaddbd 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0xc2,0x74,0x82,0x40]
+vphaddbd 64(%rdx,%rax,4), %xmm6
+
+// CHECK: vphaddbd -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0xc2,0x74,0x82,0xc0]
+vphaddbd -64(%rdx,%rax,4), %xmm6
+
+// CHECK: vphaddbd 64(%rdx,%rax), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x78,0xc2,0x7c,0x02,0x40]
+vphaddbd 64(%rdx,%rax), %xmm15
+
+// CHECK: vphaddbd 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0xc2,0x74,0x02,0x40]
+vphaddbd 64(%rdx,%rax), %xmm6
+
+// CHECK: vphaddbd 64(%rdx), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x78,0xc2,0x7a,0x40]
+vphaddbd 64(%rdx), %xmm15
+
+// CHECK: vphaddbd 64(%rdx), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0xc2,0x72,0x40]
+vphaddbd 64(%rdx), %xmm6
+
+// CHECK: vphaddbd (%rdx), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x78,0xc2,0x3a]
+vphaddbd (%rdx), %xmm15
+
+// CHECK: vphaddbd (%rdx), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0xc2,0x32]
+vphaddbd (%rdx), %xmm6
+
+// CHECK: vphaddbd %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x49,0x78,0xc2,0xff]
+vphaddbd %xmm15, %xmm15
+
+// CHECK: vphaddbd %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0xc2,0xf6]
+vphaddbd %xmm6, %xmm6
+
+// CHECK: vphaddbq 485498096, %xmm15
+// CHECK: encoding: [0x8f,0x69,0x78,0xc3,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vphaddbq 485498096, %xmm15
+
+// CHECK: vphaddbq 485498096, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0xc3,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+vphaddbq 485498096, %xmm6
+
+// CHECK: vphaddbq 64(%rdx,%rax,4), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x78,0xc3,0x7c,0x82,0x40]
+vphaddbq 64(%rdx,%rax,4), %xmm15
+
+// CHECK: vphaddbq -64(%rdx,%rax,4), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x78,0xc3,0x7c,0x82,0xc0]
+vphaddbq -64(%rdx,%rax,4), %xmm15
+
+// CHECK: vphaddbq 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0xc3,0x74,0x82,0x40]
+vphaddbq 64(%rdx,%rax,4), %xmm6
+
+// CHECK: vphaddbq -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0xc3,0x74,0x82,0xc0]
+vphaddbq -64(%rdx,%rax,4), %xmm6
+
+// CHECK: vphaddbq 64(%rdx,%rax), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x78,0xc3,0x7c,0x02,0x40]
+vphaddbq 64(%rdx,%rax), %xmm15
+
+// CHECK: vphaddbq 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0xc3,0x74,0x02,0x40]
+vphaddbq 64(%rdx,%rax), %xmm6
+
+// CHECK: vphaddbq 64(%rdx), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x78,0xc3,0x7a,0x40]
+vphaddbq 64(%rdx), %xmm15
+
+// CHECK: vphaddbq 64(%rdx), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0xc3,0x72,0x40]
+vphaddbq 64(%rdx), %xmm6
+
+// CHECK: vphaddbq (%rdx), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x78,0xc3,0x3a]
+vphaddbq (%rdx), %xmm15
+
+// CHECK: vphaddbq (%rdx), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0xc3,0x32]
+vphaddbq (%rdx), %xmm6
+
+// CHECK: vphaddbq %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x49,0x78,0xc3,0xff]
+vphaddbq %xmm15, %xmm15
+
+// CHECK: vphaddbq %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0xc3,0xf6]
+vphaddbq %xmm6, %xmm6
+
+// CHECK: vphaddbw 485498096, %xmm15
+// CHECK: encoding: [0x8f,0x69,0x78,0xc1,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vphaddbw 485498096, %xmm15
+
+// CHECK: vphaddbw 485498096, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0xc1,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+vphaddbw 485498096, %xmm6
+
+// CHECK: vphaddbw 64(%rdx,%rax,4), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x78,0xc1,0x7c,0x82,0x40]
+vphaddbw 64(%rdx,%rax,4), %xmm15
+
+// CHECK: vphaddbw -64(%rdx,%rax,4), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x78,0xc1,0x7c,0x82,0xc0]
+vphaddbw -64(%rdx,%rax,4), %xmm15
+
+// CHECK: vphaddbw 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0xc1,0x74,0x82,0x40]
+vphaddbw 64(%rdx,%rax,4), %xmm6
+
+// CHECK: vphaddbw -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0xc1,0x74,0x82,0xc0]
+vphaddbw -64(%rdx,%rax,4), %xmm6
+
+// CHECK: vphaddbw 64(%rdx,%rax), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x78,0xc1,0x7c,0x02,0x40]
+vphaddbw 64(%rdx,%rax), %xmm15
+
+// CHECK: vphaddbw 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0xc1,0x74,0x02,0x40]
+vphaddbw 64(%rdx,%rax), %xmm6
+
+// CHECK: vphaddbw 64(%rdx), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x78,0xc1,0x7a,0x40]
+vphaddbw 64(%rdx), %xmm15
+
+// CHECK: vphaddbw 64(%rdx), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0xc1,0x72,0x40]
+vphaddbw 64(%rdx), %xmm6
+
+// CHECK: vphaddbw (%rdx), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x78,0xc1,0x3a]
+vphaddbw (%rdx), %xmm15
+
+// CHECK: vphaddbw (%rdx), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0xc1,0x32]
+vphaddbw (%rdx), %xmm6
+
+// CHECK: vphaddbw %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x49,0x78,0xc1,0xff]
+vphaddbw %xmm15, %xmm15
+
+// CHECK: vphaddbw %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0xc1,0xf6]
+vphaddbw %xmm6, %xmm6
+
+// CHECK: vphadddq 485498096, %xmm15
+// CHECK: encoding: [0x8f,0x69,0x78,0xcb,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vphadddq 485498096, %xmm15
+
+// CHECK: vphadddq 485498096, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0xcb,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+vphadddq 485498096, %xmm6
+
+// CHECK: vphadddq 64(%rdx,%rax,4), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x78,0xcb,0x7c,0x82,0x40]
+vphadddq 64(%rdx,%rax,4), %xmm15
+
+// CHECK: vphadddq -64(%rdx,%rax,4), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x78,0xcb,0x7c,0x82,0xc0]
+vphadddq -64(%rdx,%rax,4), %xmm15
+
+// CHECK: vphadddq 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0xcb,0x74,0x82,0x40]
+vphadddq 64(%rdx,%rax,4), %xmm6
+
+// CHECK: vphadddq -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0xcb,0x74,0x82,0xc0]
+vphadddq -64(%rdx,%rax,4), %xmm6
+
+// CHECK: vphadddq 64(%rdx,%rax), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x78,0xcb,0x7c,0x02,0x40]
+vphadddq 64(%rdx,%rax), %xmm15
+
+// CHECK: vphadddq 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0xcb,0x74,0x02,0x40]
+vphadddq 64(%rdx,%rax), %xmm6
+
+// CHECK: vphadddq 64(%rdx), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x78,0xcb,0x7a,0x40]
+vphadddq 64(%rdx), %xmm15
+
+// CHECK: vphadddq 64(%rdx), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0xcb,0x72,0x40]
+vphadddq 64(%rdx), %xmm6
+
+// CHECK: vphadddq (%rdx), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x78,0xcb,0x3a]
+vphadddq (%rdx), %xmm15
+
+// CHECK: vphadddq (%rdx), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0xcb,0x32]
+vphadddq (%rdx), %xmm6
+
+// CHECK: vphadddq %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x49,0x78,0xcb,0xff]
+vphadddq %xmm15, %xmm15
+
+// CHECK: vphadddq %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0xcb,0xf6]
+vphadddq %xmm6, %xmm6
+
+// CHECK: vphaddubd 485498096, %xmm15
+// CHECK: encoding: [0x8f,0x69,0x78,0xd2,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vphaddubd 485498096, %xmm15
+
+// CHECK: vphaddubd 485498096, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0xd2,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+vphaddubd 485498096, %xmm6
+
+// CHECK: vphaddubd 64(%rdx,%rax,4), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x78,0xd2,0x7c,0x82,0x40]
+vphaddubd 64(%rdx,%rax,4), %xmm15
+
+// CHECK: vphaddubd -64(%rdx,%rax,4), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x78,0xd2,0x7c,0x82,0xc0]
+vphaddubd -64(%rdx,%rax,4), %xmm15
+
+// CHECK: vphaddubd 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0xd2,0x74,0x82,0x40]
+vphaddubd 64(%rdx,%rax,4), %xmm6
+
+// CHECK: vphaddubd -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0xd2,0x74,0x82,0xc0]
+vphaddubd -64(%rdx,%rax,4), %xmm6
+
+// CHECK: vphaddubd 64(%rdx,%rax), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x78,0xd2,0x7c,0x02,0x40]
+vphaddubd 64(%rdx,%rax), %xmm15
+
+// CHECK: vphaddubd 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0xd2,0x74,0x02,0x40]
+vphaddubd 64(%rdx,%rax), %xmm6
+
+// CHECK: vphaddubd 64(%rdx), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x78,0xd2,0x7a,0x40]
+vphaddubd 64(%rdx), %xmm15
+
+// CHECK: vphaddubd 64(%rdx), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0xd2,0x72,0x40]
+vphaddubd 64(%rdx), %xmm6
+
+// CHECK: vphaddubd (%rdx), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x78,0xd2,0x3a]
+vphaddubd (%rdx), %xmm15
+
+// CHECK: vphaddubd (%rdx), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0xd2,0x32]
+vphaddubd (%rdx), %xmm6
+
+// CHECK: vphaddubd %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x49,0x78,0xd2,0xff]
+vphaddubd %xmm15, %xmm15
+
+// CHECK: vphaddubd %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0xd2,0xf6]
+vphaddubd %xmm6, %xmm6
+
+// CHECK: vphaddubq 485498096, %xmm15
+// CHECK: encoding: [0x8f,0x69,0x78,0xd3,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vphaddubq 485498096, %xmm15
+
+// CHECK: vphaddubq 485498096, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0xd3,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+vphaddubq 485498096, %xmm6
+
+// CHECK: vphaddubq 64(%rdx,%rax,4), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x78,0xd3,0x7c,0x82,0x40]
+vphaddubq 64(%rdx,%rax,4), %xmm15
+
+// CHECK: vphaddubq -64(%rdx,%rax,4), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x78,0xd3,0x7c,0x82,0xc0]
+vphaddubq -64(%rdx,%rax,4), %xmm15
+
+// CHECK: vphaddubq 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0xd3,0x74,0x82,0x40]
+vphaddubq 64(%rdx,%rax,4), %xmm6
+
+// CHECK: vphaddubq -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0xd3,0x74,0x82,0xc0]
+vphaddubq -64(%rdx,%rax,4), %xmm6
+
+// CHECK: vphaddubq 64(%rdx,%rax), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x78,0xd3,0x7c,0x02,0x40]
+vphaddubq 64(%rdx,%rax), %xmm15
+
+// CHECK: vphaddubq 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0xd3,0x74,0x02,0x40]
+vphaddubq 64(%rdx,%rax), %xmm6
+
+// CHECK: vphaddubq 64(%rdx), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x78,0xd3,0x7a,0x40]
+vphaddubq 64(%rdx), %xmm15
+
+// CHECK: vphaddubq 64(%rdx), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0xd3,0x72,0x40]
+vphaddubq 64(%rdx), %xmm6
+
+// CHECK: vphaddubq (%rdx), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x78,0xd3,0x3a]
+vphaddubq (%rdx), %xmm15
+
+// CHECK: vphaddubq (%rdx), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0xd3,0x32]
+vphaddubq (%rdx), %xmm6
+
+// CHECK: vphaddubq %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x49,0x78,0xd3,0xff]
+vphaddubq %xmm15, %xmm15
+
+// CHECK: vphaddubq %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0xd3,0xf6]
+vphaddubq %xmm6, %xmm6
+
+// CHECK: vphaddubw 485498096, %xmm15
+// CHECK: encoding: [0x8f,0x69,0x78,0xd1,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vphaddubw 485498096, %xmm15
+
+// CHECK: vphaddubw 485498096, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0xd1,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+vphaddubw 485498096, %xmm6
+
+// CHECK: vphaddubw 64(%rdx,%rax,4), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x78,0xd1,0x7c,0x82,0x40]
+vphaddubw 64(%rdx,%rax,4), %xmm15
+
+// CHECK: vphaddubw -64(%rdx,%rax,4), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x78,0xd1,0x7c,0x82,0xc0]
+vphaddubw -64(%rdx,%rax,4), %xmm15
+
+// CHECK: vphaddubw 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0xd1,0x74,0x82,0x40]
+vphaddubw 64(%rdx,%rax,4), %xmm6
+
+// CHECK: vphaddubw -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0xd1,0x74,0x82,0xc0]
+vphaddubw -64(%rdx,%rax,4), %xmm6
+
+// CHECK: vphaddubw 64(%rdx,%rax), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x78,0xd1,0x7c,0x02,0x40]
+vphaddubw 64(%rdx,%rax), %xmm15
+
+// CHECK: vphaddubw 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0xd1,0x74,0x02,0x40]
+vphaddubw 64(%rdx,%rax), %xmm6
+
+// CHECK: vphaddubw 64(%rdx), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x78,0xd1,0x7a,0x40]
+vphaddubw 64(%rdx), %xmm15
+
+// CHECK: vphaddubw 64(%rdx), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0xd1,0x72,0x40]
+vphaddubw 64(%rdx), %xmm6
+
+// CHECK: vphaddubw (%rdx), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x78,0xd1,0x3a]
+vphaddubw (%rdx), %xmm15
+
+// CHECK: vphaddubw (%rdx), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0xd1,0x32]
+vphaddubw (%rdx), %xmm6
+
+// CHECK: vphaddubw %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x49,0x78,0xd1,0xff]
+vphaddubw %xmm15, %xmm15
+
+// CHECK: vphaddubw %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0xd1,0xf6]
+vphaddubw %xmm6, %xmm6
+
+// CHECK: vphaddudq 485498096, %xmm15
+// CHECK: encoding: [0x8f,0x69,0x78,0xdb,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vphaddudq 485498096, %xmm15
+
+// CHECK: vphaddudq 485498096, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0xdb,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+vphaddudq 485498096, %xmm6
+
+// CHECK: vphaddudq 64(%rdx,%rax,4), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x78,0xdb,0x7c,0x82,0x40]
+vphaddudq 64(%rdx,%rax,4), %xmm15
+
+// CHECK: vphaddudq -64(%rdx,%rax,4), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x78,0xdb,0x7c,0x82,0xc0]
+vphaddudq -64(%rdx,%rax,4), %xmm15
+
+// CHECK: vphaddudq 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0xdb,0x74,0x82,0x40]
+vphaddudq 64(%rdx,%rax,4), %xmm6
+
+// CHECK: vphaddudq -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0xdb,0x74,0x82,0xc0]
+vphaddudq -64(%rdx,%rax,4), %xmm6
+
+// CHECK: vphaddudq 64(%rdx,%rax), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x78,0xdb,0x7c,0x02,0x40]
+vphaddudq 64(%rdx,%rax), %xmm15
+
+// CHECK: vphaddudq 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0xdb,0x74,0x02,0x40]
+vphaddudq 64(%rdx,%rax), %xmm6
+
+// CHECK: vphaddudq 64(%rdx), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x78,0xdb,0x7a,0x40]
+vphaddudq 64(%rdx), %xmm15
+
+// CHECK: vphaddudq 64(%rdx), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0xdb,0x72,0x40]
+vphaddudq 64(%rdx), %xmm6
+
+// CHECK: vphaddudq (%rdx), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x78,0xdb,0x3a]
+vphaddudq (%rdx), %xmm15
+
+// CHECK: vphaddudq (%rdx), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0xdb,0x32]
+vphaddudq (%rdx), %xmm6
+
+// CHECK: vphaddudq %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x49,0x78,0xdb,0xff]
+vphaddudq %xmm15, %xmm15
+
+// CHECK: vphaddudq %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0xdb,0xf6]
+vphaddudq %xmm6, %xmm6
+
+// CHECK: vphadduwd 485498096, %xmm15
+// CHECK: encoding: [0x8f,0x69,0x78,0xd6,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vphadduwd 485498096, %xmm15
+
+// CHECK: vphadduwd 485498096, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0xd6,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+vphadduwd 485498096, %xmm6
+
+// CHECK: vphadduwd 64(%rdx,%rax,4), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x78,0xd6,0x7c,0x82,0x40]
+vphadduwd 64(%rdx,%rax,4), %xmm15
+
+// CHECK: vphadduwd -64(%rdx,%rax,4), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x78,0xd6,0x7c,0x82,0xc0]
+vphadduwd -64(%rdx,%rax,4), %xmm15
+
+// CHECK: vphadduwd 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0xd6,0x74,0x82,0x40]
+vphadduwd 64(%rdx,%rax,4), %xmm6
+
+// CHECK: vphadduwd -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0xd6,0x74,0x82,0xc0]
+vphadduwd -64(%rdx,%rax,4), %xmm6
+
+// CHECK: vphadduwd 64(%rdx,%rax), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x78,0xd6,0x7c,0x02,0x40]
+vphadduwd 64(%rdx,%rax), %xmm15
+
+// CHECK: vphadduwd 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0xd6,0x74,0x02,0x40]
+vphadduwd 64(%rdx,%rax), %xmm6
+
+// CHECK: vphadduwd 64(%rdx), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x78,0xd6,0x7a,0x40]
+vphadduwd 64(%rdx), %xmm15
+
+// CHECK: vphadduwd 64(%rdx), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0xd6,0x72,0x40]
+vphadduwd 64(%rdx), %xmm6
+
+// CHECK: vphadduwd (%rdx), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x78,0xd6,0x3a]
+vphadduwd (%rdx), %xmm15
+
+// CHECK: vphadduwd (%rdx), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0xd6,0x32]
+vphadduwd (%rdx), %xmm6
+
+// CHECK: vphadduwd %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x49,0x78,0xd6,0xff]
+vphadduwd %xmm15, %xmm15
+
+// CHECK: vphadduwd %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0xd6,0xf6]
+vphadduwd %xmm6, %xmm6
+
+// CHECK: vphadduwq 485498096, %xmm15
+// CHECK: encoding: [0x8f,0x69,0x78,0xd7,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vphadduwq 485498096, %xmm15
+
+// CHECK: vphadduwq 485498096, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0xd7,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+vphadduwq 485498096, %xmm6
+
+// CHECK: vphadduwq 64(%rdx,%rax,4), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x78,0xd7,0x7c,0x82,0x40]
+vphadduwq 64(%rdx,%rax,4), %xmm15
+
+// CHECK: vphadduwq -64(%rdx,%rax,4), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x78,0xd7,0x7c,0x82,0xc0]
+vphadduwq -64(%rdx,%rax,4), %xmm15
+
+// CHECK: vphadduwq 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0xd7,0x74,0x82,0x40]
+vphadduwq 64(%rdx,%rax,4), %xmm6
+
+// CHECK: vphadduwq -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0xd7,0x74,0x82,0xc0]
+vphadduwq -64(%rdx,%rax,4), %xmm6
+
+// CHECK: vphadduwq 64(%rdx,%rax), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x78,0xd7,0x7c,0x02,0x40]
+vphadduwq 64(%rdx,%rax), %xmm15
+
+// CHECK: vphadduwq 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0xd7,0x74,0x02,0x40]
+vphadduwq 64(%rdx,%rax), %xmm6
+
+// CHECK: vphadduwq 64(%rdx), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x78,0xd7,0x7a,0x40]
+vphadduwq 64(%rdx), %xmm15
+
+// CHECK: vphadduwq 64(%rdx), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0xd7,0x72,0x40]
+vphadduwq 64(%rdx), %xmm6
+
+// CHECK: vphadduwq (%rdx), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x78,0xd7,0x3a]
+vphadduwq (%rdx), %xmm15
+
+// CHECK: vphadduwq (%rdx), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0xd7,0x32]
+vphadduwq (%rdx), %xmm6
+
+// CHECK: vphadduwq %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x49,0x78,0xd7,0xff]
+vphadduwq %xmm15, %xmm15
+
+// CHECK: vphadduwq %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0xd7,0xf6]
+vphadduwq %xmm6, %xmm6
+
+// CHECK: vphaddwd 485498096, %xmm15
+// CHECK: encoding: [0x8f,0x69,0x78,0xc6,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vphaddwd 485498096, %xmm15
+
+// CHECK: vphaddwd 485498096, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0xc6,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+vphaddwd 485498096, %xmm6
+
+// CHECK: vphaddwd 64(%rdx,%rax,4), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x78,0xc6,0x7c,0x82,0x40]
+vphaddwd 64(%rdx,%rax,4), %xmm15
+
+// CHECK: vphaddwd -64(%rdx,%rax,4), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x78,0xc6,0x7c,0x82,0xc0]
+vphaddwd -64(%rdx,%rax,4), %xmm15
+
+// CHECK: vphaddwd 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0xc6,0x74,0x82,0x40]
+vphaddwd 64(%rdx,%rax,4), %xmm6
+
+// CHECK: vphaddwd -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0xc6,0x74,0x82,0xc0]
+vphaddwd -64(%rdx,%rax,4), %xmm6
+
+// CHECK: vphaddwd 64(%rdx,%rax), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x78,0xc6,0x7c,0x02,0x40]
+vphaddwd 64(%rdx,%rax), %xmm15
+
+// CHECK: vphaddwd 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0xc6,0x74,0x02,0x40]
+vphaddwd 64(%rdx,%rax), %xmm6
+
+// CHECK: vphaddwd 64(%rdx), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x78,0xc6,0x7a,0x40]
+vphaddwd 64(%rdx), %xmm15
+
+// CHECK: vphaddwd 64(%rdx), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0xc6,0x72,0x40]
+vphaddwd 64(%rdx), %xmm6
+
+// CHECK: vphaddwd (%rdx), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x78,0xc6,0x3a]
+vphaddwd (%rdx), %xmm15
+
+// CHECK: vphaddwd (%rdx), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0xc6,0x32]
+vphaddwd (%rdx), %xmm6
+
+// CHECK: vphaddwd %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x49,0x78,0xc6,0xff]
+vphaddwd %xmm15, %xmm15
+
+// CHECK: vphaddwd %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0xc6,0xf6]
+vphaddwd %xmm6, %xmm6
+
+// CHECK: vphaddwq 485498096, %xmm15
+// CHECK: encoding: [0x8f,0x69,0x78,0xc7,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vphaddwq 485498096, %xmm15
+
+// CHECK: vphaddwq 485498096, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0xc7,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+vphaddwq 485498096, %xmm6
+
+// CHECK: vphaddwq 64(%rdx,%rax,4), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x78,0xc7,0x7c,0x82,0x40]
+vphaddwq 64(%rdx,%rax,4), %xmm15
+
+// CHECK: vphaddwq -64(%rdx,%rax,4), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x78,0xc7,0x7c,0x82,0xc0]
+vphaddwq -64(%rdx,%rax,4), %xmm15
+
+// CHECK: vphaddwq 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0xc7,0x74,0x82,0x40]
+vphaddwq 64(%rdx,%rax,4), %xmm6
+
+// CHECK: vphaddwq -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0xc7,0x74,0x82,0xc0]
+vphaddwq -64(%rdx,%rax,4), %xmm6
+
+// CHECK: vphaddwq 64(%rdx,%rax), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x78,0xc7,0x7c,0x02,0x40]
+vphaddwq 64(%rdx,%rax), %xmm15
+
+// CHECK: vphaddwq 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0xc7,0x74,0x02,0x40]
+vphaddwq 64(%rdx,%rax), %xmm6
+
+// CHECK: vphaddwq 64(%rdx), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x78,0xc7,0x7a,0x40]
+vphaddwq 64(%rdx), %xmm15
+
+// CHECK: vphaddwq 64(%rdx), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0xc7,0x72,0x40]
+vphaddwq 64(%rdx), %xmm6
+
+// CHECK: vphaddwq (%rdx), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x78,0xc7,0x3a]
+vphaddwq (%rdx), %xmm15
+
+// CHECK: vphaddwq (%rdx), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0xc7,0x32]
+vphaddwq (%rdx), %xmm6
+
+// CHECK: vphaddwq %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x49,0x78,0xc7,0xff]
+vphaddwq %xmm15, %xmm15
+
+// CHECK: vphaddwq %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0xc7,0xf6]
+vphaddwq %xmm6, %xmm6
+
+// CHECK: vphsubbw 485498096, %xmm15
+// CHECK: encoding: [0x8f,0x69,0x78,0xe1,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vphsubbw 485498096, %xmm15
+
+// CHECK: vphsubbw 485498096, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0xe1,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+vphsubbw 485498096, %xmm6
+
+// CHECK: vphsubbw 64(%rdx,%rax,4), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x78,0xe1,0x7c,0x82,0x40]
+vphsubbw 64(%rdx,%rax,4), %xmm15
+
+// CHECK: vphsubbw -64(%rdx,%rax,4), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x78,0xe1,0x7c,0x82,0xc0]
+vphsubbw -64(%rdx,%rax,4), %xmm15
+
+// CHECK: vphsubbw 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0xe1,0x74,0x82,0x40]
+vphsubbw 64(%rdx,%rax,4), %xmm6
+
+// CHECK: vphsubbw -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0xe1,0x74,0x82,0xc0]
+vphsubbw -64(%rdx,%rax,4), %xmm6
+
+// CHECK: vphsubbw 64(%rdx,%rax), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x78,0xe1,0x7c,0x02,0x40]
+vphsubbw 64(%rdx,%rax), %xmm15
+
+// CHECK: vphsubbw 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0xe1,0x74,0x02,0x40]
+vphsubbw 64(%rdx,%rax), %xmm6
+
+// CHECK: vphsubbw 64(%rdx), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x78,0xe1,0x7a,0x40]
+vphsubbw 64(%rdx), %xmm15
+
+// CHECK: vphsubbw 64(%rdx), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0xe1,0x72,0x40]
+vphsubbw 64(%rdx), %xmm6
+
+// CHECK: vphsubbw (%rdx), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x78,0xe1,0x3a]
+vphsubbw (%rdx), %xmm15
+
+// CHECK: vphsubbw (%rdx), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0xe1,0x32]
+vphsubbw (%rdx), %xmm6
+
+// CHECK: vphsubbw %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x49,0x78,0xe1,0xff]
+vphsubbw %xmm15, %xmm15
+
+// CHECK: vphsubbw %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0xe1,0xf6]
+vphsubbw %xmm6, %xmm6
+
+// CHECK: vphsubdq 485498096, %xmm15
+// CHECK: encoding: [0x8f,0x69,0x78,0xe3,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vphsubdq 485498096, %xmm15
+
+// CHECK: vphsubdq 485498096, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0xe3,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+vphsubdq 485498096, %xmm6
+
+// CHECK: vphsubdq 64(%rdx,%rax,4), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x78,0xe3,0x7c,0x82,0x40]
+vphsubdq 64(%rdx,%rax,4), %xmm15
+
+// CHECK: vphsubdq -64(%rdx,%rax,4), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x78,0xe3,0x7c,0x82,0xc0]
+vphsubdq -64(%rdx,%rax,4), %xmm15
+
+// CHECK: vphsubdq 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0xe3,0x74,0x82,0x40]
+vphsubdq 64(%rdx,%rax,4), %xmm6
+
+// CHECK: vphsubdq -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0xe3,0x74,0x82,0xc0]
+vphsubdq -64(%rdx,%rax,4), %xmm6
+
+// CHECK: vphsubdq 64(%rdx,%rax), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x78,0xe3,0x7c,0x02,0x40]
+vphsubdq 64(%rdx,%rax), %xmm15
+
+// CHECK: vphsubdq 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0xe3,0x74,0x02,0x40]
+vphsubdq 64(%rdx,%rax), %xmm6
+
+// CHECK: vphsubdq 64(%rdx), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x78,0xe3,0x7a,0x40]
+vphsubdq 64(%rdx), %xmm15
+
+// CHECK: vphsubdq 64(%rdx), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0xe3,0x72,0x40]
+vphsubdq 64(%rdx), %xmm6
+
+// CHECK: vphsubdq (%rdx), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x78,0xe3,0x3a]
+vphsubdq (%rdx), %xmm15
+
+// CHECK: vphsubdq (%rdx), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0xe3,0x32]
+vphsubdq (%rdx), %xmm6
+
+// CHECK: vphsubdq %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x49,0x78,0xe3,0xff]
+vphsubdq %xmm15, %xmm15
+
+// CHECK: vphsubdq %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0xe3,0xf6]
+vphsubdq %xmm6, %xmm6
+
+// CHECK: vphsubwd 485498096, %xmm15
+// CHECK: encoding: [0x8f,0x69,0x78,0xe2,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vphsubwd 485498096, %xmm15
+
+// CHECK: vphsubwd 485498096, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0xe2,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+vphsubwd 485498096, %xmm6
+
+// CHECK: vphsubwd 64(%rdx,%rax,4), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x78,0xe2,0x7c,0x82,0x40]
+vphsubwd 64(%rdx,%rax,4), %xmm15
+
+// CHECK: vphsubwd -64(%rdx,%rax,4), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x78,0xe2,0x7c,0x82,0xc0]
+vphsubwd -64(%rdx,%rax,4), %xmm15
+
+// CHECK: vphsubwd 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0xe2,0x74,0x82,0x40]
+vphsubwd 64(%rdx,%rax,4), %xmm6
+
+// CHECK: vphsubwd -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0xe2,0x74,0x82,0xc0]
+vphsubwd -64(%rdx,%rax,4), %xmm6
+
+// CHECK: vphsubwd 64(%rdx,%rax), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x78,0xe2,0x7c,0x02,0x40]
+vphsubwd 64(%rdx,%rax), %xmm15
+
+// CHECK: vphsubwd 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0xe2,0x74,0x02,0x40]
+vphsubwd 64(%rdx,%rax), %xmm6
+
+// CHECK: vphsubwd 64(%rdx), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x78,0xe2,0x7a,0x40]
+vphsubwd 64(%rdx), %xmm15
+
+// CHECK: vphsubwd 64(%rdx), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0xe2,0x72,0x40]
+vphsubwd 64(%rdx), %xmm6
+
+// CHECK: vphsubwd (%rdx), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x78,0xe2,0x3a]
+vphsubwd (%rdx), %xmm15
+
+// CHECK: vphsubwd (%rdx), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0xe2,0x32]
+vphsubwd (%rdx), %xmm6
+
+// CHECK: vphsubwd %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x49,0x78,0xe2,0xff]
+vphsubwd %xmm15, %xmm15
+
+// CHECK: vphsubwd %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x78,0xe2,0xf6]
+vphsubwd %xmm6, %xmm6
+
+// CHECK: vpmacsdd %xmm15, 485498096, %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0x9e,0x3c,0x25,0xf0,0x1c,0xf0,0x1c,0xf0]
+vpmacsdd %xmm15, 485498096, %xmm15, %xmm15
+
+// CHECK: vpmacsdd %xmm15, -64(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0x
+vpmacsdd %xmm15, -64(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vpmacsdd %xmm15, 64(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0x9e,0x7c,0x82,0x40,0xf0]
+vpmacsdd %xmm15, 64(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vpmacsdd %xmm15, -64(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0x9e,0x7c,0x82,0xc0,0xf0]
+vpmacsdd %xmm15, -64(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vpmacsdd %xmm15, 64(%rdx,%rax), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0x9e,0x7c,0x02,0x40,0xf0]
+vpmacsdd %xmm15, 64(%rdx,%rax), %xmm15, %xmm15
+
+// CHECK: vpmacsdd %xmm15, 64(%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0x9e,0x7a,0x40,0xf0]
+vpmacsdd %xmm15, 64(%rdx), %xmm15, %xmm15
+
+// CHECK: vpmacsdd %xmm15, (%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0x9e,0x3a,0xf0]
+vpmacsdd %xmm15, (%rdx), %xmm15, %xmm15
+
+// CHECK: vpmacsdd %xmm15, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x48,0x00,0x9e,0xff,0xf0]
+vpmacsdd %xmm15, %xmm15, %xmm15, %xmm15
+
+// CHECK: vpmacsdd %xmm6, 485498096, %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0x9e,0x34,0x25,0xf0,0x1c,0xf0,0x1c,0x60]
+vpmacsdd %xmm6, 485498096, %xmm6, %xmm6
+
+// CHECK: vpmacsdd %xmm6, 64(%rdx,%rax,4), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0x9e,0x74,0x82,0x40,0x60]
+vpmacsdd %xmm6, 64(%rdx,%rax,4), %xmm6, %xmm6
+
+// CHECK: vpmacsdd %xmm6, -64(%rdx,%rax,4), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0x9e,0x74,0x82,0xc0,0x60]
+vpmacsdd %xmm6, -64(%rdx,%rax,4), %xmm6, %xmm6
+
+// CHECK: vpmacsdd %xmm6, 64(%rdx,%rax), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0x9e,0x74,0x02,0x40,0x60]
+vpmacsdd %xmm6, 64(%rdx,%rax), %xmm6, %xmm6
+
+// CHECK: vpmacsdd %xmm6, 64(%rdx), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0x9e,0x72,0x40,0x60]
+vpmacsdd %xmm6, 64(%rdx), %xmm6, %xmm6
+
+// CHECK: vpmacsdd %xmm6, (%rdx), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0x9e,0x32,0x60]
+vpmacsdd %xmm6, (%rdx), %xmm6, %xmm6
+
+// CHECK: vpmacsdd %xmm6, %xmm6, %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0x9e,0xf6,0x60]
+vpmacsdd %xmm6, %xmm6, %xmm6, %xmm6
+
+// CHECK: vpmacsdqh %xmm15, 485498096, %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0x9f,0x3c,0x25,0xf0,0x1c,0xf0,0x1c,0xf0]
+vpmacsdqh %xmm15, 485498096, %xmm15, %xmm15
+
+// CHECK: vpmacsdqh %xmm15, 64(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0x9f,0x7c,0x82,0x40,0xf0]
+vpmacsdqh %xmm15, 64(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vpmacsdqh %xmm15, -64(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0x9f,0x7c,0x82,0xc0,0xf0]
+vpmacsdqh %xmm15, -64(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vpmacsdqh %xmm15, 64(%rdx,%rax), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0x9f,0x7c,0x02,0x40,0xf0]
+vpmacsdqh %xmm15, 64(%rdx,%rax), %xmm15, %xmm15
+
+// CHECK: vpmacsdqh %xmm15, 64(%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0x9f,0x7a,0x40,0xf0]
+vpmacsdqh %xmm15, 64(%rdx), %xmm15, %xmm15
+
+// CHECK: vpmacsdqh %xmm15, (%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0x9f,0x3a,0xf0]
+vpmacsdqh %xmm15, (%rdx), %xmm15, %xmm15
+
+// CHECK: vpmacsdqh %xmm15, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x48,0x00,0x9f,0xff,0xf0]
+vpmacsdqh %xmm15, %xmm15, %xmm15, %xmm15
+
+// CHECK: vpmacsdqh %xmm6, 485498096, %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0x9f,0x34,0x25,0xf0,0x1c,0xf0,0x1c,0x60]
+vpmacsdqh %xmm6, 485498096, %xmm6, %xmm6
+
+// CHECK: vpmacsdqh %xmm6, 64(%rdx,%rax,4), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0x9f,0x74,0x82,0x40,0x60]
+vpmacsdqh %xmm6, 64(%rdx,%rax,4), %xmm6, %xmm6
+
+// CHECK: vpmacsdqh %xmm6, -64(%rdx,%rax,4), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0x9f,0x74,0x82,0xc0,0x60]
+vpmacsdqh %xmm6, -64(%rdx,%rax,4), %xmm6, %xmm6
+
+// CHECK: vpmacsdqh %xmm6, 64(%rdx,%rax), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0x9f,0x74,0x02,0x40,0x60]
+vpmacsdqh %xmm6, 64(%rdx,%rax), %xmm6, %xmm6
+
+// CHECK: vpmacsdqh %xmm6, 64(%rdx), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0x9f,0x72,0x40,0x60]
+vpmacsdqh %xmm6, 64(%rdx), %xmm6, %xmm6
+
+// CHECK: vpmacsdqh %xmm6, (%rdx), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0x9f,0x32,0x60]
+vpmacsdqh %xmm6, (%rdx), %xmm6, %xmm6
+
+// CHECK: vpmacsdqh %xmm6, %xmm6, %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0x9f,0xf6,0x60]
+vpmacsdqh %xmm6, %xmm6, %xmm6, %xmm6
+
+// CHECK: vpmacsdql %xmm15, 485498096, %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0x97,0x3c,0x25,0xf0,0x1c,0xf0,0x1c,0xf0]
+vpmacsdql %xmm15, 485498096, %xmm15, %xmm15
+
+// CHECK: vpmacsdql %xmm15, 64(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0x97,0x7c,0x82,0x40,0xf0]
+vpmacsdql %xmm15, 64(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vpmacsdql %xmm15, -64(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0x97,0x7c,0x82,0xc0,0xf0]
+vpmacsdql %xmm15, -64(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vpmacsdql %xmm15, 64(%rdx,%rax), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0x97,0x7c,0x02,0x40,0xf0]
+vpmacsdql %xmm15, 64(%rdx,%rax), %xmm15, %xmm15
+
+// CHECK: vpmacsdql %xmm15, 64(%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0x97,0x7a,0x40,0xf0]
+vpmacsdql %xmm15, 64(%rdx), %xmm15, %xmm15
+
+// CHECK: vpmacsdql %xmm15, (%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0x97,0x3a,0xf0]
+vpmacsdql %xmm15, (%rdx), %xmm15, %xmm15
+
+// CHECK: vpmacsdql %xmm15, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x48,0x00,0x97,0xff,0xf0]
+vpmacsdql %xmm15, %xmm15, %xmm15, %xmm15
+
+// CHECK: vpmacsdql %xmm6, 485498096, %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0x97,0x34,0x25,0xf0,0x1c,0xf0,0x1c,0x60]
+vpmacsdql %xmm6, 485498096, %xmm6, %xmm6
+
+// CHECK: vpmacsdql %xmm6, 64(%rdx,%rax,4), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0x97,0x74,0x82,0x40,0x60]
+vpmacsdql %xmm6, 64(%rdx,%rax,4), %xmm6, %xmm6
+
+// CHECK: vpmacsdql %xmm6, -64(%rdx,%rax,4), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0x97,0x74,0x82,0xc0,0x60]
+vpmacsdql %xmm6, -64(%rdx,%rax,4), %xmm6, %xmm6
+
+// CHECK: vpmacsdql %xmm6, 64(%rdx,%rax), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0x97,0x74,0x02,0x40,0x60]
+vpmacsdql %xmm6, 64(%rdx,%rax), %xmm6, %xmm6
+
+// CHECK: vpmacsdql %xmm6, 64(%rdx), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0x97,0x72,0x40,0x60]
+vpmacsdql %xmm6, 64(%rdx), %xmm6, %xmm6
+
+// CHECK: vpmacsdql %xmm6, (%rdx), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0x97,0x32,0x60]
+vpmacsdql %xmm6, (%rdx), %xmm6, %xmm6
+
+// CHECK: vpmacsdql %xmm6, %xmm6, %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0x97,0xf6,0x60]
+vpmacsdql %xmm6, %xmm6, %xmm6, %xmm6
+
+// CHECK: vpmacssdd %xmm15, 485498096, %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0x8e,0x3c,0x25,0xf0,0x1c,0xf0,0x1c,0xf0]
+vpmacssdd %xmm15, 485498096, %xmm15, %xmm15
+
+// CHECK: vpmacssdd %xmm15, 64(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0x8e,0x7c,0x82,0x40,0xf0]
+vpmacssdd %xmm15, 64(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vpmacssdd %xmm15, -64(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0x8e,0x7c,0x82,0xc0,0xf0]
+vpmacssdd %xmm15, -64(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vpmacssdd %xmm15, 64(%rdx,%rax), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0x8e,0x7c,0x02,0x40,0xf0]
+vpmacssdd %xmm15, 64(%rdx,%rax), %xmm15, %xmm15
+
+// CHECK: vpmacssdd %xmm15, 64(%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0x8e,0x7a,0x40,0xf0]
+vpmacssdd %xmm15, 64(%rdx), %xmm15, %xmm15
+
+// CHECK: vpmacssdd %xmm15, (%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0x8e,0x3a,0xf0]
+vpmacssdd %xmm15, (%rdx), %xmm15, %xmm15
+
+// CHECK: vpmacssdd %xmm15, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x48,0x00,0x8e,0xff,0xf0]
+vpmacssdd %xmm15, %xmm15, %xmm15, %xmm15
+
+// CHECK: vpmacssdd %xmm6, 485498096, %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0x8e,0x34,0x25,0xf0,0x1c,0xf0,0x1c,0x60]
+vpmacssdd %xmm6, 485498096, %xmm6, %xmm6
+
+// CHECK: vpmacssdd %xmm6, 64(%rdx,%rax,4), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0x8e,0x74,0x82,0x40,0x60]
+vpmacssdd %xmm6, 64(%rdx,%rax,4), %xmm6, %xmm6
+
+// CHECK: vpmacssdd %xmm6, -64(%rdx,%rax,4), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0x8e,0x74,0x82,0xc0,0x60]
+vpmacssdd %xmm6, -64(%rdx,%rax,4), %xmm6, %xmm6
+
+// CHECK: vpmacssdd %xmm6, 64(%rdx,%rax), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0x8e,0x74,0x02,0x40,0x60]
+vpmacssdd %xmm6, 64(%rdx,%rax), %xmm6, %xmm6
+
+// CHECK: vpmacssdd %xmm6, 64(%rdx), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0x8e,0x72,0x40,0x60]
+vpmacssdd %xmm6, 64(%rdx), %xmm6, %xmm6
+
+// CHECK: vpmacssdd %xmm6, (%rdx), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0x8e,0x32,0x60]
+vpmacssdd %xmm6, (%rdx), %xmm6, %xmm6
+
+// CHECK: vpmacssdd %xmm6, %xmm6, %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0x8e,0xf6,0x60]
+vpmacssdd %xmm6, %xmm6, %xmm6, %xmm6
+
+// CHECK: vpmacssdqh %xmm15, 485498096, %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0x8f,0x3c,0x25,0xf0,0x1c,0xf0,0x1c,0xf0]
+vpmacssdqh %xmm15, 485498096, %xmm15, %xmm15
+
+// CHECK: vpmacssdqh %xmm15, 64(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0x8f,0x7c,0x82,0x40,0xf0]
+vpmacssdqh %xmm15, 64(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vpmacssdqh %xmm15, -64(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0x8f,0x7c,0x82,0xc0,0xf0]
+vpmacssdqh %xmm15, -64(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vpmacssdqh %xmm15, 64(%rdx,%rax), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0x8f,0x7c,0x02,0x40,0xf0]
+vpmacssdqh %xmm15, 64(%rdx,%rax), %xmm15, %xmm15
+
+// CHECK: vpmacssdqh %xmm15, 64(%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0x8f,0x7a,0x40,0xf0]
+vpmacssdqh %xmm15, 64(%rdx), %xmm15, %xmm15
+
+// CHECK: vpmacssdqh %xmm15, (%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0x8f,0x3a,0xf0]
+vpmacssdqh %xmm15, (%rdx), %xmm15, %xmm15
+
+// CHECK: vpmacssdqh %xmm15, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x48,0x00,0x8f,0xff,0xf0]
+vpmacssdqh %xmm15, %xmm15, %xmm15, %xmm15
+
+// CHECK: vpmacssdqh %xmm6, 485498096, %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0x8f,0x34,0x25,0xf0,0x1c,0xf0,0x1c,0x60]
+vpmacssdqh %xmm6, 485498096, %xmm6, %xmm6
+
+// CHECK: vpmacssdqh %xmm6, 64(%rdx,%rax,4), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0x8f,0x74,0x82,0x40,0x60]
+vpmacssdqh %xmm6, 64(%rdx,%rax,4), %xmm6, %xmm6
+
+// CHECK: vpmacssdqh %xmm6, -64(%rdx,%rax,4), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0x8f,0x74,0x82,0xc0,0x60]
+vpmacssdqh %xmm6, -64(%rdx,%rax,4), %xmm6, %xmm6
+
+// CHECK: vpmacssdqh %xmm6, 64(%rdx,%rax), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0x8f,0x74,0x02,0x40,0x60]
+vpmacssdqh %xmm6, 64(%rdx,%rax), %xmm6, %xmm6
+
+// CHECK: vpmacssdqh %xmm6, 64(%rdx), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0x8f,0x72,0x40,0x60]
+vpmacssdqh %xmm6, 64(%rdx), %xmm6, %xmm6
+
+// CHECK: vpmacssdqh %xmm6, (%rdx), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0x8f,0x32,0x60]
+vpmacssdqh %xmm6, (%rdx), %xmm6, %xmm6
+
+// CHECK: vpmacssdqh %xmm6, %xmm6, %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0x8f,0xf6,0x60]
+vpmacssdqh %xmm6, %xmm6, %xmm6, %xmm6
+
+// CHECK: vpmacssdql %xmm15, 485498096, %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0x87,0x3c,0x25,0xf0,0x1c,0xf0,0x1c,0xf0]
+vpmacssdql %xmm15, 485498096, %xmm15, %xmm15
+
+// CHECK: vpmacssdql %xmm15, 64(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0x87,0x7c,0x82,0x40,0xf0]
+vpmacssdql %xmm15, 64(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vpmacssdql %xmm15, -64(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0x87,0x7c,0x82,0xc0,0xf0]
+vpmacssdql %xmm15, -64(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vpmacssdql %xmm15, 64(%rdx,%rax), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0x87,0x7c,0x02,0x40,0xf0]
+vpmacssdql %xmm15, 64(%rdx,%rax), %xmm15, %xmm15
+
+// CHECK: vpmacssdql %xmm15, 64(%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0x87,0x7a,0x40,0xf0]
+vpmacssdql %xmm15, 64(%rdx), %xmm15, %xmm15
+
+// CHECK: vpmacssdql %xmm15, (%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0x87,0x3a,0xf0]
+vpmacssdql %xmm15, (%rdx), %xmm15, %xmm15
+
+// CHECK: vpmacssdql %xmm15, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x48,0x00,0x87,0xff,0xf0]
+vpmacssdql %xmm15, %xmm15, %xmm15, %xmm15
+
+// CHECK: vpmacssdql %xmm6, 485498096, %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0x87,0x34,0x25,0xf0,0x1c,0xf0,0x1c,0x60]
+vpmacssdql %xmm6, 485498096, %xmm6, %xmm6
+
+// CHECK: vpmacssdql %xmm6, 64(%rdx,%rax,4), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0x87,0x74,0x82,0x40,0x60]
+vpmacssdql %xmm6, 64(%rdx,%rax,4), %xmm6, %xmm6
+
+// CHECK: vpmacssdql %xmm6, -64(%rdx,%rax,4), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0x87,0x74,0x82,0xc0,0x60]
+vpmacssdql %xmm6, -64(%rdx,%rax,4), %xmm6, %xmm6
+
+// CHECK: vpmacssdql %xmm6, 64(%rdx,%rax), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0x87,0x74,0x02,0x40,0x60]
+vpmacssdql %xmm6, 64(%rdx,%rax), %xmm6, %xmm6
+
+// CHECK: vpmacssdql %xmm6, 64(%rdx), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0x87,0x72,0x40,0x60]
+vpmacssdql %xmm6, 64(%rdx), %xmm6, %xmm6
+
+// CHECK: vpmacssdql %xmm6, (%rdx), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0x87,0x32,0x60]
+vpmacssdql %xmm6, (%rdx), %xmm6, %xmm6
+
+// CHECK: vpmacssdql %xmm6, %xmm6, %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0x87,0xf6,0x60]
+vpmacssdql %xmm6, %xmm6, %xmm6, %xmm6
+
+// CHECK: vpmacsswd %xmm15, 485498096, %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0x86,0x3c,0x25,0xf0,0x1c,0xf0,0x1c,0xf0]
+vpmacsswd %xmm15, 485498096, %xmm15, %xmm15
+
+// CHECK: vpmacsswd %xmm15, 64(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0x86,0x7c,0x82,0x40,0xf0]
+vpmacsswd %xmm15, 64(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vpmacsswd %xmm15, -64(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0x86,0x7c,0x82,0xc0,0xf0]
+vpmacsswd %xmm15, -64(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vpmacsswd %xmm15, 64(%rdx,%rax), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0x86,0x7c,0x02,0x40,0xf0]
+vpmacsswd %xmm15, 64(%rdx,%rax), %xmm15, %xmm15
+
+// CHECK: vpmacsswd %xmm15, 64(%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0x86,0x7a,0x40,0xf0]
+vpmacsswd %xmm15, 64(%rdx), %xmm15, %xmm15
+
+// CHECK: vpmacsswd %xmm15, (%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0x86,0x3a,0xf0]
+vpmacsswd %xmm15, (%rdx), %xmm15, %xmm15
+
+// CHECK: vpmacsswd %xmm15, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x48,0x00,0x86,0xff,0xf0]
+vpmacsswd %xmm15, %xmm15, %xmm15, %xmm15
+
+// CHECK: vpmacsswd %xmm6, 485498096, %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0x86,0x34,0x25,0xf0,0x1c,0xf0,0x1c,0x60]
+vpmacsswd %xmm6, 485498096, %xmm6, %xmm6
+
+// CHECK: vpmacsswd %xmm6, 64(%rdx,%rax,4), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0x86,0x74,0x82,0x40,0x60]
+vpmacsswd %xmm6, 64(%rdx,%rax,4), %xmm6, %xmm6
+
+// CHECK: vpmacsswd %xmm6, -64(%rdx,%rax,4), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0x86,0x74,0x82,0xc0,0x60]
+vpmacsswd %xmm6, -64(%rdx,%rax,4), %xmm6, %xmm6
+
+// CHECK: vpmacsswd %xmm6, 64(%rdx,%rax), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0x86,0x74,0x02,0x40,0x60]
+vpmacsswd %xmm6, 64(%rdx,%rax), %xmm6, %xmm6
+
+// CHECK: vpmacsswd %xmm6, 64(%rdx), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0x86,0x72,0x40,0x60]
+vpmacsswd %xmm6, 64(%rdx), %xmm6, %xmm6
+
+// CHECK: vpmacsswd %xmm6, (%rdx), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0x86,0x32,0x60]
+vpmacsswd %xmm6, (%rdx), %xmm6, %xmm6
+
+// CHECK: vpmacsswd %xmm6, %xmm6, %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0x86,0xf6,0x60]
+vpmacsswd %xmm6, %xmm6, %xmm6, %xmm6
+
+// CHECK: vpmacssww %xmm15, 485498096, %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0x85,0x3c,0x25,0xf0,0x1c,0xf0,0x1c,0xf0]
+vpmacssww %xmm15, 485498096, %xmm15, %xmm15
+
+// CHECK: vpmacssww %xmm15, 64(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0x85,0x7c,0x82,0x40,0xf0]
+vpmacssww %xmm15, 64(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vpmacssww %xmm15, -64(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0x85,0x7c,0x82,0xc0,0xf0]
+vpmacssww %xmm15, -64(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vpmacssww %xmm15, 64(%rdx,%rax), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0x85,0x7c,0x02,0x40,0xf0]
+vpmacssww %xmm15, 64(%rdx,%rax), %xmm15, %xmm15
+
+// CHECK: vpmacssww %xmm15, 64(%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0x85,0x7a,0x40,0xf0]
+vpmacssww %xmm15, 64(%rdx), %xmm15, %xmm15
+
+// CHECK: vpmacssww %xmm15, (%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0x85,0x3a,0xf0]
+vpmacssww %xmm15, (%rdx), %xmm15, %xmm15
+
+// CHECK: vpmacssww %xmm15, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x48,0x00,0x85,0xff,0xf0]
+vpmacssww %xmm15, %xmm15, %xmm15, %xmm15
+
+// CHECK: vpmacssww %xmm6, 485498096, %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0x85,0x34,0x25,0xf0,0x1c,0xf0,0x1c,0x60]
+vpmacssww %xmm6, 485498096, %xmm6, %xmm6
+
+// CHECK: vpmacssww %xmm6, 64(%rdx,%rax,4), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0x85,0x74,0x82,0x40,0x60]
+vpmacssww %xmm6, 64(%rdx,%rax,4), %xmm6, %xmm6
+
+// CHECK: vpmacssww %xmm6, -64(%rdx,%rax,4), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0x85,0x74,0x82,0xc0,0x60]
+vpmacssww %xmm6, -64(%rdx,%rax,4), %xmm6, %xmm6
+
+// CHECK: vpmacssww %xmm6, 64(%rdx,%rax), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0x85,0x74,0x02,0x40,0x60]
+vpmacssww %xmm6, 64(%rdx,%rax), %xmm6, %xmm6
+
+// CHECK: vpmacssww %xmm6, 64(%rdx), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0x85,0x72,0x40,0x60]
+vpmacssww %xmm6, 64(%rdx), %xmm6, %xmm6
+
+// CHECK: vpmacssww %xmm6, (%rdx), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0x85,0x32,0x60]
+vpmacssww %xmm6, (%rdx), %xmm6, %xmm6
+
+// CHECK: vpmacssww %xmm6, %xmm6, %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0x85,0xf6,0x60]
+vpmacssww %xmm6, %xmm6, %xmm6, %xmm6
+
+// CHECK: vpmacswd %xmm15, 485498096, %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0x96,0x3c,0x25,0xf0,0x1c,0xf0,0x1c,0xf0]
+vpmacswd %xmm15, 485498096, %xmm15, %xmm15
+
+// CHECK: vpmacswd %xmm15, 64(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0x96,0x7c,0x82,0x40,0xf0]
+vpmacswd %xmm15, 64(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vpmacswd %xmm15, -64(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0x96,0x7c,0x82,0xc0,0xf0]
+vpmacswd %xmm15, -64(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vpmacswd %xmm15, 64(%rdx,%rax), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0x96,0x7c,0x02,0x40,0xf0]
+vpmacswd %xmm15, 64(%rdx,%rax), %xmm15, %xmm15
+
+// CHECK: vpmacswd %xmm15, 64(%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0x96,0x7a,0x40,0xf0]
+vpmacswd %xmm15, 64(%rdx), %xmm15, %xmm15
+
+// CHECK: vpmacswd %xmm15, (%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0x96,0x3a,0xf0]
+vpmacswd %xmm15, (%rdx), %xmm15, %xmm15
+
+// CHECK: vpmacswd %xmm15, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x48,0x00,0x96,0xff,0xf0]
+vpmacswd %xmm15, %xmm15, %xmm15, %xmm15
+
+// CHECK: vpmacswd %xmm6, 485498096, %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0x96,0x34,0x25,0xf0,0x1c,0xf0,0x1c,0x60]
+vpmacswd %xmm6, 485498096, %xmm6, %xmm6
+
+// CHECK: vpmacswd %xmm6, 64(%rdx,%rax,4), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0x96,0x74,0x82,0x40,0x60]
+vpmacswd %xmm6, 64(%rdx,%rax,4), %xmm6, %xmm6
+
+// CHECK: vpmacswd %xmm6, -64(%rdx,%rax,4), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0x96,0x74,0x82,0xc0,0x60]
+vpmacswd %xmm6, -64(%rdx,%rax,4), %xmm6, %xmm6
+
+// CHECK: vpmacswd %xmm6, 64(%rdx,%rax), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0x96,0x74,0x02,0x40,0x60]
+vpmacswd %xmm6, 64(%rdx,%rax), %xmm6, %xmm6
+
+// CHECK: vpmacswd %xmm6, 64(%rdx), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0x96,0x72,0x40,0x60]
+vpmacswd %xmm6, 64(%rdx), %xmm6, %xmm6
+
+// CHECK: vpmacswd %xmm6, (%rdx), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0x96,0x32,0x60]
+vpmacswd %xmm6, (%rdx), %xmm6, %xmm6
+
+// CHECK: vpmacswd %xmm6, %xmm6, %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0x96,0xf6,0x60]
+vpmacswd %xmm6, %xmm6, %xmm6, %xmm6
+
+// CHECK: vpmacsww %xmm15, 485498096, %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0x95,0x3c,0x25,0xf0,0x1c,0xf0,0x1c,0xf0]
+vpmacsww %xmm15, 485498096, %xmm15, %xmm15
+
+// CHECK: vpmacsww %xmm15, 64(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0x95,0x7c,0x82,0x40,0xf0]
+vpmacsww %xmm15, 64(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vpmacsww %xmm15, -64(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0x95,0x7c,0x82,0xc0,0xf0]
+vpmacsww %xmm15, -64(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vpmacsww %xmm15, 64(%rdx,%rax), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0x95,0x7c,0x02,0x40,0xf0]
+vpmacsww %xmm15, 64(%rdx,%rax), %xmm15, %xmm15
+
+// CHECK: vpmacsww %xmm15, 64(%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0x95,0x7a,0x40,0xf0]
+vpmacsww %xmm15, 64(%rdx), %xmm15, %xmm15
+
+// CHECK: vpmacsww %xmm15, (%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0x95,0x3a,0xf0]
+vpmacsww %xmm15, (%rdx), %xmm15, %xmm15
+
+// CHECK: vpmacsww %xmm15, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x48,0x00,0x95,0xff,0xf0]
+vpmacsww %xmm15, %xmm15, %xmm15, %xmm15
+
+// CHECK: vpmacsww %xmm6, 485498096, %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0x95,0x34,0x25,0xf0,0x1c,0xf0,0x1c,0x60]
+vpmacsww %xmm6, 485498096, %xmm6, %xmm6
+
+// CHECK: vpmacsww %xmm6, 64(%rdx,%rax,4), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0x95,0x74,0x82,0x40,0x60]
+vpmacsww %xmm6, 64(%rdx,%rax,4), %xmm6, %xmm6
+
+// CHECK: vpmacsww %xmm6, -64(%rdx,%rax,4), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0x95,0x74,0x82,0xc0,0x60]
+vpmacsww %xmm6, -64(%rdx,%rax,4), %xmm6, %xmm6
+
+// CHECK: vpmacsww %xmm6, 64(%rdx,%rax), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0x95,0x74,0x02,0x40,0x60]
+vpmacsww %xmm6, 64(%rdx,%rax), %xmm6, %xmm6
+
+// CHECK: vpmacsww %xmm6, 64(%rdx), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0x95,0x72,0x40,0x60]
+vpmacsww %xmm6, 64(%rdx), %xmm6, %xmm6
+
+// CHECK: vpmacsww %xmm6, (%rdx), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0x95,0x32,0x60]
+vpmacsww %xmm6, (%rdx), %xmm6, %xmm6
+
+// CHECK: vpmacsww %xmm6, %xmm6, %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0x95,0xf6,0x60]
+vpmacsww %xmm6, %xmm6, %xmm6, %xmm6
+
+// CHECK: vpmadcsswd %xmm15, 485498096, %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0xa6,0x3c,0x25,0xf0,0x1c,0xf0,0x1c,0xf0]
+vpmadcsswd %xmm15, 485498096, %xmm15, %xmm15
+
+// CHECK: vpmadcsswd %xmm15, 64(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0xa6,0x7c,0x82,0x40,0xf0]
+vpmadcsswd %xmm15, 64(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vpmadcsswd %xmm15, -64(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0xa6,0x7c,0x82,0xc0,0xf0]
+vpmadcsswd %xmm15, -64(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vpmadcsswd %xmm15, 64(%rdx,%rax), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0xa6,0x7c,0x02,0x40,0xf0]
+vpmadcsswd %xmm15, 64(%rdx,%rax), %xmm15, %xmm15
+
+// CHECK: vpmadcsswd %xmm15, 64(%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0xa6,0x7a,0x40,0xf0]
+vpmadcsswd %xmm15, 64(%rdx), %xmm15, %xmm15
+
+// CHECK: vpmadcsswd %xmm15, (%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0xa6,0x3a,0xf0]
+vpmadcsswd %xmm15, (%rdx), %xmm15, %xmm15
+
+// CHECK: vpmadcsswd %xmm15, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x48,0x00,0xa6,0xff,0xf0]
+vpmadcsswd %xmm15, %xmm15, %xmm15, %xmm15
+
+// CHECK: vpmadcsswd %xmm6, 485498096, %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0xa6,0x34,0x25,0xf0,0x1c,0xf0,0x1c,0x60]
+vpmadcsswd %xmm6, 485498096, %xmm6, %xmm6
+
+// CHECK: vpmadcsswd %xmm6, 64(%rdx,%rax,4), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0xa6,0x74,0x82,0x40,0x60]
+vpmadcsswd %xmm6, 64(%rdx,%rax,4), %xmm6, %xmm6
+
+// CHECK: vpmadcsswd %xmm6, -64(%rdx,%rax,4), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0xa6,0x74,0x82,0xc0,0x60]
+vpmadcsswd %xmm6, -64(%rdx,%rax,4), %xmm6, %xmm6
+
+// CHECK: vpmadcsswd %xmm6, 64(%rdx,%rax), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0xa6,0x74,0x02,0x40,0x60]
+vpmadcsswd %xmm6, 64(%rdx,%rax), %xmm6, %xmm6
+
+// CHECK: vpmadcsswd %xmm6, 64(%rdx), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0xa6,0x72,0x40,0x60]
+vpmadcsswd %xmm6, 64(%rdx), %xmm6, %xmm6
+
+// CHECK: vpmadcsswd %xmm6, (%rdx), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0xa6,0x32,0x60]
+vpmadcsswd %xmm6, (%rdx), %xmm6, %xmm6
+
+// CHECK: vpmadcsswd %xmm6, %xmm6, %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0xa6,0xf6,0x60]
+vpmadcsswd %xmm6, %xmm6, %xmm6, %xmm6
+
+// CHECK: vpmadcswd %xmm15, 485498096, %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0xb6,0x3c,0x25,0xf0,0x1c,0xf0,0x1c,0xf0]
+vpmadcswd %xmm15, 485498096, %xmm15, %xmm15
+
+// CHECK: vpmadcswd %xmm15, 64(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0xb6,0x7c,0x82,0x40,0xf0]
+vpmadcswd %xmm15, 64(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vpmadcswd %xmm15, -64(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0xb6,0x7c,0x82,0xc0,0xf0]
+vpmadcswd %xmm15, -64(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vpmadcswd %xmm15, 64(%rdx,%rax), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0xb6,0x7c,0x02,0x40,0xf0]
+vpmadcswd %xmm15, 64(%rdx,%rax), %xmm15, %xmm15
+
+// CHECK: vpmadcswd %xmm15, 64(%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0xb6,0x7a,0x40,0xf0]
+vpmadcswd %xmm15, 64(%rdx), %xmm15, %xmm15
+
+// CHECK: vpmadcswd %xmm15, (%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0xb6,0x3a,0xf0]
+vpmadcswd %xmm15, (%rdx), %xmm15, %xmm15
+
+// CHECK: vpmadcswd %xmm15, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x48,0x00,0xb6,0xff,0xf0]
+vpmadcswd %xmm15, %xmm15, %xmm15, %xmm15
+
+// CHECK: vpmadcswd %xmm6, 485498096, %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0xb6,0x34,0x25,0xf0,0x1c,0xf0,0x1c,0x60]
+vpmadcswd %xmm6, 485498096, %xmm6, %xmm6
+
+// CHECK: vpmadcswd %xmm6, 64(%rdx,%rax,4), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0xb6,0x74,0x82,0x40,0x60]
+vpmadcswd %xmm6, 64(%rdx,%rax,4), %xmm6, %xmm6
+
+// CHECK: vpmadcswd %xmm6, -64(%rdx,%rax,4), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0xb6,0x74,0x82,0xc0,0x60]
+vpmadcswd %xmm6, -64(%rdx,%rax,4), %xmm6, %xmm6
+
+// CHECK: vpmadcswd %xmm6, 64(%rdx,%rax), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0xb6,0x74,0x02,0x40,0x60]
+vpmadcswd %xmm6, 64(%rdx,%rax), %xmm6, %xmm6
+
+// CHECK: vpmadcswd %xmm6, 64(%rdx), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0xb6,0x72,0x40,0x60]
+vpmadcswd %xmm6, 64(%rdx), %xmm6, %xmm6
+
+// CHECK: vpmadcswd %xmm6, (%rdx), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0xb6,0x32,0x60]
+vpmadcswd %xmm6, (%rdx), %xmm6, %xmm6
+
+// CHECK: vpmadcswd %xmm6, %xmm6, %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0xb6,0xf6,0x60]
+vpmadcswd %xmm6, %xmm6, %xmm6, %xmm6
+
+// CHECK: vpperm 485498096, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x80,0xa3,0x3c,0x25,0xf0,0x1c,0xf0,0x1c,0xf0]
+vpperm 485498096, %xmm15, %xmm15, %xmm15
+
+// CHECK: vpperm 485498096, %xmm6, %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0xc8,0xa3,0x34,0x25,0xf0,0x1c,0xf0,0x1c,0x60]
+vpperm 485498096, %xmm6, %xmm6, %xmm6
+
+// CHECK: vpperm 64(%rdx,%rax,4), %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x80,0xa3,0x7c,0x82,0x40,0xf0]
+vpperm 64(%rdx,%rax,4), %xmm15, %xmm15, %xmm15
+
+// CHECK: vpperm -64(%rdx,%rax,4), %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x80,0xa3,0x7c,0x82,0xc0,0xf0]
+vpperm -64(%rdx,%rax,4), %xmm15, %xmm15, %xmm15
+
+// CHECK: vpperm 64(%rdx,%rax,4), %xmm6, %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0xc8,0xa3,0x74,0x82,0x40,0x60]
+vpperm 64(%rdx,%rax,4), %xmm6, %xmm6, %xmm6
+
+// CHECK: vpperm -64(%rdx,%rax,4), %xmm6, %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0xc8,0xa3,0x74,0x82,0xc0,0x60]
+vpperm -64(%rdx,%rax,4), %xmm6, %xmm6, %xmm6
+
+// CHECK: vpperm 64(%rdx,%rax), %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x80,0xa3,0x7c,0x02,0x40,0xf0]
+vpperm 64(%rdx,%rax), %xmm15, %xmm15, %xmm15
+
+// CHECK: vpperm 64(%rdx,%rax), %xmm6, %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0xc8,0xa3,0x74,0x02,0x40,0x60]
+vpperm 64(%rdx,%rax), %xmm6, %xmm6, %xmm6
+
+// CHECK: vpperm 64(%rdx), %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x80,0xa3,0x7a,0x40,0xf0]
+vpperm 64(%rdx), %xmm15, %xmm15, %xmm15
+
+// CHECK: vpperm 64(%rdx), %xmm6, %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0xc8,0xa3,0x72,0x40,0x60]
+vpperm 64(%rdx), %xmm6, %xmm6, %xmm6
+
+// CHECK: vpperm (%rdx), %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x80,0xa3,0x3a,0xf0]
+vpperm (%rdx), %xmm15, %xmm15, %xmm15
+
+// CHECK: vpperm (%rdx), %xmm6, %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0xc8,0xa3,0x32,0x60]
+vpperm (%rdx), %xmm6, %xmm6, %xmm6
+
+// CHECK: vpperm %xmm15, 485498096, %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0xa3,0x3c,0x25,0xf0,0x1c,0xf0,0x1c,0xf0]
+vpperm %xmm15, 485498096, %xmm15, %xmm15
+
+// CHECK: vpperm %xmm15, 64(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0xa3,0x7c,0x82,0x40,0xf0]
+vpperm %xmm15, 64(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vpperm %xmm15, -64(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0xa3,0x7c,0x82,0xc0,0xf0]
+vpperm %xmm15, -64(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vpperm %xmm15, 64(%rdx,%rax), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0xa3,0x7c,0x02,0x40,0xf0]
+vpperm %xmm15, 64(%rdx,%rax), %xmm15, %xmm15
+
+// CHECK: vpperm %xmm15, 64(%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0xa3,0x7a,0x40,0xf0]
+vpperm %xmm15, 64(%rdx), %xmm15, %xmm15
+
+// CHECK: vpperm %xmm15, (%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x00,0xa3,0x3a,0xf0]
+vpperm %xmm15, (%rdx), %xmm15, %xmm15
+
+// CHECK: vpperm %xmm15, %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x48,0x00,0xa3,0xff,0xf0]
+vpperm %xmm15, %xmm15, %xmm15, %xmm15
+
+// CHECK: vpperm %xmm6, 485498096, %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0xa3,0x34,0x25,0xf0,0x1c,0xf0,0x1c,0x60]
+vpperm %xmm6, 485498096, %xmm6, %xmm6
+
+// CHECK: vpperm %xmm6, 64(%rdx,%rax,4), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0xa3,0x74,0x82,0x40,0x60]
+vpperm %xmm6, 64(%rdx,%rax,4), %xmm6, %xmm6
+
+// CHECK: vpperm %xmm6, -64(%rdx,%rax,4), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0xa3,0x74,0x82,0xc0,0x60]
+vpperm %xmm6, -64(%rdx,%rax,4), %xmm6, %xmm6
+
+// CHECK: vpperm %xmm6, 64(%rdx,%rax), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0xa3,0x74,0x02,0x40,0x60]
+vpperm %xmm6, 64(%rdx,%rax), %xmm6, %xmm6
+
+// CHECK: vpperm %xmm6, 64(%rdx), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0xa3,0x72,0x40,0x60]
+vpperm %xmm6, 64(%rdx), %xmm6, %xmm6
+
+// CHECK: vpperm %xmm6, (%rdx), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0xa3,0x32,0x60]
+vpperm %xmm6, (%rdx), %xmm6, %xmm6
+
+// CHECK: vpperm %xmm6, %xmm6, %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x48,0xa3,0xf6,0x60]
+vpperm %xmm6, %xmm6, %xmm6, %xmm6
+
+// CHECK: vprotb $0, 485498096, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x78,0xc0,0x3c,0x25,0xf0,0x1c,0xf0,0x1c,0x00]
+vprotb $0, 485498096, %xmm15
+
+// CHECK: vprotb $0, 485498096, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x78,0xc0,0x34,0x25,0xf0,0x1c,0xf0,0x1c,0x00]
+vprotb $0, 485498096, %xmm6
+
+// CHECK: vprotb $0, 64(%rdx,%rax,4), %xmm15
+// CHECK: encoding: [0x8f,0x68,0x78,0xc0,0x7c,0x82,0x40,0x00]
+vprotb $0, 64(%rdx,%rax,4), %xmm15
+
+// CHECK: vprotb $0, -64(%rdx,%rax,4), %xmm15
+// CHECK: encoding: [0x8f,0x68,0x78,0xc0,0x7c,0x82,0xc0,0x00]
+vprotb $0, -64(%rdx,%rax,4), %xmm15
+
+// CHECK: vprotb $0, 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x78,0xc0,0x74,0x82,0x40,0x00]
+vprotb $0, 64(%rdx,%rax,4), %xmm6
+
+// CHECK: vprotb $0, -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x78,0xc0,0x74,0x82,0xc0,0x00]
+vprotb $0, -64(%rdx,%rax,4), %xmm6
+
+// CHECK: vprotb $0, 64(%rdx,%rax), %xmm15
+// CHECK: encoding: [0x8f,0x68,0x78,0xc0,0x7c,0x02,0x40,0x00]
+vprotb $0, 64(%rdx,%rax), %xmm15
+
+// CHECK: vprotb $0, 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x78,0xc0,0x74,0x02,0x40,0x00]
+vprotb $0, 64(%rdx,%rax), %xmm6
+
+// CHECK: vprotb $0, 64(%rdx), %xmm15
+// CHECK: encoding: [0x8f,0x68,0x78,0xc0,0x7a,0x40,0x00]
+vprotb $0, 64(%rdx), %xmm15
+
+// CHECK: vprotb $0, 64(%rdx), %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x78,0xc0,0x72,0x40,0x00]
+vprotb $0, 64(%rdx), %xmm6
+
+// CHECK: vprotb $0, (%rdx), %xmm15
+// CHECK: encoding: [0x8f,0x68,0x78,0xc0,0x3a,0x00]
+vprotb $0, (%rdx), %xmm15
+
+// CHECK: vprotb $0, (%rdx), %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x78,0xc0,0x32,0x00]
+vprotb $0, (%rdx), %xmm6
+
+// CHECK: vprotb $0, %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x48,0x78,0xc0,0xff,0x00]
+vprotb $0, %xmm15, %xmm15
+
+// CHECK: vprotb $0, %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x78,0xc0,0xf6,0x00]
+vprotb $0, %xmm6, %xmm6
+
+// CHECK: vprotb 485498096, %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x69,0x80,0x90,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vprotb 485498096, %xmm15, %xmm15
+
+// CHECK: vprotb 485498096, %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0xc8,0x90,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+vprotb 485498096, %xmm6, %xmm6
+
+// CHECK: vprotb 64(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x69,0x80,0x90,0x7c,0x82,0x40]
+vprotb 64(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vprotb -64(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x69,0x80,0x90,0x7c,0x82,0xc0]
+vprotb -64(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vprotb 64(%rdx,%rax,4), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0xc8,0x90,0x74,0x82,0x40]
+vprotb 64(%rdx,%rax,4), %xmm6, %xmm6
+
+// CHECK: vprotb -64(%rdx,%rax,4), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0xc8,0x90,0x74,0x82,0xc0]
+vprotb -64(%rdx,%rax,4), %xmm6, %xmm6
+
+// CHECK: vprotb 64(%rdx,%rax), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x69,0x80,0x90,0x7c,0x02,0x40]
+vprotb 64(%rdx,%rax), %xmm15, %xmm15
+
+// CHECK: vprotb 64(%rdx,%rax), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0xc8,0x90,0x74,0x02,0x40]
+vprotb 64(%rdx,%rax), %xmm6, %xmm6
+
+// CHECK: vprotb 64(%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x69,0x80,0x90,0x7a,0x40]
+vprotb 64(%rdx), %xmm15, %xmm15
+
+// CHECK: vprotb 64(%rdx), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0xc8,0x90,0x72,0x40]
+vprotb 64(%rdx), %xmm6, %xmm6
+
+// CHECK: vprotb (%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x69,0x80,0x90,0x3a]
+vprotb (%rdx), %xmm15, %xmm15
+
+// CHECK: vprotb (%rdx), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0xc8,0x90,0x32]
+vprotb (%rdx), %xmm6, %xmm6
+
+// CHECK: vprotb %xmm15, 485498096, %xmm15
+// CHECK: encoding: [0x8f,0x69,0x00,0x90,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vprotb %xmm15, 485498096, %xmm15
+
+// CHECK: vprotb %xmm15, 64(%rdx,%rax,4), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x00,0x90,0x7c,0x82,0x40]
+vprotb %xmm15, 64(%rdx,%rax,4), %xmm15
+
+// CHECK: vprotb %xmm15, -64(%rdx,%rax,4), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x00,0x90,0x7c,0x82,0xc0]
+vprotb %xmm15, -64(%rdx,%rax,4), %xmm15
+
+// CHECK: vprotb %xmm15, 64(%rdx,%rax), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x00,0x90,0x7c,0x02,0x40]
+vprotb %xmm15, 64(%rdx,%rax), %xmm15
+
+// CHECK: vprotb %xmm15, 64(%rdx), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x00,0x90,0x7a,0x40]
+vprotb %xmm15, 64(%rdx), %xmm15
+
+// CHECK: vprotb %xmm15, (%rdx), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x00,0x90,0x3a]
+vprotb %xmm15, (%rdx), %xmm15
+
+// CHECK: vprotb %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x49,0x00,0x90,0xff]
+vprotb %xmm15, %xmm15, %xmm15
+
+// CHECK: vprotb %xmm6, 485498096, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x48,0x90,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+vprotb %xmm6, 485498096, %xmm6
+
+// CHECK: vprotb %xmm6, 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x48,0x90,0x74,0x82,0x40]
+vprotb %xmm6, 64(%rdx,%rax,4), %xmm6
+
+// CHECK: vprotb %xmm6, -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x48,0x90,0x74,0x82,0xc0]
+vprotb %xmm6, -64(%rdx,%rax,4), %xmm6
+
+// CHECK: vprotb %xmm6, 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x48,0x90,0x74,0x02,0x40]
+vprotb %xmm6, 64(%rdx,%rax), %xmm6
+
+// CHECK: vprotb %xmm6, 64(%rdx), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x48,0x90,0x72,0x40]
+vprotb %xmm6, 64(%rdx), %xmm6
+
+// CHECK: vprotb %xmm6, (%rdx), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x48,0x90,0x32]
+vprotb %xmm6, (%rdx), %xmm6
+
+// CHECK: vprotb %xmm6, %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x48,0x90,0xf6]
+vprotb %xmm6, %xmm6, %xmm6
+
+// CHECK: vprotd $0, 485498096, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x78,0xc2,0x3c,0x25,0xf0,0x1c,0xf0,0x1c,0x00]
+vprotd $0, 485498096, %xmm15
+
+// CHECK: vprotd $0, 485498096, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x78,0xc2,0x34,0x25,0xf0,0x1c,0xf0,0x1c,0x00]
+vprotd $0, 485498096, %xmm6
+
+// CHECK: vprotd $0, 64(%rdx,%rax,4), %xmm15
+// CHECK: encoding: [0x8f,0x68,0x78,0xc2,0x7c,0x82,0x40,0x00]
+vprotd $0, 64(%rdx,%rax,4), %xmm15
+
+// CHECK: vprotd $0, -64(%rdx,%rax,4), %xmm15
+// CHECK: encoding: [0x8f,0x68,0x78,0xc2,0x7c,0x82,0xc0,0x00]
+vprotd $0, -64(%rdx,%rax,4), %xmm15
+
+// CHECK: vprotd $0, 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x78,0xc2,0x74,0x82,0x40,0x00]
+vprotd $0, 64(%rdx,%rax,4), %xmm6
+
+// CHECK: vprotd $0, -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x78,0xc2,0x74,0x82,0xc0,0x00]
+vprotd $0, -64(%rdx,%rax,4), %xmm6
+
+// CHECK: vprotd $0, 64(%rdx,%rax), %xmm15
+// CHECK: encoding: [0x8f,0x68,0x78,0xc2,0x7c,0x02,0x40,0x00]
+vprotd $0, 64(%rdx,%rax), %xmm15
+
+// CHECK: vprotd $0, 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x78,0xc2,0x74,0x02,0x40,0x00]
+vprotd $0, 64(%rdx,%rax), %xmm6
+
+// CHECK: vprotd $0, 64(%rdx), %xmm15
+// CHECK: encoding: [0x8f,0x68,0x78,0xc2,0x7a,0x40,0x00]
+vprotd $0, 64(%rdx), %xmm15
+
+// CHECK: vprotd $0, 64(%rdx), %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x78,0xc2,0x72,0x40,0x00]
+vprotd $0, 64(%rdx), %xmm6
+
+// CHECK: vprotd $0, (%rdx), %xmm15
+// CHECK: encoding: [0x8f,0x68,0x78,0xc2,0x3a,0x00]
+vprotd $0, (%rdx), %xmm15
+
+// CHECK: vprotd $0, (%rdx), %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x78,0xc2,0x32,0x00]
+vprotd $0, (%rdx), %xmm6
+
+// CHECK: vprotd $0, %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x48,0x78,0xc2,0xff,0x00]
+vprotd $0, %xmm15, %xmm15
+
+// CHECK: vprotd $0, %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x78,0xc2,0xf6,0x00]
+vprotd $0, %xmm6, %xmm6
+
+// CHECK: vprotd 485498096, %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x69,0x80,0x92,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vprotd 485498096, %xmm15, %xmm15
+
+// CHECK: vprotd 485498096, %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0xc8,0x92,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+vprotd 485498096, %xmm6, %xmm6
+
+// CHECK: vprotd 64(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x69,0x80,0x92,0x7c,0x82,0x40]
+vprotd 64(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vprotd -64(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x69,0x80,0x92,0x7c,0x82,0xc0]
+vprotd -64(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vprotd 64(%rdx,%rax,4), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0xc8,0x92,0x74,0x82,0x40]
+vprotd 64(%rdx,%rax,4), %xmm6, %xmm6
+
+// CHECK: vprotd -64(%rdx,%rax,4), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0xc8,0x92,0x74,0x82,0xc0]
+vprotd -64(%rdx,%rax,4), %xmm6, %xmm6
+
+// CHECK: vprotd 64(%rdx,%rax), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x69,0x80,0x92,0x7c,0x02,0x40]
+vprotd 64(%rdx,%rax), %xmm15, %xmm15
+
+// CHECK: vprotd 64(%rdx,%rax), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0xc8,0x92,0x74,0x02,0x40]
+vprotd 64(%rdx,%rax), %xmm6, %xmm6
+
+// CHECK: vprotd 64(%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x69,0x80,0x92,0x7a,0x40]
+vprotd 64(%rdx), %xmm15, %xmm15
+
+// CHECK: vprotd 64(%rdx), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0xc8,0x92,0x72,0x40]
+vprotd 64(%rdx), %xmm6, %xmm6
+
+// CHECK: vprotd (%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x69,0x80,0x92,0x3a]
+vprotd (%rdx), %xmm15, %xmm15
+
+// CHECK: vprotd (%rdx), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0xc8,0x92,0x32]
+vprotd (%rdx), %xmm6, %xmm6
+
+// CHECK: vprotd %xmm15, 485498096, %xmm15
+// CHECK: encoding: [0x8f,0x69,0x00,0x92,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vprotd %xmm15, 485498096, %xmm15
+
+// CHECK: vprotd %xmm15, 64(%rdx,%rax,4), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x00,0x92,0x7c,0x82,0x40]
+vprotd %xmm15, 64(%rdx,%rax,4), %xmm15
+
+// CHECK: vprotd %xmm15, -64(%rdx,%rax,4), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x00,0x92,0x7c,0x82,0xc0]
+vprotd %xmm15, -64(%rdx,%rax,4), %xmm15
+
+// CHECK: vprotd %xmm15, 64(%rdx,%rax), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x00,0x92,0x7c,0x02,0x40]
+vprotd %xmm15, 64(%rdx,%rax), %xmm15
+
+// CHECK: vprotd %xmm15, 64(%rdx), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x00,0x92,0x7a,0x40]
+vprotd %xmm15, 64(%rdx), %xmm15
+
+// CHECK: vprotd %xmm15, (%rdx), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x00,0x92,0x3a]
+vprotd %xmm15, (%rdx), %xmm15
+
+// CHECK: vprotd %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x49,0x00,0x92,0xff]
+vprotd %xmm15, %xmm15, %xmm15
+
+// CHECK: vprotd %xmm6, 485498096, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x48,0x92,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+vprotd %xmm6, 485498096, %xmm6
+
+// CHECK: vprotd %xmm6, 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x48,0x92,0x74,0x82,0x40]
+vprotd %xmm6, 64(%rdx,%rax,4), %xmm6
+
+// CHECK: vprotd %xmm6, -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x48,0x92,0x74,0x82,0xc0]
+vprotd %xmm6, -64(%rdx,%rax,4), %xmm6
+
+// CHECK: vprotd %xmm6, 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x48,0x92,0x74,0x02,0x40]
+vprotd %xmm6, 64(%rdx,%rax), %xmm6
+
+// CHECK: vprotd %xmm6, 64(%rdx), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x48,0x92,0x72,0x40]
+vprotd %xmm6, 64(%rdx), %xmm6
+
+// CHECK: vprotd %xmm6, (%rdx), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x48,0x92,0x32]
+vprotd %xmm6, (%rdx), %xmm6
+
+// CHECK: vprotd %xmm6, %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x48,0x92,0xf6]
+vprotd %xmm6, %xmm6, %xmm6
+
+// CHECK: vprotq $0, 485498096, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x78,0xc3,0x3c,0x25,0xf0,0x1c,0xf0,0x1c,0x00]
+vprotq $0, 485498096, %xmm15
+
+// CHECK: vprotq $0, 485498096, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x78,0xc3,0x34,0x25,0xf0,0x1c,0xf0,0x1c,0x00]
+vprotq $0, 485498096, %xmm6
+
+// CHECK: vprotq $0, 64(%rdx,%rax,4), %xmm15
+// CHECK: encoding: [0x8f,0x68,0x78,0xc3,0x7c,0x82,0x40,0x00]
+vprotq $0, 64(%rdx,%rax,4), %xmm15
+
+// CHECK: vprotq $0, -64(%rdx,%rax,4), %xmm15
+// CHECK: encoding: [0x8f,0x68,0x78,0xc3,0x7c,0x82,0xc0,0x00]
+vprotq $0, -64(%rdx,%rax,4), %xmm15
+
+// CHECK: vprotq $0, 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x78,0xc3,0x74,0x82,0x40,0x00]
+vprotq $0, 64(%rdx,%rax,4), %xmm6
+
+// CHECK: vprotq $0, -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x78,0xc3,0x74,0x82,0xc0,0x00]
+vprotq $0, -64(%rdx,%rax,4), %xmm6
+
+// CHECK: vprotq $0, 64(%rdx,%rax), %xmm15
+// CHECK: encoding: [0x8f,0x68,0x78,0xc3,0x7c,0x02,0x40,0x00]
+vprotq $0, 64(%rdx,%rax), %xmm15
+
+// CHECK: vprotq $0, 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x78,0xc3,0x74,0x02,0x40,0x00]
+vprotq $0, 64(%rdx,%rax), %xmm6
+
+// CHECK: vprotq $0, 64(%rdx), %xmm15
+// CHECK: encoding: [0x8f,0x68,0x78,0xc3,0x7a,0x40,0x00]
+vprotq $0, 64(%rdx), %xmm15
+
+// CHECK: vprotq $0, 64(%rdx), %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x78,0xc3,0x72,0x40,0x00]
+vprotq $0, 64(%rdx), %xmm6
+
+// CHECK: vprotq $0, (%rdx), %xmm15
+// CHECK: encoding: [0x8f,0x68,0x78,0xc3,0x3a,0x00]
+vprotq $0, (%rdx), %xmm15
+
+// CHECK: vprotq $0, (%rdx), %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x78,0xc3,0x32,0x00]
+vprotq $0, (%rdx), %xmm6
+
+// CHECK: vprotq $0, %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x48,0x78,0xc3,0xff,0x00]
+vprotq $0, %xmm15, %xmm15
+
+// CHECK: vprotq $0, %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x78,0xc3,0xf6,0x00]
+vprotq $0, %xmm6, %xmm6
+
+// CHECK: vprotq 485498096, %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x69,0x80,0x93,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vprotq 485498096, %xmm15, %xmm15
+
+// CHECK: vprotq 485498096, %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0xc8,0x93,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+vprotq 485498096, %xmm6, %xmm6
+
+// CHECK: vprotq 64(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x69,0x80,0x93,0x7c,0x82,0x40]
+vprotq 64(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vprotq -64(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x69,0x80,0x93,0x7c,0x82,0xc0]
+vprotq -64(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vprotq 64(%rdx,%rax,4), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0xc8,0x93,0x74,0x82,0x40]
+vprotq 64(%rdx,%rax,4), %xmm6, %xmm6
+
+// CHECK: vprotq -64(%rdx,%rax,4), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0xc8,0x93,0x74,0x82,0xc0]
+vprotq -64(%rdx,%rax,4), %xmm6, %xmm6
+
+// CHECK: vprotq 64(%rdx,%rax), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x69,0x80,0x93,0x7c,0x02,0x40]
+vprotq 64(%rdx,%rax), %xmm15, %xmm15
+
+// CHECK: vprotq 64(%rdx,%rax), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0xc8,0x93,0x74,0x02,0x40]
+vprotq 64(%rdx,%rax), %xmm6, %xmm6
+
+// CHECK: vprotq 64(%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x69,0x80,0x93,0x7a,0x40]
+vprotq 64(%rdx), %xmm15, %xmm15
+
+// CHECK: vprotq 64(%rdx), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0xc8,0x93,0x72,0x40]
+vprotq 64(%rdx), %xmm6, %xmm6
+
+// CHECK: vprotq (%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x69,0x80,0x93,0x3a]
+vprotq (%rdx), %xmm15, %xmm15
+
+// CHECK: vprotq (%rdx), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0xc8,0x93,0x32]
+vprotq (%rdx), %xmm6, %xmm6
+
+// CHECK: vprotq %xmm15, 485498096, %xmm15
+// CHECK: encoding: [0x8f,0x69,0x00,0x93,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vprotq %xmm15, 485498096, %xmm15
+
+// CHECK: vprotq %xmm15, 64(%rdx,%rax,4), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x00,0x93,0x7c,0x82,0x40]
+vprotq %xmm15, 64(%rdx,%rax,4), %xmm15
+
+// CHECK: vprotq %xmm15, -64(%rdx,%rax,4), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x00,0x93,0x7c,0x82,0xc0]
+vprotq %xmm15, -64(%rdx,%rax,4), %xmm15
+
+// CHECK: vprotq %xmm15, 64(%rdx,%rax), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x00,0x93,0x7c,0x02,0x40]
+vprotq %xmm15, 64(%rdx,%rax), %xmm15
+
+// CHECK: vprotq %xmm15, 64(%rdx), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x00,0x93,0x7a,0x40]
+vprotq %xmm15, 64(%rdx), %xmm15
+
+// CHECK: vprotq %xmm15, (%rdx), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x00,0x93,0x3a]
+vprotq %xmm15, (%rdx), %xmm15
+
+// CHECK: vprotq %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x49,0x00,0x93,0xff]
+vprotq %xmm15, %xmm15, %xmm15
+
+// CHECK: vprotq %xmm6, 485498096, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x48,0x93,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+vprotq %xmm6, 485498096, %xmm6
+
+// CHECK: vprotq %xmm6, 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x48,0x93,0x74,0x82,0x40]
+vprotq %xmm6, 64(%rdx,%rax,4), %xmm6
+
+// CHECK: vprotq %xmm6, -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x48,0x93,0x74,0x82,0xc0]
+vprotq %xmm6, -64(%rdx,%rax,4), %xmm6
+
+// CHECK: vprotq %xmm6, 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x48,0x93,0x74,0x02,0x40]
+vprotq %xmm6, 64(%rdx,%rax), %xmm6
+
+// CHECK: vprotq %xmm6, 64(%rdx), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x48,0x93,0x72,0x40]
+vprotq %xmm6, 64(%rdx), %xmm6
+
+// CHECK: vprotq %xmm6, (%rdx), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x48,0x93,0x32]
+vprotq %xmm6, (%rdx), %xmm6
+
+// CHECK: vprotq %xmm6, %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x48,0x93,0xf6]
+vprotq %xmm6, %xmm6, %xmm6
+
+// CHECK: vprotw $0, 485498096, %xmm15
+// CHECK: encoding: [0x8f,0x68,0x78,0xc1,0x3c,0x25,0xf0,0x1c,0xf0,0x1c,0x00]
+vprotw $0, 485498096, %xmm15
+
+// CHECK: vprotw $0, 485498096, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x78,0xc1,0x34,0x25,0xf0,0x1c,0xf0,0x1c,0x00]
+vprotw $0, 485498096, %xmm6
+
+// CHECK: vprotw $0, 64(%rdx,%rax,4), %xmm15
+// CHECK: encoding: [0x8f,0x68,0x78,0xc1,0x7c,0x82,0x40,0x00]
+vprotw $0, 64(%rdx,%rax,4), %xmm15
+
+// CHECK: vprotw $0, -64(%rdx,%rax,4), %xmm15
+// CHECK: encoding: [0x8f,0x68,0x78,0xc1,0x7c,0x82,0xc0,0x00]
+vprotw $0, -64(%rdx,%rax,4), %xmm15
+
+// CHECK: vprotw $0, 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x78,0xc1,0x74,0x82,0x40,0x00]
+vprotw $0, 64(%rdx,%rax,4), %xmm6
+
+// CHECK: vprotw $0, -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x78,0xc1,0x74,0x82,0xc0,0x00]
+vprotw $0, -64(%rdx,%rax,4), %xmm6
+
+// CHECK: vprotw $0, 64(%rdx,%rax), %xmm15
+// CHECK: encoding: [0x8f,0x68,0x78,0xc1,0x7c,0x02,0x40,0x00]
+vprotw $0, 64(%rdx,%rax), %xmm15
+
+// CHECK: vprotw $0, 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x78,0xc1,0x74,0x02,0x40,0x00]
+vprotw $0, 64(%rdx,%rax), %xmm6
+
+// CHECK: vprotw $0, 64(%rdx), %xmm15
+// CHECK: encoding: [0x8f,0x68,0x78,0xc1,0x7a,0x40,0x00]
+vprotw $0, 64(%rdx), %xmm15
+
+// CHECK: vprotw $0, 64(%rdx), %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x78,0xc1,0x72,0x40,0x00]
+vprotw $0, 64(%rdx), %xmm6
+
+// CHECK: vprotw $0, (%rdx), %xmm15
+// CHECK: encoding: [0x8f,0x68,0x78,0xc1,0x3a,0x00]
+vprotw $0, (%rdx), %xmm15
+
+// CHECK: vprotw $0, (%rdx), %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x78,0xc1,0x32,0x00]
+vprotw $0, (%rdx), %xmm6
+
+// CHECK: vprotw $0, %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x48,0x78,0xc1,0xff,0x00]
+vprotw $0, %xmm15, %xmm15
+
+// CHECK: vprotw $0, %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe8,0x78,0xc1,0xf6,0x00]
+vprotw $0, %xmm6, %xmm6
+
+// CHECK: vprotw 485498096, %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x69,0x80,0x91,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vprotw 485498096, %xmm15, %xmm15
+
+// CHECK: vprotw 485498096, %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0xc8,0x91,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+vprotw 485498096, %xmm6, %xmm6
+
+// CHECK: vprotw 64(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x69,0x80,0x91,0x7c,0x82,0x40]
+vprotw 64(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vprotw -64(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x69,0x80,0x91,0x7c,0x82,0xc0]
+vprotw -64(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vprotw 64(%rdx,%rax,4), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0xc8,0x91,0x74,0x82,0x40]
+vprotw 64(%rdx,%rax,4), %xmm6, %xmm6
+
+// CHECK: vprotw -64(%rdx,%rax,4), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0xc8,0x91,0x74,0x82,0xc0]
+vprotw -64(%rdx,%rax,4), %xmm6, %xmm6
+
+// CHECK: vprotw 64(%rdx,%rax), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x69,0x80,0x91,0x7c,0x02,0x40]
+vprotw 64(%rdx,%rax), %xmm15, %xmm15
+
+// CHECK: vprotw 64(%rdx,%rax), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0xc8,0x91,0x74,0x02,0x40]
+vprotw 64(%rdx,%rax), %xmm6, %xmm6
+
+// CHECK: vprotw 64(%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x69,0x80,0x91,0x7a,0x40]
+vprotw 64(%rdx), %xmm15, %xmm15
+
+// CHECK: vprotw 64(%rdx), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0xc8,0x91,0x72,0x40]
+vprotw 64(%rdx), %xmm6, %xmm6
+
+// CHECK: vprotw (%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x69,0x80,0x91,0x3a]
+vprotw (%rdx), %xmm15, %xmm15
+
+// CHECK: vprotw (%rdx), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0xc8,0x91,0x32]
+vprotw (%rdx), %xmm6, %xmm6
+
+// CHECK: vprotw %xmm15, 485498096, %xmm15
+// CHECK: encoding: [0x8f,0x69,0x00,0x91,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vprotw %xmm15, 485498096, %xmm15
+
+// CHECK: vprotw %xmm15, 64(%rdx,%rax,4), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x00,0x91,0x7c,0x82,0x40]
+vprotw %xmm15, 64(%rdx,%rax,4), %xmm15
+
+// CHECK: vprotw %xmm15, -64(%rdx,%rax,4), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x00,0x91,0x7c,0x82,0xc0]
+vprotw %xmm15, -64(%rdx,%rax,4), %xmm15
+
+// CHECK: vprotw %xmm15, 64(%rdx,%rax), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x00,0x91,0x7c,0x02,0x40]
+vprotw %xmm15, 64(%rdx,%rax), %xmm15
+
+// CHECK: vprotw %xmm15, 64(%rdx), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x00,0x91,0x7a,0x40]
+vprotw %xmm15, 64(%rdx), %xmm15
+
+// CHECK: vprotw %xmm15, (%rdx), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x00,0x91,0x3a]
+vprotw %xmm15, (%rdx), %xmm15
+
+// CHECK: vprotw %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x49,0x00,0x91,0xff]
+vprotw %xmm15, %xmm15, %xmm15
+
+// CHECK: vprotw %xmm6, 485498096, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x48,0x91,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+vprotw %xmm6, 485498096, %xmm6
+
+// CHECK: vprotw %xmm6, 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x48,0x91,0x74,0x82,0x40]
+vprotw %xmm6, 64(%rdx,%rax,4), %xmm6
+
+// CHECK: vprotw %xmm6, -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x48,0x91,0x74,0x82,0xc0]
+vprotw %xmm6, -64(%rdx,%rax,4), %xmm6
+
+// CHECK: vprotw %xmm6, 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x48,0x91,0x74,0x02,0x40]
+vprotw %xmm6, 64(%rdx,%rax), %xmm6
+
+// CHECK: vprotw %xmm6, 64(%rdx), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x48,0x91,0x72,0x40]
+vprotw %xmm6, 64(%rdx), %xmm6
+
+// CHECK: vprotw %xmm6, (%rdx), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x48,0x91,0x32]
+vprotw %xmm6, (%rdx), %xmm6
+
+// CHECK: vprotw %xmm6, %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x48,0x91,0xf6]
+vprotw %xmm6, %xmm6, %xmm6
+
+// CHECK: vpshab 485498096, %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x69,0x80,0x98,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vpshab 485498096, %xmm15, %xmm15
+
+// CHECK: vpshab 485498096, %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0xc8,0x98,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+vpshab 485498096, %xmm6, %xmm6
+
+// CHECK: vpshab 64(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x69,0x80,0x98,0x7c,0x82,0x40]
+vpshab 64(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vpshab -64(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x69,0x80,0x98,0x7c,0x82,0xc0]
+vpshab -64(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vpshab 64(%rdx,%rax,4), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0xc8,0x98,0x74,0x82,0x40]
+vpshab 64(%rdx,%rax,4), %xmm6, %xmm6
+
+// CHECK: vpshab -64(%rdx,%rax,4), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0xc8,0x98,0x74,0x82,0xc0]
+vpshab -64(%rdx,%rax,4), %xmm6, %xmm6
+
+// CHECK: vpshab 64(%rdx,%rax), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x69,0x80,0x98,0x7c,0x02,0x40]
+vpshab 64(%rdx,%rax), %xmm15, %xmm15
+
+// CHECK: vpshab 64(%rdx,%rax), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0xc8,0x98,0x74,0x02,0x40]
+vpshab 64(%rdx,%rax), %xmm6, %xmm6
+
+// CHECK: vpshab 64(%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x69,0x80,0x98,0x7a,0x40]
+vpshab 64(%rdx), %xmm15, %xmm15
+
+// CHECK: vpshab 64(%rdx), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0xc8,0x98,0x72,0x40]
+vpshab 64(%rdx), %xmm6, %xmm6
+
+// CHECK: vpshab (%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x69,0x80,0x98,0x3a]
+vpshab (%rdx), %xmm15, %xmm15
+
+// CHECK: vpshab (%rdx), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0xc8,0x98,0x32]
+vpshab (%rdx), %xmm6, %xmm6
+
+// CHECK: vpshab %xmm15, 485498096, %xmm15
+// CHECK: encoding: [0x8f,0x69,0x00,0x98,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vpshab %xmm15, 485498096, %xmm15
+
+// CHECK: vpshab %xmm15, 64(%rdx,%rax,4), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x00,0x98,0x7c,0x82,0x40]
+vpshab %xmm15, 64(%rdx,%rax,4), %xmm15
+
+// CHECK: vpshab %xmm15, -64(%rdx,%rax,4), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x00,0x98,0x7c,0x82,0xc0]
+vpshab %xmm15, -64(%rdx,%rax,4), %xmm15
+
+// CHECK: vpshab %xmm15, 64(%rdx,%rax), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x00,0x98,0x7c,0x02,0x40]
+vpshab %xmm15, 64(%rdx,%rax), %xmm15
+
+// CHECK: vpshab %xmm15, 64(%rdx), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x00,0x98,0x7a,0x40]
+vpshab %xmm15, 64(%rdx), %xmm15
+
+// CHECK: vpshab %xmm15, (%rdx), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x00,0x98,0x3a]
+vpshab %xmm15, (%rdx), %xmm15
+
+// CHECK: vpshab %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x49,0x00,0x98,0xff]
+vpshab %xmm15, %xmm15, %xmm15
+
+// CHECK: vpshab %xmm6, 485498096, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x48,0x98,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+vpshab %xmm6, 485498096, %xmm6
+
+// CHECK: vpshab %xmm6, 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x48,0x98,0x74,0x82,0x40]
+vpshab %xmm6, 64(%rdx,%rax,4), %xmm6
+
+// CHECK: vpshab %xmm6, -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x48,0x98,0x74,0x82,0xc0]
+vpshab %xmm6, -64(%rdx,%rax,4), %xmm6
+
+// CHECK: vpshab %xmm6, 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x48,0x98,0x74,0x02,0x40]
+vpshab %xmm6, 64(%rdx,%rax), %xmm6
+
+// CHECK: vpshab %xmm6, 64(%rdx), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x48,0x98,0x72,0x40]
+vpshab %xmm6, 64(%rdx), %xmm6
+
+// CHECK: vpshab %xmm6, (%rdx), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x48,0x98,0x32]
+vpshab %xmm6, (%rdx), %xmm6
+
+// CHECK: vpshab %xmm6, %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x48,0x98,0xf6]
+vpshab %xmm6, %xmm6, %xmm6
+
+// CHECK: vpshad 485498096, %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x69,0x80,0x9a,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vpshad 485498096, %xmm15, %xmm15
+
+// CHECK: vpshad 485498096, %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0xc8,0x9a,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+vpshad 485498096, %xmm6, %xmm6
+
+// CHECK: vpshad 64(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x69,0x80,0x9a,0x7c,0x82,0x40]
+vpshad 64(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vpshad -64(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x69,0x80,0x9a,0x7c,0x82,0xc0]
+vpshad -64(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vpshad 64(%rdx,%rax,4), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0xc8,0x9a,0x74,0x82,0x40]
+vpshad 64(%rdx,%rax,4), %xmm6, %xmm6
+
+// CHECK: vpshad -64(%rdx,%rax,4), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0xc8,0x9a,0x74,0x82,0xc0]
+vpshad -64(%rdx,%rax,4), %xmm6, %xmm6
+
+// CHECK: vpshad 64(%rdx,%rax), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x69,0x80,0x9a,0x7c,0x02,0x40]
+vpshad 64(%rdx,%rax), %xmm15, %xmm15
+
+// CHECK: vpshad 64(%rdx,%rax), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0xc8,0x9a,0x74,0x02,0x40]
+vpshad 64(%rdx,%rax), %xmm6, %xmm6
+
+// CHECK: vpshad 64(%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x69,0x80,0x9a,0x7a,0x40]
+vpshad 64(%rdx), %xmm15, %xmm15
+
+// CHECK: vpshad 64(%rdx), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0xc8,0x9a,0x72,0x40]
+vpshad 64(%rdx), %xmm6, %xmm6
+
+// CHECK: vpshad (%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x69,0x80,0x9a,0x3a]
+vpshad (%rdx), %xmm15, %xmm15
+
+// CHECK: vpshad (%rdx), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0xc8,0x9a,0x32]
+vpshad (%rdx), %xmm6, %xmm6
+
+// CHECK: vpshad %xmm15, 485498096, %xmm15
+// CHECK: encoding: [0x8f,0x69,0x00,0x9a,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vpshad %xmm15, 485498096, %xmm15
+
+// CHECK: vpshad %xmm15, 64(%rdx,%rax,4), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x00,0x9a,0x7c,0x82,0x40]
+vpshad %xmm15, 64(%rdx,%rax,4), %xmm15
+
+// CHECK: vpshad %xmm15, -64(%rdx,%rax,4), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x00,0x9a,0x7c,0x82,0xc0]
+vpshad %xmm15, -64(%rdx,%rax,4), %xmm15
+
+// CHECK: vpshad %xmm15, 64(%rdx,%rax), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x00,0x9a,0x7c,0x02,0x40]
+vpshad %xmm15, 64(%rdx,%rax), %xmm15
+
+// CHECK: vpshad %xmm15, 64(%rdx), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x00,0x9a,0x7a,0x40]
+vpshad %xmm15, 64(%rdx), %xmm15
+
+// CHECK: vpshad %xmm15, (%rdx), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x00,0x9a,0x3a]
+vpshad %xmm15, (%rdx), %xmm15
+
+// CHECK: vpshad %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x49,0x00,0x9a,0xff]
+vpshad %xmm15, %xmm15, %xmm15
+
+// CHECK: vpshad %xmm6, 485498096, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x48,0x9a,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+vpshad %xmm6, 485498096, %xmm6
+
+// CHECK: vpshad %xmm6, 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x48,0x9a,0x74,0x82,0x40]
+vpshad %xmm6, 64(%rdx,%rax,4), %xmm6
+
+// CHECK: vpshad %xmm6, -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x48,0x9a,0x74,0x82,0xc0]
+vpshad %xmm6, -64(%rdx,%rax,4), %xmm6
+
+// CHECK: vpshad %xmm6, 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x48,0x9a,0x74,0x02,0x40]
+vpshad %xmm6, 64(%rdx,%rax), %xmm6
+
+// CHECK: vpshad %xmm6, 64(%rdx), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x48,0x9a,0x72,0x40]
+vpshad %xmm6, 64(%rdx), %xmm6
+
+// CHECK: vpshad %xmm6, (%rdx), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x48,0x9a,0x32]
+vpshad %xmm6, (%rdx), %xmm6
+
+// CHECK: vpshad %xmm6, %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x48,0x9a,0xf6]
+vpshad %xmm6, %xmm6, %xmm6
+
+// CHECK: vpshaq 485498096, %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x69,0x80,0x9b,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vpshaq 485498096, %xmm15, %xmm15
+
+// CHECK: vpshaq 485498096, %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0xc8,0x9b,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+vpshaq 485498096, %xmm6, %xmm6
+
+// CHECK: vpshaq 64(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x69,0x80,0x9b,0x7c,0x82,0x40]
+vpshaq 64(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vpshaq -64(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x69,0x80,0x9b,0x7c,0x82,0xc0]
+vpshaq -64(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vpshaq 64(%rdx,%rax,4), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0xc8,0x9b,0x74,0x82,0x40]
+vpshaq 64(%rdx,%rax,4), %xmm6, %xmm6
+
+// CHECK: vpshaq -64(%rdx,%rax,4), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0xc8,0x9b,0x74,0x82,0xc0]
+vpshaq -64(%rdx,%rax,4), %xmm6, %xmm6
+
+// CHECK: vpshaq 64(%rdx,%rax), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x69,0x80,0x9b,0x7c,0x02,0x40]
+vpshaq 64(%rdx,%rax), %xmm15, %xmm15
+
+// CHECK: vpshaq 64(%rdx,%rax), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0xc8,0x9b,0x74,0x02,0x40]
+vpshaq 64(%rdx,%rax), %xmm6, %xmm6
+
+// CHECK: vpshaq 64(%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x69,0x80,0x9b,0x7a,0x40]
+vpshaq 64(%rdx), %xmm15, %xmm15
+
+// CHECK: vpshaq 64(%rdx), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0xc8,0x9b,0x72,0x40]
+vpshaq 64(%rdx), %xmm6, %xmm6
+
+// CHECK: vpshaq (%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x69,0x80,0x9b,0x3a]
+vpshaq (%rdx), %xmm15, %xmm15
+
+// CHECK: vpshaq (%rdx), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0xc8,0x9b,0x32]
+vpshaq (%rdx), %xmm6, %xmm6
+
+// CHECK: vpshaq %xmm15, 485498096, %xmm15
+// CHECK: encoding: [0x8f,0x69,0x00,0x9b,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vpshaq %xmm15, 485498096, %xmm15
+
+// CHECK: vpshaq %xmm15, 64(%rdx,%rax,4), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x00,0x9b,0x7c,0x82,0x40]
+vpshaq %xmm15, 64(%rdx,%rax,4), %xmm15
+
+// CHECK: vpshaq %xmm15, -64(%rdx,%rax,4), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x00,0x9b,0x7c,0x82,0xc0]
+vpshaq %xmm15, -64(%rdx,%rax,4), %xmm15
+
+// CHECK: vpshaq %xmm15, 64(%rdx,%rax), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x00,0x9b,0x7c,0x02,0x40]
+vpshaq %xmm15, 64(%rdx,%rax), %xmm15
+
+// CHECK: vpshaq %xmm15, 64(%rdx), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x00,0x9b,0x7a,0x40]
+vpshaq %xmm15, 64(%rdx), %xmm15
+
+// CHECK: vpshaq %xmm15, (%rdx), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x00,0x9b,0x3a]
+vpshaq %xmm15, (%rdx), %xmm15
+
+// CHECK: vpshaq %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x49,0x00,0x9b,0xff]
+vpshaq %xmm15, %xmm15, %xmm15
+
+// CHECK: vpshaq %xmm6, 485498096, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x48,0x9b,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+vpshaq %xmm6, 485498096, %xmm6
+
+// CHECK: vpshaq %xmm6, 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x48,0x9b,0x74,0x82,0x40]
+vpshaq %xmm6, 64(%rdx,%rax,4), %xmm6
+
+// CHECK: vpshaq %xmm6, -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x48,0x9b,0x74,0x82,0xc0]
+vpshaq %xmm6, -64(%rdx,%rax,4), %xmm6
+
+// CHECK: vpshaq %xmm6, 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x48,0x9b,0x74,0x02,0x40]
+vpshaq %xmm6, 64(%rdx,%rax), %xmm6
+
+// CHECK: vpshaq %xmm6, 64(%rdx), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x48,0x9b,0x72,0x40]
+vpshaq %xmm6, 64(%rdx), %xmm6
+
+// CHECK: vpshaq %xmm6, (%rdx), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x48,0x9b,0x32]
+vpshaq %xmm6, (%rdx), %xmm6
+
+// CHECK: vpshaq %xmm6, %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x48,0x9b,0xf6]
+vpshaq %xmm6, %xmm6, %xmm6
+
+// CHECK: vpshaw 485498096, %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x69,0x80,0x99,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vpshaw 485498096, %xmm15, %xmm15
+
+// CHECK: vpshaw 485498096, %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0xc8,0x99,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+vpshaw 485498096, %xmm6, %xmm6
+
+// CHECK: vpshaw 64(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x69,0x80,0x99,0x7c,0x82,0x40]
+vpshaw 64(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vpshaw -64(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x69,0x80,0x99,0x7c,0x82,0xc0]
+vpshaw -64(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vpshaw 64(%rdx,%rax,4), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0xc8,0x99,0x74,0x82,0x40]
+vpshaw 64(%rdx,%rax,4), %xmm6, %xmm6
+
+// CHECK: vpshaw -64(%rdx,%rax,4), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0xc8,0x99,0x74,0x82,0xc0]
+vpshaw -64(%rdx,%rax,4), %xmm6, %xmm6
+
+// CHECK: vpshaw 64(%rdx,%rax), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x69,0x80,0x99,0x7c,0x02,0x40]
+vpshaw 64(%rdx,%rax), %xmm15, %xmm15
+
+// CHECK: vpshaw 64(%rdx,%rax), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0xc8,0x99,0x74,0x02,0x40]
+vpshaw 64(%rdx,%rax), %xmm6, %xmm6
+
+// CHECK: vpshaw 64(%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x69,0x80,0x99,0x7a,0x40]
+vpshaw 64(%rdx), %xmm15, %xmm15
+
+// CHECK: vpshaw 64(%rdx), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0xc8,0x99,0x72,0x40]
+vpshaw 64(%rdx), %xmm6, %xmm6
+
+// CHECK: vpshaw (%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x69,0x80,0x99,0x3a]
+vpshaw (%rdx), %xmm15, %xmm15
+
+// CHECK: vpshaw (%rdx), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0xc8,0x99,0x32]
+vpshaw (%rdx), %xmm6, %xmm6
+
+// CHECK: vpshaw %xmm15, 485498096, %xmm15
+// CHECK: encoding: [0x8f,0x69,0x00,0x99,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vpshaw %xmm15, 485498096, %xmm15
+
+// CHECK: vpshaw %xmm15, 64(%rdx,%rax,4), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x00,0x99,0x7c,0x82,0x40]
+vpshaw %xmm15, 64(%rdx,%rax,4), %xmm15
+
+// CHECK: vpshaw %xmm15, -64(%rdx,%rax,4), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x00,0x99,0x7c,0x82,0xc0]
+vpshaw %xmm15, -64(%rdx,%rax,4), %xmm15
+
+// CHECK: vpshaw %xmm15, 64(%rdx,%rax), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x00,0x99,0x7c,0x02,0x40]
+vpshaw %xmm15, 64(%rdx,%rax), %xmm15
+
+// CHECK: vpshaw %xmm15, 64(%rdx), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x00,0x99,0x7a,0x40]
+vpshaw %xmm15, 64(%rdx), %xmm15
+
+// CHECK: vpshaw %xmm15, (%rdx), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x00,0x99,0x3a]
+vpshaw %xmm15, (%rdx), %xmm15
+
+// CHECK: vpshaw %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x49,0x00,0x99,0xff]
+vpshaw %xmm15, %xmm15, %xmm15
+
+// CHECK: vpshaw %xmm6, 485498096, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x48,0x99,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+vpshaw %xmm6, 485498096, %xmm6
+
+// CHECK: vpshaw %xmm6, 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x48,0x99,0x74,0x82,0x40]
+vpshaw %xmm6, 64(%rdx,%rax,4), %xmm6
+
+// CHECK: vpshaw %xmm6, -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x48,0x99,0x74,0x82,0xc0]
+vpshaw %xmm6, -64(%rdx,%rax,4), %xmm6
+
+// CHECK: vpshaw %xmm6, 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x48,0x99,0x74,0x02,0x40]
+vpshaw %xmm6, 64(%rdx,%rax), %xmm6
+
+// CHECK: vpshaw %xmm6, 64(%rdx), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x48,0x99,0x72,0x40]
+vpshaw %xmm6, 64(%rdx), %xmm6
+
+// CHECK: vpshaw %xmm6, (%rdx), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x48,0x99,0x32]
+vpshaw %xmm6, (%rdx), %xmm6
+
+// CHECK: vpshaw %xmm6, %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x48,0x99,0xf6]
+vpshaw %xmm6, %xmm6, %xmm6
+
+// CHECK: vpshlb 485498096, %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x69,0x80,0x94,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vpshlb 485498096, %xmm15, %xmm15
+
+// CHECK: vpshlb 485498096, %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0xc8,0x94,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+vpshlb 485498096, %xmm6, %xmm6
+
+// CHECK: vpshlb 64(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x69,0x80,0x94,0x7c,0x82,0x40]
+vpshlb 64(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vpshlb -64(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x69,0x80,0x94,0x7c,0x82,0xc0]
+vpshlb -64(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vpshlb 64(%rdx,%rax,4), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0xc8,0x94,0x74,0x82,0x40]
+vpshlb 64(%rdx,%rax,4), %xmm6, %xmm6
+
+// CHECK: vpshlb -64(%rdx,%rax,4), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0xc8,0x94,0x74,0x82,0xc0]
+vpshlb -64(%rdx,%rax,4), %xmm6, %xmm6
+
+// CHECK: vpshlb 64(%rdx,%rax), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x69,0x80,0x94,0x7c,0x02,0x40]
+vpshlb 64(%rdx,%rax), %xmm15, %xmm15
+
+// CHECK: vpshlb 64(%rdx,%rax), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0xc8,0x94,0x74,0x02,0x40]
+vpshlb 64(%rdx,%rax), %xmm6, %xmm6
+
+// CHECK: vpshlb 64(%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x69,0x80,0x94,0x7a,0x40]
+vpshlb 64(%rdx), %xmm15, %xmm15
+
+// CHECK: vpshlb 64(%rdx), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0xc8,0x94,0x72,0x40]
+vpshlb 64(%rdx), %xmm6, %xmm6
+
+// CHECK: vpshlb (%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x69,0x80,0x94,0x3a]
+vpshlb (%rdx), %xmm15, %xmm15
+
+// CHECK: vpshlb (%rdx), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0xc8,0x94,0x32]
+vpshlb (%rdx), %xmm6, %xmm6
+
+// CHECK: vpshlb %xmm15, 485498096, %xmm15
+// CHECK: encoding: [0x8f,0x69,0x00,0x94,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vpshlb %xmm15, 485498096, %xmm15
+
+// CHECK: vpshlb %xmm15, 64(%rdx,%rax,4), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x00,0x94,0x7c,0x82,0x40]
+vpshlb %xmm15, 64(%rdx,%rax,4), %xmm15
+
+// CHECK: vpshlb %xmm15, -64(%rdx,%rax,4), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x00,0x94,0x7c,0x82,0xc0]
+vpshlb %xmm15, -64(%rdx,%rax,4), %xmm15
+
+// CHECK: vpshlb %xmm15, 64(%rdx,%rax), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x00,0x94,0x7c,0x02,0x40]
+vpshlb %xmm15, 64(%rdx,%rax), %xmm15
+
+// CHECK: vpshlb %xmm15, 64(%rdx), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x00,0x94,0x7a,0x40]
+vpshlb %xmm15, 64(%rdx), %xmm15
+
+// CHECK: vpshlb %xmm15, (%rdx), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x00,0x94,0x3a]
+vpshlb %xmm15, (%rdx), %xmm15
+
+// CHECK: vpshlb %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x49,0x00,0x94,0xff]
+vpshlb %xmm15, %xmm15, %xmm15
+
+// CHECK: vpshlb %xmm6, 485498096, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x48,0x94,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+vpshlb %xmm6, 485498096, %xmm6
+
+// CHECK: vpshlb %xmm6, 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x48,0x94,0x74,0x82,0x40]
+vpshlb %xmm6, 64(%rdx,%rax,4), %xmm6
+
+// CHECK: vpshlb %xmm6, -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x48,0x94,0x74,0x82,0xc0]
+vpshlb %xmm6, -64(%rdx,%rax,4), %xmm6
+
+// CHECK: vpshlb %xmm6, 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x48,0x94,0x74,0x02,0x40]
+vpshlb %xmm6, 64(%rdx,%rax), %xmm6
+
+// CHECK: vpshlb %xmm6, 64(%rdx), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x48,0x94,0x72,0x40]
+vpshlb %xmm6, 64(%rdx), %xmm6
+
+// CHECK: vpshlb %xmm6, (%rdx), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x48,0x94,0x32]
+vpshlb %xmm6, (%rdx), %xmm6
+
+// CHECK: vpshlb %xmm6, %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x48,0x94,0xf6]
+vpshlb %xmm6, %xmm6, %xmm6
+
+// CHECK: vpshld 485498096, %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x69,0x80,0x96,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vpshld 485498096, %xmm15, %xmm15
+
+// CHECK: vpshld 485498096, %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0xc8,0x96,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+vpshld 485498096, %xmm6, %xmm6
+
+// CHECK: vpshld 64(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x69,0x80,0x96,0x7c,0x82,0x40]
+vpshld 64(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vpshld -64(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x69,0x80,0x96,0x7c,0x82,0xc0]
+vpshld -64(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vpshld 64(%rdx,%rax,4), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0xc8,0x96,0x74,0x82,0x40]
+vpshld 64(%rdx,%rax,4), %xmm6, %xmm6
+
+// CHECK: vpshld -64(%rdx,%rax,4), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0xc8,0x96,0x74,0x82,0xc0]
+vpshld -64(%rdx,%rax,4), %xmm6, %xmm6
+
+// CHECK: vpshld 64(%rdx,%rax), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x69,0x80,0x96,0x7c,0x02,0x40]
+vpshld 64(%rdx,%rax), %xmm15, %xmm15
+
+// CHECK: vpshld 64(%rdx,%rax), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0xc8,0x96,0x74,0x02,0x40]
+vpshld 64(%rdx,%rax), %xmm6, %xmm6
+
+// CHECK: vpshld 64(%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x69,0x80,0x96,0x7a,0x40]
+vpshld 64(%rdx), %xmm15, %xmm15
+
+// CHECK: vpshld 64(%rdx), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0xc8,0x96,0x72,0x40]
+vpshld 64(%rdx), %xmm6, %xmm6
+
+// CHECK: vpshld (%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x69,0x80,0x96,0x3a]
+vpshld (%rdx), %xmm15, %xmm15
+
+// CHECK: vpshld (%rdx), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0xc8,0x96,0x32]
+vpshld (%rdx), %xmm6, %xmm6
+
+// CHECK: vpshld %xmm15, 485498096, %xmm15
+// CHECK: encoding: [0x8f,0x69,0x00,0x96,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vpshld %xmm15, 485498096, %xmm15
+
+// CHECK: vpshld %xmm15, 64(%rdx,%rax,4), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x00,0x96,0x7c,0x82,0x40]
+vpshld %xmm15, 64(%rdx,%rax,4), %xmm15
+
+// CHECK: vpshld %xmm15, -64(%rdx,%rax,4), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x00,0x96,0x7c,0x82,0xc0]
+vpshld %xmm15, -64(%rdx,%rax,4), %xmm15
+
+// CHECK: vpshld %xmm15, 64(%rdx,%rax), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x00,0x96,0x7c,0x02,0x40]
+vpshld %xmm15, 64(%rdx,%rax), %xmm15
+
+// CHECK: vpshld %xmm15, 64(%rdx), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x00,0x96,0x7a,0x40]
+vpshld %xmm15, 64(%rdx), %xmm15
+
+// CHECK: vpshld %xmm15, (%rdx), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x00,0x96,0x3a]
+vpshld %xmm15, (%rdx), %xmm15
+
+// CHECK: vpshld %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x49,0x00,0x96,0xff]
+vpshld %xmm15, %xmm15, %xmm15
+
+// CHECK: vpshld %xmm6, 485498096, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x48,0x96,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+vpshld %xmm6, 485498096, %xmm6
+
+// CHECK: vpshld %xmm6, 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x48,0x96,0x74,0x82,0x40]
+vpshld %xmm6, 64(%rdx,%rax,4), %xmm6
+
+// CHECK: vpshld %xmm6, -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x48,0x96,0x74,0x82,0xc0]
+vpshld %xmm6, -64(%rdx,%rax,4), %xmm6
+
+// CHECK: vpshld %xmm6, 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x48,0x96,0x74,0x02,0x40]
+vpshld %xmm6, 64(%rdx,%rax), %xmm6
+
+// CHECK: vpshld %xmm6, 64(%rdx), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x48,0x96,0x72,0x40]
+vpshld %xmm6, 64(%rdx), %xmm6
+
+// CHECK: vpshld %xmm6, (%rdx), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x48,0x96,0x32]
+vpshld %xmm6, (%rdx), %xmm6
+
+// CHECK: vpshld %xmm6, %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x48,0x96,0xf6]
+vpshld %xmm6, %xmm6, %xmm6
+
+// CHECK: vpshlq 485498096, %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x69,0x80,0x97,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vpshlq 485498096, %xmm15, %xmm15
+
+// CHECK: vpshlq 485498096, %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0xc8,0x97,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+vpshlq 485498096, %xmm6, %xmm6
+
+// CHECK: vpshlq 64(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x69,0x80,0x97,0x7c,0x82,0x40]
+vpshlq 64(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vpshlq -64(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x69,0x80,0x97,0x7c,0x82,0xc0]
+vpshlq -64(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vpshlq 64(%rdx,%rax,4), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0xc8,0x97,0x74,0x82,0x40]
+vpshlq 64(%rdx,%rax,4), %xmm6, %xmm6
+
+// CHECK: vpshlq -64(%rdx,%rax,4), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0xc8,0x97,0x74,0x82,0xc0]
+vpshlq -64(%rdx,%rax,4), %xmm6, %xmm6
+
+// CHECK: vpshlq 64(%rdx,%rax), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x69,0x80,0x97,0x7c,0x02,0x40]
+vpshlq 64(%rdx,%rax), %xmm15, %xmm15
+
+// CHECK: vpshlq 64(%rdx,%rax), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0xc8,0x97,0x74,0x02,0x40]
+vpshlq 64(%rdx,%rax), %xmm6, %xmm6
+
+// CHECK: vpshlq 64(%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x69,0x80,0x97,0x7a,0x40]
+vpshlq 64(%rdx), %xmm15, %xmm15
+
+// CHECK: vpshlq 64(%rdx), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0xc8,0x97,0x72,0x40]
+vpshlq 64(%rdx), %xmm6, %xmm6
+
+// CHECK: vpshlq (%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x69,0x80,0x97,0x3a]
+vpshlq (%rdx), %xmm15, %xmm15
+
+// CHECK: vpshlq (%rdx), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0xc8,0x97,0x32]
+vpshlq (%rdx), %xmm6, %xmm6
+
+// CHECK: vpshlq %xmm15, 485498096, %xmm15
+// CHECK: encoding: [0x8f,0x69,0x00,0x97,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vpshlq %xmm15, 485498096, %xmm15
+
+// CHECK: vpshlq %xmm15, 64(%rdx,%rax,4), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x00,0x97,0x7c,0x82,0x40]
+vpshlq %xmm15, 64(%rdx,%rax,4), %xmm15
+
+// CHECK: vpshlq %xmm15, -64(%rdx,%rax,4), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x00,0x97,0x7c,0x82,0xc0]
+vpshlq %xmm15, -64(%rdx,%rax,4), %xmm15
+
+// CHECK: vpshlq %xmm15, 64(%rdx,%rax), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x00,0x97,0x7c,0x02,0x40]
+vpshlq %xmm15, 64(%rdx,%rax), %xmm15
+
+// CHECK: vpshlq %xmm15, 64(%rdx), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x00,0x97,0x7a,0x40]
+vpshlq %xmm15, 64(%rdx), %xmm15
+
+// CHECK: vpshlq %xmm15, (%rdx), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x00,0x97,0x3a]
+vpshlq %xmm15, (%rdx), %xmm15
+
+// CHECK: vpshlq %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x49,0x00,0x97,0xff]
+vpshlq %xmm15, %xmm15, %xmm15
+
+// CHECK: vpshlq %xmm6, 485498096, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x48,0x97,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+vpshlq %xmm6, 485498096, %xmm6
+
+// CHECK: vpshlq %xmm6, 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x48,0x97,0x74,0x82,0x40]
+vpshlq %xmm6, 64(%rdx,%rax,4), %xmm6
+
+// CHECK: vpshlq %xmm6, -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x48,0x97,0x74,0x82,0xc0]
+vpshlq %xmm6, -64(%rdx,%rax,4), %xmm6
+
+// CHECK: vpshlq %xmm6, 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x48,0x97,0x74,0x02,0x40]
+vpshlq %xmm6, 64(%rdx,%rax), %xmm6
+
+// CHECK: vpshlq %xmm6, 64(%rdx), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x48,0x97,0x72,0x40]
+vpshlq %xmm6, 64(%rdx), %xmm6
+
+// CHECK: vpshlq %xmm6, (%rdx), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x48,0x97,0x32]
+vpshlq %xmm6, (%rdx), %xmm6
+
+// CHECK: vpshlq %xmm6, %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x48,0x97,0xf6]
+vpshlq %xmm6, %xmm6, %xmm6
+
+// CHECK: vpshlw 485498096, %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x69,0x80,0x95,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vpshlw 485498096, %xmm15, %xmm15
+
+// CHECK: vpshlw 485498096, %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0xc8,0x95,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+vpshlw 485498096, %xmm6, %xmm6
+
+// CHECK: vpshlw 64(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x69,0x80,0x95,0x7c,0x82,0x40]
+vpshlw 64(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vpshlw -64(%rdx,%rax,4), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x69,0x80,0x95,0x7c,0x82,0xc0]
+vpshlw -64(%rdx,%rax,4), %xmm15, %xmm15
+
+// CHECK: vpshlw 64(%rdx,%rax,4), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0xc8,0x95,0x74,0x82,0x40]
+vpshlw 64(%rdx,%rax,4), %xmm6, %xmm6
+
+// CHECK: vpshlw -64(%rdx,%rax,4), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0xc8,0x95,0x74,0x82,0xc0]
+vpshlw -64(%rdx,%rax,4), %xmm6, %xmm6
+
+// CHECK: vpshlw 64(%rdx,%rax), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x69,0x80,0x95,0x7c,0x02,0x40]
+vpshlw 64(%rdx,%rax), %xmm15, %xmm15
+
+// CHECK: vpshlw 64(%rdx,%rax), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0xc8,0x95,0x74,0x02,0x40]
+vpshlw 64(%rdx,%rax), %xmm6, %xmm6
+
+// CHECK: vpshlw 64(%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x69,0x80,0x95,0x7a,0x40]
+vpshlw 64(%rdx), %xmm15, %xmm15
+
+// CHECK: vpshlw 64(%rdx), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0xc8,0x95,0x72,0x40]
+vpshlw 64(%rdx), %xmm6, %xmm6
+
+// CHECK: vpshlw (%rdx), %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x69,0x80,0x95,0x3a]
+vpshlw (%rdx), %xmm15, %xmm15
+
+// CHECK: vpshlw (%rdx), %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0xc8,0x95,0x32]
+vpshlw (%rdx), %xmm6, %xmm6
+
+// CHECK: vpshlw %xmm15, 485498096, %xmm15
+// CHECK: encoding: [0x8f,0x69,0x00,0x95,0x3c,0x25,0xf0,0x1c,0xf0,0x1c]
+vpshlw %xmm15, 485498096, %xmm15
+
+// CHECK: vpshlw %xmm15, 64(%rdx,%rax,4), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x00,0x95,0x7c,0x82,0x40]
+vpshlw %xmm15, 64(%rdx,%rax,4), %xmm15
+
+// CHECK: vpshlw %xmm15, -64(%rdx,%rax,4), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x00,0x95,0x7c,0x82,0xc0]
+vpshlw %xmm15, -64(%rdx,%rax,4), %xmm15
+
+// CHECK: vpshlw %xmm15, 64(%rdx,%rax), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x00,0x95,0x7c,0x02,0x40]
+vpshlw %xmm15, 64(%rdx,%rax), %xmm15
+
+// CHECK: vpshlw %xmm15, 64(%rdx), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x00,0x95,0x7a,0x40]
+vpshlw %xmm15, 64(%rdx), %xmm15
+
+// CHECK: vpshlw %xmm15, (%rdx), %xmm15
+// CHECK: encoding: [0x8f,0x69,0x00,0x95,0x3a]
+vpshlw %xmm15, (%rdx), %xmm15
+
+// CHECK: vpshlw %xmm15, %xmm15, %xmm15
+// CHECK: encoding: [0x8f,0x49,0x00,0x95,0xff]
+vpshlw %xmm15, %xmm15, %xmm15
+
+// CHECK: vpshlw %xmm6, 485498096, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x48,0x95,0x34,0x25,0xf0,0x1c,0xf0,0x1c]
+vpshlw %xmm6, 485498096, %xmm6
+
+// CHECK: vpshlw %xmm6, 64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x48,0x95,0x74,0x82,0x40]
+vpshlw %xmm6, 64(%rdx,%rax,4), %xmm6
+
+// CHECK: vpshlw %xmm6, -64(%rdx,%rax,4), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x48,0x95,0x74,0x82,0xc0]
+vpshlw %xmm6, -64(%rdx,%rax,4), %xmm6
+
+// CHECK: vpshlw %xmm6, 64(%rdx,%rax), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x48,0x95,0x74,0x02,0x40]
+vpshlw %xmm6, 64(%rdx,%rax), %xmm6
+
+// CHECK: vpshlw %xmm6, 64(%rdx), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x48,0x95,0x72,0x40]
+vpshlw %xmm6, 64(%rdx), %xmm6
+
+// CHECK: vpshlw %xmm6, (%rdx), %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x48,0x95,0x32]
+vpshlw %xmm6, (%rdx), %xmm6
+
+// CHECK: vpshlw %xmm6, %xmm6, %xmm6
+// CHECK: encoding: [0x8f,0xe9,0x48,0x95,0xf6]
+vpshlw %xmm6, %xmm6, %xmm6
+
diff --git a/test/MC/X86/compact-unwind.s b/test/MC/X86/compact-unwind.s
index 70fc018f78860f7497fe6650c9fa662fa7734d45..03138c4034059e80b054c3a0d9d4ea998e0ccd79 100644
--- a/test/MC/X86/compact-unwind.s
+++ b/test/MC/X86/compact-unwind.s
@@ -64,6 +64,24 @@ Ltmp14:
 	retq
 	.cfi_endproc
 
+# Check that a adjustment through a push %rax is the same as a sub.
+
+# CHECK:   Entry at offset 0x40:
+# CHECK-NEXT:     start:                0x2a _testrax
+# CHECK-NEXT:     length:               0x5
+# CHECK-NEXT:     compact encoding:     0x02020000
+	.globl	_testrax
+_testrax:                               ## @testrax
+	.cfi_startproc
+## %bb.0:                               ## %entry
+	pushq	%rax
+Ltmp15:
+	.cfi_def_cfa_offset 16
+	xorl	%eax, %eax
+	popq	%rax
+	retq
+	.cfi_endproc
+
 	.section	__TEXT,__cstring,cstring_literals
 L_.str:                                 ## @.str
 	.asciz	"%d\n"
diff --git a/test/MC/X86/disassemble-zeroes.s b/test/MC/X86/disassemble-zeroes.s
new file mode 100644
index 0000000000000000000000000000000000000000..2ecfdad0c2d91b2b0374ac0748b226acc24d61df
--- /dev/null
+++ b/test/MC/X86/disassemble-zeroes.s
@@ -0,0 +1,81 @@
+// RUN: llvm-mc -filetype=obj -triple x86_64-unknown-unknown %s -o %t
+// RUN: llvm-objdump -d %t | FileCheck %s --check-prefix=NODISASM
+
+// The exact rules of skipping the bytes you can find in the code.
+// This test checks that we follow these rules and can force
+// dissasembly of zero blocks with the -z and --disassemble-zeroes options.
+
+// NODISASM:      Disassembly of section .text:
+// NODISASM-NEXT:  0000000000000000 main:
+// NODISASM-NEXT:   0:  00 00               addb %al, (%rax)
+// NODISASM-NEXT:   2:  00 00               addb %al, (%rax)
+// NODISASM-NEXT:   4:  00 00               addb %al, (%rax)
+// NODISASM-NEXT:   6:  00 90 00 00 00 00   addb %dl, (%rax)
+// NODISASM-NEXT:       ...
+// NODISASM-NEXT:   20: 90                  nop
+// NODISASM-NEXT:       ...
+// NODISASM:      0000000000000031 foo:
+// NODISASM-NEXT:   31: 00 00               addb %al, (%rax)
+// NODISASM-NEXT:   33: 00 00               addb %al, (%rax)
+// NODISASM:      0000000000000035 bar:
+// NODISASM-NEXT:       ...
+
+// Check that with -z we disassemble blocks of zeroes.
+// RUN: llvm-objdump -d -z %t | FileCheck %s --check-prefix=DISASM
+
+// DISASM: Disassembly of section .text:
+// DISASM-NEXT: 0000000000000000 main:
+// DISASM-NEXT:   0: 00 00              addb %al, (%rax)
+// DISASM-NEXT:   2: 00 00              addb %al, (%rax)
+// DISASM-NEXT:   4: 00 00              addb %al, (%rax)
+// DISASM-NEXT:   6: 00 90 00 00 00 00  addb %dl, (%rax)
+// DISASM-NEXT:   c: 00 00              addb %al, (%rax)
+// DISASM-NEXT:   e: 00 00              addb %al, (%rax)
+// DISASM-NEXT:  10: 00 00              addb %al, (%rax)
+// DISASM-NEXT:  12: 00 00              addb %al, (%rax)
+// DISASM-NEXT:  14: 00 00              addb %al, (%rax)
+// DISASM-NEXT:  16: 00 00              addb %al, (%rax)
+// DISASM-NEXT:  18: 00 00              addb %al, (%rax)
+// DISASM-NEXT:  1a: 00 00              addb %al, (%rax)
+// DISASM-NEXT:  1c: 00 00              addb %al, (%rax)
+// DISASM-NEXT:  1e: 00 00              addb %al, (%rax)
+// DISASM-NEXT:  20: 90                 nop
+// DISASM-NEXT:  21: 00 00              addb %al, (%rax)
+// DISASM-NEXT:  23: 00 00              addb %al, (%rax)
+// DISASM-NEXT:  25: 00 00              addb %al, (%rax)
+// DISASM-NEXT:  27: 00 00              addb %al, (%rax)
+// DISASM-NEXT:  29: 00 00              addb %al, (%rax)
+// DISASM-NEXT:  2b: 00 00              addb %al, (%rax)
+// DISASM-NEXT:  2d: 00 00              addb %al, (%rax)
+// DISASM-NEXT:  2f: 00 00              addb %al, (%rax)
+// DISASM:      0000000000000031 foo:
+// DISASM-NEXT:  31: 00 00              addb %al, (%rax)
+// DISASM-NEXT:  33: 00 00              addb %al, (%rax)
+// DISASM:      0000000000000035 bar:
+// DISASM-NEXT:  35: 00 00              addb %al, (%rax)
+// DISASM-NEXT:  37: 00 00              addb %al, (%rax)
+// DISASM-NEXT:  39: 00 00              addb %al, (%rax)
+// DISASM-NEXT:  3b: 00 00              addb %al, (%rax)
+
+// Check that --disassemble-zeroes work as alias for -z.
+// RUN: llvm-objdump -d --disassemble-zeroes %t | FileCheck %s --check-prefix=DISASM
+
+.text
+.globl main
+.type main, @function
+main:
+ .long 0
+ .byte 0
+ .byte 0
+ .byte 0
+ nop
+ .quad 0
+ .quad 0
+ .quad 0
+ nop
+ .quad 0
+ .quad 0
+foo:
+ .long 0
+bar:
+ .quad 0
diff --git a/test/MC/X86/intel-syntax-2.s b/test/MC/X86/intel-syntax-2.s
index aead5766db4d52322b9c7bde67192e389c708c4e..b23965ae52e88ebfd2649bbb01cb1df500f22187 100644
--- a/test/MC/X86/intel-syntax-2.s
+++ b/test/MC/X86/intel-syntax-2.s
@@ -18,14 +18,14 @@ _test2:
 
 _test3:
 fadd 
-// CHECK: faddp %st(1)
+// CHECK: faddp %st, %st(1)
 fmul
-// CHECK: fmulp %st(1)
+// CHECK: fmulp %st, %st(1)
 fsub
-// CHECK: fsubp %st(1)
+// CHECK: fsubp %st, %st(1)
 fsubr
-// CHECK: fsubrp %st(1)
+// CHECK: fsubrp %st, %st(1)
 fdiv
-// CHECK: fdivp %st(1)
+// CHECK: fdivp %st, %st(1)
 fdivr
-// CHECK: fdivrp %st(1)
+// CHECK: fdivrp %st, %st(1)
diff --git a/test/MC/X86/intel-syntax.s b/test/MC/X86/intel-syntax.s
index 428a7e4ec41f1f077f3a85b6bac32652e6f5a702..171357e1f5212e5e5ee77df895e5b4794cdf6e04 100644
--- a/test/MC/X86/intel-syntax.s
+++ b/test/MC/X86/intel-syntax.s
@@ -556,12 +556,12 @@ fnstsw
 fnstsw AX
 fnstsw WORD PTR [EAX]
 
-// CHECK: faddp %st(1)
-// CHECK: fmulp %st(1)
-// CHECK: fsubrp %st(1)
-// CHECK: fsubp %st(1)
-// CHECK: fdivrp %st(1)
-// CHECK: fdivp %st(1)
+// CHECK: faddp %st, %st(1)
+// CHECK: fmulp %st, %st(1)
+// CHECK: fsubrp %st, %st(1)
+// CHECK: fsubp %st, %st(1)
+// CHECK: fdivrp %st, %st(1)
+// CHECK: fdivp %st, %st(1)
 faddp ST(1), ST(0)
 fmulp ST(1), ST(0)
 fsubp ST(1), ST(0)
@@ -569,12 +569,12 @@ fsubrp ST(1), ST(0)
 fdivp ST(1), ST(0)
 fdivrp ST(1), ST(0)
 
-// CHECK: faddp %st(1)
-// CHECK: fmulp %st(1)
-// CHECK: fsubrp %st(1)
-// CHECK: fsubp %st(1)
-// CHECK: fdivrp %st(1)
-// CHECK: fdivp %st(1)
+// CHECK: faddp %st, %st(1)
+// CHECK: fmulp %st, %st(1)
+// CHECK: fsubrp %st, %st(1)
+// CHECK: fsubp %st, %st(1)
+// CHECK: fdivrp %st, %st(1)
+// CHECK: fdivp %st, %st(1)
 faddp ST(0), ST(1)
 fmulp ST(0), ST(1)
 fsubp ST(0), ST(1)
@@ -582,12 +582,12 @@ fsubrp ST(0), ST(1)
 fdivp ST(0), ST(1)
 fdivrp ST(0), ST(1)
 
-// CHECK: faddp %st(1)
-// CHECK: fmulp %st(1)
-// CHECK: fsubrp %st(1)
-// CHECK: fsubp %st(1)
-// CHECK: fdivrp %st(1)
-// CHECK: fdivp %st(1)
+// CHECK: faddp %st, %st(1)
+// CHECK: fmulp %st, %st(1)
+// CHECK: fsubrp %st, %st(1)
+// CHECK: fsubp %st, %st(1)
+// CHECK: fdivrp %st, %st(1)
+// CHECK: fdivp %st, %st(1)
 faddp ST(1)
 fmulp ST(1)
 fsubp ST(1)
@@ -596,12 +596,12 @@ fdivp ST(1)
 fdivrp ST(1)
 
 
-// CHECK: faddp %st(1)
-// CHECK: fmulp %st(1)
-// CHECK: fsubrp %st(1)
-// CHECK: fsubp %st(1)
-// CHECK: fdivrp %st(1)
-// CHECK: fdivp %st(1)
+// CHECK: faddp %st, %st(1)
+// CHECK: fmulp %st, %st(1)
+// CHECK: fsubrp %st, %st(1)
+// CHECK: fsubp %st, %st(1)
+// CHECK: fdivrp %st, %st(1)
+// CHECK: fdivp %st, %st(1)
 fadd 
 fmul
 fsub
@@ -609,12 +609,12 @@ fsubr
 fdiv
 fdivr
 
-// CHECK: faddp %st(1)
-// CHECK: fmulp %st(1)
-// CHECK: fsubrp %st(1)
-// CHECK: fsubp %st(1)
-// CHECK: fdivrp %st(1)
-// CHECK: fdivp %st(1)
+// CHECK: faddp %st, %st(1)
+// CHECK: fmulp %st, %st(1)
+// CHECK: fsubrp %st, %st(1)
+// CHECK: fsubp %st, %st(1)
+// CHECK: fdivrp %st, %st(1)
+// CHECK: fdivp %st, %st(1)
 faddp
 fmulp
 fsubp
@@ -622,12 +622,12 @@ fsubrp
 fdivp
 fdivrp
 
-// CHECK: fadd %st(1)
-// CHECK: fmul %st(1)
-// CHECK: fsub %st(1)
-// CHECK: fsubr %st(1)
-// CHECK: fdiv %st(1)
-// CHECK: fdivr %st(1)
+// CHECK: fadd %st(1), %st
+// CHECK: fmul %st(1), %st
+// CHECK: fsub %st(1), %st
+// CHECK: fsubr %st(1), %st
+// CHECK: fdiv %st(1), %st
+// CHECK: fdivr %st(1), %st
 fadd ST(0), ST(1)
 fmul ST(0), ST(1)
 fsub ST(0), ST(1)
@@ -635,12 +635,12 @@ fsubr ST(0), ST(1)
 fdiv ST(0), ST(1)
 fdivr ST(0), ST(1)
 
-// CHECK: fadd %st(0), %st(1)
-// CHECK: fmul %st(0), %st(1)
-// CHECK: fsubr %st(0), %st(1)
-// CHECK: fsub %st(0), %st(1)
-// CHECK: fdivr %st(0), %st(1)
-// CHECK: fdiv %st(0), %st(1)
+// CHECK: fadd %st, %st(1)
+// CHECK: fmul %st, %st(1)
+// CHECK: fsubr %st, %st(1)
+// CHECK: fsub %st, %st(1)
+// CHECK: fdivr %st, %st(1)
+// CHECK: fdiv %st, %st(1)
 fadd ST(1), ST(0)
 fmul ST(1), ST(0)
 fsub ST(1), ST(0)
@@ -648,12 +648,12 @@ fsubr ST(1), ST(0)
 fdiv ST(1), ST(0)
 fdivr ST(1), ST(0)
 
-// CHECK: fadd %st(1)
-// CHECK: fmul %st(1)
-// CHECK: fsub %st(1)
-// CHECK: fsubr %st(1)
-// CHECK: fdiv %st(1)
-// CHECK: fdivr %st(1)
+// CHECK: fadd %st(1), %st
+// CHECK: fmul %st(1), %st
+// CHECK: fsub %st(1), %st
+// CHECK: fsubr %st(1), %st
+// CHECK: fdiv %st(1), %st
+// CHECK: fdivr %st(1), %st
 fadd ST(1)
 fmul ST(1)
 fsub ST(1)
diff --git a/test/Assembler/return-column.s b/test/MC/X86/return-column.s
similarity index 100%
rename from test/Assembler/return-column.s
rename to test/MC/X86/return-column.s
diff --git a/test/MC/X86/stackmap-nops.ll b/test/MC/X86/stackmap-nops.ll
index 33ef8624f1f507c8a9079d8b6a43e787f4be4dcd..97cef8bf7e272de9f245b2d7d8c5968b3de0d831 100644
--- a/test/MC/X86/stackmap-nops.ll
+++ b/test/MC/X86/stackmap-nops.ll
@@ -1,5 +1,5 @@
-; RUN: llc -mtriple=x86_64-apple-darwin -mcpu=corei7 -disable-fp-elim -filetype=obj %s -o - | llvm-objdump -d - | FileCheck %s
-; RUN: llc -mtriple=x86_64-apple-darwin -mcpu=corei7 -disable-fp-elim -filetype=asm %s -o - | llvm-mc -triple=x86_64-apple-darwin -mcpu=corei7 -filetype=obj - | llvm-objdump -d - | FileCheck %s
+; RUN: llc -mtriple=x86_64-apple-darwin -mcpu=corei7 -frame-pointer=all -filetype=obj %s -o - | llvm-objdump -d - | FileCheck %s
+; RUN: llc -mtriple=x86_64-apple-darwin -mcpu=corei7 -frame-pointer=all -filetype=asm %s -o - | llvm-mc -triple=x86_64-apple-darwin -mcpu=corei7 -filetype=obj - | llvm-objdump -d - | FileCheck %s
 
 define void @nop_test() {
 entry:
diff --git a/test/MC/X86/x86-16.s b/test/MC/X86/x86-16.s
index 50263beaa67567da4247dd29cfefb70ba721e42c..88ee77a6d4143ab55ec708894b38bd84f98a6372 100644
--- a/test/MC/X86/x86-16.s
+++ b/test/MC/X86/x86-16.s
@@ -920,11 +920,11 @@ pshufw $90, %mm4, %mm0
 	str %eax
 
 
-// CHECK: fsubp
+// CHECK: fsubp %st, %st(1)
 // CHECK: encoding: [0xde,0xe1]
 fsubp %st,%st(1)
 
-// CHECK: fsubp	%st(2)
+// CHECK: fsubp %st, %st(2)
 // CHECK: encoding: [0xde,0xe2]
 fsubp   %st, %st(2)
 
diff --git a/test/MC/X86/x86-32-coverage.s b/test/MC/X86/x86-32-coverage.s
index 32281f41216445c49ac74b04136b5e2bad4bb403..ba59fb357b9741b8a59dbe5883527d219af3902c 100644
--- a/test/MC/X86/x86-32-coverage.s
+++ b/test/MC/X86/x86-32-coverage.s
@@ -2472,11 +2472,11 @@
 // CHECK:  encoding: [0xda,0x05,0x78,0x56,0x34,0x12]
         	fiaddl	0x12345678
 
-// CHECK: faddp	%st(2)
+// CHECK: faddp	%st, %st(2)
 // CHECK:  encoding: [0xde,0xc2]
         	faddp	%st(2)
 
-// CHECK: fsub	%st(2)
+// CHECK: fsub	%st(2), %st
 // CHECK:  encoding: [0xd8,0xe2]
         	fsub	%st(2)
 
@@ -2504,11 +2504,11 @@
 // CHECK:  encoding: [0xda,0x25,0x78,0x56,0x34,0x12]
         	fisubl	0x12345678
 
-// CHECK: fsubp	%st(2)
+// CHECK: fsubp	%st, %st(2)
 // CHECK:  encoding: [0xde,0xe2]
         	fsubp	%st(2)
 
-// CHECK: fsubr	%st(2)
+// CHECK: fsubr	%st(2), %st
 // CHECK:  encoding: [0xd8,0xea]
         	fsubr	%st(2)
 
@@ -2536,11 +2536,11 @@
 // CHECK:  encoding: [0xda,0x2d,0x78,0x56,0x34,0x12]
         	fisubrl	0x12345678
 
-// CHECK: fsubrp	%st(2)
+// CHECK: fsubrp	%st, %st(2)
 // CHECK:  encoding: [0xde,0xea]
         	fsubrp	%st(2)
 
-// CHECK: fmul	%st(2)
+// CHECK: fmul	%st(2), %st
 // CHECK:  encoding: [0xd8,0xca]
         	fmul	%st(2)
 
@@ -2568,11 +2568,11 @@
 // CHECK:  encoding: [0xda,0x0d,0x78,0x56,0x34,0x12]
         	fimull	0x12345678
 
-// CHECK: fmulp	%st(2)
+// CHECK: fmulp	%st, %st(2)
 // CHECK:  encoding: [0xde,0xca]
         	fmulp	%st(2)
 
-// CHECK: fdiv	%st(2)
+// CHECK: fdiv	%st(2), %st
 // CHECK:  encoding: [0xd8,0xf2]
         	fdiv	%st(2)
 
@@ -2600,11 +2600,11 @@
 // CHECK:  encoding: [0xda,0x35,0x78,0x56,0x34,0x12]
         	fidivl	0x12345678
 
-// CHECK: fdivp	%st(2)
+// CHECK: fdivp	%st, %st(2)
 // CHECK:  encoding: [0xde,0xf2]
         	fdivp	%st(2)
 
-// CHECK: fdivr	%st(2)
+// CHECK: fdivr	%st(2), %st
 // CHECK:  encoding: [0xd8,0xfa]
         	fdivr	%st(2)
 
@@ -2632,7 +2632,7 @@
 // CHECK:  encoding: [0xda,0x3d,0x78,0x56,0x34,0x12]
         	fidivrl	0x12345678
 
-// CHECK: fdivrp	%st(2)
+// CHECK: fdivrp	%st, %st(2)
 // CHECK:  encoding: [0xde,0xfa]
         	fdivrp	%st(2)
 
@@ -2876,35 +2876,35 @@
 // CHECK:  encoding: [0x0f,0x0b]
         	ud2
 
-// CHECK: fcmovb	%st(2), %st(0)
+// CHECK: fcmovb	%st(2), %st
 // CHECK:  encoding: [0xda,0xc2]
         	fcmovb	%st(2),%st
 
-// CHECK: fcmove	%st(2), %st(0)
+// CHECK: fcmove	%st(2), %st
 // CHECK:  encoding: [0xda,0xca]
         	fcmove	%st(2),%st
 
-// CHECK: fcmovbe	%st(2), %st(0)
+// CHECK: fcmovbe	%st(2), %st
 // CHECK:  encoding: [0xda,0xd2]
         	fcmovbe	%st(2),%st
 
-// CHECK: fcmovu	 %st(2), %st(0)
+// CHECK: fcmovu	 %st(2), %st
 // CHECK:  encoding: [0xda,0xda]
         	fcmovu	%st(2),%st
 
-// CHECK: fcmovnb	%st(2), %st(0)
+// CHECK: fcmovnb	%st(2), %st
 // CHECK:  encoding: [0xdb,0xc2]
         	fcmovnb	%st(2),%st
 
-// CHECK: fcmovne	%st(2), %st(0)
+// CHECK: fcmovne	%st(2), %st
 // CHECK:  encoding: [0xdb,0xca]
         	fcmovne	%st(2),%st
 
-// CHECK: fcmovnbe	%st(2), %st(0)
+// CHECK: fcmovnbe	%st(2), %st
 // CHECK:  encoding: [0xdb,0xd2]
         	fcmovnbe	%st(2),%st
 
-// CHECK: fcmovnu	%st(2), %st(0)
+// CHECK: fcmovnu	%st(2), %st
 // CHECK:  encoding: [0xdb,0xda]
         	fcmovnu	%st(2),%st
 
diff --git a/test/MC/X86/x86-32.s b/test/MC/X86/x86-32.s
index 69d36032593453e4870ad2aa381b91d222f1b775..1593c26b9d029733673e337d852df8fccfe95ced 100644
--- a/test/MC/X86/x86-32.s
+++ b/test/MC/X86/x86-32.s
@@ -1055,7 +1055,7 @@ pshufw $90, %mm4, %mm0
 fsubp %st,%st(1)
 
 // PR9164
-// CHECK: fsubp	%st(2)
+// CHECK: fsubp %st, %st(2)
 // CHECK: encoding: [0xde,0xe2]
 fsubp   %st, %st(2)
 
diff --git a/test/MC/X86/x86-64.s b/test/MC/X86/x86-64.s
index df811a6c4cf2ee65f6299baaf8f0a0a7cdfb4d17..e35ba19fc4f2ec2cbcf8f961df4b7c3c1923d166 100644
--- a/test/MC/X86/x86-64.s
+++ b/test/MC/X86/x86-64.s
@@ -307,13 +307,13 @@ insl	(%dx), %es:(%rdi)
 // CHECK: fxch %st(1)
 // CHECK: fucom %st(1)
 // CHECK: fucomp %st(1)
-// CHECK: faddp %st(1)
-// CHECK: faddp	%st(0)
-// CHECK: fsubp %st(1)
-// CHECK: fsubrp %st(1)
-// CHECK: fmulp %st(1)
-// CHECK: fdivp %st(1)
-// CHECK: fdivrp %st(1)
+// CHECK: faddp %st, %st(1)
+// CHECK: faddp %st, %st(0)
+// CHECK: fsubp %st, %st(1)
+// CHECK: fsubrp %st, %st(1)
+// CHECK: fmulp %st, %st(1)
+// CHECK: fdivp %st, %st(1)
+// CHECK: fdivrp %st, %st(1)
 
 fxch
 fucom
@@ -416,21 +416,21 @@ enter $0x7ace,$0x7f
 mov %cs, %ax
 
 // rdar://8456391
-fcmovb %st(1), %st(0)   // CHECK: fcmovb	%st(1), %st(0)
-fcmove %st(1), %st(0)   // CHECK: fcmove	%st(1), %st(0)
-fcmovbe %st(1), %st(0)  // CHECK: fcmovbe	%st(1), %st(0)
-fcmovu %st(1), %st(0)   // CHECK: fcmovu	 %st(1), %st(0)
+fcmovb %st(1), %st   // CHECK: fcmovb	%st(1), %st
+fcmove %st(1), %st   // CHECK: fcmove	%st(1), %st
+fcmovbe %st(1), %st  // CHECK: fcmovbe	%st(1), %st
+fcmovu %st(1), %st   // CHECK: fcmovu	 %st(1), %st
 
-fcmovnb %st(1), %st(0)  // CHECK: fcmovnb	%st(1), %st(0)
-fcmovne %st(1), %st(0)  // CHECK: fcmovne	%st(1), %st(0)
-fcmovnbe %st(1), %st(0) // CHECK: fcmovnbe	%st(1), %st(0)
-fcmovnu %st(1), %st(0)  // CHECK: fcmovnu	%st(1), %st(0)
+fcmovnb %st(1), %st  // CHECK: fcmovnb	%st(1), %st
+fcmovne %st(1), %st  // CHECK: fcmovne	%st(1), %st
+fcmovnbe %st(1), %st // CHECK: fcmovnbe	%st(1), %st
+fcmovnu %st(1), %st  // CHECK: fcmovnu	%st(1), %st
 
-fcmovnae %st(1), %st(0) // CHECK: fcmovb	%st(1), %st(0)
-fcmovna %st(1), %st(0)  // CHECK: fcmovbe	%st(1), %st(0)
+fcmovnae %st(1), %st // CHECK: fcmovb	%st(1), %st
+fcmovna %st(1), %st  // CHECK: fcmovbe	%st(1), %st
 
-fcmovae %st(1), %st(0)  // CHECK: fcmovnb	%st(1), %st(0)
-fcmova %st(1), %st(0)   // CHECK: fcmovnbe	%st(1), %st(0)
+fcmovae %st(1), %st  // CHECK: fcmovnb	%st(1), %st
+fcmova %st(1), %st   // CHECK: fcmovnbe	%st(1), %st
 
 // rdar://8456417
 .byte (88 + 1) & 15  // CHECK: .byte	9
@@ -456,20 +456,20 @@ mov %rdx, %db15
 // CHECK: encoding: [0x44,0x0f,0x23,0xfa]
 
 // rdar://8456371 - Handle commutable instructions written backward.
-// CHECK: 	faddp	%st(1)
-// CHECK:	fmulp	%st(2)
+// CHECK: 	faddp	%st, %st(1)
+// CHECK:	fmulp	%st, %st(2)
 faddp %st, %st(1)
 fmulp %st, %st(2)
 
 // rdar://8468087 - Encode these accurately, they are not synonyms.
-// CHECK: fmul	%st(0), %st(1)
+// CHECK: fmul	%st, %st(1)
 // CHECK: encoding: [0xdc,0xc9]
 // CHECK: fmul	%st(1)
 // CHECK: encoding: [0xd8,0xc9]
 fmul %st, %st(1)
 fmul %st(1), %st
 
-// CHECK: fadd	%st(0), %st(1)
+// CHECK: fadd	%st, %st(1)
 // CHECK: encoding: [0xdc,0xc1]
 // CHECK: fadd	%st(1)
 // CHECK: encoding: [0xd8,0xc1]
@@ -582,15 +582,15 @@ movmskpd	%xmm6, %eax
 // CHECK: encoding: [0x66,0x0f,0x50,0xc6]
 
 // rdar://8491845 - Gas supports commuted forms of non-commutable instructions.
-fdivrp %st(0), %st(1) // CHECK: encoding: [0xde,0xf9]
-fdivrp %st(1), %st(0) // CHECK: encoding: [0xde,0xf9]
+fdivrp %st, %st(1) // CHECK: encoding: [0xde,0xf9]
+fdivrp %st(1), %st // CHECK: encoding: [0xde,0xf9]
 
-fsubrp %st(0), %st(1) // CHECK: encoding: [0xde,0xe9]
-fsubrp %st(1), %st(0) // CHECK: encoding: [0xde,0xe9]
+fsubrp %st, %st(1) // CHECK: encoding: [0xde,0xe9]
+fsubrp %st(1), %st // CHECK: encoding: [0xde,0xe9]
 
 // also PR8861
-fdivp %st(0), %st(1) // CHECK: encoding: [0xde,0xf1]
-fdivp %st(1), %st(0) // CHECK: encoding: [0xde,0xf1]
+fdivp %st, %st(1) // CHECK: encoding: [0xde,0xf1]
+fdivp %st(1), %st // CHECK: encoding: [0xde,0xf1]
 
 
 movl	foo(%rip), %eax
@@ -1391,38 +1391,38 @@ clac
 // CHECK: encoding: [0x0f,0x01,0xcb]
 stac
 
-// CHECK: faddp %st(1)
-// CHECK: fmulp %st(1)
-// CHECK: fsubp %st(1)
-// CHECK: fsubrp %st(1)
-// CHECK: fdivp %st(1)
-// CHECK: fdivrp %st(1)
-faddp %st(0), %st(1)
-fmulp %st(0), %st(1)
-fsubp %st(0), %st(1)
-fsubrp %st(0), %st(1)
-fdivp %st(0), %st(1)
-fdivrp %st(0), %st(1)
-
-// CHECK: faddp %st(1)
-// CHECK: fmulp %st(1)
-// CHECK: fsubp %st(1)
-// CHECK: fsubrp %st(1)
-// CHECK: fdivp %st(1)
-// CHECK: fdivrp %st(1)
-faddp %st(1), %st(0)
-fmulp %st(1), %st(0)
-fsubp %st(1), %st(0)
-fsubrp %st(1), %st(0)
-fdivp %st(1), %st(0)
-fdivrp %st(1), %st(0)
-
-// CHECK: faddp %st(1)
-// CHECK: fmulp %st(1)
-// CHECK: fsubp %st(1)
-// CHECK: fsubrp %st(1)
-// CHECK: fdivp %st(1)
-// CHECK: fdivrp %st(1)
+// CHECK: faddp %st, %st(1)
+// CHECK: fmulp %st, %st(1)
+// CHECK: fsubp %st, %st(1)
+// CHECK: fsubrp %st, %st(1)
+// CHECK: fdivp %st, %st(1)
+// CHECK: fdivrp %st, %st(1)
+faddp %st, %st(1)
+fmulp %st, %st(1)
+fsubp %st, %st(1)
+fsubrp %st, %st(1)
+fdivp %st, %st(1)
+fdivrp %st, %st(1)
+
+// CHECK: faddp %st, %st(1)
+// CHECK: fmulp %st, %st(1)
+// CHECK: fsubp %st, %st(1)
+// CHECK: fsubrp %st, %st(1)
+// CHECK: fdivp %st, %st(1)
+// CHECK: fdivrp %st, %st(1)
+faddp %st(1), %st
+fmulp %st(1), %st
+fsubp %st(1), %st
+fsubrp %st(1), %st
+fdivp %st(1), %st
+fdivrp %st(1), %st
+
+// CHECK: faddp %st, %st(1)
+// CHECK: fmulp %st, %st(1)
+// CHECK: fsubp %st, %st(1)
+// CHECK: fsubrp %st, %st(1)
+// CHECK: fdivp %st, %st(1)
+// CHECK: fdivrp %st, %st(1)
 faddp %st(1)
 fmulp %st(1)
 fsubp %st(1)
@@ -1430,12 +1430,12 @@ fsubrp %st(1)
 fdivp %st(1)
 fdivrp %st(1)
 
-// CHECK: faddp %st(1)
-// CHECK: fmulp %st(1)
-// CHECK: fsubp %st(1)
-// CHECK: fsubrp %st(1)
-// CHECK: fdivp %st(1)
-// CHECK: fdivrp %st(1)
+// CHECK: faddp %st, %st(1)
+// CHECK: fmulp %st, %st(1)
+// CHECK: fsubp %st, %st(1)
+// CHECK: fsubrp %st, %st(1)
+// CHECK: fdivp %st, %st(1)
+// CHECK: fdivrp %st, %st(1)
 faddp
 fmulp
 fsubp
@@ -1449,25 +1449,25 @@ fdivrp
 // CHECK: fsubr %st(1)
 // CHECK: fdiv %st(1)
 // CHECK: fdivr %st(1)
-fadd %st(1), %st(0)
-fmul %st(1), %st(0)
-fsub %st(1), %st(0)
-fsubr %st(1), %st(0)
-fdiv %st(1), %st(0)
-fdivr %st(1), %st(0)
-
-// CHECK: fadd %st(0), %st(1)
-// CHECK: fmul %st(0), %st(1)
-// CHECK: fsub %st(0), %st(1)
-// CHECK: fsubr %st(0), %st(1)
-// CHECK: fdiv %st(0), %st(1)
-// CHECK: fdivr %st(0), %st(1)
-fadd %st(0), %st(1)
-fmul %st(0), %st(1)
-fsub %st(0), %st(1)
-fsubr %st(0), %st(1)
-fdiv %st(0), %st(1)
-fdivr %st(0), %st(1)
+fadd %st(1), %st
+fmul %st(1), %st
+fsub %st(1), %st
+fsubr %st(1), %st
+fdiv %st(1), %st
+fdivr %st(1), %st
+
+// CHECK: fadd %st, %st(1)
+// CHECK: fmul %st, %st(1)
+// CHECK: fsub %st, %st(1)
+// CHECK: fsubr %st, %st(1)
+// CHECK: fdiv %st, %st(1)
+// CHECK: fdivr %st, %st(1)
+fadd %st, %st(1)
+fmul %st, %st(1)
+fsub %st, %st(1)
+fsubr %st, %st(1)
+fdiv %st, %st(1)
+fdivr %st, %st(1)
 
 // CHECK: fadd %st(1)
 // CHECK: fmul %st(1)
diff --git a/test/MC/X86/x86_errors.s b/test/MC/X86/x86_errors.s
index 1fe0a583e59ce4894079790fec380d0c647660f2..d999591838910d19e7e2d578d81da1d76afde5bb 100644
--- a/test/MC/X86/x86_errors.s
+++ b/test/MC/X86/x86_errors.s
@@ -118,3 +118,43 @@ mov (%rax,%eiz), %ebx
 // 32: error: register %riz is only available in 64-bit mode
 // 64: error: base register is 32-bit, but index register is not
 mov (%eax,%riz), %ebx
+
+
+// Parse errors from assembler parsing. 
+
+v_ecx = %ecx
+v_eax = %eax
+v_gs  = %gs
+v_imm = 4
+$test = %ebx
+
+// 32: 7: error: expected register here
+// 64: 7: error: expected register here
+mov 4(4), %eax	
+
+// 32: 7: error: expected register here
+// 64: 7: error: expected register here
+mov 5(v_imm), %eax		
+	
+// 32: 7: error: invalid register name
+// 64: 7: error: invalid register name
+mov 6(%v_imm), %eax		
+	
+// 32: 8: warning: scale factor without index register is ignored
+// 64: 8: warning: scale factor without index register is ignored
+mov 7(,v_imm), %eax		
+
+// 64: 6: error: expected immediate expression
+mov $%eax, %ecx
+
+// 32: 6: error: expected immediate expression
+// 64: 6: error: expected immediate expression
+mov $v_eax, %ecx
+
+// 32: error: unexpected token in argument list
+// 64: error: unexpected token in argument list
+mov v_ecx(%eax), %ecx	
+
+// 32: 7: error: invalid operand for instruction
+// 64: 7: error: invalid operand for instruction
+addb (%dx), %al
diff --git a/test/MC/X86/x86_operands.s b/test/MC/X86/x86_operands.s
index 2258a952774617d9769cc4d07f3a3605f9756092..3aa1b8d7fb8fd43ceefa2e9ae6ef7298e1cf259c 100644
--- a/test/MC/X86/x86_operands.s
+++ b/test/MC/X86/x86_operands.s
@@ -61,3 +61,406 @@ foo:
 # CHECK: movl	%gs:8, %eax
 movl %gs:8, %eax
 
+# Make sure we handle parsing uses of variables assigned
+# to registers in operands.
+
+v_ecx = %ecx
+v_eax = %eax
+v_gs  = %gs
+v_imm = 4
+
+#CHECK:	movl %eax, %ecx
+	movl %eax, v_ecx
+
+#CHECK: movl $1, %gs:0
+	movl $1, v_gs:(,)
+#CHECK: movl $1, %gs:(,%eax)
+	movl $1, v_gs:(,%eax)
+#CHECK: movl $1, %gs:(,%eax,2)
+	movl $1, v_gs:(,%eax,2)
+#CHECK: movl $1, %gs:(,%eax,4)
+	movl $1, v_gs:(,%eax,v_imm)
+#CHECK: movl $1, %gs:(,%eax)
+	movl $1, v_gs:(,v_eax)
+#CHECK: movl $1, %gs:(,%eax,2)
+	movl $1, v_gs:(,v_eax,2)
+#CHECK: movl $1, %gs:(,%eax,4)
+	movl $1, v_gs:(,v_eax,v_imm)
+#CHECK: movl $1, %gs:(%ecx)
+	movl $1, v_gs:(%ecx)
+#CHECK: movl $1, %gs:(%ecx)
+	movl $1, v_gs:(%ecx,)
+#CHECK: movl $1, %gs:(%ecx,%eax)
+	movl $1, v_gs:(%ecx,%eax)
+#CHECK: movl $1, %gs:(%ecx,%eax,2)
+	movl $1, v_gs:(%ecx,%eax,2)
+#CHECK: movl $1, %gs:(%ecx,%eax,4)
+	movl $1, v_gs:(%ecx,%eax,v_imm)
+#CHECK: movl $1, %gs:(%ecx,%eax)
+	movl $1, v_gs:(%ecx,v_eax)
+#CHECK: movl $1, %gs:(%ecx,%eax,2)
+	movl $1, v_gs:(%ecx,v_eax,2)
+#CHECK: movl $1, %gs:(%ecx,%eax,4)
+	movl $1, v_gs:(%ecx,v_eax,v_imm)
+#CHECK: movl $1, %gs:(%ecx)
+	movl $1, v_gs:(v_ecx)
+#CHECK: movl $1, %gs:(%ecx)
+	movl $1, v_gs:(v_ecx,)
+#CHECK: movl $1, %gs:(%ecx,%eax)
+	movl $1, v_gs:(v_ecx,%eax)
+#CHECK: movl $1, %gs:(%ecx,%eax,2)
+	movl $1, v_gs:(v_ecx,%eax,2)
+#CHECK: movl $1, %gs:(%ecx,%eax,4)
+	movl $1, v_gs:(v_ecx,%eax,v_imm)
+#CHECK: movl $1, %gs:(%ecx,%eax)
+	movl $1, v_gs:(v_ecx,v_eax)
+#CHECK: movl $1, %gs:(%ecx,%eax,2)
+	movl $1, v_gs:(v_ecx,v_eax,2)
+#CHECK: movl $1, %gs:(%ecx,%eax,4)
+	movl $1, v_gs:(v_ecx,v_eax,v_imm)
+#CHECK: movl $1, %gs:4
+	movl $1, v_gs:4
+#CHECK: movl $1, %gs:4
+	movl $1, v_gs:4()
+#CHECK: movl $1, %gs:4
+	movl $1, v_gs:4(,)
+#CHECK: movl $1, %gs:4(,%eax)
+	movl $1, v_gs:4(,%eax)
+#CHECK: movl $1, %gs:4(,%eax,2)
+	movl $1, v_gs:4(,%eax,2)
+#CHECK: movl $1, %gs:4(,%eax,4)
+	movl $1, v_gs:4(,%eax,v_imm)
+#CHECK: movl $1, %gs:4(,%eax)
+	movl $1, v_gs:4(,v_eax)
+#CHECK: movl $1, %gs:4(,%eax,2)
+	movl $1, v_gs:4(,v_eax,2)
+#CHECK: movl $1, %gs:4(,%eax,4)
+	movl $1, v_gs:4(,v_eax,v_imm)
+#CHECK: movl $1, %gs:4(%ecx)
+	movl $1, v_gs:4(%ecx)
+#CHECK: movl $1, %gs:4(%ecx)
+	movl $1, v_gs:4(%ecx,)
+#CHECK: movl $1, %gs:4(%ecx,%eax)
+	movl $1, v_gs:4(%ecx,%eax)
+#CHECK: movl $1, %gs:4(%ecx,%eax,2)
+	movl $1, v_gs:4(%ecx,%eax,2)
+#CHECK: movl $1, %gs:4(%ecx,%eax,4)
+	movl $1, v_gs:4(%ecx,%eax,v_imm)
+#CHECK: movl $1, %gs:4(%ecx,%eax)
+	movl $1, v_gs:4(%ecx,v_eax)
+#CHECK: movl $1, %gs:4(%ecx,%eax,2)
+	movl $1, v_gs:4(%ecx,v_eax,2)
+#CHECK: movl $1, %gs:4(%ecx,%eax,4)
+	movl $1, v_gs:4(%ecx,v_eax,v_imm)
+#CHECK: movl $1, %gs:4(%ecx)
+	movl $1, v_gs:4(v_ecx)
+#CHECK: movl $1, %gs:4(%ecx)
+	movl $1, v_gs:4(v_ecx,)
+#CHECK: movl $1, %gs:4(%ecx,%eax)
+	movl $1, v_gs:4(v_ecx,%eax)
+#CHECK: movl $1, %gs:4(%ecx,%eax,2)
+	movl $1, v_gs:4(v_ecx,%eax,2)
+#CHECK: movl $1, %gs:4(%ecx,%eax,4)
+	movl $1, v_gs:4(v_ecx,%eax,v_imm)
+#CHECK: movl $1, %gs:4(%ecx,%eax)
+	movl $1, v_gs:4(v_ecx,v_eax)
+#CHECK: movl $1, %gs:4(%ecx,%eax,2)
+	movl $1, v_gs:4(v_ecx,v_eax,2)
+#CHECK: movl $1, %gs:4(%ecx,%eax,4)
+	movl $1, v_gs:4(v_ecx,v_eax,v_imm)
+#CHECK: movl $1, %gs:4
+	movl $1, v_gs:v_imm
+#CHECK: movl $1, %gs:4
+	movl $1, v_gs:v_imm()
+#CHECK: movl $1, %gs:4
+	movl $1, v_gs:v_imm(,)
+#CHECK: movl $1, %gs:4(,%eax)
+	movl $1, v_gs:v_imm(,%eax)
+#CHECK: movl $1, %gs:4(,%eax,2)
+	movl $1, v_gs:v_imm(,%eax,2)
+#CHECK: movl $1, %gs:4(,%eax,4)
+	movl $1, v_gs:v_imm(,%eax,v_imm)
+#CHECK: movl $1, %gs:4(,%eax)
+	movl $1, v_gs:v_imm(,v_eax)
+#CHECK: movl $1, %gs:4(,%eax,2)
+	movl $1, v_gs:v_imm(,v_eax,2)
+#CHECK: movl $1, %gs:4(,%eax,4)
+	movl $1, v_gs:v_imm(,v_eax,v_imm)
+#CHECK: movl $1, %gs:4(%ecx)
+	movl $1, v_gs:v_imm(%ecx)
+#CHECK: movl $1, %gs:4(%ecx)
+	movl $1, v_gs:v_imm(%ecx,)
+#CHECK: movl $1, %gs:4(%ecx,%eax)
+	movl $1, v_gs:v_imm(%ecx,%eax)
+#CHECK: movl $1, %gs:4(%ecx,%eax,2)
+	movl $1, v_gs:v_imm(%ecx,%eax,2)
+#CHECK: movl $1, %gs:4(%ecx,%eax,4)
+	movl $1, v_gs:v_imm(%ecx,%eax,v_imm)
+#CHECK: movl $1, %gs:4(%ecx,%eax)
+	movl $1, v_gs:v_imm(%ecx,v_eax)
+#CHECK: movl $1, %gs:4(%ecx,%eax,2)
+	movl $1, v_gs:v_imm(%ecx,v_eax,2)
+#CHECK: movl $1, %gs:4(%ecx,%eax,4)
+	movl $1, v_gs:v_imm(%ecx,v_eax,v_imm)
+#CHECK: movl $1, %gs:4(%ecx)
+	movl $1, v_gs:v_imm(v_ecx)
+#CHECK: movl $1, %gs:4(%ecx)
+	movl $1, v_gs:v_imm(v_ecx,)
+#CHECK: movl $1, %gs:4(%ecx,%eax)
+	movl $1, v_gs:v_imm(v_ecx,%eax)
+#CHECK: movl $1, %gs:4(%ecx,%eax,2)
+	movl $1, v_gs:v_imm(v_ecx,%eax,2)
+#CHECK: movl $1, %gs:4(%ecx,%eax,4)
+	movl $1, v_gs:v_imm(v_ecx,%eax,v_imm)
+#CHECK: movl $1, %gs:4(%ecx,%eax)
+	movl $1, v_gs:v_imm(v_ecx,v_eax)
+#CHECK: movl $1, %gs:4(%ecx,%eax,2)
+	movl $1, v_gs:v_imm(v_ecx,v_eax,2)
+#CHECK: movl $1, %gs:4(%ecx,%eax,4)
+	movl $1, v_gs:v_imm(v_ecx,v_eax,v_imm)
+#CHECK: movl $1, %gs:8
+	movl $1, v_gs:(v_imm+4)
+#CHECK: movl $1, %gs:8
+	movl $1, v_gs:(v_imm+4)()
+#CHECK: movl $1, %gs:8
+	movl $1, v_gs:(v_imm+4)(,)
+#CHECK: movl $1, %gs:8(,%eax)
+	movl $1, v_gs:(v_imm+4)(,%eax)
+#CHECK: movl $1, %gs:8(,%eax,2)
+	movl $1, v_gs:(v_imm+4)(,%eax,2)
+#CHECK: movl $1, %gs:8(,%eax,4)
+	movl $1, v_gs:(v_imm+4)(,%eax,v_imm)
+#CHECK: movl $1, %gs:8(,%eax)
+	movl $1, v_gs:(v_imm+4)(,v_eax)
+#CHECK: movl $1, %gs:8(,%eax,2)
+	movl $1, v_gs:(v_imm+4)(,v_eax,2)
+#CHECK: movl $1, %gs:8(,%eax,4)
+	movl $1, v_gs:(v_imm+4)(,v_eax,v_imm)
+#CHECK: movl $1, %gs:8(%ecx)
+	movl $1, v_gs:(v_imm+4)(%ecx)
+#CHECK: movl $1, %gs:8(%ecx)
+	movl $1, v_gs:(v_imm+4)(%ecx,)
+#CHECK: movl $1, %gs:8(%ecx,%eax)
+	movl $1, v_gs:(v_imm+4)(%ecx,%eax)
+#CHECK: movl $1, %gs:8(%ecx,%eax,2)
+	movl $1, v_gs:(v_imm+4)(%ecx,%eax,2)
+#CHECK: movl $1, %gs:8(%ecx,%eax,4)
+	movl $1, v_gs:(v_imm+4)(%ecx,%eax,v_imm)
+#CHECK: movl $1, %gs:8(%ecx,%eax)
+	movl $1, v_gs:(v_imm+4)(%ecx,v_eax)
+#CHECK: movl $1, %gs:8(%ecx,%eax,2)
+	movl $1, v_gs:(v_imm+4)(%ecx,v_eax,2)
+#CHECK: movl $1, %gs:8(%ecx,%eax,4)
+	movl $1, v_gs:(v_imm+4)(%ecx,v_eax,v_imm)
+#CHECK: movl $1, %gs:8(%ecx)
+	movl $1, v_gs:(v_imm+4)(v_ecx)
+#CHECK: movl $1, %gs:8(%ecx)
+	movl $1, v_gs:(v_imm+4)(v_ecx,)
+#CHECK: movl $1, %gs:8(%ecx,%eax)
+	movl $1, v_gs:(v_imm+4)(v_ecx,%eax)
+#CHECK: movl $1, %gs:8(%ecx,%eax,2)
+	movl $1, v_gs:(v_imm+4)(v_ecx,%eax,2)
+#CHECK: movl $1, %gs:8(%ecx,%eax,4)
+	movl $1, v_gs:(v_imm+4)(v_ecx,%eax,v_imm)
+#CHECK: movl $1, %gs:8(%ecx,%eax)
+	movl $1, v_gs:(v_imm+4)(v_ecx,v_eax)
+#CHECK: movl $1, %gs:8(%ecx,%eax,2)
+	movl $1, v_gs:(v_imm+4)(v_ecx,v_eax,2)
+#CHECK: movl $1, %gs:8(%ecx,%eax,4)
+	movl $1, v_gs:(v_imm+4)(v_ecx,v_eax,v_imm)
+#CHECK: movl $1, %fs:0
+	movl $1, %fs:(,)
+#CHECK: movl $1, %fs:(,%eax)
+	movl $1, %fs:(,%eax)
+#CHECK: movl $1, %fs:(,%eax,2)
+	movl $1, %fs:(,%eax,2)
+#CHECK: movl $1, %fs:(,%eax,4)
+	movl $1, %fs:(,%eax,v_imm)
+#CHECK: movl $1, %fs:(,%eax)
+	movl $1, %fs:(,v_eax)
+#CHECK: movl $1, %fs:(,%eax,2)
+	movl $1, %fs:(,v_eax,2)
+#CHECK: movl $1, %fs:(,%eax,4)
+	movl $1, %fs:(,v_eax,v_imm)
+#CHECK: movl $1, %fs:(%ecx)
+	movl $1, %fs:(%ecx)
+#CHECK: movl $1, %fs:(%ecx)
+	movl $1, %fs:(%ecx,)
+#CHECK: movl $1, %fs:(%ecx,%eax)
+	movl $1, %fs:(%ecx,%eax)
+#CHECK: movl $1, %fs:(%ecx,%eax,2)
+	movl $1, %fs:(%ecx,%eax,2)
+#CHECK: movl $1, %fs:(%ecx,%eax,4)
+	movl $1, %fs:(%ecx,%eax,v_imm)
+#CHECK: movl $1, %fs:(%ecx,%eax)
+	movl $1, %fs:(%ecx,v_eax)
+#CHECK: movl $1, %fs:(%ecx,%eax,2)
+	movl $1, %fs:(%ecx,v_eax,2)
+#CHECK: movl $1, %fs:(%ecx,%eax,4)
+	movl $1, %fs:(%ecx,v_eax,v_imm)
+#CHECK: movl $1, %fs:(%ecx)
+	movl $1, %fs:(v_ecx)
+#CHECK: movl $1, %fs:(%ecx)
+	movl $1, %fs:(v_ecx,)
+#CHECK: movl $1, %fs:(%ecx,%eax)
+	movl $1, %fs:(v_ecx,%eax)
+#CHECK: movl $1, %fs:(%ecx,%eax,2)
+	movl $1, %fs:(v_ecx,%eax,2)
+#CHECK: movl $1, %fs:(%ecx,%eax,4)
+	movl $1, %fs:(v_ecx,%eax,v_imm)
+#CHECK: movl $1, %fs:(%ecx,%eax)
+	movl $1, %fs:(v_ecx,v_eax)
+#CHECK: movl $1, %fs:(%ecx,%eax,2)
+	movl $1, %fs:(v_ecx,v_eax,2)
+#CHECK: movl $1, %fs:(%ecx,%eax,4)
+	movl $1, %fs:(v_ecx,v_eax,v_imm)
+#CHECK: movl $1, %fs:4
+	movl $1, %fs:4
+#CHECK: movl $1, %fs:4
+	movl $1, %fs:4()
+#CHECK: movl $1, %fs:4
+	movl $1, %fs:4(,)
+#CHECK: movl $1, %fs:4(,%eax)
+	movl $1, %fs:4(,%eax)
+#CHECK: movl $1, %fs:4(,%eax,2)
+	movl $1, %fs:4(,%eax,2)
+#CHECK: movl $1, %fs:4(,%eax,4)
+	movl $1, %fs:4(,%eax,v_imm)
+#CHECK: movl $1, %fs:4(,%eax)
+	movl $1, %fs:4(,v_eax)
+#CHECK: movl $1, %fs:4(,%eax,2)
+	movl $1, %fs:4(,v_eax,2)
+#CHECK: movl $1, %fs:4(,%eax,4)
+	movl $1, %fs:4(,v_eax,v_imm)
+#CHECK: movl $1, %fs:4(%ecx)
+	movl $1, %fs:4(%ecx)
+#CHECK: movl $1, %fs:4(%ecx)
+	movl $1, %fs:4(%ecx,)
+#CHECK: movl $1, %fs:4(%ecx,%eax)
+	movl $1, %fs:4(%ecx,%eax)
+#CHECK: movl $1, %fs:4(%ecx,%eax,2)
+	movl $1, %fs:4(%ecx,%eax,2)
+#CHECK: movl $1, %fs:4(%ecx,%eax,4)
+	movl $1, %fs:4(%ecx,%eax,v_imm)
+#CHECK: movl $1, %fs:4(%ecx,%eax)
+	movl $1, %fs:4(%ecx,v_eax)
+#CHECK: movl $1, %fs:4(%ecx,%eax,2)
+	movl $1, %fs:4(%ecx,v_eax,2)
+#CHECK: movl $1, %fs:4(%ecx,%eax,4)
+	movl $1, %fs:4(%ecx,v_eax,v_imm)
+#CHECK: movl $1, %fs:4(%ecx)
+	movl $1, %fs:4(v_ecx)
+#CHECK: movl $1, %fs:4(%ecx)
+	movl $1, %fs:4(v_ecx,)
+#CHECK: movl $1, %fs:4(%ecx,%eax)
+	movl $1, %fs:4(v_ecx,%eax)
+#CHECK: movl $1, %fs:4(%ecx,%eax,2)
+	movl $1, %fs:4(v_ecx,%eax,2)
+#CHECK: movl $1, %fs:4(%ecx,%eax,4)
+	movl $1, %fs:4(v_ecx,%eax,v_imm)
+#CHECK: movl $1, %fs:4(%ecx,%eax)
+	movl $1, %fs:4(v_ecx,v_eax)
+#CHECK: movl $1, %fs:4(%ecx,%eax,2)
+	movl $1, %fs:4(v_ecx,v_eax,2)
+#CHECK: movl $1, %fs:4(%ecx,%eax,4)
+	movl $1, %fs:4(v_ecx,v_eax,v_imm)
+#CHECK: movl $1, %fs:4
+	movl $1, %fs:v_imm
+#CHECK: movl $1, %fs:4
+	movl $1, %fs:v_imm()
+#CHECK: movl $1, %fs:4
+	movl $1, %fs:v_imm(,)
+#CHECK: movl $1, %fs:4(,%eax)
+	movl $1, %fs:v_imm(,%eax)
+#CHECK: movl $1, %fs:4(,%eax,2)
+	movl $1, %fs:v_imm(,%eax,2)
+#CHECK: movl $1, %fs:4(,%eax,4)
+	movl $1, %fs:v_imm(,%eax,v_imm)
+#CHECK: movl $1, %fs:4(,%eax)
+	movl $1, %fs:v_imm(,v_eax)
+#CHECK: movl $1, %fs:4(,%eax,2)
+	movl $1, %fs:v_imm(,v_eax,2)
+#CHECK: movl $1, %fs:4(,%eax,4)
+	movl $1, %fs:v_imm(,v_eax,v_imm)
+#CHECK: movl $1, %fs:4(%ecx)
+	movl $1, %fs:v_imm(%ecx)
+#CHECK: movl $1, %fs:4(%ecx)
+	movl $1, %fs:v_imm(%ecx,)
+#CHECK: movl $1, %fs:4(%ecx,%eax)
+	movl $1, %fs:v_imm(%ecx,%eax)
+#CHECK: movl $1, %fs:4(%ecx,%eax,2)
+	movl $1, %fs:v_imm(%ecx,%eax,2)
+#CHECK: movl $1, %fs:4(%ecx,%eax,4)
+	movl $1, %fs:v_imm(%ecx,%eax,v_imm)
+#CHECK: movl $1, %fs:4(%ecx,%eax)
+	movl $1, %fs:v_imm(%ecx,v_eax)
+#CHECK: movl $1, %fs:4(%ecx,%eax,2)
+	movl $1, %fs:v_imm(%ecx,v_eax,2)
+#CHECK: movl $1, %fs:4(%ecx,%eax,4)
+	movl $1, %fs:v_imm(%ecx,v_eax,v_imm)
+#CHECK: movl $1, %fs:4(%ecx)
+	movl $1, %fs:v_imm(v_ecx)
+#CHECK: movl $1, %fs:4(%ecx)
+	movl $1, %fs:v_imm(v_ecx,)
+#CHECK: movl $1, %fs:4(%ecx,%eax)
+	movl $1, %fs:v_imm(v_ecx,%eax)
+#CHECK: movl $1, %fs:4(%ecx,%eax,2)
+	movl $1, %fs:v_imm(v_ecx,%eax,2)
+#CHECK: movl $1, %fs:4(%ecx,%eax,4)
+	movl $1, %fs:v_imm(v_ecx,%eax,v_imm)
+#CHECK: movl $1, %fs:4(%ecx,%eax)
+	movl $1, %fs:v_imm(v_ecx,v_eax)
+#CHECK: movl $1, %fs:4(%ecx,%eax,2)
+	movl $1, %fs:v_imm(v_ecx,v_eax,2)
+#CHECK: movl $1, %fs:4(%ecx,%eax,4)
+	movl $1, %fs:v_imm(v_ecx,v_eax,v_imm)
+#CHECK: movl $1, %fs:8
+	movl $1, %fs:(v_imm+4)
+#CHECK: movl $1, %fs:8
+	movl $1, %fs:(v_imm+4)()
+#CHECK: movl $1, %fs:8
+	movl $1, %fs:(v_imm+4)(,)
+#CHECK: movl $1, %fs:8(,%eax)
+	movl $1, %fs:(v_imm+4)(,%eax)
+#CHECK: movl $1, %fs:8(,%eax,2)
+	movl $1, %fs:(v_imm+4)(,%eax,2)
+#CHECK: movl $1, %fs:8(,%eax,4)
+	movl $1, %fs:(v_imm+4)(,%eax,v_imm)
+#CHECK: movl $1, %fs:8(,%eax)
+	movl $1, %fs:(v_imm+4)(,v_eax)
+#CHECK: movl $1, %fs:8(,%eax,2)
+	movl $1, %fs:(v_imm+4)(,v_eax,2)
+#CHECK: movl $1, %fs:8(,%eax,4)
+	movl $1, %fs:(v_imm+4)(,v_eax,v_imm)
+#CHECK: movl $1, %fs:8(%ecx)
+	movl $1, %fs:(v_imm+4)(%ecx)
+#CHECK: movl $1, %fs:8(%ecx)
+	movl $1, %fs:(v_imm+4)(%ecx,)
+#CHECK: movl $1, %fs:8(%ecx,%eax)
+	movl $1, %fs:(v_imm+4)(%ecx,%eax)
+#CHECK: movl $1, %fs:8(%ecx,%eax,2)
+	movl $1, %fs:(v_imm+4)(%ecx,%eax,2)
+#CHECK: movl $1, %fs:8(%ecx,%eax,4)
+	movl $1, %fs:(v_imm+4)(%ecx,%eax,v_imm)
+#CHECK: movl $1, %fs:8(%ecx,%eax)
+	movl $1, %fs:(v_imm+4)(%ecx,v_eax)
+#CHECK: movl $1, %fs:8(%ecx,%eax,2)
+	movl $1, %fs:(v_imm+4)(%ecx,v_eax,2)
+#CHECK: movl $1, %fs:8(%ecx,%eax,4)
+	movl $1, %fs:(v_imm+4)(%ecx,v_eax,v_imm)
+#CHECK: movl $1, %fs:8(%ecx)
+	movl $1, %fs:(v_imm+4)(v_ecx)
+#CHECK: movl $1, %fs:8(%ecx)
+	movl $1, %fs:(v_imm+4)(v_ecx,)
+#CHECK: movl $1, %fs:8(%ecx,%eax)
+	movl $1, %fs:(v_imm+4)(v_ecx,%eax)
+#CHECK: movl $1, %fs:8(%ecx,%eax,2)
+	movl $1, %fs:(v_imm+4)(v_ecx,%eax,2)
+#CHECK: movl $1, %fs:8(%ecx,%eax,4)
+	movl $1, %fs:(v_imm+4)(v_ecx,%eax,v_imm)
+#CHECK: movl $1, %fs:8(%ecx,%eax)
+	movl $1, %fs:(v_imm+4)(v_ecx,v_eax)
+#CHECK: movl $1, %fs:8(%ecx,%eax,2)
+	movl $1, %fs:(v_imm+4)(v_ecx,v_eax,2)
+#CHECK: movl $1, %fs:8(%ecx,%eax,4)
+	movl $1, %fs:(v_imm+4)(v_ecx,v_eax,v_imm)
diff --git a/test/Object/AArch64/yaml2obj-elf-aarch64-rel.yaml b/test/Object/AArch64/yaml2obj-elf-aarch64-rel.yaml
index e3017a24a97d44acd0f1ae60084ac5cd40c9d57b..c1e318a7472c4ea29b84326e7761d70ea73e80d7 100644
--- a/test/Object/AArch64/yaml2obj-elf-aarch64-rel.yaml
+++ b/test/Object/AArch64/yaml2obj-elf-aarch64-rel.yaml
@@ -5,6 +5,7 @@
 # CHECK-NEXT:   Type:            SHT_RELA
 # CHECK-NEXT:    Link:            .symtab
 # CHECK-NEXT:    AddressAlign:    0x0000000000000008
+# CHECK-NEXT:    EntSize:         0x0000000000000018
 # CHECK-NEXT:    Info:            .text
 # CHECK-NEXT:    Relocations:     
 # CHECK-NEXT:      - Offset:          0x0000000000000000
diff --git a/test/Object/Inputs/darwin-m-test3.macho-x86-64 b/test/Object/Inputs/darwin-m-test3.macho-x86-64
index 18960c4f6aab699718a7dbe54620eb7c41786efc..bf72b59a2358dae863dc6ee498593da30a6b5bf9 100755
Binary files a/test/Object/Inputs/darwin-m-test3.macho-x86-64 and b/test/Object/Inputs/darwin-m-test3.macho-x86-64 differ
diff --git a/test/Object/Inputs/trivial-object-test.wasm b/test/Object/Inputs/trivial-object-test.wasm
index 8652d67f69222ab57610df1b13d3972e031460a1..a894522fa152ba86d59c6c585788e1467809f416 100644
Binary files a/test/Object/Inputs/trivial-object-test.wasm and b/test/Object/Inputs/trivial-object-test.wasm differ
diff --git a/test/Object/Mips/abi-flags.yaml b/test/Object/Mips/abi-flags.yaml
index 0d9fb369423af8e19e61b21561f8337b362bab4b..ee66638c9d11e77a33c61d554848dafe9a5dee2c 100644
--- a/test/Object/Mips/abi-flags.yaml
+++ b/test/Object/Mips/abi-flags.yaml
@@ -25,6 +25,7 @@
 # YAML-NEXT: - Name:            .MIPS.abiflags
 # YAML-NEXT:   Type:            SHT_MIPS_ABIFLAGS
 # YAML-NEXT:   AddressAlign:    0x0000000000000008
+# YAML-NEXT:   EntSize:         0x0000000000000018
 # YAML-NEXT:   ISA:             MIPS64
 # YAML-NEXT:   ISARevision:     0x05
 # YAML-NEXT:   ISAExtension:    EXT_OCTEON3
diff --git a/test/Object/X86/coff-asm.ll b/test/Object/X86/coff-asm.ll
new file mode 100644
index 0000000000000000000000000000000000000000..e6e4bc16bff71e51d99ef7753377bc840f7b5058
--- /dev/null
+++ b/test/Object/X86/coff-asm.ll
@@ -0,0 +1,13 @@
+; Check that llvm-nm doesn't crash on COFF-specific assembly directives
+; (PR36789).
+
+; RUN: llvm-as %s -o %t.bc
+; RUN: llvm-nm %t.bc
+
+target datalayout = "e-m:w-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-pc-windows-msvc19.0.24210"
+
+module asm ".text"
+module asm ".def foo; .scl 2; .type 32; .endef"
+module asm "foo:"
+module asm "  ret"
diff --git a/test/Object/ar-create.test b/test/Object/ar-create.test
index 95d994e06a48ce959e8376c772aa63bc5864b4cf..4b08c97317a8dadcb797c95af6c313abc71361a6 100644
--- a/test/Object/ar-create.test
+++ b/test/Object/ar-create.test
@@ -13,5 +13,5 @@ RUN: rm -f %t.foo.a
 RUN: llvm-ar r %t.foo.a %t 2>&1 | FileCheck --check-prefix=CREATE %s
 RUN: rm -f %t.foo.a
 
-CHECK: llvm-ar{{(.exe|.EXE)?}}: error loading '{{[^']+}}.foo.a':
+CHECK: llvm-ar{{(.exe|.EXE)?}}: error: error loading '{{[^']+}}.foo.a':
 CREATE: creating {{.*}}.foo.a
diff --git a/test/Object/ar-error.test b/test/Object/ar-error.test
index 7add9b448e1997987585dfff89c86ea915373c4b..d212555f5c39ac2696c346ff644ef31d6377a865 100644
--- a/test/Object/ar-error.test
+++ b/test/Object/ar-error.test
@@ -3,4 +3,4 @@ Test if we get a proper error with a filename that doesn't exist
 RUN: not llvm-ar r %t.out.a sparkle.o %t 2>&1 | FileCheck %s
 
 # Don't check the message "No such file or directory".
-CHECK: llvm-ar{{(.exe|.EXE)?}}: sparkle.o:
+CHECK: llvm-ar{{(.exe|.EXE)?}}: error: sparkle.o:
diff --git a/test/Object/archive-extract.test b/test/Object/archive-extract.test
index 85d853d79e792bb056b6b98ff1afa77be93f143a..916ccc254323b78eaba6454349484475809b0447 100644
--- a/test/Object/archive-extract.test
+++ b/test/Object/archive-extract.test
@@ -49,7 +49,7 @@ CHECK-GNU: 1465 Nov 19 03:01 2004 very_long_bytecode_file_name.bc
 CHECK-DARWIN: 1472 Nov 19 03:01 2004 very_long_bytecode_file_name.bc
 
 RUN: not llvm-ar x %p/Inputs/GNU.a foo.o 2>&1 | FileCheck --check-prefix=NOTFOUND %s
-NOTFOUND: foo.o was not found
+NOTFOUND: error: 'foo.o' was not found
 
 RUN: not llvm-ar x %p/Inputs/thin.a foo.o 2>&1 | FileCheck %s --check-prefix=THINEXTRACT
 THINEXTRACT: extracting from a thin archive is not supported
diff --git a/test/Object/archive-format.test b/test/Object/archive-format.test
index b1ae411161bc1bdc346f9e69c6d7c114d8bcaf39..b555bf88faca9164a5fcfafbec108a4f1e844388 100644
--- a/test/Object/archive-format.test
+++ b/test/Object/archive-format.test
@@ -15,8 +15,7 @@ RUN: llvm-ar --format=gnu rc %t.a 0123456789abcde 0123456789abcdef foo/012345678
 RUN: cat %t.a | FileCheck -strict-whitespace %s
 
 CHECK:      !<arch>
-CHECK-NEXT: //                                              36        `
-CHECK-NEXT: 0123456789abcdef/
+CHECK-NEXT: //                                              18        `
 CHECK-NEXT: 0123456789abcdef/
 CHECK-NEXT: 0123456789abcde/0           0     0     644     4         `
 CHECK-NEXT: bar.
@@ -24,7 +23,7 @@ CHECK-SAME: /0              0           0     0     644     4         `
 CHECK-NEXT: zed.
 CHECK-SAME: 0123456789abcde/0           0     0     644     4         `
 CHECK-NEXT: bar2
-CHECK-SAME: /18             0           0     0     644     4         `
+CHECK-SAME: /0              0           0     0     644     4         `
 CHECK-NEXT: zed2
 
 RUN: rm -f %t.a
diff --git a/test/Object/nm-darwin-m.test b/test/Object/nm-darwin-m.test
index 5bb19dcacd374aaf6abae417a17701a2f5120375..a8d7007006128521ffa4941af2ecbb265f41e8a8 100644
--- a/test/Object/nm-darwin-m.test
+++ b/test/Object/nm-darwin-m.test
@@ -19,8 +19,8 @@ test2:          (indirect) external _i (for __i)
 
 # This is testing is using darwin-m-test3.macho-x86-64 that is linked with
 # dylibs that have the follow set of -install_names:
-#	Foo.framework/Foo 
-#	/System/Library/Frameworks/FooPath.framework/FooPath 
+#	Foo.framework/Foo
+#	/System/Library/Frameworks/FooPath.framework/FooPath
 #	FooSuffix.framework/FooSuffix_debug
 #	/System/Library/Frameworks/FooPathSuffix.framework/FooPathSuffix_profile
 #	FooVers.framework/Versions/A/FooVers
@@ -32,8 +32,10 @@ test2:          (indirect) external _i (for __i)
 #	/usr/lib/libPathATS.A_profile.dylib
 #	QT.A.qtx
 #	/lib/QTPath.qtx
+#	/usr/lib/libfoo_bar.dylib
+#	/usr/lib/libfoo_bar_profile.dylib
 #	/usr/lib/libSystem.B.dylib
-# to test that MachOObjectFile::guessLibraryShortName() is correctly parsing 
+# to test that MachOObjectFile::guessLibraryShortName() is correctly parsing
 # them into their short names.
 test3: 0000000100000000 (__TEXT,__text) [referenced dynamically] external __mh_execute_header
 test3:                  (undefined) external _atsPathVersSuffix (from libPathATS)
@@ -44,7 +46,9 @@ test3:                  (undefined) external _fooPathSuffix (from FooPathSuffix)
 test3:                  (undefined) external _fooPathVers (from FooPathVers)
 test3:                  (undefined) external _fooSuffix (from FooSuffix)
 test3:                  (undefined) external _fooVers (from FooVers)
-test3: 0000000100000e60 (__TEXT,__text) external _main
+test3:                  (undefined) external _foo_bar (from libfoo_bar)
+test3:                  (undefined) external _foo_barSuffix (from libfoo_barSuffix)
+test3: 0000000100000e50 (__TEXT,__text) external _main
 test3:                  (undefined) external _qt (from QT)
 test3:                  (undefined) external _qtPath (from QTPath)
 test3:                  (undefined) external _x (from libx)
diff --git a/test/Object/nm-trivial-object.test b/test/Object/nm-trivial-object.test
index ca9c2782611afb23bed15692029f38a765cdc2e5..086456ef1c72ff39e94524de35dec6fa874f1ee5 100644
--- a/test/Object/nm-trivial-object.test
+++ b/test/Object/nm-trivial-object.test
@@ -18,6 +18,8 @@ RUN: llvm-nm %p/Inputs/weak.elf-x86-64 \
 RUN:         | FileCheck %s -check-prefix WEAK-ELF64
 RUN: llvm-nm %p/Inputs/absolute.elf-x86-64 \
 RUN:         | FileCheck %s -check-prefix ABSOLUTE-ELF64
+RUN: llvm-nm -a %p/Inputs/IsNAN.o \
+RUN:         | FileCheck %s -check-prefix ELF64-DEBUG-SYMS
 RUN: llvm-nm %p/Inputs/trivial-object-test.macho-i386 \
 RUN:         | FileCheck %s -check-prefix macho
 RUN: llvm-nm -U %p/Inputs/trivial-object-test.macho-i386 \
@@ -61,9 +63,9 @@ COFF32-NEXT:          U _puts
 
 
 WASM:      00000000 d .L.str
-WASM-NEXT: 00000003 t .LSomeOtherFunction_bitcast
+WASM-NEXT: 00000019 t .LSomeOtherFunction_bitcast
 WASM-NEXT:          U SomeOtherFunction
-WASM-NEXT: 00000002 T main
+WASM-NEXT: 00000001 T main
 WASM-NEXT:          U puts
 WASM-NEXT: 00000010 D var
 
@@ -113,6 +115,23 @@ WEAK-ELF64: 0000000000000000 V x2
 ABSOLUTE-ELF64: 0000000000000123 a a1
 ABSOLUTE-ELF64: 0000000000000123 A a2
 
+ELF64-DEBUG-SYMS: 00000000 b .bss
+ELF64-DEBUG-SYMS-NEXT: 00000000 n .comment
+ELF64-DEBUG-SYMS-NEXT: 00000000 d .data
+ELF64-DEBUG-SYMS-NEXT: 00000000 N .debug_abbrev
+ELF64-DEBUG-SYMS-NEXT: 00000000 N .debug_aranges
+ELF64-DEBUG-SYMS-NEXT: 00000000 N .debug_frame
+ELF64-DEBUG-SYMS-NEXT: 00000000 N .debug_info
+ELF64-DEBUG-SYMS-NEXT: 00000000 N .debug_line
+ELF64-DEBUG-SYMS-NEXT: 00000000 N .debug_pubnames
+ELF64-DEBUG-SYMS-NEXT: 00000000 n .note.GNU-stack
+ELF64-DEBUG-SYMS-NEXT: 00000000 t .text
+ELF64-DEBUG-SYMS-NEXT: 00000000 a IsNAN.cpp
+ELF64-DEBUG-SYMS-NEXT: 00000014 T _ZN4llvm5IsNANEd
+ELF64-DEBUG-SYMS-NEXT: 00000000 T _ZN4llvm5IsNANEf
+ELF64-DEBUG-SYMS-NEXT:          U __isnan
+ELF64-DEBUG-SYMS-NEXT:          U __isnanf
+
 macho:          U _SomeOtherFunction
 macho: 00000000 T _main
 macho:          U _puts
@@ -166,11 +185,11 @@ macho-o: {{.*}}/macho-text-data-bss.macho-x86_64: 0000000000000048 S _t.eh
 macho-pa: 0000000000000000 - 00 0000    SO /Volumes/SandBox/
 macho-pa: 0000000000000000 - 00 0000    SO hello.c
 macho-pa: 0000000053c8408d - 03 0001   OSO /Volumes/SandBox/hello.o
-macho-pa: 0000000100000f30 - 01 0000 BNSYM 
+macho-pa: 0000000100000f30 - 01 0000 BNSYM
 macho-pa: 0000000100000f30 - 01 0000   FUN _main
-macho-pa: 000000000000003b - 00 0000   FUN 
-macho-pa: 000000000000003b - 01 0000 ENSYM 
-macho-pa: 0000000000000000 - 01 0000    SO 
+macho-pa: 000000000000003b - 00 0000   FUN
+macho-pa: 000000000000003b - 01 0000 ENSYM
+macho-pa: 0000000000000000 - 01 0000    SO
 macho-pa: 0000000100000000 T __mh_execute_header
 macho-pa: 0000000100000f30 T _main
 macho-pa:                  U _printf
diff --git a/test/Object/obj2yaml-invalid-reloc.test b/test/Object/obj2yaml-invalid-reloc.test
index c0a5a1b5fbd89cd307d5b885207446c88ea9e8de..1c77c59277e279b3dbbe85080c28077a646fec06 100644
--- a/test/Object/obj2yaml-invalid-reloc.test
+++ b/test/Object/obj2yaml-invalid-reloc.test
@@ -17,6 +17,7 @@ CHECK-NEXT:   - Name:            .rela.text
 CHECK-NEXT:     Type:            SHT_RELA
 CHECK-NEXT:     Link:            .symtab
 CHECK-NEXT:     AddressAlign:    0x0000000000000008
+CHECK-NEXT:     EntSize:         0x0000000000000018
 CHECK-NEXT:     Info:            .text
 CHECK-NEXT:     Relocations:
 CHECK-NEXT:       - Offset:          0x0000000000000000
diff --git a/test/Object/obj2yaml.test b/test/Object/obj2yaml.test
index 46a0d77363647c85f7c15de4b83f9910dbdb1378..79f44a02b943cc0382516ff8c670fedaafbbacc3 100644
--- a/test/Object/obj2yaml.test
+++ b/test/Object/obj2yaml.test
@@ -364,6 +364,7 @@ ELF-MIPSEL-NEXT:   - Name:            .rel.text
 ELF-MIPSEL-NEXT:     Type:            SHT_REL
 ELF-MIPSEL-NEXT:     Link:            .symtab
 ELF-MIPSEL-NEXT:     AddressAlign:    0x0000000000000004
+ELF-MIPSEL-NEXT:     EntSize:         0x0000000000000008
 ELF-MIPSEL-NEXT:     Info:            .text
 ELF-MIPSEL-NEXT:     Relocations:
 ELF-MIPSEL-NEXT:       - Offset:          0x0000000000000000
@@ -402,6 +403,7 @@ ELF-MIPSEL-NEXT:   - Name:            .rodata.str1.1
 ELF-MIPSEL-NEXT:     Type:            SHT_PROGBITS
 ELF-MIPSEL-NEXT:     Flags:           [ SHF_ALLOC, SHF_MERGE, SHF_STRINGS ]
 ELF-MIPSEL-NEXT:     AddressAlign:    0x0000000000000001
+ELF-MIPSEL-NEXT:     EntSize:         0x0000000000000001
 ELF-MIPSEL-NEXT:     Content:         48656C6C6F20576F726C640A00
 ELF-MIPSEL-NEXT:   - Name:            .reginfo
 ELF-MIPSEL-NEXT:     Type:            SHT_MIPS_REGINFO
@@ -481,6 +483,7 @@ ELF-MIPS64EL-NEXT:   - Name:            .rela.data
 ELF-MIPS64EL-NEXT:     Type:            SHT_RELA
 ELF-MIPS64EL-NEXT:     Link:            .symtab
 ELF-MIPS64EL-NEXT:     AddressAlign:    0x0000000000000008
+ELF-MIPS64EL-NEXT:     EntSize:         0x0000000000000018
 ELF-MIPS64EL-NEXT:     Info:            .data
 ELF-MIPS64EL-NEXT:     Relocations:
 ELF-MIPS64EL-NEXT:       - Offset:          0
@@ -494,6 +497,7 @@ ELF-MIPS64EL-NEXT:   - Name:            .MIPS.options
 ELF-MIPS64EL-NEXT:     Type:            SHT_MIPS_OPTIONS
 ELF-MIPS64EL-NEXT:     Flags:           [ SHF_ALLOC, SHF_MIPS_NOSTRIP ]
 ELF-MIPS64EL-NEXT:     AddressAlign:    0x0000000000000008
+ELF-MIPS64EL-NEXT:     EntSize:         0x0000000000000001
 ELF-MIPS64EL-NEXT:     Content:         '01280000000000000000000000000000000000000000000000000000000000000000000000000000'
 ELF-MIPS64EL-NEXT:   - Name:            .pdr
 ELF-MIPS64EL-NEXT:     Type:            SHT_PROGBITS
@@ -549,6 +553,7 @@ ELF-X86-64-NEXT:     Type:            SHT_RELA
 ELF-X86-64-NEXT:     Address:         0x0000000000000038
 ELF-X86-64-NEXT:     Link:            .symtab
 ELF-X86-64-NEXT:     AddressAlign:    0x0000000000000008
+ELF-X86-64-NEXT:     EntSize:         0x0000000000000018
 ELF-X86-64-NEXT:     Info:            .text
 ELF-X86-64-NEXT:     Relocations:
 ELF-X86-64-NEXT:       - Offset:          0x000000000000000D
@@ -651,7 +656,7 @@ WASM-NEXT: FileHeader:
 WASM-NEXT:   Version:         0x00000001
 WASM:        - Type:            CUSTOM
 WASM-NEXT:     Name:            linking
-WASM-NEXT:     Version:         1
+WASM-NEXT:     Version:         2
 WASM-NEXT:     SymbolTable:
 WASM-NEXT:       - Index:           0
 WASM-NEXT:         Kind:            FUNCTION
@@ -675,7 +680,7 @@ WASM:              Name:            var
 WASM:          SegmentInfo:
 WASM-NEXT:       - Index:           0
 WASM-NEXT:         Name:            .rodata..L.str
-WASM-NEXT:         Alignment:       1
+WASM-NEXT:         Alignment:       0
 WASM-NEXT:         Flags:           [  ]
 WASM-NEXT:       - Index:           1
 WASM:      ...
diff --git a/test/Object/objdump-no-sectionheaders.test b/test/Object/objdump-no-sectionheaders.test
index 5130100d8ee7c8ec2c556aebe195de06a029dacc..ebefa760bf14d0bc6e8c95f968edc2cafcd85bdf 100644
--- a/test/Object/objdump-no-sectionheaders.test
+++ b/test/Object/objdump-no-sectionheaders.test
@@ -2,5 +2,5 @@
 ; RUN:              | FileCheck %s
 
 ; CHECK: Sections:
-; CHECK: Idx Name          Size      Address          Type
+; CHECK: Idx Name          Size      VMA              Type
 ; CHECK-NOT: {{.}}
diff --git a/test/Object/objdump-relocations.test b/test/Object/objdump-relocations.test
index 3a1793e3daf6e4266af5ee4b4751ac8cdd5ea67b..73e9a4866c6f0e606b7fa5d30bb80526d91ad0af 100644
--- a/test/Object/objdump-relocations.test
+++ b/test/Object/objdump-relocations.test
@@ -60,10 +60,10 @@ ELF-MIPSEL: R_MIPS_CALL16 puts
 ELF-MIPSEL: R_MIPS_CALL16 SomeOtherFunction
 
 WASM:      CODE
-WASM-NEXT: R_WEBASSEMBLY_MEMORY_ADDR_SLEB .L.str
-WASM-NEXT: R_WEBASSEMBLY_FUNCTION_INDEX_LEB puts
-WASM-NEXT: R_WEBASSEMBLY_FUNCTION_INDEX_LEB .LSomeOtherFunction_bitcast
-WASM-NEXT: R_WEBASSEMBLY_FUNCTION_INDEX_LEB SomeOtherFunction
+WASM-NEXT: R_WASM_MEMORY_ADDR_SLEB .L.str
+WASM-NEXT: R_WASM_FUNCTION_INDEX_LEB puts
+WASM-NEXT: R_WASM_FUNCTION_INDEX_LEB .LSomeOtherFunction_bitcast
+WASM-NEXT: R_WASM_FUNCTION_INDEX_LEB SomeOtherFunction
 
 ELF-complex-x86-64: .text
 ELF-complex-x86-64-NEXT: R_X86_64_8 .data-4
diff --git a/test/Object/objdump-sectionheaders.test b/test/Object/objdump-sectionheaders.test
index efdd6bb5fc95ce2afe7caf7b6a721013dc90898a..973e1ca770179c250d807a183c45d6adc859e22d 100644
--- a/test/Object/objdump-sectionheaders.test
+++ b/test/Object/objdump-sectionheaders.test
@@ -5,7 +5,7 @@
 ; results in a way that we don't emulate.
 
 ; CHECK: Sections:
-; CHECK: Idx Name          Size      Address          Type
+; CHECK: Idx Name          Size     VMA              Type
 ; CHECK:   0               00000000 0000000000000000
 ; CHECK:   1 .text         00000026 0000000000000000 TEXT
 ; CHECK:   2 .rodata.str1.1 0000000d 0000000000000026 DATA
diff --git a/test/Object/wasm-bad-metadata-version.yaml b/test/Object/wasm-bad-metadata-version.yaml
index b2970bb59eadd02ccf35413657044941f3c5044f..7bda5b8d13aa538169a7ca9284db17eb5d647b57 100644
--- a/test/Object/wasm-bad-metadata-version.yaml
+++ b/test/Object/wasm-bad-metadata-version.yaml
@@ -8,4 +8,4 @@ Sections:
     Name:            linking
     Version:         0
 
-# CHECK: {{.*}}: Unexpected metadata version: 0 (Expected: 1)
+# CHECK: {{.*}}: Unexpected metadata version: 0 (Expected: 2)
diff --git a/test/Object/yaml2obj-readobj.test b/test/Object/yaml2obj-readobj.test
index 3bd0c6b0269a75fc55e8c9e1d9d68398c3253832..e7ee2a016a40783aaa6e61801fc4d054d4b9044f 100644
--- a/test/Object/yaml2obj-readobj.test
+++ b/test/Object/yaml2obj-readobj.test
@@ -13,16 +13,19 @@ RUN:   | FileCheck %s --check-prefix COFF-I386
 // COFF-I386-NEXT:       Offset: 0xE
 // COFF-I386-NEXT:       Type: IMAGE_REL_I386_DIR32 (6)
 // COFF-I386-NEXT:       Symbol: L_.str
+// COFF-I386-NEXT:       SymbolIndex: 5
 // COFF-I386-NEXT:     }
 // COFF-I386-NEXT:     Relocation {
 // COFF-I386-NEXT:       Offset: 0x13
 // COFF-I386-NEXT:       Type: IMAGE_REL_I386_REL32 (20)
 // COFF-I386-NEXT:       Symbol: _puts
+// COFF-I386-NEXT:       SymbolIndex: 6
 // COFF-I386-NEXT:     }
 // COFF-I386-NEXT:     Relocation {
 // COFF-I386-NEXT:       Offset: 0x18
 // COFF-I386-NEXT:       Type: IMAGE_REL_I386_REL32 (20)
 // COFF-I386-NEXT:       Symbol: _SomeOtherFunction
+// COFF-I386-NEXT:       SymbolIndex: 7
 // COFF-I386-NEXT:     }
 // COFF-I386-NEXT:   }
 // COFF-I386-NEXT: ]
diff --git a/test/ObjectYAML/wasm/code_section.yaml b/test/ObjectYAML/wasm/code_section.yaml
index 0d7cf61638bccd234c21a5d4a97efee2f26d023d..0171938a6a30437fdf8015d4be3fb2e5ad363a3c 100644
--- a/test/ObjectYAML/wasm/code_section.yaml
+++ b/test/ObjectYAML/wasm/code_section.yaml
@@ -20,10 +20,10 @@ Sections:
       - 1
   - Type:            CODE
     Relocations:
-      - Type:            R_WEBASSEMBLY_TABLE_INDEX_SLEB
+      - Type:            R_WASM_TABLE_INDEX_SLEB
         Index:           0
         Offset:          0x00000006
-      - Type:            R_WEBASSEMBLY_FUNCTION_INDEX_LEB
+      - Type:            R_WASM_FUNCTION_INDEX_LEB
         Index:           1
         Offset:          0x00000025
     Functions:
@@ -39,7 +39,7 @@ Sections:
         Body:            108180808000210020000F0B
   - Type:            CUSTOM
     Name:            linking
-    Version:         1
+    Version:         2
     SymbolTable:
       - Index:           0
         Kind:            FUNCTION
@@ -69,10 +69,10 @@ Sections:
 # CHECK:          - I64
 # CHECK:  - Type:            CODE
 # CHECK:    Relocations:
-# CHECK:      - Type:            R_WEBASSEMBLY_TABLE_INDEX_SLEB
+# CHECK:      - Type:            R_WASM_TABLE_INDEX_SLEB
 # CHECK:        Index:           0
 # CHECK:        Offset:          0x00000006
-# CHECK:      - Type:            R_WEBASSEMBLY_FUNCTION_INDEX_LEB
+# CHECK:      - Type:            R_WASM_FUNCTION_INDEX_LEB
 # CHECK:        Index:           1
 # CHECK:        Offset:          0x00000025
 # CHECK:    Functions:
diff --git a/test/ObjectYAML/wasm/data_section.yaml b/test/ObjectYAML/wasm/data_section.yaml
index ef5945dcaac6887cbd1f3c25c68e99f929402834..2ebdf9c4d1c40a6f983dbe76475f37fee89f435f 100644
--- a/test/ObjectYAML/wasm/data_section.yaml
+++ b/test/ObjectYAML/wasm/data_section.yaml
@@ -14,17 +14,17 @@ Sections:
           Value:           4
         Content:         '10001000'
     Relocations:
-      - Type:            R_WEBASSEMBLY_MEMORY_ADDR_I32
+      - Type:            R_WASM_MEMORY_ADDR_I32
         Index:           0
         Offset:          0x00000006
         Addend:          8
-      - Type:            R_WEBASSEMBLY_MEMORY_ADDR_I32
+      - Type:            R_WASM_MEMORY_ADDR_I32
         Index:           0
         Offset:          0x00000006
         Addend:          -6
   - Type:            CUSTOM
     Name:            linking
-    Version:         1
+    Version:         2
     SymbolTable:
       - Index:           0
         Kind:            DATA
@@ -40,11 +40,11 @@ Sections:
 # CHECK: Sections:
 # CHECK:   - Type:            DATA
 # CHECK-NEXT:     Relocations:
-# CHECK-NEXT:      - Type:            R_WEBASSEMBLY_MEMORY_ADDR_I32
+# CHECK-NEXT:      - Type:            R_WASM_MEMORY_ADDR_I32
 # CHECK-NEXT:        Index:           0
 # CHECK-NEXT:        Offset:          0x00000006
 # CHECK-NEXT:        Addend:          8
-# CHECK-NEXT:      - Type:            R_WEBASSEMBLY_MEMORY_ADDR_I32
+# CHECK-NEXT:      - Type:            R_WASM_MEMORY_ADDR_I32
 # CHECK-NEXT:        Index:           0
 # CHECK-NEXT:        Offset:          0x00000006
 # CHECK-NEXT:        Addend:          -6
diff --git a/test/ObjectYAML/wasm/elem_section.yaml b/test/ObjectYAML/wasm/elem_section.yaml
index 684256e0a9ef824cceb8a08abef68d7baadd495f..73989dda878c06bb8a2e62cc7f9898938158b9f7 100644
--- a/test/ObjectYAML/wasm/elem_section.yaml
+++ b/test/ObjectYAML/wasm/elem_section.yaml
@@ -5,7 +5,7 @@ FileHeader:
 Sections:
   - Type:            TABLE
     Tables:         
-      - ElemType:          ANYFUNC
+      - ElemType:          FUNCREF
         Limits:
           Flags:           [ HAS_MAX ]
           Initial:         0x00000010
@@ -18,7 +18,7 @@ Sections:
         Functions:
           - 1
       - Offset:
-          Opcode:        GET_GLOBAL
+          Opcode:        GLOBAL_GET
           Index:         1
         Functions:
           - 4
@@ -34,7 +34,7 @@ Sections:
 # CHECK:           Value:            3
 # CHECK:         Functions: [ 1 ]
 # CHECK:       - Offset:
-# CHECK:           Opcode:           GET_GLOBAL
+# CHECK:           Opcode:           GLOBAL_GET
 # CHECK:           Index:            1
 # CHECK:         Functions: [ 4 ]
 # CHECK: ...
diff --git a/test/ObjectYAML/wasm/event_section.yaml b/test/ObjectYAML/wasm/event_section.yaml
index 017efda056ce14dfd2ff939dcaf446a81c9ee1c5..91683bd3da4c72ca66672348f1ba6d1e274dec84 100644
--- a/test/ObjectYAML/wasm/event_section.yaml
+++ b/test/ObjectYAML/wasm/event_section.yaml
@@ -23,7 +23,7 @@ Sections:
         SigIndex:        1
   - Type:            CODE
     Relocations:
-      - Type:            R_WEBASSEMBLY_EVENT_INDEX_LEB
+      - Type:            R_WASM_EVENT_INDEX_LEB
         Index:           1
         Offset:          0x00000006
     Functions:
@@ -32,7 +32,7 @@ Sections:
         Body:            200008808080800041000B
   - Type:            CUSTOM
     Name:            linking
-    Version:         1
+    Version:         2
     SymbolTable:
       - Index:           0
         Kind:            FUNCTION
@@ -69,7 +69,7 @@ Sections:
 # CHECK-NEXT:         SigIndex:        1
 # CHECK-NEXT:   - Type:            CODE
 # CHECK-NEXT:     Relocations:
-# CHECK-NEXT:       - Type:            R_WEBASSEMBLY_EVENT_INDEX_LEB
+# CHECK-NEXT:       - Type:            R_WASM_EVENT_INDEX_LEB
 # CHECK-NEXT:         Index:           1
 # CHECK-NEXT:         Offset:          0x00000006
 # CHECK-NEXT:     Functions:
@@ -78,7 +78,7 @@ Sections:
 # CHECK-NEXT:         Body:            200008808080800041000B
 # CHECK-NEXT:   - Type:            CUSTOM
 # CHECK-NEXT:     Name:            linking
-# CHECK-NEXT:     Version:         1
+# CHECK-NEXT:     Version:         2
 # CHECK-NEXT:     SymbolTable:
 # CHECK-NEXT:       - Index:           0
 # CHECK-NEXT:         Kind:            FUNCTION
diff --git a/test/ObjectYAML/wasm/import_section.yaml b/test/ObjectYAML/wasm/import_section.yaml
index fc75705ab2c3936ee7f2a219a9150bc707348776..90de6f0fec7f0db3f559af67928fd32a0564375b 100644
--- a/test/ObjectYAML/wasm/import_section.yaml
+++ b/test/ObjectYAML/wasm/import_section.yaml
@@ -31,7 +31,7 @@ Sections:
         Field:           imported_table
         Kind:            TABLE
         Table:
-          ElemType:      ANYFUNC
+          ElemType:      FUNCREF
           Limits:
             Flags:           [ HAS_MAX ]
             Initial:         0x00000020
@@ -63,7 +63,7 @@ Sections:
 # CHECK:         Field:           imported_table
 # CHECK:         Kind:            TABLE
 # CHECK:         Table:
-# CHECK:           ElemType:      ANYFUNC
+# CHECK:           ElemType:      FUNCREF
 # CHECK:           Limits:
 # CHECK:             Flags:           [ HAS_MAX ]
 # CHECK:             Initial:         0x00000020
diff --git a/test/ObjectYAML/wasm/invalid_global_weak.yaml b/test/ObjectYAML/wasm/invalid_global_weak.yaml
index c364075e2e45d0cbc536307d9d11afa998f375fa..4566e8f0f009de8445d910ddad7522223990d184 100644
--- a/test/ObjectYAML/wasm/invalid_global_weak.yaml
+++ b/test/ObjectYAML/wasm/invalid_global_weak.yaml
@@ -13,7 +13,7 @@ Sections:
         GlobalMutable:   false
   - Type:            CUSTOM
     Name:            linking
-    Version:         1
+    Version:         2
     SymbolTable:
       - Index:           0
         Kind:            GLOBAL
diff --git a/test/ObjectYAML/wasm/linking_section.yaml b/test/ObjectYAML/wasm/linking_section.yaml
index d1f0243037dfadc9a70a4496485122775487e9e7..69817f50970ad1239f2c8786756c7c2c6bf4dc8a 100644
--- a/test/ObjectYAML/wasm/linking_section.yaml
+++ b/test/ObjectYAML/wasm/linking_section.yaml
@@ -29,7 +29,7 @@ Sections:
         Content:         '11110000'
   - Type:            CUSTOM
     Name:            linking
-    Version:         1
+    Version:         2
     SymbolTable:
       - Index:           0
         Kind:            FUNCTION
diff --git a/test/ObjectYAML/wasm/producers_section.yaml b/test/ObjectYAML/wasm/producers_section.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..1aef68ca3e785e2cf2b4c1f1dc39a33aa0c18ab3
--- /dev/null
+++ b/test/ObjectYAML/wasm/producers_section.yaml
@@ -0,0 +1,29 @@
+# RUN: yaml2obj %s | obj2yaml | FileCheck %s
+--- !WASM
+FileHeader:
+  Version:         0x00000001
+Sections:
+  - Type:            CUSTOM
+    Name:            producers
+    Languages:
+      - Name:        C++
+        Version:     C++11
+    Tools:
+      - Name:        clang
+        Version:     123
+    SDKs:
+      - Name:        emscripten
+        Version:     9001
+...
+# CHECK:  Sections:
+# CHECK:    - Type:            CUSTOM
+# CHECK:      Name:            producers
+# CHECK:      Languages:
+# CHECK:        - Name:        'C++'
+# CHECK:          Version:     'C++11'
+# CHECK:      Tools:
+# CHECK:        - Name:        clang
+# CHECK:          Version:     '123'
+# CHECK:      SDKs:
+# CHECK:        - Name:        emscripten
+# CHECK:          Version:     '9001'
diff --git a/test/ObjectYAML/wasm/producers_section_repeat.yaml b/test/ObjectYAML/wasm/producers_section_repeat.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..9dd6b7fd1f7e0a2dd709fd83f9e43223de4d3968
--- /dev/null
+++ b/test/ObjectYAML/wasm/producers_section_repeat.yaml
@@ -0,0 +1,14 @@
+# RUN: yaml2obj %s | not obj2yaml 2>&1 | FileCheck %s
+--- !WASM
+FileHeader:
+  Version:         0x00000001
+Sections:
+  - Type:            CUSTOM
+    Name:            producers
+    Languages:
+      - Name:        C++
+        Version:     C++11
+      - Name:        C++
+        Version:     C++17
+...
+# CHECK:  Producers section contains repeated producer
diff --git a/test/ObjectYAML/wasm/table_section.yaml b/test/ObjectYAML/wasm/table_section.yaml
index 5996b63e1328c6adda4df1085d76a2edaee7be98..90e73fdd40bc6f179ce44a7e66f94f7edecd0e0e 100644
--- a/test/ObjectYAML/wasm/table_section.yaml
+++ b/test/ObjectYAML/wasm/table_section.yaml
@@ -5,7 +5,7 @@ FileHeader:
 Sections:
   - Type:            TABLE
     Tables:         
-      - ElemType:        ANYFUNC
+      - ElemType:        FUNCREF
         Limits:
           Flags:           [ HAS_MAX ]
           Initial:         0x00000010
@@ -17,7 +17,7 @@ Sections:
 # CHECK: Sections:
 # CHECK:  - Type:            TABLE
 # CHECK:    Tables:         
-# CHECK:      - ElemType:        ANYFUNC
+# CHECK:      - ElemType:        FUNCREF
 # CHECK:        Limits:
 # CHECK:          Flags:           [ HAS_MAX ]
 # CHECK:          Initial:         0x00000010
diff --git a/test/ObjectYAML/wasm/weak_symbols.yaml b/test/ObjectYAML/wasm/weak_symbols.yaml
index 3e9ca34deac066df4b5e33fc28cc45fb1230e927..fb85b829a371b9de95adb73a5cfbfcfacfc59ab2 100644
--- a/test/ObjectYAML/wasm/weak_symbols.yaml
+++ b/test/ObjectYAML/wasm/weak_symbols.yaml
@@ -36,7 +36,7 @@ Sections:
         Body:            00
   - Type:            CUSTOM
     Name:            linking
-    Version:         1
+    Version:         2
     SymbolTable:
       - Index:           0
         Kind:            FUNCTION
diff --git a/test/Other/Inputs/pass-pipelines.proftext b/test/Other/Inputs/pass-pipelines.proftext
new file mode 100644
index 0000000000000000000000000000000000000000..04a7c1c1a35aad2b9e52b10a80f1ee47178d897c
--- /dev/null
+++ b/test/Other/Inputs/pass-pipelines.proftext
@@ -0,0 +1 @@
+:ir
diff --git a/test/Other/X86/lto-hot-cold-split.ll b/test/Other/X86/lto-hot-cold-split.ll
new file mode 100644
index 0000000000000000000000000000000000000000..67de630e6e268ba3389be15ef36c1ec811efc0f5
--- /dev/null
+++ b/test/Other/X86/lto-hot-cold-split.ll
@@ -0,0 +1,10 @@
+; RUN: opt -module-summary %s -o %t.bc
+; RUN: llvm-lto -hot-cold-split=true -thinlto-action=run %t.bc -debug-pass=Structure 2>&1 | FileCheck %s -check-prefix=OLDPM-THINLTO-POSTLINK-Os
+
+; REQUIRES: asserts
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; OLDPM-THINLTO-POSTLINK-Os-LABEL: Pass Arguments
+; OLDPM-THINLTO-POSTLINK-Os-NOT: Hot Cold Splitting
diff --git a/test/Other/cgscc-libcall-update.ll b/test/Other/cgscc-libcall-update.ll
index 1bf482af7954348291872ccd968bd10d63474ab8..5aa61df6a57285fd119721a178a356508f07c963 100644
--- a/test/Other/cgscc-libcall-update.ll
+++ b/test/Other/cgscc-libcall-update.ll
@@ -15,7 +15,7 @@ bb:
   %tmp2 = getelementptr inbounds [1024 x i8], [1024 x i8]* %tmp, i64 0, i64 0
   call void @llvm.memcpy.p0i8.p0i8.i64(i8* %tmp2, i8* %arg1, i64 1024, i1 false)
 ; CHECK:         call void @llvm.memcpy
-  %tmp3 = call i64 @llvm.objectsize.i64.p0i8(i8* %tmp2, i1 false, i1 true)
+  %tmp3 = call i64 @llvm.objectsize.i64.p0i8(i8* %tmp2, i1 false, i1 true, i1 false)
   %tmp4 = call i8* @__strncpy_chk(i8* %arg2, i8* %tmp2, i64 1023, i64 %tmp3)
 ; CHECK-NOT:     call
 ; CHECK:         call i8* @strncpy(i8* %arg2, i8* nonnull %tmp2, i64 1023)
@@ -33,7 +33,7 @@ bb:
 
 declare i8* @my_special_strncpy(i8* %arg1, i8* %arg2, i64 %size)
 
-declare i64 @llvm.objectsize.i64.p0i8(i8*, i1, i1)
+declare i64 @llvm.objectsize.i64.p0i8(i8*, i1, i1, i1)
 
 declare i8* @__strncpy_chk(i8*, i8*, i64, i64)
 
diff --git a/test/Other/loop-deletion-printer.ll b/test/Other/loop-deletion-printer.ll
index d344568506cd7e80bfd7499615c4ee842a00e2f4..6cb29ae374fbfd23599aaecd328073baaca35b4a 100644
--- a/test/Other/loop-deletion-printer.ll
+++ b/test/Other/loop-deletion-printer.ll
@@ -5,10 +5,14 @@
 ; RUN:     -passes=loop-instsimplify -print-after-all  2>&1 | FileCheck %s -check-prefix=SIMPLIFY
 ; RUN: opt < %s -disable-output \
 ; RUN:     -passes=loop-deletion,loop-instsimplify -print-after-all  2>&1 | FileCheck %s -check-prefix=DELETED
+; RUN: opt < %s -disable-output \
+; RUN:     -passes=loop-deletion,loop-instsimplify -print-after-all -print-module-scope  2>&1 | FileCheck %s -check-prefix=DELETED-BUT-PRINTED
 ;
 ; SIMPLIFY: IR Dump {{.*}} LoopInstSimplifyPass
 ; DELETED-NOT: IR Dump {{.*}}LoopInstSimplifyPass
 ; DELETED-NOT: IR Dump {{.*}}LoopDeletionPass
+; DELETED-BUT-PRINTED: IR Dump {{.*}}LoopDeletionPass {{.*invalidated:}}
+; DELETED-BUT-PRINTED-NOT: IR Dump {{.*}}LoopInstSimplifyPass
 
 define void @deleteme() {
 entry:
diff --git a/test/Other/new-pm-pgo.ll b/test/Other/new-pm-pgo.ll
index 5d6ed49025051b1752926bba5264309a0e83ecac..916309e625b2c1cd79a54f1bff4510458408fa1b 100644
--- a/test/Other/new-pm-pgo.ll
+++ b/test/Other/new-pm-pgo.ll
@@ -1,17 +1,19 @@
-; RUN: opt -debug-pass-manager -passes='default<O2>' -pgo-kind=new-pm-pgo-instr-gen-pipeline -profile-file='temp' %s 2>&1 |FileCheck %s --check-prefixes=GEN
+; RUN: opt -debug-pass-manager -passes='default<O2>' -pgo-kind=pgo-instr-gen-pipeline -profile-file='temp' %s 2>&1 |FileCheck %s --check-prefixes=GEN
 ; RUN: llvm-profdata merge %S/Inputs/new-pm-pgo.proftext -o %t.profdata
-; RUN: opt -debug-pass-manager -passes='default<O2>' -pgo-kind=new-pm-pgo-instr-use-pipeline -profile-file='%t.profdata' %s 2>&1 |FileCheck %s --check-prefixes=USE
-; RUN: opt -debug-pass-manager -passes='default<O2>' -pgo-kind=new-pm-pgo-sample-use-pipeline -profile-file='%S/Inputs/new-pm-pgo.prof' %s 2>&1 \
+; RUN: opt -debug-pass-manager -passes='default<O2>' -pgo-kind=pgo-instr-use-pipeline -profile-file='%t.profdata' %s 2>&1 |FileCheck %s --check-prefixes=USE
+; RUN: opt -debug-pass-manager -passes='default<O2>' -hot-cold-split -pgo-kind=pgo-instr-use-pipeline -profile-file='%t.profdata' %s 2>&1 |FileCheck %s --check-prefixes=USE --check-prefixes=SPLIT
+; RUN: opt -debug-pass-manager -passes='default<O2>' -pgo-kind=pgo-sample-use-pipeline -profile-file='%S/Inputs/new-pm-pgo.prof' %s 2>&1 \
 ; RUN:     |FileCheck %s --check-prefixes=SAMPLE_USE,SAMPLE_USE_O
-; RUN: opt -debug-pass-manager -passes='thinlto-pre-link<O2>' -pgo-kind=new-pm-pgo-sample-use-pipeline -profile-file='%S/Inputs/new-pm-pgo.prof' %s 2>&1 \
+; RUN: opt -debug-pass-manager -passes='thinlto-pre-link<O2>' -pgo-kind=pgo-sample-use-pipeline -profile-file='%S/Inputs/new-pm-pgo.prof' %s 2>&1 \
 ; RUN:     |FileCheck %s --check-prefixes=SAMPLE_USE,SAMPLE_USE_PRE_LINK
-; RUN: opt -debug-pass-manager -passes='thinlto<O2>' -pgo-kind=new-pm-pgo-sample-use-pipeline -profile-file='%S/Inputs/new-pm-pgo.prof' %s 2>&1 \
+; RUN: opt -debug-pass-manager -passes='thinlto<O2>' -pgo-kind=pgo-sample-use-pipeline -profile-file='%S/Inputs/new-pm-pgo.prof' %s 2>&1 \
 ; RUN:     |FileCheck %s --check-prefixes=SAMPLE_USE,SAMPLE_USE_POST_LINK
 ; RUN: opt -debug-pass-manager -passes='default<O2>' -new-pm-debug-info-for-profiling %s 2>&1 |FileCheck %s --check-prefixes=SAMPLE_GEN
 ;
 ; GEN: Running pass: PGOInstrumentationGen
 ; USE: Running pass: PGOInstrumentationUse
 ; USE: Running pass: PGOIndirectCallPromotion
+; SPLIT: Running pass: HotColdSplittingPass
 ; USE: Running pass: PGOMemOPSizeOpt
 ; SAMPLE_USE_O: Running pass: ModuleToFunctionPassAdaptor<{{.*}}AddDiscriminatorsPass{{.*}}>
 ; SAMPLE_USE_PRE_LINK: Running pass: ModuleToFunctionPassAdaptor<{{.*}}AddDiscriminatorsPass{{.*}}>
diff --git a/test/Other/new-pm-thinlto-defaults.ll b/test/Other/new-pm-thinlto-defaults.ll
index 3d1c9092c5b4a382cb7fe004d6f91338915ea62e..4d05f73c5936f6671c9af11166c05ba8d0cd7b96 100644
--- a/test/Other/new-pm-thinlto-defaults.ll
+++ b/test/Other/new-pm-thinlto-defaults.ll
@@ -26,10 +26,6 @@
 ; RUN: opt -disable-verify -debug-pass-manager -new-pm-debug-info-for-profiling \
 ; RUN:     -passes='thinlto-pre-link<O2>,name-anon-globals' -S  %s 2>&1 \
 ; RUN:     | FileCheck %s --check-prefixes=CHECK-DIS,CHECK-O,CHECK-O2,CHECK-PRELINK-O,CHECK-PRELINK-O2
-; Enabling the hot-cold-split pass should not affect the ThinLTO pre-link
-; RUN: opt -disable-verify -debug-pass-manager \
-; RUN:     -passes='thinlto-pre-link<O2>,name-anon-globals' -hot-cold-split -S  %s 2>&1 \
-; RUN:     | FileCheck %s --check-prefixes=CHECK-O,CHECK-O2,CHECK-PRELINK-O,CHECK-PRELINK-O-NODIS,CHECK-PRELINK-O2
 ;
 ; Postlink pipelines:
 ; RUN: opt -disable-verify -debug-pass-manager \
@@ -59,13 +55,13 @@
 ; CHECK-EP-PIPELINE-START-NEXT: Running pass: NoOpModulePass
 ; CHECK-DIS-NEXT: Running pass: ModuleToFunctionPassAdaptor<{{.*}}AddDiscriminatorsPass{{.*}}>
 ; CHECK-DIS-NEXT: Running analysis: InnerAnalysisManagerProxy
+; CHECK-O-NEXT: Running pass: PassManager<{{.*}}Module{{.*}}>
+; CHECK-O-NEXT: Starting llvm::Module pass manager run.
 ; CHECK-POSTLINK-O-NEXT: Running pass: PGOIndirectCallPromotion
 ; CHECK-POSTLINK-O-NEXT: Running analysis: ProfileSummaryAnalysis
 ; CHECK-POSTLINK-O-NEXT: Running analysis: InnerAnalysisManagerProxy
 ; CHECK-POSTLINK-O-NEXT: Running analysis: OptimizationRemarkEmitterAnalysis
 ; CHECK-POSTLINK-O-NEXT: Running analysis: PassInstrumentationAnalysis
-; CHECK-O-NEXT: Running pass: PassManager<{{.*}}Module{{.*}}>
-; CHECK-O-NEXT: Starting llvm::Module pass manager run.
 ; CHECK-O-NEXT: Running pass: InferFunctionAttrsPass
 ; CHECK-O-NEXT: Running analysis: TargetLibraryAnalysis
 ; CHECK-O-NEXT: Running pass: ModuleToFunctionPassAdaptor<{{.*}}PassManager{{.*}}>
diff --git a/test/Other/opt-bisect-helper.py b/test/Other/opt-bisect-helper.py
index d75950f11d89958a1e20704cdc0776c00d43ceae..d2ab4ce096f93991a329f3b5309b09b5d83d3d28 100755
--- a/test/Other/opt-bisect-helper.py
+++ b/test/Other/opt-bisect-helper.py
@@ -1,5 +1,7 @@
 #!/usr/bin/env python
 
+from __future__ import print_function
+
 import os
 import sys
 import argparse
diff --git a/test/Other/opt-hot-cold-split.ll b/test/Other/opt-hot-cold-split.ll
index ab08258359ffeeaf66b7210211f1624682b0aea8..90358bf5567e1913357b528bd42a0d0f0704f64c 100644
--- a/test/Other/opt-hot-cold-split.ll
+++ b/test/Other/opt-hot-cold-split.ll
@@ -1,296 +1,22 @@
-; RUN: opt -mtriple=x86_64-- -Os -hotcoldsplit -debug-pass=Structure < %s -o /dev/null 2>&1 | FileCheck %s
+; RUN: opt -mtriple=x86_64-- -Os -hot-cold-split=true -debug-pass=Structure < %s -o /dev/null 2>&1 | FileCheck %s -check-prefix=DEFAULT-Os
+; RUN: opt -mtriple=x86_64-- -Os -hot-cold-split=true -passes='lto-pre-link<Os>' -debug-pass-manager < %s -o /dev/null 2>&1 | FileCheck %s -check-prefix=LTO-PRELINK-Os
+; RUN: opt -mtriple=x86_64-- -Os -hot-cold-split=true -passes='thinlto-pre-link<Os>' -debug-pass-manager < %s -o /dev/null 2>&1 | FileCheck %s -check-prefix=THINLTO-PRELINK-Os
+; RUN: opt -mtriple=x86_64-- -Os -hot-cold-split=true -passes='thinlto<Os>' -debug-pass-manager < %s -o /dev/null 2>&1 | FileCheck %s -check-prefix=THINLTO-POSTLINK-Os
+
 ; REQUIRES: asserts
 
-; CHECK-LABEL: Pass Arguments:
-; CHECK-NEXT: Target Transform Information
-; CHECK-NEXT: Type-Based Alias Analysis
-; CHECK-NEXT: Scoped NoAlias Alias Analysis
-; CHECK-NEXT: Assumption Cache Tracker
-; CHECK-NEXT: Target Library Information
-; CHECK-NEXT:   FunctionPass Manager
-; CHECK-NEXT:     Module Verifier
-; CHECK-NEXT:     Instrument function entry/exit with calls to e.g. mcount() (pre inlining)
-; CHECK-NEXT:     Simplify the CFG
-; CHECK-NEXT:     Dominator Tree Construction
-; CHECK-NEXT:     SROA
-; CHECK-NEXT:     Early CSE
-; CHECK-NEXT:     Lower 'expect' Intrinsics
-; CHECK-NEXT:  Pass Arguments:
-; CHECK-NEXT:  Target Library Information
-; CHECK-NEXT:  Target Transform Information
-;              Target Pass Configuration
-; CHECK:       Type-Based Alias Analysis
-; CHECK-NEXT:  Scoped NoAlias Alias Analysis
-; CHECK-NEXT:  Assumption Cache Tracker
-; CHECK-NEXT:  Profile summary info
-; CHECK-NEXT:    ModulePass Manager
-; CHECK-NEXT:      Force set function attributes
-; CHECK-NEXT:      Infer set function attributes
-; CHECK-NEXT:      Interprocedural Sparse Conditional Constant Propagation
-; CHECK-NEXT:        Unnamed pass: implement Pass::getPassName()
-; CHECK-NEXT:     Called Value Propagation
-; CHECK-NEXT:     Global Variable Optimizer
-; CHECK-NEXT:       Unnamed pass: implement Pass::getPassName()
-; CHECK-NEXT:     FunctionPass Manager
-; CHECK-NEXT:       Dominator Tree Construction
-; CHECK-NEXT:       Promote Memory to Register
-; CHECK-NEXT:     Dead Argument Elimination
-; CHECK-NEXT:     FunctionPass Manager
-; CHECK-NEXT:       Dominator Tree Construction
-; CHECK-NEXT:       Basic Alias Analysis (stateless AA impl)
-; CHECK-NEXT:       Function Alias Analysis Results
-; CHECK-NEXT:       Natural Loop Information
-; CHECK-NEXT:       Lazy Branch Probability Analysis
-; CHECK-NEXT:       Lazy Block Frequency Analysis
-; CHECK-NEXT:       Optimization Remark Emitter
-; CHECK-NEXT:       Combine redundant instructions
-; CHECK-NEXT:       Simplify the CFG
-; CHECK-NEXT:     CallGraph Construction
-; CHECK-NEXT:     Globals Alias Analysis
-; CHECK-NEXT:     Call Graph SCC Pass Manager
-; CHECK-NEXT:       Remove unused exception handling info
-; CHECK-NEXT:       Function Integration/Inlining
-; CHECK-NEXT:       Deduce function attributes
-; CHECK-NEXT:       FunctionPass Manager
-; CHECK-NEXT:         Dominator Tree Construction
-; CHECK-NEXT:         SROA
-; CHECK-NEXT:         Basic Alias Analysis (stateless AA impl)
-; CHECK-NEXT:         Function Alias Analysis Results
-; CHECK-NEXT:         Memory SSA
-; CHECK-NEXT:         Early CSE w/ MemorySSA
-; CHECK-NEXT:         Speculatively execute instructions if target has divergent branches
-; CHECK-NEXT:         Basic Alias Analysis (stateless AA impl)
-; CHECK-NEXT:         Function Alias Analysis Results
-; CHECK-NEXT:         Lazy Value Information Analysis
-; CHECK-NEXT:         Jump Threading
-; CHECK-NEXT:         Value Propagation
-; CHECK-NEXT:         Simplify the CFG
-; CHECK-NEXT:         Dominator Tree Construction
-; CHECK-NEXT:         Basic Alias Analysis (stateless AA impl)
-; CHECK-NEXT:         Function Alias Analysis Results
-; CHECK-NEXT:         Natural Loop Information
-; CHECK-NEXT:         Lazy Branch Probability Analysis
-; CHECK-NEXT:         Lazy Block Frequency Analysis
-; CHECK-NEXT:         Optimization Remark Emitter
-; CHECK-NEXT:         Combine redundant instructions
-; CHECK-NEXT:         Optimization Remark Emitter
-; CHECK-NEXT:         Tail Call Elimination
-; CHECK-NEXT:         Simplify the CFG
-; CHECK-NEXT:         Reassociate expressions
-; CHECK-NEXT:         Dominator Tree Construction
-; CHECK-NEXT:         Natural Loop Information
-; CHECK-NEXT:         Canonicalize natural loops
-; CHECK-NEXT:         LCSSA Verifier
-; CHECK-NEXT:         Loop-Closed SSA Form Pass
-; CHECK-NEXT:         Basic Alias Analysis (stateless AA impl)
-; CHECK-NEXT:         Function Alias Analysis Results
-; CHECK-NEXT:         Scalar Evolution Analysis
-; CHECK-NEXT:         Loop Pass Manager
-; CHECK-NEXT:           Rotate Loops
-; CHECK-NEXT:           Loop Invariant Code Motion
-; CHECK-NEXT:           Unswitch loops
-; CHECK-NEXT:         Simplify the CFG
-; CHECK-NEXT:         Dominator Tree Construction
-; CHECK-NEXT:         Basic Alias Analysis (stateless AA impl)
-; CHECK-NEXT:         Function Alias Analysis Results
-; CHECK-NEXT:         Natural Loop Information
-; CHECK-NEXT:         Lazy Branch Probability Analysis
-; CHECK-NEXT:         Lazy Block Frequency Analysis
-; CHECK-NEXT:         Optimization Remark Emitter
-; CHECK-NEXT:         Combine redundant instructions
-; CHECK-NEXT:         Canonicalize natural loops
-; CHECK-NEXT:         LCSSA Verifier
-; CHECK-NEXT:         Loop-Closed SSA Form Pass
-; CHECK-NEXT:         Scalar Evolution Analysis
-; CHECK-NEXT:         Loop Pass Manager
-; CHECK-NEXT:           Induction Variable Simplification
-; CHECK-NEXT:           Recognize loop idioms
-; CHECK-NEXT:           Delete dead loops
-; CHECK-NEXT:           Unroll loops
-; CHECK-NEXT:         MergedLoadStoreMotion
-; CHECK-NEXT:         Phi Values Analysis
-; CHECK-NEXT:         Basic Alias Analysis (stateless AA impl)
-; CHECK-NEXT:         Function Alias Analysis Results
-; CHECK-NEXT:         Memory Dependence Analysis
-; CHECK-NEXT:         Lazy Branch Probability Analysis
-; CHECK-NEXT:         Lazy Block Frequency Analysis
-; CHECK-NEXT:         Optimization Remark Emitter
-; CHECK-NEXT:         Global Value Numbering
-; CHECK-NEXT:         Phi Values Analysis
-; CHECK-NEXT:         Basic Alias Analysis (stateless AA impl)
-; CHECK-NEXT:         Function Alias Analysis Results
-; CHECK-NEXT:         Memory Dependence Analysis
-; CHECK-NEXT:         MemCpy Optimization
-; CHECK-NEXT:         Sparse Conditional Constant Propagation
-; CHECK-NEXT:         Demanded bits analysis
-; CHECK-NEXT:         Bit-Tracking Dead Code Elimination
-; CHECK-NEXT:         Basic Alias Analysis (stateless AA impl)
-; CHECK-NEXT:         Function Alias Analysis Results
-; CHECK-NEXT:         Natural Loop Information
-; CHECK-NEXT:         Lazy Branch Probability Analysis
-; CHECK-NEXT:         Lazy Block Frequency Analysis
-; CHECK-NEXT:         Optimization Remark Emitter
-; CHECK-NEXT:         Combine redundant instructions
-; CHECK-NEXT:         Lazy Value Information Analysis
-; CHECK-NEXT:         Jump Threading
-; CHECK-NEXT:         Value Propagation
-; CHECK-NEXT:         Basic Alias Analysis (stateless AA impl)
-; CHECK-NEXT:         Function Alias Analysis Results
-; CHECK-NEXT:         Phi Values Analysis
-; CHECK-NEXT:         Memory Dependence Analysis
-; CHECK-NEXT:         Dead Store Elimination
-; CHECK-NEXT:         Natural Loop Information
-; CHECK-NEXT:         Canonicalize natural loops
-; CHECK-NEXT:         LCSSA Verifier
-; CHECK-NEXT:         Loop-Closed SSA Form Pass
-; CHECK-NEXT:         Basic Alias Analysis (stateless AA impl)
-; CHECK-NEXT:         Function Alias Analysis Results
-; CHECK-NEXT:         Scalar Evolution Analysis
-; CHECK-NEXT:         Loop Pass Manager
-; CHECK-NEXT:           Loop Invariant Code Motion
-; CHECK-NEXT:         Post-Dominator Tree Construction
-; CHECK-NEXT:         Aggressive Dead Code Elimination
-; CHECK-NEXT:         Simplify the CFG
-; CHECK-NEXT:         Dominator Tree Construction
-; CHECK-NEXT:         Basic Alias Analysis (stateless AA impl)
-; CHECK-NEXT:         Function Alias Analysis Results
-; CHECK-NEXT:         Natural Loop Information
-; CHECK-NEXT:         Lazy Branch Probability Analysis
-; CHECK-NEXT:         Lazy Block Frequency Analysis
-; CHECK-NEXT:         Optimization Remark Emitter
-; CHECK-NEXT:         Combine redundant instructions
-; CHECK-NEXT:     A No-Op Barrier Pass
-; CHECK-NEXT:     Eliminate Available Externally Globals
-; CHECK-NEXT:     CallGraph Construction
-; CHECK-NEXT:     Deduce function attributes in RPO
-; CHECK-NEXT:     Global Variable Optimizer
-; CHECK-NEXT:       Unnamed pass: implement Pass::getPassName()
-; CHECK-NEXT:     Dead Global Elimination
-; CHECK-NEXT:     CallGraph Construction
-; CHECK-NEXT:     Globals Alias Analysis
-; CHECK-NEXT:     FunctionPass Manager
-; CHECK-NEXT:       Float to int
-; CHECK-NEXT:       Dominator Tree Construction
-; CHECK-NEXT:       Natural Loop Information
-; CHECK-NEXT:       Canonicalize natural loops
-; CHECK-NEXT:       LCSSA Verifier
-; CHECK-NEXT:       Loop-Closed SSA Form Pass
-; CHECK-NEXT:       Basic Alias Analysis (stateless AA impl)
-; CHECK-NEXT:       Function Alias Analysis Results
-; CHECK-NEXT:       Scalar Evolution Analysis
-; CHECK-NEXT:       Loop Pass Manager
-; CHECK-NEXT:         Rotate Loops
-; CHECK-NEXT:       Loop Access Analysis
-; CHECK-NEXT:       Lazy Branch Probability Analysis
-; CHECK-NEXT:       Lazy Block Frequency Analysis
-; CHECK-NEXT:       Optimization Remark Emitter
-; CHECK-NEXT:       Loop Distribution
-; CHECK-NEXT:       Branch Probability Analysis
-; CHECK-NEXT:       Block Frequency Analysis
-; CHECK-NEXT:       Scalar Evolution Analysis
-; CHECK-NEXT:       Basic Alias Analysis (stateless AA impl)
-; CHECK-NEXT:       Function Alias Analysis Results
-; CHECK-NEXT:       Loop Access Analysis
-; CHECK-NEXT:       Demanded bits analysis
-; CHECK-NEXT:       Lazy Branch Probability Analysis
-; CHECK-NEXT:       Lazy Block Frequency Analysis
-; CHECK-NEXT:       Optimization Remark Emitter
-; CHECK-NEXT:       Loop Vectorization
-; CHECK-NEXT:       Canonicalize natural loops
-; CHECK-NEXT:       Scalar Evolution Analysis
-; CHECK-NEXT:       Function Alias Analysis Results
-; CHECK-NEXT:       Loop Access Analysis
-; CHECK-NEXT:       Loop Load Elimination
-; CHECK-NEXT:       Basic Alias Analysis (stateless AA impl)
-; CHECK-NEXT:       Function Alias Analysis Results
-; CHECK-NEXT:       Lazy Branch Probability Analysis
-; CHECK-NEXT:       Lazy Block Frequency Analysis
-; CHECK-NEXT:       Optimization Remark Emitter
-; CHECK-NEXT:       Combine redundant instructions
-; CHECK-NEXT:       Simplify the CFG
-; CHECK-NEXT:       Dominator Tree Construction
-; CHECK-NEXT:       Natural Loop Information
-; CHECK-NEXT:       Scalar Evolution Analysis
-; CHECK-NEXT:       Basic Alias Analysis (stateless AA impl)
-; CHECK-NEXT:       Function Alias Analysis Results
-; CHECK-NEXT:       Demanded bits analysis
-; CHECK-NEXT:       Lazy Branch Probability Analysis
-; CHECK-NEXT:       Lazy Block Frequency Analysis
-; CHECK-NEXT:       Optimization Remark Emitter
-; CHECK-NEXT:       SLP Vectorizer
-; CHECK-NEXT:       Optimization Remark Emitter
-; CHECK-NEXT:       Combine redundant instructions
-; CHECK-NEXT:       Canonicalize natural loops
-; CHECK-NEXT:       LCSSA Verifier
-; CHECK-NEXT:       Loop-Closed SSA Form Pass
-; CHECK-NEXT:       Scalar Evolution Analysis
-; CHECK-NEXT:       Loop Pass Manager
-; CHECK-NEXT:         Unroll loops
-; CHECK-NEXT:       Lazy Branch Probability Analysis
-; CHECK-NEXT:       Lazy Block Frequency Analysis
-; CHECK-NEXT:       Optimization Remark Emitter
-; CHECK-NEXT:       Combine redundant instructions
-; CHECK-NEXT:       Canonicalize natural loops
-; CHECK-NEXT:       LCSSA Verifier
-; CHECK-NEXT:       Loop-Closed SSA Form Pass
-; CHECK-NEXT:       Scalar Evolution Analysis
-; CHECK-NEXT:       Loop Pass Manager
-; CHECK-NEXT:         Loop Invariant Code Motion
-; CHECK-NEXT:       Lazy Branch Probability Analysis
-; CHECK-NEXT:       Lazy Block Frequency Analysis
-; CHECK-NEXT:       Optimization Remark Emitter
-; CHECK-NEXT:       Warn about non-applied transformations
-; CHECK-NEXT:       Alignment from assumptions
-; CHECK-NEXT:     Strip Unused Function Prototypes
-; CHECK-NEXT:     Dead Global Elimination
-; CHECK-NEXT:     Merge Duplicate Global Constants
-; CHECK-NEXT:     FunctionPass Manager
-; CHECK-NEXT:       Dominator Tree Construction
-; CHECK-NEXT:       Natural Loop Information
-; CHECK-NEXT:       Branch Probability Analysis
-; CHECK-NEXT:       Block Frequency Analysis
-; CHECK-NEXT:       Canonicalize natural loops
-; CHECK-NEXT:       LCSSA Verifier
-; CHECK-NEXT:       Loop-Closed SSA Form Pass
-; CHECK-NEXT:       Basic Alias Analysis (stateless AA impl)
-; CHECK-NEXT:       Function Alias Analysis Results
-; CHECK-NEXT:       Scalar Evolution Analysis
-; CHECK-NEXT:       Branch Probability Analysis
-; CHECK-NEXT:       Block Frequency Analysis
-; CHECK-NEXT:       Loop Pass Manager
-; CHECK-NEXT:         Loop Sink
-; CHECK-NEXT:       Lazy Branch Probability Analysis
-; CHECK-NEXT:       Lazy Block Frequency Analysis
-; CHECK-NEXT:       Optimization Remark Emitter
-; CHECK-NEXT:       Remove redundant instructions
-; CHECK-NEXT:       Hoist/decompose integer division and remainder
-; CHECK-NEXT:       Simplify the CFG
-; CHECK-NEXT:     Hot Cold Splitting
-; CHECK-NEXT:       Unnamed pass: implement Pass::getPassName()
-; CHECK-NEXT:     FunctionPass Manager
-; CHECK-NEXT:       Module Verifier
-; CHECK-NEXT:     Bitcode Writer
-; CHECK-NEXT: Pass Arguments:  -domtree
-; CHECK-NEXT:   FunctionPass Manager
-; CHECK-NEXT:     Dominator Tree Construction
-; CHECK-NEXT: Pass Arguments:  -targetlibinfo -domtree -loops -branch-prob -block-freq
-; CHECK-NEXT: Target Library Information
-; CHECK-NEXT:   FunctionPass Manager
-; CHECK-NEXT:     Dominator Tree Construction
-; CHECK-NEXT:     Natural Loop Information
-; CHECK-NEXT:     Branch Probability Analysis
-; CHECK-NEXT:     Block Frequency Analysis
-; CHECK-NEXT: Pass Arguments:  -targetlibinfo -domtree -loops -branch-prob -block-freq
-; CHECK-NEXT: Target Library Information
-; CHECK-NEXT:   FunctionPass Manager
-; CHECK-NEXT:     Dominator Tree Construction
-; CHECK-NEXT:     Natural Loop Information
-; CHECK-NEXT:     Branch Probability Analysis
-; CHECK-NEXT:     Block Frequency Analysis
-; CHECK-NEXT: Pass Arguments:  -targetlibinfo -domtree -loops -branch-prob -block-freq
-; CHECK-NEXT: Target Library Information
-; CHECK-NEXT:   FunctionPass Manager
-; CHECK-NEXT:     Dominator Tree Construction
-; CHECK-NEXT:     Natural Loop Information
-; CHECK-NEXT:     Branch Probability Analysis
-; CHECK-NEXT:     Block Frequency Analysis
+; Splitting should occur after Mem2Reg and should be followed by InstCombine.
+
+; DEFAULT-Os: Promote Memory to Register
+; DEFAULT-Os: Hot Cold Splitting
+; DEFAULT-Os: Combine redundant instructions
+
+; LTO-PRELINK-Os-LABEL: Starting llvm::Module pass manager run.
+; LTO-PRELINK-Os: Running pass: {{.*}}PromotePass
+; LTO-PRELINK-Os: Running pass: HotColdSplittingPass
+
+; THINLTO-PRELINK-Os-LABEL: Running analysis: PassInstrumentationAnalysis
+; THINLTO-PRELINK-Os: Running pass: {{.*}}PromotePass
+; THINLTO-PRELINK-Os: Running pass: HotColdSplittingPass
+
+; THINLTO-POSTLINK-Os-NOT: HotColdSplitting
diff --git a/test/Other/pass-pipelines.ll b/test/Other/pass-pipelines.ll
index dddf1338a4d01b1b563699d8f5428e4b86ff64b8..dfddcee55e83323ffb945df4f3d10f6598e3b5e9 100644
--- a/test/Other/pass-pipelines.ll
+++ b/test/Other/pass-pipelines.ll
@@ -6,6 +6,16 @@
 ; RUN: opt -disable-output -disable-verify -debug-pass=Structure \
 ; RUN:     -O2 %s 2>&1 \
 ; RUN:     | FileCheck %s --check-prefix=CHECK-O2
+; RUN: llvm-profdata merge %S/Inputs/pass-pipelines.proftext -o %t.profdata
+; RUN: opt -disable-output -disable-verify -debug-pass=Structure \
+; RUN:     -pgo-kind=pgo-instr-use-pipeline -profile-file='%t.profdata' \
+; RUN:     -O2 %s 2>&1 \
+; RUN:     | FileCheck %s --check-prefix=CHECK-O2 --check-prefix=PGOUSE
+; RUN: opt -disable-output -disable-verify -debug-pass=Structure \
+; RUN:     -pgo-kind=pgo-instr-use-pipeline -profile-file='%t.profdata' \
+; RUN:     -hot-cold-split \
+; RUN:     -O2 %s 2>&1 \
+; RUN:     | FileCheck %s --check-prefix=CHECK-O2 --check-prefix=PGOUSE --check-prefix=SPLIT
 ;
 ; In the first pipeline there should just be a function pass manager, no other
 ; pass managers.
@@ -27,6 +37,12 @@
 ; Very carefully assert the CGSCC pass pipeline as it is fragile and unusually
 ; susceptible to phase ordering issues.
 ; CHECK-O2: CallGraph Construction
+; PGOUSE: Call Graph SCC Pass Manager
+; PGOUSE:      Function Integration/Inlining
+; PGOUSE: PGOInstrumentationUsePass
+; PGOUSE: PGOIndirectCallPromotion
+; SPLIT: Hot Cold Splitting
+; PGOUSE: CallGraph Construction
 ; CHECK-O2-NEXT: Globals Alias Analysis
 ; CHECK-O2-NEXT: Call Graph SCC Pass Manager
 ; CHECK-O2-NEXT: Remove unused exception handling info
diff --git a/test/Other/scc-deleted-printer.ll b/test/Other/scc-deleted-printer.ll
index e6323f0274a3811b07f1c9793d679706cf2141d5..f7b97c3d30eb1f2f335c43ee1fe3937e4d0a2f52 100644
--- a/test/Other/scc-deleted-printer.ll
+++ b/test/Other/scc-deleted-printer.ll
@@ -9,7 +9,7 @@
 ; INL: IR Dump After {{InlinerPass .*scc: .tester}}
 
 ; INL-MOD: IR Dump Before {{InlinerPass .*scc: .tester, foo}}
-; INL-MOD-NOT: IR Dump After {{InlinerPass}}
+; INL-MOD: IR Dump After {{InlinerPass .*invalidated: .*scc: .tester, foo}}
 ; INL-MOD: IR Dump Before {{InlinerPass .*scc: .tester}}
 ; INL-MOD: IR Dump After {{InlinerPass .*scc: .tester}}
 
diff --git a/test/TableGen/FixedLenDecoderEmitter/conflict.td b/test/TableGen/FixedLenDecoderEmitter/conflict.td
new file mode 100644
index 0000000000000000000000000000000000000000..a9a95bc05a32783f5f94be6f1c15ff7d75fc1e69
--- /dev/null
+++ b/test/TableGen/FixedLenDecoderEmitter/conflict.td
@@ -0,0 +1,35 @@
+// RUN: llvm-tblgen -gen-disassembler -I %p/../../../include %s -o - 2>%t
+// RUN: FileCheck %s < %t
+
+include "llvm/Target/Target.td"
+
+def MyTargetISA : InstrInfo;
+def MyTarget : Target { let InstructionSet = MyTargetISA; }
+
+def R0 : Register<"r0"> { let Namespace = "MyTarget"; }
+def GPR32 : RegisterClass<"MyTarget", [i32], 32, (add R0)>;
+
+class I<dag OOps, dag IOps, list<dag> Pat>
+  : Instruction {
+  let Namespace = "MyTarget";
+  let OutOperandList = OOps;
+  let InOperandList = IOps;
+  let Pattern = Pat;
+  bits<32> Inst;
+  bits<32> SoftFail;
+}
+
+def A : I<(outs GPR32:$dst), (ins GPR32:$src1), []> {
+  let Size = 4;
+  let Inst{31-0} = 0;
+}
+def B : I<(outs GPR32:$dst), (ins GPR32:$src1), []> {
+  let Size = 4;
+  let Inst{31-0} = 0;
+}
+
+// CHECK: Decoding Conflict:
+// CHECK:   00000000000000000000000000000000
+// CHECK:   ................................
+// CHECK: A 00000000000000000000000000000000
+// CHECK: B 00000000000000000000000000000000
diff --git a/test/TableGen/GlobalISelEmitterSkippedPatterns.td b/test/TableGen/GlobalISelEmitterSkippedPatterns.td
new file mode 100644
index 0000000000000000000000000000000000000000..30d858164f0de8e20a39dfebc153448cf6127269
--- /dev/null
+++ b/test/TableGen/GlobalISelEmitterSkippedPatterns.td
@@ -0,0 +1,45 @@
+// RUN: llvm-tblgen -warn-on-skipped-patterns -gen-global-isel -I %p/../../include %s -o /dev/null 2>&1 | FileCheck %s
+include "llvm/Target/Target.td"
+
+//===- Boiler plate target code -===//
+def MyTargetISA : InstrInfo;
+def MyTarget : Target { let InstructionSet = MyTargetISA; }
+
+let TargetPrefix = "mytarget" in {
+def int_mytarget_nop : Intrinsic<[llvm_i32_ty], [llvm_i32_ty], [IntrNoMem]>;
+}
+
+def R0 : Register<"r0"> { let Namespace = "MyTarget"; }
+def GPR32 : RegisterClass<"MyTarget", [i32], 32, (add R0)>;
+
+class I<dag OOps, dag IOps, list<dag> Pat>
+  : Instruction {
+  let Namespace = "MyTarget";
+  let OutOperandList = OOps;
+  let InOperandList = IOps;
+  let Pattern = Pat;
+}
+
+def complex : Operand<i32>, ComplexPattern<i32, 2, "SelectComplexPattern", []> {
+  let MIOperandInfo = (ops i32imm, i32imm);
+}
+
+def gi_complex :
+    GIComplexOperandMatcher<s32, "selectComplexPattern">,
+    GIComplexPatternEquiv<complex>;
+def complex_rr : Operand<i32>, ComplexPattern<i32, 2, "SelectComplexPatternRR", []> {
+  let MIOperandInfo = (ops GPR32, GPR32);
+}
+
+def gi_complex_rr :
+    GIComplexOperandMatcher<s32, "selectComplexPatternRR">,
+    GIComplexPatternEquiv<complex_rr>;
+
+def INSN : I<(outs GPR32:$dst), (ins GPR32:$src1, complex:$src2), []>;
+
+//===- Bail out when we define a variable twice wrt complex suboperands. -===//
+
+// CHECK: warning: Skipped pattern: Complex suboperand referenced more than once (Operand: x)
+def : Pat<(add (complex_rr GPR32:$x, GPR32:$y),
+               (complex_rr GPR32:$x, GPR32:$z)),
+               (INSN GPR32:$z, complex:$y)>;
diff --git a/test/TableGen/cond-bitlist.td b/test/TableGen/cond-bitlist.td
new file mode 100644
index 0000000000000000000000000000000000000000..bce615838dfc7c370d8865cb206ed9a90b4a6fba
--- /dev/null
+++ b/test/TableGen/cond-bitlist.td
@@ -0,0 +1,27 @@
+// RUN: llvm-tblgen %s | FileCheck %s
+// XFAIL: vg_leak
+
+class S<int s> {
+  bits<2> val = !cond(!eq(s, 8):  {0, 0},
+                      !eq(s, 16): 0b01,
+                      !eq(s, 32): 2,
+                      !eq(s, 64): {1, 1},
+                              1 : ?);
+}
+
+def D8  : S<8>;
+def D16 : S<16>;
+def D32 : S<32>;
+def D64 : S<64>;
+def D128: S<128>;
+// CHECK: def D128
+// CHECK-NEXT: bits<2> val = { ?, ? };
+// CHECK: def D16
+// CHECK-NEXT: bits<2> val = { 0, 1 };
+// CHECK: def D32
+// CHECK-NEXT: bits<2> val = { 1, 0 };
+// CHECK: def D64
+// CHECK-NEXT: bits<2> val = { 1, 1 };
+// CHECK: def D8
+// CHECK-NEXT: bits<2> val = { 0, 0 };
+
diff --git a/test/TableGen/cond-default.td b/test/TableGen/cond-default.td
new file mode 100644
index 0000000000000000000000000000000000000000..816bf10676fe3188640e6b210c08ff8efb44a5f0
--- /dev/null
+++ b/test/TableGen/cond-default.td
@@ -0,0 +1,11 @@
+// Check that not specifying a valid condition results in error
+
+// RUN: not llvm-tblgen %s 2>&1 | FileCheck %s
+// XFAIL: vg_leak
+
+class C<int x> {
+  string s  = !cond(!lt(x,0) : "negative", !gt(x,0) : "positive");
+}
+
+def Zero : C<0>;
+//CHECK: error: Zero does not have any true condition in:!cond(0: "negative", 0: "positive")
diff --git a/test/TableGen/cond-empty-list-arg.td b/test/TableGen/cond-empty-list-arg.td
new file mode 100644
index 0000000000000000000000000000000000000000..5f4ccade1697a263de5d2a2b381719fcddedadb9
--- /dev/null
+++ b/test/TableGen/cond-empty-list-arg.td
@@ -0,0 +1,8 @@
+// RUN: llvm-tblgen %s
+// XFAIL: vg_leak
+
+class C<bit cond> {
+  bit true = 1;
+  list<int> X = !cond(cond: [1, 2, 3], true : []);
+  list<int> Y = !cond(cond: [], true : [4, 5, 6]);
+}
diff --git a/test/TableGen/cond-inheritance.td b/test/TableGen/cond-inheritance.td
new file mode 100644
index 0000000000000000000000000000000000000000..4b4abdf72f388a69d9a4465815ae3ccf90163c88
--- /dev/null
+++ b/test/TableGen/cond-inheritance.td
@@ -0,0 +1,22 @@
+// Make sure !cond gets propagated across multiple layers of inheritance.
+// RUN: llvm-tblgen %s | FileCheck %s
+// XFAIL: vg_leak
+
+class getInt<int c> {
+  int ret = !cond(c: 0, 1 : 1);
+}
+
+class I1<int c> {
+  int i = getInt<c>.ret;
+}
+
+class I2<int c> : I1<c>;
+
+def DI1: I1<1>;
+// CHECK: def DI1 {     // I1
+// CHECK-NEXT: int i = 0;
+
+// CHECK: def DI2 {     // I1 I2
+// CHECK-NEXT: int i = 0;
+def DI2: I2<1>;
+
diff --git a/test/TableGen/cond-let.td b/test/TableGen/cond-let.td
new file mode 100644
index 0000000000000000000000000000000000000000..044878f2ab8e3d77a787b8a1401881e60ab1a672
--- /dev/null
+++ b/test/TableGen/cond-let.td
@@ -0,0 +1,36 @@
+// Check support for `!cond' operator as part of a `let' statement.
+// RUN: llvm-tblgen %s | FileCheck %s
+// XFAIL: vg_leak
+
+
+class C<bits<3> x, bits<4> y, bit z> {
+  bits<16> n;
+
+  let n{11}  = !cond(y{3}: 1,
+                     y{2}: x{0},
+                     y{1}: x{1},
+                     y{0}: x{2},
+                     {1} :?);
+  let n{10-9}= !cond(x{2}: y{3-2},
+                     x{1}: y{2-1},
+                     x{1}: y{1-0},
+                     {1} : ?);
+  let n{8-6} = !cond(x{2}: 0b010,  1 : 0b110);
+  let n{5-4} = !cond(x{1}: y{3-2}, 1 :  {0, 1});
+  let n{3-0} = !cond(x{0}: y{3-0}, 1 : {z, y{2}, y{1}, y{0}});
+}
+
+
+def C1 : C<{1, 0, 1}, {0, 1, 0, 1}, 0>;
+def C2 : C<{0, 1, 0}, {1, 0, 1, 0}, 1>;
+def C3 : C<{0, 0, 0}, {1, 0, 1, 0}, 0>;
+def C4 : C<{0, 0, 0}, {0, 0, 0, 0}, 0>;
+
+// CHECK: def C1
+// CHECK-NEXT: bits<16> n = { ?, ?, ?, ?, 1, 0, 1, 0, 1, 0, 0, 1, 0, 1, 0, 1 };
+// CHECK: def C2
+// CHECK-NEXT: bits<16> n = { ?, ?, ?, ?, 1, 0, 1, 1, 1, 0, 1, 0, 1, 0, 1, 0 };
+// CHECK: def C3
+// CHECK-NEXT: bits<16> n = { ?, ?, ?, ?, 1, ?, ?, 1, 1, 0, 0, 1, 0, 0, 1, 0 };
+// CHECK: def C4
+// CHECK-NEXT: bits<16> n = { ?, ?, ?, ?, ?, ?, ?, 1, 1, 0, 0, 1, 0, 0, 0, 0 };
diff --git a/test/TableGen/cond-list.td b/test/TableGen/cond-list.td
new file mode 100644
index 0000000000000000000000000000000000000000..aa013cea4e12fceac05eb6f2f4763e76dbbff83e
--- /dev/null
+++ b/test/TableGen/cond-list.td
@@ -0,0 +1,38 @@
+// RUN: llvm-tblgen %s | FileCheck %s
+// XFAIL: vg_leak
+
+
+class A<list<list<int>> vals> {
+  list<int> first = vals[0];
+  list<int> rest  = !cond(!empty(!tail(vals)): vals[0],
+                          1                 : vals[1]);
+}
+
+def A_OneEl : A<[[1,2,3]]>;
+// CHECK:      def A_OneEl {  // A
+// CHECK-NEXT: list<int> first = [1, 2, 3];
+// CHECK-NEXT: list<int> rest = [1, 2, 3];
+// CHECK-NEXT: }
+
+def A_TwoEl : A<[[1,2,3], [4,5,6]]>;
+// CHECK:      def A_TwoEl { // A
+// CHECK-NEXT: list<int> first = [1, 2, 3];
+// CHECK-NEXT: list<int> rest = [4, 5, 6];
+// CHECK-NEXT: }
+
+
+class B<list<int> v> {
+  list<int> vals = v;
+}
+class BB<list<list<int>> vals> : B<!cond(!empty(!tail(vals)): vals[0],  1 : vals[1])>;
+class BBB<list<list<int>> vals> : BB<vals>;
+
+def B_OneEl : BBB<[[1,2,3]]>;
+// CHECK:      def B_OneEl { //  B BB BBB
+// CHECK-NEXT: list<int> vals = [1, 2, 3];
+// CHECK-NEXT: }
+
+def B_TwoEl : BBB<[[1,2,3],[4,5,6]]>;
+// CHECK:      def B_TwoEl { // B BB BBB
+// CHECK-NEXT: list<int> vals = [4, 5, 6];
+// CHECK-NEXT: }
diff --git a/test/TableGen/cond-subclass.td b/test/TableGen/cond-subclass.td
new file mode 100644
index 0000000000000000000000000000000000000000..9f6f6e2cb8cc6d3bcfcf418581eae6384f49c13b
--- /dev/null
+++ b/test/TableGen/cond-subclass.td
@@ -0,0 +1,27 @@
+// Check that !cond with operands of different subtypes can
+// initialize a supertype variable.
+// RUN: llvm-tblgen %s | FileCheck %s
+// XFAIL: vg_leak
+
+class E<int dummy> {}
+class E1<int dummy> : E<dummy> {}
+class E2<int dummy> : E<dummy> {}
+
+class EX<int cc, E1 b, E2 c> {
+  E x = !cond(cc: b, 1 : c);
+}
+
+def E1d : E1<0>;
+def E2d : E2<0>;
+
+def EXd1 : EX<1, E1d, E2d>;
+def EXd2 : EX<0, E1d, E2d>;
+
+// CHECK: def EXd1 {
+// CHECK:   E x = E1d;
+// CHECK: }
+//
+// CHECK: def EXd2 {
+// CHECK:   E x = E2d;
+// CHECK: }
+
diff --git a/test/TableGen/cond-type.td b/test/TableGen/cond-type.td
new file mode 100644
index 0000000000000000000000000000000000000000..fd2a3cc52b7832deac60f034b27617d6f24ba0c7
--- /dev/null
+++ b/test/TableGen/cond-type.td
@@ -0,0 +1,11 @@
+// RUN: not llvm-tblgen %s 2>&1 | FileCheck %s
+// XFAIL: vg_leak
+
+class A<int dummy> {}
+class B<int dummy> : A<dummy> {}
+class C<int dummy> : A<dummy> {}
+
+// CHECK: Value 'x' of type 'C' is incompatible with initializer '{{.*}}' of type 'A'
+class X<int cc, B b, C c> {
+  C x = !cond(cc: b, 1 : c);
+}
diff --git a/test/TableGen/cond-usage.td b/test/TableGen/cond-usage.td
new file mode 100644
index 0000000000000000000000000000000000000000..055fd6d7c69f4239d313045d801c9ebfb574f466
--- /dev/null
+++ b/test/TableGen/cond-usage.td
@@ -0,0 +1,29 @@
+// RUN: llvm-tblgen %s | FileCheck %s
+// XFAIL: vg_leak
+
+// Check that !cond picks the first true value
+// CHECK:       class A
+// CHECK-NEXT:  string S = !cond(!eq(A:x, 10): "ten", !eq(A:x, 11): "eleven", !eq(A:x, 10): "TEN", !gt(A:x, 9): "MoreThanNine", 1: "unknown"); 
+// CHECK: B1
+// CHECK-NEXT: string S = "unknown"
+// CHECK: B10
+// CHECK-NEXT: string S = "ten";
+// CHECK: def B11
+// CHECK-NEXT: string S = "eleven";
+// CHECK: def B12
+// CHECK-NEXT:  string S = "MoreThanNine";
+// CHECK: def B9
+// CHECK-NEXT: string S = "unknown"
+
+class A<int x> {
+  string S = !cond(!eq(x,10) : "ten",
+                   !eq(x,11) : "eleven",
+                   !eq(x,10) : "TEN",
+                   !gt(x,9) : "MoreThanNine",
+                   !eq(1,1) : "unknown");
+}
+def B1  : A<1>;
+def B9  : A<9>;
+def B10 : A<10>;
+def B11 : A<11>;
+def B12 : A<12>;
diff --git a/test/TableGen/condsbit.td b/test/TableGen/condsbit.td
new file mode 100644
index 0000000000000000000000000000000000000000..e08ac97f68bf3cce556903d31e159c386ffb6fc5
--- /dev/null
+++ b/test/TableGen/condsbit.td
@@ -0,0 +1,15 @@
+// check that !cond works well with bit conditional values
+// RUN: llvm-tblgen %s | FileCheck %s
+// XFAIL: vg_leak
+// CHECK: a = 6
+// CHECK: a = 5
+
+class A<bit b = 1> {
+  bit true = 1;
+  int a = !cond(b: 5, true : 6);
+  bit c = !cond(b: 0, true : 1);
+  bits<1> d = !cond(b: 0, true : 1);
+}
+
+def X : A<0>;
+def Y : A;
diff --git a/test/ThinLTO/X86/Inputs/deadstrip.ll b/test/ThinLTO/X86/Inputs/deadstrip.ll
index a9161a31b2490e0f3cebd96b28d909a665d07a96..057305ae67e1a43a73a79075d3dc943f92d82fe8 100644
--- a/test/ThinLTO/X86/Inputs/deadstrip.ll
+++ b/test/ThinLTO/X86/Inputs/deadstrip.ll
@@ -20,3 +20,8 @@ define void @another_dead_func() {
     call void @dead_func()
     ret void
 }
+
+define linkonce_odr void @linkonceodrfuncwithalias() {
+entry:
+  ret void
+}
diff --git a/test/ThinLTO/X86/cache-icall.ll b/test/ThinLTO/X86/cache-icall.ll
index 5e64a713ffad20c51ece3036ccc8b468ee7e7384..034ff97cede2ee6778965afeb0a964280db23df2 100644
--- a/test/ThinLTO/X86/cache-icall.ll
+++ b/test/ThinLTO/X86/cache-icall.ll
@@ -4,8 +4,8 @@
 ; This affects code generated for any users of f(). Make sure that we don't pull a stale object
 ; file for %t.o from the cache.
 
-; RUN: opt -module-hash -module-summary -thinlto-bc %s -o %t.bc
-; RUN: opt -module-hash -module-summary -thinlto-bc %p/Inputs/cache-icall.ll -o %t2.bc
+; RUN: opt -module-hash -module-summary -thinlto-bc -thinlto-split-lto-unit %s -o %t.bc
+; RUN: opt -module-hash -module-summary -thinlto-bc -thinlto-split-lto-unit %p/Inputs/cache-icall.ll -o %t2.bc
 
 ; RUN: rm -Rf %t.cache && mkdir %t.cache
 
diff --git a/test/ThinLTO/X86/cfi-devirt.ll b/test/ThinLTO/X86/cfi-devirt.ll
index 7ade794d498b6e726863e3cec738f0e49fa39599..3fd0486c7e801b15789d51a4b779f8fbfc6cdc2e 100644
--- a/test/ThinLTO/X86/cfi-devirt.ll
+++ b/test/ThinLTO/X86/cfi-devirt.ll
@@ -2,7 +2,7 @@
 
 ; Test CFI devirtualization through the thin link and backend.
 
-; RUN: opt -thinlto-bc -o %t.o %s
+; RUN: opt -thinlto-bc -thinlto-split-lto-unit -o %t.o %s
 
 ; Legacy PM
 ; FIXME: Fix machine verifier issues and remove -verify-machineinstrs=0. PR39436.
@@ -20,8 +20,8 @@
 ; RUN:   -r=%t.o,_ZN1B1fEi, \
 ; RUN:   -r=%t.o,_ZN1C1fEi, \
 ; RUN:   -r=%t.o,_ZTV1B,px \
-; RUN:   -r=%t.o,_ZTV1C,px 2>&1 | FileCheck %s --check-prefix=REMARK
-; RUN: llvm-dis %t3.1.4.opt.bc -o - | FileCheck %s --check-prefix=CHECK-IR
+; RUN:   -r=%t.o,_ZTV1C,px 2>&1 | FileCheck %s --check-prefix=REMARK -dump-input=always
+; RUN: llvm-dis %t3.1.4.opt.bc -o - | FileCheck %s --check-prefix=CHECK-IR -dump-input=always
 
 ; New PM
 ; FIXME: Fix machine verifier issues and remove -verify-machineinstrs=0. PR39436.
@@ -39,11 +39,32 @@
 ; RUN:   -r=%t.o,_ZN1B1fEi, \
 ; RUN:   -r=%t.o,_ZN1C1fEi, \
 ; RUN:   -r=%t.o,_ZTV1B,px \
-; RUN:   -r=%t.o,_ZTV1C,px 2>&1 | FileCheck %s --check-prefix=REMARK
-; RUN: llvm-dis %t3.1.4.opt.bc -o - | FileCheck %s --check-prefix=CHECK-IR
+; RUN:   -r=%t.o,_ZTV1C,px 2>&1 | FileCheck %s --check-prefix=REMARK -dump-input=always
+; RUN: llvm-dis %t3.1.4.opt.bc -o - | FileCheck %s --check-prefix=CHECK-IR -dump-input=always
 
 ; REMARK: single-impl: devirtualized a call to _ZN1A1nEi
 
+; Next check that we emit an error when trying to LTO link this module
+; containing an llvm.type.checked.load (with a split LTO Unit) with one
+; that does not have a split LTO Unit.
+; RUN: opt -thinlto-bc -o %t2.o %S/Inputs/empty.ll
+; RUN: not llvm-lto2 run %t.o %t2.o -save-temps -pass-remarks=. \
+; RUN:   -verify-machineinstrs=0 -thinlto-threads=1 \
+; RUN:   -o %t3 \
+; RUN:   -r=%t.o,test,px \
+; RUN:   -r=%t.o,_ZN1A1nEi,p \
+; RUN:   -r=%t.o,_ZN1B1fEi,p \
+; RUN:   -r=%t.o,_ZN1C1fEi,p \
+; RUN:   -r=%t.o,empty,p \
+; RUN:   -r=%t.o,_ZTV1B, \
+; RUN:   -r=%t.o,_ZTV1C, \
+; RUN:   -r=%t.o,_ZN1A1nEi, \
+; RUN:   -r=%t.o,_ZN1B1fEi, \
+; RUN:   -r=%t.o,_ZN1C1fEi, \
+; RUN:   -r=%t.o,_ZTV1B,px \
+; RUN:   -r=%t.o,_ZTV1C,px 2>&1 | FileCheck %s --check-prefix=ERROR -dump-input=always
+; ERROR: LLVM ERROR: inconsistent LTO Unit splitting with llvm.type.test or llvm.type.checked.load
+
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-grtev4-linux-gnu"
 
diff --git a/test/ThinLTO/X86/cfi-distributed.ll b/test/ThinLTO/X86/cfi-distributed.ll
index 94f3f95c298a6abc35ac2f4d43ac0f89580a3536..5339228515429bec2bb69a123bacc84216682287 100644
--- a/test/ThinLTO/X86/cfi-distributed.ll
+++ b/test/ThinLTO/X86/cfi-distributed.ll
@@ -3,8 +3,8 @@
 ; Test to ensure that only referenced type ID records are emitted into
 ; each distributed index file.
 
-; RUN: opt -thinlto-bc -o %t1.o %s
-; RUN: opt -thinlto-bc -o %t2.o %p/Inputs/cfi-distributed.ll
+; RUN: opt -thinlto-bc -thinlto-split-lto-unit -o %t1.o %s
+; RUN: opt -thinlto-bc -thinlto-split-lto-unit -o %t2.o %p/Inputs/cfi-distributed.ll
 
 ; RUN: llvm-lto2 run -thinlto-distributed-indexes %t1.o %t2.o \
 ; RUN:   -o %t3 \
diff --git a/test/ThinLTO/X86/cfi-icall.ll b/test/ThinLTO/X86/cfi-icall.ll
index 1ab184f1f19988732912649532c708687f4acc83..42c26f15893105dfb41da5dfb617e924d715176b 100644
--- a/test/ThinLTO/X86/cfi-icall.ll
+++ b/test/ThinLTO/X86/cfi-icall.ll
@@ -1,4 +1,4 @@
-; RUN: opt -thinlto-bc %s -o %t1.bc
+; RUN: opt -thinlto-bc -thinlto-split-lto-unit %s -o %t1.bc
 ; RUN: llvm-lto2 run  -thinlto-distributed-indexes %t1.bc -o %t.out -save-temps \
 ; RUN:   -r %t1.bc,foo,plx \
 ; RUN:   -r %t1.bc,bar,x \
diff --git a/test/ThinLTO/X86/cfi.ll b/test/ThinLTO/X86/cfi.ll
index 9b1bde3b6158d415ba8d75d13e1ae39c4329e5c6..0edddb82be9ec650bee4f5dbe1c0bcc4e16d4bb9 100644
--- a/test/ThinLTO/X86/cfi.ll
+++ b/test/ThinLTO/X86/cfi.ll
@@ -2,7 +2,7 @@
 
 ; Test CFI through the thin link and backend.
 
-; RUN: opt -thinlto-bc -o %t.o %s
+; RUN: opt -thinlto-bc -thinlto-split-lto-unit -o %t.o %s
 
 ; Legacy PM
 ; RUN: llvm-lto2 run -save-temps %t.o \
diff --git a/test/ThinLTO/X86/deadstrip.ll b/test/ThinLTO/X86/deadstrip.ll
index 76dc09e8a9e87a2cbe7b8d1046ecf8125019451e..257cfafb7671b4886332268b69c6484e77785159 100644
--- a/test/ThinLTO/X86/deadstrip.ll
+++ b/test/ThinLTO/X86/deadstrip.ll
@@ -20,10 +20,14 @@
 ; RUN:   -r %t1.bc,_live_available_externally_func,l \
 ; RUN:   -r %t1.bc,_live_linkonce_odr_func,l \
 ; RUN:   -r %t1.bc,_live_weak_odr_func,l \
+; RUN:   -r %t1.bc,_linkonceodralias,pl \
+; RUN:   -r %t1.bc,_linkonceodrfuncwithalias,l \
+; RUN:   -r %t1.bc,_linkonceodrfuncwithalias_caller,pl \
 ; RUN:   -r %t2.bc,_baz,pl \
 ; RUN:   -r %t2.bc,_boo,pl \
 ; RUN:   -r %t2.bc,_dead_func,l \
 ; RUN:   -r %t2.bc,_another_dead_func,pl \
+; RUN:   -r %t2.bc,_linkonceodrfuncwithalias,pl \
 ; RUN:   -thinlto-threads=1 \
 ; RUN:	 -debug-only=function-import 2>&1 | FileCheck %s --check-prefix=DEBUG --check-prefix=STATS
 ; RUN: llvm-dis < %t.out.1.3.import.bc | FileCheck %s --check-prefix=LTO2
@@ -71,6 +75,11 @@
 ; llvm.global_ctors
 ; CHECK2: define void @boo()
 ; LTO2-CHECK2: define dso_local void @boo()
+
+; Make sure we keep @linkonceodrfuncwithalias in Input/deadstrip.ll alive as it
+; is reachable from @main.
+; LTO2-CHECK2: define weak_odr dso_local void @linkonceodrfuncwithalias() {
+
 ; We should have eventually removed @baz since it was internalized and unused
 ; CHECK2-NM-NOT: _baz
 
@@ -105,10 +114,14 @@
 ; RUN:   -r %t1.bc,_live_available_externally_func,l \
 ; RUN:   -r %t1.bc,_live_linkonce_odr_func,l \
 ; RUN:   -r %t1.bc,_live_weak_odr_func,l \
+; RUN:   -r %t1.bc,_linkonceodralias,pl \
+; RUN:   -r %t1.bc,_linkonceodrfuncwithalias,l \
+; RUN:   -r %t1.bc,_linkonceodrfuncwithalias_caller,pl \
 ; RUN:   -r %t3.bc,_baz,pl \
 ; RUN:   -r %t3.bc,_boo,pl \
 ; RUN:   -r %t3.bc,_dead_func,l \
-; RUN:   -r %t3.bc,_another_dead_func,pl
+; RUN:   -r %t3.bc,_another_dead_func,pl \
+; RUN:   -r %t3.bc,_linkonceodrfuncwithalias,pl
 ; RUN: llvm-dis < %t4.out.1.3.import.bc | FileCheck %s --check-prefix=CHECK-NOTDEAD
 ; RUN: llvm-nm %t4.out.0 | FileCheck %s --check-prefix=CHECK-NM-NOTDEAD
 
@@ -163,11 +176,31 @@ define available_externally void @live_available_externally_func() {
     ret void
 }
 
+; This alias will set its base object in this file (linkonceodrfuncwithalias)
+; alive.
+; We want to make sure the @linkonceodrfuncwithalias copy in Input/deadstrip.ll
+; is also scanned when computing reachability.
+@linkonceodralias = linkonce_odr alias void (), void ()* @linkonceodrfuncwithalias
+
+define linkonce_odr void @linkonceodrfuncwithalias() {
+entry:
+  ret void
+}
+
+define void @linkonceodrfuncwithalias_caller() {
+entry:
+  call void @linkonceodrfuncwithalias()
+  ret void
+}
+
+
 define void @main() {
     call void @bar()
     call void @bar_internal()
     call void @live_linkonce_odr_func()
     call void @live_weak_odr_func()
     call void @live_available_externally_func()
+    call void @linkonceodrfuncwithalias_caller()
+    call void @linkonceodralias()
     ret void
 }
diff --git a/test/ThinLTO/X86/devirt-after-icp.ll b/test/ThinLTO/X86/devirt-after-icp.ll
index 987221787e2f8a4a7aa22c555c3af66b43cdb0c9..fd5dcb728aa7333d25ee3d1c405a532447e791f0 100644
--- a/test/ThinLTO/X86/devirt-after-icp.ll
+++ b/test/ThinLTO/X86/devirt-after-icp.ll
@@ -42,7 +42,7 @@
 ; will use the same vtable pointer. Without a dominance check, we could
 ; incorrectly devirtualize a->foo() to B::foo();
 
-; RUN: opt -thinlto-bc -o %t.o %s
+; RUN: opt -thinlto-bc -thinlto-split-lto-unit -o %t.o %s
 
 ; Legacy PM
 ; FIXME: Fix machine verifier issues and remove -verify-machineinstrs=0. PR39436.
diff --git a/test/ThinLTO/X86/dot-dumper.ll b/test/ThinLTO/X86/dot-dumper.ll
index 7578212ec44489a249d22c22797ef6456876ba94..571e968c2b1a0b00cbc9399ea0824fdb3e405d2f 100644
--- a/test/ThinLTO/X86/dot-dumper.ll
+++ b/test/ThinLTO/X86/dot-dumper.ll
@@ -1,5 +1,5 @@
-; RUN: opt -module-summary %s -o %t1.bc
-; RUN: opt -module-summary %p/Inputs/dot-dumper.ll -o %t2.bc
+; RUN: opt -module-summary %s -o %t1.bc -module-summary-dot-file=%t1.dot
+; RUN: opt -module-summary %p/Inputs/dot-dumper.ll -o %t2.bc -module-summary-dot-file=%t2.dot
 ; RUN: llvm-lto2 run -save-temps %t1.bc %t2.bc -o %t3 \
 ; RUN:  -r=%t1.bc,main,px \
 ; RUN:  -r=%t1.bc,main_alias,p \
@@ -11,10 +11,33 @@
 ; RUN:  -r=%t2.bc,B,p
 
 ; Never assume specific order of clusters, nodes or edges
+; RUN: cat %t1.dot | FileCheck --check-prefix=STRUCTURE1 %s
+; RUN: cat %t1.dot | FileCheck --check-prefix=CLUSTER0 --check-prefix=PERMODULE0 %s
+; RUN: cat %t2.dot | FileCheck --check-prefix=STRUCTURE2 %s
+; RUN: cat %t2.dot | FileCheck --check-prefix=CLUSTER1 --check-prefix=PERMODULE1 %s
 ; RUN: cat %t3.index.dot | FileCheck --check-prefix=STRUCTURE %s
-; RUN: cat %t3.index.dot | FileCheck --check-prefix=CLUSTER0 %s
-; RUN: cat %t3.index.dot | FileCheck --check-prefix=CLUSTER1 %s
+; RUN: cat %t3.index.dot | FileCheck --check-prefix=CLUSTER0 --check-prefix=COMBINED0 %s
+; RUN: cat %t3.index.dot | FileCheck --check-prefix=CLUSTER1 --check-prefix=COMBINED1 %s
 
+; %t1 index
+; STRUCTURE1:        digraph Summary {
+; STRUCTURE1:          subgraph cluster_0
+; STRUCTURE1:          // Cross-module edges:
+; STRUCTURE1:          0 [label="@0"]; // defined externally
+; STRUCTURE1:          M0_{{[0-9]+}} -> 0 [style=dotted]; // alias
+; STRUCTURE1-DAG:      [[A:[0-9]+]] [label="A"]; // defined externally
+; STRUCTURE1-DAG:      [[FOO:[0-9]+]] [label="foo"]; // defined externally
+; STRUCTURE1-DAG:      M0_{{[0-9]+}} -> [[FOO]] // call
+; STRUCTURE1-DAG:      M0_{{[0-9]+}} -> [[A]] [{{.*}}]; // const-ref
+; STRUCTURE1-NEXT:   }
+
+; %t2 index
+; STRUCTURE2:        digraph Summary {
+; STRUCTURE2:          subgraph cluster_0
+; STRUCTURE2:          // Cross-module edges:
+; STRUCTURE2-NEXT:   }
+
+; Combined index
 ; STRUCTURE:        digraph Summary {
 ; STRUCTURE-DAG:      subgraph cluster_0
 ; STRUCTURE-DAG:      subgraph cluster_1
@@ -23,23 +46,27 @@
 ; STRUCTURE-DAG:      M0_{{[0-9]+}} -> M1_{{[0-9]+}} [{{.*}}]; // const-ref
 ; STRUCTURE-NEXT:   }
 
-; CLUSTER0:         // Module: {{.*}}1.bc
-; CLUSTER0-NEXT:    subgraph cluster_0 {
-; CLUSTER0-DAG:       M0_[[MAIN_ALIAS:[0-9]+]] [{{.*}}main_alias{{.*}}]; // alias, dead
-; CLUSTER0-DAG:       M0_[[MAIN:[0-9]+]] [{{.*}}main|extern{{.*}}]; // function
+; PERMODULE0:       // Module:
+; COMBINED0:	    // Module: {{.*}}1.bc
+; CLUSTER0-NEXT:    subgraph cluster_[[ID0:[0-1]]] {
+; CLUSTER0-DAG:       M[[ID0]]_[[MAIN_ALIAS:[0-9]+]] [{{.*}}main_alias{{.*}}]; // alias, dead
+; CLUSTER0-DAG:       M[[ID0]]_[[MAIN:[0-9]+]] [{{.*}}main|extern{{.*}}]; // function
 ; CLUSTER0-NEXT:      // Edges:
-; CLUSTER0-NEXT:      M0_[[MAIN_ALIAS]] -> M0_[[MAIN]] [{{.*}}]; // alias
+; COMBINED0-NEXT:     M[[ID0]]_[[MAIN_ALIAS]] -> M[[ID0]]_[[MAIN]] [{{.*}}]; // alias
 ; CLUSTER0-NEXT:    }
 
-; CLUSTER1:         // Module: {{.*}}2.bc
-; CLUSTER1-NEXT:    subgraph cluster_1 {
-; CLUSTER1-DAG:       M1_[[A:[0-9]+]] [{{.*}}A|extern{{.*}}]; // variable, immutable
-; CLUSTER1-DAG:       M1_[[FOO:[0-9]+]] [{{.*}}foo|extern{{.*}} ffl: 00001{{.*}}]; // function
-; CLUSTER1-DAG:       M1_[[B:[0-9]+]] [{{.*}}B|extern{{.*}}]; // variable, immutable
-; CLUSTER1-DAG:       M1_[[BAR:[0-9]+]] [{{.*}}bar|extern{{.*}}]; // function, dead
+; PERMODULE1:       // Module:
+; COMBINED1:	    // Module: {{.*}}2.bc
+; CLUSTER1-NEXT:    subgraph cluster_[[ID1:[0-1]]] {
+; CLUSTER1-DAG:       M[[ID1]]_[[A:[0-9]+]] [{{.*}}A|extern{{.*}}]; // variable
+; COMBINED1-SAME:	, immutable
+; CLUSTER1-DAG:       M[[ID1]]_[[FOO:[0-9]+]] [{{.*}}foo|extern{{.*}} ffl: 00001{{.*}}]; // function
+; CLUSTER1-DAG:       M[[ID1]]_[[B:[0-9]+]] [{{.*}}B|extern{{.*}}]; // variable
+; COMBINED1-SAME:	, immutable
+; CLUSTER1-DAG:       M[[ID1]]_[[BAR:[0-9]+]] [{{.*}}bar|extern{{.*}}]; // function, dead
 ; CLUSTER1-NEXT:      // Edges:
-; CLUSTER1-DAG:       M1_[[FOO]] -> M1_[[B]] [{{.*}}]; // const-ref
-; CLUSTER1-DAG:       M1_[[FOO]] -> M1_[[A]] [{{.*}}]; // const-ref
+; CLUSTER1-DAG:       M[[ID1]]_[[FOO]] -> M[[ID1]]_[[B]] [{{.*}}]; // const-ref
+; CLUSTER1-DAG:       M[[ID1]]_[[FOO]] -> M[[ID1]]_[[A]] [{{.*}}]; // const-ref
 ; CLUSTER1-DAG:     }
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
diff --git a/test/ThinLTO/X86/lazyload_metadata.ll b/test/ThinLTO/X86/lazyload_metadata.ll
index 4680e46245856545b3ce1590f8cad96685c75b0c..21ee726e148832b65911d7cb253e15b19e65a3e1 100644
--- a/test/ThinLTO/X86/lazyload_metadata.ll
+++ b/test/ThinLTO/X86/lazyload_metadata.ll
@@ -10,13 +10,13 @@
 ; RUN: llvm-lto -thinlto-action=import %t2.bc -thinlto-index=%t3.bc \
 ; RUN:          -o /dev/null -stats \
 ; RUN:  2>&1 | FileCheck %s -check-prefix=LAZY
-; LAZY: 55 bitcode-reader  - Number of Metadata records loaded
+; LAZY: 59 bitcode-reader  - Number of Metadata records loaded
 ; LAZY: 2 bitcode-reader  - Number of MDStrings loaded
 
 ; RUN: llvm-lto -thinlto-action=import %t2.bc -thinlto-index=%t3.bc \
 ; RUN:          -o /dev/null -disable-ondemand-mds-loading -stats \
 ; RUN:  2>&1 | FileCheck %s -check-prefix=NOTLAZY
-; NOTLAZY: 64 bitcode-reader  - Number of Metadata records loaded
+; NOTLAZY: 68 bitcode-reader  - Number of Metadata records loaded
 ; NOTLAZY: 7 bitcode-reader  - Number of MDStrings loaded
 
 
diff --git a/test/Transforms/ADCE/2016-09-06.ll b/test/Transforms/ADCE/2016-09-06.ll
index 82c333bb45e869b517f0c78089a8eb20e6bdf0f8..6a2d396ecdad98ebe1c6ef07f6be196868380994 100644
--- a/test/Transforms/ADCE/2016-09-06.ll
+++ b/test/Transforms/ADCE/2016-09-06.ll
@@ -52,4 +52,4 @@ attributes #0 = { nounwind uwtable "correctly-rounded-divide-sqrt-fp-math"="fals
 
 !llvm.ident = !{!0}
 
-!0 = !{!"clang version 4.0.0 (http://llvm.org/git/clang.git 5864a13abf4490e76ae2eb0896198e1305927df2)"}
+!0 = !{!"clang version 4.0.0"}
diff --git a/test/Transforms/AddDiscriminators/invoke.ll b/test/Transforms/AddDiscriminators/invoke.ll
new file mode 100644
index 0000000000000000000000000000000000000000..0a2f86926c6455259d6f86239576ade0882cad41
--- /dev/null
+++ b/test/Transforms/AddDiscriminators/invoke.ll
@@ -0,0 +1,134 @@
+; RUN: opt < %s -add-discriminators -S | FileCheck %s
+; ModuleID = 'invoke.bc'
+source_filename = "invoke.cpp"
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.14.0"
+
+; Function Attrs: ssp uwtable
+define void @_Z3foov() #0 personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) !dbg !8 {
+entry:
+  %exn.slot = alloca i8*
+  %ehselector.slot = alloca i32
+  ; CHECK: call void @_Z12bar_noexceptv({{.*}} !dbg ![[CALL1:[0-9]+]]
+  call void @_Z12bar_noexceptv() #4, !dbg !11
+  ; CHECK: call void @_Z12bar_noexceptv({{.*}} !dbg ![[CALL2:[0-9]+]]
+  call void @_Z12bar_noexceptv() #4, !dbg !13
+  invoke void @_Z3barv()
+  ; CHECK: unwind label {{.*}} !dbg ![[INVOKE:[0-9]+]]
+          to label %invoke.cont unwind label %lpad, !dbg !14
+
+invoke.cont:                                      ; preds = %entry
+  br label %try.cont, !dbg !15
+
+lpad:                                             ; preds = %entry
+  %0 = landingpad { i8*, i32 }
+          catch i8* null, !dbg !16
+  %1 = extractvalue { i8*, i32 } %0, 0, !dbg !16
+  store i8* %1, i8** %exn.slot, align 8, !dbg !16
+  %2 = extractvalue { i8*, i32 } %0, 1, !dbg !16
+  store i32 %2, i32* %ehselector.slot, align 4, !dbg !16
+  br label %catch, !dbg !16
+
+catch:                                            ; preds = %lpad
+  %exn = load i8*, i8** %exn.slot, align 8, !dbg !15
+  %3 = call i8* @__cxa_begin_catch(i8* %exn) #4, !dbg !15
+  invoke void @__cxa_rethrow() #5
+          to label %unreachable unwind label %lpad1, !dbg !17
+
+lpad1:                                            ; preds = %catch
+  %4 = landingpad { i8*, i32 }
+          cleanup, !dbg !19
+  %5 = extractvalue { i8*, i32 } %4, 0, !dbg !19
+  store i8* %5, i8** %exn.slot, align 8, !dbg !19
+  %6 = extractvalue { i8*, i32 } %4, 1, !dbg !19
+  store i32 %6, i32* %ehselector.slot, align 4, !dbg !19
+  invoke void @__cxa_end_catch()
+          to label %invoke.cont2 unwind label %terminate.lpad, !dbg !20
+
+invoke.cont2:                                     ; preds = %lpad1
+  br label %eh.resume, !dbg !20
+
+try.cont:                                         ; preds = %invoke.cont
+  ret void, !dbg !21
+
+eh.resume:                                        ; preds = %invoke.cont2
+  %exn3 = load i8*, i8** %exn.slot, align 8, !dbg !20
+  %sel = load i32, i32* %ehselector.slot, align 4, !dbg !20
+  %lpad.val = insertvalue { i8*, i32 } undef, i8* %exn3, 0, !dbg !20
+  %lpad.val4 = insertvalue { i8*, i32 } %lpad.val, i32 %sel, 1, !dbg !20
+  resume { i8*, i32 } %lpad.val4, !dbg !20
+
+terminate.lpad:                                   ; preds = %lpad1
+  %7 = landingpad { i8*, i32 }
+          catch i8* null, !dbg !20
+  %8 = extractvalue { i8*, i32 } %7, 0, !dbg !20
+  call void @__clang_call_terminate(i8* %8) #6, !dbg !20
+  unreachable, !dbg !20
+
+unreachable:                                      ; preds = %catch
+  unreachable
+}
+
+; Function Attrs: nounwind
+declare void @_Z12bar_noexceptv() #1
+
+declare void @_Z3barv() #2
+
+declare i32 @__gxx_personality_v0(...)
+
+declare i8* @__cxa_begin_catch(i8*)
+
+declare void @__cxa_rethrow()
+
+declare void @__cxa_end_catch()
+
+; Function Attrs: noinline noreturn nounwind
+define linkonce_odr hidden void @__clang_call_terminate(i8*) #3 {
+  %2 = call i8* @__cxa_begin_catch(i8* %0) #4
+  call void @_ZSt9terminatev() #6
+  unreachable
+}
+
+declare void @_ZSt9terminatev()
+
+attributes #0 = { ssp uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="penryn" "target-features"="+cx16,+fxsr,+mmx,+sahf,+sse,+sse2,+sse3,+sse4.1,+ssse3,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="penryn" "target-features"="+cx16,+fxsr,+mmx,+sahf,+sse,+sse2,+sse3,+sse4.1,+ssse3,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #2 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="penryn" "target-features"="+cx16,+fxsr,+mmx,+sahf,+sse,+sse2,+sse3,+sse4.1,+ssse3,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #3 = { noinline noreturn nounwind }
+attributes #4 = { nounwind }
+attributes #5 = { noreturn }
+attributes #6 = { noreturn nounwind }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4, !5, !6}
+!llvm.ident = !{!7}
+
+; CHECK: ![[CALL1]] = !DILocation(line: 7, column: 5, scope: ![[SCOPE1:[0-9]+]])
+; CHECK: ![[SCOPE1]] = distinct !DILexicalBlock(scope: !8, file: !1, line: 6, column: 7)
+; CHECK: ![[CALL2]] = !DILocation(line: 7, column: 21, scope: ![[SCOPE2:[0-9]+]])
+; CHECK: ![[SCOPE2]] = !DILexicalBlockFile(scope: ![[SCOPE1]], file: !1, discriminator: 2)
+; CHECK: ![[INVOKE]] = !DILocation(line: 7, column: 37, scope: ![[SCOPE3:[0-9]+]])
+; CHECK: ![[SCOPE3]] = !DILexicalBlockFile(scope: ![[SCOPE1]], file: !1, discriminator: 4)
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !1, producer: "clang version 8.0.0", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, nameTableKind: GNU)
+!1 = !DIFile(filename: "invoke.cpp", directory: "examples")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{i32 1, !"wchar_size", i32 4}
+!6 = !{i32 7, !"PIC Level", i32 2}
+!7 = !{!"clang version 8.0.0"}
+!8 = distinct !DISubprogram(name: "foo", linkageName: "_Z3foov", scope: !1, file: !1, line: 5, type: !9, scopeLine: 5, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !2)
+!9 = !DISubroutineType(types: !10)
+!10 = !{null}
+!11 = !DILocation(line: 7, column: 5, scope: !12)
+!12 = distinct !DILexicalBlock(scope: !8, file: !1, line: 6, column: 7)
+!13 = !DILocation(line: 7, column: 21, scope: !12)
+!14 = !DILocation(line: 7, column: 37, scope: !12)
+!15 = !DILocation(line: 8, column: 3, scope: !12)
+!16 = !DILocation(line: 12, column: 1, scope: !12)
+!17 = !DILocation(line: 10, column: 5, scope: !18)
+!18 = distinct !DILexicalBlock(scope: !8, file: !1, line: 9, column: 15)
+!19 = !DILocation(line: 12, column: 1, scope: !18)
+!20 = !DILocation(line: 11, column: 3, scope: !18)
+!21 = !DILocation(line: 12, column: 1, scope: !8)
diff --git a/test/Transforms/ArgumentPromotion/X86/attributes.ll b/test/Transforms/ArgumentPromotion/X86/attributes.ll
new file mode 100644
index 0000000000000000000000000000000000000000..82a264109874c8b4c932dbcdd3020779f983dd6a
--- /dev/null
+++ b/test/Transforms/ArgumentPromotion/X86/attributes.ll
@@ -0,0 +1,53 @@
+; RUN: opt -S -argpromotion < %s | FileCheck %s
+; RUN: opt -S -passes=argpromotion < %s | FileCheck %s
+; Test that we only promote arguments when the caller/callee have compatible
+; function attrubtes.
+
+target triple = "x86_64-unknown-linux-gnu"
+
+; CHECK-LABEL: @no_promote_avx2(<4 x i64>* %arg, <4 x i64>* readonly %arg1)
+define internal fastcc void @no_promote_avx2(<4 x i64>* %arg, <4 x i64>* readonly %arg1) #0 {
+bb:
+  %tmp = load <4 x i64>, <4 x i64>* %arg1
+  store <4 x i64> %tmp, <4 x i64>* %arg
+  ret void
+}
+
+define void @no_promote(<4 x i64>* %arg) #1 {
+bb:
+  %tmp = alloca <4 x i64>, align 32
+  %tmp2 = alloca <4 x i64>, align 32
+  %tmp3 = bitcast <4 x i64>* %tmp to i8*
+  call void @llvm.memset.p0i8.i64(i8* align 32 %tmp3, i8 0, i64 32, i1 false)
+  call fastcc void @no_promote_avx2(<4 x i64>* %tmp2, <4 x i64>* %tmp)
+  %tmp4 = load <4 x i64>, <4 x i64>* %tmp2, align 32
+  store <4 x i64> %tmp4, <4 x i64>* %arg, align 2
+  ret void
+}
+
+; CHECK-LABEL: @promote_avx2(<4 x i64>* %arg, <4 x i64> %
+define internal fastcc void @promote_avx2(<4 x i64>* %arg, <4 x i64>* readonly %arg1) #0 {
+bb:
+  %tmp = load <4 x i64>, <4 x i64>* %arg1
+  store <4 x i64> %tmp, <4 x i64>* %arg
+  ret void
+}
+
+define void @promote(<4 x i64>* %arg) #0 {
+bb:
+  %tmp = alloca <4 x i64>, align 32
+  %tmp2 = alloca <4 x i64>, align 32
+  %tmp3 = bitcast <4 x i64>* %tmp to i8*
+  call void @llvm.memset.p0i8.i64(i8* align 32 %tmp3, i8 0, i64 32, i1 false)
+  call fastcc void @promote_avx2(<4 x i64>* %tmp2, <4 x i64>* %tmp)
+  %tmp4 = load <4 x i64>, <4 x i64>* %tmp2, align 32
+  store <4 x i64> %tmp4, <4 x i64>* %arg, align 2
+  ret void
+}
+
+; Function Attrs: argmemonly nounwind
+declare void @llvm.memset.p0i8.i64(i8* nocapture writeonly, i8, i64, i1) #2
+
+attributes #0 = { inlinehint norecurse nounwind uwtable "target-features"="+avx2" }
+attributes #1 = { nounwind uwtable }
+attributes #2 = { argmemonly nounwind }
diff --git a/test/Transforms/ArgumentPromotion/profile.ll b/test/Transforms/ArgumentPromotion/profile.ll
index f667f9ea2c2ae3da0d98ffa52f0ae72622e2fd63..64d424cd2cc178ac4053936c290c64784bbb9bde 100644
--- a/test/Transforms/ArgumentPromotion/profile.ll
+++ b/test/Transforms/ArgumentPromotion/profile.ll
@@ -1,17 +1,18 @@
 ; RUN: opt -argpromotion -mem2reg -S < %s | FileCheck %s
 target datalayout = "E-p:64:64:64-a0:0:8-f32:32:32-f64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-v64:64:64-v128:128:128"
 
-; Checks if !prof metadata is corret in deadargelim.
+; Checks if !prof metadata is correct after argpromotion
 
 define void @caller() #0 {
   %x = alloca i32
   store i32 42, i32* %x
-  call void @promote_i32_ptr(i32* %x), !prof !0
-; CHECK: call void @promote_i32_ptr(i32 42), !prof ![[PROF:[0-9]]]
+  call void @promote_i32_ptr(i32* %x), !prof !6
+; CHECK: call void @promote_i32_ptr(i32 42), !prof ![[PROF:[0-9]+]]
   ret void
 }
 
-define internal void @promote_i32_ptr(i32* %xp) {
+; CHECK: define internal void @promote_i32_ptr(i32 %xp.val) !dbg ![[DBG:[0-9]+]] !prof ![[FUNCPROF:[0-9]+]]
+define internal void @promote_i32_ptr(i32* %xp) !dbg !7 !prof !12 {
   %x = load i32, i32* %xp
   call void @use_i32(i32 %x)
   ret void
@@ -19,5 +20,22 @@ define internal void @promote_i32_ptr(i32* %xp) {
 
 declare void @use_i32(i32)
 
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4, !5}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !1, producer: "clang version 9.0.0 (trunk 353186) (llvm/trunk 353190)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, nameTableKind: None)
+!1 = !DIFile(filename: "profile.cc", directory: ".")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{i32 1, !"wchar_size", i32 4}
 ; CHECK: ![[PROF]] = !{!"branch_weights", i32 30}
-!0 = !{!"branch_weights", i32 30}
+!6 = !{!"branch_weights", i32 30}
+; CHECK: ![[DBG]] = distinct !DISubprogram(name: "promote_i32_ptr"
+!7 = distinct !DISubprogram(name: "promote_i32_ptr", linkageName: "_ZL15promote_i32_ptrPi", scope: !1, file: !1, line: 2, type: !8, scopeLine: 2, flags: DIFlagPrototyped, spFlags: DISPFlagLocalToUnit | DISPFlagDefinition, unit: !0, retainedNodes: !2)
+!8 = !DISubroutineType(types: !9)
+!9 = !{null, !10}
+!10 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !11, size: 64)
+!11 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+; CHECK: ![[FUNCPROF]] = !{!"function_entry_count", i64 331}
+!12 = !{!"function_entry_count", i64 331}
diff --git a/test/Transforms/AtomicExpand/AArch64/atomicrmw-fp.ll b/test/Transforms/AtomicExpand/AArch64/atomicrmw-fp.ll
new file mode 100644
index 0000000000000000000000000000000000000000..d63f911a33f899a06aa6efbb885502229bc9a587
--- /dev/null
+++ b/test/Transforms/AtomicExpand/AArch64/atomicrmw-fp.ll
@@ -0,0 +1,47 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -mtriple=aarch64-linux-gnu -atomic-expand %s | FileCheck %s
+
+define float @test_atomicrmw_fadd_f32(float* %ptr, float %value) {
+; CHECK-LABEL: @test_atomicrmw_fadd_f32(
+; CHECK-NEXT:    [[TMP1:%.*]] = load float, float* [[PTR:%.*]], align 4
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; CHECK-NEXT:    [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast float* [[PTR]] to i32*
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast float [[NEW]] to i32
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast float [[LOADED]] to i32
+; CHECK-NEXT:    [[TMP5:%.*]] = cmpxchg i32* [[TMP2]], i32 [[TMP4]], i32 [[TMP3]] seq_cst seq_cst
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1
+; CHECK-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6]] = bitcast i32 [[NEWLOADED]] to float
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret float [[TMP6]]
+;
+  %res = atomicrmw fadd float* %ptr, float %value seq_cst
+  ret float %res
+}
+
+define float @test_atomicrmw_fsub_f32(float* %ptr, float %value) {
+; CHECK-LABEL: @test_atomicrmw_fsub_f32(
+; CHECK-NEXT:    [[TMP1:%.*]] = load float, float* [[PTR:%.*]], align 4
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; CHECK-NEXT:    [[NEW:%.*]] = fsub float [[LOADED]], [[VALUE:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast float* [[PTR]] to i32*
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast float [[NEW]] to i32
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast float [[LOADED]] to i32
+; CHECK-NEXT:    [[TMP5:%.*]] = cmpxchg i32* [[TMP2]], i32 [[TMP4]], i32 [[TMP3]] seq_cst seq_cst
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1
+; CHECK-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6]] = bitcast i32 [[NEWLOADED]] to float
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret float [[TMP6]]
+;
+  %res = atomicrmw fsub float* %ptr, float %value seq_cst
+  ret float %res
+}
+
diff --git a/test/Transforms/AtomicExpand/AArch64/expand-atomicrmw-xchg-fp.ll b/test/Transforms/AtomicExpand/AArch64/expand-atomicrmw-xchg-fp.ll
new file mode 100644
index 0000000000000000000000000000000000000000..d3d36eb4503915b6cf9bb98b192d851c4a81d7d6
--- /dev/null
+++ b/test/Transforms/AtomicExpand/AArch64/expand-atomicrmw-xchg-fp.ll
@@ -0,0 +1,57 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -mtriple=aarch64-- -atomic-expand %s | FileCheck %s
+
+define void @atomic_swap_f16(half* %ptr, half %val) nounwind {
+; CHECK-LABEL: @atomic_swap_f16(
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[TMP1:%.*]] = call i64 @llvm.aarch64.ldaxr.p0f16(half* [[PTR:%.*]])
+; CHECK-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i16
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i16 [[TMP2]] to half
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast half [[VAL:%.*]] to i16
+; CHECK-NEXT:    [[TMP5:%.*]] = zext i16 [[TMP4]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = call i32 @llvm.aarch64.stxr.p0f16(i64 [[TMP5]], half* [[PTR]])
+; CHECK-NEXT:    [[TRYAGAIN:%.*]] = icmp ne i32 [[TMP6]], 0
+; CHECK-NEXT:    br i1 [[TRYAGAIN]], label [[ATOMICRMW_START]], label [[ATOMICRMW_END:%.*]]
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+  %t1 = atomicrmw xchg half* %ptr, half %val acquire
+  ret void
+}
+
+define void @atomic_swap_f32(float* %ptr, float %val) nounwind {
+; CHECK-LABEL: @atomic_swap_f32(
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[TMP1:%.*]] = call i64 @llvm.aarch64.ldaxr.p0f32(float* [[PTR:%.*]])
+; CHECK-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i32
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i32 [[TMP2]] to float
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast float [[VAL:%.*]] to i32
+; CHECK-NEXT:    [[TMP5:%.*]] = zext i32 [[TMP4]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = call i32 @llvm.aarch64.stxr.p0f32(i64 [[TMP5]], float* [[PTR]])
+; CHECK-NEXT:    [[TRYAGAIN:%.*]] = icmp ne i32 [[TMP6]], 0
+; CHECK-NEXT:    br i1 [[TRYAGAIN]], label [[ATOMICRMW_START]], label [[ATOMICRMW_END:%.*]]
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+  %t1 = atomicrmw xchg float* %ptr, float %val acquire
+  ret void
+}
+
+define void @atomic_swap_f64(double* %ptr, double %val) nounwind {
+; CHECK-LABEL: @atomic_swap_f64(
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[TMP1:%.*]] = call i64 @llvm.aarch64.ldaxr.p0f64(double* [[PTR:%.*]])
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i64 [[TMP1]] to double
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast double [[VAL:%.*]] to i64
+; CHECK-NEXT:    [[TMP4:%.*]] = call i32 @llvm.aarch64.stxr.p0f64(i64 [[TMP3]], double* [[PTR]])
+; CHECK-NEXT:    [[TRYAGAIN:%.*]] = icmp ne i32 [[TMP4]], 0
+; CHECK-NEXT:    br i1 [[TRYAGAIN]], label [[ATOMICRMW_START]], label [[ATOMICRMW_END:%.*]]
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret void
+;
+  %t1 = atomicrmw xchg double* %ptr, double %val acquire
+  ret void
+}
diff --git a/test/Transforms/AtomicExpand/AArch64/lit.local.cfg b/test/Transforms/AtomicExpand/AArch64/lit.local.cfg
new file mode 100644
index 0000000000000000000000000000000000000000..cec29af5bbe478d6037c442d59b3d6e5cfee1dc5
--- /dev/null
+++ b/test/Transforms/AtomicExpand/AArch64/lit.local.cfg
@@ -0,0 +1,3 @@
+if not 'AArch64' in config.root.targets:
+    config.unsupported = True
+
diff --git a/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-rmw-fadd.ll b/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-rmw-fadd.ll
new file mode 100644
index 0000000000000000000000000000000000000000..f96cd0b29bffb186f13fcbb683c3a4e7f0204895
--- /dev/null
+++ b/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-rmw-fadd.ll
@@ -0,0 +1,264 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=hawaii -atomic-expand %s | FileCheck -check-prefix=CI %s
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -atomic-expand %s | FileCheck -check-prefix=GFX9 %s
+
+define float @test_atomicrmw_fadd_f32_flat(float* %ptr, float %value) {
+; CI-LABEL: @test_atomicrmw_fadd_f32_flat(
+; CI-NEXT:    [[TMP1:%.*]] = load float, float* [[PTR:%.*]], align 4
+; CI-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; CI:       atomicrmw.start:
+; CI-NEXT:    [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; CI-NEXT:    [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]]
+; CI-NEXT:    [[TMP2:%.*]] = bitcast float* [[PTR]] to i32*
+; CI-NEXT:    [[TMP3:%.*]] = bitcast float [[NEW]] to i32
+; CI-NEXT:    [[TMP4:%.*]] = bitcast float [[LOADED]] to i32
+; CI-NEXT:    [[TMP5:%.*]] = cmpxchg i32* [[TMP2]], i32 [[TMP4]], i32 [[TMP3]] seq_cst seq_cst
+; CI-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1
+; CI-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP5]], 0
+; CI-NEXT:    [[TMP6]] = bitcast i32 [[NEWLOADED]] to float
+; CI-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; CI:       atomicrmw.end:
+; CI-NEXT:    ret float [[TMP6]]
+;
+; GFX9-LABEL: @test_atomicrmw_fadd_f32_flat(
+; GFX9-NEXT:    [[TMP1:%.*]] = load float, float* [[PTR:%.*]], align 4
+; GFX9-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX9:       atomicrmw.start:
+; GFX9-NEXT:    [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GFX9-NEXT:    [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]]
+; GFX9-NEXT:    [[TMP2:%.*]] = bitcast float* [[PTR]] to i32*
+; GFX9-NEXT:    [[TMP3:%.*]] = bitcast float [[NEW]] to i32
+; GFX9-NEXT:    [[TMP4:%.*]] = bitcast float [[LOADED]] to i32
+; GFX9-NEXT:    [[TMP5:%.*]] = cmpxchg i32* [[TMP2]], i32 [[TMP4]], i32 [[TMP3]] seq_cst seq_cst
+; GFX9-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1
+; GFX9-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP5]], 0
+; GFX9-NEXT:    [[TMP6]] = bitcast i32 [[NEWLOADED]] to float
+; GFX9-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX9:       atomicrmw.end:
+; GFX9-NEXT:    ret float [[TMP6]]
+;
+  %res = atomicrmw fadd float* %ptr, float %value seq_cst
+  ret float %res
+}
+
+define float @test_atomicrmw_fadd_f32_global(float addrspace(1)* %ptr, float %value) {
+; CI-LABEL: @test_atomicrmw_fadd_f32_global(
+; CI-NEXT:    [[TMP1:%.*]] = load float, float addrspace(1)* [[PTR:%.*]], align 4
+; CI-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; CI:       atomicrmw.start:
+; CI-NEXT:    [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; CI-NEXT:    [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]]
+; CI-NEXT:    [[TMP2:%.*]] = bitcast float addrspace(1)* [[PTR]] to i32 addrspace(1)*
+; CI-NEXT:    [[TMP3:%.*]] = bitcast float [[NEW]] to i32
+; CI-NEXT:    [[TMP4:%.*]] = bitcast float [[LOADED]] to i32
+; CI-NEXT:    [[TMP5:%.*]] = cmpxchg i32 addrspace(1)* [[TMP2]], i32 [[TMP4]], i32 [[TMP3]] seq_cst seq_cst
+; CI-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1
+; CI-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP5]], 0
+; CI-NEXT:    [[TMP6]] = bitcast i32 [[NEWLOADED]] to float
+; CI-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; CI:       atomicrmw.end:
+; CI-NEXT:    ret float [[TMP6]]
+;
+; GFX9-LABEL: @test_atomicrmw_fadd_f32_global(
+; GFX9-NEXT:    [[TMP1:%.*]] = load float, float addrspace(1)* [[PTR:%.*]], align 4
+; GFX9-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX9:       atomicrmw.start:
+; GFX9-NEXT:    [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GFX9-NEXT:    [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]]
+; GFX9-NEXT:    [[TMP2:%.*]] = bitcast float addrspace(1)* [[PTR]] to i32 addrspace(1)*
+; GFX9-NEXT:    [[TMP3:%.*]] = bitcast float [[NEW]] to i32
+; GFX9-NEXT:    [[TMP4:%.*]] = bitcast float [[LOADED]] to i32
+; GFX9-NEXT:    [[TMP5:%.*]] = cmpxchg i32 addrspace(1)* [[TMP2]], i32 [[TMP4]], i32 [[TMP3]] seq_cst seq_cst
+; GFX9-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1
+; GFX9-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP5]], 0
+; GFX9-NEXT:    [[TMP6]] = bitcast i32 [[NEWLOADED]] to float
+; GFX9-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX9:       atomicrmw.end:
+; GFX9-NEXT:    ret float [[TMP6]]
+;
+  %res = atomicrmw fadd float addrspace(1)* %ptr, float %value seq_cst
+  ret float %res
+}
+
+define float @test_atomicrmw_fadd_f32_local(float addrspace(3)* %ptr, float %value) {
+; CI-LABEL: @test_atomicrmw_fadd_f32_local(
+; CI-NEXT:    [[TMP1:%.*]] = load float, float addrspace(3)* [[PTR:%.*]], align 4
+; CI-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; CI:       atomicrmw.start:
+; CI-NEXT:    [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; CI-NEXT:    [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]]
+; CI-NEXT:    [[TMP2:%.*]] = bitcast float addrspace(3)* [[PTR]] to i32 addrspace(3)*
+; CI-NEXT:    [[TMP3:%.*]] = bitcast float [[NEW]] to i32
+; CI-NEXT:    [[TMP4:%.*]] = bitcast float [[LOADED]] to i32
+; CI-NEXT:    [[TMP5:%.*]] = cmpxchg i32 addrspace(3)* [[TMP2]], i32 [[TMP4]], i32 [[TMP3]] seq_cst seq_cst
+; CI-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1
+; CI-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP5]], 0
+; CI-NEXT:    [[TMP6]] = bitcast i32 [[NEWLOADED]] to float
+; CI-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; CI:       atomicrmw.end:
+; CI-NEXT:    ret float [[TMP6]]
+;
+; GFX9-LABEL: @test_atomicrmw_fadd_f32_local(
+; GFX9-NEXT:    [[RES:%.*]] = atomicrmw fadd float addrspace(3)* [[PTR:%.*]], float [[VALUE:%.*]] seq_cst
+; GFX9-NEXT:    ret float [[RES]]
+;
+  %res = atomicrmw fadd float addrspace(3)* %ptr, float %value seq_cst
+  ret float %res
+}
+
+define half @test_atomicrmw_fadd_f16_flat(half* %ptr, half %value) {
+; CI-LABEL: @test_atomicrmw_fadd_f16_flat(
+; CI-NEXT:    [[RES:%.*]] = atomicrmw fadd half* [[PTR:%.*]], half [[VALUE:%.*]] seq_cst
+; CI-NEXT:    ret half [[RES]]
+;
+; GFX9-LABEL: @test_atomicrmw_fadd_f16_flat(
+; GFX9-NEXT:    [[RES:%.*]] = atomicrmw fadd half* [[PTR:%.*]], half [[VALUE:%.*]] seq_cst
+; GFX9-NEXT:    ret half [[RES]]
+;
+  %res = atomicrmw fadd half* %ptr, half %value seq_cst
+  ret half %res
+}
+
+define half @test_atomicrmw_fadd_f16_global(half addrspace(1)* %ptr, half %value) {
+; CI-LABEL: @test_atomicrmw_fadd_f16_global(
+; CI-NEXT:    [[RES:%.*]] = atomicrmw fadd half addrspace(1)* [[PTR:%.*]], half [[VALUE:%.*]] seq_cst
+; CI-NEXT:    ret half [[RES]]
+;
+; GFX9-LABEL: @test_atomicrmw_fadd_f16_global(
+; GFX9-NEXT:    [[RES:%.*]] = atomicrmw fadd half addrspace(1)* [[PTR:%.*]], half [[VALUE:%.*]] seq_cst
+; GFX9-NEXT:    ret half [[RES]]
+;
+  %res = atomicrmw fadd half addrspace(1)* %ptr, half %value seq_cst
+  ret half %res
+}
+
+define half @test_atomicrmw_fadd_f16_local(half addrspace(3)* %ptr, half %value) {
+; CI-LABEL: @test_atomicrmw_fadd_f16_local(
+; CI-NEXT:    [[RES:%.*]] = atomicrmw fadd half addrspace(3)* [[PTR:%.*]], half [[VALUE:%.*]] seq_cst
+; CI-NEXT:    ret half [[RES]]
+;
+; GFX9-LABEL: @test_atomicrmw_fadd_f16_local(
+; GFX9-NEXT:    [[RES:%.*]] = atomicrmw fadd half addrspace(3)* [[PTR:%.*]], half [[VALUE:%.*]] seq_cst
+; GFX9-NEXT:    ret half [[RES]]
+;
+  %res = atomicrmw fadd half addrspace(3)* %ptr, half %value seq_cst
+  ret half %res
+}
+
+define double @test_atomicrmw_fadd_f64_flat(double* %ptr, double %value) {
+; CI-LABEL: @test_atomicrmw_fadd_f64_flat(
+; CI-NEXT:    [[TMP1:%.*]] = load double, double* [[PTR:%.*]], align 8
+; CI-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; CI:       atomicrmw.start:
+; CI-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; CI-NEXT:    [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE:%.*]]
+; CI-NEXT:    [[TMP2:%.*]] = bitcast double* [[PTR]] to i64*
+; CI-NEXT:    [[TMP3:%.*]] = bitcast double [[NEW]] to i64
+; CI-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
+; CI-NEXT:    [[TMP5:%.*]] = cmpxchg i64* [[TMP2]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst
+; CI-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; CI-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; CI-NEXT:    [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
+; CI-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; CI:       atomicrmw.end:
+; CI-NEXT:    ret double [[TMP6]]
+;
+; GFX9-LABEL: @test_atomicrmw_fadd_f64_flat(
+; GFX9-NEXT:    [[TMP1:%.*]] = load double, double* [[PTR:%.*]], align 8
+; GFX9-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX9:       atomicrmw.start:
+; GFX9-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GFX9-NEXT:    [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE:%.*]]
+; GFX9-NEXT:    [[TMP2:%.*]] = bitcast double* [[PTR]] to i64*
+; GFX9-NEXT:    [[TMP3:%.*]] = bitcast double [[NEW]] to i64
+; GFX9-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
+; GFX9-NEXT:    [[TMP5:%.*]] = cmpxchg i64* [[TMP2]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst
+; GFX9-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; GFX9-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; GFX9-NEXT:    [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
+; GFX9-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX9:       atomicrmw.end:
+; GFX9-NEXT:    ret double [[TMP6]]
+;
+  %res = atomicrmw fadd double* %ptr, double %value seq_cst
+  ret double %res
+}
+
+define double @test_atomicrmw_fadd_f64_global(double addrspace(1)* %ptr, double %value) {
+; CI-LABEL: @test_atomicrmw_fadd_f64_global(
+; CI-NEXT:    [[TMP1:%.*]] = load double, double addrspace(1)* [[PTR:%.*]], align 8
+; CI-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; CI:       atomicrmw.start:
+; CI-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; CI-NEXT:    [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE:%.*]]
+; CI-NEXT:    [[TMP2:%.*]] = bitcast double addrspace(1)* [[PTR]] to i64 addrspace(1)*
+; CI-NEXT:    [[TMP3:%.*]] = bitcast double [[NEW]] to i64
+; CI-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
+; CI-NEXT:    [[TMP5:%.*]] = cmpxchg i64 addrspace(1)* [[TMP2]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst
+; CI-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; CI-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; CI-NEXT:    [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
+; CI-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; CI:       atomicrmw.end:
+; CI-NEXT:    ret double [[TMP6]]
+;
+; GFX9-LABEL: @test_atomicrmw_fadd_f64_global(
+; GFX9-NEXT:    [[TMP1:%.*]] = load double, double addrspace(1)* [[PTR:%.*]], align 8
+; GFX9-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX9:       atomicrmw.start:
+; GFX9-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GFX9-NEXT:    [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE:%.*]]
+; GFX9-NEXT:    [[TMP2:%.*]] = bitcast double addrspace(1)* [[PTR]] to i64 addrspace(1)*
+; GFX9-NEXT:    [[TMP3:%.*]] = bitcast double [[NEW]] to i64
+; GFX9-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
+; GFX9-NEXT:    [[TMP5:%.*]] = cmpxchg i64 addrspace(1)* [[TMP2]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst
+; GFX9-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; GFX9-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; GFX9-NEXT:    [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
+; GFX9-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX9:       atomicrmw.end:
+; GFX9-NEXT:    ret double [[TMP6]]
+;
+  %res = atomicrmw fadd double addrspace(1)* %ptr, double %value seq_cst
+  ret double %res
+}
+
+define double @test_atomicrmw_fadd_f64_local(double addrspace(3)* %ptr, double %value) {
+; CI-LABEL: @test_atomicrmw_fadd_f64_local(
+; CI-NEXT:    [[TMP1:%.*]] = load double, double addrspace(3)* [[PTR:%.*]], align 8
+; CI-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; CI:       atomicrmw.start:
+; CI-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; CI-NEXT:    [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE:%.*]]
+; CI-NEXT:    [[TMP2:%.*]] = bitcast double addrspace(3)* [[PTR]] to i64 addrspace(3)*
+; CI-NEXT:    [[TMP3:%.*]] = bitcast double [[NEW]] to i64
+; CI-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
+; CI-NEXT:    [[TMP5:%.*]] = cmpxchg i64 addrspace(3)* [[TMP2]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst
+; CI-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; CI-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; CI-NEXT:    [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
+; CI-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; CI:       atomicrmw.end:
+; CI-NEXT:    ret double [[TMP6]]
+;
+; GFX9-LABEL: @test_atomicrmw_fadd_f64_local(
+; GFX9-NEXT:    [[TMP1:%.*]] = load double, double addrspace(3)* [[PTR:%.*]], align 8
+; GFX9-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GFX9:       atomicrmw.start:
+; GFX9-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GFX9-NEXT:    [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE:%.*]]
+; GFX9-NEXT:    [[TMP2:%.*]] = bitcast double addrspace(3)* [[PTR]] to i64 addrspace(3)*
+; GFX9-NEXT:    [[TMP3:%.*]] = bitcast double [[NEW]] to i64
+; GFX9-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
+; GFX9-NEXT:    [[TMP5:%.*]] = cmpxchg i64 addrspace(3)* [[TMP2]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst
+; GFX9-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; GFX9-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; GFX9-NEXT:    [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
+; GFX9-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GFX9:       atomicrmw.end:
+; GFX9-NEXT:    ret double [[TMP6]]
+;
+  %res = atomicrmw fadd double addrspace(3)* %ptr, double %value seq_cst
+  ret double %res
+}
+
diff --git a/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-rmw-fsub.ll b/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-rmw-fsub.ll
new file mode 100644
index 0000000000000000000000000000000000000000..bfbdf9b6e8634436a933bb25bbe892c1de72ec31
--- /dev/null
+++ b/test/Transforms/AtomicExpand/AMDGPU/expand-atomic-rmw-fsub.ll
@@ -0,0 +1,201 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=hawaii -atomic-expand %s | FileCheck -check-prefix=GCN %s
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -atomic-expand %s | FileCheck -check-prefix=GCN %s
+
+define float @test_atomicrmw_fadd_f32_flat(float* %ptr, float %value) {
+; GCN-LABEL: @test_atomicrmw_fadd_f32_flat(
+; GCN-NEXT:    [[TMP1:%.*]] = load float, float* [[PTR:%.*]], align 4
+; GCN-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GCN:       atomicrmw.start:
+; GCN-NEXT:    [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GCN-NEXT:    [[NEW:%.*]] = fsub float [[LOADED]], [[VALUE:%.*]]
+; GCN-NEXT:    [[TMP2:%.*]] = bitcast float* [[PTR]] to i32*
+; GCN-NEXT:    [[TMP3:%.*]] = bitcast float [[NEW]] to i32
+; GCN-NEXT:    [[TMP4:%.*]] = bitcast float [[LOADED]] to i32
+; GCN-NEXT:    [[TMP5:%.*]] = cmpxchg i32* [[TMP2]], i32 [[TMP4]], i32 [[TMP3]] seq_cst seq_cst
+; GCN-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1
+; GCN-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP5]], 0
+; GCN-NEXT:    [[TMP6]] = bitcast i32 [[NEWLOADED]] to float
+; GCN-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GCN:       atomicrmw.end:
+; GCN-NEXT:    ret float [[TMP6]]
+;
+  %res = atomicrmw fsub float* %ptr, float %value seq_cst
+  ret float %res
+}
+
+define float @test_atomicrmw_fsub_f32_global(float addrspace(1)* %ptr, float %value) {
+; GCN-LABEL: @test_atomicrmw_fsub_f32_global(
+; GCN-NEXT:    [[TMP1:%.*]] = load float, float addrspace(1)* [[PTR:%.*]], align 4
+; GCN-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GCN:       atomicrmw.start:
+; GCN-NEXT:    [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GCN-NEXT:    [[NEW:%.*]] = fsub float [[LOADED]], [[VALUE:%.*]]
+; GCN-NEXT:    [[TMP2:%.*]] = bitcast float addrspace(1)* [[PTR]] to i32 addrspace(1)*
+; GCN-NEXT:    [[TMP3:%.*]] = bitcast float [[NEW]] to i32
+; GCN-NEXT:    [[TMP4:%.*]] = bitcast float [[LOADED]] to i32
+; GCN-NEXT:    [[TMP5:%.*]] = cmpxchg i32 addrspace(1)* [[TMP2]], i32 [[TMP4]], i32 [[TMP3]] seq_cst seq_cst
+; GCN-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1
+; GCN-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP5]], 0
+; GCN-NEXT:    [[TMP6]] = bitcast i32 [[NEWLOADED]] to float
+; GCN-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GCN:       atomicrmw.end:
+; GCN-NEXT:    ret float [[TMP6]]
+;
+  %res = atomicrmw fsub float addrspace(1)* %ptr, float %value seq_cst
+  ret float %res
+}
+
+define float @test_atomicrmw_fsub_f32_local(float addrspace(3)* %ptr, float %value) {
+; GCN-LABEL: @test_atomicrmw_fsub_f32_local(
+; GCN-NEXT:    [[TMP1:%.*]] = load float, float addrspace(3)* [[PTR:%.*]], align 4
+; GCN-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GCN:       atomicrmw.start:
+; GCN-NEXT:    [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GCN-NEXT:    [[NEW:%.*]] = fsub float [[LOADED]], [[VALUE:%.*]]
+; GCN-NEXT:    [[TMP2:%.*]] = bitcast float addrspace(3)* [[PTR]] to i32 addrspace(3)*
+; GCN-NEXT:    [[TMP3:%.*]] = bitcast float [[NEW]] to i32
+; GCN-NEXT:    [[TMP4:%.*]] = bitcast float [[LOADED]] to i32
+; GCN-NEXT:    [[TMP5:%.*]] = cmpxchg i32 addrspace(3)* [[TMP2]], i32 [[TMP4]], i32 [[TMP3]] seq_cst seq_cst
+; GCN-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1
+; GCN-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP5]], 0
+; GCN-NEXT:    [[TMP6]] = bitcast i32 [[NEWLOADED]] to float
+; GCN-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GCN:       atomicrmw.end:
+; GCN-NEXT:    ret float [[TMP6]]
+;
+  %res = atomicrmw fsub float addrspace(3)* %ptr, float %value seq_cst
+  ret float %res
+}
+
+define half @test_atomicrmw_fsub_f16_flat(half* %ptr, half %value) {
+; GCN-LABEL: @test_atomicrmw_fsub_f16_flat(
+; GCN-NEXT:    [[TMP1:%.*]] = load half, half* [[PTR:%.*]], align 2
+; GCN-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GCN:       atomicrmw.start:
+; GCN-NEXT:    [[LOADED:%.*]] = phi half [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GCN-NEXT:    [[NEW:%.*]] = fsub half [[LOADED]], [[VALUE:%.*]]
+; GCN-NEXT:    [[TMP2:%.*]] = bitcast half* [[PTR]] to i16*
+; GCN-NEXT:    [[TMP3:%.*]] = bitcast half [[NEW]] to i16
+; GCN-NEXT:    [[TMP4:%.*]] = bitcast half [[LOADED]] to i16
+; GCN-NEXT:    [[TMP5:%.*]] = cmpxchg i16* [[TMP2]], i16 [[TMP4]], i16 [[TMP3]] seq_cst seq_cst
+; GCN-NEXT:    [[SUCCESS:%.*]] = extractvalue { i16, i1 } [[TMP5]], 1
+; GCN-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i16, i1 } [[TMP5]], 0
+; GCN-NEXT:    [[TMP6]] = bitcast i16 [[NEWLOADED]] to half
+; GCN-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GCN:       atomicrmw.end:
+; GCN-NEXT:    ret half [[TMP6]]
+;
+  %res = atomicrmw fsub half* %ptr, half %value seq_cst
+  ret half %res
+}
+
+define half @test_atomicrmw_fsub_f16_global(half addrspace(1)* %ptr, half %value) {
+; GCN-LABEL: @test_atomicrmw_fsub_f16_global(
+; GCN-NEXT:    [[TMP1:%.*]] = load half, half addrspace(1)* [[PTR:%.*]], align 2
+; GCN-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GCN:       atomicrmw.start:
+; GCN-NEXT:    [[LOADED:%.*]] = phi half [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GCN-NEXT:    [[NEW:%.*]] = fsub half [[LOADED]], [[VALUE:%.*]]
+; GCN-NEXT:    [[TMP2:%.*]] = bitcast half addrspace(1)* [[PTR]] to i16 addrspace(1)*
+; GCN-NEXT:    [[TMP3:%.*]] = bitcast half [[NEW]] to i16
+; GCN-NEXT:    [[TMP4:%.*]] = bitcast half [[LOADED]] to i16
+; GCN-NEXT:    [[TMP5:%.*]] = cmpxchg i16 addrspace(1)* [[TMP2]], i16 [[TMP4]], i16 [[TMP3]] seq_cst seq_cst
+; GCN-NEXT:    [[SUCCESS:%.*]] = extractvalue { i16, i1 } [[TMP5]], 1
+; GCN-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i16, i1 } [[TMP5]], 0
+; GCN-NEXT:    [[TMP6]] = bitcast i16 [[NEWLOADED]] to half
+; GCN-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GCN:       atomicrmw.end:
+; GCN-NEXT:    ret half [[TMP6]]
+;
+  %res = atomicrmw fsub half addrspace(1)* %ptr, half %value seq_cst
+  ret half %res
+}
+
+define half @test_atomicrmw_fsub_f16_local(half addrspace(3)* %ptr, half %value) {
+; GCN-LABEL: @test_atomicrmw_fsub_f16_local(
+; GCN-NEXT:    [[TMP1:%.*]] = load half, half addrspace(3)* [[PTR:%.*]], align 2
+; GCN-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GCN:       atomicrmw.start:
+; GCN-NEXT:    [[LOADED:%.*]] = phi half [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GCN-NEXT:    [[NEW:%.*]] = fsub half [[LOADED]], [[VALUE:%.*]]
+; GCN-NEXT:    [[TMP2:%.*]] = bitcast half addrspace(3)* [[PTR]] to i16 addrspace(3)*
+; GCN-NEXT:    [[TMP3:%.*]] = bitcast half [[NEW]] to i16
+; GCN-NEXT:    [[TMP4:%.*]] = bitcast half [[LOADED]] to i16
+; GCN-NEXT:    [[TMP5:%.*]] = cmpxchg i16 addrspace(3)* [[TMP2]], i16 [[TMP4]], i16 [[TMP3]] seq_cst seq_cst
+; GCN-NEXT:    [[SUCCESS:%.*]] = extractvalue { i16, i1 } [[TMP5]], 1
+; GCN-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i16, i1 } [[TMP5]], 0
+; GCN-NEXT:    [[TMP6]] = bitcast i16 [[NEWLOADED]] to half
+; GCN-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GCN:       atomicrmw.end:
+; GCN-NEXT:    ret half [[TMP6]]
+;
+  %res = atomicrmw fsub half addrspace(3)* %ptr, half %value seq_cst
+  ret half %res
+}
+
+define double @test_atomicrmw_fsub_f64_flat(double* %ptr, double %value) {
+; GCN-LABEL: @test_atomicrmw_fsub_f64_flat(
+; GCN-NEXT:    [[TMP1:%.*]] = load double, double* [[PTR:%.*]], align 8
+; GCN-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GCN:       atomicrmw.start:
+; GCN-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GCN-NEXT:    [[NEW:%.*]] = fsub double [[LOADED]], [[VALUE:%.*]]
+; GCN-NEXT:    [[TMP2:%.*]] = bitcast double* [[PTR]] to i64*
+; GCN-NEXT:    [[TMP3:%.*]] = bitcast double [[NEW]] to i64
+; GCN-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
+; GCN-NEXT:    [[TMP5:%.*]] = cmpxchg i64* [[TMP2]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst
+; GCN-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; GCN-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; GCN-NEXT:    [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
+; GCN-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GCN:       atomicrmw.end:
+; GCN-NEXT:    ret double [[TMP6]]
+;
+  %res = atomicrmw fsub double* %ptr, double %value seq_cst
+  ret double %res
+}
+
+define double @test_atomicrmw_fsub_f64_global(double addrspace(1)* %ptr, double %value) {
+; GCN-LABEL: @test_atomicrmw_fsub_f64_global(
+; GCN-NEXT:    [[TMP1:%.*]] = load double, double addrspace(1)* [[PTR:%.*]], align 8
+; GCN-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GCN:       atomicrmw.start:
+; GCN-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GCN-NEXT:    [[NEW:%.*]] = fsub double [[LOADED]], [[VALUE:%.*]]
+; GCN-NEXT:    [[TMP2:%.*]] = bitcast double addrspace(1)* [[PTR]] to i64 addrspace(1)*
+; GCN-NEXT:    [[TMP3:%.*]] = bitcast double [[NEW]] to i64
+; GCN-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
+; GCN-NEXT:    [[TMP5:%.*]] = cmpxchg i64 addrspace(1)* [[TMP2]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst
+; GCN-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; GCN-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; GCN-NEXT:    [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
+; GCN-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GCN:       atomicrmw.end:
+; GCN-NEXT:    ret double [[TMP6]]
+;
+  %res = atomicrmw fsub double addrspace(1)* %ptr, double %value seq_cst
+  ret double %res
+}
+
+define double @test_atomicrmw_fsub_f64_local(double addrspace(3)* %ptr, double %value) {
+; GCN-LABEL: @test_atomicrmw_fsub_f64_local(
+; GCN-NEXT:    [[TMP1:%.*]] = load double, double addrspace(3)* [[PTR:%.*]], align 8
+; GCN-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; GCN:       atomicrmw.start:
+; GCN-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; GCN-NEXT:    [[NEW:%.*]] = fsub double [[LOADED]], [[VALUE:%.*]]
+; GCN-NEXT:    [[TMP2:%.*]] = bitcast double addrspace(3)* [[PTR]] to i64 addrspace(3)*
+; GCN-NEXT:    [[TMP3:%.*]] = bitcast double [[NEW]] to i64
+; GCN-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
+; GCN-NEXT:    [[TMP5:%.*]] = cmpxchg i64 addrspace(3)* [[TMP2]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst
+; GCN-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; GCN-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; GCN-NEXT:    [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
+; GCN-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; GCN:       atomicrmw.end:
+; GCN-NEXT:    ret double [[TMP6]]
+;
+  %res = atomicrmw fsub double addrspace(3)* %ptr, double %value seq_cst
+  ret double %res
+}
diff --git a/test/Transforms/AtomicExpand/ARM/atomicrmw-fp.ll b/test/Transforms/AtomicExpand/ARM/atomicrmw-fp.ll
new file mode 100644
index 0000000000000000000000000000000000000000..6f8ffc1cba21fa746d196fbeb06bedd3b7c01ebd
--- /dev/null
+++ b/test/Transforms/AtomicExpand/ARM/atomicrmw-fp.ll
@@ -0,0 +1,51 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -mtriple=armv7-apple-ios7.0 -atomic-expand %s | FileCheck %s
+
+define float @test_atomicrmw_fadd_f32(float* %ptr, float %value) {
+; CHECK-LABEL: @test_atomicrmw_fadd_f32(
+; CHECK-NEXT:    call void @llvm.arm.dmb(i32 11)
+; CHECK-NEXT:    [[TMP1:%.*]] = load float, float* [[PTR:%.*]], align 4
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; CHECK-NEXT:    [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast float* [[PTR]] to i32*
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast float [[NEW]] to i32
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast float [[LOADED]] to i32
+; CHECK-NEXT:    [[TMP5:%.*]] = cmpxchg i32* [[TMP2]], i32 [[TMP4]], i32 [[TMP3]] monotonic monotonic
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1
+; CHECK-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6]] = bitcast i32 [[NEWLOADED]] to float
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    call void @llvm.arm.dmb(i32 11)
+; CHECK-NEXT:    ret float [[TMP6]]
+;
+  %res = atomicrmw fadd float* %ptr, float %value seq_cst
+  ret float %res
+}
+
+define float @test_atomicrmw_fsub_f32(float* %ptr, float %value) {
+; CHECK-LABEL: @test_atomicrmw_fsub_f32(
+; CHECK-NEXT:    call void @llvm.arm.dmb(i32 11)
+; CHECK-NEXT:    [[TMP1:%.*]] = load float, float* [[PTR:%.*]], align 4
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; CHECK-NEXT:    [[NEW:%.*]] = fsub float [[LOADED]], [[VALUE:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast float* [[PTR]] to i32*
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast float [[NEW]] to i32
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast float [[LOADED]] to i32
+; CHECK-NEXT:    [[TMP5:%.*]] = cmpxchg i32* [[TMP2]], i32 [[TMP4]], i32 [[TMP3]] monotonic monotonic
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1
+; CHECK-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6]] = bitcast i32 [[NEWLOADED]] to float
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    call void @llvm.arm.dmb(i32 11)
+; CHECK-NEXT:    ret float [[TMP6]]
+;
+  %res = atomicrmw fsub float* %ptr, float %value seq_cst
+  ret float %res
+}
+
diff --git a/test/Transforms/AtomicExpand/Hexagon/atomicrmw-fp.ll b/test/Transforms/AtomicExpand/Hexagon/atomicrmw-fp.ll
new file mode 100644
index 0000000000000000000000000000000000000000..34026909d7642d6063e4a38d91b5f592eadd48de
--- /dev/null
+++ b/test/Transforms/AtomicExpand/Hexagon/atomicrmw-fp.ll
@@ -0,0 +1,47 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -mtriple=hexagon-- -atomic-expand %s | FileCheck %s
+
+define float @test_atomicrmw_fadd_f32(float* %ptr, float %value) {
+; CHECK-LABEL: @test_atomicrmw_fadd_f32(
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast float* [[PTR:%.*]] to i32*
+; CHECK-NEXT:    [[LARX:%.*]] = call i32 @llvm.hexagon.L2.loadw.locked(i32* [[TMP1]])
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i32 [[LARX]] to float
+; CHECK-NEXT:    [[NEW:%.*]] = fadd float [[TMP2]], [[VALUE:%.*]]
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast float* [[PTR]] to i32*
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast float [[NEW]] to i32
+; CHECK-NEXT:    [[STCX:%.*]] = call i32 @llvm.hexagon.S2.storew.locked(i32* [[TMP3]], i32 [[TMP4]])
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[STCX]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = zext i1 [[TMP5]] to i32
+; CHECK-NEXT:    [[TRYAGAIN:%.*]] = icmp ne i32 [[TMP6]], 0
+; CHECK-NEXT:    br i1 [[TRYAGAIN]], label [[ATOMICRMW_START]], label [[ATOMICRMW_END:%.*]]
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret float [[TMP2]]
+;
+  %res = atomicrmw fadd float* %ptr, float %value seq_cst
+  ret float %res
+}
+
+define float @test_atomicrmw_fsub_f32(float* %ptr, float %value) {
+; CHECK-LABEL: @test_atomicrmw_fsub_f32(
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast float* [[PTR:%.*]] to i32*
+; CHECK-NEXT:    [[LARX:%.*]] = call i32 @llvm.hexagon.L2.loadw.locked(i32* [[TMP1]])
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i32 [[LARX]] to float
+; CHECK-NEXT:    [[NEW:%.*]] = fsub float [[TMP2]], [[VALUE:%.*]]
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast float* [[PTR]] to i32*
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast float [[NEW]] to i32
+; CHECK-NEXT:    [[STCX:%.*]] = call i32 @llvm.hexagon.S2.storew.locked(i32* [[TMP3]], i32 [[TMP4]])
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[STCX]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = zext i1 [[TMP5]] to i32
+; CHECK-NEXT:    [[TRYAGAIN:%.*]] = icmp ne i32 [[TMP6]], 0
+; CHECK-NEXT:    br i1 [[TRYAGAIN]], label [[ATOMICRMW_START]], label [[ATOMICRMW_END:%.*]]
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret float [[TMP2]]
+;
+  %res = atomicrmw fsub float* %ptr, float %value seq_cst
+  ret float %res
+}
+
diff --git a/test/Transforms/AtomicExpand/Hexagon/lit.local.cfg b/test/Transforms/AtomicExpand/Hexagon/lit.local.cfg
new file mode 100644
index 0000000000000000000000000000000000000000..cc6a7edf05f370e7c39cad53a944f41cd61bd5c0
--- /dev/null
+++ b/test/Transforms/AtomicExpand/Hexagon/lit.local.cfg
@@ -0,0 +1,2 @@
+if not 'Hexagon' in config.root.targets:
+    config.unsupported = True
diff --git a/test/Transforms/AtomicExpand/Mips/atomicrmw-fp.ll b/test/Transforms/AtomicExpand/Mips/atomicrmw-fp.ll
new file mode 100644
index 0000000000000000000000000000000000000000..7931b2bb7f3d64606b5bf2014b3f29bf20d860f2
--- /dev/null
+++ b/test/Transforms/AtomicExpand/Mips/atomicrmw-fp.ll
@@ -0,0 +1,51 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -mtriple=mips64-mti-linux-gnu -atomic-expand %s | FileCheck %s
+
+define float @test_atomicrmw_fadd_f32(float* %ptr, float %value) {
+; CHECK-LABEL: @test_atomicrmw_fadd_f32(
+; CHECK-NEXT:    fence seq_cst
+; CHECK-NEXT:    [[TMP1:%.*]] = load float, float* [[PTR:%.*]], align 4
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; CHECK-NEXT:    [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast float* [[PTR]] to i32*
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast float [[NEW]] to i32
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast float [[LOADED]] to i32
+; CHECK-NEXT:    [[TMP5:%.*]] = cmpxchg i32* [[TMP2]], i32 [[TMP4]], i32 [[TMP3]] monotonic monotonic
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1
+; CHECK-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6]] = bitcast i32 [[NEWLOADED]] to float
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    fence seq_cst
+; CHECK-NEXT:    ret float [[TMP6]]
+;
+  %res = atomicrmw fadd float* %ptr, float %value seq_cst
+  ret float %res
+}
+
+define float @test_atomicrmw_fsub_f32(float* %ptr, float %value) {
+; CHECK-LABEL: @test_atomicrmw_fsub_f32(
+; CHECK-NEXT:    fence seq_cst
+; CHECK-NEXT:    [[TMP1:%.*]] = load float, float* [[PTR:%.*]], align 4
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; CHECK-NEXT:    [[NEW:%.*]] = fsub float [[LOADED]], [[VALUE:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast float* [[PTR]] to i32*
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast float [[NEW]] to i32
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast float [[LOADED]] to i32
+; CHECK-NEXT:    [[TMP5:%.*]] = cmpxchg i32* [[TMP2]], i32 [[TMP4]], i32 [[TMP3]] monotonic monotonic
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1
+; CHECK-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6]] = bitcast i32 [[NEWLOADED]] to float
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    fence seq_cst
+; CHECK-NEXT:    ret float [[TMP6]]
+;
+  %res = atomicrmw fsub float* %ptr, float %value seq_cst
+  ret float %res
+}
+
diff --git a/test/Transforms/AtomicExpand/Mips/lit.local.cfg b/test/Transforms/AtomicExpand/Mips/lit.local.cfg
new file mode 100644
index 0000000000000000000000000000000000000000..7d12f7a9c56494e6f3844dfad349aad958e933a0
--- /dev/null
+++ b/test/Transforms/AtomicExpand/Mips/lit.local.cfg
@@ -0,0 +1,2 @@
+if not 'Mips' in config.root.targets:
+    config.unsupported = True
diff --git a/test/Transforms/AtomicExpand/RISCV/atomicrmw-fp.ll b/test/Transforms/AtomicExpand/RISCV/atomicrmw-fp.ll
new file mode 100644
index 0000000000000000000000000000000000000000..7ffe404c843187067688d4670babcc1dc05db844
--- /dev/null
+++ b/test/Transforms/AtomicExpand/RISCV/atomicrmw-fp.ll
@@ -0,0 +1,59 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -mtriple=riscv32-- -atomic-expand %s | FileCheck %s
+
+define float @test_atomicrmw_fadd_f32(float* %ptr, float %value) {
+; CHECK-LABEL: @test_atomicrmw_fadd_f32(
+; CHECK-NEXT:    [[TMP1:%.*]] = alloca float, align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = load float, float* [[PTR:%.*]], align 4
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi float [ [[TMP2]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; CHECK-NEXT:    [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]]
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast float* [[PTR]] to i8*
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast float* [[TMP1]] to i8*
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0i8(i64 4, i8* [[TMP4]])
+; CHECK-NEXT:    store float [[LOADED]], float* [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast float [[NEW]] to i32
+; CHECK-NEXT:    [[TMP6:%.*]] = call zeroext i1 @__atomic_compare_exchange_4(i8* [[TMP3]], i8* [[TMP4]], i32 [[TMP5]], i32 5, i32 5)
+; CHECK-NEXT:    [[TMP7:%.*]] = load float, float* [[TMP1]], align 4
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0i8(i64 4, i8* [[TMP4]])
+; CHECK-NEXT:    [[TMP8:%.*]] = insertvalue { float, i1 } undef, float [[TMP7]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = insertvalue { float, i1 } [[TMP8]], i1 [[TMP6]], 1
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { float, i1 } [[TMP9]], 1
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { float, i1 } [[TMP9]], 0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret float [[NEWLOADED]]
+;
+  %res = atomicrmw fadd float* %ptr, float %value seq_cst
+  ret float %res
+}
+
+define float @test_atomicrmw_fsub_f32(float* %ptr, float %value) {
+; CHECK-LABEL: @test_atomicrmw_fsub_f32(
+; CHECK-NEXT:    [[TMP1:%.*]] = alloca float, align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = load float, float* [[PTR:%.*]], align 4
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi float [ [[TMP2]], [[TMP0:%.*]] ], [ [[NEWLOADED:%.*]], [[ATOMICRMW_START]] ]
+; CHECK-NEXT:    [[NEW:%.*]] = fsub float [[LOADED]], [[VALUE:%.*]]
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast float* [[PTR]] to i8*
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast float* [[TMP1]] to i8*
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0i8(i64 4, i8* [[TMP4]])
+; CHECK-NEXT:    store float [[LOADED]], float* [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast float [[NEW]] to i32
+; CHECK-NEXT:    [[TMP6:%.*]] = call zeroext i1 @__atomic_compare_exchange_4(i8* [[TMP3]], i8* [[TMP4]], i32 [[TMP5]], i32 5, i32 5)
+; CHECK-NEXT:    [[TMP7:%.*]] = load float, float* [[TMP1]], align 4
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0i8(i64 4, i8* [[TMP4]])
+; CHECK-NEXT:    [[TMP8:%.*]] = insertvalue { float, i1 } undef, float [[TMP7]], 0
+; CHECK-NEXT:    [[TMP9:%.*]] = insertvalue { float, i1 } [[TMP8]], i1 [[TMP6]], 1
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { float, i1 } [[TMP9]], 1
+; CHECK-NEXT:    [[NEWLOADED]] = extractvalue { float, i1 } [[TMP9]], 0
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret float [[NEWLOADED]]
+;
+  %res = atomicrmw fsub float* %ptr, float %value seq_cst
+  ret float %res
+}
+
diff --git a/test/Transforms/AtomicExpand/RISCV/lit.local.cfg b/test/Transforms/AtomicExpand/RISCV/lit.local.cfg
new file mode 100644
index 0000000000000000000000000000000000000000..7aaeda5a5b37908b17bf47faf4ce632cc0b9d897
--- /dev/null
+++ b/test/Transforms/AtomicExpand/RISCV/lit.local.cfg
@@ -0,0 +1,5 @@
+config.suffixes = ['.ll']
+
+targets = set(config.root.targets_to_build.split())
+if not 'RISCV' in targets:
+    config.unsupported = True
diff --git a/test/Transforms/AtomicExpand/X86/expand-atomic-rmw-fp.ll b/test/Transforms/AtomicExpand/X86/expand-atomic-rmw-fp.ll
new file mode 100644
index 0000000000000000000000000000000000000000..e37f9bb7d56971e5aa1022b7996555a9e2659dd5
--- /dev/null
+++ b/test/Transforms/AtomicExpand/X86/expand-atomic-rmw-fp.ll
@@ -0,0 +1,112 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -mtriple=i686-linux-gnu -atomic-expand %s | FileCheck %s
+
+define float @test_atomicrmw_fadd_f32(float* %ptr, float %value) {
+; CHECK-LABEL: @test_atomicrmw_fadd_f32(
+; CHECK-NEXT:    [[TMP1:%.*]] = load float, float* [[PTR:%.*]], align 4
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; CHECK-NEXT:    [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast float* [[PTR]] to i32*
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast float [[NEW]] to i32
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast float [[LOADED]] to i32
+; CHECK-NEXT:    [[TMP5:%.*]] = cmpxchg i32* [[TMP2]], i32 [[TMP4]], i32 [[TMP3]] seq_cst seq_cst
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1
+; CHECK-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6]] = bitcast i32 [[NEWLOADED]] to float
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret float [[TMP6]]
+;
+  %res = atomicrmw fadd float* %ptr, float %value seq_cst
+  ret float %res
+}
+
+define double @test_atomicrmw_fadd_f64(double* %ptr, double %value) {
+; CHECK-LABEL: @test_atomicrmw_fadd_f64(
+; CHECK-NEXT:    [[TMP1:%.*]] = load double, double* [[PTR:%.*]], align 8
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; CHECK-NEXT:    [[NEW:%.*]] = fadd double [[LOADED]], [[VALUE:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast double* [[PTR]] to i64*
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast double [[NEW]] to i64
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = cmpxchg i64* [[TMP2]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; CHECK-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret double [[TMP6]]
+;
+  %res = atomicrmw fadd double* %ptr, double %value seq_cst
+  ret double %res
+}
+
+define float @test_atomicrmw_fadd_f32_as1(float addrspace(1)* %ptr, float %value) {
+; CHECK-LABEL: @test_atomicrmw_fadd_f32_as1(
+; CHECK-NEXT:    [[TMP1:%.*]] = load float, float addrspace(1)* [[PTR:%.*]], align 4
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; CHECK-NEXT:    [[NEW:%.*]] = fadd float [[LOADED]], [[VALUE:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast float addrspace(1)* [[PTR]] to i32 addrspace(1)*
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast float [[NEW]] to i32
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast float [[LOADED]] to i32
+; CHECK-NEXT:    [[TMP5:%.*]] = cmpxchg i32 addrspace(1)* [[TMP2]], i32 [[TMP4]], i32 [[TMP3]] seq_cst seq_cst
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1
+; CHECK-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6]] = bitcast i32 [[NEWLOADED]] to float
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret float [[TMP6]]
+;
+  %res = atomicrmw fadd float addrspace(1)* %ptr, float %value seq_cst
+  ret float %res
+}
+
+define float @test_atomicrmw_fsub_f32(float* %ptr, float %value) {
+; CHECK-LABEL: @test_atomicrmw_fsub_f32(
+; CHECK-NEXT:    [[TMP1:%.*]] = load float, float* [[PTR:%.*]], align 4
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi float [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; CHECK-NEXT:    [[NEW:%.*]] = fsub float [[LOADED]], [[VALUE:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast float* [[PTR]] to i32*
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast float [[NEW]] to i32
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast float [[LOADED]] to i32
+; CHECK-NEXT:    [[TMP5:%.*]] = cmpxchg i32* [[TMP2]], i32 [[TMP4]], i32 [[TMP3]] seq_cst seq_cst
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i32, i1 } [[TMP5]], 1
+; CHECK-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i32, i1 } [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6]] = bitcast i32 [[NEWLOADED]] to float
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret float [[TMP6]]
+;
+  %res = atomicrmw fsub float* %ptr, float %value seq_cst
+  ret float %res
+}
+
+define double @test_atomicrmw_fsub_f64(double* %ptr, double %value) {
+; CHECK-LABEL: @test_atomicrmw_fsub_f64(
+; CHECK-NEXT:    [[TMP1:%.*]] = load double, double* [[PTR:%.*]], align 8
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP6:%.*]], [[ATOMICRMW_START]] ]
+; CHECK-NEXT:    [[NEW:%.*]] = fsub double [[LOADED]], [[VALUE:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast double* [[PTR]] to i64*
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast double [[NEW]] to i64
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast double [[LOADED]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = cmpxchg i64* [[TMP2]], i64 [[TMP4]], i64 [[TMP3]] seq_cst seq_cst
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP5]], 1
+; CHECK-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6]] = bitcast i64 [[NEWLOADED]] to double
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret double [[TMP6]]
+;
+  %res = atomicrmw fsub double* %ptr, double %value seq_cst
+  ret double %res
+}
diff --git a/test/Transforms/AtomicExpand/X86/expand-atomic-xchg-fp.ll b/test/Transforms/AtomicExpand/X86/expand-atomic-xchg-fp.ll
new file mode 100644
index 0000000000000000000000000000000000000000..3389cf0c19deb9c0197eed02c187ad0cbba266c8
--- /dev/null
+++ b/test/Transforms/AtomicExpand/X86/expand-atomic-xchg-fp.ll
@@ -0,0 +1,42 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -mtriple=i686-linux-gnu -atomic-expand %s | FileCheck %s
+
+define double @atomic_xchg_f64(double* %ptr) nounwind {
+; CHECK-LABEL: @atomic_xchg_f64(
+; CHECK-NEXT:    [[TMP1:%.*]] = load double, double* [[PTR:%.*]], align 8
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast double* [[PTR]] to i64*
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
+; CHECK-NEXT:    [[TMP4:%.*]] = cmpxchg i64* [[TMP2]], i64 [[TMP3]], i64 4616189618054758400 seq_cst seq_cst
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
+; CHECK-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5]] = bitcast i64 [[NEWLOADED]] to double
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret double [[TMP5]]
+;
+  %result = atomicrmw xchg double* %ptr, double 4.0 seq_cst
+  ret double %result
+}
+
+define double @atomic_xchg_f64_as1(double addrspace(1)* %ptr) nounwind {
+; CHECK-LABEL: @atomic_xchg_f64_as1(
+; CHECK-NEXT:    [[TMP1:%.*]] = load double, double addrspace(1)* [[PTR:%.*]], align 8
+; CHECK-NEXT:    br label [[ATOMICRMW_START:%.*]]
+; CHECK:       atomicrmw.start:
+; CHECK-NEXT:    [[LOADED:%.*]] = phi double [ [[TMP1]], [[TMP0:%.*]] ], [ [[TMP5:%.*]], [[ATOMICRMW_START]] ]
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast double addrspace(1)* [[PTR]] to i64 addrspace(1)*
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast double [[LOADED]] to i64
+; CHECK-NEXT:    [[TMP4:%.*]] = cmpxchg i64 addrspace(1)* [[TMP2]], i64 [[TMP3]], i64 4616189618054758400 seq_cst seq_cst
+; CHECK-NEXT:    [[SUCCESS:%.*]] = extractvalue { i64, i1 } [[TMP4]], 1
+; CHECK-NEXT:    [[NEWLOADED:%.*]] = extractvalue { i64, i1 } [[TMP4]], 0
+; CHECK-NEXT:    [[TMP5]] = bitcast i64 [[NEWLOADED]] to double
+; CHECK-NEXT:    br i1 [[SUCCESS]], label [[ATOMICRMW_END:%.*]], label [[ATOMICRMW_START]]
+; CHECK:       atomicrmw.end:
+; CHECK-NEXT:    ret double [[TMP5]]
+;
+  %result = atomicrmw xchg double addrspace(1)* %ptr, double 4.0 seq_cst
+  ret double %result
+}
diff --git a/test/Transforms/BDCE/dead-uses.ll b/test/Transforms/BDCE/dead-uses.ll
index 95c2893871f72411ba977e31e507ef76f9925872..34f8cc007f148469930c861968f230c2b2cc8cbc 100644
--- a/test/Transforms/BDCE/dead-uses.ll
+++ b/test/Transforms/BDCE/dead-uses.ll
@@ -10,7 +10,7 @@ declare <2 x i32> @llvm.fshr.v2i32(<2 x i32>, <2 x i32>, <2 x i32>)
 define i32 @pr39771_fshr_multi_use_instr(i32 %a) {
 ; CHECK-LABEL: @pr39771_fshr_multi_use_instr(
 ; CHECK-NEXT:    [[X:%.*]] = or i32 [[A:%.*]], 0
-; CHECK-NEXT:    [[B:%.*]] = tail call i32 @llvm.fshr.i32(i32 [[X]], i32 [[X]], i32 1)
+; CHECK-NEXT:    [[B:%.*]] = tail call i32 @llvm.fshr.i32(i32 0, i32 [[X]], i32 1)
 ; CHECK-NEXT:    [[C:%.*]] = lshr i32 [[B]], 23
 ; CHECK-NEXT:    [[D:%.*]] = xor i32 [[C]], [[B]]
 ; CHECK-NEXT:    [[E:%.*]] = and i32 [[D]], 31
@@ -28,7 +28,7 @@ define i32 @pr39771_fshr_multi_use_instr(i32 %a) {
 define <2 x i32> @pr39771_fshr_multi_use_instr_vec(<2 x i32> %a) {
 ; CHECK-LABEL: @pr39771_fshr_multi_use_instr_vec(
 ; CHECK-NEXT:    [[X:%.*]] = or <2 x i32> [[A:%.*]], zeroinitializer
-; CHECK-NEXT:    [[B:%.*]] = tail call <2 x i32> @llvm.fshr.v2i32(<2 x i32> [[X]], <2 x i32> [[X]], <2 x i32> <i32 1, i32 1>)
+; CHECK-NEXT:    [[B:%.*]] = tail call <2 x i32> @llvm.fshr.v2i32(<2 x i32> zeroinitializer, <2 x i32> [[X]], <2 x i32> <i32 1, i32 1>)
 ; CHECK-NEXT:    [[C:%.*]] = lshr <2 x i32> [[B]], <i32 23, i32 23>
 ; CHECK-NEXT:    [[D:%.*]] = xor <2 x i32> [[C]], [[B]]
 ; CHECK-NEXT:    [[E:%.*]] = and <2 x i32> [[D]], <i32 31, i32 31>
@@ -45,7 +45,7 @@ define <2 x i32> @pr39771_fshr_multi_use_instr_vec(<2 x i32> %a) {
 ; First fshr operand is dead, but it comes from an argument, not instruction.
 define i32 @pr39771_fshr_multi_use_arg(i32 %a) {
 ; CHECK-LABEL: @pr39771_fshr_multi_use_arg(
-; CHECK-NEXT:    [[B:%.*]] = tail call i32 @llvm.fshr.i32(i32 [[A:%.*]], i32 [[A]], i32 1)
+; CHECK-NEXT:    [[B:%.*]] = tail call i32 @llvm.fshr.i32(i32 0, i32 [[A:%.*]], i32 1)
 ; CHECK-NEXT:    [[C:%.*]] = lshr i32 [[B]], 23
 ; CHECK-NEXT:    [[D:%.*]] = xor i32 [[C]], [[B]]
 ; CHECK-NEXT:    [[E:%.*]] = and i32 [[D]], 31
@@ -58,11 +58,10 @@ define i32 @pr39771_fshr_multi_use_arg(i32 %a) {
   ret i32 %e
 }
 
-; Second or operand is dead, but BDCE does not realize this.
 define i32 @pr39771_expanded_fshr_multi_use(i32 %a) {
 ; CHECK-LABEL: @pr39771_expanded_fshr_multi_use(
 ; CHECK-NEXT:    [[TMP:%.*]] = lshr i32 [[A:%.*]], 1
-; CHECK-NEXT:    [[TMP2:%.*]] = shl i32 [[A]], 32
+; CHECK-NEXT:    [[TMP2:%.*]] = shl i32 0, 31
 ; CHECK-NEXT:    [[B:%.*]] = or i32 [[TMP]], [[TMP2]]
 ; CHECK-NEXT:    [[C:%.*]] = lshr i32 [[B]], 23
 ; CHECK-NEXT:    [[D:%.*]] = xor i32 [[C]], [[B]]
@@ -70,10 +69,35 @@ define i32 @pr39771_expanded_fshr_multi_use(i32 %a) {
 ; CHECK-NEXT:    ret i32 [[E]]
 ;
   %tmp = lshr i32 %a, 1
-  %tmp2 = shl i32 %a, 32
+  %tmp2 = shl i32 %a, 31
   %b = or i32 %tmp, %tmp2
   %c = lshr i32 %b, 23
   %d = xor i32 %c, %b
   %e = and i32 %d, 31
   ret i32 %e
 }
+
+; %b operand of %c will be dead initially, but later found live.
+define void @dead_use_invalidation(i32 %a) {
+; CHECK-LABEL: @dead_use_invalidation(
+; CHECK-NEXT:    [[B:%.*]] = or i32 [[A:%.*]], 0
+; CHECK-NEXT:    [[C:%.*]] = shl i32 [[B]], 31
+; CHECK-NEXT:    [[D:%.*]] = and i32 [[C]], 1
+; CHECK-NEXT:    [[E:%.*]] = or i32 [[C]], 0
+; CHECK-NEXT:    [[F:%.*]] = or i32 [[D]], 0
+; CHECK-NEXT:    call void @dummy(i32 [[E]])
+; CHECK-NEXT:    call void @dummy(i32 [[F]])
+; CHECK-NEXT:    call void @dummy(i32 [[B]])
+; CHECK-NEXT:    ret void
+;
+  %b = or i32 %a, 0
+  %c = shl i32 %b, 31
+  %d = and i32 %c, 1
+  %e = or i32 %c, 0
+  %f = or i32 %d, 0
+  call void @dummy(i32 %e)
+  call void @dummy(i32 %f)
+  call void @dummy(i32 %b)
+  ret void
+}
+declare void @dummy(i32)
diff --git a/test/Transforms/BDCE/invalidate-assumptions.ll b/test/Transforms/BDCE/invalidate-assumptions.ll
index d165d74be86d697ed2f396f9c5eeaac77d7cfead..69a99b33d900464b562b12d0cc87712133e484a7 100644
--- a/test/Transforms/BDCE/invalidate-assumptions.ll
+++ b/test/Transforms/BDCE/invalidate-assumptions.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -bdce %s -S | FileCheck %s
 
 ; The 'nuw' on the subtract allows us to deduce that %setbit is not demanded.
@@ -8,8 +9,7 @@
 
 define i1 @PR33695(i1 %b, i8 %x) {
 ; CHECK-LABEL: @PR33695(
-; CHECK-NEXT:    [[SETBIT:%.*]] = or i8 %x, 64
-; CHECK-NEXT:    [[LITTLE_NUMBER:%.*]] = zext i1 %b to i8
+; CHECK-NEXT:    [[LITTLE_NUMBER:%.*]] = zext i1 [[B:%.*]] to i8
 ; CHECK-NEXT:    [[BIG_NUMBER:%.*]] = shl i8 0, 1
 ; CHECK-NEXT:    [[SUB:%.*]] = sub i8 [[BIG_NUMBER]], [[LITTLE_NUMBER]]
 ; CHECK-NEXT:    [[TRUNC:%.*]] = trunc i8 [[SUB]] to i1
@@ -28,16 +28,12 @@ define i1 @PR33695(i1 %b, i8 %x) {
 
 define i64 @PR34037(i64 %m, i32 %r, i64 %j, i1 %b, i32 %k, i64 %p) {
 ; CHECK-LABEL: @PR34037(
-; CHECK-NEXT:    [[CONV:%.*]] = zext i32 %r to i64
-; CHECK-NEXT:    [[AND:%.*]] = and i64 %m, 0
-; CHECK-NEXT:    [[NEG:%.*]] = xor i64 0, 34359738367
-; CHECK-NEXT:    [[OR:%.*]] = or i64 %j, 0
 ; CHECK-NEXT:    [[SHL:%.*]] = shl i64 0, 29
-; CHECK-NEXT:    [[CONV1:%.*]] = select i1 %b, i64 7, i64 0
+; CHECK-NEXT:    [[CONV1:%.*]] = select i1 [[B:%.*]], i64 7, i64 0
 ; CHECK-NEXT:    [[SUB:%.*]] = sub i64 [[SHL]], [[CONV1]]
-; CHECK-NEXT:    [[CONV2:%.*]] = zext i32 %k to i64
+; CHECK-NEXT:    [[CONV2:%.*]] = zext i32 [[K:%.*]] to i64
 ; CHECK-NEXT:    [[MUL:%.*]] = mul i64 [[SUB]], [[CONV2]]
-; CHECK-NEXT:    [[CONV4:%.*]] = and i64 %p, 65535
+; CHECK-NEXT:    [[CONV4:%.*]] = and i64 [[P:%.*]], 65535
 ; CHECK-NEXT:    [[AND5:%.*]] = and i64 [[MUL]], [[CONV4]]
 ; CHECK-NEXT:    ret i64 [[AND5]]
 ;
@@ -63,8 +59,7 @@ declare i1 @foo(i1)
 
 define i1 @poison_on_call_user_is_ok(i1 %b, i8 %x) {
 ; CHECK-LABEL: @poison_on_call_user_is_ok(
-; CHECK-NEXT:    [[SETBIT:%.*]] = or i8 %x, 64
-; CHECK-NEXT:    [[LITTLE_NUMBER:%.*]] = zext i1 %b to i8
+; CHECK-NEXT:    [[LITTLE_NUMBER:%.*]] = zext i1 [[B:%.*]] to i8
 ; CHECK-NEXT:    [[BIG_NUMBER:%.*]] = shl i8 0, 1
 ; CHECK-NEXT:    [[SUB:%.*]] = sub i8 [[BIG_NUMBER]], [[LITTLE_NUMBER]]
 ; CHECK-NEXT:    [[TRUNC:%.*]] = trunc i8 [[SUB]] to i1
@@ -90,7 +85,7 @@ define i1 @poison_on_call_user_is_ok(i1 %b, i8 %x) {
 
 define void @PR34179(i32* %a) {
 ; CHECK-LABEL: @PR34179(
-; CHECK-NEXT:    [[T0:%.*]] = load volatile i32, i32* %a
+; CHECK-NEXT:    [[T0:%.*]] = load volatile i32, i32* [[A:%.*]]
 ; CHECK-NEXT:    ret void
 ;
   %t0 = load volatile i32, i32* %a
diff --git a/test/Transforms/BDCE/vectors.ll b/test/Transforms/BDCE/vectors.ll
index fde22fd36b7883b7f9940ec69c784112689d6387..63a858726809f36b8648810bb4fa1d3c59c6ac2a 100644
--- a/test/Transforms/BDCE/vectors.ll
+++ b/test/Transforms/BDCE/vectors.ll
@@ -1,25 +1,16 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -S -bdce -instsimplify < %s | FileCheck %s
-; RUN: opt -S -instsimplify < %s | FileCheck %s -check-prefix=CHECK-IO
-; CHECK-IO lines to ensure that transformations are not performed by only instsimplify.
+; RUN: opt -S -bdce < %s | FileCheck %s
 
 ; BDCE applied to integer vectors.
 
 define <2 x i32> @test_basic(<2 x i32> %a, <2 x i32> %b) {
 ; CHECK-LABEL: @test_basic(
+; CHECK-NEXT:    [[A3:%.*]] = and <2 x i32> zeroinitializer, <i32 4, i32 4>
 ; CHECK-NEXT:    [[B2:%.*]] = add <2 x i32> [[B:%.*]], <i32 1, i32 1>
 ; CHECK-NEXT:    [[B3:%.*]] = and <2 x i32> [[B2]], <i32 8, i32 8>
-; CHECK-NEXT:    [[D:%.*]] = ashr <2 x i32> [[B3]], <i32 3, i32 3>
+; CHECK-NEXT:    [[C:%.*]] = or <2 x i32> [[A3]], [[B3]]
+; CHECK-NEXT:    [[D:%.*]] = ashr <2 x i32> [[C]], <i32 3, i32 3>
 ; CHECK-NEXT:    ret <2 x i32> [[D]]
-;
-; CHECK-IO-LABEL: @test_basic(
-; CHECK-IO-NEXT:    [[A2:%.*]] = add <2 x i32> [[A:%.*]], <i32 1, i32 1>
-; CHECK-IO-NEXT:    [[A3:%.*]] = and <2 x i32> [[A2]], <i32 4, i32 4>
-; CHECK-IO-NEXT:    [[B2:%.*]] = add <2 x i32> [[B:%.*]], <i32 1, i32 1>
-; CHECK-IO-NEXT:    [[B3:%.*]] = and <2 x i32> [[B2]], <i32 8, i32 8>
-; CHECK-IO-NEXT:    [[C:%.*]] = or <2 x i32> [[A3]], [[B3]]
-; CHECK-IO-NEXT:    [[D:%.*]] = ashr <2 x i32> [[C]], <i32 3, i32 3>
-; CHECK-IO-NEXT:    ret <2 x i32> [[D]]
 ;
   %a2 = add <2 x i32> %a, <i32 1, i32 1>
   %a3 = and <2 x i32> %a2, <i32 4, i32 4>
@@ -33,21 +24,13 @@ define <2 x i32> @test_basic(<2 x i32> %a, <2 x i32> %b) {
 ; Going vector -> scalar
 define i32 @test_extractelement(<2 x i32> %a, <2 x i32> %b) {
 ; CHECK-LABEL: @test_extractelement(
+; CHECK-NEXT:    [[A3:%.*]] = and <2 x i32> zeroinitializer, <i32 4, i32 4>
 ; CHECK-NEXT:    [[B2:%.*]] = add <2 x i32> [[B:%.*]], <i32 1, i32 1>
 ; CHECK-NEXT:    [[B3:%.*]] = and <2 x i32> [[B2]], <i32 8, i32 8>
-; CHECK-NEXT:    [[D:%.*]] = extractelement <2 x i32> [[B3]], i32 0
+; CHECK-NEXT:    [[C:%.*]] = or <2 x i32> [[A3]], [[B3]]
+; CHECK-NEXT:    [[D:%.*]] = extractelement <2 x i32> [[C]], i32 0
 ; CHECK-NEXT:    [[E:%.*]] = ashr i32 [[D]], 3
 ; CHECK-NEXT:    ret i32 [[E]]
-;
-; CHECK-IO-LABEL: @test_extractelement(
-; CHECK-IO-NEXT:    [[A2:%.*]] = add <2 x i32> [[A:%.*]], <i32 1, i32 1>
-; CHECK-IO-NEXT:    [[A3:%.*]] = and <2 x i32> [[A2]], <i32 4, i32 4>
-; CHECK-IO-NEXT:    [[B2:%.*]] = add <2 x i32> [[B:%.*]], <i32 1, i32 1>
-; CHECK-IO-NEXT:    [[B3:%.*]] = and <2 x i32> [[B2]], <i32 8, i32 8>
-; CHECK-IO-NEXT:    [[C:%.*]] = or <2 x i32> [[A3]], [[B3]]
-; CHECK-IO-NEXT:    [[D:%.*]] = extractelement <2 x i32> [[C]], i32 0
-; CHECK-IO-NEXT:    [[E:%.*]] = ashr i32 [[D]], 3
-; CHECK-IO-NEXT:    ret i32 [[E]]
 ;
   %a2 = add <2 x i32> %a, <i32 1, i32 1>
   %a3 = and <2 x i32> %a2, <i32 4, i32 4>
@@ -62,22 +45,13 @@ define i32 @test_extractelement(<2 x i32> %a, <2 x i32> %b) {
 ; Going scalar -> vector
 define <2 x i32> @test_insertelement(i32 %a, i32 %b) {
 ; CHECK-LABEL: @test_insertelement(
+; CHECK-NEXT:    [[X3:%.*]] = and <2 x i32> zeroinitializer, <i32 4, i32 4>
 ; CHECK-NEXT:    [[Y:%.*]] = insertelement <2 x i32> undef, i32 [[B:%.*]], i32 0
 ; CHECK-NEXT:    [[Y2:%.*]] = insertelement <2 x i32> [[Y]], i32 [[A:%.*]], i32 1
 ; CHECK-NEXT:    [[Y3:%.*]] = and <2 x i32> [[Y2]], <i32 8, i32 8>
-; CHECK-NEXT:    [[U:%.*]] = ashr <2 x i32> [[Y3]], <i32 3, i32 3>
+; CHECK-NEXT:    [[Z:%.*]] = or <2 x i32> [[X3]], [[Y3]]
+; CHECK-NEXT:    [[U:%.*]] = ashr <2 x i32> [[Z]], <i32 3, i32 3>
 ; CHECK-NEXT:    ret <2 x i32> [[U]]
-;
-; CHECK-IO-LABEL: @test_insertelement(
-; CHECK-IO-NEXT:    [[X:%.*]] = insertelement <2 x i32> undef, i32 [[A:%.*]], i32 0
-; CHECK-IO-NEXT:    [[X2:%.*]] = insertelement <2 x i32> [[X]], i32 [[B:%.*]], i32 1
-; CHECK-IO-NEXT:    [[X3:%.*]] = and <2 x i32> [[X2]], <i32 4, i32 4>
-; CHECK-IO-NEXT:    [[Y:%.*]] = insertelement <2 x i32> undef, i32 [[B]], i32 0
-; CHECK-IO-NEXT:    [[Y2:%.*]] = insertelement <2 x i32> [[Y]], i32 [[A]], i32 1
-; CHECK-IO-NEXT:    [[Y3:%.*]] = and <2 x i32> [[Y2]], <i32 8, i32 8>
-; CHECK-IO-NEXT:    [[Z:%.*]] = or <2 x i32> [[X3]], [[Y3]]
-; CHECK-IO-NEXT:    [[U:%.*]] = ashr <2 x i32> [[Z]], <i32 3, i32 3>
-; CHECK-IO-NEXT:    ret <2 x i32> [[U]]
 ;
   %x = insertelement <2 x i32> undef, i32 %a, i32 0
   %x2 = insertelement <2 x i32> %x, i32 %b, i32 1
@@ -100,15 +74,6 @@ define <2 x i32> @test_conversion(<2 x i32> %a) {
 ; CHECK-NEXT:    [[Z:%.*]] = fptoui <2 x double> [[Y]] to <2 x i32>
 ; CHECK-NEXT:    [[U:%.*]] = ashr <2 x i32> [[Z]], <i32 3, i32 3>
 ; CHECK-NEXT:    ret <2 x i32> [[U]]
-;
-; CHECK-IO-LABEL: @test_conversion(
-; CHECK-IO-NEXT:    [[A2:%.*]] = add <2 x i32> [[A:%.*]], <i32 1, i32 1>
-; CHECK-IO-NEXT:    [[A3:%.*]] = and <2 x i32> [[A2]], <i32 2, i32 2>
-; CHECK-IO-NEXT:    [[X:%.*]] = uitofp <2 x i32> [[A3]] to <2 x double>
-; CHECK-IO-NEXT:    [[Y:%.*]] = fadd <2 x double> [[X]], <double 1.000000e+00, double 1.000000e+00>
-; CHECK-IO-NEXT:    [[Z:%.*]] = fptoui <2 x double> [[Y]] to <2 x i32>
-; CHECK-IO-NEXT:    [[U:%.*]] = ashr <2 x i32> [[Z]], <i32 3, i32 3>
-; CHECK-IO-NEXT:    ret <2 x i32> [[U]]
 ;
   %a2 = add <2 x i32> %a, <i32 1, i32 1>
   %a3 = and <2 x i32> %a2, <i32 2, i32 2>
@@ -123,17 +88,10 @@ define <2 x i32> @test_conversion(<2 x i32> %a) {
 define <2 x i1> @test_assumption_invalidation(<2 x i1> %b, <2 x i8> %x) {
 ; CHECK-LABEL: @test_assumption_invalidation(
 ; CHECK-NEXT:    [[LITTLE_NUMBER:%.*]] = zext <2 x i1> [[B:%.*]] to <2 x i8>
-; CHECK-NEXT:    [[SUB:%.*]] = sub <2 x i8> zeroinitializer, [[LITTLE_NUMBER]]
+; CHECK-NEXT:    [[BIG_NUMBER:%.*]] = shl <2 x i8> zeroinitializer, <i8 1, i8 1>
+; CHECK-NEXT:    [[SUB:%.*]] = sub <2 x i8> [[BIG_NUMBER]], [[LITTLE_NUMBER]]
 ; CHECK-NEXT:    [[TRUNC:%.*]] = trunc <2 x i8> [[SUB]] to <2 x i1>
 ; CHECK-NEXT:    ret <2 x i1> [[TRUNC]]
-;
-; CHECK-IO-LABEL: @test_assumption_invalidation(
-; CHECK-IO-NEXT:    [[SETBIT:%.*]] = or <2 x i8> [[X:%.*]], <i8 64, i8 64>
-; CHECK-IO-NEXT:    [[LITTLE_NUMBER:%.*]] = zext <2 x i1> [[B:%.*]] to <2 x i8>
-; CHECK-IO-NEXT:    [[BIG_NUMBER:%.*]] = shl <2 x i8> [[SETBIT]], <i8 1, i8 1>
-; CHECK-IO-NEXT:    [[SUB:%.*]] = sub nuw <2 x i8> [[BIG_NUMBER]], [[LITTLE_NUMBER]]
-; CHECK-IO-NEXT:    [[TRUNC:%.*]] = trunc <2 x i8> [[SUB]] to <2 x i1>
-; CHECK-IO-NEXT:    ret <2 x i1> [[TRUNC]]
 ;
   %setbit = or <2 x i8> %x, <i8 64, i8 64>
   %little_number = zext <2 x i1> %b to <2 x i8>
diff --git a/test/Transforms/CanonicalizeAliases/canonicalize.ll b/test/Transforms/CanonicalizeAliases/canonicalize.ll
new file mode 100644
index 0000000000000000000000000000000000000000..e762fc55953e248064fff6e2f132e55b9e394419
--- /dev/null
+++ b/test/Transforms/CanonicalizeAliases/canonicalize.ll
@@ -0,0 +1,37 @@
+; RUN: opt -S -canonicalize-aliases < %s | FileCheck %s
+; RUN: opt -prepare-for-thinlto -O0 -module-summary -o - < %s | llvm-dis -o - | FileCheck %s
+; RUN: opt -S -passes=canonicalize-aliases < %s | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; CHECK-DAG: @analias = alias void (), void ()* @aliasee
+; CHECK-DAG: @anotheralias = alias void (), void ()* @aliasee
+; CHECK-DAG: define void @aliasee()
+
+@analias = alias void (), void ()* @anotheralias
+@anotheralias = alias void (), bitcast (void ()* @aliasee to void ()*)
+
+; Function Attrs: nounwind uwtable
+define void @aliasee() #0 {
+entry:
+    ret void
+}
+
+%struct.S1 = type { i32, i32, i32 }
+
+; CHECK-DAG: @S = global %struct.S1 { i32 31, i32 32, i32 33 }
+; CHECK-DAG: @Salias = alias i32, getelementptr inbounds (%struct.S1, %struct.S1* @S, i32 0, i32 1)
+; CHECK-DAG: @Salias2 = alias i32, getelementptr inbounds (%struct.S1, %struct.S1* @S, i32 0, i32 1)
+; CHECK-DAG: @Salias3 = alias i32, getelementptr inbounds (%struct.S1, %struct.S1* @S, i32 0, i32 1)
+
+@S = global %struct.S1 { i32 31, i32 32, i32 33 }, align 4
+@Salias = alias i32, getelementptr inbounds (%struct.S1, %struct.S1* @S, i32 0, i32 1)
+@Salias2 = alias i32, i32* @Salias
+@Salias3 = alias i32, i32* @Salias2
+
+; CHECK-DAG: @Salias4 = alias %struct.S1, %struct.S1* @S
+; CHECK-DAG: @Salias5 = alias i32, getelementptr inbounds (%struct.S1, %struct.S1* @S, i32 0, i32 1)
+
+@Salias4 = alias %struct.S1, %struct.S1* @S
+@Salias5 = alias i32, getelementptr inbounds (%struct.S1, %struct.S1* @Salias4, i32 0, i32 1)
diff --git a/test/Transforms/CodeExtractor/PartialInlineAlloca4.ll b/test/Transforms/CodeExtractor/PartialInlineAlloca4.ll
index 6bb38d44f466c42ace36dbb10e0a78c2b6a566ed..04789eaad29eba9a04226241414b09554b6e8894 100644
--- a/test/Transforms/CodeExtractor/PartialInlineAlloca4.ll
+++ b/test/Transforms/CodeExtractor/PartialInlineAlloca4.ll
@@ -6,10 +6,14 @@
 
 @g = external local_unnamed_addr global i32, align 4
 
+; CHECK-LABEL: define{{.*}}@caller(
+; CHECK: call void @llvm.lifetime.start.p0i8(i64 -1, i8* %tmp.i)
+; CHECK-NEXT: call void @callee_unknown_use1.{{.*}}(i8* %tmp.i
+; CHECK-NEXT: call void @llvm.lifetime.end.p0i8(i64 -1, i8* %tmp.i)
+
 define i32 @callee_unknown_use1(i32 %arg) local_unnamed_addr #0 {
 ; CHECK-LABEL:define{{.*}}@callee_unknown_use1.{{[0-9]}}
 ; CHECK-NOT: alloca
-; CHECK: call void @llvm.lifetime
 bb:
   %tmp = alloca  i8, align 4
   %tmp2 = load i32, i32* @g, align 4, !tbaa !2
diff --git a/test/Transforms/CodeExtractor/PartialInlineAlloca5.ll b/test/Transforms/CodeExtractor/PartialInlineAlloca5.ll
index 9c53496e1ceac7243970df8215fbc11b3eb08397..0bde58fbccdb06fa5e15bd3e8b9ff1cf818fdee3 100644
--- a/test/Transforms/CodeExtractor/PartialInlineAlloca5.ll
+++ b/test/Transforms/CodeExtractor/PartialInlineAlloca5.ll
@@ -9,7 +9,6 @@
 define i32 @callee_unknown_use2(i32 %arg) local_unnamed_addr #0 {
 ; CHECK-LABEL:define{{.*}}@callee_unknown_use2.{{[0-9]}}
 ; CHECK-NOT: alloca
-; CHECK: call void @llvm.lifetime
 bb:
   %tmp = alloca i32, align 4
   %tmp1 = bitcast i32* %tmp to i8*
diff --git a/test/Transforms/CodeExtractor/PartialInlineDebug.ll b/test/Transforms/CodeExtractor/PartialInlineDebug.ll
index c0bc66db0ee65a94d96e7179f3ae1791d3ae2be5..c68a070a41dd6131aa533f97ea55ff4d17609bba 100644
--- a/test/Transforms/CodeExtractor/PartialInlineDebug.ll
+++ b/test/Transforms/CodeExtractor/PartialInlineDebug.ll
@@ -23,7 +23,8 @@ if.end:                                           ; preds = %if.then, %entry
 
 ; CHECK-LABEL: @caller
 ; CHECK: codeRepl.i:
-; CHECK-NEXT: call void @callee.2.if.then(i32 %v, i32* %mul.loc.i), !dbg ![[DBG2:[0-9]+]]
+; CHECK-NOT: br label
+; CHECK: call void @callee.2.if.then(i32 %v, i32* %mul.loc.i), !dbg ![[DBG2:[0-9]+]]
 define i32 @caller(i32 %v) !dbg !8 {
 entry:
   %call = call i32 @callee(i32 %v), !dbg !14
@@ -53,7 +54,8 @@ if.end:
 
 ; CHECK-LABEL: @caller2
 ; CHECK: codeRepl.i:
-; CHECK-NEXT: call void @callee2.1.if.then(i32 %v, i32* %sub.loc.i), !dbg ![[DBG4:[0-9]+]]
+; CHECK-NOT: br label
+; CHECK: call void @callee2.1.if.then(i32 %v, i32* %sub.loc.i), !dbg ![[DBG4:[0-9]+]]
 define i32 @caller2(i32 %v) !dbg !21 {
 entry:
   %call = call i32 @callee2(i32 %v), !dbg !22
diff --git a/test/Transforms/CodeExtractor/PartialInlineInvokeProducesOutVal.ll b/test/Transforms/CodeExtractor/PartialInlineInvokeProducesOutVal.ll
index bc6f780c5a815bb72c4b46d3fd64010c1328c2ef..2e0fbf6073ea7b28feaef2b2cee93e1690f18ef2 100644
--- a/test/Transforms/CodeExtractor/PartialInlineInvokeProducesOutVal.ll
+++ b/test/Transforms/CodeExtractor/PartialInlineInvokeProducesOutVal.ll
@@ -26,7 +26,11 @@ bb5:                                              ; preds = %bb4, %bb1, %bb
 ; CHECK-LABEL: bb:
 ; CHECK-NEXT:  [[CALL26LOC:%.*]] = alloca i8*
 ; CHECK-LABEL: codeRepl.i:
+; CHECK-NEXT:   %lt.cast.i = bitcast i8** [[CALL26LOC]] to i8*
+; CHECK-NEXT:   call void @llvm.lifetime.start.p0i8(i64 -1, i8* %lt.cast.i)
 ; CHECK-NEXT:   call void @bar.1.bb1(i8** [[CALL26LOC]])
+; CHECK-NEXT:   %call26.reload.i = load i8*, i8** [[CALL26LOC]]
+; CHECK-NEXT:   call void @llvm.lifetime.end.p0i8(i64 -1, i8* %lt.cast.i)
 define i8* @dummy_caller(i32 %arg) {
 bb:
   %tmp = tail call i8* @bar(i32 %arg)
diff --git a/test/Transforms/CodeExtractor/PartialInlineVarArgsDebug.ll b/test/Transforms/CodeExtractor/PartialInlineVarArgsDebug.ll
index 02f695d3662605fef61b25e897a26090c4d95704..d19de842b61ee76c1f30bfc48ae68439c499aa36 100644
--- a/test/Transforms/CodeExtractor/PartialInlineVarArgsDebug.ll
+++ b/test/Transforms/CodeExtractor/PartialInlineVarArgsDebug.ll
@@ -19,7 +19,8 @@ if.end:                                           ; preds = %if.then, %entry
 
 ; CHECK-LABEL: @caller
 ; CHECK: codeRepl.i:
-; CHECK-NEXT: call void (i32, i32*, ...) @callee.1.if.then(i32 %v, i32* %mul.loc.i, i32 99), !dbg ![[DBG2:[0-9]+]]
+; CHECK-NOT: br label
+; CHECK: call void (i32, i32*, ...) @callee.1.if.then(i32 %v, i32* %mul.loc.i, i32 99), !dbg ![[DBG2:[0-9]+]]
 define i32 @caller(i32 %v) !dbg !8 {
 entry:
   %call = call i32 (i32, ...) @callee(i32 %v, i32 99), !dbg !14
diff --git a/test/Transforms/CodeExtractor/extract-assume.ll b/test/Transforms/CodeExtractor/extract-assume.ll
new file mode 100644
index 0000000000000000000000000000000000000000..b79c6a6913753be323987377194acc05230df826
--- /dev/null
+++ b/test/Transforms/CodeExtractor/extract-assume.ll
@@ -0,0 +1,29 @@
+; RUN: opt -passes="function(slp-vectorizer),module(hotcoldsplit),function(slp-vectorizer,print<assumptions>)" -disable-output %s 2>&1 | FileCheck %s
+;
+; Make sure this compiles. Check that function assumption cache is refreshed
+; after extracting blocks with assume calls from the function.
+
+; CHECK:      Cached assumptions for function: fun
+; CHECK-NEXT: Cached assumptions for function: fun.cold
+; CHECK-NEXT:   %cmp = icmp uge i32 %x, 64
+
+declare void @fun2(i32) #0
+
+define void @fun(i32 %x) {
+entry:
+  br i1 undef, label %if.then, label %if.else
+
+if.then:
+  ret void
+
+if.else:
+  %cmp = icmp uge i32 %x, 64
+  call void @llvm.assume(i1 %cmp)
+  call void @fun2(i32 %x)
+  unreachable
+}
+
+declare void @llvm.assume(i1) #1
+
+attributes #0 = { alwaysinline }
+attributes #1 = { nounwind }
diff --git a/test/Transforms/CodeGenPrepare/AArch64/large-offset-gep.ll b/test/Transforms/CodeGenPrepare/AArch64/large-offset-gep.ll
index 5cc00b75962e1ee4e6a06466f724eaad2b048f8a..005ea37288ceb362299609e24c44f86e11ffa2fe 100644
--- a/test/Transforms/CodeGenPrepare/AArch64/large-offset-gep.ll
+++ b/test/Transforms/CodeGenPrepare/AArch64/large-offset-gep.ll
@@ -88,18 +88,23 @@ while_end:
 }
 
 declare %struct_type* @foo()
+declare void @foo2()
 
 define void @test4(i32 %n) personality i32 (...)* @__FrameHandler {
 ; CHECK-LABEL: test4
 entry:
-  %struct = invoke %struct_type* @foo() to label %while_cond unwind label %cleanup
+  br label %while_cond
 
 while_cond:
   %phi = phi i32 [ 0, %entry ], [ %i, %while_body ]
+  %struct = invoke %struct_type* @foo() to label %while_cond_x unwind label %cleanup
+
+while_cond_x:
 ; CHECK:     mov     w{{[0-9]+}}, #40000
 ; CHECK-NOT: mov     w{{[0-9]+}}, #40004
   %gep0 = getelementptr %struct_type, %struct_type* %struct, i64 0, i32 1
   %gep1 = getelementptr %struct_type, %struct_type* %struct, i64 0, i32 2
+  store i32 0, i32* %gep0
   %cmp = icmp slt i32 %phi, %n
   br i1 %cmp, label %while_body, label %while_end
 
@@ -114,8 +119,10 @@ while_end:
   ret void
 
 cleanup:
-  landingpad { i8*, i32 } cleanup
-  unreachable
+  %x10 = landingpad { i8*, i32 }
+          cleanup
+  call void @foo2()
+  resume { i8*, i32 } %x10
 }
 
 declare i32 @__FrameHandler(...)
diff --git a/test/Transforms/CodeGenPrepare/AArch64/sink-free-instructions.ll b/test/Transforms/CodeGenPrepare/AArch64/sink-free-instructions.ll
new file mode 100644
index 0000000000000000000000000000000000000000..394fd6cc23aae2d0a098e5d121f80923f8acfcdb
--- /dev/null
+++ b/test/Transforms/CodeGenPrepare/AArch64/sink-free-instructions.ll
@@ -0,0 +1,236 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -codegenprepare -S | FileCheck %s
+
+target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64-unknown"
+
+define <8 x i16> @sink_zext(<8 x i8> %a, <8 x i8> %b, i1 %c) {
+; CHECK-LABEL: @sink_zext(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[C:%.*]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[ZB_1:%.*]] = zext <8 x i8> [[B:%.*]] to <8 x i16>
+; CHECK-NEXT:    [[TMP0:%.*]] = zext <8 x i8> [[A:%.*]] to <8 x i16>
+; CHECK-NEXT:    [[RES_1:%.*]] = add <8 x i16> [[TMP0]], [[ZB_1]]
+; CHECK-NEXT:    ret <8 x i16> [[RES_1]]
+; CHECK:       if.else:
+; CHECK-NEXT:    [[ZB_2:%.*]] = zext <8 x i8> [[B]] to <8 x i16>
+; CHECK-NEXT:    [[TMP1:%.*]] = zext <8 x i8> [[A]] to <8 x i16>
+; CHECK-NEXT:    [[RES_2:%.*]] = sub <8 x i16> [[TMP1]], [[ZB_2]]
+; CHECK-NEXT:    ret <8 x i16> [[RES_2]]
+;
+entry:
+  %za = zext <8 x i8> %a to <8 x i16>
+  br i1 %c, label %if.then, label %if.else
+
+if.then:
+  %zb.1 = zext <8 x i8> %b to <8 x i16>
+  %res.1 = add <8 x i16> %za, %zb.1
+  ret <8 x i16> %res.1
+
+if.else:
+  %zb.2 = zext <8 x i8> %b to <8 x i16>
+  %res.2 = sub <8 x i16> %za, %zb.2
+  ret <8 x i16> %res.2
+}
+
+define <8 x i16> @sink_sext(<8 x i8> %a, <8 x i8> %b, i1 %c) {
+; CHECK-LABEL: @sink_sext(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[C:%.*]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[ZB_1:%.*]] = sext <8 x i8> [[B:%.*]] to <8 x i16>
+; CHECK-NEXT:    [[TMP0:%.*]] = sext <8 x i8> [[A:%.*]] to <8 x i16>
+; CHECK-NEXT:    [[RES_1:%.*]] = add <8 x i16> [[TMP0]], [[ZB_1]]
+; CHECK-NEXT:    ret <8 x i16> [[RES_1]]
+; CHECK:       if.else:
+; CHECK-NEXT:    [[ZB_2:%.*]] = sext <8 x i8> [[B]] to <8 x i16>
+; CHECK-NEXT:    [[TMP1:%.*]] = sext <8 x i8> [[A]] to <8 x i16>
+; CHECK-NEXT:    [[RES_2:%.*]] = sub <8 x i16> [[TMP1]], [[ZB_2]]
+; CHECK-NEXT:    ret <8 x i16> [[RES_2]]
+;
+entry:
+  %za = sext <8 x i8> %a to <8 x i16>
+  br i1 %c, label %if.then, label %if.else
+
+if.then:
+  %zb.1 = sext <8 x i8> %b to <8 x i16>
+  %res.1 = add <8 x i16> %za, %zb.1
+  ret <8 x i16> %res.1
+
+if.else:
+  %zb.2 = sext <8 x i8> %b to <8 x i16>
+  %res.2 = sub <8 x i16> %za, %zb.2
+  ret <8 x i16> %res.2
+}
+
+define <8 x i16> @do_not_sink_nonfree_zext(<8 x i8> %a, <8 x i8> %b, i1 %c) {
+; CHECK-LABEL: @do_not_sink_nonfree_zext(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[C:%.*]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[ZB_1:%.*]] = sext <8 x i8> [[B:%.*]] to <8 x i16>
+; CHECK-NEXT:    [[TMP0:%.*]] = sext <8 x i8> [[A:%.*]] to <8 x i16>
+; CHECK-NEXT:    [[RES_1:%.*]] = add <8 x i16> [[TMP0]], [[ZB_1]]
+; CHECK-NEXT:    ret <8 x i16> [[RES_1]]
+; CHECK:       if.else:
+; CHECK-NEXT:    [[ZB_2:%.*]] = sext <8 x i8> [[B]] to <8 x i16>
+; CHECK-NEXT:    ret <8 x i16> [[ZB_2]]
+;
+entry:
+  %za = sext <8 x i8> %a to <8 x i16>
+  br i1 %c, label %if.then, label %if.else
+
+if.then:
+  %zb.1 = sext <8 x i8> %b to <8 x i16>
+  %res.1 = add <8 x i16> %za, %zb.1
+  ret <8 x i16> %res.1
+
+if.else:
+  %zb.2 = sext <8 x i8> %b to <8 x i16>
+  ret <8 x i16> %zb.2
+}
+
+define <8 x i16> @do_not_sink_nonfree_sext(<8 x i8> %a, <8 x i8> %b, i1 %c) {
+; CHECK-LABEL: @do_not_sink_nonfree_sext(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[C:%.*]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[ZB_1:%.*]] = sext <8 x i8> [[B:%.*]] to <8 x i16>
+; CHECK-NEXT:    [[TMP0:%.*]] = sext <8 x i8> [[A:%.*]] to <8 x i16>
+; CHECK-NEXT:    [[RES_1:%.*]] = add <8 x i16> [[TMP0]], [[ZB_1]]
+; CHECK-NEXT:    ret <8 x i16> [[RES_1]]
+; CHECK:       if.else:
+; CHECK-NEXT:    [[ZB_2:%.*]] = sext <8 x i8> [[B]] to <8 x i16>
+; CHECK-NEXT:    ret <8 x i16> [[ZB_2]]
+;
+entry:
+  %za = sext <8 x i8> %a to <8 x i16>
+  br i1 %c, label %if.then, label %if.else
+
+if.then:
+  %zb.1 = sext <8 x i8> %b to <8 x i16>
+  %res.1 = add <8 x i16> %za, %zb.1
+  ret <8 x i16> %res.1
+
+if.else:
+  %zb.2 = sext <8 x i8> %b to <8 x i16>
+  ret <8 x i16> %zb.2
+}
+
+; The masks used are suitable for umull, sink shufflevector to users.
+define <8 x i16> @sink_shufflevector_umull(<16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: @sink_shufflevector_umull(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 undef, label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[S2:%.*]] = shufflevector <16 x i8> [[B:%.*]], <16 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[TMP0:%.*]] = shufflevector <16 x i8> [[A:%.*]], <16 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[VMULL0:%.*]] = tail call <8 x i16> @llvm.aarch64.neon.umull.v8i16(<8 x i8> [[TMP0]], <8 x i8> [[S2]])
+; CHECK-NEXT:    ret <8 x i16> [[VMULL0]]
+; CHECK:       if.else:
+; CHECK-NEXT:    [[S4:%.*]] = shufflevector <16 x i8> [[B]], <16 x i8> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <16 x i8> [[A]], <16 x i8> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    [[VMULL1:%.*]] = tail call <8 x i16> @llvm.aarch64.neon.umull.v8i16(<8 x i8> [[TMP1]], <8 x i8> [[S4]])
+; CHECK-NEXT:    ret <8 x i16> [[VMULL1]]
+;
+entry:
+  %s1 = shufflevector <16 x i8> %a, <16 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %s3 = shufflevector <16 x i8> %a, <16 x i8> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  br i1 undef, label %if.then, label %if.else
+
+if.then:
+  %s2 = shufflevector <16 x i8> %b, <16 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %vmull0 = tail call <8 x i16> @llvm.aarch64.neon.umull.v8i16(<8 x i8> %s1, <8 x i8> %s2) #3
+  ret <8 x i16> %vmull0
+
+if.else:
+  %s4 = shufflevector <16 x i8> %b, <16 x i8> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  %vmull1 = tail call <8 x i16> @llvm.aarch64.neon.umull.v8i16(<8 x i8> %s3, <8 x i8> %s4) #3
+  ret <8 x i16> %vmull1
+}
+
+; Both exts and their shufflevector operands can be sunk.
+define <8 x i16> @sink_shufflevector_ext_subadd(<16 x i8> %a, <16 x i8> %b) {
+entry:
+  %s1 = shufflevector <16 x i8> %a, <16 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %z1 = zext <8 x i8> %s1 to <8 x i16>
+  %s3 = shufflevector <16 x i8> %a, <16 x i8> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  %z3 = sext <8 x i8> %s3 to <8 x i16>
+  br i1 undef, label %if.then, label %if.else
+
+if.then:
+  %s2 = shufflevector <16 x i8> %b, <16 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %z2 = zext <8 x i8> %s2 to <8 x i16>
+  %res1 = add <8 x i16> %z1, %z2
+  ret <8 x i16> %res1
+
+if.else:
+  %s4 = shufflevector <16 x i8> %b, <16 x i8> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  %z4 = sext <8 x i8> %s4 to <8 x i16>
+  %res2 = sub <8 x i16> %z3, %z4
+  ret <8 x i16> %res2
+}
+
+
+declare void @user1(<8 x i16>)
+
+; Both exts and their shufflevector operands can be sunk.
+define <8 x i16> @sink_shufflevector_ext_subadd_multiuse(<16 x i8> %a, <16 x i8> %b) {
+entry:
+  %s1 = shufflevector <16 x i8> %a, <16 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %z1 = zext <8 x i8> %s1 to <8 x i16>
+  %s3 = shufflevector <16 x i8> %a, <16 x i8> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  %z3 = sext <8 x i8> %s3 to <8 x i16>
+  call void @user1(<8 x i16> %z3)
+  br i1 undef, label %if.then, label %if.else
+
+if.then:
+  %s2 = shufflevector <16 x i8> %b, <16 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %z2 = zext <8 x i8> %s2 to <8 x i16>
+  %res1 = add <8 x i16> %z1, %z2
+  ret <8 x i16> %res1
+
+if.else:
+  %s4 = shufflevector <16 x i8> %b, <16 x i8> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  %z4 = sext <8 x i8> %s4 to <8 x i16>
+  %res2 = sub <8 x i16> %z3, %z4
+  ret <8 x i16> %res2
+}
+
+
+; The masks used are not suitable for umull, do not sink.
+define <8 x i16> @no_sink_shufflevector_umull(<16 x i8> %a, <16 x i8> %b) {
+; CHECK-LABEL: @no_sink_shufflevector_umull(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[S1:%.*]] = shufflevector <16 x i8> [[A:%.*]], <16 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 1, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[S3:%.*]] = shufflevector <16 x i8> [[A]], <16 x i8> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    br i1 undef, label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[S2:%.*]] = shufflevector <16 x i8> [[B:%.*]], <16 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT:    [[VMULL0:%.*]] = tail call <8 x i16> @llvm.aarch64.neon.umull.v8i16(<8 x i8> [[S1]], <8 x i8> [[S2]])
+; CHECK-NEXT:    ret <8 x i16> [[VMULL0]]
+; CHECK:       if.else:
+; CHECK-NEXT:    [[S4:%.*]] = shufflevector <16 x i8> [[B]], <16 x i8> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 10, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT:    [[VMULL1:%.*]] = tail call <8 x i16> @llvm.aarch64.neon.umull.v8i16(<8 x i8> [[S3]], <8 x i8> [[S4]])
+; CHECK-NEXT:    ret <8 x i16> [[VMULL1]]
+;
+entry:
+  %s1 = shufflevector <16 x i8> %a, <16 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 1, i32 5, i32 6, i32 7>
+  %s3 = shufflevector <16 x i8> %a, <16 x i8> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  br i1 undef, label %if.then, label %if.else
+
+if.then:
+  %s2 = shufflevector <16 x i8> %b, <16 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %vmull0 = tail call <8 x i16> @llvm.aarch64.neon.umull.v8i16(<8 x i8> %s1, <8 x i8> %s2) #3
+  ret <8 x i16> %vmull0
+
+if.else:
+  %s4 = shufflevector <16 x i8> %b, <16 x i8> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 10, i32 12, i32 13, i32 14, i32 15>
+  %vmull1 = tail call <8 x i16> @llvm.aarch64.neon.umull.v8i16(<8 x i8> %s3, <8 x i8> %s4) #3
+  ret <8 x i16> %vmull1
+}
+
+
+; Function Attrs: nounwind readnone
+declare <8 x i16> @llvm.aarch64.neon.umull.v8i16(<8 x i8>, <8 x i8>) #2
diff --git a/test/Transforms/CodeGenPrepare/ARM/bitreverse-recognize.ll b/test/Transforms/CodeGenPrepare/ARM/bitreverse-recognize.ll
index 36440da21626ba23ec5ae46f9519aff26d5cf11e..f5644e4ad312a9dcc3020b950d05952629e09104 100644
--- a/test/Transforms/CodeGenPrepare/ARM/bitreverse-recognize.ll
+++ b/test/Transforms/CodeGenPrepare/ARM/bitreverse-recognize.ll
@@ -32,6 +32,6 @@ attributes #0 = { norecurse nounwind readnone "disable-tail-calls"="false" "less
 
 !0 = !{i32 1, !"wchar_size", i32 4}
 !1 = !{i32 1, !"min_enum_size", i32 4}
-!2 = !{!"clang version 3.8.0 (http://llvm.org/git/clang.git b7441a0f42c43a8eea9e3e706be187252db747fa)"}
+!2 = !{!"clang version 3.8.0"}
 !3 = distinct !{!3, !4}
 !4 = !{!"llvm.loop.unroll.full"}
diff --git a/test/Transforms/CodeGenPrepare/X86/overflow-intrinsics.ll b/test/Transforms/CodeGenPrepare/X86/overflow-intrinsics.ll
new file mode 100644
index 0000000000000000000000000000000000000000..f8ef77556909eedf9805561a043d55126d63fbb7
--- /dev/null
+++ b/test/Transforms/CodeGenPrepare/X86/overflow-intrinsics.ll
@@ -0,0 +1,346 @@
+; RUN: opt -codegenprepare -S < %s | FileCheck %s
+; RUN: opt -enable-debugify -codegenprepare -S < %s 2>&1 | FileCheck %s -check-prefix=DEBUG
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
+target triple = "x86_64-apple-darwin10.0.0"
+
+define i64 @uaddo1(i64 %a, i64 %b) nounwind ssp {
+; CHECK-LABEL: @uaddo1(
+; CHECK-NEXT:    [[TMP1:%.*]] = call { i64, i1 } @llvm.uadd.with.overflow.i64(i64 [[B:%.*]], i64 [[A:%.*]])
+; CHECK-NEXT:    [[MATH:%.*]] = extractvalue { i64, i1 } [[TMP1]], 0
+; CHECK-NEXT:    [[OV:%.*]] = extractvalue { i64, i1 } [[TMP1]], 1
+; CHECK-NEXT:    [[Q:%.*]] = select i1 [[OV]], i64 [[B]], i64 42
+; CHECK-NEXT:    ret i64 [[Q]]
+;
+  %add = add i64 %b, %a
+  %cmp = icmp ult i64 %add, %a
+  %Q = select i1 %cmp, i64 %b, i64 42
+  ret i64 %Q
+}
+
+define i64 @uaddo2(i64 %a, i64 %b) nounwind ssp {
+; CHECK-LABEL: @uaddo2(
+; CHECK-NEXT:    [[TMP1:%.*]] = call { i64, i1 } @llvm.uadd.with.overflow.i64(i64 [[B:%.*]], i64 [[A:%.*]])
+; CHECK-NEXT:    [[MATH:%.*]] = extractvalue { i64, i1 } [[TMP1]], 0
+; CHECK-NEXT:    [[OV:%.*]] = extractvalue { i64, i1 } [[TMP1]], 1
+; CHECK-NEXT:    [[Q:%.*]] = select i1 [[OV]], i64 [[B]], i64 42
+; CHECK-NEXT:    ret i64 [[Q]]
+;
+  %add = add i64 %b, %a
+  %cmp = icmp ult i64 %add, %b
+  %Q = select i1 %cmp, i64 %b, i64 42
+  ret i64 %Q
+}
+
+define i64 @uaddo3(i64 %a, i64 %b) nounwind ssp {
+; CHECK-LABEL: @uaddo3(
+; CHECK-NEXT:    [[TMP1:%.*]] = call { i64, i1 } @llvm.uadd.with.overflow.i64(i64 [[B:%.*]], i64 [[A:%.*]])
+; CHECK-NEXT:    [[MATH:%.*]] = extractvalue { i64, i1 } [[TMP1]], 0
+; CHECK-NEXT:    [[OV:%.*]] = extractvalue { i64, i1 } [[TMP1]], 1
+; CHECK-NEXT:    [[Q:%.*]] = select i1 [[OV]], i64 [[B]], i64 42
+; CHECK-NEXT:    ret i64 [[Q]]
+;
+  %add = add i64 %b, %a
+  %cmp = icmp ugt i64 %b, %add
+  %Q = select i1 %cmp, i64 %b, i64 42
+  ret i64 %Q
+}
+
+define i64 @uaddo4(i64 %a, i64 %b, i1 %c) nounwind ssp {
+; CHECK-LABEL: @uaddo4(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[C:%.*]], label [[NEXT:%.*]], label [[EXIT:%.*]]
+; CHECK:       next:
+; CHECK-NEXT:    [[TMP0:%.*]] = call { i64, i1 } @llvm.uadd.with.overflow.i64(i64 [[B:%.*]], i64 [[A:%.*]])
+; CHECK-NEXT:    [[MATH:%.*]] = extractvalue { i64, i1 } [[TMP0]], 0
+; CHECK-NEXT:    [[OV:%.*]] = extractvalue { i64, i1 } [[TMP0]], 1
+; CHECK-NEXT:    [[Q:%.*]] = select i1 [[OV]], i64 [[B]], i64 42
+; CHECK-NEXT:    ret i64 [[Q]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret i64 0
+;
+entry:
+  %add = add i64 %b, %a
+  %cmp = icmp ugt i64 %b, %add
+  br i1 %c, label %next, label %exit
+
+next:
+  %Q = select i1 %cmp, i64 %b, i64 42
+  ret i64 %Q
+
+exit:
+  ret i64 0
+}
+
+define i64 @uaddo5(i64 %a, i64 %b, i64* %ptr, i1 %c) nounwind ssp {
+; CHECK-LABEL: @uaddo5(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[ADD:%.*]] = add i64 [[B:%.*]], [[A:%.*]]
+; CHECK-NEXT:    store i64 [[ADD]], i64* [[PTR:%.*]]
+; CHECK-NEXT:    br i1 [[C:%.*]], label [[NEXT:%.*]], label [[EXIT:%.*]]
+; CHECK:       next:
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp ugt i64 [[B]], [[ADD]]
+; CHECK-NEXT:    [[Q:%.*]] = select i1 [[TMP0]], i64 [[B]], i64 42
+; CHECK-NEXT:    ret i64 [[Q]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret i64 0
+;
+entry:
+  %add = add i64 %b, %a
+  store i64 %add, i64* %ptr
+  %cmp = icmp ugt i64 %b, %add
+  br i1 %c, label %next, label %exit
+
+next:
+  %Q = select i1 %cmp, i64 %b, i64 42
+  ret i64 %Q
+
+exit:
+  ret i64 0
+}
+
+; When adding 1, the general pattern for add-overflow may be different due to icmp canonicalization.
+; PR31754: https://bugs.llvm.org/show_bug.cgi?id=31754
+
+define i1 @uaddo_i64_increment(i64 %x, i64* %p) {
+; CHECK-LABEL: @uaddo_i64_increment(
+; CHECK-NEXT:    [[TMP1:%.*]] = call { i64, i1 } @llvm.uadd.with.overflow.i64(i64 [[X:%.*]], i64 1)
+; CHECK-NEXT:    [[MATH:%.*]] = extractvalue { i64, i1 } [[TMP1]], 0
+; CHECK-NEXT:    [[OV1:%.*]] = extractvalue { i64, i1 } [[TMP1]], 1
+; CHECK-NEXT:    store i64 [[MATH]], i64* [[P:%.*]]
+; CHECK-NEXT:    ret i1 [[OV1]]
+;
+  %a = add i64 %x, 1
+  %ov = icmp eq i64 %a, 0
+  store i64 %a, i64* %p
+  ret i1 %ov
+}
+
+define i1 @uaddo_i8_increment_noncanonical_1(i8 %x, i8* %p) {
+; CHECK-LABEL: @uaddo_i8_increment_noncanonical_1(
+; CHECK-NEXT:    [[TMP1:%.*]] = call { i8, i1 } @llvm.uadd.with.overflow.i8(i8 1, i8 [[X:%.*]])
+; CHECK-NEXT:    [[MATH:%.*]] = extractvalue { i8, i1 } [[TMP1]], 0
+; CHECK-NEXT:    [[OV1:%.*]] = extractvalue { i8, i1 } [[TMP1]], 1
+; CHECK-NEXT:    store i8 [[MATH]], i8* [[P:%.*]]
+; CHECK-NEXT:    ret i1 [[OV1]]
+;
+  %a = add i8 1, %x        ; commute
+  %ov = icmp eq i8 %a, 0
+  store i8 %a, i8* %p
+  ret i1 %ov
+}
+
+define i1 @uaddo_i32_increment_noncanonical_2(i32 %x, i32* %p) {
+; CHECK-LABEL: @uaddo_i32_increment_noncanonical_2(
+; CHECK-NEXT:    [[TMP1:%.*]] = call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 [[X:%.*]], i32 1)
+; CHECK-NEXT:    [[MATH:%.*]] = extractvalue { i32, i1 } [[TMP1]], 0
+; CHECK-NEXT:    [[OV1:%.*]] = extractvalue { i32, i1 } [[TMP1]], 1
+; CHECK-NEXT:    store i32 [[MATH]], i32* [[P:%.*]]
+; CHECK-NEXT:    ret i1 [[OV1]]
+;
+  %a = add i32 %x, 1
+  %ov = icmp eq i32 0, %a   ; commute
+  store i32 %a, i32* %p
+  ret i1 %ov
+}
+
+define i1 @uaddo_i16_increment_noncanonical_3(i16 %x, i16* %p) {
+; CHECK-LABEL: @uaddo_i16_increment_noncanonical_3(
+; CHECK-NEXT:    [[TMP1:%.*]] = call { i16, i1 } @llvm.uadd.with.overflow.i16(i16 1, i16 [[X:%.*]])
+; CHECK-NEXT:    [[MATH:%.*]] = extractvalue { i16, i1 } [[TMP1]], 0
+; CHECK-NEXT:    [[OV1:%.*]] = extractvalue { i16, i1 } [[TMP1]], 1
+; CHECK-NEXT:    store i16 [[MATH]], i16* [[P:%.*]]
+; CHECK-NEXT:    ret i1 [[OV1]]
+;
+  %a = add i16 1, %x        ; commute
+  %ov = icmp eq i16 0, %a   ; commute
+  store i16 %a, i16* %p
+  ret i1 %ov
+}
+
+; No transform for illegal types.
+
+define i1 @uaddo_i42_increment_illegal_type(i42 %x, i42* %p) {
+; CHECK-LABEL: @uaddo_i42_increment_illegal_type(
+; CHECK-NEXT:    [[A:%.*]] = add i42 [[X:%.*]], 1
+; CHECK-NEXT:    [[OV:%.*]] = icmp eq i42 [[A]], 0
+; CHECK-NEXT:    store i42 [[A]], i42* [[P:%.*]]
+; CHECK-NEXT:    ret i1 [[OV]]
+;
+  %a = add i42 %x, 1
+  %ov = icmp eq i42 %a, 0
+  store i42 %a, i42* %p
+  ret i1 %ov
+}
+
+define i1 @usubo_ult_i64(i64 %x, i64 %y, i64* %p) {
+; CHECK-LABEL: @usubo_ult_i64(
+; CHECK-NEXT:    [[S:%.*]] = sub i64 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    store i64 [[S]], i64* [[P:%.*]]
+; CHECK-NEXT:    [[OV:%.*]] = icmp ult i64 [[X]], [[Y]]
+; CHECK-NEXT:    ret i1 [[OV]]
+;
+  %s = sub i64 %x, %y
+  store i64 %s, i64* %p
+  %ov = icmp ult i64 %x, %y
+  ret i1 %ov
+}
+
+; Verify insertion point for single-BB. Toggle predicate.
+
+define i1 @usubo_ugt_i32(i32 %x, i32 %y, i32* %p) {
+; CHECK-LABEL: @usubo_ugt_i32(
+; CHECK-NEXT:    [[OV:%.*]] = icmp ugt i32 [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT:    [[S:%.*]] = sub i32 [[X]], [[Y]]
+; CHECK-NEXT:    store i32 [[S]], i32* [[P:%.*]]
+; CHECK-NEXT:    ret i1 [[OV]]
+;
+  %ov = icmp ugt i32 %y, %x
+  %s = sub i32 %x, %y
+  store i32 %s, i32* %p
+  ret i1 %ov
+}
+
+; Constant operand should match.
+
+define i1 @usubo_ugt_constant_op0_i8(i8 %x, i8* %p) {
+; CHECK-LABEL: @usubo_ugt_constant_op0_i8(
+; CHECK-NEXT:    [[S:%.*]] = sub i8 42, [[X:%.*]]
+; CHECK-NEXT:    [[OV:%.*]] = icmp ugt i8 [[X]], 42
+; CHECK-NEXT:    store i8 [[S]], i8* [[P:%.*]]
+; CHECK-NEXT:    ret i1 [[OV]]
+;
+  %s = sub i8 42, %x
+  %ov = icmp ugt i8 %x, 42
+  store i8 %s, i8* %p
+  ret i1 %ov
+}
+
+; Compare with constant operand 0 is canonicalized by commuting, but verify match for non-canonical form.
+
+define i1 @usubo_ult_constant_op0_i16(i16 %x, i16* %p) {
+; CHECK-LABEL: @usubo_ult_constant_op0_i16(
+; CHECK-NEXT:    [[S:%.*]] = sub i16 43, [[X:%.*]]
+; CHECK-NEXT:    [[OV:%.*]] = icmp ult i16 43, [[X]]
+; CHECK-NEXT:    store i16 [[S]], i16* [[P:%.*]]
+; CHECK-NEXT:    ret i1 [[OV]]
+;
+  %s = sub i16 43, %x
+  %ov = icmp ult i16 43, %x
+  store i16 %s, i16* %p
+  ret i1 %ov
+}
+
+; Subtract with constant operand 1 is canonicalized to add.
+
+define i1 @usubo_ult_constant_op1_i16(i16 %x, i16* %p) {
+; CHECK-LABEL: @usubo_ult_constant_op1_i16(
+; CHECK-NEXT:    [[S:%.*]] = add i16 [[X:%.*]], -44
+; CHECK-NEXT:    [[OV:%.*]] = icmp ult i16 [[X]], 44
+; CHECK-NEXT:    store i16 [[S]], i16* [[P:%.*]]
+; CHECK-NEXT:    ret i1 [[OV]]
+;
+  %s = add i16 %x, -44
+  %ov = icmp ult i16 %x, 44
+  store i16 %s, i16* %p
+  ret i1 %ov
+}
+
+define i1 @usubo_ugt_constant_op1_i8(i8 %x, i8* %p) {
+; CHECK-LABEL: @usubo_ugt_constant_op1_i8(
+; CHECK-NEXT:    [[OV:%.*]] = icmp ugt i8 45, [[X:%.*]]
+; CHECK-NEXT:    [[S:%.*]] = add i8 [[X]], -45
+; CHECK-NEXT:    store i8 [[S]], i8* [[P:%.*]]
+; CHECK-NEXT:    ret i1 [[OV]]
+;
+  %ov = icmp ugt i8 45, %x
+  %s = add i8 %x, -45
+  store i8 %s, i8* %p
+  ret i1 %ov
+}
+
+; Special-case: subtract 1 changes the compare predicate and constant.
+
+define i1 @usubo_eq_constant1_op1_i32(i32 %x, i32* %p) {
+; CHECK-LABEL: @usubo_eq_constant1_op1_i32(
+; CHECK-NEXT:    [[S:%.*]] = add i32 [[X:%.*]], -1
+; CHECK-NEXT:    [[OV:%.*]] = icmp eq i32 [[X]], 0
+; CHECK-NEXT:    store i32 [[S]], i32* [[P:%.*]]
+; CHECK-NEXT:    ret i1 [[OV]]
+;
+  %s = add i32 %x, -1
+  %ov = icmp eq i32 %x, 0
+  store i32 %s, i32* %p
+  ret i1 %ov
+}
+
+; Verify insertion point for multi-BB.
+
+declare void @call(i1)
+
+define i1 @usubo_ult_sub_dominates_i64(i64 %x, i64 %y, i64* %p, i1 %cond) {
+; CHECK-LABEL: @usubo_ult_sub_dominates_i64(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[COND:%.*]], label [[T:%.*]], label [[F:%.*]]
+; CHECK:       t:
+; CHECK-NEXT:    [[S:%.*]] = sub i64 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    store i64 [[S]], i64* [[P:%.*]]
+; CHECK-NEXT:    br i1 [[COND]], label [[END:%.*]], label [[F]]
+; CHECK:       f:
+; CHECK-NEXT:    ret i1 [[COND]]
+; CHECK:       end:
+; CHECK-NEXT:    [[OV:%.*]] = icmp ult i64 [[X]], [[Y]]
+; CHECK-NEXT:    ret i1 [[OV]]
+;
+entry:
+  br i1 %cond, label %t, label %f
+
+t:
+  %s = sub i64 %x, %y
+  store i64 %s, i64* %p
+  br i1 %cond, label %end, label %f
+
+f:
+  ret i1 %cond
+
+end:
+  %ov = icmp ult i64 %x, %y
+  ret i1 %ov
+}
+
+define i1 @usubo_ult_cmp_dominates_i64(i64 %x, i64 %y, i64* %p, i1 %cond) {
+; CHECK-LABEL: @usubo_ult_cmp_dominates_i64(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[COND:%.*]], label [[T:%.*]], label [[F:%.*]]
+; CHECK:       t:
+; CHECK-NEXT:    [[OV:%.*]] = icmp ult i64 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    call void @call(i1 [[OV]])
+; CHECK-NEXT:    br i1 [[OV]], label [[END:%.*]], label [[F]]
+; CHECK:       f:
+; CHECK-NEXT:    ret i1 [[COND]]
+; CHECK:       end:
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp ult i64 [[X]], [[Y]]
+; CHECK-NEXT:    [[S:%.*]] = sub i64 [[X]], [[Y]]
+; CHECK-NEXT:    store i64 [[S]], i64* [[P:%.*]]
+; CHECK-NEXT:    ret i1 [[TMP0]]
+;
+entry:
+  br i1 %cond, label %t, label %f
+
+t:
+  %ov = icmp ult i64 %x, %y
+  call void @call(i1 %ov)
+  br i1 %ov, label %end, label %f
+
+f:
+  ret i1 %cond
+
+end:
+  %s = sub i64 %x, %y
+  store i64 %s, i64* %p
+  ret i1 %ov
+}
+
+; Check that every instruction inserted by -codegenprepare has a debug location.
+; DEBUG: CheckModuleDebugify: PASS
+
diff --git a/test/Transforms/CodeGenPrepare/basic.ll b/test/Transforms/CodeGenPrepare/basic.ll
index 768209abcdf38f5f07c90bab6765f8b2d9ab33ad..1a58d61b6e9611b39856c1943a41d891205ceb24 100644
--- a/test/Transforms/CodeGenPrepare/basic.ll
+++ b/test/Transforms/CodeGenPrepare/basic.ll
@@ -9,7 +9,7 @@ target triple = "x86_64-apple-darwin10.0.0"
 ; rdar://8785296
 define i32 @test1(i8* %ptr) nounwind ssp noredzone align 2 {
 entry:
-  %0 = tail call i64 @llvm.objectsize.i64(i8* %ptr, i1 false, i1 false)
+  %0 = tail call i64 @llvm.objectsize.i64(i8* %ptr, i1 false, i1 false, i1 false)
   %1 = icmp ugt i64 %0, 3
   br i1 %1, label %T, label %trap
 
@@ -29,7 +29,7 @@ T:
 define i64 @test_objectsize_null_flag(i8* %ptr) {
 entry:
   ; CHECK: ret i64 -1
-  %0 = tail call i64 @llvm.objectsize.i64(i8* null, i1 false, i1 true)
+  %0 = tail call i64 @llvm.objectsize.i64(i8* null, i1 false, i1 true, i1 false)
   ret i64 %0
 }
 
@@ -37,7 +37,7 @@ entry:
 define i64 @test_objectsize_null_flag_min(i8* %ptr) {
 entry:
   ; CHECK: ret i64 0
-  %0 = tail call i64 @llvm.objectsize.i64(i8* null, i1 true, i1 true)
+  %0 = tail call i64 @llvm.objectsize.i64(i8* null, i1 true, i1 true, i1 false)
   ret i64 %0
 }
 
@@ -48,7 +48,7 @@ define i64 @test_objectsize_null_flag_noas0() {
 entry:
   ; CHECK: ret i64 -1
   %0 = tail call i64 @llvm.objectsize.i64.p1i8(i8 addrspace(1)* null, i1 false,
-                                               i1 true)
+                                               i1 true, i1 false)
   ret i64 %0
 }
 
@@ -57,7 +57,7 @@ define i64 @test_objectsize_null_flag_min_noas0() {
 entry:
   ; CHECK: ret i64 0
   %0 = tail call i64 @llvm.objectsize.i64.p1i8(i8 addrspace(1)* null, i1 true,
-                                               i1 true)
+                                               i1 true, i1 false)
   ret i64 %0
 }
 
@@ -66,7 +66,7 @@ define i64 @test_objectsize_null_known_flag_noas0() {
 entry:
   ; CHECK: ret i64 -1
   %0 = tail call i64 @llvm.objectsize.i64.p1i8(i8 addrspace(1)* null, i1 false,
-                                               i1 false)
+                                               i1 false, i1 false)
   ret i64 %0
 }
 
@@ -75,12 +75,12 @@ define i64 @test_objectsize_null_known_flag_min_noas0() {
 entry:
   ; CHECK: ret i64 0
   %0 = tail call i64 @llvm.objectsize.i64.p1i8(i8 addrspace(1)* null, i1 true,
-                                               i1 false)
+                                               i1 false, i1 false)
   ret i64 %0
 }
 
 
-declare i64 @llvm.objectsize.i64(i8*, i1, i1) nounwind readonly
-declare i64 @llvm.objectsize.i64.p1i8(i8 addrspace(1)*, i1, i1) nounwind readonly
+declare i64 @llvm.objectsize.i64(i8*, i1, i1, i1) nounwind readonly
+declare i64 @llvm.objectsize.i64.p1i8(i8 addrspace(1)*, i1, i1, i1) nounwind readonly
 
 declare void @llvm.trap() nounwind
diff --git a/test/Transforms/CodeGenPrepare/bitreverse-hang.ll b/test/Transforms/CodeGenPrepare/bitreverse-hang.ll
index c81dcc15cae912e9cddb8d9e85159e64de4c06ec..34044055663b9037a338c4823058b66bf3a03b21 100644
--- a/test/Transforms/CodeGenPrepare/bitreverse-hang.ll
+++ b/test/Transforms/CodeGenPrepare/bitreverse-hang.ll
@@ -46,7 +46,7 @@ attributes #0 = { norecurse nounwind ssp uwtable "disable-tail-calls"="false" "l
 !llvm.ident = !{!1}
 
 !0 = !{i32 1, !"PIC Level", i32 2}
-!1 = !{!"clang version 3.8.0 (http://llvm.org/git/clang.git eb70f4e9cc9a4dc3dd57b032fb858d56b4b64a0e)"}
+!1 = !{!"clang version 3.8.0"}
 !2 = !{!3, !3, i64 0}
 !3 = !{!"int", !4, i64 0}
 !4 = !{!"omnipotent char", !5, i64 0}
diff --git a/test/Transforms/CodeGenPrepare/crash-on-large-allocas.ll b/test/Transforms/CodeGenPrepare/crash-on-large-allocas.ll
index 3808c0e61c10a0b17133e4778d1bc6c3d44fd3cf..5049207ec27e12b50fadbba958180f280c0a4cf3 100644
--- a/test/Transforms/CodeGenPrepare/crash-on-large-allocas.ll
+++ b/test/Transforms/CodeGenPrepare/crash-on-large-allocas.ll
@@ -8,9 +8,9 @@ target datalayout = "p:16:16"
 ; CHECK-LABEL: @alloca_overflow_is_unknown(
 define i16 @alloca_overflow_is_unknown() {
   %i = alloca i8, i32 65537
-  %j = call i16 @llvm.objectsize.i16.p0i8(i8* %i, i1 false, i1 false)
+  %j = call i16 @llvm.objectsize.i16.p0i8(i8* %i, i1 false, i1 false, i1 false)
   ; CHECK: ret i16 -1
   ret i16 %j
 }
 
-declare i16 @llvm.objectsize.i16.p0i8(i8*, i1, i1)
+declare i16 @llvm.objectsize.i16.p0i8(i8*, i1, i1, i1)
diff --git a/test/Transforms/CodeGenPrepare/dom-tree.ll b/test/Transforms/CodeGenPrepare/dom-tree.ll
index b012706e3047c5dd6aa84e79beead0542430fbfc..28c3b94dcfd974a37324a1b31f3e89a1b09e37fb 100644
--- a/test/Transforms/CodeGenPrepare/dom-tree.ll
+++ b/test/Transforms/CodeGenPrepare/dom-tree.ll
@@ -36,6 +36,6 @@ attributes #0 = { norecurse nounwind readnone "disable-tail-calls"="false" "less
 
 !0 = !{i32 1, !"wchar_size", i32 4}
 !1 = !{i32 1, !"min_enum_size", i32 4}
-!2 = !{!"clang version 3.8.0 (http://llvm.org/git/clang.git b7441a0f42c43a8eea9e3e706be187252db747fa)"}
+!2 = !{!"clang version 3.8.0"}
 !3 = distinct !{!3, !4}
 !4 = !{!"llvm.loop.unroll.full"}
diff --git a/test/Transforms/CodeGenPrepare/overflow-intrinsics.ll b/test/Transforms/CodeGenPrepare/overflow-intrinsics.ll
deleted file mode 100644
index 772245250d44d46fb3e845890b8ba27c5728cc33..0000000000000000000000000000000000000000
--- a/test/Transforms/CodeGenPrepare/overflow-intrinsics.ll
+++ /dev/null
@@ -1,78 +0,0 @@
-; RUN: opt -codegenprepare -S < %s | FileCheck %s
-; RUN: opt -enable-debugify -codegenprepare -S < %s 2>&1 | FileCheck %s -check-prefix=DEBUG
-
-target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64"
-target triple = "x86_64-apple-darwin10.0.0"
-
-; CHECK-LABEL: @test1(
-; CHECK: llvm.uadd.with.overflow
-; CHECK: ret i64
-define i64 @test1(i64 %a, i64 %b) nounwind ssp {
-entry:
-  %add = add i64 %b, %a
-  %cmp = icmp ult i64 %add, %a
-  %Q = select i1 %cmp, i64 %b, i64 42
-  ret i64 %Q
-}
-
-; CHECK-LABEL: @test2(
-; CHECK: llvm.uadd.with.overflow
-; CHECK: ret i64
-define i64 @test2(i64 %a, i64 %b) nounwind ssp {
-entry:
-  %add = add i64 %b, %a
-  %cmp = icmp ult i64 %add, %b
-  %Q = select i1 %cmp, i64 %b, i64 42
-  ret i64 %Q
-}
-
-; CHECK-LABEL: @test3(
-; CHECK: llvm.uadd.with.overflow
-; CHECK: ret i64
-define i64 @test3(i64 %a, i64 %b) nounwind ssp {
-entry:
-  %add = add i64 %b, %a
-  %cmp = icmp ugt i64 %b, %add
-  %Q = select i1 %cmp, i64 %b, i64 42
-  ret i64 %Q
-}
-
-; CHECK-LABEL: @test4(
-; CHECK: llvm.uadd.with.overflow
-; CHECK: extractvalue
-; CHECK: extractvalue
-; CHECK: select
-define i64 @test4(i64 %a, i64 %b, i1 %c) nounwind ssp {
-entry:
-  %add = add i64 %b, %a
-  %cmp = icmp ugt i64 %b, %add
-  br i1 %c, label %next, label %exit
-
- next:
-  %Q = select i1 %cmp, i64 %b, i64 42
-  ret i64 %Q
-
- exit:
-  ret i64 0
-}
-
-; CHECK-LABEL: @test5(
-; CHECK-NOT: llvm.uadd.with.overflow
-; CHECK: next
-define i64 @test5(i64 %a, i64 %b, i64* %ptr, i1 %c) nounwind ssp {
-entry:
-  %add = add i64 %b, %a
-  store i64 %add, i64* %ptr
-  %cmp = icmp ugt i64 %b, %add
-  br i1 %c, label %next, label %exit
-
- next:
-  %Q = select i1 %cmp, i64 %b, i64 42
-  ret i64 %Q
-
- exit:
-  ret i64 0
-}
-
-; Check that every instruction inserted by -codegenprepare has a debug location.
-; DEBUG: CheckModuleDebugify: PASS
diff --git a/test/Transforms/CodeGenPrepare/widenable-condition.ll b/test/Transforms/CodeGenPrepare/widenable-condition.ll
new file mode 100644
index 0000000000000000000000000000000000000000..a12b87ff11583315a347ef1b4ef687b09bfe130a
--- /dev/null
+++ b/test/Transforms/CodeGenPrepare/widenable-condition.ll
@@ -0,0 +1,93 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -codegenprepare -S < %s | FileCheck %s
+
+; Check the idiomatic guard pattern to ensure it's lowered correctly.
+define void @test_guard(i1 %cond_0) {
+; CHECK-LABEL: @test_guard(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[COND_0:%.*]], label [[GUARDED:%.*]], label [[DEOPT:%.*]]
+; CHECK:       deopt:
+; CHECK-NEXT:    call void @foo()
+; CHECK-NEXT:    ret void
+; CHECK:       guarded:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %widenable_cond = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond = and i1 %cond_0, %widenable_cond
+  br i1 %exiplicit_guard_cond, label %guarded, label %deopt
+
+deopt:                                            ; preds = %entry
+  call void @foo()
+  ret void
+
+guarded:
+  ret void
+}
+
+;; Test a non-guard fastpath/slowpath case
+define void @test_triangle(i1 %cond_0) {
+; CHECK-LABEL: @test_triangle(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[COND_0:%.*]], label [[FASTPATH:%.*]], label [[SLOWPATH:%.*]]
+; CHECK:       fastpath:
+; CHECK-NEXT:    call void @bar()
+; CHECK-NEXT:    br label [[MERGE:%.*]]
+; CHECK:       slowpath:
+; CHECK-NEXT:    call void @foo()
+; CHECK-NEXT:    br label [[MERGE]]
+; CHECK:       merge:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %widenable_cond = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond = and i1 %cond_0, %widenable_cond
+  br i1 %exiplicit_guard_cond, label %fastpath, label %slowpath
+
+fastpath:
+  call void @bar()
+  br label %merge
+
+slowpath:
+  call void @foo()
+  br label %merge
+
+merge:
+  ret void
+}
+
+
+; Demonstrate that resulting CFG simplifications are made
+define void @test_cfg_simplify() {
+; CHECK-LABEL: @test_cfg_simplify(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %widenable_cond3 = call i1 @llvm.experimental.widenable.condition()
+  br i1 %widenable_cond3, label %guarded2, label %deopt3
+
+deopt3:
+  call void @foo()
+  ret void
+
+guarded2:
+  %widenable_cond4 = call i1 @llvm.experimental.widenable.condition()
+  br i1 %widenable_cond4, label %merge1, label %slowpath1
+
+slowpath1:
+  call void @foo()
+  br label %merge1
+
+merge1:
+  ret void
+}
+
+
+declare void @foo()
+declare void @bar()
+
+; Function Attrs: inaccessiblememonly nounwind
+declare i1 @llvm.experimental.widenable.condition() #0
+
+attributes #0 = { inaccessiblememonly nounwind }
diff --git a/test/Transforms/ConstProp/2007-11-23-cttz.ll b/test/Transforms/ConstProp/2007-11-23-cttz.ll
deleted file mode 100644
index c5ee70c2ff1208ce17a4c63f10d3d8efbe94f42a..0000000000000000000000000000000000000000
--- a/test/Transforms/ConstProp/2007-11-23-cttz.ll
+++ /dev/null
@@ -1,8 +0,0 @@
-; RUN: opt < %s -constprop -S | grep "ret i13 13"
-; PR1816
-declare i13 @llvm.cttz.i13(i13, i1)
-
-define i13 @test() {
-	%X = call i13 @llvm.cttz.i13(i13 0, i1 false)
-	ret i13 %X
-}
diff --git a/test/Transforms/ConstProp/bitcast.ll b/test/Transforms/ConstProp/bitcast.ll
index 5e1581d801c7cb951e222f314079b18a5ab6442c..353062ee3fed0125482869bc4b9fb0548cde2058 100644
--- a/test/Transforms/ConstProp/bitcast.ll
+++ b/test/Transforms/ConstProp/bitcast.ll
@@ -1,10 +1,24 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -constprop -S | FileCheck %s
 ; PR2165
 
 define <1 x i64> @test1() {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    ret <1 x i64> <i64 63>
+;
   %A = bitcast i64 63 to <1 x i64>
   ret <1 x i64> %A
-; CHECK-LABEL: @test1(
-; CHECK: ret <1 x i64> <i64 63>
 }
 
+; FIXME: Don't try to propagate an FP source operand to an icmp.
+
+@a = external global i16, align 1
+@b = external global i16, align 1
+
+define i1 @bad_icmp_constexpr_bitcast() {
+; CHECK-LABEL: @bad_icmp_constexpr_bitcast(
+; CHECK-NEXT:    ret i1 icmp eq (float bitcast (i32 ptrtoint (i16* @a to i32) to float), float fadd (float bitcast (i32 ptrtoint (i16* @b to i32) to float), float 2.000000e+00))
+;
+  %cmp = icmp eq i32 ptrtoint (i16* @a to i32), bitcast (float fadd (float bitcast (i32 ptrtoint (i16* @b to i32) to float), float 2.0) to i32)
+  ret i1 %cmp
+}
diff --git a/test/Transforms/ConstProp/overflow-ops.ll b/test/Transforms/ConstProp/overflow-ops.ll
index 1ae3e562e7e9133c3faa517e88ebd27e57097242..303b3b90ab36cd708ee33d9f780d408ac995bfa6 100644
--- a/test/Transforms/ConstProp/overflow-ops.ll
+++ b/test/Transforms/ConstProp/overflow-ops.ll
@@ -1,6 +1,6 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -constprop -S | FileCheck %s
 
-
 declare {i8, i1} @llvm.uadd.with.overflow.i8(i8, i8)
 declare {i8, i1} @llvm.usub.with.overflow.i8(i8, i8)
 declare {i8, i1} @llvm.umul.with.overflow.i8(i8, i8)
@@ -14,21 +14,27 @@ declare {i8, i1} @llvm.smul.with.overflow.i8(i8, i8)
 ;;-----------------------------
 
 define {i8, i1} @uadd_1() nounwind {
-entry:
+; CHECK-LABEL: @uadd_1(
+; CHECK-NEXT:    ret { i8, i1 } { i8 -114, i1 false }
+;
   %t = call {i8, i1} @llvm.uadd.with.overflow.i8(i8 42, i8 100)
   ret {i8, i1} %t
-
-; CHECK-LABEL: @uadd_1(
-; CHECK: ret { i8, i1 } { i8 -114, i1 false }
 }
 
 define {i8, i1} @uadd_2() nounwind {
-entry:
+; CHECK-LABEL: @uadd_2(
+; CHECK-NEXT:    ret { i8, i1 } { i8 6, i1 true }
+;
   %t = call {i8, i1} @llvm.uadd.with.overflow.i8(i8 142, i8 120)
   ret {i8, i1} %t
+}
 
-; CHECK-LABEL: @uadd_2(
-; CHECK: ret { i8, i1 } { i8 6, i1 true }
+define {i8, i1} @uadd_undef() nounwind {
+; CHECK-LABEL: @uadd_undef(
+; CHECK-NEXT:    ret { i8, i1 } undef
+;
+  %t = call {i8, i1} @llvm.uadd.with.overflow.i8(i8 142, i8 undef)
+  ret {i8, i1} %t
 }
 
 ;;-----------------------------
@@ -36,21 +42,27 @@ entry:
 ;;-----------------------------
 
 define {i8, i1} @usub_1() nounwind {
-entry:
+; CHECK-LABEL: @usub_1(
+; CHECK-NEXT:    ret { i8, i1 } { i8 2, i1 false }
+;
   %t = call {i8, i1} @llvm.usub.with.overflow.i8(i8 4, i8 2)
   ret {i8, i1} %t
-
-; CHECK-LABEL: @usub_1(
-; CHECK: ret { i8, i1 } { i8 2, i1 false }
 }
 
 define {i8, i1} @usub_2() nounwind {
-entry:
+; CHECK-LABEL: @usub_2(
+; CHECK-NEXT:    ret { i8, i1 } { i8 -2, i1 true }
+;
   %t = call {i8, i1} @llvm.usub.with.overflow.i8(i8 4, i8 6)
   ret {i8, i1} %t
+}
 
-; CHECK-LABEL: @usub_2(
-; CHECK: ret { i8, i1 } { i8 -2, i1 true }
+define {i8, i1} @usub_undef() nounwind {
+; CHECK-LABEL: @usub_undef(
+; CHECK-NEXT:    ret { i8, i1 } undef
+;
+  %t = call {i8, i1} @llvm.usub.with.overflow.i8(i8 4, i8 undef)
+  ret {i8, i1} %t
 }
 
 ;;-----------------------------
@@ -58,21 +70,35 @@ entry:
 ;;-----------------------------
 
 define {i8, i1} @umul_1() nounwind {
-entry:
+; CHECK-LABEL: @umul_1(
+; CHECK-NEXT:    ret { i8, i1 } { i8 44, i1 true }
+;
   %t = call {i8, i1} @llvm.umul.with.overflow.i8(i8 100, i8 3)
   ret {i8, i1} %t
-
-; CHECK-LABEL: @umul_1(
-; CHECK: ret { i8, i1 } { i8 44, i1 true }
 }
 
 define {i8, i1} @umul_2() nounwind {
-entry:
+; CHECK-LABEL: @umul_2(
+; CHECK-NEXT:    ret { i8, i1 } { i8 -56, i1 false }
+;
   %t = call {i8, i1} @llvm.umul.with.overflow.i8(i8 100, i8 2)
   ret {i8, i1} %t
+}
 
-; CHECK-LABEL: @umul_2(
-; CHECK: ret { i8, i1 } { i8 -56, i1 false }
+define {i8, i1} @umul_undef() nounwind {
+; CHECK-LABEL: @umul_undef(
+; CHECK-NEXT:    ret { i8, i1 } zeroinitializer
+;
+  %t = call {i8, i1} @llvm.umul.with.overflow.i8(i8 undef, i8 2)
+  ret {i8, i1} %t
+}
+
+define {i8, i1} @umul_both_undef() nounwind {
+; CHECK-LABEL: @umul_both_undef(
+; CHECK-NEXT:    ret { i8, i1 } zeroinitializer
+;
+  %t = call {i8, i1} @llvm.umul.with.overflow.i8(i8 undef, i8 undef)
+  ret {i8, i1} %t
 }
 
 ;;-----------------------------
@@ -80,128 +106,145 @@ entry:
 ;;-----------------------------
 
 define {i8, i1} @sadd_1() nounwind {
-entry:
+; CHECK-LABEL: @sadd_1(
+; CHECK-NEXT:    ret { i8, i1 } { i8 44, i1 false }
+;
   %t = call {i8, i1} @llvm.sadd.with.overflow.i8(i8 42, i8 2)
   ret {i8, i1} %t
-
-; CHECK-LABEL: @sadd_1(
-; CHECK: ret { i8, i1 } { i8 44, i1 false }
 }
 
 define {i8, i1} @sadd_2() nounwind {
-entry:
+; CHECK-LABEL: @sadd_2(
+; CHECK-NEXT:    ret { i8, i1 } { i8 -126, i1 true }
+;
   %t = call {i8, i1} @llvm.sadd.with.overflow.i8(i8 120, i8 10)
   ret {i8, i1} %t
-
-; CHECK-LABEL: @sadd_2(
-; CHECK: ret { i8, i1 } { i8 -126, i1 true }
 }
 
 define {i8, i1} @sadd_3() nounwind {
-entry:
+; CHECK-LABEL: @sadd_3(
+; CHECK-NEXT:    ret { i8, i1 } { i8 -110, i1 false }
+;
   %t = call {i8, i1} @llvm.sadd.with.overflow.i8(i8 -120, i8 10)
   ret {i8, i1} %t
-
-; CHECK-LABEL: @sadd_3(
-; CHECK: ret { i8, i1 } { i8 -110, i1 false }
 }
 
 define {i8, i1} @sadd_4() nounwind {
-entry:
+; CHECK-LABEL: @sadd_4(
+; CHECK-NEXT:    ret { i8, i1 } { i8 126, i1 true }
+;
   %t = call {i8, i1} @llvm.sadd.with.overflow.i8(i8 -120, i8 -10)
   ret {i8, i1} %t
-
-; CHECK-LABEL: @sadd_4(
-; CHECK: ret { i8, i1 } { i8 126, i1 true }
 }
 
 define {i8, i1} @sadd_5() nounwind {
-entry:
+; CHECK-LABEL: @sadd_5(
+; CHECK-NEXT:    ret { i8, i1 } { i8 -8, i1 false }
+;
   %t = call {i8, i1} @llvm.sadd.with.overflow.i8(i8 2, i8 -10)
   ret {i8, i1} %t
-
-; CHECK-LABEL: @sadd_5(
-; CHECK: ret { i8, i1 } { i8 -8, i1 false }
 }
 
+define {i8, i1} @sadd_undef() nounwind {
+; CHECK-LABEL: @sadd_undef(
+; CHECK-NEXT:    ret { i8, i1 } undef
+;
+  %t = call {i8, i1} @llvm.sadd.with.overflow.i8(i8 undef, i8 -10)
+  ret {i8, i1} %t
+}
 
 ;;-----------------------------
 ;; ssub
 ;;-----------------------------
 
 define {i8, i1} @ssub_1() nounwind {
-entry:
+; CHECK-LABEL: @ssub_1(
+; CHECK-NEXT:    ret { i8, i1 } { i8 2, i1 false }
+;
   %t = call {i8, i1} @llvm.ssub.with.overflow.i8(i8 4, i8 2)
   ret {i8, i1} %t
-
-; CHECK-LABEL: @ssub_1(
-; CHECK: ret { i8, i1 } { i8 2, i1 false }
 }
 
 define {i8, i1} @ssub_2() nounwind {
-entry:
+; CHECK-LABEL: @ssub_2(
+; CHECK-NEXT:    ret { i8, i1 } { i8 -2, i1 false }
+;
   %t = call {i8, i1} @llvm.ssub.with.overflow.i8(i8 4, i8 6)
   ret {i8, i1} %t
-
-; CHECK-LABEL: @ssub_2(
-; CHECK: ret { i8, i1 } { i8 -2, i1 false }
 }
 
 define {i8, i1} @ssub_3() nounwind {
-entry:
+; CHECK-LABEL: @ssub_3(
+; CHECK-NEXT:    ret { i8, i1 } { i8 126, i1 true }
+;
   %t = call {i8, i1} @llvm.ssub.with.overflow.i8(i8 -10, i8 120)
   ret {i8, i1} %t
-
-; CHECK-LABEL: @ssub_3(
-; CHECK: ret { i8, i1 } { i8 126, i1 true }
 }
 
 define {i8, i1} @ssub_3b() nounwind {
-entry:
+; CHECK-LABEL: @ssub_3b(
+; CHECK-NEXT:    ret { i8, i1 } { i8 -20, i1 false }
+;
   %t = call {i8, i1} @llvm.ssub.with.overflow.i8(i8 -10, i8 10)
   ret {i8, i1} %t
-
-; CHECK-LABEL: @ssub_3b(
-; CHECK: ret { i8, i1 } { i8 -20, i1 false }
 }
 
 define {i8, i1} @ssub_4() nounwind {
-entry:
+; CHECK-LABEL: @ssub_4(
+; CHECK-NEXT:    ret { i8, i1 } { i8 -126, i1 true }
+;
   %t = call {i8, i1} @llvm.ssub.with.overflow.i8(i8 120, i8 -10)
   ret {i8, i1} %t
-
-; CHECK-LABEL: @ssub_4(
-; CHECK: ret { i8, i1 } { i8 -126, i1 true }
 }
 
 define {i8, i1} @ssub_4b() nounwind {
-entry:
+; CHECK-LABEL: @ssub_4b(
+; CHECK-NEXT:    ret { i8, i1 } { i8 30, i1 false }
+;
   %t = call {i8, i1} @llvm.ssub.with.overflow.i8(i8 20, i8 -10)
   ret {i8, i1} %t
-
-; CHECK-LABEL: @ssub_4b(
-; CHECK: ret { i8, i1 } { i8 30, i1 false }
 }
 
 define {i8, i1} @ssub_5() nounwind {
-entry:
+; CHECK-LABEL: @ssub_5(
+; CHECK-NEXT:    ret { i8, i1 } { i8 -10, i1 false }
+;
   %t = call {i8, i1} @llvm.ssub.with.overflow.i8(i8 -20, i8 -10)
   ret {i8, i1} %t
+}
 
-; CHECK-LABEL: @ssub_5(
-; CHECK: ret { i8, i1 } { i8 -10, i1 false }
+define {i8, i1} @ssub_undef() nounwind {
+; CHECK-LABEL: @ssub_undef(
+; CHECK-NEXT:    ret { i8, i1 } undef
+;
+  %t = call {i8, i1} @llvm.ssub.with.overflow.i8(i8 undef, i8 -10)
+  ret {i8, i1} %t
 }
 
 ;;-----------------------------
 ;; smul
 ;;-----------------------------
 
-; rdar://8501501
 define {i8, i1} @smul_1() nounwind {
-entry:
+; CHECK-LABEL: @smul_1(
+; CHECK-NEXT:    ret { i8, i1 } { i8 -56, i1 true }
+;
   %t = call {i8, i1} @llvm.smul.with.overflow.i8(i8 -20, i8 -10)
   ret {i8, i1} %t
+}
 
-; CHECK-LABEL: @smul_1(
-; CHECK: ret { i8, i1 } { i8 -56, i1 true }
+define {i8, i1} @smul_undef() nounwind {
+; CHECK-LABEL: @smul_undef(
+; CHECK-NEXT:    ret { i8, i1 } zeroinitializer
+;
+  %t = call {i8, i1} @llvm.smul.with.overflow.i8(i8 -20, i8 undef)
+  ret {i8, i1} %t
+}
+
+define {i8, i1} @smul_both_undef() nounwind {
+; CHECK-LABEL: @smul_both_undef(
+; CHECK-NEXT:    ret { i8, i1 } zeroinitializer
+;
+  %t = call {i8, i1} @llvm.smul.with.overflow.i8(i8 undef, i8 undef)
+  ret {i8, i1} %t
 }
diff --git a/test/Transforms/Coroutines/coro-debug.ll b/test/Transforms/Coroutines/coro-debug.ll
index fefffd5b272305aea2e22e74af405424e4926616..e9e61ed64e854479a4f6cecac78f76d7ba8eb2bf 100644
--- a/test/Transforms/Coroutines/coro-debug.ll
+++ b/test/Transforms/Coroutines/coro-debug.ll
@@ -100,12 +100,12 @@ attributes #7 = { noduplicate }
 !llvm.module.flags = !{!3, !4}
 !llvm.ident = !{!5}
 
-!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 5.0.0 (http://llvm.org/git/clang.git 97b002238b11ff30d94d0516d6a0515db5725fd8) (http://llvm.org/git/llvm.git 0cb060ba567f1aa5b4b04e86665f88e4632b528a)", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !2)
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 5.0.0", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !2)
 !1 = !DIFile(filename: "<stdin>", directory: "C:\5CGitHub\5Cllvm\5Cbuild\5CDebug\5Cbin")
 !2 = !{}
 !3 = !{i32 2, !"Dwarf Version", i32 4}
 !4 = !{i32 2, !"Debug Info Version", i32 3}
-!5 = !{!"clang version 5.0.0 (http://llvm.org/git/clang.git 97b002238b11ff30d94d0516d6a0515db5725fd8) (http://llvm.org/git/llvm.git 0cb060ba567f1aa5b4b04e86665f88e4632b528a)"}
+!5 = !{!"clang version 5.0.0"}
 !6 = distinct !DISubprogram(name: "f", linkageName: "flink", scope: !7, file: !7, line: 55, type: !8, isLocal: false, isDefinition: true, scopeLine: 55, flags: DIFlagPrototyped, isOptimized: false, unit: !0, retainedNodes: !2)
 !7 = !DIFile(filename: "simple-repro.c", directory: "C:\5CGitHub\5Cllvm\5Cbuild\5CDebug\5Cbin")
 !8 = !DISubroutineType(types: !9)
diff --git a/test/Transforms/Coroutines/coro-split-dbg.ll b/test/Transforms/Coroutines/coro-split-dbg.ll
index 328ea2c6e55ff68f31a71cb78f0c11b81370de6a..e79d87170d19db60b33a8845b0bf18081cf1122c 100644
--- a/test/Transforms/Coroutines/coro-split-dbg.ll
+++ b/test/Transforms/Coroutines/coro-split-dbg.ll
@@ -77,12 +77,12 @@ attributes #9 = { noduplicate }
 !llvm.module.flags = !{!3, !4}
 !llvm.ident = !{!5}
 
-!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 4.0.0 (http://llvm.org/git/clang.git 6c405f93921ac99ff5b8521bb1b3df4449deede4) (http://llvm.org/git/llvm.git 6e6d3247102a0f87ce14e906dcf6a3a5ed3faa65)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2)
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 4.0.0", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2)
 !1 = !DIFile(filename: "coro.c", directory: "/home/gor/build/bin")
 !2 = !{}
 !3 = !{i32 2, !"Dwarf Version", i32 4}
 !4 = !{i32 2, !"Debug Info Version", i32 3}
-!5 = !{!"clang version 4.0.0 (http://llvm.org/git/clang.git 6c405f93921ac99ff5b8521bb1b3df4449deede4) (http://llvm.org/git/llvm.git 6e6d3247102a0f87ce14e906dcf6a3a5ed3faa65)"}
+!5 = !{!"clang version 4.0.0"}
 !6 = distinct !DISubprogram(name: "print", scope: !1, file: !1, line: 6, type: !7, isLocal: false, isDefinition: true, scopeLine: 6, flags: DIFlagPrototyped, isOptimized: true, unit: !0, retainedNodes: !10)
 !7 = !DISubroutineType(types: !8)
 !8 = !{null, !9}
diff --git a/test/Transforms/CorrelatedValuePropagation/deopt.ll b/test/Transforms/CorrelatedValuePropagation/deopt.ll
index cb1f72221bc7bfc9ea7b0c11f458c2a74b0b0690..6dab3aaf2438b075cc8b61df334896a7fece1870 100644
--- a/test/Transforms/CorrelatedValuePropagation/deopt.ll
+++ b/test/Transforms/CorrelatedValuePropagation/deopt.ll
@@ -11,7 +11,7 @@ define void @test1(i1 %c, i1 %c2) {
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i64 [[SEL2]], 0
 ; CHECK-NEXT:    br i1 [[CMP]], label [[TAKEN:%.*]], label [[UNTAKEN:%.*]]
 ; CHECK:       taken:
-; CHECK-NEXT:    call void @use() [ "deopt"(i64 [[SEL2]]) ]
+; CHECK-NEXT:    call void @use() [ "deopt"(i64 1) ]
 ; CHECK-NEXT:    ret void
 ; CHECK:       untaken:
 ; CHECK-NEXT:    ret void
@@ -37,7 +37,7 @@ define void @test1_assume(i1 %c, i1 %c2) {
 ; CHECK-NEXT:    [[SEL2:%.*]] = select i1 [[C2:%.*]], i64 [[SEL]], i64 0
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i64 [[SEL2]], 0
 ; CHECK-NEXT:    call void @llvm.assume(i1 [[CMP]])
-; CHECK-NEXT:    call void @use() [ "deopt"(i64 [[SEL2]]) ]
+; CHECK-NEXT:    call void @use() [ "deopt"(i64 1) ]
 ; CHECK-NEXT:    ret void
 ;
   %sel = select i1 %c, i64 -1, i64 1
@@ -55,7 +55,7 @@ define void @test1_guard(i1 %c, i1 %c2) {
 ; CHECK-NEXT:    [[SEL2:%.*]] = select i1 [[C2:%.*]], i64 [[SEL]], i64 0
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i64 [[SEL2]], 0
 ; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[CMP]]) [ "deopt"(i64 [[SEL2]]) ]
-; CHECK-NEXT:    call void @use() [ "deopt"(i64 [[SEL2]]) ]
+; CHECK-NEXT:    call void @use() [ "deopt"(i64 1) ]
 ; CHECK-NEXT:    ret void
 ;
   %sel = select i1 %c, i64 -1, i64 1
@@ -76,7 +76,7 @@ define void @test2(i1 %c, i1 %c2) {
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i64 [[SEL2]], 0
 ; CHECK-NEXT:    br i1 [[CMP]], label [[TAKEN:%.*]], label [[UNTAKEN:%.*]]
 ; CHECK:       taken:
-; CHECK-NEXT:    call void @use() [ "deopt"(i64 [[SEL2]]) ]
+; CHECK-NEXT:    call void @use() [ "deopt"(i64 1) ]
 ; CHECK-NEXT:    ret void
 ; CHECK:       untaken:
 ; CHECK-NEXT:    ret void
@@ -98,7 +98,7 @@ define void @test3(i1 %c, i1 %c2) {
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i64 [[SEL2]], 1
 ; CHECK-NEXT:    br i1 [[CMP]], label [[TAKEN:%.*]], label [[UNTAKEN:%.*]]
 ; CHECK:       taken:
-; CHECK-NEXT:    call void @use() [ "deopt"(i64 [[SEL2]]) ]
+; CHECK-NEXT:    call void @use() [ "deopt"(i64 2) ]
 ; CHECK-NEXT:    ret void
 ; CHECK:       untaken:
 ; CHECK-NEXT:    ret void
@@ -120,10 +120,10 @@ define void @test4(i1 %c, i1 %c2) {
 ; CHECK-NEXT:    [[SEL2:%.*]] = select i1 [[C2:%.*]], i64 0, i64 1
 ; CHECK-NEXT:    [[ADD1:%.*]] = add i64 0, [[SEL]]
 ; CHECK-NEXT:    [[ADD2:%.*]] = add i64 [[ADD1]], [[SEL2]]
-; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i64 [[ADD2]], 0
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i64 [[ADD2]], 1
 ; CHECK-NEXT:    br i1 [[CMP]], label [[TAKEN:%.*]], label [[UNTAKEN:%.*]]
 ; CHECK:       taken:
-; CHECK-NEXT:    call void @use() [ "deopt"(i64 [[ADD2]]) ]
+; CHECK-NEXT:    call void @use() [ "deopt"(i64 2) ]
 ; CHECK-NEXT:    ret void
 ; CHECK:       untaken:
 ; CHECK-NEXT:    ret void
@@ -132,7 +132,7 @@ define void @test4(i1 %c, i1 %c2) {
   %sel2 = select i1 %c2, i64 0, i64 1
   %add1 = add i64 0, %sel
   %add2 = add i64 %add1, %sel2
-  %cmp = icmp sgt i64 %add2, 0
+  %cmp = icmp sgt i64 %add2, 1
   br i1 %cmp, label %taken, label %untaken
 taken:
   call void @use() ["deopt" (i64 %add2)]
diff --git a/test/Transforms/DeadStoreElimination/debuginfo.ll b/test/Transforms/DeadStoreElimination/debuginfo.ll
index 2d9b4fe3ee8367b8828b0ea8fc15fe1437221cbf..b14c779288aa16513c0e256379f097f20b08d659 100644
--- a/test/Transforms/DeadStoreElimination/debuginfo.ll
+++ b/test/Transforms/DeadStoreElimination/debuginfo.ll
@@ -6,23 +6,21 @@ declare noalias i8* @malloc(i32)
 
 declare void @test_f()
 
-define i32* @test_salvage() {
+define i32* @test_salvage(i32 %arg) {
 ; Check that all four original local variables have their values preserved.
-; CHECK-LABEL: @test_salvage()
+; CHECK-LABEL: @test_salvage(
 ; CHECK-NEXT: malloc
 ; CHECK-NEXT: @llvm.dbg.value(metadata i8* %p, metadata ![[p:.*]], metadata !DIExpression())
 ; CHECK-NEXT: bitcast
 ; CHECK-NEXT: @llvm.dbg.value(metadata i32* %P, metadata ![[P:.*]], metadata !DIExpression())
-; CHECK-NEXT: @llvm.dbg.value(metadata i32* %P, metadata ![[DEAD:.*]], metadata !DIExpression(DW_OP_deref))
-; CHECK-NEXT: call void @llvm.dbg.value(metadata i32* %P, metadata ![[DEAD2:.*]], metadata !DIExpression(DW_OP_deref, DW_OP_plus_uconst, 1, DW_OP_stack_value))
+; CHECK-NEXT: @llvm.dbg.value(metadata i32 %arg, metadata ![[DEAD:.*]], metadata !DIExpression(DW_OP_plus_uconst, 1, DW_OP_stack_value))
 ; CHECK-NEXT: call void @test_f()
 ; CHECK-NEXT: store i32 0, i32* %P
 
   %p = tail call i8* @malloc(i32 4)
   %P = bitcast i8* %p to i32*
-  %DEAD = load i32, i32* %P
-  %DEAD2 = add i32 %DEAD, 1
-  store i32 %DEAD2, i32* %P
+  %DEAD = add i32 %arg, 1
+  store i32 %DEAD, i32* %P
   call void @test_f()
   store i32 0, i32* %P
   ret i32* %P
@@ -31,4 +29,3 @@ define i32* @test_salvage() {
 ; CHECK: ![[p]] = !DILocalVariable(name: "1"
 ; CHECK: ![[P]] = !DILocalVariable(name: "2"
 ; CHECK: ![[DEAD]] = !DILocalVariable(name: "3"
-; CHECK: ![[DEAD2]] = !DILocalVariable(name: "4"
diff --git a/test/Transforms/ExpandMemCmp/X86/memcmp.ll b/test/Transforms/ExpandMemCmp/X86/memcmp.ll
index 37bd85029b9fe4586afba6d845e7bc7db9dc6095..c1cbcc3272c7f8c0b30de0a6b34fab1020370cf9 100644
--- a/test/Transforms/ExpandMemCmp/X86/memcmp.ll
+++ b/test/Transforms/ExpandMemCmp/X86/memcmp.ll
@@ -130,11 +130,11 @@ define i32 @cmp6(i8* nocapture readonly %x, i8* nocapture readonly %y)  {
 ; ALL-NEXT:    [[TMP9:%.*]] = icmp eq i32 [[TMP7]], [[TMP8]]
 ; ALL-NEXT:    br i1 [[TMP9]], label [[LOADBB1]], label [[RES_BLOCK:%.*]]
 ; ALL:       loadbb1:
-; ALL-NEXT:    [[TMP10:%.*]] = bitcast i8* [[X]] to i16*
-; ALL-NEXT:    [[TMP11:%.*]] = bitcast i8* [[Y]] to i16*
-; ALL-NEXT:    [[TMP12:%.*]] = getelementptr i16, i16* [[TMP10]], i16 2
-; ALL-NEXT:    [[TMP13:%.*]] = getelementptr i16, i16* [[TMP11]], i16 2
-; ALL-NEXT:    [[TMP14:%.*]] = load i16, i16* [[TMP12]]
+; ALL-NEXT:    [[TMP10:%.*]] = getelementptr i8, i8* [[X]], i8 4
+; ALL-NEXT:    [[TMP11:%.*]] = bitcast i8* [[TMP10]] to i16*
+; ALL-NEXT:    [[TMP12:%.*]] = getelementptr i8, i8* [[Y]], i8 4
+; ALL-NEXT:    [[TMP13:%.*]] = bitcast i8* [[TMP12]] to i16*
+; ALL-NEXT:    [[TMP14:%.*]] = load i16, i16* [[TMP11]]
 ; ALL-NEXT:    [[TMP15:%.*]] = load i16, i16* [[TMP13]]
 ; ALL-NEXT:    [[TMP16:%.*]] = call i16 @llvm.bswap.i16(i16 [[TMP14]])
 ; ALL-NEXT:    [[TMP17:%.*]] = call i16 @llvm.bswap.i16(i16 [[TMP15]])
@@ -178,11 +178,11 @@ define i32 @cmp8(i8* nocapture readonly %x, i8* nocapture readonly %y)  {
 ; X32-NEXT:    [[TMP9:%.*]] = icmp eq i32 [[TMP7]], [[TMP8]]
 ; X32-NEXT:    br i1 [[TMP9]], label [[LOADBB1]], label [[RES_BLOCK:%.*]]
 ; X32:       loadbb1:
-; X32-NEXT:    [[TMP10:%.*]] = bitcast i8* [[X]] to i32*
-; X32-NEXT:    [[TMP11:%.*]] = bitcast i8* [[Y]] to i32*
-; X32-NEXT:    [[TMP12:%.*]] = getelementptr i32, i32* [[TMP10]], i32 1
-; X32-NEXT:    [[TMP13:%.*]] = getelementptr i32, i32* [[TMP11]], i32 1
-; X32-NEXT:    [[TMP14:%.*]] = load i32, i32* [[TMP12]]
+; X32-NEXT:    [[TMP10:%.*]] = getelementptr i8, i8* [[X]], i8 4
+; X32-NEXT:    [[TMP11:%.*]] = bitcast i8* [[TMP10]] to i32*
+; X32-NEXT:    [[TMP12:%.*]] = getelementptr i8, i8* [[Y]], i8 4
+; X32-NEXT:    [[TMP13:%.*]] = bitcast i8* [[TMP12]] to i32*
+; X32-NEXT:    [[TMP14:%.*]] = load i32, i32* [[TMP11]]
 ; X32-NEXT:    [[TMP15:%.*]] = load i32, i32* [[TMP13]]
 ; X32-NEXT:    [[TMP16]] = call i32 @llvm.bswap.i32(i32 [[TMP14]])
 ; X32-NEXT:    [[TMP17]] = call i32 @llvm.bswap.i32(i32 [[TMP15]])
@@ -272,11 +272,11 @@ define i32 @cmp10(i8* nocapture readonly %x, i8* nocapture readonly %y)  {
 ; X64-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[TMP7]], [[TMP8]]
 ; X64-NEXT:    br i1 [[TMP9]], label [[LOADBB1]], label [[RES_BLOCK:%.*]]
 ; X64:       loadbb1:
-; X64-NEXT:    [[TMP10:%.*]] = bitcast i8* [[X]] to i16*
-; X64-NEXT:    [[TMP11:%.*]] = bitcast i8* [[Y]] to i16*
-; X64-NEXT:    [[TMP12:%.*]] = getelementptr i16, i16* [[TMP10]], i16 4
-; X64-NEXT:    [[TMP13:%.*]] = getelementptr i16, i16* [[TMP11]], i16 4
-; X64-NEXT:    [[TMP14:%.*]] = load i16, i16* [[TMP12]]
+; X64-NEXT:    [[TMP10:%.*]] = getelementptr i8, i8* [[X]], i8 8
+; X64-NEXT:    [[TMP11:%.*]] = bitcast i8* [[TMP10]] to i16*
+; X64-NEXT:    [[TMP12:%.*]] = getelementptr i8, i8* [[Y]], i8 8
+; X64-NEXT:    [[TMP13:%.*]] = bitcast i8* [[TMP12]] to i16*
+; X64-NEXT:    [[TMP14:%.*]] = load i16, i16* [[TMP11]]
 ; X64-NEXT:    [[TMP15:%.*]] = load i16, i16* [[TMP13]]
 ; X64-NEXT:    [[TMP16:%.*]] = call i16 @llvm.bswap.i16(i16 [[TMP14]])
 ; X64-NEXT:    [[TMP17:%.*]] = call i16 @llvm.bswap.i16(i16 [[TMP15]])
@@ -324,11 +324,11 @@ define i32 @cmp12(i8* nocapture readonly %x, i8* nocapture readonly %y)  {
 ; X64-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[TMP7]], [[TMP8]]
 ; X64-NEXT:    br i1 [[TMP9]], label [[LOADBB1]], label [[RES_BLOCK:%.*]]
 ; X64:       loadbb1:
-; X64-NEXT:    [[TMP10:%.*]] = bitcast i8* [[X]] to i32*
-; X64-NEXT:    [[TMP11:%.*]] = bitcast i8* [[Y]] to i32*
-; X64-NEXT:    [[TMP12:%.*]] = getelementptr i32, i32* [[TMP10]], i32 2
-; X64-NEXT:    [[TMP13:%.*]] = getelementptr i32, i32* [[TMP11]], i32 2
-; X64-NEXT:    [[TMP14:%.*]] = load i32, i32* [[TMP12]]
+; X64-NEXT:    [[TMP10:%.*]] = getelementptr i8, i8* [[X]], i8 8
+; X64-NEXT:    [[TMP11:%.*]] = bitcast i8* [[TMP10]] to i32*
+; X64-NEXT:    [[TMP12:%.*]] = getelementptr i8, i8* [[Y]], i8 8
+; X64-NEXT:    [[TMP13:%.*]] = bitcast i8* [[TMP12]] to i32*
+; X64-NEXT:    [[TMP14:%.*]] = load i32, i32* [[TMP11]]
 ; X64-NEXT:    [[TMP15:%.*]] = load i32, i32* [[TMP13]]
 ; X64-NEXT:    [[TMP16:%.*]] = call i32 @llvm.bswap.i32(i32 [[TMP14]])
 ; X64-NEXT:    [[TMP17:%.*]] = call i32 @llvm.bswap.i32(i32 [[TMP15]])
@@ -394,11 +394,11 @@ define i32 @cmp16(i8* nocapture readonly %x, i8* nocapture readonly %y)  {
 ; X64-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[TMP7]], [[TMP8]]
 ; X64-NEXT:    br i1 [[TMP9]], label [[LOADBB1]], label [[RES_BLOCK:%.*]]
 ; X64:       loadbb1:
-; X64-NEXT:    [[TMP10:%.*]] = bitcast i8* [[X]] to i64*
-; X64-NEXT:    [[TMP11:%.*]] = bitcast i8* [[Y]] to i64*
-; X64-NEXT:    [[TMP12:%.*]] = getelementptr i64, i64* [[TMP10]], i64 1
-; X64-NEXT:    [[TMP13:%.*]] = getelementptr i64, i64* [[TMP11]], i64 1
-; X64-NEXT:    [[TMP14:%.*]] = load i64, i64* [[TMP12]]
+; X64-NEXT:    [[TMP10:%.*]] = getelementptr i8, i8* [[X]], i8 8
+; X64-NEXT:    [[TMP11:%.*]] = bitcast i8* [[TMP10]] to i64*
+; X64-NEXT:    [[TMP12:%.*]] = getelementptr i8, i8* [[Y]], i8 8
+; X64-NEXT:    [[TMP13:%.*]] = bitcast i8* [[TMP12]] to i64*
+; X64-NEXT:    [[TMP14:%.*]] = load i64, i64* [[TMP11]]
 ; X64-NEXT:    [[TMP15:%.*]] = load i64, i64* [[TMP13]]
 ; X64-NEXT:    [[TMP16]] = call i64 @llvm.bswap.i64(i64 [[TMP14]])
 ; X64-NEXT:    [[TMP17]] = call i64 @llvm.bswap.i64(i64 [[TMP15]])
@@ -597,11 +597,11 @@ define i32 @cmp_eq6(i8* nocapture readonly %x, i8* nocapture readonly %y)  {
 ; X32-NEXT:    [[TMP3:%.*]] = load i32, i32* [[TMP1]]
 ; X32-NEXT:    [[TMP4:%.*]] = load i32, i32* [[TMP2]]
 ; X32-NEXT:    [[TMP5:%.*]] = xor i32 [[TMP3]], [[TMP4]]
-; X32-NEXT:    [[TMP6:%.*]] = bitcast i8* [[X]] to i16*
-; X32-NEXT:    [[TMP7:%.*]] = bitcast i8* [[Y]] to i16*
-; X32-NEXT:    [[TMP8:%.*]] = getelementptr i16, i16* [[TMP6]], i16 2
-; X32-NEXT:    [[TMP9:%.*]] = getelementptr i16, i16* [[TMP7]], i16 2
-; X32-NEXT:    [[TMP10:%.*]] = load i16, i16* [[TMP8]]
+; X32-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i8 4
+; X32-NEXT:    [[TMP7:%.*]] = bitcast i8* [[TMP6]] to i16*
+; X32-NEXT:    [[TMP8:%.*]] = getelementptr i8, i8* [[Y]], i8 4
+; X32-NEXT:    [[TMP9:%.*]] = bitcast i8* [[TMP8]] to i16*
+; X32-NEXT:    [[TMP10:%.*]] = load i16, i16* [[TMP7]]
 ; X32-NEXT:    [[TMP11:%.*]] = load i16, i16* [[TMP9]]
 ; X32-NEXT:    [[TMP12:%.*]] = zext i16 [[TMP10]] to i32
 ; X32-NEXT:    [[TMP13:%.*]] = zext i16 [[TMP11]] to i32
@@ -625,11 +625,11 @@ define i32 @cmp_eq6(i8* nocapture readonly %x, i8* nocapture readonly %y)  {
 ; X64_1LD-NEXT:    [[TMP5:%.*]] = icmp ne i32 [[TMP3]], [[TMP4]]
 ; X64_1LD-NEXT:    br i1 [[TMP5]], label [[RES_BLOCK:%.*]], label [[LOADBB1:%.*]]
 ; X64_1LD:       loadbb1:
-; X64_1LD-NEXT:    [[TMP6:%.*]] = bitcast i8* [[X]] to i16*
-; X64_1LD-NEXT:    [[TMP7:%.*]] = bitcast i8* [[Y]] to i16*
-; X64_1LD-NEXT:    [[TMP8:%.*]] = getelementptr i16, i16* [[TMP6]], i16 2
-; X64_1LD-NEXT:    [[TMP9:%.*]] = getelementptr i16, i16* [[TMP7]], i16 2
-; X64_1LD-NEXT:    [[TMP10:%.*]] = load i16, i16* [[TMP8]]
+; X64_1LD-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i8 4
+; X64_1LD-NEXT:    [[TMP7:%.*]] = bitcast i8* [[TMP6]] to i16*
+; X64_1LD-NEXT:    [[TMP8:%.*]] = getelementptr i8, i8* [[Y]], i8 4
+; X64_1LD-NEXT:    [[TMP9:%.*]] = bitcast i8* [[TMP8]] to i16*
+; X64_1LD-NEXT:    [[TMP10:%.*]] = load i16, i16* [[TMP7]]
 ; X64_1LD-NEXT:    [[TMP11:%.*]] = load i16, i16* [[TMP9]]
 ; X64_1LD-NEXT:    [[TMP12:%.*]] = icmp ne i16 [[TMP10]], [[TMP11]]
 ; X64_1LD-NEXT:    br i1 [[TMP12]], label [[RES_BLOCK]], label [[ENDBLOCK]]
@@ -645,11 +645,11 @@ define i32 @cmp_eq6(i8* nocapture readonly %x, i8* nocapture readonly %y)  {
 ; X64_2LD-NEXT:    [[TMP3:%.*]] = load i32, i32* [[TMP1]]
 ; X64_2LD-NEXT:    [[TMP4:%.*]] = load i32, i32* [[TMP2]]
 ; X64_2LD-NEXT:    [[TMP5:%.*]] = xor i32 [[TMP3]], [[TMP4]]
-; X64_2LD-NEXT:    [[TMP6:%.*]] = bitcast i8* [[X]] to i16*
-; X64_2LD-NEXT:    [[TMP7:%.*]] = bitcast i8* [[Y]] to i16*
-; X64_2LD-NEXT:    [[TMP8:%.*]] = getelementptr i16, i16* [[TMP6]], i16 2
-; X64_2LD-NEXT:    [[TMP9:%.*]] = getelementptr i16, i16* [[TMP7]], i16 2
-; X64_2LD-NEXT:    [[TMP10:%.*]] = load i16, i16* [[TMP8]]
+; X64_2LD-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i8 4
+; X64_2LD-NEXT:    [[TMP7:%.*]] = bitcast i8* [[TMP6]] to i16*
+; X64_2LD-NEXT:    [[TMP8:%.*]] = getelementptr i8, i8* [[Y]], i8 4
+; X64_2LD-NEXT:    [[TMP9:%.*]] = bitcast i8* [[TMP8]] to i16*
+; X64_2LD-NEXT:    [[TMP10:%.*]] = load i16, i16* [[TMP7]]
 ; X64_2LD-NEXT:    [[TMP11:%.*]] = load i16, i16* [[TMP9]]
 ; X64_2LD-NEXT:    [[TMP12:%.*]] = zext i16 [[TMP10]] to i32
 ; X64_2LD-NEXT:    [[TMP13:%.*]] = zext i16 [[TMP11]] to i32
@@ -668,11 +668,71 @@ define i32 @cmp_eq6(i8* nocapture readonly %x, i8* nocapture readonly %y)  {
 }
 
 define i32 @cmp_eq7(i8* nocapture readonly %x, i8* nocapture readonly %y)  {
-; ALL-LABEL: @cmp_eq7(
-; ALL-NEXT:    [[CALL:%.*]] = tail call i32 @memcmp(i8* [[X:%.*]], i8* [[Y:%.*]], i64 7)
-; ALL-NEXT:    [[CMP:%.*]] = icmp eq i32 [[CALL]], 0
-; ALL-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
-; ALL-NEXT:    ret i32 [[CONV]]
+; X32-LABEL: @cmp_eq7(
+; X32-NEXT:    [[TMP1:%.*]] = bitcast i8* [[X:%.*]] to i32*
+; X32-NEXT:    [[TMP2:%.*]] = bitcast i8* [[Y:%.*]] to i32*
+; X32-NEXT:    [[TMP3:%.*]] = load i32, i32* [[TMP1]]
+; X32-NEXT:    [[TMP4:%.*]] = load i32, i32* [[TMP2]]
+; X32-NEXT:    [[TMP5:%.*]] = xor i32 [[TMP3]], [[TMP4]]
+; X32-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i8 3
+; X32-NEXT:    [[TMP7:%.*]] = bitcast i8* [[TMP6]] to i32*
+; X32-NEXT:    [[TMP8:%.*]] = getelementptr i8, i8* [[Y]], i8 3
+; X32-NEXT:    [[TMP9:%.*]] = bitcast i8* [[TMP8]] to i32*
+; X32-NEXT:    [[TMP10:%.*]] = load i32, i32* [[TMP7]]
+; X32-NEXT:    [[TMP11:%.*]] = load i32, i32* [[TMP9]]
+; X32-NEXT:    [[TMP12:%.*]] = xor i32 [[TMP10]], [[TMP11]]
+; X32-NEXT:    [[TMP13:%.*]] = or i32 [[TMP5]], [[TMP12]]
+; X32-NEXT:    [[TMP14:%.*]] = icmp ne i32 [[TMP13]], 0
+; X32-NEXT:    [[TMP15:%.*]] = zext i1 [[TMP14]] to i32
+; X32-NEXT:    [[CMP:%.*]] = icmp eq i32 [[TMP15]], 0
+; X32-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
+; X32-NEXT:    ret i32 [[CONV]]
+;
+; X64_1LD-LABEL: @cmp_eq7(
+; X64_1LD-NEXT:    br label [[LOADBB:%.*]]
+; X64_1LD:       res_block:
+; X64_1LD-NEXT:    br label [[ENDBLOCK:%.*]]
+; X64_1LD:       loadbb:
+; X64_1LD-NEXT:    [[TMP1:%.*]] = bitcast i8* [[X:%.*]] to i32*
+; X64_1LD-NEXT:    [[TMP2:%.*]] = bitcast i8* [[Y:%.*]] to i32*
+; X64_1LD-NEXT:    [[TMP3:%.*]] = load i32, i32* [[TMP1]]
+; X64_1LD-NEXT:    [[TMP4:%.*]] = load i32, i32* [[TMP2]]
+; X64_1LD-NEXT:    [[TMP5:%.*]] = icmp ne i32 [[TMP3]], [[TMP4]]
+; X64_1LD-NEXT:    br i1 [[TMP5]], label [[RES_BLOCK:%.*]], label [[LOADBB1:%.*]]
+; X64_1LD:       loadbb1:
+; X64_1LD-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i8 3
+; X64_1LD-NEXT:    [[TMP7:%.*]] = bitcast i8* [[TMP6]] to i32*
+; X64_1LD-NEXT:    [[TMP8:%.*]] = getelementptr i8, i8* [[Y]], i8 3
+; X64_1LD-NEXT:    [[TMP9:%.*]] = bitcast i8* [[TMP8]] to i32*
+; X64_1LD-NEXT:    [[TMP10:%.*]] = load i32, i32* [[TMP7]]
+; X64_1LD-NEXT:    [[TMP11:%.*]] = load i32, i32* [[TMP9]]
+; X64_1LD-NEXT:    [[TMP12:%.*]] = icmp ne i32 [[TMP10]], [[TMP11]]
+; X64_1LD-NEXT:    br i1 [[TMP12]], label [[RES_BLOCK]], label [[ENDBLOCK]]
+; X64_1LD:       endblock:
+; X64_1LD-NEXT:    [[PHI_RES:%.*]] = phi i32 [ 0, [[LOADBB1]] ], [ 1, [[RES_BLOCK]] ]
+; X64_1LD-NEXT:    [[CMP:%.*]] = icmp eq i32 [[PHI_RES]], 0
+; X64_1LD-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
+; X64_1LD-NEXT:    ret i32 [[CONV]]
+;
+; X64_2LD-LABEL: @cmp_eq7(
+; X64_2LD-NEXT:    [[TMP1:%.*]] = bitcast i8* [[X:%.*]] to i32*
+; X64_2LD-NEXT:    [[TMP2:%.*]] = bitcast i8* [[Y:%.*]] to i32*
+; X64_2LD-NEXT:    [[TMP3:%.*]] = load i32, i32* [[TMP1]]
+; X64_2LD-NEXT:    [[TMP4:%.*]] = load i32, i32* [[TMP2]]
+; X64_2LD-NEXT:    [[TMP5:%.*]] = xor i32 [[TMP3]], [[TMP4]]
+; X64_2LD-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i8 3
+; X64_2LD-NEXT:    [[TMP7:%.*]] = bitcast i8* [[TMP6]] to i32*
+; X64_2LD-NEXT:    [[TMP8:%.*]] = getelementptr i8, i8* [[Y]], i8 3
+; X64_2LD-NEXT:    [[TMP9:%.*]] = bitcast i8* [[TMP8]] to i32*
+; X64_2LD-NEXT:    [[TMP10:%.*]] = load i32, i32* [[TMP7]]
+; X64_2LD-NEXT:    [[TMP11:%.*]] = load i32, i32* [[TMP9]]
+; X64_2LD-NEXT:    [[TMP12:%.*]] = xor i32 [[TMP10]], [[TMP11]]
+; X64_2LD-NEXT:    [[TMP13:%.*]] = or i32 [[TMP5]], [[TMP12]]
+; X64_2LD-NEXT:    [[TMP14:%.*]] = icmp ne i32 [[TMP13]], 0
+; X64_2LD-NEXT:    [[TMP15:%.*]] = zext i1 [[TMP14]] to i32
+; X64_2LD-NEXT:    [[CMP:%.*]] = icmp eq i32 [[TMP15]], 0
+; X64_2LD-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
+; X64_2LD-NEXT:    ret i32 [[CONV]]
 ;
   %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 7)
   %cmp = icmp eq i32 %call, 0
@@ -687,11 +747,11 @@ define i32 @cmp_eq8(i8* nocapture readonly %x, i8* nocapture readonly %y)  {
 ; X32-NEXT:    [[TMP3:%.*]] = load i32, i32* [[TMP1]]
 ; X32-NEXT:    [[TMP4:%.*]] = load i32, i32* [[TMP2]]
 ; X32-NEXT:    [[TMP5:%.*]] = xor i32 [[TMP3]], [[TMP4]]
-; X32-NEXT:    [[TMP6:%.*]] = bitcast i8* [[X]] to i32*
-; X32-NEXT:    [[TMP7:%.*]] = bitcast i8* [[Y]] to i32*
-; X32-NEXT:    [[TMP8:%.*]] = getelementptr i32, i32* [[TMP6]], i32 1
-; X32-NEXT:    [[TMP9:%.*]] = getelementptr i32, i32* [[TMP7]], i32 1
-; X32-NEXT:    [[TMP10:%.*]] = load i32, i32* [[TMP8]]
+; X32-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i8 4
+; X32-NEXT:    [[TMP7:%.*]] = bitcast i8* [[TMP6]] to i32*
+; X32-NEXT:    [[TMP8:%.*]] = getelementptr i8, i8* [[Y]], i8 4
+; X32-NEXT:    [[TMP9:%.*]] = bitcast i8* [[TMP8]] to i32*
+; X32-NEXT:    [[TMP10:%.*]] = load i32, i32* [[TMP7]]
 ; X32-NEXT:    [[TMP11:%.*]] = load i32, i32* [[TMP9]]
 ; X32-NEXT:    [[TMP12:%.*]] = xor i32 [[TMP10]], [[TMP11]]
 ; X32-NEXT:    [[TMP13:%.*]] = or i32 [[TMP5]], [[TMP12]]
@@ -794,11 +854,11 @@ define i32 @cmp_eq10(i8* nocapture readonly %x, i8* nocapture readonly %y)  {
 ; X64_1LD-NEXT:    [[TMP5:%.*]] = icmp ne i64 [[TMP3]], [[TMP4]]
 ; X64_1LD-NEXT:    br i1 [[TMP5]], label [[RES_BLOCK:%.*]], label [[LOADBB1:%.*]]
 ; X64_1LD:       loadbb1:
-; X64_1LD-NEXT:    [[TMP6:%.*]] = bitcast i8* [[X]] to i16*
-; X64_1LD-NEXT:    [[TMP7:%.*]] = bitcast i8* [[Y]] to i16*
-; X64_1LD-NEXT:    [[TMP8:%.*]] = getelementptr i16, i16* [[TMP6]], i16 4
-; X64_1LD-NEXT:    [[TMP9:%.*]] = getelementptr i16, i16* [[TMP7]], i16 4
-; X64_1LD-NEXT:    [[TMP10:%.*]] = load i16, i16* [[TMP8]]
+; X64_1LD-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i8 8
+; X64_1LD-NEXT:    [[TMP7:%.*]] = bitcast i8* [[TMP6]] to i16*
+; X64_1LD-NEXT:    [[TMP8:%.*]] = getelementptr i8, i8* [[Y]], i8 8
+; X64_1LD-NEXT:    [[TMP9:%.*]] = bitcast i8* [[TMP8]] to i16*
+; X64_1LD-NEXT:    [[TMP10:%.*]] = load i16, i16* [[TMP7]]
 ; X64_1LD-NEXT:    [[TMP11:%.*]] = load i16, i16* [[TMP9]]
 ; X64_1LD-NEXT:    [[TMP12:%.*]] = icmp ne i16 [[TMP10]], [[TMP11]]
 ; X64_1LD-NEXT:    br i1 [[TMP12]], label [[RES_BLOCK]], label [[ENDBLOCK]]
@@ -814,11 +874,11 @@ define i32 @cmp_eq10(i8* nocapture readonly %x, i8* nocapture readonly %y)  {
 ; X64_2LD-NEXT:    [[TMP3:%.*]] = load i64, i64* [[TMP1]]
 ; X64_2LD-NEXT:    [[TMP4:%.*]] = load i64, i64* [[TMP2]]
 ; X64_2LD-NEXT:    [[TMP5:%.*]] = xor i64 [[TMP3]], [[TMP4]]
-; X64_2LD-NEXT:    [[TMP6:%.*]] = bitcast i8* [[X]] to i16*
-; X64_2LD-NEXT:    [[TMP7:%.*]] = bitcast i8* [[Y]] to i16*
-; X64_2LD-NEXT:    [[TMP8:%.*]] = getelementptr i16, i16* [[TMP6]], i16 4
-; X64_2LD-NEXT:    [[TMP9:%.*]] = getelementptr i16, i16* [[TMP7]], i16 4
-; X64_2LD-NEXT:    [[TMP10:%.*]] = load i16, i16* [[TMP8]]
+; X64_2LD-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i8 8
+; X64_2LD-NEXT:    [[TMP7:%.*]] = bitcast i8* [[TMP6]] to i16*
+; X64_2LD-NEXT:    [[TMP8:%.*]] = getelementptr i8, i8* [[Y]], i8 8
+; X64_2LD-NEXT:    [[TMP9:%.*]] = bitcast i8* [[TMP8]] to i16*
+; X64_2LD-NEXT:    [[TMP10:%.*]] = load i16, i16* [[TMP7]]
 ; X64_2LD-NEXT:    [[TMP11:%.*]] = load i16, i16* [[TMP9]]
 ; X64_2LD-NEXT:    [[TMP12:%.*]] = zext i16 [[TMP10]] to i64
 ; X64_2LD-NEXT:    [[TMP13:%.*]] = zext i16 [[TMP11]] to i64
@@ -837,11 +897,57 @@ define i32 @cmp_eq10(i8* nocapture readonly %x, i8* nocapture readonly %y)  {
 }
 
 define i32 @cmp_eq11(i8* nocapture readonly %x, i8* nocapture readonly %y)  {
-; ALL-LABEL: @cmp_eq11(
-; ALL-NEXT:    [[CALL:%.*]] = tail call i32 @memcmp(i8* [[X:%.*]], i8* [[Y:%.*]], i64 11)
-; ALL-NEXT:    [[CMP:%.*]] = icmp eq i32 [[CALL]], 0
-; ALL-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
-; ALL-NEXT:    ret i32 [[CONV]]
+; X32-LABEL: @cmp_eq11(
+; X32-NEXT:    [[CALL:%.*]] = tail call i32 @memcmp(i8* [[X:%.*]], i8* [[Y:%.*]], i64 11)
+; X32-NEXT:    [[CMP:%.*]] = icmp eq i32 [[CALL]], 0
+; X32-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
+; X32-NEXT:    ret i32 [[CONV]]
+;
+; X64_1LD-LABEL: @cmp_eq11(
+; X64_1LD-NEXT:    br label [[LOADBB:%.*]]
+; X64_1LD:       res_block:
+; X64_1LD-NEXT:    br label [[ENDBLOCK:%.*]]
+; X64_1LD:       loadbb:
+; X64_1LD-NEXT:    [[TMP1:%.*]] = bitcast i8* [[X:%.*]] to i64*
+; X64_1LD-NEXT:    [[TMP2:%.*]] = bitcast i8* [[Y:%.*]] to i64*
+; X64_1LD-NEXT:    [[TMP3:%.*]] = load i64, i64* [[TMP1]]
+; X64_1LD-NEXT:    [[TMP4:%.*]] = load i64, i64* [[TMP2]]
+; X64_1LD-NEXT:    [[TMP5:%.*]] = icmp ne i64 [[TMP3]], [[TMP4]]
+; X64_1LD-NEXT:    br i1 [[TMP5]], label [[RES_BLOCK:%.*]], label [[LOADBB1:%.*]]
+; X64_1LD:       loadbb1:
+; X64_1LD-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i8 3
+; X64_1LD-NEXT:    [[TMP7:%.*]] = bitcast i8* [[TMP6]] to i64*
+; X64_1LD-NEXT:    [[TMP8:%.*]] = getelementptr i8, i8* [[Y]], i8 3
+; X64_1LD-NEXT:    [[TMP9:%.*]] = bitcast i8* [[TMP8]] to i64*
+; X64_1LD-NEXT:    [[TMP10:%.*]] = load i64, i64* [[TMP7]]
+; X64_1LD-NEXT:    [[TMP11:%.*]] = load i64, i64* [[TMP9]]
+; X64_1LD-NEXT:    [[TMP12:%.*]] = icmp ne i64 [[TMP10]], [[TMP11]]
+; X64_1LD-NEXT:    br i1 [[TMP12]], label [[RES_BLOCK]], label [[ENDBLOCK]]
+; X64_1LD:       endblock:
+; X64_1LD-NEXT:    [[PHI_RES:%.*]] = phi i32 [ 0, [[LOADBB1]] ], [ 1, [[RES_BLOCK]] ]
+; X64_1LD-NEXT:    [[CMP:%.*]] = icmp eq i32 [[PHI_RES]], 0
+; X64_1LD-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
+; X64_1LD-NEXT:    ret i32 [[CONV]]
+;
+; X64_2LD-LABEL: @cmp_eq11(
+; X64_2LD-NEXT:    [[TMP1:%.*]] = bitcast i8* [[X:%.*]] to i64*
+; X64_2LD-NEXT:    [[TMP2:%.*]] = bitcast i8* [[Y:%.*]] to i64*
+; X64_2LD-NEXT:    [[TMP3:%.*]] = load i64, i64* [[TMP1]]
+; X64_2LD-NEXT:    [[TMP4:%.*]] = load i64, i64* [[TMP2]]
+; X64_2LD-NEXT:    [[TMP5:%.*]] = xor i64 [[TMP3]], [[TMP4]]
+; X64_2LD-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i8 3
+; X64_2LD-NEXT:    [[TMP7:%.*]] = bitcast i8* [[TMP6]] to i64*
+; X64_2LD-NEXT:    [[TMP8:%.*]] = getelementptr i8, i8* [[Y]], i8 3
+; X64_2LD-NEXT:    [[TMP9:%.*]] = bitcast i8* [[TMP8]] to i64*
+; X64_2LD-NEXT:    [[TMP10:%.*]] = load i64, i64* [[TMP7]]
+; X64_2LD-NEXT:    [[TMP11:%.*]] = load i64, i64* [[TMP9]]
+; X64_2LD-NEXT:    [[TMP12:%.*]] = xor i64 [[TMP10]], [[TMP11]]
+; X64_2LD-NEXT:    [[TMP13:%.*]] = or i64 [[TMP5]], [[TMP12]]
+; X64_2LD-NEXT:    [[TMP14:%.*]] = icmp ne i64 [[TMP13]], 0
+; X64_2LD-NEXT:    [[TMP15:%.*]] = zext i1 [[TMP14]] to i32
+; X64_2LD-NEXT:    [[CMP:%.*]] = icmp eq i32 [[TMP15]], 0
+; X64_2LD-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
+; X64_2LD-NEXT:    ret i32 [[CONV]]
 ;
   %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 11)
   %cmp = icmp eq i32 %call, 0
@@ -868,11 +974,11 @@ define i32 @cmp_eq12(i8* nocapture readonly %x, i8* nocapture readonly %y)  {
 ; X64_1LD-NEXT:    [[TMP5:%.*]] = icmp ne i64 [[TMP3]], [[TMP4]]
 ; X64_1LD-NEXT:    br i1 [[TMP5]], label [[RES_BLOCK:%.*]], label [[LOADBB1:%.*]]
 ; X64_1LD:       loadbb1:
-; X64_1LD-NEXT:    [[TMP6:%.*]] = bitcast i8* [[X]] to i32*
-; X64_1LD-NEXT:    [[TMP7:%.*]] = bitcast i8* [[Y]] to i32*
-; X64_1LD-NEXT:    [[TMP8:%.*]] = getelementptr i32, i32* [[TMP6]], i32 2
-; X64_1LD-NEXT:    [[TMP9:%.*]] = getelementptr i32, i32* [[TMP7]], i32 2
-; X64_1LD-NEXT:    [[TMP10:%.*]] = load i32, i32* [[TMP8]]
+; X64_1LD-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i8 8
+; X64_1LD-NEXT:    [[TMP7:%.*]] = bitcast i8* [[TMP6]] to i32*
+; X64_1LD-NEXT:    [[TMP8:%.*]] = getelementptr i8, i8* [[Y]], i8 8
+; X64_1LD-NEXT:    [[TMP9:%.*]] = bitcast i8* [[TMP8]] to i32*
+; X64_1LD-NEXT:    [[TMP10:%.*]] = load i32, i32* [[TMP7]]
 ; X64_1LD-NEXT:    [[TMP11:%.*]] = load i32, i32* [[TMP9]]
 ; X64_1LD-NEXT:    [[TMP12:%.*]] = icmp ne i32 [[TMP10]], [[TMP11]]
 ; X64_1LD-NEXT:    br i1 [[TMP12]], label [[RES_BLOCK]], label [[ENDBLOCK]]
@@ -888,11 +994,11 @@ define i32 @cmp_eq12(i8* nocapture readonly %x, i8* nocapture readonly %y)  {
 ; X64_2LD-NEXT:    [[TMP3:%.*]] = load i64, i64* [[TMP1]]
 ; X64_2LD-NEXT:    [[TMP4:%.*]] = load i64, i64* [[TMP2]]
 ; X64_2LD-NEXT:    [[TMP5:%.*]] = xor i64 [[TMP3]], [[TMP4]]
-; X64_2LD-NEXT:    [[TMP6:%.*]] = bitcast i8* [[X]] to i32*
-; X64_2LD-NEXT:    [[TMP7:%.*]] = bitcast i8* [[Y]] to i32*
-; X64_2LD-NEXT:    [[TMP8:%.*]] = getelementptr i32, i32* [[TMP6]], i32 2
-; X64_2LD-NEXT:    [[TMP9:%.*]] = getelementptr i32, i32* [[TMP7]], i32 2
-; X64_2LD-NEXT:    [[TMP10:%.*]] = load i32, i32* [[TMP8]]
+; X64_2LD-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i8 8
+; X64_2LD-NEXT:    [[TMP7:%.*]] = bitcast i8* [[TMP6]] to i32*
+; X64_2LD-NEXT:    [[TMP8:%.*]] = getelementptr i8, i8* [[Y]], i8 8
+; X64_2LD-NEXT:    [[TMP9:%.*]] = bitcast i8* [[TMP8]] to i32*
+; X64_2LD-NEXT:    [[TMP10:%.*]] = load i32, i32* [[TMP7]]
 ; X64_2LD-NEXT:    [[TMP11:%.*]] = load i32, i32* [[TMP9]]
 ; X64_2LD-NEXT:    [[TMP12:%.*]] = zext i32 [[TMP10]] to i64
 ; X64_2LD-NEXT:    [[TMP13:%.*]] = zext i32 [[TMP11]] to i64
@@ -911,11 +1017,57 @@ define i32 @cmp_eq12(i8* nocapture readonly %x, i8* nocapture readonly %y)  {
 }
 
 define i32 @cmp_eq13(i8* nocapture readonly %x, i8* nocapture readonly %y)  {
-; ALL-LABEL: @cmp_eq13(
-; ALL-NEXT:    [[CALL:%.*]] = tail call i32 @memcmp(i8* [[X:%.*]], i8* [[Y:%.*]], i64 13)
-; ALL-NEXT:    [[CMP:%.*]] = icmp eq i32 [[CALL]], 0
-; ALL-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
-; ALL-NEXT:    ret i32 [[CONV]]
+; X32-LABEL: @cmp_eq13(
+; X32-NEXT:    [[CALL:%.*]] = tail call i32 @memcmp(i8* [[X:%.*]], i8* [[Y:%.*]], i64 13)
+; X32-NEXT:    [[CMP:%.*]] = icmp eq i32 [[CALL]], 0
+; X32-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
+; X32-NEXT:    ret i32 [[CONV]]
+;
+; X64_1LD-LABEL: @cmp_eq13(
+; X64_1LD-NEXT:    br label [[LOADBB:%.*]]
+; X64_1LD:       res_block:
+; X64_1LD-NEXT:    br label [[ENDBLOCK:%.*]]
+; X64_1LD:       loadbb:
+; X64_1LD-NEXT:    [[TMP1:%.*]] = bitcast i8* [[X:%.*]] to i64*
+; X64_1LD-NEXT:    [[TMP2:%.*]] = bitcast i8* [[Y:%.*]] to i64*
+; X64_1LD-NEXT:    [[TMP3:%.*]] = load i64, i64* [[TMP1]]
+; X64_1LD-NEXT:    [[TMP4:%.*]] = load i64, i64* [[TMP2]]
+; X64_1LD-NEXT:    [[TMP5:%.*]] = icmp ne i64 [[TMP3]], [[TMP4]]
+; X64_1LD-NEXT:    br i1 [[TMP5]], label [[RES_BLOCK:%.*]], label [[LOADBB1:%.*]]
+; X64_1LD:       loadbb1:
+; X64_1LD-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i8 5
+; X64_1LD-NEXT:    [[TMP7:%.*]] = bitcast i8* [[TMP6]] to i64*
+; X64_1LD-NEXT:    [[TMP8:%.*]] = getelementptr i8, i8* [[Y]], i8 5
+; X64_1LD-NEXT:    [[TMP9:%.*]] = bitcast i8* [[TMP8]] to i64*
+; X64_1LD-NEXT:    [[TMP10:%.*]] = load i64, i64* [[TMP7]]
+; X64_1LD-NEXT:    [[TMP11:%.*]] = load i64, i64* [[TMP9]]
+; X64_1LD-NEXT:    [[TMP12:%.*]] = icmp ne i64 [[TMP10]], [[TMP11]]
+; X64_1LD-NEXT:    br i1 [[TMP12]], label [[RES_BLOCK]], label [[ENDBLOCK]]
+; X64_1LD:       endblock:
+; X64_1LD-NEXT:    [[PHI_RES:%.*]] = phi i32 [ 0, [[LOADBB1]] ], [ 1, [[RES_BLOCK]] ]
+; X64_1LD-NEXT:    [[CMP:%.*]] = icmp eq i32 [[PHI_RES]], 0
+; X64_1LD-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
+; X64_1LD-NEXT:    ret i32 [[CONV]]
+;
+; X64_2LD-LABEL: @cmp_eq13(
+; X64_2LD-NEXT:    [[TMP1:%.*]] = bitcast i8* [[X:%.*]] to i64*
+; X64_2LD-NEXT:    [[TMP2:%.*]] = bitcast i8* [[Y:%.*]] to i64*
+; X64_2LD-NEXT:    [[TMP3:%.*]] = load i64, i64* [[TMP1]]
+; X64_2LD-NEXT:    [[TMP4:%.*]] = load i64, i64* [[TMP2]]
+; X64_2LD-NEXT:    [[TMP5:%.*]] = xor i64 [[TMP3]], [[TMP4]]
+; X64_2LD-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i8 5
+; X64_2LD-NEXT:    [[TMP7:%.*]] = bitcast i8* [[TMP6]] to i64*
+; X64_2LD-NEXT:    [[TMP8:%.*]] = getelementptr i8, i8* [[Y]], i8 5
+; X64_2LD-NEXT:    [[TMP9:%.*]] = bitcast i8* [[TMP8]] to i64*
+; X64_2LD-NEXT:    [[TMP10:%.*]] = load i64, i64* [[TMP7]]
+; X64_2LD-NEXT:    [[TMP11:%.*]] = load i64, i64* [[TMP9]]
+; X64_2LD-NEXT:    [[TMP12:%.*]] = xor i64 [[TMP10]], [[TMP11]]
+; X64_2LD-NEXT:    [[TMP13:%.*]] = or i64 [[TMP5]], [[TMP12]]
+; X64_2LD-NEXT:    [[TMP14:%.*]] = icmp ne i64 [[TMP13]], 0
+; X64_2LD-NEXT:    [[TMP15:%.*]] = zext i1 [[TMP14]] to i32
+; X64_2LD-NEXT:    [[CMP:%.*]] = icmp eq i32 [[TMP15]], 0
+; X64_2LD-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
+; X64_2LD-NEXT:    ret i32 [[CONV]]
 ;
   %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 13)
   %cmp = icmp eq i32 %call, 0
@@ -924,11 +1076,57 @@ define i32 @cmp_eq13(i8* nocapture readonly %x, i8* nocapture readonly %y)  {
 }
 
 define i32 @cmp_eq14(i8* nocapture readonly %x, i8* nocapture readonly %y)  {
-; ALL-LABEL: @cmp_eq14(
-; ALL-NEXT:    [[CALL:%.*]] = tail call i32 @memcmp(i8* [[X:%.*]], i8* [[Y:%.*]], i64 14)
-; ALL-NEXT:    [[CMP:%.*]] = icmp eq i32 [[CALL]], 0
-; ALL-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
-; ALL-NEXT:    ret i32 [[CONV]]
+; X32-LABEL: @cmp_eq14(
+; X32-NEXT:    [[CALL:%.*]] = tail call i32 @memcmp(i8* [[X:%.*]], i8* [[Y:%.*]], i64 14)
+; X32-NEXT:    [[CMP:%.*]] = icmp eq i32 [[CALL]], 0
+; X32-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
+; X32-NEXT:    ret i32 [[CONV]]
+;
+; X64_1LD-LABEL: @cmp_eq14(
+; X64_1LD-NEXT:    br label [[LOADBB:%.*]]
+; X64_1LD:       res_block:
+; X64_1LD-NEXT:    br label [[ENDBLOCK:%.*]]
+; X64_1LD:       loadbb:
+; X64_1LD-NEXT:    [[TMP1:%.*]] = bitcast i8* [[X:%.*]] to i64*
+; X64_1LD-NEXT:    [[TMP2:%.*]] = bitcast i8* [[Y:%.*]] to i64*
+; X64_1LD-NEXT:    [[TMP3:%.*]] = load i64, i64* [[TMP1]]
+; X64_1LD-NEXT:    [[TMP4:%.*]] = load i64, i64* [[TMP2]]
+; X64_1LD-NEXT:    [[TMP5:%.*]] = icmp ne i64 [[TMP3]], [[TMP4]]
+; X64_1LD-NEXT:    br i1 [[TMP5]], label [[RES_BLOCK:%.*]], label [[LOADBB1:%.*]]
+; X64_1LD:       loadbb1:
+; X64_1LD-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i8 6
+; X64_1LD-NEXT:    [[TMP7:%.*]] = bitcast i8* [[TMP6]] to i64*
+; X64_1LD-NEXT:    [[TMP8:%.*]] = getelementptr i8, i8* [[Y]], i8 6
+; X64_1LD-NEXT:    [[TMP9:%.*]] = bitcast i8* [[TMP8]] to i64*
+; X64_1LD-NEXT:    [[TMP10:%.*]] = load i64, i64* [[TMP7]]
+; X64_1LD-NEXT:    [[TMP11:%.*]] = load i64, i64* [[TMP9]]
+; X64_1LD-NEXT:    [[TMP12:%.*]] = icmp ne i64 [[TMP10]], [[TMP11]]
+; X64_1LD-NEXT:    br i1 [[TMP12]], label [[RES_BLOCK]], label [[ENDBLOCK]]
+; X64_1LD:       endblock:
+; X64_1LD-NEXT:    [[PHI_RES:%.*]] = phi i32 [ 0, [[LOADBB1]] ], [ 1, [[RES_BLOCK]] ]
+; X64_1LD-NEXT:    [[CMP:%.*]] = icmp eq i32 [[PHI_RES]], 0
+; X64_1LD-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
+; X64_1LD-NEXT:    ret i32 [[CONV]]
+;
+; X64_2LD-LABEL: @cmp_eq14(
+; X64_2LD-NEXT:    [[TMP1:%.*]] = bitcast i8* [[X:%.*]] to i64*
+; X64_2LD-NEXT:    [[TMP2:%.*]] = bitcast i8* [[Y:%.*]] to i64*
+; X64_2LD-NEXT:    [[TMP3:%.*]] = load i64, i64* [[TMP1]]
+; X64_2LD-NEXT:    [[TMP4:%.*]] = load i64, i64* [[TMP2]]
+; X64_2LD-NEXT:    [[TMP5:%.*]] = xor i64 [[TMP3]], [[TMP4]]
+; X64_2LD-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i8 6
+; X64_2LD-NEXT:    [[TMP7:%.*]] = bitcast i8* [[TMP6]] to i64*
+; X64_2LD-NEXT:    [[TMP8:%.*]] = getelementptr i8, i8* [[Y]], i8 6
+; X64_2LD-NEXT:    [[TMP9:%.*]] = bitcast i8* [[TMP8]] to i64*
+; X64_2LD-NEXT:    [[TMP10:%.*]] = load i64, i64* [[TMP7]]
+; X64_2LD-NEXT:    [[TMP11:%.*]] = load i64, i64* [[TMP9]]
+; X64_2LD-NEXT:    [[TMP12:%.*]] = xor i64 [[TMP10]], [[TMP11]]
+; X64_2LD-NEXT:    [[TMP13:%.*]] = or i64 [[TMP5]], [[TMP12]]
+; X64_2LD-NEXT:    [[TMP14:%.*]] = icmp ne i64 [[TMP13]], 0
+; X64_2LD-NEXT:    [[TMP15:%.*]] = zext i1 [[TMP14]] to i32
+; X64_2LD-NEXT:    [[CMP:%.*]] = icmp eq i32 [[TMP15]], 0
+; X64_2LD-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
+; X64_2LD-NEXT:    ret i32 [[CONV]]
 ;
   %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 14)
   %cmp = icmp eq i32 %call, 0
@@ -937,11 +1135,57 @@ define i32 @cmp_eq14(i8* nocapture readonly %x, i8* nocapture readonly %y)  {
 }
 
 define i32 @cmp_eq15(i8* nocapture readonly %x, i8* nocapture readonly %y)  {
-; ALL-LABEL: @cmp_eq15(
-; ALL-NEXT:    [[CALL:%.*]] = tail call i32 @memcmp(i8* [[X:%.*]], i8* [[Y:%.*]], i64 15)
-; ALL-NEXT:    [[CMP:%.*]] = icmp eq i32 [[CALL]], 0
-; ALL-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
-; ALL-NEXT:    ret i32 [[CONV]]
+; X32-LABEL: @cmp_eq15(
+; X32-NEXT:    [[CALL:%.*]] = tail call i32 @memcmp(i8* [[X:%.*]], i8* [[Y:%.*]], i64 15)
+; X32-NEXT:    [[CMP:%.*]] = icmp eq i32 [[CALL]], 0
+; X32-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
+; X32-NEXT:    ret i32 [[CONV]]
+;
+; X64_1LD-LABEL: @cmp_eq15(
+; X64_1LD-NEXT:    br label [[LOADBB:%.*]]
+; X64_1LD:       res_block:
+; X64_1LD-NEXT:    br label [[ENDBLOCK:%.*]]
+; X64_1LD:       loadbb:
+; X64_1LD-NEXT:    [[TMP1:%.*]] = bitcast i8* [[X:%.*]] to i64*
+; X64_1LD-NEXT:    [[TMP2:%.*]] = bitcast i8* [[Y:%.*]] to i64*
+; X64_1LD-NEXT:    [[TMP3:%.*]] = load i64, i64* [[TMP1]]
+; X64_1LD-NEXT:    [[TMP4:%.*]] = load i64, i64* [[TMP2]]
+; X64_1LD-NEXT:    [[TMP5:%.*]] = icmp ne i64 [[TMP3]], [[TMP4]]
+; X64_1LD-NEXT:    br i1 [[TMP5]], label [[RES_BLOCK:%.*]], label [[LOADBB1:%.*]]
+; X64_1LD:       loadbb1:
+; X64_1LD-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i8 7
+; X64_1LD-NEXT:    [[TMP7:%.*]] = bitcast i8* [[TMP6]] to i64*
+; X64_1LD-NEXT:    [[TMP8:%.*]] = getelementptr i8, i8* [[Y]], i8 7
+; X64_1LD-NEXT:    [[TMP9:%.*]] = bitcast i8* [[TMP8]] to i64*
+; X64_1LD-NEXT:    [[TMP10:%.*]] = load i64, i64* [[TMP7]]
+; X64_1LD-NEXT:    [[TMP11:%.*]] = load i64, i64* [[TMP9]]
+; X64_1LD-NEXT:    [[TMP12:%.*]] = icmp ne i64 [[TMP10]], [[TMP11]]
+; X64_1LD-NEXT:    br i1 [[TMP12]], label [[RES_BLOCK]], label [[ENDBLOCK]]
+; X64_1LD:       endblock:
+; X64_1LD-NEXT:    [[PHI_RES:%.*]] = phi i32 [ 0, [[LOADBB1]] ], [ 1, [[RES_BLOCK]] ]
+; X64_1LD-NEXT:    [[CMP:%.*]] = icmp eq i32 [[PHI_RES]], 0
+; X64_1LD-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
+; X64_1LD-NEXT:    ret i32 [[CONV]]
+;
+; X64_2LD-LABEL: @cmp_eq15(
+; X64_2LD-NEXT:    [[TMP1:%.*]] = bitcast i8* [[X:%.*]] to i64*
+; X64_2LD-NEXT:    [[TMP2:%.*]] = bitcast i8* [[Y:%.*]] to i64*
+; X64_2LD-NEXT:    [[TMP3:%.*]] = load i64, i64* [[TMP1]]
+; X64_2LD-NEXT:    [[TMP4:%.*]] = load i64, i64* [[TMP2]]
+; X64_2LD-NEXT:    [[TMP5:%.*]] = xor i64 [[TMP3]], [[TMP4]]
+; X64_2LD-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i8 7
+; X64_2LD-NEXT:    [[TMP7:%.*]] = bitcast i8* [[TMP6]] to i64*
+; X64_2LD-NEXT:    [[TMP8:%.*]] = getelementptr i8, i8* [[Y]], i8 7
+; X64_2LD-NEXT:    [[TMP9:%.*]] = bitcast i8* [[TMP8]] to i64*
+; X64_2LD-NEXT:    [[TMP10:%.*]] = load i64, i64* [[TMP7]]
+; X64_2LD-NEXT:    [[TMP11:%.*]] = load i64, i64* [[TMP9]]
+; X64_2LD-NEXT:    [[TMP12:%.*]] = xor i64 [[TMP10]], [[TMP11]]
+; X64_2LD-NEXT:    [[TMP13:%.*]] = or i64 [[TMP5]], [[TMP12]]
+; X64_2LD-NEXT:    [[TMP14:%.*]] = icmp ne i64 [[TMP13]], 0
+; X64_2LD-NEXT:    [[TMP15:%.*]] = zext i1 [[TMP14]] to i32
+; X64_2LD-NEXT:    [[CMP:%.*]] = icmp eq i32 [[TMP15]], 0
+; X64_2LD-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
+; X64_2LD-NEXT:    ret i32 [[CONV]]
 ;
   %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 15)
   %cmp = icmp eq i32 %call, 0
diff --git a/test/Transforms/FunctionImport/Inputs/comdat.ll b/test/Transforms/FunctionImport/Inputs/comdat.ll
new file mode 100644
index 0000000000000000000000000000000000000000..1df6f25351e58731df01c00cd4d2c6c13d5da5c0
--- /dev/null
+++ b/test/Transforms/FunctionImport/Inputs/comdat.ll
@@ -0,0 +1,10 @@
+target datalayout = "e-m:w-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-pc-windows-msvc19.0.24215"
+
+define void @main() {
+entry:
+  call i8* @lwt_fun()
+  ret void
+}
+
+declare i8* @lwt_fun()
diff --git a/test/Transforms/FunctionImport/comdat.ll b/test/Transforms/FunctionImport/comdat.ll
new file mode 100644
index 0000000000000000000000000000000000000000..29e8cb538ab66529ad983c0e7761a36fb2830ddf
--- /dev/null
+++ b/test/Transforms/FunctionImport/comdat.ll
@@ -0,0 +1,32 @@
+; Test to ensure that comdat is renamed consistently when comdat leader is
+; promoted and renamed due to an import. Required by COFF.
+
+; REQUIRES: x86-registered-target
+
+; RUN: opt -thinlto-bc -o %t1.bc %s
+; RUN: opt -thinlto-bc -o %t2.bc %S/Inputs/comdat.ll
+; RUN: llvm-lto2 run -save-temps -o %t3 %t1.bc %t2.bc \
+; RUN:          -r %t1.bc,lwt_fun,plx \
+; RUN:          -r %t2.bc,main,plx \
+; RUN:          -r %t2.bc,lwt_fun,
+; RUN: llvm-dis -o - %t3.1.3.import.bc | FileCheck %s
+
+target datalayout = "e-m:w-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-pc-windows-msvc19.0.24215"
+
+; CHECK: $lwt.llvm.[[HASH:[0-9]+]] = comdat any
+$lwt = comdat any
+
+; CHECK: @lwt_aliasee = private unnamed_addr global {{.*}}, comdat($lwt.llvm.[[HASH]])
+@lwt_aliasee = private unnamed_addr global [1 x i8*] [i8* null], comdat($lwt)
+
+; CHECK: @lwt.llvm.[[HASH]] = hidden unnamed_addr alias
+@lwt = internal unnamed_addr alias [1 x i8*], [1 x i8*]* @lwt_aliasee
+
+; Below function should get imported into other module, resulting in @lwt being
+; promoted and renamed.
+define i8* @lwt_fun() {
+  %1 = getelementptr inbounds [1 x i8*], [1 x i8*]* @lwt, i32 0, i32 0
+  %2 = load i8*, i8** %1
+  ret i8* %2
+}
diff --git a/test/Transforms/GVN/callbr-loadpre-critedge.ll b/test/Transforms/GVN/callbr-loadpre-critedge.ll
new file mode 100644
index 0000000000000000000000000000000000000000..2a6a0aa778ab0bc65179daf38182a025f5400877
--- /dev/null
+++ b/test/Transforms/GVN/callbr-loadpre-critedge.ll
@@ -0,0 +1,49 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -gvn -S | FileCheck %s
+
+; This test checks that we don't hang trying to split a critical edge in loadpre
+; when the control flow uses a callbr instruction.
+
+%struct.pluto = type <{ i8, i8 }>
+
+define void @widget(%struct.pluto** %tmp1) {
+; CHECK-LABEL: @widget(
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    callbr void asm sideeffect "", "X,X"(i8* blockaddress(@widget, [[BB5:%.*]]), i8* blockaddress(@widget, [[BB8:%.*]]))
+; CHECK-NEXT:    to label [[BB4:%.*]] [label [[BB5]], label %bb8]
+; CHECK:       bb4:
+; CHECK-NEXT:    br label [[BB5]]
+; CHECK:       bb5:
+; CHECK-NEXT:    [[TMP6:%.*]] = load %struct.pluto*, %struct.pluto** [[TMP1:%.*]]
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds [[STRUCT_PLUTO:%.*]], %struct.pluto* [[TMP6]], i64 0, i32 1
+; CHECK-NEXT:    br label [[BB8]]
+; CHECK:       bb8:
+; CHECK-NEXT:    [[TMP9:%.*]] = phi i8* [ [[TMP7]], [[BB5]] ], [ null, [[BB:%.*]] ]
+; CHECK-NEXT:    [[TMP10:%.*]] = load %struct.pluto*, %struct.pluto** [[TMP1]]
+; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr inbounds [[STRUCT_PLUTO]], %struct.pluto* [[TMP10]], i64 0, i32 0
+; CHECK-NEXT:    [[TMP12:%.*]] = load i8, i8* [[TMP11]]
+; CHECK-NEXT:    tail call void @spam(i8* [[TMP9]], i8 [[TMP12]])
+; CHECK-NEXT:    ret void
+;
+bb:
+  callbr void asm sideeffect "", "X,X"(i8* blockaddress(@widget, %bb5), i8* blockaddress(@widget, %bb8))
+  to label %bb4 [label %bb5, label %bb8]
+
+bb4:                                              ; preds = %bb
+  br label %bb5
+
+bb5:                                              ; preds = %bb4, %bb
+  %tmp6 = load %struct.pluto*, %struct.pluto** %tmp1
+  %tmp7 = getelementptr inbounds %struct.pluto, %struct.pluto* %tmp6, i64 0, i32 1
+  br label %bb8
+
+bb8:                                              ; preds = %bb5, %bb
+  %tmp9 = phi i8* [ %tmp7, %bb5 ], [ null, %bb ]
+  %tmp10 = load %struct.pluto*, %struct.pluto** %tmp1
+  %tmp11 = getelementptr inbounds %struct.pluto, %struct.pluto* %tmp10, i64 0, i32 0
+  %tmp12 = load i8, i8* %tmp11
+  tail call void @spam(i8* %tmp9, i8 %tmp12)
+  ret void
+}
+
+declare void @spam(i8*, i8)
diff --git a/test/Transforms/GVN/callbr-scalarpre-critedge.ll b/test/Transforms/GVN/callbr-scalarpre-critedge.ll
new file mode 100644
index 0000000000000000000000000000000000000000..733ba4a0cc867156b521ea7432a95deb705a4d41
--- /dev/null
+++ b/test/Transforms/GVN/callbr-scalarpre-critedge.ll
@@ -0,0 +1,43 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -gvn -S | FileCheck %s
+
+; This test checks that we don't hang trying to split a critical edge in scalar
+; PRE when the control flow uses a callbr instruction.
+
+define void @wombat(i64 %arg, i64* %arg1, i64 %arg2, i32* %arg3) {
+; CHECK-LABEL: @wombat(
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    [[TMP5:%.*]] = or i64 [[ARG2:%.*]], [[ARG:%.*]]
+; CHECK-NEXT:    callbr void asm sideeffect "", "X,X"(i8* blockaddress(@wombat, [[BB7:%.*]]), i8* blockaddress(@wombat, [[BB9:%.*]]))
+; CHECK-NEXT:    to label [[BB6:%.*]] [label [[BB7]], label %bb9]
+; CHECK:       bb6:
+; CHECK-NEXT:    br label [[BB7]]
+; CHECK:       bb7:
+; CHECK-NEXT:    [[TMP8:%.*]] = trunc i64 [[TMP5]] to i32
+; CHECK-NEXT:    tail call void @barney(i32 [[TMP8]])
+; CHECK-NEXT:    br label [[BB9]]
+; CHECK:       bb9:
+; CHECK-NEXT:    [[TMP10:%.*]] = trunc i64 [[TMP5]] to i32
+; CHECK-NEXT:    store i32 [[TMP10]], i32* [[ARG3:%.*]]
+; CHECK-NEXT:    ret void
+;
+bb:
+  %tmp5 = or i64 %arg2, %arg
+  callbr void asm sideeffect "", "X,X"(i8* blockaddress(@wombat, %bb7), i8* blockaddress(@wombat, %bb9))
+          to label %bb6 [label %bb7, label %bb9]
+
+bb6:                                              ; preds = %bb
+  br label %bb7
+
+bb7:                                              ; preds = %bb6, %bb
+  %tmp8 = trunc i64 %tmp5 to i32
+  tail call void @barney(i32 %tmp8)
+  br label %bb9
+
+bb9:                                              ; preds = %bb7, %bb
+  %tmp10 = trunc i64 %tmp5 to i32
+  store i32 %tmp10, i32* %arg3
+  ret void
+}
+
+declare void @barney(i32)
diff --git a/test/Transforms/GVN/fence.ll b/test/Transforms/GVN/fence.ll
index ab8e2bbe72bacde19457a49861eacee197b7d7ce..f39fb951865d9d6c3eb12589794ef72b7b9f0e92 100644
--- a/test/Transforms/GVN/fence.ll
+++ b/test/Transforms/GVN/fence.ll
@@ -1,4 +1,4 @@
-; RUN: opt -S -debugify -basicaa -gvn < %s | FileCheck %s
+; RUN: opt -S -basicaa -gvn < %s | FileCheck %s
 
 @a = external constant i32
 ; We can value forward across the fence since we can (semantically) 
@@ -18,10 +18,8 @@ define i32 @test(i32* %addr.i) {
 ; Same as above
 define i32 @test2(i32* %addr.i) {
 ; CHECK-LABEL: @test2
-; CHECK-NEXT: call void @llvm.dbg.value(metadata i32* %addr.i, metadata [[var_a:![0-9]+]], metadata !DIExpression(DW_OP_deref))
 ; CHECK-NEXT: fence
 ; CHECK-NOT: load
-; CHECK-NEXT: call void @llvm.dbg.value(metadata i32* %addr.i, metadata [[var_a2:![0-9]+]], metadata !DIExpression(DW_OP_deref))
 ; CHECK: ret
   %a = load i32, i32* %addr.i, align 4
   fence release
@@ -88,6 +86,3 @@ define i32 @test4(i32* %addr) {
 ; }
 ; Given we chose to forward across the release fence, we clearly can't forward
 ; across the acquire fence as well.
-
-; CHECK: [[var_a]] = !DILocalVariable
-; CHECK-NEXT: [[var_a2]] = !DILocalVariable
diff --git a/test/Transforms/GlobalOpt/cxx-dtor.ll b/test/Transforms/GlobalOpt/cxx-dtor.ll
index c94c26b534850f25d9f8e80588cfab469645937d..c43a8e2be2e7c2c2aaa9d88cb19eb6ce06d14c19 100644
--- a/test/Transforms/GlobalOpt/cxx-dtor.ll
+++ b/test/Transforms/GlobalOpt/cxx-dtor.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -globalopt -S | FileCheck %s
+; RUN: opt < %s -S -passes='cgscc(inline),function(early-cse),globalopt' | FileCheck %s
 
 %0 = type { i32, void ()* }
 %struct.A = type { i8 }
diff --git a/test/Transforms/GuardWidening/basic_widenable_condition_guards.ll b/test/Transforms/GuardWidening/basic_widenable_condition_guards.ll
new file mode 100644
index 0000000000000000000000000000000000000000..605178a7868bbc45517e7a09a787bb61e61ae9dd
--- /dev/null
+++ b/test/Transforms/GuardWidening/basic_widenable_condition_guards.ll
@@ -0,0 +1,1041 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -guard-widening-widen-branch-guards=true -guard-widening < %s        | FileCheck %s
+; RUN: opt -S -guard-widening-widen-branch-guards=true -passes=guard-widening < %s | FileCheck %s
+
+; Basic test case: we wide the first check to check both the
+; conditions.
+define void @f_0(i1 %cond_0, i1 %cond_1) {
+; CHECK-LABEL: @f_0(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[COND_0:%.*]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    [[WIDE_CHK:%.*]] = and i1 [[COND_0]], [[COND_1:%.*]]
+; CHECK-NEXT:    [[GUARD_CHK:%.*]] = and i1 [[WIDE_CHK]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[GUARD_CHK]], label [[GUARDED:%.*]], label [[DEOPT:%.*]], !prof !0
+; CHECK:       deopt:
+; CHECK-NEXT:    call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+; CHECK-NEXT:    ret void
+; CHECK:       guarded:
+; CHECK-NEXT:    [[WIDENABLE_COND3:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND4:%.*]] = and i1 [[COND_1]], [[WIDENABLE_COND3]]
+; CHECK-NEXT:    br i1 true, label [[GUARDED1:%.*]], label [[DEOPT2:%.*]], !prof !0
+; CHECK:       deopt2:
+; CHECK-NEXT:    call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+; CHECK-NEXT:    ret void
+; CHECK:       guarded1:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %widenable_cond = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond = and i1 %cond_0, %widenable_cond
+  br i1 %exiplicit_guard_cond, label %guarded, label %deopt, !prof !0
+
+deopt:                                            ; preds = %entry
+  call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+  ret void
+
+guarded:                                          ; preds = %entry
+  %widenable_cond3 = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond4 = and i1 %cond_1, %widenable_cond3
+  br i1 %exiplicit_guard_cond4, label %guarded1, label %deopt2, !prof !0
+
+deopt2:                                           ; preds = %guarded
+  call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+  ret void
+
+guarded1:                                         ; preds = %guarded
+  ret void
+}
+
+; Same as @f_0, but with using a more general notion of postdominance.
+define void @f_1(i1 %cond_0, i1 %cond_1) {
+; CHECK-LABEL: @f_1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[COND_0:%.*]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    [[WIDE_CHK:%.*]] = and i1 [[COND_0]], [[COND_1:%.*]]
+; CHECK-NEXT:    [[GUARD_CHK:%.*]] = and i1 [[WIDE_CHK]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[GUARD_CHK]], label [[GUARDED:%.*]], label [[DEOPT:%.*]], !prof !0
+; CHECK:       deopt:
+; CHECK-NEXT:    call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+; CHECK-NEXT:    ret void
+; CHECK:       guarded:
+; CHECK-NEXT:    br i1 undef, label [[LEFT:%.*]], label [[RIGHT:%.*]]
+; CHECK:       left:
+; CHECK-NEXT:    br label [[MERGE:%.*]]
+; CHECK:       right:
+; CHECK-NEXT:    br label [[MERGE]]
+; CHECK:       merge:
+; CHECK-NEXT:    [[WIDENABLE_COND3:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND4:%.*]] = and i1 [[COND_1]], [[WIDENABLE_COND3]]
+; CHECK-NEXT:    br i1 true, label [[GUARDED1:%.*]], label [[DEOPT2:%.*]], !prof !0
+; CHECK:       deopt2:
+; CHECK-NEXT:    call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+; CHECK-NEXT:    ret void
+; CHECK:       guarded1:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %widenable_cond = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond = and i1 %cond_0, %widenable_cond
+  br i1 %exiplicit_guard_cond, label %guarded, label %deopt, !prof !0
+
+deopt:                                            ; preds = %entry
+  call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+  ret void
+
+guarded:                                          ; preds = %entry
+  br i1 undef, label %left, label %right
+
+left:                                             ; preds = %guarded
+  br label %merge
+
+right:                                            ; preds = %guarded
+  br label %merge
+
+merge:                                            ; preds = %right, %left
+  %widenable_cond3 = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond4 = and i1 %cond_1, %widenable_cond3
+  br i1 %exiplicit_guard_cond4, label %guarded1, label %deopt2, !prof !0
+
+deopt2:                                           ; preds = %merge
+  call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+  ret void
+
+guarded1:                                         ; preds = %merge
+  ret void
+}
+
+; Like @f_1, but we have some code we need to hoist before we can
+; widen a dominanting check.
+define void @f_2(i32 %a, i32 %b) {
+; CHECK-LABEL: @f_2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[COND_0:%.*]] = icmp ult i32 [[A:%.*]], 10
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[COND_0]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    [[COND_1:%.*]] = icmp ult i32 [[B:%.*]], 10
+; CHECK-NEXT:    [[WIDE_CHK:%.*]] = and i1 [[COND_0]], [[COND_1]]
+; CHECK-NEXT:    [[GUARD_CHK:%.*]] = and i1 [[WIDE_CHK]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[GUARD_CHK]], label [[GUARDED:%.*]], label [[DEOPT:%.*]], !prof !0
+; CHECK:       deopt:
+; CHECK-NEXT:    call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+; CHECK-NEXT:    ret void
+; CHECK:       guarded:
+; CHECK-NEXT:    br i1 undef, label [[LEFT:%.*]], label [[RIGHT:%.*]]
+; CHECK:       left:
+; CHECK-NEXT:    br label [[MERGE:%.*]]
+; CHECK:       right:
+; CHECK-NEXT:    br label [[MERGE]]
+; CHECK:       merge:
+; CHECK-NEXT:    [[WIDENABLE_COND3:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND4:%.*]] = and i1 [[COND_1]], [[WIDENABLE_COND3]]
+; CHECK-NEXT:    br i1 true, label [[GUARDED1:%.*]], label [[DEOPT2:%.*]], !prof !0
+; CHECK:       deopt2:
+; CHECK-NEXT:    call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+; CHECK-NEXT:    ret void
+; CHECK:       guarded1:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %cond_0 = icmp ult i32 %a, 10
+  %widenable_cond = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond = and i1 %cond_0, %widenable_cond
+  br i1 %exiplicit_guard_cond, label %guarded, label %deopt, !prof !0
+
+deopt:                                            ; preds = %entry
+  call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+  ret void
+
+guarded:                                          ; preds = %entry
+  br i1 undef, label %left, label %right
+
+left:                                             ; preds = %guarded
+  br label %merge
+
+right:                                            ; preds = %guarded
+  br label %merge
+
+merge:                                            ; preds = %right, %left
+  %cond_1 = icmp ult i32 %b, 10
+  %widenable_cond3 = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond4 = and i1 %cond_1, %widenable_cond3
+  br i1 %exiplicit_guard_cond4, label %guarded1, label %deopt2, !prof !0
+
+deopt2:                                           ; preds = %merge
+  call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+  ret void
+
+guarded1:                                         ; preds = %merge
+  ret void
+}
+
+; Negative test: don't hoist stuff out of control flow
+; indiscriminately, since that can make us do more work than needed.
+define void @f_3(i32 %a, i32 %b) {
+; CHECK-LABEL: @f_3(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[COND_0:%.*]] = icmp ult i32 [[A:%.*]], 10
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[COND_0]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[EXIPLICIT_GUARD_COND]], label [[GUARDED:%.*]], label [[DEOPT:%.*]], !prof !0
+; CHECK:       deopt:
+; CHECK-NEXT:    call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+; CHECK-NEXT:    ret void
+; CHECK:       guarded:
+; CHECK-NEXT:    br i1 undef, label [[LEFT:%.*]], label [[RIGHT:%.*]]
+; CHECK:       left:
+; CHECK-NEXT:    [[COND_1:%.*]] = icmp ult i32 [[B:%.*]], 10
+; CHECK-NEXT:    [[WIDENABLE_COND3:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND4:%.*]] = and i1 [[COND_1]], [[WIDENABLE_COND3]]
+; CHECK-NEXT:    br i1 [[EXIPLICIT_GUARD_COND4]], label [[GUARDED1:%.*]], label [[DEOPT2:%.*]], !prof !0
+; CHECK:       deopt2:
+; CHECK-NEXT:    call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+; CHECK-NEXT:    ret void
+; CHECK:       guarded1:
+; CHECK-NEXT:    ret void
+; CHECK:       right:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %cond_0 = icmp ult i32 %a, 10
+  %widenable_cond = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond = and i1 %cond_0, %widenable_cond
+  br i1 %exiplicit_guard_cond, label %guarded, label %deopt, !prof !0
+
+deopt:                                            ; preds = %entry
+  call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+  ret void
+
+guarded:                                          ; preds = %entry
+  br i1 undef, label %left, label %right
+
+left:                                             ; preds = %guarded
+  %cond_1 = icmp ult i32 %b, 10
+  %widenable_cond3 = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond4 = and i1 %cond_1, %widenable_cond3
+  br i1 %exiplicit_guard_cond4, label %guarded1, label %deopt2, !prof !0
+
+deopt2:                                           ; preds = %left
+  call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+  ret void
+
+guarded1:                                         ; preds = %left
+  ret void
+
+right:                                            ; preds = %guarded
+  ret void
+}
+
+; But hoisting out of control flow is fine if it makes a loop computed
+; condition loop invariant.  This behavior may require some tuning in
+; the future.
+define void @f_4(i32 %a, i32 %b) {
+; CHECK-LABEL: @f_4(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[COND_0:%.*]] = icmp ult i32 [[A:%.*]], 10
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[COND_0]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    [[COND_1:%.*]] = icmp ult i32 [[B:%.*]], 10
+; CHECK-NEXT:    [[WIDE_CHK:%.*]] = and i1 [[COND_0]], [[COND_1]]
+; CHECK-NEXT:    [[GUARD_CHK:%.*]] = and i1 [[WIDE_CHK]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[GUARD_CHK]], label [[GUARDED:%.*]], label [[DEOPT:%.*]], !prof !0
+; CHECK:       deopt:
+; CHECK-NEXT:    call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+; CHECK-NEXT:    ret void
+; CHECK:       guarded:
+; CHECK-NEXT:    br i1 undef, label [[LOOP:%.*]], label [[LEAVE:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[WIDENABLE_COND3:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND4:%.*]] = and i1 [[COND_1]], [[WIDENABLE_COND3]]
+; CHECK-NEXT:    br i1 true, label [[GUARDED1:%.*]], label [[DEOPT2:%.*]], !prof !0
+; CHECK:       deopt2:
+; CHECK-NEXT:    call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+; CHECK-NEXT:    ret void
+; CHECK:       guarded1:
+; CHECK-NEXT:    br i1 undef, label [[LOOP]], label [[LEAVE]]
+; CHECK:       leave:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %cond_0 = icmp ult i32 %a, 10
+  %widenable_cond = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond = and i1 %cond_0, %widenable_cond
+  br i1 %exiplicit_guard_cond, label %guarded, label %deopt, !prof !0
+
+deopt:                                            ; preds = %entry
+  call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+  ret void
+
+guarded:                                          ; preds = %entry
+  br i1 undef, label %loop, label %leave
+
+loop:                                             ; preds = %guarded1, %guarded
+  %cond_1 = icmp ult i32 %b, 10
+  %widenable_cond3 = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond4 = and i1 %cond_1, %widenable_cond3
+  br i1 %exiplicit_guard_cond4, label %guarded1, label %deopt2, !prof !0
+
+deopt2:                                           ; preds = %loop
+  call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+  ret void
+
+guarded1:                                         ; preds = %loop
+  br i1 undef, label %loop, label %leave
+
+leave:                                            ; preds = %guarded1, %guarded
+  ret void
+}
+
+; Hoisting out of control flow is also fine if we can widen the
+; dominating check without doing any extra work.
+define void @f_5(i32 %a) {
+; CHECK-LABEL: @f_5(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[COND_0:%.*]] = icmp ugt i32 [[A:%.*]], 7
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[COND_0]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    [[WIDE_CHK:%.*]] = icmp uge i32 [[A]], 11
+; CHECK-NEXT:    [[GUARD_CHK:%.*]] = and i1 [[WIDE_CHK]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[GUARD_CHK]], label [[GUARDED:%.*]], label [[DEOPT:%.*]], !prof !0
+; CHECK:       deopt:
+; CHECK-NEXT:    call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+; CHECK-NEXT:    ret void
+; CHECK:       guarded:
+; CHECK-NEXT:    br i1 undef, label [[LEFT:%.*]], label [[RIGHT:%.*]]
+; CHECK:       left:
+; CHECK-NEXT:    [[COND_1:%.*]] = icmp ugt i32 [[A]], 10
+; CHECK-NEXT:    [[WIDENABLE_COND3:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND4:%.*]] = and i1 [[COND_1]], [[WIDENABLE_COND3]]
+; CHECK-NEXT:    br i1 true, label [[GUARDED1:%.*]], label [[DEOPT2:%.*]], !prof !0
+; CHECK:       deopt2:
+; CHECK-NEXT:    call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+; CHECK-NEXT:    ret void
+; CHECK:       guarded1:
+; CHECK-NEXT:    ret void
+; CHECK:       right:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %cond_0 = icmp ugt i32 %a, 7
+  %widenable_cond = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond = and i1 %cond_0, %widenable_cond
+  br i1 %exiplicit_guard_cond, label %guarded, label %deopt, !prof !0
+
+deopt:                                            ; preds = %entry
+  call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+  ret void
+
+guarded:                                          ; preds = %entry
+  br i1 undef, label %left, label %right
+
+left:                                             ; preds = %guarded
+  %cond_1 = icmp ugt i32 %a, 10
+  %widenable_cond3 = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond4 = and i1 %cond_1, %widenable_cond3
+  br i1 %exiplicit_guard_cond4, label %guarded1, label %deopt2, !prof !0
+
+deopt2:                                           ; preds = %left
+  call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+  ret void
+
+guarded1:                                         ; preds = %left
+  ret void
+
+right:                                            ; preds = %guarded
+  ret void
+}
+
+; Negative test: the load from %a can be safely speculated to before
+; the first guard, but there is no guarantee that it will produce the
+; same value.
+define void @f_6(i1* dereferenceable(32) %a, i1* %b, i1 %unknown) {
+; CHECK-LABEL: @f_6(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[COND_0:%.*]] = load i1, i1* [[A:%.*]]
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[COND_0]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[EXIPLICIT_GUARD_COND]], label [[GUARDED:%.*]], label [[DEOPT:%.*]], !prof !0
+; CHECK:       deopt:
+; CHECK-NEXT:    call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+; CHECK-NEXT:    ret void
+; CHECK:       guarded:
+; CHECK-NEXT:    store i1 [[UNKNOWN:%.*]], i1* [[B:%.*]]
+; CHECK-NEXT:    [[COND_1:%.*]] = load i1, i1* [[A]]
+; CHECK-NEXT:    [[WIDENABLE_COND3:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND4:%.*]] = and i1 [[COND_1]], [[WIDENABLE_COND3]]
+; CHECK-NEXT:    br i1 [[EXIPLICIT_GUARD_COND4]], label [[GUARDED1:%.*]], label [[DEOPT2:%.*]], !prof !0
+; CHECK:       deopt2:
+; CHECK-NEXT:    call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+; CHECK-NEXT:    ret void
+; CHECK:       guarded1:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %cond_0 = load i1, i1* %a
+  %widenable_cond = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond = and i1 %cond_0, %widenable_cond
+  br i1 %exiplicit_guard_cond, label %guarded, label %deopt, !prof !0
+
+deopt:                                            ; preds = %entry
+  call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+  ret void
+
+guarded:                                          ; preds = %entry
+  store i1 %unknown, i1* %b
+  %cond_1 = load i1, i1* %a
+  %widenable_cond3 = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond4 = and i1 %cond_1, %widenable_cond3
+  br i1 %exiplicit_guard_cond4, label %guarded1, label %deopt2, !prof !0
+
+deopt2:                                           ; preds = %guarded
+  call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+  ret void
+
+guarded1:                                         ; preds = %guarded
+  ret void
+}
+
+; All else equal, we try to widen the earliest guard we can.  This
+; heuristic can use some tuning.
+define void @f_7(i32 %a, i1* %cond_buf) {
+; CHECK-LABEL: @f_7(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[COND_1:%.*]] = load volatile i1, i1* [[COND_BUF:%.*]]
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[COND_1]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    [[COND_3:%.*]] = icmp ult i32 [[A:%.*]], 7
+; CHECK-NEXT:    [[WIDE_CHK:%.*]] = and i1 [[COND_1]], [[COND_3]]
+; CHECK-NEXT:    [[GUARD_CHK:%.*]] = and i1 [[WIDE_CHK]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[GUARD_CHK]], label [[GUARDED:%.*]], label [[DEOPT:%.*]], !prof !0
+; CHECK:       deopt:
+; CHECK-NEXT:    call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+; CHECK-NEXT:    ret void
+; CHECK:       guarded:
+; CHECK-NEXT:    [[COND_2:%.*]] = load volatile i1, i1* [[COND_BUF]]
+; CHECK-NEXT:    [[WIDENABLE_COND3:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND4:%.*]] = and i1 [[COND_2]], [[WIDENABLE_COND3]]
+; CHECK-NEXT:    br i1 [[EXIPLICIT_GUARD_COND4]], label [[GUARDED1:%.*]], label [[DEOPT2:%.*]], !prof !0
+; CHECK:       deopt2:
+; CHECK-NEXT:    call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+; CHECK-NEXT:    ret void
+; CHECK:       guarded1:
+; CHECK-NEXT:    br i1 undef, label [[LEFT:%.*]], label [[RIGHT:%.*]]
+; CHECK:       left:
+; CHECK-NEXT:    [[WIDENABLE_COND7:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND8:%.*]] = and i1 [[COND_3]], [[WIDENABLE_COND7]]
+; CHECK-NEXT:    br i1 true, label [[GUARDED5:%.*]], label [[DEOPT6:%.*]], !prof !0
+; CHECK:       deopt6:
+; CHECK-NEXT:    call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+; CHECK-NEXT:    ret void
+; CHECK:       guarded5:
+; CHECK-NEXT:    br label [[LEFT]]
+; CHECK:       right:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %cond_1 = load volatile i1, i1* %cond_buf
+  %widenable_cond = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond = and i1 %cond_1, %widenable_cond
+  br i1 %exiplicit_guard_cond, label %guarded, label %deopt, !prof !0
+
+deopt:                                            ; preds = %entry
+  call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+  ret void
+
+guarded:                                          ; preds = %entry
+  %cond_2 = load volatile i1, i1* %cond_buf
+  %widenable_cond3 = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond4 = and i1 %cond_2, %widenable_cond3
+  br i1 %exiplicit_guard_cond4, label %guarded1, label %deopt2, !prof !0
+
+deopt2:                                           ; preds = %guarded
+  call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+  ret void
+
+guarded1:                                         ; preds = %guarded
+  br i1 undef, label %left, label %right
+
+left:                                             ; preds = %guarded5, %guarded1
+  %cond_3 = icmp ult i32 %a, 7
+  %widenable_cond7 = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond8 = and i1 %cond_3, %widenable_cond7
+  br i1 %exiplicit_guard_cond8, label %guarded5, label %deopt6, !prof !0
+
+deopt6:                                           ; preds = %left
+  call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+  ret void
+
+guarded5:                                         ; preds = %left
+  br label %left
+
+right:                                            ; preds = %guarded1
+  ret void
+}
+
+; In this case the earliest dominating guard is in a loop, and we
+; don't want to put extra work in there.  This heuristic can use some
+; tuning.
+define void @f_8(i32 %a, i1 %cond_1, i1 %cond_2) {
+; CHECK-LABEL: @f_8(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[COND_1:%.*]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[EXIPLICIT_GUARD_COND]], label [[GUARDED:%.*]], label [[DEOPT:%.*]], !prof !0
+; CHECK:       deopt:
+; CHECK-NEXT:    call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+; CHECK-NEXT:    ret void
+; CHECK:       guarded:
+; CHECK-NEXT:    br i1 undef, label [[LOOP]], label [[LEAVE:%.*]]
+; CHECK:       leave:
+; CHECK-NEXT:    [[WIDENABLE_COND3:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND4:%.*]] = and i1 [[COND_2:%.*]], [[WIDENABLE_COND3]]
+; CHECK-NEXT:    [[COND_3:%.*]] = icmp ult i32 [[A:%.*]], 7
+; CHECK-NEXT:    [[WIDE_CHK:%.*]] = and i1 [[COND_2]], [[COND_3]]
+; CHECK-NEXT:    [[GUARD_CHK:%.*]] = and i1 [[WIDE_CHK]], [[WIDENABLE_COND3]]
+; CHECK-NEXT:    br i1 [[GUARD_CHK]], label [[GUARDED1:%.*]], label [[DEOPT2:%.*]], !prof !0
+; CHECK:       deopt2:
+; CHECK-NEXT:    call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+; CHECK-NEXT:    ret void
+; CHECK:       guarded1:
+; CHECK-NEXT:    br i1 undef, label [[LOOP2:%.*]], label [[LEAVE2:%.*]]
+; CHECK:       loop2:
+; CHECK-NEXT:    [[WIDENABLE_COND7:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND8:%.*]] = and i1 [[COND_3]], [[WIDENABLE_COND7]]
+; CHECK-NEXT:    br i1 true, label [[GUARDED5:%.*]], label [[DEOPT6:%.*]], !prof !0
+; CHECK:       deopt6:
+; CHECK-NEXT:    call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+; CHECK-NEXT:    ret void
+; CHECK:       guarded5:
+; CHECK-NEXT:    br label [[LOOP2]]
+; CHECK:       leave2:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+
+loop:                                             ; preds = %guarded, %entry
+  %widenable_cond = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond = and i1 %cond_1, %widenable_cond
+  br i1 %exiplicit_guard_cond, label %guarded, label %deopt, !prof !0
+
+deopt:                                            ; preds = %loop
+  call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+  ret void
+
+guarded:                                          ; preds = %loop
+  br i1 undef, label %loop, label %leave
+
+leave:                                            ; preds = %guarded
+  %widenable_cond3 = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond4 = and i1 %cond_2, %widenable_cond3
+  br i1 %exiplicit_guard_cond4, label %guarded1, label %deopt2, !prof !0
+
+deopt2:                                           ; preds = %leave
+  call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+  ret void
+
+guarded1:                                         ; preds = %leave
+  br i1 undef, label %loop2, label %leave2
+
+loop2:                                            ; preds = %guarded5, %guarded1
+  %cond_3 = icmp ult i32 %a, 7
+  %widenable_cond7 = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond8 = and i1 %cond_3, %widenable_cond7
+  br i1 %exiplicit_guard_cond8, label %guarded5, label %deopt6, !prof !0
+
+deopt6:                                           ; preds = %loop2
+  call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+  ret void
+
+guarded5:                                         ; preds = %loop2
+  br label %loop2
+
+leave2:                                           ; preds = %guarded1
+  ret void
+}
+
+; In cases like these where there isn't any "obviously profitable"
+; widening sites, we refuse to do anything.
+define void @f_9(i32 %a, i1 %cond_0, i1 %cond_1) {
+; CHECK-LABEL: @f_9(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[FIRST_LOOP:%.*]]
+; CHECK:       first_loop:
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[COND_0:%.*]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[EXIPLICIT_GUARD_COND]], label [[GUARDED:%.*]], label [[DEOPT:%.*]], !prof !0
+; CHECK:       deopt:
+; CHECK-NEXT:    call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+; CHECK-NEXT:    ret void
+; CHECK:       guarded:
+; CHECK-NEXT:    br i1 undef, label [[FIRST_LOOP]], label [[SECOND_LOOP:%.*]]
+; CHECK:       second_loop:
+; CHECK-NEXT:    [[WIDENABLE_COND3:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND4:%.*]] = and i1 [[COND_1:%.*]], [[WIDENABLE_COND3]]
+; CHECK-NEXT:    br i1 [[EXIPLICIT_GUARD_COND4]], label [[GUARDED1:%.*]], label [[DEOPT2:%.*]], !prof !0
+; CHECK:       deopt2:
+; CHECK-NEXT:    call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+; CHECK-NEXT:    ret void
+; CHECK:       guarded1:
+; CHECK-NEXT:    br label [[SECOND_LOOP]]
+;
+entry:
+  br label %first_loop
+
+first_loop:                                       ; preds = %guarded, %entry
+  %widenable_cond = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond = and i1 %cond_0, %widenable_cond
+  br i1 %exiplicit_guard_cond, label %guarded, label %deopt, !prof !0
+
+deopt:                                            ; preds = %first_loop
+  call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+  ret void
+
+guarded:                                          ; preds = %first_loop
+  br i1 undef, label %first_loop, label %second_loop
+
+second_loop:                                      ; preds = %guarded1, %guarded
+  %widenable_cond3 = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond4 = and i1 %cond_1, %widenable_cond3
+  br i1 %exiplicit_guard_cond4, label %guarded1, label %deopt2, !prof !0
+
+deopt2:                                           ; preds = %second_loop
+  call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+  ret void
+
+guarded1:                                         ; preds = %second_loop
+  br label %second_loop
+}
+
+; Same situation as in @f_9: no "obviously profitable" widening sites,
+; so we refuse to do anything.
+define void @f_10(i32 %a, i1 %cond_0, i1 %cond_1) {
+; CHECK-LABEL: @f_10(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[COND_0:%.*]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[EXIPLICIT_GUARD_COND]], label [[GUARDED:%.*]], label [[DEOPT:%.*]], !prof !0
+; CHECK:       deopt:
+; CHECK-NEXT:    call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+; CHECK-NEXT:    ret void
+; CHECK:       guarded:
+; CHECK-NEXT:    br i1 undef, label [[LOOP]], label [[NO_LOOP:%.*]]
+; CHECK:       no_loop:
+; CHECK-NEXT:    [[WIDENABLE_COND3:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND4:%.*]] = and i1 [[COND_1:%.*]], [[WIDENABLE_COND3]]
+; CHECK-NEXT:    br i1 [[EXIPLICIT_GUARD_COND4]], label [[GUARDED1:%.*]], label [[DEOPT2:%.*]], !prof !0
+; CHECK:       deopt2:
+; CHECK-NEXT:    call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+; CHECK-NEXT:    ret void
+; CHECK:       guarded1:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+
+loop:                                             ; preds = %guarded, %entry
+  %widenable_cond = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond = and i1 %cond_0, %widenable_cond
+  br i1 %exiplicit_guard_cond, label %guarded, label %deopt, !prof !0
+
+deopt:                                            ; preds = %loop
+  call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+  ret void
+
+guarded:                                          ; preds = %loop
+  br i1 undef, label %loop, label %no_loop
+
+no_loop:                                          ; preds = %guarded
+  %widenable_cond3 = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond4 = and i1 %cond_1, %widenable_cond3
+  br i1 %exiplicit_guard_cond4, label %guarded1, label %deopt2, !prof !0
+
+deopt2:                                           ; preds = %no_loop
+  call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+  ret void
+
+guarded1:                                         ; preds = %no_loop
+  ret void
+}
+
+; With guards in loops, we're okay hoisting out the guard into the
+; containing loop.
+define void @f_11(i32 %a, i1 %cond_0, i1 %cond_1) {
+; CHECK-LABEL: @f_11(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[OUTER_HEADER:%.*]]
+; CHECK:       outer_header:
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[COND_0:%.*]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    [[WIDE_CHK:%.*]] = and i1 [[COND_0]], [[COND_1:%.*]]
+; CHECK-NEXT:    [[GUARD_CHK:%.*]] = and i1 [[WIDE_CHK]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[GUARD_CHK]], label [[GUARDED:%.*]], label [[DEOPT:%.*]], !prof !0
+; CHECK:       deopt:
+; CHECK-NEXT:    call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+; CHECK-NEXT:    ret void
+; CHECK:       guarded:
+; CHECK-NEXT:    br label [[INNER:%.*]]
+; CHECK:       inner:
+; CHECK-NEXT:    [[WIDENABLE_COND3:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND4:%.*]] = and i1 [[COND_1]], [[WIDENABLE_COND3]]
+; CHECK-NEXT:    br i1 true, label [[GUARDED1:%.*]], label [[DEOPT2:%.*]], !prof !0
+; CHECK:       deopt2:
+; CHECK-NEXT:    call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+; CHECK-NEXT:    ret void
+; CHECK:       guarded1:
+; CHECK-NEXT:    br i1 undef, label [[INNER]], label [[OUTER_LATCH:%.*]]
+; CHECK:       outer_latch:
+; CHECK-NEXT:    br i1 undef, label [[OUTER_HEADER]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %outer_header
+
+outer_header:                                     ; preds = %outer_latch, %entry
+  %widenable_cond = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond = and i1 %cond_0, %widenable_cond
+  br i1 %exiplicit_guard_cond, label %guarded, label %deopt, !prof !0
+
+deopt:                                            ; preds = %outer_header
+  call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+  ret void
+
+guarded:                                          ; preds = %outer_header
+  br label %inner
+
+inner:                                            ; preds = %guarded1, %guarded
+  %widenable_cond3 = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond4 = and i1 %cond_1, %widenable_cond3
+  br i1 %exiplicit_guard_cond4, label %guarded1, label %deopt2, !prof !0
+
+deopt2:                                           ; preds = %inner
+  call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+  ret void
+
+guarded1:                                         ; preds = %inner
+  br i1 undef, label %inner, label %outer_latch
+
+outer_latch:                                      ; preds = %guarded1
+  br i1 undef, label %outer_header, label %exit
+
+exit:                                             ; preds = %outer_latch
+  ret void
+}
+
+; Checks that we are adequately guarded against exponential-time
+; behavior when hoisting code.
+define void @f_12(i32 %a0) {
+; CHECK-LABEL: @f_12(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 true, [[WIDENABLE_COND]]
+; CHECK-NEXT:    [[A1:%.*]] = mul i32 [[A0:%.*]], [[A0]]
+; CHECK-NEXT:    [[A2:%.*]] = mul i32 [[A1]], [[A1]]
+; CHECK-NEXT:    [[A3:%.*]] = mul i32 [[A2]], [[A2]]
+; CHECK-NEXT:    [[A4:%.*]] = mul i32 [[A3]], [[A3]]
+; CHECK-NEXT:    [[A5:%.*]] = mul i32 [[A4]], [[A4]]
+; CHECK-NEXT:    [[A6:%.*]] = mul i32 [[A5]], [[A5]]
+; CHECK-NEXT:    [[A7:%.*]] = mul i32 [[A6]], [[A6]]
+; CHECK-NEXT:    [[A8:%.*]] = mul i32 [[A7]], [[A7]]
+; CHECK-NEXT:    [[A9:%.*]] = mul i32 [[A8]], [[A8]]
+; CHECK-NEXT:    [[A10:%.*]] = mul i32 [[A9]], [[A9]]
+; CHECK-NEXT:    [[A11:%.*]] = mul i32 [[A10]], [[A10]]
+; CHECK-NEXT:    [[A12:%.*]] = mul i32 [[A11]], [[A11]]
+; CHECK-NEXT:    [[A13:%.*]] = mul i32 [[A12]], [[A12]]
+; CHECK-NEXT:    [[A14:%.*]] = mul i32 [[A13]], [[A13]]
+; CHECK-NEXT:    [[A15:%.*]] = mul i32 [[A14]], [[A14]]
+; CHECK-NEXT:    [[A16:%.*]] = mul i32 [[A15]], [[A15]]
+; CHECK-NEXT:    [[A17:%.*]] = mul i32 [[A16]], [[A16]]
+; CHECK-NEXT:    [[A18:%.*]] = mul i32 [[A17]], [[A17]]
+; CHECK-NEXT:    [[A19:%.*]] = mul i32 [[A18]], [[A18]]
+; CHECK-NEXT:    [[A20:%.*]] = mul i32 [[A19]], [[A19]]
+; CHECK-NEXT:    [[A21:%.*]] = mul i32 [[A20]], [[A20]]
+; CHECK-NEXT:    [[A22:%.*]] = mul i32 [[A21]], [[A21]]
+; CHECK-NEXT:    [[A23:%.*]] = mul i32 [[A22]], [[A22]]
+; CHECK-NEXT:    [[A24:%.*]] = mul i32 [[A23]], [[A23]]
+; CHECK-NEXT:    [[A25:%.*]] = mul i32 [[A24]], [[A24]]
+; CHECK-NEXT:    [[A26:%.*]] = mul i32 [[A25]], [[A25]]
+; CHECK-NEXT:    [[A27:%.*]] = mul i32 [[A26]], [[A26]]
+; CHECK-NEXT:    [[A28:%.*]] = mul i32 [[A27]], [[A27]]
+; CHECK-NEXT:    [[A29:%.*]] = mul i32 [[A28]], [[A28]]
+; CHECK-NEXT:    [[A30:%.*]] = mul i32 [[A29]], [[A29]]
+; CHECK-NEXT:    [[COND:%.*]] = trunc i32 [[A30]] to i1
+; CHECK-NEXT:    [[WIDE_CHK:%.*]] = and i1 true, [[COND]]
+; CHECK-NEXT:    [[GUARD_CHK:%.*]] = and i1 [[WIDE_CHK]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[GUARD_CHK]], label [[GUARDED:%.*]], label [[DEOPT:%.*]], !prof !0
+; CHECK:       deopt:
+; CHECK-NEXT:    call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+; CHECK-NEXT:    ret void
+; CHECK:       guarded:
+; CHECK-NEXT:    [[WIDENABLE_COND3:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND4:%.*]] = and i1 [[COND]], [[WIDENABLE_COND3]]
+; CHECK-NEXT:    br i1 true, label [[GUARDED1:%.*]], label [[DEOPT2:%.*]], !prof !0
+; CHECK:       deopt2:
+; CHECK-NEXT:    call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+; CHECK-NEXT:    ret void
+; CHECK:       guarded1:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %widenable_cond = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond = and i1 true, %widenable_cond
+  br i1 %exiplicit_guard_cond, label %guarded, label %deopt, !prof !0
+
+deopt:                                            ; preds = %entry
+  call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+  ret void
+
+guarded:                                          ; preds = %entry
+  %a1 = mul i32 %a0, %a0
+  %a2 = mul i32 %a1, %a1
+  %a3 = mul i32 %a2, %a2
+  %a4 = mul i32 %a3, %a3
+  %a5 = mul i32 %a4, %a4
+  %a6 = mul i32 %a5, %a5
+  %a7 = mul i32 %a6, %a6
+  %a8 = mul i32 %a7, %a7
+  %a9 = mul i32 %a8, %a8
+  %a10 = mul i32 %a9, %a9
+  %a11 = mul i32 %a10, %a10
+  %a12 = mul i32 %a11, %a11
+  %a13 = mul i32 %a12, %a12
+  %a14 = mul i32 %a13, %a13
+  %a15 = mul i32 %a14, %a14
+  %a16 = mul i32 %a15, %a15
+  %a17 = mul i32 %a16, %a16
+  %a18 = mul i32 %a17, %a17
+  %a19 = mul i32 %a18, %a18
+  %a20 = mul i32 %a19, %a19
+  %a21 = mul i32 %a20, %a20
+  %a22 = mul i32 %a21, %a21
+  %a23 = mul i32 %a22, %a22
+  %a24 = mul i32 %a23, %a23
+  %a25 = mul i32 %a24, %a24
+  %a26 = mul i32 %a25, %a25
+  %a27 = mul i32 %a26, %a26
+  %a28 = mul i32 %a27, %a27
+  %a29 = mul i32 %a28, %a28
+  %a30 = mul i32 %a29, %a29
+  %cond = trunc i32 %a30 to i1
+  %widenable_cond3 = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond4 = and i1 %cond, %widenable_cond3
+  br i1 %exiplicit_guard_cond4, label %guarded1, label %deopt2, !prof !0
+
+deopt2:                                           ; preds = %guarded
+  call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+  ret void
+
+guarded1:                                         ; preds = %guarded
+  ret void
+}
+
+define void @f_13(i32 %a) {
+; CHECK-LABEL: @f_13(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[COND_0:%.*]] = icmp ult i32 [[A:%.*]], 14
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[COND_0]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    [[WIDE_CHK:%.*]] = icmp ult i32 [[A]], 10
+; CHECK-NEXT:    [[GUARD_CHK:%.*]] = and i1 [[WIDE_CHK]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[GUARD_CHK]], label [[GUARDED:%.*]], label [[DEOPT:%.*]], !prof !0
+; CHECK:       deopt:
+; CHECK-NEXT:    call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+; CHECK-NEXT:    ret void
+; CHECK:       guarded:
+; CHECK-NEXT:    br i1 undef, label [[LEFT:%.*]], label [[RIGHT:%.*]]
+; CHECK:       left:
+; CHECK-NEXT:    [[COND_1:%.*]] = icmp slt i32 [[A]], 10
+; CHECK-NEXT:    [[WIDENABLE_COND3:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND4:%.*]] = and i1 [[COND_1]], [[WIDENABLE_COND3]]
+; CHECK-NEXT:    br i1 true, label [[GUARDED1:%.*]], label [[DEOPT2:%.*]], !prof !0
+; CHECK:       deopt2:
+; CHECK-NEXT:    call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+; CHECK-NEXT:    ret void
+; CHECK:       guarded1:
+; CHECK-NEXT:    ret void
+; CHECK:       right:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %cond_0 = icmp ult i32 %a, 14
+  %widenable_cond = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond = and i1 %cond_0, %widenable_cond
+  br i1 %exiplicit_guard_cond, label %guarded, label %deopt, !prof !0
+
+deopt:                                            ; preds = %entry
+  call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+  ret void
+
+guarded:                                          ; preds = %entry
+  br i1 undef, label %left, label %right
+
+left:                                             ; preds = %guarded
+  %cond_1 = icmp slt i32 %a, 10
+  %widenable_cond3 = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond4 = and i1 %cond_1, %widenable_cond3
+  br i1 %exiplicit_guard_cond4, label %guarded1, label %deopt2, !prof !0
+
+deopt2:                                           ; preds = %left
+  call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+  ret void
+
+guarded1:                                         ; preds = %left
+  ret void
+
+right:                                            ; preds = %guarded
+  ret void
+}
+
+define void @f_14(i32 %a) {
+; CHECK-LABEL: @f_14(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[COND_0:%.*]] = icmp ult i32 [[A:%.*]], 14
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[COND_0]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[EXIPLICIT_GUARD_COND]], label [[GUARDED:%.*]], label [[DEOPT:%.*]], !prof !0
+; CHECK:       deopt:
+; CHECK-NEXT:    call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+; CHECK-NEXT:    ret void
+; CHECK:       guarded:
+; CHECK-NEXT:    br i1 undef, label [[LEFT:%.*]], label [[RIGHT:%.*]]
+; CHECK:       left:
+; CHECK-NEXT:    [[COND_1:%.*]] = icmp sgt i32 [[A]], 10
+; CHECK-NEXT:    [[WIDENABLE_COND3:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND4:%.*]] = and i1 [[COND_1]], [[WIDENABLE_COND3]]
+; CHECK-NEXT:    br i1 [[EXIPLICIT_GUARD_COND4]], label [[GUARDED1:%.*]], label [[DEOPT2:%.*]], !prof !0
+; CHECK:       deopt2:
+; CHECK-NEXT:    call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+; CHECK-NEXT:    ret void
+; CHECK:       guarded1:
+; CHECK-NEXT:    ret void
+; CHECK:       right:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %cond_0 = icmp ult i32 %a, 14
+  %widenable_cond = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond = and i1 %cond_0, %widenable_cond
+  br i1 %exiplicit_guard_cond, label %guarded, label %deopt, !prof !0
+
+deopt:                                            ; preds = %entry
+  call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+  ret void
+
+guarded:                                          ; preds = %entry
+  br i1 undef, label %left, label %right
+
+left:                                             ; preds = %guarded
+  %cond_1 = icmp sgt i32 %a, 10
+  %widenable_cond3 = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond4 = and i1 %cond_1, %widenable_cond3
+  br i1 %exiplicit_guard_cond4, label %guarded1, label %deopt2, !prof !0
+
+deopt2:                                           ; preds = %left
+  call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+  ret void
+
+guarded1:                                         ; preds = %left
+  ret void
+
+right:                                            ; preds = %guarded
+  ret void
+}
+
+; Make sure we do not widen guard by trivial true conditions into something.
+define void @f_15(i1 %cond_0, i1 %cond_1) {
+; CHECK-LABEL: @f_15(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[COND_0:%.*]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[EXIPLICIT_GUARD_COND]], label [[GUARDED:%.*]], label [[DEOPT:%.*]], !prof !0
+; CHECK:       deopt:
+; CHECK-NEXT:    call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+; CHECK-NEXT:    ret void
+; CHECK:       guarded:
+; CHECK-NEXT:    [[WIDENABLE_COND3:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND4:%.*]] = and i1 true, [[WIDENABLE_COND3]]
+; CHECK-NEXT:    br i1 [[EXIPLICIT_GUARD_COND4]], label [[GUARDED1:%.*]], label [[DEOPT2:%.*]], !prof !0
+; CHECK:       deopt2:
+; CHECK-NEXT:    call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+; CHECK-NEXT:    ret void
+; CHECK:       guarded1:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %widenable_cond = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond = and i1 %cond_0, %widenable_cond
+  br i1 %exiplicit_guard_cond, label %guarded, label %deopt, !prof !0
+
+deopt:                                            ; preds = %entry
+  call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+  ret void
+
+guarded:                                          ; preds = %entry
+  %widenable_cond3 = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond4 = and i1 true, %widenable_cond3
+  br i1 %exiplicit_guard_cond4, label %guarded1, label %deopt2, !prof !0
+
+deopt2:                                           ; preds = %guarded
+  call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+  ret void
+
+guarded1:                                         ; preds = %guarded
+  ret void
+}
+
+; Make sure we do not widen guard by trivial false conditions into something.
+define void @f_16(i1 %cond_0, i1 %cond_1) {
+; CHECK-LABEL: @f_16(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[COND_0:%.*]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[EXIPLICIT_GUARD_COND]], label [[GUARDED:%.*]], label [[DEOPT:%.*]], !prof !0
+; CHECK:       deopt:
+; CHECK-NEXT:    call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+; CHECK-NEXT:    ret void
+; CHECK:       guarded:
+; CHECK-NEXT:    [[WIDENABLE_COND3:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND4:%.*]] = and i1 false, [[WIDENABLE_COND3]]
+; CHECK-NEXT:    br i1 [[EXIPLICIT_GUARD_COND4]], label [[GUARDED1:%.*]], label [[DEOPT2:%.*]], !prof !0
+; CHECK:       deopt2:
+; CHECK-NEXT:    call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+; CHECK-NEXT:    ret void
+; CHECK:       guarded1:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %widenable_cond = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond = and i1 %cond_0, %widenable_cond
+  br i1 %exiplicit_guard_cond, label %guarded, label %deopt, !prof !0
+
+deopt:                                            ; preds = %entry
+  call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+  ret void
+
+guarded:                                          ; preds = %entry
+  %widenable_cond3 = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond4 = and i1 false, %widenable_cond3
+  br i1 %exiplicit_guard_cond4, label %guarded1, label %deopt2, !prof !0
+
+deopt2:                                           ; preds = %guarded
+  call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+  ret void
+
+guarded1:                                         ; preds = %guarded
+  ret void
+}
+
+declare void @llvm.experimental.deoptimize.isVoid(...)
+
+; Function Attrs: inaccessiblememonly nounwind
+declare i1 @llvm.experimental.widenable.condition() #0
+
+attributes #0 = { inaccessiblememonly nounwind }
+
+!0 = !{!"branch_weights", i32 1048576, i32 1}
diff --git a/test/Transforms/GuardWidening/mixed_guards.ll b/test/Transforms/GuardWidening/mixed_guards.ll
new file mode 100644
index 0000000000000000000000000000000000000000..58908c0ff8066d6f17262d11ff7be498f271a1e3
--- /dev/null
+++ b/test/Transforms/GuardWidening/mixed_guards.ll
@@ -0,0 +1,74 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -guard-widening-widen-branch-guards=true -guard-widening < %s        | FileCheck %s
+; RUN: opt -S -guard-widening-widen-branch-guards=true -passes=guard-widening < %s | FileCheck %s
+
+; Interaction between intrinsic and widenable condition guards.
+
+declare void @llvm.experimental.guard(i1,...)
+
+declare void @llvm.experimental.deoptimize.isVoid(...)
+
+; Function Attrs: inaccessiblememonly nounwind
+declare i1 @llvm.experimental.widenable.condition() #0
+
+; Widen condition of intrinsic guard with a condition from widenable branch.
+define void @test_01(i1 %cond_0, i1 %cond_1) {
+; CHECK-LABEL: @test_01(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[WIDE_CHK:%.*]] = and i1 [[COND_0:%.*]], [[COND_1:%.*]]
+; CHECK-NEXT:    call void (i1, ...) @llvm.experimental.guard(i1 [[WIDE_CHK]]) [ "deopt"() ]
+; CHECK-NEXT:    [[WIDENABLE_COND3:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND4:%.*]] = and i1 [[COND_1]], [[WIDENABLE_COND3]]
+; CHECK-NEXT:    br i1 true, label [[GUARDED1:%.*]], label [[DEOPT2:%.*]], !prof !0
+; CHECK:       deopt2:
+; CHECK-NEXT:    call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+; CHECK-NEXT:    ret void
+; CHECK:       guarded1:
+; CHECK-NEXT:    ret void
+;
+entry:
+  call void (i1, ...) @llvm.experimental.guard(i1 %cond_0) [ "deopt"() ]
+  %widenable_cond3 = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond4 = and i1 %cond_1, %widenable_cond3
+  br i1 %exiplicit_guard_cond4, label %guarded1, label %deopt2, !prof !0
+
+deopt2:                                           ; preds = %guarded
+  call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+  ret void
+
+guarded1:                                         ; preds = %guarded
+  ret void
+}
+
+; Widen condition of widenable condition guard with a condition from intrinsic.
+define void @test_02(i1 %cond_0, i1 %cond_1) {
+; CHECK-LABEL: @test_02(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[COND_0:%.*]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    [[WIDE_CHK:%.*]] = and i1 [[COND_0]], [[COND_1:%.*]]
+; CHECK-NEXT:    [[GUARD_CHK:%.*]] = and i1 [[WIDE_CHK]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[GUARD_CHK]], label [[GUARDED:%.*]], label [[DEOPT:%.*]], !prof !0
+; CHECK:       deopt:
+; CHECK-NEXT:    call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+; CHECK-NEXT:    ret void
+; CHECK:       guarded:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %widenable_cond = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond = and i1 %cond_0, %widenable_cond
+  br i1 %exiplicit_guard_cond, label %guarded, label %deopt, !prof !0
+
+deopt:                                            ; preds = %entry
+  call void (...) @llvm.experimental.deoptimize.isVoid() [ "deopt"() ]
+  ret void
+
+guarded:                                          ; preds = %entry
+  call void (i1, ...) @llvm.experimental.guard(i1 %cond_1) [ "deopt"() ]
+  ret void
+}
+
+attributes #0 = { inaccessiblememonly nounwind }
+
+!0 = !{!"branch_weights", i32 1048576, i32 1}
diff --git a/test/Transforms/HotColdSplit/do-not-split.ll b/test/Transforms/HotColdSplit/X86/do-not-split.ll
similarity index 79%
rename from test/Transforms/HotColdSplit/do-not-split.ll
rename to test/Transforms/HotColdSplit/X86/do-not-split.ll
index d5a8c44cc04f2b06a9eb00672e7d74fb47d81d50..448e63ab1de2ae4985152d9f2b49e7ae99fad806 100644
--- a/test/Transforms/HotColdSplit/do-not-split.ll
+++ b/test/Transforms/HotColdSplit/X86/do-not-split.ll
@@ -1,5 +1,4 @@
-; RUN: opt -hotcoldsplit -S < %s | FileCheck %s
-; RUN: opt -passes=hotcoldsplit -S < %s | FileCheck %s
+; RUN: opt -hotcoldsplit -hotcoldsplit-threshold=2 -S < %s | FileCheck %s
 
 target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.14.0"
@@ -29,7 +28,6 @@ entry:
   br i1 undef, label %if.then, label %if.end
 
 if.then:                                          ; preds = %entry
-  call void @sink()
   call void @sink()
   ret void
 
@@ -45,6 +43,37 @@ entry:
   br i1 undef, label %if.then, label %if.end
 
 if.then:                                          ; preds = %entry
+  call void @sink()
+  br label %if.end
+
+if.end:                                           ; preds = %entry
+  ret void
+}
+
+; Do not split `noinline` functions.
+; CHECK-LABEL: @noinline_func
+; CHECK-NOT: noinline_func.cold.1
+define void @noinline_func() noinline {
+entry:
+  br i1 undef, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  call void @sink()
+  br label %if.end
+
+if.end:                                           ; preds = %entry
+  ret void
+}
+
+; Do not split `alwaysinline` functions.
+; CHECK-LABEL: @alwaysinline_func
+; CHECK-NOT: alwaysinline_func.cold.1
+define void @alwaysinline_func() alwaysinline {
+entry:
+  br i1 undef, label %if.then, label %if.end
+
+if.then:                                          ; preds = %entry
+  call void @sink()
   br label %if.end
 
 if.end:                                           ; preds = %entry
@@ -59,8 +88,6 @@ entry:
   br label %loop
 
 loop:
-  call void @sink()
-  call void @sink()
   call void @sink()
   br label %loop
 }
diff --git a/test/Transforms/HotColdSplit/X86/outline-expensive.ll b/test/Transforms/HotColdSplit/X86/outline-expensive.ll
deleted file mode 100644
index 5b0cceae2af321e0b35fdaab7d255e2d84fc631f..0000000000000000000000000000000000000000
--- a/test/Transforms/HotColdSplit/X86/outline-expensive.ll
+++ /dev/null
@@ -1,25 +0,0 @@
-; The magic number 6 comes from (1 * TCC_Expensive) + (1 * CostOfCallX86).
-; RUN: opt -hotcoldsplit -min-outlining-thresh=6 -S < %s | FileCheck %s
-
-; Test that we outline even though there are only two cold instructions. TTI
-; should determine that they are expensive in terms of code size.
-
-target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
-target triple = "x86_64-apple-macosx10.14.0"
-
-; CHECK-LABEL: @fun
-; CHECK: call void @fun.cold.1
-define void @fun(i32 %x) {
-entry:
-  br i1 undef, label %if.then, label %if.else
-
-if.then:
-  ret void
-
-if.else:
-  %y = sdiv i32 %x, 111
-  call void @sink(i32 %y)
-  ret void
-}
-
-declare void @sink(i32 %x) cold
diff --git a/test/Transforms/HotColdSplit/addr-taken.ll b/test/Transforms/HotColdSplit/addr-taken.ll
new file mode 100644
index 0000000000000000000000000000000000000000..19f1d4f1974b1417664dbb9e5fafc5e0ff859092
--- /dev/null
+++ b/test/Transforms/HotColdSplit/addr-taken.ll
@@ -0,0 +1,27 @@
+; RUN: opt -hotcoldsplit -hotcoldsplit-threshold=-1 -S < %s | FileCheck %s
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.14.0"
+
+; CHECK: define {{.*}} @foo{{.*}}#[[outlined_func_attr:[0-9]+]]
+define void @foo() noreturn cold {
+  unreachable
+}
+
+; CHECK: define {{.*}} @bar.cold.1{{.*}}#[[outlined_func_attr]]
+define void @bar() {
+  br i1 undef, label %normal, label %exit
+
+normal:
+  unreachable
+
+exit:
+  ret void
+}
+
+@take_addr_of_foo = global void ()* @foo
+@take_addr_of_bar = global void ()* @bar
+
+; CHECK: attributes #[[outlined_func_attr]] = {
+; CHECK-SAME: cold
+; CHECK-SAME: minsize
diff --git a/test/Transforms/HotColdSplit/apply-noreturn-bonus.ll b/test/Transforms/HotColdSplit/apply-noreturn-bonus.ll
new file mode 100644
index 0000000000000000000000000000000000000000..c1d9af88595c5f1fd86543decf207efdafb233b1
--- /dev/null
+++ b/test/Transforms/HotColdSplit/apply-noreturn-bonus.ll
@@ -0,0 +1,26 @@
+; REQUIRES: asserts
+; RUN: opt -hotcoldsplit -debug-only=hotcoldsplit -S < %s -o /dev/null 2>&1 | FileCheck %s
+
+declare void @sink() cold
+
+define void @foo(i32 %arg) {
+entry:
+  br i1 undef, label %cold1, label %exit
+
+cold1:
+  ; CHECK: Applying bonus for: 4 non-returning terminators
+  call void @sink()
+  br i1 undef, label %cold2, label %cold3
+
+cold2:
+  br label %cold4
+
+cold3:
+  br label %cold4
+
+cold4:
+  unreachable
+
+exit:
+  ret void
+}
diff --git a/test/Transforms/HotColdSplit/apply-penalty-for-inputs.ll b/test/Transforms/HotColdSplit/apply-penalty-for-inputs.ll
new file mode 100644
index 0000000000000000000000000000000000000000..fffd6f9f5dcf98f1f86cedb454bbd14d309c8851
--- /dev/null
+++ b/test/Transforms/HotColdSplit/apply-penalty-for-inputs.ll
@@ -0,0 +1,19 @@
+; REQUIRES: asserts
+; RUN: opt -hotcoldsplit -debug-only=hotcoldsplit -S < %s -o /dev/null 2>&1 | FileCheck %s
+
+declare void @sink(i32*, i32, i32) cold
+
+@g = global i32 0
+
+define void @foo(i32 %arg) {
+  %local = load i32, i32* @g
+  br i1 undef, label %cold, label %exit
+
+cold:
+  ; CHECK: Applying penalty for: 2 inputs
+  call void @sink(i32* @g, i32 %arg, i32 %local)
+  ret void
+
+exit:
+  ret void
+}
diff --git a/test/Transforms/HotColdSplit/apply-penalty-for-outputs.ll b/test/Transforms/HotColdSplit/apply-penalty-for-outputs.ll
new file mode 100644
index 0000000000000000000000000000000000000000..a7d9f97ab030b9adae1555bc5f7562f6c545da82
--- /dev/null
+++ b/test/Transforms/HotColdSplit/apply-penalty-for-outputs.ll
@@ -0,0 +1,22 @@
+; REQUIRES: asserts
+; RUN: opt -hotcoldsplit -debug-only=hotcoldsplit -S < %s -o /dev/null 2>&1 | FileCheck %s
+
+declare void @sink() cold
+
+@g = global i32 0
+
+define i32 @foo(i32 %arg) {
+entry:
+  br i1 undef, label %cold, label %exit
+
+cold:
+  ; CHECK: Applying penalty for: 1 output
+  ; CHECK: Applying penalty for: 1 non-region successors
+  %local = load i32, i32* @g
+  call void @sink()
+  br label %exit
+
+exit:
+  %p = phi i32 [ %local, %cold ], [ 0, %entry ]
+  ret i32 %p
+}
diff --git a/test/Transforms/HotColdSplit/apply-successor-penalty.ll b/test/Transforms/HotColdSplit/apply-successor-penalty.ll
new file mode 100644
index 0000000000000000000000000000000000000000..3886d76da01699e8e1a88ba826604b7cc0425dea
--- /dev/null
+++ b/test/Transforms/HotColdSplit/apply-successor-penalty.ll
@@ -0,0 +1,53 @@
+; REQUIRES: asserts
+; RUN: opt -hotcoldsplit -debug-only=hotcoldsplit -S < %s -o /dev/null 2>&1 | FileCheck %s
+
+declare void @sink() cold
+
+; CHECK-LABEL: Outlining in one_non_region_successor
+define void @one_non_region_successor(i32 %arg) {
+entry:
+  br i1 undef, label %cold1, label %exit
+
+cold1:
+  ; CHECK: Applying penalty for: 1 non-region successor
+  call void @sink()
+  br i1 undef, label %cold2, label %cold3
+
+cold2:
+  br i1 undef, label %cold4, label %exit
+
+cold3:
+  br i1 undef, label %cold4, label %exit
+
+cold4:
+  unreachable
+
+exit:
+  ret void
+}
+
+; CHECK-LABEL: Outlining in two_non_region_successor
+define void @two_non_region_successors(i32 %arg) {
+entry:
+  br i1 undef, label %cold1, label %exit1
+
+cold1:
+  ; CHECK: Applying penalty for: 2 non-region successors
+  call void @sink()
+  br i1 undef, label %cold2, label %cold3
+
+cold2:
+  br i1 undef, label %cold4, label %exit1
+
+cold3:
+  br i1 undef, label %cold4, label %exit2
+
+cold4:
+  unreachable
+
+exit1:
+  br label %exit2
+
+exit2:
+  ret void
+}
diff --git a/test/Transforms/HotColdSplit/coldentrycount.ll b/test/Transforms/HotColdSplit/coldentrycount.ll
new file mode 100644
index 0000000000000000000000000000000000000000..d63acc188f5441ad3413d08e23acc7ae4b531f50
--- /dev/null
+++ b/test/Transforms/HotColdSplit/coldentrycount.ll
@@ -0,0 +1,43 @@
+; Test to ensure that split cold function gets 0 entry count profile
+; metadata when compiling with pgo.
+
+; RUN: opt -hotcoldsplit -hotcoldsplit-threshold=0 -S < %s | FileCheck %s
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.14.0"
+
+; CHECK-LABEL: @fun
+; CHECK: call void @fun.cold.1
+define void @fun() !prof !14 {
+entry:
+  br i1 undef, label %if.then, label %if.else
+
+if.then:
+  ret void
+
+if.else:
+  call void @sink()
+  ret void
+}
+
+declare void @sink() cold
+
+; CHECK: define {{.*}} @fun.cold.1{{.*}} ![[PROF:[0-9]+]]
+; CHECK: ![[PROF]] = !{!"function_entry_count", i64 0}
+
+!llvm.module.flags = !{!0}
+!0 = !{i32 1, !"ProfileSummary", !1}
+!1 = !{!2, !3, !4, !5, !6, !7, !8, !9}
+!2 = !{!"ProfileFormat", !"InstrProf"}
+!3 = !{!"TotalCount", i64 10000}
+!4 = !{!"MaxCount", i64 10}
+!5 = !{!"MaxInternalCount", i64 1}
+!6 = !{!"MaxFunctionCount", i64 1000}
+!7 = !{!"NumCounts", i64 3}
+!8 = !{!"NumFunctions", i64 3}
+!9 = !{!"DetailedSummary", !10}
+!10 = !{!11, !12, !13}
+!11 = !{i32 10000, i64 100, i32 1}
+!12 = !{i32 999000, i64 100, i32 1}
+!13 = !{i32 999999, i64 1, i32 2}
+!14 = !{!"function_entry_count", i64 100}
diff --git a/test/Transforms/HotColdSplit/delete-use-without-def-dbg-val.ll b/test/Transforms/HotColdSplit/delete-use-without-def-dbg-val.ll
index 878db4863807591dbbb409f20f53b502415f6cde..97f5cb8ba6c6dd339966e23cc44a3b968b5f74a4 100644
--- a/test/Transforms/HotColdSplit/delete-use-without-def-dbg-val.ll
+++ b/test/Transforms/HotColdSplit/delete-use-without-def-dbg-val.ll
@@ -1,4 +1,4 @@
-; RUN: opt -hotcoldsplit -S < %s | FileCheck %s
+; RUN: opt -hotcoldsplit -hotcoldsplit-threshold=0 -S < %s | FileCheck %s
 
 target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.14.0"
@@ -20,8 +20,6 @@ if.end:                                           ; preds = %entry
   ; We expect this block to be outlined. That kills the definition of %var.
   %var = add i32 0, 0, !dbg !11
   call void @sink()
-  call void @sink()
-  call void @sink()
   br label %cleanup
 
 cleanup:
diff --git a/test/Transforms/HotColdSplit/duplicate-phi-preds-crash.ll b/test/Transforms/HotColdSplit/duplicate-phi-preds-crash.ll
index 5b697727e7d36c5f21d88b0798c92cd5459e45f7..fa68300b0d86b4d12ecbf09a8e4aba4b9a95d1fe 100644
--- a/test/Transforms/HotColdSplit/duplicate-phi-preds-crash.ll
+++ b/test/Transforms/HotColdSplit/duplicate-phi-preds-crash.ll
@@ -1,4 +1,4 @@
-; RUN: opt -S -hotcoldsplit < %s | FileCheck %s
+; RUN: opt -S -hotcoldsplit -hotcoldsplit-threshold=0 < %s | FileCheck %s
 
 target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.14.0"
@@ -15,7 +15,7 @@ declare void @sink() cold
 ; CHECK: call {{.*}}@sideeffect(
 ; CHECK: call {{.*}}@realloc(
 ; CHECK-LABEL: codeRepl:
-; CHECK-NEXT: call {{.*}}@realloc2.cold.1(i64 %size, i8* %ptr, i8** %retval.0.ce.loc)
+; CHECK: call {{.*}}@realloc2.cold.1(i64 %size, i8* %ptr, i8** %retval.0.ce.loc)
 ; CHECK-LABEL: cleanup:
 ; CHECK-NEXT: phi i8* [ null, %if.then ], [ %call, %if.end ], [ %retval.0.ce.reload, %codeRepl ]
 define i8* @realloc2(i8* %ptr, i64 %size) {
diff --git a/test/Transforms/HotColdSplit/eh-pads.ll b/test/Transforms/HotColdSplit/eh-pads.ll
index 1197a54bc32f9b27a110b2dca1e5275953408485..80a566c6cf1ff52efc5292d2d6e707ae4786088d 100644
--- a/test/Transforms/HotColdSplit/eh-pads.ll
+++ b/test/Transforms/HotColdSplit/eh-pads.ll
@@ -1,4 +1,4 @@
-; RUN: opt -S -hotcoldsplit < %s | FileCheck %s
+; RUN: opt -S -hotcoldsplit -hotcoldsplit-threshold=0 < %s | FileCheck %s
 
 target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.14.0"
@@ -17,7 +17,6 @@ exception:
 
 continue_exception:
   call void @sideeffect(i32 0)
-  call void @sideeffect(i32 1)
   call void @sink()
   ret void
 
@@ -44,19 +43,55 @@ continue:
 exception:
   ; Note: EH pads are not candidates for region entry points.
   %cleanup = landingpad i8 cleanup
-  ret void
+  br label %trivial-eh-handler
+
+trivial-eh-handler:
+  call void @sideeffect(i32 1)
+  br label %normal
 
 normal:
   call void @sideeffect(i32 0)
+  ret void
+}
+
+define void @baz() personality i8 0 {
+entry:
+  br i1 undef, label %exit, label %cold1
+
+exit:
+  ret void
+
+cold1:
+  ; The predecessor of a cold invoke may still be extracted (see baz.cold.2).
+  call void @sideeffect(i32 0)
+  br label %cold2
+
+cold2:
+  invoke void @sink() to label %cold3 unwind label %cold4
+
+cold3:
+  ; The successor of a cold invoke may still be extracted (see baz.cold.1).
   call void @sideeffect(i32 1)
   ret void
+
+cold4:
+  landingpad i8 cleanup
+  ret void
 }
 
 ; CHECK-LABEL: define {{.*}}@foo.cold.1(
 ; CHECK: sideeffect(i32 0)
-; CHECK: sideeffect(i32 1)
 ; CHECK: sink
 
+; CHECK-LABEL: define {{.*}}@bar.cold.1(
+; CHECK: sideeffect(i32 1)
+
+; CHECK-LABEL: define {{.*}}@baz.cold.1(
+; CHECK: sideeffect(i32 1)
+
+; CHECK-LABEL: define {{.*}}@baz.cold.2(
+; CHECK: sideeffect(i32 0)
+
 declare void @sideeffect(i32)
 
 declare void @sink() cold
diff --git a/test/Transforms/HotColdSplit/eh-typeid-for.ll b/test/Transforms/HotColdSplit/eh-typeid-for.ll
index 75f9e672332e21eef0fece6f7efffe3df16507e8..87a5bcc9131eac447cf177d3969d55875def3383 100644
--- a/test/Transforms/HotColdSplit/eh-typeid-for.ll
+++ b/test/Transforms/HotColdSplit/eh-typeid-for.ll
@@ -1,4 +1,4 @@
-; RUN: opt -hotcoldsplit -S < %s | FileCheck %s
+; RUN: opt -hotcoldsplit -hotcoldsplit-threshold=0 -S < %s | FileCheck %s
 
 ; Do not outline calls to @llvm.eh.typeid.for. See llvm.org/PR39545.
 
@@ -16,8 +16,6 @@ if.then:
 if.else:
   %t = call i32 @llvm.eh.typeid.for(i8* bitcast (i8** @_ZTIi to i8*))
   call void @sink()
-  call void @sink()
-  call void @sink()
   ret void
 }
 
diff --git a/test/Transforms/HotColdSplit/extraction-subregion-breaks-phis.ll b/test/Transforms/HotColdSplit/extraction-subregion-breaks-phis.ll
deleted file mode 100644
index 11256a443fa70245f78c72d52b946016112eb504..0000000000000000000000000000000000000000
--- a/test/Transforms/HotColdSplit/extraction-subregion-breaks-phis.ll
+++ /dev/null
@@ -1,63 +0,0 @@
-; RUN: opt -S -hotcoldsplit < %s | FileCheck %s
-
-target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
-target triple = "x86_64-apple-macosx10.14.0"
-
-; CHECK-LABEL: define {{.*}}@foo(
-; CHECK: call {{.*}}@foo.cold.1(
-; CHECK: unreachable
-
-; CHECK-LABEL: define {{.*}}@foo.cold.1(
-; CHECK: switch i32 undef, label %sw.epilog.i
-define void @foo(i32 %QMM) {
-entry:
-  switch i32 %QMM, label %entry.if.end16_crit_edge [
-    i32 1, label %if.then
-  ]
-
-entry.if.end16_crit_edge:                         ; preds = %entry
-  br label %if.end16
-
-if.then:                                          ; preds = %entry
-  br i1 undef, label %cond.true.i.i, label %_ZN10StringView8popFrontEv.exit.i
-
-cond.true.i.i:                                    ; preds = %if.then
-  ret void
-
-_ZN10StringView8popFrontEv.exit.i:                ; preds = %if.then
-  switch i32 undef, label %sw.epilog.i [
-    i32 81, label %if.end16
-    i32 82, label %sw.bb4.i
-    i32 83, label %sw.bb8.i
-    i32 84, label %sw.bb12.i
-    i32 65, label %if.end16
-    i32 66, label %sw.bb20.i
-    i32 67, label %sw.bb24.i
-    i32 68, label %sw.bb28.i
-  ]
-
-sw.bb4.i:                                         ; preds = %_ZN10StringView8popFrontEv.exit.i
-  br label %if.end16
-
-sw.bb8.i:                                         ; preds = %_ZN10StringView8popFrontEv.exit.i
-  br label %if.end16
-
-sw.bb12.i:                                        ; preds = %_ZN10StringView8popFrontEv.exit.i
-  br label %if.end16
-
-sw.bb20.i:                                        ; preds = %_ZN10StringView8popFrontEv.exit.i
-  br label %if.end16
-
-sw.bb24.i:                                        ; preds = %_ZN10StringView8popFrontEv.exit.i
-  br label %if.end16
-
-sw.bb28.i:                                        ; preds = %_ZN10StringView8popFrontEv.exit.i
-  br label %if.end16
-
-sw.epilog.i:                                      ; preds = %_ZN10StringView8popFrontEv.exit.i
-  br label %if.end16
-
-if.end16:                                         ; preds = %sw.epilog.i, %sw.bb28.i, %sw.bb24.i, %sw.bb20.i, %sw.bb12.i, %sw.bb8.i, %sw.bb4.i, %_ZN10StringView8popFrontEv.exit.i, %_ZN10StringView8popFrontEv.exit.i, %entry.if.end16_crit_edge
-  %0 = phi i8 [ 0, %entry.if.end16_crit_edge ], [ 0, %_ZN10StringView8popFrontEv.exit.i ], [ 0, %_ZN10StringView8popFrontEv.exit.i ], [ 1, %sw.bb4.i ], [ 2, %sw.bb8.i ], [ 3, %sw.bb12.i ], [ 1, %sw.bb20.i ], [ 2, %sw.bb24.i ], [ 3, %sw.bb28.i ], [ 0, %sw.epilog.i ]
-  unreachable
-}
diff --git a/test/Transforms/HotColdSplit/forward-dfs-reaches-marked-block.ll b/test/Transforms/HotColdSplit/forward-dfs-reaches-marked-block.ll
index 81a8f4231593440ef6a5970b8844d9d1a70fd24f..3c0f93b0985c961b960815a43d8a87892889db65 100644
--- a/test/Transforms/HotColdSplit/forward-dfs-reaches-marked-block.ll
+++ b/test/Transforms/HotColdSplit/forward-dfs-reaches-marked-block.ll
@@ -1,4 +1,4 @@
-; RUN: opt -hotcoldsplit -S < %s | FileCheck %s
+; RUN: opt -hotcoldsplit -hotcoldsplit-threshold=0 -S < %s | FileCheck %s
 
 target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.14.0"
diff --git a/test/Transforms/HotColdSplit/lifetime-markers-on-inputs-1.ll b/test/Transforms/HotColdSplit/lifetime-markers-on-inputs-1.ll
new file mode 100644
index 0000000000000000000000000000000000000000..e80e2b970772a32dc228b1ca4d29fa26da2f1bff
--- /dev/null
+++ b/test/Transforms/HotColdSplit/lifetime-markers-on-inputs-1.ll
@@ -0,0 +1,66 @@
+; RUN: opt -S -hotcoldsplit -hotcoldsplit-threshold=0 < %s 2>&1 | FileCheck %s
+
+declare void @llvm.lifetime.start.p0i8(i64, i8* nocapture)
+
+declare void @llvm.lifetime.end.p0i8(i64, i8* nocapture)
+
+declare void @use(i8*)
+
+declare void @cold_use2(i8*, i8*) cold
+
+; CHECK-LABEL: define {{.*}}@foo(
+define void @foo() {
+entry:
+  %local1 = alloca i256
+  %local2 = alloca i256
+  %local1_cast = bitcast i256* %local1 to i8*
+  %local2_cast = bitcast i256* %local2 to i8*
+  br i1 undef, label %normalPath, label %outlinedPath
+
+normalPath:
+  ; These two uses of stack slots are non-overlapping. Based on this alone,
+  ; the stack slots could be merged.
+  call void @llvm.lifetime.start.p0i8(i64 1, i8* %local1_cast)
+  call void @use(i8* %local1_cast)
+  call void @llvm.lifetime.end.p0i8(i64 1, i8* %local1_cast)
+  call void @llvm.lifetime.start.p0i8(i64 1, i8* %local2_cast)
+  call void @use(i8* %local2_cast)
+  call void @llvm.lifetime.end.p0i8(i64 1, i8* %local2_cast)
+  ret void
+
+; CHECK-LABEL: codeRepl:
+; CHECK: [[local1_cast:%.*]] = bitcast i256* %local1 to i8*
+; CHECK-NEXT: call void @llvm.lifetime.start.p0i8(i64 -1, i8* [[local1_cast]])
+; CHECK-NEXT: [[local2_cast:%.*]] = bitcast i256* %local2 to i8*
+; CHECK-NEXT: call void @llvm.lifetime.start.p0i8(i64 -1, i8* [[local2_cast]])
+; CHECK-NEXT: call i1 @foo.cold.1(i8* %local1_cast, i8* %local2_cast)
+; CHECK-NEXT: call void @llvm.lifetime.end.p0i8(i64 -1, i8* [[local1_cast]])
+; CHECK-NEXT: call void @llvm.lifetime.end.p0i8(i64 -1, i8* [[local2_cast]])
+; CHECK-NEXT: br i1
+
+outlinedPath:
+  ; These two uses of stack slots are overlapping. This should prevent
+  ; merging of stack slots. CodeExtractor must replicate the effects of
+  ; these markers in the caller to inhibit stack coloring.
+  %gep1 = getelementptr inbounds i8, i8* %local1_cast, i64 1
+  call void @llvm.lifetime.start.p0i8(i64 1, i8* %gep1)
+  call void @llvm.lifetime.start.p0i8(i64 1, i8* %local2_cast)
+  call void @cold_use2(i8* %local1_cast, i8* %local2_cast)
+  call void @llvm.lifetime.end.p0i8(i64 1, i8* %gep1)
+  call void @llvm.lifetime.end.p0i8(i64 1, i8* %local2_cast)
+  br i1 undef, label %outlinedPath2, label %outlinedPathExit
+
+outlinedPath2:
+  ; These extra lifetime markers are used to test that we emit only one
+  ; pair of guard markers in the caller per memory object.
+  call void @llvm.lifetime.start.p0i8(i64 1, i8* %local2_cast)
+  call void @use(i8* %local2_cast)
+  call void @llvm.lifetime.end.p0i8(i64 1, i8* %local2_cast)
+  ret void
+
+outlinedPathExit:
+  ret void
+}
+
+; CHECK-LABEL: define {{.*}}@foo.cold.1(
+; CHECK-NOT: @llvm.lifetime
diff --git a/test/Transforms/HotColdSplit/lifetime-markers-on-inputs-2.ll b/test/Transforms/HotColdSplit/lifetime-markers-on-inputs-2.ll
new file mode 100644
index 0000000000000000000000000000000000000000..d9a30f78f76552e502e4b91f2bddbde3f18e0e4a
--- /dev/null
+++ b/test/Transforms/HotColdSplit/lifetime-markers-on-inputs-2.ll
@@ -0,0 +1,135 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -hotcoldsplit -hotcoldsplit-threshold=0 < %s 2>&1 | FileCheck %s
+
+declare void @llvm.lifetime.start.p0i8(i64, i8* nocapture)
+
+declare void @llvm.lifetime.end.p0i8(i64, i8* nocapture)
+
+declare void @cold_use(i8*) cold
+
+; In this CFG, splitting will extract the blocks extract{1,2}. I.e., it will
+; extract a lifetime.start marker, but not the corresponding lifetime.end
+; marker. Make sure that a lifetime.start marker is emitted before the call to
+; the split function, and *only* that marker.
+;
+;            entry
+;          /       \
+;      extract1  no-extract1
+;     (lt.start)    |
+;    /              |
+; extract2          |
+;    \_____         |
+;          \      /
+;            exit
+;          (lt.end)
+;
+; After splitting, we should see:
+;
+;            entry
+;          /       \
+;      codeRepl  no-extract1
+;     (lt.start)   |
+;          \      /
+;            exit
+;          (lt.end)
+define void @only_lifetime_start_is_cold() {
+; CHECK-LABEL: @only_lifetime_start_is_cold(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[LOCAL1:%.*]] = alloca i256
+; CHECK-NEXT:    [[LOCAL1_CAST:%.*]] = bitcast i256* [[LOCAL1]] to i8*
+; CHECK-NEXT:    br i1 undef, label [[CODEREPL:%.*]], label [[NO_EXTRACT1:%.*]]
+; CHECK:       codeRepl:
+; CHECK-NEXT:    [[LT_CAST:%.*]] = bitcast i256* [[LOCAL1]] to i8*
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0i8(i64 -1, i8* [[LT_CAST]])
+; CHECK-NEXT:    [[TARGETBLOCK:%.*]] = call i1 @only_lifetime_start_is_cold.cold.1(i8* [[LOCAL1_CAST]]) #3
+; CHECK-NEXT:    br i1 [[TARGETBLOCK]], label [[NO_EXTRACT1]], label [[EXIT:%.*]]
+; CHECK:       no-extract1:
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0i8(i64 1, i8* [[LOCAL1_CAST]])
+; CHECK-NEXT:    ret void
+;
+entry:
+  %local1 = alloca i256
+  %local1_cast = bitcast i256* %local1 to i8*
+  br i1 undef, label %extract1, label %no-extract1
+
+extract1:
+  ; lt.start
+  call void @llvm.lifetime.start.p0i8(i64 1, i8* %local1_cast)
+  call void @cold_use(i8* %local1_cast)
+  br i1 undef, label %extract2, label %no-extract1
+
+extract2:
+  br label %exit
+
+no-extract1:
+  br label %exit
+
+exit:
+  ; lt.end
+  call void @llvm.lifetime.end.p0i8(i64 1, i8* %local1_cast)
+  ret void
+}
+
+; In this CFG, splitting will extract the block extract1. I.e., it will extract
+; a lifetime.end marker, but not the corresponding lifetime.start marker. Make
+; sure that a lifetime.end marker is emitted after the call to the split
+; function, and *only* that marker.
+;
+;            entry
+;         (lt.start)
+;        /          \
+;   no-extract1  extract1
+;    (lt.end)    (lt.end)
+;        \         /
+;            exit
+;
+; After splitting, we should see:
+;
+;            entry
+;         (lt.start)
+;        /          \
+;   no-extract1  codeRepl
+;    (lt.end)    (lt.end)
+;        \         /
+;            exit
+define void @only_lifetime_end_is_cold() {
+; CHECK-LABEL: @only_lifetime_end_is_cold(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[LOCAL1:%.*]] = alloca i256
+; CHECK-NEXT:    [[LOCAL1_CAST:%.*]] = bitcast i256* [[LOCAL1]] to i8*
+; CHECK-NEXT:    call void @llvm.lifetime.start.p0i8(i64 1, i8* [[LOCAL1_CAST]])
+; CHECK-NEXT:    br i1 undef, label [[NO_EXTRACT1:%.*]], label [[CODEREPL:%.*]]
+; CHECK:       no-extract1:
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0i8(i64 1, i8* [[LOCAL1_CAST]])
+; CHECK-NEXT:    br label [[EXIT:%.*]]
+; CHECK:       codeRepl:
+; CHECK-NEXT:    [[LT_CAST:%.*]] = bitcast i256* [[LOCAL1]] to i8*
+; CHECK-NEXT:    call void @only_lifetime_end_is_cold.cold.1(i8* [[LOCAL1_CAST]]) #3
+; CHECK-NEXT:    call void @llvm.lifetime.end.p0i8(i64 -1, i8* [[LT_CAST]])
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  ; lt.start
+  %local1 = alloca i256
+  %local1_cast = bitcast i256* %local1 to i8*
+  call void @llvm.lifetime.start.p0i8(i64 1, i8* %local1_cast)
+  br i1 undef, label %no-extract1, label %extract1
+
+no-extract1:
+  ; lt.end
+  call void @llvm.lifetime.end.p0i8(i64 1, i8* %local1_cast)
+  br label %exit
+
+extract1:
+  ; lt.end
+  call void @cold_use(i8* %local1_cast)
+  call void @llvm.lifetime.end.p0i8(i64 1, i8* %local1_cast)
+  br label %exit
+
+exit:
+  ret void
+}
diff --git a/test/Transforms/HotColdSplit/mark-the-whole-func-cold.ll b/test/Transforms/HotColdSplit/mark-the-whole-func-cold.ll
index d8bd55e46dc9cd456ae9c07250d30fd1115c156e..cc87f617b4f4dffbddfa0fd08ad12a7d0c62d51a 100644
--- a/test/Transforms/HotColdSplit/mark-the-whole-func-cold.ll
+++ b/test/Transforms/HotColdSplit/mark-the-whole-func-cold.ll
@@ -1,4 +1,4 @@
-; RUN: opt -S -hotcoldsplit < %s | FileCheck %s
+; RUN: opt -S -hotcoldsplit -hotcoldsplit-threshold=0 < %s | FileCheck %s
 
 ; Source:
 ; 
diff --git a/test/Transforms/HotColdSplit/minsize.ll b/test/Transforms/HotColdSplit/minsize.ll
index 69cd0979b94596c5458c85e713a693373bc27654..36bd03407181f81181d7a71afb15fc88341cfa78 100644
--- a/test/Transforms/HotColdSplit/minsize.ll
+++ b/test/Transforms/HotColdSplit/minsize.ll
@@ -1,4 +1,4 @@
-; RUN: opt -hotcoldsplit -S < %s | FileCheck %s
+; RUN: opt -hotcoldsplit -hotcoldsplit-threshold=0 -S < %s | FileCheck %s
 
 target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.14.0"
@@ -14,12 +14,18 @@ if.then:
 
 if.else:
   call void @sink()
-  call void @sink()
-  call void @sink()
+  ret void
+}
+
+; CHECK: define {{.*}} @foo{{.*}}#[[outlined_func_attr:[0-9]+]]
+define void @foo() cold {
   ret void
 }
 
 declare void @sink() cold
 
-; CHECK: define {{.*}} @fun.cold.1{{.*}}#[[outlined_func_attr:[0-9]+]]
-; CHECK: attributes #[[outlined_func_attr]] = { {{.*}}minsize
+; CHECK: define {{.*}} @fun.cold.1{{.*}}#[[outlined_func_attr]]
+
+; CHECK: attributes #[[outlined_func_attr]] = {
+; CHECK-SAME: cold
+; CHECK-SAME: minsize
diff --git a/test/Transforms/HotColdSplit/multiple-exits.ll b/test/Transforms/HotColdSplit/multiple-exits.ll
index 2e7cf84f72e49131c3a7891f849911d51a3c15ef..c41e4f04bf36044d4dbbf40f2d675eacf924e83b 100644
--- a/test/Transforms/HotColdSplit/multiple-exits.ll
+++ b/test/Transforms/HotColdSplit/multiple-exits.ll
@@ -1,4 +1,4 @@
-; RUN: opt -S -hotcoldsplit < %s | FileCheck %s
+; RUN: opt -S -hotcoldsplit -hotcoldsplit-threshold=0 < %s | FileCheck %s
 
 ; Source:
 ;
@@ -55,7 +55,6 @@ return:                                           ; preds = %exit2, %exit1
 }
 
 ; CHECK-LABEL: define {{.*}}@foo.cold.1(
-; TODO: Eliminate this unnecessary unconditional branch.
 ; CHECK: br
 ; CHECK: [[exit1Stub:.*]]:
 ; CHECK-NEXT: ret i1 true
diff --git a/test/Transforms/HotColdSplit/noreturn.ll b/test/Transforms/HotColdSplit/noreturn.ll
index 66de93432b87511272531bb41ebc1cd480f133b8..ca0f58815bf430932260a60dd81b11c987e5ac48 100644
--- a/test/Transforms/HotColdSplit/noreturn.ll
+++ b/test/Transforms/HotColdSplit/noreturn.ll
@@ -1,4 +1,4 @@
-; RUN: opt -hotcoldsplit -S < %s | FileCheck %s
+; RUN: opt -hotcoldsplit -hotcoldsplit-threshold=0 -S < %s | FileCheck %s
 
 target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.14.0"
@@ -33,8 +33,6 @@ define void @bar(i32) {
   br i1 %2, label %sink, label %exit
 
 sink:
-  tail call void @_Z10sideeffectv()
-  tail call void @_Z10sideeffectv()
   tail call void @_Z10sideeffectv()
   call void @llvm.trap()
   unreachable
diff --git a/test/Transforms/HotColdSplit/outline-cold-asm.ll b/test/Transforms/HotColdSplit/outline-cold-asm.ll
index f3685030c1d8b136eec0f81fcaea3c41d29e05b7..8c1a1a88ecf4c0e800be43c8532bbd0d616cfee1 100644
--- a/test/Transforms/HotColdSplit/outline-cold-asm.ll
+++ b/test/Transforms/HotColdSplit/outline-cold-asm.ll
@@ -1,4 +1,4 @@
-; RUN: opt -hotcoldsplit -S < %s | FileCheck %s
+; RUN: opt -hotcoldsplit -hotcoldsplit-threshold=0 -S < %s | FileCheck %s
 
 target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.14.0"
@@ -19,8 +19,6 @@ if.then:
 if.else:
   call void asm "", ""()
   call void @sink()
-  call void @sink()
-  call void @sink()
   ret void
 }
 
diff --git a/test/Transforms/HotColdSplit/outline-disjoint-diamonds.ll b/test/Transforms/HotColdSplit/outline-disjoint-diamonds.ll
index d10240ab3795d410cf208fe4ee4be1d7ef490e6c..b33454b5c4efed2b66ad04ec628b26174ecc70f1 100644
--- a/test/Transforms/HotColdSplit/outline-disjoint-diamonds.ll
+++ b/test/Transforms/HotColdSplit/outline-disjoint-diamonds.ll
@@ -1,4 +1,4 @@
-; RUN: opt -S -hotcoldsplit < %s 2>&1 | FileCheck %s
+; RUN: opt -S -hotcoldsplit -hotcoldsplit-threshold=-1 < %s 2>&1 | FileCheck %s
 
 ; CHECK-LABEL: define {{.*}}@fun
 ; CHECK: call {{.*}}@fun.cold.2(
diff --git a/test/Transforms/HotColdSplit/outline-if-then-else.ll b/test/Transforms/HotColdSplit/outline-if-then-else.ll
index bbde7651e28121ac1da59bdf2c61fc372398dcbb..faf663c71c54c97290da8bc6d9460bf2611f27fa 100644
--- a/test/Transforms/HotColdSplit/outline-if-then-else.ll
+++ b/test/Transforms/HotColdSplit/outline-if-then-else.ll
@@ -1,4 +1,4 @@
-; RUN: opt -S -hotcoldsplit < %s | FileCheck %s
+; RUN: opt -S -hotcoldsplit -hotcoldsplit-threshold=0 < %s | FileCheck %s
 
 ; Source:
 ;
diff --git a/test/Transforms/HotColdSplit/outline-multiple-entry-region.ll b/test/Transforms/HotColdSplit/outline-multiple-entry-region.ll
index 8fd20c0cee78cc4131bbc26eb328793b1a4dc03d..0275715fc0a10b4f1814917236dd20e2bb548d3b 100644
--- a/test/Transforms/HotColdSplit/outline-multiple-entry-region.ll
+++ b/test/Transforms/HotColdSplit/outline-multiple-entry-region.ll
@@ -1,4 +1,4 @@
-; RUN: opt -S -hotcoldsplit < %s | FileCheck %s
+; RUN: opt -S -hotcoldsplit -hotcoldsplit-threshold=0 < %s | FileCheck %s
 
 ; Source:
 ;
@@ -26,11 +26,9 @@ target triple = "x86_64-apple-macosx10.14.0"
 
 ; CHECK-LABEL: define {{.*}}@_Z3fooii.cold.1
 ; CHECK: call void @_Z10sideeffecti(i32 1)
-; CHECK: call void @_Z10sideeffecti(i32 11)
 
 ; CHECK-LABEL: define {{.*}}@_Z3fooii.cold.2
 ; CHECK: call void @_Z10sideeffecti(i32 0)
-; CHECK: call void @_Z10sideeffecti(i32 10)
 
 ; CHECK-LABEL: define {{.*}}@_Z3fooii.cold.3
 ; CHECK: call void @_Z4sinkv
@@ -50,7 +48,6 @@ define void @_Z3fooii(i32, i32) {
 
 ; <label>:8:                                      ; preds = %5
   call void @_Z10sideeffecti(i32 0)
-  call void @_Z10sideeffecti(i32 10)
   br label %14
 
 ; <label>:9:                                      ; preds = %5
@@ -60,7 +57,6 @@ define void @_Z3fooii(i32, i32) {
 
 ; <label>:12:                                     ; preds = %9
   call void @_Z10sideeffecti(i32 1)
-  call void @_Z10sideeffecti(i32 11)
   br label %14
 
 ; <label>:13:                                     ; preds = %9
diff --git a/test/Transforms/HotColdSplit/outline-while-loop.ll b/test/Transforms/HotColdSplit/outline-while-loop.ll
index 4c4e1f9284e95723217b5af970fe3fadf6823a24..6337726ca13322364874be7d8545ed9ec337a133 100644
--- a/test/Transforms/HotColdSplit/outline-while-loop.ll
+++ b/test/Transforms/HotColdSplit/outline-while-loop.ll
@@ -1,4 +1,4 @@
-; RUN: opt -S -hotcoldsplit < %s | FileCheck %s
+; RUN: opt -S -hotcoldsplit -hotcoldsplit-threshold=0 < %s | FileCheck %s
 
 ; Source:
 ;
diff --git a/test/Transforms/HotColdSplit/phi-with-distinct-outlined-values.ll b/test/Transforms/HotColdSplit/phi-with-distinct-outlined-values.ll
index 6e8b13a31223b43fa0e47005cbc9d00302693919..cb12befcf463c61af9f99562239be9a16947e623 100644
--- a/test/Transforms/HotColdSplit/phi-with-distinct-outlined-values.ll
+++ b/test/Transforms/HotColdSplit/phi-with-distinct-outlined-values.ll
@@ -1,4 +1,4 @@
-; RUN: opt -S -hotcoldsplit < %s | FileCheck %s
+; RUN: opt -S -hotcoldsplit -hotcoldsplit-threshold=0 < %s | FileCheck %s
 
 target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.14.0"
@@ -19,7 +19,6 @@ entry:
 coldbb:
   call void @sink()
   call void @sideeffect()
-  call void @sideeffect()
   br i1 undef, label %if.end, label %coldbb2
 
 coldbb2:
diff --git a/test/Transforms/HotColdSplit/region-overlap.ll b/test/Transforms/HotColdSplit/region-overlap.ll
index 8064a545200f2ac9a1d715156ef60e58c035d41b..6ab65c069ec915752040e1505dca25521ef07ccd 100644
--- a/test/Transforms/HotColdSplit/region-overlap.ll
+++ b/test/Transforms/HotColdSplit/region-overlap.ll
@@ -1,4 +1,4 @@
-; RUN: opt -S -hotcoldsplit < %s | FileCheck %s
+; RUN: opt -S -hotcoldsplit -hotcoldsplit-threshold=0 < %s | FileCheck %s
 
 ; Source:
 ;
@@ -8,14 +8,12 @@
 ;     if (cond1) {
 ;         if (cond2) { // This is the first cold region we visit.
 ;             sideeffect(0);
-;             sideeffect(10);
 ;             sink(0);
 ;         }
 ;
 ;         // There's a larger, overlapping cold region here. But we ignore it.
 ;         // This could be improved.
 ;         sideeffect(1);
-;         sideeffect(11);
 ;         sink(1);
 ;     }
 ; }
@@ -42,13 +40,11 @@ define void @_Z3fooii(i32, i32) {
 
 ; <label>:10:                                     ; preds = %7
   call void @_Z10sideeffecti(i32 0)
-  call void @_Z10sideeffecti(i32 10)
   call void @_Z4sinki(i32 0) #3
   br label %11
 
 ; <label>:11:                                     ; preds = %10, %7
   call void @_Z10sideeffecti(i32 1)
-  call void @_Z10sideeffecti(i32 11)
   call void @_Z4sinki(i32 1) #3
   br label %12
 
@@ -58,7 +54,6 @@ define void @_Z3fooii(i32, i32) {
 
 ; CHECK-LABEL: define {{.*}}@_Z3fooii.cold.1
 ; CHECK: call void @_Z10sideeffecti(i32 0)
-; CHECK: call void @_Z10sideeffecti(i32 10)
 
 declare void @_Z10sideeffecti(i32)
 
diff --git a/test/Transforms/HotColdSplit/resume.ll b/test/Transforms/HotColdSplit/resume.ll
new file mode 100644
index 0000000000000000000000000000000000000000..67d2d2419167bb615886b1a3d32731822b43e190
--- /dev/null
+++ b/test/Transforms/HotColdSplit/resume.ll
@@ -0,0 +1,26 @@
+; RUN: opt -hotcoldsplit -hotcoldsplit-threshold=-1 -S < %s | FileCheck %s
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.14.0"
+
+; Consider `resume` to be cold.
+
+; CHECK-LABEL: define {{.*}}@foo.cold.1(
+; CHECK: call {{.*}}@sink(
+
+declare void @sink() cold
+
+define i32 @foo() personality i8 0 {
+entry:
+  br i1 undef, label %pre-resume-eh, label %normal
+
+pre-resume-eh:
+  call void @sink()
+  br label %resume-eh
+
+resume-eh:
+  resume i32 undef
+
+normal:
+  ret i32 0
+}
diff --git a/test/Transforms/HotColdSplit/split-cold-2.ll b/test/Transforms/HotColdSplit/split-cold-2.ll
index 66c6fbb1b8d3cbf0a8a820472b95416c609b2035..0b228a58897ccf6858b3a3015fc30c28ded96dba 100644
--- a/test/Transforms/HotColdSplit/split-cold-2.ll
+++ b/test/Transforms/HotColdSplit/split-cold-2.ll
@@ -1,5 +1,5 @@
-; RUN: opt -hotcoldsplit -pass-remarks=hotcoldsplit -S < %s 2>&1 | FileCheck %s
-; RUN: opt -passes=hotcoldsplit -pass-remarks=hotcoldsplit -S < %s 2>&1 | FileCheck %s
+; RUN: opt -hotcoldsplit -hotcoldsplit-threshold=-1 -pass-remarks=hotcoldsplit -S < %s 2>&1 | FileCheck %s
+; RUN: opt -passes=hotcoldsplit -hotcoldsplit-threshold=-1 -pass-remarks=hotcoldsplit -S < %s 2>&1 | FileCheck %s
 
 ; Make sure this compiles. This test used to fail with an invalid phi node: the
 ; two predecessors were outlined and the SSA representation was invalid.
diff --git a/test/Transforms/HotColdSplit/split-out-dbg-val-of-arg.ll b/test/Transforms/HotColdSplit/split-out-dbg-val-of-arg.ll
index becfaf8e63d3f728c6a54c0f6035b382a1f98eca..54ca19511c086cedf6b1e2c8a3ac90f23694ec10 100644
--- a/test/Transforms/HotColdSplit/split-out-dbg-val-of-arg.ll
+++ b/test/Transforms/HotColdSplit/split-out-dbg-val-of-arg.ll
@@ -1,4 +1,4 @@
-; RUN: opt -hotcoldsplit -S < %s | FileCheck %s
+; RUN: opt -hotcoldsplit -hotcoldsplit-threshold=0 -S < %s | FileCheck %s
 
 target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.14.0"
@@ -17,8 +17,6 @@ if.then:                                          ; preds = %entry
 if.end:                                           ; preds = %entry
   call void @llvm.dbg.value(metadata i32 %arg1, metadata !9, metadata !DIExpression()), !dbg !11
   call void @sink()
-  call void @sink()
-  call void @sink()
   ret void
 }
 
diff --git a/test/Transforms/HotColdSplit/split-phis-in-exit-blocks.ll b/test/Transforms/HotColdSplit/split-phis-in-exit-blocks.ll
new file mode 100644
index 0000000000000000000000000000000000000000..2f5360ccb1e7e20cd98e282ccc571a6f4335a5f2
--- /dev/null
+++ b/test/Transforms/HotColdSplit/split-phis-in-exit-blocks.ll
@@ -0,0 +1,71 @@
+; RUN: opt -hotcoldsplit-threshold=0 -hotcoldsplit -S < %s | FileCheck %s
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.14.0"
+
+; CHECK-LABEL: define {{.*}}@pluto(
+; CHECK-NEXT: bb:
+; CHECK-NEXT:  %tmp8.ce.loc = alloca i1
+; CHECK-NEXT:  switch i8 undef, label %codeRepl [
+; CHECK-NEXT:    i8 0, label %bb7
+; CHECK-NEXT:    i8 1, label %bb7
+; CHECK-NEXT:  ]
+;
+; CHECK:  codeRepl:
+; CHECK-NEXT:    bitcast
+; CHECK-NEXT:    lifetime.start
+; CHECK-NEXT:    call void @pluto.cold.1(i1* %tmp8.ce.loc)
+; CHECK-NEXT:    %tmp8.ce.reload = load i1, i1* %tmp8.ce.loc
+; CHECK-NEXT:    lifetime.end
+; CHECK-NEXT:    br label %bb7
+;
+; CHECK:  bb7:
+; CHECK:    %tmp8 = phi i1 [ true, %bb ], [ true, %bb ], [ %tmp8.ce.reload, %codeRepl ]
+; CHECK:    ret void
+
+; CHECK-LABEL: define {{.*}}@pluto.cold.1(
+; CHECK: call {{.*}}@sideeffect(i32 1)
+; CHECK: call {{.*}}@sink(
+; CHECK: call {{.*}}@sideeffect(i32 3)
+; CHECK: call {{.*}}@sideeffect(i32 4)
+; CHECK: call {{.*}}@sideeffect(i32 5)
+define void @pluto() {
+bb:
+  switch i8 undef, label %bb1 [
+    i8 0, label %bb7
+    i8 1, label %bb7
+  ]
+
+bb1:                                              ; preds = %bb
+  call void @sideeffect(i32 1)
+  br label %bb2
+
+bb2:                                              ; preds = %bb1
+  call void @sink()
+  br i1 undef, label %bb7, label %bb3
+
+bb3:                                              ; preds = %bb2
+  call void @sideeffect(i32 3)
+  br label %bb4
+
+bb4:                                              ; preds = %bb3
+  call void @sideeffect(i32 4)
+  br i1 undef, label %bb5, label %bb6
+
+bb5:                                              ; preds = %bb4
+  call void @sideeffect(i32 5)
+  br label %bb6
+
+bb6:                                              ; preds = %bb5, %bb4
+  %tmp = phi i1 [ true, %bb5 ], [ false, %bb4 ]
+  call void @sideeffect(i32 6)
+  br label %bb7
+
+bb7:                                              ; preds = %bb6, %bb2, %bb, %bb
+  %tmp8 = phi i1 [ true, %bb ], [ true, %bb ], [ true, %bb2 ], [ %tmp, %bb6 ]
+  ret void
+}
+
+declare void @sink() cold
+
+declare void @sideeffect(i32)
diff --git a/test/Transforms/HotColdSplit/succ-block-with-self-edge.ll b/test/Transforms/HotColdSplit/succ-block-with-self-edge.ll
index f8cff9e863e246b4b3101de918ba723d81658dfe..5197f4061e6d8e51a94514a591ff72da747b2bc7 100644
--- a/test/Transforms/HotColdSplit/succ-block-with-self-edge.ll
+++ b/test/Transforms/HotColdSplit/succ-block-with-self-edge.ll
@@ -1,4 +1,4 @@
-; RUN: opt -S -hotcoldsplit < %s | FileCheck %s
+; RUN: opt -S -hotcoldsplit -hotcoldsplit-threshold=0 < %s | FileCheck %s
 
 target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.14.0"
@@ -15,7 +15,6 @@ entry:
 coldbb:
   call void @sink()
   call void @sideeffect()
-  call void @sideeffect()
   br i1 undef, label %if.end, label %coldbb2
 
 coldbb2:
@@ -39,7 +38,6 @@ entry:
 coldbb:
   call void @sink()
   call void @sideeffect()
-  call void @sideeffect()
   br i1 undef, label %if.end, label %coldbb2
 
 coldbb2:
diff --git a/test/Transforms/HotColdSplit/swifterror.ll b/test/Transforms/HotColdSplit/swifterror.ll
new file mode 100644
index 0000000000000000000000000000000000000000..b97b0fa005e716d7bf26f44b0b14bc3425109e5a
--- /dev/null
+++ b/test/Transforms/HotColdSplit/swifterror.ll
@@ -0,0 +1,43 @@
+; RUN: opt -hotcoldsplit -hotcoldsplit-threshold=0 -S < %s | FileCheck %s
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.14.0"
+
+%swift_error = type {i64, i8}
+
+declare void @sink() cold
+
+; CHECK-LABEL: define {{.*}}@in_arg(
+; CHECK: call void @in_arg.cold.1(%swift_error** swifterror
+define void @in_arg(%swift_error** swifterror %error_ptr_ref) {
+  br i1 undef, label %cold, label %exit
+
+cold:
+  store %swift_error* undef, %swift_error** %error_ptr_ref
+  call void @sink()
+  br label %exit
+
+exit:
+  ret void
+}
+
+; CHECK-LABEL: define {{.*}}@in_alloca(
+; CHECK: call void @in_alloca.cold.1(%swift_error** swifterror
+define void @in_alloca() {
+  %err = alloca swifterror %swift_error*
+  br i1 undef, label %cold, label %exit
+
+cold:
+  store %swift_error* undef, %swift_error** %err
+  call void @sink()
+  br label %exit
+
+exit:
+  ret void
+}
+
+; CHECK-LABEL: define {{.*}}@in_arg.cold.1({{.*}} swifterror
+; CHECK: call {{.*}}@sink
+
+; CHECK-LABEL: define {{.*}}@in_alloca.cold.1({{.*}} swifterror
+; CHECK: call {{.*}}@sink
diff --git a/test/Transforms/HotColdSplit/unwind.ll b/test/Transforms/HotColdSplit/unwind.ll
index 4715ef2ab8676a852233c8918f433a8de77a9025..66e2f76e327efdb776fc4a735ffb3ad8636ec774 100644
--- a/test/Transforms/HotColdSplit/unwind.ll
+++ b/test/Transforms/HotColdSplit/unwind.ll
@@ -1,14 +1,17 @@
-; RUN: opt -hotcoldsplit -S < %s | FileCheck %s
+; RUN: opt -hotcoldsplit -hotcoldsplit-threshold=0 -S < %s | FileCheck %s
 
 target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.14.0"
 
-; Do not mark outlined functions which resume exception unwinding as noreturn.
+; Do not split out `resume` instructions.
 
 ; CHECK-LABEL: define {{.*}}@foo.cold.1(
-; CHECK: resume
+; CHECK: call {{.*}}@sink(
+; CHECK-NOT: resume i32 undef
+
 ; CHECK-NOT: noreturn
-define i32 @foo(i32 %cond) personality i8 0 {
+
+define i32 @foo() personality i8 0 {
 entry:
   invoke void @llvm.donothing() to label %normal unwind label %exception
 
@@ -18,8 +21,10 @@ exception:
 
 continue_exception:
   call void @sideeffect(i32 0)
-  call void @sideeffect(i32 1)
   call void @sink()
+  br label %resume-eh
+
+resume-eh:
   resume i32 undef
 
 normal:
diff --git a/test/Transforms/IPConstantProp/arg-count-mismatch.ll b/test/Transforms/IPConstantProp/arg-count-mismatch.ll
new file mode 100644
index 0000000000000000000000000000000000000000..1f62f6465095dba3c94d7a667d60e89330965d8f
--- /dev/null
+++ b/test/Transforms/IPConstantProp/arg-count-mismatch.ll
@@ -0,0 +1,72 @@
+; RUN: opt < %s -ipconstprop -S -o - | FileCheck %s
+
+; The original C source looked like this:
+;
+;   long long a101, b101, e101;
+;   volatile long c101;
+;   int d101;
+;
+;   static inline int bar(p1, p2)
+;   {
+;       return 0;
+;   }
+;
+;   void foo(unsigned p1)
+;   {
+;       long long *f = &b101, *g = &e101;
+;       c101 = 0;
+;       (void)((*f |= a101) - (*g = bar(d101)));
+;       c101 = (*f |= a101 &= p1) == d101;
+;   }
+;
+; When compiled with Clang it gives a warning
+;   warning: too few arguments in call to 'bar'
+;
+; This ll reproducer has been reduced to only include tha call.
+;
+; Note that -lint will report this as UB, but it passes -verify.
+
+; This test is just to verify that we do not crash/assert due to mismatch in
+; argument count between the caller and callee.
+
+define dso_local void @foo(i16 %a) {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT:    [[CALL:%.*]] = call i16 bitcast (i16 (i16, i16)* @bar to i16 (i16)*)(i16 [[A:%.*]])
+; CHECK-NEXT:    ret void
+;
+  %call = call i16 bitcast (i16 (i16, i16) * @bar to i16 (i16) *)(i16 %a)
+  ret void
+}
+
+define internal i16 @bar(i16 %p1, i16 %p2) {
+; CHECK-LABEL: @bar(
+; CHECK-NEXT:    ret i16 0
+;
+  ret i16 0
+}
+
+;-------------------------------------------------------------------------------
+; Additional tests to verify that we still optimize when having a mismatch
+; in argument count due to varargs (as long as all non-variadic arguments have
+; been provided),
+
+define dso_local void @vararg_tests(i16 %a) {
+  %call1 = call i16 (i16, ...) @vararg_prop(i16 7, i16 8, i16 %a)
+  %call2 = call i16 bitcast (i16 (i16, i16, ...) * @vararg_no_prop to i16 (i16) *) (i16 7)
+  ret void
+}
+
+define internal i16 @vararg_prop(i16 %p1, ...) {
+; CHECK-LABEL: define internal i16 @vararg_prop(
+; CHECK-NEXT:    ret i16 7
+;
+  ret i16 %p1
+}
+
+define internal i16 @vararg_no_prop(i16 %p1, i16 %p2, ...) {
+; CHECK-LABEL: define internal i16 @vararg_no_prop(
+; CHECK-NEXT:    ret i16 [[P1:%.*]]
+;
+  ret i16 %p1
+}
+
diff --git a/test/Transforms/IPConstantProp/arg-type-mismatch.ll b/test/Transforms/IPConstantProp/arg-type-mismatch.ll
new file mode 100644
index 0000000000000000000000000000000000000000..ff924d73390b0fcca6b838134945d3716f809c1b
--- /dev/null
+++ b/test/Transforms/IPConstantProp/arg-type-mismatch.ll
@@ -0,0 +1,23 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -ipconstprop -S -o - | FileCheck %s
+
+; This test is just to verify that we do not crash/assert due to mismatch in
+; argument type between the caller and callee.
+
+define dso_local void @foo(i16 %a) {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT:    [[CALL:%.*]] = call i16 bitcast (i16 (i16, i16)* @bar to i16 (i16, i32)*)(i16 [[A:%.*]], i32 7)
+; CHECK-NEXT:    ret void
+;
+  %call = call i16 bitcast (i16 (i16, i16) * @bar to i16 (i16, i32) *)(i16 %a, i32 7)
+  ret void
+}
+
+define internal i16 @bar(i16 %p1, i16 %p2) {
+; CHECK-LABEL: @bar(
+; CHECK-NEXT:    ret i16 [[P2:%.*]]
+;
+  ret i16 %p2
+}
+
+
diff --git a/test/Transforms/IPConstantProp/multiple_callbacks.ll b/test/Transforms/IPConstantProp/multiple_callbacks.ll
new file mode 100644
index 0000000000000000000000000000000000000000..3288b5b94de1120b3b068cbb25724f7bca99568c
--- /dev/null
+++ b/test/Transforms/IPConstantProp/multiple_callbacks.ll
@@ -0,0 +1,87 @@
+; RUN: opt -ipconstprop -S < %s | FileCheck %s
+;
+;
+;                            /---------------------------------------|
+;                            |                /----------------------|----|
+;                            |                |                /-----|    |
+;                            V                V                V     |    |
+;    void broker(int (*cb0)(int), int (*cb1)(int), int (*cb2)(int), int, int);
+;
+;    static int cb0(int zero) {
+;      return zero;
+;    }
+;    static int cb1(int unknown) {
+;      return unknown;
+;    }
+;    static int cb2(int unknown) {
+;      cb0(0);
+;      return unknown;
+;    }
+;    static int cb3(int unknown) {
+;      return unknown;
+;    }
+;    static int cb4(int unknown) {
+;      return unknown;
+;    }
+;
+;    void foo() {
+;      cb0(0);
+;      cb3(1);
+;      broker(cb0, cb1, cb0, 0, 1);
+;      broker(cb1, cb2, cb2, 0, 1);
+;      broker(cb3, cb2, cb3, 0, 1);
+;      broker(cb4, cb4, cb4, 0, 1);
+;    }
+;
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+define internal i32 @cb0(i32 %zero) {
+entry:
+; CHECK:      @cb0
+; CHECK-NEXT: entry
+; CHECK-NEXT: ret i32 0
+  ret i32 %zero
+}
+
+define internal i32 @cb1(i32 %unknown) {
+entry:
+; CHECK: ret i32 %unknown
+  ret i32 %unknown
+}
+
+define internal i32 @cb2(i32 %unknown) {
+entry:
+  %call = call i32 @cb0(i32 0)
+; CHECK: ret i32 %unknown
+  ret i32 %unknown
+}
+
+define internal i32 @cb3(i32 %unknown) {
+entry:
+; CHECK: ret i32 %unknown
+  ret i32 %unknown
+}
+
+define internal i32 @cb4(i32 %unknown) {
+entry:
+; CHECK: ret i32 %unknown
+  ret i32 %unknown
+}
+
+define void @foo() {
+entry:
+  %call = call i32 @cb0(i32 0)
+  %call1 = call i32 @cb3(i32 1)
+  call void @broker(i32 (i32)* nonnull @cb0, i32 (i32)* nonnull @cb1, i32 (i32)* nonnull @cb0, i32 0, i32 1)
+  call void @broker(i32 (i32)* nonnull @cb1, i32 (i32)* nonnull @cb2, i32 (i32)* nonnull @cb2, i32 0, i32 1)
+  call void @broker(i32 (i32)* nonnull @cb3, i32 (i32)* nonnull @cb2, i32 (i32)* nonnull @cb3, i32 0, i32 1)
+  call void @broker(i32 (i32)* nonnull @cb4, i32 (i32)* nonnull @cb4, i32 (i32)* nonnull @cb4, i32 0, i32 1)
+  ret void
+}
+
+declare !callback !3 void @broker(i32 (i32)*, i32 (i32)*, i32 (i32)*, i32, i32)
+
+!0 = !{i64 0, i64 3, i1 false}
+!1 = !{i64 1, i64 4, i1 false}
+!2 = !{i64 2, i64 3, i1 false}
+!3 = !{!0, !2, !1}
diff --git a/test/Transforms/IPConstantProp/openmp_parallel_for.ll b/test/Transforms/IPConstantProp/openmp_parallel_for.ll
new file mode 100644
index 0000000000000000000000000000000000000000..3c7ffe27f5d2fdb6d83833be19f3e209a81c06c3
--- /dev/null
+++ b/test/Transforms/IPConstantProp/openmp_parallel_for.ll
@@ -0,0 +1,120 @@
+; RUN: opt -S -ipconstprop < %s | FileCheck %s
+;
+;    void bar(int, float, double);
+;
+;    void foo(int N) {
+;      float p = 3;
+;      double q = 5;
+;      N = 7;
+;
+;    #pragma omp parallel for firstprivate(q)
+;      for (int i = 2; i < N; i++) {
+;        bar(i, p, q);
+;      }
+;    }
+;
+; Verify the constant value of q is propagated into the outlined function.
+;
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+%struct.ident_t = type { i32, i32, i32, i32, i8* }
+
+@.str = private unnamed_addr constant [23 x i8] c";unknown;unknown;0;0;;\00", align 1
+@0 = private unnamed_addr global %struct.ident_t { i32 0, i32 514, i32 0, i32 0, i8* getelementptr inbounds ([23 x i8], [23 x i8]* @.str, i32 0, i32 0) }, align 8
+@1 = private unnamed_addr global %struct.ident_t { i32 0, i32 2, i32 0, i32 0, i8* getelementptr inbounds ([23 x i8], [23 x i8]* @.str, i32 0, i32 0) }, align 8
+
+define dso_local void @foo(i32 %N) {
+entry:
+  %N.addr = alloca i32, align 4
+  %p = alloca float, align 4
+  store i32 %N, i32* %N.addr, align 4
+  store float 3.000000e+00, float* %p, align 4
+  store i32 7, i32* %N.addr, align 4
+  call void (%struct.ident_t*, i32, void (i32*, i32*, ...)*, ...) @__kmpc_fork_call(%struct.ident_t* nonnull @1, i32 3, void (i32*, i32*, ...)* bitcast (void (i32*, i32*, i32*, float*, i64)* @.omp_outlined. to void (i32*, i32*, ...)*), i32* nonnull %N.addr, float* nonnull %p, i64 4617315517961601024)
+  ret void
+}
+
+define internal void @.omp_outlined.(i32* noalias %.global_tid., i32* noalias %.bound_tid., i32* dereferenceable(4) %N, float* dereferenceable(4) %p, i64 %q) {
+entry:
+  %q.addr = alloca i64, align 8
+  %.omp.lb = alloca i32, align 4
+  %.omp.ub = alloca i32, align 4
+  %.omp.stride = alloca i32, align 4
+  %.omp.is_last = alloca i32, align 4
+; CHECK: store i64 4617315517961601024, i64* %q.addr, align 8
+  store i64 %q, i64* %q.addr, align 8
+  %conv = bitcast i64* %q.addr to double*
+  %tmp = load i32, i32* %N, align 4
+  %sub3 = add nsw i32 %tmp, -3
+  %cmp = icmp sgt i32 %tmp, 2
+  br i1 %cmp, label %omp.precond.then, label %omp.precond.end
+
+omp.precond.then:                                 ; preds = %entry
+  store i32 0, i32* %.omp.lb, align 4
+  store i32 %sub3, i32* %.omp.ub, align 4
+  store i32 1, i32* %.omp.stride, align 4
+  store i32 0, i32* %.omp.is_last, align 4
+  %tmp5 = load i32, i32* %.global_tid., align 4
+  call void @__kmpc_for_static_init_4(%struct.ident_t* nonnull @0, i32 %tmp5, i32 34, i32* nonnull %.omp.is_last, i32* nonnull %.omp.lb, i32* nonnull %.omp.ub, i32* nonnull %.omp.stride, i32 1, i32 1)
+  %tmp6 = load i32, i32* %.omp.ub, align 4
+  %cmp6 = icmp sgt i32 %tmp6, %sub3
+  br i1 %cmp6, label %cond.true, label %cond.false
+
+cond.true:                                        ; preds = %omp.precond.then
+  br label %cond.end
+
+cond.false:                                       ; preds = %omp.precond.then
+  %tmp7 = load i32, i32* %.omp.ub, align 4
+  br label %cond.end
+
+cond.end:                                         ; preds = %cond.false, %cond.true
+  %cond = phi i32 [ %sub3, %cond.true ], [ %tmp7, %cond.false ]
+  store i32 %cond, i32* %.omp.ub, align 4
+  %tmp8 = load i32, i32* %.omp.lb, align 4
+  br label %omp.inner.for.cond
+
+omp.inner.for.cond:                               ; preds = %omp.inner.for.inc, %cond.end
+  %.omp.iv.0 = phi i32 [ %tmp8, %cond.end ], [ %add11, %omp.inner.for.inc ]
+  %tmp9 = load i32, i32* %.omp.ub, align 4
+  %cmp8 = icmp sgt i32 %.omp.iv.0, %tmp9
+  br i1 %cmp8, label %omp.inner.for.cond.cleanup, label %omp.inner.for.body
+
+omp.inner.for.cond.cleanup:                       ; preds = %omp.inner.for.cond
+  br label %omp.inner.for.end
+
+omp.inner.for.body:                               ; preds = %omp.inner.for.cond
+  %add10 = add nsw i32 %.omp.iv.0, 2
+  %tmp10 = load float, float* %p, align 4
+  %tmp11 = load double, double* %conv, align 8
+  call void @bar(i32 %add10, float %tmp10, double %tmp11)
+  br label %omp.body.continue
+
+omp.body.continue:                                ; preds = %omp.inner.for.body
+  br label %omp.inner.for.inc
+
+omp.inner.for.inc:                                ; preds = %omp.body.continue
+  %add11 = add nsw i32 %.omp.iv.0, 1
+  br label %omp.inner.for.cond
+
+omp.inner.for.end:                                ; preds = %omp.inner.for.cond.cleanup
+  br label %omp.loop.exit
+
+omp.loop.exit:                                    ; preds = %omp.inner.for.end
+  %tmp12 = load i32, i32* %.global_tid., align 4
+  call void @__kmpc_for_static_fini(%struct.ident_t* nonnull @0, i32 %tmp12)
+  br label %omp.precond.end
+
+omp.precond.end:                                  ; preds = %omp.loop.exit, %entry
+  ret void
+}
+
+declare dso_local void @__kmpc_for_static_init_4(%struct.ident_t*, i32, i32, i32*, i32*, i32*, i32*, i32, i32)
+
+declare dso_local void @bar(i32, float, double)
+
+declare dso_local void @__kmpc_for_static_fini(%struct.ident_t*, i32)
+
+declare !callback !0 dso_local void @__kmpc_fork_call(%struct.ident_t*, i32, void (i32*, i32*, ...)*, ...)
+
+!1 = !{i64 2, i64 -1, i64 -1, i1 true}
+!0 = !{!1}
diff --git a/test/Transforms/IPConstantProp/pthreads.ll b/test/Transforms/IPConstantProp/pthreads.ll
new file mode 100644
index 0000000000000000000000000000000000000000..0af2c166a0887f89fa87a7a35500d62120683983
--- /dev/null
+++ b/test/Transforms/IPConstantProp/pthreads.ll
@@ -0,0 +1,49 @@
+; RUN: opt -ipconstprop -S < %s | FileCheck %s
+;
+;    #include <pthread.h>
+;
+;    void *GlobalVPtr;
+;
+;    static void *foo(void *arg) { return arg; }
+;    static void *bar(void *arg) { return arg; }
+;
+;    int main() {
+;      pthread_t thread;
+;      pthread_create(&thread, NULL, foo, NULL);
+;      pthread_create(&thread, NULL, bar, &GlobalVPtr);
+;      return 0;
+;    }
+;
+; Verify the constant values NULL and &GlobalVPtr are propagated into foo and
+; bar, respectively.
+;
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+%union.pthread_attr_t = type { i64, [48 x i8] }
+
+@GlobalVPtr = common dso_local global i8* null, align 8
+
+define dso_local i32 @main() {
+entry:
+  %thread = alloca i64, align 8
+  %call = call i32 @pthread_create(i64* nonnull %thread, %union.pthread_attr_t* null, i8* (i8*)* nonnull @foo, i8* null)
+  %call1 = call i32 @pthread_create(i64* nonnull %thread, %union.pthread_attr_t* null, i8* (i8*)* nonnull @bar, i8* bitcast (i8** @GlobalVPtr to i8*))
+  ret i32 0
+}
+
+declare !callback !0 dso_local i32 @pthread_create(i64*, %union.pthread_attr_t*, i8* (i8*)*, i8*)
+
+define internal i8* @foo(i8* %arg) {
+entry:
+; CHECK: ret i8* null
+  ret i8* %arg
+}
+
+define internal i8* @bar(i8* %arg) {
+entry:
+; CHECK: ret i8* bitcast (i8** @GlobalVPtr to i8*)
+  ret i8* %arg
+}
+
+!1 = !{i64 2, i64 3, i1 false}
+!0 = !{!1}
diff --git a/test/Transforms/IPConstantProp/thread_local_acs.ll b/test/Transforms/IPConstantProp/thread_local_acs.ll
new file mode 100644
index 0000000000000000000000000000000000000000..0595a5ca7f13ce18138e5dd8fecfbf0c2084076f
--- /dev/null
+++ b/test/Transforms/IPConstantProp/thread_local_acs.ll
@@ -0,0 +1,42 @@
+; RUN: opt -ipconstprop -S < %s | FileCheck %s
+;
+;    #include <threads.h>
+;    thread_local int gtl = 0;
+;    int gsh = 0;
+;
+;    static int callee(int *thread_local_ptr, int *shared_ptr) {
+;      return *thread_local_ptr + *shared_ptr;
+;    }
+;
+;    void broker(int *, int (*callee)(int *, int *), int *);
+;
+;    void caller() {
+;      broker(&gtl, callee, &gsh);
+;    }
+;
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+@gtl = dso_local thread_local global i32 0, align 4
+@gsh = dso_local global i32 0, align 4
+
+define internal i32 @callee(i32* %thread_local_ptr, i32* %shared_ptr) {
+entry:
+; CHECK:  %tmp = load i32, i32* %thread_local_ptr, align 4
+; CHECK:  %tmp1 = load i32, i32* @gsh, align 4
+; CHECK:  %add = add nsw i32 %tmp, %tmp1
+  %tmp = load i32, i32* %thread_local_ptr, align 4
+  %tmp1 = load i32, i32* %shared_ptr, align 4
+  %add = add nsw i32 %tmp, %tmp1
+  ret i32 %add
+}
+
+define dso_local void @caller() {
+entry:
+  call void @broker(i32* nonnull @gtl, i32 (i32*, i32*)* nonnull @callee, i32* nonnull @gsh)
+  ret void
+}
+
+declare !callback !0 dso_local void @broker(i32*, i32 (i32*, i32*)*, i32*)
+
+!1 = !{i64 1, i64 0, i64 2, i1 false}
+!0 = !{!1}
diff --git a/test/Transforms/IRCE/only-lower-check.ll b/test/Transforms/IRCE/only-lower-check.ll
index e913ea61c03a64fd17ef278344ee61cca6a29532..ad379fb20ac4ae3041ecb56f49ee5c6d356df195 100644
--- a/test/Transforms/IRCE/only-lower-check.ll
+++ b/test/Transforms/IRCE/only-lower-check.ll
@@ -3,7 +3,6 @@
 
 ; CHECK: irce: loop has 1 inductive range checks:
 ; CHECK-NEXT: InductiveRangeCheck:
-; CHECK-NEXT:   Kind: RANGE_CHECK_LOWER
 ; CHECK-NEXT:   Begin: (-1 + %n)  Step: -1  End: 2147483647
 ; CHECK-NEXT:   CheckUse:   br i1 %abc, label %in.bounds, label %out.of.bounds, !prof !1 Operand: 0
 ; CHECK-NEXT: irce: in function only_lower_check: constrained Loop at depth 1 containing: %loop<header><exiting>,%in.bounds<latch><exiting>
diff --git a/test/Transforms/IRCE/only-upper-check.ll b/test/Transforms/IRCE/only-upper-check.ll
index 859ac4e23116dce6237dbadbcae7dc21546ddb3e..45a911b5bfd71577bf23c83faf10097d3897660a 100644
--- a/test/Transforms/IRCE/only-upper-check.ll
+++ b/test/Transforms/IRCE/only-upper-check.ll
@@ -3,7 +3,6 @@
 
 ; CHECK: irce: loop has 1 inductive range checks:
 ; CHECK-NEXT:InductiveRangeCheck:
-; CHECK-NEXT:  Kind: RANGE_CHECK_UPPER
 ; CHECK-NEXT:  Begin: %offset  Step: 1  End:   %len
 ; CHECK-NEXT:  CheckUse:   br i1 %abc, label %in.bounds, label %out.of.bounds, !prof !1 Operand: 0
 ; CHECK-NEXT: irce: in function incrementing: constrained Loop at depth 1 containing: %loop<header><exiting>,%in.bounds<latch><exiting>
diff --git a/test/Transforms/IRCE/wide_indvar.ll b/test/Transforms/IRCE/wide_indvar.ll
new file mode 100644
index 0000000000000000000000000000000000000000..55580a10a6fbc1dbf9afc9223831100a83223cf2
--- /dev/null
+++ b/test/Transforms/IRCE/wide_indvar.ll
@@ -0,0 +1,459 @@
+; RUN: opt -verify-loop-info -irce-print-changed-loops -irce -irce-allow-narrow-latch=true -S < %s 2>&1 | FileCheck %s
+; RUN: opt -verify-loop-info -irce-print-changed-loops -passes='require<branch-prob>,loop(irce)' -irce-allow-narrow-latch=true -S < %s 2>&1 | FileCheck %s
+
+; Check that we can remove trivially non-failing range check.
+define i32 @test_increasing_slt_slt_wide_simple_no_postloop() {
+
+; CHECK-LABEL: @test_increasing_slt_slt_wide_simple_no_postloop(
+; CHECK-NOT:   preloop
+; CHECK-NOT:   postloop
+; CHECK:       loop:
+; CHECK:       br i1 true, label %backedge, label %check_failed
+
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %backedge ]
+  %rc = icmp slt i64 %iv, 100
+  br i1 %rc, label %backedge, label %check_failed
+
+backedge:
+  %iv.next = add i64 %iv, 1
+  %narrow.iv = trunc i64 %iv.next to i32
+  %latch.cond = icmp slt i32 %narrow.iv, 100
+  br i1 %latch.cond, label %loop, label %exit
+
+exit:
+  ret i32 %narrow.iv
+
+check_failed:
+  ret i32 -1
+}
+
+; This range check fails on the last iteration, so it needs a postloop.
+define i32 @test_increasing_slt_slt_wide_simple_postloop() {
+
+; CHECK-LABEL: @test_increasing_slt_slt_wide_simple_postloop(
+; CHECK-NOT:   preloop
+; CHECK:       loop:
+; CHECK:       br i1 true, label %backedge, label %check_failed
+; CHECK:       backedge
+; CHECK:       [[COND:%[^ ]+]] = icmp slt i64 %wide.narrow.iv, 99
+; CHECK:       br i1 [[COND]], label %loop, label %main.exit.selector
+; CHECK:       postloop
+
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %backedge ]
+  %rc = icmp slt i64 %iv, 99
+  br i1 %rc, label %backedge, label %check_failed
+
+backedge:
+  %iv.next = add i64 %iv, 1
+  %narrow.iv = trunc i64 %iv.next to i32
+  %latch.cond = icmp slt i32 %narrow.iv, 100
+  br i1 %latch.cond, label %loop, label %exit
+
+exit:
+  ret i32 %narrow.iv
+
+check_failed:
+  ret i32 -1
+}
+
+; General case. If both %N and %M are non-negative, we do not need a preloop.
+define i32 @test_increasing_slt_slt_wide_non-negative(i32* %n_ptr, i64* %m_ptr) {
+
+; CHECK-LABEL: @test_increasing_slt_slt_wide_non-negative(
+; CHECK-NOT:   preloop
+; CHECK:       loop:
+; CHECK:       br i1 true, label %backedge, label %check_failed
+; CHECK:       backedge
+; CHECK:       [[COND:%[^ ]+]] = icmp slt i64 %wide.narrow.iv, %exit.mainloop.at
+; CHECK:       br i1 [[COND]], label %loop, label %main.exit.selector
+; CHECK:       postloop
+
+entry:
+  %N = load i32, i32* %n_ptr, !range !2
+  %M = load i64, i64* %m_ptr, !range !1
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %backedge ]
+  %rc = icmp slt i64 %iv, %M
+  br i1 %rc, label %backedge, label %check_failed
+
+backedge:
+  %iv.next = add i64 %iv, 1
+  %narrow.iv = trunc i64 %iv.next to i32
+  %latch.cond = icmp slt i32 %narrow.iv, %N
+  br i1 %latch.cond, label %loop, label %exit
+
+exit:
+  ret i32 %narrow.iv
+
+check_failed:
+  ret i32 -1
+}
+
+; General case. Even though %M may be negative, we do not need a preloop because
+; we make a non-negativity runtime check against M and do not go to main loop if
+; M was negative.
+define i32 @test_increasing_slt_slt_wide_general(i32* %n_ptr, i64* %m_ptr) {
+
+; CHECK-LABEL: @test_increasing_slt_slt_wide_general(
+; CHECK-NOT:   preloop
+; CHECK:       loop:
+; CHECK:       br i1 true, label %backedge, label %check_failed
+; CHECK:       backedge
+; CHECK:       [[COND:%[^ ]+]] = icmp slt i64
+; CHECK:       br i1 [[COND]], label %loop, label %main.exit.selector
+; CHECK:       postloop
+
+entry:
+  %N = load i32, i32* %n_ptr, !range !2
+  %M = load i64, i64* %m_ptr
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %backedge ]
+  %rc = icmp slt i64 %iv, %M
+  br i1 %rc, label %backedge, label %check_failed
+
+backedge:
+  %iv.next = add i64 %iv, 1
+  %narrow.iv = trunc i64 %iv.next to i32
+  %latch.cond = icmp slt i32 %narrow.iv, %N
+  br i1 %latch.cond, label %loop, label %exit
+
+exit:
+  ret i32 %narrow.iv
+
+check_failed:
+  ret i32 -1
+}
+
+; General case with preloop.
+define i32 @test_increasing_slt_slt_wide_general_preloop(i32* %n_ptr, i64* %m_ptr) {
+
+; CHECK-LABEL: @test_increasing_slt_slt_wide_general_preloop(
+; CHECK:       loop:
+; CHECK:       br i1 true, label %backedge, label %check_failed
+; CHECK:       backedge
+; CHECK:       [[COND:%[^ ]+]] = icmp slt i64
+; CHECK:       br i1 [[COND]], label %loop, label %main.exit.selector
+; CHECK:       preloop
+; CHECK:       postloop
+
+entry:
+  %N = load i32, i32* %n_ptr, !range !2
+  %M = load i64, i64* %m_ptr
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %backedge ]
+  %rc = icmp slt i64 %iv, %M
+  br i1 %rc, label %backedge, label %check_failed
+
+backedge:
+  %iv.next = add i64 %iv, 1
+  %narrow.iv = trunc i64 %iv to i32
+  %latch.cond = icmp slt i32 %narrow.iv, %N
+  br i1 %latch.cond, label %loop, label %exit
+
+exit:
+  ret i32 %narrow.iv
+
+check_failed:
+  ret i32 -1
+}
+
+; Same as above, multiple checks.
+define i32 @test_increasing_slt_slt_wide_multiple_checks(i32* %n_ptr, i64* %m1_ptr, i64* %m2_ptr, i64* %m3_ptr, i64* %m4_ptr) {
+; CHECK-LABEL: @test_increasing_slt_slt_wide_multiple_checks(
+; CHECK-NOT:   preloop
+; CHECK:       loop:
+; CHECK:       %c1 = and i1 true, true
+; CHECK:       %c2 = and i1 %c1, true
+; CHECK:       %rc = and i1 %c2, true
+; CHECK:       br i1 %rc, label %backedge, label %check_failed.loopexit
+; CHECK:       backedge
+; CHECK:       [[COND:%[^ ]+]] = icmp slt i64
+; CHECK:       br i1 [[COND]], label %loop, label %main.exit.selector
+; CHECK:       postloop
+
+entry:
+  %N = load i32, i32* %n_ptr, !range !2
+  %M1 = load i64, i64* %m1_ptr
+  %M2 = load i64, i64* %m2_ptr
+  %M3 = load i64, i64* %m3_ptr
+  %M4 = load i64, i64* %m4_ptr
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %backedge ]
+  %rc1 = icmp slt i64 %iv, %M1
+  %rc2 = icmp slt i64 %iv, %M2
+  %rc3 = icmp slt i64 %iv, %M3
+  %rc4 = icmp slt i64 %iv, %M4
+  %c1 = and i1 %rc1, %rc2
+  %c2 = and i1 %c1, %rc3
+  %rc = and i1 %c2, %rc4
+  br i1 %rc, label %backedge, label %check_failed
+
+backedge:
+  %iv.next = add i64 %iv, 1
+  %narrow.iv = trunc i64 %iv.next to i32
+  %latch.cond = icmp slt i32 %narrow.iv, %N
+  br i1 %latch.cond, label %loop, label %exit
+
+exit:
+  ret i32 %narrow.iv
+
+check_failed:
+  ret i32 -1
+}
+
+; Wide IV against narrow range check. We don't currently support it.
+define i32 @test_increasing_slt_slt_wide_simple_negtest_narrow_rc() {
+
+; CHECK-LABEL: @test_increasing_slt_slt_wide_simple_negtest_narrow_rc(
+; CHECK-NOT:   i1 true
+; CHECK-NOT:   main
+
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %backedge ]
+  %narrow.iv = trunc i64 %iv to i32
+  %rc = icmp slt i32 %narrow.iv, 101
+  br i1 %rc, label %backedge, label %check_failed
+
+backedge:
+  %iv.next = add i64 %iv, 1
+  %latch.cond = icmp slt i64 %iv, 100
+  br i1 %latch.cond, label %loop, label %exit
+
+exit:
+  ret i32 %narrow.iv
+
+check_failed:
+  ret i32 -1
+}
+
+; Check that we can remove trivially non-failing range check.
+define i32 @test_increasing_ult_ult_wide_simple_no_postloop() {
+
+; CHECK-LABEL: @test_increasing_ult_ult_wide_simple_no_postloop(
+; CHECK-NOT:   preloop
+; CHECK-NOT:   postloop
+; CHECK:       loop:
+; CHECK:       br i1 true, label %backedge, label %check_failed
+
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %backedge ]
+  %rc = icmp ult i64 %iv, 100
+  br i1 %rc, label %backedge, label %check_failed
+
+backedge:
+  %iv.next = add i64 %iv, 1
+  %narrow.iv = trunc i64 %iv.next to i32
+  %latch.cond = icmp ult i32 %narrow.iv, 100
+  br i1 %latch.cond, label %loop, label %exit
+
+exit:
+  ret i32 %narrow.iv
+
+check_failed:
+  ret i32 -1
+}
+
+; This range check fails on the last iteration, so it needs a postloop.
+define i32 @test_increasing_ult_ult_wide_simple_postloop() {
+
+; CHECK-LABEL: @test_increasing_ult_ult_wide_simple_postloop(
+; CHECK-NOT:   preloop
+; CHECK:       loop:
+; CHECK:       br i1 true, label %backedge, label %check_failed
+; CHECK:       backedge
+; CHECK:       [[COND:%[^ ]+]] = icmp ult i64 %wide.narrow.iv, 99
+; CHECK:       br i1 [[COND]], label %loop, label %main.exit.selector
+; CHECK:       postloop
+
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %backedge ]
+  %rc = icmp ult i64 %iv, 99
+  br i1 %rc, label %backedge, label %check_failed
+
+backedge:
+  %iv.next = add i64 %iv, 1
+  %narrow.iv = trunc i64 %iv.next to i32
+  %latch.cond = icmp ult i32 %narrow.iv, 100
+  br i1 %latch.cond, label %loop, label %exit
+
+exit:
+  ret i32 %narrow.iv
+
+check_failed:
+  ret i32 -1
+}
+
+; General case. If both %N and %M are non-negative, we do not need a preloop.
+define i32 @test_increasing_ult_ult_wide_non-negative(i32* %n_ptr, i64* %m_ptr) {
+
+; CHECK-LABEL: @test_increasing_ult_ult_wide_non-negative(
+; CHECK-NOT:   preloop
+; CHECK:       loop:
+; CHECK:       br i1 true, label %backedge, label %check_failed
+; CHECK:       backedge
+; CHECK:       [[COND:%[^ ]+]] = icmp ult i64 %wide.narrow.iv, %exit.mainloop.at
+; CHECK:       br i1 [[COND]], label %loop, label %main.exit.selector
+; CHECK:       postloop
+
+entry:
+  %N = load i32, i32* %n_ptr, !range !2
+  %M = load i64, i64* %m_ptr, !range !1
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %backedge ]
+  %rc = icmp ult i64 %iv, %M
+  br i1 %rc, label %backedge, label %check_failed
+
+backedge:
+  %iv.next = add i64 %iv, 1
+  %narrow.iv = trunc i64 %iv.next to i32
+  %latch.cond = icmp ult i32 %narrow.iv, %N
+  br i1 %latch.cond, label %loop, label %exit
+
+exit:
+  ret i32 %narrow.iv
+
+check_failed:
+  ret i32 -1
+}
+
+; General case. Even though %M may be negative, we do not need a preloop because
+; we make a non-negativity runtime check against M and do not go to main loop if
+; M was negative.
+define i32 @test_increasing_ult_ult_wide_general(i32* %n_ptr, i64* %m_ptr) {
+
+; CHECK-LABEL: @test_increasing_ult_ult_wide_general(
+; CHECK-NOT:   preloop
+; CHECK:       loop:
+; CHECK:       br i1 true, label %backedge, label %check_failed
+; CHECK:       backedge
+; CHECK:       [[COND:%[^ ]+]] = icmp ult i64
+; CHECK:       br i1 [[COND]], label %loop, label %main.exit.selector
+; CHECK:       postloop
+
+entry:
+  %N = load i32, i32* %n_ptr, !range !2
+  %M = load i64, i64* %m_ptr
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %backedge ]
+  %rc = icmp ult i64 %iv, %M
+  br i1 %rc, label %backedge, label %check_failed
+
+backedge:
+  %iv.next = add i64 %iv, 1
+  %narrow.iv = trunc i64 %iv.next to i32
+  %latch.cond = icmp ult i32 %narrow.iv, %N
+  br i1 %latch.cond, label %loop, label %exit
+
+exit:
+  ret i32 %narrow.iv
+
+check_failed:
+  ret i32 -1
+}
+
+; Same as above, multiple checks.
+define i32 @test_increasing_ult_ult_wide_multiple_checks(i32* %n_ptr, i64* %m1_ptr, i64* %m2_ptr, i64* %m3_ptr, i64* %m4_ptr) {
+; CHECK-LABEL: @test_increasing_ult_ult_wide_multiple_checks(
+; CHECK-NOT:   preloop
+; CHECK:       loop:
+; CHECK:       %c1 = and i1 true, true
+; CHECK:       %c2 = and i1 %c1, true
+; CHECK:       %rc = and i1 %c2, true
+; CHECK:       br i1 %rc, label %backedge, label %check_failed.loopexit
+; CHECK:       backedge
+; CHECK:       [[COND:%[^ ]+]] = icmp ult i64
+; CHECK:       br i1 [[COND]], label %loop, label %main.exit.selector
+; CHECK:       postloop
+
+entry:
+  %N = load i32, i32* %n_ptr, !range !2
+  %M1 = load i64, i64* %m1_ptr
+  %M2 = load i64, i64* %m2_ptr
+  %M3 = load i64, i64* %m3_ptr
+  %M4 = load i64, i64* %m4_ptr
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %backedge ]
+  %rc1 = icmp ult i64 %iv, %M1
+  %rc2 = icmp ult i64 %iv, %M2
+  %rc3 = icmp ult i64 %iv, %M3
+  %rc4 = icmp ult i64 %iv, %M4
+  %c1 = and i1 %rc1, %rc2
+  %c2 = and i1 %c1, %rc3
+  %rc = and i1 %c2, %rc4
+  br i1 %rc, label %backedge, label %check_failed
+
+backedge:
+  %iv.next = add i64 %iv, 1
+  %narrow.iv = trunc i64 %iv.next to i32
+  %latch.cond = icmp ult i32 %narrow.iv, %N
+  br i1 %latch.cond, label %loop, label %exit
+
+exit:
+  ret i32 %narrow.iv
+
+check_failed:
+  ret i32 -1
+}
+
+; Wide IV against narrow range check. We don't currently support it.
+define i32 @test_increasing_ult_ult_wide_simple_negtest_narrow_rc() {
+
+; CHECK-LABEL: @test_increasing_ult_ult_wide_simple_negtest_narrow_rc(
+; CHECK-NOT:   i1 true
+; CHECK-NOT:   main
+
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %backedge ]
+  %narrow.iv = trunc i64 %iv to i32
+  %rc = icmp ult i32 %narrow.iv, 101
+  br i1 %rc, label %backedge, label %check_failed
+
+backedge:
+  %iv.next = add i64 %iv, 1
+  %latch.cond = icmp ult i64 %iv, 100
+  br i1 %latch.cond, label %loop, label %exit
+
+exit:
+  ret i32 %narrow.iv
+
+check_failed:
+  ret i32 -1
+}
+
+!0 = !{i32 0, i32 2147483647}
+!1 = !{i64 0, i64 9223372036854775807}
+!2 = !{i32 1, i32 2147483647}
diff --git a/test/Transforms/IndVarSimplify/pr39673.ll b/test/Transforms/IndVarSimplify/pr39673.ll
new file mode 100644
index 0000000000000000000000000000000000000000..4ab79a935c91eb1c8474e99ee6110a61da880165
--- /dev/null
+++ b/test/Transforms/IndVarSimplify/pr39673.ll
@@ -0,0 +1,56 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -indvars < %s | FileCheck %s
+
+define i16 @test() {
+; CHECK-LABEL: @test(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP1:%.*]]
+; CHECK:       loop1:
+; CHECK-NEXT:    [[L1:%.*]] = phi i16 [ 0, [[ENTRY:%.*]] ], [ [[L1_ADD:%.*]], [[LOOP1]] ]
+; CHECK-NEXT:    [[L1_ADD]] = add nuw nsw i16 [[L1]], 1
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp ult i16 [[L1_ADD]], 2
+; CHECK-NEXT:    br i1 [[CMP1]], label [[LOOP1]], label [[LOOP2_PREHEADER:%.*]]
+; CHECK:       loop2.preheader:
+; CHECK-NEXT:    br label [[LOOP2:%.*]]
+; CHECK:       loop2:
+; CHECK-NEXT:    [[K2:%.*]] = phi i16 [ [[K2_ADD:%.*]], [[LOOP2]] ], [ 182, [[LOOP2_PREHEADER]] ]
+; CHECK-NEXT:    [[L2:%.*]] = phi i16 [ [[L2_ADD:%.*]], [[LOOP2]] ], [ 0, [[LOOP2_PREHEADER]] ]
+; CHECK-NEXT:    [[L2_ADD]] = add nuw nsw i16 [[L2]], 1
+; CHECK-NEXT:    tail call void @foo(i16 [[K2]])
+; CHECK-NEXT:    [[K2_ADD]] = add nuw nsw i16 [[K2]], 1
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ult i16 [[L2_ADD]], 2
+; CHECK-NEXT:    br i1 [[CMP2]], label [[LOOP2]], label [[LOOP2_END:%.*]]
+; CHECK:       loop2.end:
+; CHECK-NEXT:    [[K2_ADD_LCSSA:%.*]] = phi i16 [ [[K2_ADD]], [[LOOP2]] ]
+; CHECK-NEXT:    ret i16 [[K2_ADD_LCSSA]]
+;
+entry:
+  br label %loop1
+
+loop1:                                           ; preds = %entry, %loop1
+  %k1 = phi i16 [ 180, %entry ], [ %k1.add, %loop1 ]
+  %l1 = phi i16 [ 0, %entry ], [ %l1.add, %loop1 ]
+  %k1.add = add nuw nsw i16 %k1, 1
+  %l1.add = add nuw nsw i16 %l1, 1
+  %cmp1 = icmp ult i16 %l1.add, 2
+  br i1 %cmp1, label %loop1, label %loop2.preheader
+
+loop2.preheader:                                 ; preds = %loop1
+  %k1.add.lcssa = phi i16 [ %k1.add, %loop1 ]
+  br label %loop2
+
+loop2:                                           ; preds = %loop2.preheader, %loop2
+  %k2 = phi i16 [ %k2.add, %loop2 ], [ %k1.add.lcssa, %loop2.preheader ]
+  %l2 = phi i16 [ %l2.add, %loop2 ], [ 0, %loop2.preheader ]
+  %l2.add = add nuw i16 %l2, 1
+  tail call void @foo(i16 %k2)
+  %k2.add = add nuw nsw i16 %k2, 1
+  %cmp2 = icmp ult i16 %l2.add, 2
+  br i1 %cmp2, label %loop2, label %loop2.end
+
+loop2.end:                                       ; preds = %loop2
+  %k2.add.lcssa = phi i16 [ %k2.add, %loop2 ]
+  ret i16 %k2.add.lcssa
+}
+
+declare void @foo(i16)
diff --git a/test/Transforms/IndVarSimplify/pr40454.ll b/test/Transforms/IndVarSimplify/pr40454.ll
new file mode 100644
index 0000000000000000000000000000000000000000..c0ad01e4a7f94c2f66979c912df1a2d6d7743eaa
--- /dev/null
+++ b/test/Transforms/IndVarSimplify/pr40454.ll
@@ -0,0 +1,62 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -indvars  < %s | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+define void @test() {
+; CHECK-LABEL: @test(
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    br label [[BB2:%.*]]
+; CHECK:       bb1:
+; CHECK-NEXT:    br label [[BB2]]
+; CHECK:       bb2:
+; CHECK-NEXT:    [[TMP:%.*]] = phi i32 [ -9, [[BB:%.*]] ], [ [[TMP6:%.*]], [[BB1:%.*]] ]
+; CHECK-NEXT:    br label [[BB3:%.*]]
+; CHECK:       bb3:
+; CHECK-NEXT:    [[TMP4:%.*]] = phi i32 [ -9, [[BB2]] ], [ [[TMP6]], [[BB10:%.*]] ]
+; CHECK-NEXT:    br i1 false, label [[BB5:%.*]], label [[BB12:%.*]]
+; CHECK:       bb5:
+; CHECK-NEXT:    [[TMP6]] = add nsw i32 [[TMP4]], -1
+; CHECK-NEXT:    br i1 undef, label [[BB8:%.*]], label [[BB9:%.*]]
+; CHECK:       bb8:
+; CHECK-NEXT:    br label [[BB10]]
+; CHECK:       bb9:
+; CHECK-NEXT:    br label [[BB10]]
+; CHECK:       bb10:
+; CHECK-NEXT:    br label [[BB3]]
+; CHECK:       bb12:
+; CHECK-NEXT:    ret void
+;
+
+bb:
+  br label %bb2
+
+bb1:                                              ; No predecessors!
+  br label %bb2
+
+bb2:                                              ; preds = %bb1, %bb
+  %tmp = phi i32 [ -9, %bb ], [ %tmp6, %bb1 ]
+  br label %bb3
+
+bb3:                                              ; preds = %bb10, %bb2
+  %tmp4 = phi i32 [ -9, %bb2 ], [ %tmp6, %bb10 ]
+  br i1 undef, label %bb5, label %bb12
+
+bb5:                                              ; preds = %bb3
+  %tmp6 = add i32 %tmp4, -1
+  %tmp7 = zext i32 %tmp6 to i64
+  br i1 undef, label %bb8, label %bb9
+
+bb8:                                              ; preds = %bb5
+  br label %bb10
+
+bb9:                                              ; preds = %bb5
+  br label %bb10
+
+bb10:                                             ; preds = %bb9, %bb8
+  %tmp11 = and i64 undef, %tmp7
+  br label %bb3
+
+bb12:                                             ; preds = %bb3
+  ret void
+}
diff --git a/test/Transforms/InferAddressSpaces/AMDGPU/intrinsics.ll b/test/Transforms/InferAddressSpaces/AMDGPU/intrinsics.ll
index 723ce41588a1560d772c3270f0b4f68f7f851836..166e90ab7b698ded8ae45f66b28fa34e81588281 100644
--- a/test/Transforms/InferAddressSpaces/AMDGPU/intrinsics.ll
+++ b/test/Transforms/InferAddressSpaces/AMDGPU/intrinsics.ll
@@ -1,18 +1,18 @@
 ; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -infer-address-spaces %s | FileCheck %s
 
 ; CHECK-LABEL: @objectsize_group_to_flat_i32(
-; CHECK: %val = call i32 @llvm.objectsize.i32.p3i8(i8 addrspace(3)* %group.ptr, i1 true, i1 false)
+; CHECK: %val = call i32 @llvm.objectsize.i32.p3i8(i8 addrspace(3)* %group.ptr, i1 true, i1 false, i1 false)
 define i32 @objectsize_group_to_flat_i32(i8 addrspace(3)* %group.ptr) #0 {
   %cast = addrspacecast i8 addrspace(3)* %group.ptr to i8*
-  %val = call i32 @llvm.objectsize.i32.p0i8(i8* %cast, i1 true, i1 false)
+  %val = call i32 @llvm.objectsize.i32.p0i8(i8* %cast, i1 true, i1 false, i1 false)
   ret i32 %val
 }
 
 ; CHECK-LABEL: @objectsize_global_to_flat_i64(
-; CHECK: %val = call i64 @llvm.objectsize.i64.p3i8(i8 addrspace(3)* %global.ptr, i1 true, i1 false)
+; CHECK: %val = call i64 @llvm.objectsize.i64.p3i8(i8 addrspace(3)* %global.ptr, i1 true, i1 false, i1 false)
 define i64 @objectsize_global_to_flat_i64(i8 addrspace(3)* %global.ptr) #0 {
   %cast = addrspacecast i8 addrspace(3)* %global.ptr to i8*
-  %val = call i64 @llvm.objectsize.i64.p0i8(i8* %cast, i1 true, i1 false)
+  %val = call i64 @llvm.objectsize.i64.p0i8(i8* %cast, i1 true, i1 false, i1 false)
   ret i64 %val
 }
 
@@ -134,8 +134,8 @@ define i64 @invalid_variable_volatile_atomicinc_group_to_flat_i64(i64 addrspace(
   ret i64 %ret
 }
 
-declare i32 @llvm.objectsize.i32.p0i8(i8*, i1, i1) #1
-declare i64 @llvm.objectsize.i64.p0i8(i8*, i1, i1) #1
+declare i32 @llvm.objectsize.i32.p0i8(i8*, i1, i1, i1) #1
+declare i64 @llvm.objectsize.i64.p0i8(i8*, i1, i1, i1) #1
 declare i32 @llvm.amdgcn.atomic.inc.i32.p0i32(i32* nocapture, i32, i32, i32, i1) #2
 declare i64 @llvm.amdgcn.atomic.inc.i64.p0i64(i64* nocapture, i64, i32, i32, i1) #2
 declare i32 @llvm.amdgcn.atomic.dec.i32.p0i32(i32* nocapture, i32, i32, i32, i1) #2
diff --git a/test/Transforms/Inline/AMDGPU/inline-target-cpu.ll b/test/Transforms/Inline/AMDGPU/inline-target-cpu.ll
index 4258fc5bc828ff0501b104d3b7b5de0cafdc12b0..87330c72d442c03e381fcf41de43d21ed01c4753 100644
--- a/test/Transforms/Inline/AMDGPU/inline-target-cpu.ll
+++ b/test/Transforms/Inline/AMDGPU/inline-target-cpu.ll
@@ -78,6 +78,17 @@ define i32 @call_func_without_loadstoreopt() #9 {
   ret i32 %call
 }
 
+define i32 @enable_codeobjectv3() #10 {
+  ret i32 999
+}
+
+; CHECK-LABEL: @disable_codeobjectv3_call_codeobjectv3(
+; CHECK-NEXT: ret i32 999
+define i32 @disable_codeobjectv3_call_codeobjectv3() #11 {
+  %call = call i32 @enable_codeobjectv3()
+  ret i32 %call
+}
+
 attributes #0 = { nounwind }
 attributes #1 = { nounwind "target-cpu"="fiji" }
 attributes #2 = { nounwind "target-cpu"="fiji" "target-features"="+fp32-denormals" }
@@ -88,3 +99,5 @@ attributes #6 = { nounwind "target-features"="-half-rate-64-ops" }
 attributes #7 = { nounwind "target-features"="+half-rate-64-ops" }
 attributes #8 = { nounwind "target-features"="-load-store-opt" }
 attributes #9 = { nounwind "target-features"="+load-store-opt" }
+attributes #10 = { nounwind "target-features"="+code-object-v3" }
+attributes #11 = { nounwind "target-features"="-code-object-v3" }
diff --git a/test/Transforms/Inline/inline-remark.ll b/test/Transforms/Inline/inline-remark.ll
index d436fa6f09617f42725fe7c6f61ec17e31360518..bfb78c9ee88e504553706f7a252cb1c630c97497 100644
--- a/test/Transforms/Inline/inline-remark.ll
+++ b/test/Transforms/Inline/inline-remark.ll
@@ -1,7 +1,9 @@
-; RUN: opt < %s -inline -inline-remark-attribute --inline-threshold=-2 -S | FileCheck %s
+; RUN: opt < %s -inline -inline-remark-attribute --inline-threshold=0 -S | FileCheck %s
 
 ; Test that the inliner adds inline remark attributes to non-inlined callsites.
 
+declare void @ext();
+
 define void @foo() {
   call void @bar(i1 true)
   ret void
@@ -12,6 +14,7 @@ define void @bar(i1 %p) {
 
 bb1:
   call void @foo()
+  call void @ext()
   ret void
 
 bb2:
@@ -43,6 +46,16 @@ define void @test2(i8*) {
   ret void
 }
 
-; CHECK: attributes [[ATTR1]] = { "inline-remark"="(cost=-5, threshold=-6)" }
+;; Test 3 - InlineResult messages come from llvm::isInlineViable()
+define void @test3() {
+; CHECK-LABEL: @test3
+; CHECK-NEXT: call void @test3() [[ATTR4:#[0-9]+]]
+; CHECK-NEXT: ret void
+  call void @test3() alwaysinline
+  ret void
+}
+
+; CHECK: attributes [[ATTR1]] = { "inline-remark"="(cost=25, threshold=0)" }
 ; CHECK: attributes [[ATTR2]] = { "inline-remark"="(cost=never): recursive" }
 ; CHECK: attributes [[ATTR3]] = { "inline-remark"="unsupported operand bundle; (cost={{.*}}, threshold={{.*}})" }
+; CHECK: attributes [[ATTR4]] = { alwaysinline "inline-remark"="(cost=never): recursive call" }
diff --git a/test/Transforms/Inline/parallel-loop-md-callee.ll b/test/Transforms/Inline/parallel-loop-md-callee.ll
new file mode 100644
index 0000000000000000000000000000000000000000..4a87c00a44082e1aea24a544103f05d4d0c6e8ab
--- /dev/null
+++ b/test/Transforms/Inline/parallel-loop-md-callee.ll
@@ -0,0 +1,56 @@
+; RUN: opt -S -inline < %s | FileCheck %s
+;
+; Check that the !llvm.access.group is still present after inlining.
+;
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+define void @Body(i32* nocapture %res, i32* nocapture readnone %c, i32* nocapture readonly %d, i32* nocapture readonly %p, i32 %i) {
+entry:
+  %idxprom = sext i32 %i to i64
+  %arrayidx = getelementptr inbounds i32, i32* %p, i64 %idxprom
+  %0 = load i32, i32* %arrayidx, align 4, !llvm.access.group !0
+  %cmp = icmp eq i32 %0, 0
+  %arrayidx2 = getelementptr inbounds i32, i32* %res, i64 %idxprom
+  %1 = load i32, i32* %arrayidx2, align 4, !llvm.access.group !0
+  br i1 %cmp, label %cond.end, label %cond.false
+
+cond.false:
+  %arrayidx6 = getelementptr inbounds i32, i32* %d, i64 %idxprom
+  %2 = load i32, i32* %arrayidx6, align 4, !llvm.access.group !0
+  %add = add nsw i32 %2, %1
+  br label %cond.end
+
+cond.end:
+  %cond = phi i32 [ %add, %cond.false ], [ %1, %entry ]
+  store i32 %cond, i32* %arrayidx2, align 4
+  ret void
+}
+
+define void @Test(i32* %res, i32* %c, i32* %d, i32* %p, i32 %n) {
+entry:
+  br label %for.cond
+
+for.cond:
+  %i.0 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  %cmp = icmp slt i32 %i.0, 1600
+  br i1 %cmp, label %for.body, label %for.end
+
+for.body:
+  call void @Body(i32* %res, i32* undef, i32* %d, i32* %p, i32 %i.0), !llvm.access.group !0
+  %inc = add nsw i32 %i.0, 1
+  br label %for.cond, !llvm.loop !1
+
+for.end:
+  ret void
+}
+
+!0 = distinct !{}                                          ; access group
+!1 = distinct !{!1, !{!"llvm.loop.parallel_accesses", !0}} ; LoopID
+
+
+; CHECK-LABEL: @Test
+; CHECK: load i32,{{.*}}, !llvm.access.group !0
+; CHECK: load i32,{{.*}}, !llvm.access.group !0
+; CHECK: load i32,{{.*}}, !llvm.access.group !0
+; CHECK: store i32 {{.*}}, !llvm.access.group !0
+; CHECK: br label %for.cond, !llvm.loop !1
diff --git a/test/Transforms/Inline/parallel-loop-md-merge.ll b/test/Transforms/Inline/parallel-loop-md-merge.ll
new file mode 100644
index 0000000000000000000000000000000000000000..a53efb75302092453a7c46e134a3708d8db52276
--- /dev/null
+++ b/test/Transforms/Inline/parallel-loop-md-merge.ll
@@ -0,0 +1,78 @@
+; RUN: opt -always-inline -globalopt -S < %s | FileCheck %s
+;
+; static void __attribute__((always_inline)) callee(long n, double A[static const restrict n], long i) {
+;   for (long j = 0; j < n; j += 1)
+;     A[i * n + j] = 42;
+; }
+;
+; void caller(long n, double A[static const restrict n]) {
+;   for (long i = 0; i < n; i += 1)
+;     callee(n, A, i);
+; }
+;
+; Check that the access groups (llvm.access.group) are correctly merged.
+;
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+define internal void @callee(i64 %n, double* noalias nonnull %A, i64 %i) #0 {
+entry:
+  br label %for.cond
+
+for.cond:
+  %j.0 = phi i64 [ 0, %entry ], [ %add1, %for.body ]
+  %cmp = icmp slt i64 %j.0, %n
+  br i1 %cmp, label %for.body, label %for.end
+
+for.body:
+  %mul = mul nsw i64 %i, %n
+  %add = add nsw i64 %mul, %j.0
+  %arrayidx = getelementptr inbounds double, double* %A, i64 %add
+  store double 4.200000e+01, double* %arrayidx, align 8, !llvm.access.group !6
+  %add1 = add nuw nsw i64 %j.0, 1
+  br label %for.cond, !llvm.loop !7
+
+for.end:
+  ret void
+}
+
+attributes #0 = { alwaysinline }
+
+!6 = distinct !{}       ; access group
+!7 = distinct !{!7, !9} ; LoopID
+!9 = !{!"llvm.loop.parallel_accesses", !6}
+
+
+define void @caller(i64 %n, double* noalias nonnull %A) {
+entry:
+  br label %for.cond
+
+for.cond:
+  %i.0 = phi i64 [ 0, %entry ], [ %add, %for.body ]
+  %cmp = icmp slt i64 %i.0, %n
+  br i1 %cmp, label %for.body, label %for.end
+
+for.body:
+  call void @callee(i64 %n, double* %A, i64 %i.0), !llvm.access.group !10
+  %add = add nuw nsw i64 %i.0, 1
+  br label %for.cond, !llvm.loop !11
+
+for.end:
+  ret void
+}
+
+!10 = distinct !{}         ; access group
+!11 = distinct !{!11, !12} ; LoopID
+!12 = !{!"llvm.loop.parallel_accesses", !10}
+
+
+; CHECK: store double 4.200000e+01, {{.*}} !llvm.access.group ![[ACCESS_GROUP_LIST_3:[0-9]+]]
+; CHECK: br label %for.cond.i, !llvm.loop ![[LOOP_INNER:[0-9]+]]
+; CHECK: br label %for.cond, !llvm.loop ![[LOOP_OUTER:[0-9]+]]
+
+; CHECK: ![[ACCESS_GROUP_LIST_3]] = !{![[ACCESS_GROUP_INNER:[0-9]+]], ![[ACCESS_GROUP_OUTER:[0-9]+]]}
+; CHECK: ![[ACCESS_GROUP_INNER]] = distinct !{}
+; CHECK: ![[ACCESS_GROUP_OUTER]] = distinct !{}
+; CHECK: ![[LOOP_INNER]] = distinct !{![[LOOP_INNER]], ![[ACCESSES_INNER:[0-9]+]]}
+; CHECK: ![[ACCESSES_INNER]] = !{!"llvm.loop.parallel_accesses", ![[ACCESS_GROUP_INNER]]}
+; CHECK: ![[LOOP_OUTER]] = distinct !{![[LOOP_OUTER]], ![[ACCESSES_OUTER:[0-9]+]]}
+; CHECK: ![[ACCESSES_OUTER]] = !{!"llvm.loop.parallel_accesses", ![[ACCESS_GROUP_OUTER]]}
diff --git a/test/Transforms/Inline/parallel-loop-md.ll b/test/Transforms/Inline/parallel-loop-md.ll
index abbacc9acd099426a03b95130289c4544721eda3..a55392d6a737ffdc6f00d85dc7e7ddab6f76b63d 100644
--- a/test/Transforms/Inline/parallel-loop-md.ll
+++ b/test/Transforms/Inline/parallel-loop-md.ll
@@ -37,22 +37,22 @@ for.cond:                                         ; preds = %for.body, %entry
   br i1 %cmp, label %for.body, label %for.end
 
 for.body:                                         ; preds = %for.cond
-  call void @Body(i32* %res, i32* undef, i32* %d, i32* %p, i32 %i.0), !llvm.mem.parallel_loop_access !0
+  call void @Body(i32* %res, i32* undef, i32* %d, i32* %p, i32 %i.0), !llvm.access.group !0
   %inc = add nsw i32 %i.0, 1
-  br label %for.cond, !llvm.loop !0
+  br label %for.cond, !llvm.loop !1
 
 for.end:                                          ; preds = %for.cond
   ret void
 }
 
 ; CHECK-LABEL: @Test
-; CHECK: load i32,{{.*}}, !llvm.mem.parallel_loop_access !0
-; CHECK: load i32,{{.*}}, !llvm.mem.parallel_loop_access !0
-; CHECK: load i32,{{.*}}, !llvm.mem.parallel_loop_access !0
-; CHECK: store i32{{.*}}, !llvm.mem.parallel_loop_access !0
-; CHECK: br label %for.cond, !llvm.loop !0
+; CHECK: load i32,{{.*}}, !llvm.access.group !0
+; CHECK: load i32,{{.*}}, !llvm.access.group !0
+; CHECK: load i32,{{.*}}, !llvm.access.group !0
+; CHECK: store i32{{.*}}, !llvm.access.group !0
+; CHECK: br label %for.cond, !llvm.loop !1
 
 attributes #0 = { norecurse nounwind uwtable }
 
-!0 = distinct !{!0}
-
+!0 = distinct !{}
+!1 = distinct !{!0, !{!"llvm.loop.parallel_accesses", !0}}
diff --git a/test/Transforms/InstCombine/AMDGPU/amdgcn-demanded-vector-elts.ll b/test/Transforms/InstCombine/AMDGPU/amdgcn-demanded-vector-elts.ll
index af34a3fd3712f49b305b76c08c00d717ec78deef..6030d14c8e1ec5df675ce07cc284542e2a7d4d54 100644
--- a/test/Transforms/InstCombine/AMDGPU/amdgcn-demanded-vector-elts.ll
+++ b/test/Transforms/InstCombine/AMDGPU/amdgcn-demanded-vector-elts.ll
@@ -315,6 +315,966 @@ declare <2 x float> @llvm.amdgcn.buffer.load.format.v2f32(<4 x i32>, i32, i32, i
 declare <3 x float> @llvm.amdgcn.buffer.load.format.v3f32(<4 x i32>, i32, i32, i1, i1) #1
 declare <4 x float> @llvm.amdgcn.buffer.load.format.v4f32(<4 x i32>, i32, i32, i1, i1) #1
 
+; --------------------------------------------------------------------
+; llvm.amdgcn.raw.buffer.load
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @raw_buffer_load_f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 %coherent)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @raw_buffer_load_f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs, i32 %coherent) #0 {
+  %data = call float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 %coherent)
+  ret float %data
+}
+
+; CHECK-LABEL: @raw_buffer_load_v1f32(
+; CHECK-NEXT: %data = call <1 x float> @llvm.amdgcn.raw.buffer.load.v1f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 %coherent)
+; CHECK-NEXT: ret <1 x float> %data
+define amdgpu_ps <1 x float> @raw_buffer_load_v1f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs, i32 %coherent) #0 {
+  %data = call <1 x float> @llvm.amdgcn.raw.buffer.load.v1f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 %coherent)
+  ret <1 x float> %data
+}
+
+; CHECK-LABEL: @raw_buffer_load_v2f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.raw.buffer.load.v2f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 %coherent)
+; CHECK-NEXT: ret <2 x float> %data
+define amdgpu_ps <2 x float> @raw_buffer_load_v2f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs, i32 %coherent) #0 {
+  %data = call <2 x float> @llvm.amdgcn.raw.buffer.load.v2f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 %coherent)
+  ret <2 x float> %data
+}
+
+; CHECK-LABEL: @raw_buffer_load_v4f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.raw.buffer.load.v4f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 %coherent)
+; CHECK-NEXT: ret <4 x float> %data
+define amdgpu_ps <4 x float> @raw_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs, i32 %coherent) #0 {
+  %data = call <4 x float> @llvm.amdgcn.raw.buffer.load.v4f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 %coherent)
+  ret <4 x float> %data
+}
+
+; CHECK-LABEL: @extract_elt0_raw_buffer_load_v2f32(
+; CHECK: %data = call float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 %coherent)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_raw_buffer_load_v2f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs, i32 %coherent) #0 {
+  %data = call <2 x float> @llvm.amdgcn.raw.buffer.load.v2f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 %coherent)
+  %elt0 = extractelement <2 x float> %data, i32 0
+  ret float %elt0
+}
+
+; CHECK-LABEL: @extract_elt1_raw_buffer_load_v2f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.raw.buffer.load.v2f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 %coherent)
+; CHECK-NEXT: %elt1 = extractelement <2 x float> %data, i32 1
+; CHECK-NEXT: ret float %elt1
+define amdgpu_ps float @extract_elt1_raw_buffer_load_v2f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs, i32 %coherent) #0 {
+  %data = call <2 x float> @llvm.amdgcn.raw.buffer.load.v2f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 %coherent)
+  %elt1 = extractelement <2 x float> %data, i32 1
+  ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt0_raw_buffer_load_v4f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 %coherent)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_raw_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs, i32 %coherent) #0 {
+  %data = call <4 x float> @llvm.amdgcn.raw.buffer.load.v4f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 %coherent)
+  %elt0 = extractelement <4 x float> %data, i32 0
+  ret float %elt0
+}
+
+; CHECK-LABEL: @extract_elt1_raw_buffer_load_v4f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.raw.buffer.load.v2f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 %coherent)
+; CHECK-NEXT: %elt1 = extractelement <2 x float> %data, i32 1
+; CHECK-NEXT: ret float %elt1
+define amdgpu_ps float @extract_elt1_raw_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs, i32 %coherent) #0 {
+  %data = call <4 x float> @llvm.amdgcn.raw.buffer.load.v4f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 %coherent)
+  %elt1 = extractelement <4 x float> %data, i32 1
+  ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt2_raw_buffer_load_v4f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.raw.buffer.load.v4f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 %coherent)
+; CHECK-NEXT: %elt1 = extractelement <4 x float> %data, i32 2
+; CHECK-NEXT: ret float %elt1
+define amdgpu_ps float @extract_elt2_raw_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs, i32 %coherent) #0 {
+  %data = call <4 x float> @llvm.amdgcn.raw.buffer.load.v4f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 %coherent)
+  %elt1 = extractelement <4 x float> %data, i32 2
+  ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt3_raw_buffer_load_v4f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.raw.buffer.load.v4f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 %coherent)
+; CHECK-NEXT: %elt1 = extractelement <4 x float> %data, i32 3
+; CHECK-NEXT: ret float %elt1
+define amdgpu_ps float @extract_elt3_raw_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs, i32 %coherent) #0 {
+  %data = call <4 x float> @llvm.amdgcn.raw.buffer.load.v4f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 %coherent)
+  %elt1 = extractelement <4 x float> %data, i32 3
+  ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt0_elt1_raw_buffer_load_v4f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.raw.buffer.load.v2f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 %coherent)
+; CHECK-NEXT: ret <2 x float>
+define amdgpu_ps <2 x float> @extract_elt0_elt1_raw_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs, i32 %coherent) #0 {
+  %data = call <4 x float> @llvm.amdgcn.raw.buffer.load.v4f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 %coherent)
+  %shuf = shufflevector <4 x float> %data, <4 x float> undef, <2 x i32> <i32 0, i32 1>
+  ret <2 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt1_elt2_raw_buffer_load_v4f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.raw.buffer.load.v4f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 %coherent)
+; CHECK-NEXT: %shuf = shufflevector <4 x float> %data, <4 x float> undef, <2 x i32> <i32 1, i32 2>
+; CHECK-NEXT: ret <2 x float> %shuf
+define amdgpu_ps <2 x float> @extract_elt1_elt2_raw_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs, i32 %coherent) #0 {
+  %data = call <4 x float> @llvm.amdgcn.raw.buffer.load.v4f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 %coherent)
+  %shuf = shufflevector <4 x float> %data, <4 x float> undef, <2 x i32> <i32 1, i32 2>
+  ret <2 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt2_elt3_raw_buffer_load_v4f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.raw.buffer.load.v4f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 %coherent)
+; CHECK-NEXT: %shuf = shufflevector <4 x float> %data, <4 x float> undef, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT: ret <2 x float> %shuf
+define amdgpu_ps <2 x float> @extract_elt2_elt3_raw_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs, i32 %coherent) #0 {
+  %data = call <4 x float> @llvm.amdgcn.raw.buffer.load.v4f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 %coherent)
+  %shuf = shufflevector <4 x float> %data, <4 x float> undef, <2 x i32> <i32 2, i32 3>
+  ret <2 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt0_elt1_elt2_raw_buffer_load_v4f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.raw.buffer.load.v4f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 %coherent)
+; CHECK-NEXT: %shuf = shufflevector <4 x float> %data, <4 x float> undef, <3 x i32> <i32 0, i32 1, i32 2>
+; CHECK-NEXT: ret <3 x float> %shuf
+define amdgpu_ps <3 x float> @extract_elt0_elt1_elt2_raw_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs, i32 %coherent) #0 {
+  %data = call <4 x float> @llvm.amdgcn.raw.buffer.load.v4f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 %coherent)
+  %shuf = shufflevector <4 x float> %data, <4 x float> undef, <3 x i32> <i32 0, i32 1, i32 2>
+  ret <3 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt1_elt2_elt3_raw_buffer_load_v4f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.raw.buffer.load.v4f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 %coherent)
+; CHECK-NEXT: %shuf = shufflevector <4 x float> %data, <4 x float> undef, <3 x i32> <i32 1, i32 2, i32 3>
+; CHECK-NEXT: ret <3 x float> %shuf
+define amdgpu_ps <3 x float> @extract_elt1_elt2_elt3_raw_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs, i32 %coherent) #0 {
+  %data = call <4 x float> @llvm.amdgcn.raw.buffer.load.v4f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 %coherent)
+  %shuf = shufflevector <4 x float> %data, <4 x float> undef, <3 x i32> <i32 1, i32 2, i32 3>
+  ret <3 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt0_elt2_elt3_raw_buffer_load_v4f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.raw.buffer.load.v4f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 %coherent)
+; CHECK-NEXT: %shuf = shufflevector <4 x float> %data, <4 x float> undef, <3 x i32> <i32 0, i32 2, i32 3>
+; CHECK-NEXT: ret <3 x float> %shuf
+define amdgpu_ps <3 x float> @extract_elt0_elt2_elt3_raw_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs, i32 %coherent) #0 {
+  %data = call <4 x float> @llvm.amdgcn.raw.buffer.load.v4f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 %coherent)
+  %shuf = shufflevector <4 x float> %data, <4 x float> undef, <3 x i32> <i32 0, i32 2, i32 3>
+  ret <3 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt0_raw_buffer_load_v3f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 %coherent)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_raw_buffer_load_v3f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs, i32 %coherent) #0 {
+  %data = call <3 x float> @llvm.amdgcn.raw.buffer.load.v3f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 %coherent)
+  %elt0 = extractelement <3 x float> %data, i32 0
+  ret float %elt0
+}
+
+; CHECK-LABEL: @extract_elt1_raw_buffer_load_v3f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.raw.buffer.load.v2f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 %coherent)
+; CHECK-NEXT: %elt1 = extractelement <2 x float> %data, i32 1
+; CHECK-NEXT: ret float %elt1
+define amdgpu_ps float @extract_elt1_raw_buffer_load_v3f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs, i32 %coherent) #0 {
+  %data = call <3 x float> @llvm.amdgcn.raw.buffer.load.v3f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 %coherent)
+  %elt1 = extractelement <3 x float> %data, i32 1
+  ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt2_raw_buffer_load_v3f32(
+; CHECK-NEXT: %data = call <3 x float> @llvm.amdgcn.raw.buffer.load.v3f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 %coherent)
+; CHECK-NEXT: %elt1 = extractelement <3 x float> %data, i32 2
+; CHECK-NEXT: ret float %elt1
+define amdgpu_ps float @extract_elt2_raw_buffer_load_v3f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs, i32 %coherent) #0 {
+  %data = call <3 x float> @llvm.amdgcn.raw.buffer.load.v3f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 %coherent)
+  %elt1 = extractelement <3 x float> %data, i32 2
+  ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt0_elt1_raw_buffer_load_v3f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.raw.buffer.load.v2f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 %coherent)
+; CHECK-NEXT: ret <2 x float>
+define amdgpu_ps <2 x float> @extract_elt0_elt1_raw_buffer_load_v3f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs, i32 %coherent) #0 {
+  %data = call <3 x float> @llvm.amdgcn.raw.buffer.load.v3f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 %coherent)
+  %shuf = shufflevector <3 x float> %data, <3 x float> undef, <2 x i32> <i32 0, i32 1>
+  ret <2 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt1_elt2_raw_buffer_load_v3f32(
+; CHECK-NEXT: %data = call <3 x float> @llvm.amdgcn.raw.buffer.load.v3f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 %coherent)
+; CHECK-NEXT: %shuf = shufflevector <3 x float> %data, <3 x float> undef, <2 x i32> <i32 1, i32 2>
+; CHECK-NEXT: ret <2 x float> %shuf
+define amdgpu_ps <2 x float> @extract_elt1_elt2_raw_buffer_load_v3f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs, i32 %coherent) #0 {
+  %data = call <3 x float> @llvm.amdgcn.raw.buffer.load.v3f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 %coherent)
+  %shuf = shufflevector <3 x float> %data, <3 x float> undef, <2 x i32> <i32 1, i32 2>
+  ret <2 x float> %shuf
+}
+
+; CHECK-LABEL: @extract0_bitcast_raw_buffer_load_v4f32(
+; CHECK-NEXT: %tmp = call float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 %coherent)
+; CHECK-NEXT: %tmp2 = bitcast float %tmp to i32
+; CHECK-NEXT: ret i32 %tmp2
+define i32 @extract0_bitcast_raw_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs, i32 %coherent) #0 {
+  %tmp = call <4 x float> @llvm.amdgcn.raw.buffer.load.v4f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 %coherent)
+  %tmp1 = bitcast <4 x float> %tmp to <4 x i32>
+  %tmp2 = extractelement <4 x i32> %tmp1, i32 0
+  ret i32 %tmp2
+}
+
+; CHECK-LABEL: @extract0_bitcast_raw_buffer_load_v4i32(
+; CHECK-NEXT: %tmp = call i32 @llvm.amdgcn.raw.buffer.load.i32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 %coherent)
+; CHECK-NEXT: %tmp2 = bitcast i32 %tmp to float
+; CHECK-NEXT: ret float %tmp2
+define float @extract0_bitcast_raw_buffer_load_v4i32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs, i32 %coherent) #0 {
+  %tmp = call <4 x i32> @llvm.amdgcn.raw.buffer.load.v4i32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 %coherent)
+  %tmp1 = bitcast <4 x i32> %tmp to <4 x float>
+  %tmp2 = extractelement <4 x float> %tmp1, i32 0
+  ret float %tmp2
+}
+
+; CHECK-LABEL: @preserve_metadata_extract_elt0_raw_buffer_load_v2f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 %coherent), !fpmath !0
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @preserve_metadata_extract_elt0_raw_buffer_load_v2f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs, i32 %coherent) #0 {
+  %data = call <2 x float> @llvm.amdgcn.raw.buffer.load.v2f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 %coherent), !fpmath !0
+  %elt0 = extractelement <2 x float> %data, i32 0
+  ret float %elt0
+}
+
+declare float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32>, i32, i32, i32) #1
+declare <1 x float> @llvm.amdgcn.raw.buffer.load.v1f32(<4 x i32>, i32, i32, i32) #1
+declare <2 x float> @llvm.amdgcn.raw.buffer.load.v2f32(<4 x i32>, i32, i32, i32) #1
+declare <3 x float> @llvm.amdgcn.raw.buffer.load.v3f32(<4 x i32>, i32, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.raw.buffer.load.v4f32(<4 x i32>, i32, i32, i32) #1
+
+declare <4 x i32> @llvm.amdgcn.raw.buffer.load.v4i32(<4 x i32>, i32, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.raw.buffer.load.format
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @raw_buffer_load_format_f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.raw.buffer.load.format.f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 %coherent)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @raw_buffer_load_format_f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs, i32 %coherent) #0 {
+  %data = call float @llvm.amdgcn.raw.buffer.load.format.f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 %coherent)
+  ret float %data
+}
+
+; CHECK-LABEL: @raw_buffer_load_format_v1f32(
+; CHECK-NEXT: %data = call <1 x float> @llvm.amdgcn.raw.buffer.load.format.v1f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 %coherent)
+; CHECK-NEXT: ret <1 x float> %data
+define amdgpu_ps <1 x float> @raw_buffer_load_format_v1f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs, i32 %coherent) #0 {
+  %data = call <1 x float> @llvm.amdgcn.raw.buffer.load.format.v1f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 %coherent)
+  ret <1 x float> %data
+}
+
+; CHECK-LABEL: @raw_buffer_load_format_v2f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.raw.buffer.load.format.v2f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 %coherent)
+; CHECK-NEXT: ret <2 x float> %data
+define amdgpu_ps <2 x float> @raw_buffer_load_format_v2f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs, i32 %coherent) #0 {
+  %data = call <2 x float> @llvm.amdgcn.raw.buffer.load.format.v2f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 %coherent)
+  ret <2 x float> %data
+}
+
+; CHECK-LABEL: @raw_buffer_load_format_v4f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.raw.buffer.load.format.v4f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 %coherent)
+; CHECK-NEXT: ret <4 x float> %data
+define amdgpu_ps <4 x float> @raw_buffer_load_format_v4f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs, i32 %coherent) #0 {
+  %data = call <4 x float> @llvm.amdgcn.raw.buffer.load.format.v4f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 %coherent)
+  ret <4 x float> %data
+}
+
+; CHECK-LABEL: @extract_elt0_raw_buffer_load_format_v2f32(
+; CHECK: %data = call float @llvm.amdgcn.raw.buffer.load.format.f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 %coherent)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_raw_buffer_load_format_v2f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs, i32 %coherent) #0 {
+  %data = call <2 x float> @llvm.amdgcn.raw.buffer.load.format.v2f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 %coherent)
+  %elt0 = extractelement <2 x float> %data, i32 0
+  ret float %elt0
+}
+
+; CHECK-LABEL: @extract_elt1_raw_buffer_load_format_v2f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.raw.buffer.load.format.v2f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 %coherent)
+; CHECK-NEXT: %elt1 = extractelement <2 x float> %data, i32 1
+; CHECK-NEXT: ret float %elt1
+define amdgpu_ps float @extract_elt1_raw_buffer_load_format_v2f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs, i32 %coherent) #0 {
+  %data = call <2 x float> @llvm.amdgcn.raw.buffer.load.format.v2f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 %coherent)
+  %elt1 = extractelement <2 x float> %data, i32 1
+  ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt0_raw_buffer_load_format_v4f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.raw.buffer.load.format.f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 %coherent)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_raw_buffer_load_format_v4f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs, i32 %coherent) #0 {
+  %data = call <4 x float> @llvm.amdgcn.raw.buffer.load.format.v4f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 %coherent)
+  %elt0 = extractelement <4 x float> %data, i32 0
+  ret float %elt0
+}
+
+; CHECK-LABEL: @extract_elt1_raw_buffer_load_format_v4f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.raw.buffer.load.format.v2f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 %coherent)
+; CHECK-NEXT: %elt1 = extractelement <2 x float> %data, i32 1
+; CHECK-NEXT: ret float %elt1
+define amdgpu_ps float @extract_elt1_raw_buffer_load_format_v4f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs, i32 %coherent) #0 {
+  %data = call <4 x float> @llvm.amdgcn.raw.buffer.load.format.v4f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 %coherent)
+  %elt1 = extractelement <4 x float> %data, i32 1
+  ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt2_raw_buffer_load_format_v4f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.raw.buffer.load.format.v4f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 %coherent)
+; CHECK-NEXT: %elt1 = extractelement <4 x float> %data, i32 2
+; CHECK-NEXT: ret float %elt1
+define amdgpu_ps float @extract_elt2_raw_buffer_load_format_v4f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs, i32 %coherent) #0 {
+  %data = call <4 x float> @llvm.amdgcn.raw.buffer.load.format.v4f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 %coherent)
+  %elt1 = extractelement <4 x float> %data, i32 2
+  ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt3_raw_buffer_load_format_v4f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.raw.buffer.load.format.v4f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 %coherent)
+; CHECK-NEXT: %elt1 = extractelement <4 x float> %data, i32 3
+; CHECK-NEXT: ret float %elt1
+define amdgpu_ps float @extract_elt3_raw_buffer_load_format_v4f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs, i32 %coherent) #0 {
+  %data = call <4 x float> @llvm.amdgcn.raw.buffer.load.format.v4f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 %coherent)
+  %elt1 = extractelement <4 x float> %data, i32 3
+  ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt0_elt1_raw_buffer_load_format_v4f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.raw.buffer.load.format.v2f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 %coherent)
+; CHECK-NEXT: ret <2 x float>
+define amdgpu_ps <2 x float> @extract_elt0_elt1_raw_buffer_load_format_v4f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs, i32 %coherent) #0 {
+  %data = call <4 x float> @llvm.amdgcn.raw.buffer.load.format.v4f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 %coherent)
+  %shuf = shufflevector <4 x float> %data, <4 x float> undef, <2 x i32> <i32 0, i32 1>
+  ret <2 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt1_elt2_raw_buffer_load_format_v4f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.raw.buffer.load.format.v4f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 %coherent)
+; CHECK-NEXT: %shuf = shufflevector <4 x float> %data, <4 x float> undef, <2 x i32> <i32 1, i32 2>
+; CHECK-NEXT: ret <2 x float> %shuf
+define amdgpu_ps <2 x float> @extract_elt1_elt2_raw_buffer_load_format_v4f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs, i32 %coherent) #0 {
+  %data = call <4 x float> @llvm.amdgcn.raw.buffer.load.format.v4f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 %coherent)
+  %shuf = shufflevector <4 x float> %data, <4 x float> undef, <2 x i32> <i32 1, i32 2>
+  ret <2 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt2_elt3_raw_buffer_load_format_v4f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.raw.buffer.load.format.v4f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 %coherent)
+; CHECK-NEXT: %shuf = shufflevector <4 x float> %data, <4 x float> undef, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT: ret <2 x float> %shuf
+define amdgpu_ps <2 x float> @extract_elt2_elt3_raw_buffer_load_format_v4f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs, i32 %coherent) #0 {
+  %data = call <4 x float> @llvm.amdgcn.raw.buffer.load.format.v4f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 %coherent)
+  %shuf = shufflevector <4 x float> %data, <4 x float> undef, <2 x i32> <i32 2, i32 3>
+  ret <2 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt0_elt1_elt2_raw_buffer_load_format_v4f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.raw.buffer.load.format.v4f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 %coherent)
+; CHECK-NEXT: %shuf = shufflevector <4 x float> %data, <4 x float> undef, <3 x i32> <i32 0, i32 1, i32 2>
+; CHECK-NEXT: ret <3 x float> %shuf
+define amdgpu_ps <3 x float> @extract_elt0_elt1_elt2_raw_buffer_load_format_v4f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs, i32 %coherent) #0 {
+  %data = call <4 x float> @llvm.amdgcn.raw.buffer.load.format.v4f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 %coherent)
+  %shuf = shufflevector <4 x float> %data, <4 x float> undef, <3 x i32> <i32 0, i32 1, i32 2>
+  ret <3 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt1_elt2_elt3_raw_buffer_load_format_v4f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.raw.buffer.load.format.v4f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 %coherent)
+; CHECK-NEXT: %shuf = shufflevector <4 x float> %data, <4 x float> undef, <3 x i32> <i32 1, i32 2, i32 3>
+; CHECK-NEXT: ret <3 x float> %shuf
+define amdgpu_ps <3 x float> @extract_elt1_elt2_elt3_raw_buffer_load_format_v4f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs, i32 %coherent) #0 {
+  %data = call <4 x float> @llvm.amdgcn.raw.buffer.load.format.v4f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 %coherent)
+  %shuf = shufflevector <4 x float> %data, <4 x float> undef, <3 x i32> <i32 1, i32 2, i32 3>
+  ret <3 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt0_elt2_elt3_raw_buffer_load_format_v4f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.raw.buffer.load.format.v4f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 %coherent)
+; CHECK-NEXT: %shuf = shufflevector <4 x float> %data, <4 x float> undef, <3 x i32> <i32 0, i32 2, i32 3>
+; CHECK-NEXT: ret <3 x float> %shuf
+define amdgpu_ps <3 x float> @extract_elt0_elt2_elt3_raw_buffer_load_format_v4f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs, i32 %coherent) #0 {
+  %data = call <4 x float> @llvm.amdgcn.raw.buffer.load.format.v4f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 %coherent)
+  %shuf = shufflevector <4 x float> %data, <4 x float> undef, <3 x i32> <i32 0, i32 2, i32 3>
+  ret <3 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt0_raw_buffer_load_format_v3f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.raw.buffer.load.format.f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 %coherent)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_raw_buffer_load_format_v3f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs, i32 %coherent) #0 {
+  %data = call <3 x float> @llvm.amdgcn.raw.buffer.load.format.v3f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 %coherent)
+  %elt0 = extractelement <3 x float> %data, i32 0
+  ret float %elt0
+}
+
+; CHECK-LABEL: @extract_elt1_raw_buffer_load_format_v3f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.raw.buffer.load.format.v2f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 %coherent)
+; CHECK-NEXT: %elt1 = extractelement <2 x float> %data, i32 1
+; CHECK-NEXT: ret float %elt1
+define amdgpu_ps float @extract_elt1_raw_buffer_load_format_v3f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs, i32 %coherent) #0 {
+  %data = call <3 x float> @llvm.amdgcn.raw.buffer.load.format.v3f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 %coherent)
+  %elt1 = extractelement <3 x float> %data, i32 1
+  ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt2_raw_buffer_load_format_v3f32(
+; CHECK-NEXT: %data = call <3 x float> @llvm.amdgcn.raw.buffer.load.format.v3f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 %coherent)
+; CHECK-NEXT: %elt1 = extractelement <3 x float> %data, i32 2
+; CHECK-NEXT: ret float %elt1
+define amdgpu_ps float @extract_elt2_raw_buffer_load_format_v3f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs, i32 %coherent) #0 {
+  %data = call <3 x float> @llvm.amdgcn.raw.buffer.load.format.v3f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 %coherent)
+  %elt1 = extractelement <3 x float> %data, i32 2
+  ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt0_elt1_raw_buffer_load_format_v3f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.raw.buffer.load.format.v2f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 %coherent)
+; CHECK-NEXT: ret <2 x float>
+define amdgpu_ps <2 x float> @extract_elt0_elt1_raw_buffer_load_format_v3f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs, i32 %coherent) #0 {
+  %data = call <3 x float> @llvm.amdgcn.raw.buffer.load.format.v3f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 %coherent)
+  %shuf = shufflevector <3 x float> %data, <3 x float> undef, <2 x i32> <i32 0, i32 1>
+  ret <2 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt1_elt2_raw_buffer_load_format_v3f32(
+; CHECK-NEXT: %data = call <3 x float> @llvm.amdgcn.raw.buffer.load.format.v3f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 %coherent)
+; CHECK-NEXT: %shuf = shufflevector <3 x float> %data, <3 x float> undef, <2 x i32> <i32 1, i32 2>
+; CHECK-NEXT: ret <2 x float> %shuf
+define amdgpu_ps <2 x float> @extract_elt1_elt2_raw_buffer_load_format_v3f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs, i32 %coherent) #0 {
+  %data = call <3 x float> @llvm.amdgcn.raw.buffer.load.format.v3f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 %coherent)
+  %shuf = shufflevector <3 x float> %data, <3 x float> undef, <2 x i32> <i32 1, i32 2>
+  ret <2 x float> %shuf
+}
+
+; CHECK-LABEL: @extract0_bitcast_raw_buffer_load_format_v4f32(
+; CHECK-NEXT: %tmp = call float @llvm.amdgcn.raw.buffer.load.format.f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 %coherent)
+; CHECK-NEXT: %tmp2 = bitcast float %tmp to i32
+; CHECK-NEXT: ret i32 %tmp2
+define i32 @extract0_bitcast_raw_buffer_load_format_v4f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs, i32 %coherent) #0 {
+  %tmp = call <4 x float> @llvm.amdgcn.raw.buffer.load.format.v4f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 %coherent)
+  %tmp1 = bitcast <4 x float> %tmp to <4 x i32>
+  %tmp2 = extractelement <4 x i32> %tmp1, i32 0
+  ret i32 %tmp2
+}
+
+; CHECK-LABEL: @extract0_bitcast_raw_buffer_load_format_v4i32(
+; CHECK-NEXT: %tmp = call i32 @llvm.amdgcn.raw.buffer.load.format.i32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 %coherent)
+; CHECK-NEXT: %tmp2 = bitcast i32 %tmp to float
+; CHECK-NEXT: ret float %tmp2
+define float @extract0_bitcast_raw_buffer_load_format_v4i32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs, i32 %coherent) #0 {
+  %tmp = call <4 x i32> @llvm.amdgcn.raw.buffer.load.format.v4i32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 %coherent)
+  %tmp1 = bitcast <4 x i32> %tmp to <4 x float>
+  %tmp2 = extractelement <4 x float> %tmp1, i32 0
+  ret float %tmp2
+}
+
+; CHECK-LABEL: @preserve_metadata_extract_elt0_raw_buffer_load_format_v2f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.raw.buffer.load.format.f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 %coherent), !fpmath !0
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @preserve_metadata_extract_elt0_raw_buffer_load_format_v2f32(<4 x i32> inreg %rsrc, i32 %ofs, i32 %sofs, i32 %coherent) #0 {
+  %data = call <2 x float> @llvm.amdgcn.raw.buffer.load.format.v2f32(<4 x i32> %rsrc, i32 %ofs, i32 %sofs, i32 %coherent), !fpmath !0
+  %elt0 = extractelement <2 x float> %data, i32 0
+  ret float %elt0
+}
+
+declare float @llvm.amdgcn.raw.buffer.load.format.f32(<4 x i32>, i32, i32, i32) #1
+declare <1 x float> @llvm.amdgcn.raw.buffer.load.format.v1f32(<4 x i32>, i32, i32, i32) #1
+declare <2 x float> @llvm.amdgcn.raw.buffer.load.format.v2f32(<4 x i32>, i32, i32, i32) #1
+declare <3 x float> @llvm.amdgcn.raw.buffer.load.format.v3f32(<4 x i32>, i32, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.raw.buffer.load.format.v4f32(<4 x i32>, i32, i32, i32) #1
+
+declare <4 x i32> @llvm.amdgcn.raw.buffer.load.format.v4i32(<4 x i32>, i32, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.struct.buffer.load
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @struct_buffer_load_f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @struct_buffer_load_f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent) #0 {
+  %data = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent)
+  ret float %data
+}
+
+; CHECK-LABEL: @struct_buffer_load_v1f32(
+; CHECK-NEXT: %data = call <1 x float> @llvm.amdgcn.struct.buffer.load.v1f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent)
+; CHECK-NEXT: ret <1 x float> %data
+define amdgpu_ps <1 x float> @struct_buffer_load_v1f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent) #0 {
+  %data = call <1 x float> @llvm.amdgcn.struct.buffer.load.v1f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent)
+  ret <1 x float> %data
+}
+
+; CHECK-LABEL: @struct_buffer_load_v2f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.struct.buffer.load.v2f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent)
+; CHECK-NEXT: ret <2 x float> %data
+define amdgpu_ps <2 x float> @struct_buffer_load_v2f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent) #0 {
+  %data = call <2 x float> @llvm.amdgcn.struct.buffer.load.v2f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent)
+  ret <2 x float> %data
+}
+
+; CHECK-LABEL: @struct_buffer_load_v4f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.struct.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent)
+; CHECK-NEXT: ret <4 x float> %data
+define amdgpu_ps <4 x float> @struct_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent) #0 {
+  %data = call <4 x float> @llvm.amdgcn.struct.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent)
+  ret <4 x float> %data
+}
+
+; CHECK-LABEL: @extract_elt0_struct_buffer_load_v2f32(
+; CHECK: %data = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_struct_buffer_load_v2f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent) #0 {
+  %data = call <2 x float> @llvm.amdgcn.struct.buffer.load.v2f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent)
+  %elt0 = extractelement <2 x float> %data, i32 0
+  ret float %elt0
+}
+
+; CHECK-LABEL: @extract_elt1_struct_buffer_load_v2f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.struct.buffer.load.v2f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent)
+; CHECK-NEXT: %elt1 = extractelement <2 x float> %data, i32 1
+; CHECK-NEXT: ret float %elt1
+define amdgpu_ps float @extract_elt1_struct_buffer_load_v2f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent) #0 {
+  %data = call <2 x float> @llvm.amdgcn.struct.buffer.load.v2f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent)
+  %elt1 = extractelement <2 x float> %data, i32 1
+  ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt0_struct_buffer_load_v4f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_struct_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent) #0 {
+  %data = call <4 x float> @llvm.amdgcn.struct.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent)
+  %elt0 = extractelement <4 x float> %data, i32 0
+  ret float %elt0
+}
+
+; CHECK-LABEL: @extract_elt1_struct_buffer_load_v4f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.struct.buffer.load.v2f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent)
+; CHECK-NEXT: %elt1 = extractelement <2 x float> %data, i32 1
+; CHECK-NEXT: ret float %elt1
+define amdgpu_ps float @extract_elt1_struct_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent) #0 {
+  %data = call <4 x float> @llvm.amdgcn.struct.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent)
+  %elt1 = extractelement <4 x float> %data, i32 1
+  ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt2_struct_buffer_load_v4f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.struct.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent)
+; CHECK-NEXT: %elt1 = extractelement <4 x float> %data, i32 2
+; CHECK-NEXT: ret float %elt1
+define amdgpu_ps float @extract_elt2_struct_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent) #0 {
+  %data = call <4 x float> @llvm.amdgcn.struct.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent)
+  %elt1 = extractelement <4 x float> %data, i32 2
+  ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt3_struct_buffer_load_v4f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.struct.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent)
+; CHECK-NEXT: %elt1 = extractelement <4 x float> %data, i32 3
+; CHECK-NEXT: ret float %elt1
+define amdgpu_ps float @extract_elt3_struct_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent) #0 {
+  %data = call <4 x float> @llvm.amdgcn.struct.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent)
+  %elt1 = extractelement <4 x float> %data, i32 3
+  ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt0_elt1_struct_buffer_load_v4f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.struct.buffer.load.v2f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent)
+; CHECK-NEXT: ret <2 x float>
+define amdgpu_ps <2 x float> @extract_elt0_elt1_struct_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent) #0 {
+  %data = call <4 x float> @llvm.amdgcn.struct.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent)
+  %shuf = shufflevector <4 x float> %data, <4 x float> undef, <2 x i32> <i32 0, i32 1>
+  ret <2 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt1_elt2_struct_buffer_load_v4f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.struct.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent)
+; CHECK-NEXT: %shuf = shufflevector <4 x float> %data, <4 x float> undef, <2 x i32> <i32 1, i32 2>
+; CHECK-NEXT: ret <2 x float> %shuf
+define amdgpu_ps <2 x float> @extract_elt1_elt2_struct_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent) #0 {
+  %data = call <4 x float> @llvm.amdgcn.struct.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent)
+  %shuf = shufflevector <4 x float> %data, <4 x float> undef, <2 x i32> <i32 1, i32 2>
+  ret <2 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt2_elt3_struct_buffer_load_v4f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.struct.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent)
+; CHECK-NEXT: %shuf = shufflevector <4 x float> %data, <4 x float> undef, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT: ret <2 x float> %shuf
+define amdgpu_ps <2 x float> @extract_elt2_elt3_struct_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent) #0 {
+  %data = call <4 x float> @llvm.amdgcn.struct.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent)
+  %shuf = shufflevector <4 x float> %data, <4 x float> undef, <2 x i32> <i32 2, i32 3>
+  ret <2 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt0_elt1_elt2_struct_buffer_load_v4f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.struct.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent)
+; CHECK-NEXT: %shuf = shufflevector <4 x float> %data, <4 x float> undef, <3 x i32> <i32 0, i32 1, i32 2>
+; CHECK-NEXT: ret <3 x float> %shuf
+define amdgpu_ps <3 x float> @extract_elt0_elt1_elt2_struct_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent) #0 {
+  %data = call <4 x float> @llvm.amdgcn.struct.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent)
+  %shuf = shufflevector <4 x float> %data, <4 x float> undef, <3 x i32> <i32 0, i32 1, i32 2>
+  ret <3 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt1_elt2_elt3_struct_buffer_load_v4f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.struct.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent)
+; CHECK-NEXT: %shuf = shufflevector <4 x float> %data, <4 x float> undef, <3 x i32> <i32 1, i32 2, i32 3>
+; CHECK-NEXT: ret <3 x float> %shuf
+define amdgpu_ps <3 x float> @extract_elt1_elt2_elt3_struct_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent) #0 {
+  %data = call <4 x float> @llvm.amdgcn.struct.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent)
+  %shuf = shufflevector <4 x float> %data, <4 x float> undef, <3 x i32> <i32 1, i32 2, i32 3>
+  ret <3 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt0_elt2_elt3_struct_buffer_load_v4f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.struct.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent)
+; CHECK-NEXT: %shuf = shufflevector <4 x float> %data, <4 x float> undef, <3 x i32> <i32 0, i32 2, i32 3>
+; CHECK-NEXT: ret <3 x float> %shuf
+define amdgpu_ps <3 x float> @extract_elt0_elt2_elt3_struct_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent) #0 {
+  %data = call <4 x float> @llvm.amdgcn.struct.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent)
+  %shuf = shufflevector <4 x float> %data, <4 x float> undef, <3 x i32> <i32 0, i32 2, i32 3>
+  ret <3 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt0_struct_buffer_load_v3f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_struct_buffer_load_v3f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent) #0 {
+  %data = call <3 x float> @llvm.amdgcn.struct.buffer.load.v3f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent)
+  %elt0 = extractelement <3 x float> %data, i32 0
+  ret float %elt0
+}
+
+; CHECK-LABEL: @extract_elt1_struct_buffer_load_v3f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.struct.buffer.load.v2f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent)
+; CHECK-NEXT: %elt1 = extractelement <2 x float> %data, i32 1
+; CHECK-NEXT: ret float %elt1
+define amdgpu_ps float @extract_elt1_struct_buffer_load_v3f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent) #0 {
+  %data = call <3 x float> @llvm.amdgcn.struct.buffer.load.v3f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent)
+  %elt1 = extractelement <3 x float> %data, i32 1
+  ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt2_struct_buffer_load_v3f32(
+; CHECK-NEXT: %data = call <3 x float> @llvm.amdgcn.struct.buffer.load.v3f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent)
+; CHECK-NEXT: %elt1 = extractelement <3 x float> %data, i32 2
+; CHECK-NEXT: ret float %elt1
+define amdgpu_ps float @extract_elt2_struct_buffer_load_v3f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent) #0 {
+  %data = call <3 x float> @llvm.amdgcn.struct.buffer.load.v3f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent)
+  %elt1 = extractelement <3 x float> %data, i32 2
+  ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt0_elt1_struct_buffer_load_v3f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.struct.buffer.load.v2f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent)
+; CHECK-NEXT: ret <2 x float>
+define amdgpu_ps <2 x float> @extract_elt0_elt1_struct_buffer_load_v3f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent) #0 {
+  %data = call <3 x float> @llvm.amdgcn.struct.buffer.load.v3f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent)
+  %shuf = shufflevector <3 x float> %data, <3 x float> undef, <2 x i32> <i32 0, i32 1>
+  ret <2 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt1_elt2_struct_buffer_load_v3f32(
+; CHECK-NEXT: %data = call <3 x float> @llvm.amdgcn.struct.buffer.load.v3f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent)
+; CHECK-NEXT: %shuf = shufflevector <3 x float> %data, <3 x float> undef, <2 x i32> <i32 1, i32 2>
+; CHECK-NEXT: ret <2 x float> %shuf
+define amdgpu_ps <2 x float> @extract_elt1_elt2_struct_buffer_load_v3f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent) #0 {
+  %data = call <3 x float> @llvm.amdgcn.struct.buffer.load.v3f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent)
+  %shuf = shufflevector <3 x float> %data, <3 x float> undef, <2 x i32> <i32 1, i32 2>
+  ret <2 x float> %shuf
+}
+
+; CHECK-LABEL: @extract0_bitcast_struct_buffer_load_v4f32(
+; CHECK-NEXT: %tmp = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent)
+; CHECK-NEXT: %tmp2 = bitcast float %tmp to i32
+; CHECK-NEXT: ret i32 %tmp2
+define i32 @extract0_bitcast_struct_buffer_load_v4f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent) #0 {
+  %tmp = call <4 x float> @llvm.amdgcn.struct.buffer.load.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent)
+  %tmp1 = bitcast <4 x float> %tmp to <4 x i32>
+  %tmp2 = extractelement <4 x i32> %tmp1, i32 0
+  ret i32 %tmp2
+}
+
+; CHECK-LABEL: @extract0_bitcast_struct_buffer_load_v4i32(
+; CHECK-NEXT: %tmp = call i32 @llvm.amdgcn.struct.buffer.load.i32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent)
+; CHECK-NEXT: %tmp2 = bitcast i32 %tmp to float
+; CHECK-NEXT: ret float %tmp2
+define float @extract0_bitcast_struct_buffer_load_v4i32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent) #0 {
+  %tmp = call <4 x i32> @llvm.amdgcn.struct.buffer.load.v4i32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent)
+  %tmp1 = bitcast <4 x i32> %tmp to <4 x float>
+  %tmp2 = extractelement <4 x float> %tmp1, i32 0
+  ret float %tmp2
+}
+
+; CHECK-LABEL: @preserve_metadata_extract_elt0_struct_buffer_load_v2f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent), !fpmath !0
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @preserve_metadata_extract_elt0_struct_buffer_load_v2f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent) #0 {
+  %data = call <2 x float> @llvm.amdgcn.struct.buffer.load.v2f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent), !fpmath !0
+  %elt0 = extractelement <2 x float> %data, i32 0
+  ret float %elt0
+}
+
+declare float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32>, i32, i32, i32, i32) #1
+declare <1 x float> @llvm.amdgcn.struct.buffer.load.v1f32(<4 x i32>, i32, i32, i32, i32) #1
+declare <2 x float> @llvm.amdgcn.struct.buffer.load.v2f32(<4 x i32>, i32, i32, i32, i32) #1
+declare <3 x float> @llvm.amdgcn.struct.buffer.load.v3f32(<4 x i32>, i32, i32, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.struct.buffer.load.v4f32(<4 x i32>, i32, i32, i32, i32) #1
+
+declare <4 x i32> @llvm.amdgcn.struct.buffer.load.v4i32(<4 x i32>, i32, i32, i32, i32) #1
+
+; --------------------------------------------------------------------
+; llvm.amdgcn.struct.buffer.load.format
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @struct_buffer_load_format_f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.struct.buffer.load.format.f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @struct_buffer_load_format_f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent) #0 {
+  %data = call float @llvm.amdgcn.struct.buffer.load.format.f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent)
+  ret float %data
+}
+
+; CHECK-LABEL: @struct_buffer_load_format_v1f32(
+; CHECK-NEXT: %data = call <1 x float> @llvm.amdgcn.struct.buffer.load.format.v1f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent)
+; CHECK-NEXT: ret <1 x float> %data
+define amdgpu_ps <1 x float> @struct_buffer_load_format_v1f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent) #0 {
+  %data = call <1 x float> @llvm.amdgcn.struct.buffer.load.format.v1f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent)
+  ret <1 x float> %data
+}
+
+; CHECK-LABEL: @struct_buffer_load_format_v2f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.struct.buffer.load.format.v2f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent)
+; CHECK-NEXT: ret <2 x float> %data
+define amdgpu_ps <2 x float> @struct_buffer_load_format_v2f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent) #0 {
+  %data = call <2 x float> @llvm.amdgcn.struct.buffer.load.format.v2f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent)
+  ret <2 x float> %data
+}
+
+; CHECK-LABEL: @struct_buffer_load_format_v4f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.struct.buffer.load.format.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent)
+; CHECK-NEXT: ret <4 x float> %data
+define amdgpu_ps <4 x float> @struct_buffer_load_format_v4f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent) #0 {
+  %data = call <4 x float> @llvm.amdgcn.struct.buffer.load.format.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent)
+  ret <4 x float> %data
+}
+
+; CHECK-LABEL: @extract_elt0_struct_buffer_load_format_v2f32(
+; CHECK: %data = call float @llvm.amdgcn.struct.buffer.load.format.f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_struct_buffer_load_format_v2f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent) #0 {
+  %data = call <2 x float> @llvm.amdgcn.struct.buffer.load.format.v2f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent)
+  %elt0 = extractelement <2 x float> %data, i32 0
+  ret float %elt0
+}
+
+; CHECK-LABEL: @extract_elt1_struct_buffer_load_format_v2f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.struct.buffer.load.format.v2f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent)
+; CHECK-NEXT: %elt1 = extractelement <2 x float> %data, i32 1
+; CHECK-NEXT: ret float %elt1
+define amdgpu_ps float @extract_elt1_struct_buffer_load_format_v2f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent) #0 {
+  %data = call <2 x float> @llvm.amdgcn.struct.buffer.load.format.v2f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent)
+  %elt1 = extractelement <2 x float> %data, i32 1
+  ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt0_struct_buffer_load_format_v4f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.struct.buffer.load.format.f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_struct_buffer_load_format_v4f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent) #0 {
+  %data = call <4 x float> @llvm.amdgcn.struct.buffer.load.format.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent)
+  %elt0 = extractelement <4 x float> %data, i32 0
+  ret float %elt0
+}
+
+; CHECK-LABEL: @extract_elt1_struct_buffer_load_format_v4f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.struct.buffer.load.format.v2f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent)
+; CHECK-NEXT: %elt1 = extractelement <2 x float> %data, i32 1
+; CHECK-NEXT: ret float %elt1
+define amdgpu_ps float @extract_elt1_struct_buffer_load_format_v4f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent) #0 {
+  %data = call <4 x float> @llvm.amdgcn.struct.buffer.load.format.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent)
+  %elt1 = extractelement <4 x float> %data, i32 1
+  ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt2_struct_buffer_load_format_v4f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.struct.buffer.load.format.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent)
+; CHECK-NEXT: %elt1 = extractelement <4 x float> %data, i32 2
+; CHECK-NEXT: ret float %elt1
+define amdgpu_ps float @extract_elt2_struct_buffer_load_format_v4f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent) #0 {
+  %data = call <4 x float> @llvm.amdgcn.struct.buffer.load.format.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent)
+  %elt1 = extractelement <4 x float> %data, i32 2
+  ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt3_struct_buffer_load_format_v4f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.struct.buffer.load.format.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent)
+; CHECK-NEXT: %elt1 = extractelement <4 x float> %data, i32 3
+; CHECK-NEXT: ret float %elt1
+define amdgpu_ps float @extract_elt3_struct_buffer_load_format_v4f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent) #0 {
+  %data = call <4 x float> @llvm.amdgcn.struct.buffer.load.format.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent)
+  %elt1 = extractelement <4 x float> %data, i32 3
+  ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt0_elt1_struct_buffer_load_format_v4f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.struct.buffer.load.format.v2f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent)
+; CHECK-NEXT: ret <2 x float>
+define amdgpu_ps <2 x float> @extract_elt0_elt1_struct_buffer_load_format_v4f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent) #0 {
+  %data = call <4 x float> @llvm.amdgcn.struct.buffer.load.format.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent)
+  %shuf = shufflevector <4 x float> %data, <4 x float> undef, <2 x i32> <i32 0, i32 1>
+  ret <2 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt1_elt2_struct_buffer_load_format_v4f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.struct.buffer.load.format.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent)
+; CHECK-NEXT: %shuf = shufflevector <4 x float> %data, <4 x float> undef, <2 x i32> <i32 1, i32 2>
+; CHECK-NEXT: ret <2 x float> %shuf
+define amdgpu_ps <2 x float> @extract_elt1_elt2_struct_buffer_load_format_v4f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent) #0 {
+  %data = call <4 x float> @llvm.amdgcn.struct.buffer.load.format.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent)
+  %shuf = shufflevector <4 x float> %data, <4 x float> undef, <2 x i32> <i32 1, i32 2>
+  ret <2 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt2_elt3_struct_buffer_load_format_v4f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.struct.buffer.load.format.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent)
+; CHECK-NEXT: %shuf = shufflevector <4 x float> %data, <4 x float> undef, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT: ret <2 x float> %shuf
+define amdgpu_ps <2 x float> @extract_elt2_elt3_struct_buffer_load_format_v4f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent) #0 {
+  %data = call <4 x float> @llvm.amdgcn.struct.buffer.load.format.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent)
+  %shuf = shufflevector <4 x float> %data, <4 x float> undef, <2 x i32> <i32 2, i32 3>
+  ret <2 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt0_elt1_elt2_struct_buffer_load_format_v4f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.struct.buffer.load.format.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent)
+; CHECK-NEXT: %shuf = shufflevector <4 x float> %data, <4 x float> undef, <3 x i32> <i32 0, i32 1, i32 2>
+; CHECK-NEXT: ret <3 x float> %shuf
+define amdgpu_ps <3 x float> @extract_elt0_elt1_elt2_struct_buffer_load_format_v4f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent) #0 {
+  %data = call <4 x float> @llvm.amdgcn.struct.buffer.load.format.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent)
+  %shuf = shufflevector <4 x float> %data, <4 x float> undef, <3 x i32> <i32 0, i32 1, i32 2>
+  ret <3 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt1_elt2_elt3_struct_buffer_load_format_v4f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.struct.buffer.load.format.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent)
+; CHECK-NEXT: %shuf = shufflevector <4 x float> %data, <4 x float> undef, <3 x i32> <i32 1, i32 2, i32 3>
+; CHECK-NEXT: ret <3 x float> %shuf
+define amdgpu_ps <3 x float> @extract_elt1_elt2_elt3_struct_buffer_load_format_v4f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent) #0 {
+  %data = call <4 x float> @llvm.amdgcn.struct.buffer.load.format.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent)
+  %shuf = shufflevector <4 x float> %data, <4 x float> undef, <3 x i32> <i32 1, i32 2, i32 3>
+  ret <3 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt0_elt2_elt3_struct_buffer_load_format_v4f32(
+; CHECK-NEXT: %data = call <4 x float> @llvm.amdgcn.struct.buffer.load.format.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent)
+; CHECK-NEXT: %shuf = shufflevector <4 x float> %data, <4 x float> undef, <3 x i32> <i32 0, i32 2, i32 3>
+; CHECK-NEXT: ret <3 x float> %shuf
+define amdgpu_ps <3 x float> @extract_elt0_elt2_elt3_struct_buffer_load_format_v4f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent) #0 {
+  %data = call <4 x float> @llvm.amdgcn.struct.buffer.load.format.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent)
+  %shuf = shufflevector <4 x float> %data, <4 x float> undef, <3 x i32> <i32 0, i32 2, i32 3>
+  ret <3 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt0_struct_buffer_load_format_v3f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.struct.buffer.load.format.f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent)
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @extract_elt0_struct_buffer_load_format_v3f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent) #0 {
+  %data = call <3 x float> @llvm.amdgcn.struct.buffer.load.format.v3f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent)
+  %elt0 = extractelement <3 x float> %data, i32 0
+  ret float %elt0
+}
+
+; CHECK-LABEL: @extract_elt1_struct_buffer_load_format_v3f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.struct.buffer.load.format.v2f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent)
+; CHECK-NEXT: %elt1 = extractelement <2 x float> %data, i32 1
+; CHECK-NEXT: ret float %elt1
+define amdgpu_ps float @extract_elt1_struct_buffer_load_format_v3f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent) #0 {
+  %data = call <3 x float> @llvm.amdgcn.struct.buffer.load.format.v3f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent)
+  %elt1 = extractelement <3 x float> %data, i32 1
+  ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt2_struct_buffer_load_format_v3f32(
+; CHECK-NEXT: %data = call <3 x float> @llvm.amdgcn.struct.buffer.load.format.v3f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent)
+; CHECK-NEXT: %elt1 = extractelement <3 x float> %data, i32 2
+; CHECK-NEXT: ret float %elt1
+define amdgpu_ps float @extract_elt2_struct_buffer_load_format_v3f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent) #0 {
+  %data = call <3 x float> @llvm.amdgcn.struct.buffer.load.format.v3f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent)
+  %elt1 = extractelement <3 x float> %data, i32 2
+  ret float %elt1
+}
+
+; CHECK-LABEL: @extract_elt0_elt1_struct_buffer_load_format_v3f32(
+; CHECK-NEXT: %data = call <2 x float> @llvm.amdgcn.struct.buffer.load.format.v2f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent)
+; CHECK-NEXT: ret <2 x float>
+define amdgpu_ps <2 x float> @extract_elt0_elt1_struct_buffer_load_format_v3f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent) #0 {
+  %data = call <3 x float> @llvm.amdgcn.struct.buffer.load.format.v3f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent)
+  %shuf = shufflevector <3 x float> %data, <3 x float> undef, <2 x i32> <i32 0, i32 1>
+  ret <2 x float> %shuf
+}
+
+; CHECK-LABEL: @extract_elt1_elt2_struct_buffer_load_format_v3f32(
+; CHECK-NEXT: %data = call <3 x float> @llvm.amdgcn.struct.buffer.load.format.v3f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent)
+; CHECK-NEXT: %shuf = shufflevector <3 x float> %data, <3 x float> undef, <2 x i32> <i32 1, i32 2>
+; CHECK-NEXT: ret <2 x float> %shuf
+define amdgpu_ps <2 x float> @extract_elt1_elt2_struct_buffer_load_format_v3f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent) #0 {
+  %data = call <3 x float> @llvm.amdgcn.struct.buffer.load.format.v3f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent)
+  %shuf = shufflevector <3 x float> %data, <3 x float> undef, <2 x i32> <i32 1, i32 2>
+  ret <2 x float> %shuf
+}
+
+; CHECK-LABEL: @extract0_bitcast_struct_buffer_load_format_v4f32(
+; CHECK-NEXT: %tmp = call float @llvm.amdgcn.struct.buffer.load.format.f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent)
+; CHECK-NEXT: %tmp2 = bitcast float %tmp to i32
+; CHECK-NEXT: ret i32 %tmp2
+define i32 @extract0_bitcast_struct_buffer_load_format_v4f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent) #0 {
+  %tmp = call <4 x float> @llvm.amdgcn.struct.buffer.load.format.v4f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent)
+  %tmp1 = bitcast <4 x float> %tmp to <4 x i32>
+  %tmp2 = extractelement <4 x i32> %tmp1, i32 0
+  ret i32 %tmp2
+}
+
+; CHECK-LABEL: @extract0_bitcast_struct_buffer_load_format_v4i32(
+; CHECK-NEXT: %tmp = call i32 @llvm.amdgcn.struct.buffer.load.format.i32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent)
+; CHECK-NEXT: %tmp2 = bitcast i32 %tmp to float
+; CHECK-NEXT: ret float %tmp2
+define float @extract0_bitcast_struct_buffer_load_format_v4i32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent) #0 {
+  %tmp = call <4 x i32> @llvm.amdgcn.struct.buffer.load.format.v4i32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent)
+  %tmp1 = bitcast <4 x i32> %tmp to <4 x float>
+  %tmp2 = extractelement <4 x float> %tmp1, i32 0
+  ret float %tmp2
+}
+
+; CHECK-LABEL: @preserve_metadata_extract_elt0_struct_buffer_load_format_v2f32(
+; CHECK-NEXT: %data = call float @llvm.amdgcn.struct.buffer.load.format.f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent), !fpmath !0
+; CHECK-NEXT: ret float %data
+define amdgpu_ps float @preserve_metadata_extract_elt0_struct_buffer_load_format_v2f32(<4 x i32> inreg %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent) #0 {
+  %data = call <2 x float> @llvm.amdgcn.struct.buffer.load.format.v2f32(<4 x i32> %rsrc, i32 %idx, i32 %ofs, i32 %sofs, i32 %coherent), !fpmath !0
+  %elt0 = extractelement <2 x float> %data, i32 0
+  ret float %elt0
+}
+
+declare float @llvm.amdgcn.struct.buffer.load.format.f32(<4 x i32>, i32, i32, i32, i32) #1
+declare <1 x float> @llvm.amdgcn.struct.buffer.load.format.v1f32(<4 x i32>, i32, i32, i32, i32) #1
+declare <2 x float> @llvm.amdgcn.struct.buffer.load.format.v2f32(<4 x i32>, i32, i32, i32, i32) #1
+declare <3 x float> @llvm.amdgcn.struct.buffer.load.format.v3f32(<4 x i32>, i32, i32, i32, i32) #1
+declare <4 x float> @llvm.amdgcn.struct.buffer.load.format.v4f32(<4 x i32>, i32, i32, i32, i32) #1
+
+declare <4 x i32> @llvm.amdgcn.struct.buffer.load.format.v4i32(<4 x i32>, i32, i32, i32, i32) #1
+
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.image.sample
 ; --------------------------------------------------------------------
@@ -328,6 +1288,28 @@ define amdgpu_ps float @extract_elt0_image_sample_1d_v4f32_f32(float %vaddr, <8
   ret float %elt0
 }
 
+; Check that the intrinsic remains unchanged in the presence of TFE or LWE
+; CHECK-LABEL: @extract_elt0_image_sample_1d_v4f32_f32_tfe(
+; CHECK-NEXT: %data = call { <4 x float>, i32 } @llvm.amdgcn.image.sample.1d.sl_v4f32i32s.f32(i32 15, float %vaddr, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 1, i32 0)
+; CHECK: ret float %elt0
+define amdgpu_ps float @extract_elt0_image_sample_1d_v4f32_f32_tfe(float %vaddr, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+  %data = call {<4 x float>,i32} @llvm.amdgcn.image.sample.1d.sl_v4f32i32s.f32(i32 15, float %vaddr, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 1, i32 0)
+  %data.vec = extractvalue {<4 x float>,i32} %data, 0
+  %elt0 = extractelement <4 x float> %data.vec, i32 0
+  ret float %elt0
+}
+
+; Check that the intrinsic remains unchanged in the presence of TFE or LWE
+; CHECK-LABEL: @extract_elt0_image_sample_1d_v4f32_f32_lwe(
+; CHECK-NEXT: %data = call { <4 x float>, i32 } @llvm.amdgcn.image.sample.1d.sl_v4f32i32s.f32(i32 15, float %vaddr, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 2, i32 0)
+; CHECK: ret float %elt0
+define amdgpu_ps float @extract_elt0_image_sample_1d_v4f32_f32_lwe(float %vaddr, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 {
+  %data = call {<4 x float>,i32} @llvm.amdgcn.image.sample.1d.sl_v4f32i32s.f32(i32 15, float %vaddr, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 2, i32 0)
+  %data.vec = extractvalue {<4 x float>,i32} %data, 0
+  %elt0 = extractelement <4 x float> %data.vec, i32 0
+  ret float %elt0
+}
+
 ; CHECK-LABEL: @extract_elt0_image_sample_2d_v4f32_f32(
 ; CHECK-NEXT: %data = call float @llvm.amdgcn.image.sample.2d.f32.f32(i32 1, float %s, float %t, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0)
 ; CHECK-NEXT: ret float %data
@@ -506,6 +1488,7 @@ define amdgpu_ps <3 x float> @extract_elt0_elt1_elt2_dmask_1111_image_sample_1d_
 }
 
 declare <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
+declare {<4 x float>,i32} @llvm.amdgcn.image.sample.1d.sl_v4f32i32s.f32(i32, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 declare <4 x float> @llvm.amdgcn.image.sample.2d.v4f32.f32(i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 declare <4 x float> @llvm.amdgcn.image.sample.3d.v4f32.f32(i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
 declare <4 x float> @llvm.amdgcn.image.sample.1darray.v4f32.f32(i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1
@@ -1412,7 +2395,29 @@ define amdgpu_ps float @extract_elt0_image_getresinfo_1d_v4f32_i32(i32 %mip, <8
   ret float %elt0
 }
 
+; Verify that we don't creash on non-constant operand.
+define protected <4 x half> @__llvm_amdgcn_image_sample_d_1darray_v4f16_f32_f32(i32, float, float, float, float, <8 x i32>, <4 x i32>, i1 zeroext, i32, i32) local_unnamed_addr {
+  %tmp = tail call <4 x half> @llvm.amdgcn.image.sample.d.1darray.v4f16.f32.f32(i32 %0, float %1, float %2, float %3, float %4, <8 x i32> %5, <4 x i32> %6, i1 zeroext %7, i32 %8, i32 %9) #1
+  ret <4 x half> %tmp
+}
+
 declare <4 x float> @llvm.amdgcn.image.getresinfo.1d.v4f32.i32(i32, i32, <8 x i32>, i32, i32) #1
+declare <4 x half> @llvm.amdgcn.image.sample.d.1darray.v4f16.f32.f32(i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32)
+
+; --------------------------------------------------------------------
+; TFE / LWE
+; --------------------------------------------------------------------
+
+; CHECK-LABEL: @extract_elt0_tfe_image_load_1d_v4f32i32_i32(
+; CHECK-NEXT: %data = call { <4 x float>, i32 } @llvm.amdgcn.image.load.1d.sl_v4f32i32s.i32(i32 15, i32 %s, <8 x i32> %rsrc, i32 0, i32 1)
+define amdgpu_ps float @extract_elt0_tfe_image_load_1d_v4f32i32_i32(i32 %s, <8 x i32> inreg %rsrc) #0 {
+  %data = call { <4 x float>, i32 } @llvm.amdgcn.image.load.1d.sl_v4f32i32s.i32(i32 15, i32 %s, <8 x i32> %rsrc, i32 0, i32 1)
+  %rgba = extractvalue { <4 x float>, i32 } %data, 0
+  %elt0 = extractelement <4 x float> %rgba, i32 0
+  ret float %elt0
+}
+
+declare {<4 x float>, i32} @llvm.amdgcn.image.load.1d.sl_v4f32i32s.i32(i32, i32, <8 x i32>, i32, i32) #1
 
 attributes #0 = { nounwind }
 attributes #1 = { nounwind readonly }
diff --git a/test/Transforms/InstCombine/AMDGPU/amdgcn-intrinsics.ll b/test/Transforms/InstCombine/AMDGPU/amdgcn-intrinsics.ll
index 0a179d1f96d70f9b41819086114aaec8640aba86..b8e19e2bc94932485e099924b51d98cbc96088d8 100644
--- a/test/Transforms/InstCombine/AMDGPU/amdgcn-intrinsics.ll
+++ b/test/Transforms/InstCombine/AMDGPU/amdgcn-intrinsics.ll
@@ -1406,6 +1406,7 @@ define float @fmed3_0_1_undef_f32() {
 
 declare i64 @llvm.amdgcn.icmp.i32(i32, i32, i32) nounwind readnone convergent
 declare i64 @llvm.amdgcn.icmp.i64(i64, i64, i32) nounwind readnone convergent
+declare i64 @llvm.amdgcn.icmp.i1(i1, i1, i32) nounwind readnone convergent
 
 ; Make sure there's no crash for invalid input
 ; CHECK-LABEL: @invalid_nonconstant_icmp_code(
@@ -1815,6 +1816,198 @@ define i64 @fold_icmp_ne_0_zext_icmp_ult_i16(i16 %a, i16 %b) {
   ret i64 %mask
 }
 
+; 1-bit NE comparisons
+
+; CHECK-LABEL: @fold_icmp_i1_ne_0_icmp_eq_i1(
+; CHECK-NEXT: icmp
+; CHECK-NEXT: call i64 @llvm.amdgcn.icmp.i1(i1 %cmp, i1 false, i32 33)
+define i64 @fold_icmp_i1_ne_0_icmp_eq_i1(i32 %a, i32 %b) {
+  %cmp = icmp eq i32 %a, %b
+  %mask = call i64 @llvm.amdgcn.icmp.i1(i1 %cmp, i1 false, i32 33)
+  ret i64 %mask
+}
+
+; CHECK-LABEL: @fold_icmp_i1_ne_0_icmp_ne_i1(
+; CHECK-NEXT: icmp
+; CHECK-NEXT: call i64 @llvm.amdgcn.icmp.i1(i1 %cmp, i1 false, i32 33)
+define i64 @fold_icmp_i1_ne_0_icmp_ne_i1(i32 %a, i32 %b) {
+  %cmp = icmp ne i32 %a, %b
+  %mask = call i64 @llvm.amdgcn.icmp.i1(i1 %cmp, i1 false, i32 33)
+  ret i64 %mask
+}
+
+; CHECK-LABEL: @fold_icmp_i1_ne_0_icmp_sle_i1(
+; CHECK-NEXT: icmp
+; CHECK-NEXT: call i64 @llvm.amdgcn.icmp.i1(i1 %cmp, i1 false, i32 33)
+define i64 @fold_icmp_i1_ne_0_icmp_sle_i1(i32 %a, i32 %b) {
+  %cmp = icmp sle i32 %a, %b
+  %mask = call i64 @llvm.amdgcn.icmp.i1(i1 %cmp, i1 false, i32 33)
+  ret i64 %mask
+}
+
+; CHECK-LABEL: @fold_icmp_i1_ne_0_icmp_ugt_i64(
+; CHECK-NEXT: icmp
+; CHECK-NEXT: call i64 @llvm.amdgcn.icmp.i1(i1 %cmp, i1 false, i32 33)
+define i64 @fold_icmp_i1_ne_0_icmp_ugt_i64(i64 %a, i64 %b) {
+  %cmp = icmp ugt i64 %a, %b
+  %mask = call i64 @llvm.amdgcn.icmp.i1(i1 %cmp, i1 false, i32 33)
+  ret i64 %mask
+}
+
+; CHECK-LABEL: @fold_icmp_i1_ne_0_icmp_ult_swap_i64(
+; CHECK-NEXT: icmp
+; CHECK-NEXT: call i64 @llvm.amdgcn.icmp.i1(i1 %cmp, i1 false, i32 33)
+define i64 @fold_icmp_i1_ne_0_icmp_ult_swap_i64(i64 %a, i64 %b) {
+  %cmp = icmp ugt i64 %a, %b
+  %mask = call i64 @llvm.amdgcn.icmp.i1(i1 false, i1 %cmp, i32 33)
+  ret i64 %mask
+}
+
+; CHECK-LABEL: @fold_icmp_i1_ne_0_fcmp_oeq_f32(
+; CHECK-NEXT: fcmp
+; CHECK-NEXT: call i64 @llvm.amdgcn.icmp.i1(i1 %cmp, i1 false, i32 33)
+define i64 @fold_icmp_i1_ne_0_fcmp_oeq_f32(float %a, float %b) {
+  %cmp = fcmp oeq float %a, %b
+  %mask = call i64 @llvm.amdgcn.icmp.i1(i1 %cmp, i1 false, i32 33)
+  ret i64 %mask
+}
+
+; CHECK-LABEL: @fold_icmp_i1_ne_0_fcmp_une_f32(
+; CHECK-NEXT: fcmp
+; CHECK-NEXT: call i64 @llvm.amdgcn.icmp.i1(i1 %cmp, i1 false, i32 33)
+define i64 @fold_icmp_i1_ne_0_fcmp_une_f32(float %a, float %b) {
+  %cmp = fcmp une float %a, %b
+  %mask = call i64 @llvm.amdgcn.icmp.i1(i1 %cmp, i1 false, i32 33)
+  ret i64 %mask
+}
+
+; CHECK-LABEL: @fold_icmp_i1_ne_0_fcmp_olt_f64(
+; CHECK-NEXT: fcmp
+; CHECK-NEXT: call i64 @llvm.amdgcn.icmp.i1(i1 %cmp, i1 false, i32 33)
+define i64 @fold_icmp_i1_ne_0_fcmp_olt_f64(double %a, double %b) {
+  %cmp = fcmp olt double %a, %b
+  %mask = call i64 @llvm.amdgcn.icmp.i1(i1 %cmp, i1 false, i32 33)
+  ret i64 %mask
+}
+
+; CHECK-LABEL: @fold_icmp_i1_ne_0_icmp_eq_i4(
+; CHECK-NEXT: icmp
+; CHECK: call i64 @llvm.amdgcn.icmp.i1(i1 %cmp, i1 false, i32 33)
+define i64 @fold_icmp_i1_ne_0_icmp_eq_i4(i4 %a, i4 %b) {
+  %cmp = icmp eq i4 %a, %b
+  %mask = call i64 @llvm.amdgcn.icmp.i1(i1 %cmp, i1 false, i32 33)
+  ret i64 %mask
+}
+
+; CHECK-LABEL: @fold_icmp_i1_ne_0_icmp_eq_i8(
+; CHECK-NEXT: icmp
+; CHECK: call i64 @llvm.amdgcn.icmp.i1(i1 %cmp, i1 false, i32 33)
+define i64 @fold_icmp_i1_ne_0_icmp_eq_i8(i8 %a, i8 %b) {
+  %cmp = icmp eq i8 %a, %b
+  %mask = call i64 @llvm.amdgcn.icmp.i1(i1 %cmp, i1 false, i32 33)
+  ret i64 %mask
+}
+
+; CHECK-LABEL: @fold_icmp_i1_ne_0_icmp_eq_i16(
+; CHECK-NEXT: icmp
+; CHECK: call i64 @llvm.amdgcn.icmp.i1(i1 %cmp, i1 false, i32 33)
+define i64 @fold_icmp_i1_ne_0_icmp_eq_i16(i16 %a, i16 %b) {
+  %cmp = icmp eq i16 %a, %b
+  %mask = call i64 @llvm.amdgcn.icmp.i1(i1 %cmp, i1 false, i32 33)
+  ret i64 %mask
+}
+
+; CHECK-LABEL: @fold_icmp_i1_ne_0_icmp_eq_i36(
+; CHECK-NEXT: icmp
+; CHECK: call i64 @llvm.amdgcn.icmp.i1(i1 %cmp, i1 false, i32 33)
+define i64 @fold_icmp_i1_ne_0_icmp_eq_i36(i36 %a, i36 %b) {
+  %cmp = icmp eq i36 %a, %b
+  %mask = call i64 @llvm.amdgcn.icmp.i1(i1 %cmp, i1 false, i32 33)
+  ret i64 %mask
+}
+
+; CHECK-LABEL: @fold_icmp_i1_ne_0_icmp_eq_i128(
+; CHECK-NEXT: icmp
+; CHECK-NEXT: call i64 @llvm.amdgcn.icmp.i1(i1 %cmp, i1 false, i32 33)
+define i64 @fold_icmp_i1_ne_0_icmp_eq_i128(i128 %a, i128 %b) {
+  %cmp = icmp eq i128 %a, %b
+  %mask = call i64 @llvm.amdgcn.icmp.i1(i1 %cmp, i1 false, i32 33)
+  ret i64 %mask
+}
+
+; CHECK-LABEL: @fold_icmp_i1_ne_0_fcmp_oeq_f16(
+; CHECK-NEXT: fcmp
+; CHECK-NEXT: call i64 @llvm.amdgcn.icmp.i1(i1 %cmp, i1 false, i32 33)
+define i64 @fold_icmp_i1_ne_0_fcmp_oeq_f16(half %a, half %b) {
+  %cmp = fcmp oeq half %a, %b
+  %mask = call i64 @llvm.amdgcn.icmp.i1(i1 %cmp, i1 false, i32 33)
+  ret i64 %mask
+}
+
+; CHECK-LABEL: @fold_icmp_i1_ne_0_fcmp_oeq_f128(
+; CHECK-NEXT: fcmp
+; CHECK-NEXT: call i64 @llvm.amdgcn.icmp.i1(i1 %cmp, i1 false, i32 33)
+define i64 @fold_icmp_i1_ne_0_fcmp_oeq_f128(fp128 %a, fp128 %b) {
+;
+  %cmp = fcmp oeq fp128 %a, %b
+  %mask = call i64 @llvm.amdgcn.icmp.i1(i1 %cmp, i1 false, i32 33)
+  ret i64 %mask
+}
+
+; CHECK-LABEL: @fold_icmp_i1_ne_0_icmp_slt_i4(
+; CHECK-NEXT: icmp
+; CHECK-NEXT: call i64 @llvm.amdgcn.icmp.i1(i1 %cmp, i1 false, i32 33)
+define i64 @fold_icmp_i1_ne_0_icmp_slt_i4(i4 %a, i4 %b) {
+  %cmp = icmp slt i4 %a, %b
+  %mask = call i64 @llvm.amdgcn.icmp.i1(i1 %cmp, i1 false, i32 33)
+  ret i64 %mask
+}
+
+; CHECK-LABEL: @fold_icmp_i1_ne_0_icmp_slt_i8(
+; CHECK-NEXT: icmp
+; CHECK-NEXT: call i64 @llvm.amdgcn.icmp.i1(i1 %cmp, i1 false, i32 33)
+define i64 @fold_icmp_i1_ne_0_icmp_slt_i8(i8 %a, i8 %b) {
+  %cmp = icmp slt i8 %a, %b
+  %mask = call i64 @llvm.amdgcn.icmp.i1(i1 %cmp, i1 false, i32 33)
+  ret i64 %mask
+}
+
+; CHECK-LABEL: @fold_icmp_i1_ne_0_icmp_slt_i16(
+; CHECK-NEXT: icmp
+; CHECK-NEXT: call i64 @llvm.amdgcn.icmp.i1(i1 %cmp, i1 false, i32 33)
+define i64 @fold_icmp_i1_ne_0_icmp_slt_i16(i16 %a, i16 %b) {
+  %cmp = icmp slt i16 %a, %b
+  %mask = call i64 @llvm.amdgcn.icmp.i1(i1 %cmp, i1 false, i32 33)
+  ret i64 %mask
+}
+
+; CHECK-LABEL: @fold_icmp_i1_ne_0_icmp_ult_i4(
+; CHECK-NEXT: icmp
+; CHECK-NEXT: call i64 @llvm.amdgcn.icmp.i1(i1 %cmp, i1 false, i32 33)
+define i64 @fold_icmp_i1_ne_0_icmp_ult_i4(i4 %a, i4 %b) {
+  %cmp = icmp ult i4 %a, %b
+  %mask = call i64 @llvm.amdgcn.icmp.i1(i1 %cmp, i1 false, i32 33)
+  ret i64 %mask
+}
+
+; CHECK-LABEL: @fold_icmp_i1_ne_0_icmp_ult_i8(
+; CHECK-NEXT: icmp
+; CHECK-NEXT: call i64 @llvm.amdgcn.icmp.i1(i1 %cmp, i1 false, i32 33)
+define i64 @fold_icmp_i1_ne_0_icmp_ult_i8(i8 %a, i8 %b) {
+  %cmp = icmp ult i8 %a, %b
+  %mask = call i64 @llvm.amdgcn.icmp.i1(i1 %cmp, i1 false, i32 33)
+  ret i64 %mask
+}
+
+; CHECK-LABEL: @fold_icmp_i1_ne_0_icmp_ult_i16(
+; CHECK-NEXT: icmp
+; CHECK-NEXT: call i64 @llvm.amdgcn.icmp.i1(i1 %cmp, i1 false, i32 33)
+define i64 @fold_icmp_i1_ne_0_icmp_ult_i16(i16 %a, i16 %b) {
+  %cmp = icmp ult i16 %a, %b
+  %mask = call i64 @llvm.amdgcn.icmp.i1(i1 %cmp, i1 false, i32 33)
+  ret i64 %mask
+}
+
 ; --------------------------------------------------------------------
 ; llvm.amdgcn.fcmp
 ; --------------------------------------------------------------------
diff --git a/test/Transforms/InstCombine/X86/addcarry.ll b/test/Transforms/InstCombine/X86/addcarry.ll
new file mode 100644
index 0000000000000000000000000000000000000000..d762b4e96da5aa6022608ede55ff288037f10243
--- /dev/null
+++ b/test/Transforms/InstCombine/X86/addcarry.ll
@@ -0,0 +1,38 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+declare { i8, i32 } @llvm.x86.addcarry.32(i8, i32, i32)
+declare { i8, i64 } @llvm.x86.addcarry.64(i8, i64, i64)
+
+define i32 @no_carryin_i32(i32 %x, i32 %y, i8* %p) {
+; CHECK-LABEL: @no_carryin_i32(
+; CHECK-NEXT:    [[TMP1:%.*]] = call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 [[X:%.*]], i32 [[Y:%.*]])
+; CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { i32, i1 } [[TMP1]], 0
+; CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { i32, i1 } [[TMP1]], 1
+; CHECK-NEXT:    [[TMP4:%.*]] = zext i1 [[TMP3]] to i8
+; CHECK-NEXT:    store i8 [[TMP4]], i8* [[P:%.*]], align 1
+; CHECK-NEXT:    ret i32 [[TMP2]]
+;
+  %s = call { i8, i32 } @llvm.x86.addcarry.32(i8 0, i32 %x, i32 %y)
+  %ov = extractvalue { i8, i32 } %s, 0
+  store i8 %ov, i8* %p
+  %r = extractvalue { i8, i32 } %s, 1
+  ret i32 %r
+}
+
+define i64 @no_carryin_i64(i64 %x, i64 %y, i8* %p) {
+; CHECK-LABEL: @no_carryin_i64(
+; CHECK-NEXT:    [[TMP1:%.*]] = call { i64, i1 } @llvm.uadd.with.overflow.i64(i64 [[X:%.*]], i64 [[Y:%.*]])
+; CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { i64, i1 } [[TMP1]], 0
+; CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { i64, i1 } [[TMP1]], 1
+; CHECK-NEXT:    [[TMP4:%.*]] = zext i1 [[TMP3]] to i8
+; CHECK-NEXT:    store i8 [[TMP4]], i8* [[P:%.*]], align 1
+; CHECK-NEXT:    ret i64 [[TMP2]]
+;
+  %s = call { i8, i64 } @llvm.x86.addcarry.64(i8 0, i64 %x, i64 %y)
+  %ov = extractvalue { i8, i64 } %s, 0
+  store i8 %ov, i8* %p
+  %r = extractvalue { i8, i64 } %s, 1
+  ret i64 %r
+}
+
diff --git a/test/Transforms/InstCombine/X86/x86-adds-subs.ll b/test/Transforms/InstCombine/X86/x86-adds-subs.ll
deleted file mode 100644
index 54cccbe906802a9b1d771d4b1f4a3d2f129cf20d..0000000000000000000000000000000000000000
--- a/test/Transforms/InstCombine/X86/x86-adds-subs.ll
+++ /dev/null
@@ -1,351 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -instcombine -S | FileCheck %s
-
-define <16 x i8> @sse2_adds_b_constant() {
-; CHECK-LABEL: @sse2_adds_b_constant(
-; CHECK-NEXT: ret <16 x i8> <i8 2, i8 4, i8 6, i8 8, i8 10, i8 12, i8 14, i8 16, i8 18, i8 20, i8 22, i8 24, i8 26, i8 28, i8 30, i8 33>
-  %1 = call <16 x i8> @llvm.x86.sse2.padds.b(<16 x i8> <i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16>, <16 x i8> <i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 17>)
-  ret <16 x i8> %1
-}
-
-define <16 x i8> @sse2_adds_b_constant_underflow() {
-; CHECK-LABEL: @sse2_adds_b_constant_underflow(
-; CHECK-NEXT: ret <16 x i8> <i8 2, i8 4, i8 6, i8 8, i8 10, i8 12, i8 14, i8 -128, i8 18, i8 20, i8 22, i8 24, i8 26, i8 28, i8 30, i8 33>
-  %1 = call <16 x i8> @llvm.x86.sse2.padds.b(<16 x i8> <i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 -107, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16>, <16 x i8> <i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 -48, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 17>)
-  ret <16 x i8> %1
-}
-
-define <16 x i8> @sse2_adds_b_constant_overflow() {
-; CHECK-LABEL: @sse2_adds_b_constant_overflow(
-; CHECK-NEXT: ret <16 x i8> <i8 2, i8 4, i8 6, i8 8, i8 10, i8 12, i8 14, i8 16, i8 18, i8 20, i8 127, i8 24, i8 26, i8 28, i8 30, i8 127>
-  %1 = call <16 x i8> @llvm.x86.sse2.padds.b(<16 x i8> <i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 125, i8 12, i8 13, i8 14, i8 15, i8 16>, <16 x i8> <i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 120>)
-  ret <16 x i8> %1
-}
-
-define <16 x i8> @sse2_adds_b_constant_undefs() {
-; CHECK-LABEL: @sse2_adds_b_constant_undefs(
-; CHECK-NEXT: ret <16 x i8> <i8 undef, i8 4, i8 6, i8 8, i8 10, i8 undef, i8 14, i8 16, i8 18, i8 20, i8 22, i8 24, i8 26, i8 28, i8 30, i8 33>
-  %1 = call <16 x i8> @llvm.x86.sse2.padds.b(<16 x i8> <i8 undef, i8 2, i8 3, i8 4, i8 5, i8 undef, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16>, <16 x i8> <i8 undef, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 17>)
-  ret <16 x i8> %1
-}
-
-define <32 x i8> @avx2_adds_b_constant() {
-; CHECK-LABEL: @avx2_adds_b_constant(
-; CHECK-NEXT: ret <32 x i8> <i8 2, i8 4, i8 6, i8 8, i8 10, i8 12, i8 14, i8 16, i8 18, i8 20, i8 22, i8 24, i8 26, i8 28, i8 30, i8 33, i8 2, i8 4, i8 6, i8 8, i8 10, i8 12, i8 14, i8 16, i8 18, i8 20, i8 22, i8 24, i8 26, i8 28, i8 30, i8 33>
-  %1 = call <32 x i8> @llvm.x86.avx2.padds.b(<32 x i8> <i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16>, <32 x i8> <i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 17, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 17>)
-  ret <32 x i8> %1
-}
-
-define <32 x i8> @avx2_adds_b_constant_underflow() {
-; CHECK-LABEL: @avx2_adds_b_constant_underflow(
-; CHECK-NEXT: ret <32 x i8> <i8 2, i8 4, i8 6, i8 8, i8 10, i8 12, i8 14, i8 -128, i8 18, i8 20, i8 22, i8 24, i8 26, i8 28, i8 30, i8 33, i8 2, i8 4, i8 6, i8 8, i8 10, i8 12, i8 14, i8 -128, i8 18, i8 20, i8 22, i8 24, i8 26, i8 28, i8 30, i8 33>
-  %1 = call <32 x i8> @llvm.x86.avx2.padds.b(<32 x i8> <i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 -107, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 -107, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16>, <32 x i8> <i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 -48, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 17, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 -48, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 17>)
-  ret <32 x i8> %1
-}
-
-define <32 x i8> @avx2_adds_b_constant_overflow() {
-; CHECK-LABEL: @avx2_adds_b_constant_overflow(
-; CHECK-NEXT: ret <32 x i8> <i8 2, i8 4, i8 6, i8 8, i8 10, i8 12, i8 14, i8 16, i8 18, i8 20, i8 127, i8 24, i8 26, i8 28, i8 30, i8 127, i8 2, i8 4, i8 6, i8 8, i8 10, i8 12, i8 14, i8 16, i8 18, i8 20, i8 127, i8 24, i8 26, i8 28, i8 30, i8 127>
-  %1 = call <32 x i8> @llvm.x86.avx2.padds.b(<32 x i8> <i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 125, i8 12, i8 13, i8 14, i8 15, i8 16, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 125, i8 12, i8 13, i8 14, i8 15, i8 16>, <32 x i8> <i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 120, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 120>)
-  ret <32 x i8> %1
-}
-
-define <32 x i8> @avx2_adds_b_constant_undefs() {
-; CHECK-LABEL: @avx2_adds_b_constant_undefs(
-; CHECK-NEXT: ret <32 x i8> <i8 undef, i8 4, i8 6, i8 8, i8 10, i8 undef, i8 14, i8 16, i8 18, i8 20, i8 22, i8 24, i8 26, i8 28, i8 30, i8 33, i8 undef, i8 4, i8 6, i8 8, i8 10, i8 undef, i8 14, i8 16, i8 18, i8 20, i8 22, i8 24, i8 26, i8 28, i8 30, i8 33>
-  %1 = call <32 x i8> @llvm.x86.avx2.padds.b(<32 x i8> <i8 undef, i8 2, i8 3, i8 4, i8 5, i8 undef, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16, i8 undef, i8 2, i8 3, i8 4, i8 5, i8 undef, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16>, <32 x i8> <i8 undef, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 17, i8 undef, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 17>)
-  ret <32 x i8> %1
-}
-
-define <64 x i8> @avx512_mask_adds_b_constant() {
-; CHECK-LABEL: @avx512_mask_adds_b_constant(
-; CHECK-NEXT: ret <64 x i8> <i8 2, i8 0, i8 6, i8 8, i8 10, i8 12, i8 14, i8 16, i8 18, i8 20, i8 22, i8 24, i8 26, i8 28, i8 30, i8 33, i8 2, i8 4, i8 6, i8 8, i8 10, i8 12, i8 14, i8 16, i8 18, i8 20, i8 22, i8 24, i8 26, i8 28, i8 30, i8 33, i8 2, i8 4, i8 6, i8 8, i8 10, i8 12, i8 14, i8 16, i8 18, i8 20, i8 22, i8 24, i8 26, i8 28, i8 30, i8 33, i8 2, i8 4, i8 6, i8 8, i8 10, i8 12, i8 14, i8 16, i8 18, i8 20, i8 22, i8 24, i8 26, i8 28, i8 30, i8 33>
-  %1 = call <64 x i8> @llvm.x86.avx512.mask.padds.b.512(<64 x i8> <i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16>, <64 x i8> <i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 17, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 17, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 17, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 17>, <64 x i8> zeroinitializer, i64 -3)
-  ret <64 x i8> %1
-}
-
-define <64 x i8> @avx512_mask_adds_b_constant_underflow() {
-; CHECK-LABEL: @avx512_mask_adds_b_constant_underflow(
-; CHECK-NEXT: ret <64 x i8> <i8 2, i8 4, i8 6, i8 8, i8 10, i8 12, i8 14, i8 -128, i8 18, i8 20, i8 22, i8 24, i8 26, i8 28, i8 30, i8 33, i8 2, i8 4, i8 6, i8 8, i8 10, i8 12, i8 14, i8 -128, i8 18, i8 20, i8 22, i8 24, i8 26, i8 28, i8 30, i8 33, i8 2, i8 4, i8 6, i8 8, i8 10, i8 12, i8 14, i8 -128, i8 18, i8 20, i8 22, i8 24, i8 26, i8 28, i8 30, i8 33, i8 2, i8 4, i8 6, i8 8, i8 10, i8 12, i8 14, i8 -128, i8 18, i8 20, i8 22, i8 24, i8 26, i8 28, i8 30, i8 33>
-  %1 = call <64 x i8> @llvm.x86.avx512.mask.padds.b.512(<64 x i8> <i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 -107, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 -107, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 -107, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 -107, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16>, <64 x i8> <i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 -48, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 17, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 -48, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 17, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 -48, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 17, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 -48, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 17>, <64 x i8> zeroinitializer, i64 -1)
-  ret <64 x i8> %1
-}
-
-define <64 x i8> @avx512_mask_adds_b_constant_overflow() {
-; CHECK-LABEL: @avx512_mask_adds_b_constant_overflow(
-; CHECK-NEXT: ret <64 x i8> <i8 2, i8 4, i8 6, i8 8, i8 10, i8 12, i8 14, i8 16, i8 18, i8 20, i8 127, i8 24, i8 26, i8 28, i8 30, i8 127, i8 2, i8 4, i8 6, i8 8, i8 10, i8 12, i8 14, i8 16, i8 18, i8 20, i8 127, i8 24, i8 26, i8 28, i8 30, i8 127, i8 2, i8 4, i8 6, i8 8, i8 10, i8 12, i8 14, i8 16, i8 18, i8 20, i8 127, i8 24, i8 26, i8 28, i8 30, i8 127, i8 2, i8 4, i8 6, i8 8, i8 10, i8 12, i8 14, i8 16, i8 18, i8 20, i8 127, i8 24, i8 26, i8 28, i8 30, i8 127>
-  %1 = call <64 x i8> @llvm.x86.avx512.mask.padds.b.512(<64 x i8> <i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 125, i8 12, i8 13, i8 14, i8 15, i8 16, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 125, i8 12, i8 13, i8 14, i8 15, i8 16, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 125, i8 12, i8 13, i8 14, i8 15, i8 16, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 125, i8 12, i8 13, i8 14, i8 15, i8 16>, <64 x i8> <i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 120, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 120, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 120, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 120>, <64 x i8> zeroinitializer, i64 -1)
-  ret <64 x i8> %1
-}
-
-define <64 x i8> @avx512_mask_adds_b_constant_undefs() {
-; CHECK-LABEL: @avx512_mask_adds_b_constant_undefs(
-; CHECK-NEXT: ret <64 x i8> <i8 undef, i8 4, i8 6, i8 8, i8 10, i8 undef, i8 14, i8 16, i8 18, i8 20, i8 22, i8 24, i8 26, i8 28, i8 30, i8 33, i8 undef, i8 4, i8 6, i8 8, i8 10, i8 undef, i8 14, i8 16, i8 18, i8 20, i8 22, i8 24, i8 26, i8 28, i8 30, i8 33, i8 undef, i8 4, i8 6, i8 8, i8 10, i8 undef, i8 14, i8 16, i8 18, i8 20, i8 22, i8 24, i8 26, i8 28, i8 30, i8 33, i8 undef, i8 4, i8 6, i8 8, i8 10, i8 undef, i8 14, i8 16, i8 18, i8 20, i8 22, i8 24, i8 26, i8 28, i8 30, i8 33>
-  %1 = call <64 x i8> @llvm.x86.avx512.mask.padds.b.512(<64 x i8> <i8 undef, i8 2, i8 3, i8 4, i8 5, i8 undef, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16, i8 undef, i8 2, i8 3, i8 4, i8 5, i8 undef, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16, i8 undef, i8 2, i8 3, i8 4, i8 5, i8 undef, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16, i8 undef, i8 2, i8 3, i8 4, i8 5, i8 undef, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16>, <64 x i8> <i8 undef, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 17, i8 undef, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 17, i8 undef, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 17, i8 undef, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 17>, <64 x i8> zeroinitializer, i64 -1)
-  ret <64 x i8> %1
-}
-
-define <8 x i16> @sse2_adds_w_constant() {
-; CHECK-LABEL: @sse2_adds_w_constant(
-; CHECK-NEXT: ret <8 x i16> <i16 2, i16 4, i16 6, i16 8, i16 10, i16 12, i16 14, i16 16>
-  %1 = call <8 x i16> @llvm.x86.sse2.padds.w(<8 x i16> <i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8>, <8 x i16> <i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8>)
-  ret <8 x i16> %1
-}
-
-define <8 x i16> @sse2_adds_w_constant_underflow() {
-; CHECK-LABEL: @sse2_adds_w_constant_underflow(
-; CHECK-NEXT: ret <8 x i16> <i16 2, i16 4, i16 6, i16 8, i16 10, i16 12, i16 14, i16 -32768>
-  %1 = call <8 x i16> @llvm.x86.sse2.padds.w(<8 x i16> <i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 -32107>, <8 x i16> <i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 -12188>)
-  ret <8 x i16> %1
-}
-
-define <8 x i16> @sse2_adds_w_constant_overflow() {
-; CHECK-LABEL: @sse2_adds_w_constant_overflow(
-; CHECK-NEXT: ret <8 x i16> <i16 2, i16 32767, i16 6, i16 8, i16 10, i16 12, i16 14, i16 16>
-  %1 = call <8 x i16> @llvm.x86.sse2.padds.w(<8 x i16> <i16 1, i16 8248, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8>, <8 x i16> <i16 1, i16 25192, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8>)
-  ret <8 x i16> %1
-}
-
-define <8 x i16> @sse2_adds_w_constant_undefs() {
-; CHECK-LABEL: @sse2_adds_w_constant_undefs(
-; CHECK-NEXT: ret <8 x i16> <i16 undef, i16 4, i16 6, i16 8, i16 10, i16 undef, i16 14, i16 16>
-  %1 = call <8 x i16> @llvm.x86.sse2.padds.w(<8 x i16> <i16 undef, i16 2, i16 3, i16 4, i16 5, i16 undef, i16 7, i16 8>, <8 x i16> <i16 undef, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8>)
-  ret <8 x i16> %1
-}
-
-define <16 x i16> @avx2_adds_w_constant() {
-; CHECK-LABEL: @avx2_adds_w_constant(
-; CHECK-NEXT: ret <16 x i16> <i16 2, i16 4, i16 6, i16 8, i16 10, i16 12, i16 14, i16 16, i16 18, i16 20, i16 22, i16 24, i16 26, i16 28, i16 30, i16 33>
-  %1 = call <16 x i16> @llvm.x86.avx2.padds.w(<16 x i16> <i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15, i16 16>, <16 x i16> <i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15, i16 17>)
-  ret <16 x i16> %1
-}
-
-define <16 x i16> @avx2_adds_w_constant_underflow() {
-; CHECK-LABEL: @avx2_adds_w_constant_underflow(
-; CHECK-NEXT: ret <16 x i16> <i16 2, i16 4, i16 6, i16 8, i16 10, i16 12, i16 14, i16 -32768, i16 18, i16 20, i16 22, i16 24, i16 26, i16 28, i16 30, i16 33>
-  %1 = call <16 x i16> @llvm.x86.avx2.padds.w(<16 x i16> <i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 -21107, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15, i16 16>, <16 x i16> <i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 -15188, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15, i16 17>)
-  ret <16 x i16> %1
-}
-
-define <16 x i16> @avx2_adds_w_constant_overflow() {
-; CHECK-LABEL: @avx2_adds_w_constant_overflow(
-; CHECK-NEXT: ret <16 x i16> <i16 2, i16 4, i16 6, i16 8, i16 10, i16 12, i16 14, i16 16, i16 18, i16 20, i16 32767, i16 24, i16 26, i16 28, i16 30, i16 32767>
-  %1 = call <16 x i16> @llvm.x86.avx2.padds.w(<16 x i16> <i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 20125, i16 12, i16 13, i16 14, i16 15, i16 20160>, <16 x i16> <i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 20230, i16 12, i16 13, i16 14, i16 15, i16 20120>)
-  ret <16 x i16> %1
-}
-
-define <16 x i16> @avx2_adds_w_constant_undefs() {
-; CHECK-LABEL: @avx2_adds_w_constant_undefs(
-; CHECK-NEXT: ret <16 x i16> <i16 undef, i16 4, i16 6, i16 8, i16 10, i16 undef, i16 14, i16 16, i16 18, i16 20, i16 22, i16 24, i16 26, i16 28, i16 30, i16 33>
-  %1 = call <16 x i16> @llvm.x86.avx2.padds.w(<16 x i16> <i16 undef, i16 2, i16 3, i16 4, i16 5, i16 undef, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15, i16 16>, <16 x i16> <i16 undef, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15, i16 17>)
-  ret <16 x i16> %1
-}
-
-define <32 x i16> @avx512_mask_adds_w_constant() {
-; CHECK-LABEL: @avx512_mask_adds_w_constant(
-; CHECK-NEXT: ret <32 x i16> <i16 2, i16 0, i16 6, i16 8, i16 10, i16 12, i16 14, i16 16, i16 18, i16 20, i16 22, i16 24, i16 26, i16 28, i16 30, i16 33, i16 2, i16 4, i16 6, i16 8, i16 10, i16 12, i16 14, i16 16, i16 18, i16 20, i16 22, i16 24, i16 26, i16 28, i16 30, i16 33>
-  %1 = call <32 x i16> @llvm.x86.avx512.mask.padds.w.512(<32 x i16> <i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15, i16 16, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15, i16 16>, <32 x i16> <i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15, i16 17, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15, i16 17>, <32 x i16> zeroinitializer, i32 -3)
-  ret <32 x i16> %1
-}
-
-define <32 x i16> @avx512_mask_adds_w_constant_underflow() {
-; CHECK-LABEL: @avx512_mask_adds_w_constant_underflow(
-; CHECK-NEXT: ret <32 x i16> <i16 2, i16 4, i16 6, i16 8, i16 10, i16 12, i16 14, i16 -32768, i16 18, i16 20, i16 22, i16 24, i16 26, i16 28, i16 30, i16 33, i16 2, i16 4, i16 6, i16 8, i16 10, i16 12, i16 14, i16 -32768, i16 18, i16 20, i16 22, i16 24, i16 26, i16 28, i16 30, i16 33>
-  %1 = call <32 x i16> @llvm.x86.avx512.mask.padds.w.512(<32 x i16> <i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 -20107, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15, i16 16, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 -20107, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15, i16 16>, <32 x i16> <i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 -20168, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15, i16 17, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 -20248, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15, i16 17>, <32 x i16> zeroinitializer, i32 -1)
-  ret <32 x i16> %1
-}
-
-define <32 x i16> @avx512_mask_adds_w_constant_overflow() {
-; CHECK-LABEL: @avx512_mask_adds_w_constant_overflow(
-; CHECK-NEXT: ret <32 x i16> <i16 2, i16 4, i16 6, i16 8, i16 10, i16 12, i16 14, i16 16, i16 18, i16 20, i16 32767, i16 24, i16 26, i16 28, i16 30, i16 32767, i16 2, i16 4, i16 6, i16 8, i16 10, i16 12, i16 14, i16 16, i16 18, i16 20, i16 32767, i16 24, i16 26, i16 28, i16 30, i16 32767>
-  %1 = call <32 x i16> @llvm.x86.avx512.mask.padds.w.512(<32 x i16> <i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 20125, i16 12, i16 13, i16 14, i16 15, i16 20200, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 20125, i16 12, i16 13, i16 14, i16 15, i16 20200>, <32 x i16> <i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 20200, i16 12, i16 13, i16 14, i16 15, i16 20120, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 20211, i16 12, i16 13, i16 14, i16 15, i16 20120>, <32 x i16> zeroinitializer, i32 -1)
-  ret <32 x i16> %1
-}
-
-define <32 x i16> @avx512_mask_adds_w_constant_undefs() {
-; CHECK-LABEL: @avx512_mask_adds_w_constant_undefs(
-; CHECK-NEXT: ret <32 x i16> <i16 undef, i16 4, i16 6, i16 8, i16 10, i16 undef, i16 14, i16 16, i16 18, i16 20, i16 22, i16 24, i16 26, i16 28, i16 30, i16 33, i16 undef, i16 4, i16 6, i16 8, i16 10, i16 undef, i16 14, i16 16, i16 18, i16 20, i16 22, i16 24, i16 26, i16 28, i16 30, i16 33>
-  %1 = call <32 x i16> @llvm.x86.avx512.mask.padds.w.512(<32 x i16> <i16 undef, i16 2, i16 3, i16 4, i16 5, i16 undef, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15, i16 16, i16 undef, i16 2, i16 3, i16 4, i16 5, i16 undef, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15, i16 16>, <32 x i16> <i16 undef, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15, i16 17, i16 undef, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15, i16 17>, <32 x i16> zeroinitializer, i32 -1)
-  ret <32 x i16> %1
-}
-
-define <16 x i8> @sse2_subs_b_constant() {
-; CHECK-LABEL: @sse2_subs_b_constant(
-; CHECK-NEXT: ret <16 x i8> <i8 -2, i8 -4, i8 -6, i8 -8, i8 -10, i8 -12, i8 -14, i8 -16, i8 -18, i8 -20, i8 -22, i8 -24, i8 -26, i8 -28, i8 -30, i8 -33>
-  %1 = call <16 x i8> @llvm.x86.sse2.psubs.b(<16 x i8> <i8 -1, i8 -2, i8 -3, i8 -4, i8 -5, i8 -6, i8 -7, i8 -8, i8 -9, i8 -10, i8 -11, i8 -12, i8 -13, i8 -14, i8 -15, i8 -16>, <16 x i8> <i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 17>)
-  ret <16 x i8> %1
-}
-
-define <16 x i8> @sse2_subs_b_constant_underflow() {
-; CHECK-LABEL: @sse2_subs_b_constant_underflow(
-; CHECK-NEXT: ret <16 x i8> <i8 -2, i8 -4, i8 -6, i8 -8, i8 -10, i8 -12, i8 -14, i8 -128, i8 -18, i8 -20, i8 -22, i8 -24, i8 -26, i8 -28, i8 -30, i8 -33>
-  %1 = call <16 x i8> @llvm.x86.sse2.psubs.b(<16 x i8> <i8 -1, i8 -2, i8 -3, i8 -4, i8 -5, i8 -6, i8 -7, i8 -107, i8 -9, i8 -10, i8 -11, i8 -12, i8 -13, i8 -14, i8 -15, i8 -16>, <16 x i8> <i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 48, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 17>)
-  ret <16 x i8> %1
-}
-
-define <16 x i8> @sse2_subs_b_constant_overflow() {
-; CHECK-LABEL: @sse2_subs_b_constant_overflow(
-; CHECK-NEXT: ret <16 x i8> <i8 -2, i8 -4, i8 -6, i8 -8, i8 -10, i8 -12, i8 -14, i8 -16, i8 -18, i8 -20, i8 127, i8 -24, i8 -26, i8 -28, i8 -30, i8 127>
-  %1 = call <16 x i8> @llvm.x86.sse2.psubs.b(<16 x i8> <i8 -1, i8 -2, i8 -3, i8 -4, i8 -5, i8 -6, i8 -7, i8 -8, i8 -9, i8 -10, i8 125, i8 -12, i8 -13, i8 -14, i8 -15, i8 16>, <16 x i8> <i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 -11, i8 12, i8 13, i8 14, i8 15, i8 -120>)
-  ret <16 x i8> %1
-}
-
-define <16 x i8> @sse2_subs_b_constant_undefs() {
-; CHECK-LABEL: @sse2_subs_b_constant_undefs(
-; CHECK-NEXT: ret <16 x i8> <i8 undef, i8 -4, i8 -6, i8 -8, i8 -10, i8 undef, i8 -14, i8 -16, i8 -18, i8 -20, i8 -22, i8 -24, i8 -26, i8 -28, i8 -30, i8 -33>
-  %1 = call <16 x i8> @llvm.x86.sse2.psubs.b(<16 x i8> <i8 undef, i8 -2, i8 -3, i8 -4, i8 -5, i8 undef, i8 -7, i8 -8, i8 -9, i8 -10, i8 -11, i8 -12, i8 -13, i8 -14, i8 -15, i8 -16>, <16 x i8> <i8 undef, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 17>)
-  ret <16 x i8> %1
-}
-
-define <32 x i8> @avx2_subs_b_constant() {
-; CHECK-LABEL: @avx2_subs_b_constant(
-; CHECK-NEXT: ret <32 x i8> <i8 -2, i8 -4, i8 -6, i8 -8, i8 -10, i8 -12, i8 -14, i8 -16, i8 -18, i8 -20, i8 -22, i8 -24, i8 -26, i8 -28, i8 -30, i8 -33, i8 -2, i8 -4, i8 -6, i8 -8, i8 -10, i8 -12, i8 -14, i8 -16, i8 -18, i8 -20, i8 -22, i8 -24, i8 -26, i8 -28, i8 -30, i8 -33>
-  %1 = call <32 x i8> @llvm.x86.avx2.psubs.b(<32 x i8> <i8 -1, i8 -2, i8 -3, i8 -4, i8 -5, i8 -6, i8 -7, i8 -8, i8 -9, i8 -10, i8 -11, i8 -12, i8 -13, i8 -14, i8 -15, i8 -16, i8 -1, i8 -2, i8 -3, i8 -4, i8 -5, i8 -6, i8 -7, i8 -8, i8 -9, i8 -10, i8 -11, i8 -12, i8 -13, i8 -14, i8 -15, i8 -16>, <32 x i8> <i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 17, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 17>)
-  ret <32 x i8> %1
-}
-
-define <32 x i8> @avx2_subs_b_constant_underflow() {
-; CHECK-LABEL: @avx2_subs_b_constant_underflow(
-; CHECK-NEXT: ret <32 x i8> <i8 -2, i8 -4, i8 -6, i8 -8, i8 -10, i8 -12, i8 -14, i8 -128, i8 -18, i8 -20, i8 -22, i8 -24, i8 -26, i8 -28, i8 -30, i8 -33, i8 -2, i8 -4, i8 -6, i8 -8, i8 -10, i8 -12, i8 -14, i8 -128, i8 -18, i8 -20, i8 -22, i8 -24, i8 -26, i8 -28, i8 -30, i8 -33>
-  %1 = call <32 x i8> @llvm.x86.avx2.psubs.b(<32 x i8> <i8 -1, i8 -2, i8 -3, i8 -4, i8 -5, i8 -6, i8 -7, i8 -107, i8 -9, i8 -10, i8 -11, i8 -12, i8 -13, i8 -14, i8 -15, i8 -16, i8 -1, i8 -2, i8 -3, i8 -4, i8 -5, i8 -6, i8 -7, i8 -107, i8 -9, i8 -10, i8 -11, i8 -12, i8 -13, i8 -14, i8 -15, i8 -16>, <32 x i8> <i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 48, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 17, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 48, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 17>)
-  ret <32 x i8> %1
-}
-
-define <32 x i8> @avx2_subs_b_constant_overflow() {
-; CHECK-LABEL: @avx2_subs_b_constant_overflow(
-; CHECK-NEXT: ret <32 x i8> <i8 -2, i8 -4, i8 -6, i8 -8, i8 -10, i8 -12, i8 -14, i8 -16, i8 -18, i8 -20, i8 127, i8 -24, i8 -26, i8 -28, i8 -30, i8 127, i8 -2, i8 -4, i8 -6, i8 -8, i8 -10, i8 -12, i8 -14, i8 -16, i8 -18, i8 -20, i8 127, i8 -24, i8 -26, i8 -28, i8 -30, i8 127>
-  %1 = call <32 x i8> @llvm.x86.avx2.psubs.b(<32 x i8> <i8 -1, i8 -2, i8 -3, i8 -4, i8 -5, i8 -6, i8 -7, i8 -8, i8 -9, i8 -10, i8 125, i8 -12, i8 -13, i8 -14, i8 -15, i8 16, i8 -1, i8 -2, i8 -3, i8 -4, i8 -5, i8 -6, i8 -7, i8 -8, i8 -9, i8 -10, i8 125, i8 -12, i8 -13, i8 -14, i8 -15, i8 16>, <32 x i8> <i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 -11, i8 12, i8 13, i8 14, i8 15, i8 -120, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 -11, i8 12, i8 13, i8 14, i8 15, i8 -120>)
-  ret <32 x i8> %1
-}
-
-define <32 x i8> @avx2_subs_b_constant_undefs() {
-; CHECK-LABEL: @avx2_subs_b_constant_undefs(
-; CHECK-NEXT: ret <32 x i8> <i8 undef, i8 -4, i8 -6, i8 -8, i8 -10, i8 undef, i8 -14, i8 -16, i8 -18, i8 -20, i8 -22, i8 -24, i8 -26, i8 -28, i8 -30, i8 -33, i8 undef, i8 -4, i8 -6, i8 -8, i8 -10, i8 undef, i8 -14, i8 -16, i8 -18, i8 -20, i8 -22, i8 -24, i8 -26, i8 -28, i8 -30, i8 -33>
-  %1 = call <32 x i8> @llvm.x86.avx2.psubs.b(<32 x i8> <i8 undef, i8 -2, i8 -3, i8 -4, i8 -5, i8 undef, i8 -7, i8 -8, i8 -9, i8 -10, i8 -11, i8 -12, i8 -13, i8 -14, i8 -15, i8 -16, i8 undef, i8 -2, i8 -3, i8 -4, i8 -5, i8 undef, i8 -7, i8 -8, i8 -9, i8 -10, i8 -11, i8 -12, i8 -13, i8 -14, i8 -15, i8 -16>, <32 x i8> <i8 undef, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 17, i8 undef, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 17>)
-  ret <32 x i8> %1
-}
-
-define <64 x i8> @avx512_mask_subs_b_constant() {
-; CHECK-LABEL: @avx512_mask_subs_b_constant(
-; CHECK-NEXT: ret <64 x i8> <i8 -2, i8 0, i8 -6, i8 -8, i8 -10, i8 -12, i8 -14, i8 -16, i8 -18, i8 -20, i8 -22, i8 -24, i8 -26, i8 -28, i8 -30, i8 -33, i8 -2, i8 -4, i8 -6, i8 -8, i8 -10, i8 -12, i8 -14, i8 -16, i8 -18, i8 -20, i8 -22, i8 -24, i8 -26, i8 -28, i8 -30, i8 -33, i8 -2, i8 -4, i8 -6, i8 -8, i8 -10, i8 -12, i8 -14, i8 -16, i8 -18, i8 -20, i8 -22, i8 -24, i8 -26, i8 -28, i8 -30, i8 -33, i8 -2, i8 -4, i8 -6, i8 -8, i8 -10, i8 -12, i8 -14, i8 -16, i8 -18, i8 -20, i8 -22, i8 -24, i8 -26, i8 -28, i8 -30, i8 -33>
-  %1 = call <64 x i8> @llvm.x86.avx512.mask.psubs.b.512(<64 x i8> <i8 -1, i8 -2, i8 -3, i8 -4, i8 -5, i8 -6, i8 -7, i8 -8, i8 -9, i8 -10, i8 -11, i8 -12, i8 -13, i8 -14, i8 -15, i8 -16, i8 -1, i8 -2, i8 -3, i8 -4, i8 -5, i8 -6, i8 -7, i8 -8, i8 -9, i8 -10, i8 -11, i8 -12, i8 -13, i8 -14, i8 -15, i8 -16, i8 -1, i8 -2, i8 -3, i8 -4, i8 -5, i8 -6, i8 -7, i8 -8, i8 -9, i8 -10, i8 -11, i8 -12, i8 -13, i8 -14, i8 -15, i8 -16, i8 -1, i8 -2, i8 -3, i8 -4, i8 -5, i8 -6, i8 -7, i8 -8, i8 -9, i8 -10, i8 -11, i8 -12, i8 -13, i8 -14, i8 -15, i8 -16>, <64 x i8> <i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 17, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 17, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 17, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 17>, <64 x i8> zeroinitializer, i64 -3)
-  ret <64 x i8> %1
-}
-
-define <64 x i8> @avx512_mask_subs_b_constant_underflow() {
-; CHECK-LABEL: @avx512_mask_subs_b_constant_underflow(
-; CHECK-NEXT: ret <64 x i8> <i8 -2, i8 -4, i8 -6, i8 -8, i8 -10, i8 -12, i8 -14, i8 -128, i8 -18, i8 -20, i8 -22, i8 -24, i8 -26, i8 -28, i8 -30, i8 -33, i8 -2, i8 -4, i8 -6, i8 -8, i8 -10, i8 -12, i8 -14, i8 -128, i8 -18, i8 -20, i8 -22, i8 -24, i8 -26, i8 -28, i8 -30, i8 -33, i8 -2, i8 -4, i8 -6, i8 -8, i8 -10, i8 -12, i8 -14, i8 -128, i8 -18, i8 -20, i8 -22, i8 -24, i8 -26, i8 -28, i8 -30, i8 -33, i8 -2, i8 -4, i8 -6, i8 -8, i8 -10, i8 -12, i8 -14, i8 -128, i8 -18, i8 -20, i8 -22, i8 -24, i8 -26, i8 -28, i8 -30, i8 -33>
-  %1 = call <64 x i8> @llvm.x86.avx512.mask.psubs.b.512(<64 x i8> <i8 -1, i8 -2, i8 -3, i8 -4, i8 -5, i8 -6, i8 -7, i8 -107, i8 -9, i8 -10, i8 -11, i8 -12, i8 -13, i8 -14, i8 -15, i8 -16, i8 -1, i8 -2, i8 -3, i8 -4, i8 -5, i8 -6, i8 -7, i8 -107, i8 -9, i8 -10, i8 -11, i8 -12, i8 -13, i8 -14, i8 -15, i8 -16, i8 -1, i8 -2, i8 -3, i8 -4, i8 -5, i8 -6, i8 -7, i8 -107, i8 -9, i8 -10, i8 -11, i8 -12, i8 -13, i8 -14, i8 -15, i8 -16, i8 -1, i8 -2, i8 -3, i8 -4, i8 -5, i8 -6, i8 -7, i8 -107, i8 -9, i8 -10, i8 -11, i8 -12, i8 -13, i8 -14, i8 -15, i8 -16>, <64 x i8> <i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 48, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 17, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 48, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 17, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 48, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 17, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 48, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 17>, <64 x i8> zeroinitializer, i64 -1)
-  ret <64 x i8> %1
-}
-
-define <64 x i8> @avx512_mask_subs_b_constant_overflow() {
-; CHECK-LABEL: @avx512_mask_subs_b_constant_overflow(
-; CHECK-NEXT: ret <64 x i8> <i8 -2, i8 -4, i8 -6, i8 -8, i8 -10, i8 -12, i8 -14, i8 -16, i8 -18, i8 -20, i8 127, i8 -24, i8 -26, i8 -28, i8 -30, i8 127, i8 -2, i8 -4, i8 -6, i8 -8, i8 -10, i8 -12, i8 -14, i8 -16, i8 -18, i8 -20, i8 127, i8 -24, i8 -26, i8 -28, i8 -30, i8 127, i8 -2, i8 -4, i8 -6, i8 -8, i8 -10, i8 -12, i8 -14, i8 -16, i8 -18, i8 -20, i8 127, i8 -24, i8 -26, i8 -28, i8 -30, i8 127, i8 -2, i8 -4, i8 -6, i8 -8, i8 -10, i8 -12, i8 -14, i8 -16, i8 -18, i8 -20, i8 127, i8 -24, i8 -26, i8 -28, i8 -30, i8 127>
-  %1 = call <64 x i8> @llvm.x86.avx512.mask.psubs.b.512(<64 x i8> <i8 -1, i8 -2, i8 -3, i8 -4, i8 -5, i8 -6, i8 -7, i8 -8, i8 -9, i8 -10, i8 125, i8 -12, i8 -13, i8 -14, i8 -15, i8 16, i8 -1, i8 -2, i8 -3, i8 -4, i8 -5, i8 -6, i8 -7, i8 -8, i8 -9, i8 -10, i8 125, i8 -12, i8 -13, i8 -14, i8 -15, i8 16, i8 -1, i8 -2, i8 -3, i8 -4, i8 -5, i8 -6, i8 -7, i8 -8, i8 -9, i8 -10, i8 125, i8 -12, i8 -13, i8 -14, i8 -15, i8 16, i8 -1, i8 -2, i8 -3, i8 -4, i8 -5, i8 -6, i8 -7, i8 -8, i8 -9, i8 -10, i8 125, i8 -12, i8 -13, i8 -14, i8 -15, i8 16>, <64 x i8> <i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 -11, i8 12, i8 13, i8 14, i8 15, i8 -120, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 -11, i8 12, i8 13, i8 14, i8 15, i8 -120, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 -11, i8 12, i8 13, i8 14, i8 15, i8 -120, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 -11, i8 12, i8 13, i8 14, i8 15, i8 -120>, <64 x i8> zeroinitializer, i64 -1)
-  ret <64 x i8> %1
-}
-
-define <64 x i8> @avx512_mask_subs_b_constant_undefs() {
-; CHECK-LABEL: @avx512_mask_subs_b_constant_undefs(
-; CHECK-NEXT: ret <64 x i8> <i8 undef, i8 -4, i8 -6, i8 -8, i8 -10, i8 undef, i8 -14, i8 -16, i8 -18, i8 -20, i8 -22, i8 -24, i8 -26, i8 -28, i8 -30, i8 -33, i8 undef, i8 -4, i8 -6, i8 -8, i8 -10, i8 undef, i8 -14, i8 -16, i8 -18, i8 -20, i8 -22, i8 -24, i8 -26, i8 -28, i8 -30, i8 -33, i8 undef, i8 -4, i8 -6, i8 -8, i8 -10, i8 undef, i8 -14, i8 -16, i8 -18, i8 -20, i8 -22, i8 -24, i8 -26, i8 -28, i8 -30, i8 -33, i8 undef, i8 -4, i8 -6, i8 -8, i8 -10, i8 undef, i8 -14, i8 -16, i8 -18, i8 -20, i8 -22, i8 -24, i8 -26, i8 -28, i8 -30, i8 -33>
-  %1 = call <64 x i8> @llvm.x86.avx512.mask.psubs.b.512(<64 x i8> <i8 undef, i8 -2, i8 -3, i8 -4, i8 -5, i8 undef, i8 -7, i8 -8, i8 -9, i8 -10, i8 -11, i8 -12, i8 -13, i8 -14, i8 -15, i8 -16, i8 undef, i8 -2, i8 -3, i8 -4, i8 -5, i8 undef, i8 -7, i8 -8, i8 -9, i8 -10, i8 -11, i8 -12, i8 -13, i8 -14, i8 -15, i8 -16, i8 undef, i8 -2, i8 -3, i8 -4, i8 -5, i8 undef, i8 -7, i8 -8, i8 -9, i8 -10, i8 -11, i8 -12, i8 -13, i8 -14, i8 -15, i8 -16, i8 undef, i8 -2, i8 -3, i8 -4, i8 -5, i8 undef, i8 -7, i8 -8, i8 -9, i8 -10, i8 -11, i8 -12, i8 -13, i8 -14, i8 -15, i8 -16>, <64 x i8> <i8 undef, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 17, i8 undef, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 17, i8 undef, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 17, i8 undef, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 17>, <64 x i8> zeroinitializer, i64 -1)
-  ret <64 x i8> %1
-}
-
-define <8 x i16> @sse2_subs_w_constant() {
-; CHECK-LABEL: @sse2_subs_w_constant(
-; CHECK-NEXT: ret <8 x i16> <i16 -2, i16 -4, i16 -6, i16 -8, i16 -10, i16 -12, i16 -14, i16 -16>
-  %1 = call <8 x i16> @llvm.x86.sse2.psubs.w(<8 x i16> <i16 -1, i16 -2, i16 -3, i16 -4, i16 -5, i16 -6, i16 -7, i16 -8>, <8 x i16> <i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8>)
-  ret <8 x i16> %1
-}
-
-define <8 x i16> @sse2_subs_w_constant_underflow() {
-; CHECK-LABEL: @sse2_subs_w_constant_underflow(
-; CHECK-NEXT: ret <8 x i16> <i16 -2, i16 -4, i16 -6, i16 -8, i16 -10, i16 -12, i16 -14, i16 -32768>
-  %1 = call <8 x i16> @llvm.x86.sse2.psubs.w(<8 x i16> <i16 -1, i16 -2, i16 -3, i16 -4, i16 -5, i16 -6, i16 -7, i16 -32107>, <8 x i16> <i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 12188>)
-  ret <8 x i16> %1
-}
-
-define <8 x i16> @sse2_subs_w_constant_overflow() {
-; CHECK-LABEL: @sse2_subs_w_constant_overflow(
-; CHECK-NEXT: ret <8 x i16> <i16 -2, i16 32767, i16 -6, i16 -8, i16 -10, i16 -12, i16 -14, i16 -16>
-  %1 = call <8 x i16> @llvm.x86.sse2.psubs.w(<8 x i16> <i16 -1, i16 8248, i16 -3, i16 -4, i16 -5, i16 -6, i16 -7, i16 -8>, <8 x i16> <i16 1, i16 -25192, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8>)
-  ret <8 x i16> %1
-}
-
-define <8 x i16> @sse2_subs_w_constant_undefs() {
-; CHECK-LABEL: @sse2_subs_w_constant_undefs(
-; CHECK-NEXT: ret <8 x i16> <i16 undef, i16 -4, i16 -6, i16 -8, i16 -10, i16 undef, i16 -14, i16 -16>
-  %1 = call <8 x i16> @llvm.x86.sse2.psubs.w(<8 x i16> <i16 undef, i16 -2, i16 -3, i16 -4, i16 -5, i16 undef, i16 -7, i16 -8>, <8 x i16> <i16 undef, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8>)
-  ret <8 x i16> %1
-}
-
-define <16 x i16> @avx2_subs_w_constant() {
-; CHECK-LABEL: @avx2_subs_w_constant(
-; CHECK-NEXT: ret <16 x i16> <i16 -2, i16 -4, i16 -6, i16 -8, i16 -10, i16 -12, i16 -14, i16 -16, i16 -18, i16 -20, i16 -22, i16 -24, i16 -26, i16 -28, i16 -30, i16 -33>
-  %1 = call <16 x i16> @llvm.x86.avx2.psubs.w(<16 x i16> <i16 -1, i16 -2, i16 -3, i16 -4, i16 -5, i16 -6, i16 -7, i16 -8, i16 -9, i16 -10, i16 -11, i16 -12, i16 -13, i16 -14, i16 -15, i16 -16>, <16 x i16> <i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15, i16 17>)
-  ret <16 x i16> %1
-}
-
-define <16 x i16> @avx2_subs_w_constant_underflow() {
-; CHECK-LABEL: @avx2_subs_w_constant_underflow(
-; CHECK-NEXT: ret <16 x i16> <i16 -2, i16 -4, i16 -6, i16 -8, i16 -10, i16 -12, i16 -14, i16 -32768, i16 -18, i16 -20, i16 -22, i16 -24, i16 -26, i16 -28, i16 -30, i16 -33>
-  %1 = call <16 x i16> @llvm.x86.avx2.psubs.w(<16 x i16> <i16 -1, i16 -2, i16 -3, i16 -4, i16 -5, i16 -6, i16 -7, i16 -21107, i16 -9, i16 -10, i16 -11, i16 -12, i16 -13, i16 -14, i16 -15, i16 -16>, <16 x i16> <i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 15188, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15, i16 17>)
-  ret <16 x i16> %1
-}
-
-define <16 x i16> @avx2_subs_w_constant_overflow() {
-; CHECK-LABEL: @avx2_subs_w_constant_overflow(
-; CHECK-NEXT: ret <16 x i16> <i16 -2, i16 -4, i16 -6, i16 -8, i16 -10, i16 -12, i16 -14, i16 -16, i16 -18, i16 -20, i16 32767, i16 -24, i16 -26, i16 -28, i16 -30, i16 32767>
-  %1 = call <16 x i16> @llvm.x86.avx2.psubs.w(<16 x i16> <i16 -1, i16 -2, i16 -3, i16 -4, i16 -5, i16 -6, i16 -7, i16 -8, i16 -9, i16 -10, i16 20125, i16 -12, i16 -13, i16 -14, i16 -15, i16 20160>, <16 x i16> <i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 -20230, i16 12, i16 13, i16 14, i16 15, i16 -20120>)
-  ret <16 x i16> %1
-}
-
-define <16 x i16> @avx2_subs_w_constant_undefs() {
-; CHECK-LABEL: @avx2_subs_w_constant_undefs(
-; CHECK-NEXT: ret <16 x i16> <i16 undef, i16 -4, i16 -6, i16 -8, i16 -10, i16 undef, i16 -14, i16 -16, i16 -18, i16 -20, i16 -22, i16 -24, i16 -26, i16 -28, i16 -30, i16 -33>
-  %1 = call <16 x i16> @llvm.x86.avx2.psubs.w(<16 x i16> <i16 undef, i16 -2, i16 -3, i16 -4, i16 -5, i16 undef, i16 -7, i16 -8, i16 -9, i16 -10, i16 -11, i16 -12, i16 -13, i16 -14, i16 -15, i16 -16>, <16 x i16> <i16 undef, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15, i16 17>)
-  ret <16 x i16> %1
-}
-
-define <32 x i16> @avx512_mask_subs_w_constant() {
-; CHECK-LABEL: @avx512_mask_subs_w_constant(
-; CHECK-NEXT: ret <32 x i16> <i16 -2, i16 0, i16 -6, i16 -8, i16 -10, i16 -12, i16 -14, i16 -16, i16 -18, i16 -20, i16 -22, i16 -24, i16 -26, i16 -28, i16 -30, i16 -33, i16 -2, i16 -4, i16 -6, i16 -8, i16 -10, i16 -12, i16 -14, i16 -16, i16 -18, i16 -20, i16 -22, i16 -24, i16 -26, i16 -28, i16 -30, i16 -33>
-  %1 = call <32 x i16> @llvm.x86.avx512.mask.psubs.w.512(<32 x i16> <i16 -1, i16 -2, i16 -3, i16 -4, i16 -5, i16 -6, i16 -7, i16 -8, i16 -9, i16 -10, i16 -11, i16 -12, i16 -13, i16 -14, i16 -15, i16 -16, i16 -1, i16 -2, i16 -3, i16 -4, i16 -5, i16 -6, i16 -7, i16 -8, i16 -9, i16 -10, i16 -11, i16 -12, i16 -13, i16 -14, i16 -15, i16 -16>, <32 x i16> <i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15, i16 17, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15, i16 17>, <32 x i16> zeroinitializer, i32 -3)
-  ret <32 x i16> %1
-}
-
-define <32 x i16> @avx512_mask_subs_w_constant_underflow() {
-; CHECK-LABEL: @avx512_mask_subs_w_constant_underflow(
-; CHECK-NEXT: ret <32 x i16> <i16 -2, i16 -4, i16 -6, i16 -8, i16 -10, i16 -12, i16 -14, i16 -32768, i16 -18, i16 -20, i16 -22, i16 -24, i16 -26, i16 -28, i16 -30, i16 -33, i16 -2, i16 -4, i16 -6, i16 -8, i16 -10, i16 -12, i16 -14, i16 -32768, i16 -18, i16 -20, i16 -22, i16 -24, i16 -26, i16 -28, i16 -30, i16 -33>
-  %1 = call <32 x i16> @llvm.x86.avx512.mask.psubs.w.512(<32 x i16> <i16 -1, i16 -2, i16 -3, i16 -4, i16 -5, i16 -6, i16 -7, i16 -20107, i16 -9, i16 -10, i16 -11, i16 -12, i16 -13, i16 -14, i16 -15, i16 -16, i16 -1, i16 -2, i16 -3, i16 -4, i16 -5, i16 -6, i16 -7, i16 -20107, i16 -9, i16 -10, i16 -11, i16 -12, i16 -13, i16 -14, i16 -15, i16 -16>, <32 x i16> <i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 20168, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15, i16 17, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 20248, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15, i16 17>, <32 x i16> zeroinitializer, i32 -1)
-  ret <32 x i16> %1
-}
-
-define <32 x i16> @avx512_mask_subs_w_constant_overflow() {
-; CHECK-LABEL: @avx512_mask_subs_w_constant_overflow(
-; CHECK-NEXT: ret <32 x i16> <i16 -2, i16 -4, i16 -6, i16 -8, i16 -10, i16 -12, i16 -14, i16 -16, i16 -18, i16 -20, i16 32767, i16 -24, i16 -26, i16 -28, i16 -30, i16 32767, i16 -2, i16 -4, i16 -6, i16 -8, i16 -10, i16 -12, i16 -14, i16 -16, i16 -18, i16 -20, i16 32767, i16 -24, i16 -26, i16 -28, i16 -30, i16 32767>
-  %1 = call <32 x i16> @llvm.x86.avx512.mask.psubs.w.512(<32 x i16> <i16 -1, i16 -2, i16 -3, i16 -4, i16 -5, i16 -6, i16 -7, i16 -8, i16 -9, i16 -10, i16 20125, i16 -12, i16 -13, i16 -14, i16 -15, i16 20200, i16 -1, i16 -2, i16 -3, i16 -4, i16 -5, i16 -6, i16 -7, i16 -8, i16 -9, i16 -10, i16 20125, i16 -12, i16 -13, i16 -14, i16 -15, i16 20200>, <32 x i16> <i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 -20200, i16 12, i16 13, i16 14, i16 15, i16 -20120, i16 1, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 -20211, i16 12, i16 13, i16 14, i16 15, i16 -20120>, <32 x i16> zeroinitializer, i32 -1)
-  ret <32 x i16> %1
-}
-
-define <32 x i16> @avx512_mask_subs_w_constant_undefs() {
-; CHECK-LABEL: @avx512_mask_subs_w_constant_undefs(
-; CHECK-NEXT: ret <32 x i16> <i16 undef, i16 -4, i16 -6, i16 -8, i16 -10, i16 undef, i16 -14, i16 -16, i16 -18, i16 -20, i16 -22, i16 -24, i16 -26, i16 -28, i16 -30, i16 -33, i16 undef, i16 -4, i16 -6, i16 -8, i16 -10, i16 undef, i16 -14, i16 -16, i16 -18, i16 -20, i16 -22, i16 -24, i16 -26, i16 -28, i16 -30, i16 -33>
-  %1 = call <32 x i16> @llvm.x86.avx512.mask.psubs.w.512(<32 x i16> <i16 undef, i16 -2, i16 -3, i16 -4, i16 -5, i16 undef, i16 -7, i16 -8, i16 -9, i16 -10, i16 -11, i16 -12, i16 -13, i16 -14, i16 -15, i16 -16, i16 undef, i16 -2, i16 -3, i16 -4, i16 -5, i16 undef, i16 -7, i16 -8, i16 -9, i16 -10, i16 -11, i16 -12, i16 -13, i16 -14, i16 -15, i16 -16>, <32 x i16> <i16 undef, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15, i16 17, i16 undef, i16 2, i16 3, i16 4, i16 5, i16 6, i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14, i16 15, i16 17>, <32 x i16> zeroinitializer, i32 -1)
-  ret <32 x i16> %1
-}
-
-declare <16 x i8> @llvm.x86.sse2.padds.b(<16 x i8>, <16 x i8>) nounwind readnone
-declare <16 x i8> @llvm.x86.sse2.psubs.b(<16 x i8>, <16 x i8>) nounwind readnone
-declare <8 x i16> @llvm.x86.sse2.padds.w(<8 x i16>, <8 x i16>) nounwind readnone
-declare <8 x i16> @llvm.x86.sse2.psubs.w(<8 x i16>, <8 x i16>) nounwind readnone
-declare <32 x i8> @llvm.x86.avx2.padds.b(<32 x i8>, <32 x i8>) nounwind readnone
-declare <32 x i8> @llvm.x86.avx2.psubs.b(<32 x i8>, <32 x i8>) nounwind readnone
-declare <16 x i16> @llvm.x86.avx2.padds.w(<16 x i16>, <16 x i16>) nounwind readnone
-declare <16 x i16> @llvm.x86.avx2.psubs.w(<16 x i16>, <16 x i16>) nounwind readnone
-declare <64 x i8> @llvm.x86.avx512.mask.padds.b.512(<64 x i8>, <64 x i8>, <64 x i8>, i64) nounwind readnone
-declare <64 x i8> @llvm.x86.avx512.mask.psubs.b.512(<64 x i8>, <64 x i8>, <64 x i8>, i64) nounwind readnone
-declare <32 x i16> @llvm.x86.avx512.mask.padds.w.512(<32 x i16>, <32 x i16>, <32 x i16>, i32) nounwind readnone
-declare <32 x i16> @llvm.x86.avx512.mask.psubs.w.512(<32 x i16>, <32 x i16>, <32 x i16>, i32) nounwind readnone
diff --git a/test/Transforms/InstCombine/X86/x86-avx512.ll b/test/Transforms/InstCombine/X86/x86-avx512.ll
index ff78142493e1dfa2e5db15c429172c3515041471..9c5080c7792e16ef4a104bd3b2cb291a3faf84ce 100644
--- a/test/Transforms/InstCombine/X86/x86-avx512.ll
+++ b/test/Transforms/InstCombine/X86/x86-avx512.ll
@@ -7,7 +7,7 @@ declare <4 x float> @llvm.x86.avx512.mask.add.ss.round(<4 x float>, <4 x float>,
 define <4 x float> @test_add_ss(<4 x float> %a, <4 x float> %b) {
 ; CHECK-LABEL: @test_add_ss(
 ; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <4 x float> [[A:%.*]], i64 0
-; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <4 x float> [[B:%.*]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <4 x float> [[B:%.*]], i64 0
 ; CHECK-NEXT:    [[TMP3:%.*]] = fadd float [[TMP1]], [[TMP2]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <4 x float> [[A]], float [[TMP3]], i64 0
 ; CHECK-NEXT:    ret <4 x float> [[TMP4]]
@@ -38,7 +38,7 @@ define <4 x float> @test_add_ss_mask(<4 x float> %a, <4 x float> %b, <4 x float>
 ; CHECK-NEXT:    [[TMP3:%.*]] = fadd float [[TMP1]], [[TMP2]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
 ; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <8 x i1> [[TMP4]], i64 0
-; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x float> [[C:%.*]], i32 0
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x float> [[C:%.*]], i64 0
 ; CHECK-NEXT:    [[TMP7:%.*]] = select i1 [[TMP5]], float [[TMP3]], float [[TMP6]]
 ; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <4 x float> [[A]], float [[TMP7]], i64 0
 ; CHECK-NEXT:    ret <4 x float> [[TMP8]]
@@ -149,7 +149,7 @@ declare <4 x float> @llvm.x86.avx512.mask.sub.ss.round(<4 x float>, <4 x float>,
 define <4 x float> @test_sub_ss(<4 x float> %a, <4 x float> %b) {
 ; CHECK-LABEL: @test_sub_ss(
 ; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <4 x float> [[A:%.*]], i64 0
-; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <4 x float> [[B:%.*]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <4 x float> [[B:%.*]], i64 0
 ; CHECK-NEXT:    [[TMP3:%.*]] = fsub float [[TMP1]], [[TMP2]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <4 x float> [[A]], float [[TMP3]], i64 0
 ; CHECK-NEXT:    ret <4 x float> [[TMP4]]
@@ -180,7 +180,7 @@ define <4 x float> @test_sub_ss_mask(<4 x float> %a, <4 x float> %b, <4 x float>
 ; CHECK-NEXT:    [[TMP3:%.*]] = fsub float [[TMP1]], [[TMP2]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
 ; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <8 x i1> [[TMP4]], i64 0
-; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x float> [[C:%.*]], i32 0
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x float> [[C:%.*]], i64 0
 ; CHECK-NEXT:    [[TMP7:%.*]] = select i1 [[TMP5]], float [[TMP3]], float [[TMP6]]
 ; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <4 x float> [[A]], float [[TMP7]], i64 0
 ; CHECK-NEXT:    ret <4 x float> [[TMP8]]
@@ -291,7 +291,7 @@ declare <4 x float> @llvm.x86.avx512.mask.mul.ss.round(<4 x float>, <4 x float>,
 define <4 x float> @test_mul_ss(<4 x float> %a, <4 x float> %b) {
 ; CHECK-LABEL: @test_mul_ss(
 ; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <4 x float> [[A:%.*]], i64 0
-; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <4 x float> [[B:%.*]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <4 x float> [[B:%.*]], i64 0
 ; CHECK-NEXT:    [[TMP3:%.*]] = fmul float [[TMP1]], [[TMP2]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <4 x float> [[A]], float [[TMP3]], i64 0
 ; CHECK-NEXT:    ret <4 x float> [[TMP4]]
@@ -322,7 +322,7 @@ define <4 x float> @test_mul_ss_mask(<4 x float> %a, <4 x float> %b, <4 x float>
 ; CHECK-NEXT:    [[TMP3:%.*]] = fmul float [[TMP1]], [[TMP2]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
 ; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <8 x i1> [[TMP4]], i64 0
-; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x float> [[C:%.*]], i32 0
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x float> [[C:%.*]], i64 0
 ; CHECK-NEXT:    [[TMP7:%.*]] = select i1 [[TMP5]], float [[TMP3]], float [[TMP6]]
 ; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <4 x float> [[A]], float [[TMP7]], i64 0
 ; CHECK-NEXT:    ret <4 x float> [[TMP8]]
@@ -433,7 +433,7 @@ declare <4 x float> @llvm.x86.avx512.mask.div.ss.round(<4 x float>, <4 x float>,
 define <4 x float> @test_div_ss(<4 x float> %a, <4 x float> %b) {
 ; CHECK-LABEL: @test_div_ss(
 ; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <4 x float> [[A:%.*]], i64 0
-; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <4 x float> [[B:%.*]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <4 x float> [[B:%.*]], i64 0
 ; CHECK-NEXT:    [[TMP3:%.*]] = fdiv float [[TMP1]], [[TMP2]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <4 x float> [[A]], float [[TMP3]], i64 0
 ; CHECK-NEXT:    ret <4 x float> [[TMP4]]
@@ -464,7 +464,7 @@ define <4 x float> @test_div_ss_mask(<4 x float> %a, <4 x float> %b, <4 x float>
 ; CHECK-NEXT:    [[TMP3:%.*]] = fdiv float [[TMP1]], [[TMP2]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
 ; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <8 x i1> [[TMP4]], i64 0
-; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x float> [[C:%.*]], i32 0
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x float> [[C:%.*]], i64 0
 ; CHECK-NEXT:    [[TMP7:%.*]] = select i1 [[TMP5]], float [[TMP3]], float [[TMP6]]
 ; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <4 x float> [[A]], float [[TMP7]], i64 0
 ; CHECK-NEXT:    ret <4 x float> [[TMP8]]
@@ -1551,15 +1551,15 @@ define <4 x float> @test_mask3_vfmsub_ss(<4 x float> %a, <4 x float> %b, <4 x fl
 ; CHECK-LABEL: @test_mask3_vfmsub_ss(
 ; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <4 x float> [[A:%.*]], i32 0
 ; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <4 x float> [[B:%.*]], i32 0
-; CHECK-NEXT:    [[DOTRHS:%.*]] = extractelement <4 x float> [[C:%.*]], i64 0
-; CHECK-NEXT:    [[TMP3:%.*]] = fsub float -0.000000e+00, [[DOTRHS]]
-; CHECK-NEXT:    [[TMP4:%.*]] = call float @llvm.fma.f32(float [[TMP1]], float [[TMP2]], float [[TMP3]])
-; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <4 x float> [[C]], i64 0
-; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
-; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <8 x i1> [[TMP6]], i64 0
-; CHECK-NEXT:    [[TMP8:%.*]] = select i1 [[TMP7]], float [[TMP4]], float [[TMP5]]
-; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <4 x float> [[C]], float [[TMP8]], i64 0
-; CHECK-NEXT:    ret <4 x float> [[TMP9]]
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <4 x float> [[C:%.*]], i64 0
+; CHECK-NEXT:    [[TMP4:%.*]] = fsub float -0.000000e+00, [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = call float @llvm.fma.f32(float [[TMP1]], float [[TMP2]], float [[TMP4]])
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x float> [[C]], i64 0
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <8 x i1> [[TMP7]], i64 0
+; CHECK-NEXT:    [[TMP9:%.*]] = select i1 [[TMP8]], float [[TMP5]], float [[TMP6]]
+; CHECK-NEXT:    [[TMP10:%.*]] = insertelement <4 x float> [[C]], float [[TMP9]], i64 0
+; CHECK-NEXT:    ret <4 x float> [[TMP10]]
 ;
   %1 = insertelement <4 x float> %a, float 1.000000e+00, i32 1
   %2 = insertelement <4 x float> %1, float 2.000000e+00, i32 2
@@ -1584,14 +1584,14 @@ define float @test_mask3_vfmsub_ss_0(<4 x float> %a, <4 x float> %b, <4 x float>
 ; CHECK-LABEL: @test_mask3_vfmsub_ss_0(
 ; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <4 x float> [[A:%.*]], i64 0
 ; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <4 x float> [[B:%.*]], i64 0
-; CHECK-NEXT:    [[DOTRHS:%.*]] = extractelement <4 x float> [[C:%.*]], i32 0
-; CHECK-NEXT:    [[TMP3:%.*]] = fsub float -0.000000e+00, [[DOTRHS]]
-; CHECK-NEXT:    [[TMP4:%.*]] = call float @llvm.fma.f32(float [[TMP1]], float [[TMP2]], float [[TMP3]])
-; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <4 x float> [[C]], i32 0
-; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
-; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <8 x i1> [[TMP6]], i64 0
-; CHECK-NEXT:    [[TMP8:%.*]] = select i1 [[TMP7]], float [[TMP4]], float [[TMP5]]
-; CHECK-NEXT:    ret float [[TMP8]]
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <4 x float> [[C:%.*]], i32 0
+; CHECK-NEXT:    [[TMP4:%.*]] = fsub float -0.000000e+00, [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = call float @llvm.fma.f32(float [[TMP1]], float [[TMP2]], float [[TMP4]])
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x float> [[C]], i32 0
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <8 x i1> [[TMP7]], i64 0
+; CHECK-NEXT:    [[TMP9:%.*]] = select i1 [[TMP8]], float [[TMP5]], float [[TMP6]]
+; CHECK-NEXT:    ret float [[TMP9]]
 ;
   %1 = insertelement <4 x float> %c, float 1.000000e+00, i32 1
   %2 = insertelement <4 x float> %1, float 2.000000e+00, i32 2
@@ -1635,15 +1635,15 @@ define <2 x double> @test_mask3_vfmsub_sd(<2 x double> %a, <2 x double> %b, <2 x
 ; CHECK-LABEL: @test_mask3_vfmsub_sd(
 ; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <2 x double> [[A:%.*]], i64 0
 ; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <2 x double> [[B:%.*]], i64 0
-; CHECK-NEXT:    [[DOTRHS:%.*]] = extractelement <2 x double> [[C:%.*]], i64 0
-; CHECK-NEXT:    [[TMP3:%.*]] = fsub double -0.000000e+00, [[DOTRHS]]
-; CHECK-NEXT:    [[TMP4:%.*]] = call double @llvm.fma.f64(double [[TMP1]], double [[TMP2]], double [[TMP3]])
-; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <2 x double> [[C]], i64 0
-; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
-; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <8 x i1> [[TMP6]], i64 0
-; CHECK-NEXT:    [[TMP8:%.*]] = select i1 [[TMP7]], double [[TMP4]], double [[TMP5]]
-; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <2 x double> [[C]], double [[TMP8]], i64 0
-; CHECK-NEXT:    ret <2 x double> [[TMP9]]
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <2 x double> [[C:%.*]], i64 0
+; CHECK-NEXT:    [[TMP4:%.*]] = fsub double -0.000000e+00, [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = call double @llvm.fma.f64(double [[TMP1]], double [[TMP2]], double [[TMP4]])
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <2 x double> [[C]], i64 0
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <8 x i1> [[TMP7]], i64 0
+; CHECK-NEXT:    [[TMP9:%.*]] = select i1 [[TMP8]], double [[TMP5]], double [[TMP6]]
+; CHECK-NEXT:    [[TMP10:%.*]] = insertelement <2 x double> [[C]], double [[TMP9]], i64 0
+; CHECK-NEXT:    ret <2 x double> [[TMP10]]
 ;
   %1 = insertelement <2 x double> %a, double 1.000000e+00, i32 1
   %2 = insertelement <2 x double> %b, double 2.000000e+00, i32 1
@@ -1664,14 +1664,14 @@ define double @test_mask3_vfmsub_sd_0(<2 x double> %a, <2 x double> %b, <2 x dou
 ; CHECK-LABEL: @test_mask3_vfmsub_sd_0(
 ; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <2 x double> [[A:%.*]], i64 0
 ; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <2 x double> [[B:%.*]], i64 0
-; CHECK-NEXT:    [[DOTRHS:%.*]] = extractelement <2 x double> [[C:%.*]], i64 0
-; CHECK-NEXT:    [[TMP3:%.*]] = fsub double -0.000000e+00, [[DOTRHS]]
-; CHECK-NEXT:    [[TMP4:%.*]] = call double @llvm.fma.f64(double [[TMP1]], double [[TMP2]], double [[TMP3]])
-; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <2 x double> [[C]], i64 0
-; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
-; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <8 x i1> [[TMP6]], i64 0
-; CHECK-NEXT:    [[TMP8:%.*]] = select i1 [[TMP7]], double [[TMP4]], double [[TMP5]]
-; CHECK-NEXT:    ret double [[TMP8]]
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <2 x double> [[C:%.*]], i64 0
+; CHECK-NEXT:    [[TMP4:%.*]] = fsub double -0.000000e+00, [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = call double @llvm.fma.f64(double [[TMP1]], double [[TMP2]], double [[TMP4]])
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <2 x double> [[C]], i64 0
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <8 x i1> [[TMP7]], i64 0
+; CHECK-NEXT:    [[TMP9:%.*]] = select i1 [[TMP8]], double [[TMP5]], double [[TMP6]]
+; CHECK-NEXT:    ret double [[TMP9]]
 ;
   %1 = insertelement <2 x double> %c, double 1.000000e+00, i32 1
   %2 = fsub <2 x double> <double -0.000000e+00, double -0.000000e+00>, %1
@@ -1709,18 +1709,18 @@ define double @test_mask3_vfmsub_sd_1(<2 x double> %a, <2 x double> %b, <2 x dou
 
 define <4 x float> @test_mask3_vfnmsub_ss(<4 x float> %a, <4 x float> %b, <4 x float> %c, i8 %mask) {
 ; CHECK-LABEL: @test_mask3_vfnmsub_ss(
-; CHECK-NEXT:    [[DOTRHS:%.*]] = extractelement <4 x float> [[A:%.*]], i64 0
-; CHECK-NEXT:    [[TMP1:%.*]] = fsub float -0.000000e+00, [[DOTRHS]]
-; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <4 x float> [[B:%.*]], i32 0
-; CHECK-NEXT:    [[DOTRHS1:%.*]] = extractelement <4 x float> [[C:%.*]], i64 0
-; CHECK-NEXT:    [[TMP3:%.*]] = fsub float -0.000000e+00, [[DOTRHS1]]
-; CHECK-NEXT:    [[TMP4:%.*]] = call float @llvm.fma.f32(float [[TMP1]], float [[TMP2]], float [[TMP3]])
-; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <4 x float> [[C]], i64 0
-; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
-; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <8 x i1> [[TMP6]], i64 0
-; CHECK-NEXT:    [[TMP8:%.*]] = select i1 [[TMP7]], float [[TMP4]], float [[TMP5]]
-; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <4 x float> [[C]], float [[TMP8]], i64 0
-; CHECK-NEXT:    ret <4 x float> [[TMP9]]
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <4 x float> [[A:%.*]], i64 0
+; CHECK-NEXT:    [[TMP2:%.*]] = fsub float -0.000000e+00, [[TMP1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <4 x float> [[B:%.*]], i32 0
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <4 x float> [[C:%.*]], i64 0
+; CHECK-NEXT:    [[TMP5:%.*]] = fsub float -0.000000e+00, [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = call float @llvm.fma.f32(float [[TMP2]], float [[TMP3]], float [[TMP5]])
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <4 x float> [[C]], i64 0
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <8 x i1> [[TMP8]], i64 0
+; CHECK-NEXT:    [[TMP10:%.*]] = select i1 [[TMP9]], float [[TMP6]], float [[TMP7]]
+; CHECK-NEXT:    [[TMP11:%.*]] = insertelement <4 x float> [[C]], float [[TMP10]], i64 0
+; CHECK-NEXT:    ret <4 x float> [[TMP11]]
 ;
   %1 = insertelement <4 x float> %a, float 1.000000e+00, i32 1
   %2 = insertelement <4 x float> %1, float 2.000000e+00, i32 2
@@ -1744,17 +1744,17 @@ define <4 x float> @test_mask3_vfnmsub_ss(<4 x float> %a, <4 x float> %b, <4 x f
 
 define float @test_mask3_vfnmsub_ss_0(<4 x float> %a, <4 x float> %b, <4 x float> %c, i8 %mask) {
 ; CHECK-LABEL: @test_mask3_vfnmsub_ss_0(
-; CHECK-NEXT:    [[DOTRHS:%.*]] = extractelement <4 x float> [[A:%.*]], i64 0
-; CHECK-NEXT:    [[TMP1:%.*]] = fsub float -0.000000e+00, [[DOTRHS]]
-; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <4 x float> [[B:%.*]], i64 0
-; CHECK-NEXT:    [[DOTRHS1:%.*]] = extractelement <4 x float> [[C:%.*]], i32 0
-; CHECK-NEXT:    [[TMP3:%.*]] = fsub float -0.000000e+00, [[DOTRHS1]]
-; CHECK-NEXT:    [[TMP4:%.*]] = call float @llvm.fma.f32(float [[TMP1]], float [[TMP2]], float [[TMP3]])
-; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <4 x float> [[C]], i32 0
-; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
-; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <8 x i1> [[TMP6]], i64 0
-; CHECK-NEXT:    [[TMP8:%.*]] = select i1 [[TMP7]], float [[TMP4]], float [[TMP5]]
-; CHECK-NEXT:    ret float [[TMP8]]
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <4 x float> [[A:%.*]], i64 0
+; CHECK-NEXT:    [[TMP2:%.*]] = fsub float -0.000000e+00, [[TMP1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <4 x float> [[B:%.*]], i64 0
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <4 x float> [[C:%.*]], i32 0
+; CHECK-NEXT:    [[TMP5:%.*]] = fsub float -0.000000e+00, [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = call float @llvm.fma.f32(float [[TMP2]], float [[TMP3]], float [[TMP5]])
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <4 x float> [[C]], i32 0
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <8 x i1> [[TMP8]], i64 0
+; CHECK-NEXT:    [[TMP10:%.*]] = select i1 [[TMP9]], float [[TMP6]], float [[TMP7]]
+; CHECK-NEXT:    ret float [[TMP10]]
 ;
   %1 = insertelement <4 x float> %c, float 1.000000e+00, i32 1
   %2 = insertelement <4 x float> %1, float 2.000000e+00, i32 2
@@ -1798,18 +1798,18 @@ define float @test_mask3_vfnmsub_ss_1(<4 x float> %a, <4 x float> %b, <4 x float
 
 define <2 x double> @test_mask3_vfnmsub_sd(<2 x double> %a, <2 x double> %b, <2 x double> %c, i8 %mask) {
 ; CHECK-LABEL: @test_mask3_vfnmsub_sd(
-; CHECK-NEXT:    [[DOTRHS:%.*]] = extractelement <2 x double> [[A:%.*]], i64 0
-; CHECK-NEXT:    [[TMP1:%.*]] = fsub double -0.000000e+00, [[DOTRHS]]
-; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <2 x double> [[B:%.*]], i64 0
-; CHECK-NEXT:    [[DOTRHS1:%.*]] = extractelement <2 x double> [[C:%.*]], i64 0
-; CHECK-NEXT:    [[TMP3:%.*]] = fsub double -0.000000e+00, [[DOTRHS1]]
-; CHECK-NEXT:    [[TMP4:%.*]] = call double @llvm.fma.f64(double [[TMP1]], double [[TMP2]], double [[TMP3]])
-; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <2 x double> [[C]], i64 0
-; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
-; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <8 x i1> [[TMP6]], i64 0
-; CHECK-NEXT:    [[TMP8:%.*]] = select i1 [[TMP7]], double [[TMP4]], double [[TMP5]]
-; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <2 x double> [[C]], double [[TMP8]], i64 0
-; CHECK-NEXT:    ret <2 x double> [[TMP9]]
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <2 x double> [[A:%.*]], i64 0
+; CHECK-NEXT:    [[TMP2:%.*]] = fsub double -0.000000e+00, [[TMP1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <2 x double> [[B:%.*]], i64 0
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x double> [[C:%.*]], i64 0
+; CHECK-NEXT:    [[TMP5:%.*]] = fsub double -0.000000e+00, [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = call double @llvm.fma.f64(double [[TMP2]], double [[TMP3]], double [[TMP5]])
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <2 x double> [[C]], i64 0
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <8 x i1> [[TMP8]], i64 0
+; CHECK-NEXT:    [[TMP10:%.*]] = select i1 [[TMP9]], double [[TMP6]], double [[TMP7]]
+; CHECK-NEXT:    [[TMP11:%.*]] = insertelement <2 x double> [[C]], double [[TMP10]], i64 0
+; CHECK-NEXT:    ret <2 x double> [[TMP11]]
 ;
   %1 = insertelement <2 x double> %a, double 1.000000e+00, i32 1
   %2 = insertelement <2 x double> %b, double 2.000000e+00, i32 1
@@ -1829,17 +1829,17 @@ define <2 x double> @test_mask3_vfnmsub_sd(<2 x double> %a, <2 x double> %b, <2
 
 define double @test_mask3_vfnmsub_sd_0(<2 x double> %a, <2 x double> %b, <2 x double> %c, i8 %mask) {
 ; CHECK-LABEL: @test_mask3_vfnmsub_sd_0(
-; CHECK-NEXT:    [[DOTRHS:%.*]] = extractelement <2 x double> [[A:%.*]], i64 0
-; CHECK-NEXT:    [[TMP1:%.*]] = fsub double -0.000000e+00, [[DOTRHS]]
-; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <2 x double> [[B:%.*]], i64 0
-; CHECK-NEXT:    [[DOTRHS1:%.*]] = extractelement <2 x double> [[C:%.*]], i64 0
-; CHECK-NEXT:    [[TMP3:%.*]] = fsub double -0.000000e+00, [[DOTRHS1]]
-; CHECK-NEXT:    [[TMP4:%.*]] = call double @llvm.fma.f64(double [[TMP1]], double [[TMP2]], double [[TMP3]])
-; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <2 x double> [[C]], i64 0
-; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
-; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <8 x i1> [[TMP6]], i64 0
-; CHECK-NEXT:    [[TMP8:%.*]] = select i1 [[TMP7]], double [[TMP4]], double [[TMP5]]
-; CHECK-NEXT:    ret double [[TMP8]]
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <2 x double> [[A:%.*]], i64 0
+; CHECK-NEXT:    [[TMP2:%.*]] = fsub double -0.000000e+00, [[TMP1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <2 x double> [[B:%.*]], i64 0
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x double> [[C:%.*]], i64 0
+; CHECK-NEXT:    [[TMP5:%.*]] = fsub double -0.000000e+00, [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = call double @llvm.fma.f64(double [[TMP2]], double [[TMP3]], double [[TMP5]])
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <2 x double> [[C]], i64 0
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast i8 [[MASK:%.*]] to <8 x i1>
+; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <8 x i1> [[TMP8]], i64 0
+; CHECK-NEXT:    [[TMP10:%.*]] = select i1 [[TMP9]], double [[TMP6]], double [[TMP7]]
+; CHECK-NEXT:    ret double [[TMP10]]
 ;
   %1 = insertelement <2 x double> %c, double 1.000000e+00, i32 1
   %2 = fsub <2 x double> <double -0.000000e+00, double -0.000000e+00>, %a
diff --git a/test/Transforms/InstCombine/X86/x86-masked-memops.ll b/test/Transforms/InstCombine/X86/x86-masked-memops.ll
index 8502b1899ecb2e976e90280585097e5926902fd3..123ba1f49e1ae3ebb12b975bcef9466657cc58a3 100644
--- a/test/Transforms/InstCombine/X86/x86-masked-memops.ll
+++ b/test/Transforms/InstCombine/X86/x86-masked-memops.ll
@@ -7,7 +7,7 @@
 
 define <4 x float> @mload(i8* %f, <4 x i32> %mask) {
 ; CHECK-LABEL: @mload(
-; CHECK-NEXT:    [[LD:%.*]] = tail call <4 x float> @llvm.x86.avx.maskload.ps(i8* %f, <4 x i32> %mask)
+; CHECK-NEXT:    [[LD:%.*]] = tail call <4 x float> @llvm.x86.avx.maskload.ps(i8* [[F:%.*]], <4 x i32> [[MASK:%.*]])
 ; CHECK-NEXT:    ret <4 x float> [[LD]]
 ;
   %ld = tail call <4 x float> @llvm.x86.avx.maskload.ps(i8* %f, <4 x i32> %mask)
@@ -41,7 +41,7 @@ define <4 x float> @mload_fake_ones(i8* %f) {
 
 define <4 x float> @mload_real_ones(i8* %f) {
 ; CHECK-LABEL: @mload_real_ones(
-; CHECK-NEXT:    [[CASTVEC:%.*]] = bitcast i8* %f to <4 x float>*
+; CHECK-NEXT:    [[CASTVEC:%.*]] = bitcast i8* [[F:%.*]] to <4 x float>*
 ; CHECK-NEXT:    [[UNMASKEDLOAD:%.*]] = load <4 x float>, <4 x float>* [[CASTVEC]], align 1
 ; CHECK-NEXT:    ret <4 x float> [[UNMASKEDLOAD]]
 ;
@@ -54,7 +54,7 @@ define <4 x float> @mload_real_ones(i8* %f) {
 
 define <4 x float> @mload_one_one(i8* %f) {
 ; CHECK-LABEL: @mload_one_one(
-; CHECK-NEXT:    [[CASTVEC:%.*]] = bitcast i8* %f to <4 x float>*
+; CHECK-NEXT:    [[CASTVEC:%.*]] = bitcast i8* [[F:%.*]] to <4 x float>*
 ; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x float> @llvm.masked.load.v4f32.p0v4f32(<4 x float>* [[CASTVEC]], i32 1, <4 x i1> <i1 false, i1 false, i1 false, i1 true>, <4 x float> zeroinitializer)
 ; CHECK-NEXT:    ret <4 x float> [[TMP1]]
 ;
@@ -67,7 +67,7 @@ define <4 x float> @mload_one_one(i8* %f) {
 
 define <2 x double> @mload_one_one_double(i8* %f) {
 ; CHECK-LABEL: @mload_one_one_double(
-; CHECK-NEXT:    [[CASTVEC:%.*]] = bitcast i8* %f to <2 x double>*
+; CHECK-NEXT:    [[CASTVEC:%.*]] = bitcast i8* [[F:%.*]] to <2 x double>*
 ; CHECK-NEXT:    [[TMP1:%.*]] = call <2 x double> @llvm.masked.load.v2f64.p0v2f64(<2 x double>* [[CASTVEC]], i32 1, <2 x i1> <i1 true, i1 false>, <2 x double> zeroinitializer)
 ; CHECK-NEXT:    ret <2 x double> [[TMP1]]
 ;
@@ -80,7 +80,7 @@ define <2 x double> @mload_one_one_double(i8* %f) {
 
 define <8 x float> @mload_v8f32(i8* %f) {
 ; CHECK-LABEL: @mload_v8f32(
-; CHECK-NEXT:    [[CASTVEC:%.*]] = bitcast i8* %f to <8 x float>*
+; CHECK-NEXT:    [[CASTVEC:%.*]] = bitcast i8* [[F:%.*]] to <8 x float>*
 ; CHECK-NEXT:    [[TMP1:%.*]] = call <8 x float> @llvm.masked.load.v8f32.p0v8f32(<8 x float>* [[CASTVEC]], i32 1, <8 x i1> <i1 false, i1 false, i1 false, i1 true, i1 false, i1 false, i1 false, i1 false>, <8 x float> zeroinitializer)
 ; CHECK-NEXT:    ret <8 x float> [[TMP1]]
 ;
@@ -91,7 +91,7 @@ define <8 x float> @mload_v8f32(i8* %f) {
 
 define <4 x double> @mload_v4f64(i8* %f) {
 ; CHECK-LABEL: @mload_v4f64(
-; CHECK-NEXT:    [[CASTVEC:%.*]] = bitcast i8* %f to <4 x double>*
+; CHECK-NEXT:    [[CASTVEC:%.*]] = bitcast i8* [[F:%.*]] to <4 x double>*
 ; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x double> @llvm.masked.load.v4f64.p0v4f64(<4 x double>* [[CASTVEC]], i32 1, <4 x i1> <i1 true, i1 false, i1 false, i1 false>, <4 x double> zeroinitializer)
 ; CHECK-NEXT:    ret <4 x double> [[TMP1]]
 ;
@@ -104,7 +104,7 @@ define <4 x double> @mload_v4f64(i8* %f) {
 
 define <4 x i32> @mload_v4i32(i8* %f) {
 ; CHECK-LABEL: @mload_v4i32(
-; CHECK-NEXT:    [[CASTVEC:%.*]] = bitcast i8* %f to <4 x i32>*
+; CHECK-NEXT:    [[CASTVEC:%.*]] = bitcast i8* [[F:%.*]] to <4 x i32>*
 ; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* [[CASTVEC]], i32 1, <4 x i1> <i1 false, i1 false, i1 false, i1 true>, <4 x i32> zeroinitializer)
 ; CHECK-NEXT:    ret <4 x i32> [[TMP1]]
 ;
@@ -115,7 +115,7 @@ define <4 x i32> @mload_v4i32(i8* %f) {
 
 define <2 x i64> @mload_v2i64(i8* %f) {
 ; CHECK-LABEL: @mload_v2i64(
-; CHECK-NEXT:    [[CASTVEC:%.*]] = bitcast i8* %f to <2 x i64>*
+; CHECK-NEXT:    [[CASTVEC:%.*]] = bitcast i8* [[F:%.*]] to <2 x i64>*
 ; CHECK-NEXT:    [[TMP1:%.*]] = call <2 x i64> @llvm.masked.load.v2i64.p0v2i64(<2 x i64>* [[CASTVEC]], i32 1, <2 x i1> <i1 true, i1 false>, <2 x i64> zeroinitializer)
 ; CHECK-NEXT:    ret <2 x i64> [[TMP1]]
 ;
@@ -126,7 +126,7 @@ define <2 x i64> @mload_v2i64(i8* %f) {
 
 define <8 x i32> @mload_v8i32(i8* %f) {
 ; CHECK-LABEL: @mload_v8i32(
-; CHECK-NEXT:    [[CASTVEC:%.*]] = bitcast i8* %f to <8 x i32>*
+; CHECK-NEXT:    [[CASTVEC:%.*]] = bitcast i8* [[F:%.*]] to <8 x i32>*
 ; CHECK-NEXT:    [[TMP1:%.*]] = call <8 x i32> @llvm.masked.load.v8i32.p0v8i32(<8 x i32>* [[CASTVEC]], i32 1, <8 x i1> <i1 false, i1 false, i1 false, i1 true, i1 false, i1 false, i1 false, i1 false>, <8 x i32> zeroinitializer)
 ; CHECK-NEXT:    ret <8 x i32> [[TMP1]]
 ;
@@ -137,7 +137,7 @@ define <8 x i32> @mload_v8i32(i8* %f) {
 
 define <4 x i64> @mload_v4i64(i8* %f) {
 ; CHECK-LABEL: @mload_v4i64(
-; CHECK-NEXT:    [[CASTVEC:%.*]] = bitcast i8* %f to <4 x i64>*
+; CHECK-NEXT:    [[CASTVEC:%.*]] = bitcast i8* [[F:%.*]] to <4 x i64>*
 ; CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i64> @llvm.masked.load.v4i64.p0v4i64(<4 x i64>* [[CASTVEC]], i32 1, <4 x i1> <i1 true, i1 false, i1 false, i1 false>, <4 x i64> zeroinitializer)
 ; CHECK-NEXT:    ret <4 x i64> [[TMP1]]
 ;
@@ -153,7 +153,7 @@ define <4 x i64> @mload_v4i64(i8* %f) {
 
 define void @mstore(i8* %f, <4 x i32> %mask, <4 x float> %v) {
 ; CHECK-LABEL: @mstore(
-; CHECK-NEXT:    tail call void @llvm.x86.avx.maskstore.ps(i8* %f, <4 x i32> %mask, <4 x float> %v)
+; CHECK-NEXT:    tail call void @llvm.x86.avx.maskstore.ps(i8* [[F:%.*]], <4 x i32> [[MASK:%.*]], <4 x float> [[V:%.*]])
 ; CHECK-NEXT:    ret void
 ;
   tail call void @llvm.x86.avx.maskstore.ps(i8* %f, <4 x i32> %mask, <4 x float> %v)
@@ -187,8 +187,8 @@ define void @mstore_fake_ones(i8* %f, <4 x float> %v) {
 
 define void @mstore_real_ones(i8* %f, <4 x float> %v) {
 ; CHECK-LABEL: @mstore_real_ones(
-; CHECK-NEXT:    [[CASTVEC:%.*]] = bitcast i8* %f to <4 x float>*
-; CHECK-NEXT:    store <4 x float> %v, <4 x float>* [[CASTVEC]], align 1
+; CHECK-NEXT:    [[CASTVEC:%.*]] = bitcast i8* [[F:%.*]] to <4 x float>*
+; CHECK-NEXT:    store <4 x float> [[V:%.*]], <4 x float>* [[CASTVEC]], align 1
 ; CHECK-NEXT:    ret void
 ;
   tail call void @llvm.x86.avx.maskstore.ps(i8* %f, <4 x i32> <i32 -1, i32 -2, i32 -3, i32 -2147483648>, <4 x float> %v)
@@ -200,8 +200,8 @@ define void @mstore_real_ones(i8* %f, <4 x float> %v) {
 
 define void @mstore_one_one(i8* %f, <4 x float> %v) {
 ; CHECK-LABEL: @mstore_one_one(
-; CHECK-NEXT:    [[CASTVEC:%.*]] = bitcast i8* %f to <4 x float>*
-; CHECK-NEXT:    call void @llvm.masked.store.v4f32.p0v4f32(<4 x float> %v, <4 x float>* [[CASTVEC]], i32 1, <4 x i1> <i1 false, i1 false, i1 false, i1 true>)
+; CHECK-NEXT:    [[CASTVEC:%.*]] = bitcast i8* [[F:%.*]] to <4 x float>*
+; CHECK-NEXT:    call void @llvm.masked.store.v4f32.p0v4f32(<4 x float> [[V:%.*]], <4 x float>* [[CASTVEC]], i32 1, <4 x i1> <i1 false, i1 false, i1 false, i1 true>)
 ; CHECK-NEXT:    ret void
 ;
   tail call void @llvm.x86.avx.maskstore.ps(i8* %f, <4 x i32> <i32 0, i32 0, i32 0, i32 -1>, <4 x float> %v)
@@ -213,8 +213,8 @@ define void @mstore_one_one(i8* %f, <4 x float> %v) {
 
 define void @mstore_one_one_double(i8* %f, <2 x double> %v) {
 ; CHECK-LABEL: @mstore_one_one_double(
-; CHECK-NEXT:    [[CASTVEC:%.*]] = bitcast i8* %f to <2 x double>*
-; CHECK-NEXT:    call void @llvm.masked.store.v2f64.p0v2f64(<2 x double> %v, <2 x double>* [[CASTVEC]], i32 1, <2 x i1> <i1 true, i1 false>)
+; CHECK-NEXT:    [[CASTVEC:%.*]] = bitcast i8* [[F:%.*]] to <2 x double>*
+; CHECK-NEXT:    call void @llvm.masked.store.v2f64.p0v2f64(<2 x double> [[V:%.*]], <2 x double>* [[CASTVEC]], i32 1, <2 x i1> <i1 true, i1 false>)
 ; CHECK-NEXT:    ret void
 ;
   tail call void @llvm.x86.avx.maskstore.pd(i8* %f, <2 x i64> <i64 -1, i64 0>, <2 x double> %v)
@@ -226,8 +226,8 @@ define void @mstore_one_one_double(i8* %f, <2 x double> %v) {
 
 define void @mstore_v8f32(i8* %f, <8 x float> %v) {
 ; CHECK-LABEL: @mstore_v8f32(
-; CHECK-NEXT:    [[CASTVEC:%.*]] = bitcast i8* %f to <8 x float>*
-; CHECK-NEXT:    call void @llvm.masked.store.v8f32.p0v8f32(<8 x float> %v, <8 x float>* [[CASTVEC]], i32 1, <8 x i1> <i1 false, i1 false, i1 false, i1 false, i1 true, i1 true, i1 true, i1 true>)
+; CHECK-NEXT:    [[CASTVEC:%.*]] = bitcast i8* [[F:%.*]] to <8 x float>*
+; CHECK-NEXT:    call void @llvm.masked.store.v8f32.p0v8f32(<8 x float> [[V:%.*]], <8 x float>* [[CASTVEC]], i32 1, <8 x i1> <i1 false, i1 false, i1 false, i1 false, i1 true, i1 true, i1 true, i1 true>)
 ; CHECK-NEXT:    ret void
 ;
   tail call void @llvm.x86.avx.maskstore.ps.256(i8* %f, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 -1, i32 -2, i32 -3, i32 -4>, <8 x float> %v)
@@ -237,8 +237,8 @@ define void @mstore_v8f32(i8* %f, <8 x float> %v) {
 
 define void @mstore_v4f64(i8* %f, <4 x double> %v) {
 ; CHECK-LABEL: @mstore_v4f64(
-; CHECK-NEXT:    [[CASTVEC:%.*]] = bitcast i8* %f to <4 x double>*
-; CHECK-NEXT:    call void @llvm.masked.store.v4f64.p0v4f64(<4 x double> %v, <4 x double>* [[CASTVEC]], i32 1, <4 x i1> <i1 true, i1 false, i1 false, i1 false>)
+; CHECK-NEXT:    [[CASTVEC:%.*]] = bitcast i8* [[F:%.*]] to <4 x double>*
+; CHECK-NEXT:    call void @llvm.masked.store.v4f64.p0v4f64(<4 x double> [[V:%.*]], <4 x double>* [[CASTVEC]], i32 1, <4 x i1> <i1 true, i1 false, i1 false, i1 false>)
 ; CHECK-NEXT:    ret void
 ;
   tail call void @llvm.x86.avx.maskstore.pd.256(i8* %f, <4 x i64> <i64 -1, i64 0, i64 1, i64 2>, <4 x double> %v)
@@ -250,8 +250,8 @@ define void @mstore_v4f64(i8* %f, <4 x double> %v) {
 
 define void @mstore_v4i32(i8* %f, <4 x i32> %v) {
 ; CHECK-LABEL: @mstore_v4i32(
-; CHECK-NEXT:    [[CASTVEC:%.*]] = bitcast i8* %f to <4 x i32>*
-; CHECK-NEXT:    call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %v, <4 x i32>* [[CASTVEC]], i32 1, <4 x i1> <i1 false, i1 false, i1 true, i1 true>)
+; CHECK-NEXT:    [[CASTVEC:%.*]] = bitcast i8* [[F:%.*]] to <4 x i32>*
+; CHECK-NEXT:    call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> [[V:%.*]], <4 x i32>* [[CASTVEC]], i32 1, <4 x i1> <i1 false, i1 false, i1 true, i1 true>)
 ; CHECK-NEXT:    ret void
 ;
   tail call void @llvm.x86.avx2.maskstore.d(i8* %f, <4 x i32> <i32 0, i32 1, i32 -1, i32 -2>, <4 x i32> %v)
@@ -261,8 +261,8 @@ define void @mstore_v4i32(i8* %f, <4 x i32> %v) {
 
 define void @mstore_v2i64(i8* %f, <2 x i64> %v) {
 ; CHECK-LABEL: @mstore_v2i64(
-; CHECK-NEXT:    [[CASTVEC:%.*]] = bitcast i8* %f to <2 x i64>*
-; CHECK-NEXT:    call void @llvm.masked.store.v2i64.p0v2i64(<2 x i64> %v, <2 x i64>* [[CASTVEC]], i32 1, <2 x i1> <i1 true, i1 false>)
+; CHECK-NEXT:    [[CASTVEC:%.*]] = bitcast i8* [[F:%.*]] to <2 x i64>*
+; CHECK-NEXT:    call void @llvm.masked.store.v2i64.p0v2i64(<2 x i64> [[V:%.*]], <2 x i64>* [[CASTVEC]], i32 1, <2 x i1> <i1 true, i1 false>)
 ; CHECK-NEXT:    ret void
 ;
   tail call void @llvm.x86.avx2.maskstore.q(i8* %f, <2 x i64> <i64 -1, i64 0>, <2 x i64> %v)
@@ -272,8 +272,8 @@ define void @mstore_v2i64(i8* %f, <2 x i64> %v) {
 
 define void @mstore_v8i32(i8* %f, <8 x i32> %v) {
 ; CHECK-LABEL: @mstore_v8i32(
-; CHECK-NEXT:    [[CASTVEC:%.*]] = bitcast i8* %f to <8 x i32>*
-; CHECK-NEXT:    call void @llvm.masked.store.v8i32.p0v8i32(<8 x i32> %v, <8 x i32>* [[CASTVEC]], i32 1, <8 x i1> <i1 false, i1 false, i1 false, i1 false, i1 true, i1 true, i1 true, i1 true>)
+; CHECK-NEXT:    [[CASTVEC:%.*]] = bitcast i8* [[F:%.*]] to <8 x i32>*
+; CHECK-NEXT:    call void @llvm.masked.store.v8i32.p0v8i32(<8 x i32> [[V:%.*]], <8 x i32>* [[CASTVEC]], i32 1, <8 x i1> <i1 false, i1 false, i1 false, i1 false, i1 true, i1 true, i1 true, i1 true>)
 ; CHECK-NEXT:    ret void
 ;
   tail call void @llvm.x86.avx2.maskstore.d.256(i8* %f, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 -1, i32 -2, i32 -3, i32 -4>, <8 x i32> %v)
@@ -283,8 +283,8 @@ define void @mstore_v8i32(i8* %f, <8 x i32> %v) {
 
 define void @mstore_v4i64(i8* %f, <4 x i64> %v) {
 ; CHECK-LABEL: @mstore_v4i64(
-; CHECK-NEXT:    [[CASTVEC:%.*]] = bitcast i8* %f to <4 x i64>*
-; CHECK-NEXT:    call void @llvm.masked.store.v4i64.p0v4i64(<4 x i64> %v, <4 x i64>* [[CASTVEC]], i32 1, <4 x i1> <i1 true, i1 false, i1 false, i1 false>)
+; CHECK-NEXT:    [[CASTVEC:%.*]] = bitcast i8* [[F:%.*]] to <4 x i64>*
+; CHECK-NEXT:    call void @llvm.masked.store.v4i64.p0v4i64(<4 x i64> [[V:%.*]], <4 x i64>* [[CASTVEC]], i32 1, <4 x i1> <i1 true, i1 false, i1 false, i1 false>)
 ; CHECK-NEXT:    ret void
 ;
   tail call void @llvm.x86.avx2.maskstore.q.256(i8* %f, <4 x i64> <i64 -1, i64 0, i64 1, i64 2>, <4 x i64> %v)
diff --git a/test/Transforms/InstCombine/alloca-cast-debuginfo.ll b/test/Transforms/InstCombine/alloca-cast-debuginfo.ll
index b5a1f34ee056cc5b19a9f7c0ac12f12bdc4cfc57..c3bca700ea722429acb5b1767763452c5be45c38 100644
--- a/test/Transforms/InstCombine/alloca-cast-debuginfo.ll
+++ b/test/Transforms/InstCombine/alloca-cast-debuginfo.ll
@@ -45,7 +45,7 @@ entry:
 ; Another dbg.value for "local" would be redundant here.
 ; CHECK-NOT: call void @llvm.dbg.value(metadata i8* [[simplified]], metadata !22, metadata !DIExpression())
 ;
-; CHECK: call void @escape(i8* [[simplified]])
+; CHECK: call void @escape(i8* nonnull [[simplified]])
 ; CHECK: ret void
 
 declare void @llvm.dbg.declare(metadata, metadata, metadata)
diff --git a/test/Transforms/InstCombine/apint-shift.ll b/test/Transforms/InstCombine/apint-shift.ll
index efc088637c47fec699bfd1eddda6a168432918d7..52602dd7f4547160c6b33e7afc37c4b972f41643 100644
--- a/test/Transforms/InstCombine/apint-shift.ll
+++ b/test/Transforms/InstCombine/apint-shift.ll
@@ -1,11 +1,9 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; This test makes sure that shift instructions are properly eliminated
-; even with arbitrary precision integers.
 ; RUN: opt < %s -instcombine -S | FileCheck %s
 
 define i55 @test6(i55 %A) {
 ; CHECK-LABEL: @test6(
-; CHECK-NEXT:    [[C:%.*]] = mul i55 %A, 6
+; CHECK-NEXT:    [[C:%.*]] = mul i55 [[A:%.*]], 6
 ; CHECK-NEXT:    ret i55 [[C]]
 ;
   %B = shl i55 %A, 1
@@ -17,7 +15,7 @@ define i55 @test6(i55 %A) {
 
 define i55 @test6a(i55 %A) {
 ; CHECK-LABEL: @test6a(
-; CHECK-NEXT:    [[C:%.*]] = mul i55 %A, 6
+; CHECK-NEXT:    [[C:%.*]] = mul i55 [[A:%.*]], 6
 ; CHECK-NEXT:    ret i55 [[C]]
 ;
   %B = mul i55 %A, 3
@@ -29,7 +27,7 @@ define i55 @test6a(i55 %A) {
 
 define <2 x i55> @test6a_vec(<2 x i55> %A) {
 ; CHECK-LABEL: @test6a_vec(
-; CHECK-NEXT:    [[C:%.*]] = mul <2 x i55> %A, <i55 6, i55 48>
+; CHECK-NEXT:    [[C:%.*]] = mul <2 x i55> [[A:%.*]], <i55 6, i55 48>
 ; CHECK-NEXT:    ret <2 x i55> [[C]]
 ;
   %B = mul <2 x i55> %A, <i55 3, i55 12>
@@ -57,7 +55,7 @@ define i7 @test8(i7 %A) {
 
 define i17 @test9(i17 %A) {
 ; CHECK-LABEL: @test9(
-; CHECK-NEXT:    [[B:%.*]] = and i17 %A, 1
+; CHECK-NEXT:    [[B:%.*]] = and i17 [[A:%.*]], 1
 ; CHECK-NEXT:    ret i17 [[B]]
 ;
   %B = shl i17 %A, 16
@@ -69,7 +67,7 @@ define i17 @test9(i17 %A) {
 
 define i19 @test10(i19 %X) {
 ; CHECK-LABEL: @test10(
-; CHECK-NEXT:    [[SH1:%.*]] = and i19 %X, -262144
+; CHECK-NEXT:    [[SH1:%.*]] = and i19 [[X:%.*]], -262144
 ; CHECK-NEXT:    ret i19 [[SH1]]
 ;
   %sh1 = lshr i19 %X, 18
@@ -82,7 +80,7 @@ define i19 @test10(i19 %X) {
 
 define <2 x i19> @lshr_lshr_splat_vec(<2 x i19> %X) {
 ; CHECK-LABEL: @lshr_lshr_splat_vec(
-; CHECK-NEXT:    [[SH1:%.*]] = lshr <2 x i19> %X, <i19 5, i19 5>
+; CHECK-NEXT:    [[SH1:%.*]] = lshr <2 x i19> [[X:%.*]], <i19 5, i19 5>
 ; CHECK-NEXT:    ret <2 x i19> [[SH1]]
 ;
   %sh1 = lshr <2 x i19> %X, <i19 3, i19 3>
@@ -92,8 +90,8 @@ define <2 x i19> @lshr_lshr_splat_vec(<2 x i19> %X) {
 
 define i9 @multiuse_lshr_lshr(i9 %x) {
 ; CHECK-LABEL: @multiuse_lshr_lshr(
-; CHECK-NEXT:    [[SH1:%.*]] = lshr i9 %x, 2
-; CHECK-NEXT:    [[SH2:%.*]] = lshr i9 %x, 5
+; CHECK-NEXT:    [[SH1:%.*]] = lshr i9 [[X:%.*]], 2
+; CHECK-NEXT:    [[SH2:%.*]] = lshr i9 [[X]], 5
 ; CHECK-NEXT:    [[MUL:%.*]] = mul i9 [[SH1]], [[SH2]]
 ; CHECK-NEXT:    ret i9 [[MUL]]
 ;
@@ -105,8 +103,8 @@ define i9 @multiuse_lshr_lshr(i9 %x) {
 
 define <2 x i9> @multiuse_lshr_lshr_splat(<2 x i9> %x) {
 ; CHECK-LABEL: @multiuse_lshr_lshr_splat(
-; CHECK-NEXT:    [[SH1:%.*]] = lshr <2 x i9> %x, <i9 2, i9 2>
-; CHECK-NEXT:    [[SH2:%.*]] = lshr <2 x i9> %x, <i9 5, i9 5>
+; CHECK-NEXT:    [[SH1:%.*]] = lshr <2 x i9> [[X:%.*]], <i9 2, i9 2>
+; CHECK-NEXT:    [[SH2:%.*]] = lshr <2 x i9> [[X]], <i9 5, i9 5>
 ; CHECK-NEXT:    [[MUL:%.*]] = mul <2 x i9> [[SH1]], [[SH2]]
 ; CHECK-NEXT:    ret <2 x i9> [[MUL]]
 ;
@@ -121,7 +119,7 @@ define <2 x i9> @multiuse_lshr_lshr_splat(<2 x i9> %x) {
 
 define <2 x i19> @shl_shl_splat_vec(<2 x i19> %X) {
 ; CHECK-LABEL: @shl_shl_splat_vec(
-; CHECK-NEXT:    [[SH1:%.*]] = shl <2 x i19> %X, <i19 5, i19 5>
+; CHECK-NEXT:    [[SH1:%.*]] = shl <2 x i19> [[X:%.*]], <i19 5, i19 5>
 ; CHECK-NEXT:    ret <2 x i19> [[SH1]]
 ;
   %sh1 = shl <2 x i19> %X, <i19 3, i19 3>
@@ -131,8 +129,8 @@ define <2 x i19> @shl_shl_splat_vec(<2 x i19> %X) {
 
 define i42 @multiuse_shl_shl(i42 %x) {
 ; CHECK-LABEL: @multiuse_shl_shl(
-; CHECK-NEXT:    [[SH1:%.*]] = shl i42 %x, 8
-; CHECK-NEXT:    [[SH2:%.*]] = shl i42 %x, 17
+; CHECK-NEXT:    [[SH1:%.*]] = shl i42 [[X:%.*]], 8
+; CHECK-NEXT:    [[SH2:%.*]] = shl i42 [[X]], 17
 ; CHECK-NEXT:    [[MUL:%.*]] = mul i42 [[SH1]], [[SH2]]
 ; CHECK-NEXT:    ret i42 [[MUL]]
 ;
@@ -144,8 +142,8 @@ define i42 @multiuse_shl_shl(i42 %x) {
 
 define <2 x i42> @multiuse_shl_shl_splat(<2 x i42> %x) {
 ; CHECK-LABEL: @multiuse_shl_shl_splat(
-; CHECK-NEXT:    [[SH1:%.*]] = shl <2 x i42> %x, <i42 8, i42 8>
-; CHECK-NEXT:    [[SH2:%.*]] = shl <2 x i42> %x, <i42 17, i42 17>
+; CHECK-NEXT:    [[SH1:%.*]] = shl <2 x i42> [[X:%.*]], <i42 8, i42 8>
+; CHECK-NEXT:    [[SH2:%.*]] = shl <2 x i42> [[X]], <i42 17, i42 17>
 ; CHECK-NEXT:    [[MUL:%.*]] = mul <2 x i42> [[SH1]], [[SH2]]
 ; CHECK-NEXT:    ret <2 x i42> [[MUL]]
 ;
@@ -160,7 +158,7 @@ define <2 x i42> @multiuse_shl_shl_splat(<2 x i42> %x) {
 
 define <2 x i19> @eq_shl_lshr_splat_vec(<2 x i19> %X) {
 ; CHECK-LABEL: @eq_shl_lshr_splat_vec(
-; CHECK-NEXT:    [[SH1:%.*]] = and <2 x i19> %X, <i19 65535, i19 65535>
+; CHECK-NEXT:    [[SH1:%.*]] = and <2 x i19> [[X:%.*]], <i19 65535, i19 65535>
 ; CHECK-NEXT:    ret <2 x i19> [[SH1]]
 ;
   %sh1 = shl <2 x i19> %X, <i19 3, i19 3>
@@ -173,7 +171,7 @@ define <2 x i19> @eq_shl_lshr_splat_vec(<2 x i19> %X) {
 
 define <2 x i19> @eq_lshr_shl_splat_vec(<2 x i19> %X) {
 ; CHECK-LABEL: @eq_lshr_shl_splat_vec(
-; CHECK-NEXT:    [[SH1:%.*]] = and <2 x i19> %X, <i19 -8, i19 -8>
+; CHECK-NEXT:    [[SH1:%.*]] = and <2 x i19> [[X:%.*]], <i19 -8, i19 -8>
 ; CHECK-NEXT:    ret <2 x i19> [[SH1]]
 ;
   %sh1 = lshr <2 x i19> %X, <i19 3, i19 3>
@@ -186,7 +184,7 @@ define <2 x i19> @eq_lshr_shl_splat_vec(<2 x i19> %X) {
 
 define <2 x i7> @lshr_shl_splat_vec(<2 x i7> %X) {
 ; CHECK-LABEL: @lshr_shl_splat_vec(
-; CHECK-NEXT:    [[MUL:%.*]] = mul <2 x i7> %X, <i7 -8, i7 -8>
+; CHECK-NEXT:    [[MUL:%.*]] = mul <2 x i7> [[X:%.*]], <i7 -8, i7 -8>
 ; CHECK-NEXT:    [[SH1:%.*]] = lshr exact <2 x i7> [[MUL]], <i7 1, i7 1>
 ; CHECK-NEXT:    ret <2 x i7> [[SH1]]
 ;
@@ -201,7 +199,7 @@ define <2 x i7> @lshr_shl_splat_vec(<2 x i7> %X) {
 
 define <2 x i7> @shl_lshr_splat_vec(<2 x i7> %X) {
 ; CHECK-LABEL: @shl_lshr_splat_vec(
-; CHECK-NEXT:    [[DIV:%.*]] = udiv <2 x i7> %X, <i7 9, i7 9>
+; CHECK-NEXT:    [[DIV:%.*]] = udiv <2 x i7> [[X:%.*]], <i7 9, i7 9>
 ; CHECK-NEXT:    [[SH1:%.*]] = shl nuw nsw <2 x i7> [[DIV]], <i7 1, i7 1>
 ; CHECK-NEXT:    ret <2 x i7> [[SH1]]
 ;
@@ -212,25 +210,26 @@ define <2 x i7> @shl_lshr_splat_vec(<2 x i7> %X) {
 }
 
 ; Don't hide the shl from scalar evolution. DAGCombine will get it.
-define i23 @test11(i23 %A) {
+
+define i23 @test11(i23 %x) {
 ; CHECK-LABEL: @test11(
-; CHECK-NEXT:    [[A:%.*]] = mul i23 %A, 3
+; CHECK-NEXT:    [[A:%.*]] = mul i23 [[X:%.*]], 3
 ; CHECK-NEXT:    [[B:%.*]] = lshr i23 [[A]], 11
 ; CHECK-NEXT:    [[C:%.*]] = shl i23 [[B]], 12
 ; CHECK-NEXT:    ret i23 [[C]]
 ;
-  %a = mul i23 %A, 3
-  %B = lshr i23 %a, 11
-  %C = shl i23 %B, 12
-  ret i23 %C
+  %a = mul i23 %x, 3
+  %b = lshr i23 %a, 11
+  %c = shl i23 %b, 12
+  ret i23 %c
 }
 
 ; shl (ashr X, C), C --> and X, C'
 
 define i47 @test12(i47 %X) {
 ; CHECK-LABEL: @test12(
-; CHECK-NEXT:    [[SH11:%.*]] = and i47 %X, -256
-; CHECK-NEXT:    ret i47 [[SH11]]
+; CHECK-NEXT:    [[TMP1:%.*]] = and i47 [[X:%.*]], -256
+; CHECK-NEXT:    ret i47 [[TMP1]]
 ;
   %sh1 = ashr i47 %X, 8
   %sh2 = shl i47 %sh1, 8
@@ -239,7 +238,7 @@ define i47 @test12(i47 %X) {
 
 define <2 x i47> @test12_splat_vec(<2 x i47> %X) {
 ; CHECK-LABEL: @test12_splat_vec(
-; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i47> %X, <i47 -256, i47 -256>
+; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i47> [[X:%.*]], <i47 -256, i47 -256>
 ; CHECK-NEXT:    ret <2 x i47> [[TMP1]]
 ;
   %sh1 = ashr <2 x i47> %X, <i47 8, i47 8>
@@ -248,22 +247,23 @@ define <2 x i47> @test12_splat_vec(<2 x i47> %X) {
 }
 
 ; Don't hide the shl from scalar evolution. DAGCombine will get it.
-define i18 @test13(i18 %A) {
+
+define i18 @test13(i18 %x) {
 ; CHECK-LABEL: @test13(
-; CHECK-NEXT:    [[A:%.*]] = mul i18 %A, 3
-; CHECK-NEXT:    [[B1:%.*]] = lshr i18 [[A]], 8
-; CHECK-NEXT:    [[C:%.*]] = shl i18 [[B1]], 9
+; CHECK-NEXT:    [[A:%.*]] = mul i18 [[X:%.*]], 3
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr i18 [[A]], 8
+; CHECK-NEXT:    [[C:%.*]] = shl i18 [[TMP1]], 9
 ; CHECK-NEXT:    ret i18 [[C]]
 ;
-  %a = mul i18 %A, 3
-  %B = ashr i18 %a, 8
-  %C = shl i18 %B, 9
-  ret i18 %C
+  %a = mul i18 %x, 3
+  %b = ashr i18 %a, 8
+  %c = shl i18 %b, 9
+  ret i18 %c
 }
 
 define i35 @test14(i35 %A) {
 ; CHECK-LABEL: @test14(
-; CHECK-NEXT:    [[B:%.*]] = and i35 %A, -19760
+; CHECK-NEXT:    [[B:%.*]] = and i35 [[A:%.*]], -19760
 ; CHECK-NEXT:    [[C:%.*]] = or i35 [[B]], 19744
 ; CHECK-NEXT:    ret i35 [[C]]
 ;
@@ -275,7 +275,7 @@ define i35 @test14(i35 %A) {
 
 define i79 @test14a(i79 %A) {
 ; CHECK-LABEL: @test14a(
-; CHECK-NEXT:    [[C:%.*]] = and i79 %A, 77
+; CHECK-NEXT:    [[C:%.*]] = and i79 [[A:%.*]], 77
 ; CHECK-NEXT:    ret i79 [[C]]
 ;
   %B = shl i79 %A, 4
@@ -286,7 +286,7 @@ define i79 @test14a(i79 %A) {
 
 define i45 @test15(i1 %C) {
 ; CHECK-LABEL: @test15(
-; CHECK-NEXT:    [[A:%.*]] = select i1 %C, i45 12, i45 4
+; CHECK-NEXT:    [[A:%.*]] = select i1 [[C:%.*]], i45 12, i45 4
 ; CHECK-NEXT:    ret i45 [[A]]
 ;
   %A = select i1 %C, i45 3, i45 1
@@ -296,7 +296,7 @@ define i45 @test15(i1 %C) {
 
 define i53 @test15a(i1 %X) {
 ; CHECK-LABEL: @test15a(
-; CHECK-NEXT:    [[V:%.*]] = select i1 %X, i53 512, i53 128
+; CHECK-NEXT:    [[V:%.*]] = select i1 [[X:%.*]], i53 512, i53 128
 ; CHECK-NEXT:    ret i53 [[V]]
 ;
   %A = select i1 %X, i8 3, i8 1
@@ -307,7 +307,7 @@ define i53 @test15a(i1 %X) {
 
 define i1 @test16(i84 %X) {
 ; CHECK-LABEL: @test16(
-; CHECK-NEXT:    [[AND:%.*]] = and i84 %X, 16
+; CHECK-NEXT:    [[AND:%.*]] = and i84 [[X:%.*]], 16
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i84 [[AND]], 0
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
@@ -331,7 +331,7 @@ define <2 x i1> @test16vec(<2 x i84> %X) {
 
 define i1 @test17(i106 %A) {
 ; CHECK-LABEL: @test17(
-; CHECK-NEXT:    [[B_MASK:%.*]] = and i106 %A, -8
+; CHECK-NEXT:    [[B_MASK:%.*]] = and i106 [[A:%.*]], -8
 ; CHECK-NEXT:    [[C:%.*]] = icmp eq i106 [[B_MASK]], 9872
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
@@ -342,7 +342,7 @@ define i1 @test17(i106 %A) {
 
 define <2 x i1> @test17vec(<2 x i106> %A) {
 ; CHECK-LABEL: @test17vec(
-; CHECK-NEXT:    [[B_MASK:%.*]] = and <2 x i106> %A, <i106 -8, i106 -8>
+; CHECK-NEXT:    [[B_MASK:%.*]] = and <2 x i106> [[A:%.*]], <i106 -8, i106 -8>
 ; CHECK-NEXT:    [[C:%.*]] = icmp eq <2 x i106> [[B_MASK]], <i106 9872, i106 9872>
 ; CHECK-NEXT:    ret <2 x i1> [[C]]
 ;
@@ -362,7 +362,7 @@ define i1 @test18(i11 %A) {
 
 define i1 @test19(i37 %A) {
 ; CHECK-LABEL: @test19(
-; CHECK-NEXT:    [[C:%.*]] = icmp ult i37 %A, 4
+; CHECK-NEXT:    [[C:%.*]] = icmp ult i37 [[A:%.*]], 4
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %B = ashr i37 %A, 2
@@ -372,7 +372,7 @@ define i1 @test19(i37 %A) {
 
 define <2 x i1> @test19vec(<2 x i37> %A) {
 ; CHECK-LABEL: @test19vec(
-; CHECK-NEXT:    [[C:%.*]] = icmp ult <2 x i37> %A, <i37 4, i37 4>
+; CHECK-NEXT:    [[C:%.*]] = icmp ult <2 x i37> [[A:%.*]], <i37 4, i37 4>
 ; CHECK-NEXT:    ret <2 x i1> [[C]]
 ;
   %B = ashr <2 x i37> %A, <i37 2, i37 2>
@@ -382,7 +382,7 @@ define <2 x i1> @test19vec(<2 x i37> %A) {
 
 define i1 @test19a(i39 %A) {
 ; CHECK-LABEL: @test19a(
-; CHECK-NEXT:    [[C:%.*]] = icmp ugt i39 %A, -5
+; CHECK-NEXT:    [[C:%.*]] = icmp ugt i39 [[A:%.*]], -5
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %B = ashr i39 %A, 2
@@ -392,7 +392,7 @@ define i1 @test19a(i39 %A) {
 
 define <2 x i1> @test19a_vec(<2 x i39> %A) {
 ; CHECK-LABEL: @test19a_vec(
-; CHECK-NEXT:    [[C:%.*]] = icmp ugt <2 x i39> %A, <i39 -5, i39 -5>
+; CHECK-NEXT:    [[C:%.*]] = icmp ugt <2 x i39> [[A:%.*]], <i39 -5, i39 -5>
 ; CHECK-NEXT:    ret <2 x i1> [[C]]
 ;
   %B = ashr <2 x i39> %A, <i39 2, i39 2>
@@ -411,7 +411,7 @@ define i1 @test20(i13 %A) {
 
 define i1 @test21(i12 %A) {
 ; CHECK-LABEL: @test21(
-; CHECK-NEXT:    [[B_MASK:%.*]] = and i12 %A, 63
+; CHECK-NEXT:    [[B_MASK:%.*]] = and i12 [[A:%.*]], 63
 ; CHECK-NEXT:    [[C:%.*]] = icmp eq i12 [[B_MASK]], 62
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
@@ -422,7 +422,7 @@ define i1 @test21(i12 %A) {
 
 define i1 @test22(i14 %A) {
 ; CHECK-LABEL: @test22(
-; CHECK-NEXT:    [[B_MASK:%.*]] = and i14 %A, 127
+; CHECK-NEXT:    [[B_MASK:%.*]] = and i14 [[A:%.*]], 127
 ; CHECK-NEXT:    [[C:%.*]] = icmp eq i14 [[B_MASK]], 0
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
@@ -433,7 +433,7 @@ define i1 @test22(i14 %A) {
 
 define i11 @test23(i44 %A) {
 ; CHECK-LABEL: @test23(
-; CHECK-NEXT:    [[D:%.*]] = trunc i44 %A to i11
+; CHECK-NEXT:    [[D:%.*]] = trunc i44 [[A:%.*]] to i11
 ; CHECK-NEXT:    ret i11 [[D]]
 ;
   %B = shl i44 %A, 33
@@ -446,8 +446,8 @@ define i11 @test23(i44 %A) {
 
 define i44 @shl_lshr_eq_amt_multi_use(i44 %A) {
 ; CHECK-LABEL: @shl_lshr_eq_amt_multi_use(
-; CHECK-NEXT:    [[B:%.*]] = shl i44 %A, 33
-; CHECK-NEXT:    [[C:%.*]] = and i44 %A, 2047
+; CHECK-NEXT:    [[B:%.*]] = shl i44 [[A:%.*]], 33
+; CHECK-NEXT:    [[C:%.*]] = and i44 [[A]], 2047
 ; CHECK-NEXT:    [[D:%.*]] = or i44 [[B]], [[C]]
 ; CHECK-NEXT:    ret i44 [[D]]
 ;
@@ -461,8 +461,8 @@ define i44 @shl_lshr_eq_amt_multi_use(i44 %A) {
 
 define <2 x i44> @shl_lshr_eq_amt_multi_use_splat_vec(<2 x i44> %A) {
 ; CHECK-LABEL: @shl_lshr_eq_amt_multi_use_splat_vec(
-; CHECK-NEXT:    [[B:%.*]] = shl <2 x i44> %A, <i44 33, i44 33>
-; CHECK-NEXT:    [[C:%.*]] = and <2 x i44> %A, <i44 2047, i44 2047>
+; CHECK-NEXT:    [[B:%.*]] = shl <2 x i44> [[A:%.*]], <i44 33, i44 33>
+; CHECK-NEXT:    [[C:%.*]] = and <2 x i44> [[A]], <i44 2047, i44 2047>
 ; CHECK-NEXT:    [[D:%.*]] = or <2 x i44> [[B]], [[C]]
 ; CHECK-NEXT:    ret <2 x i44> [[D]]
 ;
@@ -476,8 +476,8 @@ define <2 x i44> @shl_lshr_eq_amt_multi_use_splat_vec(<2 x i44> %A) {
 
 define i43 @lshr_shl_eq_amt_multi_use(i43 %A) {
 ; CHECK-LABEL: @lshr_shl_eq_amt_multi_use(
-; CHECK-NEXT:    [[B:%.*]] = lshr i43 %A, 23
-; CHECK-NEXT:    [[C:%.*]] = and i43 %A, -8388608
+; CHECK-NEXT:    [[B:%.*]] = lshr i43 [[A:%.*]], 23
+; CHECK-NEXT:    [[C:%.*]] = and i43 [[A]], -8388608
 ; CHECK-NEXT:    [[D:%.*]] = mul i43 [[B]], [[C]]
 ; CHECK-NEXT:    ret i43 [[D]]
 ;
@@ -491,8 +491,8 @@ define i43 @lshr_shl_eq_amt_multi_use(i43 %A) {
 
 define <2 x i43> @lshr_shl_eq_amt_multi_use_splat_vec(<2 x i43> %A) {
 ; CHECK-LABEL: @lshr_shl_eq_amt_multi_use_splat_vec(
-; CHECK-NEXT:    [[B:%.*]] = lshr <2 x i43> %A, <i43 23, i43 23>
-; CHECK-NEXT:    [[C:%.*]] = and <2 x i43> %A, <i43 -8388608, i43 -8388608>
+; CHECK-NEXT:    [[B:%.*]] = lshr <2 x i43> [[A:%.*]], <i43 23, i43 23>
+; CHECK-NEXT:    [[C:%.*]] = and <2 x i43> [[A]], <i43 -8388608, i43 -8388608>
 ; CHECK-NEXT:    [[D:%.*]] = mul <2 x i43> [[B]], [[C]]
 ; CHECK-NEXT:    ret <2 x i43> [[D]]
 ;
@@ -504,8 +504,8 @@ define <2 x i43> @lshr_shl_eq_amt_multi_use_splat_vec(<2 x i43> %A) {
 
 define i37 @test25(i37 %tmp.2, i37 %AA) {
 ; CHECK-LABEL: @test25(
-; CHECK-NEXT:    [[TMP_3:%.*]] = and i37 %tmp.2, -131072
-; CHECK-NEXT:    [[X2:%.*]] = add i37 [[TMP_3]], %AA
+; CHECK-NEXT:    [[TMP_3:%.*]] = and i37 [[TMP_2:%.*]], -131072
+; CHECK-NEXT:    [[X2:%.*]] = add i37 [[TMP_3]], [[AA:%.*]]
 ; CHECK-NEXT:    [[TMP_6:%.*]] = and i37 [[X2]], -131072
 ; CHECK-NEXT:    ret i37 [[TMP_6]]
 ;
@@ -518,7 +518,7 @@ define i37 @test25(i37 %tmp.2, i37 %AA) {
 
 define i40 @test26(i40 %A) {
 ; CHECK-LABEL: @test26(
-; CHECK-NEXT:    [[B:%.*]] = and i40 %A, -2
+; CHECK-NEXT:    [[B:%.*]] = and i40 [[A:%.*]], -2
 ; CHECK-NEXT:    ret i40 [[B]]
 ;
   %B = lshr i40 %A, 1
diff --git a/test/Transforms/InstCombine/atomicrmw.ll b/test/Transforms/InstCombine/atomicrmw.ll
new file mode 100644
index 0000000000000000000000000000000000000000..e9474f1685cbeef1e94e89b8dd32b24791dbfe33
--- /dev/null
+++ b/test/Transforms/InstCombine/atomicrmw.ll
@@ -0,0 +1,84 @@
+; RUN: opt -instcombine -S -o - %s | FileCheck %s
+; Check that we can replace `atomicrmw <op> LHS, 0` with `load atomic LHS`.
+; This is possible when:
+; - <op> LHS, 0 == LHS
+; - the ordering of atomicrmw is compatible with a load (i.e., no release semantic)
+
+; CHECK-LABEL: atomic_add_zero
+; CHECK-NEXT: %res = load atomic i32, i32* %addr monotonic, align 4
+; CHECK-NEXT: ret i32 %res
+define i32 @atomic_add_zero(i32* %addr) {
+  %res = atomicrmw add i32* %addr, i32 0 monotonic
+  ret i32 %res
+}
+
+; Don't transform volatile atomicrmw. This would eliminate a volatile store
+; otherwise.
+; CHECK-LABEL: atomic_sub_zero_volatile
+; CHECK-NEXT: %res = atomicrmw volatile sub i64* %addr, i64 0 acquire
+; CHECK-NEXT: ret i64 %res
+define i64 @atomic_sub_zero_volatile(i64* %addr) {
+  %res = atomicrmw volatile sub i64* %addr, i64 0 acquire
+  ret i64 %res
+}
+
+
+; Check that the transformation properly preserve the syncscope.
+; CHECK-LABEL: atomic_or_zero
+; CHECK-NEXT: %res = load atomic i16, i16* %addr syncscope("some_syncscope") acquire, align 2
+; CHECK-NEXT: ret i16 %res
+define i16 @atomic_or_zero(i16* %addr) {
+  %res = atomicrmw or i16* %addr, i16 0 syncscope("some_syncscope") acquire
+  ret i16 %res
+}
+
+; Don't transform seq_cst ordering.
+; By eliminating the store part of the atomicrmw, we would get rid of the
+; release semantic, which is incorrect.
+; CHECK-LABEL: atomic_or_zero_seq_cst
+; CHECK-NEXT: %res = atomicrmw or i16* %addr, i16 0 seq_cst
+; CHECK-NEXT: ret i16 %res
+define i16 @atomic_or_zero_seq_cst(i16* %addr) {
+  %res = atomicrmw or i16* %addr, i16 0 seq_cst
+  ret i16 %res
+}
+
+; Check that the transformation does not apply when the value is changed by
+; the atomic operation (non zero constant).
+; CHECK-LABEL: atomic_or_non_zero
+; CHECK-NEXT: %res = atomicrmw or i16* %addr, i16 2 monotonic
+; CHECK-NEXT: ret i16 %res
+define i16 @atomic_or_non_zero(i16* %addr) {
+  %res = atomicrmw or i16* %addr, i16 2 monotonic
+  ret i16 %res
+}
+
+; Check that the transformation does not apply when the value is changed by
+; the atomic operation (xor operation with zero).
+; CHECK-LABEL: atomic_xor_zero
+; CHECK-NEXT: %res = atomicrmw xor i16* %addr, i16 0 monotonic
+; CHECK-NEXT: ret i16 %res
+define i16 @atomic_xor_zero(i16* %addr) {
+  %res = atomicrmw xor i16* %addr, i16 0 monotonic
+  ret i16 %res
+}
+
+; Check that the transformation does not apply when the ordering is
+; incompatible with a load (release).
+; CHECK-LABEL: atomic_or_zero_release
+; CHECK-NEXT: %res = atomicrmw or i16* %addr, i16 0 release
+; CHECK-NEXT: ret i16 %res
+define i16 @atomic_or_zero_release(i16* %addr) {
+  %res = atomicrmw or i16* %addr, i16 0 release
+  ret i16 %res
+}
+
+; Check that the transformation does not apply when the ordering is
+; incompatible with a load (acquire, release).
+; CHECK-LABEL: atomic_or_zero_acq_rel
+; CHECK-NEXT: %res = atomicrmw or i16* %addr, i16 0 acq_rel
+; CHECK-NEXT: ret i16 %res
+define i16 @atomic_or_zero_acq_rel(i16* %addr) {
+  %res = atomicrmw or i16* %addr, i16 0 acq_rel
+  ret i16 %res
+}
diff --git a/test/Transforms/InstCombine/bitcast-store.ll b/test/Transforms/InstCombine/bitcast-store.ll
index 6d7952717550c67b5c13a7a20e40b7fe281006cc..2308d77be32ca5c6d5cb57b2e6c7e6d8b8d8f5e7 100644
--- a/test/Transforms/InstCombine/bitcast-store.ll
+++ b/test/Transforms/InstCombine/bitcast-store.ll
@@ -10,11 +10,11 @@ target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f3
 @G = external constant [5 x i8*]
 
 ; CHECK-LABEL: @foo
-; CHECK: store i32 %x, i32* %{{.*}}, align 16, !noalias !0
+; CHECK: store i32 %x, i32* %{{.*}}, align 16, !noalias !0, !llvm.access.group !1
 define void @foo(i32 %x, float* %p) nounwind {
 entry:
   %x.cast = bitcast i32 %x to float
-  store float %x.cast, float* %p, align 16, !noalias !0
+  store float %x.cast, float* %p, align 16, !noalias !0, !llvm.access.group !1
   ret void
 }
 
@@ -48,3 +48,4 @@ entry:
 }
 
 !0 = !{!0}
+!1 = !{}
\ No newline at end of file
diff --git a/test/Transforms/InstCombine/bitcount.ll b/test/Transforms/InstCombine/bitcount.ll
deleted file mode 100644
index 318ca7339f1aaf58518631258cdaff343ff80591..0000000000000000000000000000000000000000
--- a/test/Transforms/InstCombine/bitcount.ll
+++ /dev/null
@@ -1,19 +0,0 @@
-; Tests to make sure bit counts of constants are folded
-; RUN: opt < %s -instcombine -S | grep "ret i32 19"
-; RUN: opt < %s -instcombine -S | \
-; RUN:   grep -v declare | not grep llvm.ct
-
-declare i31 @llvm.ctpop.i31(i31 %val) 
-declare i32 @llvm.cttz.i32(i32 %val, i1) 
-declare i33 @llvm.ctlz.i33(i33 %val, i1) 
-
-define i32 @test(i32 %A) {
-  %c1 = call i31 @llvm.ctpop.i31(i31 12415124)
-  %c2 = call i32 @llvm.cttz.i32(i32 87359874, i1 true)
-  %c3 = call i33 @llvm.ctlz.i33(i33 87359874, i1 true)
-  %t1 = zext i31 %c1 to i32
-  %t3 = trunc i33 %c3 to i32
-  %r1 = add i32 %t1, %c2
-  %r2 = add i32 %r1, %t3
-  ret i32 %r2
-}
diff --git a/test/Transforms/InstCombine/bitreverse-hang.ll b/test/Transforms/InstCombine/bitreverse-hang.ll
index 6823bd0ed65333da40e86766282fc8f1bf23333b..8e6585e995d1ab47717197d3b44b38c113f70c2c 100644
--- a/test/Transforms/InstCombine/bitreverse-hang.ll
+++ b/test/Transforms/InstCombine/bitreverse-hang.ll
@@ -46,7 +46,7 @@ attributes #0 = { norecurse nounwind ssp uwtable "disable-tail-calls"="false" "l
 !llvm.ident = !{!1}
 
 !0 = !{i32 1, !"PIC Level", i32 2}
-!1 = !{!"clang version 3.8.0 (http://llvm.org/git/clang.git eb70f4e9cc9a4dc3dd57b032fb858d56b4b64a0e)"}
+!1 = !{!"clang version 3.8.0"}
 !2 = !{!3, !3, i64 0}
 !3 = !{!"int", !4, i64 0}
 !4 = !{!"omnipotent char", !5, i64 0}
diff --git a/test/Transforms/InstCombine/builtin-dynamic-object-size.ll b/test/Transforms/InstCombine/builtin-dynamic-object-size.ll
new file mode 100644
index 0000000000000000000000000000000000000000..a0c2d139916395d7b00881a0e680f9de9d3ba707
--- /dev/null
+++ b/test/Transforms/InstCombine/builtin-dynamic-object-size.ll
@@ -0,0 +1,83 @@
+; RUN: opt -instcombine -S < %s | FileCheck %s --dump-input-on-failure
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.14.0"
+
+; Function Attrs: nounwind ssp uwtable
+define i64 @weird_identity_but_ok(i64 %sz) {
+entry:
+  %call = tail call i8* @malloc(i64 %sz)
+  %calc_size = tail call i64 @llvm.objectsize.i64.p0i8(i8* %call, i1 false, i1 true, i1 true)
+  tail call void @free(i8* %call)
+  ret i64 %calc_size
+}
+
+; CHECK:      define i64 @weird_identity_but_ok(i64 %sz)
+; CHECK-NEXT: entry:
+; CHECK-NEXT:   ret i64 %sz
+; CHECK-NEXT: }
+
+define i64 @phis_are_neat(i1 %which) {
+entry:
+  br i1 %which, label %first_label, label %second_label
+
+first_label:
+  %first_call = call i8* @malloc(i64 10)
+  br label %join_label
+
+second_label:
+  %second_call = call i8* @malloc(i64 30)
+  br label %join_label
+
+join_label:
+  %joined = phi i8* [ %first_call, %first_label ], [ %second_call, %second_label ]
+  %calc_size = tail call i64 @llvm.objectsize.i64.p0i8(i8* %joined, i1 false, i1 true, i1 true)
+  ret i64 %calc_size
+}
+
+; CHECK:      %0 = phi i64 [ 10, %first_label ], [ 30, %second_label ]
+; CHECK-NEXT: ret i64 %0
+
+define i64 @internal_pointer(i64 %sz) {
+entry:
+  %ptr = call i8* @malloc(i64 %sz)
+  %ptr2 = getelementptr inbounds i8, i8* %ptr, i32 2
+  %calc_size = call i64 @llvm.objectsize.i64.p0i8(i8* %ptr2, i1 false, i1 true, i1 true)
+  ret i64 %calc_size
+}
+
+; CHECK:      define i64 @internal_pointer(i64 %sz)
+; CHECK-NEXT: entry:
+; CHECK-NEXT:   %0 = add i64 %sz, -2
+; CHECK-NEXT:   %1 = icmp ult i64 %sz, 2
+; CHECK-NEXT:   %2 = select i1 %1, i64 0, i64 %0
+; CHECK-NEXT:   ret i64 %2
+; CHECK-NEXT: }
+
+define i64 @uses_nullptr_no_fold() {
+entry:
+  %res = call i64 @llvm.objectsize.i64.p0i8(i8* null, i1 false, i1 true, i1 true)
+  ret i64 %res
+}
+
+; CHECK: %res = call i64 @llvm.objectsize.i64.p0i8(i8* null, i1 false, i1 true, i1 true)
+
+define i64 @uses_nullptr_fold() {
+entry:
+  ; NOTE: the third parameter to this call is false, unlike above.
+  %res = call i64 @llvm.objectsize.i64.p0i8(i8* null, i1 false, i1 false, i1 true)
+  ret i64 %res
+}
+
+; CHECK: ret i64 0
+
+; Function Attrs: nounwind allocsize(0)
+declare i8* @malloc(i64)
+
+declare i8* @get_unknown_buffer()
+
+; Function Attrs: nounwind
+declare void @free(i8* nocapture)
+
+; Function Attrs: nounwind readnone speculatable
+declare i64 @llvm.objectsize.i64.p0i8(i8*, i1, i1, i1)
diff --git a/test/Transforms/InstCombine/callsite_nonnull_args_through_casts.ll b/test/Transforms/InstCombine/callsite_nonnull_args_through_casts.ll
new file mode 100644
index 0000000000000000000000000000000000000000..b7a1d1d3fb7f0835d115f76d2fc0a9a275a0752f
--- /dev/null
+++ b/test/Transforms/InstCombine/callsite_nonnull_args_through_casts.ll
@@ -0,0 +1,99 @@
+; RUN: opt -instcombine -S < %s | FileCheck %s
+;
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+declare void @foo(i8*)
+declare void @bar(i8 addrspace(1)*)
+
+define void @nonnullAfterBitCast() {
+entry:
+  %i = alloca i32, align 4
+  %tmp1 = bitcast i32* %i to i8*
+; CHECK: call void @foo(i8* nonnull %tmp1)
+  call void @foo(i8* %tmp1)
+  ret void
+}
+
+define void @nonnullAfterSExt(i8 %a) {
+entry:
+  %b = zext i8 %a to i32              ; <- %b is >= 0
+  %c = add nsw nuw i32 %b, 2          ; <- %c is > 0
+  %sext = sext i32 %c to i64          ; <- %sext cannot be 0 because %c is not 0
+  %i2p = inttoptr i64 %sext to i8*    ; <- no-op int2ptr cast
+; CHECK: call void @foo(i8* nonnull %i2p)
+  call void @foo(i8* %i2p)
+  ret void
+}
+
+define void @nonnullAfterZExt(i8 %a) {
+entry:
+  %b = zext i8 %a to i32              ; <- %b is >= 0
+  %c = add nsw nuw i32 %b, 2          ; <- %c is > 0
+  %zext = zext i32 %c to i64          ; <- %zext cannot be 0 because %c is not 0
+  %i2p = inttoptr i64 %zext to i8*    ; <- no-op int2ptr cast
+; CHECK: call void @foo(i8* nonnull %i2p)
+  call void @foo(i8* %i2p)
+  ret void
+}
+
+declare void @llvm.assume(i1 %b)
+
+define void @nonnullAfterInt2Ptr(i32 %u, i64 %lu) {
+entry:
+  %nz = sdiv exact i32 100, %u         ; %nz cannot be null
+  %i2p = inttoptr i32 %nz to i8*       ; extending int2ptr as sizeof(i32) < sizeof(i8*)
+; CHECK:  call void @foo(i8* nonnull %i2p)
+  call void @foo(i8* %i2p)
+
+  %nz.2 = sdiv exact i64 100, %lu      ; %nz.2 cannot be null
+  %i2p.2 = inttoptr i64 %nz.2 to i8*   ; no-op int2ptr as sizeof(i64) == sizeof(i8*)
+; CHECK:  call void @foo(i8* nonnull %i2p.2)
+  call void @foo(i8* %i2p.2)
+  ret void
+}
+
+define void @nonnullAfterPtr2Int() {
+entry:
+  %a = alloca i32
+  %p2i = ptrtoint i32* %a to i64      ; no-op ptr2int as sizeof(i32*) == sizeof(i64)
+  %i2p = inttoptr i64 %p2i to i8*
+; CHECK:  call void @foo(i8* nonnull %i2p)
+  call void @foo(i8* %i2p)
+  ret void
+}
+
+define void @maybenullAfterInt2Ptr(i128 %llu) {
+entry:
+  %cmp = icmp ne i128 %llu, 0
+  call void @llvm.assume(i1 %cmp)          ; %llu != 0
+  %i2p = inttoptr i128 %llu to i8*    ; truncating int2ptr as sizeof(i128) > sizeof(i8*)
+; CHECK:  call void @foo(i8* %i2p)
+  call void @foo(i8* %i2p)
+  ret void
+}
+
+define void @maybenullAfterPtr2Int() {
+entry:
+  %a = alloca i32
+  %p2i = ptrtoint i32* %a to i32      ; truncating ptr2int as sizeof(i32*) > sizeof(i32)
+  %i2p = inttoptr i32 %p2i to i8*
+; CHECK:  call void @foo(i8* %i2p)
+  call void @foo(i8* %i2p)
+  ret void
+}
+
+define void @maybenullAfterAddrspacecast(i8* nonnull %p) {
+entry:
+  %addrspcast = addrspacecast i8* %p to i8 addrspace(1)*
+
+; An address space cast can be "a no-op cast or a complex value modification,
+; depending on the target and the address space pair". As a consequence, we
+; cannot simply assume non-nullness of %p is preserved by the cast.
+;
+; CHECK:  call void @bar(i8 addrspace(1)* %addrspcast)
+  call void @bar(i8 addrspace(1)* %addrspcast)
+
+; CHECK:  call void @foo(i8* nonnull %p)
+  call void @foo(i8* %p)
+  ret void
+}
diff --git a/test/Transforms/InstCombine/cast-int-icmp-eq-0.ll b/test/Transforms/InstCombine/cast-int-icmp-eq-0.ll
index 9576c5c20ba0d7350051d1bc48edc5665198f763..f18bfe7531c25edb19923d8934eb25d797971561 100644
--- a/test/Transforms/InstCombine/cast-int-icmp-eq-0.ll
+++ b/test/Transforms/InstCombine/cast-int-icmp-eq-0.ll
@@ -651,3 +651,59 @@ define <3 x i1> @i16_cast_cmp_sgt_int_m1_sitofp_float_vec_undef(<3 x i16> %i) {
   ret <3 x i1> %cmp
 }
 
+; Verify that the various forms of this transform are not applied when the
+; bitcast changes the number of vector elements:
+;   icmp (bitcast ([su]itofp X)), Y -> icmp X, Y
+
+define <6 x i1> @i16_cast_cmp_sgt_int_m1_bitcast_vector_num_elements_sitofp(<3 x i16> %i) {
+; CHECK-LABEL: @i16_cast_cmp_sgt_int_m1_bitcast_vector_num_elements_sitofp(
+; CHECK-NEXT:    [[F:%.*]] = sitofp <3 x i16> [[I:%.*]] to <3 x float>
+; CHECK-NEXT:    [[B:%.*]] = bitcast <3 x float> [[F]] to <6 x i16>
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt <6 x i16> [[B]], <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>
+; CHECK-NEXT:    ret <6 x i1> [[CMP]]
+;
+  %f = sitofp <3 x i16> %i to  <3 x float>
+  %b = bitcast <3 x float> %f to <6 x i16>
+  %cmp = icmp sgt <6 x i16> %b, <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>
+  ret <6 x i1> %cmp
+}
+
+define i1 @i16_cast_cmp_sgt_int_m1_bitcast_vector_to_scalar_sitofp(<3 x i16> %i) {
+; CHECK-LABEL: @i16_cast_cmp_sgt_int_m1_bitcast_vector_to_scalar_sitofp(
+; CHECK-NEXT:    [[F:%.*]] = sitofp <3 x i16> [[I:%.*]] to <3 x float>
+; CHECK-NEXT:    [[B:%.*]] = bitcast <3 x float> [[F]] to i96
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i96 [[B]], -1
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %f = sitofp <3 x i16> %i to  <3 x float>
+  %b = bitcast <3 x float> %f to i96
+  %cmp = icmp sgt i96 %b, -1
+  ret i1 %cmp
+}
+
+
+define <6 x i1> @i16_cast_cmp_eq_int_0_bitcast_vector_num_elements_uitofp(<3 x i16> %i) {
+; CHECK-LABEL: @i16_cast_cmp_eq_int_0_bitcast_vector_num_elements_uitofp(
+; CHECK-NEXT:    [[F:%.*]] = uitofp <3 x i16> [[I:%.*]] to <3 x float>
+; CHECK-NEXT:    [[B:%.*]] = bitcast <3 x float> [[F]] to <6 x i16>
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <6 x i16> [[B]], zeroinitializer
+; CHECK-NEXT:    ret <6 x i1> [[CMP]]
+;
+  %f = uitofp <3 x i16> %i to <3 x float>
+  %b = bitcast <3 x float> %f to <6 x i16>
+  %cmp = icmp eq <6 x i16> %b, <i16 0, i16 0, i16 0, i16 0, i16 0, i16 0>
+  ret <6 x i1> %cmp
+}
+
+define i1 @i16_cast_cmp_eq_int_0_bitcast_vector_to_scalar_uitofp(<3 x i16> %i) {
+; CHECK-LABEL: @i16_cast_cmp_eq_int_0_bitcast_vector_to_scalar_uitofp(
+; CHECK-NEXT:    [[F:%.*]] = uitofp <3 x i16> [[I:%.*]] to <3 x float>
+; CHECK-NEXT:    [[B:%.*]] = bitcast <3 x float> [[F]] to i96
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i96 [[B]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %f = uitofp <3 x i16> %i to <3 x float>
+  %b = bitcast <3 x float> %f to i96
+  %cmp = icmp eq i96 %b, 0
+  ret i1 %cmp
+}
diff --git a/test/Transforms/InstCombine/cast_phi.ll b/test/Transforms/InstCombine/cast_phi.ll
new file mode 100644
index 0000000000000000000000000000000000000000..141ad186002d8c1facdd76d4a7de4a3ff1b2849c
--- /dev/null
+++ b/test/Transforms/InstCombine/cast_phi.ll
@@ -0,0 +1,135 @@
+; RUN: opt < %s -instcombine -S | FileCheck %s
+; RUN: opt < %s -passes=instcombine -S | FileCheck %s
+
+define void @MainKernel(i32 %iNumSteps, i32 %tid, i32 %base) {
+; CHECK-NOT: bitcast
+
+  %callA = alloca [258 x float], align 4
+  %callB = alloca [258 x float], align 4
+  %conv.i = uitofp i32 %iNumSteps to float
+  %1 = bitcast float %conv.i to i32
+  %conv.i12 = zext i32 %tid to i64
+  %arrayidx3 = getelementptr inbounds [258 x float], [258 x float]* %callA, i64 0, i64 %conv.i12
+  %2 = bitcast float* %arrayidx3 to i32*
+  store i32 %1, i32* %2, align 4
+  %arrayidx6 = getelementptr inbounds [258 x float], [258 x float]* %callB, i64 0, i64 %conv.i12
+  %3 = bitcast float* %arrayidx6 to i32*
+  store i32 %1, i32* %3, align 4
+  %cmp7 = icmp eq i32 %tid, 0
+  br i1 %cmp7, label %.bb1, label %.bb2
+
+.bb1:
+  %arrayidx10 = getelementptr inbounds [258 x float], [258 x float]* %callA, i64 0, i64 256
+  store float %conv.i, float* %arrayidx10, align 4
+  %arrayidx11 = getelementptr inbounds [258 x float], [258 x float]* %callB, i64 0, i64 256
+  store float 0.000000e+00, float* %arrayidx11, align 4
+  br label %.bb2
+
+.bb2:
+  %cmp135 = icmp sgt i32 %iNumSteps, 0
+  br i1 %cmp135, label %.bb3, label %.bb8
+
+; CHECK-LABEL: .bb3
+; CHECK: phi float
+; CHECK: phi float
+; CHECK: phi i32 {{.*}} [ %iNumSteps
+; CHECK-NOT: rA.sroa.[0-9].[0-9] = phi i32
+; CHECK-NOT: phi float
+; CHECK-NOT: phi i32
+; CHECK-LABEL: .bb4
+
+.bb3:
+  %rA.sroa.8.0 = phi i32 [ %rA.sroa.8.2, %.bb12 ], [ %1, %.bb2 ]
+  %rA.sroa.0.0 = phi i32 [ %rA.sroa.0.2, %.bb12 ], [ %1, %.bb2 ]
+  %i12.06 = phi i32 [ %sub, %.bb12 ], [ %iNumSteps, %.bb2 ]
+  %4 = icmp ugt i32 %i12.06, %base
+  %add = add i32 %i12.06, 1
+  %conv.i9 = sext i32 %add to i64
+  %arrayidx20 = getelementptr inbounds [258 x float], [258 x float]* %callA, i64 0, i64 %conv.i9
+  %5 = bitcast float* %arrayidx20 to i32*
+  %arrayidx24 = getelementptr inbounds [258 x float], [258 x float]* %callB, i64 0, i64 %conv.i9
+  %6 = bitcast float* %arrayidx24 to i32*
+  %cmp40 = icmp ult i32 %i12.06, %base
+  br i1 %4, label %.bb4, label %.bb5
+
+.bb4:
+  %7 = load i32, i32* %5, align 4
+  %8 = load i32, i32* %6, align 4
+  %9 = bitcast i32 %8 to float
+  %10 = bitcast i32 %7 to float
+  %add33 = fadd float %9, %10
+  %11 = bitcast i32 %rA.sroa.8.0 to float
+  %add33.1 = fadd float %add33, %11
+  %12 = bitcast float %add33.1 to i32
+  %13 = bitcast i32 %rA.sroa.0.0 to float
+  %add33.2 = fadd float %add33.1, %13
+  %14 = bitcast float %add33.2 to i32
+  br label %.bb5
+
+; CHECK-LABEL: .bb5
+; CHECK: phi float
+; CHECK: phi float
+; CHECK-NOT: rA.sroa.[0-9].[0-9] = phi i32
+; CHECK-NOT: phi float
+; CHECK-NOT: phi i32
+; CHECK-LABEL: .bb6
+
+.bb5:
+  %rA.sroa.8.1 = phi i32 [ %12, %.bb4 ], [ %rA.sroa.8.0, %.bb3 ]
+  %rA.sroa.0.1 = phi i32 [ %14, %.bb4 ], [ %rA.sroa.0.0, %.bb3 ]
+  br i1 %cmp40, label %.bb6, label %.bb7
+
+.bb6:
+  store i32 %rA.sroa.0.1, i32* %2, align 4
+  store i32 %rA.sroa.8.1, i32* %3, align 4
+  br label %.bb7
+
+.bb7:
+  br i1 %4, label %.bb9, label %.bb10
+
+.bb8:
+  ret void
+
+.bb9:
+  %15 = load i32, i32* %5, align 4
+  %16 = load i32, i32* %6, align 4
+  %17 = bitcast i32 %16 to float
+  %18 = bitcast i32 %15 to float
+  %add33.112 = fadd float %17, %18
+  %19 = bitcast i32 %rA.sroa.8.1 to float
+  %add33.1.1 = fadd float %add33.112, %19
+  %20 = bitcast float %add33.1.1 to i32
+  %21 = bitcast i32 %rA.sroa.0.1 to float
+  %add33.2.1 = fadd float %add33.1.1, %21
+  %22 = bitcast float %add33.2.1 to i32
+  br label %.bb10
+
+; CHECK-LABEL: .bb10
+; CHECK: phi float
+; CHECK: phi float
+; CHECK-NOT: rA.sroa.[0-9].[0-9] = phi i32
+; CHECK-NOT: phi float
+; CHECK-NOT: phi i32
+; CHECK-LABEL: .bb11
+
+.bb10:
+  %rA.sroa.8.2 = phi i32 [ %20, %.bb9 ], [ %rA.sroa.8.1, %.bb7 ]
+  %rA.sroa.0.2 = phi i32 [ %22, %.bb9 ], [ %rA.sroa.0.1, %.bb7 ]
+  br i1 %cmp40, label %.bb11, label %.bb12
+
+; CHECK-LABEL: .bb11
+; CHECK: store float
+; CHECK: store float
+; CHECK-NOT: store i32 %rA.sroa.[0-9].[0-9]
+; CHECK-LABEL: .bb12
+
+.bb11:
+  store i32 %rA.sroa.0.2, i32* %2, align 4
+  store i32 %rA.sroa.8.2, i32* %3, align 4
+  br label %.bb12
+
+.bb12:
+  %sub = add i32 %i12.06, -4
+  %cmp13 = icmp sgt i32 %sub, 0
+  br i1 %cmp13, label %.bb3, label %.bb8
+}
diff --git a/test/Transforms/InstCombine/cmp-intrinsic.ll b/test/Transforms/InstCombine/cmp-intrinsic.ll
index 1ea608d5945b79207aec7ac93a95ca6cc4cf3696..82b32ee1b7f3e85a8684d874c60ead9fb335ebc7 100644
--- a/test/Transforms/InstCombine/cmp-intrinsic.ll
+++ b/test/Transforms/InstCombine/cmp-intrinsic.ll
@@ -137,6 +137,112 @@ define <2 x i1> @ctlz_ne_bitwidth_v2i32(<2 x i32> %a) {
   ret <2 x i1> %cmp
 }
 
+define i1 @ctlz_ugt_zero_i32(i32 %x) {
+; CHECK-LABEL: @ctlz_ugt_zero_i32(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[X:%.*]], -1
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %lz = tail call i32 @llvm.ctlz.i32(i32 %x, i1 false)
+  %cmp = icmp ugt i32 %lz, 0
+  ret i1 %cmp
+}
+
+define i1 @ctlz_ugt_one_i32(i32 %x) {
+; CHECK-LABEL: @ctlz_ugt_one_i32(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[X:%.*]], 1073741824
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %lz = tail call i32 @llvm.ctlz.i32(i32 %x, i1 false)
+  %cmp = icmp ugt i32 %lz, 1
+  ret i1 %cmp
+}
+
+define i1 @ctlz_ugt_other_i32(i32 %x) {
+; CHECK-LABEL: @ctlz_ugt_other_i32(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[X:%.*]], 32768
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %lz = tail call i32 @llvm.ctlz.i32(i32 %x, i1 false)
+  %cmp = icmp ugt i32 %lz, 16
+  ret i1 %cmp
+}
+
+define i1 @ctlz_ugt_other_multiuse_i32(i32 %x, i32* %p) {
+; CHECK-LABEL: @ctlz_ugt_other_multiuse_i32(
+; CHECK-NEXT:    [[LZ:%.*]] = tail call i32 @llvm.ctlz.i32(i32 [[X:%.*]], i1 false), !range !0
+; CHECK-NEXT:    store i32 [[LZ]], i32* [[P:%.*]], align 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[X]], 32768
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %lz = tail call i32 @llvm.ctlz.i32(i32 %x, i1 false)
+  store i32 %lz, i32* %p
+  %cmp = icmp ugt i32 %lz, 16
+  ret i1 %cmp
+}
+
+define i1 @ctlz_ugt_bw_minus_one_i32(i32 %x) {
+; CHECK-LABEL: @ctlz_ugt_bw_minus_one_i32(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[X:%.*]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %lz = tail call i32 @llvm.ctlz.i32(i32 %x, i1 false)
+  %cmp = icmp ugt i32 %lz, 31
+  ret i1 %cmp
+}
+
+define <2 x i1> @ctlz_ult_one_v2i32(<2 x i32> %x) {
+; CHECK-LABEL: @ctlz_ult_one_v2i32(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt <2 x i32> [[X:%.*]], zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %lz = tail call <2 x i32> @llvm.ctlz.v2i32(<2 x i32> %x, i1 false)
+  %cmp = icmp ult <2 x i32> %lz, <i32 1, i32 1>
+  ret <2 x i1> %cmp
+}
+
+define <2 x i1> @ctlz_ult_other_v2i32(<2 x i32> %x) {
+; CHECK-LABEL: @ctlz_ult_other_v2i32(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt <2 x i32> [[X:%.*]], <i32 65535, i32 65535>
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %lz = tail call <2 x i32> @llvm.ctlz.v2i32(<2 x i32> %x, i1 false)
+  %cmp = icmp ult <2 x i32> %lz, <i32 16, i32 16>
+  ret <2 x i1> %cmp
+}
+
+define <2 x i1> @ctlz_ult_other_multiuse_v2i32(<2 x i32> %x, <2 x i32>* %p) {
+; CHECK-LABEL: @ctlz_ult_other_multiuse_v2i32(
+; CHECK-NEXT:    [[LZ:%.*]] = tail call <2 x i32> @llvm.ctlz.v2i32(<2 x i32> [[X:%.*]], i1 false)
+; CHECK-NEXT:    store <2 x i32> [[LZ]], <2 x i32>* [[P:%.*]], align 8
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt <2 x i32> [[X]], <i32 65535, i32 65535>
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %lz = tail call <2 x i32> @llvm.ctlz.v2i32(<2 x i32> %x, i1 false)
+  store <2 x i32> %lz, <2 x i32>* %p
+  %cmp = icmp ult <2 x i32> %lz, <i32 16, i32 16>
+  ret <2 x i1> %cmp
+}
+
+define <2 x i1> @ctlz_ult_bw_minus_one_v2i32(<2 x i32> %x) {
+; CHECK-LABEL: @ctlz_ult_bw_minus_one_v2i32(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt <2 x i32> [[X:%.*]], <i32 1, i32 1>
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %lz = tail call <2 x i32> @llvm.ctlz.v2i32(<2 x i32> %x, i1 false)
+  %cmp = icmp ult <2 x i32> %lz, <i32 31, i32 31>
+  ret <2 x i1> %cmp
+}
+
+define <2 x i1> @ctlz_ult_bitwidth_v2i32(<2 x i32> %x) {
+; CHECK-LABEL: @ctlz_ult_bitwidth_v2i32(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne <2 x i32> [[X:%.*]], zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %lz = tail call <2 x i32> @llvm.ctlz.v2i32(<2 x i32> %x, i1 false)
+  %cmp = icmp ult <2 x i32> %lz, <i32 32, i32 32>
+  ret <2 x i1> %cmp
+}
+
 define i1 @cttz_ne_bitwidth_i33(i33 %x) {
 ; CHECK-LABEL: @cttz_ne_bitwidth_i33(
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i33 [[X:%.*]], 0
@@ -223,17 +329,128 @@ define <2 x i1> @cttz_ne_other_v2i32(<2 x i32> %a) {
 
 define i1 @cttz_eq_other_i33_multiuse(i33 %x, i33* %p) {
 ; CHECK-LABEL: @cttz_eq_other_i33_multiuse(
-; CHECK-NEXT:    [[LZ:%.*]] = tail call i33 @llvm.cttz.i33(i33 [[X:%.*]], i1 false), !range !1
-; CHECK-NEXT:    store i33 [[LZ]], i33* [[P:%.*]], align 4
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i33 [[LZ]], 4
+; CHECK-NEXT:    [[TZ:%.*]] = tail call i33 @llvm.cttz.i33(i33 [[X:%.*]], i1 false), !range !1
+; CHECK-NEXT:    store i33 [[TZ]], i33* [[P:%.*]], align 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i33 [[TZ]], 4
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
-  %lz = tail call i33 @llvm.cttz.i33(i33 %x, i1 false)
-  store i33 %lz, i33* %p
-  %cmp = icmp eq i33 %lz, 4
+  %tz = tail call i33 @llvm.cttz.i33(i33 %x, i1 false)
+  store i33 %tz, i33* %p
+  %cmp = icmp eq i33 %tz, 4
+  ret i1 %cmp
+}
+
+define i1 @cttz_ugt_zero_i33(i33 %x) {
+; CHECK-LABEL: @cttz_ugt_zero_i33(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i33 [[X:%.*]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i33 [[TMP1]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %tz = tail call i33 @llvm.cttz.i33(i33 %x, i1 false)
+  %cmp = icmp ugt i33 %tz, 0
+  ret i1 %cmp
+}
+
+define i1 @cttz_ugt_one_i33(i33 %x) {
+; CHECK-LABEL: @cttz_ugt_one_i33(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i33 [[X:%.*]], 3
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i33 [[TMP1]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %tz = tail call i33 @llvm.cttz.i33(i33 %x, i1 false)
+  %cmp = icmp ugt i33 %tz, 1
   ret i1 %cmp
 }
 
+define i1 @cttz_ugt_other_i33(i33 %x) {
+; CHECK-LABEL: @cttz_ugt_other_i33(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i33 [[X:%.*]], 131071
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i33 [[TMP1]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %tz = tail call i33 @llvm.cttz.i33(i33 %x, i1 false)
+  %cmp = icmp ugt i33 %tz, 16
+  ret i1 %cmp
+}
+
+define i1 @cttz_ugt_other_multiuse_i33(i33 %x, i33* %p) {
+; CHECK-LABEL: @cttz_ugt_other_multiuse_i33(
+; CHECK-NEXT:    [[TZ:%.*]] = tail call i33 @llvm.cttz.i33(i33 [[X:%.*]], i1 false), !range !1
+; CHECK-NEXT:    store i33 [[TZ]], i33* [[P:%.*]], align 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i33 [[TZ]], 16
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %tz = tail call i33 @llvm.cttz.i33(i33 %x, i1 false)
+  store i33 %tz, i33* %p
+  %cmp = icmp ugt i33 %tz, 16
+  ret i1 %cmp
+}
+
+define i1 @cttz_ugt_bw_minus_one_i33(i33 %x) {
+; CHECK-LABEL: @cttz_ugt_bw_minus_one_i33(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i33 [[X:%.*]], 0
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %tz = tail call i33 @llvm.cttz.i33(i33 %x, i1 false)
+  %cmp = icmp ugt i33 %tz, 32
+  ret i1 %cmp
+}
+
+define <2 x i1> @cttz_ult_one_v2i32(<2 x i32> %x) {
+; CHECK-LABEL: @cttz_ult_one_v2i32(
+; CHECK-NEXT:    [[CMP:%.*]] = trunc <2 x i32> [[X:%.*]] to <2 x i1>
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %tz = tail call <2 x i32> @llvm.cttz.v2i32(<2 x i32> %x, i1 false)
+  %cmp = icmp ult <2 x i32> %tz, <i32 1, i32 1>
+  ret <2 x i1> %cmp
+}
+
+define <2 x i1> @cttz_ult_other_v2i32(<2 x i32> %x) {
+; CHECK-LABEL: @cttz_ult_other_v2i32(
+; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i32> [[X:%.*]], <i32 65535, i32 65535>
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne <2 x i32> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %tz = tail call <2 x i32> @llvm.cttz.v2i32(<2 x i32> %x, i1 false)
+  %cmp = icmp ult <2 x i32> %tz, <i32 16, i32 16>
+  ret <2 x i1> %cmp
+}
+
+define <2 x i1> @cttz_ult_other_multiuse_v2i32(<2 x i32> %x, <2 x i32>* %p) {
+; CHECK-LABEL: @cttz_ult_other_multiuse_v2i32(
+; CHECK-NEXT:    [[TZ:%.*]] = tail call <2 x i32> @llvm.cttz.v2i32(<2 x i32> [[X:%.*]], i1 false)
+; CHECK-NEXT:    store <2 x i32> [[TZ]], <2 x i32>* [[P:%.*]], align 8
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult <2 x i32> [[TZ]], <i32 16, i32 16>
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %tz = tail call <2 x i32> @llvm.cttz.v2i32(<2 x i32> %x, i1 false)
+  store <2 x i32> %tz, <2 x i32>* %p
+  %cmp = icmp ult <2 x i32> %tz, <i32 16, i32 16>
+  ret <2 x i1> %cmp
+}
+
+define <2 x i1> @cttz_ult_bw_minus_one_v2i32(<2 x i32> %x) {
+; CHECK-LABEL: @cttz_ult_bw_minus_one_v2i32(
+; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i32> [[X:%.*]], <i32 2147483647, i32 2147483647>
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne <2 x i32> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %tz = tail call <2 x i32> @llvm.cttz.v2i32(<2 x i32> %x, i1 false)
+  %cmp = icmp ult <2 x i32> %tz, <i32 31, i32 31>
+  ret <2 x i1> %cmp
+}
+
+define <2 x i1> @cttz_ult_bitwidth_v2i32(<2 x i32> %x) {
+; CHECK-LABEL: @cttz_ult_bitwidth_v2i32(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne <2 x i32> [[X:%.*]], zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %tz = tail call <2 x i32> @llvm.cttz.v2i32(<2 x i32> %x, i1 false)
+  %cmp = icmp ult <2 x i32> %tz, <i32 32, i32 32>
+  ret <2 x i1> %cmp
+}
+
 define i1 @ctpop_eq_zero_i11(i11 %x) {
 ; CHECK-LABEL: @ctpop_eq_zero_i11(
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i11 [[X:%.*]], 0
diff --git a/test/Transforms/InstCombine/debuginfo-dce.ll b/test/Transforms/InstCombine/debuginfo-dce.ll
index d23cc1e9952ea24ce31e5728335358025c2888b0..200ea26cdaf90bea10d1c0ab969e95c2fab7eaea 100644
--- a/test/Transforms/InstCombine/debuginfo-dce.ll
+++ b/test/Transforms/InstCombine/debuginfo-dce.ll
@@ -23,6 +23,9 @@ target triple = "x86_64-apple-macosx10.12.0"
 
 %struct.entry = type { %struct.entry* }
 
+; This salvage can't currently occur safely (PR40628), however if/when that's
+; ever fixed, then this is definitely a piece of test coverage that should
+; be maintained.
 define void @salvage_load(%struct.entry** %queue) local_unnamed_addr #0 !dbg !14 {
 entry:
   %im_not_dead = alloca %struct.entry*
@@ -31,8 +34,7 @@ entry:
   call void @llvm.dbg.value(metadata %struct.entry* %1, metadata !18, metadata !20), !dbg !19
 ; CHECK: define void @salvage_load
 ; CHECK-NEXT: entry:
-; CHECK-NEXT: call void @llvm.dbg.value(metadata %struct.entry** %queue,
-; CHECK-SAME:                           metadata !DIExpression(DW_OP_deref, DW_OP_plus_uconst, 0))
+; CHECK-NOT: dbg.value
   store %struct.entry* %1, %struct.entry** %im_not_dead, align 8
   ret void, !dbg !21
 }
diff --git a/test/Transforms/InstCombine/debuginfo-sink.ll b/test/Transforms/InstCombine/debuginfo-sink.ll
new file mode 100644
index 0000000000000000000000000000000000000000..1e6277748faea42f19196e652874f3df4cd8c762
--- /dev/null
+++ b/test/Transforms/InstCombine/debuginfo-sink.ll
@@ -0,0 +1,78 @@
+; RUN: opt  %s -instcombine -S | FileCheck %s
+
+; Test sinking of dbg.values when instcombine sinks associated instructions.
+
+declare void @llvm.dbg.value(metadata, metadata, metadata)
+
+; This GEP is sunk, but can be folded into a DIExpression. Check that it
+; gets folded. The dbg.value should be duplicated in the block its sunk
+; into, to maximise liveness.
+;
+; CHECK-LABEL: define i32 @foo(i32*
+; CHECK:       call void @llvm.dbg.value(metadata i32* %a, metadata !{{[0-9]+}},
+; CHECK-SAME:  metadata !DIExpression(DW_OP_plus_uconst, 4, DW_OP_stack_value))
+; CHECK-NEXT:  br label %sink1
+
+define i32 @foo(i32 *%a) !dbg !7 {
+entry:
+  %gep = getelementptr i32, i32 *%a, i32 1
+  call void @llvm.dbg.value(metadata i32 *%gep, metadata !16, metadata !12), !dbg !15
+  br label %sink1
+
+sink1:
+; CHECK-LABEL: sink1:
+; CHECK:       call void @llvm.dbg.value(metadata i32* %gep,
+; CHECK-SAME:                    metadata !{{[0-9]+}}, metadata !DIExpression())
+; CHECK-NEXT:  load
+  %0 = load i32, i32* %gep, align 4, !dbg !15
+  ret i32 %0, !dbg !15
+}
+
+; In this example the GEP cannot (yet) be salvaged. Check that not only is the
+; dbg.value sunk, but an undef dbg.value is left to terminate any earlier
+; value range.
+
+; CHECK-LABEL: define i32 @bar(
+; CHECK:       call void @llvm.dbg.value(metadata i32* undef,
+; CHECK-NEXT:  br label %sink2
+
+define i32 @bar(i32 *%a, i32 %b) !dbg !70 {
+entry:
+  %gep = getelementptr i32, i32 *%a, i32 %b
+  call void @llvm.dbg.value(metadata i32* %gep, metadata !73, metadata !12), !dbg !74
+  br label %sink2
+
+sink2:
+; CHECK-LABEL: sink2:
+; CHECK:       call void @llvm.dbg.value(metadata i32* %gep,
+; CHECK-SAME:                    metadata !{{[0-9]+}}, metadata !DIExpression())
+; CHECK-NEXT:  load
+; CHECK-NEXT:  ret
+  %0 = load i32, i32* %gep
+  ret i32 %0
+}
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4, !5}
+!llvm.ident = !{!6}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug)
+!1 = !DIFile(filename: "a.c", directory: ".")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{i32 1, !"PIC Level", i32 2}
+!6 = !{!"clang"}
+!7 = distinct !DISubprogram(name: "foo", scope: !1, file: !1, line: 2, type: !8, isLocal: false, isDefinition: true, scopeLine: 3, flags: DIFlagPrototyped, isOptimized: false, unit: !0, retainedNodes: !2)
+!8 = !DISubroutineType(types: !9)
+!9 = !{!10, !10}
+!10 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!11 = !DILocalVariable(name: "j", scope: !7, file: !1, line: 2, type: !10)
+!12 = !DIExpression()
+!15 = !DILocation(line: 5, column: 3, scope: !7)
+!16 = !DILocalVariable(name: "h", scope: !7, file: !1, line: 4, type: !10)
+!70 = distinct !DISubprogram(name: "bar", scope: !1, file: !1, line: 2, type: !71, isLocal: false, isDefinition: true, scopeLine: 3, flags: DIFlagPrototyped, isOptimized: false, unit: !0, retainedNodes: !2)
+!71 = !DISubroutineType(types: !72)
+!72 = !{!10, !10, !10}
+!73 = !DILocalVariable(name: "k", scope: !70, file: !1, line: 2, type: !10)
+!74 = !DILocation(line: 5, column: 3, scope: !70)
diff --git a/test/Transforms/InstCombine/double-float-shrink-1.ll b/test/Transforms/InstCombine/double-float-shrink-1.ll
index c170f2ca74ba6b21dcd42921532a3ddbf356c9e2..e8f7f720b1527351dc57f0967eb5714179b777de 100644
--- a/test/Transforms/InstCombine/double-float-shrink-1.ll
+++ b/test/Transforms/InstCombine/double-float-shrink-1.ll
@@ -1,8 +1,8 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -instcombine -S | FileCheck %s
-
-target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
-target triple = "x86_64-unknown-linux-gnu"
+; RUN: opt < %s -instcombine -S -mtriple x86_64-unknown-linux-gnu | FileCheck %s --check-prefixes=CHECK,LINUX,ISC99
+; RUN: opt < %s -instcombine -S -mtriple x86_64-pc-win32          | FileCheck %s --check-prefixes=CHECK,ISC99
+; RUN: opt < %s -instcombine -S -mtriple x86_64-pc-windows-msvc16 | FileCheck %s --check-prefixes=CHECK,MS64,ISC89
+; RUN: opt < %s -instcombine -S -mtriple i386-pc-windows-msvc     | FileCheck %s --check-prefixes=CHECK,ISC99
+; RUN: opt < %s -instcombine -S -mtriple i686-pc-windows-msvc17   | FileCheck %s --check-prefixes=CHECK,MS32,ISC89
 
 ; Check for and against shrinkage when using the
 ; unsafe-fp-math function attribute on a math lib
@@ -12,8 +12,10 @@ target triple = "x86_64-unknown-linux-gnu"
 
 define float @acos_test1(float %f)   {
 ; CHECK-LABEL: @acos_test1(
-; CHECK-NEXT:    [[ACOSF:%.*]] = call fast float @acosf(float [[F:%.*]])
-; CHECK-NEXT:    ret float [[ACOSF]]
+; LINUX-NEXT:    [[ACOSF:%.*]] = call fast float @acosf(float [[F:%.*]])
+; LINUX-NEXT:    ret float [[ACOSF]]
+; MS32:          [[ACOSF:%.*]] = call fast double @acos(double [[F:%.*]])
+; MS64-NEXT:     [[ACOSF:%.*]] = call fast float @acosf(float [[F:%.*]])
 ;
   %conv = fpext float %f to double
   %call = call fast double @acos(double %conv)
@@ -34,8 +36,9 @@ define double @acos_test2(float %f)   {
 
 define float @acosh_test1(float %f)   {
 ; CHECK-LABEL: @acosh_test1(
-; CHECK-NEXT:    [[ACOSHF:%.*]] = call fast float @acoshf(float [[F:%.*]])
-; CHECK-NEXT:    ret float [[ACOSHF]]
+; ISC99-NEXT:    [[ACOSHF:%.*]] = call fast float @acoshf(float [[F:%.*]])
+; ISC99-NEXT:    ret float [[ACOSHF]]
+; ISC89:         [[ACOSHF:%.*]] = call fast double @acosh(double [[F:%.*]])
 ;
   %conv = fpext float %f to double
   %call = call fast double @acosh(double %conv)
@@ -56,8 +59,10 @@ define double @acosh_test2(float %f)   {
 
 define float @asin_test1(float %f)   {
 ; CHECK-LABEL: @asin_test1(
-; CHECK-NEXT:    [[ASINF:%.*]] = call fast float @asinf(float [[F:%.*]])
-; CHECK-NEXT:    ret float [[ASINF]]
+; LINUX-NEXT:    [[ASINF:%.*]] = call fast float @asinf(float [[F:%.*]])
+; LINUX-NEXT:    ret float [[ASINF]]
+; MS32:          [[ASINF:%.*]] = call fast double @asin(double [[F:%.*]])
+; MS64-NEXT:     [[ASINF:%.*]] = call fast float @asinf(float [[F:%.*]])
 ;
   %conv = fpext float %f to double
   %call = call fast double @asin(double %conv)
@@ -78,8 +83,9 @@ define double @asin_test2(float %f)   {
 
 define float @asinh_test1(float %f)   {
 ; CHECK-LABEL: @asinh_test1(
-; CHECK-NEXT:    [[ASINHF:%.*]] = call fast float @asinhf(float [[F:%.*]])
-; CHECK-NEXT:    ret float [[ASINHF]]
+; ISC99-NEXT:   [[ASINHF:%.*]] = call fast float @asinhf(float [[F:%.*]])
+; ISC99-NEXT:   ret float [[ASINHF]]
+; ISC89:        [[ASINHF:%.*]] = call fast double @asinh(double [[F:%.*]])
 ;
   %conv = fpext float %f to double
   %call = call fast double @asinh(double %conv)
@@ -100,8 +106,10 @@ define double @asinh_test2(float %f)   {
 
 define float @atan_test1(float %f)   {
 ; CHECK-LABEL: @atan_test1(
-; CHECK-NEXT:    [[ATANF:%.*]] = call fast float @atanf(float [[F:%.*]])
-; CHECK-NEXT:    ret float [[ATANF]]
+; LINUX-NEXT:    [[ATANF:%.*]] = call fast float @atanf(float [[F:%.*]])
+; LINUX-NEXT:    ret float [[ATANF]]
+; MS32:          [[ATANF:%.*]] = call fast double @atan(double [[F:%.*]])
+; MS64-NEXT:     [[ATANF:%.*]] = call fast float @atanf(float [[F:%.*]])
 ;
   %conv = fpext float %f to double
   %call = call fast double @atan(double %conv)
@@ -122,8 +130,9 @@ define double @atan_test2(float %f)   {
 
 define float @atanh_test1(float %f)   {
 ; CHECK-LABEL: @atanh_test1(
-; CHECK-NEXT:    [[ATANHF:%.*]] = call fast float @atanhf(float [[F:%.*]])
-; CHECK-NEXT:    ret float [[ATANHF]]
+; ISC99-NEXT:    [[ATANHF:%.*]] = call fast float @atanhf(float [[F:%.*]])
+; ISC99-NEXT:    ret float [[ATANHF]]
+; ISC89:         [[ATANHF:%.*]] = call fast double @atanh(double [[F:%.*]])
 ;
   %conv = fpext float %f to double
   %call = call fast double @atanh(double %conv)
@@ -144,8 +153,9 @@ define double @atanh_test2(float %f)   {
 
 define float @cbrt_test1(float %f)   {
 ; CHECK-LABEL: @cbrt_test1(
-; CHECK-NEXT:    [[CBRTF:%.*]] = call fast float @cbrtf(float [[F:%.*]])
-; CHECK-NEXT:    ret float [[CBRTF]]
+; ISC99-NEXT:    [[CBRTF:%.*]] = call fast float @cbrtf(float [[F:%.*]])
+; ISC99-NEXT:    ret float [[CBRTF]]
+; ISC89:         [[CBRTF:%.*]] = call fast double @cbrt(double [[F:%.*]])
 ;
   %conv = fpext float %f to double
   %call = call fast double @cbrt(double %conv)
@@ -166,8 +176,10 @@ define double @cbrt_test2(float %f)   {
 
 define float @exp_test1(float %f)   {
 ; CHECK-LABEL: @exp_test1(
-; CHECK-NEXT:    [[EXPF:%.*]] = call fast float @expf(float [[F:%.*]])
-; CHECK-NEXT:    ret float [[EXPF]]
+; LINUX-NEXT:    [[EXPF:%.*]] = call fast float @expf(float [[F:%.*]])
+; LINUX-NEXT:    ret float [[EXPF]]
+; MS32:          [[EXPF:%.*]] = call fast double @exp(double [[F:%.*]])
+; MS64-NEXT:     [[EXPF:%.*]] = call fast float @expf(float [[F:%.*]])
 ;
   %conv = fpext float %f to double
   %call = call fast double @exp(double %conv)
@@ -188,8 +200,9 @@ define double @exp_test2(float %f)   {
 
 define float @expm1_test1(float %f)   {
 ; CHECK-LABEL: @expm1_test1(
-; CHECK-NEXT:    [[EXPM1F:%.*]] = call fast float @expm1f(float [[F:%.*]])
-; CHECK-NEXT:    ret float [[EXPM1F]]
+; ISC99-NEXT:    [[EXPM1F:%.*]] = call fast float @expm1f(float [[F:%.*]])
+; ISC99-NEXT:    ret float [[EXPM1F]]
+; ISC89:         [[EXPM1F:%.*]] = call fast double @expm1(double [[F:%.*]])
 ;
   %conv = fpext float %f to double
   %call = call fast double @expm1(double %conv)
@@ -236,8 +249,10 @@ define double @exp10_test2(float %f)   {
 
 define float @log_test1(float %f)   {
 ; CHECK-LABEL: @log_test1(
-; CHECK-NEXT:    [[LOGF:%.*]] = call fast float @logf(float [[F:%.*]])
-; CHECK-NEXT:    ret float [[LOGF]]
+; LINUX-NEXT:    [[LOGF:%.*]] = call fast float @logf(float [[F:%.*]])
+; LINUX-NEXT:    ret float [[LOGF]]
+; MS32:          [[LOGF:%.*]] = call fast double @log(double [[F:%.*]])
+; MS64-NEXT:     [[LOGF:%.*]] = call fast float @logf(float [[F:%.*]])
 ;
   %conv = fpext float %f to double
   %call = call fast double @log(double %conv)
@@ -258,8 +273,10 @@ define double @log_test2(float %f)   {
 
 define float @log10_test1(float %f)   {
 ; CHECK-LABEL: @log10_test1(
-; CHECK-NEXT:    [[LOG10F:%.*]] = call fast float @log10f(float [[F:%.*]])
-; CHECK-NEXT:    ret float [[LOG10F]]
+; LINUX-NEXT:    [[LOG10F:%.*]] = call fast float @log10f(float [[F:%.*]])
+; LINUX-NEXT:    ret float [[LOG10F]]
+; MS32:          [[LOG10F:%.*]] = call fast double @log10(double [[F:%.*]])
+; MS64-NEXT:     [[LOG10F:%.*]] = call fast float @log10f(float [[F:%.*]])
 ;
   %conv = fpext float %f to double
   %call = call fast double @log10(double %conv)
@@ -280,8 +297,9 @@ define double @log10_test2(float %f) {
 
 define float @log1p_test1(float %f)   {
 ; CHECK-LABEL: @log1p_test1(
-; CHECK-NEXT:    [[LOG1PF:%.*]] = call fast float @log1pf(float [[F:%.*]])
-; CHECK-NEXT:    ret float [[LOG1PF]]
+; ISC99-NEXT:    [[LOG1PF:%.*]] = call fast float @log1pf(float [[F:%.*]])
+; ISC99-NEXT:    ret float [[LOG1PF]]
+; ISC89:         [[LOG1PF:%.*]] = call fast double @log1p(double [[F:%.*]])
 ;
   %conv = fpext float %f to double
   %call = call fast double @log1p(double %conv)
@@ -302,8 +320,9 @@ define double @log1p_test2(float %f)   {
 
 define float @log2_test1(float %f)   {
 ; CHECK-LABEL: @log2_test1(
-; CHECK-NEXT:    [[LOG2F:%.*]] = call fast float @log2f(float [[F:%.*]])
-; CHECK-NEXT:    ret float [[LOG2F]]
+; ISC99-NEXT:    [[LOG2F:%.*]] = call fast float @log2f(float [[F:%.*]])
+; ISC99-NEXT:    ret float [[LOG2F]]
+; ISC89:         [[LOG2F:%.*]] = call fast double @log2(double [[F:%.*]])
 ;
   %conv = fpext float %f to double
   %call = call fast double @log2(double %conv)
@@ -324,8 +343,10 @@ define double @log2_test2(float %f)   {
 
 define float @logb_test1(float %f)   {
 ; CHECK-LABEL: @logb_test1(
-; CHECK-NEXT:    [[LOGBF:%.*]] = call fast float @logbf(float [[F:%.*]])
-; CHECK-NEXT:    ret float [[LOGBF]]
+; LINUX-NEXT:    [[LOGBF:%.*]] = call fast float @logbf(float [[F:%.*]])
+; LINUX-NEXT:    ret float [[LOGBF]]
+; MS32:          [[POWF:%.*]] = call fast double @logb(double [[F:%.*]])
+; MS64-NEXT:     [[LOGBF:%.*]] = call fast float @logbf(float [[F:%.*]])
 ;
   %conv = fpext float %f to double
   %call = call fast double @logb(double %conv)
@@ -346,8 +367,10 @@ define double @logb_test2(float %f)   {
 
 define float @pow_test1(float %f, float %g)   {
 ; CHECK-LABEL: @pow_test1(
-; CHECK-NEXT:    [[POWF:%.*]] = call fast float @powf(float %f, float %g)
-; CHECK-NEXT:    ret float [[POWF]]
+; LINUX-NEXT:    [[POWF:%.*]] = call fast float @powf(float %f, float %g)
+; LINUX-NEXT:    ret float [[POWF]]
+; MS32:          [[POWF:%.*]] = call fast double @pow(double %df, double %dg)
+; MS64-NEXT:     [[POWF:%.*]] = call fast float @powf(float %f, float %g)
 ;
   %df = fpext float %f to double
   %dg = fpext float %g to double
@@ -369,8 +392,10 @@ define double @pow_test2(float %f, float %g) {
 
 define float @sin_test1(float %f)   {
 ; CHECK-LABEL: @sin_test1(
-; CHECK-NEXT:    [[SINF:%.*]] = call fast float @sinf(float [[F:%.*]])
-; CHECK-NEXT:    ret float [[SINF]]
+; LINUX-NEXT:    [[SINF:%.*]] = call fast float @sinf(float [[F:%.*]])
+; LINUX-NEXT:    ret float [[SINF]]
+; MS32:          [[SINF:%.*]] = call fast double @sin(double [[F:%.*]])
+; MS64-NEXT:     [[SINF:%.*]] = call fast float @sinf(float [[F:%.*]])
 ;
   %conv = fpext float %f to double
   %call = call fast double @sin(double %conv)
@@ -391,8 +416,10 @@ define double @sin_test2(float %f) {
 
 define float @sqrt_test1(float %f) {
 ; CHECK-LABEL: @sqrt_test1(
-; CHECK-NEXT:    [[SQRTF:%.*]] = call float @sqrtf(float [[F:%.*]])
-; CHECK-NEXT:    ret float [[SQRTF]]
+; LINUX-NEXT:    [[SQRTF:%.*]] = call float @sqrtf(float [[F:%.*]])
+; LINUX-NEXT:    ret float [[SQRTF]]
+; MS32:          [[SQRTF:%.*]] = call double @sqrt(double [[F:%.*]])
+; MS64-NEXT:     [[SQRTF:%.*]] = call float @sqrtf(float [[F:%.*]])
 ;
   %conv = fpext float %f to double
   %call = call double @sqrt(double %conv)
@@ -413,8 +440,10 @@ define double @sqrt_test2(float %f) {
 
 define float @sqrt_int_test1(float %f) {
 ; CHECK-LABEL: @sqrt_int_test1(
-; CHECK-NEXT:    [[TMP1:%.*]] = call float @llvm.sqrt.f32(float [[F:%.*]])
-; CHECK-NEXT:    ret float [[TMP1]]
+; LINUX-NEXT:    [[TMP1:%.*]] = call float @llvm.sqrt.f32(float [[F:%.*]])
+; LINUX-NEXT:    ret float [[TMP1]]
+; MS32:          [[TMP1:%.*]] = call double @llvm.sqrt.f64(double [[F:%.*]])
+; MS64-NEXT:     [[TMP1:%.*]] = call float @llvm.sqrt.f32(float [[F:%.*]])
 ;
   %conv = fpext float %f to double
   %call = call double @llvm.sqrt.f64(double %conv)
@@ -435,8 +464,10 @@ define double @sqrt_int_test2(float %f) {
 
 define float @tan_test1(float %f) {
 ; CHECK-LABEL: @tan_test1(
-; CHECK-NEXT:    [[TANF:%.*]] = call fast float @tanf(float [[F:%.*]])
-; CHECK-NEXT:    ret float [[TANF]]
+; LINUX-NEXT:    [[TANF:%.*]] = call fast float @tanf(float [[F:%.*]])
+; LINUX-NEXT:    ret float [[TANF]]
+; MS32:          [[TANF:%.*]] = call fast double @tan(double [[F:%.*]])
+; MS64-NEXT:     [[TANF:%.*]] = call fast float @tanf(float [[F:%.*]])
 ;
   %conv = fpext float %f to double
   %call = call fast double @tan(double %conv)
@@ -456,8 +487,10 @@ define double @tan_test2(float %f) {
 }
 define float @tanh_test1(float %f) {
 ; CHECK-LABEL: @tanh_test1(
-; CHECK-NEXT:    [[TANHF:%.*]] = call fast float @tanhf(float [[F:%.*]])
-; CHECK-NEXT:    ret float [[TANHF]]
+; LINUX-NEXT:    [[TANHF:%.*]] = call fast float @tanhf(float [[F:%.*]])
+; LINUX-NEXT:    ret float [[TANHF]]
+; MS32:          [[TANHF:%.*]] = call fast double @tanh(double [[F:%.*]])
+; MS64-NEXT:     [[TANHF:%.*]] = call fast float @tanhf(float [[F:%.*]])
 ;
   %conv = fpext float %f to double
   %call = call fast double @tanh(double %conv)
@@ -480,8 +513,9 @@ define double @tanh_test2(float %f) {
 ; flags are propagated for shrunken *binary* double FP calls.
 define float @max1(float %a, float %b) {
 ; CHECK-LABEL: @max1(
-; CHECK-NEXT:    [[FMAXF:%.*]] = call arcp float @fmaxf(float [[A:%.*]], float [[B:%.*]])
-; CHECK-NEXT:    ret float [[FMAXF]]
+; ISC99-NEXT:    [[FMAXF:%.*]] = call arcp float @fmaxf(float [[A:%.*]], float [[B:%.*]])
+; ISC99-NEXT:    ret float [[FMAXF]]
+; ISC89:         [[FMAXF:%.*]] = call arcp double @fmax(double [[A:%.*]], double [[B:%.*]])
 ;
   %c = fpext float %a to double
   %d = fpext float %b to double
diff --git a/test/Transforms/InstCombine/double-float-shrink-2.ll b/test/Transforms/InstCombine/double-float-shrink-2.ll
index 5bdeaf7d9d7e0f4cc3b995cdbff63bf38353388e..76e497bd68fc7949e4b57336b6d2e64dc3c1195b 100644
--- a/test/Transforms/InstCombine/double-float-shrink-2.ll
+++ b/test/Transforms/InstCombine/double-float-shrink-2.ll
@@ -1,11 +1,11 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -instcombine -S -mtriple "i386-pc-linux" | FileCheck -check-prefix=ALL -check-prefix=DO-SIMPLIFY %s
-; RUN: opt < %s -instcombine -S -mtriple "i386-pc-win32" | FileCheck -check-prefix=ALL -check-prefix=DONT-SIMPLIFY %s
-; RUN: opt < %s -instcombine -S -mtriple "x86_64-pc-win32" | FileCheck -check-prefix=ALL -check-prefix=C89-SIMPLIFY %s
-; RUN: opt < %s -instcombine -S -mtriple "i386-pc-mingw32" | FileCheck -check-prefix=ALL -check-prefix=DO-SIMPLIFY %s
-; RUN: opt < %s -instcombine -S -mtriple "x86_64-pc-mingw32" | FileCheck -check-prefix=ALL -check-prefix=DO-SIMPLIFY %s
-; RUN: opt < %s -instcombine -S -mtriple "sparc-sun-solaris" | FileCheck -check-prefix=ALL -check-prefix=DO-SIMPLIFY %s
-; RUN: opt < %s -enable-debugify -instcombine -S -mtriple "x86_64-pc-win32" 2>&1 | FileCheck -check-prefix=DBG-VALID %s
+; RUN: opt < %s -instcombine -S -mtriple "i386-pc-linux"     | FileCheck %s
+; RUN: opt < %s -instcombine -S -mtriple "i386-pc-win32"     | FileCheck %s
+; RUN: opt < %s -instcombine -S -mtriple "x86_64-pc-win32"   | FileCheck %s
+; RUN: opt < %s -instcombine -S -mtriple "i386-pc-mingw32"   | FileCheck %s
+; RUN: opt < %s -instcombine -S -mtriple "x86_64-pc-mingw32" | FileCheck %s
+; RUN: opt < %s -instcombine -S -mtriple "sparc-sun-solaris" | FileCheck %s
+; RUN: opt < %s -instcombine -S -mtriple "x86_64-pc-win32" -enable-debugify 2>&1 | FileCheck --check-prefix=DBG-VALID %s
 
 declare double @floor(double)
 declare double @ceil(double)
@@ -36,9 +36,9 @@ declare double @llvm.trunc.f64(double)
 declare <2 x double> @llvm.trunc.v2f64(<2 x double>)
 
 define float @test_shrink_libcall_floor(float %C) {
-; ALL-LABEL: @test_shrink_libcall_floor(
-; ALL-NEXT:    [[F:%.*]] = call float @llvm.floor.f32(float [[C:%.*]])
-; ALL-NEXT:    ret float [[F]]
+; CHECK-LABEL: @test_shrink_libcall_floor(
+; CHECK-NEXT:    [[F:%.*]] = call float @llvm.floor.f32(float [[C:%.*]])
+; CHECK-NEXT:    ret float [[F]]
 ;
   %D = fpext float %C to double
   ; --> floorf
@@ -48,9 +48,9 @@ define float @test_shrink_libcall_floor(float %C) {
 }
 
 define float @test_shrink_libcall_ceil(float %C) {
-; ALL-LABEL: @test_shrink_libcall_ceil(
-; ALL-NEXT:    [[F:%.*]] = call float @llvm.ceil.f32(float [[C:%.*]])
-; ALL-NEXT:    ret float [[F]]
+; CHECK-LABEL: @test_shrink_libcall_ceil(
+; CHECK-NEXT:    [[F:%.*]] = call float @llvm.ceil.f32(float [[C:%.*]])
+; CHECK-NEXT:    ret float [[F]]
 ;
   %D = fpext float %C to double
   ; --> ceilf
@@ -60,21 +60,9 @@ define float @test_shrink_libcall_ceil(float %C) {
 }
 
 define float @test_shrink_libcall_round(float %C) {
-; DO-SIMPLIFY-LABEL: @test_shrink_libcall_round(
-; DO-SIMPLIFY-NEXT:    [[F:%.*]] = call float @llvm.round.f32(float [[C:%.*]])
-; DO-SIMPLIFY-NEXT:    ret float [[F]]
-;
-; DONT-SIMPLIFY-LABEL: @test_shrink_libcall_round(
-; DONT-SIMPLIFY-NEXT:    [[D:%.*]] = fpext float [[C:%.*]] to double
-; DONT-SIMPLIFY-NEXT:    [[E:%.*]] = call double @round(double [[D]])
-; DONT-SIMPLIFY-NEXT:    [[F:%.*]] = fptrunc double [[E]] to float
-; DONT-SIMPLIFY-NEXT:    ret float [[F]]
-;
-; C89-SIMPLIFY-LABEL: @test_shrink_libcall_round(
-; C89-SIMPLIFY-NEXT:    [[D:%.*]] = fpext float [[C:%.*]] to double
-; C89-SIMPLIFY-NEXT:    [[E:%.*]] = call double @round(double [[D]])
-; C89-SIMPLIFY-NEXT:    [[F:%.*]] = fptrunc double [[E]] to float
-; C89-SIMPLIFY-NEXT:    ret float [[F]]
+; CHECK-LABEL: @test_shrink_libcall_round(
+; CHECK-NEXT:    [[F:%.*]] = call float @llvm.round.f32(float [[C:%.*]])
+; CHECK-NEXT:    ret float [[F]]
 ;
   %D = fpext float %C to double
   ; --> roundf
@@ -84,21 +72,9 @@ define float @test_shrink_libcall_round(float %C) {
 }
 
 define float @test_shrink_libcall_nearbyint(float %C) {
-; DO-SIMPLIFY-LABEL: @test_shrink_libcall_nearbyint(
-; DO-SIMPLIFY-NEXT:    [[F:%.*]] = call float @llvm.nearbyint.f32(float [[C:%.*]])
-; DO-SIMPLIFY-NEXT:    ret float [[F]]
-;
-; DONT-SIMPLIFY-LABEL: @test_shrink_libcall_nearbyint(
-; DONT-SIMPLIFY-NEXT:    [[D:%.*]] = fpext float [[C:%.*]] to double
-; DONT-SIMPLIFY-NEXT:    [[E:%.*]] = call double @nearbyint(double [[D]])
-; DONT-SIMPLIFY-NEXT:    [[F:%.*]] = fptrunc double [[E]] to float
-; DONT-SIMPLIFY-NEXT:    ret float [[F]]
-;
-; C89-SIMPLIFY-LABEL: @test_shrink_libcall_nearbyint(
-; C89-SIMPLIFY-NEXT:    [[D:%.*]] = fpext float [[C:%.*]] to double
-; C89-SIMPLIFY-NEXT:    [[E:%.*]] = call double @nearbyint(double [[D]])
-; C89-SIMPLIFY-NEXT:    [[F:%.*]] = fptrunc double [[E]] to float
-; C89-SIMPLIFY-NEXT:    ret float [[F]]
+; CHECK-LABEL: @test_shrink_libcall_nearbyint(
+; CHECK-NEXT:    [[F:%.*]] = call float @llvm.nearbyint.f32(float [[C:%.*]])
+; CHECK-NEXT:    ret float [[F]]
 ;
   %D = fpext float %C to double
   ; --> nearbyintf
@@ -108,21 +84,9 @@ define float @test_shrink_libcall_nearbyint(float %C) {
 }
 
 define float @test_shrink_libcall_trunc(float %C) {
-; DO-SIMPLIFY-LABEL: @test_shrink_libcall_trunc(
-; DO-SIMPLIFY-NEXT:    [[F:%.*]] = call float @llvm.trunc.f32(float [[C:%.*]])
-; DO-SIMPLIFY-NEXT:    ret float [[F]]
-;
-; DONT-SIMPLIFY-LABEL: @test_shrink_libcall_trunc(
-; DONT-SIMPLIFY-NEXT:    [[D:%.*]] = fpext float [[C:%.*]] to double
-; DONT-SIMPLIFY-NEXT:    [[E:%.*]] = call double @trunc(double [[D]])
-; DONT-SIMPLIFY-NEXT:    [[F:%.*]] = fptrunc double [[E]] to float
-; DONT-SIMPLIFY-NEXT:    ret float [[F]]
-;
-; C89-SIMPLIFY-LABEL: @test_shrink_libcall_trunc(
-; C89-SIMPLIFY-NEXT:    [[D:%.*]] = fpext float [[C:%.*]] to double
-; C89-SIMPLIFY-NEXT:    [[E:%.*]] = call double @trunc(double [[D]])
-; C89-SIMPLIFY-NEXT:    [[F:%.*]] = fptrunc double [[E]] to float
-; C89-SIMPLIFY-NEXT:    ret float [[F]]
+; CHECK-LABEL: @test_shrink_libcall_trunc(
+; CHECK-NEXT:    [[F:%.*]] = call float @llvm.trunc.f32(float [[C:%.*]])
+; CHECK-NEXT:    ret float [[F]]
 ;
   %D = fpext float %C to double
   ; --> truncf
@@ -132,11 +96,11 @@ define float @test_shrink_libcall_trunc(float %C) {
 }
 
 ; This is replaced with the intrinsic, which does the right thing on
-; all platforms.
+; CHECK platforms.
 define float @test_shrink_libcall_fabs(float %C) {
-; ALL-LABEL: @test_shrink_libcall_fabs(
-; ALL-NEXT:    [[F:%.*]] = call float @llvm.fabs.f32(float [[C:%.*]])
-; ALL-NEXT:    ret float [[F]]
+; CHECK-LABEL: @test_shrink_libcall_fabs(
+; CHECK-NEXT:    [[F:%.*]] = call float @llvm.fabs.f32(float [[C:%.*]])
+; CHECK-NEXT:    ret float [[F]]
 ;
   %D = fpext float %C to double
   %E = call double @fabs(double %D)
@@ -146,9 +110,9 @@ define float @test_shrink_libcall_fabs(float %C) {
 
 ; Make sure fast math flags are preserved
 define float @test_shrink_libcall_fabs_fast(float %C) {
-; ALL-LABEL: @test_shrink_libcall_fabs_fast(
-; ALL-NEXT:    [[F:%.*]] = call fast float @llvm.fabs.f32(float [[C:%.*]])
-; ALL-NEXT:    ret float [[F]]
+; CHECK-LABEL: @test_shrink_libcall_fabs_fast(
+; CHECK-NEXT:    [[F:%.*]] = call fast float @llvm.fabs.f32(float [[C:%.*]])
+; CHECK-NEXT:    ret float [[F]]
 ;
   %D = fpext float %C to double
   %E = call fast double @fabs(double %D)
@@ -157,9 +121,9 @@ define float @test_shrink_libcall_fabs_fast(float %C) {
 }
 
 define float @test_shrink_intrin_ceil(float %C) {
-; ALL-LABEL: @test_shrink_intrin_ceil(
-; ALL-NEXT:    [[TMP1:%.*]] = call float @llvm.ceil.f32(float [[C:%.*]])
-; ALL-NEXT:    ret float [[TMP1]]
+; CHECK-LABEL: @test_shrink_intrin_ceil(
+; CHECK-NEXT:    [[TMP1:%.*]] = call float @llvm.ceil.f32(float [[C:%.*]])
+; CHECK-NEXT:    ret float [[TMP1]]
 ;
   %D = fpext float %C to double
   %E = call double @llvm.ceil.f64(double %D)
@@ -168,9 +132,9 @@ define float @test_shrink_intrin_ceil(float %C) {
 }
 
 define float @test_shrink_intrin_fabs(float %C) {
-; ALL-LABEL: @test_shrink_intrin_fabs(
-; ALL-NEXT:    [[TMP1:%.*]] = call float @llvm.fabs.f32(float [[C:%.*]])
-; ALL-NEXT:    ret float [[TMP1]]
+; CHECK-LABEL: @test_shrink_intrin_fabs(
+; CHECK-NEXT:    [[TMP1:%.*]] = call float @llvm.fabs.f32(float [[C:%.*]])
+; CHECK-NEXT:    ret float [[TMP1]]
 ;
   %D = fpext float %C to double
   %E = call double @llvm.fabs.f64(double %D)
@@ -179,9 +143,9 @@ define float @test_shrink_intrin_fabs(float %C) {
 }
 
 define float @test_shrink_intrin_floor(float %C) {
-; ALL-LABEL: @test_shrink_intrin_floor(
-; ALL-NEXT:    [[TMP1:%.*]] = call float @llvm.floor.f32(float [[C:%.*]])
-; ALL-NEXT:    ret float [[TMP1]]
+; CHECK-LABEL: @test_shrink_intrin_floor(
+; CHECK-NEXT:    [[TMP1:%.*]] = call float @llvm.floor.f32(float [[C:%.*]])
+; CHECK-NEXT:    ret float [[TMP1]]
 ;
   %D = fpext float %C to double
   %E = call double @llvm.floor.f64(double %D)
@@ -190,9 +154,9 @@ define float @test_shrink_intrin_floor(float %C) {
 }
 
 define float @test_shrink_intrin_nearbyint(float %C) {
-; ALL-LABEL: @test_shrink_intrin_nearbyint(
-; ALL-NEXT:    [[TMP1:%.*]] = call float @llvm.nearbyint.f32(float [[C:%.*]])
-; ALL-NEXT:    ret float [[TMP1]]
+; CHECK-LABEL: @test_shrink_intrin_nearbyint(
+; CHECK-NEXT:    [[TMP1:%.*]] = call float @llvm.nearbyint.f32(float [[C:%.*]])
+; CHECK-NEXT:    ret float [[TMP1]]
 ;
   %D = fpext float %C to double
   %E = call double @llvm.nearbyint.f64(double %D)
@@ -201,9 +165,9 @@ define float @test_shrink_intrin_nearbyint(float %C) {
 }
 
 define half @test_shrink_intrin_rint(half %C) {
-; ALL-LABEL: @test_shrink_intrin_rint(
-; ALL-NEXT:    [[TMP1:%.*]] = call half @llvm.rint.f16(half [[C:%.*]])
-; ALL-NEXT:    ret half [[TMP1]]
+; CHECK-LABEL: @test_shrink_intrin_rint(
+; CHECK-NEXT:    [[TMP1:%.*]] = call half @llvm.rint.f16(half [[C:%.*]])
+; CHECK-NEXT:    ret half [[TMP1]]
 ;
   %D = fpext half %C to float
   %E = call float @llvm.rint.f32(float %D)
@@ -212,9 +176,9 @@ define half @test_shrink_intrin_rint(half %C) {
 }
 
 define float @test_shrink_intrin_round(float %C) {
-; ALL-LABEL: @test_shrink_intrin_round(
-; ALL-NEXT:    [[TMP1:%.*]] = call float @llvm.round.f32(float [[C:%.*]])
-; ALL-NEXT:    ret float [[TMP1]]
+; CHECK-LABEL: @test_shrink_intrin_round(
+; CHECK-NEXT:    [[TMP1:%.*]] = call float @llvm.round.f32(float [[C:%.*]])
+; CHECK-NEXT:    ret float [[TMP1]]
 ;
   %D = fpext float %C to double
   %E = call double @llvm.round.f64(double %D)
@@ -223,9 +187,9 @@ define float @test_shrink_intrin_round(float %C) {
 }
 
 define float @test_shrink_intrin_trunc(float %C) {
-; ALL-LABEL: @test_shrink_intrin_trunc(
-; ALL-NEXT:    [[TMP1:%.*]] = call float @llvm.trunc.f32(float [[C:%.*]])
-; ALL-NEXT:    ret float [[TMP1]]
+; CHECK-LABEL: @test_shrink_intrin_trunc(
+; CHECK-NEXT:    [[TMP1:%.*]] = call float @llvm.trunc.f32(float [[C:%.*]])
+; CHECK-NEXT:    ret float [[TMP1]]
 ;
   %D = fpext float %C to double
   %E = call double @llvm.trunc.f64(double %D)
@@ -237,12 +201,12 @@ declare void @use_v2f64(<2 x double>)
 declare void @use_v2f32(<2 x float>)
 
 define <2 x float> @test_shrink_intrin_ceil_multi_use(<2 x float> %C) {
-; ALL-LABEL: @test_shrink_intrin_ceil_multi_use(
-; ALL-NEXT:    [[D:%.*]] = fpext <2 x float> [[C:%.*]] to <2 x double>
-; ALL-NEXT:    [[E:%.*]] = call <2 x double> @llvm.ceil.v2f64(<2 x double> [[D]])
-; ALL-NEXT:    [[F:%.*]] = fptrunc <2 x double> [[E]] to <2 x float>
-; ALL-NEXT:    call void @use_v2f64(<2 x double> [[D]])
-; ALL-NEXT:    ret <2 x float> [[F]]
+; CHECK-LABEL: @test_shrink_intrin_ceil_multi_use(
+; CHECK-NEXT:    [[D:%.*]] = fpext <2 x float> [[C:%.*]] to <2 x double>
+; CHECK-NEXT:    [[E:%.*]] = call <2 x double> @llvm.ceil.v2f64(<2 x double> [[D]])
+; CHECK-NEXT:    [[F:%.*]] = fptrunc <2 x double> [[E]] to <2 x float>
+; CHECK-NEXT:    call void @use_v2f64(<2 x double> [[D]])
+; CHECK-NEXT:    ret <2 x float> [[F]]
 ;
   %D = fpext <2 x float> %C to <2 x double>
   %E = call <2 x double> @llvm.ceil.v2f64(<2 x double> %D)
@@ -252,11 +216,11 @@ define <2 x float> @test_shrink_intrin_ceil_multi_use(<2 x float> %C) {
 }
 
 define <2 x float> @test_shrink_intrin_fabs_multi_use(<2 x float> %C) {
-; ALL-LABEL: @test_shrink_intrin_fabs_multi_use(
-; ALL-NEXT:    [[TMP1:%.*]] = call <2 x float> @llvm.fabs.v2f32(<2 x float> [[C:%.*]])
-; ALL-NEXT:    [[E:%.*]] = fpext <2 x float> [[TMP1]] to <2 x double>
-; ALL-NEXT:    call void @use_v2f64(<2 x double> [[E]])
-; ALL-NEXT:    ret <2 x float> [[TMP1]]
+; CHECK-LABEL: @test_shrink_intrin_fabs_multi_use(
+; CHECK-NEXT:    [[TMP1:%.*]] = call <2 x float> @llvm.fabs.v2f32(<2 x float> [[C:%.*]])
+; CHECK-NEXT:    [[E:%.*]] = fpext <2 x float> [[TMP1]] to <2 x double>
+; CHECK-NEXT:    call void @use_v2f64(<2 x double> [[E]])
+; CHECK-NEXT:    ret <2 x float> [[TMP1]]
 ;
   %D = fpext <2 x float> %C to <2 x double>
   %E = call <2 x double> @llvm.fabs.v2f64(<2 x double> %D)
@@ -266,13 +230,13 @@ define <2 x float> @test_shrink_intrin_fabs_multi_use(<2 x float> %C) {
 }
 
 define <2 x float> @test_shrink_intrin_floor_multi_use(<2 x float> %C) {
-; ALL-LABEL: @test_shrink_intrin_floor_multi_use(
-; ALL-NEXT:    [[D:%.*]] = fpext <2 x float> [[C:%.*]] to <2 x double>
-; ALL-NEXT:    [[E:%.*]] = call <2 x double> @llvm.floor.v2f64(<2 x double> [[D]])
-; ALL-NEXT:    [[F:%.*]] = fptrunc <2 x double> [[E]] to <2 x float>
-; ALL-NEXT:    call void @use_v2f64(<2 x double> [[D]])
-; ALL-NEXT:    call void @use_v2f64(<2 x double> [[E]])
-; ALL-NEXT:    ret <2 x float> [[F]]
+; CHECK-LABEL: @test_shrink_intrin_floor_multi_use(
+; CHECK-NEXT:    [[D:%.*]] = fpext <2 x float> [[C:%.*]] to <2 x double>
+; CHECK-NEXT:    [[E:%.*]] = call <2 x double> @llvm.floor.v2f64(<2 x double> [[D]])
+; CHECK-NEXT:    [[F:%.*]] = fptrunc <2 x double> [[E]] to <2 x float>
+; CHECK-NEXT:    call void @use_v2f64(<2 x double> [[D]])
+; CHECK-NEXT:    call void @use_v2f64(<2 x double> [[E]])
+; CHECK-NEXT:    ret <2 x float> [[F]]
 ;
   %D = fpext <2 x float> %C to <2 x double>
   %E = call <2 x double> @llvm.floor.v2f64(<2 x double> %D)
@@ -283,12 +247,12 @@ define <2 x float> @test_shrink_intrin_floor_multi_use(<2 x float> %C) {
 }
 
 define <2 x float> @test_shrink_intrin_nearbyint_multi_use(<2 x float> %C) {
-; ALL-LABEL: @test_shrink_intrin_nearbyint_multi_use(
-; ALL-NEXT:    [[D:%.*]] = fpext <2 x float> [[C:%.*]] to <2 x double>
-; ALL-NEXT:    [[E:%.*]] = call <2 x double> @llvm.nearbyint.v2f64(<2 x double> [[D]])
-; ALL-NEXT:    [[F:%.*]] = fptrunc <2 x double> [[E]] to <2 x float>
-; ALL-NEXT:    call void @use_v2f64(<2 x double> [[D]])
-; ALL-NEXT:    ret <2 x float> [[F]]
+; CHECK-LABEL: @test_shrink_intrin_nearbyint_multi_use(
+; CHECK-NEXT:    [[D:%.*]] = fpext <2 x float> [[C:%.*]] to <2 x double>
+; CHECK-NEXT:    [[E:%.*]] = call <2 x double> @llvm.nearbyint.v2f64(<2 x double> [[D]])
+; CHECK-NEXT:    [[F:%.*]] = fptrunc <2 x double> [[E]] to <2 x float>
+; CHECK-NEXT:    call void @use_v2f64(<2 x double> [[D]])
+; CHECK-NEXT:    ret <2 x float> [[F]]
 ;
   %D = fpext <2 x float> %C to <2 x double>
   %E = call <2 x double> @llvm.nearbyint.v2f64(<2 x double> %D)
@@ -298,11 +262,11 @@ define <2 x float> @test_shrink_intrin_nearbyint_multi_use(<2 x float> %C) {
 }
 
 define <2 x half> @test_shrink_intrin_rint_multi_use(<2 x half> %C) {
-; ALL-LABEL: @test_shrink_intrin_rint_multi_use(
-; ALL-NEXT:    [[TMP1:%.*]] = call <2 x half> @llvm.rint.v2f16(<2 x half> [[C:%.*]])
-; ALL-NEXT:    [[E:%.*]] = fpext <2 x half> [[TMP1]] to <2 x float>
-; ALL-NEXT:    call void @use_v2f32(<2 x float> [[E]])
-; ALL-NEXT:    ret <2 x half> [[TMP1]]
+; CHECK-LABEL: @test_shrink_intrin_rint_multi_use(
+; CHECK-NEXT:    [[TMP1:%.*]] = call <2 x half> @llvm.rint.v2f16(<2 x half> [[C:%.*]])
+; CHECK-NEXT:    [[E:%.*]] = fpext <2 x half> [[TMP1]] to <2 x float>
+; CHECK-NEXT:    call void @use_v2f32(<2 x float> [[E]])
+; CHECK-NEXT:    ret <2 x half> [[TMP1]]
 ;
   %D = fpext <2 x half> %C to <2 x float>
   %E = call <2 x float> @llvm.rint.v2f32(<2 x float> %D)
@@ -312,13 +276,13 @@ define <2 x half> @test_shrink_intrin_rint_multi_use(<2 x half> %C) {
 }
 
 define <2 x float> @test_shrink_intrin_round_multi_use(<2 x float> %C) {
-; ALL-LABEL: @test_shrink_intrin_round_multi_use(
-; ALL-NEXT:    [[D:%.*]] = fpext <2 x float> [[C:%.*]] to <2 x double>
-; ALL-NEXT:    [[E:%.*]] = call <2 x double> @llvm.round.v2f64(<2 x double> [[D]])
-; ALL-NEXT:    [[F:%.*]] = fptrunc <2 x double> [[E]] to <2 x float>
-; ALL-NEXT:    call void @use_v2f64(<2 x double> [[D]])
-; ALL-NEXT:    call void @use_v2f64(<2 x double> [[E]])
-; ALL-NEXT:    ret <2 x float> [[F]]
+; CHECK-LABEL: @test_shrink_intrin_round_multi_use(
+; CHECK-NEXT:    [[D:%.*]] = fpext <2 x float> [[C:%.*]] to <2 x double>
+; CHECK-NEXT:    [[E:%.*]] = call <2 x double> @llvm.round.v2f64(<2 x double> [[D]])
+; CHECK-NEXT:    [[F:%.*]] = fptrunc <2 x double> [[E]] to <2 x float>
+; CHECK-NEXT:    call void @use_v2f64(<2 x double> [[D]])
+; CHECK-NEXT:    call void @use_v2f64(<2 x double> [[E]])
+; CHECK-NEXT:    ret <2 x float> [[F]]
 ;
   %D = fpext <2 x float> %C to <2 x double>
   %E = call <2 x double> @llvm.round.v2f64(<2 x double> %D)
@@ -329,12 +293,12 @@ define <2 x float> @test_shrink_intrin_round_multi_use(<2 x float> %C) {
 }
 
 define <2 x float> @test_shrink_intrin_trunc_multi_use(<2 x float> %C) {
-; ALL-LABEL: @test_shrink_intrin_trunc_multi_use(
-; ALL-NEXT:    [[D:%.*]] = fpext <2 x float> [[C:%.*]] to <2 x double>
-; ALL-NEXT:    [[E:%.*]] = call <2 x double> @llvm.trunc.v2f64(<2 x double> [[D]])
-; ALL-NEXT:    [[F:%.*]] = fptrunc <2 x double> [[E]] to <2 x float>
-; ALL-NEXT:    call void @use_v2f64(<2 x double> [[D]])
-; ALL-NEXT:    ret <2 x float> [[F]]
+; CHECK-LABEL: @test_shrink_intrin_trunc_multi_use(
+; CHECK-NEXT:    [[D:%.*]] = fpext <2 x float> [[C:%.*]] to <2 x double>
+; CHECK-NEXT:    [[E:%.*]] = call <2 x double> @llvm.trunc.v2f64(<2 x double> [[D]])
+; CHECK-NEXT:    [[F:%.*]] = fptrunc <2 x double> [[E]] to <2 x float>
+; CHECK-NEXT:    call void @use_v2f64(<2 x double> [[D]])
+; CHECK-NEXT:    ret <2 x float> [[F]]
 ;
   %D = fpext <2 x float> %C to <2 x double>
   %E = call <2 x double> @llvm.trunc.v2f64(<2 x double> %D)
@@ -345,9 +309,9 @@ define <2 x float> @test_shrink_intrin_trunc_multi_use(<2 x float> %C) {
 
 ; Make sure fast math flags are preserved
 define float @test_shrink_intrin_fabs_fast(float %C) {
-; ALL-LABEL: @test_shrink_intrin_fabs_fast(
-; ALL-NEXT:    [[TMP1:%.*]] = call fast float @llvm.fabs.f32(float [[C:%.*]])
-; ALL-NEXT:    ret float [[TMP1]]
+; CHECK-LABEL: @test_shrink_intrin_fabs_fast(
+; CHECK-NEXT:    [[TMP1:%.*]] = call fast float @llvm.fabs.f32(float [[C:%.*]])
+; CHECK-NEXT:    ret float [[TMP1]]
 ;
   %D = fpext float %C to double
   %E = call fast double @llvm.fabs.f64(double %D)
@@ -356,10 +320,10 @@ define float @test_shrink_intrin_fabs_fast(float %C) {
 }
 
 define float @test_no_shrink_intrin_floor(double %D) {
-; ALL-LABEL: @test_no_shrink_intrin_floor(
-; ALL-NEXT:    [[E:%.*]] = call double @llvm.floor.f64(double [[D:%.*]])
-; ALL-NEXT:    [[F:%.*]] = fptrunc double [[E]] to float
-; ALL-NEXT:    ret float [[F]]
+; CHECK-LABEL: @test_no_shrink_intrin_floor(
+; CHECK-NEXT:    [[E:%.*]] = call double @llvm.floor.f64(double [[D:%.*]])
+; CHECK-NEXT:    [[F:%.*]] = fptrunc double [[E]] to float
+; CHECK-NEXT:    ret float [[F]]
 ;
   %E = call double @llvm.floor.f64(double %D)
   %F = fptrunc double %E to float
@@ -367,10 +331,10 @@ define float @test_no_shrink_intrin_floor(double %D) {
 }
 
 define float @test_no_shrink_intrin_ceil(double %D) {
-; ALL-LABEL: @test_no_shrink_intrin_ceil(
-; ALL-NEXT:    [[E:%.*]] = call double @llvm.ceil.f64(double [[D:%.*]])
-; ALL-NEXT:    [[F:%.*]] = fptrunc double [[E]] to float
-; ALL-NEXT:    ret float [[F]]
+; CHECK-LABEL: @test_no_shrink_intrin_ceil(
+; CHECK-NEXT:    [[E:%.*]] = call double @llvm.ceil.f64(double [[D:%.*]])
+; CHECK-NEXT:    [[F:%.*]] = fptrunc double [[E]] to float
+; CHECK-NEXT:    ret float [[F]]
 ;
   %E = call double @llvm.ceil.f64(double %D)
   %F = fptrunc double %E to float
@@ -378,10 +342,10 @@ define float @test_no_shrink_intrin_ceil(double %D) {
 }
 
 define float @test_no_shrink_intrin_round(double %D) {
-; ALL-LABEL: @test_no_shrink_intrin_round(
-; ALL-NEXT:    [[E:%.*]] = call double @llvm.round.f64(double [[D:%.*]])
-; ALL-NEXT:    [[F:%.*]] = fptrunc double [[E]] to float
-; ALL-NEXT:    ret float [[F]]
+; CHECK-LABEL: @test_no_shrink_intrin_round(
+; CHECK-NEXT:    [[E:%.*]] = call double @llvm.round.f64(double [[D:%.*]])
+; CHECK-NEXT:    [[F:%.*]] = fptrunc double [[E]] to float
+; CHECK-NEXT:    ret float [[F]]
 ;
   %E = call double @llvm.round.f64(double %D)
   %F = fptrunc double %E to float
@@ -389,10 +353,10 @@ define float @test_no_shrink_intrin_round(double %D) {
 }
 
 define float @test_no_shrink_intrin_nearbyint(double %D) {
-; ALL-LABEL: @test_no_shrink_intrin_nearbyint(
-; ALL-NEXT:    [[E:%.*]] = call double @llvm.nearbyint.f64(double [[D:%.*]])
-; ALL-NEXT:    [[F:%.*]] = fptrunc double [[E]] to float
-; ALL-NEXT:    ret float [[F]]
+; CHECK-LABEL: @test_no_shrink_intrin_nearbyint(
+; CHECK-NEXT:    [[E:%.*]] = call double @llvm.nearbyint.f64(double [[D:%.*]])
+; CHECK-NEXT:    [[F:%.*]] = fptrunc double [[E]] to float
+; CHECK-NEXT:    ret float [[F]]
 ;
   %E = call double @llvm.nearbyint.f64(double %D)
   %F = fptrunc double %E to float
@@ -400,10 +364,10 @@ define float @test_no_shrink_intrin_nearbyint(double %D) {
 }
 
 define float @test_no_shrink_intrin_trunc(double %D) {
-; ALL-LABEL: @test_no_shrink_intrin_trunc(
-; ALL-NEXT:    [[E:%.*]] = call double @llvm.trunc.f64(double [[D:%.*]])
-; ALL-NEXT:    [[F:%.*]] = fptrunc double [[E]] to float
-; ALL-NEXT:    ret float [[F]]
+; CHECK-LABEL: @test_no_shrink_intrin_trunc(
+; CHECK-NEXT:    [[E:%.*]] = call double @llvm.trunc.f64(double [[D:%.*]])
+; CHECK-NEXT:    [[F:%.*]] = fptrunc double [[E]] to float
+; CHECK-NEXT:    ret float [[F]]
 ;
   %E = call double @llvm.trunc.f64(double %D)
   %F = fptrunc double %E to float
@@ -411,10 +375,10 @@ define float @test_no_shrink_intrin_trunc(double %D) {
 }
 
 define float @test_shrink_intrin_fabs_double_src(double %D) {
-; ALL-LABEL: @test_shrink_intrin_fabs_double_src(
-; ALL-NEXT:    [[TMP1:%.*]] = fptrunc double [[D:%.*]] to float
-; ALL-NEXT:    [[F:%.*]] = call float @llvm.fabs.f32(float [[TMP1]])
-; ALL-NEXT:    ret float [[F]]
+; CHECK-LABEL: @test_shrink_intrin_fabs_double_src(
+; CHECK-NEXT:    [[TMP1:%.*]] = fptrunc double [[D:%.*]] to float
+; CHECK-NEXT:    [[F:%.*]] = call float @llvm.fabs.f32(float [[TMP1]])
+; CHECK-NEXT:    ret float [[F]]
 ;
   %E = call double @llvm.fabs.f64(double %D)
   %F = fptrunc double %E to float
@@ -423,10 +387,10 @@ define float @test_shrink_intrin_fabs_double_src(double %D) {
 
 ; Make sure fast math flags are preserved
 define float @test_shrink_intrin_fabs_fast_double_src(double %D) {
-; ALL-LABEL: @test_shrink_intrin_fabs_fast_double_src(
-; ALL-NEXT:    [[TMP1:%.*]] = fptrunc double [[D:%.*]] to float
-; ALL-NEXT:    [[F:%.*]] = call fast float @llvm.fabs.f32(float [[TMP1]])
-; ALL-NEXT:    ret float [[F]]
+; CHECK-LABEL: @test_shrink_intrin_fabs_fast_double_src(
+; CHECK-NEXT:    [[TMP1:%.*]] = fptrunc double [[D:%.*]] to float
+; CHECK-NEXT:    [[F:%.*]] = call fast float @llvm.fabs.f32(float [[TMP1]])
+; CHECK-NEXT:    ret float [[F]]
 ;
   %E = call fast double @llvm.fabs.f64(double %D)
   %F = fptrunc double %E to float
@@ -434,8 +398,8 @@ define float @test_shrink_intrin_fabs_fast_double_src(double %D) {
 }
 
 define float @test_shrink_float_convertible_constant_intrin_floor() {
-; ALL-LABEL: @test_shrink_float_convertible_constant_intrin_floor(
-; ALL-NEXT:    ret float 2.000000e+00
+; CHECK-LABEL: @test_shrink_float_convertible_constant_intrin_floor(
+; CHECK-NEXT:    ret float 2.000000e+00
 ;
   %E = call double @llvm.floor.f64(double 2.1)
   %F = fptrunc double %E to float
@@ -443,8 +407,8 @@ define float @test_shrink_float_convertible_constant_intrin_floor() {
 }
 
 define float @test_shrink_float_convertible_constant_intrin_ceil() {
-; ALL-LABEL: @test_shrink_float_convertible_constant_intrin_ceil(
-; ALL-NEXT:    ret float 3.000000e+00
+; CHECK-LABEL: @test_shrink_float_convertible_constant_intrin_ceil(
+; CHECK-NEXT:    ret float 3.000000e+00
 ;
   %E = call double @llvm.ceil.f64(double 2.1)
   %F = fptrunc double %E to float
@@ -452,8 +416,8 @@ define float @test_shrink_float_convertible_constant_intrin_ceil() {
 }
 
 define float @test_shrink_float_convertible_constant_intrin_round() {
-; ALL-LABEL: @test_shrink_float_convertible_constant_intrin_round(
-; ALL-NEXT:    ret float 2.000000e+00
+; CHECK-LABEL: @test_shrink_float_convertible_constant_intrin_round(
+; CHECK-NEXT:    ret float 2.000000e+00
 ;
   %E = call double @llvm.round.f64(double 2.1)
   %F = fptrunc double %E to float
@@ -461,8 +425,8 @@ define float @test_shrink_float_convertible_constant_intrin_round() {
 }
 
 define float @test_shrink_float_convertible_constant_intrin_nearbyint() {
-; ALL-LABEL: @test_shrink_float_convertible_constant_intrin_nearbyint(
-; ALL-NEXT:    ret float 2.000000e+00
+; CHECK-LABEL: @test_shrink_float_convertible_constant_intrin_nearbyint(
+; CHECK-NEXT:    ret float 2.000000e+00
 ;
   %E = call double @llvm.nearbyint.f64(double 2.1)
   %F = fptrunc double %E to float
@@ -470,8 +434,8 @@ define float @test_shrink_float_convertible_constant_intrin_nearbyint() {
 }
 
 define float @test_shrink_float_convertible_constant_intrin_trunc() {
-; ALL-LABEL: @test_shrink_float_convertible_constant_intrin_trunc(
-; ALL-NEXT:    ret float 2.000000e+00
+; CHECK-LABEL: @test_shrink_float_convertible_constant_intrin_trunc(
+; CHECK-NEXT:    ret float 2.000000e+00
 ;
   %E = call double @llvm.trunc.f64(double 2.1)
   %F = fptrunc double %E to float
@@ -479,8 +443,8 @@ define float @test_shrink_float_convertible_constant_intrin_trunc() {
 }
 
 define float @test_shrink_float_convertible_constant_intrin_fabs() {
-; ALL-LABEL: @test_shrink_float_convertible_constant_intrin_fabs(
-; ALL-NEXT:    ret float 0x4000CCCCC0000000
+; CHECK-LABEL: @test_shrink_float_convertible_constant_intrin_fabs(
+; CHECK-NEXT:    ret float 0x4000CCCCC0000000
 ;
   %E = call double @llvm.fabs.f64(double 2.1)
   %F = fptrunc double %E to float
@@ -489,8 +453,8 @@ define float @test_shrink_float_convertible_constant_intrin_fabs() {
 
 ; Make sure fast math flags are preserved
 define float @test_shrink_float_convertible_constant_intrin_fabs_fast() {
-; ALL-LABEL: @test_shrink_float_convertible_constant_intrin_fabs_fast(
-; ALL-NEXT:    ret float 0x4000CCCCC0000000
+; CHECK-LABEL: @test_shrink_float_convertible_constant_intrin_fabs_fast(
+; CHECK-NEXT:    ret float 0x4000CCCCC0000000
 ;
   %E = call fast double @llvm.fabs.f64(double 2.1)
   %F = fptrunc double %E to float
@@ -498,10 +462,10 @@ define float @test_shrink_float_convertible_constant_intrin_fabs_fast() {
 }
 
 define half @test_no_shrink_mismatched_type_intrin_floor(double %D) {
-; ALL-LABEL: @test_no_shrink_mismatched_type_intrin_floor(
-; ALL-NEXT:    [[E:%.*]] = call double @llvm.floor.f64(double [[D:%.*]])
-; ALL-NEXT:    [[F:%.*]] = fptrunc double [[E]] to half
-; ALL-NEXT:    ret half [[F]]
+; CHECK-LABEL: @test_no_shrink_mismatched_type_intrin_floor(
+; CHECK-NEXT:    [[E:%.*]] = call double @llvm.floor.f64(double [[D:%.*]])
+; CHECK-NEXT:    [[F:%.*]] = fptrunc double [[E]] to half
+; CHECK-NEXT:    ret half [[F]]
 ;
   %E = call double @llvm.floor.f64(double %D)
   %F = fptrunc double %E to half
@@ -509,10 +473,10 @@ define half @test_no_shrink_mismatched_type_intrin_floor(double %D) {
 }
 
 define half @test_no_shrink_mismatched_type_intrin_ceil(double %D) {
-; ALL-LABEL: @test_no_shrink_mismatched_type_intrin_ceil(
-; ALL-NEXT:    [[E:%.*]] = call double @llvm.ceil.f64(double [[D:%.*]])
-; ALL-NEXT:    [[F:%.*]] = fptrunc double [[E]] to half
-; ALL-NEXT:    ret half [[F]]
+; CHECK-LABEL: @test_no_shrink_mismatched_type_intrin_ceil(
+; CHECK-NEXT:    [[E:%.*]] = call double @llvm.ceil.f64(double [[D:%.*]])
+; CHECK-NEXT:    [[F:%.*]] = fptrunc double [[E]] to half
+; CHECK-NEXT:    ret half [[F]]
 ;
   %E = call double @llvm.ceil.f64(double %D)
   %F = fptrunc double %E to half
@@ -520,10 +484,10 @@ define half @test_no_shrink_mismatched_type_intrin_ceil(double %D) {
 }
 
 define half @test_no_shrink_mismatched_type_intrin_round(double %D) {
-; ALL-LABEL: @test_no_shrink_mismatched_type_intrin_round(
-; ALL-NEXT:    [[E:%.*]] = call double @llvm.round.f64(double [[D:%.*]])
-; ALL-NEXT:    [[F:%.*]] = fptrunc double [[E]] to half
-; ALL-NEXT:    ret half [[F]]
+; CHECK-LABEL: @test_no_shrink_mismatched_type_intrin_round(
+; CHECK-NEXT:    [[E:%.*]] = call double @llvm.round.f64(double [[D:%.*]])
+; CHECK-NEXT:    [[F:%.*]] = fptrunc double [[E]] to half
+; CHECK-NEXT:    ret half [[F]]
 ;
   %E = call double @llvm.round.f64(double %D)
   %F = fptrunc double %E to half
@@ -531,10 +495,10 @@ define half @test_no_shrink_mismatched_type_intrin_round(double %D) {
 }
 
 define half @test_no_shrink_mismatched_type_intrin_nearbyint(double %D) {
-; ALL-LABEL: @test_no_shrink_mismatched_type_intrin_nearbyint(
-; ALL-NEXT:    [[E:%.*]] = call double @llvm.nearbyint.f64(double [[D:%.*]])
-; ALL-NEXT:    [[F:%.*]] = fptrunc double [[E]] to half
-; ALL-NEXT:    ret half [[F]]
+; CHECK-LABEL: @test_no_shrink_mismatched_type_intrin_nearbyint(
+; CHECK-NEXT:    [[E:%.*]] = call double @llvm.nearbyint.f64(double [[D:%.*]])
+; CHECK-NEXT:    [[F:%.*]] = fptrunc double [[E]] to half
+; CHECK-NEXT:    ret half [[F]]
 ;
   %E = call double @llvm.nearbyint.f64(double %D)
   %F = fptrunc double %E to half
@@ -542,10 +506,10 @@ define half @test_no_shrink_mismatched_type_intrin_nearbyint(double %D) {
 }
 
 define half @test_no_shrink_mismatched_type_intrin_trunc(double %D) {
-; ALL-LABEL: @test_no_shrink_mismatched_type_intrin_trunc(
-; ALL-NEXT:    [[E:%.*]] = call double @llvm.trunc.f64(double [[D:%.*]])
-; ALL-NEXT:    [[F:%.*]] = fptrunc double [[E]] to half
-; ALL-NEXT:    ret half [[F]]
+; CHECK-LABEL: @test_no_shrink_mismatched_type_intrin_trunc(
+; CHECK-NEXT:    [[E:%.*]] = call double @llvm.trunc.f64(double [[D:%.*]])
+; CHECK-NEXT:    [[F:%.*]] = fptrunc double [[E]] to half
+; CHECK-NEXT:    ret half [[F]]
 ;
   %E = call double @llvm.trunc.f64(double %D)
   %F = fptrunc double %E to half
@@ -553,10 +517,10 @@ define half @test_no_shrink_mismatched_type_intrin_trunc(double %D) {
 }
 
 define half @test_shrink_mismatched_type_intrin_fabs_double_src(double %D) {
-; ALL-LABEL: @test_shrink_mismatched_type_intrin_fabs_double_src(
-; ALL-NEXT:    [[TMP1:%.*]] = fptrunc double [[D:%.*]] to half
-; ALL-NEXT:    [[F:%.*]] = call half @llvm.fabs.f16(half [[TMP1]])
-; ALL-NEXT:    ret half [[F]]
+; CHECK-LABEL: @test_shrink_mismatched_type_intrin_fabs_double_src(
+; CHECK-NEXT:    [[TMP1:%.*]] = fptrunc double [[D:%.*]] to half
+; CHECK-NEXT:    [[F:%.*]] = call half @llvm.fabs.f16(half [[TMP1]])
+; CHECK-NEXT:    ret half [[F]]
 ;
   %E = call double @llvm.fabs.f64(double %D)
   %F = fptrunc double %E to half
@@ -565,10 +529,10 @@ define half @test_shrink_mismatched_type_intrin_fabs_double_src(double %D) {
 
 ; Make sure fast math flags are preserved
 define half @test_mismatched_type_intrin_fabs_fast_double_src(double %D) {
-; ALL-LABEL: @test_mismatched_type_intrin_fabs_fast_double_src(
-; ALL-NEXT:    [[TMP1:%.*]] = fptrunc double [[D:%.*]] to half
-; ALL-NEXT:    [[F:%.*]] = call fast half @llvm.fabs.f16(half [[TMP1]])
-; ALL-NEXT:    ret half [[F]]
+; CHECK-LABEL: @test_mismatched_type_intrin_fabs_fast_double_src(
+; CHECK-NEXT:    [[TMP1:%.*]] = fptrunc double [[D:%.*]] to half
+; CHECK-NEXT:    [[F:%.*]] = call fast half @llvm.fabs.f16(half [[TMP1]])
+; CHECK-NEXT:    ret half [[F]]
 ;
   %E = call fast double @llvm.fabs.f64(double %D)
   %F = fptrunc double %E to half
@@ -576,10 +540,10 @@ define half @test_mismatched_type_intrin_fabs_fast_double_src(double %D) {
 }
 
 define <2 x double> @test_shrink_intrin_floor_fp16_vec(<2 x half> %C) {
-; ALL-LABEL: @test_shrink_intrin_floor_fp16_vec(
-; ALL-NEXT:    [[TMP1:%.*]] = call arcp <2 x half> @llvm.floor.v2f16(<2 x half> [[C:%.*]])
-; ALL-NEXT:    [[E:%.*]] = fpext <2 x half> [[TMP1]] to <2 x double>
-; ALL-NEXT:    ret <2 x double> [[E]]
+; CHECK-LABEL: @test_shrink_intrin_floor_fp16_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = call arcp <2 x half> @llvm.floor.v2f16(<2 x half> [[C:%.*]])
+; CHECK-NEXT:    [[E:%.*]] = fpext <2 x half> [[TMP1]] to <2 x double>
+; CHECK-NEXT:    ret <2 x double> [[E]]
 ;
   %D = fpext <2 x half> %C to <2 x double>
   %E = call arcp <2 x double> @llvm.floor.v2f64(<2 x double> %D)
@@ -587,10 +551,10 @@ define <2 x double> @test_shrink_intrin_floor_fp16_vec(<2 x half> %C) {
 }
 
 define float @test_shrink_intrin_ceil_fp16_src(half %C) {
-; ALL-LABEL: @test_shrink_intrin_ceil_fp16_src(
-; ALL-NEXT:    [[TMP1:%.*]] = call half @llvm.ceil.f16(half [[C:%.*]])
-; ALL-NEXT:    [[F:%.*]] = fpext half [[TMP1]] to float
-; ALL-NEXT:    ret float [[F]]
+; CHECK-LABEL: @test_shrink_intrin_ceil_fp16_src(
+; CHECK-NEXT:    [[TMP1:%.*]] = call half @llvm.ceil.f16(half [[C:%.*]])
+; CHECK-NEXT:    [[F:%.*]] = fpext half [[TMP1]] to float
+; CHECK-NEXT:    ret float [[F]]
 ;
   %D = fpext half %C to double
   %E = call double @llvm.ceil.f64(double %D)
@@ -599,10 +563,10 @@ define float @test_shrink_intrin_ceil_fp16_src(half %C) {
 }
 
 define <2 x double> @test_shrink_intrin_round_fp16_vec(<2 x half> %C) {
-; ALL-LABEL: @test_shrink_intrin_round_fp16_vec(
-; ALL-NEXT:    [[TMP1:%.*]] = call <2 x half> @llvm.round.v2f16(<2 x half> [[C:%.*]])
-; ALL-NEXT:    [[E:%.*]] = fpext <2 x half> [[TMP1]] to <2 x double>
-; ALL-NEXT:    ret <2 x double> [[E]]
+; CHECK-LABEL: @test_shrink_intrin_round_fp16_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = call <2 x half> @llvm.round.v2f16(<2 x half> [[C:%.*]])
+; CHECK-NEXT:    [[E:%.*]] = fpext <2 x half> [[TMP1]] to <2 x double>
+; CHECK-NEXT:    ret <2 x double> [[E]]
 ;
   %D = fpext <2 x  half> %C to <2 x double>
   %E = call <2 x double> @llvm.round.v2f64(<2 x double> %D)
@@ -610,10 +574,10 @@ define <2 x double> @test_shrink_intrin_round_fp16_vec(<2 x half> %C) {
 }
 
 define float @test_shrink_intrin_nearbyint_fp16_src(half %C) {
-; ALL-LABEL: @test_shrink_intrin_nearbyint_fp16_src(
-; ALL-NEXT:    [[TMP1:%.*]] = call half @llvm.nearbyint.f16(half [[C:%.*]])
-; ALL-NEXT:    [[F:%.*]] = fpext half [[TMP1]] to float
-; ALL-NEXT:    ret float [[F]]
+; CHECK-LABEL: @test_shrink_intrin_nearbyint_fp16_src(
+; CHECK-NEXT:    [[TMP1:%.*]] = call half @llvm.nearbyint.f16(half [[C:%.*]])
+; CHECK-NEXT:    [[F:%.*]] = fpext half [[TMP1]] to float
+; CHECK-NEXT:    ret float [[F]]
 ;
   %D = fpext half %C to double
   %E = call double @llvm.nearbyint.f64(double %D)
@@ -622,10 +586,10 @@ define float @test_shrink_intrin_nearbyint_fp16_src(half %C) {
 }
 
 define <2 x double> @test_shrink_intrin_trunc_fp16_src(<2 x half> %C) {
-; ALL-LABEL: @test_shrink_intrin_trunc_fp16_src(
-; ALL-NEXT:    [[TMP1:%.*]] = call <2 x half> @llvm.trunc.v2f16(<2 x half> [[C:%.*]])
-; ALL-NEXT:    [[E:%.*]] = fpext <2 x half> [[TMP1]] to <2 x double>
-; ALL-NEXT:    ret <2 x double> [[E]]
+; CHECK-LABEL: @test_shrink_intrin_trunc_fp16_src(
+; CHECK-NEXT:    [[TMP1:%.*]] = call <2 x half> @llvm.trunc.v2f16(<2 x half> [[C:%.*]])
+; CHECK-NEXT:    [[E:%.*]] = fpext <2 x half> [[TMP1]] to <2 x double>
+; CHECK-NEXT:    ret <2 x double> [[E]]
 ;
   %D = fpext <2 x half> %C to <2 x double>
   %E = call <2 x double> @llvm.trunc.v2f64(<2 x double> %D)
@@ -633,10 +597,10 @@ define <2 x double> @test_shrink_intrin_trunc_fp16_src(<2 x half> %C) {
 }
 
 define float @test_shrink_intrin_fabs_fp16_src(half %C) {
-; ALL-LABEL: @test_shrink_intrin_fabs_fp16_src(
-; ALL-NEXT:    [[TMP1:%.*]] = call half @llvm.fabs.f16(half [[C:%.*]])
-; ALL-NEXT:    [[F:%.*]] = fpext half [[TMP1]] to float
-; ALL-NEXT:    ret float [[F]]
+; CHECK-LABEL: @test_shrink_intrin_fabs_fp16_src(
+; CHECK-NEXT:    [[TMP1:%.*]] = call half @llvm.fabs.f16(half [[C:%.*]])
+; CHECK-NEXT:    [[F:%.*]] = fpext half [[TMP1]] to float
+; CHECK-NEXT:    ret float [[F]]
 ;
   %D = fpext half %C to double
   %E = call double @llvm.fabs.f64(double %D)
@@ -646,10 +610,10 @@ define float @test_shrink_intrin_fabs_fp16_src(half %C) {
 
 ; Make sure fast math flags are preserved
 define float @test_shrink_intrin_fabs_fast_fp16_src(half %C) {
-; ALL-LABEL: @test_shrink_intrin_fabs_fast_fp16_src(
-; ALL-NEXT:    [[TMP1:%.*]] = call fast half @llvm.fabs.f16(half [[C:%.*]])
-; ALL-NEXT:    [[F:%.*]] = fpext half [[TMP1]] to float
-; ALL-NEXT:    ret float [[F]]
+; CHECK-LABEL: @test_shrink_intrin_fabs_fast_fp16_src(
+; CHECK-NEXT:    [[TMP1:%.*]] = call fast half @llvm.fabs.f16(half [[C:%.*]])
+; CHECK-NEXT:    [[F:%.*]] = fpext half [[TMP1]] to float
+; CHECK-NEXT:    ret float [[F]]
 ;
   %D = fpext half %C to double
   %E = call fast double @llvm.fabs.f64(double %D)
@@ -658,12 +622,12 @@ define float @test_shrink_intrin_fabs_fast_fp16_src(half %C) {
 }
 
 define float @test_no_shrink_intrin_floor_multi_use_fpext(half %C) {
-; ALL-LABEL: @test_no_shrink_intrin_floor_multi_use_fpext(
-; ALL-NEXT:    [[D:%.*]] = fpext half [[C:%.*]] to double
-; ALL-NEXT:    store volatile double [[D]], double* undef, align 8
-; ALL-NEXT:    [[E:%.*]] = call double @llvm.floor.f64(double [[D]])
-; ALL-NEXT:    [[F:%.*]] = fptrunc double [[E]] to float
-; ALL-NEXT:    ret float [[F]]
+; CHECK-LABEL: @test_no_shrink_intrin_floor_multi_use_fpext(
+; CHECK-NEXT:    [[D:%.*]] = fpext half [[C:%.*]] to double
+; CHECK-NEXT:    store volatile double [[D]], double* undef, align 8
+; CHECK-NEXT:    [[E:%.*]] = call double @llvm.floor.f64(double [[D]])
+; CHECK-NEXT:    [[F:%.*]] = fptrunc double [[E]] to float
+; CHECK-NEXT:    ret float [[F]]
 ;
   %D = fpext half %C to double
   store volatile double %D, double* undef
@@ -673,12 +637,12 @@ define float @test_no_shrink_intrin_floor_multi_use_fpext(half %C) {
 }
 
 define float @test_no_shrink_intrin_fabs_multi_use_fpext(half %C) {
-; ALL-LABEL: @test_no_shrink_intrin_fabs_multi_use_fpext(
-; ALL-NEXT:    [[D:%.*]] = fpext half [[C:%.*]] to double
-; ALL-NEXT:    store volatile double [[D]], double* undef, align 8
-; ALL-NEXT:    [[E:%.*]] = call double @llvm.fabs.f64(double [[D]])
-; ALL-NEXT:    [[F:%.*]] = fptrunc double [[E]] to float
-; ALL-NEXT:    ret float [[F]]
+; CHECK-LABEL: @test_no_shrink_intrin_fabs_multi_use_fpext(
+; CHECK-NEXT:    [[D:%.*]] = fpext half [[C:%.*]] to double
+; CHECK-NEXT:    store volatile double [[D]], double* undef, align 8
+; CHECK-NEXT:    [[E:%.*]] = call double @llvm.fabs.f64(double [[D]])
+; CHECK-NEXT:    [[F:%.*]] = fptrunc double [[E]] to float
+; CHECK-NEXT:    ret float [[F]]
 ;
   %D = fpext half %C to double
   store volatile double %D, double* undef
diff --git a/test/Transforms/InstCombine/element-atomic-memintrins.ll b/test/Transforms/InstCombine/element-atomic-memintrins.ll
index 7467bc7f6c7425cfc47d92a5b87a760924f66c44..6bc62c94e3260309ebd23ead033b19e86ee267a1 100644
--- a/test/Transforms/InstCombine/element-atomic-memintrins.ll
+++ b/test/Transforms/InstCombine/element-atomic-memintrins.ll
@@ -15,12 +15,9 @@ define void @test_memset_zero_length(i8* %dest) {
 define void @test_memset_to_store(i8* %dest) {
 ; CHECK-LABEL: @test_memset_to_store(
 ; CHECK-NEXT:    store atomic i8 1, i8* [[DEST:%.*]] unordered, align 1
-; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i8* [[DEST]] to i16*
-; CHECK-NEXT:    store atomic i16 257, i16* [[TMP1]] unordered, align 1
-; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8* [[DEST]] to i32*
-; CHECK-NEXT:    store atomic i32 16843009, i32* [[TMP2]] unordered, align 1
-; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i8* [[DEST]] to i64*
-; CHECK-NEXT:    store atomic i64 72340172838076673, i64* [[TMP3]] unordered, align 1
+; CHECK-NEXT:    call void @llvm.memset.element.unordered.atomic.p0i8.i32(i8* align 1 [[DEST]], i8 1, i32 2, i32 1)
+; CHECK-NEXT:    call void @llvm.memset.element.unordered.atomic.p0i8.i32(i8* align 1 [[DEST]], i8 1, i32 4, i32 1)
+; CHECK-NEXT:    call void @llvm.memset.element.unordered.atomic.p0i8.i32(i8* align 1 [[DEST]], i8 1, i32 8, i32 1)
 ; CHECK-NEXT:    call void @llvm.memset.element.unordered.atomic.p0i8.i32(i8* align 1 [[DEST]], i8 1, i32 16, i32 1)
 ; CHECK-NEXT:    ret void
 ;
@@ -34,15 +31,15 @@ define void @test_memset_to_store(i8* %dest) {
 
 define void @test_memset_to_store_2(i8* %dest) {
 ; CHECK-LABEL: @test_memset_to_store_2(
-; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i8* [[DEST:%.*]] to i16*
+; CHECK-NEXT:    store atomic i8 1, i8* [[DEST:%.*]] unordered, align 2
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i8* [[DEST]] to i16*
 ; CHECK-NEXT:    store atomic i16 257, i16* [[TMP1]] unordered, align 2
-; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8* [[DEST]] to i32*
-; CHECK-NEXT:    store atomic i32 16843009, i32* [[TMP2]] unordered, align 2
-; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i8* [[DEST]] to i64*
-; CHECK-NEXT:    store atomic i64 72340172838076673, i64* [[TMP3]] unordered, align 2
+; CHECK-NEXT:    call void @llvm.memset.element.unordered.atomic.p0i8.i32(i8* align 2 [[DEST]], i8 1, i32 4, i32 2)
+; CHECK-NEXT:    call void @llvm.memset.element.unordered.atomic.p0i8.i32(i8* align 2 [[DEST]], i8 1, i32 8, i32 2)
 ; CHECK-NEXT:    call void @llvm.memset.element.unordered.atomic.p0i8.i32(i8* align 2 [[DEST]], i8 1, i32 16, i32 2)
 ; CHECK-NEXT:    ret void
 ;
+  call void @llvm.memset.element.unordered.atomic.p0i8.i32(i8* align 2 %dest, i8 1, i32 1, i32 1)
   call void @llvm.memset.element.unordered.atomic.p0i8.i32(i8* align 2 %dest, i8 1, i32 2, i32 2)
   call void @llvm.memset.element.unordered.atomic.p0i8.i32(i8* align 2 %dest, i8 1, i32 4, i32 2)
   call void @llvm.memset.element.unordered.atomic.p0i8.i32(i8* align 2 %dest, i8 1, i32 8, i32 2)
@@ -52,13 +49,17 @@ define void @test_memset_to_store_2(i8* %dest) {
 
 define void @test_memset_to_store_4(i8* %dest) {
 ; CHECK-LABEL: @test_memset_to_store_4(
-; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i8* [[DEST:%.*]] to i32*
-; CHECK-NEXT:    store atomic i32 16843009, i32* [[TMP1]] unordered, align 4
-; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8* [[DEST]] to i64*
-; CHECK-NEXT:    store atomic i64 72340172838076673, i64* [[TMP2]] unordered, align 4
+; CHECK-NEXT:    store atomic i8 1, i8* [[DEST:%.*]] unordered, align 4
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i8* [[DEST]] to i16*
+; CHECK-NEXT:    store atomic i16 257, i16* [[TMP1]] unordered, align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8* [[DEST]] to i32*
+; CHECK-NEXT:    store atomic i32 16843009, i32* [[TMP2]] unordered, align 4
+; CHECK-NEXT:    call void @llvm.memset.element.unordered.atomic.p0i8.i32(i8* align 4 [[DEST]], i8 1, i32 8, i32 4)
 ; CHECK-NEXT:    call void @llvm.memset.element.unordered.atomic.p0i8.i32(i8* align 4 [[DEST]], i8 1, i32 16, i32 4)
 ; CHECK-NEXT:    ret void
 ;
+  call void @llvm.memset.element.unordered.atomic.p0i8.i32(i8* align 4 %dest, i8 1, i32 1, i32 1)
+  call void @llvm.memset.element.unordered.atomic.p0i8.i32(i8* align 4 %dest, i8 1, i32 2, i32 2)
   call void @llvm.memset.element.unordered.atomic.p0i8.i32(i8* align 4 %dest, i8 1, i32 4, i32 4)
   call void @llvm.memset.element.unordered.atomic.p0i8.i32(i8* align 4 %dest, i8 1, i32 8, i32 4)
   call void @llvm.memset.element.unordered.atomic.p0i8.i32(i8* align 4 %dest, i8 1, i32 16, i32 4)
@@ -67,11 +68,19 @@ define void @test_memset_to_store_4(i8* %dest) {
 
 define void @test_memset_to_store_8(i8* %dest) {
 ; CHECK-LABEL: @test_memset_to_store_8(
-; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i8* [[DEST:%.*]] to i64*
-; CHECK-NEXT:    store atomic i64 72340172838076673, i64* [[TMP1]] unordered, align 8
+; CHECK-NEXT:    store atomic i8 1, i8* [[DEST:%.*]] unordered, align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i8* [[DEST]] to i16*
+; CHECK-NEXT:    store atomic i16 257, i16* [[TMP1]] unordered, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8* [[DEST]] to i32*
+; CHECK-NEXT:    store atomic i32 16843009, i32* [[TMP2]] unordered, align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i8* [[DEST]] to i64*
+; CHECK-NEXT:    store atomic i64 72340172838076673, i64* [[TMP3]] unordered, align 8
 ; CHECK-NEXT:    call void @llvm.memset.element.unordered.atomic.p0i8.i32(i8* align 8 [[DEST]], i8 1, i32 16, i32 8)
 ; CHECK-NEXT:    ret void
 ;
+  call void @llvm.memset.element.unordered.atomic.p0i8.i32(i8* align 8 %dest, i8 1, i32 1, i32 1)
+  call void @llvm.memset.element.unordered.atomic.p0i8.i32(i8* align 8 %dest, i8 1, i32 2, i32 2)
+  call void @llvm.memset.element.unordered.atomic.p0i8.i32(i8* align 8 %dest, i8 1, i32 4, i32 4)
   call void @llvm.memset.element.unordered.atomic.p0i8.i32(i8* align 8 %dest, i8 1, i32 8, i32 8)
   call void @llvm.memset.element.unordered.atomic.p0i8.i32(i8* align 8 %dest, i8 1, i32 16, i32 8)
   ret void
@@ -79,9 +88,20 @@ define void @test_memset_to_store_8(i8* %dest) {
 
 define void @test_memset_to_store_16(i8* %dest) {
 ; CHECK-LABEL: @test_memset_to_store_16(
-; CHECK-NEXT:    call void @llvm.memset.element.unordered.atomic.p0i8.i32(i8* align 16 [[DEST:%.*]], i8 1, i32 16, i32 16)
+; CHECK-NEXT:    store atomic i8 1, i8* [[DEST:%.*]] unordered, align 16
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i8* [[DEST]] to i16*
+; CHECK-NEXT:    store atomic i16 257, i16* [[TMP1]] unordered, align 16
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8* [[DEST]] to i32*
+; CHECK-NEXT:    store atomic i32 16843009, i32* [[TMP2]] unordered, align 16
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i8* [[DEST]] to i64*
+; CHECK-NEXT:    store atomic i64 72340172838076673, i64* [[TMP3]] unordered, align 16
+; CHECK-NEXT:    call void @llvm.memset.element.unordered.atomic.p0i8.i32(i8* align 16 [[DEST]], i8 1, i32 16, i32 16)
 ; CHECK-NEXT:    ret void
 ;
+  call void @llvm.memset.element.unordered.atomic.p0i8.i32(i8* align 16 %dest, i8 1, i32 1, i32 1)
+  call void @llvm.memset.element.unordered.atomic.p0i8.i32(i8* align 16 %dest, i8 1, i32 2, i32 2)
+  call void @llvm.memset.element.unordered.atomic.p0i8.i32(i8* align 16 %dest, i8 1, i32 4, i32 4)
+  call void @llvm.memset.element.unordered.atomic.p0i8.i32(i8* align 16 %dest, i8 1, i32 8, i32 8)
   call void @llvm.memset.element.unordered.atomic.p0i8.i32(i8* align 16 %dest, i8 1, i32 16, i32 16)
   ret void
 }
@@ -134,18 +154,9 @@ define void @test_memmove_loadstore(i8* %dest, i8* %src) {
 ; CHECK-LABEL: @test_memmove_loadstore(
 ; CHECK-NEXT:    [[TMP1:%.*]] = load atomic i8, i8* [[SRC:%.*]] unordered, align 1
 ; CHECK-NEXT:    store atomic i8 [[TMP1]], i8* [[DEST:%.*]] unordered, align 1
-; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8* [[SRC]] to i16*
-; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i8* [[DEST]] to i16*
-; CHECK-NEXT:    [[TMP4:%.*]] = load atomic i16, i16* [[TMP2]] unordered, align 1
-; CHECK-NEXT:    store atomic i16 [[TMP4]], i16* [[TMP3]] unordered, align 1
-; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8* [[SRC]] to i32*
-; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i8* [[DEST]] to i32*
-; CHECK-NEXT:    [[TMP7:%.*]] = load atomic i32, i32* [[TMP5]] unordered, align 1
-; CHECK-NEXT:    store atomic i32 [[TMP7]], i32* [[TMP6]] unordered, align 1
-; CHECK-NEXT:    [[TMP8:%.*]] = bitcast i8* [[SRC]] to i64*
-; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i8* [[DEST]] to i64*
-; CHECK-NEXT:    [[TMP10:%.*]] = load atomic i64, i64* [[TMP8]] unordered, align 1
-; CHECK-NEXT:    store atomic i64 [[TMP10]], i64* [[TMP9]] unordered, align 1
+; CHECK-NEXT:    call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i32(i8* align 1 [[DEST]], i8* align 1 [[SRC]], i32 2, i32 1)
+; CHECK-NEXT:    call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i32(i8* align 1 [[DEST]], i8* align 1 [[SRC]], i32 4, i32 1)
+; CHECK-NEXT:    call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i32(i8* align 1 [[DEST]], i8* align 1 [[SRC]], i32 8, i32 1)
 ; CHECK-NEXT:    call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i32(i8* align 1 [[DEST]], i8* align 1 [[SRC]], i32 16, i32 1)
 ; CHECK-NEXT:    ret void
 ;
@@ -159,21 +170,18 @@ define void @test_memmove_loadstore(i8* %dest, i8* %src) {
 
 define void @test_memmove_loadstore_2(i8* %dest, i8* %src) {
 ; CHECK-LABEL: @test_memmove_loadstore_2(
-; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i8* [[SRC:%.*]] to i16*
-; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8* [[DEST:%.*]] to i16*
-; CHECK-NEXT:    [[TMP3:%.*]] = load atomic i16, i16* [[TMP1]] unordered, align 2
-; CHECK-NEXT:    store atomic i16 [[TMP3]], i16* [[TMP2]] unordered, align 2
-; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i8* [[SRC]] to i32*
-; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8* [[DEST]] to i32*
-; CHECK-NEXT:    [[TMP6:%.*]] = load atomic i32, i32* [[TMP4]] unordered, align 2
-; CHECK-NEXT:    store atomic i32 [[TMP6]], i32* [[TMP5]] unordered, align 2
-; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i8* [[SRC]] to i64*
-; CHECK-NEXT:    [[TMP8:%.*]] = bitcast i8* [[DEST]] to i64*
-; CHECK-NEXT:    [[TMP9:%.*]] = load atomic i64, i64* [[TMP7]] unordered, align 2
-; CHECK-NEXT:    store atomic i64 [[TMP9]], i64* [[TMP8]] unordered, align 2
+; CHECK-NEXT:    [[TMP1:%.*]] = load atomic i8, i8* [[SRC:%.*]] unordered, align 2
+; CHECK-NEXT:    store atomic i8 [[TMP1]], i8* [[DEST:%.*]] unordered, align 2
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8* [[SRC]] to i16*
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i8* [[DEST]] to i16*
+; CHECK-NEXT:    [[TMP4:%.*]] = load atomic i16, i16* [[TMP2]] unordered, align 2
+; CHECK-NEXT:    store atomic i16 [[TMP4]], i16* [[TMP3]] unordered, align 2
+; CHECK-NEXT:    call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i32(i8* align 2 [[DEST]], i8* align 2 [[SRC]], i32 4, i32 2)
+; CHECK-NEXT:    call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i32(i8* align 2 [[DEST]], i8* align 2 [[SRC]], i32 8, i32 2)
 ; CHECK-NEXT:    call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i32(i8* align 2 [[DEST]], i8* align 2 [[SRC]], i32 16, i32 2)
 ; CHECK-NEXT:    ret void
 ;
+  call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i32(i8* align 2 %dest, i8* align 2 %src, i32 1, i32 1)
   call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i32(i8* align 2 %dest, i8* align 2 %src, i32 2, i32 2)
   call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i32(i8* align 2 %dest, i8* align 2 %src, i32 4, i32 2)
   call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i32(i8* align 2 %dest, i8* align 2 %src, i32 8, i32 2)
@@ -183,17 +191,22 @@ define void @test_memmove_loadstore_2(i8* %dest, i8* %src) {
 
 define void @test_memmove_loadstore_4(i8* %dest, i8* %src) {
 ; CHECK-LABEL: @test_memmove_loadstore_4(
-; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i8* [[SRC:%.*]] to i32*
-; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8* [[DEST:%.*]] to i32*
-; CHECK-NEXT:    [[TMP3:%.*]] = load atomic i32, i32* [[TMP1]] unordered, align 4
-; CHECK-NEXT:    store atomic i32 [[TMP3]], i32* [[TMP2]] unordered, align 4
-; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i8* [[SRC]] to i64*
-; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8* [[DEST]] to i64*
-; CHECK-NEXT:    [[TMP6:%.*]] = load atomic i64, i64* [[TMP4]] unordered, align 4
-; CHECK-NEXT:    store atomic i64 [[TMP6]], i64* [[TMP5]] unordered, align 4
+; CHECK-NEXT:    [[TMP1:%.*]] = load atomic i8, i8* [[SRC:%.*]] unordered, align 4
+; CHECK-NEXT:    store atomic i8 [[TMP1]], i8* [[DEST:%.*]] unordered, align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8* [[SRC]] to i16*
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i8* [[DEST]] to i16*
+; CHECK-NEXT:    [[TMP4:%.*]] = load atomic i16, i16* [[TMP2]] unordered, align 4
+; CHECK-NEXT:    store atomic i16 [[TMP4]], i16* [[TMP3]] unordered, align 4
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8* [[SRC]] to i32*
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i8* [[DEST]] to i32*
+; CHECK-NEXT:    [[TMP7:%.*]] = load atomic i32, i32* [[TMP5]] unordered, align 4
+; CHECK-NEXT:    store atomic i32 [[TMP7]], i32* [[TMP6]] unordered, align 4
+; CHECK-NEXT:    call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i32(i8* align 4 [[DEST]], i8* align 4 [[SRC]], i32 8, i32 4)
 ; CHECK-NEXT:    call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i32(i8* align 4 [[DEST]], i8* align 4 [[SRC]], i32 16, i32 4)
 ; CHECK-NEXT:    ret void
 ;
+  call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i32(i8* align 4 %dest, i8* align 4 %src, i32 1, i32 1)
+  call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i32(i8* align 4 %dest, i8* align 4 %src, i32 2, i32 2)
   call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i32(i8* align 4 %dest, i8* align 4 %src, i32 4, i32 4)
   call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i32(i8* align 4 %dest, i8* align 4 %src, i32 8, i32 4)
   call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i32(i8* align 4 %dest, i8* align 4 %src, i32 16, i32 4)
@@ -202,13 +215,26 @@ define void @test_memmove_loadstore_4(i8* %dest, i8* %src) {
 
 define void @test_memmove_loadstore_8(i8* %dest, i8* %src) {
 ; CHECK-LABEL: @test_memmove_loadstore_8(
-; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i8* [[SRC:%.*]] to i64*
-; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8* [[DEST:%.*]] to i64*
-; CHECK-NEXT:    [[TMP3:%.*]] = load atomic i64, i64* [[TMP1]] unordered, align 8
-; CHECK-NEXT:    store atomic i64 [[TMP3]], i64* [[TMP2]] unordered, align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = load atomic i8, i8* [[SRC:%.*]] unordered, align 8
+; CHECK-NEXT:    store atomic i8 [[TMP1]], i8* [[DEST:%.*]] unordered, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8* [[SRC]] to i16*
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i8* [[DEST]] to i16*
+; CHECK-NEXT:    [[TMP4:%.*]] = load atomic i16, i16* [[TMP2]] unordered, align 8
+; CHECK-NEXT:    store atomic i16 [[TMP4]], i16* [[TMP3]] unordered, align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8* [[SRC]] to i32*
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i8* [[DEST]] to i32*
+; CHECK-NEXT:    [[TMP7:%.*]] = load atomic i32, i32* [[TMP5]] unordered, align 8
+; CHECK-NEXT:    store atomic i32 [[TMP7]], i32* [[TMP6]] unordered, align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast i8* [[SRC]] to i64*
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i8* [[DEST]] to i64*
+; CHECK-NEXT:    [[TMP10:%.*]] = load atomic i64, i64* [[TMP8]] unordered, align 8
+; CHECK-NEXT:    store atomic i64 [[TMP10]], i64* [[TMP9]] unordered, align 8
 ; CHECK-NEXT:    call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i32(i8* align 8 [[DEST]], i8* align 8 [[SRC]], i32 16, i32 8)
 ; CHECK-NEXT:    ret void
 ;
+  call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i32(i8* align 8 %dest, i8* align 8 %src, i32 1, i32 1)
+  call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i32(i8* align 8 %dest, i8* align 8 %src, i32 2, i32 2)
+  call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i32(i8* align 8 %dest, i8* align 8 %src, i32 4, i32 4)
   call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i32(i8* align 8 %dest, i8* align 8 %src, i32 8, i32 8)
   call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i32(i8* align 8 %dest, i8* align 8 %src, i32 16, i32 8)
   ret void
@@ -216,9 +242,27 @@ define void @test_memmove_loadstore_8(i8* %dest, i8* %src) {
 
 define void @test_memmove_loadstore_16(i8* %dest, i8* %src) {
 ; CHECK-LABEL: @test_memmove_loadstore_16(
+; CHECK-NEXT:    [[TMP1:%.*]] = load atomic i8, i8* [[SRC:%.*]] unordered, align 16
+; CHECK-NEXT:    store atomic i8 [[TMP1]], i8* [[DEST:%.*]] unordered, align 16
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8* [[SRC]] to i16*
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i8* [[DEST]] to i16*
+; CHECK-NEXT:    [[TMP4:%.*]] = load atomic i16, i16* [[TMP2]] unordered, align 16
+; CHECK-NEXT:    store atomic i16 [[TMP4]], i16* [[TMP3]] unordered, align 16
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8* [[SRC]] to i32*
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i8* [[DEST]] to i32*
+; CHECK-NEXT:    [[TMP7:%.*]] = load atomic i32, i32* [[TMP5]] unordered, align 16
+; CHECK-NEXT:    store atomic i32 [[TMP7]], i32* [[TMP6]] unordered, align 16
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast i8* [[SRC]] to i64*
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i8* [[DEST]] to i64*
+; CHECK-NEXT:    [[TMP10:%.*]] = load atomic i64, i64* [[TMP8]] unordered, align 16
+; CHECK-NEXT:    store atomic i64 [[TMP10]], i64* [[TMP9]] unordered, align 16
 ; CHECK-NEXT:    call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i32(i8* align 16 [[DEST:%.*]], i8* align 16 [[SRC:%.*]], i32 16, i32 16)
 ; CHECK-NEXT:    ret void
 ;
+  call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i32(i8* align 16 %dest, i8* align 16 %src, i32 1, i32 1)
+  call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i32(i8* align 16 %dest, i8* align 16 %src, i32 2, i32 2)
+  call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i32(i8* align 16 %dest, i8* align 16 %src, i32 4, i32 4)
+  call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i32(i8* align 16 %dest, i8* align 16 %src, i32 8, i32 8)
   call void @llvm.memmove.element.unordered.atomic.p0i8.p0i8.i32(i8* align 16 %dest, i8* align 16 %src, i32 16, i32 16)
   ret void
 }
@@ -258,18 +302,9 @@ define void @test_memcpy_loadstore(i8* %dest, i8* %src) {
 ; CHECK-LABEL: @test_memcpy_loadstore(
 ; CHECK-NEXT:    [[TMP1:%.*]] = load atomic i8, i8* [[SRC:%.*]] unordered, align 1
 ; CHECK-NEXT:    store atomic i8 [[TMP1]], i8* [[DEST:%.*]] unordered, align 1
-; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8* [[SRC]] to i16*
-; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i8* [[DEST]] to i16*
-; CHECK-NEXT:    [[TMP4:%.*]] = load atomic i16, i16* [[TMP2]] unordered, align 1
-; CHECK-NEXT:    store atomic i16 [[TMP4]], i16* [[TMP3]] unordered, align 1
-; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8* [[SRC]] to i32*
-; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i8* [[DEST]] to i32*
-; CHECK-NEXT:    [[TMP7:%.*]] = load atomic i32, i32* [[TMP5]] unordered, align 1
-; CHECK-NEXT:    store atomic i32 [[TMP7]], i32* [[TMP6]] unordered, align 1
-; CHECK-NEXT:    [[TMP8:%.*]] = bitcast i8* [[SRC]] to i64*
-; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i8* [[DEST]] to i64*
-; CHECK-NEXT:    [[TMP10:%.*]] = load atomic i64, i64* [[TMP8]] unordered, align 1
-; CHECK-NEXT:    store atomic i64 [[TMP10]], i64* [[TMP9]] unordered, align 1
+; CHECK-NEXT:    call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i32(i8* align 1 [[DEST]], i8* align 1 [[SRC]], i32 2, i32 1)
+; CHECK-NEXT:    call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i32(i8* align 1 [[DEST]], i8* align 1 [[SRC]], i32 4, i32 1)
+; CHECK-NEXT:    call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i32(i8* align 1 [[DEST]], i8* align 1 [[SRC]], i32 8, i32 1)
 ; CHECK-NEXT:    call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i32(i8* align 1 [[DEST]], i8* align 1 [[SRC]], i32 16, i32 1)
 ; CHECK-NEXT:    ret void
 ;
@@ -283,21 +318,18 @@ define void @test_memcpy_loadstore(i8* %dest, i8* %src) {
 
 define void @test_memcpy_loadstore_2(i8* %dest, i8* %src) {
 ; CHECK-LABEL: @test_memcpy_loadstore_2(
-; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i8* [[SRC:%.*]] to i16*
-; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8* [[DEST:%.*]] to i16*
-; CHECK-NEXT:    [[TMP3:%.*]] = load atomic i16, i16* [[TMP1]] unordered, align 2
-; CHECK-NEXT:    store atomic i16 [[TMP3]], i16* [[TMP2]] unordered, align 2
-; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i8* [[SRC]] to i32*
-; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8* [[DEST]] to i32*
-; CHECK-NEXT:    [[TMP6:%.*]] = load atomic i32, i32* [[TMP4]] unordered, align 2
-; CHECK-NEXT:    store atomic i32 [[TMP6]], i32* [[TMP5]] unordered, align 2
-; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i8* [[SRC]] to i64*
-; CHECK-NEXT:    [[TMP8:%.*]] = bitcast i8* [[DEST]] to i64*
-; CHECK-NEXT:    [[TMP9:%.*]] = load atomic i64, i64* [[TMP7]] unordered, align 2
-; CHECK-NEXT:    store atomic i64 [[TMP9]], i64* [[TMP8]] unordered, align 2
+; CHECK-NEXT:    [[TMP1:%.*]] = load atomic i8, i8* [[SRC:%.*]] unordered, align 2
+; CHECK-NEXT:    store atomic i8 [[TMP1]], i8* [[DEST:%.*]] unordered, align 2
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8* [[SRC]] to i16*
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i8* [[DEST]] to i16*
+; CHECK-NEXT:    [[TMP4:%.*]] = load atomic i16, i16* [[TMP2]] unordered, align 2
+; CHECK-NEXT:    store atomic i16 [[TMP4]], i16* [[TMP3]] unordered, align 2
+; CHECK-NEXT:    call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i32(i8* align 2 [[DEST]], i8* align 2 [[SRC]], i32 4, i32 2)
+; CHECK-NEXT:    call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i32(i8* align 2 [[DEST]], i8* align 2 [[SRC]], i32 8, i32 2)
 ; CHECK-NEXT:    call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i32(i8* align 2 [[DEST]], i8* align 2 [[SRC]], i32 16, i32 2)
 ; CHECK-NEXT:    ret void
 ;
+  call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i32(i8* align 2 %dest, i8* align 2 %src, i32 1, i32 1)
   call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i32(i8* align 2 %dest, i8* align 2 %src, i32 2, i32 2)
   call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i32(i8* align 2 %dest, i8* align 2 %src, i32 4, i32 2)
   call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i32(i8* align 2 %dest, i8* align 2 %src, i32 8, i32 2)
@@ -307,17 +339,22 @@ define void @test_memcpy_loadstore_2(i8* %dest, i8* %src) {
 
 define void @test_memcpy_loadstore_4(i8* %dest, i8* %src) {
 ; CHECK-LABEL: @test_memcpy_loadstore_4(
-; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i8* [[SRC:%.*]] to i32*
-; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8* [[DEST:%.*]] to i32*
-; CHECK-NEXT:    [[TMP3:%.*]] = load atomic i32, i32* [[TMP1]] unordered, align 4
-; CHECK-NEXT:    store atomic i32 [[TMP3]], i32* [[TMP2]] unordered, align 4
-; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i8* [[SRC]] to i64*
-; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8* [[DEST]] to i64*
-; CHECK-NEXT:    [[TMP6:%.*]] = load atomic i64, i64* [[TMP4]] unordered, align 4
-; CHECK-NEXT:    store atomic i64 [[TMP6]], i64* [[TMP5]] unordered, align 4
+; CHECK-NEXT:    [[TMP1:%.*]] = load atomic i8, i8* [[SRC:%.*]] unordered, align 4
+; CHECK-NEXT:    store atomic i8 [[TMP1]], i8* [[DEST:%.*]] unordered, align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8* [[SRC]] to i16*
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i8* [[DEST]] to i16*
+; CHECK-NEXT:    [[TMP4:%.*]] = load atomic i16, i16* [[TMP2]] unordered, align 4
+; CHECK-NEXT:    store atomic i16 [[TMP4]], i16* [[TMP3]] unordered, align 4
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8* [[SRC]] to i32*
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i8* [[DEST]] to i32*
+; CHECK-NEXT:    [[TMP7:%.*]] = load atomic i32, i32* [[TMP5]] unordered, align 4
+; CHECK-NEXT:    store atomic i32 [[TMP7]], i32* [[TMP6]] unordered, align 4
+; CHECK-NEXT:    call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i32(i8* align 4 [[DEST]], i8* align 4 [[SRC]], i32 8, i32 4)
 ; CHECK-NEXT:    call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i32(i8* align 4 [[DEST]], i8* align 4 [[SRC]], i32 16, i32 4)
 ; CHECK-NEXT:    ret void
 ;
+  call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i32(i8* align 4 %dest, i8* align 4 %src, i32 1, i32 1)
+  call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i32(i8* align 4 %dest, i8* align 4 %src, i32 2, i32 2)
   call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i32(i8* align 4 %dest, i8* align 4 %src, i32 4, i32 4)
   call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i32(i8* align 4 %dest, i8* align 4 %src, i32 8, i32 4)
   call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i32(i8* align 4 %dest, i8* align 4 %src, i32 16, i32 4)
@@ -326,13 +363,26 @@ define void @test_memcpy_loadstore_4(i8* %dest, i8* %src) {
 
 define void @test_memcpy_loadstore_8(i8* %dest, i8* %src) {
 ; CHECK-LABEL: @test_memcpy_loadstore_8(
-; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i8* [[SRC:%.*]] to i64*
-; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8* [[DEST:%.*]] to i64*
-; CHECK-NEXT:    [[TMP3:%.*]] = load atomic i64, i64* [[TMP1]] unordered, align 8
-; CHECK-NEXT:    store atomic i64 [[TMP3]], i64* [[TMP2]] unordered, align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = load atomic i8, i8* [[SRC:%.*]] unordered, align 8
+; CHECK-NEXT:    store atomic i8 [[TMP1]], i8* [[DEST:%.*]] unordered, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8* [[SRC]] to i16*
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i8* [[DEST]] to i16*
+; CHECK-NEXT:    [[TMP4:%.*]] = load atomic i16, i16* [[TMP2]] unordered, align 8
+; CHECK-NEXT:    store atomic i16 [[TMP4]], i16* [[TMP3]] unordered, align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8* [[SRC]] to i32*
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i8* [[DEST]] to i32*
+; CHECK-NEXT:    [[TMP7:%.*]] = load atomic i32, i32* [[TMP5]] unordered, align 8
+; CHECK-NEXT:    store atomic i32 [[TMP7]], i32* [[TMP6]] unordered, align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast i8* [[SRC]] to i64*
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i8* [[DEST]] to i64*
+; CHECK-NEXT:    [[TMP10:%.*]] = load atomic i64, i64* [[TMP8]] unordered, align 8
+; CHECK-NEXT:    store atomic i64 [[TMP10]], i64* [[TMP9]] unordered, align 8
 ; CHECK-NEXT:    call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i32(i8* align 8 [[DEST]], i8* align 8 [[SRC]], i32 16, i32 8)
 ; CHECK-NEXT:    ret void
 ;
+  call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i32(i8* align 8 %dest, i8* align 8 %src, i32 1, i32 1)
+  call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i32(i8* align 8 %dest, i8* align 8 %src, i32 2, i32 2)
+  call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i32(i8* align 8 %dest, i8* align 8 %src, i32 4, i32 4)
   call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i32(i8* align 8 %dest, i8* align 8 %src, i32 8, i32 8)
   call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i32(i8* align 8 %dest, i8* align 8 %src, i32 16, i32 8)
   ret void
@@ -340,9 +390,27 @@ define void @test_memcpy_loadstore_8(i8* %dest, i8* %src) {
 
 define void @test_memcpy_loadstore_16(i8* %dest, i8* %src) {
 ; CHECK-LABEL: @test_memcpy_loadstore_16(
+; CHECK-NEXT:    [[TMP1:%.*]] = load atomic i8, i8* [[SRC:%.*]] unordered, align 16
+; CHECK-NEXT:    store atomic i8 [[TMP1]], i8* [[DEST:%.*]] unordered, align 16
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8* [[SRC]] to i16*
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i8* [[DEST]] to i16*
+; CHECK-NEXT:    [[TMP4:%.*]] = load atomic i16, i16* [[TMP2]] unordered, align 16
+; CHECK-NEXT:    store atomic i16 [[TMP4]], i16* [[TMP3]] unordered, align 16
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i8* [[SRC]] to i32*
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i8* [[DEST]] to i32*
+; CHECK-NEXT:    [[TMP7:%.*]] = load atomic i32, i32* [[TMP5]] unordered, align 16
+; CHECK-NEXT:    store atomic i32 [[TMP7]], i32* [[TMP6]] unordered, align 16
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast i8* [[SRC]] to i64*
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i8* [[DEST]] to i64*
+; CHECK-NEXT:    [[TMP10:%.*]] = load atomic i64, i64* [[TMP8]] unordered, align 16
+; CHECK-NEXT:    store atomic i64 [[TMP10]], i64* [[TMP9]] unordered, align 16
 ; CHECK-NEXT:    call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i32(i8* align 16 [[DEST:%.*]], i8* align 16 [[SRC:%.*]], i32 16, i32 16)
 ; CHECK-NEXT:    ret void
 ;
+  call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i32(i8* align 16 %dest, i8* align 16 %src, i32 1, i32 1)
+  call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i32(i8* align 16 %dest, i8* align 16 %src, i32 2, i32 2)
+  call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i32(i8* align 16 %dest, i8* align 16 %src, i32 4, i32 4)
+  call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i32(i8* align 16 %dest, i8* align 16 %src, i32 8, i32 8)
   call void @llvm.memcpy.element.unordered.atomic.p0i8.p0i8.i32(i8* align 16 %dest, i8* align 16 %src, i32 16, i32 16)
   ret void
 }
diff --git a/test/Transforms/InstCombine/fmul-exp.ll b/test/Transforms/InstCombine/fmul-exp.ll
new file mode 100644
index 0000000000000000000000000000000000000000..28542f45294896bd8cb5a314c9d35f65b5cfa1d5
--- /dev/null
+++ b/test/Transforms/InstCombine/fmul-exp.ll
@@ -0,0 +1,85 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -instcombine < %s | FileCheck %s
+
+declare double @llvm.exp.f64(double) nounwind readnone speculatable
+declare void @use(double)
+
+; exp(a) * exp(b) no reassoc flags
+define double @exp_a_exp_b(double %a, double %b) {
+; CHECK-LABEL: @exp_a_exp_b(
+; CHECK-NEXT:    [[TMP:%.*]] = call double @llvm.exp.f64(double [[A:%.*]])
+; CHECK-NEXT:    [[TMP1:%.*]] = call double @llvm.exp.f64(double [[B:%.*]])
+; CHECK-NEXT:    [[MUL:%.*]] = fmul double [[TMP]], [[TMP1]]
+; CHECK-NEXT:    ret double [[MUL]]
+;
+  %tmp = call double @llvm.exp.f64(double %a)
+  %tmp1 = call double @llvm.exp.f64(double %b)
+  %mul = fmul double %tmp, %tmp1
+  ret double %mul
+}
+
+; exp(a) * exp(b) reassoc, multiple uses
+define double @exp_a_exp_b_multiple_uses(double %a, double %b) {
+; CHECK-LABEL: @exp_a_exp_b_multiple_uses(
+; CHECK-NEXT:    [[TMP1:%.*]] = call double @llvm.exp.f64(double [[B:%.*]])
+; CHECK-NEXT:    [[TMP:%.*]] = fadd reassoc double [[A:%.*]], [[B]]
+; CHECK-NEXT:    [[TMP2:%.*]] = call reassoc double @llvm.exp.f64(double [[TMP]])
+; CHECK-NEXT:    call void @use(double [[TMP1]])
+; CHECK-NEXT:    ret double [[TMP2]]
+;
+  %tmp = call double @llvm.exp.f64(double %a)
+  %tmp1 = call double @llvm.exp.f64(double %b)
+  %mul = fmul reassoc double %tmp, %tmp1
+  call void @use(double %tmp1)
+  ret double %mul
+}
+
+; exp(a) * exp(b) reassoc, both with multiple uses
+define double @exp_a_exp_b_multiple_uses_both(double %a, double %b) {
+; CHECK-LABEL: @exp_a_exp_b_multiple_uses_both(
+; CHECK-NEXT:    [[TMP:%.*]] = call double @llvm.exp.f64(double [[A:%.*]])
+; CHECK-NEXT:    [[TMP1:%.*]] = call double @llvm.exp.f64(double [[B:%.*]])
+; CHECK-NEXT:    [[MUL:%.*]] = fmul reassoc double [[TMP]], [[TMP1]]
+; CHECK-NEXT:    call void @use(double [[TMP]])
+; CHECK-NEXT:    call void @use(double [[TMP1]])
+; CHECK-NEXT:    ret double [[MUL]]
+;
+  %tmp = call double @llvm.exp.f64(double %a)
+  %tmp1 = call double @llvm.exp.f64(double %b)
+  %mul = fmul reassoc double %tmp, %tmp1
+  call void @use(double %tmp)
+  call void @use(double %tmp1)
+  ret double %mul
+}
+
+; exp(a) * exp(b) => exp(a+b) with reassoc
+define double @exp_a_exp_b_reassoc(double %a, double %b) {
+; CHECK-LABEL: @exp_a_exp_b_reassoc(
+; CHECK-NEXT:    [[TMP:%.*]] = fadd reassoc double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call reassoc double @llvm.exp.f64(double [[TMP]])
+; CHECK-NEXT:    ret double [[TMP1]]
+;
+  %tmp = call double @llvm.exp.f64(double %a)
+  %tmp1 = call double @llvm.exp.f64(double %b)
+  %mul = fmul reassoc double %tmp, %tmp1
+  ret double %mul
+}
+
+; exp(a) * exp(b) * exp(c) * exp(d) => exp(a+b+c+d) with reassoc
+define double @exp_a_exp_b_exp_c_exp_d_fast(double %a, double %b, double %c, double %d) {
+; CHECK-LABEL: @exp_a_exp_b_exp_c_exp_d_fast(
+; CHECK-NEXT:    [[TMP:%.*]] = fadd reassoc double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = fadd reassoc double [[TMP]], [[C:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = fadd reassoc double [[TMP1]], [[D:%.*]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call reassoc double @llvm.exp.f64(double [[TMP2]])
+; CHECK-NEXT:    ret double [[TMP3]]
+;
+  %tmp = call double @llvm.exp.f64(double %a)
+  %tmp1 = call double @llvm.exp.f64(double %b)
+  %mul = fmul reassoc double %tmp, %tmp1
+  %tmp2 = call double @llvm.exp.f64(double %c)
+  %mul1 = fmul reassoc double %mul, %tmp2
+  %tmp3 = call double @llvm.exp.f64(double %d)
+  %mul2 = fmul reassoc double %mul1, %tmp3
+  ret double %mul2
+}
diff --git a/test/Transforms/InstCombine/fmul-exp2.ll b/test/Transforms/InstCombine/fmul-exp2.ll
new file mode 100644
index 0000000000000000000000000000000000000000..f09013847bb6fc960cfe00a7da1b5fcb89184754
--- /dev/null
+++ b/test/Transforms/InstCombine/fmul-exp2.ll
@@ -0,0 +1,85 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -instcombine < %s | FileCheck %s
+
+declare double @llvm.exp2.f64(double) nounwind readnone speculatable
+declare void @use(double)
+
+; exp2(a) * exp2(b) no reassoc flags
+define double @exp2_a_exp2_b(double %a, double %b) {
+; CHECK-LABEL: @exp2_a_exp2_b(
+; CHECK-NEXT:    [[TMP:%.*]] = call double @llvm.exp2.f64(double [[A:%.*]])
+; CHECK-NEXT:    [[TMP1:%.*]] = call double @llvm.exp2.f64(double [[B:%.*]])
+; CHECK-NEXT:    [[MUL:%.*]] = fmul double [[TMP]], [[TMP1]]
+; CHECK-NEXT:    ret double [[MUL]]
+;
+  %tmp = call double @llvm.exp2.f64(double %a)
+  %tmp1 = call double @llvm.exp2.f64(double %b)
+  %mul = fmul double %tmp, %tmp1
+  ret double %mul
+}
+
+; exp2(a) * exp2(b) reassoc, multiple uses
+define double @exp2_a_exp2_b_multiple_uses(double %a, double %b) {
+; CHECK-LABEL: @exp2_a_exp2_b_multiple_uses(
+; CHECK-NEXT:    [[TMP1:%.*]] = call double @llvm.exp2.f64(double [[B:%.*]])
+; CHECK-NEXT:    [[TMP:%.*]] = fadd reassoc double [[A:%.*]], [[B]]
+; CHECK-NEXT:    [[TMP2:%.*]] = call reassoc double @llvm.exp2.f64(double [[TMP]])
+; CHECK-NEXT:    call void @use(double [[TMP1]])
+; CHECK-NEXT:    ret double [[TMP2]]
+;
+  %tmp = call double @llvm.exp2.f64(double %a)
+  %tmp1 = call double @llvm.exp2.f64(double %b)
+  %mul = fmul reassoc double %tmp, %tmp1
+  call void @use(double %tmp1)
+  ret double %mul
+}
+
+; exp2(a) * exp2(b) reassoc, both with multiple uses
+define double @exp2_a_exp2_b_multiple_uses_both(double %a, double %b) {
+; CHECK-LABEL: @exp2_a_exp2_b_multiple_uses_both(
+; CHECK-NEXT:    [[TMP:%.*]] = call double @llvm.exp2.f64(double [[A:%.*]])
+; CHECK-NEXT:    [[TMP1:%.*]] = call double @llvm.exp2.f64(double [[B:%.*]])
+; CHECK-NEXT:    [[MUL:%.*]] = fmul reassoc double [[TMP]], [[TMP1]]
+; CHECK-NEXT:    call void @use(double [[TMP]])
+; CHECK-NEXT:    call void @use(double [[TMP1]])
+; CHECK-NEXT:    ret double [[MUL]]
+;
+  %tmp = call double @llvm.exp2.f64(double %a)
+  %tmp1 = call double @llvm.exp2.f64(double %b)
+  %mul = fmul reassoc double %tmp, %tmp1
+  call void @use(double %tmp)
+  call void @use(double %tmp1)
+  ret double %mul
+}
+
+; exp2(a) * exp2(b) => exp2(a+b) with reassoc
+define double @exp2_a_exp2_b_reassoc(double %a, double %b) {
+; CHECK-LABEL: @exp2_a_exp2_b_reassoc(
+; CHECK-NEXT:    [[TMP:%.*]] = fadd reassoc double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = call reassoc double @llvm.exp2.f64(double [[TMP]])
+; CHECK-NEXT:    ret double [[TMP1]]
+;
+  %tmp = call double @llvm.exp2.f64(double %a)
+  %tmp1 = call double @llvm.exp2.f64(double %b)
+  %mul = fmul reassoc double %tmp, %tmp1
+  ret double %mul
+}
+
+; exp2(a) * exp2(b) * exp2(c) * exp2(d) => exp2(a+b+c+d) with reassoc
+define double @exp2_a_exp2_b_exp2_c_exp2_d(double %a, double %b, double %c, double %d) {
+; CHECK-LABEL: @exp2_a_exp2_b_exp2_c_exp2_d(
+; CHECK-NEXT:    [[TMP:%.*]] = fadd reassoc double [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = fadd reassoc double [[TMP]], [[C:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = fadd reassoc double [[TMP1]], [[D:%.*]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call reassoc double @llvm.exp2.f64(double [[TMP2]])
+; CHECK-NEXT:    ret double [[TMP3]]
+;
+  %tmp = call double @llvm.exp2.f64(double %a)
+  %tmp1 = call double @llvm.exp2.f64(double %b)
+  %mul = fmul reassoc double %tmp, %tmp1
+  %tmp2 = call double @llvm.exp2.f64(double %c)
+  %mul1 = fmul reassoc double %mul, %tmp2
+  %tmp3 = call double @llvm.exp2.f64(double %d)
+  %mul2 = fmul reassoc double %mul1, %tmp3
+  ret double %mul2
+}
diff --git a/test/Transforms/InstCombine/fmul-pow.ll b/test/Transforms/InstCombine/fmul-pow.ll
new file mode 100644
index 0000000000000000000000000000000000000000..c85994cea71cf14a830afb2448ab47dbe59dbd15
--- /dev/null
+++ b/test/Transforms/InstCombine/fmul-pow.ll
@@ -0,0 +1,91 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -instcombine < %s | FileCheck %s
+
+declare double @llvm.pow.f64(double, double)
+
+define double @pow_ab_a(double %a, double %b)  {
+; CHECK-LABEL: @pow_ab_a(
+; CHECK-NEXT:    [[TMP1:%.*]] = call double @llvm.pow.f64(double [[A:%.*]], double [[B:%.*]])
+; CHECK-NEXT:    [[MUL:%.*]] = fmul double [[TMP1]], [[A]]
+; CHECK-NEXT:    ret double [[MUL]]
+;
+  %1 = call double @llvm.pow.f64(double %a, double %b)
+  %mul = fmul double %1, %a
+  ret double %mul
+}
+
+define double @pow_ab_a_reassoc(double %a, double %b)  {
+; CHECK-LABEL: @pow_ab_a_reassoc(
+; CHECK-NEXT:    [[TMP1:%.*]] = call double @llvm.pow.f64(double [[A:%.*]], double [[B:%.*]])
+; CHECK-NEXT:    [[MUL:%.*]] = fmul reassoc double [[TMP1]], [[A]]
+; CHECK-NEXT:    ret double [[MUL]]
+;
+  %1 = call double @llvm.pow.f64(double %a, double %b)
+  %mul = fmul reassoc double %1, %a
+  ret double %mul
+}
+
+define double @pow_ab_a_reassoc_commute(double %a, double %b)  {
+; CHECK-LABEL: @pow_ab_a_reassoc_commute(
+; CHECK-NEXT:    [[TMP1:%.*]] = fdiv double 1.000000e+00, [[A:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = call double @llvm.pow.f64(double [[A]], double [[B:%.*]])
+; CHECK-NEXT:    [[MUL:%.*]] = fmul reassoc double [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    ret double [[MUL]]
+;
+  %1 = fdiv double 1.0, %a
+  %2 = call double @llvm.pow.f64(double %a, double %b)
+  %mul = fmul reassoc double %1, %2
+  ret double %mul
+}
+
+define double @pow_ab_pow_cb(double %a, double %b, double %c) {
+; CHECK-LABEL: @pow_ab_pow_cb(
+; CHECK-NEXT:    [[TMP1:%.*]] = call double @llvm.pow.f64(double [[A:%.*]], double [[B:%.*]])
+; CHECK-NEXT:    [[TMP2:%.*]] = call double @llvm.pow.f64(double [[C:%.*]], double [[B]])
+; CHECK-NEXT:    [[MUL:%.*]] = fmul double [[TMP2]], [[TMP1]]
+; CHECK-NEXT:    ret double [[MUL]]
+;
+  %1 = call double @llvm.pow.f64(double %a, double %b)
+  %2 = call double @llvm.pow.f64(double %c, double %b)
+  %mul = fmul double %2, %1
+  ret double %mul
+}
+
+define double @pow_ab_pow_cb_reassoc(double %a, double %b, double %c) {
+; CHECK-LABEL: @pow_ab_pow_cb_reassoc(
+; CHECK-NEXT:    [[TMP1:%.*]] = call double @llvm.pow.f64(double [[A:%.*]], double [[B:%.*]])
+; CHECK-NEXT:    [[TMP2:%.*]] = call double @llvm.pow.f64(double [[C:%.*]], double [[B]])
+; CHECK-NEXT:    [[MUL:%.*]] = fmul reassoc double [[TMP2]], [[TMP1]]
+; CHECK-NEXT:    ret double [[MUL]]
+;
+  %1 = call double @llvm.pow.f64(double %a, double %b)
+  %2 = call double @llvm.pow.f64(double %c, double %b)
+  %mul = fmul reassoc double %2, %1
+  ret double %mul
+}
+
+define double @pow_ab_pow_ac(double %a, double %b, double %c) {
+; CHECK-LABEL: @pow_ab_pow_ac(
+; CHECK-NEXT:    [[TMP1:%.*]] = call double @llvm.pow.f64(double [[A:%.*]], double [[B:%.*]])
+; CHECK-NEXT:    [[TMP2:%.*]] = call double @llvm.pow.f64(double [[A]], double [[C:%.*]])
+; CHECK-NEXT:    [[MUL:%.*]] = fmul double [[TMP2]], [[TMP1]]
+; CHECK-NEXT:    ret double [[MUL]]
+;
+  %1 = call double @llvm.pow.f64(double %a, double %b)
+  %2 = call double @llvm.pow.f64(double %a, double %c)
+  %mul = fmul double %2, %1
+  ret double %mul
+}
+
+define double @pow_ab_x_pow_ac_reassoc(double %a, double %b, double %c) {
+; CHECK-LABEL: @pow_ab_x_pow_ac_reassoc(
+; CHECK-NEXT:    [[TMP1:%.*]] = call double @llvm.pow.f64(double [[A:%.*]], double [[B:%.*]])
+; CHECK-NEXT:    [[TMP2:%.*]] = call double @llvm.pow.f64(double [[A]], double [[C:%.*]])
+; CHECK-NEXT:    [[MUL:%.*]] = fmul reassoc double [[TMP2]], [[TMP1]]
+; CHECK-NEXT:    ret double [[MUL]]
+;
+  %1 = call double @llvm.pow.f64(double %a, double %b)
+  %2 = call double @llvm.pow.f64(double %a, double %c)
+  %mul = fmul reassoc double %2, %1
+  ret double %mul
+}
diff --git a/test/Transforms/InstCombine/icmp.ll b/test/Transforms/InstCombine/icmp.ll
index 7bf0cc9de8295e79c2d3d53dc92b4ac22b89eada..7f8c6a3819dbbedeff564f001b1c65b408cdce41 100644
--- a/test/Transforms/InstCombine/icmp.ll
+++ b/test/Transforms/InstCombine/icmp.ll
@@ -5,7 +5,7 @@ target datalayout = "e-p:64:64:64-p1:16:16:16-p2:32:32:32-p3:64:64:64-i1:8:8-i8:
 
 define i32 @test1(i32 %X) {
 ; CHECK-LABEL: @test1(
-; CHECK-NEXT:    [[X_LOBIT:%.*]] = lshr i32 %X, 31
+; CHECK-NEXT:    [[X_LOBIT:%.*]] = lshr i32 [[X:%.*]], 31
 ; CHECK-NEXT:    ret i32 [[X_LOBIT]]
 ;
   %a = icmp slt i32 %X, 0
@@ -25,7 +25,7 @@ define <2 x i32> @test1vec(<2 x i32> %X) {
 
 define i32 @test2(i32 %X) {
 ; CHECK-LABEL: @test2(
-; CHECK-NEXT:    [[X_LOBIT:%.*]] = lshr i32 %X, 31
+; CHECK-NEXT:    [[X_LOBIT:%.*]] = lshr i32 [[X:%.*]], 31
 ; CHECK-NEXT:    [[X_LOBIT_NOT:%.*]] = xor i32 [[X_LOBIT]], 1
 ; CHECK-NEXT:    ret i32 [[X_LOBIT_NOT]]
 ;
@@ -47,7 +47,7 @@ define <2 x i32> @test2vec(<2 x i32> %X) {
 
 define i32 @test3(i32 %X) {
 ; CHECK-LABEL: @test3(
-; CHECK-NEXT:    [[X_LOBIT:%.*]] = ashr i32 %X, 31
+; CHECK-NEXT:    [[X_LOBIT:%.*]] = ashr i32 [[X:%.*]], 31
 ; CHECK-NEXT:    ret i32 [[X_LOBIT]]
 ;
   %a = icmp slt i32 %X, 0
@@ -57,7 +57,7 @@ define i32 @test3(i32 %X) {
 
 define i32 @test4(i32 %X) {
 ; CHECK-LABEL: @test4(
-; CHECK-NEXT:    [[X_LOBIT:%.*]] = ashr i32 %X, 31
+; CHECK-NEXT:    [[X_LOBIT:%.*]] = ashr i32 [[X:%.*]], 31
 ; CHECK-NEXT:    [[X_LOBIT_NOT:%.*]] = xor i32 [[X_LOBIT]], -1
 ; CHECK-NEXT:    ret i32 [[X_LOBIT_NOT]]
 ;
@@ -77,8 +77,8 @@ define <2 x i1> @test5(<2 x i64> %x) {
 
 define i32 @test6(i32 %a, i32 %b) {
 ; CHECK-LABEL: @test6(
-; CHECK-NEXT:    [[E:%.*]] = ashr i32 %a, 31
-; CHECK-NEXT:    [[F:%.*]] = and i32 [[E]], %b
+; CHECK-NEXT:    [[E:%.*]] = ashr i32 [[A:%.*]], 31
+; CHECK-NEXT:    [[F:%.*]] = and i32 [[E]], [[B:%.*]]
 ; CHECK-NEXT:    ret i32 [[F]]
 ;
   %c = icmp sle i32 %a, -1
@@ -91,7 +91,7 @@ define i32 @test6(i32 %a, i32 %b) {
 
 define i1 @test7(i32 %x) {
 ; CHECK-LABEL: @test7(
-; CHECK-NEXT:    [[B:%.*]] = icmp ne i32 %x, 0
+; CHECK-NEXT:    [[B:%.*]] = icmp ne i32 [[X:%.*]], 0
 ; CHECK-NEXT:    ret i1 [[B]]
 ;
   %a = add i32 %x, -1
@@ -129,7 +129,7 @@ define <2 x i1> @test8_vec(<2 x i32> %x) {
 
 define i1 @test9(i32 %x) {
 ; CHECK-LABEL: @test9(
-; CHECK-NEXT:    [[B:%.*]] = icmp ugt i32 %x, 1
+; CHECK-NEXT:    [[B:%.*]] = icmp ugt i32 [[X:%.*]], 1
 ; CHECK-NEXT:    ret i1 [[B]]
 ;
   %a = add i32 %x, -2
@@ -169,7 +169,7 @@ define <2 x i1> @test9b_vec(<2 x i32> %x) {
 
 define i1 @test10(i32 %x) {
 ; CHECK-LABEL: @test10(
-; CHECK-NEXT:    [[B:%.*]] = icmp ne i32 %x, -2147483648
+; CHECK-NEXT:    [[B:%.*]] = icmp ne i32 [[X:%.*]], -2147483648
 ; CHECK-NEXT:    ret i1 [[B]]
 ;
   %a = add i32 %x, -1
@@ -228,7 +228,7 @@ define <2 x i1> @test11_vec(<2 x i32> %x) {
 ; PR6195
 define i1 @test12(i1 %A) {
 ; CHECK-LABEL: @test12(
-; CHECK-NEXT:    [[NOT_A:%.*]] = xor i1 %A, true
+; CHECK-NEXT:    [[NOT_A:%.*]] = xor i1 [[A:%.*]], true
 ; CHECK-NEXT:    ret i1 [[NOT_A]]
 ;
   %S = select i1 %A, i64 -4294967295, i64 8589934591
@@ -271,7 +271,7 @@ define i1 @test16() {
 
 define i1 @test17(i32 %x) {
 ; CHECK-LABEL: @test17(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 %x, 3
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[X:%.*]], 3
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %shl = shl i32 1, %x
@@ -282,7 +282,7 @@ define i1 @test17(i32 %x) {
 
 define <2 x i1> @test17vec(<2 x i32> %x) {
 ; CHECK-LABEL: @test17vec(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ne <2 x i32> %x, <i32 3, i32 3>
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne <2 x i32> [[X:%.*]], <i32 3, i32 3>
 ; CHECK-NEXT:    ret <2 x i1> [[CMP]]
 ;
   %shl = shl <2 x i32> <i32 1, i32 1>, %x
@@ -293,7 +293,7 @@ define <2 x i1> @test17vec(<2 x i32> %x) {
 
 define i1 @test17a(i32 %x) {
 ; CHECK-LABEL: @test17a(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i32 %x, 2
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i32 [[X:%.*]], 2
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %shl = shl i32 1, %x
@@ -304,7 +304,7 @@ define i1 @test17a(i32 %x) {
 
 define <2 x i1> @test17a_vec(<2 x i32> %x) {
 ; CHECK-LABEL: @test17a_vec(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt <2 x i32> %x, <i32 2, i32 2>
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt <2 x i32> [[X:%.*]], <i32 2, i32 2>
 ; CHECK-NEXT:    ret <2 x i1> [[CMP]]
 ;
   %shl = shl <2 x i32> <i32 1, i32 1>, %x
@@ -315,7 +315,7 @@ define <2 x i1> @test17a_vec(<2 x i32> %x) {
 
 define i1 @test18_eq(i32 %x) {
 ; CHECK-LABEL: @test18_eq(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 %x, 3
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[X:%.*]], 3
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %sh = lshr i32 8, %x
@@ -326,7 +326,7 @@ define i1 @test18_eq(i32 %x) {
 
 define <2 x i1> @test18_eq_vec(<2 x i32> %x) {
 ; CHECK-LABEL: @test18_eq_vec(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ne <2 x i32> %x, <i32 3, i32 3>
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne <2 x i32> [[X:%.*]], <i32 3, i32 3>
 ; CHECK-NEXT:    ret <2 x i1> [[CMP]]
 ;
   %sh = lshr <2 x i32> <i32 8, i32 8>, %x
@@ -337,7 +337,7 @@ define <2 x i1> @test18_eq_vec(<2 x i32> %x) {
 
 define i1 @test18_ne(i32 %x) {
 ; CHECK-LABEL: @test18_ne(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 %x, 3
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[X:%.*]], 3
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %sh = lshr i32 8, %x
@@ -348,7 +348,7 @@ define i1 @test18_ne(i32 %x) {
 
 define <2 x i1> @test18_ne_vec(<2 x i32> %x) {
 ; CHECK-LABEL: @test18_ne_vec(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <2 x i32> %x, <i32 3, i32 3>
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <2 x i32> [[X:%.*]], <i32 3, i32 3>
 ; CHECK-NEXT:    ret <2 x i1> [[CMP]]
 ;
   %sh = lshr <2 x i32> <i32 8, i32 8>, %x
@@ -359,7 +359,7 @@ define <2 x i1> @test18_ne_vec(<2 x i32> %x) {
 
 define i1 @test19(i32 %x) {
 ; CHECK-LABEL: @test19(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 %x, 3
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[X:%.*]], 3
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %shl = shl i32 1, %x
@@ -370,7 +370,7 @@ define i1 @test19(i32 %x) {
 
 define <2 x i1> @test19vec(<2 x i32> %x) {
 ; CHECK-LABEL: @test19vec(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <2 x i32> %x, <i32 3, i32 3>
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <2 x i32> [[X:%.*]], <i32 3, i32 3>
 ; CHECK-NEXT:    ret <2 x i1> [[CMP]]
 ;
   %shl = shl <2 x i32> <i32 1, i32 1>, %x
@@ -381,7 +381,7 @@ define <2 x i1> @test19vec(<2 x i32> %x) {
 
 define <2 x i1> @cmp_and_signbit_vec(<2 x i3> %x) {
 ; CHECK-LABEL: @cmp_and_signbit_vec(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp slt <2 x i3> %x, zeroinitializer
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt <2 x i3> [[X:%.*]], zeroinitializer
 ; CHECK-NEXT:    ret <2 x i1> [[CMP]]
 ;
   %and = and <2 x i3> %x, <i3 4, i3 4>
@@ -391,7 +391,7 @@ define <2 x i1> @cmp_and_signbit_vec(<2 x i3> %x) {
 
 define i1 @test20(i32 %x) {
 ; CHECK-LABEL: @test20(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 %x, 3
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[X:%.*]], 3
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %shl = shl i32 1, %x
@@ -402,7 +402,7 @@ define i1 @test20(i32 %x) {
 
 define <2 x i1> @test20vec(<2 x i32> %x) {
 ; CHECK-LABEL: @test20vec(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <2 x i32> %x, <i32 3, i32 3>
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <2 x i32> [[X:%.*]], <i32 3, i32 3>
 ; CHECK-NEXT:    ret <2 x i1> [[CMP]]
 ;
   %shl = shl <2 x i32> <i32 1, i32 1>, %x
@@ -413,7 +413,7 @@ define <2 x i1> @test20vec(<2 x i32> %x) {
 
 define i1 @test20a(i32 %x) {
 ; CHECK-LABEL: @test20a(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 %x, 3
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[X:%.*]], 3
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %shl = shl i32 1, %x
@@ -424,7 +424,7 @@ define i1 @test20a(i32 %x) {
 
 define <2 x i1> @test20a_vec(<2 x i32> %x) {
 ; CHECK-LABEL: @test20a_vec(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ult <2 x i32> %x, <i32 3, i32 3>
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult <2 x i32> [[X:%.*]], <i32 3, i32 3>
 ; CHECK-NEXT:    ret <2 x i1> [[CMP]]
 ;
   %shl = shl <2 x i32> <i32 1, i32 1>, %x
@@ -435,7 +435,7 @@ define <2 x i1> @test20a_vec(<2 x i32> %x) {
 
 define i1 @test21(i8 %x, i8 %y) {
 ; CHECK-LABEL: @test21(
-; CHECK-NEXT:    [[B:%.*]] = icmp ugt i8 %x, 3
+; CHECK-NEXT:    [[B:%.*]] = icmp ugt i8 [[X:%.*]], 3
 ; CHECK-NEXT:    ret i1 [[B]]
 ;
   %A = or i8 %x, 1
@@ -445,7 +445,7 @@ define i1 @test21(i8 %x, i8 %y) {
 
 define i1 @test22(i8 %x, i8 %y) {
 ; CHECK-LABEL: @test22(
-; CHECK-NEXT:    [[B:%.*]] = icmp ult i8 %x, 4
+; CHECK-NEXT:    [[B:%.*]] = icmp ult i8 [[X:%.*]], 4
 ; CHECK-NEXT:    ret i1 [[B]]
 ;
   %A = or i8 %x, 1
@@ -456,7 +456,7 @@ define i1 @test22(i8 %x, i8 %y) {
 ; PR2740
 define i1 @test23(i32 %x) {
 ; CHECK-LABEL: @test23(
-; CHECK-NEXT:    [[I4:%.*]] = icmp sgt i32 %x, 1328634634
+; CHECK-NEXT:    [[I4:%.*]] = icmp sgt i32 [[X:%.*]], 1328634634
 ; CHECK-NEXT:    ret i1 [[I4]]
 ;
   %i3 = sdiv i32 %x, -1328634635
@@ -466,7 +466,7 @@ define i1 @test23(i32 %x) {
 
 define <2 x i1> @test23vec(<2 x i32> %x) {
 ; CHECK-LABEL: @test23vec(
-; CHECK-NEXT:    [[I4:%.*]] = icmp sgt <2 x i32> %x, <i32 1328634634, i32 1328634634>
+; CHECK-NEXT:    [[I4:%.*]] = icmp sgt <2 x i32> [[X:%.*]], <i32 1328634634, i32 1328634634>
 ; CHECK-NEXT:    ret <2 x i1> [[I4]]
 ;
   %i3 = sdiv <2 x i32> %x, <i32 -1328634635, i32 -1328634635>
@@ -479,7 +479,7 @@ define <2 x i1> @test23vec(<2 x i32> %x) {
 ; PR8882
 define i1 @test24(i64 %i) {
 ; CHECK-LABEL: @test24(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i64 %i, 1000
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i64 [[I:%.*]], 1000
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %p1 = getelementptr inbounds i32, i32* getelementptr inbounds ([1000 x i32], [1000 x i32]* @X, i64 0, i64 0), i64 %i
@@ -491,7 +491,7 @@ define i1 @test24(i64 %i) {
 
 define i1 @test24_as1(i64 %i) {
 ; CHECK-LABEL: @test24_as1(
-; CHECK-NEXT:    [[TMP1:%.*]] = trunc i64 %i to i16
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i64 [[I:%.*]] to i16
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i16 [[TMP1]], 1000
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
@@ -502,7 +502,7 @@ define i1 @test24_as1(i64 %i) {
 
 define i1 @test25(i32 %x, i32 %y, i32 %z) {
 ; CHECK-LABEL: @test25(
-; CHECK-NEXT:    [[C:%.*]] = icmp sgt i32 %x, %y
+; CHECK-NEXT:    [[C:%.*]] = icmp sgt i32 [[X:%.*]], [[Y:%.*]]
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %lhs = add nsw i32 %x, %z
@@ -514,7 +514,7 @@ define i1 @test25(i32 %x, i32 %y, i32 %z) {
 ; X + Z > Y + Z -> X > Y if there is no overflow.
 define i1 @test26(i32 %x, i32 %y, i32 %z) {
 ; CHECK-LABEL: @test26(
-; CHECK-NEXT:    [[C:%.*]] = icmp ugt i32 %x, %y
+; CHECK-NEXT:    [[C:%.*]] = icmp ugt i32 [[X:%.*]], [[Y:%.*]]
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %lhs = add nuw i32 %x, %z
@@ -526,7 +526,7 @@ define i1 @test26(i32 %x, i32 %y, i32 %z) {
 ; X - Z > Y - Z -> X > Y if there is no overflow.
 define i1 @test27(i32 %x, i32 %y, i32 %z) {
 ; CHECK-LABEL: @test27(
-; CHECK-NEXT:    [[C:%.*]] = icmp sgt i32 %x, %y
+; CHECK-NEXT:    [[C:%.*]] = icmp sgt i32 [[X:%.*]], [[Y:%.*]]
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %lhs = sub nsw i32 %x, %z
@@ -538,7 +538,7 @@ define i1 @test27(i32 %x, i32 %y, i32 %z) {
 ; X - Z > Y - Z -> X > Y if there is no overflow.
 define i1 @test28(i32 %x, i32 %y, i32 %z) {
 ; CHECK-LABEL: @test28(
-; CHECK-NEXT:    [[C:%.*]] = icmp ugt i32 %x, %y
+; CHECK-NEXT:    [[C:%.*]] = icmp ugt i32 [[X:%.*]], [[Y:%.*]]
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %lhs = sub nuw i32 %x, %z
@@ -550,7 +550,7 @@ define i1 @test28(i32 %x, i32 %y, i32 %z) {
 ; X + Y > X -> Y > 0 if there is no overflow.
 define i1 @test29(i32 %x, i32 %y) {
 ; CHECK-LABEL: @test29(
-; CHECK-NEXT:    [[C:%.*]] = icmp sgt i32 %y, 0
+; CHECK-NEXT:    [[C:%.*]] = icmp sgt i32 [[Y:%.*]], 0
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %lhs = add nsw i32 %x, %y
@@ -561,7 +561,7 @@ define i1 @test29(i32 %x, i32 %y) {
 ; X + Y > X -> Y > 0 if there is no overflow.
 define i1 @test30(i32 %x, i32 %y) {
 ; CHECK-LABEL: @test30(
-; CHECK-NEXT:    [[C:%.*]] = icmp ne i32 %y, 0
+; CHECK-NEXT:    [[C:%.*]] = icmp ne i32 [[Y:%.*]], 0
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %lhs = add nuw i32 %x, %y
@@ -572,7 +572,7 @@ define i1 @test30(i32 %x, i32 %y) {
 ; X > X + Y -> 0 > Y if there is no overflow.
 define i1 @test31(i32 %x, i32 %y) {
 ; CHECK-LABEL: @test31(
-; CHECK-NEXT:    [[C:%.*]] = icmp slt i32 %y, 0
+; CHECK-NEXT:    [[C:%.*]] = icmp slt i32 [[Y:%.*]], 0
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %rhs = add nsw i32 %x, %y
@@ -593,7 +593,7 @@ define i1 @test32(i32 %x, i32 %y) {
 ; X - Y > X -> 0 > Y if there is no overflow.
 define i1 @test33(i32 %x, i32 %y) {
 ; CHECK-LABEL: @test33(
-; CHECK-NEXT:    [[C:%.*]] = icmp slt i32 %y, 0
+; CHECK-NEXT:    [[C:%.*]] = icmp slt i32 [[Y:%.*]], 0
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %lhs = sub nsw i32 %x, %y
@@ -614,7 +614,7 @@ define i1 @test34(i32 %x, i32 %y) {
 ; X > X - Y -> Y > 0 if there is no overflow.
 define i1 @test35(i32 %x, i32 %y) {
 ; CHECK-LABEL: @test35(
-; CHECK-NEXT:    [[C:%.*]] = icmp sgt i32 %y, 0
+; CHECK-NEXT:    [[C:%.*]] = icmp sgt i32 [[Y:%.*]], 0
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %rhs = sub nsw i32 %x, %y
@@ -625,7 +625,7 @@ define i1 @test35(i32 %x, i32 %y) {
 ; X > X - Y -> Y > 0 if there is no overflow.
 define i1 @test36(i32 %x, i32 %y) {
 ; CHECK-LABEL: @test36(
-; CHECK-NEXT:    [[C:%.*]] = icmp ne i32 %y, 0
+; CHECK-NEXT:    [[C:%.*]] = icmp ne i32 [[Y:%.*]], 0
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %rhs = sub nuw i32 %x, %y
@@ -664,7 +664,7 @@ define <2 x i1> @ult_sub(<2 x i8> %xsrc, <2 x i8> %y) {
 ; X - Y > X - Z -> Z > Y if there is no overflow.
 define i1 @test37(i32 %x, i32 %y, i32 %z) {
 ; CHECK-LABEL: @test37(
-; CHECK-NEXT:    [[C:%.*]] = icmp sgt i32 %z, %y
+; CHECK-NEXT:    [[C:%.*]] = icmp sgt i32 [[Z:%.*]], [[Y:%.*]]
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %lhs = sub nsw i32 %x, %y
@@ -676,7 +676,7 @@ define i1 @test37(i32 %x, i32 %y, i32 %z) {
 ; X - Y > X - Z -> Z > Y if there is no overflow.
 define i1 @test38(i32 %x, i32 %y, i32 %z) {
 ; CHECK-LABEL: @test38(
-; CHECK-NEXT:    [[C:%.*]] = icmp ugt i32 %z, %y
+; CHECK-NEXT:    [[C:%.*]] = icmp ugt i32 [[Z:%.*]], [[Y:%.*]]
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %lhs = sub nuw i32 %x, %y
@@ -688,7 +688,7 @@ define i1 @test38(i32 %x, i32 %y, i32 %z) {
 ; PR9343 #1
 define i1 @test39(i32 %X, i32 %Y) {
 ; CHECK-LABEL: @test39(
-; CHECK-NEXT:    [[B:%.*]] = icmp eq i32 %X, 0
+; CHECK-NEXT:    [[B:%.*]] = icmp eq i32 [[X:%.*]], 0
 ; CHECK-NEXT:    ret i1 [[B]]
 ;
   %A = ashr exact i32 %X, %Y
@@ -698,7 +698,7 @@ define i1 @test39(i32 %X, i32 %Y) {
 
 define <2 x i1> @test39vec(<2 x i32> %X, <2 x i32> %Y) {
 ; CHECK-LABEL: @test39vec(
-; CHECK-NEXT:    [[B:%.*]] = icmp eq <2 x i32> %X, zeroinitializer
+; CHECK-NEXT:    [[B:%.*]] = icmp eq <2 x i32> [[X:%.*]], zeroinitializer
 ; CHECK-NEXT:    ret <2 x i1> [[B]]
 ;
   %A = ashr exact <2 x i32> %X, %Y
@@ -708,7 +708,7 @@ define <2 x i1> @test39vec(<2 x i32> %X, <2 x i32> %Y) {
 
 define i1 @test40(i32 %X, i32 %Y) {
 ; CHECK-LABEL: @test40(
-; CHECK-NEXT:    [[B:%.*]] = icmp ne i32 %X, 0
+; CHECK-NEXT:    [[B:%.*]] = icmp ne i32 [[X:%.*]], 0
 ; CHECK-NEXT:    ret i1 [[B]]
 ;
   %A = lshr exact i32 %X, %Y
@@ -718,7 +718,7 @@ define i1 @test40(i32 %X, i32 %Y) {
 
 define <2 x i1> @test40vec(<2 x i32> %X, <2 x i32> %Y) {
 ; CHECK-LABEL: @test40vec(
-; CHECK-NEXT:    [[B:%.*]] = icmp ne <2 x i32> %X, zeroinitializer
+; CHECK-NEXT:    [[B:%.*]] = icmp ne <2 x i32> [[X:%.*]], zeroinitializer
 ; CHECK-NEXT:    ret <2 x i1> [[B]]
 ;
   %A = lshr exact <2 x i32> %X, %Y
@@ -728,7 +728,7 @@ define <2 x i1> @test40vec(<2 x i32> %X, <2 x i32> %Y) {
 
 define i1 @shr_exact(i132 %x) {
 ; CHECK-LABEL: @shr_exact(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i132 %x, 32
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i132 [[X:%.*]], 32
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %sh = ashr exact i132 %x, 4
@@ -738,7 +738,7 @@ define i1 @shr_exact(i132 %x) {
 
 define <2 x i1> @shr_exact_vec(<2 x i132> %x) {
 ; CHECK-LABEL: @shr_exact_vec(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ne <2 x i132> %x, <i132 32, i132 32>
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne <2 x i132> [[X:%.*]], <i132 32, i132 32>
 ; CHECK-NEXT:    ret <2 x i1> [[CMP]]
 ;
   %sh = lshr exact <2 x i132> %x, <i132 4, i132 4>
@@ -758,7 +758,7 @@ define i1 @test41(i32 %X, i32 %Y) {
 
 define i1 @test42(i32 %X, i32 %Y) {
 ; CHECK-LABEL: @test42(
-; CHECK-NEXT:    [[B:%.*]] = icmp sgt i32 %Y, -1
+; CHECK-NEXT:    [[B:%.*]] = icmp sgt i32 [[Y:%.*]], -1
 ; CHECK-NEXT:    ret i1 [[B]]
 ;
   %A = srem i32 %X, %Y
@@ -768,7 +768,7 @@ define i1 @test42(i32 %X, i32 %Y) {
 
 define i1 @test43(i32 %X, i32 %Y) {
 ; CHECK-LABEL: @test43(
-; CHECK-NEXT:    [[B:%.*]] = icmp slt i32 %Y, 0
+; CHECK-NEXT:    [[B:%.*]] = icmp slt i32 [[Y:%.*]], 0
 ; CHECK-NEXT:    ret i1 [[B]]
 ;
   %A = srem i32 %X, %Y
@@ -778,7 +778,7 @@ define i1 @test43(i32 %X, i32 %Y) {
 
 define i1 @test44(i32 %X, i32 %Y) {
 ; CHECK-LABEL: @test44(
-; CHECK-NEXT:    [[B:%.*]] = icmp sgt i32 %Y, -1
+; CHECK-NEXT:    [[B:%.*]] = icmp sgt i32 [[Y:%.*]], -1
 ; CHECK-NEXT:    ret i1 [[B]]
 ;
   %A = srem i32 %X, %Y
@@ -788,7 +788,7 @@ define i1 @test44(i32 %X, i32 %Y) {
 
 define i1 @test45(i32 %X, i32 %Y) {
 ; CHECK-LABEL: @test45(
-; CHECK-NEXT:    [[B:%.*]] = icmp slt i32 %Y, 0
+; CHECK-NEXT:    [[B:%.*]] = icmp slt i32 [[Y:%.*]], 0
 ; CHECK-NEXT:    ret i1 [[B]]
 ;
   %A = srem i32 %X, %Y
@@ -799,7 +799,7 @@ define i1 @test45(i32 %X, i32 %Y) {
 ; PR9343 #4
 define i1 @test46(i32 %X, i32 %Y, i32 %Z) {
 ; CHECK-LABEL: @test46(
-; CHECK-NEXT:    [[C:%.*]] = icmp ult i32 %X, %Y
+; CHECK-NEXT:    [[C:%.*]] = icmp ult i32 [[X:%.*]], [[Y:%.*]]
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %A = ashr exact i32 %X, %Z
@@ -811,7 +811,7 @@ define i1 @test46(i32 %X, i32 %Y, i32 %Z) {
 ; PR9343 #5
 define i1 @test47(i32 %X, i32 %Y, i32 %Z) {
 ; CHECK-LABEL: @test47(
-; CHECK-NEXT:    [[C:%.*]] = icmp ugt i32 %X, %Y
+; CHECK-NEXT:    [[C:%.*]] = icmp ugt i32 [[X:%.*]], [[Y:%.*]]
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %A = ashr exact i32 %X, %Z
@@ -823,7 +823,7 @@ define i1 @test47(i32 %X, i32 %Y, i32 %Z) {
 ; PR9343 #8
 define i1 @test48(i32 %X, i32 %Y, i32 %Z) {
 ; CHECK-LABEL: @test48(
-; CHECK-NEXT:    [[C:%.*]] = icmp eq i32 %X, %Y
+; CHECK-NEXT:    [[C:%.*]] = icmp eq i32 [[X:%.*]], [[Y:%.*]]
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %A = sdiv exact i32 %X, %Z
@@ -836,8 +836,8 @@ define i1 @test48(i32 %X, i32 %Y, i32 %Z) {
 
 define i1 @PR32949(i32 %X, i32 %Y, i32 %Z) {
 ; CHECK-LABEL: @PR32949(
-; CHECK-NEXT:    [[A:%.*]] = sdiv exact i32 %X, %Z
-; CHECK-NEXT:    [[B:%.*]] = sdiv exact i32 %Y, %Z
+; CHECK-NEXT:    [[A:%.*]] = sdiv exact i32 [[X:%.*]], [[Z:%.*]]
+; CHECK-NEXT:    [[B:%.*]] = sdiv exact i32 [[Y:%.*]], [[Z]]
 ; CHECK-NEXT:    [[C:%.*]] = icmp sgt i32 [[A]], [[B]]
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
@@ -872,8 +872,8 @@ define i1 @test50(i16 %X, i32 %Y) {
 
 define i1 @test51(i32 %X, i32 %Y) {
 ; CHECK-LABEL: @test51(
-; CHECK-NEXT:    [[A:%.*]] = and i32 %X, -2147483648
-; CHECK-NEXT:    [[B:%.*]] = srem i32 [[A]], %Y
+; CHECK-NEXT:    [[A:%.*]] = and i32 [[X:%.*]], -2147483648
+; CHECK-NEXT:    [[B:%.*]] = srem i32 [[A]], [[Y:%.*]]
 ; CHECK-NEXT:    [[C:%.*]] = icmp sgt i32 [[B]], -1
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
@@ -885,7 +885,7 @@ define i1 @test51(i32 %X, i32 %Y) {
 
 define i1 @test52(i32 %x1) {
 ; CHECK-LABEL: @test52(
-; CHECK-NEXT:    [[TMP1:%.*]] = and i32 %x1, 16711935
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X1:%.*]], 16711935
 ; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i32 [[TMP1]], 4980863
 ; CHECK-NEXT:    ret i1 [[TMP2]]
 ;
@@ -918,8 +918,8 @@ define i1 @test52b(i128 %x1) {
 ; PR9838
 define i1 @test53(i32 %a, i32 %b) {
 ; CHECK-LABEL: @test53(
-; CHECK-NEXT:    [[X:%.*]] = sdiv exact i32 %a, 30
-; CHECK-NEXT:    [[Y:%.*]] = sdiv i32 %b, 30
+; CHECK-NEXT:    [[X:%.*]] = sdiv exact i32 [[A:%.*]], 30
+; CHECK-NEXT:    [[Y:%.*]] = sdiv i32 [[B:%.*]], 30
 ; CHECK-NEXT:    [[Z:%.*]] = icmp eq i32 [[X]], [[Y]]
 ; CHECK-NEXT:    ret i1 [[Z]]
 ;
@@ -931,8 +931,8 @@ define i1 @test53(i32 %a, i32 %b) {
 
 define i1 @test54(i8 %a) {
 ; CHECK-LABEL: @test54(
-; CHECK-NEXT:    [[AND:%.*]] = and i8 %a, -64
-; CHECK-NEXT:    [[RET:%.*]] = icmp eq i8 [[AND]], -128
+; CHECK-NEXT:    [[TMP1:%.*]] = and i8 [[A:%.*]], -64
+; CHECK-NEXT:    [[RET:%.*]] = icmp eq i8 [[TMP1]], -128
 ; CHECK-NEXT:    ret i1 [[RET]]
 ;
   %ext = zext i8 %a to i32
@@ -943,7 +943,7 @@ define i1 @test54(i8 %a) {
 
 define i1 @test55(i32 %a) {
 ; CHECK-LABEL: @test55(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 %a, -123
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[A:%.*]], -123
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %sub = sub i32 0, %a
@@ -953,7 +953,7 @@ define i1 @test55(i32 %a) {
 
 define <2 x i1> @test55vec(<2 x i32> %a) {
 ; CHECK-LABEL: @test55vec(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <2 x i32> %a, <i32 -123, i32 -123>
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <2 x i32> [[A:%.*]], <i32 -123, i32 -123>
 ; CHECK-NEXT:    ret <2 x i1> [[CMP]]
 ;
   %sub = sub <2 x i32> zeroinitializer, %a
@@ -963,7 +963,7 @@ define <2 x i1> @test55vec(<2 x i32> %a) {
 
 define i1 @test56(i32 %a) {
 ; CHECK-LABEL: @test56(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 %a, -113
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[A:%.*]], -113
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %sub = sub i32 10, %a
@@ -973,7 +973,7 @@ define i1 @test56(i32 %a) {
 
 define <2 x i1> @test56vec(<2 x i32> %a) {
 ; CHECK-LABEL: @test56vec(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <2 x i32> %a, <i32 -113, i32 -113>
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <2 x i32> [[A:%.*]], <i32 -113, i32 -113>
 ; CHECK-NEXT:    ret <2 x i1> [[CMP]]
 ;
   %sub = sub <2 x i32> <i32 10, i32 10>, %a
@@ -985,7 +985,7 @@ define <2 x i1> @test56vec(<2 x i32> %a) {
 declare void @foo(i32)
 define i1 @test57(i32 %a) {
 ; CHECK-LABEL: @test57(
-; CHECK-NEXT:    [[AND:%.*]] = and i32 %a, -2
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[A:%.*]], -2
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[AND]], 0
 ; CHECK-NEXT:    call void @foo(i32 [[AND]])
 ; CHECK-NEXT:    ret i1 [[CMP]]
@@ -999,7 +999,7 @@ define i1 @test57(i32 %a) {
 ; rdar://problem/10482509
 define zeroext i1 @cmpabs1(i64 %val) {
 ; CHECK-LABEL: @cmpabs1(
-; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp ne i64 %val, 0
+; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp ne i64 [[VAL:%.*]], 0
 ; CHECK-NEXT:    ret i1 [[TOBOOL]]
 ;
   %sub = sub nsw i64 0, %val
@@ -1011,7 +1011,7 @@ define zeroext i1 @cmpabs1(i64 %val) {
 
 define zeroext i1 @cmpabs2(i64 %val) {
 ; CHECK-LABEL: @cmpabs2(
-; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp ne i64 %val, 0
+; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp ne i64 [[VAL:%.*]], 0
 ; CHECK-NEXT:    ret i1 [[TOBOOL]]
 ;
   %sub = sub nsw i64 0, %val
@@ -1034,7 +1034,7 @@ declare i32 @test58_d(i64)
 
 define i1 @test59(i8* %foo) {
 ; CHECK-LABEL: @test59(
-; CHECK-NEXT:    [[GEP1:%.*]] = getelementptr inbounds i8, i8* %foo, i64 8
+; CHECK-NEXT:    [[GEP1:%.*]] = getelementptr inbounds i8, i8* [[FOO:%.*]], i64 8
 ; CHECK-NEXT:    [[USE:%.*]] = ptrtoint i8* [[GEP1]] to i64
 ; CHECK-NEXT:    [[CALL:%.*]] = call i32 @test58_d(i64 [[USE]])
 ; CHECK-NEXT:    ret i1 true
@@ -1051,7 +1051,7 @@ define i1 @test59(i8* %foo) {
 
 define i1 @test59_as1(i8 addrspace(1)* %foo) {
 ; CHECK-LABEL: @test59_as1(
-; CHECK-NEXT:    [[GEP1:%.*]] = getelementptr inbounds i8, i8 addrspace(1)* %foo, i16 8
+; CHECK-NEXT:    [[GEP1:%.*]] = getelementptr inbounds i8, i8 addrspace(1)* [[FOO:%.*]], i16 8
 ; CHECK-NEXT:    [[TMP1:%.*]] = ptrtoint i8 addrspace(1)* [[GEP1]] to i16
 ; CHECK-NEXT:    [[USE:%.*]] = zext i16 [[TMP1]] to i64
 ; CHECK-NEXT:    [[CALL:%.*]] = call i32 @test58_d(i64 [[USE]])
@@ -1069,8 +1069,8 @@ define i1 @test59_as1(i8 addrspace(1)* %foo) {
 
 define i1 @test60(i8* %foo, i64 %i, i64 %j) {
 ; CHECK-LABEL: @test60(
-; CHECK-NEXT:    [[GEP1_IDX:%.*]] = shl nuw i64 %i, 2
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i64 [[GEP1_IDX]], %j
+; CHECK-NEXT:    [[GEP1_IDX:%.*]] = shl nuw i64 [[I:%.*]], 2
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i64 [[GEP1_IDX]], [[J:%.*]]
 ; CHECK-NEXT:    ret i1 [[TMP1]]
 ;
   %bit = bitcast i8* %foo to i32*
@@ -1083,8 +1083,8 @@ define i1 @test60(i8* %foo, i64 %i, i64 %j) {
 
 define i1 @test60_as1(i8 addrspace(1)* %foo, i64 %i, i64 %j) {
 ; CHECK-LABEL: @test60_as1(
-; CHECK-NEXT:    [[TMP1:%.*]] = trunc i64 %i to i16
-; CHECK-NEXT:    [[TMP2:%.*]] = trunc i64 %j to i16
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i64 [[I:%.*]] to i16
+; CHECK-NEXT:    [[TMP2:%.*]] = trunc i64 [[J:%.*]] to i16
 ; CHECK-NEXT:    [[GEP1_IDX:%.*]] = shl nuw i16 [[TMP1]], 2
 ; CHECK-NEXT:    [[TMP3:%.*]] = icmp slt i16 [[GEP1_IDX]], [[TMP2]]
 ; CHECK-NEXT:    ret i1 [[TMP3]]
@@ -1101,8 +1101,8 @@ define i1 @test60_as1(i8 addrspace(1)* %foo, i64 %i, i64 %j) {
 ; bitcast. This uses the same sized addrspace.
 define i1 @test60_addrspacecast(i8* %foo, i64 %i, i64 %j) {
 ; CHECK-LABEL: @test60_addrspacecast(
-; CHECK-NEXT:    [[GEP1_IDX:%.*]] = shl nuw i64 %i, 2
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i64 [[GEP1_IDX]], %j
+; CHECK-NEXT:    [[GEP1_IDX:%.*]] = shl nuw i64 [[I:%.*]], 2
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i64 [[GEP1_IDX]], [[J:%.*]]
 ; CHECK-NEXT:    ret i1 [[TMP1]]
 ;
   %bit = addrspacecast i8* %foo to i32 addrspace(3)*
@@ -1115,8 +1115,8 @@ define i1 @test60_addrspacecast(i8* %foo, i64 %i, i64 %j) {
 
 define i1 @test60_addrspacecast_smaller(i8* %foo, i16 %i, i64 %j) {
 ; CHECK-LABEL: @test60_addrspacecast_smaller(
-; CHECK-NEXT:    [[GEP1_IDX:%.*]] = shl nuw i16 %i, 2
-; CHECK-NEXT:    [[TMP1:%.*]] = trunc i64 %j to i16
+; CHECK-NEXT:    [[GEP1_IDX:%.*]] = shl nuw i16 [[I:%.*]], 2
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i64 [[J:%.*]] to i16
 ; CHECK-NEXT:    [[TMP2:%.*]] = icmp slt i16 [[GEP1_IDX]], [[TMP1]]
 ; CHECK-NEXT:    ret i1 [[TMP2]]
 ;
@@ -1130,9 +1130,9 @@ define i1 @test60_addrspacecast_smaller(i8* %foo, i16 %i, i64 %j) {
 
 define i1 @test60_addrspacecast_larger(i8 addrspace(1)* %foo, i32 %i, i16 %j) {
 ; CHECK-LABEL: @test60_addrspacecast_larger(
-; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 %i to i16
-; CHECK-NEXT:    [[SHL:%.*]] = shl i16 [[TMP1]], 2
-; CHECK-NEXT:    [[TMP2:%.*]] = icmp slt i16 [[SHL]], %j
+; CHECK-NEXT:    [[I_TR:%.*]] = trunc i32 [[I:%.*]] to i16
+; CHECK-NEXT:    [[TMP1:%.*]] = shl i16 [[I_TR]], 2
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp slt i16 [[TMP1]], [[J:%.*]]
 ; CHECK-NEXT:    ret i1 [[TMP2]]
 ;
   %bit = addrspacecast i8 addrspace(1)* %foo to i32 addrspace(2)*
@@ -1145,9 +1145,9 @@ define i1 @test60_addrspacecast_larger(i8 addrspace(1)* %foo, i32 %i, i16 %j) {
 
 define i1 @test61(i8* %foo, i64 %i, i64 %j) {
 ; CHECK-LABEL: @test61(
-; CHECK-NEXT:    [[BIT:%.*]] = bitcast i8* %foo to i32*
-; CHECK-NEXT:    [[GEP1:%.*]] = getelementptr i32, i32* [[BIT]], i64 %i
-; CHECK-NEXT:    [[GEP2:%.*]] = getelementptr i8, i8* %foo, i64 %j
+; CHECK-NEXT:    [[BIT:%.*]] = bitcast i8* [[FOO:%.*]] to i32*
+; CHECK-NEXT:    [[GEP1:%.*]] = getelementptr i32, i32* [[BIT]], i64 [[I:%.*]]
+; CHECK-NEXT:    [[GEP2:%.*]] = getelementptr i8, i8* [[FOO]], i64 [[J:%.*]]
 ; CHECK-NEXT:    [[CAST1:%.*]] = bitcast i32* [[GEP1]] to i8*
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i8* [[GEP2]], [[CAST1]]
 ; CHECK-NEXT:    ret i1 [[CMP]]
@@ -1163,9 +1163,9 @@ define i1 @test61(i8* %foo, i64 %i, i64 %j) {
 
 define i1 @test61_as1(i8 addrspace(1)* %foo, i16 %i, i16 %j) {
 ; CHECK-LABEL: @test61_as1(
-; CHECK-NEXT:    [[BIT:%.*]] = bitcast i8 addrspace(1)* %foo to i32 addrspace(1)*
-; CHECK-NEXT:    [[GEP1:%.*]] = getelementptr i32, i32 addrspace(1)* [[BIT]], i16 %i
-; CHECK-NEXT:    [[GEP2:%.*]] = getelementptr i8, i8 addrspace(1)* %foo, i16 %j
+; CHECK-NEXT:    [[BIT:%.*]] = bitcast i8 addrspace(1)* [[FOO:%.*]] to i32 addrspace(1)*
+; CHECK-NEXT:    [[GEP1:%.*]] = getelementptr i32, i32 addrspace(1)* [[BIT]], i16 [[I:%.*]]
+; CHECK-NEXT:    [[GEP2:%.*]] = getelementptr i8, i8 addrspace(1)* [[FOO]], i16 [[J:%.*]]
 ; CHECK-NEXT:    [[CAST1:%.*]] = bitcast i32 addrspace(1)* [[GEP1]] to i8 addrspace(1)*
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i8 addrspace(1)* [[GEP2]], [[CAST1]]
 ; CHECK-NEXT:    ret i1 [[CMP]]
@@ -1201,8 +1201,8 @@ define i1 @test62_as1(i8 addrspace(1)* %a) {
 
 define i1 @test63(i8 %a, i32 %b) {
 ; CHECK-LABEL: @test63(
-; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 %b to i8
-; CHECK-NEXT:    [[C:%.*]] = icmp eq i8 [[TMP1]], %a
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 [[B:%.*]] to i8
+; CHECK-NEXT:    [[C:%.*]] = icmp eq i8 [[TMP1]], [[A:%.*]]
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %z = zext i8 %a to i32
@@ -1213,8 +1213,8 @@ define i1 @test63(i8 %a, i32 %b) {
 
 define i1 @test64(i8 %a, i32 %b) {
 ; CHECK-LABEL: @test64(
-; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 %b to i8
-; CHECK-NEXT:    [[C:%.*]] = icmp eq i8 [[TMP1]], %a
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 [[B:%.*]] to i8
+; CHECK-NEXT:    [[C:%.*]] = icmp eq i8 [[TMP1]], [[A:%.*]]
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %t = and i32 %b, 255
@@ -1245,7 +1245,7 @@ define i1 @test66(i64 %A, i64 %B) {
 
 define i1 @test67(i32 %x) {
 ; CHECK-LABEL: @test67(
-; CHECK-NEXT:    [[AND:%.*]] = and i32 %x, 96
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 96
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[AND]], 0
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
@@ -1270,7 +1270,7 @@ define i1 @test67inverse(i32 %x) {
 
 define <2 x i1> @test67vec(<2 x i32> %x) {
 ; CHECK-LABEL: @test67vec(
-; CHECK-NEXT:    [[AND:%.*]] = and <2 x i32> %x, <i32 96, i32 96>
+; CHECK-NEXT:    [[AND:%.*]] = and <2 x i32> [[X:%.*]], <i32 96, i32 96>
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp ne <2 x i32> [[AND]], zeroinitializer
 ; CHECK-NEXT:    ret <2 x i1> [[CMP]]
 ;
@@ -1303,7 +1303,7 @@ define <2 x i1> @test67vecinverse(<2 x i32> %x) {
 
 define i1 @test68(i32 %x) {
 ; CHECK-LABEL: @test68(
-; CHECK-NEXT:    [[AND:%.*]] = and i32 %x, 127
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 127
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i32 [[AND]], 30
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
@@ -1315,7 +1315,7 @@ define i1 @test68(i32 %x) {
 ; PR15940
 define i1 @test70(i32 %X) {
 ; CHECK-LABEL: @test70(
-; CHECK-NEXT:    [[A:%.*]] = srem i32 5, %X
+; CHECK-NEXT:    [[A:%.*]] = srem i32 5, [[X:%.*]]
 ; CHECK-NEXT:    [[C:%.*]] = icmp ne i32 [[A]], 2
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
@@ -1327,7 +1327,7 @@ define i1 @test70(i32 %X) {
 
 define <2 x i1> @test70vec(<2 x i32> %X) {
 ; CHECK-LABEL: @test70vec(
-; CHECK-NEXT:    [[C:%.*]] = icmp ne <2 x i32> %X, <i32 2, i32 2>
+; CHECK-NEXT:    [[C:%.*]] = icmp ne <2 x i32> [[X:%.*]], <i32 2, i32 2>
 ; CHECK-NEXT:    ret <2 x i1> [[C]]
 ;
   %B = add <2 x i32> %X, <i32 2, i32 2>
@@ -1337,7 +1337,7 @@ define <2 x i1> @test70vec(<2 x i32> %X) {
 
 define i1 @icmp_sext16trunc(i32 %x) {
 ; CHECK-LABEL: @icmp_sext16trunc(
-; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 %x to i16
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 [[X:%.*]] to i16
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i16 [[TMP1]], 36
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
@@ -1349,7 +1349,7 @@ define i1 @icmp_sext16trunc(i32 %x) {
 
 define i1 @icmp_sext8trunc(i32 %x) {
 ; CHECK-LABEL: @icmp_sext8trunc(
-; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 %x to i8
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 [[X:%.*]] to i8
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i8 [[TMP1]], 36
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
@@ -1362,7 +1362,7 @@ define i1 @icmp_sext8trunc(i32 %x) {
 ; Vectors should fold the same way.
 define <2 x i1> @icmp_sext8trunc_vec(<2 x i32> %x) {
 ; CHECK-LABEL: @icmp_sext8trunc_vec(
-; CHECK-NEXT:    [[TMP1:%.*]] = trunc <2 x i32> %x to <2 x i8>
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc <2 x i32> [[X:%.*]] to <2 x i8>
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp slt <2 x i8> [[TMP1]], <i8 36, i8 36>
 ; CHECK-NEXT:    ret <2 x i1> [[CMP]]
 ;
@@ -1374,7 +1374,7 @@ define <2 x i1> @icmp_sext8trunc_vec(<2 x i32> %x) {
 
 define i1 @icmp_shl16(i32 %x) {
 ; CHECK-LABEL: @icmp_shl16(
-; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 %x to i16
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 [[X:%.*]] to i16
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i16 [[TMP1]], 36
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
@@ -1387,7 +1387,7 @@ define i1 @icmp_shl16(i32 %x) {
 
 define i1 @icmp_shl17(i32 %x) {
 ; CHECK-LABEL: @icmp_shl17(
-; CHECK-NEXT:    [[SHL:%.*]] = shl i32 %x, 17
+; CHECK-NEXT:    [[SHL:%.*]] = shl i32 [[X:%.*]], 17
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[SHL]], 2359296
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
@@ -1398,7 +1398,7 @@ define i1 @icmp_shl17(i32 %x) {
 
 define <2 x i1> @icmp_shl16_vec(<2 x i32> %x) {
 ; CHECK-LABEL: @icmp_shl16_vec(
-; CHECK-NEXT:    [[TMP1:%.*]] = trunc <2 x i32> %x to <2 x i16>
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc <2 x i32> [[X:%.*]] to <2 x i16>
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp slt <2 x i16> [[TMP1]], <i16 36, i16 36>
 ; CHECK-NEXT:    ret <2 x i1> [[CMP]]
 ;
@@ -1409,7 +1409,7 @@ define <2 x i1> @icmp_shl16_vec(<2 x i32> %x) {
 
 define i1 @icmp_shl24(i32 %x) {
 ; CHECK-LABEL: @icmp_shl24(
-; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 %x to i8
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 [[X:%.*]] to i8
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i8 [[TMP1]], 36
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
@@ -1420,7 +1420,7 @@ define i1 @icmp_shl24(i32 %x) {
 
 define i1 @icmp_shl_eq(i32 %x) {
 ; CHECK-LABEL: @icmp_shl_eq(
-; CHECK-NEXT:    [[MUL_MASK:%.*]] = and i32 %x, 134217727
+; CHECK-NEXT:    [[MUL_MASK:%.*]] = and i32 [[X:%.*]], 134217727
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[MUL_MASK]], 0
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
@@ -1431,7 +1431,7 @@ define i1 @icmp_shl_eq(i32 %x) {
 
 define <2 x i1> @icmp_shl_eq_vec(<2 x i32> %x) {
 ; CHECK-LABEL: @icmp_shl_eq_vec(
-; CHECK-NEXT:    [[MUL_MASK:%.*]] = and <2 x i32> %x, <i32 134217727, i32 134217727>
+; CHECK-NEXT:    [[MUL_MASK:%.*]] = and <2 x i32> [[X:%.*]], <i32 134217727, i32 134217727>
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <2 x i32> [[MUL_MASK]], zeroinitializer
 ; CHECK-NEXT:    ret <2 x i1> [[CMP]]
 ;
@@ -1442,7 +1442,7 @@ define <2 x i1> @icmp_shl_eq_vec(<2 x i32> %x) {
 
 define i1 @icmp_shl_nsw_ne(i32 %x) {
 ; CHECK-LABEL: @icmp_shl_nsw_ne(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 %x, 0
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[X:%.*]], 0
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %mul = shl nsw i32 %x, 7
@@ -1452,7 +1452,7 @@ define i1 @icmp_shl_nsw_ne(i32 %x) {
 
 define <2 x i1> @icmp_shl_nsw_ne_vec(<2 x i32> %x) {
 ; CHECK-LABEL: @icmp_shl_nsw_ne_vec(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ne <2 x i32> %x, zeroinitializer
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne <2 x i32> [[X:%.*]], zeroinitializer
 ; CHECK-NEXT:    ret <2 x i1> [[CMP]]
 ;
   %mul = shl nsw <2 x i32> %x, <i32 7, i32 7>
@@ -1462,7 +1462,7 @@ define <2 x i1> @icmp_shl_nsw_ne_vec(<2 x i32> %x) {
 
 define i1 @icmp_shl_ne(i32 %x) {
 ; CHECK-LABEL: @icmp_shl_ne(
-; CHECK-NEXT:    [[MUL_MASK:%.*]] = and i32 %x, 33554431
+; CHECK-NEXT:    [[MUL_MASK:%.*]] = and i32 [[X:%.*]], 33554431
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[MUL_MASK]], 0
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
@@ -1473,7 +1473,7 @@ define i1 @icmp_shl_ne(i32 %x) {
 
 define <2 x i1> @icmp_shl_ne_vec(<2 x i32> %x) {
 ; CHECK-LABEL: @icmp_shl_ne_vec(
-; CHECK-NEXT:    [[MUL_MASK:%.*]] = and <2 x i32> %x, <i32 33554431, i32 33554431>
+; CHECK-NEXT:    [[MUL_MASK:%.*]] = and <2 x i32> [[X:%.*]], <i32 33554431, i32 33554431>
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp ne <2 x i32> [[MUL_MASK]], zeroinitializer
 ; CHECK-NEXT:    ret <2 x i1> [[CMP]]
 ;
@@ -1484,7 +1484,7 @@ define <2 x i1> @icmp_shl_ne_vec(<2 x i32> %x) {
 
 define <2 x i1> @icmp_shl_nuw_ne_vec(<2 x i32> %x) {
 ; CHECK-LABEL: @icmp_shl_nuw_ne_vec(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ne <2 x i32> %x, <i32 2, i32 2>
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne <2 x i32> [[X:%.*]], <i32 2, i32 2>
 ; CHECK-NEXT:    ret <2 x i1> [[CMP]]
 ;
   %shl = shl nuw <2 x i32> %x, <i32 7, i32 7>
@@ -1496,7 +1496,7 @@ define <2 x i1> @icmp_shl_nuw_ne_vec(<2 x i32> %x) {
 ; compare the LHS operand instead
 define i1 @icmp_mul_nsw(i32 %x) {
 ; CHECK-LABEL: @icmp_mul_nsw(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 %x, 0
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[X:%.*]], 0
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %mul = mul nsw i32 %x, 12
@@ -1506,7 +1506,7 @@ define i1 @icmp_mul_nsw(i32 %x) {
 
 define i1 @icmp_mul_nsw1(i32 %x) {
 ; CHECK-LABEL: @icmp_mul_nsw1(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 %x, 0
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[X:%.*]], 0
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %mul = mul nsw i32 %x, 12
@@ -1516,7 +1516,7 @@ define i1 @icmp_mul_nsw1(i32 %x) {
 
 define i1 @icmp_mul_nsw_neg(i32 %x) {
 ; CHECK-LABEL: @icmp_mul_nsw_neg(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 %x, 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[X:%.*]], 1
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %mul = mul nsw i32 %x, -12
@@ -1526,7 +1526,7 @@ define i1 @icmp_mul_nsw_neg(i32 %x) {
 
 define i1 @icmp_mul_nsw_neg1(i32 %x) {
 ; CHECK-LABEL: @icmp_mul_nsw_neg1(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 %x, 0
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[X:%.*]], 0
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %mul = mul nsw i32 %x, -12
@@ -1536,7 +1536,7 @@ define i1 @icmp_mul_nsw_neg1(i32 %x) {
 
 define <2 x i1> @icmp_mul_nsw_neg1_vec(<2 x i32> %x) {
 ; CHECK-LABEL: @icmp_mul_nsw_neg1_vec(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp slt <2 x i32> %x, zeroinitializer
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt <2 x i32> [[X:%.*]], zeroinitializer
 ; CHECK-NEXT:    ret <2 x i1> [[CMP]]
 ;
   %mul = mul nsw <2 x i32> %x, <i32 -12, i32 -12>
@@ -1555,7 +1555,7 @@ define i1 @icmp_mul_nsw_0(i32 %x) {
 
 define i1 @icmp_mul(i32 %x) {
 ; CHECK-LABEL: @icmp_mul(
-; CHECK-NEXT:    [[MUL:%.*]] = mul i32 %x, -12
+; CHECK-NEXT:    [[MUL:%.*]] = mul i32 [[X:%.*]], -12
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[MUL]], -1
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
@@ -1567,7 +1567,7 @@ define i1 @icmp_mul(i32 %x) {
 ; Checks for icmp (eq|ne) (mul x, C), 0
 define i1 @icmp_mul_neq0(i32 %x) {
 ; CHECK-LABEL: @icmp_mul_neq0(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 %x, 0
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[X:%.*]], 0
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %mul = mul nsw i32 %x, -12
@@ -1577,7 +1577,7 @@ define i1 @icmp_mul_neq0(i32 %x) {
 
 define <2 x i1> @icmp_mul_neq0_vec(<2 x i32> %x) {
 ; CHECK-LABEL: @icmp_mul_neq0_vec(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ne <2 x i32> %x, zeroinitializer
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne <2 x i32> [[X:%.*]], zeroinitializer
 ; CHECK-NEXT:    ret <2 x i1> [[CMP]]
 ;
   %mul = mul nsw <2 x i32> %x, <i32 -12, i32 -12>
@@ -1587,7 +1587,7 @@ define <2 x i1> @icmp_mul_neq0_vec(<2 x i32> %x) {
 
 define i1 @icmp_mul_eq0(i32 %x) {
 ; CHECK-LABEL: @icmp_mul_eq0(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 %x, 0
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[X:%.*]], 0
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %mul = mul nsw i32 %x, 12
@@ -1615,7 +1615,7 @@ define i1 @icmp_mul0_ne0(i32 %x) {
 
 define i1 @icmp_sub1_sge(i32 %x, i32 %y) {
 ; CHECK-LABEL: @icmp_sub1_sge(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 %x, %y
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[X:%.*]], [[Y:%.*]]
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %sub = add nsw i32 %x, -1
@@ -1625,7 +1625,7 @@ define i1 @icmp_sub1_sge(i32 %x, i32 %y) {
 
 define i1 @icmp_add1_sgt(i32 %x, i32 %y) {
 ; CHECK-LABEL: @icmp_add1_sgt(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp sge i32 %x, %y
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sge i32 [[X:%.*]], [[Y:%.*]]
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %add = add nsw i32 %x, 1
@@ -1635,7 +1635,7 @@ define i1 @icmp_add1_sgt(i32 %x, i32 %y) {
 
 define i1 @icmp_sub1_slt(i32 %x, i32 %y) {
 ; CHECK-LABEL: @icmp_sub1_slt(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp sle i32 %x, %y
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sle i32 [[X:%.*]], [[Y:%.*]]
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %sub = add nsw i32 %x, -1
@@ -1645,7 +1645,7 @@ define i1 @icmp_sub1_slt(i32 %x, i32 %y) {
 
 define i1 @icmp_add1_sle(i32 %x, i32 %y) {
 ; CHECK-LABEL: @icmp_add1_sle(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 %x, %y
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[X:%.*]], [[Y:%.*]]
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %add = add nsw i32 %x, 1
@@ -1655,8 +1655,8 @@ define i1 @icmp_add1_sle(i32 %x, i32 %y) {
 
 define i1 @icmp_add20_sge_add57(i32 %x, i32 %y) {
 ; CHECK-LABEL: @icmp_add20_sge_add57(
-; CHECK-NEXT:    [[TMP1:%.*]] = add nsw i32 %y, 37
-; CHECK-NEXT:    [[CMP:%.*]] = icmp sle i32 [[TMP1]], %x
+; CHECK-NEXT:    [[TMP1:%.*]] = add nsw i32 [[Y:%.*]], 37
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sle i32 [[TMP1]], [[X:%.*]]
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %1 = add nsw i32 %x, 20
@@ -1667,8 +1667,8 @@ define i1 @icmp_add20_sge_add57(i32 %x, i32 %y) {
 
 define i1 @icmp_sub57_sge_sub20(i32 %x, i32 %y) {
 ; CHECK-LABEL: @icmp_sub57_sge_sub20(
-; CHECK-NEXT:    [[TMP1:%.*]] = add nsw i32 %x, -37
-; CHECK-NEXT:    [[CMP:%.*]] = icmp sge i32 [[TMP1]], %y
+; CHECK-NEXT:    [[TMP1:%.*]] = add nsw i32 [[X:%.*]], -37
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sge i32 [[TMP1]], [[Y:%.*]]
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %1 = add nsw i32 %x, -57
@@ -1679,8 +1679,8 @@ define i1 @icmp_sub57_sge_sub20(i32 %x, i32 %y) {
 
 define i1 @icmp_and_shl_neg_ne_0(i32 %A, i32 %B) {
 ; CHECK-LABEL: @icmp_and_shl_neg_ne_0(
-; CHECK-NEXT:    [[SHL:%.*]] = shl i32 1, %B
-; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[SHL]], %A
+; CHECK-NEXT:    [[SHL:%.*]] = shl i32 1, [[B:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[SHL]], [[A:%.*]]
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[TMP1]], 0
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
@@ -1693,8 +1693,8 @@ define i1 @icmp_and_shl_neg_ne_0(i32 %A, i32 %B) {
 
 define i1 @icmp_and_shl_neg_eq_0(i32 %A, i32 %B) {
 ; CHECK-LABEL: @icmp_and_shl_neg_eq_0(
-; CHECK-NEXT:    [[SHL:%.*]] = shl i32 1, %B
-; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[SHL]], %A
+; CHECK-NEXT:    [[SHL:%.*]] = shl i32 1, [[B:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[SHL]], [[A:%.*]]
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[TMP1]], 0
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
@@ -1707,7 +1707,7 @@ define i1 @icmp_and_shl_neg_eq_0(i32 %A, i32 %B) {
 
 define i1 @icmp_add_and_shr_ne_0(i32 %X) {
 ; CHECK-LABEL: @icmp_add_and_shr_ne_0(
-; CHECK-NEXT:    [[AND:%.*]] = and i32 %X, 240
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 240
 ; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp ne i32 [[AND]], 224
 ; CHECK-NEXT:    ret i1 [[TOBOOL]]
 ;
@@ -1720,7 +1720,7 @@ define i1 @icmp_add_and_shr_ne_0(i32 %X) {
 
 define <2 x i1> @icmp_add_and_shr_ne_0_vec(<2 x i32> %X) {
 ; CHECK-LABEL: @icmp_add_and_shr_ne_0_vec(
-; CHECK-NEXT:    [[AND:%.*]] = and <2 x i32> %X, <i32 240, i32 240>
+; CHECK-NEXT:    [[AND:%.*]] = and <2 x i32> [[X:%.*]], <i32 240, i32 240>
 ; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp ne <2 x i32> [[AND]], <i32 224, i32 224>
 ; CHECK-NEXT:    ret <2 x i1> [[TOBOOL]]
 ;
@@ -1817,7 +1817,7 @@ define i1 @test71_as1(i8 addrspace(1)* %x) {
 
 define i1 @icmp_shl_1_V_ult_32(i32 %V) {
 ; CHECK-LABEL: @icmp_shl_1_V_ult_32(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 %V, 5
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[V:%.*]], 5
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %shl = shl i32 1, %V
@@ -1827,7 +1827,7 @@ define i1 @icmp_shl_1_V_ult_32(i32 %V) {
 
 define <2 x i1> @icmp_shl_1_V_ult_32_vec(<2 x i32> %V) {
 ; CHECK-LABEL: @icmp_shl_1_V_ult_32_vec(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ult <2 x i32> %V, <i32 5, i32 5>
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult <2 x i32> [[V:%.*]], <i32 5, i32 5>
 ; CHECK-NEXT:    ret <2 x i1> [[CMP]]
 ;
   %shl = shl <2 x i32> <i32 1, i32 1>, %V
@@ -1837,7 +1837,7 @@ define <2 x i1> @icmp_shl_1_V_ult_32_vec(<2 x i32> %V) {
 
 define i1 @icmp_shl_1_V_eq_32(i32 %V) {
 ; CHECK-LABEL: @icmp_shl_1_V_eq_32(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 %V, 5
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[V:%.*]], 5
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %shl = shl i32 1, %V
@@ -1847,7 +1847,7 @@ define i1 @icmp_shl_1_V_eq_32(i32 %V) {
 
 define <2 x i1> @icmp_shl_1_V_eq_32_vec(<2 x i32> %V) {
 ; CHECK-LABEL: @icmp_shl_1_V_eq_32_vec(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <2 x i32> %V, <i32 5, i32 5>
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <2 x i32> [[V:%.*]], <i32 5, i32 5>
 ; CHECK-NEXT:    ret <2 x i1> [[CMP]]
 ;
   %shl = shl <2 x i32> <i32 1, i32 1>, %V
@@ -1857,7 +1857,7 @@ define <2 x i1> @icmp_shl_1_V_eq_32_vec(<2 x i32> %V) {
 
 define i1 @icmp_shl_1_V_ult_30(i32 %V) {
 ; CHECK-LABEL: @icmp_shl_1_V_ult_30(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 %V, 5
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[V:%.*]], 5
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %shl = shl i32 1, %V
@@ -1867,7 +1867,7 @@ define i1 @icmp_shl_1_V_ult_30(i32 %V) {
 
 define <2 x i1> @icmp_shl_1_V_ult_30_vec(<2 x i32> %V) {
 ; CHECK-LABEL: @icmp_shl_1_V_ult_30_vec(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ult <2 x i32> %V, <i32 5, i32 5>
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult <2 x i32> [[V:%.*]], <i32 5, i32 5>
 ; CHECK-NEXT:    ret <2 x i1> [[CMP]]
 ;
   %shl = shl <2 x i32> <i32 1, i32 1>, %V
@@ -1877,7 +1877,7 @@ define <2 x i1> @icmp_shl_1_V_ult_30_vec(<2 x i32> %V) {
 
 define i1 @icmp_shl_1_V_ugt_30(i32 %V) {
 ; CHECK-LABEL: @icmp_shl_1_V_ugt_30(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i32 %V, 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i32 [[V:%.*]], 4
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %shl = shl i32 1, %V
@@ -1887,7 +1887,7 @@ define i1 @icmp_shl_1_V_ugt_30(i32 %V) {
 
 define <2 x i1> @icmp_shl_1_V_ugt_30_vec(<2 x i32> %V) {
 ; CHECK-LABEL: @icmp_shl_1_V_ugt_30_vec(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt <2 x i32> %V, <i32 4, i32 4>
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt <2 x i32> [[V:%.*]], <i32 4, i32 4>
 ; CHECK-NEXT:    ret <2 x i1> [[CMP]]
 ;
   %shl = shl <2 x i32> <i32 1, i32 1>, %V
@@ -1897,7 +1897,7 @@ define <2 x i1> @icmp_shl_1_V_ugt_30_vec(<2 x i32> %V) {
 
 define i1 @icmp_shl_1_V_ule_30(i32 %V) {
 ; CHECK-LABEL: @icmp_shl_1_V_ule_30(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 %V, 5
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[V:%.*]], 5
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %shl = shl i32 1, %V
@@ -1907,7 +1907,7 @@ define i1 @icmp_shl_1_V_ule_30(i32 %V) {
 
 define <2 x i1> @icmp_shl_1_V_ule_30_vec(<2 x i32> %V) {
 ; CHECK-LABEL: @icmp_shl_1_V_ule_30_vec(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ult <2 x i32> %V, <i32 5, i32 5>
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult <2 x i32> [[V:%.*]], <i32 5, i32 5>
 ; CHECK-NEXT:    ret <2 x i1> [[CMP]]
 ;
   %shl = shl <2 x i32> <i32 1, i32 1>, %V
@@ -1917,7 +1917,7 @@ define <2 x i1> @icmp_shl_1_V_ule_30_vec(<2 x i32> %V) {
 
 define i1 @icmp_shl_1_V_uge_30(i32 %V) {
 ; CHECK-LABEL: @icmp_shl_1_V_uge_30(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i32 %V, 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i32 [[V:%.*]], 4
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %shl = shl i32 1, %V
@@ -1927,7 +1927,7 @@ define i1 @icmp_shl_1_V_uge_30(i32 %V) {
 
 define <2 x i1> @icmp_shl_1_V_uge_30_vec(<2 x i32> %V) {
 ; CHECK-LABEL: @icmp_shl_1_V_uge_30_vec(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt <2 x i32> %V, <i32 4, i32 4>
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt <2 x i32> [[V:%.*]], <i32 4, i32 4>
 ; CHECK-NEXT:    ret <2 x i1> [[CMP]]
 ;
   %shl = shl <2 x i32> <i32 1, i32 1>, %V
@@ -1937,7 +1937,7 @@ define <2 x i1> @icmp_shl_1_V_uge_30_vec(<2 x i32> %V) {
 
 define i1 @icmp_shl_1_V_uge_2147483648(i32 %V) {
 ; CHECK-LABEL: @icmp_shl_1_V_uge_2147483648(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 %V, 31
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[V:%.*]], 31
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %shl = shl i32 1, %V
@@ -1947,7 +1947,7 @@ define i1 @icmp_shl_1_V_uge_2147483648(i32 %V) {
 
 define <2 x i1> @icmp_shl_1_V_uge_2147483648_vec(<2 x i32> %V) {
 ; CHECK-LABEL: @icmp_shl_1_V_uge_2147483648_vec(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <2 x i32> %V, <i32 31, i32 31>
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <2 x i32> [[V:%.*]], <i32 31, i32 31>
 ; CHECK-NEXT:    ret <2 x i1> [[CMP]]
 ;
   %shl = shl <2 x i32> <i32 1, i32 1>, %V
@@ -1957,7 +1957,7 @@ define <2 x i1> @icmp_shl_1_V_uge_2147483648_vec(<2 x i32> %V) {
 
 define i1 @icmp_shl_1_V_ult_2147483648(i32 %V) {
 ; CHECK-LABEL: @icmp_shl_1_V_ult_2147483648(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 %V, 31
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[V:%.*]], 31
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %shl = shl i32 1, %V
@@ -1967,7 +1967,7 @@ define i1 @icmp_shl_1_V_ult_2147483648(i32 %V) {
 
 define <2 x i1> @icmp_shl_1_V_ult_2147483648_vec(<2 x i32> %V) {
 ; CHECK-LABEL: @icmp_shl_1_V_ult_2147483648_vec(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ne <2 x i32> %V, <i32 31, i32 31>
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne <2 x i32> [[V:%.*]], <i32 31, i32 31>
 ; CHECK-NEXT:    ret <2 x i1> [[CMP]]
 ;
   %shl = shl <2 x i32> <i32 1, i32 1>, %V
@@ -1977,8 +1977,8 @@ define <2 x i1> @icmp_shl_1_V_ult_2147483648_vec(<2 x i32> %V) {
 
 define i1 @or_icmp_eq_B_0_icmp_ult_A_B(i64 %a, i64 %b) {
 ; CHECK-LABEL: @or_icmp_eq_B_0_icmp_ult_A_B(
-; CHECK-NEXT:    [[TMP1:%.*]] = add i64 %b, -1
-; CHECK-NEXT:    [[TMP2:%.*]] = icmp uge i64 [[TMP1]], %a
+; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[B:%.*]], -1
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp uge i64 [[TMP1]], [[A:%.*]]
 ; CHECK-NEXT:    ret i1 [[TMP2]]
 ;
   %1 = icmp eq i64 %b, 0
@@ -1989,7 +1989,7 @@ define i1 @or_icmp_eq_B_0_icmp_ult_A_B(i64 %a, i64 %b) {
 
 define i1 @icmp_add_ult_2(i32 %X) {
 ; CHECK-LABEL: @icmp_add_ult_2(
-; CHECK-NEXT:    [[TMP1:%.*]] = and i32 %X, -2
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X:%.*]], -2
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[TMP1]], 14
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
@@ -2000,7 +2000,7 @@ define i1 @icmp_add_ult_2(i32 %X) {
 
 define <2 x i1> @icmp_add_X_-14_ult_2_vec(<2 x i32> %X) {
 ; CHECK-LABEL: @icmp_add_X_-14_ult_2_vec(
-; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i32> %X, <i32 -2, i32 -2>
+; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i32> [[X:%.*]], <i32 -2, i32 -2>
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <2 x i32> [[TMP1]], <i32 14, i32 14>
 ; CHECK-NEXT:    ret <2 x i1> [[CMP]]
 ;
@@ -2011,7 +2011,7 @@ define <2 x i1> @icmp_add_X_-14_ult_2_vec(<2 x i32> %X) {
 
 define i1 @icmp_sub_3_X_ult_2(i32 %X) {
 ; CHECK-LABEL: @icmp_sub_3_X_ult_2(
-; CHECK-NEXT:    [[TMP1:%.*]] = or i32 %X, 1
+; CHECK-NEXT:    [[TMP1:%.*]] = or i32 [[X:%.*]], 1
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[TMP1]], 3
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
@@ -2022,7 +2022,7 @@ define i1 @icmp_sub_3_X_ult_2(i32 %X) {
 
 define <2 x i1> @icmp_sub_3_X_ult_2_vec(<2 x i32> %X) {
 ; CHECK-LABEL: @icmp_sub_3_X_ult_2_vec(
-; CHECK-NEXT:    [[TMP1:%.*]] = or <2 x i32> %X, <i32 1, i32 1>
+; CHECK-NEXT:    [[TMP1:%.*]] = or <2 x i32> [[X:%.*]], <i32 1, i32 1>
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <2 x i32> [[TMP1]], <i32 3, i32 3>
 ; CHECK-NEXT:    ret <2 x i1> [[CMP]]
 ;
@@ -2033,7 +2033,7 @@ define <2 x i1> @icmp_sub_3_X_ult_2_vec(<2 x i32> %X) {
 
 define i1 @icmp_add_X_-14_uge_2(i32 %X) {
 ; CHECK-LABEL: @icmp_add_X_-14_uge_2(
-; CHECK-NEXT:    [[TMP1:%.*]] = and i32 %X, -2
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X:%.*]], -2
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[TMP1]], 14
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
@@ -2044,7 +2044,7 @@ define i1 @icmp_add_X_-14_uge_2(i32 %X) {
 
 define <2 x i1> @icmp_add_X_-14_uge_2_vec(<2 x i32> %X) {
 ; CHECK-LABEL: @icmp_add_X_-14_uge_2_vec(
-; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i32> %X, <i32 -2, i32 -2>
+; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i32> [[X:%.*]], <i32 -2, i32 -2>
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp ne <2 x i32> [[TMP1]], <i32 14, i32 14>
 ; CHECK-NEXT:    ret <2 x i1> [[CMP]]
 ;
@@ -2055,7 +2055,7 @@ define <2 x i1> @icmp_add_X_-14_uge_2_vec(<2 x i32> %X) {
 
 define i1 @icmp_sub_3_X_uge_2(i32 %X) {
 ; CHECK-LABEL: @icmp_sub_3_X_uge_2(
-; CHECK-NEXT:    [[TMP1:%.*]] = or i32 %X, 1
+; CHECK-NEXT:    [[TMP1:%.*]] = or i32 [[X:%.*]], 1
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[TMP1]], 3
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
@@ -2066,7 +2066,7 @@ define i1 @icmp_sub_3_X_uge_2(i32 %X) {
 
 define <2 x i1> @icmp_sub_3_X_uge_2_vec(<2 x i32> %X) {
 ; CHECK-LABEL: @icmp_sub_3_X_uge_2_vec(
-; CHECK-NEXT:    [[TMP1:%.*]] = or <2 x i32> %X, <i32 1, i32 1>
+; CHECK-NEXT:    [[TMP1:%.*]] = or <2 x i32> [[X:%.*]], <i32 1, i32 1>
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp ne <2 x i32> [[TMP1]], <i32 3, i32 3>
 ; CHECK-NEXT:    ret <2 x i1> [[CMP]]
 ;
@@ -2077,7 +2077,7 @@ define <2 x i1> @icmp_sub_3_X_uge_2_vec(<2 x i32> %X) {
 
 define i1 @icmp_and_X_-16_eq-16(i32 %X) {
 ; CHECK-LABEL: @icmp_and_X_-16_eq-16(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i32 %X, -17
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i32 [[X:%.*]], -17
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %and = and i32 %X, -16
@@ -2087,7 +2087,7 @@ define i1 @icmp_and_X_-16_eq-16(i32 %X) {
 
 define <2 x i1> @icmp_and_X_-16_eq-16_vec(<2 x i32> %X) {
 ; CHECK-LABEL: @icmp_and_X_-16_eq-16_vec(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt <2 x i32> %X, <i32 -17, i32 -17>
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt <2 x i32> [[X:%.*]], <i32 -17, i32 -17>
 ; CHECK-NEXT:    ret <2 x i1> [[CMP]]
 ;
   %and = and <2 x i32> %X, <i32 -16, i32 -16>
@@ -2097,7 +2097,7 @@ define <2 x i1> @icmp_and_X_-16_eq-16_vec(<2 x i32> %X) {
 
 define i1 @icmp_and_X_-16_ne-16(i32 %X) {
 ; CHECK-LABEL: @icmp_and_X_-16_ne-16(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 %X, -16
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[X:%.*]], -16
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %and = and i32 %X, -16
@@ -2107,7 +2107,7 @@ define i1 @icmp_and_X_-16_ne-16(i32 %X) {
 
 define <2 x i1> @icmp_and_X_-16_ne-16_vec(<2 x i32> %X) {
 ; CHECK-LABEL: @icmp_and_X_-16_ne-16_vec(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ult <2 x i32> %X, <i32 -16, i32 -16>
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult <2 x i32> [[X:%.*]], <i32 -16, i32 -16>
 ; CHECK-NEXT:    ret <2 x i1> [[CMP]]
 ;
   %and = and <2 x i32> %X, <i32 -16, i32 -16>
@@ -2120,7 +2120,7 @@ define <2 x i1> @icmp_and_X_-16_ne-16_vec(<2 x i32> %X) {
 
 define i1 @or1_eq1(i32 %x) {
 ; CHECK-LABEL: @or1_eq1(
-; CHECK-NEXT:    [[T1:%.*]] = icmp ult i32 %x, 2
+; CHECK-NEXT:    [[T1:%.*]] = icmp ult i32 [[X:%.*]], 2
 ; CHECK-NEXT:    ret i1 [[T1]]
 ;
   %t0 = or i32 %x, 1
@@ -2132,7 +2132,7 @@ define i1 @or1_eq1(i32 %x) {
 
 define <2 x i1> @or3_eq3_vec(<2 x i8> %x) {
 ; CHECK-LABEL: @or3_eq3_vec(
-; CHECK-NEXT:    [[T1:%.*]] = icmp ult <2 x i8> %x, <i8 4, i8 4>
+; CHECK-NEXT:    [[T1:%.*]] = icmp ult <2 x i8> [[X:%.*]], <i8 4, i8 4>
 ; CHECK-NEXT:    ret <2 x i1> [[T1]]
 ;
   %t0 = or <2 x i8> %x, <i8 3, i8 3>
@@ -2144,7 +2144,7 @@ define <2 x i1> @or3_eq3_vec(<2 x i8> %x) {
 
 define i1 @or7_ne7(i32 %x) {
 ; CHECK-LABEL: @or7_ne7(
-; CHECK-NEXT:    [[T1:%.*]] = icmp ugt i32 %x, 7
+; CHECK-NEXT:    [[T1:%.*]] = icmp ugt i32 [[X:%.*]], 7
 ; CHECK-NEXT:    ret i1 [[T1]]
 ;
   %t0 = or i32 %x, 7
@@ -2156,7 +2156,7 @@ define i1 @or7_ne7(i32 %x) {
 
 define <2 x i1> @or63_ne63_vec(<2 x i8> %x) {
 ; CHECK-LABEL: @or63_ne63_vec(
-; CHECK-NEXT:    [[T1:%.*]] = icmp ugt <2 x i8> %x, <i8 63, i8 63>
+; CHECK-NEXT:    [[T1:%.*]] = icmp ugt <2 x i8> [[X:%.*]], <i8 63, i8 63>
 ; CHECK-NEXT:    ret <2 x i1> [[T1]]
 ;
   %t0 = or <2 x i8> %x, <i8 63, i8 63>
@@ -2164,9 +2164,62 @@ define <2 x i1> @or63_ne63_vec(<2 x i8> %x) {
   ret <2 x i1> %t1
 }
 
+; PR40611: https://bugs.llvm.org/show_bug.cgi?id=40611
+; X | C == C --> (X & ~C) == 0
+
+define i1 @orC_eqC(i32 %x) {
+; CHECK-LABEL: @orC_eqC(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X:%.*]], -43
+; CHECK-NEXT:    [[T1:%.*]] = icmp eq i32 [[TMP1]], 0
+; CHECK-NEXT:    ret i1 [[T1]]
+;
+  %t0 = or i32 %x, 42
+  %t1 = icmp eq i32 %t0, 42
+  ret i1 %t1
+}
+
+; X | C == C --> (X & ~C) == 0
+
+define <2 x i1> @orC_eqC_vec(<2 x i8> %x) {
+; CHECK-LABEL: @orC_eqC_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i8> [[X:%.*]], <i8 -44, i8 -44>
+; CHECK-NEXT:    [[T1:%.*]] = icmp eq <2 x i8> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[T1]]
+;
+  %t0 = or <2 x i8> %x, <i8 43, i8 43>
+  %t1 = icmp eq <2 x i8> %t0, <i8 43, i8 43>
+  ret <2 x i1> %t1
+}
+
+; X | C != C --> (X & ~C) != 0
+
+define i1 @orC_neC(i32 %x) {
+; CHECK-LABEL: @orC_neC(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X:%.*]], 41
+; CHECK-NEXT:    [[T1:%.*]] = icmp ne i32 [[TMP1]], 0
+; CHECK-NEXT:    ret i1 [[T1]]
+;
+  %t0 = or i32 %x, -42
+  %t1 = icmp ne i32 %t0, -42
+  ret i1 %t1
+}
+
+; X | C != C --> (X & ~C) != 0
+
+define <2 x i1> @orC_neC_vec(<2 x i8> %x) {
+; CHECK-LABEL: @orC_neC_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i8> [[X:%.*]], <i8 42, i8 42>
+; CHECK-NEXT:    [[T1:%.*]] = icmp ne <2 x i8> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    ret <2 x i1> [[T1]]
+;
+  %t0 = or <2 x i8> %x, <i8 -43, i8 -43>
+  %t1 = icmp ne <2 x i8> %t0, <i8 -43, i8 -43>
+  ret <2 x i1> %t1
+}
+
 define i1 @shrink_constant(i32 %X) {
 ; CHECK-LABEL: @shrink_constant(
-; CHECK-NEXT:    [[XOR:%.*]] = xor i32 %X, -12
+; CHECK-NEXT:    [[XOR:%.*]] = xor i32 [[X:%.*]], -12
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[XOR]], 4
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
@@ -2189,7 +2242,7 @@ define <2 x i1> @shrink_constant_vec(<2 x i32> %X) {
 ; This test requires 3 different transforms to get to the result.
 define i1 @icmp_sub_-1_X_ult_4(i32 %X) {
 ; CHECK-LABEL: @icmp_sub_-1_X_ult_4(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i32 %X, -5
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i32 [[X:%.*]], -5
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %sub = sub i32 -1, %X
@@ -2199,7 +2252,7 @@ define i1 @icmp_sub_-1_X_ult_4(i32 %X) {
 
 define <2 x i1> @icmp_xor_neg4_X_ult_4_vec(<2 x i32> %X) {
 ; CHECK-LABEL: @icmp_xor_neg4_X_ult_4_vec(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt <2 x i32> %X, <i32 -5, i32 -5>
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt <2 x i32> [[X:%.*]], <i32 -5, i32 -5>
 ; CHECK-NEXT:    ret <2 x i1> [[CMP]]
 ;
   %xor = xor <2 x i32> %X, <i32 -4, i32 -4>
@@ -2209,7 +2262,7 @@ define <2 x i1> @icmp_xor_neg4_X_ult_4_vec(<2 x i32> %X) {
 
 define i1 @icmp_sub_-1_X_uge_4(i32 %X) {
 ; CHECK-LABEL: @icmp_sub_-1_X_uge_4(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 %X, -4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[X:%.*]], -4
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %sub = sub i32 -1, %X
@@ -2219,7 +2272,7 @@ define i1 @icmp_sub_-1_X_uge_4(i32 %X) {
 
 define <2 x i1> @icmp_xor_neg4_X_uge_4_vec(<2 x i32> %X) {
 ; CHECK-LABEL: @icmp_xor_neg4_X_uge_4_vec(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ult <2 x i32> %X, <i32 -4, i32 -4>
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult <2 x i32> [[X:%.*]], <i32 -4, i32 -4>
 ; CHECK-NEXT:    ret <2 x i1> [[CMP]]
 ;
   %xor = xor <2 x i32> %X, <i32 -4, i32 -4>
@@ -2276,18 +2329,18 @@ define i1 @xor_ugt_extra_use(i8 %x, i8* %p) {
 define i1 @icmp_swap_operands_for_cse(i32 %X, i32 %Y) {
 ; CHECK-LABEL: @icmp_swap_operands_for_cse(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[SUB:%.*]] = sub i32 %X, %Y
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 %X, %Y
-; CHECK-NEXT:    br i1 [[CMP]], label %true, label %false
+; CHECK-NEXT:    [[SUB:%.*]] = sub i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[X]], [[Y]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[TRUE:%.*]], label [[FALSE:%.*]]
 ; CHECK:       true:
 ; CHECK-NEXT:    [[TMP0:%.*]] = and i32 [[SUB]], 1
-; CHECK-NEXT:    br label %end
+; CHECK-NEXT:    br label [[END:%.*]]
 ; CHECK:       false:
 ; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[SUB]], 16
-; CHECK-NEXT:    br label %end
+; CHECK-NEXT:    br label [[END]]
 ; CHECK:       end:
-; CHECK-NEXT:    [[RES_IN:%.*]] = phi i32 [ [[TMP0]], %true ], [ [[TMP1]], %false ]
-; CHECK-NEXT:    [[RES:%.*]] = icmp ne i32 [[RES:%.*]].in, 0
+; CHECK-NEXT:    [[RES_IN:%.*]] = phi i32 [ [[TMP0]], [[TRUE]] ], [ [[TMP1]], [[FALSE]] ]
+; CHECK-NEXT:    [[RES:%.*]] = icmp ne i32 [[RES_IN]], 0
 ; CHECK-NEXT:    ret i1 [[RES]]
 ;
 entry:
@@ -2309,20 +2362,20 @@ end:
 define i1 @icmp_swap_operands_for_cse2(i32 %X, i32 %Y) {
 ; CHECK-LABEL: @icmp_swap_operands_for_cse2(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 %X, %Y
-; CHECK-NEXT:    br i1 [[CMP]], label %true, label %false
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[TRUE:%.*]], label [[FALSE:%.*]]
 ; CHECK:       true:
-; CHECK-NEXT:    [[SUB:%.*]] = sub i32 %X, %Y
-; CHECK-NEXT:    [[SUB1:%.*]] = sub i32 %X, %Y
+; CHECK-NEXT:    [[SUB:%.*]] = sub i32 [[X]], [[Y]]
+; CHECK-NEXT:    [[SUB1:%.*]] = sub i32 [[X]], [[Y]]
 ; CHECK-NEXT:    [[ADD:%.*]] = add i32 [[SUB]], [[SUB1]]
-; CHECK-NEXT:    br label %end
+; CHECK-NEXT:    br label [[END:%.*]]
 ; CHECK:       false:
-; CHECK-NEXT:    [[SUB2:%.*]] = sub i32 %Y, %X
-; CHECK-NEXT:    br label %end
+; CHECK-NEXT:    [[SUB2:%.*]] = sub i32 [[Y]], [[X]]
+; CHECK-NEXT:    br label [[END]]
 ; CHECK:       end:
-; CHECK-NEXT:    [[RES_IN_IN:%.*]] = phi i32 [ [[ADD]], %true ], [ [[SUB2]], %false ]
-; CHECK-NEXT:    [[RES_IN:%.*]] = and i32 [[RES_IN:%.*]].in, 1
-; CHECK-NEXT:    [[RES:%.*]] = icmp ne i32 [[RES:%.*]].in, 0
+; CHECK-NEXT:    [[RES_IN_IN:%.*]] = phi i32 [ [[ADD]], [[TRUE]] ], [ [[SUB2]], [[FALSE]] ]
+; CHECK-NEXT:    [[RES_IN:%.*]] = and i32 [[RES_IN_IN]], 1
+; CHECK-NEXT:    [[RES:%.*]] = icmp ne i32 [[RES_IN]], 0
 ; CHECK-NEXT:    ret i1 [[RES]]
 ;
 entry:
@@ -2346,18 +2399,18 @@ end:
 define i1 @icmp_do_not_swap_operands_for_cse(i32 %X, i32 %Y) {
 ; CHECK-LABEL: @icmp_do_not_swap_operands_for_cse(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i32 %Y, %X
-; CHECK-NEXT:    br i1 [[CMP]], label %true, label %false
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i32 [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[TRUE:%.*]], label [[FALSE:%.*]]
 ; CHECK:       true:
-; CHECK-NEXT:    [[SUB:%.*]] = sub i32 %X, %Y
-; CHECK-NEXT:    br label %end
+; CHECK-NEXT:    [[SUB:%.*]] = sub i32 [[X]], [[Y]]
+; CHECK-NEXT:    br label [[END:%.*]]
 ; CHECK:       false:
-; CHECK-NEXT:    [[SUB2:%.*]] = sub i32 %Y, %X
-; CHECK-NEXT:    br label %end
+; CHECK-NEXT:    [[SUB2:%.*]] = sub i32 [[Y]], [[X]]
+; CHECK-NEXT:    br label [[END]]
 ; CHECK:       end:
-; CHECK-NEXT:    [[RES_IN_IN:%.*]] = phi i32 [ [[SUB]], %true ], [ [[SUB2]], %false ]
-; CHECK-NEXT:    [[RES_IN:%.*]] = and i32 [[RES_IN:%.*]].in, 1
-; CHECK-NEXT:    [[RES:%.*]] = icmp ne i32 [[RES:%.*]].in, 0
+; CHECK-NEXT:    [[RES_IN_IN:%.*]] = phi i32 [ [[SUB]], [[TRUE]] ], [ [[SUB2]], [[FALSE]] ]
+; CHECK-NEXT:    [[RES_IN:%.*]] = and i32 [[RES_IN_IN]], 1
+; CHECK-NEXT:    [[RES:%.*]] = icmp ne i32 [[RES_IN]], 0
 ; CHECK-NEXT:    ret i1 [[RES]]
 ;
 entry:
@@ -2378,8 +2431,8 @@ end:
 
 define i1 @icmp_lshr_lshr_eq(i32 %a, i32 %b) {
 ; CHECK-LABEL: @icmp_lshr_lshr_eq(
-; CHECK-NEXT:    [[Z_UNSHIFTED:%.*]] = xor i32 %a, %b
-; CHECK-NEXT:    [[Z:%.*]] = icmp ult i32 [[Z:%.*]].unshifted, 1073741824
+; CHECK-NEXT:    [[Z_UNSHIFTED:%.*]] = xor i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[Z:%.*]] = icmp ult i32 [[Z_UNSHIFTED]], 1073741824
 ; CHECK-NEXT:    ret i1 [[Z]]
 ;
   %x = lshr i32 %a, 30
@@ -2390,8 +2443,8 @@ define i1 @icmp_lshr_lshr_eq(i32 %a, i32 %b) {
 
 define i1 @icmp_ashr_ashr_ne(i32 %a, i32 %b) {
 ; CHECK-LABEL: @icmp_ashr_ashr_ne(
-; CHECK-NEXT:    [[Z_UNSHIFTED:%.*]] = xor i32 %a, %b
-; CHECK-NEXT:    [[Z:%.*]] = icmp ugt i32 [[Z:%.*]].unshifted, 255
+; CHECK-NEXT:    [[Z_UNSHIFTED:%.*]] = xor i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[Z:%.*]] = icmp ugt i32 [[Z_UNSHIFTED]], 255
 ; CHECK-NEXT:    ret i1 [[Z]]
 ;
   %x = ashr i32 %a, 8
@@ -2402,7 +2455,7 @@ define i1 @icmp_ashr_ashr_ne(i32 %a, i32 %b) {
 
 define i1 @icmp_neg_cst_slt(i32 %a) {
 ; CHECK-LABEL: @icmp_neg_cst_slt(
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i32 %a, 10
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i32 [[A:%.*]], 10
 ; CHECK-NEXT:    ret i1 [[TMP1]]
 ;
   %1 = sub nsw i32 0, %a
@@ -2412,9 +2465,9 @@ define i1 @icmp_neg_cst_slt(i32 %a) {
 
 define i1 @icmp_and_or_lshr(i32 %x, i32 %y) {
 ; CHECK-LABEL: @icmp_and_or_lshr(
-; CHECK-NEXT:    [[SHF1:%.*]] = shl nuw i32 1, %y
+; CHECK-NEXT:    [[SHF1:%.*]] = shl nuw i32 1, [[Y:%.*]]
 ; CHECK-NEXT:    [[OR2:%.*]] = or i32 [[SHF1]], 1
-; CHECK-NEXT:    [[AND3:%.*]] = and i32 [[OR2]], %x
+; CHECK-NEXT:    [[AND3:%.*]] = and i32 [[OR2]], [[X:%.*]]
 ; CHECK-NEXT:    [[RET:%.*]] = icmp ne i32 [[AND3]], 0
 ; CHECK-NEXT:    ret i1 [[RET]]
 ;
@@ -2457,7 +2510,7 @@ define <2 x i1> @icmp_and_or_lshr_vec_commute(<2 x i32> %xp, <2 x i32> %y) {
 
 define i1 @icmp_and_or_lshr_cst(i32 %x) {
 ; CHECK-LABEL: @icmp_and_or_lshr_cst(
-; CHECK-NEXT:    [[AND1:%.*]] = and i32 %x, 3
+; CHECK-NEXT:    [[AND1:%.*]] = and i32 [[X:%.*]], 3
 ; CHECK-NEXT:    [[RET:%.*]] = icmp ne i32 [[AND1]], 0
 ; CHECK-NEXT:    ret i1 [[RET]]
 ;
@@ -2498,7 +2551,7 @@ define <2 x i1> @icmp_and_or_lshr_cst_vec_commute(<2 x i32> %xp) {
 
 define i1 @shl_ap1_zero_ap2_non_zero_2(i32 %a) {
 ; CHECK-LABEL: @shl_ap1_zero_ap2_non_zero_2(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i32 %a, 29
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i32 [[A:%.*]], 29
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %shl = shl i32 4, %a
@@ -2508,7 +2561,7 @@ define i1 @shl_ap1_zero_ap2_non_zero_2(i32 %a) {
 
 define <2 x i1> @shl_ap1_zero_ap2_non_zero_2_vec(<2 x i32> %a) {
 ; CHECK-LABEL: @shl_ap1_zero_ap2_non_zero_2_vec(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt <2 x i32> %a, <i32 29, i32 29>
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt <2 x i32> [[A:%.*]], <i32 29, i32 29>
 ; CHECK-NEXT:    ret <2 x i1> [[CMP]]
 ;
   %shl = shl <2 x i32> <i32 4, i32 4>, %a
@@ -2518,7 +2571,7 @@ define <2 x i1> @shl_ap1_zero_ap2_non_zero_2_vec(<2 x i32> %a) {
 
 define i1 @shl_ap1_zero_ap2_non_zero_4(i32 %a) {
 ; CHECK-LABEL: @shl_ap1_zero_ap2_non_zero_4(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i32 %a, 30
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i32 [[A:%.*]], 30
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %shl = shl i32 -2, %a
@@ -2528,7 +2581,7 @@ define i1 @shl_ap1_zero_ap2_non_zero_4(i32 %a) {
 
 define i1 @shl_ap1_non_zero_ap2_non_zero_both_positive(i32 %a) {
 ; CHECK-LABEL: @shl_ap1_non_zero_ap2_non_zero_both_positive(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 %a, 0
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[A:%.*]], 0
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %shl = shl i32 50, %a
@@ -2538,7 +2591,7 @@ define i1 @shl_ap1_non_zero_ap2_non_zero_both_positive(i32 %a) {
 
 define i1 @shl_ap1_non_zero_ap2_non_zero_both_negative(i32 %a) {
 ; CHECK-LABEL: @shl_ap1_non_zero_ap2_non_zero_both_negative(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 %a, 0
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[A:%.*]], 0
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %shl = shl i32 -50, %a
@@ -2557,7 +2610,7 @@ define i1 @shl_ap1_non_zero_ap2_non_zero_ap1_1(i32 %a) {
 
 define i1 @shl_ap1_non_zero_ap2_non_zero_ap1_2(i32 %a) {
 ; CHECK-LABEL: @shl_ap1_non_zero_ap2_non_zero_ap1_2(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 %a, 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[A:%.*]], 1
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %shl = shl i32 25, %a
@@ -2576,7 +2629,7 @@ define i1 @shl_ap1_non_zero_ap2_non_zero_ap1_3(i32 %a) {
 
 define i1 @icmp_sgt_zero_add_nsw(i32 %a) {
 ; CHECK-LABEL: @icmp_sgt_zero_add_nsw(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 %a, -1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[A:%.*]], -1
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %add = add nsw i32 %a, 1
@@ -2586,7 +2639,7 @@ define i1 @icmp_sgt_zero_add_nsw(i32 %a) {
 
 define i1 @icmp_sge_zero_add_nsw(i32 %a) {
 ; CHECK-LABEL: @icmp_sge_zero_add_nsw(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 %a, -2
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[A:%.*]], -2
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %add = add nsw i32 %a, 1
@@ -2596,7 +2649,7 @@ define i1 @icmp_sge_zero_add_nsw(i32 %a) {
 
 define i1 @icmp_sle_zero_add_nsw(i32 %a) {
 ; CHECK-LABEL: @icmp_sle_zero_add_nsw(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 %a, 0
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[A:%.*]], 0
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %add = add nsw i32 %a, 1
@@ -2606,8 +2659,9 @@ define i1 @icmp_sle_zero_add_nsw(i32 %a) {
 
 define zeroext i1 @icmp_cmpxchg_strong(i32* %sc, i32 %old_val, i32 %new_val) {
 ; CHECK-LABEL: @icmp_cmpxchg_strong(
-; CHECK-NEXT:    [[XCHG:%.*]] = cmpxchg i32* %sc, i32 %old_val, i32 %new_val seq_cst seq_cst
-; CHECK-NEXT:    [[ICMP:%.*]] = extractvalue { i32, i1
+; CHECK-NEXT:    [[XCHG:%.*]] = cmpxchg i32* [[SC:%.*]], i32 [[OLD_VAL:%.*]], i32 [[NEW_VAL:%.*]] seq_cst seq_cst
+; CHECK-NEXT:    [[ICMP:%.*]] = extractvalue { i32, i1 } [[XCHG]], 1
+; CHECK-NEXT:    ret i1 [[ICMP]]
 ;
   %xchg = cmpxchg i32* %sc, i32 %old_val, i32 %new_val seq_cst seq_cst
   %xtrc = extractvalue { i32, i1 } %xchg, 0
@@ -2617,7 +2671,7 @@ define zeroext i1 @icmp_cmpxchg_strong(i32* %sc, i32 %old_val, i32 %new_val) {
 
 define i1 @f1(i64 %a, i64 %b) {
 ; CHECK-LABEL: @f1(
-; CHECK-NEXT:    [[V:%.*]] = icmp sge i64 %a, %b
+; CHECK-NEXT:    [[V:%.*]] = icmp sge i64 [[A:%.*]], [[B:%.*]]
 ; CHECK-NEXT:    ret i1 [[V]]
 ;
   %t = sub nsw i64 %a, %b
@@ -2627,7 +2681,7 @@ define i1 @f1(i64 %a, i64 %b) {
 
 define <2 x i1> @f1_vec(<2 x i64> %a, <2 x i64> %b) {
 ; CHECK-LABEL: @f1_vec(
-; CHECK-NEXT:    [[V:%.*]] = icmp sge <2 x i64> %a, %b
+; CHECK-NEXT:    [[V:%.*]] = icmp sge <2 x i64> [[A:%.*]], [[B:%.*]]
 ; CHECK-NEXT:    ret <2 x i1> [[V]]
 ;
   %t = sub nsw <2 x i64> %a, %b
@@ -2637,7 +2691,7 @@ define <2 x i1> @f1_vec(<2 x i64> %a, <2 x i64> %b) {
 
 define i1 @f2(i64 %a, i64 %b) {
 ; CHECK-LABEL: @f2(
-; CHECK-NEXT:    [[V:%.*]] = icmp sgt i64 %a, %b
+; CHECK-NEXT:    [[V:%.*]] = icmp sgt i64 [[A:%.*]], [[B:%.*]]
 ; CHECK-NEXT:    ret i1 [[V]]
 ;
   %t = sub nsw i64 %a, %b
@@ -2647,7 +2701,7 @@ define i1 @f2(i64 %a, i64 %b) {
 
 define <2 x i1> @f2_vec(<2 x i64> %a, <2 x i64> %b) {
 ; CHECK-LABEL: @f2_vec(
-; CHECK-NEXT:    [[V:%.*]] = icmp sgt <2 x i64> %a, %b
+; CHECK-NEXT:    [[V:%.*]] = icmp sgt <2 x i64> [[A:%.*]], [[B:%.*]]
 ; CHECK-NEXT:    ret <2 x i1> [[V]]
 ;
   %t = sub nsw <2 x i64> %a, %b
@@ -2657,7 +2711,7 @@ define <2 x i1> @f2_vec(<2 x i64> %a, <2 x i64> %b) {
 
 define i1 @f3(i64 %a, i64 %b) {
 ; CHECK-LABEL: @f3(
-; CHECK-NEXT:    [[V:%.*]] = icmp slt i64 %a, %b
+; CHECK-NEXT:    [[V:%.*]] = icmp slt i64 [[A:%.*]], [[B:%.*]]
 ; CHECK-NEXT:    ret i1 [[V]]
 ;
   %t = sub nsw i64 %a, %b
@@ -2667,7 +2721,7 @@ define i1 @f3(i64 %a, i64 %b) {
 
 define <2 x i1> @f3_vec(<2 x i64> %a, <2 x i64> %b) {
 ; CHECK-LABEL: @f3_vec(
-; CHECK-NEXT:    [[V:%.*]] = icmp slt <2 x i64> %a, %b
+; CHECK-NEXT:    [[V:%.*]] = icmp slt <2 x i64> [[A:%.*]], [[B:%.*]]
 ; CHECK-NEXT:    ret <2 x i1> [[V]]
 ;
   %t = sub nsw <2 x i64> %a, %b
@@ -2677,7 +2731,7 @@ define <2 x i1> @f3_vec(<2 x i64> %a, <2 x i64> %b) {
 
 define i1 @f4(i64 %a, i64 %b) {
 ; CHECK-LABEL: @f4(
-; CHECK-NEXT:    [[V:%.*]] = icmp sle i64 %a, %b
+; CHECK-NEXT:    [[V:%.*]] = icmp sle i64 [[A:%.*]], [[B:%.*]]
 ; CHECK-NEXT:    ret i1 [[V]]
 ;
   %t = sub nsw i64 %a, %b
@@ -2687,7 +2741,7 @@ define i1 @f4(i64 %a, i64 %b) {
 
 define <2 x i1> @f4_vec(<2 x i64> %a, <2 x i64> %b) {
 ; CHECK-LABEL: @f4_vec(
-; CHECK-NEXT:    [[V:%.*]] = icmp sle <2 x i64> %a, %b
+; CHECK-NEXT:    [[V:%.*]] = icmp sle <2 x i64> [[A:%.*]], [[B:%.*]]
 ; CHECK-NEXT:    ret <2 x i1> [[V]]
 ;
   %t = sub nsw <2 x i64> %a, %b
@@ -2697,8 +2751,8 @@ define <2 x i1> @f4_vec(<2 x i64> %a, <2 x i64> %b) {
 
 define i32 @f5(i8 %a, i8 %b) {
 ; CHECK-LABEL: @f5(
-; CHECK-NEXT:    [[CONV:%.*]] = zext i8 %a to i32
-; CHECK-NEXT:    [[CONV3:%.*]] = zext i8 %b to i32
+; CHECK-NEXT:    [[CONV:%.*]] = zext i8 [[A:%.*]] to i32
+; CHECK-NEXT:    [[CONV3:%.*]] = zext i8 [[B:%.*]] to i32
 ; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i32 [[CONV]], [[CONV3]]
 ; CHECK-NEXT:    [[CMP4:%.*]] = icmp slt i32 [[SUB]], 0
 ; CHECK-NEXT:    [[SUB7:%.*]] = sub nsw i32 0, [[SUB]]
@@ -2716,9 +2770,9 @@ define i32 @f5(i8 %a, i8 %b) {
 
 define i32 @f6(i32 %a, i32 %b) {
 ; CHECK-LABEL: @f6(
-; CHECK-NEXT:    [[CMP_UNSHIFTED:%.*]] = xor i32 %a, %b
+; CHECK-NEXT:    [[CMP_UNSHIFTED:%.*]] = xor i32 [[A:%.*]], [[B:%.*]]
 ; CHECK-NEXT:    [[CMP_MASK:%.*]] = and i32 [[CMP_UNSHIFTED]], 255
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[CMP:%.*]].mask, 0
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[CMP_MASK]], 0
 ; CHECK-NEXT:    [[S:%.*]] = select i1 [[CMP]], i32 10000, i32 0
 ; CHECK-NEXT:    ret i32 [[S]]
 ;
@@ -2733,7 +2787,7 @@ define i32 @f6(i32 %a, i32 %b) {
 
 define i32 @f7(i32 %a, i32 %b) {
 ; CHECK-LABEL: @f7(
-; CHECK-NEXT:    [[CMP_UNSHIFTED:%.*]] = xor i32 %a, %b
+; CHECK-NEXT:    [[CMP_UNSHIFTED:%.*]] = xor i32 [[A:%.*]], [[B:%.*]]
 ; CHECK-NEXT:    [[CMP_MASK:%.*]] = and i32 [[CMP_UNSHIFTED]], 511
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[CMP_MASK]], 0
 ; CHECK-NEXT:    [[S:%.*]] = select i1 [[CMP]], i32 0, i32 10000
@@ -2748,7 +2802,7 @@ define i32 @f7(i32 %a, i32 %b) {
 
 define i1 @f8(i32 %val, i32 %lim) {
 ; CHECK-LABEL: @f8(
-; CHECK-NEXT:    [[R:%.*]] = icmp ne i32 %lim, 0
+; CHECK-NEXT:    [[R:%.*]] = icmp ne i32 [[LIM:%.*]], 0
 ; CHECK-NEXT:    ret i1 [[R]]
 ;
   %lim.sub = add i32 %lim, -1
@@ -2759,7 +2813,7 @@ define i1 @f8(i32 %val, i32 %lim) {
 
 define i1 @f9(i32 %val, i32 %lim) {
 ; CHECK-LABEL: @f9(
-; CHECK-NEXT:    [[R:%.*]] = icmp ne i32 %lim, 0
+; CHECK-NEXT:    [[R:%.*]] = icmp ne i32 [[LIM:%.*]], 0
 ; CHECK-NEXT:    ret i1 [[R]]
 ;
   %lim.sub = sub i32 %lim, 1
@@ -2770,7 +2824,7 @@ define i1 @f9(i32 %val, i32 %lim) {
 
 define i1 @f10(i16 %p) {
 ; CHECK-LABEL: @f10(
-; CHECK-NEXT:    [[CMP580:%.*]] = icmp uge i16 %p, mul (i16 zext (i8 ptrtoint (i1 (i16)* @f10 to i8) to i16), i16 zext (i8 ptrtoint (i1 (i16)* @f10 to i8) to i16))
+; CHECK-NEXT:    [[CMP580:%.*]] = icmp uge i16 [[P:%.*]], mul (i16 zext (i8 ptrtoint (i1 (i16)* @f10 to i8) to i16), i16 zext (i8 ptrtoint (i1 (i16)* @f10 to i8) to i16))
 ; CHECK-NEXT:    ret i1 [[CMP580]]
 ;
   %cmp580 = icmp ule i16 mul (i16 zext (i8 ptrtoint (i1 (i16)* @f10 to i8) to i16), i16 zext (i8 ptrtoint (i1 (i16)* @f10 to i8) to i16)), %p
@@ -2783,8 +2837,8 @@ define i1 @f10(i16 %p) {
 
 define i1 @cmp_sgt_rhs_dec(float %x, i32 %i) {
 ; CHECK-LABEL: @cmp_sgt_rhs_dec(
-; CHECK-NEXT:    [[CONV:%.*]] = fptosi float %x to i32
-; CHECK-NEXT:    [[CMP:%.*]] = icmp sge i32 [[CONV]], %i
+; CHECK-NEXT:    [[CONV:%.*]] = fptosi float [[X:%.*]] to i32
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sge i32 [[CONV]], [[I:%.*]]
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %conv = fptosi float %x to i32
@@ -2795,8 +2849,8 @@ define i1 @cmp_sgt_rhs_dec(float %x, i32 %i) {
 
 define i1 @cmp_sle_rhs_dec(float %x, i32 %i) {
 ; CHECK-LABEL: @cmp_sle_rhs_dec(
-; CHECK-NEXT:    [[CONV:%.*]] = fptosi float %x to i32
-; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[CONV]], %i
+; CHECK-NEXT:    [[CONV:%.*]] = fptosi float [[X:%.*]] to i32
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[CONV]], [[I:%.*]]
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %conv = fptosi float %x to i32
@@ -2807,8 +2861,8 @@ define i1 @cmp_sle_rhs_dec(float %x, i32 %i) {
 
 define i1 @cmp_sge_rhs_inc(float %x, i32 %i) {
 ; CHECK-LABEL: @cmp_sge_rhs_inc(
-; CHECK-NEXT:    [[CONV:%.*]] = fptosi float %x to i32
-; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[CONV]], %i
+; CHECK-NEXT:    [[CONV:%.*]] = fptosi float [[X:%.*]] to i32
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[CONV]], [[I:%.*]]
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %conv = fptosi float %x to i32
@@ -2819,8 +2873,8 @@ define i1 @cmp_sge_rhs_inc(float %x, i32 %i) {
 
 define i1 @cmp_slt_rhs_inc(float %x, i32 %i) {
 ; CHECK-LABEL: @cmp_slt_rhs_inc(
-; CHECK-NEXT:    [[CONV:%.*]] = fptosi float %x to i32
-; CHECK-NEXT:    [[CMP:%.*]] = icmp sle i32 [[CONV]], %i
+; CHECK-NEXT:    [[CONV:%.*]] = fptosi float [[X:%.*]] to i32
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sle i32 [[CONV]], [[I:%.*]]
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %conv = fptosi float %x to i32
@@ -2831,8 +2885,8 @@ define i1 @cmp_slt_rhs_inc(float %x, i32 %i) {
 
 define i1 @PR26407(i32 %x, i32 %y) {
 ; CHECK-LABEL: @PR26407(
-; CHECK-NEXT:    [[ADDX:%.*]] = add i32 %x, 2147483647
-; CHECK-NEXT:    [[ADDY:%.*]] = add i32 %y, 2147483647
+; CHECK-NEXT:    [[ADDX:%.*]] = add i32 [[X:%.*]], 2147483647
+; CHECK-NEXT:    [[ADDY:%.*]] = add i32 [[Y:%.*]], 2147483647
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp uge i32 [[ADDX]], [[ADDY]]
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
@@ -2844,7 +2898,7 @@ define i1 @PR26407(i32 %x, i32 %y) {
 
 define i1 @cmp_inverse_mask_bits_set_eq(i32 %x) {
 ; CHECK-LABEL: @cmp_inverse_mask_bits_set_eq(
-; CHECK-NEXT:    [[TMP1:%.*]] = and i32 %x, -43
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X:%.*]], -43
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[TMP1]], -43
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
@@ -2855,7 +2909,7 @@ define i1 @cmp_inverse_mask_bits_set_eq(i32 %x) {
 
 define <2 x i1> @cmp_inverse_mask_bits_set_eq_vec(<2 x i32> %x) {
 ; CHECK-LABEL: @cmp_inverse_mask_bits_set_eq_vec(
-; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i32> %x, <i32 -43, i32 -43>
+; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i32> [[X:%.*]], <i32 -43, i32 -43>
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <2 x i32> [[TMP1]], <i32 -43, i32 -43>
 ; CHECK-NEXT:    ret <2 x i1> [[CMP]]
 ;
@@ -2866,7 +2920,7 @@ define <2 x i1> @cmp_inverse_mask_bits_set_eq_vec(<2 x i32> %x) {
 
 define i1 @cmp_inverse_mask_bits_set_ne(i32 %x) {
 ; CHECK-LABEL: @cmp_inverse_mask_bits_set_ne(
-; CHECK-NEXT:    [[TMP1:%.*]] = and i32 %x, -43
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X:%.*]], -43
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[TMP1]], -43
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
@@ -2879,7 +2933,7 @@ define i1 @cmp_inverse_mask_bits_set_ne(i32 %x) {
 
 define i1 @PR27792(i128 %a) {
 ; CHECK-LABEL: @PR27792(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i128 %a, -1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i128 [[A:%.*]], -1
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %cmp = icmp sge i128 %a, 0
@@ -2888,7 +2942,7 @@ define i1 @PR27792(i128 %a) {
 
 define i1 @PR27792_2(i128 %a) {
 ; CHECK-LABEL: @PR27792_2(
-; CHECK-NEXT:    [[B:%.*]] = icmp ne i128 %a, 0
+; CHECK-NEXT:    [[B:%.*]] = icmp ne i128 [[A:%.*]], 0
 ; CHECK-NEXT:    ret i1 [[B]]
 ;
   %b = icmp uge i128 %a, 1
@@ -2897,7 +2951,7 @@ define i1 @PR27792_2(i128 %a) {
 
 define i1 @ugtMaxSignedVal(i8 %a) {
 ; CHECK-LABEL: @ugtMaxSignedVal(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i8 %a, 0
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i8 [[A:%.*]], 0
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %cmp = icmp ugt i8 %a, 127
@@ -2906,7 +2960,7 @@ define i1 @ugtMaxSignedVal(i8 %a) {
 
 define <2 x i1> @ugtMaxSignedValVec(<2 x i8> %a) {
 ; CHECK-LABEL: @ugtMaxSignedValVec(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp slt <2 x i8> %a, zeroinitializer
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt <2 x i8> [[A:%.*]], zeroinitializer
 ; CHECK-NEXT:    ret <2 x i1> [[CMP]]
 ;
   %cmp = icmp ugt <2 x i8> %a, <i8 127, i8 127>
@@ -2915,7 +2969,7 @@ define <2 x i1> @ugtMaxSignedValVec(<2 x i8> %a) {
 
 define i1 @ugtKnownBits(i8 %a) {
 ; CHECK-LABEL: @ugtKnownBits(
-; CHECK-NEXT:    [[B:%.*]] = and i8 %a, 17
+; CHECK-NEXT:    [[B:%.*]] = and i8 [[A:%.*]], 17
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 [[B]], 17
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
@@ -2926,7 +2980,7 @@ define i1 @ugtKnownBits(i8 %a) {
 
 define <2 x i1> @ugtKnownBitsVec(<2 x i8> %a) {
 ; CHECK-LABEL: @ugtKnownBitsVec(
-; CHECK-NEXT:    [[B:%.*]] = and <2 x i8> %a, <i8 17, i8 17>
+; CHECK-NEXT:    [[B:%.*]] = and <2 x i8> [[A:%.*]], <i8 17, i8 17>
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <2 x i8> [[B]], <i8 17, i8 17>
 ; CHECK-NEXT:    ret <2 x i1> [[CMP]]
 ;
@@ -2936,11 +2990,12 @@ define <2 x i1> @ugtKnownBitsVec(<2 x i8> %a) {
 }
 
 define i1 @or_ptrtoint_mismatch(i8* %p, i32* %q) {
-; CHECK-LABEL: define i1 @or_ptrtoint_mismatch(i8* %p, i32* %q)
-; CHECK: [[pc:%.*]] = icmp eq i8* %p, null
-; CHECK: [[qc:%.*]] = icmp eq i32* %q, null
-; CHECK: [[b:%.*]] = and i1 [[pc]], [[qc]]
-; CHECK: ret i1 [[b]]
+; CHECK-LABEL: @or_ptrtoint_mismatch(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i8* [[P:%.*]], null
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i32* [[Q:%.*]], null
+; CHECK-NEXT:    [[B:%.*]] = and i1 [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    ret i1 [[B]]
+;
 
   %pp = ptrtoint i8* %p to i64
   %qq = ptrtoint i32* %q to i64
@@ -2951,7 +3006,7 @@ define i1 @or_ptrtoint_mismatch(i8* %p, i32* %q) {
 
 define i1 @icmp_add1_ugt(i32 %x, i32 %y) {
 ; CHECK-LABEL: @icmp_add1_ugt(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp uge i32 %x, %y
+; CHECK-NEXT:    [[CMP:%.*]] = icmp uge i32 [[X:%.*]], [[Y:%.*]]
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %add = add nuw i32 %x, 1
@@ -2961,7 +3016,7 @@ define i1 @icmp_add1_ugt(i32 %x, i32 %y) {
 
 define i1 @icmp_add1_ule(i32 %x, i32 %y) {
 ; CHECK-LABEL: @icmp_add1_ule(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 %x, %y
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[X:%.*]], [[Y:%.*]]
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %add = add nuw i32 %x, 1
@@ -2971,8 +3026,8 @@ define i1 @icmp_add1_ule(i32 %x, i32 %y) {
 
 define i1 @cmp_uge_rhs_inc(float %x, i32 %i) {
 ; CHECK-LABEL: @cmp_uge_rhs_inc(
-; CHECK-NEXT:    [[CONV:%.*]] = fptosi float %x to i32
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i32 [[CONV]], %i
+; CHECK-NEXT:    [[CONV:%.*]] = fptosi float [[X:%.*]] to i32
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i32 [[CONV]], [[I:%.*]]
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %conv = fptosi float %x to i32
@@ -2983,8 +3038,8 @@ define i1 @cmp_uge_rhs_inc(float %x, i32 %i) {
 
 define i1 @cmp_ult_rhs_inc(float %x, i32 %i) {
 ; CHECK-LABEL: @cmp_ult_rhs_inc(
-; CHECK-NEXT:    [[CONV:%.*]] = fptosi float %x to i32
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ule i32 [[CONV]], %i
+; CHECK-NEXT:    [[CONV:%.*]] = fptosi float [[X:%.*]] to i32
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ule i32 [[CONV]], [[I:%.*]]
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %conv = fptosi float %x to i32
@@ -2995,8 +3050,8 @@ define i1 @cmp_ult_rhs_inc(float %x, i32 %i) {
 
 define i1 @cmp_sge_lhs_inc(i32 %x, i32 %y) {
 ; CHECK-LABEL: @cmp_sge_lhs_inc(
-; CHECK-NEXT:    [[INC:%.*]] = add
-; CHECK-NEXT:    [[CMP:%.*]] = icmp sge i32 [[INC]], %y
+; CHECK-NEXT:    [[INC:%.*]] = add nsw i32 [[X:%.*]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sge i32 [[INC]], [[Y:%.*]]
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %inc = add nsw i32 %x, 1
@@ -3006,8 +3061,8 @@ define i1 @cmp_sge_lhs_inc(i32 %x, i32 %y) {
 
 define i1 @cmp_uge_lhs_inc(i32 %x, i32 %y) {
 ; CHECK-LABEL: @cmp_uge_lhs_inc(
-; CHECK-NEXT:    [[INC:%.*]] = add
-; CHECK-NEXT:    [[CMP:%.*]] = icmp uge i32 [[INC]], %y
+; CHECK-NEXT:    [[INC:%.*]] = add nuw i32 [[X:%.*]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp uge i32 [[INC]], [[Y:%.*]]
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %inc = add nuw i32 %x, 1
@@ -3017,8 +3072,8 @@ define i1 @cmp_uge_lhs_inc(i32 %x, i32 %y) {
 
 define i1 @cmp_sgt_lhs_dec(i32 %x, i32 %y) {
 ; CHECK-LABEL: @cmp_sgt_lhs_dec(
-; CHECK-NEXT:    [[DEC:%.*]] = {{add|sub}}
-; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[DEC]], %y
+; CHECK-NEXT:    [[DEC:%.*]] = add nsw i32 [[X:%.*]], -1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[DEC]], [[Y:%.*]]
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %dec = sub nsw i32 %x, 1
@@ -3028,8 +3083,8 @@ define i1 @cmp_sgt_lhs_dec(i32 %x, i32 %y) {
 
 define i1 @cmp_ugt_lhs_dec(i32 %x, i32 %y) {
 ; CHECK-LABEL: @cmp_ugt_lhs_dec(
-; CHECK-NEXT:    [[DEC:%.*]] = {{add|sub}}
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i32 [[DEC]], %y
+; CHECK-NEXT:    [[DEC:%.*]] = add i32 [[X:%.*]], -1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i32 [[DEC]], [[Y:%.*]]
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %dec = sub nuw i32 %x, 1
@@ -3039,8 +3094,8 @@ define i1 @cmp_ugt_lhs_dec(i32 %x, i32 %y) {
 
 define i1 @cmp_sle_rhs_inc(float %x, i32 %y) {
 ; CHECK-LABEL: @cmp_sle_rhs_inc(
-; CHECK-NEXT:    [[CONV:%.*]] = fptosi float %x to i32
-; CHECK-NEXT:    [[INC:%.*]] = add
+; CHECK-NEXT:    [[CONV:%.*]] = fptosi float [[X:%.*]] to i32
+; CHECK-NEXT:    [[INC:%.*]] = add nsw i32 [[Y:%.*]], 1
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp sge i32 [[INC]], [[CONV]]
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
@@ -3052,8 +3107,8 @@ define i1 @cmp_sle_rhs_inc(float %x, i32 %y) {
 
 define i1 @cmp_ule_rhs_inc(float %x, i32 %y) {
 ; CHECK-LABEL: @cmp_ule_rhs_inc(
-; CHECK-NEXT:    [[CONV:%.*]] = fptosi float %x to i32
-; CHECK-NEXT:    [[INC:%.*]] = add
+; CHECK-NEXT:    [[CONV:%.*]] = fptosi float [[X:%.*]] to i32
+; CHECK-NEXT:    [[INC:%.*]] = add nuw i32 [[Y:%.*]], 1
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp uge i32 [[INC]], [[CONV]]
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
@@ -3065,8 +3120,8 @@ define i1 @cmp_ule_rhs_inc(float %x, i32 %y) {
 
 define i1 @cmp_slt_rhs_dec(float %x, i32 %y) {
 ; CHECK-LABEL: @cmp_slt_rhs_dec(
-; CHECK-NEXT:    [[CONV:%.*]] = fptosi float %x to i32
-; CHECK-NEXT:    [[DEC:%.*]] = {{add|sub}}
+; CHECK-NEXT:    [[CONV:%.*]] = fptosi float [[X:%.*]] to i32
+; CHECK-NEXT:    [[DEC:%.*]] = add nsw i32 [[Y:%.*]], -1
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[DEC]], [[CONV]]
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
@@ -3078,8 +3133,8 @@ define i1 @cmp_slt_rhs_dec(float %x, i32 %y) {
 
 define i1 @cmp_ult_rhs_dec(float %x, i32 %y) {
 ; CHECK-LABEL: @cmp_ult_rhs_dec(
-; CHECK-NEXT:    [[CONV:%.*]] = fptosi float %x to i32
-; CHECK-NEXT:    [[DEC:%.*]] = {{add|sub}}
+; CHECK-NEXT:    [[CONV:%.*]] = fptosi float [[X:%.*]] to i32
+; CHECK-NEXT:    [[DEC:%.*]] = add i32 [[Y:%.*]], -1
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i32 [[DEC]], [[CONV]]
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
@@ -3091,7 +3146,7 @@ define i1 @cmp_ult_rhs_dec(float %x, i32 %y) {
 
 define i1 @eq_add_constants(i32 %x, i32 %y) {
 ; CHECK-LABEL: @eq_add_constants(
-; CHECK-NEXT:    [[C:%.*]] = icmp eq i32 %x, %y
+; CHECK-NEXT:    [[C:%.*]] = icmp eq i32 [[X:%.*]], [[Y:%.*]]
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %A = add i32 %x, 5
@@ -3102,7 +3157,7 @@ define i1 @eq_add_constants(i32 %x, i32 %y) {
 
 define i1 @eq_mul_constants(i32 %x, i32 %y) {
 ; CHECK-LABEL: @eq_mul_constants(
-; CHECK-NEXT:    [[C:%.*]] = icmp eq i32 %x, %y
+; CHECK-NEXT:    [[C:%.*]] = icmp eq i32 [[X:%.*]], [[Y:%.*]]
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %A = mul i32 %x, 5
@@ -3113,7 +3168,7 @@ define i1 @eq_mul_constants(i32 %x, i32 %y) {
 
 define <2 x i1> @eq_mul_constants_splat(<2 x i32> %x, <2 x i32> %y) {
 ; CHECK-LABEL: @eq_mul_constants_splat(
-; CHECK-NEXT:    [[C:%.*]] = icmp ne <2 x i32> %x, %y
+; CHECK-NEXT:    [[C:%.*]] = icmp ne <2 x i32> [[X:%.*]], [[Y:%.*]]
 ; CHECK-NEXT:    ret <2 x i1> [[C]]
 ;
   %A = mul <2 x i32> %x, <i32 5, i32 5>
@@ -3128,7 +3183,7 @@ define <2 x i1> @eq_mul_constants_splat(<2 x i32> %x, <2 x i32> %y) {
 
 define i1 @eq_mul_constants_with_tz(i32 %x, i32 %y) {
 ; CHECK-LABEL: @eq_mul_constants_with_tz(
-; CHECK-NEXT:    [[TMP1:%.*]] = xor i32 %x, %y
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i32 [[X:%.*]], [[Y:%.*]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = and i32 [[TMP1]], 1073741823
 ; CHECK-NEXT:    [[C:%.*]] = icmp ne i32 [[TMP2]], 0
 ; CHECK-NEXT:    ret i1 [[C]]
@@ -3141,7 +3196,7 @@ define i1 @eq_mul_constants_with_tz(i32 %x, i32 %y) {
 
 define <2 x i1> @eq_mul_constants_with_tz_splat(<2 x i32> %x, <2 x i32> %y) {
 ; CHECK-LABEL: @eq_mul_constants_with_tz_splat(
-; CHECK-NEXT:    [[TMP1:%.*]] = xor <2 x i32> %x, %y
+; CHECK-NEXT:    [[TMP1:%.*]] = xor <2 x i32> [[X:%.*]], [[Y:%.*]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = and <2 x i32> [[TMP1]], <i32 1073741823, i32 1073741823>
 ; CHECK-NEXT:    [[C:%.*]] = icmp eq <2 x i32> [[TMP2]], zeroinitializer
 ; CHECK-NEXT:    ret <2 x i1> [[C]]
@@ -3156,7 +3211,7 @@ declare i32 @llvm.bswap.i32(i32)
 
 define i1 @bswap_ne(i32 %x, i32 %y) {
 ; CHECK-LABEL: @bswap_ne(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 %x, %y
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[X:%.*]], [[Y:%.*]]
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %swapx = call i32 @llvm.bswap.i32(i32 %x)
@@ -3169,7 +3224,7 @@ declare <8 x i16> @llvm.bswap.v8i16(<8 x i16>)
 
 define <8 x i1> @bswap_vec_eq(<8 x i16> %x, <8 x i16> %y) {
 ; CHECK-LABEL: @bswap_vec_eq(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <8 x i16> %x, %y
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <8 x i16> [[X:%.*]], [[Y:%.*]]
 ; CHECK-NEXT:    ret <8 x i1> [[CMP]]
 ;
   %swapx = call <8 x i16> @llvm.bswap.v8i16(<8 x i16> %x)
@@ -3182,7 +3237,7 @@ declare i64 @llvm.bitreverse.i64(i64)
 
 define i1 @bitreverse_eq(i64 %x, i64 %y) {
 ; CHECK-LABEL: @bitreverse_eq(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i64 %x, %y
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i64 [[X:%.*]], [[Y:%.*]]
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
   %revx = call i64 @llvm.bitreverse.i64(i64 %x)
@@ -3195,7 +3250,7 @@ declare <8 x i16> @llvm.bitreverse.v8i16(<8 x i16>)
 
 define <8 x i1> @bitreverse_vec_ne(<8 x i16> %x, <8 x i16> %y) {
 ; CHECK-LABEL: @bitreverse_vec_ne(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ne <8 x i16> %x, %y
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne <8 x i16> [[X:%.*]], [[Y:%.*]]
 ; CHECK-NEXT:    ret <8 x i1> [[CMP]]
 ;
   %revx = call <8 x i16> @llvm.bitreverse.v8i16(<8 x i16> %x)
@@ -3365,7 +3420,7 @@ define i32 @abs_preserve(i32 %x) {
 declare void @llvm.assume(i1)
 define i1 @PR35794(i32* %a) {
 ; CHECK-LABEL: @PR35794(
-; CHECK-NEXT:    [[MASKCOND:%.*]] = icmp eq i32* %a, null
+; CHECK-NEXT:    [[MASKCOND:%.*]] = icmp eq i32* [[A:%.*]], null
 ; CHECK-NEXT:    tail call void @llvm.assume(i1 [[MASKCOND]])
 ; CHECK-NEXT:    ret i1 true
 ;
@@ -3378,7 +3433,7 @@ define i1 @PR35794(i32* %a) {
 ; Don't crash by assuming the compared values are integers.
 define <2 x i1> @PR36583(<2 x i8*>)  {
 ; CHECK-LABEL: @PR36583(
-; CHECK-NEXT:    [[RES:%.*]] = icmp eq <2 x i8*> %0, zeroinitializer
+; CHECK-NEXT:    [[RES:%.*]] = icmp eq <2 x i8*> [[TMP0:%.*]], zeroinitializer
 ; CHECK-NEXT:    ret <2 x i1> [[RES]]
 ;
   %cast = ptrtoint <2 x i8*> %0 to <2 x i64>
diff --git a/test/Transforms/InstCombine/intersect-accessgroup.ll b/test/Transforms/InstCombine/intersect-accessgroup.ll
new file mode 100644
index 0000000000000000000000000000000000000000..858b9b6e59bd4aa2b18e6f93e8a3ad149fb078ea
--- /dev/null
+++ b/test/Transforms/InstCombine/intersect-accessgroup.ll
@@ -0,0 +1,113 @@
+; RUN: opt -instcombine -S < %s | FileCheck %s
+;
+; void func(long n, double A[static const restrict n]) {
+; 	for (int i = 0; i <  n; i+=1)
+; 		for (int j = 0; j <  n;j+=1)
+; 			for (int k = 0; k < n; k += 1)
+; 				for (int l = 0; l < n; l += 1) {
+; 					double *p = &A[i + j + k + l];
+; 					double x = *p;
+; 					double y = *p;
+; 					arg(x + y);
+; 				}
+; }
+;
+; Check for correctly merging access group metadata for instcombine
+; (only common loops are parallel == intersection)
+; Note that combined load would be parallel to loop !16 since both
+; origin loads are parallel to it, but it references two access groups
+; (!8 and !9), neither of which contain both loads. As such, the
+; information that the combined load is parallel to !16 is lost.
+;
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+
+declare void @arg(double)
+
+define void @func(i64 %n, double* noalias nonnull %A) {
+entry:
+  br label %for.cond
+
+for.cond:
+  %i.0 = phi i32 [ 0, %entry ], [ %add31, %for.inc30 ]
+  %conv = sext i32 %i.0 to i64
+  %cmp = icmp slt i64 %conv, %n
+  br i1 %cmp, label %for.cond2, label %for.end32
+
+for.cond2:
+  %j.0 = phi i32 [ %add28, %for.inc27 ], [ 0, %for.cond ]
+  %conv3 = sext i32 %j.0 to i64
+  %cmp4 = icmp slt i64 %conv3, %n
+  br i1 %cmp4, label %for.cond8, label %for.inc30
+
+for.cond8:
+  %k.0 = phi i32 [ %add25, %for.inc24 ], [ 0, %for.cond2 ]
+  %conv9 = sext i32 %k.0 to i64
+  %cmp10 = icmp slt i64 %conv9, %n
+  br i1 %cmp10, label %for.cond14, label %for.inc27
+
+for.cond14:
+  %l.0 = phi i32 [ %add23, %for.body19 ], [ 0, %for.cond8 ]
+  %conv15 = sext i32 %l.0 to i64
+  %cmp16 = icmp slt i64 %conv15, %n
+  br i1 %cmp16, label %for.body19, label %for.inc24
+
+for.body19:
+  %add = add nsw i32 %i.0, %j.0
+  %add20 = add nsw i32 %add, %k.0
+  %add21 = add nsw i32 %add20, %l.0
+  %idxprom = sext i32 %add21 to i64
+  %arrayidx = getelementptr inbounds double, double* %A, i64 %idxprom
+  %0 = load double, double* %arrayidx, align 8, !llvm.access.group !1
+  %1 = load double, double* %arrayidx, align 8, !llvm.access.group !2
+  %add22 = fadd double %0, %1
+  call void @arg(double %add22), !llvm.access.group !3
+  %add23 = add nsw i32 %l.0, 1
+  br label %for.cond14, !llvm.loop !11
+
+for.inc24:
+  %add25 = add nsw i32 %k.0, 1
+  br label %for.cond8, !llvm.loop !14
+
+for.inc27:
+  %add28 = add nsw i32 %j.0, 1
+  br label %for.cond2, !llvm.loop !16
+
+for.inc30:
+  %add31 = add nsw i32 %i.0, 1
+  br label %for.cond, !llvm.loop !18
+
+for.end32:
+  ret void
+}
+
+
+; access groups
+!7 = distinct !{}
+!8 = distinct !{}
+!9 = distinct !{}
+
+; access group lists
+!1 = !{!7, !9}
+!2 = !{!7, !8}
+!3 = !{!7, !8, !9}
+
+!11 = distinct !{!11, !13}
+!13 = !{!"llvm.loop.parallel_accesses", !7}
+
+!14 = distinct !{!14, !15}
+!15 = !{!"llvm.loop.parallel_accesses", !8}
+
+!16 = distinct !{!16, !17}
+!17 = !{!"llvm.loop.parallel_accesses", !8, !9}
+
+!18 = distinct !{!18, !19}
+!19 = !{!"llvm.loop.parallel_accesses", !9}
+
+
+; CHECK: load double, {{.*}} !llvm.access.group ![[ACCESSGROUP_0:[0-9]+]]
+; CHECK: br label %for.cond14, !llvm.loop ![[LOOP_4:[0-9]+]]
+
+; CHECK: ![[ACCESSGROUP_0]] = distinct !{}
+
+; CHECK: ![[LOOP_4]] = distinct !{![[LOOP_4]], ![[PARALLEL_ACCESSES_5:[0-9]+]]}
+; CHECK: ![[PARALLEL_ACCESSES_5]] = !{!"llvm.loop.parallel_accesses", ![[ACCESSGROUP_0]]}
diff --git a/test/Transforms/InstCombine/invoke.ll b/test/Transforms/InstCombine/invoke.ll
index fb54c4f77c49b69d764af888014762b40b9a2cb1..deb4a2b87a4f8dc7d5fb7a3b7847616c5099d0cf 100644
--- a/test/Transforms/InstCombine/invoke.ll
+++ b/test/Transforms/InstCombine/invoke.ll
@@ -55,7 +55,7 @@ entry:
           to label %invoke.cont unwind label %lpad
 
 invoke.cont:
-; CHECK: call i64 @llvm.objectsize.i64.p0i8(i8* %call, i1 false, i1 false)
+; CHECK: call i64 @llvm.objectsize.i64.p0i8(i8* %call, i1 false, i1 false, i1 false)
   %0 = tail call i64 @llvm.objectsize.i64(i8* %call, i1 false)
   ret i64 %0
 
diff --git a/test/Transforms/InstCombine/loadstore-metadata.ll b/test/Transforms/InstCombine/loadstore-metadata.ll
index 4856350133efda646022fcecb6159b97ef639696..5916a8d3a4c337d226dd679469ebbd6b937d0284 100644
--- a/test/Transforms/InstCombine/loadstore-metadata.ll
+++ b/test/Transforms/InstCombine/loadstore-metadata.ll
@@ -39,7 +39,7 @@ entry:
 define i32 @test_load_cast_combine_invariant(float* %ptr) {
 ; Ensure (cast (load (...))) -> (load (cast (...))) preserves invariant metadata.
 ; CHECK-LABEL: @test_load_cast_combine_invariant(
-; CHECK: load i32, i32* %{{.*}}, !invariant.load !5
+; CHECK: load i32, i32* %{{.*}}, !invariant.load !7
 entry:
   %l = load float, float* %ptr, !invariant.load !6
   %c = bitcast float %l to i32
@@ -50,7 +50,7 @@ define i32 @test_load_cast_combine_nontemporal(float* %ptr) {
 ; Ensure (cast (load (...))) -> (load (cast (...))) preserves nontemporal
 ; metadata.
 ; CHECK-LABEL: @test_load_cast_combine_nontemporal(
-; CHECK: load i32, i32* %{{.*}}, !nontemporal !6
+; CHECK: load i32, i32* %{{.*}}, !nontemporal !8
 entry:
   %l = load float, float* %ptr, !nontemporal !7
   %c = bitcast float %l to i32
@@ -61,7 +61,7 @@ define i8* @test_load_cast_combine_align(i32** %ptr) {
 ; Ensure (cast (load (...))) -> (load (cast (...))) preserves align
 ; metadata.
 ; CHECK-LABEL: @test_load_cast_combine_align(
-; CHECK: load i8*, i8** %{{.*}}, !align !7
+; CHECK: load i8*, i8** %{{.*}}, !align !9
 entry:
   %l = load i32*, i32** %ptr, !align !8
   %c = bitcast i32* %l to i8*
@@ -72,7 +72,7 @@ define i8* @test_load_cast_combine_deref(i32** %ptr) {
 ; Ensure (cast (load (...))) -> (load (cast (...))) preserves dereferenceable
 ; metadata.
 ; CHECK-LABEL: @test_load_cast_combine_deref(
-; CHECK: load i8*, i8** %{{.*}}, !dereferenceable !7
+; CHECK: load i8*, i8** %{{.*}}, !dereferenceable !9
 entry:
   %l = load i32*, i32** %ptr, !dereferenceable !8
   %c = bitcast i32* %l to i8*
@@ -83,7 +83,7 @@ define i8* @test_load_cast_combine_deref_or_null(i32** %ptr) {
 ; Ensure (cast (load (...))) -> (load (cast (...))) preserves
 ; dereferenceable_or_null metadata.
 ; CHECK-LABEL: @test_load_cast_combine_deref_or_null(
-; CHECK: load i8*, i8** %{{.*}}, !dereferenceable_or_null !7
+; CHECK: load i8*, i8** %{{.*}}, !dereferenceable_or_null !9
 entry:
   %l = load i32*, i32** %ptr, !dereferenceable_or_null !8
   %c = bitcast i32* %l to i8*
@@ -94,7 +94,7 @@ define void @test_load_cast_combine_loop(float* %src, i32* %dst, i32 %n) {
 ; Ensure (cast (load (...))) -> (load (cast (...))) preserves loop access
 ; metadata.
 ; CHECK-LABEL: @test_load_cast_combine_loop(
-; CHECK: load i32, i32* %{{.*}}, !llvm.mem.parallel_loop_access !4
+; CHECK: load i32, i32* %{{.*}}, !llvm.access.group !6
 entry:
   br label %loop
 
@@ -102,7 +102,7 @@ loop:
   %i = phi i32 [ 0, %entry ], [ %i.next, %loop ]
   %src.gep = getelementptr inbounds float, float* %src, i32 %i
   %dst.gep = getelementptr inbounds i32, i32* %dst, i32 %i
-  %l = load float, float* %src.gep, !llvm.mem.parallel_loop_access !4
+  %l = load float, float* %src.gep, !llvm.access.group !9
   %c = bitcast float %l to i32
   store i32 %c, i32* %dst.gep
   %i.next = add i32 %i, 1
@@ -142,8 +142,9 @@ entry:
 !1 = !{!"scalar type", !2}
 !2 = !{!"root"}
 !3 = distinct !{!3, !4}
-!4 = distinct !{!4}
+!4 = distinct !{!4, !{!"llvm.loop.parallel_accesses", !9}}
 !5 = !{i32 0, i32 42}
 !6 = !{}
 !7 = !{i32 1}
 !8 = !{i64 8}
+!9 = distinct !{}
diff --git a/test/Transforms/InstCombine/masked_intrinsics.ll b/test/Transforms/InstCombine/masked_intrinsics.ll
index d5403d17ddc24c8ee8540eb146c5d5aac7753de7..cd9b30bddad80f8fea2882b89bf5562be35688c0 100644
--- a/test/Transforms/InstCombine/masked_intrinsics.ll
+++ b/test/Transforms/InstCombine/masked_intrinsics.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -instcombine -S < %s | FileCheck %s
 
 declare <2 x double> @llvm.masked.load.v2f64.p0v2f64(<2 x double>* %ptrs, i32, <2 x i1> %mask, <2 x double> %src0)
@@ -6,61 +7,110 @@ declare <2 x double> @llvm.masked.gather.v2f64.v2p0f64(<2 x double*> %ptrs, i32,
 declare void @llvm.masked.scatter.v2f64.v2p0f64(<2 x double> %val, <2 x double*> %ptrs, i32, <2 x i1> %mask)
 
 define <2 x double> @load_zeromask(<2 x double>* %ptr, <2 x double> %passthru)  {
+; CHECK-LABEL: @load_zeromask(
+; CHECK-NEXT:    ret <2 x double> [[PASSTHRU:%.*]]
+;
   %res = call <2 x double> @llvm.masked.load.v2f64.p0v2f64(<2 x double>* %ptr, i32 1, <2 x i1> zeroinitializer, <2 x double> %passthru)
   ret <2 x double> %res
 
-; CHECK-LABEL: @load_zeromask(
-; CHECK-NEXT:  ret <2 x double> %passthru
 }
 
 define <2 x double> @load_onemask(<2 x double>* %ptr, <2 x double> %passthru)  {
+; CHECK-LABEL: @load_onemask(
+; CHECK-NEXT:    [[UNMASKEDLOAD:%.*]] = load <2 x double>, <2 x double>* [[PTR:%.*]], align 2
+; CHECK-NEXT:    ret <2 x double> [[UNMASKEDLOAD]]
+;
   %res = call <2 x double> @llvm.masked.load.v2f64.p0v2f64(<2 x double>* %ptr, i32 2, <2 x i1> <i1 1, i1 1>, <2 x double> %passthru)
   ret <2 x double> %res
 
-; CHECK-LABEL: @load_onemask(
-; CHECK-NEXT:  %unmaskedload = load <2 x double>, <2 x double>* %ptr, align 2
-; CHECK-NEXT:  ret <2 x double> %unmaskedload
 }
 
 define <2 x double> @load_undefmask(<2 x double>* %ptr, <2 x double> %passthru)  {
+; CHECK-LABEL: @load_undefmask(
+; CHECK-NEXT:    [[UNMASKEDLOAD:%.*]] = load <2 x double>, <2 x double>* [[PTR:%.*]], align 2
+; CHECK-NEXT:    ret <2 x double> [[UNMASKEDLOAD]]
+;
   %res = call <2 x double> @llvm.masked.load.v2f64.p0v2f64(<2 x double>* %ptr, i32 2, <2 x i1> <i1 1, i1 undef>, <2 x double> %passthru)
   ret <2 x double> %res
 
-; CHECK-LABEL: @load_undefmask(
-; CHECK-NEXT:  %unmaskedload = load <2 x double>, <2 x double>* %ptr, align 2
-; CHECK-NEXT:  ret <2 x double> %unmaskedload
 }
 
+define <2 x double> @load_lane0(<2 x double>* %ptr, double %pt)  {
+; CHECK-LABEL: @load_lane0(
+; CHECK-NEXT:    [[PTV1:%.*]] = insertelement <2 x double> undef, double [[PT:%.*]], i64 0
+; CHECK-NEXT:    [[PTV2:%.*]] = shufflevector <2 x double> [[PTV1]], <2 x double> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[RES:%.*]] = call <2 x double> @llvm.masked.load.v2f64.p0v2f64(<2 x double>* [[PTR:%.*]], i32 2, <2 x i1> <i1 true, i1 false>, <2 x double> [[PTV2]])
+; CHECK-NEXT:    ret <2 x double> [[RES]]
+;
+  %ptv1 = insertelement <2 x double> undef, double %pt, i64 0
+  %ptv2 = insertelement <2 x double> %ptv1, double %pt, i64 1
+  %res = call <2 x double> @llvm.masked.load.v2f64.p0v2f64(<2 x double>* %ptr, i32 2, <2 x i1> <i1 true, i1 false>, <2 x double> %ptv2)
+  ret <2 x double> %res
+
+}
+
+
 define void @store_zeromask(<2 x double>* %ptr, <2 x double> %val)  {
+; CHECK-LABEL: @store_zeromask(
+; CHECK-NEXT:    ret void
+;
   call void @llvm.masked.store.v2f64.p0v2f64(<2 x double> %val, <2 x double>* %ptr, i32 3, <2 x i1> zeroinitializer)
   ret void
 
-; CHECK-LABEL: @store_zeromask(
-; CHECK-NEXT:  ret void
 }
 
 define void @store_onemask(<2 x double>* %ptr, <2 x double> %val)  {
+; CHECK-LABEL: @store_onemask(
+; CHECK-NEXT:    store <2 x double> [[VAL:%.*]], <2 x double>* [[PTR:%.*]], align 4
+; CHECK-NEXT:    ret void
+;
   call void @llvm.masked.store.v2f64.p0v2f64(<2 x double> %val, <2 x double>* %ptr, i32 4, <2 x i1> <i1 1, i1 1>)
   ret void
 
-; CHECK-LABEL: @store_onemask(
-; CHECK-NEXT:  store <2 x double> %val, <2 x double>* %ptr, align 4
-; CHECK-NEXT:  ret void
 }
 
 define <2 x double> @gather_zeromask(<2 x double*> %ptrs, <2 x double> %passthru)  {
+; CHECK-LABEL: @gather_zeromask(
+; CHECK-NEXT:    ret <2 x double> [[PASSTHRU:%.*]]
+;
   %res = call <2 x double> @llvm.masked.gather.v2f64.v2p0f64(<2 x double*> %ptrs, i32 5, <2 x i1> zeroinitializer, <2 x double> %passthru)
   ret <2 x double> %res
 
-; CHECK-LABEL: @gather_zeromask(
-; CHECK-NEXT:  ret <2 x double> %passthru
+}
+
+
+define <2 x double> @gather_onemask(<2 x double*> %ptrs, <2 x double> %passthru)  {
+; CHECK-LABEL: @gather_onemask(
+; CHECK-NEXT:    [[RES:%.*]] = call <2 x double> @llvm.masked.gather.v2f64.v2p0f64(<2 x double*> [[PTRS:%.*]], i32 5, <2 x i1> <i1 true, i1 true>, <2 x double> [[PASSTHRU:%.*]])
+; CHECK-NEXT:    ret <2 x double> [[RES]]
+;
+  %res = call <2 x double> @llvm.masked.gather.v2f64.v2p0f64(<2 x double*> %ptrs, i32 5, <2 x i1> <i1 true, i1 true>, <2 x double> %passthru)
+  ret <2 x double> %res
+
+}
+
+define <2 x double> @gather_lane0(double* %base, double %pt)  {
+; CHECK-LABEL: @gather_lane0(
+; CHECK-NEXT:    [[PTRS:%.*]] = getelementptr double, double* [[BASE:%.*]], <2 x i64> <i64 0, i64 1>
+; CHECK-NEXT:    [[PT_V1:%.*]] = insertelement <2 x double> undef, double [[PT:%.*]], i64 0
+; CHECK-NEXT:    [[PT_V2:%.*]] = shufflevector <2 x double> [[PT_V1]], <2 x double> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[RES:%.*]] = call <2 x double> @llvm.masked.gather.v2f64.v2p0f64(<2 x double*> [[PTRS]], i32 5, <2 x i1> <i1 true, i1 false>, <2 x double> [[PT_V2]])
+; CHECK-NEXT:    ret <2 x double> [[RES]]
+;
+  %ptrs = getelementptr double, double *%base, <2 x i64> <i64 0, i64 1>
+  %pt_v1 = insertelement <2 x double> undef, double %pt, i64 0
+  %pt_v2 = insertelement <2 x double> %pt_v1, double %pt, i64 1
+  %res = call <2 x double> @llvm.masked.gather.v2f64.v2p0f64(<2 x double*> %ptrs, i32 5, <2 x i1> <i1 true, i1 false>, <2 x double> %pt_v2)
+  ret <2 x double> %res
+
 }
 
 define void @scatter_zeromask(<2 x double*> %ptrs, <2 x double> %val)  {
+; CHECK-LABEL: @scatter_zeromask(
+; CHECK-NEXT:    ret void
+;
   call void @llvm.masked.scatter.v2f64.v2p0f64(<2 x double> %val, <2 x double*> %ptrs, i32 6, <2 x i1> zeroinitializer)
   ret void
 
-; CHECK-LABEL: @scatter_zeromask(
-; CHECK-NEXT:  ret void
 }
 
diff --git a/test/Transforms/InstCombine/mem-par-metadata-memcpy.ll b/test/Transforms/InstCombine/mem-par-metadata-memcpy.ll
index 82231bae9810c268207befb6c87160a57b6e76f3..54fe6cb8495c022c7e789417ec4a186ad5e6899d 100644
--- a/test/Transforms/InstCombine/mem-par-metadata-memcpy.ll
+++ b/test/Transforms/InstCombine/mem-par-metadata-memcpy.ll
@@ -1,6 +1,6 @@
 ; RUN: opt < %s -instcombine -S | FileCheck %s
 ;
-; Make sure the llvm.mem.parallel_loop_access meta-data is preserved
+; Make sure the llvm.access.group meta-data is preserved
 ; when a memcpy is replaced with a load+store by instcombine
 ;
 ; #include <string.h>
@@ -13,8 +13,8 @@
 ; }
 
 ; CHECK: for.body:
-; CHECK:  %{{.*}} = load i16, i16* %{{.*}}, align 1, !llvm.mem.parallel_loop_access !1
-; CHECK:  store i16 %{{.*}}, i16* %{{.*}}, align 1, !llvm.mem.parallel_loop_access !1
+; CHECK:  %{{.*}} = load i16, i16* %{{.*}}, align 1, !llvm.access.group !1
+; CHECK:  store i16 %{{.*}}, i16* %{{.*}}, align 1, !llvm.access.group !1
 
 
 ; ModuleID = '<stdin>'
@@ -36,7 +36,7 @@ for.body:                                         ; preds = %for.cond
   %arrayidx = getelementptr inbounds i8, i8* %out, i64 %i.0
   %add = add nsw i64 %i.0, %size
   %arrayidx1 = getelementptr inbounds i8, i8* %out, i64 %add
-  call void @llvm.memcpy.p0i8.p0i8.i64(i8* %arrayidx, i8* %arrayidx1, i64 2, i1 false), !llvm.mem.parallel_loop_access !1
+  call void @llvm.memcpy.p0i8.p0i8.i64(i8* %arrayidx, i8* %arrayidx1, i64 2, i1 false), !llvm.access.group !4
   br label %for.inc
 
 for.inc:                                          ; preds = %for.body
@@ -56,6 +56,7 @@ attributes #1 = { argmemonly nounwind }
 !llvm.ident = !{!0}
 
 !0 = !{!"clang version 4.0.0 (cfe/trunk 277751)"}
-!1 = distinct !{!1, !2, !3}
+!1 = distinct !{!1, !2, !3, !{!"llvm.loop.parallel_accesses", !4}}
 !2 = distinct !{!2, !3}
 !3 = !{!"llvm.loop.vectorize.enable", i1 true}
+!4 = distinct !{} ; access group
diff --git a/test/Transforms/InstCombine/memchr.ll b/test/Transforms/InstCombine/memchr.ll
index 5a081c222fb021fe0ace94e0199ef051472768e6..83073e20676e82bfe96402670172f7538a6752ff 100644
--- a/test/Transforms/InstCombine/memchr.ll
+++ b/test/Transforms/InstCombine/memchr.ll
@@ -50,7 +50,7 @@ define void @test3() {
 
 define void @test4(i32 %chr) {
 ; CHECK-LABEL: @test4(
-; CHECK-NEXT:    [[DST:%.*]] = call i8* @memchr(i8* getelementptr inbounds ([14 x i8], [14 x i8]* @hello, i32 0, i32 0), i32 %chr, i32 14)
+; CHECK-NEXT:    [[DST:%.*]] = call i8* @memchr(i8* getelementptr inbounds ([14 x i8], [14 x i8]* @hello, i32 0, i32 0), i32 [[CHR:%.*]], i32 14)
 ; CHECK-NEXT:    store i8* [[DST]], i8** @chp, align 4
 ; CHECK-NEXT:    ret void
 ;
@@ -131,12 +131,13 @@ define void @test10() {
 ; Check transformation memchr("\r\n", C, 2) != nullptr -> (C & 9216) != 0
 define i1 @test11(i32 %C) {
 ; CHECK-LABEL: @test11(
-; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 %C to i16
-; CHECK-NEXT:    [[MEMCHR_BOUNDS:%.*]] = icmp ult i16 [[TMP1]], 16
-; CHECK-NEXT:    [[TMP2:%.*]] = shl i16 1, [[TMP1]]
-; CHECK-NEXT:    [[TMP3:%.*]] = and i16 [[TMP2]], 9216
-; CHECK-NEXT:    [[MEMCHR_BITS:%.*]] = icmp ne i16 [[TMP3]], 0
-; CHECK-NEXT:    [[MEMCHR:%.*]] = and i1 [[MEMCHR:%.*]].bounds, [[MEMCHR:%.*]].bits
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 [[C:%.*]] to i16
+; CHECK-NEXT:    [[TMP2:%.*]] = and i16 [[TMP1]], 255
+; CHECK-NEXT:    [[MEMCHR_BOUNDS:%.*]] = icmp ult i16 [[TMP2]], 16
+; CHECK-NEXT:    [[TMP3:%.*]] = shl i16 1, [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = and i16 [[TMP3]], 9216
+; CHECK-NEXT:    [[MEMCHR_BITS:%.*]] = icmp ne i16 [[TMP4]], 0
+; CHECK-NEXT:    [[MEMCHR:%.*]] = and i1 [[MEMCHR_BOUNDS]], [[MEMCHR_BITS]]
 ; CHECK-NEXT:    ret i1 [[MEMCHR]]
 ;
   %dst = call i8* @memchr(i8* getelementptr inbounds ([3 x i8], [3 x i8]* @newlines, i64 0, i64 0), i32 %C, i32 2)
@@ -147,7 +148,7 @@ define i1 @test11(i32 %C) {
 ; No 64 bits here
 define i1 @test12(i32 %C) {
 ; CHECK-LABEL: @test12(
-; CHECK-NEXT:    [[DST:%.*]] = call i8* @memchr(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @spaces, i32 0, i32 0), i32 %C, i32 3)
+; CHECK-NEXT:    [[DST:%.*]] = call i8* @memchr(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @spaces, i32 0, i32 0), i32 [[C:%.*]], i32 3)
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i8* [[DST]], null
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
@@ -158,11 +159,12 @@ define i1 @test12(i32 %C) {
 
 define i1 @test13(i32 %C) {
 ; CHECK-LABEL: @test13(
-; CHECK-NEXT:    [[MEMCHR_BOUNDS:%.*]] = icmp ult i32 %C, 32
-; CHECK-NEXT:    [[TMP1:%.*]] = shl i32 1, %C
-; CHECK-NEXT:    [[TMP2:%.*]] = and i32 [[TMP1]], -2147483647
-; CHECK-NEXT:    [[MEMCHR_BITS:%.*]] = icmp ne i32 [[TMP2]], 0
-; CHECK-NEXT:    [[MEMCHR:%.*]] = and i1 [[MEMCHR:%.*]].bounds, [[MEMCHR:%.*]].bits
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[C:%.*]], 255
+; CHECK-NEXT:    [[MEMCHR_BOUNDS:%.*]] = icmp ult i32 [[TMP1]], 32
+; CHECK-NEXT:    [[TMP2:%.*]] = shl i32 1, [[TMP1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = and i32 [[TMP2]], -2147483647
+; CHECK-NEXT:    [[MEMCHR_BITS:%.*]] = icmp ne i32 [[TMP3]], 0
+; CHECK-NEXT:    [[MEMCHR:%.*]] = and i1 [[MEMCHR_BOUNDS]], [[MEMCHR_BITS]]
 ; CHECK-NEXT:    ret i1 [[MEMCHR]]
 ;
   %dst = call i8* @memchr(i8* getelementptr inbounds ([2 x i8], [2 x i8]* @single, i64 0, i64 0), i32 %C, i32 2)
@@ -172,8 +174,9 @@ define i1 @test13(i32 %C) {
 
 define i1 @test14(i32 %C) {
 ; CHECK-LABEL: @test14(
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i32 %C, 31
-; CHECK-NEXT:    ret i1 [[TMP1]]
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[C:%.*]], 255
+; CHECK-NEXT:    [[MEMCHR_BITS:%.*]] = icmp eq i32 [[TMP1]], 31
+; CHECK-NEXT:    ret i1 [[MEMCHR_BITS]]
 ;
   %dst = call i8* @memchr(i8* getelementptr inbounds ([2 x i8], [2 x i8]* @single, i64 0, i64 0), i32 %C, i32 1)
   %cmp = icmp ne i8* %dst, null
@@ -182,7 +185,7 @@ define i1 @test14(i32 %C) {
 
 define i1 @test15(i32 %C) {
 ; CHECK-LABEL: @test15(
-; CHECK-NEXT:    [[DST:%.*]] = call i8* @memchr(i8* getelementptr inbounds ([3 x i8], [3 x i8]* @negative, i32 0, i32 0), i32 %C, i32 3)
+; CHECK-NEXT:    [[DST:%.*]] = call i8* @memchr(i8* getelementptr inbounds ([3 x i8], [3 x i8]* @negative, i32 0, i32 0), i32 [[C:%.*]], i32 3)
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i8* [[DST]], null
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
diff --git a/test/Transforms/InstCombine/memset_chk-1.ll b/test/Transforms/InstCombine/memset_chk-1.ll
index 7f680ea48b6cea3422e1744b314a9e3f53348ac0..71f95b02457849a0025b56a3551feaf2057707e3 100644
--- a/test/Transforms/InstCombine/memset_chk-1.ll
+++ b/test/Transforms/InstCombine/memset_chk-1.ll
@@ -69,7 +69,7 @@ define i32 @test_rauw(i8* %a, i8* %b, i8** %c) {
 entry:
   %call49 = call i64 @strlen(i8* %a)
   %add180 = add i64 %call49, 1
-  %yo107 = call i64 @llvm.objectsize.i64.p0i8(i8* %b, i1 false, i1 false)
+  %yo107 = call i64 @llvm.objectsize.i64.p0i8(i8* %b, i1 false, i1 false, i1 false)
   %call50 = call i8* @__memmove_chk(i8* %b, i8* %a, i64 %add180, i64 %yo107)
 ; CHECK: %strlen = call i64 @strlen(i8* %b)
 ; CHECK-NEXT: %strchr2 = getelementptr i8, i8* %b, i64 %strlen
@@ -87,7 +87,7 @@ entry:
 declare i8* @__memmove_chk(i8*, i8*, i64, i64)
 declare i8* @strrchr(i8*, i32)
 declare i64 @strlen(i8* nocapture)
-declare i64 @llvm.objectsize.i64.p0i8(i8*, i1, i1)
+declare i64 @llvm.objectsize.i64.p0i8(i8*, i1, i1, i1)
 
 declare i8* @__memset_chk(i8*, i32, i64, i64)
 
@@ -100,7 +100,7 @@ entry:
   br i1 %cmp, label %cleanup, label %if.end
 if.end:
   %bc = bitcast i8* %call to float*
-  %call2 = tail call i64 @llvm.objectsize.i64.p0i8(i8* nonnull %call, i1 false, i1 false)
+  %call2 = tail call i64 @llvm.objectsize.i64.p0i8(i8* nonnull %call, i1 false, i1 false, i1 false)
   %call3 = tail call i8* @__memset_chk(i8* nonnull %call, i32 0, i64 %size, i64 %call2) #1
   br label %cleanup
 cleanup:
@@ -114,7 +114,7 @@ cleanup:
 ; CHECK-NEXT:    br i1 %cmp, label %cleanup, label %if.end
 ; CHECK:       if.end:
 ; CHECK-NEXT:    %bc = bitcast i8* %call to float*
-; CHECK-NEXT:    %call2 = tail call i64 @llvm.objectsize.i64.p0i8(i8* nonnull %call, i1 false, i1 false)
+; CHECK-NEXT:    %call2 = tail call i64 @llvm.objectsize.i64.p0i8(i8* nonnull %call, i1 false, i1 false, i1 false)
 ; CHECK-NEXT:    %call3 = tail call i8* @__memset_chk(i8* nonnull %call, i32 0, i64 %size, i64 %call2)
 ; CHECK-NEXT:    br label %cleanup
 ; CHECK:       cleanup:
diff --git a/test/Transforms/InstCombine/mul.ll b/test/Transforms/InstCombine/mul.ll
index fc3773aa9f15dba508a51a1d22023d8410c606af..b1dc9d51332d735f0ec5ebda65f181b2377a6931 100644
--- a/test/Transforms/InstCombine/mul.ll
+++ b/test/Transforms/InstCombine/mul.ll
@@ -442,3 +442,78 @@ define i128 @test34(i128 %X) {
   %mul = mul nsw i128 %X, 2
   ret i128 %mul
 }
+
+define i32 @test_mul_canonicalize_op0(i32 %x, i32 %y) {
+; CHECK-LABEL: @test_mul_canonicalize_op0(
+; CHECK-NEXT:    [[TMP1:%.*]] = mul i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[MUL:%.*]] = sub i32 0, [[TMP1]]
+; CHECK-NEXT:    ret i32 [[MUL]]
+;
+  %neg = sub i32 0, %x
+  %mul = mul i32 %neg, %y
+  ret i32 %mul
+}
+
+define i32 @test_mul_canonicalize_op1(i32 %x, i32 %z) {
+; CHECK-LABEL: @test_mul_canonicalize_op1(
+; CHECK-NEXT:    [[Y:%.*]] = mul i32 [[Z:%.*]], 3
+; CHECK-NEXT:    [[TMP1:%.*]] = mul i32 [[Y]], [[X:%.*]]
+; CHECK-NEXT:    [[MUL:%.*]] = sub i32 0, [[TMP1]]
+; CHECK-NEXT:    ret i32 [[MUL]]
+;
+  %y = mul i32 %z, 3
+  %neg = sub i32 0, %x
+  %mul = mul i32 %y, %neg
+  ret i32 %mul
+}
+
+define i32 @test_mul_canonicalize_nsw(i32 %x, i32 %y) {
+; CHECK-LABEL: @test_mul_canonicalize_nsw(
+; CHECK-NEXT:    [[TMP1:%.*]] = mul i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[MUL:%.*]] = sub i32 0, [[TMP1]]
+; CHECK-NEXT:    ret i32 [[MUL]]
+;
+  %neg = sub nsw i32 0, %x
+  %mul = mul nsw i32 %neg, %y
+  ret i32 %mul
+}
+
+define <2 x i32> @test_mul_canonicalize_vec(<2 x i32> %x, <2 x i32> %y) {
+; CHECK-LABEL: @test_mul_canonicalize_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = mul <2 x i32> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[MUL:%.*]] = sub <2 x i32> zeroinitializer, [[TMP1]]
+; CHECK-NEXT:    ret <2 x i32> [[MUL]]
+;
+  %neg = sub <2 x i32> <i32 0, i32 0>, %x
+  %mul = mul <2 x i32> %neg, %y
+  ret <2 x i32> %mul
+}
+
+define i32 @test_mul_canonicalize_multiple_uses(i32 %x, i32 %y) {
+; CHECK-LABEL: @test_mul_canonicalize_multiple_uses(
+; CHECK-NEXT:    [[NEG:%.*]] = sub i32 0, [[X:%.*]]
+; CHECK-NEXT:    [[MUL:%.*]] = mul i32 [[NEG]], [[Y:%.*]]
+; CHECK-NEXT:    [[MUL2:%.*]] = mul i32 [[MUL]], [[NEG]]
+; CHECK-NEXT:    ret i32 [[MUL2]]
+;
+  %neg = sub i32 0, %x
+  %mul = mul i32 %neg, %y
+  %mul2 = mul i32 %mul, %neg
+  ret i32 %mul2
+}
+
+@X = global i32 5
+
+define i64 @test_mul_canonicalize_neg_is_not_undone(i64 %L1) {
+; Check we do not undo the canonicalization of 0 - (X * Y), if Y is a constant
+; expr.
+; CHECK-LABEL: @test_mul_canonicalize_neg_is_not_undone(
+; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[L1:%.*]], ptrtoint (i32* @X to i64)
+; CHECK-NEXT:    [[B4:%.*]] = sub i64 0, [[TMP1]]
+; CHECK-NEXT:    ret i64 [[B4]]
+;
+  %v1 = ptrtoint i32* @X to i64
+  %B8 = sub i64 0, %v1
+  %B4 = mul i64 %B8, %L1
+  ret i64 %B4
+}
diff --git a/test/Transforms/InstCombine/objsize.ll b/test/Transforms/InstCombine/objsize.ll
index 12262c03670cbf17e0ed2a86a0e0457f3f927929..97c708fb6bfdabae2543da5d2066ae898317541f 100644
--- a/test/Transforms/InstCombine/objsize.ll
+++ b/test/Transforms/InstCombine/objsize.ll
@@ -8,7 +8,7 @@ target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f3
 define i32 @foo() nounwind {
 ; CHECK-LABEL: @foo(
 ; CHECK-NEXT: ret i32 60
-  %1 = call i32 @llvm.objectsize.i32.p0i8(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0), i1 false, i1 false)
+  %1 = call i32 @llvm.objectsize.i32.p0i8(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0), i1 false, i1 false, i1 false)
   ret i32 %1
 }
 
@@ -16,7 +16,7 @@ define i8* @bar() nounwind {
 ; CHECK-LABEL: @bar(
 entry:
   %retval = alloca i8*
-  %0 = call i32 @llvm.objectsize.i32.p0i8(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0), i1 false, i1 false)
+  %0 = call i32 @llvm.objectsize.i32.p0i8(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0), i1 false, i1 false, i1 false)
   %cmp = icmp ne i32 %0, -1
 ; CHECK: br i1 true
   br i1 %cmp, label %cond.true, label %cond.false
@@ -33,7 +33,7 @@ cond.false:
 define i32 @f() nounwind {
 ; CHECK-LABEL: @f(
 ; CHECK-NEXT: ret i32 0
-  %1 = call i32 @llvm.objectsize.i32.p0i8(i8* getelementptr ([60 x i8], [60 x i8]* @a, i32 1, i32 0), i1 false, i1 false)
+  %1 = call i32 @llvm.objectsize.i32.p0i8(i8* getelementptr ([60 x i8], [60 x i8]* @a, i32 1, i32 0), i1 false, i1 false, i1 false)
   ret i32 %1
 }
 
@@ -42,7 +42,7 @@ define i32 @f() nounwind {
 define i1 @baz() nounwind {
 ; CHECK-LABEL: @baz(
 ; CHECK-NEXT: objectsize
-  %1 = tail call i32 @llvm.objectsize.i32.p0i8(i8* getelementptr inbounds ([0 x i8], [0 x i8]* @window, i32 0, i32 0), i1 false, i1 false)
+  %1 = tail call i32 @llvm.objectsize.i32.p0i8(i8* getelementptr inbounds ([0 x i8], [0 x i8]* @window, i32 0, i32 0), i1 false, i1 false, i1 false)
   %2 = icmp eq i32 %1, -1
   ret i1 %2
 }
@@ -51,7 +51,7 @@ define void @test1(i8* %q, i32 %x) nounwind noinline {
 ; CHECK-LABEL: @test1(
 ; CHECK: objectsize.i32.p0i8
 entry:
-  %0 = call i32 @llvm.objectsize.i32.p0i8(i8* getelementptr inbounds ([0 x i8], [0 x i8]* @window, i32 0, i32 10), i1 false, i1 false) ; <i64> [#uses=1]
+  %0 = call i32 @llvm.objectsize.i32.p0i8(i8* getelementptr inbounds ([0 x i8], [0 x i8]* @window, i32 0, i32 10), i1 false, i1 false, i1 false) ; <i64> [#uses=1]
   %1 = icmp eq i32 %0, -1                         ; <i1> [#uses=1]
   br i1 %1, label %"47", label %"46"
 
@@ -67,7 +67,7 @@ entry:
 define i32 @test2() nounwind {
 ; CHECK-LABEL: @test2(
 ; CHECK-NEXT: ret i32 34
-  %1 = call i32 @llvm.objectsize.i32.p0i8(i8* getelementptr (i8, i8* bitcast ([9 x i32]* @.str5 to i8*), i32 2), i1 false, i1 false)
+  %1 = call i32 @llvm.objectsize.i32.p0i8(i8* getelementptr (i8, i8* bitcast ([9 x i32]* @.str5 to i8*), i32 2), i1 false, i1 false, i1 false)
   ret i32 %1
 }
 
@@ -76,9 +76,9 @@ define i32 @test2() nounwind {
 
 declare i8* @__memcpy_chk(i8*, i8*, i32, i32) nounwind
 
-declare i32 @llvm.objectsize.i32.p0i8(i8*, i1, i1) nounwind readonly
+declare i32 @llvm.objectsize.i32.p0i8(i8*, i1, i1, i1) nounwind readonly
 
-declare i32 @llvm.objectsize.i32.p1i8(i8 addrspace(1)*, i1, i1) nounwind readonly
+declare i32 @llvm.objectsize.i32.p1i8(i8 addrspace(1)*, i1, i1, i1) nounwind readonly
 
 declare i8* @__inline_memcpy_chk(i8*, i8*, i32) nounwind inlinehint
 
@@ -90,7 +90,7 @@ entry:
 bb11:
   %0 = getelementptr inbounds float, float* getelementptr inbounds ([480 x float], [480 x float]* @array, i32 0, i32 128), i32 -127 ; <float*> [#uses=1]
   %1 = bitcast float* %0 to i8*                   ; <i8*> [#uses=1]
-  %2 = call i32 @llvm.objectsize.i32.p0i8(i8* %1, i1 false, i1 false) ; <i32> [#uses=1]
+  %2 = call i32 @llvm.objectsize.i32.p0i8(i8* %1, i1 false, i1 false, i1 false) ; <i32> [#uses=1]
   %3 = call i8* @__memcpy_chk(i8* undef, i8* undef, i32 512, i32 %2) nounwind ; <i8*> [#uses=0]
 ; CHECK: unreachable
   unreachable
@@ -112,9 +112,9 @@ define i32 @test4(i8** %esc) nounwind ssp {
 entry:
   %0 = alloca %struct.data, align 8
   %1 = bitcast %struct.data* %0 to i8*
-  %2 = call i32 @llvm.objectsize.i32.p0i8(i8* %1, i1 false, i1 false) nounwind
+  %2 = call i32 @llvm.objectsize.i32.p0i8(i8* %1, i1 false, i1 false, i1 false) nounwind
 ; CHECK-NOT: @llvm.objectsize
-; CHECK: @llvm.memset.p0i8.i32(i8* align 8 %1, i8 0, i32 1824, i1 false)
+; CHECK: @llvm.memset.p0i8.i32(i8* nonnull align 8 %1, i8 0, i32 1824, i1 false)
   %3 = call i8* @__memset_chk(i8* %1, i32 0, i32 1824, i32 %2) nounwind
   store i8* %1, i8** %esc
   ret i32 0
@@ -127,7 +127,7 @@ define i8* @test5(i32 %n) nounwind ssp {
 ; CHECK-LABEL: @test5(
 entry:
   %0 = tail call noalias i8* @malloc(i32 20) nounwind
-  %1 = tail call i32 @llvm.objectsize.i32.p0i8(i8* %0, i1 false, i1 false)
+  %1 = tail call i32 @llvm.objectsize.i32.p0i8(i8* %0, i1 false, i1 false, i1 false)
   %2 = load i8*, i8** @s, align 8
 ; CHECK-NOT: @llvm.objectsize
 ; CHECK: @llvm.memcpy.p0i8.p0i8.i32(i8* align 1 %0, i8* align 1 %1, i32 10, i1 false)
@@ -139,7 +139,7 @@ define void @test6(i32 %n) nounwind ssp {
 ; CHECK-LABEL: @test6(
 entry:
   %0 = tail call noalias i8* @malloc(i32 20) nounwind
-  %1 = tail call i32 @llvm.objectsize.i32.p0i8(i8* %0, i1 false, i1 false)
+  %1 = tail call i32 @llvm.objectsize.i32.p0i8(i8* %0, i1 false, i1 false, i1 false)
   %2 = load i8*, i8** @s, align 8
 ; CHECK-NOT: @llvm.objectsize
 ; CHECK: @__memcpy_chk(i8* %0, i8* %1, i32 30, i32 20)
@@ -156,7 +156,7 @@ define i32 @test7(i8** %esc) {
   %alloc = call noalias i8* @malloc(i32 48) nounwind
   store i8* %alloc, i8** %esc
   %gep = getelementptr inbounds i8, i8* %alloc, i32 16
-  %objsize = call i32 @llvm.objectsize.i32.p0i8(i8* %gep, i1 false, i1 false) nounwind readonly
+  %objsize = call i32 @llvm.objectsize.i32.p0i8(i8* %gep, i1 false, i1 false, i1 false) nounwind readonly
 ; CHECK: ret i32 32
   ret i32 %objsize
 }
@@ -168,7 +168,7 @@ define i32 @test8(i8** %esc) {
   %alloc = call noalias i8* @calloc(i32 5, i32 7) nounwind
   store i8* %alloc, i8** %esc
   %gep = getelementptr inbounds i8, i8* %alloc, i32 5
-  %objsize = call i32 @llvm.objectsize.i32.p0i8(i8* %gep, i1 false, i1 false) nounwind readonly
+  %objsize = call i32 @llvm.objectsize.i32.p0i8(i8* %gep, i1 false, i1 false, i1 false) nounwind readonly
 ; CHECK: ret i32 30
   ret i32 %objsize
 }
@@ -180,7 +180,7 @@ declare noalias i8* @strndup(i8* nocapture, i32) nounwind
 define i32 @test9(i8** %esc) {
   %call = tail call i8* @strdup(i8* getelementptr inbounds ([8 x i8], [8 x i8]* @.str, i64 0, i64 0)) nounwind
   store i8* %call, i8** %esc, align 8
-  %1 = tail call i32 @llvm.objectsize.i32.p0i8(i8* %call, i1 true, i1 false)
+  %1 = tail call i32 @llvm.objectsize.i32.p0i8(i8* %call, i1 true, i1 false, i1 false)
 ; CHECK: ret i32 8
   ret i32 %1
 }
@@ -189,7 +189,7 @@ define i32 @test9(i8** %esc) {
 define i32 @test10(i8** %esc) {
   %call = tail call i8* @strndup(i8* getelementptr inbounds ([8 x i8], [8 x i8]* @.str, i64 0, i64 0), i32 3) nounwind
   store i8* %call, i8** %esc, align 8
-  %1 = tail call i32 @llvm.objectsize.i32.p0i8(i8* %call, i1 true, i1 false)
+  %1 = tail call i32 @llvm.objectsize.i32.p0i8(i8* %call, i1 true, i1 false, i1 false)
 ; CHECK: ret i32 4
   ret i32 %1
 }
@@ -198,7 +198,7 @@ define i32 @test10(i8** %esc) {
 define i32 @test11(i8** %esc) {
   %call = tail call i8* @strndup(i8* getelementptr inbounds ([8 x i8], [8 x i8]* @.str, i64 0, i64 0), i32 7) nounwind
   store i8* %call, i8** %esc, align 8
-  %1 = tail call i32 @llvm.objectsize.i32.p0i8(i8* %call, i1 true, i1 false)
+  %1 = tail call i32 @llvm.objectsize.i32.p0i8(i8* %call, i1 true, i1 false, i1 false)
 ; CHECK: ret i32 8
   ret i32 %1
 }
@@ -207,7 +207,7 @@ define i32 @test11(i8** %esc) {
 define i32 @test12(i8** %esc) {
   %call = tail call i8* @strndup(i8* getelementptr inbounds ([8 x i8], [8 x i8]* @.str, i64 0, i64 0), i32 8) nounwind
   store i8* %call, i8** %esc, align 8
-  %1 = tail call i32 @llvm.objectsize.i32.p0i8(i8* %call, i1 true, i1 false)
+  %1 = tail call i32 @llvm.objectsize.i32.p0i8(i8* %call, i1 true, i1 false, i1 false)
 ; CHECK: ret i32 8
   ret i32 %1
 }
@@ -216,7 +216,7 @@ define i32 @test12(i8** %esc) {
 define i32 @test13(i8** %esc) {
   %call = tail call i8* @strndup(i8* getelementptr inbounds ([8 x i8], [8 x i8]* @.str, i64 0, i64 0), i32 57) nounwind
   store i8* %call, i8** %esc, align 8
-  %1 = tail call i32 @llvm.objectsize.i32.p0i8(i8* %call, i1 true, i1 false)
+  %1 = tail call i32 @llvm.objectsize.i32.p0i8(i8* %call, i1 true, i1 false, i1 false)
 ; CHECK: ret i32 8
   ret i32 %1
 }
@@ -227,7 +227,7 @@ define i32 @test13(i8** %esc) {
 ; CHECK-NEXT: ret i32 60
 define i32 @test18() {
   %bc = bitcast [60 x i8]* @globalalias to i8*
-  %1 = call i32 @llvm.objectsize.i32.p0i8(i8* %bc, i1 false, i1 false)
+  %1 = call i32 @llvm.objectsize.i32.p0i8(i8* %bc, i1 false, i1 false, i1 false)
   ret i32 %1
 }
 
@@ -237,35 +237,35 @@ define i32 @test18() {
 ; CHECK: llvm.objectsize
 define i32 @test19() {
   %bc = bitcast [60 x i8]* @globalalias2 to i8*
-  %1 = call i32 @llvm.objectsize.i32.p0i8(i8* %bc, i1 false, i1 false)
+  %1 = call i32 @llvm.objectsize.i32.p0i8(i8* %bc, i1 false, i1 false, i1 false)
   ret i32 %1
 }
 
 ; CHECK-LABEL: @test20(
 ; CHECK: ret i32 0
 define i32 @test20() {
-  %1 = call i32 @llvm.objectsize.i32.p0i8(i8* null, i1 false, i1 false)
+  %1 = call i32 @llvm.objectsize.i32.p0i8(i8* null, i1 false, i1 false, i1 false)
   ret i32 %1
 }
 
 ; CHECK-LABEL: @test21(
 ; CHECK: ret i32 0
 define i32 @test21() {
-  %1 = call i32 @llvm.objectsize.i32.p0i8(i8* null, i1 true, i1 false)
+  %1 = call i32 @llvm.objectsize.i32.p0i8(i8* null, i1 true, i1 false, i1 false)
   ret i32 %1
 }
 
 ; CHECK-LABEL: @test22(
 ; CHECK: llvm.objectsize
 define i32 @test22() {
-  %1 = call i32 @llvm.objectsize.i32.p0i8(i8* null, i1 false, i1 true)
+  %1 = call i32 @llvm.objectsize.i32.p0i8(i8* null, i1 false, i1 true, i1 false)
   ret i32 %1
 }
 
 ; CHECK-LABEL: @test23(
 ; CHECK: llvm.objectsize
 define i32 @test23() {
-  %1 = call i32 @llvm.objectsize.i32.p0i8(i8* null, i1 true, i1 true)
+  %1 = call i32 @llvm.objectsize.i32.p0i8(i8* null, i1 true, i1 true, i1 false)
   ret i32 %1
 }
 
@@ -274,7 +274,7 @@ define i32 @test23() {
 ; CHECK: llvm.objectsize
 define i32 @test24() {
   %1 = call i32 @llvm.objectsize.i32.p1i8(i8 addrspace(1)* null, i1 false,
-                                          i1 false)
+                                          i1 false, i1 false)
   ret i32 %1
 }
 
@@ -282,7 +282,7 @@ define i32 @test24() {
 ; CHECK: llvm.objectsize
 define i32 @test25() {
   %1 = call i32 @llvm.objectsize.i32.p1i8(i8 addrspace(1)* null, i1 true,
-                                          i1 false)
+                                          i1 false, i1 false)
   ret i32 %1
 }
 
@@ -290,7 +290,7 @@ define i32 @test25() {
 ; CHECK: llvm.objectsize
 define i32 @test26() {
   %1 = call i32 @llvm.objectsize.i32.p1i8(i8 addrspace(1)* null, i1 false,
-                                          i1 true)
+                                          i1 true, i1 false)
   ret i32 %1
 }
 
@@ -298,6 +298,6 @@ define i32 @test26() {
 ; CHECK: llvm.objectsize
 define i32 @test27() {
   %1 = call i32 @llvm.objectsize.i32.p1i8(i8 addrspace(1)* null, i1 true,
-                                          i1 true)
+                                          i1 true, i1 false)
   ret i32 %1
 }
diff --git a/test/Transforms/InstCombine/operand-complexity.ll b/test/Transforms/InstCombine/operand-complexity.ll
index 20abe7b48f9fd6cb4b54e917201b4fdc7c48e3c7..c67fb0803e03b692f85fe195e61d665a168d1cc6 100644
--- a/test/Transforms/InstCombine/operand-complexity.ll
+++ b/test/Transforms/InstCombine/operand-complexity.ll
@@ -1,18 +1,18 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -instcombine -S | FileCheck %s
 
-; 'Negate' is considered less complex than a normal binop, so the mul should have the binop as the first operand.
+; 'Negate' is considered less complex than a normal binop, so the xor should have the binop as the first operand.
 
 define i8 @neg(i8 %x) {
 ; CHECK-LABEL: @neg(
 ; CHECK-NEXT:    [[BO:%.*]] = udiv i8 [[X:%.*]], 42
 ; CHECK-NEXT:    [[NEGX:%.*]] = sub i8 0, [[X]]
-; CHECK-NEXT:    [[R:%.*]] = mul i8 [[BO]], [[NEGX]]
+; CHECK-NEXT:    [[R:%.*]] = xor i8 [[BO]], [[NEGX]]
 ; CHECK-NEXT:    ret i8 [[R]]
 ;
   %bo = udiv i8 %x, 42
   %negx = sub i8 0, %x
-  %r = mul i8 %negx, %bo
+  %r = xor i8 %negx, %bo
   ret i8 %r
 }
 
@@ -20,12 +20,12 @@ define <2 x i8> @neg_vec(<2 x i8> %x) {
 ; CHECK-LABEL: @neg_vec(
 ; CHECK-NEXT:    [[BO:%.*]] = udiv <2 x i8> [[X:%.*]], <i8 42, i8 -42>
 ; CHECK-NEXT:    [[NEGX:%.*]] = sub <2 x i8> zeroinitializer, [[X]]
-; CHECK-NEXT:    [[R:%.*]] = mul <2 x i8> [[BO]], [[NEGX]]
+; CHECK-NEXT:    [[R:%.*]] = xor <2 x i8> [[BO]], [[NEGX]]
 ; CHECK-NEXT:    ret <2 x i8> [[R]]
 ;
   %bo = udiv <2 x i8> %x, <i8 42, i8 -42>
   %negx = sub <2 x i8> <i8 0, i8 0>, %x
-  %r = mul <2 x i8> %negx, %bo
+  %r = xor <2 x i8> %negx, %bo
   ret <2 x i8> %r
 }
 
@@ -33,12 +33,12 @@ define <2 x i8> @neg_vec_undef(<2 x i8> %x) {
 ; CHECK-LABEL: @neg_vec_undef(
 ; CHECK-NEXT:    [[BO:%.*]] = udiv <2 x i8> [[X:%.*]], <i8 42, i8 -42>
 ; CHECK-NEXT:    [[NEGX:%.*]] = sub <2 x i8> <i8 0, i8 undef>, [[X]]
-; CHECK-NEXT:    [[R:%.*]] = mul <2 x i8> [[BO]], [[NEGX]]
+; CHECK-NEXT:    [[R:%.*]] = xor <2 x i8> [[BO]], [[NEGX]]
 ; CHECK-NEXT:    ret <2 x i8> [[R]]
 ;
   %bo = udiv <2 x i8> %x, <i8 42, i8 -42>
   %negx = sub <2 x i8> <i8 0, i8 undef>, %x
-  %r = mul <2 x i8> %negx, %bo
+  %r = xor <2 x i8> %negx, %bo
   ret <2 x i8> %r
 }
 
diff --git a/test/Transforms/InstCombine/pow-1.ll b/test/Transforms/InstCombine/pow-1.ll
index 672ec70e71ec2ac72782cc346ec8875d6bd86caa..957e2488b721b410e30c7488dd2a753969231396 100644
--- a/test/Transforms/InstCombine/pow-1.ll
+++ b/test/Transforms/InstCombine/pow-1.ll
@@ -1,15 +1,18 @@
 ; Test that the pow library call simplifier works correctly.
 ;
-; RUN: opt -instcombine -S < %s                                  | FileCheck %s --check-prefixes=ANY
-; RUN: opt -instcombine -S < %s -mtriple=x86_64-apple-macosx10.9 | FileCheck %s --check-prefixes=ANY,CHECK-EXP10
-; RUN: opt -instcombine -S < %s -mtriple=arm-apple-ios7.0        | FileCheck %s --check-prefixes=ANY,CHECK-EXP10
-; RUN: opt -instcombine -S < %s -mtriple=x86_64-apple-macosx10.8 | FileCheck %s --check-prefixes=ANY,CHECK-NO-EXP10
-; RUN: opt -instcombine -S < %s -mtriple=arm-apple-ios6.0        | FileCheck %s --check-prefixes=ANY,CHECK-NO-EXP10
-; RUN: opt -instcombine -S < %s -mtriple=x86_64-netbsd           | FileCheck %s --check-prefixes=ANY,CHECK-NO-EXP10
-; RUN: opt -instcombine -S < %s -mtriple=arm-apple-tvos9.0       | FileCheck %s --check-prefixes=ANY,CHECK-EXP10
-; RUN: opt -instcombine -S < %s -mtriple=arm-apple-watchos2.0    | FileCheck %s --check-prefixes=ANY,CHECK-EXP10
+; RUN: opt -instcombine -S < %s                                   | FileCheck %s --check-prefixes=CHECK,ANY
+; RUN: opt -instcombine -S < %s -mtriple=x86_64-apple-macosx10.9  | FileCheck %s --check-prefixes=CHECK,ANY,CHECK-EXP10
+; RUN: opt -instcombine -S < %s -mtriple=arm-apple-ios7.0         | FileCheck %s --check-prefixes=CHECK,ANY,CHECK-EXP10
+; RUN: opt -instcombine -S < %s -mtriple=x86_64-apple-macosx10.8  | FileCheck %s --check-prefixes=CHECK,ANY,CHECK-NO-EXP10
+; RUN: opt -instcombine -S < %s -mtriple=arm-apple-ios6.0         | FileCheck %s --check-prefixes=CHECK,ANY,CHECK-NO-EXP10
+; RUN: opt -instcombine -S < %s -mtriple=x86_64-netbsd            | FileCheck %s --check-prefixes=CHECK,ANY,CHECK-NO-EXP10
+; RUN: opt -instcombine -S < %s -mtriple=arm-apple-tvos9.0        | FileCheck %s --check-prefixes=CHECK,ANY,CHECK-EXP10
+; RUN: opt -instcombine -S < %s -mtriple=arm-apple-watchos2.0     | FileCheck %s --check-prefixes=CHECK,ANY,CHECK-EXP10
 ; rdar://7251832
-; RUN: opt -instcombine -S < %s -mtriple=x86_64-pc-windows-msvc  | FileCheck %s --check-prefixes=CHECK-WIN
+; RUN: opt -instcombine -S < %s -mtriple=i386-pc-windows-msvc18   | FileCheck %s --check-prefixes=CHECK,MSVC,VC32,CHECK-NO-EXP10
+; RUN: opt -instcombine -S < %s -mtriple=i386-pc-windows-msvc     | FileCheck %s --check-prefixes=CHECK,MSVC,VC51,VC19,CHECK-NO-EXP10
+; RUN: opt -instcombine -S < %s -mtriple=x86_64-pc-windows-msvc18 | FileCheck %s --check-prefixes=CHECK,MSVC,VC64,CHECK-NO-EXP10
+; RUN: opt -instcombine -S < %s -mtriple=x86_64-pc-windows-msvc   | FileCheck %s --check-prefixes=CHECK,MSVC,VC83,VC19,CHECK-NO-EXP10
 
 ; NOTE: The readonly attribute on the pow call should be preserved
 ; in the cases below where pow is transformed into another function call.
@@ -23,32 +26,39 @@ declare <2 x double> @llvm.pow.v2f64(<2 x double>, <2 x double>) nounwind readon
 ; Check pow(1.0, x) -> 1.0.
 
 define float @test_simplify1(float %x) {
-; ANY-LABEL: @test_simplify1(
+; CHECK-LABEL: @test_simplify1(
 ; ANY-NEXT:    ret float 1.000000e+00
+; VC32-NEXT:   [[POW:%.*]] = call float @powf(float 1.000000e+00, float [[X:%.*]])
+; VC32-NEXT:   ret float [[POW]]
+; VC64-NEXT:   ret float 1.000000e+00
 ;
   %retval = call float @powf(float 1.0, float %x)
   ret float %retval
 }
 
 define <2 x float> @test_simplify1v(<2 x float> %x) {
-; ANY-LABEL: @test_simplify1v(
+; CHECK-LABEL: @test_simplify1v(
 ; ANY-NEXT:    ret <2 x float> <float 1.000000e+00, float 1.000000e+00>
+; MSVC-NEXT:   [[POW:%.*]] = call <2 x float> @llvm.pow.v2f32(<2 x float> <float 1.000000e+00, float 1.000000e+00>, <2 x float> [[X:%.*]])
+; MSVC-NEXT:   ret <2 x float> [[POW]]
 ;
   %retval = call <2 x float> @llvm.pow.v2f32(<2 x float> <float 1.0, float 1.0>, <2 x float> %x)
   ret <2 x float> %retval
 }
 
 define double @test_simplify2(double %x) {
-; ANY-LABEL: @test_simplify2(
-; ANY-NEXT:    ret double 1.000000e+00
+; CHECK-LABEL: @test_simplify2(
+; CHECK-NEXT:  ret double 1.000000e+00
 ;
   %retval = call double @pow(double 1.0, double %x)
   ret double %retval
 }
 
 define <2 x double> @test_simplify2v(<2 x double> %x) {
-; ANY-LABEL: @test_simplify2v(
+; CHECK-LABEL: @test_simplify2v(
 ; ANY-NEXT:    ret <2 x double> <double 1.000000e+00, double 1.000000e+00>
+; MSVC-NEXT:   [[POW:%.*]] = call <2 x double> @llvm.pow.v2f64(<2 x double> <double 1.000000e+00, double 1.000000e+00>, <2 x double> [[X:%.*]])
+; MSVC-NEXT:   ret <2 x double> [[POW]]
 ;
   %retval = call <2 x double> @llvm.pow.v2f64(<2 x double> <double 1.0, double 1.0>, <2 x double> %x)
   ret <2 x double> %retval
@@ -57,104 +67,114 @@ define <2 x double> @test_simplify2v(<2 x double> %x) {
 ; Check pow(2.0 ** n, x) -> exp2(n * x).
 
 define float @test_simplify3(float %x) {
-; ANY-LABEL: @test_simplify3(
-; ANY-NEXT:    [[EXP2F:%.*]] = call float @exp2f(float [[X:%.*]]) [[NUW_RO:#[0-9]+]]
+; CHECK-LABEL: @test_simplify3(
+; ANY-NEXT:    [[EXP2F:%.*]] = call float @exp2f(float [[X:%.*]])
 ; ANY-NEXT:    ret float [[EXP2F]]
-;
-; CHECK-WIN-LABEL: @test_simplify3(
-; CHECK-WIN-NEXT:    [[POW:%.*]] = call float @powf(float 2.000000e+00, float [[X:%.*]])
-; CHECK-WIN-NEXT:    ret float [[POW]]
+; VC32-NEXT:   [[POW:%.*]] = call float @powf(float 2.000000e+00, float [[X:%.*]])
+; VC32-NEXT:   ret float [[POW]]
+; VC51-NEXT:   [[POW:%.*]] = call float @powf(float 2.000000e+00, float [[X:%.*]])
+; VC51-NEXT:   ret float [[POW]]
+; VC64-NEXT:   [[POW:%.*]] = call float @powf(float 2.000000e+00, float [[X:%.*]])
+; VC64-NEXT:   ret float [[POW]]
+; VC83-NEXT:   [[EXP2F:%.*]] = call float @exp2f(float [[X:%.*]])
+; VC83-NEXT:   ret float [[EXP2F]]
 ;
   %retval = call float @powf(float 2.0, float %x)
   ret float %retval
 }
 
 define double @test_simplify3n(double %x) {
-; ANY-LABEL: @test_simplify3n(
+; CHECK-LABEL: @test_simplify3n(
 ; ANY-NEXT:    [[MUL:%.*]] = fmul double [[X:%.*]], -2.000000e+00
-; ANY-NEXT:    [[EXP2:%.*]] = call double @exp2(double [[MUL]]) [[NUW_RO]]
+; ANY-NEXT:    [[EXP2:%.*]] = call double @exp2(double [[MUL]])
 ; ANY-NEXT:    ret double [[EXP2]]
-;
-; CHECK-WIN-LABEL: @test_simplify3n(
-; CHECK-WIN-NEXT:    [[POW:%.*]] = call double @pow(double 2.500000e-01, double [[X:%.*]])
-; CHECK-WIN-NEXT:    ret double [[POW]]
+; VC19-NEXT:   [[MUL:%.*]] = fmul double [[X:%.*]], -2.000000e+00
+; VC19-NEXT:   [[EXP2:%.*]] = call double @exp2(double [[MUL]])
+; VC19-NEXT:   ret double [[EXP2]]
+; VC32-NEXT:   [[POW:%.*]] = call double @pow(double 2.500000e-01, double [[X:%.*]])
+; VC32-NEXT:   ret double [[POW]]
+; VC64-NEXT:   [[POW:%.*]] = call double @pow(double 2.500000e-01, double [[X:%.*]])
+; VC64-NEXT:   ret double [[POW]]
 ;
   %retval = call double @pow(double 0.25, double %x)
   ret double %retval
 }
 
 define <2 x float> @test_simplify3v(<2 x float> %x) {
-; ANY-LABEL: @test_simplify3v(
+; CHECK-LABEL: @test_simplify3v(
 ; ANY-NEXT:    [[EXP2:%.*]] = call <2 x float> @llvm.exp2.v2f32(<2 x float> [[X:%.*]])
 ; ANY-NEXT:    ret <2 x float> [[EXP2]]
-;
-; CHECK-WIN-LABEL: @test_simplify3v(
-; CHECK-WIN-NEXT:    [[POW:%.*]] = call <2 x float> @llvm.pow.v2f32(<2 x float> <float 2.000000e+00, float 2.000000e+00>, <2 x float> [[X:%.*]])
-; CHECK-WIN-NEXT:    ret <2 x float> [[POW]]
+; MSVC-NEXT:   [[POW:%.*]] = call <2 x float> @llvm.pow.v2f32(<2 x float> <float 2.000000e+00, float 2.000000e+00>, <2 x float> [[X:%.*]])
+; MSVC-NEXT:   ret <2 x float> [[POW]]
 ;
   %retval = call <2 x float> @llvm.pow.v2f32(<2 x float> <float 2.0, float 2.0>, <2 x float> %x)
   ret <2 x float> %retval
 }
 
 define <2 x double> @test_simplify3vn(<2 x double> %x) {
-; ANY-LABEL: @test_simplify3vn(
+; CHECK-LABEL: @test_simplify3vn(
 ; ANY-NEXT:    [[MUL:%.*]] = fmul <2 x double> [[X:%.*]], <double 2.000000e+00, double 2.000000e+00>
 ; ANY-NEXT:    [[EXP2:%.*]] = call <2 x double> @llvm.exp2.v2f64(<2 x double> [[MUL]])
 ; ANY-NEXT:    ret <2 x double> [[EXP2]]
+; MSVC-NEXT:   [[POW:%.*]] = call <2 x double> @llvm.pow.v2f64(<2 x double> <double 4.000000e+00, double 4.000000e+00>, <2 x double> [[X:%.*]])
+; MSVC-NEXT:   ret <2 x double> [[POW]]
 ;
   %retval = call <2 x double> @llvm.pow.v2f64(<2 x double> <double 4.0, double 4.0>, <2 x double> %x)
   ret <2 x double> %retval
 }
 
 define double @test_simplify4(double %x) {
-; ANY-LABEL: @test_simplify4(
-; ANY-NEXT:    [[EXP2:%.*]] = call double @exp2(double [[X:%.*]]) [[NUW_RO]]
+; CHECK-LABEL: @test_simplify4(
+; ANY-NEXT:    [[EXP2:%.*]] = call double @exp2(double [[X:%.*]])
 ; ANY-NEXT:    ret double [[EXP2]]
-;
-; CHECK-WIN-LABEL: @test_simplify4(
-; CHECK-WIN-NEXT:    [[POW:%.*]] = call double @pow(double 2.000000e+00, double [[X:%.*]])
-; CHECK-WIN-NEXT:    ret double [[POW]]
+; VC19-NEXT:   [[EXP2:%.*]] = call double @exp2(double [[X:%.*]])
+; VC19-NEXT:   ret double [[EXP2]]
+; VC32-NEXT:   [[POW:%.*]] = call double @pow(double 2.000000e+00, double [[X:%.*]])
+; VC32-NEXT:   ret double [[POW]]
+; VC64-NEXT:   [[POW:%.*]] = call double @pow(double 2.000000e+00, double [[X:%.*]])
+; VC64-NEXT:   ret double [[POW]]
 ;
   %retval = call double @pow(double 2.0, double %x)
   ret double %retval
 }
 
 define float @test_simplify4n(float %x) {
-; ANY-LABEL: @test_simplify4n(
+; CHECK-LABEL: @test_simplify4n(
 ; ANY-NEXT:    [[MUL:%.*]] = fmul float [[X:%.*]], 3.000000e+00
-; ANY-NEXT:    [[EXP2F:%.*]] = call float @exp2f(float [[MUL]]) [[NUW_RO]]
+; ANY-NEXT:    [[EXP2F:%.*]] = call float @exp2f(float [[MUL]])
 ; ANY-NEXT:    ret float [[EXP2F]]
-;
-; CHECK-WIN-LABEL: @test_simplify4n(
-; CHECK-WIN-NEXT:    [[POW:%.*]] = call float @powf(float 8.000000e+00, float [[X:%.*]])
-; CHECK-WIN-NEXT:    ret float [[POW]]
+; VC32-NEXT:   [[POW:%.*]] = call float @powf(float 8.000000e+00, float [[X:%.*]])
+; VC32-NEXT:   ret float [[POW]]
+; VC51-NEXT:   [[POW:%.*]] = call float @powf(float 8.000000e+00, float [[X:%.*]])
+; VC51-NEXT:   ret float [[POW]]
+; VC64-NEXT:   [[POW:%.*]] = call float @powf(float 8.000000e+00, float [[X:%.*]])
+; VC64-NEXT:   ret float [[POW]]
+; VC83-NEXT:   [[MUL:%.*]] = fmul float [[X:%.*]], 3.000000e+00
+; VC83-NEXT:   [[EXP2F:%.*]] = call float @exp2f(float [[MUL]])
+; VC83-NEXT:   ret float [[EXP2F]]
 ;
   %retval = call float @powf(float 8.0, float %x)
   ret float %retval
 }
 
 define <2 x double> @test_simplify4v(<2 x double> %x) {
-; ANY-LABEL: @test_simplify4v(
+; CHECK-LABEL: @test_simplify4v(
 ; ANY-NEXT:    [[EXP2:%.*]] = call <2 x double> @llvm.exp2.v2f64(<2 x double> [[X:%.*]])
 ; ANY-NEXT:    ret <2 x double> [[EXP2]]
-;
-; CHECK-WIN-LABEL: @test_simplify4v(
-; CHECK-WIN-NEXT:    [[POW:%.*]] = call <2 x double> @llvm.pow.v2f64(<2 x double> <double 2.000000e+00, double 2.000000e+00>, <2 x double> [[X:%.*]])
-; CHECK-WIN-NEXT:    ret <2 x double> [[POW]]
+; MSVC-NEXT:   [[POW:%.*]] = call <2 x double> @llvm.pow.v2f64(<2 x double> <double 2.000000e+00, double 2.000000e+00>, <2 x double> [[X:%.*]])
+; MSVC-NEXT:   ret <2 x double> [[POW]]
 ;
   %retval = call <2 x double> @llvm.pow.v2f64(<2 x double> <double 2.0, double 2.0>, <2 x double> %x)
   ret <2 x double> %retval
 }
 
 define <2 x float> @test_simplify4vn(<2 x float> %x) {
-; ANY-LABEL: @test_simplify4vn(
+; CHECK-LABEL: @test_simplify4vn(
 ; ANY-NEXT:    [[MUL:%.*]] = fsub <2 x float> <float -0.000000e+00, float -0.000000e+00>, [[X:%.*]]
 ; ANY-NEXT:    [[EXP2:%.*]] = call <2 x float> @llvm.exp2.v2f32(<2 x float> [[MUL]])
 ; ANY-NEXT:    ret <2 x float> [[EXP2]]
-;
-; CHECK-WIN-LABEL: @test_simplify4vn(
-; CHECK-WIN-NEXT:    [[POW:%.*]] = call <2 x float> @llvm.pow.v2f32(<2 x float> <float 5.000000e-01, float 5.000000e-01>, <2 x float> [[X:%.*]])
-; CHECK-WIN-NEXT:    ret <2 x float> [[POW]]
+; MSVC-NEXT:   [[POW:%.*]] = call <2 x float> @llvm.pow.v2f32(<2 x float> <float 5.000000e-01, float 5.000000e-01>, <2 x float> [[X:%.*]])
+; MSVC-NEXT:   ret <2 x float> [[POW]]
 ;
   %retval = call <2 x float> @llvm.pow.v2f32(<2 x float> <float 0.5, float 0.5>, <2 x float> %x)
   ret <2 x float> %retval
@@ -163,32 +183,42 @@ define <2 x float> @test_simplify4vn(<2 x float> %x) {
 ; Check pow(x, 0.0) -> 1.0.
 
 define float @test_simplify5(float %x) {
-; ANY-LABEL: @test_simplify5(
+; CHECK-LABEL: @test_simplify5(
 ; ANY-NEXT:    ret float 1.000000e+00
+; VC32-NEXT:   [[POW:%.*]] = call float @powf(float [[X:%.*]], float 0.000000e+00)
+; VC32-NEXT:   ret float [[POW]]
+; VC51-NEXT:   [[POW:%.*]] = call float @powf(float [[X:%.*]], float 0.000000e+00)
+; VC51-NEXT:   ret float [[POW]]
+; VC64-NEXT:   ret float 1.000000e+00
+; VC83-NEXT:   ret float 1.000000e+00
 ;
   %retval = call float @powf(float %x, float 0.0)
   ret float %retval
 }
 
 define <2 x float> @test_simplify5v(<2 x float> %x) {
-; ANY-LABEL: @test_simplify5v(
+; CHECK-LABEL: @test_simplify5v(
 ; ANY-NEXT:    ret <2 x float> <float 1.000000e+00, float 1.000000e+00>
+; MSVC-NEXT:   [[POW:%.*]] = call <2 x float> @llvm.pow.v2f32(<2 x float> [[X:%.*]], <2 x float> zeroinitializer)
+; MSVC-NEXT:   ret <2 x float> [[POW]]
 ;
   %retval = call <2 x float> @llvm.pow.v2f32(<2 x float> %x, <2 x float> <float 0.0, float 0.0>)
   ret <2 x float> %retval
 }
 
 define double @test_simplify6(double %x) {
-; ANY-LABEL: @test_simplify6(
-; ANY-NEXT:    ret double 1.000000e+00
+; CHECK-LABEL: @test_simplify6(
+; CHECK-NEXT:  ret double 1.000000e+00
 ;
   %retval = call double @pow(double %x, double 0.0)
   ret double %retval
 }
 
 define <2 x double> @test_simplify6v(<2 x double> %x) {
-; ANY-LABEL: @test_simplify6v(
+; CHECK-LABEL: @test_simplify6v(
 ; ANY-NEXT:    ret <2 x double> <double 1.000000e+00, double 1.000000e+00>
+; MSVC-NEXT:   [[POW:%.*]] = call <2 x double> @llvm.pow.v2f64(<2 x double> [[X:%.*]], <2 x double> zeroinitializer)
+; MSVC-NEXT:   ret <2 x double> [[POW]]
 ;
   %retval = call <2 x double> @llvm.pow.v2f64(<2 x double> %x, <2 x double> <double 0.0, double 0.0>)
   ret <2 x double> %retval
@@ -197,24 +227,38 @@ define <2 x double> @test_simplify6v(<2 x double> %x) {
 ; Check pow(x, 0.5) -> fabs(sqrt(x)), where x != -infinity.
 
 define float @test_simplify7(float %x) {
-; ANY-LABEL: @test_simplify7(
-; ANY-NEXT:    [[SQRTF:%.*]] = call float @sqrtf(float [[X:%.*]]) [[NUW_RO]]
+; CHECK-LABEL: @test_simplify7(
+; ANY-NEXT:    [[SQRTF:%.*]] = call float @sqrtf(float [[X:%.*]])
 ; ANY-NEXT:    [[ABS:%.*]] = call float @llvm.fabs.f32(float [[SQRTF]])
 ; ANY-NEXT:    [[ISINF:%.*]] = fcmp oeq float [[X]], 0xFFF0000000000000
 ; ANY-NEXT:    [[TMP1:%.*]] = select i1 [[ISINF]], float 0x7FF0000000000000, float [[ABS]]
 ; ANY-NEXT:    ret float [[TMP1]]
+; VC32-NEXT:   [[POW:%.*]] = call float @powf(float [[X:%.*]], float 5.000000e-01)
+; VC32-NEXT:   ret float [[POW]]
+; VC51-NEXT:   [[POW:%.*]] = call float @powf(float [[X:%.*]], float 5.000000e-01)
+; VC51-NEXT:   ret float [[POW]]
+; VC64-NEXT:   [[SQRTF:%.*]] = call float @sqrtf(float [[X:%.*]])
+; VC64-NEXT:   [[ABS:%.*]] = call float @llvm.fabs.f32(float [[SQRTF]])
+; VC64-NEXT:   [[ISINF:%.*]] = fcmp oeq float [[X]], 0xFFF0000000000000
+; VC64-NEXT:   [[TMP1:%.*]] = select i1 [[ISINF]], float 0x7FF0000000000000, float [[ABS]]
+; VC64-NEXT:   ret float [[TMP1]]
+; VC83-NEXT:   [[SQRTF:%.*]] = call float @sqrtf(float [[X:%.*]])
+; VC83-NEXT:   [[ABS:%.*]] = call float @llvm.fabs.f32(float [[SQRTF]])
+; VC83-NEXT:   [[ISINF:%.*]] = fcmp oeq float [[X]], 0xFFF0000000000000
+; VC83-NEXT:   [[TMP1:%.*]] = select i1 [[ISINF]], float 0x7FF0000000000000, float [[ABS]]
+; VC83-NEXT:   ret float [[TMP1]]
 ;
   %retval = call float @powf(float %x, float 0.5)
   ret float %retval
 }
 
 define double @test_simplify8(double %x) {
-; ANY-LABEL: @test_simplify8(
-; ANY-NEXT:    [[SQRT:%.*]] = call double @sqrt(double [[X:%.*]]) [[NUW_RO]]
-; ANY-NEXT:    [[ABS:%.*]] = call double @llvm.fabs.f64(double [[SQRT]])
-; ANY-NEXT:    [[ISINF:%.*]] = fcmp oeq double [[X]], 0xFFF0000000000000
-; ANY-NEXT:    [[TMP1:%.*]] = select i1 [[ISINF]], double 0x7FF0000000000000, double [[ABS]]
-; ANY-NEXT:    ret double [[TMP1]]
+; CHECK-LABEL: @test_simplify8(
+; CHECK-NEXT:  [[SQRT:%.*]] = call double @sqrt(double [[X:%.*]])
+; CHECK-NEXT:  [[ABS:%.*]] = call double @llvm.fabs.f64(double [[SQRT]])
+; CHECK-NEXT:  [[ISINF:%.*]] = fcmp oeq double [[X]], 0xFFF0000000000000
+; CHECK-NEXT:  [[TMP1:%.*]] = select i1 [[ISINF]], double 0x7FF0000000000000, double [[ABS]]
+; CHECK-NEXT:  ret double [[TMP1]]
 ;
   %retval = call double @pow(double %x, double 0.5)
   ret double %retval
@@ -223,16 +267,22 @@ define double @test_simplify8(double %x) {
 ; Check pow(-infinity, 0.5) -> +infinity.
 
 define float @test_simplify9(float %x) {
-; ANY-LABEL: @test_simplify9(
+; CHECK-LABEL: @test_simplify9(
 ; ANY-NEXT:    ret float 0x7FF0000000000000
+; VC32-NEXT:   [[POW:%.*]] = call float @powf(float 0xFFF0000000000000, float 5.000000e-01)
+; VC32-NEXT:   ret float [[POW]]
+; VC51-NEXT:   [[POW:%.*]] = call float @powf(float 0xFFF0000000000000, float 5.000000e-01)
+; VC51-NEXT:   ret float [[POW]]
+; VC64-NEXT:   ret float 0x7FF0000000000000
+; VC83-NEXT:   ret float 0x7FF0000000000000
 ;
   %retval = call float @powf(float 0xFFF0000000000000, float 0.5)
   ret float %retval
 }
 
 define double @test_simplify10(double %x) {
-; ANY-LABEL: @test_simplify10(
-; ANY-NEXT:    ret double 0x7FF0000000000000
+; CHECK-LABEL: @test_simplify10(
+; CHECK-NEXT:  ret double 0x7FF0000000000000
 ;
   %retval = call double @pow(double 0xFFF0000000000000, double 0.5)
   ret double %retval
@@ -241,32 +291,42 @@ define double @test_simplify10(double %x) {
 ; Check pow(x, 1.0) -> x.
 
 define float @test_simplify11(float %x) {
-; ANY-LABEL: @test_simplify11(
+; CHECK-LABEL: @test_simplify11(
 ; ANY-NEXT:    ret float [[X:%.*]]
+; VC32-NEXT:   [[POW:%.*]] = call float @powf(float [[X:%.*]], float 1.000000e+00)
+; VC32-NEXT:   ret float [[POW]]
+; VC51-NEXT:   [[POW:%.*]] = call float @powf(float [[X:%.*]], float 1.000000e+00)
+; VC51-NEXT:   ret float [[POW]]
+; VC64-NEXT:   ret float [[X:%.*]]
+; VC83-NEXT:   ret float [[X:%.*]]
 ;
   %retval = call float @powf(float %x, float 1.0)
   ret float %retval
 }
 
 define <2 x float> @test_simplify11v(<2 x float> %x) {
-; ANY-LABEL: @test_simplify11v(
+; CHECK-LABEL: @test_simplify11v(
 ; ANY-NEXT:    ret <2 x float> [[X:%.*]]
+; MSVC-NEXT:   [[POW:%.*]] = call <2 x float> @llvm.pow.v2f32(<2 x float> [[X:%.*]], <2 x float> <float 1.000000e+00, float 1.000000e+00>)
+; MSVC-NEXT:   ret <2 x float> [[POW]]
 ;
   %retval = call <2 x float> @llvm.pow.v2f32(<2 x float> %x, <2 x float> <float 1.0, float 1.0>)
   ret <2 x float> %retval
 }
 
 define double @test_simplify12(double %x) {
-; ANY-LABEL: @test_simplify12(
-; ANY-NEXT:    ret double [[X:%.*]]
+; CHECK-LABEL: @test_simplify12(
+; CHECK-NEXT:  ret double [[X:%.*]]
 ;
   %retval = call double @pow(double %x, double 1.0)
   ret double %retval
 }
 
 define <2 x double> @test_simplify12v(<2 x double> %x) {
-; ANY-LABEL: @test_simplify12v(
+; CHECK-LABEL: @test_simplify12v(
 ; ANY-NEXT:    ret <2 x double> [[X:%.*]]
+; MSVC-NEXT:   [[POW:%.*]] = call <2 x double> @llvm.pow.v2f64(<2 x double> [[X:%.*]], <2 x double> <double 1.000000e+00, double 1.000000e+00>)
+; MSVC-NEXT:   ret <2 x double> [[POW]]
 ;
   %retval = call <2 x double> @llvm.pow.v2f64(<2 x double> %x, <2 x double> <double 1.0, double 1.0>)
   ret <2 x double> %retval
@@ -275,36 +335,48 @@ define <2 x double> @test_simplify12v(<2 x double> %x) {
 ; Check pow(x, 2.0) -> x*x.
 
 define float @pow2_strict(float %x) {
-; ANY-LABEL: @pow2_strict(
+; CHECK-LABEL: @pow2_strict(
 ; ANY-NEXT:    [[SQUARE:%.*]] = fmul float [[X:%.*]], [[X]]
 ; ANY-NEXT:    ret float [[SQUARE]]
+; VC32-NEXT:   [[POW:%.*]] = call float @powf(float [[X:%.*]], float 2.000000e+00)
+; VC32-NEXT:   ret float [[POW]]
+; VC51-NEXT:   [[POW:%.*]] = call float @powf(float [[X:%.*]], float 2.000000e+00)
+; VC51-NEXT:   ret float [[POW]]
+; VC64-NEXT:   [[SQUARE:%.*]] = fmul float [[X:%.*]], [[X]]
+; VC64-NEXT:   ret float [[SQUARE]]
+; VC83-NEXT:   [[SQUARE:%.*]] = fmul float [[X:%.*]], [[X]]
+; VC83-NEXT:   ret float [[SQUARE]]
 ;
   %r = call float @powf(float %x, float 2.0)
   ret float %r
 }
 
 define <2 x float> @pow2_strictv(<2 x float> %x) {
-; ANY-LABEL: @pow2_strictv(
+; CHECK-LABEL: @pow2_strictv(
 ; ANY-NEXT:    [[SQUARE:%.*]] = fmul <2 x float> [[X:%.*]], [[X]]
 ; ANY-NEXT:    ret <2 x float> [[SQUARE]]
+; MSVC-NEXT:   [[POW:%.*]] = call <2 x float> @llvm.pow.v2f32(<2 x float> [[X:%.*]], <2 x float> <float 2.000000e+00, float 2.000000e+00>)
+; MSVC-NEXT:   ret <2 x float> [[POW]]
 ;
   %r = call <2 x float> @llvm.pow.v2f32(<2 x float> %x, <2 x float> <float 2.0, float 2.0>)
   ret <2 x float> %r
 }
 
 define double @pow2_double_strict(double %x) {
-; ANY-LABEL: @pow2_double_strict(
-; ANY-NEXT:    [[SQUARE:%.*]] = fmul double [[X:%.*]], [[X]]
-; ANY-NEXT:    ret double [[SQUARE]]
+; CHECK-LABEL: @pow2_double_strict(
+; CHECK-NEXT:  [[SQUARE:%.*]] = fmul double [[X:%.*]], [[X]]
+; CHECK-NEXT:  ret double [[SQUARE]]
 ;
   %r = call double @pow(double %x, double 2.0)
   ret double %r
 }
 
 define <2 x double> @pow2_double_strictv(<2 x double> %x) {
-; ANY-LABEL: @pow2_double_strictv(
+; CHECK-LABEL: @pow2_double_strictv(
 ; ANY-NEXT:    [[SQUARE:%.*]] = fmul <2 x double> [[X:%.*]], [[X]]
 ; ANY-NEXT:    ret <2 x double> [[SQUARE]]
+; MSVC-NEXT:   [[POW:%.*]] = call <2 x double> @llvm.pow.v2f64(<2 x double> [[X:%.*]], <2 x double> <double 2.000000e+00, double 2.000000e+00>)
+; MSVC-NEXT:   ret <2 x double> [[POW]]
 ;
   %r = call <2 x double> @llvm.pow.v2f64(<2 x double> %x, <2 x double> <double 2.0, double 2.0>)
   ret <2 x double> %r
@@ -313,9 +385,17 @@ define <2 x double> @pow2_double_strictv(<2 x double> %x) {
 ; Don't drop the FMF - PR35601 ( https://bugs.llvm.org/show_bug.cgi?id=35601 )
 
 define float @pow2_fast(float %x) {
-; ANY-LABEL: @pow2_fast(
+; CHECK-LABEL: @pow2_fast(
 ; ANY-NEXT:    [[SQUARE:%.*]] = fmul fast float [[X:%.*]], [[X]]
 ; ANY-NEXT:    ret float [[SQUARE]]
+; VC32-NEXT:   [[POW:%.*]] = call fast float @powf(float [[X:%.*]], float 2.000000e+00)
+; VC32-NEXT:   ret float [[POW]]
+; VC51-NEXT:   [[POW:%.*]] = call fast float @powf(float [[X:%.*]], float 2.000000e+00)
+; VC51-NEXT:   ret float [[POW]]
+; VC64-NEXT:   [[SQUARE:%.*]] = fmul fast float [[X:%.*]], [[X]]
+; VC64-NEXT:   ret float [[SQUARE]]
+; VC83-NEXT:   [[SQUARE:%.*]] = fmul fast float [[X:%.*]], [[X]]
+; VC83-NEXT:   ret float [[SQUARE]]
 ;
   %r = call fast float @powf(float %x, float 2.0)
   ret float %r
@@ -324,48 +404,60 @@ define float @pow2_fast(float %x) {
 ; Check pow(x, -1.0) -> 1.0/x.
 
 define float @pow_neg1_strict(float %x) {
-; ANY-LABEL: @pow_neg1_strict(
+; CHECK-LABEL: @pow_neg1_strict(
 ; ANY-NEXT:    [[RECIPROCAL:%.*]] = fdiv float 1.000000e+00, [[X:%.*]]
 ; ANY-NEXT:    ret float [[RECIPROCAL]]
+; VC32-NEXT:   [[POW:%.*]] = call float @powf(float [[X:%.*]], float -1.000000e+00)
+; VC32-NEXT:   ret float [[POW]]
+; VC51-NEXT:   [[POW:%.*]] = call float @powf(float [[X:%.*]], float -1.000000e+00)
+; VC51-NEXT:   ret float [[POW]]
+; VC64-NEXT:   [[RECIPROCAL:%.*]] = fdiv float 1.000000e+00, [[X:%.*]]
+; VC64-NEXT:   ret float [[RECIPROCAL]]
+; VC83-NEXT:   [[RECIPROCAL:%.*]] = fdiv float 1.000000e+00, [[X:%.*]]
+; VC83-NEXT:   ret float [[RECIPROCAL]]
 ;
   %r = call float @powf(float %x, float -1.0)
   ret float %r
 }
 
 define <2 x float> @pow_neg1_strictv(<2 x float> %x) {
-; ANY-LABEL: @pow_neg1_strictv(
+; CHECK-LABEL: @pow_neg1_strictv(
 ; ANY-NEXT:    [[RECIPROCAL:%.*]] = fdiv <2 x float> <float 1.000000e+00, float 1.000000e+00>, [[X:%.*]]
 ; ANY-NEXT:    ret <2 x float> [[RECIPROCAL]]
+; MSVC-NEXT:   [[POW:%.*]] = call <2 x float> @llvm.pow.v2f32(<2 x float> [[X:%.*]], <2 x float> <float -1.000000e+00, float -1.000000e+00>)
+; MSVC-NEXT:   ret <2 x float> [[POW]]
 ;
   %r = call <2 x float> @llvm.pow.v2f32(<2 x float> %x, <2 x float> <float -1.0, float -1.0>)
   ret <2 x float> %r
 }
 
 define double @pow_neg1_double_fast(double %x) {
-; ANY-LABEL: @pow_neg1_double_fast(
-; ANY-NEXT:    [[RECIPROCAL:%.*]] = fdiv fast double 1.000000e+00, [[X:%.*]]
-; ANY-NEXT:    ret double [[RECIPROCAL]]
+; CHECK-LABEL: @pow_neg1_double_fast(
+; CHECK-NEXT:  [[RECIPROCAL:%.*]] = fdiv fast double 1.000000e+00, [[X:%.*]]
+; CHECK-NEXT:  ret double [[RECIPROCAL]]
 ;
   %r = call fast double @pow(double %x, double -1.0)
   ret double %r
 }
 
 define <2 x double> @pow_neg1_double_fastv(<2 x double> %x) {
-; ANY-LABEL: @pow_neg1_double_fastv(
+; CHECK-LABEL: @pow_neg1_double_fastv(
 ; ANY-NEXT:    [[RECIPROCAL:%.*]] = fdiv fast <2 x double> <double 1.000000e+00, double 1.000000e+00>, [[X:%.*]]
 ; ANY-NEXT:    ret <2 x double> [[RECIPROCAL]]
+; MSVC-NEXT:   [[POW:%.*]] = call fast <2 x double> @llvm.pow.v2f64(<2 x double> [[X:%.*]], <2 x double> <double -1.000000e+00, double -1.000000e+00>)
+; MSVC-NEXT:   ret <2 x double> [[POW]]
 ;
   %r = call fast <2 x double> @llvm.pow.v2f64(<2 x double> %x, <2 x double> <double -1.0, double -1.0>)
   ret <2 x double> %r
 }
 
 define double @test_simplify17(double %x) {
-; ANY-LABEL: @test_simplify17(
-; ANY-NEXT:    [[SQRT:%.*]] = call double @llvm.sqrt.f64(double [[X:%.*]])
-; ANY-NEXT:    [[ABS:%.*]] = call double @llvm.fabs.f64(double [[SQRT]])
-; ANY-NEXT:    [[ISINF:%.*]] = fcmp oeq double [[X]], 0xFFF0000000000000
-; ANY-NEXT:    [[TMP1:%.*]] = select i1 [[ISINF]], double 0x7FF0000000000000, double [[ABS]]
-; ANY-NEXT:    ret double [[TMP1]]
+; CHECK-LABEL: @test_simplify17(
+; CHECK-NEXT:  [[SQRT:%.*]] = call double @llvm.sqrt.f64(double [[X:%.*]])
+; CHECK-NEXT:  [[ABS:%.*]] = call double @llvm.fabs.f64(double [[SQRT]])
+; CHECK-NEXT:  [[ISINF:%.*]] = fcmp oeq double [[X]], 0xFFF0000000000000
+; CHECK-NEXT:  [[TMP1:%.*]] = select i1 [[ISINF]], double 0x7FF0000000000000, double [[ABS]]
+; CHECK-NEXT:  ret double [[TMP1]]
 ;
   %retval = call double @llvm.pow.f64(double %x, double 0.5)
   ret double %retval
@@ -374,29 +466,23 @@ define double @test_simplify17(double %x) {
 ; Check pow(10.0, x) -> __exp10(x) on OS X 10.9+ and iOS 7.0+.
 
 define float @test_simplify18(float %x) {
-; CHECK-EXP10-LABEL: @test_simplify18(
-; CHECK-EXP10-NEXT:    [[__EXP10F:%.*]] = call float @__exp10f(float [[X:%.*]]) [[NUW_RO]]
-; CHECK-EXP10-NEXT:    ret float [[__EXP10F]]
-;
-; CHECK-NO-EXP10-LABEL: @test_simplify18(
-; CHECK-NO-EXP10-NEXT:    [[RETVAL:%.*]] = call float @powf(float 1.000000e+01, float [[X:%.*]])
-; CHECK-NO-EXP10-NEXT:    ret float [[RETVAL]]
+; CHECK-LABEL:          @test_simplify18(
+; CHECK-EXP10-NEXT:     [[__EXP10F:%.*]] = call float @__exp10f(float [[X:%.*]])
+; CHECK-EXP10-NEXT:     ret float [[__EXP10F]]
+; CHECK-NO-EXP10-NEXT:  [[RETVAL:%.*]] = call float @powf(float 1.000000e+01, float [[X:%.*]])
+; CHECK-NO-EXP10-NEXT:  ret float [[RETVAL]]
 ;
   %retval = call float @powf(float 10.0, float %x)
   ret float %retval
 }
 
 define double @test_simplify19(double %x) {
-; CHECK-EXP10-LABEL: @test_simplify19(
-; CHECK-EXP10-NEXT:    [[__EXP10:%.*]] = call double @__exp10(double [[X:%.*]]) [[NUW_RO]]
-; CHECK-EXP10-NEXT:    ret double [[__EXP10]]
-;
-; CHECK-NO-EXP10-LABEL: @test_simplify19(
-; CHECK-NO-EXP10-NEXT:    [[RETVAL:%.*]] = call double @pow(double 1.000000e+01, double [[X:%.*]])
-; CHECK-NO-EXP10-NEXT:    ret double [[RETVAL]]
+; CHECK-LABEL:          @test_simplify19(
+; CHECK-EXP10-NEXT:     [[__EXP10:%.*]] = call double @__exp10(double [[X:%.*]])
+; CHECK-EXP10-NEXT:     ret double [[__EXP10]]
+; CHECK-NO-EXP10-NEXT:  [[RETVAL:%.*]] = call double @pow(double 1.000000e+01, double [[X:%.*]])
+; CHECK-NO-EXP10-NEXT:  ret double [[RETVAL]]
 ;
   %retval = call double @pow(double 10.0, double %x)
   ret double %retval
 }
-
-; CHECK-EXP10: attributes [[NUW_RO]] = { nounwind readonly }
diff --git a/test/Transforms/InstCombine/rotate.ll b/test/Transforms/InstCombine/rotate.ll
index 492817e47e22b804f51614be785b57a43b770652..6e11c68df9261516a9cfda6ffa1750cb4271f88d 100644
--- a/test/Transforms/InstCombine/rotate.ll
+++ b/test/Transforms/InstCombine/rotate.ll
@@ -206,19 +206,13 @@ define <3 x i42> @rotr_v3i42(<3 x i42> %x, <3 x i42> %y) {
   ret <3 x i42> %r
 }
 
-; TODO:
 ; This is the canonical pattern for a UB-safe rotate-by-variable with power-of-2-size scalar type.
 ; The backend expansion of funnel shift for targets that don't have a rotate instruction should
 ; match the original IR, so it is always good to canonicalize to the intrinsics for this pattern.
 
 define i32 @rotl_safe_i32(i32 %x, i32 %y) {
 ; CHECK-LABEL: @rotl_safe_i32(
-; CHECK-NEXT:    [[NEGY:%.*]] = sub i32 0, [[Y:%.*]]
-; CHECK-NEXT:    [[YMASK:%.*]] = and i32 [[Y]], 31
-; CHECK-NEXT:    [[NEGYMASK:%.*]] = and i32 [[NEGY]], 31
-; CHECK-NEXT:    [[SHL:%.*]] = shl i32 [[X:%.*]], [[YMASK]]
-; CHECK-NEXT:    [[SHR:%.*]] = lshr i32 [[X]], [[NEGYMASK]]
-; CHECK-NEXT:    [[R:%.*]] = or i32 [[SHR]], [[SHL]]
+; CHECK-NEXT:    [[R:%.*]] = call i32 @llvm.fshl.i32(i32 [[X:%.*]], i32 [[X]], i32 [[Y:%.*]])
 ; CHECK-NEXT:    ret i32 [[R]]
 ;
   %negy = sub i32 0, %y
@@ -230,18 +224,14 @@ define i32 @rotl_safe_i32(i32 %x, i32 %y) {
   ret i32 %r
 }
 
-; TODO:
 ; Extra uses don't change anything.
 
 define i16 @rotl_safe_i16_commute_extra_use(i16 %x, i16 %y, i16* %p) {
 ; CHECK-LABEL: @rotl_safe_i16_commute_extra_use(
 ; CHECK-NEXT:    [[NEGY:%.*]] = sub i16 0, [[Y:%.*]]
-; CHECK-NEXT:    [[YMASK:%.*]] = and i16 [[Y]], 15
 ; CHECK-NEXT:    [[NEGYMASK:%.*]] = and i16 [[NEGY]], 15
 ; CHECK-NEXT:    store i16 [[NEGYMASK]], i16* [[P:%.*]], align 2
-; CHECK-NEXT:    [[SHL:%.*]] = shl i16 [[X:%.*]], [[YMASK]]
-; CHECK-NEXT:    [[SHR:%.*]] = lshr i16 [[X]], [[NEGYMASK]]
-; CHECK-NEXT:    [[R:%.*]] = or i16 [[SHL]], [[SHR]]
+; CHECK-NEXT:    [[R:%.*]] = call i16 @llvm.fshl.i16(i16 [[X:%.*]], i16 [[X]], i16 [[Y]])
 ; CHECK-NEXT:    ret i16 [[R]]
 ;
   %negy = sub i16 0, %y
@@ -254,17 +244,11 @@ define i16 @rotl_safe_i16_commute_extra_use(i16 %x, i16 %y, i16* %p) {
   ret i16 %r
 }
 
-; TODO:
 ; Left/right is determined by the negation.
 
 define i64 @rotr_safe_i64(i64 %x, i64 %y) {
 ; CHECK-LABEL: @rotr_safe_i64(
-; CHECK-NEXT:    [[NEGY:%.*]] = sub i64 0, [[Y:%.*]]
-; CHECK-NEXT:    [[YMASK:%.*]] = and i64 [[Y]], 63
-; CHECK-NEXT:    [[NEGYMASK:%.*]] = and i64 [[NEGY]], 63
-; CHECK-NEXT:    [[SHL:%.*]] = shl i64 [[X:%.*]], [[NEGYMASK]]
-; CHECK-NEXT:    [[SHR:%.*]] = lshr i64 [[X]], [[YMASK]]
-; CHECK-NEXT:    [[R:%.*]] = or i64 [[SHR]], [[SHL]]
+; CHECK-NEXT:    [[R:%.*]] = call i64 @llvm.fshr.i64(i64 [[X:%.*]], i64 [[X]], i64 [[Y:%.*]])
 ; CHECK-NEXT:    ret i64 [[R]]
 ;
   %negy = sub i64 0, %y
@@ -276,7 +260,6 @@ define i64 @rotr_safe_i64(i64 %x, i64 %y) {
   ret i64 %r
 }
 
-; TODO:
 ; Extra uses don't change anything.
 
 define i8 @rotr_safe_i8_commute_extra_use(i8 %x, i8 %y, i8* %p) {
@@ -300,17 +283,11 @@ define i8 @rotr_safe_i8_commute_extra_use(i8 %x, i8 %y, i8* %p) {
   ret i8 %r
 }
 
-; TODO:
 ; Vectors follow the same rules.
 
 define <2 x i32> @rotl_safe_v2i32(<2 x i32> %x, <2 x i32> %y) {
 ; CHECK-LABEL: @rotl_safe_v2i32(
-; CHECK-NEXT:    [[NEGY:%.*]] = sub <2 x i32> zeroinitializer, [[Y:%.*]]
-; CHECK-NEXT:    [[YMASK:%.*]] = and <2 x i32> [[Y]], <i32 31, i32 31>
-; CHECK-NEXT:    [[NEGYMASK:%.*]] = and <2 x i32> [[NEGY]], <i32 31, i32 31>
-; CHECK-NEXT:    [[SHL:%.*]] = shl <2 x i32> [[X:%.*]], [[YMASK]]
-; CHECK-NEXT:    [[SHR:%.*]] = lshr <2 x i32> [[X]], [[NEGYMASK]]
-; CHECK-NEXT:    [[R:%.*]] = or <2 x i32> [[SHR]], [[SHL]]
+; CHECK-NEXT:    [[R:%.*]] = call <2 x i32> @llvm.fshl.v2i32(<2 x i32> [[X:%.*]], <2 x i32> [[X]], <2 x i32> [[Y:%.*]])
 ; CHECK-NEXT:    ret <2 x i32> [[R]]
 ;
   %negy = sub <2 x i32> zeroinitializer, %y
@@ -322,17 +299,11 @@ define <2 x i32> @rotl_safe_v2i32(<2 x i32> %x, <2 x i32> %y) {
   ret <2 x i32> %r
 }
 
-; TODO:
 ; Vectors follow the same rules.
 
 define <3 x i16> @rotr_safe_v3i16(<3 x i16> %x, <3 x i16> %y) {
 ; CHECK-LABEL: @rotr_safe_v3i16(
-; CHECK-NEXT:    [[NEGY:%.*]] = sub <3 x i16> zeroinitializer, [[Y:%.*]]
-; CHECK-NEXT:    [[YMASK:%.*]] = and <3 x i16> [[Y]], <i16 15, i16 15, i16 15>
-; CHECK-NEXT:    [[NEGYMASK:%.*]] = and <3 x i16> [[NEGY]], <i16 15, i16 15, i16 15>
-; CHECK-NEXT:    [[SHL:%.*]] = shl <3 x i16> [[X:%.*]], [[NEGYMASK]]
-; CHECK-NEXT:    [[SHR:%.*]] = lshr <3 x i16> [[X]], [[YMASK]]
-; CHECK-NEXT:    [[R:%.*]] = or <3 x i16> [[SHR]], [[SHL]]
+; CHECK-NEXT:    [[R:%.*]] = call <3 x i16> @llvm.fshr.v3i16(<3 x i16> [[X:%.*]], <3 x i16> [[X]], <3 x i16> [[Y:%.*]])
 ; CHECK-NEXT:    ret <3 x i16> [[R]]
 ;
   %negy = sub <3 x i16> zeroinitializer, %y
@@ -353,12 +324,7 @@ define <3 x i16> @rotr_safe_v3i16(<3 x i16> %x, <3 x i16> %y) {
 define i16 @rotate_left_16bit(i16 %v, i32 %shift) {
 ; CHECK-LABEL: @rotate_left_16bit(
 ; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 [[SHIFT:%.*]] to i16
-; CHECK-NEXT:    [[TMP2:%.*]] = and i16 [[TMP1]], 15
-; CHECK-NEXT:    [[TMP3:%.*]] = sub i16 0, [[TMP1]]
-; CHECK-NEXT:    [[TMP4:%.*]] = and i16 [[TMP3]], 15
-; CHECK-NEXT:    [[TMP5:%.*]] = lshr i16 [[V:%.*]], [[TMP4]]
-; CHECK-NEXT:    [[TMP6:%.*]] = shl i16 [[V]], [[TMP2]]
-; CHECK-NEXT:    [[CONV2:%.*]] = or i16 [[TMP5]], [[TMP6]]
+; CHECK-NEXT:    [[CONV2:%.*]] = call i16 @llvm.fshl.i16(i16 [[V:%.*]], i16 [[V]], i16 [[TMP1]])
 ; CHECK-NEXT:    ret i16 [[CONV2]]
 ;
   %and = and i32 %shift, 15
@@ -376,12 +342,7 @@ define i16 @rotate_left_16bit(i16 %v, i32 %shift) {
 define <2 x i16> @rotate_left_commute_16bit_vec(<2 x i16> %v, <2 x i32> %shift) {
 ; CHECK-LABEL: @rotate_left_commute_16bit_vec(
 ; CHECK-NEXT:    [[TMP1:%.*]] = trunc <2 x i32> [[SHIFT:%.*]] to <2 x i16>
-; CHECK-NEXT:    [[TMP2:%.*]] = and <2 x i16> [[TMP1]], <i16 15, i16 15>
-; CHECK-NEXT:    [[TMP3:%.*]] = sub <2 x i16> zeroinitializer, [[TMP1]]
-; CHECK-NEXT:    [[TMP4:%.*]] = and <2 x i16> [[TMP3]], <i16 15, i16 15>
-; CHECK-NEXT:    [[TMP5:%.*]] = shl <2 x i16> [[V:%.*]], [[TMP2]]
-; CHECK-NEXT:    [[TMP6:%.*]] = lshr <2 x i16> [[V]], [[TMP4]]
-; CHECK-NEXT:    [[CONV2:%.*]] = or <2 x i16> [[TMP5]], [[TMP6]]
+; CHECK-NEXT:    [[CONV2:%.*]] = call <2 x i16> @llvm.fshl.v2i16(<2 x i16> [[V:%.*]], <2 x i16> [[V]], <2 x i16> [[TMP1]])
 ; CHECK-NEXT:    ret <2 x i16> [[CONV2]]
 ;
   %and = and <2 x i32> %shift, <i32 15, i32 15>
@@ -399,11 +360,7 @@ define <2 x i16> @rotate_left_commute_16bit_vec(<2 x i16> %v, <2 x i32> %shift)
 define i8 @rotate_right_8bit(i8 %v, i3 %shift) {
 ; CHECK-LABEL: @rotate_right_8bit(
 ; CHECK-NEXT:    [[TMP1:%.*]] = zext i3 [[SHIFT:%.*]] to i8
-; CHECK-NEXT:    [[TMP2:%.*]] = sub i3 0, [[SHIFT]]
-; CHECK-NEXT:    [[TMP3:%.*]] = zext i3 [[TMP2]] to i8
-; CHECK-NEXT:    [[TMP4:%.*]] = shl i8 [[V:%.*]], [[TMP3]]
-; CHECK-NEXT:    [[TMP5:%.*]] = lshr i8 [[V]], [[TMP1]]
-; CHECK-NEXT:    [[CONV2:%.*]] = or i8 [[TMP4]], [[TMP5]]
+; CHECK-NEXT:    [[CONV2:%.*]] = call i8 @llvm.fshr.i8(i8 [[V:%.*]], i8 [[V]], i8 [[TMP1]])
 ; CHECK-NEXT:    ret i8 [[CONV2]]
 ;
   %and = zext i3 %shift to i32
@@ -423,12 +380,8 @@ define i8 @rotate_right_commute_8bit(i32 %v, i32 %shift) {
 ; CHECK-LABEL: @rotate_right_commute_8bit(
 ; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 [[SHIFT:%.*]] to i8
 ; CHECK-NEXT:    [[TMP2:%.*]] = and i8 [[TMP1]], 3
-; CHECK-NEXT:    [[TMP3:%.*]] = sub nsw i8 0, [[TMP2]]
-; CHECK-NEXT:    [[TMP4:%.*]] = and i8 [[TMP3]], 7
-; CHECK-NEXT:    [[TMP5:%.*]] = trunc i32 [[V:%.*]] to i8
-; CHECK-NEXT:    [[TMP6:%.*]] = lshr i8 [[TMP5]], [[TMP2]]
-; CHECK-NEXT:    [[TMP7:%.*]] = shl i8 [[TMP5]], [[TMP4]]
-; CHECK-NEXT:    [[CONV2:%.*]] = or i8 [[TMP6]], [[TMP7]]
+; CHECK-NEXT:    [[TMP3:%.*]] = trunc i32 [[V:%.*]] to i8
+; CHECK-NEXT:    [[CONV2:%.*]] = call i8 @llvm.fshr.i8(i8 [[TMP3]], i8 [[TMP3]], i8 [[TMP2]])
 ; CHECK-NEXT:    ret i8 [[CONV2]]
 ;
   %and = and i32 %shift, 3
@@ -447,12 +400,7 @@ define i8 @rotate_right_commute_8bit(i32 %v, i32 %shift) {
 define i8 @rotate8_not_safe(i8 %v, i32 %shamt) {
 ; CHECK-LABEL: @rotate8_not_safe(
 ; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 [[SHAMT:%.*]] to i8
-; CHECK-NEXT:    [[TMP2:%.*]] = sub i8 0, [[TMP1]]
-; CHECK-NEXT:    [[TMP3:%.*]] = and i8 [[TMP1]], 7
-; CHECK-NEXT:    [[TMP4:%.*]] = and i8 [[TMP2]], 7
-; CHECK-NEXT:    [[TMP5:%.*]] = lshr i8 [[V:%.*]], [[TMP4]]
-; CHECK-NEXT:    [[TMP6:%.*]] = shl i8 [[V]], [[TMP3]]
-; CHECK-NEXT:    [[RET:%.*]] = or i8 [[TMP5]], [[TMP6]]
+; CHECK-NEXT:    [[RET:%.*]] = call i8 @llvm.fshl.i8(i8 [[V:%.*]], i8 [[V]], i8 [[TMP1]])
 ; CHECK-NEXT:    ret i8 [[RET]]
 ;
   %conv = zext i8 %v to i32
@@ -490,12 +438,7 @@ define i9 @rotate9_not_safe(i9 %v, i32 %shamt) {
 
 define i16 @rotateleft_16_neg_mask(i16 %v, i16 %shamt) {
 ; CHECK-LABEL: @rotateleft_16_neg_mask(
-; CHECK-NEXT:    [[TMP1:%.*]] = sub i16 0, [[SHAMT:%.*]]
-; CHECK-NEXT:    [[TMP2:%.*]] = and i16 [[SHAMT]], 15
-; CHECK-NEXT:    [[TMP3:%.*]] = and i16 [[TMP1]], 15
-; CHECK-NEXT:    [[TMP4:%.*]] = lshr i16 [[V:%.*]], [[TMP3]]
-; CHECK-NEXT:    [[TMP5:%.*]] = shl i16 [[V]], [[TMP2]]
-; CHECK-NEXT:    [[RET:%.*]] = or i16 [[TMP4]], [[TMP5]]
+; CHECK-NEXT:    [[RET:%.*]] = call i16 @llvm.fshl.i16(i16 [[V:%.*]], i16 [[V]], i16 [[SHAMT:%.*]])
 ; CHECK-NEXT:    ret i16 [[RET]]
 ;
   %neg = sub i16 0, %shamt
@@ -513,12 +456,7 @@ define i16 @rotateleft_16_neg_mask(i16 %v, i16 %shamt) {
 
 define i16 @rotateleft_16_neg_mask_commute(i16 %v, i16 %shamt) {
 ; CHECK-LABEL: @rotateleft_16_neg_mask_commute(
-; CHECK-NEXT:    [[TMP1:%.*]] = sub i16 0, [[SHAMT:%.*]]
-; CHECK-NEXT:    [[TMP2:%.*]] = and i16 [[SHAMT]], 15
-; CHECK-NEXT:    [[TMP3:%.*]] = and i16 [[TMP1]], 15
-; CHECK-NEXT:    [[TMP4:%.*]] = shl i16 [[V:%.*]], [[TMP2]]
-; CHECK-NEXT:    [[TMP5:%.*]] = lshr i16 [[V]], [[TMP3]]
-; CHECK-NEXT:    [[RET:%.*]] = or i16 [[TMP4]], [[TMP5]]
+; CHECK-NEXT:    [[RET:%.*]] = call i16 @llvm.fshl.i16(i16 [[V:%.*]], i16 [[V]], i16 [[SHAMT:%.*]])
 ; CHECK-NEXT:    ret i16 [[RET]]
 ;
   %neg = sub i16 0, %shamt
@@ -536,12 +474,7 @@ define i16 @rotateleft_16_neg_mask_commute(i16 %v, i16 %shamt) {
 
 define i8 @rotateright_8_neg_mask(i8 %v, i8 %shamt) {
 ; CHECK-LABEL: @rotateright_8_neg_mask(
-; CHECK-NEXT:    [[TMP1:%.*]] = sub i8 0, [[SHAMT:%.*]]
-; CHECK-NEXT:    [[TMP2:%.*]] = and i8 [[SHAMT]], 7
-; CHECK-NEXT:    [[TMP3:%.*]] = and i8 [[TMP1]], 7
-; CHECK-NEXT:    [[TMP4:%.*]] = lshr i8 [[V:%.*]], [[TMP2]]
-; CHECK-NEXT:    [[TMP5:%.*]] = shl i8 [[V]], [[TMP3]]
-; CHECK-NEXT:    [[RET:%.*]] = or i8 [[TMP4]], [[TMP5]]
+; CHECK-NEXT:    [[RET:%.*]] = call i8 @llvm.fshr.i8(i8 [[V:%.*]], i8 [[V]], i8 [[SHAMT:%.*]])
 ; CHECK-NEXT:    ret i8 [[RET]]
 ;
   %neg = sub i8 0, %shamt
@@ -559,12 +492,7 @@ define i8 @rotateright_8_neg_mask(i8 %v, i8 %shamt) {
 
 define i8 @rotateright_8_neg_mask_commute(i8 %v, i8 %shamt) {
 ; CHECK-LABEL: @rotateright_8_neg_mask_commute(
-; CHECK-NEXT:    [[TMP1:%.*]] = sub i8 0, [[SHAMT:%.*]]
-; CHECK-NEXT:    [[TMP2:%.*]] = and i8 [[SHAMT]], 7
-; CHECK-NEXT:    [[TMP3:%.*]] = and i8 [[TMP1]], 7
-; CHECK-NEXT:    [[TMP4:%.*]] = shl i8 [[V:%.*]], [[TMP3]]
-; CHECK-NEXT:    [[TMP5:%.*]] = lshr i8 [[V]], [[TMP2]]
-; CHECK-NEXT:    [[RET:%.*]] = or i8 [[TMP4]], [[TMP5]]
+; CHECK-NEXT:    [[RET:%.*]] = call i8 @llvm.fshr.i8(i8 [[V:%.*]], i8 [[V]], i8 [[SHAMT:%.*]])
 ; CHECK-NEXT:    ret i8 [[RET]]
 ;
   %neg = sub i8 0, %shamt
@@ -586,12 +514,7 @@ define i8 @rotateright_8_neg_mask_commute(i8 %v, i8 %shamt) {
 define i16 @rotateright_16_neg_mask_wide_amount(i16 %v, i32 %shamt) {
 ; CHECK-LABEL: @rotateright_16_neg_mask_wide_amount(
 ; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 [[SHAMT:%.*]] to i16
-; CHECK-NEXT:    [[TMP2:%.*]] = sub i16 0, [[TMP1]]
-; CHECK-NEXT:    [[TMP3:%.*]] = and i16 [[TMP1]], 15
-; CHECK-NEXT:    [[TMP4:%.*]] = and i16 [[TMP2]], 15
-; CHECK-NEXT:    [[TMP5:%.*]] = lshr i16 [[V:%.*]], [[TMP3]]
-; CHECK-NEXT:    [[TMP6:%.*]] = shl i16 [[V]], [[TMP4]]
-; CHECK-NEXT:    [[RET:%.*]] = or i16 [[TMP5]], [[TMP6]]
+; CHECK-NEXT:    [[RET:%.*]] = call i16 @llvm.fshr.i16(i16 [[V:%.*]], i16 [[V]], i16 [[TMP1]])
 ; CHECK-NEXT:    ret i16 [[RET]]
 ;
   %neg = sub i32 0, %shamt
@@ -608,12 +531,7 @@ define i16 @rotateright_16_neg_mask_wide_amount(i16 %v, i32 %shamt) {
 define i16 @rotateright_16_neg_mask_wide_amount_commute(i16 %v, i32 %shamt) {
 ; CHECK-LABEL: @rotateright_16_neg_mask_wide_amount_commute(
 ; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 [[SHAMT:%.*]] to i16
-; CHECK-NEXT:    [[TMP2:%.*]] = sub i16 0, [[TMP1]]
-; CHECK-NEXT:    [[TMP3:%.*]] = and i16 [[TMP1]], 15
-; CHECK-NEXT:    [[TMP4:%.*]] = and i16 [[TMP2]], 15
-; CHECK-NEXT:    [[TMP5:%.*]] = shl i16 [[V:%.*]], [[TMP4]]
-; CHECK-NEXT:    [[TMP6:%.*]] = lshr i16 [[V]], [[TMP3]]
-; CHECK-NEXT:    [[RET:%.*]] = or i16 [[TMP5]], [[TMP6]]
+; CHECK-NEXT:    [[RET:%.*]] = call i16 @llvm.fshr.i16(i16 [[V:%.*]], i16 [[V]], i16 [[TMP1]])
 ; CHECK-NEXT:    ret i16 [[RET]]
 ;
   %neg = sub i32 0, %shamt
@@ -630,12 +548,7 @@ define i16 @rotateright_16_neg_mask_wide_amount_commute(i16 %v, i32 %shamt) {
 define i8 @rotateleft_8_neg_mask_wide_amount(i8 %v, i32 %shamt) {
 ; CHECK-LABEL: @rotateleft_8_neg_mask_wide_amount(
 ; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 [[SHAMT:%.*]] to i8
-; CHECK-NEXT:    [[TMP2:%.*]] = sub i8 0, [[TMP1]]
-; CHECK-NEXT:    [[TMP3:%.*]] = and i8 [[TMP1]], 7
-; CHECK-NEXT:    [[TMP4:%.*]] = and i8 [[TMP2]], 7
-; CHECK-NEXT:    [[TMP5:%.*]] = lshr i8 [[V:%.*]], [[TMP4]]
-; CHECK-NEXT:    [[TMP6:%.*]] = shl i8 [[V]], [[TMP3]]
-; CHECK-NEXT:    [[RET:%.*]] = or i8 [[TMP5]], [[TMP6]]
+; CHECK-NEXT:    [[RET:%.*]] = call i8 @llvm.fshl.i8(i8 [[V:%.*]], i8 [[V]], i8 [[TMP1]])
 ; CHECK-NEXT:    ret i8 [[RET]]
 ;
   %neg = sub i32 0, %shamt
@@ -652,12 +565,7 @@ define i8 @rotateleft_8_neg_mask_wide_amount(i8 %v, i32 %shamt) {
 define i8 @rotateleft_8_neg_mask_wide_amount_commute(i8 %v, i32 %shamt) {
 ; CHECK-LABEL: @rotateleft_8_neg_mask_wide_amount_commute(
 ; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 [[SHAMT:%.*]] to i8
-; CHECK-NEXT:    [[TMP2:%.*]] = sub i8 0, [[TMP1]]
-; CHECK-NEXT:    [[TMP3:%.*]] = and i8 [[TMP1]], 7
-; CHECK-NEXT:    [[TMP4:%.*]] = and i8 [[TMP2]], 7
-; CHECK-NEXT:    [[TMP5:%.*]] = shl i8 [[V:%.*]], [[TMP3]]
-; CHECK-NEXT:    [[TMP6:%.*]] = lshr i8 [[V]], [[TMP4]]
-; CHECK-NEXT:    [[RET:%.*]] = or i8 [[TMP5]], [[TMP6]]
+; CHECK-NEXT:    [[RET:%.*]] = call i8 @llvm.fshl.i8(i8 [[V:%.*]], i8 [[V]], i8 [[TMP1]])
 ; CHECK-NEXT:    ret i8 [[RET]]
 ;
   %neg = sub i32 0, %shamt
@@ -700,12 +608,7 @@ define i9 @rotateleft_9_neg_mask_wide_amount_commute(i9 %v, i33 %shamt) {
 
 define i32 @rotr_select(i32 %x, i32 %shamt) {
 ; CHECK-LABEL: @rotr_select(
-; CHECK-NEXT:    [[TMP1:%.*]] = sub i32 0, [[SHAMT:%.*]]
-; CHECK-NEXT:    [[TMP2:%.*]] = and i32 [[SHAMT]], 31
-; CHECK-NEXT:    [[TMP3:%.*]] = and i32 [[TMP1]], 31
-; CHECK-NEXT:    [[TMP4:%.*]] = lshr i32 [[X:%.*]], [[TMP2]]
-; CHECK-NEXT:    [[TMP5:%.*]] = shl i32 [[X]], [[TMP3]]
-; CHECK-NEXT:    [[R:%.*]] = or i32 [[TMP4]], [[TMP5]]
+; CHECK-NEXT:    [[R:%.*]] = call i32 @llvm.fshr.i32(i32 [[X:%.*]], i32 [[X]], i32 [[SHAMT:%.*]])
 ; CHECK-NEXT:    ret i32 [[R]]
 ;
   %cmp = icmp eq i32 %shamt, 0
@@ -721,12 +624,7 @@ define i32 @rotr_select(i32 %x, i32 %shamt) {
 
 define i8 @rotr_select_commute(i8 %x, i8 %shamt) {
 ; CHECK-LABEL: @rotr_select_commute(
-; CHECK-NEXT:    [[TMP1:%.*]] = sub i8 0, [[SHAMT:%.*]]
-; CHECK-NEXT:    [[TMP2:%.*]] = and i8 [[SHAMT]], 7
-; CHECK-NEXT:    [[TMP3:%.*]] = and i8 [[TMP1]], 7
-; CHECK-NEXT:    [[TMP4:%.*]] = shl i8 [[X:%.*]], [[TMP3]]
-; CHECK-NEXT:    [[TMP5:%.*]] = lshr i8 [[X]], [[TMP2]]
-; CHECK-NEXT:    [[R:%.*]] = or i8 [[TMP4]], [[TMP5]]
+; CHECK-NEXT:    [[R:%.*]] = call i8 @llvm.fshr.i8(i8 [[X:%.*]], i8 [[X]], i8 [[SHAMT:%.*]])
 ; CHECK-NEXT:    ret i8 [[R]]
 ;
   %cmp = icmp eq i8 %shamt, 0
@@ -742,12 +640,7 @@ define i8 @rotr_select_commute(i8 %x, i8 %shamt) {
 
 define i16 @rotl_select(i16 %x, i16 %shamt) {
 ; CHECK-LABEL: @rotl_select(
-; CHECK-NEXT:    [[TMP1:%.*]] = sub i16 0, [[SHAMT:%.*]]
-; CHECK-NEXT:    [[TMP2:%.*]] = and i16 [[SHAMT]], 15
-; CHECK-NEXT:    [[TMP3:%.*]] = and i16 [[TMP1]], 15
-; CHECK-NEXT:    [[TMP4:%.*]] = lshr i16 [[X:%.*]], [[TMP3]]
-; CHECK-NEXT:    [[TMP5:%.*]] = shl i16 [[X]], [[TMP2]]
-; CHECK-NEXT:    [[R:%.*]] = or i16 [[TMP4]], [[TMP5]]
+; CHECK-NEXT:    [[R:%.*]] = call i16 @llvm.fshl.i16(i16 [[X:%.*]], i16 [[X]], i16 [[SHAMT:%.*]])
 ; CHECK-NEXT:    ret i16 [[R]]
 ;
   %cmp = icmp eq i16 %shamt, 0
@@ -763,12 +656,7 @@ define i16 @rotl_select(i16 %x, i16 %shamt) {
 
 define <2 x i64> @rotl_select_commute(<2 x i64> %x, <2 x i64> %shamt) {
 ; CHECK-LABEL: @rotl_select_commute(
-; CHECK-NEXT:    [[TMP1:%.*]] = sub <2 x i64> zeroinitializer, [[SHAMT:%.*]]
-; CHECK-NEXT:    [[TMP2:%.*]] = and <2 x i64> [[SHAMT]], <i64 63, i64 63>
-; CHECK-NEXT:    [[TMP3:%.*]] = and <2 x i64> [[TMP1]], <i64 63, i64 63>
-; CHECK-NEXT:    [[TMP4:%.*]] = shl <2 x i64> [[X:%.*]], [[TMP2]]
-; CHECK-NEXT:    [[TMP5:%.*]] = lshr <2 x i64> [[X]], [[TMP3]]
-; CHECK-NEXT:    [[R:%.*]] = or <2 x i64> [[TMP4]], [[TMP5]]
+; CHECK-NEXT:    [[R:%.*]] = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> [[X:%.*]], <2 x i64> [[X]], <2 x i64> [[SHAMT:%.*]])
 ; CHECK-NEXT:    ret <2 x i64> [[R]]
 ;
   %cmp = icmp eq <2 x i64> %shamt, zeroinitializer
@@ -801,3 +689,17 @@ define i24 @rotl_select_weird_type(i24 %x, i24 %shamt) {
   ret i24 %r
 }
 
+; Test that the transform doesn't crash when there's an "or" with a ConstantExpr operand.
+
+@external_global = external global i8
+
+define i32 @rotl_constant_expr(i32 %shamt) {
+; CHECK-LABEL: @rotl_constant_expr(
+; CHECK-NEXT:    [[SHR:%.*]] = lshr i32 ptrtoint (i8* @external_global to i32), [[SHAMT:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = or i32 [[SHR]], shl (i32 ptrtoint (i8* @external_global to i32), i32 11)
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %shr = lshr i32 ptrtoint (i8* @external_global to i32), %shamt
+  %r = or i32 %shr, shl (i32 ptrtoint (i8* @external_global to i32), i32 11)
+  ret i32 %r
+}
diff --git a/test/Transforms/InstCombine/saturating-add-sub.ll b/test/Transforms/InstCombine/saturating-add-sub.ll
index 51f2670ec1229d130966075dd117415e98367910..f8eb7917618811005675d60785125280aa7a6dab 100644
--- a/test/Transforms/InstCombine/saturating-add-sub.ll
+++ b/test/Transforms/InstCombine/saturating-add-sub.ll
@@ -636,3 +636,71 @@ define <2 x i8> @test_vector_ssub_neg_nneg(<2 x i8> %a) {
   %r = call <2 x i8> @llvm.ssub.sat.v2i8(<2 x i8> %a_neg, <2 x i8> <i8 10, i8 20>)
   ret <2 x i8> %r
 }
+
+; Raw IR tests
+
+define i32 @uadd_sat_constant(i32 %x) {
+; CHECK-LABEL: @uadd_sat_constant(
+; CHECK-NEXT:    [[A:%.*]] = add i32 [[X:%.*]], 42
+; CHECK-NEXT:    [[C:%.*]] = icmp ugt i32 [[X]], -43
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[C]], i32 -1, i32 [[A]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %a = add i32 %x, 42
+  %c = icmp ugt i32 %x, -43
+  %r = select i1 %c, i32 -1, i32 %a
+  ret i32 %r
+}
+
+define i32 @uadd_sat_constant_commute(i32 %x) {
+; CHECK-LABEL: @uadd_sat_constant_commute(
+; CHECK-NEXT:    [[A:%.*]] = add i32 [[X:%.*]], 42
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ugt i32 [[X]], -43
+; CHECK-NEXT:    [[TMP2:%.*]] = select i1 [[TMP1]], i32 -1, i32 [[A]]
+; CHECK-NEXT:    ret i32 [[TMP2]]
+;
+  %a = add i32 %x, 42
+  %c = icmp ult i32 %x, -43
+  %r = select i1 %c, i32 %a, i32 -1
+  ret i32 %r
+}
+
+define <4 x i32> @uadd_sat_constant_vec(<4 x i32> %x) {
+; CHECK-LABEL: @uadd_sat_constant_vec(
+; CHECK-NEXT:    [[A:%.*]] = add <4 x i32> [[X:%.*]], <i32 42, i32 42, i32 42, i32 42>
+; CHECK-NEXT:    [[C:%.*]] = icmp ugt <4 x i32> [[X]], <i32 -43, i32 -43, i32 -43, i32 -43>
+; CHECK-NEXT:    [[R:%.*]] = select <4 x i1> [[C]], <4 x i32> <i32 -1, i32 -1, i32 -1, i32 -1>, <4 x i32> [[A]]
+; CHECK-NEXT:    ret <4 x i32> [[R]]
+;
+  %a = add <4 x i32> %x, <i32 42, i32 42, i32 42, i32 42>
+  %c = icmp ugt <4 x i32> %x, <i32 -43, i32 -43, i32 -43, i32 -43>
+  %r = select <4 x i1> %c, <4 x i32> <i32 -1, i32 -1, i32 -1, i32 -1>, <4 x i32> %a
+  ret <4 x i32> %r
+}
+
+define <4 x i32> @uadd_sat_constant_vec_commute(<4 x i32> %x) {
+; CHECK-LABEL: @uadd_sat_constant_vec_commute(
+; CHECK-NEXT:    [[A:%.*]] = add <4 x i32> [[X:%.*]], <i32 42, i32 42, i32 42, i32 42>
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ugt <4 x i32> [[X]], <i32 -43, i32 -43, i32 -43, i32 -43>
+; CHECK-NEXT:    [[TMP2:%.*]] = select <4 x i1> [[TMP1]], <4 x i32> <i32 -1, i32 -1, i32 -1, i32 -1>, <4 x i32> [[A]]
+; CHECK-NEXT:    ret <4 x i32> [[TMP2]]
+;
+  %a = add <4 x i32> %x, <i32 42, i32 42, i32 42, i32 42>
+  %c = icmp ult <4 x i32> %x, <i32 -43, i32 -43, i32 -43, i32 -43>
+  %r = select <4 x i1> %c, <4 x i32> %a, <4 x i32> <i32 -1, i32 -1, i32 -1, i32 -1>
+  ret <4 x i32> %r
+}
+
+define <4 x i32> @uadd_sat_constant_vec_commute_undefs(<4 x i32> %x) {
+; CHECK-LABEL: @uadd_sat_constant_vec_commute_undefs(
+; CHECK-NEXT:    [[A:%.*]] = add <4 x i32> [[X:%.*]], <i32 42, i32 42, i32 42, i32 undef>
+; CHECK-NEXT:    [[C:%.*]] = icmp ult <4 x i32> [[X]], <i32 -43, i32 -43, i32 undef, i32 -43>
+; CHECK-NEXT:    [[R:%.*]] = select <4 x i1> [[C]], <4 x i32> [[A]], <4 x i32> <i32 -1, i32 undef, i32 -1, i32 -1>
+; CHECK-NEXT:    ret <4 x i32> [[R]]
+;
+  %a = add <4 x i32> %x, <i32 42, i32 42, i32 42, i32 undef>
+  %c = icmp ult <4 x i32> %x, <i32 -43, i32 -43, i32 undef, i32 -43>
+  %r = select <4 x i1> %c, <4 x i32> %a, <4 x i32> <i32 -1, i32 undef, i32 -1, i32 -1>
+  ret <4 x i32> %r
+}
+
diff --git a/test/Transforms/InstCombine/select-cmp-cttz-ctlz.ll b/test/Transforms/InstCombine/select-cmp-cttz-ctlz.ll
index 6ccc6155e4c8661ccf6b725d5e26d8259021f3ab..606cdedca7a71fc5ee80ee7f1b1d7162a6652439 100644
--- a/test/Transforms/InstCombine/select-cmp-cttz-ctlz.ll
+++ b/test/Transforms/InstCombine/select-cmp-cttz-ctlz.ll
@@ -345,7 +345,7 @@ define i128 @test8(i128 %x) {
 
 define i32 @test_ctlz_not_bw(i32 %x) {
 ; CHECK-LABEL: @test_ctlz_not_bw(
-; CHECK-NEXT:    [[CT:%.*]] = tail call i32 @llvm.ctlz.i32(i32 [[X:%.*]], i1 false), !range !1
+; CHECK-NEXT:    [[CT:%.*]] = tail call i32 @llvm.ctlz.i32(i32 [[X:%.*]], i1 true), !range !1
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[X]], 0
 ; CHECK-NEXT:    [[RES:%.*]] = select i1 [[CMP]], i32 123, i32 [[CT]]
 ; CHECK-NEXT:    ret i32 [[RES]]
@@ -373,7 +373,7 @@ define i32 @test_ctlz_not_bw_multiuse(i32 %x) {
 
 define i32 @test_cttz_not_bw(i32 %x) {
 ; CHECK-LABEL: @test_cttz_not_bw(
-; CHECK-NEXT:    [[CT:%.*]] = tail call i32 @llvm.cttz.i32(i32 [[X:%.*]], i1 false), !range !1
+; CHECK-NEXT:    [[CT:%.*]] = tail call i32 @llvm.cttz.i32(i32 [[X:%.*]], i1 true), !range !1
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[X]], 0
 ; CHECK-NEXT:    [[RES:%.*]] = select i1 [[CMP]], i32 123, i32 [[CT]]
 ; CHECK-NEXT:    ret i32 [[RES]]
@@ -399,11 +399,61 @@ define i32 @test_cttz_not_bw_multiuse(i32 %x) {
   ret i32 %res
 }
 
+define <2 x i32> @test_ctlz_bw_vec(<2 x i32> %x) {
+; CHECK-LABEL: @test_ctlz_bw_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <2 x i32> @llvm.ctlz.v2i32(<2 x i32> [[X:%.*]], i1 false)
+; CHECK-NEXT:    ret <2 x i32> [[TMP1]]
+;
+  %ct = tail call <2 x i32> @llvm.ctlz.v2i32(<2 x i32> %x, i1 true)
+  %cmp = icmp ne <2 x i32> %x, zeroinitializer
+  %res = select <2 x i1> %cmp, <2 x i32> %ct, <2 x i32> <i32 32, i32 32>
+  ret <2 x i32> %res
+}
+
+define <2 x i32> @test_ctlz_not_bw_vec(<2 x i32> %x) {
+; CHECK-LABEL: @test_ctlz_not_bw_vec(
+; CHECK-NEXT:    [[CT:%.*]] = tail call <2 x i32> @llvm.ctlz.v2i32(<2 x i32> [[X:%.*]], i1 true)
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <2 x i32> [[X]], zeroinitializer
+; CHECK-NEXT:    [[RES:%.*]] = select <2 x i1> [[CMP]], <2 x i32> zeroinitializer, <2 x i32> [[CT]]
+; CHECK-NEXT:    ret <2 x i32> [[RES]]
+;
+  %ct = tail call <2 x i32> @llvm.ctlz.v2i32(<2 x i32> %x, i1 false)
+  %cmp = icmp ne <2 x i32> %x, zeroinitializer
+  %res = select <2 x i1> %cmp, <2 x i32> %ct, <2 x i32> <i32 0, i32 0>
+  ret <2 x i32> %res
+}
+
+define <2 x i32> @test_cttz_bw_vec(<2 x i32> %x) {
+; CHECK-LABEL: @test_cttz_bw_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call <2 x i32> @llvm.cttz.v2i32(<2 x i32> [[X:%.*]], i1 false)
+; CHECK-NEXT:    ret <2 x i32> [[TMP1]]
+;
+  %ct = tail call <2 x i32> @llvm.cttz.v2i32(<2 x i32> %x, i1 true)
+  %cmp = icmp ne <2 x i32> %x, zeroinitializer
+  %res = select <2 x i1> %cmp, <2 x i32> %ct, <2 x i32> <i32 32, i32 32>
+  ret <2 x i32> %res
+}
+
+define <2 x i32> @test_cttz_not_bw_vec(<2 x i32> %x) {
+; CHECK-LABEL: @test_cttz_not_bw_vec(
+; CHECK-NEXT:    [[CT:%.*]] = tail call <2 x i32> @llvm.cttz.v2i32(<2 x i32> [[X:%.*]], i1 true)
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <2 x i32> [[X]], zeroinitializer
+; CHECK-NEXT:    [[RES:%.*]] = select <2 x i1> [[CMP]], <2 x i32> zeroinitializer, <2 x i32> [[CT]]
+; CHECK-NEXT:    ret <2 x i32> [[RES]]
+;
+  %ct = tail call <2 x i32> @llvm.cttz.v2i32(<2 x i32> %x, i1 false)
+  %cmp = icmp ne <2 x i32> %x, zeroinitializer
+  %res = select <2 x i1> %cmp, <2 x i32> %ct, <2 x i32> <i32 0, i32 0>
+  ret <2 x i32> %res
+}
+
 declare i16 @llvm.ctlz.i16(i16, i1)
 declare i32 @llvm.ctlz.i32(i32, i1)
 declare i64 @llvm.ctlz.i64(i64, i1)
 declare i128 @llvm.ctlz.i128(i128, i1)
+declare <2 x i32> @llvm.ctlz.v2i32(<2 x i32>, i1)
 declare i16 @llvm.cttz.i16(i16, i1)
 declare i32 @llvm.cttz.i32(i32, i1)
 declare i64 @llvm.cttz.i64(i64, i1)
 declare i128 @llvm.cttz.i128(i128, i1)
+declare <2 x i32> @llvm.cttz.v2i32(<2 x i32>, i1)
diff --git a/test/Transforms/InstCombine/select.ll b/test/Transforms/InstCombine/select.ll
index 5ea0f34e39949b0c8c8b9f10f146720916405e15..c601a98f062dade86486d7f2af166ecc3b6edf89 100644
--- a/test/Transforms/InstCombine/select.ll
+++ b/test/Transforms/InstCombine/select.ll
@@ -1141,7 +1141,7 @@ define i32 @test82(i1 %flag) {
 ; CHECK-NEXT:    [[Y:%.*]] = alloca i32, align 4
 ; CHECK-NEXT:    [[X1:%.*]] = bitcast float* [[X]] to i32*
 ; CHECK-NEXT:    [[Y1:%.*]] = bitcast i32* [[Y]] to float*
-; CHECK-NEXT:    call void @scribble_on_i32(i32* [[X1]])
+; CHECK-NEXT:    call void @scribble_on_i32(i32* nonnull [[X1]])
 ; CHECK-NEXT:    call void @scribble_on_i32(i32* nonnull [[Y]])
 ; CHECK-NEXT:    [[TMP:%.*]] = load float, float* [[X]], align 4
 ; CHECK-NEXT:    store float [[TMP]], float* [[Y1]], align 4
@@ -1172,8 +1172,8 @@ define i8* @test83(i1 %flag) {
 ; CHECK-NEXT:    [[Y:%.*]] = alloca i8*, align 8
 ; CHECK-NEXT:    [[TMPCAST:%.*]] = bitcast i8** [[Y]] to i64*
 ; CHECK-NEXT:    [[X1:%.*]] = bitcast i8** [[X]] to i64*
-; CHECK-NEXT:    call void @scribble_on_i64(i64* [[X1]])
-; CHECK-NEXT:    call void @scribble_on_i64(i64* [[TMPCAST]])
+; CHECK-NEXT:    call void @scribble_on_i64(i64* nonnull [[X1]])
+; CHECK-NEXT:    call void @scribble_on_i64(i64* nonnull [[TMPCAST]])
 ; CHECK-NEXT:    [[TMP:%.*]] = load i64, i64* [[X1]], align 8
 ; CHECK-NEXT:    store i64 [[TMP]], i64* [[TMPCAST]], align 8
 ; CHECK-NEXT:    [[V:%.*]] = inttoptr i64 [[TMP]] to i8*
@@ -1200,8 +1200,8 @@ define i64 @test84(i1 %flag) {
 ; CHECK-NEXT:    [[Y:%.*]] = alloca i8*, align 8
 ; CHECK-NEXT:    [[TMPCAST:%.*]] = bitcast i8** [[Y]] to i64*
 ; CHECK-NEXT:    [[X1:%.*]] = bitcast i8** [[X]] to i64*
-; CHECK-NEXT:    call void @scribble_on_i64(i64* [[X1]])
-; CHECK-NEXT:    call void @scribble_on_i64(i64* [[TMPCAST]])
+; CHECK-NEXT:    call void @scribble_on_i64(i64* nonnull [[X1]])
+; CHECK-NEXT:    call void @scribble_on_i64(i64* nonnull [[TMPCAST]])
 ; CHECK-NEXT:    [[TMP:%.*]] = load i8*, i8** [[X]], align 8
 ; CHECK-NEXT:    store i8* [[TMP]], i8** [[Y]], align 8
 ; CHECK-NEXT:    [[V:%.*]] = ptrtoint i8* [[TMP]] to i64
@@ -1230,7 +1230,7 @@ define i8* @test85(i1 %flag) {
 ; CHECK-NEXT:    [[X1_SUB:%.*]] = getelementptr inbounds [2 x i8*], [2 x i8*]* [[X1]], i64 0, i64 0
 ; CHECK-NEXT:    [[X2:%.*]] = bitcast [2 x i8*]* [[X1]] to i128*
 ; CHECK-NEXT:    [[Y1:%.*]] = bitcast i128* [[Y]] to i8**
-; CHECK-NEXT:    call void @scribble_on_i128(i128* [[X2]])
+; CHECK-NEXT:    call void @scribble_on_i128(i128* nonnull [[X2]])
 ; CHECK-NEXT:    call void @scribble_on_i128(i128* nonnull [[Y]])
 ; CHECK-NEXT:    [[TMP:%.*]] = load i128, i128* [[X2]], align 8
 ; CHECK-NEXT:    store i128 [[TMP]], i128* [[Y]], align 8
@@ -1263,7 +1263,7 @@ define i128 @test86(i1 %flag) {
 ; CHECK-NEXT:    [[X1_SUB:%.*]] = getelementptr inbounds [2 x i8*], [2 x i8*]* [[X1]], i64 0, i64 0
 ; CHECK-NEXT:    [[X2:%.*]] = bitcast [2 x i8*]* [[X1]] to i128*
 ; CHECK-NEXT:    [[Y1:%.*]] = bitcast i128* [[Y]] to i8**
-; CHECK-NEXT:    call void @scribble_on_i128(i128* [[X2]])
+; CHECK-NEXT:    call void @scribble_on_i128(i128* nonnull [[X2]])
 ; CHECK-NEXT:    call void @scribble_on_i128(i128* nonnull [[Y]])
 ; CHECK-NEXT:    [[TMP:%.*]] = load i8*, i8** [[X1_SUB]], align 8
 ; CHECK-NEXT:    store i8* [[TMP]], i8** [[Y1]], align 8
diff --git a/test/Transforms/InstCombine/sink-alloca.ll b/test/Transforms/InstCombine/sink-alloca.ll
new file mode 100644
index 0000000000000000000000000000000000000000..f2de74ff533ba218c61b390de4090cb86beb50fa
--- /dev/null
+++ b/test/Transforms/InstCombine/sink-alloca.ll
@@ -0,0 +1,52 @@
+; RUN: opt -instcombine -S < %s | FileCheck %s
+
+target datalayout = "e-m:e-p:32:32-f64:32:64-f80:32-n8:16:32-S128"
+target triple = "i686-unknown-linux-gnu"
+
+; Check that instcombine doesn't sink dynamic allocas across llvm.stacksave.
+
+; Helper to generate branch conditions.
+declare i1 @cond()
+
+declare i32* @use_and_return(i32*)
+
+declare i8* @llvm.stacksave() #0
+
+declare void @llvm.stackrestore(i8*) #0
+
+define void @foo(i32 %x) {
+entry:
+  %c1 = call i1 @cond()
+  br i1 %c1, label %ret, label %nonentry
+
+nonentry:                                         ; preds = %entry
+  %argmem = alloca i32, i32 %x, align 4
+  %sp = call i8* @llvm.stacksave()
+  %c2 = call i1 @cond()
+  br i1 %c2, label %ret, label %sinktarget
+
+sinktarget:                                       ; preds = %nonentry
+  ; Arrange for there to be a single use of %argmem by returning it.
+  %p = call i32* @use_and_return(i32* nonnull %argmem)
+  store i32 13, i32* %p, align 4
+  call void @llvm.stackrestore(i8* %sp)
+  %0 = call i32* @use_and_return(i32* %p)
+  br label %ret
+
+ret:                                              ; preds = %sinktarget, %nonentry, %entry
+  ret void
+}
+
+; CHECK-LABEL: define void @foo(i32 %x)
+; CHECK: nonentry:
+; CHECK:   %argmem = alloca i32, i32 %x
+; CHECK:   %sp = call i8* @llvm.stacksave()
+; CHECK:   %c2 = call i1 @cond()
+; CHECK:   br i1 %c2, label %ret, label %sinktarget
+; CHECK: sinktarget:
+; CHECK:   %p = call i32* @use_and_return(i32* nonnull %argmem)
+; CHECK:   store i32 13, i32* %p
+; CHECK:   call void @llvm.stackrestore(i8* %sp)
+; CHECK:   %0 = call i32* @use_and_return(i32* %p)
+
+attributes #0 = { nounwind }
diff --git a/test/Transforms/InstCombine/stpcpy_chk-1.ll b/test/Transforms/InstCombine/stpcpy_chk-1.ll
index aae0d48e41b1b3268fc305725393ed5e0fd56ee1..b2e0416ae3dd9a725e538f2ee15976dd625c867d 100644
--- a/test/Transforms/InstCombine/stpcpy_chk-1.ll
+++ b/test/Transforms/InstCombine/stpcpy_chk-1.ll
@@ -64,10 +64,10 @@ define i8* @test_simplify5() {
   %dst = getelementptr inbounds [60 x i8], [60 x i8]* @a, i32 0, i32 0
   %src = getelementptr inbounds [12 x i8], [12 x i8]* @.str, i32 0, i32 0
 
-; CHECK-NEXT: %len = call i32 @llvm.objectsize.i32.p0i8(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0), i1 false, i1 false)
+; CHECK-NEXT: %len = call i32 @llvm.objectsize.i32.p0i8(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0), i1 false, i1 false, i1 false)
 ; CHECK-NEXT: %1 = call i8* @__memcpy_chk(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0), i8* getelementptr inbounds ([12 x i8], [12 x i8]* @.str, i32 0, i32 0), i32 12, i32 %len)
 ; CHECK-NEXT: ret i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 11)
-  %len = call i32 @llvm.objectsize.i32.p0i8(i8* %dst, i1 false, i1 false)
+  %len = call i32 @llvm.objectsize.i32.p0i8(i8* %dst, i1 false, i1 false, i1 false)
   %ret = call i8* @__stpcpy_chk(i8* %dst, i8* %src, i32 %len)
   ret i8* %ret
 }
@@ -81,7 +81,7 @@ define i8* @test_simplify6() {
 ; CHECK-NEXT: %strlen = call i32 @strlen(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0))
 ; CHECK-NEXT: %1 = getelementptr inbounds [60 x i8], [60 x i8]* @a, i32 0, i32 %strlen
 ; CHECK-NEXT: ret i8* %1
-  %len = call i32 @llvm.objectsize.i32.p0i8(i8* %dst, i1 false, i1 false)
+  %len = call i32 @llvm.objectsize.i32.p0i8(i8* %dst, i1 false, i1 false, i1 false)
   %ret = call i8* @__stpcpy_chk(i8* %dst, i8* %dst, i32 %len)
   ret i8* %ret
 }
@@ -100,4 +100,4 @@ define i8* @test_no_simplify1() {
 }
 
 declare i8* @__stpcpy_chk(i8*, i8*, i32) nounwind
-declare i32 @llvm.objectsize.i32.p0i8(i8*, i1, i1) nounwind readonly
+declare i32 @llvm.objectsize.i32.p0i8(i8*, i1, i1, i1) nounwind readonly
diff --git a/test/Transforms/InstCombine/strchr-1.ll b/test/Transforms/InstCombine/strchr-1.ll
index 6c10ebdfcc43020b9ab00f99a17300d4b379b0cb..4fce378f59f4377fe36af295dd1ebd6f2daf7184 100644
--- a/test/Transforms/InstCombine/strchr-1.ll
+++ b/test/Transforms/InstCombine/strchr-1.ll
@@ -82,8 +82,9 @@ define void @test_simplify6(i8* %str) {
 define i1 @test_simplify7(i32 %C) {
 ; CHECK-LABEL: @test_simplify7
 ; CHECK-NEXT: [[TRUNC:%.*]] = trunc i32 %C to i16
-; CHECK-NEXT: %memchr.bounds = icmp ult i16 [[TRUNC]], 16
-; CHECK-NEXT: [[SHL:%.*]] = shl i16 1, [[TRUNC]]
+; CHECK-NEXT: [[TRUNC_AND:%.*]] = and i16 [[TRUNC]], 255
+; CHECK-NEXT: %memchr.bounds = icmp ult i16 [[TRUNC_AND]], 16
+; CHECK-NEXT: [[SHL:%.*]] = shl i16 1, [[TRUNC_AND]]
 ; CHECK-NEXT: [[AND:%.*]] = and i16 [[SHL]], 9217
 ; CHECK-NEXT: %memchr.bits = icmp ne i16 [[AND]], 0
 ; CHECK-NEXT: %memchr1 = and i1 %memchr.bounds, %memchr.bits
diff --git a/test/Transforms/InstCombine/strcpy_chk-1.ll b/test/Transforms/InstCombine/strcpy_chk-1.ll
index a9a1f46b7e22eeb551e7f1f8f7e830fc38023704..859d810d3c4d79a9554420245330b27531cb8a51 100644
--- a/test/Transforms/InstCombine/strcpy_chk-1.ll
+++ b/test/Transforms/InstCombine/strcpy_chk-1.ll
@@ -64,10 +64,10 @@ define i8* @test_simplify5() {
   %dst = getelementptr inbounds [60 x i8], [60 x i8]* @a, i32 0, i32 0
   %src = getelementptr inbounds [12 x i8], [12 x i8]* @.str, i32 0, i32 0
 
-; CHECK-NEXT: %len = call i32 @llvm.objectsize.i32.p0i8(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0), i1 false, i1 false)
+; CHECK-NEXT: %len = call i32 @llvm.objectsize.i32.p0i8(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0), i1 false, i1 false, i1 false)
 ; CHECK-NEXT: %1 = call i8* @__memcpy_chk(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0), i8* getelementptr inbounds ([12 x i8], [12 x i8]* @.str, i32 0, i32 0), i32 12, i32 %len)
 ; CHECK-NEXT: ret i8* %1
-  %len = call i32 @llvm.objectsize.i32.p0i8(i8* %dst, i1 false, i1 false)
+  %len = call i32 @llvm.objectsize.i32.p0i8(i8* %dst, i1 false, i1 false, i1 false)
   %ret = call i8* @__strcpy_chk(i8* %dst, i8* %src, i32 %len)
   ret i8* %ret
 }
@@ -78,10 +78,10 @@ define i8* @test_simplify6() {
 ; CHECK-LABEL: @test_simplify6(
   %dst = getelementptr inbounds [60 x i8], [60 x i8]* @a, i32 0, i32 0
 
-; CHECK-NEXT: %len = call i32 @llvm.objectsize.i32.p0i8(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0), i1 false, i1 false)
+; CHECK-NEXT: %len = call i32 @llvm.objectsize.i32.p0i8(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0), i1 false, i1 false, i1 false)
 ; CHECK-NEXT: %ret = call i8* @__strcpy_chk(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0), i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0), i32 %len)
 ; CHECK-NEXT: ret i8* %ret
-  %len = call i32 @llvm.objectsize.i32.p0i8(i8* %dst, i1 false, i1 false)
+  %len = call i32 @llvm.objectsize.i32.p0i8(i8* %dst, i1 false, i1 false, i1 false)
   %ret = call i8* @__strcpy_chk(i8* %dst, i8* %dst, i32 %len)
   ret i8* %ret
 }
@@ -100,4 +100,4 @@ define i8* @test_no_simplify1() {
 }
 
 declare i8* @__strcpy_chk(i8*, i8*, i32) nounwind
-declare i32 @llvm.objectsize.i32.p0i8(i8*, i1, i1) nounwind readonly
+declare i32 @llvm.objectsize.i32.p0i8(i8*, i1, i1, i1) nounwind readonly
diff --git a/test/Transforms/InstCombine/vec_demanded_elts.ll b/test/Transforms/InstCombine/vec_demanded_elts.ll
index 838bc535888f7256bfa6d8a00cfe407abe9d8927..bc4b0737938d1a12ad34246f72a4cad554f97f33 100644
--- a/test/Transforms/InstCombine/vec_demanded_elts.ll
+++ b/test/Transforms/InstCombine/vec_demanded_elts.ll
@@ -496,3 +496,149 @@ define <3 x float> @shuf_frem_const_op1(<3 x float> %x) {
   ret <3 x float> %r
 }
 
+;; TODO: getelementptr tests below show missing simplifications for
+;; vector demanded elements on vector geps.
+
+define i32* @gep_vbase_w_s_idx(<2 x i32*> %base) {
+; CHECK-LABEL: @gep_vbase_w_s_idx(
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i32, <2 x i32*> [[BASE:%.*]], i64 1
+; CHECK-NEXT:    [[EE:%.*]] = extractelement <2 x i32*> [[GEP]], i32 1
+; CHECK-NEXT:    ret i32* [[EE]]
+;
+  %gep = getelementptr i32, <2 x i32*> %base, i64 1
+  %ee = extractelement <2 x i32*> %gep, i32 1
+  ret i32* %ee
+}
+
+define i32* @gep_splat_base_w_s_idx(i32* %base) {
+; CHECK-LABEL: @gep_splat_base_w_s_idx(
+; CHECK-NEXT:    [[BASEVEC2:%.*]] = insertelement <2 x i32*> undef, i32* [[BASE:%.*]], i32 1
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i32, <2 x i32*> [[BASEVEC2]], i64 1
+; CHECK-NEXT:    [[EE:%.*]] = extractelement <2 x i32*> [[GEP]], i32 1
+; CHECK-NEXT:    ret i32* [[EE]]
+;
+  %basevec1 = insertelement <2 x i32*> undef, i32* %base, i32 0
+  %basevec2 = shufflevector <2 x i32*> %basevec1, <2 x i32*> undef, <2 x i32> zeroinitializer
+  %gep = getelementptr i32, <2 x i32*> %basevec2, i64 1
+  %ee = extractelement <2 x i32*> %gep, i32 1
+  ret i32* %ee
+}
+
+
+define i32* @gep_splat_base_w_cv_idx(i32* %base) {
+; CHECK-LABEL: @gep_splat_base_w_cv_idx(
+; CHECK-NEXT:    [[BASEVEC2:%.*]] = insertelement <2 x i32*> undef, i32* [[BASE:%.*]], i32 1
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i32, <2 x i32*> [[BASEVEC2]], <2 x i64> <i64 undef, i64 1>
+; CHECK-NEXT:    [[EE:%.*]] = extractelement <2 x i32*> [[GEP]], i32 1
+; CHECK-NEXT:    ret i32* [[EE]]
+;
+  %basevec1 = insertelement <2 x i32*> undef, i32* %base, i32 0
+  %basevec2 = shufflevector <2 x i32*> %basevec1, <2 x i32*> undef, <2 x i32> zeroinitializer
+  %gep = getelementptr i32, <2 x i32*> %basevec2, <2 x i64> <i64 0, i64 1>
+  %ee = extractelement <2 x i32*> %gep, i32 1
+  ret i32* %ee
+}
+
+define i32* @gep_splat_base_w_vidx(i32* %base, <2 x i64> %idxvec) {
+; CHECK-LABEL: @gep_splat_base_w_vidx(
+; CHECK-NEXT:    [[BASEVEC2:%.*]] = insertelement <2 x i32*> undef, i32* [[BASE:%.*]], i32 1
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i32, <2 x i32*> [[BASEVEC2]], <2 x i64> [[IDXVEC:%.*]]
+; CHECK-NEXT:    [[EE:%.*]] = extractelement <2 x i32*> [[GEP]], i32 1
+; CHECK-NEXT:    ret i32* [[EE]]
+;
+  %basevec1 = insertelement <2 x i32*> undef, i32* %base, i32 0
+  %basevec2 = shufflevector <2 x i32*> %basevec1, <2 x i32*> undef, <2 x i32> zeroinitializer
+  %gep = getelementptr i32, <2 x i32*> %basevec2, <2 x i64> %idxvec
+  %ee = extractelement <2 x i32*> %gep, i32 1
+  ret i32* %ee
+}
+
+
+@GLOBAL = internal global i32 zeroinitializer
+
+define i32* @gep_cvbase_w_s_idx(<2 x i32*> %base, i64 %raw_addr) {
+; CHECK-LABEL: @gep_cvbase_w_s_idx(
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i32, <2 x i32*> <i32* undef, i32* @GLOBAL>, i64 [[RAW_ADDR:%.*]]
+; CHECK-NEXT:    [[EE:%.*]] = extractelement <2 x i32*> [[GEP]], i32 1
+; CHECK-NEXT:    ret i32* [[EE]]
+;
+  %gep = getelementptr i32, <2 x i32*> <i32* @GLOBAL, i32* @GLOBAL>, i64 %raw_addr
+  %ee = extractelement <2 x i32*> %gep, i32 1
+  ret i32* %ee
+}
+
+define i32* @gep_cvbase_w_cv_idx(<2 x i32*> %base, i64 %raw_addr) {
+; CHECK-LABEL: @gep_cvbase_w_cv_idx(
+; CHECK-NEXT:    ret i32* extractelement (<2 x i32*> getelementptr (i32, <2 x i32*> <i32* @GLOBAL, i32* @GLOBAL>, <2 x i64> <i64 0, i64 1>), i32 1)
+;
+  %gep = getelementptr i32, <2 x i32*> <i32* @GLOBAL, i32* @GLOBAL>, <2 x i64> <i64 0, i64 1>
+  %ee = extractelement <2 x i32*> %gep, i32 1
+  ret i32* %ee
+}
+
+
+define i32* @gep_sbase_w_cv_idx(i32* %base) {
+; CHECK-LABEL: @gep_sbase_w_cv_idx(
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i32, i32* [[BASE:%.*]], <2 x i64> <i64 undef, i64 1>
+; CHECK-NEXT:    [[EE:%.*]] = extractelement <2 x i32*> [[GEP]], i32 1
+; CHECK-NEXT:    ret i32* [[EE]]
+;
+  %gep = getelementptr i32, i32* %base, <2 x i64> <i64 0, i64 1>
+  %ee = extractelement <2 x i32*> %gep, i32 1
+  ret i32* %ee
+}
+
+define i32* @gep_sbase_w_splat_idx(i32* %base, i64 %idx) {
+; CHECK-LABEL: @gep_sbase_w_splat_idx(
+; CHECK-NEXT:    [[IDXVEC2:%.*]] = insertelement <2 x i64> undef, i64 [[IDX:%.*]], i32 1
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i32, i32* [[BASE:%.*]], <2 x i64> [[IDXVEC2]]
+; CHECK-NEXT:    [[EE:%.*]] = extractelement <2 x i32*> [[GEP]], i32 1
+; CHECK-NEXT:    ret i32* [[EE]]
+;
+  %idxvec1 = insertelement <2 x i64> undef, i64 %idx, i32 0
+  %idxvec2 = shufflevector <2 x i64> %idxvec1, <2 x i64> undef, <2 x i32> zeroinitializer
+  %gep = getelementptr i32, i32* %base, <2 x i64> %idxvec2
+  %ee = extractelement <2 x i32*> %gep, i32 1
+  ret i32* %ee
+}
+define i32* @gep_splat_both(i32* %base, i64 %idx) {
+; CHECK-LABEL: @gep_splat_both(
+; CHECK-NEXT:    [[BASEVEC2:%.*]] = insertelement <2 x i32*> undef, i32* [[BASE:%.*]], i32 1
+; CHECK-NEXT:    [[IDXVEC2:%.*]] = insertelement <2 x i64> undef, i64 [[IDX:%.*]], i32 1
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i32, <2 x i32*> [[BASEVEC2]], <2 x i64> [[IDXVEC2]]
+; CHECK-NEXT:    [[EE:%.*]] = extractelement <2 x i32*> [[GEP]], i32 1
+; CHECK-NEXT:    ret i32* [[EE]]
+;
+  %basevec1 = insertelement <2 x i32*> undef, i32* %base, i32 0
+  %basevec2 = shufflevector <2 x i32*> %basevec1, <2 x i32*> undef, <2 x i32> zeroinitializer
+  %idxvec1 = insertelement <2 x i64> undef, i64 %idx, i32 0
+  %idxvec2 = shufflevector <2 x i64> %idxvec1, <2 x i64> undef, <2 x i32> zeroinitializer
+  %gep = getelementptr i32, <2 x i32*> %basevec2, <2 x i64> %idxvec2
+  %ee = extractelement <2 x i32*> %gep, i32 1
+  ret i32* %ee
+}
+
+define <2 x i32*> @gep_all_lanes_undef(i32* %base, i64 %idx) {;
+; CHECK-LABEL: @gep_all_lanes_undef(
+; CHECK-NEXT:    [[BASEVEC:%.*]] = insertelement <2 x i32*> undef, i32* [[BASE:%.*]], i32 0
+; CHECK-NEXT:    [[IDXVEC:%.*]] = insertelement <2 x i64> undef, i64 [[IDX:%.*]], i32 1
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i32, <2 x i32*> [[BASEVEC]], <2 x i64> [[IDXVEC]]
+; CHECK-NEXT:    ret <2 x i32*> [[GEP]]
+;
+  %basevec = insertelement <2 x i32*> undef, i32* %base, i32 0
+  %idxvec = insertelement <2 x i64> undef, i64 %idx, i32 1
+  %gep = getelementptr i32, <2 x i32*> %basevec, <2 x i64> %idxvec
+  ret <2 x i32*> %gep
+}
+
+define i32* @gep_demanded_lane_undef(i32* %base, i64 %idx) {
+; CHECK-LABEL: @gep_demanded_lane_undef(
+; CHECK-NEXT:    ret i32* undef
+;
+  %basevec = insertelement <2 x i32*> undef, i32* %base, i32 0
+  %idxvec = insertelement <2 x i64> undef, i64 %idx, i32 1
+  %gep = getelementptr i32, <2 x i32*> %basevec, <2 x i64> %idxvec
+  %ee = extractelement <2 x i32*> %gep, i32 1
+  ret i32* %ee
+}
+
diff --git a/test/Transforms/InstCombine/vec_shuffle.ll b/test/Transforms/InstCombine/vec_shuffle.ll
index b82c8117eebf58bf98df5c1039f3a1503888c53c..aae337d0de3f8d12db38df2422a9ef8e361191b7 100644
--- a/test/Transforms/InstCombine/vec_shuffle.ll
+++ b/test/Transforms/InstCombine/vec_shuffle.ll
@@ -182,17 +182,28 @@ define <2 x i8> @extract_subvector_of_shuffle(<2 x i8> %x, <2 x i8> %y) {
   ret <2 x i8> %extract_subv
 }
 
-; Extra uses are ok.
 ; Undef elements in either mask are ok. Undefs from the 2nd shuffle mask should propagate to the new shuffle.
 ; The type of the inputs does not have to match the output type.
 
+define <4 x i8> @extract_subvector_of_shuffle_undefs_types(<2 x i8> %x, <2 x i8> %y) {
+; CHECK-LABEL: @extract_subvector_of_shuffle_undefs_types(
+; CHECK-NEXT:    [[EXTRACT_SUBV:%.*]] = shufflevector <2 x i8> [[X:%.*]], <2 x i8> [[Y:%.*]], <4 x i32> <i32 undef, i32 2, i32 0, i32 undef>
+; CHECK-NEXT:    ret <4 x i8> [[EXTRACT_SUBV]]
+;
+  %shuf = shufflevector <2 x i8> %x, <2 x i8> %y, <5 x i32> <i32 undef, i32 2, i32 0, i32 1, i32 0>
+  %extract_subv = shufflevector <5 x i8> %shuf, <5 x i8> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 undef>
+  ret <4 x i8> %extract_subv
+}
+
+; Extra uses are not ok - we only do the transform when we can eliminate an instruction.
+
 declare void @use_v5i8(<5 x i8>)
 
 define <4 x i8> @extract_subvector_of_shuffle_extra_use(<2 x i8> %x, <2 x i8> %y) {
 ; CHECK-LABEL: @extract_subvector_of_shuffle_extra_use(
 ; CHECK-NEXT:    [[SHUF:%.*]] = shufflevector <2 x i8> [[X:%.*]], <2 x i8> [[Y:%.*]], <5 x i32> <i32 undef, i32 2, i32 0, i32 1, i32 0>
 ; CHECK-NEXT:    call void @use_v5i8(<5 x i8> [[SHUF]])
-; CHECK-NEXT:    [[EXTRACT_SUBV:%.*]] = shufflevector <2 x i8> [[X]], <2 x i8> [[Y]], <4 x i32> <i32 undef, i32 2, i32 0, i32 undef>
+; CHECK-NEXT:    [[EXTRACT_SUBV:%.*]] = shufflevector <5 x i8> [[SHUF]], <5 x i8> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 undef>
 ; CHECK-NEXT:    ret <4 x i8> [[EXTRACT_SUBV]]
 ;
   %shuf = shufflevector <2 x i8> %x, <2 x i8> %y, <5 x i32> <i32 undef, i32 2, i32 0, i32 1, i32 0>
@@ -723,8 +734,8 @@ define <8 x i8> @pr19730(<16 x i8> %in0) {
 
 define i32 @pr19737(<4 x i32> %in0) {
 ; CHECK-LABEL: @pr19737(
-; CHECK-NEXT:    [[RV_RHS:%.*]] = extractelement <4 x i32> [[IN0:%.*]], i32 0
-; CHECK-NEXT:    ret i32 [[RV_RHS]]
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <4 x i32> [[IN0:%.*]], i32 0
+; CHECK-NEXT:    ret i32 [[TMP1]]
 ;
   %shuffle.i = shufflevector <4 x i32> zeroinitializer, <4 x i32> %in0, <4 x i32> <i32 0, i32 4, i32 2, i32 6>
   %neg.i = xor <4 x i32> %shuffle.i, <i32 -1, i32 -1, i32 -1, i32 -1>
diff --git a/test/Transforms/InstCombine/win-math.ll b/test/Transforms/InstCombine/win-math.ll
index 36947791393d9a4b5fa271db0a35978ec60a9b19..38ed949e949dcef4f01822c81ede9495cbde3805 100644
--- a/test/Transforms/InstCombine/win-math.ll
+++ b/test/Transforms/InstCombine/win-math.ll
@@ -1,17 +1,21 @@
-; RUN: opt -O2 -S -mtriple=i386-pc-win32 < %s | FileCheck %s -check-prefix=WIN32
-; RUN: opt -O2 -S -mtriple=x86_64-pc-win32 < %s | FileCheck %s -check-prefix=WIN64
-; RUN: opt -O2 -S -mtriple=i386-pc-mingw32 < %s | FileCheck %s -check-prefix=MINGW32
-; RUN: opt -O2 -S -mtriple=x86_64-pc-mingw32 < %s | FileCheck %s -check-prefix=MINGW64
+; RUN: opt < %s -O2 -S -mtriple=i386-pc-windows-msvc18   | FileCheck %s --check-prefixes=CHECK,MSVCXX,MSVC32
+; RUN: opt < %s -O2 -S -mtriple=i386-pc-windows-msvc     | FileCheck %s --check-prefixes=CHECK,MSVC19,MSVC51
+; RUN: opt < %s -O2 -S -mtriple=x86_64-pc-windows-msvc17 | FileCheck %s --check-prefixes=CHECK,MSVCXX,MSVC64
+; RUN: opt < %s -O2 -S -mtriple=x86_64-pc-win32          | FileCheck %s --check-prefixes=CHECK,MSVC19,MSVC83
+; RUN: opt < %s -O2 -S -mtriple=i386-pc-mingw32          | FileCheck %s --check-prefixes=CHECK,MINGW32
+; RUN: opt < %s -O2 -S -mtriple=x86_64-pc-mingw32        | FileCheck %s --check-prefixes=CHECK,MINGW64
 
 ; x86 win32 msvcrt does not provide entry points for single-precision libm.
-; x86-64 win32 msvcrt does (except for fabsf)
-; msvcrt does not provide C99 math, but mingw32 does.
+; x86-64 win32 msvcrt does, but with exceptions
+; msvcrt does not provide all of C99 math, but mingw32 does.
 
 declare double @acos(double %x)
 define float @float_acos(float %x) nounwind readnone {
-; WIN32-LABEL: @float_acos(
-; WIN32-NOT: float @acosf
-; WIN32: double @acos
+; CHECK-LABEL: @float_acos(
+; MSVCXX-NOT: float @acosf
+; MSVCXX: double @acos
+; MSVC19-NOT: float @acosf
+; MSVC19: double @acos
     %1 = fpext float %x to double
     %2 = call double @acos(double %1)
     %3 = fptrunc double %2 to float
@@ -20,9 +24,11 @@ define float @float_acos(float %x) nounwind readnone {
 
 declare double @asin(double %x)
 define float @float_asin(float %x) nounwind readnone {
-; WIN32-LABEL: @float_asin(
-; WIN32-NOT: float @asinf
-; WIN32: double @asin
+; CHECK-LABEL: @float_asin(
+; MSVCXX-NOT: float @asinf
+; MSVCXX: double @asin
+; MSVC19-NOT: float @asinf
+; MSVC19: double @asin
     %1 = fpext float %x to double
     %2 = call double @asin(double %1)
     %3 = fptrunc double %2 to float
@@ -31,9 +37,11 @@ define float @float_asin(float %x) nounwind readnone {
 
 declare double @atan(double %x)
 define float @float_atan(float %x) nounwind readnone {
-; WIN32-LABEL: @float_atan(
-; WIN32-NOT: float @atanf
-; WIN32: double @atan
+; CHECK-LABEL: @float_atan(
+; MSVCXX-NOT: float @atanf
+; MSVCXX: double @atan
+; MSVC19-NOT: float @atanf
+; MSVC19: double @atan
     %1 = fpext float %x to double
     %2 = call double @atan(double %1)
     %3 = fptrunc double %2 to float
@@ -42,9 +50,11 @@ define float @float_atan(float %x) nounwind readnone {
 
 declare double @atan2(double %x, double %y)
 define float @float_atan2(float %x, float %y) nounwind readnone {
-; WIN32-LABEL: @float_atan2(
-; WIN32-NOT: float @atan2f
-; WIN32: double @atan2
+; CHECK-LABEL: @float_atan2(
+; MSVCXX-NOT: float @atan2f
+; MSVCXX: double @atan2
+; MSVC19-NOT: float @atan2f
+; MSVC19: double @atan2
     %1 = fpext float %x to double
     %2 = fpext float %y to double
     %3 = call double @atan2(double %1, double %2)
@@ -54,18 +64,15 @@ define float @float_atan2(float %x, float %y) nounwind readnone {
 
 declare double @ceil(double %x)
 define float @float_ceil(float %x) nounwind readnone {
-; WIN32-LABEL: @float_ceil(
-; WIN32-NOT: float @ceilf
-; WIN32: float @llvm.ceil.f32
-; WIN64-LABEL: @float_ceil(
-; WIN64: float @llvm.ceil.f32
-; WIN64-NOT: double @ceil
-; MINGW32-LABEL: @float_ceil(
-; MINGW32: float @llvm.ceil.f32
+; CHECK-LABEL: @float_ceil(
+; MSVCXX-NOT: float @ceilf
+; MSVCXX: float @llvm.ceil.f32
+; MSVC19-NOT: double @ceil
+; MSVC19: float @llvm.ceil.f32
 ; MINGW32-NOT: double @ceil
-; MINGW64-LABEL: @float_ceil(
-; MINGW64: float @llvm.ceil.f32
+; MINGW32: float @llvm.ceil.f32
 ; MINGW64-NOT: double @ceil
+; MINGW64: float @llvm.ceil.f32
     %1 = fpext float %x to double
     %2 = call double @ceil(double %1)
     %3 = fptrunc double %2 to float
@@ -74,10 +81,11 @@ define float @float_ceil(float %x) nounwind readnone {
 
 declare double @_copysign(double %x)
 define float @float_copysign(float %x) nounwind readnone {
-; WIN32-LABEL: @float_copysign(
-; WIN32-NOT: float @copysignf
-; WIN32-NOT: float @_copysignf
-; WIN32: double @_copysign
+; CHECK-LABEL: @float_copysign(
+; MSVCXX-NOT: float @_copysignf
+; MSVCXX: double @_copysign
+; MSVC19-NOT: float @_copysignf
+; MSVC19: double @_copysign
     %1 = fpext float %x to double
     %2 = call double @_copysign(double %1)
     %3 = fptrunc double %2 to float
@@ -86,9 +94,11 @@ define float @float_copysign(float %x) nounwind readnone {
 
 declare double @cos(double %x)
 define float @float_cos(float %x) nounwind readnone {
-; WIN32-LABEL: @float_cos(
-; WIN32-NOT: float @cosf
-; WIN32: double @cos
+; CHECK-LABEL: @float_cos(
+; MSVCXX-NOT: float @cosf
+; MSVCXX: double @cos
+; MSVC19-NOT: float @cosf
+; MSVC19: double @cos
     %1 = fpext float %x to double
     %2 = call double @cos(double %1)
     %3 = fptrunc double %2 to float
@@ -97,9 +107,11 @@ define float @float_cos(float %x) nounwind readnone {
 
 declare double @cosh(double %x)
 define float @float_cosh(float %x) nounwind readnone {
-; WIN32-LABEL: @float_cosh(
-; WIN32-NOT: float @coshf
-; WIN32: double @cosh
+; CHECK-LABEL: @float_cosh(
+; MSVCXX-NOT: float @coshf
+; MSVCXX: double @cosh
+; MSVC19-NOT: float @coshf
+; MSVC19: double @cosh
     %1 = fpext float %x to double
     %2 = call double @cosh(double %1)
     %3 = fptrunc double %2 to float
@@ -108,9 +120,11 @@ define float @float_cosh(float %x) nounwind readnone {
 
 declare double @exp(double %x, double %y)
 define float @float_exp(float %x, float %y) nounwind readnone {
-; WIN32-LABEL: @float_exp(
-; WIN32-NOT: float @expf
-; WIN32: double @exp
+; CHECK-LABEL: @float_exp(
+; MSVCXX-NOT: float @expf
+; MSVCXX: double @exp
+; MSVC19-NOT: float @expf
+; MSVC19: double @exp
     %1 = fpext float %x to double
     %2 = fpext float %y to double
     %3 = call double @exp(double %1, double %2)
@@ -120,12 +134,11 @@ define float @float_exp(float %x, float %y) nounwind readnone {
 
 declare double @fabs(double %x, double %y)
 define float @float_fabs(float %x, float %y) nounwind readnone {
-; WIN32-LABEL: @float_fabs(
-; WIN32-NOT: float @fabsf
-; WIN32: double @fabs
-; WIN64-LABEL: @float_fabs(
-; WIN64-NOT: float @fabsf
-; WIN64: double @fabs
+; CHECK-LABEL: @float_fabs(
+; MSVCXX-NOT: float @fabsf
+; MSVCXX: double @fabs
+; MSVC19-NOT: float @fabsf
+; MSVC19: double @fabs
     %1 = fpext float %x to double
     %2 = fpext float %y to double
     %3 = call double @fabs(double %1, double %2)
@@ -135,18 +148,15 @@ define float @float_fabs(float %x, float %y) nounwind readnone {
 
 declare double @floor(double %x)
 define float @float_floor(float %x) nounwind readnone {
-; WIN32-LABEL: @float_floor(
-; WIN32-NOT: float @floorf
-; WIN32: float @llvm.floor.f32
-; WIN64-LABEL: @float_floor(
-; WIN64: float @llvm.floor.f32
-; WIN64-NOT: double @floor
-; MINGW32-LABEL: @float_floor(
-; MINGW32: float @llvm.floor.f32
+; CHECK-LABEL: @float_floor(
+; MSVCXX-NOT: float @floorf
+; MSVCXX: float @llvm.floor.f32
+; MSVC19-NOT: double @floor
+; MSVC19: float @llvm.floor.f32
 ; MINGW32-NOT: double @floor
-; MINGW64-LABEL: @float_floor(
-; MINGW64: float @llvm.floor.f32
+; MINGW32: float @llvm.floor.f32
 ; MINGW64-NOT: double @floor
+; MINGW64: float @llvm.floor.f32
     %1 = fpext float %x to double
     %2 = call double @floor(double %1)
     %3 = fptrunc double %2 to float
@@ -155,9 +165,11 @@ define float @float_floor(float %x) nounwind readnone {
 
 declare double @fmod(double %x, double %y)
 define float @float_fmod(float %x, float %y) nounwind readnone {
-; WIN32-LABEL: @float_fmod(
-; WIN32-NOT: float @fmodf
-; WIN32: double @fmod
+; MSVCXX-LABEL: @float_fmod(
+; MSVCXX-NOT: float @fmodf
+; MSVCXX: double @fmod
+; MSVC19-NOT: float @fmodf
+; MSVC19: double @fmod
     %1 = fpext float %x to double
     %2 = fpext float %y to double
     %3 = call double @fmod(double %1, double %2)
@@ -167,20 +179,37 @@ define float @float_fmod(float %x, float %y) nounwind readnone {
 
 declare double @log(double %x)
 define float @float_log(float %x) nounwind readnone {
-; WIN32-LABEL: @float_log(
-; WIN32-NOT: float @logf
-; WIN32: double @log
+; CHECK-LABEL: @float_log(
+; MSVCXX-NOT: float @logf
+; MSVCXX: double @log
+; MSVC19-NOT: float @logf
+; MSVC19: double @log
     %1 = fpext float %x to double
     %2 = call double @log(double %1)
     %3 = fptrunc double %2 to float
     ret float %3
 }
 
+declare double @logb(double %x)
+define float @float_logb(float %x) nounwind readnone {
+; CHECK-LABEL: @float_logb(
+; MSVCXX-NOT: float @logbf
+; MSVCXX: double @logb
+; MSVC19-NOT: float @logbf
+; MSVC19: double @logb
+    %1 = fpext float %x to double
+    %2 = call double @logb(double %1)
+    %3 = fptrunc double %2 to float
+    ret float %3
+}
+
 declare double @pow(double %x, double %y)
 define float @float_pow(float %x, float %y) nounwind readnone {
-; WIN32-LABEL: @float_pow(
-; WIN32-NOT: float @powf
-; WIN32: double @pow
+; CHECK-LABEL: @float_pow(
+; MSVCXX-NOT: float @powf
+; MSVCXX: double @pow
+; MSVC19-NOT: float @powf
+; MSVC19: double @pow
     %1 = fpext float %x to double
     %2 = fpext float %y to double
     %3 = call double @pow(double %1, double %2)
@@ -190,9 +219,11 @@ define float @float_pow(float %x, float %y) nounwind readnone {
 
 declare double @sin(double %x)
 define float @float_sin(float %x) nounwind readnone {
-; WIN32-LABEL: @float_sin(
-; WIN32-NOT: float @sinf
-; WIN32: double @sin
+; CHECK-LABEL: @float_sin(
+; MSVCXX-NOT: float @sinf
+; MSVCXX: double @sin
+; MSVC19-NOT: float @sinf
+; MSVC19: double @sin
     %1 = fpext float %x to double
     %2 = call double @sin(double %1)
     %3 = fptrunc double %2 to float
@@ -201,9 +232,11 @@ define float @float_sin(float %x) nounwind readnone {
 
 declare double @sinh(double %x)
 define float @float_sinh(float %x) nounwind readnone {
-; WIN32-LABEL: @float_sinh(
-; WIN32-NOT: float @sinhf
-; WIN32: double @sinh
+; CHECK-LABEL: @float_sinh(
+; MSVCXX-NOT: float @sinhf
+; MSVCXX: double @sinh
+; MSVC19-NOT: float @sinhf
+; MSVC19: double @sinh
     %1 = fpext float %x to double
     %2 = call double @sinh(double %1)
     %3 = fptrunc double %2 to float
@@ -212,18 +245,19 @@ define float @float_sinh(float %x) nounwind readnone {
 
 declare double @sqrt(double %x)
 define float @float_sqrt(float %x) nounwind readnone {
-; WIN32-LABEL: @float_sqrt(
-; WIN32-NOT: float @sqrtf
-; WIN32: double @sqrt
-; WIN64-LABEL: @float_sqrt(
-; WIN64: float @sqrtf
-; WIN64-NOT: double @sqrt
-; MINGW32-LABEL: @float_sqrt(
-; MINGW32: float @sqrtf
+; CHECK-LABEL: @float_sqrt(
+; MSVC32-NOT: float @sqrtf
+; MSVC32: double @sqrt
+; MSVC51-NOT: float @sqrtf
+; MSVC51: double @sqrt
+; MSVC64-NOT: double @sqrt
+; MSVC64: float @sqrtf
+; MSVC83-NOT: double @sqrt
+; MSVC83: float @sqrtf
 ; MINGW32-NOT: double @sqrt
-; MINGW64-LABEL: @float_sqrt(
-; MINGW64: float @sqrtf
+; MINGW32: float @sqrtf
 ; MINGW64-NOT: double @sqrt
+; MINGW64: float @sqrtf
     %1 = fpext float %x to double
     %2 = call double @sqrt(double %1)
     %3 = fptrunc double %2 to float
@@ -232,9 +266,11 @@ define float @float_sqrt(float %x) nounwind readnone {
 
 declare double @tan(double %x)
 define float @float_tan(float %x) nounwind readnone {
-; WIN32-LABEL: @float_tan(
-; WIN32-NOT: float @tanf
-; WIN32: double @tan
+; CHECK-LABEL: @float_tan(
+; MSVCXX-NOT: float @tanf
+; MSVCXX: double @tan
+; MSVC19-NOT: float @tanf
+; MSVC19: double @tan
     %1 = fpext float %x to double
     %2 = call double @tan(double %1)
     %3 = fptrunc double %2 to float
@@ -243,30 +279,29 @@ define float @float_tan(float %x) nounwind readnone {
 
 declare double @tanh(double %x)
 define float @float_tanh(float %x) nounwind readnone {
-; WIN32-LABEL: @float_tanh(
-; WIN32-NOT: float @tanhf
-; WIN32: double @tanh
+; CHECK-LABEL: @float_tanh(
+; MSVCXX-NOT: float @tanhf
+; MSVCXX: double @tanh
+; MSVC19-NOT: float @tanhf
+; MSVC19: double @tanh
     %1 = fpext float %x to double
     %2 = call double @tanh(double %1)
     %3 = fptrunc double %2 to float
     ret float %3
 }
 
-; win32 does not have round; mingw32 does
+; win32 does not have roundf; mingw32 does
 declare double @round(double %x)
 define float @float_round(float %x) nounwind readnone {
-; WIN32-LABEL: @float_round(
-; WIN32-NOT: float @roundf
-; WIN32: double @round
-; WIN64-LABEL: @float_round(
-; WIN64-NOT: float @roundf
-; WIN64: double @round
-; MINGW32-LABEL: @float_round(
-; MINGW32: float @llvm.round.f32
+; CHECK-LABEL: @float_round(
+; MSVCXX-NOT: double @roundf
+; MSVCXX: double @round
+; MSVC19-NOT: double @round
+; MSVC19: float @llvm.round.f32
 ; MINGW32-NOT: double @round
-; MINGW64-LABEL: @float_round(
-; MINGW64: float @llvm.round.f32
+; MINGW32: float @llvm.round.f32
 ; MINGW64-NOT: double @round
+; MINGW64: float @llvm.round.f32
     %1 = fpext float %x to double
     %2 = call double @round(double %1)
     %3 = fptrunc double %2 to float
@@ -275,26 +310,26 @@ define float @float_round(float %x) nounwind readnone {
 
 declare float @powf(float, float)
 
-; win32 lacks sqrtf&fabsf, win64 lacks fabsf, but
+; win32 lacks sqrtf & fabsf, win64 lacks fabsf, but
 ; calls to the intrinsics can be emitted instead.
 define float @float_powsqrt(float %x) nounwind readnone {
-; WIN32-LABEL: @float_powsqrt(
-; WIN32-NOT: float @sqrtf
-; WIN32: float @powf
-
-; WIN64-LABEL: @float_powsqrt(
-; WIN64: float @sqrtf
-; WIN64: float @llvm.fabs.f32(
-; WIN64-NOT: float @powf
-
-; MINGW32-LABEL: @float_powsqrt(
+; CHECK-LABEL: @float_powsqrt(
+; MSVC32-NOT: float @sqrtf
+; MSVC32: float @powf
+; MSVC51-NOT: float @sqrtf
+; MSVC51: float @powf
+; MSVC64-NOT: float @powf
+; MSVC64: float @sqrtf
+; MSVC64: float @llvm.fabs.f32(
+; MSVC83-NOT: float @powf
+; MSVC83: float @sqrtf
+; MSVC83: float @llvm.fabs.f32(
+; MINGW32-NOT: float @powf
 ; MINGW32: float @sqrtf
 ; MINGW32: float @llvm.fabs.f32
-; MINGW32-NOT: float @powf
-; MINGW64-LABEL: @float_powsqrt(
+; MINGW64-NOT: float @powf
 ; MINGW64: float @sqrtf
 ; MINGW64: float @llvm.fabs.f32(
-; MINGW64-NOT: float @powf
     %1 = call float @powf(float %x, float 0.5)
     ret float %1
 }
diff --git a/test/Transforms/InstCombine/zext-bool-add-sub.ll b/test/Transforms/InstCombine/zext-bool-add-sub.ll
index 3fe5bddd1178817e70f8c7f41c7e658b1e54fa2f..d009aa9ed975d5c463f939163847e04a831d549a 100644
--- a/test/Transforms/InstCombine/zext-bool-add-sub.ll
+++ b/test/Transforms/InstCombine/zext-bool-add-sub.ll
@@ -315,3 +315,90 @@ define i8 @sext_sub_nuw(i8 %x, i1 %y) {
   ret i8 %sub
 }
 
+define i32 @sextbool_add(i1 %c, i32 %x) {
+; CHECK-LABEL: @sextbool_add(
+; CHECK-NEXT:    [[B:%.*]] = sext i1 [[C:%.*]] to i32
+; CHECK-NEXT:    [[S:%.*]] = add i32 [[B]], [[X:%.*]]
+; CHECK-NEXT:    ret i32 [[S]]
+;
+  %b = sext i1 %c to i32
+  %s = add i32 %b, %x
+  ret i32 %s
+}
+
+define i32 @sextbool_add_commute(i1 %c, i32 %px) {
+; CHECK-LABEL: @sextbool_add_commute(
+; CHECK-NEXT:    [[X:%.*]] = urem i32 [[PX:%.*]], 42
+; CHECK-NEXT:    [[B:%.*]] = sext i1 [[C:%.*]] to i32
+; CHECK-NEXT:    [[S:%.*]] = add nsw i32 [[X]], [[B]]
+; CHECK-NEXT:    ret i32 [[S]]
+;
+  %x = urem i32 %px, 42 ; thwart complexity-based canonicalization
+  %b = sext i1 %c to i32
+  %s = add i32 %x, %b
+  ret i32 %s
+}
+
+; Negative test - extra use prevents canonicalization.
+
+declare void @use32(i32)
+
+define i32 @sextbool_add_uses(i1 %c, i32 %x) {
+; CHECK-LABEL: @sextbool_add_uses(
+; CHECK-NEXT:    [[B:%.*]] = sext i1 [[C:%.*]] to i32
+; CHECK-NEXT:    call void @use32(i32 [[B]])
+; CHECK-NEXT:    [[S:%.*]] = add i32 [[B]], [[X:%.*]]
+; CHECK-NEXT:    ret i32 [[S]]
+;
+  %b = sext i1 %c to i32
+  call void @use32(i32 %b)
+  %s = add i32 %b, %x
+  ret i32 %s
+}
+
+define <4 x i32> @sextbool_add_vector(<4 x i1> %c, <4 x i32> %x) {
+; CHECK-LABEL: @sextbool_add_vector(
+; CHECK-NEXT:    [[B:%.*]] = sext <4 x i1> [[C:%.*]] to <4 x i32>
+; CHECK-NEXT:    [[S:%.*]] = add <4 x i32> [[B]], [[X:%.*]]
+; CHECK-NEXT:    ret <4 x i32> [[S]]
+;
+  %b = sext <4 x i1> %c to <4 x i32>
+  %s = add <4 x i32> %x, %b
+  ret <4 x i32> %s
+}
+
+define i32 @zextbool_sub(i1 %c, i32 %x) {
+; CHECK-LABEL: @zextbool_sub(
+; CHECK-NEXT:    [[B:%.*]] = zext i1 [[C:%.*]] to i32
+; CHECK-NEXT:    [[S:%.*]] = sub i32 [[B]], [[X:%.*]]
+; CHECK-NEXT:    ret i32 [[S]]
+;
+  %b = zext i1 %c to i32
+  %s = sub i32 %b, %x
+  ret i32 %s
+}
+
+define i32 @zextbool_sub_uses(i1 %c, i32 %x) {
+; CHECK-LABEL: @zextbool_sub_uses(
+; CHECK-NEXT:    [[B:%.*]] = zext i1 [[C:%.*]] to i32
+; CHECK-NEXT:    call void @use32(i32 [[B]])
+; CHECK-NEXT:    [[S:%.*]] = sub i32 [[X:%.*]], [[B]]
+; CHECK-NEXT:    ret i32 [[S]]
+;
+  %b = zext i1 %c to i32
+  call void @use32(i32 %b)
+  %s = sub i32 %x, %b
+  ret i32 %s
+}
+
+define <4 x i32> @zextbool_sub_vector(<4 x i1> %c, <4 x i32> %x) {
+; CHECK-LABEL: @zextbool_sub_vector(
+; CHECK-NEXT:    [[B:%.*]] = zext <4 x i1> [[C:%.*]] to <4 x i32>
+; CHECK-NEXT:    [[S:%.*]] = sub <4 x i32> [[X:%.*]], [[B]]
+; CHECK-NEXT:    ret <4 x i32> [[S]]
+;
+  %b = zext <4 x i1> %c to <4 x i32>
+  %s = sub <4 x i32> %x, %b
+  ret <4 x i32> %s
+}
+
diff --git a/test/Transforms/InstSimplify/call.ll b/test/Transforms/InstSimplify/call.ll
index b3a2f96870e53f4e12527f827174f24a70dcd6a9..9d63d375afb00ff9334e6dbcef9dfb56c5c60130 100644
--- a/test/Transforms/InstSimplify/call.ll
+++ b/test/Transforms/InstSimplify/call.ll
@@ -189,36 +189,6 @@ define {i8, i1} @test_smul4(i8 %V) {
   ret {i8, i1} %x
 }
 
-declare i256 @llvm.cttz.i256(i256 %src, i1 %is_zero_undef)
-
-define i256 @test_cttz() {
-; CHECK-LABEL: @test_cttz(
-; CHECK-NEXT:    ret i256 1
-;
-  %x = call i256 @llvm.cttz.i256(i256 10, i1 false)
-  ret i256 %x
-}
-
-declare <2 x i256> @llvm.cttz.v2i256(<2 x i256> %src, i1 %is_zero_undef)
-
-define <2 x i256> @test_cttz_vec() {
-; CHECK-LABEL: @test_cttz_vec(
-; CHECK-NEXT:    ret <2 x i256> <i256 1, i256 1>
-;
-  %x = call <2 x i256> @llvm.cttz.v2i256(<2 x i256> <i256 10, i256 10>, i1 false)
-  ret <2 x i256> %x
-}
-
-declare i256 @llvm.ctpop.i256(i256 %src)
-
-define i256 @test_ctpop() {
-; CHECK-LABEL: @test_ctpop(
-; CHECK-NEXT:    ret i256 2
-;
-  %x = call i256 @llvm.ctpop.i256(i256 10)
-  ret i256 %x
-}
-
 ; Test a non-intrinsic that we know about as a library call.
 declare float @fabs(float %x)
 
diff --git a/test/Transforms/InstSimplify/log10-pow10-intrinsic.ll b/test/Transforms/InstSimplify/log10-pow10-intrinsic.ll
new file mode 100644
index 0000000000000000000000000000000000000000..a5b7afde1a80b9050c47dbb2d356276942487862
--- /dev/null
+++ b/test/Transforms/InstSimplify/log10-pow10-intrinsic.ll
@@ -0,0 +1,45 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -instsimplify < %s | FileCheck %s
+
+declare double @llvm.log10.f64(double)
+declare double @llvm.pow.f64(double, double)
+
+define double @log10_pow10(double %x) {
+; CHECK-LABEL: @log10_pow10(
+; CHECK-NEXT:    [[TMP1:%.*]] = call double @llvm.pow.f64(double 1.000000e+01, double [[X:%.*]])
+; CHECK-NEXT:    [[TMP2:%.*]] = call double @llvm.log10.f64(double [[TMP1]])
+; CHECK-NEXT:    ret double [[TMP2]]
+;
+  %tmp = call double @llvm.pow.f64(double 1.000000e+01, double %x)
+  %tmp1 = call double @llvm.log10.f64(double %tmp)
+  ret double %tmp1
+}
+
+define double @log10_strict_pow10_reassoc(double %x) {
+; CHECK-LABEL: @log10_strict_pow10_reassoc(
+; CHECK-NEXT:    [[TMP1:%.*]] = call reassoc double @llvm.pow.f64(double 1.000000e+01, double [[X:%.*]])
+; CHECK-NEXT:    [[TMP2:%.*]] = call double @llvm.log10.f64(double [[TMP1]])
+; CHECK-NEXT:    ret double [[TMP2]]
+;
+  %tmp = call reassoc double @llvm.pow.f64(double 1.000000e+01, double %x)
+  %tmp1 = call double @llvm.log10.f64(double %tmp)
+  ret double %tmp1
+}
+
+define double @log10_reassoc_pow10_strict(double %x) {
+; CHECK-LABEL: @log10_reassoc_pow10_strict(
+; CHECK-NEXT:    ret double [[X:%.*]]
+;
+  %tmp = call double @llvm.pow.f64(double 1.000000e+01, double %x)
+  %tmp1 = call reassoc double @llvm.log10.f64(double %tmp)
+  ret double %tmp1
+}
+
+define double @log10_pow10_reassoc(double %x) {
+; CHECK-LABEL: @log10_pow10_reassoc(
+; CHECK-NEXT:    ret double [[X:%.*]]
+;
+  %tmp = call reassoc double @llvm.pow.f64(double 1.000000e+01, double %x)
+  %tmp1 = call reassoc double @llvm.log10.f64(double %tmp)
+  ret double %tmp1
+}
diff --git a/test/Transforms/InstSimplify/log2-pow2-intrinsic.ll b/test/Transforms/InstSimplify/log2-pow2-intrinsic.ll
new file mode 100644
index 0000000000000000000000000000000000000000..8d8ef80db72ee3c70dea2cdfe95cf770b39fca71
--- /dev/null
+++ b/test/Transforms/InstSimplify/log2-pow2-intrinsic.ll
@@ -0,0 +1,45 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -instsimplify < %s | FileCheck %s
+
+declare double @llvm.log2.f64(double)
+declare double @llvm.pow.f64(double, double)
+
+define double @log2_pow2(double %x) {
+; CHECK-LABEL: @log2_pow2(
+; CHECK-NEXT:    [[TMP:%.*]] = call double @llvm.pow.f64(double 2.000000e+00, double [[X:%.*]])
+; CHECK-NEXT:    [[TMP1:%.*]] = call double @llvm.log2.f64(double [[TMP]])
+; CHECK-NEXT:    ret double [[TMP1]]
+;
+  %tmp = call double @llvm.pow.f64(double 2.000000e+00, double %x)
+  %tmp1 = call double @llvm.log2.f64(double %tmp)
+  ret double %tmp1
+}
+
+define double @log2_strict_pow2_reassoc(double %x) {
+; CHECK-LABEL: @log2_strict_pow2_reassoc(
+; CHECK-NEXT:    [[TMP:%.*]] = call reassoc double @llvm.pow.f64(double 2.000000e+00, double [[X:%.*]])
+; CHECK-NEXT:    [[TMP1:%.*]] = call double @llvm.log2.f64(double [[TMP]])
+; CHECK-NEXT:    ret double [[TMP1]]
+;
+  %tmp = call reassoc double @llvm.pow.f64(double 2.000000e+00, double %x)
+  %tmp1 = call double @llvm.log2.f64(double %tmp)
+  ret double %tmp1
+}
+
+define double @log2_reassoc_pow2_strict(double %x) {
+; CHECK-LABEL: @log2_reassoc_pow2_strict(
+; CHECK-NEXT:    ret double [[X:%.*]]
+;
+  %tmp = call double @llvm.pow.f64(double 2.000000e+00, double %x)
+  %tmp1 = call reassoc double @llvm.log2.f64(double %tmp)
+  ret double %tmp1
+}
+
+define double @log2_pow2_reassoc(double %x) {
+; CHECK-LABEL: @log2_pow2_reassoc(
+; CHECK-NEXT:    ret double [[X:%.*]]
+;
+  %tmp = call reassoc double @llvm.pow.f64(double 2.000000e+00, double %x)
+  %tmp1 = call reassoc double @llvm.log2.f64(double %tmp)
+  ret double %tmp1
+}
diff --git a/test/Transforms/JumpThreading/callbr-edge-split.ll b/test/Transforms/JumpThreading/callbr-edge-split.ll
new file mode 100644
index 0000000000000000000000000000000000000000..a341f73dee5a88ba903fb8e9a63f89b15d242b98
--- /dev/null
+++ b/test/Transforms/JumpThreading/callbr-edge-split.ll
@@ -0,0 +1,58 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -S -jump-threading | FileCheck %s
+
+; This test used to cause jump threading to try to split an edge of a callbr.
+
+@a = global i32 0
+
+define i32 @c() {
+; CHECK-LABEL: @c(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* @a
+; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[TMP0]], 0
+; CHECK-NEXT:    br i1 [[TOBOOL]], label [[IF_ELSE:%.*]], label [[IF_THEN:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @b()
+; CHECK-NEXT:    [[PHITMP:%.*]] = icmp ne i32 [[CALL]], 0
+; CHECK-NEXT:    br i1 [[PHITMP]], label [[IF_THEN2:%.*]], label [[IF_END4:%.*]]
+; CHECK:       if.else:
+; CHECK-NEXT:    callbr void asm sideeffect "", "X"(i8* blockaddress(@c, [[IF_THEN2]]))
+; CHECK-NEXT:    to label [[IF_END_THREAD:%.*]] [label %if.then2]
+; CHECK:       if.end.thread:
+; CHECK-NEXT:    br label [[IF_THEN2]]
+; CHECK:       if.then2:
+; CHECK-NEXT:    [[CALL3:%.*]] = call i32 @b()
+; CHECK-NEXT:    br label [[IF_END4]]
+; CHECK:       if.end4:
+; CHECK-NEXT:    ret i32 undef
+;
+entry:
+  %0 = load i32, i32* @a
+  %tobool = icmp eq i32 %0, 0
+  br i1 %tobool, label %if.else, label %if.then
+
+if.then:                                          ; preds = %entry
+  %call = call i32 @b() #2
+  %phitmp = icmp ne i32 %call, 0
+  br label %if.end
+
+if.else:                                          ; preds = %entry
+  callbr void asm sideeffect "", "X"(i8* blockaddress(@c, %if.end)) #2
+  to label %normal [label %if.end]
+
+normal:                                           ; preds = %if.else
+  br label %if.end
+
+if.end:                                           ; preds = %if.else, %normal, %if.then
+  %d.0 = phi i1 [ %phitmp, %if.then ], [ undef, %normal ], [ undef, %if.else ]
+  br i1 %d.0, label %if.then2, label %if.end4
+
+if.then2:                                         ; preds = %if.end
+  %call3 = call i32 @b()
+  br label %if.end4
+
+if.end4:                                          ; preds = %if.then2, %if.end
+  ret i32 undef
+}
+
+declare i32 @b()
diff --git a/test/Transforms/JumpThreading/select.ll b/test/Transforms/JumpThreading/select.ll
index dd96e757b09348659240dd23183e9ff384c481c5..7557a6c814b6d9800cfaf9c8999ab77d44f056b5 100644
--- a/test/Transforms/JumpThreading/select.ll
+++ b/test/Transforms/JumpThreading/select.ll
@@ -363,3 +363,81 @@ cond.false.15.i:                                  ; preds = %cond.false.10.i
 ; CHECK: br i1 %cmp13.i, label %.exit, label %cond.false.15.i
 ; CHECK: br label %.exit
 }
+
+; When a select has a constant operand in one branch, and it feeds a phi node
+; and the phi node feeds a switch we unfold the select
+define void @test_func(i32* nocapture readonly %a, i32* nocapture readonly %b, i32* nocapture readonly %c, i32 %n) local_unnamed_addr #0 {
+entry:
+  br label %for.cond
+
+for.cond:                                         ; preds = %sw.default, %entry
+  %i.0 = phi i32 [ 0, %entry ], [ %inc, %sw.default ]
+  %cmp = icmp slt i32 %i.0, %n
+  br i1 %cmp, label %for.body, label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %for.cond
+  ret void
+
+for.body:                                         ; preds = %for.cond
+  %0 = zext i32 %i.0 to i64
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %0
+  %1 = load i32, i32* %arrayidx, align 4
+  %cmp1 = icmp eq i32 %1, 4
+  br i1 %cmp1, label %land.lhs.true, label %if.end
+
+land.lhs.true:                                    ; preds = %for.body
+  %arrayidx3 = getelementptr inbounds i32, i32* %b, i64 %0
+  %2 = load i32, i32* %arrayidx3, align 4
+  %arrayidx5 = getelementptr inbounds i32, i32* %c, i64 %0
+  %3 = load i32, i32* %arrayidx5, align 4
+  %cmp6 = icmp eq i32 %2, %3
+  %spec.select = select i1 %cmp6, i32 2, i32 4
+  br label %if.end
+
+if.end:                                           ; preds = %land.lhs.true, %for.body
+  %local_var.0 = phi i32 [ %1, %for.body ], [ %spec.select, %land.lhs.true ]
+  switch i32 %local_var.0, label %sw.default [
+    i32 2, label %sw.bb
+    i32 4, label %sw.bb7
+    i32 5, label %sw.bb8
+    i32 7, label %sw.bb9
+  ]
+
+sw.bb:                                            ; preds = %if.end
+  call void @foo()
+  br label %sw.bb7
+
+sw.bb7:                                           ; preds = %if.end, %sw.bb
+  call void @bar()
+  br label %sw.bb8
+
+sw.bb8:                                           ; preds = %if.end, %sw.bb7
+  call void @baz()
+  br label %sw.bb9
+
+sw.bb9:                                           ; preds = %if.end, %sw.bb8
+  call void @quux()
+  br label %sw.default
+
+sw.default:                                       ; preds = %if.end, %sw.bb9
+  call void @baz()
+  %inc = add nuw nsw i32 %i.0, 1
+  br label %for.cond
+
+; CHECK-LABEL: @test_func(
+; CHECK: [[REG:%[0-9]+]] = load
+; CHECK-NOT: select
+; CHECK: br i1
+; CHECK-NOT: select
+; CHECK: br i1 {{.*}}, label [[DEST1:%.*]], label [[DEST2:%.*]]
+
+; The following line checks existence of a phi node, and makes sure
+; it only has one incoming value. To do this, we check every '%'. Note
+; that REG and REG2 each contain one '%;. There is another one in the
+; beginning of the incoming block name. After that there should be no other '%'.
+
+; CHECK: [[REG2:%.*]] = phi i32 {{[^%]*}}[[REG]]{{[^%]*%[^%]*}}
+; CHECK: switch i32 [[REG2]]
+; CHECK: i32 2, label [[DEST1]]
+; CHECK: i32 4, label [[DEST2]]
+}
diff --git a/test/Transforms/LCSSA/avoid-intrinsics-in-catchswitch.ll b/test/Transforms/LCSSA/avoid-intrinsics-in-catchswitch.ll
index d1c25c0c728b4eb0f0768a472aa82d538b4139ba..55357f1fee55769371c1fbf138510dd299f94ee6 100644
--- a/test/Transforms/LCSSA/avoid-intrinsics-in-catchswitch.ll
+++ b/test/Transforms/LCSSA/avoid-intrinsics-in-catchswitch.ll
@@ -85,7 +85,7 @@ declare %class.f* @"\01??0f@@QEAA@H@Z"(%class.f* returned, i32) unnamed_addr
 
 define internal i32 @"\01?filt$0@0@m@@"(i8* %exception_pointers, i8* %frame_pointer) personality i8* bitcast (i32 (...)* @__C_specific_handler to i8*) {
 entry:
-  %0 = tail call i8* @llvm.x86.seh.recoverfp(i8* bitcast (i32 ()* @"\01?m@@YAJXZ" to i8*), i8* %frame_pointer)
+  %0 = tail call i8* @llvm.eh.recoverfp(i8* bitcast (i32 ()* @"\01?m@@YAJXZ" to i8*), i8* %frame_pointer)
   %1 = tail call i8* @llvm.localrecover(i8* bitcast (i32 ()* @"\01?m@@YAJXZ" to i8*), i8* %0, i32 0)
   %2 = tail call i8* @llvm.localrecover(i8* bitcast (i32 ()* @"\01?m@@YAJXZ" to i8*), i8* %0, i32 1)
   %status = bitcast i8* %2 to i32*
@@ -112,7 +112,7 @@ ehcleanup:                                        ; preds = %entry
   cleanupret from %9 unwind to caller
 }
 
-declare i8* @llvm.x86.seh.recoverfp(i8*, i8*)
+declare i8* @llvm.eh.recoverfp(i8*, i8*)
 declare i8* @llvm.localrecover(i8*, i8*, i32)
 declare i32 @"\01?j@@YAJVf@@JPEAUk@@PEAH@Z"(i8, i32, %struct.k*, i32*) local_unnamed_addr
 declare i32 @__C_specific_handler(...)
diff --git a/test/Transforms/LCSSA/rewrite-existing-dbg-values.ll b/test/Transforms/LCSSA/rewrite-existing-dbg-values.ll
index 563a75f407f0e38ff725fc18d6bdc58440dacda6..bedcc780d31f0cfa0fe59826f3db1970598b069c 100644
--- a/test/Transforms/LCSSA/rewrite-existing-dbg-values.ll
+++ b/test/Transforms/LCSSA/rewrite-existing-dbg-values.ll
@@ -1,54 +1,118 @@
 ; RUN: opt -S -lcssa < %s | FileCheck %s
 
-target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
-target triple = "x86_64-unknown-linux-gnu"
-
 ; Reproducer for PR39019.
 ;
-; Verify that the llvm.dbg.value in the %for.cond.cleanup2 block is rewritten
-; to use the PHI node for %add that is created by LCSSA.
+; Verify that the llvm.dbg.values are updated to use the PHI nodes inserted by
+; LCSSA.
 
-; CHECK-LABEL: for.cond.cleanup2:
-; CHECK-NEXT: [[PN:%[^ ]*]] = phi i32 [ %add.lcssa, %for.cond.cleanup1 ]
-; CHECK-NEXT: call void @llvm.dbg.value(metadata i32 [[PN]], metadata [[VAR:![0-9]+]], metadata !DIExpression())
-; CHECK-NEXT: call void @bar(i32 [[PN]])
+; For the test case @single_exit, we can rewrite all llvm.dbg.value calls
+; to use the inserted PHI.
 
-; CHECK-LABEL: for.body:
+; CHECK-LABEL: @single_exit(
+
+; CHECK-LABEL: inner.body:
 ; CHECK: %add = add nsw i32 0, 2
-; CHECK: call void @llvm.dbg.value(metadata i32 %add, metadata [[VAR]], metadata !DIExpression())
+; CHECK: call void @llvm.dbg.value(metadata i32 %add, metadata [[VAR:![0-9]+]], metadata !DIExpression())
 
-; CHECK: [[VAR]] = !DILocalVariable(name: "sum",
 
-; Function Attrs: nounwind
-define void @foo() #0 !dbg !6 {
+; CHECK-LABEL: outer.exit:
+; CHECK-NEXT: [[PN:%[^ ]*]] = phi i32 [ %add.lcssa, %outer.latch ]
+; CHECK-NEXT: call void @llvm.dbg.value(metadata i32 [[PN]], metadata [[VAR]], metadata !DIExpression())
+; CHECK-NEXT: call void @bar(i32 [[PN]])
+
+; CHECK-LABEL: exit:
+; CHECK-NEXT: call void @llvm.dbg.value(metadata i32 [[PN]], metadata [[VAR]], metadata !DIExpression())
+
+define void @single_exit()  !dbg !6 {
 entry:
-  br label %for.cond.preheader, !dbg !12
+  br label %outer.header, !dbg !12
 
-for.cond.preheader:                               ; preds = %for.cond.cleanup1, %entry
-  br label %for.body, !dbg !12
+outer.header:                                     ; preds = %outer.latch, %entry
+  br label %inner.body, !dbg !12
 
-for.cond.cleanup2:                                ; preds = %for.cond.cleanup1
+inner.body:                                       ; preds = %inner.body, %outer.header
+  %add = add nsw i32 0, 2, !dbg !12
   call void @llvm.dbg.value(metadata i32 %add, metadata !9, metadata !DIExpression()), !dbg !12
-  tail call void @bar(i32 %add) #0, !dbg !12
-  ret void, !dbg !12
+  br i1 false, label %inner.body, label %inner.exit, !dbg !12
 
-for.cond.cleanup1:                                ; preds = %for.body
-  br i1 false, label %for.cond.preheader, label %for.cond.cleanup2, !dbg !12
+inner.exit:                                       ; preds = %inner.body
+  br label %outer.latch
 
-for.body:                                         ; preds = %for.body, %for.cond.preheader
-  %add = add nsw i32 0, 2, !dbg !12
+outer.latch:                                      ; preds = %inner.exit
+  br i1 false, label %outer.header, label %outer.exit, !dbg !12
+
+outer.exit:                                       ; preds = %outer.latch
+  call void @llvm.dbg.value(metadata i32 %add, metadata !9, metadata !DIExpression()), !dbg !12
+  tail call void @bar(i32 %add), !dbg !12
+  br label %exit
+
+exit:                                             ; preds = %outer.exit
   call void @llvm.dbg.value(metadata i32 %add, metadata !9, metadata !DIExpression()), !dbg !12
-  br i1 false, label %for.body, label %for.cond.cleanup1, !dbg !12
+  ret void, !dbg !12
 }
 
-; Function Attrs: nounwind
-declare void @bar(i32) #0
+; For the test case @multi_exit, we cannot update the llvm.dbg.value call in exit,
+; because LCSSA did not insert a PHI node in %exit, as there is no non-debug
+; use.
+
+; CHECK-LABEL: @multi_exit()
+
+; CHECK-LABEL: for.header:
+; CHECK-NEXT: %add = add nsw i32 0, 2
+; CHECK-NEXT: call void @llvm.dbg.value(metadata i32 %add, metadata [[VAR2:![0-9]+]], metadata !DIExpression())
+
+; CHECK-LABEL: for.exit1:
+; CHECK-NEXT: [[PN1:%[^ ]*]] = phi i32 [ %add, %for.header ]
+; CHECK-NEXT: br label %for.exit1.succ
+
+; CHECK-LABEL: for.exit1.succ:
+; CHECK-NEXT: call void @llvm.dbg.value(metadata i32 [[PN1]], metadata [[VAR2]], metadata !DIExpression())
+; CHECK-NEXT: call void @bar(i32 [[PN1]])
+
+; CHECK-LABEL: for.exit2:
+; CHECK-NEXT: [[PN2:%[^ ]*]] = phi i32 [ %add, %for.latch ]
+; CHECK-NEXT: call void @llvm.dbg.value(metadata i32 [[PN2]], metadata [[VAR2]], metadata !DIExpression())
+; CHECK-NEXT: call void @bar(i32 [[PN2]])
+
+; CHECK-LABEL: exit:
+; CHECK-NEXT: call void @llvm.dbg.value(metadata i32 %add, metadata [[VAR2]], metadata !DIExpression())
+
+define void @multi_exit()  !dbg !13 {
+entry:
+  br label %for.header, !dbg !14
+
+for.header:                                       ; preds = %for.latch, %entry
+  %add = add nsw i32 0, 2, !dbg !14
+  call void @llvm.dbg.value(metadata i32 %add, metadata !16, metadata !DIExpression()), !dbg !14
+  br i1 false, label %for.latch, label %for.exit1, !dbg !14
+
+for.latch:                                        ; preds = %for.header
+  br i1 false, label %for.header, label %for.exit2, !dbg !14
+
+for.exit1:                                        ; preds = %for.header
+  br label %for.exit1.succ
+
+for.exit1.succ:                                   ; preds = %for.exit1
+  call void @llvm.dbg.value(metadata i32 %add, metadata !16, metadata !DIExpression()), !dbg !14
+  tail call void @bar(i32 %add), !dbg !14
+  br label %exit
+
+for.exit2:                                        ; preds = %for.latch
+  call void @llvm.dbg.value(metadata i32 %add, metadata !16, metadata !DIExpression()), !dbg !14
+  tail call void @bar(i32 %add), !dbg !14
+  br label %exit
+
+exit:                                             ; preds = %for.exit2, %for.exit1.succ
+  call void @llvm.dbg.value(metadata i32 %add, metadata !16, metadata !DIExpression()), !dbg !14
+  ret void, !dbg !14
+}
+
+; CHECK: [[VAR]] = !DILocalVariable(name: "sum",
+; CHECK: [[VAR2]] = !DILocalVariable(name: "sum2",
 
-; Function Attrs: nounwind readnone speculatable
-declare void @llvm.dbg.value(metadata, metadata, metadata) #1
+declare void @bar(i32)
 
-attributes #0 = { nounwind }
-attributes #1 = { nounwind readnone speculatable }
+declare void @llvm.dbg.value(metadata, metadata, metadata)
 
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!3, !4}
@@ -60,10 +124,14 @@ attributes #1 = { nounwind readnone speculatable }
 !3 = !{i32 2, !"Dwarf Version", i32 4}
 !4 = !{i32 2, !"Debug Info Version", i32 3}
 !5 = !{!"clang version 8.0.0"}
-!6 = distinct !DISubprogram(name: "foo", scope: !1, file: !1, line: 10, type: !7, isLocal: false, isDefinition: true, scopeLine: 10, isOptimized: true, unit: !0, retainedNodes: !8)
+!6 = distinct !DISubprogram(name: "foo", scope: !1, file: !1, line: 10, type: !7, scopeLine: 10, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !8)
 !7 = !DISubroutineType(types: !2)
 !8 = !{!9}
 !9 = !DILocalVariable(name: "sum", scope: !10, file: !1, line: 11, type: !11)
 !10 = !DILexicalBlockFile(scope: !6, file: !1, discriminator: 0)
 !11 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
 !12 = !DILocation(line: 0, scope: !10)
+!13 = distinct !DISubprogram(name: "multi_exit", scope: !1, file: !1, line: 10, type: !7, scopeLine: 10, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !8)
+!14 = !DILocation(line: 0, scope: !15)
+!15 = !DILexicalBlockFile(scope: !13, file: !1, discriminator: 0)
+!16 = !DILocalVariable(name: "sum2", scope: !15, file: !1, line: 11, type: !11)
diff --git a/test/Transforms/LICM/argmemonly-call.ll b/test/Transforms/LICM/argmemonly-call.ll
index b9e0818e80bebae070a8d4f839e237d6a3ac29ec..86c92a558fdd7849a36f0c697b4602150e2e8c02 100644
--- a/test/Transforms/LICM/argmemonly-call.ll
+++ b/test/Transforms/LICM/argmemonly-call.ll
@@ -2,6 +2,7 @@
 ; RUN: opt -licm -basicaa -licm-n2-threshold=200 < %s -S | FileCheck %s --check-prefix=ALIAS-N2
 ; RUN: opt -aa-pipeline=basic-aa -licm-n2-threshold=0 -passes='require<aa>,require<targetir>,require<scalar-evolution>,require<opt-remark-emit>,loop(licm)' < %s -S | FileCheck %s
 ; RUN: opt -aa-pipeline=basic-aa -licm-n2-threshold=200 -passes='require<aa>,require<targetir>,require<scalar-evolution>,require<opt-remark-emit>,loop(licm)' < %s -S | FileCheck %s --check-prefix=ALIAS-N2
+; RUN: opt -S -basicaa -licm -licm-n2-threshold=0 -enable-mssa-loop-dependency=true -verify-memoryssa %s | FileCheck %s --check-prefix=ALIAS-N2
 
 declare i32 @foo() readonly argmemonly nounwind
 declare i32 @foo2() readonly nounwind
@@ -11,6 +12,9 @@ define void @test(i32* %loc) {
 ; CHECK-LABEL: @test
 ; CHECK: @foo
 ; CHECK-LABEL: loop:
+; ALIAS-N2-LABEL: @test
+; ALIAS-N2: @foo
+; ALIAS-N2-LABEL: loop:
   br label %loop
 
 loop:
@@ -24,6 +28,9 @@ define void @test_neg(i32* %loc) {
 ; CHECK-LABEL: @test_neg
 ; CHECK-LABEL: loop:
 ; CHECK: @foo
+; ALIAS-N2-LABEL: @test_neg
+; ALIAS-N2-LABEL: loop:
+; ALIAS-N2: @foo
   br label %loop
 
 loop:
@@ -36,6 +43,9 @@ define void @test2(i32* noalias %loc, i32* noalias %loc2) {
 ; CHECK-LABEL: @test2
 ; CHECK: @bar
 ; CHECK-LABEL: loop:
+; ALIAS-N2-LABEL: @test2
+; ALIAS-N2: @bar
+; ALIAS-N2-LABEL: loop:
   br label %loop
 
 loop:
@@ -49,6 +59,9 @@ define void @test3(i32* %loc) {
 ; CHECK-LABEL: @test3
 ; CHECK-LABEL: loop:
 ; CHECK: @bar
+; ALIAS-N2-LABEL: @test3
+; ALIAS-N2-LABEL: loop:
+; ALIAS-N2: @bar
   br label %loop
 
 loop:
@@ -64,6 +77,9 @@ define void @test4(i32* %loc, i32* %loc2) {
 ; CHECK-LABEL: @test4
 ; CHECK-LABEL: loop:
 ; CHECK: @bar
+; ALIAS-N2-LABEL: @test4
+; ALIAS-N2-LABEL: loop:
+; ALIAS-N2: @bar
   br label %loop
 
 loop:
@@ -77,6 +93,7 @@ declare i32 @foo_new(i32*) readonly
 ; we clump foo_new with bar.
 ; With the N2 Alias analysis diagnostic tool, we are able to hoist the
 ; argmemonly bar call out of the loop.
+; Using MemorySSA we can also hoist bar.
 
 define void @test5(i32* %loc2, i32* noalias %loc) {
 ; ALIAS-N2-LABEL: @test5
@@ -103,6 +120,10 @@ define void @test6(i32* noalias %loc, i32* noalias %loc2) {
 ; CHECK: %val = load i32, i32* %loc2
 ; CHECK-LABEL: loop:
 ; CHECK: @llvm.memcpy
+; ALIAS-N2-LABEL: @test6
+; ALIAS-N2: %val = load i32, i32* %loc2
+; ALIAS-N2-LABEL: loop:
+; ALIAS-N2: @llvm.memcpy
   br label %loop
 
 loop:
@@ -119,6 +140,10 @@ define void @test7(i32* noalias %loc, i32* noalias %loc2) {
 ; CHECK: %val = load i32, i32* %loc2
 ; CHECK-LABEL: loop:
 ; CHECK: @custom_memcpy
+; ALIAS-N2-LABEL: @test7
+; ALIAS-N2: %val = load i32, i32* %loc2
+; ALIAS-N2-LABEL: loop:
+; ALIAS-N2: @custom_memcpy
   br label %loop
 
 loop:
diff --git a/test/Transforms/LICM/hoist-bitcast-load.ll b/test/Transforms/LICM/hoist-bitcast-load.ll
index 956c7283be315588ab2347aa82ebd6576d89125a..fcd6d08f2b47e16a9ed756173c2b23a05085f51b 100644
--- a/test/Transforms/LICM/hoist-bitcast-load.ll
+++ b/test/Transforms/LICM/hoist-bitcast-load.ll
@@ -1,5 +1,6 @@
 ; RUN: opt -S -basicaa -licm < %s | FileCheck %s
 ; RUN: opt -aa-pipeline=basic-aa -passes='require<opt-remark-emit>,loop(simplify-cfg,licm)' -S < %s | FileCheck %s
+; RUN: opt -S -basicaa -licm -enable-mssa-loop-dependency=true -verify-memoryssa < %s | FileCheck %s
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
diff --git a/test/Transforms/LICM/hoist-debuginvariant.ll b/test/Transforms/LICM/hoist-debuginvariant.ll
index 95c4018e46c1cd9202d667fb70aed63baf64b150..71aeb7549cef8bc2c82aa64f2e112b588cf684c0 100644
--- a/test/Transforms/LICM/hoist-debuginvariant.ll
+++ b/test/Transforms/LICM/hoist-debuginvariant.ll
@@ -1,5 +1,6 @@
 ; RUN: opt < %s -licm -S | FileCheck %s
 ; RUN: opt < %s -strip-debug -licm -S | FileCheck %s
+; RUN: opt < %s -licm -enable-mssa-loop-dependency=true -verify-memoryssa -S | FileCheck %s --check-prefixes=CHECK,MSSA
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
@@ -16,7 +17,9 @@ define void @fn1() !dbg !6 {
 ; CHECK-NEXT: [[_TMP2:%.*]] = load i32, i32* @a, align 4
 ; CHECK-NEXT: [[_TMP3:%.*]] = load i32, i32* @b, align 4
 ; CHECK-NEXT: [[_TMP4:%.*]] = sdiv i32 [[_TMP2]], [[_TMP3]]
+; MSSA-NEXT: store i32 [[_TMP4:%.*]], i32* @c, align 4
 ; CHECK-NEXT: br label [[BB3:%.*]]
+
   br label %bb3
 
 bb3:                                              ; preds = %bb3, %0
diff --git a/test/Transforms/LICM/hoist-deref-load.ll b/test/Transforms/LICM/hoist-deref-load.ll
index 6c9a8177408b44159407c05782f6bc0217727932..aacff88ac12ec78d6104570be5222be2312499af 100644
--- a/test/Transforms/LICM/hoist-deref-load.ll
+++ b/test/Transforms/LICM/hoist-deref-load.ll
@@ -1,5 +1,7 @@
 ; RUN: opt -S -basicaa -licm < %s | FileCheck %s
 ; RUN: opt -aa-pipeline=basic-aa -passes='require<opt-remark-emit>,loop(simplify-cfg,licm)' -S < %s | FileCheck %s
+; RUN: opt -S -basicaa -licm -enable-mssa-loop-dependency=true -verify-memoryssa < %s | FileCheck %s
+; RUN: opt -aa-pipeline=basic-aa -passes='require<opt-remark-emit>,loop(simplify-cfg,licm)' -enable-mssa-loop-dependency=true -verify-memoryssa -S < %s | FileCheck %s
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
diff --git a/test/Transforms/LICM/hoist-fast-fdiv.ll b/test/Transforms/LICM/hoist-fast-fdiv.ll
index 57df241a2cea33195ee9d509b0a45c5a37e4902f..bdefcf9e8f46217a326e9d0c907cb580efaaa6db 100644
--- a/test/Transforms/LICM/hoist-fast-fdiv.ll
+++ b/test/Transforms/LICM/hoist-fast-fdiv.ll
@@ -1,4 +1,5 @@
 ; RUN: opt -licm -S < %s | FileCheck %s
+; RUN: opt -licm -enable-mssa-loop-dependency=true -verify-memoryssa -S < %s | FileCheck %s
 
 ; Function Attrs: noinline norecurse nounwind readnone ssp uwtable
 define zeroext i1 @invariant_denom(double %v) #0 {
diff --git a/test/Transforms/LICM/hoist-invariant-load.ll b/test/Transforms/LICM/hoist-invariant-load.ll
index ed669f383fc3fb06d5af3252822df2d1b6af1ece..909652b8f46c934203bebc836bad969f92ec8e50 100644
--- a/test/Transforms/LICM/hoist-invariant-load.ll
+++ b/test/Transforms/LICM/hoist-invariant-load.ll
@@ -1,5 +1,6 @@
 ; REQUIRES: asserts
 ; RUN: opt < %s -licm -disable-basicaa -stats -S 2>&1 | grep "1 licm"
+; RUN: opt < %s -licm -enable-mssa-loop-dependency=true -verify-memoryssa -disable-basicaa -stats -S 2>&1 | grep "1 licm"
 
 @"\01L_OBJC_METH_VAR_NAME_" = internal global [4 x i8] c"foo\00", section "__TEXT,__objc_methname,cstring_literals", align 1
 @"\01L_OBJC_SELECTOR_REFERENCES_" = internal global i8* getelementptr inbounds ([4 x i8], [4 x i8]* @"\01L_OBJC_METH_VAR_NAME_", i32 0, i32 0), section "__DATA, __objc_selrefs, literal_pointers, no_dead_strip"
diff --git a/test/Transforms/LICM/hoist-nounwind.ll b/test/Transforms/LICM/hoist-nounwind.ll
index d53e4043af14086b6becec3b1f108e9b517fc80b..a582173a156a9c66f91bd678d6d157c05424e1be 100644
--- a/test/Transforms/LICM/hoist-nounwind.ll
+++ b/test/Transforms/LICM/hoist-nounwind.ll
@@ -1,5 +1,6 @@
 ; RUN: opt -S -basicaa -licm < %s | FileCheck %s
 ; RUN: opt -aa-pipeline=basic-aa -passes='require<opt-remark-emit>,loop(licm)' -S %s | FileCheck %s
+; RUN: opt -S -basicaa -licm -enable-mssa-loop-dependency=true -verify-memoryssa < %s | FileCheck %s
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
diff --git a/test/Transforms/LICM/hoist-phi.ll b/test/Transforms/LICM/hoist-phi.ll
index 26d0c0114f6be1be3fda63dce451bf31e55b79d0..f3a137db5f3ca393c9c1b71bfecc10bb4eec2469 100644
--- a/test/Transforms/LICM/hoist-phi.ll
+++ b/test/Transforms/LICM/hoist-phi.ll
@@ -5,6 +5,11 @@
 ; RUN: opt -passes='require<opt-remark-emit>,loop(licm)' -licm-control-flow-hoisting=1 -S < %s | FileCheck %s -check-prefixes=CHECK,CHECK-ENABLED
 ; RUN: opt -passes='require<opt-remark-emit>,loop(licm)' -licm-control-flow-hoisting=0 -S < %s | FileCheck %s -check-prefixes=CHECK,CHECK-DISABLED
 
+; RUN: opt -passes='require<opt-remark-emit>,loop(licm)' -licm-control-flow-hoisting=1 -enable-mssa-loop-dependency=true -verify-memoryssa -S < %s | FileCheck %s -check-prefixes=CHECK,CHECK-ENABLED
+; Enable run below when adding promotion. e.g. "store i32 %phi, i32* %p" is promoted to phi.lcssa.
+; opt -passes='require<opt-remark-emit>,loop(licm)' -licm-control-flow-hoisting=0 -enable-mssa-loop-dependency=true -verify-memoryssa -S < %s | FileCheck %s -check-prefixes=CHECK,CHECK-DISABLED
+
+
 ; CHECK-LABEL: @triangle_phi
 define void @triangle_phi(i32 %x, i32* %p) {
 ; CHECK-LABEL: entry:
@@ -1349,3 +1354,167 @@ if.then3:
 loop.backedge:
   br label %loop
 }
+
+; The order that we hoist instructions from the loop is different to the textual
+; order in the function. Check that we can rehoist this correctly.
+; CHECK-LABEL: @rehoist_wrong_order_1
+define void @rehoist_wrong_order_1(i32* %ptr) {
+; CHECK-LABEL: entry
+; CHECK-DAG: %gep2 = getelementptr inbounds i32, i32* %ptr, i64 2
+; CHECK-DAG: %gep3 = getelementptr inbounds i32, i32* %ptr, i64 3
+; CHECK-DAG: %gep1 = getelementptr inbounds i32, i32* %ptr, i64 1
+; CHECK-ENABLED: br i1 undef, label %[[IF1_LICM:.*]], label %[[ELSE1_LICM:.*]]
+entry:
+  br label %loop
+
+; CHECK-ENABLED: [[IF1_LICM]]:
+; CHECK-ENABLED: br label %[[LOOP_BACKEDGE_LICM:.*]]
+
+; CHECK-ENABLED: [[ELSE1_LICM]]:
+; CHECK-ENABLED: br label %[[LOOP_BACKEDGE_LICM]]
+
+; CHECK-ENABLED: [[LOOP_BACKEDGE_LICM]]:
+; CHECK-ENABLED: br i1 undef, label %[[IF3_LICM:.*]], label %[[END_LICM:.*]]
+
+; CHECK-ENABLED: [[IF3_LICM]]:
+; CHECK-ENABLED: br label %[[END_LICM]]
+
+; CHECK-ENABLED: [[END_LICM]]:
+; CHECK: br label %loop
+
+loop:
+  br i1 undef, label %if1, label %else1
+
+if1:
+  %gep1 = getelementptr inbounds i32, i32* %ptr, i64 1
+  store i32 0, i32* %gep1, align 4
+  br label %loop.backedge
+
+else1:
+  %gep2 = getelementptr inbounds i32, i32* %ptr, i64 2
+  store i32 0, i32* %gep2, align 4
+  br i1 undef, label %if2, label %loop.backedge
+
+if2:
+  br i1 undef, label %if3, label %end
+
+if3:
+  %gep3 = getelementptr inbounds i32, i32* %ptr, i64 3
+  store i32 0, i32* %gep3, align 4
+  br label %end
+
+end:
+  br label %loop.backedge
+
+loop.backedge:
+  br label %loop
+
+}
+
+; CHECK-LABEL: @rehoist_wrong_order_2
+define void @rehoist_wrong_order_2(i32* %ptr) {
+; CHECK-LABEL: entry
+; CHECK-DAG: %gep2 = getelementptr inbounds i32, i32* %ptr, i64 2
+; CHECK-DAG: %gep3 = getelementptr inbounds i32, i32* %gep2, i64 3
+; CHECK-DAG: %gep1 = getelementptr inbounds i32, i32* %ptr, i64 1
+; CHECK-ENABLED: br i1 undef, label %[[IF1_LICM:.*]], label %[[ELSE1_LICM:.*]]
+entry:
+  br label %loop
+
+; CHECK-ENABLED: [[IF1_LICM]]:
+; CHECK-ENABLED: br label %[[LOOP_BACKEDGE_LICM:.*]]
+
+; CHECK-ENABLED: [[ELSE1_LICM]]:
+; CHECK-ENABLED: br label %[[LOOP_BACKEDGE_LICM]]
+
+; CHECK-ENABLED: [[LOOP_BACKEDGE_LICM]]:
+; CHECK-ENABLED: br i1 undef, label %[[IF3_LICM:.*]], label %[[END_LICM:.*]]
+
+; CHECK-ENABLED: [[IF3_LICM]]:
+; CHECK-ENABLED: br label %[[END_LICM]]
+
+; CHECK-ENABLED: [[END_LICM]]:
+; CHECK: br label %loop
+
+loop:
+  br i1 undef, label %if1, label %else1
+
+if1:
+  %gep1 = getelementptr inbounds i32, i32* %ptr, i64 1
+  store i32 0, i32* %gep1, align 4
+  br label %loop.backedge
+
+else1:
+  %gep2 = getelementptr inbounds i32, i32* %ptr, i64 2
+  store i32 0, i32* %gep2, align 4
+  br i1 undef, label %if2, label %loop.backedge
+
+if2:
+  br i1 undef, label %if3, label %end
+
+if3:
+  %gep3 = getelementptr inbounds i32, i32* %gep2, i64 3
+  store i32 0, i32* %gep3, align 4
+  br label %end
+
+end:
+  br label %loop.backedge
+
+loop.backedge:
+  br label %loop
+}
+
+; CHECK-LABEL: @rehoist_wrong_order_3
+define void @rehoist_wrong_order_3(i32* %ptr) {
+; CHECK-LABEL: entry
+; CHECK-DAG: %gep2 = getelementptr inbounds i32, i32* %ptr, i64 2
+; CHECK-DAG: %gep1 = getelementptr inbounds i32, i32* %ptr, i64 1
+; CHECK-ENABLED: br i1 undef, label %[[IF1_LICM:.*]], label %[[ELSE1_LICM:.*]]
+entry:
+  br label %loop
+
+; CHECK-ENABLED: [[IF1_LICM]]:
+; CHECK-ENABLED: br label %[[IF2_LICM:.*]]
+
+; CHECK-ENABLED: [[ELSE1_LICM]]:
+; CHECK-ENABLED: br label %[[IF2_LICM]]
+
+; CHECK-ENABLED: [[IF2_LICM]]:
+; CHECK-ENABLED: %phi = phi i32* [ %gep1, %[[IF1_LICM]] ], [ %gep2, %[[ELSE1_LICM]] ]
+; CHECK-ENABLED: %gep3 = getelementptr inbounds i32, i32* %phi, i64 3
+; CHECK-ENABLED: br i1 undef, label %[[IF3_LICM:.*]], label %[[END_LICM:.*]]
+
+; CHECK-ENABLED: [[IF3_LICM]]:
+; CHECK-ENABLED: br label %[[END_LICM]]
+
+; CHECK-ENABLED: [[END_LICM]]:
+; CHECK: br label %loop
+
+loop:
+  br i1 undef, label %if1, label %else1
+
+if1:
+  %gep1 = getelementptr inbounds i32, i32* %ptr, i64 1
+  store i32 0, i32* %gep1, align 4
+  br label %if2
+
+else1:
+  %gep2 = getelementptr inbounds i32, i32* %ptr, i64 2
+  store i32 0, i32* %gep2, align 4
+  br i1 undef, label %if2, label %loop.backedge
+
+if2:
+  %phi = phi i32* [ %gep1, %if1 ], [ %gep2, %else1 ]
+  br i1 undef, label %if3, label %end
+
+if3:
+  %gep3 = getelementptr inbounds i32, i32* %phi, i64 3
+  store i32 0, i32* %gep3, align 4
+  br label %end
+
+end:
+  br label %loop.backedge
+
+loop.backedge:
+  br label %loop
+}
diff --git a/test/Transforms/LICM/hoist-round.ll b/test/Transforms/LICM/hoist-round.ll
index 35851f39d2596000a4d4e7b7f87991a285921a98..10f75be4d3270304d647b270b40120eeb4794700 100644
--- a/test/Transforms/LICM/hoist-round.ll
+++ b/test/Transforms/LICM/hoist-round.ll
@@ -1,5 +1,6 @@
 ; RUN: opt -S -licm < %s | FileCheck %s
 ; RUN: opt -aa-pipeline=basic-aa -passes='require<aa>,require<targetir>,require<scalar-evolution>,require<opt-remark-emit>,loop(licm)' -S %s | FileCheck %s
+; RUN: opt -S -licm -enable-mssa-loop-dependency=true -verify-memoryssa < %s | FileCheck %s
 
 target datalayout = "E-m:e-p:32:32-i8:8:8-i16:16:16-i64:32:32-f64:32:32-v64:32:32-v128:32:32-a0:0:32-n32"
 
diff --git a/test/Transforms/LICM/hoisting.ll b/test/Transforms/LICM/hoisting.ll
index 4572ad38d3cd85bbf2e68a66445a567718730572..f65b965d5550f35b66be9d118f8d965f246d83bf 100644
--- a/test/Transforms/LICM/hoisting.ll
+++ b/test/Transforms/LICM/hoisting.ll
@@ -1,5 +1,6 @@
 ; RUN: opt < %s -licm -S | FileCheck %s
 ; RUN: opt < %s -aa-pipeline=basic-aa -passes='require<opt-remark-emit>,loop(licm)' -S | FileCheck %s
+; RUN: opt < %s -licm -enable-mssa-loop-dependency=true -verify-memoryssa -S | FileCheck %s
 
 @X = global i32 0		; <i32*> [#uses=1]
 
diff --git a/test/Transforms/LICM/pr40317.ll b/test/Transforms/LICM/pr40317.ll
new file mode 100644
index 0000000000000000000000000000000000000000..88afbdc496caf810b3135f1ca58b0aa0d7423cf5
--- /dev/null
+++ b/test/Transforms/LICM/pr40317.ll
@@ -0,0 +1,62 @@
+; RUN: opt -S -march=z13 -tbaa -licm -enable-mssa-loop-dependency -licm-control-flow-hoisting -verify-memoryssa < %s | FileCheck %s
+
+target datalayout = "E-m:e-i1:8:16-i8:8:16-i64:64-f128:64-v128:64-a:8:16-n32:64"
+target triple = "s390x-ibm-linux"
+
+%0 = type { %1, %1, i16, %2 }
+%1 = type <{ i16, i8, i32, i32, i32, i64, i64 }>
+%2 = type { i8, i16, i16, [2 x i8] }
+
+@0 = internal global %0 { %1 <{ i16 22437, i8 117, i32 2017322857, i32 900074563, i32 -1390364, i64 0, i64 0 }>, %1 <{ i16 0, i8 7, i32 -387299562, i32 925371866, i32 -1, i64 4826244575317081679, i64 1 }>, i16 8, %2 { i8 0, i16 0, i16 3, [2 x i8] undef } }, align 2
+@g_18 = external dso_local global i64, align 8
+
+; Function Attrs: argmemonly nounwind
+declare void @llvm.memset.p0i8.i64(i8* nocapture writeonly, i8, i64, i1) #1
+
+; CHECK-LABEL: @func_94
+; CHECK: bb:
+; CHECK: tail call void @llvm.memset.p0i8.i64
+; CHECK: load i32
+; CHECK: bb6.licm:
+; Function Attrs: noreturn nounwind
+define dso_local void @func_94(i16 %arg, i64* nocapture %arg1) local_unnamed_addr #3 {
+bb:
+  tail call void @llvm.memset.p0i8.i64(i8* align 8 undef, i8 0, i64 80, i1 false)
+  br label %bb3
+
+bb3:                                              ; preds = %bb13, %bb
+  %tmp5 = icmp eq i16 %arg, 0
+  br i1 %tmp5, label %bb6, label %bb13
+
+bb6:                                              ; preds = %bb3
+  %tmp7 = load i32, i32* getelementptr inbounds (%0, %0* @0, i64 0, i32 1, i32 2), align 1, !tbaa !11
+  %tmp8 = zext i32 %tmp7 to i64
+  %sext = shl i64 %tmp8, 56
+  %tmp10 = ashr exact i64 %sext, 56
+  store i64 %tmp10, i64* %arg1, align 8, !tbaa !12
+  br label %bb13
+
+bb13:                                             ; preds = %bb3, %bb6
+  br label %bb3
+}
+
+attributes #0 = { "use-soft-float"="false" }
+attributes #1 = { argmemonly nounwind }
+attributes #2 = { norecurse nounwind readnone "use-soft-float"="false" }
+attributes #3 = { noreturn nounwind "use-soft-float"="false" }
+
+!llvm.ident = !{!0}
+
+!0 = !{!"clang version 8.0.0 (http://llvm.org/git/clang.git e593a791f2cf19db84237b0b9d632e9966a00a39) (http://llvm.org/git/llvm.git fe0523d1bd7def3ef62cfb3dd37a8b1941aafa81)"}
+!1 = !{!2, !8, i64 46}
+!2 = !{!"S5", !3, i64 0, !3, i64 31, !4, i64 62, !9, i64 64}
+!3 = !{!"S2", !4, i64 0, !5, i64 2, !7, i64 3, !7, i64 7, !7, i64 11, !8, i64 15, !8, i64 23}
+!4 = !{!"short", !5, i64 0}
+!5 = !{!"omnipotent char", !6, i64 0}
+!6 = !{!"Simple C/C++ TBAA"}
+!7 = !{!"int", !5, i64 0}
+!8 = !{!"long", !5, i64 0}
+!9 = !{!"S3", !7, i64 0, !4, i64 2, !4, i64 4}
+!10 = !{!2, !7, i64 42}
+!11 = !{!2, !7, i64 34}
+!12 = !{!8, !8, i64 0}
diff --git a/test/Transforms/LICM/sink-promote.ll b/test/Transforms/LICM/sink-promote.ll
new file mode 100644
index 0000000000000000000000000000000000000000..45024c46da958e8e96302072dec8ff8f5a57ab2a
--- /dev/null
+++ b/test/Transforms/LICM/sink-promote.ll
@@ -0,0 +1,50 @@
+; RUN: opt < %s -basicaa -licm -S | FileCheck %s
+
+; Test moved from sinking.ll, as it tests sinking of a store who alone touches
+; a memory location in a loop.
+; Store can be sunk out of exit block containing indirectbr instructions after
+; D50925. Updated to use an argument instead of undef, due to PR38989.
+define void @test12(i32* %ptr) {
+; CHECK-LABEL: @test12
+; CHECK: store
+; CHECK-NEXT: br label %lab4
+  br label %lab4
+
+lab4:
+  br label %lab20
+
+lab5:
+  br label %lab20
+
+lab6:
+  br label %lab4
+
+lab7:
+  br i1 undef, label %lab8, label %lab13
+
+lab8:
+  br i1 undef, label %lab13, label %lab10
+
+lab10:
+  br label %lab7
+
+lab13:
+  ret void
+
+lab20:
+  br label %lab21
+
+lab21:
+; CHECK: lab21:
+; CHECK-NOT: store
+; CHECK: br i1 false, label %lab21, label %lab22
+  store i32 36127957, i32* %ptr, align 4
+  br i1 undef, label %lab21, label %lab22
+
+lab22:
+; CHECK: lab22:
+; CHECK-NOT: store
+; CHECK-NEXT: indirectbr i8* undef
+  indirectbr i8* undef, [label %lab5, label %lab6, label %lab7]
+}
+
diff --git a/test/Transforms/LICM/sink.ll b/test/Transforms/LICM/sink.ll
index 70fa6fa13e3e245bb4db9c325069dbf21b280586..17170f5af19651fe4803b623ba7753a001177c16 100644
--- a/test/Transforms/LICM/sink.ll
+++ b/test/Transforms/LICM/sink.ll
@@ -2,6 +2,7 @@
 ; RUN: opt -S -licm < %s | opt -S -loop-sink | FileCheck %s --check-prefix=CHECK-SINK
 ; RUN: opt -S < %s -passes='require<opt-remark-emit>,loop(licm),loop-sink' \
 ; RUN:     | FileCheck %s --check-prefix=CHECK-SINK
+; RUN: opt -S -licm -enable-mssa-loop-dependency=true -verify-memoryssa < %s | FileCheck %s --check-prefix=CHECK-LICM
 
 ; Original source code:
 ; int g;
diff --git a/test/Transforms/LICM/sinking.ll b/test/Transforms/LICM/sinking.ll
index 3bcbc5d67db735bdbe65642d4d47b713b7543771..cc30494a518e4c6e3acbbe18d38d7dc309c96440 100644
--- a/test/Transforms/LICM/sinking.ll
+++ b/test/Transforms/LICM/sinking.ll
@@ -1,5 +1,7 @@
 ; RUN: opt < %s -basicaa -licm -S | FileCheck %s
 ; RUN: opt < %s -debugify -basicaa -licm -S | FileCheck %s -check-prefix=DEBUGIFY
+; RUN: opt < %s -basicaa -licm -S -enable-mssa-loop-dependency=true -verify-memoryssa | FileCheck %s
+
 
 declare i32 @strlen(i8*) readonly nounwind
 
@@ -358,50 +360,7 @@ exit:
   ret i32 %lcssa
 }
 
-; Can't sink stores out of exit blocks containing indirectbr instructions
-; because loop simplify does not create dedicated exits for such blocks. Test
-; that by sinking the store from lab21 to lab22, but not further.
-define void @test12() {
-; CHECK-LABEL: @test12
-  br label %lab4
-
-lab4:
-  br label %lab20
-
-lab5:
-  br label %lab20
-
-lab6:
-  br label %lab4
-
-lab7:
-  br i1 undef, label %lab8, label %lab13
-
-lab8:
-  br i1 undef, label %lab13, label %lab10
-
-lab10:
-  br label %lab7
-
-lab13:
-  ret void
-
-lab20:
-  br label %lab21
-
-lab21:
-; CHECK: lab21:
-; CHECK-NOT: store
-; CHECK: br i1 false, label %lab21, label %lab22
-  store i32 36127957, i32* undef, align 4
-  br i1 undef, label %lab21, label %lab22
-
-lab22:
-; CHECK: lab22:
-; CHECK: store
-; CHECK-NEXT: indirectbr i8* undef
-  indirectbr i8* undef, [label %lab5, label %lab6, label %lab7]
-}
+; @test12 moved to sink-promote.ll, as it tests sinking and promotion.
 
 ; Test that we don't crash when trying to sink stores and there's no preheader
 ; available (which is used for creating loads that may be used by the SSA
diff --git a/test/Transforms/LICM/store-hoisting.ll b/test/Transforms/LICM/store-hoisting.ll
index 1b1497857a29c5cbbe9d573e0dbe631fe86772ac..3f7f3042febda447ab8657de38d8e06cfeabc885 100644
--- a/test/Transforms/LICM/store-hoisting.ll
+++ b/test/Transforms/LICM/store-hoisting.ll
@@ -1,5 +1,6 @@
-; RUN: opt -S -basicaa -licm %s | FileCheck %s
-; RUN: opt -aa-pipeline=basic-aa -passes='require<aa>,require<targetir>,require<scalar-evolution>,require<opt-remark-emit>,loop(licm)' < %s -S | FileCheck %s
+; RUN: opt -S -basicaa -licm %s | FileCheck -check-prefixes=CHECK,AST %s
+; RUN: opt -S -basicaa -licm -enable-mssa-loop-dependency=true %s | FileCheck  -check-prefixes=CHECK,MSSA %s
+; RUN: opt -aa-pipeline=basic-aa -passes='require<aa>,require<targetir>,require<scalar-evolution>,require<opt-remark-emit>,loop(licm)' < %s -S | FileCheck -check-prefixes=CHECK,AST %s
 
 define void @test(i32* %loc) {
 ; CHECK-LABEL: @test
@@ -46,8 +47,11 @@ exit2:
 
 define i32* @false_negative_2use(i32* %loc) {
 ; CHECK-LABEL: @false_negative_2use
-; CHECK-LABEL: exit:
-; CHECK: store i32 0, i32* %loc
+; AST-LABEL: exit:
+; AST: store i32 0, i32* %loc
+; MSSA-LABEL: entry:
+; MSSA: store i32 0, i32* %loc
+; MSSA-LABEL: loop:
 entry:
   br label %loop
 
@@ -119,6 +123,7 @@ exit:
   ret void
 }
 
+; Hoisting the store is actually valid here, as it dominates the load.
 define void @neg_ref(i32* %loc) {
 ; CHECK-LABEL: @neg_ref
 ; CHECK-LABEL: exit1:
@@ -146,6 +151,35 @@ exit2:
   ret void
 }
 
+; Hoisting the store here leads to a miscompile.
+define void @neg_ref2(i32* %loc) {
+; CHECK-LABEL: @neg_ref2
+; CHECK-LABEL: exit1:
+; CHECK: store i32 0, i32* %loc
+; CHECK-LABEL: exit2:
+; CHECK: store i32 0, i32* %loc
+entry:
+  store i32 198, i32* %loc
+  br label %loop
+
+loop:
+  %iv = phi i32 [0, %entry], [%iv.next, %backedge]
+  %v = load i32, i32* %loc
+  store i32 0, i32* %loc
+  %earlycnd = icmp eq i32 %v, 198
+  br i1 %earlycnd, label %exit1, label %backedge
+  
+backedge:
+  %iv.next = add i32 %iv, 1
+  %cmp = icmp slt i32 %iv, 200
+  br i1 %cmp, label %loop, label %exit2
+
+exit1:
+  ret void
+exit2:
+  ret void
+}
+
 declare void @modref()
 
 define void @neg_modref(i32* %loc) {
diff --git a/test/Transforms/LICM/volatile-alias.ll b/test/Transforms/LICM/volatile-alias.ll
index f387012015c72340e1face9b78d1dbd189796bfd..c1f727c4e38d5eb107ba91fd9cd56f05f105bd09 100644
--- a/test/Transforms/LICM/volatile-alias.ll
+++ b/test/Transforms/LICM/volatile-alias.ll
@@ -1,5 +1,6 @@
 ; RUN: opt -basicaa -sroa -loop-rotate -licm -S < %s | FileCheck %s
 ; RUN: opt -basicaa -sroa -loop-rotate %s | opt -aa-pipeline=basic-aa -passes='require<aa>,require<targetir>,require<scalar-evolution>,require<opt-remark-emit>,loop(licm)' -S | FileCheck %s
+; RUN: opt -basicaa -sroa -loop-rotate -licm -enable-mssa-loop-dependency=true -verify-memoryssa -S < %s | FileCheck %s
 ; The objects *p and *q are aliased to each other, but even though *q is
 ; volatile, *p can be considered invariant in the loop. Check if it is moved
 ; out of the loop.
diff --git a/test/Transforms/LoopDeletion/crashbc.ll b/test/Transforms/LoopDeletion/crashbc.ll
index 056c61f77aea18cd49b3d2b65f34af2ec18ce587..f385a29c6cdbfc4686866d3ddd9e3633f893a16c 100644
--- a/test/Transforms/LoopDeletion/crashbc.ll
+++ b/test/Transforms/LoopDeletion/crashbc.ll
@@ -1,5 +1,5 @@
 ; Make sure we don't crash when writing bitcode.
-; RUN: opt -loop-deletion -o /dev/null
+; RUN: opt < %s -loop-deletion -o /dev/null
 
 define void @f() {
   br label %bb1
diff --git a/test/Transforms/LoopIdiom/X86/cttz.ll b/test/Transforms/LoopIdiom/X86/cttz.ll
new file mode 100644
index 0000000000000000000000000000000000000000..e18fecabe86efdcb7f0fe4127cb4e046cb8ed099
--- /dev/null
+++ b/test/Transforms/LoopIdiom/X86/cttz.ll
@@ -0,0 +1,82 @@
+; RUN: opt -loop-idiom -mtriple=x86_64 -mcpu=core-avx2 < %s -S | FileCheck --check-prefix=ALL %s
+; RUN: opt -loop-idiom -mtriple=x86_64 -mcpu=corei7 < %s -S | FileCheck --check-prefix=ALL %s
+
+; Recognize CTTZ builtin pattern.
+; Here it will replace the loop -
+; assume builtin is always profitable.
+;
+; int cttz_zero_check(int n)
+; {
+;   int i = 0;
+;   while(n) {
+;     n <<= 1;
+;     i++;
+;   }
+;   return i;
+; }
+;
+; ALL-LABEL: @cttz_zero_check
+; ALL:       %0 = call i32 @llvm.cttz.i32(i32 %n, i1 true)
+; ALL-NEXT:  %1 = sub i32 32, %0
+;
+; Function Attrs: norecurse nounwind readnone uwtable
+define i32 @cttz_zero_check(i32 %n) {
+entry:
+  %tobool4 = icmp eq i32 %n, 0
+  br i1 %tobool4, label %while.end, label %while.body.preheader
+
+while.body.preheader:                             ; preds = %entry
+  br label %while.body
+
+while.body:                                       ; preds = %while.body.preheader, %while.body
+  %i.06 = phi i32 [ %inc, %while.body ], [ 0, %while.body.preheader ]
+  %n.addr.05 = phi i32 [ %shl, %while.body ], [ %n, %while.body.preheader ]
+  %shl = shl i32 %n.addr.05, 1
+  %inc = add nsw i32 %i.06, 1
+  %tobool = icmp eq i32 %shl, 0
+  br i1 %tobool, label %while.end.loopexit, label %while.body
+
+while.end.loopexit:                               ; preds = %while.body
+  br label %while.end
+
+while.end:                                        ; preds = %while.end.loopexit, %entry
+  %i.0.lcssa = phi i32 [ 0, %entry ], [ %inc, %while.end.loopexit ]
+  ret i32 %i.0.lcssa
+}
+
+; Recognize CTTZ builtin pattern.
+; Here it will replace the loop -
+; assume builtin is always profitable.
+;
+; int cttz(int n)
+; {
+;   int i = 0;
+;   while(n <<= 1) {
+;     i++;
+;   }
+;   return i;
+; }
+;
+; ALL-LABEL: @cttz
+; ALL:      %0 = shl i32 %n, 1
+; ALL-NEXT: %1 = call i32 @llvm.cttz.i32(i32 %0, i1 false)
+; ALL-NEXT: %2 = sub i32 32, %1
+; ALL-NEXT: %3 = add i32 %2, 1
+;
+; Function Attrs: norecurse nounwind readnone uwtable
+define i32 @cttz(i32 %n) {
+entry:
+  br label %while.cond
+
+while.cond:                                       ; preds = %while.cond, %entry
+  %n.addr.0 = phi i32 [ %n, %entry ], [ %shl, %while.cond ]
+  %i.0 = phi i32 [ 0, %entry ], [ %inc, %while.cond ]
+  %shl = shl i32 %n.addr.0, 1
+  %tobool = icmp eq i32 %shl, 0
+  %inc = add nsw i32 %i.0, 1
+  br i1 %tobool, label %while.end, label %while.cond
+
+while.end:                                        ; preds = %while.cond
+  ret i32 %i.0
+}
+
diff --git a/test/Transforms/LoopIdiom/dbginfo-cost.ll b/test/Transforms/LoopIdiom/dbginfo-cost.ll
new file mode 100644
index 0000000000000000000000000000000000000000..b8c33a5b1a8bf21823d8812db4ed5e60b1233bbb
--- /dev/null
+++ b/test/Transforms/LoopIdiom/dbginfo-cost.ll
@@ -0,0 +1,68 @@
+; RUN: opt -S -loop-idiom -mtriple=systemz-unknown -mcpu=z13 %s | FileCheck %s
+
+; CHECK: @llvm.ctlz.i32
+
+declare void @llvm.dbg.declare(metadata, metadata, metadata) #0
+define dso_local i32 @CeilLog2(i32 %arg) local_unnamed_addr #1 !dbg !38 {
+bb:
+  %tmp4 = add i32 %arg, -1, !dbg !45
+  call void @llvm.dbg.value(metadata i32 0, metadata !44, metadata !DIExpression()), !dbg !45
+  %tmp71 = icmp eq i32 %tmp4, 0, !dbg !45
+  br i1 %tmp71, label %bb13, label %bb8.preheader, !dbg !48
+
+bb8.preheader:                                    ; preds = %bb
+  br label %bb8, !dbg !49
+
+bb8:                                              ; preds = %bb8.preheader, %bb8
+  %tmp2.03 = phi i32 [ %tmp12, %bb8 ], [ 0, %bb8.preheader ]
+  %tmp1.02 = phi i32 [ %tmp10, %bb8 ], [ %tmp4, %bb8.preheader ]
+  call void @llvm.dbg.value(metadata i32 %tmp2.03, metadata !44, metadata !DIExpression()), !dbg !45
+  %tmp10 = lshr i32 %tmp1.02, 1, !dbg !49
+  %tmp12 = add nuw nsw i32 %tmp2.03, 1, !dbg !51
+  call void @llvm.dbg.value(metadata i32 %tmp12, metadata !44, metadata !DIExpression()), !dbg !45
+  %tmp7 = icmp eq i32 %tmp10, 0, !dbg !45
+  br i1 %tmp7, label %bb13.loopexit, label %bb8, !dbg !48, !llvm.loop !52
+
+bb13.loopexit:                                    ; preds = %bb8
+  %tmp12.lcssa = phi i32 [ %tmp12, %bb8 ], !dbg !51
+  br label %bb13, !dbg !54
+
+bb13:                                             ; preds = %bb13.loopexit, %bb
+  %tmp2.0.lcssa = phi i32 [ 0, %bb ], [ %tmp12.lcssa, %bb13.loopexit ], !dbg !55
+  call void @llvm.dbg.value(metadata i32 %tmp2.0.lcssa, metadata !44, metadata !DIExpression()), !dbg !45
+  ret i32 %tmp2.0.lcssa, !dbg !54
+}
+
+; Function Attrs: nounwind readnone speculatable
+declare void @llvm.dbg.value(metadata, metadata, metadata) #2
+
+attributes #0 = { nounwind readnone speculatable "target-cpu"="z13" }
+attributes #1 = { norecurse nounwind readnone "target-cpu"="z13" "use-soft-float"="false" }
+attributes #2 = { nounwind readnone speculatable }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!36, !37}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 9.0.0 (ijonpan@m35lp38.lnxne.boe:llvm/llvm-dev-2/tools/clang a87ff88c6466fbedd6281513b9480a2cad6c08c8) (llvm/llvm-dev-2 922a3b1b3254bf3310c467e880a5419c1e13c87f)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, retainedTypes: !2, globals: !2, nameTableKind: None)
+!1 = !DIFile(filename: "configfile.c", directory: "/home/ijonpan/minispec-2006/spec-llvm/464.h264ref/build")
+!2 = !{}
+!4 = !DIFile(filename: "./global.h", directory: "/home/ijonpan/minispec-2006/spec-llvm/464.h264ref/build")
+!5 = !DIBasicType(name: "unsigned int", size: 32, encoding: DW_ATE_unsigned)
+!36 = !{i32 2, !"Debug Info Version", i32 3}
+!37 = !{i32 1, !"wchar_size", i32 4}
+!38 = distinct !DISubprogram(name: "CeilLog2", scope: !1, file: !1, line: 599, type: !39, scopeLine: 600, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !41)
+!39 = !DISubroutineType(types: !40)
+!40 = !{!5, !5}
+!41 = !{!42, !43, !44}
+!42 = !DILocalVariable(name: "uiVal", arg: 1, scope: !38, file: !1, line: 599, type: !5)
+!43 = !DILocalVariable(name: "uiTmp", scope: !38, file: !1, line: 601, type: !5)
+!44 = !DILocalVariable(name: "uiRet", scope: !38, file: !1, line: 602, type: !5)
+!45 = !DILocation(line: 601, column: 25, scope: !38)
+!48 = !DILocation(line: 604, column: 3, scope: !38)
+!49 = !DILocation(line: 606, column: 11, scope: !50)
+!50 = distinct !DILexicalBlock(scope: !38, file: !1, line: 605, column: 3)
+!51 = !DILocation(line: 607, column: 10, scope: !50)
+!52 = distinct !{!52, !48, !53}
+!53 = !DILocation(line: 608, column: 3, scope: !38)
+!54 = !DILocation(line: 609, column: 3, scope: !38)
+!55 = !DILocation(line: 0, scope: !38)
diff --git a/test/Transforms/LoopPredication/basic_widenable_branch_guards.ll b/test/Transforms/LoopPredication/basic_widenable_branch_guards.ll
new file mode 100644
index 0000000000000000000000000000000000000000..c836e5f4d835965aff1c9c62a16c58905b04c3b1
--- /dev/null
+++ b/test/Transforms/LoopPredication/basic_widenable_branch_guards.ll
@@ -0,0 +1,1486 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -loop-predication -loop-predication-predicate-widenable-branches-to-deopt=true < %s 2>&1 | FileCheck %s
+; RUN: opt -S -passes='require<scalar-evolution>,loop(loop-predication)' -loop-predication-predicate-widenable-branches-to-deopt=true < %s 2>&1 | FileCheck %s
+
+declare void @llvm.experimental.guard(i1, ...)
+
+define i32 @unsigned_loop_0_to_n_ult_check(i32* %array, i32 %length, i32 %n) {
+; CHECK-LABEL: @unsigned_loop_0_to_n_ult_check(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP5]], label [[EXIT:%.*]], label [[LOOP_PREHEADER:%.*]]
+; CHECK:       loop.preheader:
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp ule i32 [[N]], [[LENGTH:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i32 0, [[LENGTH]]
+; CHECK-NEXT:    [[TMP2:%.*]] = and i1 [[TMP1]], [[TMP0]]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[GUARDED:%.*]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[GUARDED]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[WITHIN_BOUNDS:%.*]] = icmp ult i32 [[I]], [[LENGTH]]
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[WITHIN_BOUNDS]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    [[TMP3:%.*]] = and i1 [[TMP2]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[TMP3]], label [[GUARDED]], label [[DEOPT:%.*]], !prof !0
+;
+entry:
+  %tmp5 = icmp eq i32 %n, 0
+  br i1 %tmp5, label %exit, label %loop.preheader
+
+loop.preheader:                                   ; preds = %entry
+  br label %loop
+
+loop:                                             ; preds = %guarded, %loop.preheader
+  %loop.acc = phi i32 [ %loop.acc.next, %guarded ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %guarded ], [ 0, %loop.preheader ]
+  %within.bounds = icmp ult i32 %i, %length
+  %widenable_cond = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond = and i1 %within.bounds, %widenable_cond
+  br i1 %exiplicit_guard_cond, label %guarded, label %deopt, !prof !0
+
+deopt:                                            ; preds = %loop
+  %deoptcall = call i32 (...) @llvm.experimental.deoptimize.i32(i32 9) [ "deopt"() ]
+  ret i32 %deoptcall
+
+guarded:                                          ; preds = %loop
+  %i.i64 = zext i32 %i to i64
+  %array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i.i64
+  %array.i = load i32, i32* %array.i.ptr, align 4
+  %loop.acc.next = add i32 %loop.acc, %array.i
+  %i.next = add nuw i32 %i, 1
+  %continue = icmp ult i32 %i.next, %n
+  br i1 %continue, label %loop, label %exit
+
+exit:                                             ; preds = %guarded, %entry
+  %result = phi i32 [ 0, %entry ], [ %loop.acc.next, %guarded ]
+  ret i32 %result
+}
+
+define i32 @unsigned_loop_0_to_n_ule_latch_ult_check(i32* %array, i32 %length, i32 %n) {
+; CHECK-LABEL: @unsigned_loop_0_to_n_ule_latch_ult_check(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP5]], label [[EXIT:%.*]], label [[LOOP_PREHEADER:%.*]]
+; CHECK:       loop.preheader:
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp ult i32 [[N]], [[LENGTH:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i32 0, [[LENGTH]]
+; CHECK-NEXT:    [[TMP2:%.*]] = and i1 [[TMP1]], [[TMP0]]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[GUARDED:%.*]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[GUARDED]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[WITHIN_BOUNDS:%.*]] = icmp ult i32 [[I]], [[LENGTH]]
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[WITHIN_BOUNDS]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    [[TMP3:%.*]] = and i1 [[TMP2]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[TMP3]], label [[GUARDED]], label [[DEOPT:%.*]], !prof !0
+;
+entry:
+  %tmp5 = icmp eq i32 %n, 0
+  br i1 %tmp5, label %exit, label %loop.preheader
+
+loop.preheader:                                   ; preds = %entry
+  br label %loop
+
+loop:                                             ; preds = %guarded, %loop.preheader
+  %loop.acc = phi i32 [ %loop.acc.next, %guarded ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %guarded ], [ 0, %loop.preheader ]
+  %within.bounds = icmp ult i32 %i, %length
+  %widenable_cond = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond = and i1 %within.bounds, %widenable_cond
+  br i1 %exiplicit_guard_cond, label %guarded, label %deopt, !prof !0
+
+deopt:                                            ; preds = %loop
+  %deoptcall = call i32 (...) @llvm.experimental.deoptimize.i32(i32 9) [ "deopt"() ]
+  ret i32 %deoptcall
+
+guarded:                                          ; preds = %loop
+  %i.i64 = zext i32 %i to i64
+  %array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i.i64
+  %array.i = load i32, i32* %array.i.ptr, align 4
+  %loop.acc.next = add i32 %loop.acc, %array.i
+  %i.next = add nuw i32 %i, 1
+  %continue = icmp ule i32 %i.next, %n
+  br i1 %continue, label %loop, label %exit
+
+exit:                                             ; preds = %guarded, %entry
+  %result = phi i32 [ 0, %entry ], [ %loop.acc.next, %guarded ]
+  ret i32 %result
+}
+
+define i32 @unsigned_loop_0_to_n_ugt_check(i32* %array, i32 %length, i32 %n) {
+; CHECK-LABEL: @unsigned_loop_0_to_n_ugt_check(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP5]], label [[EXIT:%.*]], label [[LOOP_PREHEADER:%.*]]
+; CHECK:       loop.preheader:
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp ule i32 [[N]], [[LENGTH:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i32 0, [[LENGTH]]
+; CHECK-NEXT:    [[TMP2:%.*]] = and i1 [[TMP1]], [[TMP0]]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[GUARDED:%.*]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[GUARDED]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[WITHIN_BOUNDS:%.*]] = icmp ugt i32 [[LENGTH]], [[I]]
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[WITHIN_BOUNDS]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    [[TMP3:%.*]] = and i1 [[TMP2]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[TMP3]], label [[GUARDED]], label [[DEOPT:%.*]], !prof !0
+;
+entry:
+  %tmp5 = icmp eq i32 %n, 0
+  br i1 %tmp5, label %exit, label %loop.preheader
+
+loop.preheader:                                   ; preds = %entry
+  br label %loop
+
+loop:                                             ; preds = %guarded, %loop.preheader
+  %loop.acc = phi i32 [ %loop.acc.next, %guarded ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %guarded ], [ 0, %loop.preheader ]
+  %within.bounds = icmp ugt i32 %length, %i
+  %widenable_cond = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond = and i1 %within.bounds, %widenable_cond
+  br i1 %exiplicit_guard_cond, label %guarded, label %deopt, !prof !0
+
+deopt:                                            ; preds = %loop
+  %deoptcall = call i32 (...) @llvm.experimental.deoptimize.i32(i32 9) [ "deopt"() ]
+  ret i32 %deoptcall
+
+guarded:                                          ; preds = %loop
+  %i.i64 = zext i32 %i to i64
+  %array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i.i64
+  %array.i = load i32, i32* %array.i.ptr, align 4
+  %loop.acc.next = add i32 %loop.acc, %array.i
+  %i.next = add nuw i32 %i, 1
+  %continue = icmp ult i32 %i.next, %n
+  br i1 %continue, label %loop, label %exit
+
+exit:                                             ; preds = %guarded, %entry
+  %result = phi i32 [ 0, %entry ], [ %loop.acc.next, %guarded ]
+  ret i32 %result
+}
+
+define i32 @signed_loop_0_to_n_ult_check(i32* %array, i32 %length, i32 %n) {
+; CHECK-LABEL: @signed_loop_0_to_n_ult_check(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp sle i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP5]], label [[EXIT:%.*]], label [[LOOP_PREHEADER:%.*]]
+; CHECK:       loop.preheader:
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp sle i32 [[N]], [[LENGTH:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i32 0, [[LENGTH]]
+; CHECK-NEXT:    [[TMP2:%.*]] = and i1 [[TMP1]], [[TMP0]]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[GUARDED:%.*]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[GUARDED]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[WITHIN_BOUNDS:%.*]] = icmp ult i32 [[I]], [[LENGTH]]
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[WITHIN_BOUNDS]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    [[TMP3:%.*]] = and i1 [[TMP2]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[TMP3]], label [[GUARDED]], label [[DEOPT:%.*]], !prof !0
+
+;
+entry:
+  %tmp5 = icmp sle i32 %n, 0
+  br i1 %tmp5, label %exit, label %loop.preheader
+
+loop.preheader:                                   ; preds = %entry
+  br label %loop
+
+loop:                                             ; preds = %guarded, %loop.preheader
+  %loop.acc = phi i32 [ %loop.acc.next, %guarded ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %guarded ], [ 0, %loop.preheader ]
+  %within.bounds = icmp ult i32 %i, %length
+  %widenable_cond = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond = and i1 %within.bounds, %widenable_cond
+  br i1 %exiplicit_guard_cond, label %guarded, label %deopt, !prof !0
+
+deopt:                                            ; preds = %loop
+  %deoptcall = call i32 (...) @llvm.experimental.deoptimize.i32(i32 9) [ "deopt"() ]
+  ret i32 %deoptcall
+
+guarded:                                          ; preds = %loop
+  %i.i64 = zext i32 %i to i64
+  %array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i.i64
+  %array.i = load i32, i32* %array.i.ptr, align 4
+  %loop.acc.next = add i32 %loop.acc, %array.i
+  %i.next = add nuw i32 %i, 1
+  %continue = icmp slt i32 %i.next, %n
+  br i1 %continue, label %loop, label %exit
+
+exit:                                             ; preds = %guarded, %entry
+  %result = phi i32 [ 0, %entry ], [ %loop.acc.next, %guarded ]
+  ret i32 %result
+}
+
+define i32 @signed_loop_0_to_n_ult_check_length_range_known(i32* %array, i32* %length.ptr, i32 %n) {
+; CHECK-LABEL: @signed_loop_0_to_n_ult_check_length_range_known(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp sle i32 [[N:%.*]], 0
+; CHECK-NEXT:    [[LENGTH:%.*]] = load i32, i32* [[LENGTH_PTR:%.*]], !range !1
+; CHECK-NEXT:    br i1 [[TMP5]], label [[EXIT:%.*]], label [[LOOP_PREHEADER:%.*]]
+; CHECK:       loop.preheader:
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp sle i32 [[N]], [[LENGTH]]
+; CHECK-NEXT:    [[TMP1:%.*]] = and i1 true, [[TMP0]]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[GUARDED:%.*]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[GUARDED]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[WITHIN_BOUNDS:%.*]] = icmp ult i32 [[I]], [[LENGTH]]
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[WITHIN_BOUNDS]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    [[TMP2:%.*]] = and i1 [[TMP1]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[TMP2]], label [[GUARDED]], label [[DEOPT:%.*]], !prof !0
+;
+entry:
+  %tmp5 = icmp sle i32 %n, 0
+  %length = load i32, i32* %length.ptr, !range !1
+  br i1 %tmp5, label %exit, label %loop.preheader
+
+loop.preheader:                                   ; preds = %entry
+  br label %loop
+
+loop:                                             ; preds = %guarded, %loop.preheader
+  %loop.acc = phi i32 [ %loop.acc.next, %guarded ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %guarded ], [ 0, %loop.preheader ]
+  %within.bounds = icmp ult i32 %i, %length
+  %widenable_cond = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond = and i1 %within.bounds, %widenable_cond
+  br i1 %exiplicit_guard_cond, label %guarded, label %deopt, !prof !0
+
+deopt:                                            ; preds = %loop
+  %deoptcall = call i32 (...) @llvm.experimental.deoptimize.i32(i32 9) [ "deopt"() ]
+  ret i32 %deoptcall
+
+guarded:                                          ; preds = %loop
+  %i.i64 = zext i32 %i to i64
+  %array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i.i64
+  %array.i = load i32, i32* %array.i.ptr, align 4
+  %loop.acc.next = add i32 %loop.acc, %array.i
+  %i.next = add nuw i32 %i, 1
+  %continue = icmp slt i32 %i.next, %n
+  br i1 %continue, label %loop, label %exit
+
+exit:                                             ; preds = %guarded, %entry
+  %result = phi i32 [ 0, %entry ], [ %loop.acc.next, %guarded ]
+  ret i32 %result
+}
+
+define i32 @signed_loop_0_to_n_inverse_latch_predicate(i32* %array, i32 %length, i32 %n) {
+; CHECK-LABEL: @signed_loop_0_to_n_inverse_latch_predicate(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp sle i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP5]], label [[EXIT:%.*]], label [[LOOP_PREHEADER:%.*]]
+; CHECK:       loop.preheader:
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp slt i32 [[N]], [[LENGTH:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i32 0, [[LENGTH]]
+; CHECK-NEXT:    [[TMP2:%.*]] = and i1 [[TMP1]], [[TMP0]]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[GUARDED:%.*]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[GUARDED]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[WITHIN_BOUNDS:%.*]] = icmp ult i32 [[I]], [[LENGTH]]
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[WITHIN_BOUNDS]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    [[TMP3:%.*]] = and i1 [[TMP2]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[TMP3]], label [[GUARDED]], label [[DEOPT:%.*]], !prof !0
+;
+entry:
+  %tmp5 = icmp sle i32 %n, 0
+  br i1 %tmp5, label %exit, label %loop.preheader
+
+loop.preheader:                                   ; preds = %entry
+  br label %loop
+
+loop:                                             ; preds = %guarded, %loop.preheader
+  %loop.acc = phi i32 [ %loop.acc.next, %guarded ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %guarded ], [ 0, %loop.preheader ]
+  %within.bounds = icmp ult i32 %i, %length
+  %widenable_cond = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond = and i1 %within.bounds, %widenable_cond
+  br i1 %exiplicit_guard_cond, label %guarded, label %deopt, !prof !0
+
+deopt:                                            ; preds = %loop
+  %deoptcall = call i32 (...) @llvm.experimental.deoptimize.i32(i32 9) [ "deopt"() ]
+  ret i32 %deoptcall
+
+guarded:                                          ; preds = %loop
+  %i.i64 = zext i32 %i to i64
+  %array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i.i64
+  %array.i = load i32, i32* %array.i.ptr, align 4
+  %loop.acc.next = add i32 %loop.acc, %array.i
+  %i.next = add nuw i32 %i, 1
+  %continue = icmp sgt i32 %i.next, %n
+  br i1 %continue, label %exit, label %loop
+
+exit:                                             ; preds = %guarded, %entry
+  %result = phi i32 [ 0, %entry ], [ %loop.acc.next, %guarded ]
+  ret i32 %result
+}
+
+define i32 @signed_loop_0_to_n_sle_latch_ult_check(i32* %array, i32 %length, i32 %n) {
+; CHECK-LABEL: @signed_loop_0_to_n_sle_latch_ult_check(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp sle i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP5]], label [[EXIT:%.*]], label [[LOOP_PREHEADER:%.*]]
+; CHECK:       loop.preheader:
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp slt i32 [[N]], [[LENGTH:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i32 0, [[LENGTH]]
+; CHECK-NEXT:    [[TMP2:%.*]] = and i1 [[TMP1]], [[TMP0]]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[GUARDED:%.*]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[GUARDED]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[WITHIN_BOUNDS:%.*]] = icmp ult i32 [[I]], [[LENGTH]]
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[WITHIN_BOUNDS]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    [[TMP3:%.*]] = and i1 [[TMP2]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[TMP3]], label [[GUARDED]], label [[DEOPT:%.*]], !prof !0
+;
+entry:
+  %tmp5 = icmp sle i32 %n, 0
+  br i1 %tmp5, label %exit, label %loop.preheader
+
+loop.preheader:                                   ; preds = %entry
+  br label %loop
+
+loop:                                             ; preds = %guarded, %loop.preheader
+  %loop.acc = phi i32 [ %loop.acc.next, %guarded ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %guarded ], [ 0, %loop.preheader ]
+  %within.bounds = icmp ult i32 %i, %length
+  %widenable_cond = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond = and i1 %within.bounds, %widenable_cond
+  br i1 %exiplicit_guard_cond, label %guarded, label %deopt, !prof !0
+
+deopt:                                            ; preds = %loop
+  %deoptcall = call i32 (...) @llvm.experimental.deoptimize.i32(i32 9) [ "deopt"() ]
+  ret i32 %deoptcall
+
+guarded:                                          ; preds = %loop
+  %i.i64 = zext i32 %i to i64
+  %array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i.i64
+  %array.i = load i32, i32* %array.i.ptr, align 4
+  %loop.acc.next = add i32 %loop.acc, %array.i
+  %i.next = add nuw i32 %i, 1
+  %continue = icmp sle i32 %i.next, %n
+  br i1 %continue, label %loop, label %exit
+
+exit:                                             ; preds = %guarded, %entry
+  %result = phi i32 [ 0, %entry ], [ %loop.acc.next, %guarded ]
+  ret i32 %result
+}
+
+define i32 @signed_loop_0_to_n_preincrement_latch_check(i32* %array, i32 %length, i32 %n) {
+; CHECK-LABEL: @signed_loop_0_to_n_preincrement_latch_check(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp sle i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP5]], label [[EXIT:%.*]], label [[LOOP_PREHEADER:%.*]]
+; CHECK:       loop.preheader:
+; CHECK-NEXT:    [[TMP0:%.*]] = add i32 [[LENGTH:%.*]], -1
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sle i32 [[N]], [[TMP0]]
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ult i32 0, [[LENGTH]]
+; CHECK-NEXT:    [[TMP3:%.*]] = and i1 [[TMP2]], [[TMP1]]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[GUARDED:%.*]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[GUARDED]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[WITHIN_BOUNDS:%.*]] = icmp ult i32 [[I]], [[LENGTH]]
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[WITHIN_BOUNDS]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    [[TMP4:%.*]] = and i1 [[TMP3]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[TMP4]], label [[GUARDED]], label [[DEOPT:%.*]], !prof !0
+;
+entry:
+  %tmp5 = icmp sle i32 %n, 0
+  br i1 %tmp5, label %exit, label %loop.preheader
+
+loop.preheader:                                   ; preds = %entry
+  br label %loop
+
+loop:                                             ; preds = %guarded, %loop.preheader
+  %loop.acc = phi i32 [ %loop.acc.next, %guarded ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %guarded ], [ 0, %loop.preheader ]
+  %within.bounds = icmp ult i32 %i, %length
+  %widenable_cond = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond = and i1 %within.bounds, %widenable_cond
+  br i1 %exiplicit_guard_cond, label %guarded, label %deopt, !prof !0
+
+deopt:                                            ; preds = %loop
+  %deoptcall = call i32 (...) @llvm.experimental.deoptimize.i32(i32 9) [ "deopt"() ]
+  ret i32 %deoptcall
+
+guarded:                                          ; preds = %loop
+  %i.i64 = zext i32 %i to i64
+  %array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i.i64
+  %array.i = load i32, i32* %array.i.ptr, align 4
+  %loop.acc.next = add i32 %loop.acc, %array.i
+  %i.next = add i32 %i, 1
+  %continue = icmp slt i32 %i, %n
+  br i1 %continue, label %loop, label %exit
+
+exit:                                             ; preds = %guarded, %entry
+  %result = phi i32 [ 0, %entry ], [ %loop.acc.next, %guarded ]
+  ret i32 %result
+}
+
+define i32 @signed_loop_0_to_n_preincrement_latch_check_postincrement_guard_check(i32* %array, i32 %length, i32 %n) {
+; CHECK-LABEL: @signed_loop_0_to_n_preincrement_latch_check_postincrement_guard_check(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp sle i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP5]], label [[EXIT:%.*]], label [[LOOP_PREHEADER:%.*]]
+; CHECK:       loop.preheader:
+; CHECK-NEXT:    [[TMP0:%.*]] = add i32 [[LENGTH:%.*]], -2
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sle i32 [[N]], [[TMP0]]
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ult i32 1, [[LENGTH]]
+; CHECK-NEXT:    [[TMP3:%.*]] = and i1 [[TMP2]], [[TMP1]]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[GUARDED:%.*]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[GUARDED]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[I_NEXT]] = add i32 [[I]], 1
+; CHECK-NEXT:    [[WITHIN_BOUNDS:%.*]] = icmp ult i32 [[I_NEXT]], [[LENGTH]]
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[WITHIN_BOUNDS]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    [[TMP4:%.*]] = and i1 [[TMP3]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[TMP4]], label [[GUARDED]], label [[DEOPT:%.*]], !prof !0
+;
+entry:
+  %tmp5 = icmp sle i32 %n, 0
+  br i1 %tmp5, label %exit, label %loop.preheader
+
+loop.preheader:                                   ; preds = %entry
+  br label %loop
+
+loop:                                             ; preds = %guarded, %loop.preheader
+  %loop.acc = phi i32 [ %loop.acc.next, %guarded ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %guarded ], [ 0, %loop.preheader ]
+  %i.next = add i32 %i, 1
+  %within.bounds = icmp ult i32 %i.next, %length
+  %widenable_cond = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond = and i1 %within.bounds, %widenable_cond
+  br i1 %exiplicit_guard_cond, label %guarded, label %deopt, !prof !0
+
+deopt:                                            ; preds = %loop
+  %deoptcall = call i32 (...) @llvm.experimental.deoptimize.i32(i32 9) [ "deopt"() ]
+  ret i32 %deoptcall
+
+guarded:                                          ; preds = %loop
+  %i.i64 = zext i32 %i to i64
+  %array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i.i64
+  %array.i = load i32, i32* %array.i.ptr, align 4
+  %loop.acc.next = add i32 %loop.acc, %array.i
+  %continue = icmp slt i32 %i, %n
+  br i1 %continue, label %loop, label %exit
+
+exit:                                             ; preds = %guarded, %entry
+  %result = phi i32 [ 0, %entry ], [ %loop.acc.next, %guarded ]
+  ret i32 %result
+}
+
+define i32 @signed_loop_0_to_n_sle_latch_offset_ult_check(i32* %array, i32 %length, i32 %n) {
+; CHECK-LABEL: @signed_loop_0_to_n_sle_latch_offset_ult_check(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp sle i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP5]], label [[EXIT:%.*]], label [[LOOP_PREHEADER:%.*]]
+; CHECK:       loop.preheader:
+; CHECK-NEXT:    [[TMP0:%.*]] = add i32 [[LENGTH:%.*]], -1
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i32 [[N]], [[TMP0]]
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ult i32 1, [[LENGTH]]
+; CHECK-NEXT:    [[TMP3:%.*]] = and i1 [[TMP2]], [[TMP1]]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[GUARDED:%.*]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[GUARDED]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[I_OFFSET:%.*]] = add i32 [[I]], 1
+; CHECK-NEXT:    [[WITHIN_BOUNDS:%.*]] = icmp ult i32 [[I_OFFSET]], [[LENGTH]]
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[WITHIN_BOUNDS]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    [[TMP4:%.*]] = and i1 [[TMP3]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[TMP4]], label [[GUARDED]], label [[DEOPT:%.*]], !prof !0
+;
+entry:
+  %tmp5 = icmp sle i32 %n, 0
+  br i1 %tmp5, label %exit, label %loop.preheader
+
+loop.preheader:                                   ; preds = %entry
+  br label %loop
+
+loop:                                             ; preds = %guarded, %loop.preheader
+  %loop.acc = phi i32 [ %loop.acc.next, %guarded ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %guarded ], [ 0, %loop.preheader ]
+  %i.offset = add i32 %i, 1
+  %within.bounds = icmp ult i32 %i.offset, %length
+  %widenable_cond = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond = and i1 %within.bounds, %widenable_cond
+  br i1 %exiplicit_guard_cond, label %guarded, label %deopt, !prof !0
+
+deopt:                                            ; preds = %loop
+  %deoptcall = call i32 (...) @llvm.experimental.deoptimize.i32(i32 9) [ "deopt"() ]
+  ret i32 %deoptcall
+
+guarded:                                          ; preds = %loop
+  %i.i64 = zext i32 %i to i64
+  %array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i.i64
+  %array.i = load i32, i32* %array.i.ptr, align 4
+  %loop.acc.next = add i32 %loop.acc, %array.i
+  %i.next = add i32 %i, 1
+  %continue = icmp sle i32 %i.next, %n
+  br i1 %continue, label %loop, label %exit
+
+exit:                                             ; preds = %guarded, %entry
+  %result = phi i32 [ 0, %entry ], [ %loop.acc.next, %guarded ]
+  ret i32 %result
+}
+
+define i32 @signed_loop_0_to_n_offset_sle_latch_offset_ult_check(i32* %array, i32 %length, i32 %n) {
+; CHECK-LABEL: @signed_loop_0_to_n_offset_sle_latch_offset_ult_check(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp sle i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP5]], label [[EXIT:%.*]], label [[LOOP_PREHEADER:%.*]]
+; CHECK:       loop.preheader:
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp slt i32 [[N]], [[LENGTH:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i32 1, [[LENGTH]]
+; CHECK-NEXT:    [[TMP2:%.*]] = and i1 [[TMP1]], [[TMP0]]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[GUARDED:%.*]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[GUARDED]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[I_OFFSET:%.*]] = add i32 [[I]], 1
+; CHECK-NEXT:    [[WITHIN_BOUNDS:%.*]] = icmp ult i32 [[I_OFFSET]], [[LENGTH]]
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[WITHIN_BOUNDS]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    [[TMP3:%.*]] = and i1 [[TMP2]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[TMP3]], label [[GUARDED]], label [[DEOPT:%.*]], !prof !0
+;
+entry:
+  %tmp5 = icmp sle i32 %n, 0
+  br i1 %tmp5, label %exit, label %loop.preheader
+
+loop.preheader:                                   ; preds = %entry
+  br label %loop
+
+loop:                                             ; preds = %guarded, %loop.preheader
+  %loop.acc = phi i32 [ %loop.acc.next, %guarded ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %guarded ], [ 0, %loop.preheader ]
+  %i.offset = add i32 %i, 1
+  %within.bounds = icmp ult i32 %i.offset, %length
+  %widenable_cond = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond = and i1 %within.bounds, %widenable_cond
+  br i1 %exiplicit_guard_cond, label %guarded, label %deopt, !prof !0
+
+deopt:                                            ; preds = %loop
+  %deoptcall = call i32 (...) @llvm.experimental.deoptimize.i32(i32 9) [ "deopt"() ]
+  ret i32 %deoptcall
+
+guarded:                                          ; preds = %loop
+  %i.i64 = zext i32 %i to i64
+  %array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i.i64
+  %array.i = load i32, i32* %array.i.ptr, align 4
+  %loop.acc.next = add i32 %loop.acc, %array.i
+  %i.next = add i32 %i, 1
+  %i.next.offset = add i32 %i.next, 1
+  %continue = icmp sle i32 %i.next.offset, %n
+  br i1 %continue, label %loop, label %exit
+
+exit:                                             ; preds = %guarded, %entry
+  %result = phi i32 [ 0, %entry ], [ %loop.acc.next, %guarded ]
+  ret i32 %result
+}
+
+define i32 @unsupported_latch_pred_loop_0_to_n(i32* %array, i32 %length, i32 %n) {
+; CHECK-LABEL: @unsupported_latch_pred_loop_0_to_n(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp sle i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP5]], label [[EXIT:%.*]], label [[LOOP_PREHEADER:%.*]]
+; CHECK:       loop.preheader:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[GUARDED:%.*]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[GUARDED]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[WITHIN_BOUNDS:%.*]] = icmp ult i32 [[I]], [[LENGTH:%.*]]
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[WITHIN_BOUNDS]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[EXIPLICIT_GUARD_COND]], label [[GUARDED]], label [[DEOPT:%.*]], !prof !0
+;
+entry:
+  %tmp5 = icmp sle i32 %n, 0
+  br i1 %tmp5, label %exit, label %loop.preheader
+
+loop.preheader:                                   ; preds = %entry
+  br label %loop
+
+loop:                                             ; preds = %guarded, %loop.preheader
+  %loop.acc = phi i32 [ %loop.acc.next, %guarded ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %guarded ], [ 0, %loop.preheader ]
+  %within.bounds = icmp ult i32 %i, %length
+  %widenable_cond = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond = and i1 %within.bounds, %widenable_cond
+  br i1 %exiplicit_guard_cond, label %guarded, label %deopt, !prof !0
+
+deopt:                                            ; preds = %loop
+  %deoptcall = call i32 (...) @llvm.experimental.deoptimize.i32(i32 9) [ "deopt"() ]
+  ret i32 %deoptcall
+
+guarded:                                          ; preds = %loop
+  %i.i64 = zext i32 %i to i64
+  %array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i.i64
+  %array.i = load i32, i32* %array.i.ptr, align 4
+  %loop.acc.next = add i32 %loop.acc, %array.i
+  %i.next = add nsw i32 %i, 1
+  %continue = icmp ne i32 %i.next, %n
+  br i1 %continue, label %loop, label %exit
+
+exit:                                             ; preds = %guarded, %entry
+  %result = phi i32 [ 0, %entry ], [ %loop.acc.next, %guarded ]
+  ret i32 %result
+}
+
+define i32 @signed_loop_0_to_n_unsupported_iv_step(i32* %array, i32 %length, i32 %n) {
+; CHECK-LABEL: @signed_loop_0_to_n_unsupported_iv_step(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp sle i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP5]], label [[EXIT:%.*]], label [[LOOP_PREHEADER:%.*]]
+; CHECK:       loop.preheader:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[GUARDED:%.*]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[GUARDED]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[WITHIN_BOUNDS:%.*]] = icmp ult i32 [[I]], [[LENGTH:%.*]]
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[WITHIN_BOUNDS]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[EXIPLICIT_GUARD_COND]], label [[GUARDED]], label [[DEOPT:%.*]], !prof !0
+;
+entry:
+  %tmp5 = icmp sle i32 %n, 0
+  br i1 %tmp5, label %exit, label %loop.preheader
+
+loop.preheader:                                   ; preds = %entry
+  br label %loop
+
+loop:                                             ; preds = %guarded, %loop.preheader
+  %loop.acc = phi i32 [ %loop.acc.next, %guarded ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %guarded ], [ 0, %loop.preheader ]
+  %within.bounds = icmp ult i32 %i, %length
+  %widenable_cond = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond = and i1 %within.bounds, %widenable_cond
+  br i1 %exiplicit_guard_cond, label %guarded, label %deopt, !prof !0
+
+deopt:                                            ; preds = %loop
+  %deoptcall = call i32 (...) @llvm.experimental.deoptimize.i32(i32 9) [ "deopt"() ]
+  ret i32 %deoptcall
+
+guarded:                                          ; preds = %loop
+  %i.i64 = zext i32 %i to i64
+  %array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i.i64
+  %array.i = load i32, i32* %array.i.ptr, align 4
+  %loop.acc.next = add i32 %loop.acc, %array.i
+  %i.next = add nsw i32 %i, 2
+  %continue = icmp slt i32 %i.next, %n
+  br i1 %continue, label %loop, label %exit
+
+exit:                                             ; preds = %guarded, %entry
+  %result = phi i32 [ 0, %entry ], [ %loop.acc.next, %guarded ]
+  ret i32 %result
+}
+
+define i32 @signed_loop_0_to_n_equal_iv_range_check(i32* %array, i32 %length, i32 %n) {
+; CHECK-LABEL: @signed_loop_0_to_n_equal_iv_range_check(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp sle i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP5]], label [[EXIT:%.*]], label [[LOOP_PREHEADER:%.*]]
+; CHECK:       loop.preheader:
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp sle i32 [[N]], [[LENGTH:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i32 0, [[LENGTH]]
+; CHECK-NEXT:    [[TMP2:%.*]] = and i1 [[TMP1]], [[TMP0]]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[GUARDED:%.*]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[GUARDED]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[J:%.*]] = phi i32 [ [[J_NEXT:%.*]], [[GUARDED]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[WITHIN_BOUNDS:%.*]] = icmp ult i32 [[J]], [[LENGTH]]
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[WITHIN_BOUNDS]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    [[TMP3:%.*]] = and i1 [[TMP2]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[TMP3]], label [[GUARDED]], label [[DEOPT:%.*]], !prof !0
+;
+entry:
+  %tmp5 = icmp sle i32 %n, 0
+  br i1 %tmp5, label %exit, label %loop.preheader
+
+loop.preheader:                                   ; preds = %entry
+  br label %loop
+
+loop:                                             ; preds = %guarded, %loop.preheader
+  %loop.acc = phi i32 [ %loop.acc.next, %guarded ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %guarded ], [ 0, %loop.preheader ]
+  %j = phi i32 [ %j.next, %guarded ], [ 0, %loop.preheader ]
+  %within.bounds = icmp ult i32 %j, %length
+  %widenable_cond = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond = and i1 %within.bounds, %widenable_cond
+  br i1 %exiplicit_guard_cond, label %guarded, label %deopt, !prof !0
+
+deopt:                                            ; preds = %loop
+  %deoptcall = call i32 (...) @llvm.experimental.deoptimize.i32(i32 9) [ "deopt"() ]
+  ret i32 %deoptcall
+
+guarded:                                          ; preds = %loop
+  %i.i64 = zext i32 %i to i64
+  %array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i.i64
+  %array.i = load i32, i32* %array.i.ptr, align 4
+  %loop.acc.next = add i32 %loop.acc, %array.i
+  %j.next = add nsw i32 %j, 1
+  %i.next = add nsw i32 %i, 1
+  %continue = icmp slt i32 %i.next, %n
+  br i1 %continue, label %loop, label %exit
+
+exit:                                             ; preds = %guarded, %entry
+  %result = phi i32 [ 0, %entry ], [ %loop.acc.next, %guarded ]
+  ret i32 %result
+}
+
+define i32 @signed_loop_start_to_n_offset_iv_range_check(i32* %array, i32 %start.i, i32 %start.j, i32 %length, i32 %n) {
+; CHECK-LABEL: @signed_loop_start_to_n_offset_iv_range_check(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp sle i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP5]], label [[EXIT:%.*]], label [[LOOP_PREHEADER:%.*]]
+; CHECK:       loop.preheader:
+; CHECK-NEXT:    [[TMP0:%.*]] = add i32 [[LENGTH:%.*]], [[START_I:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = sub i32 [[TMP0]], [[START_J:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp sle i32 [[N]], [[TMP1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ult i32 [[START_J]], [[LENGTH]]
+; CHECK-NEXT:    [[TMP4:%.*]] = and i1 [[TMP3]], [[TMP2]]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[GUARDED:%.*]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[GUARDED]] ], [ [[START_I]], [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[J:%.*]] = phi i32 [ [[J_NEXT:%.*]], [[GUARDED]] ], [ [[START_J]], [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[WITHIN_BOUNDS:%.*]] = icmp ult i32 [[J]], [[LENGTH]]
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[WITHIN_BOUNDS]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    [[TMP5:%.*]] = and i1 [[TMP4]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[TMP5]], label [[GUARDED]], label [[DEOPT:%.*]], !prof !0
+;
+entry:
+  %tmp5 = icmp sle i32 %n, 0
+  br i1 %tmp5, label %exit, label %loop.preheader
+
+loop.preheader:                                   ; preds = %entry
+  br label %loop
+
+loop:                                             ; preds = %guarded, %loop.preheader
+  %loop.acc = phi i32 [ %loop.acc.next, %guarded ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %guarded ], [ %start.i, %loop.preheader ]
+  %j = phi i32 [ %j.next, %guarded ], [ %start.j, %loop.preheader ]
+  %within.bounds = icmp ult i32 %j, %length
+  %widenable_cond = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond = and i1 %within.bounds, %widenable_cond
+  br i1 %exiplicit_guard_cond, label %guarded, label %deopt, !prof !0
+
+deopt:                                            ; preds = %loop
+  %deoptcall = call i32 (...) @llvm.experimental.deoptimize.i32(i32 9) [ "deopt"() ]
+  ret i32 %deoptcall
+
+guarded:                                          ; preds = %loop
+  %i.i64 = zext i32 %i to i64
+  %array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i.i64
+  %array.i = load i32, i32* %array.i.ptr, align 4
+  %loop.acc.next = add i32 %loop.acc, %array.i
+  %j.next = add i32 %j, 1
+  %i.next = add i32 %i, 1
+  %continue = icmp slt i32 %i.next, %n
+  br i1 %continue, label %loop, label %exit
+
+exit:                                             ; preds = %guarded, %entry
+  %result = phi i32 [ 0, %entry ], [ %loop.acc.next, %guarded ]
+  ret i32 %result
+}
+
+define i32 @signed_loop_0_to_n_different_iv_types(i32* %array, i16 %length, i32 %n) {
+; CHECK-LABEL: @signed_loop_0_to_n_different_iv_types(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp sle i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP5]], label [[EXIT:%.*]], label [[LOOP_PREHEADER:%.*]]
+; CHECK:       loop.preheader:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[GUARDED:%.*]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[GUARDED]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[J:%.*]] = phi i16 [ [[J_NEXT:%.*]], [[GUARDED]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[WITHIN_BOUNDS:%.*]] = icmp ult i16 [[J]], [[LENGTH:%.*]]
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[WITHIN_BOUNDS]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[EXIPLICIT_GUARD_COND]], label [[GUARDED]], label [[DEOPT:%.*]], !prof !0
+;
+entry:
+  %tmp5 = icmp sle i32 %n, 0
+  br i1 %tmp5, label %exit, label %loop.preheader
+
+loop.preheader:                                   ; preds = %entry
+  br label %loop
+
+loop:                                             ; preds = %guarded, %loop.preheader
+  %loop.acc = phi i32 [ %loop.acc.next, %guarded ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %guarded ], [ 0, %loop.preheader ]
+  %j = phi i16 [ %j.next, %guarded ], [ 0, %loop.preheader ]
+  %within.bounds = icmp ult i16 %j, %length
+  %widenable_cond = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond = and i1 %within.bounds, %widenable_cond
+  br i1 %exiplicit_guard_cond, label %guarded, label %deopt, !prof !0
+
+deopt:                                            ; preds = %loop
+  %deoptcall = call i32 (...) @llvm.experimental.deoptimize.i32(i32 9) [ "deopt"() ]
+  ret i32 %deoptcall
+
+guarded:                                          ; preds = %loop
+  %i.i64 = zext i32 %i to i64
+  %array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i.i64
+  %array.i = load i32, i32* %array.i.ptr, align 4
+  %loop.acc.next = add i32 %loop.acc, %array.i
+  %j.next = add i16 %j, 1
+  %i.next = add i32 %i, 1
+  %continue = icmp slt i32 %i.next, %n
+  br i1 %continue, label %loop, label %exit
+
+exit:                                             ; preds = %guarded, %entry
+  %result = phi i32 [ 0, %entry ], [ %loop.acc.next, %guarded ]
+  ret i32 %result
+}
+
+define i32 @signed_loop_0_to_n_different_iv_strides(i32* %array, i32 %length, i32 %n) {
+; CHECK-LABEL: @signed_loop_0_to_n_different_iv_strides(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp sle i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP5]], label [[EXIT:%.*]], label [[LOOP_PREHEADER:%.*]]
+; CHECK:       loop.preheader:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[GUARDED:%.*]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[GUARDED]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[J:%.*]] = phi i32 [ [[J_NEXT:%.*]], [[GUARDED]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[WITHIN_BOUNDS:%.*]] = icmp ult i32 [[J]], [[LENGTH:%.*]]
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[WITHIN_BOUNDS]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[EXIPLICIT_GUARD_COND]], label [[GUARDED]], label [[DEOPT:%.*]], !prof !0
+;
+entry:
+  %tmp5 = icmp sle i32 %n, 0
+  br i1 %tmp5, label %exit, label %loop.preheader
+
+loop.preheader:                                   ; preds = %entry
+  br label %loop
+
+loop:                                             ; preds = %guarded, %loop.preheader
+  %loop.acc = phi i32 [ %loop.acc.next, %guarded ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %guarded ], [ 0, %loop.preheader ]
+  %j = phi i32 [ %j.next, %guarded ], [ 0, %loop.preheader ]
+  %within.bounds = icmp ult i32 %j, %length
+  %widenable_cond = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond = and i1 %within.bounds, %widenable_cond
+  br i1 %exiplicit_guard_cond, label %guarded, label %deopt, !prof !0
+
+deopt:                                            ; preds = %loop
+  %deoptcall = call i32 (...) @llvm.experimental.deoptimize.i32(i32 9) [ "deopt"() ]
+  ret i32 %deoptcall
+
+guarded:                                          ; preds = %loop
+  %i.i64 = zext i32 %i to i64
+  %array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i.i64
+  %array.i = load i32, i32* %array.i.ptr, align 4
+  %loop.acc.next = add i32 %loop.acc, %array.i
+  %j.next = add nsw i32 %j, 2
+  %i.next = add nsw i32 %i, 1
+  %continue = icmp slt i32 %i.next, %n
+  br i1 %continue, label %loop, label %exit
+
+exit:                                             ; preds = %guarded, %entry
+  %result = phi i32 [ 0, %entry ], [ %loop.acc.next, %guarded ]
+  ret i32 %result
+}
+
+define i32 @two_range_checks(i32* %array.1, i32 %length.1, i32* %array.2, i32 %length.2, i32 %n) {
+; CHECK-LABEL: @two_range_checks(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP5]], label [[EXIT:%.*]], label [[LOOP_PREHEADER:%.*]]
+; CHECK:       loop.preheader:
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp ule i32 [[N]], [[LENGTH_2:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i32 0, [[LENGTH_2]]
+; CHECK-NEXT:    [[TMP2:%.*]] = and i1 [[TMP1]], [[TMP0]]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ule i32 [[N]], [[LENGTH_1:%.*]]
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ult i32 0, [[LENGTH_1]]
+; CHECK-NEXT:    [[TMP5:%.*]] = and i1 [[TMP4]], [[TMP3]]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[GUARDED:%.*]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[GUARDED]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[WITHIN_BOUNDS_1:%.*]] = icmp ult i32 [[I]], [[LENGTH_1]]
+; CHECK-NEXT:    [[WITHIN_BOUNDS_2:%.*]] = icmp ult i32 [[I]], [[LENGTH_2]]
+; CHECK-NEXT:    [[WITHIN_BOUNDS:%.*]] = and i1 [[WITHIN_BOUNDS_1]], [[WITHIN_BOUNDS_2]]
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[WITHIN_BOUNDS]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    [[TMP6:%.*]] = and i1 [[TMP2]], [[TMP5]]
+; CHECK-NEXT:    [[TMP7:%.*]] = and i1 [[TMP6]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[TMP7]], label [[GUARDED]], label [[DEOPT:%.*]], !prof !0
+;
+entry:
+  %tmp5 = icmp eq i32 %n, 0
+  br i1 %tmp5, label %exit, label %loop.preheader
+
+loop.preheader:                                   ; preds = %entry
+  br label %loop
+
+loop:                                             ; preds = %guarded, %loop.preheader
+  %loop.acc = phi i32 [ %loop.acc.next, %guarded ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %guarded ], [ 0, %loop.preheader ]
+  %within.bounds.1 = icmp ult i32 %i, %length.1
+  %within.bounds.2 = icmp ult i32 %i, %length.2
+  %within.bounds = and i1 %within.bounds.1, %within.bounds.2
+  %widenable_cond = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond = and i1 %within.bounds, %widenable_cond
+  br i1 %exiplicit_guard_cond, label %guarded, label %deopt, !prof !0
+
+deopt:                                            ; preds = %loop
+  %deoptcall = call i32 (...) @llvm.experimental.deoptimize.i32(i32 9) [ "deopt"() ]
+  ret i32 %deoptcall
+
+guarded:                                          ; preds = %loop
+  %i.i64 = zext i32 %i to i64
+  %array.1.i.ptr = getelementptr inbounds i32, i32* %array.1, i64 %i.i64
+  %array.1.i = load i32, i32* %array.1.i.ptr, align 4
+  %loop.acc.1 = add i32 %loop.acc, %array.1.i
+  %array.2.i.ptr = getelementptr inbounds i32, i32* %array.2, i64 %i.i64
+  %array.2.i = load i32, i32* %array.2.i.ptr, align 4
+  %loop.acc.next = add i32 %loop.acc.1, %array.2.i
+  %i.next = add nuw i32 %i, 1
+  %continue = icmp ult i32 %i.next, %n
+  br i1 %continue, label %loop, label %exit
+
+exit:                                             ; preds = %guarded, %entry
+  %result = phi i32 [ 0, %entry ], [ %loop.acc.next, %guarded ]
+  ret i32 %result
+}
+
+define i32 @three_range_checks(i32* %array.1, i32 %length.1, i32* %array.2, i32 %length.2, i32* %array.3, i32 %length.3, i32 %n) {
+; CHECK-LABEL: @three_range_checks(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP5]], label [[EXIT:%.*]], label [[LOOP_PREHEADER:%.*]]
+; CHECK:       loop.preheader:
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp ule i32 [[N]], [[LENGTH_3:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i32 0, [[LENGTH_3]]
+; CHECK-NEXT:    [[TMP2:%.*]] = and i1 [[TMP1]], [[TMP0]]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ule i32 [[N]], [[LENGTH_2:%.*]]
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ult i32 0, [[LENGTH_2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = and i1 [[TMP4]], [[TMP3]]
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp ule i32 [[N]], [[LENGTH_1:%.*]]
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp ult i32 0, [[LENGTH_1]]
+; CHECK-NEXT:    [[TMP8:%.*]] = and i1 [[TMP7]], [[TMP6]]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[GUARDED:%.*]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[GUARDED]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[WITHIN_BOUNDS_1:%.*]] = icmp ult i32 [[I]], [[LENGTH_1]]
+; CHECK-NEXT:    [[WITHIN_BOUNDS_2:%.*]] = icmp ult i32 [[I]], [[LENGTH_2]]
+; CHECK-NEXT:    [[WITHIN_BOUNDS_3:%.*]] = icmp ult i32 [[I]], [[LENGTH_3]]
+; CHECK-NEXT:    [[WITHIN_BOUNDS_1_AND_2:%.*]] = and i1 [[WITHIN_BOUNDS_1]], [[WITHIN_BOUNDS_2]]
+; CHECK-NEXT:    [[WITHIN_BOUNDS:%.*]] = and i1 [[WITHIN_BOUNDS_1_AND_2]], [[WITHIN_BOUNDS_3]]
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[WITHIN_BOUNDS]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    [[TMP9:%.*]] = and i1 [[TMP2]], [[TMP5]]
+; CHECK-NEXT:    [[TMP10:%.*]] = and i1 [[TMP9]], [[TMP8]]
+; CHECK-NEXT:    [[TMP11:%.*]] = and i1 [[TMP10]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[TMP11]], label [[GUARDED]], label [[DEOPT:%.*]], !prof !0
+;
+entry:
+  %tmp5 = icmp eq i32 %n, 0
+  br i1 %tmp5, label %exit, label %loop.preheader
+
+loop.preheader:                                   ; preds = %entry
+  br label %loop
+
+loop:                                             ; preds = %guarded, %loop.preheader
+  %loop.acc = phi i32 [ %loop.acc.next, %guarded ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %guarded ], [ 0, %loop.preheader ]
+  %within.bounds.1 = icmp ult i32 %i, %length.1
+  %within.bounds.2 = icmp ult i32 %i, %length.2
+  %within.bounds.3 = icmp ult i32 %i, %length.3
+  %within.bounds.1.and.2 = and i1 %within.bounds.1, %within.bounds.2
+  %within.bounds = and i1 %within.bounds.1.and.2, %within.bounds.3
+  %widenable_cond = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond = and i1 %within.bounds, %widenable_cond
+  br i1 %exiplicit_guard_cond, label %guarded, label %deopt, !prof !0
+
+deopt:                                            ; preds = %loop
+  %deoptcall = call i32 (...) @llvm.experimental.deoptimize.i32(i32 9) [ "deopt"() ]
+  ret i32 %deoptcall
+
+guarded:                                          ; preds = %loop
+  %i.i64 = zext i32 %i to i64
+  %array.1.i.ptr = getelementptr inbounds i32, i32* %array.1, i64 %i.i64
+  %array.1.i = load i32, i32* %array.1.i.ptr, align 4
+  %loop.acc.1 = add i32 %loop.acc, %array.1.i
+  %array.2.i.ptr = getelementptr inbounds i32, i32* %array.2, i64 %i.i64
+  %array.2.i = load i32, i32* %array.2.i.ptr, align 4
+  %loop.acc.2 = add i32 %loop.acc.1, %array.2.i
+  %array.3.i.ptr = getelementptr inbounds i32, i32* %array.3, i64 %i.i64
+  %array.3.i = load i32, i32* %array.3.i.ptr, align 4
+  %loop.acc.next = add i32 %loop.acc.2, %array.3.i
+  %i.next = add nuw i32 %i, 1
+  %continue = icmp ult i32 %i.next, %n
+  br i1 %continue, label %loop, label %exit
+
+exit:                                             ; preds = %guarded, %entry
+  %result = phi i32 [ 0, %entry ], [ %loop.acc.next, %guarded ]
+  ret i32 %result
+}
+
+define i32 @three_guards(i32* %array.1, i32 %length.1, i32* %array.2, i32 %length.2, i32* %array.3, i32 %length.3, i32 %n) {
+; CHECK-LABEL: @three_guards(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP5]], label [[EXIT:%.*]], label [[LOOP_PREHEADER:%.*]]
+; CHECK:       loop.preheader:
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp ule i32 [[N]], [[LENGTH_1:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i32 0, [[LENGTH_1]]
+; CHECK-NEXT:    [[TMP2:%.*]] = and i1 [[TMP1]], [[TMP0]]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ule i32 [[N]], [[LENGTH_2:%.*]]
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ult i32 0, [[LENGTH_2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = and i1 [[TMP4]], [[TMP3]]
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp ule i32 [[N]], [[LENGTH_3:%.*]]
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp ult i32 0, [[LENGTH_3]]
+; CHECK-NEXT:    [[TMP8:%.*]] = and i1 [[TMP7]], [[TMP6]]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[GUARDED6:%.*]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[GUARDED6]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[WITHIN_BOUNDS_1:%.*]] = icmp ult i32 [[I]], [[LENGTH_1]]
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[WITHIN_BOUNDS_1]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    [[TMP9:%.*]] = and i1 [[TMP2]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[TMP9]], label [[GUARDED:%.*]], label [[DEOPT:%.*]], !prof !0
+; CHECK:       deopt:
+; CHECK-NEXT:    [[DEOPTCALL:%.*]] = call i32 (...) @llvm.experimental.deoptimize.i32(i32 9) [ "deopt"() ]
+; CHECK-NEXT:    ret i32 [[DEOPTCALL]]
+; CHECK:       guarded:
+; CHECK-NEXT:    [[I_I64:%.*]] = zext i32 [[I]] to i64
+; CHECK-NEXT:    [[ARRAY_1_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY_1:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    [[ARRAY_1_I:%.*]] = load i32, i32* [[ARRAY_1_I_PTR]], align 4
+; CHECK-NEXT:    [[LOOP_ACC_1:%.*]] = add i32 [[LOOP_ACC]], [[ARRAY_1_I]]
+; CHECK-NEXT:    [[WITHIN_BOUNDS_2:%.*]] = icmp ult i32 [[I]], [[LENGTH_2]]
+; CHECK-NEXT:    [[WIDENABLE_COND4:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND5:%.*]] = and i1 [[WITHIN_BOUNDS_2]], [[WIDENABLE_COND4]]
+; CHECK-NEXT:    [[TMP10:%.*]] = and i1 [[TMP5]], [[WIDENABLE_COND4]]
+; CHECK-NEXT:    br i1 [[TMP10]], label [[GUARDED1:%.*]], label [[DEOPT2:%.*]], !prof !0
+; CHECK:       deopt2:
+; CHECK-NEXT:    [[DEOPTCALL3:%.*]] = call i32 (...) @llvm.experimental.deoptimize.i32(i32 9) [ "deopt"() ]
+; CHECK-NEXT:    ret i32 [[DEOPTCALL3]]
+; CHECK:       guarded1:
+; CHECK-NEXT:    [[ARRAY_2_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY_2:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    [[ARRAY_2_I:%.*]] = load i32, i32* [[ARRAY_2_I_PTR]], align 4
+; CHECK-NEXT:    [[LOOP_ACC_2:%.*]] = add i32 [[LOOP_ACC_1]], [[ARRAY_2_I]]
+; CHECK-NEXT:    [[WITHIN_BOUNDS_3:%.*]] = icmp ult i32 [[I]], [[LENGTH_3]]
+; CHECK-NEXT:    [[WIDENABLE_COND9:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND10:%.*]] = and i1 [[WITHIN_BOUNDS_3]], [[WIDENABLE_COND9]]
+; CHECK-NEXT:    [[TMP11:%.*]] = and i1 [[TMP8]], [[WIDENABLE_COND9]]
+; CHECK-NEXT:    br i1 [[TMP11]], label [[GUARDED6]], label [[DEOPT7:%.*]], !prof !0
+; CHECK:       deopt7:
+; CHECK-NEXT:    [[DEOPTCALL8:%.*]] = call i32 (...) @llvm.experimental.deoptimize.i32(i32 9) [ "deopt"() ]
+; CHECK-NEXT:    ret i32 [[DEOPTCALL8]]
+; CHECK:       guarded6:
+; CHECK-NEXT:    [[ARRAY_3_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY_3:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    [[ARRAY_3_I:%.*]] = load i32, i32* [[ARRAY_3_I_PTR]], align 4
+; CHECK-NEXT:    [[LOOP_ACC_NEXT]] = add i32 [[LOOP_ACC_2]], [[ARRAY_3_I]]
+; CHECK-NEXT:    [[I_NEXT]] = add nuw i32 [[I]], 1
+; CHECK-NEXT:    [[CONTINUE:%.*]] = icmp ult i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CONTINUE]], label [[LOOP]], label [[EXIT_LOOPEXIT:%.*]]
+; CHECK:       exit.loopexit:
+; CHECK-NEXT:    [[LOOP_ACC_NEXT_LCSSA:%.*]] = phi i32 [ [[LOOP_ACC_NEXT]], [[GUARDED6]] ]
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RESULT:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[LOOP_ACC_NEXT_LCSSA]], [[EXIT_LOOPEXIT]] ]
+; CHECK-NEXT:    ret i32 [[RESULT]]
+;
+entry:
+  %tmp5 = icmp eq i32 %n, 0
+  br i1 %tmp5, label %exit, label %loop.preheader
+
+loop.preheader:                                   ; preds = %entry
+  br label %loop
+
+loop:                                             ; preds = %guarded6, %loop.preheader
+  %loop.acc = phi i32 [ %loop.acc.next, %guarded6 ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %guarded6 ], [ 0, %loop.preheader ]
+  %within.bounds.1 = icmp ult i32 %i, %length.1
+  %widenable_cond = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond = and i1 %within.bounds.1, %widenable_cond
+  br i1 %exiplicit_guard_cond, label %guarded, label %deopt, !prof !0
+
+deopt:                                            ; preds = %loop
+  %deoptcall = call i32 (...) @llvm.experimental.deoptimize.i32(i32 9) [ "deopt"() ]
+  ret i32 %deoptcall
+
+guarded:                                          ; preds = %loop
+  %i.i64 = zext i32 %i to i64
+  %array.1.i.ptr = getelementptr inbounds i32, i32* %array.1, i64 %i.i64
+  %array.1.i = load i32, i32* %array.1.i.ptr, align 4
+  %loop.acc.1 = add i32 %loop.acc, %array.1.i
+  %within.bounds.2 = icmp ult i32 %i, %length.2
+  %widenable_cond4 = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond5 = and i1 %within.bounds.2, %widenable_cond4
+  br i1 %exiplicit_guard_cond5, label %guarded1, label %deopt2, !prof !0
+
+deopt2:                                           ; preds = %guarded
+  %deoptcall3 = call i32 (...) @llvm.experimental.deoptimize.i32(i32 9) [ "deopt"() ]
+  ret i32 %deoptcall3
+
+guarded1:                                         ; preds = %guarded
+  %array.2.i.ptr = getelementptr inbounds i32, i32* %array.2, i64 %i.i64
+  %array.2.i = load i32, i32* %array.2.i.ptr, align 4
+  %loop.acc.2 = add i32 %loop.acc.1, %array.2.i
+  %within.bounds.3 = icmp ult i32 %i, %length.3
+  %widenable_cond9 = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond10 = and i1 %within.bounds.3, %widenable_cond9
+  br i1 %exiplicit_guard_cond10, label %guarded6, label %deopt7, !prof !0
+
+deopt7:                                           ; preds = %guarded1
+  %deoptcall8 = call i32 (...) @llvm.experimental.deoptimize.i32(i32 9) [ "deopt"() ]
+  ret i32 %deoptcall8
+
+guarded6:                                         ; preds = %guarded1
+  %array.3.i.ptr = getelementptr inbounds i32, i32* %array.3, i64 %i.i64
+  %array.3.i = load i32, i32* %array.3.i.ptr, align 4
+  %loop.acc.next = add i32 %loop.acc.2, %array.3.i
+  %i.next = add nuw i32 %i, 1
+  %continue = icmp ult i32 %i.next, %n
+  br i1 %continue, label %loop, label %exit
+
+exit:                                             ; preds = %guarded6, %entry
+  %result = phi i32 [ 0, %entry ], [ %loop.acc.next, %guarded6 ]
+  ret i32 %result
+}
+
+define i32 @unsigned_loop_0_to_n_unrelated_condition(i32* %array, i32 %length, i32 %n, i32 %x) {
+; CHECK-LABEL: @unsigned_loop_0_to_n_unrelated_condition(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP5]], label [[EXIT:%.*]], label [[LOOP_PREHEADER:%.*]]
+; CHECK:       loop.preheader:
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp ule i32 [[N]], [[LENGTH:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i32 0, [[LENGTH]]
+; CHECK-NEXT:    [[TMP2:%.*]] = and i1 [[TMP1]], [[TMP0]]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[GUARDED:%.*]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[GUARDED]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[WITHIN_BOUNDS:%.*]] = icmp ult i32 [[I]], [[LENGTH]]
+; CHECK-NEXT:    [[UNRELATED_COND:%.*]] = icmp ult i32 [[X:%.*]], [[LENGTH]]
+; CHECK-NEXT:    [[GUARD_COND:%.*]] = and i1 [[WITHIN_BOUNDS]], [[UNRELATED_COND]]
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[GUARD_COND]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    [[TMP3:%.*]] = and i1 [[UNRELATED_COND]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = and i1 [[TMP3]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[TMP4]], label [[GUARDED]], label [[DEOPT:%.*]], !prof !0
+;
+entry:
+  %tmp5 = icmp eq i32 %n, 0
+  br i1 %tmp5, label %exit, label %loop.preheader
+
+loop.preheader:                                   ; preds = %entry
+  br label %loop
+
+loop:                                             ; preds = %guarded, %loop.preheader
+  %loop.acc = phi i32 [ %loop.acc.next, %guarded ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %guarded ], [ 0, %loop.preheader ]
+  %within.bounds = icmp ult i32 %i, %length
+  %unrelated.cond = icmp ult i32 %x, %length
+  %guard.cond = and i1 %within.bounds, %unrelated.cond
+  %widenable_cond = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond = and i1 %guard.cond, %widenable_cond
+  br i1 %exiplicit_guard_cond, label %guarded, label %deopt, !prof !0
+
+deopt:                                            ; preds = %loop
+  %deoptcall = call i32 (...) @llvm.experimental.deoptimize.i32(i32 9) [ "deopt"() ]
+  ret i32 %deoptcall
+
+guarded:                                          ; preds = %loop
+  %i.i64 = zext i32 %i to i64
+  %array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i.i64
+  %array.i = load i32, i32* %array.i.ptr, align 4
+  %loop.acc.next = add i32 %loop.acc, %array.i
+  %i.next = add nuw i32 %i, 1
+  %continue = icmp ult i32 %i.next, %n
+  br i1 %continue, label %loop, label %exit
+
+exit:                                             ; preds = %guarded, %entry
+  %result = phi i32 [ 0, %entry ], [ %loop.acc.next, %guarded ]
+  ret i32 %result
+}
+
+define i32 @test_no_widened_conditions(i32* %array, i32 %length, i32 %n, i32 %x1, i32 %x2, i32 %x3) {
+; CHECK-LABEL: @test_no_widened_conditions(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP5]], label [[EXIT:%.*]], label [[LOOP_PREHEADER:%.*]]
+; CHECK:       loop.preheader:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[GUARDED:%.*]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[GUARDED]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[UNRELATED_COND_1:%.*]] = icmp eq i32 [[X1:%.*]], [[I]]
+; CHECK-NEXT:    [[UNRELATED_COND_2:%.*]] = icmp eq i32 [[X2:%.*]], [[I]]
+; CHECK-NEXT:    [[UNRELATED_COND_3:%.*]] = icmp eq i32 [[X3:%.*]], [[I]]
+; CHECK-NEXT:    [[UNRELATED_COND_AND_1:%.*]] = and i1 [[UNRELATED_COND_1]], [[UNRELATED_COND_2]]
+; CHECK-NEXT:    [[GUARD_COND:%.*]] = and i1 [[UNRELATED_COND_AND_1]], [[UNRELATED_COND_3]]
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[GUARD_COND]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[EXIPLICIT_GUARD_COND]], label [[GUARDED]], label [[DEOPT:%.*]], !prof !0
+;
+entry:
+  %tmp5 = icmp eq i32 %n, 0
+  br i1 %tmp5, label %exit, label %loop.preheader
+
+loop.preheader:                                   ; preds = %entry
+  br label %loop
+
+loop:                                             ; preds = %guarded, %loop.preheader
+  %loop.acc = phi i32 [ %loop.acc.next, %guarded ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %guarded ], [ 0, %loop.preheader ]
+  %unrelated.cond.1 = icmp eq i32 %x1, %i
+  %unrelated.cond.2 = icmp eq i32 %x2, %i
+  %unrelated.cond.3 = icmp eq i32 %x3, %i
+  %unrelated.cond.and.1 = and i1 %unrelated.cond.1, %unrelated.cond.2
+  %guard.cond = and i1 %unrelated.cond.and.1, %unrelated.cond.3
+  %widenable_cond = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond = and i1 %guard.cond, %widenable_cond
+  br i1 %exiplicit_guard_cond, label %guarded, label %deopt, !prof !0
+
+deopt:                                            ; preds = %loop
+  %deoptcall = call i32 (...) @llvm.experimental.deoptimize.i32(i32 9) [ "deopt"() ]
+  ret i32 %deoptcall
+
+guarded:                                          ; preds = %loop
+  %i.i64 = zext i32 %i to i64
+  %array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i.i64
+  %array.i = load i32, i32* %array.i.ptr, align 4
+  %loop.acc.next = add i32 %loop.acc, %array.i
+  %i.next = add nuw i32 %i, 1
+  %continue = icmp ult i32 %i.next, %n
+  br i1 %continue, label %loop, label %exit
+
+exit:                                             ; preds = %guarded, %entry
+  %result = phi i32 [ 0, %entry ], [ %loop.acc.next, %guarded ]
+  ret i32 %result
+}
+
+define i32 @signed_loop_start_to_n_loop_variant_bound(i32* %array, i32 %x, i32 %start, i32 %n) {
+; CHECK-LABEL: @signed_loop_start_to_n_loop_variant_bound(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp sle i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP5]], label [[EXIT:%.*]], label [[LOOP_PREHEADER:%.*]]
+; CHECK:       loop.preheader:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[GUARDED:%.*]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[GUARDED]] ], [ [[START:%.*]], [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[BOUND:%.*]] = add i32 [[I]], [[X:%.*]]
+; CHECK-NEXT:    [[WITHIN_BOUNDS:%.*]] = icmp ult i32 [[I]], [[BOUND]]
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[WITHIN_BOUNDS]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[EXIPLICIT_GUARD_COND]], label [[GUARDED]], label [[DEOPT:%.*]], !prof !0
+;
+entry:
+  %tmp5 = icmp sle i32 %n, 0
+  br i1 %tmp5, label %exit, label %loop.preheader
+
+loop.preheader:                                   ; preds = %entry
+  br label %loop
+
+loop:                                             ; preds = %guarded, %loop.preheader
+  %loop.acc = phi i32 [ %loop.acc.next, %guarded ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %guarded ], [ %start, %loop.preheader ]
+  %bound = add i32 %i, %x
+  %within.bounds = icmp ult i32 %i, %bound
+  %widenable_cond = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond = and i1 %within.bounds, %widenable_cond
+  br i1 %exiplicit_guard_cond, label %guarded, label %deopt, !prof !0
+
+deopt:                                            ; preds = %loop
+  %deoptcall = call i32 (...) @llvm.experimental.deoptimize.i32(i32 9) [ "deopt"() ]
+  ret i32 %deoptcall
+
+guarded:                                          ; preds = %loop
+  %i.i64 = zext i32 %i to i64
+  %array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i.i64
+  %array.i = load i32, i32* %array.i.ptr, align 4
+  %loop.acc.next = add i32 %loop.acc, %array.i
+  %i.next = add nsw i32 %i, 1
+  %continue = icmp slt i32 %i.next, %n
+  br i1 %continue, label %loop, label %exit
+
+exit:                                             ; preds = %guarded, %entry
+  %result = phi i32 [ 0, %entry ], [ %loop.acc.next, %guarded ]
+  ret i32 %result
+}
+
+define i32 @signed_loop_start_to_n_non_monotonic_predicate(i32* %array, i32 %x, i32 %start, i32 %n) {
+; CHECK-LABEL: @signed_loop_start_to_n_non_monotonic_predicate(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp sle i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP5]], label [[EXIT:%.*]], label [[LOOP_PREHEADER:%.*]]
+; CHECK:       loop.preheader:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[GUARDED:%.*]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[GUARDED]] ], [ [[START:%.*]], [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[GUARD_COND:%.*]] = icmp eq i32 [[I]], [[X:%.*]]
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[GUARD_COND]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[EXIPLICIT_GUARD_COND]], label [[GUARDED]], label [[DEOPT:%.*]], !prof !0
+;
+entry:
+  %tmp5 = icmp sle i32 %n, 0
+  br i1 %tmp5, label %exit, label %loop.preheader
+
+loop.preheader:                                   ; preds = %entry
+  br label %loop
+
+loop:                                             ; preds = %guarded, %loop.preheader
+  %loop.acc = phi i32 [ %loop.acc.next, %guarded ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %guarded ], [ %start, %loop.preheader ]
+  %guard.cond = icmp eq i32 %i, %x
+  %widenable_cond = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond = and i1 %guard.cond, %widenable_cond
+  br i1 %exiplicit_guard_cond, label %guarded, label %deopt, !prof !0
+
+deopt:                                            ; preds = %loop
+  %deoptcall = call i32 (...) @llvm.experimental.deoptimize.i32(i32 9) [ "deopt"() ]
+  ret i32 %deoptcall
+
+guarded:                                          ; preds = %loop
+  %i.i64 = zext i32 %i to i64
+  %array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i.i64
+  %array.i = load i32, i32* %array.i.ptr, align 4
+  %loop.acc.next = add i32 %loop.acc, %array.i
+  %i.next = add nsw i32 %i, 1
+  %continue = icmp slt i32 %i.next, %n
+  br i1 %continue, label %loop, label %exit
+
+exit:                                             ; preds = %guarded, %entry
+  %result = phi i32 [ 0, %entry ], [ %loop.acc.next, %guarded ]
+  ret i32 %result
+}
+
+define i32 @unsigned_loop_0_to_n_hoist_length(i32* %array, i16 %length.i16, i32 %n) {
+; CHECK-LABEL: @unsigned_loop_0_to_n_hoist_length(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP5]], label [[EXIT:%.*]], label [[LOOP_PREHEADER:%.*]]
+; CHECK:       loop.preheader:
+; CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[LENGTH_I16:%.*]] to i32
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ule i32 [[N]], [[TMP0]]
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ult i32 0, [[TMP0]]
+; CHECK-NEXT:    [[TMP3:%.*]] = and i1 [[TMP2]], [[TMP1]]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[GUARDED:%.*]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[GUARDED]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[LENGTH:%.*]] = zext i16 [[LENGTH_I16]] to i32
+; CHECK-NEXT:    [[WITHIN_BOUNDS:%.*]] = icmp ult i32 [[I]], [[LENGTH]]
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[WITHIN_BOUNDS]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    [[TMP4:%.*]] = and i1 [[TMP3]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[TMP4]], label [[GUARDED]], label [[DEOPT:%.*]], !prof !0
+;
+entry:
+  %tmp5 = icmp eq i32 %n, 0
+  br i1 %tmp5, label %exit, label %loop.preheader
+
+loop.preheader:                                   ; preds = %entry
+  br label %loop
+
+loop:                                             ; preds = %guarded, %loop.preheader
+  %loop.acc = phi i32 [ %loop.acc.next, %guarded ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %guarded ], [ 0, %loop.preheader ]
+  %length = zext i16 %length.i16 to i32
+  %within.bounds = icmp ult i32 %i, %length
+  %widenable_cond = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond = and i1 %within.bounds, %widenable_cond
+  br i1 %exiplicit_guard_cond, label %guarded, label %deopt, !prof !0
+
+deopt:                                            ; preds = %loop
+  %deoptcall = call i32 (...) @llvm.experimental.deoptimize.i32(i32 9) [ "deopt"() ]
+  ret i32 %deoptcall
+
+guarded:                                          ; preds = %loop
+  %i.i64 = zext i32 %i to i64
+  %array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i.i64
+  %array.i = load i32, i32* %array.i.ptr, align 4
+  %loop.acc.next = add i32 %loop.acc, %array.i
+  %i.next = add nuw i32 %i, 1
+  %continue = icmp ult i32 %i.next, %n
+  br i1 %continue, label %loop, label %exit
+
+exit:                                             ; preds = %guarded, %entry
+  %result = phi i32 [ 0, %entry ], [ %loop.acc.next, %guarded ]
+  ret i32 %result
+}
+
+define i32 @unsigned_loop_0_to_n_cant_hoist_length(i32* %array, i32 %length, i32 %divider, i32 %n) {
+; CHECK-LABEL: @unsigned_loop_0_to_n_cant_hoist_length(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP5]], label [[EXIT:%.*]], label [[LOOP_PREHEADER:%.*]]
+; CHECK:       loop.preheader:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[GUARDED:%.*]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[GUARDED]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[LENGTH_UDIV:%.*]] = udiv i32 [[LENGTH:%.*]], [[DIVIDER:%.*]]
+; CHECK-NEXT:    [[WITHIN_BOUNDS:%.*]] = icmp ult i32 [[I]], [[LENGTH_UDIV]]
+; CHECK-NEXT:    [[WIDENABLE_COND:%.*]] = call i1 @llvm.experimental.widenable.condition()
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[WITHIN_BOUNDS]], [[WIDENABLE_COND]]
+; CHECK-NEXT:    br i1 [[EXIPLICIT_GUARD_COND]], label [[GUARDED]], label [[DEOPT:%.*]], !prof !0
+;
+entry:
+  %tmp5 = icmp eq i32 %n, 0
+  br i1 %tmp5, label %exit, label %loop.preheader
+
+loop.preheader:                                   ; preds = %entry
+  br label %loop
+
+loop:                                             ; preds = %guarded, %loop.preheader
+  %loop.acc = phi i32 [ %loop.acc.next, %guarded ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %guarded ], [ 0, %loop.preheader ]
+  %length.udiv = udiv i32 %length, %divider
+  %within.bounds = icmp ult i32 %i, %length.udiv
+  %widenable_cond = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond = and i1 %within.bounds, %widenable_cond
+  br i1 %exiplicit_guard_cond, label %guarded, label %deopt, !prof !0
+
+deopt:                                            ; preds = %loop
+  %deoptcall = call i32 (...) @llvm.experimental.deoptimize.i32(i32 9) [ "deopt"() ]
+  ret i32 %deoptcall
+
+guarded:                                          ; preds = %loop
+  %i.i64 = zext i32 %i to i64
+  %array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i.i64
+  %array.i = load i32, i32* %array.i.ptr, align 4
+  %loop.acc.next = add i32 %loop.acc, %array.i
+  %i.next = add nuw i32 %i, 1
+  %continue = icmp ult i32 %i.next, %n
+  br i1 %continue, label %loop, label %exit
+
+exit:                                             ; preds = %guarded, %entry
+  %result = phi i32 [ 0, %entry ], [ %loop.acc.next, %guarded ]
+  ret i32 %result
+}
+
+declare i32 @llvm.experimental.deoptimize.i32(...)
+
+; Function Attrs: inaccessiblememonly nounwind
+declare i1 @llvm.experimental.widenable.condition() #0
+
+attributes #0 = { inaccessiblememonly nounwind }
+
+!0 = !{!"branch_weights", i32 1048576, i32 1}
+!1 = !{i32 1, i32 -2147483648}
diff --git a/test/Transforms/LoopReroll/basic.ll b/test/Transforms/LoopReroll/basic.ll
index 6e2f2fcabf1316cdeffe69145c031bbd55862019..b415b2653ac87bfec0d952d1aa8cc039a2b8562e 100644
--- a/test/Transforms/LoopReroll/basic.ll
+++ b/test/Transforms/LoopReroll/basic.ll
@@ -785,6 +785,30 @@ bb19:                                             ; preds = %bb3
   ret void
 }
 
+define void @bad_step(i32* nocapture readnone %x) #0 {
+entry:
+  br label %for.body
+
+for.body:                                         ; preds = %for.body, %entry
+  %i.08 = phi i32 [ 0, %entry ], [ %add3, %for.body ]
+  %call = tail call i32 @foo(i32 %i.08) #1
+  %add = add nsw i32 %i.08, 2
+  %call1 = tail call i32 @foo(i32 %add) #1
+  %add2 = add nsw i32 %i.08, 3
+  %call3 = tail call i32 @foo(i32 %add2) #1
+  %add3 = add nsw i32 %i.08, 6
+  %exitcond = icmp sge i32 %add3, 500
+  br i1 %exitcond, label %for.end, label %for.body
+
+; CHECK-LABEL: @bad_step
+; CHECK: %add = add nsw i32 %i.08, 2
+; CHECK: %add2 = add nsw i32 %i.08, 3
+; CHECK: %add3 = add nsw i32 %i.08, 6
+
+for.end:                                          ; preds = %for.body
+  ret void
+}
+
 attributes #0 = { nounwind uwtable }
 attributes #1 = { nounwind }
 
diff --git a/test/Transforms/LoopReroll/reroll_with_dbg.ll b/test/Transforms/LoopReroll/reroll_with_dbg.ll
index 9351846c7756397dce682e7e82b65871cfd765af..3a752b23b4e408239ff069967a8767350b22db97 100644
--- a/test/Transforms/LoopReroll/reroll_with_dbg.ll
+++ b/test/Transforms/LoopReroll/reroll_with_dbg.ll
@@ -85,7 +85,7 @@ attributes #1 = { nounwind readnone }
 !llvm.module.flags = !{!17, !18, !19, !20}
 !llvm.ident = !{!21}
 
-!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 3.8.0 (http://llvm.org/git/clang.git b1fbc23058e7fa1cdd954ab97ba84f1c549c9879) (http://llvm.org/git/llvm.git 054da58c5398a721d4dab7af63d7de8d7a1e1a1c)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2)
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 3.8.0", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2)
 !1 = !DIFile(filename: "test.c", directory: "/home/weimingz/llvm-build/release/community-tip")
 !2 = !{}
 !4 = distinct !DISubprogram(name: "foo", scope: !1, file: !1, line: 1, type: !5, isLocal: false, isDefinition: true, scopeLine: 1, flags: DIFlagPrototyped, isOptimized: true, unit: !0, retainedNodes: !11)
@@ -105,7 +105,7 @@ attributes #1 = { nounwind readnone }
 !18 = !{i32 2, !"Debug Info Version", i32 3}
 !19 = !{i32 1, !"wchar_size", i32 4}
 !20 = !{i32 1, !"min_enum_size", i32 4}
-!21 = !{!"clang version 3.8.0 (http://llvm.org/git/clang.git b1fbc23058e7fa1cdd954ab97ba84f1c549c9879) (http://llvm.org/git/llvm.git 054da58c5398a721d4dab7af63d7de8d7a1e1a1c)"}
+!21 = !{!"clang version 3.8.0"}
 !22 = !DIExpression()
 !23 = !DILocation(line: 1, column: 27, scope: !4)
 !24 = !DILocation(line: 1, column: 47, scope: !4)
diff --git a/test/Transforms/LoopRotate/phi-dbgvalue.ll b/test/Transforms/LoopRotate/phi-dbgvalue.ll
index 1f7e129c26e3a60ce86becddad5eb26314efd15b..c4e13dc710e7d995f5a2094b2326d3a90f50450f 100644
--- a/test/Transforms/LoopRotate/phi-dbgvalue.ll
+++ b/test/Transforms/LoopRotate/phi-dbgvalue.ll
@@ -48,12 +48,12 @@ attributes #3 = { nounwind }
 !llvm.module.flags = !{!3, !4}
 !llvm.ident = !{!5}
 
-!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 5.0.0 (http://llvm.org/git/clang.git 0f3ed908c1f13f83da4b240f7595eb8d05e0a754) (http://llvm.org/git/llvm.git 8e270f5a6b8ceb0f3ac3ef1ffb83c5e29b44ae68)", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !2)
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 5.0.0", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !2)
 !1 = !DIFile(filename: "debug-phi.c", directory: "/work/projects/src/tests/debug")
 !2 = !{}
 !3 = !{i32 2, !"Dwarf Version", i32 4}
 !4 = !{i32 2, !"Debug Info Version", i32 3}
-!5 = !{!"clang version 5.0.0 (http://llvm.org/git/clang.git 0f3ed908c1f13f83da4b240f7595eb8d05e0a754) (http://llvm.org/git/llvm.git 8e270f5a6b8ceb0f3ac3ef1ffb83c5e29b44ae68)"}
+!5 = !{!"clang version 5.0.0"}
 !6 = distinct !DISubprogram(name: "func", scope: !1, file: !1, line: 2, type: !7, isLocal: false, isDefinition: true, scopeLine: 2, flags: DIFlagPrototyped, isOptimized: false, unit: !0, retainedNodes: !2)
 !7 = !DISubroutineType(types: !8)
 !8 = !{null, !9}
diff --git a/test/Transforms/LoopSimplifyCFG/constant-fold-branch.ll b/test/Transforms/LoopSimplifyCFG/constant-fold-branch.ll
index 7998aa360ddb24a6127a4a6c6a49b553640b3cbd..d89193edf93e04dc27649ca9becc40271a71dd28 100644
--- a/test/Transforms/LoopSimplifyCFG/constant-fold-branch.ll
+++ b/test/Transforms/LoopSimplifyCFG/constant-fold-branch.ll
@@ -88,18 +88,12 @@ define i32 @dead_block_test_branch_loop(i32 %end) {
 ; CHECK-NEXT:  preheader:
 ; CHECK-NEXT:    br label [[HEADER:%.*]]
 ; CHECK:       header:
-; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, [[PREHEADER:%.*]] ], [ [[I_INC:%.*]], [[BACKEDGE:%.*]] ]
-; CHECK-NEXT:    br i1 true, label [[BACKEDGE]], label [[DEAD:%.*]]
-; CHECK:       dead:
-; CHECK-NEXT:    [[I_2:%.*]] = add i32 [[I]], 1
-; CHECK-NEXT:    br label [[BACKEDGE]]
-; CHECK:       backedge:
-; CHECK-NEXT:    [[I_1:%.*]] = phi i32 [ [[I]], [[HEADER]] ], [ [[I_2]], [[DEAD]] ]
-; CHECK-NEXT:    [[I_INC]] = add i32 [[I_1]], 1
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, [[PREHEADER:%.*]] ], [ [[I_INC:%.*]], [[HEADER]] ]
+; CHECK-NEXT:    [[I_INC]] = add i32 [[I]], 1
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[I_INC]], [[END:%.*]]
 ; CHECK-NEXT:    br i1 [[CMP]], label [[HEADER]], label [[EXIT:%.*]]
 ; CHECK:       exit:
-; CHECK-NEXT:    [[I_INC_LCSSA:%.*]] = phi i32 [ [[I_INC]], [[BACKEDGE]] ]
+; CHECK-NEXT:    [[I_INC_LCSSA:%.*]] = phi i32 [ [[I_INC]], [[HEADER]] ]
 ; CHECK-NEXT:    ret i32 [[I_INC_LCSSA]]
 ;
 preheader:
@@ -129,22 +123,12 @@ define i32 @dead_block_test_switch_loop(i32 %end) {
 ; CHECK-NEXT:  preheader:
 ; CHECK-NEXT:    br label [[HEADER:%.*]]
 ; CHECK:       header:
-; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, [[PREHEADER:%.*]] ], [ [[I_INC:%.*]], [[BACKEDGE:%.*]] ]
-; CHECK-NEXT:    switch i32 1, label [[DEAD:%.*]] [
-; CHECK-NEXT:    i32 0, label [[DEAD]]
-; CHECK-NEXT:    i32 1, label [[BACKEDGE]]
-; CHECK-NEXT:    i32 2, label [[DEAD]]
-; CHECK-NEXT:    ]
-; CHECK:       dead:
-; CHECK-NEXT:    [[I_2:%.*]] = add i32 [[I]], 1
-; CHECK-NEXT:    br label [[BACKEDGE]]
-; CHECK:       backedge:
-; CHECK-NEXT:    [[I_1:%.*]] = phi i32 [ [[I]], [[HEADER]] ], [ [[I_2]], [[DEAD]] ]
-; CHECK-NEXT:    [[I_INC]] = add i32 [[I_1]], 1
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, [[PREHEADER:%.*]] ], [ [[I_INC:%.*]], [[HEADER]] ]
+; CHECK-NEXT:    [[I_INC]] = add i32 [[I]], 1
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[I_INC]], [[END:%.*]]
 ; CHECK-NEXT:    br i1 [[CMP]], label [[HEADER]], label [[EXIT:%.*]]
 ; CHECK:       exit:
-; CHECK-NEXT:    [[I_INC_LCSSA:%.*]] = phi i32 [ [[I_INC]], [[BACKEDGE]] ]
+; CHECK-NEXT:    [[I_INC_LCSSA:%.*]] = phi i32 [ [[I_INC]], [[HEADER]] ]
 ; CHECK-NEXT:    ret i32 [[I_INC_LCSSA]]
 ;
 preheader:
@@ -175,18 +159,12 @@ define i32 @dead_block_propogate_test_branch_loop(i32 %end) {
 ; CHECK-NEXT:  preheader:
 ; CHECK-NEXT:    br label [[HEADER:%.*]]
 ; CHECK:       header:
-; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, [[PREHEADER:%.*]] ], [ [[I_INC:%.*]], [[BACKEDGE:%.*]] ]
-; CHECK-NEXT:    br i1 true, label [[BACKEDGE]], label [[DEAD:%.*]]
-; CHECK:       dead:
-; CHECK-NEXT:    [[I_2:%.*]] = add i32 [[I]], 1
-; CHECK-NEXT:    br label [[BACKEDGE]]
-; CHECK:       backedge:
-; CHECK-NEXT:    [[I_1:%.*]] = phi i32 [ [[I]], [[HEADER]] ], [ [[I_2]], [[DEAD]] ]
-; CHECK-NEXT:    [[I_INC]] = add i32 [[I_1]], 1
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, [[PREHEADER:%.*]] ], [ [[I_INC:%.*]], [[HEADER]] ]
+; CHECK-NEXT:    [[I_INC]] = add i32 [[I]], 1
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[I_INC]], [[END:%.*]]
 ; CHECK-NEXT:    br i1 [[CMP]], label [[HEADER]], label [[EXIT:%.*]]
 ; CHECK:       exit:
-; CHECK-NEXT:    [[I_INC_LCSSA:%.*]] = phi i32 [ [[I_INC]], [[BACKEDGE]] ]
+; CHECK-NEXT:    [[I_INC_LCSSA:%.*]] = phi i32 [ [[I_INC]], [[HEADER]] ]
 ; CHECK-NEXT:    ret i32 [[I_INC_LCSSA]]
 ;
 preheader:
@@ -219,22 +197,12 @@ define i32 @dead_block_propogate_test_switch_loop(i32 %end) {
 ; CHECK-NEXT:  preheader:
 ; CHECK-NEXT:    br label [[HEADER:%.*]]
 ; CHECK:       header:
-; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, [[PREHEADER:%.*]] ], [ [[I_INC:%.*]], [[BACKEDGE:%.*]] ]
-; CHECK-NEXT:    switch i32 1, label [[DEAD:%.*]] [
-; CHECK-NEXT:    i32 0, label [[DEAD]]
-; CHECK-NEXT:    i32 1, label [[BACKEDGE]]
-; CHECK-NEXT:    i32 2, label [[DEAD]]
-; CHECK-NEXT:    ]
-; CHECK:       dead:
-; CHECK-NEXT:    [[I_2:%.*]] = add i32 [[I]], 1
-; CHECK-NEXT:    br label [[BACKEDGE]]
-; CHECK:       backedge:
-; CHECK-NEXT:    [[I_1:%.*]] = phi i32 [ [[I]], [[HEADER]] ], [ [[I_2]], [[DEAD]] ]
-; CHECK-NEXT:    [[I_INC]] = add i32 [[I_1]], 1
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, [[PREHEADER:%.*]] ], [ [[I_INC:%.*]], [[HEADER]] ]
+; CHECK-NEXT:    [[I_INC]] = add i32 [[I]], 1
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[I_INC]], [[END:%.*]]
 ; CHECK-NEXT:    br i1 [[CMP]], label [[HEADER]], label [[EXIT:%.*]]
 ; CHECK:       exit:
-; CHECK-NEXT:    [[I_INC_LCSSA:%.*]] = phi i32 [ [[I_INC]], [[BACKEDGE]] ]
+; CHECK-NEXT:    [[I_INC_LCSSA:%.*]] = phi i32 [ [[I_INC]], [[HEADER]] ]
 ; CHECK-NEXT:    ret i32 [[I_INC_LCSSA]]
 ;
 preheader:
@@ -268,24 +236,25 @@ exit:
 define i32 @dead_exit_test_branch_loop(i32 %end) {
 ; CHECK-LABEL: @dead_exit_test_branch_loop(
 ; CHECK-NEXT:  preheader:
+; CHECK-NEXT:    switch i32 0, label [[PREHEADER_SPLIT:%.*]] [
+; CHECK-NEXT:    i32 1, label [[DEAD:%.*]]
+; CHECK-NEXT:    ]
+; CHECK:       preheader-split:
 ; CHECK-NEXT:    br label [[HEADER:%.*]]
 ; CHECK:       header:
-; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, [[PREHEADER:%.*]] ], [ [[I_INC:%.*]], [[BACKEDGE:%.*]] ]
-; CHECK-NEXT:    br i1 true, label [[BACKEDGE]], label [[DEAD:%.*]]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, [[PREHEADER_SPLIT]] ], [ [[I_INC:%.*]], [[HEADER]] ]
+; CHECK-NEXT:    [[I_INC]] = add i32 [[I]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[I_INC]], [[END:%.*]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[HEADER]], label [[EXIT_LOOPEXIT:%.*]]
 ; CHECK:       dead:
-; CHECK-NEXT:    [[I_LCSSA:%.*]] = phi i32 [ [[I]], [[HEADER]] ]
 ; CHECK-NEXT:    br label [[DUMMY:%.*]]
 ; CHECK:       dummy:
 ; CHECK-NEXT:    br label [[EXIT:%.*]]
-; CHECK:       backedge:
-; CHECK-NEXT:    [[I_INC]] = add i32 [[I]], 1
-; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[I_INC]], [[END:%.*]]
-; CHECK-NEXT:    br i1 [[CMP]], label [[HEADER]], label [[EXIT_LOOPEXIT:%.*]]
 ; CHECK:       exit.loopexit:
-; CHECK-NEXT:    [[I_INC_LCSSA:%.*]] = phi i32 [ [[I_INC]], [[BACKEDGE]] ]
+; CHECK-NEXT:    [[I_INC_LCSSA:%.*]] = phi i32 [ [[I_INC]], [[HEADER]] ]
 ; CHECK-NEXT:    br label [[EXIT]]
 ; CHECK:       exit:
-; CHECK-NEXT:    [[I_1:%.*]] = phi i32 [ [[I_LCSSA]], [[DUMMY]] ], [ [[I_INC_LCSSA]], [[EXIT_LOOPEXIT]] ]
+; CHECK-NEXT:    [[I_1:%.*]] = phi i32 [ undef, [[DUMMY]] ], [ [[I_INC_LCSSA]], [[EXIT_LOOPEXIT]] ]
 ; CHECK-NEXT:    ret i32 [[I_1]]
 ;
 preheader:
@@ -316,28 +285,25 @@ exit:
 define i32 @dead_exit_test_switch_loop(i32 %end) {
 ; CHECK-LABEL: @dead_exit_test_switch_loop(
 ; CHECK-NEXT:  preheader:
+; CHECK-NEXT:    switch i32 0, label [[PREHEADER_SPLIT:%.*]] [
+; CHECK-NEXT:    i32 1, label [[DEAD:%.*]]
+; CHECK-NEXT:    ]
+; CHECK:       preheader-split:
 ; CHECK-NEXT:    br label [[HEADER:%.*]]
 ; CHECK:       header:
-; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, [[PREHEADER:%.*]] ], [ [[I_INC:%.*]], [[BACKEDGE:%.*]] ]
-; CHECK-NEXT:    switch i32 1, label [[DEAD:%.*]] [
-; CHECK-NEXT:    i32 0, label [[DEAD]]
-; CHECK-NEXT:    i32 1, label [[BACKEDGE]]
-; CHECK-NEXT:    i32 2, label [[DEAD]]
-; CHECK-NEXT:    ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, [[PREHEADER_SPLIT]] ], [ [[I_INC:%.*]], [[HEADER]] ]
+; CHECK-NEXT:    [[I_INC]] = add i32 [[I]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[I_INC]], [[END:%.*]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[HEADER]], label [[EXIT_LOOPEXIT:%.*]]
 ; CHECK:       dead:
-; CHECK-NEXT:    [[I_LCSSA:%.*]] = phi i32 [ [[I]], [[HEADER]] ], [ [[I]], [[HEADER]] ], [ [[I]], [[HEADER]] ]
 ; CHECK-NEXT:    br label [[DUMMY:%.*]]
 ; CHECK:       dummy:
 ; CHECK-NEXT:    br label [[EXIT:%.*]]
-; CHECK:       backedge:
-; CHECK-NEXT:    [[I_INC]] = add i32 [[I]], 1
-; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[I_INC]], [[END:%.*]]
-; CHECK-NEXT:    br i1 [[CMP]], label [[HEADER]], label [[EXIT_LOOPEXIT:%.*]]
 ; CHECK:       exit.loopexit:
-; CHECK-NEXT:    [[I_INC_LCSSA:%.*]] = phi i32 [ [[I_INC]], [[BACKEDGE]] ]
+; CHECK-NEXT:    [[I_INC_LCSSA:%.*]] = phi i32 [ [[I_INC]], [[HEADER]] ]
 ; CHECK-NEXT:    br label [[EXIT]]
 ; CHECK:       exit:
-; CHECK-NEXT:    [[I_1:%.*]] = phi i32 [ [[I_LCSSA]], [[DUMMY]] ], [ [[I_INC_LCSSA]], [[EXIT_LOOPEXIT]] ]
+; CHECK-NEXT:    [[I_1:%.*]] = phi i32 [ undef, [[DUMMY]] ], [ [[I_INC_LCSSA]], [[EXIT_LOOPEXIT]] ]
 ; CHECK-NEXT:    ret i32 [[I_1]]
 ;
 preheader:
@@ -463,32 +429,19 @@ define i32 @dead_sub_loop_test_branch_loop(i32 %end) {
 ; CHECK-NEXT:  preheader:
 ; CHECK-NEXT:    br label [[HEADER:%.*]]
 ; CHECK:       header:
-; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, [[PREHEADER:%.*]] ], [ [[I_INC:%.*]], [[BACKEDGE:%.*]] ]
-; CHECK-NEXT:    br i1 true, label [[LIVE_PREHEADER:%.*]], label [[DEAD_PREHEADER:%.*]]
-; CHECK:       live_preheader:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, [[PREHEADER:%.*]] ], [ [[I_INC:%.*]], [[EXIT_A:%.*]] ]
 ; CHECK-NEXT:    br label [[LIVE_LOOP:%.*]]
 ; CHECK:       live_loop:
-; CHECK-NEXT:    [[A:%.*]] = phi i32 [ 0, [[LIVE_PREHEADER]] ], [ [[A_INC:%.*]], [[LIVE_LOOP]] ]
+; CHECK-NEXT:    [[A:%.*]] = phi i32 [ 0, [[HEADER]] ], [ [[A_INC:%.*]], [[LIVE_LOOP]] ]
 ; CHECK-NEXT:    [[A_INC]] = add i32 [[A]], 1
 ; CHECK-NEXT:    [[CMP_A:%.*]] = icmp slt i32 [[A_INC]], [[END:%.*]]
-; CHECK-NEXT:    br i1 [[CMP_A]], label [[LIVE_LOOP]], label [[EXIT_A:%.*]]
+; CHECK-NEXT:    br i1 [[CMP_A]], label [[LIVE_LOOP]], label [[EXIT_A]]
 ; CHECK:       exit.a:
-; CHECK-NEXT:    br label [[BACKEDGE]]
-; CHECK:       dead_preheader:
-; CHECK-NEXT:    br label [[DEAD_LOOP:%.*]]
-; CHECK:       dead_loop:
-; CHECK-NEXT:    [[B:%.*]] = phi i32 [ 0, [[DEAD_PREHEADER]] ], [ [[B_INC:%.*]], [[DEAD_LOOP]] ]
-; CHECK-NEXT:    [[B_INC]] = add i32 [[B]], 1
-; CHECK-NEXT:    [[CMP_B:%.*]] = icmp slt i32 [[B_INC]], [[END]]
-; CHECK-NEXT:    br i1 [[CMP_B]], label [[DEAD_LOOP]], label [[EXIT_B:%.*]]
-; CHECK:       exit.b:
-; CHECK-NEXT:    br label [[BACKEDGE]]
-; CHECK:       backedge:
 ; CHECK-NEXT:    [[I_INC]] = add i32 [[I]], 1
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[I_INC]], [[END]]
 ; CHECK-NEXT:    br i1 [[CMP]], label [[HEADER]], label [[EXIT:%.*]]
 ; CHECK:       exit:
-; CHECK-NEXT:    [[I_INC_LCSSA:%.*]] = phi i32 [ [[I_INC]], [[BACKEDGE]] ]
+; CHECK-NEXT:    [[I_INC_LCSSA:%.*]] = phi i32 [ [[I_INC]], [[EXIT_A]] ]
 ; CHECK-NEXT:    ret i32 [[I_INC_LCSSA]]
 ;
 preheader:
@@ -536,36 +489,19 @@ define i32 @dead_sub_loop_test_switch_loop(i32 %end) {
 ; CHECK-NEXT:  preheader:
 ; CHECK-NEXT:    br label [[HEADER:%.*]]
 ; CHECK:       header:
-; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, [[PREHEADER:%.*]] ], [ [[I_INC:%.*]], [[BACKEDGE:%.*]] ]
-; CHECK-NEXT:    switch i32 1, label [[DEAD_PREHEADER:%.*]] [
-; CHECK-NEXT:    i32 0, label [[DEAD_PREHEADER]]
-; CHECK-NEXT:    i32 1, label [[LIVE_PREHEADER:%.*]]
-; CHECK-NEXT:    i32 2, label [[DEAD_PREHEADER]]
-; CHECK-NEXT:    ]
-; CHECK:       live_preheader:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, [[PREHEADER:%.*]] ], [ [[I_INC:%.*]], [[EXIT_A:%.*]] ]
 ; CHECK-NEXT:    br label [[LIVE_LOOP:%.*]]
 ; CHECK:       live_loop:
-; CHECK-NEXT:    [[A:%.*]] = phi i32 [ 0, [[LIVE_PREHEADER]] ], [ [[A_INC:%.*]], [[LIVE_LOOP]] ]
+; CHECK-NEXT:    [[A:%.*]] = phi i32 [ 0, [[HEADER]] ], [ [[A_INC:%.*]], [[LIVE_LOOP]] ]
 ; CHECK-NEXT:    [[A_INC]] = add i32 [[A]], 1
 ; CHECK-NEXT:    [[CMP_A:%.*]] = icmp slt i32 [[A_INC]], [[END:%.*]]
-; CHECK-NEXT:    br i1 [[CMP_A]], label [[LIVE_LOOP]], label [[EXIT_A:%.*]]
+; CHECK-NEXT:    br i1 [[CMP_A]], label [[LIVE_LOOP]], label [[EXIT_A]]
 ; CHECK:       exit.a:
-; CHECK-NEXT:    br label [[BACKEDGE]]
-; CHECK:       dead_preheader:
-; CHECK-NEXT:    br label [[DEAD_LOOP:%.*]]
-; CHECK:       dead_loop:
-; CHECK-NEXT:    [[B:%.*]] = phi i32 [ 0, [[DEAD_PREHEADER]] ], [ [[B_INC:%.*]], [[DEAD_LOOP]] ]
-; CHECK-NEXT:    [[B_INC]] = add i32 [[B]], 1
-; CHECK-NEXT:    [[CMP_B:%.*]] = icmp slt i32 [[B_INC]], [[END]]
-; CHECK-NEXT:    br i1 [[CMP_B]], label [[DEAD_LOOP]], label [[EXIT_B:%.*]]
-; CHECK:       exit.b:
-; CHECK-NEXT:    br label [[BACKEDGE]]
-; CHECK:       backedge:
 ; CHECK-NEXT:    [[I_INC]] = add i32 [[I]], 1
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[I_INC]], [[END]]
 ; CHECK-NEXT:    br i1 [[CMP]], label [[HEADER]], label [[EXIT:%.*]]
 ; CHECK:       exit:
-; CHECK-NEXT:    [[I_INC_LCSSA:%.*]] = phi i32 [ [[I_INC]], [[BACKEDGE]] ]
+; CHECK-NEXT:    [[I_INC_LCSSA:%.*]] = phi i32 [ [[I_INC]], [[EXIT_A]] ]
 ; CHECK-NEXT:    ret i32 [[I_INC_LCSSA]]
 ;
 preheader:
@@ -615,21 +551,18 @@ exit:
 define i32 @inf_loop_test_branch_loop(i32 %end) {
 ; CHECK-LABEL: @inf_loop_test_branch_loop(
 ; CHECK-NEXT:  preheader:
+; CHECK-NEXT:    switch i32 0, label [[PREHEADER_SPLIT:%.*]] [
+; CHECK-NEXT:    i32 1, label [[EXIT:%.*]]
+; CHECK-NEXT:    ]
+; CHECK:       preheader-split:
 ; CHECK-NEXT:    br label [[HEADER:%.*]]
 ; CHECK:       header:
-; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, [[PREHEADER:%.*]] ], [ [[I_INC:%.*]], [[BACKEDGE:%.*]] ]
-; CHECK-NEXT:    br i1 true, label [[BACKEDGE]], label [[DEAD:%.*]]
-; CHECK:       dead:
-; CHECK-NEXT:    [[I_2:%.*]] = add i32 [[I]], 1
-; CHECK-NEXT:    br label [[BACKEDGE]]
-; CHECK:       backedge:
-; CHECK-NEXT:    [[I_1:%.*]] = phi i32 [ [[I]], [[HEADER]] ], [ [[I_2]], [[DEAD]] ]
-; CHECK-NEXT:    [[I_INC]] = add i32 [[I_1]], 1
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, [[PREHEADER_SPLIT]] ], [ [[I_INC:%.*]], [[HEADER]] ]
+; CHECK-NEXT:    [[I_INC]] = add i32 [[I]], 1
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[I_INC]], [[END:%.*]]
-; CHECK-NEXT:    br i1 true, label [[HEADER]], label [[EXIT:%.*]]
+; CHECK-NEXT:    br label [[HEADER]]
 ; CHECK:       exit:
-; CHECK-NEXT:    [[I_INC_LCSSA:%.*]] = phi i32 [ [[I_INC]], [[BACKEDGE]] ]
-; CHECK-NEXT:    ret i32 [[I_INC_LCSSA]]
+; CHECK-NEXT:    ret i32 undef
 ;
 preheader:
   br label %header
@@ -658,25 +591,18 @@ exit:
 define i32 @inf_loop_test_switch_loop(i32 %end) {
 ; CHECK-LABEL: @inf_loop_test_switch_loop(
 ; CHECK-NEXT:  preheader:
+; CHECK-NEXT:    switch i32 0, label [[PREHEADER_SPLIT:%.*]] [
+; CHECK-NEXT:    i32 1, label [[EXIT:%.*]]
+; CHECK-NEXT:    ]
+; CHECK:       preheader-split:
 ; CHECK-NEXT:    br label [[HEADER:%.*]]
 ; CHECK:       header:
-; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, [[PREHEADER:%.*]] ], [ [[I_INC:%.*]], [[BACKEDGE:%.*]] ]
-; CHECK-NEXT:    switch i32 1, label [[DEAD:%.*]] [
-; CHECK-NEXT:    i32 0, label [[DEAD]]
-; CHECK-NEXT:    i32 1, label [[BACKEDGE]]
-; CHECK-NEXT:    i32 2, label [[DEAD]]
-; CHECK-NEXT:    ]
-; CHECK:       dead:
-; CHECK-NEXT:    [[I_2:%.*]] = add i32 [[I]], 1
-; CHECK-NEXT:    br label [[BACKEDGE]]
-; CHECK:       backedge:
-; CHECK-NEXT:    [[I_1:%.*]] = phi i32 [ [[I]], [[HEADER]] ], [ [[I_2]], [[DEAD]] ]
-; CHECK-NEXT:    [[I_INC]] = add i32 [[I_1]], 1
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, [[PREHEADER_SPLIT]] ], [ [[I_INC:%.*]], [[HEADER]] ]
+; CHECK-NEXT:    [[I_INC]] = add i32 [[I]], 1
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[I_INC]], [[END:%.*]]
-; CHECK-NEXT:    br i1 true, label [[HEADER]], label [[EXIT:%.*]]
+; CHECK-NEXT:    br label [[HEADER]]
 ; CHECK:       exit:
-; CHECK-NEXT:    [[I_INC_LCSSA:%.*]] = phi i32 [ [[I_INC]], [[BACKEDGE]] ]
-; CHECK-NEXT:    ret i32 [[I_INC_LCSSA]]
+; CHECK-NEXT:    ret i32 undef
 ;
 preheader:
   br label %header
@@ -898,18 +824,12 @@ define i32 @partial_sub_loop_test_branch_loop(i32 %end) {
 ; CHECK-NEXT:    [[J:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[J_INC:%.*]], [[OUTER_BACKEDGE:%.*]] ]
 ; CHECK-NEXT:    br label [[HEADER:%.*]]
 ; CHECK:       header:
-; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, [[OUTER_HEADER]] ], [ [[I_INC:%.*]], [[BACKEDGE:%.*]] ]
-; CHECK-NEXT:    br i1 true, label [[BACKEDGE]], label [[DEAD:%.*]]
-; CHECK:       dead:
-; CHECK-NEXT:    [[I_2:%.*]] = add i32 [[I]], 1
-; CHECK-NEXT:    br label [[BACKEDGE]]
-; CHECK:       backedge:
-; CHECK-NEXT:    [[I_1:%.*]] = phi i32 [ [[I]], [[HEADER]] ], [ [[I_2]], [[DEAD]] ]
-; CHECK-NEXT:    [[I_INC]] = add i32 [[I_1]], 1
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, [[OUTER_HEADER]] ], [ [[I_INC:%.*]], [[HEADER]] ]
+; CHECK-NEXT:    [[I_INC]] = add i32 [[I]], 1
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[I_INC]], [[END:%.*]]
 ; CHECK-NEXT:    br i1 [[CMP]], label [[HEADER]], label [[OUTER_BACKEDGE]]
 ; CHECK:       outer_backedge:
-; CHECK-NEXT:    [[I_INC_LCSSA:%.*]] = phi i32 [ [[I_INC]], [[BACKEDGE]] ]
+; CHECK-NEXT:    [[I_INC_LCSSA:%.*]] = phi i32 [ [[I_INC]], [[HEADER]] ]
 ; CHECK-NEXT:    [[J_INC]] = add i32 [[J]], 1
 ; CHECK-NEXT:    [[CMP_J:%.*]] = icmp slt i32 [[J_INC]], [[END]]
 ; CHECK-NEXT:    br i1 [[CMP_J]], label [[OUTER_HEADER]], label [[EXIT:%.*]]
@@ -958,22 +878,12 @@ define i32 @partial_sub_loop_test_switch_loop(i32 %end) {
 ; CHECK-NEXT:    [[J:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[J_INC:%.*]], [[OUTER_BACKEDGE:%.*]] ]
 ; CHECK-NEXT:    br label [[HEADER:%.*]]
 ; CHECK:       header:
-; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, [[OUTER_HEADER]] ], [ [[I_INC:%.*]], [[BACKEDGE:%.*]] ]
-; CHECK-NEXT:    switch i32 1, label [[DEAD:%.*]] [
-; CHECK-NEXT:    i32 0, label [[DEAD]]
-; CHECK-NEXT:    i32 1, label [[BACKEDGE]]
-; CHECK-NEXT:    i32 2, label [[DEAD]]
-; CHECK-NEXT:    ]
-; CHECK:       dead:
-; CHECK-NEXT:    [[I_2:%.*]] = add i32 [[I]], 1
-; CHECK-NEXT:    br label [[BACKEDGE]]
-; CHECK:       backedge:
-; CHECK-NEXT:    [[I_1:%.*]] = phi i32 [ [[I]], [[HEADER]] ], [ [[I_2]], [[DEAD]] ]
-; CHECK-NEXT:    [[I_INC]] = add i32 [[I_1]], 1
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, [[OUTER_HEADER]] ], [ [[I_INC:%.*]], [[HEADER]] ]
+; CHECK-NEXT:    [[I_INC]] = add i32 [[I]], 1
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[I_INC]], [[END:%.*]]
 ; CHECK-NEXT:    br i1 [[CMP]], label [[HEADER]], label [[OUTER_BACKEDGE]]
 ; CHECK:       outer_backedge:
-; CHECK-NEXT:    [[I_INC_LCSSA:%.*]] = phi i32 [ [[I_INC]], [[BACKEDGE]] ]
+; CHECK-NEXT:    [[I_INC_LCSSA:%.*]] = phi i32 [ [[I_INC]], [[HEADER]] ]
 ; CHECK-NEXT:    [[J_INC]] = add i32 [[J]], 1
 ; CHECK-NEXT:    [[CMP_J:%.*]] = icmp slt i32 [[J_INC]], [[END]]
 ; CHECK-NEXT:    br i1 [[CMP_J]], label [[OUTER_HEADER]], label [[EXIT:%.*]]
@@ -1284,25 +1194,23 @@ define i32 @full_sub_loop_test_branch_loop_inverse_2(i32 %end) {
 ; CHECK-NEXT:    br label [[OUTER_HEADER:%.*]]
 ; CHECK:       outer_header:
 ; CHECK-NEXT:    [[J:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[J_INC:%.*]], [[OUTER_BACKEDGE:%.*]] ]
+; CHECK-NEXT:    switch i32 0, label [[PREHEADER_SPLIT:%.*]] [
+; CHECK-NEXT:    i32 1, label [[OUTER_BACKEDGE]]
+; CHECK-NEXT:    ]
+; CHECK:       preheader-split:
 ; CHECK-NEXT:    br label [[HEADER:%.*]]
 ; CHECK:       header:
-; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, [[OUTER_HEADER]] ], [ [[I_INC:%.*]], [[BACKEDGE:%.*]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, [[PREHEADER_SPLIT]] ], [ [[I_INC:%.*]], [[HEADER]] ]
 ; CHECK-NEXT:    [[MUL:%.*]] = mul i32 [[I]], [[I]]
-; CHECK-NEXT:    br i1 false, label [[BACKEDGE]], label [[DEAD:%.*]]
-; CHECK:       dead:
 ; CHECK-NEXT:    [[I_2:%.*]] = add i32 [[I]], 1
-; CHECK-NEXT:    br label [[BACKEDGE]]
-; CHECK:       backedge:
-; CHECK-NEXT:    [[I_1:%.*]] = phi i32 [ [[I]], [[HEADER]] ], [ [[I_2]], [[DEAD]] ]
-; CHECK-NEXT:    [[I_INC]] = add i32 [[I_1]], 1
-; CHECK-NEXT:    br i1 true, label [[HEADER]], label [[OUTER_BACKEDGE]]
+; CHECK-NEXT:    [[I_INC]] = add i32 [[I_2]], 1
+; CHECK-NEXT:    br label [[HEADER]]
 ; CHECK:       outer_backedge:
-; CHECK-NEXT:    [[I_INC_LCSSA:%.*]] = phi i32 [ [[I_INC]], [[BACKEDGE]] ]
 ; CHECK-NEXT:    [[J_INC]] = add i32 [[J]], 1
 ; CHECK-NEXT:    [[CMP_J:%.*]] = icmp slt i32 [[J_INC]], [[END:%.*]]
 ; CHECK-NEXT:    br i1 [[CMP_J]], label [[OUTER_HEADER]], label [[EXIT:%.*]]
 ; CHECK:       exit:
-; CHECK-NEXT:    [[I_INC_LCSSA_LCSSA:%.*]] = phi i32 [ [[I_INC_LCSSA]], [[OUTER_BACKEDGE]] ]
+; CHECK-NEXT:    [[I_INC_LCSSA_LCSSA:%.*]] = phi i32 [ undef, [[OUTER_BACKEDGE]] ]
 ; CHECK-NEXT:    ret i32 [[I_INC_LCSSA_LCSSA]]
 ;
 entry:
@@ -1347,29 +1255,23 @@ define i32 @full_sub_loop_test_switch_loop_inverse_2(i32 %end) {
 ; CHECK-NEXT:    br label [[OUTER_HEADER:%.*]]
 ; CHECK:       outer_header:
 ; CHECK-NEXT:    [[J:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[J_INC:%.*]], [[OUTER_BACKEDGE:%.*]] ]
+; CHECK-NEXT:    switch i32 0, label [[PREHEADER_SPLIT:%.*]] [
+; CHECK-NEXT:    i32 1, label [[OUTER_BACKEDGE]]
+; CHECK-NEXT:    ]
+; CHECK:       preheader-split:
 ; CHECK-NEXT:    br label [[HEADER:%.*]]
 ; CHECK:       header:
-; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, [[OUTER_HEADER]] ], [ [[I_INC:%.*]], [[BACKEDGE:%.*]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, [[PREHEADER_SPLIT]] ], [ [[I_INC:%.*]], [[HEADER]] ]
 ; CHECK-NEXT:    [[MUL:%.*]] = mul i32 [[I]], [[I]]
-; CHECK-NEXT:    switch i32 1, label [[DEAD:%.*]] [
-; CHECK-NEXT:    i32 0, label [[BACKEDGE]]
-; CHECK-NEXT:    ]
-; CHECK:       dead:
 ; CHECK-NEXT:    [[I_2:%.*]] = add i32 [[I]], 1
-; CHECK-NEXT:    br label [[BACKEDGE]]
-; CHECK:       backedge:
-; CHECK-NEXT:    [[I_1:%.*]] = phi i32 [ [[I]], [[HEADER]] ], [ [[I_2]], [[DEAD]] ]
-; CHECK-NEXT:    [[I_INC]] = add i32 [[I_1]], 1
-; CHECK-NEXT:    switch i32 1, label [[HEADER]] [
-; CHECK-NEXT:    i32 0, label [[OUTER_BACKEDGE]]
-; CHECK-NEXT:    ]
+; CHECK-NEXT:    [[I_INC]] = add i32 [[I_2]], 1
+; CHECK-NEXT:    br label [[HEADER]]
 ; CHECK:       outer_backedge:
-; CHECK-NEXT:    [[I_INC_LCSSA:%.*]] = phi i32 [ [[I_INC]], [[BACKEDGE]] ]
 ; CHECK-NEXT:    [[J_INC]] = add i32 [[J]], 1
 ; CHECK-NEXT:    [[CMP_J:%.*]] = icmp slt i32 [[J_INC]], [[END:%.*]]
 ; CHECK-NEXT:    br i1 [[CMP_J]], label [[OUTER_HEADER]], label [[EXIT:%.*]]
 ; CHECK:       exit:
-; CHECK-NEXT:    [[I_INC_LCSSA_LCSSA:%.*]] = phi i32 [ [[I_INC_LCSSA]], [[OUTER_BACKEDGE]] ]
+; CHECK-NEXT:    [[I_INC_LCSSA_LCSSA:%.*]] = phi i32 [ undef, [[OUTER_BACKEDGE]] ]
 ; CHECK-NEXT:    ret i32 [[I_INC_LCSSA_LCSSA]]
 ;
 entry:
@@ -1415,25 +1317,22 @@ define i32 @full_sub_loop_test_branch_loop_inverse_3(i32 %end) {
 ; CHECK-NEXT:    br label [[OUTER_HEADER:%.*]]
 ; CHECK:       outer_header:
 ; CHECK-NEXT:    [[J:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[J_INC:%.*]], [[OUTER_BACKEDGE:%.*]] ]
+; CHECK-NEXT:    switch i32 0, label [[PREHEADER_SPLIT:%.*]] [
+; CHECK-NEXT:    i32 1, label [[OUTER_BACKEDGE]]
+; CHECK-NEXT:    ]
+; CHECK:       preheader-split:
 ; CHECK-NEXT:    br label [[HEADER:%.*]]
 ; CHECK:       header:
-; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, [[OUTER_HEADER]] ], [ [[I_INC:%.*]], [[BACKEDGE:%.*]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, [[PREHEADER_SPLIT]] ], [ [[I_INC:%.*]], [[HEADER]] ]
 ; CHECK-NEXT:    [[MUL:%.*]] = mul i32 [[I]], [[I]]
-; CHECK-NEXT:    br i1 true, label [[BACKEDGE]], label [[DEAD:%.*]]
-; CHECK:       dead:
-; CHECK-NEXT:    [[I_2:%.*]] = add i32 [[I]], 1
-; CHECK-NEXT:    br label [[BACKEDGE]]
-; CHECK:       backedge:
-; CHECK-NEXT:    [[I_1:%.*]] = phi i32 [ [[I]], [[HEADER]] ], [ [[I_2]], [[DEAD]] ]
-; CHECK-NEXT:    [[I_INC]] = add i32 [[I_1]], 1
-; CHECK-NEXT:    br i1 true, label [[HEADER]], label [[OUTER_BACKEDGE]]
+; CHECK-NEXT:    [[I_INC]] = add i32 [[I]], 1
+; CHECK-NEXT:    br label [[HEADER]]
 ; CHECK:       outer_backedge:
-; CHECK-NEXT:    [[I_INC_LCSSA:%.*]] = phi i32 [ [[I_INC]], [[BACKEDGE]] ]
 ; CHECK-NEXT:    [[J_INC]] = add i32 [[J]], 1
 ; CHECK-NEXT:    [[CMP_J:%.*]] = icmp slt i32 [[J_INC]], [[END:%.*]]
 ; CHECK-NEXT:    br i1 [[CMP_J]], label [[OUTER_HEADER]], label [[EXIT:%.*]]
 ; CHECK:       exit:
-; CHECK-NEXT:    [[I_INC_LCSSA_LCSSA:%.*]] = phi i32 [ [[I_INC_LCSSA]], [[OUTER_BACKEDGE]] ]
+; CHECK-NEXT:    [[I_INC_LCSSA_LCSSA:%.*]] = phi i32 [ undef, [[OUTER_BACKEDGE]] ]
 ; CHECK-NEXT:    ret i32 [[I_INC_LCSSA_LCSSA]]
 ;
 entry:
@@ -1478,29 +1377,22 @@ define i32 @full_sub_loop_test_switch_loop_inverse_3(i32 %end) {
 ; CHECK-NEXT:    br label [[OUTER_HEADER:%.*]]
 ; CHECK:       outer_header:
 ; CHECK-NEXT:    [[J:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[J_INC:%.*]], [[OUTER_BACKEDGE:%.*]] ]
+; CHECK-NEXT:    switch i32 0, label [[PREHEADER_SPLIT:%.*]] [
+; CHECK-NEXT:    i32 1, label [[OUTER_BACKEDGE]]
+; CHECK-NEXT:    ]
+; CHECK:       preheader-split:
 ; CHECK-NEXT:    br label [[HEADER:%.*]]
 ; CHECK:       header:
-; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, [[OUTER_HEADER]] ], [ [[I_INC:%.*]], [[BACKEDGE:%.*]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ 0, [[PREHEADER_SPLIT]] ], [ [[I_INC:%.*]], [[HEADER]] ]
 ; CHECK-NEXT:    [[MUL:%.*]] = mul i32 [[I]], [[I]]
-; CHECK-NEXT:    switch i32 1, label [[BACKEDGE]] [
-; CHECK-NEXT:    i32 0, label [[DEAD:%.*]]
-; CHECK-NEXT:    ]
-; CHECK:       dead:
-; CHECK-NEXT:    [[I_2:%.*]] = add i32 [[I]], 1
-; CHECK-NEXT:    br label [[BACKEDGE]]
-; CHECK:       backedge:
-; CHECK-NEXT:    [[I_1:%.*]] = phi i32 [ [[I]], [[HEADER]] ], [ [[I_2]], [[DEAD]] ]
-; CHECK-NEXT:    [[I_INC]] = add i32 [[I_1]], 1
-; CHECK-NEXT:    switch i32 1, label [[HEADER]] [
-; CHECK-NEXT:    i32 0, label [[OUTER_BACKEDGE]]
-; CHECK-NEXT:    ]
+; CHECK-NEXT:    [[I_INC]] = add i32 [[I]], 1
+; CHECK-NEXT:    br label [[HEADER]]
 ; CHECK:       outer_backedge:
-; CHECK-NEXT:    [[I_INC_LCSSA:%.*]] = phi i32 [ [[I_INC]], [[BACKEDGE]] ]
 ; CHECK-NEXT:    [[J_INC]] = add i32 [[J]], 1
 ; CHECK-NEXT:    [[CMP_J:%.*]] = icmp slt i32 [[J_INC]], [[END:%.*]]
 ; CHECK-NEXT:    br i1 [[CMP_J]], label [[OUTER_HEADER]], label [[EXIT:%.*]]
 ; CHECK:       exit:
-; CHECK-NEXT:    [[I_INC_LCSSA_LCSSA:%.*]] = phi i32 [ [[I_INC_LCSSA]], [[OUTER_BACKEDGE]] ]
+; CHECK-NEXT:    [[I_INC_LCSSA_LCSSA:%.*]] = phi i32 [ undef, [[OUTER_BACKEDGE]] ]
 ; CHECK-NEXT:    ret i32 [[I_INC_LCSSA_LCSSA]]
 ;
 entry:
@@ -1548,13 +1440,17 @@ define i32 @exit_branch_from_inner_to_grandparent(i1 %cond1, i1 %cond2, i32 %N)
 ; CHECK-NEXT:    br label [[LOOP_2:%.*]]
 ; CHECK:       loop_2:
 ; CHECK-NEXT:    [[J:%.*]] = phi i32 [ 0, [[LOOP_1]] ], [ [[J_NEXT:%.*]], [[LOOP_2_BACKEDGE:%.*]] ]
+; CHECK-NEXT:    switch i32 0, label [[LOOP_2_SPLIT:%.*]] [
+; CHECK-NEXT:    i32 1, label [[LOOP_2_BACKEDGE]]
+; CHECK-NEXT:    ]
+; CHECK:       loop_2-split:
 ; CHECK-NEXT:    br label [[LOOP_3:%.*]]
 ; CHECK:       loop_3:
-; CHECK-NEXT:    [[K:%.*]] = phi i32 [ 0, [[LOOP_2]] ], [ [[K_NEXT:%.*]], [[LOOP_3_BACKEDGE:%.*]] ]
+; CHECK-NEXT:    [[K:%.*]] = phi i32 [ 0, [[LOOP_2_SPLIT]] ], [ [[K_NEXT:%.*]], [[LOOP_3_BACKEDGE:%.*]] ]
 ; CHECK-NEXT:    br i1 [[COND1:%.*]], label [[LOOP_3_BACKEDGE]], label [[LOOP_1_BACKEDGE_LOOPEXIT:%.*]]
 ; CHECK:       loop_3_backedge:
 ; CHECK-NEXT:    [[K_NEXT]] = add i32 [[K]], 1
-; CHECK-NEXT:    br i1 true, label [[LOOP_3]], label [[LOOP_2_BACKEDGE]]
+; CHECK-NEXT:    br label [[LOOP_3]]
 ; CHECK:       loop_2_backedge:
 ; CHECK-NEXT:    [[J_NEXT]] = add i32 [[J]], 1
 ; CHECK-NEXT:    [[C_2:%.*]] = icmp slt i32 [[J_NEXT]], [[N:%.*]]
@@ -1613,15 +1509,17 @@ define i32 @exit_switch_from_inner_to_grandparent(i1 %cond1, i1 %cond2, i32 %N)
 ; CHECK-NEXT:    br label [[LOOP_2:%.*]]
 ; CHECK:       loop_2:
 ; CHECK-NEXT:    [[J:%.*]] = phi i32 [ 0, [[LOOP_1]] ], [ [[J_NEXT:%.*]], [[LOOP_2_BACKEDGE:%.*]] ]
+; CHECK-NEXT:    switch i32 0, label [[LOOP_2_SPLIT:%.*]] [
+; CHECK-NEXT:    i32 1, label [[LOOP_2_BACKEDGE]]
+; CHECK-NEXT:    ]
+; CHECK:       loop_2-split:
 ; CHECK-NEXT:    br label [[LOOP_3:%.*]]
 ; CHECK:       loop_3:
-; CHECK-NEXT:    [[K:%.*]] = phi i32 [ 0, [[LOOP_2]] ], [ [[K_NEXT:%.*]], [[LOOP_3_BACKEDGE:%.*]] ]
+; CHECK-NEXT:    [[K:%.*]] = phi i32 [ 0, [[LOOP_2_SPLIT]] ], [ [[K_NEXT:%.*]], [[LOOP_3_BACKEDGE:%.*]] ]
 ; CHECK-NEXT:    br i1 [[COND1:%.*]], label [[LOOP_3_BACKEDGE]], label [[LOOP_1_BACKEDGE_LOOPEXIT:%.*]]
 ; CHECK:       loop_3_backedge:
 ; CHECK-NEXT:    [[K_NEXT]] = add i32 [[K]], 1
-; CHECK-NEXT:    switch i32 1, label [[LOOP_3]] [
-; CHECK-NEXT:    i32 0, label [[LOOP_2_BACKEDGE]]
-; CHECK-NEXT:    ]
+; CHECK-NEXT:    br label [[LOOP_3]]
 ; CHECK:       loop_2_backedge:
 ; CHECK-NEXT:    [[J_NEXT]] = add i32 [[J]], 1
 ; CHECK-NEXT:    [[C_2:%.*]] = icmp slt i32 [[J_NEXT]], [[N:%.*]]
@@ -2607,3 +2505,270 @@ loop_1_backedge:
 exit:
   ret i32 %i
 }
+
+define i32 @complex_dead_subloop_branch(i1 %cond1, i1 %cond2, i1 %cond3) {
+; CHECK-LABEL: @complex_dead_subloop_branch(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    br i1 [[COND3:%.*]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RESULT_LCSSA:%.*]] = phi i32 [ 0, [[LOOP]] ]
+; CHECK-NEXT:    ret i32 [[RESULT_LCSSA]]
+;
+entry:
+  br label %loop
+
+loop:
+  br i1 true, label %latch, label %subloop
+
+subloop:
+  br i1 %cond1, label %x, label %y
+
+x:
+  br label %subloop_latch
+
+y:
+  br label %subloop_latch
+
+subloop_latch:
+  %dead_phi = phi i32 [ 1, %x ], [ 2, %y ]
+  br i1 %cond2, label %latch, label %subloop
+
+latch:
+  %result = phi i32 [ 0, %loop ], [ %dead_phi, %subloop_latch ]
+  br i1 %cond3, label %loop, label %exit
+
+exit:
+  ret i32 %result
+}
+
+define i32 @complex_dead_subloop_switch(i1 %cond1, i1 %cond2, i1 %cond3) {
+; CHECK-LABEL: @complex_dead_subloop_switch(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    br i1 [[COND3:%.*]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RESULT_LCSSA:%.*]] = phi i32 [ 0, [[LOOP]] ]
+; CHECK-NEXT:    ret i32 [[RESULT_LCSSA]]
+;
+entry:
+  br label %loop
+
+loop:
+  switch i32 1, label %latch [ i32 0, label %subloop ]
+
+subloop:
+  br i1 %cond1, label %x, label %y
+
+x:
+  br label %subloop_latch
+
+y:
+  br label %subloop_latch
+
+subloop_latch:
+  %dead_phi = phi i32 [ 1, %x ], [ 2, %y ]
+  br i1 %cond2, label %latch, label %subloop
+
+latch:
+  %result = phi i32 [ 0, %loop ], [ %dead_phi, %subloop_latch ]
+  br i1 %cond3, label %loop, label %exit
+
+exit:
+  ret i32 %result
+}
+
+define void @test_crash_01() {
+; CHECK-LABEL: @test_crash_01(
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    br label [[BB1:%.*]]
+; CHECK:       bb1:
+; CHECK-NEXT:    br i1 undef, label [[BB17:%.*]], label [[BB2:%.*]]
+; CHECK:       bb2:
+; CHECK-NEXT:    switch i32 0, label [[BB2_SPLIT:%.*]] [
+; CHECK-NEXT:    i32 1, label [[BB19:%.*]]
+; CHECK-NEXT:    ]
+; CHECK:       bb2-split:
+; CHECK-NEXT:    br label [[BB3:%.*]]
+; CHECK:       bb3:
+; CHECK-NEXT:    switch i32 undef, label [[BB16:%.*]] [
+; CHECK-NEXT:    i32 0, label [[BB15:%.*]]
+; CHECK-NEXT:    i32 1, label [[BB14:%.*]]
+; CHECK-NEXT:    i32 2, label [[BB13:%.*]]
+; CHECK-NEXT:    i32 3, label [[BB12:%.*]]
+; CHECK-NEXT:    i32 4, label [[BB11:%.*]]
+; CHECK-NEXT:    i32 5, label [[BB8:%.*]]
+; CHECK-NEXT:    i32 6, label [[BB10:%.*]]
+; CHECK-NEXT:    i32 7, label [[BB9:%.*]]
+; CHECK-NEXT:    i32 8, label [[BB7:%.*]]
+; CHECK-NEXT:    ]
+; CHECK:       bb7:
+; CHECK-NEXT:    unreachable
+; CHECK:       bb8:
+; CHECK-NEXT:    switch i32 undef, label [[BB28:%.*]] [
+; CHECK-NEXT:    i32 0, label [[BB27:%.*]]
+; CHECK-NEXT:    i32 1, label [[BB26:%.*]]
+; CHECK-NEXT:    i32 2, label [[BB23:%.*]]
+; CHECK-NEXT:    i32 3, label [[BB24:%.*]]
+; CHECK-NEXT:    i32 4, label [[BB25:%.*]]
+; CHECK-NEXT:    i32 5, label [[BB29:%.*]]
+; CHECK-NEXT:    i32 6, label [[BB22:%.*]]
+; CHECK-NEXT:    i32 7, label [[BB20:%.*]]
+; CHECK-NEXT:    i32 8, label [[BB21:%.*]]
+; CHECK-NEXT:    ]
+; CHECK:       bb9:
+; CHECK-NEXT:    unreachable
+; CHECK:       bb10:
+; CHECK-NEXT:    unreachable
+; CHECK:       bb11:
+; CHECK-NEXT:    br label [[BB8]]
+; CHECK:       bb12:
+; CHECK-NEXT:    unreachable
+; CHECK:       bb13:
+; CHECK-NEXT:    unreachable
+; CHECK:       bb14:
+; CHECK-NEXT:    unreachable
+; CHECK:       bb15:
+; CHECK-NEXT:    unreachable
+; CHECK:       bb16:
+; CHECK-NEXT:    unreachable
+; CHECK:       bb17:
+; CHECK-NEXT:    ret void
+; CHECK:       bb19:
+; CHECK-NEXT:    ret void
+; CHECK:       bb20:
+; CHECK-NEXT:    unreachable
+; CHECK:       bb21:
+; CHECK-NEXT:    unreachable
+; CHECK:       bb22:
+; CHECK-NEXT:    unreachable
+; CHECK:       bb23:
+; CHECK-NEXT:    unreachable
+; CHECK:       bb24:
+; CHECK-NEXT:    unreachable
+; CHECK:       bb25:
+; CHECK-NEXT:    unreachable
+; CHECK:       bb26:
+; CHECK-NEXT:    unreachable
+; CHECK:       bb27:
+; CHECK-NEXT:    unreachable
+; CHECK:       bb28:
+; CHECK-NEXT:    unreachable
+; CHECK:       bb29:
+; CHECK-NEXT:    br label [[BB3]]
+;
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb
+  br i1 undef, label %bb17, label %bb2
+
+bb2:                                              ; preds = %bb1
+  br label %bb3
+
+bb3:                                              ; preds = %bb6, %bb2
+  br label %bb4
+
+bb4:                                              ; preds = %bb3
+  switch i32 0, label %bb5 [
+  i32 1, label %bb19
+  i32 2, label %bb18
+  ]
+
+bb5:                                              ; preds = %bb4
+  switch i32 undef, label %bb16 [
+  i32 0, label %bb15
+  i32 1, label %bb14
+  i32 2, label %bb13
+  i32 3, label %bb12
+  i32 4, label %bb11
+  i32 5, label %bb8
+  i32 6, label %bb10
+  i32 7, label %bb9
+  i32 8, label %bb7
+  ]
+
+bb6:                                              ; preds = %bb29, %bb18
+  br label %bb3
+
+bb7:                                              ; preds = %bb5
+  unreachable
+
+bb8:                                              ; preds = %bb11, %bb5
+  switch i32 undef, label %bb28 [
+  i32 0, label %bb27
+  i32 1, label %bb26
+  i32 2, label %bb23
+  i32 3, label %bb24
+  i32 4, label %bb25
+  i32 5, label %bb29
+  i32 6, label %bb22
+  i32 7, label %bb20
+  i32 8, label %bb21
+  ]
+
+bb9:                                              ; preds = %bb5
+  unreachable
+
+bb10:                                             ; preds = %bb5
+  unreachable
+
+bb11:                                             ; preds = %bb5
+  br label %bb8
+
+bb12:                                             ; preds = %bb5
+  unreachable
+
+bb13:                                             ; preds = %bb5
+  unreachable
+
+bb14:                                             ; preds = %bb5
+  unreachable
+
+bb15:                                             ; preds = %bb5
+  unreachable
+
+bb16:                                             ; preds = %bb5
+  unreachable
+
+bb17:                                             ; preds = %bb1
+  ret void
+
+bb18:                                             ; preds = %bb4
+  br label %bb6
+
+bb19:                                             ; preds = %bb4
+  ret void
+
+bb20:                                             ; preds = %bb8
+  unreachable
+
+bb21:                                             ; preds = %bb8
+  unreachable
+
+bb22:                                             ; preds = %bb8
+  unreachable
+
+bb23:                                             ; preds = %bb8
+  unreachable
+
+bb24:                                             ; preds = %bb8
+  unreachable
+
+bb25:                                             ; preds = %bb8
+  unreachable
+
+bb26:                                             ; preds = %bb8
+  unreachable
+
+bb27:                                             ; preds = %bb8
+  unreachable
+
+bb28:                                             ; preds = %bb8
+  unreachable
+
+bb29:                                             ; preds = %bb8
+  br label %bb6
+}
diff --git a/test/Transforms/LoopSimplifyCFG/lcssa.ll b/test/Transforms/LoopSimplifyCFG/lcssa.ll
index 82920f153044bcfdf578e5da27592ce696b96068..8281f36d97893be165308427fcdacf416dec42b5 100644
--- a/test/Transforms/LoopSimplifyCFG/lcssa.ll
+++ b/test/Transforms/LoopSimplifyCFG/lcssa.ll
@@ -50,3 +50,107 @@ if.end.7:                                         ; preds = %if.end
 if.end.8:                                         ; preds = %if.end.7
   br label %for.cond
 }
+
+define void @test_01() {
+; CHECK-LABEL: @test_01(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[FOR_COND:%.*]]
+; CHECK:       for.cond.loopexit:
+; CHECK-NEXT:    br label [[FOR_COND]]
+; CHECK:       for.cond:
+; CHECK-NEXT:    [[INC41_LCSSA3:%.*]] = phi i16 [ undef, [[FOR_COND_LOOPEXIT:%.*]] ], [ undef, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    switch i32 0, label [[FOR_COND_SPLIT:%.*]] [
+; CHECK-NEXT:    i32 1, label [[FOR_COND_LOOPEXIT]]
+; CHECK-NEXT:    ]
+; CHECK:       for.cond-split:
+; CHECK-NEXT:    [[INC41_LCSSA3_LCSSA:%.*]] = phi i16 [ [[INC41_LCSSA3]], [[FOR_COND]] ]
+; CHECK-NEXT:    br label [[WHILE_COND:%.*]]
+; CHECK:       while.cond:
+; CHECK-NEXT:    [[INC41:%.*]] = phi i16 [ [[INC4:%.*]], [[WHILE_COND]] ], [ [[INC41_LCSSA3_LCSSA]], [[FOR_COND_SPLIT]] ]
+; CHECK-NEXT:    [[INC4]] = add nsw i16 [[INC41]], 1
+; CHECK-NEXT:    br label [[WHILE_COND]]
+;
+entry:
+  br label %for.cond
+
+for.cond.loopexit:                                ; preds = %while.cond
+  %inc41.lcssa = phi i16 [ %inc41, %while.cond ]
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.cond.loopexit, %entry
+  %inc41.lcssa3 = phi i16 [ %inc41.lcssa, %for.cond.loopexit ], [ undef, %entry ]
+  br label %while.cond
+
+while.cond:                                       ; preds = %while.body, %for.cond
+  %inc41 = phi i16 [ %inc4, %while.body ], [ %inc41.lcssa3, %for.cond ]
+  br i1 true, label %while.body, label %for.cond.loopexit
+
+while.body:                                       ; preds = %while.cond
+  %inc4 = add nsw i16 %inc41, 1
+  br label %while.cond
+}
+
+define void @bar() {
+; CHECK-LABEL: @bar(
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    switch i32 0, label [[BB_SPLIT:%.*]] [
+; CHECK-NEXT:    i32 1, label [[BB10:%.*]]
+; CHECK-NEXT:    ]
+; CHECK:       bb-split:
+; CHECK-NEXT:    br label [[BB1:%.*]]
+; CHECK:       bb1:
+; CHECK-NEXT:    [[TMP:%.*]] = phi i32 [ [[TMP7:%.*]], [[BB6:%.*]] ], [ undef, [[BB_SPLIT]] ]
+; CHECK-NEXT:    switch i32 undef, label [[BB5:%.*]] [
+; CHECK-NEXT:    i32 0, label [[BB6]]
+; CHECK-NEXT:    i32 1, label [[BB8:%.*]]
+; CHECK-NEXT:    ]
+; CHECK:       bb5:
+; CHECK-NEXT:    ret void
+; CHECK:       bb6:
+; CHECK-NEXT:    [[TMP7]] = add i32 undef, 123
+; CHECK-NEXT:    br label [[BB1]]
+; CHECK:       bb8:
+; CHECK-NEXT:    [[TMP9:%.*]] = phi i32 [ [[TMP]], [[BB1]] ]
+; CHECK-NEXT:    [[USE:%.*]] = add i32 [[TMP9]], 1
+; CHECK-NEXT:    ret void
+; CHECK:       bb10:
+; CHECK-NEXT:    ret void
+;
+
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb6, %bb
+  %tmp = phi i32 [ %tmp7, %bb6 ], [ undef, %bb ]
+  br i1 false, label %bb2, label %bb4
+
+bb2:                                              ; preds = %bb1
+  switch i32 undef, label %bb10 [
+  i32 0, label %bb3
+  i32 1, label %bb8
+  ]
+
+bb3:                                              ; preds = %bb2
+  br label %bb6
+
+bb4:                                              ; preds = %bb1
+  switch i32 undef, label %bb5 [
+  i32 0, label %bb6
+  i32 1, label %bb8
+  ]
+
+bb5:                                              ; preds = %bb4
+  ret void
+
+bb6:                                              ; preds = %bb4, %bb3
+  %tmp7 = add i32 undef, 123
+  br label %bb1
+
+bb8:                                              ; preds = %bb4, %bb2
+  %tmp9 = phi i32 [ %tmp, %bb2 ], [ %tmp, %bb4 ]
+  %use = add i32 %tmp9, 1
+  ret void
+
+bb10:                                             ; preds = %bb2
+  ret void
+}
diff --git a/test/Transforms/LoopSimplifyCFG/live_block_marking.ll b/test/Transforms/LoopSimplifyCFG/live_block_marking.ll
new file mode 100644
index 0000000000000000000000000000000000000000..344871c79ae913e77b3379585abbb6f0aacd3beb
--- /dev/null
+++ b/test/Transforms/LoopSimplifyCFG/live_block_marking.ll
@@ -0,0 +1,62 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; REQUIRES: asserts
+; RUN: opt -S -enable-loop-simplifycfg-term-folding=true -indvars -loop-simplifycfg -debug-only=loop-simplifycfg -verify-loop-info -verify-dom-info -verify-loop-lcssa 2>&1 < %s | FileCheck %s
+; RUN: opt -S -enable-loop-simplifycfg-term-folding=true -passes='require<domtree>,loop(indvars,simplify-cfg)' -debug-only=loop-simplifycfg -verify-loop-info -verify-dom-info -verify-loop-lcssa 2>&1 < %s | FileCheck %s
+; RUN: opt -S -enable-loop-simplifycfg-term-folding=true -indvars -loop-simplifycfg -enable-mssa-loop-dependency=true -verify-memoryssa -debug-only=loop-simplifycfg -verify-loop-info -verify-dom-info -verify-loop-lcssa 2>&1 < %s | FileCheck %s
+
+define void @test(i1 %c) {
+; CHECK-LABEL: @test(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    switch i32 0, label [[ENTRY_SPLIT:%.*]] [
+; CHECK-NEXT:    i32 1, label [[DEAD_EXIT:%.*]]
+; CHECK-NEXT:    ]
+; CHECK:       entry-split:
+; CHECK-NEXT:    br label [[OUTER:%.*]]
+; CHECK:       outer:
+; CHECK-NEXT:    br i1 [[C:%.*]], label [[TO_FOLD:%.*]], label [[LATCH:%.*]]
+; CHECK:       to_fold:
+; CHECK-NEXT:    br i1 [[C]], label [[LATCH]], label [[INNER_PREHEADER:%.*]]
+; CHECK:       inner.preheader:
+; CHECK-NEXT:    br label [[INNER:%.*]]
+; CHECK:       inner:
+; CHECK-NEXT:    br i1 false, label [[INNER_LATCH:%.*]], label [[UNDEAD:%.*]]
+; CHECK:       inner_latch:
+; CHECK-NEXT:    br i1 true, label [[INNER]], label [[LATCH_LOOPEXIT:%.*]]
+; CHECK:       undead:
+; CHECK-NEXT:    br label [[LATCH]]
+; CHECK:       latch.loopexit:
+; CHECK-NEXT:    br label [[LATCH]]
+; CHECK:       latch:
+; CHECK-NEXT:    br label [[OUTER]]
+; CHECK:       dead_exit:
+; CHECK-NEXT:    ret void
+;
+
+entry:
+  br label %outer
+
+outer:
+  br i1 %c, label %to_fold, label %latch
+
+to_fold:
+  br i1 %c, label %latch, label %inner
+
+inner:
+  %iv = phi i32 [0, %to_fold], [%iv.next, %inner_latch]
+  %never = icmp sgt i32 %iv, 40
+  br i1 %never, label %inner_latch, label %undead
+
+inner_latch:
+  %iv.next = add i32 %iv, 1
+  %cmp = icmp slt i32 %iv.next, 10
+  br i1 %cmp, label %inner, label %latch
+
+undead:
+  br label %latch
+
+latch:
+  br i1 true, label %outer, label %dead_exit
+
+dead_exit:
+  ret void
+}
diff --git a/test/Transforms/LoopSimplifyCFG/update_parents.ll b/test/Transforms/LoopSimplifyCFG/update_parents.ll
new file mode 100644
index 0000000000000000000000000000000000000000..377fbb20e85997784dcca51c8b25c1cf75bcaa85
--- /dev/null
+++ b/test/Transforms/LoopSimplifyCFG/update_parents.ll
@@ -0,0 +1,119 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; REQUIRES: asserts
+; RUN: opt -S -enable-loop-simplifycfg-term-folding=true -loop-simplifycfg -debug-only=loop-simplifycfg -verify-loop-info -verify-dom-info -verify-loop-lcssa 2>&1 < %s | FileCheck %s
+; RUN: opt -S -enable-loop-simplifycfg-term-folding=true -passes='require<domtree>,loop(simplify-cfg)' -debug-only=loop-simplifycfg -verify-loop-info -verify-dom-info -verify-loop-lcssa 2>&1 < %s | FileCheck %s
+; RUN: opt -S -enable-loop-simplifycfg-term-folding=true -loop-simplifycfg -enable-mssa-loop-dependency=true -verify-memoryssa -debug-only=loop-simplifycfg -verify-loop-info -verify-dom-info -verify-loop-lcssa 2>&1 < %s | FileCheck %s
+
+target triple = "x86_64-unknown-linux-gnu"
+
+define void @test() {
+; CHECK-LABEL: @test(
+; CHECK-NEXT:    br label [[BB1:%.*]]
+; CHECK:       bb1.loopexit:
+; CHECK-NEXT:    br label [[BB1]]
+; CHECK:       bb1:
+; CHECK-NEXT:    br label [[BB2:%.*]]
+; CHECK:       bb2.loopexit:
+; CHECK-NEXT:    br label [[BB2]]
+; CHECK:       bb2:
+; CHECK-NEXT:    switch i32 0, label [[BB2_SPLIT:%.*]] [
+; CHECK-NEXT:    i32 1, label [[BB1_LOOPEXIT:%.*]]
+; CHECK-NEXT:    i32 2, label [[BB2_LOOPEXIT:%.*]]
+; CHECK-NEXT:    ]
+; CHECK:       bb2-split:
+; CHECK-NEXT:    br label [[BB3:%.*]]
+; CHECK:       bb3:
+; CHECK-NEXT:    br label [[BB3]]
+;
+
+  br label %bb1
+
+bb1:                                              ; preds = %bb4, %0
+  br label %bb2
+
+bb2:                                              ; preds = %bb6, %bb1
+  br label %bb3
+
+bb3:                                              ; preds = %bb8, %bb3, %bb2
+  br i1 false, label %bb4, label %bb3
+
+bb4:                                              ; preds = %bb8, %bb3
+  br i1 undef, label %bb1, label %bb6
+
+bb6:                                              ; preds = %bb4
+  br i1 undef, label %bb2, label %bb8
+
+bb8:                                              ; preds = %bb6
+  br i1 true, label %bb4, label %bb3
+}
+
+define void @test_many_subloops(i1 %c) {
+; CHECK-LABEL: @test_many_subloops(
+; CHECK-NEXT:    br label [[BB1:%.*]]
+; CHECK:       bb1.loopexit:
+; CHECK-NEXT:    br label [[BB1]]
+; CHECK:       bb1:
+; CHECK-NEXT:    br label [[BB2:%.*]]
+; CHECK:       bb2.loopexit:
+; CHECK-NEXT:    br label [[BB2]]
+; CHECK:       bb2:
+; CHECK-NEXT:    switch i32 0, label [[BB2_SPLIT:%.*]] [
+; CHECK-NEXT:    i32 1, label [[BB1_LOOPEXIT:%.*]]
+; CHECK-NEXT:    i32 2, label [[BB2_LOOPEXIT:%.*]]
+; CHECK-NEXT:    ]
+; CHECK:       bb2-split:
+; CHECK-NEXT:    br label [[BB3:%.*]]
+; CHECK:       bb3:
+; CHECK-NEXT:    br label [[BB3]]
+;
+
+  br label %bb1
+
+bb1:
+  br label %bb2
+
+bb2:
+  br label %bb3
+
+bb3:
+  br i1 false, label %bb4, label %bb3
+
+bb4:
+  br i1 undef, label %bb1, label %subloop1
+
+subloop1:
+  br i1 %c, label %subloop2, label %subloop11
+
+subloop11:
+  br i1 %c, label %subloop11, label %subloop12
+
+subloop12:
+  br i1 %c, label %subloop12, label %subloop13
+
+subloop13:
+  br i1 %c, label %subloop13, label %subloop1_latch
+
+subloop1_latch:
+  br label %subloop1
+
+subloop2:
+  br i1 %c, label %bb6, label %subloop21
+
+subloop21:
+  br i1 %c, label %subloop21, label %subloop22
+
+subloop22:
+  br i1 %c, label %subloop22, label %subloop23
+
+subloop23:
+  br i1 %c, label %subloop23, label %subloop2_latch
+
+subloop2_latch:
+  br label %subloop2
+
+bb6:
+  br i1 undef, label %bb2, label %bb8
+
+bb8:
+  br i1 true, label %bb4, label %bb3
+}
diff --git a/test/Transforms/LoopStrengthReduce/ARM/complexity.ll b/test/Transforms/LoopStrengthReduce/ARM/complexity.ll
index f2cc0a5a6f8ffb693124712e33a9e380617debf0..197bb53ab517905cc7558231de7c1a099b797c37 100644
--- a/test/Transforms/LoopStrengthReduce/ARM/complexity.ll
+++ b/test/Transforms/LoopStrengthReduce/ARM/complexity.ll
@@ -1,21 +1,15 @@
 target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64"
 
-; RUN: opt -mtriple=thumbv7em %s -S -loop-reduce -lsr-complexity-limit=65536 -o - | FileCheck %s --check-prefix=CHECK-DEFAULT
-; RUN: opt -mtriple=thumbv7em %s -S -loop-reduce -lsr-complexity-limit=2147483647 -o - | FileCheck %s --check-prefix=CHECK-COMPLEX
+; RUN: opt -mtriple=thumbv7em %s -S -loop-reduce -lsr-complexity-limit=65536 -o - | FileCheck %s
+; RUN: opt -mtriple=thumbv7em %s -S -loop-reduce -lsr-complexity-limit=2147483647 -o - | FileCheck %s
 
-; CHECK-DEFAULT-LABEL: for.body12.us.us:
-; CHECK-DEFAULT: phi i32
-; CHECK-DEFAULT: [[LSR_IV:%[^ ]+]] = phi i32 [ [[LSR_IV_NEXT:%[^ ]+]], %for.body12.us.us ], [ 0, %for.cond9.preheader.us.us ]
-; CHECK-DEFAULT: phi i32
-; CHECK-DEFAULT: [[LSR_IV_NEXT]] = add i32 [[LSR_IV]], 8
-
-; CHECK-COMPLEX-LABEL: for.body12.us.us:
-; CHECK-COMPLEX: phi i32
-; CHECK-COMPLEX: [[LSR_IV6:%[^ ]+]] = phi i16* [ [[SCEVGEP7:%[^ ]+]], %for.body12.us.us ], [ [[SCEVGEP5:%[^ ]+]], %for.cond9.preheader.us.us ]
-; CHECK-COMPLEX: [[LSR_IV:%[^ ]+]] = phi i16* [ [[SCEVGEP1:%[^ ]+]], %for.body12.us.us ], [ [[SCEVGEP:%[^ ]+]], %for.cond9.preheader.us.us ]
-; CHECK-COMPLEX: phi i32
-; CHECK-COMPLEX: [[SCEVGEP1]] = getelementptr i16, i16* [[LSR_IV]], i32 4
-; CHECK-COMPLEX: [[SCEVGEP7]] = getelementptr i16, i16* [[LSR_IV6]], i32 4
+; CHECK-LABEL: for.body12.us.us:
+; CHECK: [[LSR_IV6:%[^ ]+]] = phi i16* [ [[SCEVGEP7:%[^ ]+]], %for.body12.us.us ], [ [[SCEVGEP5:%[^ ]+]], %for.cond9.preheader.us.us ]
+; CHECK: phi i32
+; CHECK: [[LSR_IV:%[^ ]+]] = phi i16* [ [[SCEVGEP1:%[^ ]+]], %for.body12.us.us ], [ [[SCEVGEP:%[^ ]+]], %for.cond9.preheader.us.us ]
+; CHECK: phi i32
+; CHECK: [[SCEVGEP1]] = getelementptr i16, i16* [[LSR_IV]], i32 4
+; CHECK: [[SCEVGEP7]] = getelementptr i16, i16* [[LSR_IV6]], i32 4
 
 define void @convolve(i16** nocapture readonly %input_image, i16** nocapture readonly %filter, i32 %filter_dim, i32 %out_width, i32 %out_height, i32** nocapture readonly %convolved) {
 entry:
diff --git a/test/Transforms/LoopStrengthReduce/X86/bin_power.ll b/test/Transforms/LoopStrengthReduce/X86/bin_power.ll
index 35cb28da7d55dfbe3dec1c78a5e56b45ac9fea8f..c9781241c96993017d143bf4e9cc1d8c717e9842 100644
--- a/test/Transforms/LoopStrengthReduce/X86/bin_power.ll
+++ b/test/Transforms/LoopStrengthReduce/X86/bin_power.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -loop-reduce -S | FileCheck %s
+; RUN: opt < %s -scalar-evolution-huge-expr-threshold=1000000 -loop-reduce -S | FileCheck %s
 
 target datalayout = "e-m:e-i32:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
diff --git a/test/Transforms/LoopStrengthReduce/X86/pr40514.ll b/test/Transforms/LoopStrengthReduce/X86/pr40514.ll
new file mode 100644
index 0000000000000000000000000000000000000000..4ffcfd8549e03aaf0faf06d71b38e9930e316086
--- /dev/null
+++ b/test/Transforms/LoopStrengthReduce/X86/pr40514.ll
@@ -0,0 +1,57 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -loop-reduce -S | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128-ni:1"
+target triple = "x86_64-unknown-linux-gnu"
+
+define i32 @pluto(i32 %arg) #0 {
+; CHECK-LABEL: @pluto(
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    br label [[BB10:%.*]]
+; CHECK:       bb1:
+; CHECK-NEXT:    store i64 [[LSR_IV_NEXT2:%.*]], i64 addrspace(1)* undef, align 8
+; CHECK-NEXT:    ret i32 [[LSR_IV_NEXT:%.*]]
+; CHECK:       bb10:
+; CHECK-NEXT:    [[LSR_IV1:%.*]] = phi i64 [ [[LSR_IV_NEXT2]], [[BB10]] ], [ 9, [[BB:%.*]] ]
+; CHECK-NEXT:    [[LSR_IV:%.*]] = phi i32 [ [[LSR_IV_NEXT]], [[BB10]] ], [ undef, [[BB]] ]
+; CHECK-NEXT:    [[LSR_IV_NEXT]] = add i32 [[LSR_IV]], 1
+; CHECK-NEXT:    [[LSR_IV_NEXT2]] = add nuw nsw i64 [[LSR_IV1]], 1
+; CHECK-NEXT:    br i1 true, label [[BB1:%.*]], label [[BB10]]
+;
+
+bb:
+  br label %bb10
+
+bb1:                                              ; preds = %bb10
+  %tmp = and i64 %tmp24, 4294967295
+  %tmp2 = shl i64 %tmp23, 33
+  %tmp3 = ashr exact i64 %tmp2, 32
+  %tmp4 = add i64 undef, %tmp
+  %tmp5 = add i64 %tmp4, %tmp3
+  %tmp6 = add i64 %tmp5, undef
+  %tmp7 = add i64 %tmp6, undef
+  %tmp8 = add i64 undef, %tmp7
+  store i64 %tmp8, i64 addrspace(1)* undef, align 8
+  %tmp9 = trunc i64 %tmp7 to i32
+  ret i32 %tmp9
+
+bb10:                                             ; preds = %bb10, %bb
+  %tmp11 = phi i64 [ 9, %bb ], [ %tmp24, %bb10 ]
+  %tmp12 = shl i64 undef, 1
+  %tmp13 = mul i64 %tmp12, %tmp12
+  %tmp14 = shl i64 %tmp13, 1
+  %tmp15 = mul i64 %tmp14, %tmp14
+  %tmp16 = shl i64 %tmp15, 1
+  %tmp17 = mul i64 %tmp16, %tmp16
+  %tmp18 = shl i64 %tmp17, 1
+  %tmp19 = mul i64 %tmp18, %tmp18
+  %tmp20 = shl i64 %tmp19, 1
+  %tmp21 = mul i64 %tmp20, %tmp20
+  %tmp22 = shl i64 %tmp21, 1
+  %tmp23 = mul i64 %tmp22, %tmp22
+  %tmp24 = add nuw nsw i64 %tmp11, 1
+  br i1 undef, label %bb1, label %bb10
+}
+
+
+attributes #0 = { "target-cpu"="broadwell" "target-features"="+sse2,+cx16,+sahf,-tbm,-avx512ifma,-sha,-gfni,-fma4,-vpclmulqdq,+prfchw,+bmi2,-cldemote,+fsgsbase,-ptwrite,-xsavec,+popcnt,+aes,-avx512bitalg,-movdiri,-xsaves,-avx512er,-avx512vnni,-avx512vpopcntdq,-pconfig,-clwb,-avx512f,-clzero,-pku,+mmx,-lwp,-rdpid,-xop,+rdseed,-waitpkg,-movdir64b,-sse4a,-avx512bw,-clflushopt,+xsave,-avx512vbmi2,+64bit,-avx512vl,+invpcid,-avx512cd,+avx,-vaes,+rtm,+fma,+bmi,+rdrnd,-mwaitx,+sse4.1,+sse4.2,+avx2,-wbnoinvd,+sse,+lzcnt,+pclmul,-prefetchwt1,+f16c,+ssse3,-sgx,-shstk,+cmov,-avx512vbmi,+movbe,+xsaveopt,-avx512dq,+adx,-avx512pf,+sse3" }
diff --git a/test/Transforms/LoopTransformWarning/enable_and_isvectorized.ll b/test/Transforms/LoopTransformWarning/enable_and_isvectorized.ll
new file mode 100644
index 0000000000000000000000000000000000000000..77d09ad53f7887a095fc75c03796058b3aab36b8
--- /dev/null
+++ b/test/Transforms/LoopTransformWarning/enable_and_isvectorized.ll
@@ -0,0 +1,33 @@
+; RUN: opt -transform-warning -disable-output < %s 2>&1 | FileCheck -allow-empty %s
+;
+; llvm.org/PR40546
+; Do not warn about about leftover llvm.loop.vectorize.enable for already
+; vectorized loops.
+
+target datalayout = "e-m:w-i64:64-f80:128-n8:16:32:64-S128"
+
+define void @test(i32 %n) {
+entry:
+  %cmp = icmp eq i32 %n, 0
+  br i1 %cmp, label %simd.if.end, label %omp.inner.for.body.preheader
+
+omp.inner.for.body.preheader:
+  %wide.trip.count = zext i32 %n to i64
+  br label %omp.inner.for.body
+
+omp.inner.for.body:
+  %indvars.iv = phi i64 [ 0, %omp.inner.for.body.preheader ], [ %indvars.iv.next, %omp.inner.for.body ]
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, %wide.trip.count
+  br i1 %exitcond, label %simd.if.end, label %omp.inner.for.body, !llvm.loop !0
+
+simd.if.end:
+  ret void
+}
+
+!0 = distinct !{!0, !1, !2}
+!1 = !{!"llvm.loop.vectorize.enable", i1 true}
+!2 = !{!"llvm.loop.isvectorized"}
+
+
+; CHECK-NOT: loop not vectorized
diff --git a/test/Transforms/LoopUnroll/runtime-loop-multiexit-dom-verify.ll b/test/Transforms/LoopUnroll/runtime-loop-multiexit-dom-verify.ll
index 6014775028ee54026b2e1e5472181e5a17aeccb4..b3c0039dc155ff6936f8f277d7e077b47e1c5ca4 100644
--- a/test/Transforms/LoopUnroll/runtime-loop-multiexit-dom-verify.ll
+++ b/test/Transforms/LoopUnroll/runtime-loop-multiexit-dom-verify.ll
@@ -124,3 +124,152 @@ latchexit:                                              ; preds = %latch
 exitsucc:                                              ; preds = %headerexit
   ret i64 96
 }
+
+; exit block (%default) has an exiting block and another exit block as predecessors.
+define void @test4(i16 %c3) {
+; CHECK-LABEL: test4
+
+; CHECK-LABEL: exiting.prol:
+; CHECK-NEXT:   switch i16 %c3, label %default.loopexit.loopexit1 [
+
+; CHECK-LABEL: exiting:
+; CHECK-NEXT:   switch i16 %c3, label %default.loopexit.loopexit [
+
+; CHECK-LABEL: default.loopexit.loopexit:
+; CHECK-NEXT:   br label %default.loopexit
+
+; CHECK-LABEL: default.loopexit.loopexit1:
+; CHECK-NEXT:   br label %default.loopexit
+
+; CHECK-LABEL: default.loopexit:
+; CHECK-NEXT:   br label %default
+preheader:
+  %c1 = zext i32 undef to i64
+  br label %header
+
+header:                                       ; preds = %latch, %preheader
+  %indvars.iv = phi i64 [ 0, %preheader ], [ %indvars.iv.next, %latch ]
+  br label %exiting
+
+exiting:                                           ; preds = %header
+  switch i16 %c3, label %default [
+    i16 45, label %otherexit
+    i16 95, label %latch
+  ]
+
+latch:                                          ; preds = %exiting
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %c2 = icmp ult i64 %indvars.iv.next, %c1
+  br i1 %c2, label %header, label %latchexit
+
+latchexit:                                          ; preds = %latch
+  ret void
+
+default:                                          ; preds = %otherexit, %exiting
+  ret void
+
+otherexit:                                           ; preds = %exiting
+  br label %default
+}
+
+; exit block (%exitB) has an exiting block and another exit block as predecessors.
+; exiting block comes from inner loop.
+define void @test5() {
+; CHECK-LABEL: test5
+; CHECK-LABEL: bb1:
+; CHECK-NEXT:   br i1 false, label %outerH.prol.preheader, label %outerH.prol.loopexit
+
+; CHECK-LABEL: outerH.prol.preheader:
+; CHECK-NEXT:   br label %outerH.prol
+
+; CHECK-LABEL: outerH.prol:
+; CHECK-NEXT:   %tmp4.prol = phi i32 [ %tmp6.prol, %outerLatch.prol ], [ undef, %outerH.prol.preheader ]
+; CHECK-NEXT:   %prol.iter = phi i32 [ 0, %outerH.prol.preheader ], [ %prol.iter.sub, %outerLatch.prol ]
+; CHECK-NEXT:   br label %innerH.prol
+bb:
+  %tmp = icmp sgt i32 undef, 79
+  br i1 %tmp, label %outerLatchExit, label %bb1
+
+bb1:                                              ; preds = %bb
+  br label %outerH
+
+outerH:                                              ; preds = %outerLatch, %bb1
+  %tmp4 = phi i32 [ %tmp6, %outerLatch ], [ undef, %bb1 ]
+  br label %innerH
+
+innerH:                                              ; preds = %innerLatch, %outerH
+  br i1 undef, label %innerexiting, label %otherexitB
+
+innerexiting:                                             ; preds = %innerH
+  br i1 undef, label %innerLatch, label %exitB
+
+innerLatch:                                             ; preds = %innerexiting
+  %tmp13 = fcmp olt double undef, 2.000000e+00
+  br i1 %tmp13, label %innerH, label %outerLatch
+
+outerLatch:                                              ; preds = %innerLatch
+  %tmp6 = add i32 %tmp4, 1
+  %tmp7 = icmp sgt i32 %tmp6, 79
+  br i1 %tmp7, label %outerLatchExit, label %outerH
+
+outerLatchExit:                                              ; preds = %outerLatch, %bb
+  ret void
+
+exitB:                                             ; preds = %innerexiting, %otherexitB
+  ret void
+
+otherexitB:                                              ; preds = %innerH
+  br label %exitB
+
+}
+
+; Blocks reachable from exits (not_zero44) have the IDom as the block within the loop (Header).
+; Update the IDom to the preheader.
+define void @test6() {
+; CHECK-LABEL: test6
+; CHECK-LABEL: header.prol.preheader:
+; CHECK-NEXT:    br label %header.prol
+
+; CHECK-LABEL: header.prol:
+; CHECK-NEXT:    %indvars.iv.prol = phi i64 [ undef, %header.prol.preheader ], [ %indvars.iv.next.prol, %latch.prol ]
+; CHECK-NEXT:    %prol.iter = phi i64 [ 1, %header.prol.preheader ], [ %prol.iter.sub, %latch.prol ]
+; CHECK-NEXT:    br i1 false, label %latch.prol, label %otherexit.loopexit1
+
+; CHECK-LABEL: header.prol.loopexit.unr-lcssa:
+; CHECK-NEXT:    %indvars.iv.unr.ph = phi i64 [ %indvars.iv.next.prol, %latch.prol ]
+; CHECK-NEXT:    br label %header.prol.loopexit
+
+; CHECK-LABEL: header.prol.loopexit:
+; CHECK-NEXT:    %indvars.iv.unr = phi i64 [ undef, %entry ], [ %indvars.iv.unr.ph, %header.prol.loopexit.unr-lcssa ]
+; CHECK-NEXT:    br i1 true, label %latchexit, label %entry.new
+
+; CHECK-LABEL: entry.new:
+; CHECK-NEXT:    br label %header
+entry:
+  br label %header
+
+header:                                          ; preds = %latch, %entry
+  %indvars.iv = phi i64 [ undef, %entry ], [ %indvars.iv.next, %latch ]
+  br i1 undef, label %latch, label %otherexit
+
+latch:                                         ; preds = %header
+  %indvars.iv.next = add nsw i64 %indvars.iv, 2
+  %0 = icmp slt i64 %indvars.iv.next, 616
+  br i1 %0, label %header, label %latchexit
+
+latchexit:                                          ; preds = %latch
+  br label %latchexitsucc
+
+otherexit:                                 ; preds = %header
+  br label %otherexitsucc
+
+otherexitsucc:                                          ; preds = %otherexit
+  br label %not_zero44
+
+not_zero44:                                       ; preds = %latchexitsucc, %otherexitsucc
+  unreachable
+
+latchexitsucc:                                      ; preds = %latchexit
+  br label %not_zero44
+}
+
diff --git a/test/Transforms/LoopUnroll/runtime-loop-multiple-exits.ll b/test/Transforms/LoopUnroll/runtime-loop-multiple-exits.ll
index ef983dffcd2b11e2386f757122909794eb758664..397e907a8e99d306c45b2dbf3eaea0a54b28a36c 100644
--- a/test/Transforms/LoopUnroll/runtime-loop-multiple-exits.ll
+++ b/test/Transforms/LoopUnroll/runtime-loop-multiple-exits.ll
@@ -1,8 +1,8 @@
-; RUN: opt < %s -loop-unroll -unroll-runtime=true -unroll-runtime-epilog=true -unroll-runtime-multi-exit=true -verify-dom-info -verify-loop-info -S | FileCheck %s -check-prefix=EPILOG-NO-IC
-; RUN: opt < %s -loop-unroll -unroll-runtime=true -unroll-runtime-epilog=true -unroll-runtime-multi-exit=true -verify-dom-info -verify-loop-info -instcombine -S | FileCheck %s -check-prefix=EPILOG
-; RUN: opt < %s -loop-unroll -unroll-runtime -unroll-count=2 -unroll-runtime-epilog=true -unroll-runtime-multi-exit=true -verify-dom-info -verify-loop-info -instcombine
-; RUN: opt < %s -loop-unroll -unroll-runtime=true -unroll-runtime-epilog=false -unroll-runtime-multi-exit=true -verify-dom-info -verify-loop-info -instcombine -S | FileCheck %s -check-prefix=PROLOG
-; RUN: opt < %s -loop-unroll -unroll-runtime -unroll-runtime-epilog=false -unroll-count=2 -unroll-runtime-multi-exit=true -verify-dom-info -verify-loop-info -instcombine
+; RUN: opt < %s -loop-unroll -unroll-runtime=true -unroll-runtime-epilog=true -unroll-runtime-multi-exit=true -verify-loop-lcssa -verify-dom-info -verify-loop-info -S | FileCheck %s -check-prefix=EPILOG-NO-IC
+; RUN: opt < %s -loop-unroll -unroll-runtime=true -unroll-runtime-epilog=true -unroll-runtime-multi-exit=true -verify-loop-lcssa -verify-dom-info -verify-loop-info -instcombine -S | FileCheck %s -check-prefix=EPILOG
+; RUN: opt < %s -loop-unroll -unroll-runtime -unroll-count=2 -unroll-runtime-epilog=true -unroll-runtime-multi-exit=true -verify-loop-lcssa -verify-dom-info -verify-loop-info -instcombine
+; RUN: opt < %s -loop-unroll -unroll-runtime=true -unroll-runtime-epilog=false -unroll-runtime-multi-exit=true -verify-loop-lcssa -verify-dom-info -verify-loop-info -instcombine -S | FileCheck %s -check-prefix=PROLOG
+; RUN: opt < %s -loop-unroll -unroll-runtime -unroll-runtime-epilog=false -unroll-count=2 -unroll-runtime-multi-exit=true -verify-loop-lcssa -verify-dom-info -verify-loop-info -instcombine
 
 ; REQUIRES: asserts
 
@@ -207,43 +207,165 @@ exit2.loopexit:
 }
 
 ; FIXME: Support multiple exiting blocks to the same latch exit block.
-define i32 @test4(i32* nocapture %a, i64 %n, i1 %cond) {
-; EPILOG: test4(
+; Three exiting blocks where header and latch exit to same LatchExit.
+define i32 @hdr_latch_same_exit(i32* nocapture %a, i64 %n, i1 %cond) {
+; EPILOG: hdr_latch_same_exit(
 ; EPILOG-NOT: .unr
 ; EPILOG-NOT: .epil
 
-; PROLOG: test4(
+; PROLOG: hdr_latch_same_exit(
 ; PROLOG-NOT: .unr
 ; PROLOG-NOT: .prol
 entry:
   br label %header
 
 header:
-  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
-  %sum.02 = phi i32 [ %add, %for.body ], [ 0, %entry ]
-  br i1 %cond, label %for.end, label %for.exiting_block
+  %indvars.iv = phi i64 [ %indvars.iv.next, %latch ], [ 0, %entry ]
+  %sum.02 = phi i32 [ %add, %latch ], [ 0, %entry ]
+  br i1 %cond, label %latchExit, label %for.exiting_block
 
 for.exiting_block:
  %cmp = icmp eq i64 %n, 42
- br i1 %cmp, label %for.exit2, label %for.body
+ br i1 %cmp, label %for.exit2, label %latch
 
-for.body:                                         ; preds = %for.body, %entry
+latch:                                         ; preds = %latch, %entry
   %arrayidx = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
   %0 = load i32, i32* %arrayidx, align 4
   %add = add nsw i32 %0, %sum.02
   %indvars.iv.next = add i64 %indvars.iv, 1
   %exitcond = icmp eq i64 %indvars.iv.next, %n
-  br i1 %exitcond, label %for.end, label %header
+  br i1 %exitcond, label %latchExit, label %header
 
-for.end:                                          ; preds = %for.body, %entry
-  %sum.0.lcssa = phi i32 [ 0, %header ], [ %add, %for.body ]
-  ret i32 %sum.0.lcssa
+latchExit:                                          ; preds = %latch, %entry
+  %result = phi i32 [ 0, %header ], [ %add, %latch ]
+  ret i32 %result
 
 for.exit2:
   ret i32 42
 }
 
-; FIXME: Support multiple exiting blocks to the unique exit block.
+; Two exiting blocks to latch where the exiting blocks are Latch and a
+; non-header
+; FIXME: We should unroll this loop.
+define i32 @otherblock_latch_same_exit(i32* nocapture %a, i64 %n, i1 %cond) {
+; EPILOG: otherblock_latch_same_exit(
+; EPILOG-NOT: .unr
+; EPILOG-NOT: .epil
+
+; PROLOG: otherblock_latch_same_exit(
+; PROLOG-NOT: .unr
+; PROLOG-NOT: .prol
+entry:
+  br label %header
+
+header:
+  %indvars.iv = phi i64 [ %indvars.iv.next, %latch ], [ 0, %entry ]
+  %sum.02 = phi i32 [ %add, %latch ], [ 0, %entry ]
+  br i1 %cond, label %for.exit2, label %for.exiting_block
+
+for.exiting_block:
+ %cmp = icmp eq i64 %n, 42
+ br i1 %cmp, label %latchExit, label %latch
+
+latch:                                         ; preds = %latch, %entry
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %add = add nsw i32 %0, %sum.02
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, %n
+  br i1 %exitcond, label %latchExit, label %header
+
+latchExit:                                          ; preds = %latch, %entry
+  %result = phi i32 [ 2, %for.exiting_block ], [ %add, %latch ]
+  ret i32 %result
+
+for.exit2:
+  ret i32 42
+}
+
+; Two exiting blocks to latch where the exiting blocks are Latch and a
+; non-header
+; Same as above test except the incoming value for latch Phi is from the header
+; FIXME: We should be able to runtime unroll.
+define i32 @otherblock_latch_same_exit2(i32* nocapture %a, i64 %n, i1 %cond) {
+; EPILOG: otherblock_latch_same_exit2(
+; EPILOG-NOT: .unr
+; EPILOG-NOT: .epil
+
+; PROLOG: otherblock_latch_same_exit2(
+; PROLOG-NOT: .unr
+; PROLOG-NOT: .prol
+entry:
+  br label %header
+
+header:
+  %indvars.iv = phi i64 [ %indvars.iv.next, %latch ], [ 0, %entry ]
+  %sum.02 = phi i32 [ %add, %latch ], [ 0, %entry ]
+  br i1 %cond, label %for.exit2, label %for.exiting_block
+
+for.exiting_block:
+ %cmp = icmp eq i64 %n, 42
+ br i1 %cmp, label %latchExit, label %latch
+
+latch:                                         ; preds = %latch, %entry
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %add = add nsw i32 %0, %sum.02
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, %n
+  br i1 %exitcond, label %latchExit, label %header
+
+latchExit:                                          ; preds = %latch, %entry
+  %result = phi i32 [ %sum.02, %for.exiting_block ], [ %add, %latch ]
+  ret i32 %result
+
+for.exit2:
+  ret i32 42
+}
+
+; Two exiting blocks to latch where the exiting blocks are Latch and a
+; non-header
+; Same as above test except the incoming value for cloned latch Phi is from the
+; for.exiting_block.
+; FIXME: We should be able to runtime unroll.
+define i32 @otherblock_latch_same_exit3(i32* nocapture %a, i64 %n, i1 %cond) {
+; EPILOG: otherblock_latch_same_exit3(
+; EPILOG-NOT: .unr
+; EPILOG-NOT: .epil
+
+; PROLOG: otherblock_latch_same_exit3(
+; PROLOG-NOT: .unr
+; PROLOG-NOT: .prol
+entry:
+  br label %header
+
+header:
+  %indvars.iv = phi i64 [ %indvars.iv.next, %latch ], [ 0, %entry ]
+  %sum.02 = phi i32 [ %add, %latch ], [ 0, %entry ]
+  br i1 %cond, label %for.exit2, label %for.exiting_block
+
+for.exiting_block:
+ %arrayidx = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
+ %0 = load i32, i32* %arrayidx, align 4
+ %add = add nsw i32 %0, %sum.02
+ %cmp = icmp eq i64 %n, 42
+ br i1 %cmp, label %latchExit, label %latch
+
+latch:                                         ; preds = %latch, %entry
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, %n
+  br i1 %exitcond, label %latchExit, label %header
+
+latchExit:                                          ; preds = %latch, %entry
+  %result = phi i32 [ %sum.02, %for.exiting_block ], [ %add, %latch ]
+  ret i32 %result
+
+for.exit2:
+  ret i32 42
+}
+
+; FIXME: Support multiple exiting blocks to the unique exit block (LatchExit).
+; Only 2 blocks in loop: header and latch where both exit to same LatchExit.
 define void @unique_exit(i32 %arg) {
 ; EPILOG: unique_exit(
 ; EPILOG-NOT: .unr
@@ -259,22 +381,22 @@ entry:
 preheader:                                 ; preds = %entry
   br label %header
 
-LoopExit:                                ; preds = %header, %latch
-  %tmp2.ph = phi i32 [ %tmp4, %header ], [ -1, %latch ]
-  br label %returnblock
-
-returnblock:                                         ; preds = %LoopExit, %entry
-  %tmp2 = phi i32 [ -1, %entry ], [ %tmp2.ph, %LoopExit ]
-  ret void
-
 header:                                           ; preds = %preheader, %latch
   %tmp4 = phi i32 [ %inc, %latch ], [ %arg, %preheader ]
   %inc = add nsw i32 %tmp4, 1
-  br i1 true, label %LoopExit, label %latch
+  br i1 true, label %latchExit, label %latch
 
 latch:                                            ; preds = %header
   %cmp = icmp slt i32 %inc, undef
-  br i1 %cmp, label %header, label %LoopExit
+  br i1 %cmp, label %header, label %latchExit
+
+latchExit:                                ; preds = %header, %latch
+  %tmp2.ph = phi i32 [ %tmp4, %header ], [ -1, %latch ]
+  br label %returnblock
+
+returnblock:                                         ; preds = %latchExit, %entry
+  %tmp2 = phi i32 [ -1, %entry ], [ %tmp2.ph, %latchExit ]
+  ret void
 }
 
 ; two exiting and two exit blocks.
diff --git a/test/Transforms/LoopVectorize/X86/force-ifcvt.ll b/test/Transforms/LoopVectorize/X86/force-ifcvt.ll
index 00764943556de313a1c272b5af3f5e8738509239..07b98b4fb00104821fa16a7af55c984e97b2902c 100644
--- a/test/Transforms/LoopVectorize/X86/force-ifcvt.ll
+++ b/test/Transforms/LoopVectorize/X86/force-ifcvt.ll
@@ -13,21 +13,21 @@ entry:
 for.body:                                         ; preds = %cond.end, %entry
   %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %cond.end ]
   %arrayidx = getelementptr inbounds i32, i32* %p, i64 %indvars.iv
-  %0 = load i32, i32* %arrayidx, align 4, !llvm.mem.parallel_loop_access !0
+  %0 = load i32, i32* %arrayidx, align 4, !llvm.access.group !1
   %cmp1 = icmp eq i32 %0, 0
   %arrayidx3 = getelementptr inbounds i32, i32* %res, i64 %indvars.iv
-  %1 = load i32, i32* %arrayidx3, align 4, !llvm.mem.parallel_loop_access !0
+  %1 = load i32, i32* %arrayidx3, align 4, !llvm.access.group !1
   br i1 %cmp1, label %cond.end, label %cond.false
 
 cond.false:                                       ; preds = %for.body
   %arrayidx7 = getelementptr inbounds i32, i32* %d, i64 %indvars.iv
-  %2 = load i32, i32* %arrayidx7, align 4, !llvm.mem.parallel_loop_access !0
+  %2 = load i32, i32* %arrayidx7, align 4, !llvm.access.group !1
   %add = add nsw i32 %2, %1
   br label %cond.end
 
 cond.end:                                         ; preds = %for.body, %cond.false
   %cond = phi i32 [ %add, %cond.false ], [ %1, %for.body ]
-  store i32 %cond, i32* %arrayidx3, align 4, !llvm.mem.parallel_loop_access !0
+  store i32 %cond, i32* %arrayidx3, align 4, !llvm.access.group !1
   %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
   %exitcond = icmp eq i64 %indvars.iv.next, 16
   br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !0
@@ -38,4 +38,5 @@ for.end:                                          ; preds = %cond.end
 
 attributes #0 = { norecurse nounwind uwtable "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" }
 
-!0 = distinct !{!0}
+!0 = distinct !{!0, !{!"llvm.loop.parallel_accesses", !1}}
+!1 = distinct !{}
diff --git a/test/Transforms/LoopVectorize/X86/gather_scatter.ll b/test/Transforms/LoopVectorize/X86/gather_scatter.ll
index 95be7eb76b3a3549058da6285475e992fcfcfacb..21ed2f2fb39917c211884b49c3ecd9c3176e8e7f 100644
--- a/test/Transforms/LoopVectorize/X86/gather_scatter.ll
+++ b/test/Transforms/LoopVectorize/X86/gather_scatter.ll
@@ -47,7 +47,7 @@ define void @foo1(float* noalias %in, float* noalias %out, i32* noalias %trigger
 ; AVX512-NEXT:    [[TMP12:%.*]] = icmp sgt <16 x i32> [[WIDE_LOAD_1]], zeroinitializer
 ; AVX512-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i32, i32* [[INDEX]], i64 [[INDEX_NEXT]]
 ; AVX512-NEXT:    [[TMP14:%.*]] = bitcast i32* [[TMP13]] to <16 x i32>*
-; AVX512-NEXT:    [[WIDE_MASKED_LOAD_1:%.*]] = call <16 x i32> @llvm.masked.load.v16i32.p0v16i32(<16 x i32>* [[TMP14]], i32 4, <16 x i1> [[TMP12]], <16 x i32> undef)
+; AVX512-NEXT:    [[WIDE_MASKED_LOAD_1:%.*]] = call <16 x i32> @llvm.masked.load.v16i32.p0v16i32(<16 x i32>* nonnull [[TMP14]], i32 4, <16 x i1> [[TMP12]], <16 x i32> undef)
 ; AVX512-NEXT:    [[TMP15:%.*]] = sext <16 x i32> [[WIDE_MASKED_LOAD_1]] to <16 x i64>
 ; AVX512-NEXT:    [[TMP16:%.*]] = getelementptr inbounds float, float* [[IN]], <16 x i64> [[TMP15]]
 ; AVX512-NEXT:    [[WIDE_MASKED_GATHER_1:%.*]] = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> [[TMP16]], i32 4, <16 x i1> [[TMP12]], <16 x float> undef)
@@ -62,7 +62,7 @@ define void @foo1(float* noalias %in, float* noalias %out, i32* noalias %trigger
 ; AVX512-NEXT:    [[TMP22:%.*]] = icmp sgt <16 x i32> [[WIDE_LOAD_2]], zeroinitializer
 ; AVX512-NEXT:    [[TMP23:%.*]] = getelementptr inbounds i32, i32* [[INDEX]], i64 [[INDEX_NEXT_1]]
 ; AVX512-NEXT:    [[TMP24:%.*]] = bitcast i32* [[TMP23]] to <16 x i32>*
-; AVX512-NEXT:    [[WIDE_MASKED_LOAD_2:%.*]] = call <16 x i32> @llvm.masked.load.v16i32.p0v16i32(<16 x i32>* [[TMP24]], i32 4, <16 x i1> [[TMP22]], <16 x i32> undef)
+; AVX512-NEXT:    [[WIDE_MASKED_LOAD_2:%.*]] = call <16 x i32> @llvm.masked.load.v16i32.p0v16i32(<16 x i32>* nonnull [[TMP24]], i32 4, <16 x i1> [[TMP22]], <16 x i32> undef)
 ; AVX512-NEXT:    [[TMP25:%.*]] = sext <16 x i32> [[WIDE_MASKED_LOAD_2]] to <16 x i64>
 ; AVX512-NEXT:    [[TMP26:%.*]] = getelementptr inbounds float, float* [[IN]], <16 x i64> [[TMP25]]
 ; AVX512-NEXT:    [[WIDE_MASKED_GATHER_2:%.*]] = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> [[TMP26]], i32 4, <16 x i1> [[TMP22]], <16 x float> undef)
@@ -77,7 +77,7 @@ define void @foo1(float* noalias %in, float* noalias %out, i32* noalias %trigger
 ; AVX512-NEXT:    [[TMP32:%.*]] = icmp sgt <16 x i32> [[WIDE_LOAD_3]], zeroinitializer
 ; AVX512-NEXT:    [[TMP33:%.*]] = getelementptr inbounds i32, i32* [[INDEX]], i64 [[INDEX_NEXT_2]]
 ; AVX512-NEXT:    [[TMP34:%.*]] = bitcast i32* [[TMP33]] to <16 x i32>*
-; AVX512-NEXT:    [[WIDE_MASKED_LOAD_3:%.*]] = call <16 x i32> @llvm.masked.load.v16i32.p0v16i32(<16 x i32>* [[TMP34]], i32 4, <16 x i1> [[TMP32]], <16 x i32> undef)
+; AVX512-NEXT:    [[WIDE_MASKED_LOAD_3:%.*]] = call <16 x i32> @llvm.masked.load.v16i32.p0v16i32(<16 x i32>* nonnull [[TMP34]], i32 4, <16 x i1> [[TMP32]], <16 x i32> undef)
 ; AVX512-NEXT:    [[TMP35:%.*]] = sext <16 x i32> [[WIDE_MASKED_LOAD_3]] to <16 x i64>
 ; AVX512-NEXT:    [[TMP36:%.*]] = getelementptr inbounds float, float* [[IN]], <16 x i64> [[TMP35]]
 ; AVX512-NEXT:    [[WIDE_MASKED_GATHER_3:%.*]] = call <16 x float> @llvm.masked.gather.v16f32.v16p0f32(<16 x float*> [[TMP36]], i32 4, <16 x i1> [[TMP32]], <16 x float> undef)
@@ -117,7 +117,7 @@ define void @foo1(float* noalias %in, float* noalias %out, i32* noalias %trigger
 ; FVW2-NEXT:    [[TMP12:%.*]] = icmp sgt <2 x i32> [[WIDE_LOAD_1]], zeroinitializer
 ; FVW2-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i32, i32* [[INDEX]], i64 [[INDEX_NEXT]]
 ; FVW2-NEXT:    [[TMP14:%.*]] = bitcast i32* [[TMP13]] to <2 x i32>*
-; FVW2-NEXT:    [[WIDE_MASKED_LOAD_1:%.*]] = call <2 x i32> @llvm.masked.load.v2i32.p0v2i32(<2 x i32>* [[TMP14]], i32 4, <2 x i1> [[TMP12]], <2 x i32> undef)
+; FVW2-NEXT:    [[WIDE_MASKED_LOAD_1:%.*]] = call <2 x i32> @llvm.masked.load.v2i32.p0v2i32(<2 x i32>* nonnull [[TMP14]], i32 4, <2 x i1> [[TMP12]], <2 x i32> undef)
 ; FVW2-NEXT:    [[TMP15:%.*]] = sext <2 x i32> [[WIDE_MASKED_LOAD_1]] to <2 x i64>
 ; FVW2-NEXT:    [[TMP16:%.*]] = getelementptr inbounds float, float* [[IN]], <2 x i64> [[TMP15]]
 ; FVW2-NEXT:    [[WIDE_MASKED_GATHER_1:%.*]] = call <2 x float> @llvm.masked.gather.v2f32.v2p0f32(<2 x float*> [[TMP16]], i32 4, <2 x i1> [[TMP12]], <2 x float> undef)
@@ -132,7 +132,7 @@ define void @foo1(float* noalias %in, float* noalias %out, i32* noalias %trigger
 ; FVW2-NEXT:    [[TMP22:%.*]] = icmp sgt <2 x i32> [[WIDE_LOAD_2]], zeroinitializer
 ; FVW2-NEXT:    [[TMP23:%.*]] = getelementptr inbounds i32, i32* [[INDEX]], i64 [[INDEX_NEXT_1]]
 ; FVW2-NEXT:    [[TMP24:%.*]] = bitcast i32* [[TMP23]] to <2 x i32>*
-; FVW2-NEXT:    [[WIDE_MASKED_LOAD_2:%.*]] = call <2 x i32> @llvm.masked.load.v2i32.p0v2i32(<2 x i32>* [[TMP24]], i32 4, <2 x i1> [[TMP22]], <2 x i32> undef)
+; FVW2-NEXT:    [[WIDE_MASKED_LOAD_2:%.*]] = call <2 x i32> @llvm.masked.load.v2i32.p0v2i32(<2 x i32>* nonnull [[TMP24]], i32 4, <2 x i1> [[TMP22]], <2 x i32> undef)
 ; FVW2-NEXT:    [[TMP25:%.*]] = sext <2 x i32> [[WIDE_MASKED_LOAD_2]] to <2 x i64>
 ; FVW2-NEXT:    [[TMP26:%.*]] = getelementptr inbounds float, float* [[IN]], <2 x i64> [[TMP25]]
 ; FVW2-NEXT:    [[WIDE_MASKED_GATHER_2:%.*]] = call <2 x float> @llvm.masked.gather.v2f32.v2p0f32(<2 x float*> [[TMP26]], i32 4, <2 x i1> [[TMP22]], <2 x float> undef)
@@ -147,7 +147,7 @@ define void @foo1(float* noalias %in, float* noalias %out, i32* noalias %trigger
 ; FVW2-NEXT:    [[TMP32:%.*]] = icmp sgt <2 x i32> [[WIDE_LOAD_3]], zeroinitializer
 ; FVW2-NEXT:    [[TMP33:%.*]] = getelementptr inbounds i32, i32* [[INDEX]], i64 [[INDEX_NEXT_2]]
 ; FVW2-NEXT:    [[TMP34:%.*]] = bitcast i32* [[TMP33]] to <2 x i32>*
-; FVW2-NEXT:    [[WIDE_MASKED_LOAD_3:%.*]] = call <2 x i32> @llvm.masked.load.v2i32.p0v2i32(<2 x i32>* [[TMP34]], i32 4, <2 x i1> [[TMP32]], <2 x i32> undef)
+; FVW2-NEXT:    [[WIDE_MASKED_LOAD_3:%.*]] = call <2 x i32> @llvm.masked.load.v2i32.p0v2i32(<2 x i32>* nonnull [[TMP34]], i32 4, <2 x i1> [[TMP32]], <2 x i32> undef)
 ; FVW2-NEXT:    [[TMP35:%.*]] = sext <2 x i32> [[WIDE_MASKED_LOAD_3]] to <2 x i64>
 ; FVW2-NEXT:    [[TMP36:%.*]] = getelementptr inbounds float, float* [[IN]], <2 x i64> [[TMP35]]
 ; FVW2-NEXT:    [[WIDE_MASKED_GATHER_3:%.*]] = call <2 x float> @llvm.masked.gather.v2f32.v2p0f32(<2 x float*> [[TMP36]], i32 4, <2 x i1> [[TMP32]], <2 x float> undef)
diff --git a/test/Transforms/LoopVectorize/X86/imprecise-through-phis.ll b/test/Transforms/LoopVectorize/X86/imprecise-through-phis.ll
index 25e164fc1c931b92c0b59105e326c96d6a9ad21d..f9ccbf146fca1cad22e4bda64f0a9b892295a435 100644
--- a/test/Transforms/LoopVectorize/X86/imprecise-through-phis.ll
+++ b/test/Transforms/LoopVectorize/X86/imprecise-through-phis.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -S -loop-vectorize -mtriple=x86_64-apple-darwin %s | FileCheck %s
+; RUN: opt -S -loop-vectorize -mtriple=x86_64-apple-darwin %s | FileCheck %s --check-prefixes=CHECK,SSE
+; RUN: opt -S -loop-vectorize -mtriple=x86_64-apple-darwin -mattr=+avx %s | FileCheck %s --check-prefixes=CHECK,AVX
 
 ; Two mostly identical functions. The only difference is the presence of
 ; fast-math flags on the second. The loop is a pretty simple reduction:
@@ -64,59 +65,85 @@ done:
 }
 
 define double @sumIfVector(double* nocapture readonly %arr) {
-; CHECK-LABEL: @sumIfVector(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; CHECK:       vector.ph:
-; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
-; CHECK:       vector.body:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <2 x double> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PREDPHI:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i32> undef, i32 [[INDEX]], i32 0
-; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i32> [[BROADCAST_SPLATINSERT]], <2 x i32> undef, <2 x i32> zeroinitializer
-; CHECK-NEXT:    [[INDUCTION:%.*]] = add <2 x i32> [[BROADCAST_SPLAT]], <i32 0, i32 1>
-; CHECK-NEXT:    [[TMP0:%.*]] = add i32 [[INDEX]], 0
-; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr double, double* [[ARR:%.*]], i32 [[TMP0]]
-; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr double, double* [[TMP1]], i32 0
-; CHECK-NEXT:    [[TMP3:%.*]] = bitcast double* [[TMP2]] to <2 x double>*
-; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x double>, <2 x double>* [[TMP3]], align 8
-; CHECK-NEXT:    [[TMP4:%.*]] = fcmp fast une <2 x double> [[WIDE_LOAD]], <double 4.200000e+01, double 4.200000e+01>
-; CHECK-NEXT:    [[TMP5:%.*]] = fadd fast <2 x double> [[VEC_PHI]], [[WIDE_LOAD]]
-; CHECK-NEXT:    [[TMP6:%.*]] = xor <2 x i1> [[TMP4]], <i1 true, i1 true>
-; CHECK-NEXT:    [[PREDPHI]] = select <2 x i1> [[TMP4]], <2 x double> [[TMP5]], <2 x double> [[VEC_PHI]]
-; CHECK-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 2
-; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq i32 [[INDEX_NEXT]], 32
-; CHECK-NEXT:    br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop !0
-; CHECK:       middle.block:
-; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <2 x double> [[PREDPHI]], <2 x double> undef, <2 x i32> <i32 1, i32 undef>
-; CHECK-NEXT:    [[BIN_RDX:%.*]] = fadd fast <2 x double> [[PREDPHI]], [[RDX_SHUF]]
-; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <2 x double> [[BIN_RDX]], i32 0
-; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i32 32, 32
-; CHECK-NEXT:    br i1 [[CMP_N]], label [[DONE:%.*]], label [[SCALAR_PH]]
-; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 32, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi double [ 0.000000e+00, [[ENTRY]] ], [ [[TMP8]], [[MIDDLE_BLOCK]] ]
-; CHECK-NEXT:    br label [[LOOP:%.*]]
-; CHECK:       loop:
-; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[I_NEXT:%.*]], [[NEXT_ITER:%.*]] ]
-; CHECK-NEXT:    [[TOT:%.*]] = phi double [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[TOT_NEXT:%.*]], [[NEXT_ITER]] ]
-; CHECK-NEXT:    [[ADDR:%.*]] = getelementptr double, double* [[ARR]], i32 [[I]]
-; CHECK-NEXT:    [[NEXTVAL:%.*]] = load double, double* [[ADDR]]
-; CHECK-NEXT:    [[TST:%.*]] = fcmp fast une double [[NEXTVAL]], 4.200000e+01
-; CHECK-NEXT:    br i1 [[TST]], label [[DO_ADD:%.*]], label [[NO_ADD:%.*]]
-; CHECK:       do.add:
-; CHECK-NEXT:    [[TOT_NEW:%.*]] = fadd fast double [[TOT]], [[NEXTVAL]]
-; CHECK-NEXT:    br label [[NEXT_ITER]]
-; CHECK:       no.add:
-; CHECK-NEXT:    br label [[NEXT_ITER]]
-; CHECK:       next.iter:
-; CHECK-NEXT:    [[TOT_NEXT]] = phi double [ [[TOT]], [[NO_ADD]] ], [ [[TOT_NEW]], [[DO_ADD]] ]
-; CHECK-NEXT:    [[I_NEXT]] = add i32 [[I]], 1
-; CHECK-NEXT:    [[AGAIN:%.*]] = icmp ult i32 [[I_NEXT]], 32
-; CHECK-NEXT:    br i1 [[AGAIN]], label [[LOOP]], label [[DONE]], !llvm.loop !2
-; CHECK:       done:
-; CHECK-NEXT:    [[TOT_NEXT_LCSSA:%.*]] = phi double [ [[TOT_NEXT]], [[NEXT_ITER]] ], [ [[TMP8]], [[MIDDLE_BLOCK]] ]
-; CHECK-NEXT:    ret double [[TOT_NEXT_LCSSA]]
+; SSE-LABEL: @sumIfVector(
+; SSE-NEXT:  entry:
+; SSE-NEXT:    br label [[LOOP:%.*]]
+; SSE:       loop:
+; SSE-NEXT:    [[I:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[I_NEXT:%.*]], [[NEXT_ITER:%.*]] ]
+; SSE-NEXT:    [[TOT:%.*]] = phi double [ 0.000000e+00, [[ENTRY]] ], [ [[TOT_NEXT:%.*]], [[NEXT_ITER]] ]
+; SSE-NEXT:    [[ADDR:%.*]] = getelementptr double, double* [[ARR:%.*]], i32 [[I]]
+; SSE-NEXT:    [[NEXTVAL:%.*]] = load double, double* [[ADDR]]
+; SSE-NEXT:    [[TST:%.*]] = fcmp fast une double [[NEXTVAL]], 4.200000e+01
+; SSE-NEXT:    br i1 [[TST]], label [[DO_ADD:%.*]], label [[NO_ADD:%.*]]
+; SSE:       do.add:
+; SSE-NEXT:    [[TOT_NEW:%.*]] = fadd fast double [[TOT]], [[NEXTVAL]]
+; SSE-NEXT:    br label [[NEXT_ITER]]
+; SSE:       no.add:
+; SSE-NEXT:    br label [[NEXT_ITER]]
+; SSE:       next.iter:
+; SSE-NEXT:    [[TOT_NEXT]] = phi double [ [[TOT]], [[NO_ADD]] ], [ [[TOT_NEW]], [[DO_ADD]] ]
+; SSE-NEXT:    [[I_NEXT]] = add i32 [[I]], 1
+; SSE-NEXT:    [[AGAIN:%.*]] = icmp ult i32 [[I_NEXT]], 32
+; SSE-NEXT:    br i1 [[AGAIN]], label [[LOOP]], label [[DONE:%.*]]
+; SSE:       done:
+; SSE-NEXT:    [[TOT_NEXT_LCSSA:%.*]] = phi double [ [[TOT_NEXT]], [[NEXT_ITER]] ]
+; SSE-NEXT:    ret double [[TOT_NEXT_LCSSA]]
+;
+; AVX-LABEL: @sumIfVector(
+; AVX-NEXT:  entry:
+; AVX-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; AVX:       vector.ph:
+; AVX-NEXT:    br label [[VECTOR_BODY:%.*]]
+; AVX:       vector.body:
+; AVX-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; AVX-NEXT:    [[VEC_PHI:%.*]] = phi <4 x double> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PREDPHI:%.*]], [[VECTOR_BODY]] ]
+; AVX-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> undef, i32 [[INDEX]], i32 0
+; AVX-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> undef, <4 x i32> zeroinitializer
+; AVX-NEXT:    [[INDUCTION:%.*]] = add <4 x i32> [[BROADCAST_SPLAT]], <i32 0, i32 1, i32 2, i32 3>
+; AVX-NEXT:    [[TMP0:%.*]] = add i32 [[INDEX]], 0
+; AVX-NEXT:    [[TMP1:%.*]] = getelementptr double, double* [[ARR:%.*]], i32 [[TMP0]]
+; AVX-NEXT:    [[TMP2:%.*]] = getelementptr double, double* [[TMP1]], i32 0
+; AVX-NEXT:    [[TMP3:%.*]] = bitcast double* [[TMP2]] to <4 x double>*
+; AVX-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x double>, <4 x double>* [[TMP3]], align 8
+; AVX-NEXT:    [[TMP4:%.*]] = fcmp fast une <4 x double> [[WIDE_LOAD]], <double 4.200000e+01, double 4.200000e+01, double 4.200000e+01, double 4.200000e+01>
+; AVX-NEXT:    [[TMP5:%.*]] = fadd fast <4 x double> [[VEC_PHI]], [[WIDE_LOAD]]
+; AVX-NEXT:    [[TMP6:%.*]] = xor <4 x i1> [[TMP4]], <i1 true, i1 true, i1 true, i1 true>
+; AVX-NEXT:    [[PREDPHI]] = select <4 x i1> [[TMP4]], <4 x double> [[TMP5]], <4 x double> [[VEC_PHI]]
+; AVX-NEXT:    [[INDEX_NEXT]] = add i32 [[INDEX]], 4
+; AVX-NEXT:    [[TMP7:%.*]] = icmp eq i32 [[INDEX_NEXT]], 32
+; AVX-NEXT:    br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop !0
+; AVX:       middle.block:
+; AVX-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x double> [[PREDPHI]], <4 x double> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+; AVX-NEXT:    [[BIN_RDX:%.*]] = fadd fast <4 x double> [[PREDPHI]], [[RDX_SHUF]]
+; AVX-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <4 x double> [[BIN_RDX]], <4 x double> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; AVX-NEXT:    [[BIN_RDX2:%.*]] = fadd fast <4 x double> [[BIN_RDX]], [[RDX_SHUF1]]
+; AVX-NEXT:    [[TMP8:%.*]] = extractelement <4 x double> [[BIN_RDX2]], i32 0
+; AVX-NEXT:    [[CMP_N:%.*]] = icmp eq i32 32, 32
+; AVX-NEXT:    br i1 [[CMP_N]], label [[DONE:%.*]], label [[SCALAR_PH]]
+; AVX:       scalar.ph:
+; AVX-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 32, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
+; AVX-NEXT:    [[BC_MERGE_RDX:%.*]] = phi double [ 0.000000e+00, [[ENTRY]] ], [ [[TMP8]], [[MIDDLE_BLOCK]] ]
+; AVX-NEXT:    br label [[LOOP:%.*]]
+; AVX:       loop:
+; AVX-NEXT:    [[I:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[I_NEXT:%.*]], [[NEXT_ITER:%.*]] ]
+; AVX-NEXT:    [[TOT:%.*]] = phi double [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[TOT_NEXT:%.*]], [[NEXT_ITER]] ]
+; AVX-NEXT:    [[ADDR:%.*]] = getelementptr double, double* [[ARR]], i32 [[I]]
+; AVX-NEXT:    [[NEXTVAL:%.*]] = load double, double* [[ADDR]]
+; AVX-NEXT:    [[TST:%.*]] = fcmp fast une double [[NEXTVAL]], 4.200000e+01
+; AVX-NEXT:    br i1 [[TST]], label [[DO_ADD:%.*]], label [[NO_ADD:%.*]]
+; AVX:       do.add:
+; AVX-NEXT:    [[TOT_NEW:%.*]] = fadd fast double [[TOT]], [[NEXTVAL]]
+; AVX-NEXT:    br label [[NEXT_ITER]]
+; AVX:       no.add:
+; AVX-NEXT:    br label [[NEXT_ITER]]
+; AVX:       next.iter:
+; AVX-NEXT:    [[TOT_NEXT]] = phi double [ [[TOT]], [[NO_ADD]] ], [ [[TOT_NEW]], [[DO_ADD]] ]
+; AVX-NEXT:    [[I_NEXT]] = add i32 [[I]], 1
+; AVX-NEXT:    [[AGAIN:%.*]] = icmp ult i32 [[I_NEXT]], 32
+; AVX-NEXT:    br i1 [[AGAIN]], label [[LOOP]], label [[DONE]], !llvm.loop !2
+; AVX:       done:
+; AVX-NEXT:    [[TOT_NEXT_LCSSA:%.*]] = phi double [ [[TOT_NEXT]], [[NEXT_ITER]] ], [ [[TMP8]], [[MIDDLE_BLOCK]] ]
+; AVX-NEXT:    ret double [[TOT_NEXT_LCSSA]]
 ;
 entry:
   br label %loop
diff --git a/test/Transforms/LoopVectorize/X86/masked_load_store.ll b/test/Transforms/LoopVectorize/X86/masked_load_store.ll
index 71038feec7bb07b1c1aa16d8c2b5600e4bd7c2c2..cdb985952a79d667b2ba0ab904677338f04ff525 100644
--- a/test/Transforms/LoopVectorize/X86/masked_load_store.ll
+++ b/test/Transforms/LoopVectorize/X86/masked_load_store.ll
@@ -52,7 +52,7 @@ define void @foo1(i32* %A, i32* %B, i32* %trigger) {
 ; AVX1-NEXT:    [[TMP10:%.*]] = icmp slt <8 x i32> [[WIDE_LOAD_1]], <i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100>
 ; AVX1-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDEX_NEXT]]
 ; AVX1-NEXT:    [[TMP12:%.*]] = bitcast i32* [[TMP11]] to <8 x i32>*
-; AVX1-NEXT:    [[WIDE_MASKED_LOAD_1:%.*]] = call <8 x i32> @llvm.masked.load.v8i32.p0v8i32(<8 x i32>* [[TMP12]], i32 4, <8 x i1> [[TMP10]], <8 x i32> undef), !alias.scope !3
+; AVX1-NEXT:    [[WIDE_MASKED_LOAD_1:%.*]] = call <8 x i32> @llvm.masked.load.v8i32.p0v8i32(<8 x i32>* nonnull [[TMP12]], i32 4, <8 x i1> [[TMP10]], <8 x i32> undef), !alias.scope !3
 ; AVX1-NEXT:    [[TMP13:%.*]] = add nsw <8 x i32> [[WIDE_MASKED_LOAD_1]], [[WIDE_LOAD_1]]
 ; AVX1-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[INDEX_NEXT]]
 ; AVX1-NEXT:    [[TMP15:%.*]] = bitcast i32* [[TMP14]] to <8 x i32>*
@@ -129,13 +129,13 @@ define void @foo1(i32* %A, i32* %B, i32* %trigger) {
 ; AVX2-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <8 x i32> @llvm.masked.load.v8i32.p0v8i32(<8 x i32>* [[TMP13]], i32 4, <8 x i1> [[TMP8]], <8 x i32> undef), !alias.scope !3
 ; AVX2-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i32, i32* [[TMP12]], i64 8
 ; AVX2-NEXT:    [[TMP15:%.*]] = bitcast i32* [[TMP14]] to <8 x i32>*
-; AVX2-NEXT:    [[WIDE_MASKED_LOAD25:%.*]] = call <8 x i32> @llvm.masked.load.v8i32.p0v8i32(<8 x i32>* [[TMP15]], i32 4, <8 x i1> [[TMP9]], <8 x i32> undef), !alias.scope !3
+; AVX2-NEXT:    [[WIDE_MASKED_LOAD25:%.*]] = call <8 x i32> @llvm.masked.load.v8i32.p0v8i32(<8 x i32>* nonnull [[TMP15]], i32 4, <8 x i1> [[TMP9]], <8 x i32> undef), !alias.scope !3
 ; AVX2-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i32, i32* [[TMP12]], i64 16
 ; AVX2-NEXT:    [[TMP17:%.*]] = bitcast i32* [[TMP16]] to <8 x i32>*
-; AVX2-NEXT:    [[WIDE_MASKED_LOAD26:%.*]] = call <8 x i32> @llvm.masked.load.v8i32.p0v8i32(<8 x i32>* [[TMP17]], i32 4, <8 x i1> [[TMP10]], <8 x i32> undef), !alias.scope !3
+; AVX2-NEXT:    [[WIDE_MASKED_LOAD26:%.*]] = call <8 x i32> @llvm.masked.load.v8i32.p0v8i32(<8 x i32>* nonnull [[TMP17]], i32 4, <8 x i1> [[TMP10]], <8 x i32> undef), !alias.scope !3
 ; AVX2-NEXT:    [[TMP18:%.*]] = getelementptr inbounds i32, i32* [[TMP12]], i64 24
 ; AVX2-NEXT:    [[TMP19:%.*]] = bitcast i32* [[TMP18]] to <8 x i32>*
-; AVX2-NEXT:    [[WIDE_MASKED_LOAD27:%.*]] = call <8 x i32> @llvm.masked.load.v8i32.p0v8i32(<8 x i32>* [[TMP19]], i32 4, <8 x i1> [[TMP11]], <8 x i32> undef), !alias.scope !3
+; AVX2-NEXT:    [[WIDE_MASKED_LOAD27:%.*]] = call <8 x i32> @llvm.masked.load.v8i32.p0v8i32(<8 x i32>* nonnull [[TMP19]], i32 4, <8 x i1> [[TMP11]], <8 x i32> undef), !alias.scope !3
 ; AVX2-NEXT:    [[TMP20:%.*]] = add nsw <8 x i32> [[WIDE_MASKED_LOAD]], [[WIDE_LOAD]]
 ; AVX2-NEXT:    [[TMP21:%.*]] = add nsw <8 x i32> [[WIDE_MASKED_LOAD25]], [[WIDE_LOAD22]]
 ; AVX2-NEXT:    [[TMP22:%.*]] = add nsw <8 x i32> [[WIDE_MASKED_LOAD26]], [[WIDE_LOAD23]]
@@ -171,16 +171,16 @@ define void @foo1(i32* %A, i32* %B, i32* %trigger) {
 ; AVX2-NEXT:    [[TMP43:%.*]] = icmp slt <8 x i32> [[WIDE_LOAD24_1]], <i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100>
 ; AVX2-NEXT:    [[TMP44:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDEX_NEXT]]
 ; AVX2-NEXT:    [[TMP45:%.*]] = bitcast i32* [[TMP44]] to <8 x i32>*
-; AVX2-NEXT:    [[WIDE_MASKED_LOAD_1:%.*]] = call <8 x i32> @llvm.masked.load.v8i32.p0v8i32(<8 x i32>* [[TMP45]], i32 4, <8 x i1> [[TMP40]], <8 x i32> undef), !alias.scope !3
+; AVX2-NEXT:    [[WIDE_MASKED_LOAD_1:%.*]] = call <8 x i32> @llvm.masked.load.v8i32.p0v8i32(<8 x i32>* nonnull [[TMP45]], i32 4, <8 x i1> [[TMP40]], <8 x i32> undef), !alias.scope !3
 ; AVX2-NEXT:    [[TMP46:%.*]] = getelementptr inbounds i32, i32* [[TMP44]], i64 8
 ; AVX2-NEXT:    [[TMP47:%.*]] = bitcast i32* [[TMP46]] to <8 x i32>*
-; AVX2-NEXT:    [[WIDE_MASKED_LOAD25_1:%.*]] = call <8 x i32> @llvm.masked.load.v8i32.p0v8i32(<8 x i32>* [[TMP47]], i32 4, <8 x i1> [[TMP41]], <8 x i32> undef), !alias.scope !3
+; AVX2-NEXT:    [[WIDE_MASKED_LOAD25_1:%.*]] = call <8 x i32> @llvm.masked.load.v8i32.p0v8i32(<8 x i32>* nonnull [[TMP47]], i32 4, <8 x i1> [[TMP41]], <8 x i32> undef), !alias.scope !3
 ; AVX2-NEXT:    [[TMP48:%.*]] = getelementptr inbounds i32, i32* [[TMP44]], i64 16
 ; AVX2-NEXT:    [[TMP49:%.*]] = bitcast i32* [[TMP48]] to <8 x i32>*
-; AVX2-NEXT:    [[WIDE_MASKED_LOAD26_1:%.*]] = call <8 x i32> @llvm.masked.load.v8i32.p0v8i32(<8 x i32>* [[TMP49]], i32 4, <8 x i1> [[TMP42]], <8 x i32> undef), !alias.scope !3
+; AVX2-NEXT:    [[WIDE_MASKED_LOAD26_1:%.*]] = call <8 x i32> @llvm.masked.load.v8i32.p0v8i32(<8 x i32>* nonnull [[TMP49]], i32 4, <8 x i1> [[TMP42]], <8 x i32> undef), !alias.scope !3
 ; AVX2-NEXT:    [[TMP50:%.*]] = getelementptr inbounds i32, i32* [[TMP44]], i64 24
 ; AVX2-NEXT:    [[TMP51:%.*]] = bitcast i32* [[TMP50]] to <8 x i32>*
-; AVX2-NEXT:    [[WIDE_MASKED_LOAD27_1:%.*]] = call <8 x i32> @llvm.masked.load.v8i32.p0v8i32(<8 x i32>* [[TMP51]], i32 4, <8 x i1> [[TMP43]], <8 x i32> undef), !alias.scope !3
+; AVX2-NEXT:    [[WIDE_MASKED_LOAD27_1:%.*]] = call <8 x i32> @llvm.masked.load.v8i32.p0v8i32(<8 x i32>* nonnull [[TMP51]], i32 4, <8 x i1> [[TMP43]], <8 x i32> undef), !alias.scope !3
 ; AVX2-NEXT:    [[TMP52:%.*]] = add nsw <8 x i32> [[WIDE_MASKED_LOAD_1]], [[WIDE_LOAD_1]]
 ; AVX2-NEXT:    [[TMP53:%.*]] = add nsw <8 x i32> [[WIDE_MASKED_LOAD25_1]], [[WIDE_LOAD22_1]]
 ; AVX2-NEXT:    [[TMP54:%.*]] = add nsw <8 x i32> [[WIDE_MASKED_LOAD26_1]], [[WIDE_LOAD23_1]]
@@ -298,13 +298,13 @@ define void @foo1(i32* %A, i32* %B, i32* %trigger) {
 ; AVX512-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <16 x i32> @llvm.masked.load.v16i32.p0v16i32(<16 x i32>* [[TMP13]], i32 4, <16 x i1> [[TMP8]], <16 x i32> undef), !alias.scope !3
 ; AVX512-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i32, i32* [[TMP12]], i64 16
 ; AVX512-NEXT:    [[TMP15:%.*]] = bitcast i32* [[TMP14]] to <16 x i32>*
-; AVX512-NEXT:    [[WIDE_MASKED_LOAD25:%.*]] = call <16 x i32> @llvm.masked.load.v16i32.p0v16i32(<16 x i32>* [[TMP15]], i32 4, <16 x i1> [[TMP9]], <16 x i32> undef), !alias.scope !3
+; AVX512-NEXT:    [[WIDE_MASKED_LOAD25:%.*]] = call <16 x i32> @llvm.masked.load.v16i32.p0v16i32(<16 x i32>* nonnull [[TMP15]], i32 4, <16 x i1> [[TMP9]], <16 x i32> undef), !alias.scope !3
 ; AVX512-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i32, i32* [[TMP12]], i64 32
 ; AVX512-NEXT:    [[TMP17:%.*]] = bitcast i32* [[TMP16]] to <16 x i32>*
-; AVX512-NEXT:    [[WIDE_MASKED_LOAD26:%.*]] = call <16 x i32> @llvm.masked.load.v16i32.p0v16i32(<16 x i32>* [[TMP17]], i32 4, <16 x i1> [[TMP10]], <16 x i32> undef), !alias.scope !3
+; AVX512-NEXT:    [[WIDE_MASKED_LOAD26:%.*]] = call <16 x i32> @llvm.masked.load.v16i32.p0v16i32(<16 x i32>* nonnull [[TMP17]], i32 4, <16 x i1> [[TMP10]], <16 x i32> undef), !alias.scope !3
 ; AVX512-NEXT:    [[TMP18:%.*]] = getelementptr inbounds i32, i32* [[TMP12]], i64 48
 ; AVX512-NEXT:    [[TMP19:%.*]] = bitcast i32* [[TMP18]] to <16 x i32>*
-; AVX512-NEXT:    [[WIDE_MASKED_LOAD27:%.*]] = call <16 x i32> @llvm.masked.load.v16i32.p0v16i32(<16 x i32>* [[TMP19]], i32 4, <16 x i1> [[TMP11]], <16 x i32> undef), !alias.scope !3
+; AVX512-NEXT:    [[WIDE_MASKED_LOAD27:%.*]] = call <16 x i32> @llvm.masked.load.v16i32.p0v16i32(<16 x i32>* nonnull [[TMP19]], i32 4, <16 x i1> [[TMP11]], <16 x i32> undef), !alias.scope !3
 ; AVX512-NEXT:    [[TMP20:%.*]] = add nsw <16 x i32> [[WIDE_MASKED_LOAD]], [[WIDE_LOAD]]
 ; AVX512-NEXT:    [[TMP21:%.*]] = add nsw <16 x i32> [[WIDE_MASKED_LOAD25]], [[WIDE_LOAD22]]
 ; AVX512-NEXT:    [[TMP22:%.*]] = add nsw <16 x i32> [[WIDE_MASKED_LOAD26]], [[WIDE_LOAD23]]
@@ -340,16 +340,16 @@ define void @foo1(i32* %A, i32* %B, i32* %trigger) {
 ; AVX512-NEXT:    [[TMP43:%.*]] = icmp slt <16 x i32> [[WIDE_LOAD24_1]], <i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100, i32 100>
 ; AVX512-NEXT:    [[TMP44:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[INDEX_NEXT]]
 ; AVX512-NEXT:    [[TMP45:%.*]] = bitcast i32* [[TMP44]] to <16 x i32>*
-; AVX512-NEXT:    [[WIDE_MASKED_LOAD_1:%.*]] = call <16 x i32> @llvm.masked.load.v16i32.p0v16i32(<16 x i32>* [[TMP45]], i32 4, <16 x i1> [[TMP40]], <16 x i32> undef), !alias.scope !3
+; AVX512-NEXT:    [[WIDE_MASKED_LOAD_1:%.*]] = call <16 x i32> @llvm.masked.load.v16i32.p0v16i32(<16 x i32>* nonnull [[TMP45]], i32 4, <16 x i1> [[TMP40]], <16 x i32> undef), !alias.scope !3
 ; AVX512-NEXT:    [[TMP46:%.*]] = getelementptr inbounds i32, i32* [[TMP44]], i64 16
 ; AVX512-NEXT:    [[TMP47:%.*]] = bitcast i32* [[TMP46]] to <16 x i32>*
-; AVX512-NEXT:    [[WIDE_MASKED_LOAD25_1:%.*]] = call <16 x i32> @llvm.masked.load.v16i32.p0v16i32(<16 x i32>* [[TMP47]], i32 4, <16 x i1> [[TMP41]], <16 x i32> undef), !alias.scope !3
+; AVX512-NEXT:    [[WIDE_MASKED_LOAD25_1:%.*]] = call <16 x i32> @llvm.masked.load.v16i32.p0v16i32(<16 x i32>* nonnull [[TMP47]], i32 4, <16 x i1> [[TMP41]], <16 x i32> undef), !alias.scope !3
 ; AVX512-NEXT:    [[TMP48:%.*]] = getelementptr inbounds i32, i32* [[TMP44]], i64 32
 ; AVX512-NEXT:    [[TMP49:%.*]] = bitcast i32* [[TMP48]] to <16 x i32>*
-; AVX512-NEXT:    [[WIDE_MASKED_LOAD26_1:%.*]] = call <16 x i32> @llvm.masked.load.v16i32.p0v16i32(<16 x i32>* [[TMP49]], i32 4, <16 x i1> [[TMP42]], <16 x i32> undef), !alias.scope !3
+; AVX512-NEXT:    [[WIDE_MASKED_LOAD26_1:%.*]] = call <16 x i32> @llvm.masked.load.v16i32.p0v16i32(<16 x i32>* nonnull [[TMP49]], i32 4, <16 x i1> [[TMP42]], <16 x i32> undef), !alias.scope !3
 ; AVX512-NEXT:    [[TMP50:%.*]] = getelementptr inbounds i32, i32* [[TMP44]], i64 48
 ; AVX512-NEXT:    [[TMP51:%.*]] = bitcast i32* [[TMP50]] to <16 x i32>*
-; AVX512-NEXT:    [[WIDE_MASKED_LOAD27_1:%.*]] = call <16 x i32> @llvm.masked.load.v16i32.p0v16i32(<16 x i32>* [[TMP51]], i32 4, <16 x i1> [[TMP43]], <16 x i32> undef), !alias.scope !3
+; AVX512-NEXT:    [[WIDE_MASKED_LOAD27_1:%.*]] = call <16 x i32> @llvm.masked.load.v16i32.p0v16i32(<16 x i32>* nonnull [[TMP51]], i32 4, <16 x i1> [[TMP43]], <16 x i32> undef), !alias.scope !3
 ; AVX512-NEXT:    [[TMP52:%.*]] = add nsw <16 x i32> [[WIDE_MASKED_LOAD_1]], [[WIDE_LOAD_1]]
 ; AVX512-NEXT:    [[TMP53:%.*]] = add nsw <16 x i32> [[WIDE_MASKED_LOAD25_1]], [[WIDE_LOAD22_1]]
 ; AVX512-NEXT:    [[TMP54:%.*]] = add nsw <16 x i32> [[WIDE_MASKED_LOAD26_1]], [[WIDE_LOAD23_1]]
@@ -1012,13 +1012,13 @@ define void @foo2(float* %A, float* %B, i32* %trigger) {
 ; AVX1-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <8 x float> @llvm.masked.load.v8f32.p0v8f32(<8 x float>* [[TMP15]], i32 4, <8 x i1> [[TMP10]], <8 x float> undef), !alias.scope !24
 ; AVX1-NEXT:    [[TMP16:%.*]] = getelementptr inbounds float, float* [[TMP14]], i64 8
 ; AVX1-NEXT:    [[TMP17:%.*]] = bitcast float* [[TMP16]] to <8 x float>*
-; AVX1-NEXT:    [[WIDE_MASKED_LOAD25:%.*]] = call <8 x float> @llvm.masked.load.v8f32.p0v8f32(<8 x float>* [[TMP17]], i32 4, <8 x i1> [[TMP11]], <8 x float> undef), !alias.scope !24
+; AVX1-NEXT:    [[WIDE_MASKED_LOAD25:%.*]] = call <8 x float> @llvm.masked.load.v8f32.p0v8f32(<8 x float>* nonnull [[TMP17]], i32 4, <8 x i1> [[TMP11]], <8 x float> undef), !alias.scope !24
 ; AVX1-NEXT:    [[TMP18:%.*]] = getelementptr inbounds float, float* [[TMP14]], i64 16
 ; AVX1-NEXT:    [[TMP19:%.*]] = bitcast float* [[TMP18]] to <8 x float>*
-; AVX1-NEXT:    [[WIDE_MASKED_LOAD26:%.*]] = call <8 x float> @llvm.masked.load.v8f32.p0v8f32(<8 x float>* [[TMP19]], i32 4, <8 x i1> [[TMP12]], <8 x float> undef), !alias.scope !24
+; AVX1-NEXT:    [[WIDE_MASKED_LOAD26:%.*]] = call <8 x float> @llvm.masked.load.v8f32.p0v8f32(<8 x float>* nonnull [[TMP19]], i32 4, <8 x i1> [[TMP12]], <8 x float> undef), !alias.scope !24
 ; AVX1-NEXT:    [[TMP20:%.*]] = getelementptr inbounds float, float* [[TMP14]], i64 24
 ; AVX1-NEXT:    [[TMP21:%.*]] = bitcast float* [[TMP20]] to <8 x float>*
-; AVX1-NEXT:    [[WIDE_MASKED_LOAD27:%.*]] = call <8 x float> @llvm.masked.load.v8f32.p0v8f32(<8 x float>* [[TMP21]], i32 4, <8 x i1> [[TMP13]], <8 x float> undef), !alias.scope !24
+; AVX1-NEXT:    [[WIDE_MASKED_LOAD27:%.*]] = call <8 x float> @llvm.masked.load.v8f32.p0v8f32(<8 x float>* nonnull [[TMP21]], i32 4, <8 x i1> [[TMP13]], <8 x float> undef), !alias.scope !24
 ; AVX1-NEXT:    [[TMP22:%.*]] = sitofp <8 x i32> [[WIDE_LOAD]] to <8 x float>
 ; AVX1-NEXT:    [[TMP23:%.*]] = sitofp <8 x i32> [[WIDE_LOAD22]] to <8 x float>
 ; AVX1-NEXT:    [[TMP24:%.*]] = sitofp <8 x i32> [[WIDE_LOAD23]] to <8 x float>
@@ -1118,13 +1118,13 @@ define void @foo2(float* %A, float* %B, i32* %trigger) {
 ; AVX2-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <8 x float> @llvm.masked.load.v8f32.p0v8f32(<8 x float>* [[TMP15]], i32 4, <8 x i1> [[TMP10]], <8 x float> undef), !alias.scope !24
 ; AVX2-NEXT:    [[TMP16:%.*]] = getelementptr inbounds float, float* [[TMP14]], i64 8
 ; AVX2-NEXT:    [[TMP17:%.*]] = bitcast float* [[TMP16]] to <8 x float>*
-; AVX2-NEXT:    [[WIDE_MASKED_LOAD25:%.*]] = call <8 x float> @llvm.masked.load.v8f32.p0v8f32(<8 x float>* [[TMP17]], i32 4, <8 x i1> [[TMP11]], <8 x float> undef), !alias.scope !24
+; AVX2-NEXT:    [[WIDE_MASKED_LOAD25:%.*]] = call <8 x float> @llvm.masked.load.v8f32.p0v8f32(<8 x float>* nonnull [[TMP17]], i32 4, <8 x i1> [[TMP11]], <8 x float> undef), !alias.scope !24
 ; AVX2-NEXT:    [[TMP18:%.*]] = getelementptr inbounds float, float* [[TMP14]], i64 16
 ; AVX2-NEXT:    [[TMP19:%.*]] = bitcast float* [[TMP18]] to <8 x float>*
-; AVX2-NEXT:    [[WIDE_MASKED_LOAD26:%.*]] = call <8 x float> @llvm.masked.load.v8f32.p0v8f32(<8 x float>* [[TMP19]], i32 4, <8 x i1> [[TMP12]], <8 x float> undef), !alias.scope !24
+; AVX2-NEXT:    [[WIDE_MASKED_LOAD26:%.*]] = call <8 x float> @llvm.masked.load.v8f32.p0v8f32(<8 x float>* nonnull [[TMP19]], i32 4, <8 x i1> [[TMP12]], <8 x float> undef), !alias.scope !24
 ; AVX2-NEXT:    [[TMP20:%.*]] = getelementptr inbounds float, float* [[TMP14]], i64 24
 ; AVX2-NEXT:    [[TMP21:%.*]] = bitcast float* [[TMP20]] to <8 x float>*
-; AVX2-NEXT:    [[WIDE_MASKED_LOAD27:%.*]] = call <8 x float> @llvm.masked.load.v8f32.p0v8f32(<8 x float>* [[TMP21]], i32 4, <8 x i1> [[TMP13]], <8 x float> undef), !alias.scope !24
+; AVX2-NEXT:    [[WIDE_MASKED_LOAD27:%.*]] = call <8 x float> @llvm.masked.load.v8f32.p0v8f32(<8 x float>* nonnull [[TMP21]], i32 4, <8 x i1> [[TMP13]], <8 x float> undef), !alias.scope !24
 ; AVX2-NEXT:    [[TMP22:%.*]] = sitofp <8 x i32> [[WIDE_LOAD]] to <8 x float>
 ; AVX2-NEXT:    [[TMP23:%.*]] = sitofp <8 x i32> [[WIDE_LOAD22]] to <8 x float>
 ; AVX2-NEXT:    [[TMP24:%.*]] = sitofp <8 x i32> [[WIDE_LOAD23]] to <8 x float>
@@ -1252,13 +1252,13 @@ define void @foo2(float* %A, float* %B, i32* %trigger) {
 ; AVX512-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <16 x float> @llvm.masked.load.v16f32.p0v16f32(<16 x float>* [[TMP15]], i32 4, <16 x i1> [[TMP10]], <16 x float> undef), !alias.scope !24
 ; AVX512-NEXT:    [[TMP16:%.*]] = getelementptr inbounds float, float* [[TMP14]], i64 16
 ; AVX512-NEXT:    [[TMP17:%.*]] = bitcast float* [[TMP16]] to <16 x float>*
-; AVX512-NEXT:    [[WIDE_MASKED_LOAD25:%.*]] = call <16 x float> @llvm.masked.load.v16f32.p0v16f32(<16 x float>* [[TMP17]], i32 4, <16 x i1> [[TMP11]], <16 x float> undef), !alias.scope !24
+; AVX512-NEXT:    [[WIDE_MASKED_LOAD25:%.*]] = call <16 x float> @llvm.masked.load.v16f32.p0v16f32(<16 x float>* nonnull [[TMP17]], i32 4, <16 x i1> [[TMP11]], <16 x float> undef), !alias.scope !24
 ; AVX512-NEXT:    [[TMP18:%.*]] = getelementptr inbounds float, float* [[TMP14]], i64 32
 ; AVX512-NEXT:    [[TMP19:%.*]] = bitcast float* [[TMP18]] to <16 x float>*
-; AVX512-NEXT:    [[WIDE_MASKED_LOAD26:%.*]] = call <16 x float> @llvm.masked.load.v16f32.p0v16f32(<16 x float>* [[TMP19]], i32 4, <16 x i1> [[TMP12]], <16 x float> undef), !alias.scope !24
+; AVX512-NEXT:    [[WIDE_MASKED_LOAD26:%.*]] = call <16 x float> @llvm.masked.load.v16f32.p0v16f32(<16 x float>* nonnull [[TMP19]], i32 4, <16 x i1> [[TMP12]], <16 x float> undef), !alias.scope !24
 ; AVX512-NEXT:    [[TMP20:%.*]] = getelementptr inbounds float, float* [[TMP14]], i64 48
 ; AVX512-NEXT:    [[TMP21:%.*]] = bitcast float* [[TMP20]] to <16 x float>*
-; AVX512-NEXT:    [[WIDE_MASKED_LOAD27:%.*]] = call <16 x float> @llvm.masked.load.v16f32.p0v16f32(<16 x float>* [[TMP21]], i32 4, <16 x i1> [[TMP13]], <16 x float> undef), !alias.scope !24
+; AVX512-NEXT:    [[WIDE_MASKED_LOAD27:%.*]] = call <16 x float> @llvm.masked.load.v16f32.p0v16f32(<16 x float>* nonnull [[TMP21]], i32 4, <16 x i1> [[TMP13]], <16 x float> undef), !alias.scope !24
 ; AVX512-NEXT:    [[TMP22:%.*]] = sitofp <16 x i32> [[WIDE_LOAD]] to <16 x float>
 ; AVX512-NEXT:    [[TMP23:%.*]] = sitofp <16 x i32> [[WIDE_LOAD22]] to <16 x float>
 ; AVX512-NEXT:    [[TMP24:%.*]] = sitofp <16 x i32> [[WIDE_LOAD23]] to <16 x float>
@@ -1457,13 +1457,13 @@ define void @foo3(double* %A, double* %B, i32* %trigger) #0 {
 ; AVX1-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <4 x double> @llvm.masked.load.v4f64.p0v4f64(<4 x double>* [[TMP15]], i32 8, <4 x i1> [[TMP10]], <4 x double> undef), !alias.scope !34
 ; AVX1-NEXT:    [[TMP16:%.*]] = getelementptr inbounds double, double* [[TMP14]], i64 4
 ; AVX1-NEXT:    [[TMP17:%.*]] = bitcast double* [[TMP16]] to <4 x double>*
-; AVX1-NEXT:    [[WIDE_MASKED_LOAD25:%.*]] = call <4 x double> @llvm.masked.load.v4f64.p0v4f64(<4 x double>* [[TMP17]], i32 8, <4 x i1> [[TMP11]], <4 x double> undef), !alias.scope !34
+; AVX1-NEXT:    [[WIDE_MASKED_LOAD25:%.*]] = call <4 x double> @llvm.masked.load.v4f64.p0v4f64(<4 x double>* nonnull [[TMP17]], i32 8, <4 x i1> [[TMP11]], <4 x double> undef), !alias.scope !34
 ; AVX1-NEXT:    [[TMP18:%.*]] = getelementptr inbounds double, double* [[TMP14]], i64 8
 ; AVX1-NEXT:    [[TMP19:%.*]] = bitcast double* [[TMP18]] to <4 x double>*
-; AVX1-NEXT:    [[WIDE_MASKED_LOAD26:%.*]] = call <4 x double> @llvm.masked.load.v4f64.p0v4f64(<4 x double>* [[TMP19]], i32 8, <4 x i1> [[TMP12]], <4 x double> undef), !alias.scope !34
+; AVX1-NEXT:    [[WIDE_MASKED_LOAD26:%.*]] = call <4 x double> @llvm.masked.load.v4f64.p0v4f64(<4 x double>* nonnull [[TMP19]], i32 8, <4 x i1> [[TMP12]], <4 x double> undef), !alias.scope !34
 ; AVX1-NEXT:    [[TMP20:%.*]] = getelementptr inbounds double, double* [[TMP14]], i64 12
 ; AVX1-NEXT:    [[TMP21:%.*]] = bitcast double* [[TMP20]] to <4 x double>*
-; AVX1-NEXT:    [[WIDE_MASKED_LOAD27:%.*]] = call <4 x double> @llvm.masked.load.v4f64.p0v4f64(<4 x double>* [[TMP21]], i32 8, <4 x i1> [[TMP13]], <4 x double> undef), !alias.scope !34
+; AVX1-NEXT:    [[WIDE_MASKED_LOAD27:%.*]] = call <4 x double> @llvm.masked.load.v4f64.p0v4f64(<4 x double>* nonnull [[TMP21]], i32 8, <4 x i1> [[TMP13]], <4 x double> undef), !alias.scope !34
 ; AVX1-NEXT:    [[TMP22:%.*]] = sitofp <4 x i32> [[WIDE_LOAD]] to <4 x double>
 ; AVX1-NEXT:    [[TMP23:%.*]] = sitofp <4 x i32> [[WIDE_LOAD22]] to <4 x double>
 ; AVX1-NEXT:    [[TMP24:%.*]] = sitofp <4 x i32> [[WIDE_LOAD23]] to <4 x double>
@@ -1560,13 +1560,13 @@ define void @foo3(double* %A, double* %B, i32* %trigger) #0 {
 ; AVX2-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <4 x double> @llvm.masked.load.v4f64.p0v4f64(<4 x double>* [[TMP15]], i32 8, <4 x i1> [[TMP10]], <4 x double> undef), !alias.scope !34
 ; AVX2-NEXT:    [[TMP16:%.*]] = getelementptr inbounds double, double* [[TMP14]], i64 4
 ; AVX2-NEXT:    [[TMP17:%.*]] = bitcast double* [[TMP16]] to <4 x double>*
-; AVX2-NEXT:    [[WIDE_MASKED_LOAD25:%.*]] = call <4 x double> @llvm.masked.load.v4f64.p0v4f64(<4 x double>* [[TMP17]], i32 8, <4 x i1> [[TMP11]], <4 x double> undef), !alias.scope !34
+; AVX2-NEXT:    [[WIDE_MASKED_LOAD25:%.*]] = call <4 x double> @llvm.masked.load.v4f64.p0v4f64(<4 x double>* nonnull [[TMP17]], i32 8, <4 x i1> [[TMP11]], <4 x double> undef), !alias.scope !34
 ; AVX2-NEXT:    [[TMP18:%.*]] = getelementptr inbounds double, double* [[TMP14]], i64 8
 ; AVX2-NEXT:    [[TMP19:%.*]] = bitcast double* [[TMP18]] to <4 x double>*
-; AVX2-NEXT:    [[WIDE_MASKED_LOAD26:%.*]] = call <4 x double> @llvm.masked.load.v4f64.p0v4f64(<4 x double>* [[TMP19]], i32 8, <4 x i1> [[TMP12]], <4 x double> undef), !alias.scope !34
+; AVX2-NEXT:    [[WIDE_MASKED_LOAD26:%.*]] = call <4 x double> @llvm.masked.load.v4f64.p0v4f64(<4 x double>* nonnull [[TMP19]], i32 8, <4 x i1> [[TMP12]], <4 x double> undef), !alias.scope !34
 ; AVX2-NEXT:    [[TMP20:%.*]] = getelementptr inbounds double, double* [[TMP14]], i64 12
 ; AVX2-NEXT:    [[TMP21:%.*]] = bitcast double* [[TMP20]] to <4 x double>*
-; AVX2-NEXT:    [[WIDE_MASKED_LOAD27:%.*]] = call <4 x double> @llvm.masked.load.v4f64.p0v4f64(<4 x double>* [[TMP21]], i32 8, <4 x i1> [[TMP13]], <4 x double> undef), !alias.scope !34
+; AVX2-NEXT:    [[WIDE_MASKED_LOAD27:%.*]] = call <4 x double> @llvm.masked.load.v4f64.p0v4f64(<4 x double>* nonnull [[TMP21]], i32 8, <4 x i1> [[TMP13]], <4 x double> undef), !alias.scope !34
 ; AVX2-NEXT:    [[TMP22:%.*]] = sitofp <4 x i32> [[WIDE_LOAD]] to <4 x double>
 ; AVX2-NEXT:    [[TMP23:%.*]] = sitofp <4 x i32> [[WIDE_LOAD22]] to <4 x double>
 ; AVX2-NEXT:    [[TMP24:%.*]] = sitofp <4 x i32> [[WIDE_LOAD23]] to <4 x double>
@@ -1691,13 +1691,13 @@ define void @foo3(double* %A, double* %B, i32* %trigger) #0 {
 ; AVX512-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <8 x double> @llvm.masked.load.v8f64.p0v8f64(<8 x double>* [[TMP15]], i32 8, <8 x i1> [[TMP10]], <8 x double> undef), !alias.scope !34
 ; AVX512-NEXT:    [[TMP16:%.*]] = getelementptr inbounds double, double* [[TMP14]], i64 8
 ; AVX512-NEXT:    [[TMP17:%.*]] = bitcast double* [[TMP16]] to <8 x double>*
-; AVX512-NEXT:    [[WIDE_MASKED_LOAD25:%.*]] = call <8 x double> @llvm.masked.load.v8f64.p0v8f64(<8 x double>* [[TMP17]], i32 8, <8 x i1> [[TMP11]], <8 x double> undef), !alias.scope !34
+; AVX512-NEXT:    [[WIDE_MASKED_LOAD25:%.*]] = call <8 x double> @llvm.masked.load.v8f64.p0v8f64(<8 x double>* nonnull [[TMP17]], i32 8, <8 x i1> [[TMP11]], <8 x double> undef), !alias.scope !34
 ; AVX512-NEXT:    [[TMP18:%.*]] = getelementptr inbounds double, double* [[TMP14]], i64 16
 ; AVX512-NEXT:    [[TMP19:%.*]] = bitcast double* [[TMP18]] to <8 x double>*
-; AVX512-NEXT:    [[WIDE_MASKED_LOAD26:%.*]] = call <8 x double> @llvm.masked.load.v8f64.p0v8f64(<8 x double>* [[TMP19]], i32 8, <8 x i1> [[TMP12]], <8 x double> undef), !alias.scope !34
+; AVX512-NEXT:    [[WIDE_MASKED_LOAD26:%.*]] = call <8 x double> @llvm.masked.load.v8f64.p0v8f64(<8 x double>* nonnull [[TMP19]], i32 8, <8 x i1> [[TMP12]], <8 x double> undef), !alias.scope !34
 ; AVX512-NEXT:    [[TMP20:%.*]] = getelementptr inbounds double, double* [[TMP14]], i64 24
 ; AVX512-NEXT:    [[TMP21:%.*]] = bitcast double* [[TMP20]] to <8 x double>*
-; AVX512-NEXT:    [[WIDE_MASKED_LOAD27:%.*]] = call <8 x double> @llvm.masked.load.v8f64.p0v8f64(<8 x double>* [[TMP21]], i32 8, <8 x i1> [[TMP13]], <8 x double> undef), !alias.scope !34
+; AVX512-NEXT:    [[WIDE_MASKED_LOAD27:%.*]] = call <8 x double> @llvm.masked.load.v8f64.p0v8f64(<8 x double>* nonnull [[TMP21]], i32 8, <8 x i1> [[TMP13]], <8 x double> undef), !alias.scope !34
 ; AVX512-NEXT:    [[TMP22:%.*]] = sitofp <8 x i32> [[WIDE_LOAD]] to <8 x double>
 ; AVX512-NEXT:    [[TMP23:%.*]] = sitofp <8 x i32> [[WIDE_LOAD22]] to <8 x double>
 ; AVX512-NEXT:    [[TMP24:%.*]] = sitofp <8 x i32> [[WIDE_LOAD23]] to <8 x double>
@@ -2453,22 +2453,22 @@ define void @foo6(double* %in, double* %out, i32 %size, i32* %trigger) {
 ; AVX1-NEXT:    [[TMP19:%.*]] = getelementptr inbounds double, double* [[TMP18]], i64 -3
 ; AVX1-NEXT:    [[REVERSE26:%.*]] = shufflevector <4 x i1> [[TMP14]], <4 x i1> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
 ; AVX1-NEXT:    [[TMP20:%.*]] = bitcast double* [[TMP19]] to <4 x double>*
-; AVX1-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <4 x double> @llvm.masked.load.v4f64.p0v4f64(<4 x double>* [[TMP20]], i32 8, <4 x i1> [[REVERSE26]], <4 x double> undef), !alias.scope !44
+; AVX1-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <4 x double> @llvm.masked.load.v4f64.p0v4f64(<4 x double>* nonnull [[TMP20]], i32 8, <4 x i1> [[REVERSE26]], <4 x double> undef), !alias.scope !44
 ; AVX1-NEXT:    [[TMP21:%.*]] = getelementptr inbounds double, double* [[TMP18]], i64 -4
 ; AVX1-NEXT:    [[TMP22:%.*]] = getelementptr inbounds double, double* [[TMP21]], i64 -3
 ; AVX1-NEXT:    [[REVERSE28:%.*]] = shufflevector <4 x i1> [[TMP15]], <4 x i1> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
 ; AVX1-NEXT:    [[TMP23:%.*]] = bitcast double* [[TMP22]] to <4 x double>*
-; AVX1-NEXT:    [[WIDE_MASKED_LOAD29:%.*]] = call <4 x double> @llvm.masked.load.v4f64.p0v4f64(<4 x double>* [[TMP23]], i32 8, <4 x i1> [[REVERSE28]], <4 x double> undef), !alias.scope !44
+; AVX1-NEXT:    [[WIDE_MASKED_LOAD29:%.*]] = call <4 x double> @llvm.masked.load.v4f64.p0v4f64(<4 x double>* nonnull [[TMP23]], i32 8, <4 x i1> [[REVERSE28]], <4 x double> undef), !alias.scope !44
 ; AVX1-NEXT:    [[TMP24:%.*]] = getelementptr inbounds double, double* [[TMP18]], i64 -8
 ; AVX1-NEXT:    [[TMP25:%.*]] = getelementptr inbounds double, double* [[TMP24]], i64 -3
 ; AVX1-NEXT:    [[REVERSE31:%.*]] = shufflevector <4 x i1> [[TMP16]], <4 x i1> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
 ; AVX1-NEXT:    [[TMP26:%.*]] = bitcast double* [[TMP25]] to <4 x double>*
-; AVX1-NEXT:    [[WIDE_MASKED_LOAD32:%.*]] = call <4 x double> @llvm.masked.load.v4f64.p0v4f64(<4 x double>* [[TMP26]], i32 8, <4 x i1> [[REVERSE31]], <4 x double> undef), !alias.scope !44
+; AVX1-NEXT:    [[WIDE_MASKED_LOAD32:%.*]] = call <4 x double> @llvm.masked.load.v4f64.p0v4f64(<4 x double>* nonnull [[TMP26]], i32 8, <4 x i1> [[REVERSE31]], <4 x double> undef), !alias.scope !44
 ; AVX1-NEXT:    [[TMP27:%.*]] = getelementptr inbounds double, double* [[TMP18]], i64 -12
 ; AVX1-NEXT:    [[TMP28:%.*]] = getelementptr inbounds double, double* [[TMP27]], i64 -3
 ; AVX1-NEXT:    [[REVERSE34:%.*]] = shufflevector <4 x i1> [[TMP17]], <4 x i1> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
 ; AVX1-NEXT:    [[TMP29:%.*]] = bitcast double* [[TMP28]] to <4 x double>*
-; AVX1-NEXT:    [[WIDE_MASKED_LOAD35:%.*]] = call <4 x double> @llvm.masked.load.v4f64.p0v4f64(<4 x double>* [[TMP29]], i32 8, <4 x i1> [[REVERSE34]], <4 x double> undef), !alias.scope !44
+; AVX1-NEXT:    [[WIDE_MASKED_LOAD35:%.*]] = call <4 x double> @llvm.masked.load.v4f64.p0v4f64(<4 x double>* nonnull [[TMP29]], i32 8, <4 x i1> [[REVERSE34]], <4 x double> undef), !alias.scope !44
 ; AVX1-NEXT:    [[TMP30:%.*]] = fadd <4 x double> [[WIDE_MASKED_LOAD]], <double 5.000000e-01, double 5.000000e-01, double 5.000000e-01, double 5.000000e-01>
 ; AVX1-NEXT:    [[TMP31:%.*]] = fadd <4 x double> [[WIDE_MASKED_LOAD29]], <double 5.000000e-01, double 5.000000e-01, double 5.000000e-01, double 5.000000e-01>
 ; AVX1-NEXT:    [[TMP32:%.*]] = fadd <4 x double> [[WIDE_MASKED_LOAD32]], <double 5.000000e-01, double 5.000000e-01, double 5.000000e-01, double 5.000000e-01>
@@ -2571,22 +2571,22 @@ define void @foo6(double* %in, double* %out, i32 %size, i32* %trigger) {
 ; AVX2-NEXT:    [[TMP19:%.*]] = getelementptr inbounds double, double* [[TMP18]], i64 -3
 ; AVX2-NEXT:    [[REVERSE26:%.*]] = shufflevector <4 x i1> [[TMP14]], <4 x i1> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
 ; AVX2-NEXT:    [[TMP20:%.*]] = bitcast double* [[TMP19]] to <4 x double>*
-; AVX2-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <4 x double> @llvm.masked.load.v4f64.p0v4f64(<4 x double>* [[TMP20]], i32 8, <4 x i1> [[REVERSE26]], <4 x double> undef), !alias.scope !44
+; AVX2-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <4 x double> @llvm.masked.load.v4f64.p0v4f64(<4 x double>* nonnull [[TMP20]], i32 8, <4 x i1> [[REVERSE26]], <4 x double> undef), !alias.scope !44
 ; AVX2-NEXT:    [[TMP21:%.*]] = getelementptr inbounds double, double* [[TMP18]], i64 -4
 ; AVX2-NEXT:    [[TMP22:%.*]] = getelementptr inbounds double, double* [[TMP21]], i64 -3
 ; AVX2-NEXT:    [[REVERSE28:%.*]] = shufflevector <4 x i1> [[TMP15]], <4 x i1> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
 ; AVX2-NEXT:    [[TMP23:%.*]] = bitcast double* [[TMP22]] to <4 x double>*
-; AVX2-NEXT:    [[WIDE_MASKED_LOAD29:%.*]] = call <4 x double> @llvm.masked.load.v4f64.p0v4f64(<4 x double>* [[TMP23]], i32 8, <4 x i1> [[REVERSE28]], <4 x double> undef), !alias.scope !44
+; AVX2-NEXT:    [[WIDE_MASKED_LOAD29:%.*]] = call <4 x double> @llvm.masked.load.v4f64.p0v4f64(<4 x double>* nonnull [[TMP23]], i32 8, <4 x i1> [[REVERSE28]], <4 x double> undef), !alias.scope !44
 ; AVX2-NEXT:    [[TMP24:%.*]] = getelementptr inbounds double, double* [[TMP18]], i64 -8
 ; AVX2-NEXT:    [[TMP25:%.*]] = getelementptr inbounds double, double* [[TMP24]], i64 -3
 ; AVX2-NEXT:    [[REVERSE31:%.*]] = shufflevector <4 x i1> [[TMP16]], <4 x i1> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
 ; AVX2-NEXT:    [[TMP26:%.*]] = bitcast double* [[TMP25]] to <4 x double>*
-; AVX2-NEXT:    [[WIDE_MASKED_LOAD32:%.*]] = call <4 x double> @llvm.masked.load.v4f64.p0v4f64(<4 x double>* [[TMP26]], i32 8, <4 x i1> [[REVERSE31]], <4 x double> undef), !alias.scope !44
+; AVX2-NEXT:    [[WIDE_MASKED_LOAD32:%.*]] = call <4 x double> @llvm.masked.load.v4f64.p0v4f64(<4 x double>* nonnull [[TMP26]], i32 8, <4 x i1> [[REVERSE31]], <4 x double> undef), !alias.scope !44
 ; AVX2-NEXT:    [[TMP27:%.*]] = getelementptr inbounds double, double* [[TMP18]], i64 -12
 ; AVX2-NEXT:    [[TMP28:%.*]] = getelementptr inbounds double, double* [[TMP27]], i64 -3
 ; AVX2-NEXT:    [[REVERSE34:%.*]] = shufflevector <4 x i1> [[TMP17]], <4 x i1> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
 ; AVX2-NEXT:    [[TMP29:%.*]] = bitcast double* [[TMP28]] to <4 x double>*
-; AVX2-NEXT:    [[WIDE_MASKED_LOAD35:%.*]] = call <4 x double> @llvm.masked.load.v4f64.p0v4f64(<4 x double>* [[TMP29]], i32 8, <4 x i1> [[REVERSE34]], <4 x double> undef), !alias.scope !44
+; AVX2-NEXT:    [[WIDE_MASKED_LOAD35:%.*]] = call <4 x double> @llvm.masked.load.v4f64.p0v4f64(<4 x double>* nonnull [[TMP29]], i32 8, <4 x i1> [[REVERSE34]], <4 x double> undef), !alias.scope !44
 ; AVX2-NEXT:    [[TMP30:%.*]] = fadd <4 x double> [[WIDE_MASKED_LOAD]], <double 5.000000e-01, double 5.000000e-01, double 5.000000e-01, double 5.000000e-01>
 ; AVX2-NEXT:    [[TMP31:%.*]] = fadd <4 x double> [[WIDE_MASKED_LOAD29]], <double 5.000000e-01, double 5.000000e-01, double 5.000000e-01, double 5.000000e-01>
 ; AVX2-NEXT:    [[TMP32:%.*]] = fadd <4 x double> [[WIDE_MASKED_LOAD32]], <double 5.000000e-01, double 5.000000e-01, double 5.000000e-01, double 5.000000e-01>
@@ -2715,22 +2715,22 @@ define void @foo6(double* %in, double* %out, i32 %size, i32* %trigger) {
 ; AVX512-NEXT:    [[TMP19:%.*]] = getelementptr inbounds double, double* [[TMP18]], i64 -7
 ; AVX512-NEXT:    [[REVERSE26:%.*]] = shufflevector <8 x i1> [[TMP14]], <8 x i1> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
 ; AVX512-NEXT:    [[TMP20:%.*]] = bitcast double* [[TMP19]] to <8 x double>*
-; AVX512-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <8 x double> @llvm.masked.load.v8f64.p0v8f64(<8 x double>* [[TMP20]], i32 8, <8 x i1> [[REVERSE26]], <8 x double> undef), !alias.scope !56
+; AVX512-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <8 x double> @llvm.masked.load.v8f64.p0v8f64(<8 x double>* nonnull [[TMP20]], i32 8, <8 x i1> [[REVERSE26]], <8 x double> undef), !alias.scope !56
 ; AVX512-NEXT:    [[TMP21:%.*]] = getelementptr inbounds double, double* [[TMP18]], i64 -8
 ; AVX512-NEXT:    [[TMP22:%.*]] = getelementptr inbounds double, double* [[TMP21]], i64 -7
 ; AVX512-NEXT:    [[REVERSE28:%.*]] = shufflevector <8 x i1> [[TMP15]], <8 x i1> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
 ; AVX512-NEXT:    [[TMP23:%.*]] = bitcast double* [[TMP22]] to <8 x double>*
-; AVX512-NEXT:    [[WIDE_MASKED_LOAD29:%.*]] = call <8 x double> @llvm.masked.load.v8f64.p0v8f64(<8 x double>* [[TMP23]], i32 8, <8 x i1> [[REVERSE28]], <8 x double> undef), !alias.scope !56
+; AVX512-NEXT:    [[WIDE_MASKED_LOAD29:%.*]] = call <8 x double> @llvm.masked.load.v8f64.p0v8f64(<8 x double>* nonnull [[TMP23]], i32 8, <8 x i1> [[REVERSE28]], <8 x double> undef), !alias.scope !56
 ; AVX512-NEXT:    [[TMP24:%.*]] = getelementptr inbounds double, double* [[TMP18]], i64 -16
 ; AVX512-NEXT:    [[TMP25:%.*]] = getelementptr inbounds double, double* [[TMP24]], i64 -7
 ; AVX512-NEXT:    [[REVERSE31:%.*]] = shufflevector <8 x i1> [[TMP16]], <8 x i1> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
 ; AVX512-NEXT:    [[TMP26:%.*]] = bitcast double* [[TMP25]] to <8 x double>*
-; AVX512-NEXT:    [[WIDE_MASKED_LOAD32:%.*]] = call <8 x double> @llvm.masked.load.v8f64.p0v8f64(<8 x double>* [[TMP26]], i32 8, <8 x i1> [[REVERSE31]], <8 x double> undef), !alias.scope !56
+; AVX512-NEXT:    [[WIDE_MASKED_LOAD32:%.*]] = call <8 x double> @llvm.masked.load.v8f64.p0v8f64(<8 x double>* nonnull [[TMP26]], i32 8, <8 x i1> [[REVERSE31]], <8 x double> undef), !alias.scope !56
 ; AVX512-NEXT:    [[TMP27:%.*]] = getelementptr inbounds double, double* [[TMP18]], i64 -24
 ; AVX512-NEXT:    [[TMP28:%.*]] = getelementptr inbounds double, double* [[TMP27]], i64 -7
 ; AVX512-NEXT:    [[REVERSE34:%.*]] = shufflevector <8 x i1> [[TMP17]], <8 x i1> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
 ; AVX512-NEXT:    [[TMP29:%.*]] = bitcast double* [[TMP28]] to <8 x double>*
-; AVX512-NEXT:    [[WIDE_MASKED_LOAD35:%.*]] = call <8 x double> @llvm.masked.load.v8f64.p0v8f64(<8 x double>* [[TMP29]], i32 8, <8 x i1> [[REVERSE34]], <8 x double> undef), !alias.scope !56
+; AVX512-NEXT:    [[WIDE_MASKED_LOAD35:%.*]] = call <8 x double> @llvm.masked.load.v8f64.p0v8f64(<8 x double>* nonnull [[TMP29]], i32 8, <8 x i1> [[REVERSE34]], <8 x double> undef), !alias.scope !56
 ; AVX512-NEXT:    [[TMP30:%.*]] = fadd <8 x double> [[WIDE_MASKED_LOAD]], <double 5.000000e-01, double 5.000000e-01, double 5.000000e-01, double 5.000000e-01, double 5.000000e-01, double 5.000000e-01, double 5.000000e-01, double 5.000000e-01>
 ; AVX512-NEXT:    [[TMP31:%.*]] = fadd <8 x double> [[WIDE_MASKED_LOAD29]], <double 5.000000e-01, double 5.000000e-01, double 5.000000e-01, double 5.000000e-01, double 5.000000e-01, double 5.000000e-01, double 5.000000e-01, double 5.000000e-01>
 ; AVX512-NEXT:    [[TMP32:%.*]] = fadd <8 x double> [[WIDE_MASKED_LOAD32]], <double 5.000000e-01, double 5.000000e-01, double 5.000000e-01, double 5.000000e-01, double 5.000000e-01, double 5.000000e-01, double 5.000000e-01, double 5.000000e-01>
@@ -2910,13 +2910,13 @@ define void @foo7(double* noalias %out, double** noalias %in, i8* noalias %trigg
 ; AVX-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <4 x double*> @llvm.masked.load.v4p0f64.p0v4p0f64(<4 x double*>* [[TMP13]], i32 8, <4 x i1> [[TMP8]], <4 x double*> undef)
 ; AVX-NEXT:    [[TMP14:%.*]] = getelementptr inbounds double*, double** [[TMP12]], i64 4
 ; AVX-NEXT:    [[TMP15:%.*]] = bitcast double** [[TMP14]] to <4 x double*>*
-; AVX-NEXT:    [[WIDE_MASKED_LOAD13:%.*]] = call <4 x double*> @llvm.masked.load.v4p0f64.p0v4p0f64(<4 x double*>* [[TMP15]], i32 8, <4 x i1> [[TMP9]], <4 x double*> undef)
+; AVX-NEXT:    [[WIDE_MASKED_LOAD13:%.*]] = call <4 x double*> @llvm.masked.load.v4p0f64.p0v4p0f64(<4 x double*>* nonnull [[TMP15]], i32 8, <4 x i1> [[TMP9]], <4 x double*> undef)
 ; AVX-NEXT:    [[TMP16:%.*]] = getelementptr inbounds double*, double** [[TMP12]], i64 8
 ; AVX-NEXT:    [[TMP17:%.*]] = bitcast double** [[TMP16]] to <4 x double*>*
-; AVX-NEXT:    [[WIDE_MASKED_LOAD14:%.*]] = call <4 x double*> @llvm.masked.load.v4p0f64.p0v4p0f64(<4 x double*>* [[TMP17]], i32 8, <4 x i1> [[TMP10]], <4 x double*> undef)
+; AVX-NEXT:    [[WIDE_MASKED_LOAD14:%.*]] = call <4 x double*> @llvm.masked.load.v4p0f64.p0v4p0f64(<4 x double*>* nonnull [[TMP17]], i32 8, <4 x i1> [[TMP10]], <4 x double*> undef)
 ; AVX-NEXT:    [[TMP18:%.*]] = getelementptr inbounds double*, double** [[TMP12]], i64 12
 ; AVX-NEXT:    [[TMP19:%.*]] = bitcast double** [[TMP18]] to <4 x double*>*
-; AVX-NEXT:    [[WIDE_MASKED_LOAD15:%.*]] = call <4 x double*> @llvm.masked.load.v4p0f64.p0v4p0f64(<4 x double*>* [[TMP19]], i32 8, <4 x i1> [[TMP11]], <4 x double*> undef)
+; AVX-NEXT:    [[WIDE_MASKED_LOAD15:%.*]] = call <4 x double*> @llvm.masked.load.v4p0f64.p0v4p0f64(<4 x double*>* nonnull [[TMP19]], i32 8, <4 x i1> [[TMP11]], <4 x double*> undef)
 ; AVX-NEXT:    [[TMP20:%.*]] = icmp ne <4 x double*> [[WIDE_MASKED_LOAD]], zeroinitializer
 ; AVX-NEXT:    [[TMP21:%.*]] = icmp ne <4 x double*> [[WIDE_MASKED_LOAD13]], zeroinitializer
 ; AVX-NEXT:    [[TMP22:%.*]] = icmp ne <4 x double*> [[WIDE_MASKED_LOAD14]], zeroinitializer
@@ -3003,13 +3003,13 @@ define void @foo7(double* noalias %out, double** noalias %in, i8* noalias %trigg
 ; AVX512-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <8 x double*> @llvm.masked.load.v8p0f64.p0v8p0f64(<8 x double*>* [[TMP13]], i32 8, <8 x i1> [[TMP8]], <8 x double*> undef)
 ; AVX512-NEXT:    [[TMP14:%.*]] = getelementptr inbounds double*, double** [[TMP12]], i64 8
 ; AVX512-NEXT:    [[TMP15:%.*]] = bitcast double** [[TMP14]] to <8 x double*>*
-; AVX512-NEXT:    [[WIDE_MASKED_LOAD13:%.*]] = call <8 x double*> @llvm.masked.load.v8p0f64.p0v8p0f64(<8 x double*>* [[TMP15]], i32 8, <8 x i1> [[TMP9]], <8 x double*> undef)
+; AVX512-NEXT:    [[WIDE_MASKED_LOAD13:%.*]] = call <8 x double*> @llvm.masked.load.v8p0f64.p0v8p0f64(<8 x double*>* nonnull [[TMP15]], i32 8, <8 x i1> [[TMP9]], <8 x double*> undef)
 ; AVX512-NEXT:    [[TMP16:%.*]] = getelementptr inbounds double*, double** [[TMP12]], i64 16
 ; AVX512-NEXT:    [[TMP17:%.*]] = bitcast double** [[TMP16]] to <8 x double*>*
-; AVX512-NEXT:    [[WIDE_MASKED_LOAD14:%.*]] = call <8 x double*> @llvm.masked.load.v8p0f64.p0v8p0f64(<8 x double*>* [[TMP17]], i32 8, <8 x i1> [[TMP10]], <8 x double*> undef)
+; AVX512-NEXT:    [[WIDE_MASKED_LOAD14:%.*]] = call <8 x double*> @llvm.masked.load.v8p0f64.p0v8p0f64(<8 x double*>* nonnull [[TMP17]], i32 8, <8 x i1> [[TMP10]], <8 x double*> undef)
 ; AVX512-NEXT:    [[TMP18:%.*]] = getelementptr inbounds double*, double** [[TMP12]], i64 24
 ; AVX512-NEXT:    [[TMP19:%.*]] = bitcast double** [[TMP18]] to <8 x double*>*
-; AVX512-NEXT:    [[WIDE_MASKED_LOAD15:%.*]] = call <8 x double*> @llvm.masked.load.v8p0f64.p0v8p0f64(<8 x double*>* [[TMP19]], i32 8, <8 x i1> [[TMP11]], <8 x double*> undef)
+; AVX512-NEXT:    [[WIDE_MASKED_LOAD15:%.*]] = call <8 x double*> @llvm.masked.load.v8p0f64.p0v8p0f64(<8 x double*>* nonnull [[TMP19]], i32 8, <8 x i1> [[TMP11]], <8 x double*> undef)
 ; AVX512-NEXT:    [[TMP20:%.*]] = icmp ne <8 x double*> [[WIDE_MASKED_LOAD]], zeroinitializer
 ; AVX512-NEXT:    [[TMP21:%.*]] = icmp ne <8 x double*> [[WIDE_MASKED_LOAD13]], zeroinitializer
 ; AVX512-NEXT:    [[TMP22:%.*]] = icmp ne <8 x double*> [[WIDE_MASKED_LOAD14]], zeroinitializer
@@ -3163,13 +3163,13 @@ define void @foo8(double* noalias %out, i32 ()** noalias %in, i8* noalias %trigg
 ; AVX-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <4 x i32 ()*> @llvm.masked.load.v4p0f_i32f.p0v4p0f_i32f(<4 x i32 ()*>* [[TMP13]], i32 8, <4 x i1> [[TMP8]], <4 x i32 ()*> undef)
 ; AVX-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i32 ()*, i32 ()** [[TMP12]], i64 4
 ; AVX-NEXT:    [[TMP15:%.*]] = bitcast i32 ()** [[TMP14]] to <4 x i32 ()*>*
-; AVX-NEXT:    [[WIDE_MASKED_LOAD13:%.*]] = call <4 x i32 ()*> @llvm.masked.load.v4p0f_i32f.p0v4p0f_i32f(<4 x i32 ()*>* [[TMP15]], i32 8, <4 x i1> [[TMP9]], <4 x i32 ()*> undef)
+; AVX-NEXT:    [[WIDE_MASKED_LOAD13:%.*]] = call <4 x i32 ()*> @llvm.masked.load.v4p0f_i32f.p0v4p0f_i32f(<4 x i32 ()*>* nonnull [[TMP15]], i32 8, <4 x i1> [[TMP9]], <4 x i32 ()*> undef)
 ; AVX-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i32 ()*, i32 ()** [[TMP12]], i64 8
 ; AVX-NEXT:    [[TMP17:%.*]] = bitcast i32 ()** [[TMP16]] to <4 x i32 ()*>*
-; AVX-NEXT:    [[WIDE_MASKED_LOAD14:%.*]] = call <4 x i32 ()*> @llvm.masked.load.v4p0f_i32f.p0v4p0f_i32f(<4 x i32 ()*>* [[TMP17]], i32 8, <4 x i1> [[TMP10]], <4 x i32 ()*> undef)
+; AVX-NEXT:    [[WIDE_MASKED_LOAD14:%.*]] = call <4 x i32 ()*> @llvm.masked.load.v4p0f_i32f.p0v4p0f_i32f(<4 x i32 ()*>* nonnull [[TMP17]], i32 8, <4 x i1> [[TMP10]], <4 x i32 ()*> undef)
 ; AVX-NEXT:    [[TMP18:%.*]] = getelementptr inbounds i32 ()*, i32 ()** [[TMP12]], i64 12
 ; AVX-NEXT:    [[TMP19:%.*]] = bitcast i32 ()** [[TMP18]] to <4 x i32 ()*>*
-; AVX-NEXT:    [[WIDE_MASKED_LOAD15:%.*]] = call <4 x i32 ()*> @llvm.masked.load.v4p0f_i32f.p0v4p0f_i32f(<4 x i32 ()*>* [[TMP19]], i32 8, <4 x i1> [[TMP11]], <4 x i32 ()*> undef)
+; AVX-NEXT:    [[WIDE_MASKED_LOAD15:%.*]] = call <4 x i32 ()*> @llvm.masked.load.v4p0f_i32f.p0v4p0f_i32f(<4 x i32 ()*>* nonnull [[TMP19]], i32 8, <4 x i1> [[TMP11]], <4 x i32 ()*> undef)
 ; AVX-NEXT:    [[TMP20:%.*]] = icmp ne <4 x i32 ()*> [[WIDE_MASKED_LOAD]], zeroinitializer
 ; AVX-NEXT:    [[TMP21:%.*]] = icmp ne <4 x i32 ()*> [[WIDE_MASKED_LOAD13]], zeroinitializer
 ; AVX-NEXT:    [[TMP22:%.*]] = icmp ne <4 x i32 ()*> [[WIDE_MASKED_LOAD14]], zeroinitializer
@@ -3256,13 +3256,13 @@ define void @foo8(double* noalias %out, i32 ()** noalias %in, i8* noalias %trigg
 ; AVX512-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <8 x i32 ()*> @llvm.masked.load.v8p0f_i32f.p0v8p0f_i32f(<8 x i32 ()*>* [[TMP13]], i32 8, <8 x i1> [[TMP8]], <8 x i32 ()*> undef)
 ; AVX512-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i32 ()*, i32 ()** [[TMP12]], i64 8
 ; AVX512-NEXT:    [[TMP15:%.*]] = bitcast i32 ()** [[TMP14]] to <8 x i32 ()*>*
-; AVX512-NEXT:    [[WIDE_MASKED_LOAD13:%.*]] = call <8 x i32 ()*> @llvm.masked.load.v8p0f_i32f.p0v8p0f_i32f(<8 x i32 ()*>* [[TMP15]], i32 8, <8 x i1> [[TMP9]], <8 x i32 ()*> undef)
+; AVX512-NEXT:    [[WIDE_MASKED_LOAD13:%.*]] = call <8 x i32 ()*> @llvm.masked.load.v8p0f_i32f.p0v8p0f_i32f(<8 x i32 ()*>* nonnull [[TMP15]], i32 8, <8 x i1> [[TMP9]], <8 x i32 ()*> undef)
 ; AVX512-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i32 ()*, i32 ()** [[TMP12]], i64 16
 ; AVX512-NEXT:    [[TMP17:%.*]] = bitcast i32 ()** [[TMP16]] to <8 x i32 ()*>*
-; AVX512-NEXT:    [[WIDE_MASKED_LOAD14:%.*]] = call <8 x i32 ()*> @llvm.masked.load.v8p0f_i32f.p0v8p0f_i32f(<8 x i32 ()*>* [[TMP17]], i32 8, <8 x i1> [[TMP10]], <8 x i32 ()*> undef)
+; AVX512-NEXT:    [[WIDE_MASKED_LOAD14:%.*]] = call <8 x i32 ()*> @llvm.masked.load.v8p0f_i32f.p0v8p0f_i32f(<8 x i32 ()*>* nonnull [[TMP17]], i32 8, <8 x i1> [[TMP10]], <8 x i32 ()*> undef)
 ; AVX512-NEXT:    [[TMP18:%.*]] = getelementptr inbounds i32 ()*, i32 ()** [[TMP12]], i64 24
 ; AVX512-NEXT:    [[TMP19:%.*]] = bitcast i32 ()** [[TMP18]] to <8 x i32 ()*>*
-; AVX512-NEXT:    [[WIDE_MASKED_LOAD15:%.*]] = call <8 x i32 ()*> @llvm.masked.load.v8p0f_i32f.p0v8p0f_i32f(<8 x i32 ()*>* [[TMP19]], i32 8, <8 x i1> [[TMP11]], <8 x i32 ()*> undef)
+; AVX512-NEXT:    [[WIDE_MASKED_LOAD15:%.*]] = call <8 x i32 ()*> @llvm.masked.load.v8p0f_i32f.p0v8p0f_i32f(<8 x i32 ()*>* nonnull [[TMP19]], i32 8, <8 x i1> [[TMP11]], <8 x i32 ()*> undef)
 ; AVX512-NEXT:    [[TMP20:%.*]] = icmp ne <8 x i32 ()*> [[WIDE_MASKED_LOAD]], zeroinitializer
 ; AVX512-NEXT:    [[TMP21:%.*]] = icmp ne <8 x i32 ()*> [[WIDE_MASKED_LOAD13]], zeroinitializer
 ; AVX512-NEXT:    [[TMP22:%.*]] = icmp ne <8 x i32 ()*> [[WIDE_MASKED_LOAD14]], zeroinitializer
diff --git a/test/Transforms/LoopVectorize/X86/parallel-loops-after-reg2mem.ll b/test/Transforms/LoopVectorize/X86/parallel-loops-after-reg2mem.ll
index 631361c9d47d8c815867121990ee94f6d38a6a65..0e6f06497e306e995cf85a2ddc6b3bc93a4f7a66 100644
--- a/test/Transforms/LoopVectorize/X86/parallel-loops-after-reg2mem.ll
+++ b/test/Transforms/LoopVectorize/X86/parallel-loops-after-reg2mem.ll
@@ -19,19 +19,19 @@ entry:
 for.body:                                         ; preds = %for.body.for.body_crit_edge, %entry
   %indvars.iv.reload = load i64, i64* %indvars.iv.reg2mem
   %arrayidx = getelementptr inbounds i32, i32* %b, i64 %indvars.iv.reload
-  %0 = load i32, i32* %arrayidx, align 4, !llvm.mem.parallel_loop_access !3
+  %0 = load i32, i32* %arrayidx, align 4, !llvm.access.group !4
   %arrayidx2 = getelementptr inbounds i32, i32* %a, i64 %indvars.iv.reload
-  %1 = load i32, i32* %arrayidx2, align 4, !llvm.mem.parallel_loop_access !3
+  %1 = load i32, i32* %arrayidx2, align 4, !llvm.access.group !4
   %idxprom3 = sext i32 %1 to i64
   %arrayidx4 = getelementptr inbounds i32, i32* %a, i64 %idxprom3
-  store i32 %0, i32* %arrayidx4, align 4, !llvm.mem.parallel_loop_access !3
+  store i32 %0, i32* %arrayidx4, align 4, !llvm.access.group !4
   %indvars.iv.next = add i64 %indvars.iv.reload, 1
   ; A new store without the parallel metadata here:
   store i64 %indvars.iv.next, i64* %indvars.iv.next.reg2mem
   %indvars.iv.next.reload1 = load i64, i64* %indvars.iv.next.reg2mem
   %arrayidx6 = getelementptr inbounds i32, i32* %b, i64 %indvars.iv.next.reload1
-  %2 = load i32, i32* %arrayidx6, align 4, !llvm.mem.parallel_loop_access !3
-  store i32 %2, i32* %arrayidx2, align 4, !llvm.mem.parallel_loop_access !3
+  %2 = load i32, i32* %arrayidx6, align 4, !llvm.access.group !4
+  store i32 %2, i32* %arrayidx2, align 4, !llvm.access.group !4
   %indvars.iv.next.reload = load i64, i64* %indvars.iv.next.reg2mem
   %lftr.wideiv = trunc i64 %indvars.iv.next.reload to i32
   %exitcond = icmp eq i32 %lftr.wideiv, 512
@@ -46,4 +46,5 @@ for.end:                                          ; preds = %for.body
   ret void
 }
 
-!3 = !{!3}
+!3 = !{!3, !{!"llvm.loop.parallel_accesses", !4}}
+!4 = distinct !{}
diff --git a/test/Transforms/LoopVectorize/X86/parallel-loops.ll b/test/Transforms/LoopVectorize/X86/parallel-loops.ll
index 53061edc35c5d9ac02bcd7bd80b08b4394fecf3b..be778850096a58f629d017d1397bb60cb7f54424 100644
--- a/test/Transforms/LoopVectorize/X86/parallel-loops.ll
+++ b/test/Transforms/LoopVectorize/X86/parallel-loops.ll
@@ -51,18 +51,18 @@ entry:
 for.body:                                         ; preds = %for.body, %entry
   %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
   %arrayidx = getelementptr inbounds i32, i32* %b, i64 %indvars.iv
-  %0 = load i32, i32* %arrayidx, align 4, !llvm.mem.parallel_loop_access !3
+  %0 = load i32, i32* %arrayidx, align 4, !llvm.access.group !13
   %arrayidx2 = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
-  %1 = load i32, i32* %arrayidx2, align 4, !llvm.mem.parallel_loop_access !3
+  %1 = load i32, i32* %arrayidx2, align 4, !llvm.access.group !13
   %idxprom3 = sext i32 %1 to i64
   %arrayidx4 = getelementptr inbounds i32, i32* %a, i64 %idxprom3
   ; This store might have originated from inlining a function with a parallel
   ; loop. Refers to a list with the "original loop reference" (!4) also included.
-  store i32 %0, i32* %arrayidx4, align 4, !llvm.mem.parallel_loop_access !5
+  store i32 %0, i32* %arrayidx4, align 4, !llvm.access.group !15
   %indvars.iv.next = add i64 %indvars.iv, 1
   %arrayidx6 = getelementptr inbounds i32, i32* %b, i64 %indvars.iv.next
-  %2 = load i32, i32* %arrayidx6, align 4, !llvm.mem.parallel_loop_access !3
-  store i32 %2, i32* %arrayidx2, align 4, !llvm.mem.parallel_loop_access !3
+  %2 = load i32, i32* %arrayidx6, align 4, !llvm.access.group !13
+  store i32 %2, i32* %arrayidx2, align 4, !llvm.access.group !13
   %lftr.wideiv = trunc i64 %indvars.iv.next to i32
   %exitcond = icmp eq i32 %lftr.wideiv, 512
   br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !3
@@ -84,18 +84,18 @@ entry:
 for.body:                                         ; preds = %for.body, %entry
   %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
   %arrayidx = getelementptr inbounds i32, i32* %b, i64 %indvars.iv
-  %0 = load i32, i32* %arrayidx, align 4, !llvm.mem.parallel_loop_access !6
+  %0 = load i32, i32* %arrayidx, align 4, !llvm.access.group !16
   %arrayidx2 = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
-  %1 = load i32, i32* %arrayidx2, align 4, !llvm.mem.parallel_loop_access !6
+  %1 = load i32, i32* %arrayidx2, align 4, !llvm.access.group !16
   %idxprom3 = sext i32 %1 to i64
   %arrayidx4 = getelementptr inbounds i32, i32* %a, i64 %idxprom3
   ; This refers to the loop marked with !7 which we are not in at the moment.
   ; It should prevent detecting as a parallel loop.
-  store i32 %0, i32* %arrayidx4, align 4, !llvm.mem.parallel_loop_access !7
+  store i32 %0, i32* %arrayidx4, align 4, !llvm.access.group !17
   %indvars.iv.next = add i64 %indvars.iv, 1
   %arrayidx6 = getelementptr inbounds i32, i32* %b, i64 %indvars.iv.next
-  %2 = load i32, i32* %arrayidx6, align 4, !llvm.mem.parallel_loop_access !6
-  store i32 %2, i32* %arrayidx2, align 4, !llvm.mem.parallel_loop_access !6
+  %2 = load i32, i32* %arrayidx6, align 4, !llvm.access.group !16
+  store i32 %2, i32* %arrayidx2, align 4, !llvm.access.group !16
   %lftr.wideiv = trunc i64 %indvars.iv.next to i32
   %exitcond = icmp eq i32 %lftr.wideiv, 512
   br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !6
@@ -104,8 +104,12 @@ for.end:                                          ; preds = %for.body
   ret void
 }
 
-!3 = !{!3}
-!4 = !{!4}
-!5 = !{!3, !4}
-!6 = !{!6}
-!7 = !{!7}
+!3 = !{!3, !{!"llvm.loop.parallel_accesses", !13, !15}}
+!4 = !{!4, !{!"llvm.loop.parallel_accesses", !14, !15}}
+!6 = !{!6, !{!"llvm.loop.parallel_accesses", !16}}
+!7 = !{!7, !{!"llvm.loop.parallel_accesses", !17}}
+!13 = distinct !{}
+!14 = distinct !{}
+!15 = distinct !{}
+!16 = distinct !{}
+!17 = distinct !{}
diff --git a/test/Transforms/LoopVectorize/X86/pr34438.ll b/test/Transforms/LoopVectorize/X86/pr34438.ll
index 6699985c0a0abc3c94c19c361e0a23978f16f7e2..efbfecc2f54cd397e036506ea50128e60928f71a 100644
--- a/test/Transforms/LoopVectorize/X86/pr34438.ll
+++ b/test/Transforms/LoopVectorize/X86/pr34438.ll
@@ -18,11 +18,11 @@ entry:
 for.body:
   %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
   %arrayidx = getelementptr inbounds float, float* %B, i64 %indvars.iv
-  %0 = load float, float* %arrayidx, align 4, !llvm.mem.parallel_loop_access !3
+  %0 = load float, float* %arrayidx, align 4, !llvm.access.group !5
   %arrayidx2 = getelementptr inbounds float, float* %A, i64 %indvars.iv
-  %1 = load float, float* %arrayidx2, align 4, !llvm.mem.parallel_loop_access !3
+  %1 = load float, float* %arrayidx2, align 4, !llvm.access.group !5
   %add = fadd fast float %0, %1
-  store float %add, float* %arrayidx2, align 4, !llvm.mem.parallel_loop_access !3
+  store float %add, float* %arrayidx2, align 4, !llvm.access.group !5
   %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
   %exitcond = icmp eq i64 %indvars.iv.next, 8
   br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !4
@@ -31,5 +31,6 @@ for.end:
   ret void
 }
 
-!3 = !{!3}
+!3 = !{!3, !{!"llvm.loop.parallel_accesses", !5}}
 !4 = !{!4}
+!5 = distinct !{}
diff --git a/test/Transforms/LoopVectorize/X86/vect.omp.force.ll b/test/Transforms/LoopVectorize/X86/vect.omp.force.ll
index c2a0fed1fe6f234bb2b41ea227303d57c93dcd70..90114672c8f531fbaed5548d4e506f1027179e39 100644
--- a/test/Transforms/LoopVectorize/X86/vect.omp.force.ll
+++ b/test/Transforms/LoopVectorize/X86/vect.omp.force.ll
@@ -32,10 +32,10 @@ entry:
 for.body:
   %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
   %arrayidx = getelementptr inbounds float, float* %B, i64 %indvars.iv
-  %0 = load float, float* %arrayidx, align 4, !llvm.mem.parallel_loop_access !1
+  %0 = load float, float* %arrayidx, align 4, !llvm.access.group !11
   %call = tail call float @llvm.sin.f32(float %0)
   %arrayidx2 = getelementptr inbounds float, float* %A, i64 %indvars.iv
-  store float %call, float* %arrayidx2, align 4, !llvm.mem.parallel_loop_access !1
+  store float %call, float* %arrayidx2, align 4, !llvm.access.group !11
   %indvars.iv.next = add nuw nsw i64 %indvars.iv, 2
   %lftr.wideiv = trunc i64 %indvars.iv.next to i32
   %exitcond = icmp eq i32 %lftr.wideiv, 1000
@@ -48,8 +48,9 @@ for.end:
   ret void
 }
 
-!1 = !{!1, !2}
+!1 = !{!1, !2, !{!"llvm.loop.parallel_accesses", !11}}
 !2 = !{!"llvm.loop.vectorize.enable", i1 true}
+!11 = distinct !{}
 
 ;
 ; This method will not be vectorized, as scalar cost is lower than any of vector costs.
@@ -62,10 +63,10 @@ entry:
 for.body:
   %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
   %arrayidx = getelementptr inbounds float, float* %B, i64 %indvars.iv
-  %0 = load float, float* %arrayidx, align 4, !llvm.mem.parallel_loop_access !3
+  %0 = load float, float* %arrayidx, align 4, !llvm.access.group !13
   %call = tail call float @llvm.sin.f32(float %0)
   %arrayidx2 = getelementptr inbounds float, float* %A, i64 %indvars.iv
-  store float %call, float* %arrayidx2, align 4, !llvm.mem.parallel_loop_access !3
+  store float %call, float* %arrayidx2, align 4, !llvm.access.group !13
   %indvars.iv.next = add nuw nsw i64 %indvars.iv, 2
   %lftr.wideiv = trunc i64 %indvars.iv.next to i32
   %exitcond = icmp eq i32 %lftr.wideiv, 1000
@@ -81,5 +82,6 @@ for.end:
 declare float @llvm.sin.f32(float) nounwind readnone
 
 ; Dummy metadata
-!3 = !{!3}
+!3 = !{!3, !{!"llvm.loop.parallel_accesses", !13}}
+!13 = distinct !{}
 
diff --git a/test/Transforms/LoopVectorize/X86/vect.omp.force.small-tc.ll b/test/Transforms/LoopVectorize/X86/vect.omp.force.small-tc.ll
index 81f3113bf22fe47b8a6d1759f731e25e6b74890b..7c249a1b4228b2bcfb33aa988ecce6702160120d 100644
--- a/test/Transforms/LoopVectorize/X86/vect.omp.force.small-tc.ll
+++ b/test/Transforms/LoopVectorize/X86/vect.omp.force.small-tc.ll
@@ -41,7 +41,7 @@ define void @vectorized(float* noalias nocapture %A, float* noalias nocapture re
 ; CHECK-NEXT:    store <8 x float> [[TMP7]], <8 x float>* [[TMP8]], align 4
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 8
 ; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], 16
-; CHECK-NEXT:    br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop !0
+; CHECK-NEXT:    br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop !1
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 20, 16
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
@@ -51,11 +51,11 @@ define void @vectorized(float* noalias nocapture %A, float* noalias nocapture re
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, float* [[B]], i64 [[INDVARS_IV]]
-; CHECK-NEXT:    [[TMP10:%.*]] = load float, float* [[ARRAYIDX]], align 4, !llvm.mem.parallel_loop_access !3
+; CHECK-NEXT:    [[TMP10:%.*]] = load float, float* [[ARRAYIDX]], align 4, !llvm.access.group !0
 ; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds float, float* [[A]], i64 [[INDVARS_IV]]
-; CHECK-NEXT:    [[TMP11:%.*]] = load float, float* [[ARRAYIDX2]], align 4, !llvm.mem.parallel_loop_access !3
+; CHECK-NEXT:    [[TMP11:%.*]] = load float, float* [[ARRAYIDX2]], align 4, !llvm.access.group !0
 ; CHECK-NEXT:    [[ADD:%.*]] = fadd fast float [[TMP10]], [[TMP11]]
-; CHECK-NEXT:    store float [[ADD]], float* [[ARRAYIDX2]], align 4, !llvm.mem.parallel_loop_access !3
+; CHECK-NEXT:    store float [[ADD]], float* [[ARRAYIDX2]], align 4, !llvm.access.group !0
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 20
 ; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop !4
@@ -68,11 +68,11 @@ entry:
 for.body:
   %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
   %arrayidx = getelementptr inbounds float, float* %B, i64 %indvars.iv
-  %0 = load float, float* %arrayidx, align 4, !llvm.mem.parallel_loop_access !1
+  %0 = load float, float* %arrayidx, align 4, !llvm.access.group !11
   %arrayidx2 = getelementptr inbounds float, float* %A, i64 %indvars.iv
-  %1 = load float, float* %arrayidx2, align 4, !llvm.mem.parallel_loop_access !1
+  %1 = load float, float* %arrayidx2, align 4, !llvm.access.group !11
   %add = fadd fast float %0, %1
-  store float %add, float* %arrayidx2, align 4, !llvm.mem.parallel_loop_access !1
+  store float %add, float* %arrayidx2, align 4, !llvm.access.group !11
   %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
   %exitcond = icmp eq i64 %indvars.iv.next, 20
   br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !1
@@ -81,8 +81,9 @@ for.end:
   ret void
 }
 
-!1 = !{!1, !2}
+!1 = !{!1, !2, !{!"llvm.loop.parallel_accesses", !11}}
 !2 = !{!"llvm.loop.vectorize.enable", i1 true}
+!11 = distinct !{}
 
 ;
 ; This loop will be vectorized as the trip count is below the threshold but no
@@ -114,7 +115,7 @@ define void @vectorized1(float* noalias nocapture %A, float* noalias nocapture r
 ; CHECK-NEXT:    call void @llvm.masked.store.v8f32.p0v8f32(<8 x float> [[TMP7]], <8 x float>* [[TMP9]], i32 4, <8 x i1> [[TMP8]])
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 8
 ; CHECK-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], 24
-; CHECK-NEXT:    br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop !6
+; CHECK-NEXT:    br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop !7
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br i1 true, label [[FOR_END:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
@@ -127,11 +128,11 @@ entry:
 for.body:
   %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
   %arrayidx = getelementptr inbounds float, float* %B, i64 %indvars.iv
-  %0 = load float, float* %arrayidx, align 4, !llvm.mem.parallel_loop_access !3
+  %0 = load float, float* %arrayidx, align 4, !llvm.access.group !13
   %arrayidx2 = getelementptr inbounds float, float* %A, i64 %indvars.iv
-  %1 = load float, float* %arrayidx2, align 4, !llvm.mem.parallel_loop_access !3
+  %1 = load float, float* %arrayidx2, align 4, !llvm.access.group !13
   %add = fadd fast float %0, %1
-  store float %add, float* %arrayidx2, align 4, !llvm.mem.parallel_loop_access !3
+  store float %add, float* %arrayidx2, align 4, !llvm.access.group !13
   %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
   %exitcond = icmp eq i64 %indvars.iv.next, 20
   br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !3
@@ -140,7 +141,8 @@ for.end:
   ret void
 }
 
-!3 = !{!3}
+!3 = !{!3, !{!"llvm.loop.parallel_accesses", !13}}
+!13 = distinct !{}
 
 ;
 ; This loop will be vectorized as the trip count is below the threshold but no
@@ -171,7 +173,7 @@ define void @vectorized2(float* noalias nocapture %A, float* noalias nocapture r
 ; CHECK-NEXT:    store <8 x float> [[TMP7]], <8 x float>* [[TMP8]], align 4
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 8
 ; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], 16
-; CHECK-NEXT:    br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop !9
+; CHECK-NEXT:    br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop !10
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 16, 16
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
@@ -181,14 +183,14 @@ define void @vectorized2(float* noalias nocapture %A, float* noalias nocapture r
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, float* [[B]], i64 [[INDVARS_IV]]
-; CHECK-NEXT:    [[TMP10:%.*]] = load float, float* [[ARRAYIDX]], align 4, !llvm.mem.parallel_loop_access !7
+; CHECK-NEXT:    [[TMP10:%.*]] = load float, float* [[ARRAYIDX]], align 4, !llvm.access.group !6
 ; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds float, float* [[A]], i64 [[INDVARS_IV]]
-; CHECK-NEXT:    [[TMP11:%.*]] = load float, float* [[ARRAYIDX2]], align 4, !llvm.mem.parallel_loop_access !7
+; CHECK-NEXT:    [[TMP11:%.*]] = load float, float* [[ARRAYIDX2]], align 4, !llvm.access.group !6
 ; CHECK-NEXT:    [[ADD:%.*]] = fadd fast float [[TMP10]], [[TMP11]]
-; CHECK-NEXT:    store float [[ADD]], float* [[ARRAYIDX2]], align 4, !llvm.mem.parallel_loop_access !7
+; CHECK-NEXT:    store float [[ADD]], float* [[ARRAYIDX2]], align 4, !llvm.access.group !6
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 16
-; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop !10
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop !11
 ; CHECK:       for.end:
 ; CHECK-NEXT:    ret void
 ;
@@ -198,11 +200,11 @@ entry:
 for.body:
   %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
   %arrayidx = getelementptr inbounds float, float* %B, i64 %indvars.iv
-  %0 = load float, float* %arrayidx, align 4, !llvm.mem.parallel_loop_access !3
+  %0 = load float, float* %arrayidx, align 4, !llvm.access.group !13
   %arrayidx2 = getelementptr inbounds float, float* %A, i64 %indvars.iv
-  %1 = load float, float* %arrayidx2, align 4, !llvm.mem.parallel_loop_access !3
+  %1 = load float, float* %arrayidx2, align 4, !llvm.access.group !13
   %add = fadd fast float %0, %1
-  store float %add, float* %arrayidx2, align 4, !llvm.mem.parallel_loop_access !3
+  store float %add, float* %arrayidx2, align 4, !llvm.access.group !13
   %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
   %exitcond = icmp eq i64 %indvars.iv.next, 16
   br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !4
diff --git a/test/Transforms/LoopVectorize/X86/vector-scalar-select-cost.ll b/test/Transforms/LoopVectorize/X86/vector-scalar-select-cost.ll
index 5efabe16d32b619cb16c98cd7dce32b968ffec0a..5b3f5c58c79050b61324f7a252b166b324264fc5 100644
--- a/test/Transforms/LoopVectorize/X86/vector-scalar-select-cost.ll
+++ b/test/Transforms/LoopVectorize/X86/vector-scalar-select-cost.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s  -loop-vectorize -mcpu=core2 -debug-only=loop-vectorize 2>&1 -S | FileCheck %s
+; RUN: opt < %s  -loop-vectorize -mattr=+sse4.2 -debug-only=loop-vectorize 2>&1 -S | FileCheck %s
 ; REQUIRES: asserts
 ; Make sure we use the right select kind when querying select costs.
 
diff --git a/test/Transforms/LoopVectorize/X86/vector_max_bandwidth.ll b/test/Transforms/LoopVectorize/X86/vector_max_bandwidth.ll
index 34c6d745df583758fb6c914c7748899c8e42203e..2d2082eec3aa7123a005dc5ef386f6d66a06881c 100644
--- a/test/Transforms/LoopVectorize/X86/vector_max_bandwidth.ll
+++ b/test/Transforms/LoopVectorize/X86/vector_max_bandwidth.ll
@@ -58,11 +58,11 @@ entry:
 for.body:
   %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
   %arrayidx = getelementptr inbounds i8, i8* %B, i64 %indvars.iv
-  %l1 = load i8, i8* %arrayidx, align 4, !llvm.mem.parallel_loop_access !3
+  %l1 = load i8, i8* %arrayidx, align 4, !llvm.access.group !13
   %arrayidx2 = getelementptr inbounds i8, i8* %A, i64 %indvars.iv
-  %l2 = load i8, i8* %arrayidx2, align 4, !llvm.mem.parallel_loop_access !3
+  %l2 = load i8, i8* %arrayidx2, align 4, !llvm.access.group !13
   %add = add i8 %l1, %l2
-  store i8 %add, i8* %arrayidx2, align 4, !llvm.mem.parallel_loop_access !3
+  store i8 %add, i8* %arrayidx2, align 4, !llvm.access.group !13
   %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
   %exitcond = icmp eq i64 %indvars.iv.next, 16
   br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !4
@@ -70,5 +70,6 @@ for.body:
 for.end:
   ret void
 }
-!3 = !{!3}
+!3 = !{!3, !{!"llvm.loop.parallel_accesses", !13}}
 !4 = !{!4}
+!13 = distinct !{}
diff --git a/test/Transforms/LoopVectorize/libcall-remark.ll b/test/Transforms/LoopVectorize/libcall-remark.ll
new file mode 100644
index 0000000000000000000000000000000000000000..a1a7e461c705ff67866198ec16456a628c4f93f1
--- /dev/null
+++ b/test/Transforms/LoopVectorize/libcall-remark.ll
@@ -0,0 +1,52 @@
+; RUN: opt -S -loop-vectorize < %s 2>&1 -pass-remarks-analysis=.* | FileCheck %s
+
+; Test the optimization remark emitter for recognition 
+; of a mathlib function vs. an arbitrary function.
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.14.0"
+@data = external local_unnamed_addr global [32768 x float], align 16
+
+; CHECK: loop not vectorized: library call cannot be vectorized
+
+define void @libcall_blocks_vectorization() {
+entry:
+  br label %for.body
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds [32768 x float], [32768 x float]* @data, i64 0, i64 %indvars.iv
+  %t0 = load float, float* %arrayidx, align 4
+  %sqrtf = tail call float @sqrtf(float %t0)
+  store float %sqrtf, float* %arrayidx, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 32768
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+}
+
+; CHECK: loop not vectorized: call instruction cannot be vectorized
+
+define void @arbitrary_call_blocks_vectorization() {
+entry:
+  br label %for.body
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds [32768 x float], [32768 x float]* @data, i64 0, i64 %indvars.iv
+  %t0 = load float, float* %arrayidx, align 4
+  %sqrtf = tail call float @arbitrary(float %t0)
+  store float %sqrtf, float* %arrayidx, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 32768
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+}
+
+declare float @sqrtf(float)
+declare float @arbitrary(float)
+
diff --git a/test/Transforms/LoopVectorize/no-interleave-up-front.ll b/test/Transforms/LoopVectorize/no-interleave-up-front.ll
new file mode 100644
index 0000000000000000000000000000000000000000..a0c53450482289a4bb8edc5465aead455da06934
--- /dev/null
+++ b/test/Transforms/LoopVectorize/no-interleave-up-front.ll
@@ -0,0 +1,35 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -loop-vectorize -S | FileCheck %s
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+
+; Test case based on reproducer for
+; http://bugs.chromium.org/p/oss-fuzz/issues/detail?id=6477
+
+define void @test1(i32 %n) #0 {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    br i1 false, label [[DOTLR_PH_PREHEADER:%.*]], label [[DOT_CRIT_EDGE:%.*]]
+; CHECK:       .lr.ph.preheader:
+; CHECK-NEXT:    br label [[DOTLR_PH:%.*]]
+; CHECK:       .lr.ph:
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[DOTLR_PH]] ], [ 0, [[DOTLR_PH_PREHEADER]] ]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add i64 [[INDVARS_IV]], 1
+; CHECK-NEXT:    br i1 true, label [[DOT_CRIT_EDGE_LOOPEXIT:%.*]], label [[DOTLR_PH]], !llvm.loop !0
+; CHECK:       ._crit_edge.loopexit:
+; CHECK-NEXT:    br label [[DOT_CRIT_EDGE]]
+; CHECK:       ._crit_edge:
+; CHECK-NEXT:    ret void
+;
+  br i1 false, label %.lr.ph, label %._crit_edge
+
+.lr.ph:                                           ; preds = %.lr.ph, %0
+  %indvars.iv = phi i64 [ %indvars.iv.next, %.lr.ph ], [ 0, %0 ]
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  br i1 true, label %._crit_edge, label %.lr.ph, !llvm.loop !0
+
+._crit_edge:                                      ; preds = %.lr.ph, %0
+  ret void
+}
+
+!0 = distinct !{!0, !1}
+!1 = !{!"llvm.loop.interleave.count", i32 2}
diff --git a/test/Transforms/LoopVectorize/no_switch_disable_vectorization.ll b/test/Transforms/LoopVectorize/no_switch_disable_vectorization.ll
new file mode 100644
index 0000000000000000000000000000000000000000..424ef3846224a32fe75774dee3c6471822ce05a9
--- /dev/null
+++ b/test/Transforms/LoopVectorize/no_switch_disable_vectorization.ll
@@ -0,0 +1,95 @@
+; RUN: opt < %s -loop-vectorize -force-vector-width=4 -transform-warning -S 2>&1 | FileCheck %s
+; RUN: opt < %s -loop-vectorize -force-vector-width=1 -transform-warning -S 2>&1 | FileCheck %s -check-prefix=NOANALYSIS
+; RUN: opt < %s -loop-vectorize -force-vector-width=4 -transform-warning -pass-remarks-missed='loop-vectorize' -S 2>&1 | FileCheck %s -check-prefix=MOREINFO
+
+; This test is a copy of no_switch.ll, with the "llvm.loop.vectorize.enable" metadata set to false.
+; It tests that vectorization is explicitly disabled and no warnings are emitted.
+
+; CHECK-NOT: remark: source.cpp:4:5: loop not vectorized: loop contains a switch statement
+; CHECK-NOT: warning: source.cpp:4:5: loop not vectorized: the optimizer was unable to perform the requested transformation; the transformation might be disabled or specified as part of an unsupported transformation ordering
+
+; NOANALYSIS-NOT: remark: {{.*}}
+; NOANALYSIS-NOT: warning: source.cpp:4:5: loop not vectorized: the optimizer was unable to perform the requested transformation; the transformation might be disabled or specified as part of an unsupported transformation ordering
+
+; MOREINFO: remark: source.cpp:4:5: loop not vectorized: vectorization is explicitly disabled
+; MOREINFO-NOT: warning: source.cpp:4:5: loop not vectorized: the optimizer was unable to perform the requested transformation; the transformation might be disabled or specified as part of an unsupported transformation ordering
+
+; CHECK: _Z11test_switchPii
+; CHECK-NOT: x i32>
+; CHECK: ret
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+
+; Function Attrs: nounwind optsize ssp uwtable
+define void @_Z11test_switchPii(i32* nocapture %A, i32 %Length) #0 !dbg !4 {
+entry:
+  %cmp18 = icmp sgt i32 %Length, 0, !dbg !10
+  br i1 %cmp18, label %for.body.preheader, label %for.end, !dbg !10, !llvm.loop !12
+
+for.body.preheader:                               ; preds = %entry
+  br label %for.body, !dbg !14
+
+for.body:                                         ; preds = %for.body.preheader, %for.inc
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.inc ], [ 0, %for.body.preheader ]
+  %arrayidx = getelementptr inbounds i32, i32* %A, i64 %indvars.iv, !dbg !14
+  %0 = load i32, i32* %arrayidx, align 4, !dbg !14, !tbaa !16
+  switch i32 %0, label %for.inc [
+    i32 0, label %sw.bb
+    i32 1, label %sw.bb3
+  ], !dbg !14
+
+sw.bb:                                            ; preds = %for.body
+  %1 = trunc i64 %indvars.iv to i32, !dbg !20
+  %mul = shl nsw i32 %1, 1, !dbg !20
+  br label %for.inc, !dbg !22
+
+sw.bb3:                                           ; preds = %for.body
+  %2 = trunc i64 %indvars.iv to i32, !dbg !23
+  store i32 %2, i32* %arrayidx, align 4, !dbg !23, !tbaa !16
+  br label %for.inc, !dbg !23
+
+for.inc:                                          ; preds = %sw.bb3, %for.body, %sw.bb
+  %storemerge = phi i32 [ %mul, %sw.bb ], [ 0, %for.body ], [ 0, %sw.bb3 ]
+  store i32 %storemerge, i32* %arrayidx, align 4, !dbg !20, !tbaa !16
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1, !dbg !10
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32, !dbg !10
+  %exitcond = icmp eq i32 %lftr.wideiv, %Length, !dbg !10
+  br i1 %exitcond, label %for.end.loopexit, label %for.body, !dbg !10, !llvm.loop !12
+
+for.end.loopexit:                                 ; preds = %for.inc
+  br label %for.end
+
+for.end:                                          ; preds = %for.end.loopexit, %entry
+  ret void, !dbg !24
+}
+
+attributes #0 = { nounwind }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!7, !8}
+!llvm.ident = !{!9}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5.0", isOptimized: true, runtimeVersion: 6, emissionKind: LineTablesOnly, file: !1, enums: !2, retainedTypes: !2, globals: !2, imports: !2)
+!1 = !DIFile(filename: "source.cpp", directory: ".")
+!2 = !{}
+!4 = distinct !DISubprogram(name: "test_switch", line: 1, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: true, unit: !0, scopeLine: 1, file: !1, scope: !5, type: !6, retainedNodes: !2)
+!5 = !DIFile(filename: "source.cpp", directory: ".")
+!6 = !DISubroutineType(types: !2)
+!7 = !{i32 2, !"Dwarf Version", i32 2}
+!8 = !{i32 2, !"Debug Info Version", i32 3}
+!9 = !{!"clang version 3.5.0"}
+!10 = !DILocation(line: 3, column: 8, scope: !11)
+!11 = distinct !DILexicalBlock(line: 3, column: 3, file: !1, scope: !4)
+!12 = !{!12, !13, !13}
+!13 = !{!"llvm.loop.vectorize.enable", i1 false}
+!14 = !DILocation(line: 4, column: 5, scope: !15)
+!15 = distinct !DILexicalBlock(line: 3, column: 36, file: !1, scope: !11)
+!16 = !{!17, !17, i64 0}
+!17 = !{!"int", !18, i64 0}
+!18 = !{!"omnipotent char", !19, i64 0}
+!19 = !{!"Simple C/C++ TBAA"}
+!20 = !DILocation(line: 6, column: 7, scope: !21)
+!21 = distinct !DILexicalBlock(line: 4, column: 18, file: !1, scope: !15)
+!22 = !DILocation(line: 7, column: 5, scope: !21)
+!23 = !DILocation(line: 9, column: 7, scope: !21)
+!24 = !DILocation(line: 14, column: 1, scope: !4)
diff --git a/test/Transforms/LoopVectorize/remove_metadata.ll b/test/Transforms/LoopVectorize/remove_metadata.ll
new file mode 100644
index 0000000000000000000000000000000000000000..793c134dd7dc8f793131aede8ab094fae2528a95
--- /dev/null
+++ b/test/Transforms/LoopVectorize/remove_metadata.ll
@@ -0,0 +1,32 @@
+; RUN: opt -loop-vectorize -force-vector-interleave=1 -force-vector-width=2 -S < %s | FileCheck %s
+;
+; Check that llvm.loop.vectorize.* metadata is removed after vectorization.
+;
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+; CHECK-LABEL: @disable_nonforced_enable(
+; CHECK: store <2 x i32>
+define void @disable_nonforced_enable(i32* nocapture %a, i32 %n) {
+entry:
+  %cmp4 = icmp sgt i32 %n, 0
+  br i1 %cmp4, label %for.body, label %for.end
+
+for.body:
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 %indvars.iv
+  %0 = trunc i64 %indvars.iv to i32
+  store i32 %0, i32* %arrayidx, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  %lftr.wideiv = trunc i64 %indvars.iv.next to i32
+  %exitcond = icmp eq i32 %lftr.wideiv, %n
+  br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !0
+
+for.end:
+  ret void
+}
+
+!0 = !{!0, !{!"llvm.loop.vectorize.some_property"}, !{!"llvm.loop.vectorize.enable", i32 1}}
+
+; CHECK-NOT: llvm.loop.vectorize.
+; CHECK: {!"llvm.loop.isvectorized", i32 1}
+; CHECK-NOT: llvm.loop.vectorize.
diff --git a/test/Transforms/LoopVectorize/runtime-check.ll b/test/Transforms/LoopVectorize/runtime-check.ll
index 19bfa50f769b4299e0faf5846551ddff4b5b4109..332e3efb65d30fc3ebfc8d7ea8335cf6576ff28e 100644
--- a/test/Transforms/LoopVectorize/runtime-check.ll
+++ b/test/Transforms/LoopVectorize/runtime-check.ll
@@ -117,7 +117,9 @@ loopexit:
   ret void
 }
 
-; Check we do generate unnecessary runtime checks. They will always fail.
+; Check we do not generate runtime checks if we found a known dependence preventing
+; vectorization. In this case, it is a read of c[i-1] followed by a write of c[i].
+; The runtime checks would always fail.
 
 ; void test_runtime_check2(float *a, float b, unsigned offset, unsigned offset2, unsigned n, float *c) {
 ;   for (unsigned i = 1; i < n; i++) {
@@ -127,7 +129,7 @@ loopexit:
 ; }
 ;
 ; CHECK-LABEL: test_runtime_check2
-; CHECK:      <4 x float>
+; CHECK-NOT:      <4 x float>
 define void @test_runtime_check2(float* %a, float %b, i64 %offset, i64 %offset2, i64 %n, float* %c) {
 entry:
   br label %for.body
diff --git a/test/Transforms/LowerWidenableCondition/basic.ll b/test/Transforms/LowerWidenableCondition/basic.ll
new file mode 100644
index 0000000000000000000000000000000000000000..9779efc8846ed9db4399dfe966d7ccbc16476b81
--- /dev/null
+++ b/test/Transforms/LowerWidenableCondition/basic.ll
@@ -0,0 +1,44 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -lower-widenable-condition < %s        | FileCheck %s
+; RUN: opt -S -passes=lower-widenable-condition < %s | FileCheck %s
+
+; Basic test case: make sure that all widenable conditions turn into i1 true.
+define void @f_0(i1 %cond_0, i1 %cond_1) {
+; CHECK-LABEL: @f_0(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND:%.*]] = and i1 [[COND_0:%.*]], true
+; CHECK-NEXT:    br i1 [[EXIPLICIT_GUARD_COND]], label [[GUARDED:%.*]], label [[DEOPT:%.*]]
+; CHECK:       deopt:
+; CHECK-NEXT:    unreachable
+; CHECK:       guarded:
+; CHECK-NEXT:    [[EXIPLICIT_GUARD_COND4:%.*]] = and i1 [[COND_1:%.*]], true
+; CHECK-NEXT:    br i1 [[EXIPLICIT_GUARD_COND4]], label [[GUARDED1:%.*]], label [[DEOPT2:%.*]]
+; CHECK:       deopt2:
+; CHECK-NEXT:    unreachable
+; CHECK:       guarded1:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %widenable_cond = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond = and i1 %cond_0, %widenable_cond
+  br i1 %exiplicit_guard_cond, label %guarded, label %deopt
+
+deopt:                                            ; preds = %entry
+  unreachable
+
+guarded:                                          ; preds = %entry
+  %widenable_cond3 = call i1 @llvm.experimental.widenable.condition()
+  %exiplicit_guard_cond4 = and i1 %cond_1, %widenable_cond3
+  br i1 %exiplicit_guard_cond4, label %guarded1, label %deopt2
+
+deopt2:                                           ; preds = %guarded
+  unreachable
+
+guarded1:                                         ; preds = %guarded
+  ret void
+}
+
+; Function Attrs: inaccessiblememonly nounwind
+declare i1 @llvm.experimental.widenable.condition() #0
+
+attributes #0 = { inaccessiblememonly nounwind }
diff --git a/test/Transforms/MemCpyOpt/memcpy-to-memset-with-lifetimes.ll b/test/Transforms/MemCpyOpt/memcpy-to-memset-with-lifetimes.ll
index 1b8e8bc2828d89cabca7c88eeb1b8eb99ed99415..73d567a00ae2d761df45656c77108d3d9a6ac2b2 100644
--- a/test/Transforms/MemCpyOpt/memcpy-to-memset-with-lifetimes.ll
+++ b/test/Transforms/MemCpyOpt/memcpy-to-memset-with-lifetimes.ll
@@ -16,7 +16,7 @@ entry-block:
 
 ; CHECK-LABEL: @foo(
 ; CHECK:         %[[sret_cast:[^=]+]] = bitcast [8 x i64]* %sret to i8*
-; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* align 8 %[[sret_cast]], i8 0, i64 64
+; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* nonnull align 8 %[[sret_cast]], i8 0, i64 64
 ; CHECK-NOT: call void @llvm.memcpy
 ; CHECK: ret void
 }
@@ -38,12 +38,12 @@ entry-block:
 ; CHECK-LABEL: @bar(
 ; CHECK:         %[[a:[^=]+]] = alloca [8 x i64]
 ; CHECK:         %[[a_cast:[^=]+]] = bitcast [8 x i64]* %[[a]] to i8*
-; CHECK:         call void @llvm.memset.p0i8.i64(i8* align 8 %[[a_cast]], i8 0, i64 64
+; CHECK:         call void @llvm.memset.p0i8.i64(i8* nonnull align 8 %[[a_cast]], i8 0, i64 64
 ; CHECK:         %[[sret_cast:[^=]+]] = bitcast [8 x i64]* %sret to i8*
-; CHECK:         call void @llvm.memset.p0i8.i64(i8* align 8 %[[sret_cast]], i8 0, i64 64
-; CHECK:         call void @llvm.memset.p0i8.i64(i8* align 8 %[[a_cast]], i8 42, i64 32
+; CHECK:         call void @llvm.memset.p0i8.i64(i8* nonnull align 8 %[[sret_cast]], i8 0, i64 64
+; CHECK:         call void @llvm.memset.p0i8.i64(i8* nonnull align 8 %[[a_cast]], i8 42, i64 32
 ; CHECK:         %[[out_cast:[^=]+]] = bitcast [8 x i64]* %out to i8*
-; CHECK:         call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 8 %[[out_cast]], i8* align 8 %[[a_cast]], i64 64
+; CHECK:         call void @llvm.memcpy.p0i8.p0i8.i64(i8* nonnull align 8 %[[out_cast]], i8* nonnull align 8 %[[a_cast]], i64 64
 ; CHECK-NOT: call void @llvm.memcpy
 ; CHECK: ret void
 }
diff --git a/test/Transforms/MemCpyOpt/stackrestore.ll b/test/Transforms/MemCpyOpt/stackrestore.ll
new file mode 100644
index 0000000000000000000000000000000000000000..4bead3381ccdadfd34e1f4e44106aacc292a0195
--- /dev/null
+++ b/test/Transforms/MemCpyOpt/stackrestore.ll
@@ -0,0 +1,74 @@
+; RUN: opt -S -memcpyopt < %s | FileCheck %s
+
+; PR40118: BasicAA didn't realize that stackrestore ends the lifetime of
+; unescaped dynamic allocas, such as those that might come from inalloca.
+
+source_filename = "t.cpp"
+target datalayout = "e-m:x-p:32:32-i64:64-f80:32-n8:16:32-a:0:32-S32"
+target triple = "i686-unknown-windows-msvc19.14.26433"
+
+@str = internal constant [9 x i8] c"abcdxxxxx"
+
+
+; Test that we can propagate memcpy through an unescaped dynamic alloca across
+; a call to @external.
+
+define i32 @test_norestore(i32 %n) {
+  %tmpmem = alloca [10 x i8], align 4
+  %tmp = getelementptr inbounds [10 x i8], [10 x i8]* %tmpmem, i32 0, i32 0
+
+  ; Make a dynamic alloca, initialize it.
+  %p = alloca i8, i32 %n, align 4
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* %p, i8* align 1 getelementptr inbounds ([9 x i8], [9 x i8]* @str, i32 0, i32 0), i32 9, i1 false)
+
+  ; This extra byte exists to prevent memcpyopt from propagating @str.
+  %p10 = getelementptr inbounds i8, i8* %p, i32 9
+  store i8 0, i8* %p10
+
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* %tmp, i8* %p, i32 10, i1 false)
+  call void @external()
+  %heap = call i8* @malloc(i32 9)
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* %heap, i8* %tmp, i32 9, i1 false)
+  call void @useit(i8* %heap)
+  ret i32 0
+}
+
+; CHECK-LABEL: define i32 @test_norestore(i32 %n)
+; CHECK: call void @llvm.memcpy.p0i8.p0i8.i32(i8* %p, i8* align 1 getelementptr inbounds ([9 x i8], [9 x i8]* @str, i32 0, i32 0), i32 9, i1 false)
+; CHECK: call void @llvm.memcpy.p0i8.p0i8.i32(i8* %tmp, i8* %p, i32 10, i1 false)
+; CHECK: call void @llvm.memcpy.p0i8.p0i8.i32(i8* %heap, i8* %p, i32 9, i1 false)
+
+
+; Do not propagate memcpy from %p across the stackrestore.
+
+define i32 @test_stackrestore() {
+  %tmpmem = alloca [10 x i8], align 4
+  %tmp = getelementptr inbounds [10 x i8], [10 x i8]* %tmpmem, i32 0, i32 0
+  %inalloca.save = tail call i8* @llvm.stacksave()
+  %argmem = alloca inalloca [10 x i8], align 4
+  %p = getelementptr inbounds [10 x i8], [10 x i8]* %argmem, i32 0, i32 0
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* %p, i8* align 1 getelementptr inbounds ([9 x i8], [9 x i8]* @str, i32 0, i32 0), i32 9, i1 false)
+
+  ; This extra byte exists to prevent memcpyopt from propagating @str.
+  %p10 = getelementptr inbounds [10 x i8], [10 x i8]* %argmem, i32 0, i32 9
+  store i8 0, i8* %p10
+
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* %tmp, i8* %p, i32 10, i1 false)
+  call void @llvm.stackrestore(i8* %inalloca.save)
+  %heap = call i8* @malloc(i32 9)
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* %heap, i8* %tmp, i32 9, i1 false)
+  call void @useit(i8* %heap)
+  ret i32 0
+}
+
+; CHECK-LABEL: define i32 @test_stackrestore()
+; CHECK: call void @llvm.memcpy.p0i8.p0i8.i32(i8* %p, i8* align 1 getelementptr inbounds ([9 x i8], [9 x i8]* @str, i32 0, i32 0), i32 9, i1 false)
+; CHECK: call void @llvm.memcpy.p0i8.p0i8.i32(i8* %tmp, i8* %p, i32 10, i1 false)
+; CHECK: call void @llvm.memcpy.p0i8.p0i8.i32(i8* %heap, i8* %tmp, i32 9, i1 false)
+
+declare void @llvm.memcpy.p0i8.p0i8.i32(i8* nocapture writeonly, i8* nocapture readonly, i32, i1)
+declare i8* @llvm.stacksave()
+declare void @llvm.stackrestore(i8*)
+declare i8* @malloc(i32)
+declare void @useit(i8*)
+declare void @external()
diff --git a/test/Transforms/MergeFunc/call-and-invoke-with-ranges.ll b/test/Transforms/MergeFunc/call-and-invoke-with-ranges.ll
index f138ac429144ebd948ccbc9527a39ea14b9af3de..4b8d6bb53e41c1b4ae12306a594e8e9fa73bebe6 100644
--- a/test/Transforms/MergeFunc/call-and-invoke-with-ranges.ll
+++ b/test/Transforms/MergeFunc/call-and-invoke-with-ranges.ll
@@ -63,14 +63,6 @@ lpad:
   resume { i8*, i32 } zeroinitializer
 }
 
-define i8 @call_with_same_range() {
-; CHECK-LABEL: @call_with_same_range
-; CHECK: tail call i8 @call_with_range
-  bitcast i8 0 to i8
-  %out = call i8 @dummy(), !range !0
-  ret i8 %out
-}
-
 define i8 @invoke_with_same_range() personality i8* undef {
 ; CHECK-LABEL: @invoke_with_same_range()
 ; CHECK: tail call i8 @invoke_with_range()
@@ -84,6 +76,13 @@ lpad:
   resume { i8*, i32 } zeroinitializer
 }
 
+define i8 @call_with_same_range() {
+; CHECK-LABEL: @call_with_same_range
+; CHECK: tail call i8 @call_with_range
+  bitcast i8 0 to i8
+  %out = call i8 @dummy(), !range !0
+  ret i8 %out
+}
 
 
 declare i8 @dummy();
diff --git a/test/Transforms/MergeFunc/inline-asm.ll b/test/Transforms/MergeFunc/inline-asm.ll
index 370d3c56f060ebbcbad51fc3b9059b0ceaf0f2ab..15760242cf6920572096bbc0a7e1807ef9593edc 100644
--- a/test/Transforms/MergeFunc/inline-asm.ll
+++ b/test/Transforms/MergeFunc/inline-asm.ll
@@ -3,13 +3,13 @@
 ; CHECK-LABEL: @int_ptr_arg_different
 ; CHECK-NEXT: call void asm
 
+; CHECK-LABEL: @int_ptr_null
+; CHECK-NEXT: tail call void @float_ptr_null()
+
 ; CHECK-LABEL: @int_ptr_arg_same
 ; CHECK-NEXT: %2 = bitcast i32* %0 to float*
 ; CHECK-NEXT: tail call void @float_ptr_arg_same(float* %2)
 
-; CHECK-LABEL: @int_ptr_null
-; CHECK-NEXT: tail call void @float_ptr_null()
-
 ; Used to satisfy minimum size limit
 declare void @stuff()
 
diff --git a/test/Transforms/MergeFunc/linkonce_odr.ll b/test/Transforms/MergeFunc/linkonce_odr.ll
index 1ad0d727f83b8a7e6442201e722b17cec205de58..825f9054d7b7de53c3766ad1a751ddfaeaa5573f 100644
--- a/test/Transforms/MergeFunc/linkonce_odr.ll
+++ b/test/Transforms/MergeFunc/linkonce_odr.ll
@@ -1,4 +1,4 @@
-; RUN: opt -S -mergefunc < %s | FileCheck %s
+; RUN: opt -S -mergefunc < %s | FileCheck %s -implicit-check-not=funC
 
 ; Replacments should be totally ordered on the function name.
 ; If we don't do this we  can end up with one module defining a thunk for @funA
@@ -15,6 +15,13 @@
 ; CHECK-NEXT:    tail call i32 @funA(i32 %0, i32 %1)
 ; CHECK-NEXT:    ret
 
+define linkonce_odr i32 @funC(i32 %x, i32 %y) {
+  %sum = add i32 %x, %y
+  %sum2 = add i32 %x, %sum
+  %sum3 = add i32 %x, %sum2
+  ret i32 %sum3
+}
+
 define linkonce_odr i32 @funB(i32 %x, i32 %y) {
   %sum = add i32 %x, %y
   %sum2 = add i32 %x, %sum
@@ -28,3 +35,8 @@ define linkonce_odr i32 @funA(i32 %x, i32 %y) {
   %sum3 = add i32 %x, %sum2
   ret i32 %sum3
 }
+
+; This creates a use of @funB, preventing -mergefunc from deleting it.
+; @funC, however, can safely be deleted as it has no uses, and is discardable
+; if unused.
+@take_addr_of_funB = global i8* bitcast (i32 (i32, i32)* @funB to i8*)
diff --git a/test/Transforms/MergeFunc/tailcall.ll b/test/Transforms/MergeFunc/tailcall.ll
new file mode 100644
index 0000000000000000000000000000000000000000..8adf45ab2ff47a885210dbda671dcefe9a1261eb
--- /dev/null
+++ b/test/Transforms/MergeFunc/tailcall.ll
@@ -0,0 +1,21 @@
+; RUN: opt -mergefunc -S < %s | FileCheck %s
+
+declare void @dummy()
+
+; CHECK-LABEL: define{{.*}}@foo
+; CHECK: call {{.*}}@dummy
+; CHECK: musttail {{.*}}@dummy
+define void @foo() {
+  call void @dummy()
+  musttail call void @dummy()
+  ret void
+}
+
+; CHECK-LABEL: define{{.*}}@bar
+; CHECK: call {{.*}}@dummy
+; CHECK: call {{.*}}@dummy
+define void @bar() {
+  call void @dummy()
+  call void @dummy()
+  ret void
+}
diff --git a/test/Transforms/MergeFunc/va_arg.ll b/test/Transforms/MergeFunc/va_arg.ll
new file mode 100644
index 0000000000000000000000000000000000000000..1a48a6395ede71a12cf8d7e5e1e61aef6062659c
--- /dev/null
+++ b/test/Transforms/MergeFunc/va_arg.ll
@@ -0,0 +1,93 @@
+; RUN: opt -S -mergefunc < %s | FileCheck %s
+; RUN: opt -S -mergefunc -mergefunc-use-aliases < %s | FileCheck %s -check-prefix=ALIAS
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; ALIAS: @_Z9simple_vaPKcz = unnamed_addr alias void (i8*, ...), void (i8*, ...)* @_Z10simple_va2PKcz
+; ALIAS-NOT: @_Z9simple_vaPKcz
+
+%struct.__va_list_tag = type { i32, i32, i8*, i8* }
+
+; CHECK-LABEL: define {{.*}}@_Z9simple_vaPKcz
+; CHECK: call void @llvm.va_start
+; CHECK: call void @llvm.va_end
+define dso_local void @_Z9simple_vaPKcz(i8* nocapture readnone, ...) unnamed_addr {
+  %2 = alloca [1 x %struct.__va_list_tag], align 16
+  %3 = bitcast [1 x %struct.__va_list_tag]* %2 to i8*
+  call void @llvm.va_start(i8* nonnull %3)
+  %4 = getelementptr inbounds [1 x %struct.__va_list_tag], [1 x %struct.__va_list_tag]* %2, i64 0, i64 0, i32 0
+  %5 = load i32, i32* %4, align 16
+  %6 = icmp ult i32 %5, 41
+  br i1 %6, label %7, label %13
+
+; <label>:7:                                      ; preds = %1
+  %8 = getelementptr inbounds [1 x %struct.__va_list_tag], [1 x %struct.__va_list_tag]* %2, i64 0, i64 0, i32 3
+  %9 = load i8*, i8** %8, align 16
+  %10 = sext i32 %5 to i64
+  %11 = getelementptr i8, i8* %9, i64 %10
+  %12 = add i32 %5, 8
+  store i32 %12, i32* %4, align 16
+  br label %17
+
+; <label>:13:                                     ; preds = %1
+  %14 = getelementptr inbounds [1 x %struct.__va_list_tag], [1 x %struct.__va_list_tag]* %2, i64 0, i64 0, i32 2
+  %15 = load i8*, i8** %14, align 8
+  %16 = getelementptr i8, i8* %15, i64 8
+  store i8* %16, i8** %14, align 8
+  br label %17
+
+; <label>:17:                                     ; preds = %13, %7
+  %18 = phi i8* [ %11, %7 ], [ %15, %13 ]
+  %19 = bitcast i8* %18 to i32*
+  %20 = load i32, i32* %19, align 4
+  call void @_Z6escapei(i32 %20)
+  call void @llvm.va_end(i8* nonnull %3)
+  ret void
+}
+
+; Function Attrs: nounwind
+declare void @llvm.va_start(i8*)
+
+; Function Attrs: minsize optsize
+declare dso_local void @_Z6escapei(i32) local_unnamed_addr
+
+; Function Attrs: nounwind
+declare void @llvm.va_end(i8*)
+
+; CHECK-LABEL: define {{.*}}@_Z10simple_va2PKcz
+; CHECK: call void @llvm.va_start
+; CHECK: call void @llvm.va_end
+define dso_local void @_Z10simple_va2PKcz(i8* nocapture readnone, ...) unnamed_addr {
+  %2 = alloca [1 x %struct.__va_list_tag], align 16
+  %3 = bitcast [1 x %struct.__va_list_tag]* %2 to i8*
+  call void @llvm.va_start(i8* nonnull %3)
+  %4 = getelementptr inbounds [1 x %struct.__va_list_tag], [1 x %struct.__va_list_tag]* %2, i64 0, i64 0, i32 0
+  %5 = load i32, i32* %4, align 16
+  %6 = icmp ult i32 %5, 41
+  br i1 %6, label %7, label %13
+
+; <label>:7:                                      ; preds = %1
+  %8 = getelementptr inbounds [1 x %struct.__va_list_tag], [1 x %struct.__va_list_tag]* %2, i64 0, i64 0, i32 3
+  %9 = load i8*, i8** %8, align 16
+  %10 = sext i32 %5 to i64
+  %11 = getelementptr i8, i8* %9, i64 %10
+  %12 = add i32 %5, 8
+  store i32 %12, i32* %4, align 16
+  br label %17
+
+; <label>:13:                                     ; preds = %1
+  %14 = getelementptr inbounds [1 x %struct.__va_list_tag], [1 x %struct.__va_list_tag]* %2, i64 0, i64 0, i32 2
+  %15 = load i8*, i8** %14, align 8
+  %16 = getelementptr i8, i8* %15, i64 8
+  store i8* %16, i8** %14, align 8
+  br label %17
+
+; <label>:17:                                     ; preds = %13, %7
+  %18 = phi i8* [ %11, %7 ], [ %15, %13 ]
+  %19 = bitcast i8* %18 to i32*
+  %20 = load i32, i32* %19, align 4
+  call void @_Z6escapei(i32 %20)
+  call void @llvm.va_end(i8* nonnull %3)
+  ret void
+}
diff --git a/test/Transforms/NewGVN/memory-handling.ll b/test/Transforms/NewGVN/memory-handling.ll
index 12e882bd0897861322e5bf363dc4bb75b8efa4ea..dfb52d617038f661d340ae29adada8533c24285b 100644
--- a/test/Transforms/NewGVN/memory-handling.ll
+++ b/test/Transforms/NewGVN/memory-handling.ll
@@ -176,7 +176,7 @@ attributes #5 = { nounwind readonly }
 
 !llvm.ident = !{!0}
 
-!0 = !{!"clang version 4.0.0 (http://llvm.org/git/clang.git 9b9db7fa41a1905899dbcbcc6cbdd05d2511da8e) (/Users/dannyb/sources/llvm-clean a3908a41623f6ac14ba8c04613d6c64e0544bb5d)"}
+!0 = !{!"clang version 4.0.0"}
 !1 = !{!2, !2, i64 0}
 !2 = !{!"int", !3, i64 0}
 !3 = !{!"omnipotent char", !4, i64 0}
diff --git a/test/Transforms/NewGVN/phi-edge-handling.ll b/test/Transforms/NewGVN/phi-edge-handling.ll
index 6451006a69490d242aec7c1679a97cd92d1ea2c5..890958c97870f3fd5a76aee550d02fe0e4ce38b4 100644
--- a/test/Transforms/NewGVN/phi-edge-handling.ll
+++ b/test/Transforms/NewGVN/phi-edge-handling.ll
@@ -55,6 +55,6 @@ attributes #0 = { ssp uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "d
 
 !llvm.ident = !{!0}
 
-!0 = !{!"clang version 5.0.0 (http://llvm.org/git/clang.git a8b933d4d1d133594fdaed35ee5814514b738f6d) (/Users/dannyb/sources/llvm-clean fc630a9b5613f544c07a8f16abcc173793df62cf)"}
+!0 = !{!"clang version 5.0.0"}
 !1 = distinct !{!1, !2}
 !2 = !{!"llvm.loop.unroll.disable"}
diff --git a/test/Transforms/NewGVN/pr31501.ll b/test/Transforms/NewGVN/pr31501.ll
index 7122ade56eeb64b66a9e34f1bf230f270639216d..bc95969ea76c4801c1ec77374384ea938abcfcb7 100644
--- a/test/Transforms/NewGVN/pr31501.ll
+++ b/test/Transforms/NewGVN/pr31501.ll
@@ -123,7 +123,7 @@ attributes #0 = { norecurse nounwind ssp uwtable "correctly-rounded-divide-sqrt-
 !llvm.ident = !{!1}
 
 !0 = !{i32 1, !"PIC Level", i32 2}
-!1 = !{!"clang version 4.0.0 (http://llvm.org/git/clang.git b63fa9e2bb8aac0a80c3e3467991c6b1a4b01e6a) (llvm/trunk 290779)"}
+!1 = !{!"clang version 4.0.0"}
 !2 = !{!3, !4, i64 0}
 !3 = !{!"_ZTSN4llvm15SmallVectorBaseE", !4, i64 0, !4, i64 8, !4, i64 16}
 !4 = !{!"any pointer", !5, i64 0}
diff --git a/test/Transforms/NewGVN/pr33305.ll b/test/Transforms/NewGVN/pr33305.ll
index 96717b84cf83da7ed839166a99ed7138f8f23450..9cbc10c098f7cade3638c2e4165100db975ea642 100644
--- a/test/Transforms/NewGVN/pr33305.ll
+++ b/test/Transforms/NewGVN/pr33305.ll
@@ -176,7 +176,7 @@ attributes #3 = { noreturn nounwind optsize }
 
 !0 = !{i32 1, !"wchar_size", i32 4}
 !1 = !{i32 7, !"PIC Level", i32 2}
-!2 = !{!"clang version 5.0.0 (http://llvm.org/git/clang.git e97b4dda83fd49e0218ea06ba4e37796a81b2027) (/Users/dannyb/sources/llvm-clean b38f051979e4ac2aa6513e40046d120fd472cb96)"}
+!2 = !{!"clang version 5.0.0"}
 !3 = !{!4, !4, i64 0}
 !4 = !{!"int", !5, i64 0}
 !5 = !{!"omnipotent char", !6, i64 0}
diff --git a/test/Transforms/NewGVN/pr34430.ll b/test/Transforms/NewGVN/pr34430.ll
index 81d18371cb99bdb4b96866c100f54c424d28bea2..0b59dfc72e1349cd7dee0c7ab8437d12e4a568be 100644
--- a/test/Transforms/NewGVN/pr34430.ll
+++ b/test/Transforms/NewGVN/pr34430.ll
@@ -45,4 +45,4 @@ attributes #0 = { norecurse noreturn nounwind ssp uwtable "correctly-rounded-div
 
 !llvm.ident = !{!0}
 
-!0 = !{!"clang version 6.0.0 (http://llvm.org/git/clang.git e649d902285b23af8ba58cb92a739f3bad2723df) (/Users/dannyb/sources/llvm-clean 0abfd30028cbb294ff2c2dd5e2df4ec3fdb6c591)"}
+!0 = !{!"clang version 6.0.0"}
diff --git a/test/Transforms/ObjCARC/rv.ll b/test/Transforms/ObjCARC/rv.ll
index 604ef59f119d16dcf54f9176cecf428b7cbff9cc..3d0d56ca0e536418abf112dee95515abd0047fd5 100644
--- a/test/Transforms/ObjCARC/rv.ll
+++ b/test/Transforms/ObjCARC/rv.ll
@@ -61,6 +61,11 @@ define void @test2() {
   call i8* @llvm.objc.retainAutoreleasedReturnValue(i8* null)
   call i8* @llvm.objc.autoreleaseReturnValue(i8* null)
   ; call i8* @llvm.objc.retainAutoreleaseReturnValue(i8* null) ; TODO
+  %bitcast = bitcast i32* null to i8*
+  %rb = call i8* @llvm.objc.retainBlock(i8* %bitcast)
+  call void @use_pointer(i8* %rb)
+  %rb2 = call i8* @llvm.objc.retainBlock(i8* undef)
+  call void @use_pointer(i8* %rb2)
   ret void
 }
 
@@ -234,6 +239,24 @@ define i8* @test19(i8* %p) {
   ret i8* %p
 }
 
+; Delete autoreleaseRV+retainRV pairs when they have equivalent PHIs as inputs
+
+; CHECK: define i8* @test19phi(i8* %p) {
+; CHECK-NEXT: entry:
+; CHECK-NEXT: br label %test19bb
+; CHECK: test19bb:
+; CHECK-NEXT: ret i8* %p
+define i8* @test19phi(i8* %p) {
+entry:
+  br label %test19bb
+test19bb:
+  %phi1 = phi i8* [ %p, %entry ]
+  %phi2 = phi i8* [ %p, %entry ]
+  call i8* @llvm.objc.autoreleaseReturnValue(i8* %phi1)
+  call i8* @llvm.objc.retainAutoreleasedReturnValue(i8* %phi2)
+  ret i8* %p
+}
+
 ; Like test19 but with plain autorelease.
 
 ; CHECK: define i8* @test20(i8* %p) {
diff --git a/test/Transforms/PGOProfile/preinline.ll b/test/Transforms/PGOProfile/preinline.ll
index e0deafd78767dede8f7e9852dcd7daaa046aac85..277baf99c33b9ee24757aa5f3f54d648d45b6214 100644
--- a/test/Transforms/PGOProfile/preinline.ll
+++ b/test/Transforms/PGOProfile/preinline.ll
@@ -1,5 +1,5 @@
-; RUN: opt < %s -O2 -profile-generate -S | FileCheck %s --check-prefix=GEN
-; RUN: opt < %s -O2 -profile-generate -profile-generate-file=default.profraw -S | FileCheck %s --check-prefix=GEN
+; RUN: opt < %s -O2 -pgo-kind=pgo-instr-gen-pipeline -S | FileCheck %s --check-prefix=GEN
+; RUN: opt < %s -O2 -pgo-kind=pgo-instr-gen-pipeline -profile-file=default.profraw -S | FileCheck %s --check-prefix=GEN
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
diff --git a/test/Transforms/PhaseOrdering/rotate.ll b/test/Transforms/PhaseOrdering/rotate.ll
index 190807668ee8d6bfa5921fcae581245b9d79be8f..e10a46cb8300e0ce10719e6084cccc7a9c91a170 100644
--- a/test/Transforms/PhaseOrdering/rotate.ll
+++ b/test/Transforms/PhaseOrdering/rotate.ll
@@ -5,6 +5,9 @@
 ; This should become a single funnel shift through a combination
 ; of aggressive-instcombine, simplifycfg, and instcombine.
 ; https://bugs.llvm.org/show_bug.cgi?id=34924
+; These are equivalent, but the value name with the new-pm shows a bug -
+; this code should not have been converted to a speculative select with
+; an intermediate transform.
 
 define i32 @rotl(i32 %a, i32 %b) {
 ; OLDPM-LABEL: @rotl(
@@ -14,12 +17,7 @@ define i32 @rotl(i32 %a, i32 %b) {
 ;
 ; NEWPM-LABEL: @rotl(
 ; NEWPM-NEXT:  entry:
-; NEWPM-NEXT:    [[TMP0:%.*]] = sub i32 0, [[B:%.*]]
-; NEWPM-NEXT:    [[TMP1:%.*]] = and i32 [[B]], 31
-; NEWPM-NEXT:    [[TMP2:%.*]] = and i32 [[TMP0]], 31
-; NEWPM-NEXT:    [[TMP3:%.*]] = lshr i32 [[A:%.*]], [[TMP2]]
-; NEWPM-NEXT:    [[TMP4:%.*]] = shl i32 [[A]], [[TMP1]]
-; NEWPM-NEXT:    [[SPEC_SELECT:%.*]] = or i32 [[TMP3]], [[TMP4]]
+; NEWPM-NEXT:    [[SPEC_SELECT:%.*]] = tail call i32 @llvm.fshl.i32(i32 [[A:%.*]], i32 [[A]], i32 [[B:%.*]])
 ; NEWPM-NEXT:    ret i32 [[SPEC_SELECT]]
 ;
 entry:
diff --git a/test/Transforms/RewriteStatepointsForGC/scalar-base-vector.ll b/test/Transforms/RewriteStatepointsForGC/scalar-base-vector.ll
new file mode 100644
index 0000000000000000000000000000000000000000..1dc48a444376162effe54615d1b343817337d826
--- /dev/null
+++ b/test/Transforms/RewriteStatepointsForGC/scalar-base-vector.ll
@@ -0,0 +1,143 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -rewrite-statepoints-for-gc -S | FileCheck  %s
+; RUN: opt < %s -passes=rewrite-statepoints-for-gc -S | FileCheck  %s
+
+declare void @do_safepoint()
+declare i8 addrspace(1)* @def_ptr()
+
+define i32 addrspace(1)* @test1(i8 addrspace(1)* %base1, <2 x i64> %offsets) gc "statepoint-example" {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 undef, label [[FIRST:%.*]], label [[SECOND:%.*]]
+; CHECK:       first:
+; CHECK-NEXT:    [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, i8 addrspace(1)* ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_p1i8f(i64 2882400000, i32 0, i8 addrspace(1)* ()* @def_ptr, i32 0, i32 0, i32 0, i32 5, i32 0, i32 -1, i32 0, i32 0, i32 0)
+; CHECK-NEXT:    [[BASE21:%.*]] = call i8 addrspace(1)* @llvm.experimental.gc.result.p1i8(token [[STATEPOINT_TOKEN]])
+; CHECK-NEXT:    br label [[SECOND]]
+; CHECK:       second:
+; CHECK-NEXT:    [[PHI_BASE:%.*]] = phi i8 addrspace(1)* [ [[BASE1:%.*]], [[ENTRY:%.*]] ], [ [[BASE21]], [[FIRST]] ], !is_base_value !0
+; CHECK-NEXT:    [[PHI:%.*]] = phi i8 addrspace(1)* [ [[BASE1]], [[ENTRY]] ], [ [[BASE21]], [[FIRST]] ]
+; CHECK-NEXT:    [[BASE_I32:%.*]] = bitcast i8 addrspace(1)* [[PHI]] to i32 addrspace(1)*
+; CHECK-NEXT:    [[CAST:%.*]] = bitcast i8 addrspace(1)* [[PHI_BASE]] to i32 addrspace(1)*
+; CHECK-NEXT:    [[DOTSPLATINSERT_BASE:%.*]] = insertelement <2 x i32 addrspace(1)*> zeroinitializer, i32 addrspace(1)* [[CAST]], i32 0, !is_base_value !0
+; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <2 x i32 addrspace(1)*> undef, i32 addrspace(1)* [[BASE_I32]], i32 0
+; CHECK-NEXT:    [[DOTSPLAT_BASE:%.*]] = shufflevector <2 x i32 addrspace(1)*> [[DOTSPLATINSERT_BASE]], <2 x i32 addrspace(1)*> zeroinitializer, <2 x i32> zeroinitializer, !is_base_value !0
+; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <2 x i32 addrspace(1)*> [[DOTSPLATINSERT]], <2 x i32 addrspace(1)*> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[VEC:%.*]] = getelementptr i32, <2 x i32 addrspace(1)*> [[DOTSPLAT]], <2 x i64> [[OFFSETS:%.*]]
+; CHECK-NEXT:    [[PTR_BASE:%.*]] = extractelement <2 x i32 addrspace(1)*> [[DOTSPLAT_BASE]], i32 1, !is_base_value !0
+; CHECK-NEXT:    [[PTR:%.*]] = extractelement <2 x i32 addrspace(1)*> [[VEC]], i32 1
+; CHECK-NEXT:    [[STATEPOINT_TOKEN2:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @do_safepoint, i32 0, i32 0, i32 0, i32 5, i32 0, i32 -1, i32 0, i32 0, i32 0, i32 addrspace(1)* [[PTR]], i32 addrspace(1)* [[PTR_BASE]])
+; CHECK-NEXT:    [[PTR_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN2]], i32 13, i32 12)
+; CHECK-NEXT:    [[PTR_RELOCATED_CASTED:%.*]] = bitcast i8 addrspace(1)* [[PTR_RELOCATED]] to i32 addrspace(1)*
+; CHECK-NEXT:    [[PTR_BASE_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN2]], i32 13, i32 13)
+; CHECK-NEXT:    [[PTR_BASE_RELOCATED_CASTED:%.*]] = bitcast i8 addrspace(1)* [[PTR_BASE_RELOCATED]] to i32 addrspace(1)*
+; CHECK-NEXT:    ret i32 addrspace(1)* [[PTR_RELOCATED_CASTED]]
+;
+entry:
+  br i1 undef, label %first, label %second
+
+first:
+  %base2 = call i8 addrspace(1)* @def_ptr() [ "deopt"(i32 0, i32 -1, i32 0, i32 0, i32 0) ]
+  br label %second
+
+second:
+  %phi = phi i8 addrspace(1)* [ %base1, %entry ], [ %base2, %first ]
+  %base.i32 = bitcast i8 addrspace(1)* %phi to i32 addrspace(1)*
+  %vec = getelementptr i32, i32 addrspace(1)* %base.i32, <2 x i64> %offsets
+  %ptr = extractelement <2 x i32 addrspace(1)*> %vec, i32 1
+  call void @do_safepoint() [ "deopt"(i32 0, i32 -1, i32 0, i32 0, i32 0) ]
+  ret i32 addrspace(1)* %ptr
+}
+
+define i32 addrspace(1)* @test2(i8 addrspace(1)* %base, <2 x i64> %offsets) gc "statepoint-example" {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[BASE_I32:%.*]] = bitcast i8 addrspace(1)* [[BASE:%.*]] to i32 addrspace(1)*
+; CHECK-NEXT:    [[CAST:%.*]] = bitcast i8 addrspace(1)* [[BASE]] to i32 addrspace(1)*
+; CHECK-NEXT:    [[DOTSPLATINSERT_BASE:%.*]] = insertelement <2 x i32 addrspace(1)*> zeroinitializer, i32 addrspace(1)* [[CAST]], i32 0, !is_base_value !0
+; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <2 x i32 addrspace(1)*> undef, i32 addrspace(1)* [[BASE_I32]], i32 0
+; CHECK-NEXT:    [[DOTSPLAT_BASE:%.*]] = shufflevector <2 x i32 addrspace(1)*> [[DOTSPLATINSERT_BASE]], <2 x i32 addrspace(1)*> zeroinitializer, <2 x i32> zeroinitializer, !is_base_value !0
+; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <2 x i32 addrspace(1)*> [[DOTSPLATINSERT]], <2 x i32 addrspace(1)*> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[VEC:%.*]] = getelementptr i32, <2 x i32 addrspace(1)*> [[DOTSPLAT]], <2 x i64> [[OFFSETS:%.*]]
+; CHECK-NEXT:    [[PTR_BASE:%.*]] = extractelement <2 x i32 addrspace(1)*> [[DOTSPLAT_BASE]], i32 1, !is_base_value !0
+; CHECK-NEXT:    [[PTR:%.*]] = extractelement <2 x i32 addrspace(1)*> [[VEC]], i32 1
+; CHECK-NEXT:    [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @do_safepoint, i32 0, i32 0, i32 0, i32 0, i32 addrspace(1)* [[PTR]], i32 addrspace(1)* [[PTR_BASE]])
+; CHECK-NEXT:    [[PTR_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 8, i32 7)
+; CHECK-NEXT:    [[PTR_RELOCATED_CASTED:%.*]] = bitcast i8 addrspace(1)* [[PTR_RELOCATED]] to i32 addrspace(1)*
+; CHECK-NEXT:    [[PTR_BASE_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 8, i32 8)
+; CHECK-NEXT:    [[PTR_BASE_RELOCATED_CASTED:%.*]] = bitcast i8 addrspace(1)* [[PTR_BASE_RELOCATED]] to i32 addrspace(1)*
+; CHECK-NEXT:    ret i32 addrspace(1)* [[PTR_RELOCATED_CASTED]]
+;
+entry:
+  %base.i32 = bitcast i8 addrspace(1)* %base to i32 addrspace(1)*
+  %vec = getelementptr i32, i32 addrspace(1)* %base.i32, <2 x i64> %offsets
+  %ptr = extractelement <2 x i32 addrspace(1)*> %vec, i32 1
+  call void @do_safepoint()
+  ret i32 addrspace(1)* %ptr
+}
+
+define i32 addrspace(1)* @test3(<2 x i8 addrspace(1)*> %base, <2 x i64> %offsets) gc "statepoint-example" {
+; CHECK-LABEL: @test3(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[BASE_I32:%.*]] = bitcast <2 x i8 addrspace(1)*> [[BASE:%.*]] to <2 x i32 addrspace(1)*>
+; CHECK-NEXT:    [[VEC:%.*]] = getelementptr i32, <2 x i32 addrspace(1)*> [[BASE_I32]], <2 x i64> [[OFFSETS:%.*]]
+; CHECK-NEXT:    [[BASE_EE:%.*]] = extractelement <2 x i8 addrspace(1)*> [[BASE]], i32 1, !is_base_value !0
+; CHECK-NEXT:    [[PTR:%.*]] = extractelement <2 x i32 addrspace(1)*> [[VEC]], i32 1
+; CHECK-NEXT:    [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @do_safepoint, i32 0, i32 0, i32 0, i32 0, i32 addrspace(1)* [[PTR]], i8 addrspace(1)* [[BASE_EE]])
+; CHECK-NEXT:    [[PTR_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 8, i32 7)
+; CHECK-NEXT:    [[PTR_RELOCATED_CASTED:%.*]] = bitcast i8 addrspace(1)* [[PTR_RELOCATED]] to i32 addrspace(1)*
+; CHECK-NEXT:    [[BASE_EE_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 8, i32 8)
+; CHECK-NEXT:    ret i32 addrspace(1)* [[PTR_RELOCATED_CASTED]]
+;
+entry:
+  %base.i32 = bitcast <2 x i8 addrspace(1)*> %base to <2 x i32 addrspace(1)*>
+  %vec = getelementptr i32, <2 x i32 addrspace(1)*> %base.i32, <2 x i64> %offsets
+  %ptr = extractelement <2 x i32 addrspace(1)*> %vec, i32 1
+  call void @do_safepoint()
+  ret i32 addrspace(1)* %ptr
+}
+
+define i32 addrspace(1)* @test4(i8 addrspace(1)* %base, <2 x i64> %offsets) gc "statepoint-example" {
+; CHECK-LABEL: @test4(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[BASE_I32:%.*]] = bitcast i8 addrspace(1)* [[BASE:%.*]] to i32 addrspace(1)*
+; CHECK-NEXT:    [[CAST:%.*]] = bitcast i8 addrspace(1)* [[BASE]] to i32 addrspace(1)*
+; CHECK-NEXT:    [[DOTSPLATINSERT_BASE:%.*]] = insertelement <2 x i32 addrspace(1)*> zeroinitializer, i32 addrspace(1)* [[CAST]], i32 0, !is_base_value !0
+; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <2 x i32 addrspace(1)*> undef, i32 addrspace(1)* [[BASE_I32]], i32 0
+; CHECK-NEXT:    [[DOTSPLAT_BASE:%.*]] = shufflevector <2 x i32 addrspace(1)*> [[DOTSPLATINSERT_BASE]], <2 x i32 addrspace(1)*> zeroinitializer, <2 x i32> zeroinitializer, !is_base_value !0
+; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <2 x i32 addrspace(1)*> [[DOTSPLATINSERT]], <2 x i32 addrspace(1)*> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[VEC:%.*]] = getelementptr i32, <2 x i32 addrspace(1)*> [[DOTSPLAT]], <2 x i64> [[OFFSETS:%.*]]
+; CHECK-NEXT:    [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @do_safepoint, i32 0, i32 0, i32 0, i32 0, <2 x i32 addrspace(1)*> [[VEC]], <2 x i32 addrspace(1)*> [[DOTSPLAT_BASE]])
+; CHECK-NEXT:    [[VEC_RELOCATED:%.*]] = call coldcc <2 x i8 addrspace(1)*> @llvm.experimental.gc.relocate.v2p1i8(token [[STATEPOINT_TOKEN]], i32 8, i32 7)
+; CHECK-NEXT:    [[VEC_RELOCATED_CASTED:%.*]] = bitcast <2 x i8 addrspace(1)*> [[VEC_RELOCATED]] to <2 x i32 addrspace(1)*>
+; CHECK-NEXT:    [[DOTSPLAT_BASE_RELOCATED:%.*]] = call coldcc <2 x i8 addrspace(1)*> @llvm.experimental.gc.relocate.v2p1i8(token [[STATEPOINT_TOKEN]], i32 8, i32 8)
+; CHECK-NEXT:    [[DOTSPLAT_BASE_RELOCATED_CASTED:%.*]] = bitcast <2 x i8 addrspace(1)*> [[DOTSPLAT_BASE_RELOCATED]] to <2 x i32 addrspace(1)*>
+; CHECK-NEXT:    [[PTR:%.*]] = extractelement <2 x i32 addrspace(1)*> [[VEC_RELOCATED_CASTED]], i32 1
+; CHECK-NEXT:    ret i32 addrspace(1)* [[PTR]]
+;
+entry:
+  %base.i32 = bitcast i8 addrspace(1)* %base to i32 addrspace(1)*
+  %vec = getelementptr i32, i32 addrspace(1)* %base.i32, <2 x i64> %offsets
+  call void @do_safepoint()
+  %ptr = extractelement <2 x i32 addrspace(1)*> %vec, i32 1
+  ret i32 addrspace(1)* %ptr
+}
+
+define i32 addrspace(1)* @test5(<2 x i8 addrspace(1)*> %base, <2 x i64> %offsets) gc "statepoint-example" {
+; CHECK-LABEL: @test5(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[BASE_I32:%.*]] = bitcast <2 x i8 addrspace(1)*> [[BASE:%.*]] to <2 x i32 addrspace(1)*>
+; CHECK-NEXT:    [[VEC:%.*]] = getelementptr i32, <2 x i32 addrspace(1)*> [[BASE_I32]], <2 x i64> [[OFFSETS:%.*]]
+; CHECK-NEXT:    [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @do_safepoint, i32 0, i32 0, i32 0, i32 0, <2 x i8 addrspace(1)*> [[BASE]])
+; CHECK-NEXT:    [[BASE_RELOCATED:%.*]] = call coldcc <2 x i8 addrspace(1)*> @llvm.experimental.gc.relocate.v2p1i8(token [[STATEPOINT_TOKEN]], i32 7, i32 7)
+; CHECK-NEXT:    [[BASE_I32_REMAT:%.*]] = bitcast <2 x i8 addrspace(1)*> [[BASE_RELOCATED]] to <2 x i32 addrspace(1)*>
+; CHECK-NEXT:    [[VEC_REMAT:%.*]] = getelementptr i32, <2 x i32 addrspace(1)*> [[BASE_I32_REMAT]], <2 x i64> [[OFFSETS]]
+; CHECK-NEXT:    [[PTR:%.*]] = extractelement <2 x i32 addrspace(1)*> [[VEC_REMAT]], i32 0
+; CHECK-NEXT:    ret i32 addrspace(1)* [[PTR]]
+;
+entry:
+  %base.i32 = bitcast <2 x i8 addrspace(1)*> %base to <2 x i32 addrspace(1)*>
+  %vec = getelementptr i32, <2 x i32 addrspace(1)*> %base.i32, <2 x i64> %offsets
+  call void @do_safepoint()
+  %ptr = extractelement <2 x i32 addrspace(1)*> %vec, i32 0
+  ret i32 addrspace(1)* %ptr
+}
diff --git a/test/Transforms/SLPVectorizer/AArch64/64-bit-vector.ll b/test/Transforms/SLPVectorizer/AArch64/64-bit-vector.ll
index edc8042a217df59cda07404679e7f408640ce6db..ad970b2bec1bf4da5790682a2e9a96af4ab35684 100644
--- a/test/Transforms/SLPVectorizer/AArch64/64-bit-vector.ll
+++ b/test/Transforms/SLPVectorizer/AArch64/64-bit-vector.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -S -slp-vectorizer -mtriple=aarch64--linux-gnu -mcpu=generic < %s | FileCheck %s
 ; RUN: opt -S -slp-vectorizer -mtriple=aarch64-apple-ios -mcpu=cyclone < %s | FileCheck %s
 ; Currently disabled for a few subtargets (e.g. Kryo):
@@ -5,14 +6,36 @@
 ; RUN: opt -S -slp-vectorizer -mtriple=aarch64--linux-gnu -mcpu=generic -slp-min-reg-size=128 < %s | FileCheck --check-prefix=NO_SLP %s
 
 define void @f(float* %r, float* %w) {
+; CHECK-LABEL: @f(
+; CHECK-NEXT:    [[R0:%.*]] = getelementptr inbounds float, float* [[R:%.*]], i64 0
+; CHECK-NEXT:    [[R1:%.*]] = getelementptr inbounds float, float* [[R]], i64 1
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast float* [[R0]] to <2 x float>*
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x float>, <2 x float>* [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = fadd <2 x float> [[TMP2]], [[TMP2]]
+; CHECK-NEXT:    [[W0:%.*]] = getelementptr inbounds float, float* [[W:%.*]], i64 0
+; CHECK-NEXT:    [[W1:%.*]] = getelementptr inbounds float, float* [[W]], i64 1
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast float* [[W0]] to <2 x float>*
+; CHECK-NEXT:    store <2 x float> [[TMP3]], <2 x float>* [[TMP4]], align 4
+; CHECK-NEXT:    ret void
+;
+; NO_SLP-LABEL: @f(
+; NO_SLP-NEXT:    [[R0:%.*]] = getelementptr inbounds float, float* [[R:%.*]], i64 0
+; NO_SLP-NEXT:    [[R1:%.*]] = getelementptr inbounds float, float* [[R]], i64 1
+; NO_SLP-NEXT:    [[F0:%.*]] = load float, float* [[R0]]
+; NO_SLP-NEXT:    [[F1:%.*]] = load float, float* [[R1]]
+; NO_SLP-NEXT:    [[ADD0:%.*]] = fadd float [[F0]], [[F0]]
+; NO_SLP-NEXT:    [[ADD1:%.*]] = fadd float [[F1]], [[F1]]
+; NO_SLP-NEXT:    [[W0:%.*]] = getelementptr inbounds float, float* [[W:%.*]], i64 0
+; NO_SLP-NEXT:    [[W1:%.*]] = getelementptr inbounds float, float* [[W]], i64 1
+; NO_SLP-NEXT:    store float [[ADD0]], float* [[W0]]
+; NO_SLP-NEXT:    store float [[ADD1]], float* [[W1]]
+; NO_SLP-NEXT:    ret void
+;
   %r0 = getelementptr inbounds float, float* %r, i64 0
   %r1 = getelementptr inbounds float, float* %r, i64 1
   %f0 = load float, float* %r0
   %f1 = load float, float* %r1
   %add0 = fadd float %f0, %f0
-; CHECK:  fadd <2 x float>
-; NO_SLP: fadd float
-; NO_SLP: fadd float
   %add1 = fadd float %f1, %f1
   %w0 = getelementptr inbounds float, float* %w, i64 0
   %w1 = getelementptr inbounds float, float* %w, i64 1
diff --git a/test/Transforms/SLPVectorizer/AArch64/commute.ll b/test/Transforms/SLPVectorizer/AArch64/commute.ll
index 2bce59c62000350ead31c1a09ace739f2b411acb..3f831fd2cc7d8ed2f372f63130241e5278ef789b 100644
--- a/test/Transforms/SLPVectorizer/AArch64/commute.ll
+++ b/test/Transforms/SLPVectorizer/AArch64/commute.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -S -slp-vectorizer %s -slp-threshold=-10 | FileCheck %s
 target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
 target triple = "aarch64--linux-gnu"
@@ -5,17 +6,27 @@ target triple = "aarch64--linux-gnu"
 %structA = type { [2 x float] }
 
 define void @test1(%structA* nocapture readonly %J, i32 %xmin, i32 %ymin) {
-; CHECK-LABEL: test1
-; CHECK: %arrayidx4 = getelementptr inbounds %structA, %structA* %J, i64 0, i32 0, i64 0
-; CHECK: %arrayidx9 = getelementptr inbounds %structA, %structA* %J, i64 0, i32 0, i64 1
-; CHECK: %3 = bitcast float* %arrayidx4 to <2 x float>*
-; CHECK: %4 = load <2 x float>, <2 x float>* %3, align 4
-; CHECK: %5 = fsub fast <2 x float> %2, %4
-; CHECK: %6 = fmul fast <2 x float> %5, %5
-; CHECK: %7 = extractelement <2 x float> %6, i32 0
-; CHECK: %8 = extractelement <2 x float> %6, i32 1
-; CHECK: %add = fadd fast float %7, %8
-; CHECK: %cmp = fcmp oeq float %add, 0.000000e+00
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <2 x i32> undef, i32 [[XMIN:%.*]], i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x i32> [[TMP0]], i32 [[YMIN:%.*]], i32 1
+; CHECK-NEXT:    br label [[FOR_BODY3_LR_PH:%.*]]
+; CHECK:       for.body3.lr.ph:
+; CHECK-NEXT:    [[TMP2:%.*]] = sitofp <2 x i32> [[TMP1]] to <2 x float>
+; CHECK-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds [[STRUCTA:%.*]], %structA* [[J:%.*]], i64 0, i32 0, i64 0
+; CHECK-NEXT:    [[ARRAYIDX9:%.*]] = getelementptr inbounds [[STRUCTA]], %structA* [[J]], i64 0, i32 0, i64 1
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast float* [[ARRAYIDX4]] to <2 x float>*
+; CHECK-NEXT:    [[TMP4:%.*]] = load <2 x float>, <2 x float>* [[TMP3]], align 4
+; CHECK-NEXT:    [[TMP5:%.*]] = fsub fast <2 x float> [[TMP2]], [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = fmul fast <2 x float> [[TMP5]], [[TMP5]]
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <2 x float> [[TMP6]], i32 0
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <2 x float> [[TMP6]], i32 1
+; CHECK-NEXT:    [[ADD:%.*]] = fadd fast float [[TMP7]], [[TMP8]]
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp oeq float [[ADD]], 0.000000e+00
+; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY3_LR_PH]], label [[FOR_END27:%.*]]
+; CHECK:       for.end27:
+; CHECK-NEXT:    ret void
+;
 
 entry:
   br label %for.body3.lr.ph
@@ -40,17 +51,27 @@ for.end27:
 }
 
 define void @test2(%structA* nocapture readonly %J, i32 %xmin, i32 %ymin) {
-; CHECK-LABEL: test2
-; CHECK: %arrayidx4 = getelementptr inbounds %structA, %structA* %J, i64 0, i32 0, i64 0
-; CHECK: %arrayidx9 = getelementptr inbounds %structA, %structA* %J, i64 0, i32 0, i64 1
-; CHECK: %3 = bitcast float* %arrayidx4 to <2 x float>*
-; CHECK: %4 = load <2 x float>, <2 x float>* %3, align 4
-; CHECK: %5 = fsub fast <2 x float> %2, %4
-; CHECK: %6 = fmul fast <2 x float> %5, %5
-; CHECK: %7 = extractelement <2 x float> %6, i32 0
-; CHECK: %8 = extractelement <2 x float> %6, i32 1
-; CHECK: %add = fadd fast float %8, %7
-; CHECK: %cmp = fcmp oeq float %add, 0.000000e+00
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <2 x i32> undef, i32 [[XMIN:%.*]], i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x i32> [[TMP0]], i32 [[YMIN:%.*]], i32 1
+; CHECK-NEXT:    br label [[FOR_BODY3_LR_PH:%.*]]
+; CHECK:       for.body3.lr.ph:
+; CHECK-NEXT:    [[TMP2:%.*]] = sitofp <2 x i32> [[TMP1]] to <2 x float>
+; CHECK-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds [[STRUCTA:%.*]], %structA* [[J:%.*]], i64 0, i32 0, i64 0
+; CHECK-NEXT:    [[ARRAYIDX9:%.*]] = getelementptr inbounds [[STRUCTA]], %structA* [[J]], i64 0, i32 0, i64 1
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast float* [[ARRAYIDX4]] to <2 x float>*
+; CHECK-NEXT:    [[TMP4:%.*]] = load <2 x float>, <2 x float>* [[TMP3]], align 4
+; CHECK-NEXT:    [[TMP5:%.*]] = fsub fast <2 x float> [[TMP2]], [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = fmul fast <2 x float> [[TMP5]], [[TMP5]]
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <2 x float> [[TMP6]], i32 0
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <2 x float> [[TMP6]], i32 1
+; CHECK-NEXT:    [[ADD:%.*]] = fadd fast float [[TMP8]], [[TMP7]]
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp oeq float [[ADD]], 0.000000e+00
+; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY3_LR_PH]], label [[FOR_END27:%.*]]
+; CHECK:       for.end27:
+; CHECK-NEXT:    ret void
+;
 
 entry:
   br label %for.body3.lr.ph
diff --git a/test/Transforms/SLPVectorizer/AArch64/gather-cost.ll b/test/Transforms/SLPVectorizer/AArch64/gather-cost.ll
index a4c655ca3cb0b74d99ac09829dbfe3698cda6187..401776aa270d250547ef727f603dd10fa638b5a6 100644
--- a/test/Transforms/SLPVectorizer/AArch64/gather-cost.ll
+++ b/test/Transforms/SLPVectorizer/AArch64/gather-cost.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -S -slp-vectorizer -instcombine -pass-remarks-output=%t | FileCheck %s
 ; RUN: cat %t | FileCheck -check-prefix=REMARK %s
 ; RUN: opt < %s -S -passes='slp-vectorizer,instcombine' -pass-remarks-output=%t | FileCheck %s
@@ -6,25 +7,25 @@
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 target triple = "aarch64--linux-gnu"
 
-; CHECK-LABEL:  @gather_multiple_use(
-; CHECK-NEXT:     [[TMP1:%.*]] = insertelement <4 x i32> undef, i32 [[C:%.*]], i32 0
-; CHECK-NEXT:     [[TMP2:%.*]] = insertelement <4 x i32> [[TMP1]], i32 [[A:%.*]], i32 1
-; CHECK-NEXT:     [[TMP3:%.*]] = insertelement <4 x i32> [[TMP2]], i32 [[B:%.*]], i32 2
-; CHECK-NEXT:     [[TMP4:%.*]] = insertelement <4 x i32> [[TMP3]], i32 [[D:%.*]], i32 3
-; CHECK-NEXT:     [[TMP5:%.*]] = lshr <4 x i32> [[TMP4]], <i32 15, i32 15, i32 15, i32 15>
-; CHECK-NEXT:     [[TMP6:%.*]] = and <4 x i32> [[TMP5]], <i32 65537, i32 65537, i32 65537, i32 65537>
-; CHECK-NEXT:     [[TMP7:%.*]] = mul nuw <4 x i32> [[TMP6]], <i32 65535, i32 65535, i32 65535, i32 65535>
-; CHECK-NEXT:     [[TMP8:%.*]] = add <4 x i32> [[TMP4]], [[TMP7]]
-; CHECK-NEXT:     [[TMP9:%.*]] = xor <4 x i32> [[TMP8]], [[TMP7]]
-; CHECK-NEXT:     [[TMP10:%.*]] = call i32 @llvm.experimental.vector.reduce.add.i32.v4i32(<4 x i32> [[TMP9]])
-; CHECK-NEXT:     ret i32 [[TMP10]]
-;
 ; REMARK-LABEL: Function: gather_multiple_use
 ; REMARK:       Args:
 ; REMARK-NEXT:    - String: 'Vectorized horizontal reduction with cost '
 ; REMARK-NEXT:    - Cost: '-7'
 ;
 define internal i32 @gather_multiple_use(i32 %a, i32 %b, i32 %c, i32 %d) {
+; CHECK-LABEL: @gather_multiple_use(
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x i32> undef, i32 [[C:%.*]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <4 x i32> [[TMP1]], i32 [[A:%.*]], i32 1
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <4 x i32> [[TMP2]], i32 [[B:%.*]], i32 2
+; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <4 x i32> [[TMP3]], i32 [[D:%.*]], i32 3
+; CHECK-NEXT:    [[TMP5:%.*]] = lshr <4 x i32> [[TMP4]], <i32 15, i32 15, i32 15, i32 15>
+; CHECK-NEXT:    [[TMP6:%.*]] = and <4 x i32> [[TMP5]], <i32 65537, i32 65537, i32 65537, i32 65537>
+; CHECK-NEXT:    [[TMP7:%.*]] = mul nuw <4 x i32> [[TMP6]], <i32 65535, i32 65535, i32 65535, i32 65535>
+; CHECK-NEXT:    [[TMP8:%.*]] = add <4 x i32> [[TMP4]], [[TMP7]]
+; CHECK-NEXT:    [[TMP9:%.*]] = xor <4 x i32> [[TMP8]], [[TMP7]]
+; CHECK-NEXT:    [[TMP10:%.*]] = call i32 @llvm.experimental.vector.reduce.add.i32.v4i32(<4 x i32> [[TMP9]])
+; CHECK-NEXT:    ret i32 [[TMP10]]
+;
   %tmp00 = lshr i32 %a, 15
   %tmp01 = and i32 %tmp00, 65537
   %tmp02 = mul nuw i32 %tmp01, 65535
diff --git a/test/Transforms/SLPVectorizer/AArch64/getelementptr.ll b/test/Transforms/SLPVectorizer/AArch64/getelementptr.ll
index 3f61875b91825a94de2356c46c12cafdfe84b089..db02f55dcc7734ddb86c9dbbd53db4dfaa3e7163 100644
--- a/test/Transforms/SLPVectorizer/AArch64/getelementptr.ll
+++ b/test/Transforms/SLPVectorizer/AArch64/getelementptr.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -S -slp-vectorizer -slp-threshold=-18 -dce -instcombine -pass-remarks-output=%t < %s | FileCheck %s
 ; RUN: cat %t | FileCheck -check-prefix=YAML %s
 ; RUN: opt -S -passes='slp-vectorizer,dce,instcombine' -slp-threshold=-18 -pass-remarks-output=%t < %s | FileCheck %s
@@ -22,12 +23,6 @@ target triple = "aarch64--linux-gnu"
 ; }
 ;
 
-; CHECK-LABEL: @getelementptr_4x32
-;
-; CHECK: [[A:%[a-zA-Z0-9.]+]] = add nsw <4 x i32>
-; CHECK: [[X:%[a-zA-Z0-9.]+]] = extractelement <4 x i32> [[A]]
-; CHECK: sext i32 [[X]] to i64
-
 ; YAML:      --- !Passed
 ; YAML-NEXT: Pass:            slp-vectorizer
 ; YAML-NEXT: Name:            VectorizedList
@@ -49,6 +44,56 @@ target triple = "aarch64--linux-gnu"
 ; YAML-NEXT:   - TreeSize:        '3'
 
 define i32 @getelementptr_4x32(i32* nocapture readonly %g, i32 %n, i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @getelementptr_4x32(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP31:%.*]] = icmp sgt i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[CMP31]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]]
+; CHECK:       for.body.preheader:
+; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>, i32 [[X:%.*]], i32 1
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x i32> [[TMP0]], i32 [[Y:%.*]], i32 2
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <4 x i32> [[TMP1]], i32 [[Z:%.*]], i32 3
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.cond.cleanup.loopexit:
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <2 x i32> [[TMP21:%.*]], i32 1
+; CHECK-NEXT:    br label [[FOR_COND_CLEANUP]]
+; CHECK:       for.cond.cleanup:
+; CHECK-NEXT:    [[SUM_0_LCSSA:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[TMP3]], [[FOR_COND_CLEANUP_LOOPEXIT:%.*]] ]
+; CHECK-NEXT:    ret i32 [[SUM_0_LCSSA]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[TMP4:%.*]] = phi <2 x i32> [ zeroinitializer, [[FOR_BODY_PREHEADER]] ], [ [[TMP21]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <2 x i32> [[TMP4]], i32 0
+; CHECK-NEXT:    [[T4:%.*]] = shl nsw i32 [[TMP5]], 1
+; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <4 x i32> undef, i32 [[T4]], i32 0
+; CHECK-NEXT:    [[TMP7:%.*]] = shufflevector <4 x i32> [[TMP6]], <4 x i32> undef, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = add nsw <4 x i32> [[TMP2]], [[TMP7]]
+; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <4 x i32> [[TMP8]], i32 0
+; CHECK-NEXT:    [[TMP10:%.*]] = sext i32 [[TMP9]] to i64
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[G:%.*]], i64 [[TMP10]]
+; CHECK-NEXT:    [[T6:%.*]] = load i32, i32* [[ARRAYIDX]], align 4
+; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <2 x i32> [[TMP4]], i32 1
+; CHECK-NEXT:    [[ADD1:%.*]] = add nsw i32 [[T6]], [[TMP11]]
+; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <4 x i32> [[TMP8]], i32 1
+; CHECK-NEXT:    [[TMP13:%.*]] = sext i32 [[TMP12]] to i64
+; CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds i32, i32* [[G]], i64 [[TMP13]]
+; CHECK-NEXT:    [[T8:%.*]] = load i32, i32* [[ARRAYIDX5]], align 4
+; CHECK-NEXT:    [[ADD6:%.*]] = add nsw i32 [[ADD1]], [[T8]]
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <4 x i32> [[TMP8]], i32 2
+; CHECK-NEXT:    [[TMP15:%.*]] = sext i32 [[TMP14]] to i64
+; CHECK-NEXT:    [[ARRAYIDX10:%.*]] = getelementptr inbounds i32, i32* [[G]], i64 [[TMP15]]
+; CHECK-NEXT:    [[T10:%.*]] = load i32, i32* [[ARRAYIDX10]], align 4
+; CHECK-NEXT:    [[ADD11:%.*]] = add nsw i32 [[ADD6]], [[T10]]
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <4 x i32> [[TMP8]], i32 3
+; CHECK-NEXT:    [[TMP17:%.*]] = sext i32 [[TMP16]] to i64
+; CHECK-NEXT:    [[ARRAYIDX15:%.*]] = getelementptr inbounds i32, i32* [[G]], i64 [[TMP17]]
+; CHECK-NEXT:    [[T12:%.*]] = load i32, i32* [[ARRAYIDX15]], align 4
+; CHECK-NEXT:    [[TMP18:%.*]] = insertelement <2 x i32> <i32 1, i32 undef>, i32 [[ADD11]], i32 1
+; CHECK-NEXT:    [[TMP19:%.*]] = insertelement <2 x i32> undef, i32 [[TMP5]], i32 0
+; CHECK-NEXT:    [[TMP20:%.*]] = insertelement <2 x i32> [[TMP19]], i32 [[T12]], i32 1
+; CHECK-NEXT:    [[TMP21]] = add nsw <2 x i32> [[TMP18]], [[TMP20]]
+; CHECK-NEXT:    [[TMP22:%.*]] = extractelement <2 x i32> [[TMP21]], i32 0
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[TMP22]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP_LOOPEXIT]], label [[FOR_BODY]]
+;
 entry:
   %cmp31 = icmp sgt i32 %n, 0
   br i1 %cmp31, label %for.body.preheader, label %for.cond.cleanup
@@ -88,12 +133,6 @@ for.body:
   br i1 %exitcond, label %for.cond.cleanup.loopexit, label %for.body
 }
 
-; CHECK-LABEL: @getelementptr_2x32
-;
-; CHECK: [[A:%[a-zA-Z0-9.]+]] = add nsw <2 x i32>
-; CHECK: [[X:%[a-zA-Z0-9.]+]] = extractelement <2 x i32> [[A]]
-; CHECK: sext i32 [[X]] to i64
-
 ; YAML:      --- !Passed
 ; YAML-NEXT: Pass:            slp-vectorizer
 ; YAML-NEXT: Name:            VectorizedList
@@ -115,6 +154,54 @@ for.body:
 ; YAML-NEXT:   - TreeSize:        '3'
 
 define i32 @getelementptr_2x32(i32* nocapture readonly %g, i32 %n, i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @getelementptr_2x32(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP31:%.*]] = icmp sgt i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[CMP31]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]]
+; CHECK:       for.body.preheader:
+; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <2 x i32> undef, i32 [[Y:%.*]], i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x i32> [[TMP0]], i32 [[Z:%.*]], i32 1
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.cond.cleanup.loopexit:
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <2 x i32> [[TMP18:%.*]], i32 1
+; CHECK-NEXT:    br label [[FOR_COND_CLEANUP]]
+; CHECK:       for.cond.cleanup:
+; CHECK-NEXT:    [[SUM_0_LCSSA:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[TMP2]], [[FOR_COND_CLEANUP_LOOPEXIT:%.*]] ]
+; CHECK-NEXT:    ret i32 [[SUM_0_LCSSA]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[TMP3:%.*]] = phi <2 x i32> [ zeroinitializer, [[FOR_BODY_PREHEADER]] ], [ [[TMP18]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x i32> [[TMP3]], i32 0
+; CHECK-NEXT:    [[T4:%.*]] = shl nsw i32 [[TMP4]], 1
+; CHECK-NEXT:    [[TMP5:%.*]] = sext i32 [[T4]] to i64
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[G:%.*]], i64 [[TMP5]]
+; CHECK-NEXT:    [[T6:%.*]] = load i32, i32* [[ARRAYIDX]], align 4
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <2 x i32> [[TMP3]], i32 1
+; CHECK-NEXT:    [[ADD1:%.*]] = add nsw i32 [[T6]], [[TMP6]]
+; CHECK-NEXT:    [[T7:%.*]] = or i32 [[T4]], 1
+; CHECK-NEXT:    [[TMP7:%.*]] = sext i32 [[T7]] to i64
+; CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds i32, i32* [[G]], i64 [[TMP7]]
+; CHECK-NEXT:    [[T8:%.*]] = load i32, i32* [[ARRAYIDX5]], align 4
+; CHECK-NEXT:    [[ADD6:%.*]] = add nsw i32 [[ADD1]], [[T8]]
+; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <2 x i32> undef, i32 [[T4]], i32 0
+; CHECK-NEXT:    [[TMP9:%.*]] = shufflevector <2 x i32> [[TMP8]], <2 x i32> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP10:%.*]] = add nsw <2 x i32> [[TMP1]], [[TMP9]]
+; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <2 x i32> [[TMP10]], i32 0
+; CHECK-NEXT:    [[TMP12:%.*]] = sext i32 [[TMP11]] to i64
+; CHECK-NEXT:    [[ARRAYIDX10:%.*]] = getelementptr inbounds i32, i32* [[G]], i64 [[TMP12]]
+; CHECK-NEXT:    [[T10:%.*]] = load i32, i32* [[ARRAYIDX10]], align 4
+; CHECK-NEXT:    [[ADD11:%.*]] = add nsw i32 [[ADD6]], [[T10]]
+; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <2 x i32> [[TMP10]], i32 1
+; CHECK-NEXT:    [[TMP14:%.*]] = sext i32 [[TMP13]] to i64
+; CHECK-NEXT:    [[ARRAYIDX15:%.*]] = getelementptr inbounds i32, i32* [[G]], i64 [[TMP14]]
+; CHECK-NEXT:    [[T12:%.*]] = load i32, i32* [[ARRAYIDX15]], align 4
+; CHECK-NEXT:    [[TMP15:%.*]] = insertelement <2 x i32> <i32 1, i32 undef>, i32 [[ADD11]], i32 1
+; CHECK-NEXT:    [[TMP16:%.*]] = insertelement <2 x i32> undef, i32 [[TMP4]], i32 0
+; CHECK-NEXT:    [[TMP17:%.*]] = insertelement <2 x i32> [[TMP16]], i32 [[T12]], i32 1
+; CHECK-NEXT:    [[TMP18]] = add nsw <2 x i32> [[TMP15]], [[TMP17]]
+; CHECK-NEXT:    [[TMP19:%.*]] = extractelement <2 x i32> [[TMP18]], i32 0
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[TMP19]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP_LOOPEXIT]], label [[FOR_BODY]]
+;
 entry:
   %cmp31 = icmp sgt i32 %n, 0
   br i1 %cmp31, label %for.body.preheader, label %for.cond.cleanup
diff --git a/test/Transforms/SLPVectorizer/AArch64/horizontal.ll b/test/Transforms/SLPVectorizer/AArch64/horizontal.ll
index 02cf09d3434bc2404947356baa5017bafd7fdc95..c4a584fcbf3fa109388aa45f638d0ec3eb054ad5 100644
--- a/test/Transforms/SLPVectorizer/AArch64/horizontal.ll
+++ b/test/Transforms/SLPVectorizer/AArch64/horizontal.ll
@@ -1,6 +1,5 @@
-; RUN: opt -slp-vectorizer -slp-threshold=-6 -S -pass-remarks-output=%t <  %s | FileCheck %s
-; RUN: cat %t | FileCheck -check-prefix=YAML %s
-; RUN: opt -passes=slp-vectorizer -slp-threshold=-6 -S -pass-remarks-output=%t <  %s | FileCheck %s
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -slp-vectorizer -slp-threshold=-6 -S -pass-remarks-output=%t < %s | FileCheck %s
 ; RUN: cat %t | FileCheck -check-prefix=YAML %s
 
 
@@ -10,11 +9,6 @@
 target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
 target triple = "aarch64--linux"
 
-; CHECK-LABEL: test_select
-; CHECK: load <4 x i32>
-; CHECK: load <4 x i32>
-; CHECK: select <4 x i1>
-
 ; YAML:      --- !Passed
 ; YAML-NEXT: Pass:            slp-vectorizer
 ; YAML-NEXT: Name:            VectorizedHorizontalReduction
@@ -26,6 +20,49 @@ target triple = "aarch64--linux"
 ; YAML-NEXT:   - TreeSize:        '8'
 
 define i32 @test_select(i32* noalias nocapture readonly %blk1, i32* noalias nocapture readonly %blk2, i32 %lx, i32 %h) {
+; CHECK-LABEL: @test_select(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP_22:%.*]] = icmp sgt i32 [[H:%.*]], 0
+; CHECK-NEXT:    br i1 [[CMP_22]], label [[FOR_BODY_LR_PH:%.*]], label [[FOR_END:%.*]]
+; CHECK:       for.body.lr.ph:
+; CHECK-NEXT:    [[IDX_EXT:%.*]] = sext i32 [[LX:%.*]] to i64
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[S_026:%.*]] = phi i32 [ 0, [[FOR_BODY_LR_PH]] ], [ [[OP_EXTRA:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[J_025:%.*]] = phi i32 [ 0, [[FOR_BODY_LR_PH]] ], [ [[INC:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[P2_024:%.*]] = phi i32* [ [[BLK2:%.*]], [[FOR_BODY_LR_PH]] ], [ [[ADD_PTR29:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[P1_023:%.*]] = phi i32* [ [[BLK1:%.*]], [[FOR_BODY_LR_PH]] ], [ [[ADD_PTR:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds i32, i32* [[P1_023]], i64 1
+; CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds i32, i32* [[P2_024]], i64 1
+; CHECK-NEXT:    [[ARRAYIDX12:%.*]] = getelementptr inbounds i32, i32* [[P1_023]], i64 2
+; CHECK-NEXT:    [[ARRAYIDX13:%.*]] = getelementptr inbounds i32, i32* [[P2_024]], i64 2
+; CHECK-NEXT:    [[ARRAYIDX20:%.*]] = getelementptr inbounds i32, i32* [[P1_023]], i64 3
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast i32* [[P1_023]] to <4 x i32>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, <4 x i32>* [[TMP0]], align 4
+; CHECK-NEXT:    [[ARRAYIDX21:%.*]] = getelementptr inbounds i32, i32* [[P2_024]], i64 3
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i32* [[P2_024]] to <4 x i32>*
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, <4 x i32>* [[TMP2]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = sub nsw <4 x i32> [[TMP1]], [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp slt <4 x i32> [[TMP4]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = sub nsw <4 x i32> zeroinitializer, [[TMP4]]
+; CHECK-NEXT:    [[TMP7:%.*]] = select <4 x i1> [[TMP5]], <4 x i32> [[TMP6]], <4 x i32> [[TMP4]]
+; CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 undef, [[S_026]]
+; CHECK-NEXT:    [[ADD11:%.*]] = add nsw i32 [[ADD]], undef
+; CHECK-NEXT:    [[ADD19:%.*]] = add nsw i32 [[ADD11]], undef
+; CHECK-NEXT:    [[TMP8:%.*]] = call i32 @llvm.experimental.vector.reduce.add.i32.v4i32(<4 x i32> [[TMP7]])
+; CHECK-NEXT:    [[OP_EXTRA]] = add nsw i32 [[TMP8]], [[S_026]]
+; CHECK-NEXT:    [[ADD27:%.*]] = add nsw i32 [[ADD19]], undef
+; CHECK-NEXT:    [[ADD_PTR]] = getelementptr inbounds i32, i32* [[P1_023]], i64 [[IDX_EXT]]
+; CHECK-NEXT:    [[ADD_PTR29]] = getelementptr inbounds i32, i32* [[P2_024]], i64 [[IDX_EXT]]
+; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[J_025]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], [[H]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_END_LOOPEXIT:%.*]], label [[FOR_BODY]]
+; CHECK:       for.end.loopexit:
+; CHECK-NEXT:    br label [[FOR_END]]
+; CHECK:       for.end:
+; CHECK-NEXT:    [[S_0_LCSSA:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[OP_EXTRA]], [[FOR_END_LOOPEXIT]] ]
+; CHECK-NEXT:    ret i32 [[S_0_LCSSA]]
+;
 entry:
   %cmp.22 = icmp sgt i32 %h, 0
   br i1 %cmp.22, label %for.body.lr.ph, label %for.end
@@ -104,11 +141,6 @@ for.end:                                          ; preds = %for.end.loopexit, %
 ;;   p2 += lx;
 ;; }
 define i32 @reduction_with_br(i32* noalias nocapture readonly %blk1, i32* noalias nocapture readonly %blk2, i32 %lx, i32 %h, i32 %lim) {
-; CHECK-LABEL: reduction_with_br
-; CHECK: load <4 x i32>
-; CHECK: load <4 x i32>
-; CHECK: mul nsw <4 x i32>
-
 ; YAML:      --- !Passed
 ; YAML-NEXT: Pass:            slp-vectorizer
 ; YAML-NEXT: Name:            VectorizedHorizontalReduction
@@ -118,7 +150,49 @@ define i32 @reduction_with_br(i32* noalias nocapture readonly %blk1, i32* noalia
 ; YAML-NEXT:   - Cost:            '-11'
 ; YAML-NEXT:   - String:          ' and with tree size '
 ; YAML-NEXT:   - TreeSize:        '3'
-
+; CHECK-LABEL: @reduction_with_br(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP_16:%.*]] = icmp sgt i32 [[H:%.*]], 0
+; CHECK-NEXT:    br i1 [[CMP_16]], label [[FOR_BODY_LR_PH:%.*]], label [[FOR_END:%.*]]
+; CHECK:       for.body.lr.ph:
+; CHECK-NEXT:    [[IDX_EXT:%.*]] = sext i32 [[LX:%.*]] to i64
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[S_020:%.*]] = phi i32 [ 0, [[FOR_BODY_LR_PH]] ], [ [[OP_EXTRA:%.*]], [[IF_END:%.*]] ]
+; CHECK-NEXT:    [[J_019:%.*]] = phi i32 [ 0, [[FOR_BODY_LR_PH]] ], [ [[INC:%.*]], [[IF_END]] ]
+; CHECK-NEXT:    [[P2_018:%.*]] = phi i32* [ [[BLK2:%.*]], [[FOR_BODY_LR_PH]] ], [ [[ADD_PTR16:%.*]], [[IF_END]] ]
+; CHECK-NEXT:    [[P1_017:%.*]] = phi i32* [ [[BLK1:%.*]], [[FOR_BODY_LR_PH]] ], [ [[ADD_PTR:%.*]], [[IF_END]] ]
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, i32* [[P1_017]], i64 1
+; CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds i32, i32* [[P2_018]], i64 1
+; CHECK-NEXT:    [[ARRAYIDX6:%.*]] = getelementptr inbounds i32, i32* [[P1_017]], i64 2
+; CHECK-NEXT:    [[ARRAYIDX7:%.*]] = getelementptr inbounds i32, i32* [[P2_018]], i64 2
+; CHECK-NEXT:    [[ARRAYIDX10:%.*]] = getelementptr inbounds i32, i32* [[P1_017]], i64 3
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast i32* [[P1_017]] to <4 x i32>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, <4 x i32>* [[TMP0]], align 4
+; CHECK-NEXT:    [[ARRAYIDX11:%.*]] = getelementptr inbounds i32, i32* [[P2_018]], i64 3
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i32* [[P2_018]] to <4 x i32>*
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, <4 x i32>* [[TMP2]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = mul nsw <4 x i32> [[TMP3]], [[TMP1]]
+; CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 undef, [[S_020]]
+; CHECK-NEXT:    [[ADD5:%.*]] = add nsw i32 [[ADD]], undef
+; CHECK-NEXT:    [[ADD9:%.*]] = add nsw i32 [[ADD5]], undef
+; CHECK-NEXT:    [[TMP5:%.*]] = call i32 @llvm.experimental.vector.reduce.add.i32.v4i32(<4 x i32> [[TMP4]])
+; CHECK-NEXT:    [[OP_EXTRA]] = add nsw i32 [[TMP5]], [[S_020]]
+; CHECK-NEXT:    [[ADD13:%.*]] = add nsw i32 [[ADD9]], undef
+; CHECK-NEXT:    [[CMP14:%.*]] = icmp slt i32 [[OP_EXTRA]], [[LIM:%.*]]
+; CHECK-NEXT:    br i1 [[CMP14]], label [[IF_END]], label [[FOR_END_LOOPEXIT:%.*]]
+; CHECK:       if.end:
+; CHECK-NEXT:    [[ADD_PTR]] = getelementptr inbounds i32, i32* [[P1_017]], i64 [[IDX_EXT]]
+; CHECK-NEXT:    [[ADD_PTR16]] = getelementptr inbounds i32, i32* [[P2_018]], i64 [[IDX_EXT]]
+; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[J_019]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[INC]], [[H]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_END_LOOPEXIT]]
+; CHECK:       for.end.loopexit:
+; CHECK-NEXT:    br label [[FOR_END]]
+; CHECK:       for.end:
+; CHECK-NEXT:    [[S_1:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[OP_EXTRA]], [[FOR_END_LOOPEXIT]] ]
+; CHECK-NEXT:    ret i32 [[S_1]]
+;
 entry:
   %cmp.16 = icmp sgt i32 %h, 0
   br i1 %cmp.16, label %for.body.lr.ph, label %for.end
@@ -172,11 +246,6 @@ for.end:                                          ; preds = %for.end.loopexit, %
   ret i32 %s.1
 }
 
-; CHECK: test_unrolled_select
-; CHECK: load <8 x i8>
-; CHECK: load <8 x i8>
-; CHECK: select <8 x i1>
-
 ; YAML:      --- !Passed
 ; YAML-NEXT: Pass:            slp-vectorizer
 ; YAML-NEXT: Name:            VectorizedHorizontalReduction
@@ -188,6 +257,66 @@ for.end:                                          ; preds = %for.end.loopexit, %
 ; YAML-NEXT:   - TreeSize:        '10'
 
 define i32 @test_unrolled_select(i8* noalias nocapture readonly %blk1, i8* noalias nocapture readonly %blk2, i32 %lx, i32 %h, i32 %lim) #0 {
+; CHECK-LABEL: @test_unrolled_select(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP_43:%.*]] = icmp sgt i32 [[H:%.*]], 0
+; CHECK-NEXT:    br i1 [[CMP_43]], label [[FOR_BODY_LR_PH:%.*]], label [[FOR_END:%.*]]
+; CHECK:       for.body.lr.ph:
+; CHECK-NEXT:    [[IDX_EXT:%.*]] = sext i32 [[LX:%.*]] to i64
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[S_047:%.*]] = phi i32 [ 0, [[FOR_BODY_LR_PH]] ], [ [[OP_EXTRA:%.*]], [[IF_END_86:%.*]] ]
+; CHECK-NEXT:    [[J_046:%.*]] = phi i32 [ 0, [[FOR_BODY_LR_PH]] ], [ [[INC:%.*]], [[IF_END_86]] ]
+; CHECK-NEXT:    [[P2_045:%.*]] = phi i8* [ [[BLK2:%.*]], [[FOR_BODY_LR_PH]] ], [ [[ADD_PTR88:%.*]], [[IF_END_86]] ]
+; CHECK-NEXT:    [[P1_044:%.*]] = phi i8* [ [[BLK1:%.*]], [[FOR_BODY_LR_PH]] ], [ [[ADD_PTR:%.*]], [[IF_END_86]] ]
+; CHECK-NEXT:    [[ARRAYIDX6:%.*]] = getelementptr inbounds i8, i8* [[P1_044]], i64 1
+; CHECK-NEXT:    [[ARRAYIDX8:%.*]] = getelementptr inbounds i8, i8* [[P2_045]], i64 1
+; CHECK-NEXT:    [[ARRAYIDX17:%.*]] = getelementptr inbounds i8, i8* [[P1_044]], i64 2
+; CHECK-NEXT:    [[ARRAYIDX19:%.*]] = getelementptr inbounds i8, i8* [[P2_045]], i64 2
+; CHECK-NEXT:    [[ARRAYIDX28:%.*]] = getelementptr inbounds i8, i8* [[P1_044]], i64 3
+; CHECK-NEXT:    [[ARRAYIDX30:%.*]] = getelementptr inbounds i8, i8* [[P2_045]], i64 3
+; CHECK-NEXT:    [[ARRAYIDX39:%.*]] = getelementptr inbounds i8, i8* [[P1_044]], i64 4
+; CHECK-NEXT:    [[ARRAYIDX41:%.*]] = getelementptr inbounds i8, i8* [[P2_045]], i64 4
+; CHECK-NEXT:    [[ARRAYIDX50:%.*]] = getelementptr inbounds i8, i8* [[P1_044]], i64 5
+; CHECK-NEXT:    [[ARRAYIDX52:%.*]] = getelementptr inbounds i8, i8* [[P2_045]], i64 5
+; CHECK-NEXT:    [[ARRAYIDX61:%.*]] = getelementptr inbounds i8, i8* [[P1_044]], i64 6
+; CHECK-NEXT:    [[ARRAYIDX63:%.*]] = getelementptr inbounds i8, i8* [[P2_045]], i64 6
+; CHECK-NEXT:    [[ARRAYIDX72:%.*]] = getelementptr inbounds i8, i8* [[P1_044]], i64 7
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast i8* [[P1_044]] to <8 x i8>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i8>, <8 x i8>* [[TMP0]], align 1
+; CHECK-NEXT:    [[TMP2:%.*]] = zext <8 x i8> [[TMP1]] to <8 x i32>
+; CHECK-NEXT:    [[ARRAYIDX74:%.*]] = getelementptr inbounds i8, i8* [[P2_045]], i64 7
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i8* [[P2_045]] to <8 x i8>*
+; CHECK-NEXT:    [[TMP4:%.*]] = load <8 x i8>, <8 x i8>* [[TMP3]], align 1
+; CHECK-NEXT:    [[TMP5:%.*]] = zext <8 x i8> [[TMP4]] to <8 x i32>
+; CHECK-NEXT:    [[TMP6:%.*]] = sub nsw <8 x i32> [[TMP2]], [[TMP5]]
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp slt <8 x i32> [[TMP6]], zeroinitializer
+; CHECK-NEXT:    [[TMP8:%.*]] = sub nsw <8 x i32> zeroinitializer, [[TMP6]]
+; CHECK-NEXT:    [[TMP9:%.*]] = select <8 x i1> [[TMP7]], <8 x i32> [[TMP8]], <8 x i32> [[TMP6]]
+; CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 undef, [[S_047]]
+; CHECK-NEXT:    [[ADD16:%.*]] = add nsw i32 [[ADD]], undef
+; CHECK-NEXT:    [[ADD27:%.*]] = add nsw i32 [[ADD16]], undef
+; CHECK-NEXT:    [[ADD38:%.*]] = add nsw i32 [[ADD27]], undef
+; CHECK-NEXT:    [[ADD49:%.*]] = add nsw i32 [[ADD38]], undef
+; CHECK-NEXT:    [[ADD60:%.*]] = add nsw i32 [[ADD49]], undef
+; CHECK-NEXT:    [[ADD71:%.*]] = add nsw i32 [[ADD60]], undef
+; CHECK-NEXT:    [[TMP10:%.*]] = call i32 @llvm.experimental.vector.reduce.add.i32.v8i32(<8 x i32> [[TMP9]])
+; CHECK-NEXT:    [[OP_EXTRA]] = add nsw i32 [[TMP10]], [[S_047]]
+; CHECK-NEXT:    [[ADD82:%.*]] = add nsw i32 [[ADD71]], undef
+; CHECK-NEXT:    [[CMP83:%.*]] = icmp slt i32 [[OP_EXTRA]], [[LIM:%.*]]
+; CHECK-NEXT:    br i1 [[CMP83]], label [[IF_END_86]], label [[FOR_END_LOOPEXIT:%.*]]
+; CHECK:       if.end.86:
+; CHECK-NEXT:    [[ADD_PTR]] = getelementptr inbounds i8, i8* [[P1_044]], i64 [[IDX_EXT]]
+; CHECK-NEXT:    [[ADD_PTR88]] = getelementptr inbounds i8, i8* [[P2_045]], i64 [[IDX_EXT]]
+; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[J_046]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[INC]], [[H]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_END_LOOPEXIT]]
+; CHECK:       for.end.loopexit:
+; CHECK-NEXT:    br label [[FOR_END]]
+; CHECK:       for.end:
+; CHECK-NEXT:    [[S_1:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[OP_EXTRA]], [[FOR_END_LOOPEXIT]] ]
+; CHECK-NEXT:    ret i32 [[S_1]]
+;
 entry:
   %cmp.43 = icmp sgt i32 %h, 0
   br i1 %cmp.43, label %for.body.lr.ph, label %for.end
diff --git a/test/Transforms/SLPVectorizer/AArch64/matmul.ll b/test/Transforms/SLPVectorizer/AArch64/matmul.ll
index 17beea4f42ebb41b7df1d0399a8db75bb03645e2..fdd12c1e807e010d65284dc57fd143820e31489f 100644
--- a/test/Transforms/SLPVectorizer/AArch64/matmul.ll
+++ b/test/Transforms/SLPVectorizer/AArch64/matmul.ll
@@ -4,7 +4,7 @@
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 
 ; This test is reduced from the matrix multiplication benchmark in the test-suite:
-; http://www.llvm.org/viewvc/llvm-project/test-suite/trunk/SingleSource/Benchmarks/Misc/matmul_f64_4x4.c
+; https://github.com/llvm/llvm-test-suite/tree/master/SingleSource/Benchmarks/Misc/matmul_f64_4x4.c
 ; The operations here are expected to be vectorized to <2 x double>.
 ; Otherwise, performance will suffer on Cortex-A53.
 
diff --git a/test/Transforms/SLPVectorizer/AArch64/minimum-sizes.ll b/test/Transforms/SLPVectorizer/AArch64/minimum-sizes.ll
index 7e1d670956645eaca91b703fab843d236f2a4dd6..0429665c0aef271623446e1c0d774bfda231dfcf 100644
--- a/test/Transforms/SLPVectorizer/AArch64/minimum-sizes.ll
+++ b/test/Transforms/SLPVectorizer/AArch64/minimum-sizes.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -S -slp-vectorizer < %s | FileCheck %s
 
 target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
@@ -7,10 +8,19 @@ target triple = "aarch64--linux-gnu"
 ; should not compute a smaller size for %k.13 since it is in a use-def cycle
 ; and cannot be demoted.
 ;
-; CHECK-LABEL: @PR26364
-; CHECK: %k.13 = phi i32
-;
 define fastcc void @PR26364() {
+; CHECK-LABEL: @PR26364(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 undef, label [[FOR_END11:%.*]], label [[FOR_COND4:%.*]]
+; CHECK:       for.cond4:
+; CHECK-NEXT:    [[K_13:%.*]] = phi i32 [ undef, [[ENTRY:%.*]] ], [ [[K_3:%.*]], [[FOR_COND4]] ]
+; CHECK-NEXT:    [[E_02:%.*]] = phi i32 [ 1, [[ENTRY]] ], [ 0, [[FOR_COND4]] ]
+; CHECK-NEXT:    [[E_1:%.*]] = select i1 undef, i32 [[E_02]], i32 0
+; CHECK-NEXT:    [[K_3]] = select i1 undef, i32 [[K_13]], i32 undef
+; CHECK-NEXT:    br label [[FOR_COND4]]
+; CHECK:       for.end11:
+; CHECK-NEXT:    ret void
+;
 entry:
   br i1 undef, label %for.end11, label %for.cond4
 
@@ -29,10 +39,25 @@ for.end11:
 ; every root in the vectorizable tree when computing minimum sizes since one
 ; root may require fewer bits than another.
 ;
-; CHECK-LABEL: @PR26629
-; CHECK-NOT: {{.*}} and <2 x i72>
-;
 define void @PR26629(i32* %c) {
+; CHECK-LABEL: @PR26629(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 undef, label [[FOR_PH:%.*]], label [[FOR_END:%.*]]
+; CHECK:       for.ph:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[C:%.*]], align 4
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[D:%.*]] = phi i72 [ 576507472957710340, [[FOR_PH]] ], [ [[BF_SET17:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[SUB:%.*]] = sub i32 [[TMP0]], undef
+; CHECK-NEXT:    [[BF_CLEAR13:%.*]] = and i72 [[D]], -576460748008464384
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i32 [[SUB]] to i72
+; CHECK-NEXT:    [[BF_VALUE15:%.*]] = and i72 [[TMP1]], 8191
+; CHECK-NEXT:    [[BF_CLEAR16:%.*]] = or i72 [[BF_VALUE15]], [[BF_CLEAR13]]
+; CHECK-NEXT:    [[BF_SET17]] = or i72 [[BF_CLEAR16]], undef
+; CHECK-NEXT:    br label [[FOR_BODY]]
+; CHECK:       for.end:
+; CHECK-NEXT:    ret void
+;
 entry:
   br i1 undef, label %for.ph, label %for.end
 
diff --git a/test/Transforms/SLPVectorizer/AArch64/mismatched-intrinsics.ll b/test/Transforms/SLPVectorizer/AArch64/mismatched-intrinsics.ll
index 3d6da124fc45ff4dc047fd8b528eff09397229d3..64b8743f8bd2a97a8818b79de20859270ddce90d 100644
--- a/test/Transforms/SLPVectorizer/AArch64/mismatched-intrinsics.ll
+++ b/test/Transforms/SLPVectorizer/AArch64/mismatched-intrinsics.ll
@@ -1,11 +1,16 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -S -slp-vectorizer %s | FileCheck %s
 target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
 target triple = "arm64-apple-ios5.0.0"
 
 define i64 @mismatched_intrinsics(<4 x i32> %in1, <2 x i32> %in2) nounwind {
-; CHECK-LABEL: @mismatched_intrinsics
-; CHECK: call i64 @llvm.arm64.neon.saddlv.i64.v4i32
-; CHECK: call i64 @llvm.arm64.neon.saddlv.i64.v2i32
+; CHECK-LABEL: @mismatched_intrinsics(
+; CHECK-NEXT:    [[VADDLVQ_S32_I:%.*]] = tail call i64 @llvm.arm64.neon.saddlv.i64.v4i32(<4 x i32> [[IN1:%.*]])
+; CHECK-NEXT:    [[VADDLV_S32_I:%.*]] = tail call i64 @llvm.arm64.neon.saddlv.i64.v2i32(<2 x i32> [[IN2:%.*]])
+; CHECK-NEXT:    [[TST:%.*]] = icmp sgt i64 [[VADDLVQ_S32_I]], [[VADDLV_S32_I]]
+; CHECK-NEXT:    [[EQUAL:%.*]] = sext i1 [[TST]] to i64
+; CHECK-NEXT:    ret i64 [[EQUAL]]
+;
 
   %vaddlvq_s32.i = tail call i64 @llvm.arm64.neon.saddlv.i64.v4i32(<4 x i32> %in1) #2
   %vaddlv_s32.i = tail call i64 @llvm.arm64.neon.saddlv.i64.v2i32(<2 x i32> %in2) #2
diff --git a/test/Transforms/SLPVectorizer/AArch64/nontemporal.ll b/test/Transforms/SLPVectorizer/AArch64/nontemporal.ll
index 87d021d534cff274ce4cfdb80cf9376d3c7fa605..98c033262a080b64ff4b37b1621661fe1f23b4bc 100644
--- a/test/Transforms/SLPVectorizer/AArch64/nontemporal.ll
+++ b/test/Transforms/SLPVectorizer/AArch64/nontemporal.ll
@@ -1,12 +1,22 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -S -basicaa -slp-vectorizer -dce < %s | FileCheck %s
 target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
 target triple = "arm64-apple-ios5.0.0"
 
-; CHECK-LABEL: @foo
 define void @foo(float* noalias %a, float* noalias %b, float* noalias %c) {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast float* [[B:%.*]] to <4 x float>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x float>, <4 x float>* [[TMP0]], align 4, !nontemporal !0
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast float* [[C:%.*]] to <4 x float>*
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x float>, <4 x float>* [[TMP2]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = fadd <4 x float> [[TMP1]], [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast float* [[A:%.*]] to <4 x float>*
+; CHECK-NEXT:    store <4 x float> [[TMP4]], <4 x float>* [[TMP5]], align 4, !nontemporal !0
+; CHECK-NEXT:    ret void
+;
 entry:
 ; Check that we don't lose !nontemporal hint when vectorizing loads.
-; CHECK: %{{[0-9]*}} = load <4 x float>, <4 x float>* %{{[0-9]+}}, align 4, !nontemporal !0
   %b1 = load float, float* %b, align 4, !nontemporal !0
   %arrayidx.1 = getelementptr inbounds float, float* %b, i64 1
   %b2 = load float, float* %arrayidx.1, align 4, !nontemporal !0
@@ -16,7 +26,6 @@ entry:
   %b4 = load float, float* %arrayidx.3, align 4, !nontemporal !0
 
 ; Check that we don't introduce !nontemporal hint when the original scalar loads didn't have it.
-; CHECK: %{{[0-9]*}} = load <4 x float>, <4 x float>* %{{[0-9]+}}, align 4{{$}}
   %c1 = load float, float* %c, align 4
   %arrayidx2.1 = getelementptr inbounds float, float* %c, i64 1
   %c2 = load float, float* %arrayidx2.1, align 4
@@ -31,7 +40,6 @@ entry:
   %a4 = fadd float %b4, %c4
 
 ; Check that we don't lose !nontemporal hint when vectorizing stores.
-; CHECK: store <4 x float> %{{[0-9]+}}, <4 x float>* %{{[0-9]+}}, align 4, !nontemporal !0
   store float %a1, float* %a, align 4, !nontemporal !0
   %arrayidx3.1 = getelementptr inbounds float, float* %a, i64 1
   store float %a2, float* %arrayidx3.1, align 4, !nontemporal !0
@@ -40,16 +48,21 @@ entry:
   %arrayidx3.3 = getelementptr inbounds float, float* %a, i64 3
   store float %a4, float* %arrayidx3.3, align 4, !nontemporal !0
 
-; CHECK: ret void
   ret void
 }
 
-; CHECK-LABEL: @foo2
 define void @foo2(float* noalias %a, float* noalias %b) {
+; CHECK-LABEL: @foo2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast float* [[B:%.*]] to <4 x float>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x float>, <4 x float>* [[TMP0]], align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast float* [[A:%.*]] to <4 x float>*
+; CHECK-NEXT:    store <4 x float> [[TMP1]], <4 x float>* [[TMP2]], align 4
+; CHECK-NEXT:    ret void
+;
 entry:
 ; Check that we don't mark vector load with !nontemporal attribute if some of
 ; the original scalar loads don't have it.
-; CHECK: %{{[0-9]*}} = load <4 x float>, <4 x float>* %{{[0-9]+}}, align 4{{$}}
   %b1 = load float, float* %b, align 4, !nontemporal !0
   %arrayidx.1 = getelementptr inbounds float, float* %b, i64 1
   %b2 = load float, float* %arrayidx.1, align 4
@@ -60,7 +73,6 @@ entry:
 
 ; Check that we don't mark vector store with !nontemporal attribute if some of
 ; the original scalar stores don't have it.
-; CHECK: store <4 x float> %{{[0-9]+}}, <4 x float>* %{{[0-9]+}}, align 4{{$}}
   store float %b1, float* %a, align 4, !nontemporal !0
   %arrayidx3.1 = getelementptr inbounds float, float* %a, i64 1
   store float %b2, float* %arrayidx3.1, align 4
@@ -69,7 +81,6 @@ entry:
   %arrayidx3.3 = getelementptr inbounds float, float* %a, i64 3
   store float %b4, float* %arrayidx3.3, align 4, !nontemporal !0
 
-; CHECK: ret void
   ret void
 }
 
diff --git a/test/Transforms/SLPVectorizer/AArch64/sdiv-pow2.ll b/test/Transforms/SLPVectorizer/AArch64/sdiv-pow2.ll
index 72c70823e6988e667410dbbfc6c587909c7df8cd..8311f20514927478c9d27f70baf544265f45801c 100644
--- a/test/Transforms/SLPVectorizer/AArch64/sdiv-pow2.ll
+++ b/test/Transforms/SLPVectorizer/AArch64/sdiv-pow2.ll
@@ -1,13 +1,30 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -basicaa -slp-vectorizer -S -mtriple=aarch64-unknown-linux-gnu -mcpu=cortex-a57 | FileCheck %s
 target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
 target triple = "aarch64--linux-gnu"
 
-; CHECK-LABEL: @test1
-; CHECK: load <4 x i32>
-; CHECK: add nsw <4 x i32>
-; CHECK: sdiv <4 x i32>
-
 define void @test1(i32* noalias nocapture %a, i32* noalias nocapture readonly %b, i32* noalias nocapture readonly %c) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds i32, i32* [[B:%.*]], i64 1
+; CHECK-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds i32, i32* [[C:%.*]], i64 1
+; CHECK-NEXT:    [[ARRAYIDX7:%.*]] = getelementptr inbounds i32, i32* [[A:%.*]], i64 1
+; CHECK-NEXT:    [[ARRAYIDX8:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 2
+; CHECK-NEXT:    [[ARRAYIDX9:%.*]] = getelementptr inbounds i32, i32* [[C]], i64 2
+; CHECK-NEXT:    [[ARRAYIDX12:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 2
+; CHECK-NEXT:    [[ARRAYIDX13:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 3
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast i32* [[B]] to <4 x i32>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, <4 x i32>* [[TMP0]], align 4
+; CHECK-NEXT:    [[ARRAYIDX14:%.*]] = getelementptr inbounds i32, i32* [[C]], i64 3
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i32* [[C]] to <4 x i32>*
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, <4 x i32>* [[TMP2]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = add nsw <4 x i32> [[TMP3]], [[TMP1]]
+; CHECK-NEXT:    [[TMP5:%.*]] = sdiv <4 x i32> [[TMP4]], <i32 2, i32 2, i32 2, i32 2>
+; CHECK-NEXT:    [[ARRAYIDX17:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 3
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i32* [[A]] to <4 x i32>*
+; CHECK-NEXT:    store <4 x i32> [[TMP5]], <4 x i32>* [[TMP6]], align 4
+; CHECK-NEXT:    ret void
+;
 entry:
   %0 = load i32, i32* %b, align 4
   %1 = load i32, i32* %c, align 4
diff --git a/test/Transforms/SLPVectorizer/AArch64/tsc-s352.ll b/test/Transforms/SLPVectorizer/AArch64/tsc-s352.ll
index 9ca039a00b250329a7011dc12d0642bfc152fdba..6d7a6a200dd0f7f7347eca6b42004f599ab650b5 100644
--- a/test/Transforms/SLPVectorizer/AArch64/tsc-s352.ll
+++ b/test/Transforms/SLPVectorizer/AArch64/tsc-s352.ll
@@ -4,7 +4,7 @@
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
 
 ; This test is reduced from the TSVC evaluation of vectorizers:
-; http://www.llvm.org/viewvc/llvm-project/test-suite/trunk/MultiSource/Benchmarks/TSVC/LoopRerolling-flt/tsc.c?view=log
+; https://github.com/llvm/llvm-test-suite/commits/master/MultiSource/Benchmarks/TSVC/LoopRerolling-flt/tsc.c
 ; Two loads and an fmul are expected to be vectorized to <2 x float>.
 ; Otherwise, performance will suffer on Cortex-A53.
 ; See https://bugs.llvm.org/show_bug.cgi?id=36280 for more details.
diff --git a/test/Transforms/SLPVectorizer/ARM/memory.ll b/test/Transforms/SLPVectorizer/ARM/memory.ll
index 57d7cceac6b9bee2d667f6039f702e323d18742f..70e87033207a4e8ccca53e3746de29411ddcc449 100644
--- a/test/Transforms/SLPVectorizer/ARM/memory.ll
+++ b/test/Transforms/SLPVectorizer/ARM/memory.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -basicaa -slp-vectorizer -S -mtriple=thumbv7-apple-ios3.0.0 -mcpu=swift | FileCheck %s
 
 target datalayout = "e-p:32:32:32-i1:8:32-i8:8:32-i16:16:32-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:32:64-v128:32:128-a0:0:32-n32-S32"
@@ -5,10 +6,17 @@ target datalayout = "e-p:32:32:32-i1:8:32-i8:8:32-i16:16:32-i32:32:32-i64:32:64-
 ; On swift unaligned <2 x double> stores need 4uops and it is there for cheaper
 ; to do this scalar.
 
-; CHECK-LABEL: expensive_double_store
-; CHECK-NOT: load <2 x double>
-; CHECK-NOT: store <2 x double>
 define void @expensive_double_store(double* noalias %dst, double* noalias %src, i64 %count) {
+; CHECK-LABEL: @expensive_double_store(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load double, double* [[SRC:%.*]], align 8
+; CHECK-NEXT:    store double [[TMP0]], double* [[DST:%.*]], align 8
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds double, double* [[SRC]], i64 1
+; CHECK-NEXT:    [[TMP1:%.*]] = load double, double* [[ARRAYIDX2]], align 8
+; CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds double, double* [[DST]], i64 1
+; CHECK-NEXT:    store double [[TMP1]], double* [[ARRAYIDX3]], align 8
+; CHECK-NEXT:    ret void
+;
 entry:
   %0 = load double, double* %src, align 8
   store double %0, double* %dst, align 8
diff --git a/test/Transforms/SLPVectorizer/ARM/sroa.ll b/test/Transforms/SLPVectorizer/ARM/sroa.ll
index 65e02608b2905a90b1f719d923e20190e7db5f4c..c43e8f10a59b8b01dbd00ad0d994513b30137d33 100644
--- a/test/Transforms/SLPVectorizer/ARM/sroa.ll
+++ b/test/Transforms/SLPVectorizer/ARM/sroa.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -S -mcpu=swift -mtriple=thumbv7-apple-ios -basicaa -slp-vectorizer < %s | FileCheck %s
 
 target datalayout = "e-p:32:32:32-i1:8:32-i8:8:32-i16:16:32-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:32:64-v128:32:128-a0:0:32-n32-S32"
@@ -8,11 +9,45 @@ target datalayout = "e-p:32:32:32-i1:8:32-i8:8:32-i16:16:32-i32:32:32-i64:32:64-
 ; because the scalar version of the shl/or are handled by the
 ; backend and disappear, the vectorized code stays.
 
-; CHECK-LABEL: SROAed
-; CHECK-NOT: shl nuw <2 x i64>
-; CHECK-NOT: or <2 x i64>
-
 define void @SROAed(%class.Complex* noalias nocapture sret %agg.result, [4 x i32] %a.coerce, [4 x i32] %b.coerce) {
+; CHECK-LABEL: @SROAed(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[A_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [4 x i32] [[A_COERCE:%.*]], 0
+; CHECK-NEXT:    [[A_SROA_0_0_INSERT_EXT:%.*]] = zext i32 [[A_COERCE_FCA_0_EXTRACT]] to i64
+; CHECK-NEXT:    [[A_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [4 x i32] [[A_COERCE]], 1
+; CHECK-NEXT:    [[A_SROA_0_4_INSERT_EXT:%.*]] = zext i32 [[A_COERCE_FCA_1_EXTRACT]] to i64
+; CHECK-NEXT:    [[A_SROA_0_4_INSERT_SHIFT:%.*]] = shl nuw i64 [[A_SROA_0_4_INSERT_EXT]], 32
+; CHECK-NEXT:    [[A_SROA_0_4_INSERT_INSERT:%.*]] = or i64 [[A_SROA_0_4_INSERT_SHIFT]], [[A_SROA_0_0_INSERT_EXT]]
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast i64 [[A_SROA_0_4_INSERT_INSERT]] to double
+; CHECK-NEXT:    [[A_COERCE_FCA_2_EXTRACT:%.*]] = extractvalue [4 x i32] [[A_COERCE]], 2
+; CHECK-NEXT:    [[A_SROA_3_8_INSERT_EXT:%.*]] = zext i32 [[A_COERCE_FCA_2_EXTRACT]] to i64
+; CHECK-NEXT:    [[A_COERCE_FCA_3_EXTRACT:%.*]] = extractvalue [4 x i32] [[A_COERCE]], 3
+; CHECK-NEXT:    [[A_SROA_3_12_INSERT_EXT:%.*]] = zext i32 [[A_COERCE_FCA_3_EXTRACT]] to i64
+; CHECK-NEXT:    [[A_SROA_3_12_INSERT_SHIFT:%.*]] = shl nuw i64 [[A_SROA_3_12_INSERT_EXT]], 32
+; CHECK-NEXT:    [[A_SROA_3_12_INSERT_INSERT:%.*]] = or i64 [[A_SROA_3_12_INSERT_SHIFT]], [[A_SROA_3_8_INSERT_EXT]]
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i64 [[A_SROA_3_12_INSERT_INSERT]] to double
+; CHECK-NEXT:    [[B_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [4 x i32] [[B_COERCE:%.*]], 0
+; CHECK-NEXT:    [[B_SROA_0_0_INSERT_EXT:%.*]] = zext i32 [[B_COERCE_FCA_0_EXTRACT]] to i64
+; CHECK-NEXT:    [[B_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [4 x i32] [[B_COERCE]], 1
+; CHECK-NEXT:    [[B_SROA_0_4_INSERT_EXT:%.*]] = zext i32 [[B_COERCE_FCA_1_EXTRACT]] to i64
+; CHECK-NEXT:    [[B_SROA_0_4_INSERT_SHIFT:%.*]] = shl nuw i64 [[B_SROA_0_4_INSERT_EXT]], 32
+; CHECK-NEXT:    [[B_SROA_0_4_INSERT_INSERT:%.*]] = or i64 [[B_SROA_0_4_INSERT_SHIFT]], [[B_SROA_0_0_INSERT_EXT]]
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i64 [[B_SROA_0_4_INSERT_INSERT]] to double
+; CHECK-NEXT:    [[B_COERCE_FCA_2_EXTRACT:%.*]] = extractvalue [4 x i32] [[B_COERCE]], 2
+; CHECK-NEXT:    [[B_SROA_3_8_INSERT_EXT:%.*]] = zext i32 [[B_COERCE_FCA_2_EXTRACT]] to i64
+; CHECK-NEXT:    [[B_COERCE_FCA_3_EXTRACT:%.*]] = extractvalue [4 x i32] [[B_COERCE]], 3
+; CHECK-NEXT:    [[B_SROA_3_12_INSERT_EXT:%.*]] = zext i32 [[B_COERCE_FCA_3_EXTRACT]] to i64
+; CHECK-NEXT:    [[B_SROA_3_12_INSERT_SHIFT:%.*]] = shl nuw i64 [[B_SROA_3_12_INSERT_EXT]], 32
+; CHECK-NEXT:    [[B_SROA_3_12_INSERT_INSERT:%.*]] = or i64 [[B_SROA_3_12_INSERT_SHIFT]], [[B_SROA_3_8_INSERT_EXT]]
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i64 [[B_SROA_3_12_INSERT_INSERT]] to double
+; CHECK-NEXT:    [[ADD:%.*]] = fadd double [[TMP0]], [[TMP2]]
+; CHECK-NEXT:    [[ADD3:%.*]] = fadd double [[TMP1]], [[TMP3]]
+; CHECK-NEXT:    [[RE_I_I:%.*]] = getelementptr inbounds [[CLASS_COMPLEX:%.*]], %class.Complex* [[AGG_RESULT:%.*]], i32 0, i32 0
+; CHECK-NEXT:    store double [[ADD]], double* [[RE_I_I]], align 4
+; CHECK-NEXT:    [[IM_I_I:%.*]] = getelementptr inbounds [[CLASS_COMPLEX]], %class.Complex* [[AGG_RESULT]], i32 0, i32 1
+; CHECK-NEXT:    store double [[ADD3]], double* [[IM_I_I]], align 4
+; CHECK-NEXT:    ret void
+;
 entry:
   %a.coerce.fca.0.extract = extractvalue [4 x i32] %a.coerce, 0
   %a.sroa.0.0.insert.ext = zext i32 %a.coerce.fca.0.extract to i64
diff --git a/test/Transforms/SLPVectorizer/X86/non-vectorizable-intrinsic.ll b/test/Transforms/SLPVectorizer/NVPTX/non-vectorizable-intrinsic.ll
similarity index 53%
rename from test/Transforms/SLPVectorizer/X86/non-vectorizable-intrinsic.ll
rename to test/Transforms/SLPVectorizer/NVPTX/non-vectorizable-intrinsic.ll
index b250735874c5881a645b0d7ed11aec2f98eed2c8..3eae2d0b010896182b578f6814bfe3a7e776a20b 100644
--- a/test/Transforms/SLPVectorizer/X86/non-vectorizable-intrinsic.ll
+++ b/test/Transforms/SLPVectorizer/NVPTX/non-vectorizable-intrinsic.ll
@@ -1,4 +1,5 @@
-; RUN: opt < %s -slp-vectorizer -o - -S -slp-threshold=-1000
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -slp-vectorizer -o - -S -slp-threshold=-1000 | FileCheck %s
 
 target datalayout = "e-p:32:32-i64:64-v16:16-v32:32-n16:32:64"
 target triple = "nvptx--nvidiacl"
@@ -9,6 +10,16 @@ target triple = "nvptx--nvidiacl"
 ; Test causes an assert if LLVM tries to vectorize CTLZ.
 
 define <2 x i8> @cltz_test(<2 x i8> %x) #0 {
+; CHECK-LABEL: @cltz_test(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = extractelement <2 x i8> [[X:%.*]], i32 0
+; CHECK-NEXT:    [[CALL_I:%.*]] = call i8 @llvm.ctlz.i8(i8 [[TMP0]], i1 false)
+; CHECK-NEXT:    [[VECINIT:%.*]] = insertelement <2 x i8> undef, i8 [[CALL_I]], i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <2 x i8> [[X]], i32 1
+; CHECK-NEXT:    [[CALL_I4:%.*]] = call i8 @llvm.ctlz.i8(i8 [[TMP1]], i1 false)
+; CHECK-NEXT:    [[VECINIT2:%.*]] = insertelement <2 x i8> [[VECINIT]], i8 [[CALL_I4]], i32 1
+; CHECK-NEXT:    ret <2 x i8> [[VECINIT2]]
+;
 entry:
   %0 = extractelement <2 x i8> %x, i32 0
   %call.i = call i8 @llvm.ctlz.i8(i8 %0, i1 false)
@@ -20,6 +31,16 @@ entry:
 }
 
 define <2 x i8> @cltz_test2(<2 x i8> %x) #1 {
+; CHECK-LABEL: @cltz_test2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = extractelement <2 x i8> [[X:%.*]], i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <2 x i8> [[X]], i32 1
+; CHECK-NEXT:    [[CALL_I:%.*]] = call i8 @llvm.ctlz.i8(i8 [[TMP0]], i1 false)
+; CHECK-NEXT:    [[CALL_I4:%.*]] = call i8 @llvm.ctlz.i8(i8 [[TMP1]], i1 false)
+; CHECK-NEXT:    [[VECINIT:%.*]] = insertelement <2 x i8> undef, i8 [[CALL_I]], i32 0
+; CHECK-NEXT:    [[VECINIT2:%.*]] = insertelement <2 x i8> [[VECINIT]], i8 [[CALL_I4]], i32 1
+; CHECK-NEXT:    ret <2 x i8> [[VECINIT2]]
+;
 entry:
   %0 = extractelement <2 x i8> %x, i32 0
   %1 = extractelement <2 x i8> %x, i32 1
diff --git a/test/Transforms/SLPVectorizer/NVPTX/v2f16.ll b/test/Transforms/SLPVectorizer/NVPTX/v2f16.ll
index d8b80f437b92195848866963623d31493daec823..7038b0f8e2762ec3c946f38dbd2e6523e696ca3f 100644
--- a/test/Transforms/SLPVectorizer/NVPTX/v2f16.ll
+++ b/test/Transforms/SLPVectorizer/NVPTX/v2f16.ll
@@ -1,20 +1,50 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -basicaa -slp-vectorizer -S -mtriple=nvptx64-nvidia-cuda -mcpu=sm_70 | FileCheck %s
 ; RUN: opt < %s -basicaa -slp-vectorizer -S -mtriple=nvptx64-nvidia-cuda -mcpu=sm_40 | FileCheck %s -check-prefix=NOVECTOR
 
-; CHECK-LABEL: @fusion
-; CHECK: load <2 x half>, <2 x half>*
-; CHECK: fmul fast <2 x half>
-; CHECK: fadd fast <2 x half>
-; CHECK: store <2 x half> %4, <2 x half>
-
-; NOVECTOR-LABEL: @fusion
-; NOVECTOR: load half
-; NOVECTOR: fmul fast half
-; NOVECTOR: fadd fast half
-; NOVECTOR: fmul fast half
-; NOVECTOR: fadd fast half
-; NOVECTOR: store half
 define void @fusion(i8* noalias nocapture align 256 dereferenceable(19267584) %arg, i8* noalias nocapture readonly align 256 dereferenceable(19267584) %arg1, i32 %arg2, i32 %arg3) local_unnamed_addr #0 {
+; CHECK-LABEL: @fusion(
+; CHECK-NEXT:    [[TMP:%.*]] = shl nuw nsw i32 [[ARG2:%.*]], 6
+; CHECK-NEXT:    [[TMP4:%.*]] = or i32 [[TMP]], [[ARG3:%.*]]
+; CHECK-NEXT:    [[TMP5:%.*]] = shl nuw nsw i32 [[TMP4]], 2
+; CHECK-NEXT:    [[TMP6:%.*]] = zext i32 [[TMP5]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = or i64 [[TMP6]], 1
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i8* [[ARG1:%.*]] to half*
+; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr inbounds half, half* [[TMP10]], i64 [[TMP6]]
+; CHECK-NEXT:    [[TMP15:%.*]] = bitcast i8* [[ARG:%.*]] to half*
+; CHECK-NEXT:    [[TMP16:%.*]] = getelementptr inbounds half, half* [[TMP15]], i64 [[TMP6]]
+; CHECK-NEXT:    [[TMP17:%.*]] = getelementptr inbounds half, half* [[TMP10]], i64 [[TMP7]]
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast half* [[TMP11]] to <2 x half>*
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x half>, <2 x half>* [[TMP1]], align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = fmul fast <2 x half> <half 0xH5380, half 0xH5380>, [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = fadd fast <2 x half> <half 0xH57F0, half 0xH57F0>, [[TMP3]]
+; CHECK-NEXT:    [[TMP21:%.*]] = getelementptr inbounds half, half* [[TMP15]], i64 [[TMP7]]
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast half* [[TMP16]] to <2 x half>*
+; CHECK-NEXT:    store <2 x half> [[TMP4]], <2 x half>* [[TMP5]], align 8
+; CHECK-NEXT:    ret void
+;
+; NOVECTOR-LABEL: @fusion(
+; NOVECTOR-NEXT:    [[TMP:%.*]] = shl nuw nsw i32 [[ARG2:%.*]], 6
+; NOVECTOR-NEXT:    [[TMP4:%.*]] = or i32 [[TMP]], [[ARG3:%.*]]
+; NOVECTOR-NEXT:    [[TMP5:%.*]] = shl nuw nsw i32 [[TMP4]], 2
+; NOVECTOR-NEXT:    [[TMP6:%.*]] = zext i32 [[TMP5]] to i64
+; NOVECTOR-NEXT:    [[TMP7:%.*]] = or i64 [[TMP6]], 1
+; NOVECTOR-NEXT:    [[TMP10:%.*]] = bitcast i8* [[ARG1:%.*]] to half*
+; NOVECTOR-NEXT:    [[TMP11:%.*]] = getelementptr inbounds half, half* [[TMP10]], i64 [[TMP6]]
+; NOVECTOR-NEXT:    [[TMP12:%.*]] = load half, half* [[TMP11]], align 8
+; NOVECTOR-NEXT:    [[TMP13:%.*]] = fmul fast half [[TMP12]], 0xH5380
+; NOVECTOR-NEXT:    [[TMP14:%.*]] = fadd fast half [[TMP13]], 0xH57F0
+; NOVECTOR-NEXT:    [[TMP15:%.*]] = bitcast i8* [[ARG:%.*]] to half*
+; NOVECTOR-NEXT:    [[TMP16:%.*]] = getelementptr inbounds half, half* [[TMP15]], i64 [[TMP6]]
+; NOVECTOR-NEXT:    store half [[TMP14]], half* [[TMP16]], align 8
+; NOVECTOR-NEXT:    [[TMP17:%.*]] = getelementptr inbounds half, half* [[TMP10]], i64 [[TMP7]]
+; NOVECTOR-NEXT:    [[TMP18:%.*]] = load half, half* [[TMP17]], align 2
+; NOVECTOR-NEXT:    [[TMP19:%.*]] = fmul fast half [[TMP18]], 0xH5380
+; NOVECTOR-NEXT:    [[TMP20:%.*]] = fadd fast half [[TMP19]], 0xH57F0
+; NOVECTOR-NEXT:    [[TMP21:%.*]] = getelementptr inbounds half, half* [[TMP15]], i64 [[TMP7]]
+; NOVECTOR-NEXT:    store half [[TMP20]], half* [[TMP21]], align 2
+; NOVECTOR-NEXT:    ret void
+;
   %tmp = shl nuw nsw i32 %arg2, 6
   %tmp4 = or i32 %tmp, %arg3
   %tmp5 = shl nuw nsw i32 %tmp4, 2
diff --git a/test/Transforms/SLPVectorizer/PowerPC/pr27897.ll b/test/Transforms/SLPVectorizer/PowerPC/pr27897.ll
index dabb3380ef1c08453d3780664b0cda5573f3f9e9..7f7df82a38fc3bc17d1ac6f703db791bd97fa6cf 100644
--- a/test/Transforms/SLPVectorizer/PowerPC/pr27897.ll
+++ b/test/Transforms/SLPVectorizer/PowerPC/pr27897.ll
@@ -1,8 +1,28 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -S -mtriple=powerpc64-linux-gnu -mcpu=pwr8 -mattr=+vsx -slp-vectorizer < %s | FileCheck %s
 
 %struct.A = type { i8*, i8* }
 
 define i64 @foo(%struct.A* nocapture readonly %this) {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[END_I:%.*]] = getelementptr inbounds [[STRUCT_A:%.*]], %struct.A* [[THIS:%.*]], i64 0, i32 1
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast i8** [[END_I]] to i64*
+; CHECK-NEXT:    [[TMP1:%.*]] = load i64, i64* [[TMP0]], align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast %struct.A* [[THIS]] to i64*
+; CHECK-NEXT:    [[TMP3:%.*]] = load i64, i64* [[TMP2]], align 8
+; CHECK-NEXT:    [[SUB_PTR_SUB_I:%.*]] = sub i64 [[TMP1]], [[TMP3]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i64 [[SUB_PTR_SUB_I]], 9
+; CHECK-NEXT:    br i1 [[CMP]], label [[RETURN:%.*]], label [[LOR_LHS_FALSE:%.*]]
+; CHECK:       lor.lhs.false:
+; CHECK-NEXT:    [[TMP4:%.*]] = inttoptr i64 [[TMP3]] to i8*
+; CHECK-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[TMP1]] to i8*
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp ugt i8* [[TMP5]], [[TMP4]]
+; CHECK-NEXT:    [[DOT:%.*]] = select i1 [[CMP2]], i64 2, i64 -1
+; CHECK-NEXT:    ret i64 [[DOT]]
+; CHECK:       return:
+; CHECK-NEXT:    ret i64 2
+;
 entry:
   %end.i = getelementptr inbounds %struct.A, %struct.A* %this, i64 0, i32 1
   %0 = bitcast i8** %end.i to i64*
@@ -24,6 +44,3 @@ return:
   ret i64 2
 }
 
-; CHECK: load i64
-; CHECK-NOT: load <2 x i64>
-; CHECK-NOT: extractelement
diff --git a/test/Transforms/SLPVectorizer/PowerPC/short-to-double.ll b/test/Transforms/SLPVectorizer/PowerPC/short-to-double.ll
new file mode 100644
index 0000000000000000000000000000000000000000..203f47c76d4095bb7a1d1e27ae9da6104cc2d066
--- /dev/null
+++ b/test/Transforms/SLPVectorizer/PowerPC/short-to-double.ll
@@ -0,0 +1,39 @@
+; RUN: opt -S -mtriple=powerpc64-linux-gnu -mcpu=pwr9 -mattr=+vsx -slp-vectorizer < %s | FileCheck %s --check-prefix=CHECK-P9
+; RUN: opt -S -mtriple=powerpc64-linux-gnu -mcpu=pwr8 -mattr=+vsx -slp-vectorizer < %s | FileCheck %s --check-prefix=CHECK-P8
+
+%struct._pp = type { i16, i16, i16, i16 }
+
+; Function Attrs: norecurse nounwind readonly
+define [5 x double] @foo(double %k, i64 %n, %struct._pp* nocapture readonly %p) local_unnamed_addr #0 {
+entry:
+  %cmp17 = icmp sgt i64 %n, 0
+  br i1 %cmp17, label %for.body, label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %for.body, %entry
+  %retval.sroa.0.0.lcssa = phi double [ 0.000000e+00, %entry ], [ %add, %for.body ]
+  %retval.sroa.4.0.lcssa = phi double [ 0.000000e+00, %entry ], [ %add10, %for.body ]
+  %.fca.0.insert = insertvalue [5 x double] undef, double %retval.sroa.0.0.lcssa, 0
+  %.fca.1.insert = insertvalue [5 x double] %.fca.0.insert, double %retval.sroa.4.0.lcssa, 1
+  ret [5 x double] %.fca.1.insert
+
+for.body:                                         ; preds = %entry, %for.body
+  %i.020 = phi i64 [ %inc, %for.body ], [ 0, %entry ]
+  %retval.sroa.4.019 = phi double [ %add10, %for.body ], [ 0.000000e+00, %entry ]
+  %retval.sroa.0.018 = phi double [ %add, %for.body ], [ 0.000000e+00, %entry ]
+  %r1 = getelementptr inbounds %struct._pp, %struct._pp* %p, i64 %i.020, i32 2
+  %0 = load i16, i16* %r1, align 2
+  %conv2 = uitofp i16 %0 to double
+  %mul = fmul double %conv2, %k
+  %add = fadd double %retval.sroa.0.018, %mul
+  %g5 = getelementptr inbounds %struct._pp, %struct._pp* %p, i64 %i.020, i32 1
+  %1 = load i16, i16* %g5, align 2
+  %conv7 = uitofp i16 %1 to double
+  %mul8 = fmul double %conv7, %k
+  %add10 = fadd double %retval.sroa.4.019, %mul8
+  %inc = add nuw nsw i64 %i.020, 1
+  %exitcond = icmp eq i64 %inc, %n
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+}
+
+; CHECK-P8: load <2 x i16>
+; CHECK-P9-NOT: load <2 x i16>
diff --git a/test/Transforms/SLPVectorizer/X86/PR39774.ll b/test/Transforms/SLPVectorizer/X86/PR39774.ll
index 8405117090b46faa0a024773403fc5cabb010c95..67717a54659c36ac76ecd6ffbfa739b49de70909 100644
--- a/test/Transforms/SLPVectorizer/X86/PR39774.ll
+++ b/test/Transforms/SLPVectorizer/X86/PR39774.ll
@@ -3,93 +3,185 @@
 ; RUN: opt -slp-vectorizer -S < %s -mtriple=x86_64-unknown-linux-gnu -mcpu=skylake -slp-threshold=-8 -slp-min-tree-size=6 | FileCheck %s --check-prefixes=ALL,FORCE_REDUCTION
 
 define void @Test(i32) {
-; ALL-LABEL: @Test(
-; ALL-NEXT:  entry:
-; ALL-NEXT:    br label [[LOOP:%.*]]
-; ALL:       loop:
-; ALL-NEXT:    [[TMP1:%.*]] = phi <2 x i32> [ [[TMP11:%.*]], [[LOOP]] ], [ zeroinitializer, [[ENTRY:%.*]] ]
-; ALL-NEXT:    [[SHUFFLE:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> undef, <8 x i32> <i32 0, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
-; ALL-NEXT:    [[TMP2:%.*]] = extractelement <8 x i32> [[SHUFFLE]], i32 1
-; ALL-NEXT:    [[TMP3:%.*]] = add <8 x i32> <i32 0, i32 55, i32 285, i32 1240, i32 1496, i32 8555, i32 12529, i32 13685>, [[SHUFFLE]]
-; ALL-NEXT:    [[VAL_1:%.*]] = and i32 [[TMP2]], undef
-; ALL-NEXT:    [[VAL_2:%.*]] = and i32 [[VAL_1]], [[TMP0:%.*]]
-; ALL-NEXT:    [[VAL_3:%.*]] = and i32 [[VAL_2]], [[TMP0]]
-; ALL-NEXT:    [[VAL_4:%.*]] = and i32 [[VAL_3]], [[TMP0]]
-; ALL-NEXT:    [[VAL_5:%.*]] = and i32 [[VAL_4]], [[TMP0]]
-; ALL-NEXT:    [[VAL_7:%.*]] = and i32 [[VAL_5]], undef
-; ALL-NEXT:    [[VAL_8:%.*]] = and i32 [[VAL_7]], [[TMP0]]
-; ALL-NEXT:    [[VAL_9:%.*]] = and i32 [[VAL_8]], [[TMP0]]
-; ALL-NEXT:    [[VAL_10:%.*]] = and i32 [[VAL_9]], [[TMP0]]
-; ALL-NEXT:    [[VAL_12:%.*]] = and i32 [[VAL_10]], undef
-; ALL-NEXT:    [[VAL_13:%.*]] = and i32 [[VAL_12]], [[TMP0]]
-; ALL-NEXT:    [[VAL_14:%.*]] = and i32 [[VAL_13]], [[TMP0]]
-; ALL-NEXT:    [[VAL_15:%.*]] = and i32 [[VAL_14]], [[TMP0]]
-; ALL-NEXT:    [[VAL_16:%.*]] = and i32 [[VAL_15]], [[TMP0]]
-; ALL-NEXT:    [[VAL_17:%.*]] = and i32 [[VAL_16]], [[TMP0]]
-; ALL-NEXT:    [[VAL_19:%.*]] = and i32 [[VAL_17]], undef
-; ALL-NEXT:    [[VAL_21:%.*]] = and i32 [[VAL_19]], undef
-; ALL-NEXT:    [[VAL_22:%.*]] = and i32 [[VAL_21]], [[TMP0]]
-; ALL-NEXT:    [[VAL_23:%.*]] = and i32 [[VAL_22]], [[TMP0]]
-; ALL-NEXT:    [[VAL_24:%.*]] = and i32 [[VAL_23]], [[TMP0]]
-; ALL-NEXT:    [[VAL_25:%.*]] = and i32 [[VAL_24]], [[TMP0]]
-; ALL-NEXT:    [[VAL_26:%.*]] = and i32 [[VAL_25]], [[TMP0]]
-; ALL-NEXT:    [[VAL_27:%.*]] = and i32 [[VAL_26]], [[TMP0]]
-; ALL-NEXT:    [[VAL_28:%.*]] = and i32 [[VAL_27]], [[TMP0]]
-; ALL-NEXT:    [[VAL_29:%.*]] = and i32 [[VAL_28]], [[TMP0]]
-; ALL-NEXT:    [[VAL_30:%.*]] = and i32 [[VAL_29]], [[TMP0]]
-; ALL-NEXT:    [[VAL_31:%.*]] = and i32 [[VAL_30]], [[TMP0]]
-; ALL-NEXT:    [[VAL_32:%.*]] = and i32 [[VAL_31]], [[TMP0]]
-; ALL-NEXT:    [[VAL_33:%.*]] = and i32 [[VAL_32]], [[TMP0]]
-; ALL-NEXT:    [[VAL_35:%.*]] = and i32 [[VAL_33]], undef
-; ALL-NEXT:    [[VAL_36:%.*]] = and i32 [[VAL_35]], [[TMP0]]
-; ALL-NEXT:    [[VAL_37:%.*]] = and i32 [[VAL_36]], [[TMP0]]
-; ALL-NEXT:    [[VAL_38:%.*]] = and i32 [[VAL_37]], [[TMP0]]
-; ALL-NEXT:    [[VAL_40:%.*]] = and i32 [[VAL_38]], undef
-; ALL-NEXT:    [[TMP4:%.*]] = insertelement <2 x i32> undef, i32 [[VAL_40]], i32 0
-; ALL-NEXT:    [[TMP5:%.*]] = insertelement <2 x i32> [[TMP4]], i32 [[TMP2]], i32 1
-; ALL-NEXT:    [[TMP6:%.*]] = extractelement <8 x i32> [[TMP3]], i32 7
-; ALL-NEXT:    [[TMP7:%.*]] = insertelement <2 x i32> undef, i32 [[TMP6]], i32 0
-; ALL-NEXT:    [[TMP8:%.*]] = insertelement <2 x i32> [[TMP7]], i32 14910, i32 1
-; ALL-NEXT:    [[TMP9:%.*]] = and <2 x i32> [[TMP5]], [[TMP8]]
-; ALL-NEXT:    [[TMP10:%.*]] = add <2 x i32> [[TMP5]], [[TMP8]]
-; ALL-NEXT:    [[TMP11]] = shufflevector <2 x i32> [[TMP9]], <2 x i32> [[TMP10]], <2 x i32> <i32 0, i32 3>
-; ALL-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <8 x i32> [[TMP3]], <8 x i32> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
-; ALL-NEXT:    [[BIN_RDX:%.*]] = and <8 x i32> [[TMP3]], [[RDX_SHUF]]
-; ALL-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <8 x i32> [[BIN_RDX]], <8 x i32> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; ALL-NEXT:    [[BIN_RDX2:%.*]] = and <8 x i32> [[BIN_RDX]], [[RDX_SHUF1]]
-; ALL-NEXT:    [[RDX_SHUF3:%.*]] = shufflevector <8 x i32> [[BIN_RDX2]], <8 x i32> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; ALL-NEXT:    [[BIN_RDX4:%.*]] = and <8 x i32> [[BIN_RDX2]], [[RDX_SHUF3]]
-; ALL-NEXT:    [[TMP12:%.*]] = extractelement <8 x i32> [[BIN_RDX4]], i32 0
-; ALL-NEXT:    [[OP_EXTRA:%.*]] = and i32 [[TMP12]], [[TMP0]]
-; ALL-NEXT:    [[OP_EXTRA5:%.*]] = and i32 [[OP_EXTRA]], [[TMP0]]
-; ALL-NEXT:    [[OP_EXTRA6:%.*]] = and i32 [[OP_EXTRA5]], [[TMP0]]
-; ALL-NEXT:    [[OP_EXTRA7:%.*]] = and i32 [[OP_EXTRA6]], [[TMP0]]
-; ALL-NEXT:    [[OP_EXTRA8:%.*]] = and i32 [[OP_EXTRA7]], [[TMP0]]
-; ALL-NEXT:    [[OP_EXTRA9:%.*]] = and i32 [[OP_EXTRA8]], [[TMP0]]
-; ALL-NEXT:    [[OP_EXTRA10:%.*]] = and i32 [[OP_EXTRA9]], [[TMP0]]
-; ALL-NEXT:    [[OP_EXTRA11:%.*]] = and i32 [[OP_EXTRA10]], [[TMP0]]
-; ALL-NEXT:    [[OP_EXTRA12:%.*]] = and i32 [[OP_EXTRA11]], [[TMP0]]
-; ALL-NEXT:    [[OP_EXTRA13:%.*]] = and i32 [[OP_EXTRA12]], [[TMP0]]
-; ALL-NEXT:    [[OP_EXTRA14:%.*]] = and i32 [[OP_EXTRA13]], [[TMP0]]
-; ALL-NEXT:    [[OP_EXTRA15:%.*]] = and i32 [[OP_EXTRA14]], [[TMP0]]
-; ALL-NEXT:    [[OP_EXTRA16:%.*]] = and i32 [[OP_EXTRA15]], [[TMP0]]
-; ALL-NEXT:    [[OP_EXTRA17:%.*]] = and i32 [[OP_EXTRA16]], [[TMP0]]
-; ALL-NEXT:    [[OP_EXTRA18:%.*]] = and i32 [[OP_EXTRA17]], [[TMP0]]
-; ALL-NEXT:    [[OP_EXTRA19:%.*]] = and i32 [[OP_EXTRA18]], [[TMP0]]
-; ALL-NEXT:    [[OP_EXTRA20:%.*]] = and i32 [[OP_EXTRA19]], [[TMP0]]
-; ALL-NEXT:    [[OP_EXTRA21:%.*]] = and i32 [[OP_EXTRA20]], [[TMP0]]
-; ALL-NEXT:    [[OP_EXTRA22:%.*]] = and i32 [[OP_EXTRA21]], [[TMP0]]
-; ALL-NEXT:    [[OP_EXTRA23:%.*]] = and i32 [[OP_EXTRA22]], [[TMP0]]
-; ALL-NEXT:    [[OP_EXTRA24:%.*]] = and i32 [[OP_EXTRA23]], [[TMP0]]
-; ALL-NEXT:    [[OP_EXTRA25:%.*]] = and i32 [[OP_EXTRA24]], [[TMP0]]
-; ALL-NEXT:    [[OP_EXTRA26:%.*]] = and i32 [[OP_EXTRA25]], [[TMP0]]
-; ALL-NEXT:    [[OP_EXTRA27:%.*]] = and i32 [[OP_EXTRA26]], [[TMP0]]
-; ALL-NEXT:    [[OP_EXTRA28:%.*]] = and i32 [[OP_EXTRA27]], [[TMP0]]
-; ALL-NEXT:    [[OP_EXTRA29:%.*]] = and i32 [[OP_EXTRA28]], [[TMP0]]
-; ALL-NEXT:    [[OP_EXTRA30:%.*]] = and i32 [[OP_EXTRA29]], [[TMP0]]
-; ALL-NEXT:    [[OP_EXTRA31:%.*]] = and i32 [[OP_EXTRA30]], [[TMP2]]
-; ALL-NEXT:    [[TMP13:%.*]] = extractelement <2 x i32> [[TMP11]], i32 0
-; ALL-NEXT:    br label [[LOOP]]
+; CHECK-LABEL: @Test(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[TMP1:%.*]] = phi <2 x i32> [ [[TMP15:%.*]], [[LOOP]] ], [ zeroinitializer, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> undef, <8 x i32> <i32 0, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <8 x i32> [[SHUFFLE]], i32 1
+; CHECK-NEXT:    [[TMP3:%.*]] = add <8 x i32> <i32 0, i32 55, i32 285, i32 1240, i32 1496, i32 8555, i32 12529, i32 13685>, [[SHUFFLE]]
+; CHECK-NEXT:    [[VAL_1:%.*]] = and i32 [[TMP2]], undef
+; CHECK-NEXT:    [[VAL_2:%.*]] = and i32 [[VAL_1]], [[TMP0:%.*]]
+; CHECK-NEXT:    [[VAL_3:%.*]] = and i32 [[VAL_2]], [[TMP0]]
+; CHECK-NEXT:    [[VAL_4:%.*]] = and i32 [[VAL_3]], [[TMP0]]
+; CHECK-NEXT:    [[VAL_5:%.*]] = and i32 [[VAL_4]], [[TMP0]]
+; CHECK-NEXT:    [[VAL_7:%.*]] = and i32 [[VAL_5]], undef
+; CHECK-NEXT:    [[VAL_8:%.*]] = and i32 [[VAL_7]], [[TMP0]]
+; CHECK-NEXT:    [[VAL_9:%.*]] = and i32 [[VAL_8]], [[TMP0]]
+; CHECK-NEXT:    [[VAL_10:%.*]] = and i32 [[VAL_9]], [[TMP0]]
+; CHECK-NEXT:    [[VAL_12:%.*]] = and i32 [[VAL_10]], undef
+; CHECK-NEXT:    [[VAL_13:%.*]] = and i32 [[VAL_12]], [[TMP0]]
+; CHECK-NEXT:    [[VAL_14:%.*]] = and i32 [[VAL_13]], [[TMP0]]
+; CHECK-NEXT:    [[VAL_15:%.*]] = and i32 [[VAL_14]], [[TMP0]]
+; CHECK-NEXT:    [[VAL_16:%.*]] = and i32 [[VAL_15]], [[TMP0]]
+; CHECK-NEXT:    [[VAL_17:%.*]] = and i32 [[VAL_16]], [[TMP0]]
+; CHECK-NEXT:    [[VAL_19:%.*]] = and i32 [[VAL_17]], undef
+; CHECK-NEXT:    [[VAL_21:%.*]] = and i32 [[VAL_19]], undef
+; CHECK-NEXT:    [[VAL_22:%.*]] = and i32 [[VAL_21]], [[TMP0]]
+; CHECK-NEXT:    [[VAL_23:%.*]] = and i32 [[VAL_22]], [[TMP0]]
+; CHECK-NEXT:    [[VAL_24:%.*]] = and i32 [[VAL_23]], [[TMP0]]
+; CHECK-NEXT:    [[VAL_25:%.*]] = and i32 [[VAL_24]], [[TMP0]]
+; CHECK-NEXT:    [[VAL_26:%.*]] = and i32 [[VAL_25]], [[TMP0]]
+; CHECK-NEXT:    [[VAL_27:%.*]] = and i32 [[VAL_26]], [[TMP0]]
+; CHECK-NEXT:    [[VAL_28:%.*]] = and i32 [[VAL_27]], [[TMP0]]
+; CHECK-NEXT:    [[VAL_29:%.*]] = and i32 [[VAL_28]], [[TMP0]]
+; CHECK-NEXT:    [[VAL_30:%.*]] = and i32 [[VAL_29]], [[TMP0]]
+; CHECK-NEXT:    [[VAL_31:%.*]] = and i32 [[VAL_30]], [[TMP0]]
+; CHECK-NEXT:    [[VAL_32:%.*]] = and i32 [[VAL_31]], [[TMP0]]
+; CHECK-NEXT:    [[VAL_33:%.*]] = and i32 [[VAL_32]], [[TMP0]]
+; CHECK-NEXT:    [[VAL_35:%.*]] = and i32 [[VAL_33]], undef
+; CHECK-NEXT:    [[VAL_36:%.*]] = and i32 [[VAL_35]], [[TMP0]]
+; CHECK-NEXT:    [[VAL_37:%.*]] = and i32 [[VAL_36]], [[TMP0]]
+; CHECK-NEXT:    [[VAL_38:%.*]] = and i32 [[VAL_37]], [[TMP0]]
+; CHECK-NEXT:    [[VAL_40:%.*]] = and i32 [[VAL_38]], undef
+; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <8 x i32> [[TMP3]], <8 x i32> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX:%.*]] = and <8 x i32> [[TMP3]], [[RDX_SHUF]]
+; CHECK-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <8 x i32> [[BIN_RDX]], <8 x i32> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX2:%.*]] = and <8 x i32> [[BIN_RDX]], [[RDX_SHUF1]]
+; CHECK-NEXT:    [[RDX_SHUF3:%.*]] = shufflevector <8 x i32> [[BIN_RDX2]], <8 x i32> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX4:%.*]] = and <8 x i32> [[BIN_RDX2]], [[RDX_SHUF3]]
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <8 x i32> [[BIN_RDX4]], i32 0
+; CHECK-NEXT:    [[OP_EXTRA:%.*]] = and i32 [[TMP4]], [[TMP0]]
+; CHECK-NEXT:    [[OP_EXTRA5:%.*]] = and i32 [[OP_EXTRA]], [[TMP0]]
+; CHECK-NEXT:    [[OP_EXTRA6:%.*]] = and i32 [[OP_EXTRA5]], [[TMP0]]
+; CHECK-NEXT:    [[OP_EXTRA7:%.*]] = and i32 [[OP_EXTRA6]], [[TMP0]]
+; CHECK-NEXT:    [[OP_EXTRA8:%.*]] = and i32 [[OP_EXTRA7]], [[TMP0]]
+; CHECK-NEXT:    [[OP_EXTRA9:%.*]] = and i32 [[OP_EXTRA8]], [[TMP0]]
+; CHECK-NEXT:    [[OP_EXTRA10:%.*]] = and i32 [[OP_EXTRA9]], [[TMP0]]
+; CHECK-NEXT:    [[OP_EXTRA11:%.*]] = and i32 [[OP_EXTRA10]], [[TMP0]]
+; CHECK-NEXT:    [[OP_EXTRA12:%.*]] = and i32 [[OP_EXTRA11]], [[TMP0]]
+; CHECK-NEXT:    [[OP_EXTRA13:%.*]] = and i32 [[OP_EXTRA12]], [[TMP0]]
+; CHECK-NEXT:    [[OP_EXTRA14:%.*]] = and i32 [[OP_EXTRA13]], [[TMP0]]
+; CHECK-NEXT:    [[OP_EXTRA15:%.*]] = and i32 [[OP_EXTRA14]], [[TMP0]]
+; CHECK-NEXT:    [[OP_EXTRA16:%.*]] = and i32 [[OP_EXTRA15]], [[TMP0]]
+; CHECK-NEXT:    [[OP_EXTRA17:%.*]] = and i32 [[OP_EXTRA16]], [[TMP0]]
+; CHECK-NEXT:    [[OP_EXTRA18:%.*]] = and i32 [[OP_EXTRA17]], [[TMP0]]
+; CHECK-NEXT:    [[OP_EXTRA19:%.*]] = and i32 [[OP_EXTRA18]], [[TMP0]]
+; CHECK-NEXT:    [[OP_EXTRA20:%.*]] = and i32 [[OP_EXTRA19]], [[TMP0]]
+; CHECK-NEXT:    [[OP_EXTRA21:%.*]] = and i32 [[OP_EXTRA20]], [[TMP0]]
+; CHECK-NEXT:    [[OP_EXTRA22:%.*]] = and i32 [[OP_EXTRA21]], [[TMP0]]
+; CHECK-NEXT:    [[OP_EXTRA23:%.*]] = and i32 [[OP_EXTRA22]], [[TMP0]]
+; CHECK-NEXT:    [[OP_EXTRA24:%.*]] = and i32 [[OP_EXTRA23]], [[TMP0]]
+; CHECK-NEXT:    [[OP_EXTRA25:%.*]] = and i32 [[OP_EXTRA24]], [[TMP0]]
+; CHECK-NEXT:    [[OP_EXTRA26:%.*]] = and i32 [[OP_EXTRA25]], [[TMP0]]
+; CHECK-NEXT:    [[OP_EXTRA27:%.*]] = and i32 [[OP_EXTRA26]], [[TMP0]]
+; CHECK-NEXT:    [[OP_EXTRA28:%.*]] = and i32 [[OP_EXTRA27]], [[TMP0]]
+; CHECK-NEXT:    [[OP_EXTRA29:%.*]] = and i32 [[OP_EXTRA28]], [[TMP0]]
+; CHECK-NEXT:    [[OP_EXTRA30:%.*]] = and i32 [[OP_EXTRA29]], [[TMP0]]
+; CHECK-NEXT:    [[VAL_42:%.*]] = and i32 [[VAL_40]], undef
+; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <2 x i32> undef, i32 [[OP_EXTRA30]], i32 0
+; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <2 x i32> [[TMP5]], i32 [[TMP2]], i32 1
+; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <2 x i32> undef, i32 [[TMP2]], i32 0
+; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <2 x i32> [[TMP7]], i32 14910, i32 1
+; CHECK-NEXT:    [[TMP9:%.*]] = and <2 x i32> [[TMP6]], [[TMP8]]
+; CHECK-NEXT:    [[TMP10:%.*]] = add <2 x i32> [[TMP6]], [[TMP8]]
+; CHECK-NEXT:    [[TMP11:%.*]] = shufflevector <2 x i32> [[TMP9]], <2 x i32> [[TMP10]], <2 x i32> <i32 0, i32 3>
+; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <2 x i32> [[TMP11]], i32 0
+; CHECK-NEXT:    [[TMP13:%.*]] = insertelement <2 x i32> undef, i32 [[TMP12]], i32 0
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <2 x i32> [[TMP11]], i32 1
+; CHECK-NEXT:    [[TMP15]] = insertelement <2 x i32> [[TMP13]], i32 [[TMP14]], i32 1
+; CHECK-NEXT:    br label [[LOOP]]
+;
+; FORCE_REDUCTION-LABEL: @Test(
+; FORCE_REDUCTION-NEXT:  entry:
+; FORCE_REDUCTION-NEXT:    br label [[LOOP:%.*]]
+; FORCE_REDUCTION:       loop:
+; FORCE_REDUCTION-NEXT:    [[TMP1:%.*]] = phi <2 x i32> [ [[TMP13:%.*]], [[LOOP]] ], [ zeroinitializer, [[ENTRY:%.*]] ]
+; FORCE_REDUCTION-NEXT:    [[SHUFFLE:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> undef, <4 x i32> <i32 0, i32 1, i32 1, i32 1>
+; FORCE_REDUCTION-NEXT:    [[TMP2:%.*]] = extractelement <4 x i32> [[SHUFFLE]], i32 1
+; FORCE_REDUCTION-NEXT:    [[TMP3:%.*]] = add <4 x i32> <i32 0, i32 55, i32 285, i32 1240>, [[SHUFFLE]]
+; FORCE_REDUCTION-NEXT:    [[VAL_1:%.*]] = and i32 [[TMP2]], undef
+; FORCE_REDUCTION-NEXT:    [[VAL_2:%.*]] = and i32 [[VAL_1]], [[TMP0:%.*]]
+; FORCE_REDUCTION-NEXT:    [[VAL_3:%.*]] = and i32 [[VAL_2]], [[TMP0]]
+; FORCE_REDUCTION-NEXT:    [[VAL_4:%.*]] = and i32 [[VAL_3]], [[TMP0]]
+; FORCE_REDUCTION-NEXT:    [[VAL_5:%.*]] = and i32 [[VAL_4]], [[TMP0]]
+; FORCE_REDUCTION-NEXT:    [[VAL_7:%.*]] = and i32 [[VAL_5]], undef
+; FORCE_REDUCTION-NEXT:    [[VAL_8:%.*]] = and i32 [[VAL_7]], [[TMP0]]
+; FORCE_REDUCTION-NEXT:    [[VAL_9:%.*]] = and i32 [[VAL_8]], [[TMP0]]
+; FORCE_REDUCTION-NEXT:    [[VAL_10:%.*]] = and i32 [[VAL_9]], [[TMP0]]
+; FORCE_REDUCTION-NEXT:    [[VAL_12:%.*]] = and i32 [[VAL_10]], undef
+; FORCE_REDUCTION-NEXT:    [[VAL_13:%.*]] = and i32 [[VAL_12]], [[TMP0]]
+; FORCE_REDUCTION-NEXT:    [[VAL_14:%.*]] = and i32 [[VAL_13]], [[TMP0]]
+; FORCE_REDUCTION-NEXT:    [[VAL_15:%.*]] = and i32 [[VAL_14]], [[TMP0]]
+; FORCE_REDUCTION-NEXT:    [[VAL_16:%.*]] = and i32 [[VAL_15]], [[TMP0]]
+; FORCE_REDUCTION-NEXT:    [[VAL_17:%.*]] = and i32 [[VAL_16]], [[TMP0]]
+; FORCE_REDUCTION-NEXT:    [[VAL_19:%.*]] = and i32 [[VAL_17]], undef
+; FORCE_REDUCTION-NEXT:    [[VAL_20:%.*]] = add i32 [[TMP2]], 1496
+; FORCE_REDUCTION-NEXT:    [[VAL_21:%.*]] = and i32 [[VAL_19]], [[VAL_20]]
+; FORCE_REDUCTION-NEXT:    [[VAL_22:%.*]] = and i32 [[VAL_21]], [[TMP0]]
+; FORCE_REDUCTION-NEXT:    [[VAL_23:%.*]] = and i32 [[VAL_22]], [[TMP0]]
+; FORCE_REDUCTION-NEXT:    [[VAL_24:%.*]] = and i32 [[VAL_23]], [[TMP0]]
+; FORCE_REDUCTION-NEXT:    [[VAL_25:%.*]] = and i32 [[VAL_24]], [[TMP0]]
+; FORCE_REDUCTION-NEXT:    [[VAL_26:%.*]] = and i32 [[VAL_25]], [[TMP0]]
+; FORCE_REDUCTION-NEXT:    [[VAL_27:%.*]] = and i32 [[VAL_26]], [[TMP0]]
+; FORCE_REDUCTION-NEXT:    [[VAL_28:%.*]] = and i32 [[VAL_27]], [[TMP0]]
+; FORCE_REDUCTION-NEXT:    [[VAL_29:%.*]] = and i32 [[VAL_28]], [[TMP0]]
+; FORCE_REDUCTION-NEXT:    [[VAL_30:%.*]] = and i32 [[VAL_29]], [[TMP0]]
+; FORCE_REDUCTION-NEXT:    [[VAL_31:%.*]] = and i32 [[VAL_30]], [[TMP0]]
+; FORCE_REDUCTION-NEXT:    [[VAL_32:%.*]] = and i32 [[VAL_31]], [[TMP0]]
+; FORCE_REDUCTION-NEXT:    [[VAL_33:%.*]] = and i32 [[VAL_32]], [[TMP0]]
+; FORCE_REDUCTION-NEXT:    [[VAL_34:%.*]] = add i32 [[TMP2]], 8555
+; FORCE_REDUCTION-NEXT:    [[VAL_35:%.*]] = and i32 [[VAL_33]], [[VAL_34]]
+; FORCE_REDUCTION-NEXT:    [[VAL_36:%.*]] = and i32 [[VAL_35]], [[TMP0]]
+; FORCE_REDUCTION-NEXT:    [[VAL_37:%.*]] = and i32 [[VAL_36]], [[TMP0]]
+; FORCE_REDUCTION-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+; FORCE_REDUCTION-NEXT:    [[BIN_RDX:%.*]] = and <4 x i32> [[TMP3]], [[RDX_SHUF]]
+; FORCE_REDUCTION-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <4 x i32> [[BIN_RDX]], <4 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; FORCE_REDUCTION-NEXT:    [[BIN_RDX2:%.*]] = and <4 x i32> [[BIN_RDX]], [[RDX_SHUF1]]
+; FORCE_REDUCTION-NEXT:    [[TMP4:%.*]] = extractelement <4 x i32> [[BIN_RDX2]], i32 0
+; FORCE_REDUCTION-NEXT:    [[TMP5:%.*]] = and i32 [[TMP4]], [[VAL_20]]
+; FORCE_REDUCTION-NEXT:    [[TMP6:%.*]] = and i32 [[TMP5]], [[VAL_34]]
+; FORCE_REDUCTION-NEXT:    [[OP_EXTRA:%.*]] = and i32 [[TMP6]], [[TMP0]]
+; FORCE_REDUCTION-NEXT:    [[OP_EXTRA3:%.*]] = and i32 [[OP_EXTRA]], [[TMP0]]
+; FORCE_REDUCTION-NEXT:    [[OP_EXTRA4:%.*]] = and i32 [[OP_EXTRA3]], [[TMP0]]
+; FORCE_REDUCTION-NEXT:    [[OP_EXTRA5:%.*]] = and i32 [[OP_EXTRA4]], [[TMP0]]
+; FORCE_REDUCTION-NEXT:    [[OP_EXTRA6:%.*]] = and i32 [[OP_EXTRA5]], [[TMP0]]
+; FORCE_REDUCTION-NEXT:    [[OP_EXTRA7:%.*]] = and i32 [[OP_EXTRA6]], [[TMP0]]
+; FORCE_REDUCTION-NEXT:    [[OP_EXTRA8:%.*]] = and i32 [[OP_EXTRA7]], [[TMP0]]
+; FORCE_REDUCTION-NEXT:    [[OP_EXTRA9:%.*]] = and i32 [[OP_EXTRA8]], [[TMP0]]
+; FORCE_REDUCTION-NEXT:    [[OP_EXTRA10:%.*]] = and i32 [[OP_EXTRA9]], [[TMP0]]
+; FORCE_REDUCTION-NEXT:    [[OP_EXTRA11:%.*]] = and i32 [[OP_EXTRA10]], [[TMP0]]
+; FORCE_REDUCTION-NEXT:    [[OP_EXTRA12:%.*]] = and i32 [[OP_EXTRA11]], [[TMP0]]
+; FORCE_REDUCTION-NEXT:    [[OP_EXTRA13:%.*]] = and i32 [[OP_EXTRA12]], [[TMP0]]
+; FORCE_REDUCTION-NEXT:    [[OP_EXTRA14:%.*]] = and i32 [[OP_EXTRA13]], [[TMP0]]
+; FORCE_REDUCTION-NEXT:    [[OP_EXTRA15:%.*]] = and i32 [[OP_EXTRA14]], [[TMP0]]
+; FORCE_REDUCTION-NEXT:    [[OP_EXTRA16:%.*]] = and i32 [[OP_EXTRA15]], [[TMP0]]
+; FORCE_REDUCTION-NEXT:    [[OP_EXTRA17:%.*]] = and i32 [[OP_EXTRA16]], [[TMP0]]
+; FORCE_REDUCTION-NEXT:    [[OP_EXTRA18:%.*]] = and i32 [[OP_EXTRA17]], [[TMP0]]
+; FORCE_REDUCTION-NEXT:    [[OP_EXTRA19:%.*]] = and i32 [[OP_EXTRA18]], [[TMP0]]
+; FORCE_REDUCTION-NEXT:    [[OP_EXTRA20:%.*]] = and i32 [[OP_EXTRA19]], [[TMP0]]
+; FORCE_REDUCTION-NEXT:    [[OP_EXTRA21:%.*]] = and i32 [[OP_EXTRA20]], [[TMP0]]
+; FORCE_REDUCTION-NEXT:    [[OP_EXTRA22:%.*]] = and i32 [[OP_EXTRA21]], [[TMP0]]
+; FORCE_REDUCTION-NEXT:    [[OP_EXTRA23:%.*]] = and i32 [[OP_EXTRA22]], [[TMP0]]
+; FORCE_REDUCTION-NEXT:    [[OP_EXTRA24:%.*]] = and i32 [[OP_EXTRA23]], [[TMP0]]
+; FORCE_REDUCTION-NEXT:    [[OP_EXTRA25:%.*]] = and i32 [[OP_EXTRA24]], [[TMP0]]
+; FORCE_REDUCTION-NEXT:    [[OP_EXTRA26:%.*]] = and i32 [[OP_EXTRA25]], [[TMP0]]
+; FORCE_REDUCTION-NEXT:    [[OP_EXTRA27:%.*]] = and i32 [[OP_EXTRA26]], [[TMP0]]
+; FORCE_REDUCTION-NEXT:    [[OP_EXTRA28:%.*]] = and i32 [[OP_EXTRA27]], [[TMP0]]
+; FORCE_REDUCTION-NEXT:    [[OP_EXTRA29:%.*]] = and i32 [[OP_EXTRA28]], [[TMP2]]
+; FORCE_REDUCTION-NEXT:    [[VAL_38:%.*]] = and i32 [[VAL_37]], [[TMP0]]
+; FORCE_REDUCTION-NEXT:    [[VAL_39:%.*]] = add i32 [[TMP2]], 12529
+; FORCE_REDUCTION-NEXT:    [[VAL_40:%.*]] = and i32 [[OP_EXTRA29]], [[VAL_39]]
+; FORCE_REDUCTION-NEXT:    [[VAL_41:%.*]] = add i32 [[TMP2]], 13685
+; FORCE_REDUCTION-NEXT:    [[TMP7:%.*]] = insertelement <2 x i32> undef, i32 [[VAL_40]], i32 0
+; FORCE_REDUCTION-NEXT:    [[TMP8:%.*]] = insertelement <2 x i32> [[TMP7]], i32 [[TMP2]], i32 1
+; FORCE_REDUCTION-NEXT:    [[TMP9:%.*]] = insertelement <2 x i32> undef, i32 [[VAL_41]], i32 0
+; FORCE_REDUCTION-NEXT:    [[TMP10:%.*]] = insertelement <2 x i32> [[TMP9]], i32 14910, i32 1
+; FORCE_REDUCTION-NEXT:    [[TMP11:%.*]] = and <2 x i32> [[TMP8]], [[TMP10]]
+; FORCE_REDUCTION-NEXT:    [[TMP12:%.*]] = add <2 x i32> [[TMP8]], [[TMP10]]
+; FORCE_REDUCTION-NEXT:    [[TMP13]] = shufflevector <2 x i32> [[TMP11]], <2 x i32> [[TMP12]], <2 x i32> <i32 0, i32 3>
+; FORCE_REDUCTION-NEXT:    br label [[LOOP]]
 ;
 entry:
   br label %loop
diff --git a/test/Transforms/SLPVectorizer/X86/PR40310.ll b/test/Transforms/SLPVectorizer/X86/PR40310.ll
new file mode 100644
index 0000000000000000000000000000000000000000..ad1434146a5b3626cb1ecc6343a7439d414d3d61
--- /dev/null
+++ b/test/Transforms/SLPVectorizer/X86/PR40310.ll
@@ -0,0 +1,93 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -slp-vectorizer -S -mtriple=x86_64-unknown-linux-gnu -mcpu=skylake < %s | FileCheck %s
+
+define void @mainTest(i32 %param, i32 * %vals, i32 %len) {
+; CHECK-LABEL: @mainTest(
+; CHECK-NEXT:  bci_15.preheader:
+; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <2 x i32> <i32 31, i32 undef>, i32 [[PARAM:%.*]], i32 1
+; CHECK-NEXT:    br label [[BCI_15:%.*]]
+; CHECK:       bci_15:
+; CHECK-NEXT:    [[TMP1:%.*]] = phi <2 x i32> [ [[TMP7:%.*]], [[BCI_15]] ], [ [[TMP0]], [[BCI_15_PREHEADER:%.*]] ]
+; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> undef, <16 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 1>
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <16 x i32> [[SHUFFLE]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <16 x i32> [[SHUFFLE]], i32 15
+; CHECK-NEXT:    store atomic i32 [[TMP3]], i32* [[VALS:%.*]] unordered, align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = add <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 -1>, [[SHUFFLE]]
+; CHECK-NEXT:    [[V14:%.*]] = and i32 [[TMP2]], undef
+; CHECK-NEXT:    [[V16:%.*]] = and i32 undef, [[V14]]
+; CHECK-NEXT:    [[V18:%.*]] = and i32 undef, [[V16]]
+; CHECK-NEXT:    [[V20:%.*]] = and i32 undef, [[V18]]
+; CHECK-NEXT:    [[V22:%.*]] = and i32 undef, [[V20]]
+; CHECK-NEXT:    [[V24:%.*]] = and i32 undef, [[V22]]
+; CHECK-NEXT:    [[V26:%.*]] = and i32 undef, [[V24]]
+; CHECK-NEXT:    [[V28:%.*]] = and i32 undef, [[V26]]
+; CHECK-NEXT:    [[V30:%.*]] = and i32 undef, [[V28]]
+; CHECK-NEXT:    [[V32:%.*]] = and i32 undef, [[V30]]
+; CHECK-NEXT:    [[V34:%.*]] = and i32 undef, [[V32]]
+; CHECK-NEXT:    [[V36:%.*]] = and i32 undef, [[V34]]
+; CHECK-NEXT:    [[V38:%.*]] = and i32 undef, [[V36]]
+; CHECK-NEXT:    [[V40:%.*]] = and i32 undef, [[V38]]
+; CHECK-NEXT:    [[V42:%.*]] = and i32 undef, [[V40]]
+; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <16 x i32> [[TMP4]], <16 x i32> undef, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX:%.*]] = and <16 x i32> [[TMP4]], [[RDX_SHUF]]
+; CHECK-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <16 x i32> [[BIN_RDX]], <16 x i32> undef, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX2:%.*]] = and <16 x i32> [[BIN_RDX]], [[RDX_SHUF1]]
+; CHECK-NEXT:    [[RDX_SHUF3:%.*]] = shufflevector <16 x i32> [[BIN_RDX2]], <16 x i32> undef, <16 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX4:%.*]] = and <16 x i32> [[BIN_RDX2]], [[RDX_SHUF3]]
+; CHECK-NEXT:    [[RDX_SHUF5:%.*]] = shufflevector <16 x i32> [[BIN_RDX4]], <16 x i32> undef, <16 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX6:%.*]] = and <16 x i32> [[BIN_RDX4]], [[RDX_SHUF5]]
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <16 x i32> [[BIN_RDX6]], i32 0
+; CHECK-NEXT:    [[OP_EXTRA:%.*]] = and i32 [[TMP5]], [[TMP2]]
+; CHECK-NEXT:    [[V43:%.*]] = and i32 undef, [[V42]]
+; CHECK-NEXT:    [[V44:%.*]] = add i32 [[TMP2]], 16
+; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <2 x i32> undef, i32 [[V44]], i32 0
+; CHECK-NEXT:    [[TMP7]] = insertelement <2 x i32> [[TMP6]], i32 [[OP_EXTRA]], i32 1
+; CHECK-NEXT:    br i1 true, label [[BCI_15]], label [[LOOPEXIT:%.*]]
+; CHECK:       loopexit:
+; CHECK-NEXT:    ret void
+;
+bci_15.preheader:
+  br label %bci_15
+
+bci_15:                                       ; preds = %bci_15.preheader, %bci_15
+  %local_0_ = phi i32 [ %v43, %bci_15 ], [ %param, %bci_15.preheader ]
+  %local_4_ = phi i32 [ %v44, %bci_15 ], [ 31, %bci_15.preheader ]
+  %v12 = add i32 %local_0_, -1
+  store atomic i32 %local_0_, i32 * %vals unordered, align 4
+  %v13 = add i32 %local_4_, 1
+  %v14 = and i32 %local_4_, %v12
+  %v15 = add i32 %local_4_, 2
+  %v16 = and i32 %v13, %v14
+  %v17 = add i32 %local_4_, 3
+  %v18 = and i32 %v15, %v16
+  %v19 = add i32 %local_4_, 4
+  %v20 = and i32 %v17, %v18
+  %v21 = add i32 %local_4_, 5
+  %v22 = and i32 %v19, %v20
+  %v23 = add i32 %local_4_, 6
+  %v24 = and i32 %v21, %v22
+  %v25 = add i32 %local_4_, 7
+  %v26 = and i32 %v23, %v24
+  %v27 = add i32 %local_4_, 8
+  %v28 = and i32 %v25, %v26
+  %v29 = add i32 %local_4_, 9
+  %v30 = and i32 %v27, %v28
+  %v31 = add i32 %local_4_, 10
+  %v32 = and i32 %v29, %v30
+  %v33 = add i32 %local_4_, 11
+  %v34 = and i32 %v31, %v32
+  %v35 = add i32 %local_4_, 12
+  %v36 = and i32 %v33, %v34
+  %v37 = add i32 %local_4_, 13
+  %v38 = and i32 %v35, %v36
+  %v39 = add i32 %local_4_, 14
+  %v40 = and i32 %v37, %v38
+  %v41 = add i32 %local_4_, 15
+  %v42 = and i32 %v39, %v40
+  %v43 = and i32 %v41, %v42
+  %v44 = add i32 %local_4_, 16
+  br i1 true, label %bci_15, label %loopexit
+
+loopexit:
+  ret void
+}
diff --git a/test/Transforms/SLPVectorizer/X86/addsub.ll b/test/Transforms/SLPVectorizer/X86/addsub.ll
index 510bc9e5b20f6da02151573081068b300112fcf6..6ab5e1632587832af57a8ae15fedf78dff436e42 100644
--- a/test/Transforms/SLPVectorizer/X86/addsub.ll
+++ b/test/Transforms/SLPVectorizer/X86/addsub.ll
@@ -348,22 +348,18 @@ define void @reorder_alt_rightsubTree(double* nocapture %c, double* noalias noca
 
 define void @no_vec_shuff_reorder() #0 {
 ; CHECK-LABEL: @no_vec_shuff_reorder(
-; CHECK-NEXT:    [[TMP1:%.*]] = load float, float* getelementptr inbounds ([4 x float], [4 x float]* @fb, i32 0, i64 0), align 4
-; CHECK-NEXT:    [[TMP2:%.*]] = load float, float* getelementptr inbounds ([4 x float], [4 x float]* @fa, i32 0, i64 0), align 4
-; CHECK-NEXT:    [[TMP3:%.*]] = fadd float [[TMP1]], [[TMP2]]
-; CHECK-NEXT:    store float [[TMP3]], float* getelementptr inbounds ([4 x float], [4 x float]* @fc, i32 0, i64 0), align 4
-; CHECK-NEXT:    [[TMP4:%.*]] = load float, float* getelementptr inbounds ([4 x float], [4 x float]* @fa, i32 0, i64 1), align 4
-; CHECK-NEXT:    [[TMP5:%.*]] = load float, float* getelementptr inbounds ([4 x float], [4 x float]* @fb, i32 0, i64 1), align 4
-; CHECK-NEXT:    [[TMP6:%.*]] = fsub float [[TMP4]], [[TMP5]]
-; CHECK-NEXT:    store float [[TMP6]], float* getelementptr inbounds ([4 x float], [4 x float]* @fc, i32 0, i64 1), align 4
-; CHECK-NEXT:    [[TMP7:%.*]] = load float, float* getelementptr inbounds ([4 x float], [4 x float]* @fa, i32 0, i64 2), align 4
-; CHECK-NEXT:    [[TMP8:%.*]] = load float, float* getelementptr inbounds ([4 x float], [4 x float]* @fb, i32 0, i64 2), align 4
-; CHECK-NEXT:    [[TMP9:%.*]] = fadd float [[TMP7]], [[TMP8]]
-; CHECK-NEXT:    store float [[TMP9]], float* getelementptr inbounds ([4 x float], [4 x float]* @fc, i32 0, i64 2), align 4
-; CHECK-NEXT:    [[TMP10:%.*]] = load float, float* getelementptr inbounds ([4 x float], [4 x float]* @fb, i32 0, i64 3), align 4
-; CHECK-NEXT:    [[TMP11:%.*]] = load float, float* getelementptr inbounds ([4 x float], [4 x float]* @fa, i32 0, i64 3), align 4
-; CHECK-NEXT:    [[TMP12:%.*]] = fsub float [[TMP10]], [[TMP11]]
-; CHECK-NEXT:    store float [[TMP12]], float* getelementptr inbounds ([4 x float], [4 x float]* @fc, i32 0, i64 3), align 4
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x float>, <2 x float>* bitcast ([4 x float]* @fa to <2 x float>*), align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x float>, <2 x float>* bitcast ([4 x float]* @fb to <2 x float>*), align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = fadd <2 x float> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = fsub <2 x float> [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <2 x float> [[TMP3]], <2 x float> [[TMP4]], <2 x i32> <i32 0, i32 3>
+; CHECK-NEXT:    store <2 x float> [[TMP5]], <2 x float>* bitcast ([4 x float]* @fc to <2 x float>*), align 4
+; CHECK-NEXT:    [[TMP6:%.*]] = load <2 x float>, <2 x float>* bitcast (float* getelementptr inbounds ([4 x float], [4 x float]* @fb, i32 0, i64 2) to <2 x float>*), align 4
+; CHECK-NEXT:    [[TMP7:%.*]] = load <2 x float>, <2 x float>* bitcast (float* getelementptr inbounds ([4 x float], [4 x float]* @fa, i32 0, i64 2) to <2 x float>*), align 4
+; CHECK-NEXT:    [[TMP8:%.*]] = fadd <2 x float> [[TMP6]], [[TMP7]]
+; CHECK-NEXT:    [[TMP9:%.*]] = fsub <2 x float> [[TMP6]], [[TMP7]]
+; CHECK-NEXT:    [[TMP10:%.*]] = shufflevector <2 x float> [[TMP8]], <2 x float> [[TMP9]], <2 x i32> <i32 0, i32 3>
+; CHECK-NEXT:    store <2 x float> [[TMP10]], <2 x float>* bitcast (float* getelementptr inbounds ([4 x float], [4 x float]* @fc, i32 0, i64 2) to <2 x float>*), align 4
 ; CHECK-NEXT:    ret void
 ;
   %1 = load float, float* getelementptr inbounds ([4 x float], [4 x float]* @fb, i32 0, i64 0), align 4
diff --git a/test/Transforms/SLPVectorizer/X86/align.ll b/test/Transforms/SLPVectorizer/X86/align.ll
index b74b70900ee96b809464b0998dfb740c97a952c4..5c7c4ceae8173a0f7398b1a48f306f9be2e9f3e2 100644
--- a/test/Transforms/SLPVectorizer/X86/align.ll
+++ b/test/Transforms/SLPVectorizer/X86/align.ll
@@ -1,16 +1,31 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -basicaa -slp-vectorizer -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.8.0"
 
 ; Simple 3-pair chain with loads and stores
-; CHECK-LABEL: @test1
 define void @test1(double* %a, double* %b, double* %c) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[AGG_TMP_I_I_SROA_0:%.*]] = alloca [3 x double], align 16
+; CHECK-NEXT:    [[STORE1:%.*]] = getelementptr inbounds [3 x double], [3 x double]* [[AGG_TMP_I_I_SROA_0]], i64 0, i64 1
+; CHECK-NEXT:    [[STORE2:%.*]] = getelementptr inbounds [3 x double], [3 x double]* [[AGG_TMP_I_I_SROA_0]], i64 0, i64 2
+; CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds double, double* [[A:%.*]], i64 1
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast double* [[A]] to <2 x double>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x double>, <2 x double>* [[TMP0]], align 8
+; CHECK-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds double, double* [[B:%.*]], i64 1
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast double* [[B]] to <2 x double>*
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x double>, <2 x double>* [[TMP2]], align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = fmul <2 x double> [[TMP1]], [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast double* [[STORE1]] to <2 x double>*
+; CHECK-NEXT:    store <2 x double> [[TMP4]], <2 x double>* [[TMP5]], align 8
+; CHECK-NEXT:    ret void
+;
 entry:
   %agg.tmp.i.i.sroa.0 = alloca [3 x double], align 16
-; CHECK: %[[V0:[0-9]+]] = load <2 x double>, <2 x double>* %[[V2:[0-9]+]], align 8
-  %i0 = load double, double* %a 
-  %i1 = load double, double* %b 
+  %i0 = load double, double* %a
+  %i1 = load double, double* %b
   %mul = fmul double %i0, %i1
   %store1 = getelementptr inbounds [3 x double], [3 x double]* %agg.tmp.i.i.sroa.0, i64 0, i64 1
   %store2 = getelementptr inbounds [3 x double], [3 x double]* %agg.tmp.i.i.sroa.0, i64 0, i64 2
@@ -19,23 +34,29 @@ entry:
   %arrayidx4 = getelementptr inbounds double, double* %b, i64 1
   %i4 = load double, double* %arrayidx4, align 8
   %mul5 = fmul double %i3, %i4
-; CHECK: store <2 x double> %[[V1:[0-9]+]], <2 x double>* %[[V2:[0-9]+]], align 8
   store double %mul, double* %store1
   store double %mul5, double* %store2, align 16
-; CHECK: ret
   ret void
 }
 
 ; Float has 4 byte abi alignment on x86_64. We must use the alignmnet of the
 ; value being loaded/stored not the alignment of the pointer type.
 
-; CHECK-LABEL: @test2
-; CHECK-NOT: align 8
-; CHECK: load <4 x float>{{.*}}, align 4
-; CHECK: store <4 x float>{{.*}}, align 4
-; CHECK: ret
-
 define void @test2(float * %a, float * %b) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[A1:%.*]] = getelementptr inbounds float, float* [[A:%.*]], i64 1
+; CHECK-NEXT:    [[A2:%.*]] = getelementptr inbounds float, float* [[A]], i64 2
+; CHECK-NEXT:    [[A3:%.*]] = getelementptr inbounds float, float* [[A]], i64 3
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast float* [[A]] to <4 x float>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x float>, <4 x float>* [[TMP0]], align 4
+; CHECK-NEXT:    [[B1:%.*]] = getelementptr inbounds float, float* [[B:%.*]], i64 1
+; CHECK-NEXT:    [[B2:%.*]] = getelementptr inbounds float, float* [[B]], i64 2
+; CHECK-NEXT:    [[B3:%.*]] = getelementptr inbounds float, float* [[B]], i64 3
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast float* [[B]] to <4 x float>*
+; CHECK-NEXT:    store <4 x float> [[TMP1]], <4 x float>* [[TMP2]], align 4
+; CHECK-NEXT:    ret void
+;
 entry:
   %l0 = load float, float* %a
   %a1 = getelementptr inbounds float, float* %a, i64 1
diff --git a/test/Transforms/SLPVectorizer/X86/alternate-fp.ll b/test/Transforms/SLPVectorizer/X86/alternate-fp.ll
index 9602314c6d49ddcd1eae379c7a7cea84c4c86fd2..585381fad5ea192f4a727d288bdaffd749cc6e86 100644
--- a/test/Transforms/SLPVectorizer/X86/alternate-fp.ll
+++ b/test/Transforms/SLPVectorizer/X86/alternate-fp.ll
@@ -125,14 +125,15 @@ define <4 x float> @fmul_fdiv_v4f32_const(<4 x float> %a) {
 ; SSE-NEXT:    ret <4 x float> [[TMP1]]
 ;
 ; SLM-LABEL: @fmul_fdiv_v4f32_const(
-; SLM-NEXT:    [[A0:%.*]] = extractelement <4 x float> [[A:%.*]], i32 0
-; SLM-NEXT:    [[A1:%.*]] = extractelement <4 x float> [[A]], i32 1
-; SLM-NEXT:    [[A2:%.*]] = extractelement <4 x float> [[A]], i32 2
+; SLM-NEXT:    [[A2:%.*]] = extractelement <4 x float> [[A:%.*]], i32 2
 ; SLM-NEXT:    [[A3:%.*]] = extractelement <4 x float> [[A]], i32 3
-; SLM-NEXT:    [[AB0:%.*]] = fmul float [[A0]], 2.000000e+00
+; SLM-NEXT:    [[TMP1:%.*]] = shufflevector <4 x float> [[A]], <4 x float> undef, <2 x i32> <i32 0, i32 1>
+; SLM-NEXT:    [[TMP2:%.*]] = fmul <2 x float> [[TMP1]], <float 2.000000e+00, float 1.000000e+00>
 ; SLM-NEXT:    [[AB3:%.*]] = fmul float [[A3]], 2.000000e+00
-; SLM-NEXT:    [[R0:%.*]] = insertelement <4 x float> undef, float [[AB0]], i32 0
-; SLM-NEXT:    [[R1:%.*]] = insertelement <4 x float> [[R0]], float [[A1]], i32 1
+; SLM-NEXT:    [[TMP3:%.*]] = extractelement <2 x float> [[TMP2]], i32 0
+; SLM-NEXT:    [[R0:%.*]] = insertelement <4 x float> undef, float [[TMP3]], i32 0
+; SLM-NEXT:    [[TMP4:%.*]] = extractelement <2 x float> [[TMP2]], i32 1
+; SLM-NEXT:    [[R1:%.*]] = insertelement <4 x float> [[R0]], float [[TMP4]], i32 1
 ; SLM-NEXT:    [[R2:%.*]] = insertelement <4 x float> [[R1]], float [[A2]], i32 2
 ; SLM-NEXT:    [[R3:%.*]] = insertelement <4 x float> [[R2]], float [[AB3]], i32 3
 ; SLM-NEXT:    ret <4 x float> [[R3]]
diff --git a/test/Transforms/SLPVectorizer/X86/alternate-int.ll b/test/Transforms/SLPVectorizer/X86/alternate-int.ll
index fe8ae4d1c3db4f0f42ae61067ded68f4ad8ac893..3faabe4bf52fd9b42a7489cb05c133eac59b96be 100644
--- a/test/Transforms/SLPVectorizer/X86/alternate-int.ll
+++ b/test/Transforms/SLPVectorizer/X86/alternate-int.ll
@@ -83,20 +83,17 @@ define <4 x i32> @add_mul_v4i32(<4 x i32> %a, <4 x i32> %b) {
 ;
 ; SLM-LABEL: @add_mul_v4i32(
 ; SLM-NEXT:    [[A0:%.*]] = extractelement <4 x i32> [[A:%.*]], i32 0
-; SLM-NEXT:    [[A1:%.*]] = extractelement <4 x i32> [[A]], i32 1
-; SLM-NEXT:    [[A2:%.*]] = extractelement <4 x i32> [[A]], i32 2
 ; SLM-NEXT:    [[A3:%.*]] = extractelement <4 x i32> [[A]], i32 3
 ; SLM-NEXT:    [[B0:%.*]] = extractelement <4 x i32> [[B:%.*]], i32 0
-; SLM-NEXT:    [[B1:%.*]] = extractelement <4 x i32> [[B]], i32 1
-; SLM-NEXT:    [[B2:%.*]] = extractelement <4 x i32> [[B]], i32 2
 ; SLM-NEXT:    [[B3:%.*]] = extractelement <4 x i32> [[B]], i32 3
 ; SLM-NEXT:    [[AB0:%.*]] = mul i32 [[A0]], [[B0]]
-; SLM-NEXT:    [[AB1:%.*]] = add i32 [[A1]], [[B1]]
-; SLM-NEXT:    [[AB2:%.*]] = add i32 [[A2]], [[B2]]
+; SLM-NEXT:    [[TMP1:%.*]] = add <4 x i32> [[A]], [[B]]
 ; SLM-NEXT:    [[AB3:%.*]] = mul i32 [[A3]], [[B3]]
 ; SLM-NEXT:    [[R0:%.*]] = insertelement <4 x i32> undef, i32 [[AB0]], i32 0
-; SLM-NEXT:    [[R1:%.*]] = insertelement <4 x i32> [[R0]], i32 [[AB1]], i32 1
-; SLM-NEXT:    [[R2:%.*]] = insertelement <4 x i32> [[R1]], i32 [[AB2]], i32 2
+; SLM-NEXT:    [[TMP2:%.*]] = extractelement <4 x i32> [[TMP1]], i32 1
+; SLM-NEXT:    [[R1:%.*]] = insertelement <4 x i32> [[R0]], i32 [[TMP2]], i32 1
+; SLM-NEXT:    [[TMP3:%.*]] = extractelement <4 x i32> [[TMP1]], i32 2
+; SLM-NEXT:    [[R2:%.*]] = insertelement <4 x i32> [[R1]], i32 [[TMP3]], i32 2
 ; SLM-NEXT:    [[R3:%.*]] = insertelement <4 x i32> [[R2]], i32 [[AB3]], i32 3
 ; SLM-NEXT:    ret <4 x i32> [[R3]]
 ;
@@ -274,34 +271,28 @@ define <8 x i32> @ashr_lshr_shl_v8i32(<8 x i32> %a, <8 x i32> %b) {
 ; SSE-LABEL: @ashr_lshr_shl_v8i32(
 ; SSE-NEXT:    [[A0:%.*]] = extractelement <8 x i32> [[A:%.*]], i32 0
 ; SSE-NEXT:    [[A1:%.*]] = extractelement <8 x i32> [[A]], i32 1
-; SSE-NEXT:    [[A2:%.*]] = extractelement <8 x i32> [[A]], i32 2
-; SSE-NEXT:    [[A3:%.*]] = extractelement <8 x i32> [[A]], i32 3
-; SSE-NEXT:    [[A4:%.*]] = extractelement <8 x i32> [[A]], i32 4
-; SSE-NEXT:    [[A5:%.*]] = extractelement <8 x i32> [[A]], i32 5
 ; SSE-NEXT:    [[A6:%.*]] = extractelement <8 x i32> [[A]], i32 6
 ; SSE-NEXT:    [[A7:%.*]] = extractelement <8 x i32> [[A]], i32 7
 ; SSE-NEXT:    [[B0:%.*]] = extractelement <8 x i32> [[B:%.*]], i32 0
 ; SSE-NEXT:    [[B1:%.*]] = extractelement <8 x i32> [[B]], i32 1
-; SSE-NEXT:    [[B2:%.*]] = extractelement <8 x i32> [[B]], i32 2
-; SSE-NEXT:    [[B3:%.*]] = extractelement <8 x i32> [[B]], i32 3
-; SSE-NEXT:    [[B4:%.*]] = extractelement <8 x i32> [[B]], i32 4
-; SSE-NEXT:    [[B5:%.*]] = extractelement <8 x i32> [[B]], i32 5
 ; SSE-NEXT:    [[B6:%.*]] = extractelement <8 x i32> [[B]], i32 6
 ; SSE-NEXT:    [[B7:%.*]] = extractelement <8 x i32> [[B]], i32 7
 ; SSE-NEXT:    [[AB0:%.*]] = ashr i32 [[A0]], [[B0]]
 ; SSE-NEXT:    [[AB1:%.*]] = ashr i32 [[A1]], [[B1]]
-; SSE-NEXT:    [[AB2:%.*]] = lshr i32 [[A2]], [[B2]]
-; SSE-NEXT:    [[AB3:%.*]] = lshr i32 [[A3]], [[B3]]
-; SSE-NEXT:    [[AB4:%.*]] = lshr i32 [[A4]], [[B4]]
-; SSE-NEXT:    [[AB5:%.*]] = lshr i32 [[A5]], [[B5]]
+; SSE-NEXT:    [[TMP1:%.*]] = lshr <8 x i32> [[A]], [[B]]
+; SSE-NEXT:    [[TMP2:%.*]] = lshr <8 x i32> [[A]], [[B]]
 ; SSE-NEXT:    [[AB6:%.*]] = shl i32 [[A6]], [[B6]]
 ; SSE-NEXT:    [[AB7:%.*]] = shl i32 [[A7]], [[B7]]
 ; SSE-NEXT:    [[R0:%.*]] = insertelement <8 x i32> undef, i32 [[AB0]], i32 0
 ; SSE-NEXT:    [[R1:%.*]] = insertelement <8 x i32> [[R0]], i32 [[AB1]], i32 1
-; SSE-NEXT:    [[R2:%.*]] = insertelement <8 x i32> [[R1]], i32 [[AB2]], i32 2
-; SSE-NEXT:    [[R3:%.*]] = insertelement <8 x i32> [[R2]], i32 [[AB3]], i32 3
-; SSE-NEXT:    [[R4:%.*]] = insertelement <8 x i32> [[R3]], i32 [[AB4]], i32 4
-; SSE-NEXT:    [[R5:%.*]] = insertelement <8 x i32> [[R4]], i32 [[AB5]], i32 5
+; SSE-NEXT:    [[TMP3:%.*]] = extractelement <8 x i32> [[TMP1]], i32 2
+; SSE-NEXT:    [[R2:%.*]] = insertelement <8 x i32> [[R1]], i32 [[TMP3]], i32 2
+; SSE-NEXT:    [[TMP4:%.*]] = extractelement <8 x i32> [[TMP1]], i32 3
+; SSE-NEXT:    [[R3:%.*]] = insertelement <8 x i32> [[R2]], i32 [[TMP4]], i32 3
+; SSE-NEXT:    [[TMP5:%.*]] = extractelement <8 x i32> [[TMP2]], i32 4
+; SSE-NEXT:    [[R4:%.*]] = insertelement <8 x i32> [[R3]], i32 [[TMP5]], i32 4
+; SSE-NEXT:    [[TMP6:%.*]] = extractelement <8 x i32> [[TMP2]], i32 5
+; SSE-NEXT:    [[R5:%.*]] = insertelement <8 x i32> [[R4]], i32 [[TMP6]], i32 5
 ; SSE-NEXT:    [[R6:%.*]] = insertelement <8 x i32> [[R5]], i32 [[AB6]], i32 6
 ; SSE-NEXT:    [[R7:%.*]] = insertelement <8 x i32> [[R6]], i32 [[AB7]], i32 7
 ; SSE-NEXT:    ret <8 x i32> [[R7]]
@@ -486,26 +477,110 @@ define <8 x i32> @add_v8i32_undefs(<8 x i32> %a) {
 }
 
 define <8 x i32> @sdiv_v8i32_undefs(<8 x i32> %a) {
-; CHECK-LABEL: @sdiv_v8i32_undefs(
-; CHECK-NEXT:    [[A1:%.*]] = extractelement <8 x i32> [[A:%.*]], i32 1
-; CHECK-NEXT:    [[A2:%.*]] = extractelement <8 x i32> [[A]], i32 2
-; CHECK-NEXT:    [[A3:%.*]] = extractelement <8 x i32> [[A]], i32 3
-; CHECK-NEXT:    [[A5:%.*]] = extractelement <8 x i32> [[A]], i32 5
-; CHECK-NEXT:    [[A6:%.*]] = extractelement <8 x i32> [[A]], i32 6
-; CHECK-NEXT:    [[A7:%.*]] = extractelement <8 x i32> [[A]], i32 7
-; CHECK-NEXT:    [[AB1:%.*]] = sdiv i32 [[A1]], 4
-; CHECK-NEXT:    [[AB2:%.*]] = sdiv i32 [[A2]], 8
-; CHECK-NEXT:    [[AB3:%.*]] = sdiv i32 [[A3]], 16
-; CHECK-NEXT:    [[AB5:%.*]] = sdiv i32 [[A5]], 4
-; CHECK-NEXT:    [[AB6:%.*]] = sdiv i32 [[A6]], 8
-; CHECK-NEXT:    [[AB7:%.*]] = sdiv i32 [[A7]], 16
-; CHECK-NEXT:    [[R1:%.*]] = insertelement <8 x i32> undef, i32 [[AB1]], i32 1
-; CHECK-NEXT:    [[R2:%.*]] = insertelement <8 x i32> [[R1]], i32 [[AB2]], i32 2
-; CHECK-NEXT:    [[R3:%.*]] = insertelement <8 x i32> [[R2]], i32 [[AB3]], i32 3
-; CHECK-NEXT:    [[R5:%.*]] = insertelement <8 x i32> [[R3]], i32 [[AB5]], i32 5
-; CHECK-NEXT:    [[R6:%.*]] = insertelement <8 x i32> [[R5]], i32 [[AB6]], i32 6
-; CHECK-NEXT:    [[R7:%.*]] = insertelement <8 x i32> [[R6]], i32 [[AB7]], i32 7
-; CHECK-NEXT:    ret <8 x i32> [[R7]]
+; SSE-LABEL: @sdiv_v8i32_undefs(
+; SSE-NEXT:    [[A1:%.*]] = extractelement <8 x i32> [[A:%.*]], i32 1
+; SSE-NEXT:    [[A2:%.*]] = extractelement <8 x i32> [[A]], i32 2
+; SSE-NEXT:    [[A3:%.*]] = extractelement <8 x i32> [[A]], i32 3
+; SSE-NEXT:    [[A5:%.*]] = extractelement <8 x i32> [[A]], i32 5
+; SSE-NEXT:    [[A6:%.*]] = extractelement <8 x i32> [[A]], i32 6
+; SSE-NEXT:    [[A7:%.*]] = extractelement <8 x i32> [[A]], i32 7
+; SSE-NEXT:    [[AB1:%.*]] = sdiv i32 [[A1]], 4
+; SSE-NEXT:    [[AB2:%.*]] = sdiv i32 [[A2]], 8
+; SSE-NEXT:    [[AB3:%.*]] = sdiv i32 [[A3]], 16
+; SSE-NEXT:    [[AB5:%.*]] = sdiv i32 [[A5]], 4
+; SSE-NEXT:    [[AB6:%.*]] = sdiv i32 [[A6]], 8
+; SSE-NEXT:    [[AB7:%.*]] = sdiv i32 [[A7]], 16
+; SSE-NEXT:    [[R1:%.*]] = insertelement <8 x i32> undef, i32 [[AB1]], i32 1
+; SSE-NEXT:    [[R2:%.*]] = insertelement <8 x i32> [[R1]], i32 [[AB2]], i32 2
+; SSE-NEXT:    [[R3:%.*]] = insertelement <8 x i32> [[R2]], i32 [[AB3]], i32 3
+; SSE-NEXT:    [[R5:%.*]] = insertelement <8 x i32> [[R3]], i32 [[AB5]], i32 5
+; SSE-NEXT:    [[R6:%.*]] = insertelement <8 x i32> [[R5]], i32 [[AB6]], i32 6
+; SSE-NEXT:    [[R7:%.*]] = insertelement <8 x i32> [[R6]], i32 [[AB7]], i32 7
+; SSE-NEXT:    ret <8 x i32> [[R7]]
+;
+; SLM-LABEL: @sdiv_v8i32_undefs(
+; SLM-NEXT:    [[A1:%.*]] = extractelement <8 x i32> [[A:%.*]], i32 1
+; SLM-NEXT:    [[A2:%.*]] = extractelement <8 x i32> [[A]], i32 2
+; SLM-NEXT:    [[A3:%.*]] = extractelement <8 x i32> [[A]], i32 3
+; SLM-NEXT:    [[A5:%.*]] = extractelement <8 x i32> [[A]], i32 5
+; SLM-NEXT:    [[A6:%.*]] = extractelement <8 x i32> [[A]], i32 6
+; SLM-NEXT:    [[A7:%.*]] = extractelement <8 x i32> [[A]], i32 7
+; SLM-NEXT:    [[AB1:%.*]] = sdiv i32 [[A1]], 4
+; SLM-NEXT:    [[AB2:%.*]] = sdiv i32 [[A2]], 8
+; SLM-NEXT:    [[AB3:%.*]] = sdiv i32 [[A3]], 16
+; SLM-NEXT:    [[AB5:%.*]] = sdiv i32 [[A5]], 4
+; SLM-NEXT:    [[AB6:%.*]] = sdiv i32 [[A6]], 8
+; SLM-NEXT:    [[AB7:%.*]] = sdiv i32 [[A7]], 16
+; SLM-NEXT:    [[R1:%.*]] = insertelement <8 x i32> undef, i32 [[AB1]], i32 1
+; SLM-NEXT:    [[R2:%.*]] = insertelement <8 x i32> [[R1]], i32 [[AB2]], i32 2
+; SLM-NEXT:    [[R3:%.*]] = insertelement <8 x i32> [[R2]], i32 [[AB3]], i32 3
+; SLM-NEXT:    [[R5:%.*]] = insertelement <8 x i32> [[R3]], i32 [[AB5]], i32 5
+; SLM-NEXT:    [[R6:%.*]] = insertelement <8 x i32> [[R5]], i32 [[AB6]], i32 6
+; SLM-NEXT:    [[R7:%.*]] = insertelement <8 x i32> [[R6]], i32 [[AB7]], i32 7
+; SLM-NEXT:    ret <8 x i32> [[R7]]
+;
+; AVX1-LABEL: @sdiv_v8i32_undefs(
+; AVX1-NEXT:    [[A1:%.*]] = extractelement <8 x i32> [[A:%.*]], i32 1
+; AVX1-NEXT:    [[A2:%.*]] = extractelement <8 x i32> [[A]], i32 2
+; AVX1-NEXT:    [[A3:%.*]] = extractelement <8 x i32> [[A]], i32 3
+; AVX1-NEXT:    [[A5:%.*]] = extractelement <8 x i32> [[A]], i32 5
+; AVX1-NEXT:    [[A6:%.*]] = extractelement <8 x i32> [[A]], i32 6
+; AVX1-NEXT:    [[A7:%.*]] = extractelement <8 x i32> [[A]], i32 7
+; AVX1-NEXT:    [[AB1:%.*]] = sdiv i32 [[A1]], 4
+; AVX1-NEXT:    [[AB2:%.*]] = sdiv i32 [[A2]], 8
+; AVX1-NEXT:    [[AB3:%.*]] = sdiv i32 [[A3]], 16
+; AVX1-NEXT:    [[AB5:%.*]] = sdiv i32 [[A5]], 4
+; AVX1-NEXT:    [[AB6:%.*]] = sdiv i32 [[A6]], 8
+; AVX1-NEXT:    [[AB7:%.*]] = sdiv i32 [[A7]], 16
+; AVX1-NEXT:    [[R1:%.*]] = insertelement <8 x i32> undef, i32 [[AB1]], i32 1
+; AVX1-NEXT:    [[R2:%.*]] = insertelement <8 x i32> [[R1]], i32 [[AB2]], i32 2
+; AVX1-NEXT:    [[R3:%.*]] = insertelement <8 x i32> [[R2]], i32 [[AB3]], i32 3
+; AVX1-NEXT:    [[R5:%.*]] = insertelement <8 x i32> [[R3]], i32 [[AB5]], i32 5
+; AVX1-NEXT:    [[R6:%.*]] = insertelement <8 x i32> [[R5]], i32 [[AB6]], i32 6
+; AVX1-NEXT:    [[R7:%.*]] = insertelement <8 x i32> [[R6]], i32 [[AB7]], i32 7
+; AVX1-NEXT:    ret <8 x i32> [[R7]]
+;
+; AVX2-LABEL: @sdiv_v8i32_undefs(
+; AVX2-NEXT:    [[A3:%.*]] = extractelement <8 x i32> [[A:%.*]], i32 3
+; AVX2-NEXT:    [[A7:%.*]] = extractelement <8 x i32> [[A]], i32 7
+; AVX2-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> undef, <2 x i32> <i32 1, i32 2>
+; AVX2-NEXT:    [[TMP2:%.*]] = sdiv <2 x i32> [[TMP1]], <i32 4, i32 8>
+; AVX2-NEXT:    [[AB3:%.*]] = sdiv i32 [[A3]], 16
+; AVX2-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> undef, <2 x i32> <i32 5, i32 6>
+; AVX2-NEXT:    [[TMP4:%.*]] = sdiv <2 x i32> [[TMP3]], <i32 4, i32 8>
+; AVX2-NEXT:    [[AB7:%.*]] = sdiv i32 [[A7]], 16
+; AVX2-NEXT:    [[TMP5:%.*]] = extractelement <2 x i32> [[TMP2]], i32 0
+; AVX2-NEXT:    [[R1:%.*]] = insertelement <8 x i32> undef, i32 [[TMP5]], i32 1
+; AVX2-NEXT:    [[TMP6:%.*]] = extractelement <2 x i32> [[TMP2]], i32 1
+; AVX2-NEXT:    [[R2:%.*]] = insertelement <8 x i32> [[R1]], i32 [[TMP6]], i32 2
+; AVX2-NEXT:    [[R3:%.*]] = insertelement <8 x i32> [[R2]], i32 [[AB3]], i32 3
+; AVX2-NEXT:    [[TMP7:%.*]] = extractelement <2 x i32> [[TMP4]], i32 0
+; AVX2-NEXT:    [[R5:%.*]] = insertelement <8 x i32> [[R3]], i32 [[TMP7]], i32 5
+; AVX2-NEXT:    [[TMP8:%.*]] = extractelement <2 x i32> [[TMP4]], i32 1
+; AVX2-NEXT:    [[R6:%.*]] = insertelement <8 x i32> [[R5]], i32 [[TMP8]], i32 6
+; AVX2-NEXT:    [[R7:%.*]] = insertelement <8 x i32> [[R6]], i32 [[AB7]], i32 7
+; AVX2-NEXT:    ret <8 x i32> [[R7]]
+;
+; AVX512-LABEL: @sdiv_v8i32_undefs(
+; AVX512-NEXT:    [[A3:%.*]] = extractelement <8 x i32> [[A:%.*]], i32 3
+; AVX512-NEXT:    [[A7:%.*]] = extractelement <8 x i32> [[A]], i32 7
+; AVX512-NEXT:    [[TMP1:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> undef, <2 x i32> <i32 1, i32 2>
+; AVX512-NEXT:    [[TMP2:%.*]] = sdiv <2 x i32> [[TMP1]], <i32 4, i32 8>
+; AVX512-NEXT:    [[AB3:%.*]] = sdiv i32 [[A3]], 16
+; AVX512-NEXT:    [[TMP3:%.*]] = shufflevector <8 x i32> [[A]], <8 x i32> undef, <2 x i32> <i32 5, i32 6>
+; AVX512-NEXT:    [[TMP4:%.*]] = sdiv <2 x i32> [[TMP3]], <i32 4, i32 8>
+; AVX512-NEXT:    [[AB7:%.*]] = sdiv i32 [[A7]], 16
+; AVX512-NEXT:    [[TMP5:%.*]] = extractelement <2 x i32> [[TMP2]], i32 0
+; AVX512-NEXT:    [[R1:%.*]] = insertelement <8 x i32> undef, i32 [[TMP5]], i32 1
+; AVX512-NEXT:    [[TMP6:%.*]] = extractelement <2 x i32> [[TMP2]], i32 1
+; AVX512-NEXT:    [[R2:%.*]] = insertelement <8 x i32> [[R1]], i32 [[TMP6]], i32 2
+; AVX512-NEXT:    [[R3:%.*]] = insertelement <8 x i32> [[R2]], i32 [[AB3]], i32 3
+; AVX512-NEXT:    [[TMP7:%.*]] = extractelement <2 x i32> [[TMP4]], i32 0
+; AVX512-NEXT:    [[R5:%.*]] = insertelement <8 x i32> [[R3]], i32 [[TMP7]], i32 5
+; AVX512-NEXT:    [[TMP8:%.*]] = extractelement <2 x i32> [[TMP4]], i32 1
+; AVX512-NEXT:    [[R6:%.*]] = insertelement <8 x i32> [[R5]], i32 [[TMP8]], i32 6
+; AVX512-NEXT:    [[R7:%.*]] = insertelement <8 x i32> [[R6]], i32 [[AB7]], i32 7
+; AVX512-NEXT:    ret <8 x i32> [[R7]]
 ;
   %a0 = extractelement <8 x i32> %a, i32 0
   %a1 = extractelement <8 x i32> %a, i32 1
diff --git a/test/Transforms/SLPVectorizer/X86/arith-add-ssat.ll b/test/Transforms/SLPVectorizer/X86/arith-add-ssat.ll
new file mode 100644
index 0000000000000000000000000000000000000000..0bf3ea20d6c524c7cdd5cf6782631b593ed44b1a
--- /dev/null
+++ b/test/Transforms/SLPVectorizer/X86/arith-add-ssat.ll
@@ -0,0 +1,775 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -mtriple=x86_64-unknown -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SSE
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SLM
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX1
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512F
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -mattr=-prefer-256-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512BW
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -mattr=+prefer-256-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX256BW
+
+@a64 = common global [8 x i64] zeroinitializer, align 64
+@b64 = common global [8 x i64] zeroinitializer, align 64
+@c64 = common global [8 x i64] zeroinitializer, align 64
+@a32 = common global [16 x i32] zeroinitializer, align 64
+@b32 = common global [16 x i32] zeroinitializer, align 64
+@c32 = common global [16 x i32] zeroinitializer, align 64
+@a16 = common global [32 x i16] zeroinitializer, align 64
+@b16 = common global [32 x i16] zeroinitializer, align 64
+@c16 = common global [32 x i16] zeroinitializer, align 64
+@a8  = common global [64 x i8] zeroinitializer, align 64
+@b8  = common global [64 x i8] zeroinitializer, align 64
+@c8  = common global [64 x i8] zeroinitializer, align 64
+
+declare i64 @llvm.sadd.sat.i64(i64, i64)
+declare i32 @llvm.sadd.sat.i32(i32, i32)
+declare i16 @llvm.sadd.sat.i16(i16, i16)
+declare i8  @llvm.sadd.sat.i8 (i8 , i8 )
+
+define void @add_v8i64() {
+; SSE-LABEL: @add_v8i64(
+; SSE-NEXT:    [[A0:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 0), align 8
+; SSE-NEXT:    [[A1:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 1), align 8
+; SSE-NEXT:    [[A2:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 2), align 8
+; SSE-NEXT:    [[A3:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 3), align 8
+; SSE-NEXT:    [[A4:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 4), align 8
+; SSE-NEXT:    [[A5:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 5), align 8
+; SSE-NEXT:    [[A6:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 6), align 8
+; SSE-NEXT:    [[A7:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 7), align 8
+; SSE-NEXT:    [[B0:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 0), align 8
+; SSE-NEXT:    [[B1:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 1), align 8
+; SSE-NEXT:    [[B2:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 2), align 8
+; SSE-NEXT:    [[B3:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 3), align 8
+; SSE-NEXT:    [[B4:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 4), align 8
+; SSE-NEXT:    [[B5:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 5), align 8
+; SSE-NEXT:    [[B6:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 6), align 8
+; SSE-NEXT:    [[B7:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 7), align 8
+; SSE-NEXT:    [[R0:%.*]] = call i64 @llvm.sadd.sat.i64(i64 [[A0]], i64 [[B0]])
+; SSE-NEXT:    [[R1:%.*]] = call i64 @llvm.sadd.sat.i64(i64 [[A1]], i64 [[B1]])
+; SSE-NEXT:    [[R2:%.*]] = call i64 @llvm.sadd.sat.i64(i64 [[A2]], i64 [[B2]])
+; SSE-NEXT:    [[R3:%.*]] = call i64 @llvm.sadd.sat.i64(i64 [[A3]], i64 [[B3]])
+; SSE-NEXT:    [[R4:%.*]] = call i64 @llvm.sadd.sat.i64(i64 [[A4]], i64 [[B4]])
+; SSE-NEXT:    [[R5:%.*]] = call i64 @llvm.sadd.sat.i64(i64 [[A5]], i64 [[B5]])
+; SSE-NEXT:    [[R6:%.*]] = call i64 @llvm.sadd.sat.i64(i64 [[A6]], i64 [[B6]])
+; SSE-NEXT:    [[R7:%.*]] = call i64 @llvm.sadd.sat.i64(i64 [[A7]], i64 [[B7]])
+; SSE-NEXT:    store i64 [[R0]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 0), align 8
+; SSE-NEXT:    store i64 [[R1]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 1), align 8
+; SSE-NEXT:    store i64 [[R2]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 2), align 8
+; SSE-NEXT:    store i64 [[R3]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 3), align 8
+; SSE-NEXT:    store i64 [[R4]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 4), align 8
+; SSE-NEXT:    store i64 [[R5]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 5), align 8
+; SSE-NEXT:    store i64 [[R6]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 6), align 8
+; SSE-NEXT:    store i64 [[R7]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 7), align 8
+; SSE-NEXT:    ret void
+;
+; SLM-LABEL: @add_v8i64(
+; SLM-NEXT:    [[A0:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 0), align 8
+; SLM-NEXT:    [[A1:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 1), align 8
+; SLM-NEXT:    [[A2:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 2), align 8
+; SLM-NEXT:    [[A3:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 3), align 8
+; SLM-NEXT:    [[A4:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 4), align 8
+; SLM-NEXT:    [[A5:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 5), align 8
+; SLM-NEXT:    [[A6:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 6), align 8
+; SLM-NEXT:    [[A7:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 7), align 8
+; SLM-NEXT:    [[B0:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 0), align 8
+; SLM-NEXT:    [[B1:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 1), align 8
+; SLM-NEXT:    [[B2:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 2), align 8
+; SLM-NEXT:    [[B3:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 3), align 8
+; SLM-NEXT:    [[B4:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 4), align 8
+; SLM-NEXT:    [[B5:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 5), align 8
+; SLM-NEXT:    [[B6:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 6), align 8
+; SLM-NEXT:    [[B7:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 7), align 8
+; SLM-NEXT:    [[R0:%.*]] = call i64 @llvm.sadd.sat.i64(i64 [[A0]], i64 [[B0]])
+; SLM-NEXT:    [[R1:%.*]] = call i64 @llvm.sadd.sat.i64(i64 [[A1]], i64 [[B1]])
+; SLM-NEXT:    [[R2:%.*]] = call i64 @llvm.sadd.sat.i64(i64 [[A2]], i64 [[B2]])
+; SLM-NEXT:    [[R3:%.*]] = call i64 @llvm.sadd.sat.i64(i64 [[A3]], i64 [[B3]])
+; SLM-NEXT:    [[R4:%.*]] = call i64 @llvm.sadd.sat.i64(i64 [[A4]], i64 [[B4]])
+; SLM-NEXT:    [[R5:%.*]] = call i64 @llvm.sadd.sat.i64(i64 [[A5]], i64 [[B5]])
+; SLM-NEXT:    [[R6:%.*]] = call i64 @llvm.sadd.sat.i64(i64 [[A6]], i64 [[B6]])
+; SLM-NEXT:    [[R7:%.*]] = call i64 @llvm.sadd.sat.i64(i64 [[A7]], i64 [[B7]])
+; SLM-NEXT:    store i64 [[R0]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 0), align 8
+; SLM-NEXT:    store i64 [[R1]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 1), align 8
+; SLM-NEXT:    store i64 [[R2]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 2), align 8
+; SLM-NEXT:    store i64 [[R3]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 3), align 8
+; SLM-NEXT:    store i64 [[R4]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 4), align 8
+; SLM-NEXT:    store i64 [[R5]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 5), align 8
+; SLM-NEXT:    store i64 [[R6]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 6), align 8
+; SLM-NEXT:    store i64 [[R7]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 7), align 8
+; SLM-NEXT:    ret void
+;
+; AVX1-LABEL: @add_v8i64(
+; AVX1-NEXT:    [[TMP1:%.*]] = load <2 x i64>, <2 x i64>* bitcast ([8 x i64]* @a64 to <2 x i64>*), align 8
+; AVX1-NEXT:    [[TMP2:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 2) to <2 x i64>*), align 8
+; AVX1-NEXT:    [[TMP3:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 4) to <2 x i64>*), align 8
+; AVX1-NEXT:    [[TMP4:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 6) to <2 x i64>*), align 8
+; AVX1-NEXT:    [[TMP5:%.*]] = load <2 x i64>, <2 x i64>* bitcast ([8 x i64]* @b64 to <2 x i64>*), align 8
+; AVX1-NEXT:    [[TMP6:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 2) to <2 x i64>*), align 8
+; AVX1-NEXT:    [[TMP7:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 4) to <2 x i64>*), align 8
+; AVX1-NEXT:    [[TMP8:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 6) to <2 x i64>*), align 8
+; AVX1-NEXT:    [[TMP9:%.*]] = call <2 x i64> @llvm.sadd.sat.v2i64(<2 x i64> [[TMP1]], <2 x i64> [[TMP5]])
+; AVX1-NEXT:    [[TMP10:%.*]] = call <2 x i64> @llvm.sadd.sat.v2i64(<2 x i64> [[TMP2]], <2 x i64> [[TMP6]])
+; AVX1-NEXT:    [[TMP11:%.*]] = call <2 x i64> @llvm.sadd.sat.v2i64(<2 x i64> [[TMP3]], <2 x i64> [[TMP7]])
+; AVX1-NEXT:    [[TMP12:%.*]] = call <2 x i64> @llvm.sadd.sat.v2i64(<2 x i64> [[TMP4]], <2 x i64> [[TMP8]])
+; AVX1-NEXT:    store <2 x i64> [[TMP9]], <2 x i64>* bitcast ([8 x i64]* @c64 to <2 x i64>*), align 8
+; AVX1-NEXT:    store <2 x i64> [[TMP10]], <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 2) to <2 x i64>*), align 8
+; AVX1-NEXT:    store <2 x i64> [[TMP11]], <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 4) to <2 x i64>*), align 8
+; AVX1-NEXT:    store <2 x i64> [[TMP12]], <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 6) to <2 x i64>*), align 8
+; AVX1-NEXT:    ret void
+;
+; AVX2-LABEL: @add_v8i64(
+; AVX2-NEXT:    [[TMP1:%.*]] = load <4 x i64>, <4 x i64>* bitcast ([8 x i64]* @a64 to <4 x i64>*), align 8
+; AVX2-NEXT:    [[TMP2:%.*]] = load <4 x i64>, <4 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 4) to <4 x i64>*), align 8
+; AVX2-NEXT:    [[TMP3:%.*]] = load <4 x i64>, <4 x i64>* bitcast ([8 x i64]* @b64 to <4 x i64>*), align 8
+; AVX2-NEXT:    [[TMP4:%.*]] = load <4 x i64>, <4 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 4) to <4 x i64>*), align 8
+; AVX2-NEXT:    [[TMP5:%.*]] = call <4 x i64> @llvm.sadd.sat.v4i64(<4 x i64> [[TMP1]], <4 x i64> [[TMP3]])
+; AVX2-NEXT:    [[TMP6:%.*]] = call <4 x i64> @llvm.sadd.sat.v4i64(<4 x i64> [[TMP2]], <4 x i64> [[TMP4]])
+; AVX2-NEXT:    store <4 x i64> [[TMP5]], <4 x i64>* bitcast ([8 x i64]* @c64 to <4 x i64>*), align 8
+; AVX2-NEXT:    store <4 x i64> [[TMP6]], <4 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 4) to <4 x i64>*), align 8
+; AVX2-NEXT:    ret void
+;
+; AVX512-LABEL: @add_v8i64(
+; AVX512-NEXT:    [[TMP1:%.*]] = load <8 x i64>, <8 x i64>* bitcast ([8 x i64]* @a64 to <8 x i64>*), align 8
+; AVX512-NEXT:    [[TMP2:%.*]] = load <8 x i64>, <8 x i64>* bitcast ([8 x i64]* @b64 to <8 x i64>*), align 8
+; AVX512-NEXT:    [[TMP3:%.*]] = call <8 x i64> @llvm.sadd.sat.v8i64(<8 x i64> [[TMP1]], <8 x i64> [[TMP2]])
+; AVX512-NEXT:    store <8 x i64> [[TMP3]], <8 x i64>* bitcast ([8 x i64]* @c64 to <8 x i64>*), align 8
+; AVX512-NEXT:    ret void
+;
+; AVX256BW-LABEL: @add_v8i64(
+; AVX256BW-NEXT:    [[TMP1:%.*]] = load <4 x i64>, <4 x i64>* bitcast ([8 x i64]* @a64 to <4 x i64>*), align 8
+; AVX256BW-NEXT:    [[TMP2:%.*]] = load <4 x i64>, <4 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 4) to <4 x i64>*), align 8
+; AVX256BW-NEXT:    [[TMP3:%.*]] = load <4 x i64>, <4 x i64>* bitcast ([8 x i64]* @b64 to <4 x i64>*), align 8
+; AVX256BW-NEXT:    [[TMP4:%.*]] = load <4 x i64>, <4 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 4) to <4 x i64>*), align 8
+; AVX256BW-NEXT:    [[TMP5:%.*]] = call <4 x i64> @llvm.sadd.sat.v4i64(<4 x i64> [[TMP1]], <4 x i64> [[TMP3]])
+; AVX256BW-NEXT:    [[TMP6:%.*]] = call <4 x i64> @llvm.sadd.sat.v4i64(<4 x i64> [[TMP2]], <4 x i64> [[TMP4]])
+; AVX256BW-NEXT:    store <4 x i64> [[TMP5]], <4 x i64>* bitcast ([8 x i64]* @c64 to <4 x i64>*), align 8
+; AVX256BW-NEXT:    store <4 x i64> [[TMP6]], <4 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 4) to <4 x i64>*), align 8
+; AVX256BW-NEXT:    ret void
+;
+  %a0 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 0), align 8
+  %a1 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 1), align 8
+  %a2 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 2), align 8
+  %a3 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 3), align 8
+  %a4 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 4), align 8
+  %a5 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 5), align 8
+  %a6 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 6), align 8
+  %a7 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 7), align 8
+  %b0 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 0), align 8
+  %b1 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 1), align 8
+  %b2 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 2), align 8
+  %b3 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 3), align 8
+  %b4 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 4), align 8
+  %b5 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 5), align 8
+  %b6 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 6), align 8
+  %b7 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 7), align 8
+  %r0 = call i64 @llvm.sadd.sat.i64(i64 %a0, i64 %b0)
+  %r1 = call i64 @llvm.sadd.sat.i64(i64 %a1, i64 %b1)
+  %r2 = call i64 @llvm.sadd.sat.i64(i64 %a2, i64 %b2)
+  %r3 = call i64 @llvm.sadd.sat.i64(i64 %a3, i64 %b3)
+  %r4 = call i64 @llvm.sadd.sat.i64(i64 %a4, i64 %b4)
+  %r5 = call i64 @llvm.sadd.sat.i64(i64 %a5, i64 %b5)
+  %r6 = call i64 @llvm.sadd.sat.i64(i64 %a6, i64 %b6)
+  %r7 = call i64 @llvm.sadd.sat.i64(i64 %a7, i64 %b7)
+  store i64 %r0, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 0), align 8
+  store i64 %r1, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 1), align 8
+  store i64 %r2, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 2), align 8
+  store i64 %r3, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 3), align 8
+  store i64 %r4, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 4), align 8
+  store i64 %r5, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 5), align 8
+  store i64 %r6, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 6), align 8
+  store i64 %r7, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 7), align 8
+  ret void
+}
+
+define void @add_v16i32() {
+; SSE-LABEL: @add_v16i32(
+; SSE-NEXT:    [[TMP1:%.*]] = load <4 x i32>, <4 x i32>* bitcast ([16 x i32]* @a32 to <4 x i32>*), align 4
+; SSE-NEXT:    [[TMP2:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 4) to <4 x i32>*), align 4
+; SSE-NEXT:    [[TMP3:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 8) to <4 x i32>*), align 4
+; SSE-NEXT:    [[TMP4:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 12) to <4 x i32>*), align 4
+; SSE-NEXT:    [[TMP5:%.*]] = load <4 x i32>, <4 x i32>* bitcast ([16 x i32]* @b32 to <4 x i32>*), align 4
+; SSE-NEXT:    [[TMP6:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 4) to <4 x i32>*), align 4
+; SSE-NEXT:    [[TMP7:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 8) to <4 x i32>*), align 4
+; SSE-NEXT:    [[TMP8:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 12) to <4 x i32>*), align 4
+; SSE-NEXT:    [[TMP9:%.*]] = call <4 x i32> @llvm.sadd.sat.v4i32(<4 x i32> [[TMP1]], <4 x i32> [[TMP5]])
+; SSE-NEXT:    [[TMP10:%.*]] = call <4 x i32> @llvm.sadd.sat.v4i32(<4 x i32> [[TMP2]], <4 x i32> [[TMP6]])
+; SSE-NEXT:    [[TMP11:%.*]] = call <4 x i32> @llvm.sadd.sat.v4i32(<4 x i32> [[TMP3]], <4 x i32> [[TMP7]])
+; SSE-NEXT:    [[TMP12:%.*]] = call <4 x i32> @llvm.sadd.sat.v4i32(<4 x i32> [[TMP4]], <4 x i32> [[TMP8]])
+; SSE-NEXT:    store <4 x i32> [[TMP9]], <4 x i32>* bitcast ([16 x i32]* @c32 to <4 x i32>*), align 4
+; SSE-NEXT:    store <4 x i32> [[TMP10]], <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 4) to <4 x i32>*), align 4
+; SSE-NEXT:    store <4 x i32> [[TMP11]], <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 8) to <4 x i32>*), align 4
+; SSE-NEXT:    store <4 x i32> [[TMP12]], <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 12) to <4 x i32>*), align 4
+; SSE-NEXT:    ret void
+;
+; SLM-LABEL: @add_v16i32(
+; SLM-NEXT:    [[TMP1:%.*]] = load <4 x i32>, <4 x i32>* bitcast ([16 x i32]* @a32 to <4 x i32>*), align 4
+; SLM-NEXT:    [[TMP2:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 4) to <4 x i32>*), align 4
+; SLM-NEXT:    [[TMP3:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 8) to <4 x i32>*), align 4
+; SLM-NEXT:    [[TMP4:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 12) to <4 x i32>*), align 4
+; SLM-NEXT:    [[TMP5:%.*]] = load <4 x i32>, <4 x i32>* bitcast ([16 x i32]* @b32 to <4 x i32>*), align 4
+; SLM-NEXT:    [[TMP6:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 4) to <4 x i32>*), align 4
+; SLM-NEXT:    [[TMP7:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 8) to <4 x i32>*), align 4
+; SLM-NEXT:    [[TMP8:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 12) to <4 x i32>*), align 4
+; SLM-NEXT:    [[TMP9:%.*]] = call <4 x i32> @llvm.sadd.sat.v4i32(<4 x i32> [[TMP1]], <4 x i32> [[TMP5]])
+; SLM-NEXT:    [[TMP10:%.*]] = call <4 x i32> @llvm.sadd.sat.v4i32(<4 x i32> [[TMP2]], <4 x i32> [[TMP6]])
+; SLM-NEXT:    [[TMP11:%.*]] = call <4 x i32> @llvm.sadd.sat.v4i32(<4 x i32> [[TMP3]], <4 x i32> [[TMP7]])
+; SLM-NEXT:    [[TMP12:%.*]] = call <4 x i32> @llvm.sadd.sat.v4i32(<4 x i32> [[TMP4]], <4 x i32> [[TMP8]])
+; SLM-NEXT:    store <4 x i32> [[TMP9]], <4 x i32>* bitcast ([16 x i32]* @c32 to <4 x i32>*), align 4
+; SLM-NEXT:    store <4 x i32> [[TMP10]], <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 4) to <4 x i32>*), align 4
+; SLM-NEXT:    store <4 x i32> [[TMP11]], <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 8) to <4 x i32>*), align 4
+; SLM-NEXT:    store <4 x i32> [[TMP12]], <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 12) to <4 x i32>*), align 4
+; SLM-NEXT:    ret void
+;
+; AVX-LABEL: @add_v16i32(
+; AVX-NEXT:    [[TMP1:%.*]] = load <8 x i32>, <8 x i32>* bitcast ([16 x i32]* @a32 to <8 x i32>*), align 4
+; AVX-NEXT:    [[TMP2:%.*]] = load <8 x i32>, <8 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 8) to <8 x i32>*), align 4
+; AVX-NEXT:    [[TMP3:%.*]] = load <8 x i32>, <8 x i32>* bitcast ([16 x i32]* @b32 to <8 x i32>*), align 4
+; AVX-NEXT:    [[TMP4:%.*]] = load <8 x i32>, <8 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 8) to <8 x i32>*), align 4
+; AVX-NEXT:    [[TMP5:%.*]] = call <8 x i32> @llvm.sadd.sat.v8i32(<8 x i32> [[TMP1]], <8 x i32> [[TMP3]])
+; AVX-NEXT:    [[TMP6:%.*]] = call <8 x i32> @llvm.sadd.sat.v8i32(<8 x i32> [[TMP2]], <8 x i32> [[TMP4]])
+; AVX-NEXT:    store <8 x i32> [[TMP5]], <8 x i32>* bitcast ([16 x i32]* @c32 to <8 x i32>*), align 4
+; AVX-NEXT:    store <8 x i32> [[TMP6]], <8 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 8) to <8 x i32>*), align 4
+; AVX-NEXT:    ret void
+;
+; AVX512-LABEL: @add_v16i32(
+; AVX512-NEXT:    [[TMP1:%.*]] = load <16 x i32>, <16 x i32>* bitcast ([16 x i32]* @a32 to <16 x i32>*), align 4
+; AVX512-NEXT:    [[TMP2:%.*]] = load <16 x i32>, <16 x i32>* bitcast ([16 x i32]* @b32 to <16 x i32>*), align 4
+; AVX512-NEXT:    [[TMP3:%.*]] = call <16 x i32> @llvm.sadd.sat.v16i32(<16 x i32> [[TMP1]], <16 x i32> [[TMP2]])
+; AVX512-NEXT:    store <16 x i32> [[TMP3]], <16 x i32>* bitcast ([16 x i32]* @c32 to <16 x i32>*), align 4
+; AVX512-NEXT:    ret void
+;
+  %a0  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 0 ), align 4
+  %a1  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 1 ), align 4
+  %a2  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 2 ), align 4
+  %a3  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 3 ), align 4
+  %a4  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 4 ), align 4
+  %a5  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 5 ), align 4
+  %a6  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 6 ), align 4
+  %a7  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 7 ), align 4
+  %a8  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 8 ), align 4
+  %a9  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 9 ), align 4
+  %a10 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 10), align 4
+  %a11 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 11), align 4
+  %a12 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 12), align 4
+  %a13 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 13), align 4
+  %a14 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 14), align 4
+  %a15 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 15), align 4
+  %b0  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 0 ), align 4
+  %b1  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 1 ), align 4
+  %b2  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 2 ), align 4
+  %b3  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 3 ), align 4
+  %b4  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 4 ), align 4
+  %b5  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 5 ), align 4
+  %b6  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 6 ), align 4
+  %b7  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 7 ), align 4
+  %b8  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 8 ), align 4
+  %b9  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 9 ), align 4
+  %b10 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 10), align 4
+  %b11 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 11), align 4
+  %b12 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 12), align 4
+  %b13 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 13), align 4
+  %b14 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 14), align 4
+  %b15 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 15), align 4
+  %r0  = call i32 @llvm.sadd.sat.i32(i32 %a0 , i32 %b0 )
+  %r1  = call i32 @llvm.sadd.sat.i32(i32 %a1 , i32 %b1 )
+  %r2  = call i32 @llvm.sadd.sat.i32(i32 %a2 , i32 %b2 )
+  %r3  = call i32 @llvm.sadd.sat.i32(i32 %a3 , i32 %b3 )
+  %r4  = call i32 @llvm.sadd.sat.i32(i32 %a4 , i32 %b4 )
+  %r5  = call i32 @llvm.sadd.sat.i32(i32 %a5 , i32 %b5 )
+  %r6  = call i32 @llvm.sadd.sat.i32(i32 %a6 , i32 %b6 )
+  %r7  = call i32 @llvm.sadd.sat.i32(i32 %a7 , i32 %b7 )
+  %r8  = call i32 @llvm.sadd.sat.i32(i32 %a8 , i32 %b8 )
+  %r9  = call i32 @llvm.sadd.sat.i32(i32 %a9 , i32 %b9 )
+  %r10 = call i32 @llvm.sadd.sat.i32(i32 %a10, i32 %b10)
+  %r11 = call i32 @llvm.sadd.sat.i32(i32 %a11, i32 %b11)
+  %r12 = call i32 @llvm.sadd.sat.i32(i32 %a12, i32 %b12)
+  %r13 = call i32 @llvm.sadd.sat.i32(i32 %a13, i32 %b13)
+  %r14 = call i32 @llvm.sadd.sat.i32(i32 %a14, i32 %b14)
+  %r15 = call i32 @llvm.sadd.sat.i32(i32 %a15, i32 %b15)
+  store i32 %r0 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 0 ), align 4
+  store i32 %r1 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 1 ), align 4
+  store i32 %r2 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 2 ), align 4
+  store i32 %r3 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 3 ), align 4
+  store i32 %r4 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 4 ), align 4
+  store i32 %r5 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 5 ), align 4
+  store i32 %r6 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 6 ), align 4
+  store i32 %r7 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 7 ), align 4
+  store i32 %r8 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 8 ), align 4
+  store i32 %r9 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 9 ), align 4
+  store i32 %r10, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 10), align 4
+  store i32 %r11, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 11), align 4
+  store i32 %r12, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 12), align 4
+  store i32 %r13, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 13), align 4
+  store i32 %r14, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 14), align 4
+  store i32 %r15, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 15), align 4
+  ret void
+}
+
+define void @add_v32i16() {
+; SSE-LABEL: @add_v32i16(
+; SSE-NEXT:    [[TMP1:%.*]] = load <8 x i16>, <8 x i16>* bitcast ([32 x i16]* @a16 to <8 x i16>*), align 2
+; SSE-NEXT:    [[TMP2:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 8) to <8 x i16>*), align 2
+; SSE-NEXT:    [[TMP3:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 16) to <8 x i16>*), align 2
+; SSE-NEXT:    [[TMP4:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 24) to <8 x i16>*), align 2
+; SSE-NEXT:    [[TMP5:%.*]] = load <8 x i16>, <8 x i16>* bitcast ([32 x i16]* @b16 to <8 x i16>*), align 2
+; SSE-NEXT:    [[TMP6:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 8) to <8 x i16>*), align 2
+; SSE-NEXT:    [[TMP7:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 16) to <8 x i16>*), align 2
+; SSE-NEXT:    [[TMP8:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 24) to <8 x i16>*), align 2
+; SSE-NEXT:    [[TMP9:%.*]] = call <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16> [[TMP1]], <8 x i16> [[TMP5]])
+; SSE-NEXT:    [[TMP10:%.*]] = call <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16> [[TMP2]], <8 x i16> [[TMP6]])
+; SSE-NEXT:    [[TMP11:%.*]] = call <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16> [[TMP3]], <8 x i16> [[TMP7]])
+; SSE-NEXT:    [[TMP12:%.*]] = call <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16> [[TMP4]], <8 x i16> [[TMP8]])
+; SSE-NEXT:    store <8 x i16> [[TMP9]], <8 x i16>* bitcast ([32 x i16]* @c16 to <8 x i16>*), align 2
+; SSE-NEXT:    store <8 x i16> [[TMP10]], <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 8) to <8 x i16>*), align 2
+; SSE-NEXT:    store <8 x i16> [[TMP11]], <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 16) to <8 x i16>*), align 2
+; SSE-NEXT:    store <8 x i16> [[TMP12]], <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 24) to <8 x i16>*), align 2
+; SSE-NEXT:    ret void
+;
+; SLM-LABEL: @add_v32i16(
+; SLM-NEXT:    [[TMP1:%.*]] = load <8 x i16>, <8 x i16>* bitcast ([32 x i16]* @a16 to <8 x i16>*), align 2
+; SLM-NEXT:    [[TMP2:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 8) to <8 x i16>*), align 2
+; SLM-NEXT:    [[TMP3:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 16) to <8 x i16>*), align 2
+; SLM-NEXT:    [[TMP4:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 24) to <8 x i16>*), align 2
+; SLM-NEXT:    [[TMP5:%.*]] = load <8 x i16>, <8 x i16>* bitcast ([32 x i16]* @b16 to <8 x i16>*), align 2
+; SLM-NEXT:    [[TMP6:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 8) to <8 x i16>*), align 2
+; SLM-NEXT:    [[TMP7:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 16) to <8 x i16>*), align 2
+; SLM-NEXT:    [[TMP8:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 24) to <8 x i16>*), align 2
+; SLM-NEXT:    [[TMP9:%.*]] = call <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16> [[TMP1]], <8 x i16> [[TMP5]])
+; SLM-NEXT:    [[TMP10:%.*]] = call <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16> [[TMP2]], <8 x i16> [[TMP6]])
+; SLM-NEXT:    [[TMP11:%.*]] = call <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16> [[TMP3]], <8 x i16> [[TMP7]])
+; SLM-NEXT:    [[TMP12:%.*]] = call <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16> [[TMP4]], <8 x i16> [[TMP8]])
+; SLM-NEXT:    store <8 x i16> [[TMP9]], <8 x i16>* bitcast ([32 x i16]* @c16 to <8 x i16>*), align 2
+; SLM-NEXT:    store <8 x i16> [[TMP10]], <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 8) to <8 x i16>*), align 2
+; SLM-NEXT:    store <8 x i16> [[TMP11]], <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 16) to <8 x i16>*), align 2
+; SLM-NEXT:    store <8 x i16> [[TMP12]], <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 24) to <8 x i16>*), align 2
+; SLM-NEXT:    ret void
+;
+; AVX-LABEL: @add_v32i16(
+; AVX-NEXT:    [[TMP1:%.*]] = load <16 x i16>, <16 x i16>* bitcast ([32 x i16]* @a16 to <16 x i16>*), align 2
+; AVX-NEXT:    [[TMP2:%.*]] = load <16 x i16>, <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 16) to <16 x i16>*), align 2
+; AVX-NEXT:    [[TMP3:%.*]] = load <16 x i16>, <16 x i16>* bitcast ([32 x i16]* @b16 to <16 x i16>*), align 2
+; AVX-NEXT:    [[TMP4:%.*]] = load <16 x i16>, <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 16) to <16 x i16>*), align 2
+; AVX-NEXT:    [[TMP5:%.*]] = call <16 x i16> @llvm.sadd.sat.v16i16(<16 x i16> [[TMP1]], <16 x i16> [[TMP3]])
+; AVX-NEXT:    [[TMP6:%.*]] = call <16 x i16> @llvm.sadd.sat.v16i16(<16 x i16> [[TMP2]], <16 x i16> [[TMP4]])
+; AVX-NEXT:    store <16 x i16> [[TMP5]], <16 x i16>* bitcast ([32 x i16]* @c16 to <16 x i16>*), align 2
+; AVX-NEXT:    store <16 x i16> [[TMP6]], <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 16) to <16 x i16>*), align 2
+; AVX-NEXT:    ret void
+;
+; AVX512-LABEL: @add_v32i16(
+; AVX512-NEXT:    [[TMP1:%.*]] = load <16 x i16>, <16 x i16>* bitcast ([32 x i16]* @a16 to <16 x i16>*), align 2
+; AVX512-NEXT:    [[TMP2:%.*]] = load <16 x i16>, <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 16) to <16 x i16>*), align 2
+; AVX512-NEXT:    [[TMP3:%.*]] = load <16 x i16>, <16 x i16>* bitcast ([32 x i16]* @b16 to <16 x i16>*), align 2
+; AVX512-NEXT:    [[TMP4:%.*]] = load <16 x i16>, <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 16) to <16 x i16>*), align 2
+; AVX512-NEXT:    [[TMP5:%.*]] = call <16 x i16> @llvm.sadd.sat.v16i16(<16 x i16> [[TMP1]], <16 x i16> [[TMP3]])
+; AVX512-NEXT:    [[TMP6:%.*]] = call <16 x i16> @llvm.sadd.sat.v16i16(<16 x i16> [[TMP2]], <16 x i16> [[TMP4]])
+; AVX512-NEXT:    store <16 x i16> [[TMP5]], <16 x i16>* bitcast ([32 x i16]* @c16 to <16 x i16>*), align 2
+; AVX512-NEXT:    store <16 x i16> [[TMP6]], <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 16) to <16 x i16>*), align 2
+; AVX512-NEXT:    ret void
+;
+  %a0  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 0 ), align 2
+  %a1  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 1 ), align 2
+  %a2  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 2 ), align 2
+  %a3  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 3 ), align 2
+  %a4  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 4 ), align 2
+  %a5  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 5 ), align 2
+  %a6  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 6 ), align 2
+  %a7  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 7 ), align 2
+  %a8  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 8 ), align 2
+  %a9  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 9 ), align 2
+  %a10 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 10), align 2
+  %a11 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 11), align 2
+  %a12 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 12), align 2
+  %a13 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 13), align 2
+  %a14 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 14), align 2
+  %a15 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 15), align 2
+  %a16 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 16), align 2
+  %a17 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 17), align 2
+  %a18 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 18), align 2
+  %a19 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 19), align 2
+  %a20 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 20), align 2
+  %a21 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 21), align 2
+  %a22 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 22), align 2
+  %a23 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 23), align 2
+  %a24 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 24), align 2
+  %a25 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 25), align 2
+  %a26 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 26), align 2
+  %a27 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 27), align 2
+  %a28 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 28), align 2
+  %a29 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 29), align 2
+  %a30 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 30), align 2
+  %a31 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 31), align 2
+  %b0  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 0 ), align 2
+  %b1  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 1 ), align 2
+  %b2  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 2 ), align 2
+  %b3  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 3 ), align 2
+  %b4  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 4 ), align 2
+  %b5  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 5 ), align 2
+  %b6  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 6 ), align 2
+  %b7  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 7 ), align 2
+  %b8  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 8 ), align 2
+  %b9  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 9 ), align 2
+  %b10 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 10), align 2
+  %b11 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 11), align 2
+  %b12 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 12), align 2
+  %b13 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 13), align 2
+  %b14 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 14), align 2
+  %b15 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 15), align 2
+  %b16 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 16), align 2
+  %b17 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 17), align 2
+  %b18 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 18), align 2
+  %b19 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 19), align 2
+  %b20 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 20), align 2
+  %b21 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 21), align 2
+  %b22 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 22), align 2
+  %b23 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 23), align 2
+  %b24 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 24), align 2
+  %b25 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 25), align 2
+  %b26 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 26), align 2
+  %b27 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 27), align 2
+  %b28 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 28), align 2
+  %b29 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 29), align 2
+  %b30 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 30), align 2
+  %b31 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 31), align 2
+  %r0  = call i16 @llvm.sadd.sat.i16(i16 %a0 , i16 %b0 )
+  %r1  = call i16 @llvm.sadd.sat.i16(i16 %a1 , i16 %b1 )
+  %r2  = call i16 @llvm.sadd.sat.i16(i16 %a2 , i16 %b2 )
+  %r3  = call i16 @llvm.sadd.sat.i16(i16 %a3 , i16 %b3 )
+  %r4  = call i16 @llvm.sadd.sat.i16(i16 %a4 , i16 %b4 )
+  %r5  = call i16 @llvm.sadd.sat.i16(i16 %a5 , i16 %b5 )
+  %r6  = call i16 @llvm.sadd.sat.i16(i16 %a6 , i16 %b6 )
+  %r7  = call i16 @llvm.sadd.sat.i16(i16 %a7 , i16 %b7 )
+  %r8  = call i16 @llvm.sadd.sat.i16(i16 %a8 , i16 %b8 )
+  %r9  = call i16 @llvm.sadd.sat.i16(i16 %a9 , i16 %b9 )
+  %r10 = call i16 @llvm.sadd.sat.i16(i16 %a10, i16 %b10)
+  %r11 = call i16 @llvm.sadd.sat.i16(i16 %a11, i16 %b11)
+  %r12 = call i16 @llvm.sadd.sat.i16(i16 %a12, i16 %b12)
+  %r13 = call i16 @llvm.sadd.sat.i16(i16 %a13, i16 %b13)
+  %r14 = call i16 @llvm.sadd.sat.i16(i16 %a14, i16 %b14)
+  %r15 = call i16 @llvm.sadd.sat.i16(i16 %a15, i16 %b15)
+  %r16 = call i16 @llvm.sadd.sat.i16(i16 %a16, i16 %b16)
+  %r17 = call i16 @llvm.sadd.sat.i16(i16 %a17, i16 %b17)
+  %r18 = call i16 @llvm.sadd.sat.i16(i16 %a18, i16 %b18)
+  %r19 = call i16 @llvm.sadd.sat.i16(i16 %a19, i16 %b19)
+  %r20 = call i16 @llvm.sadd.sat.i16(i16 %a20, i16 %b20)
+  %r21 = call i16 @llvm.sadd.sat.i16(i16 %a21, i16 %b21)
+  %r22 = call i16 @llvm.sadd.sat.i16(i16 %a22, i16 %b22)
+  %r23 = call i16 @llvm.sadd.sat.i16(i16 %a23, i16 %b23)
+  %r24 = call i16 @llvm.sadd.sat.i16(i16 %a24, i16 %b24)
+  %r25 = call i16 @llvm.sadd.sat.i16(i16 %a25, i16 %b25)
+  %r26 = call i16 @llvm.sadd.sat.i16(i16 %a26, i16 %b26)
+  %r27 = call i16 @llvm.sadd.sat.i16(i16 %a27, i16 %b27)
+  %r28 = call i16 @llvm.sadd.sat.i16(i16 %a28, i16 %b28)
+  %r29 = call i16 @llvm.sadd.sat.i16(i16 %a29, i16 %b29)
+  %r30 = call i16 @llvm.sadd.sat.i16(i16 %a30, i16 %b30)
+  %r31 = call i16 @llvm.sadd.sat.i16(i16 %a31, i16 %b31)
+  store i16 %r0 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 0 ), align 2
+  store i16 %r1 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 1 ), align 2
+  store i16 %r2 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 2 ), align 2
+  store i16 %r3 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 3 ), align 2
+  store i16 %r4 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 4 ), align 2
+  store i16 %r5 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 5 ), align 2
+  store i16 %r6 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 6 ), align 2
+  store i16 %r7 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 7 ), align 2
+  store i16 %r8 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 8 ), align 2
+  store i16 %r9 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 9 ), align 2
+  store i16 %r10, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 10), align 2
+  store i16 %r11, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 11), align 2
+  store i16 %r12, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 12), align 2
+  store i16 %r13, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 13), align 2
+  store i16 %r14, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 14), align 2
+  store i16 %r15, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 15), align 2
+  store i16 %r16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 16), align 2
+  store i16 %r17, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 17), align 2
+  store i16 %r18, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 18), align 2
+  store i16 %r19, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 19), align 2
+  store i16 %r20, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 20), align 2
+  store i16 %r21, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 21), align 2
+  store i16 %r22, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 22), align 2
+  store i16 %r23, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 23), align 2
+  store i16 %r24, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 24), align 2
+  store i16 %r25, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 25), align 2
+  store i16 %r26, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 26), align 2
+  store i16 %r27, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 27), align 2
+  store i16 %r28, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 28), align 2
+  store i16 %r29, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 29), align 2
+  store i16 %r30, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 30), align 2
+  store i16 %r31, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 31), align 2
+  ret void
+}
+
+define void @add_v64i8() {
+; CHECK-LABEL: @add_v64i8(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i8>, <16 x i8>* bitcast ([64 x i8]* @a8 to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 16) to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 32) to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 48) to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP5:%.*]] = load <16 x i8>, <16 x i8>* bitcast ([64 x i8]* @b8 to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP6:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 16) to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP7:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 32) to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP8:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 48) to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP9:%.*]] = call <16 x i8> @llvm.sadd.sat.v16i8(<16 x i8> [[TMP1]], <16 x i8> [[TMP5]])
+; CHECK-NEXT:    [[TMP10:%.*]] = call <16 x i8> @llvm.sadd.sat.v16i8(<16 x i8> [[TMP2]], <16 x i8> [[TMP6]])
+; CHECK-NEXT:    [[TMP11:%.*]] = call <16 x i8> @llvm.sadd.sat.v16i8(<16 x i8> [[TMP3]], <16 x i8> [[TMP7]])
+; CHECK-NEXT:    [[TMP12:%.*]] = call <16 x i8> @llvm.sadd.sat.v16i8(<16 x i8> [[TMP4]], <16 x i8> [[TMP8]])
+; CHECK-NEXT:    store <16 x i8> [[TMP9]], <16 x i8>* bitcast ([64 x i8]* @c8 to <16 x i8>*), align 1
+; CHECK-NEXT:    store <16 x i8> [[TMP10]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 16) to <16 x i8>*), align 1
+; CHECK-NEXT:    store <16 x i8> [[TMP11]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 32) to <16 x i8>*), align 1
+; CHECK-NEXT:    store <16 x i8> [[TMP12]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 48) to <16 x i8>*), align 1
+; CHECK-NEXT:    ret void
+;
+  %a0  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 0 ), align 1
+  %a1  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 1 ), align 1
+  %a2  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 2 ), align 1
+  %a3  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 3 ), align 1
+  %a4  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 4 ), align 1
+  %a5  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 5 ), align 1
+  %a6  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 6 ), align 1
+  %a7  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 7 ), align 1
+  %a8  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 8 ), align 1
+  %a9  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 9 ), align 1
+  %a10 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 10), align 1
+  %a11 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 11), align 1
+  %a12 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 12), align 1
+  %a13 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 13), align 1
+  %a14 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 14), align 1
+  %a15 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 15), align 1
+  %a16 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 16), align 1
+  %a17 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 17), align 1
+  %a18 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 18), align 1
+  %a19 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 19), align 1
+  %a20 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 20), align 1
+  %a21 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 21), align 1
+  %a22 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 22), align 1
+  %a23 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 23), align 1
+  %a24 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 24), align 1
+  %a25 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 25), align 1
+  %a26 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 26), align 1
+  %a27 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 27), align 1
+  %a28 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 28), align 1
+  %a29 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 29), align 1
+  %a30 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 30), align 1
+  %a31 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 31), align 1
+  %a32 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 32), align 1
+  %a33 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 33), align 1
+  %a34 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 34), align 1
+  %a35 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 35), align 1
+  %a36 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 36), align 1
+  %a37 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 37), align 1
+  %a38 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 38), align 1
+  %a39 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 39), align 1
+  %a40 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 40), align 1
+  %a41 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 41), align 1
+  %a42 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 42), align 1
+  %a43 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 43), align 1
+  %a44 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 44), align 1
+  %a45 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 45), align 1
+  %a46 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 46), align 1
+  %a47 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 47), align 1
+  %a48 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 48), align 1
+  %a49 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 49), align 1
+  %a50 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 50), align 1
+  %a51 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 51), align 1
+  %a52 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 52), align 1
+  %a53 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 53), align 1
+  %a54 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 54), align 1
+  %a55 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 55), align 1
+  %a56 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 56), align 1
+  %a57 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 57), align 1
+  %a58 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 58), align 1
+  %a59 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 59), align 1
+  %a60 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 60), align 1
+  %a61 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 61), align 1
+  %a62 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 62), align 1
+  %a63 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 63), align 1
+  %b0  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 0 ), align 1
+  %b1  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 1 ), align 1
+  %b2  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 2 ), align 1
+  %b3  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 3 ), align 1
+  %b4  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 4 ), align 1
+  %b5  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 5 ), align 1
+  %b6  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 6 ), align 1
+  %b7  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 7 ), align 1
+  %b8  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 8 ), align 1
+  %b9  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 9 ), align 1
+  %b10 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 10), align 1
+  %b11 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 11), align 1
+  %b12 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 12), align 1
+  %b13 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 13), align 1
+  %b14 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 14), align 1
+  %b15 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 15), align 1
+  %b16 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 16), align 1
+  %b17 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 17), align 1
+  %b18 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 18), align 1
+  %b19 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 19), align 1
+  %b20 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 20), align 1
+  %b21 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 21), align 1
+  %b22 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 22), align 1
+  %b23 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 23), align 1
+  %b24 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 24), align 1
+  %b25 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 25), align 1
+  %b26 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 26), align 1
+  %b27 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 27), align 1
+  %b28 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 28), align 1
+  %b29 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 29), align 1
+  %b30 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 30), align 1
+  %b31 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 31), align 1
+  %b32 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 32), align 1
+  %b33 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 33), align 1
+  %b34 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 34), align 1
+  %b35 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 35), align 1
+  %b36 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 36), align 1
+  %b37 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 37), align 1
+  %b38 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 38), align 1
+  %b39 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 39), align 1
+  %b40 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 40), align 1
+  %b41 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 41), align 1
+  %b42 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 42), align 1
+  %b43 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 43), align 1
+  %b44 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 44), align 1
+  %b45 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 45), align 1
+  %b46 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 46), align 1
+  %b47 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 47), align 1
+  %b48 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 48), align 1
+  %b49 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 49), align 1
+  %b50 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 50), align 1
+  %b51 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 51), align 1
+  %b52 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 52), align 1
+  %b53 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 53), align 1
+  %b54 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 54), align 1
+  %b55 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 55), align 1
+  %b56 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 56), align 1
+  %b57 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 57), align 1
+  %b58 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 58), align 1
+  %b59 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 59), align 1
+  %b60 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 60), align 1
+  %b61 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 61), align 1
+  %b62 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 62), align 1
+  %b63 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 63), align 1
+  %r0  = call i8 @llvm.sadd.sat.i8(i8 %a0 , i8 %b0 )
+  %r1  = call i8 @llvm.sadd.sat.i8(i8 %a1 , i8 %b1 )
+  %r2  = call i8 @llvm.sadd.sat.i8(i8 %a2 , i8 %b2 )
+  %r3  = call i8 @llvm.sadd.sat.i8(i8 %a3 , i8 %b3 )
+  %r4  = call i8 @llvm.sadd.sat.i8(i8 %a4 , i8 %b4 )
+  %r5  = call i8 @llvm.sadd.sat.i8(i8 %a5 , i8 %b5 )
+  %r6  = call i8 @llvm.sadd.sat.i8(i8 %a6 , i8 %b6 )
+  %r7  = call i8 @llvm.sadd.sat.i8(i8 %a7 , i8 %b7 )
+  %r8  = call i8 @llvm.sadd.sat.i8(i8 %a8 , i8 %b8 )
+  %r9  = call i8 @llvm.sadd.sat.i8(i8 %a9 , i8 %b9 )
+  %r10 = call i8 @llvm.sadd.sat.i8(i8 %a10, i8 %b10)
+  %r11 = call i8 @llvm.sadd.sat.i8(i8 %a11, i8 %b11)
+  %r12 = call i8 @llvm.sadd.sat.i8(i8 %a12, i8 %b12)
+  %r13 = call i8 @llvm.sadd.sat.i8(i8 %a13, i8 %b13)
+  %r14 = call i8 @llvm.sadd.sat.i8(i8 %a14, i8 %b14)
+  %r15 = call i8 @llvm.sadd.sat.i8(i8 %a15, i8 %b15)
+  %r16 = call i8 @llvm.sadd.sat.i8(i8 %a16, i8 %b16)
+  %r17 = call i8 @llvm.sadd.sat.i8(i8 %a17, i8 %b17)
+  %r18 = call i8 @llvm.sadd.sat.i8(i8 %a18, i8 %b18)
+  %r19 = call i8 @llvm.sadd.sat.i8(i8 %a19, i8 %b19)
+  %r20 = call i8 @llvm.sadd.sat.i8(i8 %a20, i8 %b20)
+  %r21 = call i8 @llvm.sadd.sat.i8(i8 %a21, i8 %b21)
+  %r22 = call i8 @llvm.sadd.sat.i8(i8 %a22, i8 %b22)
+  %r23 = call i8 @llvm.sadd.sat.i8(i8 %a23, i8 %b23)
+  %r24 = call i8 @llvm.sadd.sat.i8(i8 %a24, i8 %b24)
+  %r25 = call i8 @llvm.sadd.sat.i8(i8 %a25, i8 %b25)
+  %r26 = call i8 @llvm.sadd.sat.i8(i8 %a26, i8 %b26)
+  %r27 = call i8 @llvm.sadd.sat.i8(i8 %a27, i8 %b27)
+  %r28 = call i8 @llvm.sadd.sat.i8(i8 %a28, i8 %b28)
+  %r29 = call i8 @llvm.sadd.sat.i8(i8 %a29, i8 %b29)
+  %r30 = call i8 @llvm.sadd.sat.i8(i8 %a30, i8 %b30)
+  %r31 = call i8 @llvm.sadd.sat.i8(i8 %a31, i8 %b31)
+  %r32 = call i8 @llvm.sadd.sat.i8(i8 %a32, i8 %b32)
+  %r33 = call i8 @llvm.sadd.sat.i8(i8 %a33, i8 %b33)
+  %r34 = call i8 @llvm.sadd.sat.i8(i8 %a34, i8 %b34)
+  %r35 = call i8 @llvm.sadd.sat.i8(i8 %a35, i8 %b35)
+  %r36 = call i8 @llvm.sadd.sat.i8(i8 %a36, i8 %b36)
+  %r37 = call i8 @llvm.sadd.sat.i8(i8 %a37, i8 %b37)
+  %r38 = call i8 @llvm.sadd.sat.i8(i8 %a38, i8 %b38)
+  %r39 = call i8 @llvm.sadd.sat.i8(i8 %a39, i8 %b39)
+  %r40 = call i8 @llvm.sadd.sat.i8(i8 %a40, i8 %b40)
+  %r41 = call i8 @llvm.sadd.sat.i8(i8 %a41, i8 %b41)
+  %r42 = call i8 @llvm.sadd.sat.i8(i8 %a42, i8 %b42)
+  %r43 = call i8 @llvm.sadd.sat.i8(i8 %a43, i8 %b43)
+  %r44 = call i8 @llvm.sadd.sat.i8(i8 %a44, i8 %b44)
+  %r45 = call i8 @llvm.sadd.sat.i8(i8 %a45, i8 %b45)
+  %r46 = call i8 @llvm.sadd.sat.i8(i8 %a46, i8 %b46)
+  %r47 = call i8 @llvm.sadd.sat.i8(i8 %a47, i8 %b47)
+  %r48 = call i8 @llvm.sadd.sat.i8(i8 %a48, i8 %b48)
+  %r49 = call i8 @llvm.sadd.sat.i8(i8 %a49, i8 %b49)
+  %r50 = call i8 @llvm.sadd.sat.i8(i8 %a50, i8 %b50)
+  %r51 = call i8 @llvm.sadd.sat.i8(i8 %a51, i8 %b51)
+  %r52 = call i8 @llvm.sadd.sat.i8(i8 %a52, i8 %b52)
+  %r53 = call i8 @llvm.sadd.sat.i8(i8 %a53, i8 %b53)
+  %r54 = call i8 @llvm.sadd.sat.i8(i8 %a54, i8 %b54)
+  %r55 = call i8 @llvm.sadd.sat.i8(i8 %a55, i8 %b55)
+  %r56 = call i8 @llvm.sadd.sat.i8(i8 %a56, i8 %b56)
+  %r57 = call i8 @llvm.sadd.sat.i8(i8 %a57, i8 %b57)
+  %r58 = call i8 @llvm.sadd.sat.i8(i8 %a58, i8 %b58)
+  %r59 = call i8 @llvm.sadd.sat.i8(i8 %a59, i8 %b59)
+  %r60 = call i8 @llvm.sadd.sat.i8(i8 %a60, i8 %b60)
+  %r61 = call i8 @llvm.sadd.sat.i8(i8 %a61, i8 %b61)
+  %r62 = call i8 @llvm.sadd.sat.i8(i8 %a62, i8 %b62)
+  %r63 = call i8 @llvm.sadd.sat.i8(i8 %a63, i8 %b63)
+  store i8 %r0 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 0 ), align 1
+  store i8 %r1 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 1 ), align 1
+  store i8 %r2 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 2 ), align 1
+  store i8 %r3 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 3 ), align 1
+  store i8 %r4 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 4 ), align 1
+  store i8 %r5 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 5 ), align 1
+  store i8 %r6 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 6 ), align 1
+  store i8 %r7 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 7 ), align 1
+  store i8 %r8 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 8 ), align 1
+  store i8 %r9 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 9 ), align 1
+  store i8 %r10, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 10), align 1
+  store i8 %r11, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 11), align 1
+  store i8 %r12, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 12), align 1
+  store i8 %r13, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 13), align 1
+  store i8 %r14, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 14), align 1
+  store i8 %r15, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 15), align 1
+  store i8 %r16, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 16), align 1
+  store i8 %r17, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 17), align 1
+  store i8 %r18, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 18), align 1
+  store i8 %r19, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 19), align 1
+  store i8 %r20, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 20), align 1
+  store i8 %r21, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 21), align 1
+  store i8 %r22, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 22), align 1
+  store i8 %r23, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 23), align 1
+  store i8 %r24, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 24), align 1
+  store i8 %r25, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 25), align 1
+  store i8 %r26, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 26), align 1
+  store i8 %r27, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 27), align 1
+  store i8 %r28, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 28), align 1
+  store i8 %r29, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 29), align 1
+  store i8 %r30, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 30), align 1
+  store i8 %r31, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 31), align 1
+  store i8 %r32, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 32), align 1
+  store i8 %r33, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 33), align 1
+  store i8 %r34, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 34), align 1
+  store i8 %r35, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 35), align 1
+  store i8 %r36, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 36), align 1
+  store i8 %r37, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 37), align 1
+  store i8 %r38, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 38), align 1
+  store i8 %r39, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 39), align 1
+  store i8 %r40, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 40), align 1
+  store i8 %r41, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 41), align 1
+  store i8 %r42, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 42), align 1
+  store i8 %r43, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 43), align 1
+  store i8 %r44, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 44), align 1
+  store i8 %r45, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 45), align 1
+  store i8 %r46, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 46), align 1
+  store i8 %r47, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 47), align 1
+  store i8 %r48, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 48), align 1
+  store i8 %r49, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 49), align 1
+  store i8 %r50, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 50), align 1
+  store i8 %r51, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 51), align 1
+  store i8 %r52, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 52), align 1
+  store i8 %r53, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 53), align 1
+  store i8 %r54, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 54), align 1
+  store i8 %r55, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 55), align 1
+  store i8 %r56, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 56), align 1
+  store i8 %r57, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 57), align 1
+  store i8 %r58, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 58), align 1
+  store i8 %r59, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 59), align 1
+  store i8 %r60, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 60), align 1
+  store i8 %r61, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 61), align 1
+  store i8 %r62, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 62), align 1
+  store i8 %r63, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 63), align 1
+  ret void
+}
diff --git a/test/Transforms/SLPVectorizer/X86/arith-add-usat.ll b/test/Transforms/SLPVectorizer/X86/arith-add-usat.ll
new file mode 100644
index 0000000000000000000000000000000000000000..4bd84d846e1158ddfaab72b00ca83763522f0d56
--- /dev/null
+++ b/test/Transforms/SLPVectorizer/X86/arith-add-usat.ll
@@ -0,0 +1,729 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -mtriple=x86_64-unknown -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SSE
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SLM
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX1
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512F
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -mattr=-prefer-256-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512BW
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -mattr=+prefer-256-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX256BW
+
+@a64 = common global [8 x i64] zeroinitializer, align 64
+@b64 = common global [8 x i64] zeroinitializer, align 64
+@c64 = common global [8 x i64] zeroinitializer, align 64
+@a32 = common global [16 x i32] zeroinitializer, align 64
+@b32 = common global [16 x i32] zeroinitializer, align 64
+@c32 = common global [16 x i32] zeroinitializer, align 64
+@a16 = common global [32 x i16] zeroinitializer, align 64
+@b16 = common global [32 x i16] zeroinitializer, align 64
+@c16 = common global [32 x i16] zeroinitializer, align 64
+@a8  = common global [64 x i8] zeroinitializer, align 64
+@b8  = common global [64 x i8] zeroinitializer, align 64
+@c8  = common global [64 x i8] zeroinitializer, align 64
+
+declare i64 @llvm.uadd.sat.i64(i64, i64)
+declare i32 @llvm.uadd.sat.i32(i32, i32)
+declare i16 @llvm.uadd.sat.i16(i16, i16)
+declare i8  @llvm.uadd.sat.i8 (i8 , i8 )
+
+define void @add_v8i64() {
+; SSE-LABEL: @add_v8i64(
+; SSE-NEXT:    [[A0:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 0), align 8
+; SSE-NEXT:    [[A1:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 1), align 8
+; SSE-NEXT:    [[A2:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 2), align 8
+; SSE-NEXT:    [[A3:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 3), align 8
+; SSE-NEXT:    [[A4:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 4), align 8
+; SSE-NEXT:    [[A5:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 5), align 8
+; SSE-NEXT:    [[A6:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 6), align 8
+; SSE-NEXT:    [[A7:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 7), align 8
+; SSE-NEXT:    [[B0:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 0), align 8
+; SSE-NEXT:    [[B1:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 1), align 8
+; SSE-NEXT:    [[B2:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 2), align 8
+; SSE-NEXT:    [[B3:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 3), align 8
+; SSE-NEXT:    [[B4:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 4), align 8
+; SSE-NEXT:    [[B5:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 5), align 8
+; SSE-NEXT:    [[B6:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 6), align 8
+; SSE-NEXT:    [[B7:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 7), align 8
+; SSE-NEXT:    [[R0:%.*]] = call i64 @llvm.uadd.sat.i64(i64 [[A0]], i64 [[B0]])
+; SSE-NEXT:    [[R1:%.*]] = call i64 @llvm.uadd.sat.i64(i64 [[A1]], i64 [[B1]])
+; SSE-NEXT:    [[R2:%.*]] = call i64 @llvm.uadd.sat.i64(i64 [[A2]], i64 [[B2]])
+; SSE-NEXT:    [[R3:%.*]] = call i64 @llvm.uadd.sat.i64(i64 [[A3]], i64 [[B3]])
+; SSE-NEXT:    [[R4:%.*]] = call i64 @llvm.uadd.sat.i64(i64 [[A4]], i64 [[B4]])
+; SSE-NEXT:    [[R5:%.*]] = call i64 @llvm.uadd.sat.i64(i64 [[A5]], i64 [[B5]])
+; SSE-NEXT:    [[R6:%.*]] = call i64 @llvm.uadd.sat.i64(i64 [[A6]], i64 [[B6]])
+; SSE-NEXT:    [[R7:%.*]] = call i64 @llvm.uadd.sat.i64(i64 [[A7]], i64 [[B7]])
+; SSE-NEXT:    store i64 [[R0]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 0), align 8
+; SSE-NEXT:    store i64 [[R1]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 1), align 8
+; SSE-NEXT:    store i64 [[R2]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 2), align 8
+; SSE-NEXT:    store i64 [[R3]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 3), align 8
+; SSE-NEXT:    store i64 [[R4]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 4), align 8
+; SSE-NEXT:    store i64 [[R5]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 5), align 8
+; SSE-NEXT:    store i64 [[R6]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 6), align 8
+; SSE-NEXT:    store i64 [[R7]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 7), align 8
+; SSE-NEXT:    ret void
+;
+; SLM-LABEL: @add_v8i64(
+; SLM-NEXT:    [[TMP1:%.*]] = load <2 x i64>, <2 x i64>* bitcast ([8 x i64]* @a64 to <2 x i64>*), align 8
+; SLM-NEXT:    [[TMP2:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 2) to <2 x i64>*), align 8
+; SLM-NEXT:    [[TMP3:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 4) to <2 x i64>*), align 8
+; SLM-NEXT:    [[TMP4:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 6) to <2 x i64>*), align 8
+; SLM-NEXT:    [[TMP5:%.*]] = load <2 x i64>, <2 x i64>* bitcast ([8 x i64]* @b64 to <2 x i64>*), align 8
+; SLM-NEXT:    [[TMP6:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 2) to <2 x i64>*), align 8
+; SLM-NEXT:    [[TMP7:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 4) to <2 x i64>*), align 8
+; SLM-NEXT:    [[TMP8:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 6) to <2 x i64>*), align 8
+; SLM-NEXT:    [[TMP9:%.*]] = call <2 x i64> @llvm.uadd.sat.v2i64(<2 x i64> [[TMP1]], <2 x i64> [[TMP5]])
+; SLM-NEXT:    [[TMP10:%.*]] = call <2 x i64> @llvm.uadd.sat.v2i64(<2 x i64> [[TMP2]], <2 x i64> [[TMP6]])
+; SLM-NEXT:    [[TMP11:%.*]] = call <2 x i64> @llvm.uadd.sat.v2i64(<2 x i64> [[TMP3]], <2 x i64> [[TMP7]])
+; SLM-NEXT:    [[TMP12:%.*]] = call <2 x i64> @llvm.uadd.sat.v2i64(<2 x i64> [[TMP4]], <2 x i64> [[TMP8]])
+; SLM-NEXT:    store <2 x i64> [[TMP9]], <2 x i64>* bitcast ([8 x i64]* @c64 to <2 x i64>*), align 8
+; SLM-NEXT:    store <2 x i64> [[TMP10]], <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 2) to <2 x i64>*), align 8
+; SLM-NEXT:    store <2 x i64> [[TMP11]], <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 4) to <2 x i64>*), align 8
+; SLM-NEXT:    store <2 x i64> [[TMP12]], <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 6) to <2 x i64>*), align 8
+; SLM-NEXT:    ret void
+;
+; AVX-LABEL: @add_v8i64(
+; AVX-NEXT:    [[TMP1:%.*]] = load <4 x i64>, <4 x i64>* bitcast ([8 x i64]* @a64 to <4 x i64>*), align 8
+; AVX-NEXT:    [[TMP2:%.*]] = load <4 x i64>, <4 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 4) to <4 x i64>*), align 8
+; AVX-NEXT:    [[TMP3:%.*]] = load <4 x i64>, <4 x i64>* bitcast ([8 x i64]* @b64 to <4 x i64>*), align 8
+; AVX-NEXT:    [[TMP4:%.*]] = load <4 x i64>, <4 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 4) to <4 x i64>*), align 8
+; AVX-NEXT:    [[TMP5:%.*]] = call <4 x i64> @llvm.uadd.sat.v4i64(<4 x i64> [[TMP1]], <4 x i64> [[TMP3]])
+; AVX-NEXT:    [[TMP6:%.*]] = call <4 x i64> @llvm.uadd.sat.v4i64(<4 x i64> [[TMP2]], <4 x i64> [[TMP4]])
+; AVX-NEXT:    store <4 x i64> [[TMP5]], <4 x i64>* bitcast ([8 x i64]* @c64 to <4 x i64>*), align 8
+; AVX-NEXT:    store <4 x i64> [[TMP6]], <4 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 4) to <4 x i64>*), align 8
+; AVX-NEXT:    ret void
+;
+; AVX512-LABEL: @add_v8i64(
+; AVX512-NEXT:    [[TMP1:%.*]] = load <8 x i64>, <8 x i64>* bitcast ([8 x i64]* @a64 to <8 x i64>*), align 8
+; AVX512-NEXT:    [[TMP2:%.*]] = load <8 x i64>, <8 x i64>* bitcast ([8 x i64]* @b64 to <8 x i64>*), align 8
+; AVX512-NEXT:    [[TMP3:%.*]] = call <8 x i64> @llvm.uadd.sat.v8i64(<8 x i64> [[TMP1]], <8 x i64> [[TMP2]])
+; AVX512-NEXT:    store <8 x i64> [[TMP3]], <8 x i64>* bitcast ([8 x i64]* @c64 to <8 x i64>*), align 8
+; AVX512-NEXT:    ret void
+;
+  %a0 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 0), align 8
+  %a1 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 1), align 8
+  %a2 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 2), align 8
+  %a3 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 3), align 8
+  %a4 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 4), align 8
+  %a5 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 5), align 8
+  %a6 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 6), align 8
+  %a7 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 7), align 8
+  %b0 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 0), align 8
+  %b1 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 1), align 8
+  %b2 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 2), align 8
+  %b3 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 3), align 8
+  %b4 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 4), align 8
+  %b5 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 5), align 8
+  %b6 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 6), align 8
+  %b7 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 7), align 8
+  %r0 = call i64 @llvm.uadd.sat.i64(i64 %a0, i64 %b0)
+  %r1 = call i64 @llvm.uadd.sat.i64(i64 %a1, i64 %b1)
+  %r2 = call i64 @llvm.uadd.sat.i64(i64 %a2, i64 %b2)
+  %r3 = call i64 @llvm.uadd.sat.i64(i64 %a3, i64 %b3)
+  %r4 = call i64 @llvm.uadd.sat.i64(i64 %a4, i64 %b4)
+  %r5 = call i64 @llvm.uadd.sat.i64(i64 %a5, i64 %b5)
+  %r6 = call i64 @llvm.uadd.sat.i64(i64 %a6, i64 %b6)
+  %r7 = call i64 @llvm.uadd.sat.i64(i64 %a7, i64 %b7)
+  store i64 %r0, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 0), align 8
+  store i64 %r1, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 1), align 8
+  store i64 %r2, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 2), align 8
+  store i64 %r3, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 3), align 8
+  store i64 %r4, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 4), align 8
+  store i64 %r5, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 5), align 8
+  store i64 %r6, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 6), align 8
+  store i64 %r7, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 7), align 8
+  ret void
+}
+
+define void @add_v16i32() {
+; SSE-LABEL: @add_v16i32(
+; SSE-NEXT:    [[TMP1:%.*]] = load <4 x i32>, <4 x i32>* bitcast ([16 x i32]* @a32 to <4 x i32>*), align 4
+; SSE-NEXT:    [[TMP2:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 4) to <4 x i32>*), align 4
+; SSE-NEXT:    [[TMP3:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 8) to <4 x i32>*), align 4
+; SSE-NEXT:    [[TMP4:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 12) to <4 x i32>*), align 4
+; SSE-NEXT:    [[TMP5:%.*]] = load <4 x i32>, <4 x i32>* bitcast ([16 x i32]* @b32 to <4 x i32>*), align 4
+; SSE-NEXT:    [[TMP6:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 4) to <4 x i32>*), align 4
+; SSE-NEXT:    [[TMP7:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 8) to <4 x i32>*), align 4
+; SSE-NEXT:    [[TMP8:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 12) to <4 x i32>*), align 4
+; SSE-NEXT:    [[TMP9:%.*]] = call <4 x i32> @llvm.uadd.sat.v4i32(<4 x i32> [[TMP1]], <4 x i32> [[TMP5]])
+; SSE-NEXT:    [[TMP10:%.*]] = call <4 x i32> @llvm.uadd.sat.v4i32(<4 x i32> [[TMP2]], <4 x i32> [[TMP6]])
+; SSE-NEXT:    [[TMP11:%.*]] = call <4 x i32> @llvm.uadd.sat.v4i32(<4 x i32> [[TMP3]], <4 x i32> [[TMP7]])
+; SSE-NEXT:    [[TMP12:%.*]] = call <4 x i32> @llvm.uadd.sat.v4i32(<4 x i32> [[TMP4]], <4 x i32> [[TMP8]])
+; SSE-NEXT:    store <4 x i32> [[TMP9]], <4 x i32>* bitcast ([16 x i32]* @c32 to <4 x i32>*), align 4
+; SSE-NEXT:    store <4 x i32> [[TMP10]], <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 4) to <4 x i32>*), align 4
+; SSE-NEXT:    store <4 x i32> [[TMP11]], <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 8) to <4 x i32>*), align 4
+; SSE-NEXT:    store <4 x i32> [[TMP12]], <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 12) to <4 x i32>*), align 4
+; SSE-NEXT:    ret void
+;
+; SLM-LABEL: @add_v16i32(
+; SLM-NEXT:    [[TMP1:%.*]] = load <4 x i32>, <4 x i32>* bitcast ([16 x i32]* @a32 to <4 x i32>*), align 4
+; SLM-NEXT:    [[TMP2:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 4) to <4 x i32>*), align 4
+; SLM-NEXT:    [[TMP3:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 8) to <4 x i32>*), align 4
+; SLM-NEXT:    [[TMP4:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 12) to <4 x i32>*), align 4
+; SLM-NEXT:    [[TMP5:%.*]] = load <4 x i32>, <4 x i32>* bitcast ([16 x i32]* @b32 to <4 x i32>*), align 4
+; SLM-NEXT:    [[TMP6:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 4) to <4 x i32>*), align 4
+; SLM-NEXT:    [[TMP7:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 8) to <4 x i32>*), align 4
+; SLM-NEXT:    [[TMP8:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 12) to <4 x i32>*), align 4
+; SLM-NEXT:    [[TMP9:%.*]] = call <4 x i32> @llvm.uadd.sat.v4i32(<4 x i32> [[TMP1]], <4 x i32> [[TMP5]])
+; SLM-NEXT:    [[TMP10:%.*]] = call <4 x i32> @llvm.uadd.sat.v4i32(<4 x i32> [[TMP2]], <4 x i32> [[TMP6]])
+; SLM-NEXT:    [[TMP11:%.*]] = call <4 x i32> @llvm.uadd.sat.v4i32(<4 x i32> [[TMP3]], <4 x i32> [[TMP7]])
+; SLM-NEXT:    [[TMP12:%.*]] = call <4 x i32> @llvm.uadd.sat.v4i32(<4 x i32> [[TMP4]], <4 x i32> [[TMP8]])
+; SLM-NEXT:    store <4 x i32> [[TMP9]], <4 x i32>* bitcast ([16 x i32]* @c32 to <4 x i32>*), align 4
+; SLM-NEXT:    store <4 x i32> [[TMP10]], <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 4) to <4 x i32>*), align 4
+; SLM-NEXT:    store <4 x i32> [[TMP11]], <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 8) to <4 x i32>*), align 4
+; SLM-NEXT:    store <4 x i32> [[TMP12]], <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 12) to <4 x i32>*), align 4
+; SLM-NEXT:    ret void
+;
+; AVX-LABEL: @add_v16i32(
+; AVX-NEXT:    [[TMP1:%.*]] = load <8 x i32>, <8 x i32>* bitcast ([16 x i32]* @a32 to <8 x i32>*), align 4
+; AVX-NEXT:    [[TMP2:%.*]] = load <8 x i32>, <8 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 8) to <8 x i32>*), align 4
+; AVX-NEXT:    [[TMP3:%.*]] = load <8 x i32>, <8 x i32>* bitcast ([16 x i32]* @b32 to <8 x i32>*), align 4
+; AVX-NEXT:    [[TMP4:%.*]] = load <8 x i32>, <8 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 8) to <8 x i32>*), align 4
+; AVX-NEXT:    [[TMP5:%.*]] = call <8 x i32> @llvm.uadd.sat.v8i32(<8 x i32> [[TMP1]], <8 x i32> [[TMP3]])
+; AVX-NEXT:    [[TMP6:%.*]] = call <8 x i32> @llvm.uadd.sat.v8i32(<8 x i32> [[TMP2]], <8 x i32> [[TMP4]])
+; AVX-NEXT:    store <8 x i32> [[TMP5]], <8 x i32>* bitcast ([16 x i32]* @c32 to <8 x i32>*), align 4
+; AVX-NEXT:    store <8 x i32> [[TMP6]], <8 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 8) to <8 x i32>*), align 4
+; AVX-NEXT:    ret void
+;
+; AVX512-LABEL: @add_v16i32(
+; AVX512-NEXT:    [[TMP1:%.*]] = load <16 x i32>, <16 x i32>* bitcast ([16 x i32]* @a32 to <16 x i32>*), align 4
+; AVX512-NEXT:    [[TMP2:%.*]] = load <16 x i32>, <16 x i32>* bitcast ([16 x i32]* @b32 to <16 x i32>*), align 4
+; AVX512-NEXT:    [[TMP3:%.*]] = call <16 x i32> @llvm.uadd.sat.v16i32(<16 x i32> [[TMP1]], <16 x i32> [[TMP2]])
+; AVX512-NEXT:    store <16 x i32> [[TMP3]], <16 x i32>* bitcast ([16 x i32]* @c32 to <16 x i32>*), align 4
+; AVX512-NEXT:    ret void
+;
+  %a0  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 0 ), align 4
+  %a1  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 1 ), align 4
+  %a2  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 2 ), align 4
+  %a3  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 3 ), align 4
+  %a4  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 4 ), align 4
+  %a5  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 5 ), align 4
+  %a6  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 6 ), align 4
+  %a7  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 7 ), align 4
+  %a8  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 8 ), align 4
+  %a9  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 9 ), align 4
+  %a10 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 10), align 4
+  %a11 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 11), align 4
+  %a12 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 12), align 4
+  %a13 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 13), align 4
+  %a14 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 14), align 4
+  %a15 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 15), align 4
+  %b0  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 0 ), align 4
+  %b1  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 1 ), align 4
+  %b2  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 2 ), align 4
+  %b3  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 3 ), align 4
+  %b4  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 4 ), align 4
+  %b5  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 5 ), align 4
+  %b6  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 6 ), align 4
+  %b7  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 7 ), align 4
+  %b8  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 8 ), align 4
+  %b9  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 9 ), align 4
+  %b10 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 10), align 4
+  %b11 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 11), align 4
+  %b12 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 12), align 4
+  %b13 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 13), align 4
+  %b14 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 14), align 4
+  %b15 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 15), align 4
+  %r0  = call i32 @llvm.uadd.sat.i32(i32 %a0 , i32 %b0 )
+  %r1  = call i32 @llvm.uadd.sat.i32(i32 %a1 , i32 %b1 )
+  %r2  = call i32 @llvm.uadd.sat.i32(i32 %a2 , i32 %b2 )
+  %r3  = call i32 @llvm.uadd.sat.i32(i32 %a3 , i32 %b3 )
+  %r4  = call i32 @llvm.uadd.sat.i32(i32 %a4 , i32 %b4 )
+  %r5  = call i32 @llvm.uadd.sat.i32(i32 %a5 , i32 %b5 )
+  %r6  = call i32 @llvm.uadd.sat.i32(i32 %a6 , i32 %b6 )
+  %r7  = call i32 @llvm.uadd.sat.i32(i32 %a7 , i32 %b7 )
+  %r8  = call i32 @llvm.uadd.sat.i32(i32 %a8 , i32 %b8 )
+  %r9  = call i32 @llvm.uadd.sat.i32(i32 %a9 , i32 %b9 )
+  %r10 = call i32 @llvm.uadd.sat.i32(i32 %a10, i32 %b10)
+  %r11 = call i32 @llvm.uadd.sat.i32(i32 %a11, i32 %b11)
+  %r12 = call i32 @llvm.uadd.sat.i32(i32 %a12, i32 %b12)
+  %r13 = call i32 @llvm.uadd.sat.i32(i32 %a13, i32 %b13)
+  %r14 = call i32 @llvm.uadd.sat.i32(i32 %a14, i32 %b14)
+  %r15 = call i32 @llvm.uadd.sat.i32(i32 %a15, i32 %b15)
+  store i32 %r0 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 0 ), align 4
+  store i32 %r1 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 1 ), align 4
+  store i32 %r2 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 2 ), align 4
+  store i32 %r3 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 3 ), align 4
+  store i32 %r4 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 4 ), align 4
+  store i32 %r5 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 5 ), align 4
+  store i32 %r6 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 6 ), align 4
+  store i32 %r7 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 7 ), align 4
+  store i32 %r8 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 8 ), align 4
+  store i32 %r9 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 9 ), align 4
+  store i32 %r10, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 10), align 4
+  store i32 %r11, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 11), align 4
+  store i32 %r12, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 12), align 4
+  store i32 %r13, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 13), align 4
+  store i32 %r14, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 14), align 4
+  store i32 %r15, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 15), align 4
+  ret void
+}
+
+define void @add_v32i16() {
+; SSE-LABEL: @add_v32i16(
+; SSE-NEXT:    [[TMP1:%.*]] = load <8 x i16>, <8 x i16>* bitcast ([32 x i16]* @a16 to <8 x i16>*), align 2
+; SSE-NEXT:    [[TMP2:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 8) to <8 x i16>*), align 2
+; SSE-NEXT:    [[TMP3:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 16) to <8 x i16>*), align 2
+; SSE-NEXT:    [[TMP4:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 24) to <8 x i16>*), align 2
+; SSE-NEXT:    [[TMP5:%.*]] = load <8 x i16>, <8 x i16>* bitcast ([32 x i16]* @b16 to <8 x i16>*), align 2
+; SSE-NEXT:    [[TMP6:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 8) to <8 x i16>*), align 2
+; SSE-NEXT:    [[TMP7:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 16) to <8 x i16>*), align 2
+; SSE-NEXT:    [[TMP8:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 24) to <8 x i16>*), align 2
+; SSE-NEXT:    [[TMP9:%.*]] = call <8 x i16> @llvm.uadd.sat.v8i16(<8 x i16> [[TMP1]], <8 x i16> [[TMP5]])
+; SSE-NEXT:    [[TMP10:%.*]] = call <8 x i16> @llvm.uadd.sat.v8i16(<8 x i16> [[TMP2]], <8 x i16> [[TMP6]])
+; SSE-NEXT:    [[TMP11:%.*]] = call <8 x i16> @llvm.uadd.sat.v8i16(<8 x i16> [[TMP3]], <8 x i16> [[TMP7]])
+; SSE-NEXT:    [[TMP12:%.*]] = call <8 x i16> @llvm.uadd.sat.v8i16(<8 x i16> [[TMP4]], <8 x i16> [[TMP8]])
+; SSE-NEXT:    store <8 x i16> [[TMP9]], <8 x i16>* bitcast ([32 x i16]* @c16 to <8 x i16>*), align 2
+; SSE-NEXT:    store <8 x i16> [[TMP10]], <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 8) to <8 x i16>*), align 2
+; SSE-NEXT:    store <8 x i16> [[TMP11]], <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 16) to <8 x i16>*), align 2
+; SSE-NEXT:    store <8 x i16> [[TMP12]], <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 24) to <8 x i16>*), align 2
+; SSE-NEXT:    ret void
+;
+; SLM-LABEL: @add_v32i16(
+; SLM-NEXT:    [[TMP1:%.*]] = load <8 x i16>, <8 x i16>* bitcast ([32 x i16]* @a16 to <8 x i16>*), align 2
+; SLM-NEXT:    [[TMP2:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 8) to <8 x i16>*), align 2
+; SLM-NEXT:    [[TMP3:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 16) to <8 x i16>*), align 2
+; SLM-NEXT:    [[TMP4:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 24) to <8 x i16>*), align 2
+; SLM-NEXT:    [[TMP5:%.*]] = load <8 x i16>, <8 x i16>* bitcast ([32 x i16]* @b16 to <8 x i16>*), align 2
+; SLM-NEXT:    [[TMP6:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 8) to <8 x i16>*), align 2
+; SLM-NEXT:    [[TMP7:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 16) to <8 x i16>*), align 2
+; SLM-NEXT:    [[TMP8:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 24) to <8 x i16>*), align 2
+; SLM-NEXT:    [[TMP9:%.*]] = call <8 x i16> @llvm.uadd.sat.v8i16(<8 x i16> [[TMP1]], <8 x i16> [[TMP5]])
+; SLM-NEXT:    [[TMP10:%.*]] = call <8 x i16> @llvm.uadd.sat.v8i16(<8 x i16> [[TMP2]], <8 x i16> [[TMP6]])
+; SLM-NEXT:    [[TMP11:%.*]] = call <8 x i16> @llvm.uadd.sat.v8i16(<8 x i16> [[TMP3]], <8 x i16> [[TMP7]])
+; SLM-NEXT:    [[TMP12:%.*]] = call <8 x i16> @llvm.uadd.sat.v8i16(<8 x i16> [[TMP4]], <8 x i16> [[TMP8]])
+; SLM-NEXT:    store <8 x i16> [[TMP9]], <8 x i16>* bitcast ([32 x i16]* @c16 to <8 x i16>*), align 2
+; SLM-NEXT:    store <8 x i16> [[TMP10]], <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 8) to <8 x i16>*), align 2
+; SLM-NEXT:    store <8 x i16> [[TMP11]], <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 16) to <8 x i16>*), align 2
+; SLM-NEXT:    store <8 x i16> [[TMP12]], <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 24) to <8 x i16>*), align 2
+; SLM-NEXT:    ret void
+;
+; AVX-LABEL: @add_v32i16(
+; AVX-NEXT:    [[TMP1:%.*]] = load <16 x i16>, <16 x i16>* bitcast ([32 x i16]* @a16 to <16 x i16>*), align 2
+; AVX-NEXT:    [[TMP2:%.*]] = load <16 x i16>, <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 16) to <16 x i16>*), align 2
+; AVX-NEXT:    [[TMP3:%.*]] = load <16 x i16>, <16 x i16>* bitcast ([32 x i16]* @b16 to <16 x i16>*), align 2
+; AVX-NEXT:    [[TMP4:%.*]] = load <16 x i16>, <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 16) to <16 x i16>*), align 2
+; AVX-NEXT:    [[TMP5:%.*]] = call <16 x i16> @llvm.uadd.sat.v16i16(<16 x i16> [[TMP1]], <16 x i16> [[TMP3]])
+; AVX-NEXT:    [[TMP6:%.*]] = call <16 x i16> @llvm.uadd.sat.v16i16(<16 x i16> [[TMP2]], <16 x i16> [[TMP4]])
+; AVX-NEXT:    store <16 x i16> [[TMP5]], <16 x i16>* bitcast ([32 x i16]* @c16 to <16 x i16>*), align 2
+; AVX-NEXT:    store <16 x i16> [[TMP6]], <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 16) to <16 x i16>*), align 2
+; AVX-NEXT:    ret void
+;
+; AVX512-LABEL: @add_v32i16(
+; AVX512-NEXT:    [[TMP1:%.*]] = load <16 x i16>, <16 x i16>* bitcast ([32 x i16]* @a16 to <16 x i16>*), align 2
+; AVX512-NEXT:    [[TMP2:%.*]] = load <16 x i16>, <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 16) to <16 x i16>*), align 2
+; AVX512-NEXT:    [[TMP3:%.*]] = load <16 x i16>, <16 x i16>* bitcast ([32 x i16]* @b16 to <16 x i16>*), align 2
+; AVX512-NEXT:    [[TMP4:%.*]] = load <16 x i16>, <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 16) to <16 x i16>*), align 2
+; AVX512-NEXT:    [[TMP5:%.*]] = call <16 x i16> @llvm.uadd.sat.v16i16(<16 x i16> [[TMP1]], <16 x i16> [[TMP3]])
+; AVX512-NEXT:    [[TMP6:%.*]] = call <16 x i16> @llvm.uadd.sat.v16i16(<16 x i16> [[TMP2]], <16 x i16> [[TMP4]])
+; AVX512-NEXT:    store <16 x i16> [[TMP5]], <16 x i16>* bitcast ([32 x i16]* @c16 to <16 x i16>*), align 2
+; AVX512-NEXT:    store <16 x i16> [[TMP6]], <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 16) to <16 x i16>*), align 2
+; AVX512-NEXT:    ret void
+;
+  %a0  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 0 ), align 2
+  %a1  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 1 ), align 2
+  %a2  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 2 ), align 2
+  %a3  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 3 ), align 2
+  %a4  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 4 ), align 2
+  %a5  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 5 ), align 2
+  %a6  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 6 ), align 2
+  %a7  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 7 ), align 2
+  %a8  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 8 ), align 2
+  %a9  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 9 ), align 2
+  %a10 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 10), align 2
+  %a11 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 11), align 2
+  %a12 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 12), align 2
+  %a13 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 13), align 2
+  %a14 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 14), align 2
+  %a15 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 15), align 2
+  %a16 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 16), align 2
+  %a17 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 17), align 2
+  %a18 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 18), align 2
+  %a19 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 19), align 2
+  %a20 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 20), align 2
+  %a21 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 21), align 2
+  %a22 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 22), align 2
+  %a23 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 23), align 2
+  %a24 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 24), align 2
+  %a25 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 25), align 2
+  %a26 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 26), align 2
+  %a27 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 27), align 2
+  %a28 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 28), align 2
+  %a29 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 29), align 2
+  %a30 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 30), align 2
+  %a31 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 31), align 2
+  %b0  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 0 ), align 2
+  %b1  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 1 ), align 2
+  %b2  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 2 ), align 2
+  %b3  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 3 ), align 2
+  %b4  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 4 ), align 2
+  %b5  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 5 ), align 2
+  %b6  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 6 ), align 2
+  %b7  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 7 ), align 2
+  %b8  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 8 ), align 2
+  %b9  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 9 ), align 2
+  %b10 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 10), align 2
+  %b11 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 11), align 2
+  %b12 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 12), align 2
+  %b13 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 13), align 2
+  %b14 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 14), align 2
+  %b15 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 15), align 2
+  %b16 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 16), align 2
+  %b17 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 17), align 2
+  %b18 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 18), align 2
+  %b19 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 19), align 2
+  %b20 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 20), align 2
+  %b21 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 21), align 2
+  %b22 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 22), align 2
+  %b23 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 23), align 2
+  %b24 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 24), align 2
+  %b25 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 25), align 2
+  %b26 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 26), align 2
+  %b27 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 27), align 2
+  %b28 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 28), align 2
+  %b29 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 29), align 2
+  %b30 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 30), align 2
+  %b31 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 31), align 2
+  %r0  = call i16 @llvm.uadd.sat.i16(i16 %a0 , i16 %b0 )
+  %r1  = call i16 @llvm.uadd.sat.i16(i16 %a1 , i16 %b1 )
+  %r2  = call i16 @llvm.uadd.sat.i16(i16 %a2 , i16 %b2 )
+  %r3  = call i16 @llvm.uadd.sat.i16(i16 %a3 , i16 %b3 )
+  %r4  = call i16 @llvm.uadd.sat.i16(i16 %a4 , i16 %b4 )
+  %r5  = call i16 @llvm.uadd.sat.i16(i16 %a5 , i16 %b5 )
+  %r6  = call i16 @llvm.uadd.sat.i16(i16 %a6 , i16 %b6 )
+  %r7  = call i16 @llvm.uadd.sat.i16(i16 %a7 , i16 %b7 )
+  %r8  = call i16 @llvm.uadd.sat.i16(i16 %a8 , i16 %b8 )
+  %r9  = call i16 @llvm.uadd.sat.i16(i16 %a9 , i16 %b9 )
+  %r10 = call i16 @llvm.uadd.sat.i16(i16 %a10, i16 %b10)
+  %r11 = call i16 @llvm.uadd.sat.i16(i16 %a11, i16 %b11)
+  %r12 = call i16 @llvm.uadd.sat.i16(i16 %a12, i16 %b12)
+  %r13 = call i16 @llvm.uadd.sat.i16(i16 %a13, i16 %b13)
+  %r14 = call i16 @llvm.uadd.sat.i16(i16 %a14, i16 %b14)
+  %r15 = call i16 @llvm.uadd.sat.i16(i16 %a15, i16 %b15)
+  %r16 = call i16 @llvm.uadd.sat.i16(i16 %a16, i16 %b16)
+  %r17 = call i16 @llvm.uadd.sat.i16(i16 %a17, i16 %b17)
+  %r18 = call i16 @llvm.uadd.sat.i16(i16 %a18, i16 %b18)
+  %r19 = call i16 @llvm.uadd.sat.i16(i16 %a19, i16 %b19)
+  %r20 = call i16 @llvm.uadd.sat.i16(i16 %a20, i16 %b20)
+  %r21 = call i16 @llvm.uadd.sat.i16(i16 %a21, i16 %b21)
+  %r22 = call i16 @llvm.uadd.sat.i16(i16 %a22, i16 %b22)
+  %r23 = call i16 @llvm.uadd.sat.i16(i16 %a23, i16 %b23)
+  %r24 = call i16 @llvm.uadd.sat.i16(i16 %a24, i16 %b24)
+  %r25 = call i16 @llvm.uadd.sat.i16(i16 %a25, i16 %b25)
+  %r26 = call i16 @llvm.uadd.sat.i16(i16 %a26, i16 %b26)
+  %r27 = call i16 @llvm.uadd.sat.i16(i16 %a27, i16 %b27)
+  %r28 = call i16 @llvm.uadd.sat.i16(i16 %a28, i16 %b28)
+  %r29 = call i16 @llvm.uadd.sat.i16(i16 %a29, i16 %b29)
+  %r30 = call i16 @llvm.uadd.sat.i16(i16 %a30, i16 %b30)
+  %r31 = call i16 @llvm.uadd.sat.i16(i16 %a31, i16 %b31)
+  store i16 %r0 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 0 ), align 2
+  store i16 %r1 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 1 ), align 2
+  store i16 %r2 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 2 ), align 2
+  store i16 %r3 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 3 ), align 2
+  store i16 %r4 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 4 ), align 2
+  store i16 %r5 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 5 ), align 2
+  store i16 %r6 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 6 ), align 2
+  store i16 %r7 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 7 ), align 2
+  store i16 %r8 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 8 ), align 2
+  store i16 %r9 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 9 ), align 2
+  store i16 %r10, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 10), align 2
+  store i16 %r11, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 11), align 2
+  store i16 %r12, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 12), align 2
+  store i16 %r13, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 13), align 2
+  store i16 %r14, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 14), align 2
+  store i16 %r15, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 15), align 2
+  store i16 %r16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 16), align 2
+  store i16 %r17, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 17), align 2
+  store i16 %r18, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 18), align 2
+  store i16 %r19, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 19), align 2
+  store i16 %r20, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 20), align 2
+  store i16 %r21, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 21), align 2
+  store i16 %r22, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 22), align 2
+  store i16 %r23, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 23), align 2
+  store i16 %r24, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 24), align 2
+  store i16 %r25, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 25), align 2
+  store i16 %r26, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 26), align 2
+  store i16 %r27, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 27), align 2
+  store i16 %r28, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 28), align 2
+  store i16 %r29, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 29), align 2
+  store i16 %r30, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 30), align 2
+  store i16 %r31, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 31), align 2
+  ret void
+}
+
+define void @add_v64i8() {
+; CHECK-LABEL: @add_v64i8(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i8>, <16 x i8>* bitcast ([64 x i8]* @a8 to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 16) to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 32) to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 48) to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP5:%.*]] = load <16 x i8>, <16 x i8>* bitcast ([64 x i8]* @b8 to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP6:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 16) to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP7:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 32) to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP8:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 48) to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP9:%.*]] = call <16 x i8> @llvm.uadd.sat.v16i8(<16 x i8> [[TMP1]], <16 x i8> [[TMP5]])
+; CHECK-NEXT:    [[TMP10:%.*]] = call <16 x i8> @llvm.uadd.sat.v16i8(<16 x i8> [[TMP2]], <16 x i8> [[TMP6]])
+; CHECK-NEXT:    [[TMP11:%.*]] = call <16 x i8> @llvm.uadd.sat.v16i8(<16 x i8> [[TMP3]], <16 x i8> [[TMP7]])
+; CHECK-NEXT:    [[TMP12:%.*]] = call <16 x i8> @llvm.uadd.sat.v16i8(<16 x i8> [[TMP4]], <16 x i8> [[TMP8]])
+; CHECK-NEXT:    store <16 x i8> [[TMP9]], <16 x i8>* bitcast ([64 x i8]* @c8 to <16 x i8>*), align 1
+; CHECK-NEXT:    store <16 x i8> [[TMP10]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 16) to <16 x i8>*), align 1
+; CHECK-NEXT:    store <16 x i8> [[TMP11]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 32) to <16 x i8>*), align 1
+; CHECK-NEXT:    store <16 x i8> [[TMP12]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 48) to <16 x i8>*), align 1
+; CHECK-NEXT:    ret void
+;
+  %a0  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 0 ), align 1
+  %a1  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 1 ), align 1
+  %a2  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 2 ), align 1
+  %a3  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 3 ), align 1
+  %a4  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 4 ), align 1
+  %a5  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 5 ), align 1
+  %a6  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 6 ), align 1
+  %a7  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 7 ), align 1
+  %a8  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 8 ), align 1
+  %a9  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 9 ), align 1
+  %a10 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 10), align 1
+  %a11 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 11), align 1
+  %a12 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 12), align 1
+  %a13 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 13), align 1
+  %a14 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 14), align 1
+  %a15 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 15), align 1
+  %a16 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 16), align 1
+  %a17 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 17), align 1
+  %a18 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 18), align 1
+  %a19 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 19), align 1
+  %a20 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 20), align 1
+  %a21 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 21), align 1
+  %a22 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 22), align 1
+  %a23 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 23), align 1
+  %a24 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 24), align 1
+  %a25 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 25), align 1
+  %a26 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 26), align 1
+  %a27 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 27), align 1
+  %a28 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 28), align 1
+  %a29 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 29), align 1
+  %a30 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 30), align 1
+  %a31 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 31), align 1
+  %a32 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 32), align 1
+  %a33 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 33), align 1
+  %a34 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 34), align 1
+  %a35 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 35), align 1
+  %a36 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 36), align 1
+  %a37 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 37), align 1
+  %a38 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 38), align 1
+  %a39 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 39), align 1
+  %a40 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 40), align 1
+  %a41 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 41), align 1
+  %a42 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 42), align 1
+  %a43 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 43), align 1
+  %a44 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 44), align 1
+  %a45 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 45), align 1
+  %a46 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 46), align 1
+  %a47 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 47), align 1
+  %a48 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 48), align 1
+  %a49 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 49), align 1
+  %a50 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 50), align 1
+  %a51 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 51), align 1
+  %a52 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 52), align 1
+  %a53 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 53), align 1
+  %a54 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 54), align 1
+  %a55 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 55), align 1
+  %a56 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 56), align 1
+  %a57 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 57), align 1
+  %a58 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 58), align 1
+  %a59 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 59), align 1
+  %a60 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 60), align 1
+  %a61 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 61), align 1
+  %a62 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 62), align 1
+  %a63 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 63), align 1
+  %b0  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 0 ), align 1
+  %b1  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 1 ), align 1
+  %b2  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 2 ), align 1
+  %b3  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 3 ), align 1
+  %b4  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 4 ), align 1
+  %b5  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 5 ), align 1
+  %b6  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 6 ), align 1
+  %b7  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 7 ), align 1
+  %b8  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 8 ), align 1
+  %b9  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 9 ), align 1
+  %b10 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 10), align 1
+  %b11 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 11), align 1
+  %b12 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 12), align 1
+  %b13 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 13), align 1
+  %b14 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 14), align 1
+  %b15 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 15), align 1
+  %b16 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 16), align 1
+  %b17 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 17), align 1
+  %b18 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 18), align 1
+  %b19 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 19), align 1
+  %b20 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 20), align 1
+  %b21 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 21), align 1
+  %b22 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 22), align 1
+  %b23 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 23), align 1
+  %b24 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 24), align 1
+  %b25 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 25), align 1
+  %b26 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 26), align 1
+  %b27 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 27), align 1
+  %b28 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 28), align 1
+  %b29 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 29), align 1
+  %b30 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 30), align 1
+  %b31 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 31), align 1
+  %b32 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 32), align 1
+  %b33 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 33), align 1
+  %b34 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 34), align 1
+  %b35 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 35), align 1
+  %b36 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 36), align 1
+  %b37 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 37), align 1
+  %b38 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 38), align 1
+  %b39 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 39), align 1
+  %b40 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 40), align 1
+  %b41 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 41), align 1
+  %b42 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 42), align 1
+  %b43 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 43), align 1
+  %b44 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 44), align 1
+  %b45 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 45), align 1
+  %b46 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 46), align 1
+  %b47 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 47), align 1
+  %b48 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 48), align 1
+  %b49 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 49), align 1
+  %b50 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 50), align 1
+  %b51 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 51), align 1
+  %b52 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 52), align 1
+  %b53 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 53), align 1
+  %b54 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 54), align 1
+  %b55 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 55), align 1
+  %b56 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 56), align 1
+  %b57 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 57), align 1
+  %b58 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 58), align 1
+  %b59 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 59), align 1
+  %b60 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 60), align 1
+  %b61 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 61), align 1
+  %b62 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 62), align 1
+  %b63 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 63), align 1
+  %r0  = call i8 @llvm.uadd.sat.i8(i8 %a0 , i8 %b0 )
+  %r1  = call i8 @llvm.uadd.sat.i8(i8 %a1 , i8 %b1 )
+  %r2  = call i8 @llvm.uadd.sat.i8(i8 %a2 , i8 %b2 )
+  %r3  = call i8 @llvm.uadd.sat.i8(i8 %a3 , i8 %b3 )
+  %r4  = call i8 @llvm.uadd.sat.i8(i8 %a4 , i8 %b4 )
+  %r5  = call i8 @llvm.uadd.sat.i8(i8 %a5 , i8 %b5 )
+  %r6  = call i8 @llvm.uadd.sat.i8(i8 %a6 , i8 %b6 )
+  %r7  = call i8 @llvm.uadd.sat.i8(i8 %a7 , i8 %b7 )
+  %r8  = call i8 @llvm.uadd.sat.i8(i8 %a8 , i8 %b8 )
+  %r9  = call i8 @llvm.uadd.sat.i8(i8 %a9 , i8 %b9 )
+  %r10 = call i8 @llvm.uadd.sat.i8(i8 %a10, i8 %b10)
+  %r11 = call i8 @llvm.uadd.sat.i8(i8 %a11, i8 %b11)
+  %r12 = call i8 @llvm.uadd.sat.i8(i8 %a12, i8 %b12)
+  %r13 = call i8 @llvm.uadd.sat.i8(i8 %a13, i8 %b13)
+  %r14 = call i8 @llvm.uadd.sat.i8(i8 %a14, i8 %b14)
+  %r15 = call i8 @llvm.uadd.sat.i8(i8 %a15, i8 %b15)
+  %r16 = call i8 @llvm.uadd.sat.i8(i8 %a16, i8 %b16)
+  %r17 = call i8 @llvm.uadd.sat.i8(i8 %a17, i8 %b17)
+  %r18 = call i8 @llvm.uadd.sat.i8(i8 %a18, i8 %b18)
+  %r19 = call i8 @llvm.uadd.sat.i8(i8 %a19, i8 %b19)
+  %r20 = call i8 @llvm.uadd.sat.i8(i8 %a20, i8 %b20)
+  %r21 = call i8 @llvm.uadd.sat.i8(i8 %a21, i8 %b21)
+  %r22 = call i8 @llvm.uadd.sat.i8(i8 %a22, i8 %b22)
+  %r23 = call i8 @llvm.uadd.sat.i8(i8 %a23, i8 %b23)
+  %r24 = call i8 @llvm.uadd.sat.i8(i8 %a24, i8 %b24)
+  %r25 = call i8 @llvm.uadd.sat.i8(i8 %a25, i8 %b25)
+  %r26 = call i8 @llvm.uadd.sat.i8(i8 %a26, i8 %b26)
+  %r27 = call i8 @llvm.uadd.sat.i8(i8 %a27, i8 %b27)
+  %r28 = call i8 @llvm.uadd.sat.i8(i8 %a28, i8 %b28)
+  %r29 = call i8 @llvm.uadd.sat.i8(i8 %a29, i8 %b29)
+  %r30 = call i8 @llvm.uadd.sat.i8(i8 %a30, i8 %b30)
+  %r31 = call i8 @llvm.uadd.sat.i8(i8 %a31, i8 %b31)
+  %r32 = call i8 @llvm.uadd.sat.i8(i8 %a32, i8 %b32)
+  %r33 = call i8 @llvm.uadd.sat.i8(i8 %a33, i8 %b33)
+  %r34 = call i8 @llvm.uadd.sat.i8(i8 %a34, i8 %b34)
+  %r35 = call i8 @llvm.uadd.sat.i8(i8 %a35, i8 %b35)
+  %r36 = call i8 @llvm.uadd.sat.i8(i8 %a36, i8 %b36)
+  %r37 = call i8 @llvm.uadd.sat.i8(i8 %a37, i8 %b37)
+  %r38 = call i8 @llvm.uadd.sat.i8(i8 %a38, i8 %b38)
+  %r39 = call i8 @llvm.uadd.sat.i8(i8 %a39, i8 %b39)
+  %r40 = call i8 @llvm.uadd.sat.i8(i8 %a40, i8 %b40)
+  %r41 = call i8 @llvm.uadd.sat.i8(i8 %a41, i8 %b41)
+  %r42 = call i8 @llvm.uadd.sat.i8(i8 %a42, i8 %b42)
+  %r43 = call i8 @llvm.uadd.sat.i8(i8 %a43, i8 %b43)
+  %r44 = call i8 @llvm.uadd.sat.i8(i8 %a44, i8 %b44)
+  %r45 = call i8 @llvm.uadd.sat.i8(i8 %a45, i8 %b45)
+  %r46 = call i8 @llvm.uadd.sat.i8(i8 %a46, i8 %b46)
+  %r47 = call i8 @llvm.uadd.sat.i8(i8 %a47, i8 %b47)
+  %r48 = call i8 @llvm.uadd.sat.i8(i8 %a48, i8 %b48)
+  %r49 = call i8 @llvm.uadd.sat.i8(i8 %a49, i8 %b49)
+  %r50 = call i8 @llvm.uadd.sat.i8(i8 %a50, i8 %b50)
+  %r51 = call i8 @llvm.uadd.sat.i8(i8 %a51, i8 %b51)
+  %r52 = call i8 @llvm.uadd.sat.i8(i8 %a52, i8 %b52)
+  %r53 = call i8 @llvm.uadd.sat.i8(i8 %a53, i8 %b53)
+  %r54 = call i8 @llvm.uadd.sat.i8(i8 %a54, i8 %b54)
+  %r55 = call i8 @llvm.uadd.sat.i8(i8 %a55, i8 %b55)
+  %r56 = call i8 @llvm.uadd.sat.i8(i8 %a56, i8 %b56)
+  %r57 = call i8 @llvm.uadd.sat.i8(i8 %a57, i8 %b57)
+  %r58 = call i8 @llvm.uadd.sat.i8(i8 %a58, i8 %b58)
+  %r59 = call i8 @llvm.uadd.sat.i8(i8 %a59, i8 %b59)
+  %r60 = call i8 @llvm.uadd.sat.i8(i8 %a60, i8 %b60)
+  %r61 = call i8 @llvm.uadd.sat.i8(i8 %a61, i8 %b61)
+  %r62 = call i8 @llvm.uadd.sat.i8(i8 %a62, i8 %b62)
+  %r63 = call i8 @llvm.uadd.sat.i8(i8 %a63, i8 %b63)
+  store i8 %r0 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 0 ), align 1
+  store i8 %r1 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 1 ), align 1
+  store i8 %r2 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 2 ), align 1
+  store i8 %r3 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 3 ), align 1
+  store i8 %r4 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 4 ), align 1
+  store i8 %r5 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 5 ), align 1
+  store i8 %r6 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 6 ), align 1
+  store i8 %r7 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 7 ), align 1
+  store i8 %r8 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 8 ), align 1
+  store i8 %r9 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 9 ), align 1
+  store i8 %r10, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 10), align 1
+  store i8 %r11, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 11), align 1
+  store i8 %r12, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 12), align 1
+  store i8 %r13, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 13), align 1
+  store i8 %r14, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 14), align 1
+  store i8 %r15, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 15), align 1
+  store i8 %r16, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 16), align 1
+  store i8 %r17, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 17), align 1
+  store i8 %r18, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 18), align 1
+  store i8 %r19, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 19), align 1
+  store i8 %r20, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 20), align 1
+  store i8 %r21, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 21), align 1
+  store i8 %r22, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 22), align 1
+  store i8 %r23, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 23), align 1
+  store i8 %r24, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 24), align 1
+  store i8 %r25, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 25), align 1
+  store i8 %r26, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 26), align 1
+  store i8 %r27, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 27), align 1
+  store i8 %r28, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 28), align 1
+  store i8 %r29, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 29), align 1
+  store i8 %r30, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 30), align 1
+  store i8 %r31, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 31), align 1
+  store i8 %r32, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 32), align 1
+  store i8 %r33, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 33), align 1
+  store i8 %r34, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 34), align 1
+  store i8 %r35, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 35), align 1
+  store i8 %r36, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 36), align 1
+  store i8 %r37, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 37), align 1
+  store i8 %r38, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 38), align 1
+  store i8 %r39, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 39), align 1
+  store i8 %r40, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 40), align 1
+  store i8 %r41, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 41), align 1
+  store i8 %r42, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 42), align 1
+  store i8 %r43, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 43), align 1
+  store i8 %r44, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 44), align 1
+  store i8 %r45, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 45), align 1
+  store i8 %r46, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 46), align 1
+  store i8 %r47, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 47), align 1
+  store i8 %r48, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 48), align 1
+  store i8 %r49, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 49), align 1
+  store i8 %r50, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 50), align 1
+  store i8 %r51, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 51), align 1
+  store i8 %r52, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 52), align 1
+  store i8 %r53, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 53), align 1
+  store i8 %r54, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 54), align 1
+  store i8 %r55, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 55), align 1
+  store i8 %r56, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 56), align 1
+  store i8 %r57, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 57), align 1
+  store i8 %r58, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 58), align 1
+  store i8 %r59, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 59), align 1
+  store i8 %r60, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 60), align 1
+  store i8 %r61, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 61), align 1
+  store i8 %r62, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 62), align 1
+  store i8 %r63, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 63), align 1
+  ret void
+}
diff --git a/test/Transforms/SLPVectorizer/X86/arith-sub-ssat.ll b/test/Transforms/SLPVectorizer/X86/arith-sub-ssat.ll
new file mode 100644
index 0000000000000000000000000000000000000000..31c8c4243f177a270424c7e1bc2fca2c0557116c
--- /dev/null
+++ b/test/Transforms/SLPVectorizer/X86/arith-sub-ssat.ll
@@ -0,0 +1,775 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -mtriple=x86_64-unknown -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SSE
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SLM
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX1
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512F
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -mattr=-prefer-256-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512BW
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -mattr=+prefer-256-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX256BW
+
+@a64 = common global [8 x i64] zeroinitializer, align 64
+@b64 = common global [8 x i64] zeroinitializer, align 64
+@c64 = common global [8 x i64] zeroinitializer, align 64
+@a32 = common global [16 x i32] zeroinitializer, align 64
+@b32 = common global [16 x i32] zeroinitializer, align 64
+@c32 = common global [16 x i32] zeroinitializer, align 64
+@a16 = common global [32 x i16] zeroinitializer, align 64
+@b16 = common global [32 x i16] zeroinitializer, align 64
+@c16 = common global [32 x i16] zeroinitializer, align 64
+@a8  = common global [64 x i8] zeroinitializer, align 64
+@b8  = common global [64 x i8] zeroinitializer, align 64
+@c8  = common global [64 x i8] zeroinitializer, align 64
+
+declare i64 @llvm.ssub.sat.i64(i64, i64)
+declare i32 @llvm.ssub.sat.i32(i32, i32)
+declare i16 @llvm.ssub.sat.i16(i16, i16)
+declare i8  @llvm.ssub.sat.i8 (i8 , i8 )
+
+define void @sub_v8i64() {
+; SSE-LABEL: @sub_v8i64(
+; SSE-NEXT:    [[A0:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 0), align 8
+; SSE-NEXT:    [[A1:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 1), align 8
+; SSE-NEXT:    [[A2:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 2), align 8
+; SSE-NEXT:    [[A3:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 3), align 8
+; SSE-NEXT:    [[A4:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 4), align 8
+; SSE-NEXT:    [[A5:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 5), align 8
+; SSE-NEXT:    [[A6:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 6), align 8
+; SSE-NEXT:    [[A7:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 7), align 8
+; SSE-NEXT:    [[B0:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 0), align 8
+; SSE-NEXT:    [[B1:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 1), align 8
+; SSE-NEXT:    [[B2:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 2), align 8
+; SSE-NEXT:    [[B3:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 3), align 8
+; SSE-NEXT:    [[B4:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 4), align 8
+; SSE-NEXT:    [[B5:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 5), align 8
+; SSE-NEXT:    [[B6:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 6), align 8
+; SSE-NEXT:    [[B7:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 7), align 8
+; SSE-NEXT:    [[R0:%.*]] = call i64 @llvm.ssub.sat.i64(i64 [[A0]], i64 [[B0]])
+; SSE-NEXT:    [[R1:%.*]] = call i64 @llvm.ssub.sat.i64(i64 [[A1]], i64 [[B1]])
+; SSE-NEXT:    [[R2:%.*]] = call i64 @llvm.ssub.sat.i64(i64 [[A2]], i64 [[B2]])
+; SSE-NEXT:    [[R3:%.*]] = call i64 @llvm.ssub.sat.i64(i64 [[A3]], i64 [[B3]])
+; SSE-NEXT:    [[R4:%.*]] = call i64 @llvm.ssub.sat.i64(i64 [[A4]], i64 [[B4]])
+; SSE-NEXT:    [[R5:%.*]] = call i64 @llvm.ssub.sat.i64(i64 [[A5]], i64 [[B5]])
+; SSE-NEXT:    [[R6:%.*]] = call i64 @llvm.ssub.sat.i64(i64 [[A6]], i64 [[B6]])
+; SSE-NEXT:    [[R7:%.*]] = call i64 @llvm.ssub.sat.i64(i64 [[A7]], i64 [[B7]])
+; SSE-NEXT:    store i64 [[R0]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 0), align 8
+; SSE-NEXT:    store i64 [[R1]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 1), align 8
+; SSE-NEXT:    store i64 [[R2]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 2), align 8
+; SSE-NEXT:    store i64 [[R3]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 3), align 8
+; SSE-NEXT:    store i64 [[R4]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 4), align 8
+; SSE-NEXT:    store i64 [[R5]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 5), align 8
+; SSE-NEXT:    store i64 [[R6]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 6), align 8
+; SSE-NEXT:    store i64 [[R7]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 7), align 8
+; SSE-NEXT:    ret void
+;
+; SLM-LABEL: @sub_v8i64(
+; SLM-NEXT:    [[A0:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 0), align 8
+; SLM-NEXT:    [[A1:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 1), align 8
+; SLM-NEXT:    [[A2:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 2), align 8
+; SLM-NEXT:    [[A3:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 3), align 8
+; SLM-NEXT:    [[A4:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 4), align 8
+; SLM-NEXT:    [[A5:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 5), align 8
+; SLM-NEXT:    [[A6:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 6), align 8
+; SLM-NEXT:    [[A7:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 7), align 8
+; SLM-NEXT:    [[B0:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 0), align 8
+; SLM-NEXT:    [[B1:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 1), align 8
+; SLM-NEXT:    [[B2:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 2), align 8
+; SLM-NEXT:    [[B3:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 3), align 8
+; SLM-NEXT:    [[B4:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 4), align 8
+; SLM-NEXT:    [[B5:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 5), align 8
+; SLM-NEXT:    [[B6:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 6), align 8
+; SLM-NEXT:    [[B7:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 7), align 8
+; SLM-NEXT:    [[R0:%.*]] = call i64 @llvm.ssub.sat.i64(i64 [[A0]], i64 [[B0]])
+; SLM-NEXT:    [[R1:%.*]] = call i64 @llvm.ssub.sat.i64(i64 [[A1]], i64 [[B1]])
+; SLM-NEXT:    [[R2:%.*]] = call i64 @llvm.ssub.sat.i64(i64 [[A2]], i64 [[B2]])
+; SLM-NEXT:    [[R3:%.*]] = call i64 @llvm.ssub.sat.i64(i64 [[A3]], i64 [[B3]])
+; SLM-NEXT:    [[R4:%.*]] = call i64 @llvm.ssub.sat.i64(i64 [[A4]], i64 [[B4]])
+; SLM-NEXT:    [[R5:%.*]] = call i64 @llvm.ssub.sat.i64(i64 [[A5]], i64 [[B5]])
+; SLM-NEXT:    [[R6:%.*]] = call i64 @llvm.ssub.sat.i64(i64 [[A6]], i64 [[B6]])
+; SLM-NEXT:    [[R7:%.*]] = call i64 @llvm.ssub.sat.i64(i64 [[A7]], i64 [[B7]])
+; SLM-NEXT:    store i64 [[R0]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 0), align 8
+; SLM-NEXT:    store i64 [[R1]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 1), align 8
+; SLM-NEXT:    store i64 [[R2]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 2), align 8
+; SLM-NEXT:    store i64 [[R3]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 3), align 8
+; SLM-NEXT:    store i64 [[R4]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 4), align 8
+; SLM-NEXT:    store i64 [[R5]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 5), align 8
+; SLM-NEXT:    store i64 [[R6]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 6), align 8
+; SLM-NEXT:    store i64 [[R7]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 7), align 8
+; SLM-NEXT:    ret void
+;
+; AVX1-LABEL: @sub_v8i64(
+; AVX1-NEXT:    [[TMP1:%.*]] = load <2 x i64>, <2 x i64>* bitcast ([8 x i64]* @a64 to <2 x i64>*), align 8
+; AVX1-NEXT:    [[TMP2:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 2) to <2 x i64>*), align 8
+; AVX1-NEXT:    [[TMP3:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 4) to <2 x i64>*), align 8
+; AVX1-NEXT:    [[TMP4:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 6) to <2 x i64>*), align 8
+; AVX1-NEXT:    [[TMP5:%.*]] = load <2 x i64>, <2 x i64>* bitcast ([8 x i64]* @b64 to <2 x i64>*), align 8
+; AVX1-NEXT:    [[TMP6:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 2) to <2 x i64>*), align 8
+; AVX1-NEXT:    [[TMP7:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 4) to <2 x i64>*), align 8
+; AVX1-NEXT:    [[TMP8:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 6) to <2 x i64>*), align 8
+; AVX1-NEXT:    [[TMP9:%.*]] = call <2 x i64> @llvm.ssub.sat.v2i64(<2 x i64> [[TMP1]], <2 x i64> [[TMP5]])
+; AVX1-NEXT:    [[TMP10:%.*]] = call <2 x i64> @llvm.ssub.sat.v2i64(<2 x i64> [[TMP2]], <2 x i64> [[TMP6]])
+; AVX1-NEXT:    [[TMP11:%.*]] = call <2 x i64> @llvm.ssub.sat.v2i64(<2 x i64> [[TMP3]], <2 x i64> [[TMP7]])
+; AVX1-NEXT:    [[TMP12:%.*]] = call <2 x i64> @llvm.ssub.sat.v2i64(<2 x i64> [[TMP4]], <2 x i64> [[TMP8]])
+; AVX1-NEXT:    store <2 x i64> [[TMP9]], <2 x i64>* bitcast ([8 x i64]* @c64 to <2 x i64>*), align 8
+; AVX1-NEXT:    store <2 x i64> [[TMP10]], <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 2) to <2 x i64>*), align 8
+; AVX1-NEXT:    store <2 x i64> [[TMP11]], <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 4) to <2 x i64>*), align 8
+; AVX1-NEXT:    store <2 x i64> [[TMP12]], <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 6) to <2 x i64>*), align 8
+; AVX1-NEXT:    ret void
+;
+; AVX2-LABEL: @sub_v8i64(
+; AVX2-NEXT:    [[TMP1:%.*]] = load <4 x i64>, <4 x i64>* bitcast ([8 x i64]* @a64 to <4 x i64>*), align 8
+; AVX2-NEXT:    [[TMP2:%.*]] = load <4 x i64>, <4 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 4) to <4 x i64>*), align 8
+; AVX2-NEXT:    [[TMP3:%.*]] = load <4 x i64>, <4 x i64>* bitcast ([8 x i64]* @b64 to <4 x i64>*), align 8
+; AVX2-NEXT:    [[TMP4:%.*]] = load <4 x i64>, <4 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 4) to <4 x i64>*), align 8
+; AVX2-NEXT:    [[TMP5:%.*]] = call <4 x i64> @llvm.ssub.sat.v4i64(<4 x i64> [[TMP1]], <4 x i64> [[TMP3]])
+; AVX2-NEXT:    [[TMP6:%.*]] = call <4 x i64> @llvm.ssub.sat.v4i64(<4 x i64> [[TMP2]], <4 x i64> [[TMP4]])
+; AVX2-NEXT:    store <4 x i64> [[TMP5]], <4 x i64>* bitcast ([8 x i64]* @c64 to <4 x i64>*), align 8
+; AVX2-NEXT:    store <4 x i64> [[TMP6]], <4 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 4) to <4 x i64>*), align 8
+; AVX2-NEXT:    ret void
+;
+; AVX512-LABEL: @sub_v8i64(
+; AVX512-NEXT:    [[TMP1:%.*]] = load <8 x i64>, <8 x i64>* bitcast ([8 x i64]* @a64 to <8 x i64>*), align 8
+; AVX512-NEXT:    [[TMP2:%.*]] = load <8 x i64>, <8 x i64>* bitcast ([8 x i64]* @b64 to <8 x i64>*), align 8
+; AVX512-NEXT:    [[TMP3:%.*]] = call <8 x i64> @llvm.ssub.sat.v8i64(<8 x i64> [[TMP1]], <8 x i64> [[TMP2]])
+; AVX512-NEXT:    store <8 x i64> [[TMP3]], <8 x i64>* bitcast ([8 x i64]* @c64 to <8 x i64>*), align 8
+; AVX512-NEXT:    ret void
+;
+; AVX256BW-LABEL: @sub_v8i64(
+; AVX256BW-NEXT:    [[TMP1:%.*]] = load <4 x i64>, <4 x i64>* bitcast ([8 x i64]* @a64 to <4 x i64>*), align 8
+; AVX256BW-NEXT:    [[TMP2:%.*]] = load <4 x i64>, <4 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 4) to <4 x i64>*), align 8
+; AVX256BW-NEXT:    [[TMP3:%.*]] = load <4 x i64>, <4 x i64>* bitcast ([8 x i64]* @b64 to <4 x i64>*), align 8
+; AVX256BW-NEXT:    [[TMP4:%.*]] = load <4 x i64>, <4 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 4) to <4 x i64>*), align 8
+; AVX256BW-NEXT:    [[TMP5:%.*]] = call <4 x i64> @llvm.ssub.sat.v4i64(<4 x i64> [[TMP1]], <4 x i64> [[TMP3]])
+; AVX256BW-NEXT:    [[TMP6:%.*]] = call <4 x i64> @llvm.ssub.sat.v4i64(<4 x i64> [[TMP2]], <4 x i64> [[TMP4]])
+; AVX256BW-NEXT:    store <4 x i64> [[TMP5]], <4 x i64>* bitcast ([8 x i64]* @c64 to <4 x i64>*), align 8
+; AVX256BW-NEXT:    store <4 x i64> [[TMP6]], <4 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 4) to <4 x i64>*), align 8
+; AVX256BW-NEXT:    ret void
+;
+  %a0 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 0), align 8
+  %a1 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 1), align 8
+  %a2 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 2), align 8
+  %a3 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 3), align 8
+  %a4 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 4), align 8
+  %a5 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 5), align 8
+  %a6 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 6), align 8
+  %a7 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 7), align 8
+  %b0 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 0), align 8
+  %b1 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 1), align 8
+  %b2 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 2), align 8
+  %b3 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 3), align 8
+  %b4 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 4), align 8
+  %b5 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 5), align 8
+  %b6 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 6), align 8
+  %b7 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 7), align 8
+  %r0 = call i64 @llvm.ssub.sat.i64(i64 %a0, i64 %b0)
+  %r1 = call i64 @llvm.ssub.sat.i64(i64 %a1, i64 %b1)
+  %r2 = call i64 @llvm.ssub.sat.i64(i64 %a2, i64 %b2)
+  %r3 = call i64 @llvm.ssub.sat.i64(i64 %a3, i64 %b3)
+  %r4 = call i64 @llvm.ssub.sat.i64(i64 %a4, i64 %b4)
+  %r5 = call i64 @llvm.ssub.sat.i64(i64 %a5, i64 %b5)
+  %r6 = call i64 @llvm.ssub.sat.i64(i64 %a6, i64 %b6)
+  %r7 = call i64 @llvm.ssub.sat.i64(i64 %a7, i64 %b7)
+  store i64 %r0, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 0), align 8
+  store i64 %r1, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 1), align 8
+  store i64 %r2, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 2), align 8
+  store i64 %r3, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 3), align 8
+  store i64 %r4, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 4), align 8
+  store i64 %r5, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 5), align 8
+  store i64 %r6, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 6), align 8
+  store i64 %r7, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 7), align 8
+  ret void
+}
+
+define void @sub_v16i32() {
+; SSE-LABEL: @sub_v16i32(
+; SSE-NEXT:    [[TMP1:%.*]] = load <4 x i32>, <4 x i32>* bitcast ([16 x i32]* @a32 to <4 x i32>*), align 4
+; SSE-NEXT:    [[TMP2:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 4) to <4 x i32>*), align 4
+; SSE-NEXT:    [[TMP3:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 8) to <4 x i32>*), align 4
+; SSE-NEXT:    [[TMP4:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 12) to <4 x i32>*), align 4
+; SSE-NEXT:    [[TMP5:%.*]] = load <4 x i32>, <4 x i32>* bitcast ([16 x i32]* @b32 to <4 x i32>*), align 4
+; SSE-NEXT:    [[TMP6:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 4) to <4 x i32>*), align 4
+; SSE-NEXT:    [[TMP7:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 8) to <4 x i32>*), align 4
+; SSE-NEXT:    [[TMP8:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 12) to <4 x i32>*), align 4
+; SSE-NEXT:    [[TMP9:%.*]] = call <4 x i32> @llvm.ssub.sat.v4i32(<4 x i32> [[TMP1]], <4 x i32> [[TMP5]])
+; SSE-NEXT:    [[TMP10:%.*]] = call <4 x i32> @llvm.ssub.sat.v4i32(<4 x i32> [[TMP2]], <4 x i32> [[TMP6]])
+; SSE-NEXT:    [[TMP11:%.*]] = call <4 x i32> @llvm.ssub.sat.v4i32(<4 x i32> [[TMP3]], <4 x i32> [[TMP7]])
+; SSE-NEXT:    [[TMP12:%.*]] = call <4 x i32> @llvm.ssub.sat.v4i32(<4 x i32> [[TMP4]], <4 x i32> [[TMP8]])
+; SSE-NEXT:    store <4 x i32> [[TMP9]], <4 x i32>* bitcast ([16 x i32]* @c32 to <4 x i32>*), align 4
+; SSE-NEXT:    store <4 x i32> [[TMP10]], <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 4) to <4 x i32>*), align 4
+; SSE-NEXT:    store <4 x i32> [[TMP11]], <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 8) to <4 x i32>*), align 4
+; SSE-NEXT:    store <4 x i32> [[TMP12]], <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 12) to <4 x i32>*), align 4
+; SSE-NEXT:    ret void
+;
+; SLM-LABEL: @sub_v16i32(
+; SLM-NEXT:    [[TMP1:%.*]] = load <4 x i32>, <4 x i32>* bitcast ([16 x i32]* @a32 to <4 x i32>*), align 4
+; SLM-NEXT:    [[TMP2:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 4) to <4 x i32>*), align 4
+; SLM-NEXT:    [[TMP3:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 8) to <4 x i32>*), align 4
+; SLM-NEXT:    [[TMP4:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 12) to <4 x i32>*), align 4
+; SLM-NEXT:    [[TMP5:%.*]] = load <4 x i32>, <4 x i32>* bitcast ([16 x i32]* @b32 to <4 x i32>*), align 4
+; SLM-NEXT:    [[TMP6:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 4) to <4 x i32>*), align 4
+; SLM-NEXT:    [[TMP7:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 8) to <4 x i32>*), align 4
+; SLM-NEXT:    [[TMP8:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 12) to <4 x i32>*), align 4
+; SLM-NEXT:    [[TMP9:%.*]] = call <4 x i32> @llvm.ssub.sat.v4i32(<4 x i32> [[TMP1]], <4 x i32> [[TMP5]])
+; SLM-NEXT:    [[TMP10:%.*]] = call <4 x i32> @llvm.ssub.sat.v4i32(<4 x i32> [[TMP2]], <4 x i32> [[TMP6]])
+; SLM-NEXT:    [[TMP11:%.*]] = call <4 x i32> @llvm.ssub.sat.v4i32(<4 x i32> [[TMP3]], <4 x i32> [[TMP7]])
+; SLM-NEXT:    [[TMP12:%.*]] = call <4 x i32> @llvm.ssub.sat.v4i32(<4 x i32> [[TMP4]], <4 x i32> [[TMP8]])
+; SLM-NEXT:    store <4 x i32> [[TMP9]], <4 x i32>* bitcast ([16 x i32]* @c32 to <4 x i32>*), align 4
+; SLM-NEXT:    store <4 x i32> [[TMP10]], <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 4) to <4 x i32>*), align 4
+; SLM-NEXT:    store <4 x i32> [[TMP11]], <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 8) to <4 x i32>*), align 4
+; SLM-NEXT:    store <4 x i32> [[TMP12]], <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 12) to <4 x i32>*), align 4
+; SLM-NEXT:    ret void
+;
+; AVX-LABEL: @sub_v16i32(
+; AVX-NEXT:    [[TMP1:%.*]] = load <8 x i32>, <8 x i32>* bitcast ([16 x i32]* @a32 to <8 x i32>*), align 4
+; AVX-NEXT:    [[TMP2:%.*]] = load <8 x i32>, <8 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 8) to <8 x i32>*), align 4
+; AVX-NEXT:    [[TMP3:%.*]] = load <8 x i32>, <8 x i32>* bitcast ([16 x i32]* @b32 to <8 x i32>*), align 4
+; AVX-NEXT:    [[TMP4:%.*]] = load <8 x i32>, <8 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 8) to <8 x i32>*), align 4
+; AVX-NEXT:    [[TMP5:%.*]] = call <8 x i32> @llvm.ssub.sat.v8i32(<8 x i32> [[TMP1]], <8 x i32> [[TMP3]])
+; AVX-NEXT:    [[TMP6:%.*]] = call <8 x i32> @llvm.ssub.sat.v8i32(<8 x i32> [[TMP2]], <8 x i32> [[TMP4]])
+; AVX-NEXT:    store <8 x i32> [[TMP5]], <8 x i32>* bitcast ([16 x i32]* @c32 to <8 x i32>*), align 4
+; AVX-NEXT:    store <8 x i32> [[TMP6]], <8 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 8) to <8 x i32>*), align 4
+; AVX-NEXT:    ret void
+;
+; AVX512-LABEL: @sub_v16i32(
+; AVX512-NEXT:    [[TMP1:%.*]] = load <16 x i32>, <16 x i32>* bitcast ([16 x i32]* @a32 to <16 x i32>*), align 4
+; AVX512-NEXT:    [[TMP2:%.*]] = load <16 x i32>, <16 x i32>* bitcast ([16 x i32]* @b32 to <16 x i32>*), align 4
+; AVX512-NEXT:    [[TMP3:%.*]] = call <16 x i32> @llvm.ssub.sat.v16i32(<16 x i32> [[TMP1]], <16 x i32> [[TMP2]])
+; AVX512-NEXT:    store <16 x i32> [[TMP3]], <16 x i32>* bitcast ([16 x i32]* @c32 to <16 x i32>*), align 4
+; AVX512-NEXT:    ret void
+;
+  %a0  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 0 ), align 4
+  %a1  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 1 ), align 4
+  %a2  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 2 ), align 4
+  %a3  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 3 ), align 4
+  %a4  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 4 ), align 4
+  %a5  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 5 ), align 4
+  %a6  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 6 ), align 4
+  %a7  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 7 ), align 4
+  %a8  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 8 ), align 4
+  %a9  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 9 ), align 4
+  %a10 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 10), align 4
+  %a11 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 11), align 4
+  %a12 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 12), align 4
+  %a13 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 13), align 4
+  %a14 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 14), align 4
+  %a15 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 15), align 4
+  %b0  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 0 ), align 4
+  %b1  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 1 ), align 4
+  %b2  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 2 ), align 4
+  %b3  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 3 ), align 4
+  %b4  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 4 ), align 4
+  %b5  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 5 ), align 4
+  %b6  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 6 ), align 4
+  %b7  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 7 ), align 4
+  %b8  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 8 ), align 4
+  %b9  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 9 ), align 4
+  %b10 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 10), align 4
+  %b11 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 11), align 4
+  %b12 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 12), align 4
+  %b13 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 13), align 4
+  %b14 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 14), align 4
+  %b15 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 15), align 4
+  %r0  = call i32 @llvm.ssub.sat.i32(i32 %a0 , i32 %b0 )
+  %r1  = call i32 @llvm.ssub.sat.i32(i32 %a1 , i32 %b1 )
+  %r2  = call i32 @llvm.ssub.sat.i32(i32 %a2 , i32 %b2 )
+  %r3  = call i32 @llvm.ssub.sat.i32(i32 %a3 , i32 %b3 )
+  %r4  = call i32 @llvm.ssub.sat.i32(i32 %a4 , i32 %b4 )
+  %r5  = call i32 @llvm.ssub.sat.i32(i32 %a5 , i32 %b5 )
+  %r6  = call i32 @llvm.ssub.sat.i32(i32 %a6 , i32 %b6 )
+  %r7  = call i32 @llvm.ssub.sat.i32(i32 %a7 , i32 %b7 )
+  %r8  = call i32 @llvm.ssub.sat.i32(i32 %a8 , i32 %b8 )
+  %r9  = call i32 @llvm.ssub.sat.i32(i32 %a9 , i32 %b9 )
+  %r10 = call i32 @llvm.ssub.sat.i32(i32 %a10, i32 %b10)
+  %r11 = call i32 @llvm.ssub.sat.i32(i32 %a11, i32 %b11)
+  %r12 = call i32 @llvm.ssub.sat.i32(i32 %a12, i32 %b12)
+  %r13 = call i32 @llvm.ssub.sat.i32(i32 %a13, i32 %b13)
+  %r14 = call i32 @llvm.ssub.sat.i32(i32 %a14, i32 %b14)
+  %r15 = call i32 @llvm.ssub.sat.i32(i32 %a15, i32 %b15)
+  store i32 %r0 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 0 ), align 4
+  store i32 %r1 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 1 ), align 4
+  store i32 %r2 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 2 ), align 4
+  store i32 %r3 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 3 ), align 4
+  store i32 %r4 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 4 ), align 4
+  store i32 %r5 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 5 ), align 4
+  store i32 %r6 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 6 ), align 4
+  store i32 %r7 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 7 ), align 4
+  store i32 %r8 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 8 ), align 4
+  store i32 %r9 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 9 ), align 4
+  store i32 %r10, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 10), align 4
+  store i32 %r11, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 11), align 4
+  store i32 %r12, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 12), align 4
+  store i32 %r13, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 13), align 4
+  store i32 %r14, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 14), align 4
+  store i32 %r15, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 15), align 4
+  ret void
+}
+
+define void @sub_v32i16() {
+; SSE-LABEL: @sub_v32i16(
+; SSE-NEXT:    [[TMP1:%.*]] = load <8 x i16>, <8 x i16>* bitcast ([32 x i16]* @a16 to <8 x i16>*), align 2
+; SSE-NEXT:    [[TMP2:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 8) to <8 x i16>*), align 2
+; SSE-NEXT:    [[TMP3:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 16) to <8 x i16>*), align 2
+; SSE-NEXT:    [[TMP4:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 24) to <8 x i16>*), align 2
+; SSE-NEXT:    [[TMP5:%.*]] = load <8 x i16>, <8 x i16>* bitcast ([32 x i16]* @b16 to <8 x i16>*), align 2
+; SSE-NEXT:    [[TMP6:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 8) to <8 x i16>*), align 2
+; SSE-NEXT:    [[TMP7:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 16) to <8 x i16>*), align 2
+; SSE-NEXT:    [[TMP8:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 24) to <8 x i16>*), align 2
+; SSE-NEXT:    [[TMP9:%.*]] = call <8 x i16> @llvm.ssub.sat.v8i16(<8 x i16> [[TMP1]], <8 x i16> [[TMP5]])
+; SSE-NEXT:    [[TMP10:%.*]] = call <8 x i16> @llvm.ssub.sat.v8i16(<8 x i16> [[TMP2]], <8 x i16> [[TMP6]])
+; SSE-NEXT:    [[TMP11:%.*]] = call <8 x i16> @llvm.ssub.sat.v8i16(<8 x i16> [[TMP3]], <8 x i16> [[TMP7]])
+; SSE-NEXT:    [[TMP12:%.*]] = call <8 x i16> @llvm.ssub.sat.v8i16(<8 x i16> [[TMP4]], <8 x i16> [[TMP8]])
+; SSE-NEXT:    store <8 x i16> [[TMP9]], <8 x i16>* bitcast ([32 x i16]* @c16 to <8 x i16>*), align 2
+; SSE-NEXT:    store <8 x i16> [[TMP10]], <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 8) to <8 x i16>*), align 2
+; SSE-NEXT:    store <8 x i16> [[TMP11]], <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 16) to <8 x i16>*), align 2
+; SSE-NEXT:    store <8 x i16> [[TMP12]], <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 24) to <8 x i16>*), align 2
+; SSE-NEXT:    ret void
+;
+; SLM-LABEL: @sub_v32i16(
+; SLM-NEXT:    [[TMP1:%.*]] = load <8 x i16>, <8 x i16>* bitcast ([32 x i16]* @a16 to <8 x i16>*), align 2
+; SLM-NEXT:    [[TMP2:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 8) to <8 x i16>*), align 2
+; SLM-NEXT:    [[TMP3:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 16) to <8 x i16>*), align 2
+; SLM-NEXT:    [[TMP4:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 24) to <8 x i16>*), align 2
+; SLM-NEXT:    [[TMP5:%.*]] = load <8 x i16>, <8 x i16>* bitcast ([32 x i16]* @b16 to <8 x i16>*), align 2
+; SLM-NEXT:    [[TMP6:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 8) to <8 x i16>*), align 2
+; SLM-NEXT:    [[TMP7:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 16) to <8 x i16>*), align 2
+; SLM-NEXT:    [[TMP8:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 24) to <8 x i16>*), align 2
+; SLM-NEXT:    [[TMP9:%.*]] = call <8 x i16> @llvm.ssub.sat.v8i16(<8 x i16> [[TMP1]], <8 x i16> [[TMP5]])
+; SLM-NEXT:    [[TMP10:%.*]] = call <8 x i16> @llvm.ssub.sat.v8i16(<8 x i16> [[TMP2]], <8 x i16> [[TMP6]])
+; SLM-NEXT:    [[TMP11:%.*]] = call <8 x i16> @llvm.ssub.sat.v8i16(<8 x i16> [[TMP3]], <8 x i16> [[TMP7]])
+; SLM-NEXT:    [[TMP12:%.*]] = call <8 x i16> @llvm.ssub.sat.v8i16(<8 x i16> [[TMP4]], <8 x i16> [[TMP8]])
+; SLM-NEXT:    store <8 x i16> [[TMP9]], <8 x i16>* bitcast ([32 x i16]* @c16 to <8 x i16>*), align 2
+; SLM-NEXT:    store <8 x i16> [[TMP10]], <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 8) to <8 x i16>*), align 2
+; SLM-NEXT:    store <8 x i16> [[TMP11]], <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 16) to <8 x i16>*), align 2
+; SLM-NEXT:    store <8 x i16> [[TMP12]], <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 24) to <8 x i16>*), align 2
+; SLM-NEXT:    ret void
+;
+; AVX-LABEL: @sub_v32i16(
+; AVX-NEXT:    [[TMP1:%.*]] = load <16 x i16>, <16 x i16>* bitcast ([32 x i16]* @a16 to <16 x i16>*), align 2
+; AVX-NEXT:    [[TMP2:%.*]] = load <16 x i16>, <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 16) to <16 x i16>*), align 2
+; AVX-NEXT:    [[TMP3:%.*]] = load <16 x i16>, <16 x i16>* bitcast ([32 x i16]* @b16 to <16 x i16>*), align 2
+; AVX-NEXT:    [[TMP4:%.*]] = load <16 x i16>, <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 16) to <16 x i16>*), align 2
+; AVX-NEXT:    [[TMP5:%.*]] = call <16 x i16> @llvm.ssub.sat.v16i16(<16 x i16> [[TMP1]], <16 x i16> [[TMP3]])
+; AVX-NEXT:    [[TMP6:%.*]] = call <16 x i16> @llvm.ssub.sat.v16i16(<16 x i16> [[TMP2]], <16 x i16> [[TMP4]])
+; AVX-NEXT:    store <16 x i16> [[TMP5]], <16 x i16>* bitcast ([32 x i16]* @c16 to <16 x i16>*), align 2
+; AVX-NEXT:    store <16 x i16> [[TMP6]], <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 16) to <16 x i16>*), align 2
+; AVX-NEXT:    ret void
+;
+; AVX512-LABEL: @sub_v32i16(
+; AVX512-NEXT:    [[TMP1:%.*]] = load <16 x i16>, <16 x i16>* bitcast ([32 x i16]* @a16 to <16 x i16>*), align 2
+; AVX512-NEXT:    [[TMP2:%.*]] = load <16 x i16>, <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 16) to <16 x i16>*), align 2
+; AVX512-NEXT:    [[TMP3:%.*]] = load <16 x i16>, <16 x i16>* bitcast ([32 x i16]* @b16 to <16 x i16>*), align 2
+; AVX512-NEXT:    [[TMP4:%.*]] = load <16 x i16>, <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 16) to <16 x i16>*), align 2
+; AVX512-NEXT:    [[TMP5:%.*]] = call <16 x i16> @llvm.ssub.sat.v16i16(<16 x i16> [[TMP1]], <16 x i16> [[TMP3]])
+; AVX512-NEXT:    [[TMP6:%.*]] = call <16 x i16> @llvm.ssub.sat.v16i16(<16 x i16> [[TMP2]], <16 x i16> [[TMP4]])
+; AVX512-NEXT:    store <16 x i16> [[TMP5]], <16 x i16>* bitcast ([32 x i16]* @c16 to <16 x i16>*), align 2
+; AVX512-NEXT:    store <16 x i16> [[TMP6]], <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 16) to <16 x i16>*), align 2
+; AVX512-NEXT:    ret void
+;
+  %a0  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 0 ), align 2
+  %a1  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 1 ), align 2
+  %a2  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 2 ), align 2
+  %a3  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 3 ), align 2
+  %a4  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 4 ), align 2
+  %a5  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 5 ), align 2
+  %a6  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 6 ), align 2
+  %a7  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 7 ), align 2
+  %a8  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 8 ), align 2
+  %a9  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 9 ), align 2
+  %a10 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 10), align 2
+  %a11 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 11), align 2
+  %a12 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 12), align 2
+  %a13 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 13), align 2
+  %a14 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 14), align 2
+  %a15 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 15), align 2
+  %a16 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 16), align 2
+  %a17 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 17), align 2
+  %a18 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 18), align 2
+  %a19 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 19), align 2
+  %a20 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 20), align 2
+  %a21 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 21), align 2
+  %a22 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 22), align 2
+  %a23 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 23), align 2
+  %a24 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 24), align 2
+  %a25 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 25), align 2
+  %a26 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 26), align 2
+  %a27 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 27), align 2
+  %a28 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 28), align 2
+  %a29 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 29), align 2
+  %a30 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 30), align 2
+  %a31 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 31), align 2
+  %b0  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 0 ), align 2
+  %b1  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 1 ), align 2
+  %b2  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 2 ), align 2
+  %b3  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 3 ), align 2
+  %b4  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 4 ), align 2
+  %b5  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 5 ), align 2
+  %b6  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 6 ), align 2
+  %b7  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 7 ), align 2
+  %b8  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 8 ), align 2
+  %b9  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 9 ), align 2
+  %b10 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 10), align 2
+  %b11 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 11), align 2
+  %b12 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 12), align 2
+  %b13 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 13), align 2
+  %b14 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 14), align 2
+  %b15 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 15), align 2
+  %b16 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 16), align 2
+  %b17 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 17), align 2
+  %b18 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 18), align 2
+  %b19 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 19), align 2
+  %b20 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 20), align 2
+  %b21 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 21), align 2
+  %b22 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 22), align 2
+  %b23 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 23), align 2
+  %b24 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 24), align 2
+  %b25 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 25), align 2
+  %b26 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 26), align 2
+  %b27 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 27), align 2
+  %b28 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 28), align 2
+  %b29 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 29), align 2
+  %b30 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 30), align 2
+  %b31 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 31), align 2
+  %r0  = call i16 @llvm.ssub.sat.i16(i16 %a0 , i16 %b0 )
+  %r1  = call i16 @llvm.ssub.sat.i16(i16 %a1 , i16 %b1 )
+  %r2  = call i16 @llvm.ssub.sat.i16(i16 %a2 , i16 %b2 )
+  %r3  = call i16 @llvm.ssub.sat.i16(i16 %a3 , i16 %b3 )
+  %r4  = call i16 @llvm.ssub.sat.i16(i16 %a4 , i16 %b4 )
+  %r5  = call i16 @llvm.ssub.sat.i16(i16 %a5 , i16 %b5 )
+  %r6  = call i16 @llvm.ssub.sat.i16(i16 %a6 , i16 %b6 )
+  %r7  = call i16 @llvm.ssub.sat.i16(i16 %a7 , i16 %b7 )
+  %r8  = call i16 @llvm.ssub.sat.i16(i16 %a8 , i16 %b8 )
+  %r9  = call i16 @llvm.ssub.sat.i16(i16 %a9 , i16 %b9 )
+  %r10 = call i16 @llvm.ssub.sat.i16(i16 %a10, i16 %b10)
+  %r11 = call i16 @llvm.ssub.sat.i16(i16 %a11, i16 %b11)
+  %r12 = call i16 @llvm.ssub.sat.i16(i16 %a12, i16 %b12)
+  %r13 = call i16 @llvm.ssub.sat.i16(i16 %a13, i16 %b13)
+  %r14 = call i16 @llvm.ssub.sat.i16(i16 %a14, i16 %b14)
+  %r15 = call i16 @llvm.ssub.sat.i16(i16 %a15, i16 %b15)
+  %r16 = call i16 @llvm.ssub.sat.i16(i16 %a16, i16 %b16)
+  %r17 = call i16 @llvm.ssub.sat.i16(i16 %a17, i16 %b17)
+  %r18 = call i16 @llvm.ssub.sat.i16(i16 %a18, i16 %b18)
+  %r19 = call i16 @llvm.ssub.sat.i16(i16 %a19, i16 %b19)
+  %r20 = call i16 @llvm.ssub.sat.i16(i16 %a20, i16 %b20)
+  %r21 = call i16 @llvm.ssub.sat.i16(i16 %a21, i16 %b21)
+  %r22 = call i16 @llvm.ssub.sat.i16(i16 %a22, i16 %b22)
+  %r23 = call i16 @llvm.ssub.sat.i16(i16 %a23, i16 %b23)
+  %r24 = call i16 @llvm.ssub.sat.i16(i16 %a24, i16 %b24)
+  %r25 = call i16 @llvm.ssub.sat.i16(i16 %a25, i16 %b25)
+  %r26 = call i16 @llvm.ssub.sat.i16(i16 %a26, i16 %b26)
+  %r27 = call i16 @llvm.ssub.sat.i16(i16 %a27, i16 %b27)
+  %r28 = call i16 @llvm.ssub.sat.i16(i16 %a28, i16 %b28)
+  %r29 = call i16 @llvm.ssub.sat.i16(i16 %a29, i16 %b29)
+  %r30 = call i16 @llvm.ssub.sat.i16(i16 %a30, i16 %b30)
+  %r31 = call i16 @llvm.ssub.sat.i16(i16 %a31, i16 %b31)
+  store i16 %r0 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 0 ), align 2
+  store i16 %r1 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 1 ), align 2
+  store i16 %r2 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 2 ), align 2
+  store i16 %r3 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 3 ), align 2
+  store i16 %r4 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 4 ), align 2
+  store i16 %r5 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 5 ), align 2
+  store i16 %r6 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 6 ), align 2
+  store i16 %r7 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 7 ), align 2
+  store i16 %r8 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 8 ), align 2
+  store i16 %r9 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 9 ), align 2
+  store i16 %r10, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 10), align 2
+  store i16 %r11, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 11), align 2
+  store i16 %r12, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 12), align 2
+  store i16 %r13, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 13), align 2
+  store i16 %r14, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 14), align 2
+  store i16 %r15, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 15), align 2
+  store i16 %r16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 16), align 2
+  store i16 %r17, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 17), align 2
+  store i16 %r18, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 18), align 2
+  store i16 %r19, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 19), align 2
+  store i16 %r20, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 20), align 2
+  store i16 %r21, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 21), align 2
+  store i16 %r22, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 22), align 2
+  store i16 %r23, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 23), align 2
+  store i16 %r24, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 24), align 2
+  store i16 %r25, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 25), align 2
+  store i16 %r26, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 26), align 2
+  store i16 %r27, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 27), align 2
+  store i16 %r28, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 28), align 2
+  store i16 %r29, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 29), align 2
+  store i16 %r30, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 30), align 2
+  store i16 %r31, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 31), align 2
+  ret void
+}
+
+define void @sub_v64i8() {
+; CHECK-LABEL: @sub_v64i8(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i8>, <16 x i8>* bitcast ([64 x i8]* @a8 to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 16) to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 32) to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 48) to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP5:%.*]] = load <16 x i8>, <16 x i8>* bitcast ([64 x i8]* @b8 to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP6:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 16) to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP7:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 32) to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP8:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 48) to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP9:%.*]] = call <16 x i8> @llvm.ssub.sat.v16i8(<16 x i8> [[TMP1]], <16 x i8> [[TMP5]])
+; CHECK-NEXT:    [[TMP10:%.*]] = call <16 x i8> @llvm.ssub.sat.v16i8(<16 x i8> [[TMP2]], <16 x i8> [[TMP6]])
+; CHECK-NEXT:    [[TMP11:%.*]] = call <16 x i8> @llvm.ssub.sat.v16i8(<16 x i8> [[TMP3]], <16 x i8> [[TMP7]])
+; CHECK-NEXT:    [[TMP12:%.*]] = call <16 x i8> @llvm.ssub.sat.v16i8(<16 x i8> [[TMP4]], <16 x i8> [[TMP8]])
+; CHECK-NEXT:    store <16 x i8> [[TMP9]], <16 x i8>* bitcast ([64 x i8]* @c8 to <16 x i8>*), align 1
+; CHECK-NEXT:    store <16 x i8> [[TMP10]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 16) to <16 x i8>*), align 1
+; CHECK-NEXT:    store <16 x i8> [[TMP11]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 32) to <16 x i8>*), align 1
+; CHECK-NEXT:    store <16 x i8> [[TMP12]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 48) to <16 x i8>*), align 1
+; CHECK-NEXT:    ret void
+;
+  %a0  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 0 ), align 1
+  %a1  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 1 ), align 1
+  %a2  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 2 ), align 1
+  %a3  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 3 ), align 1
+  %a4  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 4 ), align 1
+  %a5  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 5 ), align 1
+  %a6  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 6 ), align 1
+  %a7  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 7 ), align 1
+  %a8  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 8 ), align 1
+  %a9  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 9 ), align 1
+  %a10 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 10), align 1
+  %a11 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 11), align 1
+  %a12 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 12), align 1
+  %a13 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 13), align 1
+  %a14 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 14), align 1
+  %a15 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 15), align 1
+  %a16 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 16), align 1
+  %a17 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 17), align 1
+  %a18 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 18), align 1
+  %a19 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 19), align 1
+  %a20 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 20), align 1
+  %a21 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 21), align 1
+  %a22 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 22), align 1
+  %a23 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 23), align 1
+  %a24 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 24), align 1
+  %a25 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 25), align 1
+  %a26 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 26), align 1
+  %a27 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 27), align 1
+  %a28 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 28), align 1
+  %a29 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 29), align 1
+  %a30 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 30), align 1
+  %a31 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 31), align 1
+  %a32 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 32), align 1
+  %a33 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 33), align 1
+  %a34 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 34), align 1
+  %a35 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 35), align 1
+  %a36 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 36), align 1
+  %a37 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 37), align 1
+  %a38 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 38), align 1
+  %a39 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 39), align 1
+  %a40 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 40), align 1
+  %a41 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 41), align 1
+  %a42 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 42), align 1
+  %a43 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 43), align 1
+  %a44 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 44), align 1
+  %a45 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 45), align 1
+  %a46 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 46), align 1
+  %a47 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 47), align 1
+  %a48 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 48), align 1
+  %a49 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 49), align 1
+  %a50 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 50), align 1
+  %a51 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 51), align 1
+  %a52 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 52), align 1
+  %a53 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 53), align 1
+  %a54 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 54), align 1
+  %a55 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 55), align 1
+  %a56 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 56), align 1
+  %a57 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 57), align 1
+  %a58 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 58), align 1
+  %a59 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 59), align 1
+  %a60 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 60), align 1
+  %a61 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 61), align 1
+  %a62 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 62), align 1
+  %a63 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 63), align 1
+  %b0  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 0 ), align 1
+  %b1  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 1 ), align 1
+  %b2  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 2 ), align 1
+  %b3  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 3 ), align 1
+  %b4  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 4 ), align 1
+  %b5  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 5 ), align 1
+  %b6  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 6 ), align 1
+  %b7  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 7 ), align 1
+  %b8  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 8 ), align 1
+  %b9  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 9 ), align 1
+  %b10 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 10), align 1
+  %b11 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 11), align 1
+  %b12 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 12), align 1
+  %b13 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 13), align 1
+  %b14 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 14), align 1
+  %b15 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 15), align 1
+  %b16 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 16), align 1
+  %b17 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 17), align 1
+  %b18 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 18), align 1
+  %b19 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 19), align 1
+  %b20 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 20), align 1
+  %b21 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 21), align 1
+  %b22 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 22), align 1
+  %b23 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 23), align 1
+  %b24 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 24), align 1
+  %b25 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 25), align 1
+  %b26 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 26), align 1
+  %b27 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 27), align 1
+  %b28 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 28), align 1
+  %b29 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 29), align 1
+  %b30 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 30), align 1
+  %b31 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 31), align 1
+  %b32 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 32), align 1
+  %b33 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 33), align 1
+  %b34 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 34), align 1
+  %b35 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 35), align 1
+  %b36 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 36), align 1
+  %b37 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 37), align 1
+  %b38 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 38), align 1
+  %b39 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 39), align 1
+  %b40 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 40), align 1
+  %b41 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 41), align 1
+  %b42 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 42), align 1
+  %b43 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 43), align 1
+  %b44 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 44), align 1
+  %b45 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 45), align 1
+  %b46 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 46), align 1
+  %b47 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 47), align 1
+  %b48 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 48), align 1
+  %b49 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 49), align 1
+  %b50 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 50), align 1
+  %b51 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 51), align 1
+  %b52 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 52), align 1
+  %b53 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 53), align 1
+  %b54 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 54), align 1
+  %b55 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 55), align 1
+  %b56 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 56), align 1
+  %b57 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 57), align 1
+  %b58 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 58), align 1
+  %b59 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 59), align 1
+  %b60 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 60), align 1
+  %b61 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 61), align 1
+  %b62 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 62), align 1
+  %b63 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 63), align 1
+  %r0  = call i8 @llvm.ssub.sat.i8(i8 %a0 , i8 %b0 )
+  %r1  = call i8 @llvm.ssub.sat.i8(i8 %a1 , i8 %b1 )
+  %r2  = call i8 @llvm.ssub.sat.i8(i8 %a2 , i8 %b2 )
+  %r3  = call i8 @llvm.ssub.sat.i8(i8 %a3 , i8 %b3 )
+  %r4  = call i8 @llvm.ssub.sat.i8(i8 %a4 , i8 %b4 )
+  %r5  = call i8 @llvm.ssub.sat.i8(i8 %a5 , i8 %b5 )
+  %r6  = call i8 @llvm.ssub.sat.i8(i8 %a6 , i8 %b6 )
+  %r7  = call i8 @llvm.ssub.sat.i8(i8 %a7 , i8 %b7 )
+  %r8  = call i8 @llvm.ssub.sat.i8(i8 %a8 , i8 %b8 )
+  %r9  = call i8 @llvm.ssub.sat.i8(i8 %a9 , i8 %b9 )
+  %r10 = call i8 @llvm.ssub.sat.i8(i8 %a10, i8 %b10)
+  %r11 = call i8 @llvm.ssub.sat.i8(i8 %a11, i8 %b11)
+  %r12 = call i8 @llvm.ssub.sat.i8(i8 %a12, i8 %b12)
+  %r13 = call i8 @llvm.ssub.sat.i8(i8 %a13, i8 %b13)
+  %r14 = call i8 @llvm.ssub.sat.i8(i8 %a14, i8 %b14)
+  %r15 = call i8 @llvm.ssub.sat.i8(i8 %a15, i8 %b15)
+  %r16 = call i8 @llvm.ssub.sat.i8(i8 %a16, i8 %b16)
+  %r17 = call i8 @llvm.ssub.sat.i8(i8 %a17, i8 %b17)
+  %r18 = call i8 @llvm.ssub.sat.i8(i8 %a18, i8 %b18)
+  %r19 = call i8 @llvm.ssub.sat.i8(i8 %a19, i8 %b19)
+  %r20 = call i8 @llvm.ssub.sat.i8(i8 %a20, i8 %b20)
+  %r21 = call i8 @llvm.ssub.sat.i8(i8 %a21, i8 %b21)
+  %r22 = call i8 @llvm.ssub.sat.i8(i8 %a22, i8 %b22)
+  %r23 = call i8 @llvm.ssub.sat.i8(i8 %a23, i8 %b23)
+  %r24 = call i8 @llvm.ssub.sat.i8(i8 %a24, i8 %b24)
+  %r25 = call i8 @llvm.ssub.sat.i8(i8 %a25, i8 %b25)
+  %r26 = call i8 @llvm.ssub.sat.i8(i8 %a26, i8 %b26)
+  %r27 = call i8 @llvm.ssub.sat.i8(i8 %a27, i8 %b27)
+  %r28 = call i8 @llvm.ssub.sat.i8(i8 %a28, i8 %b28)
+  %r29 = call i8 @llvm.ssub.sat.i8(i8 %a29, i8 %b29)
+  %r30 = call i8 @llvm.ssub.sat.i8(i8 %a30, i8 %b30)
+  %r31 = call i8 @llvm.ssub.sat.i8(i8 %a31, i8 %b31)
+  %r32 = call i8 @llvm.ssub.sat.i8(i8 %a32, i8 %b32)
+  %r33 = call i8 @llvm.ssub.sat.i8(i8 %a33, i8 %b33)
+  %r34 = call i8 @llvm.ssub.sat.i8(i8 %a34, i8 %b34)
+  %r35 = call i8 @llvm.ssub.sat.i8(i8 %a35, i8 %b35)
+  %r36 = call i8 @llvm.ssub.sat.i8(i8 %a36, i8 %b36)
+  %r37 = call i8 @llvm.ssub.sat.i8(i8 %a37, i8 %b37)
+  %r38 = call i8 @llvm.ssub.sat.i8(i8 %a38, i8 %b38)
+  %r39 = call i8 @llvm.ssub.sat.i8(i8 %a39, i8 %b39)
+  %r40 = call i8 @llvm.ssub.sat.i8(i8 %a40, i8 %b40)
+  %r41 = call i8 @llvm.ssub.sat.i8(i8 %a41, i8 %b41)
+  %r42 = call i8 @llvm.ssub.sat.i8(i8 %a42, i8 %b42)
+  %r43 = call i8 @llvm.ssub.sat.i8(i8 %a43, i8 %b43)
+  %r44 = call i8 @llvm.ssub.sat.i8(i8 %a44, i8 %b44)
+  %r45 = call i8 @llvm.ssub.sat.i8(i8 %a45, i8 %b45)
+  %r46 = call i8 @llvm.ssub.sat.i8(i8 %a46, i8 %b46)
+  %r47 = call i8 @llvm.ssub.sat.i8(i8 %a47, i8 %b47)
+  %r48 = call i8 @llvm.ssub.sat.i8(i8 %a48, i8 %b48)
+  %r49 = call i8 @llvm.ssub.sat.i8(i8 %a49, i8 %b49)
+  %r50 = call i8 @llvm.ssub.sat.i8(i8 %a50, i8 %b50)
+  %r51 = call i8 @llvm.ssub.sat.i8(i8 %a51, i8 %b51)
+  %r52 = call i8 @llvm.ssub.sat.i8(i8 %a52, i8 %b52)
+  %r53 = call i8 @llvm.ssub.sat.i8(i8 %a53, i8 %b53)
+  %r54 = call i8 @llvm.ssub.sat.i8(i8 %a54, i8 %b54)
+  %r55 = call i8 @llvm.ssub.sat.i8(i8 %a55, i8 %b55)
+  %r56 = call i8 @llvm.ssub.sat.i8(i8 %a56, i8 %b56)
+  %r57 = call i8 @llvm.ssub.sat.i8(i8 %a57, i8 %b57)
+  %r58 = call i8 @llvm.ssub.sat.i8(i8 %a58, i8 %b58)
+  %r59 = call i8 @llvm.ssub.sat.i8(i8 %a59, i8 %b59)
+  %r60 = call i8 @llvm.ssub.sat.i8(i8 %a60, i8 %b60)
+  %r61 = call i8 @llvm.ssub.sat.i8(i8 %a61, i8 %b61)
+  %r62 = call i8 @llvm.ssub.sat.i8(i8 %a62, i8 %b62)
+  %r63 = call i8 @llvm.ssub.sat.i8(i8 %a63, i8 %b63)
+  store i8 %r0 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 0 ), align 1
+  store i8 %r1 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 1 ), align 1
+  store i8 %r2 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 2 ), align 1
+  store i8 %r3 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 3 ), align 1
+  store i8 %r4 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 4 ), align 1
+  store i8 %r5 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 5 ), align 1
+  store i8 %r6 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 6 ), align 1
+  store i8 %r7 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 7 ), align 1
+  store i8 %r8 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 8 ), align 1
+  store i8 %r9 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 9 ), align 1
+  store i8 %r10, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 10), align 1
+  store i8 %r11, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 11), align 1
+  store i8 %r12, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 12), align 1
+  store i8 %r13, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 13), align 1
+  store i8 %r14, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 14), align 1
+  store i8 %r15, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 15), align 1
+  store i8 %r16, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 16), align 1
+  store i8 %r17, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 17), align 1
+  store i8 %r18, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 18), align 1
+  store i8 %r19, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 19), align 1
+  store i8 %r20, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 20), align 1
+  store i8 %r21, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 21), align 1
+  store i8 %r22, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 22), align 1
+  store i8 %r23, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 23), align 1
+  store i8 %r24, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 24), align 1
+  store i8 %r25, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 25), align 1
+  store i8 %r26, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 26), align 1
+  store i8 %r27, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 27), align 1
+  store i8 %r28, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 28), align 1
+  store i8 %r29, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 29), align 1
+  store i8 %r30, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 30), align 1
+  store i8 %r31, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 31), align 1
+  store i8 %r32, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 32), align 1
+  store i8 %r33, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 33), align 1
+  store i8 %r34, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 34), align 1
+  store i8 %r35, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 35), align 1
+  store i8 %r36, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 36), align 1
+  store i8 %r37, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 37), align 1
+  store i8 %r38, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 38), align 1
+  store i8 %r39, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 39), align 1
+  store i8 %r40, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 40), align 1
+  store i8 %r41, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 41), align 1
+  store i8 %r42, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 42), align 1
+  store i8 %r43, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 43), align 1
+  store i8 %r44, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 44), align 1
+  store i8 %r45, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 45), align 1
+  store i8 %r46, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 46), align 1
+  store i8 %r47, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 47), align 1
+  store i8 %r48, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 48), align 1
+  store i8 %r49, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 49), align 1
+  store i8 %r50, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 50), align 1
+  store i8 %r51, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 51), align 1
+  store i8 %r52, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 52), align 1
+  store i8 %r53, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 53), align 1
+  store i8 %r54, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 54), align 1
+  store i8 %r55, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 55), align 1
+  store i8 %r56, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 56), align 1
+  store i8 %r57, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 57), align 1
+  store i8 %r58, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 58), align 1
+  store i8 %r59, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 59), align 1
+  store i8 %r60, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 60), align 1
+  store i8 %r61, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 61), align 1
+  store i8 %r62, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 62), align 1
+  store i8 %r63, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 63), align 1
+  ret void
+}
diff --git a/test/Transforms/SLPVectorizer/X86/arith-sub-usat.ll b/test/Transforms/SLPVectorizer/X86/arith-sub-usat.ll
new file mode 100644
index 0000000000000000000000000000000000000000..ec554ff4fb514c5cc4ebd6191ee69df3c6f1cc90
--- /dev/null
+++ b/test/Transforms/SLPVectorizer/X86/arith-sub-usat.ll
@@ -0,0 +1,729 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -mtriple=x86_64-unknown -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SSE
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SLM
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX1
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX2
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512F
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -mattr=-prefer-256-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512BW
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=skx -mattr=+prefer-256-bit -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX256BW
+
+@a64 = common global [8 x i64] zeroinitializer, align 64
+@b64 = common global [8 x i64] zeroinitializer, align 64
+@c64 = common global [8 x i64] zeroinitializer, align 64
+@a32 = common global [16 x i32] zeroinitializer, align 64
+@b32 = common global [16 x i32] zeroinitializer, align 64
+@c32 = common global [16 x i32] zeroinitializer, align 64
+@a16 = common global [32 x i16] zeroinitializer, align 64
+@b16 = common global [32 x i16] zeroinitializer, align 64
+@c16 = common global [32 x i16] zeroinitializer, align 64
+@a8  = common global [64 x i8] zeroinitializer, align 64
+@b8  = common global [64 x i8] zeroinitializer, align 64
+@c8  = common global [64 x i8] zeroinitializer, align 64
+
+declare i64 @llvm.usub.sat.i64(i64, i64)
+declare i32 @llvm.usub.sat.i32(i32, i32)
+declare i16 @llvm.usub.sat.i16(i16, i16)
+declare i8  @llvm.usub.sat.i8 (i8 , i8 )
+
+define void @sub_v8i64() {
+; SSE-LABEL: @sub_v8i64(
+; SSE-NEXT:    [[A0:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 0), align 8
+; SSE-NEXT:    [[A1:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 1), align 8
+; SSE-NEXT:    [[A2:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 2), align 8
+; SSE-NEXT:    [[A3:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 3), align 8
+; SSE-NEXT:    [[A4:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 4), align 8
+; SSE-NEXT:    [[A5:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 5), align 8
+; SSE-NEXT:    [[A6:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 6), align 8
+; SSE-NEXT:    [[A7:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 7), align 8
+; SSE-NEXT:    [[B0:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 0), align 8
+; SSE-NEXT:    [[B1:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 1), align 8
+; SSE-NEXT:    [[B2:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 2), align 8
+; SSE-NEXT:    [[B3:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 3), align 8
+; SSE-NEXT:    [[B4:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 4), align 8
+; SSE-NEXT:    [[B5:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 5), align 8
+; SSE-NEXT:    [[B6:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 6), align 8
+; SSE-NEXT:    [[B7:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 7), align 8
+; SSE-NEXT:    [[R0:%.*]] = call i64 @llvm.usub.sat.i64(i64 [[A0]], i64 [[B0]])
+; SSE-NEXT:    [[R1:%.*]] = call i64 @llvm.usub.sat.i64(i64 [[A1]], i64 [[B1]])
+; SSE-NEXT:    [[R2:%.*]] = call i64 @llvm.usub.sat.i64(i64 [[A2]], i64 [[B2]])
+; SSE-NEXT:    [[R3:%.*]] = call i64 @llvm.usub.sat.i64(i64 [[A3]], i64 [[B3]])
+; SSE-NEXT:    [[R4:%.*]] = call i64 @llvm.usub.sat.i64(i64 [[A4]], i64 [[B4]])
+; SSE-NEXT:    [[R5:%.*]] = call i64 @llvm.usub.sat.i64(i64 [[A5]], i64 [[B5]])
+; SSE-NEXT:    [[R6:%.*]] = call i64 @llvm.usub.sat.i64(i64 [[A6]], i64 [[B6]])
+; SSE-NEXT:    [[R7:%.*]] = call i64 @llvm.usub.sat.i64(i64 [[A7]], i64 [[B7]])
+; SSE-NEXT:    store i64 [[R0]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 0), align 8
+; SSE-NEXT:    store i64 [[R1]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 1), align 8
+; SSE-NEXT:    store i64 [[R2]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 2), align 8
+; SSE-NEXT:    store i64 [[R3]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 3), align 8
+; SSE-NEXT:    store i64 [[R4]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 4), align 8
+; SSE-NEXT:    store i64 [[R5]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 5), align 8
+; SSE-NEXT:    store i64 [[R6]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 6), align 8
+; SSE-NEXT:    store i64 [[R7]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 7), align 8
+; SSE-NEXT:    ret void
+;
+; SLM-LABEL: @sub_v8i64(
+; SLM-NEXT:    [[TMP1:%.*]] = load <2 x i64>, <2 x i64>* bitcast ([8 x i64]* @a64 to <2 x i64>*), align 8
+; SLM-NEXT:    [[TMP2:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 2) to <2 x i64>*), align 8
+; SLM-NEXT:    [[TMP3:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 4) to <2 x i64>*), align 8
+; SLM-NEXT:    [[TMP4:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 6) to <2 x i64>*), align 8
+; SLM-NEXT:    [[TMP5:%.*]] = load <2 x i64>, <2 x i64>* bitcast ([8 x i64]* @b64 to <2 x i64>*), align 8
+; SLM-NEXT:    [[TMP6:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 2) to <2 x i64>*), align 8
+; SLM-NEXT:    [[TMP7:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 4) to <2 x i64>*), align 8
+; SLM-NEXT:    [[TMP8:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 6) to <2 x i64>*), align 8
+; SLM-NEXT:    [[TMP9:%.*]] = call <2 x i64> @llvm.usub.sat.v2i64(<2 x i64> [[TMP1]], <2 x i64> [[TMP5]])
+; SLM-NEXT:    [[TMP10:%.*]] = call <2 x i64> @llvm.usub.sat.v2i64(<2 x i64> [[TMP2]], <2 x i64> [[TMP6]])
+; SLM-NEXT:    [[TMP11:%.*]] = call <2 x i64> @llvm.usub.sat.v2i64(<2 x i64> [[TMP3]], <2 x i64> [[TMP7]])
+; SLM-NEXT:    [[TMP12:%.*]] = call <2 x i64> @llvm.usub.sat.v2i64(<2 x i64> [[TMP4]], <2 x i64> [[TMP8]])
+; SLM-NEXT:    store <2 x i64> [[TMP9]], <2 x i64>* bitcast ([8 x i64]* @c64 to <2 x i64>*), align 8
+; SLM-NEXT:    store <2 x i64> [[TMP10]], <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 2) to <2 x i64>*), align 8
+; SLM-NEXT:    store <2 x i64> [[TMP11]], <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 4) to <2 x i64>*), align 8
+; SLM-NEXT:    store <2 x i64> [[TMP12]], <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 6) to <2 x i64>*), align 8
+; SLM-NEXT:    ret void
+;
+; AVX-LABEL: @sub_v8i64(
+; AVX-NEXT:    [[TMP1:%.*]] = load <4 x i64>, <4 x i64>* bitcast ([8 x i64]* @a64 to <4 x i64>*), align 8
+; AVX-NEXT:    [[TMP2:%.*]] = load <4 x i64>, <4 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 4) to <4 x i64>*), align 8
+; AVX-NEXT:    [[TMP3:%.*]] = load <4 x i64>, <4 x i64>* bitcast ([8 x i64]* @b64 to <4 x i64>*), align 8
+; AVX-NEXT:    [[TMP4:%.*]] = load <4 x i64>, <4 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 4) to <4 x i64>*), align 8
+; AVX-NEXT:    [[TMP5:%.*]] = call <4 x i64> @llvm.usub.sat.v4i64(<4 x i64> [[TMP1]], <4 x i64> [[TMP3]])
+; AVX-NEXT:    [[TMP6:%.*]] = call <4 x i64> @llvm.usub.sat.v4i64(<4 x i64> [[TMP2]], <4 x i64> [[TMP4]])
+; AVX-NEXT:    store <4 x i64> [[TMP5]], <4 x i64>* bitcast ([8 x i64]* @c64 to <4 x i64>*), align 8
+; AVX-NEXT:    store <4 x i64> [[TMP6]], <4 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 4) to <4 x i64>*), align 8
+; AVX-NEXT:    ret void
+;
+; AVX512-LABEL: @sub_v8i64(
+; AVX512-NEXT:    [[TMP1:%.*]] = load <8 x i64>, <8 x i64>* bitcast ([8 x i64]* @a64 to <8 x i64>*), align 8
+; AVX512-NEXT:    [[TMP2:%.*]] = load <8 x i64>, <8 x i64>* bitcast ([8 x i64]* @b64 to <8 x i64>*), align 8
+; AVX512-NEXT:    [[TMP3:%.*]] = call <8 x i64> @llvm.usub.sat.v8i64(<8 x i64> [[TMP1]], <8 x i64> [[TMP2]])
+; AVX512-NEXT:    store <8 x i64> [[TMP3]], <8 x i64>* bitcast ([8 x i64]* @c64 to <8 x i64>*), align 8
+; AVX512-NEXT:    ret void
+;
+  %a0 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 0), align 8
+  %a1 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 1), align 8
+  %a2 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 2), align 8
+  %a3 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 3), align 8
+  %a4 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 4), align 8
+  %a5 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 5), align 8
+  %a6 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 6), align 8
+  %a7 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 7), align 8
+  %b0 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 0), align 8
+  %b1 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 1), align 8
+  %b2 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 2), align 8
+  %b3 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 3), align 8
+  %b4 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 4), align 8
+  %b5 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 5), align 8
+  %b6 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 6), align 8
+  %b7 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 7), align 8
+  %r0 = call i64 @llvm.usub.sat.i64(i64 %a0, i64 %b0)
+  %r1 = call i64 @llvm.usub.sat.i64(i64 %a1, i64 %b1)
+  %r2 = call i64 @llvm.usub.sat.i64(i64 %a2, i64 %b2)
+  %r3 = call i64 @llvm.usub.sat.i64(i64 %a3, i64 %b3)
+  %r4 = call i64 @llvm.usub.sat.i64(i64 %a4, i64 %b4)
+  %r5 = call i64 @llvm.usub.sat.i64(i64 %a5, i64 %b5)
+  %r6 = call i64 @llvm.usub.sat.i64(i64 %a6, i64 %b6)
+  %r7 = call i64 @llvm.usub.sat.i64(i64 %a7, i64 %b7)
+  store i64 %r0, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 0), align 8
+  store i64 %r1, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 1), align 8
+  store i64 %r2, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 2), align 8
+  store i64 %r3, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 3), align 8
+  store i64 %r4, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 4), align 8
+  store i64 %r5, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 5), align 8
+  store i64 %r6, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 6), align 8
+  store i64 %r7, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 7), align 8
+  ret void
+}
+
+define void @sub_v16i32() {
+; SSE-LABEL: @sub_v16i32(
+; SSE-NEXT:    [[TMP1:%.*]] = load <4 x i32>, <4 x i32>* bitcast ([16 x i32]* @a32 to <4 x i32>*), align 4
+; SSE-NEXT:    [[TMP2:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 4) to <4 x i32>*), align 4
+; SSE-NEXT:    [[TMP3:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 8) to <4 x i32>*), align 4
+; SSE-NEXT:    [[TMP4:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 12) to <4 x i32>*), align 4
+; SSE-NEXT:    [[TMP5:%.*]] = load <4 x i32>, <4 x i32>* bitcast ([16 x i32]* @b32 to <4 x i32>*), align 4
+; SSE-NEXT:    [[TMP6:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 4) to <4 x i32>*), align 4
+; SSE-NEXT:    [[TMP7:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 8) to <4 x i32>*), align 4
+; SSE-NEXT:    [[TMP8:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 12) to <4 x i32>*), align 4
+; SSE-NEXT:    [[TMP9:%.*]] = call <4 x i32> @llvm.usub.sat.v4i32(<4 x i32> [[TMP1]], <4 x i32> [[TMP5]])
+; SSE-NEXT:    [[TMP10:%.*]] = call <4 x i32> @llvm.usub.sat.v4i32(<4 x i32> [[TMP2]], <4 x i32> [[TMP6]])
+; SSE-NEXT:    [[TMP11:%.*]] = call <4 x i32> @llvm.usub.sat.v4i32(<4 x i32> [[TMP3]], <4 x i32> [[TMP7]])
+; SSE-NEXT:    [[TMP12:%.*]] = call <4 x i32> @llvm.usub.sat.v4i32(<4 x i32> [[TMP4]], <4 x i32> [[TMP8]])
+; SSE-NEXT:    store <4 x i32> [[TMP9]], <4 x i32>* bitcast ([16 x i32]* @c32 to <4 x i32>*), align 4
+; SSE-NEXT:    store <4 x i32> [[TMP10]], <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 4) to <4 x i32>*), align 4
+; SSE-NEXT:    store <4 x i32> [[TMP11]], <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 8) to <4 x i32>*), align 4
+; SSE-NEXT:    store <4 x i32> [[TMP12]], <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 12) to <4 x i32>*), align 4
+; SSE-NEXT:    ret void
+;
+; SLM-LABEL: @sub_v16i32(
+; SLM-NEXT:    [[TMP1:%.*]] = load <4 x i32>, <4 x i32>* bitcast ([16 x i32]* @a32 to <4 x i32>*), align 4
+; SLM-NEXT:    [[TMP2:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 4) to <4 x i32>*), align 4
+; SLM-NEXT:    [[TMP3:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 8) to <4 x i32>*), align 4
+; SLM-NEXT:    [[TMP4:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 12) to <4 x i32>*), align 4
+; SLM-NEXT:    [[TMP5:%.*]] = load <4 x i32>, <4 x i32>* bitcast ([16 x i32]* @b32 to <4 x i32>*), align 4
+; SLM-NEXT:    [[TMP6:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 4) to <4 x i32>*), align 4
+; SLM-NEXT:    [[TMP7:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 8) to <4 x i32>*), align 4
+; SLM-NEXT:    [[TMP8:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 12) to <4 x i32>*), align 4
+; SLM-NEXT:    [[TMP9:%.*]] = call <4 x i32> @llvm.usub.sat.v4i32(<4 x i32> [[TMP1]], <4 x i32> [[TMP5]])
+; SLM-NEXT:    [[TMP10:%.*]] = call <4 x i32> @llvm.usub.sat.v4i32(<4 x i32> [[TMP2]], <4 x i32> [[TMP6]])
+; SLM-NEXT:    [[TMP11:%.*]] = call <4 x i32> @llvm.usub.sat.v4i32(<4 x i32> [[TMP3]], <4 x i32> [[TMP7]])
+; SLM-NEXT:    [[TMP12:%.*]] = call <4 x i32> @llvm.usub.sat.v4i32(<4 x i32> [[TMP4]], <4 x i32> [[TMP8]])
+; SLM-NEXT:    store <4 x i32> [[TMP9]], <4 x i32>* bitcast ([16 x i32]* @c32 to <4 x i32>*), align 4
+; SLM-NEXT:    store <4 x i32> [[TMP10]], <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 4) to <4 x i32>*), align 4
+; SLM-NEXT:    store <4 x i32> [[TMP11]], <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 8) to <4 x i32>*), align 4
+; SLM-NEXT:    store <4 x i32> [[TMP12]], <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 12) to <4 x i32>*), align 4
+; SLM-NEXT:    ret void
+;
+; AVX-LABEL: @sub_v16i32(
+; AVX-NEXT:    [[TMP1:%.*]] = load <8 x i32>, <8 x i32>* bitcast ([16 x i32]* @a32 to <8 x i32>*), align 4
+; AVX-NEXT:    [[TMP2:%.*]] = load <8 x i32>, <8 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 8) to <8 x i32>*), align 4
+; AVX-NEXT:    [[TMP3:%.*]] = load <8 x i32>, <8 x i32>* bitcast ([16 x i32]* @b32 to <8 x i32>*), align 4
+; AVX-NEXT:    [[TMP4:%.*]] = load <8 x i32>, <8 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 8) to <8 x i32>*), align 4
+; AVX-NEXT:    [[TMP5:%.*]] = call <8 x i32> @llvm.usub.sat.v8i32(<8 x i32> [[TMP1]], <8 x i32> [[TMP3]])
+; AVX-NEXT:    [[TMP6:%.*]] = call <8 x i32> @llvm.usub.sat.v8i32(<8 x i32> [[TMP2]], <8 x i32> [[TMP4]])
+; AVX-NEXT:    store <8 x i32> [[TMP5]], <8 x i32>* bitcast ([16 x i32]* @c32 to <8 x i32>*), align 4
+; AVX-NEXT:    store <8 x i32> [[TMP6]], <8 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 8) to <8 x i32>*), align 4
+; AVX-NEXT:    ret void
+;
+; AVX512-LABEL: @sub_v16i32(
+; AVX512-NEXT:    [[TMP1:%.*]] = load <16 x i32>, <16 x i32>* bitcast ([16 x i32]* @a32 to <16 x i32>*), align 4
+; AVX512-NEXT:    [[TMP2:%.*]] = load <16 x i32>, <16 x i32>* bitcast ([16 x i32]* @b32 to <16 x i32>*), align 4
+; AVX512-NEXT:    [[TMP3:%.*]] = call <16 x i32> @llvm.usub.sat.v16i32(<16 x i32> [[TMP1]], <16 x i32> [[TMP2]])
+; AVX512-NEXT:    store <16 x i32> [[TMP3]], <16 x i32>* bitcast ([16 x i32]* @c32 to <16 x i32>*), align 4
+; AVX512-NEXT:    ret void
+;
+  %a0  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 0 ), align 4
+  %a1  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 1 ), align 4
+  %a2  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 2 ), align 4
+  %a3  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 3 ), align 4
+  %a4  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 4 ), align 4
+  %a5  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 5 ), align 4
+  %a6  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 6 ), align 4
+  %a7  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 7 ), align 4
+  %a8  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 8 ), align 4
+  %a9  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 9 ), align 4
+  %a10 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 10), align 4
+  %a11 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 11), align 4
+  %a12 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 12), align 4
+  %a13 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 13), align 4
+  %a14 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 14), align 4
+  %a15 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 15), align 4
+  %b0  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 0 ), align 4
+  %b1  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 1 ), align 4
+  %b2  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 2 ), align 4
+  %b3  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 3 ), align 4
+  %b4  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 4 ), align 4
+  %b5  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 5 ), align 4
+  %b6  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 6 ), align 4
+  %b7  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 7 ), align 4
+  %b8  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 8 ), align 4
+  %b9  = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 9 ), align 4
+  %b10 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 10), align 4
+  %b11 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 11), align 4
+  %b12 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 12), align 4
+  %b13 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 13), align 4
+  %b14 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 14), align 4
+  %b15 = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 15), align 4
+  %r0  = call i32 @llvm.usub.sat.i32(i32 %a0 , i32 %b0 )
+  %r1  = call i32 @llvm.usub.sat.i32(i32 %a1 , i32 %b1 )
+  %r2  = call i32 @llvm.usub.sat.i32(i32 %a2 , i32 %b2 )
+  %r3  = call i32 @llvm.usub.sat.i32(i32 %a3 , i32 %b3 )
+  %r4  = call i32 @llvm.usub.sat.i32(i32 %a4 , i32 %b4 )
+  %r5  = call i32 @llvm.usub.sat.i32(i32 %a5 , i32 %b5 )
+  %r6  = call i32 @llvm.usub.sat.i32(i32 %a6 , i32 %b6 )
+  %r7  = call i32 @llvm.usub.sat.i32(i32 %a7 , i32 %b7 )
+  %r8  = call i32 @llvm.usub.sat.i32(i32 %a8 , i32 %b8 )
+  %r9  = call i32 @llvm.usub.sat.i32(i32 %a9 , i32 %b9 )
+  %r10 = call i32 @llvm.usub.sat.i32(i32 %a10, i32 %b10)
+  %r11 = call i32 @llvm.usub.sat.i32(i32 %a11, i32 %b11)
+  %r12 = call i32 @llvm.usub.sat.i32(i32 %a12, i32 %b12)
+  %r13 = call i32 @llvm.usub.sat.i32(i32 %a13, i32 %b13)
+  %r14 = call i32 @llvm.usub.sat.i32(i32 %a14, i32 %b14)
+  %r15 = call i32 @llvm.usub.sat.i32(i32 %a15, i32 %b15)
+  store i32 %r0 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 0 ), align 4
+  store i32 %r1 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 1 ), align 4
+  store i32 %r2 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 2 ), align 4
+  store i32 %r3 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 3 ), align 4
+  store i32 %r4 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 4 ), align 4
+  store i32 %r5 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 5 ), align 4
+  store i32 %r6 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 6 ), align 4
+  store i32 %r7 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 7 ), align 4
+  store i32 %r8 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 8 ), align 4
+  store i32 %r9 , i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 9 ), align 4
+  store i32 %r10, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 10), align 4
+  store i32 %r11, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 11), align 4
+  store i32 %r12, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 12), align 4
+  store i32 %r13, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 13), align 4
+  store i32 %r14, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 14), align 4
+  store i32 %r15, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 15), align 4
+  ret void
+}
+
+define void @sub_v32i16() {
+; SSE-LABEL: @sub_v32i16(
+; SSE-NEXT:    [[TMP1:%.*]] = load <8 x i16>, <8 x i16>* bitcast ([32 x i16]* @a16 to <8 x i16>*), align 2
+; SSE-NEXT:    [[TMP2:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 8) to <8 x i16>*), align 2
+; SSE-NEXT:    [[TMP3:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 16) to <8 x i16>*), align 2
+; SSE-NEXT:    [[TMP4:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 24) to <8 x i16>*), align 2
+; SSE-NEXT:    [[TMP5:%.*]] = load <8 x i16>, <8 x i16>* bitcast ([32 x i16]* @b16 to <8 x i16>*), align 2
+; SSE-NEXT:    [[TMP6:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 8) to <8 x i16>*), align 2
+; SSE-NEXT:    [[TMP7:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 16) to <8 x i16>*), align 2
+; SSE-NEXT:    [[TMP8:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 24) to <8 x i16>*), align 2
+; SSE-NEXT:    [[TMP9:%.*]] = call <8 x i16> @llvm.usub.sat.v8i16(<8 x i16> [[TMP1]], <8 x i16> [[TMP5]])
+; SSE-NEXT:    [[TMP10:%.*]] = call <8 x i16> @llvm.usub.sat.v8i16(<8 x i16> [[TMP2]], <8 x i16> [[TMP6]])
+; SSE-NEXT:    [[TMP11:%.*]] = call <8 x i16> @llvm.usub.sat.v8i16(<8 x i16> [[TMP3]], <8 x i16> [[TMP7]])
+; SSE-NEXT:    [[TMP12:%.*]] = call <8 x i16> @llvm.usub.sat.v8i16(<8 x i16> [[TMP4]], <8 x i16> [[TMP8]])
+; SSE-NEXT:    store <8 x i16> [[TMP9]], <8 x i16>* bitcast ([32 x i16]* @c16 to <8 x i16>*), align 2
+; SSE-NEXT:    store <8 x i16> [[TMP10]], <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 8) to <8 x i16>*), align 2
+; SSE-NEXT:    store <8 x i16> [[TMP11]], <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 16) to <8 x i16>*), align 2
+; SSE-NEXT:    store <8 x i16> [[TMP12]], <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 24) to <8 x i16>*), align 2
+; SSE-NEXT:    ret void
+;
+; SLM-LABEL: @sub_v32i16(
+; SLM-NEXT:    [[TMP1:%.*]] = load <8 x i16>, <8 x i16>* bitcast ([32 x i16]* @a16 to <8 x i16>*), align 2
+; SLM-NEXT:    [[TMP2:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 8) to <8 x i16>*), align 2
+; SLM-NEXT:    [[TMP3:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 16) to <8 x i16>*), align 2
+; SLM-NEXT:    [[TMP4:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 24) to <8 x i16>*), align 2
+; SLM-NEXT:    [[TMP5:%.*]] = load <8 x i16>, <8 x i16>* bitcast ([32 x i16]* @b16 to <8 x i16>*), align 2
+; SLM-NEXT:    [[TMP6:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 8) to <8 x i16>*), align 2
+; SLM-NEXT:    [[TMP7:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 16) to <8 x i16>*), align 2
+; SLM-NEXT:    [[TMP8:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 24) to <8 x i16>*), align 2
+; SLM-NEXT:    [[TMP9:%.*]] = call <8 x i16> @llvm.usub.sat.v8i16(<8 x i16> [[TMP1]], <8 x i16> [[TMP5]])
+; SLM-NEXT:    [[TMP10:%.*]] = call <8 x i16> @llvm.usub.sat.v8i16(<8 x i16> [[TMP2]], <8 x i16> [[TMP6]])
+; SLM-NEXT:    [[TMP11:%.*]] = call <8 x i16> @llvm.usub.sat.v8i16(<8 x i16> [[TMP3]], <8 x i16> [[TMP7]])
+; SLM-NEXT:    [[TMP12:%.*]] = call <8 x i16> @llvm.usub.sat.v8i16(<8 x i16> [[TMP4]], <8 x i16> [[TMP8]])
+; SLM-NEXT:    store <8 x i16> [[TMP9]], <8 x i16>* bitcast ([32 x i16]* @c16 to <8 x i16>*), align 2
+; SLM-NEXT:    store <8 x i16> [[TMP10]], <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 8) to <8 x i16>*), align 2
+; SLM-NEXT:    store <8 x i16> [[TMP11]], <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 16) to <8 x i16>*), align 2
+; SLM-NEXT:    store <8 x i16> [[TMP12]], <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 24) to <8 x i16>*), align 2
+; SLM-NEXT:    ret void
+;
+; AVX-LABEL: @sub_v32i16(
+; AVX-NEXT:    [[TMP1:%.*]] = load <16 x i16>, <16 x i16>* bitcast ([32 x i16]* @a16 to <16 x i16>*), align 2
+; AVX-NEXT:    [[TMP2:%.*]] = load <16 x i16>, <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 16) to <16 x i16>*), align 2
+; AVX-NEXT:    [[TMP3:%.*]] = load <16 x i16>, <16 x i16>* bitcast ([32 x i16]* @b16 to <16 x i16>*), align 2
+; AVX-NEXT:    [[TMP4:%.*]] = load <16 x i16>, <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 16) to <16 x i16>*), align 2
+; AVX-NEXT:    [[TMP5:%.*]] = call <16 x i16> @llvm.usub.sat.v16i16(<16 x i16> [[TMP1]], <16 x i16> [[TMP3]])
+; AVX-NEXT:    [[TMP6:%.*]] = call <16 x i16> @llvm.usub.sat.v16i16(<16 x i16> [[TMP2]], <16 x i16> [[TMP4]])
+; AVX-NEXT:    store <16 x i16> [[TMP5]], <16 x i16>* bitcast ([32 x i16]* @c16 to <16 x i16>*), align 2
+; AVX-NEXT:    store <16 x i16> [[TMP6]], <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 16) to <16 x i16>*), align 2
+; AVX-NEXT:    ret void
+;
+; AVX512-LABEL: @sub_v32i16(
+; AVX512-NEXT:    [[TMP1:%.*]] = load <16 x i16>, <16 x i16>* bitcast ([32 x i16]* @a16 to <16 x i16>*), align 2
+; AVX512-NEXT:    [[TMP2:%.*]] = load <16 x i16>, <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 16) to <16 x i16>*), align 2
+; AVX512-NEXT:    [[TMP3:%.*]] = load <16 x i16>, <16 x i16>* bitcast ([32 x i16]* @b16 to <16 x i16>*), align 2
+; AVX512-NEXT:    [[TMP4:%.*]] = load <16 x i16>, <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 16) to <16 x i16>*), align 2
+; AVX512-NEXT:    [[TMP5:%.*]] = call <16 x i16> @llvm.usub.sat.v16i16(<16 x i16> [[TMP1]], <16 x i16> [[TMP3]])
+; AVX512-NEXT:    [[TMP6:%.*]] = call <16 x i16> @llvm.usub.sat.v16i16(<16 x i16> [[TMP2]], <16 x i16> [[TMP4]])
+; AVX512-NEXT:    store <16 x i16> [[TMP5]], <16 x i16>* bitcast ([32 x i16]* @c16 to <16 x i16>*), align 2
+; AVX512-NEXT:    store <16 x i16> [[TMP6]], <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 16) to <16 x i16>*), align 2
+; AVX512-NEXT:    ret void
+;
+  %a0  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 0 ), align 2
+  %a1  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 1 ), align 2
+  %a2  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 2 ), align 2
+  %a3  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 3 ), align 2
+  %a4  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 4 ), align 2
+  %a5  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 5 ), align 2
+  %a6  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 6 ), align 2
+  %a7  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 7 ), align 2
+  %a8  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 8 ), align 2
+  %a9  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 9 ), align 2
+  %a10 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 10), align 2
+  %a11 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 11), align 2
+  %a12 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 12), align 2
+  %a13 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 13), align 2
+  %a14 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 14), align 2
+  %a15 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 15), align 2
+  %a16 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 16), align 2
+  %a17 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 17), align 2
+  %a18 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 18), align 2
+  %a19 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 19), align 2
+  %a20 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 20), align 2
+  %a21 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 21), align 2
+  %a22 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 22), align 2
+  %a23 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 23), align 2
+  %a24 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 24), align 2
+  %a25 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 25), align 2
+  %a26 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 26), align 2
+  %a27 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 27), align 2
+  %a28 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 28), align 2
+  %a29 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 29), align 2
+  %a30 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 30), align 2
+  %a31 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 31), align 2
+  %b0  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 0 ), align 2
+  %b1  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 1 ), align 2
+  %b2  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 2 ), align 2
+  %b3  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 3 ), align 2
+  %b4  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 4 ), align 2
+  %b5  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 5 ), align 2
+  %b6  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 6 ), align 2
+  %b7  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 7 ), align 2
+  %b8  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 8 ), align 2
+  %b9  = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 9 ), align 2
+  %b10 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 10), align 2
+  %b11 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 11), align 2
+  %b12 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 12), align 2
+  %b13 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 13), align 2
+  %b14 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 14), align 2
+  %b15 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 15), align 2
+  %b16 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 16), align 2
+  %b17 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 17), align 2
+  %b18 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 18), align 2
+  %b19 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 19), align 2
+  %b20 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 20), align 2
+  %b21 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 21), align 2
+  %b22 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 22), align 2
+  %b23 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 23), align 2
+  %b24 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 24), align 2
+  %b25 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 25), align 2
+  %b26 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 26), align 2
+  %b27 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 27), align 2
+  %b28 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 28), align 2
+  %b29 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 29), align 2
+  %b30 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 30), align 2
+  %b31 = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 31), align 2
+  %r0  = call i16 @llvm.usub.sat.i16(i16 %a0 , i16 %b0 )
+  %r1  = call i16 @llvm.usub.sat.i16(i16 %a1 , i16 %b1 )
+  %r2  = call i16 @llvm.usub.sat.i16(i16 %a2 , i16 %b2 )
+  %r3  = call i16 @llvm.usub.sat.i16(i16 %a3 , i16 %b3 )
+  %r4  = call i16 @llvm.usub.sat.i16(i16 %a4 , i16 %b4 )
+  %r5  = call i16 @llvm.usub.sat.i16(i16 %a5 , i16 %b5 )
+  %r6  = call i16 @llvm.usub.sat.i16(i16 %a6 , i16 %b6 )
+  %r7  = call i16 @llvm.usub.sat.i16(i16 %a7 , i16 %b7 )
+  %r8  = call i16 @llvm.usub.sat.i16(i16 %a8 , i16 %b8 )
+  %r9  = call i16 @llvm.usub.sat.i16(i16 %a9 , i16 %b9 )
+  %r10 = call i16 @llvm.usub.sat.i16(i16 %a10, i16 %b10)
+  %r11 = call i16 @llvm.usub.sat.i16(i16 %a11, i16 %b11)
+  %r12 = call i16 @llvm.usub.sat.i16(i16 %a12, i16 %b12)
+  %r13 = call i16 @llvm.usub.sat.i16(i16 %a13, i16 %b13)
+  %r14 = call i16 @llvm.usub.sat.i16(i16 %a14, i16 %b14)
+  %r15 = call i16 @llvm.usub.sat.i16(i16 %a15, i16 %b15)
+  %r16 = call i16 @llvm.usub.sat.i16(i16 %a16, i16 %b16)
+  %r17 = call i16 @llvm.usub.sat.i16(i16 %a17, i16 %b17)
+  %r18 = call i16 @llvm.usub.sat.i16(i16 %a18, i16 %b18)
+  %r19 = call i16 @llvm.usub.sat.i16(i16 %a19, i16 %b19)
+  %r20 = call i16 @llvm.usub.sat.i16(i16 %a20, i16 %b20)
+  %r21 = call i16 @llvm.usub.sat.i16(i16 %a21, i16 %b21)
+  %r22 = call i16 @llvm.usub.sat.i16(i16 %a22, i16 %b22)
+  %r23 = call i16 @llvm.usub.sat.i16(i16 %a23, i16 %b23)
+  %r24 = call i16 @llvm.usub.sat.i16(i16 %a24, i16 %b24)
+  %r25 = call i16 @llvm.usub.sat.i16(i16 %a25, i16 %b25)
+  %r26 = call i16 @llvm.usub.sat.i16(i16 %a26, i16 %b26)
+  %r27 = call i16 @llvm.usub.sat.i16(i16 %a27, i16 %b27)
+  %r28 = call i16 @llvm.usub.sat.i16(i16 %a28, i16 %b28)
+  %r29 = call i16 @llvm.usub.sat.i16(i16 %a29, i16 %b29)
+  %r30 = call i16 @llvm.usub.sat.i16(i16 %a30, i16 %b30)
+  %r31 = call i16 @llvm.usub.sat.i16(i16 %a31, i16 %b31)
+  store i16 %r0 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 0 ), align 2
+  store i16 %r1 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 1 ), align 2
+  store i16 %r2 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 2 ), align 2
+  store i16 %r3 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 3 ), align 2
+  store i16 %r4 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 4 ), align 2
+  store i16 %r5 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 5 ), align 2
+  store i16 %r6 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 6 ), align 2
+  store i16 %r7 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 7 ), align 2
+  store i16 %r8 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 8 ), align 2
+  store i16 %r9 , i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 9 ), align 2
+  store i16 %r10, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 10), align 2
+  store i16 %r11, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 11), align 2
+  store i16 %r12, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 12), align 2
+  store i16 %r13, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 13), align 2
+  store i16 %r14, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 14), align 2
+  store i16 %r15, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 15), align 2
+  store i16 %r16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 16), align 2
+  store i16 %r17, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 17), align 2
+  store i16 %r18, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 18), align 2
+  store i16 %r19, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 19), align 2
+  store i16 %r20, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 20), align 2
+  store i16 %r21, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 21), align 2
+  store i16 %r22, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 22), align 2
+  store i16 %r23, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 23), align 2
+  store i16 %r24, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 24), align 2
+  store i16 %r25, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 25), align 2
+  store i16 %r26, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 26), align 2
+  store i16 %r27, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 27), align 2
+  store i16 %r28, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 28), align 2
+  store i16 %r29, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 29), align 2
+  store i16 %r30, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 30), align 2
+  store i16 %r31, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 31), align 2
+  ret void
+}
+
+define void @sub_v64i8() {
+; CHECK-LABEL: @sub_v64i8(
+; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i8>, <16 x i8>* bitcast ([64 x i8]* @a8 to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 16) to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP3:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 32) to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP4:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 48) to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP5:%.*]] = load <16 x i8>, <16 x i8>* bitcast ([64 x i8]* @b8 to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP6:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 16) to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP7:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 32) to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP8:%.*]] = load <16 x i8>, <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 48) to <16 x i8>*), align 1
+; CHECK-NEXT:    [[TMP9:%.*]] = call <16 x i8> @llvm.usub.sat.v16i8(<16 x i8> [[TMP1]], <16 x i8> [[TMP5]])
+; CHECK-NEXT:    [[TMP10:%.*]] = call <16 x i8> @llvm.usub.sat.v16i8(<16 x i8> [[TMP2]], <16 x i8> [[TMP6]])
+; CHECK-NEXT:    [[TMP11:%.*]] = call <16 x i8> @llvm.usub.sat.v16i8(<16 x i8> [[TMP3]], <16 x i8> [[TMP7]])
+; CHECK-NEXT:    [[TMP12:%.*]] = call <16 x i8> @llvm.usub.sat.v16i8(<16 x i8> [[TMP4]], <16 x i8> [[TMP8]])
+; CHECK-NEXT:    store <16 x i8> [[TMP9]], <16 x i8>* bitcast ([64 x i8]* @c8 to <16 x i8>*), align 1
+; CHECK-NEXT:    store <16 x i8> [[TMP10]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 16) to <16 x i8>*), align 1
+; CHECK-NEXT:    store <16 x i8> [[TMP11]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 32) to <16 x i8>*), align 1
+; CHECK-NEXT:    store <16 x i8> [[TMP12]], <16 x i8>* bitcast (i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 48) to <16 x i8>*), align 1
+; CHECK-NEXT:    ret void
+;
+  %a0  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 0 ), align 1
+  %a1  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 1 ), align 1
+  %a2  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 2 ), align 1
+  %a3  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 3 ), align 1
+  %a4  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 4 ), align 1
+  %a5  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 5 ), align 1
+  %a6  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 6 ), align 1
+  %a7  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 7 ), align 1
+  %a8  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 8 ), align 1
+  %a9  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 9 ), align 1
+  %a10 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 10), align 1
+  %a11 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 11), align 1
+  %a12 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 12), align 1
+  %a13 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 13), align 1
+  %a14 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 14), align 1
+  %a15 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 15), align 1
+  %a16 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 16), align 1
+  %a17 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 17), align 1
+  %a18 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 18), align 1
+  %a19 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 19), align 1
+  %a20 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 20), align 1
+  %a21 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 21), align 1
+  %a22 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 22), align 1
+  %a23 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 23), align 1
+  %a24 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 24), align 1
+  %a25 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 25), align 1
+  %a26 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 26), align 1
+  %a27 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 27), align 1
+  %a28 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 28), align 1
+  %a29 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 29), align 1
+  %a30 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 30), align 1
+  %a31 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 31), align 1
+  %a32 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 32), align 1
+  %a33 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 33), align 1
+  %a34 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 34), align 1
+  %a35 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 35), align 1
+  %a36 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 36), align 1
+  %a37 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 37), align 1
+  %a38 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 38), align 1
+  %a39 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 39), align 1
+  %a40 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 40), align 1
+  %a41 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 41), align 1
+  %a42 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 42), align 1
+  %a43 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 43), align 1
+  %a44 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 44), align 1
+  %a45 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 45), align 1
+  %a46 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 46), align 1
+  %a47 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 47), align 1
+  %a48 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 48), align 1
+  %a49 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 49), align 1
+  %a50 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 50), align 1
+  %a51 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 51), align 1
+  %a52 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 52), align 1
+  %a53 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 53), align 1
+  %a54 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 54), align 1
+  %a55 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 55), align 1
+  %a56 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 56), align 1
+  %a57 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 57), align 1
+  %a58 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 58), align 1
+  %a59 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 59), align 1
+  %a60 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 60), align 1
+  %a61 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 61), align 1
+  %a62 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 62), align 1
+  %a63 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @a8, i32 0, i64 63), align 1
+  %b0  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 0 ), align 1
+  %b1  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 1 ), align 1
+  %b2  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 2 ), align 1
+  %b3  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 3 ), align 1
+  %b4  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 4 ), align 1
+  %b5  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 5 ), align 1
+  %b6  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 6 ), align 1
+  %b7  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 7 ), align 1
+  %b8  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 8 ), align 1
+  %b9  = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 9 ), align 1
+  %b10 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 10), align 1
+  %b11 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 11), align 1
+  %b12 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 12), align 1
+  %b13 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 13), align 1
+  %b14 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 14), align 1
+  %b15 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 15), align 1
+  %b16 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 16), align 1
+  %b17 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 17), align 1
+  %b18 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 18), align 1
+  %b19 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 19), align 1
+  %b20 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 20), align 1
+  %b21 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 21), align 1
+  %b22 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 22), align 1
+  %b23 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 23), align 1
+  %b24 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 24), align 1
+  %b25 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 25), align 1
+  %b26 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 26), align 1
+  %b27 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 27), align 1
+  %b28 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 28), align 1
+  %b29 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 29), align 1
+  %b30 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 30), align 1
+  %b31 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 31), align 1
+  %b32 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 32), align 1
+  %b33 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 33), align 1
+  %b34 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 34), align 1
+  %b35 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 35), align 1
+  %b36 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 36), align 1
+  %b37 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 37), align 1
+  %b38 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 38), align 1
+  %b39 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 39), align 1
+  %b40 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 40), align 1
+  %b41 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 41), align 1
+  %b42 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 42), align 1
+  %b43 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 43), align 1
+  %b44 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 44), align 1
+  %b45 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 45), align 1
+  %b46 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 46), align 1
+  %b47 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 47), align 1
+  %b48 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 48), align 1
+  %b49 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 49), align 1
+  %b50 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 50), align 1
+  %b51 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 51), align 1
+  %b52 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 52), align 1
+  %b53 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 53), align 1
+  %b54 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 54), align 1
+  %b55 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 55), align 1
+  %b56 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 56), align 1
+  %b57 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 57), align 1
+  %b58 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 58), align 1
+  %b59 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 59), align 1
+  %b60 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 60), align 1
+  %b61 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 61), align 1
+  %b62 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 62), align 1
+  %b63 = load i8, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @b8, i32 0, i64 63), align 1
+  %r0  = call i8 @llvm.usub.sat.i8(i8 %a0 , i8 %b0 )
+  %r1  = call i8 @llvm.usub.sat.i8(i8 %a1 , i8 %b1 )
+  %r2  = call i8 @llvm.usub.sat.i8(i8 %a2 , i8 %b2 )
+  %r3  = call i8 @llvm.usub.sat.i8(i8 %a3 , i8 %b3 )
+  %r4  = call i8 @llvm.usub.sat.i8(i8 %a4 , i8 %b4 )
+  %r5  = call i8 @llvm.usub.sat.i8(i8 %a5 , i8 %b5 )
+  %r6  = call i8 @llvm.usub.sat.i8(i8 %a6 , i8 %b6 )
+  %r7  = call i8 @llvm.usub.sat.i8(i8 %a7 , i8 %b7 )
+  %r8  = call i8 @llvm.usub.sat.i8(i8 %a8 , i8 %b8 )
+  %r9  = call i8 @llvm.usub.sat.i8(i8 %a9 , i8 %b9 )
+  %r10 = call i8 @llvm.usub.sat.i8(i8 %a10, i8 %b10)
+  %r11 = call i8 @llvm.usub.sat.i8(i8 %a11, i8 %b11)
+  %r12 = call i8 @llvm.usub.sat.i8(i8 %a12, i8 %b12)
+  %r13 = call i8 @llvm.usub.sat.i8(i8 %a13, i8 %b13)
+  %r14 = call i8 @llvm.usub.sat.i8(i8 %a14, i8 %b14)
+  %r15 = call i8 @llvm.usub.sat.i8(i8 %a15, i8 %b15)
+  %r16 = call i8 @llvm.usub.sat.i8(i8 %a16, i8 %b16)
+  %r17 = call i8 @llvm.usub.sat.i8(i8 %a17, i8 %b17)
+  %r18 = call i8 @llvm.usub.sat.i8(i8 %a18, i8 %b18)
+  %r19 = call i8 @llvm.usub.sat.i8(i8 %a19, i8 %b19)
+  %r20 = call i8 @llvm.usub.sat.i8(i8 %a20, i8 %b20)
+  %r21 = call i8 @llvm.usub.sat.i8(i8 %a21, i8 %b21)
+  %r22 = call i8 @llvm.usub.sat.i8(i8 %a22, i8 %b22)
+  %r23 = call i8 @llvm.usub.sat.i8(i8 %a23, i8 %b23)
+  %r24 = call i8 @llvm.usub.sat.i8(i8 %a24, i8 %b24)
+  %r25 = call i8 @llvm.usub.sat.i8(i8 %a25, i8 %b25)
+  %r26 = call i8 @llvm.usub.sat.i8(i8 %a26, i8 %b26)
+  %r27 = call i8 @llvm.usub.sat.i8(i8 %a27, i8 %b27)
+  %r28 = call i8 @llvm.usub.sat.i8(i8 %a28, i8 %b28)
+  %r29 = call i8 @llvm.usub.sat.i8(i8 %a29, i8 %b29)
+  %r30 = call i8 @llvm.usub.sat.i8(i8 %a30, i8 %b30)
+  %r31 = call i8 @llvm.usub.sat.i8(i8 %a31, i8 %b31)
+  %r32 = call i8 @llvm.usub.sat.i8(i8 %a32, i8 %b32)
+  %r33 = call i8 @llvm.usub.sat.i8(i8 %a33, i8 %b33)
+  %r34 = call i8 @llvm.usub.sat.i8(i8 %a34, i8 %b34)
+  %r35 = call i8 @llvm.usub.sat.i8(i8 %a35, i8 %b35)
+  %r36 = call i8 @llvm.usub.sat.i8(i8 %a36, i8 %b36)
+  %r37 = call i8 @llvm.usub.sat.i8(i8 %a37, i8 %b37)
+  %r38 = call i8 @llvm.usub.sat.i8(i8 %a38, i8 %b38)
+  %r39 = call i8 @llvm.usub.sat.i8(i8 %a39, i8 %b39)
+  %r40 = call i8 @llvm.usub.sat.i8(i8 %a40, i8 %b40)
+  %r41 = call i8 @llvm.usub.sat.i8(i8 %a41, i8 %b41)
+  %r42 = call i8 @llvm.usub.sat.i8(i8 %a42, i8 %b42)
+  %r43 = call i8 @llvm.usub.sat.i8(i8 %a43, i8 %b43)
+  %r44 = call i8 @llvm.usub.sat.i8(i8 %a44, i8 %b44)
+  %r45 = call i8 @llvm.usub.sat.i8(i8 %a45, i8 %b45)
+  %r46 = call i8 @llvm.usub.sat.i8(i8 %a46, i8 %b46)
+  %r47 = call i8 @llvm.usub.sat.i8(i8 %a47, i8 %b47)
+  %r48 = call i8 @llvm.usub.sat.i8(i8 %a48, i8 %b48)
+  %r49 = call i8 @llvm.usub.sat.i8(i8 %a49, i8 %b49)
+  %r50 = call i8 @llvm.usub.sat.i8(i8 %a50, i8 %b50)
+  %r51 = call i8 @llvm.usub.sat.i8(i8 %a51, i8 %b51)
+  %r52 = call i8 @llvm.usub.sat.i8(i8 %a52, i8 %b52)
+  %r53 = call i8 @llvm.usub.sat.i8(i8 %a53, i8 %b53)
+  %r54 = call i8 @llvm.usub.sat.i8(i8 %a54, i8 %b54)
+  %r55 = call i8 @llvm.usub.sat.i8(i8 %a55, i8 %b55)
+  %r56 = call i8 @llvm.usub.sat.i8(i8 %a56, i8 %b56)
+  %r57 = call i8 @llvm.usub.sat.i8(i8 %a57, i8 %b57)
+  %r58 = call i8 @llvm.usub.sat.i8(i8 %a58, i8 %b58)
+  %r59 = call i8 @llvm.usub.sat.i8(i8 %a59, i8 %b59)
+  %r60 = call i8 @llvm.usub.sat.i8(i8 %a60, i8 %b60)
+  %r61 = call i8 @llvm.usub.sat.i8(i8 %a61, i8 %b61)
+  %r62 = call i8 @llvm.usub.sat.i8(i8 %a62, i8 %b62)
+  %r63 = call i8 @llvm.usub.sat.i8(i8 %a63, i8 %b63)
+  store i8 %r0 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 0 ), align 1
+  store i8 %r1 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 1 ), align 1
+  store i8 %r2 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 2 ), align 1
+  store i8 %r3 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 3 ), align 1
+  store i8 %r4 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 4 ), align 1
+  store i8 %r5 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 5 ), align 1
+  store i8 %r6 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 6 ), align 1
+  store i8 %r7 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 7 ), align 1
+  store i8 %r8 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 8 ), align 1
+  store i8 %r9 , i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 9 ), align 1
+  store i8 %r10, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 10), align 1
+  store i8 %r11, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 11), align 1
+  store i8 %r12, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 12), align 1
+  store i8 %r13, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 13), align 1
+  store i8 %r14, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 14), align 1
+  store i8 %r15, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 15), align 1
+  store i8 %r16, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 16), align 1
+  store i8 %r17, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 17), align 1
+  store i8 %r18, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 18), align 1
+  store i8 %r19, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 19), align 1
+  store i8 %r20, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 20), align 1
+  store i8 %r21, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 21), align 1
+  store i8 %r22, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 22), align 1
+  store i8 %r23, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 23), align 1
+  store i8 %r24, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 24), align 1
+  store i8 %r25, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 25), align 1
+  store i8 %r26, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 26), align 1
+  store i8 %r27, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 27), align 1
+  store i8 %r28, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 28), align 1
+  store i8 %r29, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 29), align 1
+  store i8 %r30, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 30), align 1
+  store i8 %r31, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 31), align 1
+  store i8 %r32, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 32), align 1
+  store i8 %r33, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 33), align 1
+  store i8 %r34, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 34), align 1
+  store i8 %r35, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 35), align 1
+  store i8 %r36, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 36), align 1
+  store i8 %r37, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 37), align 1
+  store i8 %r38, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 38), align 1
+  store i8 %r39, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 39), align 1
+  store i8 %r40, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 40), align 1
+  store i8 %r41, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 41), align 1
+  store i8 %r42, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 42), align 1
+  store i8 %r43, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 43), align 1
+  store i8 %r44, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 44), align 1
+  store i8 %r45, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 45), align 1
+  store i8 %r46, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 46), align 1
+  store i8 %r47, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 47), align 1
+  store i8 %r48, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 48), align 1
+  store i8 %r49, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 49), align 1
+  store i8 %r50, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 50), align 1
+  store i8 %r51, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 51), align 1
+  store i8 %r52, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 52), align 1
+  store i8 %r53, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 53), align 1
+  store i8 %r54, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 54), align 1
+  store i8 %r55, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 55), align 1
+  store i8 %r56, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 56), align 1
+  store i8 %r57, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 57), align 1
+  store i8 %r58, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 58), align 1
+  store i8 %r59, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 59), align 1
+  store i8 %r60, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 60), align 1
+  store i8 %r61, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 61), align 1
+  store i8 %r62, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 62), align 1
+  store i8 %r63, i8* getelementptr inbounds ([64 x i8], [64 x i8]* @c8, i32 0, i64 63), align 1
+  ret void
+}
diff --git a/test/Transforms/SLPVectorizer/X86/atomics.ll b/test/Transforms/SLPVectorizer/X86/atomics.ll
index a48b793922f4b599f021056c3e2304404dcd97e7..c7f05491a18597221ff74b2419a960a61e7d1fe0 100644
--- a/test/Transforms/SLPVectorizer/X86/atomics.ll
+++ b/test/Transforms/SLPVectorizer/X86/atomics.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -basicaa -slp-vectorizer -S |FileCheck %s
 target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
 
@@ -7,16 +8,19 @@ target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
 ; The SLPVectorizer should not vectorize atomic stores and it should not
 ; schedule regular stores around atomic stores.
 
-; CHECK-LABEL: test
-; CHECK: store i32
-; CHECK: store atomic i32
-; CHECK: store i32
-; CHECK: store atomic i32
-; CHECK: store i32
-; CHECK: store atomic i32
-; CHECK: store i32
-; CHECK: store atomic i32
 define void @test() {
+; CHECK-LABEL: @test(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    store i32 0, i32* getelementptr inbounds ([4 x i32], [4 x i32]* @a, i64 0, i64 0), align 16
+; CHECK-NEXT:    store atomic i32 0, i32* getelementptr inbounds ([4 x i32], [4 x i32]* @x, i64 0, i64 0) release, align 16
+; CHECK-NEXT:    store i32 0, i32* getelementptr inbounds ([4 x i32], [4 x i32]* @a, i64 0, i64 1), align 4
+; CHECK-NEXT:    store atomic i32 1, i32* getelementptr inbounds ([4 x i32], [4 x i32]* @x, i64 0, i64 1) release, align 4
+; CHECK-NEXT:    store i32 0, i32* getelementptr inbounds ([4 x i32], [4 x i32]* @a, i64 0, i64 2), align 8
+; CHECK-NEXT:    store atomic i32 2, i32* getelementptr inbounds ([4 x i32], [4 x i32]* @x, i64 0, i64 2) release, align 8
+; CHECK-NEXT:    store i32 0, i32* getelementptr inbounds ([4 x i32], [4 x i32]* @a, i64 0, i64 3), align 4
+; CHECK-NEXT:    store atomic i32 3, i32* getelementptr inbounds ([4 x i32], [4 x i32]* @x, i64 0, i64 3) release, align 4
+; CHECK-NEXT:    ret void
+;
 entry:
   store i32 0, i32* getelementptr inbounds ([4 x i32], [4 x i32]* @a, i64 0, i64 0), align 16
   store atomic i32 0, i32* getelementptr inbounds ([4 x i32], [4 x i32]* @x, i64 0, i64 0) release, align 16
diff --git a/test/Transforms/SLPVectorizer/X86/bad_types.ll b/test/Transforms/SLPVectorizer/X86/bad_types.ll
index 98c29068bb96aa88792e309e4558dd805d0bf722..d229acd17894a749efb1924b5b89ebf3a907fc41 100644
--- a/test/Transforms/SLPVectorizer/X86/bad_types.ll
+++ b/test/Transforms/SLPVectorizer/X86/bad_types.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -basicaa -slp-vectorizer -S -mcpu=corei7-avx | FileCheck %s
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
@@ -7,10 +8,17 @@ define void @test1(x86_mmx %a, x86_mmx %b, i64* %ptr) {
 ; Ensure we can handle x86_mmx values which are primitive and can be bitcast
 ; with integer types but can't be put into a vector.
 ;
-; CHECK-LABEL: @test1
-; CHECK:         store i64
-; CHECK:         store i64
-; CHECK:         ret void
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[A_CAST:%.*]] = bitcast x86_mmx [[A:%.*]] to i64
+; CHECK-NEXT:    [[B_CAST:%.*]] = bitcast x86_mmx [[B:%.*]] to i64
+; CHECK-NEXT:    [[A_AND:%.*]] = and i64 [[A_CAST]], 42
+; CHECK-NEXT:    [[B_AND:%.*]] = and i64 [[B_CAST]], 42
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i64, i64* [[PTR:%.*]], i32 1
+; CHECK-NEXT:    store i64 [[A_AND]], i64* [[PTR]]
+; CHECK-NEXT:    store i64 [[B_AND]], i64* [[GEP]]
+; CHECK-NEXT:    ret void
+;
 entry:
   %a.cast = bitcast x86_mmx %a to i64
   %b.cast = bitcast x86_mmx %b to i64
@@ -26,10 +34,21 @@ define void @test2(x86_mmx %a, x86_mmx %b) {
 ; Same as @test1 but using phi-input vectorization instead of store
 ; vectorization.
 ;
-; CHECK-LABEL: @test2
-; CHECK:         and i64
-; CHECK:         and i64
-; CHECK:         ret void
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 undef, label [[IF_THEN:%.*]], label [[EXIT:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[A_CAST:%.*]] = bitcast x86_mmx [[A:%.*]] to i64
+; CHECK-NEXT:    [[B_CAST:%.*]] = bitcast x86_mmx [[B:%.*]] to i64
+; CHECK-NEXT:    [[A_AND:%.*]] = and i64 [[A_CAST]], 42
+; CHECK-NEXT:    [[B_AND:%.*]] = and i64 [[B_CAST]], 42
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[A_PHI:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[A_AND]], [[IF_THEN]] ]
+; CHECK-NEXT:    [[B_PHI:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[B_AND]], [[IF_THEN]] ]
+; CHECK-NEXT:    tail call void @f(i64 [[A_PHI]], i64 [[B_PHI]])
+; CHECK-NEXT:    ret void
+;
 entry:
   br i1 undef, label %if.then, label %exit
 
@@ -50,9 +69,26 @@ exit:
 define i8 @test3(i8 *%addr) {
 ; Check that we do not vectorize types that are padded to a bigger ones.
 ;
-; CHECK-LABEL: @test3
-; CHECK-NOT:   <4 x i2>
-; CHECK:       ret i8
+; CHECK-LABEL: @test3(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[A:%.*]] = bitcast i8* [[ADDR:%.*]] to i2*
+; CHECK-NEXT:    [[A0:%.*]] = getelementptr inbounds i2, i2* [[A]], i64 0
+; CHECK-NEXT:    [[A1:%.*]] = getelementptr inbounds i2, i2* [[A]], i64 1
+; CHECK-NEXT:    [[A2:%.*]] = getelementptr inbounds i2, i2* [[A]], i64 2
+; CHECK-NEXT:    [[A3:%.*]] = getelementptr inbounds i2, i2* [[A]], i64 3
+; CHECK-NEXT:    [[L0:%.*]] = load i2, i2* [[A0]], align 1
+; CHECK-NEXT:    [[L1:%.*]] = load i2, i2* [[A1]], align 1
+; CHECK-NEXT:    [[L2:%.*]] = load i2, i2* [[A2]], align 1
+; CHECK-NEXT:    [[L3:%.*]] = load i2, i2* [[A3]], align 1
+; CHECK-NEXT:    br label [[BB1:%.*]]
+; CHECK:       bb1:
+; CHECK-NEXT:    [[P0:%.*]] = phi i2 [ [[L0]], [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[P1:%.*]] = phi i2 [ [[L1]], [[ENTRY]] ]
+; CHECK-NEXT:    [[P2:%.*]] = phi i2 [ [[L2]], [[ENTRY]] ]
+; CHECK-NEXT:    [[P3:%.*]] = phi i2 [ [[L3]], [[ENTRY]] ]
+; CHECK-NEXT:    [[R:%.*]] = zext i2 [[P2]] to i8
+; CHECK-NEXT:    ret i8 [[R]]
+;
 entry:
   %a = bitcast i8* %addr to i2*
   %a0 = getelementptr inbounds i2, i2* %a, i64 0
diff --git a/test/Transforms/SLPVectorizer/X86/barriercall.ll b/test/Transforms/SLPVectorizer/X86/barriercall.ll
index 382a43fddf588ee4de7628628e241cc0da18e768..7378b8bcb1c93bd816b4e32912161edf3db452f2 100644
--- a/test/Transforms/SLPVectorizer/X86/barriercall.ll
+++ b/test/Transforms/SLPVectorizer/X86/barriercall.ll
@@ -1,12 +1,25 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -basicaa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.8.0"
 
-;CHECK-LABEL: @foo(
-;CHECK: store <4 x i32>
-;CHECK: ret
 define i32 @foo(i32* nocapture %A, i32 %n) {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CALL:%.*]] = tail call i32 (...) @bar()
+; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <4 x i32> undef, i32 [[N:%.*]], i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x i32> [[TMP0]], i32 [[N]], i32 1
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <4 x i32> [[TMP1]], i32 [[N]], i32 2
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <4 x i32> [[TMP2]], i32 [[N]], i32 3
+; CHECK-NEXT:    [[TMP4:%.*]] = mul nsw <4 x i32> [[TMP3]], <i32 5, i32 9, i32 3, i32 10>
+; CHECK-NEXT:    [[TMP5:%.*]] = shl <4 x i32> [[TMP3]], <i32 5, i32 9, i32 3, i32 10>
+; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <4 x i32> [[TMP4]], <4 x i32> [[TMP5]], <4 x i32> <i32 0, i32 1, i32 6, i32 3>
+; CHECK-NEXT:    [[TMP7:%.*]] = add nsw <4 x i32> <i32 9, i32 9, i32 9, i32 9>, [[TMP6]]
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast i32* [[A:%.*]] to <4 x i32>*
+; CHECK-NEXT:    store <4 x i32> [[TMP7]], <4 x i32>* [[TMP8]], align 4
+; CHECK-NEXT:    ret i32 undef
+;
 entry:
   %call = tail call i32 (...) @bar() #2
   %mul = mul nsw i32 %n, 5
diff --git a/test/Transforms/SLPVectorizer/X86/call.ll b/test/Transforms/SLPVectorizer/X86/call.ll
index 8397d348483cf4c908178c64f12a4a9f139b23a5..c93397c0337509d1b64655abdc3626d80e4c2310 100644
--- a/test/Transforms/SLPVectorizer/X86/call.ll
+++ b/test/Transforms/SLPVectorizer/X86/call.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -basicaa -slp-vectorizer -slp-threshold=-999 -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
@@ -13,10 +14,10 @@ declare i64 @round(i64)
 
 define void @sin_libm(double* %a, double* %b) {
 ; CHECK-LABEL: @sin_libm(
-; CHECK-NEXT:    [[TMP1:%.*]] = bitcast double* %a to <2 x double>*
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast double* [[A:%.*]] to <2 x double>*
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x double>, <2 x double>* [[TMP1]], align 8
 ; CHECK-NEXT:    [[TMP3:%.*]] = call <2 x double> @llvm.sin.v2f64(<2 x double> [[TMP2]])
-; CHECK-NEXT:    [[TMP4:%.*]] = bitcast double* %b to <2 x double>*
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast double* [[B:%.*]] to <2 x double>*
 ; CHECK-NEXT:    store <2 x double> [[TMP3]], <2 x double>* [[TMP4]], align 8
 ; CHECK-NEXT:    ret void
 ;
@@ -33,10 +34,10 @@ define void @sin_libm(double* %a, double* %b) {
 
 define void @cos_libm(double* %a, double* %b) {
 ; CHECK-LABEL: @cos_libm(
-; CHECK-NEXT:    [[TMP1:%.*]] = bitcast double* %a to <2 x double>*
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast double* [[A:%.*]] to <2 x double>*
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x double>, <2 x double>* [[TMP1]], align 8
 ; CHECK-NEXT:    [[TMP3:%.*]] = call <2 x double> @llvm.cos.v2f64(<2 x double> [[TMP2]])
-; CHECK-NEXT:    [[TMP4:%.*]] = bitcast double* %b to <2 x double>*
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast double* [[B:%.*]] to <2 x double>*
 ; CHECK-NEXT:    store <2 x double> [[TMP3]], <2 x double>* [[TMP4]], align 8
 ; CHECK-NEXT:    ret void
 ;
@@ -53,10 +54,10 @@ define void @cos_libm(double* %a, double* %b) {
 
 define void @pow_libm(double* %a, double* %b) {
 ; CHECK-LABEL: @pow_libm(
-; CHECK-NEXT:    [[TMP1:%.*]] = bitcast double* %a to <2 x double>*
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast double* [[A:%.*]] to <2 x double>*
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x double>, <2 x double>* [[TMP1]], align 8
 ; CHECK-NEXT:    [[TMP3:%.*]] = call <2 x double> @llvm.pow.v2f64(<2 x double> [[TMP2]], <2 x double> [[TMP2]])
-; CHECK-NEXT:    [[TMP4:%.*]] = bitcast double* %b to <2 x double>*
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast double* [[B:%.*]] to <2 x double>*
 ; CHECK-NEXT:    store <2 x double> [[TMP3]], <2 x double>* [[TMP4]], align 8
 ; CHECK-NEXT:    ret void
 ;
@@ -73,10 +74,10 @@ define void @pow_libm(double* %a, double* %b) {
 
 define void @exp_libm(double* %a, double* %b) {
 ; CHECK-LABEL: @exp_libm(
-; CHECK-NEXT:    [[TMP1:%.*]] = bitcast double* %a to <2 x double>*
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast double* [[A:%.*]] to <2 x double>*
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x double>, <2 x double>* [[TMP1]], align 8
 ; CHECK-NEXT:    [[TMP3:%.*]] = call <2 x double> @llvm.exp2.v2f64(<2 x double> [[TMP2]])
-; CHECK-NEXT:    [[TMP4:%.*]] = bitcast double* %b to <2 x double>*
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast double* [[B:%.*]] to <2 x double>*
 ; CHECK-NEXT:    store <2 x double> [[TMP3]], <2 x double>* [[TMP4]], align 8
 ; CHECK-NEXT:    ret void
 ;
@@ -91,15 +92,15 @@ define void @exp_libm(double* %a, double* %b) {
   ret void
 }
 
-; No fast-math-flags are required to convert sqrt library calls to an intrinsic. 
+; No fast-math-flags are required to convert sqrt library calls to an intrinsic.
 ; We just need to know that errno is not set (readnone).
 
 define void @sqrt_libm_no_errno(double* %a, double* %b) {
 ; CHECK-LABEL: @sqrt_libm_no_errno(
-; CHECK-NEXT:    [[TMP1:%.*]] = bitcast double* %a to <2 x double>*
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast double* [[A:%.*]] to <2 x double>*
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x double>, <2 x double>* [[TMP1]], align 8
 ; CHECK-NEXT:    [[TMP3:%.*]] = call <2 x double> @llvm.sqrt.v2f64(<2 x double> [[TMP2]])
-; CHECK-NEXT:    [[TMP4:%.*]] = bitcast double* %b to <2 x double>*
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast double* [[B:%.*]] to <2 x double>*
 ; CHECK-NEXT:    store <2 x double> [[TMP3]], <2 x double>* [[TMP4]], align 8
 ; CHECK-NEXT:    ret void
 ;
@@ -116,18 +117,18 @@ define void @sqrt_libm_no_errno(double* %a, double* %b) {
 
 ; The sqrt intrinsic does not set errno, but a non-constant sqrt call might, so this can't vectorize.
 ; The nnan on the call does not matter because there's no guarantee in the C standard that a negative
-; input would result in a nan output ("On a domain error, the function returns an 
+; input would result in a nan output ("On a domain error, the function returns an
 ; implementation-defined value.")
 
 define void @sqrt_libm_errno(double* %a, double* %b) {
 ; CHECK-LABEL: @sqrt_libm_errno(
-; CHECK-NEXT:    [[A0:%.*]] = load double, double* %a, align 8
-; CHECK-NEXT:    [[IDX1:%.*]] = getelementptr inbounds double, double* %a, i64 1
+; CHECK-NEXT:    [[A0:%.*]] = load double, double* [[A:%.*]], align 8
+; CHECK-NEXT:    [[IDX1:%.*]] = getelementptr inbounds double, double* [[A]], i64 1
 ; CHECK-NEXT:    [[A1:%.*]] = load double, double* [[IDX1]], align 8
 ; CHECK-NEXT:    [[SQRT1:%.*]] = tail call nnan double @sqrt(double [[A0]]) #2
 ; CHECK-NEXT:    [[SQRT2:%.*]] = tail call nnan double @sqrt(double [[A1]]) #2
-; CHECK-NEXT:    store double [[SQRT1]], double* %b, align 8
-; CHECK-NEXT:    [[IDX2:%.*]] = getelementptr inbounds double, double* %b, i64 1
+; CHECK-NEXT:    store double [[SQRT1]], double* [[B:%.*]], align 8
+; CHECK-NEXT:    [[IDX2:%.*]] = getelementptr inbounds double, double* [[B]], i64 1
 ; CHECK-NEXT:    store double [[SQRT2]], double* [[IDX2]], align 8
 ; CHECK-NEXT:    ret void
 ;
@@ -145,13 +146,13 @@ define void @sqrt_libm_errno(double* %a, double* %b) {
 ; Negative test case
 define void @round_custom(i64* %a, i64* %b) {
 ; CHECK-LABEL: @round_custom(
-; CHECK-NEXT:    [[A0:%.*]] = load i64, i64* %a, align 8
-; CHECK-NEXT:    [[IDX1:%.*]] = getelementptr inbounds i64, i64* %a, i64 1
+; CHECK-NEXT:    [[A0:%.*]] = load i64, i64* [[A:%.*]], align 8
+; CHECK-NEXT:    [[IDX1:%.*]] = getelementptr inbounds i64, i64* [[A]], i64 1
 ; CHECK-NEXT:    [[A1:%.*]] = load i64, i64* [[IDX1]], align 8
 ; CHECK-NEXT:    [[ROUND1:%.*]] = tail call i64 @round(i64 [[A0]]) #3
 ; CHECK-NEXT:    [[ROUND2:%.*]] = tail call i64 @round(i64 [[A1]]) #3
-; CHECK-NEXT:    store i64 [[ROUND1]], i64* %b, align 8
-; CHECK-NEXT:    [[IDX2:%.*]] = getelementptr inbounds i64, i64* %b, i64 1
+; CHECK-NEXT:    store i64 [[ROUND1]], i64* [[B:%.*]], align 8
+; CHECK-NEXT:    [[IDX2:%.*]] = getelementptr inbounds i64, i64* [[B]], i64 1
 ; CHECK-NEXT:    store i64 [[ROUND2]], i64* [[IDX2]], align 8
 ; CHECK-NEXT:    ret void
 ;
diff --git a/test/Transforms/SLPVectorizer/X86/commutativity.ll b/test/Transforms/SLPVectorizer/X86/commutativity.ll
index 2798ccb15e48ab131c192284023a7f89c0f3bc6f..9af59efd345349b8000390c051e5000a703942ae 100644
--- a/test/Transforms/SLPVectorizer/X86/commutativity.ll
+++ b/test/Transforms/SLPVectorizer/X86/commutativity.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -slp-vectorizer < %s -S | FileCheck %s
 
 ; Verify that the SLP vectorizer is able to figure out that commutativity
@@ -16,9 +17,31 @@ target triple = "x86_64-apple-macosx10.11.0"
 ; Check that we correctly detect a splat/broadcast by leveraging the
 ; commutativity property of `xor`.
 
-; CHECK-LABEL:  @splat
-; CHECK:  store <16 x i8>
 define void @splat(i8 %a, i8 %b, i8 %c) {
+; CHECK-LABEL: @splat(
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <16 x i8> undef, i8 [[C:%.*]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <16 x i8> [[TMP1]], i8 [[C]], i32 1
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <16 x i8> [[TMP2]], i8 [[C]], i32 2
+; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <16 x i8> [[TMP3]], i8 [[C]], i32 3
+; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <16 x i8> [[TMP4]], i8 [[C]], i32 4
+; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <16 x i8> [[TMP5]], i8 [[C]], i32 5
+; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <16 x i8> [[TMP6]], i8 [[C]], i32 6
+; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <16 x i8> [[TMP7]], i8 [[C]], i32 7
+; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <16 x i8> [[TMP8]], i8 [[C]], i32 8
+; CHECK-NEXT:    [[TMP10:%.*]] = insertelement <16 x i8> [[TMP9]], i8 [[C]], i32 9
+; CHECK-NEXT:    [[TMP11:%.*]] = insertelement <16 x i8> [[TMP10]], i8 [[C]], i32 10
+; CHECK-NEXT:    [[TMP12:%.*]] = insertelement <16 x i8> [[TMP11]], i8 [[C]], i32 11
+; CHECK-NEXT:    [[TMP13:%.*]] = insertelement <16 x i8> [[TMP12]], i8 [[C]], i32 12
+; CHECK-NEXT:    [[TMP14:%.*]] = insertelement <16 x i8> [[TMP13]], i8 [[C]], i32 13
+; CHECK-NEXT:    [[TMP15:%.*]] = insertelement <16 x i8> [[TMP14]], i8 [[C]], i32 14
+; CHECK-NEXT:    [[TMP16:%.*]] = insertelement <16 x i8> [[TMP15]], i8 [[C]], i32 15
+; CHECK-NEXT:    [[TMP17:%.*]] = insertelement <2 x i8> undef, i8 [[A:%.*]], i32 0
+; CHECK-NEXT:    [[TMP18:%.*]] = insertelement <2 x i8> [[TMP17]], i8 [[B:%.*]], i32 1
+; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <2 x i8> [[TMP18]], <2 x i8> undef, <16 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 1, i32 0, i32 1, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0>
+; CHECK-NEXT:    [[TMP19:%.*]] = xor <16 x i8> [[TMP16]], [[SHUFFLE]]
+; CHECK-NEXT:    store <16 x i8> [[TMP19]], <16 x i8>* bitcast ([32 x i8]* @cle to <16 x i8>*), align 16
+; CHECK-NEXT:    ret void
+;
   %1 = xor i8 %c, %a
   store i8 %1, i8* getelementptr inbounds ([32 x i8], [32 x i8]* @cle, i64 0, i64 0), align 16
   %2 = xor i8 %a, %c
@@ -59,9 +82,24 @@ define void @splat(i8 %a, i8 %b, i8 %c) {
 ; Check that we correctly detect that we can have the same opcode on one side by
 ; leveraging the commutativity property of `xor`.
 
-; CHECK-LABEL:  @same_opcode_on_one_side
-; CHECK:  store <4 x i32>
 define void @same_opcode_on_one_side(i32 %a, i32 %b, i32 %c) {
+; CHECK-LABEL: @same_opcode_on_one_side(
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x i32> undef, i32 [[C:%.*]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <4 x i32> [[TMP1]], i32 [[C]], i32 1
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <4 x i32> [[TMP2]], i32 [[C]], i32 2
+; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <4 x i32> [[TMP3]], i32 [[C]], i32 3
+; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <4 x i32> undef, i32 [[A:%.*]], i32 0
+; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <4 x i32> [[TMP5]], i32 [[A]], i32 1
+; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <4 x i32> [[TMP6]], i32 [[A]], i32 2
+; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <4 x i32> [[TMP7]], i32 [[A]], i32 3
+; CHECK-NEXT:    [[TMP9:%.*]] = add <4 x i32> [[TMP4]], [[TMP8]]
+; CHECK-NEXT:    [[TMP10:%.*]] = insertelement <4 x i32> [[TMP5]], i32 [[B:%.*]], i32 1
+; CHECK-NEXT:    [[TMP11:%.*]] = insertelement <4 x i32> [[TMP10]], i32 [[C]], i32 2
+; CHECK-NEXT:    [[TMP12:%.*]] = insertelement <4 x i32> [[TMP11]], i32 [[A]], i32 3
+; CHECK-NEXT:    [[TMP13:%.*]] = xor <4 x i32> [[TMP12]], [[TMP9]]
+; CHECK-NEXT:    store <4 x i32> [[TMP13]], <4 x i32>* bitcast ([32 x i32]* @cle32 to <4 x i32>*), align 16
+; CHECK-NEXT:    ret void
+;
   %add1 = add i32 %c, %a
   %add2 = add i32 %c, %a
   %add3 = add i32 %a, %c
diff --git a/test/Transforms/SLPVectorizer/X86/consecutive-access.ll b/test/Transforms/SLPVectorizer/X86/consecutive-access.ll
index aa682fc09f68c71675730697152fdc4c61103432..f394dc7439791289a6034b97eb3608c8a1132208 100644
--- a/test/Transforms/SLPVectorizer/X86/consecutive-access.ll
+++ b/test/Transforms/SLPVectorizer/X86/consecutive-access.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -basicaa -slp-vectorizer -S | FileCheck %s
 target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.9.0"
@@ -11,10 +12,38 @@ target triple = "x86_64-apple-macosx10.9.0"
 ; A[3*i], A[3*i+1] and A[3*i+2] are consecutive, but in future
 ; that would hopefully be fixed. For now, check that this isn't
 ; vectorized.
-; CHECK-LABEL: foo_3double
-; CHECK-NOT: x double>
 ; Function Attrs: nounwind ssp uwtable
 define void @foo_3double(i32 %u) #0 {
+; CHECK-LABEL: @foo_3double(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[U_ADDR:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    store i32 [[U:%.*]], i32* [[U_ADDR]], align 4
+; CHECK-NEXT:    [[MUL:%.*]] = mul nsw i32 [[U]], 3
+; CHECK-NEXT:    [[IDXPROM:%.*]] = sext i32 [[MUL]] to i64
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [2000 x double], [2000 x double]* @A, i32 0, i64 [[IDXPROM]]
+; CHECK-NEXT:    [[TMP0:%.*]] = load double, double* [[ARRAYIDX]], align 8
+; CHECK-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds [2000 x double], [2000 x double]* @B, i32 0, i64 [[IDXPROM]]
+; CHECK-NEXT:    [[TMP1:%.*]] = load double, double* [[ARRAYIDX4]], align 8
+; CHECK-NEXT:    [[ADD5:%.*]] = fadd double [[TMP0]], [[TMP1]]
+; CHECK-NEXT:    store double [[ADD5]], double* [[ARRAYIDX]], align 8
+; CHECK-NEXT:    [[ADD11:%.*]] = add nsw i32 [[MUL]], 1
+; CHECK-NEXT:    [[IDXPROM12:%.*]] = sext i32 [[ADD11]] to i64
+; CHECK-NEXT:    [[ARRAYIDX13:%.*]] = getelementptr inbounds [2000 x double], [2000 x double]* @A, i32 0, i64 [[IDXPROM12]]
+; CHECK-NEXT:    [[TMP2:%.*]] = load double, double* [[ARRAYIDX13]], align 8
+; CHECK-NEXT:    [[ARRAYIDX17:%.*]] = getelementptr inbounds [2000 x double], [2000 x double]* @B, i32 0, i64 [[IDXPROM12]]
+; CHECK-NEXT:    [[TMP3:%.*]] = load double, double* [[ARRAYIDX17]], align 8
+; CHECK-NEXT:    [[ADD18:%.*]] = fadd double [[TMP2]], [[TMP3]]
+; CHECK-NEXT:    store double [[ADD18]], double* [[ARRAYIDX13]], align 8
+; CHECK-NEXT:    [[ADD24:%.*]] = add nsw i32 [[MUL]], 2
+; CHECK-NEXT:    [[IDXPROM25:%.*]] = sext i32 [[ADD24]] to i64
+; CHECK-NEXT:    [[ARRAYIDX26:%.*]] = getelementptr inbounds [2000 x double], [2000 x double]* @A, i32 0, i64 [[IDXPROM25]]
+; CHECK-NEXT:    [[TMP4:%.*]] = load double, double* [[ARRAYIDX26]], align 8
+; CHECK-NEXT:    [[ARRAYIDX30:%.*]] = getelementptr inbounds [2000 x double], [2000 x double]* @B, i32 0, i64 [[IDXPROM25]]
+; CHECK-NEXT:    [[TMP5:%.*]] = load double, double* [[ARRAYIDX30]], align 8
+; CHECK-NEXT:    [[ADD31:%.*]] = fadd double [[TMP4]], [[TMP5]]
+; CHECK-NEXT:    store double [[ADD31]], double* [[ARRAYIDX26]], align 8
+; CHECK-NEXT:    ret void
+;
 entry:
   %u.addr = alloca i32, align 4
   store i32 %u, i32* %u.addr, align 4
@@ -48,10 +77,29 @@ entry:
 ; SCEV should be able to tell that accesses A[C1 + C2*i], A[C1 + C2*i], ...
 ; A[C1 + C2*i] are consecutive, if C2 is a power of 2, and C2 > C1 > 0.
 ; Thus, the following code should be vectorized.
-; CHECK-LABEL: foo_2double
-; CHECK: x double>
 ; Function Attrs: nounwind ssp uwtable
 define void @foo_2double(i32 %u) #0 {
+; CHECK-LABEL: @foo_2double(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[U_ADDR:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    store i32 [[U:%.*]], i32* [[U_ADDR]], align 4
+; CHECK-NEXT:    [[MUL:%.*]] = mul nsw i32 [[U]], 2
+; CHECK-NEXT:    [[IDXPROM:%.*]] = sext i32 [[MUL]] to i64
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [2000 x double], [2000 x double]* @A, i32 0, i64 [[IDXPROM]]
+; CHECK-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds [2000 x double], [2000 x double]* @B, i32 0, i64 [[IDXPROM]]
+; CHECK-NEXT:    [[ADD11:%.*]] = add nsw i32 [[MUL]], 1
+; CHECK-NEXT:    [[IDXPROM12:%.*]] = sext i32 [[ADD11]] to i64
+; CHECK-NEXT:    [[ARRAYIDX13:%.*]] = getelementptr inbounds [2000 x double], [2000 x double]* @A, i32 0, i64 [[IDXPROM12]]
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast double* [[ARRAYIDX]] to <2 x double>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x double>, <2 x double>* [[TMP0]], align 8
+; CHECK-NEXT:    [[ARRAYIDX17:%.*]] = getelementptr inbounds [2000 x double], [2000 x double]* @B, i32 0, i64 [[IDXPROM12]]
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast double* [[ARRAYIDX4]] to <2 x double>*
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x double>, <2 x double>* [[TMP2]], align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = fadd <2 x double> [[TMP1]], [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast double* [[ARRAYIDX]] to <2 x double>*
+; CHECK-NEXT:    store <2 x double> [[TMP4]], <2 x double>* [[TMP5]], align 8
+; CHECK-NEXT:    ret void
+;
 entry:
   %u.addr = alloca i32, align 4
   store i32 %u, i32* %u.addr, align 4
@@ -75,10 +123,37 @@ entry:
 }
 
 ; Similar to the previous test, but with different datatype.
-; CHECK-LABEL: foo_4float
-; CHECK: x float>
 ; Function Attrs: nounwind ssp uwtable
 define void @foo_4float(i32 %u) #0 {
+; CHECK-LABEL: @foo_4float(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[U_ADDR:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    store i32 [[U:%.*]], i32* [[U_ADDR]], align 4
+; CHECK-NEXT:    [[MUL:%.*]] = mul nsw i32 [[U]], 4
+; CHECK-NEXT:    [[IDXPROM:%.*]] = sext i32 [[MUL]] to i64
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [2000 x float], [2000 x float]* @C, i32 0, i64 [[IDXPROM]]
+; CHECK-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds [2000 x float], [2000 x float]* @D, i32 0, i64 [[IDXPROM]]
+; CHECK-NEXT:    [[ADD11:%.*]] = add nsw i32 [[MUL]], 1
+; CHECK-NEXT:    [[IDXPROM12:%.*]] = sext i32 [[ADD11]] to i64
+; CHECK-NEXT:    [[ARRAYIDX13:%.*]] = getelementptr inbounds [2000 x float], [2000 x float]* @C, i32 0, i64 [[IDXPROM12]]
+; CHECK-NEXT:    [[ARRAYIDX17:%.*]] = getelementptr inbounds [2000 x float], [2000 x float]* @D, i32 0, i64 [[IDXPROM12]]
+; CHECK-NEXT:    [[ADD24:%.*]] = add nsw i32 [[MUL]], 2
+; CHECK-NEXT:    [[IDXPROM25:%.*]] = sext i32 [[ADD24]] to i64
+; CHECK-NEXT:    [[ARRAYIDX26:%.*]] = getelementptr inbounds [2000 x float], [2000 x float]* @C, i32 0, i64 [[IDXPROM25]]
+; CHECK-NEXT:    [[ARRAYIDX30:%.*]] = getelementptr inbounds [2000 x float], [2000 x float]* @D, i32 0, i64 [[IDXPROM25]]
+; CHECK-NEXT:    [[ADD37:%.*]] = add nsw i32 [[MUL]], 3
+; CHECK-NEXT:    [[IDXPROM38:%.*]] = sext i32 [[ADD37]] to i64
+; CHECK-NEXT:    [[ARRAYIDX39:%.*]] = getelementptr inbounds [2000 x float], [2000 x float]* @C, i32 0, i64 [[IDXPROM38]]
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast float* [[ARRAYIDX]] to <4 x float>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x float>, <4 x float>* [[TMP0]], align 4
+; CHECK-NEXT:    [[ARRAYIDX43:%.*]] = getelementptr inbounds [2000 x float], [2000 x float]* @D, i32 0, i64 [[IDXPROM38]]
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast float* [[ARRAYIDX4]] to <4 x float>*
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x float>, <4 x float>* [[TMP2]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = fadd <4 x float> [[TMP1]], [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast float* [[ARRAYIDX]] to <4 x float>*
+; CHECK-NEXT:    store <4 x float> [[TMP4]], <4 x float>* [[TMP5]], align 4
+; CHECK-NEXT:    ret void
+;
 entry:
   %u.addr = alloca i32, align 4
   store i32 %u, i32* %u.addr, align 4
@@ -118,10 +193,51 @@ entry:
 }
 
 ; Similar to the previous tests, but now we are dealing with AddRec SCEV.
-; CHECK-LABEL: foo_loop
-; CHECK: x double>
 ; Function Attrs: nounwind ssp uwtable
 define i32 @foo_loop(double* %A, i32 %n) #0 {
+; CHECK-LABEL: @foo_loop(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[A_ADDR:%.*]] = alloca double*, align 8
+; CHECK-NEXT:    [[N_ADDR:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    [[SUM:%.*]] = alloca double, align 8
+; CHECK-NEXT:    [[I:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    store double* [[A:%.*]], double** [[A_ADDR]], align 8
+; CHECK-NEXT:    store i32 [[N:%.*]], i32* [[N_ADDR]], align 4
+; CHECK-NEXT:    store double 0.000000e+00, double* [[SUM]], align 8
+; CHECK-NEXT:    store i32 0, i32* [[I]], align 4
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp slt i32 0, [[N]]
+; CHECK-NEXT:    br i1 [[CMP1]], label [[FOR_BODY_LR_PH:%.*]], label [[FOR_END:%.*]]
+; CHECK:       for.body.lr.ph:
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[TMP0:%.*]] = phi i32 [ 0, [[FOR_BODY_LR_PH]] ], [ [[INC:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[TMP1:%.*]] = phi double [ 0.000000e+00, [[FOR_BODY_LR_PH]] ], [ [[ADD7:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP0]], 2
+; CHECK-NEXT:    [[IDXPROM:%.*]] = sext i32 [[MUL]] to i64
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds double, double* [[A]], i64 [[IDXPROM]]
+; CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 [[MUL]], 1
+; CHECK-NEXT:    [[IDXPROM3:%.*]] = sext i32 [[ADD]] to i64
+; CHECK-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds double, double* [[A]], i64 [[IDXPROM3]]
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast double* [[ARRAYIDX]] to <2 x double>*
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x double>, <2 x double>* [[TMP2]], align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = fmul <2 x double> <double 7.000000e+00, double 7.000000e+00>, [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <2 x double> [[TMP4]], i32 0
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <2 x double> [[TMP4]], i32 1
+; CHECK-NEXT:    [[ADD6:%.*]] = fadd double [[TMP5]], [[TMP6]]
+; CHECK-NEXT:    [[ADD7]] = fadd double [[TMP1]], [[ADD6]]
+; CHECK-NEXT:    store double [[ADD7]], double* [[SUM]], align 8
+; CHECK-NEXT:    [[INC]] = add nsw i32 [[TMP0]], 1
+; CHECK-NEXT:    store i32 [[INC]], i32* [[I]], align 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[INC]], [[N]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_COND_FOR_END_CRIT_EDGE:%.*]]
+; CHECK:       for.cond.for.end_crit_edge:
+; CHECK-NEXT:    [[SPLIT:%.*]] = phi double [ [[ADD7]], [[FOR_BODY]] ]
+; CHECK-NEXT:    br label [[FOR_END]]
+; CHECK:       for.end:
+; CHECK-NEXT:    [[DOTLCSSA:%.*]] = phi double [ [[SPLIT]], [[FOR_COND_FOR_END_CRIT_EDGE]] ], [ 0.000000e+00, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[CONV:%.*]] = fptosi double [[DOTLCSSA]] to i32
+; CHECK-NEXT:    ret i32 [[CONV]]
+;
 entry:
   %A.addr = alloca double*, align 8
   %n.addr = alloca i32, align 4
@@ -170,11 +286,30 @@ for.end:                                          ; preds = %for.cond.for.end_cr
 
 ; Similar to foo_2double but with a non-power-of-2 factor and potential
 ; wrapping (both indices wrap or both don't in the same time)
-; CHECK-LABEL: foo_2double_non_power_of_2
-; CHECK: load <2 x double>
-; CHECK: load <2 x double>
 ; Function Attrs: nounwind ssp uwtable
 define void @foo_2double_non_power_of_2(i32 %u) #0 {
+; CHECK-LABEL: @foo_2double_non_power_of_2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[U_ADDR:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    store i32 [[U:%.*]], i32* [[U_ADDR]], align 4
+; CHECK-NEXT:    [[MUL:%.*]] = mul i32 [[U]], 6
+; CHECK-NEXT:    [[ADD6:%.*]] = add i32 [[MUL]], 6
+; CHECK-NEXT:    [[IDXPROM:%.*]] = sext i32 [[ADD6]] to i64
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [2000 x double], [2000 x double]* @A, i32 0, i64 [[IDXPROM]]
+; CHECK-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds [2000 x double], [2000 x double]* @B, i32 0, i64 [[IDXPROM]]
+; CHECK-NEXT:    [[ADD7:%.*]] = add i32 [[MUL]], 7
+; CHECK-NEXT:    [[IDXPROM12:%.*]] = sext i32 [[ADD7]] to i64
+; CHECK-NEXT:    [[ARRAYIDX13:%.*]] = getelementptr inbounds [2000 x double], [2000 x double]* @A, i32 0, i64 [[IDXPROM12]]
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast double* [[ARRAYIDX]] to <2 x double>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x double>, <2 x double>* [[TMP0]], align 8
+; CHECK-NEXT:    [[ARRAYIDX17:%.*]] = getelementptr inbounds [2000 x double], [2000 x double]* @B, i32 0, i64 [[IDXPROM12]]
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast double* [[ARRAYIDX4]] to <2 x double>*
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x double>, <2 x double>* [[TMP2]], align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = fadd <2 x double> [[TMP1]], [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast double* [[ARRAYIDX]] to <2 x double>*
+; CHECK-NEXT:    store <2 x double> [[TMP4]], <2 x double>* [[TMP5]], align 8
+; CHECK-NEXT:    ret void
+;
 entry:
   %u.addr = alloca i32, align 4
   store i32 %u, i32* %u.addr, align 4
@@ -199,11 +334,30 @@ entry:
 }
 
 ; Similar to foo_2double_non_power_of_2 but with zext's instead of sext's
-; CHECK-LABEL: foo_2double_non_power_of_2_zext
-; CHECK: load <2 x double>
-; CHECK: load <2 x double>
 ; Function Attrs: nounwind ssp uwtable
 define void @foo_2double_non_power_of_2_zext(i32 %u) #0 {
+; CHECK-LABEL: @foo_2double_non_power_of_2_zext(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[U_ADDR:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    store i32 [[U:%.*]], i32* [[U_ADDR]], align 4
+; CHECK-NEXT:    [[MUL:%.*]] = mul i32 [[U]], 6
+; CHECK-NEXT:    [[ADD6:%.*]] = add i32 [[MUL]], 6
+; CHECK-NEXT:    [[IDXPROM:%.*]] = zext i32 [[ADD6]] to i64
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [2000 x double], [2000 x double]* @A, i32 0, i64 [[IDXPROM]]
+; CHECK-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds [2000 x double], [2000 x double]* @B, i32 0, i64 [[IDXPROM]]
+; CHECK-NEXT:    [[ADD7:%.*]] = add i32 [[MUL]], 7
+; CHECK-NEXT:    [[IDXPROM12:%.*]] = zext i32 [[ADD7]] to i64
+; CHECK-NEXT:    [[ARRAYIDX13:%.*]] = getelementptr inbounds [2000 x double], [2000 x double]* @A, i32 0, i64 [[IDXPROM12]]
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast double* [[ARRAYIDX]] to <2 x double>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x double>, <2 x double>* [[TMP0]], align 8
+; CHECK-NEXT:    [[ARRAYIDX17:%.*]] = getelementptr inbounds [2000 x double], [2000 x double]* @B, i32 0, i64 [[IDXPROM12]]
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast double* [[ARRAYIDX4]] to <2 x double>*
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x double>, <2 x double>* [[TMP2]], align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = fadd <2 x double> [[TMP1]], [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast double* [[ARRAYIDX]] to <2 x double>*
+; CHECK-NEXT:    store <2 x double> [[TMP4]], <2 x double>* [[TMP5]], align 8
+; CHECK-NEXT:    ret void
+;
 entry:
   %u.addr = alloca i32, align 4
   store i32 %u, i32* %u.addr, align 4
@@ -230,10 +384,52 @@ entry:
 ; Similar to foo_2double_non_power_of_2, but now we are dealing with AddRec SCEV.
 ; Alternatively, this is like foo_loop, but with a non-power-of-2 factor and
 ; potential wrapping (both indices wrap or both don't in the same time)
-; CHECK-LABEL: foo_loop_non_power_of_2
-; CHECK: <2 x double>
 ; Function Attrs: nounwind ssp uwtable
 define i32 @foo_loop_non_power_of_2(double* %A, i32 %n) #0 {
+; CHECK-LABEL: @foo_loop_non_power_of_2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[A_ADDR:%.*]] = alloca double*, align 8
+; CHECK-NEXT:    [[N_ADDR:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    [[SUM:%.*]] = alloca double, align 8
+; CHECK-NEXT:    [[I:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    store double* [[A:%.*]], double** [[A_ADDR]], align 8
+; CHECK-NEXT:    store i32 [[N:%.*]], i32* [[N_ADDR]], align 4
+; CHECK-NEXT:    store double 0.000000e+00, double* [[SUM]], align 8
+; CHECK-NEXT:    store i32 0, i32* [[I]], align 4
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp slt i32 0, [[N]]
+; CHECK-NEXT:    br i1 [[CMP1]], label [[FOR_BODY_LR_PH:%.*]], label [[FOR_END:%.*]]
+; CHECK:       for.body.lr.ph:
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[TMP0:%.*]] = phi i32 [ 0, [[FOR_BODY_LR_PH]] ], [ [[INC:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[TMP1:%.*]] = phi double [ 0.000000e+00, [[FOR_BODY_LR_PH]] ], [ [[ADD7:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[MUL:%.*]] = mul i32 [[TMP0]], 12
+; CHECK-NEXT:    [[ADD_5:%.*]] = add i32 [[MUL]], 5
+; CHECK-NEXT:    [[IDXPROM:%.*]] = sext i32 [[ADD_5]] to i64
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds double, double* [[A]], i64 [[IDXPROM]]
+; CHECK-NEXT:    [[ADD_6:%.*]] = add i32 [[MUL]], 6
+; CHECK-NEXT:    [[IDXPROM3:%.*]] = sext i32 [[ADD_6]] to i64
+; CHECK-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds double, double* [[A]], i64 [[IDXPROM3]]
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast double* [[ARRAYIDX]] to <2 x double>*
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x double>, <2 x double>* [[TMP2]], align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = fmul <2 x double> <double 7.000000e+00, double 7.000000e+00>, [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <2 x double> [[TMP4]], i32 0
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <2 x double> [[TMP4]], i32 1
+; CHECK-NEXT:    [[ADD6:%.*]] = fadd double [[TMP5]], [[TMP6]]
+; CHECK-NEXT:    [[ADD7]] = fadd double [[TMP1]], [[ADD6]]
+; CHECK-NEXT:    store double [[ADD7]], double* [[SUM]], align 8
+; CHECK-NEXT:    [[INC]] = add i32 [[TMP0]], 1
+; CHECK-NEXT:    store i32 [[INC]], i32* [[I]], align 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[INC]], [[N]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_COND_FOR_END_CRIT_EDGE:%.*]]
+; CHECK:       for.cond.for.end_crit_edge:
+; CHECK-NEXT:    [[SPLIT:%.*]] = phi double [ [[ADD7]], [[FOR_BODY]] ]
+; CHECK-NEXT:    br label [[FOR_END]]
+; CHECK:       for.end:
+; CHECK-NEXT:    [[DOTLCSSA:%.*]] = phi double [ [[SPLIT]], [[FOR_COND_FOR_END_CRIT_EDGE]] ], [ 0.000000e+00, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[CONV:%.*]] = fptosi double [[DOTLCSSA]] to i32
+; CHECK-NEXT:    ret i32 [[CONV]]
+;
 entry:
   %A.addr = alloca double*, align 8
   %n.addr = alloca i32, align 4
@@ -299,9 +495,32 @@ for.end:                                          ; preds = %for.cond.for.end_cr
 ;
 ; Make sure we are able to vectorize this from now on:
 ;
-; CHECK-LABEL: @bar
-; CHECK: load <2 x double>
 define double @bar(double* nocapture readonly %a, i32 %n) local_unnamed_addr #0 {
+; CHECK-LABEL: @bar(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP15:%.*]] = icmp eq i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[CMP15]], label [[FOR_COND_CLEANUP:%.*]], label [[FOR_BODY:%.*]]
+; CHECK:       for.cond.cleanup:
+; CHECK-NEXT:    [[TMP0:%.*]] = phi <2 x double> [ zeroinitializer, [[ENTRY:%.*]] ], [ [[TMP6:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <2 x double> [[TMP0]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <2 x double> [[TMP0]], i32 1
+; CHECK-NEXT:    [[MUL:%.*]] = fmul double [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    ret double [[MUL]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[I_018:%.*]] = phi i32 [ [[ADD5:%.*]], [[FOR_BODY]] ], [ 0, [[ENTRY]] ]
+; CHECK-NEXT:    [[TMP3:%.*]] = phi <2 x double> [ [[TMP6]], [[FOR_BODY]] ], [ zeroinitializer, [[ENTRY]] ]
+; CHECK-NEXT:    [[IDXPROM:%.*]] = zext i32 [[I_018]] to i64
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds double, double* [[A:%.*]], i64 [[IDXPROM]]
+; CHECK-NEXT:    [[ADD1:%.*]] = or i32 [[I_018]], 1
+; CHECK-NEXT:    [[IDXPROM2:%.*]] = zext i32 [[ADD1]] to i64
+; CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds double, double* [[A]], i64 [[IDXPROM2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast double* [[ARRAYIDX]] to <2 x double>*
+; CHECK-NEXT:    [[TMP5:%.*]] = load <2 x double>, <2 x double>* [[TMP4]], align 8
+; CHECK-NEXT:    [[TMP6]] = fadd <2 x double> [[TMP3]], [[TMP5]]
+; CHECK-NEXT:    [[ADD5]] = add i32 [[I_018]], 2
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[ADD5]], [[N]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_COND_CLEANUP]]
+;
 entry:
   %cmp15 = icmp eq i32 %n, 0
   br i1 %cmp15, label %for.cond.cleanup, label %for.body
diff --git a/test/Transforms/SLPVectorizer/X86/continue_vectorizing.ll b/test/Transforms/SLPVectorizer/X86/continue_vectorizing.ll
index ecae70ecc918568fe7c33b113295de28f83b6e22..060cb054b7f00e1fc45c6b17619e67d99b46907c 100644
--- a/test/Transforms/SLPVectorizer/X86/continue_vectorizing.ll
+++ b/test/Transforms/SLPVectorizer/X86/continue_vectorizing.ll
@@ -1,13 +1,32 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -basicaa -slp-vectorizer -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.8.0"
 
 ; We will keep trying to vectorize the basic block even we already find vectorized store.
-; CHECK: test1
-; CHECK: store <2 x double>
-; CHECK: ret
 define void @test1(double* %a, double* %b, double* %c, double* %d) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds double, double* [[A:%.*]], i64 1
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast double* [[A]] to <2 x double>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x double>, <2 x double>* [[TMP0]], align 8
+; CHECK-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds double, double* [[B:%.*]], i64 1
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast double* [[B]] to <2 x double>*
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x double>, <2 x double>* [[TMP2]], align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = fmul <2 x double> [[TMP1]], [[TMP3]]
+; CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds double, double* [[C:%.*]], i64 1
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast double* [[C]] to <2 x double>*
+; CHECK-NEXT:    store <2 x double> [[TMP4]], <2 x double>* [[TMP5]], align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast double* [[A]] to <4 x i32>*
+; CHECK-NEXT:    [[TMP7:%.*]] = load <4 x i32>, <4 x i32>* [[TMP6]], align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast double* [[B]] to <4 x i32>*
+; CHECK-NEXT:    [[TMP9:%.*]] = load <4 x i32>, <4 x i32>* [[TMP8]], align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = mul <4 x i32> [[TMP7]], [[TMP9]]
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast double* [[D:%.*]] to <4 x i32>*
+; CHECK-NEXT:    store <4 x i32> [[TMP10]], <4 x i32>* [[TMP11]], align 8
+; CHECK-NEXT:    ret void
+;
 entry:
   %i0 = load double, double* %a, align 8
   %i1 = load double, double* %b, align 8
diff --git a/test/Transforms/SLPVectorizer/X86/crash_7zip.ll b/test/Transforms/SLPVectorizer/X86/crash_7zip.ll
index 54e5d5a5308021cc4e9658ddf215d3b673539c72..548a0fab6ad2ee69248c2d387a5691e1e47ca652 100644
--- a/test/Transforms/SLPVectorizer/X86/crash_7zip.ll
+++ b/test/Transforms/SLPVectorizer/X86/crash_7zip.ll
@@ -1,4 +1,5 @@
-; RUN: opt < %s -basicaa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -basicaa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7 | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.8.0"
@@ -7,6 +8,28 @@ target triple = "x86_64-apple-macosx10.8.0"
 %struct._CLzmaProps.0.27.54.81.102.123.144.165.180.195.228.258.333 = type { i32, i32, i32, i32 }
 
 define fastcc void @LzmaDec_DecodeReal2(%struct.CLzmaDec.1.28.55.82.103.124.145.166.181.196.229.259.334* %p) {
+; CHECK-LABEL: @LzmaDec_DecodeReal2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[RANGE20_I:%.*]] = getelementptr inbounds [[STRUCT_CLZMADEC_1_28_55_82_103_124_145_166_181_196_229_259_334:%.*]], %struct.CLzmaDec.1.28.55.82.103.124.145.166.181.196.229.259.334* [[P:%.*]], i64 0, i32 4
+; CHECK-NEXT:    br label [[DO_BODY66_I:%.*]]
+; CHECK:       do.body66.i:
+; CHECK-NEXT:    [[TMP0:%.*]] = phi <2 x i32> [ [[TMP5:%.*]], [[DO_COND_I:%.*]] ], [ undef, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[TMP1:%.*]] = select <2 x i1> undef, <2 x i32> undef, <2 x i32> [[TMP0]]
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <2 x i32> [[TMP1]], i32 1
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <2 x i32> undef, i32 [[TMP2]], i32 1
+; CHECK-NEXT:    br i1 undef, label [[DO_COND_I]], label [[IF_ELSE_I:%.*]]
+; CHECK:       if.else.i:
+; CHECK-NEXT:    [[TMP4:%.*]] = sub <2 x i32> [[TMP1]], undef
+; CHECK-NEXT:    br label [[DO_COND_I]]
+; CHECK:       do.cond.i:
+; CHECK-NEXT:    [[TMP5]] = phi <2 x i32> [ [[TMP4]], [[IF_ELSE_I]] ], [ [[TMP3]], [[DO_BODY66_I]] ]
+; CHECK-NEXT:    br i1 undef, label [[DO_BODY66_I]], label [[DO_END1006_I:%.*]]
+; CHECK:       do.end1006.i:
+; CHECK-NEXT:    [[TMP6:%.*]] = select <2 x i1> undef, <2 x i32> undef, <2 x i32> [[TMP5]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i32* [[RANGE20_I]] to <2 x i32>*
+; CHECK-NEXT:    store <2 x i32> [[TMP6]], <2 x i32>* [[TMP7]], align 4
+; CHECK-NEXT:    ret void
+;
 entry:
   %range20.i = getelementptr inbounds %struct.CLzmaDec.1.28.55.82.103.124.145.166.181.196.229.259.334, %struct.CLzmaDec.1.28.55.82.103.124.145.166.181.196.229.259.334* %p, i64 0, i32 4
   %code21.i = getelementptr inbounds %struct.CLzmaDec.1.28.55.82.103.124.145.166.181.196.229.259.334, %struct.CLzmaDec.1.28.55.82.103.124.145.166.181.196.229.259.334* %p, i64 0, i32 5
diff --git a/test/Transforms/SLPVectorizer/X86/crash_binaryop.ll b/test/Transforms/SLPVectorizer/X86/crash_binaryop.ll
index 9046c35628216fbfb8482ed1ebec606156818a45..eec53739b6796f6658a83eafb0dc6617ae4299e1 100644
--- a/test/Transforms/SLPVectorizer/X86/crash_binaryop.ll
+++ b/test/Transforms/SLPVectorizer/X86/crash_binaryop.ll
@@ -1,4 +1,5 @@
-; RUN: opt < %s -basicaa -slp-vectorizer -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -basicaa -slp-vectorizer -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7 | FileCheck %s
 
 target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-darwin13.3.0"
@@ -6,6 +7,28 @@ target triple = "x86_64-apple-darwin13.3.0"
 @a = common global double 0.000000e+00, align 8
 
 define i32 @fn1() {
+; CHECK-LABEL: @fn1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[INIT:%.*]] = load double, double* @a, align 8
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[PHI:%.*]] = phi double [ [[ADD2:%.*]], [[LOOP]] ], [ [[INIT]], [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[POSTADD1_PHI:%.*]] = phi double [ [[POSTADD1:%.*]], [[LOOP]] ], [ [[INIT]], [[ENTRY]] ]
+; CHECK-NEXT:    [[POSTADD2_PHI:%.*]] = phi double [ [[POSTADD2:%.*]], [[LOOP]] ], [ [[INIT]], [[ENTRY]] ]
+; CHECK-NEXT:    [[ADD1:%.*]] = fadd double [[POSTADD1_PHI]], undef
+; CHECK-NEXT:    [[ADD2]] = fadd double [[POSTADD2_PHI]], [[PHI]]
+; CHECK-NEXT:    [[MUL2:%.*]] = fmul double [[ADD2]], 0.000000e+00
+; CHECK-NEXT:    [[BINARYOP_B:%.*]] = fadd double [[POSTADD1_PHI]], [[MUL2]]
+; CHECK-NEXT:    [[MUL1:%.*]] = fmul double [[ADD1]], 0.000000e+00
+; CHECK-NEXT:    [[TMP:%.*]] = fadd double [[POSTADD2_PHI]], 0.000000e+00
+; CHECK-NEXT:    [[BINARY_V:%.*]] = fadd double [[MUL1]], [[BINARYOP_B]]
+; CHECK-NEXT:    [[POSTADD1]] = fadd double [[BINARY_V]], 0.000000e+00
+; CHECK-NEXT:    [[POSTADD2]] = fadd double [[TMP]], 1.000000e+00
+; CHECK-NEXT:    [[TOBOOL:%.*]] = fcmp une double [[POSTADD1]], 0.000000e+00
+; CHECK-NEXT:    br i1 [[TOBOOL]], label [[EXIT:%.*]], label [[LOOP]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret i32 1
+;
 entry:
   %init = load double, double* @a, align 8
   br label %loop
diff --git a/test/Transforms/SLPVectorizer/X86/crash_bullet.ll b/test/Transforms/SLPVectorizer/X86/crash_bullet.ll
index 1bad671fd82414ec63e37b3bcfd768f2aaca9e43..11dbbdaa16505cc92aa30fa541f5aaebbf4103ff 100644
--- a/test/Transforms/SLPVectorizer/X86/crash_bullet.ll
+++ b/test/Transforms/SLPVectorizer/X86/crash_bullet.ll
@@ -1,4 +1,5 @@
-; RUN: opt < %s -basicaa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -basicaa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7 | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.8.0"
@@ -6,6 +7,27 @@ target triple = "x86_64-apple-macosx10.8.0"
 %"struct.btTypedConstraint::btConstraintInfo1.17.157.357.417.477.960" = type { i32, i32 }
 
 define void @_ZN23btGeneric6DofConstraint8getInfo1EPN17btTypedConstraint17btConstraintInfo1E(%"struct.btTypedConstraint::btConstraintInfo1.17.157.357.417.477.960"* nocapture %info) {
+; CHECK-LABEL: @_ZN23btGeneric6DofConstraint8getInfo1EPN17btTypedConstraint17btConstraintInfo1E(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 undef, label [[IF_ELSE:%.*]], label [[IF_THEN:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    ret void
+; CHECK:       if.else:
+; CHECK-NEXT:    [[M_NUMCONSTRAINTROWS4:%.*]] = getelementptr inbounds %"struct.btTypedConstraint::btConstraintInfo1.17.157.357.417.477.960", %"struct.btTypedConstraint::btConstraintInfo1.17.157.357.417.477.960"* [[INFO:%.*]], i64 0, i32 0
+; CHECK-NEXT:    br i1 undef, label [[LAND_LHS_TRUE_I_1:%.*]], label [[IF_THEN7_1:%.*]]
+; CHECK:       land.lhs.true.i.1:
+; CHECK-NEXT:    br i1 undef, label [[FOR_INC_1:%.*]], label [[IF_THEN7_1]]
+; CHECK:       if.then7.1:
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast i32* [[M_NUMCONSTRAINTROWS4]] to <2 x i32>*
+; CHECK-NEXT:    store <2 x i32> <i32 1, i32 5>, <2 x i32>* [[TMP0]], align 4
+; CHECK-NEXT:    br label [[FOR_INC_1]]
+; CHECK:       for.inc.1:
+; CHECK-NEXT:    [[TMP1:%.*]] = phi <2 x i32> [ <i32 1, i32 5>, [[IF_THEN7_1]] ], [ <i32 0, i32 6>, [[LAND_LHS_TRUE_I_1]] ]
+; CHECK-NEXT:    [[TMP2:%.*]] = add nsw <2 x i32> <i32 1, i32 -1>, [[TMP1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i32* [[M_NUMCONSTRAINTROWS4]] to <2 x i32>*
+; CHECK-NEXT:    store <2 x i32> [[TMP2]], <2 x i32>* [[TMP3]], align 4
+; CHECK-NEXT:    unreachable
+;
 entry:
   br i1 undef, label %if.else, label %if.then
 
@@ -42,6 +64,29 @@ for.inc.1:                                        ; preds = %if.then7.1, %land.l
 %class.btVector4.7.32.67.92.117.142.177.187.262.282.331 = type { %class.btVector3.5.30.65.90.115.140.175.185.260.280.330 }
 
 define void @_ZN30GIM_TRIANGLE_CALCULATION_CACHE18triangle_collisionERK9btVector3S2_S2_fS2_S2_S2_fR25GIM_TRIANGLE_CONTACT_DATA(%class.GIM_TRIANGLE_CALCULATION_CACHE.9.34.69.94.119.144.179.189.264.284.332* %this) {
+; CHECK-LABEL: @_ZN30GIM_TRIANGLE_CALCULATION_CACHE18triangle_collisionERK9btVector3S2_S2_fS2_S2_S2_fR25GIM_TRIANGLE_CONTACT_DATA(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[ARRAYIDX26:%.*]] = getelementptr inbounds [[CLASS_GIM_TRIANGLE_CALCULATION_CACHE_9_34_69_94_119_144_179_189_264_284_332:%.*]], %class.GIM_TRIANGLE_CALCULATION_CACHE.9.34.69.94.119.144.179.189.264.284.332* [[THIS:%.*]], i64 0, i32 2, i64 0, i32 0, i64 1
+; CHECK-NEXT:    [[ARRAYIDX36:%.*]] = getelementptr inbounds [[CLASS_GIM_TRIANGLE_CALCULATION_CACHE_9_34_69_94_119_144_179_189_264_284_332]], %class.GIM_TRIANGLE_CALCULATION_CACHE.9.34.69.94.119.144.179.189.264.284.332* [[THIS]], i64 0, i32 2, i64 0, i32 0, i64 2
+; CHECK-NEXT:    [[TMP0:%.*]] = load float, float* [[ARRAYIDX36]], align 4
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x float> undef, float [[TMP0]], i32 1
+; CHECK-NEXT:    [[TMP2:%.*]] = fadd <2 x float> [[TMP1]], undef
+; CHECK-NEXT:    [[TMP3:%.*]] = fsub <2 x float> [[TMP2]], undef
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x float> [[TMP3]], i32 0
+; CHECK-NEXT:    store float [[TMP4]], float* undef, align 4
+; CHECK-NEXT:    [[TMP5:%.*]] = fsub <2 x float> [[TMP2]], [[TMP3]]
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast float* [[ARRAYIDX26]] to <2 x float>*
+; CHECK-NEXT:    store <2 x float> [[TMP5]], <2 x float>* [[TMP6]], align 4
+; CHECK-NEXT:    br i1 undef, label [[IF_ELSE1609:%.*]], label [[IF_THEN1595:%.*]]
+; CHECK:       if.then1595:
+; CHECK-NEXT:    br i1 undef, label [[RETURN:%.*]], label [[FOR_BODY_LR_PH_I_I1702:%.*]]
+; CHECK:       for.body.lr.ph.i.i1702:
+; CHECK-NEXT:    unreachable
+; CHECK:       if.else1609:
+; CHECK-NEXT:    unreachable
+; CHECK:       return:
+; CHECK-NEXT:    ret void
+;
 entry:
   %arrayidx26 = getelementptr inbounds %class.GIM_TRIANGLE_CALCULATION_CACHE.9.34.69.94.119.144.179.189.264.284.332, %class.GIM_TRIANGLE_CALCULATION_CACHE.9.34.69.94.119.144.179.189.264.284.332* %this, i64 0, i32 2, i64 0, i32 0, i64 1
   %arrayidx36 = getelementptr inbounds %class.GIM_TRIANGLE_CALCULATION_CACHE.9.34.69.94.119.144.179.189.264.284.332, %class.GIM_TRIANGLE_CALCULATION_CACHE.9.34.69.94.119.144.179.189.264.284.332* %this, i64 0, i32 2, i64 0, i32 0, i64 2
@@ -71,6 +116,40 @@ return:                                           ; preds = %if.then1595
 }
 
 define void @_Z8dBoxBox2RK9btVector3PKfS1_S1_S3_S1_RS_PfPiiP12dContactGeomiRN36btDiscreteCollisionDetectorInterface6ResultE() {
+; CHECK-LABEL: @_Z8dBoxBox2RK9btVector3PKfS1_S1_S3_S1_RS_PfPiiP12dContactGeomiRN36btDiscreteCollisionDetectorInterface6ResultE(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 undef, label [[RETURN:%.*]], label [[IF_END:%.*]]
+; CHECK:       if.end:
+; CHECK-NEXT:    br i1 undef, label [[RETURN]], label [[IF_END111:%.*]]
+; CHECK:       if.end111:
+; CHECK-NEXT:    br i1 undef, label [[RETURN]], label [[IF_END136:%.*]]
+; CHECK:       if.end136:
+; CHECK-NEXT:    br i1 undef, label [[RETURN]], label [[IF_END162:%.*]]
+; CHECK:       if.end162:
+; CHECK-NEXT:    br i1 undef, label [[RETURN]], label [[IF_END189:%.*]]
+; CHECK:       if.end189:
+; CHECK-NEXT:    br i1 undef, label [[RETURN]], label [[IF_END216:%.*]]
+; CHECK:       if.end216:
+; CHECK-NEXT:    br i1 undef, label [[IF_THEN218:%.*]], label [[IF_END225:%.*]]
+; CHECK:       if.then218:
+; CHECK-NEXT:    br label [[IF_END225]]
+; CHECK:       if.end225:
+; CHECK-NEXT:    br i1 undef, label [[RETURN]], label [[IF_END248:%.*]]
+; CHECK:       if.end248:
+; CHECK-NEXT:    br i1 undef, label [[RETURN]], label [[IF_END304:%.*]]
+; CHECK:       if.end304:
+; CHECK-NEXT:    br i1 undef, label [[RETURN]], label [[IF_END361:%.*]]
+; CHECK:       if.end361:
+; CHECK-NEXT:    br i1 undef, label [[IF_THEN370:%.*]], label [[IF_END395:%.*]]
+; CHECK:       if.then370:
+; CHECK-NEXT:    br i1 undef, label [[IF_THEN374:%.*]], label [[IF_END395]]
+; CHECK:       if.then374:
+; CHECK-NEXT:    br label [[IF_END395]]
+; CHECK:       if.end395:
+; CHECK-NEXT:    unreachable
+; CHECK:       return:
+; CHECK-NEXT:    ret void
+;
 entry:
   %add8.i2343 = fadd float undef, undef
   %add8.i2381 = fadd float undef, undef
diff --git a/test/Transforms/SLPVectorizer/X86/crash_bullet3.ll b/test/Transforms/SLPVectorizer/X86/crash_bullet3.ll
index 8102769295307d5e6a1ca61bd2f882d62aba8014..677247a87883b6ba322a82e0974e8d275067cc8b 100644
--- a/test/Transforms/SLPVectorizer/X86/crash_bullet3.ll
+++ b/test/Transforms/SLPVectorizer/X86/crash_bullet3.ll
@@ -1,4 +1,5 @@
-; RUN: opt < %s -basicaa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -basicaa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7 | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.8.0"
@@ -7,6 +8,58 @@ target triple = "x86_64-apple-macosx10.8.0"
 
 ; Function Attrs: ssp uwtable
 define void @_ZN11HullLibrary15CleanupVerticesEjPK9btVector3jRjPS0_fRS0_(%class.btVector3.23.221.463.485.507.573.595.683.727.749.815.837.991.1585.1607.1629.1651.1849.2047.2069.2091.2113* %vertices) #0 align 2 {
+; CHECK-LABEL: @_ZN11HullLibrary15CleanupVerticesEjPK9btVector3jRjPS0_fRS0_(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 undef, label [[RETURN:%.*]], label [[IF_END:%.*]]
+; CHECK:       if.end:
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    br i1 undef, label [[IF_THEN17_1:%.*]], label [[IF_END22_1:%.*]]
+; CHECK:       for.end36:
+; CHECK-NEXT:    br label [[FOR_BODY144:%.*]]
+; CHECK:       for.body144:
+; CHECK-NEXT:    br i1 undef, label [[FOR_END227:%.*]], label [[FOR_BODY144]]
+; CHECK:       for.end227:
+; CHECK-NEXT:    br i1 undef, label [[FOR_END271:%.*]], label [[FOR_BODY233:%.*]]
+; CHECK:       for.body233:
+; CHECK-NEXT:    br i1 undef, label [[FOR_BODY233]], label [[FOR_END271]]
+; CHECK:       for.end271:
+; CHECK-NEXT:    [[TMP0:%.*]] = phi <2 x float> [ <float 0x47EFFFFFE0000000, float 0x47EFFFFFE0000000>, [[FOR_END227]] ], [ undef, [[FOR_BODY233]] ]
+; CHECK-NEXT:    [[TMP1:%.*]] = fsub <2 x float> undef, [[TMP0]]
+; CHECK-NEXT:    br i1 undef, label [[IF_THEN291:%.*]], label [[RETURN]]
+; CHECK:       if.then291:
+; CHECK-NEXT:    [[TMP2:%.*]] = fmul <2 x float> <float 5.000000e-01, float 5.000000e-01>, [[TMP1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = fadd <2 x float> [[TMP0]], [[TMP2]]
+; CHECK-NEXT:    br i1 undef, label [[IF_END332:%.*]], label [[IF_ELSE319:%.*]]
+; CHECK:       if.else319:
+; CHECK-NEXT:    br i1 undef, label [[IF_THEN325:%.*]], label [[IF_END327:%.*]]
+; CHECK:       if.then325:
+; CHECK-NEXT:    br label [[IF_END327]]
+; CHECK:       if.end327:
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x float> [[TMP1]], i32 0
+; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <2 x float> undef, float [[TMP4]], i32 0
+; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <2 x float> [[TMP5]], float undef, i32 1
+; CHECK-NEXT:    br i1 undef, label [[IF_THEN329:%.*]], label [[IF_END332]]
+; CHECK:       if.then329:
+; CHECK-NEXT:    br label [[IF_END332]]
+; CHECK:       if.end332:
+; CHECK-NEXT:    [[TMP7:%.*]] = phi <2 x float> [ [[TMP6]], [[IF_THEN329]] ], [ [[TMP6]], [[IF_END327]] ], [ <float 0x3F847AE140000000, float 0x3F847AE140000000>, [[IF_THEN291]] ]
+; CHECK-NEXT:    [[TMP8:%.*]] = fsub <2 x float> [[TMP3]], [[TMP7]]
+; CHECK-NEXT:    [[ARRAYIDX_I_I606:%.*]] = getelementptr inbounds [[CLASS_BTVECTOR3_23_221_463_485_507_573_595_683_727_749_815_837_991_1585_1607_1629_1651_1849_2047_2069_2091_2113:%.*]], %class.btVector3.23.221.463.485.507.573.595.683.727.749.815.837.991.1585.1607.1629.1651.1849.2047.2069.2091.2113* [[VERTICES:%.*]], i64 0, i32 0, i64 0
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast float* [[ARRAYIDX_I_I606]] to <2 x float>*
+; CHECK-NEXT:    store <2 x float> [[TMP8]], <2 x float>* [[TMP9]], align 4
+; CHECK-NEXT:    br label [[RETURN]]
+; CHECK:       return:
+; CHECK-NEXT:    ret void
+; CHECK:       if.then17.1:
+; CHECK-NEXT:    br label [[IF_END22_1]]
+; CHECK:       if.end22.1:
+; CHECK-NEXT:    br i1 undef, label [[IF_THEN17_2:%.*]], label [[IF_END22_2:%.*]]
+; CHECK:       if.then17.2:
+; CHECK-NEXT:    br label [[IF_END22_2]]
+; CHECK:       if.end22.2:
+; CHECK-NEXT:    br i1 undef, label [[FOR_END36:%.*]], label [[FOR_BODY]]
+;
 entry:
   br i1 undef, label %return, label %if.end
 
diff --git a/test/Transforms/SLPVectorizer/X86/crash_cmpop.ll b/test/Transforms/SLPVectorizer/X86/crash_cmpop.ll
index f10c8626d417099e0bbebeaefa5c52c7f622529c..7948a9154c8252435aa205e8e0fb3328a92d58d3 100644
--- a/test/Transforms/SLPVectorizer/X86/crash_cmpop.ll
+++ b/test/Transforms/SLPVectorizer/X86/crash_cmpop.ll
@@ -1,9 +1,91 @@
-; RUN: opt < %s -basicaa -slp-vectorizer -S
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -basicaa -slp-vectorizer -S | FileCheck %s -check-prefixes=CHECK,SSE
+; RUN: opt < %s -basicaa -slp-vectorizer -S -mattr=+avx | FileCheck %s -check-prefixes=CHECK,AVX
 
 target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.10.0"
 
 define void @testfunc(float* nocapture %dest, float* nocapture readonly %src) {
+; SSE-LABEL: @testfunc(
+; SSE-NEXT:  entry:
+; SSE-NEXT:    br label [[FOR_BODY:%.*]]
+; SSE:       for.body:
+; SSE-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
+; SSE-NEXT:    [[ACC1_056:%.*]] = phi float [ 0.000000e+00, [[ENTRY]] ], [ [[ADD13:%.*]], [[FOR_BODY]] ]
+; SSE-NEXT:    [[S1_055:%.*]] = phi float [ 0.000000e+00, [[ENTRY]] ], [ [[COND_I40:%.*]], [[FOR_BODY]] ]
+; SSE-NEXT:    [[S0_054:%.*]] = phi float [ 0.000000e+00, [[ENTRY]] ], [ [[COND_I44:%.*]], [[FOR_BODY]] ]
+; SSE-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, float* [[SRC:%.*]], i64 [[INDVARS_IV]]
+; SSE-NEXT:    [[TMP0:%.*]] = load float, float* [[ARRAYIDX]], align 4
+; SSE-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; SSE-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds float, float* [[DEST:%.*]], i64 [[INDVARS_IV]]
+; SSE-NEXT:    store float [[ACC1_056]], float* [[ARRAYIDX2]], align 4
+; SSE-NEXT:    [[ADD:%.*]] = fadd float [[S0_054]], [[TMP0]]
+; SSE-NEXT:    [[ADD3:%.*]] = fadd float [[S1_055]], [[TMP0]]
+; SSE-NEXT:    [[MUL:%.*]] = fmul float [[S0_054]], 0.000000e+00
+; SSE-NEXT:    [[ADD4:%.*]] = fadd float [[MUL]], [[ADD3]]
+; SSE-NEXT:    [[MUL5:%.*]] = fmul float [[S1_055]], 0.000000e+00
+; SSE-NEXT:    [[ADD6:%.*]] = fadd float [[MUL5]], [[ADD]]
+; SSE-NEXT:    [[CMP_I:%.*]] = fcmp olt float [[ADD6]], 1.000000e+00
+; SSE-NEXT:    [[COND_I:%.*]] = select i1 [[CMP_I]], float [[ADD6]], float 1.000000e+00
+; SSE-NEXT:    [[CMP_I51:%.*]] = fcmp olt float [[COND_I]], -1.000000e+00
+; SSE-NEXT:    [[CMP_I49:%.*]] = fcmp olt float [[ADD4]], 1.000000e+00
+; SSE-NEXT:    [[COND_I50:%.*]] = select i1 [[CMP_I49]], float [[ADD4]], float 1.000000e+00
+; SSE-NEXT:    [[CMP_I47:%.*]] = fcmp olt float [[COND_I50]], -1.000000e+00
+; SSE-NEXT:    [[COND_I_OP:%.*]] = fmul float [[COND_I]], 0.000000e+00
+; SSE-NEXT:    [[MUL10:%.*]] = select i1 [[CMP_I51]], float -0.000000e+00, float [[COND_I_OP]]
+; SSE-NEXT:    [[COND_I50_OP:%.*]] = fmul float [[COND_I50]], 0.000000e+00
+; SSE-NEXT:    [[MUL11:%.*]] = select i1 [[CMP_I47]], float -0.000000e+00, float [[COND_I50_OP]]
+; SSE-NEXT:    [[ADD13]] = fadd float [[MUL10]], [[MUL11]]
+; SSE-NEXT:    [[CMP_I45:%.*]] = fcmp olt float [[ADD13]], 1.000000e+00
+; SSE-NEXT:    [[COND_I46:%.*]] = select i1 [[CMP_I45]], float [[ADD13]], float 1.000000e+00
+; SSE-NEXT:    [[CMP_I43:%.*]] = fcmp olt float [[COND_I46]], -1.000000e+00
+; SSE-NEXT:    [[COND_I44]] = select i1 [[CMP_I43]], float -1.000000e+00, float [[COND_I46]]
+; SSE-NEXT:    [[CMP_I41:%.*]] = fcmp olt float [[MUL11]], 1.000000e+00
+; SSE-NEXT:    [[COND_I42:%.*]] = select i1 [[CMP_I41]], float [[MUL11]], float 1.000000e+00
+; SSE-NEXT:    [[CMP_I39:%.*]] = fcmp olt float [[COND_I42]], -1.000000e+00
+; SSE-NEXT:    [[COND_I40]] = select i1 [[CMP_I39]], float -1.000000e+00, float [[COND_I42]]
+; SSE-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 32
+; SSE-NEXT:    br i1 [[EXITCOND]], label [[FOR_END:%.*]], label [[FOR_BODY]]
+; SSE:       for.end:
+; SSE-NEXT:    ret void
+;
+; AVX-LABEL: @testfunc(
+; AVX-NEXT:  entry:
+; AVX-NEXT:    br label [[FOR_BODY:%.*]]
+; AVX:       for.body:
+; AVX-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
+; AVX-NEXT:    [[ACC1_056:%.*]] = phi float [ 0.000000e+00, [[ENTRY]] ], [ [[ADD13:%.*]], [[FOR_BODY]] ]
+; AVX-NEXT:    [[TMP0:%.*]] = phi <2 x float> [ zeroinitializer, [[ENTRY]] ], [ [[TMP19:%.*]], [[FOR_BODY]] ]
+; AVX-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, float* [[SRC:%.*]], i64 [[INDVARS_IV]]
+; AVX-NEXT:    [[TMP1:%.*]] = load float, float* [[ARRAYIDX]], align 4
+; AVX-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; AVX-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds float, float* [[DEST:%.*]], i64 [[INDVARS_IV]]
+; AVX-NEXT:    store float [[ACC1_056]], float* [[ARRAYIDX2]], align 4
+; AVX-NEXT:    [[REORDER_SHUFFLE:%.*]] = shufflevector <2 x float> [[TMP0]], <2 x float> undef, <2 x i32> <i32 1, i32 0>
+; AVX-NEXT:    [[TMP2:%.*]] = insertelement <2 x float> undef, float [[TMP1]], i32 0
+; AVX-NEXT:    [[TMP3:%.*]] = insertelement <2 x float> [[TMP2]], float [[TMP1]], i32 1
+; AVX-NEXT:    [[TMP4:%.*]] = fadd <2 x float> [[REORDER_SHUFFLE]], [[TMP3]]
+; AVX-NEXT:    [[TMP5:%.*]] = fmul <2 x float> zeroinitializer, [[TMP0]]
+; AVX-NEXT:    [[TMP6:%.*]] = fadd <2 x float> [[TMP5]], [[TMP4]]
+; AVX-NEXT:    [[TMP7:%.*]] = fcmp olt <2 x float> [[TMP6]], <float 1.000000e+00, float 1.000000e+00>
+; AVX-NEXT:    [[TMP8:%.*]] = select <2 x i1> [[TMP7]], <2 x float> [[TMP6]], <2 x float> <float 1.000000e+00, float 1.000000e+00>
+; AVX-NEXT:    [[TMP9:%.*]] = fcmp olt <2 x float> [[TMP8]], <float -1.000000e+00, float -1.000000e+00>
+; AVX-NEXT:    [[TMP10:%.*]] = fmul <2 x float> zeroinitializer, [[TMP8]]
+; AVX-NEXT:    [[TMP11:%.*]] = select <2 x i1> [[TMP9]], <2 x float> <float -0.000000e+00, float -0.000000e+00>, <2 x float> [[TMP10]]
+; AVX-NEXT:    [[TMP12:%.*]] = extractelement <2 x float> [[TMP11]], i32 0
+; AVX-NEXT:    [[TMP13:%.*]] = extractelement <2 x float> [[TMP11]], i32 1
+; AVX-NEXT:    [[ADD13]] = fadd float [[TMP12]], [[TMP13]]
+; AVX-NEXT:    [[TMP14:%.*]] = insertelement <2 x float> undef, float [[TMP13]], i32 0
+; AVX-NEXT:    [[TMP15:%.*]] = insertelement <2 x float> [[TMP14]], float [[ADD13]], i32 1
+; AVX-NEXT:    [[TMP16:%.*]] = fcmp olt <2 x float> [[TMP15]], <float 1.000000e+00, float 1.000000e+00>
+; AVX-NEXT:    [[TMP17:%.*]] = select <2 x i1> [[TMP16]], <2 x float> [[TMP15]], <2 x float> <float 1.000000e+00, float 1.000000e+00>
+; AVX-NEXT:    [[TMP18:%.*]] = fcmp olt <2 x float> [[TMP17]], <float -1.000000e+00, float -1.000000e+00>
+; AVX-NEXT:    [[TMP19]] = select <2 x i1> [[TMP18]], <2 x float> <float -1.000000e+00, float -1.000000e+00>, <2 x float> [[TMP17]]
+; AVX-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 32
+; AVX-NEXT:    br i1 [[EXITCOND]], label [[FOR_END:%.*]], label [[FOR_BODY]]
+; AVX:       for.end:
+; AVX-NEXT:    ret void
+;
 entry:
   br label %for.body
 
diff --git a/test/Transforms/SLPVectorizer/X86/crash_dequeue.ll b/test/Transforms/SLPVectorizer/X86/crash_dequeue.ll
index 28b7aa3c4de3ca298a4d53601525b227e209d06a..b5f736af62445f0efdf1ea3e822b0f4a7d629942 100644
--- a/test/Transforms/SLPVectorizer/X86/crash_dequeue.ll
+++ b/test/Transforms/SLPVectorizer/X86/crash_dequeue.ll
@@ -1,4 +1,5 @@
-; RUN: opt < %s -basicaa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -basicaa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7 | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.8.0"
@@ -6,6 +7,33 @@ target triple = "x86_64-apple-macosx10.8.0"
 
 ; Function Attrs: nounwind ssp uwtable
 define void @_ZSt6uniqueISt15_Deque_iteratorIdRdPdEET_S4_S4_(%"struct.std::_Deque_iterator.4.157.174.208.259.276.344.731"* %__first, %"struct.std::_Deque_iterator.4.157.174.208.259.276.344.731"* nocapture %__last) {
+; CHECK-LABEL: @_ZSt6uniqueISt15_Deque_iteratorIdRdPdEET_S4_S4_(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[_M_CUR2_I_I:%.*]] = getelementptr inbounds %"struct.std::_Deque_iterator.4.157.174.208.259.276.344.731", %"struct.std::_Deque_iterator.4.157.174.208.259.276.344.731"* [[__FIRST:%.*]], i64 0, i32 0
+; CHECK-NEXT:    [[TMP0:%.*]] = load double*, double** [[_M_CUR2_I_I]], align 8
+; CHECK-NEXT:    [[_M_FIRST3_I_I:%.*]] = getelementptr inbounds %"struct.std::_Deque_iterator.4.157.174.208.259.276.344.731", %"struct.std::_Deque_iterator.4.157.174.208.259.276.344.731"* [[__FIRST]], i64 0, i32 1
+; CHECK-NEXT:    [[_M_CUR2_I_I81:%.*]] = getelementptr inbounds %"struct.std::_Deque_iterator.4.157.174.208.259.276.344.731", %"struct.std::_Deque_iterator.4.157.174.208.259.276.344.731"* [[__LAST:%.*]], i64 0, i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = load double*, double** [[_M_CUR2_I_I81]], align 8
+; CHECK-NEXT:    [[_M_FIRST3_I_I83:%.*]] = getelementptr inbounds %"struct.std::_Deque_iterator.4.157.174.208.259.276.344.731", %"struct.std::_Deque_iterator.4.157.174.208.259.276.344.731"* [[__LAST]], i64 0, i32 1
+; CHECK-NEXT:    [[TMP2:%.*]] = load double*, double** [[_M_FIRST3_I_I83]], align 8
+; CHECK-NEXT:    br i1 undef, label [[_ZST13ADJACENT_FINDIST15_DEQUE_ITERATORIDRDPDEET_S4_S4__EXIT:%.*]], label [[WHILE_COND_I_PREHEADER:%.*]]
+; CHECK:       while.cond.i.preheader:
+; CHECK-NEXT:    br label [[WHILE_COND_I:%.*]]
+; CHECK:       while.cond.i:
+; CHECK-NEXT:    br i1 undef, label [[_ZST13ADJACENT_FINDIST15_DEQUE_ITERATORIDRDPDEET_S4_S4__EXIT]], label [[WHILE_BODY_I:%.*]]
+; CHECK:       while.body.i:
+; CHECK-NEXT:    br i1 undef, label [[_ZST13ADJACENT_FINDIST15_DEQUE_ITERATORIDRDPDEET_S4_S4__EXIT]], label [[WHILE_COND_I]]
+; CHECK:       _ZSt13adjacent_findISt15_Deque_iteratorIdRdPdEET_S4_S4_.exit:
+; CHECK-NEXT:    [[TMP3:%.*]] = phi double* [ [[TMP2]], [[ENTRY:%.*]] ], [ [[TMP2]], [[WHILE_COND_I]] ], [ undef, [[WHILE_BODY_I]] ]
+; CHECK-NEXT:    [[TMP4:%.*]] = phi double* [ [[TMP0]], [[ENTRY]] ], [ [[TMP1]], [[WHILE_COND_I]] ], [ undef, [[WHILE_BODY_I]] ]
+; CHECK-NEXT:    store double* [[TMP4]], double** [[_M_CUR2_I_I]], align 8
+; CHECK-NEXT:    store double* [[TMP3]], double** [[_M_FIRST3_I_I]], align 8
+; CHECK-NEXT:    br i1 undef, label [[IF_THEN_I55:%.*]], label [[WHILE_COND:%.*]]
+; CHECK:       if.then.i55:
+; CHECK-NEXT:    br label [[WHILE_COND]]
+; CHECK:       while.cond:
+; CHECK-NEXT:    br label [[WHILE_COND]]
+;
 entry:
   %_M_cur2.i.i = getelementptr inbounds %"struct.std::_Deque_iterator.4.157.174.208.259.276.344.731", %"struct.std::_Deque_iterator.4.157.174.208.259.276.344.731"* %__first, i64 0, i32 0
   %0 = load double*, double** %_M_cur2.i.i, align 8
diff --git a/test/Transforms/SLPVectorizer/X86/crash_flop7.ll b/test/Transforms/SLPVectorizer/X86/crash_flop7.ll
index e11be488f795b23a98b578dddaa6eb9c5682d819..d149c27b7c7ee3342db619d00e57cf944320de4e 100644
--- a/test/Transforms/SLPVectorizer/X86/crash_flop7.ll
+++ b/test/Transforms/SLPVectorizer/X86/crash_flop7.ll
@@ -1,10 +1,41 @@
-; RUN: opt < %s -basicaa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -basicaa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7 | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.8.0"
 
 ; Function Attrs: nounwind ssp uwtable
 define void @main() #0 {
+; CHECK-LABEL: @main(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 undef, label [[WHILE_BODY:%.*]], label [[WHILE_END:%.*]]
+; CHECK:       while.body:
+; CHECK-NEXT:    unreachable
+; CHECK:       while.end:
+; CHECK-NEXT:    br i1 undef, label [[FOR_END80:%.*]], label [[FOR_BODY75_LR_PH:%.*]]
+; CHECK:       for.body75.lr.ph:
+; CHECK-NEXT:    br label [[FOR_BODY75:%.*]]
+; CHECK:       for.body75:
+; CHECK-NEXT:    br label [[FOR_BODY75]]
+; CHECK:       for.end80:
+; CHECK-NEXT:    br i1 undef, label [[FOR_END300:%.*]], label [[FOR_BODY267_LR_PH:%.*]]
+; CHECK:       for.body267.lr.ph:
+; CHECK-NEXT:    br label [[FOR_BODY267:%.*]]
+; CHECK:       for.body267:
+; CHECK-NEXT:    [[S_71010:%.*]] = phi double [ 0.000000e+00, [[FOR_BODY267_LR_PH]] ], [ [[ADD297:%.*]], [[FOR_BODY267]] ]
+; CHECK-NEXT:    [[MUL269:%.*]] = fmul double undef, undef
+; CHECK-NEXT:    [[MUL270:%.*]] = fmul double [[MUL269]], [[MUL269]]
+; CHECK-NEXT:    [[ADD282:%.*]] = fadd double undef, undef
+; CHECK-NEXT:    [[MUL283:%.*]] = fmul double [[MUL269]], [[ADD282]]
+; CHECK-NEXT:    [[ADD293:%.*]] = fadd double undef, undef
+; CHECK-NEXT:    [[MUL294:%.*]] = fmul double [[MUL270]], [[ADD293]]
+; CHECK-NEXT:    [[ADD295:%.*]] = fadd double undef, [[MUL294]]
+; CHECK-NEXT:    [[DIV296:%.*]] = fdiv double [[MUL283]], [[ADD295]]
+; CHECK-NEXT:    [[ADD297]] = fadd double [[S_71010]], [[DIV296]]
+; CHECK-NEXT:    br i1 undef, label [[FOR_BODY267]], label [[FOR_END300]]
+; CHECK:       for.end300:
+; CHECK-NEXT:    unreachable
+;
 entry:
   br i1 undef, label %while.body, label %while.end
 
diff --git a/test/Transforms/SLPVectorizer/X86/crash_gep.ll b/test/Transforms/SLPVectorizer/X86/crash_gep.ll
index bd1e8f7cc19d010a4c7a6c496175707e5ed7d302..eca21e7d3a1a01c5b931c8f2eb6f61bb058ca108 100644
--- a/test/Transforms/SLPVectorizer/X86/crash_gep.ll
+++ b/test/Transforms/SLPVectorizer/X86/crash_gep.ll
@@ -1,4 +1,5 @@
-; RUN: opt < %s -basicaa -slp-vectorizer -dce -S -mtriple=x86_64-unknown-linux-gnu
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -basicaa -slp-vectorizer -dce -S -mtriple=x86_64-unknown-linux-gnu | FileCheck %s
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
@@ -7,6 +8,17 @@ target triple = "x86_64-unknown-linux-gnu"
 
 ; Function Attrs: nounwind uwtable
 define i32 @fn1() {
+; CHECK-LABEL: @fn1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i64*, i64** @a, align 8
+; CHECK-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds i64, i64* [[TMP0]], i64 1
+; CHECK-NEXT:    [[TMP1:%.*]] = ptrtoint i64* [[ADD_PTR]] to i64
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i64, i64* [[TMP0]], i64 2
+; CHECK-NEXT:    store i64 [[TMP1]], i64* [[ARRAYIDX]], align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = ptrtoint i64* [[ARRAYIDX]] to i64
+; CHECK-NEXT:    store i64 [[TMP2]], i64* [[ADD_PTR]], align 8
+; CHECK-NEXT:    ret i32 undef
+;
 entry:
   %0 = load i64*, i64** @a, align 8
   %add.ptr = getelementptr inbounds i64, i64* %0, i64 1
diff --git a/test/Transforms/SLPVectorizer/X86/crash_lencod.ll b/test/Transforms/SLPVectorizer/X86/crash_lencod.ll
index 70b13fd75f14912304c820b17fb0be45bcd1545f..eb1cb32a6387177546c095068fbb223441edfea7 100644
--- a/test/Transforms/SLPVectorizer/X86/crash_lencod.ll
+++ b/test/Transforms/SLPVectorizer/X86/crash_lencod.ll
@@ -1,10 +1,45 @@
-; RUN: opt < %s -basicaa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -basicaa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7 | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.8.0"
 
 ; Function Attrs: nounwind ssp uwtable
 define void @RCModelEstimator() {
+; CHECK-LABEL: @RCModelEstimator(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 undef, label [[FOR_BODY_LR_PH:%.*]], label [[FOR_END_THREAD:%.*]]
+; CHECK:       for.end.thread:
+; CHECK-NEXT:    unreachable
+; CHECK:       for.body.lr.ph:
+; CHECK-NEXT:    br i1 undef, label [[FOR_END:%.*]], label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    br i1 undef, label [[FOR_END]], label [[FOR_BODY]]
+; CHECK:       for.end:
+; CHECK-NEXT:    br i1 undef, label [[FOR_BODY3:%.*]], label [[IF_END103:%.*]]
+; CHECK:       for.cond14.preheader:
+; CHECK-NEXT:    br i1 undef, label [[FOR_BODY16_LR_PH:%.*]], label [[IF_END103]]
+; CHECK:       for.body16.lr.ph:
+; CHECK-NEXT:    br label [[FOR_BODY16:%.*]]
+; CHECK:       for.body3:
+; CHECK-NEXT:    br i1 undef, label [[IF_THEN7:%.*]], label [[FOR_INC11:%.*]]
+; CHECK:       if.then7:
+; CHECK-NEXT:    br label [[FOR_INC11]]
+; CHECK:       for.inc11:
+; CHECK-NEXT:    br i1 false, label [[FOR_COND14_PREHEADER:%.*]], label [[FOR_BODY3]]
+; CHECK:       for.body16:
+; CHECK-NEXT:    br i1 undef, label [[FOR_END39:%.*]], label [[FOR_BODY16]]
+; CHECK:       for.end39:
+; CHECK-NEXT:    br i1 undef, label [[IF_END103]], label [[FOR_COND45_PREHEADER:%.*]]
+; CHECK:       for.cond45.preheader:
+; CHECK-NEXT:    br i1 undef, label [[IF_THEN88:%.*]], label [[IF_ELSE:%.*]]
+; CHECK:       if.then88:
+; CHECK-NEXT:    br label [[IF_END103]]
+; CHECK:       if.else:
+; CHECK-NEXT:    br label [[IF_END103]]
+; CHECK:       if.end103:
+; CHECK-NEXT:    ret void
+;
 entry:
   br i1 undef, label %for.body.lr.ph, label %for.end.thread
 
@@ -66,6 +101,17 @@ if.end103:                                        ; preds = %if.else, %if.then88
 
 
 define void @intrapred_luma() {
+; CHECK-LABEL: @intrapred_luma(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CONV153:%.*]] = trunc i32 undef to i16
+; CHECK-NEXT:    [[ARRAYIDX154:%.*]] = getelementptr inbounds [13 x i16], [13 x i16]* undef, i64 0, i64 12
+; CHECK-NEXT:    store i16 [[CONV153]], i16* [[ARRAYIDX154]], align 8
+; CHECK-NEXT:    [[ARRAYIDX155:%.*]] = getelementptr inbounds [13 x i16], [13 x i16]* undef, i64 0, i64 11
+; CHECK-NEXT:    store i16 [[CONV153]], i16* [[ARRAYIDX155]], align 2
+; CHECK-NEXT:    [[ARRAYIDX156:%.*]] = getelementptr inbounds [13 x i16], [13 x i16]* undef, i64 0, i64 10
+; CHECK-NEXT:    store i16 [[CONV153]], i16* [[ARRAYIDX156]], align 4
+; CHECK-NEXT:    ret void
+;
 entry:
   %conv153 = trunc i32 undef to i16
   %arrayidx154 = getelementptr inbounds [13 x i16], [13 x i16]* undef, i64 0, i64 12
@@ -78,6 +124,18 @@ entry:
 }
 
 define fastcc void @dct36(double* %inbuf) {
+; CHECK-LABEL: @dct36(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[ARRAYIDX41:%.*]] = getelementptr inbounds double, double* [[INBUF:%.*]], i64 2
+; CHECK-NEXT:    [[ARRAYIDX44:%.*]] = getelementptr inbounds double, double* [[INBUF]], i64 1
+; CHECK-NEXT:    [[TMP0:%.*]] = load double, double* [[ARRAYIDX44]], align 8
+; CHECK-NEXT:    [[ADD46:%.*]] = fadd double [[TMP0]], undef
+; CHECK-NEXT:    store double [[ADD46]], double* [[ARRAYIDX41]], align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = load double, double* [[INBUF]], align 8
+; CHECK-NEXT:    [[ADD49:%.*]] = fadd double [[TMP1]], [[TMP0]]
+; CHECK-NEXT:    store double [[ADD49]], double* [[ARRAYIDX44]], align 8
+; CHECK-NEXT:    ret void
+;
 entry:
   %arrayidx41 = getelementptr inbounds double, double* %inbuf, i64 2
   %arrayidx44 = getelementptr inbounds double, double* %inbuf, i64 1
diff --git a/test/Transforms/SLPVectorizer/X86/crash_mandeltext.ll b/test/Transforms/SLPVectorizer/X86/crash_mandeltext.ll
index f82343fb433f503f051937ae27c3a26fdee5cd4d..f12de2ad199c44f02c9fbf3305a3fbb3ff3b686d 100644
--- a/test/Transforms/SLPVectorizer/X86/crash_mandeltext.ll
+++ b/test/Transforms/SLPVectorizer/X86/crash_mandeltext.ll
@@ -1,9 +1,46 @@
-; RUN: opt < %s -basicaa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -basicaa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7 | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.8.0"
 
 define void @main() {
+; CHECK-LABEL: @main(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    br label [[FOR_COND4_PREHEADER:%.*]]
+; CHECK:       for.cond4.preheader:
+; CHECK-NEXT:    br label [[FOR_BODY6:%.*]]
+; CHECK:       for.body6:
+; CHECK-NEXT:    br label [[FOR_BODY12:%.*]]
+; CHECK:       for.body12:
+; CHECK-NEXT:    [[FZIMG_069:%.*]] = phi double [ undef, [[FOR_BODY6]] ], [ [[ADD19:%.*]], [[IF_END:%.*]] ]
+; CHECK-NEXT:    [[FZREAL_068:%.*]] = phi double [ undef, [[FOR_BODY6]] ], [ [[ADD20:%.*]], [[IF_END]] ]
+; CHECK-NEXT:    [[MUL13:%.*]] = fmul double [[FZREAL_068]], [[FZREAL_068]]
+; CHECK-NEXT:    [[MUL14:%.*]] = fmul double [[FZIMG_069]], [[FZIMG_069]]
+; CHECK-NEXT:    [[ADD15:%.*]] = fadd double [[MUL13]], [[MUL14]]
+; CHECK-NEXT:    [[CMP16:%.*]] = fcmp ogt double [[ADD15]], 4.000000e+00
+; CHECK-NEXT:    br i1 [[CMP16]], label [[FOR_INC21:%.*]], label [[IF_END]]
+; CHECK:       if.end:
+; CHECK-NEXT:    [[MUL18:%.*]] = fmul double undef, [[FZIMG_069]]
+; CHECK-NEXT:    [[ADD19]] = fadd double undef, [[MUL18]]
+; CHECK-NEXT:    [[SUB:%.*]] = fsub double [[MUL13]], [[MUL14]]
+; CHECK-NEXT:    [[ADD20]] = fadd double undef, [[SUB]]
+; CHECK-NEXT:    br i1 undef, label [[FOR_BODY12]], label [[FOR_INC21]]
+; CHECK:       for.inc21:
+; CHECK-NEXT:    br i1 undef, label [[FOR_END23:%.*]], label [[FOR_BODY6]]
+; CHECK:       for.end23:
+; CHECK-NEXT:    br i1 undef, label [[IF_THEN25:%.*]], label [[IF_THEN26:%.*]]
+; CHECK:       if.then25:
+; CHECK-NEXT:    br i1 undef, label [[FOR_END44:%.*]], label [[FOR_COND4_PREHEADER]]
+; CHECK:       if.then26:
+; CHECK-NEXT:    unreachable
+; CHECK:       for.end44:
+; CHECK-NEXT:    br i1 undef, label [[FOR_END48:%.*]], label [[FOR_BODY]]
+; CHECK:       for.end48:
+; CHECK-NEXT:    ret void
+;
 entry:
   br label %for.body
 
@@ -54,6 +91,26 @@ for.end48:                                        ; preds = %for.end44
 %struct.hoge = type { double, double, double}
 
 define void @zot(%struct.hoge* %arg) {
+; CHECK-LABEL: @zot(
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    [[TMP:%.*]] = load double, double* undef, align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = load double, double* undef, align 8
+; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <2 x double> undef, double [[TMP2]], i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x double> [[TMP0]], double [[TMP]], i32 1
+; CHECK-NEXT:    [[TMP2:%.*]] = fsub <2 x double> [[TMP1]], undef
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds [[STRUCT_HOGE:%.*]], %struct.hoge* [[ARG:%.*]], i64 0, i32 1
+; CHECK-NEXT:    [[TMP3:%.*]] = fmul <2 x double> undef, [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = fsub <2 x double> [[TMP3]], undef
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast double* [[TMP7]] to <2 x double>*
+; CHECK-NEXT:    store <2 x double> [[TMP4]], <2 x double>* [[TMP5]], align 8
+; CHECK-NEXT:    br i1 undef, label [[BB11:%.*]], label [[BB12:%.*]]
+; CHECK:       bb11:
+; CHECK-NEXT:    br label [[BB14:%.*]]
+; CHECK:       bb12:
+; CHECK-NEXT:    br label [[BB14]]
+; CHECK:       bb14:
+; CHECK-NEXT:    ret void
+;
 bb:
   %tmp = load double, double* undef, align 8
   %tmp1 = fsub double %tmp, undef
@@ -85,6 +142,22 @@ bb14:                                             ; preds = %bb12, %bb11
 %struct.rc4_state.0.24 = type { i32, i32, [256 x i32] }
 
 define void @rc4_crypt(%struct.rc4_state.0.24* nocapture %s) {
+; CHECK-LABEL: @rc4_crypt(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[X1:%.*]] = getelementptr inbounds [[STRUCT_RC4_STATE_0_24:%.*]], %struct.rc4_state.0.24* [[S:%.*]], i64 0, i32 0
+; CHECK-NEXT:    [[Y2:%.*]] = getelementptr inbounds [[STRUCT_RC4_STATE_0_24]], %struct.rc4_state.0.24* [[S]], i64 0, i32 1
+; CHECK-NEXT:    br i1 undef, label [[FOR_BODY:%.*]], label [[FOR_END:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[CONV4:%.*]] = and i32 undef, 255
+; CHECK-NEXT:    [[CONV7:%.*]] = and i32 undef, 255
+; CHECK-NEXT:    br i1 undef, label [[FOR_END]], label [[FOR_BODY]]
+; CHECK:       for.end:
+; CHECK-NEXT:    [[X_0_LCSSA:%.*]] = phi i32 [ undef, [[ENTRY:%.*]] ], [ [[CONV4]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[Y_0_LCSSA:%.*]] = phi i32 [ undef, [[ENTRY]] ], [ [[CONV7]], [[FOR_BODY]] ]
+; CHECK-NEXT:    store i32 [[X_0_LCSSA]], i32* [[X1]], align 4
+; CHECK-NEXT:    store i32 [[Y_0_LCSSA]], i32* [[Y2]], align 4
+; CHECK-NEXT:    ret void
+;
 entry:
   %x1 = getelementptr inbounds %struct.rc4_state.0.24, %struct.rc4_state.0.24* %s, i64 0, i32 0
   %y2 = getelementptr inbounds %struct.rc4_state.0.24, %struct.rc4_state.0.24* %s, i64 0, i32 1
diff --git a/test/Transforms/SLPVectorizer/X86/crash_netbsd_decompress.ll b/test/Transforms/SLPVectorizer/X86/crash_netbsd_decompress.ll
index e1df98d83bdbded0a63182422addefb4d727cd06..a52ec6ba9de005bf804b1bea8b59ee9237b3bac7 100644
--- a/test/Transforms/SLPVectorizer/X86/crash_netbsd_decompress.ll
+++ b/test/Transforms/SLPVectorizer/X86/crash_netbsd_decompress.ll
@@ -1,4 +1,5 @@
-; RUN: opt < %s -basicaa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -basicaa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7 | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.8.0"
@@ -12,6 +13,30 @@ target triple = "x86_64-apple-macosx10.8.0"
 @e = common global i32 0, align 4
 
 define i32 @fn1() {
+; CHECK-LABEL: @fn1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* getelementptr inbounds (%struct.DState, %struct.DState* @b, i32 0, i32 0), align 4
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, i32* getelementptr inbounds (%struct.DState, %struct.DState* @b, i32 0, i32 1), align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, i32* @d, align 4
+; CHECK-NEXT:    [[COND:%.*]] = icmp eq i32 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[COND]], label [[SW_BB:%.*]], label [[SAVE_STATE_AND_RETURN:%.*]]
+; CHECK:       sw.bb:
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, i32* @c, align 4
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[TMP3]], 7
+; CHECK-NEXT:    store i32 [[AND]], i32* @a, align 4
+; CHECK-NEXT:    switch i32 [[AND]], label [[IF_END:%.*]] [
+; CHECK-NEXT:    i32 7, label [[SAVE_STATE_AND_RETURN]]
+; CHECK-NEXT:    i32 0, label [[SAVE_STATE_AND_RETURN]]
+; CHECK-NEXT:    ]
+; CHECK:       if.end:
+; CHECK-NEXT:    br label [[SAVE_STATE_AND_RETURN]]
+; CHECK:       save_state_and_return:
+; CHECK-NEXT:    [[T_0:%.*]] = phi i32 [ 0, [[IF_END]] ], [ [[TMP0]], [[ENTRY:%.*]] ], [ [[TMP0]], [[SW_BB]] ], [ [[TMP0]], [[SW_BB]] ]
+; CHECK-NEXT:    [[F_0:%.*]] = phi i32 [ 0, [[IF_END]] ], [ [[TMP1]], [[ENTRY]] ], [ 0, [[SW_BB]] ], [ 0, [[SW_BB]] ]
+; CHECK-NEXT:    store i32 [[T_0]], i32* getelementptr inbounds (%struct.DState, %struct.DState* @b, i32 0, i32 0), align 4
+; CHECK-NEXT:    store i32 [[F_0]], i32* getelementptr inbounds (%struct.DState, %struct.DState* @b, i32 0, i32 1), align 4
+; CHECK-NEXT:    ret i32 undef
+;
 entry:
   %0 = load i32, i32* getelementptr inbounds (%struct.DState, %struct.DState* @b, i32 0, i32 0), align 4
   %1 = load i32, i32* getelementptr inbounds (%struct.DState, %struct.DState* @b, i32 0, i32 1), align 4
@@ -24,8 +49,8 @@ sw.bb:                                            ; preds = %entry
   %and = and i32 %3, 7
   store i32 %and, i32* @a, align 4
   switch i32 %and, label %if.end [
-    i32 7, label %save_state_and_return
-    i32 0, label %save_state_and_return
+  i32 7, label %save_state_and_return
+  i32 0, label %save_state_and_return
   ]
 
 if.end:                                           ; preds = %sw.bb
diff --git a/test/Transforms/SLPVectorizer/X86/crash_scheduling.ll b/test/Transforms/SLPVectorizer/X86/crash_scheduling.ll
index 916772ca8ccd55422c4eac3563ac90ca86c571af..9108e84a3a64da6cd38971f0aa577a910e9c8467 100644
--- a/test/Transforms/SLPVectorizer/X86/crash_scheduling.ll
+++ b/test/Transforms/SLPVectorizer/X86/crash_scheduling.ll
@@ -1,9 +1,41 @@
-; RUN: opt < %s -basicaa -disable-verify -slp-vectorizer -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -basicaa -disable-verify -slp-vectorizer -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7 | FileCheck %s
 
 target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-darwin13.3.0"
 
 define void @_foo(double %p1, double %p2, double %p3) #0 {
+; CHECK-LABEL: @_foo(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TAB1:%.*]] = alloca [256 x i32], align 16
+; CHECK-NEXT:    [[TAB2:%.*]] = alloca [256 x i32], align 16
+; CHECK-NEXT:    br label [[BB1:%.*]]
+; CHECK:       bb1:
+; CHECK-NEXT:    [[MUL19:%.*]] = fmul double [[P1:%.*]], 1.638400e+04
+; CHECK-NEXT:    [[MUL20:%.*]] = fmul double [[P3:%.*]], 1.638400e+04
+; CHECK-NEXT:    [[ADD:%.*]] = fadd double [[MUL20]], 8.192000e+03
+; CHECK-NEXT:    [[MUL21:%.*]] = fmul double [[P2:%.*]], 1.638400e+04
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[INDVARS_IV266:%.*]] = phi i64 [ 0, [[BB1]] ], [ [[INDVARS_IV_NEXT267:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[T_0259:%.*]] = phi double [ 0.000000e+00, [[BB1]] ], [ [[ADD27:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[P3_ADDR_0258:%.*]] = phi double [ [[ADD]], [[BB1]] ], [ [[ADD28:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[VECINIT_I_I237:%.*]] = insertelement <2 x double> undef, double [[T_0259]], i32 0
+; CHECK-NEXT:    [[X13:%.*]] = tail call i32 @_xfn(<2 x double> [[VECINIT_I_I237]])
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [256 x i32], [256 x i32]* [[TAB1]], i64 0, i64 [[INDVARS_IV266]]
+; CHECK-NEXT:    store i32 [[X13]], i32* [[ARRAYIDX]], align 4, !tbaa !0
+; CHECK-NEXT:    [[VECINIT_I_I:%.*]] = insertelement <2 x double> undef, double [[P3_ADDR_0258]], i32 0
+; CHECK-NEXT:    [[X14:%.*]] = tail call i32 @_xfn(<2 x double> [[VECINIT_I_I]])
+; CHECK-NEXT:    [[ARRAYIDX26:%.*]] = getelementptr inbounds [256 x i32], [256 x i32]* [[TAB2]], i64 0, i64 [[INDVARS_IV266]]
+; CHECK-NEXT:    store i32 [[X14]], i32* [[ARRAYIDX26]], align 4, !tbaa !0
+; CHECK-NEXT:    [[ADD27]] = fadd double [[MUL19]], [[T_0259]]
+; CHECK-NEXT:    [[ADD28]] = fadd double [[MUL21]], [[P3_ADDR_0258]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT267]] = add nuw nsw i64 [[INDVARS_IV266]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT267]], 256
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[RETURN:%.*]], label [[FOR_BODY]]
+; CHECK:       return:
+; CHECK-NEXT:    ret void
+;
 entry:
   %tab1 = alloca [256 x i32], align 16
   %tab2 = alloca [256 x i32], align 16
diff --git a/test/Transforms/SLPVectorizer/X86/crash_sim4b1.ll b/test/Transforms/SLPVectorizer/X86/crash_sim4b1.ll
index 5a576c2a7222ebcbe3226855d4383fc1dbc4d41c..8e4c282ecdc228408f7f4ab1cba715837221347d 100644
--- a/test/Transforms/SLPVectorizer/X86/crash_sim4b1.ll
+++ b/test/Transforms/SLPVectorizer/X86/crash_sim4b1.ll
@@ -1,4 +1,5 @@
-; RUN: opt < %s -basicaa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -basicaa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7 | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.8.0"
@@ -6,6 +7,80 @@ target triple = "x86_64-apple-macosx10.8.0"
 %struct._exon_t.12.103.220.363.480.649.740.857.1039.1065.1078.1091.1117.1130.1156.1169.1195.1221.1234.1286.1299.1312.1338.1429.1455.1468.1494.1520.1884.1897.1975.2066.2105.2170.2171 = type { i32, i32, i32, i32, i32, i32, [8 x i8] }
 
 define void @SIM4() {
+; CHECK-LABEL: @SIM4(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 undef, label [[RETURN:%.*]], label [[LOR_LHS_FALSE:%.*]]
+; CHECK:       lor.lhs.false:
+; CHECK-NEXT:    br i1 undef, label [[RETURN]], label [[IF_END:%.*]]
+; CHECK:       if.end:
+; CHECK-NEXT:    br i1 undef, label [[FOR_END605:%.*]], label [[FOR_BODY_LR_PH:%.*]]
+; CHECK:       for.body.lr.ph:
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    br i1 undef, label [[FOR_INC603:%.*]], label [[IF_END12:%.*]]
+; CHECK:       if.end12:
+; CHECK-NEXT:    br i1 undef, label [[LAND_LHS_TRUE:%.*]], label [[LAND_LHS_TRUE167:%.*]]
+; CHECK:       land.lhs.true:
+; CHECK-NEXT:    br i1 undef, label [[IF_THEN17:%.*]], label [[LAND_LHS_TRUE167]]
+; CHECK:       if.then17:
+; CHECK-NEXT:    br i1 undef, label [[IF_END98:%.*]], label [[LAND_RHS_LR_PH:%.*]]
+; CHECK:       land.rhs.lr.ph:
+; CHECK-NEXT:    unreachable
+; CHECK:       if.end98:
+; CHECK-NEXT:    br i1 undef, label [[LAND_LHS_TRUE167]], label [[IF_THEN103:%.*]]
+; CHECK:       if.then103:
+; CHECK-NEXT:    [[DOTSUB100:%.*]] = select i1 undef, i32 250, i32 undef
+; CHECK-NEXT:    [[MUL114:%.*]] = shl nsw i32 [[DOTSUB100]], 2
+; CHECK-NEXT:    [[FROM1115:%.*]] = getelementptr inbounds [[STRUCT__EXON_T_12_103_220_363_480_649_740_857_1039_1065_1078_1091_1117_1130_1156_1169_1195_1221_1234_1286_1299_1312_1338_1429_1455_1468_1494_1520_1884_1897_1975_2066_2105_2170_2171:%.*]], %struct._exon_t.12.103.220.363.480.649.740.857.1039.1065.1078.1091.1117.1130.1156.1169.1195.1221.1234.1286.1299.1312.1338.1429.1455.1468.1494.1520.1884.1897.1975.2066.2105.2170.2171* undef, i64 0, i32 0
+; CHECK-NEXT:    [[COND125:%.*]] = select i1 undef, i32 undef, i32 [[MUL114]]
+; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <2 x i32> undef, i32 [[COND125]], i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x i32> [[TMP0]], i32 [[DOTSUB100]], i32 1
+; CHECK-NEXT:    br label [[FOR_COND_I:%.*]]
+; CHECK:       for.cond.i:
+; CHECK-NEXT:    [[TMP2:%.*]] = phi <2 x i32> [ undef, [[LAND_RHS_I874:%.*]] ], [ [[TMP1]], [[IF_THEN103]] ]
+; CHECK-NEXT:    br i1 undef, label [[LAND_RHS_I874]], label [[FOR_END_I:%.*]]
+; CHECK:       land.rhs.i874:
+; CHECK-NEXT:    br i1 undef, label [[FOR_COND_I]], label [[FOR_END_I]]
+; CHECK:       for.end.i:
+; CHECK-NEXT:    br i1 undef, label [[IF_THEN_I:%.*]], label [[IF_END_I:%.*]]
+; CHECK:       if.then.i:
+; CHECK-NEXT:    [[TMP3:%.*]] = add nsw <2 x i32> undef, [[TMP2]]
+; CHECK-NEXT:    br label [[EXTEND_BW_EXIT:%.*]]
+; CHECK:       if.end.i:
+; CHECK-NEXT:    [[ADD16_I:%.*]] = add i32 [[COND125]], [[DOTSUB100]]
+; CHECK-NEXT:    [[CMP26514_I:%.*]] = icmp slt i32 [[ADD16_I]], 0
+; CHECK-NEXT:    br i1 [[CMP26514_I]], label [[FOR_END33_I:%.*]], label [[FOR_BODY28_LR_PH_I:%.*]]
+; CHECK:       for.body28.lr.ph.i:
+; CHECK-NEXT:    br label [[FOR_END33_I]]
+; CHECK:       for.end33.i:
+; CHECK-NEXT:    br i1 undef, label [[FOR_END58_I:%.*]], label [[FOR_BODY52_LR_PH_I:%.*]]
+; CHECK:       for.body52.lr.ph.i:
+; CHECK-NEXT:    br label [[FOR_END58_I]]
+; CHECK:       for.end58.i:
+; CHECK-NEXT:    br label [[WHILE_COND260_I:%.*]]
+; CHECK:       while.cond260.i:
+; CHECK-NEXT:    br i1 undef, label [[LAND_RHS263_I:%.*]], label [[WHILE_END275_I:%.*]]
+; CHECK:       land.rhs263.i:
+; CHECK-NEXT:    br i1 undef, label [[WHILE_COND260_I]], label [[WHILE_END275_I]]
+; CHECK:       while.end275.i:
+; CHECK-NEXT:    br label [[EXTEND_BW_EXIT]]
+; CHECK:       extend_bw.exit:
+; CHECK-NEXT:    [[TMP4:%.*]] = phi <2 x i32> [ [[TMP3]], [[IF_THEN_I]] ], [ undef, [[WHILE_END275_I]] ]
+; CHECK-NEXT:    br i1 false, label [[IF_THEN157:%.*]], label [[LAND_LHS_TRUE167]]
+; CHECK:       if.then157:
+; CHECK-NEXT:    [[TMP5:%.*]] = add nsw <2 x i32> <i32 1, i32 1>, [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i32* [[FROM1115]] to <2 x i32>*
+; CHECK-NEXT:    store <2 x i32> [[TMP5]], <2 x i32>* [[TMP6]], align 4
+; CHECK-NEXT:    br label [[LAND_LHS_TRUE167]]
+; CHECK:       land.lhs.true167:
+; CHECK-NEXT:    unreachable
+; CHECK:       for.inc603:
+; CHECK-NEXT:    br i1 undef, label [[FOR_BODY]], label [[FOR_END605]]
+; CHECK:       for.end605:
+; CHECK-NEXT:    unreachable
+; CHECK:       return:
+; CHECK-NEXT:    ret void
+;
 entry:
   br i1 undef, label %return, label %lor.lhs.false
 
diff --git a/test/Transforms/SLPVectorizer/X86/crash_smallpt.ll b/test/Transforms/SLPVectorizer/X86/crash_smallpt.ll
index 273584c11a9c5aa9150f9568b6119da5483c1e6f..e2d36376f5ea78526c099421f5793cb7644ac475 100644
--- a/test/Transforms/SLPVectorizer/X86/crash_smallpt.ll
+++ b/test/Transforms/SLPVectorizer/X86/crash_smallpt.ll
@@ -1,4 +1,5 @@
-; RUN: opt < %s -basicaa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -basicaa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7 | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.8.0"
@@ -8,6 +9,45 @@ target triple = "x86_64-apple-macosx10.8.0"
 
 ; Function Attrs: ssp uwtable
 define void @main() #0 {
+; CHECK-LABEL: @main(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 undef, label [[COND_TRUE:%.*]], label [[COND_END:%.*]]
+; CHECK:       cond.true:
+; CHECK-NEXT:    unreachable
+; CHECK:       cond.end:
+; CHECK-NEXT:    br label [[INVOKE_CONT:%.*]]
+; CHECK:       invoke.cont:
+; CHECK-NEXT:    br i1 undef, label [[ARRAYCTOR_CONT:%.*]], label [[INVOKE_CONT]]
+; CHECK:       arrayctor.cont:
+; CHECK-NEXT:    [[AGG_TMP99208_SROA_0_0_IDX:%.*]] = getelementptr inbounds [[STRUCT_RAY_5_11_53_113_119_137_149_185_329_389_416:%.*]], %struct.Ray.5.11.53.113.119.137.149.185.329.389.416* undef, i64 0, i32 0, i32 0
+; CHECK-NEXT:    [[AGG_TMP101211_SROA_0_0_IDX:%.*]] = getelementptr inbounds [[STRUCT_RAY_5_11_53_113_119_137_149_185_329_389_416]], %struct.Ray.5.11.53.113.119.137.149.185.329.389.416* undef, i64 0, i32 1, i32 0
+; CHECK-NEXT:    br label [[FOR_COND36_PREHEADER:%.*]]
+; CHECK:       for.cond36.preheader:
+; CHECK-NEXT:    br i1 undef, label [[FOR_BODY42_LR_PH_US:%.*]], label [[_Z5CLAMPD_EXIT_1:%.*]]
+; CHECK:       cond.false51.us:
+; CHECK-NEXT:    unreachable
+; CHECK:       cond.true48.us:
+; CHECK-NEXT:    br i1 undef, label [[COND_TRUE63_US:%.*]], label [[COND_FALSE66_US:%.*]]
+; CHECK:       cond.false66.us:
+; CHECK-NEXT:    [[ADD_I276_US:%.*]] = fadd double 0.000000e+00, undef
+; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <2 x double> undef, double [[ADD_I276_US]], i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x double> [[TMP0]], double 0xBFA5CC2D1960285F, i32 1
+; CHECK-NEXT:    [[TMP2:%.*]] = fadd <2 x double> <double 0.000000e+00, double undef>, [[TMP1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = fmul <2 x double> <double 1.400000e+02, double 1.400000e+02>, [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = fadd <2 x double> <double 5.000000e+01, double 5.200000e+01>, [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = fmul <2 x double> undef, [[TMP2]]
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast double* [[AGG_TMP99208_SROA_0_0_IDX]] to <2 x double>*
+; CHECK-NEXT:    store <2 x double> [[TMP4]], <2 x double>* [[TMP6]], align 8
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast double* [[AGG_TMP101211_SROA_0_0_IDX]] to <2 x double>*
+; CHECK-NEXT:    store <2 x double> [[TMP5]], <2 x double>* [[TMP7]], align 8
+; CHECK-NEXT:    unreachable
+; CHECK:       cond.true63.us:
+; CHECK-NEXT:    unreachable
+; CHECK:       for.body42.lr.ph.us:
+; CHECK-NEXT:    br i1 undef, label [[COND_TRUE48_US:%.*]], label [[COND_FALSE51_US:%.*]]
+; CHECK:       _Z5clampd.exit.1:
+; CHECK-NEXT:    br label [[FOR_COND36_PREHEADER]]
+;
 entry:
   br i1 undef, label %cond.true, label %cond.end
 
@@ -67,6 +107,27 @@ _Z5clampd.exit.1:                                 ; preds = %for.cond36.preheade
 %struct.Vec.0.6.48.90.132.186.192.198.234.252.258.264.270.276.282.288.378.432.438.450.456.594.600 = type { double, double, double }
 
 define void @_Z8radianceRK3RayiPt() #0 {
+; CHECK-LABEL: @_Z8radianceRK3RayiPt(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 undef, label [[IF_THEN78:%.*]], label [[IF_THEN38:%.*]]
+; CHECK:       if.then38:
+; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <2 x double> undef, double undef, i32 1
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul <2 x double> undef, [[TMP0]]
+; CHECK-NEXT:    [[TMP2:%.*]] = fsub <2 x double> undef, [[TMP1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = fmul <2 x double> undef, [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = fmul <2 x double> undef, [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = fadd <2 x double> undef, [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = fadd <2 x double> undef, [[TMP5]]
+; CHECK-NEXT:    [[TMP7:%.*]] = fmul <2 x double> undef, [[TMP6]]
+; CHECK-NEXT:    [[AGG_TMP74663_SROA_0_0_IDX:%.*]] = getelementptr inbounds [[STRUCT_RAY_5_11_53_95_137_191_197_203_239_257_263_269_275_281_287_293_383_437_443_455_461_599_601:%.*]], %struct.Ray.5.11.53.95.137.191.197.203.239.257.263.269.275.281.287.293.383.437.443.455.461.599.601* undef, i64 0, i32 1, i32 0
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast double* [[AGG_TMP74663_SROA_0_0_IDX]] to <2 x double>*
+; CHECK-NEXT:    store <2 x double> [[TMP7]], <2 x double>* [[TMP8]], align 8
+; CHECK-NEXT:    br label [[RETURN:%.*]]
+; CHECK:       if.then78:
+; CHECK-NEXT:    br label [[RETURN]]
+; CHECK:       return:
+; CHECK-NEXT:    ret void
+;
 entry:
   br i1 undef, label %if.then78, label %if.then38
 
diff --git a/test/Transforms/SLPVectorizer/X86/crash_vectorizeTree.ll b/test/Transforms/SLPVectorizer/X86/crash_vectorizeTree.ll
index 45ca99a3ea161a29fa9a83ae509b3d795ac98ea9..24f5bad039894a290b168540b5ee91f3da245242 100644
--- a/test/Transforms/SLPVectorizer/X86/crash_vectorizeTree.ll
+++ b/test/Transforms/SLPVectorizer/X86/crash_vectorizeTree.ll
@@ -1,14 +1,14 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -basicaa -slp-vectorizer -mtriple=x86_64-apple-macosx10.9.0 -mcpu=corei7-avx -S < %s | FileCheck %s
 target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.9.0"
 
 
 ; This test used to crash because we were following phi chains incorrectly.
-; We used indices to get the incoming value of two phi nodes rather than 
+; We used indices to get the incoming value of two phi nodes rather than
 ; incoming block lookup.
 ; This can give wrong results when the ordering of incoming
 ; edges in the two phi nodes don't match.
-;CHECK-LABEL: bar
 
 %0 = type { %1, %2 }
 %1 = type { double, double }
@@ -17,6 +17,36 @@ target triple = "x86_64-apple-macosx10.9.0"
 
 ;define fastcc void @bar() {
 define void @bar() {
+; CHECK-LABEL: @bar(
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds [[TMP0:%.*]], %0* undef, i64 0, i32 1, i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds [[TMP0]], %0* undef, i64 0, i32 1, i32 1
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds [[TMP0]], %0* undef, i64 0, i32 1, i32 0
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds [[TMP0]], %0* undef, i64 0, i32 1, i32 1
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds [[TMP0]], %0* undef, i64 0, i32 1, i32 0
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds [[TMP0]], %0* undef, i64 0, i32 1, i32 1
+; CHECK-NEXT:    br label [[TMP7:%.*]]
+; CHECK:         [[TMP8:%.*]] = phi <2 x double> [ <double 1.800000e+01, double 2.800000e+01>, [[TMP0]] ], [ [[TMP11:%.*]], [[TMP21:%.*]] ], [ [[TMP11]], [[TMP18:%.*]] ], [ [[TMP11]], [[TMP18]] ]
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast double* [[TMP1]] to <2 x double>*
+; CHECK-NEXT:    store <2 x double> [[TMP8]], <2 x double>* [[TMP9]], align 8
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast double* [[TMP3]] to <2 x double>*
+; CHECK-NEXT:    [[TMP11]] = load <2 x double>, <2 x double>* [[TMP10]], align 8
+; CHECK-NEXT:    br i1 undef, label [[TMP12:%.*]], label [[TMP13:%.*]]
+; CHECK:         ret void
+; CHECK:         [[TMP14:%.*]] = bitcast double* [[TMP5]] to <2 x double>*
+; CHECK-NEXT:    store <2 x double> [[TMP11]], <2 x double>* [[TMP14]], align 8
+; CHECK-NEXT:    br i1 undef, label [[TMP15:%.*]], label [[TMP16:%.*]]
+; CHECK:         br label [[TMP16]]
+; CHECK:         br i1 undef, label [[TMP17:%.*]], label [[TMP18]]
+; CHECK:         unreachable
+; CHECK:         [[TMP19:%.*]] = extractelement <2 x double> [[TMP11]], i32 0
+; CHECK-NEXT:    [[TMP20:%.*]] = extractelement <2 x double> [[TMP11]], i32 1
+; CHECK-NEXT:    switch i32 undef, label [[TMP21]] [
+; CHECK-NEXT:    i32 32, label [[TMP7]]
+; CHECK-NEXT:    i32 103, label [[TMP7]]
+; CHECK-NEXT:    ]
+; CHECK:         br i1 undef, label [[TMP7]], label [[TMP22:%.*]]
+; CHECK:         unreachable
+;
   %1 = getelementptr inbounds %0, %0* undef, i64 0, i32 1, i32 0
   %2 = getelementptr inbounds %0, %0* undef, i64 0, i32 1, i32 1
   %3 = getelementptr inbounds %0, %0* undef, i64 0, i32 1, i32 0
@@ -53,8 +83,8 @@ define void @bar() {
 
 ; <label>:17                                      ; preds = %15
   switch i32 undef, label %18 [
-    i32 32, label %7
-    i32 103, label %7
+  i32 32, label %7
+  i32 103, label %7
   ]
 
 ; <label>:18                                      ; preds = %17
diff --git a/test/Transforms/SLPVectorizer/X86/cross_block_slp.ll b/test/Transforms/SLPVectorizer/X86/cross_block_slp.ll
index eac20a00645007f932d0cabf6848e3c27c2c2e51..98db3edd90ead0d11dc06a6b2841aafc59cd1b1e 100644
--- a/test/Transforms/SLPVectorizer/X86/cross_block_slp.ll
+++ b/test/Transforms/SLPVectorizer/X86/cross_block_slp.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -basicaa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
@@ -16,15 +17,26 @@ target triple = "x86_64-apple-macosx10.8.0"
 ; }
 
 
-;CHECK-LABEL: @foo(
-;CHECK: load <2 x float>
-;CHECK: fadd <2 x float>
-;CHECK: call i32
-;CHECK: load <2 x double>
-;CHECK: fadd <2 x double>
-;CHECK: store <2 x double>
-;CHECK: ret
 define i32 @foo(double* nocapture %A, float* nocapture %B, i32 %g) {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast float* [[B:%.*]] to <2 x float>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x float>, <2 x float>* [[TMP0]], align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = fadd <2 x float> <float 5.000000e+00, float 8.000000e+00>, [[TMP1]]
+; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[G:%.*]], 0
+; CHECK-NEXT:    br i1 [[TOBOOL]], label [[IF_END:%.*]], label [[IF_THEN:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[CALL:%.*]] = tail call i32 (...) @bar()
+; CHECK-NEXT:    br label [[IF_END]]
+; CHECK:       if.end:
+; CHECK-NEXT:    [[TMP3:%.*]] = fpext <2 x float> [[TMP2]] to <2 x double>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast double* [[A:%.*]] to <2 x double>*
+; CHECK-NEXT:    [[TMP5:%.*]] = load <2 x double>, <2 x double>* [[TMP4]], align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = fadd <2 x double> [[TMP3]], [[TMP5]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast double* [[A]] to <2 x double>*
+; CHECK-NEXT:    store <2 x double> [[TMP6]], <2 x double>* [[TMP7]], align 8
+; CHECK-NEXT:    ret i32 undef
+;
 entry:
   %0 = load float, float* %B, align 4
   %arrayidx1 = getelementptr inbounds float, float* %B, i64 1
diff --git a/test/Transforms/SLPVectorizer/X86/cycle_dup.ll b/test/Transforms/SLPVectorizer/X86/cycle_dup.ll
index 0a4e961c2e88540b13d31174711e514112ed47dc..ac6933304780fe0957cb4c58eefd3fd4e135c839 100644
--- a/test/Transforms/SLPVectorizer/X86/cycle_dup.ll
+++ b/test/Transforms/SLPVectorizer/X86/cycle_dup.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -basicaa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
@@ -11,17 +12,28 @@ target triple = "x86_64-apple-macosx10.9.0"
 ;   A[0] = r; A[1] = g; A[2] = b; A[3] = a;
 ; }
 
-;CHECK-LABEL: @foo
-;CHECK: bitcast i32* %A to <4 x i32>*
-;CHECK-NEXT: load <4 x i32>
-;CHECK: phi <4 x i32>
-;CHECK-NEXT: mul nsw <4 x i32>
-;CHECK-NOT: mul
-;CHECK: phi <4 x i32>
-;CHECK: bitcast i32* %A to <4 x i32>*
-;CHECK-NEXT: store <4 x i32>
-;CHECK-NEXT:ret i32 undef
 define i32 @foo(i32* nocapture %A) #0 {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast i32* [[A:%.*]] to <4 x i32>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, <4 x i32>* [[TMP0]], align 4
+; CHECK-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 13
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, i32* [[ARRAYIDX4]], align 4
+; CHECK-NEXT:    [[CMP24:%.*]] = icmp sgt i32 [[TMP2]], 0
+; CHECK-NEXT:    br i1 [[CMP24]], label [[FOR_BODY:%.*]], label [[FOR_END:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[I_029:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[TMP3:%.*]] = phi <4 x i32> [ [[TMP4:%.*]], [[FOR_BODY]] ], [ [[TMP1]], [[ENTRY]] ]
+; CHECK-NEXT:    [[TMP4]] = mul nsw <4 x i32> <i32 18, i32 19, i32 12, i32 9>, [[TMP3]]
+; CHECK-NEXT:    [[INC]] = add nsw i32 [[I_029]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[INC]], [[TMP2]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_END]]
+; CHECK:       for.end:
+; CHECK-NEXT:    [[TMP5:%.*]] = phi <4 x i32> [ [[TMP1]], [[ENTRY]] ], [ [[TMP4]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i32* [[A]] to <4 x i32>*
+; CHECK-NEXT:    store <4 x i32> [[TMP5]], <4 x i32>* [[TMP6]], align 4
+; CHECK-NEXT:    ret i32 undef
+;
 entry:
   %0 = load i32, i32* %A, align 4
   %arrayidx1 = getelementptr inbounds i32, i32* %A, i64 1
diff --git a/test/Transforms/SLPVectorizer/X86/debug_info.ll b/test/Transforms/SLPVectorizer/X86/debug_info.ll
index cdf2455d66598a734aa41b53634e8ec5a5d3ebb2..0fe399e725d724ed8d1a885816d2d99b1bb4dc73 100644
--- a/test/Transforms/SLPVectorizer/X86/debug_info.ll
+++ b/test/Transforms/SLPVectorizer/X86/debug_info.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -basicaa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
@@ -12,16 +13,28 @@ target triple = "x86_64-apple-macosx10.7.0"
 ;   A[8] = y0; A[8+1] = y1;
 ; }
 
-;CHECK: @depth
-;CHECK: getelementptr inbounds {{.*}}, !dbg ![[LOC:[0-9]+]]
-;CHECK: bitcast double* {{.*}}, !dbg ![[LOC]]
-;CHECK: load <2 x double>, <2 x double>* {{.*}}, !dbg ![[LOC]]
-;CHECK: store <2 x double> {{.*}}, !dbg ![[LOC2:[0-9]+]]
-;CHECK: ret
-;CHECK: ![[LOC]] = !DILocation(line: 4, scope:
-;CHECK: ![[LOC2]] = !DILocation(line: 7, scope:
-
 define i32 @depth(double* nocapture %A, i32 %m) #0 !dbg !4 {
+; CHECK-LABEL: @depth(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    call void @llvm.dbg.value(metadata double* [[A:%.*]], metadata !12, metadata !DIExpression()), !dbg !18
+; CHECK-NEXT:    call void @llvm.dbg.value(metadata i32 [[M:%.*]], metadata !13, metadata !DIExpression()), !dbg !18
+; CHECK-NEXT:    call void @llvm.dbg.value(metadata double 0.000000e+00, metadata !14, metadata !DIExpression()), !dbg !19
+; CHECK-NEXT:    call void @llvm.dbg.value(metadata double 2.000000e-01, metadata !15, metadata !DIExpression()), !dbg !19
+; CHECK-NEXT:    call void @llvm.dbg.value(metadata i32 0, metadata !16, metadata !DIExpression()), !dbg !20
+; CHECK-NEXT:    [[CMP8:%.*]] = icmp sgt i32 [[M]], 0, !dbg !20
+; CHECK-NEXT:    br i1 [[CMP8]], label [[FOR_BODY_LR_PH:%.*]], label [[FOR_END:%.*]], !dbg !20
+; CHECK:       for.body.lr.ph:
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds double, double* [[A]], i64 4, !dbg !21
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast double* [[ARRAYIDX]] to <2 x double>*, !dbg !21
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x double>, <2 x double>* [[TMP0]], align 8, !dbg !21
+; CHECK-NEXT:    br label [[FOR_END]], !dbg !20
+; CHECK:       for.end:
+; CHECK-NEXT:    [[TMP2:%.*]] = phi <2 x double> [ [[TMP1]], [[FOR_BODY_LR_PH]] ], [ <double 0.000000e+00, double 1.000000e+00>, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds double, double* [[A]], i64 8, !dbg !23
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast double* [[ARRAYIDX2]] to <2 x double>*, !dbg !23
+; CHECK-NEXT:    store <2 x double> [[TMP2]], <2 x double>* [[TMP3]], align 8, !dbg !23
+; CHECK-NEXT:    ret i32 undef, !dbg !24
+;
 entry:
   tail call void @llvm.dbg.value(metadata double* %A, i64 0, metadata !12, metadata !DIExpression()), !dbg !19
   tail call void @llvm.dbg.value(metadata i32 %m, i64 0, metadata !13, metadata !DIExpression()), !dbg !19
diff --git a/test/Transforms/SLPVectorizer/X86/diamond.ll b/test/Transforms/SLPVectorizer/X86/diamond.ll
index 4e2c02f6965a808fab125230555ca0386073a39e..3cba8ea00f7cdc12b35db905fb12810da51aca4d 100644
--- a/test/Transforms/SLPVectorizer/X86/diamond.ll
+++ b/test/Transforms/SLPVectorizer/X86/diamond.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -basicaa -slp-vectorizer -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
@@ -11,12 +12,27 @@ target triple = "x86_64-apple-macosx10.8.0"
 ;   return 0;
 ; }
 
-; CHECK-LABEL: @foo(
-; CHECK: load <4 x i32>
-; CHECK: mul <4 x i32>
-; CHECK: store <4 x i32>
-; CHECK: ret
 define i32 @foo(i32* noalias nocapture %B, i32* noalias nocapture %A, i32 %n, i32 %m) #0 {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[MUL238:%.*]] = add i32 [[M:%.*]], [[N:%.*]]
+; CHECK-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds i32, i32* [[A:%.*]], i64 1
+; CHECK-NEXT:    [[ARRAYIDX9:%.*]] = getelementptr inbounds i32, i32* [[B:%.*]], i64 1
+; CHECK-NEXT:    [[ARRAYIDX10:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 2
+; CHECK-NEXT:    [[ARRAYIDX15:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 2
+; CHECK-NEXT:    [[ARRAYIDX16:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 3
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast i32* [[A]] to <4 x i32>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, <4 x i32>* [[TMP0]], align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <4 x i32> undef, i32 [[MUL238]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <4 x i32> [[TMP2]], i32 [[MUL238]], i32 1
+; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <4 x i32> [[TMP3]], i32 [[MUL238]], i32 2
+; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <4 x i32> [[TMP4]], i32 [[MUL238]], i32 3
+; CHECK-NEXT:    [[TMP6:%.*]] = mul <4 x i32> [[TMP1]], [[TMP5]]
+; CHECK-NEXT:    [[ARRAYIDX21:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 3
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i32* [[B]] to <4 x i32>*
+; CHECK-NEXT:    store <4 x i32> [[TMP6]], <4 x i32>* [[TMP7]], align 4
+; CHECK-NEXT:    ret i32 0
+;
 entry:
   %0 = load i32, i32* %A, align 4
   %mul238 = add i32 %m, %n
@@ -49,12 +65,28 @@ entry:
 ;   return A[0];
 ; }
 
-; CHECK-LABEL: @extr_user(
-; CHECK: load <4 x i32>
-; CHECK: store <4 x i32>
-; CHECK: extractelement <4 x i32>
-; CHECK-NEXT: ret
 define i32 @extr_user(i32* noalias nocapture %B, i32* noalias nocapture %A, i32 %n, i32 %m) {
+; CHECK-LABEL: @extr_user(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[MUL238:%.*]] = add i32 [[M:%.*]], [[N:%.*]]
+; CHECK-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds i32, i32* [[A:%.*]], i64 1
+; CHECK-NEXT:    [[ARRAYIDX9:%.*]] = getelementptr inbounds i32, i32* [[B:%.*]], i64 1
+; CHECK-NEXT:    [[ARRAYIDX10:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 2
+; CHECK-NEXT:    [[ARRAYIDX15:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 2
+; CHECK-NEXT:    [[ARRAYIDX16:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 3
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast i32* [[A]] to <4 x i32>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, <4 x i32>* [[TMP0]], align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <4 x i32> undef, i32 [[MUL238]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <4 x i32> [[TMP2]], i32 [[MUL238]], i32 1
+; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <4 x i32> [[TMP3]], i32 [[MUL238]], i32 2
+; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <4 x i32> [[TMP4]], i32 [[MUL238]], i32 3
+; CHECK-NEXT:    [[TMP6:%.*]] = mul <4 x i32> [[TMP1]], [[TMP5]]
+; CHECK-NEXT:    [[ARRAYIDX21:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 3
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i32* [[B]] to <4 x i32>*
+; CHECK-NEXT:    store <4 x i32> [[TMP6]], <4 x i32>* [[TMP7]], align 4
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <4 x i32> [[TMP1]], i32 0
+; CHECK-NEXT:    ret i32 [[TMP8]]
+;
 entry:
   %0 = load i32, i32* %A, align 4
   %mul238 = add i32 %m, %n
@@ -79,12 +111,28 @@ entry:
 }
 
 ; In this example we have an external user that is not the first element in the vector.
-; CHECK-LABEL: @extr_user1(
-; CHECK: load <4 x i32>
-; CHECK: store <4 x i32>
-; CHECK: extractelement <4 x i32>
-; CHECK-NEXT: ret
 define i32 @extr_user1(i32* noalias nocapture %B, i32* noalias nocapture %A, i32 %n, i32 %m) {
+; CHECK-LABEL: @extr_user1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[MUL238:%.*]] = add i32 [[M:%.*]], [[N:%.*]]
+; CHECK-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds i32, i32* [[A:%.*]], i64 1
+; CHECK-NEXT:    [[ARRAYIDX9:%.*]] = getelementptr inbounds i32, i32* [[B:%.*]], i64 1
+; CHECK-NEXT:    [[ARRAYIDX10:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 2
+; CHECK-NEXT:    [[ARRAYIDX15:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 2
+; CHECK-NEXT:    [[ARRAYIDX16:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 3
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast i32* [[A]] to <4 x i32>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, <4 x i32>* [[TMP0]], align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <4 x i32> undef, i32 [[MUL238]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <4 x i32> [[TMP2]], i32 [[MUL238]], i32 1
+; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <4 x i32> [[TMP3]], i32 [[MUL238]], i32 2
+; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <4 x i32> [[TMP4]], i32 [[MUL238]], i32 3
+; CHECK-NEXT:    [[TMP6:%.*]] = mul <4 x i32> [[TMP1]], [[TMP5]]
+; CHECK-NEXT:    [[ARRAYIDX21:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 3
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i32* [[B]] to <4 x i32>*
+; CHECK-NEXT:    store <4 x i32> [[TMP6]], <4 x i32>* [[TMP7]], align 4
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <4 x i32> [[TMP1]], i32 1
+; CHECK-NEXT:    ret i32 [[TMP8]]
+;
 entry:
   %0 = load i32, i32* %A, align 4
   %mul238 = add i32 %m, %n
diff --git a/test/Transforms/SLPVectorizer/X86/external_user.ll b/test/Transforms/SLPVectorizer/X86/external_user.ll
index bf2febda86b43aeea396176b27d647fd5138ddf3..8ee644f939ba015dbc1fa11091821abe9ec03f30 100644
--- a/test/Transforms/SLPVectorizer/X86/external_user.ll
+++ b/test/Transforms/SLPVectorizer/X86/external_user.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -basicaa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
@@ -22,16 +23,27 @@ target triple = "x86_64-apple-macosx10.8.0"
 ;   return x; <-- must extract here!
 ; }
 
-;CHECK: ext_user
-;CHECK: phi <2 x double>
-;CHECK: fadd <2 x double>
-;CHECK: fmul <2 x double>
-;CHECK: br
-;CHECK: store <2 x double>
-;CHECK: extractelement <2 x double>
-;CHECK: ret double
-
 define double @ext_user(double* noalias nocapture %B, double* noalias nocapture %A, i32 %n, i32 %m) {
+; CHECK-LABEL: @ext_user(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast double* [[A:%.*]] to <2 x double>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x double>, <2 x double>* [[TMP0]], align 8
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[I_020:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[TMP2:%.*]] = phi <2 x double> [ [[TMP1]], [[ENTRY]] ], [ [[TMP5:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[TMP3:%.*]] = fadd <2 x double> <double 1.000000e+01, double 1.000000e+01>, [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = fmul <2 x double> <double 4.000000e+00, double 4.000000e+00>, [[TMP3]]
+; CHECK-NEXT:    [[TMP5]] = fadd <2 x double> <double 4.000000e+00, double 4.000000e+00>, [[TMP4]]
+; CHECK-NEXT:    [[INC]] = add nsw i32 [[I_020]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], 100
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_END:%.*]], label [[FOR_BODY]]
+; CHECK:       for.end:
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast double* [[B:%.*]] to <2 x double>*
+; CHECK-NEXT:    store <2 x double> [[TMP5]], <2 x double>* [[TMP6]], align 8
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <2 x double> [[TMP4]], i32 0
+; CHECK-NEXT:    ret double [[TMP7]]
+;
 entry:
   %arrayidx = getelementptr inbounds double, double* %A, i64 1
   %0 = load double, double* %arrayidx, align 8
@@ -65,9 +77,33 @@ for.end:                                          ; preds = %for.body
 ; This test would assert because we would keep the scalar fpext and fadd alive.
 ; PR18129
 
-; CHECK-LABEL: needtogather
 define i32 @needtogather(double *noalias %a, i32 *noalias %b,  float * noalias %c,
-                i32 * noalias %d) {
+; CHECK-LABEL: @needtogather(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[D:%.*]], align 4
+; CHECK-NEXT:    [[CONV:%.*]] = sitofp i32 [[TMP0]] to float
+; CHECK-NEXT:    [[TMP1:%.*]] = load float, float* [[C:%.*]]
+; CHECK-NEXT:    [[SUB:%.*]] = fsub float 0.000000e+00, [[TMP1]]
+; CHECK-NEXT:    [[MUL:%.*]] = fmul float [[SUB]], 0.000000e+00
+; CHECK-NEXT:    [[ADD:%.*]] = fadd float [[CONV]], [[MUL]]
+; CHECK-NEXT:    [[CONV1:%.*]] = fpext float [[ADD]] to double
+; CHECK-NEXT:    [[SUB3:%.*]] = fsub float 1.000000e+00, [[TMP1]]
+; CHECK-NEXT:    [[MUL4:%.*]] = fmul float [[SUB3]], 0.000000e+00
+; CHECK-NEXT:    [[ADD5:%.*]] = fadd float [[CONV]], [[MUL4]]
+; CHECK-NEXT:    [[CONV6:%.*]] = fpext float [[ADD5]] to double
+; CHECK-NEXT:    [[TOBOOL:%.*]] = fcmp une float [[ADD]], 0.000000e+00
+; CHECK-NEXT:    br i1 [[TOBOOL]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    br label [[IF_END]]
+; CHECK:       if.end:
+; CHECK-NEXT:    [[STOREMERGE:%.*]] = phi double [ [[CONV6]], [[IF_THEN]] ], [ [[CONV1]], [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[E_0:%.*]] = phi double [ [[CONV1]], [[IF_THEN]] ], [ [[CONV6]], [[ENTRY]] ]
+; CHECK-NEXT:    store double [[STOREMERGE]], double* [[A:%.*]], align 8
+; CHECK-NEXT:    [[CONV7:%.*]] = fptosi double [[E_0]] to i32
+; CHECK-NEXT:    store i32 [[CONV7]], i32* [[B:%.*]], align 4
+; CHECK-NEXT:    ret i32 undef
+;
+  i32 * noalias %d) {
 entry:
   %0 = load i32, i32* %d, align 4
   %conv = sitofp i32 %0 to float
diff --git a/test/Transforms/SLPVectorizer/X86/extractcost.ll b/test/Transforms/SLPVectorizer/X86/extractcost.ll
index 164ddf38c1929e1d445be91a168d4e4ac99a8190..c9fae4460e5708ec2435641722d9d1abb6279d74 100644
--- a/test/Transforms/SLPVectorizer/X86/extractcost.ll
+++ b/test/Transforms/SLPVectorizer/X86/extractcost.ll
@@ -1,12 +1,28 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -basicaa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.8.0"
 
-;CHECK-LABEL: @foo(
-;CHECK: store <4 x i32>
-;CHECK: ret
 define i32 @foo(i32* nocapture %A, i32 %n, i32 %m) {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <4 x i32> undef, i32 [[N:%.*]], i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <4 x i32> [[TMP0]], i32 [[N]], i32 1
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <4 x i32> [[TMP1]], i32 [[N]], i32 2
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <4 x i32> [[TMP2]], i32 [[N]], i32 3
+; CHECK-NEXT:    [[TMP4:%.*]] = mul nsw <4 x i32> [[TMP3]], <i32 5, i32 9, i32 3, i32 10>
+; CHECK-NEXT:    [[TMP5:%.*]] = shl <4 x i32> [[TMP3]], <i32 5, i32 9, i32 3, i32 10>
+; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <4 x i32> [[TMP4]], <4 x i32> [[TMP5]], <4 x i32> <i32 0, i32 1, i32 6, i32 3>
+; CHECK-NEXT:    [[TMP7:%.*]] = add nsw <4 x i32> <i32 9, i32 9, i32 9, i32 9>, [[TMP6]]
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast i32* [[A:%.*]] to <4 x i32>*
+; CHECK-NEXT:    store <4 x i32> [[TMP7]], <4 x i32>* [[TMP8]], align 4
+; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <4 x i32> [[TMP7]], i32 0
+; CHECK-NEXT:    [[EXTERNALUSE1:%.*]] = add nsw i32 [[TMP9]], [[M:%.*]]
+; CHECK-NEXT:    [[EXTERNALUSE2:%.*]] = mul nsw i32 [[TMP9]], [[M]]
+; CHECK-NEXT:    [[ADD10:%.*]] = add nsw i32 [[EXTERNALUSE1]], [[EXTERNALUSE2]]
+; CHECK-NEXT:    ret i32 [[ADD10]]
+;
 entry:
   %mul = mul nsw i32 %n, 5
   %add = add nsw i32 %mul, 9
@@ -23,7 +39,7 @@ entry:
   %add8 = add nsw i32 %mul7, 9
   %arrayidx9 = getelementptr inbounds i32, i32* %A, i64 3
   store i32 %add8, i32* %arrayidx9, align 4
-  %externaluse1 = add nsw i32 %add, %m  
+  %externaluse1 = add nsw i32 %add, %m
   %externaluse2 = mul nsw i32 %add, %m  ; we should add the extract cost only once and the store will be vectorized
   %add10 = add nsw i32 %externaluse1, %externaluse2
   ret i32 %add10
diff --git a/test/Transforms/SLPVectorizer/X86/fabs-cost-softfp.ll b/test/Transforms/SLPVectorizer/X86/fabs-cost-softfp.ll
index add4858b7c53bcb3430aeb70b26b38c09f8913c1..25394f488493fbf0bbbf1a1ecc3e5697fd8401db 100644
--- a/test/Transforms/SLPVectorizer/X86/fabs-cost-softfp.ll
+++ b/test/Transforms/SLPVectorizer/X86/fabs-cost-softfp.ll
@@ -1,14 +1,29 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; Regression test from https://bugs.llvm.org/show_bug.cgi?id=39168
 ; Based on code from `compiler-rt/lib/builtins/multc3.c`
 ; On plaforms where fp128 lowers to an interger type (soft-fp) we
 ; shouldn't be calling isFAbsFree() on the legalized type.
 
 ; RUN: opt -slp-vectorizer -slp-threshold=-10 -S %s | FileCheck %s
-; CHECK: call <2 x fp128> @llvm.fabs.v2f128(<2 x fp128
 
 target triple = "i686-unknown-linux-gnu"
 
 define void @vectorize_fp128(fp128 %c, fp128 %d) #0 {
+; CHECK-LABEL: @vectorize_fp128(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <2 x fp128> undef, fp128 [[C:%.*]], i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x fp128> [[TMP0]], fp128 [[D:%.*]], i32 1
+; CHECK-NEXT:    [[TMP2:%.*]] = call <2 x fp128> @llvm.fabs.v2f128(<2 x fp128> [[TMP1]])
+; CHECK-NEXT:    [[TMP3:%.*]] = fcmp oeq <2 x fp128> [[TMP2]], <fp128 0xL00000000000000007FFF000000000000, fp128 0xL00000000000000007FFF000000000000>
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x i1> [[TMP3]], i32 0
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <2 x i1> [[TMP3]], i32 1
+; CHECK-NEXT:    [[OR_COND39:%.*]] = or i1 [[TMP4]], [[TMP5]]
+; CHECK-NEXT:    br i1 [[OR_COND39]], label [[IF_THEN13:%.*]], label [[IF_END24:%.*]]
+; CHECK:       if.then13:
+; CHECK-NEXT:    unreachable
+; CHECK:       if.end24:
+; CHECK-NEXT:    ret void
+;
 entry:
   %0 = tail call fp128 @llvm.fabs.f128(fp128 %c)
   %cmpinf10 = fcmp oeq fp128 %0, 0xL00000000000000007FFF000000000000
diff --git a/test/Transforms/SLPVectorizer/X86/flag.ll b/test/Transforms/SLPVectorizer/X86/flag.ll
index 7db8d75c20a7f1d9794818e4bba44ecf6fe1f20a..e695c02ef1042d71f622a1ac35692b0ab8394c5d 100644
--- a/test/Transforms/SLPVectorizer/X86/flag.ll
+++ b/test/Transforms/SLPVectorizer/X86/flag.ll
@@ -1,14 +1,50 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -basicaa -slp-vectorizer -slp-threshold=1000 -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.8.0"
 
 ; Check that the command line flag works.
-;CHECK:rollable
-;CHECK-NOT:load <4 x i32>
-;CHECK: ret
-
 define i32 @rollable(i32* noalias nocapture %in, i32* noalias nocapture %out, i64 %n) {
+; CHECK-LABEL: @rollable(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i64 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP1]], label [[DOT_CRIT_EDGE:%.*]], label [[DOTLR_PH:%.*]]
+; CHECK:       .lr.ph:
+; CHECK-NEXT:    [[I_019:%.*]] = phi i64 [ [[TMP26:%.*]], [[DOTLR_PH]] ], [ 0, [[TMP0:%.*]] ]
+; CHECK-NEXT:    [[TMP2:%.*]] = shl i64 [[I_019]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, i32* [[IN:%.*]], i64 [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = load i32, i32* [[TMP3]], align 4
+; CHECK-NEXT:    [[TMP5:%.*]] = or i64 [[TMP2]], 1
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, i32* [[IN]], i64 [[TMP5]]
+; CHECK-NEXT:    [[TMP7:%.*]] = load i32, i32* [[TMP6]], align 4
+; CHECK-NEXT:    [[TMP8:%.*]] = or i64 [[TMP2]], 2
+; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i32, i32* [[IN]], i64 [[TMP8]]
+; CHECK-NEXT:    [[TMP10:%.*]] = load i32, i32* [[TMP9]], align 4
+; CHECK-NEXT:    [[TMP11:%.*]] = or i64 [[TMP2]], 3
+; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i32, i32* [[IN]], i64 [[TMP11]]
+; CHECK-NEXT:    [[TMP13:%.*]] = load i32, i32* [[TMP12]], align 4
+; CHECK-NEXT:    [[TMP14:%.*]] = mul i32 [[TMP4]], 7
+; CHECK-NEXT:    [[TMP15:%.*]] = add i32 [[TMP14]], 7
+; CHECK-NEXT:    [[TMP16:%.*]] = mul i32 [[TMP7]], 7
+; CHECK-NEXT:    [[TMP17:%.*]] = add i32 [[TMP16]], 14
+; CHECK-NEXT:    [[TMP18:%.*]] = mul i32 [[TMP10]], 7
+; CHECK-NEXT:    [[TMP19:%.*]] = add i32 [[TMP18]], 21
+; CHECK-NEXT:    [[TMP20:%.*]] = mul i32 [[TMP13]], 7
+; CHECK-NEXT:    [[TMP21:%.*]] = add i32 [[TMP20]], 28
+; CHECK-NEXT:    [[TMP22:%.*]] = getelementptr inbounds i32, i32* [[OUT:%.*]], i64 [[TMP2]]
+; CHECK-NEXT:    store i32 [[TMP15]], i32* [[TMP22]], align 4
+; CHECK-NEXT:    [[TMP23:%.*]] = getelementptr inbounds i32, i32* [[OUT]], i64 [[TMP5]]
+; CHECK-NEXT:    store i32 [[TMP17]], i32* [[TMP23]], align 4
+; CHECK-NEXT:    [[TMP24:%.*]] = getelementptr inbounds i32, i32* [[OUT]], i64 [[TMP8]]
+; CHECK-NEXT:    store i32 [[TMP19]], i32* [[TMP24]], align 4
+; CHECK-NEXT:    [[TMP25:%.*]] = getelementptr inbounds i32, i32* [[OUT]], i64 [[TMP11]]
+; CHECK-NEXT:    store i32 [[TMP21]], i32* [[TMP25]], align 4
+; CHECK-NEXT:    [[TMP26]] = add i64 [[I_019]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[TMP26]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[DOT_CRIT_EDGE]], label [[DOTLR_PH]]
+; CHECK:       ._crit_edge:
+; CHECK-NEXT:    ret i32 undef
+;
   %1 = icmp eq i64 %n, 0
   br i1 %1, label %._crit_edge, label %.lr.ph
 
diff --git a/test/Transforms/SLPVectorizer/X86/fptosi.ll b/test/Transforms/SLPVectorizer/X86/fptosi.ll
index 69b9f32b73337304d9918492e55a5e9f7b6aaf6a..790a2f4092440f13065b2cd28e03e9ed4cd10f4c 100644
--- a/test/Transforms/SLPVectorizer/X86/fptosi.ll
+++ b/test/Transforms/SLPVectorizer/X86/fptosi.ll
@@ -221,32 +221,38 @@ define void @fptosi_8f64_8i16() #0 {
 }
 
 define void @fptosi_8f64_8i8() #0 {
-; CHECK-LABEL: @fptosi_8f64_8i8(
-; CHECK-NEXT:    [[A0:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 0), align 8
-; CHECK-NEXT:    [[A1:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 1), align 8
-; CHECK-NEXT:    [[A2:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 2), align 8
-; CHECK-NEXT:    [[A3:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 3), align 8
-; CHECK-NEXT:    [[A4:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 4), align 8
-; CHECK-NEXT:    [[A5:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 5), align 8
-; CHECK-NEXT:    [[A6:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 6), align 8
-; CHECK-NEXT:    [[A7:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 7), align 8
-; CHECK-NEXT:    [[CVT0:%.*]] = fptosi double [[A0]] to i8
-; CHECK-NEXT:    [[CVT1:%.*]] = fptosi double [[A1]] to i8
-; CHECK-NEXT:    [[CVT2:%.*]] = fptosi double [[A2]] to i8
-; CHECK-NEXT:    [[CVT3:%.*]] = fptosi double [[A3]] to i8
-; CHECK-NEXT:    [[CVT4:%.*]] = fptosi double [[A4]] to i8
-; CHECK-NEXT:    [[CVT5:%.*]] = fptosi double [[A5]] to i8
-; CHECK-NEXT:    [[CVT6:%.*]] = fptosi double [[A6]] to i8
-; CHECK-NEXT:    [[CVT7:%.*]] = fptosi double [[A7]] to i8
-; CHECK-NEXT:    store i8 [[CVT0]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 0), align 1
-; CHECK-NEXT:    store i8 [[CVT1]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 1), align 1
-; CHECK-NEXT:    store i8 [[CVT2]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 2), align 1
-; CHECK-NEXT:    store i8 [[CVT3]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 3), align 1
-; CHECK-NEXT:    store i8 [[CVT4]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 4), align 1
-; CHECK-NEXT:    store i8 [[CVT5]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 5), align 1
-; CHECK-NEXT:    store i8 [[CVT6]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 6), align 1
-; CHECK-NEXT:    store i8 [[CVT7]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 7), align 1
-; CHECK-NEXT:    ret void
+; SSE-LABEL: @fptosi_8f64_8i8(
+; SSE-NEXT:    [[A0:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 0), align 8
+; SSE-NEXT:    [[A1:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 1), align 8
+; SSE-NEXT:    [[A2:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 2), align 8
+; SSE-NEXT:    [[A3:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 3), align 8
+; SSE-NEXT:    [[A4:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 4), align 8
+; SSE-NEXT:    [[A5:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 5), align 8
+; SSE-NEXT:    [[A6:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 6), align 8
+; SSE-NEXT:    [[A7:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 7), align 8
+; SSE-NEXT:    [[CVT0:%.*]] = fptosi double [[A0]] to i8
+; SSE-NEXT:    [[CVT1:%.*]] = fptosi double [[A1]] to i8
+; SSE-NEXT:    [[CVT2:%.*]] = fptosi double [[A2]] to i8
+; SSE-NEXT:    [[CVT3:%.*]] = fptosi double [[A3]] to i8
+; SSE-NEXT:    [[CVT4:%.*]] = fptosi double [[A4]] to i8
+; SSE-NEXT:    [[CVT5:%.*]] = fptosi double [[A5]] to i8
+; SSE-NEXT:    [[CVT6:%.*]] = fptosi double [[A6]] to i8
+; SSE-NEXT:    [[CVT7:%.*]] = fptosi double [[A7]] to i8
+; SSE-NEXT:    store i8 [[CVT0]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 0), align 1
+; SSE-NEXT:    store i8 [[CVT1]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 1), align 1
+; SSE-NEXT:    store i8 [[CVT2]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 2), align 1
+; SSE-NEXT:    store i8 [[CVT3]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 3), align 1
+; SSE-NEXT:    store i8 [[CVT4]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 4), align 1
+; SSE-NEXT:    store i8 [[CVT5]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 5), align 1
+; SSE-NEXT:    store i8 [[CVT6]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 6), align 1
+; SSE-NEXT:    store i8 [[CVT7]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 7), align 1
+; SSE-NEXT:    ret void
+;
+; AVX-LABEL: @fptosi_8f64_8i8(
+; AVX-NEXT:    [[TMP1:%.*]] = load <8 x double>, <8 x double>* bitcast ([8 x double]* @src64 to <8 x double>*), align 8
+; AVX-NEXT:    [[TMP2:%.*]] = fptosi <8 x double> [[TMP1]] to <8 x i8>
+; AVX-NEXT:    store <8 x i8> [[TMP2]], <8 x i8>* bitcast ([64 x i8]* @dst8 to <8 x i8>*), align 1
+; AVX-NEXT:    ret void
 ;
   %a0 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 0), align 8
   %a1 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 1), align 8
@@ -455,30 +461,9 @@ define void @fptosi_8f32_8i16() #0 {
 
 define void @fptosi_8f32_8i8() #0 {
 ; CHECK-LABEL: @fptosi_8f32_8i8(
-; CHECK-NEXT:    [[A0:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 0), align 4
-; CHECK-NEXT:    [[A1:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 1), align 4
-; CHECK-NEXT:    [[A2:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 2), align 4
-; CHECK-NEXT:    [[A3:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 3), align 4
-; CHECK-NEXT:    [[A4:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 4), align 4
-; CHECK-NEXT:    [[A5:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 5), align 4
-; CHECK-NEXT:    [[A6:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 6), align 4
-; CHECK-NEXT:    [[A7:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 7), align 4
-; CHECK-NEXT:    [[CVT0:%.*]] = fptosi float [[A0]] to i8
-; CHECK-NEXT:    [[CVT1:%.*]] = fptosi float [[A1]] to i8
-; CHECK-NEXT:    [[CVT2:%.*]] = fptosi float [[A2]] to i8
-; CHECK-NEXT:    [[CVT3:%.*]] = fptosi float [[A3]] to i8
-; CHECK-NEXT:    [[CVT4:%.*]] = fptosi float [[A4]] to i8
-; CHECK-NEXT:    [[CVT5:%.*]] = fptosi float [[A5]] to i8
-; CHECK-NEXT:    [[CVT6:%.*]] = fptosi float [[A6]] to i8
-; CHECK-NEXT:    [[CVT7:%.*]] = fptosi float [[A7]] to i8
-; CHECK-NEXT:    store i8 [[CVT0]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 0), align 1
-; CHECK-NEXT:    store i8 [[CVT1]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 1), align 1
-; CHECK-NEXT:    store i8 [[CVT2]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 2), align 1
-; CHECK-NEXT:    store i8 [[CVT3]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 3), align 1
-; CHECK-NEXT:    store i8 [[CVT4]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 4), align 1
-; CHECK-NEXT:    store i8 [[CVT5]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 5), align 1
-; CHECK-NEXT:    store i8 [[CVT6]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 6), align 1
-; CHECK-NEXT:    store i8 [[CVT7]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 7), align 1
+; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x float>, <8 x float>* bitcast ([16 x float]* @src32 to <8 x float>*), align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = fptosi <8 x float> [[TMP1]] to <8 x i8>
+; CHECK-NEXT:    store <8 x i8> [[TMP2]], <8 x i8>* bitcast ([64 x i8]* @dst8 to <8 x i8>*), align 1
 ; CHECK-NEXT:    ret void
 ;
   %a0 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 0), align 4
diff --git a/test/Transforms/SLPVectorizer/X86/fptoui.ll b/test/Transforms/SLPVectorizer/X86/fptoui.ll
index 33d998ffbbc7e5eaf06823c6b12ff0d1d269500f..a5c5fb04dd049a99cf0dd24891f549dcd1ba127d 100644
--- a/test/Transforms/SLPVectorizer/X86/fptoui.ll
+++ b/test/Transforms/SLPVectorizer/X86/fptoui.ll
@@ -305,32 +305,71 @@ define void @fptoui_8f64_8i16() #0 {
 }
 
 define void @fptoui_8f64_8i8() #0 {
-; CHECK-LABEL: @fptoui_8f64_8i8(
-; CHECK-NEXT:    [[A0:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 0), align 8
-; CHECK-NEXT:    [[A1:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 1), align 8
-; CHECK-NEXT:    [[A2:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 2), align 8
-; CHECK-NEXT:    [[A3:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 3), align 8
-; CHECK-NEXT:    [[A4:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 4), align 8
-; CHECK-NEXT:    [[A5:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 5), align 8
-; CHECK-NEXT:    [[A6:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 6), align 8
-; CHECK-NEXT:    [[A7:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 7), align 8
-; CHECK-NEXT:    [[CVT0:%.*]] = fptoui double [[A0]] to i8
-; CHECK-NEXT:    [[CVT1:%.*]] = fptoui double [[A1]] to i8
-; CHECK-NEXT:    [[CVT2:%.*]] = fptoui double [[A2]] to i8
-; CHECK-NEXT:    [[CVT3:%.*]] = fptoui double [[A3]] to i8
-; CHECK-NEXT:    [[CVT4:%.*]] = fptoui double [[A4]] to i8
-; CHECK-NEXT:    [[CVT5:%.*]] = fptoui double [[A5]] to i8
-; CHECK-NEXT:    [[CVT6:%.*]] = fptoui double [[A6]] to i8
-; CHECK-NEXT:    [[CVT7:%.*]] = fptoui double [[A7]] to i8
-; CHECK-NEXT:    store i8 [[CVT0]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 0), align 1
-; CHECK-NEXT:    store i8 [[CVT1]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 1), align 1
-; CHECK-NEXT:    store i8 [[CVT2]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 2), align 1
-; CHECK-NEXT:    store i8 [[CVT3]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 3), align 1
-; CHECK-NEXT:    store i8 [[CVT4]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 4), align 1
-; CHECK-NEXT:    store i8 [[CVT5]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 5), align 1
-; CHECK-NEXT:    store i8 [[CVT6]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 6), align 1
-; CHECK-NEXT:    store i8 [[CVT7]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 7), align 1
-; CHECK-NEXT:    ret void
+; SSE-LABEL: @fptoui_8f64_8i8(
+; SSE-NEXT:    [[A0:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 0), align 8
+; SSE-NEXT:    [[A1:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 1), align 8
+; SSE-NEXT:    [[A2:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 2), align 8
+; SSE-NEXT:    [[A3:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 3), align 8
+; SSE-NEXT:    [[A4:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 4), align 8
+; SSE-NEXT:    [[A5:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 5), align 8
+; SSE-NEXT:    [[A6:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 6), align 8
+; SSE-NEXT:    [[A7:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 7), align 8
+; SSE-NEXT:    [[CVT0:%.*]] = fptoui double [[A0]] to i8
+; SSE-NEXT:    [[CVT1:%.*]] = fptoui double [[A1]] to i8
+; SSE-NEXT:    [[CVT2:%.*]] = fptoui double [[A2]] to i8
+; SSE-NEXT:    [[CVT3:%.*]] = fptoui double [[A3]] to i8
+; SSE-NEXT:    [[CVT4:%.*]] = fptoui double [[A4]] to i8
+; SSE-NEXT:    [[CVT5:%.*]] = fptoui double [[A5]] to i8
+; SSE-NEXT:    [[CVT6:%.*]] = fptoui double [[A6]] to i8
+; SSE-NEXT:    [[CVT7:%.*]] = fptoui double [[A7]] to i8
+; SSE-NEXT:    store i8 [[CVT0]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 0), align 1
+; SSE-NEXT:    store i8 [[CVT1]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 1), align 1
+; SSE-NEXT:    store i8 [[CVT2]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 2), align 1
+; SSE-NEXT:    store i8 [[CVT3]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 3), align 1
+; SSE-NEXT:    store i8 [[CVT4]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 4), align 1
+; SSE-NEXT:    store i8 [[CVT5]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 5), align 1
+; SSE-NEXT:    store i8 [[CVT6]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 6), align 1
+; SSE-NEXT:    store i8 [[CVT7]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 7), align 1
+; SSE-NEXT:    ret void
+;
+; AVX256NODQ-LABEL: @fptoui_8f64_8i8(
+; AVX256NODQ-NEXT:    [[A0:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 0), align 8
+; AVX256NODQ-NEXT:    [[A1:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 1), align 8
+; AVX256NODQ-NEXT:    [[A2:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 2), align 8
+; AVX256NODQ-NEXT:    [[A3:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 3), align 8
+; AVX256NODQ-NEXT:    [[A4:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 4), align 8
+; AVX256NODQ-NEXT:    [[A5:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 5), align 8
+; AVX256NODQ-NEXT:    [[A6:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 6), align 8
+; AVX256NODQ-NEXT:    [[A7:%.*]] = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 7), align 8
+; AVX256NODQ-NEXT:    [[CVT0:%.*]] = fptoui double [[A0]] to i8
+; AVX256NODQ-NEXT:    [[CVT1:%.*]] = fptoui double [[A1]] to i8
+; AVX256NODQ-NEXT:    [[CVT2:%.*]] = fptoui double [[A2]] to i8
+; AVX256NODQ-NEXT:    [[CVT3:%.*]] = fptoui double [[A3]] to i8
+; AVX256NODQ-NEXT:    [[CVT4:%.*]] = fptoui double [[A4]] to i8
+; AVX256NODQ-NEXT:    [[CVT5:%.*]] = fptoui double [[A5]] to i8
+; AVX256NODQ-NEXT:    [[CVT6:%.*]] = fptoui double [[A6]] to i8
+; AVX256NODQ-NEXT:    [[CVT7:%.*]] = fptoui double [[A7]] to i8
+; AVX256NODQ-NEXT:    store i8 [[CVT0]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 0), align 1
+; AVX256NODQ-NEXT:    store i8 [[CVT1]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 1), align 1
+; AVX256NODQ-NEXT:    store i8 [[CVT2]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 2), align 1
+; AVX256NODQ-NEXT:    store i8 [[CVT3]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 3), align 1
+; AVX256NODQ-NEXT:    store i8 [[CVT4]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 4), align 1
+; AVX256NODQ-NEXT:    store i8 [[CVT5]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 5), align 1
+; AVX256NODQ-NEXT:    store i8 [[CVT6]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 6), align 1
+; AVX256NODQ-NEXT:    store i8 [[CVT7]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 7), align 1
+; AVX256NODQ-NEXT:    ret void
+;
+; AVX512-LABEL: @fptoui_8f64_8i8(
+; AVX512-NEXT:    [[TMP1:%.*]] = load <8 x double>, <8 x double>* bitcast ([8 x double]* @src64 to <8 x double>*), align 8
+; AVX512-NEXT:    [[TMP2:%.*]] = fptoui <8 x double> [[TMP1]] to <8 x i8>
+; AVX512-NEXT:    store <8 x i8> [[TMP2]], <8 x i8>* bitcast ([64 x i8]* @dst8 to <8 x i8>*), align 1
+; AVX512-NEXT:    ret void
+;
+; AVX256DQ-LABEL: @fptoui_8f64_8i8(
+; AVX256DQ-NEXT:    [[TMP1:%.*]] = load <8 x double>, <8 x double>* bitcast ([8 x double]* @src64 to <8 x double>*), align 8
+; AVX256DQ-NEXT:    [[TMP2:%.*]] = fptoui <8 x double> [[TMP1]] to <8 x i8>
+; AVX256DQ-NEXT:    store <8 x i8> [[TMP2]], <8 x i8>* bitcast ([64 x i8]* @dst8 to <8 x i8>*), align 1
+; AVX256DQ-NEXT:    ret void
 ;
   %a0 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 0), align 8
   %a1 = load double, double* getelementptr inbounds ([8 x double], [8 x double]* @src64, i32 0, i64 1), align 8
@@ -616,32 +655,38 @@ define void @fptoui_8f32_8i16() #0 {
 }
 
 define void @fptoui_8f32_8i8() #0 {
-; CHECK-LABEL: @fptoui_8f32_8i8(
-; CHECK-NEXT:    [[A0:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 0), align 4
-; CHECK-NEXT:    [[A1:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 1), align 4
-; CHECK-NEXT:    [[A2:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 2), align 4
-; CHECK-NEXT:    [[A3:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 3), align 4
-; CHECK-NEXT:    [[A4:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 4), align 4
-; CHECK-NEXT:    [[A5:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 5), align 4
-; CHECK-NEXT:    [[A6:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 6), align 4
-; CHECK-NEXT:    [[A7:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 7), align 4
-; CHECK-NEXT:    [[CVT0:%.*]] = fptoui float [[A0]] to i8
-; CHECK-NEXT:    [[CVT1:%.*]] = fptoui float [[A1]] to i8
-; CHECK-NEXT:    [[CVT2:%.*]] = fptoui float [[A2]] to i8
-; CHECK-NEXT:    [[CVT3:%.*]] = fptoui float [[A3]] to i8
-; CHECK-NEXT:    [[CVT4:%.*]] = fptoui float [[A4]] to i8
-; CHECK-NEXT:    [[CVT5:%.*]] = fptoui float [[A5]] to i8
-; CHECK-NEXT:    [[CVT6:%.*]] = fptoui float [[A6]] to i8
-; CHECK-NEXT:    [[CVT7:%.*]] = fptoui float [[A7]] to i8
-; CHECK-NEXT:    store i8 [[CVT0]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 0), align 1
-; CHECK-NEXT:    store i8 [[CVT1]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 1), align 1
-; CHECK-NEXT:    store i8 [[CVT2]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 2), align 1
-; CHECK-NEXT:    store i8 [[CVT3]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 3), align 1
-; CHECK-NEXT:    store i8 [[CVT4]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 4), align 1
-; CHECK-NEXT:    store i8 [[CVT5]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 5), align 1
-; CHECK-NEXT:    store i8 [[CVT6]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 6), align 1
-; CHECK-NEXT:    store i8 [[CVT7]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 7), align 1
-; CHECK-NEXT:    ret void
+; SSE-LABEL: @fptoui_8f32_8i8(
+; SSE-NEXT:    [[A0:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 0), align 4
+; SSE-NEXT:    [[A1:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 1), align 4
+; SSE-NEXT:    [[A2:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 2), align 4
+; SSE-NEXT:    [[A3:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 3), align 4
+; SSE-NEXT:    [[A4:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 4), align 4
+; SSE-NEXT:    [[A5:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 5), align 4
+; SSE-NEXT:    [[A6:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 6), align 4
+; SSE-NEXT:    [[A7:%.*]] = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 7), align 4
+; SSE-NEXT:    [[CVT0:%.*]] = fptoui float [[A0]] to i8
+; SSE-NEXT:    [[CVT1:%.*]] = fptoui float [[A1]] to i8
+; SSE-NEXT:    [[CVT2:%.*]] = fptoui float [[A2]] to i8
+; SSE-NEXT:    [[CVT3:%.*]] = fptoui float [[A3]] to i8
+; SSE-NEXT:    [[CVT4:%.*]] = fptoui float [[A4]] to i8
+; SSE-NEXT:    [[CVT5:%.*]] = fptoui float [[A5]] to i8
+; SSE-NEXT:    [[CVT6:%.*]] = fptoui float [[A6]] to i8
+; SSE-NEXT:    [[CVT7:%.*]] = fptoui float [[A7]] to i8
+; SSE-NEXT:    store i8 [[CVT0]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 0), align 1
+; SSE-NEXT:    store i8 [[CVT1]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 1), align 1
+; SSE-NEXT:    store i8 [[CVT2]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 2), align 1
+; SSE-NEXT:    store i8 [[CVT3]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 3), align 1
+; SSE-NEXT:    store i8 [[CVT4]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 4), align 1
+; SSE-NEXT:    store i8 [[CVT5]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 5), align 1
+; SSE-NEXT:    store i8 [[CVT6]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 6), align 1
+; SSE-NEXT:    store i8 [[CVT7]], i8* getelementptr inbounds ([64 x i8], [64 x i8]* @dst8, i32 0, i64 7), align 1
+; SSE-NEXT:    ret void
+;
+; AVX-LABEL: @fptoui_8f32_8i8(
+; AVX-NEXT:    [[TMP1:%.*]] = load <8 x float>, <8 x float>* bitcast ([16 x float]* @src32 to <8 x float>*), align 4
+; AVX-NEXT:    [[TMP2:%.*]] = fptoui <8 x float> [[TMP1]] to <8 x i8>
+; AVX-NEXT:    store <8 x i8> [[TMP2]], <8 x i8>* bitcast ([64 x i8]* @dst8 to <8 x i8>*), align 1
+; AVX-NEXT:    ret void
 ;
   %a0 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 0), align 4
   %a1 = load float, float* getelementptr inbounds ([16 x float], [16 x float]* @src32, i32 0, i64 1), align 4
diff --git a/test/Transforms/SLPVectorizer/X86/gep.ll b/test/Transforms/SLPVectorizer/X86/gep.ll
index 60b0a114c516c70e8d2d8b73c49a2c924e0b6e46..a3cce8b0c5ee0e03107fe4e75259bed02f43236d 100644
--- a/test/Transforms/SLPVectorizer/X86/gep.ll
+++ b/test/Transforms/SLPVectorizer/X86/gep.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -basicaa -slp-vectorizer -S |FileCheck %s
 ; RUN: opt < %s -aa-pipeline=basic-aa -passes=slp-vectorizer -S |FileCheck %s
 target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
@@ -8,9 +9,19 @@ target triple = "x86_64-unknown-unknown"
 ;   x->first  = y->first  + 16
 ;   x->second = y->second + 16
 
-; CHECK-LABEL: foo1
-; CHECK: <2 x i32*>
 define void @foo1 ({ i32*, i32* }* noalias %x, { i32*, i32* }* noalias %y) {
+; CHECK-LABEL: @foo1(
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds { i32*, i32* }, { i32*, i32* }* [[Y:%.*]], i64 0, i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds { i32*, i32* }, { i32*, i32* }* [[X:%.*]], i64 0, i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds { i32*, i32* }, { i32*, i32* }* [[Y]], i64 0, i32 1
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i32** [[TMP1]] to <2 x i32*>*
+; CHECK-NEXT:    [[TMP5:%.*]] = load <2 x i32*>, <2 x i32*>* [[TMP4]], align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr i32, <2 x i32*> [[TMP5]], <2 x i64> <i64 16, i64 16>
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds { i32*, i32* }, { i32*, i32* }* [[X]], i64 0, i32 1
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast i32** [[TMP2]] to <2 x i32*>*
+; CHECK-NEXT:    store <2 x i32*> [[TMP6]], <2 x i32*>* [[TMP8]], align 8
+; CHECK-NEXT:    ret void
+;
   %1 = getelementptr inbounds { i32*, i32* }, { i32*, i32* }* %y, i64 0, i32 0
   %2 = load i32*, i32** %1, align 8
   %3 = getelementptr inbounds i32, i32* %2, i64 16
@@ -26,9 +37,20 @@ define void @foo1 ({ i32*, i32* }* noalias %x, { i32*, i32* }* noalias %y) {
 
 ; Test that we don't vectorize GEP expressions if indexes are not constants.
 ; We can't produce an efficient code in that case.
-; CHECK-LABEL: foo2
-; CHECK-NOT: <2 x i32*>
 define void @foo2 ({ i32*, i32* }* noalias %x, { i32*, i32* }* noalias %y, i32 %i) {
+; CHECK-LABEL: @foo2(
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds { i32*, i32* }, { i32*, i32* }* [[Y:%.*]], i64 0, i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32*, i32** [[TMP1]], align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, i32* [[TMP2]], i32 [[I:%.*]]
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds { i32*, i32* }, { i32*, i32* }* [[X:%.*]], i64 0, i32 0
+; CHECK-NEXT:    store i32* [[TMP3]], i32** [[TMP4]], align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds { i32*, i32* }, { i32*, i32* }* [[Y]], i64 0, i32 1
+; CHECK-NEXT:    [[TMP6:%.*]] = load i32*, i32** [[TMP5]], align 8
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i32, i32* [[TMP6]], i32 [[I]]
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds { i32*, i32* }, { i32*, i32* }* [[X]], i64 0, i32 1
+; CHECK-NEXT:    store i32* [[TMP7]], i32** [[TMP8]], align 8
+; CHECK-NEXT:    ret void
+;
   %1 = getelementptr inbounds { i32*, i32* }, { i32*, i32* }* %y, i64 0, i32 0
   %2 = load i32*, i32** %1, align 8
   %3 = getelementptr inbounds i32, i32* %2, i32 %i
diff --git a/test/Transforms/SLPVectorizer/X86/gep_mismatch.ll b/test/Transforms/SLPVectorizer/X86/gep_mismatch.ll
index 1cd28a909f79076f7f0d90df49a8b6e9184db45f..f9b9995066a22e26de4b62169bf24866e0f5855c 100644
--- a/test/Transforms/SLPVectorizer/X86/gep_mismatch.ll
+++ b/test/Transforms/SLPVectorizer/X86/gep_mismatch.ll
@@ -1,4 +1,5 @@
-; RUN: opt < %s -S -slp-vectorizer
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -S -slp-vectorizer | FileCheck %s
 
 ; This code has GEPs with different index types, which should not
 ; matter for the SLPVectorizer.
@@ -6,6 +7,19 @@
 target triple = "x86_64--linux"
 
 define void @foo() {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[BB1:%.*]]
+; CHECK:       bb1:
+; CHECK-NEXT:    [[LS1_PH:%.*]] = phi float* [ [[_TMP1:%.*]], [[BB1]] ], [ undef, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[LS2_PH:%.*]] = phi float* [ [[_TMP2:%.*]], [[BB1]] ], [ undef, [[ENTRY]] ]
+; CHECK-NEXT:    store float undef, float* [[LS1_PH]]
+; CHECK-NEXT:    [[_TMP1]] = getelementptr float, float* [[LS1_PH]], i32 1
+; CHECK-NEXT:    [[_TMP2]] = getelementptr float, float* [[LS2_PH]], i64 4
+; CHECK-NEXT:    br i1 false, label [[BB1]], label [[BB2:%.*]]
+; CHECK:       bb2:
+; CHECK-NEXT:    ret void
+;
 entry:
   br label %bb1
 
diff --git a/test/Transforms/SLPVectorizer/X86/horizontal-minmax.ll b/test/Transforms/SLPVectorizer/X86/horizontal-minmax.ll
index d49d557916c9b3489646f2dec55a6d948f10377a..271244e54fc13da087c8229f6f5826564a9dd87b 100644
--- a/test/Transforms/SLPVectorizer/X86/horizontal-minmax.ll
+++ b/test/Transforms/SLPVectorizer/X86/horizontal-minmax.ll
@@ -341,114 +341,33 @@ define i32 @maxi32(i32) {
 }
 
 define float @maxf8(float) {
-; SSE-LABEL: @maxf8(
-; SSE-NEXT:    [[TMP2:%.*]] = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr1, i64 0, i64 0), align 16
-; SSE-NEXT:    [[TMP3:%.*]] = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr1, i64 0, i64 1), align 4
-; SSE-NEXT:    [[TMP4:%.*]] = fcmp fast ogt float [[TMP2]], [[TMP3]]
-; SSE-NEXT:    [[TMP5:%.*]] = select i1 [[TMP4]], float [[TMP2]], float [[TMP3]]
-; SSE-NEXT:    [[TMP6:%.*]] = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr1, i64 0, i64 2), align 8
-; SSE-NEXT:    [[TMP7:%.*]] = fcmp fast ogt float [[TMP5]], [[TMP6]]
-; SSE-NEXT:    [[TMP8:%.*]] = select i1 [[TMP7]], float [[TMP5]], float [[TMP6]]
-; SSE-NEXT:    [[TMP9:%.*]] = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr1, i64 0, i64 3), align 4
-; SSE-NEXT:    [[TMP10:%.*]] = fcmp fast ogt float [[TMP8]], [[TMP9]]
-; SSE-NEXT:    [[TMP11:%.*]] = select i1 [[TMP10]], float [[TMP8]], float [[TMP9]]
-; SSE-NEXT:    [[TMP12:%.*]] = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr1, i64 0, i64 4), align 16
-; SSE-NEXT:    [[TMP13:%.*]] = fcmp fast ogt float [[TMP11]], [[TMP12]]
-; SSE-NEXT:    [[TMP14:%.*]] = select i1 [[TMP13]], float [[TMP11]], float [[TMP12]]
-; SSE-NEXT:    [[TMP15:%.*]] = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr1, i64 0, i64 5), align 4
-; SSE-NEXT:    [[TMP16:%.*]] = fcmp fast ogt float [[TMP14]], [[TMP15]]
-; SSE-NEXT:    [[TMP17:%.*]] = select i1 [[TMP16]], float [[TMP14]], float [[TMP15]]
-; SSE-NEXT:    [[TMP18:%.*]] = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr1, i64 0, i64 6), align 8
-; SSE-NEXT:    [[TMP19:%.*]] = fcmp fast ogt float [[TMP17]], [[TMP18]]
-; SSE-NEXT:    [[TMP20:%.*]] = select i1 [[TMP19]], float [[TMP17]], float [[TMP18]]
-; SSE-NEXT:    [[TMP21:%.*]] = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr1, i64 0, i64 7), align 4
-; SSE-NEXT:    [[TMP22:%.*]] = fcmp fast ogt float [[TMP20]], [[TMP21]]
-; SSE-NEXT:    [[TMP23:%.*]] = select i1 [[TMP22]], float [[TMP20]], float [[TMP21]]
-; SSE-NEXT:    ret float [[TMP23]]
-;
-; AVX-LABEL: @maxf8(
-; AVX-NEXT:    [[TMP2:%.*]] = load <8 x float>, <8 x float>* bitcast ([32 x float]* @arr1 to <8 x float>*), align 16
-; AVX-NEXT:    [[TMP3:%.*]] = fcmp fast ogt float undef, undef
-; AVX-NEXT:    [[TMP4:%.*]] = select i1 [[TMP3]], float undef, float undef
-; AVX-NEXT:    [[TMP5:%.*]] = fcmp fast ogt float [[TMP4]], undef
-; AVX-NEXT:    [[TMP6:%.*]] = select i1 [[TMP5]], float [[TMP4]], float undef
-; AVX-NEXT:    [[TMP7:%.*]] = fcmp fast ogt float [[TMP6]], undef
-; AVX-NEXT:    [[TMP8:%.*]] = select i1 [[TMP7]], float [[TMP6]], float undef
-; AVX-NEXT:    [[TMP9:%.*]] = fcmp fast ogt float [[TMP8]], undef
-; AVX-NEXT:    [[TMP10:%.*]] = select i1 [[TMP9]], float [[TMP8]], float undef
-; AVX-NEXT:    [[TMP11:%.*]] = fcmp fast ogt float [[TMP10]], undef
-; AVX-NEXT:    [[TMP12:%.*]] = select i1 [[TMP11]], float [[TMP10]], float undef
-; AVX-NEXT:    [[TMP13:%.*]] = fcmp fast ogt float [[TMP12]], undef
-; AVX-NEXT:    [[TMP14:%.*]] = select i1 [[TMP13]], float [[TMP12]], float undef
-; AVX-NEXT:    [[TMP15:%.*]] = fcmp fast ogt float [[TMP14]], undef
-; AVX-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <8 x float> [[TMP2]], <8 x float> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX-NEXT:    [[RDX_MINMAX_CMP:%.*]] = fcmp fast ogt <8 x float> [[TMP2]], [[RDX_SHUF]]
-; AVX-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = select <8 x i1> [[RDX_MINMAX_CMP]], <8 x float> [[TMP2]], <8 x float> [[RDX_SHUF]]
-; AVX-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <8 x float> [[RDX_MINMAX_SELECT]], <8 x float> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX-NEXT:    [[RDX_MINMAX_CMP2:%.*]] = fcmp fast ogt <8 x float> [[RDX_MINMAX_SELECT]], [[RDX_SHUF1]]
-; AVX-NEXT:    [[RDX_MINMAX_SELECT3:%.*]] = select <8 x i1> [[RDX_MINMAX_CMP2]], <8 x float> [[RDX_MINMAX_SELECT]], <8 x float> [[RDX_SHUF1]]
-; AVX-NEXT:    [[RDX_SHUF4:%.*]] = shufflevector <8 x float> [[RDX_MINMAX_SELECT3]], <8 x float> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX-NEXT:    [[RDX_MINMAX_CMP5:%.*]] = fcmp fast ogt <8 x float> [[RDX_MINMAX_SELECT3]], [[RDX_SHUF4]]
-; AVX-NEXT:    [[RDX_MINMAX_SELECT6:%.*]] = select <8 x i1> [[RDX_MINMAX_CMP5]], <8 x float> [[RDX_MINMAX_SELECT3]], <8 x float> [[RDX_SHUF4]]
-; AVX-NEXT:    [[TMP16:%.*]] = extractelement <8 x float> [[RDX_MINMAX_SELECT6]], i32 0
-; AVX-NEXT:    [[TMP17:%.*]] = select i1 [[TMP15]], float [[TMP14]], float undef
-; AVX-NEXT:    ret float [[TMP16]]
-;
-; AVX2-LABEL: @maxf8(
-; AVX2-NEXT:    [[TMP2:%.*]] = load <8 x float>, <8 x float>* bitcast ([32 x float]* @arr1 to <8 x float>*), align 16
-; AVX2-NEXT:    [[TMP3:%.*]] = fcmp fast ogt float undef, undef
-; AVX2-NEXT:    [[TMP4:%.*]] = select i1 [[TMP3]], float undef, float undef
-; AVX2-NEXT:    [[TMP5:%.*]] = fcmp fast ogt float [[TMP4]], undef
-; AVX2-NEXT:    [[TMP6:%.*]] = select i1 [[TMP5]], float [[TMP4]], float undef
-; AVX2-NEXT:    [[TMP7:%.*]] = fcmp fast ogt float [[TMP6]], undef
-; AVX2-NEXT:    [[TMP8:%.*]] = select i1 [[TMP7]], float [[TMP6]], float undef
-; AVX2-NEXT:    [[TMP9:%.*]] = fcmp fast ogt float [[TMP8]], undef
-; AVX2-NEXT:    [[TMP10:%.*]] = select i1 [[TMP9]], float [[TMP8]], float undef
-; AVX2-NEXT:    [[TMP11:%.*]] = fcmp fast ogt float [[TMP10]], undef
-; AVX2-NEXT:    [[TMP12:%.*]] = select i1 [[TMP11]], float [[TMP10]], float undef
-; AVX2-NEXT:    [[TMP13:%.*]] = fcmp fast ogt float [[TMP12]], undef
-; AVX2-NEXT:    [[TMP14:%.*]] = select i1 [[TMP13]], float [[TMP12]], float undef
-; AVX2-NEXT:    [[TMP15:%.*]] = fcmp fast ogt float [[TMP14]], undef
-; AVX2-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <8 x float> [[TMP2]], <8 x float> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX2-NEXT:    [[RDX_MINMAX_CMP:%.*]] = fcmp fast ogt <8 x float> [[TMP2]], [[RDX_SHUF]]
-; AVX2-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = select <8 x i1> [[RDX_MINMAX_CMP]], <8 x float> [[TMP2]], <8 x float> [[RDX_SHUF]]
-; AVX2-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <8 x float> [[RDX_MINMAX_SELECT]], <8 x float> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX2-NEXT:    [[RDX_MINMAX_CMP2:%.*]] = fcmp fast ogt <8 x float> [[RDX_MINMAX_SELECT]], [[RDX_SHUF1]]
-; AVX2-NEXT:    [[RDX_MINMAX_SELECT3:%.*]] = select <8 x i1> [[RDX_MINMAX_CMP2]], <8 x float> [[RDX_MINMAX_SELECT]], <8 x float> [[RDX_SHUF1]]
-; AVX2-NEXT:    [[RDX_SHUF4:%.*]] = shufflevector <8 x float> [[RDX_MINMAX_SELECT3]], <8 x float> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX2-NEXT:    [[RDX_MINMAX_CMP5:%.*]] = fcmp fast ogt <8 x float> [[RDX_MINMAX_SELECT3]], [[RDX_SHUF4]]
-; AVX2-NEXT:    [[RDX_MINMAX_SELECT6:%.*]] = select <8 x i1> [[RDX_MINMAX_CMP5]], <8 x float> [[RDX_MINMAX_SELECT3]], <8 x float> [[RDX_SHUF4]]
-; AVX2-NEXT:    [[TMP16:%.*]] = extractelement <8 x float> [[RDX_MINMAX_SELECT6]], i32 0
-; AVX2-NEXT:    [[TMP17:%.*]] = select i1 [[TMP15]], float [[TMP14]], float undef
-; AVX2-NEXT:    ret float [[TMP16]]
-;
-; SKX-LABEL: @maxf8(
-; SKX-NEXT:    [[TMP2:%.*]] = load <8 x float>, <8 x float>* bitcast ([32 x float]* @arr1 to <8 x float>*), align 16
-; SKX-NEXT:    [[TMP3:%.*]] = fcmp fast ogt float undef, undef
-; SKX-NEXT:    [[TMP4:%.*]] = select i1 [[TMP3]], float undef, float undef
-; SKX-NEXT:    [[TMP5:%.*]] = fcmp fast ogt float [[TMP4]], undef
-; SKX-NEXT:    [[TMP6:%.*]] = select i1 [[TMP5]], float [[TMP4]], float undef
-; SKX-NEXT:    [[TMP7:%.*]] = fcmp fast ogt float [[TMP6]], undef
-; SKX-NEXT:    [[TMP8:%.*]] = select i1 [[TMP7]], float [[TMP6]], float undef
-; SKX-NEXT:    [[TMP9:%.*]] = fcmp fast ogt float [[TMP8]], undef
-; SKX-NEXT:    [[TMP10:%.*]] = select i1 [[TMP9]], float [[TMP8]], float undef
-; SKX-NEXT:    [[TMP11:%.*]] = fcmp fast ogt float [[TMP10]], undef
-; SKX-NEXT:    [[TMP12:%.*]] = select i1 [[TMP11]], float [[TMP10]], float undef
-; SKX-NEXT:    [[TMP13:%.*]] = fcmp fast ogt float [[TMP12]], undef
-; SKX-NEXT:    [[TMP14:%.*]] = select i1 [[TMP13]], float [[TMP12]], float undef
-; SKX-NEXT:    [[TMP15:%.*]] = fcmp fast ogt float [[TMP14]], undef
-; SKX-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <8 x float> [[TMP2]], <8 x float> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
-; SKX-NEXT:    [[RDX_MINMAX_CMP:%.*]] = fcmp fast ogt <8 x float> [[TMP2]], [[RDX_SHUF]]
-; SKX-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = select <8 x i1> [[RDX_MINMAX_CMP]], <8 x float> [[TMP2]], <8 x float> [[RDX_SHUF]]
-; SKX-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <8 x float> [[RDX_MINMAX_SELECT]], <8 x float> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SKX-NEXT:    [[RDX_MINMAX_CMP2:%.*]] = fcmp fast ogt <8 x float> [[RDX_MINMAX_SELECT]], [[RDX_SHUF1]]
-; SKX-NEXT:    [[RDX_MINMAX_SELECT3:%.*]] = select <8 x i1> [[RDX_MINMAX_CMP2]], <8 x float> [[RDX_MINMAX_SELECT]], <8 x float> [[RDX_SHUF1]]
-; SKX-NEXT:    [[RDX_SHUF4:%.*]] = shufflevector <8 x float> [[RDX_MINMAX_SELECT3]], <8 x float> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SKX-NEXT:    [[RDX_MINMAX_CMP5:%.*]] = fcmp fast ogt <8 x float> [[RDX_MINMAX_SELECT3]], [[RDX_SHUF4]]
-; SKX-NEXT:    [[RDX_MINMAX_SELECT6:%.*]] = select <8 x i1> [[RDX_MINMAX_CMP5]], <8 x float> [[RDX_MINMAX_SELECT3]], <8 x float> [[RDX_SHUF4]]
-; SKX-NEXT:    [[TMP16:%.*]] = extractelement <8 x float> [[RDX_MINMAX_SELECT6]], i32 0
-; SKX-NEXT:    [[TMP17:%.*]] = select i1 [[TMP15]], float [[TMP14]], float undef
-; SKX-NEXT:    ret float [[TMP16]]
+; CHECK-LABEL: @maxf8(
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x float>, <8 x float>* bitcast ([32 x float]* @arr1 to <8 x float>*), align 16
+; CHECK-NEXT:    [[TMP3:%.*]] = fcmp fast ogt float undef, undef
+; CHECK-NEXT:    [[TMP4:%.*]] = select i1 [[TMP3]], float undef, float undef
+; CHECK-NEXT:    [[TMP5:%.*]] = fcmp fast ogt float [[TMP4]], undef
+; CHECK-NEXT:    [[TMP6:%.*]] = select i1 [[TMP5]], float [[TMP4]], float undef
+; CHECK-NEXT:    [[TMP7:%.*]] = fcmp fast ogt float [[TMP6]], undef
+; CHECK-NEXT:    [[TMP8:%.*]] = select i1 [[TMP7]], float [[TMP6]], float undef
+; CHECK-NEXT:    [[TMP9:%.*]] = fcmp fast ogt float [[TMP8]], undef
+; CHECK-NEXT:    [[TMP10:%.*]] = select i1 [[TMP9]], float [[TMP8]], float undef
+; CHECK-NEXT:    [[TMP11:%.*]] = fcmp fast ogt float [[TMP10]], undef
+; CHECK-NEXT:    [[TMP12:%.*]] = select i1 [[TMP11]], float [[TMP10]], float undef
+; CHECK-NEXT:    [[TMP13:%.*]] = fcmp fast ogt float [[TMP12]], undef
+; CHECK-NEXT:    [[TMP14:%.*]] = select i1 [[TMP13]], float [[TMP12]], float undef
+; CHECK-NEXT:    [[TMP15:%.*]] = fcmp fast ogt float [[TMP14]], undef
+; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <8 x float> [[TMP2]], <8 x float> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[RDX_MINMAX_CMP:%.*]] = fcmp fast ogt <8 x float> [[TMP2]], [[RDX_SHUF]]
+; CHECK-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = select <8 x i1> [[RDX_MINMAX_CMP]], <8 x float> [[TMP2]], <8 x float> [[RDX_SHUF]]
+; CHECK-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <8 x float> [[RDX_MINMAX_SELECT]], <8 x float> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[RDX_MINMAX_CMP2:%.*]] = fcmp fast ogt <8 x float> [[RDX_MINMAX_SELECT]], [[RDX_SHUF1]]
+; CHECK-NEXT:    [[RDX_MINMAX_SELECT3:%.*]] = select <8 x i1> [[RDX_MINMAX_CMP2]], <8 x float> [[RDX_MINMAX_SELECT]], <8 x float> [[RDX_SHUF1]]
+; CHECK-NEXT:    [[RDX_SHUF4:%.*]] = shufflevector <8 x float> [[RDX_MINMAX_SELECT3]], <8 x float> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[RDX_MINMAX_CMP5:%.*]] = fcmp fast ogt <8 x float> [[RDX_MINMAX_SELECT3]], [[RDX_SHUF4]]
+; CHECK-NEXT:    [[RDX_MINMAX_SELECT6:%.*]] = select <8 x i1> [[RDX_MINMAX_CMP5]], <8 x float> [[RDX_MINMAX_SELECT3]], <8 x float> [[RDX_SHUF4]]
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <8 x float> [[RDX_MINMAX_SELECT6]], i32 0
+; CHECK-NEXT:    [[TMP17:%.*]] = select i1 [[TMP15]], float [[TMP14]], float undef
+; CHECK-NEXT:    ret float [[TMP16]]
 ;
   %2 = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr1, i64 0, i64 0), align 16
   %3 = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr1, i64 0, i64 1), align 4
@@ -476,195 +395,52 @@ define float @maxf8(float) {
 }
 
 define float @maxf16(float) {
-; SSE-LABEL: @maxf16(
-; SSE-NEXT:    [[TMP2:%.*]] = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr1, i64 0, i64 0), align 16
-; SSE-NEXT:    [[TMP3:%.*]] = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr1, i64 0, i64 1), align 4
-; SSE-NEXT:    [[TMP4:%.*]] = fcmp fast ogt float [[TMP2]], [[TMP3]]
-; SSE-NEXT:    [[TMP5:%.*]] = select i1 [[TMP4]], float [[TMP2]], float [[TMP3]]
-; SSE-NEXT:    [[TMP6:%.*]] = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr1, i64 0, i64 2), align 8
-; SSE-NEXT:    [[TMP7:%.*]] = fcmp fast ogt float [[TMP5]], [[TMP6]]
-; SSE-NEXT:    [[TMP8:%.*]] = select i1 [[TMP7]], float [[TMP5]], float [[TMP6]]
-; SSE-NEXT:    [[TMP9:%.*]] = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr1, i64 0, i64 3), align 4
-; SSE-NEXT:    [[TMP10:%.*]] = fcmp fast ogt float [[TMP8]], [[TMP9]]
-; SSE-NEXT:    [[TMP11:%.*]] = select i1 [[TMP10]], float [[TMP8]], float [[TMP9]]
-; SSE-NEXT:    [[TMP12:%.*]] = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr1, i64 0, i64 4), align 16
-; SSE-NEXT:    [[TMP13:%.*]] = fcmp fast ogt float [[TMP11]], [[TMP12]]
-; SSE-NEXT:    [[TMP14:%.*]] = select i1 [[TMP13]], float [[TMP11]], float [[TMP12]]
-; SSE-NEXT:    [[TMP15:%.*]] = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr1, i64 0, i64 5), align 4
-; SSE-NEXT:    [[TMP16:%.*]] = fcmp fast ogt float [[TMP14]], [[TMP15]]
-; SSE-NEXT:    [[TMP17:%.*]] = select i1 [[TMP16]], float [[TMP14]], float [[TMP15]]
-; SSE-NEXT:    [[TMP18:%.*]] = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr1, i64 0, i64 6), align 8
-; SSE-NEXT:    [[TMP19:%.*]] = fcmp fast ogt float [[TMP17]], [[TMP18]]
-; SSE-NEXT:    [[TMP20:%.*]] = select i1 [[TMP19]], float [[TMP17]], float [[TMP18]]
-; SSE-NEXT:    [[TMP21:%.*]] = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr1, i64 0, i64 7), align 4
-; SSE-NEXT:    [[TMP22:%.*]] = fcmp fast ogt float [[TMP20]], [[TMP21]]
-; SSE-NEXT:    [[TMP23:%.*]] = select i1 [[TMP22]], float [[TMP20]], float [[TMP21]]
-; SSE-NEXT:    [[TMP24:%.*]] = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr1, i64 0, i64 8), align 16
-; SSE-NEXT:    [[TMP25:%.*]] = fcmp fast ogt float [[TMP23]], [[TMP24]]
-; SSE-NEXT:    [[TMP26:%.*]] = select i1 [[TMP25]], float [[TMP23]], float [[TMP24]]
-; SSE-NEXT:    [[TMP27:%.*]] = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr1, i64 0, i64 9), align 4
-; SSE-NEXT:    [[TMP28:%.*]] = fcmp fast ogt float [[TMP26]], [[TMP27]]
-; SSE-NEXT:    [[TMP29:%.*]] = select i1 [[TMP28]], float [[TMP26]], float [[TMP27]]
-; SSE-NEXT:    [[TMP30:%.*]] = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr1, i64 0, i64 10), align 8
-; SSE-NEXT:    [[TMP31:%.*]] = fcmp fast ogt float [[TMP29]], [[TMP30]]
-; SSE-NEXT:    [[TMP32:%.*]] = select i1 [[TMP31]], float [[TMP29]], float [[TMP30]]
-; SSE-NEXT:    [[TMP33:%.*]] = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr1, i64 0, i64 11), align 4
-; SSE-NEXT:    [[TMP34:%.*]] = fcmp fast ogt float [[TMP32]], [[TMP33]]
-; SSE-NEXT:    [[TMP35:%.*]] = select i1 [[TMP34]], float [[TMP32]], float [[TMP33]]
-; SSE-NEXT:    [[TMP36:%.*]] = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr1, i64 0, i64 12), align 16
-; SSE-NEXT:    [[TMP37:%.*]] = fcmp fast ogt float [[TMP35]], [[TMP36]]
-; SSE-NEXT:    [[TMP38:%.*]] = select i1 [[TMP37]], float [[TMP35]], float [[TMP36]]
-; SSE-NEXT:    [[TMP39:%.*]] = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr1, i64 0, i64 13), align 4
-; SSE-NEXT:    [[TMP40:%.*]] = fcmp fast ogt float [[TMP38]], [[TMP39]]
-; SSE-NEXT:    [[TMP41:%.*]] = select i1 [[TMP40]], float [[TMP38]], float [[TMP39]]
-; SSE-NEXT:    [[TMP42:%.*]] = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr1, i64 0, i64 14), align 8
-; SSE-NEXT:    [[TMP43:%.*]] = fcmp fast ogt float [[TMP41]], [[TMP42]]
-; SSE-NEXT:    [[TMP44:%.*]] = select i1 [[TMP43]], float [[TMP41]], float [[TMP42]]
-; SSE-NEXT:    [[TMP45:%.*]] = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr1, i64 0, i64 15), align 4
-; SSE-NEXT:    [[TMP46:%.*]] = fcmp fast ogt float [[TMP44]], [[TMP45]]
-; SSE-NEXT:    [[TMP47:%.*]] = select i1 [[TMP46]], float [[TMP44]], float [[TMP45]]
-; SSE-NEXT:    ret float [[TMP47]]
-;
-; AVX-LABEL: @maxf16(
-; AVX-NEXT:    [[TMP2:%.*]] = load <16 x float>, <16 x float>* bitcast ([32 x float]* @arr1 to <16 x float>*), align 16
-; AVX-NEXT:    [[TMP3:%.*]] = fcmp fast ogt float undef, undef
-; AVX-NEXT:    [[TMP4:%.*]] = select i1 [[TMP3]], float undef, float undef
-; AVX-NEXT:    [[TMP5:%.*]] = fcmp fast ogt float [[TMP4]], undef
-; AVX-NEXT:    [[TMP6:%.*]] = select i1 [[TMP5]], float [[TMP4]], float undef
-; AVX-NEXT:    [[TMP7:%.*]] = fcmp fast ogt float [[TMP6]], undef
-; AVX-NEXT:    [[TMP8:%.*]] = select i1 [[TMP7]], float [[TMP6]], float undef
-; AVX-NEXT:    [[TMP9:%.*]] = fcmp fast ogt float [[TMP8]], undef
-; AVX-NEXT:    [[TMP10:%.*]] = select i1 [[TMP9]], float [[TMP8]], float undef
-; AVX-NEXT:    [[TMP11:%.*]] = fcmp fast ogt float [[TMP10]], undef
-; AVX-NEXT:    [[TMP12:%.*]] = select i1 [[TMP11]], float [[TMP10]], float undef
-; AVX-NEXT:    [[TMP13:%.*]] = fcmp fast ogt float [[TMP12]], undef
-; AVX-NEXT:    [[TMP14:%.*]] = select i1 [[TMP13]], float [[TMP12]], float undef
-; AVX-NEXT:    [[TMP15:%.*]] = fcmp fast ogt float [[TMP14]], undef
-; AVX-NEXT:    [[TMP16:%.*]] = select i1 [[TMP15]], float [[TMP14]], float undef
-; AVX-NEXT:    [[TMP17:%.*]] = fcmp fast ogt float [[TMP16]], undef
-; AVX-NEXT:    [[TMP18:%.*]] = select i1 [[TMP17]], float [[TMP16]], float undef
-; AVX-NEXT:    [[TMP19:%.*]] = fcmp fast ogt float [[TMP18]], undef
-; AVX-NEXT:    [[TMP20:%.*]] = select i1 [[TMP19]], float [[TMP18]], float undef
-; AVX-NEXT:    [[TMP21:%.*]] = fcmp fast ogt float [[TMP20]], undef
-; AVX-NEXT:    [[TMP22:%.*]] = select i1 [[TMP21]], float [[TMP20]], float undef
-; AVX-NEXT:    [[TMP23:%.*]] = fcmp fast ogt float [[TMP22]], undef
-; AVX-NEXT:    [[TMP24:%.*]] = select i1 [[TMP23]], float [[TMP22]], float undef
-; AVX-NEXT:    [[TMP25:%.*]] = fcmp fast ogt float [[TMP24]], undef
-; AVX-NEXT:    [[TMP26:%.*]] = select i1 [[TMP25]], float [[TMP24]], float undef
-; AVX-NEXT:    [[TMP27:%.*]] = fcmp fast ogt float [[TMP26]], undef
-; AVX-NEXT:    [[TMP28:%.*]] = select i1 [[TMP27]], float [[TMP26]], float undef
-; AVX-NEXT:    [[TMP29:%.*]] = fcmp fast ogt float [[TMP28]], undef
-; AVX-NEXT:    [[TMP30:%.*]] = select i1 [[TMP29]], float [[TMP28]], float undef
-; AVX-NEXT:    [[TMP31:%.*]] = fcmp fast ogt float [[TMP30]], undef
-; AVX-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <16 x float> [[TMP2]], <16 x float> undef, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX-NEXT:    [[RDX_MINMAX_CMP:%.*]] = fcmp fast ogt <16 x float> [[TMP2]], [[RDX_SHUF]]
-; AVX-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = select <16 x i1> [[RDX_MINMAX_CMP]], <16 x float> [[TMP2]], <16 x float> [[RDX_SHUF]]
-; AVX-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <16 x float> [[RDX_MINMAX_SELECT]], <16 x float> undef, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX-NEXT:    [[RDX_MINMAX_CMP2:%.*]] = fcmp fast ogt <16 x float> [[RDX_MINMAX_SELECT]], [[RDX_SHUF1]]
-; AVX-NEXT:    [[RDX_MINMAX_SELECT3:%.*]] = select <16 x i1> [[RDX_MINMAX_CMP2]], <16 x float> [[RDX_MINMAX_SELECT]], <16 x float> [[RDX_SHUF1]]
-; AVX-NEXT:    [[RDX_SHUF4:%.*]] = shufflevector <16 x float> [[RDX_MINMAX_SELECT3]], <16 x float> undef, <16 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX-NEXT:    [[RDX_MINMAX_CMP5:%.*]] = fcmp fast ogt <16 x float> [[RDX_MINMAX_SELECT3]], [[RDX_SHUF4]]
-; AVX-NEXT:    [[RDX_MINMAX_SELECT6:%.*]] = select <16 x i1> [[RDX_MINMAX_CMP5]], <16 x float> [[RDX_MINMAX_SELECT3]], <16 x float> [[RDX_SHUF4]]
-; AVX-NEXT:    [[RDX_SHUF7:%.*]] = shufflevector <16 x float> [[RDX_MINMAX_SELECT6]], <16 x float> undef, <16 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX-NEXT:    [[RDX_MINMAX_CMP8:%.*]] = fcmp fast ogt <16 x float> [[RDX_MINMAX_SELECT6]], [[RDX_SHUF7]]
-; AVX-NEXT:    [[RDX_MINMAX_SELECT9:%.*]] = select <16 x i1> [[RDX_MINMAX_CMP8]], <16 x float> [[RDX_MINMAX_SELECT6]], <16 x float> [[RDX_SHUF7]]
-; AVX-NEXT:    [[TMP32:%.*]] = extractelement <16 x float> [[RDX_MINMAX_SELECT9]], i32 0
-; AVX-NEXT:    [[TMP33:%.*]] = select i1 [[TMP31]], float [[TMP30]], float undef
-; AVX-NEXT:    ret float [[TMP32]]
-;
-; AVX2-LABEL: @maxf16(
-; AVX2-NEXT:    [[TMP2:%.*]] = load <16 x float>, <16 x float>* bitcast ([32 x float]* @arr1 to <16 x float>*), align 16
-; AVX2-NEXT:    [[TMP3:%.*]] = fcmp fast ogt float undef, undef
-; AVX2-NEXT:    [[TMP4:%.*]] = select i1 [[TMP3]], float undef, float undef
-; AVX2-NEXT:    [[TMP5:%.*]] = fcmp fast ogt float [[TMP4]], undef
-; AVX2-NEXT:    [[TMP6:%.*]] = select i1 [[TMP5]], float [[TMP4]], float undef
-; AVX2-NEXT:    [[TMP7:%.*]] = fcmp fast ogt float [[TMP6]], undef
-; AVX2-NEXT:    [[TMP8:%.*]] = select i1 [[TMP7]], float [[TMP6]], float undef
-; AVX2-NEXT:    [[TMP9:%.*]] = fcmp fast ogt float [[TMP8]], undef
-; AVX2-NEXT:    [[TMP10:%.*]] = select i1 [[TMP9]], float [[TMP8]], float undef
-; AVX2-NEXT:    [[TMP11:%.*]] = fcmp fast ogt float [[TMP10]], undef
-; AVX2-NEXT:    [[TMP12:%.*]] = select i1 [[TMP11]], float [[TMP10]], float undef
-; AVX2-NEXT:    [[TMP13:%.*]] = fcmp fast ogt float [[TMP12]], undef
-; AVX2-NEXT:    [[TMP14:%.*]] = select i1 [[TMP13]], float [[TMP12]], float undef
-; AVX2-NEXT:    [[TMP15:%.*]] = fcmp fast ogt float [[TMP14]], undef
-; AVX2-NEXT:    [[TMP16:%.*]] = select i1 [[TMP15]], float [[TMP14]], float undef
-; AVX2-NEXT:    [[TMP17:%.*]] = fcmp fast ogt float [[TMP16]], undef
-; AVX2-NEXT:    [[TMP18:%.*]] = select i1 [[TMP17]], float [[TMP16]], float undef
-; AVX2-NEXT:    [[TMP19:%.*]] = fcmp fast ogt float [[TMP18]], undef
-; AVX2-NEXT:    [[TMP20:%.*]] = select i1 [[TMP19]], float [[TMP18]], float undef
-; AVX2-NEXT:    [[TMP21:%.*]] = fcmp fast ogt float [[TMP20]], undef
-; AVX2-NEXT:    [[TMP22:%.*]] = select i1 [[TMP21]], float [[TMP20]], float undef
-; AVX2-NEXT:    [[TMP23:%.*]] = fcmp fast ogt float [[TMP22]], undef
-; AVX2-NEXT:    [[TMP24:%.*]] = select i1 [[TMP23]], float [[TMP22]], float undef
-; AVX2-NEXT:    [[TMP25:%.*]] = fcmp fast ogt float [[TMP24]], undef
-; AVX2-NEXT:    [[TMP26:%.*]] = select i1 [[TMP25]], float [[TMP24]], float undef
-; AVX2-NEXT:    [[TMP27:%.*]] = fcmp fast ogt float [[TMP26]], undef
-; AVX2-NEXT:    [[TMP28:%.*]] = select i1 [[TMP27]], float [[TMP26]], float undef
-; AVX2-NEXT:    [[TMP29:%.*]] = fcmp fast ogt float [[TMP28]], undef
-; AVX2-NEXT:    [[TMP30:%.*]] = select i1 [[TMP29]], float [[TMP28]], float undef
-; AVX2-NEXT:    [[TMP31:%.*]] = fcmp fast ogt float [[TMP30]], undef
-; AVX2-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <16 x float> [[TMP2]], <16 x float> undef, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX2-NEXT:    [[RDX_MINMAX_CMP:%.*]] = fcmp fast ogt <16 x float> [[TMP2]], [[RDX_SHUF]]
-; AVX2-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = select <16 x i1> [[RDX_MINMAX_CMP]], <16 x float> [[TMP2]], <16 x float> [[RDX_SHUF]]
-; AVX2-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <16 x float> [[RDX_MINMAX_SELECT]], <16 x float> undef, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX2-NEXT:    [[RDX_MINMAX_CMP2:%.*]] = fcmp fast ogt <16 x float> [[RDX_MINMAX_SELECT]], [[RDX_SHUF1]]
-; AVX2-NEXT:    [[RDX_MINMAX_SELECT3:%.*]] = select <16 x i1> [[RDX_MINMAX_CMP2]], <16 x float> [[RDX_MINMAX_SELECT]], <16 x float> [[RDX_SHUF1]]
-; AVX2-NEXT:    [[RDX_SHUF4:%.*]] = shufflevector <16 x float> [[RDX_MINMAX_SELECT3]], <16 x float> undef, <16 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX2-NEXT:    [[RDX_MINMAX_CMP5:%.*]] = fcmp fast ogt <16 x float> [[RDX_MINMAX_SELECT3]], [[RDX_SHUF4]]
-; AVX2-NEXT:    [[RDX_MINMAX_SELECT6:%.*]] = select <16 x i1> [[RDX_MINMAX_CMP5]], <16 x float> [[RDX_MINMAX_SELECT3]], <16 x float> [[RDX_SHUF4]]
-; AVX2-NEXT:    [[RDX_SHUF7:%.*]] = shufflevector <16 x float> [[RDX_MINMAX_SELECT6]], <16 x float> undef, <16 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; AVX2-NEXT:    [[RDX_MINMAX_CMP8:%.*]] = fcmp fast ogt <16 x float> [[RDX_MINMAX_SELECT6]], [[RDX_SHUF7]]
-; AVX2-NEXT:    [[RDX_MINMAX_SELECT9:%.*]] = select <16 x i1> [[RDX_MINMAX_CMP8]], <16 x float> [[RDX_MINMAX_SELECT6]], <16 x float> [[RDX_SHUF7]]
-; AVX2-NEXT:    [[TMP32:%.*]] = extractelement <16 x float> [[RDX_MINMAX_SELECT9]], i32 0
-; AVX2-NEXT:    [[TMP33:%.*]] = select i1 [[TMP31]], float [[TMP30]], float undef
-; AVX2-NEXT:    ret float [[TMP32]]
-;
-; SKX-LABEL: @maxf16(
-; SKX-NEXT:    [[TMP2:%.*]] = load <16 x float>, <16 x float>* bitcast ([32 x float]* @arr1 to <16 x float>*), align 16
-; SKX-NEXT:    [[TMP3:%.*]] = fcmp fast ogt float undef, undef
-; SKX-NEXT:    [[TMP4:%.*]] = select i1 [[TMP3]], float undef, float undef
-; SKX-NEXT:    [[TMP5:%.*]] = fcmp fast ogt float [[TMP4]], undef
-; SKX-NEXT:    [[TMP6:%.*]] = select i1 [[TMP5]], float [[TMP4]], float undef
-; SKX-NEXT:    [[TMP7:%.*]] = fcmp fast ogt float [[TMP6]], undef
-; SKX-NEXT:    [[TMP8:%.*]] = select i1 [[TMP7]], float [[TMP6]], float undef
-; SKX-NEXT:    [[TMP9:%.*]] = fcmp fast ogt float [[TMP8]], undef
-; SKX-NEXT:    [[TMP10:%.*]] = select i1 [[TMP9]], float [[TMP8]], float undef
-; SKX-NEXT:    [[TMP11:%.*]] = fcmp fast ogt float [[TMP10]], undef
-; SKX-NEXT:    [[TMP12:%.*]] = select i1 [[TMP11]], float [[TMP10]], float undef
-; SKX-NEXT:    [[TMP13:%.*]] = fcmp fast ogt float [[TMP12]], undef
-; SKX-NEXT:    [[TMP14:%.*]] = select i1 [[TMP13]], float [[TMP12]], float undef
-; SKX-NEXT:    [[TMP15:%.*]] = fcmp fast ogt float [[TMP14]], undef
-; SKX-NEXT:    [[TMP16:%.*]] = select i1 [[TMP15]], float [[TMP14]], float undef
-; SKX-NEXT:    [[TMP17:%.*]] = fcmp fast ogt float [[TMP16]], undef
-; SKX-NEXT:    [[TMP18:%.*]] = select i1 [[TMP17]], float [[TMP16]], float undef
-; SKX-NEXT:    [[TMP19:%.*]] = fcmp fast ogt float [[TMP18]], undef
-; SKX-NEXT:    [[TMP20:%.*]] = select i1 [[TMP19]], float [[TMP18]], float undef
-; SKX-NEXT:    [[TMP21:%.*]] = fcmp fast ogt float [[TMP20]], undef
-; SKX-NEXT:    [[TMP22:%.*]] = select i1 [[TMP21]], float [[TMP20]], float undef
-; SKX-NEXT:    [[TMP23:%.*]] = fcmp fast ogt float [[TMP22]], undef
-; SKX-NEXT:    [[TMP24:%.*]] = select i1 [[TMP23]], float [[TMP22]], float undef
-; SKX-NEXT:    [[TMP25:%.*]] = fcmp fast ogt float [[TMP24]], undef
-; SKX-NEXT:    [[TMP26:%.*]] = select i1 [[TMP25]], float [[TMP24]], float undef
-; SKX-NEXT:    [[TMP27:%.*]] = fcmp fast ogt float [[TMP26]], undef
-; SKX-NEXT:    [[TMP28:%.*]] = select i1 [[TMP27]], float [[TMP26]], float undef
-; SKX-NEXT:    [[TMP29:%.*]] = fcmp fast ogt float [[TMP28]], undef
-; SKX-NEXT:    [[TMP30:%.*]] = select i1 [[TMP29]], float [[TMP28]], float undef
-; SKX-NEXT:    [[TMP31:%.*]] = fcmp fast ogt float [[TMP30]], undef
-; SKX-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <16 x float> [[TMP2]], <16 x float> undef, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SKX-NEXT:    [[RDX_MINMAX_CMP:%.*]] = fcmp fast ogt <16 x float> [[TMP2]], [[RDX_SHUF]]
-; SKX-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = select <16 x i1> [[RDX_MINMAX_CMP]], <16 x float> [[TMP2]], <16 x float> [[RDX_SHUF]]
-; SKX-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <16 x float> [[RDX_MINMAX_SELECT]], <16 x float> undef, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SKX-NEXT:    [[RDX_MINMAX_CMP2:%.*]] = fcmp fast ogt <16 x float> [[RDX_MINMAX_SELECT]], [[RDX_SHUF1]]
-; SKX-NEXT:    [[RDX_MINMAX_SELECT3:%.*]] = select <16 x i1> [[RDX_MINMAX_CMP2]], <16 x float> [[RDX_MINMAX_SELECT]], <16 x float> [[RDX_SHUF1]]
-; SKX-NEXT:    [[RDX_SHUF4:%.*]] = shufflevector <16 x float> [[RDX_MINMAX_SELECT3]], <16 x float> undef, <16 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SKX-NEXT:    [[RDX_MINMAX_CMP5:%.*]] = fcmp fast ogt <16 x float> [[RDX_MINMAX_SELECT3]], [[RDX_SHUF4]]
-; SKX-NEXT:    [[RDX_MINMAX_SELECT6:%.*]] = select <16 x i1> [[RDX_MINMAX_CMP5]], <16 x float> [[RDX_MINMAX_SELECT3]], <16 x float> [[RDX_SHUF4]]
-; SKX-NEXT:    [[RDX_SHUF7:%.*]] = shufflevector <16 x float> [[RDX_MINMAX_SELECT6]], <16 x float> undef, <16 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
-; SKX-NEXT:    [[RDX_MINMAX_CMP8:%.*]] = fcmp fast ogt <16 x float> [[RDX_MINMAX_SELECT6]], [[RDX_SHUF7]]
-; SKX-NEXT:    [[RDX_MINMAX_SELECT9:%.*]] = select <16 x i1> [[RDX_MINMAX_CMP8]], <16 x float> [[RDX_MINMAX_SELECT6]], <16 x float> [[RDX_SHUF7]]
-; SKX-NEXT:    [[TMP32:%.*]] = extractelement <16 x float> [[RDX_MINMAX_SELECT9]], i32 0
-; SKX-NEXT:    [[TMP33:%.*]] = select i1 [[TMP31]], float [[TMP30]], float undef
-; SKX-NEXT:    ret float [[TMP32]]
+; CHECK-LABEL: @maxf16(
+; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x float>, <16 x float>* bitcast ([32 x float]* @arr1 to <16 x float>*), align 16
+; CHECK-NEXT:    [[TMP3:%.*]] = fcmp fast ogt float undef, undef
+; CHECK-NEXT:    [[TMP4:%.*]] = select i1 [[TMP3]], float undef, float undef
+; CHECK-NEXT:    [[TMP5:%.*]] = fcmp fast ogt float [[TMP4]], undef
+; CHECK-NEXT:    [[TMP6:%.*]] = select i1 [[TMP5]], float [[TMP4]], float undef
+; CHECK-NEXT:    [[TMP7:%.*]] = fcmp fast ogt float [[TMP6]], undef
+; CHECK-NEXT:    [[TMP8:%.*]] = select i1 [[TMP7]], float [[TMP6]], float undef
+; CHECK-NEXT:    [[TMP9:%.*]] = fcmp fast ogt float [[TMP8]], undef
+; CHECK-NEXT:    [[TMP10:%.*]] = select i1 [[TMP9]], float [[TMP8]], float undef
+; CHECK-NEXT:    [[TMP11:%.*]] = fcmp fast ogt float [[TMP10]], undef
+; CHECK-NEXT:    [[TMP12:%.*]] = select i1 [[TMP11]], float [[TMP10]], float undef
+; CHECK-NEXT:    [[TMP13:%.*]] = fcmp fast ogt float [[TMP12]], undef
+; CHECK-NEXT:    [[TMP14:%.*]] = select i1 [[TMP13]], float [[TMP12]], float undef
+; CHECK-NEXT:    [[TMP15:%.*]] = fcmp fast ogt float [[TMP14]], undef
+; CHECK-NEXT:    [[TMP16:%.*]] = select i1 [[TMP15]], float [[TMP14]], float undef
+; CHECK-NEXT:    [[TMP17:%.*]] = fcmp fast ogt float [[TMP16]], undef
+; CHECK-NEXT:    [[TMP18:%.*]] = select i1 [[TMP17]], float [[TMP16]], float undef
+; CHECK-NEXT:    [[TMP19:%.*]] = fcmp fast ogt float [[TMP18]], undef
+; CHECK-NEXT:    [[TMP20:%.*]] = select i1 [[TMP19]], float [[TMP18]], float undef
+; CHECK-NEXT:    [[TMP21:%.*]] = fcmp fast ogt float [[TMP20]], undef
+; CHECK-NEXT:    [[TMP22:%.*]] = select i1 [[TMP21]], float [[TMP20]], float undef
+; CHECK-NEXT:    [[TMP23:%.*]] = fcmp fast ogt float [[TMP22]], undef
+; CHECK-NEXT:    [[TMP24:%.*]] = select i1 [[TMP23]], float [[TMP22]], float undef
+; CHECK-NEXT:    [[TMP25:%.*]] = fcmp fast ogt float [[TMP24]], undef
+; CHECK-NEXT:    [[TMP26:%.*]] = select i1 [[TMP25]], float [[TMP24]], float undef
+; CHECK-NEXT:    [[TMP27:%.*]] = fcmp fast ogt float [[TMP26]], undef
+; CHECK-NEXT:    [[TMP28:%.*]] = select i1 [[TMP27]], float [[TMP26]], float undef
+; CHECK-NEXT:    [[TMP29:%.*]] = fcmp fast ogt float [[TMP28]], undef
+; CHECK-NEXT:    [[TMP30:%.*]] = select i1 [[TMP29]], float [[TMP28]], float undef
+; CHECK-NEXT:    [[TMP31:%.*]] = fcmp fast ogt float [[TMP30]], undef
+; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <16 x float> [[TMP2]], <16 x float> undef, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[RDX_MINMAX_CMP:%.*]] = fcmp fast ogt <16 x float> [[TMP2]], [[RDX_SHUF]]
+; CHECK-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = select <16 x i1> [[RDX_MINMAX_CMP]], <16 x float> [[TMP2]], <16 x float> [[RDX_SHUF]]
+; CHECK-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <16 x float> [[RDX_MINMAX_SELECT]], <16 x float> undef, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[RDX_MINMAX_CMP2:%.*]] = fcmp fast ogt <16 x float> [[RDX_MINMAX_SELECT]], [[RDX_SHUF1]]
+; CHECK-NEXT:    [[RDX_MINMAX_SELECT3:%.*]] = select <16 x i1> [[RDX_MINMAX_CMP2]], <16 x float> [[RDX_MINMAX_SELECT]], <16 x float> [[RDX_SHUF1]]
+; CHECK-NEXT:    [[RDX_SHUF4:%.*]] = shufflevector <16 x float> [[RDX_MINMAX_SELECT3]], <16 x float> undef, <16 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[RDX_MINMAX_CMP5:%.*]] = fcmp fast ogt <16 x float> [[RDX_MINMAX_SELECT3]], [[RDX_SHUF4]]
+; CHECK-NEXT:    [[RDX_MINMAX_SELECT6:%.*]] = select <16 x i1> [[RDX_MINMAX_CMP5]], <16 x float> [[RDX_MINMAX_SELECT3]], <16 x float> [[RDX_SHUF4]]
+; CHECK-NEXT:    [[RDX_SHUF7:%.*]] = shufflevector <16 x float> [[RDX_MINMAX_SELECT6]], <16 x float> undef, <16 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[RDX_MINMAX_CMP8:%.*]] = fcmp fast ogt <16 x float> [[RDX_MINMAX_SELECT6]], [[RDX_SHUF7]]
+; CHECK-NEXT:    [[RDX_MINMAX_SELECT9:%.*]] = select <16 x i1> [[RDX_MINMAX_CMP8]], <16 x float> [[RDX_MINMAX_SELECT6]], <16 x float> [[RDX_SHUF7]]
+; CHECK-NEXT:    [[TMP32:%.*]] = extractelement <16 x float> [[RDX_MINMAX_SELECT9]], i32 0
+; CHECK-NEXT:    [[TMP33:%.*]] = select i1 [[TMP31]], float [[TMP30]], float undef
+; CHECK-NEXT:    ret float [[TMP32]]
 ;
   %2 = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr1, i64 0, i64 0), align 16
   %3 = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr1, i64 0, i64 1), align 4
@@ -901,35 +677,27 @@ define i32 @maxi8_mutiple_uses(i32) {
 ; SSE-NEXT:    [[TMP3:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 1), align 4
 ; SSE-NEXT:    [[TMP4:%.*]] = icmp sgt i32 [[TMP2]], [[TMP3]]
 ; SSE-NEXT:    [[TMP5:%.*]] = select i1 [[TMP4]], i32 [[TMP2]], i32 [[TMP3]]
-; SSE-NEXT:    [[TMP6:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 2) to <4 x i32>*), align 8
-; SSE-NEXT:    [[TMP7:%.*]] = icmp sgt i32 [[TMP5]], undef
-; SSE-NEXT:    [[TMP8:%.*]] = select i1 [[TMP7]], i32 [[TMP5]], i32 undef
-; SSE-NEXT:    [[TMP9:%.*]] = icmp sgt i32 [[TMP8]], undef
-; SSE-NEXT:    [[TMP10:%.*]] = select i1 [[TMP9]], i32 [[TMP8]], i32 undef
-; SSE-NEXT:    [[TMP11:%.*]] = icmp sgt i32 [[TMP10]], undef
-; SSE-NEXT:    [[TMP12:%.*]] = select i1 [[TMP11]], i32 [[TMP10]], i32 undef
-; SSE-NEXT:    [[TMP13:%.*]] = icmp sgt i32 [[TMP12]], undef
-; SSE-NEXT:    [[TMP14:%.*]] = select i1 [[TMP13]], i32 [[TMP12]], i32 undef
-; SSE-NEXT:    [[TMP15:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 6), align 8
+; SSE-NEXT:    [[TMP6:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 2), align 8
+; SSE-NEXT:    [[TMP7:%.*]] = icmp sgt i32 [[TMP5]], [[TMP6]]
+; SSE-NEXT:    [[TMP8:%.*]] = select i1 [[TMP7]], i32 [[TMP5]], i32 [[TMP6]]
+; SSE-NEXT:    [[TMP9:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 3), align 4
+; SSE-NEXT:    [[TMP10:%.*]] = icmp sgt i32 [[TMP8]], [[TMP9]]
+; SSE-NEXT:    [[TMP11:%.*]] = select i1 [[TMP10]], i32 [[TMP8]], i32 [[TMP9]]
+; SSE-NEXT:    [[TMP12:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 4), align 16
+; SSE-NEXT:    [[TMP13:%.*]] = icmp sgt i32 [[TMP11]], [[TMP12]]
+; SSE-NEXT:    [[TMP14:%.*]] = select i1 [[TMP13]], i32 [[TMP11]], i32 [[TMP12]]
+; SSE-NEXT:    [[TMP15:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 5), align 4
 ; SSE-NEXT:    [[TMP16:%.*]] = icmp sgt i32 [[TMP14]], [[TMP15]]
-; SSE-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x i32> [[TMP6]], <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
-; SSE-NEXT:    [[RDX_MINMAX_CMP:%.*]] = icmp sgt <4 x i32> [[TMP6]], [[RDX_SHUF]]
-; SSE-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP]], <4 x i32> [[TMP6]], <4 x i32> [[RDX_SHUF]]
-; SSE-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <4 x i32> [[RDX_MINMAX_SELECT]], <4 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
-; SSE-NEXT:    [[RDX_MINMAX_CMP2:%.*]] = icmp sgt <4 x i32> [[RDX_MINMAX_SELECT]], [[RDX_SHUF1]]
-; SSE-NEXT:    [[RDX_MINMAX_SELECT3:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP2]], <4 x i32> [[RDX_MINMAX_SELECT]], <4 x i32> [[RDX_SHUF1]]
-; SSE-NEXT:    [[TMP17:%.*]] = extractelement <4 x i32> [[RDX_MINMAX_SELECT3]], i32 0
-; SSE-NEXT:    [[TMP18:%.*]] = icmp sgt i32 [[TMP17]], [[TMP15]]
-; SSE-NEXT:    [[TMP19:%.*]] = select i1 [[TMP18]], i32 [[TMP17]], i32 [[TMP15]]
-; SSE-NEXT:    [[TMP20:%.*]] = icmp sgt i32 [[TMP19]], [[TMP5]]
-; SSE-NEXT:    [[OP_EXTRA:%.*]] = select i1 [[TMP20]], i32 [[TMP19]], i32 [[TMP5]]
-; SSE-NEXT:    [[TMP21:%.*]] = select i1 [[TMP16]], i32 [[TMP14]], i32 [[TMP15]]
-; SSE-NEXT:    [[TMP22:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 7), align 4
-; SSE-NEXT:    [[TMP23:%.*]] = icmp sgt i32 [[OP_EXTRA]], [[TMP22]]
-; SSE-NEXT:    [[TMP24:%.*]] = select i1 [[TMP23]], i32 [[OP_EXTRA]], i32 [[TMP22]]
-; SSE-NEXT:    [[TMP25:%.*]] = select i1 [[TMP4]], i32 3, i32 4
-; SSE-NEXT:    store i32 [[TMP25]], i32* @var, align 8
-; SSE-NEXT:    ret i32 [[TMP24]]
+; SSE-NEXT:    [[TMP17:%.*]] = select i1 [[TMP16]], i32 [[TMP14]], i32 [[TMP15]]
+; SSE-NEXT:    [[TMP18:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 6), align 8
+; SSE-NEXT:    [[TMP19:%.*]] = icmp sgt i32 [[TMP17]], [[TMP18]]
+; SSE-NEXT:    [[TMP20:%.*]] = select i1 [[TMP19]], i32 [[TMP17]], i32 [[TMP18]]
+; SSE-NEXT:    [[TMP21:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 7), align 4
+; SSE-NEXT:    [[TMP22:%.*]] = icmp sgt i32 [[TMP20]], [[TMP21]]
+; SSE-NEXT:    [[TMP23:%.*]] = select i1 [[TMP22]], i32 [[TMP20]], i32 [[TMP21]]
+; SSE-NEXT:    [[TMP24:%.*]] = select i1 [[TMP4]], i32 3, i32 4
+; SSE-NEXT:    store i32 [[TMP24]], i32* @var, align 8
+; SSE-NEXT:    ret i32 [[TMP23]]
 ;
 ; AVX-LABEL: @maxi8_mutiple_uses(
 ; AVX-NEXT:    [[TMP2:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 0), align 16
@@ -1077,35 +845,25 @@ define i32 @maxi8_wrong_parent(i32) {
 ; SSE-NEXT:    br label [[PP:%.*]]
 ; SSE:       pp:
 ; SSE-NEXT:    [[TMP5:%.*]] = select i1 [[TMP4]], i32 [[TMP2]], i32 [[TMP3]]
-; SSE-NEXT:    [[TMP6:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 2) to <4 x i32>*), align 8
-; SSE-NEXT:    [[TMP7:%.*]] = icmp sgt i32 [[TMP5]], undef
-; SSE-NEXT:    [[TMP8:%.*]] = select i1 [[TMP7]], i32 [[TMP5]], i32 undef
-; SSE-NEXT:    [[TMP9:%.*]] = icmp sgt i32 [[TMP8]], undef
-; SSE-NEXT:    [[TMP10:%.*]] = select i1 [[TMP9]], i32 [[TMP8]], i32 undef
-; SSE-NEXT:    [[TMP11:%.*]] = icmp sgt i32 [[TMP10]], undef
-; SSE-NEXT:    [[TMP12:%.*]] = select i1 [[TMP11]], i32 [[TMP10]], i32 undef
-; SSE-NEXT:    [[TMP13:%.*]] = icmp sgt i32 [[TMP12]], undef
-; SSE-NEXT:    [[TMP14:%.*]] = select i1 [[TMP13]], i32 [[TMP12]], i32 undef
-; SSE-NEXT:    [[TMP15:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 6), align 8
+; SSE-NEXT:    [[TMP6:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 2), align 8
+; SSE-NEXT:    [[TMP7:%.*]] = icmp sgt i32 [[TMP5]], [[TMP6]]
+; SSE-NEXT:    [[TMP8:%.*]] = select i1 [[TMP7]], i32 [[TMP5]], i32 [[TMP6]]
+; SSE-NEXT:    [[TMP9:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 3), align 4
+; SSE-NEXT:    [[TMP10:%.*]] = icmp sgt i32 [[TMP8]], [[TMP9]]
+; SSE-NEXT:    [[TMP11:%.*]] = select i1 [[TMP10]], i32 [[TMP8]], i32 [[TMP9]]
+; SSE-NEXT:    [[TMP12:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 4), align 16
+; SSE-NEXT:    [[TMP13:%.*]] = icmp sgt i32 [[TMP11]], [[TMP12]]
+; SSE-NEXT:    [[TMP14:%.*]] = select i1 [[TMP13]], i32 [[TMP11]], i32 [[TMP12]]
+; SSE-NEXT:    [[TMP15:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 5), align 4
 ; SSE-NEXT:    [[TMP16:%.*]] = icmp sgt i32 [[TMP14]], [[TMP15]]
 ; SSE-NEXT:    [[TMP17:%.*]] = select i1 [[TMP16]], i32 [[TMP14]], i32 [[TMP15]]
-; SSE-NEXT:    [[TMP18:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 7), align 4
+; SSE-NEXT:    [[TMP18:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 6), align 8
 ; SSE-NEXT:    [[TMP19:%.*]] = icmp sgt i32 [[TMP17]], [[TMP18]]
-; SSE-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x i32> [[TMP6]], <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
-; SSE-NEXT:    [[RDX_MINMAX_CMP:%.*]] = icmp sgt <4 x i32> [[TMP6]], [[RDX_SHUF]]
-; SSE-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP]], <4 x i32> [[TMP6]], <4 x i32> [[RDX_SHUF]]
-; SSE-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <4 x i32> [[RDX_MINMAX_SELECT]], <4 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
-; SSE-NEXT:    [[RDX_MINMAX_CMP2:%.*]] = icmp sgt <4 x i32> [[RDX_MINMAX_SELECT]], [[RDX_SHUF1]]
-; SSE-NEXT:    [[RDX_MINMAX_SELECT3:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP2]], <4 x i32> [[RDX_MINMAX_SELECT]], <4 x i32> [[RDX_SHUF1]]
-; SSE-NEXT:    [[TMP20:%.*]] = extractelement <4 x i32> [[RDX_MINMAX_SELECT3]], i32 0
-; SSE-NEXT:    [[TMP21:%.*]] = icmp sgt i32 [[TMP20]], [[TMP15]]
-; SSE-NEXT:    [[TMP22:%.*]] = select i1 [[TMP21]], i32 [[TMP20]], i32 [[TMP15]]
-; SSE-NEXT:    [[TMP23:%.*]] = icmp sgt i32 [[TMP22]], [[TMP18]]
-; SSE-NEXT:    [[TMP24:%.*]] = select i1 [[TMP23]], i32 [[TMP22]], i32 [[TMP18]]
-; SSE-NEXT:    [[TMP25:%.*]] = icmp sgt i32 [[TMP24]], [[TMP5]]
-; SSE-NEXT:    [[OP_EXTRA:%.*]] = select i1 [[TMP25]], i32 [[TMP24]], i32 [[TMP5]]
-; SSE-NEXT:    [[TMP26:%.*]] = select i1 [[TMP19]], i32 [[TMP17]], i32 [[TMP18]]
-; SSE-NEXT:    ret i32 [[OP_EXTRA]]
+; SSE-NEXT:    [[TMP20:%.*]] = select i1 [[TMP19]], i32 [[TMP17]], i32 [[TMP18]]
+; SSE-NEXT:    [[TMP21:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 7), align 4
+; SSE-NEXT:    [[TMP22:%.*]] = icmp sgt i32 [[TMP20]], [[TMP21]]
+; SSE-NEXT:    [[TMP23:%.*]] = select i1 [[TMP22]], i32 [[TMP20]], i32 [[TMP21]]
+; SSE-NEXT:    ret i32 [[TMP23]]
 ;
 ; AVX-LABEL: @maxi8_wrong_parent(
 ; AVX-NEXT:    [[TMP2:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 0), align 16
diff --git a/test/Transforms/SLPVectorizer/X86/implicitfloat.ll b/test/Transforms/SLPVectorizer/X86/implicitfloat.ll
index f7283f0d0271918bb00b4fed18280abd57445d57..2fbb8f0477f23ae434369218ba8d00810ddc0338 100644
--- a/test/Transforms/SLPVectorizer/X86/implicitfloat.ll
+++ b/test/Transforms/SLPVectorizer/X86/implicitfloat.ll
@@ -1,13 +1,26 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -basicaa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.8.0"
 
 ; Don't vectorize when noimplicitfloat is used.
-; CHECK: test1
-; CHECK-NOT: store <2 x double>
-; CHECK: ret
 define void @test1(double* %a, double* %b, double* %c) noimplicitfloat { ; <------ noimplicitfloat attribute here!
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[I0:%.*]] = load double, double* [[A:%.*]], align 8
+; CHECK-NEXT:    [[I1:%.*]] = load double, double* [[B:%.*]], align 8
+; CHECK-NEXT:    [[MUL:%.*]] = fmul double [[I0]], [[I1]]
+; CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds double, double* [[A]], i64 1
+; CHECK-NEXT:    [[I3:%.*]] = load double, double* [[ARRAYIDX3]], align 8
+; CHECK-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds double, double* [[B]], i64 1
+; CHECK-NEXT:    [[I4:%.*]] = load double, double* [[ARRAYIDX4]], align 8
+; CHECK-NEXT:    [[MUL5:%.*]] = fmul double [[I3]], [[I4]]
+; CHECK-NEXT:    store double [[MUL]], double* [[C:%.*]], align 8
+; CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds double, double* [[C]], i64 1
+; CHECK-NEXT:    store double [[MUL5]], double* [[ARRAYIDX5]], align 8
+; CHECK-NEXT:    ret void
+;
 entry:
   %i0 = load double, double* %a, align 8
   %i1 = load double, double* %b, align 8
diff --git a/test/Transforms/SLPVectorizer/X86/insertvalue.ll b/test/Transforms/SLPVectorizer/X86/insertvalue.ll
index 1af11609fe6f71e955e3aedfd69287291398158e..2e4fa364ed82525df4147de7da7f0f81f6765dfa 100644
--- a/test/Transforms/SLPVectorizer/X86/insertvalue.ll
+++ b/test/Transforms/SLPVectorizer/X86/insertvalue.ll
@@ -216,24 +216,21 @@ top:
 define void @julia_load_array_of_i16([4 x i16]* %a, [4 x i16]* %b, [4 x i16]* %c) {
 ; CHECK-LABEL: @julia_load_array_of_i16(
 ; CHECK-NEXT:  top:
-; CHECK-NEXT:    [[A_ARR:%.*]] = load [4 x i16], [4 x i16]* [[A:%.*]], align 4
-; CHECK-NEXT:    [[A0:%.*]] = extractvalue [4 x i16] [[A_ARR]], 0
-; CHECK-NEXT:    [[A2:%.*]] = extractvalue [4 x i16] [[A_ARR]], 2
-; CHECK-NEXT:    [[A1:%.*]] = extractvalue [4 x i16] [[A_ARR]], 1
-; CHECK-NEXT:    [[B_ARR:%.*]] = load [4 x i16], [4 x i16]* [[B:%.*]], align 4
-; CHECK-NEXT:    [[B0:%.*]] = extractvalue [4 x i16] [[B_ARR]], 0
-; CHECK-NEXT:    [[B2:%.*]] = extractvalue [4 x i16] [[B_ARR]], 2
-; CHECK-NEXT:    [[B1:%.*]] = extractvalue [4 x i16] [[B_ARR]], 1
-; CHECK-NEXT:    [[A3:%.*]] = extractvalue [4 x i16] [[A_ARR]], 3
-; CHECK-NEXT:    [[C1:%.*]] = sub i16 [[A1]], [[B1]]
-; CHECK-NEXT:    [[B3:%.*]] = extractvalue [4 x i16] [[B_ARR]], 3
-; CHECK-NEXT:    [[C0:%.*]] = sub i16 [[A0]], [[B0]]
-; CHECK-NEXT:    [[C2:%.*]] = sub i16 [[A2]], [[B2]]
-; CHECK-NEXT:    [[C_ARR0:%.*]] = insertvalue [4 x i16] undef, i16 [[C0]], 0
-; CHECK-NEXT:    [[C_ARR1:%.*]] = insertvalue [4 x i16] [[C_ARR0]], i16 [[C1]], 1
-; CHECK-NEXT:    [[C3:%.*]] = sub i16 [[A3]], [[B3]]
-; CHECK-NEXT:    [[C_ARR2:%.*]] = insertvalue [4 x i16] [[C_ARR1]], i16 [[C2]], 2
-; CHECK-NEXT:    [[C_ARR3:%.*]] = insertvalue [4 x i16] [[C_ARR2]], i16 [[C3]], 3
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast [4 x i16]* [[A:%.*]] to <4 x i16>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i16>, <4 x i16>* [[TMP0]], align 4
+; CHECK-NEXT:    [[A_ARR:%.*]] = load [4 x i16], [4 x i16]* [[A]], align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast [4 x i16]* [[B:%.*]] to <4 x i16>*
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i16>, <4 x i16>* [[TMP2]], align 4
+; CHECK-NEXT:    [[B_ARR:%.*]] = load [4 x i16], [4 x i16]* [[B]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = sub <4 x i16> [[TMP1]], [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <4 x i16> [[TMP4]], i32 0
+; CHECK-NEXT:    [[C_ARR0:%.*]] = insertvalue [4 x i16] undef, i16 [[TMP5]], 0
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x i16> [[TMP4]], i32 1
+; CHECK-NEXT:    [[C_ARR1:%.*]] = insertvalue [4 x i16] [[C_ARR0]], i16 [[TMP6]], 1
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <4 x i16> [[TMP4]], i32 2
+; CHECK-NEXT:    [[C_ARR2:%.*]] = insertvalue [4 x i16] [[C_ARR1]], i16 [[TMP7]], 2
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <4 x i16> [[TMP4]], i32 3
+; CHECK-NEXT:    [[C_ARR3:%.*]] = insertvalue [4 x i16] [[C_ARR2]], i16 [[TMP8]], 3
 ; CHECK-NEXT:    store [4 x i16] [[C_ARR3]], [4 x i16]* [[C:%.*]], align 4
 ; CHECK-NEXT:    ret void
 ;
diff --git a/test/Transforms/SLPVectorizer/X86/intrinsic.ll b/test/Transforms/SLPVectorizer/X86/intrinsic.ll
index cc5a4afe43d91499b4ab3935e8c52ec3ce7ea7f1..fbce7551b00b2c872736d12cbb244b4f79578df0 100644
--- a/test/Transforms/SLPVectorizer/X86/intrinsic.ll
+++ b/test/Transforms/SLPVectorizer/X86/intrinsic.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -basicaa -slp-vectorizer -slp-threshold=-999 -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
@@ -5,13 +6,19 @@ target triple = "x86_64-apple-macosx10.8.0"
 
 declare double @llvm.fabs.f64(double) nounwind readnone
 
-;CHECK-LABEL: @vec_fabs_f64(
-;CHECK: load <2 x double>
-;CHECK: load <2 x double>
-;CHECK: call <2 x double> @llvm.fabs.v2f64
-;CHECK: store <2 x double>
-;CHECK: ret
 define void @vec_fabs_f64(double* %a, double* %b, double* %c) {
+; CHECK-LABEL: @vec_fabs_f64(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast double* [[A:%.*]] to <2 x double>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x double>, <2 x double>* [[TMP0]], align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast double* [[B:%.*]] to <2 x double>*
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x double>, <2 x double>* [[TMP2]], align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = fmul <2 x double> [[TMP1]], [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = call <2 x double> @llvm.fabs.v2f64(<2 x double> [[TMP4]])
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast double* [[C:%.*]] to <2 x double>*
+; CHECK-NEXT:    store <2 x double> [[TMP5]], <2 x double>* [[TMP6]], align 8
+; CHECK-NEXT:    ret void
+;
 entry:
   %i0 = load double, double* %a, align 8
   %i1 = load double, double* %b, align 8
@@ -31,13 +38,18 @@ entry:
 
 declare float @llvm.copysign.f32(float, float) nounwind readnone
 
-;CHECK-LABEL: @vec_copysign_f32(
-;CHECK: load <4 x float>
-;CHECK: load <4 x float>
-;CHECK: call <4 x float> @llvm.copysign.v4f32
-;CHECK: store <4 x float>
-;CHECK: ret
 define void @vec_copysign_f32(float* %a, float* %b, float* noalias %c) {
+; CHECK-LABEL: @vec_copysign_f32(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast float* [[A:%.*]] to <4 x float>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x float>, <4 x float>* [[TMP0]], align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast float* [[B:%.*]] to <4 x float>*
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x float>, <4 x float>* [[TMP2]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = call <4 x float> @llvm.copysign.v4f32(<4 x float> [[TMP1]], <4 x float> [[TMP3]])
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast float* [[C:%.*]] to <4 x float>*
+; CHECK-NEXT:    store <4 x float> [[TMP4]], <4 x float>* [[TMP5]], align 4
+; CHECK-NEXT:    ret void
+;
 entry:
   %0 = load float, float* %a, align 4
   %1 = load float, float* %b, align 4
@@ -74,6 +86,18 @@ entry:
 declare i32 @llvm.bswap.i32(i32) nounwind readnone
 
 define void @vec_bswap_i32(i32* %a, i32* %b, i32* %c) {
+; CHECK-LABEL: @vec_bswap_i32(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast i32* [[A:%.*]] to <4 x i32>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, <4 x i32>* [[TMP0]], align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i32* [[B:%.*]] to <4 x i32>*
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, <4 x i32>* [[TMP2]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = add <4 x i32> [[TMP1]], [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = call <4 x i32> @llvm.bswap.v4i32(<4 x i32> [[TMP4]])
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i32* [[C:%.*]] to <4 x i32>*
+; CHECK-NEXT:    store <4 x i32> [[TMP5]], <4 x i32>* [[TMP6]], align 4
+; CHECK-NEXT:    ret void
+;
 entry:
   %i0 = load i32, i32* %a, align 4
   %i1 = load i32, i32* %b, align 4
@@ -110,17 +134,23 @@ entry:
   store i32 %call4, i32* %arrayidx10, align 4
   ret void
 
-; CHECK-LABEL: @vec_bswap_i32(
-; CHECK: load <4 x i32>
-; CHECK: load <4 x i32>
-; CHECK: call <4 x i32> @llvm.bswap.v4i32
-; CHECK: store <4 x i32>
-; CHECK: ret
 }
 
 declare i32 @llvm.ctlz.i32(i32,i1) nounwind readnone
 
 define void @vec_ctlz_i32(i32* %a, i32* %b, i32* %c, i1) {
+; CHECK-LABEL: @vec_ctlz_i32(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i32* [[A:%.*]] to <4 x i32>*
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, <4 x i32>* [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i32* [[B:%.*]] to <4 x i32>*
+; CHECK-NEXT:    [[TMP4:%.*]] = load <4 x i32>, <4 x i32>* [[TMP3]], align 4
+; CHECK-NEXT:    [[TMP5:%.*]] = add <4 x i32> [[TMP2]], [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = call <4 x i32> @llvm.ctlz.v4i32(<4 x i32> [[TMP5]], i1 true)
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i32* [[C:%.*]] to <4 x i32>*
+; CHECK-NEXT:    store <4 x i32> [[TMP6]], <4 x i32>* [[TMP7]], align 4
+; CHECK-NEXT:    ret void
+;
 entry:
   %i0 = load i32, i32* %a, align 4
   %i1 = load i32, i32* %b, align 4
@@ -157,15 +187,42 @@ entry:
   store i32 %call4, i32* %arrayidx10, align 4
   ret void
 
-; CHECK-LABEL: @vec_ctlz_i32(
-; CHECK: load <4 x i32>
-; CHECK: load <4 x i32>
-; CHECK: call <4 x i32> @llvm.ctlz.v4i32
-; CHECK: store <4 x i32>
-; CHECK: ret
 }
 
 define void @vec_ctlz_i32_neg(i32* %a, i32* %b, i32* %c, i1) {
+; CHECK-LABEL: @vec_ctlz_i32_neg(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[I0:%.*]] = load i32, i32* [[A:%.*]], align 4
+; CHECK-NEXT:    [[I1:%.*]] = load i32, i32* [[B:%.*]], align 4
+; CHECK-NEXT:    [[ADD1:%.*]] = add i32 [[I0]], [[I1]]
+; CHECK-NEXT:    [[CALL1:%.*]] = tail call i32 @llvm.ctlz.i32(i32 [[ADD1]], i1 true) #3
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, i32* [[A]], i32 1
+; CHECK-NEXT:    [[I2:%.*]] = load i32, i32* [[ARRAYIDX2]], align 4
+; CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds i32, i32* [[B]], i32 1
+; CHECK-NEXT:    [[I3:%.*]] = load i32, i32* [[ARRAYIDX3]], align 4
+; CHECK-NEXT:    [[ADD2:%.*]] = add i32 [[I2]], [[I3]]
+; CHECK-NEXT:    [[CALL2:%.*]] = tail call i32 @llvm.ctlz.i32(i32 [[ADD2]], i1 false) #3
+; CHECK-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds i32, i32* [[A]], i32 2
+; CHECK-NEXT:    [[I4:%.*]] = load i32, i32* [[ARRAYIDX4]], align 4
+; CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds i32, i32* [[B]], i32 2
+; CHECK-NEXT:    [[I5:%.*]] = load i32, i32* [[ARRAYIDX5]], align 4
+; CHECK-NEXT:    [[ADD3:%.*]] = add i32 [[I4]], [[I5]]
+; CHECK-NEXT:    [[CALL3:%.*]] = tail call i32 @llvm.ctlz.i32(i32 [[ADD3]], i1 true) #3
+; CHECK-NEXT:    [[ARRAYIDX6:%.*]] = getelementptr inbounds i32, i32* [[A]], i32 3
+; CHECK-NEXT:    [[I6:%.*]] = load i32, i32* [[ARRAYIDX6]], align 4
+; CHECK-NEXT:    [[ARRAYIDX7:%.*]] = getelementptr inbounds i32, i32* [[B]], i32 3
+; CHECK-NEXT:    [[I7:%.*]] = load i32, i32* [[ARRAYIDX7]], align 4
+; CHECK-NEXT:    [[ADD4:%.*]] = add i32 [[I6]], [[I7]]
+; CHECK-NEXT:    [[CALL4:%.*]] = tail call i32 @llvm.ctlz.i32(i32 [[ADD4]], i1 false) #3
+; CHECK-NEXT:    store i32 [[CALL1]], i32* [[C:%.*]], align 4
+; CHECK-NEXT:    [[ARRAYIDX8:%.*]] = getelementptr inbounds i32, i32* [[C]], i32 1
+; CHECK-NEXT:    store i32 [[CALL2]], i32* [[ARRAYIDX8]], align 4
+; CHECK-NEXT:    [[ARRAYIDX9:%.*]] = getelementptr inbounds i32, i32* [[C]], i32 2
+; CHECK-NEXT:    store i32 [[CALL3]], i32* [[ARRAYIDX9]], align 4
+; CHECK-NEXT:    [[ARRAYIDX10:%.*]] = getelementptr inbounds i32, i32* [[C]], i32 3
+; CHECK-NEXT:    store i32 [[CALL4]], i32* [[ARRAYIDX10]], align 4
+; CHECK-NEXT:    ret void
+;
 entry:
   %i0 = load i32, i32* %a, align 4
   %i1 = load i32, i32* %b, align 4
@@ -202,8 +259,6 @@ entry:
   store i32 %call4, i32* %arrayidx10, align 4
   ret void
 
-; CHECK-LABEL: @vec_ctlz_i32_neg(
-; CHECK-NOT: call <4 x i32> @llvm.ctlz.v4i32
 
 }
 
@@ -211,6 +266,18 @@ entry:
 declare i32 @llvm.cttz.i32(i32,i1) nounwind readnone
 
 define void @vec_cttz_i32(i32* %a, i32* %b, i32* %c, i1) {
+; CHECK-LABEL: @vec_cttz_i32(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i32* [[A:%.*]] to <4 x i32>*
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, <4 x i32>* [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i32* [[B:%.*]] to <4 x i32>*
+; CHECK-NEXT:    [[TMP4:%.*]] = load <4 x i32>, <4 x i32>* [[TMP3]], align 4
+; CHECK-NEXT:    [[TMP5:%.*]] = add <4 x i32> [[TMP2]], [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = call <4 x i32> @llvm.cttz.v4i32(<4 x i32> [[TMP5]], i1 true)
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i32* [[C:%.*]] to <4 x i32>*
+; CHECK-NEXT:    store <4 x i32> [[TMP6]], <4 x i32>* [[TMP7]], align 4
+; CHECK-NEXT:    ret void
+;
 entry:
   %i0 = load i32, i32* %a, align 4
   %i1 = load i32, i32* %b, align 4
@@ -247,15 +314,42 @@ entry:
   store i32 %call4, i32* %arrayidx10, align 4
   ret void
 
-; CHECK-LABEL: @vec_cttz_i32(
-; CHECK: load <4 x i32>
-; CHECK: load <4 x i32>
-; CHECK: call <4 x i32> @llvm.cttz.v4i32
-; CHECK: store <4 x i32>
-; CHECK: ret
 }
 
 define void @vec_cttz_i32_neg(i32* %a, i32* %b, i32* %c, i1) {
+; CHECK-LABEL: @vec_cttz_i32_neg(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[I0:%.*]] = load i32, i32* [[A:%.*]], align 4
+; CHECK-NEXT:    [[I1:%.*]] = load i32, i32* [[B:%.*]], align 4
+; CHECK-NEXT:    [[ADD1:%.*]] = add i32 [[I0]], [[I1]]
+; CHECK-NEXT:    [[CALL1:%.*]] = tail call i32 @llvm.cttz.i32(i32 [[ADD1]], i1 true) #3
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, i32* [[A]], i32 1
+; CHECK-NEXT:    [[I2:%.*]] = load i32, i32* [[ARRAYIDX2]], align 4
+; CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds i32, i32* [[B]], i32 1
+; CHECK-NEXT:    [[I3:%.*]] = load i32, i32* [[ARRAYIDX3]], align 4
+; CHECK-NEXT:    [[ADD2:%.*]] = add i32 [[I2]], [[I3]]
+; CHECK-NEXT:    [[CALL2:%.*]] = tail call i32 @llvm.cttz.i32(i32 [[ADD2]], i1 false) #3
+; CHECK-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds i32, i32* [[A]], i32 2
+; CHECK-NEXT:    [[I4:%.*]] = load i32, i32* [[ARRAYIDX4]], align 4
+; CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds i32, i32* [[B]], i32 2
+; CHECK-NEXT:    [[I5:%.*]] = load i32, i32* [[ARRAYIDX5]], align 4
+; CHECK-NEXT:    [[ADD3:%.*]] = add i32 [[I4]], [[I5]]
+; CHECK-NEXT:    [[CALL3:%.*]] = tail call i32 @llvm.cttz.i32(i32 [[ADD3]], i1 true) #3
+; CHECK-NEXT:    [[ARRAYIDX6:%.*]] = getelementptr inbounds i32, i32* [[A]], i32 3
+; CHECK-NEXT:    [[I6:%.*]] = load i32, i32* [[ARRAYIDX6]], align 4
+; CHECK-NEXT:    [[ARRAYIDX7:%.*]] = getelementptr inbounds i32, i32* [[B]], i32 3
+; CHECK-NEXT:    [[I7:%.*]] = load i32, i32* [[ARRAYIDX7]], align 4
+; CHECK-NEXT:    [[ADD4:%.*]] = add i32 [[I6]], [[I7]]
+; CHECK-NEXT:    [[CALL4:%.*]] = tail call i32 @llvm.cttz.i32(i32 [[ADD4]], i1 false) #3
+; CHECK-NEXT:    store i32 [[CALL1]], i32* [[C:%.*]], align 4
+; CHECK-NEXT:    [[ARRAYIDX8:%.*]] = getelementptr inbounds i32, i32* [[C]], i32 1
+; CHECK-NEXT:    store i32 [[CALL2]], i32* [[ARRAYIDX8]], align 4
+; CHECK-NEXT:    [[ARRAYIDX9:%.*]] = getelementptr inbounds i32, i32* [[C]], i32 2
+; CHECK-NEXT:    store i32 [[CALL3]], i32* [[ARRAYIDX9]], align 4
+; CHECK-NEXT:    [[ARRAYIDX10:%.*]] = getelementptr inbounds i32, i32* [[C]], i32 3
+; CHECK-NEXT:    store i32 [[CALL4]], i32* [[ARRAYIDX10]], align 4
+; CHECK-NEXT:    ret void
+;
 entry:
   %i0 = load i32, i32* %a, align 4
   %i1 = load i32, i32* %b, align 4
@@ -292,13 +386,23 @@ entry:
   store i32 %call4, i32* %arrayidx10, align 4
   ret void
 
-; CHECK-LABEL: @vec_cttz_i32_neg(
-; CHECK-NOT: call <4 x i32> @llvm.cttz.v4i32
 }
 
 
 declare float @llvm.powi.f32(float, i32)
 define void @vec_powi_f32(float* %a, float* %b, float* %c, i32 %P) {
+; CHECK-LABEL: @vec_powi_f32(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast float* [[A:%.*]] to <4 x float>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x float>, <4 x float>* [[TMP0]], align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast float* [[B:%.*]] to <4 x float>*
+; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x float>, <4 x float>* [[TMP2]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = fadd <4 x float> [[TMP1]], [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = call <4 x float> @llvm.powi.v4f32(<4 x float> [[TMP4]], i32 [[P:%.*]])
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast float* [[C:%.*]] to <4 x float>*
+; CHECK-NEXT:    store <4 x float> [[TMP5]], <4 x float>* [[TMP6]], align 4
+; CHECK-NEXT:    ret void
+;
 entry:
   %i0 = load float, float* %a, align 4
   %i1 = load float, float* %b, align 4
@@ -335,16 +439,43 @@ entry:
   store float %call4, float* %arrayidx10, align 4
   ret void
 
-; CHECK-LABEL: @vec_powi_f32(
-; CHECK: load <4 x float>
-; CHECK: load <4 x float>
-; CHECK: call <4 x float> @llvm.powi.v4f32
-; CHECK: store <4 x float>
-; CHECK: ret
 }
 
 
 define void @vec_powi_f32_neg(float* %a, float* %b, float* %c, i32 %P, i32 %Q) {
+; CHECK-LABEL: @vec_powi_f32_neg(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[I0:%.*]] = load float, float* [[A:%.*]], align 4
+; CHECK-NEXT:    [[I1:%.*]] = load float, float* [[B:%.*]], align 4
+; CHECK-NEXT:    [[ADD1:%.*]] = fadd float [[I0]], [[I1]]
+; CHECK-NEXT:    [[CALL1:%.*]] = tail call float @llvm.powi.f32(float [[ADD1]], i32 [[P:%.*]]) #3
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds float, float* [[A]], i32 1
+; CHECK-NEXT:    [[I2:%.*]] = load float, float* [[ARRAYIDX2]], align 4
+; CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds float, float* [[B]], i32 1
+; CHECK-NEXT:    [[I3:%.*]] = load float, float* [[ARRAYIDX3]], align 4
+; CHECK-NEXT:    [[ADD2:%.*]] = fadd float [[I2]], [[I3]]
+; CHECK-NEXT:    [[CALL2:%.*]] = tail call float @llvm.powi.f32(float [[ADD2]], i32 [[Q:%.*]]) #3
+; CHECK-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds float, float* [[A]], i32 2
+; CHECK-NEXT:    [[I4:%.*]] = load float, float* [[ARRAYIDX4]], align 4
+; CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds float, float* [[B]], i32 2
+; CHECK-NEXT:    [[I5:%.*]] = load float, float* [[ARRAYIDX5]], align 4
+; CHECK-NEXT:    [[ADD3:%.*]] = fadd float [[I4]], [[I5]]
+; CHECK-NEXT:    [[CALL3:%.*]] = tail call float @llvm.powi.f32(float [[ADD3]], i32 [[P]]) #3
+; CHECK-NEXT:    [[ARRAYIDX6:%.*]] = getelementptr inbounds float, float* [[A]], i32 3
+; CHECK-NEXT:    [[I6:%.*]] = load float, float* [[ARRAYIDX6]], align 4
+; CHECK-NEXT:    [[ARRAYIDX7:%.*]] = getelementptr inbounds float, float* [[B]], i32 3
+; CHECK-NEXT:    [[I7:%.*]] = load float, float* [[ARRAYIDX7]], align 4
+; CHECK-NEXT:    [[ADD4:%.*]] = fadd float [[I6]], [[I7]]
+; CHECK-NEXT:    [[CALL4:%.*]] = tail call float @llvm.powi.f32(float [[ADD4]], i32 [[Q]]) #3
+; CHECK-NEXT:    store float [[CALL1]], float* [[C:%.*]], align 4
+; CHECK-NEXT:    [[ARRAYIDX8:%.*]] = getelementptr inbounds float, float* [[C]], i32 1
+; CHECK-NEXT:    store float [[CALL2]], float* [[ARRAYIDX8]], align 4
+; CHECK-NEXT:    [[ARRAYIDX9:%.*]] = getelementptr inbounds float, float* [[C]], i32 2
+; CHECK-NEXT:    store float [[CALL3]], float* [[ARRAYIDX9]], align 4
+; CHECK-NEXT:    [[ARRAYIDX10:%.*]] = getelementptr inbounds float, float* [[C]], i32 3
+; CHECK-NEXT:    store float [[CALL4]], float* [[ARRAYIDX10]], align 4
+; CHECK-NEXT:    ret void
+;
 entry:
   %i0 = load float, float* %a, align 4
   %i1 = load float, float* %b, align 4
@@ -381,6 +512,4 @@ entry:
   store float %call4, float* %arrayidx10, align 4
   ret void
 
-; CHECK-LABEL: @vec_powi_f32_neg(
-; CHECK-NOT: call <4 x float> @llvm.powi.v4f32
 }
diff --git a/test/Transforms/SLPVectorizer/X86/long_chains.ll b/test/Transforms/SLPVectorizer/X86/long_chains.ll
index f87dabf4c9feb1e469e1d31a182412da5f767887..99b340addb925182cfcf53ef964ef3a1f3aece8b 100644
--- a/test/Transforms/SLPVectorizer/X86/long_chains.ll
+++ b/test/Transforms/SLPVectorizer/X86/long_chains.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -basicaa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
@@ -5,13 +6,31 @@ target triple = "x86_64-apple-macosx10.8.0"
 
 ; At this point we can't vectorize only parts of the tree.
 
-; CHECK: test
-; CHECK: insertelement <2 x i8>
-; CHECK: insertelement <2 x i8>
-; CHECK: sitofp <2 x i8>
-; CHECK: fmul <2 x double>
-; CHECK: ret
 define i32 @test(double* nocapture %A, i8* nocapture %B) {
+; CHECK-LABEL: @test(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast i8* [[B:%.*]] to <2 x i8>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i8>, <2 x i8>* [[TMP0]], align 1
+; CHECK-NEXT:    [[TMP2:%.*]] = add <2 x i8> <i8 3, i8 3>, [[TMP1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <2 x i8> [[TMP2]], i32 0
+; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <2 x i8> undef, i8 [[TMP3]], i32 0
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <2 x i8> [[TMP2]], i32 1
+; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <2 x i8> [[TMP4]], i8 [[TMP5]], i32 1
+; CHECK-NEXT:    [[TMP7:%.*]] = sitofp <2 x i8> [[TMP6]] to <2 x double>
+; CHECK-NEXT:    [[TMP8:%.*]] = fmul <2 x double> [[TMP7]], [[TMP7]]
+; CHECK-NEXT:    [[TMP9:%.*]] = fadd <2 x double> <double 1.000000e+00, double 1.000000e+00>, [[TMP8]]
+; CHECK-NEXT:    [[TMP10:%.*]] = fmul <2 x double> [[TMP9]], [[TMP9]]
+; CHECK-NEXT:    [[TMP11:%.*]] = fadd <2 x double> <double 1.000000e+00, double 1.000000e+00>, [[TMP10]]
+; CHECK-NEXT:    [[TMP12:%.*]] = fmul <2 x double> [[TMP11]], [[TMP11]]
+; CHECK-NEXT:    [[TMP13:%.*]] = fadd <2 x double> <double 1.000000e+00, double 1.000000e+00>, [[TMP12]]
+; CHECK-NEXT:    [[TMP14:%.*]] = fmul <2 x double> [[TMP13]], [[TMP13]]
+; CHECK-NEXT:    [[TMP15:%.*]] = fadd <2 x double> <double 1.000000e+00, double 1.000000e+00>, [[TMP14]]
+; CHECK-NEXT:    [[TMP16:%.*]] = fmul <2 x double> [[TMP15]], [[TMP15]]
+; CHECK-NEXT:    [[TMP17:%.*]] = fadd <2 x double> <double 1.000000e+00, double 1.000000e+00>, [[TMP16]]
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast double* [[A:%.*]] to <2 x double>*
+; CHECK-NEXT:    store <2 x double> [[TMP17]], <2 x double>* [[TMP18]], align 8
+; CHECK-NEXT:    ret i32 undef
+;
 entry:
   %0 = load i8, i8* %B, align 1
   %arrayidx1 = getelementptr inbounds i8, i8* %B, i64 1
@@ -19,7 +38,7 @@ entry:
   %add = add i8 %0, 3
   %add4 = add i8 %1, 3
   %conv6 = sitofp i8 %add to double
-  %conv7 = sitofp i8 %add4 to double 
+  %conv7 = sitofp i8 %add4 to double
   %mul = fmul double %conv6, %conv6
   %add8 = fadd double %mul, 1.000000e+00
   %mul9 = fmul double %conv7, %conv7
diff --git a/test/Transforms/SLPVectorizer/X86/loopinvariant.ll b/test/Transforms/SLPVectorizer/X86/loopinvariant.ll
index dace4b35b8711d6248aec8934b5c8fba50a14503..1b19aeae03776ec8385f71f43ba6ffb6b1ec6caa 100644
--- a/test/Transforms/SLPVectorizer/X86/loopinvariant.ll
+++ b/test/Transforms/SLPVectorizer/X86/loopinvariant.ll
@@ -1,14 +1,51 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -basicaa -slp-vectorizer -S -mcpu=corei7-avx | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.8.0"
 
-;CHECK-LABEL: @foo(
-;CHECK: load <8 x i32>
-;CHECK: add nsw <8 x i32>
-;CHECK: store <8 x i32>
-;CHECK: ret
 define i32 @foo(i32* nocapture %A, i32 %n) {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP62:%.*]] = icmp sgt i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[CMP62]], label [[FOR_BODY:%.*]], label [[FOR_END:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[A:%.*]], i64 [[INDVARS_IV]]
+; CHECK-NEXT:    [[TMP0:%.*]] = or i64 [[INDVARS_IV]], 1
+; CHECK-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP1:%.*]] = or i64 [[INDVARS_IV]], 2
+; CHECK-NEXT:    [[ARRAYIDX8:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[TMP1]]
+; CHECK-NEXT:    [[TMP2:%.*]] = or i64 [[INDVARS_IV]], 3
+; CHECK-NEXT:    [[ARRAYIDX12:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[TMP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = or i64 [[INDVARS_IV]], 4
+; CHECK-NEXT:    [[ARRAYIDX16:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[TMP3]]
+; CHECK-NEXT:    [[TMP4:%.*]] = or i64 [[INDVARS_IV]], 5
+; CHECK-NEXT:    [[ARRAYIDX20:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[TMP4]]
+; CHECK-NEXT:    [[TMP5:%.*]] = or i64 [[INDVARS_IV]], 6
+; CHECK-NEXT:    [[ARRAYIDX24:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[TMP5]]
+; CHECK-NEXT:    [[TMP6:%.*]] = or i64 [[INDVARS_IV]], 7
+; CHECK-NEXT:    [[ARRAYIDX28:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 [[TMP6]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i32* [[ARRAYIDX]] to <8 x i32>*
+; CHECK-NEXT:    [[TMP8:%.*]] = load <8 x i32>, <8 x i32>* [[TMP7]], align 4
+; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <8 x i32> undef, i32 [[N]], i32 0
+; CHECK-NEXT:    [[TMP10:%.*]] = insertelement <8 x i32> [[TMP9]], i32 [[N]], i32 1
+; CHECK-NEXT:    [[TMP11:%.*]] = insertelement <8 x i32> [[TMP10]], i32 [[N]], i32 2
+; CHECK-NEXT:    [[TMP12:%.*]] = insertelement <8 x i32> [[TMP11]], i32 [[N]], i32 3
+; CHECK-NEXT:    [[TMP13:%.*]] = insertelement <8 x i32> [[TMP12]], i32 [[N]], i32 4
+; CHECK-NEXT:    [[TMP14:%.*]] = insertelement <8 x i32> [[TMP13]], i32 [[N]], i32 5
+; CHECK-NEXT:    [[TMP15:%.*]] = insertelement <8 x i32> [[TMP14]], i32 [[N]], i32 6
+; CHECK-NEXT:    [[TMP16:%.*]] = insertelement <8 x i32> [[TMP15]], i32 [[N]], i32 7
+; CHECK-NEXT:    [[TMP17:%.*]] = add nsw <8 x i32> [[TMP16]], [[TMP8]]
+; CHECK-NEXT:    [[TMP18:%.*]] = bitcast i32* [[ARRAYIDX]] to <8 x i32>*
+; CHECK-NEXT:    store <8 x i32> [[TMP17]], <8 x i32>* [[TMP18]], align 4
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add i64 [[INDVARS_IV]], 8
+; CHECK-NEXT:    [[TMP19:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP19]], [[N]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_END]]
+; CHECK:       for.end:
+; CHECK-NEXT:    ret i32 undef
+;
 entry:
   %cmp62 = icmp sgt i32 %n, 0
   br i1 %cmp62, label %for.body, label %for.end
diff --git a/test/Transforms/SLPVectorizer/X86/metadata.ll b/test/Transforms/SLPVectorizer/X86/metadata.ll
index ebef6b53c6a4c70e2e108bf5429061058fcc9988..fdfd032f3ce421a587f3124b73f6d3f5a9fc2712 100644
--- a/test/Transforms/SLPVectorizer/X86/metadata.ll
+++ b/test/Transforms/SLPVectorizer/X86/metadata.ll
@@ -1,16 +1,21 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -basicaa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.8.0"
 
-;CHECK-LABEL: test1
-;CHECK: load <2 x double>{{.*}}!tbaa ![[TBAA:[0-9]+]]
-;CHECK: load <2 x double>{{.*}}!tbaa ![[TBAA]]
-;CHECK: fmul <2 x double>{{.*}}!fpmath ![[FP1:[0-9]+]]
-;CHECK: store <2 x double>{{.*}}!tbaa ![[TBAA]]
-;CHECK: ret void
-
 define void @test1(double* %a, double* %b, double* %c) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast double* [[A:%.*]] to <2 x double>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x double>, <2 x double>* [[TMP0]], align 8, !tbaa !0
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast double* [[B:%.*]] to <2 x double>*
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x double>, <2 x double>* [[TMP2]], align 8, !tbaa !0
+; CHECK-NEXT:    [[TMP4:%.*]] = fmul <2 x double> [[TMP1]], [[TMP3]], !fpmath !4
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast double* [[C:%.*]] to <2 x double>*
+; CHECK-NEXT:    store <2 x double> [[TMP4]], <2 x double>* [[TMP5]], align 8, !tbaa !0
+; CHECK-NEXT:    ret void
+;
 entry:
   %i0 = load double, double* %a, align 8, !tbaa !4
   %i1 = load double, double* %b, align 8, !tbaa !4
@@ -26,14 +31,19 @@ entry:
   ret void
 }
 
-;CHECK-LABEL: test2
-;CHECK: load <2 x double>{{.*}}!tbaa ![[TBAA]]
-;CHECK: load <2 x double>{{.*}}!tbaa ![[TBAA]]
-;CHECK: fmul <2 x double>{{.*}}!fpmath ![[FP2:[0-9]+]]
-;CHECK: store <2 x double>{{.*}}!tbaa ![[TBAA]]
-;CHECK: ret void
-
 define void @test2(double* %a, double* %b, i8* %e) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast double* [[A:%.*]] to <2 x double>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x double>, <2 x double>* [[TMP0]], align 8, !tbaa !0
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast double* [[B:%.*]] to <2 x double>*
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x double>, <2 x double>* [[TMP2]], align 8, !tbaa !0
+; CHECK-NEXT:    [[TMP4:%.*]] = fmul <2 x double> [[TMP1]], [[TMP3]], !fpmath !5
+; CHECK-NEXT:    [[C:%.*]] = bitcast i8* [[E:%.*]] to double*
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast double* [[C]] to <2 x double>*
+; CHECK-NEXT:    store <2 x double> [[TMP4]], <2 x double>* [[TMP5]], align 8, !tbaa !0
+; CHECK-NEXT:    ret void
+;
 entry:
   %i0 = load double, double* %a, align 8, !tbaa !4
   %i1 = load double, double* %b, align 8, !tbaa !4
@@ -51,9 +61,9 @@ entry:
   ret void
 }
 
-;CHECK-DAG: ![[TBAA]] = !{[[TYPEC:!.*]], [[TYPEC]], i64 0}
-;CHECK-DAG: ![[FP1]] = !{float 5.000000e+00}
-;CHECK-DAG: ![[FP2]] = !{float 2.500000e+00}
+;CHECK-DAG: !0 = !{[[TYPEC:!.*]], [[TYPEC]], i64 0}
+;CHECK-DAG: !4 = !{float 5.000000e+00}
+;CHECK-DAG: !5 = !{float 2.500000e+00}
 !0 = !{ float 5.0 }
 !1 = !{ float 2.5 }
 !2 = !{!"Simple C/C++ TBAA"}
diff --git a/test/Transforms/SLPVectorizer/X86/multi_block.ll b/test/Transforms/SLPVectorizer/X86/multi_block.ll
index b381d06688bdeae48b5bfd4a43907c62ee1a05d8..d0216103d42ae46dd648a43821aa58121762a41d 100644
--- a/test/Transforms/SLPVectorizer/X86/multi_block.ll
+++ b/test/Transforms/SLPVectorizer/X86/multi_block.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -basicaa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
@@ -16,15 +17,23 @@ target triple = "x86_64-apple-macosx10.7.0"
 ; }
 
 
-;CHECK-LABEL: @bar(
-;CHECK: load <2 x double>
-;CHECK: fptrunc <2 x double>
-;CHECK: call i32
-;CHECK: fadd <2 x float>
-;CHECK: fpext <2 x float>
-;CHECK: store <2 x double>
-;CHECK: ret
 define i32 @bar(double* nocapture %A, i32 %d) {
+; CHECK-LABEL: @bar(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast double* [[A:%.*]] to <2 x double>*
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x double>, <2 x double>* [[TMP1]], align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = fptrunc <2 x double> [[TMP2]] to <2 x float>
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq i32 [[D:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP4]], label [[TMP7:%.*]], label [[TMP5:%.*]]
+; CHECK:         [[TMP6:%.*]] = tail call i32 (...) @foo()
+; CHECK-NEXT:    br label [[TMP7]]
+; CHECK:         [[TMP8:%.*]] = fadd <2 x float> <float 4.000000e+00, float 5.000000e+00>, [[TMP3]]
+; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds double, double* [[A]], i64 8
+; CHECK-NEXT:    [[TMP10:%.*]] = fpext <2 x float> [[TMP8]] to <2 x double>
+; CHECK-NEXT:    [[TMP11:%.*]] = fadd <2 x double> <double 9.000000e+00, double 5.000000e+00>, [[TMP10]]
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast double* [[TMP9]] to <2 x double>*
+; CHECK-NEXT:    store <2 x double> [[TMP11]], <2 x double>* [[TMP12]], align 8
+; CHECK-NEXT:    ret i32 undef
+;
   %1 = load double, double* %A, align 8
   %2 = getelementptr inbounds double, double* %A, i64 1
   %3 = load double, double* %2, align 8
diff --git a/test/Transforms/SLPVectorizer/X86/multi_user.ll b/test/Transforms/SLPVectorizer/X86/multi_user.ll
index 3197f6db266f24dd74996c5de3fb03ff6fb1dfe4..ce8594ea84d77c9ff8134526424860d45b015b77 100644
--- a/test/Transforms/SLPVectorizer/X86/multi_user.ll
+++ b/test/Transforms/SLPVectorizer/X86/multi_user.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -basicaa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
@@ -11,13 +12,26 @@ target triple = "x86_64-apple-macosx10.7.0"
 ;  A[4] += n * 5 + 11;
 ;}
 
-;CHECK-LABEL: @foo(
-;CHECK: insertelement <4 x i32>
-;CHECK: load <4 x i32>
-;CHECK: add nsw <4 x i32>
-;CHECK: store <4 x i32>
-;CHECK: ret
 define i32 @foo(i32* nocapture %A, i32 %n) {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT:    [[TMP1:%.*]] = mul nsw i32 [[N:%.*]], 5
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <4 x i32> undef, i32 [[TMP1]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <4 x i32> [[TMP2]], i32 [[TMP1]], i32 1
+; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <4 x i32> [[TMP3]], i32 [[TMP1]], i32 2
+; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <4 x i32> [[TMP4]], i32 [[TMP1]], i32 3
+; CHECK-NEXT:    [[TMP6:%.*]] = add nsw <4 x i32> <i32 7, i32 8, i32 9, i32 10>, [[TMP5]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i32* [[A:%.*]] to <4 x i32>*
+; CHECK-NEXT:    [[TMP8:%.*]] = load <4 x i32>, <4 x i32>* [[TMP7]], align 4
+; CHECK-NEXT:    [[TMP9:%.*]] = add nsw <4 x i32> [[TMP6]], [[TMP8]]
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i32* [[A]] to <4 x i32>*
+; CHECK-NEXT:    store <4 x i32> [[TMP9]], <4 x i32>* [[TMP10]], align 4
+; CHECK-NEXT:    [[TMP11:%.*]] = add nsw i32 [[TMP1]], 11
+; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i32, i32* [[A]], i64 4
+; CHECK-NEXT:    [[TMP13:%.*]] = load i32, i32* [[TMP12]], align 4
+; CHECK-NEXT:    [[TMP14:%.*]] = add nsw i32 [[TMP11]], [[TMP13]]
+; CHECK-NEXT:    store i32 [[TMP14]], i32* [[TMP12]], align 4
+; CHECK-NEXT:    ret i32 undef
+;
   %1 = mul nsw i32 %n, 5
   %2 = add nsw i32 %1, 7
   %3 = load i32, i32* %A, align 4
diff --git a/test/Transforms/SLPVectorizer/X86/operandorder.ll b/test/Transforms/SLPVectorizer/X86/operandorder.ll
index dab7f296e20a6b7650e14c183932d56d80756b1d..2354ebd29879fd225374c1827cdf689f06ade9b3 100644
--- a/test/Transforms/SLPVectorizer/X86/operandorder.ll
+++ b/test/Transforms/SLPVectorizer/X86/operandorder.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -basicaa -slp-vectorizer -slp-threshold=-100 -instcombine -dce -S -mtriple=i386-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
 
 target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128-n8:16:32-S128"
@@ -7,12 +8,18 @@ target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f3
 ; Make sure we order the operands of commutative operations so that we get
 ; bigger vectorizable trees.
 
-; CHECK-LABEL: shuffle_operands1
-; CHECK:         load <2 x double>
-; CHECK:         fadd <2 x double>
-
 define void @shuffle_operands1(double * noalias %from, double * noalias %to,
-                               double %v1, double %v2) {
+; CHECK-LABEL: @shuffle_operands1(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast double* [[FROM:%.*]] to <2 x double>*
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x double>, <2 x double>* [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <2 x double> undef, double [[V1:%.*]], i32 0
+; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <2 x double> [[TMP3]], double [[V2:%.*]], i32 1
+; CHECK-NEXT:    [[TMP5:%.*]] = fadd <2 x double> [[TMP4]], [[TMP2]]
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast double* [[TO:%.*]] to <2 x double>*
+; CHECK-NEXT:    store <2 x double> [[TMP5]], <2 x double>* [[TMP6]], align 4
+; CHECK-NEXT:    ret void
+;
+  double %v1, double %v2) {
   %from_1 = getelementptr double, double *%from, i64 1
   %v0_1 = load double , double * %from
   %v0_2 = load double , double * %from_1
@@ -24,12 +31,28 @@ define void @shuffle_operands1(double * noalias %from, double * noalias %to,
   ret void
 }
 
-; CHECK-LABEL: shuffle_preserve_broadcast
-; CHECK: %[[BCAST:[a-z0-9]+]] = insertelement <2 x double> undef, double %v0_1
-; CHECK:                      = shufflevector <2 x double> %[[BCAST]], <2 x double> undef, <2 x i32> zeroinitializer
 define void @shuffle_preserve_broadcast(double * noalias %from,
-                                        double * noalias %to,
-                                        double %v1, double %v2) {
+; CHECK-LABEL: @shuffle_preserve_broadcast(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LP:%.*]]
+; CHECK:       lp:
+; CHECK-NEXT:    [[P:%.*]] = phi double [ 1.000000e+00, [[LP]] ], [ 0.000000e+00, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[FROM_1:%.*]] = getelementptr double, double* [[FROM:%.*]], i32 1
+; CHECK-NEXT:    [[V0_1:%.*]] = load double, double* [[FROM]], align 4
+; CHECK-NEXT:    [[V0_2:%.*]] = load double, double* [[FROM_1]], align 4
+; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <2 x double> undef, double [[V0_1]], i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x double> [[TMP0]], <2 x double> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> undef, double [[P]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <2 x double> [[TMP2]], double [[V0_2]], i32 1
+; CHECK-NEXT:    [[TMP4:%.*]] = fadd <2 x double> [[TMP1]], [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast double* [[TO:%.*]] to <2 x double>*
+; CHECK-NEXT:    store <2 x double> [[TMP4]], <2 x double>* [[TMP5]], align 4
+; CHECK-NEXT:    br i1 undef, label [[LP]], label [[EXT:%.*]]
+; CHECK:       ext:
+; CHECK-NEXT:    ret void
+;
+  double * noalias %to,
+  double %v1, double %v2) {
 entry:
 br label %lp
 
@@ -49,12 +72,28 @@ ext:
   ret void
 }
 
-; CHECK-LABEL: shuffle_preserve_broadcast2
-; CHECK: %[[BCAST:[a-z0-9]+]] = insertelement <2 x double> undef, double %v0_1
-; CHECK:                      = shufflevector <2 x double> %[[BCAST]], <2 x double> undef, <2 x i32> zeroinitializer
 define void @shuffle_preserve_broadcast2(double * noalias %from,
-                                        double * noalias %to,
-                                        double %v1, double %v2) {
+; CHECK-LABEL: @shuffle_preserve_broadcast2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LP:%.*]]
+; CHECK:       lp:
+; CHECK-NEXT:    [[P:%.*]] = phi double [ 1.000000e+00, [[LP]] ], [ 0.000000e+00, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[FROM_1:%.*]] = getelementptr double, double* [[FROM:%.*]], i32 1
+; CHECK-NEXT:    [[V0_1:%.*]] = load double, double* [[FROM]], align 4
+; CHECK-NEXT:    [[V0_2:%.*]] = load double, double* [[FROM_1]], align 4
+; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <2 x double> undef, double [[P]], i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x double> [[TMP0]], double [[V0_2]], i32 1
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> undef, double [[V0_1]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <2 x double> [[TMP2]], <2 x double> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = fadd <2 x double> [[TMP1]], [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast double* [[TO:%.*]] to <2 x double>*
+; CHECK-NEXT:    store <2 x double> [[TMP4]], <2 x double>* [[TMP5]], align 4
+; CHECK-NEXT:    br i1 undef, label [[LP]], label [[EXT:%.*]]
+; CHECK:       ext:
+; CHECK-NEXT:    ret void
+;
+  double * noalias %to,
+  double %v1, double %v2) {
 entry:
 br label %lp
 
@@ -74,12 +113,28 @@ ext:
   ret void
 }
 
-; CHECK-LABEL: shuffle_preserve_broadcast3
-; CHECK: %[[BCAST:[a-z0-9]+]] = insertelement <2 x double> undef, double %v0_1
-; CHECK:                      = shufflevector <2 x double> %[[BCAST]], <2 x double> undef, <2 x i32> zeroinitializer
 define void @shuffle_preserve_broadcast3(double * noalias %from,
-                                        double * noalias %to,
-                                        double %v1, double %v2) {
+; CHECK-LABEL: @shuffle_preserve_broadcast3(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LP:%.*]]
+; CHECK:       lp:
+; CHECK-NEXT:    [[P:%.*]] = phi double [ 1.000000e+00, [[LP]] ], [ 0.000000e+00, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[FROM_1:%.*]] = getelementptr double, double* [[FROM:%.*]], i32 1
+; CHECK-NEXT:    [[V0_1:%.*]] = load double, double* [[FROM]], align 4
+; CHECK-NEXT:    [[V0_2:%.*]] = load double, double* [[FROM_1]], align 4
+; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <2 x double> undef, double [[P]], i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x double> [[TMP0]], double [[V0_2]], i32 1
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> undef, double [[V0_1]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <2 x double> [[TMP2]], <2 x double> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = fadd <2 x double> [[TMP1]], [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast double* [[TO:%.*]] to <2 x double>*
+; CHECK-NEXT:    store <2 x double> [[TMP4]], <2 x double>* [[TMP5]], align 4
+; CHECK-NEXT:    br i1 undef, label [[LP]], label [[EXT:%.*]]
+; CHECK:       ext:
+; CHECK-NEXT:    ret void
+;
+  double * noalias %to,
+  double %v1, double %v2) {
 entry:
 br label %lp
 
@@ -100,12 +155,28 @@ ext:
 }
 
 
-; CHECK-LABEL: shuffle_preserve_broadcast4
-; CHECK: %[[BCAST:[a-z0-9]+]] = insertelement <2 x double> undef, double %v0_1
-; CHECK:                      = shufflevector <2 x double> %[[BCAST]], <2 x double> undef, <2 x i32> zeroinitializer
 define void @shuffle_preserve_broadcast4(double * noalias %from,
-                                        double * noalias %to,
-                                        double %v1, double %v2) {
+; CHECK-LABEL: @shuffle_preserve_broadcast4(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LP:%.*]]
+; CHECK:       lp:
+; CHECK-NEXT:    [[P:%.*]] = phi double [ 1.000000e+00, [[LP]] ], [ 0.000000e+00, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[FROM_1:%.*]] = getelementptr double, double* [[FROM:%.*]], i32 1
+; CHECK-NEXT:    [[V0_1:%.*]] = load double, double* [[FROM]], align 4
+; CHECK-NEXT:    [[V0_2:%.*]] = load double, double* [[FROM_1]], align 4
+; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <2 x double> undef, double [[V0_2]], i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x double> [[TMP0]], double [[P]], i32 1
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> undef, double [[V0_1]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <2 x double> [[TMP2]], <2 x double> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = fadd <2 x double> [[TMP1]], [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast double* [[TO:%.*]] to <2 x double>*
+; CHECK-NEXT:    store <2 x double> [[TMP4]], <2 x double>* [[TMP5]], align 4
+; CHECK-NEXT:    br i1 undef, label [[LP]], label [[EXT:%.*]]
+; CHECK:       ext:
+; CHECK-NEXT:    ret void
+;
+  double * noalias %to,
+  double %v1, double %v2) {
 entry:
 br label %lp
 
@@ -125,12 +196,28 @@ ext:
   ret void
 }
 
-; CHECK-LABEL: shuffle_preserve_broadcast5
-; CHECK: %[[BCAST:[a-z0-9]+]] = insertelement <2 x double> undef, double %v0_1
-; CHECK:                      = shufflevector <2 x double> %[[BCAST]], <2 x double> undef, <2 x i32> zeroinitializer
 define void @shuffle_preserve_broadcast5(double * noalias %from,
-                                        double * noalias %to,
-                                        double %v1, double %v2) {
+; CHECK-LABEL: @shuffle_preserve_broadcast5(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LP:%.*]]
+; CHECK:       lp:
+; CHECK-NEXT:    [[P:%.*]] = phi double [ 1.000000e+00, [[LP]] ], [ 0.000000e+00, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[FROM_1:%.*]] = getelementptr double, double* [[FROM:%.*]], i32 1
+; CHECK-NEXT:    [[V0_1:%.*]] = load double, double* [[FROM]], align 4
+; CHECK-NEXT:    [[V0_2:%.*]] = load double, double* [[FROM_1]], align 4
+; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <2 x double> undef, double [[V0_1]], i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x double> [[TMP0]], <2 x double> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> undef, double [[V0_2]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <2 x double> [[TMP2]], double [[P]], i32 1
+; CHECK-NEXT:    [[TMP4:%.*]] = fadd <2 x double> [[TMP1]], [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast double* [[TO:%.*]] to <2 x double>*
+; CHECK-NEXT:    store <2 x double> [[TMP4]], <2 x double>* [[TMP5]], align 4
+; CHECK-NEXT:    br i1 undef, label [[LP]], label [[EXT:%.*]]
+; CHECK:       ext:
+; CHECK-NEXT:    ret void
+;
+  double * noalias %to,
+  double %v1, double %v2) {
 entry:
 br label %lp
 
@@ -151,12 +238,28 @@ ext:
 }
 
 
-; CHECK-LABEL: shuffle_preserve_broadcast6
-; CHECK: %[[BCAST:[a-z0-9]+]] = insertelement <2 x double> undef, double %v0_1
-; CHECK:                      = shufflevector <2 x double> %[[BCAST]], <2 x double> undef, <2 x i32> zeroinitializer
 define void @shuffle_preserve_broadcast6(double * noalias %from,
-                                        double * noalias %to,
-                                        double %v1, double %v2) {
+; CHECK-LABEL: @shuffle_preserve_broadcast6(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LP:%.*]]
+; CHECK:       lp:
+; CHECK-NEXT:    [[P:%.*]] = phi double [ 1.000000e+00, [[LP]] ], [ 0.000000e+00, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[FROM_1:%.*]] = getelementptr double, double* [[FROM:%.*]], i32 1
+; CHECK-NEXT:    [[V0_1:%.*]] = load double, double* [[FROM]], align 4
+; CHECK-NEXT:    [[V0_2:%.*]] = load double, double* [[FROM_1]], align 4
+; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <2 x double> undef, double [[V0_1]], i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x double> [[TMP0]], <2 x double> undef, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> undef, double [[V0_2]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <2 x double> [[TMP2]], double [[P]], i32 1
+; CHECK-NEXT:    [[TMP4:%.*]] = fadd <2 x double> [[TMP1]], [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast double* [[TO:%.*]] to <2 x double>*
+; CHECK-NEXT:    store <2 x double> [[TMP4]], <2 x double>* [[TMP5]], align 4
+; CHECK-NEXT:    br i1 undef, label [[LP]], label [[EXT:%.*]]
+; CHECK:       ext:
+; CHECK-NEXT:    ret void
+;
+  double * noalias %to,
+  double %v1, double %v2) {
 entry:
 br label %lp
 
@@ -179,16 +282,46 @@ ext:
 ; Make sure we don't scramble operands when we reorder them and destroy
 ; 'good' source order.
 
-; CHECK-LABEL: good_load_order
-
-; CHECK: %[[V1:[0-9]+]] = load <4 x float>, <4 x float>*
-; CHECK: %[[V2:[0-9]+]] = insertelement <4 x float> undef, float %1, i32 0
-; CHECK: %[[V3:[0-9]+]] = shufflevector <4 x float> %[[V2]], <4 x float> %[[V1]], <4 x i32> <i32 0, i32 4, i32 5, i32 6>
-; CHECK:                = fmul <4 x float> %[[V1]], %[[V3]]
-
 @a = common global [32000 x float] zeroinitializer, align 16
 
 define void @good_load_order() {
+; CHECK-LABEL: @good_load_order(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[FOR_COND1_PREHEADER:%.*]]
+; CHECK:       for.cond1.preheader:
+; CHECK-NEXT:    [[TMP0:%.*]] = load float, float* getelementptr inbounds ([32000 x float], [32000 x float]* @a, i32 0, i32 0), align 16
+; CHECK-NEXT:    br label [[FOR_BODY3:%.*]]
+; CHECK:       for.body3:
+; CHECK-NEXT:    [[TMP1:%.*]] = phi float [ [[TMP0]], [[FOR_COND1_PREHEADER]] ], [ [[TMP14:%.*]], [[FOR_BODY3]] ]
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[FOR_COND1_PREHEADER]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY3]] ]
+; CHECK-NEXT:    [[TMP2:%.*]] = trunc i64 [[INDVARS_IV]] to i32
+; CHECK-NEXT:    [[TMP3:%.*]] = add i32 [[TMP2]], 1
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [32000 x float], [32000 x float]* @a, i32 0, i32 [[TMP3]]
+; CHECK-NEXT:    [[TMP4:%.*]] = trunc i64 [[INDVARS_IV]] to i32
+; CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds [32000 x float], [32000 x float]* @a, i32 0, i32 [[TMP4]]
+; CHECK-NEXT:    [[TMP5:%.*]] = trunc i64 [[INDVARS_IV]] to i32
+; CHECK-NEXT:    [[TMP6:%.*]] = add i32 [[TMP5]], 4
+; CHECK-NEXT:    [[ARRAYIDX31:%.*]] = getelementptr inbounds [32000 x float], [32000 x float]* @a, i32 0, i32 [[TMP6]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast float* [[ARRAYIDX]] to <4 x float>*
+; CHECK-NEXT:    [[TMP8:%.*]] = load <4 x float>, <4 x float>* [[TMP7]], align 4
+; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <4 x float> undef, float [[TMP1]], i32 0
+; CHECK-NEXT:    [[TMP10:%.*]] = shufflevector <4 x float> [[TMP9]], <4 x float> [[TMP8]], <4 x i32> <i32 0, i32 4, i32 5, i32 6>
+; CHECK-NEXT:    [[TMP11:%.*]] = fmul <4 x float> [[TMP8]], [[TMP10]]
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast float* [[ARRAYIDX5]] to <4 x float>*
+; CHECK-NEXT:    store <4 x float> [[TMP11]], <4 x float>* [[TMP12]], align 4
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 5
+; CHECK-NEXT:    [[TMP13:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
+; CHECK-NEXT:    [[ARRAYIDX41:%.*]] = getelementptr inbounds [32000 x float], [32000 x float]* @a, i32 0, i32 [[TMP13]]
+; CHECK-NEXT:    [[TMP14]] = load float, float* [[ARRAYIDX41]], align 4
+; CHECK-NEXT:    [[TMP15:%.*]] = extractelement <4 x float> [[TMP8]], i32 3
+; CHECK-NEXT:    [[MUL45:%.*]] = fmul float [[TMP14]], [[TMP15]]
+; CHECK-NEXT:    store float [[MUL45]], float* [[ARRAYIDX31]], align 4
+; CHECK-NEXT:    [[TMP16:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp slt i32 [[TMP16]], 31995
+; CHECK-NEXT:    br i1 [[CMP2]], label [[FOR_BODY3]], label [[FOR_END:%.*]]
+; CHECK:       for.end:
+; CHECK-NEXT:    ret void
+;
 entry:
   br label %for.cond1.preheader
 
@@ -237,10 +370,17 @@ for.end:
 ;  c[0] = a[0]+b[0];
 ;  c[1] = b[1]+a[1]; // swapped b[1] and a[1]
 
-; CHECK-LABEL: load_reorder_double
-; CHECK: load <2 x double>, <2 x double>*
-; CHECK: fadd <2 x double>
 define void @load_reorder_double(double* nocapture %c, double* noalias nocapture readonly %a, double* noalias nocapture readonly %b){
+; CHECK-LABEL: @load_reorder_double(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast double* [[B:%.*]] to <2 x double>*
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x double>, <2 x double>* [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast double* [[A:%.*]] to <2 x double>*
+; CHECK-NEXT:    [[TMP4:%.*]] = load <2 x double>, <2 x double>* [[TMP3]], align 4
+; CHECK-NEXT:    [[TMP5:%.*]] = fadd <2 x double> [[TMP4]], [[TMP2]]
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast double* [[C:%.*]] to <2 x double>*
+; CHECK-NEXT:    store <2 x double> [[TMP5]], <2 x double>* [[TMP6]], align 4
+; CHECK-NEXT:    ret void
+;
   %1 = load double, double* %a
   %2 = load double, double* %b
   %3 = fadd double %1, %2
@@ -261,10 +401,17 @@ define void @load_reorder_double(double* nocapture %c, double* noalias nocapture
 ;  c[2] = a[2]+b[2];
 ;  c[3] = a[3]+b[3];
 
-; CHECK-LABEL: load_reorder_float
-; CHECK: load <4 x float>, <4 x float>*
-; CHECK: fadd <4 x float>
 define void @load_reorder_float(float* nocapture %c, float* noalias nocapture readonly %a, float* noalias nocapture readonly %b){
+; CHECK-LABEL: @load_reorder_float(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast float* [[A:%.*]] to <4 x float>*
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x float>, <4 x float>* [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast float* [[B:%.*]] to <4 x float>*
+; CHECK-NEXT:    [[TMP4:%.*]] = load <4 x float>, <4 x float>* [[TMP3]], align 4
+; CHECK-NEXT:    [[TMP5:%.*]] = fadd <4 x float> [[TMP2]], [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast float* [[C:%.*]] to <4 x float>*
+; CHECK-NEXT:    store <4 x float> [[TMP5]], <4 x float>* [[TMP6]], align 4
+; CHECK-NEXT:    ret void
+;
   %1 = load float, float* %a
   %2 = load float, float* %b
   %3 = fadd float %1, %2
@@ -299,11 +446,21 @@ define void @load_reorder_float(float* nocapture %c, float* noalias nocapture re
 ; a[2] = (b[2]+c[2])+d[2];
 ; a[3] = (b[3]+c[3])+d[3];
 
-; CHECK-LABEL: opcode_reorder
-; CHECK: load <4 x float>, <4 x float>*
-; CHECK: fadd <4 x float>
-define void @opcode_reorder(float* noalias nocapture %a, float* noalias nocapture readonly %b, 
-                            float* noalias nocapture readonly %c,float* noalias nocapture readonly %d){
+define void @opcode_reorder(float* noalias nocapture %a, float* noalias nocapture readonly %b,
+; CHECK-LABEL: @opcode_reorder(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast float* [[B:%.*]] to <4 x float>*
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x float>, <4 x float>* [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast float* [[C:%.*]] to <4 x float>*
+; CHECK-NEXT:    [[TMP4:%.*]] = load <4 x float>, <4 x float>* [[TMP3]], align 4
+; CHECK-NEXT:    [[TMP5:%.*]] = fadd <4 x float> [[TMP2]], [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast float* [[D:%.*]] to <4 x float>*
+; CHECK-NEXT:    [[TMP7:%.*]] = load <4 x float>, <4 x float>* [[TMP6]], align 4
+; CHECK-NEXT:    [[TMP8:%.*]] = fadd <4 x float> [[TMP5]], [[TMP7]]
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast float* [[A:%.*]] to <4 x float>*
+; CHECK-NEXT:    store <4 x float> [[TMP8]], <4 x float>* [[TMP9]], align 4
+; CHECK-NEXT:    ret void
+;
+  float* noalias nocapture readonly %c,float* noalias nocapture readonly %d){
   %1 = load float, float* %b
   %2 = load float, float* %c
   %3 = fadd float %1, %2
diff --git a/test/Transforms/SLPVectorizer/X86/opt.ll b/test/Transforms/SLPVectorizer/X86/opt.ll
index 824e9992af03c04dbb136b926fe0d45862f2b889..97aa601b071f8effb82ec895c4363309ee0d98ae 100644
--- a/test/Transforms/SLPVectorizer/X86/opt.ll
+++ b/test/Transforms/SLPVectorizer/X86/opt.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -O3 -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s --check-prefix=SLP
 ; RUN: opt < %s -O3 -disable-slp-vectorization -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s --check-prefix=NOSLP
 
@@ -6,14 +7,33 @@ target triple = "x86_64-apple-macosx10.8.0"
 
 ; Make sure we can disable slp vectorization in opt.
 
-; SLP-LABEL: test1
-; SLP: store <2 x double>
-
-; NOSLP-LABEL: test1
-; NOSLP-NOT: store <2 x double>
-
-
 define void @test1(double* %a, double* %b, double* %c) {
+; SLP-LABEL: @test1(
+; SLP-NEXT:  entry:
+; SLP-NEXT:    [[TMP0:%.*]] = bitcast double* [[A:%.*]] to <2 x double>*
+; SLP-NEXT:    [[TMP1:%.*]] = load <2 x double>, <2 x double>* [[TMP0]], align 8
+; SLP-NEXT:    [[TMP2:%.*]] = bitcast double* [[B:%.*]] to <2 x double>*
+; SLP-NEXT:    [[TMP3:%.*]] = load <2 x double>, <2 x double>* [[TMP2]], align 8
+; SLP-NEXT:    [[TMP4:%.*]] = fmul <2 x double> [[TMP1]], [[TMP3]]
+; SLP-NEXT:    [[TMP5:%.*]] = bitcast double* [[C:%.*]] to <2 x double>*
+; SLP-NEXT:    store <2 x double> [[TMP4]], <2 x double>* [[TMP5]], align 8
+; SLP-NEXT:    ret void
+;
+; NOSLP-LABEL: @test1(
+; NOSLP-NEXT:  entry:
+; NOSLP-NEXT:    [[I0:%.*]] = load double, double* [[A:%.*]], align 8
+; NOSLP-NEXT:    [[I1:%.*]] = load double, double* [[B:%.*]], align 8
+; NOSLP-NEXT:    [[MUL:%.*]] = fmul double [[I0]], [[I1]]
+; NOSLP-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds double, double* [[A]], i64 1
+; NOSLP-NEXT:    [[I3:%.*]] = load double, double* [[ARRAYIDX3]], align 8
+; NOSLP-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds double, double* [[B]], i64 1
+; NOSLP-NEXT:    [[I4:%.*]] = load double, double* [[ARRAYIDX4]], align 8
+; NOSLP-NEXT:    [[MUL5:%.*]] = fmul double [[I3]], [[I4]]
+; NOSLP-NEXT:    store double [[MUL]], double* [[C:%.*]], align 8
+; NOSLP-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds double, double* [[C]], i64 1
+; NOSLP-NEXT:    store double [[MUL5]], double* [[ARRAYIDX5]], align 8
+; NOSLP-NEXT:    ret void
+;
 entry:
   %i0 = load double, double* %a, align 8
   %i1 = load double, double* %b, align 8
diff --git a/test/Transforms/SLPVectorizer/X86/ordering.ll b/test/Transforms/SLPVectorizer/X86/ordering.ll
index 11f5a3ddc74181770170c4cb1f05ca0b2c66aa65..bc6ef182c98d0706007ac03ef57cf1d748239c66 100644
--- a/test/Transforms/SLPVectorizer/X86/ordering.ll
+++ b/test/Transforms/SLPVectorizer/X86/ordering.ll
@@ -1,9 +1,14 @@
-; RUN: opt < %s -basicaa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -basicaa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7 | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.8.0"
 
 define void @updateModelQPFrame(i32 %m_Bits) {
+; CHECK-LABEL: @updateModelQPFrame(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret void
+;
 entry:
   %0 = load double, double* undef, align 8
   %mul = fmul double undef, %0
@@ -22,35 +27,52 @@ declare i8* @objc_msgSend(i8*, i8*, ...)
 declare i32 @personality_v0(...)
 
 define void @invoketest() personality i8* bitcast (i32 (...)* @personality_v0 to i8*) {
+; CHECK-LABEL: @invoketest(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 undef, label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
+; CHECK:       cond.true:
+; CHECK-NEXT:    [[CALL49:%.*]] = invoke double bitcast (i8* (i8*, i8*, ...)* @objc_msgSend to double (i8*, i8*)*)(i8* undef, i8* undef)
+; CHECK-NEXT:    to label [[COND_TRUE54:%.*]] unwind label [[LPAD:%.*]]
+; CHECK:       cond.false:
+; CHECK-NEXT:    [[CALL51:%.*]] = invoke double bitcast (i8* (i8*, i8*, ...)* @objc_msgSend to double (i8*, i8*)*)(i8* undef, i8* undef)
+; CHECK-NEXT:    to label [[COND_FALSE57:%.*]] unwind label [[LPAD]]
+; CHECK:       cond.true54:
+; CHECK-NEXT:    [[CALL56:%.*]] = invoke double bitcast (i8* (i8*, i8*, ...)* @objc_msgSend to double (i8*, i8*)*)(i8* undef, i8* undef)
+; CHECK-NEXT:    to label [[COND_END60:%.*]] unwind label [[LPAD]]
+; CHECK:       cond.false57:
+; CHECK-NEXT:    [[CALL59:%.*]] = invoke double bitcast (i8* (i8*, i8*, ...)* @objc_msgSend to double (i8*, i8*)*)(i8* undef, i8* undef)
+; CHECK-NEXT:    to label [[COND_END60]] unwind label [[LPAD]]
+; CHECK:       cond.end60:
+; CHECK-NEXT:    br i1 undef, label [[IF_END98:%.*]], label [[IF_THEN63:%.*]]
+; CHECK:       if.then63:
+; CHECK-NEXT:    br label [[IF_END98]]
+; CHECK:       lpad:
+; CHECK-NEXT:    [[L:%.*]] = landingpad { i8*, i32 }
+; CHECK-NEXT:    cleanup
+; CHECK-NEXT:    resume { i8*, i32 } [[L]]
+; CHECK:       if.end98:
+; CHECK-NEXT:    br label [[IF_END99:%.*]]
+; CHECK:       if.end99:
+; CHECK-NEXT:    ret void
+;
 entry:
   br i1 undef, label %cond.true, label %cond.false
 
 cond.true:
-  %call49 = invoke double bitcast (i8* (i8*, i8*, ...)* @objc_msgSend to double (i8*, i8*)*)(i8* undef, i8* undef) 
-          to label %cond.true54 unwind label %lpad
+  %call49 = invoke double bitcast (i8* (i8*, i8*, ...)* @objc_msgSend to double (i8*, i8*)*)(i8* undef, i8* undef)
+  to label %cond.true54 unwind label %lpad
 
 cond.false:
   %call51 = invoke double bitcast (i8* (i8*, i8*, ...)* @objc_msgSend to double (i8*, i8*)*)(i8* undef, i8* undef)
-          to label %cond.false57 unwind label %lpad
+  to label %cond.false57 unwind label %lpad
 
 cond.true54:
-  %call56 = invoke double bitcast (i8* (i8*, i8*, ...)* @objc_msgSend to double (i8*, i8*)*)(i8* undef, i8* undef) 
-          to label %cond.end60 unwind label %lpad
+  %call56 = invoke double bitcast (i8* (i8*, i8*, ...)* @objc_msgSend to double (i8*, i8*)*)(i8* undef, i8* undef)
+  to label %cond.end60 unwind label %lpad
 
 cond.false57:
   %call59 = invoke double bitcast (i8* (i8*, i8*, ...)* @objc_msgSend to double (i8*, i8*)*)(i8* undef, i8* undef)
-          to label %cond.end60 unwind label %lpad
-
-; Make sure we don't vectorize these phis - they have invokes as inputs.
-
-; RUN: opt < %s -slp-vectorizer -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7 | FileCheck %s
-
-; CHECK-LABEL: invoketest
-
-; CHECK-LABEL: cond.end60
-; CHECK-NOT: phi <2 x double>
-; CHECK: insertelement
-; CHECK-LABEL: if.then63
+  to label %cond.end60 unwind label %lpad
 
 cond.end60:
   %cond126 = phi double [ %call49, %cond.true54 ], [ %call51, %cond.false57 ]
@@ -68,7 +90,7 @@ if.then63:
 
 lpad:
   %l = landingpad { i8*, i32 }
-          cleanup
+  cleanup
   resume { i8*, i32 } %l
 
 if.end98:
diff --git a/test/Transforms/SLPVectorizer/X86/phi.ll b/test/Transforms/SLPVectorizer/X86/phi.ll
index ef94467f509a1a9f28f2b1eb5ed714acdf0067b6..be63297284bef54d82621744c859b65299c39e37 100644
--- a/test/Transforms/SLPVectorizer/X86/phi.ll
+++ b/test/Transforms/SLPVectorizer/X86/phi.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -basicaa -slp-vectorizer -slp-threshold=-100 -dce -S -mtriple=i386-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
 
 target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128-n8:16:32-S128"
@@ -18,12 +19,22 @@ target triple = "i386-apple-macosx10.9.0"
 ;}
 
 
-;CHECK: i32 @foo
-;CHECK: load <2 x double>
-;CHECK: phi <2 x double>
-;CHECK: store <2 x double>
-;CHECK: ret i32 undef
 define i32 @foo(double* nocapture %A, i32 %k) {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[K:%.*]], 0
+; CHECK-NEXT:    br i1 [[TOBOOL]], label [[IF_ELSE:%.*]], label [[IF_END:%.*]]
+; CHECK:       if.else:
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds double, double* [[A:%.*]], i64 10
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast double* [[ARRAYIDX]] to <2 x double>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x double>, <2 x double>* [[TMP0]], align 8
+; CHECK-NEXT:    br label [[IF_END]]
+; CHECK:       if.end:
+; CHECK-NEXT:    [[TMP2:%.*]] = phi <2 x double> [ [[TMP1]], [[IF_ELSE]] ], [ <double 3.000000e+00, double 5.000000e+00>, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast double* [[A]] to <2 x double>*
+; CHECK-NEXT:    store <2 x double> [[TMP2]], <2 x double>* [[TMP3]], align 8
+; CHECK-NEXT:    ret i32 undef
+;
 entry:
   %tobool = icmp eq i32 %k, 0
   br i1 %tobool, label %if.else, label %if.end
@@ -61,13 +72,26 @@ if.end:                                           ; preds = %entry, %if.else
 ;  return 0;
 ;}
 
-;CHECK: foo2
-;CHECK: load <2 x double>
-;CHECK: phi <2 x double>
-;CHECK: fmul <2 x double>
-;CHECK: store <2 x double>
-;CHECK: ret
 define i32 @foo2(double* noalias nocapture %B, double* noalias nocapture %A, i32 %n, i32 %m) #0 {
+; CHECK-LABEL: @foo2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast double* [[A:%.*]] to <2 x double>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x double>, <2 x double>* [[TMP0]], align 8
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[I_019:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[TMP2:%.*]] = phi <2 x double> [ [[TMP1]], [[ENTRY]] ], [ [[TMP5:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[TMP3:%.*]] = fadd <2 x double> <double 1.000000e+01, double 1.000000e+01>, [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = fmul <2 x double> <double 4.000000e+00, double 4.000000e+00>, [[TMP3]]
+; CHECK-NEXT:    [[TMP5]] = fadd <2 x double> <double 4.000000e+00, double 4.000000e+00>, [[TMP4]]
+; CHECK-NEXT:    [[INC]] = add nsw i32 [[I_019]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], 100
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_END:%.*]], label [[FOR_BODY]]
+; CHECK:       for.end:
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast double* [[B:%.*]] to <2 x double>*
+; CHECK-NEXT:    store <2 x double> [[TMP5]], <2 x double>* [[TMP6]], align 8
+; CHECK-NEXT:    ret i32 0
+;
 entry:
   %arrayidx = getelementptr inbounds double, double* %A, i64 1
   %0 = load double, double* %arrayidx, align 8
@@ -113,15 +137,60 @@ for.end:                                          ; preds = %for.body
 ;   return R+G+B+Y+P;
 ; }
 
-;CHECK: foo3
-;CHECK: phi <4 x float>
-;CHECK: fmul <4 x float>
-;CHECK: fadd <4 x float>
-;CHECK-NOT: phi <5 x float>
-;CHECK-NOT: fmul <5 x float>
-;CHECK-NOT: fadd <5 x float>
-
 define float @foo3(float* nocapture readonly %A) #0 {
+; CHECK-LABEL: @foo3(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load float, float* [[A:%.*]], align 4
+; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds float, float* [[A]], i64 1
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast float* [[ARRAYIDX1]] to <4 x float>*
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x float>, <4 x float>* [[TMP1]], align 4
+; CHECK-NEXT:    [[REORDER_SHUFFLE:%.*]] = shufflevector <4 x float> [[TMP2]], <4 x float> undef, <4 x i32> <i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <4 x float> [[REORDER_SHUFFLE]], i32 3
+; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <2 x float> undef, float [[TMP0]], i32 0
+; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <2 x float> [[TMP4]], float [[TMP3]], i32 1
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[TMP6:%.*]] = phi float [ [[TMP0]], [[ENTRY]] ], [ [[TMP18:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[TMP7:%.*]] = phi <4 x float> [ [[REORDER_SHUFFLE]], [[ENTRY]] ], [ [[TMP23:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[TMP8:%.*]] = phi <2 x float> [ [[TMP5]], [[ENTRY]] ], [ [[TMP26:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[MUL:%.*]] = fmul float [[TMP6]], 7.000000e+00
+; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <2 x float> [[TMP8]], i32 0
+; CHECK-NEXT:    [[ADD6:%.*]] = fadd float [[TMP9]], [[MUL]]
+; CHECK-NEXT:    [[TMP10:%.*]] = add nsw i64 [[INDVARS_IV]], 2
+; CHECK-NEXT:    [[ARRAYIDX14:%.*]] = getelementptr inbounds float, float* [[A]], i64 [[TMP10]]
+; CHECK-NEXT:    [[TMP11:%.*]] = load float, float* [[ARRAYIDX14]], align 4
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 3
+; CHECK-NEXT:    [[ARRAYIDX19:%.*]] = getelementptr inbounds float, float* [[A]], i64 [[INDVARS_IV_NEXT]]
+; CHECK-NEXT:    [[TMP12:%.*]] = bitcast float* [[ARRAYIDX19]] to <2 x float>*
+; CHECK-NEXT:    [[TMP13:%.*]] = load <2 x float>, <2 x float>* [[TMP12]], align 4
+; CHECK-NEXT:    [[REORDER_SHUFFLE1:%.*]] = shufflevector <2 x float> [[TMP13]], <2 x float> undef, <2 x i32> <i32 1, i32 0>
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <2 x float> [[TMP8]], i32 1
+; CHECK-NEXT:    [[TMP15:%.*]] = insertelement <4 x float> <float 1.100000e+01, float 1.000000e+01, float 9.000000e+00, float undef>, float [[TMP14]], i32 3
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <2 x float> [[REORDER_SHUFFLE1]], i32 0
+; CHECK-NEXT:    [[TMP17:%.*]] = insertelement <4 x float> undef, float [[TMP16]], i32 0
+; CHECK-NEXT:    [[TMP18]] = extractelement <2 x float> [[REORDER_SHUFFLE1]], i32 1
+; CHECK-NEXT:    [[TMP19:%.*]] = insertelement <4 x float> [[TMP17]], float [[TMP18]], i32 1
+; CHECK-NEXT:    [[TMP20:%.*]] = insertelement <4 x float> [[TMP19]], float [[TMP11]], i32 2
+; CHECK-NEXT:    [[TMP21:%.*]] = insertelement <4 x float> [[TMP20]], float 8.000000e+00, i32 3
+; CHECK-NEXT:    [[TMP22:%.*]] = fmul <4 x float> [[TMP15]], [[TMP21]]
+; CHECK-NEXT:    [[TMP23]] = fadd <4 x float> [[TMP7]], [[TMP22]]
+; CHECK-NEXT:    [[TMP24:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP24]], 121
+; CHECK-NEXT:    [[TMP25:%.*]] = insertelement <2 x float> undef, float [[ADD6]], i32 0
+; CHECK-NEXT:    [[TMP26]] = insertelement <2 x float> [[TMP25]], float [[TMP16]], i32 1
+; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_END:%.*]]
+; CHECK:       for.end:
+; CHECK-NEXT:    [[TMP27:%.*]] = extractelement <4 x float> [[TMP23]], i32 3
+; CHECK-NEXT:    [[ADD28:%.*]] = fadd float [[ADD6]], [[TMP27]]
+; CHECK-NEXT:    [[TMP28:%.*]] = extractelement <4 x float> [[TMP23]], i32 2
+; CHECK-NEXT:    [[ADD29:%.*]] = fadd float [[ADD28]], [[TMP28]]
+; CHECK-NEXT:    [[TMP29:%.*]] = extractelement <4 x float> [[TMP23]], i32 1
+; CHECK-NEXT:    [[ADD30:%.*]] = fadd float [[ADD29]], [[TMP29]]
+; CHECK-NEXT:    [[TMP30:%.*]] = extractelement <4 x float> [[TMP23]], i32 0
+; CHECK-NEXT:    [[ADD31:%.*]] = fadd float [[ADD30]], [[TMP30]]
+; CHECK-NEXT:    ret float [[ADD31]]
+;
 entry:
   %0 = load float, float* %A, align 4
   %arrayidx1 = getelementptr inbounds float, float* %A, i64 1
@@ -176,11 +245,35 @@ for.end:                                          ; preds = %for.body
 
 ; Make sure the order of phi nodes of different types does not prevent
 ; vectorization of same typed phi nodes.
-; CHECK-LABEL: sort_phi_type
-; CHECK: phi <4 x float>
-; CHECK: fmul <4 x float>
-
 define float @sort_phi_type(float* nocapture readonly %A) {
+; CHECK-LABEL: @sort_phi_type(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = phi <4 x float> [ <float 1.000000e+01, float 1.000000e+01, float 1.000000e+01, float 1.000000e+01>, [[ENTRY]] ], [ [[TMP9:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <4 x float> [[TMP0]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <4 x float> undef, float [[TMP1]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
+; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <4 x float> [[TMP2]], float [[TMP3]], i32 1
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <4 x float> [[TMP0]], i32 3
+; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <4 x float> [[TMP4]], float [[TMP5]], i32 2
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <4 x float> [[TMP0]], i32 2
+; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <4 x float> [[TMP6]], float [[TMP7]], i32 3
+; CHECK-NEXT:    [[TMP9]] = fmul <4 x float> <float 8.000000e+00, float 9.000000e+00, float 1.000000e+02, float 1.110000e+02>, [[TMP8]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nsw i64 [[INDVARS_IV]], 4
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i64 [[INDVARS_IV_NEXT]], 128
+; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY]], label [[FOR_END:%.*]]
+; CHECK:       for.end:
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <4 x float> [[TMP9]], i32 0
+; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <4 x float> [[TMP9]], i32 1
+; CHECK-NEXT:    [[ADD29:%.*]] = fadd float [[TMP10]], [[TMP11]]
+; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <4 x float> [[TMP9]], i32 2
+; CHECK-NEXT:    [[ADD30:%.*]] = fadd float [[ADD29]], [[TMP12]]
+; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <4 x float> [[TMP9]], i32 3
+; CHECK-NEXT:    [[ADD31:%.*]] = fadd float [[ADD30]], [[TMP13]]
+; CHECK-NEXT:    ret float [[ADD31]]
+;
 entry:
   br label %for.body
 
@@ -208,20 +301,33 @@ for.end:                                          ; preds = %for.body
 
 define void @test(x86_fp80* %i1, x86_fp80* %i2, x86_fp80* %o) {
 ; CHECK-LABEL: @test(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[I1_0:%.*]] = load x86_fp80, x86_fp80* [[I1:%.*]], align 16
+; CHECK-NEXT:    [[I1_GEP1:%.*]] = getelementptr x86_fp80, x86_fp80* [[I1]], i64 1
+; CHECK-NEXT:    [[I1_1:%.*]] = load x86_fp80, x86_fp80* [[I1_GEP1]], align 16
+; CHECK-NEXT:    br i1 undef, label [[THEN:%.*]], label [[END:%.*]]
+; CHECK:       then:
+; CHECK-NEXT:    [[I2_GEP0:%.*]] = getelementptr inbounds x86_fp80, x86_fp80* [[I2:%.*]], i64 0
+; CHECK-NEXT:    [[I2_0:%.*]] = load x86_fp80, x86_fp80* [[I2_GEP0]], align 16
+; CHECK-NEXT:    [[I2_GEP1:%.*]] = getelementptr inbounds x86_fp80, x86_fp80* [[I2]], i64 1
+; CHECK-NEXT:    [[I2_1:%.*]] = load x86_fp80, x86_fp80* [[I2_GEP1]], align 16
+; CHECK-NEXT:    br label [[END]]
+; CHECK:       end:
+; CHECK-NEXT:    [[PHI0:%.*]] = phi x86_fp80 [ [[I1_0]], [[ENTRY:%.*]] ], [ [[I2_0]], [[THEN]] ]
+; CHECK-NEXT:    [[PHI1:%.*]] = phi x86_fp80 [ [[I1_1]], [[ENTRY]] ], [ [[I2_1]], [[THEN]] ]
+; CHECK-NEXT:    store x86_fp80 [[PHI0]], x86_fp80* [[O:%.*]], align 16
+; CHECK-NEXT:    [[O_GEP1:%.*]] = getelementptr inbounds x86_fp80, x86_fp80* [[O]], i64 1
+; CHECK-NEXT:    store x86_fp80 [[PHI1]], x86_fp80* [[O_GEP1]], align 16
+; CHECK-NEXT:    ret void
 ;
 ; Test that we correctly recognize the discontiguous memory in arrays where the
 ; size is less than the alignment, and through various different GEP formations.
-;
-; We disable the vectorization of x86_fp80 for now. 
+; We disable the vectorization of x86_fp80 for now.
 
 entry:
   %i1.0 = load x86_fp80, x86_fp80* %i1, align 16
   %i1.gep1 = getelementptr x86_fp80, x86_fp80* %i1, i64 1
   %i1.1 = load x86_fp80, x86_fp80* %i1.gep1, align 16
-; CHECK: load x86_fp80, x86_fp80*
-; CHECK: load x86_fp80, x86_fp80*
-; CHECK-NOT: insertelement <2 x x86_fp80>
-; CHECK-NOT: insertelement <2 x x86_fp80>
   br i1 undef, label %then, label %end
 
 then:
@@ -229,18 +335,11 @@ then:
   %i2.0 = load x86_fp80, x86_fp80* %i2.gep0, align 16
   %i2.gep1 = getelementptr inbounds x86_fp80, x86_fp80* %i2, i64 1
   %i2.1 = load x86_fp80, x86_fp80* %i2.gep1, align 16
-; CHECK: load x86_fp80, x86_fp80*
-; CHECK: load x86_fp80, x86_fp80*
-; CHECK-NOT: insertelement <2 x x86_fp80>
-; CHECK-NOT: insertelement <2 x x86_fp80>
   br label %end
 
 end:
   %phi0 = phi x86_fp80 [ %i1.0, %entry ], [ %i2.0, %then ]
   %phi1 = phi x86_fp80 [ %i1.1, %entry ], [ %i2.1, %then ]
-; CHECK-NOT: phi <2 x x86_fp80>
-; CHECK-NOT: extractelement <2 x x86_fp80>
-; CHECK-NOT: extractelement <2 x x86_fp80>
   store x86_fp80 %phi0, x86_fp80* %o, align 16
   %o.gep1 = getelementptr inbounds x86_fp80, x86_fp80* %o, i64 1
   store x86_fp80 %phi1, x86_fp80* %o.gep1, align 16
diff --git a/test/Transforms/SLPVectorizer/X86/phi3.ll b/test/Transforms/SLPVectorizer/X86/phi3.ll
index 61628301aec79de300790f5ebe8bccab6124de9d..b08d9abf9ff55d55e713930cf00af859b553a8a1 100644
--- a/test/Transforms/SLPVectorizer/X86/phi3.ll
+++ b/test/Transforms/SLPVectorizer/X86/phi3.ll
@@ -1,4 +1,5 @@
-; RUN: opt < %s -basicaa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -basicaa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7 | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.8.0"
@@ -10,6 +11,24 @@ target triple = "x86_64-apple-macosx10.8.0"
 declare %struct.GPar.0.16.26* @Rf_gpptr(...)
 
 define void @Rf_GReset() {
+; CHECK-LABEL: @Rf_GReset(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load double, double* @d, align 8
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x double> undef, double [[TMP0]], i32 1
+; CHECK-NEXT:    [[TMP2:%.*]] = fsub <2 x double> <double -0.000000e+00, double -0.000000e+00>, [[TMP1]]
+; CHECK-NEXT:    br i1 icmp eq (%struct.GPar.0.16.26* (...)* inttoptr (i64 115 to %struct.GPar.0.16.26* (...)*), %struct.GPar.0.16.26* (...)* @Rf_gpptr), label [[IF_THEN:%.*]], label [[IF_END7:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[TMP3:%.*]] = fsub <2 x double> [[TMP2]], undef
+; CHECK-NEXT:    [[TMP4:%.*]] = fdiv <2 x double> [[TMP3]], undef
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <2 x double> [[TMP4]], i32 0
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <2 x double> [[TMP4]], i32 1
+; CHECK-NEXT:    [[CMP:%.*]] = fcmp ogt double [[TMP5]], [[TMP6]]
+; CHECK-NEXT:    br i1 [[CMP]], label [[IF_THEN6:%.*]], label [[IF_END7]]
+; CHECK:       if.then6:
+; CHECK-NEXT:    br label [[IF_END7]]
+; CHECK:       if.end7:
+; CHECK-NEXT:    ret void
+;
 entry:
   %sub = fsub double -0.000000e+00, undef
   %0 = load double, double* @d, align 8
diff --git a/test/Transforms/SLPVectorizer/X86/phi_landingpad.ll b/test/Transforms/SLPVectorizer/X86/phi_landingpad.ll
index b47a6ce2a263a4a11a2d9b97deaeac82afe2bd50..0a752889065f0ff14600ac20d68b1c6f7a023a11 100644
--- a/test/Transforms/SLPVectorizer/X86/phi_landingpad.ll
+++ b/test/Transforms/SLPVectorizer/X86/phi_landingpad.ll
@@ -1,18 +1,35 @@
-; RUN: opt < %s -slp-vectorizer -mtriple=x86_64-apple-macosx10.9.0 -disable-output
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -slp-vectorizer -mtriple=x86_64-apple-macosx10.9.0 -S -o - | FileCheck %s
 
 target datalayout = "f64:64:64-v64:64:64"
 
 define void @test_phi_in_landingpad() personality i8*
-          bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
+; CHECK-LABEL: @test_phi_in_landingpad(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    invoke void @foo()
+; CHECK-NEXT:    to label [[INNER:%.*]] unwind label [[LPAD:%.*]]
+; CHECK:       inner:
+; CHECK-NEXT:    invoke void @foo()
+; CHECK-NEXT:    to label [[DONE:%.*]] unwind label [[LPAD]]
+; CHECK:       lpad:
+; CHECK-NEXT:    [[TMP0:%.*]] = phi <2 x double> [ undef, [[ENTRY:%.*]] ], [ undef, [[INNER]] ]
+; CHECK-NEXT:    [[TMP1:%.*]] = landingpad { i8*, i32 }
+; CHECK-NEXT:    catch i8* null
+; CHECK-NEXT:    br label [[DONE]]
+; CHECK:       done:
+; CHECK-NEXT:    [[TMP2:%.*]] = phi <2 x double> [ undef, [[INNER]] ], [ [[TMP0]], [[LPAD]] ]
+; CHECK-NEXT:    ret void
+;
+  bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
 entry:
   invoke void @foo()
-          to label %inner unwind label %lpad
+  to label %inner unwind label %lpad
 
 inner:
   %x0 = fsub double undef, undef
   %y0 = fsub double undef, undef
   invoke void @foo()
-          to label %done unwind label %lpad
+  to label %done unwind label %lpad
 
 lpad:
   %x1 = phi double [ undef, %entry ], [ undef, %inner ]
diff --git a/test/Transforms/SLPVectorizer/X86/phi_overalignedtype.ll b/test/Transforms/SLPVectorizer/X86/phi_overalignedtype.ll
index fa08effcd648ba6154493e473e4683d8a92dc54c..f708341877af55e8d977c1af2305a9173bf3d3a6 100644
--- a/test/Transforms/SLPVectorizer/X86/phi_overalignedtype.ll
+++ b/test/Transforms/SLPVectorizer/X86/phi_overalignedtype.ll
@@ -1,12 +1,36 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -basicaa -slp-vectorizer -slp-threshold=-100 -dce -S -mtriple=i386-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
 
-; We purposely over-align f64 to 128bit here. 
+; We purposely over-align f64 to 128bit here.
 target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:128:128-v64:64:64-v128:128:128-a0:0:64-f80:128:128-n8:16:32-S128"
 target triple = "i386-apple-macosx10.9.0"
 
 
 define void @test(double* %i1, double* %i2, double* %o) {
 ; CHECK-LABEL: @test(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[I1_0:%.*]] = load double, double* [[I1:%.*]], align 16
+; CHECK-NEXT:    [[I1_GEP1:%.*]] = getelementptr double, double* [[I1]], i64 1
+; CHECK-NEXT:    [[I1_1:%.*]] = load double, double* [[I1_GEP1]], align 16
+; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <2 x double> undef, double [[I1_0]], i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <2 x double> [[TMP0]], double [[I1_1]], i32 1
+; CHECK-NEXT:    br i1 undef, label [[THEN:%.*]], label [[END:%.*]]
+; CHECK:       then:
+; CHECK-NEXT:    [[I2_GEP0:%.*]] = getelementptr inbounds double, double* [[I2:%.*]], i64 0
+; CHECK-NEXT:    [[I2_0:%.*]] = load double, double* [[I2_GEP0]], align 16
+; CHECK-NEXT:    [[I2_GEP1:%.*]] = getelementptr inbounds double, double* [[I2]], i64 1
+; CHECK-NEXT:    [[I2_1:%.*]] = load double, double* [[I2_GEP1]], align 16
+; CHECK-NEXT:    [[TMP2:%.*]] = insertelement <2 x double> undef, double [[I2_0]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <2 x double> [[TMP2]], double [[I2_1]], i32 1
+; CHECK-NEXT:    br label [[END]]
+; CHECK:       end:
+; CHECK-NEXT:    [[TMP4:%.*]] = phi <2 x double> [ [[TMP1]], [[ENTRY:%.*]] ], [ [[TMP3]], [[THEN]] ]
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <2 x double> [[TMP4]], i32 0
+; CHECK-NEXT:    store double [[TMP5]], double* [[O:%.*]], align 16
+; CHECK-NEXT:    [[O_GEP1:%.*]] = getelementptr inbounds double, double* [[O]], i64 1
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <2 x double> [[TMP4]], i32 1
+; CHECK-NEXT:    store double [[TMP6]], double* [[O_GEP1]], align 16
+; CHECK-NEXT:    ret void
 ;
 ; Test that we correctly recognize the discontiguous memory in arrays where the
 ; size is less than the alignment, and through various different GEP formations.
@@ -15,10 +39,6 @@ entry:
   %i1.0 = load double, double* %i1, align 16
   %i1.gep1 = getelementptr double, double* %i1, i64 1
   %i1.1 = load double, double* %i1.gep1, align 16
-; CHECK: load double, double*
-; CHECK: load double, double*
-; CHECK: insertelement <2 x double>
-; CHECK: insertelement <2 x double>
   br i1 undef, label %then, label %end
 
 then:
@@ -26,18 +46,11 @@ then:
   %i2.0 = load double, double* %i2.gep0, align 16
   %i2.gep1 = getelementptr inbounds double, double* %i2, i64 1
   %i2.1 = load double, double* %i2.gep1, align 16
-; CHECK: load double, double*
-; CHECK: load double, double*
-; CHECK: insertelement <2 x double>
-; CHECK: insertelement <2 x double>
   br label %end
 
 end:
   %phi0 = phi double [ %i1.0, %entry ], [ %i2.0, %then ]
   %phi1 = phi double [ %i1.1, %entry ], [ %i2.1, %then ]
-; CHECK: phi <2 x double>
-; CHECK: extractelement <2 x double>
-; CHECK: extractelement <2 x double>
   store double %phi0, double* %o, align 16
   %o.gep1 = getelementptr inbounds double, double* %o, i64 1
   store double %phi1, double* %o.gep1, align 16
diff --git a/test/Transforms/SLPVectorizer/X86/pr16628.ll b/test/Transforms/SLPVectorizer/X86/pr16628.ll
index 06abe9127e6f1ca804062f57a66a05e6d4bccc8e..9f195649c6330945db1c491dd05977e064e3370a 100644
--- a/test/Transforms/SLPVectorizer/X86/pr16628.ll
+++ b/test/Transforms/SLPVectorizer/X86/pr16628.ll
@@ -1,4 +1,5 @@
-; RUN: opt < %s -basicaa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -basicaa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7 | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.9.0"
@@ -9,6 +10,21 @@ target triple = "x86_64-apple-macosx10.9.0"
 
 ; Function Attrs: nounwind ssp uwtable
 define void @f() {
+; CHECK-LABEL: @f(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CALL:%.*]] = tail call i32 (...) @g()
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* @c, align 4
+; CHECK-NEXT:    [[LNOT:%.*]] = icmp eq i32 [[TMP0]], 0
+; CHECK-NEXT:    [[LNOT_EXT:%.*]] = zext i1 [[LNOT]] to i32
+; CHECK-NEXT:    [[TMP1:%.*]] = load i16, i16* @a, align 2
+; CHECK-NEXT:    [[LNOT2:%.*]] = icmp eq i16 [[TMP1]], 0
+; CHECK-NEXT:    [[LNOT_EXT3:%.*]] = zext i1 [[LNOT2]] to i32
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[LNOT_EXT3]], [[LNOT_EXT]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[CALL]], [[OR]]
+; CHECK-NEXT:    [[CONV4:%.*]] = zext i1 [[CMP]] to i16
+; CHECK-NEXT:    store i16 [[CONV4]], i16* @b, align 2
+; CHECK-NEXT:    ret void
+;
 entry:
   %call = tail call i32 (...) @g()
   %0 = load i32, i32* @c, align 4
diff --git a/test/Transforms/SLPVectorizer/X86/pr16899.ll b/test/Transforms/SLPVectorizer/X86/pr16899.ll
index 0de14ec3585696415ba095c151e9c2cd2b8d6e4d..5b91c3072b1981000345f2229ea49c5f1ddfea92 100644
--- a/test/Transforms/SLPVectorizer/X86/pr16899.ll
+++ b/test/Transforms/SLPVectorizer/X86/pr16899.ll
@@ -1,4 +1,5 @@
-; RUN: opt < %s  -slp-vectorizer -S -mtriple=i386--netbsd -mcpu=i486
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s  -slp-vectorizer -S -mtriple=i386--netbsd -mcpu=i486 | FileCheck %s
 target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:32:32-n8:16:32-S128"
 target triple = "i386--netbsd"
 
@@ -6,6 +7,20 @@ target triple = "i386--netbsd"
 
 ; Function Attrs: noreturn nounwind readonly
 define i32 @fn1() #0 {
+; CHECK-LABEL: @fn1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32*, i32** @a, align 4, !tbaa !0
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, i32* [[TMP0]], align 4, !tbaa !4
+; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i32, i32* [[TMP0]], i32 1
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, i32* [[ARRAYIDX1]], align 4, !tbaa !4
+; CHECK-NEXT:    br label [[DO_BODY:%.*]]
+; CHECK:       do.body:
+; CHECK-NEXT:    [[C_0:%.*]] = phi i32 [ [[TMP2]], [[ENTRY:%.*]] ], [ [[ADD2:%.*]], [[DO_BODY]] ]
+; CHECK-NEXT:    [[B_0:%.*]] = phi i32 [ [[TMP1]], [[ENTRY]] ], [ [[ADD:%.*]], [[DO_BODY]] ]
+; CHECK-NEXT:    [[ADD]] = add nsw i32 [[B_0]], [[C_0]]
+; CHECK-NEXT:    [[ADD2]] = add nsw i32 [[ADD]], 1
+; CHECK-NEXT:    br label [[DO_BODY]]
+;
 entry:
   %0 = load i32*, i32** @a, align 4, !tbaa !4
   %1 = load i32, i32* %0, align 4, !tbaa !5
diff --git a/test/Transforms/SLPVectorizer/X86/pr18060.ll b/test/Transforms/SLPVectorizer/X86/pr18060.ll
index e6813f3b315d5547c043545242d93d9c06761b4f..0af5d0fa0531db367e3380810e8369bc088c108a 100644
--- a/test/Transforms/SLPVectorizer/X86/pr18060.ll
+++ b/test/Transforms/SLPVectorizer/X86/pr18060.ll
@@ -1,19 +1,53 @@
-; RUN: opt < %s -slp-vectorizer -S -mtriple=i386-pc-linux
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -slp-vectorizer -S -mtriple=i386-pc-linux | FileCheck %s
 
 target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:32:32-n8:16:32-S128"
 target triple = "i386-pc-linux"
 
 ; Function Attrs: nounwind
 define i32 @_Z16adjustFixupValueyj(i64 %Value, i32 %Kind) {
+; CHECK-LABEL: @_Z16adjustFixupValueyj(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[EXTRACT_T:%.*]] = trunc i64 [[VALUE:%.*]] to i32
+; CHECK-NEXT:    [[EXTRACT:%.*]] = lshr i64 [[VALUE]], 12
+; CHECK-NEXT:    [[EXTRACT_T6:%.*]] = trunc i64 [[EXTRACT]] to i32
+; CHECK-NEXT:    switch i32 [[KIND:%.*]], label [[SW_DEFAULT:%.*]] [
+; CHECK-NEXT:    i32 0, label [[RETURN:%.*]]
+; CHECK-NEXT:    i32 1, label [[RETURN]]
+; CHECK-NEXT:    i32 129, label [[SW_BB1:%.*]]
+; CHECK-NEXT:    i32 130, label [[SW_BB2:%.*]]
+; CHECK-NEXT:    ]
+; CHECK:       sw.default:
+; CHECK-NEXT:    call void @_Z25llvm_unreachable_internalv()
+; CHECK-NEXT:    unreachable
+; CHECK:       sw.bb1:
+; CHECK-NEXT:    [[SHR:%.*]] = lshr i64 [[VALUE]], 16
+; CHECK-NEXT:    [[EXTRACT_T5:%.*]] = trunc i64 [[SHR]] to i32
+; CHECK-NEXT:    [[EXTRACT7:%.*]] = lshr i64 [[VALUE]], 28
+; CHECK-NEXT:    [[EXTRACT_T8:%.*]] = trunc i64 [[EXTRACT7]] to i32
+; CHECK-NEXT:    br label [[SW_BB2]]
+; CHECK:       sw.bb2:
+; CHECK-NEXT:    [[VALUE_ADDR_0_OFF0:%.*]] = phi i32 [ [[EXTRACT_T]], [[ENTRY:%.*]] ], [ [[EXTRACT_T5]], [[SW_BB1]] ]
+; CHECK-NEXT:    [[VALUE_ADDR_0_OFF12:%.*]] = phi i32 [ [[EXTRACT_T6]], [[ENTRY]] ], [ [[EXTRACT_T8]], [[SW_BB1]] ]
+; CHECK-NEXT:    [[CONV6:%.*]] = and i32 [[VALUE_ADDR_0_OFF0]], 4095
+; CHECK-NEXT:    [[CONV4:%.*]] = shl i32 [[VALUE_ADDR_0_OFF12]], 16
+; CHECK-NEXT:    [[SHL:%.*]] = and i32 [[CONV4]], 983040
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[SHL]], [[CONV6]]
+; CHECK-NEXT:    [[OR11:%.*]] = or i32 [[OR]], 8388608
+; CHECK-NEXT:    br label [[RETURN]]
+; CHECK:       return:
+; CHECK-NEXT:    [[RETVAL_0:%.*]] = phi i32 [ [[OR11]], [[SW_BB2]] ], [ [[EXTRACT_T]], [[ENTRY]] ], [ [[EXTRACT_T]], [[ENTRY]] ]
+; CHECK-NEXT:    ret i32 [[RETVAL_0]]
+;
 entry:
   %extract.t = trunc i64 %Value to i32
   %extract = lshr i64 %Value, 12
   %extract.t6 = trunc i64 %extract to i32
   switch i32 %Kind, label %sw.default [
-    i32 0, label %return
-    i32 1, label %return
-    i32 129, label %sw.bb1
-    i32 130, label %sw.bb2
+  i32 0, label %return
+  i32 1, label %return
+  i32 129, label %sw.bb1
+  i32 130, label %sw.bb2
   ]
 
 sw.default:                                       ; preds = %entry
diff --git a/test/Transforms/SLPVectorizer/X86/pr23510.ll b/test/Transforms/SLPVectorizer/X86/pr23510.ll
index efdb0ecd9996d72dfaf678521dd5a576ae11177a..420fdde3631bbd8afc2ef70e91f821b99e1c15b9 100644
--- a/test/Transforms/SLPVectorizer/X86/pr23510.ll
+++ b/test/Transforms/SLPVectorizer/X86/pr23510.ll
@@ -1,16 +1,37 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; PR23510
 ; RUN: opt < %s -basicaa -slp-vectorizer -S | FileCheck %s
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
-; CHECK-LABEL: @_Z3fooPml(
-; CHECK: lshr <2 x i64>
-; CHECK: lshr <2 x i64>
-
 @total = global i64 0, align 8
 
 define void @_Z3fooPml(i64* nocapture %a, i64 %i) {
+; CHECK-LABEL: @_Z3fooPml(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i64, i64* [[A:%.*]], i64 1
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast i64* [[A]] to <2 x i64>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, <2 x i64>* [[TMP0]], align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = lshr <2 x i64> [[TMP1]], <i64 4, i64 4>
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i64* [[A]] to <2 x i64>*
+; CHECK-NEXT:    store <2 x i64> [[TMP2]], <2 x i64>* [[TMP3]], align 8
+; CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds i64, i64* [[A]], i64 [[I:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, i64* [[ARRAYIDX3]], align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = load i64, i64* @total, align 8
+; CHECK-NEXT:    [[ADD:%.*]] = add i64 [[TMP3]], [[TMP2]]
+; CHECK-NEXT:    store i64 [[ADD]], i64* @total, align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i64* [[A]] to <2 x i64>*
+; CHECK-NEXT:    [[TMP5:%.*]] = load <2 x i64>, <2 x i64>* [[TMP4]], align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = lshr <2 x i64> [[TMP5]], <i64 4, i64 4>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i64* [[A]] to <2 x i64>*
+; CHECK-NEXT:    store <2 x i64> [[TMP6]], <2 x i64>* [[TMP7]], align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = load i64, i64* [[ARRAYIDX3]], align 8
+; CHECK-NEXT:    [[TMP7:%.*]] = load i64, i64* @total, align 8
+; CHECK-NEXT:    [[ADD9:%.*]] = add i64 [[TMP7]], [[TMP6]]
+; CHECK-NEXT:    store i64 [[ADD9]], i64* @total, align 8
+; CHECK-NEXT:    ret void
+;
 entry:
   %tmp = load i64, i64* %a, align 8
   %shr = lshr i64 %tmp, 4
diff --git a/test/Transforms/SLPVectorizer/X86/pr27163.ll b/test/Transforms/SLPVectorizer/X86/pr27163.ll
index 2b8480ef82c6d998915be1769aa6e9baa7f6d32a..b1c1d9540d01fea6536627e01ac022bd2ea2dd5a 100644
--- a/test/Transforms/SLPVectorizer/X86/pr27163.ll
+++ b/test/Transforms/SLPVectorizer/X86/pr27163.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -slp-vectorizer -S < %s | FileCheck %s
 target datalayout = "e-m:w-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-pc-windows-msvc18.0.0"
@@ -5,6 +6,31 @@ target triple = "x86_64-pc-windows-msvc18.0.0"
 %struct.B = type { i64, i64 }
 
 define void @test1(%struct.B* %p) personality i32 (...)* @__CxxFrameHandler3 {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:  invoke.cont:
+; CHECK-NEXT:    [[GEP1:%.*]] = getelementptr inbounds [[STRUCT_B:%.*]], %struct.B* [[P:%.*]], i64 0, i32 0
+; CHECK-NEXT:    [[GEP2:%.*]] = getelementptr inbounds [[STRUCT_B]], %struct.B* [[P]], i64 0, i32 1
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast i64* [[GEP1]] to <2 x i64>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, <2 x i64>* [[TMP0]], align 8
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <2 x i64> [[TMP1]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i64* [[GEP1]] to <2 x i64>*
+; CHECK-NEXT:    store <2 x i64> [[TMP1]], <2 x i64>* [[TMP3]], align 8
+; CHECK-NEXT:    invoke void @throw()
+; CHECK-NEXT:    to label [[UNREACHABLE:%.*]] unwind label [[CATCH_DISPATCH:%.*]]
+; CHECK:       catch.dispatch:
+; CHECK-NEXT:    [[CS:%.*]] = catchswitch within none [label %invoke.cont1] unwind label [[EHCLEANUP:%.*]]
+; CHECK:       invoke.cont1:
+; CHECK-NEXT:    [[CATCH:%.*]] = catchpad within [[CS]] [i8* null, i32 64, i8* null]
+; CHECK-NEXT:    invoke void @throw() [ "funclet"(token [[CATCH]]) ]
+; CHECK-NEXT:    to label [[UNREACHABLE]] unwind label [[EHCLEANUP]]
+; CHECK:       ehcleanup:
+; CHECK-NEXT:    [[PHI:%.*]] = phi i64 [ [[TMP2]], [[CATCH_DISPATCH]] ], [ 9, [[INVOKE_CONT1:%.*]] ]
+; CHECK-NEXT:    [[CLEANUP:%.*]] = cleanuppad within none []
+; CHECK-NEXT:    call void @release(i64 [[PHI]]) [ "funclet"(token [[CLEANUP]]) ]
+; CHECK-NEXT:    cleanupret from [[CLEANUP]] unwind to caller
+; CHECK:       unreachable:
+; CHECK-NEXT:    unreachable
+;
 invoke.cont:
   %gep1 = getelementptr inbounds %struct.B, %struct.B* %p, i64 0, i32 0
   %gep2 = getelementptr inbounds %struct.B, %struct.B* %p, i64 0, i32 1
@@ -13,7 +39,7 @@ invoke.cont:
   store i64 %load1, i64* %gep1, align 8
   store i64 %load2, i64* %gep2, align 8
   invoke void @throw()
-          to label %unreachable unwind label %catch.dispatch
+  to label %unreachable unwind label %catch.dispatch
 
 catch.dispatch:                                   ; preds = %invoke.cont
   %cs = catchswitch within none [label %invoke.cont1] unwind label %ehcleanup
@@ -21,7 +47,7 @@ catch.dispatch:                                   ; preds = %invoke.cont
 invoke.cont1:                                     ; preds = %catch.dispatch
   %catch = catchpad within %cs [i8* null, i32 64, i8* null]
   invoke void @throw() [ "funclet"(token %catch) ]
-          to label %unreachable unwind label %ehcleanup
+  to label %unreachable unwind label %ehcleanup
 
 ehcleanup:                                        ; preds = %invoke.cont1, %catch.dispatch
   %phi = phi i64 [ %load1, %catch.dispatch ], [ 9, %invoke.cont1 ]
@@ -33,16 +59,6 @@ unreachable:                                      ; preds = %invoke.cont1, %invo
   unreachable
 }
 
-
-; CHECK-LABEL: define void @test1(
-; CHECK: %[[gep:.*]] = getelementptr inbounds %struct.B, %struct.B* %p, i64 0, i32 0
-; CHECK: %[[bc:.*]]  = bitcast i64* %[[gep]] to <2 x i64>*
-; CHECK: %[[ld:.*]]  = load <2 x i64>, <2 x i64>* %[[bc]], align 8
-; CHECK: %[[ee:.*]]  = extractelement <2 x i64> %[[ld]], i32 0
-
-; CHECK: %[[phi:.*]] = phi i64 [ %[[ee]], {{.*}} ], [ 9, {{.*}} ]
-; CHECK: call void @release(i64 %[[phi]])
-
 declare i32 @__CxxFrameHandler3(...)
 
 declare void @throw()
diff --git a/test/Transforms/SLPVectorizer/X86/propagate_ir_flags.ll b/test/Transforms/SLPVectorizer/X86/propagate_ir_flags.ll
index 28217fcba17e174dbc38b36714d6d94ed934fc6e..7cc0194c730200a24ea8f4650cb22896f813b42d 100644
--- a/test/Transforms/SLPVectorizer/X86/propagate_ir_flags.ll
+++ b/test/Transforms/SLPVectorizer/X86/propagate_ir_flags.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -basicaa -slp-vectorizer -S | FileCheck %s
 
 ; Check propagation of optional IR flags (PR20802). For a flag to
@@ -7,9 +8,19 @@
 target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-unknown"
 
-; CHECK-LABEL: @exact(
-; CHECK: lshr exact <4 x i32>
 define void @exact(i32* %x) {
+; CHECK-LABEL: @exact(
+; CHECK-NEXT:    [[IDX1:%.*]] = getelementptr inbounds i32, i32* [[X:%.*]], i64 0
+; CHECK-NEXT:    [[IDX2:%.*]] = getelementptr inbounds i32, i32* [[X]], i64 1
+; CHECK-NEXT:    [[IDX3:%.*]] = getelementptr inbounds i32, i32* [[X]], i64 2
+; CHECK-NEXT:    [[IDX4:%.*]] = getelementptr inbounds i32, i32* [[X]], i64 3
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i32* [[IDX1]] to <4 x i32>*
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, <4 x i32>* [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = lshr exact <4 x i32> [[TMP2]], <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i32* [[IDX1]] to <4 x i32>*
+; CHECK-NEXT:    store <4 x i32> [[TMP3]], <4 x i32>* [[TMP4]], align 4
+; CHECK-NEXT:    ret void
+;
   %idx1 = getelementptr inbounds i32, i32* %x, i64 0
   %idx2 = getelementptr inbounds i32, i32* %x, i64 1
   %idx3 = getelementptr inbounds i32, i32* %x, i64 2
@@ -33,9 +44,19 @@ define void @exact(i32* %x) {
   ret void
 }
 
-; CHECK-LABEL: @not_exact(
-; CHECK: lshr <4 x i32>
 define void @not_exact(i32* %x) {
+; CHECK-LABEL: @not_exact(
+; CHECK-NEXT:    [[IDX1:%.*]] = getelementptr inbounds i32, i32* [[X:%.*]], i64 0
+; CHECK-NEXT:    [[IDX2:%.*]] = getelementptr inbounds i32, i32* [[X]], i64 1
+; CHECK-NEXT:    [[IDX3:%.*]] = getelementptr inbounds i32, i32* [[X]], i64 2
+; CHECK-NEXT:    [[IDX4:%.*]] = getelementptr inbounds i32, i32* [[X]], i64 3
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i32* [[IDX1]] to <4 x i32>*
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, <4 x i32>* [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = lshr <4 x i32> [[TMP2]], <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i32* [[IDX1]] to <4 x i32>*
+; CHECK-NEXT:    store <4 x i32> [[TMP3]], <4 x i32>* [[TMP4]], align 4
+; CHECK-NEXT:    ret void
+;
   %idx1 = getelementptr inbounds i32, i32* %x, i64 0
   %idx2 = getelementptr inbounds i32, i32* %x, i64 1
   %idx3 = getelementptr inbounds i32, i32* %x, i64 2
@@ -59,9 +80,19 @@ define void @not_exact(i32* %x) {
   ret void
 }
 
-; CHECK-LABEL: @nsw(
-; CHECK: add nsw <4 x i32>
 define void @nsw(i32* %x) {
+; CHECK-LABEL: @nsw(
+; CHECK-NEXT:    [[IDX1:%.*]] = getelementptr inbounds i32, i32* [[X:%.*]], i64 0
+; CHECK-NEXT:    [[IDX2:%.*]] = getelementptr inbounds i32, i32* [[X]], i64 1
+; CHECK-NEXT:    [[IDX3:%.*]] = getelementptr inbounds i32, i32* [[X]], i64 2
+; CHECK-NEXT:    [[IDX4:%.*]] = getelementptr inbounds i32, i32* [[X]], i64 3
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i32* [[IDX1]] to <4 x i32>*
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, <4 x i32>* [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = add nsw <4 x i32> <i32 1, i32 1, i32 1, i32 1>, [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i32* [[IDX1]] to <4 x i32>*
+; CHECK-NEXT:    store <4 x i32> [[TMP3]], <4 x i32>* [[TMP4]], align 4
+; CHECK-NEXT:    ret void
+;
   %idx1 = getelementptr inbounds i32, i32* %x, i64 0
   %idx2 = getelementptr inbounds i32, i32* %x, i64 1
   %idx3 = getelementptr inbounds i32, i32* %x, i64 2
@@ -85,9 +116,19 @@ define void @nsw(i32* %x) {
   ret void
 }
 
-; CHECK-LABEL: @not_nsw(
-; CHECK: add <4 x i32>
 define void @not_nsw(i32* %x) {
+; CHECK-LABEL: @not_nsw(
+; CHECK-NEXT:    [[IDX1:%.*]] = getelementptr inbounds i32, i32* [[X:%.*]], i64 0
+; CHECK-NEXT:    [[IDX2:%.*]] = getelementptr inbounds i32, i32* [[X]], i64 1
+; CHECK-NEXT:    [[IDX3:%.*]] = getelementptr inbounds i32, i32* [[X]], i64 2
+; CHECK-NEXT:    [[IDX4:%.*]] = getelementptr inbounds i32, i32* [[X]], i64 3
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i32* [[IDX1]] to <4 x i32>*
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, <4 x i32>* [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = add <4 x i32> <i32 1, i32 1, i32 1, i32 1>, [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i32* [[IDX1]] to <4 x i32>*
+; CHECK-NEXT:    store <4 x i32> [[TMP3]], <4 x i32>* [[TMP4]], align 4
+; CHECK-NEXT:    ret void
+;
   %idx1 = getelementptr inbounds i32, i32* %x, i64 0
   %idx2 = getelementptr inbounds i32, i32* %x, i64 1
   %idx3 = getelementptr inbounds i32, i32* %x, i64 2
@@ -111,9 +152,19 @@ define void @not_nsw(i32* %x) {
   ret void
 }
 
-; CHECK-LABEL: @nuw(
-; CHECK: add nuw <4 x i32>
 define void @nuw(i32* %x) {
+; CHECK-LABEL: @nuw(
+; CHECK-NEXT:    [[IDX1:%.*]] = getelementptr inbounds i32, i32* [[X:%.*]], i64 0
+; CHECK-NEXT:    [[IDX2:%.*]] = getelementptr inbounds i32, i32* [[X]], i64 1
+; CHECK-NEXT:    [[IDX3:%.*]] = getelementptr inbounds i32, i32* [[X]], i64 2
+; CHECK-NEXT:    [[IDX4:%.*]] = getelementptr inbounds i32, i32* [[X]], i64 3
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i32* [[IDX1]] to <4 x i32>*
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, <4 x i32>* [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = add nuw <4 x i32> <i32 1, i32 1, i32 1, i32 1>, [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i32* [[IDX1]] to <4 x i32>*
+; CHECK-NEXT:    store <4 x i32> [[TMP3]], <4 x i32>* [[TMP4]], align 4
+; CHECK-NEXT:    ret void
+;
   %idx1 = getelementptr inbounds i32, i32* %x, i64 0
   %idx2 = getelementptr inbounds i32, i32* %x, i64 1
   %idx3 = getelementptr inbounds i32, i32* %x, i64 2
@@ -136,10 +187,20 @@ define void @nuw(i32* %x) {
 
   ret void
 }
- 
-; CHECK-LABEL: @not_nuw(
-; CHECK: add <4 x i32>
+
 define void @not_nuw(i32* %x) {
+; CHECK-LABEL: @not_nuw(
+; CHECK-NEXT:    [[IDX1:%.*]] = getelementptr inbounds i32, i32* [[X:%.*]], i64 0
+; CHECK-NEXT:    [[IDX2:%.*]] = getelementptr inbounds i32, i32* [[X]], i64 1
+; CHECK-NEXT:    [[IDX3:%.*]] = getelementptr inbounds i32, i32* [[X]], i64 2
+; CHECK-NEXT:    [[IDX4:%.*]] = getelementptr inbounds i32, i32* [[X]], i64 3
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i32* [[IDX1]] to <4 x i32>*
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, <4 x i32>* [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = add <4 x i32> <i32 1, i32 1, i32 1, i32 1>, [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i32* [[IDX1]] to <4 x i32>*
+; CHECK-NEXT:    store <4 x i32> [[TMP3]], <4 x i32>* [[TMP4]], align 4
+; CHECK-NEXT:    ret void
+;
   %idx1 = getelementptr inbounds i32, i32* %x, i64 0
   %idx2 = getelementptr inbounds i32, i32* %x, i64 1
   %idx3 = getelementptr inbounds i32, i32* %x, i64 2
@@ -162,10 +223,20 @@ define void @not_nuw(i32* %x) {
 
   ret void
 }
- 
-; CHECK-LABEL: @nnan(
-; CHECK: fadd nnan <4 x float>
+
 define void @nnan(float* %x) {
+; CHECK-LABEL: @nnan(
+; CHECK-NEXT:    [[IDX1:%.*]] = getelementptr inbounds float, float* [[X:%.*]], i64 0
+; CHECK-NEXT:    [[IDX2:%.*]] = getelementptr inbounds float, float* [[X]], i64 1
+; CHECK-NEXT:    [[IDX3:%.*]] = getelementptr inbounds float, float* [[X]], i64 2
+; CHECK-NEXT:    [[IDX4:%.*]] = getelementptr inbounds float, float* [[X]], i64 3
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast float* [[IDX1]] to <4 x float>*
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x float>, <4 x float>* [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = fadd nnan <4 x float> <float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00>, [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast float* [[IDX1]] to <4 x float>*
+; CHECK-NEXT:    store <4 x float> [[TMP3]], <4 x float>* [[TMP4]], align 4
+; CHECK-NEXT:    ret void
+;
   %idx1 = getelementptr inbounds float, float* %x, i64 0
   %idx2 = getelementptr inbounds float, float* %x, i64 1
   %idx3 = getelementptr inbounds float, float* %x, i64 2
@@ -188,10 +259,20 @@ define void @nnan(float* %x) {
 
   ret void
 }
- 
-; CHECK-LABEL: @not_nnan(
-; CHECK: fadd <4 x float>
+
 define void @not_nnan(float* %x) {
+; CHECK-LABEL: @not_nnan(
+; CHECK-NEXT:    [[IDX1:%.*]] = getelementptr inbounds float, float* [[X:%.*]], i64 0
+; CHECK-NEXT:    [[IDX2:%.*]] = getelementptr inbounds float, float* [[X]], i64 1
+; CHECK-NEXT:    [[IDX3:%.*]] = getelementptr inbounds float, float* [[X]], i64 2
+; CHECK-NEXT:    [[IDX4:%.*]] = getelementptr inbounds float, float* [[X]], i64 3
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast float* [[IDX1]] to <4 x float>*
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x float>, <4 x float>* [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = fadd <4 x float> <float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00>, [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast float* [[IDX1]] to <4 x float>*
+; CHECK-NEXT:    store <4 x float> [[TMP3]], <4 x float>* [[TMP4]], align 4
+; CHECK-NEXT:    ret void
+;
   %idx1 = getelementptr inbounds float, float* %x, i64 0
   %idx2 = getelementptr inbounds float, float* %x, i64 1
   %idx3 = getelementptr inbounds float, float* %x, i64 2
@@ -214,10 +295,20 @@ define void @not_nnan(float* %x) {
 
   ret void
 }
- 
-; CHECK-LABEL: @only_fast(
-; CHECK: fadd fast <4 x float>
+
 define void @only_fast(float* %x) {
+; CHECK-LABEL: @only_fast(
+; CHECK-NEXT:    [[IDX1:%.*]] = getelementptr inbounds float, float* [[X:%.*]], i64 0
+; CHECK-NEXT:    [[IDX2:%.*]] = getelementptr inbounds float, float* [[X]], i64 1
+; CHECK-NEXT:    [[IDX3:%.*]] = getelementptr inbounds float, float* [[X]], i64 2
+; CHECK-NEXT:    [[IDX4:%.*]] = getelementptr inbounds float, float* [[X]], i64 3
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast float* [[IDX1]] to <4 x float>*
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x float>, <4 x float>* [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = fadd fast <4 x float> <float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00>, [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast float* [[IDX1]] to <4 x float>*
+; CHECK-NEXT:    store <4 x float> [[TMP3]], <4 x float>* [[TMP4]], align 4
+; CHECK-NEXT:    ret void
+;
   %idx1 = getelementptr inbounds float, float* %x, i64 0
   %idx2 = getelementptr inbounds float, float* %x, i64 1
   %idx3 = getelementptr inbounds float, float* %x, i64 2
@@ -240,10 +331,20 @@ define void @only_fast(float* %x) {
 
   ret void
 }
- 
-; CHECK-LABEL: @only_arcp(
-; CHECK: fadd arcp <4 x float>
+
 define void @only_arcp(float* %x) {
+; CHECK-LABEL: @only_arcp(
+; CHECK-NEXT:    [[IDX1:%.*]] = getelementptr inbounds float, float* [[X:%.*]], i64 0
+; CHECK-NEXT:    [[IDX2:%.*]] = getelementptr inbounds float, float* [[X]], i64 1
+; CHECK-NEXT:    [[IDX3:%.*]] = getelementptr inbounds float, float* [[X]], i64 2
+; CHECK-NEXT:    [[IDX4:%.*]] = getelementptr inbounds float, float* [[X]], i64 3
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast float* [[IDX1]] to <4 x float>*
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x float>, <4 x float>* [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = fadd arcp <4 x float> <float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00>, [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast float* [[IDX1]] to <4 x float>*
+; CHECK-NEXT:    store <4 x float> [[TMP3]], <4 x float>* [[TMP4]], align 4
+; CHECK-NEXT:    ret void
+;
   %idx1 = getelementptr inbounds float, float* %x, i64 0
   %idx2 = getelementptr inbounds float, float* %x, i64 1
   %idx3 = getelementptr inbounds float, float* %x, i64 2
@@ -267,10 +368,21 @@ define void @only_arcp(float* %x) {
   ret void
 }
 
-; CHECK-LABEL: @addsub_all_nsw
-; CHECK: add nsw <4 x i32>
-; CHECK: sub nsw <4 x i32>
 define void @addsub_all_nsw(i32* %x) {
+; CHECK-LABEL: @addsub_all_nsw(
+; CHECK-NEXT:    [[IDX1:%.*]] = getelementptr inbounds i32, i32* [[X:%.*]], i64 0
+; CHECK-NEXT:    [[IDX2:%.*]] = getelementptr inbounds i32, i32* [[X]], i64 1
+; CHECK-NEXT:    [[IDX3:%.*]] = getelementptr inbounds i32, i32* [[X]], i64 2
+; CHECK-NEXT:    [[IDX4:%.*]] = getelementptr inbounds i32, i32* [[X]], i64 3
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i32* [[IDX1]] to <4 x i32>*
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, <4 x i32>* [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = add nsw <4 x i32> [[TMP2]], <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    [[TMP4:%.*]] = sub nsw <4 x i32> [[TMP2]], <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> [[TMP4]], <4 x i32> <i32 0, i32 5, i32 2, i32 7>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i32* [[IDX1]] to <4 x i32>*
+; CHECK-NEXT:    store <4 x i32> [[TMP5]], <4 x i32>* [[TMP6]], align 4
+; CHECK-NEXT:    ret void
+;
   %idx1 = getelementptr inbounds i32, i32* %x, i64 0
   %idx2 = getelementptr inbounds i32, i32* %x, i64 1
   %idx3 = getelementptr inbounds i32, i32* %x, i64 2
@@ -293,11 +405,22 @@ define void @addsub_all_nsw(i32* %x) {
 
   ret void
 }
- 
-; CHECK-LABEL: @addsub_some_nsw
-; CHECK: add nsw <4 x i32>
-; CHECK: sub <4 x i32>
+
 define void @addsub_some_nsw(i32* %x) {
+; CHECK-LABEL: @addsub_some_nsw(
+; CHECK-NEXT:    [[IDX1:%.*]] = getelementptr inbounds i32, i32* [[X:%.*]], i64 0
+; CHECK-NEXT:    [[IDX2:%.*]] = getelementptr inbounds i32, i32* [[X]], i64 1
+; CHECK-NEXT:    [[IDX3:%.*]] = getelementptr inbounds i32, i32* [[X]], i64 2
+; CHECK-NEXT:    [[IDX4:%.*]] = getelementptr inbounds i32, i32* [[X]], i64 3
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i32* [[IDX1]] to <4 x i32>*
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, <4 x i32>* [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = add nsw <4 x i32> [[TMP2]], <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    [[TMP4:%.*]] = sub <4 x i32> [[TMP2]], <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> [[TMP4]], <4 x i32> <i32 0, i32 5, i32 2, i32 7>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i32* [[IDX1]] to <4 x i32>*
+; CHECK-NEXT:    store <4 x i32> [[TMP5]], <4 x i32>* [[TMP6]], align 4
+; CHECK-NEXT:    ret void
+;
   %idx1 = getelementptr inbounds i32, i32* %x, i64 0
   %idx2 = getelementptr inbounds i32, i32* %x, i64 1
   %idx3 = getelementptr inbounds i32, i32* %x, i64 2
@@ -320,11 +443,22 @@ define void @addsub_some_nsw(i32* %x) {
 
   ret void
 }
- 
-; CHECK-LABEL: @addsub_no_nsw
-; CHECK: add <4 x i32>
-; CHECK: sub <4 x i32>
+
 define void @addsub_no_nsw(i32* %x) {
+; CHECK-LABEL: @addsub_no_nsw(
+; CHECK-NEXT:    [[IDX1:%.*]] = getelementptr inbounds i32, i32* [[X:%.*]], i64 0
+; CHECK-NEXT:    [[IDX2:%.*]] = getelementptr inbounds i32, i32* [[X]], i64 1
+; CHECK-NEXT:    [[IDX3:%.*]] = getelementptr inbounds i32, i32* [[X]], i64 2
+; CHECK-NEXT:    [[IDX4:%.*]] = getelementptr inbounds i32, i32* [[X]], i64 3
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i32* [[IDX1]] to <4 x i32>*
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i32>, <4 x i32>* [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = add <4 x i32> [[TMP2]], <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    [[TMP4:%.*]] = sub <4 x i32> [[TMP2]], <i32 1, i32 1, i32 1, i32 1>
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> [[TMP4]], <4 x i32> <i32 0, i32 5, i32 2, i32 7>
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast i32* [[IDX1]] to <4 x i32>*
+; CHECK-NEXT:    store <4 x i32> [[TMP5]], <4 x i32>* [[TMP6]], align 4
+; CHECK-NEXT:    ret void
+;
   %idx1 = getelementptr inbounds i32, i32* %x, i64 0
   %idx2 = getelementptr inbounds i32, i32* %x, i64 1
   %idx3 = getelementptr inbounds i32, i32* %x, i64 2
@@ -347,11 +481,20 @@ define void @addsub_no_nsw(i32* %x) {
 
   ret void
 }
- 
-; CHECK-LABEL: @fcmp_fast
-; CHECK: fcmp fast oge <2 x double>
-; CHECK: sub fast <2 x double>
+
 define void @fcmp_fast(double* %x) #1 {
+; CHECK-LABEL: @fcmp_fast(
+; CHECK-NEXT:    [[IDX1:%.*]] = getelementptr inbounds double, double* [[X:%.*]], i64 0
+; CHECK-NEXT:    [[IDX2:%.*]] = getelementptr inbounds double, double* [[X]], i64 1
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast double* [[IDX1]] to <2 x double>*
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x double>, <2 x double>* [[TMP1]], align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = fcmp fast oge <2 x double> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = fsub fast <2 x double> <double -0.000000e+00, double -0.000000e+00>, [[TMP2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = select <2 x i1> [[TMP3]], <2 x double> [[TMP2]], <2 x double> [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast double* [[IDX1]] to <2 x double>*
+; CHECK-NEXT:    store <2 x double> [[TMP5]], <2 x double>* [[TMP6]], align 8
+; CHECK-NEXT:    ret void
+;
   %idx1 = getelementptr inbounds double, double* %x, i64 0
   %idx2 = getelementptr inbounds double, double* %x, i64 1
 
@@ -373,10 +516,19 @@ define void @fcmp_fast(double* %x) #1 {
   ret void
 }
 
-; CHECK-LABEL: @fcmp_no_fast
-; CHECK: fcmp oge <2 x double>
-; CHECK: sub <2 x double>
 define void @fcmp_no_fast(double* %x) #1 {
+; CHECK-LABEL: @fcmp_no_fast(
+; CHECK-NEXT:    [[IDX1:%.*]] = getelementptr inbounds double, double* [[X:%.*]], i64 0
+; CHECK-NEXT:    [[IDX2:%.*]] = getelementptr inbounds double, double* [[X]], i64 1
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast double* [[IDX1]] to <2 x double>*
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x double>, <2 x double>* [[TMP1]], align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = fcmp oge <2 x double> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = fsub <2 x double> <double -0.000000e+00, double -0.000000e+00>, [[TMP2]]
+; CHECK-NEXT:    [[TMP5:%.*]] = select <2 x i1> [[TMP3]], <2 x double> [[TMP2]], <2 x double> [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast double* [[IDX1]] to <2 x double>*
+; CHECK-NEXT:    store <2 x double> [[TMP5]], <2 x double>* [[TMP6]], align 8
+; CHECK-NEXT:    ret void
+;
   %idx1 = getelementptr inbounds double, double* %x, i64 0
   %idx2 = getelementptr inbounds double, double* %x, i64 1
 
@@ -400,9 +552,17 @@ define void @fcmp_no_fast(double* %x) #1 {
 
 declare double @llvm.fabs.f64(double) nounwind readnone
 
-;CHECK-LABEL: @call_fast(
-;CHECK: call fast <2 x double> @llvm.fabs.v2f64
 define void @call_fast(double* %x) {
+; CHECK-LABEL: @call_fast(
+; CHECK-NEXT:    [[IDX1:%.*]] = getelementptr inbounds double, double* [[X:%.*]], i64 0
+; CHECK-NEXT:    [[IDX2:%.*]] = getelementptr inbounds double, double* [[X]], i64 1
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast double* [[IDX1]] to <2 x double>*
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x double>, <2 x double>* [[TMP1]], align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = call fast <2 x double> @llvm.fabs.v2f64(<2 x double> [[TMP2]])
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast double* [[IDX1]] to <2 x double>*
+; CHECK-NEXT:    store <2 x double> [[TMP3]], <2 x double>* [[TMP4]], align 8
+; CHECK-NEXT:    ret void
+;
   %idx1 = getelementptr inbounds double, double* %x, i64 0
   %idx2 = getelementptr inbounds double, double* %x, i64 1
 
@@ -418,9 +578,17 @@ define void @call_fast(double* %x) {
   ret void
 }
 
-;CHECK-LABEL: @call_no_fast(
-;CHECK: call <2 x double> @llvm.fabs.v2f64
 define void @call_no_fast(double* %x) {
+; CHECK-LABEL: @call_no_fast(
+; CHECK-NEXT:    [[IDX1:%.*]] = getelementptr inbounds double, double* [[X:%.*]], i64 0
+; CHECK-NEXT:    [[IDX2:%.*]] = getelementptr inbounds double, double* [[X]], i64 1
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast double* [[IDX1]] to <2 x double>*
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x double>, <2 x double>* [[TMP1]], align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = call <2 x double> @llvm.fabs.v2f64(<2 x double> [[TMP2]])
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast double* [[IDX1]] to <2 x double>*
+; CHECK-NEXT:    store <2 x double> [[TMP3]], <2 x double>* [[TMP4]], align 8
+; CHECK-NEXT:    ret void
+;
   %idx1 = getelementptr inbounds double, double* %x, i64 0
   %idx2 = getelementptr inbounds double, double* %x, i64 1
 
diff --git a/test/Transforms/SLPVectorizer/X86/reduction.ll b/test/Transforms/SLPVectorizer/X86/reduction.ll
index 4c5f1266336b61cd08f0335c2276ec29d544c040..03b7f67ae4cabe5f06377b8b4ef715a02cf6f272 100644
--- a/test/Transforms/SLPVectorizer/X86/reduction.ll
+++ b/test/Transforms/SLPVectorizer/X86/reduction.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -basicaa -slp-vectorizer -dce -S -mtriple=i386-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
 
 target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128-n8:16:32-S128"
@@ -10,11 +11,33 @@ target triple = "i386-apple-macosx10.8.0"
 ;   return sum;
 ; }
 
-;CHECK: reduce
-;CHECK: load <2 x double>
-;CHECK: fmul <2 x double>
-;CHECK: ret
 define i32 @reduce(double* nocapture %A, i32 %n, i32 %m) {
+; CHECK-LABEL: @reduce(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP13:%.*]] = icmp sgt i32 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[CMP13]], label [[FOR_BODY:%.*]], label [[FOR_END:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[I_015:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[SUM_014:%.*]] = phi double [ [[ADD6:%.*]], [[FOR_BODY]] ], [ 0.000000e+00, [[ENTRY]] ]
+; CHECK-NEXT:    [[MUL:%.*]] = shl nsw i32 [[I_015]], 1
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds double, double* [[A:%.*]], i32 [[MUL]]
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast double* [[ARRAYIDX]] to <2 x double>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x double>, <2 x double>* [[TMP0]], align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = fmul <2 x double> <double 7.000000e+00, double 7.000000e+00>, [[TMP1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <2 x double> [[TMP2]], i32 0
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x double> [[TMP2]], i32 1
+; CHECK-NEXT:    [[ADD5:%.*]] = fadd double [[TMP3]], [[TMP4]]
+; CHECK-NEXT:    [[ADD6]] = fadd double [[SUM_014]], [[ADD5]]
+; CHECK-NEXT:    [[INC]] = add nsw i32 [[I_015]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_FOR_END_CRIT_EDGE:%.*]], label [[FOR_BODY]]
+; CHECK:       for.cond.for.end_crit_edge:
+; CHECK-NEXT:    [[PHITMP:%.*]] = fptosi double [[ADD6]] to i32
+; CHECK-NEXT:    br label [[FOR_END]]
+; CHECK:       for.end:
+; CHECK-NEXT:    [[SUM_0_LCSSA:%.*]] = phi i32 [ [[PHITMP]], [[FOR_COND_FOR_END_CRIT_EDGE]] ], [ 0, [[ENTRY]] ]
+; CHECK-NEXT:    ret i32 [[SUM_0_LCSSA]]
+;
 entry:
   %cmp13 = icmp sgt i32 %n, 0
   br i1 %cmp13, label %for.body, label %for.end
diff --git a/test/Transforms/SLPVectorizer/X86/reduction2.ll b/test/Transforms/SLPVectorizer/X86/reduction2.ll
index 507a61aa16f479203a4d371bd103ebe27c369307..87a6af792f50c4f71cbb810cbe56a231be6cf4fa 100644
--- a/test/Transforms/SLPVectorizer/X86/reduction2.ll
+++ b/test/Transforms/SLPVectorizer/X86/reduction2.ll
@@ -1,12 +1,29 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -basicaa -slp-vectorizer -dce -S -mtriple=i386-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
 
 target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128-n8:16:32-S128"
 target triple = "i386-apple-macosx10.8.0"
 
-;CHECK-LABEL: @foo(
-;CHECK: load <2 x double>
-;CHECK: ret
 define double @foo(double* nocapture %D) {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT:    br label [[TMP1:%.*]]
+; CHECK:         [[I_02:%.*]] = phi i32 [ 0, [[TMP0:%.*]] ], [ [[TMP12:%.*]], [[TMP1]] ]
+; CHECK-NEXT:    [[SUM_01:%.*]] = phi double [ 0.000000e+00, [[TMP0]] ], [ [[TMP11:%.*]], [[TMP1]] ]
+; CHECK-NEXT:    [[TMP2:%.*]] = shl nsw i32 [[I_02]], 1
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds double, double* [[D:%.*]], i32 [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast double* [[TMP3]] to <2 x double>*
+; CHECK-NEXT:    [[TMP5:%.*]] = load <2 x double>, <2 x double>* [[TMP4]], align 4
+; CHECK-NEXT:    [[TMP6:%.*]] = fmul <2 x double> [[TMP5]], [[TMP5]]
+; CHECK-NEXT:    [[TMP7:%.*]] = fmul <2 x double> [[TMP6]], [[TMP6]]
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <2 x double> [[TMP7]], i32 0
+; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <2 x double> [[TMP7]], i32 1
+; CHECK-NEXT:    [[TMP10:%.*]] = fadd double [[TMP8]], [[TMP9]]
+; CHECK-NEXT:    [[TMP11]] = fadd double [[SUM_01]], [[TMP10]]
+; CHECK-NEXT:    [[TMP12]] = add nsw i32 [[I_02]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[TMP12]], 100
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[TMP13:%.*]], label [[TMP1]]
+; CHECK:         ret double [[TMP11]]
+;
   br label %1
 
 ; <label>:1                                       ; preds = %1, %0
diff --git a/test/Transforms/SLPVectorizer/X86/remark_horcost.ll b/test/Transforms/SLPVectorizer/X86/remark_horcost.ll
index 96c8d7f3b68753af8f5457497b8450c1d45441ca..27997f6af3d73b988c32b2d2df8a1add92fcb546 100644
--- a/test/Transforms/SLPVectorizer/X86/remark_horcost.ll
+++ b/test/Transforms/SLPVectorizer/X86/remark_horcost.ll
@@ -1,7 +1,59 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -S -mtriple=x86_64-pc-linux-gnu -mcpu=generic -slp-vectorizer -pass-remarks-output=%t < %s | FileCheck %s
 ; RUN: FileCheck --input-file=%t --check-prefix=YAML %s
 
 define i32 @foo(i32* %diff) #0 {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[M2:%.*]] = alloca [8 x [8 x i32]], align 16
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast [8 x [8 x i32]]* [[M2]] to i8*
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[A_088:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[OP_EXTRA:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[INDVARS_IV]], 3
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[DIFF:%.*]], i64 [[TMP1]]
+; CHECK-NEXT:    [[TMP2:%.*]] = or i64 [[TMP1]], 4
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, i32* [[DIFF]], i64 [[TMP2]]
+; CHECK-NEXT:    [[ARRAYIDX6:%.*]] = getelementptr inbounds [8 x [8 x i32]], [8 x [8 x i32]]* [[M2]], i64 0, i64 [[INDVARS_IV]], i64 0
+; CHECK-NEXT:    [[TMP3:%.*]] = or i64 [[TMP1]], 1
+; CHECK-NEXT:    [[ARRAYIDX13:%.*]] = getelementptr inbounds i32, i32* [[DIFF]], i64 [[TMP3]]
+; CHECK-NEXT:    [[TMP4:%.*]] = or i64 [[TMP1]], 5
+; CHECK-NEXT:    [[ARRAYIDX16:%.*]] = getelementptr inbounds i32, i32* [[DIFF]], i64 [[TMP4]]
+; CHECK-NEXT:    [[TMP5:%.*]] = or i64 [[TMP1]], 2
+; CHECK-NEXT:    [[ARRAYIDX27:%.*]] = getelementptr inbounds i32, i32* [[DIFF]], i64 [[TMP5]]
+; CHECK-NEXT:    [[TMP6:%.*]] = or i64 [[TMP1]], 6
+; CHECK-NEXT:    [[ARRAYIDX30:%.*]] = getelementptr inbounds i32, i32* [[DIFF]], i64 [[TMP6]]
+; CHECK-NEXT:    [[TMP7:%.*]] = or i64 [[TMP1]], 3
+; CHECK-NEXT:    [[ARRAYIDX41:%.*]] = getelementptr inbounds i32, i32* [[DIFF]], i64 [[TMP7]]
+; CHECK-NEXT:    [[TMP8:%.*]] = bitcast i32* [[ARRAYIDX]] to <4 x i32>*
+; CHECK-NEXT:    [[TMP9:%.*]] = load <4 x i32>, <4 x i32>* [[TMP8]], align 4
+; CHECK-NEXT:    [[TMP10:%.*]] = or i64 [[TMP1]], 7
+; CHECK-NEXT:    [[ARRAYIDX44:%.*]] = getelementptr inbounds i32, i32* [[DIFF]], i64 [[TMP10]]
+; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i32* [[ARRAYIDX2]] to <4 x i32>*
+; CHECK-NEXT:    [[TMP12:%.*]] = load <4 x i32>, <4 x i32>* [[TMP11]], align 4
+; CHECK-NEXT:    [[TMP13:%.*]] = add nsw <4 x i32> [[TMP12]], [[TMP9]]
+; CHECK-NEXT:    [[ADD10:%.*]] = add nsw i32 undef, [[A_088]]
+; CHECK-NEXT:    [[ARRAYIDX20:%.*]] = getelementptr inbounds [8 x [8 x i32]], [8 x [8 x i32]]* [[M2]], i64 0, i64 [[INDVARS_IV]], i64 1
+; CHECK-NEXT:    [[ADD24:%.*]] = add nsw i32 [[ADD10]], undef
+; CHECK-NEXT:    [[ARRAYIDX34:%.*]] = getelementptr inbounds [8 x [8 x i32]], [8 x [8 x i32]]* [[M2]], i64 0, i64 [[INDVARS_IV]], i64 2
+; CHECK-NEXT:    [[ADD38:%.*]] = add nsw i32 [[ADD24]], undef
+; CHECK-NEXT:    [[ARRAYIDX48:%.*]] = getelementptr inbounds [8 x [8 x i32]], [8 x [8 x i32]]* [[M2]], i64 0, i64 [[INDVARS_IV]], i64 3
+; CHECK-NEXT:    [[TMP14:%.*]] = bitcast i32* [[ARRAYIDX6]] to <4 x i32>*
+; CHECK-NEXT:    store <4 x i32> [[TMP13]], <4 x i32>* [[TMP14]], align 16
+; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x i32> [[TMP13]], <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX:%.*]] = add nsw <4 x i32> [[TMP13]], [[RDX_SHUF]]
+; CHECK-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <4 x i32> [[BIN_RDX]], <4 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX2:%.*]] = add nsw <4 x i32> [[BIN_RDX]], [[RDX_SHUF1]]
+; CHECK-NEXT:    [[TMP15:%.*]] = extractelement <4 x i32> [[BIN_RDX2]], i32 0
+; CHECK-NEXT:    [[OP_EXTRA]] = add nsw i32 [[TMP15]], [[A_088]]
+; CHECK-NEXT:    [[ADD52:%.*]] = add nsw i32 [[ADD38]], undef
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 8
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_END:%.*]], label [[FOR_BODY]]
+; CHECK:       for.end:
+; CHECK-NEXT:    ret i32 [[OP_EXTRA]]
+;
 entry:
   %m2 = alloca [8 x [8 x i32]], align 16
   %0 = bitcast [8 x [8 x i32]]* %m2 to i8*
@@ -19,7 +71,7 @@ for.body:                                         ; preds = %for.body, %entry
   %add3 = add nsw i32 %4, %2
   %arrayidx6 = getelementptr inbounds [8 x [8 x i32]], [8 x [8 x i32]]* %m2, i64 0, i64 %indvars.iv, i64 0
   store i32 %add3, i32* %arrayidx6, align 16
- 
+
   %add10 = add nsw i32 %add3, %a.088
   %5 = or i64 %1, 1
   %arrayidx13 = getelementptr inbounds i32, i32* %diff, i64 %5
@@ -30,7 +82,7 @@ for.body:                                         ; preds = %for.body, %entry
   %add17 = add nsw i32 %8, %6
   %arrayidx20 = getelementptr inbounds [8 x [8 x i32]], [8 x [8 x i32]]* %m2, i64 0, i64 %indvars.iv, i64 1
   store i32 %add17, i32* %arrayidx20, align 4
-  
+
   %add24 = add nsw i32 %add10, %add17
   %9 = or i64 %1, 2
   %arrayidx27 = getelementptr inbounds i32, i32* %diff, i64 %9
@@ -41,7 +93,7 @@ for.body:                                         ; preds = %for.body, %entry
   %add31 = add nsw i32 %12, %10
   %arrayidx34 = getelementptr inbounds [8 x [8 x i32]], [8 x [8 x i32]]* %m2, i64 0, i64 %indvars.iv, i64 2
   store i32 %add31, i32* %arrayidx34, align 8
-  
+
   %add38 = add nsw i32 %add24, %add31
   %13 = or i64 %1, 3
   %arrayidx41 = getelementptr inbounds i32, i32* %diff, i64 %13
@@ -49,35 +101,33 @@ for.body:                                         ; preds = %for.body, %entry
   %15 = or i64 %1, 7
   %arrayidx44 = getelementptr inbounds i32, i32* %diff, i64 %15
   %16 = load i32, i32* %arrayidx44, align 4
-  
+
   %add45 = add nsw i32 %16, %14
   %arrayidx48 = getelementptr inbounds [8 x [8 x i32]], [8 x [8 x i32]]* %m2, i64 0, i64 %indvars.iv, i64 3
   store i32 %add45, i32* %arrayidx48, align 4
-  
+
   %add52 = add nsw i32 %add38, %add45
- ; CHECK: add nsw <{{[0-9]+}} x i32>
- ; CHECK: add nsw <{{[0-9]+}} x i32>
- 
- ; YAML:      --- !Passed
- ; YAML-NEXT: Pass:            slp-vectorizer
- ; YAML-NEXT: Name:            StoresVectorized
- ; YAML-NEXT: Function:        foo
- ; YAML-NEXT: Args:
- ; YAML-NEXT:   - String:          'Stores SLP vectorized with cost '
- ; YAML-NEXT:   - Cost:            '-8'
- ; YAML-NEXT:   - String:          ' and with tree size '
- ; YAML-NEXT:   - TreeSize:        '4'
 
- ; YAML:      --- !Passed
- ; YAML-NEXT: Pass:            slp-vectorizer
- ; YAML-NEXT: Name:            VectorizedHorizontalReduction
- ; YAML-NEXT: Function:        foo
- ; YAML-NEXT: Args:
- ; YAML-NEXT:   - String:          'Vectorized horizontal reduction with cost '
- ; YAML-NEXT:   - Cost:            '-2'
- ; YAML-NEXT:   - String:          ' and with tree size '
- ; YAML-NEXT:   - TreeSize:        '1'
- 
+  ; YAML:      --- !Passed
+  ; YAML-NEXT: Pass:            slp-vectorizer
+  ; YAML-NEXT: Name:            StoresVectorized
+  ; YAML-NEXT: Function:        foo
+  ; YAML-NEXT: Args:
+  ; YAML-NEXT:   - String:          'Stores SLP vectorized with cost '
+  ; YAML-NEXT:   - Cost:            '-8'
+  ; YAML-NEXT:   - String:          ' and with tree size '
+  ; YAML-NEXT:   - TreeSize:        '4'
+
+  ; YAML:      --- !Passed
+  ; YAML-NEXT: Pass:            slp-vectorizer
+  ; YAML-NEXT: Name:            VectorizedHorizontalReduction
+  ; YAML-NEXT: Function:        foo
+  ; YAML-NEXT: Args:
+  ; YAML-NEXT:   - String:          'Vectorized horizontal reduction with cost '
+  ; YAML-NEXT:   - Cost:            '-2'
+  ; YAML-NEXT:   - String:          ' and with tree size '
+  ; YAML-NEXT:   - TreeSize:        '1'
+
   %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
   %exitcond = icmp eq i64 %indvars.iv.next, 8
   br i1 %exitcond, label %for.end, label %for.body
diff --git a/test/Transforms/SLPVectorizer/X86/remark_listcost.ll b/test/Transforms/SLPVectorizer/X86/remark_listcost.ll
index 6f6e00f9c89bfa31b2d13d18dbcb48f893c00270..ed2cd870cf6f0c56edba2117cf20f9c9fff2fa30 100644
--- a/test/Transforms/SLPVectorizer/X86/remark_listcost.ll
+++ b/test/Transforms/SLPVectorizer/X86/remark_listcost.ll
@@ -1,7 +1,26 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -S -mtriple=x86_64-pc-linux-gnu -mcpu=generic -slp-vectorizer -pass-remarks-output=%t < %s | FileCheck %s
 ; RUN: FileCheck --input-file=%t --check-prefix=YAML %s
 
 define void @vsub2_test(i32* %pin1, i32* %pin2, i32* %pout) #0 {
+; CHECK-LABEL: @vsub2_test(
+; CHECK-NEXT:    br label [[TMP1:%.*]]
+; CHECK:         [[IDX_04:%.*]] = phi i32 [ 0, [[TMP0:%.*]] ], [ [[TMP8:%.*]], [[TMP1]] ]
+; CHECK-NEXT:    [[PO_03:%.*]] = phi i32* [ [[POUT:%.*]], [[TMP0]] ], [ [[TMP7:%.*]], [[TMP1]] ]
+; CHECK-NEXT:    [[PTMPI2_02:%.*]] = phi i32* [ [[PIN2:%.*]], [[TMP0]] ], [ [[TMP4:%.*]], [[TMP1]] ]
+; CHECK-NEXT:    [[PTMPI1_01:%.*]] = phi i32* [ [[PIN1:%.*]], [[TMP0]] ], [ [[TMP2:%.*]], [[TMP1]] ]
+; CHECK-NEXT:    [[TMP2]] = getelementptr inbounds i32, i32* [[PTMPI1_01]], i64 1
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, i32* [[PTMPI1_01]], align 4, !tbaa !1
+; CHECK-NEXT:    [[TMP4]] = getelementptr inbounds i32, i32* [[PTMPI2_02]], i64 1
+; CHECK-NEXT:    [[TMP5:%.*]] = load i32, i32* [[PTMPI2_02]], align 4, !tbaa !1
+; CHECK-NEXT:    [[TMP6:%.*]] = sub nsw i32 [[TMP3]], [[TMP5]]
+; CHECK-NEXT:    [[TMP7]] = getelementptr inbounds i32, i32* [[PO_03]], i64 1
+; CHECK-NEXT:    store i32 [[TMP6]], i32* [[PO_03]], align 4, !tbaa !1
+; CHECK-NEXT:    [[TMP8]] = add nuw nsw i32 [[IDX_04]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[TMP8]], 64
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[TMP9:%.*]], label [[TMP1]], !llvm.loop !5
+; CHECK:         ret void
+;
   br label %1
 
   %idx.04 = phi i32 [ 0, %0 ], [ %8, %1 ]
@@ -14,15 +33,14 @@ define void @vsub2_test(i32* %pin1, i32* %pin2, i32* %pout) #0 {
   %5 = load i32, i32* %ptmpi2.02, align 4, !tbaa !1
   %6 = sub nsw i32 %3, %5
   %7 = getelementptr inbounds i32, i32* %po.03, i64 1
- ; CHECK-NOT: <{{[0-9]+}} x i32>
- ; YAML:      Pass:            slp-vectorizer
- ; YAML-NEXT: Name:            NotBeneficial
- ; YAML-NEXT: Function:        vsub2_test
- ; YAML-NEXT: Args:
- ; YAML-NEXT:   - String:          'List vectorization was possible but not beneficial with cost '
- ; YAML-NEXT:   - Cost:            '0'
- ; YAML-NEXT:   - String:          ' >= '
- ; YAML-NEXT:   - Treshold:        '0'
+  ; YAML:      Pass:            slp-vectorizer
+  ; YAML-NEXT: Name:            NotBeneficial
+  ; YAML-NEXT: Function:        vsub2_test
+  ; YAML-NEXT: Args:
+  ; YAML-NEXT:   - String:          'List vectorization was possible but not beneficial with cost '
+  ; YAML-NEXT:   - Cost:            '0'
+  ; YAML-NEXT:   - String:          ' >= '
+  ; YAML-NEXT:   - Treshold:        '0'
   store i32 %6, i32* %po.03, align 4, !tbaa !1
   %8 = add nuw nsw i32 %idx.04, 1
   %exitcond = icmp eq i32 %8, 64
diff --git a/test/Transforms/SLPVectorizer/X86/remark_not_all_parts.ll b/test/Transforms/SLPVectorizer/X86/remark_not_all_parts.ll
index 59d4bb0710e65c3f5fe7a59e20dfd1f8bf27e507..e6029fb5a9794b4cbd7ccf3e45f8a0d7f077ff8b 100644
--- a/test/Transforms/SLPVectorizer/X86/remark_not_all_parts.ll
+++ b/test/Transforms/SLPVectorizer/X86/remark_not_all_parts.ll
@@ -1,7 +1,43 @@
-; RUN: opt -S -mtriple=x86_64-pc-linux-gnu -mcpu=generic -slp-vectorizer -pass-remarks-output=%t < %s | FileCheck %s
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -mtriple=x86_64-pc-linux-gnu -mcpu=generic -slp-vectorizer -slp-min-reg-size=128 -pass-remarks-output=%t < %s | FileCheck %s
 ; RUN: FileCheck --input-file=%t --check-prefix=YAML %s
 
 define i32 @foo(i32* nocapture readonly %diff) #0 {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[M2:%.*]] = alloca [8 x [8 x i32]], align 16
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast [8 x [8 x i32]]* [[M2]] to i8*
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[A_088:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[ADD24:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[INDVARS_IV]], 3
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[DIFF:%.*]], i64 [[TMP1]]
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, i32* [[ARRAYIDX]], align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = or i64 [[TMP1]], 4
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, i32* [[DIFF]], i64 [[TMP3]]
+; CHECK-NEXT:    [[TMP4:%.*]] = load i32, i32* [[ARRAYIDX2]], align 4
+; CHECK-NEXT:    [[ADD3:%.*]] = add nsw i32 [[TMP4]], [[TMP2]]
+; CHECK-NEXT:    [[ARRAYIDX6:%.*]] = getelementptr inbounds [8 x [8 x i32]], [8 x [8 x i32]]* [[M2]], i64 0, i64 [[INDVARS_IV]], i64 0
+; CHECK-NEXT:    store i32 [[ADD3]], i32* [[ARRAYIDX6]], align 16
+; CHECK-NEXT:    [[ADD10:%.*]] = add nsw i32 [[ADD3]], [[A_088]]
+; CHECK-NEXT:    [[TMP5:%.*]] = or i64 [[TMP1]], 1
+; CHECK-NEXT:    [[ARRAYIDX13:%.*]] = getelementptr inbounds i32, i32* [[DIFF]], i64 [[TMP5]]
+; CHECK-NEXT:    [[TMP6:%.*]] = load i32, i32* [[ARRAYIDX13]], align 4
+; CHECK-NEXT:    [[TMP7:%.*]] = or i64 [[TMP1]], 5
+; CHECK-NEXT:    [[ARRAYIDX16:%.*]] = getelementptr inbounds i32, i32* [[DIFF]], i64 [[TMP7]]
+; CHECK-NEXT:    [[TMP8:%.*]] = load i32, i32* [[ARRAYIDX16]], align 4
+; CHECK-NEXT:    [[ADD17:%.*]] = add nsw i32 [[TMP8]], [[TMP6]]
+; CHECK-NEXT:    [[ARRAYIDX20:%.*]] = getelementptr inbounds [8 x [8 x i32]], [8 x [8 x i32]]* [[M2]], i64 0, i64 [[INDVARS_IV]], i64 1
+; CHECK-NEXT:    store i32 [[ADD17]], i32* [[ARRAYIDX20]], align 4
+; CHECK-NEXT:    [[ADD24]] = add nsw i32 [[ADD10]], [[ADD17]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 8
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_END:%.*]], label [[FOR_BODY]]
+; CHECK:       for.end:
+; CHECK-NEXT:    [[ARRAYDECAY:%.*]] = getelementptr inbounds [8 x [8 x i32]], [8 x [8 x i32]]* [[M2]], i64 0, i64 0
+; CHECK-NEXT:    ret i32 [[ADD24]]
+;
 entry:
   %m2 = alloca [8 x [8 x i32]], align 16
   %0 = bitcast [8 x [8 x i32]]* %m2 to i8*
@@ -31,13 +67,12 @@ for.body:                                         ; preds = %for.body, %entry
   store i32 %add17, i32* %arrayidx20, align 4
   %add24 = add nsw i32 %add10, %add17
 
- ; CHECK-NOT: add nsw <{{[0-9]+}} x i32> 
- ; YAML:      Pass:            slp-vectorizer
- ; YAML-NEXT: Name:            NotPossible
- ; YAML-NEXT: Function:        foo
- ; YAML-NEXT: Args:
- ; YAML-NEXT:   - String:          'Cannot SLP vectorize list: vectorization was impossible'
- ; YAML-NEXT:   - String:          ' with available vectorization factors'
+  ; YAML:      Pass:            slp-vectorizer
+  ; YAML-NEXT: Name:            NotPossible
+  ; YAML-NEXT: Function:        foo
+  ; YAML-NEXT: Args:
+  ; YAML-NEXT:   - String:          'Cannot SLP vectorize list: vectorization was impossible'
+  ; YAML-NEXT:   - String:          ' with available vectorization factors'
 
   %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
   %exitcond = icmp eq i64 %indvars.iv.next, 8
diff --git a/test/Transforms/SLPVectorizer/X86/remark_unsupported.ll b/test/Transforms/SLPVectorizer/X86/remark_unsupported.ll
index d78d122cef7661f9f424bffded2669dc30aa4f27..a134aec00bbbcaba13a5cc175558903d97aeab57 100644
--- a/test/Transforms/SLPVectorizer/X86/remark_unsupported.ll
+++ b/test/Transforms/SLPVectorizer/X86/remark_unsupported.ll
@@ -1,9 +1,29 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -S -mtriple=x86_64-pc-linux-gnu -mcpu=generic -slp-vectorizer -pass-remarks-output=%t < %s | FileCheck %s
 ; RUN: FileCheck --input-file=%t --check-prefix=YAML %s
 
 ; This type is not supported by SLP
 define void @test(x86_fp80* %i1, x86_fp80* %i2, x86_fp80* %o) {
-
+; CHECK-LABEL: @test(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[I1_0:%.*]] = load x86_fp80, x86_fp80* [[I1:%.*]], align 16
+; CHECK-NEXT:    [[I1_GEP1:%.*]] = getelementptr x86_fp80, x86_fp80* [[I1]], i64 1
+; CHECK-NEXT:    [[I1_1:%.*]] = load x86_fp80, x86_fp80* [[I1_GEP1]], align 16
+; CHECK-NEXT:    br i1 undef, label [[THEN:%.*]], label [[END:%.*]]
+; CHECK:       then:
+; CHECK-NEXT:    [[I2_GEP0:%.*]] = getelementptr inbounds x86_fp80, x86_fp80* [[I2:%.*]], i64 0
+; CHECK-NEXT:    [[I2_0:%.*]] = load x86_fp80, x86_fp80* [[I2_GEP0]], align 16
+; CHECK-NEXT:    [[I2_GEP1:%.*]] = getelementptr inbounds x86_fp80, x86_fp80* [[I2]], i64 1
+; CHECK-NEXT:    [[I2_1:%.*]] = load x86_fp80, x86_fp80* [[I2_GEP1]], align 16
+; CHECK-NEXT:    br label [[END]]
+; CHECK:       end:
+; CHECK-NEXT:    [[PHI0:%.*]] = phi x86_fp80 [ [[I1_0]], [[ENTRY:%.*]] ], [ [[I2_0]], [[THEN]] ]
+; CHECK-NEXT:    [[PHI1:%.*]] = phi x86_fp80 [ [[I1_1]], [[ENTRY]] ], [ [[I2_1]], [[THEN]] ]
+; CHECK-NEXT:    store x86_fp80 [[PHI0]], x86_fp80* [[O:%.*]], align 16
+; CHECK-NEXT:    [[O_GEP1:%.*]] = getelementptr inbounds x86_fp80, x86_fp80* [[O]], i64 1
+; CHECK-NEXT:    store x86_fp80 [[PHI1]], x86_fp80* [[O_GEP1]], align 16
+; CHECK-NEXT:    ret void
+;
 entry:
   %i1.0 = load x86_fp80, x86_fp80* %i1, align 16
   %i1.gep1 = getelementptr x86_fp80, x86_fp80* %i1, i64 1
@@ -22,13 +42,12 @@ end:
   store x86_fp80 %phi0, x86_fp80* %o, align 16
   %o.gep1 = getelementptr inbounds x86_fp80, x86_fp80* %o, i64 1
   store x86_fp80 %phi1, x86_fp80* %o.gep1, align 16
- ; CHECK-NOT: <{{[0-9]+}} x x86_fp80>
- ; YAML:      Pass:            slp-vectorizer
- ; YAML-NEXT: Name:            UnsupportedType
- ; YAML-NEXT: Function:        test
- ; YAML-NEXT: Args:
- ; YAML-NEXT:   - String:          'Cannot SLP vectorize list: type '
- ; YAML-NEXT:   - String:          x86_fp80 is unsupported by vectorizer
+  ; YAML:      Pass:            slp-vectorizer
+  ; YAML-NEXT: Name:            UnsupportedType
+  ; YAML-NEXT: Function:        test
+  ; YAML-NEXT: Args:
+  ; YAML-NEXT:   - String:          'Cannot SLP vectorize list: type '
+  ; YAML-NEXT:   - String:          x86_fp80 is unsupported by vectorizer
 
   ret void
 }
diff --git a/test/Transforms/SLPVectorizer/X86/reorder_phi.ll b/test/Transforms/SLPVectorizer/X86/reorder_phi.ll
index afee26f0d40763328c6e44633b883b14a4017d2f..333ee0df6a1a1e1ed873f8574d9ed82094ff5c93 100644
--- a/test/Transforms/SLPVectorizer/X86/reorder_phi.ll
+++ b/test/Transforms/SLPVectorizer/X86/reorder_phi.ll
@@ -9,33 +9,38 @@ define  void @foo (%struct.complex* %A, %struct.complex* %B, %struct.complex* %R
 ; CHECK-NEXT:    [[TMP0:%.*]] = add i64 256, 0
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[TMP1:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[TMP20:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[TMP2:%.*]] = phi float [ 0.000000e+00, [[ENTRY]] ], [ [[TMP19:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[TMP3:%.*]] = phi float [ 0.000000e+00, [[ENTRY]] ], [ [[TMP18:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds [[STRUCT_COMPLEX:%.*]], %struct.complex* [[A:%.*]], i64 [[TMP1]], i32 0
-; CHECK-NEXT:    [[TMP5:%.*]] = load float, float* [[TMP4]], align 4
-; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds [[STRUCT_COMPLEX]], %struct.complex* [[A]], i64 [[TMP1]], i32 1
-; CHECK-NEXT:    [[TMP7:%.*]] = load float, float* [[TMP6]], align 4
-; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds [[STRUCT_COMPLEX]], %struct.complex* [[B:%.*]], i64 [[TMP1]], i32 0
-; CHECK-NEXT:    [[TMP9:%.*]] = load float, float* [[TMP8]], align 4
-; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr inbounds [[STRUCT_COMPLEX]], %struct.complex* [[B]], i64 [[TMP1]], i32 1
-; CHECK-NEXT:    [[TMP11:%.*]] = load float, float* [[TMP10]], align 4
-; CHECK-NEXT:    [[TMP12:%.*]] = fmul float [[TMP5]], [[TMP9]]
-; CHECK-NEXT:    [[TMP13:%.*]] = fmul float [[TMP7]], [[TMP11]]
-; CHECK-NEXT:    [[TMP14:%.*]] = fsub float [[TMP12]], [[TMP13]]
-; CHECK-NEXT:    [[TMP15:%.*]] = fmul float [[TMP7]], [[TMP9]]
-; CHECK-NEXT:    [[TMP16:%.*]] = fmul float [[TMP5]], [[TMP11]]
-; CHECK-NEXT:    [[TMP17:%.*]] = fadd float [[TMP15]], [[TMP16]]
-; CHECK-NEXT:    [[TMP18]] = fadd float [[TMP3]], [[TMP14]]
-; CHECK-NEXT:    [[TMP19]] = fadd float [[TMP2]], [[TMP17]]
-; CHECK-NEXT:    [[TMP20]] = add nuw nsw i64 [[TMP1]], 1
-; CHECK-NEXT:    [[TMP21:%.*]] = icmp eq i64 [[TMP20]], [[TMP0]]
-; CHECK-NEXT:    br i1 [[TMP21]], label [[EXIT:%.*]], label [[LOOP]]
+; CHECK-NEXT:    [[TMP1:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[TMP25:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[TMP2:%.*]] = phi <2 x float> [ zeroinitializer, [[ENTRY]] ], [ [[TMP24:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds [[STRUCT_COMPLEX:%.*]], %struct.complex* [[A:%.*]], i64 [[TMP1]], i32 0
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds [[STRUCT_COMPLEX]], %struct.complex* [[A]], i64 [[TMP1]], i32 1
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast float* [[TMP3]] to <2 x float>*
+; CHECK-NEXT:    [[TMP6:%.*]] = load <2 x float>, <2 x float>* [[TMP5]], align 4
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds [[STRUCT_COMPLEX]], %struct.complex* [[B:%.*]], i64 [[TMP1]], i32 0
+; CHECK-NEXT:    [[TMP8:%.*]] = load float, float* [[TMP7]], align 4
+; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds [[STRUCT_COMPLEX]], %struct.complex* [[B]], i64 [[TMP1]], i32 1
+; CHECK-NEXT:    [[TMP10:%.*]] = load float, float* [[TMP9]], align 4
+; CHECK-NEXT:    [[TMP11:%.*]] = insertelement <2 x float> undef, float [[TMP8]], i32 0
+; CHECK-NEXT:    [[TMP12:%.*]] = insertelement <2 x float> [[TMP11]], float [[TMP8]], i32 1
+; CHECK-NEXT:    [[TMP13:%.*]] = fmul <2 x float> [[TMP6]], [[TMP12]]
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <2 x float> [[TMP6]], i32 1
+; CHECK-NEXT:    [[TMP15:%.*]] = insertelement <2 x float> undef, float [[TMP14]], i32 0
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <2 x float> [[TMP6]], i32 0
+; CHECK-NEXT:    [[TMP17:%.*]] = insertelement <2 x float> [[TMP15]], float [[TMP16]], i32 1
+; CHECK-NEXT:    [[TMP18:%.*]] = insertelement <2 x float> undef, float [[TMP10]], i32 0
+; CHECK-NEXT:    [[TMP19:%.*]] = insertelement <2 x float> [[TMP18]], float [[TMP10]], i32 1
+; CHECK-NEXT:    [[TMP20:%.*]] = fmul <2 x float> [[TMP17]], [[TMP19]]
+; CHECK-NEXT:    [[TMP21:%.*]] = fsub <2 x float> [[TMP13]], [[TMP20]]
+; CHECK-NEXT:    [[TMP22:%.*]] = fadd <2 x float> [[TMP13]], [[TMP20]]
+; CHECK-NEXT:    [[TMP23:%.*]] = shufflevector <2 x float> [[TMP21]], <2 x float> [[TMP22]], <2 x i32> <i32 0, i32 3>
+; CHECK-NEXT:    [[TMP24]] = fadd <2 x float> [[TMP2]], [[TMP23]]
+; CHECK-NEXT:    [[TMP25]] = add nuw nsw i64 [[TMP1]], 1
+; CHECK-NEXT:    [[TMP26:%.*]] = icmp eq i64 [[TMP25]], [[TMP0]]
+; CHECK-NEXT:    br i1 [[TMP26]], label [[EXIT:%.*]], label [[LOOP]]
 ; CHECK:       exit:
-; CHECK-NEXT:    [[TMP22:%.*]] = getelementptr inbounds [[STRUCT_COMPLEX]], %struct.complex* [[RESULT:%.*]], i32 0, i32 0
-; CHECK-NEXT:    store float [[TMP18]], float* [[TMP22]], align 4
-; CHECK-NEXT:    [[TMP23:%.*]] = getelementptr inbounds [[STRUCT_COMPLEX]], %struct.complex* [[RESULT]], i32 0, i32 1
-; CHECK-NEXT:    store float [[TMP19]], float* [[TMP23]], align 4
+; CHECK-NEXT:    [[TMP27:%.*]] = getelementptr inbounds [[STRUCT_COMPLEX]], %struct.complex* [[RESULT:%.*]], i32 0, i32 0
+; CHECK-NEXT:    [[TMP28:%.*]] = getelementptr inbounds [[STRUCT_COMPLEX]], %struct.complex* [[RESULT]], i32 0, i32 1
+; CHECK-NEXT:    [[TMP29:%.*]] = bitcast float* [[TMP27]] to <2 x float>*
+; CHECK-NEXT:    store <2 x float> [[TMP24]], <2 x float>* [[TMP29]], align 4
 ; CHECK-NEXT:    ret void
 ;
 entry:
diff --git a/test/Transforms/SLPVectorizer/X86/resched.ll b/test/Transforms/SLPVectorizer/X86/resched.ll
index b8b1ff00db41653c2162ea276bb08c6c4c5ec3d6..848b70c4d86d202881917c8a0c6976cdf3ed01c2 100644
--- a/test/Transforms/SLPVectorizer/X86/resched.ll
+++ b/test/Transforms/SLPVectorizer/X86/resched.ll
@@ -38,44 +38,47 @@ define fastcc void @_ZN12_GLOBAL__N_127PolynomialMultiplyRecognize9recognizeEv()
 ; CHECK-NEXT:    [[TMP13:%.*]] = insertelement <4 x i32> [[TMP12]], i32 [[CONV31_I]], i32 3
 ; CHECK-NEXT:    [[TMP14:%.*]] = lshr <4 x i32> [[TMP13]], <i32 9, i32 10, i32 11, i32 12>
 ; CHECK-NEXT:    [[ARRAYIDX_I_I7_12_I_I:%.*]] = getelementptr inbounds %"struct.std::array", %"struct.std::array"* undef, i64 0, i32 0, i64 12
-; CHECK-NEXT:    [[SHR_12_I_I:%.*]] = lshr i32 [[CONV31_I]], 13
 ; CHECK-NEXT:    [[ARRAYIDX_I_I7_13_I_I:%.*]] = getelementptr inbounds %"struct.std::array", %"struct.std::array"* undef, i64 0, i32 0, i64 13
-; CHECK-NEXT:    [[SHR_13_I_I:%.*]] = lshr i32 [[CONV31_I]], 14
+; CHECK-NEXT:    [[TMP15:%.*]] = insertelement <2 x i32> undef, i32 [[CONV31_I]], i32 0
+; CHECK-NEXT:    [[TMP16:%.*]] = insertelement <2 x i32> [[TMP15]], i32 [[CONV31_I]], i32 1
+; CHECK-NEXT:    [[TMP17:%.*]] = lshr <2 x i32> [[TMP16]], <i32 13, i32 14>
 ; CHECK-NEXT:    [[ARRAYIDX_I_I7_14_I_I:%.*]] = getelementptr inbounds %"struct.std::array", %"struct.std::array"* undef, i64 0, i32 0, i64 14
 ; CHECK-NEXT:    [[SHR_14_I_I:%.*]] = lshr i32 [[CONV31_I]], 15
-; CHECK-NEXT:    [[TMP15:%.*]] = insertelement <16 x i32> undef, i32 [[SUB_I]], i32 0
-; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <8 x i32> [[TMP9]], i32 0
-; CHECK-NEXT:    [[TMP17:%.*]] = insertelement <16 x i32> [[TMP15]], i32 [[TMP16]], i32 1
-; CHECK-NEXT:    [[TMP18:%.*]] = extractelement <8 x i32> [[TMP9]], i32 1
-; CHECK-NEXT:    [[TMP19:%.*]] = insertelement <16 x i32> [[TMP17]], i32 [[TMP18]], i32 2
-; CHECK-NEXT:    [[TMP20:%.*]] = extractelement <8 x i32> [[TMP9]], i32 2
-; CHECK-NEXT:    [[TMP21:%.*]] = insertelement <16 x i32> [[TMP19]], i32 [[TMP20]], i32 3
-; CHECK-NEXT:    [[TMP22:%.*]] = extractelement <8 x i32> [[TMP9]], i32 3
-; CHECK-NEXT:    [[TMP23:%.*]] = insertelement <16 x i32> [[TMP21]], i32 [[TMP22]], i32 4
-; CHECK-NEXT:    [[TMP24:%.*]] = extractelement <8 x i32> [[TMP9]], i32 4
-; CHECK-NEXT:    [[TMP25:%.*]] = insertelement <16 x i32> [[TMP23]], i32 [[TMP24]], i32 5
-; CHECK-NEXT:    [[TMP26:%.*]] = extractelement <8 x i32> [[TMP9]], i32 5
-; CHECK-NEXT:    [[TMP27:%.*]] = insertelement <16 x i32> [[TMP25]], i32 [[TMP26]], i32 6
-; CHECK-NEXT:    [[TMP28:%.*]] = extractelement <8 x i32> [[TMP9]], i32 6
-; CHECK-NEXT:    [[TMP29:%.*]] = insertelement <16 x i32> [[TMP27]], i32 [[TMP28]], i32 7
-; CHECK-NEXT:    [[TMP30:%.*]] = extractelement <8 x i32> [[TMP9]], i32 7
-; CHECK-NEXT:    [[TMP31:%.*]] = insertelement <16 x i32> [[TMP29]], i32 [[TMP30]], i32 8
-; CHECK-NEXT:    [[TMP32:%.*]] = extractelement <4 x i32> [[TMP14]], i32 0
-; CHECK-NEXT:    [[TMP33:%.*]] = insertelement <16 x i32> [[TMP31]], i32 [[TMP32]], i32 9
-; CHECK-NEXT:    [[TMP34:%.*]] = extractelement <4 x i32> [[TMP14]], i32 1
-; CHECK-NEXT:    [[TMP35:%.*]] = insertelement <16 x i32> [[TMP33]], i32 [[TMP34]], i32 10
-; CHECK-NEXT:    [[TMP36:%.*]] = extractelement <4 x i32> [[TMP14]], i32 2
-; CHECK-NEXT:    [[TMP37:%.*]] = insertelement <16 x i32> [[TMP35]], i32 [[TMP36]], i32 11
-; CHECK-NEXT:    [[TMP38:%.*]] = extractelement <4 x i32> [[TMP14]], i32 3
-; CHECK-NEXT:    [[TMP39:%.*]] = insertelement <16 x i32> [[TMP37]], i32 [[TMP38]], i32 12
-; CHECK-NEXT:    [[TMP40:%.*]] = insertelement <16 x i32> [[TMP39]], i32 [[SHR_12_I_I]], i32 13
-; CHECK-NEXT:    [[TMP41:%.*]] = insertelement <16 x i32> [[TMP40]], i32 [[SHR_13_I_I]], i32 14
-; CHECK-NEXT:    [[TMP42:%.*]] = insertelement <16 x i32> [[TMP41]], i32 [[SHR_14_I_I]], i32 15
-; CHECK-NEXT:    [[TMP43:%.*]] = trunc <16 x i32> [[TMP42]] to <16 x i8>
-; CHECK-NEXT:    [[TMP44:%.*]] = and <16 x i8> <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1>, [[TMP43]]
+; CHECK-NEXT:    [[TMP18:%.*]] = insertelement <16 x i32> undef, i32 [[SUB_I]], i32 0
+; CHECK-NEXT:    [[TMP19:%.*]] = extractelement <8 x i32> [[TMP9]], i32 0
+; CHECK-NEXT:    [[TMP20:%.*]] = insertelement <16 x i32> [[TMP18]], i32 [[TMP19]], i32 1
+; CHECK-NEXT:    [[TMP21:%.*]] = extractelement <8 x i32> [[TMP9]], i32 1
+; CHECK-NEXT:    [[TMP22:%.*]] = insertelement <16 x i32> [[TMP20]], i32 [[TMP21]], i32 2
+; CHECK-NEXT:    [[TMP23:%.*]] = extractelement <8 x i32> [[TMP9]], i32 2
+; CHECK-NEXT:    [[TMP24:%.*]] = insertelement <16 x i32> [[TMP22]], i32 [[TMP23]], i32 3
+; CHECK-NEXT:    [[TMP25:%.*]] = extractelement <8 x i32> [[TMP9]], i32 3
+; CHECK-NEXT:    [[TMP26:%.*]] = insertelement <16 x i32> [[TMP24]], i32 [[TMP25]], i32 4
+; CHECK-NEXT:    [[TMP27:%.*]] = extractelement <8 x i32> [[TMP9]], i32 4
+; CHECK-NEXT:    [[TMP28:%.*]] = insertelement <16 x i32> [[TMP26]], i32 [[TMP27]], i32 5
+; CHECK-NEXT:    [[TMP29:%.*]] = extractelement <8 x i32> [[TMP9]], i32 5
+; CHECK-NEXT:    [[TMP30:%.*]] = insertelement <16 x i32> [[TMP28]], i32 [[TMP29]], i32 6
+; CHECK-NEXT:    [[TMP31:%.*]] = extractelement <8 x i32> [[TMP9]], i32 6
+; CHECK-NEXT:    [[TMP32:%.*]] = insertelement <16 x i32> [[TMP30]], i32 [[TMP31]], i32 7
+; CHECK-NEXT:    [[TMP33:%.*]] = extractelement <8 x i32> [[TMP9]], i32 7
+; CHECK-NEXT:    [[TMP34:%.*]] = insertelement <16 x i32> [[TMP32]], i32 [[TMP33]], i32 8
+; CHECK-NEXT:    [[TMP35:%.*]] = extractelement <4 x i32> [[TMP14]], i32 0
+; CHECK-NEXT:    [[TMP36:%.*]] = insertelement <16 x i32> [[TMP34]], i32 [[TMP35]], i32 9
+; CHECK-NEXT:    [[TMP37:%.*]] = extractelement <4 x i32> [[TMP14]], i32 1
+; CHECK-NEXT:    [[TMP38:%.*]] = insertelement <16 x i32> [[TMP36]], i32 [[TMP37]], i32 10
+; CHECK-NEXT:    [[TMP39:%.*]] = extractelement <4 x i32> [[TMP14]], i32 2
+; CHECK-NEXT:    [[TMP40:%.*]] = insertelement <16 x i32> [[TMP38]], i32 [[TMP39]], i32 11
+; CHECK-NEXT:    [[TMP41:%.*]] = extractelement <4 x i32> [[TMP14]], i32 3
+; CHECK-NEXT:    [[TMP42:%.*]] = insertelement <16 x i32> [[TMP40]], i32 [[TMP41]], i32 12
+; CHECK-NEXT:    [[TMP43:%.*]] = extractelement <2 x i32> [[TMP17]], i32 0
+; CHECK-NEXT:    [[TMP44:%.*]] = insertelement <16 x i32> [[TMP42]], i32 [[TMP43]], i32 13
+; CHECK-NEXT:    [[TMP45:%.*]] = extractelement <2 x i32> [[TMP17]], i32 1
+; CHECK-NEXT:    [[TMP46:%.*]] = insertelement <16 x i32> [[TMP44]], i32 [[TMP45]], i32 14
+; CHECK-NEXT:    [[TMP47:%.*]] = insertelement <16 x i32> [[TMP46]], i32 [[SHR_14_I_I]], i32 15
+; CHECK-NEXT:    [[TMP48:%.*]] = trunc <16 x i32> [[TMP47]] to <16 x i8>
+; CHECK-NEXT:    [[TMP49:%.*]] = and <16 x i8> <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1>, [[TMP48]]
 ; CHECK-NEXT:    [[ARRAYIDX_I_I7_15_I_I:%.*]] = getelementptr inbounds %"struct.std::array", %"struct.std::array"* undef, i64 0, i32 0, i64 15
-; CHECK-NEXT:    [[TMP45:%.*]] = bitcast i8* [[TMP0]] to <16 x i8>*
-; CHECK-NEXT:    store <16 x i8> [[TMP44]], <16 x i8>* [[TMP45]], align 1
+; CHECK-NEXT:    [[TMP50:%.*]] = bitcast i8* [[TMP0]] to <16 x i8>*
+; CHECK-NEXT:    store <16 x i8> [[TMP49]], <16 x i8>* [[TMP50]], align 1
 ; CHECK-NEXT:    unreachable
 ; CHECK:       if.end50.i:
 ; CHECK-NEXT:    ret void
diff --git a/test/Transforms/SLPVectorizer/X86/rgb_phi.ll b/test/Transforms/SLPVectorizer/X86/rgb_phi.ll
index 0bdb7dab172684ae15d69427bf4b819e7e9cd325..dcb730ff2fd2a1ff3dc9dc159e24f84cfe17cd0c 100644
--- a/test/Transforms/SLPVectorizer/X86/rgb_phi.ll
+++ b/test/Transforms/SLPVectorizer/X86/rgb_phi.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -basicaa -slp-vectorizer -dce -S -mtriple=i386-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
 
 target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:128:128-n8:16:32-S128"
@@ -19,19 +20,44 @@ target triple = "i386-apple-macosx10.9.0"
 ;   return R+G+B;
 ; }
 
-;CHECK-LABEL: @foo(
-;CHECK: br
-;CHECK-NOT: phi <3 x float>
-;CHECK-NOT: fmul <3 x float>
-;CHECK-NOT: fadd <3 x float>
-; At the moment we don't sink extractelements.
-;CHECK: br
-;CHECK-NOT: extractelement
-;CHECK-NOT: extractelement
-;CHECK-NOT: extractelement
-;CHECK: ret
-
 define float @foo(float* nocapture readonly %A) {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load float, float* [[A:%.*]], align 4
+; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds float, float* [[A]], i64 1
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast float* [[ARRAYIDX1]] to <2 x float>*
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x float>, <2 x float>* [[TMP1]], align 4
+; CHECK-NEXT:    [[REORDER_SHUFFLE:%.*]] = shufflevector <2 x float> [[TMP2]], <2 x float> undef, <2 x i32> <i32 1, i32 0>
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.body:
+; CHECK-NEXT:    [[TMP3:%.*]] = phi float [ [[TMP0]], [[ENTRY:%.*]] ], [ [[DOTPRE:%.*]], [[FOR_BODY_FOR_BODY_CRIT_EDGE:%.*]] ]
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY_FOR_BODY_CRIT_EDGE]] ]
+; CHECK-NEXT:    [[R_030:%.*]] = phi float [ [[TMP0]], [[ENTRY]] ], [ [[ADD4:%.*]], [[FOR_BODY_FOR_BODY_CRIT_EDGE]] ]
+; CHECK-NEXT:    [[TMP4:%.*]] = phi <2 x float> [ [[REORDER_SHUFFLE]], [[ENTRY]] ], [ [[TMP9:%.*]], [[FOR_BODY_FOR_BODY_CRIT_EDGE]] ]
+; CHECK-NEXT:    [[MUL:%.*]] = fmul float [[TMP3]], 7.000000e+00
+; CHECK-NEXT:    [[ADD4]] = fadd float [[R_030]], [[MUL]]
+; CHECK-NEXT:    [[TMP5:%.*]] = add nsw i64 [[INDVARS_IV]], 1
+; CHECK-NEXT:    [[ARRAYIDX7:%.*]] = getelementptr inbounds float, float* [[A]], i64 [[TMP5]]
+; CHECK-NEXT:    [[TMP6:%.*]] = bitcast float* [[ARRAYIDX7]] to <2 x float>*
+; CHECK-NEXT:    [[TMP7:%.*]] = load <2 x float>, <2 x float>* [[TMP6]], align 4
+; CHECK-NEXT:    [[REORDER_SHUFFLE1:%.*]] = shufflevector <2 x float> [[TMP7]], <2 x float> undef, <2 x i32> <i32 1, i32 0>
+; CHECK-NEXT:    [[TMP8:%.*]] = fmul <2 x float> <float 9.000000e+00, float 8.000000e+00>, [[REORDER_SHUFFLE1]]
+; CHECK-NEXT:    [[TMP9]] = fadd <2 x float> [[TMP4]], [[TMP8]]
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add i64 [[INDVARS_IV]], 3
+; CHECK-NEXT:    [[TMP10:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP10]], 121
+; CHECK-NEXT:    br i1 [[CMP]], label [[FOR_BODY_FOR_BODY_CRIT_EDGE]], label [[FOR_END:%.*]]
+; CHECK:       for.body.for.body_crit_edge:
+; CHECK-NEXT:    [[ARRAYIDX3_PHI_TRANS_INSERT:%.*]] = getelementptr inbounds float, float* [[A]], i64 [[INDVARS_IV_NEXT]]
+; CHECK-NEXT:    [[DOTPRE]] = load float, float* [[ARRAYIDX3_PHI_TRANS_INSERT]], align 4
+; CHECK-NEXT:    br label [[FOR_BODY]]
+; CHECK:       for.end:
+; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <2 x float> [[TMP9]], i32 1
+; CHECK-NEXT:    [[ADD16:%.*]] = fadd float [[ADD4]], [[TMP11]]
+; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <2 x float> [[TMP9]], i32 0
+; CHECK-NEXT:    [[ADD17:%.*]] = fadd float [[ADD16]], [[TMP12]]
+; CHECK-NEXT:    ret float [[ADD17]]
+;
 entry:
   %0 = load float, float* %A, align 4
   %arrayidx1 = getelementptr inbounds float, float* %A, i64 1
diff --git a/test/Transforms/SLPVectorizer/X86/saxpy.ll b/test/Transforms/SLPVectorizer/X86/saxpy.ll
index a9ca093c0cdf8cc4ca8dfbec595d08d73c398b2c..944dfac2c9648330c404e879931d7c83de080518 100644
--- a/test/Transforms/SLPVectorizer/X86/saxpy.ll
+++ b/test/Transforms/SLPVectorizer/X86/saxpy.ll
@@ -1,14 +1,28 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -basicaa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.8.0"
 
 ; SLP vectorization example from http://cs.stanford.edu/people/eschkufz/research/asplos291-schkufza.pdf
-;CHECK: SAXPY
-;CHECK: mul nsw <4 x i32>
-;CHECK: ret
-
 define void @SAXPY(i32* noalias nocapture %x, i32* noalias nocapture %y, i32 %a, i64 %i) {
+; CHECK-LABEL: @SAXPY(
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, i32* [[X:%.*]], i64 [[I:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, i32* [[Y:%.*]], i64 [[I]]
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i32* [[TMP1]] to <4 x i32>*
+; CHECK-NEXT:    [[TMP4:%.*]] = load <4 x i32>, <4 x i32>* [[TMP3]], align 4
+; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <4 x i32> undef, i32 [[A:%.*]], i32 0
+; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <4 x i32> [[TMP5]], i32 [[A]], i32 1
+; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <4 x i32> [[TMP6]], i32 [[A]], i32 2
+; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <4 x i32> [[TMP7]], i32 [[A]], i32 3
+; CHECK-NEXT:    [[TMP9:%.*]] = mul nsw <4 x i32> [[TMP8]], [[TMP4]]
+; CHECK-NEXT:    [[TMP10:%.*]] = bitcast i32* [[TMP2]] to <4 x i32>*
+; CHECK-NEXT:    [[TMP11:%.*]] = load <4 x i32>, <4 x i32>* [[TMP10]], align 4
+; CHECK-NEXT:    [[TMP12:%.*]] = add nsw <4 x i32> [[TMP9]], [[TMP11]]
+; CHECK-NEXT:    [[TMP13:%.*]] = bitcast i32* [[TMP1]] to <4 x i32>*
+; CHECK-NEXT:    store <4 x i32> [[TMP12]], <4 x i32>* [[TMP13]], align 4
+; CHECK-NEXT:    ret void
+;
   %1 = getelementptr inbounds i32, i32* %x, i64 %i
   %2 = load i32, i32* %1, align 4
   %3 = mul nsw i32 %2, %a
@@ -45,6 +59,17 @@ define void @SAXPY(i32* noalias nocapture %x, i32* noalias nocapture %y, i32 %a,
 
 ; Make sure we don't crash on this one.
 define void @SAXPY_crash(i32* noalias nocapture %x, i32* noalias nocapture %y, i64 %i) {
+; CHECK-LABEL: @SAXPY_crash(
+; CHECK-NEXT:    [[TMP1:%.*]] = add i64 [[I:%.*]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, i32* [[X:%.*]], i64 [[TMP1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, i32* [[Y:%.*]], i64 [[TMP1]]
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i32* [[TMP3]] to <2 x i32>*
+; CHECK-NEXT:    [[TMP5:%.*]] = load <2 x i32>, <2 x i32>* [[TMP4]], align 4
+; CHECK-NEXT:    [[TMP6:%.*]] = add nsw <2 x i32> undef, [[TMP5]]
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i32* [[TMP2]] to <2 x i32>*
+; CHECK-NEXT:    store <2 x i32> [[TMP6]], <2 x i32>* [[TMP7]], align 4
+; CHECK-NEXT:    ret void
+;
   %1 = add i64 %i, 1
   %2 = getelementptr inbounds i32, i32* %x, i64 %1
   %3 = getelementptr inbounds i32, i32* %y, i64 %1
diff --git a/test/Transforms/SLPVectorizer/X86/schedule-bundle.ll b/test/Transforms/SLPVectorizer/X86/schedule-bundle.ll
index 3abde37048fdcabecba7f5adfacd86351457e766..60b2aa5ba73b66257eb0b929e61af7f5a4b1f7aa 100644
--- a/test/Transforms/SLPVectorizer/X86/schedule-bundle.ll
+++ b/test/Transforms/SLPVectorizer/X86/schedule-bundle.ll
@@ -14,14 +14,10 @@ define i32 @slp_schedule_bundle() local_unnamed_addr #0 {
 ; CHECK-NEXT:    [[TMP1:%.*]] = lshr <4 x i32> [[TMP0]], <i32 31, i32 31, i32 31, i32 31>
 ; CHECK-NEXT:    [[TMP2:%.*]] = xor <4 x i32> <i32 1, i32 1, i32 1, i32 1>, [[TMP1]]
 ; CHECK-NEXT:    store <4 x i32> [[TMP2]], <4 x i32>* bitcast ([1 x i32]* @a to <4 x i32>*), align 4
-; CHECK-NEXT:    [[TMP3:%.*]] = load i32, i32* getelementptr ([1 x i32], [1 x i32]* @b, i64 4, i64 0), align 4
-; CHECK-NEXT:    [[DOTLOBIT_4:%.*]] = lshr i32 [[TMP3]], 31
-; CHECK-NEXT:    [[DOTLOBIT_NOT_4:%.*]] = xor i32 [[DOTLOBIT_4]], 1
-; CHECK-NEXT:    store i32 [[DOTLOBIT_NOT_4]], i32* getelementptr ([1 x i32], [1 x i32]* @a, i64 4, i64 0), align 4
-; CHECK-NEXT:    [[TMP4:%.*]] = load i32, i32* getelementptr ([1 x i32], [1 x i32]* @b, i64 5, i64 0), align 4
-; CHECK-NEXT:    [[DOTLOBIT_5:%.*]] = lshr i32 [[TMP4]], 31
-; CHECK-NEXT:    [[DOTLOBIT_NOT_5:%.*]] = xor i32 [[DOTLOBIT_5]], 1
-; CHECK-NEXT:    store i32 [[DOTLOBIT_NOT_5]], i32* getelementptr ([1 x i32], [1 x i32]* @a, i64 5, i64 0), align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i32>, <2 x i32>* bitcast (i32* getelementptr ([1 x i32], [1 x i32]* @b, i64 4, i64 0) to <2 x i32>*), align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = lshr <2 x i32> [[TMP3]], <i32 31, i32 31>
+; CHECK-NEXT:    [[TMP5:%.*]] = xor <2 x i32> <i32 1, i32 1>, [[TMP4]]
+; CHECK-NEXT:    store <2 x i32> [[TMP5]], <2 x i32>* bitcast (i32* getelementptr ([1 x i32], [1 x i32]* @a, i64 4, i64 0) to <2 x i32>*), align 4
 ; CHECK-NEXT:    ret i32 undef
 ;
 entry:
diff --git a/test/Transforms/SLPVectorizer/X86/schedule_budget.ll b/test/Transforms/SLPVectorizer/X86/schedule_budget.ll
index 2cb2373381c7f79a5ba502d6e143da1ee8695e1e..0cd08f0e8c338eca210a309d93d1bbfcb3575da9 100644
--- a/test/Transforms/SLPVectorizer/X86/schedule_budget.ll
+++ b/test/Transforms/SLPVectorizer/X86/schedule_budget.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -basicaa -slp-vectorizer -S  -slp-schedule-budget=16 -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
 
 target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
@@ -8,19 +9,63 @@ target triple = "x86_64-apple-macosx10.9.0"
 
 declare void @unknown()
 
-; CHECK-LABEL: @test
-; CHECK: load float
-; CHECK: load float
-; CHECK: load float
-; CHECK: load float
-; CHECK: call void @unknown
-; CHECK: store float
-; CHECK: store float
-; CHECK: store float
-; CHECK: store float
-; CHECK: load <4 x float>
-; CHECK: store <4 x float>
 define void @test(float * %a, float * %b, float * %c, float * %d) {
+; CHECK-LABEL: @test(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[L0:%.*]] = load float, float* [[A:%.*]]
+; CHECK-NEXT:    [[A1:%.*]] = getelementptr inbounds float, float* [[A]], i64 1
+; CHECK-NEXT:    [[L1:%.*]] = load float, float* [[A1]]
+; CHECK-NEXT:    [[A2:%.*]] = getelementptr inbounds float, float* [[A]], i64 2
+; CHECK-NEXT:    [[L2:%.*]] = load float, float* [[A2]]
+; CHECK-NEXT:    [[A3:%.*]] = getelementptr inbounds float, float* [[A]], i64 3
+; CHECK-NEXT:    [[L3:%.*]] = load float, float* [[A3]]
+; CHECK-NEXT:    call void @unknown()
+; CHECK-NEXT:    call void @unknown()
+; CHECK-NEXT:    call void @unknown()
+; CHECK-NEXT:    call void @unknown()
+; CHECK-NEXT:    call void @unknown()
+; CHECK-NEXT:    call void @unknown()
+; CHECK-NEXT:    call void @unknown()
+; CHECK-NEXT:    call void @unknown()
+; CHECK-NEXT:    call void @unknown()
+; CHECK-NEXT:    call void @unknown()
+; CHECK-NEXT:    call void @unknown()
+; CHECK-NEXT:    call void @unknown()
+; CHECK-NEXT:    call void @unknown()
+; CHECK-NEXT:    call void @unknown()
+; CHECK-NEXT:    call void @unknown()
+; CHECK-NEXT:    call void @unknown()
+; CHECK-NEXT:    call void @unknown()
+; CHECK-NEXT:    call void @unknown()
+; CHECK-NEXT:    call void @unknown()
+; CHECK-NEXT:    call void @unknown()
+; CHECK-NEXT:    call void @unknown()
+; CHECK-NEXT:    call void @unknown()
+; CHECK-NEXT:    call void @unknown()
+; CHECK-NEXT:    call void @unknown()
+; CHECK-NEXT:    call void @unknown()
+; CHECK-NEXT:    call void @unknown()
+; CHECK-NEXT:    call void @unknown()
+; CHECK-NEXT:    call void @unknown()
+; CHECK-NEXT:    store float [[L0]], float* [[B:%.*]]
+; CHECK-NEXT:    [[B1:%.*]] = getelementptr inbounds float, float* [[B]], i64 1
+; CHECK-NEXT:    store float [[L1]], float* [[B1]]
+; CHECK-NEXT:    [[B2:%.*]] = getelementptr inbounds float, float* [[B]], i64 2
+; CHECK-NEXT:    store float [[L2]], float* [[B2]]
+; CHECK-NEXT:    [[B3:%.*]] = getelementptr inbounds float, float* [[B]], i64 3
+; CHECK-NEXT:    store float [[L3]], float* [[B3]]
+; CHECK-NEXT:    [[C1:%.*]] = getelementptr inbounds float, float* [[C:%.*]], i64 1
+; CHECK-NEXT:    [[C2:%.*]] = getelementptr inbounds float, float* [[C]], i64 2
+; CHECK-NEXT:    [[C3:%.*]] = getelementptr inbounds float, float* [[C]], i64 3
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast float* [[C]] to <4 x float>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x float>, <4 x float>* [[TMP0]], align 4
+; CHECK-NEXT:    [[D1:%.*]] = getelementptr inbounds float, float* [[D:%.*]], i64 1
+; CHECK-NEXT:    [[D2:%.*]] = getelementptr inbounds float, float* [[D]], i64 2
+; CHECK-NEXT:    [[D3:%.*]] = getelementptr inbounds float, float* [[D]], i64 3
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast float* [[D]] to <4 x float>*
+; CHECK-NEXT:    store <4 x float> [[TMP1]], <4 x float>* [[TMP2]], align 4
+; CHECK-NEXT:    ret void
+;
 entry:
   ; Don't vectorize these loads.
   %l0 = load float, float* %a
diff --git a/test/Transforms/SLPVectorizer/X86/sext.ll b/test/Transforms/SLPVectorizer/X86/sext.ll
index 75406c927d913de8b819b2cfd6992ae17f31c3db..e940bb56f8ec7ffd888349fde973e7579117ef27 100644
--- a/test/Transforms/SLPVectorizer/X86/sext.ll
+++ b/test/Transforms/SLPVectorizer/X86/sext.ll
@@ -808,18 +808,20 @@ define <4 x i64> @loadext_4i32_to_4i64(i32* %p0) {
 ; SSE2-NEXT:    [[P1:%.*]] = getelementptr inbounds i32, i32* [[P0:%.*]], i64 1
 ; SSE2-NEXT:    [[P2:%.*]] = getelementptr inbounds i32, i32* [[P0]], i64 2
 ; SSE2-NEXT:    [[P3:%.*]] = getelementptr inbounds i32, i32* [[P0]], i64 3
-; SSE2-NEXT:    [[I0:%.*]] = load i32, i32* [[P0]], align 1
-; SSE2-NEXT:    [[I1:%.*]] = load i32, i32* [[P1]], align 1
-; SSE2-NEXT:    [[I2:%.*]] = load i32, i32* [[P2]], align 1
-; SSE2-NEXT:    [[I3:%.*]] = load i32, i32* [[P3]], align 1
-; SSE2-NEXT:    [[X0:%.*]] = sext i32 [[I0]] to i64
-; SSE2-NEXT:    [[X1:%.*]] = sext i32 [[I1]] to i64
-; SSE2-NEXT:    [[X2:%.*]] = sext i32 [[I2]] to i64
-; SSE2-NEXT:    [[X3:%.*]] = sext i32 [[I3]] to i64
-; SSE2-NEXT:    [[V0:%.*]] = insertelement <4 x i64> undef, i64 [[X0]], i32 0
-; SSE2-NEXT:    [[V1:%.*]] = insertelement <4 x i64> [[V0]], i64 [[X1]], i32 1
-; SSE2-NEXT:    [[V2:%.*]] = insertelement <4 x i64> [[V1]], i64 [[X2]], i32 2
-; SSE2-NEXT:    [[V3:%.*]] = insertelement <4 x i64> [[V2]], i64 [[X3]], i32 3
+; SSE2-NEXT:    [[TMP1:%.*]] = bitcast i32* [[P0]] to <2 x i32>*
+; SSE2-NEXT:    [[TMP2:%.*]] = load <2 x i32>, <2 x i32>* [[TMP1]], align 1
+; SSE2-NEXT:    [[TMP3:%.*]] = bitcast i32* [[P2]] to <2 x i32>*
+; SSE2-NEXT:    [[TMP4:%.*]] = load <2 x i32>, <2 x i32>* [[TMP3]], align 1
+; SSE2-NEXT:    [[TMP5:%.*]] = sext <2 x i32> [[TMP2]] to <2 x i64>
+; SSE2-NEXT:    [[TMP6:%.*]] = sext <2 x i32> [[TMP4]] to <2 x i64>
+; SSE2-NEXT:    [[TMP7:%.*]] = extractelement <2 x i64> [[TMP5]], i32 0
+; SSE2-NEXT:    [[V0:%.*]] = insertelement <4 x i64> undef, i64 [[TMP7]], i32 0
+; SSE2-NEXT:    [[TMP8:%.*]] = extractelement <2 x i64> [[TMP5]], i32 1
+; SSE2-NEXT:    [[V1:%.*]] = insertelement <4 x i64> [[V0]], i64 [[TMP8]], i32 1
+; SSE2-NEXT:    [[TMP9:%.*]] = extractelement <2 x i64> [[TMP6]], i32 0
+; SSE2-NEXT:    [[V2:%.*]] = insertelement <4 x i64> [[V1]], i64 [[TMP9]], i32 2
+; SSE2-NEXT:    [[TMP10:%.*]] = extractelement <2 x i64> [[TMP6]], i32 1
+; SSE2-NEXT:    [[V3:%.*]] = insertelement <4 x i64> [[V2]], i64 [[TMP10]], i32 3
 ; SSE2-NEXT:    ret <4 x i64> [[V3]]
 ;
 ; SLM-LABEL: @loadext_4i32_to_4i64(
@@ -845,17 +847,18 @@ define <4 x i64> @loadext_4i32_to_4i64(i32* %p0) {
 ; AVX1-NEXT:    [[P3:%.*]] = getelementptr inbounds i32, i32* [[P0]], i64 3
 ; AVX1-NEXT:    [[TMP1:%.*]] = bitcast i32* [[P0]] to <2 x i32>*
 ; AVX1-NEXT:    [[TMP2:%.*]] = load <2 x i32>, <2 x i32>* [[TMP1]], align 1
-; AVX1-NEXT:    [[I2:%.*]] = load i32, i32* [[P2]], align 1
-; AVX1-NEXT:    [[I3:%.*]] = load i32, i32* [[P3]], align 1
-; AVX1-NEXT:    [[TMP3:%.*]] = sext <2 x i32> [[TMP2]] to <2 x i64>
-; AVX1-NEXT:    [[X2:%.*]] = sext i32 [[I2]] to i64
-; AVX1-NEXT:    [[X3:%.*]] = sext i32 [[I3]] to i64
-; AVX1-NEXT:    [[TMP4:%.*]] = extractelement <2 x i64> [[TMP3]], i32 0
-; AVX1-NEXT:    [[V0:%.*]] = insertelement <4 x i64> undef, i64 [[TMP4]], i32 0
-; AVX1-NEXT:    [[TMP5:%.*]] = extractelement <2 x i64> [[TMP3]], i32 1
-; AVX1-NEXT:    [[V1:%.*]] = insertelement <4 x i64> [[V0]], i64 [[TMP5]], i32 1
-; AVX1-NEXT:    [[V2:%.*]] = insertelement <4 x i64> [[V1]], i64 [[X2]], i32 2
-; AVX1-NEXT:    [[V3:%.*]] = insertelement <4 x i64> [[V2]], i64 [[X3]], i32 3
+; AVX1-NEXT:    [[TMP3:%.*]] = bitcast i32* [[P2]] to <2 x i32>*
+; AVX1-NEXT:    [[TMP4:%.*]] = load <2 x i32>, <2 x i32>* [[TMP3]], align 1
+; AVX1-NEXT:    [[TMP5:%.*]] = sext <2 x i32> [[TMP2]] to <2 x i64>
+; AVX1-NEXT:    [[TMP6:%.*]] = sext <2 x i32> [[TMP4]] to <2 x i64>
+; AVX1-NEXT:    [[TMP7:%.*]] = extractelement <2 x i64> [[TMP5]], i32 0
+; AVX1-NEXT:    [[V0:%.*]] = insertelement <4 x i64> undef, i64 [[TMP7]], i32 0
+; AVX1-NEXT:    [[TMP8:%.*]] = extractelement <2 x i64> [[TMP5]], i32 1
+; AVX1-NEXT:    [[V1:%.*]] = insertelement <4 x i64> [[V0]], i64 [[TMP8]], i32 1
+; AVX1-NEXT:    [[TMP9:%.*]] = extractelement <2 x i64> [[TMP6]], i32 0
+; AVX1-NEXT:    [[V2:%.*]] = insertelement <4 x i64> [[V1]], i64 [[TMP9]], i32 2
+; AVX1-NEXT:    [[TMP10:%.*]] = extractelement <2 x i64> [[TMP6]], i32 1
+; AVX1-NEXT:    [[V3:%.*]] = insertelement <4 x i64> [[V2]], i64 [[TMP10]], i32 3
 ; AVX1-NEXT:    ret <4 x i64> [[V3]]
 ;
 ; AVX2-LABEL: @loadext_4i32_to_4i64(
diff --git a/test/Transforms/SLPVectorizer/X86/shift-lshr.ll b/test/Transforms/SLPVectorizer/X86/shift-lshr.ll
index 237673da5a9c59ce1656cf654144bd76e9a815cb..ceabea78fd8ddc13d1e5f3b8404682a28a4df81b 100644
--- a/test/Transforms/SLPVectorizer/X86/shift-lshr.ll
+++ b/test/Transforms/SLPVectorizer/X86/shift-lshr.ll
@@ -125,70 +125,38 @@ define void @lshr_v8i64() {
 
 define void @lshr_v16i32() {
 ; SSE-LABEL: @lshr_v16i32(
-; SSE-NEXT:    [[A0:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 0), align 4
-; SSE-NEXT:    [[A1:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 1), align 4
-; SSE-NEXT:    [[A2:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 2), align 4
-; SSE-NEXT:    [[A3:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 3), align 4
-; SSE-NEXT:    [[A4:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 4), align 4
-; SSE-NEXT:    [[A5:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 5), align 4
-; SSE-NEXT:    [[A6:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 6), align 4
-; SSE-NEXT:    [[A7:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 7), align 4
-; SSE-NEXT:    [[A8:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 8), align 4
-; SSE-NEXT:    [[A9:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 9), align 4
-; SSE-NEXT:    [[A10:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 10), align 4
-; SSE-NEXT:    [[A11:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 11), align 4
-; SSE-NEXT:    [[A12:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 12), align 4
-; SSE-NEXT:    [[A13:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 13), align 4
-; SSE-NEXT:    [[A14:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 14), align 4
-; SSE-NEXT:    [[A15:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 15), align 4
-; SSE-NEXT:    [[B0:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 0), align 4
-; SSE-NEXT:    [[B1:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 1), align 4
-; SSE-NEXT:    [[B2:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 2), align 4
-; SSE-NEXT:    [[B3:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 3), align 4
-; SSE-NEXT:    [[B4:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 4), align 4
-; SSE-NEXT:    [[B5:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 5), align 4
-; SSE-NEXT:    [[B6:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 6), align 4
-; SSE-NEXT:    [[B7:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 7), align 4
-; SSE-NEXT:    [[B8:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 8), align 4
-; SSE-NEXT:    [[B9:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 9), align 4
-; SSE-NEXT:    [[B10:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 10), align 4
-; SSE-NEXT:    [[B11:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 11), align 4
-; SSE-NEXT:    [[B12:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 12), align 4
-; SSE-NEXT:    [[B13:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 13), align 4
-; SSE-NEXT:    [[B14:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 14), align 4
-; SSE-NEXT:    [[B15:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 15), align 4
-; SSE-NEXT:    [[R0:%.*]] = lshr i32 [[A0]], [[B0]]
-; SSE-NEXT:    [[R1:%.*]] = lshr i32 [[A1]], [[B1]]
-; SSE-NEXT:    [[R2:%.*]] = lshr i32 [[A2]], [[B2]]
-; SSE-NEXT:    [[R3:%.*]] = lshr i32 [[A3]], [[B3]]
-; SSE-NEXT:    [[R4:%.*]] = lshr i32 [[A4]], [[B4]]
-; SSE-NEXT:    [[R5:%.*]] = lshr i32 [[A5]], [[B5]]
-; SSE-NEXT:    [[R6:%.*]] = lshr i32 [[A6]], [[B6]]
-; SSE-NEXT:    [[R7:%.*]] = lshr i32 [[A7]], [[B7]]
-; SSE-NEXT:    [[R8:%.*]] = lshr i32 [[A8]], [[B8]]
-; SSE-NEXT:    [[R9:%.*]] = lshr i32 [[A9]], [[B9]]
-; SSE-NEXT:    [[R10:%.*]] = lshr i32 [[A10]], [[B10]]
-; SSE-NEXT:    [[R11:%.*]] = lshr i32 [[A11]], [[B11]]
-; SSE-NEXT:    [[R12:%.*]] = lshr i32 [[A12]], [[B12]]
-; SSE-NEXT:    [[R13:%.*]] = lshr i32 [[A13]], [[B13]]
-; SSE-NEXT:    [[R14:%.*]] = lshr i32 [[A14]], [[B14]]
-; SSE-NEXT:    [[R15:%.*]] = lshr i32 [[A15]], [[B15]]
-; SSE-NEXT:    store i32 [[R0]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 0), align 4
-; SSE-NEXT:    store i32 [[R1]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 1), align 4
-; SSE-NEXT:    store i32 [[R2]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 2), align 4
-; SSE-NEXT:    store i32 [[R3]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 3), align 4
-; SSE-NEXT:    store i32 [[R4]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 4), align 4
-; SSE-NEXT:    store i32 [[R5]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 5), align 4
-; SSE-NEXT:    store i32 [[R6]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 6), align 4
-; SSE-NEXT:    store i32 [[R7]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 7), align 4
-; SSE-NEXT:    store i32 [[R8]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 8), align 4
-; SSE-NEXT:    store i32 [[R9]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 9), align 4
-; SSE-NEXT:    store i32 [[R10]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 10), align 4
-; SSE-NEXT:    store i32 [[R11]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 11), align 4
-; SSE-NEXT:    store i32 [[R12]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 12), align 4
-; SSE-NEXT:    store i32 [[R13]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 13), align 4
-; SSE-NEXT:    store i32 [[R14]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 14), align 4
-; SSE-NEXT:    store i32 [[R15]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 15), align 4
+; SSE-NEXT:    [[TMP1:%.*]] = load <2 x i32>, <2 x i32>* bitcast ([16 x i32]* @a32 to <2 x i32>*), align 4
+; SSE-NEXT:    [[TMP2:%.*]] = load <2 x i32>, <2 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 2) to <2 x i32>*), align 4
+; SSE-NEXT:    [[TMP3:%.*]] = load <2 x i32>, <2 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 4) to <2 x i32>*), align 4
+; SSE-NEXT:    [[TMP4:%.*]] = load <2 x i32>, <2 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 6) to <2 x i32>*), align 4
+; SSE-NEXT:    [[TMP5:%.*]] = load <2 x i32>, <2 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 8) to <2 x i32>*), align 4
+; SSE-NEXT:    [[TMP6:%.*]] = load <2 x i32>, <2 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 10) to <2 x i32>*), align 4
+; SSE-NEXT:    [[TMP7:%.*]] = load <2 x i32>, <2 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 12) to <2 x i32>*), align 4
+; SSE-NEXT:    [[TMP8:%.*]] = load <2 x i32>, <2 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 14) to <2 x i32>*), align 4
+; SSE-NEXT:    [[TMP9:%.*]] = load <2 x i32>, <2 x i32>* bitcast ([16 x i32]* @b32 to <2 x i32>*), align 4
+; SSE-NEXT:    [[TMP10:%.*]] = load <2 x i32>, <2 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 2) to <2 x i32>*), align 4
+; SSE-NEXT:    [[TMP11:%.*]] = load <2 x i32>, <2 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 4) to <2 x i32>*), align 4
+; SSE-NEXT:    [[TMP12:%.*]] = load <2 x i32>, <2 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 6) to <2 x i32>*), align 4
+; SSE-NEXT:    [[TMP13:%.*]] = load <2 x i32>, <2 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 8) to <2 x i32>*), align 4
+; SSE-NEXT:    [[TMP14:%.*]] = load <2 x i32>, <2 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 10) to <2 x i32>*), align 4
+; SSE-NEXT:    [[TMP15:%.*]] = load <2 x i32>, <2 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 12) to <2 x i32>*), align 4
+; SSE-NEXT:    [[TMP16:%.*]] = load <2 x i32>, <2 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 14) to <2 x i32>*), align 4
+; SSE-NEXT:    [[TMP17:%.*]] = lshr <2 x i32> [[TMP1]], [[TMP9]]
+; SSE-NEXT:    [[TMP18:%.*]] = lshr <2 x i32> [[TMP2]], [[TMP10]]
+; SSE-NEXT:    [[TMP19:%.*]] = lshr <2 x i32> [[TMP3]], [[TMP11]]
+; SSE-NEXT:    [[TMP20:%.*]] = lshr <2 x i32> [[TMP4]], [[TMP12]]
+; SSE-NEXT:    [[TMP21:%.*]] = lshr <2 x i32> [[TMP5]], [[TMP13]]
+; SSE-NEXT:    [[TMP22:%.*]] = lshr <2 x i32> [[TMP6]], [[TMP14]]
+; SSE-NEXT:    [[TMP23:%.*]] = lshr <2 x i32> [[TMP7]], [[TMP15]]
+; SSE-NEXT:    [[TMP24:%.*]] = lshr <2 x i32> [[TMP8]], [[TMP16]]
+; SSE-NEXT:    store <2 x i32> [[TMP17]], <2 x i32>* bitcast ([16 x i32]* @c32 to <2 x i32>*), align 4
+; SSE-NEXT:    store <2 x i32> [[TMP18]], <2 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 2) to <2 x i32>*), align 4
+; SSE-NEXT:    store <2 x i32> [[TMP19]], <2 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 4) to <2 x i32>*), align 4
+; SSE-NEXT:    store <2 x i32> [[TMP20]], <2 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 6) to <2 x i32>*), align 4
+; SSE-NEXT:    store <2 x i32> [[TMP21]], <2 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 8) to <2 x i32>*), align 4
+; SSE-NEXT:    store <2 x i32> [[TMP22]], <2 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 10) to <2 x i32>*), align 4
+; SSE-NEXT:    store <2 x i32> [[TMP23]], <2 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 12) to <2 x i32>*), align 4
+; SSE-NEXT:    store <2 x i32> [[TMP24]], <2 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 14) to <2 x i32>*), align 4
 ; SSE-NEXT:    ret void
 ;
 ; AVX-LABEL: @lshr_v16i32(
diff --git a/test/Transforms/SLPVectorizer/X86/shift-shl.ll b/test/Transforms/SLPVectorizer/X86/shift-shl.ll
index 8eadd04d1d0b7e8aa13286cbc1e3888ea85a5904..6993184fb353ab7979b9bcc2722cd0d99a41d688 100644
--- a/test/Transforms/SLPVectorizer/X86/shift-shl.ll
+++ b/test/Transforms/SLPVectorizer/X86/shift-shl.ll
@@ -241,134 +241,38 @@ define void @shl_v16i32() {
 
 define void @shl_v32i16() {
 ; SSE-LABEL: @shl_v32i16(
-; SSE-NEXT:    [[A0:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 0), align 2
-; SSE-NEXT:    [[A1:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 1), align 2
-; SSE-NEXT:    [[A2:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 2), align 2
-; SSE-NEXT:    [[A3:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 3), align 2
-; SSE-NEXT:    [[A4:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 4), align 2
-; SSE-NEXT:    [[A5:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 5), align 2
-; SSE-NEXT:    [[A6:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 6), align 2
-; SSE-NEXT:    [[A7:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 7), align 2
-; SSE-NEXT:    [[A8:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 8), align 2
-; SSE-NEXT:    [[A9:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 9), align 2
-; SSE-NEXT:    [[A10:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 10), align 2
-; SSE-NEXT:    [[A11:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 11), align 2
-; SSE-NEXT:    [[A12:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 12), align 2
-; SSE-NEXT:    [[A13:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 13), align 2
-; SSE-NEXT:    [[A14:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 14), align 2
-; SSE-NEXT:    [[A15:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 15), align 2
-; SSE-NEXT:    [[A16:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 16), align 2
-; SSE-NEXT:    [[A17:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 17), align 2
-; SSE-NEXT:    [[A18:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 18), align 2
-; SSE-NEXT:    [[A19:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 19), align 2
-; SSE-NEXT:    [[A20:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 20), align 2
-; SSE-NEXT:    [[A21:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 21), align 2
-; SSE-NEXT:    [[A22:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 22), align 2
-; SSE-NEXT:    [[A23:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 23), align 2
-; SSE-NEXT:    [[A24:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 24), align 2
-; SSE-NEXT:    [[A25:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 25), align 2
-; SSE-NEXT:    [[A26:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 26), align 2
-; SSE-NEXT:    [[A27:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 27), align 2
-; SSE-NEXT:    [[A28:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 28), align 2
-; SSE-NEXT:    [[A29:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 29), align 2
-; SSE-NEXT:    [[A30:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 30), align 2
-; SSE-NEXT:    [[A31:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 31), align 2
-; SSE-NEXT:    [[B0:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 0), align 2
-; SSE-NEXT:    [[B1:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 1), align 2
-; SSE-NEXT:    [[B2:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 2), align 2
-; SSE-NEXT:    [[B3:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 3), align 2
-; SSE-NEXT:    [[B4:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 4), align 2
-; SSE-NEXT:    [[B5:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 5), align 2
-; SSE-NEXT:    [[B6:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 6), align 2
-; SSE-NEXT:    [[B7:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 7), align 2
-; SSE-NEXT:    [[B8:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 8), align 2
-; SSE-NEXT:    [[B9:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 9), align 2
-; SSE-NEXT:    [[B10:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 10), align 2
-; SSE-NEXT:    [[B11:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 11), align 2
-; SSE-NEXT:    [[B12:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 12), align 2
-; SSE-NEXT:    [[B13:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 13), align 2
-; SSE-NEXT:    [[B14:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 14), align 2
-; SSE-NEXT:    [[B15:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 15), align 2
-; SSE-NEXT:    [[B16:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 16), align 2
-; SSE-NEXT:    [[B17:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 17), align 2
-; SSE-NEXT:    [[B18:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 18), align 2
-; SSE-NEXT:    [[B19:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 19), align 2
-; SSE-NEXT:    [[B20:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 20), align 2
-; SSE-NEXT:    [[B21:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 21), align 2
-; SSE-NEXT:    [[B22:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 22), align 2
-; SSE-NEXT:    [[B23:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 23), align 2
-; SSE-NEXT:    [[B24:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 24), align 2
-; SSE-NEXT:    [[B25:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 25), align 2
-; SSE-NEXT:    [[B26:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 26), align 2
-; SSE-NEXT:    [[B27:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 27), align 2
-; SSE-NEXT:    [[B28:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 28), align 2
-; SSE-NEXT:    [[B29:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 29), align 2
-; SSE-NEXT:    [[B30:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 30), align 2
-; SSE-NEXT:    [[B31:%.*]] = load i16, i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 31), align 2
-; SSE-NEXT:    [[R0:%.*]] = shl i16 [[A0]], [[B0]]
-; SSE-NEXT:    [[R1:%.*]] = shl i16 [[A1]], [[B1]]
-; SSE-NEXT:    [[R2:%.*]] = shl i16 [[A2]], [[B2]]
-; SSE-NEXT:    [[R3:%.*]] = shl i16 [[A3]], [[B3]]
-; SSE-NEXT:    [[R4:%.*]] = shl i16 [[A4]], [[B4]]
-; SSE-NEXT:    [[R5:%.*]] = shl i16 [[A5]], [[B5]]
-; SSE-NEXT:    [[R6:%.*]] = shl i16 [[A6]], [[B6]]
-; SSE-NEXT:    [[R7:%.*]] = shl i16 [[A7]], [[B7]]
-; SSE-NEXT:    [[R8:%.*]] = shl i16 [[A8]], [[B8]]
-; SSE-NEXT:    [[R9:%.*]] = shl i16 [[A9]], [[B9]]
-; SSE-NEXT:    [[R10:%.*]] = shl i16 [[A10]], [[B10]]
-; SSE-NEXT:    [[R11:%.*]] = shl i16 [[A11]], [[B11]]
-; SSE-NEXT:    [[R12:%.*]] = shl i16 [[A12]], [[B12]]
-; SSE-NEXT:    [[R13:%.*]] = shl i16 [[A13]], [[B13]]
-; SSE-NEXT:    [[R14:%.*]] = shl i16 [[A14]], [[B14]]
-; SSE-NEXT:    [[R15:%.*]] = shl i16 [[A15]], [[B15]]
-; SSE-NEXT:    [[R16:%.*]] = shl i16 [[A16]], [[B16]]
-; SSE-NEXT:    [[R17:%.*]] = shl i16 [[A17]], [[B17]]
-; SSE-NEXT:    [[R18:%.*]] = shl i16 [[A18]], [[B18]]
-; SSE-NEXT:    [[R19:%.*]] = shl i16 [[A19]], [[B19]]
-; SSE-NEXT:    [[R20:%.*]] = shl i16 [[A20]], [[B20]]
-; SSE-NEXT:    [[R21:%.*]] = shl i16 [[A21]], [[B21]]
-; SSE-NEXT:    [[R22:%.*]] = shl i16 [[A22]], [[B22]]
-; SSE-NEXT:    [[R23:%.*]] = shl i16 [[A23]], [[B23]]
-; SSE-NEXT:    [[R24:%.*]] = shl i16 [[A24]], [[B24]]
-; SSE-NEXT:    [[R25:%.*]] = shl i16 [[A25]], [[B25]]
-; SSE-NEXT:    [[R26:%.*]] = shl i16 [[A26]], [[B26]]
-; SSE-NEXT:    [[R27:%.*]] = shl i16 [[A27]], [[B27]]
-; SSE-NEXT:    [[R28:%.*]] = shl i16 [[A28]], [[B28]]
-; SSE-NEXT:    [[R29:%.*]] = shl i16 [[A29]], [[B29]]
-; SSE-NEXT:    [[R30:%.*]] = shl i16 [[A30]], [[B30]]
-; SSE-NEXT:    [[R31:%.*]] = shl i16 [[A31]], [[B31]]
-; SSE-NEXT:    store i16 [[R0]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 0), align 2
-; SSE-NEXT:    store i16 [[R1]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 1), align 2
-; SSE-NEXT:    store i16 [[R2]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 2), align 2
-; SSE-NEXT:    store i16 [[R3]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 3), align 2
-; SSE-NEXT:    store i16 [[R4]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 4), align 2
-; SSE-NEXT:    store i16 [[R5]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 5), align 2
-; SSE-NEXT:    store i16 [[R6]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 6), align 2
-; SSE-NEXT:    store i16 [[R7]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 7), align 2
-; SSE-NEXT:    store i16 [[R8]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 8), align 2
-; SSE-NEXT:    store i16 [[R9]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 9), align 2
-; SSE-NEXT:    store i16 [[R10]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 10), align 2
-; SSE-NEXT:    store i16 [[R11]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 11), align 2
-; SSE-NEXT:    store i16 [[R12]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 12), align 2
-; SSE-NEXT:    store i16 [[R13]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 13), align 2
-; SSE-NEXT:    store i16 [[R14]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 14), align 2
-; SSE-NEXT:    store i16 [[R15]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 15), align 2
-; SSE-NEXT:    store i16 [[R16]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 16), align 2
-; SSE-NEXT:    store i16 [[R17]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 17), align 2
-; SSE-NEXT:    store i16 [[R18]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 18), align 2
-; SSE-NEXT:    store i16 [[R19]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 19), align 2
-; SSE-NEXT:    store i16 [[R20]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 20), align 2
-; SSE-NEXT:    store i16 [[R21]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 21), align 2
-; SSE-NEXT:    store i16 [[R22]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 22), align 2
-; SSE-NEXT:    store i16 [[R23]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 23), align 2
-; SSE-NEXT:    store i16 [[R24]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 24), align 2
-; SSE-NEXT:    store i16 [[R25]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 25), align 2
-; SSE-NEXT:    store i16 [[R26]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 26), align 2
-; SSE-NEXT:    store i16 [[R27]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 27), align 2
-; SSE-NEXT:    store i16 [[R28]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 28), align 2
-; SSE-NEXT:    store i16 [[R29]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 29), align 2
-; SSE-NEXT:    store i16 [[R30]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 30), align 2
-; SSE-NEXT:    store i16 [[R31]], i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 31), align 2
+; SSE-NEXT:    [[TMP1:%.*]] = load <4 x i16>, <4 x i16>* bitcast ([32 x i16]* @a16 to <4 x i16>*), align 2
+; SSE-NEXT:    [[TMP2:%.*]] = load <4 x i16>, <4 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 4) to <4 x i16>*), align 2
+; SSE-NEXT:    [[TMP3:%.*]] = load <4 x i16>, <4 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 8) to <4 x i16>*), align 2
+; SSE-NEXT:    [[TMP4:%.*]] = load <4 x i16>, <4 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 12) to <4 x i16>*), align 2
+; SSE-NEXT:    [[TMP5:%.*]] = load <4 x i16>, <4 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 16) to <4 x i16>*), align 2
+; SSE-NEXT:    [[TMP6:%.*]] = load <4 x i16>, <4 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 20) to <4 x i16>*), align 2
+; SSE-NEXT:    [[TMP7:%.*]] = load <4 x i16>, <4 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 24) to <4 x i16>*), align 2
+; SSE-NEXT:    [[TMP8:%.*]] = load <4 x i16>, <4 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 28) to <4 x i16>*), align 2
+; SSE-NEXT:    [[TMP9:%.*]] = load <4 x i16>, <4 x i16>* bitcast ([32 x i16]* @b16 to <4 x i16>*), align 2
+; SSE-NEXT:    [[TMP10:%.*]] = load <4 x i16>, <4 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 4) to <4 x i16>*), align 2
+; SSE-NEXT:    [[TMP11:%.*]] = load <4 x i16>, <4 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 8) to <4 x i16>*), align 2
+; SSE-NEXT:    [[TMP12:%.*]] = load <4 x i16>, <4 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 12) to <4 x i16>*), align 2
+; SSE-NEXT:    [[TMP13:%.*]] = load <4 x i16>, <4 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 16) to <4 x i16>*), align 2
+; SSE-NEXT:    [[TMP14:%.*]] = load <4 x i16>, <4 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 20) to <4 x i16>*), align 2
+; SSE-NEXT:    [[TMP15:%.*]] = load <4 x i16>, <4 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 24) to <4 x i16>*), align 2
+; SSE-NEXT:    [[TMP16:%.*]] = load <4 x i16>, <4 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 28) to <4 x i16>*), align 2
+; SSE-NEXT:    [[TMP17:%.*]] = shl <4 x i16> [[TMP1]], [[TMP9]]
+; SSE-NEXT:    [[TMP18:%.*]] = shl <4 x i16> [[TMP2]], [[TMP10]]
+; SSE-NEXT:    [[TMP19:%.*]] = shl <4 x i16> [[TMP3]], [[TMP11]]
+; SSE-NEXT:    [[TMP20:%.*]] = shl <4 x i16> [[TMP4]], [[TMP12]]
+; SSE-NEXT:    [[TMP21:%.*]] = shl <4 x i16> [[TMP5]], [[TMP13]]
+; SSE-NEXT:    [[TMP22:%.*]] = shl <4 x i16> [[TMP6]], [[TMP14]]
+; SSE-NEXT:    [[TMP23:%.*]] = shl <4 x i16> [[TMP7]], [[TMP15]]
+; SSE-NEXT:    [[TMP24:%.*]] = shl <4 x i16> [[TMP8]], [[TMP16]]
+; SSE-NEXT:    store <4 x i16> [[TMP17]], <4 x i16>* bitcast ([32 x i16]* @c16 to <4 x i16>*), align 2
+; SSE-NEXT:    store <4 x i16> [[TMP18]], <4 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 4) to <4 x i16>*), align 2
+; SSE-NEXT:    store <4 x i16> [[TMP19]], <4 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 8) to <4 x i16>*), align 2
+; SSE-NEXT:    store <4 x i16> [[TMP20]], <4 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 12) to <4 x i16>*), align 2
+; SSE-NEXT:    store <4 x i16> [[TMP21]], <4 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 16) to <4 x i16>*), align 2
+; SSE-NEXT:    store <4 x i16> [[TMP22]], <4 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 20) to <4 x i16>*), align 2
+; SSE-NEXT:    store <4 x i16> [[TMP23]], <4 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 24) to <4 x i16>*), align 2
+; SSE-NEXT:    store <4 x i16> [[TMP24]], <4 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 28) to <4 x i16>*), align 2
 ; SSE-NEXT:    ret void
 ;
 ; AVX-LABEL: @shl_v32i16(
diff --git a/test/Transforms/SLPVectorizer/X86/simple-loop.ll b/test/Transforms/SLPVectorizer/X86/simple-loop.ll
index c9bb88497ac10eb5d2f5583b3646e4fc3c61b8ee..975a1af7576a3ab4514d1c9821e0b5cad67f6ffe 100644
--- a/test/Transforms/SLPVectorizer/X86/simple-loop.ll
+++ b/test/Transforms/SLPVectorizer/X86/simple-loop.ll
@@ -1,10 +1,30 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -basicaa -slp-vectorizer -dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.8.0"
 
-;CHECK:rollable
 define i32 @rollable(i32* noalias nocapture %in, i32* noalias nocapture %out, i64 %n) {
+; CHECK-LABEL: @rollable(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i64 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP1]], label [[DOT_CRIT_EDGE:%.*]], label [[DOTLR_PH:%.*]]
+; CHECK:       .lr.ph:
+; CHECK-NEXT:    [[I_019:%.*]] = phi i64 [ [[TMP10:%.*]], [[DOTLR_PH]] ], [ 0, [[TMP0:%.*]] ]
+; CHECK-NEXT:    [[TMP2:%.*]] = shl i64 [[I_019]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, i32* [[IN:%.*]], i64 [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i32* [[TMP3]] to <4 x i32>*
+; CHECK-NEXT:    [[TMP5:%.*]] = load <4 x i32>, <4 x i32>* [[TMP4]], align 4
+; CHECK-NEXT:    [[TMP6:%.*]] = mul <4 x i32> <i32 7, i32 7, i32 7, i32 7>, [[TMP5]]
+; CHECK-NEXT:    [[TMP7:%.*]] = add <4 x i32> <i32 7, i32 14, i32 21, i32 28>, [[TMP6]]
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i32, i32* [[OUT:%.*]], i64 [[TMP2]]
+; CHECK-NEXT:    [[TMP9:%.*]] = bitcast i32* [[TMP8]] to <4 x i32>*
+; CHECK-NEXT:    store <4 x i32> [[TMP7]], <4 x i32>* [[TMP9]], align 4
+; CHECK-NEXT:    [[TMP10]] = add i64 [[I_019]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[TMP10]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[DOT_CRIT_EDGE]], label [[DOTLR_PH]]
+; CHECK:       ._crit_edge:
+; CHECK-NEXT:    ret i32 undef
+;
   %1 = icmp eq i64 %n, 0
   br i1 %1, label %._crit_edge, label %.lr.ph
 
@@ -12,7 +32,6 @@ define i32 @rollable(i32* noalias nocapture %in, i32* noalias nocapture %out, i6
   %i.019 = phi i64 [ %26, %.lr.ph ], [ 0, %0 ]
   %2 = shl i64 %i.019, 2
   %3 = getelementptr inbounds i32, i32* %in, i64 %2
-;CHECK:load <4 x i32>
   %4 = load i32, i32* %3, align 4
   %5 = or i64 %2, 1
   %6 = getelementptr inbounds i32, i32* %in, i64 %5
@@ -23,9 +42,7 @@ define i32 @rollable(i32* noalias nocapture %in, i32* noalias nocapture %out, i6
   %11 = or i64 %2, 3
   %12 = getelementptr inbounds i32, i32* %in, i64 %11
   %13 = load i32, i32* %12, align 4
-;CHECK:mul <4 x i32>
   %14 = mul i32 %4, 7
-;CHECK:add <4 x i32>
   %15 = add i32 %14, 7
   %16 = mul i32 %7, 7
   %17 = add i32 %16, 14
@@ -34,7 +51,6 @@ define i32 @rollable(i32* noalias nocapture %in, i32* noalias nocapture %out, i6
   %20 = mul i32 %13, 7
   %21 = add i32 %20, 28
   %22 = getelementptr inbounds i32, i32* %out, i64 %2
-;CHECK:store <4 x i32>
   store i32 %15, i32* %22, align 4
   %23 = getelementptr inbounds i32, i32* %out, i64 %5
   store i32 %17, i32* %23, align 4
@@ -47,14 +63,50 @@ define i32 @rollable(i32* noalias nocapture %in, i32* noalias nocapture %out, i6
   br i1 %exitcond, label %._crit_edge, label %.lr.ph
 
 ._crit_edge:                                      ; preds = %.lr.ph, %0
-;CHECK: ret
   ret i32 undef
 }
 
-;CHECK:unrollable
-;CHECK-NOT: <4 x i32>
-;CHECK: ret
 define i32 @unrollable(i32* %in, i32* %out, i64 %n) nounwind ssp uwtable {
+; CHECK-LABEL: @unrollable(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i64 [[N:%.*]], 0
+; CHECK-NEXT:    br i1 [[TMP1]], label [[DOT_CRIT_EDGE:%.*]], label [[DOTLR_PH:%.*]]
+; CHECK:       .lr.ph:
+; CHECK-NEXT:    [[I_019:%.*]] = phi i64 [ [[TMP26:%.*]], [[DOTLR_PH]] ], [ 0, [[TMP0:%.*]] ]
+; CHECK-NEXT:    [[TMP2:%.*]] = shl i64 [[I_019]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i32, i32* [[IN:%.*]], i64 [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = load i32, i32* [[TMP3]], align 4
+; CHECK-NEXT:    [[TMP5:%.*]] = or i64 [[TMP2]], 1
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i32, i32* [[IN]], i64 [[TMP5]]
+; CHECK-NEXT:    [[TMP7:%.*]] = load i32, i32* [[TMP6]], align 4
+; CHECK-NEXT:    [[TMP8:%.*]] = or i64 [[TMP2]], 2
+; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i32, i32* [[IN]], i64 [[TMP8]]
+; CHECK-NEXT:    [[TMP10:%.*]] = load i32, i32* [[TMP9]], align 4
+; CHECK-NEXT:    [[TMP11:%.*]] = or i64 [[TMP2]], 3
+; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i32, i32* [[IN]], i64 [[TMP11]]
+; CHECK-NEXT:    [[TMP13:%.*]] = load i32, i32* [[TMP12]], align 4
+; CHECK-NEXT:    [[TMP14:%.*]] = mul i32 [[TMP4]], 7
+; CHECK-NEXT:    [[TMP15:%.*]] = add i32 [[TMP14]], 7
+; CHECK-NEXT:    [[TMP16:%.*]] = mul i32 [[TMP7]], 7
+; CHECK-NEXT:    [[TMP17:%.*]] = add i32 [[TMP16]], 14
+; CHECK-NEXT:    [[TMP18:%.*]] = mul i32 [[TMP10]], 7
+; CHECK-NEXT:    [[TMP19:%.*]] = add i32 [[TMP18]], 21
+; CHECK-NEXT:    [[TMP20:%.*]] = mul i32 [[TMP13]], 7
+; CHECK-NEXT:    [[TMP21:%.*]] = add i32 [[TMP20]], 28
+; CHECK-NEXT:    [[TMP22:%.*]] = getelementptr inbounds i32, i32* [[OUT:%.*]], i64 [[TMP2]]
+; CHECK-NEXT:    store i32 [[TMP15]], i32* [[TMP22]], align 4
+; CHECK-NEXT:    [[TMP23:%.*]] = getelementptr inbounds i32, i32* [[OUT]], i64 [[TMP5]]
+; CHECK-NEXT:    store i32 [[TMP17]], i32* [[TMP23]], align 4
+; CHECK-NEXT:    [[BARRIER:%.*]] = call i32 @goo(i32 0)
+; CHECK-NEXT:    [[TMP24:%.*]] = getelementptr inbounds i32, i32* [[OUT]], i64 [[TMP8]]
+; CHECK-NEXT:    store i32 [[TMP19]], i32* [[TMP24]], align 4
+; CHECK-NEXT:    [[TMP25:%.*]] = getelementptr inbounds i32, i32* [[OUT]], i64 [[TMP11]]
+; CHECK-NEXT:    store i32 [[TMP21]], i32* [[TMP25]], align 4
+; CHECK-NEXT:    [[TMP26]] = add i64 [[I_019]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[TMP26]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[DOT_CRIT_EDGE]], label [[DOTLR_PH]]
+; CHECK:       ._crit_edge:
+; CHECK-NEXT:    ret i32 undef
+;
   %1 = icmp eq i64 %n, 0
   br i1 %1, label %._crit_edge, label %.lr.ph
 
diff --git a/test/Transforms/SLPVectorizer/X86/sitofp.ll b/test/Transforms/SLPVectorizer/X86/sitofp.ll
index d4c4be5bbf6a4c3c1f7197b243ee5eb3a3552e17..67f554275b47ab585f2b3997c8685145e92ba957 100644
--- a/test/Transforms/SLPVectorizer/X86/sitofp.ll
+++ b/test/Transforms/SLPVectorizer/X86/sitofp.ll
@@ -598,14 +598,35 @@ define void @sitofp_8i8_8f64() #0 {
 ;
 
 define void @sitofp_2i64_2f32() #0 {
-; CHECK-LABEL: @sitofp_2i64_2f32(
-; CHECK-NEXT:    [[LD0:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 0), align 64
-; CHECK-NEXT:    [[LD1:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 1), align 8
-; CHECK-NEXT:    [[CVT0:%.*]] = sitofp i64 [[LD0]] to float
-; CHECK-NEXT:    [[CVT1:%.*]] = sitofp i64 [[LD1]] to float
-; CHECK-NEXT:    store float [[CVT0]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 0), align 64
-; CHECK-NEXT:    store float [[CVT1]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 1), align 4
-; CHECK-NEXT:    ret void
+; SSE-LABEL: @sitofp_2i64_2f32(
+; SSE-NEXT:    [[LD0:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 0), align 64
+; SSE-NEXT:    [[LD1:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 1), align 8
+; SSE-NEXT:    [[CVT0:%.*]] = sitofp i64 [[LD0]] to float
+; SSE-NEXT:    [[CVT1:%.*]] = sitofp i64 [[LD1]] to float
+; SSE-NEXT:    store float [[CVT0]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 0), align 64
+; SSE-NEXT:    store float [[CVT1]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 1), align 4
+; SSE-NEXT:    ret void
+;
+; AVX256NODQ-LABEL: @sitofp_2i64_2f32(
+; AVX256NODQ-NEXT:    [[LD0:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 0), align 64
+; AVX256NODQ-NEXT:    [[LD1:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 1), align 8
+; AVX256NODQ-NEXT:    [[CVT0:%.*]] = sitofp i64 [[LD0]] to float
+; AVX256NODQ-NEXT:    [[CVT1:%.*]] = sitofp i64 [[LD1]] to float
+; AVX256NODQ-NEXT:    store float [[CVT0]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 0), align 64
+; AVX256NODQ-NEXT:    store float [[CVT1]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 1), align 4
+; AVX256NODQ-NEXT:    ret void
+;
+; AVX512-LABEL: @sitofp_2i64_2f32(
+; AVX512-NEXT:    [[TMP1:%.*]] = load <2 x i64>, <2 x i64>* bitcast ([8 x i64]* @src64 to <2 x i64>*), align 64
+; AVX512-NEXT:    [[TMP2:%.*]] = sitofp <2 x i64> [[TMP1]] to <2 x float>
+; AVX512-NEXT:    store <2 x float> [[TMP2]], <2 x float>* bitcast ([16 x float]* @dst32 to <2 x float>*), align 64
+; AVX512-NEXT:    ret void
+;
+; AVX256DQ-LABEL: @sitofp_2i64_2f32(
+; AVX256DQ-NEXT:    [[TMP1:%.*]] = load <2 x i64>, <2 x i64>* bitcast ([8 x i64]* @src64 to <2 x i64>*), align 64
+; AVX256DQ-NEXT:    [[TMP2:%.*]] = sitofp <2 x i64> [[TMP1]] to <2 x float>
+; AVX256DQ-NEXT:    store <2 x float> [[TMP2]], <2 x float>* bitcast ([16 x float]* @dst32 to <2 x float>*), align 64
+; AVX256DQ-NEXT:    ret void
 ;
   %ld0 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 0), align 64
   %ld1 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 1), align 8
diff --git a/test/Transforms/SLPVectorizer/X86/tiny-tree.ll b/test/Transforms/SLPVectorizer/X86/tiny-tree.ll
index 19f06805ff50e6dd5eb1487bdf977a3348052e76..95b7d6bf55e6e99f92710acc4fe8d20ff462c1d0 100644
--- a/test/Transforms/SLPVectorizer/X86/tiny-tree.ll
+++ b/test/Transforms/SLPVectorizer/X86/tiny-tree.ll
@@ -172,13 +172,13 @@ define void @tiny_tree_not_fully_vectorizable2(float* noalias nocapture %dst, fl
 ; CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds float, float* [[DST_ADDR_022]], i64 1
 ; CHECK-NEXT:    store float [[TMP1]], float* [[ARRAYIDX3]], align 4
 ; CHECK-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds float, float* [[SRC_ADDR_021]], i64 2
-; CHECK-NEXT:    [[TMP2:%.*]] = load float, float* [[ARRAYIDX4]], align 4
 ; CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds float, float* [[DST_ADDR_022]], i64 2
-; CHECK-NEXT:    store float [[TMP2]], float* [[ARRAYIDX5]], align 4
 ; CHECK-NEXT:    [[ARRAYIDX6:%.*]] = getelementptr inbounds float, float* [[SRC_ADDR_021]], i64 3
-; CHECK-NEXT:    [[TMP3:%.*]] = load float, float* [[ARRAYIDX6]], align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast float* [[ARRAYIDX4]] to <2 x float>*
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x float>, <2 x float>* [[TMP2]], align 4
 ; CHECK-NEXT:    [[ARRAYIDX7:%.*]] = getelementptr inbounds float, float* [[DST_ADDR_022]], i64 3
-; CHECK-NEXT:    store float [[TMP3]], float* [[ARRAYIDX7]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast float* [[ARRAYIDX5]] to <2 x float>*
+; CHECK-NEXT:    store <2 x float> [[TMP3]], <2 x float>* [[TMP4]], align 4
 ; CHECK-NEXT:    [[ADD_PTR]] = getelementptr inbounds float, float* [[SRC_ADDR_021]], i64 [[I_023]]
 ; CHECK-NEXT:    [[ADD_PTR8]] = getelementptr inbounds float, float* [[DST_ADDR_022]], i64 [[I_023]]
 ; CHECK-NEXT:    [[INC]] = add i64 [[I_023]], 1
diff --git a/test/Transforms/SLPVectorizer/X86/uitofp.ll b/test/Transforms/SLPVectorizer/X86/uitofp.ll
index 092918115ebac48c5d5b01f4a8c03390e212c524..92ff1fbaae86d72042b241931e5b12b193a7c768 100644
--- a/test/Transforms/SLPVectorizer/X86/uitofp.ll
+++ b/test/Transforms/SLPVectorizer/X86/uitofp.ll
@@ -550,14 +550,35 @@ define void @uitofp_8i8_8f64() #0 {
 ;
 
 define void @uitofp_2i64_2f32() #0 {
-; CHECK-LABEL: @uitofp_2i64_2f32(
-; CHECK-NEXT:    [[LD0:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 0), align 64
-; CHECK-NEXT:    [[LD1:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 1), align 8
-; CHECK-NEXT:    [[CVT0:%.*]] = uitofp i64 [[LD0]] to float
-; CHECK-NEXT:    [[CVT1:%.*]] = uitofp i64 [[LD1]] to float
-; CHECK-NEXT:    store float [[CVT0]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 0), align 64
-; CHECK-NEXT:    store float [[CVT1]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 1), align 4
-; CHECK-NEXT:    ret void
+; SSE-LABEL: @uitofp_2i64_2f32(
+; SSE-NEXT:    [[LD0:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 0), align 64
+; SSE-NEXT:    [[LD1:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 1), align 8
+; SSE-NEXT:    [[CVT0:%.*]] = uitofp i64 [[LD0]] to float
+; SSE-NEXT:    [[CVT1:%.*]] = uitofp i64 [[LD1]] to float
+; SSE-NEXT:    store float [[CVT0]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 0), align 64
+; SSE-NEXT:    store float [[CVT1]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 1), align 4
+; SSE-NEXT:    ret void
+;
+; AVX256NODQ-LABEL: @uitofp_2i64_2f32(
+; AVX256NODQ-NEXT:    [[LD0:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 0), align 64
+; AVX256NODQ-NEXT:    [[LD1:%.*]] = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 1), align 8
+; AVX256NODQ-NEXT:    [[CVT0:%.*]] = uitofp i64 [[LD0]] to float
+; AVX256NODQ-NEXT:    [[CVT1:%.*]] = uitofp i64 [[LD1]] to float
+; AVX256NODQ-NEXT:    store float [[CVT0]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 0), align 64
+; AVX256NODQ-NEXT:    store float [[CVT1]], float* getelementptr inbounds ([16 x float], [16 x float]* @dst32, i32 0, i64 1), align 4
+; AVX256NODQ-NEXT:    ret void
+;
+; AVX512-LABEL: @uitofp_2i64_2f32(
+; AVX512-NEXT:    [[TMP1:%.*]] = load <2 x i64>, <2 x i64>* bitcast ([8 x i64]* @src64 to <2 x i64>*), align 64
+; AVX512-NEXT:    [[TMP2:%.*]] = uitofp <2 x i64> [[TMP1]] to <2 x float>
+; AVX512-NEXT:    store <2 x float> [[TMP2]], <2 x float>* bitcast ([16 x float]* @dst32 to <2 x float>*), align 64
+; AVX512-NEXT:    ret void
+;
+; AVX256DQ-LABEL: @uitofp_2i64_2f32(
+; AVX256DQ-NEXT:    [[TMP1:%.*]] = load <2 x i64>, <2 x i64>* bitcast ([8 x i64]* @src64 to <2 x i64>*), align 64
+; AVX256DQ-NEXT:    [[TMP2:%.*]] = uitofp <2 x i64> [[TMP1]] to <2 x float>
+; AVX256DQ-NEXT:    store <2 x float> [[TMP2]], <2 x float>* bitcast ([16 x float]* @dst32 to <2 x float>*), align 64
+; AVX256DQ-NEXT:    ret void
 ;
   %ld0 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 0), align 64
   %ld1 = load i64, i64* getelementptr inbounds ([8 x i64], [8 x i64]* @src64, i32 0, i64 1), align 8
diff --git a/test/Transforms/SLPVectorizer/X86/unreachable.ll b/test/Transforms/SLPVectorizer/X86/unreachable.ll
index f29f69d7e82f281d1f4d7196f0446aa6d0fa5358..cc1f1fca8cac6cda3a3173959564807f2eee3006 100644
--- a/test/Transforms/SLPVectorizer/X86/unreachable.ll
+++ b/test/Transforms/SLPVectorizer/X86/unreachable.ll
@@ -1,4 +1,5 @@
-; RUN: opt < %s -basicaa -slp-vectorizer -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -basicaa -slp-vectorizer -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7 | FileCheck %s
 
 ; Check if the SLPVectorizer does not crash when handling
 ; unreachable blocks with unscheduleable instructions.
@@ -7,6 +8,34 @@ target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-apple-macosx10.9.0"
 
 define void @foo(i32* nocapture %x) #0 {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[BB2:%.*]]
+; CHECK:       bb1:
+; CHECK-NEXT:    [[T3:%.*]] = getelementptr inbounds i32, i32* [[X:%.*]], i64 4
+; CHECK-NEXT:    [[T4:%.*]] = load i32, i32* [[T3]], align 4
+; CHECK-NEXT:    [[T5:%.*]] = getelementptr inbounds i32, i32* [[X]], i64 5
+; CHECK-NEXT:    [[T6:%.*]] = load i32, i32* [[T5]], align 4
+; CHECK-NEXT:    [[BAD:%.*]] = fadd float [[BAD]], 0.000000e+00
+; CHECK-NEXT:    [[T7:%.*]] = getelementptr inbounds i32, i32* [[X]], i64 6
+; CHECK-NEXT:    [[T8:%.*]] = load i32, i32* [[T7]], align 4
+; CHECK-NEXT:    [[T9:%.*]] = getelementptr inbounds i32, i32* [[X]], i64 7
+; CHECK-NEXT:    [[T10:%.*]] = load i32, i32* [[T9]], align 4
+; CHECK-NEXT:    br label [[BB2]]
+; CHECK:       bb2:
+; CHECK-NEXT:    [[T1_0:%.*]] = phi i32 [ [[T4]], [[BB1:%.*]] ], [ 2, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[T2_0:%.*]] = phi i32 [ [[T6]], [[BB1]] ], [ 2, [[ENTRY]] ]
+; CHECK-NEXT:    [[T3_0:%.*]] = phi i32 [ [[T8]], [[BB1]] ], [ 2, [[ENTRY]] ]
+; CHECK-NEXT:    [[T4_0:%.*]] = phi i32 [ [[T10]], [[BB1]] ], [ 2, [[ENTRY]] ]
+; CHECK-NEXT:    store i32 [[T1_0]], i32* [[X]], align 4
+; CHECK-NEXT:    [[T12:%.*]] = getelementptr inbounds i32, i32* [[X]], i64 1
+; CHECK-NEXT:    store i32 [[T2_0]], i32* [[T12]], align 4
+; CHECK-NEXT:    [[T13:%.*]] = getelementptr inbounds i32, i32* [[X]], i64 2
+; CHECK-NEXT:    store i32 [[T3_0]], i32* [[T13]], align 4
+; CHECK-NEXT:    [[T14:%.*]] = getelementptr inbounds i32, i32* [[X]], i64 3
+; CHECK-NEXT:    store i32 [[T4_0]], i32* [[T14]], align 4
+; CHECK-NEXT:    ret void
+;
 entry:
   br label %bb2
 
@@ -16,7 +45,7 @@ bb1:                                    ; an unreachable block
   %t5 = getelementptr inbounds i32, i32* %x, i64 5
   %t6 = load i32, i32* %t5, align 4
   %bad = fadd float %bad, 0.000000e+00  ; <- an instruction with self dependency,
-                                        ;    but legal in unreachable code
+  ;    but legal in unreachable code
   %t7 = getelementptr inbounds i32, i32* %x, i64 6
   %t8 = load i32, i32* %t7, align 4
   %t9 = getelementptr inbounds i32, i32* %x, i64 7
diff --git a/test/Transforms/SLPVectorizer/X86/vec-reg-64bit.ll b/test/Transforms/SLPVectorizer/X86/vec-reg-64bit.ll
new file mode 100644
index 0000000000000000000000000000000000000000..f71f3a7d5c1d834e59d343bd483ac187b26f71b3
--- /dev/null
+++ b/test/Transforms/SLPVectorizer/X86/vec-reg-64bit.ll
@@ -0,0 +1,51 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -basicaa -slp-vectorizer -mcpu=btver2 -S | FileCheck %s --check-prefix=VECT
+; RUN: opt < %s -basicaa -slp-vectorizer -mcpu=btver2 -slp-min-reg-size=128 -S | FileCheck %s --check-prefix=NOVECT
+
+; Check SLPVectorizer works for packed horizontal 128-bit instrs.
+; See llvm.org/PR32433
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define void @add_pairs_128(<4 x float>, float* nocapture) #0 {
+; VECT-LABEL: @add_pairs_128(
+; VECT-NEXT:    [[TMP3:%.*]] = extractelement <4 x float> [[TMP0:%.*]], i32 0
+; VECT-NEXT:    [[TMP4:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
+; VECT-NEXT:    [[TMP5:%.*]] = extractelement <4 x float> [[TMP0]], i32 2
+; VECT-NEXT:    [[TMP6:%.*]] = extractelement <4 x float> [[TMP0]], i32 3
+; VECT-NEXT:    [[TMP7:%.*]] = insertelement <2 x float> undef, float [[TMP3]], i32 0
+; VECT-NEXT:    [[TMP8:%.*]] = insertelement <2 x float> [[TMP7]], float [[TMP5]], i32 1
+; VECT-NEXT:    [[TMP9:%.*]] = insertelement <2 x float> undef, float [[TMP4]], i32 0
+; VECT-NEXT:    [[TMP10:%.*]] = insertelement <2 x float> [[TMP9]], float [[TMP6]], i32 1
+; VECT-NEXT:    [[TMP11:%.*]] = fadd <2 x float> [[TMP8]], [[TMP10]]
+; VECT-NEXT:    [[TMP12:%.*]] = getelementptr inbounds float, float* [[TMP1:%.*]], i64 1
+; VECT-NEXT:    [[TMP13:%.*]] = bitcast float* [[TMP1]] to <2 x float>*
+; VECT-NEXT:    store <2 x float> [[TMP11]], <2 x float>* [[TMP13]], align 4
+; VECT-NEXT:    ret void
+;
+; NOVECT-LABEL: @add_pairs_128(
+; NOVECT-NEXT:    [[TMP3:%.*]] = extractelement <4 x float> [[TMP0:%.*]], i32 0
+; NOVECT-NEXT:    [[TMP4:%.*]] = extractelement <4 x float> [[TMP0]], i32 1
+; NOVECT-NEXT:    [[TMP5:%.*]] = fadd float [[TMP3]], [[TMP4]]
+; NOVECT-NEXT:    store float [[TMP5]], float* [[TMP1:%.*]], align 4
+; NOVECT-NEXT:    [[TMP6:%.*]] = extractelement <4 x float> [[TMP0]], i32 2
+; NOVECT-NEXT:    [[TMP7:%.*]] = extractelement <4 x float> [[TMP0]], i32 3
+; NOVECT-NEXT:    [[TMP8:%.*]] = fadd float [[TMP6]], [[TMP7]]
+; NOVECT-NEXT:    [[TMP9:%.*]] = getelementptr inbounds float, float* [[TMP1]], i64 1
+; NOVECT-NEXT:    store float [[TMP8]], float* [[TMP9]], align 4
+; NOVECT-NEXT:    ret void
+;
+  %3 = extractelement <4 x float> %0, i32 0
+  %4 = extractelement <4 x float> %0, i32 1
+  %5 = fadd float %3, %4
+  store float %5, float* %1, align 4
+  %6 = extractelement <4 x float> %0, i32 2
+  %7 = extractelement <4 x float> %0, i32 3
+  %8 = fadd float %6, %7
+  %9 = getelementptr inbounds float, float* %1, i64 1
+  store float %8, float* %9, align 4
+  ret void
+}
+
+attributes #0 = { nounwind }
diff --git a/test/Transforms/SLPVectorizer/X86/vect_copyable_in_binops.ll b/test/Transforms/SLPVectorizer/X86/vect_copyable_in_binops.ll
index 2b593b78652fdbd0ce5858b1fce6f5e76b4e24ae..0df2e751cebbfab249a8ecdc46fc5f5976ace9ad 100644
--- a/test/Transforms/SLPVectorizer/X86/vect_copyable_in_binops.ll
+++ b/test/Transforms/SLPVectorizer/X86/vect_copyable_in_binops.ll
@@ -47,17 +47,16 @@ define void @add1(i32* noalias %dst, i32* noalias %src) {
 ; CHECK-NEXT:    [[INCDEC_PTR1:%.*]] = getelementptr inbounds i32, i32* [[DST:%.*]], i64 1
 ; CHECK-NEXT:    store i32 [[TMP0]], i32* [[DST]], align 4
 ; CHECK-NEXT:    [[INCDEC_PTR2:%.*]] = getelementptr inbounds i32, i32* [[SRC]], i64 2
-; CHECK-NEXT:    [[TMP1:%.*]] = load i32, i32* [[INCDEC_PTR]], align 4
-; CHECK-NEXT:    [[ADD3:%.*]] = add nsw i32 [[TMP1]], 1
 ; CHECK-NEXT:    [[INCDEC_PTR4:%.*]] = getelementptr inbounds i32, i32* [[DST]], i64 2
-; CHECK-NEXT:    store i32 [[ADD3]], i32* [[INCDEC_PTR1]], align 4
 ; CHECK-NEXT:    [[INCDEC_PTR5:%.*]] = getelementptr inbounds i32, i32* [[SRC]], i64 3
-; CHECK-NEXT:    [[TMP2:%.*]] = load i32, i32* [[INCDEC_PTR2]], align 4
-; CHECK-NEXT:    [[ADD6:%.*]] = add nsw i32 [[TMP2]], 2
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i32* [[INCDEC_PTR]] to <2 x i32>*
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i32>, <2 x i32>* [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = add nsw <2 x i32> <i32 1, i32 2>, [[TMP2]]
 ; CHECK-NEXT:    [[INCDEC_PTR7:%.*]] = getelementptr inbounds i32, i32* [[DST]], i64 3
-; CHECK-NEXT:    store i32 [[ADD6]], i32* [[INCDEC_PTR4]], align 4
-; CHECK-NEXT:    [[TMP3:%.*]] = load i32, i32* [[INCDEC_PTR5]], align 4
-; CHECK-NEXT:    [[ADD9:%.*]] = add nsw i32 [[TMP3]], 3
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i32* [[INCDEC_PTR1]] to <2 x i32>*
+; CHECK-NEXT:    store <2 x i32> [[TMP3]], <2 x i32>* [[TMP4]], align 4
+; CHECK-NEXT:    [[TMP5:%.*]] = load i32, i32* [[INCDEC_PTR5]], align 4
+; CHECK-NEXT:    [[ADD9:%.*]] = add nsw i32 [[TMP5]], 3
 ; CHECK-NEXT:    store i32 [[ADD9]], i32* [[INCDEC_PTR7]], align 4
 ; CHECK-NEXT:    ret void
 ;
@@ -95,13 +94,12 @@ define void @sub0(i32* noalias %dst, i32* noalias %src) {
 ; CHECK-NEXT:    [[INCDEC_PTR3:%.*]] = getelementptr inbounds i32, i32* [[DST]], i64 2
 ; CHECK-NEXT:    store i32 [[TMP1]], i32* [[INCDEC_PTR1]], align 4
 ; CHECK-NEXT:    [[INCDEC_PTR4:%.*]] = getelementptr inbounds i32, i32* [[SRC]], i64 3
-; CHECK-NEXT:    [[TMP2:%.*]] = load i32, i32* [[INCDEC_PTR2]], align 4
-; CHECK-NEXT:    [[SUB5:%.*]] = add nsw i32 [[TMP2]], -2
 ; CHECK-NEXT:    [[INCDEC_PTR6:%.*]] = getelementptr inbounds i32, i32* [[DST]], i64 3
-; CHECK-NEXT:    store i32 [[SUB5]], i32* [[INCDEC_PTR3]], align 4
-; CHECK-NEXT:    [[TMP3:%.*]] = load i32, i32* [[INCDEC_PTR4]], align 4
-; CHECK-NEXT:    [[SUB8:%.*]] = add nsw i32 [[TMP3]], -3
-; CHECK-NEXT:    store i32 [[SUB8]], i32* [[INCDEC_PTR6]], align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i32* [[INCDEC_PTR2]] to <2 x i32>*
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i32>, <2 x i32>* [[TMP2]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = add nsw <2 x i32> <i32 -2, i32 -3>, [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i32* [[INCDEC_PTR3]] to <2 x i32>*
+; CHECK-NEXT:    store <2 x i32> [[TMP4]], <2 x i32>* [[TMP5]], align 4
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -214,13 +212,14 @@ define void @addsub0(i32* noalias %dst, i32* noalias %src) {
 ; CHECK-NEXT:    [[INCDEC_PTR3:%.*]] = getelementptr inbounds i32, i32* [[DST]], i64 2
 ; CHECK-NEXT:    store i32 [[TMP1]], i32* [[INCDEC_PTR1]], align 4
 ; CHECK-NEXT:    [[INCDEC_PTR4:%.*]] = getelementptr inbounds i32, i32* [[SRC]], i64 3
-; CHECK-NEXT:    [[TMP2:%.*]] = load i32, i32* [[INCDEC_PTR2]], align 4
-; CHECK-NEXT:    [[SUB5:%.*]] = add nsw i32 [[TMP2]], -2
 ; CHECK-NEXT:    [[INCDEC_PTR6:%.*]] = getelementptr inbounds i32, i32* [[DST]], i64 3
-; CHECK-NEXT:    store i32 [[SUB5]], i32* [[INCDEC_PTR3]], align 4
-; CHECK-NEXT:    [[TMP3:%.*]] = load i32, i32* [[INCDEC_PTR4]], align 4
-; CHECK-NEXT:    [[SUB8:%.*]] = sub nsw i32 [[TMP3]], -3
-; CHECK-NEXT:    store i32 [[SUB8]], i32* [[INCDEC_PTR6]], align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i32* [[INCDEC_PTR2]] to <2 x i32>*
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x i32>, <2 x i32>* [[TMP2]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = add nsw <2 x i32> [[TMP3]], <i32 -2, i32 -3>
+; CHECK-NEXT:    [[TMP5:%.*]] = sub nsw <2 x i32> [[TMP3]], <i32 -2, i32 -3>
+; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <2 x i32> [[TMP4]], <2 x i32> [[TMP5]], <2 x i32> <i32 0, i32 3>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast i32* [[INCDEC_PTR3]] to <2 x i32>*
+; CHECK-NEXT:    store <2 x i32> [[TMP6]], <2 x i32>* [[TMP7]], align 4
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -248,21 +247,22 @@ define void @addsub1(i32* noalias %dst, i32* noalias %src) {
 ; CHECK-LABEL: @addsub1(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[INCDEC_PTR:%.*]] = getelementptr inbounds i32, i32* [[SRC:%.*]], i64 1
-; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* [[SRC]], align 4
-; CHECK-NEXT:    [[SUB:%.*]] = add nsw i32 [[TMP0]], -1
 ; CHECK-NEXT:    [[INCDEC_PTR1:%.*]] = getelementptr inbounds i32, i32* [[DST:%.*]], i64 1
-; CHECK-NEXT:    store i32 [[SUB]], i32* [[DST]], align 4
 ; CHECK-NEXT:    [[INCDEC_PTR2:%.*]] = getelementptr inbounds i32, i32* [[SRC]], i64 2
-; CHECK-NEXT:    [[TMP1:%.*]] = load i32, i32* [[INCDEC_PTR]], align 4
-; CHECK-NEXT:    [[SUB1:%.*]] = sub nsw i32 [[TMP1]], -1
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast i32* [[SRC]] to <2 x i32>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i32>, <2 x i32>* [[TMP0]], align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = add nsw <2 x i32> [[TMP1]], <i32 -1, i32 -1>
+; CHECK-NEXT:    [[TMP3:%.*]] = sub nsw <2 x i32> [[TMP1]], <i32 -1, i32 -1>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <2 x i32> [[TMP2]], <2 x i32> [[TMP3]], <2 x i32> <i32 0, i32 3>
 ; CHECK-NEXT:    [[INCDEC_PTR3:%.*]] = getelementptr inbounds i32, i32* [[DST]], i64 2
-; CHECK-NEXT:    store i32 [[SUB1]], i32* [[INCDEC_PTR1]], align 4
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast i32* [[DST]] to <2 x i32>*
+; CHECK-NEXT:    store <2 x i32> [[TMP4]], <2 x i32>* [[TMP5]], align 4
 ; CHECK-NEXT:    [[INCDEC_PTR4:%.*]] = getelementptr inbounds i32, i32* [[SRC]], i64 3
-; CHECK-NEXT:    [[TMP2:%.*]] = load i32, i32* [[INCDEC_PTR2]], align 4
+; CHECK-NEXT:    [[TMP6:%.*]] = load i32, i32* [[INCDEC_PTR2]], align 4
 ; CHECK-NEXT:    [[INCDEC_PTR6:%.*]] = getelementptr inbounds i32, i32* [[DST]], i64 3
-; CHECK-NEXT:    store i32 [[TMP2]], i32* [[INCDEC_PTR3]], align 4
-; CHECK-NEXT:    [[TMP3:%.*]] = load i32, i32* [[INCDEC_PTR4]], align 4
-; CHECK-NEXT:    [[SUB8:%.*]] = sub nsw i32 [[TMP3]], -3
+; CHECK-NEXT:    store i32 [[TMP6]], i32* [[INCDEC_PTR3]], align 4
+; CHECK-NEXT:    [[TMP7:%.*]] = load i32, i32* [[INCDEC_PTR4]], align 4
+; CHECK-NEXT:    [[SUB8:%.*]] = sub nsw i32 [[TMP7]], -3
 ; CHECK-NEXT:    store i32 [[SUB8]], i32* [[INCDEC_PTR6]], align 4
 ; CHECK-NEXT:    ret void
 ;
@@ -338,17 +338,16 @@ define void @shl0(i32* noalias %dst, i32* noalias %src) {
 ; CHECK-NEXT:    [[INCDEC_PTR1:%.*]] = getelementptr inbounds i32, i32* [[DST:%.*]], i64 1
 ; CHECK-NEXT:    store i32 [[TMP0]], i32* [[DST]], align 4
 ; CHECK-NEXT:    [[INCDEC_PTR2:%.*]] = getelementptr inbounds i32, i32* [[SRC]], i64 2
-; CHECK-NEXT:    [[TMP1:%.*]] = load i32, i32* [[INCDEC_PTR]], align 4
-; CHECK-NEXT:    [[SHL:%.*]] = shl i32 [[TMP1]], 1
 ; CHECK-NEXT:    [[INCDEC_PTR3:%.*]] = getelementptr inbounds i32, i32* [[DST]], i64 2
-; CHECK-NEXT:    store i32 [[SHL]], i32* [[INCDEC_PTR1]], align 4
 ; CHECK-NEXT:    [[INCDEC_PTR4:%.*]] = getelementptr inbounds i32, i32* [[SRC]], i64 3
-; CHECK-NEXT:    [[TMP2:%.*]] = load i32, i32* [[INCDEC_PTR2]], align 4
-; CHECK-NEXT:    [[SHL5:%.*]] = shl i32 [[TMP2]], 2
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i32* [[INCDEC_PTR]] to <2 x i32>*
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x i32>, <2 x i32>* [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = shl <2 x i32> [[TMP2]], <i32 1, i32 2>
 ; CHECK-NEXT:    [[INCDEC_PTR6:%.*]] = getelementptr inbounds i32, i32* [[DST]], i64 3
-; CHECK-NEXT:    store i32 [[SHL5]], i32* [[INCDEC_PTR3]], align 4
-; CHECK-NEXT:    [[TMP3:%.*]] = load i32, i32* [[INCDEC_PTR4]], align 4
-; CHECK-NEXT:    [[SHL8:%.*]] = shl i32 [[TMP3]], 3
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast i32* [[INCDEC_PTR1]] to <2 x i32>*
+; CHECK-NEXT:    store <2 x i32> [[TMP3]], <2 x i32>* [[TMP4]], align 4
+; CHECK-NEXT:    [[TMP5:%.*]] = load i32, i32* [[INCDEC_PTR4]], align 4
+; CHECK-NEXT:    [[SHL8:%.*]] = shl i32 [[TMP5]], 3
 ; CHECK-NEXT:    store i32 [[SHL8]], i32* [[INCDEC_PTR6]], align 4
 ; CHECK-NEXT:    ret void
 ;
@@ -457,17 +456,16 @@ define void @add1f(float* noalias %dst, float* noalias %src) {
 ; CHECK-NEXT:    [[INCDEC_PTR1:%.*]] = getelementptr inbounds float, float* [[DST:%.*]], i64 1
 ; CHECK-NEXT:    store float [[TMP0]], float* [[DST]], align 4
 ; CHECK-NEXT:    [[INCDEC_PTR2:%.*]] = getelementptr inbounds float, float* [[SRC]], i64 2
-; CHECK-NEXT:    [[TMP1:%.*]] = load float, float* [[INCDEC_PTR]], align 4
-; CHECK-NEXT:    [[ADD3:%.*]] = fadd fast float [[TMP1]], 1.000000e+00
 ; CHECK-NEXT:    [[INCDEC_PTR4:%.*]] = getelementptr inbounds float, float* [[DST]], i64 2
-; CHECK-NEXT:    store float [[ADD3]], float* [[INCDEC_PTR1]], align 4
 ; CHECK-NEXT:    [[INCDEC_PTR5:%.*]] = getelementptr inbounds float, float* [[SRC]], i64 3
-; CHECK-NEXT:    [[TMP2:%.*]] = load float, float* [[INCDEC_PTR2]], align 4
-; CHECK-NEXT:    [[ADD6:%.*]] = fadd fast float [[TMP2]], 2.000000e+00
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast float* [[INCDEC_PTR]] to <2 x float>*
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x float>, <2 x float>* [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = fadd fast <2 x float> <float 1.000000e+00, float 2.000000e+00>, [[TMP2]]
 ; CHECK-NEXT:    [[INCDEC_PTR7:%.*]] = getelementptr inbounds float, float* [[DST]], i64 3
-; CHECK-NEXT:    store float [[ADD6]], float* [[INCDEC_PTR4]], align 4
-; CHECK-NEXT:    [[TMP3:%.*]] = load float, float* [[INCDEC_PTR5]], align 4
-; CHECK-NEXT:    [[ADD9:%.*]] = fadd fast float [[TMP3]], 3.000000e+00
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast float* [[INCDEC_PTR1]] to <2 x float>*
+; CHECK-NEXT:    store <2 x float> [[TMP3]], <2 x float>* [[TMP4]], align 4
+; CHECK-NEXT:    [[TMP5:%.*]] = load float, float* [[INCDEC_PTR5]], align 4
+; CHECK-NEXT:    [[ADD9:%.*]] = fadd fast float [[TMP5]], 3.000000e+00
 ; CHECK-NEXT:    store float [[ADD9]], float* [[INCDEC_PTR7]], align 4
 ; CHECK-NEXT:    ret void
 ;
@@ -505,13 +503,12 @@ define void @sub0f(float* noalias %dst, float* noalias %src) {
 ; CHECK-NEXT:    [[INCDEC_PTR4:%.*]] = getelementptr inbounds float, float* [[DST]], i64 2
 ; CHECK-NEXT:    store float [[TMP1]], float* [[INCDEC_PTR1]], align 4
 ; CHECK-NEXT:    [[INCDEC_PTR5:%.*]] = getelementptr inbounds float, float* [[SRC]], i64 3
-; CHECK-NEXT:    [[TMP2:%.*]] = load float, float* [[INCDEC_PTR2]], align 4
-; CHECK-NEXT:    [[ADD6:%.*]] = fadd fast float [[TMP2]], -2.000000e+00
 ; CHECK-NEXT:    [[INCDEC_PTR7:%.*]] = getelementptr inbounds float, float* [[DST]], i64 3
-; CHECK-NEXT:    store float [[ADD6]], float* [[INCDEC_PTR4]], align 4
-; CHECK-NEXT:    [[TMP3:%.*]] = load float, float* [[INCDEC_PTR5]], align 4
-; CHECK-NEXT:    [[ADD9:%.*]] = fadd fast float [[TMP3]], -3.000000e+00
-; CHECK-NEXT:    store float [[ADD9]], float* [[INCDEC_PTR7]], align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast float* [[INCDEC_PTR2]] to <2 x float>*
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x float>, <2 x float>* [[TMP2]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = fadd fast <2 x float> <float -2.000000e+00, float -3.000000e+00>, [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast float* [[INCDEC_PTR4]] to <2 x float>*
+; CHECK-NEXT:    store <2 x float> [[TMP4]], <2 x float>* [[TMP5]], align 4
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -624,13 +621,14 @@ define void @addsub0f(float* noalias %dst, float* noalias %src) {
 ; CHECK-NEXT:    [[INCDEC_PTR3:%.*]] = getelementptr inbounds float, float* [[DST]], i64 2
 ; CHECK-NEXT:    store float [[TMP1]], float* [[INCDEC_PTR1]], align 4
 ; CHECK-NEXT:    [[INCDEC_PTR4:%.*]] = getelementptr inbounds float, float* [[SRC]], i64 3
-; CHECK-NEXT:    [[TMP2:%.*]] = load float, float* [[INCDEC_PTR2]], align 4
-; CHECK-NEXT:    [[SUB5:%.*]] = fadd fast float [[TMP2]], -2.000000e+00
 ; CHECK-NEXT:    [[INCDEC_PTR6:%.*]] = getelementptr inbounds float, float* [[DST]], i64 3
-; CHECK-NEXT:    store float [[SUB5]], float* [[INCDEC_PTR3]], align 4
-; CHECK-NEXT:    [[TMP3:%.*]] = load float, float* [[INCDEC_PTR4]], align 4
-; CHECK-NEXT:    [[SUB8:%.*]] = fsub fast float [[TMP3]], -3.000000e+00
-; CHECK-NEXT:    store float [[SUB8]], float* [[INCDEC_PTR6]], align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast float* [[INCDEC_PTR2]] to <2 x float>*
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x float>, <2 x float>* [[TMP2]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = fadd fast <2 x float> [[TMP3]], <float -2.000000e+00, float -3.000000e+00>
+; CHECK-NEXT:    [[TMP5:%.*]] = fsub fast <2 x float> [[TMP3]], <float -2.000000e+00, float -3.000000e+00>
+; CHECK-NEXT:    [[TMP6:%.*]] = shufflevector <2 x float> [[TMP4]], <2 x float> [[TMP5]], <2 x i32> <i32 0, i32 3>
+; CHECK-NEXT:    [[TMP7:%.*]] = bitcast float* [[INCDEC_PTR3]] to <2 x float>*
+; CHECK-NEXT:    store <2 x float> [[TMP6]], <2 x float>* [[TMP7]], align 4
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -658,21 +656,22 @@ define void @addsub1f(float* noalias %dst, float* noalias %src) {
 ; CHECK-LABEL: @addsub1f(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[INCDEC_PTR:%.*]] = getelementptr inbounds float, float* [[SRC:%.*]], i64 1
-; CHECK-NEXT:    [[TMP0:%.*]] = load float, float* [[SRC]], align 4
-; CHECK-NEXT:    [[SUB:%.*]] = fadd fast float [[TMP0]], -1.000000e+00
 ; CHECK-NEXT:    [[INCDEC_PTR1:%.*]] = getelementptr inbounds float, float* [[DST:%.*]], i64 1
-; CHECK-NEXT:    store float [[SUB]], float* [[DST]], align 4
 ; CHECK-NEXT:    [[INCDEC_PTR2:%.*]] = getelementptr inbounds float, float* [[SRC]], i64 2
-; CHECK-NEXT:    [[TMP1:%.*]] = load float, float* [[INCDEC_PTR]], align 4
-; CHECK-NEXT:    [[SUB1:%.*]] = fsub fast float [[TMP1]], -1.000000e+00
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast float* [[SRC]] to <2 x float>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x float>, <2 x float>* [[TMP0]], align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = fadd fast <2 x float> [[TMP1]], <float -1.000000e+00, float -1.000000e+00>
+; CHECK-NEXT:    [[TMP3:%.*]] = fsub fast <2 x float> [[TMP1]], <float -1.000000e+00, float -1.000000e+00>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <2 x float> [[TMP2]], <2 x float> [[TMP3]], <2 x i32> <i32 0, i32 3>
 ; CHECK-NEXT:    [[INCDEC_PTR3:%.*]] = getelementptr inbounds float, float* [[DST]], i64 2
-; CHECK-NEXT:    store float [[SUB1]], float* [[INCDEC_PTR1]], align 4
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast float* [[DST]] to <2 x float>*
+; CHECK-NEXT:    store <2 x float> [[TMP4]], <2 x float>* [[TMP5]], align 4
 ; CHECK-NEXT:    [[INCDEC_PTR4:%.*]] = getelementptr inbounds float, float* [[SRC]], i64 3
-; CHECK-NEXT:    [[TMP2:%.*]] = load float, float* [[INCDEC_PTR2]], align 4
+; CHECK-NEXT:    [[TMP6:%.*]] = load float, float* [[INCDEC_PTR2]], align 4
 ; CHECK-NEXT:    [[INCDEC_PTR6:%.*]] = getelementptr inbounds float, float* [[DST]], i64 3
-; CHECK-NEXT:    store float [[TMP2]], float* [[INCDEC_PTR3]], align 4
-; CHECK-NEXT:    [[TMP3:%.*]] = load float, float* [[INCDEC_PTR4]], align 4
-; CHECK-NEXT:    [[SUB8:%.*]] = fsub fast float [[TMP3]], -3.000000e+00
+; CHECK-NEXT:    store float [[TMP6]], float* [[INCDEC_PTR3]], align 4
+; CHECK-NEXT:    [[TMP7:%.*]] = load float, float* [[INCDEC_PTR4]], align 4
+; CHECK-NEXT:    [[SUB8:%.*]] = fsub fast float [[TMP7]], -3.000000e+00
 ; CHECK-NEXT:    store float [[SUB8]], float* [[INCDEC_PTR6]], align 4
 ; CHECK-NEXT:    ret void
 ;
@@ -701,21 +700,20 @@ define void @mulf(float* noalias %dst, float* noalias %src) {
 ; CHECK-LABEL: @mulf(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[INCDEC_PTR:%.*]] = getelementptr inbounds float, float* [[SRC:%.*]], i64 1
-; CHECK-NEXT:    [[TMP0:%.*]] = load float, float* [[SRC]], align 4
-; CHECK-NEXT:    [[SUB:%.*]] = fmul fast float [[TMP0]], 2.570000e+02
 ; CHECK-NEXT:    [[INCDEC_PTR1:%.*]] = getelementptr inbounds float, float* [[DST:%.*]], i64 1
-; CHECK-NEXT:    store float [[SUB]], float* [[DST]], align 4
 ; CHECK-NEXT:    [[INCDEC_PTR2:%.*]] = getelementptr inbounds float, float* [[SRC]], i64 2
-; CHECK-NEXT:    [[TMP1:%.*]] = load float, float* [[INCDEC_PTR]], align 4
-; CHECK-NEXT:    [[SUB3:%.*]] = fmul fast float [[TMP1]], -3.000000e+00
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast float* [[SRC]] to <2 x float>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x float>, <2 x float>* [[TMP0]], align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = fmul fast <2 x float> <float 2.570000e+02, float -3.000000e+00>, [[TMP1]]
 ; CHECK-NEXT:    [[INCDEC_PTR4:%.*]] = getelementptr inbounds float, float* [[DST]], i64 2
-; CHECK-NEXT:    store float [[SUB3]], float* [[INCDEC_PTR1]], align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast float* [[DST]] to <2 x float>*
+; CHECK-NEXT:    store <2 x float> [[TMP2]], <2 x float>* [[TMP3]], align 4
 ; CHECK-NEXT:    [[INCDEC_PTR5:%.*]] = getelementptr inbounds float, float* [[SRC]], i64 3
-; CHECK-NEXT:    [[TMP2:%.*]] = load float, float* [[INCDEC_PTR2]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = load float, float* [[INCDEC_PTR2]], align 4
 ; CHECK-NEXT:    [[INCDEC_PTR7:%.*]] = getelementptr inbounds float, float* [[DST]], i64 3
-; CHECK-NEXT:    store float [[TMP2]], float* [[INCDEC_PTR4]], align 4
-; CHECK-NEXT:    [[TMP3:%.*]] = load float, float* [[INCDEC_PTR5]], align 4
-; CHECK-NEXT:    [[SUB9:%.*]] = fmul fast float [[TMP3]], -9.000000e+00
+; CHECK-NEXT:    store float [[TMP4]], float* [[INCDEC_PTR4]], align 4
+; CHECK-NEXT:    [[TMP5:%.*]] = load float, float* [[INCDEC_PTR5]], align 4
+; CHECK-NEXT:    [[SUB9:%.*]] = fmul fast float [[TMP5]], -9.000000e+00
 ; CHECK-NEXT:    store float [[SUB9]], float* [[INCDEC_PTR7]], align 4
 ; CHECK-NEXT:    ret void
 ;
@@ -786,17 +784,16 @@ define void @add1fn(float* noalias %dst, float* noalias %src) {
 ; CHECK-NEXT:    [[INCDEC_PTR1:%.*]] = getelementptr inbounds float, float* [[DST:%.*]], i64 1
 ; CHECK-NEXT:    store float [[TMP0]], float* [[DST]], align 4
 ; CHECK-NEXT:    [[INCDEC_PTR2:%.*]] = getelementptr inbounds float, float* [[SRC]], i64 2
-; CHECK-NEXT:    [[TMP1:%.*]] = load float, float* [[INCDEC_PTR]], align 4
-; CHECK-NEXT:    [[ADD3:%.*]] = fadd float [[TMP1]], 1.000000e+00
 ; CHECK-NEXT:    [[INCDEC_PTR4:%.*]] = getelementptr inbounds float, float* [[DST]], i64 2
-; CHECK-NEXT:    store float [[ADD3]], float* [[INCDEC_PTR1]], align 4
 ; CHECK-NEXT:    [[INCDEC_PTR5:%.*]] = getelementptr inbounds float, float* [[SRC]], i64 3
-; CHECK-NEXT:    [[TMP2:%.*]] = load float, float* [[INCDEC_PTR2]], align 4
-; CHECK-NEXT:    [[ADD6:%.*]] = fadd float [[TMP2]], 2.000000e+00
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast float* [[INCDEC_PTR]] to <2 x float>*
+; CHECK-NEXT:    [[TMP2:%.*]] = load <2 x float>, <2 x float>* [[TMP1]], align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = fadd <2 x float> <float 1.000000e+00, float 2.000000e+00>, [[TMP2]]
 ; CHECK-NEXT:    [[INCDEC_PTR7:%.*]] = getelementptr inbounds float, float* [[DST]], i64 3
-; CHECK-NEXT:    store float [[ADD6]], float* [[INCDEC_PTR4]], align 4
-; CHECK-NEXT:    [[TMP3:%.*]] = load float, float* [[INCDEC_PTR5]], align 4
-; CHECK-NEXT:    [[ADD9:%.*]] = fadd float [[TMP3]], 3.000000e+00
+; CHECK-NEXT:    [[TMP4:%.*]] = bitcast float* [[INCDEC_PTR1]] to <2 x float>*
+; CHECK-NEXT:    store <2 x float> [[TMP3]], <2 x float>* [[TMP4]], align 4
+; CHECK-NEXT:    [[TMP5:%.*]] = load float, float* [[INCDEC_PTR5]], align 4
+; CHECK-NEXT:    [[ADD9:%.*]] = fadd float [[TMP5]], 3.000000e+00
 ; CHECK-NEXT:    store float [[ADD9]], float* [[INCDEC_PTR7]], align 4
 ; CHECK-NEXT:    ret void
 ;
@@ -834,13 +831,12 @@ define void @sub0fn(float* noalias %dst, float* noalias %src) {
 ; CHECK-NEXT:    [[INCDEC_PTR4:%.*]] = getelementptr inbounds float, float* [[DST]], i64 2
 ; CHECK-NEXT:    store float [[TMP1]], float* [[INCDEC_PTR1]], align 4
 ; CHECK-NEXT:    [[INCDEC_PTR5:%.*]] = getelementptr inbounds float, float* [[SRC]], i64 3
-; CHECK-NEXT:    [[TMP2:%.*]] = load float, float* [[INCDEC_PTR2]], align 4
-; CHECK-NEXT:    [[ADD6:%.*]] = fadd float [[TMP2]], -2.000000e+00
 ; CHECK-NEXT:    [[INCDEC_PTR7:%.*]] = getelementptr inbounds float, float* [[DST]], i64 3
-; CHECK-NEXT:    store float [[ADD6]], float* [[INCDEC_PTR4]], align 4
-; CHECK-NEXT:    [[TMP3:%.*]] = load float, float* [[INCDEC_PTR5]], align 4
-; CHECK-NEXT:    [[ADD9:%.*]] = fadd float [[TMP3]], -3.000000e+00
-; CHECK-NEXT:    store float [[ADD9]], float* [[INCDEC_PTR7]], align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast float* [[INCDEC_PTR2]] to <2 x float>*
+; CHECK-NEXT:    [[TMP3:%.*]] = load <2 x float>, <2 x float>* [[TMP2]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = fadd <2 x float> <float -2.000000e+00, float -3.000000e+00>, [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = bitcast float* [[INCDEC_PTR4]] to <2 x float>*
+; CHECK-NEXT:    store <2 x float> [[TMP4]], <2 x float>* [[TMP5]], align 4
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -944,21 +940,20 @@ define void @mulfn(float* noalias %dst, float* noalias %src) {
 ; CHECK-LABEL: @mulfn(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[INCDEC_PTR:%.*]] = getelementptr inbounds float, float* [[SRC:%.*]], i64 1
-; CHECK-NEXT:    [[TMP0:%.*]] = load float, float* [[SRC]], align 4
-; CHECK-NEXT:    [[SUB:%.*]] = fmul float [[TMP0]], 2.570000e+02
 ; CHECK-NEXT:    [[INCDEC_PTR1:%.*]] = getelementptr inbounds float, float* [[DST:%.*]], i64 1
-; CHECK-NEXT:    store float [[SUB]], float* [[DST]], align 4
 ; CHECK-NEXT:    [[INCDEC_PTR2:%.*]] = getelementptr inbounds float, float* [[SRC]], i64 2
-; CHECK-NEXT:    [[TMP1:%.*]] = load float, float* [[INCDEC_PTR]], align 4
-; CHECK-NEXT:    [[SUB3:%.*]] = fmul float [[TMP1]], -3.000000e+00
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast float* [[SRC]] to <2 x float>*
+; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x float>, <2 x float>* [[TMP0]], align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = fmul <2 x float> <float 2.570000e+02, float -3.000000e+00>, [[TMP1]]
 ; CHECK-NEXT:    [[INCDEC_PTR4:%.*]] = getelementptr inbounds float, float* [[DST]], i64 2
-; CHECK-NEXT:    store float [[SUB3]], float* [[INCDEC_PTR1]], align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast float* [[DST]] to <2 x float>*
+; CHECK-NEXT:    store <2 x float> [[TMP2]], <2 x float>* [[TMP3]], align 4
 ; CHECK-NEXT:    [[INCDEC_PTR5:%.*]] = getelementptr inbounds float, float* [[SRC]], i64 3
-; CHECK-NEXT:    [[TMP2:%.*]] = load float, float* [[INCDEC_PTR2]], align 4
+; CHECK-NEXT:    [[TMP4:%.*]] = load float, float* [[INCDEC_PTR2]], align 4
 ; CHECK-NEXT:    [[INCDEC_PTR7:%.*]] = getelementptr inbounds float, float* [[DST]], i64 3
-; CHECK-NEXT:    store float [[TMP2]], float* [[INCDEC_PTR4]], align 4
-; CHECK-NEXT:    [[TMP3:%.*]] = load float, float* [[INCDEC_PTR5]], align 4
-; CHECK-NEXT:    [[SUB9:%.*]] = fmul fast float [[TMP3]], -9.000000e+00
+; CHECK-NEXT:    store float [[TMP4]], float* [[INCDEC_PTR4]], align 4
+; CHECK-NEXT:    [[TMP5:%.*]] = load float, float* [[INCDEC_PTR5]], align 4
+; CHECK-NEXT:    [[SUB9:%.*]] = fmul fast float [[TMP5]], -9.000000e+00
 ; CHECK-NEXT:    store float [[SUB9]], float* [[INCDEC_PTR7]], align 4
 ; CHECK-NEXT:    ret void
 ;
diff --git a/test/Transforms/SLPVectorizer/X86/vector_gep.ll b/test/Transforms/SLPVectorizer/X86/vector_gep.ll
index 595a77f0cfbf412520a9a0a5afb69b9ab77cba3e..436f091d7242433fe9b0a09012d685d6f13950cb 100644
--- a/test/Transforms/SLPVectorizer/X86/vector_gep.ll
+++ b/test/Transforms/SLPVectorizer/X86/vector_gep.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ;RUN: opt < %s -slp-vectorizer -S | FileCheck %s
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
@@ -8,9 +9,14 @@ target triple = "x86_64-unknown-linux-gnu"
 
 ; Function Attrs: noreturn readonly uwtable
 define void @_Z3fn1v(i32 %x, <16 x i32*>%y) local_unnamed_addr #0 {
-; CHECK-LABEL: _Z3fn1v
-; CHECK: getelementptr i32, <16 x i32*>
-; CHECK: getelementptr i32, <16 x i32*>
+; CHECK-LABEL: @_Z3fn1v(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CONV42_LE:%.*]] = sext i32 [[X:%.*]] to i64
+; CHECK-NEXT:    [[CONV36109_LE:%.*]] = zext i32 2 to i64
+; CHECK-NEXT:    [[VECTORGEP:%.*]] = getelementptr i32, <16 x i32*> [[Y:%.*]], i64 [[CONV36109_LE]]
+; CHECK-NEXT:    [[VECTORGEP208:%.*]] = getelementptr i32, <16 x i32*> [[Y]], i64 [[CONV42_LE]]
+; CHECK-NEXT:    unreachable
+;
 
 entry:
   %conv42.le = sext i32 %x to i64
diff --git a/test/Transforms/SLPVectorizer/X86/zext.ll b/test/Transforms/SLPVectorizer/X86/zext.ll
index a05e41861349f740de0e963f0aef88915dbff429..02684e18d9b2437709a26faf43fdbb4188d96239 100644
--- a/test/Transforms/SLPVectorizer/X86/zext.ll
+++ b/test/Transforms/SLPVectorizer/X86/zext.ll
@@ -682,18 +682,20 @@ define <4 x i64> @loadext_4i32_to_4i64(i32* %p0) {
 ; SSE2-NEXT:    [[P1:%.*]] = getelementptr inbounds i32, i32* [[P0:%.*]], i64 1
 ; SSE2-NEXT:    [[P2:%.*]] = getelementptr inbounds i32, i32* [[P0]], i64 2
 ; SSE2-NEXT:    [[P3:%.*]] = getelementptr inbounds i32, i32* [[P0]], i64 3
-; SSE2-NEXT:    [[I0:%.*]] = load i32, i32* [[P0]], align 1
-; SSE2-NEXT:    [[I1:%.*]] = load i32, i32* [[P1]], align 1
-; SSE2-NEXT:    [[I2:%.*]] = load i32, i32* [[P2]], align 1
-; SSE2-NEXT:    [[I3:%.*]] = load i32, i32* [[P3]], align 1
-; SSE2-NEXT:    [[X0:%.*]] = zext i32 [[I0]] to i64
-; SSE2-NEXT:    [[X1:%.*]] = zext i32 [[I1]] to i64
-; SSE2-NEXT:    [[X2:%.*]] = zext i32 [[I2]] to i64
-; SSE2-NEXT:    [[X3:%.*]] = zext i32 [[I3]] to i64
-; SSE2-NEXT:    [[V0:%.*]] = insertelement <4 x i64> undef, i64 [[X0]], i32 0
-; SSE2-NEXT:    [[V1:%.*]] = insertelement <4 x i64> [[V0]], i64 [[X1]], i32 1
-; SSE2-NEXT:    [[V2:%.*]] = insertelement <4 x i64> [[V1]], i64 [[X2]], i32 2
-; SSE2-NEXT:    [[V3:%.*]] = insertelement <4 x i64> [[V2]], i64 [[X3]], i32 3
+; SSE2-NEXT:    [[TMP1:%.*]] = bitcast i32* [[P0]] to <2 x i32>*
+; SSE2-NEXT:    [[TMP2:%.*]] = load <2 x i32>, <2 x i32>* [[TMP1]], align 1
+; SSE2-NEXT:    [[TMP3:%.*]] = bitcast i32* [[P2]] to <2 x i32>*
+; SSE2-NEXT:    [[TMP4:%.*]] = load <2 x i32>, <2 x i32>* [[TMP3]], align 1
+; SSE2-NEXT:    [[TMP5:%.*]] = zext <2 x i32> [[TMP2]] to <2 x i64>
+; SSE2-NEXT:    [[TMP6:%.*]] = zext <2 x i32> [[TMP4]] to <2 x i64>
+; SSE2-NEXT:    [[TMP7:%.*]] = extractelement <2 x i64> [[TMP5]], i32 0
+; SSE2-NEXT:    [[V0:%.*]] = insertelement <4 x i64> undef, i64 [[TMP7]], i32 0
+; SSE2-NEXT:    [[TMP8:%.*]] = extractelement <2 x i64> [[TMP5]], i32 1
+; SSE2-NEXT:    [[V1:%.*]] = insertelement <4 x i64> [[V0]], i64 [[TMP8]], i32 1
+; SSE2-NEXT:    [[TMP9:%.*]] = extractelement <2 x i64> [[TMP6]], i32 0
+; SSE2-NEXT:    [[V2:%.*]] = insertelement <4 x i64> [[V1]], i64 [[TMP9]], i32 2
+; SSE2-NEXT:    [[TMP10:%.*]] = extractelement <2 x i64> [[TMP6]], i32 1
+; SSE2-NEXT:    [[V3:%.*]] = insertelement <4 x i64> [[V2]], i64 [[TMP10]], i32 3
 ; SSE2-NEXT:    ret <4 x i64> [[V3]]
 ;
 ; SLM-LABEL: @loadext_4i32_to_4i64(
@@ -719,17 +721,18 @@ define <4 x i64> @loadext_4i32_to_4i64(i32* %p0) {
 ; AVX1-NEXT:    [[P3:%.*]] = getelementptr inbounds i32, i32* [[P0]], i64 3
 ; AVX1-NEXT:    [[TMP1:%.*]] = bitcast i32* [[P0]] to <2 x i32>*
 ; AVX1-NEXT:    [[TMP2:%.*]] = load <2 x i32>, <2 x i32>* [[TMP1]], align 1
-; AVX1-NEXT:    [[I2:%.*]] = load i32, i32* [[P2]], align 1
-; AVX1-NEXT:    [[I3:%.*]] = load i32, i32* [[P3]], align 1
-; AVX1-NEXT:    [[TMP3:%.*]] = zext <2 x i32> [[TMP2]] to <2 x i64>
-; AVX1-NEXT:    [[X2:%.*]] = zext i32 [[I2]] to i64
-; AVX1-NEXT:    [[X3:%.*]] = zext i32 [[I3]] to i64
-; AVX1-NEXT:    [[TMP4:%.*]] = extractelement <2 x i64> [[TMP3]], i32 0
-; AVX1-NEXT:    [[V0:%.*]] = insertelement <4 x i64> undef, i64 [[TMP4]], i32 0
-; AVX1-NEXT:    [[TMP5:%.*]] = extractelement <2 x i64> [[TMP3]], i32 1
-; AVX1-NEXT:    [[V1:%.*]] = insertelement <4 x i64> [[V0]], i64 [[TMP5]], i32 1
-; AVX1-NEXT:    [[V2:%.*]] = insertelement <4 x i64> [[V1]], i64 [[X2]], i32 2
-; AVX1-NEXT:    [[V3:%.*]] = insertelement <4 x i64> [[V2]], i64 [[X3]], i32 3
+; AVX1-NEXT:    [[TMP3:%.*]] = bitcast i32* [[P2]] to <2 x i32>*
+; AVX1-NEXT:    [[TMP4:%.*]] = load <2 x i32>, <2 x i32>* [[TMP3]], align 1
+; AVX1-NEXT:    [[TMP5:%.*]] = zext <2 x i32> [[TMP2]] to <2 x i64>
+; AVX1-NEXT:    [[TMP6:%.*]] = zext <2 x i32> [[TMP4]] to <2 x i64>
+; AVX1-NEXT:    [[TMP7:%.*]] = extractelement <2 x i64> [[TMP5]], i32 0
+; AVX1-NEXT:    [[V0:%.*]] = insertelement <4 x i64> undef, i64 [[TMP7]], i32 0
+; AVX1-NEXT:    [[TMP8:%.*]] = extractelement <2 x i64> [[TMP5]], i32 1
+; AVX1-NEXT:    [[V1:%.*]] = insertelement <4 x i64> [[V0]], i64 [[TMP8]], i32 1
+; AVX1-NEXT:    [[TMP9:%.*]] = extractelement <2 x i64> [[TMP6]], i32 0
+; AVX1-NEXT:    [[V2:%.*]] = insertelement <4 x i64> [[V1]], i64 [[TMP9]], i32 2
+; AVX1-NEXT:    [[TMP10:%.*]] = extractelement <2 x i64> [[TMP6]], i32 1
+; AVX1-NEXT:    [[V3:%.*]] = insertelement <4 x i64> [[V2]], i64 [[TMP10]], i32 3
 ; AVX1-NEXT:    ret <4 x i64> [[V3]]
 ;
 ; AVX2-LABEL: @loadext_4i32_to_4i64(
diff --git a/test/Transforms/SLPVectorizer/XCore/no-vector-registers.ll b/test/Transforms/SLPVectorizer/XCore/no-vector-registers.ll
index efd5386f520e0c7b49068bd6dd227b6bad0966db..d84a4993ec304fdf44c062fbaced7cd15ae40c9e 100644
--- a/test/Transforms/SLPVectorizer/XCore/no-vector-registers.ll
+++ b/test/Transforms/SLPVectorizer/XCore/no-vector-registers.ll
@@ -1,12 +1,26 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -basicaa -slp-vectorizer -dce -S -mtriple=xcore  | FileCheck %s
 
 target datalayout = "e-p:32:32:32-a0:0:32-n32-i1:8:32-i8:8:32-i16:16:32-i32:32:32-i64:32:32-f16:16:32-f32:32:32-f64:32:32"
 target triple = "xcore"
 
 ; Simple 3-pair chain with loads and stores
-; CHECK: test1
-; CHECK-NOT: <2 x double>
 define void @test1(double* %a, double* %b, double* %c) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[I0:%.*]] = load double, double* [[A:%.*]], align 8
+; CHECK-NEXT:    [[I1:%.*]] = load double, double* [[B:%.*]], align 8
+; CHECK-NEXT:    [[MUL:%.*]] = fmul double [[I0]], [[I1]]
+; CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds double, double* [[A]], i64 1
+; CHECK-NEXT:    [[I3:%.*]] = load double, double* [[ARRAYIDX3]], align 8
+; CHECK-NEXT:    [[ARRAYIDX4:%.*]] = getelementptr inbounds double, double* [[B]], i64 1
+; CHECK-NEXT:    [[I4:%.*]] = load double, double* [[ARRAYIDX4]], align 8
+; CHECK-NEXT:    [[MUL5:%.*]] = fmul double [[I3]], [[I4]]
+; CHECK-NEXT:    store double [[MUL]], double* [[C:%.*]], align 8
+; CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds double, double* [[C]], i64 1
+; CHECK-NEXT:    store double [[MUL5]], double* [[ARRAYIDX5]], align 8
+; CHECK-NEXT:    ret void
+;
 entry:
   %i0 = load double, double* %a, align 8
   %i1 = load double, double* %b, align 8
diff --git a/test/Transforms/SLPVectorizer/int_sideeffect.ll b/test/Transforms/SLPVectorizer/int_sideeffect.ll
index a6123c11d92569337a17a6145dd5cb7aadbbdb8d..aab3f028f8d165dc506672461661726904b553d6 100644
--- a/test/Transforms/SLPVectorizer/int_sideeffect.ll
+++ b/test/Transforms/SLPVectorizer/int_sideeffect.ll
@@ -1,25 +1,37 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -S < %s -slp-vectorizer -slp-max-reg-size=128 -slp-min-reg-size=128 | FileCheck %s
 
 declare void @llvm.sideeffect()
 
 ; SLP vectorization across a @llvm.sideeffect.
 
-; CHECK-LABEL: test
-; CHECK: store <4 x float>
 define void @test(float* %p) {
-    %p0 = getelementptr float, float* %p, i64 0
-    %p1 = getelementptr float, float* %p, i64 1
-    %p2 = getelementptr float, float* %p, i64 2
-    %p3 = getelementptr float, float* %p, i64 3
-    %l0 = load float, float* %p0
-    %l1 = load float, float* %p1
-    %l2 = load float, float* %p2
-    call void @llvm.sideeffect()
-    %l3 = load float, float* %p3
-    store float %l0, float* %p0
-    call void @llvm.sideeffect()
-    store float %l1, float* %p1
-    store float %l2, float* %p2
-    store float %l3, float* %p3
-    ret void
+; CHECK-LABEL: @test(
+; CHECK-NEXT:    [[P0:%.*]] = getelementptr float, float* [[P:%.*]], i64 0
+; CHECK-NEXT:    [[P1:%.*]] = getelementptr float, float* [[P]], i64 1
+; CHECK-NEXT:    [[P2:%.*]] = getelementptr float, float* [[P]], i64 2
+; CHECK-NEXT:    [[P3:%.*]] = getelementptr float, float* [[P]], i64 3
+; CHECK-NEXT:    call void @llvm.sideeffect()
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast float* [[P0]] to <4 x float>*
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x float>, <4 x float>* [[TMP1]], align 4
+; CHECK-NEXT:    call void @llvm.sideeffect()
+; CHECK-NEXT:    [[TMP3:%.*]] = bitcast float* [[P0]] to <4 x float>*
+; CHECK-NEXT:    store <4 x float> [[TMP2]], <4 x float>* [[TMP3]], align 4
+; CHECK-NEXT:    ret void
+;
+  %p0 = getelementptr float, float* %p, i64 0
+  %p1 = getelementptr float, float* %p, i64 1
+  %p2 = getelementptr float, float* %p, i64 2
+  %p3 = getelementptr float, float* %p, i64 3
+  %l0 = load float, float* %p0
+  %l1 = load float, float* %p1
+  %l2 = load float, float* %p2
+  call void @llvm.sideeffect()
+  %l3 = load float, float* %p3
+  store float %l0, float* %p0
+  call void @llvm.sideeffect()
+  store float %l1, float* %p1
+  store float %l2, float* %p2
+  store float %l3, float* %p3
+  ret void
 }
diff --git a/test/Transforms/SROA/basictest.ll b/test/Transforms/SROA/basictest.ll
index a72da6399f22c1cdcf619c69c264087981d3f743..2c5829d6fcecbec9978b55efdcbae07d3c0a9d64 100644
--- a/test/Transforms/SROA/basictest.ll
+++ b/test/Transforms/SROA/basictest.ll
@@ -1745,6 +1745,55 @@ entry:
   ret void
 }
 
+declare void @llvm.lifetime.start.isVoid.i64.p0i8(i64, [10 x float]* nocapture)
+declare void @llvm.lifetime.end.isVoid.i64.p0i8(i64, [10 x float]* nocapture)
+@array = dso_local global [10 x float] undef, align 4
+
+define void @test29(i32 %num, i32 %tid) {
+; CHECK-LABEL: @test29(
+; CHECK-NOT: alloca [10 x float]
+; CHECK: ret void
+
+entry:
+  %ra = alloca [10 x float], align 4
+  call void @llvm.lifetime.start.isVoid.i64.p0i8(i64 40, [10 x float]* nonnull %ra)
+
+  %cmp1 = icmp sgt i32 %num, 0
+  br i1 %cmp1, label %bb1, label %bb7
+
+bb1:
+  %tobool = icmp eq i32 %tid, 0
+  %conv.i = zext i32 %tid to i64
+  %0 = bitcast [10 x float]* %ra to i32*
+  %1 = load i32, i32* %0, align 4
+  %arrayidx5 = getelementptr inbounds [10 x float], [10 x float]* @array, i64 0, i64 %conv.i
+  %2 = bitcast float* %arrayidx5 to i32*
+  br label %bb2
+
+bb2:
+  %i.02 = phi i32 [ %num, %bb1 ], [ %sub, %bb5 ]
+  br i1 %tobool, label %bb3, label %bb4
+
+bb3:
+  br label %bb5
+
+bb4:
+  store i32 %1, i32* %2, align 4
+  br label %bb5
+
+bb5:
+  %sub = add i32 %i.02, -1
+  %cmp = icmp sgt i32 %sub, 0
+  br i1 %cmp, label %bb2, label %bb6
+
+bb6:
+  br label %bb7
+
+bb7:
+  call void @llvm.lifetime.end.isVoid.i64.p0i8(i64 40, [10 x float]* nonnull %ra)
+  ret void
+}
+
 !0 = !{!1, !1, i64 0, i64 1}
 !1 = !{!2, i64 1, !"type_0"}
 !2 = !{!"root"}
diff --git a/test/Transforms/SROA/mem-par-metadata-sroa.ll b/test/Transforms/SROA/mem-par-metadata-sroa.ll
index 32ea8f5732ab0beb422c9fc079c02db9f3fe4c77..577245cda60416ade856bc24c52d77e638872b7e 100644
--- a/test/Transforms/SROA/mem-par-metadata-sroa.ll
+++ b/test/Transforms/SROA/mem-par-metadata-sroa.ll
@@ -1,6 +1,6 @@
 ; RUN: opt < %s -sroa -S | FileCheck %s
 ;
-; Make sure the llvm.mem.parallel_loop_access meta-data is preserved
+; Make sure the llvm.access.group meta-data is preserved
 ; when a load/store is replaced with another load/store by sroa
 ;
 ; class Complex {
@@ -33,9 +33,9 @@
 
 ; CHECK: for.body:
 ; CHECK-NOT:  store i32 %{{.*}}, i32* %{{.*}}, align 4
-; CHECK: store i32 %{{.*}}, i32* %{{.*}}, align 4, !llvm.mem.parallel_loop_access !1
+; CHECK: store i32 %{{.*}}, i32* %{{.*}}, align 4, !llvm.access.group !1
 ; CHECK-NOT:  store i32 %{{.*}}, i32* %{{.*}}, align 4
-; CHECK: store i32 %{{.*}}, i32* %{{.*}}, align 4, !llvm.mem.parallel_loop_access !1
+; CHECK: store i32 %{{.*}}, i32* %{{.*}}, align 4, !llvm.access.group !1
 ; CHECK-NOT:  store i32 %{{.*}}, i32* %{{.*}}, align 4
 ; CHECK: br label
 
@@ -63,30 +63,30 @@ for.body:                                         ; preds = %for.cond
   %arrayidx = getelementptr inbounds %class.Complex, %class.Complex* %out, i64 %offset.0
   %real_.i = getelementptr inbounds %class.Complex, %class.Complex* %t0, i64 0, i32 0
   %real_.i.i = getelementptr inbounds %class.Complex, %class.Complex* %arrayidx, i64 0, i32 0
-  %0 = load float, float* %real_.i.i, align 4, !llvm.mem.parallel_loop_access !1
-  store float %0, float* %real_.i, align 4, !llvm.mem.parallel_loop_access !1
+  %0 = load float, float* %real_.i.i, align 4, !llvm.access.group !11
+  store float %0, float* %real_.i, align 4, !llvm.access.group !11
   %imaginary_.i = getelementptr inbounds %class.Complex, %class.Complex* %t0, i64 0, i32 1
   %imaginary_.i.i = getelementptr inbounds %class.Complex, %class.Complex* %arrayidx, i64 0, i32 1
-  %1 = load float, float* %imaginary_.i.i, align 4, !llvm.mem.parallel_loop_access !1
-  store float %1, float* %imaginary_.i, align 4, !llvm.mem.parallel_loop_access !1
+  %1 = load float, float* %imaginary_.i.i, align 4, !llvm.access.group !11
+  store float %1, float* %imaginary_.i, align 4, !llvm.access.group !11
   %arrayidx1 = getelementptr inbounds %class.Complex, %class.Complex* %out, i64 %offset.0
   %real_.i1 = getelementptr inbounds %class.Complex, %class.Complex* %t0, i64 0, i32 0
-  %2 = load float, float* %real_.i1, align 4, !noalias !3, !llvm.mem.parallel_loop_access !1
+  %2 = load float, float* %real_.i1, align 4, !noalias !3, !llvm.access.group !11
   %real_2.i = getelementptr inbounds %class.Complex, %class.Complex* %t0, i64 0, i32 0
-  %3 = load float, float* %real_2.i, align 4, !noalias !3, !llvm.mem.parallel_loop_access !1
+  %3 = load float, float* %real_2.i, align 4, !noalias !3, !llvm.access.group !11
   %add.i = fadd float %2, %3
   %imaginary_.i2 = getelementptr inbounds %class.Complex, %class.Complex* %t0, i64 0, i32 1
-  %4 = load float, float* %imaginary_.i2, align 4, !noalias !3, !llvm.mem.parallel_loop_access !1
+  %4 = load float, float* %imaginary_.i2, align 4, !noalias !3, !llvm.access.group !11
   %imaginary_3.i = getelementptr inbounds %class.Complex, %class.Complex* %t0, i64 0, i32 1
-  %5 = load float, float* %imaginary_3.i, align 4, !noalias !3, !llvm.mem.parallel_loop_access !1
+  %5 = load float, float* %imaginary_3.i, align 4, !noalias !3, !llvm.access.group !11
   %add4.i = fadd float %4, %5
   %real_.i.i3 = getelementptr inbounds %class.Complex, %class.Complex* %tmpcast, i64 0, i32 0
-  store float %add.i, float* %real_.i.i3, align 4, !alias.scope !3, !llvm.mem.parallel_loop_access !1
+  store float %add.i, float* %real_.i.i3, align 4, !alias.scope !3, !llvm.access.group !11
   %imaginary_.i.i4 = getelementptr inbounds %class.Complex, %class.Complex* %tmpcast, i64 0, i32 1
-  store float %add4.i, float* %imaginary_.i.i4, align 4, !alias.scope !3, !llvm.mem.parallel_loop_access !1
+  store float %add4.i, float* %imaginary_.i.i4, align 4, !alias.scope !3, !llvm.access.group !11
   %6 = bitcast %class.Complex* %arrayidx1 to i64*
-  %7 = load i64, i64* %ref.tmp, align 8, !llvm.mem.parallel_loop_access !1
-  store i64 %7, i64* %6, align 4, !llvm.mem.parallel_loop_access !1
+  %7 = load i64, i64* %ref.tmp, align 8, !llvm.access.group !11
+  store i64 %7, i64* %6, align 4, !llvm.access.group !11
   %inc = add nsw i64 %offset.0, 1
   br label %for.cond, !llvm.loop !1
 
@@ -103,8 +103,9 @@ attributes #1 = { argmemonly nounwind }
 !llvm.ident = !{!0}
 
 !0 = !{!"clang version 4.0.0 (cfe/trunk 277751)"}
-!1 = distinct !{!1, !2}
+!1 = distinct !{!1, !2, !{!"llvm.loop.parallel_accesses", !11}}
 !2 = !{!"llvm.loop.vectorize.enable", i1 true}
 !3 = !{!4}
 !4 = distinct !{!4, !5, !"_ZNK7ComplexplERKS_: %agg.result"}
 !5 = distinct !{!5, !"_ZNK7ComplexplERKS_"}
+!11 = distinct !{}
diff --git a/test/Transforms/SROA/phi-and-select.ll b/test/Transforms/SROA/phi-and-select.ll
index e7ba2e89d7956d5356ca845db9e938bd11cec881..d0904cecd9f18a24bb1c1a62c1851ec4133b53ea 100644
--- a/test/Transforms/SROA/phi-and-select.ll
+++ b/test/Transforms/SROA/phi-and-select.ll
@@ -632,3 +632,15 @@ exit:
   %result = load i32, i32* %phi, align 4
   ret i32 %result
 }
+
+; Don't speculate a load based on an earlier volatile operation.
+define i8 @volatile_select(i8* %p, i1 %b) {
+; CHECK-LABEL: @volatile_select(
+; CHECK: select i1 %b, i8* %p, i8* %p2
+  %p2 = alloca i8
+  store i8 0, i8* %p2
+  store volatile i8 0, i8* %p
+  %px = select i1 %b, i8* %p, i8* %p2
+  %v2 = load i8, i8* %px
+  ret i8 %v2
+}
diff --git a/test/Transforms/SampleProfile/Inputs/entry_counts_cold.prof b/test/Transforms/SampleProfile/Inputs/entry_counts_cold.prof
new file mode 100644
index 0000000000000000000000000000000000000000..cd7e871d6c21bfc0d4972cc5cd951baef22be7b5
--- /dev/null
+++ b/test/Transforms/SampleProfile/Inputs/entry_counts_cold.prof
@@ -0,0 +1,20 @@
+top:200:100
+ 1: 100 foo:100
+ 2: 100
+ 3: 2
+ 4: 100
+ 1: foo:100
+    2: 100
+    3: 100 bar:100
+    4: 100
+ 3: bar:2
+    1: 2
+    2: 2
+foo:200:150
+ 2: 150
+ 3: 150  bar:150
+ 4: 150
+bar:450:300
+ 1: 300 baz:300
+ 2: 300
+ 3: 300
diff --git a/test/Transforms/SampleProfile/Inputs/flattened.prof b/test/Transforms/SampleProfile/Inputs/flattened.prof
new file mode 100644
index 0000000000000000000000000000000000000000..962bc6e58e5509b7d30e49473efb6b85c322a39e
--- /dev/null
+++ b/test/Transforms/SampleProfile/Inputs/flattened.prof
@@ -0,0 +1,2 @@
+foo:100:100
+ 1: 100
diff --git a/test/Transforms/SampleProfile/entry_counts_cold.ll b/test/Transforms/SampleProfile/entry_counts_cold.ll
new file mode 100644
index 0000000000000000000000000000000000000000..da60ebec6d597d889bc57c1a13474a972d2712e3
--- /dev/null
+++ b/test/Transforms/SampleProfile/entry_counts_cold.ll
@@ -0,0 +1,170 @@
+; RUN: opt < %s -sample-profile -sample-profile-file=%S/Inputs/entry_counts_cold.prof -S | FileCheck %s
+; ModuleID = 'temp.bc'
+source_filename = "temp.c"
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.14.0"
+
+; Function Attrs: nounwind ssp uwtable
+; CHECK: define i32 @top({{.*}} !prof [[TOP:![0-9]+]] 
+define i32 @top(i32* %p) #0 !dbg !8 {
+entry:
+  %p.addr = alloca i32*, align 8
+  store i32* %p, i32** %p.addr, align 8, !tbaa !15
+  call void @llvm.dbg.declare(metadata i32** %p.addr, metadata !14, metadata !DIExpression()), !dbg !19
+  %0 = load i32*, i32** %p.addr, align 8, !dbg !20, !tbaa !15
+  %call = call i32 @foo(i32* %0), !dbg !21
+; foo is inlined
+; CHECK-NOT: call i32 @foo
+; CHECK: call i32 @bar
+  %1 = load i32*, i32** %p.addr, align 8, !dbg !22, !tbaa !15
+  %2 = load i32, i32* %1, align 4, !dbg !24, !tbaa !25
+  %tobool = icmp ne i32 %2, 0, !dbg !24
+  br i1 %tobool, label %if.then, label %if.end, !dbg !27
+
+if.then:                                          ; preds = %entry
+  %3 = load i32*, i32** %p.addr, align 8, !dbg !28, !tbaa !15
+; bar is not inlined
+; CHECK: call i32 @bar
+  %call1 = call i32 @bar(i32* %3), !dbg !29
+  br label %if.end, !dbg !29
+
+if.end:                                           ; preds = %if.then, %entry
+  ret i32 0, !dbg !30
+}
+
+; Function Attrs: nounwind readnone speculatable
+declare void @llvm.dbg.declare(metadata, metadata, metadata) #1
+
+; Function Attrs: nounwind ssp uwtable
+; CHECK: define i32 @foo({{.*}} !prof [[FOO:![0-9]+]] 
+define i32 @foo(i32* %p) #0 !dbg !31 {
+entry:
+  %p.addr = alloca i32*, align 8
+  %a = alloca i32, align 4
+  store i32* %p, i32** %p.addr, align 8, !tbaa !15
+  call void @llvm.dbg.declare(metadata i32** %p.addr, metadata !33, metadata !DIExpression()), !dbg !35
+  %0 = bitcast i32* %a to i8*, !dbg !36
+  call void @llvm.lifetime.start.p0i8(i64 4, i8* %0) #4, !dbg !36
+  call void @llvm.dbg.declare(metadata i32* %a, metadata !34, metadata !DIExpression()), !dbg !37
+  %1 = load i32*, i32** %p.addr, align 8, !dbg !38, !tbaa !15
+  %arrayidx = getelementptr inbounds i32, i32* %1, i64 3, !dbg !38
+  %2 = load i32, i32* %arrayidx, align 4, !dbg !38, !tbaa !25
+  %3 = load i32*, i32** %p.addr, align 8, !dbg !39, !tbaa !15
+  %arrayidx1 = getelementptr inbounds i32, i32* %3, i64 2, !dbg !39
+  %4 = load i32, i32* %arrayidx1, align 4, !dbg !40, !tbaa !25
+  %add = add nsw i32 %4, %2, !dbg !40
+  store i32 %add, i32* %arrayidx1, align 4, !dbg !40, !tbaa !25
+  %5 = load i32*, i32** %p.addr, align 8, !dbg !41, !tbaa !15
+  %call = call i32 @bar(i32* %5), !dbg !42
+  store i32 %call, i32* %a, align 4, !dbg !43, !tbaa !25
+  %6 = load i32, i32* %a, align 4, !dbg !44, !tbaa !25
+  %add2 = add nsw i32 %6, 1, !dbg !45
+  %7 = bitcast i32* %a to i8*, !dbg !46
+  call void @llvm.lifetime.end.p0i8(i64 4, i8* %7) #4, !dbg !46
+  ret i32 %add2, !dbg !47
+}
+
+; Function Attrs: nounwind ssp uwtable
+; CHECK: define i32 @bar({{.*}} !prof [[BAR:![0-9]+]] 
+define i32 @bar(i32* %p) #0 !dbg !48 {
+entry:
+  %p.addr = alloca i32*, align 8
+  store i32* %p, i32** %p.addr, align 8, !tbaa !15
+  call void @llvm.dbg.declare(metadata i32** %p.addr, metadata !50, metadata !DIExpression()), !dbg !51
+  ; CHECK: call void (...) @baz{{.*}} !prof [[BAZ:![0-9]+]]
+  call void (...) @baz(), !dbg !52
+  %0 = load i32*, i32** %p.addr, align 8, !dbg !53, !tbaa !15
+  %arrayidx = getelementptr inbounds i32, i32* %0, i64 2, !dbg !53
+  %1 = load i32, i32* %arrayidx, align 4, !dbg !53, !tbaa !25
+  %2 = load i32*, i32** %p.addr, align 8, !dbg !54, !tbaa !15
+  %arrayidx1 = getelementptr inbounds i32, i32* %2, i64 1, !dbg !54
+  %3 = load i32, i32* %arrayidx1, align 4, !dbg !55, !tbaa !25
+  %add = add nsw i32 %3, %1, !dbg !55
+  store i32 %add, i32* %arrayidx1, align 4, !dbg !55, !tbaa !25
+  %4 = load i32*, i32** %p.addr, align 8, !dbg !56, !tbaa !15
+  %arrayidx2 = getelementptr inbounds i32, i32* %4, i64 3, !dbg !56
+  %5 = load i32, i32* %arrayidx2, align 4, !dbg !56, !tbaa !25
+  ret i32 %5, !dbg !57
+}
+
+; Function Attrs: argmemonly nounwind
+declare void @llvm.lifetime.start.p0i8(i64, i8* nocapture) #2
+
+; Function Attrs: argmemonly nounwind
+declare void @llvm.lifetime.end.p0i8(i64, i8* nocapture) #2
+
+declare void @baz(...) #3
+
+attributes #0 = { nounwind ssp uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="penryn" "target-features"="+cx16,+fxsr,+mmx,+sahf,+sse,+sse2,+sse3,+sse4.1,+ssse3,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind readnone speculatable }
+attributes #2 = { argmemonly nounwind }
+attributes #3 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="penryn" "target-features"="+cx16,+fxsr,+mmx,+sahf,+sse,+sse2,+sse3,+sse4.1,+ssse3,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #4 = { nounwind }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4, !5, !6}
+!llvm.ident = !{!7}
+
+; CHECK: [[TOP]] = !{!"function_entry_count", i64 101}
+; CHECK: [[FOO]] = !{!"function_entry_count", i64 151}
+; CHECK: [[BAR]] = !{!"function_entry_count", i64 303}
+; CHECK: [[BAZ]] = !{!"branch_weights", i64 303}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 8.0.0", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, nameTableKind: GNU)
+!1 = !DIFile(filename: "temp.c", directory: "llvm/test/Transforms/SampleProfile")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{i32 1, !"wchar_size", i32 4}
+!6 = !{i32 7, !"PIC Level", i32 2}
+!7 = !{!"clang version 8.0.0"}
+!8 = distinct !DISubprogram(name: "top", scope: !1, file: !1, line: 5, type: !9, scopeLine: 5, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !13)
+!9 = !DISubroutineType(types: !10)
+!10 = !{!11, !12}
+!11 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!12 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !11, size: 64)
+!13 = !{!14}
+!14 = !DILocalVariable(name: "p", arg: 1, scope: !8, file: !1, line: 5, type: !12)
+!15 = !{!16, !16, i64 0}
+!16 = !{!"any pointer", !17, i64 0}
+!17 = !{!"omnipotent char", !18, i64 0}
+!18 = !{!"Simple C/C++ TBAA"}
+!19 = !DILocation(line: 5, column: 14, scope: !8)
+!20 = !DILocation(line: 6, column: 7, scope: !8)
+!21 = !DILocation(line: 6, column: 3, scope: !8)
+!22 = !DILocation(line: 7, column: 8, scope: !23)
+!23 = distinct !DILexicalBlock(scope: !8, file: !1, line: 7, column: 7)
+!24 = !DILocation(line: 7, column: 7, scope: !23)
+!25 = !{!26, !26, i64 0}
+!26 = !{!"int", !17, i64 0}
+!27 = !DILocation(line: 7, column: 7, scope: !8)
+!28 = !DILocation(line: 8, column: 9, scope: !23)
+!29 = !DILocation(line: 8, column: 5, scope: !23)
+!30 = !DILocation(line: 9, column: 3, scope: !8)
+!31 = distinct !DISubprogram(name: "foo", scope: !1, file: !1, line: 12, type: !9, scopeLine: 12, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !32)
+!32 = !{!33, !34}
+!33 = !DILocalVariable(name: "p", arg: 1, scope: !31, file: !1, line: 12, type: !12)
+!34 = !DILocalVariable(name: "a", scope: !31, file: !1, line: 13, type: !11)
+!35 = !DILocation(line: 12, column: 14, scope: !31)
+!36 = !DILocation(line: 13, column: 3, scope: !31)
+!37 = !DILocation(line: 13, column: 7, scope: !31)
+!38 = !DILocation(line: 14, column: 11, scope: !31)
+!39 = !DILocation(line: 14, column: 3, scope: !31)
+!40 = !DILocation(line: 14, column: 8, scope: !31)
+!41 = !DILocation(line: 15, column: 11, scope: !31)
+!42 = !DILocation(line: 15, column: 7, scope: !31)
+!43 = !DILocation(line: 15, column: 5, scope: !31)
+!44 = !DILocation(line: 16, column: 10, scope: !31)
+!45 = !DILocation(line: 16, column: 11, scope: !31)
+!46 = !DILocation(line: 17, column: 1, scope: !31)
+!47 = !DILocation(line: 16, column: 3, scope: !31)
+!48 = distinct !DISubprogram(name: "bar", scope: !1, file: !1, line: 19, type: !9, scopeLine: 19, flags: DIFlagPrototyped | DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !49)
+!49 = !{!50}
+!50 = !DILocalVariable(name: "p", arg: 1, scope: !48, file: !1, line: 19, type: !12)
+!51 = !DILocation(line: 19, column: 15, scope: !48)
+!52 = !DILocation(line: 20, column: 3, scope: !48)
+!53 = !DILocation(line: 21, column: 11, scope: !48)
+!54 = !DILocation(line: 21, column: 3, scope: !48)
+!55 = !DILocation(line: 21, column: 8, scope: !48)
+!56 = !DILocation(line: 22, column: 10, scope: !48)
+!57 = !DILocation(line: 22, column: 3, scope: !48)
diff --git a/test/Transforms/SampleProfile/flattened.ll b/test/Transforms/SampleProfile/flattened.ll
new file mode 100644
index 0000000000000000000000000000000000000000..7a1e53b966c7cc659e7928c1cae73f6827484dd7
--- /dev/null
+++ b/test/Transforms/SampleProfile/flattened.ll
@@ -0,0 +1,39 @@
+; Check flattened profile will not be read in thinlto postlink.
+; RUN: opt < %s -O2 -flattened-profile-used -pgo-kind=pgo-sample-use-pipeline -profile-file=%S/Inputs/flattened.prof -enable-chr=false -perform-thinlto=true -S | FileCheck %s
+; RUN: opt < %s -passes='thinlto<O2>' -pgo-kind=pgo-sample-use-pipeline -profile-file=%S/Inputs/flattened.prof -flattened-profile-used -S | FileCheck %s
+;
+; Check flattened profile will be read in thinlto prelink.
+; RUN: opt < %s -O2 -flattened-profile-used -pgo-kind=pgo-sample-use-pipeline -profile-file=%S/Inputs/flattened.prof -enable-chr=false -prepare-for-thinlto=true -S | FileCheck %s --check-prefix=PRELINK
+; RUN: opt < %s -passes='thinlto-pre-link<O2>' -pgo-kind=pgo-sample-use-pipeline -profile-file=%S/Inputs/flattened.prof -flattened-profile-used -S | FileCheck %s --check-prefix=PRELINK
+;
+; Check flattened profile will be read in non-thinlto mode.
+; RUN: opt < %s -O2 -flattened-profile-used -pgo-kind=pgo-sample-use-pipeline -profile-file=%S/Inputs/flattened.prof -enable-chr=false -S | FileCheck %s --check-prefix=NOTHINLTO
+; RUN: opt < %s -passes='default<O2>' -pgo-kind=pgo-sample-use-pipeline -profile-file=%S/Inputs/flattened.prof -flattened-profile-used -S | FileCheck %s --check-prefix=NOTHINLTO
+;
+; CHECK-NOT: !{!"ProfileFormat", !"SampleProfile"}
+; PRELINK:   !{!"ProfileFormat", !"SampleProfile"}
+; NOTHINLTO: !{!"ProfileFormat", !"SampleProfile"}
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; Function Attrs: norecurse nounwind readnone uwtable
+define dso_local i32 @foo() local_unnamed_addr !dbg !7 {
+entry:
+  ret i32 -1, !dbg !9
+}
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4, !5}
+!llvm.ident = !{!6}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 8.0.0 (trunk 345241)", isOptimized: true, runtimeVersion: 0, emissionKind: LineTablesOnly, enums: !2, nameTableKind: None)
+!1 = !DIFile(filename: "a.c", directory: "")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{i32 1, !"wchar_size", i32 4}
+!6 = !{!"clang version 8.0.0 (trunk 345241)"}
+!7 = distinct !DISubprogram(name: "foo", scope: !1, file: !1, line: 1, type: !8, isLocal: false, isDefinition: true, scopeLine: 1, isOptimized: true, unit: !0, retainedNodes: !2)
+!8 = !DISubroutineType(types: !2)
+!9 = !DILocation(line: 2, column: 3, scope: !7)
diff --git a/test/Transforms/SampleProfile/function_metadata.ll b/test/Transforms/SampleProfile/function_metadata.ll
index dca6660415e569937ddad9b66fba6a2d42986d58..0e772e86f72b974b8cbe5b7e8b7531510024d1cc 100644
--- a/test/Transforms/SampleProfile/function_metadata.ll
+++ b/test/Transforms/SampleProfile/function_metadata.ll
@@ -1,5 +1,5 @@
-; RUN: opt < %s -passes='thinlto-pre-link<O2>' -pgo-kind=new-pm-pgo-sample-use-pipeline -profile-file=%S/Inputs/function_metadata.prof -S | FileCheck %s
-; RUN: opt < %s -passes='thinlto-pre-link<O2>' -pgo-kind=new-pm-pgo-sample-use-pipeline -profile-file=%S/Inputs/function_metadata.compact.afdo -S | FileCheck %s
+; RUN: opt < %s -passes='thinlto-pre-link<O2>' -pgo-kind=pgo-sample-use-pipeline -profile-file=%S/Inputs/function_metadata.prof -S | FileCheck %s
+; RUN: opt < %s -passes='thinlto-pre-link<O2>' -pgo-kind=pgo-sample-use-pipeline -profile-file=%S/Inputs/function_metadata.compact.afdo -S | FileCheck %s
 
 ; Tests whether the functions in the inline stack are added to the
 ; function_entry_count metadata.
diff --git a/test/Transforms/Scalarizer/basic.ll b/test/Transforms/Scalarizer/basic.ll
index c3ba1fe83fb9cefec9110f72975e80ac55b1b8ca..29a82fd8090c4f121878e08d0327f3514283531b 100644
--- a/test/Transforms/Scalarizer/basic.ll
+++ b/test/Transforms/Scalarizer/basic.ll
@@ -206,17 +206,17 @@ define void @f4(<4 x i32> *%src, <4 x i32> *%dst) {
   ret void
 }
 
-; Check that llvm.mem.parallel_loop_access information is preserved.
+; Check that llvm.access.group information is preserved.
 define void @f5(i32 %count, <4 x i32> *%src, <4 x i32> *%dst) {
 ; CHECK-LABEL: @f5(
-; CHECK: %val.i0 = load i32, i32* %this_src.i0, align 16, !llvm.mem.parallel_loop_access ![[TAG:[0-9]*]]
-; CHECK: %val.i1 = load i32, i32* %this_src.i1, align 4, !llvm.mem.parallel_loop_access ![[TAG]]
-; CHECK: %val.i2 = load i32, i32* %this_src.i2, align 8, !llvm.mem.parallel_loop_access ![[TAG]]
-; CHECK: %val.i3 = load i32, i32* %this_src.i3, align 4, !llvm.mem.parallel_loop_access ![[TAG]]
-; CHECK: store i32 %add.i0, i32* %this_dst.i0, align 16, !llvm.mem.parallel_loop_access ![[TAG]]
-; CHECK: store i32 %add.i1, i32* %this_dst.i1, align 4, !llvm.mem.parallel_loop_access ![[TAG]]
-; CHECK: store i32 %add.i2, i32* %this_dst.i2, align 8, !llvm.mem.parallel_loop_access ![[TAG]]
-; CHECK: store i32 %add.i3, i32* %this_dst.i3, align 4, !llvm.mem.parallel_loop_access ![[TAG]]
+; CHECK: %val.i0 = load i32, i32* %this_src.i0, align 16, !llvm.access.group ![[TAG:[0-9]*]]
+; CHECK: %val.i1 = load i32, i32* %this_src.i1, align 4, !llvm.access.group ![[TAG]]
+; CHECK: %val.i2 = load i32, i32* %this_src.i2, align 8, !llvm.access.group ![[TAG]]
+; CHECK: %val.i3 = load i32, i32* %this_src.i3, align 4, !llvm.access.group ![[TAG]]
+; CHECK: store i32 %add.i0, i32* %this_dst.i0, align 16, !llvm.access.group ![[TAG]]
+; CHECK: store i32 %add.i1, i32* %this_dst.i1, align 4, !llvm.access.group ![[TAG]]
+; CHECK: store i32 %add.i2, i32* %this_dst.i2, align 8, !llvm.access.group ![[TAG]]
+; CHECK: store i32 %add.i3, i32* %this_dst.i3, align 4, !llvm.access.group ![[TAG]]
 ; CHECK: ret void
 entry:
   br label %loop
@@ -225,9 +225,9 @@ loop:
   %index = phi i32 [ 0, %entry ], [ %next_index, %loop ]
   %this_src = getelementptr <4 x i32>, <4 x i32> *%src, i32 %index
   %this_dst = getelementptr <4 x i32>, <4 x i32> *%dst, i32 %index
-  %val = load <4 x i32> , <4 x i32> *%this_src, !llvm.mem.parallel_loop_access !3
+  %val = load <4 x i32> , <4 x i32> *%this_src, !llvm.access.group !13
   %add = add <4 x i32> %val, %val
-  store <4 x i32> %add, <4 x i32> *%this_dst, !llvm.mem.parallel_loop_access !3
+  store <4 x i32> %add, <4 x i32> *%this_dst, !llvm.access.group !13
   %next_index = add i32 %index, -1
   %continue = icmp ne i32 %next_index, %count
   br i1 %continue, label %loop, label %end, !llvm.loop !3
@@ -447,6 +447,7 @@ exit:
 !0 = !{ !"root" }
 !1 = !{ !"set1", !0 }
 !2 = !{ !"set2", !0 }
-!3 = !{ !3 }
+!3 = !{ !3, !{!"llvm.loop.parallel_accesses", !13} }
 !4 = !{ float 4.0 }
 !5 = !{ i64 0, i64 8, null }
+!13 = distinct !{}
diff --git a/test/Transforms/SimplifyCFG/bbi-23595.ll b/test/Transforms/SimplifyCFG/bbi-23595.ll
new file mode 100644
index 0000000000000000000000000000000000000000..676a80bb33f30441ca239f784050dcb81a752b89
--- /dev/null
+++ b/test/Transforms/SimplifyCFG/bbi-23595.ll
@@ -0,0 +1,50 @@
+; RUN: opt < %s -S -simplifycfg | FileCheck %s
+
+; In 'simplifycfg', during the flattening of a 'br', the instructions for the
+; 'true' and 'false' parts, are moved out from their respective basic blocks.
+; Their original debug locations (DILocations) and debug intrinsic instructions
+; (dbg.values) are removed.
+; As those basic blocks are now empty, their associated labels are removed.
+;
+; For the given test case, the labels 'W' and 'cleanup4' are removed.
+; We're expecting the dbg.label associated with 'W' to disappear, because
+; the 'W' label was removed.
+
+; CHECK-LABEL: _Z7test_itv()
+; CHECK:       entry:
+; CHECK-NEXT:    %retval.0 = select i1 undef, i16 1, i16 0
+; CHECK-NEXT:    ret i16 0
+
+define i16 @_Z7test_itv() {
+entry:
+  br label %sw.bb
+
+sw.bb:                                            ; preds = %entry
+  br i1 undef, label %W, label %cleanup4
+
+W:                                                ; preds = %sw.bb
+  call void @llvm.dbg.label(metadata !1), !dbg !8
+  br label %cleanup4
+
+cleanup4:                                         ; preds = %W, %sw.bb
+  %retval.0 = phi i16 [ 1, %W ], [ 0, %sw.bb ]
+  ret i16 0
+}
+
+; Function Attrs: nounwind readnone speculatable
+declare void @llvm.dbg.label(metadata) #0
+
+attributes #0 = { nounwind readnone speculatable }
+
+!llvm.dbg.cu = !{}
+!llvm.module.flags = !{!0}
+
+!0 = !{i32 2, !"Debug Info Version", i32 3}
+!1 = !DILabel(scope: !2, name: "W", file: !3, line: 47)
+!2 = distinct !DILexicalBlock(scope: !4, file: !3, line: 40, column: 3)
+!3 = !DIFile(filename: "foo.c", directory: "./")
+!4 = distinct !DISubprogram(name: "test_it", scope: !3, file: !3, line: 35, type: !5, scopeLine: 36, unit: !7)
+!5 = !DISubroutineType(types: !6)
+!6 = !{}
+!7 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !3, producer: "clang", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug)
+!8 = !DILocation(line: 47, column: 2, scope: !2)
diff --git a/test/Transforms/SimplifyCFG/combine-parallel-mem-md.ll b/test/Transforms/SimplifyCFG/combine-parallel-mem-md.ll
index 7afde1ff8d9a2ae1c12a7545562b35bb6326b315..d3ff927825f718ae5d9140f5ccf3548dc2c175f4 100644
--- a/test/Transforms/SimplifyCFG/combine-parallel-mem-md.ll
+++ b/test/Transforms/SimplifyCFG/combine-parallel-mem-md.ll
@@ -8,39 +8,39 @@ entry:
   br label %for.body
 
 ; CHECK-LABEL: @Test
-; CHECK: load i32, i32* {{.*}}, align 4, !llvm.mem.parallel_loop_access !0
-; CHECK: load i32, i32* {{.*}}, align 4, !llvm.mem.parallel_loop_access !0
-; CHECK: store i32 {{.*}}, align 4, !llvm.mem.parallel_loop_access !0
+; CHECK: load i32, i32* {{.*}}, align 4, !llvm.access.group !0
+; CHECK: load i32, i32* {{.*}}, align 4, !llvm.access.group !0
+; CHECK: store i32 {{.*}}, align 4, !llvm.access.group !0
 ; CHECK-NOT: load
 ; CHECK-NOT: store
 
 for.body:                                         ; preds = %cond.end, %entry
   %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %cond.end ]
   %arrayidx = getelementptr inbounds i32, i32* %p, i64 %indvars.iv
-  %0 = load i32, i32* %arrayidx, align 4, !llvm.mem.parallel_loop_access !0
+  %0 = load i32, i32* %arrayidx, align 4, !llvm.access.group !0
   %cmp1 = icmp eq i32 %0, 0
   br i1 %cmp1, label %cond.true, label %cond.false
 
 cond.false:                                       ; preds = %for.body
   %arrayidx3 = getelementptr inbounds i32, i32* %res, i64 %indvars.iv
-  %v = load i32, i32* %arrayidx3, align 4, !llvm.mem.parallel_loop_access !0
+  %v = load i32, i32* %arrayidx3, align 4, !llvm.access.group !0
   %arrayidx7 = getelementptr inbounds i32, i32* %d, i64 %indvars.iv
-  %1 = load i32, i32* %arrayidx7, align 4, !llvm.mem.parallel_loop_access !0
+  %1 = load i32, i32* %arrayidx7, align 4, !llvm.access.group !0
   %add = add nsw i32 %1, %v
   br label %cond.end
 
 cond.true:                                       ; preds = %for.body
   %arrayidx4 = getelementptr inbounds i32, i32* %res, i64 %indvars.iv
-  %w = load i32, i32* %arrayidx4, align 4, !llvm.mem.parallel_loop_access !0
+  %w = load i32, i32* %arrayidx4, align 4, !llvm.access.group !0
   %arrayidx8 = getelementptr inbounds i32, i32* %d, i64 %indvars.iv
-  %2 = load i32, i32* %arrayidx8, align 4, !llvm.mem.parallel_loop_access !0
+  %2 = load i32, i32* %arrayidx8, align 4, !llvm.access.group !0
   %add2 = add nsw i32 %2, %w
   br label %cond.end
 
 cond.end:                                         ; preds = %for.body, %cond.false
   %cond = phi i32 [ %add, %cond.false ], [ %add2, %cond.true ]
   %arrayidx9 = getelementptr inbounds i32, i32* %res, i64 %indvars.iv
-  store i32 %cond, i32* %arrayidx9, align 4, !llvm.mem.parallel_loop_access !0
+  store i32 %cond, i32* %arrayidx9, align 4, !llvm.access.group !0
   %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
   %exitcond = icmp eq i64 %indvars.iv.next, 16
   br i1 %exitcond, label %for.end, label %for.body, !llvm.loop !0
@@ -51,5 +51,6 @@ for.end:                                          ; preds = %cond.end
 
 attributes #0 = { norecurse nounwind uwtable }
 
-!0 = distinct !{!0, !1}
+!0 = distinct !{!0, !1, !{!"llvm.loop.parallel_accesses", !10}}
 !1 = !{!"llvm.loop.vectorize.enable", i1 true}
+!10 = distinct !{}
diff --git a/test/Transforms/ThinLTOBitcodeWriter/circular-reference.ll b/test/Transforms/ThinLTOBitcodeWriter/circular-reference.ll
index eeda79324497f08de8ad223144e62f592122373e..fb239b0a8a2ed0bf321b122ee9b74b104f02ff43 100644
--- a/test/Transforms/ThinLTOBitcodeWriter/circular-reference.ll
+++ b/test/Transforms/ThinLTOBitcodeWriter/circular-reference.ll
@@ -1,4 +1,4 @@
-; RUN: opt -thinlto-bc -o %t %s
+; RUN: opt -thinlto-bc -thinlto-split-lto-unit -o %t %s
 ; RUN: llvm-modextract -b -n 0 -o - %t | llvm-dis | FileCheck --check-prefix=M0 %s
 ; RUN: llvm-modextract -b -n 1 -o - %t | llvm-dis | FileCheck --check-prefix=M1 %s
 
diff --git a/test/Transforms/ThinLTOBitcodeWriter/comdat.ll b/test/Transforms/ThinLTOBitcodeWriter/comdat.ll
index caea48e0a5439a8724049f103e931516609810f2..a43fa1cf3eb0567eaf1cea2f6ced594e9a72a9ea 100644
--- a/test/Transforms/ThinLTOBitcodeWriter/comdat.ll
+++ b/test/Transforms/ThinLTOBitcodeWriter/comdat.ll
@@ -1,4 +1,4 @@
-; RUN: opt -thinlto-bc -o %t %s
+; RUN: opt -thinlto-bc -thinlto-split-lto-unit -o %t %s
 ; RUN: llvm-modextract -n 0 -o - %t | llvm-dis | FileCheck --check-prefix=THIN %s
 ; RUN: llvm-modextract -n 1 -o - %t | llvm-dis | FileCheck --check-prefix=MERGED %s
 
diff --git a/test/Transforms/ThinLTOBitcodeWriter/filter-alias.ll b/test/Transforms/ThinLTOBitcodeWriter/filter-alias.ll
index eb0cbe78a731804e32498aa54b54f7f0f7f40627..200d494f2474e9f88444dce76407256efd971c63 100644
--- a/test/Transforms/ThinLTOBitcodeWriter/filter-alias.ll
+++ b/test/Transforms/ThinLTOBitcodeWriter/filter-alias.ll
@@ -1,4 +1,4 @@
-; RUN: opt -thinlto-bc -o %t %s
+; RUN: opt -thinlto-bc -thinlto-split-lto-unit -o %t %s
 ; RUN: llvm-modextract -n 0 -o - %t | llvm-dis | FileCheck --check-prefix=CHECK0 %s
 ; RUN: llvm-modextract -n 1 -o - %t | llvm-dis | FileCheck --check-prefix=CHECK1 %s
 ; CHECK0-NOT: @{{.*}}anon{{.*}}=
diff --git a/test/Transforms/ThinLTOBitcodeWriter/function-alias.ll b/test/Transforms/ThinLTOBitcodeWriter/function-alias.ll
index 119b8219bab7c697d189511ba5961c38fbea2c46..a1dbd963296ec96681157bbbddb8fb257914ccb4 100644
--- a/test/Transforms/ThinLTOBitcodeWriter/function-alias.ll
+++ b/test/Transforms/ThinLTOBitcodeWriter/function-alias.ll
@@ -1,4 +1,4 @@
-; RUN: opt -thinlto-bc -o %t %s
+; RUN: opt -thinlto-bc -thinlto-split-lto-unit -o %t %s
 ; RUN: llvm-modextract -n 1 -o - %t | llvm-dis | FileCheck --check-prefix=CHECK1 %s
 
 target triple = "x86_64-unknown-linux-gnu"
diff --git a/test/Transforms/ThinLTOBitcodeWriter/pr33536.ll b/test/Transforms/ThinLTOBitcodeWriter/pr33536.ll
index 661d0739401a7035b2b8ac592f3f06a3c209d081..c405c369e05707347c38cd3e80578ffd23640c8c 100644
--- a/test/Transforms/ThinLTOBitcodeWriter/pr33536.ll
+++ b/test/Transforms/ThinLTOBitcodeWriter/pr33536.ll
@@ -1,7 +1,7 @@
 ; Test for a bug specific to the new pass manager where we may build a domtree
 ; to make more precise AA queries for functions.
 ;
-; RUN: opt -aa-pipeline=default -passes='no-op-module' -debug-pass-manager -thinlto-bc -o %t %s
+; RUN: opt -aa-pipeline=default -passes='no-op-module' -debug-pass-manager -thinlto-bc -thinlto-split-lto-unit -o %t %s
 ; RUN: llvm-modextract -b -n 0 -o - %t | llvm-dis | FileCheck --check-prefix=M0 %s
 ; RUN: llvm-modextract -b -n 1 -o - %t | llvm-dis | FileCheck --check-prefix=M1 %s
 
diff --git a/test/Transforms/ThinLTOBitcodeWriter/split-internal-typeid.ll b/test/Transforms/ThinLTOBitcodeWriter/split-internal-typeid.ll
index a43db9a3755a1deff72f863657421a5e93c31f25..290df00fa9eb36bc68304df9c843677df5909014 100644
--- a/test/Transforms/ThinLTOBitcodeWriter/split-internal-typeid.ll
+++ b/test/Transforms/ThinLTOBitcodeWriter/split-internal-typeid.ll
@@ -1,4 +1,4 @@
-; RUN: opt -thinlto-bc -o %t %s
+; RUN: opt -thinlto-bc -thinlto-split-lto-unit -o %t %s
 ; RUN: llvm-modextract -b -n 0 -o %t0 %t
 ; RUN: llvm-modextract -b -n 1 -o %t1 %t
 ; RUN: not llvm-modextract -b -n 2 -o - %t 2>&1 | FileCheck --check-prefix=ERROR %s
diff --git a/test/Transforms/ThinLTOBitcodeWriter/split-internal1.ll b/test/Transforms/ThinLTOBitcodeWriter/split-internal1.ll
index 6d18c4f6f65eada946916febbc8a6c3cdb595397..42a06bdf2bf86c0a82ce9cd56f26d3da13a933ca 100644
--- a/test/Transforms/ThinLTOBitcodeWriter/split-internal1.ll
+++ b/test/Transforms/ThinLTOBitcodeWriter/split-internal1.ll
@@ -1,4 +1,4 @@
-; RUN: opt -thinlto-bc -o %t %s
+; RUN: opt -thinlto-bc -thinlto-split-lto-unit -o %t %s
 ; RUN: llvm-modextract -b -n 0 -o %t0 %t
 ; RUN: llvm-modextract -b -n 1 -o %t1 %t
 ; RUN: not llvm-modextract -b -n 2 -o - %t 2>&1 | FileCheck --check-prefix=ERROR %s
diff --git a/test/Transforms/ThinLTOBitcodeWriter/split-internal2.ll b/test/Transforms/ThinLTOBitcodeWriter/split-internal2.ll
index fbe618f08e3e794c0bae97b63bcb770696e47450..02fc3d18850930f713d195bf6043ba4ec811100d 100644
--- a/test/Transforms/ThinLTOBitcodeWriter/split-internal2.ll
+++ b/test/Transforms/ThinLTOBitcodeWriter/split-internal2.ll
@@ -1,4 +1,4 @@
-; RUN: opt -thinlto-bc -o %t %s
+; RUN: opt -thinlto-bc -thinlto-split-lto-unit -o %t %s
 ; RUN: llvm-modextract -b -n 0 -o %t0 %t
 ; RUN: llvm-modextract -b -n 1 -o %t1 %t
 ; RUN: not llvm-modextract -b -n 2 -o - %t 2>&1 | FileCheck --check-prefix=ERROR %s
diff --git a/test/Transforms/ThinLTOBitcodeWriter/split-vfunc-internal.ll b/test/Transforms/ThinLTOBitcodeWriter/split-vfunc-internal.ll
index 087796b5031c8b6fefdb115468407e2be3bb01a9..7ebb30ae1aaba2475f07131b79ef117a4af6615a 100644
--- a/test/Transforms/ThinLTOBitcodeWriter/split-vfunc-internal.ll
+++ b/test/Transforms/ThinLTOBitcodeWriter/split-vfunc-internal.ll
@@ -1,4 +1,4 @@
-; RUN: opt -thinlto-bc -o %t %s
+; RUN: opt -thinlto-bc -thinlto-split-lto-unit -o %t %s
 ; RUN: llvm-modextract -b -n 0 -o - %t | llvm-dis | FileCheck --check-prefix=M0 %s
 ; RUN: llvm-modextract -b -n 1 -o - %t | llvm-dis | FileCheck --check-prefix=M1 %s
 
diff --git a/test/Transforms/ThinLTOBitcodeWriter/split-vfunc.ll b/test/Transforms/ThinLTOBitcodeWriter/split-vfunc.ll
index 66d37d5e8ae69d25cd169044ec0d2cfacd122869..fcf575188f7912be8ef0dfbbfe04ea1167e94a11 100644
--- a/test/Transforms/ThinLTOBitcodeWriter/split-vfunc.ll
+++ b/test/Transforms/ThinLTOBitcodeWriter/split-vfunc.ll
@@ -1,4 +1,4 @@
-; RUN: opt -thinlto-bc -o %t %s
+; RUN: opt -thinlto-bc -thinlto-split-lto-unit -o %t %s
 ; RUN: llvm-modextract -b -n 0 -o - %t | llvm-dis | FileCheck --check-prefix=M0 %s
 ; RUN: llvm-modextract -b -n 1 -o - %t | llvm-dis | FileCheck --check-prefix=M1 %s
 
diff --git a/test/Transforms/ThinLTOBitcodeWriter/split.ll b/test/Transforms/ThinLTOBitcodeWriter/split.ll
index 08ed92e7ebe2e99496c5d7fa75b0138d171fb31c..5502f7a263237fe03885982d46c29b6959f4cc27 100644
--- a/test/Transforms/ThinLTOBitcodeWriter/split.ll
+++ b/test/Transforms/ThinLTOBitcodeWriter/split.ll
@@ -1,6 +1,6 @@
 ; Generate bitcode files with summary, as well as minimized bitcode without
 ; the debug metadata for the thin link.
-; RUN: opt -thinlto-bc -thin-link-bitcode-file=%t2 -o %t %s
+; RUN: opt -thinlto-bc -thin-link-bitcode-file=%t2 -thinlto-split-lto-unit -o %t %s
 ; RUN: llvm-modextract -b -n 0 -o %t0.bc %t
 ; RUN: llvm-modextract -b -n 1 -o %t1.bc %t
 ; RUN: llvm-modextract -b -n 0 -o %t0.thinlink.bc %t2
diff --git a/test/Transforms/ThinLTOBitcodeWriter/symver.ll b/test/Transforms/ThinLTOBitcodeWriter/symver.ll
index 078825cf468b658069494117e22c620c4b515650..8acdd0c6a688b8039940c5267ebcc4ef591a09fc 100644
--- a/test/Transforms/ThinLTOBitcodeWriter/symver.ll
+++ b/test/Transforms/ThinLTOBitcodeWriter/symver.ll
@@ -1,4 +1,4 @@
-; RUN: opt -thinlto-bc -o %t %s
+; RUN: opt -thinlto-bc -thinlto-split-lto-unit -o %t %s
 ; RUN: llvm-modextract -n 1 -o - %t | llvm-dis | FileCheck %s
 
 ; The target assembly parser is required to parse the symver directives
diff --git a/test/Transforms/ThinLTOBitcodeWriter/unsplittable.ll b/test/Transforms/ThinLTOBitcodeWriter/unsplittable.ll
index 5413e0f61882c5000b66b0c52221882da4abf672..46c87bc4e1f543e6b19888aefc7a6e3dcc23dd9c 100644
--- a/test/Transforms/ThinLTOBitcodeWriter/unsplittable.ll
+++ b/test/Transforms/ThinLTOBitcodeWriter/unsplittable.ll
@@ -1,4 +1,4 @@
-; RUN: opt -thinlto-bc -thin-link-bitcode-file=%t2 -o %t %s
+; RUN: opt -thinlto-bc -thin-link-bitcode-file=%t2 -thinlto-split-lto-unit -o %t %s
 ; RUN: llvm-dis -o - %t | FileCheck %s
 ; RUN: llvm-bcanalyzer -dump %t | FileCheck --check-prefix=BCA %s
 ; When not splitting the module, the thin link bitcode file should simply be a
@@ -28,7 +28,8 @@ define void @h() comdat {
   ret void
 }
 
-; CHECK: !llvm.module.flags = !{![[FLAG:[0-9]+]]}
-; CHECK: ![[FLAG]] = !{i32 1, !"ThinLTO", i32 0}
+; CHECK: !llvm.module.flags = !{![[FLAG1:[0-9]+]], ![[FLAG2:[0-9]+]]}
+; CHECK: ![[FLAG1]] = !{i32 1, !"EnableSplitLTOUnit", i32 1}
+; CHECK: ![[FLAG2]] = !{i32 1, !"ThinLTO", i32 0}
 
 !0 = !{i32 0, !"typeid"}
diff --git a/test/Transforms/ThinLTOBitcodeWriter/x86/module-asm.ll b/test/Transforms/ThinLTOBitcodeWriter/x86/module-asm.ll
index 15e47785cbe262f3ccb1f4b360ad89987297027b..587ab3fbb5020e107e232ce7d2f784d5da115b66 100644
--- a/test/Transforms/ThinLTOBitcodeWriter/x86/module-asm.ll
+++ b/test/Transforms/ThinLTOBitcodeWriter/x86/module-asm.ll
@@ -1,4 +1,4 @@
-; RUN: opt -thinlto-bc -o %t %s
+; RUN: opt -thinlto-bc -thinlto-split-lto-unit -o %t %s
 ; RUN: llvm-modextract -b -n 0 -o - %t | llvm-dis | FileCheck --check-prefix=M0 %s
 ; RUN: llvm-modextract -b -n 1 -o - %t | llvm-dis | FileCheck --check-prefix=M1 %s
 
diff --git a/test/Transforms/Util/store-first-op.ll b/test/Transforms/Util/store-first-op.ll
index 397925dbb645bbf7f8ed24f5308191449b12906f..c4ef2a14262e35a25453efe39fcefff9008108bb 100644
--- a/test/Transforms/Util/store-first-op.ll
+++ b/test/Transforms/Util/store-first-op.ll
@@ -24,7 +24,7 @@ attributes #1 = { nounwind readnone }
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!2}
 
-!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !1, producer: "clang version 3.8.0 (https://github.com/llvm-mirror/clang 89dda3855cda574f355e6defa1d77bdae5053994) (llvm/trunk 257597)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug)
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !1, producer: "clang version 3.8.0", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug)
 !1 = !DIFile(filename: "none", directory: ".")
 !2 = !{i32 2, !"Debug Info Version", i32 3}
 !3 = !DILocalVariable(name: "getU", scope: !4, file: !1, line: 25, type: !5)
diff --git a/test/Unit/lit.cfg.py b/test/Unit/lit.cfg.py
index 609408d1dd807ade01a8f162d6d63bd2242de56f..3198ab2c9539b5c696404bc7d73b2b72e8355176 100644
--- a/test/Unit/lit.cfg.py
+++ b/test/Unit/lit.cfg.py
@@ -31,6 +31,11 @@ if 'TMP' in os.environ:
 if 'TEMP' in os.environ:
     config.environment['TEMP'] = os.environ['TEMP']
 
+# Propagate HOME as it can be used to override incorrect homedir in passwd
+# that causes the tests to fail.
+if 'HOME' in os.environ:
+    config.environment['HOME'] = os.environ['HOME']
+
 # Propagate path to symbolizer for ASan/MSan.
 for symbolizer in ['ASAN_SYMBOLIZER_PATH', 'MSAN_SYMBOLIZER_PATH']:
     if symbolizer in os.environ:
diff --git a/test/Verifier/di-subroutine-localvar.ll b/test/Verifier/di-subroutine-localvar.ll
new file mode 100644
index 0000000000000000000000000000000000000000..477e63362127f5bc32551b4b6ca913cdd0b9a3d2
--- /dev/null
+++ b/test/Verifier/di-subroutine-localvar.ll
@@ -0,0 +1,42 @@
+; RUN: opt %s -verify 2>&1 | FileCheck %s
+; CHECK: invalid type
+; CHECK: !20 = !DILocalVariable(name: "f", scope: !21, file: !13, line: 970, type: !14)
+; CHECK: !14 = !DISubroutineType(types: !15)
+
+
+%timespec.0.1.2.3.0.1.2 = type { i64, i64 }
+define internal i64 @init_vdso_clock_gettime(i32, %timespec.0.1.2.3.0.1.2* nonnull) unnamed_addr !dbg !142 {
+  call void @llvm.dbg.value(metadata i64 (i32, %timespec.0.1.2.3.0.1.2*)* null, metadata !162, metadata !DIExpression()), !dbg !167
+  ret i64 -38, !dbg !168
+}
+declare void @llvm.dbg.value(metadata, metadata, metadata) #0
+!llvm.module.flags = !{!0}
+!llvm.dbg.cu = !{!1}
+!0 = !{i32 2, !"Debug Info Version", i32 3}
+!1 = distinct !DICompileUnit(language: DW_LANG_C99, file: !2, producer: "zig 0.3.0", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !3, globals: !76)
+!2 = !DIFile(filename: "test", directory: ".")
+!3 = !{!4}
+!4 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "Arch", scope: !5, file: !5, line: 44, baseType: !6, size: 8, align: 8, elements: !7)
+!5 = !DIFile(filename: "builtin.zig", directory: "/home/andy/.local/share/zig/stage1/builtin/ugMGxVES9OkDAffv3xhJS3KQVy0Wm1xPM3Bc6x4MBuup5aetdi5pVTrGRG2aDAn0")
+!6 = !DIBasicType(name: "u7", size: 8, encoding: DW_ATE_unsigned)
+!7 = !{!8}
+!8 = !DIEnumerator(name: "armv8_5a", value: 0)
+!76 = !{!77}
+!77 = !DIGlobalVariableExpression(var: !78, expr: !DIExpression())
+!78 = distinct !DIGlobalVariable(name: "arch", linkageName: "arch", scope: !5, file: !5, line: 437, type: !4, isLocal: true, isDefinition: true)
+!81 = !DIFile(filename: "index.zig", directory: "/store/dev/zig/build-llvm8-debug/lib/zig/std/os/linux")
+!142 = distinct !DISubprogram(name: "init_vdso_clock_gettime", scope: !81, file: !81, line: 968, type: !143, scopeLine: 968, flags: DIFlagStaticMember, spFlags: DISPFlagLocalToUnit | DISPFlagDefinition | DISPFlagOptimized, unit: !1, retainedNodes: !153)
+!143 = !DISubroutineType(types: !144)
+!144 = !{!145}
+!145 = !DIBasicType(name: "usize", size: 64, encoding: DW_ATE_unsigned)
+!146 = !DIBasicType(name: "i32", size: 32, encoding: DW_ATE_signed)
+!153 = !{!154}
+!154 = !DILocalVariable(name: "clk", arg: 1, scope: !142, file: !81, line: 968, type: !146)
+!162 = !DILocalVariable(name: "f", scope: !163, file: !81, line: 970, type: !143)
+!163 = distinct !DILexicalBlock(scope: !164, file: !81, line: 969, column: 5)
+!164 = distinct !DILexicalBlock(scope: !165, file: !81, line: 968, column: 66)
+!165 = distinct !DILexicalBlock(scope: !166, file: !81, line: 968, column: 45)
+!166 = distinct !DILexicalBlock(scope: !142, file: !81, line: 968, column: 35)
+!167 = !DILocation(line: 970, column: 5, scope: !163)
+!168 = !DILocation(line: 972, column: 28, scope: !169)
+!169 = distinct !DILexicalBlock(scope: !163, file: !81, line: 970, column: 5)
diff --git a/test/Verifier/test_g_add.mir b/test/Verifier/test_g_add.mir
new file mode 100644
index 0000000000000000000000000000000000000000..9cd990bb8cc116aa33a431f165b39a0ce7c6eec2
--- /dev/null
+++ b/test/Verifier/test_g_add.mir
@@ -0,0 +1,37 @@
+#RUN: not llc -march=aarch64 -o - -global-isel -run-pass=none -verify-machineinstrs %s 2>&1 | FileCheck %s
+# REQUIRES: global-isel, aarch64-registered-target
+
+---
+name:            test_add
+legalized:       true
+regBankSelected: false
+selected:        false
+tracksRegLiveness: true
+liveins:
+body:             |
+  bb.0:
+
+    %0:_(s32) = G_CONSTANT i32 0
+    %1:_(s32) = G_CONSTANT i32 1
+
+    ; CHECK: Bad machine code: Too few operands
+    %2:_(s32) = G_ADD
+
+    ; CHECK: Bad machine code: Too few operands
+    %3:_(s32) = G_ADD %0
+    %4:_(s32) = G_ADD %0, %1
+
+    ; CHECK: Bad machine code: Too few operands
+    ; CHECK: Bad machine code: Explicit definition marked as use
+    G_ADD %0, %1
+
+    ; CHECK: Bad machine code: generic instruction must use register operands
+    %5:_(s32) = G_ADD %0, 1
+
+    %6:_(s64) = G_CONSTANT i64 0
+
+    ; CHECK: Bad machine code: Type mismatch in generic instruction
+    ; CHECK: Bad machine code: Generic virtual register does not allow subregister index
+    %8:_(s32) = G_ADD %6.sub_32:_(s64), %0
+
+...
diff --git a/test/Verifier/test_g_addrspacecast.mir b/test/Verifier/test_g_addrspacecast.mir
new file mode 100644
index 0000000000000000000000000000000000000000..44504108132adc71966147abaf152383734fd21b
--- /dev/null
+++ b/test/Verifier/test_g_addrspacecast.mir
@@ -0,0 +1,57 @@
+#RUN: not llc -o -  -run-pass=none -verify-machineinstrs %s 2>&1 | FileCheck %s
+# REQUIRES: global-isel, aarch64-registered-target
+
+---
+name:            test_addrspacecast
+legalized:       true
+regBankSelected: false
+selected:        false
+tracksRegLiveness: true
+liveins:
+body:             |
+  bb.0:
+
+    %0:_(s64) = G_IMPLICIT_DEF
+    %1:_(p0) = G_IMPLICIT_DEF
+    %2:_(<2 x s64>) = G_IMPLICIT_DEF
+    %3:_(<2 x p0>) = G_IMPLICIT_DEF
+
+    ; CHECK: Bad machine code: Too few operands
+    %4:_(s64) = G_ADDRSPACE_CAST
+
+    ; CHECK: Bad machine code: Too few operands
+    ; CHECK: Bad machine code: Explicit definition marked as use
+    G_ADDRSPACE_CAST %1
+
+    ; CHECK: Bad machine code: addrspacecast types must be pointers
+    %5:_(p0) = G_ADDRSPACE_CAST %0
+
+    ; CHECK: Bad machine code: addrspacecast types must be pointers
+    %6:_(s64) = G_ADDRSPACE_CAST %1
+
+    ; CHECK: Bad machine code: addrspacecast types must be pointers
+    %7:_(<2 x s64>) = G_ADDRSPACE_CAST %1
+
+    ; CHECK: Bad machine code: addrspacecast types must be pointers
+    %8:_(<2 x p0>) = G_ADDRSPACE_CAST %2
+
+    ; CHECK: Bad machine code: operand types must be all-vector or all-scalar
+    %9:_(<2 x p1>) = G_ADDRSPACE_CAST %1
+
+    ; CHECK: Bad machine code: operand types must be all-vector or all-scalar
+    %10:_(p1) = G_ADDRSPACE_CAST %3
+
+    ; CHECK: Bad machine code: operand types must preserve number of vector elements
+    %11:_(<4 x p1>) = G_ADDRSPACE_CAST %3
+
+    ; CHECK: Bad machine code: operand types must preserve number of vector elements
+    %12:_(<4 x p1>) = G_IMPLICIT_DEF
+    %13:_(<2 x p0>) = G_ADDRSPACE_CAST %12
+
+    ; CHECK: Bad machine code: addrspacecast must convert different address spaces
+    %14:_(p0) = G_ADDRSPACE_CAST %1
+
+    ; CHECK: Bad machine code: addrspacecast must convert different address spaces
+    %15:_(<2 x p0>) = G_ADDRSPACE_CAST %3
+
+...
diff --git a/test/Verifier/test_g_bitcast.mir b/test/Verifier/test_g_bitcast.mir
new file mode 100644
index 0000000000000000000000000000000000000000..3446d5fc86eb2bacd0a06bca6e7e6ca5cc7446f0
--- /dev/null
+++ b/test/Verifier/test_g_bitcast.mir
@@ -0,0 +1,37 @@
+#RUN: not llc -mtriple=amdgcn-amd-amdhsa -o - -global-isel -run-pass=none -verify-machineinstrs %s 2>&1 | FileCheck %s
+# REQUIRES: global-isel, amdgpu-registered-target
+
+---
+name:            test_bitcast
+legalized:       true
+regBankSelected: false
+selected:        false
+tracksRegLiveness: true
+liveins:
+body:             |
+  bb.0:
+   ; CHECK:  Bad machine code: bitcast cannot convert between pointers and other types
+    %0:_(s64) = G_IMPLICIT_DEF
+    %1:_(p0) = G_BITCAST %0
+
+    ; CHECK:  Bad machine code: bitcast cannot convert between pointers and other
+    %2:_(p0) = G_IMPLICIT_DEF
+    %3:_(s64) = G_BITCAST %2
+
+    ; CHECK: Bad machine code: bitcast sizes must match
+    %4:_(s32) = G_IMPLICIT_DEF
+    %5:_(s64) = G_BITCAST %4
+
+    ; CHECK: Bad machine code: bitcast sizes must match
+    %6:_(s32) = G_IMPLICIT_DEF
+    %7:_(<3 x s8>) = G_BITCAST %6
+
+    ; CHECK: Bad machine code: bitcast sizes must match
+    %8:_(p1) = G_IMPLICIT_DEF
+    %9:_(p3) = G_BITCAST %8
+
+    ; CHECK: Bad machine code: bitcast sizes must match
+    %10:_(p1) = G_IMPLICIT_DEF
+    %11:_(p3) = G_BITCAST %8
+
+...
diff --git a/test/Verifier/test_g_constant.mir b/test/Verifier/test_g_constant.mir
new file mode 100644
index 0000000000000000000000000000000000000000..fa5daff870b0dca96cb0063e6d9ad4bd7c561e25
--- /dev/null
+++ b/test/Verifier/test_g_constant.mir
@@ -0,0 +1,42 @@
+#RUN: not llc -march=aarch64 -o /dev/null -run-pass=none -verify-machineinstrs %s 2>&1 | FileCheck %s
+# REQUIRES: global-isel, aarch64-registered-target
+
+---
+name:            test_constant
+legalized:       true
+regBankSelected: false
+selected:        false
+tracksRegLiveness: true
+liveins:
+body:             |
+  bb.0:
+   ; CHECK: Bad machine code: Instruction cannot use a vector result type
+   ; CHECK: Bad machine code: inconsistent constant size
+    %0:_(<2 x s32>) = G_CONSTANT i32 0
+
+    ; CHECK: Bad machine code: Too few operands
+    %1:_(s32) = G_CONSTANT
+
+    ; Not a cimm
+    ; CHECK: Bad machine code:  G_CONSTANT operand must be cimm
+    %2:_(s32) = G_CONSTANT 0
+
+    ; Not a cimm
+    ; CHECK: Bad machine code:  G_CONSTANT operand must be cimm
+    %3:_(s32) = G_CONSTANT float 1.0
+
+    ;  Size is bigger than result
+    ; CHECK: Bad machine code: inconsistent constant size
+    %4:_(s32) = G_CONSTANT i64 0
+
+    ; Size is smaller than result
+    ; CHECK: Bad machine code: inconsistent constant size
+    %4:_(s32) = G_CONSTANT i16 0
+
+    ; CHECK: Bad machine code: inconsistent constant size
+    %5:_(p0) = G_CONSTANT i32 0
+
+    ; CHECK: Bad machine code: inconsistent constant size
+    %6:_(p0) = G_CONSTANT i128 0
+
+...
diff --git a/test/Verifier/test_g_extract.mir b/test/Verifier/test_g_extract.mir
new file mode 100644
index 0000000000000000000000000000000000000000..62064ae80217858677988c48cb614c362061cc02
--- /dev/null
+++ b/test/Verifier/test_g_extract.mir
@@ -0,0 +1,35 @@
+# RUN: not llc -march=aarch64 -o /dev/null -run-pass=none -verify-machineinstrs %s 2>&1 | FileCheck %s
+# REQUIRES: global-isel, aarch64-registered-target
+
+---
+name:            test_extract
+legalized:       true
+regBankSelected: false
+selected:        false
+tracksRegLiveness: true
+liveins:
+body:             |
+  bb.0:
+
+    ; CHECK: Bad machine code: Too few operands
+    %0:_(s32) = G_EXTRACT
+
+    ; CHECK: Bad machine code: Too few operands
+    %1:_(s32) = G_EXTRACT 0
+
+    ; CHECK: Bad machine code: extract source must be a register
+    %2:_(s32) = G_EXTRACT 0, 1
+
+    ; CHECK: Bad machine code: extract offset must be a constant
+    %3:_(s32) = G_IMPLICIT_DEF
+    %4:_(s32) = G_CONSTANT i32 0
+    %5:_(s16) = G_EXTRACT %3, %4
+
+    ; CHECK: Bad machine code: extract source must be larger than result
+    %6:_(s32) = G_IMPLICIT_DEF
+    %7:_(s32) = G_EXTRACT %6, 0
+
+    ; CHECK: Bad machine code: extract reads past end of register
+    %8:_(s1) = G_EXTRACT %6, 32
+
+...
diff --git a/test/Verifier/test_g_fcmp.mir b/test/Verifier/test_g_fcmp.mir
new file mode 100644
index 0000000000000000000000000000000000000000..baaa24835364de15cd43d8ae6e11795f087172f3
--- /dev/null
+++ b/test/Verifier/test_g_fcmp.mir
@@ -0,0 +1,40 @@
+#RUN: not llc -o - -global-isel -run-pass=none -verify-machineinstrs %s 2>&1 | FileCheck %s
+# REQUIRES: global-isel, aarch64-registered-target
+
+---
+name:            test_fcmp
+legalized:       true
+regBankSelected: false
+selected:        false
+tracksRegLiveness: true
+liveins:
+body:             |
+  bb.0:
+
+    %0:_(s32) = G_FCONSTANT float 0.0
+    %1:_(s32) = G_FCONSTANT float 1.0
+
+   ; Vector result, scalar source
+   ; CHECK: Bad machine code: Generic vector icmp/fcmp must preserve number of lanes
+    %2:_(<2 x s1>) = G_FCMP floatpred(oeq), %0, %1
+
+    ; Scalar result, vector source
+    ; CHECK: Bad machine code: Generic vector icmp/fcmp must preserve number of lanes
+    %3:_(<2 x s32>) = G_IMPLICIT_DEF
+    %4:_(<2 x s32>) = G_IMPLICIT_DEF
+    %5:_(s1) = G_FCMP floatpred(oeq), %3, %4
+
+    ; mismatched element count
+    ; CHECK: Bad machine code: Generic vector icmp/fcmp must preserve number of
+    %6:_(<2 x s32>) = G_IMPLICIT_DEF
+    %7:_(<2 x s32>) = G_IMPLICIT_DEF
+    %8:_(<4 x s1>) = G_FCMP floatpred(oeq), %6, %7
+
+
+    ; mismatched scalar element type
+    ; CHECK: *** Bad machine code: Type mismatch in generic instruction ***
+    %9:_(s32) = G_FCONSTANT float 0.0
+    %10:_(s64) = G_FCONSTANT float 1.0
+    %11:_(s1) = G_FCMP floatpred(oeq), %9, %10
+
+...
diff --git a/test/Verifier/test_g_fconstant.mir b/test/Verifier/test_g_fconstant.mir
new file mode 100644
index 0000000000000000000000000000000000000000..d917d9c846fffb0bd8cdf1c6c90bdfa619731361
--- /dev/null
+++ b/test/Verifier/test_g_fconstant.mir
@@ -0,0 +1,35 @@
+#RUN: not llc -march=aarch64 -o /dev/null -run-pass=none -verify-machineinstrs %s 2>&1 | FileCheck %s
+# REQUIRES: global-isel, aarch64-registered-target
+
+---
+name:            test_fconstant
+legalized:       true
+regBankSelected: false
+selected:        false
+tracksRegLiveness: true
+liveins:
+body:             |
+  bb.0:
+    ; CHECK: Bad machine code: Instruction cannot use a vector result type
+    %0:_(<2 x s32>) = G_FCONSTANT float 0.0
+
+    ; CHECK: Bad machine code: Too few operands
+    %1:_(s32) = G_FCONSTANT
+
+    ; Not a cimm
+    ; CHECK: Bad machine code:  G_FCONSTANT operand must be fpimm
+    %2:_(s32) = G_FCONSTANT 0
+
+    ; Not a cimm
+    ; CHECK: Bad machine code:  G_FCONSTANT operand must be fpimm
+    %3:_(s32) = G_FCONSTANT i32 0
+
+   ;  Size is bigger than result
+   ; CHECK: Bad machine code: inconsistent constant size
+    %4:_(s32) = G_FCONSTANT double 1.0
+
+    ; Size is smaller than result
+    ; CHECK: Bad machine code: inconsistent constant size
+    %4:_(s32) = G_FCONSTANT half 1.0
+
+...
diff --git a/test/Verifier/test_g_gep.mir b/test/Verifier/test_g_gep.mir
new file mode 100644
index 0000000000000000000000000000000000000000..5e34625d1e745398cc920d0529c743e5f010eab9
--- /dev/null
+++ b/test/Verifier/test_g_gep.mir
@@ -0,0 +1,32 @@
+#RUN: not llc -o -  -run-pass=none -verify-machineinstrs %s 2>&1 | FileCheck %s
+# REQUIRES: global-isel, aarch64-registered-target
+
+---
+name:            test_gep
+legalized:       true
+regBankSelected: false
+selected:        false
+tracksRegLiveness: true
+liveins:
+body:             |
+  bb.0:
+
+    %0:_(p0) = G_IMPLICIT_DEF
+    %1:_(s64) = G_IMPLICIT_DEF
+
+    ; CHECK:  Bad machine code: Type mismatch in generic instruction
+    %2:_(s64) = G_GEP %0, %1
+
+    ; CHECK:  Bad machine code: Type mismatch in generic instruction
+    %3:_(p0) = G_GEP %1, %1
+
+    ; CHECK: Bad machine code: gep offset operand must not be a pointer
+    %4:_(p0) = G_GEP %0, %0
+
+    ; CHECK: Bad machine code: Type mismatch in generic instruction
+    %5:_(p1) = G_GEP %0, %1
+
+    ; CHECK: Bad machine code: gep first operand must be a pointer
+    %6:_(s64) = G_GEP %1, %1
+
+...
diff --git a/test/Verifier/test_g_icmp.mir b/test/Verifier/test_g_icmp.mir
new file mode 100644
index 0000000000000000000000000000000000000000..e2d9d6419cce201b0a302ad5e521ce1980929df9
--- /dev/null
+++ b/test/Verifier/test_g_icmp.mir
@@ -0,0 +1,40 @@
+#RUN: not llc -o - -global-isel -run-pass=none -verify-machineinstrs %s 2>&1 | FileCheck %s
+# REQUIRES: global-isel, aarch64-registered-target
+
+---
+name:            test_icmp
+legalized:       true
+regBankSelected: false
+selected:        false
+tracksRegLiveness: true
+liveins:
+body:             |
+  bb.0:
+
+    %0:_(s32) = G_CONSTANT i32 0
+    %1:_(s32) = G_CONSTANT i32 1
+
+   ; Vector result, scalar source
+   ; CHECK: Bad machine code: Generic vector icmp/fcmp must preserve number of lanes
+    %2:_(<2 x s1>) = G_ICMP intpred(eq), %0, %1
+
+    ; Scalar result, vector source
+    ; CHECK: Bad machine code: Generic vector icmp/fcmp must preserve number of lanes
+    %3:_(<2 x s32>) = G_IMPLICIT_DEF
+    %4:_(<2 x s32>) = G_IMPLICIT_DEF
+    %5:_(s1) = G_ICMP intpred(eq), %3, %4
+
+    ; mismatched element count
+    ; CHECK: Bad machine code: Generic vector icmp/fcmp must preserve number of
+    %6:_(<2 x s32>) = G_IMPLICIT_DEF
+    %7:_(<2 x s32>) = G_IMPLICIT_DEF
+    %8:_(<4 x s1>) = G_ICMP intpred(eq), %6, %7
+
+
+    ; mismatched scalar element type
+    ; CHECK: *** Bad machine code: Type mismatch in generic instruction ***
+    %9:_(s32) = G_CONSTANT i32 0
+    %10:_(s64) = G_CONSTANT i32 1
+    %11:_(s1) = G_ICMP intpred(eq), %9, %10
+
+...
diff --git a/test/Verifier/test_g_inttoptr.mir b/test/Verifier/test_g_inttoptr.mir
new file mode 100644
index 0000000000000000000000000000000000000000..f3a201147f1aa93aa34d0e29bc297ca3696a8f57
--- /dev/null
+++ b/test/Verifier/test_g_inttoptr.mir
@@ -0,0 +1,45 @@
+#RUN: not llc -o -  -run-pass=none -verify-machineinstrs %s 2>&1 | FileCheck %s
+# REQUIRES: global-isel, aarch64-registered-target
+
+---
+name:            test_inttoptr
+legalized:       true
+regBankSelected: false
+selected:        false
+tracksRegLiveness: true
+liveins:
+body:             |
+  bb.0:
+
+    %0:_(s64) = G_IMPLICIT_DEF
+    %1:_(p0) = G_IMPLICIT_DEF
+    %2:_(<2 x s64>) = G_IMPLICIT_DEF
+    %3:_(<2 x p0>) = G_IMPLICIT_DEF
+
+    ; CHECK: Bad machine code: Too few operands
+    %4:_(p0) = G_INTTOPTR
+
+    ; CHECK: Bad machine code: Too few operands
+    ; CHECK: Bad machine code: Explicit definition marked as use
+    G_INTTOPTR %0
+
+    ; CHECK: Bad machine code: inttoptr result type must be a pointer
+    %5:_(s64) = G_INTTOPTR %0
+
+    ; CHECK: Bad machine code: inttoptr result type must be a pointer
+    %6:_(<2 x s64>) = G_INTTOPTR %2
+
+    ; CHECK: Bad machine code: operand types must be all-vector or all-scalar
+    %7:_(<2 x p0>) = G_INTTOPTR %0
+
+    ; CHECK: Bad machine code: operand types must be all-vector or all-scalar
+    %8:_(p0) = G_INTTOPTR %2
+
+    ; CHECK: Bad machine code: operand types must preserve number of vector elements
+    %9:_(<4 x p0>) = G_INTTOPTR %2
+
+    ; CHECK: Bad machine code: operand types must preserve number of vector elements
+    %10:_(<4 x s64>) = G_IMPLICIT_DEF
+    %11:_(<2 x p0>) = G_INTTOPTR %10
+
+...
diff --git a/test/Verifier/test_g_load.mir b/test/Verifier/test_g_load.mir
new file mode 100644
index 0000000000000000000000000000000000000000..4378e862f0ef2c85872704ea713d12e4461e11b3
--- /dev/null
+++ b/test/Verifier/test_g_load.mir
@@ -0,0 +1,23 @@
+#RUN: not llc -o - -run-pass=none -verify-machineinstrs %s 2>&1 | FileCheck %s
+# REQUIRES: global-isel, aarch64-registered-target
+
+---
+name:            test_load
+legalized:       true
+regBankSelected: false
+selected:        false
+tracksRegLiveness: true
+liveins:
+body:             |
+  bb.0:
+
+    ; CHECK: Bad machine code: Generic memory instruction must access a pointer
+    %0:_(s64) = G_CONSTANT i32 0
+    %1:_(s32) = G_LOAD %0 :: (load 4)
+
+    %2:_(p0) = G_IMPLICIT_DEF
+
+    ; CHECK: Bad machine code: load memory size cannot exceed result size
+    %3:_(s8) = G_LOAD %2 :: (load 2)
+
+...
diff --git a/test/Verifier/test_g_ptrtoint.mir b/test/Verifier/test_g_ptrtoint.mir
new file mode 100644
index 0000000000000000000000000000000000000000..c6a969a9c377db6982fa30d90953d698c0b500b6
--- /dev/null
+++ b/test/Verifier/test_g_ptrtoint.mir
@@ -0,0 +1,45 @@
+#RUN: not llc -o -  -run-pass=none -verify-machineinstrs %s 2>&1 | FileCheck %s
+# REQUIRES: global-isel, aarch64-registered-target
+
+---
+name:            test_ptrtoint
+legalized:       true
+regBankSelected: false
+selected:        false
+tracksRegLiveness: true
+liveins:
+body:             |
+  bb.0:
+
+    %0:_(s64) = G_IMPLICIT_DEF
+    %1:_(p0) = G_IMPLICIT_DEF
+    %2:_(<2 x s64>) = G_IMPLICIT_DEF
+    %3:_(<2 x p0>) = G_IMPLICIT_DEF
+
+    ; CHECK: Bad machine code: Too few operands
+    %4:_(s64) = G_PTRTOINT
+
+    ; CHECK: Bad machine code: Too few operands
+    ; CHECK: Bad machine code: Explicit definition marked as use
+    G_PTRTOINT %1
+
+    ; CHECK: Bad machine code: ptrtoint result type must not be a pointer
+    %5:_(p0) = G_PTRTOINT %1
+
+    ; CHECK: Bad machine code: ptrtoint result type must not be a pointer
+    %6:_(<2 x p0>) = G_PTRTOINT %0
+
+    ; CHECK: Bad machine code: ptrtoint source type must be a pointer
+    %7:_(<2 x s64>) = G_PTRTOINT %2
+
+    ; CHECK: Bad machine code: operand types must be all-vector or all-scalar
+    %8:_(s64) = G_PTRTOINT %3
+
+    ; CHECK: Bad machine code: operand types must preserve number of vector elements
+    %9:_(<4 x s64>) = G_INTTOPTR %3
+
+    ; CHECK: Bad machine code: operand types must preserve number of vector elements
+    %10:_(<4 x p0>) = G_IMPLICIT_DEF
+    %11:_(<2 x s64>) = G_PTRTOINT %10
+
+...
diff --git a/test/Verifier/test_g_select.mir b/test/Verifier/test_g_select.mir
new file mode 100644
index 0000000000000000000000000000000000000000..d40b276c9ba2a2f8f8ff10c6b1f023386c6bbdb9
--- /dev/null
+++ b/test/Verifier/test_g_select.mir
@@ -0,0 +1,31 @@
+#RUN: not llc -march=aarch64 -run-pass=none -verify-machineinstrs -o /dev/null %s 2>&1 | FileCheck %s
+# REQUIRES: global-isel, aarch64-registered-target
+
+---
+name:            test_select
+legalized:       true
+regBankSelected: false
+selected:        false
+tracksRegLiveness: true
+liveins:
+body:             |
+  bb.0:
+
+    %0:_(s32) = G_CONSTANT i32 0
+    %1:_(s32) = G_CONSTANT i32 1
+    %2:_(s1) = G_CONSTANT i32 0
+    %3:_(<2 x s32>) = G_IMPLICIT_DEF
+    %4:_(<4 x s32>) = G_IMPLICIT_DEF
+    %5:_(<2 x s1>) = G_IMPLICIT_DEF
+    %6:_(<4 x s1>) = G_IMPLICIT_DEF
+
+    ; CHECK: Bad machine code: operand types must be all-vector or all-scalar
+    %7:_(s32) = G_SELECT %5, %0, %1
+
+    ; CHECK: Bad machine code: operand types must preserve number of vector elements
+    %8:_(<2 x s32>) = G_SELECT %6, %3, %3
+
+    ; CHECK: Bad machine code: operand types must preserve number of vector elements
+    %9:_(<4 x s32>) = G_SELECT %5, %4, %4
+
+...
diff --git a/test/Verifier/test_g_sextload.mir b/test/Verifier/test_g_sextload.mir
new file mode 100644
index 0000000000000000000000000000000000000000..b195481d1e607ea7604f9d99ed3135fb76e805fc
--- /dev/null
+++ b/test/Verifier/test_g_sextload.mir
@@ -0,0 +1,28 @@
+# RUN: not llc -o - -global-isel -run-pass=none -verify-machineinstrs %s 2>&1 | FileCheck %s
+# REQUIRES: global-isel, aarch64-registered-target
+
+---
+name:            test_sextload
+legalized:       true
+regBankSelected: false
+selected:        false
+tracksRegLiveness: true
+liveins:
+body:             |
+  bb.0:
+
+    ; CHECK: Bad machine code: Generic memory instruction must access a pointer
+    %0:_(s64) = G_CONSTANT i32 0
+    %1:_(s32) = G_SEXTLOAD %0 :: (load 1)
+
+    ; CHECK: *** Bad machine code: Generic instruction accessing memory must have one mem operand ***
+    %2:_(p0) = G_IMPLICIT_DEF
+    %3:_(s64) = G_SEXTLOAD %2
+
+    ; CHECK: Bad machine code: Generic extload must have a narrower memory type
+    ; CHECK: Bad machine code: Generic extload must have a narrower memory type
+
+    %4:_(s64) = G_SEXTLOAD %2 :: (load 8)
+    %5:_(s64) = G_SEXTLOAD %2 :: (load 16)
+
+...
diff --git a/test/Verifier/test_g_store.mir b/test/Verifier/test_g_store.mir
new file mode 100644
index 0000000000000000000000000000000000000000..8cf144f1e31137f29546ff6848fa559405351fca
--- /dev/null
+++ b/test/Verifier/test_g_store.mir
@@ -0,0 +1,24 @@
+# RUN: not llc -o - -global-isel -run-pass=none -verify-machineinstrs %s 2>&1 | FileCheck %s
+# REQUIRES: global-isel, aarch64-registered-target
+
+---
+name:            test_store
+legalized:       true
+regBankSelected: false
+selected:        false
+tracksRegLiveness: true
+liveins:
+body:             |
+  bb.0:
+
+    ; CHECK: Bad machine code: Generic memory instruction must access a pointer
+    %0:_(s64) = G_CONSTANT i32 0
+    %1:_(s32) = G_CONSTANT i32 1
+    G_STORE %1, %0 :: (store 4)
+
+    %2:_(p0) = G_IMPLICIT_DEF
+    %3:_(s8) = G_IMPLICIT_DEF
+    ; CHECK: Bad machine code: store memory size cannot exceed value size
+    G_STORE %3, %2 :: (store 2)
+
+...
diff --git a/test/Verifier/test_g_trunc.mir b/test/Verifier/test_g_trunc.mir
new file mode 100644
index 0000000000000000000000000000000000000000..3a196bb7bbd74d52083f18b27eb79c048fdf23d9
--- /dev/null
+++ b/test/Verifier/test_g_trunc.mir
@@ -0,0 +1,23 @@
+# RUN: not llc -o - -global-isel -run-pass=none -verify-machineinstrs %s 2>&1 | FileCheck %s
+# REQUIRES: global-isel, aarch64-registered-target
+
+---
+name:            test_trunc
+legalized:       true
+regBankSelected: false
+selected:        false
+tracksRegLiveness: true
+liveins:
+body:             |
+  bb.0:
+
+    ; CHECK: Bad machine code: Too few operands
+    %0:_(s32) = G_TRUNC
+
+    %1:_(s64) = G_IMPLICIT_DEF
+
+    ; CHECK: Bad machine code: Too few operands
+    ; CHECK: Bad machine code: Explicit definition marked as use
+    G_TRUNC %1
+
+...
diff --git a/test/Verifier/test_g_zextload.mir b/test/Verifier/test_g_zextload.mir
new file mode 100644
index 0000000000000000000000000000000000000000..a2bd321bfa3abac7c4141546a01fb9e62967fa91
--- /dev/null
+++ b/test/Verifier/test_g_zextload.mir
@@ -0,0 +1,28 @@
+# RUN: not llc -o - -global-isel -run-pass=none -verify-machineinstrs %s 2>&1 | FileCheck %s
+# REQUIRES: global-isel, aarch64-registered-target
+
+---
+name:            test_zextload
+legalized:       true
+regBankSelected: false
+selected:        false
+tracksRegLiveness: true
+liveins:
+body:             |
+  bb.0:
+
+    ; CHECK: Bad machine code: Generic memory instruction must access a pointer
+    %0:_(s64) = G_CONSTANT i32 0
+    %1:_(s32) = G_ZEXTLOAD %0 :: (load 1)
+
+    ; CHECK: *** Bad machine code: Generic instruction accessing memory must have one mem operand ***
+    %2:_(p0) = G_IMPLICIT_DEF
+    %3:_(s64) = G_ZEXTLOAD %2
+
+    ; CHECK: Bad machine code: Generic extload must have a narrower memory type
+    ; CHECK: Bad machine code: Generic extload must have a narrower memory type
+
+    %4:_(s64) = G_ZEXTLOAD %2 :: (load 8)
+    %5:_(s64) = G_ZEXTLOAD %2 :: (load 16)
+
+...
diff --git a/test/Verifier/test_phis_precede_nonphis.mir b/test/Verifier/test_phis_precede_nonphis.mir
index 29409798d1b774038bce871381f431ec3cd15f5f..fa69b40fddf12b38c524b1f6816a1a23bb01d862 100644
--- a/test/Verifier/test_phis_precede_nonphis.mir
+++ b/test/Verifier/test_phis_precede_nonphis.mir
@@ -44,7 +44,7 @@ body: |
     G_BR %bb.3.exit
   bb.2.else:
     successors: %bb.3(0x80000000)
-    %6:_(s32) = G_CONSTANT i8 1
+    %6:_(s32) = G_CONSTANT i32 1
     %7:_(s8) = G_TRUNC %6
     G_BR %bb.3.exit
   bb.3.exit:
diff --git a/test/lit.cfg.py b/test/lit.cfg.py
index 3dff11bd0d2fc6aefe1dff949836d35e3daa67bf..0cd67655eb3067fca4e679bbf0e24c96ac1fc527 100644
--- a/test/lit.cfg.py
+++ b/test/lit.cfg.py
@@ -89,7 +89,6 @@ llvm_config.use_default_substitutions()
 config.substitutions.append(('%llvmshlibdir', config.llvm_shlib_dir))
 config.substitutions.append(('%shlibext', config.llvm_shlib_ext))
 config.substitutions.append(('%exeext', config.llvm_exe_ext))
-config.substitutions.append(('%host_cc', config.host_cc))
 
 
 lli_args = []
@@ -279,7 +278,10 @@ if have_ld_plugin_support():
 
 
 def have_ld64_plugin_support():
-    if not config.llvm_tool_lto_build or config.ld64_executable == '':
+    if not os.path.exists(os.path.join(config.llvm_shlib_dir, 'libLTO' + config.llvm_shlib_ext)):
+        return False
+
+    if config.ld64_executable == '':
         return False
 
     ld_cmd = subprocess.Popen(
@@ -322,7 +324,7 @@ if config.have_libxar:
 if config.enable_threads:
     config.available_features.add('thread_support')
 
-if config.llvm_libxml2_enabled == '1':
+if config.llvm_libxml2_enabled:
     config.available_features.add('libxml2')
 
 if config.have_opt_viewer_modules:
diff --git a/test/lit.site.cfg.py.in b/test/lit.site.cfg.py.in
index 6592b3c450c233825e1add6363bb696ca3d76e30..62c90b7c66b8b17cc5be0feb09de91e910e6dee3 100644
--- a/test/lit.site.cfg.py.in
+++ b/test/lit.site.cfg.py.in
@@ -15,7 +15,6 @@ config.lit_tools_dir = "@LLVM_LIT_TOOLS_DIR@"
 config.python_executable = "@PYTHON_EXECUTABLE@"
 config.gold_executable = "@GOLD_EXECUTABLE@"
 config.ld64_executable = "@LD64_EXECUTABLE@"
-config.llvm_tool_lto_build = @LLVM_TOOL_LTO_BUILD@
 config.ocamlfind_executable = "@OCAMLFIND@"
 config.have_ocamlopt = @HAVE_OCAMLOPT@
 config.have_ocaml_ounit = @HAVE_OCAML_OUNIT@
@@ -40,7 +39,7 @@ config.enable_ffi = @LLVM_ENABLE_FFI@
 config.enable_threads = @LLVM_ENABLE_THREADS@
 config.build_shared_libs = @BUILD_SHARED_LIBS@
 config.link_llvm_dylib = @LLVM_LINK_LLVM_DYLIB@
-config.llvm_libxml2_enabled = "@LLVM_LIBXML2_ENABLED@"
+config.llvm_libxml2_enabled = @LLVM_LIBXML2_ENABLED@
 config.llvm_host_triple = '@LLVM_HOST_TRIPLE@'
 config.host_arch = "@HOST_ARCH@"
 config.have_opt_viewer_modules = @LLVM_HAVE_OPT_VIEWER_MODULES@
diff --git a/test/tools/dsymutil/ARM/obfuscated.test b/test/tools/dsymutil/ARM/obfuscated.test
new file mode 100644
index 0000000000000000000000000000000000000000..9ce684cfb7e5e404a7a392cb3b0859c45f224546
--- /dev/null
+++ b/test/tools/dsymutil/ARM/obfuscated.test
@@ -0,0 +1,166 @@
+REQUIRES: system-darwin
+
+RUN: dsymutil --symbol-map %p/../Inputs/obfuscated.map %p/../Inputs/obfuscated.arm64 -f -o - \
+RUN:     | llvm-dwarfdump -v - \
+RUN:     | FileCheck %s
+
+RUN: dsymutil --symbol-map %p/../Inputs/obfuscated.map %p/../Inputs/obfuscated.arm64 -f -o - \
+RUN:     | llvm-dwarfdump -v - \
+RUN:     | FileCheck --check-prefix=NOHIDDEN %s
+
+RUN: dsymutil --symbol-map %p/../Inputs/obfuscated.2.map %p/../Inputs/obfuscated.2.arm64 -f -o - \
+RUN:     | llvm-dwarfdump -v - \
+RUN:     | FileCheck --check-prefix=NOHIDDEN %s
+
+// Run with plist and make sure dsymutil finds it.
+RUN: mkdir -p %t.dSYM/Contents/Resources/DWARF/
+RUN: mkdir -p %t.mapdir
+RUN: cp %p/../Inputs/obfuscated.arm64 %t.dSYM/Contents/Resources/DWARF/
+RUN: cp %p/../Inputs/E828A486-8433-3A5E-B6DB-A6294D28133D.plist %t.dSYM/Contents/Resources/
+RUN: cp %p/../Inputs/obfuscated.map %t.mapdir/506AA50A-6B26-3B37-86D2-DC6EBD57B720.bcsymbolmap
+RUN: dsymutil --symbol-map %t.mapdir %t.dSYM 2>&1 | FileCheck --check-prefix=OBFUSCATING %s
+
+// Run without plist and make sure dsymutil doesn't crash.
+RUN: rm %t.dSYM/Contents/Resources/E828A486-8433-3A5E-B6DB-A6294D28133D.plist
+RUN: dsymutil --symbol-map %t.mapdir %t.dSYM 2>&1 | FileCheck --check-prefix=NOTOBFUSCATING %s
+
+OBFUSCATING-NOT: not unobfuscating
+
+NOTOBFUSCATING: not unobfuscating
+
+NOHIDDEN-NOT: __hidden#
+
+CHECK: .debug_info contents:
+
+CHECK: DW_TAG_compile_unit [1] *
+CHECK:    DW_AT_producer [DW_FORM_strp]    ( {{.*}} "Apple LLVM version 7.0.0 (clang-700.2.38.2)")
+CHECK:    DW_AT_name [DW_FORM_strp]        ( {{.*}} "main.c")
+CHECK:    DW_AT_comp_dir [DW_FORM_strp]    ( {{.*}} "/Users/steven/dev/alpena/tests/src")
+CHECK:    DW_TAG_subprogram [2]
+CHECK:      DW_AT_name [DW_FORM_strp]      ( {{.*}} "main")
+
+CHECK:  DW_TAG_compile_unit [1] *
+CHECK:    DW_AT_producer [DW_FORM_strp]    ( {{.*}} "Apple LLVM version 7.0.0 (clang-700.2.38.2)")
+CHECK:    DW_AT_name [DW_FORM_strp]        ( {{.*}} "one.c")
+CHECK:    DW_AT_comp_dir [DW_FORM_strp]    ( {{.*}} "/Users/steven/dev/alpena/tests/src")
+CHECK:    DW_TAG_subprogram [2]
+CHECK:      DW_AT_name [DW_FORM_strp]      ( {{.*}} "one")
+
+CHECK:  DW_TAG_compile_unit [1] *
+CHECK:    DW_AT_producer [DW_FORM_strp]    ( {{.*}} "Apple LLVM version 7.0.0 (clang-700.2.38.2)")
+CHECK:    DW_AT_name [DW_FORM_strp]        ( {{.*}} "two.c")
+CHECK:    DW_AT_comp_dir [DW_FORM_strp]    ( {{.*}} "/Users/steven/dev/alpena/tests/src")
+CHECK:    DW_TAG_subprogram [2]
+CHECK:      DW_AT_name [DW_FORM_strp]      ( {{.*}} "two")
+
+CHECK:  DW_TAG_compile_unit [1] *
+CHECK:    DW_AT_producer [DW_FORM_strp]    ( {{.*}} "Apple LLVM version 7.0.0 (clang-700.2.38.2)")
+CHECK:    DW_AT_name [DW_FORM_strp]        ( {{.*}} "three.c")
+CHECK:    DW_AT_comp_dir [DW_FORM_strp]    ( {{.*}} "/Users/steven/dev/alpena/tests/src")
+CHECK:    DW_TAG_subprogram [2]
+CHECK:      DW_AT_name [DW_FORM_strp]      ( {{.*}} "three")
+
+CHECK:  DW_TAG_compile_unit [1] *
+CHECK:    DW_AT_producer [DW_FORM_strp]    ( {{.*}} "Apple LLVM version 7.0.0 (clang-700.2.38.2)")
+CHECK:    DW_AT_name [DW_FORM_strp]        ( {{.*}} "four.c")
+CHECK:    DW_AT_stmt_list [DW_FORM_data4]  (0x0000011e)
+CHECK:    DW_AT_comp_dir [DW_FORM_strp]    ( {{.*}} "/Users/steven/dev/alpena/tests/src")
+CHECK:    DW_TAG_subprogram [2]
+CHECK:      DW_AT_name [DW_FORM_strp]      ( {{.*}} "four")
+
+CHECK:  DW_TAG_compile_unit [1] *
+CHECK:    DW_AT_producer [DW_FORM_strp]    ( {{.*}} "Apple LLVM version 7.0.0 (clang-700.2.38.2)")
+CHECK:    DW_AT_name [DW_FORM_strp]        ( {{.*}} "five.c")
+CHECK:    DW_AT_comp_dir [DW_FORM_strp]    ( {{.*}} "/Users/steven/dev/alpena/tests/src")
+CHECK:    DW_TAG_subprogram [2]
+CHECK:      DW_AT_name [DW_FORM_strp]      ( {{.*}} "five")
+
+CHECK:  DW_TAG_compile_unit [1] *
+CHECK:    DW_AT_producer [DW_FORM_strp]    ( {{.*}} "Apple LLVM version 7.0.0 (clang-700.2.38.2)")
+CHECK:    DW_AT_name [DW_FORM_strp]        ( {{.*}} "six.c")
+CHECK:    DW_AT_comp_dir [DW_FORM_strp]    ( {{.*}} "/Users/steven/dev/alpena/tests/src")
+CHECK:    DW_TAG_subprogram [2]
+CHECK:      DW_AT_name [DW_FORM_strp]      ( {{.*}} "six")
+
+CHECK: .debug_line contents:
+CHECK: file_names[  1]:
+CHECK:            name: "main.c"
+CHECK:       dir_index: 0
+CHECK:        mod_time: 0x00000000
+CHECK: file_names[  1]:
+CHECK:            name: "one.c"
+CHECK:       dir_index: 0
+CHECK:        mod_time: 0x00000000
+CHECK:          length: 0x00000000
+CHECK: file_names[  1]:
+CHECK:            name: "two.c"
+CHECK:       dir_index: 0
+CHECK:        mod_time: 0x00000000
+CHECK:          length: 0x00000000
+CHECK: file_names[  1]:
+CHECK:            name: "three.c"
+CHECK:       dir_index: 0
+CHECK:        mod_time: 0x00000000
+CHECK:          length: 0x00000000
+CHECK: file_names[  1]:
+CHECK:            name: "four.c"
+CHECK:       dir_index: 0
+CHECK:        mod_time: 0x00000000
+CHECK:          length: 0x00000000
+CHECK: file_names[  1]:
+CHECK:            name: "five.c"
+CHECK:       dir_index: 0
+CHECK:        mod_time: 0x00000000
+CHECK:          length: 0x00000000
+CHECK: file_names[  1]:
+CHECK:            name: "six.c"
+CHECK:       dir_index: 0
+CHECK:        mod_time: 0x00000000
+CHECK:          length: 0x00000000
+
+CHECK: .debug_pubnames contents:
+CHECK: length = 0x00000017 version = 0x0002 unit_offset = 0x00000000 unit_size = 0x00000044
+CHECK: 0x0000002e "main"
+CHECK: length = 0x00000016 version = 0x0002 unit_offset = 0x00000044 unit_size = 0x00000044
+CHECK: 0x0000002e "one"
+CHECK: length = 0x00000016 version = 0x0002 unit_offset = 0x00000088 unit_size = 0x00000044
+CHECK: 0x0000002e "two"
+CHECK: length = 0x00000018 version = 0x0002 unit_offset = 0x000000cc unit_size = 0x00000044
+CHECK: 0x0000002e "three"
+CHECK: length = 0x00000017 version = 0x0002 unit_offset = 0x00000110 unit_size = 0x00000044
+CHECK: 0x0000002e "four"
+CHECK: length = 0x00000017 version = 0x0002 unit_offset = 0x00000154 unit_size = 0x00000044
+CHECK: 0x0000002e "five"
+CHECK: length = 0x00000016 version = 0x0002 unit_offset = 0x00000198 unit_size = 0x00000044
+CHECK: 0x0000002e "six"
+
+CHECK: .apple_names contents:
+
+CHECK: String: 0x00000091 "five"
+CHECK-NEXT: Data 0 [
+CHECK-NEXT:   Atom[0]: 0x00000182
+CHECK-NEXT: ]
+CHECK: String: 0x0000009c "six"
+CHECK-NEXT: Data 0 [
+CHECK-NEXT:   Atom[0]: 0x000001c6
+CHECK-NEXT: ]
+CHECK: String: 0x00000078 "three"
+CHECK-NEXT: Data 0 [
+CHECK-NEXT:   Atom[0]: 0x000000fa
+CHECK-NEXT: ]
+CHECK: String: 0x0000006c "two"
+CHECK-NEXT: Data 0 [
+CHECK-NEXT:   Atom[0]: 0x000000b6
+CHECK-NEXT: ]
+CHECK: String: 0x00000057 "main"
+CHECK-NEXT: Data 0 [
+CHECK-NEXT:   Atom[0]: 0x0000002e
+CHECK-NEXT: ]
+CHECK: String: 0x00000085 "four"
+CHECK-NEXT: Data 0 [
+CHECK-NEXT:   Atom[0]: 0x0000013e
+CHECK-NEXT: ]
+CHECK: String: 0x00000062 "one"
+CHECK-NEXT: Data 0 [
+CHECK-NEXT:   Atom[0]: 0x00000072
+CHECK-NEXT: ]
diff --git a/test/tools/dsymutil/Inputs/E828A486-8433-3A5E-B6DB-A6294D28133D.plist b/test/tools/dsymutil/Inputs/E828A486-8433-3A5E-B6DB-A6294D28133D.plist
new file mode 100644
index 0000000000000000000000000000000000000000..adf7dbf0e027bb0ec5d2b4cefae043081090deb2
--- /dev/null
+++ b/test/tools/dsymutil/Inputs/E828A486-8433-3A5E-B6DB-A6294D28133D.plist
@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="UTF-8"?><!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+   <key>DBGOriginalUUID</key>
+   <string>506AA50A-6B26-3B37-86D2-DC6EBD57B720</string>
+</dict>
+</plist>
\ No newline at end of file
diff --git a/test/tools/dsymutil/Inputs/obfuscated.2.arm64 b/test/tools/dsymutil/Inputs/obfuscated.2.arm64
new file mode 100644
index 0000000000000000000000000000000000000000..b40e023eb4916cca1fe38388ce12b030c1a51eda
Binary files /dev/null and b/test/tools/dsymutil/Inputs/obfuscated.2.arm64 differ
diff --git a/test/tools/dsymutil/Inputs/obfuscated.2.map b/test/tools/dsymutil/Inputs/obfuscated.2.map
new file mode 100644
index 0000000000000000000000000000000000000000..6efca5960311d63c3603770aa77c2e52b0d5c74d
--- /dev/null
+++ b/test/tools/dsymutil/Inputs/obfuscated.2.map
@@ -0,0 +1,22 @@
+BCSymbolMap Version: 2.0
+_two
+_three
+_four
+_five
+_six
+LLVM version 3.9.0 (ssh://git@stash.sd.apple.com/devtools/clang.git c74ae34bd917b77f9c848bd599dfde2813fb509f)
+main
+main.c
+/Volumes/Data/dev/BitcodeBuildTests/unit
+one
+one.c
+two
+two.c
+three
+three.c
+four
+four.c
+five
+five.c
+six
+six.c
diff --git a/test/tools/dsymutil/Inputs/obfuscated.arm64 b/test/tools/dsymutil/Inputs/obfuscated.arm64
new file mode 100644
index 0000000000000000000000000000000000000000..8395798be17ff1aaea4397cd38c2b0f26f465cb0
Binary files /dev/null and b/test/tools/dsymutil/Inputs/obfuscated.arm64 differ
diff --git a/test/tools/dsymutil/Inputs/obfuscated.map b/test/tools/dsymutil/Inputs/obfuscated.map
new file mode 100644
index 0000000000000000000000000000000000000000..30fed8bf9b5a574f7562d53633bc328e7d394f51
--- /dev/null
+++ b/test/tools/dsymutil/Inputs/obfuscated.map
@@ -0,0 +1,17 @@
+one
+two
+three
+four
+five
+six
+.str
+Apple LLVM version 7.0.0 (clang-700.2.38.2)
+main
+main.c
+/Users/steven/dev/alpena/tests/src
+one.c
+two.c
+three.c
+four.c
+five.c
+six.c
diff --git a/test/tools/dsymutil/X86/empty-CU.test b/test/tools/dsymutil/X86/empty-CU.test
index c55a3ca8659eab7b14df2a7e7814822989eaae06..e2acefcdccfd2b0f956a7e217d213f6cbbe7aecd 100644
--- a/test/tools/dsymutil/X86/empty-CU.test
+++ b/test/tools/dsymutil/X86/empty-CU.test
@@ -4,4 +4,3 @@ CHECK: .debug_info contents:
 CHECK: 0x00000000: Compile Unit: length = 0x00000008 version = 0x0003 abbr_offset = 0x0000 addr_size = 0x04 (next unit at 0x0000000c)
 
 CHECK: 0x0000000b: DW_TAG_compile_unit [1]
-CHECK: 0x0000000c: Compile Unit: length = 0x00000007 version = 0x0003 abbr_offset = 0x0000 addr_size = 0x04 (next unit at 0x00000017)
diff --git a/test/tools/dsymutil/X86/generate-empty-CU.test b/test/tools/dsymutil/X86/generate-empty-CU.test
index 5b1dd75159354bede5bd990acd433aed1bc38d59..02914181b3356d19bf85f62080e44aca7c578144 100644
--- a/test/tools/dsymutil/X86/generate-empty-CU.test
+++ b/test/tools/dsymutil/X86/generate-empty-CU.test
@@ -1,8 +1,8 @@
 # RUN: dsymutil -f -o - -oso-prepend-path=%p/.. -y %s | llvm-dwarfdump -v - | FileCheck %s
 
 # This test on links the Dwarf for an LTO binary and on purpose doesn't retain
-# any symbol in the second CU out of 3. This is the only case where dsymutil
-# will generate an empty CU and it requires special handling.
+# any symbol in the second CU out of 3. To be valid DWARF ssymutil must not
+# generate an empty CU but omit it.
 
 ---
 triple:          'x86_64-apple-darwin'
@@ -22,9 +22,7 @@ CHECK:        DW_AT_name {{.*}} "basic1.c"
 CHECK:   DW_TAG_subprogram
                 DW_AT_name {{.*}} "main"
 
-CHECK: Compile Unit: length = 0x00000007 version = 0x0002 abbr_offset = 0x0000 addr_size = 0x08 (next unit at 0x0000008c)
-
-CHECK: Compile Unit: length = 0x00000089 version = 0x0002 abbr_offset = 0x0000 addr_size = 0x08 (next unit at 0x00000119)
+CHECK: 0x00000081: Compile Unit: length = 0x00000089 version = 0x0002 abbr_offset = 0x0000 addr_size = 0x08 (next unit at 0x0000010e)
 
 CHECK: DW_TAG_compile_unit
 CHECK:        DW_AT_name {{.*}} "basic3.c"
diff --git a/test/tools/dsymutil/X86/lc_build_version.test b/test/tools/dsymutil/X86/lc_build_version.test
index d0a8598b03628b1b1928c8421979bfe6140354e1..2c91cbb0aa0e5c0941936b8007d15f94089a8ed0 100644
--- a/test/tools/dsymutil/X86/lc_build_version.test
+++ b/test/tools/dsymutil/X86/lc_build_version.test
@@ -1,5 +1,5 @@
 # RUN: dsymutil -f %p/../Inputs/lc_build_version.x86_64 -o - \
-# RUN:   | obj2yaml | FileCheck %s
+# RUN:   -oso-prepend-path=%p/.. | obj2yaml | FileCheck %s
 
 CHECK: LoadCommands:
 CHECK:   - cmd:             LC_BUILD_VERSION
diff --git a/test/tools/dsymutil/X86/module-warnings.test b/test/tools/dsymutil/X86/module-warnings.test
index 2304e9787115b006bff876727772c56cda0e6c88..0249a30d0a0e06828f20154fd21d27e83ed9c3d8 100644
--- a/test/tools/dsymutil/X86/module-warnings.test
+++ b/test/tools/dsymutil/X86/module-warnings.test
@@ -23,7 +23,7 @@
 # RUN: cp %p/../Inputs/module-warnings/1.o %t.dir
 # RUN: cp %p/../Inputs/module-warnings/Foo.pcm %t.dir/ModuleCache
 #
-# RUN: dsymutil -f -oso-prepend-path=%t.dir -y \
+# RUN: dsymutil -verify -f -oso-prepend-path=%t.dir -y \
 # RUN:   %p/dummy-debug-map.map -o %t 2>&1 | FileCheck %s
 #
 # Module-not-found should be reported only once.
@@ -32,12 +32,12 @@
 # CHECK-NOT: warning: {{.*}}Bar.pcm:
 #
 # RUN: cp %p/../Inputs/module-warnings/libstatic.a %t.dir
-# RUN: dsymutil -f -oso-prepend-path=%t.dir -y %s -o %t 2>&1 | FileCheck %s
+# RUN: dsymutil -verify -f -oso-prepend-path=%t.dir -y %s -o %t 2>&1 | FileCheck %s
 # CHECK: rebuild the module cache
 # CHECK-NOT: static libraries
 #
 # RUN: rm -rf %t.dir/ModuleCache
-# RUN: dsymutil -f -oso-prepend-path=%t.dir -y %s -o %t 2>&1 \
+# RUN: dsymutil -verify -f -oso-prepend-path=%t.dir -y %s -o %t 2>&1 \
 # RUN:   | FileCheck %s --check-prefix=STATIC
 # STATIC: warning: {{.*}}Bar.pcm:
 # STATIC: note: Linking a static library
diff --git a/test/tools/dsymutil/X86/modules-dwarf-version.m b/test/tools/dsymutil/X86/modules-dwarf-version.m
index 2a49eeb5040ed996da71aa47e3d288a2587744c0..7c395f22e5168d315d3a7d540d42b9f544836b13 100644
--- a/test/tools/dsymutil/X86/modules-dwarf-version.m
+++ b/test/tools/dsymutil/X86/modules-dwarf-version.m
@@ -8,7 +8,7 @@
 // RUN: mkdir %t.dir
 // RUN: cp %p/../Inputs/modules/Bar.pcm %t.dir
 // RUN: cp %p/../Inputs/modules-dwarf-version/1.o %t.dir
-// RUN: dsymutil -f -oso-prepend-path=%t.dir \
+// RUN: dsymutil -verify -f -oso-prepend-path=%t.dir \
 // RUN:   -y %p/dummy-debug-map.map -o - \
 // RUN:     | llvm-dwarfdump --debug-info - | FileCheck %s
 
diff --git a/test/tools/dsymutil/X86/modules-empty.m b/test/tools/dsymutil/X86/modules-empty.m
index 6f956772465d49ce6e60eb4e1495b3b6917764ec..ce5ab427af57d388047fea81a89efc381b6dcecf 100644
--- a/test/tools/dsymutil/X86/modules-empty.m
+++ b/test/tools/dsymutil/X86/modules-empty.m
@@ -14,6 +14,7 @@ EOF
 // RUN: mkdir %t.dir
 // RUN: cp %p/../Inputs/modules-empty/1.o %p/../Inputs/modules-empty/Empty.pcm %t.dir
 // RUN: dsymutil -f -oso-prepend-path=%t.dir \
+// RUN:   -verify \
 // RUN:   -y %p/dummy-debug-map.map -o - \
 // RUN:     | llvm-dwarfdump --debug-info - | FileCheck %s
 
diff --git a/test/tools/dsymutil/X86/modules-pruning.cpp b/test/tools/dsymutil/X86/modules-pruning.cpp
index 7f331f183f0a5e22b8aeedd6fcd4d0aef446cf97..2df39e173c92efd3167ca2776f8990f539f0dcc7 100644
--- a/test/tools/dsymutil/X86/modules-pruning.cpp
+++ b/test/tools/dsymutil/X86/modules-pruning.cpp
@@ -1,4 +1,5 @@
 // RUN: dsymutil -f -oso-prepend-path=%p/../Inputs/modules-pruning \
+// RUN:   -verify \
 // RUN:   -y %p/dummy-debug-map.map -o - \
 // RUN:     | llvm-dwarfdump --name isRef -p - | FileCheck %s
 
diff --git a/test/tools/dsymutil/cmdline.test b/test/tools/dsymutil/cmdline.test
index c2ddead320a844b6e32ffbe1da1a176f40b57d40..60a1a0a2d10f9cd90eeca76c473b3e87559f8276 100644
--- a/test/tools/dsymutil/cmdline.test
+++ b/test/tools/dsymutil/cmdline.test
@@ -17,6 +17,7 @@ HELP: -num-threads=<n>
 HELP: -o=<filename>
 HELP: -oso-prepend-path=<path>
 HELP: -papertrail
+HELP: -symbol-map
 HELP: -symtab
 HELP: -toolchain
 HELP: -update
diff --git a/test/tools/gold/X86/emit-asm.ll b/test/tools/gold/X86/emit-asm.ll
new file mode 100644
index 0000000000000000000000000000000000000000..40ff71f6143e2a31babe9a8a37201d6db76f17ea
--- /dev/null
+++ b/test/tools/gold/X86/emit-asm.ll
@@ -0,0 +1,25 @@
+; RUN: llvm-as %s -o %t.o
+
+; RUN: %gold -plugin %llvmshlibdir/LLVMgold%shlibext \
+; RUN:    -m elf_x86_64 --plugin-opt=emit-asm \
+; RUN:    -shared %t.o -o %t2.s
+; RUN: FileCheck --input-file %t2.s %s
+
+; RUN: %gold -plugin %llvmshlibdir/LLVMgold%shlibext \
+; RUN:    -m elf_x86_64 --plugin-opt=emit-asm --plugin-opt=lto-partitions=2\
+; RUN:    -shared %t.o -o %t2.s
+; RUN: cat %t2.s %t2.s1 > %t3.s
+; RUN: FileCheck --input-file %t3.s %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; CHECK-DAG: f1:
+define void @f1() {
+  ret void
+}
+
+; CHECK-DAG: f2:
+define void @f2() {
+  ret void
+}
diff --git a/test/tools/gold/X86/split-dwarf.ll b/test/tools/gold/X86/split-dwarf.ll
index ce7ebc946a9dc1cf9d6e40068935a2d12554899e..6746dfd0b38943e691b1d2d99ab1d1acddb08f4d 100644
--- a/test/tools/gold/X86/split-dwarf.ll
+++ b/test/tools/gold/X86/split-dwarf.ll
@@ -47,13 +47,13 @@ attributes #0 = { noinline nounwind optnone uwtable "correctly-rounded-divide-sq
 !llvm.module.flags = !{!3, !4, !5}
 !llvm.ident = !{!6}
 
-!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 7.0.0 (https://github.com/llvm-mirror/clang.git b641d31365414ba3ea0305fdaa80369a9efb6bd9) (https://github.com/llvm-mirror/llvm.git 6165a776d1a8bb181be93f2dc97088f7a1abc405)", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !2)
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 7.0.0", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !2)
 !1 = !DIFile(filename: "split-dwarf.c", directory: "/usr/local/google/home/yunlian/dwp/build/bin")
 !2 = !{}
 !3 = !{i32 2, !"Dwarf Version", i32 4}
 !4 = !{i32 2, !"Debug Info Version", i32 3}
 !5 = !{i32 1, !"wchar_size", i32 4}
-!6 = !{!"clang version 7.0.0 (https://github.com/llvm-mirror/clang.git b641d31365414ba3ea0305fdaa80369a9efb6bd9) (https://github.com/llvm-mirror/llvm.git 6165a776d1a8bb181be93f2dc97088f7a1abc405)"}
+!6 = !{!"clang version 7.0.0"}
 !7 = distinct !DISubprogram(name: "split_dwarf", scope: !1, file: !1, line: 1, type: !8, isLocal: false, isDefinition: true, scopeLine: 1, isOptimized: false, unit: !0, retainedNodes: !2)
 !8 = !DISubroutineType(types: !9)
 !9 = !{!10}
diff --git a/test/tools/llvm-ar/Inputs/a-plus-b.a b/test/tools/llvm-ar/Inputs/a-plus-b.a
new file mode 100644
index 0000000000000000000000000000000000000000..a58406edbfb3db2db504c40d5bec5cc54f3ca1d5
--- /dev/null
+++ b/test/tools/llvm-ar/Inputs/a-plus-b.a
@@ -0,0 +1,6 @@
+!<thin>
+//                                              14        `
+a.txt/
+b.txt/
+/0              0           0     0     644     11        `
+/7              0           0     0     644     11        `
diff --git a/test/tools/llvm-ar/Inputs/a.txt b/test/tools/llvm-ar/Inputs/a.txt
new file mode 100644
index 0000000000000000000000000000000000000000..12d91b617797f7d932b225e00d569e4979c054d4
--- /dev/null
+++ b/test/tools/llvm-ar/Inputs/a.txt
@@ -0,0 +1 @@
+a-contents
diff --git a/test/tools/llvm-ar/Inputs/b.txt b/test/tools/llvm-ar/Inputs/b.txt
new file mode 100644
index 0000000000000000000000000000000000000000..aee476f3937d382551aaca472b5cfa075ae91253
--- /dev/null
+++ b/test/tools/llvm-ar/Inputs/b.txt
@@ -0,0 +1 @@
+b-contents
diff --git a/test/tools/llvm-ar/Inputs/c.txt b/test/tools/llvm-ar/Inputs/c.txt
new file mode 100644
index 0000000000000000000000000000000000000000..a80e36bc3f05eeba7446ad46e97b201acba9a13a
--- /dev/null
+++ b/test/tools/llvm-ar/Inputs/c.txt
@@ -0,0 +1 @@
+c-contents
diff --git a/test/tools/llvm-ar/Inputs/d.txt b/test/tools/llvm-ar/Inputs/d.txt
new file mode 100644
index 0000000000000000000000000000000000000000..20d7aa0cf37b2be3ccb7f8de7338db787b69cc98
--- /dev/null
+++ b/test/tools/llvm-ar/Inputs/d.txt
@@ -0,0 +1 @@
+d-contents
diff --git a/test/tools/llvm-ar/Inputs/nested-thin-archive.a b/test/tools/llvm-ar/Inputs/nested-thin-archive.a
new file mode 100644
index 0000000000000000000000000000000000000000..312e8de8876524d267ed3130d9927c096d25a97d
--- /dev/null
+++ b/test/tools/llvm-ar/Inputs/nested-thin-archive.a
@@ -0,0 +1,7 @@
+!<thin>
+//                                              20        `
+a-plus-b.a/
+c.txt/
+
+/0              0           0     0     644     202       `
+/12             0           0     0     644     11        `
diff --git a/test/tools/llvm-ar/flatten-thin-archive-recursive.test b/test/tools/llvm-ar/flatten-thin-archive-recursive.test
new file mode 100644
index 0000000000000000000000000000000000000000..fdd752d48b24fcec16dfee727d844fe10f824a36
--- /dev/null
+++ b/test/tools/llvm-ar/flatten-thin-archive-recursive.test
@@ -0,0 +1,13 @@
+# Since llvm-ar cannot create thin archives that contain any thin archives,
+# nested-thin-archive.a is a manually constructed thin archive that contains
+# another (unflattened) thin archive.
+# This test ensures that flat archives are recursively flattened.
+
+RUN: rm -f %t.a
+RUN: llvm-ar rcsT %t.a %S/Inputs/nested-thin-archive.a %S/Inputs/d.txt
+RUN: llvm-ar t %t.a | FileCheck %s
+
+CHECK:      a.txt
+CHECK-NEXT: b.txt
+CHECK-NEXT: c.txt
+CHECK-NEXT: d.txt
diff --git a/test/tools/llvm-ar/flatten-thin-archive.test b/test/tools/llvm-ar/flatten-thin-archive.test
new file mode 100644
index 0000000000000000000000000000000000000000..430f48fe7c499d209757b5316d8ca780975cd0e7
--- /dev/null
+++ b/test/tools/llvm-ar/flatten-thin-archive.test
@@ -0,0 +1,18 @@
+# This test creates a thin archive that contains a thin archive, a regular
+# archive, and a file.
+#
+# The inner thin archive should be flattened, but the regular archive should
+# not. The order of members in the archive should match the input order, with
+# flattened members appearing together.
+
+RUN: touch %t-a.txt %t-b.txt %t-c.txt %t-d.txt %t-e.txt
+RUN: rm -f %t-a-plus-b.a %t.a
+RUN: llvm-ar rcsT %t-a-plus-b.a %t-a.txt %t-b.txt
+RUN: llvm-ar rcs %t-d-plus-e.a %t-d.txt %t-e.txt
+RUN: llvm-ar rcsT %t.a %t-a-plus-b.a %t-c.txt %t-d-plus-e.a
+RUN: llvm-ar t %t.a | FileCheck %s
+
+CHECK:      a.txt
+CHECK-NEXT: b.txt
+CHECK-NEXT: c.txt
+CHECK-NEXT: -d-plus-e.a
diff --git a/test/tools/llvm-ar/print-thin-archive-path.test b/test/tools/llvm-ar/print-thin-archive-path.test
new file mode 100644
index 0000000000000000000000000000000000000000..f6cb0cf75183ed5ccc78e138fe3469013355dbf6
--- /dev/null
+++ b/test/tools/llvm-ar/print-thin-archive-path.test
@@ -0,0 +1,10 @@
+RUN: mkdir -p %t/foo
+RUN: touch %t/foo/a.txt
+RUN: rm -f %t/foo/archive.a
+
+RUN: cd %t && llvm-ar rcST foo/archive.a foo/a.txt
+RUN: cd %t && llvm-ar t foo/archive.a | FileCheck %s --match-full-lines --check-prefix=PARENT-DIR
+RUN: cd %t/foo && llvm-ar t archive.a | FileCheck %s --match-full-lines --check-prefix=CHILD-DIR
+
+PARENT-DIR: foo/a.txt
+CHILD-DIR:  a.txt
diff --git a/test/tools/llvm-config/booleans.test b/test/tools/llvm-config/booleans.test
index b28f293666f35d12424541066cddd822999212ea..1db3383020ccec26c2377affd2e743c2e49f62ae 100644
--- a/test/tools/llvm-config/booleans.test
+++ b/test/tools/llvm-config/booleans.test
@@ -18,7 +18,7 @@ CHECK-BUILD-MODE-NOT: error:
 CHECK-BUILD-MODE-NOT: warning
 
 RUN: llvm-config --build-system 2>&1 | FileCheck --check-prefix=CHECK-BUILD-SYSTEM %s
-CHECK-BUILD-SYSTEM: cmake
+CHECK-BUILD-SYSTEM: {{cmake|gn}}
 CHECK-BUILD-SYSTEM-NOT: error:
 CHECK-BUILD-SYSTEM-NOT: warning
 
diff --git a/test/tools/llvm-cxxfilt/simple.test b/test/tools/llvm-cxxfilt/simple.test
index 76e70dbba303f6511b67d898167fd3a6e64f84d1..90dd18d58dcba7d6eb28fa9ce28f4d11c6095f68 100644
--- a/test/tools/llvm-cxxfilt/simple.test
+++ b/test/tools/llvm-cxxfilt/simple.test
@@ -1,4 +1,10 @@
 RUN: llvm-cxxfilt _Z1fi abc | FileCheck %s
+RUN: echo "Mangled _Z1fi and _Z3foov in string." | llvm-cxxfilt \
+RUN:   | FileCheck %s --check-prefix=CHECK-STRING
+RUN: llvm-cxxfilt "CLI remains mangled _Z1fi" \
+RUN:   | FileCheck %s --check-prefix=CHECK-MANGLED
 
 CHECK: f(int)
 CHECK-NEXT: abc
+CHECK-STRING: Mangled f(int) and foo() in string.
+CHECK-MANGLED: CLI remains mangled _Z1fi
diff --git a/test/tools/llvm-cxxfilt/types.test b/test/tools/llvm-cxxfilt/types.test
index 4f0b2ecaab16320b96d275a73a961c5e1372a17c..2213c386f82e360fee42cc2de635b26cada684d6 100644
--- a/test/tools/llvm-cxxfilt/types.test
+++ b/test/tools/llvm-cxxfilt/types.test
@@ -1,5 +1,6 @@
 RUN: llvm-cxxfilt -t f i | FileCheck %s
+RUN: echo "f i" | llvm-cxxfilt -t | FileCheck %s --check-prefix="CHECK-STRING"
 
 CHECK: float
 CHECK-NEXT: int
-
+CHECK-STRING: float int
diff --git a/test/tools/llvm-dwarfdump/X86/debug_info_addrx.s b/test/tools/llvm-dwarfdump/X86/debug_info_addrx.s
new file mode 100644
index 0000000000000000000000000000000000000000..e0e468c9153cb44c39160d063a8a97e778fa3174
--- /dev/null
+++ b/test/tools/llvm-dwarfdump/X86/debug_info_addrx.s
@@ -0,0 +1,77 @@
+# RUN: llvm-mc %s -filetype obj -triple x86_64-pc-linux -o %t.o
+# RUN: llvm-dwarfdump -debug-info %t.o | FileCheck %s
+# RUN: llvm-dwarfdump -debug-info %t.o -v | FileCheck --check-prefix=VERBOSE %s
+
+# CHECK: DW_TAG_compile_unit
+# CHECK:   DW_AT_low_pc                                              (0x0000000000000000)
+# VERBOSE: DW_AT_low_pc [DW_FORM_addrx] (indexed (00000000) address = 0x0000000000000000 ".text")
+# FIXME: There is a debug_addr section, it's just that the index is outside its 
+#        bounds (both of the section, and the range defined by the header for the
+#        debug_addr contribution for this CU)
+# CHECK:   DW_AT_low_pc                 (indexed (00000001) address = <no .debug_addr section>)
+# VERBOSE: DW_AT_low_pc [DW_FORM_addrx] (indexed (00000001) address = <no .debug_addr section>)
+
+# CHECK: DW_TAG_compile_unit
+# FIXME: Should error "no debug_addr contribution" - rather than parsing debug_addr
+#        from the start, incorrectly interpreting the header bytes as an address.
+# CHECK:   DW_AT_low_pc                                              (0x000800050000000c)
+# VERBOSE: DW_AT_low_pc [DW_FORM_addrx] (indexed (00000000) address = 0x000800050000000c)
+
+	.globl	foo                     # -- Begin function foo
+foo:                                    # @foo
+.Lfunc_begin0:
+	retq
+.Lfunc_end0:
+	.section	.debug_abbrev,"",@progbits
+	.byte	1                       # Abbreviation Code
+	.byte	17                      # DW_TAG_compile_unit
+	.byte	0                       # DW_CHILDREN_no
+	.byte	115                     # DW_AT_addr_base
+	.byte	23                      # DW_FORM_sec_offset
+	.byte	17                      # DW_AT_low_pc
+	.byte	27                      # DW_FORM_addrx
+	.byte	17                      # DW_AT_low_pc
+	.byte	27                      # DW_FORM_addrx
+	.byte	0                       # EOM(1)
+	.byte	0                       # EOM(2)
+	.byte	2                       # Abbreviation Code
+	.byte	17                      # DW_TAG_compile_unit
+	.byte	0                       # DW_CHILDREN_no
+	.byte	17                      # DW_AT_low_pc
+	.byte	27                      # DW_FORM_addrx
+	.byte	0                       # EOM(1)
+	.byte	0                       # EOM(2)
+	.byte	0                       # EOM(3)
+	.section	.debug_info,"",@progbits
+.Lcu_begin0:
+	.long	.Ldebug_info_end0-.Ldebug_info_start0 # Length of Unit
+.Ldebug_info_start0:
+	.short	5                       # DWARF version number
+	.byte	1                       # DWARF Unit Type
+	.byte	8                       # Address Size (in bytes)
+	.long	.debug_abbrev           # Offset Into Abbrev. Section
+	.byte	1                       # Abbrev [1] 0xc:0x23 DW_TAG_compile_unit
+	.long	.Laddr_table_base0      # DW_AT_addr_base
+	.byte	0                       # DW_AT_low_pc
+	.byte	1                       # DW_AT_low_pc
+.Ldebug_info_end0:
+.Lcu_begin1:
+	.long	.Ldebug_info_end1-.Ldebug_info_start1 # Length of Unit
+.Ldebug_info_start1:
+	.short	5                       # DWARF version number
+	.byte	1                       # DWARF Unit Type
+	.byte	8                       # Address Size (in bytes)
+	.long	.debug_abbrev           # Offset Into Abbrev. Section
+	.byte	2                       # Abbrev [2] 0xc:0x23 DW_TAG_compile_unit
+	.long	.Laddr_table_base0      # DW_AT_addr_base
+	.byte	0                       # DW_AT_low_pc
+.Ldebug_info_end1:
+	.section	.debug_macinfo,"",@progbits
+	.byte	0                       # End Of Macro List Mark
+	.section	.debug_addr,"",@progbits
+	.long	12                      # Length of Pool
+	.short	5                       # DWARF version number
+	.byte	8                       # Address Size (in bytes)
+	.byte	0                       # Segment selector size
+.Laddr_table_base0:
+	.quad	.Lfunc_begin0
diff --git a/test/tools/llvm-dwarfdump/X86/debug_loclists_startx_length.s b/test/tools/llvm-dwarfdump/X86/debug_loclists_startx_length.s
index 07c68ab2618f157256856ac01c0a1d96ed375728..0b2ae5f8e7a4fb56ec20082f9b55ee787ddb6dbe 100644
--- a/test/tools/llvm-dwarfdump/X86/debug_loclists_startx_length.s
+++ b/test/tools/llvm-dwarfdump/X86/debug_loclists_startx_length.s
@@ -6,7 +6,7 @@
 # the final version which uses ULEB128 and not the U32.
 
 # CHECK:         .debug_loclists contents:
-# CHECK-NEXT:    0x00000000: locations list header: length = 0x0000000f, version = 0x0005, addr_size = 0x08, seg_size = 0x00, offset_entry_count = 0x00000000
+# CHECK-NEXT:    0x00000000: locations list header: length = 0x0000000e, version = 0x0005, addr_size = 0x08, seg_size = 0x00, offset_entry_count = 0x00000000
 # CHECK-NEXT:    0x00000000:
 # CHECK-NEXT:    Addr idx 1 (w/ length 16): DW_OP_reg5 RDI
 
@@ -21,7 +21,7 @@
  .byte 3          # DW_LLE_startx_length
  .byte 0x01       # Index
  .uleb128 0x10    # Length
- .short 1         # Loc expr size
+ .byte 1          # Loc expr size
  .byte 85         # DW_OP_reg5
  .byte 0          # DW_LLE_end_of_list
 .Ldebug_loclist_table_end0:
diff --git a/test/tools/llvm-dwarfdump/X86/prettyprint_types.s b/test/tools/llvm-dwarfdump/X86/prettyprint_types.s
index b3e871e7c82e270d77b6aa2430e51049f34a4a16..afeee4cfba644e58441be211a408a8ee8426e0bc 100644
--- a/test/tools/llvm-dwarfdump/X86/prettyprint_types.s
+++ b/test/tools/llvm-dwarfdump/X86/prettyprint_types.s
@@ -19,13 +19,41 @@
 # CHECK:   DW_AT_type{{.*}}"int foo::*"
 
 # array_type
-# Testing lower_bound, upper_bound, lower and upper, lower and count, and count separately.
-# CHECK:   DW_AT_type{{.*}}"int[1-][2][1-2][1-3][2]"
+# CHECK:   DW_AT_type{{.*}}"int
+# Testing with a default lower bound of 0 and the following explicit bounds:
+#   lower_bound(1)
+# CHECK-NOT: {{.}}
+# CHECK-SAME: {{\[}}[1, ?)]
+#   upper_bound(2)
+# CHECK-NOT: {{.}}
+# CHECK-SAME: [3]
+#   lower(1) and upper(2)
+# CHECK-NOT: {{.}}
+# CHECK-SAME: {{\[}}[1, 3)]
+#   lower(1) and count(3)
+# CHECK-NOT: {{.}}
+# CHECK-SAME: {{\[}}[1, 4)]
+#   lower(0) and count(4) - testing that the lower bound matching language
+#   default is not rendered
+# CHECK-NOT: {{.}}
+# CHECK-SAME: [4]
+#   count(2)
+# CHECK-SAME: [2]
+#   no attributes
+# CHECK-NOT: {{.}}
+# CHECK-SAME: []{{"\)$}}
+
 
 # subroutine types
 # CHECK:   DW_AT_type{{.*}}"int()"
 # CHECK:   DW_AT_type{{.*}}"void(int)"
 # CHECK:   DW_AT_type{{.*}}"void(int, int)"
+
+# array_type with a language with a default lower bound of 1 instead of 0 and
+# an upper bound of 2. This describes an array with 2 elements (whereas with a
+# default lower bound of 0 it would be an array of 3 elements)
+# CHECK: DW_AT_type{{.*}}"int[2]"
+
 	.section	.debug_str,"MS",@progbits,1
 .Lint_name:
 	.asciz	"int"
@@ -155,6 +183,11 @@
 	.byte	19                      # DW_FORM_ref4
 	.byte	0                       # EOM(1)
 	.byte	0                       # EOM(2)
+	.byte	18                      # Abbreviation Code
+	.byte	0x21                    # DW_TAG_subrange_type
+	.byte	0                       # DW_CHILDREN_no
+	.byte	0                       # EOM(1)
+	.byte	0                       # EOM(2)
 	.byte	0                       # EOM(3)
 	.section	.debug_info,"",@progbits
 .Lcu_begin:
@@ -196,9 +229,13 @@
 	.byte   2                       #     DW_AT_upper_bound
 	.byte	12                      #   DW_AT_subrange_type
 	.byte   1                       #     DW_AT_lower_bound
-	.byte   2                       #     DW_AT_count
+	.byte   3                       #     DW_AT_count
+	.byte	12                      #   DW_AT_subrange_type
+	.byte   0                       #     DW_AT_lower_bound
+	.byte   4                       #     DW_AT_count
 	.byte	13                      #   DW_AT_subrange_type
 	.byte   2                       #     DW_AT_count
+	.byte	18                      #   DW_AT_subrange_type
 	.byte	0                       # End Of Children Mark
 .Lsub_int_empty_type:
 	.byte	15                      # DW_TAG_subroutine_type
@@ -236,3 +273,22 @@
 	.long	.Lsub_void_int_int_type - .Lcu_begin #   DW_AT_type
 	.byte	0                       # End Of Children Mark
 .Lunit_end:
+.Lcu2_begin:
+	.long	.Lcu2_unit_end - .Lcu2_unit_start # Length of Unit
+.Lcu2_unit_start:
+	.short	4                       # DWARF version number
+	.long	.debug_abbrev           # Offset Into Abbrev. Section
+	.byte	8                       # Address Size (in bytes)
+	.byte	1                       # DW_TAG_compile_unit
+	.short	13                      #   DW_AT_language
+.Lcu2_int_type:
+	.byte	2                       # DW_TAG_base_type
+	.long	.Lint_name              #   DW_AT_name
+.Lcu2_array_type:
+	.byte	8                       # DW_TAG_array_type
+	.long	.Lcu2_int_type - .Lcu2_begin #   DW_AT_type
+	.byte	10                      #   DW_AT_subrange_type
+	.byte   2                       #     DW_AT_upper_bound
+	.byte	3                       # DW_TAG_variable
+	.long	.Lcu2_array_type - .Lcu2_begin #   DW_AT_type
+.Lcu2_unit_end:
diff --git a/test/tools/llvm-dwarfdump/X86/statistics.ll b/test/tools/llvm-dwarfdump/X86/statistics.ll
index 0ddae7e39acb4a8fc6e8d5ab8307998cb945b690..73c56dabf87e4a3c2571b1f7e3cf8da0d130d774 100644
--- a/test/tools/llvm-dwarfdump/X86/statistics.ll
+++ b/test/tools/llvm-dwarfdump/X86/statistics.ll
@@ -1,6 +1,6 @@
 ; RUN: llc -O0 %s -o - -filetype=obj \
 ; RUN:   | llvm-dwarfdump -statistics - | FileCheck %s
-; CHECK: "version":1
+; CHECK: "version":2
 
 ; int GlobalConst = 42;
 ; int Global;
diff --git a/test/tools/llvm-dwarfdump/X86/stats-inlining-multi-cu.ll b/test/tools/llvm-dwarfdump/X86/stats-inlining-multi-cu.ll
index f8656e6a7b718fe0fcb7e61a56ab3c3bf726af62..e6e193c2004dab89034887796cb61eee8f2ea4ad 100644
--- a/test/tools/llvm-dwarfdump/X86/stats-inlining-multi-cu.ll
+++ b/test/tools/llvm-dwarfdump/X86/stats-inlining-multi-cu.ll
@@ -4,7 +4,7 @@
 ; Test that abstract origins in multiple CUs are uniqued.
 
 ; CHECK:      "source functions":4,
-; CHECK-SAME: "inlined functions":5,
+; CHECK-SAME: "inlined functions":2,
 ; CHECK-SAME: "unique source variables":4
 ; CHECK-SAME: "source variables":6
 ; CHECK-SAME: "variables with location":6
diff --git a/test/tools/llvm-dwarfdump/X86/stats-inlining-single-cu.ll b/test/tools/llvm-dwarfdump/X86/stats-inlining-single-cu.ll
index 14ef709613278e56b86859dd6f298792c6955b75..5a4c3541a5c44c9210a143f447b3f0da7b422da4 100644
--- a/test/tools/llvm-dwarfdump/X86/stats-inlining-single-cu.ll
+++ b/test/tools/llvm-dwarfdump/X86/stats-inlining-single-cu.ll
@@ -5,7 +5,7 @@
 ; The results for both tests should be identical.
 
 ; CHECK:      "source functions":4,
-; CHECK-SAME: "inlined functions":5,
+; CHECK-SAME: "inlined functions":2,
 ; CHECK-SAME: "unique source variables":4
 ; CHECK-SAME: "source variables":6
 ; CHECK-SAME: "variables with location":6
diff --git a/test/tools/llvm-dwarfdump/X86/verbose.test b/test/tools/llvm-dwarfdump/X86/verbose.test
index 2d52247f15f2da96fcba2040d624cdac40edbc9f..a05edb9a2ae13d1530d20ca6b0ac5e8d7b998f1e 100644
--- a/test/tools/llvm-dwarfdump/X86/verbose.test
+++ b/test/tools/llvm-dwarfdump/X86/verbose.test
@@ -16,7 +16,7 @@
 # CHECK: 0x0000000000000000      1      0      1   0             0  is_stmt
 # CHECK: 05 DW_LNS_set_column (12)
 # CHECK: 0a DW_LNS_set_prologue_end
-# CHECK: 66 address += 89,  line += 0
+# CHECK: 66 address += 6,  line += 0
 # CHECK: 0x0000000000000006      1     12      1   0             0  is_stmt prologue_end
 # CHECK: 02 DW_LNS_advance_pc (2)
 # CHECK: 00 DW_LNE_end_sequence
diff --git a/test/tools/llvm-dwarfdump/X86/verify_debug_info2.s b/test/tools/llvm-dwarfdump/X86/verify_debug_info2.s
index 71b955798396de6721819dd98528d96d6601c5f1..8ac10521b22526340ae57cce8cafe6270e8dcf63 100644
--- a/test/tools/llvm-dwarfdump/X86/verify_debug_info2.s
+++ b/test/tools/llvm-dwarfdump/X86/verify_debug_info2.s
@@ -14,7 +14,7 @@
   .byte  0x4   # DW_AT_language [DW_FORM_data1] (DW_LANG_C_plus_plus)
   .long  0     # DW_AT_name [DW_FORM_strp] ( .debug_str[0x00000000] = "test")
   .long  0     # DW_AT_comp_dir [DW_FORM_strp] ( .debug_str[0x00000000] = "test")
-  .long  0     # DW_AT_low_pc [DW_FORM_addr] (0x0000000000000000)
+  .long  0     # DW_AT_low_pc [DW_FORM_addr] (0x0000000000000000 ".text")
   .long  0     # DW_AT_high_pc [DW_FORM_data4] (0x00000000)
 
 .section  .debug_abbrev,"",@progbits
diff --git a/test/tools/llvm-dwp/Inputs/missing_dwo_id.dwo b/test/tools/llvm-dwp/Inputs/missing_dwo_id.dwo
new file mode 100644
index 0000000000000000000000000000000000000000..f92b4583c507e89eec4520e46ba04b23e49c77a4
Binary files /dev/null and b/test/tools/llvm-dwp/Inputs/missing_dwo_id.dwo differ
diff --git a/test/tools/llvm-dwp/X86/invalid_string_form.test b/test/tools/llvm-dwp/X86/invalid_string_form.test
index e78a145dcb59e38f578089401d94439260a9656e..2d3cf6fe28c6cf116da9199ade3d167673ba4a4c 100644
--- a/test/tools/llvm-dwp/X86/invalid_string_form.test
+++ b/test/tools/llvm-dwp/X86/invalid_string_form.test
@@ -1,3 +1,3 @@
 RUN: not llvm-dwp %p/../Inputs/invalid_string_form.dwo -o %t 2>&1 | FileCheck %s
 
-CHECK: error: string field encoded without DW_FORM_string or DW_FORM_GNU_str_index
+CHECK: error: {{.*}}invalid_string_form.dwo': string field encoded without DW_FORM_string or DW_FORM_GNU_str_index
diff --git a/test/tools/llvm-dwp/X86/missing_dwo_id.test b/test/tools/llvm-dwp/X86/missing_dwo_id.test
new file mode 100644
index 0000000000000000000000000000000000000000..bb5eddccd2272e47b703af372b88904d9dda95a3
--- /dev/null
+++ b/test/tools/llvm-dwp/X86/missing_dwo_id.test
@@ -0,0 +1,7 @@
+RUN: rm -f %t
+RUN: not llvm-dwp %p/../Inputs/missing_dwo_id.dwo -o %t 2>&1 | FileCheck %s
+
+Make sure we did not leave behind a temporary file:
+RUN: not ls %t
+
+CHECK: error: {{.*}}missing_dwo_id.dwo': compile unit missing dwo_id
diff --git a/test/tools/llvm-dwp/X86/non_cu_top_level.test b/test/tools/llvm-dwp/X86/non_cu_top_level.test
index 60b8742cdc25eb4f3250f31399314cccb10036a3..92f53a703a0929d9f6738dad6408c16c7192c8ce 100644
--- a/test/tools/llvm-dwp/X86/non_cu_top_level.test
+++ b/test/tools/llvm-dwp/X86/non_cu_top_level.test
@@ -1,3 +1,3 @@
 RUN: not llvm-dwp %p/../Inputs/non_cu_top_level.dwo -o %t 2>&1 | FileCheck %s
 
-CHECK: error: top level DIE is not a compile unit
+CHECK: error: {{.*}}non_cu_top_level.dwo': top level DIE is not a compile unit
diff --git a/test/tools/llvm-elfabi/Inputs/gnu_hash.so b/test/tools/llvm-elfabi/Inputs/gnu_hash.so
new file mode 100755
index 0000000000000000000000000000000000000000..3970e587d7255b2bc3f40a94e59ef187062b7eae
Binary files /dev/null and b/test/tools/llvm-elfabi/Inputs/gnu_hash.so differ
diff --git a/test/tools/llvm-elfabi/Inputs/sysv_hash.so b/test/tools/llvm-elfabi/Inputs/sysv_hash.so
new file mode 100755
index 0000000000000000000000000000000000000000..46b3b2abd1e8a2e5605bfcfc418ce09d33529728
Binary files /dev/null and b/test/tools/llvm-elfabi/Inputs/sysv_hash.so differ
diff --git a/test/tools/llvm-elfabi/binary-read-add-soname.test b/test/tools/llvm-elfabi/binary-read-add-soname.test
new file mode 100644
index 0000000000000000000000000000000000000000..744fa51d0561a5db563c73368e3bf2114932c167
--- /dev/null
+++ b/test/tools/llvm-elfabi/binary-read-add-soname.test
@@ -0,0 +1,53 @@
+# RUN: yaml2obj %s > %t
+# RUN: llvm-elfabi --elf %t --emit-tbe=- --soname=best.so | FileCheck %s
+
+!ELF
+FileHeader:
+  Class:           ELFCLASS64
+  Data:            ELFDATA2LSB
+  Type:            ET_DYN
+  Machine:         EM_AARCH64
+Sections:
+  - Name:            .dynstr
+    Type:            SHT_STRTAB
+    Flags:           [ SHF_ALLOC ]
+    Address:         0x0000
+    Content:         "00"
+  - Name:            .dynamic
+    Type:            SHT_DYNAMIC
+    Flags:           [ SHF_ALLOC ]
+    Address:         0x0000000000000008
+    Link:            .dynstr
+    AddressAlign:    0x0000000000000008
+    EntSize:         0x0000000000000010
+    Entries:
+      - Tag:             DT_STRSZ
+        Value:           0x0000000000000001
+      - Tag:             DT_STRTAB
+        Value:           0x0000000000000000
+      - Tag:             DT_SYMTAB
+        Value:           0x0000000000000000
+      - Tag:             DT_NULL
+        Value:           0x0000000000000000
+ProgramHeaders:
+  - Type: PT_LOAD
+    Flags: [ PF_R ]
+    VAddr: 0x0000
+    PAddr: 0x0000
+    Align: 8
+    Sections:
+      - Section: .dynstr
+      - Section: .dynamic
+  - Type: PT_DYNAMIC
+    Flags: [ PF_X, PF_R ]
+    VAddr: 0x0008
+    PAddr: 0x0008
+    Sections:
+      - Section: .dynamic
+
+# CHECK:      --- !tapi-tbe
+# CHECK-NEXT: TbeVersion: {{[1-9]\d*\.(0|([1-9]\d*))}}
+# CHECK-NEXT: SoName: best.so{{$}}
+# CHECK-NEXT: Arch: AArch64
+# CHECK-NEXT: Symbols: {}
+# CHECK-NEXT: ...
diff --git a/test/tools/llvm-elfabi/binary-read-arch.test b/test/tools/llvm-elfabi/binary-read-arch.test
new file mode 100644
index 0000000000000000000000000000000000000000..918a0c667d9038e19fc0278f07dcfdaa7115a9eb
--- /dev/null
+++ b/test/tools/llvm-elfabi/binary-read-arch.test
@@ -0,0 +1,52 @@
+# RUN: yaml2obj %s > %t
+# RUN: llvm-elfabi --elf %t --emit-tbe=- | FileCheck %s
+
+!ELF
+FileHeader:
+  Class:           ELFCLASS64
+  Data:            ELFDATA2LSB
+  Type:            ET_DYN
+  Machine:         EM_X86_64
+Sections:
+  - Name:            .dynstr
+    Type:            SHT_STRTAB
+    Flags:           [ SHF_ALLOC ]
+    Address:         0x0000
+    Content:         "00"
+  - Name:            .dynamic
+    Type:            SHT_DYNAMIC
+    Flags:           [ SHF_ALLOC ]
+    Address:         0x0000000000000008
+    Link:            .dynstr
+    AddressAlign:    0x0000000000000008
+    EntSize:         0x0000000000000010
+    Entries:
+      - Tag:             DT_STRSZ
+        Value:           0x0000000000000001
+      - Tag:             DT_STRTAB
+        Value:           0x0000000000000000
+      - Tag:             DT_SYMTAB
+        Value:           0x0000000000000000
+      - Tag:             DT_NULL
+        Value:           0x0000000000000000
+ProgramHeaders:
+  - Type: PT_LOAD
+    Flags: [ PF_R ]
+    VAddr: 0x0000
+    PAddr: 0x0000
+    Align: 8
+    Sections:
+      - Section: .dynstr
+      - Section: .dynamic
+  - Type: PT_DYNAMIC
+    Flags: [ PF_X, PF_R ]
+    VAddr: 0x0008
+    PAddr: 0x0008
+    Sections:
+      - Section: .dynamic
+
+# CHECK:      --- !tapi-tbe
+# CHECK-NEXT: TbeVersion: {{[1-9]\d*\.(0|([1-9]\d*))}}
+# CHECK-NEXT: Arch: x86_64
+# CHECK-NEXT: Symbols: {}
+# CHECK-NEXT: ...
diff --git a/test/tools/llvm-elfabi/binary-read-bad-soname.test b/test/tools/llvm-elfabi/binary-read-bad-soname.test
new file mode 100644
index 0000000000000000000000000000000000000000..a92d61e607d0b3374b372f9dd14e0b5d2e21144c
--- /dev/null
+++ b/test/tools/llvm-elfabi/binary-read-bad-soname.test
@@ -0,0 +1,50 @@
+# RUN: yaml2obj %s > %t
+# RUN: not llvm-elfabi --elf %t --emit-tbe=%t.tbe 2>&1 | FileCheck %s
+
+!ELF
+FileHeader:
+  Class:           ELFCLASS64
+  Data:            ELFDATA2LSB
+  Type:            ET_DYN
+  Machine:         EM_X86_64
+Sections:
+  - Name:            .dynstr
+    Type:            SHT_STRTAB
+    Flags:           [ SHF_ALLOC ]
+    Address:         0x0000
+    Content:         "00"
+  - Name:            .dynamic
+    Type:            SHT_DYNAMIC
+    Flags:           [ SHF_ALLOC ]
+    Address:         0x0000000000000008
+    Link:            .dynstr
+    AddressAlign:    0x0000000000000008
+    EntSize:         0x0000000000000010
+    Entries:
+      - Tag:             DT_SONAME
+        Value:           0x000000000000000D
+      - Tag:             DT_STRSZ
+        Value:           0x0000000000000001
+      - Tag:             DT_STRTAB
+        Value:           0x0000000000000000
+      - Tag:             DT_SYMTAB
+        Value:           0x0000000000000000
+      - Tag:             DT_NULL
+        Value:           0x0000000000000000
+ProgramHeaders:
+  - Type: PT_LOAD
+    Flags: [ PF_R ]
+    VAddr: 0x0000
+    PAddr: 0x0000
+    Align: 8
+    Sections:
+      - Section: .dynstr
+      - Section: .dynamic
+  - Type: PT_DYNAMIC
+    Flags: [ PF_X, PF_R ]
+    VAddr: 0x0008
+    PAddr: 0x0008
+    Sections:
+      - Section: .dynamic
+
+# CHECK: DT_SONAME string offset (0x000000000000000d) outside of dynamic string table
diff --git a/test/tools/llvm-elfabi/binary-read-bad-vaddr.test b/test/tools/llvm-elfabi/binary-read-bad-vaddr.test
new file mode 100644
index 0000000000000000000000000000000000000000..b8918b980e8412d061804e5b63e460ce39193775
--- /dev/null
+++ b/test/tools/llvm-elfabi/binary-read-bad-vaddr.test
@@ -0,0 +1,50 @@
+# RUN: yaml2obj %s > %t
+# RUN: not llvm-elfabi --elf %t --emit-tbe=%t.tbe 2>&1 | FileCheck %s
+
+!ELF
+FileHeader:
+  Class:           ELFCLASS64
+  Data:            ELFDATA2LSB
+  Type:            ET_DYN
+  Machine:         EM_X86_64
+Sections:
+  - Name:            .dynstr
+    Type:            SHT_STRTAB
+    Flags:           [ SHF_ALLOC ]
+    Address:         0x1000
+    Content:         "00"
+  - Name:            .dynamic
+    Type:            SHT_DYNAMIC
+    Flags:           [ SHF_ALLOC ]
+    Address:         0x0000000000001008
+    Link:            .dynstr
+    AddressAlign:    0x0000000000000008
+    EntSize:         0x0000000000000010
+    Entries:
+      - Tag:             DT_SONAME
+        Value:           0x0000000000000000
+      - Tag:             DT_STRSZ
+        Value:           0x0000000000000001
+      - Tag:             DT_STRTAB
+        Value:           0x0000000000000260 # Bad vaddr (no PT_LOAD for 0x0000 to 0x0FFF)
+      - Tag:             DT_SYMTAB
+        Value:           0x0000000000001000
+      - Tag:             DT_NULL
+        Value:           0x0000000000000000
+ProgramHeaders:
+  - Type: PT_LOAD
+    Flags: [ PF_R ]
+    VAddr: 0x1000
+    PAddr: 0x1000
+    Align: 8
+    Sections:
+      - Section: .dynstr
+      - Section: .dynamic
+  - Type: PT_DYNAMIC
+    Flags: [ PF_X, PF_R ]
+    VAddr: 0x1008
+    PAddr: 0x1008
+    Sections:
+      - Section: .dynamic
+
+# CHECK: Virtual address is not in any segment when locating .dynstr section contents
diff --git a/test/tools/llvm-elfabi/binary-read-neededlibs-bad-offset.test b/test/tools/llvm-elfabi/binary-read-neededlibs-bad-offset.test
new file mode 100644
index 0000000000000000000000000000000000000000..6b7ba8b2d1989bc3a7fc2a4a7e0c714158efd5b8
--- /dev/null
+++ b/test/tools/llvm-elfabi/binary-read-neededlibs-bad-offset.test
@@ -0,0 +1,53 @@
+# RUN: yaml2obj %s > %t
+# RUN: not llvm-elfabi --elf %t --emit-tbe=%t.tbe 2>&1 | FileCheck %s
+
+!ELF
+FileHeader:
+  Class:           ELFCLASS64
+  Data:            ELFDATA2LSB
+  Type:            ET_DYN
+  Machine:         EM_X86_64
+Sections:
+  - Name:            .dynstr
+    Type:            SHT_STRTAB
+    Flags:           [ SHF_ALLOC ]
+    Address:         0x1000
+    #                "\0libfoo.so\0libbar.so\0somelib.so\0foo\0"
+    Content:         "006c6962666f6f2e736f006c69626261722e736f00736f6d656c69622e736f00666f6f00"
+  - Name:            .dynamic
+    Type:            SHT_DYNAMIC
+    Flags:           [ SHF_ALLOC ]
+    Address:         0x0000000000001024
+    EntSize:         0x0000000000000010
+    Entries:
+      - Tag:             DT_NEEDED
+        Value:           0x0000000000000001
+      - Tag:             DT_SONAME
+        Value:           0x0000000000000015
+      # Bad DT_NEEDED entry (offset outside string table):
+      - Tag:             DT_NEEDED
+        Value:           0x000000000000FFFF
+      - Tag:             DT_STRSZ
+        Value:           0x0000000000000024
+      - Tag:             DT_STRTAB
+        Value:           0x0000000000001000
+      - Tag:             DT_SYMTAB
+        Value:           0x0000000000001000
+      - Tag:             DT_NULL
+        Value:           0x0000000000000000
+ProgramHeaders:
+  - Type: PT_LOAD
+    Flags: [ PF_R ]
+    VAddr: 0x1000
+    PAddr: 0x1000
+    Align: 8
+    Sections:
+      - Section: .dynstr
+      - Section: .dynamic
+  - Type: PT_DYNAMIC
+    Flags: [ PF_X, PF_R ]
+    VAddr: 0x1024
+    PAddr: 0x1024
+    Sections:
+
+# CHECK: DT_NEEDED string offset (0x000000000000ffff) outside of dynamic string table
diff --git a/test/tools/llvm-elfabi/binary-read-neededlibs.test b/test/tools/llvm-elfabi/binary-read-neededlibs.test
new file mode 100644
index 0000000000000000000000000000000000000000..09567e00e3599b2fdb222c17f1cd7e292b06b6d3
--- /dev/null
+++ b/test/tools/llvm-elfabi/binary-read-neededlibs.test
@@ -0,0 +1,55 @@
+# RUN: yaml2obj %s -o %t
+# RUN: llvm-elfabi --elf %t --emit-tbe=- | FileCheck %s
+
+!ELF
+FileHeader:
+  Class:           ELFCLASS64
+  Data:            ELFDATA2LSB
+  Type:            ET_DYN
+  Machine:         EM_X86_64
+Sections:
+  - Name:            .dynstr
+    Type:            SHT_STRTAB
+    Flags:           [ SHF_ALLOC ]
+    Address:         0x1000
+    #                "\0libfoo.so\0libbar.so\0somelib.so\0foo\0"
+    Content:         "006c6962666f6f2e736f006c69626261722e736f00736f6d656c69622e736f00666f6f00"
+  - Name:            .dynamic
+    Type:            SHT_DYNAMIC
+    Flags:           [ SHF_ALLOC ]
+    Address:         0x0000000000001024
+    EntSize:         0x0000000000000010
+    Entries:
+      - Tag:             DT_NEEDED
+        Value:           0x0000000000000001
+      - Tag:             DT_SONAME
+        Value:           0x0000000000000015
+      - Tag:             DT_NEEDED
+        Value:           0x000000000000000B
+      - Tag:             DT_STRSZ
+        Value:           0x0000000000000024
+      - Tag:             DT_STRTAB
+        Value:           0x0000000000001000
+      - Tag:             DT_SYMTAB
+        Value:           0x0000000000001000
+      - Tag:             DT_NULL
+        Value:           0x0000000000000000
+ProgramHeaders:
+  - Type: PT_LOAD
+    Flags: [ PF_R ]
+    VAddr: 0x1000
+    PAddr: 0x1000
+    Align: 8
+    Sections:
+      - Section: .dynstr
+      - Section: .dynamic
+  - Type: PT_DYNAMIC
+    Flags: [ PF_X, PF_R ]
+    VAddr: 0x1024
+    PAddr: 0x1024
+    Sections:
+
+# CHECK:      NeededLibs:
+# CHECK-NEXT:   - libfoo.so{{$}}
+# CHECK-NEXT:   - libbar.so{{$}}
+# CHECK-NEXT: Symbols: {}
diff --git a/test/tools/llvm-elfabi/binary-read-no-dt-strsz.test b/test/tools/llvm-elfabi/binary-read-no-dt-strsz.test
new file mode 100644
index 0000000000000000000000000000000000000000..99e06b2ca87b2a79dd5efaad441135672caa80c5
--- /dev/null
+++ b/test/tools/llvm-elfabi/binary-read-no-dt-strsz.test
@@ -0,0 +1,45 @@
+# RUN: yaml2obj %s > %t
+# RUN: not llvm-elfabi --elf %t --emit-tbe=%t.tbe 2>&1 | FileCheck %s
+
+!ELF
+FileHeader:
+  Class:           ELFCLASS64
+  Data:            ELFDATA2LSB
+  Type:            ET_DYN
+  Machine:         EM_X86_64
+Sections:
+  - Name:            .dynstr
+    Type:            SHT_STRTAB
+    Flags:           [ SHF_ALLOC ]
+    Address:         0x0000
+#                     \0 b a z\0
+    Content:         "0062617a00"
+  - Name:            .dynamic
+    Type:            SHT_DYNAMIC
+    Flags:           [ SHF_ALLOC ]
+    Address:         0x0000000000000008
+    Link:            .dynstr
+    AddressAlign:    0x0000000000000008
+    EntSize:         0x0000000000000010
+    Entries:
+      - Tag:             DT_STRTAB
+        Value:           0x0000000000000000
+      - Tag:             DT_NULL
+        Value:           0x0000000000000000
+ProgramHeaders:
+  - Type: PT_LOAD
+    Flags: [ PF_R ]
+    VAddr: 0x0000
+    PAddr: 0x0000
+    Align: 8
+    Sections:
+      - Section: .dynstr
+      - Section: .dynamic
+  - Type: PT_DYNAMIC
+    Flags: [ PF_X, PF_R ]
+    VAddr: 0x0008
+    PAddr: 0x0008
+    Sections:
+      - Section: .dynamic
+
+# CHECK: Couldn't determine dynamic string table size (no DT_STRSZ entry)
diff --git a/test/tools/llvm-elfabi/binary-read-no-dt-strtab.test b/test/tools/llvm-elfabi/binary-read-no-dt-strtab.test
new file mode 100644
index 0000000000000000000000000000000000000000..79f4496cc9f6e512dc5623e12cffb52a71071cf5
--- /dev/null
+++ b/test/tools/llvm-elfabi/binary-read-no-dt-strtab.test
@@ -0,0 +1,44 @@
+# RUN: yaml2obj %s > %t
+# RUN: not llvm-elfabi --elf %t --emit-tbe=%t.tbe 2>&1 | FileCheck %s
+
+!ELF
+FileHeader:
+  Class:           ELFCLASS64
+  Data:            ELFDATA2LSB
+  Type:            ET_DYN
+  Machine:         EM_X86_64
+Sections:
+  - Name:            .dynstr
+    Type:            SHT_STRTAB
+    Flags:           [ SHF_ALLOC ]
+    Address:         0x0000
+    Content:         "00"
+  - Name:            .dynamic
+    Type:            SHT_DYNAMIC
+    Flags:           [ SHF_ALLOC ]
+    Address:         0x0000000000000008
+    Link:            .dynstr
+    AddressAlign:    0x0000000000000008
+    EntSize:         0x0000000000000010
+    Entries:
+      - Tag:             DT_STRSZ
+        Value:           0x0000000000000001
+      - Tag:             DT_NULL
+        Value:           0x0000000000000000
+ProgramHeaders:
+  - Type: PT_LOAD
+    Flags: [ PF_R ]
+    VAddr: 0x0000
+    PAddr: 0x0000
+    Align: 8
+    Sections:
+      - Section: .dynstr
+      - Section: .dynamic
+  - Type: PT_DYNAMIC
+    Flags: [ PF_X, PF_R ]
+    VAddr: 0x0008
+    PAddr: 0x0008
+    Sections:
+      - Section: .dynamic
+
+# CHECK: Couldn't locate dynamic string table (no DT_STRTAB entry)
diff --git a/test/tools/llvm-elfabi/binary-read-no-dynamic.test b/test/tools/llvm-elfabi/binary-read-no-dynamic.test
new file mode 100644
index 0000000000000000000000000000000000000000..475d7bf7b8421373d14565db33f91c51693e1a89
--- /dev/null
+++ b/test/tools/llvm-elfabi/binary-read-no-dynamic.test
@@ -0,0 +1,11 @@
+# RUN: yaml2obj %s > %t
+# RUN: not llvm-elfabi --elf %t --emit-tbe=%t.tbe 2>&1 | FileCheck %s
+
+!ELF
+FileHeader:
+  Class:           ELFCLASS64
+  Data:            ELFDATA2LSB
+  Type:            ET_DYN
+  Machine:         EM_X86_64
+
+# CHECK: No .dynamic section found
diff --git a/test/tools/llvm-elfabi/binary-read-replace-soname.test b/test/tools/llvm-elfabi/binary-read-replace-soname.test
new file mode 100644
index 0000000000000000000000000000000000000000..762dcbb913461e50dccbbdc4cd38f9fe8c5bf1f5
--- /dev/null
+++ b/test/tools/llvm-elfabi/binary-read-replace-soname.test
@@ -0,0 +1,54 @@
+# RUN: yaml2obj %s > %t
+# RUN: llvm-elfabi --elf %t --emit-tbe=- | FileCheck %s --check-prefix=ORIGINAL
+# RUN: llvm-elfabi --elf %t --emit-tbe=- --soname=libbest.so | FileCheck %s --check-prefix=REPLACED
+
+!ELF
+FileHeader:
+  Class:           ELFCLASS64
+  Data:            ELFDATA2LSB
+  Type:            ET_DYN
+  Machine:         EM_X86_64
+Sections:
+  - Name:            .dynstr
+    Type:            SHT_STRTAB
+    Flags:           [ SHF_ALLOC ]
+    Address:         0x1000
+#                     \0 b a z\0 s o m e l i b . s o\0 f o o\0
+    Content:         "0062617a00736f6d656c69622e736f00666f6f00"
+  - Name:            .dynamic
+    Type:            SHT_DYNAMIC
+    Flags:           [ SHF_ALLOC ]
+    Address:         0x0000000000001018
+    Link:            .dynstr
+    AddressAlign:    0x0000000000000008
+    EntSize:         0x0000000000000010
+    Entries:
+      - Tag:             DT_SONAME
+        Value:           0x0000000000000005
+      - Tag:             DT_STRSZ
+        Value:           0x0000000000000014
+      - Tag:             DT_STRTAB
+        Value:           0x0000000000001000
+      - Tag:             DT_SYMTAB
+        Value:           0x0000000000001000
+      - Tag:             DT_NULL
+        Value:           0x0000000000000000
+ProgramHeaders:
+  - Type: PT_LOAD
+    Flags: [ PF_R ]
+    VAddr: 0x1000
+    PAddr: 0x1000
+    Align: 8
+    Sections:
+      - Section: .dynstr
+      - Section: .dynamic
+  - Type: PT_DYNAMIC
+    Flags: [ PF_X, PF_R ]
+    VAddr: 0x1018
+    PAddr: 0x1018
+    Sections:
+      - Section: .dynamic
+
+# ORIGINAL: SoName: somelib.so{{$}}
+
+# REPLACED: SoName: libbest.so{{$}}
diff --git a/test/tools/llvm-elfabi/binary-read-soname-no-null.test b/test/tools/llvm-elfabi/binary-read-soname-no-null.test
new file mode 100644
index 0000000000000000000000000000000000000000..30aa6794298d9a5d2f86ee12ce12af97d8a1c48f
--- /dev/null
+++ b/test/tools/llvm-elfabi/binary-read-soname-no-null.test
@@ -0,0 +1,51 @@
+# RUN: yaml2obj %s > %t
+# RUN: not llvm-elfabi --elf %t --emit-tbe=%t.tbe 2>&1 | FileCheck %s
+
+!ELF
+FileHeader:
+  Class:           ELFCLASS64
+  Data:            ELFDATA2LSB
+  Type:            ET_DYN
+  Machine:         EM_X86_64
+Sections:
+  - Name:            .dynstr
+    Type:            SHT_STRTAB
+    Flags:           [ SHF_ALLOC ]
+    Address:         0x1000
+#                     \0 b a z\0 s o m e l i b . s o z z z
+    Content:         "0062617a00736f6d656c69622e736f7a7a7a"
+  - Name:            .dynamic
+    Type:            SHT_DYNAMIC
+    Flags:           [ SHF_ALLOC ]
+    Address:         0x0000000000001018
+    Link:            .dynstr
+    AddressAlign:    0x0000000000000008
+    EntSize:         0x0000000000000010
+    Entries:
+      - Tag:             DT_SONAME
+        Value:           0x0000000000000005
+      - Tag:             DT_STRSZ
+        Value:           0x000000000000000F
+      - Tag:             DT_STRTAB
+        Value:           0x0000000000001000
+      - Tag:             DT_SYMTAB
+        Value:           0x0000000000001000
+      - Tag:             DT_NULL
+        Value:           0x0000000000000000
+ProgramHeaders:
+  - Type: PT_LOAD
+    Flags: [ PF_R ]
+    VAddr: 0x1000
+    PAddr: 0x1000
+    Align: 8
+    Sections:
+      - Section: .dynstr
+      - Section: .dynamic
+  - Type: PT_DYNAMIC
+    Flags: [ PF_X, PF_R ]
+    VAddr: 0x1018
+    PAddr: 0x1018
+    Sections:
+      - Section: .dynamic
+
+# CHECK: String overran bounds of string table (no null terminator) when reading DT_SONAME
diff --git a/test/tools/llvm-elfabi/binary-read-soname.test b/test/tools/llvm-elfabi/binary-read-soname.test
new file mode 100644
index 0000000000000000000000000000000000000000..ef35b893dc019aa5bd7277e393ac212ce4111db0
--- /dev/null
+++ b/test/tools/llvm-elfabi/binary-read-soname.test
@@ -0,0 +1,56 @@
+# RUN: yaml2obj %s > %t
+# RUN: llvm-elfabi --elf %t --emit-tbe=- | FileCheck %s
+
+!ELF
+FileHeader:
+  Class:           ELFCLASS64
+  Data:            ELFDATA2LSB
+  Type:            ET_DYN
+  Machine:         EM_X86_64
+Sections:
+  - Name:            .dynstr
+    Type:            SHT_STRTAB
+    Flags:           [ SHF_ALLOC ]
+    Address:         0x1000
+#                     \0 b a z\0 s o m e l i b . s o\0 f o o\0
+    Content:         "0062617a00736f6d656c69622e736f00666f6f00"
+  - Name:            .dynamic
+    Type:            SHT_DYNAMIC
+    Flags:           [ SHF_ALLOC ]
+    Address:         0x0000000000001018
+    Link:            .dynstr
+    AddressAlign:    0x0000000000000008
+    EntSize:         0x0000000000000010
+    Entries:
+      - Tag:             DT_SONAME
+        Value:           0x0000000000000005
+      - Tag:             DT_STRSZ
+        Value:           0x0000000000000014
+      - Tag:             DT_STRTAB
+        Value:           0x0000000000001000
+      - Tag:             DT_SYMTAB
+        Value:           0x0000000000001000
+      - Tag:             DT_NULL
+        Value:           0x0000000000000000
+ProgramHeaders:
+  - Type: PT_LOAD
+    Flags: [ PF_R ]
+    VAddr: 0x1000
+    PAddr: 0x1000
+    Align: 8
+    Sections:
+      - Section: .dynstr
+      - Section: .dynamic
+  - Type: PT_DYNAMIC
+    Flags: [ PF_X, PF_R ]
+    VAddr: 0x1018
+    PAddr: 0x1018
+    Sections:
+      - Section: .dynamic
+
+# CHECK:      --- !tapi-tbe
+# CHECK-NEXT: TbeVersion: {{[1-9]\d*\.(0|([1-9]\d*))}}
+# CHECK-NEXT: SoName: somelib.so{{$}}
+# CHECK-NEXT: Arch: x86_64
+# CHECK-NEXT: Symbols: {}
+# CHECK-NEXT: ...
diff --git a/test/tools/llvm-elfabi/binary-read-syms-gnu-hash.test b/test/tools/llvm-elfabi/binary-read-syms-gnu-hash.test
new file mode 100644
index 0000000000000000000000000000000000000000..777f2714b28f7a0b500b5187fa91295c88d99477
--- /dev/null
+++ b/test/tools/llvm-elfabi/binary-read-syms-gnu-hash.test
@@ -0,0 +1,22 @@
+# RUN: llvm-elfabi --elf %p/Inputs/gnu_hash.so --emit-tbe=- | FileCheck %s
+
+# CHECK:      --- !tapi-tbe
+# CHECK-NEXT: TbeVersion:      1.0
+# CHECK-NEXT: SoName:          libsomething.so
+# CHECK-NEXT: Arch:            x86_64
+# CHECK-NEXT: NeededLibs:
+# CHECK-NEXT:   - libm.so.6
+# CHECK-NEXT:   - libc.so.6
+# CHECK-NEXT:   - ld-linux-x86-64.so.2
+# CHECK-NEXT: Symbols:
+# CHECK-NEXT:   AGlobalInteger:  { Type: Object, Size: 4 }
+# CHECK-NEXT:   AThreadLocalLongInteger: { Type: TLS, Size: 8 }
+# CHECK-NEXT:   _ITM_deregisterTMCloneTable: { Type: NoType, Undefined: true, Weak: true }
+# CHECK-NEXT:   _ITM_registerTMCloneTable: { Type: NoType, Undefined: true, Weak: true }
+# CHECK-NEXT:   _Z11rotateArrayPii: { Type: Func }
+# CHECK-NEXT:   __cxa_finalize:  { Type: Func, Undefined: true, Weak: true }
+# CHECK-NEXT:   __gmon_start__:  { Type: NoType, Undefined: true, Weak: true }
+# CHECK-NEXT:   __tls_get_addr:  { Type: Func, Undefined: true }
+# CHECK-NEXT:   _fini:           { Type: Func }
+# CHECK-NEXT:   _init:           { Type: Func }
+# CHECK-NEXT: ...
diff --git a/test/tools/llvm-elfabi/binary-read-syms-sysv-hash.test b/test/tools/llvm-elfabi/binary-read-syms-sysv-hash.test
new file mode 100644
index 0000000000000000000000000000000000000000..0f2908529989d964394ec3752ce743e42b0c3823
--- /dev/null
+++ b/test/tools/llvm-elfabi/binary-read-syms-sysv-hash.test
@@ -0,0 +1,22 @@
+# RUN: llvm-elfabi --elf %p/Inputs/sysv_hash.so --emit-tbe=- | FileCheck %s
+
+# CHECK:      --- !tapi-tbe
+# CHECK-NEXT: TbeVersion:      1.0
+# CHECK-NEXT: SoName:          libsomething.so
+# CHECK-NEXT: Arch:            x86_64
+# CHECK-NEXT: NeededLibs:
+# CHECK-NEXT:   - libm.so.6
+# CHECK-NEXT:   - libc.so.6
+# CHECK-NEXT:   - ld-linux-x86-64.so.2
+# CHECK-NEXT: Symbols:
+# CHECK-NEXT:   AGlobalInteger:  { Type: Object, Size: 4 }
+# CHECK-NEXT:   AThreadLocalLongInteger: { Type: TLS, Size: 8 }
+# CHECK-NEXT:   _ITM_deregisterTMCloneTable: { Type: NoType, Undefined: true, Weak: true }
+# CHECK-NEXT:   _ITM_registerTMCloneTable: { Type: NoType, Undefined: true, Weak: true }
+# CHECK-NEXT:   _Z11rotateArrayPii: { Type: Func }
+# CHECK-NEXT:   __cxa_finalize:  { Type: Func, Undefined: true, Weak: true }
+# CHECK-NEXT:   __gmon_start__:  { Type: NoType, Undefined: true, Weak: true }
+# CHECK-NEXT:   __tls_get_addr:  { Type: Func, Undefined: true }
+# CHECK-NEXT:   _fini:           { Type: Func }
+# CHECK-NEXT:   _init:           { Type: Func }
+# CHECK-NEXT: ...
diff --git a/test/tools/llvm-elfabi/fail-file-open.test b/test/tools/llvm-elfabi/fail-file-open.test
new file mode 100644
index 0000000000000000000000000000000000000000..b4019af9feb9e618b61441c474e48b1ddcb4f2c1
--- /dev/null
+++ b/test/tools/llvm-elfabi/fail-file-open.test
@@ -0,0 +1,5 @@
+# RUN: not llvm-elfabi %s.NotAFileInTestingDir --emit-tbe=%t 2>&1 | FileCheck %s
+
+This file will not be read. An invalid file path is fed to llvm-elfabi.
+
+# CHECK: error: Could not open `{{.*}}.NotAFileInTestingDir`
diff --git a/test/tools/llvm-elfabi/read-tbe-as-elf.test b/test/tools/llvm-elfabi/read-tbe-as-elf.test
new file mode 100644
index 0000000000000000000000000000000000000000..eaddd38867a5bd3325688542506c91bb452a7dcd
--- /dev/null
+++ b/test/tools/llvm-elfabi/read-tbe-as-elf.test
@@ -0,0 +1,16 @@
+# RUN: not llvm-elfabi --elf %s --emit-tbe=%t  2>&1 | FileCheck %s
+
+--- !tapi-tbe
+SoName: somelib.so
+TbeVersion: 1.0
+Arch: x86_64
+Symbols:
+  foo: { Type: Func }
+  bar: { Type: Object, Size: 42 }
+  baz: { Type: Object, Size: 8 }
+  not: { Type: Object, Undefined: true, Size: 128 }
+  nor: { Type: Func, Undefined: true }
+...
+
+# CHECK: The file was not recognized as a valid object file
+# CHECK: No file readers succeeded reading `{{.*}}read-tbe-as-elf.test` (unsupported/malformed file?)
diff --git a/test/tools/llvm-elfabi/read-tbe-as-tbe.test b/test/tools/llvm-elfabi/read-tbe-as-tbe.test
new file mode 100644
index 0000000000000000000000000000000000000000..a5e2d44134e10fbc77c5b640db5ea98217ad3e0d
--- /dev/null
+++ b/test/tools/llvm-elfabi/read-tbe-as-tbe.test
@@ -0,0 +1,13 @@
+# RUN: llvm-elfabi --tbe %s --emit-tbe=- | FileCheck %s
+
+--- !tapi-tbe
+TbeVersion: 1.0
+Arch: AArch64
+Symbols: {}
+...
+
+# CHECK:      --- !tapi-tbe
+# CHECK-NEXT: TbeVersion: {{[1-9]\d*\.(0|([1-9]\d*))}}
+# CHECK-NEXT: Arch: AArch64
+# CHECK-NEXT: Symbols: {}
+# CHECK-NEXT: ...
diff --git a/test/tools/llvm-elfabi/read-unsupported-file.test b/test/tools/llvm-elfabi/read-unsupported-file.test
new file mode 100644
index 0000000000000000000000000000000000000000..4ebe1bcc6d41957826d3fe1d7bcfff525ecd8f14
--- /dev/null
+++ b/test/tools/llvm-elfabi/read-unsupported-file.test
@@ -0,0 +1,7 @@
+# RUN: not llvm-elfabi %s --emit-tbe=%t 2>&1 | FileCheck %s
+
+This is just some text that cannot be read by llvm-elfabi.
+
+# CHECK: The file was not recognized as a valid object file
+# CHECK: YAML failed reading as TBE
+# CHECK: No file readers succeeded reading `{{.*}}` (unsupported/malformed file?)
diff --git a/test/tools/llvm-elfabi/tbe-emits-current-version.test b/test/tools/llvm-elfabi/tbe-emits-current-version.test
new file mode 100644
index 0000000000000000000000000000000000000000..12a5476175a00a90ea319d6820f5fa191e23ea65
--- /dev/null
+++ b/test/tools/llvm-elfabi/tbe-emits-current-version.test
@@ -0,0 +1,13 @@
+# RUN: llvm-elfabi %s --emit-tbe=- | FileCheck %s
+
+--- !tapi-tbe
+TbeVersion: 1.0
+Arch: AArch64
+Symbols: {}
+...
+
+# As the tbe reader/writer is updated, update this check to ensure --emit-tbe
+# uses the latest tbe writer by default.
+
+# CHECK:      --- !tapi-tbe
+# CHECK-NEXT: TbeVersion: 1.0
diff --git a/test/tools/llvm-elfabi/tbe-read-basic.test b/test/tools/llvm-elfabi/tbe-read-basic.test
new file mode 100644
index 0000000000000000000000000000000000000000..1599f5a891e9ffd7ed4162ae91e0884dd0e8f436
--- /dev/null
+++ b/test/tools/llvm-elfabi/tbe-read-basic.test
@@ -0,0 +1,25 @@
+# RUN: llvm-elfabi %s --emit-tbe=- | FileCheck %s
+
+--- !tapi-tbe
+SoName: somelib.so
+TbeVersion: 1.0
+Arch: x86_64
+Symbols:
+  foo: { Type: Func }
+  bar: { Type: Object, Size: 42 }
+  baz: { Type: Object, Size: 8 }
+  not: { Type: Object, Undefined: true, Size: 128 }
+  nor: { Type: Func, Undefined: true }
+...
+
+# CHECK:      --- !tapi-tbe
+# CHECK-NEXT: TbeVersion: {{[1-9]\d*\.(0|([1-9]\d*))}}
+# CHECK-NEXT: SoName: somelib.so
+# CHECK-NEXT: Arch: x86_64
+# CHECK-NEXT: Symbols:
+# CHECK-NEXT:   bar: { Type: Object, Size: 42 }
+# CHECK-NEXT:   baz: { Type: Object, Size: 8 }
+# CHECK-NEXT:   foo: { Type: Func }
+# CHECK-NEXT:   nor: { Type: Func, Undefined: true }
+# CHECK-NEXT:   not: { Type: Object, Size: 128, Undefined: true }
+# CHECK-NEXT: ...
diff --git a/test/tools/llvm-exegesis/X86/analysis-uops-backwards.test b/test/tools/llvm-exegesis/X86/analysis-uops-backwards.test
index 4c758ba9379d8f0adffefaa6da2d52d53021b638..ac806da4869e5d37aa152b0566a9416943e6ac5a 100644
--- a/test/tools/llvm-exegesis/X86/analysis-uops-backwards.test
+++ b/test/tools/llvm-exegesis/X86/analysis-uops-backwards.test
@@ -1,5 +1,5 @@
 # This tests backwards-compatibility of the yaml schema (see PR39082).
-# RUN: llvm-exegesis -mode=analysis -benchmarks-file=%s -analysis-inconsistencies-output-file="" -analysis-numpoints=1 | FileCheck %s
+# RUN: llvm-exegesis -mode=analysis -benchmarks-file=%s -analysis-clusters-output-file=- -analysis-numpoints=1 | FileCheck %s
 
 # CHECK: cluster_id,opcode_name,config,sched_class,HWPort0,HWPort1,HWPort2,HWPort3,HWPort4,HWPort5,HWPort6,HWPort7,NumMicroOps
 # CHECK-NEXT: vzeroall
diff --git a/test/tools/llvm-exegesis/X86/analysis-uops-variant.test b/test/tools/llvm-exegesis/X86/analysis-uops-variant.test
index 507e979e5dc1775725ba235318257d8eeb300719..f1efa3af05013f9e6a84c1d3025b00bfd3a18503 100644
--- a/test/tools/llvm-exegesis/X86/analysis-uops-variant.test
+++ b/test/tools/llvm-exegesis/X86/analysis-uops-variant.test
@@ -1,4 +1,4 @@
-# RUN: llvm-exegesis -mode=analysis -benchmarks-file=%s -analysis-inconsistencies-output-file="" -analysis-numpoints=1 | FileCheck %s
+# RUN: llvm-exegesis -mode=analysis -benchmarks-file=%s -analysis-clusters-output-file=- -analysis-numpoints=1 | FileCheck %s
 
 # REQUIRES: asserts
 # (sched class name is NDEBUG only)
diff --git a/test/tools/llvm-exegesis/X86/analysis-uops.test b/test/tools/llvm-exegesis/X86/analysis-uops.test
index f1aae9750c4040e615144cdaeaba83f612051eb3..95737e5d545af7fcdfd4282501eef96c101739fc 100644
--- a/test/tools/llvm-exegesis/X86/analysis-uops.test
+++ b/test/tools/llvm-exegesis/X86/analysis-uops.test
@@ -1,4 +1,4 @@
-# RUN: llvm-exegesis -mode=analysis -benchmarks-file=%s -analysis-inconsistencies-output-file="" -analysis-numpoints=1 | FileCheck %s
+# RUN: llvm-exegesis -mode=analysis -benchmarks-file=%s -analysis-clusters-output-file=- -analysis-numpoints=1 | FileCheck %s
 
 # CHECK: cluster_id,opcode_name,config,sched_class,HWPort0,HWPort1,HWPort2,HWPort3,HWPort4,HWPort5,HWPort6,HWPort7,NumMicroOps
 # CHECK-NEXT: vzeroall
diff --git a/test/tools/llvm-exegesis/X86/inverse_throughput-by-opcode-name.s b/test/tools/llvm-exegesis/X86/inverse_throughput-by-opcode-name.s
new file mode 100644
index 0000000000000000000000000000000000000000..49cb84792034d0ca25aba075ec3fcceae5d89f33
--- /dev/null
+++ b/test/tools/llvm-exegesis/X86/inverse_throughput-by-opcode-name.s
@@ -0,0 +1,8 @@
+# RUN: llvm-exegesis -mode=inverse_throughput -opcode-name=ADD32rr | FileCheck %s
+
+CHECK:      ---
+CHECK-NEXT: mode: inverse_throughput
+CHECK-NEXT: key:
+CHECK-NEXT:   instructions:
+CHECK-NEXT:     ADD32rr
+CHECK: key: inverse_throughput
diff --git a/test/tools/llvm-lib/list.test b/test/tools/llvm-lib/list.test
new file mode 100644
index 0000000000000000000000000000000000000000..3b59180d1acee76b8dc9834b59fc1befc8d22c95
--- /dev/null
+++ b/test/tools/llvm-lib/list.test
@@ -0,0 +1,14 @@
+RUN: rm -rf %t && mkdir -p %t
+RUN: llvm-mc -triple=x86_64-pc-windows-msvc -filetype=obj -o %t/a.obj %S/Inputs/a.s
+RUN: llvm-mc -triple=x86_64-pc-windows-msvc -filetype=obj -o %t/b.obj %S/Inputs/b.s
+
+RUN: rm -f %t/l.lib
+RUN: llvm-lib /out:%t/foo.lib %t/a.obj %t/b.obj
+
+# This should ignore the two .obj files, then print the first .lib file,
+# then ignore the 2nd one.
+RUN: llvm-lib /list %t/a.obj %t/b.obj %t/foo.lib %t/foo.lib | FileCheck %s
+
+CHECK: a.obj
+CHECK: b.obj
+CHECK-NOT: a.obj
diff --git a/test/tools/llvm-mca/AArch64/CortexA57/direct-branch.s b/test/tools/llvm-mca/AArch64/Cortex/direct-branch.s
similarity index 100%
rename from test/tools/llvm-mca/AArch64/CortexA57/direct-branch.s
rename to test/tools/llvm-mca/AArch64/Cortex/direct-branch.s
diff --git a/test/tools/llvm-mca/AArch64/CortexA57/shifted-register.s b/test/tools/llvm-mca/AArch64/Cortex/shifted-register.s
similarity index 100%
rename from test/tools/llvm-mca/AArch64/CortexA57/shifted-register.s
rename to test/tools/llvm-mca/AArch64/Cortex/shifted-register.s
diff --git a/test/tools/llvm-mca/AArch64/Exynos/direct-branch.s b/test/tools/llvm-mca/AArch64/Exynos/direct-branch.s
index 7c3cc8fa48c1d796142699be372a77afccfdfe0d..cd31d0b39fba730de5e4c7e1885ea408888f6472 100644
--- a/test/tools/llvm-mca/AArch64/Exynos/direct-branch.s
+++ b/test/tools/llvm-mca/AArch64/Exynos/direct-branch.s
@@ -1,6 +1,7 @@
 # NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
 # RUN: llvm-mca -mtriple=aarch64-linux-gnu -mcpu=exynos-m1 -resource-pressure=false < %s | FileCheck %s -check-prefixes=ALL,M1
 # RUN: llvm-mca -mtriple=aarch64-linux-gnu -mcpu=exynos-m3 -resource-pressure=false < %s | FileCheck %s -check-prefixes=ALL,M3
+# RUN: llvm-mca -mtriple=aarch64-linux-gnu -mcpu=exynos-m4 -resource-pressure=false < %s | FileCheck %s -check-prefixes=ALL,M4
 
   b	main
 
@@ -9,6 +10,7 @@
 
 # M1-NEXT:  Total Cycles:      26
 # M3-NEXT:  Total Cycles:      18
+# M4-NEXT:  Total Cycles:      18
 
 # ALL-NEXT: Total uOps:        100
 
@@ -22,6 +24,11 @@
 # M3-NEXT:  IPC:               5.56
 # M3-NEXT:  Block RThroughput: 0.2
 
+# M4:       Dispatch Width:    6
+# M4-NEXT:  uOps Per Cycle:    5.56
+# M4-NEXT:  IPC:               5.56
+# M4-NEXT:  Block RThroughput: 0.2
+
 # ALL:      Instruction Info:
 # ALL-NEXT: [1]: #uOps
 # ALL-NEXT: [2]: Latency
@@ -34,3 +41,4 @@
 
 # M1-NEXT:   1      0     0.25                        b	main
 # M3-NEXT:   1      0     0.17                        b	main
+# M4-NEXT:   1      0     0.17                        b	main
diff --git a/test/tools/llvm-mca/AArch64/Exynos/extended-register.s b/test/tools/llvm-mca/AArch64/Exynos/extended-register.s
index f136bd35d8603e1f7e2d64a5d52c21b50ae630c6..7044f64318ff061e6b901b292568255ec605598c 100644
--- a/test/tools/llvm-mca/AArch64/Exynos/extended-register.s
+++ b/test/tools/llvm-mca/AArch64/Exynos/extended-register.s
@@ -1,6 +1,7 @@
 # NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
 # RUN: llvm-mca -march=aarch64 -mcpu=exynos-m1 -resource-pressure=false < %s | FileCheck %s -check-prefixes=ALL,EM1
 # RUN: llvm-mca -march=aarch64 -mcpu=exynos-m3 -resource-pressure=false < %s | FileCheck %s -check-prefixes=ALL,EM3
+# RUN: llvm-mca -march=aarch64 -mcpu=exynos-m4 -resource-pressure=false < %s | FileCheck %s -check-prefixes=ALL,EM4
 
   sub	w0, w1, w2, sxtb #0
   add	x3, x4, w5, sxth #1
@@ -16,6 +17,7 @@
 
 # EM1-NEXT: Total Cycles:      403
 # EM3-NEXT: Total Cycles:      304
+# EM4-NEXT: Total Cycles:      304
 
 # ALL-NEXT: Total uOps:        800
 
@@ -29,6 +31,11 @@
 # EM3-NEXT: IPC:               2.63
 # EM3-NEXT: Block RThroughput: 3.0
 
+# EM4:      Dispatch Width:    6
+# EM4-NEXT: uOps Per Cycle:    2.63
+# EM4-NEXT: IPC:               2.63
+# EM4-NEXT: Block RThroughput: 3.0
+
 # ALL:      Instruction Info:
 # ALL-NEXT: [1]: #uOps
 # ALL-NEXT: [2]: Latency
@@ -56,3 +63,12 @@
 # EM3-NEXT:  1      2     0.50                        add	x15, x16, w17, uxth #1
 # EM3-NEXT:  1      2     0.50                        subs	x18, x19, w20, sxtw #2
 # EM3-NEXT:  1      2     0.50                        adds	x21, x22, x23, sxtx #3
+
+# EM4-NEXT:  1      1     0.25                        sub	w0, w1, w2, sxtb
+# EM4-NEXT:  1      2     0.50                        add	x3, x4, w5, sxth #1
+# EM4-NEXT:  1      1     0.25                        subs	x6, x7, w8, uxtw #2
+# EM4-NEXT:  1      1     0.25                        adds	x9, x10, x11, uxtx #3
+# EM4-NEXT:  1      1     0.25                        sub	w12, w13, w14, uxtb
+# EM4-NEXT:  1      2     0.50                        add	x15, x16, w17, uxth #1
+# EM4-NEXT:  1      2     0.50                        subs	x18, x19, w20, sxtw #2
+# EM4-NEXT:  1      2     0.50                        adds	x21, x22, x23, sxtx #3
diff --git a/test/tools/llvm-mca/AArch64/Exynos/register-offset.s b/test/tools/llvm-mca/AArch64/Exynos/register-offset.s
deleted file mode 100644
index 4fc4cf48fdcc1baa2018d241e72686299548976c..0000000000000000000000000000000000000000
--- a/test/tools/llvm-mca/AArch64/Exynos/register-offset.s
+++ /dev/null
@@ -1,81 +0,0 @@
-# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
-# RUN: llvm-mca -march=aarch64 -mcpu=exynos-m1 -resource-pressure=false < %s | FileCheck %s -check-prefixes=ALL,EM1
-# RUN: llvm-mca -march=aarch64 -mcpu=exynos-m3 -resource-pressure=false < %s | FileCheck %s -check-prefixes=ALL,EM3
-
-  ldrb	w0, [x1, x2, lsl #0]
-  ldrh	w3, [x4, x5, sxtx #1]
-  ldr	w6, [x7, w8, uxtw #2]
-  ldr	x9, [x10, w11, sxtw #3]
-  ldr	b12, [x13, w14, sxtw #0]
-  ldr	h15, [x16, w17, uxtw #1]
-  ldr	s18, [x19, x20, sxtx #2]
-  ldr	d21, [x22, x23, lsl #3]
-  ldr	q24, [x25, x26, lsl #4]
-
-  strb	w0, [x1, x2, lsl #0]
-  strh	w3, [x4, x5, sxtx #1]
-  str	w6, [x7, w8, uxtw #2]
-  str	x9, [x10, w11, sxtw #3]
-  str	b12, [x13, w14, sxtw #0]
-  str	h15, [x16, w17, uxtw #1]
-  str	s18, [x19, x20, sxtx #2]
-  str	d21, [x22, x23, lsl #3]
-  str	q24, [x25, x26, lsl #4]
-
-# ALL:      Iterations:        100
-# ALL-NEXT: Instructions:      1800
-
-# EM1-NEXT: Total Cycles:      1719
-# EM3-NEXT: Total Cycles:      1713
-
-# ALL-NEXT: Total uOps:        2800
-
-# EM1:      Dispatch Width:    4
-# EM1-NEXT: uOps Per Cycle:    1.63
-# EM1-NEXT: IPC:               1.05
-# EM1-NEXT: Block RThroughput: 12.0
-
-# EM3:      Dispatch Width:    6
-# EM3-NEXT: uOps Per Cycle:    1.63
-# EM3-NEXT: IPC:               1.05
-# EM3-NEXT: Block RThroughput: 9.0
-
-# ALL:      Instruction Info:
-# ALL-NEXT: [1]: #uOps
-# ALL-NEXT: [2]: Latency
-# ALL-NEXT: [3]: RThroughput
-# ALL-NEXT: [4]: MayLoad
-# ALL-NEXT: [5]: MayStore
-# ALL-NEXT: [6]: HasSideEffects (U)
-
-# ALL:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
-
-# EM1-NEXT:  1      5     1.00    *                   ldrb	w0, [x1, x2, lsl #0]
-# EM1-NEXT:  1      5     1.00    *                   ldrh	w3, [x4, x5, sxtx #1]
-# EM1-NEXT:  2      5     1.00    *                   ldr	w6, [x7, w8, uxtw #2]
-# EM1-NEXT:  2      5     1.00    *                   ldr	x9, [x10, w11, sxtw #3]
-# EM1-NEXT:  2      6     2.00    *                   ldr	b12, [x13, w14, sxtw #0]
-# EM1-NEXT:  2      6     2.00    *                   ldr	h15, [x16, w17, uxtw #1]
-# EM1-NEXT:  1      5     1.00    *                   ldr	s18, [x19, x20, sxtx #2]
-# EM1-NEXT:  1      5     1.00    *                   ldr	d21, [x22, x23, lsl #3]
-# EM1-NEXT:  2      6     2.00    *                   ldr	q24, [x25, x26, lsl #4]
-
-# EM3-NEXT:  1      5     0.50    *                   ldrb	w0, [x1, x2, lsl #0]
-# EM3-NEXT:  1      5     0.50    *                   ldrh	w3, [x4, x5, sxtx #1]
-# EM3-NEXT:  2      5     0.50    *                   ldr	w6, [x7, w8, uxtw #2]
-# EM3-NEXT:  2      5     0.50    *                   ldr	x9, [x10, w11, sxtw #3]
-# EM3-NEXT:  2      6     0.50    *                   ldr	b12, [x13, w14, sxtw #0]
-# EM3-NEXT:  2      6     0.50    *                   ldr	h15, [x16, w17, uxtw #1]
-# EM3-NEXT:  1      5     0.50    *                   ldr	s18, [x19, x20, sxtx #2]
-# EM3-NEXT:  1      5     0.50    *                   ldr	d21, [x22, x23, lsl #3]
-# EM3-NEXT:  2      6     0.50    *                   ldr	q24, [x25, x26, lsl #4]
-
-# ALL-NEXT:  1      1     1.00           *            strb	w0, [x1, x2, lsl #0]
-# ALL-NEXT:  1      1     1.00           *            strh	w3, [x4, x5, sxtx #1]
-# ALL-NEXT:  2      2     1.00           *            str	w6, [x7, w8, uxtw #2]
-# ALL-NEXT:  2      2     1.00           *            str	x9, [x10, w11, sxtw #3]
-# ALL-NEXT:  2      3     1.00           *            str	b12, [x13, w14, sxtw #0]
-# ALL-NEXT:  2      3     1.00           *            str	h15, [x16, w17, uxtw #1]
-# ALL-NEXT:  1      1     1.00           *            str	s18, [x19, x20, sxtx #2]
-# ALL-NEXT:  1      1     1.00           *            str	d21, [x22, x23, lsl #3]
-# ALL-NEXT:  2      3     1.00           *            str	q24, [x25, x26, lsl #4]
diff --git a/test/tools/llvm-mca/AArch64/Exynos/scheduler-queue-usage.s b/test/tools/llvm-mca/AArch64/Exynos/scheduler-queue-usage.s
index bccae7d41b15cfe96d60d4f6cae8a7c21c19edc0..8a2c7804ef32c077ac8700aeaeb0342fc32b5003 100644
--- a/test/tools/llvm-mca/AArch64/Exynos/scheduler-queue-usage.s
+++ b/test/tools/llvm-mca/AArch64/Exynos/scheduler-queue-usage.s
@@ -1,6 +1,7 @@
 # NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
 # RUN: llvm-mca -march=aarch64 -mcpu=exynos-m1 -iterations=1 -scheduler-stats -resource-pressure=false -instruction-info=false < %s | FileCheck %s -check-prefixes=ALL,M1
 # RUN: llvm-mca -march=aarch64 -mcpu=exynos-m3 -iterations=1 -scheduler-stats -resource-pressure=false -instruction-info=false < %s | FileCheck %s -check-prefixes=ALL,M3
+# RUN: llvm-mca -march=aarch64 -mcpu=exynos-m4 -iterations=1 -scheduler-stats -resource-pressure=false -instruction-info=false < %s | FileCheck %s -check-prefixes=ALL,M4
 
   b	main
 
@@ -19,6 +20,11 @@
 # M3-NEXT:  IPC:               0.50
 # M3-NEXT:  Block RThroughput: 0.2
 
+# M4:       Dispatch Width:    6
+# M4-NEXT:  uOps Per Cycle:    0.50
+# M4-NEXT:  IPC:               0.50
+# M4-NEXT:  Block RThroughput: 0.2
+
 # ALL:      Schedulers - number of cycles where we saw N instructions issued:
 # ALL-NEXT: [# issued], [# cycles]
 # ALL-NEXT:  0,          1  (50.0%)
diff --git a/test/tools/llvm-mca/AArch64/Exynos/shifted-register.s b/test/tools/llvm-mca/AArch64/Exynos/shifted-register.s
index a7ff007129fb9555d3bce3c96a692b0ebda75925..5dfdc1e4ca14a754dd8c88ff1679b59c947dfa5e 100644
--- a/test/tools/llvm-mca/AArch64/Exynos/shifted-register.s
+++ b/test/tools/llvm-mca/AArch64/Exynos/shifted-register.s
@@ -1,21 +1,23 @@
 # NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
 # RUN: llvm-mca -march=aarch64 -mcpu=exynos-m1 -resource-pressure=false < %s | FileCheck %s -check-prefixes=ALL,EM1
 # RUN: llvm-mca -march=aarch64 -mcpu=exynos-m3 -resource-pressure=false < %s | FileCheck %s -check-prefixes=ALL,EM3
+# RUN: llvm-mca -march=aarch64 -mcpu=exynos-m4 -resource-pressure=false < %s | FileCheck %s -check-prefixes=ALL,EM4
 
-  add	w0, w1, w2, lsl #0
+  adds	w0, w1, w2, lsl #0
   sub	x3, x4, x5, lsr #1
-  adds	x6, x7, x8, lsl #2
-  subs	w9, w10, w11, asr #3
-  add	w12, w13, w14, lsl #4
+  ands	x6, x7, x8, lsl #2
+  orr	w9, w10, w11, asr #3
+  adds	w12, w13, w14, lsl #4
   sub	x15, x16, x17, lsr #6
-  adds	x18, x19, x20, lsl #8
-  subs	w21, w22, w23, asr #10
+  ands	x18, x19, x20, lsl #8
+  orr	w21, w22, w23, asr #10
 
 # ALL:      Iterations:        100
 # ALL-NEXT: Instructions:      800
 
 # EM1-NEXT: Total Cycles:      470
 # EM3-NEXT: Total Cycles:      354
+# EM4-NEXT: Total Cycles:      329
 
 # ALL-NEXT: Total uOps:        800
 
@@ -29,6 +31,11 @@
 # EM3-NEXT: IPC:               2.26
 # EM3-NEXT: Block RThroughput: 3.5
 
+# EM4:      Dispatch Width:    6
+# EM4-NEXT: uOps Per Cycle:    2.43
+# EM4-NEXT: IPC:               2.43
+# EM4-NEXT: Block RThroughput: 3.3
+
 # ALL:      Instruction Info:
 # ALL-NEXT: [1]: #uOps
 # ALL-NEXT: [2]: Latency
@@ -39,20 +46,29 @@
 
 # ALL:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
 
-# EM1-NEXT:  1      1     0.33                        add	w0, w1, w2
+# EM1-NEXT:  1      1     0.33                        adds	w0, w1, w2
 # EM1-NEXT:  1      2     0.67                        sub	x3, x4, x5, lsr #1
-# EM1-NEXT:  1      1     0.33                        adds	x6, x7, x8, lsl #2
-# EM1-NEXT:  1      2     0.67                        subs	w9, w10, w11, asr #3
-# EM1-NEXT:  1      2     0.67                        add	w12, w13, w14, lsl #4
+# EM1-NEXT:  1      1     0.33                        ands	x6, x7, x8, lsl #2
+# EM1-NEXT:  1      2     0.67                        orr	w9, w10, w11, asr #3
+# EM1-NEXT:  1      2     0.67                        adds	w12, w13, w14, lsl #4
 # EM1-NEXT:  1      2     0.67                        sub	x15, x16, x17, lsr #6
-# EM1-NEXT:  1      2     0.67                        adds	x18, x19, x20, lsl #8
-# EM1-NEXT:  1      2     0.67                        subs	w21, w22, w23, asr #10
+# EM1-NEXT:  1      2     0.67                        ands	x18, x19, x20, lsl #8
+# EM1-NEXT:  1      2     0.67                        orr	w21, w22, w23, asr #10
 
-# EM3-NEXT:  1      1     0.25                        add	w0, w1, w2
+# EM3-NEXT:  1      1     0.25                        adds	w0, w1, w2
 # EM3-NEXT:  1      2     0.50                        sub	x3, x4, x5, lsr #1
-# EM3-NEXT:  1      1     0.25                        adds	x6, x7, x8, lsl #2
-# EM3-NEXT:  1      2     0.50                        subs	w9, w10, w11, asr #3
-# EM3-NEXT:  1      2     0.50                        add	w12, w13, w14, lsl #4
+# EM3-NEXT:  1      1     0.25                        ands	x6, x7, x8, lsl #2
+# EM3-NEXT:  1      2     0.50                        orr	w9, w10, w11, asr #3
+# EM3-NEXT:  1      2     0.50                        adds	w12, w13, w14, lsl #4
 # EM3-NEXT:  1      2     0.50                        sub	x15, x16, x17, lsr #6
-# EM3-NEXT:  1      2     0.50                        adds	x18, x19, x20, lsl #8
-# EM3-NEXT:  1      2     0.50                        subs	w21, w22, w23, asr #10
+# EM3-NEXT:  1      2     0.50                        ands	x18, x19, x20, lsl #8
+# EM3-NEXT:  1      2     0.50                        orr	w21, w22, w23, asr #10
+
+# EM4-NEXT:  1      1     0.25                        adds	w0, w1, w2
+# EM4-NEXT:  1      2     0.50                        sub	x3, x4, x5, lsr #1
+# EM4-NEXT:  1      1     0.25                        ands	x6, x7, x8, lsl #2
+# EM4-NEXT:  1      2     0.50                        orr	w9, w10, w11, asr #3
+# EM4-NEXT:  1      2     0.50                        adds	w12, w13, w14, lsl #4
+# EM4-NEXT:  1      2     0.50                        sub	x15, x16, x17, lsr #6
+# EM4-NEXT:  1      1     0.25                        ands	x18, x19, x20, lsl #8
+# EM4-NEXT:  1      2     0.50                        orr	w21, w22, w23, asr #10
diff --git a/test/tools/llvm-mca/X86/Atom/resources-sse2.s b/test/tools/llvm-mca/X86/Atom/resources-sse2.s
index e1e903943ece8d75a4c7b6fc4bedc44074991956..3f65619e0f59f404e48f0be7c49c1c15915b1284 100644
--- a/test/tools/llvm-mca/X86/Atom/resources-sse2.s
+++ b/test/tools/llvm-mca/X86/Atom/resources-sse2.s
@@ -94,6 +94,8 @@ maxpd       (%rax), %xmm2
 maxsd       %xmm0, %xmm2
 maxsd       (%rax), %xmm2
 
+mfence
+
 minpd       %xmm0, %xmm2
 minpd       (%rax), %xmm2
 
@@ -229,6 +231,9 @@ pcmpgtw     (%rax), %xmm2
 
 pextrw      $1, %xmm0, %rcx
 
+pinsrw      $1, %rax, %xmm0
+pinsrw      $1, (%rax), %xmm0
+
 pmaddwd     %xmm0, %xmm2
 pmaddwd     (%rax), %xmm2
 
@@ -465,6 +470,7 @@ xorpd       (%rax), %xmm2
 # CHECK-NEXT:  1      7     3.50    *                   maxpd	(%rax), %xmm2
 # CHECK-NEXT:  1      5     5.00                        maxsd	%xmm0, %xmm2
 # CHECK-NEXT:  1      5     5.00    *                   maxsd	(%rax), %xmm2
+# CHECK-NEXT:  1      1     1.00    *      *      U     mfence
 # CHECK-NEXT:  1      6     3.00                        minpd	%xmm0, %xmm2
 # CHECK-NEXT:  1      7     3.50    *                   minpd	(%rax), %xmm2
 # CHECK-NEXT:  1      5     5.00                        minsd	%xmm0, %xmm2
@@ -555,6 +561,8 @@ xorpd       (%rax), %xmm2
 # CHECK-NEXT:  1      1     0.50                        pcmpgtw	%xmm0, %xmm2
 # CHECK-NEXT:  1      1     1.00    *                   pcmpgtw	(%rax), %xmm2
 # CHECK-NEXT:  1      4     2.00                        pextrw	$1, %xmm0, %ecx
+# CHECK-NEXT:  1      1     1.00                        pinsrw	$1, %eax, %xmm0
+# CHECK-NEXT:  1      1     1.00    *                   pinsrw	$1, (%rax), %xmm0
 # CHECK-NEXT:  1      5     5.00                        pmaddwd	%xmm0, %xmm2
 # CHECK-NEXT:  1      5     5.00    *                   pmaddwd	(%rax), %xmm2
 # CHECK-NEXT:  1      1     0.50                        pmaxsw	%xmm0, %xmm2
@@ -673,7 +681,7 @@ xorpd       (%rax), %xmm2
 
 # CHECK:      Resource pressure per iteration:
 # CHECK-NEXT: [0]    [1]
-# CHECK-NEXT: 868.50 638.50
+# CHECK-NEXT: 871.50 638.50
 
 # CHECK:      Resource pressure by instruction:
 # CHECK-NEXT: [0]    [1]    Instructions:
@@ -740,6 +748,7 @@ xorpd       (%rax), %xmm2
 # CHECK-NEXT: 3.50   3.50   maxpd	(%rax), %xmm2
 # CHECK-NEXT: 5.00    -     maxsd	%xmm0, %xmm2
 # CHECK-NEXT: 5.00    -     maxsd	(%rax), %xmm2
+# CHECK-NEXT: 1.00    -     mfence
 # CHECK-NEXT: 3.00   3.00   minpd	%xmm0, %xmm2
 # CHECK-NEXT: 3.50   3.50   minpd	(%rax), %xmm2
 # CHECK-NEXT: 5.00    -     minsd	%xmm0, %xmm2
@@ -830,6 +839,8 @@ xorpd       (%rax), %xmm2
 # CHECK-NEXT: 0.50   0.50   pcmpgtw	%xmm0, %xmm2
 # CHECK-NEXT: 1.00    -     pcmpgtw	(%rax), %xmm2
 # CHECK-NEXT: 2.00   2.00   pextrw	$1, %xmm0, %ecx
+# CHECK-NEXT: 1.00    -     pinsrw	$1, %eax, %xmm0
+# CHECK-NEXT: 1.00    -     pinsrw	$1, (%rax), %xmm0
 # CHECK-NEXT: 5.00    -     pmaddwd	%xmm0, %xmm2
 # CHECK-NEXT: 5.00    -     pmaddwd	(%rax), %xmm2
 # CHECK-NEXT: 0.50   0.50   pmaxsw	%xmm0, %xmm2
diff --git a/test/tools/llvm-mca/X86/Atom/resources-sse3.s b/test/tools/llvm-mca/X86/Atom/resources-sse3.s
index 8ecbbba79d73b6e7429453b533f03f8fea39a410..c0b4b3048d270b1fc9dea5dcfb898ddd207bd29e 100644
--- a/test/tools/llvm-mca/X86/Atom/resources-sse3.s
+++ b/test/tools/llvm-mca/X86/Atom/resources-sse3.s
@@ -21,6 +21,8 @@ hsubps    (%rax), %xmm2
 
 lddqu     (%rax), %xmm2
 
+monitor
+
 movddup   %xmm0, %xmm2
 movddup   (%rax), %xmm2
 
@@ -30,6 +32,8 @@ movshdup  (%rax), %xmm2
 movsldup  %xmm0, %xmm2
 movsldup  (%rax), %xmm2
 
+mwait
+
 # CHECK:      Instruction Info:
 # CHECK-NEXT: [1]: #uOps
 # CHECK-NEXT: [2]: Latency
@@ -52,12 +56,14 @@ movsldup  (%rax), %xmm2
 # CHECK-NEXT:  1      8     4.00                        hsubps	%xmm0, %xmm2
 # CHECK-NEXT:  1      9     4.50    *                   hsubps	(%rax), %xmm2
 # CHECK-NEXT:  1      3     1.50    *                   lddqu	(%rax), %xmm2
+# CHECK-NEXT:  1      45    22.50                 U     monitor
 # CHECK-NEXT:  1      1     1.00                        movddup	%xmm0, %xmm2
 # CHECK-NEXT:  1      1     1.00    *                   movddup	(%rax), %xmm2
 # CHECK-NEXT:  1      1     1.00                        movshdup	%xmm0, %xmm2
 # CHECK-NEXT:  1      1     1.00    *                   movshdup	(%rax), %xmm2
 # CHECK-NEXT:  1      1     1.00                        movsldup	%xmm0, %xmm2
 # CHECK-NEXT:  1      1     1.00    *                   movsldup	(%rax), %xmm2
+# CHECK-NEXT:  1      46    23.00   *      *      U     mwait
 
 # CHECK:      Resources:
 # CHECK-NEXT: [0]   - AtomPort0
@@ -65,7 +71,7 @@ movsldup  (%rax), %xmm2
 
 # CHECK:      Resource pressure per iteration:
 # CHECK-NEXT: [0]    [1]
-# CHECK-NEXT: 58.00  42.00
+# CHECK-NEXT: 103.50 87.50
 
 # CHECK:      Resource pressure by instruction:
 # CHECK-NEXT: [0]    [1]    Instructions:
@@ -82,9 +88,11 @@ movsldup  (%rax), %xmm2
 # CHECK-NEXT: 4.00   4.00   hsubps	%xmm0, %xmm2
 # CHECK-NEXT: 4.50   4.50   hsubps	(%rax), %xmm2
 # CHECK-NEXT: 1.50   1.50   lddqu	(%rax), %xmm2
+# CHECK-NEXT: 22.50  22.50  monitor
 # CHECK-NEXT: 1.00    -     movddup	%xmm0, %xmm2
 # CHECK-NEXT: 1.00    -     movddup	(%rax), %xmm2
 # CHECK-NEXT: 1.00    -     movshdup	%xmm0, %xmm2
 # CHECK-NEXT: 1.00    -     movshdup	(%rax), %xmm2
 # CHECK-NEXT: 1.00    -     movsldup	%xmm0, %xmm2
 # CHECK-NEXT: 1.00    -     movsldup	(%rax), %xmm2
+# CHECK-NEXT: 23.00  23.00  mwait
diff --git a/test/tools/llvm-mca/X86/Atom/resources-x86_64.s b/test/tools/llvm-mca/X86/Atom/resources-x86_64.s
index e757844153c4e18341b1b0f52840fa985becc981..04d2f191fe3260d884992a942f2a6c45de944b95 100644
--- a/test/tools/llvm-mca/X86/Atom/resources-x86_64.s
+++ b/test/tools/llvm-mca/X86/Atom/resources-x86_64.s
@@ -254,6 +254,8 @@ divl (%rax)
 divq %rcx
 divq (%rax)
 
+enter $7, $4095
+
 idivb %dil
 idivb (%rax)
 idivw %si
@@ -315,8 +317,13 @@ insl
 
 int $7
 
+invlpg  (%rax)
+invlpga %rax, %ecx
+
 lahf
 
+leave
+
 lodsb
 lodsw
 lodsl
@@ -486,6 +493,11 @@ rcrq %cl, %rdi
 rclq %cl, (%rax)
 rcrq %cl, (%rax)
 
+rdmsr
+rdpmc
+rdtsc
+rdtscp
+
 rolb %dil
 rorb %dil
 rolb (%rax)
@@ -789,6 +801,8 @@ testq %rsi, (%rax)
 
 ud2
 
+wrmsr
+
 xaddb %bl, %cl
 xaddb %bl, (%rcx)
 
@@ -1081,6 +1095,7 @@ xorq (%rax), %rdi
 # CHECK-NEXT:  1      50    25.00   *             U     divl	(%rax)
 # CHECK-NEXT:  1      130   65.00                 U     divq	%rcx
 # CHECK-NEXT:  1      130   65.00   *             U     divq	(%rax)
+# CHECK-NEXT:  1      32    16.00                 U     enter	$7, $4095
 # CHECK-NEXT:  1      62    31.00                 U     idivb	%dil
 # CHECK-NEXT:  1      62    31.00   *             U     idivb	(%rax)
 # CHECK-NEXT:  1      62    31.00                 U     idivw	%si
@@ -1133,7 +1148,10 @@ xorq (%rax), %rdi
 # CHECK-NEXT:  1      59    29.50                 U     insw	%dx, %es:(%rdi)
 # CHECK-NEXT:  1      59    29.50                 U     insl	%dx, %es:(%rdi)
 # CHECK-NEXT:  1      127   63.50   *      *      U     int	$7
+# CHECK-NEXT:  1      71    35.50                 U     invlpg	(%rax)
+# CHECK-NEXT:  1      71    35.50                 U     invlpga	%rax, %ecx
 # CHECK-NEXT:  1      2     1.00                        lahf
+# CHECK-NEXT:  1      2     1.00    *                   leave
 # CHECK-NEXT:  1      2     1.00                  U     lodsb	(%rsi), %al
 # CHECK-NEXT:  1      2     1.00                  U     lodsw	(%rsi), %ax
 # CHECK-NEXT:  1      2     1.00                  U     lodsl	(%rsi), %eax
@@ -1283,6 +1301,10 @@ xorq (%rax), %rdi
 # CHECK-NEXT:  1      1     1.00                        rcrq	%cl, %rdi
 # CHECK-NEXT:  1      1     1.00           *            rclq	%cl, (%rax)
 # CHECK-NEXT:  1      1     1.00           *            rcrq	%cl, (%rax)
+# CHECK-NEXT:  1      78    39.00                 U     rdmsr
+# CHECK-NEXT:  1      46    23.00                 U     rdpmc
+# CHECK-NEXT:  1      30    15.00                 U     rdtsc
+# CHECK-NEXT:  1      30    15.00                 U     rdtscp
 # CHECK-NEXT:  1      1     1.00                        rolb	%dil
 # CHECK-NEXT:  1      1     1.00                        rorb	%dil
 # CHECK-NEXT:  1      1     1.00    *      *            rolb	(%rax)
@@ -1557,6 +1579,7 @@ xorq (%rax), %rdi
 # CHECK-NEXT:  1      1     0.50                        testq	%rsi, %rdi
 # CHECK-NEXT:  1      1     1.00    *                   testq	%rsi, (%rax)
 # CHECK-NEXT:  1      100   0.50    *             U     ud2
+# CHECK-NEXT:  1      202   101.00                U     wrmsr
 # CHECK-NEXT:  1      2     1.00                        xaddb	%bl, %cl
 # CHECK-NEXT:  1      3     1.50    *      *            xaddb	%bl, (%rcx)
 # CHECK-NEXT:  1      2     1.00                        xaddw	%bx, %cx
@@ -1614,7 +1637,7 @@ xorq (%rax), %rdi
 
 # CHECK:      Resource pressure per iteration:
 # CHECK-NEXT: [0]    [1]
-# CHECK-NEXT: 2393.50 2049.50
+# CHECK-NEXT: 2674.50 2330.50
 
 # CHECK:      Resource pressure by instruction:
 # CHECK-NEXT: [0]    [1]    Instructions:
@@ -1838,6 +1861,7 @@ xorq (%rax), %rdi
 # CHECK-NEXT: 25.00  25.00  divl	(%rax)
 # CHECK-NEXT: 65.00  65.00  divq	%rcx
 # CHECK-NEXT: 65.00  65.00  divq	(%rax)
+# CHECK-NEXT: 16.00  16.00  enter	$7, $4095
 # CHECK-NEXT: 31.00  31.00  idivb	%dil
 # CHECK-NEXT: 31.00  31.00  idivb	(%rax)
 # CHECK-NEXT: 31.00  31.00  idivw	%si
@@ -1890,7 +1914,10 @@ xorq (%rax), %rdi
 # CHECK-NEXT: 29.50  29.50  insw	%dx, %es:(%rdi)
 # CHECK-NEXT: 29.50  29.50  insl	%dx, %es:(%rdi)
 # CHECK-NEXT: 63.50  63.50  int	$7
+# CHECK-NEXT: 35.50  35.50  invlpg	(%rax)
+# CHECK-NEXT: 35.50  35.50  invlpga	%rax, %ecx
 # CHECK-NEXT: 1.00   1.00   lahf
+# CHECK-NEXT: 1.00   1.00   leave
 # CHECK-NEXT: 1.00   1.00   lodsb	(%rsi), %al
 # CHECK-NEXT: 1.00   1.00   lodsw	(%rsi), %ax
 # CHECK-NEXT: 1.00   1.00   lodsl	(%rsi), %eax
@@ -2040,6 +2067,10 @@ xorq (%rax), %rdi
 # CHECK-NEXT: 1.00    -     rcrq	%cl, %rdi
 # CHECK-NEXT: 1.00    -     rclq	%cl, (%rax)
 # CHECK-NEXT: 1.00    -     rcrq	%cl, (%rax)
+# CHECK-NEXT: 39.00  39.00  rdmsr
+# CHECK-NEXT: 23.00  23.00  rdpmc
+# CHECK-NEXT: 15.00  15.00  rdtsc
+# CHECK-NEXT: 15.00  15.00  rdtscp
 # CHECK-NEXT: 1.00    -     rolb	%dil
 # CHECK-NEXT: 1.00    -     rorb	%dil
 # CHECK-NEXT: 1.00    -     rolb	(%rax)
@@ -2314,6 +2345,7 @@ xorq (%rax), %rdi
 # CHECK-NEXT: 0.50   0.50   testq	%rsi, %rdi
 # CHECK-NEXT: 1.00    -     testq	%rsi, (%rax)
 # CHECK-NEXT: 0.50   0.50   ud2
+# CHECK-NEXT: 101.00 101.00 wrmsr
 # CHECK-NEXT: 1.00   1.00   xaddb	%bl, %cl
 # CHECK-NEXT: 1.50   1.50   xaddb	%bl, (%rcx)
 # CHECK-NEXT: 1.00   1.00   xaddw	%bx, %cx
diff --git a/test/tools/llvm-mca/X86/Atom/resources-x87.s b/test/tools/llvm-mca/X86/Atom/resources-x87.s
index c7a990a892eb67faa16d3ff23c779c05bbf291c6..eda0fdebb7d3c23a300d37416fbf1c431419cd29 100644
--- a/test/tools/llvm-mca/X86/Atom/resources-x87.s
+++ b/test/tools/llvm-mca/X86/Atom/resources-x87.s
@@ -5,7 +5,7 @@ f2xm1
 
 fabs
 
-fadd %st(0), %st(1)
+fadd %st, %st(1)
 fadd %st(2)
 fadds (%ecx)
 faddl (%ecx)
@@ -21,14 +21,14 @@ fchs
 
 fnclex
 
-fcmovb %st(1), %st(0)
-fcmovbe %st(1), %st(0)
-fcmove %st(1), %st(0)
-fcmovnb %st(1), %st(0)
-fcmovnbe %st(1), %st(0)
-fcmovne %st(1), %st(0)
-fcmovnu %st(1), %st(0)
-fcmovu %st(1), %st(0)
+fcmovb %st(1), %st
+fcmovbe %st(1), %st
+fcmove %st(1), %st
+fcmovnb %st(1), %st
+fcmovnbe %st(1), %st
+fcmovne %st(1), %st
+fcmovnu %st(1), %st
+fcmovu %st(1), %st
 
 fcom %st(1)
 fcom %st(3)
@@ -47,7 +47,7 @@ fcos
 
 fdecstp
 
-fdiv %st(0), %st(1)
+fdiv %st, %st(1)
 fdiv %st(2)
 fdivs (%ecx)
 fdivl (%eax)
@@ -56,7 +56,7 @@ fdivp %st(2)
 fidivs (%ecx)
 fidivl (%eax)
 
-fdivr %st(0), %st(1)
+fdivr %st, %st(1)
 fdivr %st(2)
 fdivrs (%ecx)
 fdivrl (%eax)
@@ -106,7 +106,7 @@ fldln2
 fldpi
 fldz
 
-fmul %st(0), %st(1)
+fmul %st, %st(1)
 fmul %st(2)
 fmuls (%ecx)
 fmull (%eax)
@@ -153,7 +153,7 @@ fnstsw (%eax)
 frstor (%eax)
 fsave (%eax)
 
-fsub %st(0), %st(1)
+fsub %st, %st(1)
 fsub %st(2)
 fsubs (%ecx)
 fsubl (%eax)
@@ -162,7 +162,7 @@ fsubp %st(2)
 fisubs (%ecx)
 fisubl (%eax)
 
-fsubr %st(0), %st(1)
+fsubr %st, %st(1)
 fsubr %st(2)
 fsubrs (%ecx)
 fsubrl (%eax)
@@ -208,26 +208,26 @@ fyl2xp1
 # CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
 # CHECK-NEXT:  1      99    49.50                 U     f2xm1
 # CHECK-NEXT:  1      1     1.00                  U     fabs
-# CHECK-NEXT:  1      5     5.00                  U     fadd	%st(0), %st(1)
-# CHECK-NEXT:  1      5     5.00                  U     fadd	%st(2)
+# CHECK-NEXT:  1      5     5.00                  U     fadd	%st, %st(1)
+# CHECK-NEXT:  1      5     5.00                  U     fadd	%st(2), %st
 # CHECK-NEXT:  1      5     5.00    *             U     fadds	(%ecx)
 # CHECK-NEXT:  1      5     5.00    *             U     faddl	(%ecx)
-# CHECK-NEXT:  1      5     5.00                  U     faddp	%st(1)
-# CHECK-NEXT:  1      5     5.00                  U     faddp	%st(2)
+# CHECK-NEXT:  1      5     5.00                  U     faddp	%st, %st(1)
+# CHECK-NEXT:  1      5     5.00                  U     faddp	%st, %st(2)
 # CHECK-NEXT:  1      5     5.00    *             U     fiadds	(%ecx)
 # CHECK-NEXT:  1      5     5.00    *             U     fiaddl	(%ecx)
 # CHECK-NEXT:  1      100   0.50                  U     fbld	(%ecx)
 # CHECK-NEXT:  1      100   0.50                  U     fbstp	(%eax)
 # CHECK-NEXT:  1      1     1.00                  U     fchs
 # CHECK-NEXT:  1      25    12.50                 U     fnclex
-# CHECK-NEXT:  1      9     4.50                  U     fcmovb	%st(1), %st(0)
-# CHECK-NEXT:  1      9     4.50                  U     fcmovbe	%st(1), %st(0)
-# CHECK-NEXT:  1      9     4.50                  U     fcmove	%st(1), %st(0)
-# CHECK-NEXT:  1      9     4.50                  U     fcmovnb	%st(1), %st(0)
-# CHECK-NEXT:  1      9     4.50                  U     fcmovnbe	%st(1), %st(0)
-# CHECK-NEXT:  1      9     4.50                  U     fcmovne	%st(1), %st(0)
-# CHECK-NEXT:  1      9     4.50                  U     fcmovnu	%st(1), %st(0)
-# CHECK-NEXT:  1      9     4.50                  U     fcmovu	%st(1), %st(0)
+# CHECK-NEXT:  1      9     4.50                  U     fcmovb	%st(1), %st
+# CHECK-NEXT:  1      9     4.50                  U     fcmovbe	%st(1), %st
+# CHECK-NEXT:  1      9     4.50                  U     fcmove	%st(1), %st
+# CHECK-NEXT:  1      9     4.50                  U     fcmovnb	%st(1), %st
+# CHECK-NEXT:  1      9     4.50                  U     fcmovnbe	%st(1), %st
+# CHECK-NEXT:  1      9     4.50                  U     fcmovne	%st(1), %st
+# CHECK-NEXT:  1      9     4.50                  U     fcmovnu	%st(1), %st
+# CHECK-NEXT:  1      9     4.50                  U     fcmovu	%st(1), %st
 # CHECK-NEXT:  1      5     5.00                  U     fcom	%st(1)
 # CHECK-NEXT:  1      5     5.00                  U     fcom	%st(3)
 # CHECK-NEXT:  1      5     5.00                  U     fcoms	(%ecx)
@@ -237,24 +237,24 @@ fyl2xp1
 # CHECK-NEXT:  1      5     5.00                  U     fcomps	(%ecx)
 # CHECK-NEXT:  1      5     5.00                  U     fcompl	(%eax)
 # CHECK-NEXT:  1      1     1.00                  U     fcompp
-# CHECK-NEXT:  1      9     4.50                  U     fcomi	%st(3)
-# CHECK-NEXT:  1      9     4.50                  U     fcompi	%st(3)
+# CHECK-NEXT:  1      9     4.50                  U     fcomi	%st(3), %st
+# CHECK-NEXT:  1      9     4.50                  U     fcompi	%st(3), %st
 # CHECK-NEXT:  1      174   87.00                 U     fcos
 # CHECK-NEXT:  1      1     0.50                  U     fdecstp
-# CHECK-NEXT:  1      34    17.00                 U     fdiv	%st(0), %st(1)
-# CHECK-NEXT:  1      34    17.00                 U     fdiv	%st(2)
+# CHECK-NEXT:  1      34    17.00                 U     fdiv	%st, %st(1)
+# CHECK-NEXT:  1      34    17.00                 U     fdiv	%st(2), %st
 # CHECK-NEXT:  1      34    17.00   *             U     fdivs	(%ecx)
 # CHECK-NEXT:  1      34    17.00   *             U     fdivl	(%eax)
-# CHECK-NEXT:  1      34    17.00                 U     fdivp	%st(1)
-# CHECK-NEXT:  1      34    17.00                 U     fdivp	%st(2)
+# CHECK-NEXT:  1      34    17.00                 U     fdivp	%st, %st(1)
+# CHECK-NEXT:  1      34    17.00                 U     fdivp	%st, %st(2)
 # CHECK-NEXT:  1      34    17.00   *             U     fidivs	(%ecx)
 # CHECK-NEXT:  1      34    17.00   *             U     fidivl	(%eax)
-# CHECK-NEXT:  1      34    17.00                 U     fdivr	%st(0), %st(1)
-# CHECK-NEXT:  1      34    17.00                 U     fdivr	%st(2)
+# CHECK-NEXT:  1      34    17.00                 U     fdivr	%st, %st(1)
+# CHECK-NEXT:  1      34    17.00                 U     fdivr	%st(2), %st
 # CHECK-NEXT:  1      34    17.00   *             U     fdivrs	(%ecx)
 # CHECK-NEXT:  1      34    17.00   *             U     fdivrl	(%eax)
-# CHECK-NEXT:  1      34    17.00                 U     fdivrp	%st(1)
-# CHECK-NEXT:  1      34    17.00                 U     fdivrp	%st(2)
+# CHECK-NEXT:  1      34    17.00                 U     fdivrp	%st, %st(1)
+# CHECK-NEXT:  1      34    17.00                 U     fdivrp	%st, %st(2)
 # CHECK-NEXT:  1      34    17.00   *             U     fidivrs	(%ecx)
 # CHECK-NEXT:  1      34    17.00   *             U     fidivrl	(%eax)
 # CHECK-NEXT:  1      1     0.50                  U     ffree	%st(0)
@@ -288,12 +288,12 @@ fyl2xp1
 # CHECK-NEXT:  1      10    5.00                  U     fldln2
 # CHECK-NEXT:  1      10    5.00                  U     fldpi
 # CHECK-NEXT:  1      1     0.50                  U     fldz
-# CHECK-NEXT:  1      4     4.00                  U     fmul	%st(0), %st(1)
-# CHECK-NEXT:  1      4     4.00                  U     fmul	%st(2)
+# CHECK-NEXT:  1      4     4.00                  U     fmul	%st, %st(1)
+# CHECK-NEXT:  1      4     4.00                  U     fmul	%st(2), %st
 # CHECK-NEXT:  1      4     4.00    *             U     fmuls	(%ecx)
 # CHECK-NEXT:  1      4     4.00    *             U     fmull	(%eax)
-# CHECK-NEXT:  1      4     4.00                  U     fmulp	%st(1)
-# CHECK-NEXT:  1      4     4.00                  U     fmulp	%st(2)
+# CHECK-NEXT:  1      4     4.00                  U     fmulp	%st, %st(1)
+# CHECK-NEXT:  1      4     4.00                  U     fmulp	%st, %st(2)
 # CHECK-NEXT:  1      4     4.00    *             U     fimuls	(%ecx)
 # CHECK-NEXT:  1      4     4.00    *             U     fimull	(%eax)
 # CHECK-NEXT:  1      1     0.50                  U     fnop
@@ -321,20 +321,20 @@ fyl2xp1
 # CHECK-NEXT:  1      100   0.50                  U     frstor	(%eax)
 # CHECK-NEXT:  1      1     0.50                  U     wait
 # CHECK-NEXT:  1      100   0.50                  U     fnsave	(%eax)
-# CHECK-NEXT:  1      5     5.00                  U     fsub	%st(0), %st(1)
-# CHECK-NEXT:  1      5     5.00                  U     fsub	%st(2)
+# CHECK-NEXT:  1      5     5.00                  U     fsub	%st, %st(1)
+# CHECK-NEXT:  1      5     5.00                  U     fsub	%st(2), %st
 # CHECK-NEXT:  1      5     5.00    *             U     fsubs	(%ecx)
 # CHECK-NEXT:  1      5     5.00    *             U     fsubl	(%eax)
-# CHECK-NEXT:  1      5     5.00                  U     fsubp	%st(1)
-# CHECK-NEXT:  1      5     5.00                  U     fsubp	%st(2)
+# CHECK-NEXT:  1      5     5.00                  U     fsubp	%st, %st(1)
+# CHECK-NEXT:  1      5     5.00                  U     fsubp	%st, %st(2)
 # CHECK-NEXT:  1      5     5.00    *             U     fisubs	(%ecx)
 # CHECK-NEXT:  1      5     5.00    *             U     fisubl	(%eax)
-# CHECK-NEXT:  1      5     5.00                  U     fsubr	%st(0), %st(1)
-# CHECK-NEXT:  1      5     5.00                  U     fsubr	%st(2)
+# CHECK-NEXT:  1      5     5.00                  U     fsubr	%st, %st(1)
+# CHECK-NEXT:  1      5     5.00                  U     fsubr	%st(2), %st
 # CHECK-NEXT:  1      5     5.00    *             U     fsubrs	(%ecx)
 # CHECK-NEXT:  1      5     5.00    *             U     fsubrl	(%eax)
-# CHECK-NEXT:  1      5     5.00                  U     fsubrp	%st(1)
-# CHECK-NEXT:  1      5     5.00                  U     fsubrp	%st(2)
+# CHECK-NEXT:  1      5     5.00                  U     fsubrp	%st, %st(1)
+# CHECK-NEXT:  1      5     5.00                  U     fsubrp	%st, %st(2)
 # CHECK-NEXT:  1      5     5.00    *             U     fisubrs	(%ecx)
 # CHECK-NEXT:  1      5     5.00    *             U     fisubrl	(%eax)
 # CHECK-NEXT:  1      9     4.50                  U     ftst
@@ -343,8 +343,8 @@ fyl2xp1
 # CHECK-NEXT:  1      1     1.00                  U     fucomp	%st(1)
 # CHECK-NEXT:  1      1     1.00                  U     fucomp	%st(3)
 # CHECK-NEXT:  1      1     1.00                  U     fucompp
-# CHECK-NEXT:  1      9     4.50                  U     fucomi	%st(3)
-# CHECK-NEXT:  1      9     4.50                  U     fucompi	%st(3)
+# CHECK-NEXT:  1      9     4.50                  U     fucomi	%st(3), %st
+# CHECK-NEXT:  1      9     4.50                  U     fucompi	%st(3), %st
 # CHECK-NEXT:  1      1     0.50                  U     wait
 # CHECK-NEXT:  1      1     1.00                  U     fxam
 # CHECK-NEXT:  1      1     1.00                  U     fxch	%st(1)
@@ -367,26 +367,26 @@ fyl2xp1
 # CHECK-NEXT: [0]    [1]    Instructions:
 # CHECK-NEXT: 49.50  49.50  f2xm1
 # CHECK-NEXT:  -     1.00   fabs
-# CHECK-NEXT: 5.00    -     fadd	%st(0), %st(1)
-# CHECK-NEXT: 5.00    -     fadd	%st(2)
+# CHECK-NEXT: 5.00    -     fadd	%st, %st(1)
+# CHECK-NEXT: 5.00    -     fadd	%st(2), %st
 # CHECK-NEXT: 5.00    -     fadds	(%ecx)
 # CHECK-NEXT: 5.00    -     faddl	(%ecx)
-# CHECK-NEXT: 5.00    -     faddp	%st(1)
-# CHECK-NEXT: 5.00    -     faddp	%st(2)
+# CHECK-NEXT: 5.00    -     faddp	%st, %st(1)
+# CHECK-NEXT: 5.00    -     faddp	%st, %st(2)
 # CHECK-NEXT: 5.00    -     fiadds	(%ecx)
 # CHECK-NEXT: 5.00    -     fiaddl	(%ecx)
 # CHECK-NEXT: 0.50   0.50   fbld	(%ecx)
 # CHECK-NEXT: 0.50   0.50   fbstp	(%eax)
 # CHECK-NEXT:  -     1.00   fchs
 # CHECK-NEXT: 12.50  12.50  fnclex
-# CHECK-NEXT: 4.50   4.50   fcmovb	%st(1), %st(0)
-# CHECK-NEXT: 4.50   4.50   fcmovbe	%st(1), %st(0)
-# CHECK-NEXT: 4.50   4.50   fcmove	%st(1), %st(0)
-# CHECK-NEXT: 4.50   4.50   fcmovnb	%st(1), %st(0)
-# CHECK-NEXT: 4.50   4.50   fcmovnbe	%st(1), %st(0)
-# CHECK-NEXT: 4.50   4.50   fcmovne	%st(1), %st(0)
-# CHECK-NEXT: 4.50   4.50   fcmovnu	%st(1), %st(0)
-# CHECK-NEXT: 4.50   4.50   fcmovu	%st(1), %st(0)
+# CHECK-NEXT: 4.50   4.50   fcmovb	%st(1), %st
+# CHECK-NEXT: 4.50   4.50   fcmovbe	%st(1), %st
+# CHECK-NEXT: 4.50   4.50   fcmove	%st(1), %st
+# CHECK-NEXT: 4.50   4.50   fcmovnb	%st(1), %st
+# CHECK-NEXT: 4.50   4.50   fcmovnbe	%st(1), %st
+# CHECK-NEXT: 4.50   4.50   fcmovne	%st(1), %st
+# CHECK-NEXT: 4.50   4.50   fcmovnu	%st(1), %st
+# CHECK-NEXT: 4.50   4.50   fcmovu	%st(1), %st
 # CHECK-NEXT: 5.00    -     fcom	%st(1)
 # CHECK-NEXT: 5.00    -     fcom	%st(3)
 # CHECK-NEXT: 5.00    -     fcoms	(%ecx)
@@ -396,24 +396,24 @@ fyl2xp1
 # CHECK-NEXT: 5.00    -     fcomps	(%ecx)
 # CHECK-NEXT: 5.00    -     fcompl	(%eax)
 # CHECK-NEXT:  -     1.00   fcompp
-# CHECK-NEXT: 4.50   4.50   fcomi	%st(3)
-# CHECK-NEXT: 4.50   4.50   fcompi	%st(3)
+# CHECK-NEXT: 4.50   4.50   fcomi	%st(3), %st
+# CHECK-NEXT: 4.50   4.50   fcompi	%st(3), %st
 # CHECK-NEXT: 87.00  87.00  fcos
 # CHECK-NEXT: 0.50   0.50   fdecstp
-# CHECK-NEXT: 17.00  17.00  fdiv	%st(0), %st(1)
-# CHECK-NEXT: 17.00  17.00  fdiv	%st(2)
+# CHECK-NEXT: 17.00  17.00  fdiv	%st, %st(1)
+# CHECK-NEXT: 17.00  17.00  fdiv	%st(2), %st
 # CHECK-NEXT: 17.00  17.00  fdivs	(%ecx)
 # CHECK-NEXT: 17.00  17.00  fdivl	(%eax)
-# CHECK-NEXT: 17.00  17.00  fdivp	%st(1)
-# CHECK-NEXT: 17.00  17.00  fdivp	%st(2)
+# CHECK-NEXT: 17.00  17.00  fdivp	%st, %st(1)
+# CHECK-NEXT: 17.00  17.00  fdivp	%st, %st(2)
 # CHECK-NEXT: 17.00  17.00  fidivs	(%ecx)
 # CHECK-NEXT: 17.00  17.00  fidivl	(%eax)
-# CHECK-NEXT: 17.00  17.00  fdivr	%st(0), %st(1)
-# CHECK-NEXT: 17.00  17.00  fdivr	%st(2)
+# CHECK-NEXT: 17.00  17.00  fdivr	%st, %st(1)
+# CHECK-NEXT: 17.00  17.00  fdivr	%st(2), %st
 # CHECK-NEXT: 17.00  17.00  fdivrs	(%ecx)
 # CHECK-NEXT: 17.00  17.00  fdivrl	(%eax)
-# CHECK-NEXT: 17.00  17.00  fdivrp	%st(1)
-# CHECK-NEXT: 17.00  17.00  fdivrp	%st(2)
+# CHECK-NEXT: 17.00  17.00  fdivrp	%st, %st(1)
+# CHECK-NEXT: 17.00  17.00  fdivrp	%st, %st(2)
 # CHECK-NEXT: 17.00  17.00  fidivrs	(%ecx)
 # CHECK-NEXT: 17.00  17.00  fidivrl	(%eax)
 # CHECK-NEXT: 0.50   0.50   ffree	%st(0)
@@ -447,12 +447,12 @@ fyl2xp1
 # CHECK-NEXT: 5.00   5.00   fldln2
 # CHECK-NEXT: 5.00   5.00   fldpi
 # CHECK-NEXT: 0.50   0.50   fldz
-# CHECK-NEXT: 4.00    -     fmul	%st(0), %st(1)
-# CHECK-NEXT: 4.00    -     fmul	%st(2)
+# CHECK-NEXT: 4.00    -     fmul	%st, %st(1)
+# CHECK-NEXT: 4.00    -     fmul	%st(2), %st
 # CHECK-NEXT: 4.00    -     fmuls	(%ecx)
 # CHECK-NEXT: 4.00    -     fmull	(%eax)
-# CHECK-NEXT: 4.00    -     fmulp	%st(1)
-# CHECK-NEXT: 4.00    -     fmulp	%st(2)
+# CHECK-NEXT: 4.00    -     fmulp	%st, %st(1)
+# CHECK-NEXT: 4.00    -     fmulp	%st, %st(2)
 # CHECK-NEXT: 4.00    -     fimuls	(%ecx)
 # CHECK-NEXT: 4.00    -     fimull	(%eax)
 # CHECK-NEXT: 0.50   0.50   fnop
@@ -480,20 +480,20 @@ fyl2xp1
 # CHECK-NEXT: 0.50   0.50   frstor	(%eax)
 # CHECK-NEXT: 0.50   0.50   wait
 # CHECK-NEXT: 0.50   0.50   fnsave	(%eax)
-# CHECK-NEXT: 5.00    -     fsub	%st(0), %st(1)
-# CHECK-NEXT: 5.00    -     fsub	%st(2)
+# CHECK-NEXT: 5.00    -     fsub	%st, %st(1)
+# CHECK-NEXT: 5.00    -     fsub	%st(2), %st
 # CHECK-NEXT: 5.00    -     fsubs	(%ecx)
 # CHECK-NEXT: 5.00    -     fsubl	(%eax)
-# CHECK-NEXT: 5.00    -     fsubp	%st(1)
-# CHECK-NEXT: 5.00    -     fsubp	%st(2)
+# CHECK-NEXT: 5.00    -     fsubp	%st, %st(1)
+# CHECK-NEXT: 5.00    -     fsubp	%st, %st(2)
 # CHECK-NEXT: 5.00    -     fisubs	(%ecx)
 # CHECK-NEXT: 5.00    -     fisubl	(%eax)
-# CHECK-NEXT: 5.00    -     fsubr	%st(0), %st(1)
-# CHECK-NEXT: 5.00    -     fsubr	%st(2)
+# CHECK-NEXT: 5.00    -     fsubr	%st, %st(1)
+# CHECK-NEXT: 5.00    -     fsubr	%st(2), %st
 # CHECK-NEXT: 5.00    -     fsubrs	(%ecx)
 # CHECK-NEXT: 5.00    -     fsubrl	(%eax)
-# CHECK-NEXT: 5.00    -     fsubrp	%st(1)
-# CHECK-NEXT: 5.00    -     fsubrp	%st(2)
+# CHECK-NEXT: 5.00    -     fsubrp	%st, %st(1)
+# CHECK-NEXT: 5.00    -     fsubrp	%st, %st(2)
 # CHECK-NEXT: 5.00    -     fisubrs	(%ecx)
 # CHECK-NEXT: 5.00    -     fisubrl	(%eax)
 # CHECK-NEXT: 4.50   4.50   ftst
@@ -502,8 +502,8 @@ fyl2xp1
 # CHECK-NEXT:  -     1.00   fucomp	%st(1)
 # CHECK-NEXT:  -     1.00   fucomp	%st(3)
 # CHECK-NEXT:  -     1.00   fucompp
-# CHECK-NEXT: 4.50   4.50   fucomi	%st(3)
-# CHECK-NEXT: 4.50   4.50   fucompi	%st(3)
+# CHECK-NEXT: 4.50   4.50   fucomi	%st(3), %st
+# CHECK-NEXT: 4.50   4.50   fucompi	%st(3), %st
 # CHECK-NEXT: 0.50   0.50   wait
 # CHECK-NEXT: 1.00    -     fxam
 # CHECK-NEXT: 1.00   1.00   fxch	%st(1)
diff --git a/test/tools/llvm-mca/X86/BdVer2/int-to-fpu-forwarding-1.s b/test/tools/llvm-mca/X86/BdVer2/int-to-fpu-forwarding-1.s
new file mode 100644
index 0000000000000000000000000000000000000000..8fb9630e351e7a425200cc808977dd834952eb85
--- /dev/null
+++ b/test/tools/llvm-mca/X86/BdVer2/int-to-fpu-forwarding-1.s
@@ -0,0 +1,254 @@
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=x86_64-unknown-unknown -mcpu=bdver2 -iterations=500 < %s | FileCheck %s
+
+# LLVM-MCA-BEGIN
+vpinsrb $0, %eax, %xmm0, %xmm0
+vpinsrb $1, %eax, %xmm0, %xmm0
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN
+vpinsrw $0, %eax, %xmm0, %xmm0
+vpinsrw $1, %eax, %xmm0, %xmm0
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN
+vpinsrd $0, %eax, %xmm0, %xmm0
+vpinsrd $1, %eax, %xmm0, %xmm0
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN
+vpinsrq $0, %rax, %xmm0, %xmm0
+vpinsrq $1, %rax, %xmm0, %xmm0
+# LLVM-MCA-END
+
+# CHECK:      [0] Code Region
+
+# CHECK:      Iterations:        500
+# CHECK-NEXT: Instructions:      1000
+# CHECK-NEXT: Total Cycles:      2003
+# CHECK-NEXT: Total uOps:        2000
+
+# CHECK:      Dispatch Width:    4
+# CHECK-NEXT: uOps Per Cycle:    1.00
+# CHECK-NEXT: IPC:               0.50
+# CHECK-NEXT: Block RThroughput: 1.0
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  2      12    0.50                        vpinsrb	$0, %eax, %xmm0, %xmm0
+# CHECK-NEXT:  2      12    0.50                        vpinsrb	$1, %eax, %xmm0, %xmm0
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0.0] - PdAGLU01
+# CHECK-NEXT: [0.1] - PdAGLU01
+# CHECK-NEXT: [1]   - PdBranch
+# CHECK-NEXT: [2]   - PdCount
+# CHECK-NEXT: [3]   - PdDiv
+# CHECK-NEXT: [4]   - PdEX0
+# CHECK-NEXT: [5]   - PdEX1
+# CHECK-NEXT: [6]   - PdFPCVT
+# CHECK-NEXT: [7.0] - PdFPFMA
+# CHECK-NEXT: [7.1] - PdFPFMA
+# CHECK-NEXT: [8.0] - PdFPMAL
+# CHECK-NEXT: [8.1] - PdFPMAL
+# CHECK-NEXT: [9]   - PdFPMMA
+# CHECK-NEXT: [10]  - PdFPSTO
+# CHECK-NEXT: [11]  - PdFPU0
+# CHECK-NEXT: [12]  - PdFPU1
+# CHECK-NEXT: [13]  - PdFPU2
+# CHECK-NEXT: [14]  - PdFPU3
+# CHECK-NEXT: [15]  - PdFPXBR
+# CHECK-NEXT: [16.0] - PdLoad
+# CHECK-NEXT: [16.1] - PdLoad
+# CHECK-NEXT: [17]  - PdMul
+# CHECK-NEXT: [18]  - PdStore
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0.0]  [0.1]  [1]    [2]    [3]    [4]    [5]    [6]    [7.0]  [7.1]  [8.0]  [8.1]  [9]    [10]   [11]   [12]   [13]   [14]   [15]   [16.0] [16.1] [17]   [18]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     1.00   1.00    -      -      -      -      -      -      -
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0.0]  [0.1]  [1]    [2]    [3]    [4]    [5]    [6]    [7.0]  [7.1]  [8.0]  [8.1]  [9]    [10]   [11]   [12]   [13]   [14]   [15]   [16.0] [16.1] [17]   [18]   Instructions:
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -      -      -      -      -     vpinsrb	$0, %eax, %xmm0, %xmm0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -      -      -      -      -      -     vpinsrb	$1, %eax, %xmm0, %xmm0
+
+# CHECK:      [1] Code Region
+
+# CHECK:      Iterations:        500
+# CHECK-NEXT: Instructions:      1000
+# CHECK-NEXT: Total Cycles:      2003
+# CHECK-NEXT: Total uOps:        2000
+
+# CHECK:      Dispatch Width:    4
+# CHECK-NEXT: uOps Per Cycle:    1.00
+# CHECK-NEXT: IPC:               0.50
+# CHECK-NEXT: Block RThroughput: 1.0
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  2      12    0.50                        vpinsrw	$0, %eax, %xmm0, %xmm0
+# CHECK-NEXT:  2      12    0.50                        vpinsrw	$1, %eax, %xmm0, %xmm0
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0.0] - PdAGLU01
+# CHECK-NEXT: [0.1] - PdAGLU01
+# CHECK-NEXT: [1]   - PdBranch
+# CHECK-NEXT: [2]   - PdCount
+# CHECK-NEXT: [3]   - PdDiv
+# CHECK-NEXT: [4]   - PdEX0
+# CHECK-NEXT: [5]   - PdEX1
+# CHECK-NEXT: [6]   - PdFPCVT
+# CHECK-NEXT: [7.0] - PdFPFMA
+# CHECK-NEXT: [7.1] - PdFPFMA
+# CHECK-NEXT: [8.0] - PdFPMAL
+# CHECK-NEXT: [8.1] - PdFPMAL
+# CHECK-NEXT: [9]   - PdFPMMA
+# CHECK-NEXT: [10]  - PdFPSTO
+# CHECK-NEXT: [11]  - PdFPU0
+# CHECK-NEXT: [12]  - PdFPU1
+# CHECK-NEXT: [13]  - PdFPU2
+# CHECK-NEXT: [14]  - PdFPU3
+# CHECK-NEXT: [15]  - PdFPXBR
+# CHECK-NEXT: [16.0] - PdLoad
+# CHECK-NEXT: [16.1] - PdLoad
+# CHECK-NEXT: [17]  - PdMul
+# CHECK-NEXT: [18]  - PdStore
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0.0]  [0.1]  [1]    [2]    [3]    [4]    [5]    [6]    [7.0]  [7.1]  [8.0]  [8.1]  [9]    [10]   [11]   [12]   [13]   [14]   [15]   [16.0] [16.1] [17]   [18]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     1.00   1.00    -      -      -      -      -      -      -
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0.0]  [0.1]  [1]    [2]    [3]    [4]    [5]    [6]    [7.0]  [7.1]  [8.0]  [8.1]  [9]    [10]   [11]   [12]   [13]   [14]   [15]   [16.0] [16.1] [17]   [18]   Instructions:
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -      -      -      -      -     vpinsrw	$0, %eax, %xmm0, %xmm0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -      -      -      -      -      -     vpinsrw	$1, %eax, %xmm0, %xmm0
+
+# CHECK:      [2] Code Region
+
+# CHECK:      Iterations:        500
+# CHECK-NEXT: Instructions:      1000
+# CHECK-NEXT: Total Cycles:      2003
+# CHECK-NEXT: Total uOps:        2000
+
+# CHECK:      Dispatch Width:    4
+# CHECK-NEXT: uOps Per Cycle:    1.00
+# CHECK-NEXT: IPC:               0.50
+# CHECK-NEXT: Block RThroughput: 1.0
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  2      12    0.50                        vpinsrd	$0, %eax, %xmm0, %xmm0
+# CHECK-NEXT:  2      12    0.50                        vpinsrd	$1, %eax, %xmm0, %xmm0
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0.0] - PdAGLU01
+# CHECK-NEXT: [0.1] - PdAGLU01
+# CHECK-NEXT: [1]   - PdBranch
+# CHECK-NEXT: [2]   - PdCount
+# CHECK-NEXT: [3]   - PdDiv
+# CHECK-NEXT: [4]   - PdEX0
+# CHECK-NEXT: [5]   - PdEX1
+# CHECK-NEXT: [6]   - PdFPCVT
+# CHECK-NEXT: [7.0] - PdFPFMA
+# CHECK-NEXT: [7.1] - PdFPFMA
+# CHECK-NEXT: [8.0] - PdFPMAL
+# CHECK-NEXT: [8.1] - PdFPMAL
+# CHECK-NEXT: [9]   - PdFPMMA
+# CHECK-NEXT: [10]  - PdFPSTO
+# CHECK-NEXT: [11]  - PdFPU0
+# CHECK-NEXT: [12]  - PdFPU1
+# CHECK-NEXT: [13]  - PdFPU2
+# CHECK-NEXT: [14]  - PdFPU3
+# CHECK-NEXT: [15]  - PdFPXBR
+# CHECK-NEXT: [16.0] - PdLoad
+# CHECK-NEXT: [16.1] - PdLoad
+# CHECK-NEXT: [17]  - PdMul
+# CHECK-NEXT: [18]  - PdStore
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0.0]  [0.1]  [1]    [2]    [3]    [4]    [5]    [6]    [7.0]  [7.1]  [8.0]  [8.1]  [9]    [10]   [11]   [12]   [13]   [14]   [15]   [16.0] [16.1] [17]   [18]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     1.00   1.00    -      -      -      -      -      -      -
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0.0]  [0.1]  [1]    [2]    [3]    [4]    [5]    [6]    [7.0]  [7.1]  [8.0]  [8.1]  [9]    [10]   [11]   [12]   [13]   [14]   [15]   [16.0] [16.1] [17]   [18]   Instructions:
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -      -      -      -      -     vpinsrd	$0, %eax, %xmm0, %xmm0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -      -      -      -      -      -     vpinsrd	$1, %eax, %xmm0, %xmm0
+
+# CHECK:      [3] Code Region
+
+# CHECK:      Iterations:        500
+# CHECK-NEXT: Instructions:      1000
+# CHECK-NEXT: Total Cycles:      2003
+# CHECK-NEXT: Total uOps:        2000
+
+# CHECK:      Dispatch Width:    4
+# CHECK-NEXT: uOps Per Cycle:    1.00
+# CHECK-NEXT: IPC:               0.50
+# CHECK-NEXT: Block RThroughput: 1.0
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  2      12    0.50                        vpinsrq	$0, %rax, %xmm0, %xmm0
+# CHECK-NEXT:  2      12    0.50                        vpinsrq	$1, %rax, %xmm0, %xmm0
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0.0] - PdAGLU01
+# CHECK-NEXT: [0.1] - PdAGLU01
+# CHECK-NEXT: [1]   - PdBranch
+# CHECK-NEXT: [2]   - PdCount
+# CHECK-NEXT: [3]   - PdDiv
+# CHECK-NEXT: [4]   - PdEX0
+# CHECK-NEXT: [5]   - PdEX1
+# CHECK-NEXT: [6]   - PdFPCVT
+# CHECK-NEXT: [7.0] - PdFPFMA
+# CHECK-NEXT: [7.1] - PdFPFMA
+# CHECK-NEXT: [8.0] - PdFPMAL
+# CHECK-NEXT: [8.1] - PdFPMAL
+# CHECK-NEXT: [9]   - PdFPMMA
+# CHECK-NEXT: [10]  - PdFPSTO
+# CHECK-NEXT: [11]  - PdFPU0
+# CHECK-NEXT: [12]  - PdFPU1
+# CHECK-NEXT: [13]  - PdFPU2
+# CHECK-NEXT: [14]  - PdFPU3
+# CHECK-NEXT: [15]  - PdFPXBR
+# CHECK-NEXT: [16.0] - PdLoad
+# CHECK-NEXT: [16.1] - PdLoad
+# CHECK-NEXT: [17]  - PdMul
+# CHECK-NEXT: [18]  - PdStore
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0.0]  [0.1]  [1]    [2]    [3]    [4]    [5]    [6]    [7.0]  [7.1]  [8.0]  [8.1]  [9]    [10]   [11]   [12]   [13]   [14]   [15]   [16.0] [16.1] [17]   [18]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00   1.00    -      -     1.00   1.00    -      -      -      -      -      -      -
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0.0]  [0.1]  [1]    [2]    [3]    [4]    [5]    [6]    [7.0]  [7.1]  [8.0]  [8.1]  [9]    [10]   [11]   [12]   [13]   [14]   [15]   [16.0] [16.1] [17]   [18]   Instructions:
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -      -      -      -      -     vpinsrq	$0, %rax, %xmm0, %xmm0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -      -      -      -      -      -     vpinsrq	$1, %rax, %xmm0, %xmm0
diff --git a/test/tools/llvm-mca/X86/BdVer2/int-to-fpu-forwarding-2.s b/test/tools/llvm-mca/X86/BdVer2/int-to-fpu-forwarding-2.s
new file mode 100644
index 0000000000000000000000000000000000000000..2b31da317fb45a5426d02748e2496484ebf9c8ff
--- /dev/null
+++ b/test/tools/llvm-mca/X86/BdVer2/int-to-fpu-forwarding-2.s
@@ -0,0 +1,362 @@
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=x86_64-unknown-unknown -mcpu=bdver2 -iterations=500 < %s | FileCheck %s
+
+# LLVM-MCA-BEGIN
+vcvtsi2ss %ecx, %xmm0, %xmm0
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN
+vcvtsi2sd %ecx, %xmm0, %xmm0
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN
+cvtsi2ss %ecx, %xmm0
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN
+cvtsi2sd %ecx, %xmm0
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN
+movd %ecx, %xmm0
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN
+movq %rcx, %xmm0
+# LLVM-MCA-END
+
+# CHECK:      [0] Code Region
+
+# CHECK:      Iterations:        500
+# CHECK-NEXT: Instructions:      500
+# CHECK-NEXT: Total Cycles:      2003
+# CHECK-NEXT: Total uOps:        1000
+
+# CHECK:      Dispatch Width:    4
+# CHECK-NEXT: uOps Per Cycle:    0.50
+# CHECK-NEXT: IPC:               0.25
+# CHECK-NEXT: Block RThroughput: 1.0
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  2      14    1.00                        vcvtsi2ssl	%ecx, %xmm0, %xmm0
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0.0] - PdAGLU01
+# CHECK-NEXT: [0.1] - PdAGLU01
+# CHECK-NEXT: [1]   - PdBranch
+# CHECK-NEXT: [2]   - PdCount
+# CHECK-NEXT: [3]   - PdDiv
+# CHECK-NEXT: [4]   - PdEX0
+# CHECK-NEXT: [5]   - PdEX1
+# CHECK-NEXT: [6]   - PdFPCVT
+# CHECK-NEXT: [7.0] - PdFPFMA
+# CHECK-NEXT: [7.1] - PdFPFMA
+# CHECK-NEXT: [8.0] - PdFPMAL
+# CHECK-NEXT: [8.1] - PdFPMAL
+# CHECK-NEXT: [9]   - PdFPMMA
+# CHECK-NEXT: [10]  - PdFPSTO
+# CHECK-NEXT: [11]  - PdFPU0
+# CHECK-NEXT: [12]  - PdFPU1
+# CHECK-NEXT: [13]  - PdFPU2
+# CHECK-NEXT: [14]  - PdFPU3
+# CHECK-NEXT: [15]  - PdFPXBR
+# CHECK-NEXT: [16.0] - PdLoad
+# CHECK-NEXT: [16.1] - PdLoad
+# CHECK-NEXT: [17]  - PdMul
+# CHECK-NEXT: [18]  - PdStore
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0.0]  [0.1]  [1]    [2]    [3]    [4]    [5]    [6]    [7.0]  [7.1]  [8.0]  [8.1]  [9]    [10]   [11]   [12]   [13]   [14]   [15]   [16.0] [16.1] [17]   [18]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -      -      -      -      -      -      -
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0.0]  [0.1]  [1]    [2]    [3]    [4]    [5]    [6]    [7.0]  [7.1]  [8.0]  [8.1]  [9]    [10]   [11]   [12]   [13]   [14]   [15]   [16.0] [16.1] [17]   [18]   Instructions:
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -      -      -      -      -      -      -     vcvtsi2ssl	%ecx, %xmm0, %xmm0
+
+# CHECK:      [1] Code Region
+
+# CHECK:      Iterations:        500
+# CHECK-NEXT: Instructions:      500
+# CHECK-NEXT: Total Cycles:      2003
+# CHECK-NEXT: Total uOps:        1000
+
+# CHECK:      Dispatch Width:    4
+# CHECK-NEXT: uOps Per Cycle:    0.50
+# CHECK-NEXT: IPC:               0.25
+# CHECK-NEXT: Block RThroughput: 1.0
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  2      14    1.00                        vcvtsi2sdl	%ecx, %xmm0, %xmm0
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0.0] - PdAGLU01
+# CHECK-NEXT: [0.1] - PdAGLU01
+# CHECK-NEXT: [1]   - PdBranch
+# CHECK-NEXT: [2]   - PdCount
+# CHECK-NEXT: [3]   - PdDiv
+# CHECK-NEXT: [4]   - PdEX0
+# CHECK-NEXT: [5]   - PdEX1
+# CHECK-NEXT: [6]   - PdFPCVT
+# CHECK-NEXT: [7.0] - PdFPFMA
+# CHECK-NEXT: [7.1] - PdFPFMA
+# CHECK-NEXT: [8.0] - PdFPMAL
+# CHECK-NEXT: [8.1] - PdFPMAL
+# CHECK-NEXT: [9]   - PdFPMMA
+# CHECK-NEXT: [10]  - PdFPSTO
+# CHECK-NEXT: [11]  - PdFPU0
+# CHECK-NEXT: [12]  - PdFPU1
+# CHECK-NEXT: [13]  - PdFPU2
+# CHECK-NEXT: [14]  - PdFPU3
+# CHECK-NEXT: [15]  - PdFPXBR
+# CHECK-NEXT: [16.0] - PdLoad
+# CHECK-NEXT: [16.1] - PdLoad
+# CHECK-NEXT: [17]  - PdMul
+# CHECK-NEXT: [18]  - PdStore
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0.0]  [0.1]  [1]    [2]    [3]    [4]    [5]    [6]    [7.0]  [7.1]  [8.0]  [8.1]  [9]    [10]   [11]   [12]   [13]   [14]   [15]   [16.0] [16.1] [17]   [18]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -      -      -      -      -      -      -
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0.0]  [0.1]  [1]    [2]    [3]    [4]    [5]    [6]    [7.0]  [7.1]  [8.0]  [8.1]  [9]    [10]   [11]   [12]   [13]   [14]   [15]   [16.0] [16.1] [17]   [18]   Instructions:
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -      -      -      -      -      -      -     vcvtsi2sdl	%ecx, %xmm0, %xmm0
+
+# CHECK:      [2] Code Region
+
+# CHECK:      Iterations:        500
+# CHECK-NEXT: Instructions:      500
+# CHECK-NEXT: Total Cycles:      506
+# CHECK-NEXT: Total uOps:        1000
+
+# CHECK:      Dispatch Width:    4
+# CHECK-NEXT: uOps Per Cycle:    1.98
+# CHECK-NEXT: IPC:               0.99
+# CHECK-NEXT: Block RThroughput: 1.0
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  2      14    1.00                        cvtsi2ssl	%ecx, %xmm0
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0.0] - PdAGLU01
+# CHECK-NEXT: [0.1] - PdAGLU01
+# CHECK-NEXT: [1]   - PdBranch
+# CHECK-NEXT: [2]   - PdCount
+# CHECK-NEXT: [3]   - PdDiv
+# CHECK-NEXT: [4]   - PdEX0
+# CHECK-NEXT: [5]   - PdEX1
+# CHECK-NEXT: [6]   - PdFPCVT
+# CHECK-NEXT: [7.0] - PdFPFMA
+# CHECK-NEXT: [7.1] - PdFPFMA
+# CHECK-NEXT: [8.0] - PdFPMAL
+# CHECK-NEXT: [8.1] - PdFPMAL
+# CHECK-NEXT: [9]   - PdFPMMA
+# CHECK-NEXT: [10]  - PdFPSTO
+# CHECK-NEXT: [11]  - PdFPU0
+# CHECK-NEXT: [12]  - PdFPU1
+# CHECK-NEXT: [13]  - PdFPU2
+# CHECK-NEXT: [14]  - PdFPU3
+# CHECK-NEXT: [15]  - PdFPXBR
+# CHECK-NEXT: [16.0] - PdLoad
+# CHECK-NEXT: [16.1] - PdLoad
+# CHECK-NEXT: [17]  - PdMul
+# CHECK-NEXT: [18]  - PdStore
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0.0]  [0.1]  [1]    [2]    [3]    [4]    [5]    [6]    [7.0]  [7.1]  [8.0]  [8.1]  [9]    [10]   [11]   [12]   [13]   [14]   [15]   [16.0] [16.1] [17]   [18]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -      -      -      -      -      -      -
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0.0]  [0.1]  [1]    [2]    [3]    [4]    [5]    [6]    [7.0]  [7.1]  [8.0]  [8.1]  [9]    [10]   [11]   [12]   [13]   [14]   [15]   [16.0] [16.1] [17]   [18]   Instructions:
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -      -      -      -      -      -      -     cvtsi2ssl	%ecx, %xmm0
+
+# CHECK:      [3] Code Region
+
+# CHECK:      Iterations:        500
+# CHECK-NEXT: Instructions:      500
+# CHECK-NEXT: Total Cycles:      506
+# CHECK-NEXT: Total uOps:        1000
+
+# CHECK:      Dispatch Width:    4
+# CHECK-NEXT: uOps Per Cycle:    1.98
+# CHECK-NEXT: IPC:               0.99
+# CHECK-NEXT: Block RThroughput: 1.0
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  2      14    1.00                        cvtsi2sdl	%ecx, %xmm0
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0.0] - PdAGLU01
+# CHECK-NEXT: [0.1] - PdAGLU01
+# CHECK-NEXT: [1]   - PdBranch
+# CHECK-NEXT: [2]   - PdCount
+# CHECK-NEXT: [3]   - PdDiv
+# CHECK-NEXT: [4]   - PdEX0
+# CHECK-NEXT: [5]   - PdEX1
+# CHECK-NEXT: [6]   - PdFPCVT
+# CHECK-NEXT: [7.0] - PdFPFMA
+# CHECK-NEXT: [7.1] - PdFPFMA
+# CHECK-NEXT: [8.0] - PdFPMAL
+# CHECK-NEXT: [8.1] - PdFPMAL
+# CHECK-NEXT: [9]   - PdFPMMA
+# CHECK-NEXT: [10]  - PdFPSTO
+# CHECK-NEXT: [11]  - PdFPU0
+# CHECK-NEXT: [12]  - PdFPU1
+# CHECK-NEXT: [13]  - PdFPU2
+# CHECK-NEXT: [14]  - PdFPU3
+# CHECK-NEXT: [15]  - PdFPXBR
+# CHECK-NEXT: [16.0] - PdLoad
+# CHECK-NEXT: [16.1] - PdLoad
+# CHECK-NEXT: [17]  - PdMul
+# CHECK-NEXT: [18]  - PdStore
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0.0]  [0.1]  [1]    [2]    [3]    [4]    [5]    [6]    [7.0]  [7.1]  [8.0]  [8.1]  [9]    [10]   [11]   [12]   [13]   [14]   [15]   [16.0] [16.1] [17]   [18]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -      -      -      -      -      -      -
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0.0]  [0.1]  [1]    [2]    [3]    [4]    [5]    [6]    [7.0]  [7.1]  [8.0]  [8.1]  [9]    [10]   [11]   [12]   [13]   [14]   [15]   [16.0] [16.1] [17]   [18]   Instructions:
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -      -      -      -      -      -      -     cvtsi2sdl	%ecx, %xmm0
+
+# CHECK:      [4] Code Region
+
+# CHECK:      Iterations:        500
+# CHECK-NEXT: Instructions:      500
+# CHECK-NEXT: Total Cycles:      262
+# CHECK-NEXT: Total uOps:        1000
+
+# CHECK:      Dispatch Width:    4
+# CHECK-NEXT: uOps Per Cycle:    3.82
+# CHECK-NEXT: IPC:               1.91
+# CHECK-NEXT: Block RThroughput: 0.5
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  2      10    0.50                        movd	%ecx, %xmm0
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0.0] - PdAGLU01
+# CHECK-NEXT: [0.1] - PdAGLU01
+# CHECK-NEXT: [1]   - PdBranch
+# CHECK-NEXT: [2]   - PdCount
+# CHECK-NEXT: [3]   - PdDiv
+# CHECK-NEXT: [4]   - PdEX0
+# CHECK-NEXT: [5]   - PdEX1
+# CHECK-NEXT: [6]   - PdFPCVT
+# CHECK-NEXT: [7.0] - PdFPFMA
+# CHECK-NEXT: [7.1] - PdFPFMA
+# CHECK-NEXT: [8.0] - PdFPMAL
+# CHECK-NEXT: [8.1] - PdFPMAL
+# CHECK-NEXT: [9]   - PdFPMMA
+# CHECK-NEXT: [10]  - PdFPSTO
+# CHECK-NEXT: [11]  - PdFPU0
+# CHECK-NEXT: [12]  - PdFPU1
+# CHECK-NEXT: [13]  - PdFPU2
+# CHECK-NEXT: [14]  - PdFPU3
+# CHECK-NEXT: [15]  - PdFPXBR
+# CHECK-NEXT: [16.0] - PdLoad
+# CHECK-NEXT: [16.1] - PdLoad
+# CHECK-NEXT: [17]  - PdMul
+# CHECK-NEXT: [18]  - PdStore
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0.0]  [0.1]  [1]    [2]    [3]    [4]    [5]    [6]    [7.0]  [7.1]  [8.0]  [8.1]  [9]    [10]   [11]   [12]   [13]   [14]   [15]   [16.0] [16.1] [17]   [18]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -      -      -      -      -      -
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0.0]  [0.1]  [1]    [2]    [3]    [4]    [5]    [6]    [7.0]  [7.1]  [8.0]  [8.1]  [9]    [10]   [11]   [12]   [13]   [14]   [15]   [16.0] [16.1] [17]   [18]   Instructions:
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     movd	%ecx, %xmm0
+
+# CHECK:      [5] Code Region
+
+# CHECK:      Iterations:        500
+# CHECK-NEXT: Instructions:      500
+# CHECK-NEXT: Total Cycles:      262
+# CHECK-NEXT: Total uOps:        1000
+
+# CHECK:      Dispatch Width:    4
+# CHECK-NEXT: uOps Per Cycle:    3.82
+# CHECK-NEXT: IPC:               1.91
+# CHECK-NEXT: Block RThroughput: 0.5
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  2      10    0.50                        movq	%rcx, %xmm0
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0.0] - PdAGLU01
+# CHECK-NEXT: [0.1] - PdAGLU01
+# CHECK-NEXT: [1]   - PdBranch
+# CHECK-NEXT: [2]   - PdCount
+# CHECK-NEXT: [3]   - PdDiv
+# CHECK-NEXT: [4]   - PdEX0
+# CHECK-NEXT: [5]   - PdEX1
+# CHECK-NEXT: [6]   - PdFPCVT
+# CHECK-NEXT: [7.0] - PdFPFMA
+# CHECK-NEXT: [7.1] - PdFPFMA
+# CHECK-NEXT: [8.0] - PdFPMAL
+# CHECK-NEXT: [8.1] - PdFPMAL
+# CHECK-NEXT: [9]   - PdFPMMA
+# CHECK-NEXT: [10]  - PdFPSTO
+# CHECK-NEXT: [11]  - PdFPU0
+# CHECK-NEXT: [12]  - PdFPU1
+# CHECK-NEXT: [13]  - PdFPU2
+# CHECK-NEXT: [14]  - PdFPU3
+# CHECK-NEXT: [15]  - PdFPXBR
+# CHECK-NEXT: [16.0] - PdLoad
+# CHECK-NEXT: [16.1] - PdLoad
+# CHECK-NEXT: [17]  - PdMul
+# CHECK-NEXT: [18]  - PdStore
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0.0]  [0.1]  [1]    [2]    [3]    [4]    [5]    [6]    [7.0]  [7.1]  [8.0]  [8.1]  [9]    [10]   [11]   [12]   [13]   [14]   [15]   [16.0] [16.1] [17]   [18]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -      -      -      -      -      -
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0.0]  [0.1]  [1]    [2]    [3]    [4]    [5]    [6]    [7.0]  [7.1]  [8.0]  [8.1]  [9]    [10]   [11]   [12]   [13]   [14]   [15]   [16.0] [16.1] [17]   [18]   Instructions:
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     movq	%rcx, %xmm0
diff --git a/test/tools/llvm-mca/X86/BdVer2/int-to-fpu-forwarding-3.s b/test/tools/llvm-mca/X86/BdVer2/int-to-fpu-forwarding-3.s
new file mode 100644
index 0000000000000000000000000000000000000000..24ac77d5c930a847cb7b2111b558bc6aa46a18fd
--- /dev/null
+++ b/test/tools/llvm-mca/X86/BdVer2/int-to-fpu-forwarding-3.s
@@ -0,0 +1,89 @@
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=x86_64-unknown-unknown -mcpu=bdver2 -iterations=500 -timeline -timeline-max-iterations=3 < %s | FileCheck %s
+
+add %eax, %eax
+vpinsrb $0, %eax, %xmm0, %xmm0
+vpinsrb $1, %eax, %xmm0, %xmm0
+
+# CHECK:      Iterations:        500
+# CHECK-NEXT: Instructions:      1500
+# CHECK-NEXT: Total Cycles:      2014
+# CHECK-NEXT: Total uOps:        2500
+
+# CHECK:      Dispatch Width:    4
+# CHECK-NEXT: uOps Per Cycle:    1.24
+# CHECK-NEXT: IPC:               0.74
+# CHECK-NEXT: Block RThroughput: 1.3
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  1      1     0.50                        addl	%eax, %eax
+# CHECK-NEXT:  2      12    0.50                        vpinsrb	$0, %eax, %xmm0, %xmm0
+# CHECK-NEXT:  2      12    0.50                        vpinsrb	$1, %eax, %xmm0, %xmm0
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0.0] - PdAGLU01
+# CHECK-NEXT: [0.1] - PdAGLU01
+# CHECK-NEXT: [1]   - PdBranch
+# CHECK-NEXT: [2]   - PdCount
+# CHECK-NEXT: [3]   - PdDiv
+# CHECK-NEXT: [4]   - PdEX0
+# CHECK-NEXT: [5]   - PdEX1
+# CHECK-NEXT: [6]   - PdFPCVT
+# CHECK-NEXT: [7.0] - PdFPFMA
+# CHECK-NEXT: [7.1] - PdFPFMA
+# CHECK-NEXT: [8.0] - PdFPMAL
+# CHECK-NEXT: [8.1] - PdFPMAL
+# CHECK-NEXT: [9]   - PdFPMMA
+# CHECK-NEXT: [10]  - PdFPSTO
+# CHECK-NEXT: [11]  - PdFPU0
+# CHECK-NEXT: [12]  - PdFPU1
+# CHECK-NEXT: [13]  - PdFPU2
+# CHECK-NEXT: [14]  - PdFPU3
+# CHECK-NEXT: [15]  - PdFPXBR
+# CHECK-NEXT: [16.0] - PdLoad
+# CHECK-NEXT: [16.1] - PdLoad
+# CHECK-NEXT: [17]  - PdMul
+# CHECK-NEXT: [18]  - PdStore
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0.0]  [0.1]  [1]    [2]    [3]    [4]    [5]    [6]    [7.0]  [7.1]  [8.0]  [8.1]  [9]    [10]   [11]   [12]   [13]   [14]   [15]   [16.0] [16.1] [17]   [18]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -     1.00   1.00    -      -     1.00   1.00    -      -      -      -      -      -      -
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0.0]  [0.1]  [1]    [2]    [3]    [4]    [5]    [6]    [7.0]  [7.1]  [8.0]  [8.1]  [9]    [10]   [11]   [12]   [13]   [14]   [15]   [16.0] [16.1] [17]   [18]   Instructions:
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     addl	%eax, %eax
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -      -      -      -      -     vpinsrb	$0, %eax, %xmm0, %xmm0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -      -      -      -      -      -     vpinsrb	$1, %eax, %xmm0, %xmm0
+
+# CHECK:      Timeline view:
+# CHECK-NEXT:                     0123456789
+# CHECK-NEXT: Index     0123456789          012345
+
+# CHECK:      [0,0]     DeER .    .    .    .    .   addl	%eax, %eax
+# CHECK-NEXT: [0,1]     D===========eeER    .    .   vpinsrb	$0, %eax, %xmm0, %xmm0
+# CHECK-NEXT: [0,2]     .D============eeER  .    .   vpinsrb	$1, %eax, %xmm0, %xmm0
+# CHECK-NEXT: [1,0]     .DeE-------------R  .    .   addl	%eax, %eax
+# CHECK-NEXT: [1,1]     . D=============eeER.    .   vpinsrb	$0, %eax, %xmm0, %xmm0
+# CHECK-NEXT: [1,2]     . D===============eeER   .   vpinsrb	$1, %eax, %xmm0, %xmm0
+# CHECK-NEXT: [2,0]     .  DeE---------------R   .   addl	%eax, %eax
+# CHECK-NEXT: [2,1]     .  D================eeER .   vpinsrb	$0, %eax, %xmm0, %xmm0
+# CHECK-NEXT: [2,2]     .   D=================eeER   vpinsrb	$1, %eax, %xmm0, %xmm0
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     3     1.0    0.7    9.3       addl	%eax, %eax
+# CHECK-NEXT: 1.     3     14.3   0.0    0.0       vpinsrb	$0, %eax, %xmm0, %xmm0
+# CHECK-NEXT: 2.     3     15.7   0.0    0.0       vpinsrb	$1, %eax, %xmm0, %xmm0
diff --git a/test/tools/llvm-mca/X86/BdVer2/resources-avx1.s b/test/tools/llvm-mca/X86/BdVer2/resources-avx1.s
index 91d230dda1591d38b1c63a770fa55270281ea87c..5235eab449ca06d1a4cd6adc3b0d0f8e3157c7c6 100644
--- a/test/tools/llvm-mca/X86/BdVer2/resources-avx1.s
+++ b/test/tools/llvm-mca/X86/BdVer2/resources-avx1.s
@@ -1144,12 +1144,12 @@ vzeroupper
 # CHECK-NEXT:  2      18    1.00    *                   vcvtsd2si	(%rax), %rcx
 # CHECK-NEXT:  1      4     1.00                        vcvtsd2ss	%xmm0, %xmm1, %xmm2
 # CHECK-NEXT:  1      9     1.00    *                   vcvtsd2ss	(%rax), %xmm1, %xmm2
-# CHECK-NEXT:  2      4     1.00                        vcvtsi2sdl	%ecx, %xmm0, %xmm2
-# CHECK-NEXT:  2      4     1.00                        vcvtsi2sdq	%rcx, %xmm0, %xmm2
+# CHECK-NEXT:  2      14    1.00                        vcvtsi2sdl	%ecx, %xmm0, %xmm2
+# CHECK-NEXT:  2      14    1.00                        vcvtsi2sdq	%rcx, %xmm0, %xmm2
 # CHECK-NEXT:  2      9     1.00    *                   vcvtsi2sdl	(%rax), %xmm0, %xmm2
 # CHECK-NEXT:  2      9     1.00    *                   vcvtsi2sdq	(%rax), %xmm0, %xmm2
-# CHECK-NEXT:  2      4     1.00                        vcvtsi2ssl	%ecx, %xmm0, %xmm2
-# CHECK-NEXT:  2      4     1.00                        vcvtsi2ssq	%rcx, %xmm0, %xmm2
+# CHECK-NEXT:  2      14    1.00                        vcvtsi2ssl	%ecx, %xmm0, %xmm2
+# CHECK-NEXT:  2      14    1.00                        vcvtsi2ssq	%rcx, %xmm0, %xmm2
 # CHECK-NEXT:  2      9     1.00    *                   vcvtsi2ssl	(%rax), %xmm0, %xmm2
 # CHECK-NEXT:  2      9     1.00    *                   vcvtsi2ssq	(%rax), %xmm0, %xmm2
 # CHECK-NEXT:  1      4     1.00                        vcvtss2sd	%xmm0, %xmm1, %xmm2
@@ -1469,13 +1469,13 @@ vzeroupper
 # CHECK-NEXT:  4      10    0.50    *                   vphsubsw	(%rax), %xmm1, %xmm2
 # CHECK-NEXT:  3      5     0.50                        vphsubw	%xmm0, %xmm1, %xmm2
 # CHECK-NEXT:  4      10    0.50    *                   vphsubw	(%rax), %xmm1, %xmm2
-# CHECK-NEXT:  2      2     0.50                        vpinsrb	$1, %eax, %xmm1, %xmm2
+# CHECK-NEXT:  2      12    0.50                        vpinsrb	$1, %eax, %xmm1, %xmm2
 # CHECK-NEXT:  2      6     0.50    *                   vpinsrb	$1, (%rax), %xmm1, %xmm2
-# CHECK-NEXT:  2      2     0.50                        vpinsrd	$1, %eax, %xmm1, %xmm2
+# CHECK-NEXT:  2      12    0.50                        vpinsrd	$1, %eax, %xmm1, %xmm2
 # CHECK-NEXT:  2      6     0.50    *                   vpinsrd	$1, (%rax), %xmm1, %xmm2
-# CHECK-NEXT:  2      2     0.50                        vpinsrq	$1, %rax, %xmm1, %xmm2
+# CHECK-NEXT:  2      12    0.50                        vpinsrq	$1, %rax, %xmm1, %xmm2
 # CHECK-NEXT:  2      6     0.50    *                   vpinsrq	$1, (%rax), %xmm1, %xmm2
-# CHECK-NEXT:  2      2     0.50                        vpinsrw	$1, %eax, %xmm1, %xmm2
+# CHECK-NEXT:  2      12    0.50                        vpinsrw	$1, %eax, %xmm1, %xmm2
 # CHECK-NEXT:  2      6     0.50    *                   vpinsrw	$1, (%rax), %xmm1, %xmm2
 # CHECK-NEXT:  1      4     1.00                        vpmaddubsw	%xmm0, %xmm1, %xmm2
 # CHECK-NEXT:  1      9     1.00    *                   vpmaddubsw	(%rax), %xmm1, %xmm2
diff --git a/test/tools/llvm-mca/X86/BdVer2/resources-bmi1.s b/test/tools/llvm-mca/X86/BdVer2/resources-bmi1.s
index c67d73cd43e3a40d014a317b30e234890077c74b..34f4e6ced7f45a3e8c855266355228821887399a 100644
--- a/test/tools/llvm-mca/X86/BdVer2/resources-bmi1.s
+++ b/test/tools/llvm-mca/X86/BdVer2/resources-bmi1.s
@@ -31,6 +31,9 @@ blsr        (%rax), %ecx
 blsr        %rax, %rcx
 blsr        (%rax), %rcx
 
+tzcnt       %ax, %cx
+tzcnt       (%rax), %cx
+
 tzcnt       %eax, %ecx
 tzcnt       (%rax), %ecx
 
@@ -66,6 +69,8 @@ tzcnt       (%rax), %rcx
 # CHECK-NEXT:  2      6     0.50    *                   blsrl	(%rax), %ecx
 # CHECK-NEXT:  2      2     0.50                        blsrq	%rax, %rcx
 # CHECK-NEXT:  2      6     0.50    *                   blsrq	(%rax), %rcx
+# CHECK-NEXT:  2      2     1.00                        tzcntw	%ax, %cx
+# CHECK-NEXT:  2      6     1.00    *                   tzcntw	(%rax), %cx
 # CHECK-NEXT:  2      2     1.00                        tzcntl	%eax, %ecx
 # CHECK-NEXT:  2      6     1.00    *                   tzcntl	(%rax), %ecx
 # CHECK-NEXT:  2      2     1.00                        tzcntq	%rax, %rcx
@@ -98,7 +103,7 @@ tzcnt       (%rax), %rcx
 
 # CHECK:      Resource pressure per iteration:
 # CHECK-NEXT: [0.0]  [0.1]  [1]    [2]    [3]    [4]    [5]    [6]    [7.0]  [7.1]  [8.0]  [8.1]  [9]    [10]   [11]   [12]   [13]   [14]   [15]   [16.0] [16.1] [17]   [18]
-# CHECK-NEXT: 6.00   6.00    -      -      -     14.00  14.00   -      -      -      -      -      -      -      -      -      -      -      -     6.00   6.00    -      -
+# CHECK-NEXT: 6.50   6.50    -      -      -     16.00  16.00   -      -      -      -      -      -      -      -      -      -      -      -     6.50   6.50    -      -
 
 # CHECK:      Resource pressure by instruction:
 # CHECK-NEXT: [0.0]  [0.1]  [1]    [2]    [3]    [4]    [5]    [6]    [7.0]  [7.1]  [8.0]  [8.1]  [9]    [10]   [11]   [12]   [13]   [14]   [15]   [16.0] [16.1] [17]   [18]   Instructions:
@@ -122,6 +127,8 @@ tzcnt       (%rax), %rcx
 # CHECK-NEXT: 0.50   0.50    -      -      -     0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     blsrl	(%rax), %ecx
 # CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     blsrq	%rax, %rcx
 # CHECK-NEXT: 0.50   0.50    -      -      -     0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     blsrq	(%rax), %rcx
+# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     tzcntw	%ax, %cx
+# CHECK-NEXT: 0.50   0.50    -      -      -     1.00   1.00    -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     tzcntw	(%rax), %cx
 # CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     tzcntl	%eax, %ecx
 # CHECK-NEXT: 0.50   0.50    -      -      -     1.00   1.00    -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     tzcntl	(%rax), %ecx
 # CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     tzcntq	%rax, %rcx
diff --git a/test/tools/llvm-mca/X86/BdVer2/resources-lwp.s b/test/tools/llvm-mca/X86/BdVer2/resources-lwp.s
new file mode 100644
index 0000000000000000000000000000000000000000..b14a72d32046f526ad8e401e80beec8999de0099
--- /dev/null
+++ b/test/tools/llvm-mca/X86/BdVer2/resources-lwp.s
@@ -0,0 +1,86 @@
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=x86_64-unknown-unknown -mcpu=bdver2 -instruction-tables < %s | FileCheck %s
+
+llwpcb %edi
+llwpcb %rdi
+
+lwpins $-1985229329, %esi, %edi
+lwpins $-1985229329, (%rsi), %edi
+
+lwpins $-1985229329, %esi, %rdi
+lwpins $-1985229329, (%rsi), %rdi
+
+lwpval $-1985229329, %esi, %edi
+lwpval $-1985229329, (%rsi), %edi
+
+lwpval $-1985229329, %esi, %rdi
+lwpval $-1985229329, (%rsi), %rdi
+
+slwpcb %edi
+slwpcb %rdi
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  1      100   0.50    *      *      U     llwpcb	%edi
+# CHECK-NEXT:  1      100   0.50    *      *      U     llwpcb	%rdi
+# CHECK-NEXT:  1      100   0.50    *      *      U     lwpins	$-1985229329, %esi, %edi
+# CHECK-NEXT:  1      100   0.50    *      *      U     lwpins	$-1985229329, (%rsi), %edi
+# CHECK-NEXT:  1      100   0.50    *      *      U     lwpins	$-1985229329, %esi, %rdi
+# CHECK-NEXT:  1      100   0.50    *      *      U     lwpins	$-1985229329, (%rsi), %rdi
+# CHECK-NEXT:  1      100   0.50    *      *      U     lwpval	$-1985229329, %esi, %edi
+# CHECK-NEXT:  1      100   0.50    *      *      U     lwpval	$-1985229329, (%rsi), %edi
+# CHECK-NEXT:  1      100   0.50    *      *      U     lwpval	$-1985229329, %esi, %rdi
+# CHECK-NEXT:  1      100   0.50    *      *      U     lwpval	$-1985229329, (%rsi), %rdi
+# CHECK-NEXT:  1      100   0.50    *      *      U     slwpcb	%edi
+# CHECK-NEXT:  1      100   0.50    *      *      U     slwpcb	%rdi
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0.0] - PdAGLU01
+# CHECK-NEXT: [0.1] - PdAGLU01
+# CHECK-NEXT: [1]   - PdBranch
+# CHECK-NEXT: [2]   - PdCount
+# CHECK-NEXT: [3]   - PdDiv
+# CHECK-NEXT: [4]   - PdEX0
+# CHECK-NEXT: [5]   - PdEX1
+# CHECK-NEXT: [6]   - PdFPCVT
+# CHECK-NEXT: [7.0] - PdFPFMA
+# CHECK-NEXT: [7.1] - PdFPFMA
+# CHECK-NEXT: [8.0] - PdFPMAL
+# CHECK-NEXT: [8.1] - PdFPMAL
+# CHECK-NEXT: [9]   - PdFPMMA
+# CHECK-NEXT: [10]  - PdFPSTO
+# CHECK-NEXT: [11]  - PdFPU0
+# CHECK-NEXT: [12]  - PdFPU1
+# CHECK-NEXT: [13]  - PdFPU2
+# CHECK-NEXT: [14]  - PdFPU3
+# CHECK-NEXT: [15]  - PdFPXBR
+# CHECK-NEXT: [16.0] - PdLoad
+# CHECK-NEXT: [16.1] - PdLoad
+# CHECK-NEXT: [17]  - PdMul
+# CHECK-NEXT: [18]  - PdStore
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0.0]  [0.1]  [1]    [2]    [3]    [4]    [5]    [6]    [7.0]  [7.1]  [8.0]  [8.1]  [9]    [10]   [11]   [12]   [13]   [14]   [15]   [16.0] [16.1] [17]   [18]
+# CHECK-NEXT:  -      -      -      -      -     6.00   6.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0.0]  [0.1]  [1]    [2]    [3]    [4]    [5]    [6]    [7.0]  [7.1]  [8.0]  [8.1]  [9]    [10]   [11]   [12]   [13]   [14]   [15]   [16.0] [16.1] [17]   [18]   Instructions:
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     llwpcb	%edi
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     llwpcb	%rdi
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     lwpins	$-1985229329, %esi, %edi
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     lwpins	$-1985229329, (%rsi), %edi
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     lwpins	$-1985229329, %esi, %rdi
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     lwpins	$-1985229329, (%rsi), %rdi
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     lwpval	$-1985229329, %esi, %edi
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     lwpval	$-1985229329, (%rsi), %edi
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     lwpval	$-1985229329, %esi, %rdi
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     lwpval	$-1985229329, (%rsi), %rdi
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     slwpcb	%edi
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     slwpcb	%rdi
diff --git a/test/tools/llvm-mca/X86/BdVer2/resources-sse1.s b/test/tools/llvm-mca/X86/BdVer2/resources-sse1.s
index 7fcb7fb49f4155f3f161302ce79ca015eca9c6a8..8916504b18ea9bc015e9fb5402cb256b5b9b7d88 100644
--- a/test/tools/llvm-mca/X86/BdVer2/resources-sse1.s
+++ b/test/tools/llvm-mca/X86/BdVer2/resources-sse1.s
@@ -212,7 +212,7 @@ xorps       (%rax), %xmm2
 # CHECK-NEXT:  1      9     1.00    *                   cvtpi2ps	(%rax), %xmm2
 # CHECK-NEXT:  1      4     1.00                        cvtps2pi	%xmm0, %mm2
 # CHECK-NEXT:  1      9     1.00    *                   cvtps2pi	(%rax), %mm2
-# CHECK-NEXT:  2      4     1.00                        cvtsi2ssl	%ecx, %xmm2
+# CHECK-NEXT:  2      14    1.00                        cvtsi2ssl	%ecx, %xmm2
 # CHECK-NEXT:  2      13    1.00                        cvtsi2ssq	%rcx, %xmm2
 # CHECK-NEXT:  2      9     1.00    *                   cvtsi2ssl	(%rax), %xmm2
 # CHECK-NEXT:  2      9     1.00    *                   cvtsi2ssl	(%rax), %xmm2
@@ -269,7 +269,7 @@ xorps       (%rax), %xmm2
 # CHECK-NEXT:  1      2     0.50                        pavgw	%mm0, %mm2
 # CHECK-NEXT:  1      7     0.50    *                   pavgw	(%rax), %mm2
 # CHECK-NEXT:  2      13    1.00                        pextrw	$1, %mm0, %ecx
-# CHECK-NEXT:  2      2     0.50                        pinsrw	$1, %eax, %mm2
+# CHECK-NEXT:  2      12    0.50                        pinsrw	$1, %eax, %mm2
 # CHECK-NEXT:  2      6     0.50    *                   pinsrw	$1, (%rax), %mm2
 # CHECK-NEXT:  1      2     0.50                        pmaxsw	%mm0, %mm2
 # CHECK-NEXT:  1      7     0.50    *                   pmaxsw	(%rax), %mm2
diff --git a/test/tools/llvm-mca/X86/BdVer2/resources-sse2.s b/test/tools/llvm-mca/X86/BdVer2/resources-sse2.s
index 84bf65664c86bb97905134f115e71cb3907f3066..093a6be03592c13c173ffa51cd89c59b109d0c2f 100644
--- a/test/tools/llvm-mca/X86/BdVer2/resources-sse2.s
+++ b/test/tools/llvm-mca/X86/BdVer2/resources-sse2.s
@@ -94,6 +94,8 @@ maxpd       (%rax), %xmm2
 maxsd       %xmm0, %xmm2
 maxsd       (%rax), %xmm2
 
+mfence
+
 minpd       %xmm0, %xmm2
 minpd       (%rax), %xmm2
 
@@ -229,6 +231,9 @@ pcmpgtw     (%rax), %xmm2
 
 pextrw      $1, %xmm0, %rcx
 
+pinsrw      $1, %rax, %xmm0
+pinsrw      $1, (%rax), %xmm0
+
 pmaddwd     %xmm0, %xmm2
 pmaddwd     (%rax), %xmm2
 
@@ -439,7 +444,7 @@ xorpd       (%rax), %xmm2
 # CHECK-NEXT:  2      18    1.00    *                   cvtsd2si	(%rax), %rcx
 # CHECK-NEXT:  1      4     1.00                        cvtsd2ss	%xmm0, %xmm2
 # CHECK-NEXT:  1      9     1.00    *                   cvtsd2ss	(%rax), %xmm2
-# CHECK-NEXT:  2      4     1.00                        cvtsi2sdl	%ecx, %xmm2
+# CHECK-NEXT:  2      14    1.00                        cvtsi2sdl	%ecx, %xmm2
 # CHECK-NEXT:  2      13    1.00                        cvtsi2sdq	%rcx, %xmm2
 # CHECK-NEXT:  2      9     1.00    *                   cvtsi2sdl	(%rax), %xmm2
 # CHECK-NEXT:  2      9     1.00    *                   cvtsi2sdl	(%rax), %xmm2
@@ -465,6 +470,7 @@ xorpd       (%rax), %xmm2
 # CHECK-NEXT:  1      7     1.00    *                   maxpd	(%rax), %xmm2
 # CHECK-NEXT:  1      2     1.00                        maxsd	%xmm0, %xmm2
 # CHECK-NEXT:  1      7     1.00    *                   maxsd	(%rax), %xmm2
+# CHECK-NEXT:  1      1     1.00    *      *      U     mfence
 # CHECK-NEXT:  1      2     1.00                        minpd	%xmm0, %xmm2
 # CHECK-NEXT:  1      7     1.00    *                   minpd	(%rax), %xmm2
 # CHECK-NEXT:  1      2     1.00                        minsd	%xmm0, %xmm2
@@ -555,6 +561,8 @@ xorpd       (%rax), %xmm2
 # CHECK-NEXT:  1      2     0.50                        pcmpgtw	%xmm0, %xmm2
 # CHECK-NEXT:  1      7     0.50    *                   pcmpgtw	(%rax), %xmm2
 # CHECK-NEXT:  2      13    1.00                        pextrw	$1, %xmm0, %ecx
+# CHECK-NEXT:  2      12    0.50                        pinsrw	$1, %eax, %xmm0
+# CHECK-NEXT:  2      6     0.50    *                   pinsrw	$1, (%rax), %xmm0
 # CHECK-NEXT:  1      4     1.00                        pmaddwd	%xmm0, %xmm2
 # CHECK-NEXT:  1      9     1.00    *                   pmaddwd	(%rax), %xmm2
 # CHECK-NEXT:  1      2     0.50                        pmaxsw	%xmm0, %xmm2
@@ -694,7 +702,7 @@ xorpd       (%rax), %xmm2
 
 # CHECK:      Resource pressure per iteration:
 # CHECK-NEXT: [0.0]  [0.1]  [1]    [2]    [3]    [4]    [5]    [6]    [7.0]  [7.1]  [8.0]  [8.1]  [9]    [10]   [11]   [12]   [13]   [14]   [15]   [16.0] [16.1] [17]   [18]
-# CHECK-NEXT: 66.00  66.00   -      -      -     17.00   -      -     124.50 124.50 66.50  66.50  12.00  50.00  119.50 140.50  -      -      -     58.50  58.50   -     15.00
+# CHECK-NEXT: 67.00  67.00   -      -      -     17.00   -      -     124.50 124.50 67.50  67.50  12.00  50.00  120.50 141.50  -      -      -     59.00  59.00   -     16.00
 
 # CHECK:      Resource pressure by instruction:
 # CHECK-NEXT: [0.0]  [0.1]  [1]    [2]    [3]    [4]    [5]    [6]    [7.0]  [7.1]  [8.0]  [8.1]  [9]    [10]   [11]   [12]   [13]   [14]   [15]   [16.0] [16.1] [17]   [18]   Instructions:
@@ -761,6 +769,7 @@ xorpd       (%rax), %xmm2
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -      -      -     1.00    -      -      -      -     0.50   0.50    -      -     maxpd	(%rax), %xmm2
 # CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     1.00    -      -      -      -      -      -      -      -     maxsd	%xmm0, %xmm2
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -      -      -     1.00    -      -      -      -     0.50   0.50    -      -     maxsd	(%rax), %xmm2
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   mfence
 # CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     1.00    -      -      -      -      -      -      -      -     minpd	%xmm0, %xmm2
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -      -      -     1.00    -      -      -      -     0.50   0.50    -      -     minpd	(%rax), %xmm2
 # CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     1.00    -      -      -      -      -      -      -      -     minsd	%xmm0, %xmm2
@@ -851,6 +860,8 @@ xorpd       (%rax), %xmm2
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     0.50   0.50    -      -      -      -      -      -      -     pcmpgtw	%xmm0, %xmm2
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -     0.50   0.50    -      -     0.50   0.50    -      -      -     0.50   0.50    -      -     pcmpgtw	(%rax), %xmm2
 # CHECK-NEXT:  -      -      -      -      -     1.00    -      -     0.50   0.50    -      -      -      -     1.00    -      -      -      -      -      -      -      -     pextrw	$1, %xmm0, %ecx
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     0.50   0.50    -      -      -      -      -      -      -     pinsrw	$1, %eax, %xmm0
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -     0.50   0.50    -      -     0.50   0.50    -      -      -     0.50   0.50    -      -     pinsrw	$1, (%rax), %xmm0
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -      -      -      -      -      -      -      -     pmaddwd	%xmm0, %xmm2
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -      -      -      -     0.50   0.50    -      -     pmaddwd	(%rax), %xmm2
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     0.50   0.50    -      -      -      -      -      -      -     pmaxsw	%xmm0, %xmm2
diff --git a/test/tools/llvm-mca/X86/BdVer2/resources-sse3.s b/test/tools/llvm-mca/X86/BdVer2/resources-sse3.s
index aedbd80c033e039adea444b2936668ce74c21417..a50f2cb3699f12f214333e5d5cdde6757faa6699 100644
--- a/test/tools/llvm-mca/X86/BdVer2/resources-sse3.s
+++ b/test/tools/llvm-mca/X86/BdVer2/resources-sse3.s
@@ -21,6 +21,8 @@ hsubps    (%rax), %xmm2
 
 lddqu     (%rax), %xmm2
 
+monitor
+
 movddup   %xmm0, %xmm2
 movddup   (%rax), %xmm2
 
@@ -30,6 +32,8 @@ movshdup  (%rax), %xmm2
 movsldup  %xmm0, %xmm2
 movsldup  (%rax), %xmm2
 
+mwait
+
 # CHECK:      Instruction Info:
 # CHECK-NEXT: [1]: #uOps
 # CHECK-NEXT: [2]: Latency
@@ -52,12 +56,14 @@ movsldup  (%rax), %xmm2
 # CHECK-NEXT:  3      11    1.00                        hsubps	%xmm0, %xmm2
 # CHECK-NEXT:  4      16    1.00    *                   hsubps	(%rax), %xmm2
 # CHECK-NEXT:  1      5     0.50    *                   lddqu	(%rax), %xmm2
+# CHECK-NEXT:  1      100   0.50                  U     monitor
 # CHECK-NEXT:  1      2     0.50                        movddup	%xmm0, %xmm2
 # CHECK-NEXT:  1      7     0.50    *                   movddup	(%rax), %xmm2
 # CHECK-NEXT:  1      2     0.50                        movshdup	%xmm0, %xmm2
 # CHECK-NEXT:  1      7     0.50    *                   movshdup	(%rax), %xmm2
 # CHECK-NEXT:  1      2     0.50                        movsldup	%xmm0, %xmm2
 # CHECK-NEXT:  1      7     0.50    *                   movsldup	(%rax), %xmm2
+# CHECK-NEXT:  1      100   0.50    *      *      U     mwait
 
 # CHECK:      Resources:
 # CHECK-NEXT: [0.0] - PdAGLU01
@@ -86,7 +92,7 @@ movsldup  (%rax), %xmm2
 
 # CHECK:      Resource pressure per iteration:
 # CHECK-NEXT: [0.0]  [0.1]  [1]    [2]    [3]    [4]    [5]    [6]    [7.0]  [7.1]  [8.0]  [8.1]  [9]    [10]   [11]   [12]   [13]   [14]   [15]   [16.0] [16.1] [17]   [18]
-# CHECK-NEXT: 5.00   5.00    -      -      -      -      -      -     9.00   9.00   0.50   0.50    -      -     15.50  3.50    -      -      -     5.00   5.00    -      -
+# CHECK-NEXT: 5.00   5.00    -      -      -     1.00   1.00    -     9.00   9.00   0.50   0.50    -      -     15.50  3.50    -      -      -     5.00   5.00    -      -
 
 # CHECK:      Resource pressure by instruction:
 # CHECK-NEXT: [0.0]  [0.1]  [1]    [2]    [3]    [4]    [5]    [6]    [7.0]  [7.1]  [8.0]  [8.1]  [9]    [10]   [11]   [12]   [13]   [14]   [15]   [16.0] [16.1] [17]   [18]   Instructions:
@@ -103,9 +109,11 @@ movsldup  (%rax), %xmm2
 # CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     1.00    -      -      -      -      -      -      -      -     hsubps	%xmm0, %xmm2
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -      -      -     1.00    -      -      -      -     0.50   0.50    -      -     hsubps	(%rax), %xmm2
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -     0.50   0.50    -      -     0.50   0.50    -      -      -     0.50   0.50    -      -     lddqu	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     monitor
 # CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     movddup	%xmm0, %xmm2
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -      -     0.50   0.50    -      -     movddup	(%rax), %xmm2
 # CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     movshdup	%xmm0, %xmm2
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -      -     0.50   0.50    -      -     movshdup	(%rax), %xmm2
 # CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     movsldup	%xmm0, %xmm2
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -      -      -     0.50   0.50    -      -      -     0.50   0.50    -      -     movsldup	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     mwait
diff --git a/test/tools/llvm-mca/X86/BdVer2/resources-sse41.s b/test/tools/llvm-mca/X86/BdVer2/resources-sse41.s
index 4aa29de694e014087dda774575df80cca3776957..1e919a58d1c5343111864242a60bb0cc54f54907 100644
--- a/test/tools/llvm-mca/X86/BdVer2/resources-sse41.s
+++ b/test/tools/llvm-mca/X86/BdVer2/resources-sse41.s
@@ -191,11 +191,11 @@ roundss     $1, (%rax), %xmm2
 # CHECK-NEXT:  2      13    1.00           *            pextrw	$1, %xmm0, (%rax)
 # CHECK-NEXT:  2      4     1.00                        phminposuw	%xmm0, %xmm2
 # CHECK-NEXT:  2      9     1.00    *                   phminposuw	(%rax), %xmm2
-# CHECK-NEXT:  2      2     0.50                        pinsrb	$1, %eax, %xmm1
+# CHECK-NEXT:  2      12    0.50                        pinsrb	$1, %eax, %xmm1
 # CHECK-NEXT:  2      6     0.50    *                   pinsrb	$1, (%rax), %xmm1
-# CHECK-NEXT:  2      2     0.50                        pinsrd	$1, %eax, %xmm1
+# CHECK-NEXT:  2      12    0.50                        pinsrd	$1, %eax, %xmm1
 # CHECK-NEXT:  2      6     0.50    *                   pinsrd	$1, (%rax), %xmm1
-# CHECK-NEXT:  2      2     0.50                        pinsrq	$1, %rax, %xmm1
+# CHECK-NEXT:  2      12    0.50                        pinsrq	$1, %rax, %xmm1
 # CHECK-NEXT:  2      6     0.50    *                   pinsrq	$1, (%rax), %xmm1
 # CHECK-NEXT:  1      2     0.50                        pmaxsb	%xmm0, %xmm2
 # CHECK-NEXT:  1      7     0.50    *                   pmaxsb	(%rax), %xmm2
diff --git a/test/tools/llvm-mca/X86/BdVer2/resources-x86_64.s b/test/tools/llvm-mca/X86/BdVer2/resources-x86_64.s
index 84eec15a04923351c410e69b08989adaa6e64f01..cf53596018800f774e8b61e4b4d3995ed869eaa9 100644
--- a/test/tools/llvm-mca/X86/BdVer2/resources-x86_64.s
+++ b/test/tools/llvm-mca/X86/BdVer2/resources-x86_64.s
@@ -254,6 +254,8 @@ divl (%rax)
 divq %rcx
 divq (%rax)
 
+enter $7, $4095
+
 idivb %dil
 idivb (%rax)
 idivw %si
@@ -315,8 +317,13 @@ insl
 
 int $7
 
+invlpg  (%rax)
+invlpga %rax, %ecx
+
 lahf
 
+leave
+
 lodsb
 lodsw
 lodsl
@@ -486,6 +493,11 @@ rcrq %cl, %rdi
 rclq %cl, (%rax)
 rcrq %cl, (%rax)
 
+rdmsr
+rdpmc
+rdtsc
+rdtscp
+
 rolb %dil
 rorb %dil
 rolb (%rax)
@@ -789,6 +801,8 @@ testq %rsi, (%rax)
 
 ud2
 
+wrmsr
+
 xaddb %bl, %cl
 xaddb %bl, (%rcx)
 
@@ -1081,6 +1095,7 @@ xorq (%rax), %rdi
 # CHECK-NEXT:  2      18    14.00   *             U     divl	(%rax)
 # CHECK-NEXT:  2      14    14.00                 U     divq	%rcx
 # CHECK-NEXT:  2      18    14.00   *             U     divq	(%rax)
+# CHECK-NEXT:  1      100   0.50                  U     enter	$7, $4095
 # CHECK-NEXT:  1      12    12.00                 U     idivb	%dil
 # CHECK-NEXT:  1      16    12.00   *             U     idivb	(%rax)
 # CHECK-NEXT:  2      15    17.00                 U     idivw	%si
@@ -1133,7 +1148,10 @@ xorq (%rax), %rdi
 # CHECK-NEXT:  1      100   0.50                  U     insw	%dx, %es:(%rdi)
 # CHECK-NEXT:  1      100   0.50                  U     insl	%dx, %es:(%rdi)
 # CHECK-NEXT:  1      100   0.50    *      *      U     int	$7
+# CHECK-NEXT:  1      100   0.50                  U     invlpg	(%rax)
+# CHECK-NEXT:  1      100   0.50                  U     invlpga	%rax, %ecx
 # CHECK-NEXT:  4      2     0.50                        lahf
+# CHECK-NEXT:  1      1     0.50    *                   leave
 # CHECK-NEXT:  1      100   0.50                  U     lodsb	(%rsi), %al
 # CHECK-NEXT:  1      100   0.50                  U     lodsw	(%rsi), %ax
 # CHECK-NEXT:  1      100   0.50                  U     lodsl	(%rsi), %eax
@@ -1283,6 +1301,10 @@ xorq (%rax), %rdi
 # CHECK-NEXT:  16     7     0.50                        rcrq	%cl, %rdi
 # CHECK-NEXT:  2      5     1.00           *            rclq	%cl, (%rax)
 # CHECK-NEXT:  2      5     1.00           *            rcrq	%cl, (%rax)
+# CHECK-NEXT:  1      100   0.50                  U     rdmsr
+# CHECK-NEXT:  1      100   0.50                  U     rdpmc
+# CHECK-NEXT:  1      100   0.50                  U     rdtsc
+# CHECK-NEXT:  1      100   0.50                  U     rdtscp
 # CHECK-NEXT:  1      1     0.50                        rolb	%dil
 # CHECK-NEXT:  1      1     0.50                        rorb	%dil
 # CHECK-NEXT:  2      5     1.00    *      *            rolb	(%rax)
@@ -1557,6 +1579,7 @@ xorq (%rax), %rdi
 # CHECK-NEXT:  1      1     0.50                        testq	%rsi, %rdi
 # CHECK-NEXT:  1      5     0.50    *                   testq	%rsi, (%rax)
 # CHECK-NEXT:  1      100   0.50    *             U     ud2
+# CHECK-NEXT:  1      100   0.50                  U     wrmsr
 # CHECK-NEXT:  4      2     1.00                        xaddb	%bl, %cl
 # CHECK-NEXT:  4      6     1.00    *      *            xaddb	%bl, (%rcx)
 # CHECK-NEXT:  4      2     1.00                        xaddw	%bx, %cx
@@ -1635,7 +1658,7 @@ xorq (%rax), %rdi
 
 # CHECK:      Resource pressure per iteration:
 # CHECK-NEXT: [0.0]  [0.1]  [1]    [2]    [3]    [4]    [5]    [6]    [7.0]  [7.1]  [8.0]  [8.1]  [9]    [10]   [11]   [12]   [13]   [14]   [15]   [16.0] [16.1] [17]   [18]
-# CHECK-NEXT: 262.00 262.00  -      -     246.00 547.50 622.50  -      -      -      -      -      -      -      -      -      -      -      -     156.50 156.50 64.00  211.00
+# CHECK-NEXT: 262.00 262.00  -      -     246.00 552.00 627.00  -      -      -      -      -      -      -      -      -      -      -      -     156.50 156.50 64.00  211.00
 
 # CHECK:      Resource pressure by instruction:
 # CHECK-NEXT: [0.0]  [0.1]  [1]    [2]    [3]    [4]    [5]    [6]    [7.0]  [7.1]  [8.0]  [8.1]  [9]    [10]   [11]   [12]   [13]   [14]   [15]   [16.0] [16.1] [17]   [18]   Instructions:
@@ -1859,6 +1882,7 @@ xorq (%rax), %rdi
 # CHECK-NEXT: 0.50   0.50    -      -     14.00   -     1.00    -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     divl	(%rax)
 # CHECK-NEXT:  -      -      -      -     14.00   -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     divq	%rcx
 # CHECK-NEXT: 0.50   0.50    -      -     14.00   -     1.00    -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     divq	(%rax)
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     enter	$7, $4095
 # CHECK-NEXT:  -      -      -      -     12.00   -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     idivb	%dil
 # CHECK-NEXT: 0.50   0.50    -      -     12.00   -     1.00    -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     idivb	(%rax)
 # CHECK-NEXT:  -      -      -      -     17.00   -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     idivw	%si
@@ -1911,7 +1935,10 @@ xorq (%rax), %rdi
 # CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     insw	%dx, %es:(%rdi)
 # CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     insl	%dx, %es:(%rdi)
 # CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     int	$7
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     invlpg	(%rax)
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     invlpga	%rax, %ecx
 # CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     lahf
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     leave
 # CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     lodsb	(%rsi), %al
 # CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     lodsw	(%rsi), %ax
 # CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     lodsl	(%rsi), %eax
@@ -2061,6 +2088,10 @@ xorq (%rax), %rdi
 # CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     rcrq	%cl, %rdi
 # CHECK-NEXT: 1.00   1.00    -      -      -     0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     1.00   rclq	%cl, (%rax)
 # CHECK-NEXT: 1.00   1.00    -      -      -     0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     1.00   rcrq	%cl, (%rax)
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     rdmsr
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     rdpmc
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     rdtsc
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     rdtscp
 # CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     rolb	%dil
 # CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     rorb	%dil
 # CHECK-NEXT: 1.00   1.00    -      -      -     0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     1.00   rolb	(%rax)
@@ -2335,6 +2366,7 @@ xorq (%rax), %rdi
 # CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     testq	%rsi, %rdi
 # CHECK-NEXT: 0.50   0.50    -      -      -     0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -      -     testq	%rsi, (%rax)
 # CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     ud2
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     wrmsr
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     xaddb	%bl, %cl
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     xaddb	%bl, (%rcx)
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     xaddw	%bx, %cx
diff --git a/test/tools/llvm-mca/X86/BdVer2/resources-x87.s b/test/tools/llvm-mca/X86/BdVer2/resources-x87.s
index ad72714c74c1bbaf47a96b3b57ef85430ef9125a..4cdddf01104b04959322a557dc183fd02d3ebc55 100644
--- a/test/tools/llvm-mca/X86/BdVer2/resources-x87.s
+++ b/test/tools/llvm-mca/X86/BdVer2/resources-x87.s
@@ -5,7 +5,7 @@ f2xm1
 
 fabs
 
-fadd %st(0), %st(1)
+fadd %st, %st(1)
 fadd %st(2)
 fadds (%ecx)
 faddl (%ecx)
@@ -21,14 +21,14 @@ fchs
 
 fnclex
 
-fcmovb %st(1), %st(0)
-fcmovbe %st(1), %st(0)
-fcmove %st(1), %st(0)
-fcmovnb %st(1), %st(0)
-fcmovnbe %st(1), %st(0)
-fcmovne %st(1), %st(0)
-fcmovnu %st(1), %st(0)
-fcmovu %st(1), %st(0)
+fcmovb %st(1), %st
+fcmovbe %st(1), %st
+fcmove %st(1), %st
+fcmovnb %st(1), %st
+fcmovnbe %st(1), %st
+fcmovne %st(1), %st
+fcmovnu %st(1), %st
+fcmovu %st(1), %st
 
 fcom %st(1)
 fcom %st(3)
@@ -47,7 +47,7 @@ fcos
 
 fdecstp
 
-fdiv %st(0), %st(1)
+fdiv %st, %st(1)
 fdiv %st(2)
 fdivs (%ecx)
 fdivl (%eax)
@@ -56,7 +56,7 @@ fdivp %st(2)
 fidivs (%ecx)
 fidivl (%eax)
 
-fdivr %st(0), %st(1)
+fdivr %st, %st(1)
 fdivr %st(2)
 fdivrs (%ecx)
 fdivrl (%eax)
@@ -106,7 +106,7 @@ fldln2
 fldpi
 fldz
 
-fmul %st(0), %st(1)
+fmul %st, %st(1)
 fmul %st(2)
 fmuls (%ecx)
 fmull (%eax)
@@ -153,7 +153,7 @@ fnstsw (%eax)
 frstor (%eax)
 fsave (%eax)
 
-fsub %st(0), %st(1)
+fsub %st, %st(1)
 fsub %st(2)
 fsubs (%ecx)
 fsubl (%eax)
@@ -162,7 +162,7 @@ fsubp %st(2)
 fisubs (%ecx)
 fisubl (%eax)
 
-fsubr %st(0), %st(1)
+fsubr %st, %st(1)
 fsubr %st(2)
 fsubrs (%ecx)
 fsubrl (%eax)
@@ -208,26 +208,26 @@ fyl2xp1
 # CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
 # CHECK-NEXT:  1      100   0.50                  U     f2xm1
 # CHECK-NEXT:  1      1     1.00                  U     fabs
-# CHECK-NEXT:  1      5     1.00                  U     fadd	%st(0), %st(1)
-# CHECK-NEXT:  1      5     1.00                  U     fadd	%st(2)
+# CHECK-NEXT:  1      5     1.00                  U     fadd	%st, %st(1)
+# CHECK-NEXT:  1      5     1.00                  U     fadd	%st(2), %st
 # CHECK-NEXT:  1      10    1.00    *             U     fadds	(%ecx)
 # CHECK-NEXT:  1      10    1.00    *             U     faddl	(%ecx)
-# CHECK-NEXT:  1      5     1.00                  U     faddp	%st(1)
-# CHECK-NEXT:  1      5     1.00                  U     faddp	%st(2)
+# CHECK-NEXT:  1      5     1.00                  U     faddp	%st, %st(1)
+# CHECK-NEXT:  1      5     1.00                  U     faddp	%st, %st(2)
 # CHECK-NEXT:  1      10    1.00    *             U     fiadds	(%ecx)
 # CHECK-NEXT:  1      10    1.00    *             U     fiaddl	(%ecx)
 # CHECK-NEXT:  1      100   0.50                  U     fbld	(%ecx)
 # CHECK-NEXT:  1      100   0.50                  U     fbstp	(%eax)
 # CHECK-NEXT:  1      1     1.00                  U     fchs
 # CHECK-NEXT:  1      100   0.50                  U     fnclex
-# CHECK-NEXT:  1      1     1.00                  U     fcmovb	%st(1), %st(0)
-# CHECK-NEXT:  1      1     1.00                  U     fcmovbe	%st(1), %st(0)
-# CHECK-NEXT:  1      1     1.00                  U     fcmove	%st(1), %st(0)
-# CHECK-NEXT:  1      1     1.00                  U     fcmovnb	%st(1), %st(0)
-# CHECK-NEXT:  1      1     1.00                  U     fcmovnbe	%st(1), %st(0)
-# CHECK-NEXT:  1      1     1.00                  U     fcmovne	%st(1), %st(0)
-# CHECK-NEXT:  1      1     1.00                  U     fcmovnu	%st(1), %st(0)
-# CHECK-NEXT:  1      1     1.00                  U     fcmovu	%st(1), %st(0)
+# CHECK-NEXT:  1      1     1.00                  U     fcmovb	%st(1), %st
+# CHECK-NEXT:  1      1     1.00                  U     fcmovbe	%st(1), %st
+# CHECK-NEXT:  1      1     1.00                  U     fcmove	%st(1), %st
+# CHECK-NEXT:  1      1     1.00                  U     fcmovnb	%st(1), %st
+# CHECK-NEXT:  1      1     1.00                  U     fcmovnbe	%st(1), %st
+# CHECK-NEXT:  1      1     1.00                  U     fcmovne	%st(1), %st
+# CHECK-NEXT:  1      1     1.00                  U     fcmovnu	%st(1), %st
+# CHECK-NEXT:  1      1     1.00                  U     fcmovu	%st(1), %st
 # CHECK-NEXT:  2      1     1.00                  U     fcom	%st(1)
 # CHECK-NEXT:  2      1     1.00                  U     fcom	%st(3)
 # CHECK-NEXT:  1      6     1.00                  U     fcoms	(%ecx)
@@ -237,24 +237,24 @@ fyl2xp1
 # CHECK-NEXT:  1      6     1.00                  U     fcomps	(%ecx)
 # CHECK-NEXT:  1      6     1.00                  U     fcompl	(%eax)
 # CHECK-NEXT:  1      100   0.50                  U     fcompp
-# CHECK-NEXT:  2      1     1.00                  U     fcomi	%st(3)
-# CHECK-NEXT:  2      1     1.00                  U     fcompi	%st(3)
+# CHECK-NEXT:  2      1     1.00                  U     fcomi	%st(3), %st
+# CHECK-NEXT:  2      1     1.00                  U     fcompi	%st(3), %st
 # CHECK-NEXT:  1      100   0.50                  U     fcos
 # CHECK-NEXT:  1      100   0.50                  U     fdecstp
-# CHECK-NEXT:  1      9     9.50                  U     fdiv	%st(0), %st(1)
-# CHECK-NEXT:  1      9     9.50                  U     fdiv	%st(2)
+# CHECK-NEXT:  1      9     9.50                  U     fdiv	%st, %st(1)
+# CHECK-NEXT:  1      9     9.50                  U     fdiv	%st(2), %st
 # CHECK-NEXT:  1      14    9.50    *             U     fdivs	(%ecx)
 # CHECK-NEXT:  1      14    9.50    *             U     fdivl	(%eax)
-# CHECK-NEXT:  1      9     9.50                  U     fdivp	%st(1)
-# CHECK-NEXT:  1      9     9.50                  U     fdivp	%st(2)
+# CHECK-NEXT:  1      9     9.50                  U     fdivp	%st, %st(1)
+# CHECK-NEXT:  1      9     9.50                  U     fdivp	%st, %st(2)
 # CHECK-NEXT:  1      14    9.50    *             U     fidivs	(%ecx)
 # CHECK-NEXT:  1      14    9.50    *             U     fidivl	(%eax)
-# CHECK-NEXT:  1      9     9.50                  U     fdivr	%st(0), %st(1)
-# CHECK-NEXT:  1      9     9.50                  U     fdivr	%st(2)
+# CHECK-NEXT:  1      9     9.50                  U     fdivr	%st, %st(1)
+# CHECK-NEXT:  1      9     9.50                  U     fdivr	%st(2), %st
 # CHECK-NEXT:  1      14    9.50    *             U     fdivrs	(%ecx)
 # CHECK-NEXT:  1      14    9.50    *             U     fdivrl	(%eax)
-# CHECK-NEXT:  1      9     9.50                  U     fdivrp	%st(1)
-# CHECK-NEXT:  1      9     9.50                  U     fdivrp	%st(2)
+# CHECK-NEXT:  1      9     9.50                  U     fdivrp	%st, %st(1)
+# CHECK-NEXT:  1      9     9.50                  U     fdivrp	%st, %st(2)
 # CHECK-NEXT:  1      14    9.50    *             U     fidivrs	(%ecx)
 # CHECK-NEXT:  1      14    9.50    *             U     fidivrl	(%eax)
 # CHECK-NEXT:  1      100   0.50                  U     ffree	%st(0)
@@ -288,12 +288,12 @@ fyl2xp1
 # CHECK-NEXT:  1      3     1.00                  U     fldln2
 # CHECK-NEXT:  1      3     1.00                  U     fldpi
 # CHECK-NEXT:  1      3     1.00                  U     fldz
-# CHECK-NEXT:  1      5     1.00                  U     fmul	%st(0), %st(1)
-# CHECK-NEXT:  1      5     1.00                  U     fmul	%st(2)
+# CHECK-NEXT:  1      5     1.00                  U     fmul	%st, %st(1)
+# CHECK-NEXT:  1      5     1.00                  U     fmul	%st(2), %st
 # CHECK-NEXT:  1      10    1.00    *             U     fmuls	(%ecx)
 # CHECK-NEXT:  1      10    1.00    *             U     fmull	(%eax)
-# CHECK-NEXT:  1      5     1.00                  U     fmulp	%st(1)
-# CHECK-NEXT:  1      5     1.00                  U     fmulp	%st(2)
+# CHECK-NEXT:  1      5     1.00                  U     fmulp	%st, %st(1)
+# CHECK-NEXT:  1      5     1.00                  U     fmulp	%st, %st(2)
 # CHECK-NEXT:  1      10    1.00    *             U     fimuls	(%ecx)
 # CHECK-NEXT:  1      10    1.00    *             U     fimull	(%eax)
 # CHECK-NEXT:  1      1     0.50                  U     fnop
@@ -321,20 +321,20 @@ fyl2xp1
 # CHECK-NEXT:  1      100   0.50                  U     frstor	(%eax)
 # CHECK-NEXT:  1      100   0.50                  U     wait
 # CHECK-NEXT:  1      100   0.50                  U     fnsave	(%eax)
-# CHECK-NEXT:  1      5     1.00                  U     fsub	%st(0), %st(1)
-# CHECK-NEXT:  1      5     1.00                  U     fsub	%st(2)
+# CHECK-NEXT:  1      5     1.00                  U     fsub	%st, %st(1)
+# CHECK-NEXT:  1      5     1.00                  U     fsub	%st(2), %st
 # CHECK-NEXT:  1      10    1.00    *             U     fsubs	(%ecx)
 # CHECK-NEXT:  1      10    1.00    *             U     fsubl	(%eax)
-# CHECK-NEXT:  1      5     1.00                  U     fsubp	%st(1)
-# CHECK-NEXT:  1      5     1.00                  U     fsubp	%st(2)
+# CHECK-NEXT:  1      5     1.00                  U     fsubp	%st, %st(1)
+# CHECK-NEXT:  1      5     1.00                  U     fsubp	%st, %st(2)
 # CHECK-NEXT:  1      10    1.00    *             U     fisubs	(%ecx)
 # CHECK-NEXT:  1      10    1.00    *             U     fisubl	(%eax)
-# CHECK-NEXT:  1      5     1.00                  U     fsubr	%st(0), %st(1)
-# CHECK-NEXT:  1      5     1.00                  U     fsubr	%st(2)
+# CHECK-NEXT:  1      5     1.00                  U     fsubr	%st, %st(1)
+# CHECK-NEXT:  1      5     1.00                  U     fsubr	%st(2), %st
 # CHECK-NEXT:  1      10    1.00    *             U     fsubrs	(%ecx)
 # CHECK-NEXT:  1      10    1.00    *             U     fsubrl	(%eax)
-# CHECK-NEXT:  1      5     1.00                  U     fsubrp	%st(1)
-# CHECK-NEXT:  1      5     1.00                  U     fsubrp	%st(2)
+# CHECK-NEXT:  1      5     1.00                  U     fsubrp	%st, %st(1)
+# CHECK-NEXT:  1      5     1.00                  U     fsubrp	%st, %st(2)
 # CHECK-NEXT:  1      10    1.00    *             U     fisubrs	(%ecx)
 # CHECK-NEXT:  1      10    1.00    *             U     fisubrl	(%eax)
 # CHECK-NEXT:  1      1     1.00                  U     ftst
@@ -343,8 +343,8 @@ fyl2xp1
 # CHECK-NEXT:  2      1     1.00                  U     fucomp	%st(1)
 # CHECK-NEXT:  2      1     1.00                  U     fucomp	%st(3)
 # CHECK-NEXT:  1      1     1.00                  U     fucompp
-# CHECK-NEXT:  2      1     1.00                  U     fucomi	%st(3)
-# CHECK-NEXT:  2      1     1.00                  U     fucompi	%st(3)
+# CHECK-NEXT:  2      1     1.00                  U     fucomi	%st(3), %st
+# CHECK-NEXT:  2      1     1.00                  U     fucompi	%st(3), %st
 # CHECK-NEXT:  1      100   0.50                  U     wait
 # CHECK-NEXT:  1      100   0.50                  U     fxam
 # CHECK-NEXT:  1      1     0.50                  U     fxch	%st(1)
@@ -388,26 +388,26 @@ fyl2xp1
 # CHECK-NEXT: [0.0]  [0.1]  [1]    [2]    [3]    [4]    [5]    [6]    [7.0]  [7.1]  [8.0]  [8.1]  [9]    [10]   [11]   [12]   [13]   [14]   [15]   [16.0] [16.1] [17]   [18]   Instructions:
 # CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     f2xm1
 # CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -     1.00    -      -      -      -      -      -      -     fabs
-# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     1.00    -      -      -      -      -      -      -      -     fadd	%st(0), %st(1)
-# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     1.00    -      -      -      -      -      -      -      -     fadd	%st(2)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     1.00    -      -      -      -      -      -      -      -     fadd	%st, %st(1)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     1.00    -      -      -      -      -      -      -      -     fadd	%st(2), %st
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -      -      -     1.00    -      -      -      -     0.50   0.50    -      -     fadds	(%ecx)
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -      -      -     1.00    -      -      -      -     0.50   0.50    -      -     faddl	(%ecx)
-# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     1.00    -      -      -      -      -      -      -      -     faddp	%st(1)
-# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     1.00    -      -      -      -      -      -      -      -     faddp	%st(2)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     1.00    -      -      -      -      -      -      -      -     faddp	%st, %st(1)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     1.00    -      -      -      -      -      -      -      -     faddp	%st, %st(2)
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -      -      -     1.00    -      -      -      -     0.50   0.50    -      -     fiadds	(%ecx)
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -      -      -     1.00    -      -      -      -     0.50   0.50    -      -     fiaddl	(%ecx)
 # CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     fbld	(%ecx)
 # CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     fbstp	(%eax)
 # CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -     1.00    -      -      -      -      -      -      -     fchs
 # CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     fnclex
-# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     1.00    -      -      -      -      -      -      -      -     fcmovb	%st(1), %st(0)
-# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     1.00    -      -      -      -      -      -      -      -     fcmovbe	%st(1), %st(0)
-# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     1.00    -      -      -      -      -      -      -      -     fcmove	%st(1), %st(0)
-# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     1.00    -      -      -      -      -      -      -      -     fcmovnb	%st(1), %st(0)
-# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     1.00    -      -      -      -      -      -      -      -     fcmovnbe	%st(1), %st(0)
-# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     1.00    -      -      -      -      -      -      -      -     fcmovne	%st(1), %st(0)
-# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     1.00    -      -      -      -      -      -      -      -     fcmovnu	%st(1), %st(0)
-# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     1.00    -      -      -      -      -      -      -      -     fcmovu	%st(1), %st(0)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     1.00    -      -      -      -      -      -      -      -     fcmovb	%st(1), %st
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     1.00    -      -      -      -      -      -      -      -     fcmovbe	%st(1), %st
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     1.00    -      -      -      -      -      -      -      -     fcmove	%st(1), %st
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     1.00    -      -      -      -      -      -      -      -     fcmovnb	%st(1), %st
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     1.00    -      -      -      -      -      -      -      -     fcmovnbe	%st(1), %st
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     1.00    -      -      -      -      -      -      -      -     fcmovne	%st(1), %st
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     1.00    -      -      -      -      -      -      -      -     fcmovnu	%st(1), %st
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     1.00    -      -      -      -      -      -      -      -     fcmovu	%st(1), %st
 # CHECK-NEXT:  -      -      -      -      -     1.00    -      -     0.50   0.50    -      -      -      -     1.00    -      -      -      -      -      -      -      -     fcom	%st(1)
 # CHECK-NEXT:  -      -      -      -      -     1.00    -      -     0.50   0.50    -      -      -      -     1.00    -      -      -      -      -      -      -      -     fcom	%st(3)
 # CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -     1.00    -      -      -      -      -      -      -     fcoms	(%ecx)
@@ -417,24 +417,24 @@ fyl2xp1
 # CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -     1.00    -      -      -      -      -      -      -     fcomps	(%ecx)
 # CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -     1.00    -      -      -      -      -      -      -     fcompl	(%eax)
 # CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     fcompp
-# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     0.50   0.50    -      -      -      -     1.00    -      -      -      -      -      -      -      -     fcomi	%st(3)
-# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     0.50   0.50    -      -      -      -     1.00    -      -      -      -      -      -      -      -     fcompi	%st(3)
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     0.50   0.50    -      -      -      -     1.00    -      -      -      -      -      -      -      -     fcomi	%st(3), %st
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     0.50   0.50    -      -      -      -     1.00    -      -      -      -      -      -      -      -     fcompi	%st(3), %st
 # CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     fcos
 # CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     fdecstp
-# CHECK-NEXT:  -      -      -      -      -      -      -      -     9.50   9.50    -      -      -      -      -     1.00    -      -      -      -      -      -      -     fdiv	%st(0), %st(1)
-# CHECK-NEXT:  -      -      -      -      -      -      -      -     9.50   9.50    -      -      -      -      -     1.00    -      -      -      -      -      -      -     fdiv	%st(2)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     9.50   9.50    -      -      -      -      -     1.00    -      -      -      -      -      -      -     fdiv	%st, %st(1)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     9.50   9.50    -      -      -      -      -     1.00    -      -      -      -      -      -      -     fdiv	%st(2), %st
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -     9.50   9.50    -      -      -      -      -     1.00    -      -      -     0.50   0.50    -      -     fdivs	(%ecx)
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -     9.50   9.50    -      -      -      -      -     1.00    -      -      -     0.50   0.50    -      -     fdivl	(%eax)
-# CHECK-NEXT:  -      -      -      -      -      -      -      -     9.50   9.50    -      -      -      -      -     1.00    -      -      -      -      -      -      -     fdivp	%st(1)
-# CHECK-NEXT:  -      -      -      -      -      -      -      -     9.50   9.50    -      -      -      -      -     1.00    -      -      -      -      -      -      -     fdivp	%st(2)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     9.50   9.50    -      -      -      -      -     1.00    -      -      -      -      -      -      -     fdivp	%st, %st(1)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     9.50   9.50    -      -      -      -      -     1.00    -      -      -      -      -      -      -     fdivp	%st, %st(2)
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -     9.50   9.50    -      -      -      -      -     1.00    -      -      -     0.50   0.50    -      -     fidivs	(%ecx)
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -     9.50   9.50    -      -      -      -      -     1.00    -      -      -     0.50   0.50    -      -     fidivl	(%eax)
-# CHECK-NEXT:  -      -      -      -      -      -      -      -     9.50   9.50    -      -      -      -      -     1.00    -      -      -      -      -      -      -     fdivr	%st(0), %st(1)
-# CHECK-NEXT:  -      -      -      -      -      -      -      -     9.50   9.50    -      -      -      -      -     1.00    -      -      -      -      -      -      -     fdivr	%st(2)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     9.50   9.50    -      -      -      -      -     1.00    -      -      -      -      -      -      -     fdivr	%st, %st(1)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     9.50   9.50    -      -      -      -      -     1.00    -      -      -      -      -      -      -     fdivr	%st(2), %st
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -     9.50   9.50    -      -      -      -      -     1.00    -      -      -     0.50   0.50    -      -     fdivrs	(%ecx)
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -     9.50   9.50    -      -      -      -      -     1.00    -      -      -     0.50   0.50    -      -     fdivrl	(%eax)
-# CHECK-NEXT:  -      -      -      -      -      -      -      -     9.50   9.50    -      -      -      -      -     1.00    -      -      -      -      -      -      -     fdivrp	%st(1)
-# CHECK-NEXT:  -      -      -      -      -      -      -      -     9.50   9.50    -      -      -      -      -     1.00    -      -      -      -      -      -      -     fdivrp	%st(2)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     9.50   9.50    -      -      -      -      -     1.00    -      -      -      -      -      -      -     fdivrp	%st, %st(1)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     9.50   9.50    -      -      -      -      -     1.00    -      -      -      -      -      -      -     fdivrp	%st, %st(2)
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -     9.50   9.50    -      -      -      -      -     1.00    -      -      -     0.50   0.50    -      -     fidivrs	(%ecx)
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -     9.50   9.50    -      -      -      -      -     1.00    -      -      -     0.50   0.50    -      -     fidivrl	(%eax)
 # CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     ffree	%st(0)
@@ -468,12 +468,12 @@ fyl2xp1
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -      -      -      -      -      -      -     fldln2
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -      -      -      -      -      -      -     fldpi
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00    -     1.00    -      -      -      -      -      -      -     fldz
-# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -     1.00    -      -      -      -      -      -      -     fmul	%st(0), %st(1)
-# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -     1.00    -      -      -      -      -      -      -     fmul	%st(2)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -     1.00    -      -      -      -      -      -      -     fmul	%st, %st(1)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -     1.00    -      -      -      -      -      -      -     fmul	%st(2), %st
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -      -      -      -     1.00    -      -      -     0.50   0.50    -      -     fmuls	(%ecx)
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -      -      -      -     1.00    -      -      -     0.50   0.50    -      -     fmull	(%eax)
-# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -     1.00    -      -      -      -      -      -      -     fmulp	%st(1)
-# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -     1.00    -      -      -      -      -      -      -     fmulp	%st(2)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -     1.00    -      -      -      -      -      -      -     fmulp	%st, %st(1)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -     1.00    -      -      -      -      -      -      -     fmulp	%st, %st(2)
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -      -      -      -     1.00    -      -      -     0.50   0.50    -      -     fimuls	(%ecx)
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -      -      -      -     1.00    -      -      -     0.50   0.50    -      -     fimull	(%eax)
 # CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     fnop
@@ -501,20 +501,20 @@ fyl2xp1
 # CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     frstor	(%eax)
 # CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     wait
 # CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     fnsave	(%eax)
-# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     1.00    -      -      -      -      -      -      -      -     fsub	%st(0), %st(1)
-# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     1.00    -      -      -      -      -      -      -      -     fsub	%st(2)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     1.00    -      -      -      -      -      -      -      -     fsub	%st, %st(1)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     1.00    -      -      -      -      -      -      -      -     fsub	%st(2), %st
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -      -      -     1.00    -      -      -      -     0.50   0.50    -      -     fsubs	(%ecx)
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -      -      -     1.00    -      -      -      -     0.50   0.50    -      -     fsubl	(%eax)
-# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     1.00    -      -      -      -      -      -      -      -     fsubp	%st(1)
-# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     1.00    -      -      -      -      -      -      -      -     fsubp	%st(2)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     1.00    -      -      -      -      -      -      -      -     fsubp	%st, %st(1)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     1.00    -      -      -      -      -      -      -      -     fsubp	%st, %st(2)
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -      -      -     1.00    -      -      -      -     0.50   0.50    -      -     fisubs	(%ecx)
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -      -      -     1.00    -      -      -      -     0.50   0.50    -      -     fisubl	(%eax)
-# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     1.00    -      -      -      -      -      -      -      -     fsubr	%st(0), %st(1)
-# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     1.00    -      -      -      -      -      -      -      -     fsubr	%st(2)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     1.00    -      -      -      -      -      -      -      -     fsubr	%st, %st(1)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     1.00    -      -      -      -      -      -      -      -     fsubr	%st(2), %st
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -      -      -     1.00    -      -      -      -     0.50   0.50    -      -     fsubrs	(%ecx)
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -      -      -     1.00    -      -      -      -     0.50   0.50    -      -     fsubrl	(%eax)
-# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     1.00    -      -      -      -      -      -      -      -     fsubrp	%st(1)
-# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     1.00    -      -      -      -      -      -      -      -     fsubrp	%st(2)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     1.00    -      -      -      -      -      -      -      -     fsubrp	%st, %st(1)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -     1.00    -      -      -      -      -      -      -      -     fsubrp	%st, %st(2)
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -      -      -     1.00    -      -      -      -     0.50   0.50    -      -     fisubrs	(%ecx)
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -      -      -     1.00    -      -      -      -     0.50   0.50    -      -     fisubrl	(%eax)
 # CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -     1.00    -      -      -      -      -      -      -     ftst
@@ -523,8 +523,8 @@ fyl2xp1
 # CHECK-NEXT:  -      -      -      -      -     1.00    -      -     0.50   0.50    -      -      -      -     1.00    -      -      -      -      -      -      -      -     fucomp	%st(1)
 # CHECK-NEXT:  -      -      -      -      -     1.00    -      -     0.50   0.50    -      -      -      -     1.00    -      -      -      -      -      -      -      -     fucomp	%st(3)
 # CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -      -      -      -     1.00    -      -      -      -      -      -      -     fucompp
-# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     0.50   0.50    -      -      -      -     1.00    -      -      -      -      -      -      -      -     fucomi	%st(3)
-# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     0.50   0.50    -      -      -      -     1.00    -      -      -      -      -      -      -      -     fucompi	%st(3)
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     0.50   0.50    -      -      -      -     1.00    -      -      -      -      -      -      -      -     fucomi	%st(3), %st
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     0.50   0.50    -      -      -      -     1.00    -      -      -      -      -      -      -      -     fucompi	%st(3), %st
 # CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     wait
 # CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     fxam
 # CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -      -      -      -      -     fxch	%st(1)
diff --git a/test/tools/llvm-mca/X86/Broadwell/resources-avx2.s b/test/tools/llvm-mca/X86/Broadwell/resources-avx2.s
index 7dc3b71474113faf3183020e8c59e695adbf4b55..6286511acd0ed31d9e5a5ec8f5a871fca465a680 100644
--- a/test/tools/llvm-mca/X86/Broadwell/resources-avx2.s
+++ b/test/tools/llvm-mca/X86/Broadwell/resources-avx2.s
@@ -149,6 +149,9 @@ vpcmpgtq        (%rax), %ymm1, %ymm2
 vpcmpgtw        %ymm0, %ymm1, %ymm2
 vpcmpgtw        (%rax), %ymm1, %ymm2
 
+vperm2i128      $1, %ymm0, %ymm1, %ymm2
+vperm2i128      $1, (%rax), %ymm1, %ymm2
+
 vpermd          %ymm0, %ymm1, %ymm2
 vpermd          (%rax), %ymm1, %ymm2
 
@@ -555,6 +558,8 @@ vpxor           (%rax), %ymm1, %ymm2
 # CHECK-NEXT:  2      11    1.00    *                   vpcmpgtq	(%rax), %ymm1, %ymm2
 # CHECK-NEXT:  1      1     0.50                        vpcmpgtw	%ymm0, %ymm1, %ymm2
 # CHECK-NEXT:  2      7     0.50    *                   vpcmpgtw	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      3     1.00                        vperm2i128	$1, %ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  2      9     1.00    *                   vperm2i128	$1, (%rax), %ymm1, %ymm2
 # CHECK-NEXT:  1      3     1.00                        vpermd	%ymm0, %ymm1, %ymm2
 # CHECK-NEXT:  2      9     1.00    *                   vpermd	(%rax), %ymm1, %ymm2
 # CHECK-NEXT:  1      3     1.00                        vpermpd	$1, %ymm0, %ymm2
@@ -771,7 +776,7 @@ vpxor           (%rax), %ymm1, %ymm2
 
 # CHECK:      Resource pressure per iteration:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]
-# CHECK-NEXT:  -      -     94.67  58.67  85.17  85.17  13.00  235.67 2.00   1.67
+# CHECK-NEXT:  -      -     94.67  58.67  85.67  85.67  13.00  237.67 2.00   1.67
 
 # CHECK:      Resource pressure by instruction:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    Instructions:
@@ -873,6 +878,8 @@ vpxor           (%rax), %ymm1, %ymm2
 # CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -      -      -      -     vpcmpgtq	(%rax), %ymm1, %ymm2
 # CHECK-NEXT:  -      -      -     0.50    -      -      -     0.50    -      -     vpcmpgtw	%ymm0, %ymm1, %ymm2
 # CHECK-NEXT:  -      -      -     0.50   0.50   0.50    -     0.50    -      -     vpcmpgtw	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vperm2i128	$1, %ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vperm2i128	$1, (%rax), %ymm1, %ymm2
 # CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpermd	%ymm0, %ymm1, %ymm2
 # CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermd	(%rax), %ymm1, %ymm2
 # CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpermpd	$1, %ymm0, %ymm2
diff --git a/test/tools/llvm-mca/X86/Broadwell/resources-bmi1.s b/test/tools/llvm-mca/X86/Broadwell/resources-bmi1.s
index b2f16ff3b0834eb5fcfd9c6277ac949229ecf7a7..31e8095137f541a84b8dbb5b1e985df70b47fb74 100644
--- a/test/tools/llvm-mca/X86/Broadwell/resources-bmi1.s
+++ b/test/tools/llvm-mca/X86/Broadwell/resources-bmi1.s
@@ -31,6 +31,9 @@ blsr        (%rax), %ecx
 blsr        %rax, %rcx
 blsr        (%rax), %rcx
 
+tzcnt       %ax, %cx
+tzcnt       (%rax), %cx
+
 tzcnt       %eax, %ecx
 tzcnt       (%rax), %ecx
 
@@ -66,6 +69,8 @@ tzcnt       (%rax), %rcx
 # CHECK-NEXT:  2      6     0.50    *                   blsrl	(%rax), %ecx
 # CHECK-NEXT:  1      1     0.50                        blsrq	%rax, %rcx
 # CHECK-NEXT:  2      6     0.50    *                   blsrq	(%rax), %rcx
+# CHECK-NEXT:  1      3     1.00                        tzcntw	%ax, %cx
+# CHECK-NEXT:  2      8     1.00    *                   tzcntw	(%rax), %cx
 # CHECK-NEXT:  1      3     1.00                        tzcntl	%eax, %ecx
 # CHECK-NEXT:  2      8     1.00    *                   tzcntl	(%rax), %ecx
 # CHECK-NEXT:  1      3     1.00                        tzcntq	%rax, %rcx
@@ -85,7 +90,7 @@ tzcnt       (%rax), %rcx
 
 # CHECK:      Resource pressure per iteration:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]
-# CHECK-NEXT:  -      -     2.00   14.00  6.00   6.00    -     10.00  2.00    -
+# CHECK-NEXT:  -      -     2.00   16.00  6.50   6.50    -     10.00  2.00    -
 
 # CHECK:      Resource pressure by instruction:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    Instructions:
@@ -109,6 +114,8 @@ tzcnt       (%rax), %rcx
 # CHECK-NEXT:  -      -      -     0.50   0.50   0.50    -     0.50    -      -     blsrl	(%rax), %ecx
 # CHECK-NEXT:  -      -      -     0.50    -      -      -     0.50    -      -     blsrq	%rax, %rcx
 # CHECK-NEXT:  -      -      -     0.50   0.50   0.50    -     0.50    -      -     blsrq	(%rax), %rcx
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     tzcntw	%ax, %cx
+# CHECK-NEXT:  -      -      -     1.00   0.50   0.50    -      -      -      -     tzcntw	(%rax), %cx
 # CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     tzcntl	%eax, %ecx
 # CHECK-NEXT:  -      -      -     1.00   0.50   0.50    -      -      -      -     tzcntl	(%rax), %ecx
 # CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     tzcntq	%rax, %rcx
diff --git a/test/tools/llvm-mca/X86/Broadwell/resources-fsgsbase.s b/test/tools/llvm-mca/X86/Broadwell/resources-fsgsbase.s
new file mode 100644
index 0000000000000000000000000000000000000000..a6dd21a2ebfacc9bbf4f816d9e56194cb7951a1f
--- /dev/null
+++ b/test/tools/llvm-mca/X86/Broadwell/resources-fsgsbase.s
@@ -0,0 +1,59 @@
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=x86_64-unknown-unknown -mcpu=broadwell -instruction-tables < %s | FileCheck %s
+
+rdfsbase %eax
+rdfsbase %rax
+
+rdgsbase %eax
+rdgsbase %rax
+
+wrfsbase %edi
+wrfsbase %rdi
+
+wrgsbase %edi
+wrgsbase %rdi
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  1      100   0.25    *      *      U     rdfsbasel	%eax
+# CHECK-NEXT:  1      100   0.25    *      *      U     rdfsbaseq	%rax
+# CHECK-NEXT:  1      100   0.25    *      *      U     rdgsbasel	%eax
+# CHECK-NEXT:  1      100   0.25    *      *      U     rdgsbaseq	%rax
+# CHECK-NEXT:  1      100   0.25    *      *      U     wrfsbasel	%edi
+# CHECK-NEXT:  1      100   0.25    *      *      U     wrfsbaseq	%rdi
+# CHECK-NEXT:  1      100   0.25    *      *      U     wrgsbasel	%edi
+# CHECK-NEXT:  1      100   0.25    *      *      U     wrgsbaseq	%rdi
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0]   - BWDivider
+# CHECK-NEXT: [1]   - BWFPDivider
+# CHECK-NEXT: [2]   - BWPort0
+# CHECK-NEXT: [3]   - BWPort1
+# CHECK-NEXT: [4]   - BWPort2
+# CHECK-NEXT: [5]   - BWPort3
+# CHECK-NEXT: [6]   - BWPort4
+# CHECK-NEXT: [7]   - BWPort5
+# CHECK-NEXT: [8]   - BWPort6
+# CHECK-NEXT: [9]   - BWPort7
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]
+# CHECK-NEXT:  -      -     2.00   2.00    -      -      -     2.00   2.00    -
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    Instructions:
+# CHECK-NEXT:  -      -     0.25   0.25    -      -      -     0.25   0.25    -     rdfsbasel	%eax
+# CHECK-NEXT:  -      -     0.25   0.25    -      -      -     0.25   0.25    -     rdfsbaseq	%rax
+# CHECK-NEXT:  -      -     0.25   0.25    -      -      -     0.25   0.25    -     rdgsbasel	%eax
+# CHECK-NEXT:  -      -     0.25   0.25    -      -      -     0.25   0.25    -     rdgsbaseq	%rax
+# CHECK-NEXT:  -      -     0.25   0.25    -      -      -     0.25   0.25    -     wrfsbasel	%edi
+# CHECK-NEXT:  -      -     0.25   0.25    -      -      -     0.25   0.25    -     wrfsbaseq	%rdi
+# CHECK-NEXT:  -      -     0.25   0.25    -      -      -     0.25   0.25    -     wrgsbasel	%edi
+# CHECK-NEXT:  -      -     0.25   0.25    -      -      -     0.25   0.25    -     wrgsbaseq	%rdi
diff --git a/test/tools/llvm-mca/X86/Broadwell/resources-sse2.s b/test/tools/llvm-mca/X86/Broadwell/resources-sse2.s
index 26c3cd7565876363d52db5c1187f507c4e460e76..e4b2d680d0452fcc9d4bae267f6435173b9171d8 100644
--- a/test/tools/llvm-mca/X86/Broadwell/resources-sse2.s
+++ b/test/tools/llvm-mca/X86/Broadwell/resources-sse2.s
@@ -94,6 +94,8 @@ maxpd       (%rax), %xmm2
 maxsd       %xmm0, %xmm2
 maxsd       (%rax), %xmm2
 
+mfence
+
 minpd       %xmm0, %xmm2
 minpd       (%rax), %xmm2
 
@@ -229,6 +231,9 @@ pcmpgtw     (%rax), %xmm2
 
 pextrw      $1, %xmm0, %rcx
 
+pinsrw      $1, %rax, %xmm0
+pinsrw      $1, (%rax), %xmm0
+
 pmaddwd     %xmm0, %xmm2
 pmaddwd     (%rax), %xmm2
 
@@ -465,6 +470,7 @@ xorpd       (%rax), %xmm2
 # CHECK-NEXT:  2      8     1.00    *                   maxpd	(%rax), %xmm2
 # CHECK-NEXT:  1      3     1.00                        maxsd	%xmm0, %xmm2
 # CHECK-NEXT:  2      8     1.00    *                   maxsd	(%rax), %xmm2
+# CHECK-NEXT:  2      2     0.50    *      *      U     mfence
 # CHECK-NEXT:  1      3     1.00                        minpd	%xmm0, %xmm2
 # CHECK-NEXT:  2      8     1.00    *                   minpd	(%rax), %xmm2
 # CHECK-NEXT:  1      3     1.00                        minsd	%xmm0, %xmm2
@@ -555,6 +561,8 @@ xorpd       (%rax), %xmm2
 # CHECK-NEXT:  1      1     0.50                        pcmpgtw	%xmm0, %xmm2
 # CHECK-NEXT:  2      6     0.50    *                   pcmpgtw	(%rax), %xmm2
 # CHECK-NEXT:  2      2     1.00                        pextrw	$1, %xmm0, %ecx
+# CHECK-NEXT:  2      2     2.00                        pinsrw	$1, %eax, %xmm0
+# CHECK-NEXT:  2      6     1.00    *                   pinsrw	$1, (%rax), %xmm0
 # CHECK-NEXT:  1      5     1.00                        pmaddwd	%xmm0, %xmm2
 # CHECK-NEXT:  2      10    1.00    *                   pmaddwd	(%rax), %xmm2
 # CHECK-NEXT:  1      1     0.50                        pmaxsw	%xmm0, %xmm2
@@ -681,7 +689,7 @@ xorpd       (%rax), %xmm2
 
 # CHECK:      Resource pressure per iteration:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]
-# CHECK-NEXT:  -     78.00  70.25  95.25  62.67  62.67  14.00  123.75 1.75   4.67
+# CHECK-NEXT:  -     78.00  70.75  95.75  63.17  63.17  14.00  127.25 2.25   4.67
 
 # CHECK:      Resource pressure by instruction:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    Instructions:
@@ -748,6 +756,7 @@ xorpd       (%rax), %xmm2
 # CHECK-NEXT:  -      -      -     1.00   0.50   0.50    -      -      -      -     maxpd	(%rax), %xmm2
 # CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     maxsd	%xmm0, %xmm2
 # CHECK-NEXT:  -      -      -     1.00   0.50   0.50    -      -      -      -     maxsd	(%rax), %xmm2
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -     0.50   0.50    -     mfence
 # CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     minpd	%xmm0, %xmm2
 # CHECK-NEXT:  -      -      -     1.00   0.50   0.50    -      -      -      -     minpd	(%rax), %xmm2
 # CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     minsd	%xmm0, %xmm2
@@ -838,6 +847,8 @@ xorpd       (%rax), %xmm2
 # CHECK-NEXT:  -      -      -     0.50    -      -      -     0.50    -      -     pcmpgtw	%xmm0, %xmm2
 # CHECK-NEXT:  -      -      -     0.50   0.50   0.50    -     0.50    -      -     pcmpgtw	(%rax), %xmm2
 # CHECK-NEXT:  -      -     1.00    -      -      -      -     1.00    -      -     pextrw	$1, %xmm0, %ecx
+# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -     pinsrw	$1, %eax, %xmm0
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     pinsrw	$1, (%rax), %xmm0
 # CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     pmaddwd	%xmm0, %xmm2
 # CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -      -      -      -     pmaddwd	(%rax), %xmm2
 # CHECK-NEXT:  -      -      -     0.50    -      -      -     0.50    -      -     pmaxsw	%xmm0, %xmm2
diff --git a/test/tools/llvm-mca/X86/Broadwell/resources-sse3.s b/test/tools/llvm-mca/X86/Broadwell/resources-sse3.s
index e660799931c00f480f4cd36263b4f9abbe3fc37f..4f97d12ee6d70df6bee62d27e7e4c797337d568f 100644
--- a/test/tools/llvm-mca/X86/Broadwell/resources-sse3.s
+++ b/test/tools/llvm-mca/X86/Broadwell/resources-sse3.s
@@ -21,6 +21,8 @@ hsubps    (%rax), %xmm2
 
 lddqu     (%rax), %xmm2
 
+monitor
+
 movddup   %xmm0, %xmm2
 movddup   (%rax), %xmm2
 
@@ -30,6 +32,8 @@ movshdup  (%rax), %xmm2
 movsldup  %xmm0, %xmm2
 movsldup  (%rax), %xmm2
 
+mwait
+
 # CHECK:      Instruction Info:
 # CHECK-NEXT: [1]: #uOps
 # CHECK-NEXT: [2]: Latency
@@ -52,12 +56,14 @@ movsldup  (%rax), %xmm2
 # CHECK-NEXT:  3      5     2.00                        hsubps	%xmm0, %xmm2
 # CHECK-NEXT:  4      10    2.00    *                   hsubps	(%rax), %xmm2
 # CHECK-NEXT:  1      5     0.50    *                   lddqu	(%rax), %xmm2
+# CHECK-NEXT:  1      100   0.25                  U     monitor
 # CHECK-NEXT:  1      1     1.00                        movddup	%xmm0, %xmm2
 # CHECK-NEXT:  1      5     0.50    *                   movddup	(%rax), %xmm2
 # CHECK-NEXT:  1      1     1.00                        movshdup	%xmm0, %xmm2
 # CHECK-NEXT:  1      5     0.50    *                   movshdup	(%rax), %xmm2
 # CHECK-NEXT:  1      1     1.00                        movsldup	%xmm0, %xmm2
 # CHECK-NEXT:  1      5     0.50    *                   movsldup	(%rax), %xmm2
+# CHECK-NEXT:  1      100   0.25    *      *      U     mwait
 
 # CHECK:      Resources:
 # CHECK-NEXT: [0]   - BWDivider
@@ -73,7 +79,7 @@ movsldup  (%rax), %xmm2
 
 # CHECK:      Resource pressure per iteration:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]
-# CHECK-NEXT:  -      -      -     12.00  5.00   5.00    -     19.00   -      -
+# CHECK-NEXT:  -      -     0.50   12.50  5.00   5.00    -     19.50  0.50    -
 
 # CHECK:      Resource pressure by instruction:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    Instructions:
@@ -90,9 +96,11 @@ movsldup  (%rax), %xmm2
 # CHECK-NEXT:  -      -      -     1.00    -      -      -     2.00    -      -     hsubps	%xmm0, %xmm2
 # CHECK-NEXT:  -      -      -     1.00   0.50   0.50    -     2.00    -      -     hsubps	(%rax), %xmm2
 # CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -     lddqu	(%rax), %xmm2
+# CHECK-NEXT:  -      -     0.25   0.25    -      -      -     0.25   0.25    -     monitor
 # CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     movddup	%xmm0, %xmm2
 # CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -     movddup	(%rax), %xmm2
 # CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     movshdup	%xmm0, %xmm2
 # CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -     movshdup	(%rax), %xmm2
 # CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     movsldup	%xmm0, %xmm2
 # CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -     movsldup	(%rax), %xmm2
+# CHECK-NEXT:  -      -     0.25   0.25    -      -      -     0.25   0.25    -     mwait
diff --git a/test/tools/llvm-mca/X86/Broadwell/resources-x86_64.s b/test/tools/llvm-mca/X86/Broadwell/resources-x86_64.s
index b22edb4a9041dad2d06526cd4efead7e5557f46e..d55c4a342cb63de5ec56bb11dc8128821623c906 100644
--- a/test/tools/llvm-mca/X86/Broadwell/resources-x86_64.s
+++ b/test/tools/llvm-mca/X86/Broadwell/resources-x86_64.s
@@ -254,6 +254,8 @@ divl (%rax)
 divq %rcx
 divq (%rax)
 
+enter $7, $4095
+
 idivb %dil
 idivb (%rax)
 idivw %si
@@ -315,8 +317,13 @@ insl
 
 int $7
 
+invlpg  (%rax)
+invlpga %rax, %ecx
+
 lahf
 
+leave
+
 lodsb
 lodsw
 lodsl
@@ -486,6 +493,11 @@ rcrq %cl, %rdi
 rclq %cl, (%rax)
 rcrq %cl, (%rax)
 
+rdmsr
+rdpmc
+rdtsc
+rdtscp
+
 rolb %dil
 rorb %dil
 rolb (%rax)
@@ -789,6 +801,8 @@ testq %rsi, (%rax)
 
 ud2
 
+wrmsr
+
 xaddb %bl, %cl
 xaddb %bl, (%rcx)
 
@@ -1081,6 +1095,7 @@ xorq (%rax), %rdi
 # CHECK-NEXT:  8      34    2.00    *             U     divl	(%rax)
 # CHECK-NEXT:  32     80    8.00                  U     divq	%rcx
 # CHECK-NEXT:  8      34    2.00    *             U     divq	(%rax)
+# CHECK-NEXT:  1      100   0.25                  U     enter	$7, $4095
 # CHECK-NEXT:  1      25    10.00                 U     idivb	%dil
 # CHECK-NEXT:  8      35    2.00    *             U     idivb	(%rax)
 # CHECK-NEXT:  1      25    10.00                 U     idivw	%si
@@ -1133,7 +1148,10 @@ xorq (%rax), %rdi
 # CHECK-NEXT:  8      20    1.25                  U     insw	%dx, %es:(%rdi)
 # CHECK-NEXT:  8      20    1.25                  U     insl	%dx, %es:(%rdi)
 # CHECK-NEXT:  1      100   0.25    *      *      U     int	$7
+# CHECK-NEXT:  1      100   0.25                  U     invlpg	(%rax)
+# CHECK-NEXT:  1      100   0.25                  U     invlpga	%rax, %ecx
 # CHECK-NEXT:  1      1     0.50                        lahf
+# CHECK-NEXT:  3      7     0.50    *                   leave
 # CHECK-NEXT:  1      100   0.25                  U     lodsb	(%rsi), %al
 # CHECK-NEXT:  1      100   0.25                  U     lodsw	(%rsi), %ax
 # CHECK-NEXT:  1      100   0.25                  U     lodsl	(%rsi), %eax
@@ -1283,6 +1301,10 @@ xorq (%rax), %rdi
 # CHECK-NEXT:  7      11    2.00                        rcrq	%cl, %rdi
 # CHECK-NEXT:  10     15    2.00           *            rclq	%cl, (%rax)
 # CHECK-NEXT:  11     18    2.25           *            rcrq	%cl, (%rax)
+# CHECK-NEXT:  1      100   0.25                  U     rdmsr
+# CHECK-NEXT:  1      100   0.25                  U     rdpmc
+# CHECK-NEXT:  8      18    2.00                  U     rdtsc
+# CHECK-NEXT:  22     42    5.50                  U     rdtscp
 # CHECK-NEXT:  2      2     1.00                        rolb	%dil
 # CHECK-NEXT:  2      2     1.00                        rorb	%dil
 # CHECK-NEXT:  5      7     1.00    *      *            rolb	(%rax)
@@ -1557,6 +1579,7 @@ xorq (%rax), %rdi
 # CHECK-NEXT:  1      1     0.25                        testq	%rsi, %rdi
 # CHECK-NEXT:  2      6     0.50    *                   testq	%rsi, (%rax)
 # CHECK-NEXT:  1      100   0.25    *             U     ud2
+# CHECK-NEXT:  1      100   0.25                  U     wrmsr
 # CHECK-NEXT:  3      2     0.75                        xaddb	%bl, %cl
 # CHECK-NEXT:  5      7     1.00    *      *            xaddb	%bl, (%rcx)
 # CHECK-NEXT:  3      2     0.75                        xaddw	%bx, %cx
@@ -1622,7 +1645,7 @@ xorq (%rax), %rdi
 
 # CHECK:      Resource pressure per iteration:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]
-# CHECK-NEXT: 50.00   -     494.25 334.75 262.17 262.17 206.00 244.25 526.75 82.67
+# CHECK-NEXT: 50.00   -     503.00 343.00 262.67 262.67 206.00 255.50 536.50 82.67
 
 # CHECK:      Resource pressure by instruction:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    Instructions:
@@ -1846,6 +1869,7 @@ xorq (%rax), %rdi
 # CHECK-NEXT:  -      -     2.25   2.25   0.50   0.50    -     2.25   0.25    -     divl	(%rax)
 # CHECK-NEXT:  -      -     10.25  10.25   -      -      -     5.75   5.75    -     divq	%rcx
 # CHECK-NEXT:  -      -     2.25   2.25   0.50   0.50    -     2.25   0.25    -     divq	(%rax)
+# CHECK-NEXT:  -      -     0.25   0.25    -      -      -     0.25   0.25    -     enter	$7, $4095
 # CHECK-NEXT: 10.00   -     1.00    -      -      -      -      -      -      -     idivb	%dil
 # CHECK-NEXT:  -      -     2.25   2.25   0.50   0.50    -     2.25   0.25    -     idivb	(%rax)
 # CHECK-NEXT: 10.00   -     1.00    -      -      -      -      -      -      -     idivw	%si
@@ -1898,7 +1922,10 @@ xorq (%rax), %rdi
 # CHECK-NEXT:  -      -     1.00   0.50   0.83   0.83   1.00   1.50   2.00   0.33   insw	%dx, %es:(%rdi)
 # CHECK-NEXT:  -      -     1.00   0.50   0.83   0.83   1.00   1.50   2.00   0.33   insl	%dx, %es:(%rdi)
 # CHECK-NEXT:  -      -     0.25   0.25    -      -      -     0.25   0.25    -     int	$7
+# CHECK-NEXT:  -      -     0.25   0.25    -      -      -     0.25   0.25    -     invlpg	(%rax)
+# CHECK-NEXT:  -      -     0.25   0.25    -      -      -     0.25   0.25    -     invlpga	%rax, %ecx
 # CHECK-NEXT:  -      -     0.50    -      -      -      -      -     0.50    -     lahf
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -     0.50   0.50    -     leave
 # CHECK-NEXT:  -      -     0.25   0.25    -      -      -     0.25   0.25    -     lodsb	(%rsi), %al
 # CHECK-NEXT:  -      -     0.25   0.25    -      -      -     0.25   0.25    -     lodsw	(%rsi), %ax
 # CHECK-NEXT:  -      -     0.25   0.25    -      -      -     0.25   0.25    -     lodsl	(%rsi), %eax
@@ -2048,6 +2075,10 @@ xorq (%rax), %rdi
 # CHECK-NEXT:  -      -     1.75   2.75    -      -      -     0.75   1.75    -     rcrq	%cl, %rdi
 # CHECK-NEXT:  -      -     2.50   2.00   0.83   0.83    -     1.00   2.50   0.33   rclq	%cl, (%rax)
 # CHECK-NEXT:  -      -     2.25   3.25   0.83   0.83    -     1.25   2.25   0.33   rcrq	%cl, (%rax)
+# CHECK-NEXT:  -      -     0.25   0.25    -      -      -     0.25   0.25    -     rdmsr
+# CHECK-NEXT:  -      -     0.25   0.25    -      -      -     0.25   0.25    -     rdpmc
+# CHECK-NEXT:  -      -     1.75   1.25    -      -      -     2.25   2.75    -     rdtsc
+# CHECK-NEXT:  -      -     5.00   5.00    -      -      -     7.00   5.00    -     rdtscp
 # CHECK-NEXT:  -      -     1.00    -      -      -      -      -     1.00    -     rolb	%dil
 # CHECK-NEXT:  -      -     1.00    -      -      -      -      -     1.00    -     rorb	%dil
 # CHECK-NEXT:  -      -     1.00    -     0.83   0.83   1.00    -     1.00   0.33   rolb	(%rax)
@@ -2322,6 +2353,7 @@ xorq (%rax), %rdi
 # CHECK-NEXT:  -      -     0.25   0.25    -      -      -     0.25   0.25    -     testq	%rsi, %rdi
 # CHECK-NEXT:  -      -     0.25   0.25   0.50   0.50    -     0.25   0.25    -     testq	%rsi, (%rax)
 # CHECK-NEXT:  -      -     0.25   0.25    -      -      -     0.25   0.25    -     ud2
+# CHECK-NEXT:  -      -     0.25   0.25    -      -      -     0.25   0.25    -     wrmsr
 # CHECK-NEXT:  -      -     0.75   0.75    -      -      -     0.75   0.75    -     xaddb	%bl, %cl
 # CHECK-NEXT:  -      -     0.50   0.50   0.83   0.83   1.00   0.50   0.50   0.33   xaddb	%bl, (%rcx)
 # CHECK-NEXT:  -      -     0.75   0.75    -      -      -     0.75   0.75    -     xaddw	%bx, %cx
diff --git a/test/tools/llvm-mca/X86/Broadwell/resources-x87.s b/test/tools/llvm-mca/X86/Broadwell/resources-x87.s
index 5cb92be47eabb01d5f9e9d97eeb96e157e18b0b0..2f3a69da99d074855e7e3f28f1c783381aca74a0 100644
--- a/test/tools/llvm-mca/X86/Broadwell/resources-x87.s
+++ b/test/tools/llvm-mca/X86/Broadwell/resources-x87.s
@@ -5,7 +5,7 @@ f2xm1
 
 fabs
 
-fadd %st(0), %st(1)
+fadd %st, %st(1)
 fadd %st(2)
 fadds (%ecx)
 faddl (%ecx)
@@ -21,14 +21,14 @@ fchs
 
 fnclex
 
-fcmovb %st(1), %st(0)
-fcmovbe %st(1), %st(0)
-fcmove %st(1), %st(0)
-fcmovnb %st(1), %st(0)
-fcmovnbe %st(1), %st(0)
-fcmovne %st(1), %st(0)
-fcmovnu %st(1), %st(0)
-fcmovu %st(1), %st(0)
+fcmovb %st(1), %st
+fcmovbe %st(1), %st
+fcmove %st(1), %st
+fcmovnb %st(1), %st
+fcmovnbe %st(1), %st
+fcmovne %st(1), %st
+fcmovnu %st(1), %st
+fcmovu %st(1), %st
 
 fcom %st(1)
 fcom %st(3)
@@ -47,7 +47,7 @@ fcos
 
 fdecstp
 
-fdiv %st(0), %st(1)
+fdiv %st, %st(1)
 fdiv %st(2)
 fdivs (%ecx)
 fdivl (%eax)
@@ -56,7 +56,7 @@ fdivp %st(2)
 fidivs (%ecx)
 fidivl (%eax)
 
-fdivr %st(0), %st(1)
+fdivr %st, %st(1)
 fdivr %st(2)
 fdivrs (%ecx)
 fdivrl (%eax)
@@ -106,7 +106,7 @@ fldln2
 fldpi
 fldz
 
-fmul %st(0), %st(1)
+fmul %st, %st(1)
 fmul %st(2)
 fmuls (%ecx)
 fmull (%eax)
@@ -153,7 +153,7 @@ fnstsw (%eax)
 frstor (%eax)
 fsave (%eax)
 
-fsub %st(0), %st(1)
+fsub %st, %st(1)
 fsub %st(2)
 fsubs (%ecx)
 fsubl (%eax)
@@ -162,7 +162,7 @@ fsubp %st(2)
 fisubs (%ecx)
 fisubl (%eax)
 
-fsubr %st(0), %st(1)
+fsubr %st, %st(1)
 fsubr %st(2)
 fsubrs (%ecx)
 fsubrl (%eax)
@@ -208,26 +208,26 @@ fyl2xp1
 # CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
 # CHECK-NEXT:  1      100   0.25                  U     f2xm1
 # CHECK-NEXT:  1      1     1.00                  U     fabs
-# CHECK-NEXT:  1      3     1.00                  U     fadd	%st(0), %st(1)
-# CHECK-NEXT:  1      3     1.00                  U     fadd	%st(2)
+# CHECK-NEXT:  1      3     1.00                  U     fadd	%st, %st(1)
+# CHECK-NEXT:  1      3     1.00                  U     fadd	%st(2), %st
 # CHECK-NEXT:  2      9     1.00    *             U     fadds	(%ecx)
 # CHECK-NEXT:  2      9     1.00    *             U     faddl	(%ecx)
-# CHECK-NEXT:  1      3     1.00                  U     faddp	%st(1)
-# CHECK-NEXT:  1      3     1.00                  U     faddp	%st(2)
+# CHECK-NEXT:  1      3     1.00                  U     faddp	%st, %st(1)
+# CHECK-NEXT:  1      3     1.00                  U     faddp	%st, %st(2)
 # CHECK-NEXT:  3      12    2.00    *             U     fiadds	(%ecx)
 # CHECK-NEXT:  3      12    2.00    *             U     fiaddl	(%ecx)
 # CHECK-NEXT:  1      100   0.25                  U     fbld	(%ecx)
 # CHECK-NEXT:  2      1     1.00                  U     fbstp	(%eax)
 # CHECK-NEXT:  1      1     1.00                  U     fchs
 # CHECK-NEXT:  4      4     1.00                  U     fnclex
-# CHECK-NEXT:  1      3     1.00                  U     fcmovb	%st(1), %st(0)
-# CHECK-NEXT:  1      3     1.00                  U     fcmovbe	%st(1), %st(0)
-# CHECK-NEXT:  1      3     1.00                  U     fcmove	%st(1), %st(0)
-# CHECK-NEXT:  1      3     1.00                  U     fcmovnb	%st(1), %st(0)
-# CHECK-NEXT:  1      3     1.00                  U     fcmovnbe	%st(1), %st(0)
-# CHECK-NEXT:  1      3     1.00                  U     fcmovne	%st(1), %st(0)
-# CHECK-NEXT:  1      3     1.00                  U     fcmovnu	%st(1), %st(0)
-# CHECK-NEXT:  1      3     1.00                  U     fcmovu	%st(1), %st(0)
+# CHECK-NEXT:  1      3     1.00                  U     fcmovb	%st(1), %st
+# CHECK-NEXT:  1      3     1.00                  U     fcmovbe	%st(1), %st
+# CHECK-NEXT:  1      3     1.00                  U     fcmove	%st(1), %st
+# CHECK-NEXT:  1      3     1.00                  U     fcmovnb	%st(1), %st
+# CHECK-NEXT:  1      3     1.00                  U     fcmovnbe	%st(1), %st
+# CHECK-NEXT:  1      3     1.00                  U     fcmovne	%st(1), %st
+# CHECK-NEXT:  1      3     1.00                  U     fcmovnu	%st(1), %st
+# CHECK-NEXT:  1      3     1.00                  U     fcmovu	%st(1), %st
 # CHECK-NEXT:  1      1     1.00                  U     fcom	%st(1)
 # CHECK-NEXT:  1      1     1.00                  U     fcom	%st(3)
 # CHECK-NEXT:  2      7     1.00                  U     fcoms	(%ecx)
@@ -237,24 +237,24 @@ fyl2xp1
 # CHECK-NEXT:  2      7     1.00                  U     fcomps	(%ecx)
 # CHECK-NEXT:  2      7     1.00                  U     fcompl	(%eax)
 # CHECK-NEXT:  1      100   0.25                  U     fcompp
-# CHECK-NEXT:  1      3     1.00                  U     fcomi	%st(3)
-# CHECK-NEXT:  1      3     1.00                  U     fcompi	%st(3)
+# CHECK-NEXT:  1      3     1.00                  U     fcomi	%st(3), %st
+# CHECK-NEXT:  1      3     1.00                  U     fcompi	%st(3), %st
 # CHECK-NEXT:  1      100   0.25                  U     fcos
 # CHECK-NEXT:  2      2     1.00                  U     fdecstp
-# CHECK-NEXT:  1      15    1.00                  U     fdiv	%st(0), %st(1)
-# CHECK-NEXT:  1      20    1.00                  U     fdiv	%st(2)
+# CHECK-NEXT:  1      15    1.00                  U     fdiv	%st, %st(1)
+# CHECK-NEXT:  1      20    1.00                  U     fdiv	%st(2), %st
 # CHECK-NEXT:  2      21    1.00    *             U     fdivs	(%ecx)
 # CHECK-NEXT:  2      21    1.00    *             U     fdivl	(%eax)
-# CHECK-NEXT:  1      15    1.00                  U     fdivp	%st(1)
-# CHECK-NEXT:  1      15    1.00                  U     fdivp	%st(2)
+# CHECK-NEXT:  1      15    1.00                  U     fdivp	%st, %st(1)
+# CHECK-NEXT:  1      15    1.00                  U     fdivp	%st, %st(2)
 # CHECK-NEXT:  3      24    1.00    *             U     fidivs	(%ecx)
 # CHECK-NEXT:  3      24    1.00    *             U     fidivl	(%eax)
-# CHECK-NEXT:  1      20    1.00                  U     fdivr	%st(0), %st(1)
-# CHECK-NEXT:  1      15    1.00                  U     fdivr	%st(2)
+# CHECK-NEXT:  1      20    1.00                  U     fdivr	%st, %st(1)
+# CHECK-NEXT:  1      15    1.00                  U     fdivr	%st(2), %st
 # CHECK-NEXT:  2      26    1.00    *             U     fdivrs	(%ecx)
 # CHECK-NEXT:  2      26    1.00    *             U     fdivrl	(%eax)
-# CHECK-NEXT:  1      20    1.00                  U     fdivrp	%st(1)
-# CHECK-NEXT:  1      20    1.00                  U     fdivrp	%st(2)
+# CHECK-NEXT:  1      20    1.00                  U     fdivrp	%st, %st(1)
+# CHECK-NEXT:  1      20    1.00                  U     fdivrp	%st, %st(2)
 # CHECK-NEXT:  3      29    1.00    *             U     fidivrs	(%ecx)
 # CHECK-NEXT:  3      29    1.00    *             U     fidivrl	(%eax)
 # CHECK-NEXT:  1      100   0.25                  U     ffree	%st(0)
@@ -288,12 +288,12 @@ fyl2xp1
 # CHECK-NEXT:  2      1     1.00                  U     fldln2
 # CHECK-NEXT:  2      1     1.00                  U     fldpi
 # CHECK-NEXT:  1      1     0.50                  U     fldz
-# CHECK-NEXT:  1      5     1.00                  U     fmul	%st(0), %st(1)
-# CHECK-NEXT:  1      5     1.00                  U     fmul	%st(2)
+# CHECK-NEXT:  1      5     1.00                  U     fmul	%st, %st(1)
+# CHECK-NEXT:  1      5     1.00                  U     fmul	%st(2), %st
 # CHECK-NEXT:  2      11    1.00    *             U     fmuls	(%ecx)
 # CHECK-NEXT:  2      11    1.00    *             U     fmull	(%eax)
-# CHECK-NEXT:  1      5     1.00                  U     fmulp	%st(1)
-# CHECK-NEXT:  1      5     1.00                  U     fmulp	%st(2)
+# CHECK-NEXT:  1      5     1.00                  U     fmulp	%st, %st(1)
+# CHECK-NEXT:  1      5     1.00                  U     fmulp	%st, %st(2)
 # CHECK-NEXT:  3      14    1.00    *             U     fimuls	(%ecx)
 # CHECK-NEXT:  3      14    1.00    *             U     fimull	(%eax)
 # CHECK-NEXT:  1      1     0.50                  U     fnop
@@ -321,20 +321,20 @@ fyl2xp1
 # CHECK-NEXT:  1      100   0.25                  U     frstor	(%eax)
 # CHECK-NEXT:  2      2     0.50                  U     wait
 # CHECK-NEXT:  1      100   0.25                  U     fnsave	(%eax)
-# CHECK-NEXT:  1      3     1.00                  U     fsub	%st(0), %st(1)
-# CHECK-NEXT:  1      3     1.00                  U     fsub	%st(2)
+# CHECK-NEXT:  1      3     1.00                  U     fsub	%st, %st(1)
+# CHECK-NEXT:  1      3     1.00                  U     fsub	%st(2), %st
 # CHECK-NEXT:  2      9     1.00    *             U     fsubs	(%ecx)
 # CHECK-NEXT:  2      9     1.00    *             U     fsubl	(%eax)
-# CHECK-NEXT:  1      3     1.00                  U     fsubp	%st(1)
-# CHECK-NEXT:  1      3     1.00                  U     fsubp	%st(2)
+# CHECK-NEXT:  1      3     1.00                  U     fsubp	%st, %st(1)
+# CHECK-NEXT:  1      3     1.00                  U     fsubp	%st, %st(2)
 # CHECK-NEXT:  3      12    2.00    *             U     fisubs	(%ecx)
 # CHECK-NEXT:  3      12    2.00    *             U     fisubl	(%eax)
-# CHECK-NEXT:  1      3     1.00                  U     fsubr	%st(0), %st(1)
-# CHECK-NEXT:  1      3     1.00                  U     fsubr	%st(2)
+# CHECK-NEXT:  1      3     1.00                  U     fsubr	%st, %st(1)
+# CHECK-NEXT:  1      3     1.00                  U     fsubr	%st(2), %st
 # CHECK-NEXT:  2      9     1.00    *             U     fsubrs	(%ecx)
 # CHECK-NEXT:  2      9     1.00    *             U     fsubrl	(%eax)
-# CHECK-NEXT:  1      3     1.00                  U     fsubrp	%st(1)
-# CHECK-NEXT:  1      3     1.00                  U     fsubrp	%st(2)
+# CHECK-NEXT:  1      3     1.00                  U     fsubrp	%st, %st(1)
+# CHECK-NEXT:  1      3     1.00                  U     fsubrp	%st, %st(2)
 # CHECK-NEXT:  3      12    2.00    *             U     fisubrs	(%ecx)
 # CHECK-NEXT:  3      12    2.00    *             U     fisubrl	(%eax)
 # CHECK-NEXT:  1      3     1.00                  U     ftst
@@ -343,8 +343,8 @@ fyl2xp1
 # CHECK-NEXT:  1      1     1.00                  U     fucomp	%st(1)
 # CHECK-NEXT:  1      1     1.00                  U     fucomp	%st(3)
 # CHECK-NEXT:  1      3     1.00                  U     fucompp
-# CHECK-NEXT:  1      3     1.00                  U     fucomi	%st(3)
-# CHECK-NEXT:  1      3     1.00                  U     fucompi	%st(3)
+# CHECK-NEXT:  1      3     1.00                  U     fucomi	%st(3), %st
+# CHECK-NEXT:  1      3     1.00                  U     fucompi	%st(3), %st
 # CHECK-NEXT:  2      2     0.50                  U     wait
 # CHECK-NEXT:  1      100   0.25                  U     fxam
 # CHECK-NEXT:  12     14    4.00                  U     fxch	%st(1)
@@ -375,26 +375,26 @@ fyl2xp1
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    Instructions:
 # CHECK-NEXT:  -      -     0.25   0.25    -      -      -     0.25   0.25    -     f2xm1
 # CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     fabs
-# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     fadd	%st(0), %st(1)
-# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     fadd	%st(2)
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     fadd	%st, %st(1)
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     fadd	%st(2), %st
 # CHECK-NEXT:  -      -      -     1.00   0.50   0.50    -      -      -      -     fadds	(%ecx)
 # CHECK-NEXT:  -      -      -     1.00   0.50   0.50    -      -      -      -     faddl	(%ecx)
-# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     faddp	%st(1)
-# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     faddp	%st(2)
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     faddp	%st, %st(1)
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     faddp	%st, %st(2)
 # CHECK-NEXT:  -      -      -     2.00   0.50   0.50    -      -      -      -     fiadds	(%ecx)
 # CHECK-NEXT:  -      -      -     2.00   0.50   0.50    -      -      -      -     fiaddl	(%ecx)
 # CHECK-NEXT:  -      -     0.25   0.25    -      -      -     0.25   0.25    -     fbld	(%ecx)
 # CHECK-NEXT:  -      -      -      -     0.33   0.33   1.00    -      -     0.33   fbstp	(%eax)
 # CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     fchs
 # CHECK-NEXT:  -      -     1.00   1.00    -      -      -     1.00   1.00    -     fnclex
-# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     fcmovb	%st(1), %st(0)
-# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     fcmovbe	%st(1), %st(0)
-# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     fcmove	%st(1), %st(0)
-# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     fcmovnb	%st(1), %st(0)
-# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     fcmovnbe	%st(1), %st(0)
-# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     fcmovne	%st(1), %st(0)
-# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     fcmovnu	%st(1), %st(0)
-# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     fcmovu	%st(1), %st(0)
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     fcmovb	%st(1), %st
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     fcmovbe	%st(1), %st
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     fcmove	%st(1), %st
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     fcmovnb	%st(1), %st
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     fcmovnbe	%st(1), %st
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     fcmovne	%st(1), %st
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     fcmovnu	%st(1), %st
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     fcmovu	%st(1), %st
 # CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     fcom	%st(1)
 # CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     fcom	%st(3)
 # CHECK-NEXT:  -      -      -     1.00   0.50   0.50    -      -      -      -     fcoms	(%ecx)
@@ -404,24 +404,24 @@ fyl2xp1
 # CHECK-NEXT:  -      -      -     1.00   0.50   0.50    -      -      -      -     fcomps	(%ecx)
 # CHECK-NEXT:  -      -      -     1.00   0.50   0.50    -      -      -      -     fcompl	(%eax)
 # CHECK-NEXT:  -      -     0.25   0.25    -      -      -     0.25   0.25    -     fcompp
-# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     fcomi	%st(3)
-# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     fcompi	%st(3)
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     fcomi	%st(3), %st
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     fcompi	%st(3), %st
 # CHECK-NEXT:  -      -     0.25   0.25    -      -      -     0.25   0.25    -     fcos
 # CHECK-NEXT:  -      -     1.00   1.00    -      -      -      -      -      -     fdecstp
-# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     fdiv	%st(0), %st(1)
-# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     fdiv	%st(2)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     fdiv	%st, %st(1)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     fdiv	%st(2), %st
 # CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -      -      -      -     fdivs	(%ecx)
 # CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -      -      -      -     fdivl	(%eax)
-# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     fdivp	%st(1)
-# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     fdivp	%st(2)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     fdivp	%st, %st(1)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     fdivp	%st, %st(2)
 # CHECK-NEXT:  -      -     1.00   1.00   0.50   0.50    -      -      -      -     fidivs	(%ecx)
 # CHECK-NEXT:  -      -     1.00   1.00   0.50   0.50    -      -      -      -     fidivl	(%eax)
-# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     fdivr	%st(0), %st(1)
-# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     fdivr	%st(2)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     fdivr	%st, %st(1)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     fdivr	%st(2), %st
 # CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -      -      -      -     fdivrs	(%ecx)
 # CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -      -      -      -     fdivrl	(%eax)
-# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     fdivrp	%st(1)
-# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     fdivrp	%st(2)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     fdivrp	%st, %st(1)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     fdivrp	%st, %st(2)
 # CHECK-NEXT:  -      -     1.00   1.00   0.50   0.50    -      -      -      -     fidivrs	(%ecx)
 # CHECK-NEXT:  -      -     1.00   1.00   0.50   0.50    -      -      -      -     fidivrl	(%eax)
 # CHECK-NEXT:  -      -     0.25   0.25    -      -      -     0.25   0.25    -     ffree	%st(0)
@@ -455,12 +455,12 @@ fyl2xp1
 # CHECK-NEXT:  -      -     1.00   1.00    -      -      -      -      -      -     fldln2
 # CHECK-NEXT:  -      -     1.00   1.00    -      -      -      -      -      -     fldpi
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -     fldz
-# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     fmul	%st(0), %st(1)
-# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     fmul	%st(2)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     fmul	%st, %st(1)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     fmul	%st(2), %st
 # CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -      -      -      -     fmuls	(%ecx)
 # CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -      -      -      -     fmull	(%eax)
-# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     fmulp	%st(1)
-# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     fmulp	%st(2)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     fmulp	%st, %st(1)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     fmulp	%st, %st(2)
 # CHECK-NEXT:  -      -     1.00   1.00   0.50   0.50    -      -      -      -     fimuls	(%ecx)
 # CHECK-NEXT:  -      -     1.00   1.00   0.50   0.50    -      -      -      -     fimull	(%eax)
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -     fnop
@@ -488,20 +488,20 @@ fyl2xp1
 # CHECK-NEXT:  -      -     0.25   0.25    -      -      -     0.25   0.25    -     frstor	(%eax)
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -     0.50   0.50    -     wait
 # CHECK-NEXT:  -      -     0.25   0.25    -      -      -     0.25   0.25    -     fnsave	(%eax)
-# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     fsub	%st(0), %st(1)
-# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     fsub	%st(2)
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     fsub	%st, %st(1)
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     fsub	%st(2), %st
 # CHECK-NEXT:  -      -      -     1.00   0.50   0.50    -      -      -      -     fsubs	(%ecx)
 # CHECK-NEXT:  -      -      -     1.00   0.50   0.50    -      -      -      -     fsubl	(%eax)
-# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     fsubp	%st(1)
-# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     fsubp	%st(2)
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     fsubp	%st, %st(1)
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     fsubp	%st, %st(2)
 # CHECK-NEXT:  -      -      -     2.00   0.50   0.50    -      -      -      -     fisubs	(%ecx)
 # CHECK-NEXT:  -      -      -     2.00   0.50   0.50    -      -      -      -     fisubl	(%eax)
-# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     fsubr	%st(0), %st(1)
-# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     fsubr	%st(2)
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     fsubr	%st, %st(1)
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     fsubr	%st(2), %st
 # CHECK-NEXT:  -      -      -     1.00   0.50   0.50    -      -      -      -     fsubrs	(%ecx)
 # CHECK-NEXT:  -      -      -     1.00   0.50   0.50    -      -      -      -     fsubrl	(%eax)
-# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     fsubrp	%st(1)
-# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     fsubrp	%st(2)
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     fsubrp	%st, %st(1)
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     fsubrp	%st, %st(2)
 # CHECK-NEXT:  -      -      -     2.00   0.50   0.50    -      -      -      -     fisubrs	(%ecx)
 # CHECK-NEXT:  -      -      -     2.00   0.50   0.50    -      -      -      -     fisubrl	(%eax)
 # CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     ftst
@@ -510,8 +510,8 @@ fyl2xp1
 # CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     fucomp	%st(1)
 # CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     fucomp	%st(3)
 # CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     fucompp
-# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     fucomi	%st(3)
-# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     fucompi	%st(3)
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     fucomi	%st(3), %st
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     fucompi	%st(3), %st
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -     0.50   0.50    -     wait
 # CHECK-NEXT:  -      -     0.25   0.25    -      -      -     0.25   0.25    -     fxam
 # CHECK-NEXT:  -      -     3.25   2.25    -      -      -     1.25   5.25    -     fxch	%st(1)
diff --git a/test/tools/llvm-mca/X86/BtVer2/dot-product.s b/test/tools/llvm-mca/X86/BtVer2/dot-product.s
index 643e456450cdbd2f778d047ff71dd55f07c1a4a2..a43b8285a5186b5f83ed5fee1aec34c128a16941 100644
--- a/test/tools/llvm-mca/X86/BtVer2/dot-product.s
+++ b/test/tools/llvm-mca/X86/BtVer2/dot-product.s
@@ -7,12 +7,12 @@ vhaddps  %xmm3, %xmm3, %xmm4
 
 # CHECK:      Iterations:        300
 # CHECK-NEXT: Instructions:      900
-# CHECK-NEXT: Total Cycles:      610
+# CHECK-NEXT: Total Cycles:      611
 # CHECK-NEXT: Total uOps:        900
 
 # CHECK:      Dispatch Width:    2
-# CHECK-NEXT: uOps Per Cycle:    1.48
-# CHECK-NEXT: IPC:               1.48
+# CHECK-NEXT: uOps Per Cycle:    1.47
+# CHECK-NEXT: IPC:               1.47
 # CHECK-NEXT: Block RThroughput: 2.0
 
 # CHECK:      Instruction Info:
@@ -25,8 +25,8 @@ vhaddps  %xmm3, %xmm3, %xmm4
 
 # CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
 # CHECK-NEXT:  1      2     1.00                        vmulps	%xmm0, %xmm1, %xmm2
-# CHECK-NEXT:  1      3     1.00                        vhaddps	%xmm2, %xmm2, %xmm3
-# CHECK-NEXT:  1      3     1.00                        vhaddps	%xmm3, %xmm3, %xmm4
+# CHECK-NEXT:  1      4     1.00                        vhaddps	%xmm2, %xmm2, %xmm3
+# CHECK-NEXT:  1      4     1.00                        vhaddps	%xmm3, %xmm3, %xmm4
 
 # CHECK:      Resources:
 # CHECK-NEXT: [0]   - JALU0
@@ -59,14 +59,14 @@ vhaddps  %xmm3, %xmm3, %xmm4
 # CHECK-NEXT: Index     0123456789
 
 # CHECK:      [0,0]     DeeER.    .    .   vmulps	%xmm0, %xmm1, %xmm2
-# CHECK-NEXT: [0,1]     D==eeeER  .    .   vhaddps	%xmm2, %xmm2, %xmm3
-# CHECK-NEXT: [0,2]     .D====eeeER    .   vhaddps	%xmm3, %xmm3, %xmm4
-# CHECK-NEXT: [1,0]     .DeeE-----R    .   vmulps	%xmm0, %xmm1, %xmm2
-# CHECK-NEXT: [1,1]     . D=eeeE---R   .   vhaddps	%xmm2, %xmm2, %xmm3
-# CHECK-NEXT: [1,2]     . D====eeeER   .   vhaddps	%xmm3, %xmm3, %xmm4
-# CHECK-NEXT: [2,0]     .  DeeE-----R  .   vmulps	%xmm0, %xmm1, %xmm2
-# CHECK-NEXT: [2,1]     .  D====eeeER  .   vhaddps	%xmm2, %xmm2, %xmm3
-# CHECK-NEXT: [2,2]     .   D======eeeER   vhaddps	%xmm3, %xmm3, %xmm4
+# CHECK-NEXT: [0,1]     D==eeeeER .    .   vhaddps	%xmm2, %xmm2, %xmm3
+# CHECK-NEXT: [0,2]     .D=====eeeeER  .   vhaddps	%xmm3, %xmm3, %xmm4
+# CHECK-NEXT: [1,0]     .DeeE-------R  .   vmulps	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT: [1,1]     . D=eeeeE----R .   vhaddps	%xmm2, %xmm2, %xmm3
+# CHECK-NEXT: [1,2]     . D=====eeeeER .   vhaddps	%xmm3, %xmm3, %xmm4
+# CHECK-NEXT: [2,0]     .  DeeE-------R.   vmulps	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT: [2,1]     .  D==eeeeE---R.   vhaddps	%xmm2, %xmm2, %xmm3
+# CHECK-NEXT: [2,2]     .   D=====eeeeER   vhaddps	%xmm3, %xmm3, %xmm4
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
@@ -75,6 +75,6 @@ vhaddps  %xmm3, %xmm3, %xmm4
 # CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
 
 # CHECK:            [0]    [1]    [2]    [3]
-# CHECK-NEXT: 0.     3     1.0    1.0    3.3       vmulps	%xmm0, %xmm1, %xmm2
-# CHECK-NEXT: 1.     3     3.3    0.7    1.0       vhaddps	%xmm2, %xmm2, %xmm3
-# CHECK-NEXT: 2.     3     5.7    0.0    0.0       vhaddps	%xmm3, %xmm3, %xmm4
+# CHECK-NEXT: 0.     3     1.0    1.0    4.7       vmulps	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT: 1.     3     2.7    0.0    2.3       vhaddps	%xmm2, %xmm2, %xmm3
+# CHECK-NEXT: 2.     3     6.0    0.0    0.0       vhaddps	%xmm3, %xmm3, %xmm4
diff --git a/test/tools/llvm-mca/X86/BtVer2/hadd-read-after-ld-1.s b/test/tools/llvm-mca/X86/BtVer2/hadd-read-after-ld-1.s
index 87862a6e5a33c9b024f8cf494823e48e7d419da0..197c1dce581513b7874383fbab48169c8d2fbcc2 100644
--- a/test/tools/llvm-mca/X86/BtVer2/hadd-read-after-ld-1.s
+++ b/test/tools/llvm-mca/X86/BtVer2/hadd-read-after-ld-1.s
@@ -6,12 +6,12 @@ vhaddps (%rdi), %xmm1, %xmm2
 
 # CHECK:      Iterations:        1
 # CHECK-NEXT: Instructions:      2
-# CHECK-NEXT: Total Cycles:      11
+# CHECK-NEXT: Total Cycles:      12
 # CHECK-NEXT: Total uOps:        2
 
 # CHECK:      Dispatch Width:    2
-# CHECK-NEXT: uOps Per Cycle:    0.18
-# CHECK-NEXT: IPC:               0.18
+# CHECK-NEXT: uOps Per Cycle:    0.17
+# CHECK-NEXT: IPC:               0.17
 # CHECK-NEXT: Block RThroughput: 1.0
 
 # CHECK:      Instruction Info:
@@ -24,14 +24,14 @@ vhaddps (%rdi), %xmm1, %xmm2
 
 # CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
 # CHECK-NEXT:  1      1     0.50                        vshufps	$0, %xmm0, %xmm1, %xmm1
-# CHECK-NEXT:  1      8     1.00    *                   vhaddps	(%rdi), %xmm1, %xmm2
+# CHECK-NEXT:  1      9     1.00    *                   vhaddps	(%rdi), %xmm1, %xmm2
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     0
+# CHECK-NEXT:                     01
 # CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeER .    .   vshufps	$0, %xmm0, %xmm1, %xmm1
-# CHECK-NEXT: [0,1]     DeeeeeeeeER   vhaddps	(%rdi), %xmm1, %xmm2
+# CHECK:      [0,0]     DeER .    ..   vshufps	$0, %xmm0, %xmm1, %xmm1
+# CHECK-NEXT: [0,1]     DeeeeeeeeeER   vhaddps	(%rdi), %xmm1, %xmm2
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
diff --git a/test/tools/llvm-mca/X86/BtVer2/hadd-read-after-ld-2.s b/test/tools/llvm-mca/X86/BtVer2/hadd-read-after-ld-2.s
index 80d5109d07ee1247a86e5aaf085ab8066b5cf99e..e64ee28103f31467f8798afbcdd84ea88c25e9c2 100644
--- a/test/tools/llvm-mca/X86/BtVer2/hadd-read-after-ld-2.s
+++ b/test/tools/llvm-mca/X86/BtVer2/hadd-read-after-ld-2.s
@@ -6,12 +6,12 @@ vhaddps (%rdi), %ymm1, %ymm2
 
 # CHECK:      Iterations:        1
 # CHECK-NEXT: Instructions:      2
-# CHECK-NEXT: Total Cycles:      12
+# CHECK-NEXT: Total Cycles:      13
 # CHECK-NEXT: Total uOps:        3
 
 # CHECK:      Dispatch Width:    2
-# CHECK-NEXT: uOps Per Cycle:    0.25
-# CHECK-NEXT: IPC:               0.17
+# CHECK-NEXT: uOps Per Cycle:    0.23
+# CHECK-NEXT: IPC:               0.15
 # CHECK-NEXT: Block RThroughput: 2.0
 
 # CHECK:      Instruction Info:
@@ -24,14 +24,14 @@ vhaddps (%rdi), %ymm1, %ymm2
 
 # CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
 # CHECK-NEXT:  1      1     0.50                        vshufps	$0, %xmm0, %xmm1, %xmm1
-# CHECK-NEXT:  2      8     2.00    *                   vhaddps	(%rdi), %ymm1, %ymm2
+# CHECK-NEXT:  2      9     2.00    *                   vhaddps	(%rdi), %ymm1, %ymm2
 
 # CHECK:      Timeline view:
-# CHECK-NEXT:                     01
+# CHECK-NEXT:                     012
 # CHECK-NEXT: Index     0123456789
 
-# CHECK:      [0,0]     DeER .    ..   vshufps	$0, %xmm0, %xmm1, %xmm1
-# CHECK-NEXT: [0,1]     .DeeeeeeeeER   vhaddps	(%rdi), %ymm1, %ymm2
+# CHECK:      [0,0]     DeER .    . .   vshufps	$0, %xmm0, %xmm1, %xmm1
+# CHECK-NEXT: [0,1]     .DeeeeeeeeeER   vhaddps	(%rdi), %ymm1, %ymm2
 
 # CHECK:      Average Wait times (based on the timeline view):
 # CHECK-NEXT: [0]: Executions
diff --git a/test/tools/llvm-mca/X86/BtVer2/instruction-info-view.s b/test/tools/llvm-mca/X86/BtVer2/instruction-info-view.s
index fa19b55c1d189aa2440514c49c6a594f958fd8ac..8f17c44e0ef4f33a463e00f9c6785a03d00dc6d9 100644
--- a/test/tools/llvm-mca/X86/BtVer2/instruction-info-view.s
+++ b/test/tools/llvm-mca/X86/BtVer2/instruction-info-view.s
@@ -13,13 +13,13 @@ vhaddps  %xmm3, %xmm3, %xmm4
 
 # ENABLED:       Iterations:        100
 # ENABLED-NEXT:  Instructions:      300
-# ENABLED-NEXT:  Total Cycles:      209
+# ENABLED-NEXT:  Total Cycles:      211
 # ENABLED-NEXT:  Total uOps:        300
 
 
 # ENABLED:       Dispatch Width:    2
-# ENABLED-NEXT:  uOps Per Cycle:    1.44
-# ENABLED-NEXT:  IPC:               1.44
+# ENABLED-NEXT:  uOps Per Cycle:    1.42
+# ENABLED-NEXT:  IPC:               1.42
 # ENABLED-NEXT:  Block RThroughput: 2.0
 
 # ENABLED:       Instruction Info:
@@ -32,5 +32,5 @@ vhaddps  %xmm3, %xmm3, %xmm4
 
 # ENABLED:       [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
 # ENABLED-NEXT:   1      2     1.00                        vmulps	%xmm0, %xmm1, %xmm2
-# ENABLED-NEXT:   1      3     1.00                        vhaddps	%xmm2, %xmm2, %xmm3
-# ENABLED-NEXT:   1      3     1.00                        vhaddps	%xmm3, %xmm3, %xmm4
+# ENABLED-NEXT:   1      4     1.00                        vhaddps	%xmm2, %xmm2, %xmm3
+# ENABLED-NEXT:   1      4     1.00                        vhaddps	%xmm3, %xmm3, %xmm4
diff --git a/test/tools/llvm-mca/X86/BtVer2/int-to-fpu-forwarding-1.s b/test/tools/llvm-mca/X86/BtVer2/int-to-fpu-forwarding-1.s
new file mode 100644
index 0000000000000000000000000000000000000000..398a52a84791317573ce0a7821a2ab3bc03fa35b
--- /dev/null
+++ b/test/tools/llvm-mca/X86/BtVer2/int-to-fpu-forwarding-1.s
@@ -0,0 +1,220 @@
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=x86_64-unknown-unknown -mcpu=btver2 -iterations=500 < %s | FileCheck %s
+
+# Throughput for all the code snippet below should be 1.00 IPC.
+
+# LLVM-MCA-BEGIN
+vpinsrb $0, %eax, %xmm0, %xmm0
+vpinsrb $1, %eax, %xmm0, %xmm0
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN
+vpinsrw $0, %eax, %xmm0, %xmm0
+vpinsrw $1, %eax, %xmm0, %xmm0
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN
+vpinsrd $0, %eax, %xmm0, %xmm0
+vpinsrd $1, %eax, %xmm0, %xmm0
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN
+vpinsrq $0, %rax, %xmm0, %xmm0
+vpinsrq $1, %rax, %xmm0, %xmm0
+# LLVM-MCA-END
+
+# CHECK:      [0] Code Region
+
+# CHECK:      Iterations:        500
+# CHECK-NEXT: Instructions:      1000
+# CHECK-NEXT: Total Cycles:      1003
+# CHECK-NEXT: Total uOps:        2000
+
+# CHECK:      Dispatch Width:    2
+# CHECK-NEXT: uOps Per Cycle:    1.99
+# CHECK-NEXT: IPC:               1.00
+# CHECK-NEXT: Block RThroughput: 2.0
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  2      7     0.50                        vpinsrb	$0, %eax, %xmm0, %xmm0
+# CHECK-NEXT:  2      7     0.50                        vpinsrb	$1, %eax, %xmm0, %xmm0
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0]   - JALU0
+# CHECK-NEXT: [1]   - JALU1
+# CHECK-NEXT: [2]   - JDiv
+# CHECK-NEXT: [3]   - JFPA
+# CHECK-NEXT: [4]   - JFPM
+# CHECK-NEXT: [5]   - JFPU0
+# CHECK-NEXT: [6]   - JFPU1
+# CHECK-NEXT: [7]   - JLAGU
+# CHECK-NEXT: [8]   - JMul
+# CHECK-NEXT: [9]   - JSAGU
+# CHECK-NEXT: [10]  - JSTC
+# CHECK-NEXT: [11]  - JVALU0
+# CHECK-NEXT: [12]  - JVALU1
+# CHECK-NEXT: [13]  - JVIMUL
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]
+# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -     1.00   1.00    -
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   Instructions:
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -     vpinsrb	$0, %eax, %xmm0, %xmm0
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -     vpinsrb	$1, %eax, %xmm0, %xmm0
+
+# CHECK:      [1] Code Region
+
+# CHECK:      Iterations:        500
+# CHECK-NEXT: Instructions:      1000
+# CHECK-NEXT: Total Cycles:      1003
+# CHECK-NEXT: Total uOps:        2000
+
+# CHECK:      Dispatch Width:    2
+# CHECK-NEXT: uOps Per Cycle:    1.99
+# CHECK-NEXT: IPC:               1.00
+# CHECK-NEXT: Block RThroughput: 2.0
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  2      7     0.50                        vpinsrw	$0, %eax, %xmm0, %xmm0
+# CHECK-NEXT:  2      7     0.50                        vpinsrw	$1, %eax, %xmm0, %xmm0
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0]   - JALU0
+# CHECK-NEXT: [1]   - JALU1
+# CHECK-NEXT: [2]   - JDiv
+# CHECK-NEXT: [3]   - JFPA
+# CHECK-NEXT: [4]   - JFPM
+# CHECK-NEXT: [5]   - JFPU0
+# CHECK-NEXT: [6]   - JFPU1
+# CHECK-NEXT: [7]   - JLAGU
+# CHECK-NEXT: [8]   - JMul
+# CHECK-NEXT: [9]   - JSAGU
+# CHECK-NEXT: [10]  - JSTC
+# CHECK-NEXT: [11]  - JVALU0
+# CHECK-NEXT: [12]  - JVALU1
+# CHECK-NEXT: [13]  - JVIMUL
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]
+# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -     1.00   1.00    -
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   Instructions:
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -     vpinsrw	$0, %eax, %xmm0, %xmm0
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -     vpinsrw	$1, %eax, %xmm0, %xmm0
+
+# CHECK:      [2] Code Region
+
+# CHECK:      Iterations:        500
+# CHECK-NEXT: Instructions:      1000
+# CHECK-NEXT: Total Cycles:      1003
+# CHECK-NEXT: Total uOps:        2000
+
+# CHECK:      Dispatch Width:    2
+# CHECK-NEXT: uOps Per Cycle:    1.99
+# CHECK-NEXT: IPC:               1.00
+# CHECK-NEXT: Block RThroughput: 2.0
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  2      7     0.50                        vpinsrd	$0, %eax, %xmm0, %xmm0
+# CHECK-NEXT:  2      7     0.50                        vpinsrd	$1, %eax, %xmm0, %xmm0
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0]   - JALU0
+# CHECK-NEXT: [1]   - JALU1
+# CHECK-NEXT: [2]   - JDiv
+# CHECK-NEXT: [3]   - JFPA
+# CHECK-NEXT: [4]   - JFPM
+# CHECK-NEXT: [5]   - JFPU0
+# CHECK-NEXT: [6]   - JFPU1
+# CHECK-NEXT: [7]   - JLAGU
+# CHECK-NEXT: [8]   - JMul
+# CHECK-NEXT: [9]   - JSAGU
+# CHECK-NEXT: [10]  - JSTC
+# CHECK-NEXT: [11]  - JVALU0
+# CHECK-NEXT: [12]  - JVALU1
+# CHECK-NEXT: [13]  - JVIMUL
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]
+# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -     1.00   1.00    -
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   Instructions:
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -     vpinsrd	$0, %eax, %xmm0, %xmm0
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -     vpinsrd	$1, %eax, %xmm0, %xmm0
+
+# CHECK:      [3] Code Region
+
+# CHECK:      Iterations:        500
+# CHECK-NEXT: Instructions:      1000
+# CHECK-NEXT: Total Cycles:      1003
+# CHECK-NEXT: Total uOps:        2000
+
+# CHECK:      Dispatch Width:    2
+# CHECK-NEXT: uOps Per Cycle:    1.99
+# CHECK-NEXT: IPC:               1.00
+# CHECK-NEXT: Block RThroughput: 2.0
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  2      7     0.50                        vpinsrq	$0, %rax, %xmm0, %xmm0
+# CHECK-NEXT:  2      7     0.50                        vpinsrq	$1, %rax, %xmm0, %xmm0
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0]   - JALU0
+# CHECK-NEXT: [1]   - JALU1
+# CHECK-NEXT: [2]   - JDiv
+# CHECK-NEXT: [3]   - JFPA
+# CHECK-NEXT: [4]   - JFPM
+# CHECK-NEXT: [5]   - JFPU0
+# CHECK-NEXT: [6]   - JFPU1
+# CHECK-NEXT: [7]   - JLAGU
+# CHECK-NEXT: [8]   - JMul
+# CHECK-NEXT: [9]   - JSAGU
+# CHECK-NEXT: [10]  - JSTC
+# CHECK-NEXT: [11]  - JVALU0
+# CHECK-NEXT: [12]  - JVALU1
+# CHECK-NEXT: [13]  - JVIMUL
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]
+# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -     1.00   1.00    -
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   Instructions:
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -     vpinsrq	$0, %rax, %xmm0, %xmm0
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -     vpinsrq	$1, %rax, %xmm0, %xmm0
diff --git a/test/tools/llvm-mca/X86/BtVer2/int-to-fpu-forwarding-2.s b/test/tools/llvm-mca/X86/BtVer2/int-to-fpu-forwarding-2.s
new file mode 100644
index 0000000000000000000000000000000000000000..dc7749102c946f8271a88e5d42ff2b76f45362da
--- /dev/null
+++ b/test/tools/llvm-mca/X86/BtVer2/int-to-fpu-forwarding-2.s
@@ -0,0 +1,311 @@
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=x86_64-unknown-unknown -mcpu=btver2 -iterations=500 < %s | FileCheck %s
+
+# Throughput for the AVX code snippets below should tend to 0.25 IPC.
+# Throughput for the SSE code snippets below should tend to 1.00 IPC.
+
+# LLVM-MCA-BEGIN
+vcvtsi2ss %ecx, %xmm0, %xmm0
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN
+vcvtsi2sd %ecx, %xmm0, %xmm0
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN
+cvtsi2ss %ecx, %xmm0
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN
+cvtsi2sd %ecx, %xmm0
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN
+movd %ecx, %xmm0
+# LLVM-MCA-END
+
+# LLVM-MCA-BEGIN
+movq %rcx, %xmm0
+# LLVM-MCA-END
+
+# CHECK:      [0] Code Region
+
+# CHECK:      Iterations:        500
+# CHECK-NEXT: Instructions:      500
+# CHECK-NEXT: Total Cycles:      2003
+# CHECK-NEXT: Total uOps:        1000
+
+# CHECK:      Dispatch Width:    2
+# CHECK-NEXT: uOps Per Cycle:    0.50
+# CHECK-NEXT: IPC:               0.25
+# CHECK-NEXT: Block RThroughput: 1.0
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  2      10    1.00                        vcvtsi2ssl	%ecx, %xmm0, %xmm0
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0]   - JALU0
+# CHECK-NEXT: [1]   - JALU1
+# CHECK-NEXT: [2]   - JDiv
+# CHECK-NEXT: [3]   - JFPA
+# CHECK-NEXT: [4]   - JFPM
+# CHECK-NEXT: [5]   - JFPU0
+# CHECK-NEXT: [6]   - JFPU1
+# CHECK-NEXT: [7]   - JLAGU
+# CHECK-NEXT: [8]   - JMul
+# CHECK-NEXT: [9]   - JSAGU
+# CHECK-NEXT: [10]  - JSTC
+# CHECK-NEXT: [11]  - JVALU0
+# CHECK-NEXT: [12]  - JVALU1
+# CHECK-NEXT: [13]  - JVIMUL
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   Instructions:
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     vcvtsi2ssl	%ecx, %xmm0, %xmm0
+
+# CHECK:      [1] Code Region
+
+# CHECK:      Iterations:        500
+# CHECK-NEXT: Instructions:      500
+# CHECK-NEXT: Total Cycles:      2003
+# CHECK-NEXT: Total uOps:        1000
+
+# CHECK:      Dispatch Width:    2
+# CHECK-NEXT: uOps Per Cycle:    0.50
+# CHECK-NEXT: IPC:               0.25
+# CHECK-NEXT: Block RThroughput: 1.0
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  2      10    1.00                        vcvtsi2sdl	%ecx, %xmm0, %xmm0
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0]   - JALU0
+# CHECK-NEXT: [1]   - JALU1
+# CHECK-NEXT: [2]   - JDiv
+# CHECK-NEXT: [3]   - JFPA
+# CHECK-NEXT: [4]   - JFPM
+# CHECK-NEXT: [5]   - JFPU0
+# CHECK-NEXT: [6]   - JFPU1
+# CHECK-NEXT: [7]   - JLAGU
+# CHECK-NEXT: [8]   - JMul
+# CHECK-NEXT: [9]   - JSAGU
+# CHECK-NEXT: [10]  - JSTC
+# CHECK-NEXT: [11]  - JVALU0
+# CHECK-NEXT: [12]  - JVALU1
+# CHECK-NEXT: [13]  - JVIMUL
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   Instructions:
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     vcvtsi2sdl	%ecx, %xmm0, %xmm0
+
+# CHECK:      [2] Code Region
+
+# CHECK:      Iterations:        500
+# CHECK-NEXT: Instructions:      500
+# CHECK-NEXT: Total Cycles:      506
+# CHECK-NEXT: Total uOps:        1000
+
+# CHECK:      Dispatch Width:    2
+# CHECK-NEXT: uOps Per Cycle:    1.98
+# CHECK-NEXT: IPC:               0.99
+# CHECK-NEXT: Block RThroughput: 1.0
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  2      10    1.00                        cvtsi2ssl	%ecx, %xmm0
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0]   - JALU0
+# CHECK-NEXT: [1]   - JALU1
+# CHECK-NEXT: [2]   - JDiv
+# CHECK-NEXT: [3]   - JFPA
+# CHECK-NEXT: [4]   - JFPM
+# CHECK-NEXT: [5]   - JFPU0
+# CHECK-NEXT: [6]   - JFPU1
+# CHECK-NEXT: [7]   - JLAGU
+# CHECK-NEXT: [8]   - JMul
+# CHECK-NEXT: [9]   - JSAGU
+# CHECK-NEXT: [10]  - JSTC
+# CHECK-NEXT: [11]  - JVALU0
+# CHECK-NEXT: [12]  - JVALU1
+# CHECK-NEXT: [13]  - JVIMUL
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   Instructions:
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cvtsi2ssl	%ecx, %xmm0
+
+# CHECK:      [3] Code Region
+
+# CHECK:      Iterations:        500
+# CHECK-NEXT: Instructions:      500
+# CHECK-NEXT: Total Cycles:      506
+# CHECK-NEXT: Total uOps:        1000
+
+# CHECK:      Dispatch Width:    2
+# CHECK-NEXT: uOps Per Cycle:    1.98
+# CHECK-NEXT: IPC:               0.99
+# CHECK-NEXT: Block RThroughput: 1.0
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  2      10    1.00                        cvtsi2sdl	%ecx, %xmm0
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0]   - JALU0
+# CHECK-NEXT: [1]   - JALU1
+# CHECK-NEXT: [2]   - JDiv
+# CHECK-NEXT: [3]   - JFPA
+# CHECK-NEXT: [4]   - JFPM
+# CHECK-NEXT: [5]   - JFPU0
+# CHECK-NEXT: [6]   - JFPU1
+# CHECK-NEXT: [7]   - JLAGU
+# CHECK-NEXT: [8]   - JMul
+# CHECK-NEXT: [9]   - JSAGU
+# CHECK-NEXT: [10]  - JSTC
+# CHECK-NEXT: [11]  - JVALU0
+# CHECK-NEXT: [12]  - JVALU1
+# CHECK-NEXT: [13]  - JVIMUL
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   Instructions:
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     cvtsi2sdl	%ecx, %xmm0
+
+# CHECK:      [4] Code Region
+
+# CHECK:      Iterations:        500
+# CHECK-NEXT: Instructions:      500
+# CHECK-NEXT: Total Cycles:      510
+# CHECK-NEXT: Total uOps:        1000
+
+# CHECK:      Dispatch Width:    2
+# CHECK-NEXT: uOps Per Cycle:    1.96
+# CHECK-NEXT: IPC:               0.98
+# CHECK-NEXT: Block RThroughput: 1.0
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  2      8     0.50                        movd	%ecx, %xmm0
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0]   - JALU0
+# CHECK-NEXT: [1]   - JALU1
+# CHECK-NEXT: [2]   - JDiv
+# CHECK-NEXT: [3]   - JFPA
+# CHECK-NEXT: [4]   - JFPM
+# CHECK-NEXT: [5]   - JFPU0
+# CHECK-NEXT: [6]   - JFPU1
+# CHECK-NEXT: [7]   - JLAGU
+# CHECK-NEXT: [8]   - JMul
+# CHECK-NEXT: [9]   - JSAGU
+# CHECK-NEXT: [10]  - JSTC
+# CHECK-NEXT: [11]  - JVALU0
+# CHECK-NEXT: [12]  - JVALU1
+# CHECK-NEXT: [13]  - JVIMUL
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]
+# CHECK-NEXT:  -      -      -     0.50   0.50   0.50   0.50    -      -      -      -      -      -      -
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   Instructions:
+# CHECK-NEXT:  -      -      -     0.50   0.50   0.50   0.50    -      -      -      -      -      -      -     movd	%ecx, %xmm0
+
+# CHECK:      [5] Code Region
+
+# CHECK:      Iterations:        500
+# CHECK-NEXT: Instructions:      500
+# CHECK-NEXT: Total Cycles:      510
+# CHECK-NEXT: Total uOps:        1000
+
+# CHECK:      Dispatch Width:    2
+# CHECK-NEXT: uOps Per Cycle:    1.96
+# CHECK-NEXT: IPC:               0.98
+# CHECK-NEXT: Block RThroughput: 1.0
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  2      8     0.50                        movq	%rcx, %xmm0
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0]   - JALU0
+# CHECK-NEXT: [1]   - JALU1
+# CHECK-NEXT: [2]   - JDiv
+# CHECK-NEXT: [3]   - JFPA
+# CHECK-NEXT: [4]   - JFPM
+# CHECK-NEXT: [5]   - JFPU0
+# CHECK-NEXT: [6]   - JFPU1
+# CHECK-NEXT: [7]   - JLAGU
+# CHECK-NEXT: [8]   - JMul
+# CHECK-NEXT: [9]   - JSAGU
+# CHECK-NEXT: [10]  - JSTC
+# CHECK-NEXT: [11]  - JVALU0
+# CHECK-NEXT: [12]  - JVALU1
+# CHECK-NEXT: [13]  - JVIMUL
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]
+# CHECK-NEXT:  -      -      -     0.50   0.50   0.50   0.50    -      -      -      -      -      -      -
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   Instructions:
+# CHECK-NEXT:  -      -      -     0.50   0.50   0.50   0.50    -      -      -      -      -      -      -     movq	%rcx, %xmm0
diff --git a/test/tools/llvm-mca/X86/BtVer2/int-to-fpu-forwarding-3.s b/test/tools/llvm-mca/X86/BtVer2/int-to-fpu-forwarding-3.s
new file mode 100644
index 0000000000000000000000000000000000000000..00c13f9ef59a9039d04361c3d829c927be5400d2
--- /dev/null
+++ b/test/tools/llvm-mca/X86/BtVer2/int-to-fpu-forwarding-3.s
@@ -0,0 +1,82 @@
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=x86_64-unknown-unknown -mcpu=btver2 -iterations=500 -timeline -timeline-max-iterations=3 < %s | FileCheck %s
+
+# Throughput for the code snippet below should tend to 1.00 IPC.
+
+add %eax, %eax
+vpinsrb $0, %eax, %xmm0, %xmm0
+vpinsrb $1, %eax, %xmm0, %xmm0
+
+# CHECK:      Iterations:        500
+# CHECK-NEXT: Instructions:      1500
+# CHECK-NEXT: Total Cycles:      1509
+# CHECK-NEXT: Total uOps:        2500
+
+# CHECK:      Dispatch Width:    2
+# CHECK-NEXT: uOps Per Cycle:    1.66
+# CHECK-NEXT: IPC:               0.99
+# CHECK-NEXT: Block RThroughput: 2.5
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  1      1     0.50                        addl	%eax, %eax
+# CHECK-NEXT:  2      7     0.50                        vpinsrb	$0, %eax, %xmm0, %xmm0
+# CHECK-NEXT:  2      7     0.50                        vpinsrb	$1, %eax, %xmm0, %xmm0
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0]   - JALU0
+# CHECK-NEXT: [1]   - JALU1
+# CHECK-NEXT: [2]   - JDiv
+# CHECK-NEXT: [3]   - JFPA
+# CHECK-NEXT: [4]   - JFPM
+# CHECK-NEXT: [5]   - JFPU0
+# CHECK-NEXT: [6]   - JFPU1
+# CHECK-NEXT: [7]   - JLAGU
+# CHECK-NEXT: [8]   - JMul
+# CHECK-NEXT: [9]   - JSAGU
+# CHECK-NEXT: [10]  - JSTC
+# CHECK-NEXT: [11]  - JVALU0
+# CHECK-NEXT: [12]  - JVALU1
+# CHECK-NEXT: [13]  - JVIMUL
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]
+# CHECK-NEXT: 0.50   0.50    -      -      -     1.00   1.00    -      -      -      -     1.00   1.00    -
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   Instructions:
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -     addl	%eax, %eax
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -     1.00    -     vpinsrb	$0, %eax, %xmm0, %xmm0
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -      -      -      -     1.00    -      -     vpinsrb	$1, %eax, %xmm0, %xmm0
+
+# CHECK:      Timeline view:
+# CHECK-NEXT:                     01234567
+# CHECK-NEXT: Index     0123456789
+
+# CHECK:      [0,0]     DeER .    .    . .   addl	%eax, %eax
+# CHECK-NEXT: [0,1]     .D======eER    . .   vpinsrb	$0, %eax, %xmm0, %xmm0
+# CHECK-NEXT: [0,2]     . D======eER   . .   vpinsrb	$1, %eax, %xmm0, %xmm0
+# CHECK-NEXT: [1,0]     .  DeE-----R   . .   addl	%eax, %eax
+# CHECK-NEXT: [1,1]     .   D======eER . .   vpinsrb	$0, %eax, %xmm0, %xmm0
+# CHECK-NEXT: [1,2]     .    D======eER. .   vpinsrb	$1, %eax, %xmm0, %xmm0
+# CHECK-NEXT: [2,0]     .    .DeE-----R. .   addl	%eax, %eax
+# CHECK-NEXT: [2,1]     .    . D======eER.   vpinsrb	$0, %eax, %xmm0, %xmm0
+# CHECK-NEXT: [2,2]     .    .  D======eER   vpinsrb	$1, %eax, %xmm0, %xmm0
+
+# CHECK:      Average Wait times (based on the timeline view):
+# CHECK-NEXT: [0]: Executions
+# CHECK-NEXT: [1]: Average time spent waiting in a scheduler's queue
+# CHECK-NEXT: [2]: Average time spent waiting in a scheduler's queue while ready
+# CHECK-NEXT: [3]: Average time elapsed from WB until retire stage
+
+# CHECK:            [0]    [1]    [2]    [3]
+# CHECK-NEXT: 0.     3     1.0    1.0    3.3       addl	%eax, %eax
+# CHECK-NEXT: 1.     3     7.0    0.0    0.0       vpinsrb	$0, %eax, %xmm0, %xmm0
+# CHECK-NEXT: 2.     3     7.0    0.0    0.0       vpinsrb	$1, %eax, %xmm0, %xmm0
diff --git a/test/tools/llvm-mca/X86/BtVer2/resources-avx1.s b/test/tools/llvm-mca/X86/BtVer2/resources-avx1.s
index 3db3470e0c4f60f09b2b213e35b9157a989ddc16..959d2eac3ba72407b87eb0e816ad4fe763294bba 100644
--- a/test/tools/llvm-mca/X86/BtVer2/resources-avx1.s
+++ b/test/tools/llvm-mca/X86/BtVer2/resources-avx1.s
@@ -1144,14 +1144,14 @@ vzeroupper
 # CHECK-NEXT:  2      12    1.00    *                   vcvtsd2si	(%rax), %rcx
 # CHECK-NEXT:  2      7     2.00                        vcvtsd2ss	%xmm0, %xmm1, %xmm2
 # CHECK-NEXT:  2      12    2.00    *                   vcvtsd2ss	(%rax), %xmm1, %xmm2
-# CHECK-NEXT:  2      9     1.00                        vcvtsi2sdl	%ecx, %xmm0, %xmm2
-# CHECK-NEXT:  2      9     1.00                        vcvtsi2sdq	%rcx, %xmm0, %xmm2
-# CHECK-NEXT:  2      14    1.00    *                   vcvtsi2sdl	(%rax), %xmm0, %xmm2
-# CHECK-NEXT:  2      14    1.00    *                   vcvtsi2sdq	(%rax), %xmm0, %xmm2
-# CHECK-NEXT:  2      9     1.00                        vcvtsi2ssl	%ecx, %xmm0, %xmm2
-# CHECK-NEXT:  2      9     1.00                        vcvtsi2ssq	%rcx, %xmm0, %xmm2
-# CHECK-NEXT:  2      14    1.00    *                   vcvtsi2ssl	(%rax), %xmm0, %xmm2
-# CHECK-NEXT:  2      14    1.00    *                   vcvtsi2ssq	(%rax), %xmm0, %xmm2
+# CHECK-NEXT:  2      10    1.00                        vcvtsi2sdl	%ecx, %xmm0, %xmm2
+# CHECK-NEXT:  2      10    1.00                        vcvtsi2sdq	%rcx, %xmm0, %xmm2
+# CHECK-NEXT:  1      9     1.00    *                   vcvtsi2sdl	(%rax), %xmm0, %xmm2
+# CHECK-NEXT:  1      9     1.00    *                   vcvtsi2sdq	(%rax), %xmm0, %xmm2
+# CHECK-NEXT:  2      10    1.00                        vcvtsi2ssl	%ecx, %xmm0, %xmm2
+# CHECK-NEXT:  2      10    1.00                        vcvtsi2ssq	%rcx, %xmm0, %xmm2
+# CHECK-NEXT:  1      9     1.00    *                   vcvtsi2ssl	(%rax), %xmm0, %xmm2
+# CHECK-NEXT:  1      9     1.00    *                   vcvtsi2ssq	(%rax), %xmm0, %xmm2
 # CHECK-NEXT:  2      7     2.00                        vcvtss2sd	%xmm0, %xmm1, %xmm2
 # CHECK-NEXT:  2      12    2.00    *                   vcvtss2sd	(%rax), %xmm1, %xmm2
 # CHECK-NEXT:  2      7     1.00                        vcvtss2si	%xmm0, %ecx
@@ -1196,29 +1196,29 @@ vzeroupper
 # CHECK-NEXT:  1      1     1.00           *            vextractf128	$1, %ymm0, (%rax)
 # CHECK-NEXT:  1      3     1.00                        vextractps	$1, %xmm0, %ecx
 # CHECK-NEXT:  1      3     1.00           *            vextractps	$1, %xmm0, (%rax)
-# CHECK-NEXT:  1      3     1.00                        vhaddpd	%xmm0, %xmm1, %xmm2
-# CHECK-NEXT:  1      8     1.00    *                   vhaddpd	(%rax), %xmm1, %xmm2
-# CHECK-NEXT:  2      3     2.00                        vhaddpd	%ymm0, %ymm1, %ymm2
-# CHECK-NEXT:  2      8     2.00    *                   vhaddpd	(%rax), %ymm1, %ymm2
-# CHECK-NEXT:  1      3     1.00                        vhaddps	%xmm0, %xmm1, %xmm2
-# CHECK-NEXT:  1      8     1.00    *                   vhaddps	(%rax), %xmm1, %xmm2
-# CHECK-NEXT:  2      3     2.00                        vhaddps	%ymm0, %ymm1, %ymm2
-# CHECK-NEXT:  2      8     2.00    *                   vhaddps	(%rax), %ymm1, %ymm2
-# CHECK-NEXT:  1      3     1.00                        vhsubpd	%xmm0, %xmm1, %xmm2
-# CHECK-NEXT:  1      8     1.00    *                   vhsubpd	(%rax), %xmm1, %xmm2
-# CHECK-NEXT:  2      3     2.00                        vhsubpd	%ymm0, %ymm1, %ymm2
-# CHECK-NEXT:  2      8     2.00    *                   vhsubpd	(%rax), %ymm1, %ymm2
-# CHECK-NEXT:  1      3     1.00                        vhsubps	%xmm0, %xmm1, %xmm2
-# CHECK-NEXT:  1      8     1.00    *                   vhsubps	(%rax), %xmm1, %xmm2
-# CHECK-NEXT:  2      3     2.00                        vhsubps	%ymm0, %ymm1, %ymm2
-# CHECK-NEXT:  2      8     2.00    *                   vhsubps	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      4     1.00                        vhaddpd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      9     1.00    *                   vhaddpd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  2      4     2.00                        vhaddpd	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  2      9     2.00    *                   vhaddpd	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      4     1.00                        vhaddps	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      9     1.00    *                   vhaddps	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  2      4     2.00                        vhaddps	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  2      9     2.00    *                   vhaddps	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      4     1.00                        vhsubpd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      9     1.00    *                   vhsubpd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  2      4     2.00                        vhsubpd	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  2      9     2.00    *                   vhsubpd	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      4     1.00                        vhsubps	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      9     1.00    *                   vhsubps	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  2      4     2.00                        vhsubps	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  2      9     2.00    *                   vhsubps	(%rax), %ymm1, %ymm2
 # CHECK-NEXT:  2      1     1.00                        vinsertf128	$1, %xmm0, %ymm1, %ymm2
 # CHECK-NEXT:  2      6     1.00    *                   vinsertf128	$1, (%rax), %ymm1, %ymm2
 # CHECK-NEXT:  1      1     0.50                        vinsertps	$1, %xmm0, %xmm1, %xmm2
 # CHECK-NEXT:  1      6     1.00    *                   vinsertps	$1, (%rax), %xmm1, %xmm2
 # CHECK-NEXT:  1      5     1.00    *                   vlddqu	(%rax), %xmm2
 # CHECK-NEXT:  1      5     1.00    *                   vlddqu	(%rax), %ymm2
-# CHECK-NEXT:  1      5     1.00    *      *      U     vldmxcsr	(%rax)
+# CHECK-NEXT:  1      3     1.00    *      *      U     vldmxcsr	(%rax)
 # CHECK-NEXT:  1      1     1.00    *      *      U     vmaskmovdqu	%xmm0, %xmm1
 # CHECK-NEXT:  1      6     1.00    *                   vmaskmovpd	(%rax), %xmm0, %xmm2
 # CHECK-NEXT:  2      6     2.00    *                   vmaskmovpd	(%rax), %ymm0, %ymm2
@@ -1433,20 +1433,20 @@ vzeroupper
 # CHECK-NEXT:  2      6     1.00    *                   vperm2f128	$1, (%rax), %ymm1, %ymm2
 # CHECK-NEXT:  1      1     0.50                        vpermilpd	$1, %xmm0, %xmm2
 # CHECK-NEXT:  1      6     1.00    *                   vpermilpd	$1, (%rax), %xmm2
-# CHECK-NEXT:  3      2     2.00                        vpermilpd	%xmm0, %xmm1, %xmm2
-# CHECK-NEXT:  3      7     2.00    *                   vpermilpd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  3      3     2.00                        vpermilpd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  3      8     2.00    *                   vpermilpd	(%rax), %xmm1, %xmm2
 # CHECK-NEXT:  2      1     1.00                        vpermilpd	$1, %ymm0, %ymm2
 # CHECK-NEXT:  2      6     2.00    *                   vpermilpd	$1, (%rax), %ymm2
-# CHECK-NEXT:  6      3     3.00                        vpermilpd	%ymm0, %ymm1, %ymm2
-# CHECK-NEXT:  6      8     3.00    *                   vpermilpd	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  6      4     3.00                        vpermilpd	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  6      9     3.00    *                   vpermilpd	(%rax), %ymm1, %ymm2
 # CHECK-NEXT:  1      1     0.50                        vpermilps	$1, %xmm0, %xmm2
 # CHECK-NEXT:  1      6     1.00    *                   vpermilps	$1, (%rax), %xmm2
-# CHECK-NEXT:  3      2     2.00                        vpermilps	%xmm0, %xmm1, %xmm2
-# CHECK-NEXT:  3      7     2.00    *                   vpermilps	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  3      3     2.00                        vpermilps	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  3      8     2.00    *                   vpermilps	(%rax), %xmm1, %xmm2
 # CHECK-NEXT:  2      1     1.00                        vpermilps	$1, %ymm0, %ymm2
 # CHECK-NEXT:  2      6     2.00    *                   vpermilps	$1, (%rax), %ymm2
-# CHECK-NEXT:  6      3     3.00                        vpermilps	%ymm0, %ymm1, %ymm2
-# CHECK-NEXT:  6      8     3.00    *                   vpermilps	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  6      4     3.00                        vpermilps	%ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  6      9     3.00    *                   vpermilps	(%rax), %ymm1, %ymm2
 # CHECK-NEXT:  1      3     1.00                        vpextrb	$1, %xmm0, %ecx
 # CHECK-NEXT:  1      3     1.00           *            vpextrb	$1, %xmm0, (%rax)
 # CHECK-NEXT:  1      3     1.00                        vpextrd	$1, %xmm0, %ecx
@@ -1455,20 +1455,20 @@ vzeroupper
 # CHECK-NEXT:  1      3     1.00           *            vpextrq	$1, %xmm0, (%rax)
 # CHECK-NEXT:  1      3     1.00                        vpextrw	$1, %xmm0, %ecx
 # CHECK-NEXT:  1      3     1.00           *            vpextrw	$1, %xmm0, (%rax)
-# CHECK-NEXT:  1      1     0.50                        vphaddd	%xmm0, %xmm1, %xmm2
-# CHECK-NEXT:  1      6     1.00    *                   vphaddd	(%rax), %xmm1, %xmm2
-# CHECK-NEXT:  1      1     0.50                        vphaddsw	%xmm0, %xmm1, %xmm2
-# CHECK-NEXT:  1      6     1.00    *                   vphaddsw	(%rax), %xmm1, %xmm2
-# CHECK-NEXT:  1      1     0.50                        vphaddw	%xmm0, %xmm1, %xmm2
-# CHECK-NEXT:  1      6     1.00    *                   vphaddw	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      2     0.50                        vphaddd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      7     1.00    *                   vphaddd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      2     0.50                        vphaddsw	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      7     1.00    *                   vphaddsw	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      2     0.50                        vphaddw	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      7     1.00    *                   vphaddw	(%rax), %xmm1, %xmm2
 # CHECK-NEXT:  1      2     0.50                        vphminposuw	%xmm0, %xmm2
 # CHECK-NEXT:  1      7     1.00    *                   vphminposuw	(%rax), %xmm2
-# CHECK-NEXT:  1      1     0.50                        vphsubd	%xmm0, %xmm1, %xmm2
-# CHECK-NEXT:  1      6     1.00    *                   vphsubd	(%rax), %xmm1, %xmm2
-# CHECK-NEXT:  1      1     0.50                        vphsubsw	%xmm0, %xmm1, %xmm2
-# CHECK-NEXT:  1      6     1.00    *                   vphsubsw	(%rax), %xmm1, %xmm2
-# CHECK-NEXT:  1      1     0.50                        vphsubw	%xmm0, %xmm1, %xmm2
-# CHECK-NEXT:  1      6     1.00    *                   vphsubw	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      2     0.50                        vphsubd	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      7     1.00    *                   vphsubd	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      2     0.50                        vphsubsw	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      7     1.00    *                   vphsubsw	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      2     0.50                        vphsubw	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      7     1.00    *                   vphsubw	(%rax), %xmm1, %xmm2
 # CHECK-NEXT:  2      7     0.50                        vpinsrb	$1, %eax, %xmm1, %xmm2
 # CHECK-NEXT:  1      4     1.00    *                   vpinsrb	$1, (%rax), %xmm1, %xmm2
 # CHECK-NEXT:  2      7     0.50                        vpinsrd	$1, %eax, %xmm1, %xmm2
@@ -1562,32 +1562,32 @@ vzeroupper
 # CHECK-NEXT:  1      6     1.00    *                   vpsignd	(%rax), %xmm1, %xmm2
 # CHECK-NEXT:  1      1     0.50                        vpsignw	%xmm0, %xmm1, %xmm2
 # CHECK-NEXT:  1      6     1.00    *                   vpsignw	(%rax), %xmm1, %xmm2
-# CHECK-NEXT:  1      1     0.50                        vpslld	$1, %xmm0, %xmm2
-# CHECK-NEXT:  1      1     0.50                        vpslld	%xmm0, %xmm1, %xmm2
-# CHECK-NEXT:  1      6     1.00    *                   vpslld	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      2     0.50                        vpslld	$1, %xmm0, %xmm2
+# CHECK-NEXT:  1      2     0.50                        vpslld	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      7     1.00    *                   vpslld	(%rax), %xmm1, %xmm2
 # CHECK-NEXT:  1      1     0.50                        vpslldq	$1, %xmm1, %xmm2
-# CHECK-NEXT:  1      1     0.50                        vpsllq	$1, %xmm0, %xmm2
-# CHECK-NEXT:  1      1     0.50                        vpsllq	%xmm0, %xmm1, %xmm2
-# CHECK-NEXT:  1      6     1.00    *                   vpsllq	(%rax), %xmm1, %xmm2
-# CHECK-NEXT:  1      1     0.50                        vpsllw	$1, %xmm0, %xmm2
-# CHECK-NEXT:  1      1     0.50                        vpsllw	%xmm0, %xmm1, %xmm2
-# CHECK-NEXT:  1      6     1.00    *                   vpsllw	(%rax), %xmm1, %xmm2
-# CHECK-NEXT:  1      1     0.50                        vpsrad	$1, %xmm0, %xmm2
-# CHECK-NEXT:  1      1     0.50                        vpsrad	%xmm0, %xmm1, %xmm2
-# CHECK-NEXT:  1      6     1.00    *                   vpsrad	(%rax), %xmm1, %xmm2
-# CHECK-NEXT:  1      1     0.50                        vpsraw	$1, %xmm0, %xmm2
-# CHECK-NEXT:  1      1     0.50                        vpsraw	%xmm0, %xmm1, %xmm2
-# CHECK-NEXT:  1      6     1.00    *                   vpsraw	(%rax), %xmm1, %xmm2
-# CHECK-NEXT:  1      1     0.50                        vpsrld	$1, %xmm0, %xmm2
-# CHECK-NEXT:  1      1     0.50                        vpsrld	%xmm0, %xmm1, %xmm2
-# CHECK-NEXT:  1      6     1.00    *                   vpsrld	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      2     0.50                        vpsllq	$1, %xmm0, %xmm2
+# CHECK-NEXT:  1      2     0.50                        vpsllq	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      7     1.00    *                   vpsllq	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      2     0.50                        vpsllw	$1, %xmm0, %xmm2
+# CHECK-NEXT:  1      2     0.50                        vpsllw	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      7     1.00    *                   vpsllw	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      2     0.50                        vpsrad	$1, %xmm0, %xmm2
+# CHECK-NEXT:  1      2     0.50                        vpsrad	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      7     1.00    *                   vpsrad	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      2     0.50                        vpsraw	$1, %xmm0, %xmm2
+# CHECK-NEXT:  1      2     0.50                        vpsraw	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      7     1.00    *                   vpsraw	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      2     0.50                        vpsrld	$1, %xmm0, %xmm2
+# CHECK-NEXT:  1      2     0.50                        vpsrld	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      7     1.00    *                   vpsrld	(%rax), %xmm1, %xmm2
 # CHECK-NEXT:  1      1     0.50                        vpsrldq	$1, %xmm1, %xmm2
-# CHECK-NEXT:  1      1     0.50                        vpsrlq	$1, %xmm0, %xmm2
-# CHECK-NEXT:  1      1     0.50                        vpsrlq	%xmm0, %xmm1, %xmm2
-# CHECK-NEXT:  1      6     1.00    *                   vpsrlq	(%rax), %xmm1, %xmm2
-# CHECK-NEXT:  1      1     0.50                        vpsrlw	$1, %xmm0, %xmm2
-# CHECK-NEXT:  1      1     0.50                        vpsrlw	%xmm0, %xmm1, %xmm2
-# CHECK-NEXT:  1      6     1.00    *                   vpsrlw	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      2     0.50                        vpsrlq	$1, %xmm0, %xmm2
+# CHECK-NEXT:  1      2     0.50                        vpsrlq	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      7     1.00    *                   vpsrlq	(%rax), %xmm1, %xmm2
+# CHECK-NEXT:  1      2     0.50                        vpsrlw	$1, %xmm0, %xmm2
+# CHECK-NEXT:  1      2     0.50                        vpsrlw	%xmm0, %xmm1, %xmm2
+# CHECK-NEXT:  1      7     1.00    *                   vpsrlw	(%rax), %xmm1, %xmm2
 # CHECK-NEXT:  1      1     0.50                        vpsubb	%xmm0, %xmm1, %xmm2
 # CHECK-NEXT:  1      6     1.00    *                   vpsubb	(%rax), %xmm1, %xmm2
 # CHECK-NEXT:  1      1     0.50                        vpsubd	%xmm0, %xmm1, %xmm2
diff --git a/test/tools/llvm-mca/X86/BtVer2/resources-bmi1.s b/test/tools/llvm-mca/X86/BtVer2/resources-bmi1.s
index f2ea5ea89a1cc1812ae211352dee54f738bb810b..3227ff8773aa2837912fbd8ea92c192d56f5da38 100644
--- a/test/tools/llvm-mca/X86/BtVer2/resources-bmi1.s
+++ b/test/tools/llvm-mca/X86/BtVer2/resources-bmi1.s
@@ -31,6 +31,9 @@ blsr        (%rax), %ecx
 blsr        %rax, %rcx
 blsr        (%rax), %rcx
 
+tzcnt       %ax, %cx
+tzcnt       (%rax), %cx
+
 tzcnt       %eax, %ecx
 tzcnt       (%rax), %ecx
 
@@ -66,6 +69,8 @@ tzcnt       (%rax), %rcx
 # CHECK-NEXT:  2      5     1.00    *                   blsrl	(%rax), %ecx
 # CHECK-NEXT:  2      2     1.00                        blsrq	%rax, %rcx
 # CHECK-NEXT:  2      5     1.00    *                   blsrq	(%rax), %rcx
+# CHECK-NEXT:  2      2     1.00                        tzcntw	%ax, %cx
+# CHECK-NEXT:  2      5     1.00    *                   tzcntw	(%rax), %cx
 # CHECK-NEXT:  2      2     1.00                        tzcntl	%eax, %ecx
 # CHECK-NEXT:  2      5     1.00    *                   tzcntl	(%rax), %ecx
 # CHECK-NEXT:  2      2     1.00                        tzcntq	%rax, %rcx
@@ -89,7 +94,7 @@ tzcnt       (%rax), %rcx
 
 # CHECK:      Resource pressure per iteration:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]
-# CHECK-NEXT: 20.00  20.00   -      -      -      -      -     12.00   -      -      -      -      -      -
+# CHECK-NEXT: 22.00  22.00   -      -      -      -      -     13.00   -      -      -      -      -      -
 
 # CHECK:      Resource pressure by instruction:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   Instructions:
@@ -113,6 +118,8 @@ tzcnt       (%rax), %rcx
 # CHECK-NEXT: 1.00   1.00    -      -      -      -      -     1.00    -      -      -      -      -      -     blsrl	(%rax), %ecx
 # CHECK-NEXT: 1.00   1.00    -      -      -      -      -      -      -      -      -      -      -      -     blsrq	%rax, %rcx
 # CHECK-NEXT: 1.00   1.00    -      -      -      -      -     1.00    -      -      -      -      -      -     blsrq	(%rax), %rcx
+# CHECK-NEXT: 1.00   1.00    -      -      -      -      -      -      -      -      -      -      -      -     tzcntw	%ax, %cx
+# CHECK-NEXT: 1.00   1.00    -      -      -      -      -     1.00    -      -      -      -      -      -     tzcntw	(%rax), %cx
 # CHECK-NEXT: 1.00   1.00    -      -      -      -      -      -      -      -      -      -      -      -     tzcntl	%eax, %ecx
 # CHECK-NEXT: 1.00   1.00    -      -      -      -      -     1.00    -      -      -      -      -      -     tzcntl	(%rax), %ecx
 # CHECK-NEXT: 1.00   1.00    -      -      -      -      -      -      -      -      -      -      -      -     tzcntq	%rax, %rcx
diff --git a/test/tools/llvm-mca/X86/BtVer2/resources-prefetchw.s b/test/tools/llvm-mca/X86/BtVer2/resources-prefetchw.s
index 2a94e77e52f2621b5d36e1f33f63b71fd1ba03b8..c1cb62865f700cbe8ed3dc862c07a39e86dadbd0 100644
--- a/test/tools/llvm-mca/X86/BtVer2/resources-prefetchw.s
+++ b/test/tools/llvm-mca/X86/BtVer2/resources-prefetchw.s
@@ -13,8 +13,8 @@ prefetchw   (%rax)
 # CHECK-NEXT: [6]: HasSideEffects (U)
 
 # CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
-# CHECK-NEXT:  1      5     1.00    *      *            prefetch	(%rax)
-# CHECK-NEXT:  1      5     1.00    *      *            prefetchw	(%rax)
+# CHECK-NEXT:  1      3     1.00    *      *            prefetch	(%rax)
+# CHECK-NEXT:  1      3     1.00    *      *            prefetchw	(%rax)
 
 # CHECK:      Resources:
 # CHECK-NEXT: [0]   - JALU0
diff --git a/test/tools/llvm-mca/X86/BtVer2/resources-sse1.s b/test/tools/llvm-mca/X86/BtVer2/resources-sse1.s
index ef20da259ca9fdf6c2f90a2b6facd1d0c9b32be9..2094e26ba92f0828384302c77507eb96fb115787 100644
--- a/test/tools/llvm-mca/X86/BtVer2/resources-sse1.s
+++ b/test/tools/llvm-mca/X86/BtVer2/resources-sse1.s
@@ -212,10 +212,10 @@ xorps       (%rax), %xmm2
 # CHECK-NEXT:  1      8     1.00    *                   cvtpi2ps	(%rax), %xmm2
 # CHECK-NEXT:  1      3     1.00                        cvtps2pi	%xmm0, %mm2
 # CHECK-NEXT:  1      8     1.00    *                   cvtps2pi	(%rax), %mm2
-# CHECK-NEXT:  2      9     1.00                        cvtsi2ssl	%ecx, %xmm2
-# CHECK-NEXT:  2      9     1.00                        cvtsi2ssq	%rcx, %xmm2
-# CHECK-NEXT:  2      14    1.00    *                   cvtsi2ssl	(%rax), %xmm2
-# CHECK-NEXT:  2      14    1.00    *                   cvtsi2ssl	(%rax), %xmm2
+# CHECK-NEXT:  2      10    1.00                        cvtsi2ssl	%ecx, %xmm2
+# CHECK-NEXT:  2      10    1.00                        cvtsi2ssq	%rcx, %xmm2
+# CHECK-NEXT:  1      9     1.00    *                   cvtsi2ssl	(%rax), %xmm2
+# CHECK-NEXT:  1      9     1.00    *                   cvtsi2ssl	(%rax), %xmm2
 # CHECK-NEXT:  2      7     1.00                        cvtss2si	%xmm0, %ecx
 # CHECK-NEXT:  2      7     1.00                        cvtss2si	%xmm0, %rcx
 # CHECK-NEXT:  2      12    1.00    *                   cvtss2si	(%rax), %ecx
@@ -230,7 +230,7 @@ xorps       (%rax), %xmm2
 # CHECK-NEXT:  1      24    19.00   *                   divps	(%rax), %xmm2
 # CHECK-NEXT:  1      19    19.00                       divss	%xmm0, %xmm2
 # CHECK-NEXT:  1      24    19.00   *                   divss	(%rax), %xmm2
-# CHECK-NEXT:  1      5     1.00    *      *      U     ldmxcsr	(%rax)
+# CHECK-NEXT:  1      3     1.00    *      *      U     ldmxcsr	(%rax)
 # CHECK-NEXT:  1      1     0.50    *      *      U     maskmovq	%mm0, %mm1
 # CHECK-NEXT:  1      2     1.00                        maxps	%xmm0, %xmm2
 # CHECK-NEXT:  1      7     1.00    *                   maxps	(%rax), %xmm2
@@ -282,10 +282,10 @@ xorps       (%rax), %xmm2
 # CHECK-NEXT:  1      3     1.00                        pmovmskb	%xmm0, %ecx
 # CHECK-NEXT:  1      2     1.00                        pmulhuw	%mm0, %mm2
 # CHECK-NEXT:  1      7     1.00    *                   pmulhuw	(%rax), %mm2
-# CHECK-NEXT:  1      5     1.00    *      *            prefetcht0	(%rax)
-# CHECK-NEXT:  1      5     1.00    *      *            prefetcht1	(%rax)
-# CHECK-NEXT:  1      5     1.00    *      *            prefetcht2	(%rax)
-# CHECK-NEXT:  1      5     1.00    *      *            prefetchnta	(%rax)
+# CHECK-NEXT:  1      3     1.00    *      *            prefetcht0	(%rax)
+# CHECK-NEXT:  1      3     1.00    *      *            prefetcht1	(%rax)
+# CHECK-NEXT:  1      3     1.00    *      *            prefetcht2	(%rax)
+# CHECK-NEXT:  1      3     1.00    *      *            prefetchnta	(%rax)
 # CHECK-NEXT:  1      2     0.50                        psadbw	%mm0, %mm2
 # CHECK-NEXT:  1      7     1.00    *                   psadbw	(%rax), %mm2
 # CHECK-NEXT:  1      1     0.50                        pshufw	$1, %mm0, %mm2
diff --git a/test/tools/llvm-mca/X86/BtVer2/resources-sse2.s b/test/tools/llvm-mca/X86/BtVer2/resources-sse2.s
index 13f1966cf5b9aa6d26c2fcb8c769235e59e0cbc8..9ac305d07a87fa5fc954bf0a9ed995eaa59250de 100644
--- a/test/tools/llvm-mca/X86/BtVer2/resources-sse2.s
+++ b/test/tools/llvm-mca/X86/BtVer2/resources-sse2.s
@@ -94,6 +94,8 @@ maxpd       (%rax), %xmm2
 maxsd       %xmm0, %xmm2
 maxsd       (%rax), %xmm2
 
+mfence
+
 minpd       %xmm0, %xmm2
 minpd       (%rax), %xmm2
 
@@ -229,6 +231,9 @@ pcmpgtw     (%rax), %xmm2
 
 pextrw      $1, %xmm0, %rcx
 
+pinsrw      $1, %rax, %xmm0
+pinsrw      $1, (%rax), %xmm0
+
 pmaddwd     %xmm0, %xmm2
 pmaddwd     (%rax), %xmm2
 
@@ -410,7 +415,7 @@ xorpd       (%rax), %xmm2
 # CHECK-NEXT:  1      6     1.00    *                   andnpd	(%rax), %xmm2
 # CHECK-NEXT:  1      1     0.50                        andpd	%xmm0, %xmm2
 # CHECK-NEXT:  1      6     1.00    *                   andpd	(%rax), %xmm2
-# CHECK-NEXT:  1      5     1.00    *      *      U     clflush	(%rax)
+# CHECK-NEXT:  1      3     1.00    *      *      U     clflush	(%rax)
 # CHECK-NEXT:  1      2     1.00                        cmppd	$0, %xmm0, %xmm2
 # CHECK-NEXT:  1      7     1.00    *                   cmppd	$0, (%rax), %xmm2
 # CHECK-NEXT:  1      2     1.00                        cmpsd	$0, %xmm0, %xmm2
@@ -439,10 +444,10 @@ xorpd       (%rax), %xmm2
 # CHECK-NEXT:  2      12    1.00    *                   cvtsd2si	(%rax), %rcx
 # CHECK-NEXT:  2      7     2.00                        cvtsd2ss	%xmm0, %xmm2
 # CHECK-NEXT:  2      12    2.00    *                   cvtsd2ss	(%rax), %xmm2
-# CHECK-NEXT:  2      9     1.00                        cvtsi2sdl	%ecx, %xmm2
-# CHECK-NEXT:  2      9     1.00                        cvtsi2sdq	%rcx, %xmm2
-# CHECK-NEXT:  2      14    1.00    *                   cvtsi2sdl	(%rax), %xmm2
-# CHECK-NEXT:  2      14    1.00    *                   cvtsi2sdl	(%rax), %xmm2
+# CHECK-NEXT:  2      10    1.00                        cvtsi2sdl	%ecx, %xmm2
+# CHECK-NEXT:  2      10    1.00                        cvtsi2sdq	%rcx, %xmm2
+# CHECK-NEXT:  1      9     1.00    *                   cvtsi2sdl	(%rax), %xmm2
+# CHECK-NEXT:  1      9     1.00    *                   cvtsi2sdl	(%rax), %xmm2
 # CHECK-NEXT:  2      7     2.00                        cvtss2sd	%xmm0, %xmm2
 # CHECK-NEXT:  2      12    2.00    *                   cvtss2sd	(%rax), %xmm2
 # CHECK-NEXT:  1      3     1.00                        cvttpd2dq	%xmm0, %xmm2
@@ -465,6 +470,7 @@ xorpd       (%rax), %xmm2
 # CHECK-NEXT:  1      7     1.00    *                   maxpd	(%rax), %xmm2
 # CHECK-NEXT:  1      2     1.00                        maxsd	%xmm0, %xmm2
 # CHECK-NEXT:  1      7     1.00    *                   maxsd	(%rax), %xmm2
+# CHECK-NEXT:  1      1     1.00    *      *      U     mfence
 # CHECK-NEXT:  1      2     1.00                        minpd	%xmm0, %xmm2
 # CHECK-NEXT:  1      7     1.00    *                   minpd	(%rax), %xmm2
 # CHECK-NEXT:  1      2     1.00                        minsd	%xmm0, %xmm2
@@ -555,6 +561,8 @@ xorpd       (%rax), %xmm2
 # CHECK-NEXT:  1      1     0.50                        pcmpgtw	%xmm0, %xmm2
 # CHECK-NEXT:  1      6     1.00    *                   pcmpgtw	(%rax), %xmm2
 # CHECK-NEXT:  1      3     1.00                        pextrw	$1, %xmm0, %ecx
+# CHECK-NEXT:  2      7     0.50                        pinsrw	$1, %eax, %xmm0
+# CHECK-NEXT:  1      4     1.00    *                   pinsrw	$1, (%rax), %xmm0
 # CHECK-NEXT:  1      2     1.00                        pmaddwd	%xmm0, %xmm2
 # CHECK-NEXT:  1      7     1.00    *                   pmaddwd	(%rax), %xmm2
 # CHECK-NEXT:  1      1     0.50                        pmaxsw	%xmm0, %xmm2
@@ -586,32 +594,32 @@ xorpd       (%rax), %xmm2
 # CHECK-NEXT:  1      6     1.00    *                   pshufhw	$1, (%rax), %xmm2
 # CHECK-NEXT:  1      1     0.50                        pshuflw	$1, %xmm0, %xmm2
 # CHECK-NEXT:  1      6     1.00    *                   pshuflw	$1, (%rax), %xmm2
-# CHECK-NEXT:  1      1     0.50                        pslld	$1, %xmm2
-# CHECK-NEXT:  1      1     0.50                        pslld	%xmm0, %xmm2
-# CHECK-NEXT:  1      6     1.00    *                   pslld	(%rax), %xmm2
+# CHECK-NEXT:  1      2     0.50                        pslld	$1, %xmm2
+# CHECK-NEXT:  1      2     0.50                        pslld	%xmm0, %xmm2
+# CHECK-NEXT:  1      7     1.00    *                   pslld	(%rax), %xmm2
 # CHECK-NEXT:  1      1     0.50                        pslldq	$1, %xmm2
-# CHECK-NEXT:  1      1     0.50                        psllq	$1, %xmm2
-# CHECK-NEXT:  1      1     0.50                        psllq	%xmm0, %xmm2
-# CHECK-NEXT:  1      6     1.00    *                   psllq	(%rax), %xmm2
-# CHECK-NEXT:  1      1     0.50                        psllw	$1, %xmm2
-# CHECK-NEXT:  1      1     0.50                        psllw	%xmm0, %xmm2
-# CHECK-NEXT:  1      6     1.00    *                   psllw	(%rax), %xmm2
-# CHECK-NEXT:  1      1     0.50                        psrad	$1, %xmm2
-# CHECK-NEXT:  1      1     0.50                        psrad	%xmm0, %xmm2
-# CHECK-NEXT:  1      6     1.00    *                   psrad	(%rax), %xmm2
-# CHECK-NEXT:  1      1     0.50                        psraw	$1, %xmm2
-# CHECK-NEXT:  1      1     0.50                        psraw	%xmm0, %xmm2
-# CHECK-NEXT:  1      6     1.00    *                   psraw	(%rax), %xmm2
-# CHECK-NEXT:  1      1     0.50                        psrld	$1, %xmm2
-# CHECK-NEXT:  1      1     0.50                        psrld	%xmm0, %xmm2
-# CHECK-NEXT:  1      6     1.00    *                   psrld	(%rax), %xmm2
+# CHECK-NEXT:  1      2     0.50                        psllq	$1, %xmm2
+# CHECK-NEXT:  1      2     0.50                        psllq	%xmm0, %xmm2
+# CHECK-NEXT:  1      7     1.00    *                   psllq	(%rax), %xmm2
+# CHECK-NEXT:  1      2     0.50                        psllw	$1, %xmm2
+# CHECK-NEXT:  1      2     0.50                        psllw	%xmm0, %xmm2
+# CHECK-NEXT:  1      7     1.00    *                   psllw	(%rax), %xmm2
+# CHECK-NEXT:  1      2     0.50                        psrad	$1, %xmm2
+# CHECK-NEXT:  1      2     0.50                        psrad	%xmm0, %xmm2
+# CHECK-NEXT:  1      7     1.00    *                   psrad	(%rax), %xmm2
+# CHECK-NEXT:  1      2     0.50                        psraw	$1, %xmm2
+# CHECK-NEXT:  1      2     0.50                        psraw	%xmm0, %xmm2
+# CHECK-NEXT:  1      7     1.00    *                   psraw	(%rax), %xmm2
+# CHECK-NEXT:  1      2     0.50                        psrld	$1, %xmm2
+# CHECK-NEXT:  1      2     0.50                        psrld	%xmm0, %xmm2
+# CHECK-NEXT:  1      7     1.00    *                   psrld	(%rax), %xmm2
 # CHECK-NEXT:  1      1     0.50                        psrldq	$1, %xmm2
-# CHECK-NEXT:  1      1     0.50                        psrlq	$1, %xmm2
-# CHECK-NEXT:  1      1     0.50                        psrlq	%xmm0, %xmm2
-# CHECK-NEXT:  1      6     1.00    *                   psrlq	(%rax), %xmm2
-# CHECK-NEXT:  1      1     0.50                        psrlw	$1, %xmm2
-# CHECK-NEXT:  1      1     0.50                        psrlw	%xmm0, %xmm2
-# CHECK-NEXT:  1      6     1.00    *                   psrlw	(%rax), %xmm2
+# CHECK-NEXT:  1      2     0.50                        psrlq	$1, %xmm2
+# CHECK-NEXT:  1      2     0.50                        psrlq	%xmm0, %xmm2
+# CHECK-NEXT:  1      7     1.00    *                   psrlq	(%rax), %xmm2
+# CHECK-NEXT:  1      2     0.50                        psrlw	$1, %xmm2
+# CHECK-NEXT:  1      2     0.50                        psrlw	%xmm0, %xmm2
+# CHECK-NEXT:  1      7     1.00    *                   psrlw	(%rax), %xmm2
 # CHECK-NEXT:  1      1     0.50                        psubb	%xmm0, %xmm2
 # CHECK-NEXT:  1      6     1.00    *                   psubb	(%rax), %xmm2
 # CHECK-NEXT:  1      1     0.50                        psubd	%xmm0, %xmm2
@@ -685,7 +693,7 @@ xorpd       (%rax), %xmm2
 
 # CHECK:      Resource pressure per iteration:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]
-# CHECK-NEXT: 17.00   -      -     49.00  204.00 127.50 140.50 117.00  -     15.00  54.00  66.50  66.50  12.00
+# CHECK-NEXT: 17.00   -      -     49.00  204.00 128.50 141.50 118.00  -     16.00  54.00  67.50  67.50  12.00
 
 # CHECK:      Resource pressure by instruction:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   Instructions:
@@ -752,6 +760,7 @@ xorpd       (%rax), %xmm2
 # CHECK-NEXT:  -      -      -     1.00    -     1.00    -     1.00    -      -      -      -      -      -     maxpd	(%rax), %xmm2
 # CHECK-NEXT:  -      -      -     1.00    -     1.00    -      -      -      -      -      -      -      -     maxsd	%xmm0, %xmm2
 # CHECK-NEXT:  -      -      -     1.00    -     1.00    -     1.00    -      -      -      -      -      -     maxsd	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     mfence
 # CHECK-NEXT:  -      -      -     1.00    -     1.00    -      -      -      -      -      -      -      -     minpd	%xmm0, %xmm2
 # CHECK-NEXT:  -      -      -     1.00    -     1.00    -     1.00    -      -      -      -      -      -     minpd	(%rax), %xmm2
 # CHECK-NEXT:  -      -      -     1.00    -     1.00    -      -      -      -      -      -      -      -     minsd	%xmm0, %xmm2
@@ -842,6 +851,8 @@ xorpd       (%rax), %xmm2
 # CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.50   0.50    -     pcmpgtw	%xmm0, %xmm2
 # CHECK-NEXT:  -      -      -      -      -     0.50   0.50   1.00    -      -      -     0.50   0.50    -     pcmpgtw	(%rax), %xmm2
 # CHECK-NEXT: 1.00    -      -     1.00    -     1.00    -      -      -      -      -      -      -      -     pextrw	$1, %xmm0, %ecx
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.50   0.50    -     pinsrw	$1, %eax, %xmm0
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50   1.00    -      -      -     0.50   0.50    -     pinsrw	$1, (%rax), %xmm0
 # CHECK-NEXT:  -      -      -      -      -     1.00    -      -      -      -      -      -      -     1.00   pmaddwd	%xmm0, %xmm2
 # CHECK-NEXT:  -      -      -      -      -     1.00    -     1.00    -      -      -      -      -     1.00   pmaddwd	(%rax), %xmm2
 # CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -     0.50   0.50    -     pmaxsw	%xmm0, %xmm2
diff --git a/test/tools/llvm-mca/X86/BtVer2/resources-sse3.s b/test/tools/llvm-mca/X86/BtVer2/resources-sse3.s
index 3fd2f406a98bbe5f46a66068ea1a2930434ee925..581af93d5ea3b7587b2e8f2e0a46392bc8d99cfb 100644
--- a/test/tools/llvm-mca/X86/BtVer2/resources-sse3.s
+++ b/test/tools/llvm-mca/X86/BtVer2/resources-sse3.s
@@ -21,6 +21,8 @@ hsubps    (%rax), %xmm2
 
 lddqu     (%rax), %xmm2
 
+monitor
+
 movddup   %xmm0, %xmm2
 movddup   (%rax), %xmm2
 
@@ -30,6 +32,8 @@ movshdup  (%rax), %xmm2
 movsldup  %xmm0, %xmm2
 movsldup  (%rax), %xmm2
 
+mwait
+
 # CHECK:      Instruction Info:
 # CHECK-NEXT: [1]: #uOps
 # CHECK-NEXT: [2]: Latency
@@ -43,21 +47,23 @@ movsldup  (%rax), %xmm2
 # CHECK-NEXT:  1      8     1.00    *                   addsubpd	(%rax), %xmm2
 # CHECK-NEXT:  1      3     1.00                        addsubps	%xmm0, %xmm2
 # CHECK-NEXT:  1      8     1.00    *                   addsubps	(%rax), %xmm2
-# CHECK-NEXT:  1      3     1.00                        haddpd	%xmm0, %xmm2
-# CHECK-NEXT:  1      8     1.00    *                   haddpd	(%rax), %xmm2
-# CHECK-NEXT:  1      3     1.00                        haddps	%xmm0, %xmm2
-# CHECK-NEXT:  1      8     1.00    *                   haddps	(%rax), %xmm2
-# CHECK-NEXT:  1      3     1.00                        hsubpd	%xmm0, %xmm2
-# CHECK-NEXT:  1      8     1.00    *                   hsubpd	(%rax), %xmm2
-# CHECK-NEXT:  1      3     1.00                        hsubps	%xmm0, %xmm2
-# CHECK-NEXT:  1      8     1.00    *                   hsubps	(%rax), %xmm2
+# CHECK-NEXT:  1      4     1.00                        haddpd	%xmm0, %xmm2
+# CHECK-NEXT:  1      9     1.00    *                   haddpd	(%rax), %xmm2
+# CHECK-NEXT:  1      4     1.00                        haddps	%xmm0, %xmm2
+# CHECK-NEXT:  1      9     1.00    *                   haddps	(%rax), %xmm2
+# CHECK-NEXT:  1      4     1.00                        hsubpd	%xmm0, %xmm2
+# CHECK-NEXT:  1      9     1.00    *                   hsubpd	(%rax), %xmm2
+# CHECK-NEXT:  1      4     1.00                        hsubps	%xmm0, %xmm2
+# CHECK-NEXT:  1      9     1.00    *                   hsubps	(%rax), %xmm2
 # CHECK-NEXT:  1      5     1.00    *                   lddqu	(%rax), %xmm2
+# CHECK-NEXT:  1      100   0.50                  U     monitor
 # CHECK-NEXT:  1      1     0.50                        movddup	%xmm0, %xmm2
 # CHECK-NEXT:  1      6     1.00    *                   movddup	(%rax), %xmm2
 # CHECK-NEXT:  1      1     0.50                        movshdup	%xmm0, %xmm2
 # CHECK-NEXT:  1      6     1.00    *                   movshdup	(%rax), %xmm2
 # CHECK-NEXT:  1      1     0.50                        movsldup	%xmm0, %xmm2
 # CHECK-NEXT:  1      6     1.00    *                   movsldup	(%rax), %xmm2
+# CHECK-NEXT:  1      100   0.50    *      *      U     mwait
 
 # CHECK:      Resources:
 # CHECK-NEXT: [0]   - JALU0
@@ -77,7 +83,7 @@ movsldup  (%rax), %xmm2
 
 # CHECK:      Resource pressure per iteration:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]
-# CHECK-NEXT:  -      -      -     15.00  3.00   15.50  3.50   10.00   -      -      -     0.50   0.50    -
+# CHECK-NEXT: 1.00   1.00    -     15.00  3.00   15.50  3.50   10.00   -      -      -     0.50   0.50    -
 
 # CHECK:      Resource pressure by instruction:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   Instructions:
@@ -94,9 +100,11 @@ movsldup  (%rax), %xmm2
 # CHECK-NEXT:  -      -      -     1.00    -     1.00    -      -      -      -      -      -      -      -     hsubps	%xmm0, %xmm2
 # CHECK-NEXT:  -      -      -     1.00    -     1.00    -     1.00    -      -      -      -      -      -     hsubps	(%rax), %xmm2
 # CHECK-NEXT:  -      -      -      -      -     0.50   0.50   1.00    -      -      -     0.50   0.50    -     lddqu	(%rax), %xmm2
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -     monitor
 # CHECK-NEXT:  -      -      -     0.50   0.50   0.50   0.50    -      -      -      -      -      -      -     movddup	%xmm0, %xmm2
 # CHECK-NEXT:  -      -      -     0.50   0.50   0.50   0.50   1.00    -      -      -      -      -      -     movddup	(%rax), %xmm2
 # CHECK-NEXT:  -      -      -     0.50   0.50   0.50   0.50    -      -      -      -      -      -      -     movshdup	%xmm0, %xmm2
 # CHECK-NEXT:  -      -      -     0.50   0.50   0.50   0.50   1.00    -      -      -      -      -      -     movshdup	(%rax), %xmm2
 # CHECK-NEXT:  -      -      -     0.50   0.50   0.50   0.50    -      -      -      -      -      -      -     movsldup	%xmm0, %xmm2
 # CHECK-NEXT:  -      -      -     0.50   0.50   0.50   0.50   1.00    -      -      -      -      -      -     movsldup	(%rax), %xmm2
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -     mwait
diff --git a/test/tools/llvm-mca/X86/BtVer2/resources-ssse3.s b/test/tools/llvm-mca/X86/BtVer2/resources-ssse3.s
index ff7ff3f93bac1e271b5452a08ae52f7f2bda06ef..e35a474514966521462131f6a95a2fb5d6cc21b2 100644
--- a/test/tools/llvm-mca/X86/BtVer2/resources-ssse3.s
+++ b/test/tools/llvm-mca/X86/BtVer2/resources-ssse3.s
@@ -124,28 +124,28 @@ psignw      (%rax), %xmm2
 # CHECK-NEXT:  1      6     1.00    *                   palignr	$1, (%rax), %xmm2
 # CHECK-NEXT:  1      1     0.50                        phaddd	%mm0, %mm2
 # CHECK-NEXT:  1      6     1.00    *                   phaddd	(%rax), %mm2
-# CHECK-NEXT:  1      1     0.50                        phaddd	%xmm0, %xmm2
-# CHECK-NEXT:  1      6     1.00    *                   phaddd	(%rax), %xmm2
+# CHECK-NEXT:  1      2     0.50                        phaddd	%xmm0, %xmm2
+# CHECK-NEXT:  1      7     1.00    *                   phaddd	(%rax), %xmm2
 # CHECK-NEXT:  1      1     0.50                        phaddsw	%mm0, %mm2
 # CHECK-NEXT:  1      6     1.00    *                   phaddsw	(%rax), %mm2
-# CHECK-NEXT:  1      1     0.50                        phaddsw	%xmm0, %xmm2
-# CHECK-NEXT:  1      6     1.00    *                   phaddsw	(%rax), %xmm2
+# CHECK-NEXT:  1      2     0.50                        phaddsw	%xmm0, %xmm2
+# CHECK-NEXT:  1      7     1.00    *                   phaddsw	(%rax), %xmm2
 # CHECK-NEXT:  1      1     0.50                        phaddw	%mm0, %mm2
 # CHECK-NEXT:  1      6     1.00    *                   phaddw	(%rax), %mm2
-# CHECK-NEXT:  1      1     0.50                        phaddw	%xmm0, %xmm2
-# CHECK-NEXT:  1      6     1.00    *                   phaddw	(%rax), %xmm2
+# CHECK-NEXT:  1      2     0.50                        phaddw	%xmm0, %xmm2
+# CHECK-NEXT:  1      7     1.00    *                   phaddw	(%rax), %xmm2
 # CHECK-NEXT:  1      1     0.50                        phsubd	%mm0, %mm2
 # CHECK-NEXT:  1      6     1.00    *                   phsubd	(%rax), %mm2
-# CHECK-NEXT:  1      1     0.50                        phsubd	%xmm0, %xmm2
-# CHECK-NEXT:  1      6     1.00    *                   phsubd	(%rax), %xmm2
+# CHECK-NEXT:  1      2     0.50                        phsubd	%xmm0, %xmm2
+# CHECK-NEXT:  1      7     1.00    *                   phsubd	(%rax), %xmm2
 # CHECK-NEXT:  1      1     0.50                        phsubsw	%mm0, %mm2
 # CHECK-NEXT:  1      6     1.00    *                   phsubsw	(%rax), %mm2
-# CHECK-NEXT:  1      1     0.50                        phsubsw	%xmm0, %xmm2
-# CHECK-NEXT:  1      6     1.00    *                   phsubsw	(%rax), %xmm2
+# CHECK-NEXT:  1      2     0.50                        phsubsw	%xmm0, %xmm2
+# CHECK-NEXT:  1      7     1.00    *                   phsubsw	(%rax), %xmm2
 # CHECK-NEXT:  1      1     0.50                        phsubw	%mm0, %mm2
 # CHECK-NEXT:  1      6     1.00    *                   phsubw	(%rax), %mm2
-# CHECK-NEXT:  1      1     0.50                        phsubw	%xmm0, %xmm2
-# CHECK-NEXT:  1      6     1.00    *                   phsubw	(%rax), %xmm2
+# CHECK-NEXT:  1      2     0.50                        phsubw	%xmm0, %xmm2
+# CHECK-NEXT:  1      7     1.00    *                   phsubw	(%rax), %xmm2
 # CHECK-NEXT:  1      2     1.00                        pmaddubsw	%mm0, %mm2
 # CHECK-NEXT:  1      7     1.00    *                   pmaddubsw	(%rax), %mm2
 # CHECK-NEXT:  1      2     1.00                        pmaddubsw	%xmm0, %xmm2
diff --git a/test/tools/llvm-mca/X86/BtVer2/resources-x86_64.s b/test/tools/llvm-mca/X86/BtVer2/resources-x86_64.s
index e1bafd8aa2b48e6401a48274c2e29abcc3441155..f5585a5dbca4e7a1e56dc947e52873a1efe43784 100644
--- a/test/tools/llvm-mca/X86/BtVer2/resources-x86_64.s
+++ b/test/tools/llvm-mca/X86/BtVer2/resources-x86_64.s
@@ -254,6 +254,8 @@ divl (%rax)
 divq %rcx
 divq (%rax)
 
+enter $7, $4095
+
 idivb %dil
 idivb (%rax)
 idivw %si
@@ -315,8 +317,13 @@ insl
 
 int $7
 
+invlpg  (%rax)
+invlpga %rax, %ecx
+
 lahf
 
+leave
+
 lodsb
 lodsw
 lodsl
@@ -486,6 +493,11 @@ rcrq %cl, %rdi
 rclq %cl, (%rax)
 rcrq %cl, (%rax)
 
+rdmsr
+rdpmc
+rdtsc
+rdtscp
+
 rolb %dil
 rorb %dil
 rolb (%rax)
@@ -789,6 +801,8 @@ testq %rsi, (%rax)
 
 ud2
 
+wrmsr
+
 xaddb %bl, %cl
 xaddb %bl, (%rcx)
 
@@ -1081,6 +1095,7 @@ xorq (%rax), %rdi
 # CHECK-NEXT:  2      28    25.00   *             U     divl	(%rax)
 # CHECK-NEXT:  2      41    41.00                 U     divq	%rcx
 # CHECK-NEXT:  2      44    41.00   *             U     divq	(%rax)
+# CHECK-NEXT:  1      100   0.50                  U     enter	$7, $4095
 # CHECK-NEXT:  1      12    12.00                 U     idivb	%dil
 # CHECK-NEXT:  1      15    12.00   *             U     idivb	(%rax)
 # CHECK-NEXT:  2      17    17.00                 U     idivw	%si
@@ -1133,7 +1148,10 @@ xorq (%rax), %rdi
 # CHECK-NEXT:  1      100   0.50                  U     insw	%dx, %es:(%rdi)
 # CHECK-NEXT:  1      100   0.50                  U     insl	%dx, %es:(%rdi)
 # CHECK-NEXT:  1      100   0.50    *      *      U     int	$7
+# CHECK-NEXT:  1      100   0.50                  U     invlpg	(%rax)
+# CHECK-NEXT:  1      100   0.50                  U     invlpga	%rax, %ecx
 # CHECK-NEXT:  1      1     0.50                        lahf
+# CHECK-NEXT:  1      1     0.50    *                   leave
 # CHECK-NEXT:  1      100   0.50                  U     lodsb	(%rsi), %al
 # CHECK-NEXT:  1      100   0.50                  U     lodsw	(%rsi), %ax
 # CHECK-NEXT:  1      100   0.50                  U     lodsl	(%rsi), %eax
@@ -1283,6 +1301,10 @@ xorq (%rax), %rdi
 # CHECK-NEXT:  1      1     0.50                        rcrq	%cl, %rdi
 # CHECK-NEXT:  1      4     1.00           *            rclq	%cl, (%rax)
 # CHECK-NEXT:  1      4     1.00           *            rcrq	%cl, (%rax)
+# CHECK-NEXT:  1      100   0.50                  U     rdmsr
+# CHECK-NEXT:  1      100   0.50                  U     rdpmc
+# CHECK-NEXT:  1      100   0.50                  U     rdtsc
+# CHECK-NEXT:  1      100   0.50                  U     rdtscp
 # CHECK-NEXT:  1      1     0.50                        rolb	%dil
 # CHECK-NEXT:  1      1     0.50                        rorb	%dil
 # CHECK-NEXT:  1      4     1.00    *      *            rolb	(%rax)
@@ -1557,6 +1579,7 @@ xorq (%rax), %rdi
 # CHECK-NEXT:  1      1     0.50                        testq	%rsi, %rdi
 # CHECK-NEXT:  1      4     1.00    *                   testq	%rsi, (%rax)
 # CHECK-NEXT:  1      100   0.50    *             U     ud2
+# CHECK-NEXT:  1      100   0.50                  U     wrmsr
 # CHECK-NEXT:  1      1     0.50                        xaddb	%bl, %cl
 # CHECK-NEXT:  1      4     1.00    *      *            xaddb	%bl, (%rcx)
 # CHECK-NEXT:  1      1     0.50                        xaddw	%bx, %cx
@@ -1576,7 +1599,7 @@ xorq (%rax), %rdi
 # CHECK-NEXT:  1      1     0.50                        xchgq	%rbx, %rax
 # CHECK-NEXT:  1      1     0.50                        xchgq	%rbx, %rcx
 # CHECK-NEXT:  1      4     1.00    *      *            xchgq	%rax, (%rbx)
-# CHECK-NEXT:  1      5     1.00    *                   xlatb
+# CHECK-NEXT:  1      3     1.00    *                   xlatb
 # CHECK-NEXT:  1      1     0.50                        xorb	$7, %al
 # CHECK-NEXT:  1      1     0.50                        xorb	$7, %dil
 # CHECK-NEXT:  1      5     1.00    *      *            xorb	$7, (%rax)
@@ -1626,7 +1649,7 @@ xorq (%rax), %rdi
 
 # CHECK:      Resource pressure per iteration:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]
-# CHECK-NEXT: 581.50 631.50 380.00  -      -      -      -     322.00 64.00  223.00  -      -      -      -
+# CHECK-NEXT: 586.00 636.00 380.00  -      -      -      -     322.00 64.00  223.00  -      -      -      -
 
 # CHECK:      Resource pressure by instruction:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   Instructions:
@@ -1850,6 +1873,7 @@ xorq (%rax), %rdi
 # CHECK-NEXT:  -     1.00   25.00   -      -      -      -     1.00    -      -      -      -      -      -     divl	(%rax)
 # CHECK-NEXT:  -     1.00   41.00   -      -      -      -      -      -      -      -      -      -      -     divq	%rcx
 # CHECK-NEXT:  -     1.00   41.00   -      -      -      -     1.00    -      -      -      -      -      -     divq	(%rax)
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -     enter	$7, $4095
 # CHECK-NEXT:  -     1.00   12.00   -      -      -      -      -      -      -      -      -      -      -     idivb	%dil
 # CHECK-NEXT:  -     1.00   12.00   -      -      -      -     1.00    -      -      -      -      -      -     idivb	(%rax)
 # CHECK-NEXT:  -     1.00   17.00   -      -      -      -      -      -      -      -      -      -      -     idivw	%si
@@ -1902,7 +1926,10 @@ xorq (%rax), %rdi
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -     insw	%dx, %es:(%rdi)
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -     insl	%dx, %es:(%rdi)
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -     int	$7
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -     invlpg	(%rax)
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -     invlpga	%rax, %ecx
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -     lahf
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -     leave
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -     lodsb	(%rsi), %al
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -     lodsw	(%rsi), %ax
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -     lodsl	(%rsi), %eax
@@ -2052,6 +2079,10 @@ xorq (%rax), %rdi
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -     rcrq	%cl, %rdi
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -     1.00    -     1.00    -      -      -      -     rclq	%cl, (%rax)
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -     1.00    -     1.00    -      -      -      -     rcrq	%cl, (%rax)
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -     rdmsr
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -     rdpmc
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -     rdtsc
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -     rdtscp
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -     rolb	%dil
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -     rorb	%dil
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -     1.00    -     1.00    -      -      -      -     rolb	(%rax)
@@ -2326,6 +2357,7 @@ xorq (%rax), %rdi
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -     testq	%rsi, %rdi
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -     1.00    -      -      -      -      -      -     testq	%rsi, (%rax)
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -     ud2
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -     wrmsr
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -     xaddb	%bl, %cl
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -     1.00    -     1.00    -      -      -      -     xaddb	%bl, (%rcx)
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -     xaddw	%bx, %cx
diff --git a/test/tools/llvm-mca/X86/BtVer2/resources-x87.s b/test/tools/llvm-mca/X86/BtVer2/resources-x87.s
index a0e431f6dfe45d8e24481bc12f7aab99bdcc8037..f738be857c4928b5b25f3d4f29b12d27824a251c 100644
--- a/test/tools/llvm-mca/X86/BtVer2/resources-x87.s
+++ b/test/tools/llvm-mca/X86/BtVer2/resources-x87.s
@@ -5,7 +5,7 @@ f2xm1
 
 fabs
 
-fadd %st(0), %st(1)
+fadd %st, %st(1)
 fadd %st(2)
 fadds (%ecx)
 faddl (%ecx)
@@ -21,14 +21,14 @@ fchs
 
 fnclex
 
-fcmovb %st(1), %st(0)
-fcmovbe %st(1), %st(0)
-fcmove %st(1), %st(0)
-fcmovnb %st(1), %st(0)
-fcmovnbe %st(1), %st(0)
-fcmovne %st(1), %st(0)
-fcmovnu %st(1), %st(0)
-fcmovu %st(1), %st(0)
+fcmovb %st(1), %st
+fcmovbe %st(1), %st
+fcmove %st(1), %st
+fcmovnb %st(1), %st
+fcmovnbe %st(1), %st
+fcmovne %st(1), %st
+fcmovnu %st(1), %st
+fcmovu %st(1), %st
 
 fcom %st(1)
 fcom %st(3)
@@ -47,7 +47,7 @@ fcos
 
 fdecstp
 
-fdiv %st(0), %st(1)
+fdiv %st, %st(1)
 fdiv %st(2)
 fdivs (%ecx)
 fdivl (%eax)
@@ -56,7 +56,7 @@ fdivp %st(2)
 fidivs (%ecx)
 fidivl (%eax)
 
-fdivr %st(0), %st(1)
+fdivr %st, %st(1)
 fdivr %st(2)
 fdivrs (%ecx)
 fdivrl (%eax)
@@ -106,7 +106,7 @@ fldln2
 fldpi
 fldz
 
-fmul %st(0), %st(1)
+fmul %st, %st(1)
 fmul %st(2)
 fmuls (%ecx)
 fmull (%eax)
@@ -153,7 +153,7 @@ fnstsw (%eax)
 frstor (%eax)
 fsave (%eax)
 
-fsub %st(0), %st(1)
+fsub %st, %st(1)
 fsub %st(2)
 fsubs (%ecx)
 fsubl (%eax)
@@ -162,7 +162,7 @@ fsubp %st(2)
 fisubs (%ecx)
 fisubl (%eax)
 
-fsubr %st(0), %st(1)
+fsubr %st, %st(1)
 fsubr %st(2)
 fsubrs (%ecx)
 fsubrl (%eax)
@@ -208,26 +208,26 @@ fyl2xp1
 # CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
 # CHECK-NEXT:  1      100   0.50                  U     f2xm1
 # CHECK-NEXT:  1      2     1.00                  U     fabs
-# CHECK-NEXT:  1      3     1.00                  U     fadd	%st(0), %st(1)
-# CHECK-NEXT:  1      3     1.00                  U     fadd	%st(2)
+# CHECK-NEXT:  1      3     1.00                  U     fadd	%st, %st(1)
+# CHECK-NEXT:  1      3     1.00                  U     fadd	%st(2), %st
 # CHECK-NEXT:  1      8     1.00    *             U     fadds	(%ecx)
 # CHECK-NEXT:  1      8     1.00    *             U     faddl	(%ecx)
-# CHECK-NEXT:  1      3     1.00                  U     faddp	%st(1)
-# CHECK-NEXT:  1      3     1.00                  U     faddp	%st(2)
+# CHECK-NEXT:  1      3     1.00                  U     faddp	%st, %st(1)
+# CHECK-NEXT:  1      3     1.00                  U     faddp	%st, %st(2)
 # CHECK-NEXT:  1      8     1.00    *             U     fiadds	(%ecx)
 # CHECK-NEXT:  1      8     1.00    *             U     fiaddl	(%ecx)
 # CHECK-NEXT:  1      100   0.50                  U     fbld	(%ecx)
 # CHECK-NEXT:  1      100   0.50                  U     fbstp	(%eax)
 # CHECK-NEXT:  1      2     1.00                  U     fchs
 # CHECK-NEXT:  1      100   0.50                  U     fnclex
-# CHECK-NEXT:  1      3     1.00                  U     fcmovb	%st(1), %st(0)
-# CHECK-NEXT:  1      3     1.00                  U     fcmovbe	%st(1), %st(0)
-# CHECK-NEXT:  1      3     1.00                  U     fcmove	%st(1), %st(0)
-# CHECK-NEXT:  1      3     1.00                  U     fcmovnb	%st(1), %st(0)
-# CHECK-NEXT:  1      3     1.00                  U     fcmovnbe	%st(1), %st(0)
-# CHECK-NEXT:  1      3     1.00                  U     fcmovne	%st(1), %st(0)
-# CHECK-NEXT:  1      3     1.00                  U     fcmovnu	%st(1), %st(0)
-# CHECK-NEXT:  1      3     1.00                  U     fcmovu	%st(1), %st(0)
+# CHECK-NEXT:  1      3     1.00                  U     fcmovb	%st(1), %st
+# CHECK-NEXT:  1      3     1.00                  U     fcmovbe	%st(1), %st
+# CHECK-NEXT:  1      3     1.00                  U     fcmove	%st(1), %st
+# CHECK-NEXT:  1      3     1.00                  U     fcmovnb	%st(1), %st
+# CHECK-NEXT:  1      3     1.00                  U     fcmovnbe	%st(1), %st
+# CHECK-NEXT:  1      3     1.00                  U     fcmovne	%st(1), %st
+# CHECK-NEXT:  1      3     1.00                  U     fcmovnu	%st(1), %st
+# CHECK-NEXT:  1      3     1.00                  U     fcmovu	%st(1), %st
 # CHECK-NEXT:  1      3     1.00                  U     fcom	%st(1)
 # CHECK-NEXT:  1      3     1.00                  U     fcom	%st(3)
 # CHECK-NEXT:  1      8     1.00                  U     fcoms	(%ecx)
@@ -237,24 +237,24 @@ fyl2xp1
 # CHECK-NEXT:  1      8     1.00                  U     fcomps	(%ecx)
 # CHECK-NEXT:  1      8     1.00                  U     fcompl	(%eax)
 # CHECK-NEXT:  1      100   0.50                  U     fcompp
-# CHECK-NEXT:  1      3     1.00                  U     fcomi	%st(3)
-# CHECK-NEXT:  1      3     1.00                  U     fcompi	%st(3)
+# CHECK-NEXT:  1      3     1.00                  U     fcomi	%st(3), %st
+# CHECK-NEXT:  1      3     1.00                  U     fcompi	%st(3), %st
 # CHECK-NEXT:  1      100   0.50                  U     fcos
 # CHECK-NEXT:  1      100   0.50                  U     fdecstp
-# CHECK-NEXT:  1      19    19.00                 U     fdiv	%st(0), %st(1)
-# CHECK-NEXT:  1      19    19.00                 U     fdiv	%st(2)
+# CHECK-NEXT:  1      19    19.00                 U     fdiv	%st, %st(1)
+# CHECK-NEXT:  1      19    19.00                 U     fdiv	%st(2), %st
 # CHECK-NEXT:  1      24    19.00   *             U     fdivs	(%ecx)
 # CHECK-NEXT:  1      24    19.00   *             U     fdivl	(%eax)
-# CHECK-NEXT:  1      19    19.00                 U     fdivp	%st(1)
-# CHECK-NEXT:  1      19    19.00                 U     fdivp	%st(2)
+# CHECK-NEXT:  1      19    19.00                 U     fdivp	%st, %st(1)
+# CHECK-NEXT:  1      19    19.00                 U     fdivp	%st, %st(2)
 # CHECK-NEXT:  1      24    19.00   *             U     fidivs	(%ecx)
 # CHECK-NEXT:  1      24    19.00   *             U     fidivl	(%eax)
-# CHECK-NEXT:  1      19    19.00                 U     fdivr	%st(0), %st(1)
-# CHECK-NEXT:  1      19    19.00                 U     fdivr	%st(2)
+# CHECK-NEXT:  1      19    19.00                 U     fdivr	%st, %st(1)
+# CHECK-NEXT:  1      19    19.00                 U     fdivr	%st(2), %st
 # CHECK-NEXT:  1      24    19.00   *             U     fdivrs	(%ecx)
 # CHECK-NEXT:  1      24    19.00   *             U     fdivrl	(%eax)
-# CHECK-NEXT:  1      19    19.00                 U     fdivrp	%st(1)
-# CHECK-NEXT:  1      19    19.00                 U     fdivrp	%st(2)
+# CHECK-NEXT:  1      19    19.00                 U     fdivrp	%st, %st(1)
+# CHECK-NEXT:  1      19    19.00                 U     fdivrp	%st, %st(2)
 # CHECK-NEXT:  1      24    19.00   *             U     fidivrs	(%ecx)
 # CHECK-NEXT:  1      24    19.00   *             U     fidivrl	(%eax)
 # CHECK-NEXT:  1      100   0.50                  U     ffree	%st(0)
@@ -262,9 +262,9 @@ fyl2xp1
 # CHECK-NEXT:  1      8     1.00                  U     ficoml	(%eax)
 # CHECK-NEXT:  1      8     1.00                  U     ficomps	(%ecx)
 # CHECK-NEXT:  1      8     1.00                  U     ficompl	(%eax)
-# CHECK-NEXT:  1      5     1.00    *             U     filds	(%edx)
-# CHECK-NEXT:  1      5     1.00    *             U     fildl	(%ecx)
-# CHECK-NEXT:  1      5     1.00    *             U     fildll	(%eax)
+# CHECK-NEXT:  1      3     1.00    *             U     filds	(%edx)
+# CHECK-NEXT:  1      3     1.00    *             U     fildl	(%ecx)
+# CHECK-NEXT:  1      3     1.00    *             U     fildll	(%eax)
 # CHECK-NEXT:  1      100   0.50                  U     fincstp
 # CHECK-NEXT:  1      100   0.50                  U     fninit
 # CHECK-NEXT:  1      1     1.00           *      U     fists	(%edx)
@@ -276,10 +276,10 @@ fyl2xp1
 # CHECK-NEXT:  1      1     1.00           *      U     fisttpl	(%ecx)
 # CHECK-NEXT:  1      1     1.00           *      U     fisttpll	(%eax)
 # CHECK-NEXT:  1      1     0.50                  U     fld	%st(0)
-# CHECK-NEXT:  1      5     1.00    *             U     flds	(%edx)
-# CHECK-NEXT:  1      5     1.00    *             U     fldl	(%ecx)
-# CHECK-NEXT:  1      5     1.00    *             U     fldt	(%eax)
-# CHECK-NEXT:  1      5     1.00    *             U     fldcw	(%eax)
+# CHECK-NEXT:  1      3     1.00    *             U     flds	(%edx)
+# CHECK-NEXT:  1      3     1.00    *             U     fldl	(%ecx)
+# CHECK-NEXT:  1      3     1.00    *             U     fldt	(%eax)
+# CHECK-NEXT:  1      3     1.00    *             U     fldcw	(%eax)
 # CHECK-NEXT:  1      100   0.50                  U     fldenv	(%eax)
 # CHECK-NEXT:  1      3     1.00                  U     fld1
 # CHECK-NEXT:  1      3     1.00                  U     fldl2e
@@ -288,12 +288,12 @@ fyl2xp1
 # CHECK-NEXT:  1      3     1.00                  U     fldln2
 # CHECK-NEXT:  1      3     1.00                  U     fldpi
 # CHECK-NEXT:  1      3     1.00                  U     fldz
-# CHECK-NEXT:  1      2     1.00                  U     fmul	%st(0), %st(1)
-# CHECK-NEXT:  1      2     1.00                  U     fmul	%st(2)
+# CHECK-NEXT:  1      2     1.00                  U     fmul	%st, %st(1)
+# CHECK-NEXT:  1      2     1.00                  U     fmul	%st(2), %st
 # CHECK-NEXT:  1      7     1.00    *             U     fmuls	(%ecx)
 # CHECK-NEXT:  1      7     1.00    *             U     fmull	(%eax)
-# CHECK-NEXT:  1      2     1.00                  U     fmulp	%st(1)
-# CHECK-NEXT:  1      2     1.00                  U     fmulp	%st(2)
+# CHECK-NEXT:  1      2     1.00                  U     fmulp	%st, %st(1)
+# CHECK-NEXT:  1      2     1.00                  U     fmulp	%st, %st(2)
 # CHECK-NEXT:  1      7     1.00    *             U     fimuls	(%ecx)
 # CHECK-NEXT:  1      7     1.00    *             U     fimull	(%eax)
 # CHECK-NEXT:  1      1     0.50                  U     fnop
@@ -321,20 +321,20 @@ fyl2xp1
 # CHECK-NEXT:  1      100   0.50                  U     frstor	(%eax)
 # CHECK-NEXT:  1      100   0.50                  U     wait
 # CHECK-NEXT:  1      100   0.50                  U     fnsave	(%eax)
-# CHECK-NEXT:  1      3     1.00                  U     fsub	%st(0), %st(1)
-# CHECK-NEXT:  1      3     1.00                  U     fsub	%st(2)
+# CHECK-NEXT:  1      3     1.00                  U     fsub	%st, %st(1)
+# CHECK-NEXT:  1      3     1.00                  U     fsub	%st(2), %st
 # CHECK-NEXT:  1      8     1.00    *             U     fsubs	(%ecx)
 # CHECK-NEXT:  1      8     1.00    *             U     fsubl	(%eax)
-# CHECK-NEXT:  1      3     1.00                  U     fsubp	%st(1)
-# CHECK-NEXT:  1      3     1.00                  U     fsubp	%st(2)
+# CHECK-NEXT:  1      3     1.00                  U     fsubp	%st, %st(1)
+# CHECK-NEXT:  1      3     1.00                  U     fsubp	%st, %st(2)
 # CHECK-NEXT:  1      8     1.00    *             U     fisubs	(%ecx)
 # CHECK-NEXT:  1      8     1.00    *             U     fisubl	(%eax)
-# CHECK-NEXT:  1      3     1.00                  U     fsubr	%st(0), %st(1)
-# CHECK-NEXT:  1      3     1.00                  U     fsubr	%st(2)
+# CHECK-NEXT:  1      3     1.00                  U     fsubr	%st, %st(1)
+# CHECK-NEXT:  1      3     1.00                  U     fsubr	%st(2), %st
 # CHECK-NEXT:  1      8     1.00    *             U     fsubrs	(%ecx)
 # CHECK-NEXT:  1      8     1.00    *             U     fsubrl	(%eax)
-# CHECK-NEXT:  1      3     1.00                  U     fsubrp	%st(1)
-# CHECK-NEXT:  1      3     1.00                  U     fsubrp	%st(2)
+# CHECK-NEXT:  1      3     1.00                  U     fsubrp	%st, %st(1)
+# CHECK-NEXT:  1      3     1.00                  U     fsubrp	%st, %st(2)
 # CHECK-NEXT:  1      8     1.00    *             U     fisubrs	(%ecx)
 # CHECK-NEXT:  1      8     1.00    *             U     fisubrl	(%eax)
 # CHECK-NEXT:  1      3     1.00                  U     ftst
@@ -343,8 +343,8 @@ fyl2xp1
 # CHECK-NEXT:  1      3     1.00                  U     fucomp	%st(1)
 # CHECK-NEXT:  1      3     1.00                  U     fucomp	%st(3)
 # CHECK-NEXT:  1      3     1.00                  U     fucompp
-# CHECK-NEXT:  1      3     1.00                  U     fucomi	%st(3)
-# CHECK-NEXT:  1      3     1.00                  U     fucompi	%st(3)
+# CHECK-NEXT:  1      3     1.00                  U     fucomi	%st(3), %st
+# CHECK-NEXT:  1      3     1.00                  U     fucompi	%st(3), %st
 # CHECK-NEXT:  1      100   0.50                  U     wait
 # CHECK-NEXT:  1      100   0.50                  U     fxam
 # CHECK-NEXT:  1      1     0.50                  U     fxch	%st(1)
@@ -379,26 +379,26 @@ fyl2xp1
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   [12]   [13]   Instructions:
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -     f2xm1
 # CHECK-NEXT:  -      -      -      -     1.00    -     1.00    -      -      -      -      -      -      -     fabs
-# CHECK-NEXT:  -      -      -     1.00    -     1.00    -      -      -      -      -      -      -      -     fadd	%st(0), %st(1)
-# CHECK-NEXT:  -      -      -     1.00    -     1.00    -      -      -      -      -      -      -      -     fadd	%st(2)
+# CHECK-NEXT:  -      -      -     1.00    -     1.00    -      -      -      -      -      -      -      -     fadd	%st, %st(1)
+# CHECK-NEXT:  -      -      -     1.00    -     1.00    -      -      -      -      -      -      -      -     fadd	%st(2), %st
 # CHECK-NEXT:  -      -      -     1.00    -     1.00    -     1.00    -      -      -      -      -      -     fadds	(%ecx)
 # CHECK-NEXT:  -      -      -     1.00    -     1.00    -     1.00    -      -      -      -      -      -     faddl	(%ecx)
-# CHECK-NEXT:  -      -      -     1.00    -     1.00    -      -      -      -      -      -      -      -     faddp	%st(1)
-# CHECK-NEXT:  -      -      -     1.00    -     1.00    -      -      -      -      -      -      -      -     faddp	%st(2)
+# CHECK-NEXT:  -      -      -     1.00    -     1.00    -      -      -      -      -      -      -      -     faddp	%st, %st(1)
+# CHECK-NEXT:  -      -      -     1.00    -     1.00    -      -      -      -      -      -      -      -     faddp	%st, %st(2)
 # CHECK-NEXT:  -      -      -     1.00    -     1.00    -     1.00    -      -      -      -      -      -     fiadds	(%ecx)
 # CHECK-NEXT:  -      -      -     1.00    -     1.00    -     1.00    -      -      -      -      -      -     fiaddl	(%ecx)
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -     fbld	(%ecx)
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -     fbstp	(%eax)
 # CHECK-NEXT:  -      -      -      -     1.00    -     1.00    -      -      -      -      -      -      -     fchs
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -     fnclex
-# CHECK-NEXT:  -      -      -     1.00    -     1.00    -      -      -      -      -      -      -      -     fcmovb	%st(1), %st(0)
-# CHECK-NEXT:  -      -      -     1.00    -     1.00    -      -      -      -      -      -      -      -     fcmovbe	%st(1), %st(0)
-# CHECK-NEXT:  -      -      -     1.00    -     1.00    -      -      -      -      -      -      -      -     fcmove	%st(1), %st(0)
-# CHECK-NEXT:  -      -      -     1.00    -     1.00    -      -      -      -      -      -      -      -     fcmovnb	%st(1), %st(0)
-# CHECK-NEXT:  -      -      -     1.00    -     1.00    -      -      -      -      -      -      -      -     fcmovnbe	%st(1), %st(0)
-# CHECK-NEXT:  -      -      -     1.00    -     1.00    -      -      -      -      -      -      -      -     fcmovne	%st(1), %st(0)
-# CHECK-NEXT:  -      -      -     1.00    -     1.00    -      -      -      -      -      -      -      -     fcmovnu	%st(1), %st(0)
-# CHECK-NEXT:  -      -      -     1.00    -     1.00    -      -      -      -      -      -      -      -     fcmovu	%st(1), %st(0)
+# CHECK-NEXT:  -      -      -     1.00    -     1.00    -      -      -      -      -      -      -      -     fcmovb	%st(1), %st
+# CHECK-NEXT:  -      -      -     1.00    -     1.00    -      -      -      -      -      -      -      -     fcmovbe	%st(1), %st
+# CHECK-NEXT:  -      -      -     1.00    -     1.00    -      -      -      -      -      -      -      -     fcmove	%st(1), %st
+# CHECK-NEXT:  -      -      -     1.00    -     1.00    -      -      -      -      -      -      -      -     fcmovnb	%st(1), %st
+# CHECK-NEXT:  -      -      -     1.00    -     1.00    -      -      -      -      -      -      -      -     fcmovnbe	%st(1), %st
+# CHECK-NEXT:  -      -      -     1.00    -     1.00    -      -      -      -      -      -      -      -     fcmovne	%st(1), %st
+# CHECK-NEXT:  -      -      -     1.00    -     1.00    -      -      -      -      -      -      -      -     fcmovnu	%st(1), %st
+# CHECK-NEXT:  -      -      -     1.00    -     1.00    -      -      -      -      -      -      -      -     fcmovu	%st(1), %st
 # CHECK-NEXT: 1.00    -      -     1.00    -     1.00    -      -      -      -      -      -      -      -     fcom	%st(1)
 # CHECK-NEXT: 1.00    -      -     1.00    -     1.00    -      -      -      -      -      -      -      -     fcom	%st(3)
 # CHECK-NEXT: 1.00    -      -     1.00    -     1.00    -     1.00    -      -      -      -      -      -     fcoms	(%ecx)
@@ -408,24 +408,24 @@ fyl2xp1
 # CHECK-NEXT: 1.00    -      -     1.00    -     1.00    -     1.00    -      -      -      -      -      -     fcomps	(%ecx)
 # CHECK-NEXT: 1.00    -      -     1.00    -     1.00    -     1.00    -      -      -      -      -      -     fcompl	(%eax)
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -     fcompp
-# CHECK-NEXT: 1.00    -      -     1.00    -     1.00    -      -      -      -      -      -      -      -     fcomi	%st(3)
-# CHECK-NEXT: 1.00    -      -     1.00    -     1.00    -      -      -      -      -      -      -      -     fcompi	%st(3)
+# CHECK-NEXT: 1.00    -      -     1.00    -     1.00    -      -      -      -      -      -      -      -     fcomi	%st(3), %st
+# CHECK-NEXT: 1.00    -      -     1.00    -     1.00    -      -      -      -      -      -      -      -     fcompi	%st(3), %st
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -     fcos
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -     fdecstp
-# CHECK-NEXT:  -      -      -      -     19.00   -     1.00    -      -      -      -      -      -      -     fdiv	%st(0), %st(1)
-# CHECK-NEXT:  -      -      -      -     19.00   -     1.00    -      -      -      -      -      -      -     fdiv	%st(2)
+# CHECK-NEXT:  -      -      -      -     19.00   -     1.00    -      -      -      -      -      -      -     fdiv	%st, %st(1)
+# CHECK-NEXT:  -      -      -      -     19.00   -     1.00    -      -      -      -      -      -      -     fdiv	%st(2), %st
 # CHECK-NEXT:  -      -      -      -     19.00   -     1.00   1.00    -      -      -      -      -      -     fdivs	(%ecx)
 # CHECK-NEXT:  -      -      -      -     19.00   -     1.00   1.00    -      -      -      -      -      -     fdivl	(%eax)
-# CHECK-NEXT:  -      -      -      -     19.00   -     1.00    -      -      -      -      -      -      -     fdivp	%st(1)
-# CHECK-NEXT:  -      -      -      -     19.00   -     1.00    -      -      -      -      -      -      -     fdivp	%st(2)
+# CHECK-NEXT:  -      -      -      -     19.00   -     1.00    -      -      -      -      -      -      -     fdivp	%st, %st(1)
+# CHECK-NEXT:  -      -      -      -     19.00   -     1.00    -      -      -      -      -      -      -     fdivp	%st, %st(2)
 # CHECK-NEXT:  -      -      -      -     19.00   -     1.00   1.00    -      -      -      -      -      -     fidivs	(%ecx)
 # CHECK-NEXT:  -      -      -      -     19.00   -     1.00   1.00    -      -      -      -      -      -     fidivl	(%eax)
-# CHECK-NEXT:  -      -      -      -     19.00   -     1.00    -      -      -      -      -      -      -     fdivr	%st(0), %st(1)
-# CHECK-NEXT:  -      -      -      -     19.00   -     1.00    -      -      -      -      -      -      -     fdivr	%st(2)
+# CHECK-NEXT:  -      -      -      -     19.00   -     1.00    -      -      -      -      -      -      -     fdivr	%st, %st(1)
+# CHECK-NEXT:  -      -      -      -     19.00   -     1.00    -      -      -      -      -      -      -     fdivr	%st(2), %st
 # CHECK-NEXT:  -      -      -      -     19.00   -     1.00   1.00    -      -      -      -      -      -     fdivrs	(%ecx)
 # CHECK-NEXT:  -      -      -      -     19.00   -     1.00   1.00    -      -      -      -      -      -     fdivrl	(%eax)
-# CHECK-NEXT:  -      -      -      -     19.00   -     1.00    -      -      -      -      -      -      -     fdivrp	%st(1)
-# CHECK-NEXT:  -      -      -      -     19.00   -     1.00    -      -      -      -      -      -      -     fdivrp	%st(2)
+# CHECK-NEXT:  -      -      -      -     19.00   -     1.00    -      -      -      -      -      -      -     fdivrp	%st, %st(1)
+# CHECK-NEXT:  -      -      -      -     19.00   -     1.00    -      -      -      -      -      -      -     fdivrp	%st, %st(2)
 # CHECK-NEXT:  -      -      -      -     19.00   -     1.00   1.00    -      -      -      -      -      -     fidivrs	(%ecx)
 # CHECK-NEXT:  -      -      -      -     19.00   -     1.00   1.00    -      -      -      -      -      -     fidivrl	(%eax)
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -     ffree	%st(0)
@@ -459,12 +459,12 @@ fyl2xp1
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     fldln2
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     fldpi
 # CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -     1.00    -      -      -     fldz
-# CHECK-NEXT:  -      -      -      -     1.00    -     1.00    -      -      -      -      -      -      -     fmul	%st(0), %st(1)
-# CHECK-NEXT:  -      -      -      -     1.00    -     1.00    -      -      -      -      -      -      -     fmul	%st(2)
+# CHECK-NEXT:  -      -      -      -     1.00    -     1.00    -      -      -      -      -      -      -     fmul	%st, %st(1)
+# CHECK-NEXT:  -      -      -      -     1.00    -     1.00    -      -      -      -      -      -      -     fmul	%st(2), %st
 # CHECK-NEXT:  -      -      -      -     1.00    -     1.00   1.00    -      -      -      -      -      -     fmuls	(%ecx)
 # CHECK-NEXT:  -      -      -      -     1.00    -     1.00   1.00    -      -      -      -      -      -     fmull	(%eax)
-# CHECK-NEXT:  -      -      -      -     1.00    -     1.00    -      -      -      -      -      -      -     fmulp	%st(1)
-# CHECK-NEXT:  -      -      -      -     1.00    -     1.00    -      -      -      -      -      -      -     fmulp	%st(2)
+# CHECK-NEXT:  -      -      -      -     1.00    -     1.00    -      -      -      -      -      -      -     fmulp	%st, %st(1)
+# CHECK-NEXT:  -      -      -      -     1.00    -     1.00    -      -      -      -      -      -      -     fmulp	%st, %st(2)
 # CHECK-NEXT:  -      -      -      -     1.00    -     1.00   1.00    -      -      -      -      -      -     fimuls	(%ecx)
 # CHECK-NEXT:  -      -      -      -     1.00    -     1.00   1.00    -      -      -      -      -      -     fimull	(%eax)
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -     fnop
@@ -492,20 +492,20 @@ fyl2xp1
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -     frstor	(%eax)
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -     wait
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -     fnsave	(%eax)
-# CHECK-NEXT:  -      -      -     1.00    -     1.00    -      -      -      -      -      -      -      -     fsub	%st(0), %st(1)
-# CHECK-NEXT:  -      -      -     1.00    -     1.00    -      -      -      -      -      -      -      -     fsub	%st(2)
+# CHECK-NEXT:  -      -      -     1.00    -     1.00    -      -      -      -      -      -      -      -     fsub	%st, %st(1)
+# CHECK-NEXT:  -      -      -     1.00    -     1.00    -      -      -      -      -      -      -      -     fsub	%st(2), %st
 # CHECK-NEXT:  -      -      -     1.00    -     1.00    -     1.00    -      -      -      -      -      -     fsubs	(%ecx)
 # CHECK-NEXT:  -      -      -     1.00    -     1.00    -     1.00    -      -      -      -      -      -     fsubl	(%eax)
-# CHECK-NEXT:  -      -      -     1.00    -     1.00    -      -      -      -      -      -      -      -     fsubp	%st(1)
-# CHECK-NEXT:  -      -      -     1.00    -     1.00    -      -      -      -      -      -      -      -     fsubp	%st(2)
+# CHECK-NEXT:  -      -      -     1.00    -     1.00    -      -      -      -      -      -      -      -     fsubp	%st, %st(1)
+# CHECK-NEXT:  -      -      -     1.00    -     1.00    -      -      -      -      -      -      -      -     fsubp	%st, %st(2)
 # CHECK-NEXT:  -      -      -     1.00    -     1.00    -     1.00    -      -      -      -      -      -     fisubs	(%ecx)
 # CHECK-NEXT:  -      -      -     1.00    -     1.00    -     1.00    -      -      -      -      -      -     fisubl	(%eax)
-# CHECK-NEXT:  -      -      -     1.00    -     1.00    -      -      -      -      -      -      -      -     fsubr	%st(0), %st(1)
-# CHECK-NEXT:  -      -      -     1.00    -     1.00    -      -      -      -      -      -      -      -     fsubr	%st(2)
+# CHECK-NEXT:  -      -      -     1.00    -     1.00    -      -      -      -      -      -      -      -     fsubr	%st, %st(1)
+# CHECK-NEXT:  -      -      -     1.00    -     1.00    -      -      -      -      -      -      -      -     fsubr	%st(2), %st
 # CHECK-NEXT:  -      -      -     1.00    -     1.00    -     1.00    -      -      -      -      -      -     fsubrs	(%ecx)
 # CHECK-NEXT:  -      -      -     1.00    -     1.00    -     1.00    -      -      -      -      -      -     fsubrl	(%eax)
-# CHECK-NEXT:  -      -      -     1.00    -     1.00    -      -      -      -      -      -      -      -     fsubrp	%st(1)
-# CHECK-NEXT:  -      -      -     1.00    -     1.00    -      -      -      -      -      -      -      -     fsubrp	%st(2)
+# CHECK-NEXT:  -      -      -     1.00    -     1.00    -      -      -      -      -      -      -      -     fsubrp	%st, %st(1)
+# CHECK-NEXT:  -      -      -     1.00    -     1.00    -      -      -      -      -      -      -      -     fsubrp	%st, %st(2)
 # CHECK-NEXT:  -      -      -     1.00    -     1.00    -     1.00    -      -      -      -      -      -     fisubrs	(%ecx)
 # CHECK-NEXT:  -      -      -     1.00    -     1.00    -     1.00    -      -      -      -      -      -     fisubrl	(%eax)
 # CHECK-NEXT: 1.00    -      -     1.00    -     1.00    -      -      -      -      -      -      -      -     ftst
@@ -514,8 +514,8 @@ fyl2xp1
 # CHECK-NEXT: 1.00    -      -     1.00    -     1.00    -      -      -      -      -      -      -      -     fucomp	%st(1)
 # CHECK-NEXT: 1.00    -      -     1.00    -     1.00    -      -      -      -      -      -      -      -     fucomp	%st(3)
 # CHECK-NEXT: 1.00    -      -     1.00    -     1.00    -      -      -      -      -      -      -      -     fucompp
-# CHECK-NEXT: 1.00    -      -     1.00    -     1.00    -      -      -      -      -      -      -      -     fucomi	%st(3)
-# CHECK-NEXT: 1.00    -      -     1.00    -     1.00    -      -      -      -      -      -      -      -     fucompi	%st(3)
+# CHECK-NEXT: 1.00    -      -     1.00    -     1.00    -      -      -      -      -      -      -      -     fucomi	%st(3), %st
+# CHECK-NEXT: 1.00    -      -     1.00    -     1.00    -      -      -      -      -      -      -      -     fucompi	%st(3), %st
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -     wait
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -     fxam
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -      -      -     fxch	%st(1)
diff --git a/test/tools/llvm-mca/X86/Generic/resources-avx2.s b/test/tools/llvm-mca/X86/Generic/resources-avx2.s
index f01f92909134bd37b9ce1c352e38fd4aaf1464c6..301292d3f2f94f83402a908c145a65ad51d41ada 100644
--- a/test/tools/llvm-mca/X86/Generic/resources-avx2.s
+++ b/test/tools/llvm-mca/X86/Generic/resources-avx2.s
@@ -149,6 +149,9 @@ vpcmpgtq        (%rax), %ymm1, %ymm2
 vpcmpgtw        %ymm0, %ymm1, %ymm2
 vpcmpgtw        (%rax), %ymm1, %ymm2
 
+vperm2i128      $1, %ymm0, %ymm1, %ymm2
+vperm2i128      $1, (%rax), %ymm1, %ymm2
+
 vpermd          %ymm0, %ymm1, %ymm2
 vpermd          (%rax), %ymm1, %ymm2
 
@@ -555,6 +558,8 @@ vpxor           (%rax), %ymm1, %ymm2
 # CHECK-NEXT:  2      8     0.50    *                   vpcmpgtq	(%rax), %ymm1, %ymm2
 # CHECK-NEXT:  1      1     0.50                        vpcmpgtw	%ymm0, %ymm1, %ymm2
 # CHECK-NEXT:  2      8     0.50    *                   vpcmpgtw	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      1     1.00                        vperm2i128	$1, %ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  2      8     1.00    *                   vperm2i128	$1, (%rax), %ymm1, %ymm2
 # CHECK-NEXT:  1      1     1.00                        vpermd	%ymm0, %ymm1, %ymm2
 # CHECK-NEXT:  2      8     1.00    *                   vpermd	(%rax), %ymm1, %ymm2
 # CHECK-NEXT:  1      1     1.00                        vpermpd	$1, %ymm0, %ymm2
@@ -769,7 +774,7 @@ vpxor           (%rax), %ymm1, %ymm2
 
 # CHECK:      Resource pressure per iteration:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6.0]  [6.1]
-# CHECK-NEXT:  -      -     75.67  88.67  5.00   166.67 78.00  78.00
+# CHECK-NEXT:  -      -     75.67  88.67  5.00   168.67 78.50  78.50
 
 # CHECK:      Resource pressure by instruction:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6.0]  [6.1]  Instructions:
@@ -871,6 +876,8 @@ vpxor           (%rax), %ymm1, %ymm2
 # CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpcmpgtq	(%rax), %ymm1, %ymm2
 # CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpcmpgtw	%ymm0, %ymm1, %ymm2
 # CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpcmpgtw	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vperm2i128	$1, %ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vperm2i128	$1, (%rax), %ymm1, %ymm2
 # CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpermd	%ymm0, %ymm1, %ymm2
 # CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermd	(%rax), %ymm1, %ymm2
 # CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpermpd	$1, %ymm0, %ymm2
diff --git a/test/tools/llvm-mca/X86/Generic/resources-avx512.s b/test/tools/llvm-mca/X86/Generic/resources-avx512.s
index b5d40ed766eb2eb0715f8cb8b0714ebfb77a9d6d..1bbd0f5ed704236aa26923f9905506b388e67a1f 100644
--- a/test/tools/llvm-mca/X86/Generic/resources-avx512.s
+++ b/test/tools/llvm-mca/X86/Generic/resources-avx512.s
@@ -3,129 +3,517 @@
 
 vaddpd            %zmm16, %zmm17, %zmm19
 vaddpd            (%rax), %zmm17, %zmm19
+vaddpd            (%rax){1to8}, %zmm17, %zmm19
 vaddpd            %zmm16, %zmm17, %zmm19 {k1}
 vaddpd            (%rax), %zmm17, %zmm19 {k1}
+vaddpd            (%rax){1to8}, %zmm17, %zmm19 {k1}
 vaddpd            %zmm16, %zmm17, %zmm19 {z}{k1}
 vaddpd            (%rax), %zmm17, %zmm19 {z}{k1}
+vaddpd            (%rax){1to8}, %zmm17, %zmm19 {z}{k1}
 
 vaddps            %zmm16, %zmm17, %zmm19
 vaddps            (%rax), %zmm17, %zmm19
+vaddps            (%rax){1to16}, %zmm17, %zmm19
 vaddps            %zmm16, %zmm17, %zmm19 {k1}
 vaddps            (%rax), %zmm17, %zmm19 {k1}
+vaddps            (%rax){1to16}, %zmm17, %zmm19 {k1}
 vaddps            %zmm16, %zmm17, %zmm19 {z}{k1}
 vaddps            (%rax), %zmm17, %zmm19 {z}{k1}
+vaddps            (%rax){1to16}, %zmm17, %zmm19 {z}{k1}
+
+vcvtdq2pd         %ymm16, %zmm19
+vcvtdq2pd         (%rax), %zmm19
+vcvtdq2pd         (%rax){1to8}, %zmm19
+vcvtdq2pd         %ymm16, %zmm19 {k1}
+vcvtdq2pd         (%rax), %zmm19 {k1}
+vcvtdq2pd         (%rax){1to8}, %zmm19 {k1}
+vcvtdq2pd         %ymm16, %zmm19 {z}{k1}
+vcvtdq2pd         (%rax), %zmm19 {z}{k1}
+vcvtdq2pd         (%rax){1to8}, %zmm19 {z}{k1}
+
+vcvtdq2ps         %zmm16, %zmm19
+vcvtdq2ps         (%rax), %zmm19
+vcvtdq2ps         (%rax){1to16}, %zmm19
+vcvtdq2ps         %zmm16, %zmm19 {k1}
+vcvtdq2ps         (%rax), %zmm19 {k1}
+vcvtdq2ps         (%rax){1to16}, %zmm19 {k1}
+vcvtdq2ps         %zmm16, %zmm19 {z}{k1}
+vcvtdq2ps         (%rax), %zmm19 {z}{k1}
+vcvtdq2ps         (%rax){1to16}, %zmm19 {z}{k1}
+
+vcvtps2dq         %zmm16, %zmm19
+vcvtps2dq         (%rax), %zmm19
+vcvtps2dq         (%rax){1to16}, %zmm19
+vcvtps2dq         %zmm16, %zmm19 {k1}
+vcvtps2dq         (%rax), %zmm19 {k1}
+vcvtps2dq         (%rax){1to16}, %zmm19 {k1}
+vcvtps2dq         %zmm16, %zmm19 {z}{k1}
+vcvtps2dq         (%rax), %zmm19 {z}{k1}
+vcvtps2dq         (%rax){1to16}, %zmm19 {z}{k1}
+
+vcvttps2dq        %zmm16, %zmm19
+vcvttps2dq        (%rax), %zmm19
+vcvttps2dq        (%rax){1to16}, %zmm19
+vcvttps2dq        %zmm16, %zmm19 {k1}
+vcvttps2dq        (%rax), %zmm19 {k1}
+vcvttps2dq        (%rax){1to16}, %zmm19 {k1}
+vcvttps2dq        %zmm16, %zmm19 {z}{k1}
+vcvttps2dq        (%rax), %zmm19 {z}{k1}
+vcvttps2dq        (%rax){1to16}, %zmm19 {z}{k1}
 
 vdivpd            %zmm16, %zmm17, %zmm19
 vdivpd            (%rax), %zmm17, %zmm19
+vdivpd            (%rax){1to8}, %zmm17, %zmm19
 vdivpd            %zmm16, %zmm17, %zmm19 {k1}
 vdivpd            (%rax), %zmm17, %zmm19 {k1}
+vdivpd            (%rax){1to8}, %zmm17, %zmm19 {k1}
 vdivpd            %zmm16, %zmm17, %zmm19 {z}{k1}
 vdivpd            (%rax), %zmm17, %zmm19 {z}{k1}
+vdivpd            (%rax){1to8}, %zmm17, %zmm19 {z}{k1}
 
 vdivps            %zmm16, %zmm17, %zmm19
 vdivps            (%rax), %zmm17, %zmm19
+vdivps            (%rax){1to16}, %zmm17, %zmm19
 vdivps            %zmm16, %zmm17, %zmm19 {k1}
 vdivps            (%rax), %zmm17, %zmm19 {k1}
+vdivps            (%rax){1to16}, %zmm17, %zmm19 {k1}
 vdivps            %zmm16, %zmm17, %zmm19 {z}{k1}
 vdivps            (%rax), %zmm17, %zmm19 {z}{k1}
+vdivps            (%rax){1to16}, %zmm17, %zmm19 {z}{k1}
+
+vmaxpd            %zmm16, %zmm17, %zmm19
+vmaxpd            (%rax), %zmm17, %zmm19
+vmaxpd            (%rax){1to8}, %zmm17, %zmm19
+vmaxpd            %zmm16, %zmm17, %zmm19 {k1}
+vmaxpd            (%rax), %zmm17, %zmm19 {k1}
+vmaxpd            (%rax){1to8}, %zmm17, %zmm19 {k1}
+vmaxpd            %zmm16, %zmm17, %zmm19 {z}{k1}
+vmaxpd            (%rax), %zmm17, %zmm19 {z}{k1}
+vmaxpd            (%rax){1to8}, %zmm17, %zmm19 {z}{k1}
+
+vmaxps            %zmm16, %zmm17, %zmm19
+vmaxps            (%rax), %zmm17, %zmm19
+vmaxps            (%rax){1to16}, %zmm17, %zmm19
+vmaxps            %zmm16, %zmm17, %zmm19 {k1}
+vmaxps            (%rax), %zmm17, %zmm19 {k1}
+vmaxps            (%rax){1to16}, %zmm17, %zmm19 {k1}
+vmaxps            %zmm16, %zmm17, %zmm19 {z}{k1}
+vmaxps            (%rax), %zmm17, %zmm19 {z}{k1}
+vmaxps            (%rax){1to16}, %zmm17, %zmm19 {z}{k1}
+
+vminpd            %zmm16, %zmm17, %zmm19
+vminpd            (%rax), %zmm17, %zmm19
+vminpd            (%rax){1to8}, %zmm17, %zmm19
+vminpd            %zmm16, %zmm17, %zmm19 {k1}
+vminpd            (%rax), %zmm17, %zmm19 {k1}
+vminpd            (%rax){1to8}, %zmm17, %zmm19 {k1}
+vminpd            %zmm16, %zmm17, %zmm19 {z}{k1}
+vminpd            (%rax), %zmm17, %zmm19 {z}{k1}
+vminpd            (%rax){1to8}, %zmm17, %zmm19 {z}{k1}
+
+vminps            %zmm16, %zmm17, %zmm19
+vminps            (%rax), %zmm17, %zmm19
+vminps            (%rax){1to16}, %zmm17, %zmm19
+vminps            %zmm16, %zmm17, %zmm19 {k1}
+vminps            (%rax), %zmm17, %zmm19 {k1}
+vminps            (%rax){1to16}, %zmm17, %zmm19 {k1}
+vminps            %zmm16, %zmm17, %zmm19 {z}{k1}
+vminps            (%rax), %zmm17, %zmm19 {z}{k1}
+vminps            (%rax){1to16}, %zmm17, %zmm19 {z}{k1}
 
 vmulpd            %zmm16, %zmm17, %zmm19
 vmulpd            (%rax), %zmm17, %zmm19
+vmulpd            (%rax){1to8}, %zmm17, %zmm19
 vmulpd            %zmm16, %zmm17, %zmm19 {k1}
 vmulpd            (%rax), %zmm17, %zmm19 {k1}
+vmulpd            (%rax){1to8}, %zmm17, %zmm19 {k1}
 vmulpd            %zmm16, %zmm17, %zmm19 {z}{k1}
 vmulpd            (%rax), %zmm17, %zmm19 {z}{k1}
+vmulpd            (%rax){1to8}, %zmm17, %zmm19 {z}{k1}
 
 vmulps            %zmm16, %zmm17, %zmm19
 vmulps            (%rax), %zmm17, %zmm19
+vmulps            (%rax){1to16}, %zmm17, %zmm19
 vmulps            %zmm16, %zmm17, %zmm19 {k1}
 vmulps            (%rax), %zmm17, %zmm19 {k1}
+vmulps            (%rax){1to16}, %zmm17, %zmm19 {k1}
 vmulps            %zmm16, %zmm17, %zmm19 {z}{k1}
 vmulps            (%rax), %zmm17, %zmm19 {z}{k1}
+vmulps            (%rax){1to16}, %zmm17, %zmm19 {z}{k1}
 
 vpabsd            %zmm16, %zmm19
 vpabsd            (%rax), %zmm19
+vpabsd            (%rax){1to16}, %zmm19
 vpabsd            %zmm16, %zmm19 {k1}
 vpabsd            (%rax), %zmm19 {k1}
+vpabsd            (%rax){1to16}, %zmm19 {k1}
 vpabsd            %zmm16, %zmm19 {z}{k1}
 vpabsd            (%rax), %zmm19 {z}{k1}
+vpabsd            (%rax){1to16}, %zmm19 {z}{k1}
 
 vpabsq            %zmm16, %zmm19
 vpabsq            (%rax), %zmm19
+vpabsq            (%rax){1to8}, %zmm19
 vpabsq            %zmm16, %zmm19 {k1}
 vpabsq            (%rax), %zmm19 {k1}
+vpabsq            (%rax){1to8}, %zmm19 {k1}
 vpabsq            %zmm16, %zmm19 {z}{k1}
 vpabsq            (%rax), %zmm19 {z}{k1}
-
-vpaddb            %zmm16, %zmm17, %zmm19
-vpaddb            (%rax), %zmm17, %zmm19
-vpaddb            %zmm16, %zmm17, %zmm19 {k1}
-vpaddb            (%rax), %zmm17, %zmm19 {k1}
-vpaddb            %zmm16, %zmm17, %zmm19 {z}{k1}
-vpaddb            (%rax), %zmm17, %zmm19 {z}{k1}
+vpabsq            (%rax){1to8}, %zmm19 {z}{k1}
 
 vpaddd            %zmm16, %zmm17, %zmm19
 vpaddd            (%rax), %zmm17, %zmm19
+vpaddd            (%rax){1to16}, %zmm17, %zmm19
 vpaddd            %zmm16, %zmm17, %zmm19 {k1}
 vpaddd            (%rax), %zmm17, %zmm19 {k1}
+vpaddd            (%rax){1to16}, %zmm17, %zmm19 {k1}
 vpaddd            %zmm16, %zmm17, %zmm19 {z}{k1}
 vpaddd            (%rax), %zmm17, %zmm19 {z}{k1}
+vpaddd            (%rax){1to16}, %zmm17, %zmm19 {z}{k1}
 
 vpaddq            %zmm16, %zmm17, %zmm19
 vpaddq            (%rax), %zmm17, %zmm19
+vpaddq            (%rax){1to8}, %zmm17, %zmm19
 vpaddq            %zmm16, %zmm17, %zmm19 {k1}
 vpaddq            (%rax), %zmm17, %zmm19 {k1}
+vpaddq            (%rax){1to8}, %zmm17, %zmm19 {k1}
 vpaddq            %zmm16, %zmm17, %zmm19 {z}{k1}
 vpaddq            (%rax), %zmm17, %zmm19 {z}{k1}
+vpaddq            (%rax){1to8}, %zmm17, %zmm19 {z}{k1}
+
+vpmulld           %zmm16, %zmm17, %zmm19
+vpmulld           (%rax), %zmm17, %zmm19
+vpmulld           (%rax){1to16}, %zmm17, %zmm19
+vpmulld           %zmm16, %zmm17, %zmm19 {k1}
+vpmulld           (%rax), %zmm17, %zmm19 {k1}
+vpmulld           (%rax){1to16}, %zmm17, %zmm19 {k1}
+vpmulld           %zmm16, %zmm17, %zmm19 {z}{k1}
+vpmulld           (%rax), %zmm17, %zmm19 {z}{k1}
+vpmulld           (%rax){1to16}, %zmm17, %zmm19 {z}{k1}
+
+vpermd            %zmm16, %zmm17, %zmm19
+vpermd            (%rax), %zmm17, %zmm19
+vpermd            (%rax){1to16}, %zmm17, %zmm19
+vpermd            %zmm16, %zmm17, %zmm19 {k1}
+vpermd            (%rax), %zmm17, %zmm19 {k1}
+vpermd            (%rax){1to16}, %zmm17, %zmm19 {k1}
+vpermd            %zmm16, %zmm17, %zmm19 {z}{k1}
+vpermd            (%rax), %zmm17, %zmm19 {z}{k1}
+vpermd            (%rax){1to16}, %zmm17, %zmm19 {z}{k1}
+
+vpermilpd         $0, %zmm16, %zmm19
+vpermilpd         $0, (%rax), %zmm19
+vpermilpd         $0, (%rax){1to8}, %zmm19
+vpermilpd         $0, %zmm16, %zmm19 {k1}
+vpermilpd         $0, (%rax), %zmm19 {k1}
+vpermilpd         $0, (%rax){1to8}, %zmm19 {k1}
+vpermilpd         $0, %zmm16, %zmm19 {z}{k1}
+vpermilpd         $0, (%rax), %zmm19 {z}{k1}
+vpermilpd         $0, (%rax){1to8}, %zmm19 {z}{k1}
+
+vpermilpd         %zmm16, %zmm17, %zmm19
+vpermilpd         (%rax), %zmm17, %zmm19
+vpermilpd         (%rax){1to8}, %zmm17, %zmm19
+vpermilpd         %zmm16, %zmm17, %zmm19 {k1}
+vpermilpd         (%rax), %zmm17, %zmm19 {k1}
+vpermilpd         (%rax){1to8}, %zmm17, %zmm19 {k1}
+vpermilpd         %zmm16, %zmm17, %zmm19 {z}{k1}
+vpermilpd         (%rax), %zmm17, %zmm19 {z}{k1}
+vpermilpd         (%rax){1to8}, %zmm17, %zmm19 {z}{k1}
+
+vpermilps         $0, %zmm16, %zmm19
+vpermilps         $0, (%rax), %zmm19
+vpermilps         $0, (%rax){1to16}, %zmm19
+vpermilps         $0, %zmm16, %zmm19 {k1}
+vpermilps         $0, (%rax), %zmm19 {k1}
+vpermilps         $0, (%rax){1to16}, %zmm19 {k1}
+vpermilps         $0, %zmm16, %zmm19 {z}{k1}
+vpermilps         $0, (%rax), %zmm19 {z}{k1}
+vpermilps         $0, (%rax){1to16}, %zmm19 {z}{k1}
+
+vpermilps         %zmm16, %zmm17, %zmm19
+vpermilps         (%rax), %zmm17, %zmm19
+vpermilps         (%rax){1to16}, %zmm17, %zmm19
+vpermilps         %zmm16, %zmm17, %zmm19 {k1}
+vpermilps         (%rax), %zmm17, %zmm19 {k1}
+vpermilps         (%rax){1to16}, %zmm17, %zmm19 {k1}
+vpermilps         %zmm16, %zmm17, %zmm19 {z}{k1}
+vpermilps         (%rax), %zmm17, %zmm19 {z}{k1}
+vpermilps         (%rax){1to16}, %zmm17, %zmm19 {z}{k1}
+
+vpermpd           $0, %zmm16, %zmm19
+vpermpd           $0, (%rax), %zmm19
+vpermpd           $0, (%rax){1to8}, %zmm19
+vpermpd           $0, %zmm16, %zmm19 {k1}
+vpermpd           $0, (%rax), %zmm19 {k1}
+vpermpd           $0, (%rax){1to8}, %zmm19 {k1}
+vpermpd           $0, %zmm16, %zmm19 {z}{k1}
+vpermpd           $0, (%rax), %zmm19 {z}{k1}
+vpermpd           $0, (%rax){1to8}, %zmm19 {z}{k1}
+
+vpermpd           %zmm16, %zmm17, %zmm19
+vpermpd           (%rax), %zmm17, %zmm19
+vpermpd           (%rax){1to8}, %zmm17, %zmm19
+vpermpd           %zmm16, %zmm17, %zmm19 {k1}
+vpermpd           (%rax), %zmm17, %zmm19 {k1}
+vpermpd           (%rax){1to8}, %zmm17, %zmm19 {k1}
+vpermpd           %zmm16, %zmm17, %zmm19 {z}{k1}
+vpermpd           (%rax), %zmm17, %zmm19 {z}{k1}
+vpermpd           (%rax){1to8}, %zmm17, %zmm19 {z}{k1}
+
+vpermps           %zmm16, %zmm17, %zmm19
+vpermps           (%rax), %zmm17, %zmm19
+vpermps           (%rax){1to16}, %zmm17, %zmm19
+vpermps           %zmm16, %zmm17, %zmm19 {k1}
+vpermps           (%rax), %zmm17, %zmm19 {k1}
+vpermps           (%rax){1to16}, %zmm17, %zmm19 {k1}
+vpermps           %zmm16, %zmm17, %zmm19 {z}{k1}
+vpermps           (%rax), %zmm17, %zmm19 {z}{k1}
+vpermps           (%rax){1to16}, %zmm17, %zmm19 {z}{k1}
 
-vpaddw            %zmm16, %zmm17, %zmm19
-vpaddw            (%rax), %zmm17, %zmm19
-vpaddw            %zmm16, %zmm17, %zmm19 {k1}
-vpaddw            (%rax), %zmm17, %zmm19 {k1}
-vpaddw            %zmm16, %zmm17, %zmm19 {z}{k1}
-vpaddw            (%rax), %zmm17, %zmm19 {z}{k1}
+vpermq            $0, %zmm16, %zmm19
+vpermq            $0, (%rax), %zmm19
+vpermq            $0, (%rax){1to8}, %zmm19
+vpermq            $0, %zmm16, %zmm19 {k1}
+vpermq            $0, (%rax), %zmm19 {k1}
+vpermq            $0, (%rax){1to8}, %zmm19 {k1}
+vpermq            $0, %zmm16, %zmm19 {z}{k1}
+vpermq            $0, (%rax), %zmm19 {z}{k1}
+vpermq            $0, (%rax){1to8}, %zmm19 {z}{k1}
 
-vpsubb            %zmm16, %zmm17, %zmm19
-vpsubb            (%rax), %zmm17, %zmm19
-vpsubb            %zmm16, %zmm17, %zmm19 {k1}
-vpsubb            (%rax), %zmm17, %zmm19 {k1}
-vpsubb            %zmm16, %zmm17, %zmm19 {z}{k1}
-vpsubb            (%rax), %zmm17, %zmm19 {z}{k1}
+vpermq            %zmm16, %zmm17, %zmm19
+vpermq            (%rax), %zmm17, %zmm19
+vpermq            (%rax){1to8}, %zmm17, %zmm19
+vpermq            %zmm16, %zmm17, %zmm19 {k1}
+vpermq            (%rax), %zmm17, %zmm19 {k1}
+vpermq            (%rax){1to8}, %zmm17, %zmm19 {k1}
+vpermq            %zmm16, %zmm17, %zmm19 {z}{k1}
+vpermq            (%rax), %zmm17, %zmm19 {z}{k1}
+vpermq            (%rax){1to8}, %zmm17, %zmm19 {z}{k1}
+
+vpshufd           $0, %zmm16, %zmm19
+vpshufd           $0, (%rax), %zmm19
+vpshufd           $0, (%rax){1to16}, %zmm19
+vpshufd           $0, %zmm16, %zmm19 {k1}
+vpshufd           $0, (%rax), %zmm19 {k1}
+vpshufd           $0, (%rax){1to16}, %zmm19 {k1}
+vpshufd           $0, %zmm16, %zmm19 {z}{k1}
+vpshufd           $0, (%rax), %zmm19 {z}{k1}
+vpshufd           $0, (%rax){1to16}, %zmm19 {z}{k1}
 
 vpsubd            %zmm16, %zmm17, %zmm19
 vpsubd            (%rax), %zmm17, %zmm19
+vpsubd            (%rax){1to16}, %zmm17, %zmm19
 vpsubd            %zmm16, %zmm17, %zmm19 {k1}
 vpsubd            (%rax), %zmm17, %zmm19 {k1}
+vpsubd            (%rax){1to16}, %zmm17, %zmm19 {k1}
 vpsubd            %zmm16, %zmm17, %zmm19 {z}{k1}
 vpsubd            (%rax), %zmm17, %zmm19 {z}{k1}
+vpsubd            (%rax){1to16}, %zmm17, %zmm19 {z}{k1}
 
 vpsubq            %zmm16, %zmm17, %zmm19
 vpsubq            (%rax), %zmm17, %zmm19
+vpsubq            (%rax){1to8}, %zmm17, %zmm19
 vpsubq            %zmm16, %zmm17, %zmm19 {k1}
 vpsubq            (%rax), %zmm17, %zmm19 {k1}
+vpsubq            (%rax){1to8}, %zmm17, %zmm19 {k1}
 vpsubq            %zmm16, %zmm17, %zmm19 {z}{k1}
 vpsubq            (%rax), %zmm17, %zmm19 {z}{k1}
+vpsubq            (%rax){1to8}, %zmm17, %zmm19 {z}{k1}
+
+vpunpckhdq        %zmm16, %zmm17, %zmm19
+vpunpckhdq        (%rax), %zmm17, %zmm19
+vpunpckhdq        (%rax){1to16}, %zmm17, %zmm19
+vpunpckhdq        %zmm16, %zmm17, %zmm19 {k1}
+vpunpckhdq        (%rax), %zmm17, %zmm19 {k1}
+vpunpckhdq        (%rax){1to16}, %zmm17, %zmm19 {k1}
+vpunpckhdq        %zmm16, %zmm17, %zmm19 {z}{k1}
+vpunpckhdq        (%rax), %zmm17, %zmm19 {z}{k1}
+vpunpckhdq        (%rax){1to16}, %zmm17, %zmm19 {z}{k1}
+
+vpunpckhqdq       %zmm16, %zmm17, %zmm19
+vpunpckhqdq       (%rax), %zmm17, %zmm19
+vpunpckhqdq       (%rax){1to8}, %zmm17, %zmm19
+vpunpckhqdq       %zmm16, %zmm17, %zmm19 {k1}
+vpunpckhqdq       (%rax), %zmm17, %zmm19 {k1}
+vpunpckhqdq       (%rax){1to8}, %zmm17, %zmm19 {k1}
+vpunpckhqdq       %zmm16, %zmm17, %zmm19 {z}{k1}
+vpunpckhqdq       (%rax), %zmm17, %zmm19 {z}{k1}
+vpunpckhqdq       (%rax){1to8}, %zmm17, %zmm19 {z}{k1}
+
+vpunpckldq        %zmm16, %zmm17, %zmm19
+vpunpckldq        (%rax), %zmm17, %zmm19
+vpunpckldq        (%rax){1to16}, %zmm17, %zmm19
+vpunpckldq        %zmm16, %zmm17, %zmm19 {k1}
+vpunpckldq        (%rax), %zmm17, %zmm19 {k1}
+vpunpckldq        (%rax){1to16}, %zmm17, %zmm19 {k1}
+vpunpckldq        %zmm16, %zmm17, %zmm19 {z}{k1}
+vpunpckldq        (%rax), %zmm17, %zmm19 {z}{k1}
+vpunpckldq        (%rax){1to16}, %zmm17, %zmm19 {z}{k1}
+
+vpunpcklqdq       %zmm16, %zmm17, %zmm19
+vpunpcklqdq       (%rax), %zmm17, %zmm19
+vpunpcklqdq       (%rax){1to8}, %zmm17, %zmm19
+vpunpcklqdq       %zmm16, %zmm17, %zmm19 {k1}
+vpunpcklqdq       (%rax), %zmm17, %zmm19 {k1}
+vpunpcklqdq       (%rax){1to8}, %zmm17, %zmm19 {k1}
+vpunpcklqdq       %zmm16, %zmm17, %zmm19 {z}{k1}
+vpunpcklqdq       (%rax), %zmm17, %zmm19 {z}{k1}
+vpunpcklqdq       (%rax){1to8}, %zmm17, %zmm19 {z}{k1}
+
+vshuff32x4        $0, %zmm16, %zmm17, %zmm19
+vshuff32x4        $0, (%rax), %zmm17, %zmm19
+vshuff32x4        $0, (%rax){1to16}, %zmm17, %zmm19
+vshuff32x4        $0, %zmm16, %zmm17, %zmm19 {k1}
+vshuff32x4        $0, (%rax), %zmm17, %zmm19 {k1}
+vshuff32x4        $0, (%rax){1to16}, %zmm17, %zmm19 {k1}
+vshuff32x4        $0, %zmm16, %zmm17, %zmm19 {z}{k1}
+vshuff32x4        $0, (%rax), %zmm17, %zmm19 {z}{k1}
+vshuff32x4        $0, (%rax){1to16}, %zmm17, %zmm19 {z}{k1}
+
+vshuff64x2        $0, %zmm16, %zmm17, %zmm19
+vshuff64x2        $0, (%rax), %zmm17, %zmm19
+vshuff64x2        $0, (%rax){1to8}, %zmm17, %zmm19
+vshuff64x2        $0, %zmm16, %zmm17, %zmm19 {k1}
+vshuff64x2        $0, (%rax), %zmm17, %zmm19 {k1}
+vshuff64x2        $0, (%rax){1to8}, %zmm17, %zmm19 {k1}
+vshuff64x2        $0, %zmm16, %zmm17, %zmm19 {z}{k1}
+vshuff64x2        $0, (%rax), %zmm17, %zmm19 {z}{k1}
+vshuff64x2        $0, (%rax){1to8}, %zmm17, %zmm19 {z}{k1}
+
+vshufi32x4        $0, %zmm16, %zmm17, %zmm19
+vshufi32x4        $0, (%rax), %zmm17, %zmm19
+vshufi32x4        $0, (%rax){1to16}, %zmm17, %zmm19
+vshufi32x4        $0, %zmm16, %zmm17, %zmm19 {k1}
+vshufi32x4        $0, (%rax), %zmm17, %zmm19 {k1}
+vshufi32x4        $0, (%rax){1to16}, %zmm17, %zmm19 {k1}
+vshufi32x4        $0, %zmm16, %zmm17, %zmm19 {z}{k1}
+vshufi32x4        $0, (%rax), %zmm17, %zmm19 {z}{k1}
+vshufi32x4        $0, (%rax){1to16}, %zmm17, %zmm19 {z}{k1}
+
+vshufi64x2        $0, %zmm16, %zmm17, %zmm19
+vshufi64x2        $0, (%rax), %zmm17, %zmm19
+vshufi64x2        $0, (%rax){1to8}, %zmm17, %zmm19
+vshufi64x2        $0, %zmm16, %zmm17, %zmm19 {k1}
+vshufi64x2        $0, (%rax), %zmm17, %zmm19 {k1}
+vshufi64x2        $0, (%rax){1to8}, %zmm17, %zmm19 {k1}
+vshufi64x2        $0, %zmm16, %zmm17, %zmm19 {z}{k1}
+vshufi64x2        $0, (%rax), %zmm17, %zmm19 {z}{k1}
+vshufi64x2        $0, (%rax){1to8}, %zmm17, %zmm19 {z}{k1}
+
+vsqrtpd           %zmm16, %zmm19
+vsqrtpd           (%rax), %zmm19
+vsqrtpd           (%rax){1to8}, %zmm19
+vsqrtpd           %zmm16, %zmm19 {k1}
+vsqrtpd           (%rax), %zmm19 {k1}
+vsqrtpd           (%rax){1to8}, %zmm19 {k1}
+vsqrtpd           %zmm16, %zmm19 {z}{k1}
+vsqrtpd           (%rax), %zmm19 {z}{k1}
+vsqrtpd           (%rax){1to8}, %zmm19 {z}{k1}
+
+vsqrtps           %zmm16, %zmm19
+vsqrtps           (%rax), %zmm19
+vsqrtps           (%rax){1to16}, %zmm19
+vsqrtps           %zmm16, %zmm19 {k1}
+vsqrtps           (%rax), %zmm19 {k1}
+vsqrtps           (%rax){1to16}, %zmm19 {k1}
+vsqrtps           %zmm16, %zmm19 {z}{k1}
+vsqrtps           (%rax), %zmm19 {z}{k1}
+vsqrtps           (%rax){1to16}, %zmm19 {z}{k1}
+
+vsqrtsd           %xmm16, %xmm17, %xmm19
+vsqrtsd           (%rax), %xmm17, %xmm19
+vsqrtsd           %xmm16, %xmm17, %xmm19 {k1}
+vsqrtsd           (%rax), %xmm17, %xmm19 {k1}
+vsqrtsd           %xmm16, %xmm17, %xmm19 {z}{k1}
+vsqrtsd           (%rax), %xmm17, %xmm19 {z}{k1}
+
+vsqrtss           %xmm16, %xmm17, %xmm19
+vsqrtss           (%rax), %xmm17, %xmm19
+vsqrtss           %xmm16, %xmm17, %xmm19 {k1}
+vsqrtss           (%rax), %xmm17, %xmm19 {k1}
+vsqrtss           %xmm16, %xmm17, %xmm19 {z}{k1}
+vsqrtss           (%rax), %xmm17, %xmm19 {z}{k1}
+
+vpsubd            %zmm16, %zmm17, %zmm19
+vpsubd            (%rax), %zmm17, %zmm19
+vpsubd            (%rax){1to16}, %zmm17, %zmm19
+vpsubd            %zmm16, %zmm17, %zmm19 {k1}
+vpsubd            (%rax), %zmm17, %zmm19 {k1}
+vpsubd            (%rax){1to16}, %zmm17, %zmm19 {k1}
+vpsubd            %zmm16, %zmm17, %zmm19 {z}{k1}
+vpsubd            (%rax), %zmm17, %zmm19 {z}{k1}
+vpsubd            (%rax){1to16}, %zmm17, %zmm19 {z}{k1}
 
-vpsubw            %zmm16, %zmm17, %zmm19
-vpsubw            (%rax), %zmm17, %zmm19
-vpsubw            %zmm16, %zmm17, %zmm19 {k1}
-vpsubw            (%rax), %zmm17, %zmm19 {k1}
-vpsubw            %zmm16, %zmm17, %zmm19 {z}{k1}
-vpsubw            (%rax), %zmm17, %zmm19 {z}{k1}
+vpsubq            %zmm16, %zmm17, %zmm19
+vpsubq            (%rax), %zmm17, %zmm19
+vpsubq            (%rax){1to8}, %zmm17, %zmm19
+vpsubq            %zmm16, %zmm17, %zmm19 {k1}
+vpsubq            (%rax), %zmm17, %zmm19 {k1}
+vpsubq            (%rax){1to8}, %zmm17, %zmm19 {k1}
+vpsubq            %zmm16, %zmm17, %zmm19 {z}{k1}
+vpsubq            (%rax), %zmm17, %zmm19 {z}{k1}
+vpsubq            (%rax){1to8}, %zmm17, %zmm19 {z}{k1}
 
 vsubpd            %zmm16, %zmm17, %zmm19
 vsubpd            (%rax), %zmm17, %zmm19
+vsubpd            (%rax){1to8}, %zmm17, %zmm19
 vsubpd            %zmm16, %zmm17, %zmm19 {k1}
 vsubpd            (%rax), %zmm17, %zmm19 {k1}
+vsubpd            (%rax){1to8}, %zmm17, %zmm19 {k1}
 vsubpd            %zmm16, %zmm17, %zmm19 {z}{k1}
 vsubpd            (%rax), %zmm17, %zmm19 {z}{k1}
+vsubpd            (%rax){1to8}, %zmm17, %zmm19 {z}{k1}
 
 vsubps            %zmm16, %zmm17, %zmm19
 vsubps            (%rax), %zmm17, %zmm19
+vsubps            (%rax){1to16}, %zmm17, %zmm19
 vsubps            %zmm16, %zmm17, %zmm19 {k1}
 vsubps            (%rax), %zmm17, %zmm19 {k1}
+vsubps            (%rax){1to16}, %zmm17, %zmm19 {k1}
 vsubps            %zmm16, %zmm17, %zmm19 {z}{k1}
 vsubps            (%rax), %zmm17, %zmm19 {z}{k1}
+vsubps            (%rax){1to16}, %zmm17, %zmm19 {z}{k1}
+
+vunpckhpd         %zmm16, %zmm17, %zmm19
+vunpckhpd         (%rax), %zmm17, %zmm19
+vunpckhpd         (%rax){1to8}, %zmm17, %zmm19
+vunpckhpd         %zmm16, %zmm17, %zmm19 {k1}
+vunpckhpd         (%rax), %zmm17, %zmm19 {k1}
+vunpckhpd         (%rax){1to8}, %zmm17, %zmm19 {k1}
+vunpckhpd         %zmm16, %zmm17, %zmm19 {z}{k1}
+vunpckhpd         (%rax), %zmm17, %zmm19 {z}{k1}
+vunpckhpd         (%rax){1to8}, %zmm17, %zmm19 {z}{k1}
+
+vunpckhps         %zmm16, %zmm17, %zmm19
+vunpckhps         (%rax), %zmm17, %zmm19
+vunpckhps         (%rax){1to16}, %zmm17, %zmm19
+vunpckhps         %zmm16, %zmm17, %zmm19 {k1}
+vunpckhps         (%rax), %zmm17, %zmm19 {k1}
+vunpckhps         (%rax){1to16}, %zmm17, %zmm19 {k1}
+vunpckhps         %zmm16, %zmm17, %zmm19 {z}{k1}
+vunpckhps         (%rax), %zmm17, %zmm19 {z}{k1}
+vunpckhps         (%rax){1to16}, %zmm17, %zmm19 {z}{k1}
+
+vunpcklpd         %zmm16, %zmm17, %zmm19
+vunpcklpd         (%rax), %zmm17, %zmm19
+vunpcklpd         (%rax){1to8}, %zmm17, %zmm19
+vunpcklpd         %zmm16, %zmm17, %zmm19 {k1}
+vunpcklpd         (%rax), %zmm17, %zmm19 {k1}
+vunpcklpd         (%rax){1to8}, %zmm17, %zmm19 {k1}
+vunpcklpd         %zmm16, %zmm17, %zmm19 {z}{k1}
+vunpcklpd         (%rax), %zmm17, %zmm19 {z}{k1}
+vunpcklpd         (%rax){1to8}, %zmm17, %zmm19 {z}{k1}
+
+vunpcklps         %zmm16, %zmm17, %zmm19
+vunpcklps         (%rax), %zmm17, %zmm19
+vunpcklps         (%rax){1to16}, %zmm17, %zmm19
+vunpcklps         %zmm16, %zmm17, %zmm19 {k1}
+vunpcklps         (%rax), %zmm17, %zmm19 {k1}
+vunpcklps         (%rax){1to16}, %zmm17, %zmm19 {k1}
+vunpcklps         %zmm16, %zmm17, %zmm19 {z}{k1}
+vunpcklps         (%rax), %zmm17, %zmm19 {z}{k1}
+vunpcklps         (%rax){1to16}, %zmm17, %zmm19 {z}{k1}
 
 # CHECK:      Instruction Info:
 # CHECK-NEXT: [1]: #uOps
@@ -138,112 +526,466 @@ vsubps            (%rax), %zmm17, %zmm19 {z}{k1}
 # CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
 # CHECK-NEXT:  1      3     1.00                        vaddpd	%zmm16, %zmm17, %zmm19
 # CHECK-NEXT:  2      10    1.00    *                   vaddpd	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  2      10    1.00    *                   vaddpd	(%rax){1to8}, %zmm17, %zmm19
 # CHECK-NEXT:  1      3     1.00                        vaddpd	%zmm16, %zmm17, %zmm19 {%k1}
 # CHECK-NEXT:  2      10    1.00    *                   vaddpd	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      10    1.00    *                   vaddpd	(%rax){1to8}, %zmm17, %zmm19 {%k1}
 # CHECK-NEXT:  1      3     1.00                        vaddpd	%zmm16, %zmm17, %zmm19 {%k1} {z}
 # CHECK-NEXT:  2      10    1.00    *                   vaddpd	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      10    1.00    *                   vaddpd	(%rax){1to8}, %zmm17, %zmm19 {%k1} {z}
 # CHECK-NEXT:  1      3     1.00                        vaddps	%zmm16, %zmm17, %zmm19
 # CHECK-NEXT:  2      10    1.00    *                   vaddps	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  2      10    1.00    *                   vaddps	(%rax){1to16}, %zmm17, %zmm19
 # CHECK-NEXT:  1      3     1.00                        vaddps	%zmm16, %zmm17, %zmm19 {%k1}
 # CHECK-NEXT:  2      10    1.00    *                   vaddps	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      10    1.00    *                   vaddps	(%rax){1to16}, %zmm17, %zmm19 {%k1}
 # CHECK-NEXT:  1      3     1.00                        vaddps	%zmm16, %zmm17, %zmm19 {%k1} {z}
 # CHECK-NEXT:  2      10    1.00    *                   vaddps	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      10    1.00    *                   vaddps	(%rax){1to16}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      4     1.00                        vcvtdq2pd	%ymm16, %zmm19
+# CHECK-NEXT:  3      10    1.00    *                   vcvtdq2pd	(%rax), %zmm19
+# CHECK-NEXT:  3      10    1.00    *                   vcvtdq2pd	(%rax){1to8}, %zmm19
+# CHECK-NEXT:  2      4     1.00                        vcvtdq2pd	%ymm16, %zmm19 {%k1}
+# CHECK-NEXT:  3      10    1.00    *                   vcvtdq2pd	(%rax), %zmm19 {%k1}
+# CHECK-NEXT:  3      10    1.00    *                   vcvtdq2pd	(%rax){1to8}, %zmm19 {%k1}
+# CHECK-NEXT:  2      4     1.00                        vcvtdq2pd	%ymm16, %zmm19 {%k1} {z}
+# CHECK-NEXT:  3      10    1.00    *                   vcvtdq2pd	(%rax), %zmm19 {%k1} {z}
+# CHECK-NEXT:  3      10    1.00    *                   vcvtdq2pd	(%rax){1to8}, %zmm19 {%k1} {z}
+# CHECK-NEXT:  1      3     1.00                        vcvtdq2ps	%zmm16, %zmm19
+# CHECK-NEXT:  2      10    1.00    *                   vcvtdq2ps	(%rax), %zmm19
+# CHECK-NEXT:  2      10    1.00    *                   vcvtdq2ps	(%rax){1to16}, %zmm19
+# CHECK-NEXT:  1      3     1.00                        vcvtdq2ps	%zmm16, %zmm19 {%k1}
+# CHECK-NEXT:  2      10    1.00    *                   vcvtdq2ps	(%rax), %zmm19 {%k1}
+# CHECK-NEXT:  2      10    1.00    *                   vcvtdq2ps	(%rax){1to16}, %zmm19 {%k1}
+# CHECK-NEXT:  1      3     1.00                        vcvtdq2ps	%zmm16, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      10    1.00    *                   vcvtdq2ps	(%rax), %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      10    1.00    *                   vcvtdq2ps	(%rax){1to16}, %zmm19 {%k1} {z}
+# CHECK-NEXT:  1      3     1.00                        vcvtps2dq	%zmm16, %zmm19
+# CHECK-NEXT:  2      10    1.00    *                   vcvtps2dq	(%rax), %zmm19
+# CHECK-NEXT:  2      10    1.00    *                   vcvtps2dq	(%rax){1to16}, %zmm19
+# CHECK-NEXT:  1      3     1.00                        vcvtps2dq	%zmm16, %zmm19 {%k1}
+# CHECK-NEXT:  2      10    1.00    *                   vcvtps2dq	(%rax), %zmm19 {%k1}
+# CHECK-NEXT:  2      10    1.00    *                   vcvtps2dq	(%rax){1to16}, %zmm19 {%k1}
+# CHECK-NEXT:  1      3     1.00                        vcvtps2dq	%zmm16, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      10    1.00    *                   vcvtps2dq	(%rax), %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      10    1.00    *                   vcvtps2dq	(%rax){1to16}, %zmm19 {%k1} {z}
+# CHECK-NEXT:  1      3     1.00                        vcvttps2dq	%zmm16, %zmm19
+# CHECK-NEXT:  2      10    1.00    *                   vcvttps2dq	(%rax), %zmm19
+# CHECK-NEXT:  2      10    1.00    *                   vcvttps2dq	(%rax){1to16}, %zmm19
+# CHECK-NEXT:  1      3     1.00                        vcvttps2dq	%zmm16, %zmm19 {%k1}
+# CHECK-NEXT:  2      10    1.00    *                   vcvttps2dq	(%rax), %zmm19 {%k1}
+# CHECK-NEXT:  2      10    1.00    *                   vcvttps2dq	(%rax){1to16}, %zmm19 {%k1}
+# CHECK-NEXT:  1      3     1.00                        vcvttps2dq	%zmm16, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      10    1.00    *                   vcvttps2dq	(%rax), %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      10    1.00    *                   vcvttps2dq	(%rax){1to16}, %zmm19 {%k1} {z}
 # CHECK-NEXT:  3      45    44.00                       vdivpd	%zmm16, %zmm17, %zmm19
 # CHECK-NEXT:  4      52    44.00   *                   vdivpd	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  4      52    44.00   *                   vdivpd	(%rax){1to8}, %zmm17, %zmm19
 # CHECK-NEXT:  3      45    44.00                       vdivpd	%zmm16, %zmm17, %zmm19 {%k1}
 # CHECK-NEXT:  4      52    44.00   *                   vdivpd	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  4      52    44.00   *                   vdivpd	(%rax){1to8}, %zmm17, %zmm19 {%k1}
 # CHECK-NEXT:  3      45    44.00                       vdivpd	%zmm16, %zmm17, %zmm19 {%k1} {z}
 # CHECK-NEXT:  4      52    44.00   *                   vdivpd	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  4      52    44.00   *                   vdivpd	(%rax){1to8}, %zmm17, %zmm19 {%k1} {z}
 # CHECK-NEXT:  3      29    28.00                       vdivps	%zmm16, %zmm17, %zmm19
 # CHECK-NEXT:  4      36    28.00   *                   vdivps	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  4      36    28.00   *                   vdivps	(%rax){1to16}, %zmm17, %zmm19
 # CHECK-NEXT:  3      29    28.00                       vdivps	%zmm16, %zmm17, %zmm19 {%k1}
 # CHECK-NEXT:  4      36    28.00   *                   vdivps	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  4      36    28.00   *                   vdivps	(%rax){1to16}, %zmm17, %zmm19 {%k1}
 # CHECK-NEXT:  3      29    28.00                       vdivps	%zmm16, %zmm17, %zmm19 {%k1} {z}
 # CHECK-NEXT:  4      36    28.00   *                   vdivps	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  4      36    28.00   *                   vdivps	(%rax){1to16}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  1      3     1.00                        vmaxpd	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  2      10    1.00    *                   vmaxpd	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  2      10    1.00    *                   vmaxpd	(%rax){1to8}, %zmm17, %zmm19
+# CHECK-NEXT:  1      3     1.00                        vmaxpd	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      10    1.00    *                   vmaxpd	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      10    1.00    *                   vmaxpd	(%rax){1to8}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  1      3     1.00                        vmaxpd	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      10    1.00    *                   vmaxpd	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      10    1.00    *                   vmaxpd	(%rax){1to8}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  1      3     1.00                        vmaxps	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  2      10    1.00    *                   vmaxps	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  2      10    1.00    *                   vmaxps	(%rax){1to16}, %zmm17, %zmm19
+# CHECK-NEXT:  1      3     1.00                        vmaxps	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      10    1.00    *                   vmaxps	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      10    1.00    *                   vmaxps	(%rax){1to16}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  1      3     1.00                        vmaxps	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      10    1.00    *                   vmaxps	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      10    1.00    *                   vmaxps	(%rax){1to16}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  1      3     1.00                        vminpd	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  2      10    1.00    *                   vminpd	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  2      10    1.00    *                   vminpd	(%rax){1to8}, %zmm17, %zmm19
+# CHECK-NEXT:  1      3     1.00                        vminpd	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      10    1.00    *                   vminpd	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      10    1.00    *                   vminpd	(%rax){1to8}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  1      3     1.00                        vminpd	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      10    1.00    *                   vminpd	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      10    1.00    *                   vminpd	(%rax){1to8}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  1      3     1.00                        vminps	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  2      10    1.00    *                   vminps	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  2      10    1.00    *                   vminps	(%rax){1to16}, %zmm17, %zmm19
+# CHECK-NEXT:  1      3     1.00                        vminps	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      10    1.00    *                   vminps	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      10    1.00    *                   vminps	(%rax){1to16}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  1      3     1.00                        vminps	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      10    1.00    *                   vminps	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      10    1.00    *                   vminps	(%rax){1to16}, %zmm17, %zmm19 {%k1} {z}
 # CHECK-NEXT:  1      5     1.00                        vmulpd	%zmm16, %zmm17, %zmm19
 # CHECK-NEXT:  2      12    1.00    *                   vmulpd	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  2      12    1.00    *                   vmulpd	(%rax){1to8}, %zmm17, %zmm19
 # CHECK-NEXT:  1      5     1.00                        vmulpd	%zmm16, %zmm17, %zmm19 {%k1}
 # CHECK-NEXT:  2      12    1.00    *                   vmulpd	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      12    1.00    *                   vmulpd	(%rax){1to8}, %zmm17, %zmm19 {%k1}
 # CHECK-NEXT:  1      5     1.00                        vmulpd	%zmm16, %zmm17, %zmm19 {%k1} {z}
 # CHECK-NEXT:  2      12    1.00    *                   vmulpd	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      12    1.00    *                   vmulpd	(%rax){1to8}, %zmm17, %zmm19 {%k1} {z}
 # CHECK-NEXT:  1      5     1.00                        vmulps	%zmm16, %zmm17, %zmm19
 # CHECK-NEXT:  2      12    1.00    *                   vmulps	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  2      12    1.00    *                   vmulps	(%rax){1to16}, %zmm17, %zmm19
 # CHECK-NEXT:  1      5     1.00                        vmulps	%zmm16, %zmm17, %zmm19 {%k1}
 # CHECK-NEXT:  2      12    1.00    *                   vmulps	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      12    1.00    *                   vmulps	(%rax){1to16}, %zmm17, %zmm19 {%k1}
 # CHECK-NEXT:  1      5     1.00                        vmulps	%zmm16, %zmm17, %zmm19 {%k1} {z}
 # CHECK-NEXT:  2      12    1.00    *                   vmulps	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      12    1.00    *                   vmulps	(%rax){1to16}, %zmm17, %zmm19 {%k1} {z}
 # CHECK-NEXT:  1      1     0.50                        vpabsd	%zmm16, %zmm19
 # CHECK-NEXT:  2      8     0.50    *                   vpabsd	(%rax), %zmm19
+# CHECK-NEXT:  2      8     0.50    *                   vpabsd	(%rax){1to16}, %zmm19
 # CHECK-NEXT:  1      1     0.50                        vpabsd	%zmm16, %zmm19 {%k1}
 # CHECK-NEXT:  2      8     0.50    *                   vpabsd	(%rax), %zmm19 {%k1}
+# CHECK-NEXT:  2      8     0.50    *                   vpabsd	(%rax){1to16}, %zmm19 {%k1}
 # CHECK-NEXT:  1      1     0.50                        vpabsd	%zmm16, %zmm19 {%k1} {z}
 # CHECK-NEXT:  2      8     0.50    *                   vpabsd	(%rax), %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     0.50    *                   vpabsd	(%rax){1to16}, %zmm19 {%k1} {z}
 # CHECK-NEXT:  1      1     0.50                        vpabsq	%zmm16, %zmm19
 # CHECK-NEXT:  2      8     0.50    *                   vpabsq	(%rax), %zmm19
+# CHECK-NEXT:  2      8     0.50    *                   vpabsq	(%rax){1to8}, %zmm19
 # CHECK-NEXT:  1      1     0.50                        vpabsq	%zmm16, %zmm19 {%k1}
 # CHECK-NEXT:  2      8     0.50    *                   vpabsq	(%rax), %zmm19 {%k1}
+# CHECK-NEXT:  2      8     0.50    *                   vpabsq	(%rax){1to8}, %zmm19 {%k1}
 # CHECK-NEXT:  1      1     0.50                        vpabsq	%zmm16, %zmm19 {%k1} {z}
 # CHECK-NEXT:  2      8     0.50    *                   vpabsq	(%rax), %zmm19 {%k1} {z}
-# CHECK-NEXT:  1      1     0.50                        vpaddb	%zmm16, %zmm17, %zmm19
-# CHECK-NEXT:  2      8     0.50    *                   vpaddb	(%rax), %zmm17, %zmm19
-# CHECK-NEXT:  1      1     0.50                        vpaddb	%zmm16, %zmm17, %zmm19 {%k1}
-# CHECK-NEXT:  2      8     0.50    *                   vpaddb	(%rax), %zmm17, %zmm19 {%k1}
-# CHECK-NEXT:  1      1     0.50                        vpaddb	%zmm16, %zmm17, %zmm19 {%k1} {z}
-# CHECK-NEXT:  2      8     0.50    *                   vpaddb	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     0.50    *                   vpabsq	(%rax){1to8}, %zmm19 {%k1} {z}
 # CHECK-NEXT:  1      1     0.50                        vpaddd	%zmm16, %zmm17, %zmm19
 # CHECK-NEXT:  2      8     0.50    *                   vpaddd	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  2      8     0.50    *                   vpaddd	(%rax){1to16}, %zmm17, %zmm19
 # CHECK-NEXT:  1      1     0.50                        vpaddd	%zmm16, %zmm17, %zmm19 {%k1}
 # CHECK-NEXT:  2      8     0.50    *                   vpaddd	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     0.50    *                   vpaddd	(%rax){1to16}, %zmm17, %zmm19 {%k1}
 # CHECK-NEXT:  1      1     0.50                        vpaddd	%zmm16, %zmm17, %zmm19 {%k1} {z}
 # CHECK-NEXT:  2      8     0.50    *                   vpaddd	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     0.50    *                   vpaddd	(%rax){1to16}, %zmm17, %zmm19 {%k1} {z}
 # CHECK-NEXT:  1      1     0.50                        vpaddq	%zmm16, %zmm17, %zmm19
 # CHECK-NEXT:  2      8     0.50    *                   vpaddq	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  2      8     0.50    *                   vpaddq	(%rax){1to8}, %zmm17, %zmm19
 # CHECK-NEXT:  1      1     0.50                        vpaddq	%zmm16, %zmm17, %zmm19 {%k1}
 # CHECK-NEXT:  2      8     0.50    *                   vpaddq	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     0.50    *                   vpaddq	(%rax){1to8}, %zmm17, %zmm19 {%k1}
 # CHECK-NEXT:  1      1     0.50                        vpaddq	%zmm16, %zmm17, %zmm19 {%k1} {z}
 # CHECK-NEXT:  2      8     0.50    *                   vpaddq	(%rax), %zmm17, %zmm19 {%k1} {z}
-# CHECK-NEXT:  1      1     0.50                        vpaddw	%zmm16, %zmm17, %zmm19
-# CHECK-NEXT:  2      8     0.50    *                   vpaddw	(%rax), %zmm17, %zmm19
-# CHECK-NEXT:  1      1     0.50                        vpaddw	%zmm16, %zmm17, %zmm19 {%k1}
-# CHECK-NEXT:  2      8     0.50    *                   vpaddw	(%rax), %zmm17, %zmm19 {%k1}
-# CHECK-NEXT:  1      1     0.50                        vpaddw	%zmm16, %zmm17, %zmm19 {%k1} {z}
-# CHECK-NEXT:  2      8     0.50    *                   vpaddw	(%rax), %zmm17, %zmm19 {%k1} {z}
-# CHECK-NEXT:  1      1     0.50                        vpsubb	%zmm16, %zmm17, %zmm19
-# CHECK-NEXT:  2      8     0.50    *                   vpsubb	(%rax), %zmm17, %zmm19
-# CHECK-NEXT:  1      1     0.50                        vpsubb	%zmm16, %zmm17, %zmm19 {%k1}
-# CHECK-NEXT:  2      8     0.50    *                   vpsubb	(%rax), %zmm17, %zmm19 {%k1}
-# CHECK-NEXT:  1      1     0.50                        vpsubb	%zmm16, %zmm17, %zmm19 {%k1} {z}
-# CHECK-NEXT:  2      8     0.50    *                   vpsubb	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     0.50    *                   vpaddq	(%rax){1to8}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  1      5     1.00                        vpmulld	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  2      12    1.00    *                   vpmulld	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  2      12    1.00    *                   vpmulld	(%rax){1to16}, %zmm17, %zmm19
+# CHECK-NEXT:  1      5     1.00                        vpmulld	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      12    1.00    *                   vpmulld	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      12    1.00    *                   vpmulld	(%rax){1to16}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  1      5     1.00                        vpmulld	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      12    1.00    *                   vpmulld	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      12    1.00    *                   vpmulld	(%rax){1to16}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vpermd	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  2      8     1.00    *                   vpermd	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  2      8     1.00    *                   vpermd	(%rax){1to16}, %zmm17, %zmm19
+# CHECK-NEXT:  1      1     1.00                        vpermd	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vpermd	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vpermd	(%rax){1to16}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vpermd	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vpermd	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vpermd	(%rax){1to16}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vpermilpd	$0, %zmm16, %zmm19
+# CHECK-NEXT:  2      8     1.00    *                   vpermilpd	$0, (%rax), %zmm19
+# CHECK-NEXT:  2      8     1.00    *                   vpermilpd	$0, (%rax){1to8}, %zmm19
+# CHECK-NEXT:  1      1     1.00                        vpermilpd	$0, %zmm16, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vpermilpd	$0, (%rax), %zmm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vpermilpd	$0, (%rax){1to8}, %zmm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vpermilpd	$0, %zmm16, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vpermilpd	$0, (%rax), %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vpermilpd	$0, (%rax){1to8}, %zmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vpermilpd	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  2      8     1.00    *                   vpermilpd	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  2      8     1.00    *                   vpermilpd	(%rax){1to8}, %zmm17, %zmm19
+# CHECK-NEXT:  1      1     1.00                        vpermilpd	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vpermilpd	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vpermilpd	(%rax){1to8}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vpermilpd	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vpermilpd	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vpermilpd	(%rax){1to8}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vpermilps	$0, %zmm16, %zmm19
+# CHECK-NEXT:  2      8     1.00    *                   vpermilps	$0, (%rax), %zmm19
+# CHECK-NEXT:  2      8     1.00    *                   vpermilps	$0, (%rax){1to16}, %zmm19
+# CHECK-NEXT:  1      1     1.00                        vpermilps	$0, %zmm16, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vpermilps	$0, (%rax), %zmm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vpermilps	$0, (%rax){1to16}, %zmm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vpermilps	$0, %zmm16, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vpermilps	$0, (%rax), %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vpermilps	$0, (%rax){1to16}, %zmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vpermilps	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  2      8     1.00    *                   vpermilps	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  2      8     1.00    *                   vpermilps	(%rax){1to16}, %zmm17, %zmm19
+# CHECK-NEXT:  1      1     1.00                        vpermilps	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vpermilps	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vpermilps	(%rax){1to16}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vpermilps	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vpermilps	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vpermilps	(%rax){1to16}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vpermpd	$0, %zmm16, %zmm19
+# CHECK-NEXT:  2      8     1.00    *                   vpermpd	$0, (%rax), %zmm19
+# CHECK-NEXT:  2      8     1.00    *                   vpermpd	$0, (%rax){1to8}, %zmm19
+# CHECK-NEXT:  1      1     1.00                        vpermpd	$0, %zmm16, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vpermpd	$0, (%rax), %zmm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vpermpd	$0, (%rax){1to8}, %zmm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vpermpd	$0, %zmm16, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vpermpd	$0, (%rax), %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vpermpd	$0, (%rax){1to8}, %zmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vpermpd	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  2      8     1.00    *                   vpermpd	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  2      8     1.00    *                   vpermpd	(%rax){1to8}, %zmm17, %zmm19
+# CHECK-NEXT:  1      1     1.00                        vpermpd	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vpermpd	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vpermpd	(%rax){1to8}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vpermpd	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vpermpd	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vpermpd	(%rax){1to8}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vpermps	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  2      8     1.00    *                   vpermps	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  2      8     1.00    *                   vpermps	(%rax){1to16}, %zmm17, %zmm19
+# CHECK-NEXT:  1      1     1.00                        vpermps	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vpermps	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vpermps	(%rax){1to16}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vpermps	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vpermps	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vpermps	(%rax){1to16}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vpermq	$0, %zmm16, %zmm19
+# CHECK-NEXT:  2      8     1.00    *                   vpermq	$0, (%rax), %zmm19
+# CHECK-NEXT:  2      8     1.00    *                   vpermq	$0, (%rax){1to8}, %zmm19
+# CHECK-NEXT:  1      1     1.00                        vpermq	$0, %zmm16, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vpermq	$0, (%rax), %zmm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vpermq	$0, (%rax){1to8}, %zmm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vpermq	$0, %zmm16, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vpermq	$0, (%rax), %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vpermq	$0, (%rax){1to8}, %zmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vpermq	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  2      8     1.00    *                   vpermq	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  2      8     1.00    *                   vpermq	(%rax){1to8}, %zmm17, %zmm19
+# CHECK-NEXT:  1      1     1.00                        vpermq	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vpermq	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vpermq	(%rax){1to8}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vpermq	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vpermq	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vpermq	(%rax){1to8}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vpshufd	$0, %zmm16, %zmm19
+# CHECK-NEXT:  2      8     1.00    *                   vpshufd	$0, (%rax), %zmm19
+# CHECK-NEXT:  2      8     1.00    *                   vpshufd	$0, (%rax){1to16}, %zmm19
+# CHECK-NEXT:  1      1     1.00                        vpshufd	$0, %zmm16, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vpshufd	$0, (%rax), %zmm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vpshufd	$0, (%rax){1to16}, %zmm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vpshufd	$0, %zmm16, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vpshufd	$0, (%rax), %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vpshufd	$0, (%rax){1to16}, %zmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     0.50                        vpsubd	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  2      8     0.50    *                   vpsubd	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  2      8     0.50    *                   vpsubd	(%rax){1to16}, %zmm17, %zmm19
+# CHECK-NEXT:  1      1     0.50                        vpsubd	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     0.50    *                   vpsubd	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     0.50    *                   vpsubd	(%rax){1to16}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  1      1     0.50                        vpsubd	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     0.50    *                   vpsubd	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     0.50    *                   vpsubd	(%rax){1to16}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     0.50                        vpsubq	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  2      8     0.50    *                   vpsubq	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  2      8     0.50    *                   vpsubq	(%rax){1to8}, %zmm17, %zmm19
+# CHECK-NEXT:  1      1     0.50                        vpsubq	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     0.50    *                   vpsubq	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     0.50    *                   vpsubq	(%rax){1to8}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  1      1     0.50                        vpsubq	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     0.50    *                   vpsubq	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     0.50    *                   vpsubq	(%rax){1to8}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vpunpckhdq	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  2      8     1.00    *                   vpunpckhdq	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  2      8     1.00    *                   vpunpckhdq	(%rax){1to16}, %zmm17, %zmm19
+# CHECK-NEXT:  1      1     1.00                        vpunpckhdq	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vpunpckhdq	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vpunpckhdq	(%rax){1to16}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vpunpckhdq	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vpunpckhdq	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vpunpckhdq	(%rax){1to16}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vpunpckhqdq	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  2      8     1.00    *                   vpunpckhqdq	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  2      8     1.00    *                   vpunpckhqdq	(%rax){1to8}, %zmm17, %zmm19
+# CHECK-NEXT:  1      1     1.00                        vpunpckhqdq	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vpunpckhqdq	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vpunpckhqdq	(%rax){1to8}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vpunpckhqdq	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vpunpckhqdq	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vpunpckhqdq	(%rax){1to8}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vpunpckldq	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  2      8     1.00    *                   vpunpckldq	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  2      8     1.00    *                   vpunpckldq	(%rax){1to16}, %zmm17, %zmm19
+# CHECK-NEXT:  1      1     1.00                        vpunpckldq	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vpunpckldq	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vpunpckldq	(%rax){1to16}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vpunpckldq	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vpunpckldq	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vpunpckldq	(%rax){1to16}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vpunpcklqdq	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  2      8     1.00    *                   vpunpcklqdq	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  2      8     1.00    *                   vpunpcklqdq	(%rax){1to8}, %zmm17, %zmm19
+# CHECK-NEXT:  1      1     1.00                        vpunpcklqdq	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vpunpcklqdq	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vpunpcklqdq	(%rax){1to8}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vpunpcklqdq	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vpunpcklqdq	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vpunpcklqdq	(%rax){1to8}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vshuff32x4	$0, %zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  2      8     1.00    *                   vshuff32x4	$0, (%rax), %zmm17, %zmm19
+# CHECK-NEXT:  2      8     1.00    *                   vshuff32x4	$0, (%rax){1to16}, %zmm17, %zmm19
+# CHECK-NEXT:  1      1     1.00                        vshuff32x4	$0, %zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vshuff32x4	$0, (%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vshuff32x4	$0, (%rax){1to16}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vshuff32x4	$0, %zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vshuff32x4	$0, (%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vshuff32x4	$0, (%rax){1to16}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vshuff64x2	$0, %zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  2      8     1.00    *                   vshuff64x2	$0, (%rax), %zmm17, %zmm19
+# CHECK-NEXT:  2      8     1.00    *                   vshuff64x2	$0, (%rax){1to8}, %zmm17, %zmm19
+# CHECK-NEXT:  1      1     1.00                        vshuff64x2	$0, %zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vshuff64x2	$0, (%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vshuff64x2	$0, (%rax){1to8}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vshuff64x2	$0, %zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vshuff64x2	$0, (%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vshuff64x2	$0, (%rax){1to8}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vshufi32x4	$0, %zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  2      8     1.00    *                   vshufi32x4	$0, (%rax), %zmm17, %zmm19
+# CHECK-NEXT:  2      8     1.00    *                   vshufi32x4	$0, (%rax){1to16}, %zmm17, %zmm19
+# CHECK-NEXT:  1      1     1.00                        vshufi32x4	$0, %zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vshufi32x4	$0, (%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vshufi32x4	$0, (%rax){1to16}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vshufi32x4	$0, %zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vshufi32x4	$0, (%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vshufi32x4	$0, (%rax){1to16}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vshufi64x2	$0, %zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  2      8     1.00    *                   vshufi64x2	$0, (%rax), %zmm17, %zmm19
+# CHECK-NEXT:  2      8     1.00    *                   vshufi64x2	$0, (%rax){1to8}, %zmm17, %zmm19
+# CHECK-NEXT:  1      1     1.00                        vshufi64x2	$0, %zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vshufi64x2	$0, (%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vshufi64x2	$0, (%rax){1to8}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vshufi64x2	$0, %zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vshufi64x2	$0, (%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vshufi64x2	$0, (%rax){1to8}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  3      45    44.00                       vsqrtpd	%zmm16, %zmm19
+# CHECK-NEXT:  4      52    44.00   *                   vsqrtpd	(%rax), %zmm19
+# CHECK-NEXT:  4      52    44.00   *                   vsqrtpd	(%rax){1to8}, %zmm19
+# CHECK-NEXT:  3      45    44.00                       vsqrtpd	%zmm16, %zmm19 {%k1}
+# CHECK-NEXT:  4      52    44.00   *                   vsqrtpd	(%rax), %zmm19 {%k1}
+# CHECK-NEXT:  4      52    44.00   *                   vsqrtpd	(%rax){1to8}, %zmm19 {%k1}
+# CHECK-NEXT:  3      45    44.00                       vsqrtpd	%zmm16, %zmm19 {%k1} {z}
+# CHECK-NEXT:  4      52    44.00   *                   vsqrtpd	(%rax), %zmm19 {%k1} {z}
+# CHECK-NEXT:  4      52    44.00   *                   vsqrtpd	(%rax){1to8}, %zmm19 {%k1} {z}
+# CHECK-NEXT:  3      29    28.00                       vsqrtps	%zmm16, %zmm19
+# CHECK-NEXT:  4      36    28.00   *                   vsqrtps	(%rax), %zmm19
+# CHECK-NEXT:  4      36    28.00   *                   vsqrtps	(%rax){1to16}, %zmm19
+# CHECK-NEXT:  3      29    28.00                       vsqrtps	%zmm16, %zmm19 {%k1}
+# CHECK-NEXT:  4      36    28.00   *                   vsqrtps	(%rax), %zmm19 {%k1}
+# CHECK-NEXT:  4      36    28.00   *                   vsqrtps	(%rax){1to16}, %zmm19 {%k1}
+# CHECK-NEXT:  3      29    28.00                       vsqrtps	%zmm16, %zmm19 {%k1} {z}
+# CHECK-NEXT:  4      36    28.00   *                   vsqrtps	(%rax), %zmm19 {%k1} {z}
+# CHECK-NEXT:  4      36    28.00   *                   vsqrtps	(%rax){1to16}, %zmm19 {%k1} {z}
+# CHECK-NEXT:  1      21    21.00                       vsqrtsd	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  2      27    21.00   *                   vsqrtsd	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  1      21    21.00                       vsqrtsd	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      27    21.00   *                   vsqrtsd	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  1      21    21.00                       vsqrtsd	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      27    21.00   *                   vsqrtsd	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  1      14    14.00                       vsqrtss	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  2      20    14.00   *                   vsqrtss	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  1      14    14.00                       vsqrtss	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      20    14.00   *                   vsqrtss	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  1      14    14.00                       vsqrtss	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      20    14.00   *                   vsqrtss	(%rax), %xmm17, %xmm19 {%k1} {z}
 # CHECK-NEXT:  1      1     0.50                        vpsubd	%zmm16, %zmm17, %zmm19
 # CHECK-NEXT:  2      8     0.50    *                   vpsubd	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  2      8     0.50    *                   vpsubd	(%rax){1to16}, %zmm17, %zmm19
 # CHECK-NEXT:  1      1     0.50                        vpsubd	%zmm16, %zmm17, %zmm19 {%k1}
 # CHECK-NEXT:  2      8     0.50    *                   vpsubd	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     0.50    *                   vpsubd	(%rax){1to16}, %zmm17, %zmm19 {%k1}
 # CHECK-NEXT:  1      1     0.50                        vpsubd	%zmm16, %zmm17, %zmm19 {%k1} {z}
 # CHECK-NEXT:  2      8     0.50    *                   vpsubd	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     0.50    *                   vpsubd	(%rax){1to16}, %zmm17, %zmm19 {%k1} {z}
 # CHECK-NEXT:  1      1     0.50                        vpsubq	%zmm16, %zmm17, %zmm19
 # CHECK-NEXT:  2      8     0.50    *                   vpsubq	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  2      8     0.50    *                   vpsubq	(%rax){1to8}, %zmm17, %zmm19
 # CHECK-NEXT:  1      1     0.50                        vpsubq	%zmm16, %zmm17, %zmm19 {%k1}
 # CHECK-NEXT:  2      8     0.50    *                   vpsubq	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     0.50    *                   vpsubq	(%rax){1to8}, %zmm17, %zmm19 {%k1}
 # CHECK-NEXT:  1      1     0.50                        vpsubq	%zmm16, %zmm17, %zmm19 {%k1} {z}
 # CHECK-NEXT:  2      8     0.50    *                   vpsubq	(%rax), %zmm17, %zmm19 {%k1} {z}
-# CHECK-NEXT:  1      1     0.50                        vpsubw	%zmm16, %zmm17, %zmm19
-# CHECK-NEXT:  2      8     0.50    *                   vpsubw	(%rax), %zmm17, %zmm19
-# CHECK-NEXT:  1      1     0.50                        vpsubw	%zmm16, %zmm17, %zmm19 {%k1}
-# CHECK-NEXT:  2      8     0.50    *                   vpsubw	(%rax), %zmm17, %zmm19 {%k1}
-# CHECK-NEXT:  1      1     0.50                        vpsubw	%zmm16, %zmm17, %zmm19 {%k1} {z}
-# CHECK-NEXT:  2      8     0.50    *                   vpsubw	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     0.50    *                   vpsubq	(%rax){1to8}, %zmm17, %zmm19 {%k1} {z}
 # CHECK-NEXT:  1      3     1.00                        vsubpd	%zmm16, %zmm17, %zmm19
 # CHECK-NEXT:  2      10    1.00    *                   vsubpd	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  2      10    1.00    *                   vsubpd	(%rax){1to8}, %zmm17, %zmm19
 # CHECK-NEXT:  1      3     1.00                        vsubpd	%zmm16, %zmm17, %zmm19 {%k1}
 # CHECK-NEXT:  2      10    1.00    *                   vsubpd	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      10    1.00    *                   vsubpd	(%rax){1to8}, %zmm17, %zmm19 {%k1}
 # CHECK-NEXT:  1      3     1.00                        vsubpd	%zmm16, %zmm17, %zmm19 {%k1} {z}
 # CHECK-NEXT:  2      10    1.00    *                   vsubpd	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      10    1.00    *                   vsubpd	(%rax){1to8}, %zmm17, %zmm19 {%k1} {z}
 # CHECK-NEXT:  1      3     1.00                        vsubps	%zmm16, %zmm17, %zmm19
 # CHECK-NEXT:  2      10    1.00    *                   vsubps	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  2      10    1.00    *                   vsubps	(%rax){1to16}, %zmm17, %zmm19
 # CHECK-NEXT:  1      3     1.00                        vsubps	%zmm16, %zmm17, %zmm19 {%k1}
 # CHECK-NEXT:  2      10    1.00    *                   vsubps	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      10    1.00    *                   vsubps	(%rax){1to16}, %zmm17, %zmm19 {%k1}
 # CHECK-NEXT:  1      3     1.00                        vsubps	%zmm16, %zmm17, %zmm19 {%k1} {z}
 # CHECK-NEXT:  2      10    1.00    *                   vsubps	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      10    1.00    *                   vsubps	(%rax){1to16}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vunpckhpd	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  2      8     1.00    *                   vunpckhpd	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  2      8     1.00    *                   vunpckhpd	(%rax){1to8}, %zmm17, %zmm19
+# CHECK-NEXT:  1      1     1.00                        vunpckhpd	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vunpckhpd	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vunpckhpd	(%rax){1to8}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vunpckhpd	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vunpckhpd	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vunpckhpd	(%rax){1to8}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vunpckhps	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  2      8     1.00    *                   vunpckhps	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  2      8     1.00    *                   vunpckhps	(%rax){1to16}, %zmm17, %zmm19
+# CHECK-NEXT:  1      1     1.00                        vunpckhps	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vunpckhps	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vunpckhps	(%rax){1to16}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vunpckhps	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vunpckhps	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vunpckhps	(%rax){1to16}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vunpcklpd	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  2      8     1.00    *                   vunpcklpd	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  2      8     1.00    *                   vunpcklpd	(%rax){1to8}, %zmm17, %zmm19
+# CHECK-NEXT:  1      1     1.00                        vunpcklpd	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vunpcklpd	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vunpcklpd	(%rax){1to8}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vunpcklpd	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vunpcklpd	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vunpcklpd	(%rax){1to8}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vunpcklps	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  2      8     1.00    *                   vunpcklps	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  2      8     1.00    *                   vunpcklps	(%rax){1to16}, %zmm17, %zmm19
+# CHECK-NEXT:  1      1     1.00                        vunpcklps	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vunpcklps	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vunpcklps	(%rax){1to16}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vunpcklps	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vunpcklps	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vunpcklps	(%rax){1to16}, %zmm17, %zmm19 {%k1} {z}
 
 # CHECK:      Resources:
 # CHECK-NEXT: [0]   - SBDivider
@@ -257,115 +999,469 @@ vsubps            (%rax), %zmm17, %zmm19 {z}{k1}
 
 # CHECK:      Resource pressure per iteration:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6.0]  [6.1]
-# CHECK-NEXT:  -     432.00 42.00  54.00   -     36.00  27.00  27.00
+# CHECK-NEXT:  -     1506.00 129.00 144.00  -    270.00 153.00 153.00
 
 # CHECK:      Resource pressure by instruction:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6.0]  [6.1]  Instructions:
 # CHECK-NEXT:  -      -      -     1.00    -      -      -      -     vaddpd	%zmm16, %zmm17, %zmm19
 # CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vaddpd	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vaddpd	(%rax){1to8}, %zmm17, %zmm19
 # CHECK-NEXT:  -      -      -     1.00    -      -      -      -     vaddpd	%zmm16, %zmm17, %zmm19 {%k1}
 # CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vaddpd	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vaddpd	(%rax){1to8}, %zmm17, %zmm19 {%k1}
 # CHECK-NEXT:  -      -      -     1.00    -      -      -      -     vaddpd	%zmm16, %zmm17, %zmm19 {%k1} {z}
 # CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vaddpd	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vaddpd	(%rax){1to8}, %zmm17, %zmm19 {%k1} {z}
 # CHECK-NEXT:  -      -      -     1.00    -      -      -      -     vaddps	%zmm16, %zmm17, %zmm19
 # CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vaddps	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vaddps	(%rax){1to16}, %zmm17, %zmm19
 # CHECK-NEXT:  -      -      -     1.00    -      -      -      -     vaddps	%zmm16, %zmm17, %zmm19 {%k1}
 # CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vaddps	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vaddps	(%rax){1to16}, %zmm17, %zmm19 {%k1}
 # CHECK-NEXT:  -      -      -     1.00    -      -      -      -     vaddps	%zmm16, %zmm17, %zmm19 {%k1} {z}
 # CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vaddps	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vaddps	(%rax){1to16}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     1.00    -     1.00    -      -     vcvtdq2pd	%ymm16, %zmm19
+# CHECK-NEXT:  -      -      -     1.00    -     1.00   0.50   0.50   vcvtdq2pd	(%rax), %zmm19
+# CHECK-NEXT:  -      -      -     1.00    -     1.00   0.50   0.50   vcvtdq2pd	(%rax){1to8}, %zmm19
+# CHECK-NEXT:  -      -      -     1.00    -     1.00    -      -     vcvtdq2pd	%ymm16, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -     1.00    -     1.00   0.50   0.50   vcvtdq2pd	(%rax), %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -     1.00    -     1.00   0.50   0.50   vcvtdq2pd	(%rax){1to8}, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -     1.00    -     1.00    -      -     vcvtdq2pd	%ymm16, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     1.00    -     1.00   0.50   0.50   vcvtdq2pd	(%rax), %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     1.00    -     1.00   0.50   0.50   vcvtdq2pd	(%rax){1to8}, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -     vcvtdq2ps	%zmm16, %zmm19
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vcvtdq2ps	(%rax), %zmm19
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vcvtdq2ps	(%rax){1to16}, %zmm19
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -     vcvtdq2ps	%zmm16, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vcvtdq2ps	(%rax), %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vcvtdq2ps	(%rax){1to16}, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -     vcvtdq2ps	%zmm16, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vcvtdq2ps	(%rax), %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vcvtdq2ps	(%rax){1to16}, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -     vcvtps2dq	%zmm16, %zmm19
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vcvtps2dq	(%rax), %zmm19
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vcvtps2dq	(%rax){1to16}, %zmm19
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -     vcvtps2dq	%zmm16, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vcvtps2dq	(%rax), %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vcvtps2dq	(%rax){1to16}, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -     vcvtps2dq	%zmm16, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vcvtps2dq	(%rax), %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vcvtps2dq	(%rax){1to16}, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -     vcvttps2dq	%zmm16, %zmm19
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vcvttps2dq	(%rax), %zmm19
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vcvttps2dq	(%rax){1to16}, %zmm19
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -     vcvttps2dq	%zmm16, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vcvttps2dq	(%rax), %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vcvttps2dq	(%rax){1to16}, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -     vcvttps2dq	%zmm16, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vcvttps2dq	(%rax), %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vcvttps2dq	(%rax){1to16}, %zmm19 {%k1} {z}
 # CHECK-NEXT:  -     44.00  2.50    -      -     0.50    -      -     vdivpd	%zmm16, %zmm17, %zmm19
 # CHECK-NEXT:  -     44.00  2.50    -      -     0.50   0.50   0.50   vdivpd	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  -     44.00  2.50    -      -     0.50   0.50   0.50   vdivpd	(%rax){1to8}, %zmm17, %zmm19
 # CHECK-NEXT:  -     44.00  2.50    -      -     0.50    -      -     vdivpd	%zmm16, %zmm17, %zmm19 {%k1}
 # CHECK-NEXT:  -     44.00  2.50    -      -     0.50   0.50   0.50   vdivpd	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -     44.00  2.50    -      -     0.50   0.50   0.50   vdivpd	(%rax){1to8}, %zmm17, %zmm19 {%k1}
 # CHECK-NEXT:  -     44.00  2.50    -      -     0.50    -      -     vdivpd	%zmm16, %zmm17, %zmm19 {%k1} {z}
 # CHECK-NEXT:  -     44.00  2.50    -      -     0.50   0.50   0.50   vdivpd	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -     44.00  2.50    -      -     0.50   0.50   0.50   vdivpd	(%rax){1to8}, %zmm17, %zmm19 {%k1} {z}
 # CHECK-NEXT:  -     28.00  2.50    -      -     0.50    -      -     vdivps	%zmm16, %zmm17, %zmm19
 # CHECK-NEXT:  -     28.00  2.50    -      -     0.50   0.50   0.50   vdivps	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  -     28.00  2.50    -      -     0.50   0.50   0.50   vdivps	(%rax){1to16}, %zmm17, %zmm19
 # CHECK-NEXT:  -     28.00  2.50    -      -     0.50    -      -     vdivps	%zmm16, %zmm17, %zmm19 {%k1}
 # CHECK-NEXT:  -     28.00  2.50    -      -     0.50   0.50   0.50   vdivps	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -     28.00  2.50    -      -     0.50   0.50   0.50   vdivps	(%rax){1to16}, %zmm17, %zmm19 {%k1}
 # CHECK-NEXT:  -     28.00  2.50    -      -     0.50    -      -     vdivps	%zmm16, %zmm17, %zmm19 {%k1} {z}
 # CHECK-NEXT:  -     28.00  2.50    -      -     0.50   0.50   0.50   vdivps	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -     28.00  2.50    -      -     0.50   0.50   0.50   vdivps	(%rax){1to16}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -     vmaxpd	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vmaxpd	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vmaxpd	(%rax){1to8}, %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -     vmaxpd	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vmaxpd	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vmaxpd	(%rax){1to8}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -     vmaxpd	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vmaxpd	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vmaxpd	(%rax){1to8}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -     vmaxps	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vmaxps	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vmaxps	(%rax){1to16}, %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -     vmaxps	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vmaxps	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vmaxps	(%rax){1to16}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -     vmaxps	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vmaxps	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vmaxps	(%rax){1to16}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -     vminpd	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vminpd	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vminpd	(%rax){1to8}, %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -     vminpd	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vminpd	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vminpd	(%rax){1to8}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -     vminpd	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vminpd	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vminpd	(%rax){1to8}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -     vminps	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vminps	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vminps	(%rax){1to16}, %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -     vminps	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vminps	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vminps	(%rax){1to16}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -     vminps	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vminps	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vminps	(%rax){1to16}, %zmm17, %zmm19 {%k1} {z}
 # CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vmulpd	%zmm16, %zmm17, %zmm19
 # CHECK-NEXT:  -      -     1.00    -      -      -     0.50   0.50   vmulpd	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  -      -     1.00    -      -      -     0.50   0.50   vmulpd	(%rax){1to8}, %zmm17, %zmm19
 # CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vmulpd	%zmm16, %zmm17, %zmm19 {%k1}
 # CHECK-NEXT:  -      -     1.00    -      -      -     0.50   0.50   vmulpd	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -     1.00    -      -      -     0.50   0.50   vmulpd	(%rax){1to8}, %zmm17, %zmm19 {%k1}
 # CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vmulpd	%zmm16, %zmm17, %zmm19 {%k1} {z}
 # CHECK-NEXT:  -      -     1.00    -      -      -     0.50   0.50   vmulpd	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     1.00    -      -      -     0.50   0.50   vmulpd	(%rax){1to8}, %zmm17, %zmm19 {%k1} {z}
 # CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vmulps	%zmm16, %zmm17, %zmm19
 # CHECK-NEXT:  -      -     1.00    -      -      -     0.50   0.50   vmulps	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  -      -     1.00    -      -      -     0.50   0.50   vmulps	(%rax){1to16}, %zmm17, %zmm19
 # CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vmulps	%zmm16, %zmm17, %zmm19 {%k1}
 # CHECK-NEXT:  -      -     1.00    -      -      -     0.50   0.50   vmulps	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -     1.00    -      -      -     0.50   0.50   vmulps	(%rax){1to16}, %zmm17, %zmm19 {%k1}
 # CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vmulps	%zmm16, %zmm17, %zmm19 {%k1} {z}
 # CHECK-NEXT:  -      -     1.00    -      -      -     0.50   0.50   vmulps	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     1.00    -      -      -     0.50   0.50   vmulps	(%rax){1to16}, %zmm17, %zmm19 {%k1} {z}
 # CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpabsd	%zmm16, %zmm19
 # CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpabsd	(%rax), %zmm19
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpabsd	(%rax){1to16}, %zmm19
 # CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpabsd	%zmm16, %zmm19 {%k1}
 # CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpabsd	(%rax), %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpabsd	(%rax){1to16}, %zmm19 {%k1}
 # CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpabsd	%zmm16, %zmm19 {%k1} {z}
 # CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpabsd	(%rax), %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpabsd	(%rax){1to16}, %zmm19 {%k1} {z}
 # CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpabsq	%zmm16, %zmm19
 # CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpabsq	(%rax), %zmm19
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpabsq	(%rax){1to8}, %zmm19
 # CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpabsq	%zmm16, %zmm19 {%k1}
 # CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpabsq	(%rax), %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpabsq	(%rax){1to8}, %zmm19 {%k1}
 # CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpabsq	%zmm16, %zmm19 {%k1} {z}
 # CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpabsq	(%rax), %zmm19 {%k1} {z}
-# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpaddb	%zmm16, %zmm17, %zmm19
-# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpaddb	(%rax), %zmm17, %zmm19
-# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpaddb	%zmm16, %zmm17, %zmm19 {%k1}
-# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpaddb	(%rax), %zmm17, %zmm19 {%k1}
-# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpaddb	%zmm16, %zmm17, %zmm19 {%k1} {z}
-# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpaddb	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpabsq	(%rax){1to8}, %zmm19 {%k1} {z}
 # CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpaddd	%zmm16, %zmm17, %zmm19
 # CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpaddd	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpaddd	(%rax){1to16}, %zmm17, %zmm19
 # CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpaddd	%zmm16, %zmm17, %zmm19 {%k1}
 # CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpaddd	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpaddd	(%rax){1to16}, %zmm17, %zmm19 {%k1}
 # CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpaddd	%zmm16, %zmm17, %zmm19 {%k1} {z}
 # CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpaddd	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpaddd	(%rax){1to16}, %zmm17, %zmm19 {%k1} {z}
 # CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpaddq	%zmm16, %zmm17, %zmm19
 # CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpaddq	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpaddq	(%rax){1to8}, %zmm17, %zmm19
 # CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpaddq	%zmm16, %zmm17, %zmm19 {%k1}
 # CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpaddq	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpaddq	(%rax){1to8}, %zmm17, %zmm19 {%k1}
 # CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpaddq	%zmm16, %zmm17, %zmm19 {%k1} {z}
 # CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpaddq	(%rax), %zmm17, %zmm19 {%k1} {z}
-# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpaddw	%zmm16, %zmm17, %zmm19
-# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpaddw	(%rax), %zmm17, %zmm19
-# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpaddw	%zmm16, %zmm17, %zmm19 {%k1}
-# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpaddw	(%rax), %zmm17, %zmm19 {%k1}
-# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpaddw	%zmm16, %zmm17, %zmm19 {%k1} {z}
-# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpaddw	(%rax), %zmm17, %zmm19 {%k1} {z}
-# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpsubb	%zmm16, %zmm17, %zmm19
-# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpsubb	(%rax), %zmm17, %zmm19
-# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpsubb	%zmm16, %zmm17, %zmm19 {%k1}
-# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpsubb	(%rax), %zmm17, %zmm19 {%k1}
-# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpsubb	%zmm16, %zmm17, %zmm19 {%k1} {z}
-# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpsubb	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpaddq	(%rax){1to8}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vpmulld	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  -      -     1.00    -      -      -     0.50   0.50   vpmulld	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  -      -     1.00    -      -      -     0.50   0.50   vpmulld	(%rax){1to16}, %zmm17, %zmm19
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vpmulld	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -     1.00    -      -      -     0.50   0.50   vpmulld	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -     1.00    -      -      -     0.50   0.50   vpmulld	(%rax){1to16}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vpmulld	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     1.00    -      -      -     0.50   0.50   vpmulld	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     1.00    -      -      -     0.50   0.50   vpmulld	(%rax){1to16}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpermd	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermd	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermd	(%rax){1to16}, %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpermd	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermd	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermd	(%rax){1to16}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpermd	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermd	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermd	(%rax){1to16}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpermilpd	$0, %zmm16, %zmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermilpd	$0, (%rax), %zmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermilpd	$0, (%rax){1to8}, %zmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpermilpd	$0, %zmm16, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermilpd	$0, (%rax), %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermilpd	$0, (%rax){1to8}, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpermilpd	$0, %zmm16, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermilpd	$0, (%rax), %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermilpd	$0, (%rax){1to8}, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpermilpd	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermilpd	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermilpd	(%rax){1to8}, %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpermilpd	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermilpd	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermilpd	(%rax){1to8}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpermilpd	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermilpd	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermilpd	(%rax){1to8}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpermilps	$0, %zmm16, %zmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermilps	$0, (%rax), %zmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermilps	$0, (%rax){1to16}, %zmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpermilps	$0, %zmm16, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermilps	$0, (%rax), %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermilps	$0, (%rax){1to16}, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpermilps	$0, %zmm16, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermilps	$0, (%rax), %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermilps	$0, (%rax){1to16}, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpermilps	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermilps	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermilps	(%rax){1to16}, %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpermilps	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermilps	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermilps	(%rax){1to16}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpermilps	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermilps	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermilps	(%rax){1to16}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpermpd	$0, %zmm16, %zmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermpd	$0, (%rax), %zmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermpd	$0, (%rax){1to8}, %zmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpermpd	$0, %zmm16, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermpd	$0, (%rax), %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermpd	$0, (%rax){1to8}, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpermpd	$0, %zmm16, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermpd	$0, (%rax), %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermpd	$0, (%rax){1to8}, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpermpd	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermpd	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermpd	(%rax){1to8}, %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpermpd	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermpd	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermpd	(%rax){1to8}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpermpd	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermpd	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermpd	(%rax){1to8}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpermps	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermps	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermps	(%rax){1to16}, %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpermps	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermps	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermps	(%rax){1to16}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpermps	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermps	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermps	(%rax){1to16}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpermq	$0, %zmm16, %zmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermq	$0, (%rax), %zmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermq	$0, (%rax){1to8}, %zmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpermq	$0, %zmm16, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermq	$0, (%rax), %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermq	$0, (%rax){1to8}, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpermq	$0, %zmm16, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermq	$0, (%rax), %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermq	$0, (%rax){1to8}, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpermq	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermq	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermq	(%rax){1to8}, %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpermq	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermq	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermq	(%rax){1to8}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpermq	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermq	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermq	(%rax){1to8}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpshufd	$0, %zmm16, %zmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpshufd	$0, (%rax), %zmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpshufd	$0, (%rax){1to16}, %zmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpshufd	$0, %zmm16, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpshufd	$0, (%rax), %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpshufd	$0, (%rax){1to16}, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpshufd	$0, %zmm16, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpshufd	$0, (%rax), %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpshufd	$0, (%rax){1to16}, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpsubd	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpsubd	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpsubd	(%rax){1to16}, %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpsubd	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpsubd	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpsubd	(%rax){1to16}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpsubd	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpsubd	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpsubd	(%rax){1to16}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpsubq	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpsubq	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpsubq	(%rax){1to8}, %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpsubq	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpsubq	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpsubq	(%rax){1to8}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpsubq	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpsubq	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpsubq	(%rax){1to8}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpunpckhdq	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpunpckhdq	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpunpckhdq	(%rax){1to16}, %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpunpckhdq	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpunpckhdq	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpunpckhdq	(%rax){1to16}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpunpckhdq	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpunpckhdq	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpunpckhdq	(%rax){1to16}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpunpckhqdq	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpunpckhqdq	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpunpckhqdq	(%rax){1to8}, %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpunpckhqdq	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpunpckhqdq	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpunpckhqdq	(%rax){1to8}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpunpckhqdq	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpunpckhqdq	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpunpckhqdq	(%rax){1to8}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpunpckldq	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpunpckldq	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpunpckldq	(%rax){1to16}, %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpunpckldq	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpunpckldq	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpunpckldq	(%rax){1to16}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpunpckldq	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpunpckldq	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpunpckldq	(%rax){1to16}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpunpcklqdq	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpunpcklqdq	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpunpcklqdq	(%rax){1to8}, %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpunpcklqdq	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpunpcklqdq	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpunpcklqdq	(%rax){1to8}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpunpcklqdq	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpunpcklqdq	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpunpcklqdq	(%rax){1to8}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vshuff32x4	$0, %zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vshuff32x4	$0, (%rax), %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vshuff32x4	$0, (%rax){1to16}, %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vshuff32x4	$0, %zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vshuff32x4	$0, (%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vshuff32x4	$0, (%rax){1to16}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vshuff32x4	$0, %zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vshuff32x4	$0, (%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vshuff32x4	$0, (%rax){1to16}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vshuff64x2	$0, %zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vshuff64x2	$0, (%rax), %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vshuff64x2	$0, (%rax){1to8}, %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vshuff64x2	$0, %zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vshuff64x2	$0, (%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vshuff64x2	$0, (%rax){1to8}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vshuff64x2	$0, %zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vshuff64x2	$0, (%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vshuff64x2	$0, (%rax){1to8}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vshufi32x4	$0, %zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vshufi32x4	$0, (%rax), %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vshufi32x4	$0, (%rax){1to16}, %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vshufi32x4	$0, %zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vshufi32x4	$0, (%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vshufi32x4	$0, (%rax){1to16}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vshufi32x4	$0, %zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vshufi32x4	$0, (%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vshufi32x4	$0, (%rax){1to16}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vshufi64x2	$0, %zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vshufi64x2	$0, (%rax), %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vshufi64x2	$0, (%rax){1to8}, %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vshufi64x2	$0, %zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vshufi64x2	$0, (%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vshufi64x2	$0, (%rax){1to8}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vshufi64x2	$0, %zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vshufi64x2	$0, (%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vshufi64x2	$0, (%rax){1to8}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -     44.00  2.50    -      -     0.50    -      -     vsqrtpd	%zmm16, %zmm19
+# CHECK-NEXT:  -     44.00  2.50    -      -     0.50   0.50   0.50   vsqrtpd	(%rax), %zmm19
+# CHECK-NEXT:  -     44.00  2.50    -      -     0.50   0.50   0.50   vsqrtpd	(%rax){1to8}, %zmm19
+# CHECK-NEXT:  -     44.00  2.50    -      -     0.50    -      -     vsqrtpd	%zmm16, %zmm19 {%k1}
+# CHECK-NEXT:  -     44.00  2.50    -      -     0.50   0.50   0.50   vsqrtpd	(%rax), %zmm19 {%k1}
+# CHECK-NEXT:  -     44.00  2.50    -      -     0.50   0.50   0.50   vsqrtpd	(%rax){1to8}, %zmm19 {%k1}
+# CHECK-NEXT:  -     44.00  2.50    -      -     0.50    -      -     vsqrtpd	%zmm16, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -     44.00  2.50    -      -     0.50   0.50   0.50   vsqrtpd	(%rax), %zmm19 {%k1} {z}
+# CHECK-NEXT:  -     44.00  2.50    -      -     0.50   0.50   0.50   vsqrtpd	(%rax){1to8}, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -     28.00  2.50    -      -     0.50    -      -     vsqrtps	%zmm16, %zmm19
+# CHECK-NEXT:  -     28.00  2.50    -      -     0.50   0.50   0.50   vsqrtps	(%rax), %zmm19
+# CHECK-NEXT:  -     28.00  2.50    -      -     0.50   0.50   0.50   vsqrtps	(%rax){1to16}, %zmm19
+# CHECK-NEXT:  -     28.00  2.50    -      -     0.50    -      -     vsqrtps	%zmm16, %zmm19 {%k1}
+# CHECK-NEXT:  -     28.00  2.50    -      -     0.50   0.50   0.50   vsqrtps	(%rax), %zmm19 {%k1}
+# CHECK-NEXT:  -     28.00  2.50    -      -     0.50   0.50   0.50   vsqrtps	(%rax){1to16}, %zmm19 {%k1}
+# CHECK-NEXT:  -     28.00  2.50    -      -     0.50    -      -     vsqrtps	%zmm16, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -     28.00  2.50    -      -     0.50   0.50   0.50   vsqrtps	(%rax), %zmm19 {%k1} {z}
+# CHECK-NEXT:  -     28.00  2.50    -      -     0.50   0.50   0.50   vsqrtps	(%rax){1to16}, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -     21.00  1.00    -      -      -      -      -     vsqrtsd	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  -     21.00  1.00    -      -      -     0.50   0.50   vsqrtsd	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  -     21.00  1.00    -      -      -      -      -     vsqrtsd	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -     21.00  1.00    -      -      -     0.50   0.50   vsqrtsd	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -     21.00  1.00    -      -      -      -      -     vsqrtsd	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -     21.00  1.00    -      -      -     0.50   0.50   vsqrtsd	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -     14.00  1.00    -      -      -      -      -     vsqrtss	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  -     14.00  1.00    -      -      -     0.50   0.50   vsqrtss	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  -     14.00  1.00    -      -      -      -      -     vsqrtss	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -     14.00  1.00    -      -      -     0.50   0.50   vsqrtss	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -     14.00  1.00    -      -      -      -      -     vsqrtss	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -     14.00  1.00    -      -      -     0.50   0.50   vsqrtss	(%rax), %xmm17, %xmm19 {%k1} {z}
 # CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpsubd	%zmm16, %zmm17, %zmm19
 # CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpsubd	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpsubd	(%rax){1to16}, %zmm17, %zmm19
 # CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpsubd	%zmm16, %zmm17, %zmm19 {%k1}
 # CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpsubd	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpsubd	(%rax){1to16}, %zmm17, %zmm19 {%k1}
 # CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpsubd	%zmm16, %zmm17, %zmm19 {%k1} {z}
 # CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpsubd	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpsubd	(%rax){1to16}, %zmm17, %zmm19 {%k1} {z}
 # CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpsubq	%zmm16, %zmm17, %zmm19
 # CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpsubq	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpsubq	(%rax){1to8}, %zmm17, %zmm19
 # CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpsubq	%zmm16, %zmm17, %zmm19 {%k1}
 # CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpsubq	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpsubq	(%rax){1to8}, %zmm17, %zmm19 {%k1}
 # CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpsubq	%zmm16, %zmm17, %zmm19 {%k1} {z}
 # CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpsubq	(%rax), %zmm17, %zmm19 {%k1} {z}
-# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpsubw	%zmm16, %zmm17, %zmm19
-# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpsubw	(%rax), %zmm17, %zmm19
-# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpsubw	%zmm16, %zmm17, %zmm19 {%k1}
-# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpsubw	(%rax), %zmm17, %zmm19 {%k1}
-# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpsubw	%zmm16, %zmm17, %zmm19 {%k1} {z}
-# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpsubw	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpsubq	(%rax){1to8}, %zmm17, %zmm19 {%k1} {z}
 # CHECK-NEXT:  -      -      -     1.00    -      -      -      -     vsubpd	%zmm16, %zmm17, %zmm19
 # CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vsubpd	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vsubpd	(%rax){1to8}, %zmm17, %zmm19
 # CHECK-NEXT:  -      -      -     1.00    -      -      -      -     vsubpd	%zmm16, %zmm17, %zmm19 {%k1}
 # CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vsubpd	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vsubpd	(%rax){1to8}, %zmm17, %zmm19 {%k1}
 # CHECK-NEXT:  -      -      -     1.00    -      -      -      -     vsubpd	%zmm16, %zmm17, %zmm19 {%k1} {z}
 # CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vsubpd	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vsubpd	(%rax){1to8}, %zmm17, %zmm19 {%k1} {z}
 # CHECK-NEXT:  -      -      -     1.00    -      -      -      -     vsubps	%zmm16, %zmm17, %zmm19
 # CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vsubps	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vsubps	(%rax){1to16}, %zmm17, %zmm19
 # CHECK-NEXT:  -      -      -     1.00    -      -      -      -     vsubps	%zmm16, %zmm17, %zmm19 {%k1}
 # CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vsubps	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vsubps	(%rax){1to16}, %zmm17, %zmm19 {%k1}
 # CHECK-NEXT:  -      -      -     1.00    -      -      -      -     vsubps	%zmm16, %zmm17, %zmm19 {%k1} {z}
 # CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vsubps	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vsubps	(%rax){1to16}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vunpckhpd	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vunpckhpd	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vunpckhpd	(%rax){1to8}, %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vunpckhpd	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vunpckhpd	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vunpckhpd	(%rax){1to8}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vunpckhpd	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vunpckhpd	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vunpckhpd	(%rax){1to8}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vunpckhps	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vunpckhps	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vunpckhps	(%rax){1to16}, %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vunpckhps	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vunpckhps	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vunpckhps	(%rax){1to16}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vunpckhps	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vunpckhps	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vunpckhps	(%rax){1to16}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vunpcklpd	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vunpcklpd	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vunpcklpd	(%rax){1to8}, %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vunpcklpd	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vunpcklpd	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vunpcklpd	(%rax){1to8}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vunpcklpd	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vunpcklpd	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vunpcklpd	(%rax){1to8}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vunpcklps	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vunpcklps	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vunpcklps	(%rax){1to16}, %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vunpcklps	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vunpcklps	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vunpcklps	(%rax){1to16}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vunpcklps	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vunpcklps	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vunpcklps	(%rax){1to16}, %zmm17, %zmm19 {%k1} {z}
diff --git a/test/tools/llvm-mca/X86/Generic/resources-avx512bw.s b/test/tools/llvm-mca/X86/Generic/resources-avx512bw.s
new file mode 100644
index 0000000000000000000000000000000000000000..145192932f5bbba2d96710bdb88d3c36bb3aed81
--- /dev/null
+++ b/test/tools/llvm-mca/X86/Generic/resources-avx512bw.s
@@ -0,0 +1,295 @@
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -instruction-tables < %s | FileCheck %s
+
+vpabsb            %zmm16, %zmm19
+vpabsb            (%rax), %zmm19
+vpabsb            %zmm16, %zmm19 {k1}
+vpabsb            (%rax), %zmm19 {k1}
+vpabsb            %zmm16, %zmm19 {z}{k1}
+vpabsb            (%rax), %zmm19 {z}{k1}
+
+vpabsw            %zmm16, %zmm19
+vpabsw            (%rax), %zmm19
+vpabsw            %zmm16, %zmm19 {k1}
+vpabsw            (%rax), %zmm19 {k1}
+vpabsw            %zmm16, %zmm19 {z}{k1}
+vpabsw            (%rax), %zmm19 {z}{k1}
+
+vpaddb            %zmm16, %zmm17, %zmm19
+vpaddb            (%rax), %zmm17, %zmm19
+vpaddb            %zmm16, %zmm17, %zmm19 {k1}
+vpaddb            (%rax), %zmm17, %zmm19 {k1}
+vpaddb            %zmm16, %zmm17, %zmm19 {z}{k1}
+vpaddb            (%rax), %zmm17, %zmm19 {z}{k1}
+
+vpaddw            %zmm16, %zmm17, %zmm19
+vpaddw            (%rax), %zmm17, %zmm19
+vpaddw            %zmm16, %zmm17, %zmm19 {k1}
+vpaddw            (%rax), %zmm17, %zmm19 {k1}
+vpaddw            %zmm16, %zmm17, %zmm19 {z}{k1}
+vpaddw            (%rax), %zmm17, %zmm19 {z}{k1}
+
+vpermw            %zmm16, %zmm17, %zmm19
+vpermw            (%rax), %zmm17, %zmm19
+vpermw            %zmm16, %zmm17, %zmm19 {k1}
+vpermw            (%rax), %zmm17, %zmm19 {k1}
+vpermw            %zmm16, %zmm17, %zmm19 {z}{k1}
+vpermw            (%rax), %zmm17, %zmm19 {z}{k1}
+
+vpshufb           %zmm16, %zmm17, %zmm19
+vpshufb           (%rax), %zmm17, %zmm19
+vpshufb           %zmm16, %zmm17, %zmm19 {k1}
+vpshufb           (%rax), %zmm17, %zmm19 {k1}
+vpshufb           %zmm16, %zmm17, %zmm19 {z}{k1}
+vpshufb           (%rax), %zmm17, %zmm19 {z}{k1}
+
+vpshufhw          $0, %zmm16, %zmm19
+vpshufhw          $0, (%rax), %zmm19
+vpshufhw          $0, %zmm16, %zmm19 {k1}
+vpshufhw          $0, (%rax), %zmm19 {k1}
+vpshufhw          $0, %zmm16, %zmm19 {z}{k1}
+vpshufhw          $0, (%rax), %zmm19 {z}{k1}
+
+vpshuflw          $0, %zmm16, %zmm19
+vpshuflw          $0, (%rax), %zmm19
+vpshuflw          $0, %zmm16, %zmm19 {k1}
+vpshuflw          $0, (%rax), %zmm19 {k1}
+vpshuflw          $0, %zmm16, %zmm19 {z}{k1}
+vpshuflw          $0, (%rax), %zmm19 {z}{k1}
+
+vpsubb            %zmm16, %zmm17, %zmm19
+vpsubb            (%rax), %zmm17, %zmm19
+vpsubb            %zmm16, %zmm17, %zmm19 {k1}
+vpsubb            (%rax), %zmm17, %zmm19 {k1}
+vpsubb            %zmm16, %zmm17, %zmm19 {z}{k1}
+vpsubb            (%rax), %zmm17, %zmm19 {z}{k1}
+
+vpsubw            %zmm16, %zmm17, %zmm19
+vpsubw            (%rax), %zmm17, %zmm19
+vpsubw            %zmm16, %zmm17, %zmm19 {k1}
+vpsubw            (%rax), %zmm17, %zmm19 {k1}
+vpsubw            %zmm16, %zmm17, %zmm19 {z}{k1}
+vpsubw            (%rax), %zmm17, %zmm19 {z}{k1}
+
+vpunpckhbw        %zmm16, %zmm17, %zmm19
+vpunpckhbw        (%rax), %zmm17, %zmm19
+vpunpckhbw        %zmm16, %zmm17, %zmm19 {k1}
+vpunpckhbw        (%rax), %zmm17, %zmm19 {k1}
+vpunpckhbw        %zmm16, %zmm17, %zmm19 {z}{k1}
+vpunpckhbw        (%rax), %zmm17, %zmm19 {z}{k1}
+
+vpunpckhwd        %zmm16, %zmm17, %zmm19
+vpunpckhwd        (%rax), %zmm17, %zmm19
+vpunpckhwd        %zmm16, %zmm17, %zmm19 {k1}
+vpunpckhwd        (%rax), %zmm17, %zmm19 {k1}
+vpunpckhwd        %zmm16, %zmm17, %zmm19 {z}{k1}
+vpunpckhwd        (%rax), %zmm17, %zmm19 {z}{k1}
+
+vpunpcklbw        %zmm16, %zmm17, %zmm19
+vpunpcklbw        (%rax), %zmm17, %zmm19
+vpunpcklbw        %zmm16, %zmm17, %zmm19 {k1}
+vpunpcklbw        (%rax), %zmm17, %zmm19 {k1}
+vpunpcklbw        %zmm16, %zmm17, %zmm19 {z}{k1}
+vpunpcklbw        (%rax), %zmm17, %zmm19 {z}{k1}
+
+vpunpcklwd        %zmm16, %zmm17, %zmm19
+vpunpcklwd        (%rax), %zmm17, %zmm19
+vpunpcklwd        %zmm16, %zmm17, %zmm19 {k1}
+vpunpcklwd        (%rax), %zmm17, %zmm19 {k1}
+vpunpcklwd        %zmm16, %zmm17, %zmm19 {z}{k1}
+vpunpcklwd        (%rax), %zmm17, %zmm19 {z}{k1}
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  1      1     0.50                        vpabsb	%zmm16, %zmm19
+# CHECK-NEXT:  2      8     0.50    *                   vpabsb	(%rax), %zmm19
+# CHECK-NEXT:  1      1     0.50                        vpabsb	%zmm16, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     0.50    *                   vpabsb	(%rax), %zmm19 {%k1}
+# CHECK-NEXT:  1      1     0.50                        vpabsb	%zmm16, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     0.50    *                   vpabsb	(%rax), %zmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     0.50                        vpabsw	%zmm16, %zmm19
+# CHECK-NEXT:  2      8     0.50    *                   vpabsw	(%rax), %zmm19
+# CHECK-NEXT:  1      1     0.50                        vpabsw	%zmm16, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     0.50    *                   vpabsw	(%rax), %zmm19 {%k1}
+# CHECK-NEXT:  1      1     0.50                        vpabsw	%zmm16, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     0.50    *                   vpabsw	(%rax), %zmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     0.50                        vpaddb	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  2      8     0.50    *                   vpaddb	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  1      1     0.50                        vpaddb	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     0.50    *                   vpaddb	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  1      1     0.50                        vpaddb	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     0.50    *                   vpaddb	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     0.50                        vpaddw	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  2      8     0.50    *                   vpaddw	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  1      1     0.50                        vpaddw	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     0.50    *                   vpaddw	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  1      1     0.50                        vpaddw	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     0.50    *                   vpaddw	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vpermw	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  2      8     1.00    *                   vpermw	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  1      1     1.00                        vpermw	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vpermw	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vpermw	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vpermw	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     0.50                        vpshufb	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  2      8     0.50    *                   vpshufb	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  1      1     0.50                        vpshufb	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     0.50    *                   vpshufb	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  1      1     0.50                        vpshufb	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     0.50    *                   vpshufb	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vpshufhw	$0, %zmm16, %zmm19
+# CHECK-NEXT:  2      8     1.00    *                   vpshufhw	$0, (%rax), %zmm19
+# CHECK-NEXT:  1      1     1.00                        vpshufhw	$0, %zmm16, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vpshufhw	$0, (%rax), %zmm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vpshufhw	$0, %zmm16, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vpshufhw	$0, (%rax), %zmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vpshuflw	$0, %zmm16, %zmm19
+# CHECK-NEXT:  2      8     1.00    *                   vpshuflw	$0, (%rax), %zmm19
+# CHECK-NEXT:  1      1     1.00                        vpshuflw	$0, %zmm16, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vpshuflw	$0, (%rax), %zmm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vpshuflw	$0, %zmm16, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vpshuflw	$0, (%rax), %zmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     0.50                        vpsubb	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  2      8     0.50    *                   vpsubb	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  1      1     0.50                        vpsubb	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     0.50    *                   vpsubb	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  1      1     0.50                        vpsubb	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     0.50    *                   vpsubb	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     0.50                        vpsubw	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  2      8     0.50    *                   vpsubw	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  1      1     0.50                        vpsubw	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     0.50    *                   vpsubw	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  1      1     0.50                        vpsubw	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     0.50    *                   vpsubw	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vpunpckhbw	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  2      8     1.00    *                   vpunpckhbw	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  1      1     1.00                        vpunpckhbw	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vpunpckhbw	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vpunpckhbw	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vpunpckhbw	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vpunpckhwd	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  2      8     1.00    *                   vpunpckhwd	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  1      1     1.00                        vpunpckhwd	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vpunpckhwd	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vpunpckhwd	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vpunpckhwd	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vpunpcklbw	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  2      8     1.00    *                   vpunpcklbw	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  1      1     1.00                        vpunpcklbw	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vpunpcklbw	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vpunpcklbw	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vpunpcklbw	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vpunpcklwd	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  2      8     1.00    *                   vpunpcklwd	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  1      1     1.00                        vpunpcklwd	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vpunpcklwd	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vpunpcklwd	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vpunpcklwd	(%rax), %zmm17, %zmm19 {%k1} {z}
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0]   - SBDivider
+# CHECK-NEXT: [1]   - SBFPDivider
+# CHECK-NEXT: [2]   - SBPort0
+# CHECK-NEXT: [3]   - SBPort1
+# CHECK-NEXT: [4]   - SBPort4
+# CHECK-NEXT: [5]   - SBPort5
+# CHECK-NEXT: [6.0] - SBPort23
+# CHECK-NEXT: [6.1] - SBPort23
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6.0]  [6.1]
+# CHECK-NEXT:  -      -      -     21.00   -     63.00  21.00  21.00
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6.0]  [6.1]  Instructions:
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpabsb	%zmm16, %zmm19
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpabsb	(%rax), %zmm19
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpabsb	%zmm16, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpabsb	(%rax), %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpabsb	%zmm16, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpabsb	(%rax), %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpabsw	%zmm16, %zmm19
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpabsw	(%rax), %zmm19
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpabsw	%zmm16, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpabsw	(%rax), %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpabsw	%zmm16, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpabsw	(%rax), %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpaddb	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpaddb	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpaddb	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpaddb	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpaddb	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpaddb	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpaddw	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpaddw	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpaddw	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpaddw	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpaddw	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpaddw	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpermw	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermw	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpermw	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermw	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpermw	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermw	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpshufb	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpshufb	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpshufb	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpshufb	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpshufb	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpshufb	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpshufhw	$0, %zmm16, %zmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpshufhw	$0, (%rax), %zmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpshufhw	$0, %zmm16, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpshufhw	$0, (%rax), %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpshufhw	$0, %zmm16, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpshufhw	$0, (%rax), %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpshuflw	$0, %zmm16, %zmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpshuflw	$0, (%rax), %zmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpshuflw	$0, %zmm16, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpshuflw	$0, (%rax), %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpshuflw	$0, %zmm16, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpshuflw	$0, (%rax), %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpsubb	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpsubb	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpsubb	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpsubb	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpsubb	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpsubb	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpsubw	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpsubw	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpsubw	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpsubw	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpsubw	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpsubw	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpunpckhbw	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpunpckhbw	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpunpckhbw	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpunpckhbw	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpunpckhbw	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpunpckhbw	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpunpckhwd	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpunpckhwd	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpunpckhwd	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpunpckhwd	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpunpckhwd	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpunpckhwd	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpunpcklbw	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpunpcklbw	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpunpcklbw	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpunpcklbw	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpunpcklbw	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpunpcklbw	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpunpcklwd	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpunpcklwd	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpunpcklwd	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpunpcklwd	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpunpcklwd	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpunpcklwd	(%rax), %zmm17, %zmm19 {%k1} {z}
diff --git a/test/tools/llvm-mca/X86/Generic/resources-avx512bwvl.s b/test/tools/llvm-mca/X86/Generic/resources-avx512bwvl.s
new file mode 100644
index 0000000000000000000000000000000000000000..71c54b8f9411c112c01dc4c90bc8595ea2ea23f3
--- /dev/null
+++ b/test/tools/llvm-mca/X86/Generic/resources-avx512bwvl.s
@@ -0,0 +1,523 @@
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -instruction-tables < %s | FileCheck %s
+
+vpabsb            %xmm16, %xmm19
+vpabsb            (%rax), %xmm19
+vpabsb            %xmm16, %xmm19 {k1}
+vpabsb            (%rax), %xmm19 {k1}
+vpabsb            %xmm16, %xmm19 {z}{k1}
+vpabsb            (%rax), %xmm19 {z}{k1}
+
+vpabsb            %ymm16, %ymm19
+vpabsb            (%rax), %ymm19
+vpabsb            %ymm16, %ymm19 {k1}
+vpabsb            (%rax), %ymm19 {k1}
+vpabsb            %ymm16, %ymm19 {z}{k1}
+vpabsb            (%rax), %ymm19 {z}{k1}
+
+vpabsw            %xmm16, %xmm19
+vpabsw            (%rax), %xmm19
+vpabsw            %xmm16, %xmm19 {k1}
+vpabsw            (%rax), %xmm19 {k1}
+vpabsw            %xmm16, %xmm19 {z}{k1}
+vpabsw            (%rax), %xmm19 {z}{k1}
+
+vpabsw            %ymm16, %ymm19
+vpabsw            (%rax), %ymm19
+vpabsw            %ymm16, %ymm19 {k1}
+vpabsw            (%rax), %ymm19 {k1}
+vpabsw            %ymm16, %ymm19 {z}{k1}
+vpabsw            (%rax), %ymm19 {z}{k1}
+
+vpaddb            %xmm16, %xmm17, %xmm19
+vpaddb            (%rax), %xmm17, %xmm19
+vpaddb            %xmm16, %xmm17, %xmm19 {k1}
+vpaddb            (%rax), %xmm17, %xmm19 {k1}
+vpaddb            %xmm16, %xmm17, %xmm19 {z}{k1}
+vpaddb            (%rax), %xmm17, %xmm19 {z}{k1}
+
+vpaddb            %ymm16, %ymm17, %ymm19
+vpaddb            (%rax), %ymm17, %ymm19
+vpaddb            %ymm16, %ymm17, %ymm19 {k1}
+vpaddb            (%rax), %ymm17, %ymm19 {k1}
+vpaddb            %ymm16, %ymm17, %ymm19 {z}{k1}
+vpaddb            (%rax), %ymm17, %ymm19 {z}{k1}
+
+vpaddw            %xmm16, %xmm17, %xmm19
+vpaddw            (%rax), %xmm17, %xmm19
+vpaddw            %xmm16, %xmm17, %xmm19 {k1}
+vpaddw            (%rax), %xmm17, %xmm19 {k1}
+vpaddw            %xmm16, %xmm17, %xmm19 {z}{k1}
+vpaddw            (%rax), %xmm17, %xmm19 {z}{k1}
+
+vpaddw            %ymm16, %ymm17, %ymm19
+vpaddw            (%rax), %ymm17, %ymm19
+vpaddw            %ymm16, %ymm17, %ymm19 {k1}
+vpaddw            (%rax), %ymm17, %ymm19 {k1}
+vpaddw            %ymm16, %ymm17, %ymm19 {z}{k1}
+vpaddw            (%rax), %ymm17, %ymm19 {z}{k1}
+
+vpermw            %xmm16, %xmm17, %xmm19
+vpermw            (%rax), %xmm17, %xmm19
+vpermw            %xmm16, %xmm17, %xmm19 {k1}
+vpermw            (%rax), %xmm17, %xmm19 {k1}
+vpermw            %xmm16, %xmm17, %xmm19 {z}{k1}
+vpermw            (%rax), %xmm17, %xmm19 {z}{k1}
+
+vpermw            %ymm16, %ymm17, %ymm19
+vpermw            (%rax), %ymm17, %ymm19
+vpermw            %ymm16, %ymm17, %ymm19 {k1}
+vpermw            (%rax), %ymm17, %ymm19 {k1}
+vpermw            %ymm16, %ymm17, %ymm19 {z}{k1}
+vpermw            (%rax), %ymm17, %ymm19 {z}{k1}
+
+vpshufb           %xmm16, %xmm17, %xmm19
+vpshufb           (%rax), %xmm17, %xmm19
+vpshufb           %xmm16, %xmm17, %xmm19 {k1}
+vpshufb           (%rax), %xmm17, %xmm19 {k1}
+vpshufb           %xmm16, %xmm17, %xmm19 {z}{k1}
+vpshufb           (%rax), %xmm17, %xmm19 {z}{k1}
+
+vpshufb           %ymm16, %ymm17, %ymm19
+vpshufb           (%rax), %ymm17, %ymm19
+vpshufb           %ymm16, %ymm17, %ymm19 {k1}
+vpshufb           (%rax), %ymm17, %ymm19 {k1}
+vpshufb           %ymm16, %ymm17, %ymm19 {z}{k1}
+vpshufb           (%rax), %ymm17, %ymm19 {z}{k1}
+
+vpshufhw          $0, %xmm16, %xmm19
+vpshufhw          $0, (%rax), %xmm19
+vpshufhw          $0, %xmm16, %xmm19 {k1}
+vpshufhw          $0, (%rax), %xmm19 {k1}
+vpshufhw          $0, %xmm16, %xmm19 {z}{k1}
+vpshufhw          $0, (%rax), %xmm19 {z}{k1}
+
+vpshufhw          $0, %ymm16, %ymm19
+vpshufhw          $0, (%rax), %ymm19
+vpshufhw          $0, %ymm16, %ymm19 {k1}
+vpshufhw          $0, (%rax), %ymm19 {k1}
+vpshufhw          $0, %ymm16, %ymm19 {z}{k1}
+vpshufhw          $0, (%rax), %ymm19 {z}{k1}
+
+vpsubb            %xmm16, %xmm17, %xmm19
+vpsubb            (%rax), %xmm17, %xmm19
+vpsubb            %xmm16, %xmm17, %xmm19 {k1}
+vpsubb            (%rax), %xmm17, %xmm19 {k1}
+vpsubb            %xmm16, %xmm17, %xmm19 {z}{k1}
+vpsubb            (%rax), %xmm17, %xmm19 {z}{k1}
+
+vpsubb            %ymm16, %ymm17, %ymm19
+vpsubb            (%rax), %ymm17, %ymm19
+vpsubb            %ymm16, %ymm17, %ymm19 {k1}
+vpsubb            (%rax), %ymm17, %ymm19 {k1}
+vpsubb            %ymm16, %ymm17, %ymm19 {z}{k1}
+vpsubb            (%rax), %ymm17, %ymm19 {z}{k1}
+
+vpsubw            %xmm16, %xmm17, %xmm19
+vpsubw            (%rax), %xmm17, %xmm19
+vpsubw            %xmm16, %xmm17, %xmm19 {k1}
+vpsubw            (%rax), %xmm17, %xmm19 {k1}
+vpsubw            %xmm16, %xmm17, %xmm19 {z}{k1}
+vpsubw            (%rax), %xmm17, %xmm19 {z}{k1}
+
+vpsubw            %ymm16, %ymm17, %ymm19
+vpsubw            (%rax), %ymm17, %ymm19
+vpsubw            %ymm16, %ymm17, %ymm19 {k1}
+vpsubw            (%rax), %ymm17, %ymm19 {k1}
+vpsubw            %ymm16, %ymm17, %ymm19 {z}{k1}
+vpsubw            (%rax), %ymm17, %ymm19 {z}{k1}
+
+vpunpckhbw         %xmm16, %xmm17, %xmm19
+vpunpckhbw         (%rax), %xmm17, %xmm19
+vpunpckhbw         %xmm16, %xmm17, %xmm19 {k1}
+vpunpckhbw         (%rax), %xmm17, %xmm19 {k1}
+vpunpckhbw         %xmm16, %xmm17, %xmm19 {z}{k1}
+vpunpckhbw         (%rax), %xmm17, %xmm19 {z}{k1}
+
+vpunpckhbw         %ymm16, %ymm17, %ymm19
+vpunpckhbw         (%rax), %ymm17, %ymm19
+vpunpckhbw         %ymm16, %ymm17, %ymm19 {k1}
+vpunpckhbw         (%rax), %ymm17, %ymm19 {k1}
+vpunpckhbw         %ymm16, %ymm17, %ymm19 {z}{k1}
+vpunpckhbw         (%rax), %ymm17, %ymm19 {z}{k1}
+
+vpunpckhwd         %xmm16, %xmm17, %xmm19
+vpunpckhwd         (%rax), %xmm17, %xmm19
+vpunpckhwd         %xmm16, %xmm17, %xmm19 {k1}
+vpunpckhwd         (%rax), %xmm17, %xmm19 {k1}
+vpunpckhwd         %xmm16, %xmm17, %xmm19 {z}{k1}
+vpunpckhwd         (%rax), %xmm17, %xmm19 {z}{k1}
+
+vpunpckhwd         %ymm16, %ymm17, %ymm19
+vpunpckhwd         (%rax), %ymm17, %ymm19
+vpunpckhwd         %ymm16, %ymm17, %ymm19 {k1}
+vpunpckhwd         (%rax), %ymm17, %ymm19 {k1}
+vpunpckhwd         %ymm16, %ymm17, %ymm19 {z}{k1}
+vpunpckhwd         (%rax), %ymm17, %ymm19 {z}{k1}
+
+vpunpcklbw         %xmm16, %xmm17, %xmm19
+vpunpcklbw         (%rax), %xmm17, %xmm19
+vpunpcklbw         %xmm16, %xmm17, %xmm19 {k1}
+vpunpcklbw         (%rax), %xmm17, %xmm19 {k1}
+vpunpcklbw         %xmm16, %xmm17, %xmm19 {z}{k1}
+vpunpcklbw         (%rax), %xmm17, %xmm19 {z}{k1}
+
+vpunpcklbw         %ymm16, %ymm17, %ymm19
+vpunpcklbw         (%rax), %ymm17, %ymm19
+vpunpcklbw         %ymm16, %ymm17, %ymm19 {k1}
+vpunpcklbw         (%rax), %ymm17, %ymm19 {k1}
+vpunpcklbw         %ymm16, %ymm17, %ymm19 {z}{k1}
+vpunpcklbw         (%rax), %ymm17, %ymm19 {z}{k1}
+
+vpunpcklwd         %xmm16, %xmm17, %xmm19
+vpunpcklwd         (%rax), %xmm17, %xmm19
+vpunpcklwd         %xmm16, %xmm17, %xmm19 {k1}
+vpunpcklwd         (%rax), %xmm17, %xmm19 {k1}
+vpunpcklwd         %xmm16, %xmm17, %xmm19 {z}{k1}
+vpunpcklwd         (%rax), %xmm17, %xmm19 {z}{k1}
+
+vpunpcklwd         %ymm16, %ymm17, %ymm19
+vpunpcklwd         (%rax), %ymm17, %ymm19
+vpunpcklwd         %ymm16, %ymm17, %ymm19 {k1}
+vpunpcklwd         (%rax), %ymm17, %ymm19 {k1}
+vpunpcklwd         %ymm16, %ymm17, %ymm19 {z}{k1}
+vpunpcklwd         (%rax), %ymm17, %ymm19 {z}{k1}
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  1      1     0.50                        vpabsb	%xmm16, %xmm19
+# CHECK-NEXT:  2      7     0.50    *                   vpabsb	(%rax), %xmm19
+# CHECK-NEXT:  1      1     0.50                        vpabsb	%xmm16, %xmm19 {%k1}
+# CHECK-NEXT:  2      7     0.50    *                   vpabsb	(%rax), %xmm19 {%k1}
+# CHECK-NEXT:  1      1     0.50                        vpabsb	%xmm16, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      7     0.50    *                   vpabsb	(%rax), %xmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     0.50                        vpabsb	%ymm16, %ymm19
+# CHECK-NEXT:  2      8     0.50    *                   vpabsb	(%rax), %ymm19
+# CHECK-NEXT:  1      1     0.50                        vpabsb	%ymm16, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     0.50    *                   vpabsb	(%rax), %ymm19 {%k1}
+# CHECK-NEXT:  1      1     0.50                        vpabsb	%ymm16, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     0.50    *                   vpabsb	(%rax), %ymm19 {%k1} {z}
+# CHECK-NEXT:  1      1     0.50                        vpabsw	%xmm16, %xmm19
+# CHECK-NEXT:  2      7     0.50    *                   vpabsw	(%rax), %xmm19
+# CHECK-NEXT:  1      1     0.50                        vpabsw	%xmm16, %xmm19 {%k1}
+# CHECK-NEXT:  2      7     0.50    *                   vpabsw	(%rax), %xmm19 {%k1}
+# CHECK-NEXT:  1      1     0.50                        vpabsw	%xmm16, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      7     0.50    *                   vpabsw	(%rax), %xmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     0.50                        vpabsw	%ymm16, %ymm19
+# CHECK-NEXT:  2      8     0.50    *                   vpabsw	(%rax), %ymm19
+# CHECK-NEXT:  1      1     0.50                        vpabsw	%ymm16, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     0.50    *                   vpabsw	(%rax), %ymm19 {%k1}
+# CHECK-NEXT:  1      1     0.50                        vpabsw	%ymm16, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     0.50    *                   vpabsw	(%rax), %ymm19 {%k1} {z}
+# CHECK-NEXT:  1      1     0.50                        vpaddb	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  2      7     0.50    *                   vpaddb	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  1      1     0.50                        vpaddb	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      7     0.50    *                   vpaddb	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  1      1     0.50                        vpaddb	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      7     0.50    *                   vpaddb	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     0.50                        vpaddb	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  2      8     0.50    *                   vpaddb	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  1      1     0.50                        vpaddb	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     0.50    *                   vpaddb	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  1      1     0.50                        vpaddb	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     0.50    *                   vpaddb	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  1      1     0.50                        vpaddw	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  2      7     0.50    *                   vpaddw	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  1      1     0.50                        vpaddw	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      7     0.50    *                   vpaddw	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  1      1     0.50                        vpaddw	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      7     0.50    *                   vpaddw	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     0.50                        vpaddw	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  2      8     0.50    *                   vpaddw	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  1      1     0.50                        vpaddw	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     0.50    *                   vpaddw	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  1      1     0.50                        vpaddw	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     0.50    *                   vpaddw	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vpermw	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  2      8     1.00    *                   vpermw	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  1      1     1.00                        vpermw	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vpermw	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vpermw	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vpermw	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vpermw	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  2      8     1.00    *                   vpermw	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  1      1     1.00                        vpermw	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vpermw	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vpermw	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vpermw	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  1      1     0.50                        vpshufb	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  2      7     0.50    *                   vpshufb	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  1      1     0.50                        vpshufb	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      7     0.50    *                   vpshufb	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  1      1     0.50                        vpshufb	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      7     0.50    *                   vpshufb	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     0.50                        vpshufb	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  2      8     0.50    *                   vpshufb	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  1      1     0.50                        vpshufb	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     0.50    *                   vpshufb	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  1      1     0.50                        vpshufb	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     0.50    *                   vpshufb	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  1      1     0.50                        vpshufhw	$0, %xmm16, %xmm19
+# CHECK-NEXT:  2      7     0.50    *                   vpshufhw	$0, (%rax), %xmm19
+# CHECK-NEXT:  1      1     0.50                        vpshufhw	$0, %xmm16, %xmm19 {%k1}
+# CHECK-NEXT:  2      7     0.50    *                   vpshufhw	$0, (%rax), %xmm19 {%k1}
+# CHECK-NEXT:  1      1     0.50                        vpshufhw	$0, %xmm16, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      7     0.50    *                   vpshufhw	$0, (%rax), %xmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vpshufhw	$0, %ymm16, %ymm19
+# CHECK-NEXT:  2      8     1.00    *                   vpshufhw	$0, (%rax), %ymm19
+# CHECK-NEXT:  1      1     1.00                        vpshufhw	$0, %ymm16, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vpshufhw	$0, (%rax), %ymm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vpshufhw	$0, %ymm16, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vpshufhw	$0, (%rax), %ymm19 {%k1} {z}
+# CHECK-NEXT:  1      1     0.50                        vpsubb	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  2      7     0.50    *                   vpsubb	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  1      1     0.50                        vpsubb	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      7     0.50    *                   vpsubb	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  1      1     0.50                        vpsubb	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      7     0.50    *                   vpsubb	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     0.50                        vpsubb	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  2      8     0.50    *                   vpsubb	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  1      1     0.50                        vpsubb	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     0.50    *                   vpsubb	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  1      1     0.50                        vpsubb	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     0.50    *                   vpsubb	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  1      1     0.50                        vpsubw	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  2      7     0.50    *                   vpsubw	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  1      1     0.50                        vpsubw	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      7     0.50    *                   vpsubw	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  1      1     0.50                        vpsubw	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      7     0.50    *                   vpsubw	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     0.50                        vpsubw	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  2      8     0.50    *                   vpsubw	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  1      1     0.50                        vpsubw	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     0.50    *                   vpsubw	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  1      1     0.50                        vpsubw	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     0.50    *                   vpsubw	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  1      1     0.50                        vpunpckhbw	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  2      7     0.50    *                   vpunpckhbw	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  1      1     0.50                        vpunpckhbw	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      7     0.50    *                   vpunpckhbw	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  1      1     0.50                        vpunpckhbw	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      7     0.50    *                   vpunpckhbw	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vpunpckhbw	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  2      8     1.00    *                   vpunpckhbw	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  1      1     1.00                        vpunpckhbw	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vpunpckhbw	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vpunpckhbw	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vpunpckhbw	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  1      1     0.50                        vpunpckhwd	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  2      7     0.50    *                   vpunpckhwd	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  1      1     0.50                        vpunpckhwd	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      7     0.50    *                   vpunpckhwd	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  1      1     0.50                        vpunpckhwd	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      7     0.50    *                   vpunpckhwd	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vpunpckhwd	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  2      8     1.00    *                   vpunpckhwd	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  1      1     1.00                        vpunpckhwd	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vpunpckhwd	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vpunpckhwd	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vpunpckhwd	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  1      1     0.50                        vpunpcklbw	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  2      7     0.50    *                   vpunpcklbw	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  1      1     0.50                        vpunpcklbw	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      7     0.50    *                   vpunpcklbw	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  1      1     0.50                        vpunpcklbw	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      7     0.50    *                   vpunpcklbw	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vpunpcklbw	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  2      8     1.00    *                   vpunpcklbw	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  1      1     1.00                        vpunpcklbw	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vpunpcklbw	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vpunpcklbw	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vpunpcklbw	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  1      1     0.50                        vpunpcklwd	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  2      7     0.50    *                   vpunpcklwd	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  1      1     0.50                        vpunpcklwd	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      7     0.50    *                   vpunpcklwd	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  1      1     0.50                        vpunpcklwd	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      7     0.50    *                   vpunpcklwd	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vpunpcklwd	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  2      8     1.00    *                   vpunpcklwd	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  1      1     1.00                        vpunpcklwd	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vpunpcklwd	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vpunpcklwd	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vpunpcklwd	(%rax), %ymm17, %ymm19 {%k1} {z}
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0]   - SBDivider
+# CHECK-NEXT: [1]   - SBFPDivider
+# CHECK-NEXT: [2]   - SBPort0
+# CHECK-NEXT: [3]   - SBPort1
+# CHECK-NEXT: [4]   - SBPort4
+# CHECK-NEXT: [5]   - SBPort5
+# CHECK-NEXT: [6.0] - SBPort23
+# CHECK-NEXT: [6.1] - SBPort23
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6.0]  [6.1]
+# CHECK-NEXT:  -      -      -     57.00   -     99.00  39.00  39.00
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6.0]  [6.1]  Instructions:
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpabsb	%xmm16, %xmm19
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpabsb	(%rax), %xmm19
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpabsb	%xmm16, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpabsb	(%rax), %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpabsb	%xmm16, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpabsb	(%rax), %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpabsb	%ymm16, %ymm19
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpabsb	(%rax), %ymm19
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpabsb	%ymm16, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpabsb	(%rax), %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpabsb	%ymm16, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpabsb	(%rax), %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpabsw	%xmm16, %xmm19
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpabsw	(%rax), %xmm19
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpabsw	%xmm16, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpabsw	(%rax), %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpabsw	%xmm16, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpabsw	(%rax), %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpabsw	%ymm16, %ymm19
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpabsw	(%rax), %ymm19
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpabsw	%ymm16, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpabsw	(%rax), %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpabsw	%ymm16, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpabsw	(%rax), %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpaddb	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpaddb	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpaddb	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpaddb	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpaddb	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpaddb	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpaddb	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpaddb	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpaddb	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpaddb	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpaddb	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpaddb	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpaddw	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpaddw	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpaddw	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpaddw	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpaddw	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpaddw	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpaddw	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpaddw	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpaddw	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpaddw	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpaddw	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpaddw	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpermw	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermw	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpermw	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermw	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpermw	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermw	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpermw	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermw	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpermw	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermw	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpermw	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermw	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpshufb	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpshufb	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpshufb	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpshufb	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpshufb	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpshufb	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpshufb	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpshufb	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpshufb	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpshufb	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpshufb	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpshufb	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpshufhw	$0, %xmm16, %xmm19
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpshufhw	$0, (%rax), %xmm19
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpshufhw	$0, %xmm16, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpshufhw	$0, (%rax), %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpshufhw	$0, %xmm16, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpshufhw	$0, (%rax), %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpshufhw	$0, %ymm16, %ymm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpshufhw	$0, (%rax), %ymm19
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpshufhw	$0, %ymm16, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpshufhw	$0, (%rax), %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpshufhw	$0, %ymm16, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpshufhw	$0, (%rax), %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpsubb	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpsubb	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpsubb	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpsubb	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpsubb	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpsubb	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpsubb	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpsubb	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpsubb	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpsubb	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpsubb	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpsubb	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpsubw	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpsubw	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpsubw	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpsubw	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpsubw	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpsubw	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpsubw	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpsubw	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpsubw	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpsubw	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpsubw	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpsubw	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpunpckhbw	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpunpckhbw	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpunpckhbw	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpunpckhbw	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpunpckhbw	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpunpckhbw	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpunpckhbw	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpunpckhbw	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpunpckhbw	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpunpckhbw	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpunpckhbw	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpunpckhbw	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpunpckhwd	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpunpckhwd	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpunpckhwd	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpunpckhwd	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpunpckhwd	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpunpckhwd	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpunpckhwd	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpunpckhwd	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpunpckhwd	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpunpckhwd	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpunpckhwd	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpunpckhwd	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpunpcklbw	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpunpcklbw	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpunpcklbw	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpunpcklbw	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpunpcklbw	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpunpcklbw	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpunpcklbw	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpunpcklbw	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpunpcklbw	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpunpcklbw	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpunpcklbw	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpunpcklbw	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpunpcklwd	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpunpcklwd	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpunpcklwd	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpunpcklwd	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpunpcklwd	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpunpcklwd	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpunpcklwd	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpunpcklwd	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpunpcklwd	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpunpcklwd	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpunpcklwd	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpunpcklwd	(%rax), %ymm17, %ymm19 {%k1} {z}
diff --git a/test/tools/llvm-mca/X86/Generic/resources-avx512dq.s b/test/tools/llvm-mca/X86/Generic/resources-avx512dq.s
new file mode 100644
index 0000000000000000000000000000000000000000..1947562b5cf0e542d7ee90161c1cdccdca996934
--- /dev/null
+++ b/test/tools/llvm-mca/X86/Generic/resources-avx512dq.s
@@ -0,0 +1,328 @@
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -instruction-tables < %s | FileCheck %s
+
+vandnpd           %zmm16, %zmm17, %zmm19
+vandnpd           (%rax), %zmm17, %zmm19
+vandnpd           (%rax){1to8}, %zmm17, %zmm19
+vandnpd           %zmm16, %zmm17, %zmm19 {k1}
+vandnpd           (%rax), %zmm17, %zmm19 {k1}
+vandnpd           (%rax){1to8}, %zmm17, %zmm19 {k1}
+vandnpd           %zmm16, %zmm17, %zmm19 {z}{k1}
+vandnpd           (%rax), %zmm17, %zmm19 {z}{k1}
+vandnpd           (%rax){1to8}, %zmm17, %zmm19 {z}{k1}
+
+vandnps           %zmm16, %zmm17, %zmm19
+vandnps           (%rax), %zmm17, %zmm19
+vandnps           (%rax){1to16}, %zmm17, %zmm19
+vandnps           %zmm16, %zmm17, %zmm19 {k1}
+vandnps           (%rax), %zmm17, %zmm19 {k1}
+vandnps           (%rax){1to16}, %zmm17, %zmm19 {k1}
+vandnps           %zmm16, %zmm17, %zmm19 {z}{k1}
+vandnps           (%rax), %zmm17, %zmm19 {z}{k1}
+vandnps           (%rax){1to16}, %zmm17, %zmm19 {z}{k1}
+
+vandpd            %zmm16, %zmm17, %zmm19
+vandpd            (%rax), %zmm17, %zmm19
+vandpd            (%rax){1to8}, %zmm17, %zmm19
+vandpd            %zmm16, %zmm17, %zmm19 {k1}
+vandpd            (%rax), %zmm17, %zmm19 {k1}
+vandpd            (%rax){1to8}, %zmm17, %zmm19 {k1}
+vandpd            %zmm16, %zmm17, %zmm19 {z}{k1}
+vandpd            (%rax), %zmm17, %zmm19 {z}{k1}
+vandpd            (%rax){1to8}, %zmm17, %zmm19 {z}{k1}
+
+vandps            %zmm16, %zmm17, %zmm19
+vandps            (%rax), %zmm17, %zmm19
+vandps            (%rax){1to16}, %zmm17, %zmm19
+vandps            %zmm16, %zmm17, %zmm19 {k1}
+vandps            (%rax), %zmm17, %zmm19 {k1}
+vandps            (%rax){1to16}, %zmm17, %zmm19 {k1}
+vandps            %zmm16, %zmm17, %zmm19 {z}{k1}
+vandps            (%rax), %zmm17, %zmm19 {z}{k1}
+vandps            (%rax){1to16}, %zmm17, %zmm19 {z}{k1}
+
+vcvtqq2pd         %zmm16, %zmm19
+vcvtqq2pd         (%rax), %zmm19
+vcvtqq2pd         (%rax){1to8}, %zmm19
+vcvtqq2pd         %zmm16, %zmm19 {k1}
+vcvtqq2pd         (%rax), %zmm19 {k1}
+vcvtqq2pd         (%rax){1to8}, %zmm19 {k1}
+vcvtqq2pd         %zmm16, %zmm19 {z}{k1}
+vcvtqq2pd         (%rax), %zmm19 {z}{k1}
+vcvtqq2pd         (%rax){1to8}, %zmm19 {z}{k1}
+
+vcvtqq2ps         %zmm16, %ymm19
+vcvtqq2ps         (%rax), %ymm19
+vcvtqq2ps         (%rax){1to8}, %ymm19
+vcvtqq2ps         %zmm16, %ymm19 {k1}
+vcvtqq2ps         (%rax), %ymm19 {k1}
+vcvtqq2ps         (%rax){1to8}, %ymm19 {k1}
+vcvtqq2ps         %zmm16, %ymm19 {z}{k1}
+vcvtqq2ps         (%rax), %ymm19 {z}{k1}
+vcvtqq2ps         (%rax){1to8}, %ymm19 {z}{k1}
+
+vorpd             %zmm16, %zmm17, %zmm19
+vorpd             (%rax), %zmm17, %zmm19
+vorpd             (%rax){1to8}, %zmm17, %zmm19
+vorpd             %zmm16, %zmm17, %zmm19 {k1}
+vorpd             (%rax), %zmm17, %zmm19 {k1}
+vorpd             (%rax){1to8}, %zmm17, %zmm19 {k1}
+vorpd             %zmm16, %zmm17, %zmm19 {z}{k1}
+vorpd             (%rax), %zmm17, %zmm19 {z}{k1}
+vorpd             (%rax){1to8}, %zmm17, %zmm19 {z}{k1}
+
+vorps             %zmm16, %zmm17, %zmm19
+vorps             (%rax), %zmm17, %zmm19
+vorps             (%rax){1to16}, %zmm17, %zmm19
+vorps             %zmm16, %zmm17, %zmm19 {k1}
+vorps             (%rax), %zmm17, %zmm19 {k1}
+vorps             (%rax){1to16}, %zmm17, %zmm19 {k1}
+vorps             %zmm16, %zmm17, %zmm19 {z}{k1}
+vorps             (%rax), %zmm17, %zmm19 {z}{k1}
+vorps             (%rax){1to16}, %zmm17, %zmm19 {z}{k1}
+
+vpmullq           %zmm16, %zmm17, %zmm19
+vpmullq           (%rax), %zmm17, %zmm19
+vpmullq           %zmm16, %zmm17, %zmm19 {k1}
+vpmullq           (%rax), %zmm17, %zmm19 {k1}
+vpmullq           %zmm16, %zmm17, %zmm19 {z}{k1}
+vpmullq           (%rax), %zmm17, %zmm19 {z}{k1}
+
+vxorpd            %zmm16, %zmm17, %zmm19
+vxorpd            (%rax), %zmm17, %zmm19
+vxorpd            (%rax){1to8}, %zmm17, %zmm19
+vxorpd            %zmm16, %zmm17, %zmm19 {k1}
+vxorpd            (%rax), %zmm17, %zmm19 {k1}
+vxorpd            (%rax){1to8}, %zmm17, %zmm19 {k1}
+vxorpd            %zmm16, %zmm17, %zmm19 {z}{k1}
+vxorpd            (%rax), %zmm17, %zmm19 {z}{k1}
+vxorpd            (%rax){1to8}, %zmm17, %zmm19 {z}{k1}
+
+vxorps            %zmm16, %zmm17, %zmm19
+vxorps            (%rax), %zmm17, %zmm19
+vxorps            (%rax){1to16}, %zmm17, %zmm19
+vxorps            %zmm16, %zmm17, %zmm19 {k1}
+vxorps            (%rax), %zmm17, %zmm19 {k1}
+vxorps            (%rax){1to16}, %zmm17, %zmm19 {k1}
+vxorps            %zmm16, %zmm17, %zmm19 {z}{k1}
+vxorps            (%rax), %zmm17, %zmm19 {z}{k1}
+vxorps            (%rax){1to16}, %zmm17, %zmm19 {z}{k1}
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  1      1     1.00                        vandnpd	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  2      8     1.00    *                   vandnpd	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  2      8     1.00    *                   vandnpd	(%rax){1to8}, %zmm17, %zmm19
+# CHECK-NEXT:  1      1     1.00                        vandnpd	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vandnpd	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vandnpd	(%rax){1to8}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vandnpd	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vandnpd	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vandnpd	(%rax){1to8}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vandnps	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  2      8     1.00    *                   vandnps	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  2      8     1.00    *                   vandnps	(%rax){1to16}, %zmm17, %zmm19
+# CHECK-NEXT:  1      1     1.00                        vandnps	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vandnps	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vandnps	(%rax){1to16}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vandnps	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vandnps	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vandnps	(%rax){1to16}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vandpd	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  2      8     1.00    *                   vandpd	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  2      8     1.00    *                   vandpd	(%rax){1to8}, %zmm17, %zmm19
+# CHECK-NEXT:  1      1     1.00                        vandpd	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vandpd	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vandpd	(%rax){1to8}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vandpd	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vandpd	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vandpd	(%rax){1to8}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vandps	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  2      8     1.00    *                   vandps	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  2      8     1.00    *                   vandps	(%rax){1to16}, %zmm17, %zmm19
+# CHECK-NEXT:  1      1     1.00                        vandps	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vandps	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vandps	(%rax){1to16}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vandps	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vandps	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vandps	(%rax){1to16}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      4     1.00                        vcvtqq2pd	%zmm16, %zmm19
+# CHECK-NEXT:  3      10    1.00    *                   vcvtqq2pd	(%rax), %zmm19
+# CHECK-NEXT:  3      10    1.00    *                   vcvtqq2pd	(%rax){1to8}, %zmm19
+# CHECK-NEXT:  2      4     1.00                        vcvtqq2pd	%zmm16, %zmm19 {%k1}
+# CHECK-NEXT:  3      10    1.00    *                   vcvtqq2pd	(%rax), %zmm19 {%k1}
+# CHECK-NEXT:  3      10    1.00    *                   vcvtqq2pd	(%rax){1to8}, %zmm19 {%k1}
+# CHECK-NEXT:  2      4     1.00                        vcvtqq2pd	%zmm16, %zmm19 {%k1} {z}
+# CHECK-NEXT:  3      10    1.00    *                   vcvtqq2pd	(%rax), %zmm19 {%k1} {z}
+# CHECK-NEXT:  3      10    1.00    *                   vcvtqq2pd	(%rax){1to8}, %zmm19 {%k1} {z}
+# CHECK-NEXT:  1      3     1.00                        vcvtqq2ps	%zmm16, %ymm19
+# CHECK-NEXT:  2      10    1.00    *                   vcvtqq2ps	(%rax), %ymm19
+# CHECK-NEXT:  2      10    1.00    *                   vcvtqq2ps	(%rax){1to8}, %ymm19
+# CHECK-NEXT:  1      3     1.00                        vcvtqq2ps	%zmm16, %ymm19 {%k1}
+# CHECK-NEXT:  2      10    1.00    *                   vcvtqq2ps	(%rax), %ymm19 {%k1}
+# CHECK-NEXT:  2      10    1.00    *                   vcvtqq2ps	(%rax){1to8}, %ymm19 {%k1}
+# CHECK-NEXT:  1      3     1.00                        vcvtqq2ps	%zmm16, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      10    1.00    *                   vcvtqq2ps	(%rax), %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      10    1.00    *                   vcvtqq2ps	(%rax){1to8}, %ymm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vorpd	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  2      8     1.00    *                   vorpd	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  2      8     1.00    *                   vorpd	(%rax){1to8}, %zmm17, %zmm19
+# CHECK-NEXT:  1      1     1.00                        vorpd	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vorpd	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vorpd	(%rax){1to8}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vorpd	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vorpd	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vorpd	(%rax){1to8}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vorps	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  2      8     1.00    *                   vorps	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  2      8     1.00    *                   vorps	(%rax){1to16}, %zmm17, %zmm19
+# CHECK-NEXT:  1      1     1.00                        vorps	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vorps	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vorps	(%rax){1to16}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vorps	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vorps	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vorps	(%rax){1to16}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  1      5     1.00                        vpmullq	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  2      12    1.00    *                   vpmullq	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  1      5     1.00                        vpmullq	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      12    1.00    *                   vpmullq	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  1      5     1.00                        vpmullq	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      12    1.00    *                   vpmullq	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vxorpd	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  2      8     1.00    *                   vxorpd	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  2      8     1.00    *                   vxorpd	(%rax){1to8}, %zmm17, %zmm19
+# CHECK-NEXT:  1      1     1.00                        vxorpd	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vxorpd	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vxorpd	(%rax){1to8}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vxorpd	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vxorpd	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vxorpd	(%rax){1to8}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vxorps	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  2      8     1.00    *                   vxorps	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  2      8     1.00    *                   vxorps	(%rax){1to16}, %zmm17, %zmm19
+# CHECK-NEXT:  1      1     1.00                        vxorps	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vxorps	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vxorps	(%rax){1to16}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vxorps	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vxorps	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vxorps	(%rax){1to16}, %zmm17, %zmm19 {%k1} {z}
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0]   - SBDivider
+# CHECK-NEXT: [1]   - SBFPDivider
+# CHECK-NEXT: [2]   - SBPort0
+# CHECK-NEXT: [3]   - SBPort1
+# CHECK-NEXT: [4]   - SBPort4
+# CHECK-NEXT: [5]   - SBPort5
+# CHECK-NEXT: [6.0] - SBPort23
+# CHECK-NEXT: [6.1] - SBPort23
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6.0]  [6.1]
+# CHECK-NEXT:  -      -     6.00   18.00   -     81.00  31.50  31.50
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6.0]  [6.1]  Instructions:
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vandnpd	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vandnpd	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vandnpd	(%rax){1to8}, %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vandnpd	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vandnpd	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vandnpd	(%rax){1to8}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vandnpd	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vandnpd	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vandnpd	(%rax){1to8}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vandnps	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vandnps	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vandnps	(%rax){1to16}, %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vandnps	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vandnps	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vandnps	(%rax){1to16}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vandnps	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vandnps	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vandnps	(%rax){1to16}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vandpd	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vandpd	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vandpd	(%rax){1to8}, %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vandpd	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vandpd	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vandpd	(%rax){1to8}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vandpd	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vandpd	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vandpd	(%rax){1to8}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vandps	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vandps	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vandps	(%rax){1to16}, %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vandps	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vandps	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vandps	(%rax){1to16}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vandps	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vandps	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vandps	(%rax){1to16}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     1.00    -     1.00    -      -     vcvtqq2pd	%zmm16, %zmm19
+# CHECK-NEXT:  -      -      -     1.00    -     1.00   0.50   0.50   vcvtqq2pd	(%rax), %zmm19
+# CHECK-NEXT:  -      -      -     1.00    -     1.00   0.50   0.50   vcvtqq2pd	(%rax){1to8}, %zmm19
+# CHECK-NEXT:  -      -      -     1.00    -     1.00    -      -     vcvtqq2pd	%zmm16, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -     1.00    -     1.00   0.50   0.50   vcvtqq2pd	(%rax), %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -     1.00    -     1.00   0.50   0.50   vcvtqq2pd	(%rax){1to8}, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -     1.00    -     1.00    -      -     vcvtqq2pd	%zmm16, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     1.00    -     1.00   0.50   0.50   vcvtqq2pd	(%rax), %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     1.00    -     1.00   0.50   0.50   vcvtqq2pd	(%rax){1to8}, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -     vcvtqq2ps	%zmm16, %ymm19
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vcvtqq2ps	(%rax), %ymm19
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vcvtqq2ps	(%rax){1to8}, %ymm19
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -     vcvtqq2ps	%zmm16, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vcvtqq2ps	(%rax), %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vcvtqq2ps	(%rax){1to8}, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -     vcvtqq2ps	%zmm16, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vcvtqq2ps	(%rax), %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vcvtqq2ps	(%rax){1to8}, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vorpd	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vorpd	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vorpd	(%rax){1to8}, %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vorpd	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vorpd	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vorpd	(%rax){1to8}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vorpd	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vorpd	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vorpd	(%rax){1to8}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vorps	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vorps	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vorps	(%rax){1to16}, %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vorps	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vorps	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vorps	(%rax){1to16}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vorps	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vorps	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vorps	(%rax){1to16}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vpmullq	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  -      -     1.00    -      -      -     0.50   0.50   vpmullq	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vpmullq	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -     1.00    -      -      -     0.50   0.50   vpmullq	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vpmullq	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     1.00    -      -      -     0.50   0.50   vpmullq	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vxorpd	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vxorpd	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vxorpd	(%rax){1to8}, %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vxorpd	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vxorpd	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vxorpd	(%rax){1to8}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vxorpd	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vxorpd	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vxorpd	(%rax){1to8}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vxorps	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vxorps	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vxorps	(%rax){1to16}, %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vxorps	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vxorps	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vxorps	(%rax){1to16}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vxorps	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vxorps	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vxorps	(%rax){1to16}, %zmm17, %zmm19 {%k1} {z}
diff --git a/test/tools/llvm-mca/X86/Generic/resources-avx512dqvl.s b/test/tools/llvm-mca/X86/Generic/resources-avx512dqvl.s
new file mode 100644
index 0000000000000000000000000000000000000000..90c25753e37b219ce0f97d86d1ce7513113dbfcc
--- /dev/null
+++ b/test/tools/llvm-mca/X86/Generic/resources-avx512dqvl.s
@@ -0,0 +1,515 @@
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -instruction-tables < %s | FileCheck %s
+
+vandnpd           %xmm16, %xmm17, %xmm19
+vandnpd           (%rax), %xmm17, %xmm19
+vandnpd           (%rax){1to2}, %xmm17, %xmm19
+vandnpd           %xmm16, %xmm17, %xmm19 {k1}
+vandnpd           (%rax), %xmm17, %xmm19 {k1}
+vandnpd           (%rax){1to2}, %xmm17, %xmm19 {k1}
+vandnpd           %xmm16, %xmm17, %xmm19 {z}{k1}
+vandnpd           (%rax), %xmm17, %xmm19 {z}{k1}
+vandnpd           (%rax){1to2}, %xmm17, %xmm19 {z}{k1}
+
+vandnpd           %ymm16, %ymm17, %ymm19
+vandnpd           (%rax), %ymm17, %ymm19
+vandnpd           (%rax){1to4}, %ymm17, %ymm19
+vandnpd           %ymm16, %ymm17, %ymm19 {k1}
+vandnpd           (%rax), %ymm17, %ymm19 {k1}
+vandnpd           (%rax){1to4}, %ymm17, %ymm19 {k1}
+vandnpd           %ymm16, %ymm17, %ymm19 {z}{k1}
+vandnpd           (%rax), %ymm17, %ymm19 {z}{k1}
+vandnpd           (%rax){1to4}, %ymm17, %ymm19 {z}{k1}
+
+vandpd            %xmm16, %xmm17, %xmm19
+vandpd            (%rax), %xmm17, %xmm19
+vandpd            (%rax){1to2}, %xmm17, %xmm19
+vandpd            %xmm16, %xmm17, %xmm19 {k1}
+vandpd            (%rax), %xmm17, %xmm19 {k1}
+vandpd            (%rax){1to2}, %xmm17, %xmm19 {k1}
+vandpd            %xmm16, %xmm17, %xmm19 {z}{k1}
+vandpd            (%rax), %xmm17, %xmm19 {z}{k1}
+vandpd            (%rax){1to2}, %xmm17, %xmm19 {z}{k1}
+
+vandpd            %ymm16, %ymm17, %ymm19
+vandpd            (%rax), %ymm17, %ymm19
+vandpd            (%rax){1to4}, %ymm17, %ymm19
+vandpd            %ymm16, %ymm17, %ymm19 {k1}
+vandpd            (%rax), %ymm17, %ymm19 {k1}
+vandpd            (%rax){1to4}, %ymm17, %ymm19 {k1}
+vandpd            %ymm16, %ymm17, %ymm19 {z}{k1}
+vandpd            (%rax), %ymm17, %ymm19 {z}{k1}
+vandpd            (%rax){1to4}, %ymm17, %ymm19 {z}{k1}
+
+vandps            %xmm16, %xmm17, %xmm19
+vandps            (%rax), %xmm17, %xmm19
+vandps            (%rax){1to4}, %xmm17, %xmm19
+vandps            %xmm16, %xmm17, %xmm19 {k1}
+vandps            (%rax), %xmm17, %xmm19 {k1}
+vandps            (%rax){1to4}, %xmm17, %xmm19 {k1}
+vandps            %xmm16, %xmm17, %xmm19 {z}{k1}
+vandps            (%rax), %xmm17, %xmm19 {z}{k1}
+vandps            (%rax){1to4}, %xmm17, %xmm19 {z}{k1}
+
+vandps            %ymm16, %ymm17, %ymm19
+vandps            (%rax), %ymm17, %ymm19
+vandps            (%rax){1to8}, %ymm17, %ymm19
+vandps            %ymm16, %ymm17, %ymm19 {k1}
+vandps            (%rax), %ymm17, %ymm19 {k1}
+vandps            (%rax){1to8}, %ymm17, %ymm19 {k1}
+vandps            %ymm16, %ymm17, %ymm19 {z}{k1}
+vandps            (%rax), %ymm17, %ymm19 {z}{k1}
+vandps            (%rax){1to8}, %ymm17, %ymm19 {z}{k1}
+
+vcvtqq2pd         %xmm16, %xmm19
+vcvtqq2pd         (%rax), %xmm19
+vcvtqq2pd         (%rax){1to2}, %xmm19
+vcvtqq2pd         %xmm16, %xmm19 {k1}
+vcvtqq2pd         (%rax), %xmm19 {k1}
+vcvtqq2pd         (%rax){1to2}, %xmm19 {k1}
+vcvtqq2pd         %xmm16, %xmm19 {z}{k1}
+vcvtqq2pd         (%rax), %xmm19 {z}{k1}
+vcvtqq2pd         (%rax){1to2}, %xmm19 {z}{k1}
+
+vcvtqq2pd         %ymm16, %ymm19
+vcvtqq2pd         (%rax), %ymm19
+vcvtqq2pd         (%rax){1to4}, %ymm19
+vcvtqq2pd         %ymm16, %ymm19 {k1}
+vcvtqq2pd         (%rax), %ymm19 {k1}
+vcvtqq2pd         (%rax){1to4}, %ymm19 {k1}
+vcvtqq2pd         %ymm16, %ymm19 {z}{k1}
+vcvtqq2pd         (%rax), %ymm19 {z}{k1}
+vcvtqq2pd         (%rax){1to4}, %ymm19 {z}{k1}
+
+vorpd             %xmm16, %xmm17, %xmm19
+vorpd             (%rax), %xmm17, %xmm19
+vorpd             (%rax){1to2}, %xmm17, %xmm19
+vorpd             %xmm16, %xmm17, %xmm19 {k1}
+vorpd             (%rax), %xmm17, %xmm19 {k1}
+vorpd             (%rax){1to2}, %xmm17, %xmm19 {k1}
+vorpd             %xmm16, %xmm17, %xmm19 {z}{k1}
+vorpd             (%rax), %xmm17, %xmm19 {z}{k1}
+vorpd             (%rax){1to2}, %xmm17, %xmm19 {z}{k1}
+
+vorpd             %ymm16, %ymm17, %ymm19
+vorpd             (%rax), %ymm17, %ymm19
+vorpd             (%rax){1to4}, %ymm17, %ymm19
+vorpd             %ymm16, %ymm17, %ymm19 {k1}
+vorpd             (%rax), %ymm17, %ymm19 {k1}
+vorpd             (%rax){1to4}, %ymm17, %ymm19 {k1}
+vorpd             %ymm16, %ymm17, %ymm19 {z}{k1}
+vorpd             (%rax), %ymm17, %ymm19 {z}{k1}
+vorpd             (%rax){1to4}, %ymm17, %ymm19 {z}{k1}
+
+vorps             %xmm16, %xmm17, %xmm19
+vorps             (%rax), %xmm17, %xmm19
+vorps             (%rax){1to4}, %xmm17, %xmm19
+vorps             %xmm16, %xmm17, %xmm19 {k1}
+vorps             (%rax), %xmm17, %xmm19 {k1}
+vorps             (%rax){1to4}, %xmm17, %xmm19 {k1}
+vorps             %xmm16, %xmm17, %xmm19 {z}{k1}
+vorps             (%rax), %xmm17, %xmm19 {z}{k1}
+vorps             (%rax){1to4}, %xmm17, %xmm19 {z}{k1}
+
+vorps             %ymm16, %ymm17, %ymm19
+vorps             (%rax), %ymm17, %ymm19
+vorps             (%rax){1to8}, %ymm17, %ymm19
+vorps             %ymm16, %ymm17, %ymm19 {k1}
+vorps             (%rax), %ymm17, %ymm19 {k1}
+vorps             (%rax){1to8}, %ymm17, %ymm19 {k1}
+vorps             %ymm16, %ymm17, %ymm19 {z}{k1}
+vorps             (%rax), %ymm17, %ymm19 {z}{k1}
+vorps             (%rax){1to8}, %ymm17, %ymm19 {z}{k1}
+
+vpmullq           %xmm16, %xmm17, %xmm19
+vpmullq           (%rax), %xmm17, %xmm19
+vpmullq           %xmm16, %xmm17, %xmm19 {k1}
+vpmullq           (%rax), %xmm17, %xmm19 {k1}
+vpmullq           %xmm16, %xmm17, %xmm19 {z}{k1}
+vpmullq           (%rax), %xmm17, %xmm19 {z}{k1}
+
+vpmullq           %ymm16, %ymm17, %ymm19
+vpmullq           (%rax), %ymm17, %ymm19
+vpmullq           %ymm16, %ymm17, %ymm19 {k1}
+vpmullq           (%rax), %ymm17, %ymm19 {k1}
+vpmullq           %ymm16, %ymm17, %ymm19 {z}{k1}
+vpmullq           (%rax), %ymm17, %ymm19 {z}{k1}
+
+vxorpd            %xmm16, %xmm17, %xmm19
+vxorpd            (%rax), %xmm17, %xmm19
+vxorpd            (%rax){1to2}, %xmm17, %xmm19
+vxorpd            %xmm16, %xmm17, %xmm19 {k1}
+vxorpd            (%rax), %xmm17, %xmm19 {k1}
+vxorpd            (%rax){1to2}, %xmm17, %xmm19 {k1}
+vxorpd            %xmm16, %xmm17, %xmm19 {z}{k1}
+vxorpd            (%rax), %xmm17, %xmm19 {z}{k1}
+vxorpd            (%rax){1to2}, %xmm17, %xmm19 {z}{k1}
+
+vxorpd            %ymm16, %ymm17, %ymm19
+vxorpd            (%rax), %ymm17, %ymm19
+vxorpd            (%rax){1to4}, %ymm17, %ymm19
+vxorpd            %ymm16, %ymm17, %ymm19 {k1}
+vxorpd            (%rax), %ymm17, %ymm19 {k1}
+vxorpd            (%rax){1to4}, %ymm17, %ymm19 {k1}
+vxorpd            %ymm16, %ymm17, %ymm19 {z}{k1}
+vxorpd            (%rax), %ymm17, %ymm19 {z}{k1}
+vxorpd            (%rax){1to4}, %ymm17, %ymm19 {z}{k1}
+
+vxorps            %xmm16, %xmm17, %xmm19
+vxorps            (%rax), %xmm17, %xmm19
+vxorps            (%rax){1to4}, %xmm17, %xmm19
+vxorps            %xmm16, %xmm17, %xmm19 {k1}
+vxorps            (%rax), %xmm17, %xmm19 {k1}
+vxorps            (%rax){1to4}, %xmm17, %xmm19 {k1}
+vxorps            %xmm16, %xmm17, %xmm19 {z}{k1}
+vxorps            (%rax), %xmm17, %xmm19 {z}{k1}
+vxorps            (%rax){1to4}, %xmm17, %xmm19 {z}{k1}
+
+vxorps            %ymm16, %ymm17, %ymm19
+vxorps            (%rax), %ymm17, %ymm19
+vxorps            (%rax){1to8}, %ymm17, %ymm19
+vxorps            %ymm16, %ymm17, %ymm19 {k1}
+vxorps            (%rax), %ymm17, %ymm19 {k1}
+vxorps            (%rax){1to8}, %ymm17, %ymm19 {k1}
+vxorps            %ymm16, %ymm17, %ymm19 {z}{k1}
+vxorps            (%rax), %ymm17, %ymm19 {z}{k1}
+vxorps            (%rax){1to8}, %ymm17, %ymm19 {z}{k1}
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  1      1     1.00                        vandnpd	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  2      7     1.00    *                   vandnpd	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  2      7     1.00    *                   vandnpd	(%rax){1to2}, %xmm17, %xmm19
+# CHECK-NEXT:  1      1     1.00                        vandnpd	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      7     1.00    *                   vandnpd	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      7     1.00    *                   vandnpd	(%rax){1to2}, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vandnpd	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      7     1.00    *                   vandnpd	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      7     1.00    *                   vandnpd	(%rax){1to2}, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vandnpd	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  2      8     1.00    *                   vandnpd	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  2      8     1.00    *                   vandnpd	(%rax){1to4}, %ymm17, %ymm19
+# CHECK-NEXT:  1      1     1.00                        vandnpd	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vandnpd	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vandnpd	(%rax){1to4}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vandnpd	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vandnpd	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vandnpd	(%rax){1to4}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vandpd	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  2      7     1.00    *                   vandpd	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  2      7     1.00    *                   vandpd	(%rax){1to2}, %xmm17, %xmm19
+# CHECK-NEXT:  1      1     1.00                        vandpd	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      7     1.00    *                   vandpd	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      7     1.00    *                   vandpd	(%rax){1to2}, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vandpd	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      7     1.00    *                   vandpd	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      7     1.00    *                   vandpd	(%rax){1to2}, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vandpd	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  2      8     1.00    *                   vandpd	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  2      8     1.00    *                   vandpd	(%rax){1to4}, %ymm17, %ymm19
+# CHECK-NEXT:  1      1     1.00                        vandpd	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vandpd	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vandpd	(%rax){1to4}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vandpd	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vandpd	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vandpd	(%rax){1to4}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vandps	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  2      7     1.00    *                   vandps	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  2      7     1.00    *                   vandps	(%rax){1to4}, %xmm17, %xmm19
+# CHECK-NEXT:  1      1     1.00                        vandps	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      7     1.00    *                   vandps	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      7     1.00    *                   vandps	(%rax){1to4}, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vandps	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      7     1.00    *                   vandps	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      7     1.00    *                   vandps	(%rax){1to4}, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vandps	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  2      8     1.00    *                   vandps	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  2      8     1.00    *                   vandps	(%rax){1to8}, %ymm17, %ymm19
+# CHECK-NEXT:  1      1     1.00                        vandps	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vandps	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vandps	(%rax){1to8}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vandps	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vandps	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vandps	(%rax){1to8}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      4     1.00                        vcvtqq2pd	%xmm16, %xmm19
+# CHECK-NEXT:  3      10    1.00    *                   vcvtqq2pd	(%rax), %xmm19
+# CHECK-NEXT:  3      10    1.00    *                   vcvtqq2pd	(%rax){1to2}, %xmm19
+# CHECK-NEXT:  2      4     1.00                        vcvtqq2pd	%xmm16, %xmm19 {%k1}
+# CHECK-NEXT:  3      10    1.00    *                   vcvtqq2pd	(%rax), %xmm19 {%k1}
+# CHECK-NEXT:  3      10    1.00    *                   vcvtqq2pd	(%rax){1to2}, %xmm19 {%k1}
+# CHECK-NEXT:  2      4     1.00                        vcvtqq2pd	%xmm16, %xmm19 {%k1} {z}
+# CHECK-NEXT:  3      10    1.00    *                   vcvtqq2pd	(%rax), %xmm19 {%k1} {z}
+# CHECK-NEXT:  3      10    1.00    *                   vcvtqq2pd	(%rax){1to2}, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      4     1.00                        vcvtqq2pd	%ymm16, %ymm19
+# CHECK-NEXT:  3      10    1.00    *                   vcvtqq2pd	(%rax), %ymm19
+# CHECK-NEXT:  3      10    1.00    *                   vcvtqq2pd	(%rax){1to4}, %ymm19
+# CHECK-NEXT:  2      4     1.00                        vcvtqq2pd	%ymm16, %ymm19 {%k1}
+# CHECK-NEXT:  3      10    1.00    *                   vcvtqq2pd	(%rax), %ymm19 {%k1}
+# CHECK-NEXT:  3      10    1.00    *                   vcvtqq2pd	(%rax){1to4}, %ymm19 {%k1}
+# CHECK-NEXT:  2      4     1.00                        vcvtqq2pd	%ymm16, %ymm19 {%k1} {z}
+# CHECK-NEXT:  3      10    1.00    *                   vcvtqq2pd	(%rax), %ymm19 {%k1} {z}
+# CHECK-NEXT:  3      10    1.00    *                   vcvtqq2pd	(%rax){1to4}, %ymm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vorpd	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  2      7     1.00    *                   vorpd	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  2      7     1.00    *                   vorpd	(%rax){1to2}, %xmm17, %xmm19
+# CHECK-NEXT:  1      1     1.00                        vorpd	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      7     1.00    *                   vorpd	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      7     1.00    *                   vorpd	(%rax){1to2}, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vorpd	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      7     1.00    *                   vorpd	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      7     1.00    *                   vorpd	(%rax){1to2}, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vorpd	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  2      8     1.00    *                   vorpd	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  2      8     1.00    *                   vorpd	(%rax){1to4}, %ymm17, %ymm19
+# CHECK-NEXT:  1      1     1.00                        vorpd	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vorpd	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vorpd	(%rax){1to4}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vorpd	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vorpd	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vorpd	(%rax){1to4}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vorps	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  2      7     1.00    *                   vorps	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  2      7     1.00    *                   vorps	(%rax){1to4}, %xmm17, %xmm19
+# CHECK-NEXT:  1      1     1.00                        vorps	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      7     1.00    *                   vorps	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      7     1.00    *                   vorps	(%rax){1to4}, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vorps	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      7     1.00    *                   vorps	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      7     1.00    *                   vorps	(%rax){1to4}, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vorps	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  2      8     1.00    *                   vorps	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  2      8     1.00    *                   vorps	(%rax){1to8}, %ymm17, %ymm19
+# CHECK-NEXT:  1      1     1.00                        vorps	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vorps	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vorps	(%rax){1to8}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vorps	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vorps	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vorps	(%rax){1to8}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  1      5     1.00                        vpmullq	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  2      11    1.00    *                   vpmullq	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  1      5     1.00                        vpmullq	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      11    1.00    *                   vpmullq	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  1      5     1.00                        vpmullq	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      11    1.00    *                   vpmullq	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  1      5     1.00                        vpmullq	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  2      12    1.00    *                   vpmullq	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  1      5     1.00                        vpmullq	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      12    1.00    *                   vpmullq	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  1      5     1.00                        vpmullq	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      12    1.00    *                   vpmullq	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vxorpd	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  2      7     1.00    *                   vxorpd	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  2      7     1.00    *                   vxorpd	(%rax){1to2}, %xmm17, %xmm19
+# CHECK-NEXT:  1      1     1.00                        vxorpd	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      7     1.00    *                   vxorpd	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      7     1.00    *                   vxorpd	(%rax){1to2}, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vxorpd	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      7     1.00    *                   vxorpd	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      7     1.00    *                   vxorpd	(%rax){1to2}, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vxorpd	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  2      8     1.00    *                   vxorpd	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  2      8     1.00    *                   vxorpd	(%rax){1to4}, %ymm17, %ymm19
+# CHECK-NEXT:  1      1     1.00                        vxorpd	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vxorpd	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vxorpd	(%rax){1to4}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vxorpd	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vxorpd	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vxorpd	(%rax){1to4}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vxorps	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  2      7     1.00    *                   vxorps	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  2      7     1.00    *                   vxorps	(%rax){1to4}, %xmm17, %xmm19
+# CHECK-NEXT:  1      1     1.00                        vxorps	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      7     1.00    *                   vxorps	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      7     1.00    *                   vxorps	(%rax){1to4}, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vxorps	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      7     1.00    *                   vxorps	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      7     1.00    *                   vxorps	(%rax){1to4}, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vxorps	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  2      8     1.00    *                   vxorps	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  2      8     1.00    *                   vxorps	(%rax){1to8}, %ymm17, %ymm19
+# CHECK-NEXT:  1      1     1.00                        vxorps	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vxorps	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vxorps	(%rax){1to8}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vxorps	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vxorps	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vxorps	(%rax){1to8}, %ymm17, %ymm19 {%k1} {z}
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0]   - SBDivider
+# CHECK-NEXT: [1]   - SBFPDivider
+# CHECK-NEXT: [2]   - SBPort0
+# CHECK-NEXT: [3]   - SBPort1
+# CHECK-NEXT: [4]   - SBPort4
+# CHECK-NEXT: [5]   - SBPort5
+# CHECK-NEXT: [6.0] - SBPort23
+# CHECK-NEXT: [6.1] - SBPort23
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6.0]  [6.1]
+# CHECK-NEXT:  -      -     12.00  18.00   -     144.00 51.00  51.00
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6.0]  [6.1]  Instructions:
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vandnpd	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vandnpd	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vandnpd	(%rax){1to2}, %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vandnpd	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vandnpd	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vandnpd	(%rax){1to2}, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vandnpd	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vandnpd	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vandnpd	(%rax){1to2}, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vandnpd	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vandnpd	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vandnpd	(%rax){1to4}, %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vandnpd	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vandnpd	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vandnpd	(%rax){1to4}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vandnpd	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vandnpd	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vandnpd	(%rax){1to4}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vandpd	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vandpd	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vandpd	(%rax){1to2}, %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vandpd	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vandpd	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vandpd	(%rax){1to2}, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vandpd	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vandpd	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vandpd	(%rax){1to2}, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vandpd	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vandpd	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vandpd	(%rax){1to4}, %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vandpd	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vandpd	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vandpd	(%rax){1to4}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vandpd	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vandpd	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vandpd	(%rax){1to4}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vandps	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vandps	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vandps	(%rax){1to4}, %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vandps	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vandps	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vandps	(%rax){1to4}, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vandps	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vandps	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vandps	(%rax){1to4}, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vandps	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vandps	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vandps	(%rax){1to8}, %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vandps	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vandps	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vandps	(%rax){1to8}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vandps	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vandps	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vandps	(%rax){1to8}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     1.00    -     1.00    -      -     vcvtqq2pd	%xmm16, %xmm19
+# CHECK-NEXT:  -      -      -     1.00    -     1.00   0.50   0.50   vcvtqq2pd	(%rax), %xmm19
+# CHECK-NEXT:  -      -      -     1.00    -     1.00   0.50   0.50   vcvtqq2pd	(%rax){1to2}, %xmm19
+# CHECK-NEXT:  -      -      -     1.00    -     1.00    -      -     vcvtqq2pd	%xmm16, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -     1.00    -     1.00   0.50   0.50   vcvtqq2pd	(%rax), %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -     1.00    -     1.00   0.50   0.50   vcvtqq2pd	(%rax){1to2}, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -     1.00    -     1.00    -      -     vcvtqq2pd	%xmm16, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     1.00    -     1.00   0.50   0.50   vcvtqq2pd	(%rax), %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     1.00    -     1.00   0.50   0.50   vcvtqq2pd	(%rax){1to2}, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     1.00    -     1.00    -      -     vcvtqq2pd	%ymm16, %ymm19
+# CHECK-NEXT:  -      -      -     1.00    -     1.00   0.50   0.50   vcvtqq2pd	(%rax), %ymm19
+# CHECK-NEXT:  -      -      -     1.00    -     1.00   0.50   0.50   vcvtqq2pd	(%rax){1to4}, %ymm19
+# CHECK-NEXT:  -      -      -     1.00    -     1.00    -      -     vcvtqq2pd	%ymm16, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -     1.00    -     1.00   0.50   0.50   vcvtqq2pd	(%rax), %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -     1.00    -     1.00   0.50   0.50   vcvtqq2pd	(%rax){1to4}, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -     1.00    -     1.00    -      -     vcvtqq2pd	%ymm16, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     1.00    -     1.00   0.50   0.50   vcvtqq2pd	(%rax), %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     1.00    -     1.00   0.50   0.50   vcvtqq2pd	(%rax){1to4}, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vorpd	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vorpd	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vorpd	(%rax){1to2}, %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vorpd	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vorpd	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vorpd	(%rax){1to2}, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vorpd	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vorpd	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vorpd	(%rax){1to2}, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vorpd	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vorpd	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vorpd	(%rax){1to4}, %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vorpd	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vorpd	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vorpd	(%rax){1to4}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vorpd	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vorpd	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vorpd	(%rax){1to4}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vorps	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vorps	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vorps	(%rax){1to4}, %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vorps	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vorps	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vorps	(%rax){1to4}, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vorps	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vorps	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vorps	(%rax){1to4}, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vorps	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vorps	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vorps	(%rax){1to8}, %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vorps	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vorps	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vorps	(%rax){1to8}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vorps	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vorps	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vorps	(%rax){1to8}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vpmullq	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  -      -     1.00    -      -      -     0.50   0.50   vpmullq	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vpmullq	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -     1.00    -      -      -     0.50   0.50   vpmullq	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vpmullq	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     1.00    -      -      -     0.50   0.50   vpmullq	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vpmullq	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  -      -     1.00    -      -      -     0.50   0.50   vpmullq	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vpmullq	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -     1.00    -      -      -     0.50   0.50   vpmullq	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vpmullq	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -     1.00    -      -      -     0.50   0.50   vpmullq	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vxorpd	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vxorpd	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vxorpd	(%rax){1to2}, %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vxorpd	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vxorpd	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vxorpd	(%rax){1to2}, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vxorpd	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vxorpd	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vxorpd	(%rax){1to2}, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vxorpd	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vxorpd	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vxorpd	(%rax){1to4}, %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vxorpd	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vxorpd	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vxorpd	(%rax){1to4}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vxorpd	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vxorpd	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vxorpd	(%rax){1to4}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vxorps	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vxorps	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vxorps	(%rax){1to4}, %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vxorps	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vxorps	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vxorps	(%rax){1to4}, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vxorps	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vxorps	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vxorps	(%rax){1to4}, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vxorps	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vxorps	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vxorps	(%rax){1to8}, %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vxorps	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vxorps	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vxorps	(%rax){1to8}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vxorps	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vxorps	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vxorps	(%rax){1to8}, %ymm17, %ymm19 {%k1} {z}
diff --git a/test/tools/llvm-mca/X86/Generic/resources-avx512vl.s b/test/tools/llvm-mca/X86/Generic/resources-avx512vl.s
index 7bfcd8a5db2f0204fc552fbc18e0d4e260176c1f..6ac3448c01953d9247a3e158df40f23390c09c9f 100644
--- a/test/tools/llvm-mca/X86/Generic/resources-avx512vl.s
+++ b/test/tools/llvm-mca/X86/Generic/resources-avx512vl.s
@@ -3,115 +3,743 @@
 
 vaddpd            %xmm16, %xmm17, %xmm19
 vaddpd            (%rax), %xmm17, %xmm19
+vaddpd            (%rax){1to2}, %xmm17, %xmm19
 vaddpd            %xmm16, %xmm17, %xmm19 {k1}
 vaddpd            (%rax), %xmm17, %xmm19 {k1}
+vaddpd            (%rax){1to2}, %xmm17, %xmm19 {k1}
 vaddpd            %xmm16, %xmm17, %xmm19 {z}{k1}
 vaddpd            (%rax), %xmm17, %xmm19 {z}{k1}
+vaddpd            (%rax){1to2}, %xmm17, %xmm19 {z}{k1}
 
 vaddpd            %ymm16, %ymm17, %ymm19
 vaddpd            (%rax), %ymm17, %ymm19
+vaddpd            (%rax){1to4}, %ymm17, %ymm19
 vaddpd            %ymm16, %ymm17, %ymm19 {k1}
 vaddpd            (%rax), %ymm17, %ymm19 {k1}
+vaddpd            (%rax){1to4}, %ymm17, %ymm19 {k1}
 vaddpd            %ymm16, %ymm17, %ymm19 {z}{k1}
 vaddpd            (%rax), %ymm17, %ymm19 {z}{k1}
+vaddpd            (%rax){1to4}, %ymm17, %ymm19 {z}{k1}
 
 vaddps            %xmm16, %xmm17, %xmm19
 vaddps            (%rax), %xmm17, %xmm19
+vaddps            (%rax){1to4}, %xmm17, %xmm19
 vaddps            %xmm16, %xmm17, %xmm19 {k1}
 vaddps            (%rax), %xmm17, %xmm19 {k1}
+vaddps            (%rax){1to4}, %xmm17, %xmm19 {k1}
 vaddps            %xmm16, %xmm17, %xmm19 {z}{k1}
 vaddps            (%rax), %xmm17, %xmm19 {z}{k1}
+vaddps            (%rax){1to4}, %xmm17, %xmm19 {z}{k1}
 
 vaddps            %ymm16, %ymm17, %ymm19
 vaddps            (%rax), %ymm17, %ymm19
+vaddps            (%rax){1to8}, %ymm17, %ymm19
 vaddps            %ymm16, %ymm17, %ymm19 {k1}
 vaddps            (%rax), %ymm17, %ymm19 {k1}
+vaddps            (%rax){1to8}, %ymm17, %ymm19 {k1}
 vaddps            %ymm16, %ymm17, %ymm19 {z}{k1}
 vaddps            (%rax), %ymm17, %ymm19 {z}{k1}
+vaddps            (%rax){1to8}, %ymm17, %ymm19 {z}{k1}
+
+vcvtdq2pd         %xmm16, %xmm19
+vcvtdq2pd         (%rax), %xmm19
+vcvtdq2pd         (%rax){1to2}, %xmm19
+vcvtdq2pd         %xmm16, %xmm19 {k1}
+vcvtdq2pd         (%rax), %xmm19 {k1}
+vcvtdq2pd         (%rax){1to2}, %xmm19 {k1}
+vcvtdq2pd         %xmm16, %xmm19 {z}{k1}
+vcvtdq2pd         (%rax), %xmm19 {z}{k1}
+vcvtdq2pd         (%rax){1to2}, %xmm19 {z}{k1}
+
+vcvtdq2pd         %xmm16, %ymm19
+vcvtdq2pd         (%rax), %ymm19
+vcvtdq2pd         (%rax){1to4}, %ymm19
+vcvtdq2pd         %xmm16, %ymm19 {k1}
+vcvtdq2pd         (%rax), %ymm19 {k1}
+vcvtdq2pd         (%rax){1to4}, %ymm19 {k1}
+vcvtdq2pd         %xmm16, %ymm19 {z}{k1}
+vcvtdq2pd         (%rax), %ymm19 {z}{k1}
+vcvtdq2pd         (%rax){1to4}, %ymm19 {z}{k1}
+
+vcvtdq2ps         %xmm16, %xmm19
+vcvtdq2ps         (%rax), %xmm19
+vcvtdq2ps         (%rax){1to4}, %xmm19
+vcvtdq2ps         %xmm16, %xmm19 {k1}
+vcvtdq2ps         (%rax), %xmm19 {k1}
+vcvtdq2ps         (%rax){1to4},%xmm19 {k1}
+vcvtdq2ps         %xmm16, %xmm19 {z}{k1}
+vcvtdq2ps         (%rax), %xmm19 {z}{k1}
+vcvtdq2ps         (%rax){1to4}, %xmm19 {z}{k1}
+
+vcvtdq2ps         %ymm16, %ymm19
+vcvtdq2ps         (%rax), %ymm19
+vcvtdq2ps         (%rax){1to8}, %ymm19
+vcvtdq2ps         %ymm16,%ymm19 {k1}
+vcvtdq2ps         (%rax),%ymm19 {k1}
+vcvtdq2ps         (%rax){1to8}, %ymm19 {k1}
+vcvtdq2ps         %ymm16, %ymm19 {z}{k1}
+vcvtdq2ps         (%rax), %ymm19 {z}{k1}
+vcvtdq2ps         (%rax){1to8}, %ymm19 {z}{k1}
 
 vdivpd            %xmm16, %xmm17, %xmm19
 vdivpd            (%rax), %xmm17, %xmm19
+vdivpd            (%rax){1to2}, %xmm17, %xmm19
 vdivpd            %xmm16, %xmm17, %xmm19 {k1}
 vdivpd            (%rax), %xmm17, %xmm19 {k1}
+vdivpd            (%rax){1to2}, %xmm17, %xmm19 {k1}
 vdivpd            %xmm16, %xmm17, %xmm19 {z}{k1}
 vdivpd            (%rax), %xmm17, %xmm19 {z}{k1}
+vdivpd            (%rax){1to2}, %xmm17, %xmm19 {z}{k1}
 
 vdivpd            %ymm16, %ymm17, %ymm19
 vdivpd            (%rax), %ymm17, %ymm19
+vdivpd            (%rax){1to4}, %ymm17, %ymm19
 vdivpd            %ymm16, %ymm17, %ymm19 {k1}
 vdivpd            (%rax), %ymm17, %ymm19 {k1}
+vdivpd            (%rax){1to4}, %ymm17, %ymm19 {k1}
 vdivpd            %ymm16, %ymm17, %ymm19 {z}{k1}
 vdivpd            (%rax), %ymm17, %ymm19 {z}{k1}
+vdivpd            (%rax){1to4}, %ymm17, %ymm19 {z}{k1}
 
 vdivps            %xmm16, %xmm17, %xmm19
 vdivps            (%rax), %xmm17, %xmm19
+vdivps            (%rax){1to4}, %xmm17, %xmm19
 vdivps            %xmm16, %xmm17, %xmm19 {k1}
 vdivps            (%rax), %xmm17, %xmm19 {k1}
+vdivps            (%rax){1to4}, %xmm17, %xmm19 {k1}
 vdivps            %xmm16, %xmm17, %xmm19 {z}{k1}
 vdivps            (%rax), %xmm17, %xmm19 {z}{k1}
+vdivps            (%rax){1to4}, %xmm17, %xmm19 {z}{k1}
 
 vdivps            %ymm16, %ymm17, %ymm19
 vdivps            (%rax), %ymm17, %ymm19
+vdivps            (%rax){1to8}, %ymm17, %ymm19
 vdivps            %ymm16, %ymm17, %ymm19 {k1}
 vdivps            (%rax), %ymm17, %ymm19 {k1}
+vdivps            (%rax){1to8}, %ymm17, %ymm19 {k1}
 vdivps            %ymm16, %ymm17, %ymm19 {z}{k1}
 vdivps            (%rax), %ymm17, %ymm19 {z}{k1}
+vdivps            (%rax){1to8}, %ymm17, %ymm19 {z}{k1}
+
+vmaxpd            %xmm16, %xmm17, %xmm19
+vmaxpd            (%rax), %xmm17, %xmm19
+vmaxpd            (%rax){1to2}, %xmm17, %xmm19
+vmaxpd            %xmm16, %xmm17, %xmm19 {k1}
+vmaxpd            (%rax), %xmm17, %xmm19 {k1}
+vmaxpd            (%rax){1to2}, %xmm17, %xmm19 {k1}
+vmaxpd            %xmm16, %xmm17, %xmm19 {z}{k1}
+vmaxpd            (%rax), %xmm17, %xmm19 {z}{k1}
+vmaxpd            (%rax){1to2}, %xmm17, %xmm19 {z}{k1}
+
+vmaxpd            %ymm16, %ymm17, %ymm19
+vmaxpd            (%rax), %ymm17, %ymm19
+vmaxpd            (%rax){1to4}, %ymm17, %ymm19
+vmaxpd            %ymm16, %ymm17, %ymm19 {k1}
+vmaxpd            (%rax), %ymm17, %ymm19 {k1}
+vmaxpd            (%rax){1to4}, %ymm17, %ymm19 {k1}
+vmaxpd            %ymm16, %ymm17, %ymm19 {z}{k1}
+vmaxpd            (%rax), %ymm17, %ymm19 {z}{k1}
+vmaxpd            (%rax){1to4}, %ymm17, %ymm19 {z}{k1}
+
+vmaxps            %xmm16, %xmm17, %xmm19
+vmaxps            (%rax), %xmm17, %xmm19
+vmaxps            (%rax){1to4}, %xmm17, %xmm19
+vmaxps            %xmm16, %xmm17, %xmm19 {k1}
+vmaxps            (%rax), %xmm17, %xmm19 {k1}
+vmaxps            (%rax){1to4}, %xmm17, %xmm19 {k1}
+vmaxps            %xmm16, %xmm17, %xmm19 {z}{k1}
+vmaxps            (%rax), %xmm17, %xmm19 {z}{k1}
+vmaxps            (%rax){1to4}, %xmm17, %xmm19 {z}{k1}
+
+vmaxps            %ymm16, %ymm17, %ymm19
+vmaxps            (%rax), %ymm17, %ymm19
+vmaxps            (%rax){1to8}, %ymm17, %ymm19
+vmaxps            %ymm16, %ymm17, %ymm19 {k1}
+vmaxps            (%rax), %ymm17, %ymm19 {k1}
+vmaxps            (%rax){1to8}, %ymm17, %ymm19 {k1}
+vmaxps            %ymm16, %ymm17, %ymm19 {z}{k1}
+vmaxps            (%rax), %ymm17, %ymm19 {z}{k1}
+vmaxps            (%rax){1to8}, %ymm17, %ymm19 {z}{k1}
+
+vminpd            %xmm16, %xmm17, %xmm19
+vminpd            (%rax), %xmm17, %xmm19
+vminpd            (%rax){1to2}, %xmm17, %xmm19
+vminpd            %xmm16, %xmm17, %xmm19 {k1}
+vminpd            (%rax), %xmm17, %xmm19 {k1}
+vminpd            (%rax){1to2}, %xmm17, %xmm19 {k1}
+vminpd            %xmm16, %xmm17, %xmm19 {z}{k1}
+vminpd            (%rax), %xmm17, %xmm19 {z}{k1}
+vminpd            (%rax){1to2}, %xmm17, %xmm19 {z}{k1}
+
+vminpd            %ymm16, %ymm17, %ymm19
+vminpd            (%rax), %ymm17, %ymm19
+vminpd            (%rax){1to4}, %ymm17, %ymm19
+vminpd            %ymm16, %ymm17, %ymm19 {k1}
+vminpd            (%rax), %ymm17, %ymm19 {k1}
+vminpd            (%rax){1to4}, %ymm17, %ymm19 {k1}
+vminpd            %ymm16, %ymm17, %ymm19 {z}{k1}
+vminpd            (%rax), %ymm17, %ymm19 {z}{k1}
+vminpd            (%rax){1to4}, %ymm17, %ymm19 {z}{k1}
+
+vminps            %xmm16, %xmm17, %xmm19
+vminps            (%rax), %xmm17, %xmm19
+vminps            (%rax){1to4}, %xmm17, %xmm19
+vminps            %xmm16, %xmm17, %xmm19 {k1}
+vminps            (%rax), %xmm17, %xmm19 {k1}
+vminps            (%rax){1to4}, %xmm17, %xmm19 {k1}
+vminps            %xmm16, %xmm17, %xmm19 {z}{k1}
+vminps            (%rax), %xmm17, %xmm19 {z}{k1}
+vminps            (%rax){1to4}, %xmm17, %xmm19 {z}{k1}
+
+vminps            %ymm16, %ymm17, %ymm19
+vminps            (%rax), %ymm17, %ymm19
+vminps            (%rax){1to8}, %ymm17, %ymm19
+vminps            %ymm16, %ymm17, %ymm19 {k1}
+vminps            (%rax), %ymm17, %ymm19 {k1}
+vminps            (%rax){1to8}, %ymm17, %ymm19 {k1}
+vminps            %ymm16, %ymm17, %ymm19 {z}{k1}
+vminps            (%rax), %ymm17, %ymm19 {z}{k1}
+vminps            (%rax){1to8}, %ymm17, %ymm19 {z}{k1}
 
 vmulpd            %xmm16, %xmm17, %xmm19
 vmulpd            (%rax), %xmm17, %xmm19
+vmulpd            (%rax){1to2}, %xmm17, %xmm19
 vmulpd            %xmm16, %xmm17, %xmm19 {k1}
 vmulpd            (%rax), %xmm17, %xmm19 {k1}
+vmulpd            (%rax){1to2}, %xmm17, %xmm19 {k1}
 vmulpd            %xmm16, %xmm17, %xmm19 {z}{k1}
 vmulpd            (%rax), %xmm17, %xmm19 {z}{k1}
+vmulpd            (%rax){1to2}, %xmm17, %xmm19 {z}{k1}
 
 vmulpd            %ymm16, %ymm17, %ymm19
 vmulpd            (%rax), %ymm17, %ymm19
+vmulpd            (%rax){1to4}, %ymm17, %ymm19
 vmulpd            %ymm16, %ymm17, %ymm19 {k1}
 vmulpd            (%rax), %ymm17, %ymm19 {k1}
+vmulpd            (%rax){1to4}, %ymm17, %ymm19 {k1}
 vmulpd            %ymm16, %ymm17, %ymm19 {z}{k1}
 vmulpd            (%rax), %ymm17, %ymm19 {z}{k1}
+vmulpd            (%rax){1to4}, %ymm17, %ymm19 {z}{k1}
 
 vmulps            %xmm16, %xmm17, %xmm19
 vmulps            (%rax), %xmm17, %xmm19
+vmulps            (%rax){1to4}, %xmm17, %xmm19
 vmulps            %xmm16, %xmm17, %xmm19 {k1}
 vmulps            (%rax), %xmm17, %xmm19 {k1}
+vmulps            (%rax){1to4}, %xmm17, %xmm19 {k1}
 vmulps            %xmm16, %xmm17, %xmm19 {z}{k1}
 vmulps            (%rax), %xmm17, %xmm19 {z}{k1}
+vmulps            (%rax){1to4}, %xmm17, %xmm19 {z}{k1}
 
 vmulps            %ymm16, %ymm17, %ymm19
 vmulps            (%rax), %ymm17, %ymm19
+vmulps            (%rax){1to8}, %ymm17, %ymm19
 vmulps            %ymm16, %ymm17, %ymm19 {k1}
 vmulps            (%rax), %ymm17, %ymm19 {k1}
+vmulps            (%rax){1to8}, %ymm17, %ymm19 {k1}
 vmulps            %ymm16, %ymm17, %ymm19 {z}{k1}
 vmulps            (%rax), %ymm17, %ymm19 {z}{k1}
+vmulps            (%rax){1to8}, %ymm17, %ymm19 {z}{k1}
+
+vpaddd            %xmm16, %xmm17, %xmm19
+vpaddd            (%rax), %xmm17, %xmm19
+vpaddd            (%rax){1to4}, %xmm17, %xmm19
+vpaddd            %xmm16, %xmm17, %xmm19 {k1}
+vpaddd            (%rax), %xmm17, %xmm19 {k1}
+vpaddd            (%rax){1to4}, %xmm17, %xmm19 {k1}
+vpaddd            %xmm16, %xmm17, %xmm19 {z}{k1}
+vpaddd            (%rax), %xmm17, %xmm19 {z}{k1}
+vpaddd            (%rax){1to4}, %xmm17, %xmm19 {z}{k1}
+
+vpaddd            %ymm16, %ymm17, %ymm19
+vpaddd            (%rax), %ymm17, %ymm19
+vpaddd            (%rax){1to8}, %ymm17, %ymm19
+vpaddd            %ymm16, %ymm17, %ymm19 {k1}
+vpaddd            (%rax), %ymm17, %ymm19 {k1}
+vpaddd            (%rax){1to8}, %ymm17, %ymm19 {k1}
+vpaddd            %ymm16, %ymm17, %ymm19 {z}{k1}
+vpaddd            (%rax), %ymm17, %ymm19 {z}{k1}
+vpaddd            (%rax){1to8}, %ymm17, %ymm19 {z}{k1}
+
+vpaddq            %xmm16, %xmm17, %xmm19
+vpaddq            (%rax), %xmm17, %xmm19
+vpaddq            (%rax){1to2}, %xmm17, %xmm19
+vpaddq            %xmm16, %xmm17, %xmm19 {k1}
+vpaddq            (%rax), %xmm17, %xmm19 {k1}
+vpaddq            (%rax){1to2}, %xmm17, %xmm19 {k1}
+vpaddq            %xmm16, %xmm17, %xmm19 {z}{k1}
+vpaddq            (%rax), %xmm17, %xmm19 {z}{k1}
+vpaddq            (%rax){1to2}, %xmm17, %xmm19 {z}{k1}
+
+vpaddq            %ymm16, %ymm17, %ymm19
+vpaddq            (%rax), %ymm17, %ymm19
+vpaddq            (%rax){1to4}, %ymm17, %ymm19
+vpaddq            %ymm16, %ymm17, %ymm19 {k1}
+vpaddq            (%rax), %ymm17, %ymm19 {k1}
+vpaddq            (%rax){1to4}, %ymm17, %ymm19 {k1}
+vpaddq            %ymm16, %ymm17, %ymm19 {z}{k1}
+vpaddq            (%rax), %ymm17, %ymm19 {z}{k1}
+vpaddq            (%rax){1to4}, %ymm17, %ymm19 {z}{k1}
+
+vpermd            %ymm16, %ymm17, %ymm19
+vpermd            (%rax), %ymm17, %ymm19
+vpermd            (%rax){1to8}, %ymm17, %ymm19
+vpermd            %ymm16, %ymm17, %ymm19 {k1}
+vpermd            (%rax), %ymm17, %ymm19 {k1}
+vpermd            (%rax){1to8}, %ymm17, %ymm19 {k1}
+vpermd            %ymm16, %ymm17, %ymm19 {z}{k1}
+vpermd            (%rax), %ymm17, %ymm19 {z}{k1}
+vpermd            (%rax){1to8}, %ymm17, %ymm19 {z}{k1}
+
+vpermilpd         $0, %xmm16, %xmm19
+vpermilpd         $0, (%rax), %xmm19
+vpermilpd         $0, (%rax){1to2}, %xmm19
+vpermilpd         $0, %xmm16, %xmm19 {k1}
+vpermilpd         $0, (%rax), %xmm19 {k1}
+vpermilpd         $0, (%rax){1to2}, %xmm19 {k1}
+vpermilpd         $0, %xmm16, %xmm19 {z}{k1}
+vpermilpd         $0, (%rax), %xmm19 {z}{k1}
+vpermilpd         $0, (%rax){1to2}, %xmm19 {z}{k1}
+
+vpermilpd         $0, %ymm16, %ymm19
+vpermilpd         $0, (%rax), %ymm19
+vpermilpd         $0, (%rax){1to4}, %ymm19
+vpermilpd         $0, %ymm16, %ymm19 {k1}
+vpermilpd         $0, (%rax), %ymm19 {k1}
+vpermilpd         $0, (%rax){1to4}, %ymm19 {k1}
+vpermilpd         $0, %ymm16, %ymm19 {z}{k1}
+vpermilpd         $0, (%rax), %ymm19 {z}{k1}
+vpermilpd         $0, (%rax){1to4}, %ymm19 {z}{k1}
+
+vpermilpd         %xmm16, %xmm17, %xmm19
+vpermilpd         (%rax), %xmm17, %xmm19
+vpermilpd         (%rax){1to2}, %xmm17, %xmm19
+vpermilpd         %xmm16, %xmm17, %xmm19 {k1}
+vpermilpd         (%rax), %xmm17, %xmm19 {k1}
+vpermilpd         (%rax){1to2}, %xmm17, %xmm19 {k1}
+vpermilpd         %xmm16, %xmm17, %xmm19 {z}{k1}
+vpermilpd         (%rax), %xmm17, %xmm19 {z}{k1}
+vpermilpd         (%rax){1to2}, %xmm17, %xmm19 {z}{k1}
+
+vpermilpd         %ymm16, %ymm17, %ymm19
+vpermilpd         (%rax), %ymm17, %ymm19
+vpermilpd         (%rax){1to4}, %ymm17, %ymm19
+vpermilpd         %ymm16, %ymm17, %ymm19 {k1}
+vpermilpd         (%rax), %ymm17, %ymm19 {k1}
+vpermilpd         (%rax){1to4}, %ymm17, %ymm19 {k1}
+vpermilpd         %ymm16, %ymm17, %ymm19 {z}{k1}
+vpermilpd         (%rax), %ymm17, %ymm19 {z}{k1}
+vpermilpd         (%rax){1to4}, %ymm17, %ymm19 {z}{k1}
+
+vpermilps         $0, %xmm16, %xmm19
+vpermilps         $0, (%rax), %xmm19
+vpermilps         $0, (%rax){1to4}, %xmm19
+vpermilps         $0, %xmm16, %xmm19 {k1}
+vpermilps         $0, (%rax), %xmm19 {k1}
+vpermilps         $0, (%rax){1to4}, %xmm19 {k1}
+vpermilps         $0, %xmm16, %xmm19 {z}{k1}
+vpermilps         $0, (%rax), %xmm19 {z}{k1}
+vpermilps         $0, (%rax){1to4}, %xmm19 {z}{k1}
+
+vpermilps         $0, %ymm16, %ymm19
+vpermilps         $0, (%rax), %ymm19
+vpermilps         $0, (%rax){1to8}, %ymm19
+vpermilps         $0, %ymm16, %ymm19 {k1}
+vpermilps         $0, (%rax), %ymm19 {k1}
+vpermilps         $0, (%rax){1to8}, %ymm19 {k1}
+vpermilps         $0, %ymm16, %ymm19 {z}{k1}
+vpermilps         $0, (%rax), %ymm19 {z}{k1}
+vpermilps         $0, (%rax){1to8}, %ymm19 {z}{k1}
+
+vpermilps         %xmm16, %xmm17, %xmm19
+vpermilps         (%rax), %xmm17, %xmm19
+vpermilps         (%rax){1to4}, %xmm17, %xmm19
+vpermilps         %xmm16, %xmm17, %xmm19 {k1}
+vpermilps         (%rax), %xmm17, %xmm19 {k1}
+vpermilps         (%rax){1to4}, %xmm17, %xmm19 {k1}
+vpermilps         %xmm16, %xmm17, %xmm19 {z}{k1}
+vpermilps         (%rax), %xmm17, %xmm19 {z}{k1}
+vpermilps         (%rax){1to4}, %xmm17, %xmm19 {z}{k1}
+
+vpermilps         %ymm16, %ymm17, %ymm19
+vpermilps         (%rax), %ymm17, %ymm19
+vpermilps         (%rax){1to8}, %ymm17, %ymm19
+vpermilps         %ymm16, %ymm17, %ymm19 {k1}
+vpermilps         (%rax), %ymm17, %ymm19 {k1}
+vpermilps         (%rax){1to8}, %ymm17, %ymm19 {k1}
+vpermilps         %ymm16, %ymm17, %ymm19 {z}{k1}
+vpermilps         (%rax), %ymm17, %ymm19 {z}{k1}
+vpermilps         (%rax){1to8}, %ymm17, %ymm19 {z}{k1}
+
+vpermpd           $0, %ymm16, %ymm19
+vpermpd           $0, (%rax), %ymm19
+vpermpd           $0, (%rax){1to4}, %ymm19
+vpermpd           $0, %ymm16, %ymm19 {k1}
+vpermpd           $0, (%rax), %ymm19 {k1}
+vpermpd           $0, (%rax){1to4}, %ymm19 {k1}
+vpermpd           $0, %ymm16, %ymm19 {z}{k1}
+vpermpd           $0, (%rax), %ymm19 {z}{k1}
+vpermpd           $0, (%rax){1to4}, %ymm19 {z}{k1}
+
+vpermpd           %ymm16, %ymm17, %ymm19
+vpermpd           (%rax), %ymm17, %ymm19
+vpermpd           (%rax){1to4}, %ymm17, %ymm19
+vpermpd           %ymm16, %ymm17, %ymm19 {k1}
+vpermpd           (%rax), %ymm17, %ymm19 {k1}
+vpermpd           (%rax){1to4}, %ymm17, %ymm19 {k1}
+vpermpd           %ymm16, %ymm17, %ymm19 {z}{k1}
+vpermpd           (%rax), %ymm17, %ymm19 {z}{k1}
+vpermpd           (%rax){1to4}, %ymm17, %ymm19 {z}{k1}
+
+vpermps           %ymm16, %ymm17, %ymm19
+vpermps           (%rax), %ymm17, %ymm19
+vpermps           (%rax){1to8}, %ymm17, %ymm19
+vpermps           %ymm16, %ymm17, %ymm19 {k1}
+vpermps           (%rax), %ymm17, %ymm19 {k1}
+vpermps           (%rax){1to8}, %ymm17, %ymm19 {k1}
+vpermps           %ymm16, %ymm17, %ymm19 {z}{k1}
+vpermps           (%rax), %ymm17, %ymm19 {z}{k1}
+vpermps           (%rax){1to8}, %ymm17, %ymm19 {z}{k1}
+
+vpermq            $0, %ymm16, %ymm19
+vpermq            $0, (%rax), %ymm19
+vpermq            $0, (%rax){1to4}, %ymm19
+vpermq            $0, %ymm16, %ymm19 {k1}
+vpermq            $0, (%rax), %ymm19 {k1}
+vpermq            $0, (%rax){1to4}, %ymm19 {k1}
+vpermq            $0, %ymm16, %ymm19 {z}{k1}
+vpermq            $0, (%rax), %ymm19 {z}{k1}
+vpermq            $0, (%rax){1to4}, %ymm19 {z}{k1}
+
+vpermq            %ymm16, %ymm17, %ymm19
+vpermq            (%rax), %ymm17, %ymm19
+vpermq            (%rax){1to4}, %ymm17, %ymm19
+vpermq            %ymm16, %ymm17, %ymm19 {k1}
+vpermq            (%rax), %ymm17, %ymm19 {k1}
+vpermq            (%rax){1to4}, %ymm17, %ymm19 {k1}
+vpermq            %ymm16, %ymm17, %ymm19 {z}{k1}
+vpermq            (%rax), %ymm17, %ymm19 {z}{k1}
+vpermq            (%rax){1to4}, %ymm17, %ymm19 {z}{k1}
+
+vpmulld           %xmm16, %xmm17, %xmm19
+vpmulld           (%rax), %xmm17, %xmm19
+vpmulld           (%rax){1to4}, %xmm17, %xmm19
+vpmulld           %xmm16, %xmm17, %xmm19 {k1}
+vpmulld           (%rax), %xmm17, %xmm19 {k1}
+vpmulld           (%rax){1to4}, %xmm17, %xmm19 {k1}
+vpmulld           %xmm16, %xmm17, %xmm19 {z}{k1}
+vpmulld           (%rax), %xmm17, %xmm19 {z}{k1}
+vpmulld           (%rax){1to4}, %xmm17, %xmm19 {z}{k1}
+
+vpmulld           %ymm16, %ymm17, %ymm19
+vpmulld           (%rax), %ymm17, %ymm19
+vpmulld           (%rax){1to8}, %ymm17, %ymm19
+vpmulld           %ymm16, %ymm17, %ymm19 {k1}
+vpmulld           (%rax), %ymm17, %ymm19 {k1}
+vpmulld           (%rax){1to8}, %ymm17, %ymm19 {k1}
+vpmulld           %ymm16, %ymm17, %ymm19 {z}{k1}
+vpmulld           (%rax), %ymm17, %ymm19 {z}{k1}
+vpmulld           (%rax){1to8}, %ymm17, %ymm19 {z}{k1}
+
+vpshufd           $0, %xmm16, %xmm19
+vpshufd           $0, (%rax), %xmm19
+vpshufd           $0, (%rax){1to4}, %xmm19
+vpshufd           $0, %xmm16, %xmm19 {k1}
+vpshufd           $0, (%rax), %xmm19 {k1}
+vpshufd           $0, (%rax){1to4}, %xmm19 {k1}
+vpshufd           $0, %xmm16, %xmm19 {z}{k1}
+vpshufd           $0, (%rax), %xmm19 {z}{k1}
+vpshufd           $0, (%rax){1to4}, %xmm19 {z}{k1}
+
+vpshufd           $0, %ymm16, %ymm19
+vpshufd           $0, (%rax), %ymm19
+vpshufd           $0, (%rax){1to8}, %ymm19
+vpshufd           $0, %ymm16, %ymm19 {k1}
+vpshufd           $0, (%rax), %ymm19 {k1}
+vpshufd           $0, (%rax){1to8}, %ymm19 {k1}
+vpshufd           $0, %ymm16, %ymm19 {z}{k1}
+vpshufd           $0, (%rax), %ymm19 {z}{k1}
+vpshufd           $0, (%rax){1to8}, %ymm19 {z}{k1}
+
+vpsubd            %xmm16, %xmm17, %xmm19
+vpsubd            (%rax), %xmm17, %xmm19
+vpsubd            (%rax){1to4}, %xmm17, %xmm19
+vpsubd            %xmm16, %xmm17, %xmm19 {k1}
+vpsubd            (%rax), %xmm17, %xmm19 {k1}
+vpsubd            (%rax){1to4}, %xmm17, %xmm19 {k1}
+vpsubd            %xmm16, %xmm17, %xmm19 {z}{k1}
+vpsubd            (%rax), %xmm17, %xmm19 {z}{k1}
+vpsubd            (%rax){1to4}, %xmm17, %xmm19 {z}{k1}
+
+vpsubd            %ymm16, %ymm17, %ymm19
+vpsubd            (%rax), %ymm17, %ymm19
+vpsubd            (%rax){1to8}, %ymm17, %ymm19
+vpsubd            %ymm16, %ymm17, %ymm19 {k1}
+vpsubd            (%rax), %ymm17, %ymm19 {k1}
+vpsubd            (%rax){1to8}, %ymm17, %ymm19 {k1}
+vpsubd            %ymm16, %ymm17, %ymm19 {z}{k1}
+vpsubd            (%rax), %ymm17, %ymm19 {z}{k1}
+vpsubd            (%rax){1to8}, %ymm17, %ymm19 {z}{k1}
+
+vpsubq            %xmm16, %xmm17, %xmm19
+vpsubq            (%rax), %xmm17, %xmm19
+vpsubq            (%rax){1to2}, %xmm17, %xmm19
+vpsubq            %xmm16, %xmm17, %xmm19 {k1}
+vpsubq            (%rax), %xmm17, %xmm19 {k1}
+vpsubq            (%rax){1to2}, %xmm17, %xmm19 {k1}
+vpsubq            %xmm16, %xmm17, %xmm19 {z}{k1}
+vpsubq            (%rax), %xmm17, %xmm19 {z}{k1}
+vpsubq            (%rax){1to2}, %xmm17, %xmm19 {z}{k1}
+
+vpsubq            %ymm16, %ymm17, %ymm19
+vpsubq            (%rax), %ymm17, %ymm19
+vpsubq            (%rax){1to4}, %ymm17, %ymm19
+vpsubq            %ymm16, %ymm17, %ymm19 {k1}
+vpsubq            (%rax), %ymm17, %ymm19 {k1}
+vpsubq            (%rax){1to4}, %ymm17, %ymm19 {k1}
+vpsubq            %ymm16, %ymm17, %ymm19 {z}{k1}
+vpsubq            (%rax), %ymm17, %ymm19 {z}{k1}
+vpsubq            (%rax){1to4}, %ymm17, %ymm19 {z}{k1}
+
+vpunpckhdq        %xmm16, %xmm17, %xmm19
+vpunpckhdq        (%rax), %xmm17, %xmm19
+vpunpckhdq        (%rax){1to4}, %xmm17, %xmm19
+vpunpckhdq        %xmm16, %xmm17, %xmm19 {k1}
+vpunpckhdq        (%rax), %xmm17, %xmm19 {k1}
+vpunpckhdq        (%rax){1to4}, %xmm17, %xmm19 {k1}
+vpunpckhdq        %xmm16, %xmm17, %xmm19 {z}{k1}
+vpunpckhdq        (%rax), %xmm17, %xmm19 {z}{k1}
+vpunpckhdq        (%rax){1to4}, %xmm17, %xmm19 {z}{k1}
+
+vpunpckhdq        %ymm16, %ymm17, %ymm19
+vpunpckhdq        (%rax), %ymm17, %ymm19
+vpunpckhdq        (%rax){1to8}, %ymm17, %ymm19
+vpunpckhdq        %ymm16, %ymm17, %ymm19 {k1}
+vpunpckhdq        (%rax), %ymm17, %ymm19 {k1}
+vpunpckhdq        (%rax){1to8}, %ymm17, %ymm19 {k1}
+vpunpckhdq        %ymm16, %ymm17, %ymm19 {z}{k1}
+vpunpckhdq        (%rax), %ymm17, %ymm19 {z}{k1}
+vpunpckhdq        (%rax){1to8}, %ymm17, %ymm19 {z}{k1}
+
+vpunpckldq        %xmm16, %xmm17, %xmm19
+vpunpckldq        (%rax), %xmm17, %xmm19
+vpunpckldq        (%rax){1to4}, %xmm17, %xmm19
+vpunpckldq        %xmm16, %xmm17, %xmm19 {k1}
+vpunpckldq        (%rax), %xmm17, %xmm19 {k1}
+vpunpckldq        (%rax){1to4}, %xmm17, %xmm19 {k1}
+vpunpckldq        %xmm16, %xmm17, %xmm19 {z}{k1}
+vpunpckldq        (%rax), %xmm17, %xmm19 {z}{k1}
+vpunpckldq        (%rax){1to4}, %xmm17, %xmm19 {z}{k1}
+
+vpunpckldq        %ymm16, %ymm17, %ymm19
+vpunpckldq        (%rax), %ymm17, %ymm19
+vpunpckldq        (%rax){1to8}, %ymm17, %ymm19
+vpunpckldq        %ymm16, %ymm17, %ymm19 {k1}
+vpunpckldq        (%rax), %ymm17, %ymm19 {k1}
+vpunpckldq        (%rax){1to8}, %ymm17, %ymm19 {k1}
+vpunpckldq        %ymm16, %ymm17, %ymm19 {z}{k1}
+vpunpckldq        (%rax), %ymm17, %ymm19 {z}{k1}
+vpunpckldq        (%rax){1to8}, %ymm17, %ymm19 {z}{k1}
+
+vshuff32x4        $0, %ymm16, %ymm17, %ymm19
+vshuff32x4        $0, (%rax), %ymm17, %ymm19
+vshuff32x4        $0, (%rax){1to8}, %ymm17, %ymm19
+vshuff32x4        $0, %ymm16, %ymm17, %ymm19 {k1}
+vshuff32x4        $0, (%rax), %ymm17, %ymm19 {k1}
+vshuff32x4        $0, (%rax){1to8}, %ymm17, %ymm19 {k1}
+vshuff32x4        $0, %ymm16, %ymm17, %ymm19 {z}{k1}
+vshuff32x4        $0, (%rax), %ymm17, %ymm19 {z}{k1}
+vshuff32x4        $0, (%rax){1to8}, %ymm17, %ymm19 {z}{k1}
+
+vshuff64x2        $0, %ymm16, %ymm17, %ymm19
+vshuff64x2        $0, (%rax), %ymm17, %ymm19
+vshuff64x2        $0, (%rax){1to4}, %ymm17, %ymm19
+vshuff64x2        $0, %ymm16, %ymm17, %ymm19 {k1}
+vshuff64x2        $0, (%rax), %ymm17, %ymm19 {k1}
+vshuff64x2        $0, (%rax){1to4}, %ymm17, %ymm19 {k1}
+vshuff64x2        $0, %ymm16, %ymm17, %ymm19 {z}{k1}
+vshuff64x2        $0, (%rax), %ymm17, %ymm19 {z}{k1}
+vshuff64x2        $0, (%rax){1to4}, %ymm17, %ymm19 {z}{k1}
+
+vshufi32x4        $0, %ymm16, %ymm17, %ymm19
+vshufi32x4        $0, (%rax), %ymm17, %ymm19
+vshufi32x4        $0, (%rax){1to8}, %ymm17, %ymm19
+vshufi32x4        $0, %ymm16, %ymm17, %ymm19 {k1}
+vshufi32x4        $0, (%rax), %ymm17, %ymm19 {k1}
+vshufi32x4        $0, (%rax){1to8}, %ymm17, %ymm19 {k1}
+vshufi32x4        $0, %ymm16, %ymm17, %ymm19 {z}{k1}
+vshufi32x4        $0, (%rax), %ymm17, %ymm19 {z}{k1}
+vshufi32x4        $0, (%rax){1to8}, %ymm17, %ymm19 {z}{k1}
+
+vshufi64x2        $0, %ymm16, %ymm17, %ymm19
+vshufi64x2        $0, (%rax), %ymm17, %ymm19
+vshufi64x2        $0, (%rax){1to4}, %ymm17, %ymm19
+vshufi64x2        $0, %ymm16, %ymm17, %ymm19 {k1}
+vshufi64x2        $0, (%rax), %ymm17, %ymm19 {k1}
+vshufi64x2        $0, (%rax){1to4}, %ymm17, %ymm19 {k1}
+vshufi64x2        $0, %ymm16, %ymm17, %ymm19 {z}{k1}
+vshufi64x2        $0, (%rax), %ymm17, %ymm19 {z}{k1}
+vshufi64x2        $0, (%rax){1to4}, %ymm17, %ymm19 {z}{k1}
+
+vsqrtpd           %xmm16, %xmm19
+vsqrtpd           (%rax), %xmm19
+vsqrtpd           (%rax){1to2}, %xmm19
+vsqrtpd           %xmm16, %xmm19 {k1}
+vsqrtpd           (%rax), %xmm19 {k1}
+vsqrtpd           (%rax){1to2}, %xmm19 {k1}
+vsqrtpd           %xmm16, %xmm19 {z}{k1}
+vsqrtpd           (%rax), %xmm19 {z}{k1}
+vsqrtpd           (%rax){1to2}, %xmm19 {z}{k1}
+
+vsqrtpd           %ymm16, %ymm19
+vsqrtpd           (%rax), %ymm19
+vsqrtpd           (%rax){1to4}, %ymm19
+vsqrtpd           %ymm16, %ymm19 {k1}
+vsqrtpd           (%rax), %ymm19 {k1}
+vsqrtpd           (%rax){1to4}, %ymm19 {k1}
+vsqrtpd           %ymm16, %ymm19 {z}{k1}
+vsqrtpd           (%rax), %ymm19 {z}{k1}
+vsqrtpd           (%rax){1to4}, %ymm19 {z}{k1}
+
+vsqrtps           %xmm16, %xmm19
+vsqrtps           (%rax), %xmm19
+vsqrtps           (%rax){1to4}, %xmm19
+vsqrtps           %xmm16, %xmm19 {k1}
+vsqrtps           (%rax), %xmm19 {k1}
+vsqrtps           (%rax){1to4}, %xmm19 {k1}
+vsqrtps           %xmm16, %xmm19 {z}{k1}
+vsqrtps           (%rax), %xmm19 {z}{k1}
+vsqrtps           (%rax){1to4}, %xmm19 {z}{k1}
+
+vsqrtps           %ymm16, %ymm19
+vsqrtps           (%rax), %ymm19
+vsqrtps           (%rax){1to8}, %ymm19
+vsqrtps           %ymm16, %ymm19 {k1}
+vsqrtps           (%rax), %ymm19 {k1}
+vsqrtps           (%rax){1to8}, %ymm19 {k1}
+vsqrtps           %ymm16, %ymm19 {z}{k1}
+vsqrtps           (%rax), %ymm19 {z}{k1}
+vsqrtps           (%rax){1to8}, %ymm19 {z}{k1}
 
 vsubpd            %xmm16, %xmm17, %xmm19
 vsubpd            (%rax), %xmm17, %xmm19
+vsubpd            (%rax){1to2}, %xmm17, %xmm19
 vsubpd            %xmm16, %xmm17, %xmm19 {k1}
 vsubpd            (%rax), %xmm17, %xmm19 {k1}
+vsubpd            (%rax){1to2}, %xmm17, %xmm19 {k1}
 vsubpd            %xmm16, %xmm17, %xmm19 {z}{k1}
 vsubpd            (%rax), %xmm17, %xmm19 {z}{k1}
+vsubpd            (%rax){1to2}, %xmm17, %xmm19 {z}{k1}
 
 vsubpd            %ymm16, %ymm17, %ymm19
 vsubpd            (%rax), %ymm17, %ymm19
+vsubpd            (%rax){1to4}, %ymm17, %ymm19
 vsubpd            %ymm16, %ymm17, %ymm19 {k1}
 vsubpd            (%rax), %ymm17, %ymm19 {k1}
+vsubpd            (%rax){1to4}, %ymm17, %ymm19 {k1}
 vsubpd            %ymm16, %ymm17, %ymm19 {z}{k1}
 vsubpd            (%rax), %ymm17, %ymm19 {z}{k1}
+vsubpd            (%rax){1to4}, %ymm17, %ymm19 {z}{k1}
 
 vsubps            %xmm16, %xmm17, %xmm19
 vsubps            (%rax), %xmm17, %xmm19
+vsubps            (%rax){1to4}, %xmm17, %xmm19
 vsubps            %xmm16, %xmm17, %xmm19 {k1}
 vsubps            (%rax), %xmm17, %xmm19 {k1}
+vsubps            (%rax){1to4}, %xmm17, %xmm19 {k1}
 vsubps            %xmm16, %xmm17, %xmm19 {z}{k1}
 vsubps            (%rax), %xmm17, %xmm19 {z}{k1}
+vsubps            (%rax){1to4}, %xmm17, %xmm19 {z}{k1}
 
 vsubps            %ymm16, %ymm17, %ymm19
 vsubps            (%rax), %ymm17, %ymm19
+vsubps            (%rax){1to8}, %ymm17, %ymm19
 vsubps            %ymm16, %ymm17, %ymm19 {k1}
 vsubps            (%rax), %ymm17, %ymm19 {k1}
+vsubps            (%rax){1to8}, %ymm17, %ymm19 {k1}
 vsubps            %ymm16, %ymm17, %ymm19 {z}{k1}
 vsubps            (%rax), %ymm17, %ymm19 {z}{k1}
+vsubps            (%rax){1to8}, %ymm17, %ymm19 {z}{k1}
+
+vunpckhpd         %xmm16, %xmm17, %xmm19
+vunpckhpd         (%rax), %xmm17, %xmm19
+vunpckhpd         (%rax){1to2}, %xmm17, %xmm19
+vunpckhpd         %xmm16, %xmm17, %xmm19 {k1}
+vunpckhpd         (%rax), %xmm17, %xmm19 {k1}
+vunpckhpd         (%rax){1to2}, %xmm17, %xmm19 {k1}
+vunpckhpd         %xmm16, %xmm17, %xmm19 {z}{k1}
+vunpckhpd         (%rax), %xmm17, %xmm19 {z}{k1}
+vunpckhpd         (%rax){1to2}, %xmm17, %xmm19 {z}{k1}
+
+vunpckhpd         %ymm16, %ymm17, %ymm19
+vunpckhpd         (%rax), %ymm17, %ymm19
+vunpckhpd         (%rax){1to4}, %ymm17, %ymm19
+vunpckhpd         %ymm16, %ymm17, %ymm19 {k1}
+vunpckhpd         (%rax), %ymm17, %ymm19 {k1}
+vunpckhpd         (%rax){1to4}, %ymm17, %ymm19 {k1}
+vunpckhpd         %ymm16, %ymm17, %ymm19 {z}{k1}
+vunpckhpd         (%rax), %ymm17, %ymm19 {z}{k1}
+vunpckhpd         (%rax){1to4}, %ymm17, %ymm19 {z}{k1}
+
+vunpckhps         %xmm16, %xmm17, %xmm19
+vunpckhps         (%rax), %xmm17, %xmm19
+vunpckhps         (%rax){1to4}, %xmm17, %xmm19
+vunpckhps         %xmm16, %xmm17, %xmm19 {k1}
+vunpckhps         (%rax), %xmm17, %xmm19 {k1}
+vunpckhps         (%rax){1to4}, %xmm17, %xmm19 {k1}
+vunpckhps         %xmm16, %xmm17, %xmm19 {z}{k1}
+vunpckhps         (%rax), %xmm17, %xmm19 {z}{k1}
+vunpckhps         (%rax){1to4}, %xmm17, %xmm19 {z}{k1}
+
+vunpckhps         %ymm16, %ymm17, %ymm19
+vunpckhps         (%rax), %ymm17, %ymm19
+vunpckhps         (%rax){1to8}, %ymm17, %ymm19
+vunpckhps         %ymm16, %ymm17, %ymm19 {k1}
+vunpckhps         (%rax), %ymm17, %ymm19 {k1}
+vunpckhps         (%rax){1to8}, %ymm17, %ymm19 {k1}
+vunpckhps         %ymm16, %ymm17, %ymm19 {z}{k1}
+vunpckhps         (%rax), %ymm17, %ymm19 {z}{k1}
+vunpckhps         (%rax){1to8}, %ymm17, %ymm19 {z}{k1}
+
+vunpcklpd         %xmm16, %xmm17, %xmm19
+vunpcklpd         (%rax), %xmm17, %xmm19
+vunpcklpd         (%rax){1to2}, %xmm17, %xmm19
+vunpcklpd         %xmm16, %xmm17, %xmm19 {k1}
+vunpcklpd         (%rax), %xmm17, %xmm19 {k1}
+vunpcklpd         (%rax){1to2}, %xmm17, %xmm19 {k1}
+vunpcklpd         %xmm16, %xmm17, %xmm19 {z}{k1}
+vunpcklpd         (%rax), %xmm17, %xmm19 {z}{k1}
+vunpcklpd         (%rax){1to2}, %xmm17, %xmm19 {z}{k1}
+
+vunpcklpd         %ymm16, %ymm17, %ymm19
+vunpcklpd         (%rax), %ymm17, %ymm19
+vunpcklpd         (%rax){1to4}, %ymm17, %ymm19
+vunpcklpd         %ymm16, %ymm17, %ymm19 {k1}
+vunpcklpd         (%rax), %ymm17, %ymm19 {k1}
+vunpcklpd         (%rax){1to4}, %ymm17, %ymm19 {k1}
+vunpcklpd         %ymm16, %ymm17, %ymm19 {z}{k1}
+vunpcklpd         (%rax), %ymm17, %ymm19 {z}{k1}
+vunpcklpd         (%rax){1to4}, %ymm17, %ymm19 {z}{k1}
+
+vunpcklps         %xmm16, %xmm17, %xmm19
+vunpcklps         (%rax), %xmm17, %xmm19
+vunpcklps         (%rax){1to4}, %xmm17, %xmm19
+vunpcklps         %xmm16, %xmm17, %xmm19 {k1}
+vunpcklps         (%rax), %xmm17, %xmm19 {k1}
+vunpcklps         (%rax){1to4}, %xmm17, %xmm19 {k1}
+vunpcklps         %xmm16, %xmm17, %xmm19 {z}{k1}
+vunpcklps         (%rax), %xmm17, %xmm19 {z}{k1}
+vunpcklps         (%rax){1to4}, %xmm17, %xmm19 {z}{k1}
+
+vunpcklps         %ymm16, %ymm17, %ymm19
+vunpcklps         (%rax), %ymm17, %ymm19
+vunpcklps         (%rax){1to8}, %ymm17, %ymm19
+vunpcklps         %ymm16, %ymm17, %ymm19 {k1}
+vunpcklps         (%rax), %ymm17, %ymm19 {k1}
+vunpcklps         (%rax){1to8}, %ymm17, %ymm19 {k1}
+vunpcklps         %ymm16, %ymm17, %ymm19 {z}{k1}
+vunpcklps         (%rax), %ymm17, %ymm19 {z}{k1}
+vunpcklps         (%rax){1to8}, %ymm17, %ymm19 {z}{k1}
 
 # CHECK:      Instruction Info:
 # CHECK-NEXT: [1]: #uOps
@@ -124,100 +752,670 @@ vsubps            (%rax), %ymm17, %ymm19 {z}{k1}
 # CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
 # CHECK-NEXT:  1      3     1.00                        vaddpd	%xmm16, %xmm17, %xmm19
 # CHECK-NEXT:  2      9     1.00    *                   vaddpd	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  2      9     1.00    *                   vaddpd	(%rax){1to2}, %xmm17, %xmm19
 # CHECK-NEXT:  1      3     1.00                        vaddpd	%xmm16, %xmm17, %xmm19 {%k1}
 # CHECK-NEXT:  2      9     1.00    *                   vaddpd	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      9     1.00    *                   vaddpd	(%rax){1to2}, %xmm17, %xmm19 {%k1}
 # CHECK-NEXT:  1      3     1.00                        vaddpd	%xmm16, %xmm17, %xmm19 {%k1} {z}
 # CHECK-NEXT:  2      9     1.00    *                   vaddpd	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      9     1.00    *                   vaddpd	(%rax){1to2}, %xmm17, %xmm19 {%k1} {z}
 # CHECK-NEXT:  1      3     1.00                        vaddpd	%ymm16, %ymm17, %ymm19
 # CHECK-NEXT:  2      10    1.00    *                   vaddpd	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  2      10    1.00    *                   vaddpd	(%rax){1to4}, %ymm17, %ymm19
 # CHECK-NEXT:  1      3     1.00                        vaddpd	%ymm16, %ymm17, %ymm19 {%k1}
 # CHECK-NEXT:  2      10    1.00    *                   vaddpd	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      10    1.00    *                   vaddpd	(%rax){1to4}, %ymm17, %ymm19 {%k1}
 # CHECK-NEXT:  1      3     1.00                        vaddpd	%ymm16, %ymm17, %ymm19 {%k1} {z}
 # CHECK-NEXT:  2      10    1.00    *                   vaddpd	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      10    1.00    *                   vaddpd	(%rax){1to4}, %ymm17, %ymm19 {%k1} {z}
 # CHECK-NEXT:  1      3     1.00                        vaddps	%xmm16, %xmm17, %xmm19
 # CHECK-NEXT:  2      9     1.00    *                   vaddps	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  2      9     1.00    *                   vaddps	(%rax){1to4}, %xmm17, %xmm19
 # CHECK-NEXT:  1      3     1.00                        vaddps	%xmm16, %xmm17, %xmm19 {%k1}
 # CHECK-NEXT:  2      9     1.00    *                   vaddps	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      9     1.00    *                   vaddps	(%rax){1to4}, %xmm17, %xmm19 {%k1}
 # CHECK-NEXT:  1      3     1.00                        vaddps	%xmm16, %xmm17, %xmm19 {%k1} {z}
 # CHECK-NEXT:  2      9     1.00    *                   vaddps	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      9     1.00    *                   vaddps	(%rax){1to4}, %xmm17, %xmm19 {%k1} {z}
 # CHECK-NEXT:  1      3     1.00                        vaddps	%ymm16, %ymm17, %ymm19
 # CHECK-NEXT:  2      10    1.00    *                   vaddps	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  2      10    1.00    *                   vaddps	(%rax){1to8}, %ymm17, %ymm19
 # CHECK-NEXT:  1      3     1.00                        vaddps	%ymm16, %ymm17, %ymm19 {%k1}
 # CHECK-NEXT:  2      10    1.00    *                   vaddps	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      10    1.00    *                   vaddps	(%rax){1to8}, %ymm17, %ymm19 {%k1}
 # CHECK-NEXT:  1      3     1.00                        vaddps	%ymm16, %ymm17, %ymm19 {%k1} {z}
 # CHECK-NEXT:  2      10    1.00    *                   vaddps	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      10    1.00    *                   vaddps	(%rax){1to8}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      4     1.00                        vcvtdq2pd	%xmm16, %xmm19
+# CHECK-NEXT:  3      10    1.00    *                   vcvtdq2pd	(%rax), %xmm19
+# CHECK-NEXT:  3      10    1.00    *                   vcvtdq2pd	(%rax){1to2}, %xmm19
+# CHECK-NEXT:  2      4     1.00                        vcvtdq2pd	%xmm16, %xmm19 {%k1}
+# CHECK-NEXT:  3      10    1.00    *                   vcvtdq2pd	(%rax), %xmm19 {%k1}
+# CHECK-NEXT:  3      10    1.00    *                   vcvtdq2pd	(%rax){1to2}, %xmm19 {%k1}
+# CHECK-NEXT:  2      4     1.00                        vcvtdq2pd	%xmm16, %xmm19 {%k1} {z}
+# CHECK-NEXT:  3      10    1.00    *                   vcvtdq2pd	(%rax), %xmm19 {%k1} {z}
+# CHECK-NEXT:  3      10    1.00    *                   vcvtdq2pd	(%rax){1to2}, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      4     1.00                        vcvtdq2pd	%xmm16, %ymm19
+# CHECK-NEXT:  3      10    1.00    *                   vcvtdq2pd	(%rax), %ymm19
+# CHECK-NEXT:  3      10    1.00    *                   vcvtdq2pd	(%rax){1to4}, %ymm19
+# CHECK-NEXT:  2      4     1.00                        vcvtdq2pd	%xmm16, %ymm19 {%k1}
+# CHECK-NEXT:  3      10    1.00    *                   vcvtdq2pd	(%rax), %ymm19 {%k1}
+# CHECK-NEXT:  3      10    1.00    *                   vcvtdq2pd	(%rax){1to4}, %ymm19 {%k1}
+# CHECK-NEXT:  2      4     1.00                        vcvtdq2pd	%xmm16, %ymm19 {%k1} {z}
+# CHECK-NEXT:  3      10    1.00    *                   vcvtdq2pd	(%rax), %ymm19 {%k1} {z}
+# CHECK-NEXT:  3      10    1.00    *                   vcvtdq2pd	(%rax){1to4}, %ymm19 {%k1} {z}
+# CHECK-NEXT:  1      3     1.00                        vcvtdq2ps	%xmm16, %xmm19
+# CHECK-NEXT:  2      9     1.00    *                   vcvtdq2ps	(%rax), %xmm19
+# CHECK-NEXT:  2      9     1.00    *                   vcvtdq2ps	(%rax){1to4}, %xmm19
+# CHECK-NEXT:  1      3     1.00                        vcvtdq2ps	%xmm16, %xmm19 {%k1}
+# CHECK-NEXT:  2      9     1.00    *                   vcvtdq2ps	(%rax), %xmm19 {%k1}
+# CHECK-NEXT:  2      9     1.00    *                   vcvtdq2ps	(%rax){1to4}, %xmm19 {%k1}
+# CHECK-NEXT:  1      3     1.00                        vcvtdq2ps	%xmm16, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      9     1.00    *                   vcvtdq2ps	(%rax), %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      9     1.00    *                   vcvtdq2ps	(%rax){1to4}, %xmm19 {%k1} {z}
+# CHECK-NEXT:  1      3     1.00                        vcvtdq2ps	%ymm16, %ymm19
+# CHECK-NEXT:  2      10    1.00    *                   vcvtdq2ps	(%rax), %ymm19
+# CHECK-NEXT:  2      10    1.00    *                   vcvtdq2ps	(%rax){1to8}, %ymm19
+# CHECK-NEXT:  1      3     1.00                        vcvtdq2ps	%ymm16, %ymm19 {%k1}
+# CHECK-NEXT:  2      10    1.00    *                   vcvtdq2ps	(%rax), %ymm19 {%k1}
+# CHECK-NEXT:  2      10    1.00    *                   vcvtdq2ps	(%rax){1to8}, %ymm19 {%k1}
+# CHECK-NEXT:  1      3     1.00                        vcvtdq2ps	%ymm16, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      10    1.00    *                   vcvtdq2ps	(%rax), %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      10    1.00    *                   vcvtdq2ps	(%rax){1to8}, %ymm19 {%k1} {z}
 # CHECK-NEXT:  1      22    22.00                       vdivpd	%xmm16, %xmm17, %xmm19
 # CHECK-NEXT:  2      28    22.00   *                   vdivpd	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  2      28    22.00   *                   vdivpd	(%rax){1to2}, %xmm17, %xmm19
 # CHECK-NEXT:  1      22    22.00                       vdivpd	%xmm16, %xmm17, %xmm19 {%k1}
 # CHECK-NEXT:  2      28    22.00   *                   vdivpd	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      28    22.00   *                   vdivpd	(%rax){1to2}, %xmm17, %xmm19 {%k1}
 # CHECK-NEXT:  1      22    22.00                       vdivpd	%xmm16, %xmm17, %xmm19 {%k1} {z}
 # CHECK-NEXT:  2      28    22.00   *                   vdivpd	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      28    22.00   *                   vdivpd	(%rax){1to2}, %xmm17, %xmm19 {%k1} {z}
 # CHECK-NEXT:  3      45    44.00                       vdivpd	%ymm16, %ymm17, %ymm19
 # CHECK-NEXT:  4      52    44.00   *                   vdivpd	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  4      52    44.00   *                   vdivpd	(%rax){1to4}, %ymm17, %ymm19
 # CHECK-NEXT:  3      45    44.00                       vdivpd	%ymm16, %ymm17, %ymm19 {%k1}
 # CHECK-NEXT:  4      52    44.00   *                   vdivpd	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  4      52    44.00   *                   vdivpd	(%rax){1to4}, %ymm17, %ymm19 {%k1}
 # CHECK-NEXT:  3      45    44.00                       vdivpd	%ymm16, %ymm17, %ymm19 {%k1} {z}
 # CHECK-NEXT:  4      52    44.00   *                   vdivpd	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  4      52    44.00   *                   vdivpd	(%rax){1to4}, %ymm17, %ymm19 {%k1} {z}
 # CHECK-NEXT:  1      14    14.00                       vdivps	%xmm16, %xmm17, %xmm19
 # CHECK-NEXT:  2      20    14.00   *                   vdivps	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  2      20    14.00   *                   vdivps	(%rax){1to4}, %xmm17, %xmm19
 # CHECK-NEXT:  1      14    14.00                       vdivps	%xmm16, %xmm17, %xmm19 {%k1}
 # CHECK-NEXT:  2      20    14.00   *                   vdivps	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      20    14.00   *                   vdivps	(%rax){1to4}, %xmm17, %xmm19 {%k1}
 # CHECK-NEXT:  1      14    14.00                       vdivps	%xmm16, %xmm17, %xmm19 {%k1} {z}
 # CHECK-NEXT:  2      20    14.00   *                   vdivps	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      20    14.00   *                   vdivps	(%rax){1to4}, %xmm17, %xmm19 {%k1} {z}
 # CHECK-NEXT:  3      29    28.00                       vdivps	%ymm16, %ymm17, %ymm19
 # CHECK-NEXT:  4      36    28.00   *                   vdivps	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  4      36    28.00   *                   vdivps	(%rax){1to8}, %ymm17, %ymm19
 # CHECK-NEXT:  3      29    28.00                       vdivps	%ymm16, %ymm17, %ymm19 {%k1}
 # CHECK-NEXT:  4      36    28.00   *                   vdivps	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  4      36    28.00   *                   vdivps	(%rax){1to8}, %ymm17, %ymm19 {%k1}
 # CHECK-NEXT:  3      29    28.00                       vdivps	%ymm16, %ymm17, %ymm19 {%k1} {z}
 # CHECK-NEXT:  4      36    28.00   *                   vdivps	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  4      36    28.00   *                   vdivps	(%rax){1to8}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  1      3     1.00                        vmaxpd	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  2      9     1.00    *                   vmaxpd	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  2      9     1.00    *                   vmaxpd	(%rax){1to2}, %xmm17, %xmm19
+# CHECK-NEXT:  1      3     1.00                        vmaxpd	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      9     1.00    *                   vmaxpd	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      9     1.00    *                   vmaxpd	(%rax){1to2}, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  1      3     1.00                        vmaxpd	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      9     1.00    *                   vmaxpd	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      9     1.00    *                   vmaxpd	(%rax){1to2}, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  1      3     1.00                        vmaxpd	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  2      10    1.00    *                   vmaxpd	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  2      10    1.00    *                   vmaxpd	(%rax){1to4}, %ymm17, %ymm19
+# CHECK-NEXT:  1      3     1.00                        vmaxpd	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      10    1.00    *                   vmaxpd	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      10    1.00    *                   vmaxpd	(%rax){1to4}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  1      3     1.00                        vmaxpd	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      10    1.00    *                   vmaxpd	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      10    1.00    *                   vmaxpd	(%rax){1to4}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  1      3     1.00                        vmaxps	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  2      9     1.00    *                   vmaxps	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  2      9     1.00    *                   vmaxps	(%rax){1to4}, %xmm17, %xmm19
+# CHECK-NEXT:  1      3     1.00                        vmaxps	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      9     1.00    *                   vmaxps	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      9     1.00    *                   vmaxps	(%rax){1to4}, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  1      3     1.00                        vmaxps	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      9     1.00    *                   vmaxps	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      9     1.00    *                   vmaxps	(%rax){1to4}, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  1      3     1.00                        vmaxps	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  2      10    1.00    *                   vmaxps	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  2      10    1.00    *                   vmaxps	(%rax){1to8}, %ymm17, %ymm19
+# CHECK-NEXT:  1      3     1.00                        vmaxps	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      10    1.00    *                   vmaxps	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      10    1.00    *                   vmaxps	(%rax){1to8}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  1      3     1.00                        vmaxps	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      10    1.00    *                   vmaxps	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      10    1.00    *                   vmaxps	(%rax){1to8}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  1      3     1.00                        vminpd	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  2      9     1.00    *                   vminpd	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  2      9     1.00    *                   vminpd	(%rax){1to2}, %xmm17, %xmm19
+# CHECK-NEXT:  1      3     1.00                        vminpd	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      9     1.00    *                   vminpd	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      9     1.00    *                   vminpd	(%rax){1to2}, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  1      3     1.00                        vminpd	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      9     1.00    *                   vminpd	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      9     1.00    *                   vminpd	(%rax){1to2}, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  1      3     1.00                        vminpd	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  2      10    1.00    *                   vminpd	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  2      10    1.00    *                   vminpd	(%rax){1to4}, %ymm17, %ymm19
+# CHECK-NEXT:  1      3     1.00                        vminpd	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      10    1.00    *                   vminpd	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      10    1.00    *                   vminpd	(%rax){1to4}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  1      3     1.00                        vminpd	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      10    1.00    *                   vminpd	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      10    1.00    *                   vminpd	(%rax){1to4}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  1      3     1.00                        vminps	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  2      9     1.00    *                   vminps	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  2      9     1.00    *                   vminps	(%rax){1to4}, %xmm17, %xmm19
+# CHECK-NEXT:  1      3     1.00                        vminps	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      9     1.00    *                   vminps	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      9     1.00    *                   vminps	(%rax){1to4}, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  1      3     1.00                        vminps	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      9     1.00    *                   vminps	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      9     1.00    *                   vminps	(%rax){1to4}, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  1      3     1.00                        vminps	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  2      10    1.00    *                   vminps	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  2      10    1.00    *                   vminps	(%rax){1to8}, %ymm17, %ymm19
+# CHECK-NEXT:  1      3     1.00                        vminps	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      10    1.00    *                   vminps	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      10    1.00    *                   vminps	(%rax){1to8}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  1      3     1.00                        vminps	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      10    1.00    *                   vminps	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      10    1.00    *                   vminps	(%rax){1to8}, %ymm17, %ymm19 {%k1} {z}
 # CHECK-NEXT:  1      5     1.00                        vmulpd	%xmm16, %xmm17, %xmm19
 # CHECK-NEXT:  2      11    1.00    *                   vmulpd	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  2      11    1.00    *                   vmulpd	(%rax){1to2}, %xmm17, %xmm19
 # CHECK-NEXT:  1      5     1.00                        vmulpd	%xmm16, %xmm17, %xmm19 {%k1}
 # CHECK-NEXT:  2      11    1.00    *                   vmulpd	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      11    1.00    *                   vmulpd	(%rax){1to2}, %xmm17, %xmm19 {%k1}
 # CHECK-NEXT:  1      5     1.00                        vmulpd	%xmm16, %xmm17, %xmm19 {%k1} {z}
 # CHECK-NEXT:  2      11    1.00    *                   vmulpd	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      11    1.00    *                   vmulpd	(%rax){1to2}, %xmm17, %xmm19 {%k1} {z}
 # CHECK-NEXT:  1      5     1.00                        vmulpd	%ymm16, %ymm17, %ymm19
 # CHECK-NEXT:  2      12    1.00    *                   vmulpd	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  2      12    1.00    *                   vmulpd	(%rax){1to4}, %ymm17, %ymm19
 # CHECK-NEXT:  1      5     1.00                        vmulpd	%ymm16, %ymm17, %ymm19 {%k1}
 # CHECK-NEXT:  2      12    1.00    *                   vmulpd	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      12    1.00    *                   vmulpd	(%rax){1to4}, %ymm17, %ymm19 {%k1}
 # CHECK-NEXT:  1      5     1.00                        vmulpd	%ymm16, %ymm17, %ymm19 {%k1} {z}
 # CHECK-NEXT:  2      12    1.00    *                   vmulpd	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      12    1.00    *                   vmulpd	(%rax){1to4}, %ymm17, %ymm19 {%k1} {z}
 # CHECK-NEXT:  1      5     1.00                        vmulps	%xmm16, %xmm17, %xmm19
 # CHECK-NEXT:  2      11    1.00    *                   vmulps	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  2      11    1.00    *                   vmulps	(%rax){1to4}, %xmm17, %xmm19
 # CHECK-NEXT:  1      5     1.00                        vmulps	%xmm16, %xmm17, %xmm19 {%k1}
 # CHECK-NEXT:  2      11    1.00    *                   vmulps	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      11    1.00    *                   vmulps	(%rax){1to4}, %xmm17, %xmm19 {%k1}
 # CHECK-NEXT:  1      5     1.00                        vmulps	%xmm16, %xmm17, %xmm19 {%k1} {z}
 # CHECK-NEXT:  2      11    1.00    *                   vmulps	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      11    1.00    *                   vmulps	(%rax){1to4}, %xmm17, %xmm19 {%k1} {z}
 # CHECK-NEXT:  1      5     1.00                        vmulps	%ymm16, %ymm17, %ymm19
 # CHECK-NEXT:  2      12    1.00    *                   vmulps	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  2      12    1.00    *                   vmulps	(%rax){1to8}, %ymm17, %ymm19
 # CHECK-NEXT:  1      5     1.00                        vmulps	%ymm16, %ymm17, %ymm19 {%k1}
 # CHECK-NEXT:  2      12    1.00    *                   vmulps	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      12    1.00    *                   vmulps	(%rax){1to8}, %ymm17, %ymm19 {%k1}
 # CHECK-NEXT:  1      5     1.00                        vmulps	%ymm16, %ymm17, %ymm19 {%k1} {z}
 # CHECK-NEXT:  2      12    1.00    *                   vmulps	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      12    1.00    *                   vmulps	(%rax){1to8}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  1      1     0.50                        vpaddd	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  2      7     0.50    *                   vpaddd	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  2      7     0.50    *                   vpaddd	(%rax){1to4}, %xmm17, %xmm19
+# CHECK-NEXT:  1      1     0.50                        vpaddd	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      7     0.50    *                   vpaddd	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      7     0.50    *                   vpaddd	(%rax){1to4}, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  1      1     0.50                        vpaddd	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      7     0.50    *                   vpaddd	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      7     0.50    *                   vpaddd	(%rax){1to4}, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     0.50                        vpaddd	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  2      8     0.50    *                   vpaddd	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  2      8     0.50    *                   vpaddd	(%rax){1to8}, %ymm17, %ymm19
+# CHECK-NEXT:  1      1     0.50                        vpaddd	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     0.50    *                   vpaddd	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     0.50    *                   vpaddd	(%rax){1to8}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  1      1     0.50                        vpaddd	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     0.50    *                   vpaddd	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     0.50    *                   vpaddd	(%rax){1to8}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  1      1     0.50                        vpaddq	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  2      7     0.50    *                   vpaddq	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  2      7     0.50    *                   vpaddq	(%rax){1to2}, %xmm17, %xmm19
+# CHECK-NEXT:  1      1     0.50                        vpaddq	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      7     0.50    *                   vpaddq	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      7     0.50    *                   vpaddq	(%rax){1to2}, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  1      1     0.50                        vpaddq	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      7     0.50    *                   vpaddq	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      7     0.50    *                   vpaddq	(%rax){1to2}, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     0.50                        vpaddq	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  2      8     0.50    *                   vpaddq	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  2      8     0.50    *                   vpaddq	(%rax){1to4}, %ymm17, %ymm19
+# CHECK-NEXT:  1      1     0.50                        vpaddq	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     0.50    *                   vpaddq	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     0.50    *                   vpaddq	(%rax){1to4}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  1      1     0.50                        vpaddq	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     0.50    *                   vpaddq	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     0.50    *                   vpaddq	(%rax){1to4}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vpermd	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  2      8     1.00    *                   vpermd	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  2      8     1.00    *                   vpermd	(%rax){1to8}, %ymm17, %ymm19
+# CHECK-NEXT:  1      1     1.00                        vpermd	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vpermd	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vpermd	(%rax){1to8}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vpermd	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vpermd	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vpermd	(%rax){1to8}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vpermilpd	$0, %xmm16, %xmm19
+# CHECK-NEXT:  2      7     1.00    *                   vpermilpd	$0, (%rax), %xmm19
+# CHECK-NEXT:  2      7     1.00    *                   vpermilpd	$0, (%rax){1to2}, %xmm19
+# CHECK-NEXT:  1      1     1.00                        vpermilpd	$0, %xmm16, %xmm19 {%k1}
+# CHECK-NEXT:  2      7     1.00    *                   vpermilpd	$0, (%rax), %xmm19 {%k1}
+# CHECK-NEXT:  2      7     1.00    *                   vpermilpd	$0, (%rax){1to2}, %xmm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vpermilpd	$0, %xmm16, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      7     1.00    *                   vpermilpd	$0, (%rax), %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      7     1.00    *                   vpermilpd	$0, (%rax){1to2}, %xmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vpermilpd	$0, %ymm16, %ymm19
+# CHECK-NEXT:  2      8     1.00    *                   vpermilpd	$0, (%rax), %ymm19
+# CHECK-NEXT:  2      8     1.00    *                   vpermilpd	$0, (%rax){1to4}, %ymm19
+# CHECK-NEXT:  1      1     1.00                        vpermilpd	$0, %ymm16, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vpermilpd	$0, (%rax), %ymm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vpermilpd	$0, (%rax){1to4}, %ymm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vpermilpd	$0, %ymm16, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vpermilpd	$0, (%rax), %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vpermilpd	$0, (%rax){1to4}, %ymm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vpermilpd	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  2      7     1.00    *                   vpermilpd	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  2      7     1.00    *                   vpermilpd	(%rax){1to2}, %xmm17, %xmm19
+# CHECK-NEXT:  1      1     1.00                        vpermilpd	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      7     1.00    *                   vpermilpd	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      7     1.00    *                   vpermilpd	(%rax){1to2}, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vpermilpd	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      7     1.00    *                   vpermilpd	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      7     1.00    *                   vpermilpd	(%rax){1to2}, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vpermilpd	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  2      8     1.00    *                   vpermilpd	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  2      8     1.00    *                   vpermilpd	(%rax){1to4}, %ymm17, %ymm19
+# CHECK-NEXT:  1      1     1.00                        vpermilpd	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vpermilpd	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vpermilpd	(%rax){1to4}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vpermilpd	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vpermilpd	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vpermilpd	(%rax){1to4}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vpermilps	$0, %xmm16, %xmm19
+# CHECK-NEXT:  2      7     1.00    *                   vpermilps	$0, (%rax), %xmm19
+# CHECK-NEXT:  2      7     1.00    *                   vpermilps	$0, (%rax){1to4}, %xmm19
+# CHECK-NEXT:  1      1     1.00                        vpermilps	$0, %xmm16, %xmm19 {%k1}
+# CHECK-NEXT:  2      7     1.00    *                   vpermilps	$0, (%rax), %xmm19 {%k1}
+# CHECK-NEXT:  2      7     1.00    *                   vpermilps	$0, (%rax){1to4}, %xmm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vpermilps	$0, %xmm16, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      7     1.00    *                   vpermilps	$0, (%rax), %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      7     1.00    *                   vpermilps	$0, (%rax){1to4}, %xmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vpermilps	$0, %ymm16, %ymm19
+# CHECK-NEXT:  2      8     1.00    *                   vpermilps	$0, (%rax), %ymm19
+# CHECK-NEXT:  2      8     1.00    *                   vpermilps	$0, (%rax){1to8}, %ymm19
+# CHECK-NEXT:  1      1     1.00                        vpermilps	$0, %ymm16, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vpermilps	$0, (%rax), %ymm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vpermilps	$0, (%rax){1to8}, %ymm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vpermilps	$0, %ymm16, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vpermilps	$0, (%rax), %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vpermilps	$0, (%rax){1to8}, %ymm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vpermilps	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  2      7     1.00    *                   vpermilps	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  2      7     1.00    *                   vpermilps	(%rax){1to4}, %xmm17, %xmm19
+# CHECK-NEXT:  1      1     1.00                        vpermilps	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      7     1.00    *                   vpermilps	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      7     1.00    *                   vpermilps	(%rax){1to4}, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vpermilps	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      7     1.00    *                   vpermilps	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      7     1.00    *                   vpermilps	(%rax){1to4}, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vpermilps	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  2      8     1.00    *                   vpermilps	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  2      8     1.00    *                   vpermilps	(%rax){1to8}, %ymm17, %ymm19
+# CHECK-NEXT:  1      1     1.00                        vpermilps	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vpermilps	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vpermilps	(%rax){1to8}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vpermilps	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vpermilps	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vpermilps	(%rax){1to8}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vpermpd	$0, %ymm16, %ymm19
+# CHECK-NEXT:  2      8     1.00    *                   vpermpd	$0, (%rax), %ymm19
+# CHECK-NEXT:  2      8     1.00    *                   vpermpd	$0, (%rax){1to4}, %ymm19
+# CHECK-NEXT:  1      1     1.00                        vpermpd	$0, %ymm16, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vpermpd	$0, (%rax), %ymm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vpermpd	$0, (%rax){1to4}, %ymm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vpermpd	$0, %ymm16, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vpermpd	$0, (%rax), %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vpermpd	$0, (%rax){1to4}, %ymm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vpermpd	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  2      8     1.00    *                   vpermpd	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  2      8     1.00    *                   vpermpd	(%rax){1to4}, %ymm17, %ymm19
+# CHECK-NEXT:  1      1     1.00                        vpermpd	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vpermpd	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vpermpd	(%rax){1to4}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vpermpd	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vpermpd	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vpermpd	(%rax){1to4}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vpermps	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  2      8     1.00    *                   vpermps	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  2      8     1.00    *                   vpermps	(%rax){1to8}, %ymm17, %ymm19
+# CHECK-NEXT:  1      1     1.00                        vpermps	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vpermps	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vpermps	(%rax){1to8}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vpermps	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vpermps	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vpermps	(%rax){1to8}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vpermq	$0, %ymm16, %ymm19
+# CHECK-NEXT:  2      8     1.00    *                   vpermq	$0, (%rax), %ymm19
+# CHECK-NEXT:  2      8     1.00    *                   vpermq	$0, (%rax){1to4}, %ymm19
+# CHECK-NEXT:  1      1     1.00                        vpermq	$0, %ymm16, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vpermq	$0, (%rax), %ymm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vpermq	$0, (%rax){1to4}, %ymm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vpermq	$0, %ymm16, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vpermq	$0, (%rax), %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vpermq	$0, (%rax){1to4}, %ymm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vpermq	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  2      8     1.00    *                   vpermq	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  2      8     1.00    *                   vpermq	(%rax){1to4}, %ymm17, %ymm19
+# CHECK-NEXT:  1      1     1.00                        vpermq	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vpermq	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vpermq	(%rax){1to4}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vpermq	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vpermq	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vpermq	(%rax){1to4}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  1      5     1.00                        vpmulld	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  2      11    1.00    *                   vpmulld	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  2      11    1.00    *                   vpmulld	(%rax){1to4}, %xmm17, %xmm19
+# CHECK-NEXT:  1      5     1.00                        vpmulld	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      11    1.00    *                   vpmulld	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      11    1.00    *                   vpmulld	(%rax){1to4}, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  1      5     1.00                        vpmulld	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      11    1.00    *                   vpmulld	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      11    1.00    *                   vpmulld	(%rax){1to4}, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  1      5     1.00                        vpmulld	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  2      12    1.00    *                   vpmulld	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  2      12    1.00    *                   vpmulld	(%rax){1to8}, %ymm17, %ymm19
+# CHECK-NEXT:  1      5     1.00                        vpmulld	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      12    1.00    *                   vpmulld	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      12    1.00    *                   vpmulld	(%rax){1to8}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  1      5     1.00                        vpmulld	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      12    1.00    *                   vpmulld	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      12    1.00    *                   vpmulld	(%rax){1to8}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  1      1     0.50                        vpshufd	$0, %xmm16, %xmm19
+# CHECK-NEXT:  2      7     0.50    *                   vpshufd	$0, (%rax), %xmm19
+# CHECK-NEXT:  2      7     0.50    *                   vpshufd	$0, (%rax){1to4}, %xmm19
+# CHECK-NEXT:  1      1     0.50                        vpshufd	$0, %xmm16, %xmm19 {%k1}
+# CHECK-NEXT:  2      7     0.50    *                   vpshufd	$0, (%rax), %xmm19 {%k1}
+# CHECK-NEXT:  2      7     0.50    *                   vpshufd	$0, (%rax){1to4}, %xmm19 {%k1}
+# CHECK-NEXT:  1      1     0.50                        vpshufd	$0, %xmm16, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      7     0.50    *                   vpshufd	$0, (%rax), %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      7     0.50    *                   vpshufd	$0, (%rax){1to4}, %xmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vpshufd	$0, %ymm16, %ymm19
+# CHECK-NEXT:  2      8     1.00    *                   vpshufd	$0, (%rax), %ymm19
+# CHECK-NEXT:  2      8     1.00    *                   vpshufd	$0, (%rax){1to8}, %ymm19
+# CHECK-NEXT:  1      1     1.00                        vpshufd	$0, %ymm16, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vpshufd	$0, (%rax), %ymm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vpshufd	$0, (%rax){1to8}, %ymm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vpshufd	$0, %ymm16, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vpshufd	$0, (%rax), %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vpshufd	$0, (%rax){1to8}, %ymm19 {%k1} {z}
+# CHECK-NEXT:  1      1     0.50                        vpsubd	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  2      7     0.50    *                   vpsubd	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  2      7     0.50    *                   vpsubd	(%rax){1to4}, %xmm17, %xmm19
+# CHECK-NEXT:  1      1     0.50                        vpsubd	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      7     0.50    *                   vpsubd	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      7     0.50    *                   vpsubd	(%rax){1to4}, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  1      1     0.50                        vpsubd	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      7     0.50    *                   vpsubd	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      7     0.50    *                   vpsubd	(%rax){1to4}, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     0.50                        vpsubd	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  2      8     0.50    *                   vpsubd	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  2      8     0.50    *                   vpsubd	(%rax){1to8}, %ymm17, %ymm19
+# CHECK-NEXT:  1      1     0.50                        vpsubd	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     0.50    *                   vpsubd	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     0.50    *                   vpsubd	(%rax){1to8}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  1      1     0.50                        vpsubd	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     0.50    *                   vpsubd	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     0.50    *                   vpsubd	(%rax){1to8}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  1      1     0.50                        vpsubq	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  2      7     0.50    *                   vpsubq	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  2      7     0.50    *                   vpsubq	(%rax){1to2}, %xmm17, %xmm19
+# CHECK-NEXT:  1      1     0.50                        vpsubq	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      7     0.50    *                   vpsubq	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      7     0.50    *                   vpsubq	(%rax){1to2}, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  1      1     0.50                        vpsubq	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      7     0.50    *                   vpsubq	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      7     0.50    *                   vpsubq	(%rax){1to2}, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     0.50                        vpsubq	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  2      8     0.50    *                   vpsubq	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  2      8     0.50    *                   vpsubq	(%rax){1to4}, %ymm17, %ymm19
+# CHECK-NEXT:  1      1     0.50                        vpsubq	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     0.50    *                   vpsubq	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     0.50    *                   vpsubq	(%rax){1to4}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  1      1     0.50                        vpsubq	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     0.50    *                   vpsubq	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     0.50    *                   vpsubq	(%rax){1to4}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  1      1     0.50                        vpunpckhdq	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  2      7     0.50    *                   vpunpckhdq	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  2      7     0.50    *                   vpunpckhdq	(%rax){1to4}, %xmm17, %xmm19
+# CHECK-NEXT:  1      1     0.50                        vpunpckhdq	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      7     0.50    *                   vpunpckhdq	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      7     0.50    *                   vpunpckhdq	(%rax){1to4}, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  1      1     0.50                        vpunpckhdq	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      7     0.50    *                   vpunpckhdq	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      7     0.50    *                   vpunpckhdq	(%rax){1to4}, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vpunpckhdq	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  2      8     1.00    *                   vpunpckhdq	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  2      8     1.00    *                   vpunpckhdq	(%rax){1to8}, %ymm17, %ymm19
+# CHECK-NEXT:  1      1     1.00                        vpunpckhdq	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vpunpckhdq	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vpunpckhdq	(%rax){1to8}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vpunpckhdq	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vpunpckhdq	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vpunpckhdq	(%rax){1to8}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  1      1     0.50                        vpunpckldq	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  2      7     0.50    *                   vpunpckldq	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  2      7     0.50    *                   vpunpckldq	(%rax){1to4}, %xmm17, %xmm19
+# CHECK-NEXT:  1      1     0.50                        vpunpckldq	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      7     0.50    *                   vpunpckldq	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      7     0.50    *                   vpunpckldq	(%rax){1to4}, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  1      1     0.50                        vpunpckldq	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      7     0.50    *                   vpunpckldq	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      7     0.50    *                   vpunpckldq	(%rax){1to4}, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vpunpckldq	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  2      8     1.00    *                   vpunpckldq	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  2      8     1.00    *                   vpunpckldq	(%rax){1to8}, %ymm17, %ymm19
+# CHECK-NEXT:  1      1     1.00                        vpunpckldq	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vpunpckldq	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vpunpckldq	(%rax){1to8}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vpunpckldq	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vpunpckldq	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vpunpckldq	(%rax){1to8}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vshuff32x4	$0, %ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  2      8     1.00    *                   vshuff32x4	$0, (%rax), %ymm17, %ymm19
+# CHECK-NEXT:  2      8     1.00    *                   vshuff32x4	$0, (%rax){1to8}, %ymm17, %ymm19
+# CHECK-NEXT:  1      1     1.00                        vshuff32x4	$0, %ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vshuff32x4	$0, (%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vshuff32x4	$0, (%rax){1to8}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vshuff32x4	$0, %ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vshuff32x4	$0, (%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vshuff32x4	$0, (%rax){1to8}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vshuff64x2	$0, %ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  2      8     1.00    *                   vshuff64x2	$0, (%rax), %ymm17, %ymm19
+# CHECK-NEXT:  2      8     1.00    *                   vshuff64x2	$0, (%rax){1to4}, %ymm17, %ymm19
+# CHECK-NEXT:  1      1     1.00                        vshuff64x2	$0, %ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vshuff64x2	$0, (%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vshuff64x2	$0, (%rax){1to4}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vshuff64x2	$0, %ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vshuff64x2	$0, (%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vshuff64x2	$0, (%rax){1to4}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vshufi32x4	$0, %ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  2      8     1.00    *                   vshufi32x4	$0, (%rax), %ymm17, %ymm19
+# CHECK-NEXT:  2      8     1.00    *                   vshufi32x4	$0, (%rax){1to8}, %ymm17, %ymm19
+# CHECK-NEXT:  1      1     1.00                        vshufi32x4	$0, %ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vshufi32x4	$0, (%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vshufi32x4	$0, (%rax){1to8}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vshufi32x4	$0, %ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vshufi32x4	$0, (%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vshufi32x4	$0, (%rax){1to8}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vshufi64x2	$0, %ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  2      8     1.00    *                   vshufi64x2	$0, (%rax), %ymm17, %ymm19
+# CHECK-NEXT:  2      8     1.00    *                   vshufi64x2	$0, (%rax){1to4}, %ymm17, %ymm19
+# CHECK-NEXT:  1      1     1.00                        vshufi64x2	$0, %ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vshufi64x2	$0, (%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vshufi64x2	$0, (%rax){1to4}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vshufi64x2	$0, %ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vshufi64x2	$0, (%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vshufi64x2	$0, (%rax){1to4}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  1      21    21.00                       vsqrtpd	%xmm16, %xmm19
+# CHECK-NEXT:  2      27    21.00   *                   vsqrtpd	(%rax), %xmm19
+# CHECK-NEXT:  2      27    21.00   *                   vsqrtpd	(%rax){1to2}, %xmm19
+# CHECK-NEXT:  1      21    21.00                       vsqrtpd	%xmm16, %xmm19 {%k1}
+# CHECK-NEXT:  2      27    21.00   *                   vsqrtpd	(%rax), %xmm19 {%k1}
+# CHECK-NEXT:  2      27    21.00   *                   vsqrtpd	(%rax){1to2}, %xmm19 {%k1}
+# CHECK-NEXT:  1      21    21.00                       vsqrtpd	%xmm16, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      27    21.00   *                   vsqrtpd	(%rax), %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      27    21.00   *                   vsqrtpd	(%rax){1to2}, %xmm19 {%k1} {z}
+# CHECK-NEXT:  3      45    44.00                       vsqrtpd	%ymm16, %ymm19
+# CHECK-NEXT:  4      52    44.00   *                   vsqrtpd	(%rax), %ymm19
+# CHECK-NEXT:  4      52    44.00   *                   vsqrtpd	(%rax){1to4}, %ymm19
+# CHECK-NEXT:  3      45    44.00                       vsqrtpd	%ymm16, %ymm19 {%k1}
+# CHECK-NEXT:  4      52    44.00   *                   vsqrtpd	(%rax), %ymm19 {%k1}
+# CHECK-NEXT:  4      52    44.00   *                   vsqrtpd	(%rax){1to4}, %ymm19 {%k1}
+# CHECK-NEXT:  3      45    44.00                       vsqrtpd	%ymm16, %ymm19 {%k1} {z}
+# CHECK-NEXT:  4      52    44.00   *                   vsqrtpd	(%rax), %ymm19 {%k1} {z}
+# CHECK-NEXT:  4      52    44.00   *                   vsqrtpd	(%rax){1to4}, %ymm19 {%k1} {z}
+# CHECK-NEXT:  1      14    14.00                       vsqrtps	%xmm16, %xmm19
+# CHECK-NEXT:  2      20    14.00   *                   vsqrtps	(%rax), %xmm19
+# CHECK-NEXT:  2      20    14.00   *                   vsqrtps	(%rax){1to4}, %xmm19
+# CHECK-NEXT:  1      14    14.00                       vsqrtps	%xmm16, %xmm19 {%k1}
+# CHECK-NEXT:  2      20    14.00   *                   vsqrtps	(%rax), %xmm19 {%k1}
+# CHECK-NEXT:  2      20    14.00   *                   vsqrtps	(%rax){1to4}, %xmm19 {%k1}
+# CHECK-NEXT:  1      14    14.00                       vsqrtps	%xmm16, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      20    14.00   *                   vsqrtps	(%rax), %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      20    14.00   *                   vsqrtps	(%rax){1to4}, %xmm19 {%k1} {z}
+# CHECK-NEXT:  3      29    28.00                       vsqrtps	%ymm16, %ymm19
+# CHECK-NEXT:  4      36    28.00   *                   vsqrtps	(%rax), %ymm19
+# CHECK-NEXT:  4      36    28.00   *                   vsqrtps	(%rax){1to8}, %ymm19
+# CHECK-NEXT:  3      29    28.00                       vsqrtps	%ymm16, %ymm19 {%k1}
+# CHECK-NEXT:  4      36    28.00   *                   vsqrtps	(%rax), %ymm19 {%k1}
+# CHECK-NEXT:  4      36    28.00   *                   vsqrtps	(%rax){1to8}, %ymm19 {%k1}
+# CHECK-NEXT:  3      29    28.00                       vsqrtps	%ymm16, %ymm19 {%k1} {z}
+# CHECK-NEXT:  4      36    28.00   *                   vsqrtps	(%rax), %ymm19 {%k1} {z}
+# CHECK-NEXT:  4      36    28.00   *                   vsqrtps	(%rax){1to8}, %ymm19 {%k1} {z}
 # CHECK-NEXT:  1      3     1.00                        vsubpd	%xmm16, %xmm17, %xmm19
 # CHECK-NEXT:  2      9     1.00    *                   vsubpd	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  2      9     1.00    *                   vsubpd	(%rax){1to2}, %xmm17, %xmm19
 # CHECK-NEXT:  1      3     1.00                        vsubpd	%xmm16, %xmm17, %xmm19 {%k1}
 # CHECK-NEXT:  2      9     1.00    *                   vsubpd	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      9     1.00    *                   vsubpd	(%rax){1to2}, %xmm17, %xmm19 {%k1}
 # CHECK-NEXT:  1      3     1.00                        vsubpd	%xmm16, %xmm17, %xmm19 {%k1} {z}
 # CHECK-NEXT:  2      9     1.00    *                   vsubpd	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      9     1.00    *                   vsubpd	(%rax){1to2}, %xmm17, %xmm19 {%k1} {z}
 # CHECK-NEXT:  1      3     1.00                        vsubpd	%ymm16, %ymm17, %ymm19
 # CHECK-NEXT:  2      10    1.00    *                   vsubpd	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  2      10    1.00    *                   vsubpd	(%rax){1to4}, %ymm17, %ymm19
 # CHECK-NEXT:  1      3     1.00                        vsubpd	%ymm16, %ymm17, %ymm19 {%k1}
 # CHECK-NEXT:  2      10    1.00    *                   vsubpd	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      10    1.00    *                   vsubpd	(%rax){1to4}, %ymm17, %ymm19 {%k1}
 # CHECK-NEXT:  1      3     1.00                        vsubpd	%ymm16, %ymm17, %ymm19 {%k1} {z}
 # CHECK-NEXT:  2      10    1.00    *                   vsubpd	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      10    1.00    *                   vsubpd	(%rax){1to4}, %ymm17, %ymm19 {%k1} {z}
 # CHECK-NEXT:  1      3     1.00                        vsubps	%xmm16, %xmm17, %xmm19
 # CHECK-NEXT:  2      9     1.00    *                   vsubps	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  2      9     1.00    *                   vsubps	(%rax){1to4}, %xmm17, %xmm19
 # CHECK-NEXT:  1      3     1.00                        vsubps	%xmm16, %xmm17, %xmm19 {%k1}
 # CHECK-NEXT:  2      9     1.00    *                   vsubps	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      9     1.00    *                   vsubps	(%rax){1to4}, %xmm17, %xmm19 {%k1}
 # CHECK-NEXT:  1      3     1.00                        vsubps	%xmm16, %xmm17, %xmm19 {%k1} {z}
 # CHECK-NEXT:  2      9     1.00    *                   vsubps	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      9     1.00    *                   vsubps	(%rax){1to4}, %xmm17, %xmm19 {%k1} {z}
 # CHECK-NEXT:  1      3     1.00                        vsubps	%ymm16, %ymm17, %ymm19
 # CHECK-NEXT:  2      10    1.00    *                   vsubps	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  2      10    1.00    *                   vsubps	(%rax){1to8}, %ymm17, %ymm19
 # CHECK-NEXT:  1      3     1.00                        vsubps	%ymm16, %ymm17, %ymm19 {%k1}
 # CHECK-NEXT:  2      10    1.00    *                   vsubps	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      10    1.00    *                   vsubps	(%rax){1to8}, %ymm17, %ymm19 {%k1}
 # CHECK-NEXT:  1      3     1.00                        vsubps	%ymm16, %ymm17, %ymm19 {%k1} {z}
 # CHECK-NEXT:  2      10    1.00    *                   vsubps	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      10    1.00    *                   vsubps	(%rax){1to8}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vunpckhpd	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  2      7     1.00    *                   vunpckhpd	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  2      7     1.00    *                   vunpckhpd	(%rax){1to2}, %xmm17, %xmm19
+# CHECK-NEXT:  1      1     1.00                        vunpckhpd	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      7     1.00    *                   vunpckhpd	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      7     1.00    *                   vunpckhpd	(%rax){1to2}, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vunpckhpd	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      7     1.00    *                   vunpckhpd	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      7     1.00    *                   vunpckhpd	(%rax){1to2}, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vunpckhpd	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  2      8     1.00    *                   vunpckhpd	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  2      8     1.00    *                   vunpckhpd	(%rax){1to4}, %ymm17, %ymm19
+# CHECK-NEXT:  1      1     1.00                        vunpckhpd	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vunpckhpd	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vunpckhpd	(%rax){1to4}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vunpckhpd	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vunpckhpd	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vunpckhpd	(%rax){1to4}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vunpckhps	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  2      7     1.00    *                   vunpckhps	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  2      7     1.00    *                   vunpckhps	(%rax){1to4}, %xmm17, %xmm19
+# CHECK-NEXT:  1      1     1.00                        vunpckhps	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      7     1.00    *                   vunpckhps	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      7     1.00    *                   vunpckhps	(%rax){1to4}, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vunpckhps	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      7     1.00    *                   vunpckhps	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      7     1.00    *                   vunpckhps	(%rax){1to4}, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vunpckhps	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  2      8     1.00    *                   vunpckhps	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  2      8     1.00    *                   vunpckhps	(%rax){1to8}, %ymm17, %ymm19
+# CHECK-NEXT:  1      1     1.00                        vunpckhps	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vunpckhps	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vunpckhps	(%rax){1to8}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vunpckhps	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vunpckhps	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vunpckhps	(%rax){1to8}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vunpcklpd	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  2      7     1.00    *                   vunpcklpd	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  2      7     1.00    *                   vunpcklpd	(%rax){1to2}, %xmm17, %xmm19
+# CHECK-NEXT:  1      1     1.00                        vunpcklpd	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      7     1.00    *                   vunpcklpd	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      7     1.00    *                   vunpcklpd	(%rax){1to2}, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vunpcklpd	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      7     1.00    *                   vunpcklpd	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      7     1.00    *                   vunpcklpd	(%rax){1to2}, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vunpcklpd	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  2      8     1.00    *                   vunpcklpd	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  2      8     1.00    *                   vunpcklpd	(%rax){1to4}, %ymm17, %ymm19
+# CHECK-NEXT:  1      1     1.00                        vunpcklpd	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vunpcklpd	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vunpcklpd	(%rax){1to4}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vunpcklpd	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vunpcklpd	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vunpcklpd	(%rax){1to4}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vunpcklps	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  2      7     1.00    *                   vunpcklps	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  2      7     1.00    *                   vunpcklps	(%rax){1to4}, %xmm17, %xmm19
+# CHECK-NEXT:  1      1     1.00                        vunpcklps	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      7     1.00    *                   vunpcklps	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      7     1.00    *                   vunpcklps	(%rax){1to4}, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vunpcklps	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      7     1.00    *                   vunpcklps	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      7     1.00    *                   vunpcklps	(%rax){1to4}, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vunpcklps	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  2      8     1.00    *                   vunpcklps	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  2      8     1.00    *                   vunpcklps	(%rax){1to8}, %ymm17, %ymm19
+# CHECK-NEXT:  1      1     1.00                        vunpcklps	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vunpcklps	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vunpcklps	(%rax){1to8}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vunpcklps	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vunpcklps	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vunpcklps	(%rax){1to8}, %ymm17, %ymm19 {%k1} {z}
 
 # CHECK:      Resources:
 # CHECK-NEXT: [0]   - SBDivider
@@ -231,103 +1429,673 @@ vsubps            (%rax), %ymm17, %ymm19 {z}{k1}
 
 # CHECK:      Resource pressure per iteration:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6.0]  [6.1]
-# CHECK-NEXT:  -     648.00 66.00  48.00   -     6.00   24.00  24.00
+# CHECK-NEXT:  -     1935.00 180.00 229.50  -    346.50 222.00 222.00
 
 # CHECK:      Resource pressure by instruction:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6.0]  [6.1]  Instructions:
 # CHECK-NEXT:  -      -      -     1.00    -      -      -      -     vaddpd	%xmm16, %xmm17, %xmm19
 # CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vaddpd	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vaddpd	(%rax){1to2}, %xmm17, %xmm19
 # CHECK-NEXT:  -      -      -     1.00    -      -      -      -     vaddpd	%xmm16, %xmm17, %xmm19 {%k1}
 # CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vaddpd	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vaddpd	(%rax){1to2}, %xmm17, %xmm19 {%k1}
 # CHECK-NEXT:  -      -      -     1.00    -      -      -      -     vaddpd	%xmm16, %xmm17, %xmm19 {%k1} {z}
 # CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vaddpd	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vaddpd	(%rax){1to2}, %xmm17, %xmm19 {%k1} {z}
 # CHECK-NEXT:  -      -      -     1.00    -      -      -      -     vaddpd	%ymm16, %ymm17, %ymm19
 # CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vaddpd	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vaddpd	(%rax){1to4}, %ymm17, %ymm19
 # CHECK-NEXT:  -      -      -     1.00    -      -      -      -     vaddpd	%ymm16, %ymm17, %ymm19 {%k1}
 # CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vaddpd	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vaddpd	(%rax){1to4}, %ymm17, %ymm19 {%k1}
 # CHECK-NEXT:  -      -      -     1.00    -      -      -      -     vaddpd	%ymm16, %ymm17, %ymm19 {%k1} {z}
 # CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vaddpd	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vaddpd	(%rax){1to4}, %ymm17, %ymm19 {%k1} {z}
 # CHECK-NEXT:  -      -      -     1.00    -      -      -      -     vaddps	%xmm16, %xmm17, %xmm19
 # CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vaddps	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vaddps	(%rax){1to4}, %xmm17, %xmm19
 # CHECK-NEXT:  -      -      -     1.00    -      -      -      -     vaddps	%xmm16, %xmm17, %xmm19 {%k1}
 # CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vaddps	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vaddps	(%rax){1to4}, %xmm17, %xmm19 {%k1}
 # CHECK-NEXT:  -      -      -     1.00    -      -      -      -     vaddps	%xmm16, %xmm17, %xmm19 {%k1} {z}
 # CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vaddps	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vaddps	(%rax){1to4}, %xmm17, %xmm19 {%k1} {z}
 # CHECK-NEXT:  -      -      -     1.00    -      -      -      -     vaddps	%ymm16, %ymm17, %ymm19
 # CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vaddps	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vaddps	(%rax){1to8}, %ymm17, %ymm19
 # CHECK-NEXT:  -      -      -     1.00    -      -      -      -     vaddps	%ymm16, %ymm17, %ymm19 {%k1}
 # CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vaddps	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vaddps	(%rax){1to8}, %ymm17, %ymm19 {%k1}
 # CHECK-NEXT:  -      -      -     1.00    -      -      -      -     vaddps	%ymm16, %ymm17, %ymm19 {%k1} {z}
 # CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vaddps	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vaddps	(%rax){1to8}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     1.00    -     1.00    -      -     vcvtdq2pd	%xmm16, %xmm19
+# CHECK-NEXT:  -      -      -     1.00    -     1.00   0.50   0.50   vcvtdq2pd	(%rax), %xmm19
+# CHECK-NEXT:  -      -      -     1.00    -     1.00   0.50   0.50   vcvtdq2pd	(%rax){1to2}, %xmm19
+# CHECK-NEXT:  -      -      -     1.00    -     1.00    -      -     vcvtdq2pd	%xmm16, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -     1.00    -     1.00   0.50   0.50   vcvtdq2pd	(%rax), %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -     1.00    -     1.00   0.50   0.50   vcvtdq2pd	(%rax){1to2}, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -     1.00    -     1.00    -      -     vcvtdq2pd	%xmm16, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     1.00    -     1.00   0.50   0.50   vcvtdq2pd	(%rax), %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     1.00    -     1.00   0.50   0.50   vcvtdq2pd	(%rax){1to2}, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     1.00    -     1.00    -      -     vcvtdq2pd	%xmm16, %ymm19
+# CHECK-NEXT:  -      -      -     1.00    -     1.00   0.50   0.50   vcvtdq2pd	(%rax), %ymm19
+# CHECK-NEXT:  -      -      -     1.00    -     1.00   0.50   0.50   vcvtdq2pd	(%rax){1to4}, %ymm19
+# CHECK-NEXT:  -      -      -     1.00    -     1.00    -      -     vcvtdq2pd	%xmm16, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -     1.00    -     1.00   0.50   0.50   vcvtdq2pd	(%rax), %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -     1.00    -     1.00   0.50   0.50   vcvtdq2pd	(%rax){1to4}, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -     1.00    -     1.00    -      -     vcvtdq2pd	%xmm16, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     1.00    -     1.00   0.50   0.50   vcvtdq2pd	(%rax), %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     1.00    -     1.00   0.50   0.50   vcvtdq2pd	(%rax){1to4}, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -     vcvtdq2ps	%xmm16, %xmm19
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vcvtdq2ps	(%rax), %xmm19
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vcvtdq2ps	(%rax){1to4}, %xmm19
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -     vcvtdq2ps	%xmm16, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vcvtdq2ps	(%rax), %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vcvtdq2ps	(%rax){1to4}, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -     vcvtdq2ps	%xmm16, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vcvtdq2ps	(%rax), %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vcvtdq2ps	(%rax){1to4}, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -     vcvtdq2ps	%ymm16, %ymm19
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vcvtdq2ps	(%rax), %ymm19
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vcvtdq2ps	(%rax){1to8}, %ymm19
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -     vcvtdq2ps	%ymm16, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vcvtdq2ps	(%rax), %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vcvtdq2ps	(%rax){1to8}, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -     vcvtdq2ps	%ymm16, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vcvtdq2ps	(%rax), %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vcvtdq2ps	(%rax){1to8}, %ymm19 {%k1} {z}
 # CHECK-NEXT:  -     22.00  1.00    -      -      -      -      -     vdivpd	%xmm16, %xmm17, %xmm19
 # CHECK-NEXT:  -     22.00  1.00    -      -      -     0.50   0.50   vdivpd	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  -     22.00  1.00    -      -      -     0.50   0.50   vdivpd	(%rax){1to2}, %xmm17, %xmm19
 # CHECK-NEXT:  -     22.00  1.00    -      -      -      -      -     vdivpd	%xmm16, %xmm17, %xmm19 {%k1}
 # CHECK-NEXT:  -     22.00  1.00    -      -      -     0.50   0.50   vdivpd	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -     22.00  1.00    -      -      -     0.50   0.50   vdivpd	(%rax){1to2}, %xmm17, %xmm19 {%k1}
 # CHECK-NEXT:  -     22.00  1.00    -      -      -      -      -     vdivpd	%xmm16, %xmm17, %xmm19 {%k1} {z}
 # CHECK-NEXT:  -     22.00  1.00    -      -      -     0.50   0.50   vdivpd	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -     22.00  1.00    -      -      -     0.50   0.50   vdivpd	(%rax){1to2}, %xmm17, %xmm19 {%k1} {z}
 # CHECK-NEXT:  -     44.00  2.50    -      -     0.50    -      -     vdivpd	%ymm16, %ymm17, %ymm19
 # CHECK-NEXT:  -     44.00  2.50    -      -     0.50   0.50   0.50   vdivpd	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  -     44.00  2.50    -      -     0.50   0.50   0.50   vdivpd	(%rax){1to4}, %ymm17, %ymm19
 # CHECK-NEXT:  -     44.00  2.50    -      -     0.50    -      -     vdivpd	%ymm16, %ymm17, %ymm19 {%k1}
 # CHECK-NEXT:  -     44.00  2.50    -      -     0.50   0.50   0.50   vdivpd	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -     44.00  2.50    -      -     0.50   0.50   0.50   vdivpd	(%rax){1to4}, %ymm17, %ymm19 {%k1}
 # CHECK-NEXT:  -     44.00  2.50    -      -     0.50    -      -     vdivpd	%ymm16, %ymm17, %ymm19 {%k1} {z}
 # CHECK-NEXT:  -     44.00  2.50    -      -     0.50   0.50   0.50   vdivpd	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -     44.00  2.50    -      -     0.50   0.50   0.50   vdivpd	(%rax){1to4}, %ymm17, %ymm19 {%k1} {z}
 # CHECK-NEXT:  -     14.00  1.00    -      -      -      -      -     vdivps	%xmm16, %xmm17, %xmm19
 # CHECK-NEXT:  -     14.00  1.00    -      -      -     0.50   0.50   vdivps	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  -     14.00  1.00    -      -      -     0.50   0.50   vdivps	(%rax){1to4}, %xmm17, %xmm19
 # CHECK-NEXT:  -     14.00  1.00    -      -      -      -      -     vdivps	%xmm16, %xmm17, %xmm19 {%k1}
 # CHECK-NEXT:  -     14.00  1.00    -      -      -     0.50   0.50   vdivps	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -     14.00  1.00    -      -      -     0.50   0.50   vdivps	(%rax){1to4}, %xmm17, %xmm19 {%k1}
 # CHECK-NEXT:  -     14.00  1.00    -      -      -      -      -     vdivps	%xmm16, %xmm17, %xmm19 {%k1} {z}
 # CHECK-NEXT:  -     14.00  1.00    -      -      -     0.50   0.50   vdivps	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -     14.00  1.00    -      -      -     0.50   0.50   vdivps	(%rax){1to4}, %xmm17, %xmm19 {%k1} {z}
 # CHECK-NEXT:  -     28.00  2.50    -      -     0.50    -      -     vdivps	%ymm16, %ymm17, %ymm19
 # CHECK-NEXT:  -     28.00  2.50    -      -     0.50   0.50   0.50   vdivps	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  -     28.00  2.50    -      -     0.50   0.50   0.50   vdivps	(%rax){1to8}, %ymm17, %ymm19
 # CHECK-NEXT:  -     28.00  2.50    -      -     0.50    -      -     vdivps	%ymm16, %ymm17, %ymm19 {%k1}
 # CHECK-NEXT:  -     28.00  2.50    -      -     0.50   0.50   0.50   vdivps	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -     28.00  2.50    -      -     0.50   0.50   0.50   vdivps	(%rax){1to8}, %ymm17, %ymm19 {%k1}
 # CHECK-NEXT:  -     28.00  2.50    -      -     0.50    -      -     vdivps	%ymm16, %ymm17, %ymm19 {%k1} {z}
 # CHECK-NEXT:  -     28.00  2.50    -      -     0.50   0.50   0.50   vdivps	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -     28.00  2.50    -      -     0.50   0.50   0.50   vdivps	(%rax){1to8}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -     vmaxpd	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vmaxpd	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vmaxpd	(%rax){1to2}, %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -     vmaxpd	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vmaxpd	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vmaxpd	(%rax){1to2}, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -     vmaxpd	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vmaxpd	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vmaxpd	(%rax){1to2}, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -     vmaxpd	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vmaxpd	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vmaxpd	(%rax){1to4}, %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -     vmaxpd	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vmaxpd	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vmaxpd	(%rax){1to4}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -     vmaxpd	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vmaxpd	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vmaxpd	(%rax){1to4}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -     vmaxps	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vmaxps	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vmaxps	(%rax){1to4}, %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -     vmaxps	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vmaxps	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vmaxps	(%rax){1to4}, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -     vmaxps	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vmaxps	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vmaxps	(%rax){1to4}, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -     vmaxps	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vmaxps	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vmaxps	(%rax){1to8}, %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -     vmaxps	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vmaxps	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vmaxps	(%rax){1to8}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -     vmaxps	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vmaxps	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vmaxps	(%rax){1to8}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -     vminpd	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vminpd	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vminpd	(%rax){1to2}, %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -     vminpd	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vminpd	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vminpd	(%rax){1to2}, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -     vminpd	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vminpd	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vminpd	(%rax){1to2}, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -     vminpd	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vminpd	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vminpd	(%rax){1to4}, %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -     vminpd	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vminpd	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vminpd	(%rax){1to4}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -     vminpd	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vminpd	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vminpd	(%rax){1to4}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -     vminps	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vminps	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vminps	(%rax){1to4}, %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -     vminps	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vminps	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vminps	(%rax){1to4}, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -     vminps	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vminps	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vminps	(%rax){1to4}, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -     vminps	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vminps	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vminps	(%rax){1to8}, %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -     vminps	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vminps	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vminps	(%rax){1to8}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -     vminps	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vminps	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vminps	(%rax){1to8}, %ymm17, %ymm19 {%k1} {z}
 # CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vmulpd	%xmm16, %xmm17, %xmm19
 # CHECK-NEXT:  -      -     1.00    -      -      -     0.50   0.50   vmulpd	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  -      -     1.00    -      -      -     0.50   0.50   vmulpd	(%rax){1to2}, %xmm17, %xmm19
 # CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vmulpd	%xmm16, %xmm17, %xmm19 {%k1}
 # CHECK-NEXT:  -      -     1.00    -      -      -     0.50   0.50   vmulpd	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -     1.00    -      -      -     0.50   0.50   vmulpd	(%rax){1to2}, %xmm17, %xmm19 {%k1}
 # CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vmulpd	%xmm16, %xmm17, %xmm19 {%k1} {z}
 # CHECK-NEXT:  -      -     1.00    -      -      -     0.50   0.50   vmulpd	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     1.00    -      -      -     0.50   0.50   vmulpd	(%rax){1to2}, %xmm17, %xmm19 {%k1} {z}
 # CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vmulpd	%ymm16, %ymm17, %ymm19
 # CHECK-NEXT:  -      -     1.00    -      -      -     0.50   0.50   vmulpd	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  -      -     1.00    -      -      -     0.50   0.50   vmulpd	(%rax){1to4}, %ymm17, %ymm19
 # CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vmulpd	%ymm16, %ymm17, %ymm19 {%k1}
 # CHECK-NEXT:  -      -     1.00    -      -      -     0.50   0.50   vmulpd	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -     1.00    -      -      -     0.50   0.50   vmulpd	(%rax){1to4}, %ymm17, %ymm19 {%k1}
 # CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vmulpd	%ymm16, %ymm17, %ymm19 {%k1} {z}
 # CHECK-NEXT:  -      -     1.00    -      -      -     0.50   0.50   vmulpd	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -     1.00    -      -      -     0.50   0.50   vmulpd	(%rax){1to4}, %ymm17, %ymm19 {%k1} {z}
 # CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vmulps	%xmm16, %xmm17, %xmm19
 # CHECK-NEXT:  -      -     1.00    -      -      -     0.50   0.50   vmulps	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  -      -     1.00    -      -      -     0.50   0.50   vmulps	(%rax){1to4}, %xmm17, %xmm19
 # CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vmulps	%xmm16, %xmm17, %xmm19 {%k1}
 # CHECK-NEXT:  -      -     1.00    -      -      -     0.50   0.50   vmulps	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -     1.00    -      -      -     0.50   0.50   vmulps	(%rax){1to4}, %xmm17, %xmm19 {%k1}
 # CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vmulps	%xmm16, %xmm17, %xmm19 {%k1} {z}
 # CHECK-NEXT:  -      -     1.00    -      -      -     0.50   0.50   vmulps	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     1.00    -      -      -     0.50   0.50   vmulps	(%rax){1to4}, %xmm17, %xmm19 {%k1} {z}
 # CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vmulps	%ymm16, %ymm17, %ymm19
 # CHECK-NEXT:  -      -     1.00    -      -      -     0.50   0.50   vmulps	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  -      -     1.00    -      -      -     0.50   0.50   vmulps	(%rax){1to8}, %ymm17, %ymm19
 # CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vmulps	%ymm16, %ymm17, %ymm19 {%k1}
 # CHECK-NEXT:  -      -     1.00    -      -      -     0.50   0.50   vmulps	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -     1.00    -      -      -     0.50   0.50   vmulps	(%rax){1to8}, %ymm17, %ymm19 {%k1}
 # CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vmulps	%ymm16, %ymm17, %ymm19 {%k1} {z}
 # CHECK-NEXT:  -      -     1.00    -      -      -     0.50   0.50   vmulps	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -     1.00    -      -      -     0.50   0.50   vmulps	(%rax){1to8}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpaddd	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpaddd	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpaddd	(%rax){1to4}, %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpaddd	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpaddd	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpaddd	(%rax){1to4}, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpaddd	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpaddd	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpaddd	(%rax){1to4}, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpaddd	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpaddd	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpaddd	(%rax){1to8}, %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpaddd	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpaddd	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpaddd	(%rax){1to8}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpaddd	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpaddd	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpaddd	(%rax){1to8}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpaddq	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpaddq	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpaddq	(%rax){1to2}, %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpaddq	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpaddq	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpaddq	(%rax){1to2}, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpaddq	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpaddq	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpaddq	(%rax){1to2}, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpaddq	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpaddq	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpaddq	(%rax){1to4}, %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpaddq	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpaddq	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpaddq	(%rax){1to4}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpaddq	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpaddq	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpaddq	(%rax){1to4}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpermd	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermd	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermd	(%rax){1to8}, %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpermd	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermd	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermd	(%rax){1to8}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpermd	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermd	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermd	(%rax){1to8}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpermilpd	$0, %xmm16, %xmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermilpd	$0, (%rax), %xmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermilpd	$0, (%rax){1to2}, %xmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpermilpd	$0, %xmm16, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermilpd	$0, (%rax), %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermilpd	$0, (%rax){1to2}, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpermilpd	$0, %xmm16, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermilpd	$0, (%rax), %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermilpd	$0, (%rax){1to2}, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpermilpd	$0, %ymm16, %ymm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermilpd	$0, (%rax), %ymm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermilpd	$0, (%rax){1to4}, %ymm19
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpermilpd	$0, %ymm16, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermilpd	$0, (%rax), %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermilpd	$0, (%rax){1to4}, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpermilpd	$0, %ymm16, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermilpd	$0, (%rax), %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermilpd	$0, (%rax){1to4}, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpermilpd	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermilpd	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermilpd	(%rax){1to2}, %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpermilpd	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermilpd	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermilpd	(%rax){1to2}, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpermilpd	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermilpd	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermilpd	(%rax){1to2}, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpermilpd	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermilpd	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermilpd	(%rax){1to4}, %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpermilpd	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermilpd	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermilpd	(%rax){1to4}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpermilpd	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermilpd	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermilpd	(%rax){1to4}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpermilps	$0, %xmm16, %xmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermilps	$0, (%rax), %xmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermilps	$0, (%rax){1to4}, %xmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpermilps	$0, %xmm16, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermilps	$0, (%rax), %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermilps	$0, (%rax){1to4}, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpermilps	$0, %xmm16, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermilps	$0, (%rax), %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermilps	$0, (%rax){1to4}, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpermilps	$0, %ymm16, %ymm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermilps	$0, (%rax), %ymm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermilps	$0, (%rax){1to8}, %ymm19
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpermilps	$0, %ymm16, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermilps	$0, (%rax), %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermilps	$0, (%rax){1to8}, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpermilps	$0, %ymm16, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermilps	$0, (%rax), %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermilps	$0, (%rax){1to8}, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpermilps	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermilps	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermilps	(%rax){1to4}, %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpermilps	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermilps	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermilps	(%rax){1to4}, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpermilps	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermilps	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermilps	(%rax){1to4}, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpermilps	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermilps	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermilps	(%rax){1to8}, %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpermilps	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermilps	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermilps	(%rax){1to8}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpermilps	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermilps	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermilps	(%rax){1to8}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpermpd	$0, %ymm16, %ymm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermpd	$0, (%rax), %ymm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermpd	$0, (%rax){1to4}, %ymm19
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpermpd	$0, %ymm16, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermpd	$0, (%rax), %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermpd	$0, (%rax){1to4}, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpermpd	$0, %ymm16, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermpd	$0, (%rax), %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermpd	$0, (%rax){1to4}, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpermpd	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermpd	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermpd	(%rax){1to4}, %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpermpd	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermpd	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermpd	(%rax){1to4}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpermpd	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermpd	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermpd	(%rax){1to4}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpermps	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermps	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermps	(%rax){1to8}, %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpermps	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermps	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermps	(%rax){1to8}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpermps	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermps	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermps	(%rax){1to8}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpermq	$0, %ymm16, %ymm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermq	$0, (%rax), %ymm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermq	$0, (%rax){1to4}, %ymm19
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpermq	$0, %ymm16, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermq	$0, (%rax), %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermq	$0, (%rax){1to4}, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpermq	$0, %ymm16, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermq	$0, (%rax), %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermq	$0, (%rax){1to4}, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpermq	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermq	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermq	(%rax){1to4}, %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpermq	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermq	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermq	(%rax){1to4}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpermq	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermq	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpermq	(%rax){1to4}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vpmulld	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  -      -     1.00    -      -      -     0.50   0.50   vpmulld	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  -      -     1.00    -      -      -     0.50   0.50   vpmulld	(%rax){1to4}, %xmm17, %xmm19
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vpmulld	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -     1.00    -      -      -     0.50   0.50   vpmulld	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -     1.00    -      -      -     0.50   0.50   vpmulld	(%rax){1to4}, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vpmulld	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     1.00    -      -      -     0.50   0.50   vpmulld	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     1.00    -      -      -     0.50   0.50   vpmulld	(%rax){1to4}, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vpmulld	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  -      -     1.00    -      -      -     0.50   0.50   vpmulld	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  -      -     1.00    -      -      -     0.50   0.50   vpmulld	(%rax){1to8}, %ymm17, %ymm19
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vpmulld	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -     1.00    -      -      -     0.50   0.50   vpmulld	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -     1.00    -      -      -     0.50   0.50   vpmulld	(%rax){1to8}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     vpmulld	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -     1.00    -      -      -     0.50   0.50   vpmulld	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -     1.00    -      -      -     0.50   0.50   vpmulld	(%rax){1to8}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpshufd	$0, %xmm16, %xmm19
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpshufd	$0, (%rax), %xmm19
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpshufd	$0, (%rax){1to4}, %xmm19
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpshufd	$0, %xmm16, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpshufd	$0, (%rax), %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpshufd	$0, (%rax){1to4}, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpshufd	$0, %xmm16, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpshufd	$0, (%rax), %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpshufd	$0, (%rax){1to4}, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpshufd	$0, %ymm16, %ymm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpshufd	$0, (%rax), %ymm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpshufd	$0, (%rax){1to8}, %ymm19
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpshufd	$0, %ymm16, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpshufd	$0, (%rax), %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpshufd	$0, (%rax){1to8}, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpshufd	$0, %ymm16, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpshufd	$0, (%rax), %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpshufd	$0, (%rax){1to8}, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpsubd	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpsubd	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpsubd	(%rax){1to4}, %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpsubd	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpsubd	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpsubd	(%rax){1to4}, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpsubd	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpsubd	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpsubd	(%rax){1to4}, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpsubd	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpsubd	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpsubd	(%rax){1to8}, %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpsubd	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpsubd	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpsubd	(%rax){1to8}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpsubd	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpsubd	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpsubd	(%rax){1to8}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpsubq	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpsubq	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpsubq	(%rax){1to2}, %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpsubq	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpsubq	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpsubq	(%rax){1to2}, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpsubq	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpsubq	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpsubq	(%rax){1to2}, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpsubq	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpsubq	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpsubq	(%rax){1to4}, %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpsubq	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpsubq	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpsubq	(%rax){1to4}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpsubq	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpsubq	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpsubq	(%rax){1to4}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpunpckhdq	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpunpckhdq	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpunpckhdq	(%rax){1to4}, %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpunpckhdq	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpunpckhdq	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpunpckhdq	(%rax){1to4}, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpunpckhdq	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpunpckhdq	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpunpckhdq	(%rax){1to4}, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpunpckhdq	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpunpckhdq	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpunpckhdq	(%rax){1to8}, %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpunpckhdq	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpunpckhdq	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpunpckhdq	(%rax){1to8}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpunpckhdq	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpunpckhdq	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpunpckhdq	(%rax){1to8}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpunpckldq	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpunpckldq	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpunpckldq	(%rax){1to4}, %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpunpckldq	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpunpckldq	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpunpckldq	(%rax){1to4}, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpunpckldq	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpunpckldq	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpunpckldq	(%rax){1to4}, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpunpckldq	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpunpckldq	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpunpckldq	(%rax){1to8}, %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpunpckldq	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpunpckldq	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpunpckldq	(%rax){1to8}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vpunpckldq	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpunpckldq	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vpunpckldq	(%rax){1to8}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vshuff32x4	$0, %ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vshuff32x4	$0, (%rax), %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vshuff32x4	$0, (%rax){1to8}, %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vshuff32x4	$0, %ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vshuff32x4	$0, (%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vshuff32x4	$0, (%rax){1to8}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vshuff32x4	$0, %ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vshuff32x4	$0, (%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vshuff32x4	$0, (%rax){1to8}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vshuff64x2	$0, %ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vshuff64x2	$0, (%rax), %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vshuff64x2	$0, (%rax){1to4}, %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vshuff64x2	$0, %ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vshuff64x2	$0, (%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vshuff64x2	$0, (%rax){1to4}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vshuff64x2	$0, %ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vshuff64x2	$0, (%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vshuff64x2	$0, (%rax){1to4}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vshufi32x4	$0, %ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vshufi32x4	$0, (%rax), %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vshufi32x4	$0, (%rax){1to8}, %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vshufi32x4	$0, %ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vshufi32x4	$0, (%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vshufi32x4	$0, (%rax){1to8}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vshufi32x4	$0, %ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vshufi32x4	$0, (%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vshufi32x4	$0, (%rax){1to8}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vshufi64x2	$0, %ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vshufi64x2	$0, (%rax), %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vshufi64x2	$0, (%rax){1to4}, %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vshufi64x2	$0, %ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vshufi64x2	$0, (%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vshufi64x2	$0, (%rax){1to4}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vshufi64x2	$0, %ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vshufi64x2	$0, (%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vshufi64x2	$0, (%rax){1to4}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -     21.00  1.00    -      -      -      -      -     vsqrtpd	%xmm16, %xmm19
+# CHECK-NEXT:  -     21.00  1.00    -      -      -     0.50   0.50   vsqrtpd	(%rax), %xmm19
+# CHECK-NEXT:  -     21.00  1.00    -      -      -     0.50   0.50   vsqrtpd	(%rax){1to2}, %xmm19
+# CHECK-NEXT:  -     21.00  1.00    -      -      -      -      -     vsqrtpd	%xmm16, %xmm19 {%k1}
+# CHECK-NEXT:  -     21.00  1.00    -      -      -     0.50   0.50   vsqrtpd	(%rax), %xmm19 {%k1}
+# CHECK-NEXT:  -     21.00  1.00    -      -      -     0.50   0.50   vsqrtpd	(%rax){1to2}, %xmm19 {%k1}
+# CHECK-NEXT:  -     21.00  1.00    -      -      -      -      -     vsqrtpd	%xmm16, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -     21.00  1.00    -      -      -     0.50   0.50   vsqrtpd	(%rax), %xmm19 {%k1} {z}
+# CHECK-NEXT:  -     21.00  1.00    -      -      -     0.50   0.50   vsqrtpd	(%rax){1to2}, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -     44.00  2.50    -      -     0.50    -      -     vsqrtpd	%ymm16, %ymm19
+# CHECK-NEXT:  -     44.00  2.50    -      -     0.50   0.50   0.50   vsqrtpd	(%rax), %ymm19
+# CHECK-NEXT:  -     44.00  2.50    -      -     0.50   0.50   0.50   vsqrtpd	(%rax){1to4}, %ymm19
+# CHECK-NEXT:  -     44.00  2.50    -      -     0.50    -      -     vsqrtpd	%ymm16, %ymm19 {%k1}
+# CHECK-NEXT:  -     44.00  2.50    -      -     0.50   0.50   0.50   vsqrtpd	(%rax), %ymm19 {%k1}
+# CHECK-NEXT:  -     44.00  2.50    -      -     0.50   0.50   0.50   vsqrtpd	(%rax){1to4}, %ymm19 {%k1}
+# CHECK-NEXT:  -     44.00  2.50    -      -     0.50    -      -     vsqrtpd	%ymm16, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -     44.00  2.50    -      -     0.50   0.50   0.50   vsqrtpd	(%rax), %ymm19 {%k1} {z}
+# CHECK-NEXT:  -     44.00  2.50    -      -     0.50   0.50   0.50   vsqrtpd	(%rax){1to4}, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -     14.00  1.00    -      -      -      -      -     vsqrtps	%xmm16, %xmm19
+# CHECK-NEXT:  -     14.00  1.00    -      -      -     0.50   0.50   vsqrtps	(%rax), %xmm19
+# CHECK-NEXT:  -     14.00  1.00    -      -      -     0.50   0.50   vsqrtps	(%rax){1to4}, %xmm19
+# CHECK-NEXT:  -     14.00  1.00    -      -      -      -      -     vsqrtps	%xmm16, %xmm19 {%k1}
+# CHECK-NEXT:  -     14.00  1.00    -      -      -     0.50   0.50   vsqrtps	(%rax), %xmm19 {%k1}
+# CHECK-NEXT:  -     14.00  1.00    -      -      -     0.50   0.50   vsqrtps	(%rax){1to4}, %xmm19 {%k1}
+# CHECK-NEXT:  -     14.00  1.00    -      -      -      -      -     vsqrtps	%xmm16, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -     14.00  1.00    -      -      -     0.50   0.50   vsqrtps	(%rax), %xmm19 {%k1} {z}
+# CHECK-NEXT:  -     14.00  1.00    -      -      -     0.50   0.50   vsqrtps	(%rax){1to4}, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -     28.00  2.50    -      -     0.50    -      -     vsqrtps	%ymm16, %ymm19
+# CHECK-NEXT:  -     28.00  2.50    -      -     0.50   0.50   0.50   vsqrtps	(%rax), %ymm19
+# CHECK-NEXT:  -     28.00  2.50    -      -     0.50   0.50   0.50   vsqrtps	(%rax){1to8}, %ymm19
+# CHECK-NEXT:  -     28.00  2.50    -      -     0.50    -      -     vsqrtps	%ymm16, %ymm19 {%k1}
+# CHECK-NEXT:  -     28.00  2.50    -      -     0.50   0.50   0.50   vsqrtps	(%rax), %ymm19 {%k1}
+# CHECK-NEXT:  -     28.00  2.50    -      -     0.50   0.50   0.50   vsqrtps	(%rax){1to8}, %ymm19 {%k1}
+# CHECK-NEXT:  -     28.00  2.50    -      -     0.50    -      -     vsqrtps	%ymm16, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -     28.00  2.50    -      -     0.50   0.50   0.50   vsqrtps	(%rax), %ymm19 {%k1} {z}
+# CHECK-NEXT:  -     28.00  2.50    -      -     0.50   0.50   0.50   vsqrtps	(%rax){1to8}, %ymm19 {%k1} {z}
 # CHECK-NEXT:  -      -      -     1.00    -      -      -      -     vsubpd	%xmm16, %xmm17, %xmm19
 # CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vsubpd	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vsubpd	(%rax){1to2}, %xmm17, %xmm19
 # CHECK-NEXT:  -      -      -     1.00    -      -      -      -     vsubpd	%xmm16, %xmm17, %xmm19 {%k1}
 # CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vsubpd	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vsubpd	(%rax){1to2}, %xmm17, %xmm19 {%k1}
 # CHECK-NEXT:  -      -      -     1.00    -      -      -      -     vsubpd	%xmm16, %xmm17, %xmm19 {%k1} {z}
 # CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vsubpd	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vsubpd	(%rax){1to2}, %xmm17, %xmm19 {%k1} {z}
 # CHECK-NEXT:  -      -      -     1.00    -      -      -      -     vsubpd	%ymm16, %ymm17, %ymm19
 # CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vsubpd	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vsubpd	(%rax){1to4}, %ymm17, %ymm19
 # CHECK-NEXT:  -      -      -     1.00    -      -      -      -     vsubpd	%ymm16, %ymm17, %ymm19 {%k1}
 # CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vsubpd	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vsubpd	(%rax){1to4}, %ymm17, %ymm19 {%k1}
 # CHECK-NEXT:  -      -      -     1.00    -      -      -      -     vsubpd	%ymm16, %ymm17, %ymm19 {%k1} {z}
 # CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vsubpd	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vsubpd	(%rax){1to4}, %ymm17, %ymm19 {%k1} {z}
 # CHECK-NEXT:  -      -      -     1.00    -      -      -      -     vsubps	%xmm16, %xmm17, %xmm19
 # CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vsubps	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vsubps	(%rax){1to4}, %xmm17, %xmm19
 # CHECK-NEXT:  -      -      -     1.00    -      -      -      -     vsubps	%xmm16, %xmm17, %xmm19 {%k1}
 # CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vsubps	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vsubps	(%rax){1to4}, %xmm17, %xmm19 {%k1}
 # CHECK-NEXT:  -      -      -     1.00    -      -      -      -     vsubps	%xmm16, %xmm17, %xmm19 {%k1} {z}
 # CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vsubps	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vsubps	(%rax){1to4}, %xmm17, %xmm19 {%k1} {z}
 # CHECK-NEXT:  -      -      -     1.00    -      -      -      -     vsubps	%ymm16, %ymm17, %ymm19
 # CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vsubps	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vsubps	(%rax){1to8}, %ymm17, %ymm19
 # CHECK-NEXT:  -      -      -     1.00    -      -      -      -     vsubps	%ymm16, %ymm17, %ymm19 {%k1}
 # CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vsubps	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vsubps	(%rax){1to8}, %ymm17, %ymm19 {%k1}
 # CHECK-NEXT:  -      -      -     1.00    -      -      -      -     vsubps	%ymm16, %ymm17, %ymm19 {%k1} {z}
 # CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vsubps	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   vsubps	(%rax){1to8}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vunpckhpd	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vunpckhpd	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vunpckhpd	(%rax){1to2}, %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vunpckhpd	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vunpckhpd	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vunpckhpd	(%rax){1to2}, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vunpckhpd	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vunpckhpd	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vunpckhpd	(%rax){1to2}, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vunpckhpd	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vunpckhpd	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vunpckhpd	(%rax){1to4}, %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vunpckhpd	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vunpckhpd	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vunpckhpd	(%rax){1to4}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vunpckhpd	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vunpckhpd	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vunpckhpd	(%rax){1to4}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vunpckhps	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vunpckhps	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vunpckhps	(%rax){1to4}, %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vunpckhps	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vunpckhps	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vunpckhps	(%rax){1to4}, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vunpckhps	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vunpckhps	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vunpckhps	(%rax){1to4}, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vunpckhps	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vunpckhps	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vunpckhps	(%rax){1to8}, %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vunpckhps	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vunpckhps	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vunpckhps	(%rax){1to8}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vunpckhps	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vunpckhps	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vunpckhps	(%rax){1to8}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vunpcklpd	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vunpcklpd	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vunpcklpd	(%rax){1to2}, %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vunpcklpd	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vunpcklpd	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vunpcklpd	(%rax){1to2}, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vunpcklpd	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vunpcklpd	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vunpcklpd	(%rax){1to2}, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vunpcklpd	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vunpcklpd	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vunpcklpd	(%rax){1to4}, %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vunpcklpd	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vunpcklpd	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vunpcklpd	(%rax){1to4}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vunpcklpd	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vunpcklpd	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vunpcklpd	(%rax){1to4}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vunpcklps	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vunpcklps	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vunpcklps	(%rax){1to4}, %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vunpcklps	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vunpcklps	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vunpcklps	(%rax){1to4}, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vunpcklps	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vunpcklps	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vunpcklps	(%rax){1to4}, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vunpcklps	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vunpcklps	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vunpcklps	(%rax){1to8}, %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vunpcklps	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vunpcklps	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vunpcklps	(%rax){1to8}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -     1.00    -      -     vunpcklps	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vunpcklps	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -     1.00   0.50   0.50   vunpcklps	(%rax){1to8}, %ymm17, %ymm19 {%k1} {z}
diff --git a/test/tools/llvm-mca/X86/Generic/resources-avx512vpopcntdq.s b/test/tools/llvm-mca/X86/Generic/resources-avx512vpopcntdq.s
new file mode 100644
index 0000000000000000000000000000000000000000..f754e63cb8045e2cbdcb0cd9c77ce725b6fb7c6a
--- /dev/null
+++ b/test/tools/llvm-mca/X86/Generic/resources-avx512vpopcntdq.s
@@ -0,0 +1,89 @@
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -instruction-tables < %s | FileCheck %s
+
+vpopcntd    %zmm1, %zmm0
+vpopcntd    (%rdi), %zmm0
+vpopcntd    (%rdi){1to16}, %zmm0
+
+vpopcntd    %zmm1, %zmm0 {%k1}
+vpopcntd    (%rdi), %zmm0 {%k1}
+vpopcntd    (%rdi){1to16}, %zmm0 {%k1}
+
+vpopcntd    %zmm1, %zmm0 {%k1} {z}
+vpopcntd    (%rdi), %zmm0 {%k1} {z}
+vpopcntd    (%rdi){1to16}, %zmm0 {%k1} {z}
+
+vpopcntq    %zmm1, %zmm0
+vpopcntq    (%rdi), %zmm0
+vpopcntq    (%rdi){1to8}, %zmm0
+
+vpopcntq    %zmm1, %zmm0 {%k1}
+vpopcntq    (%rdi), %zmm0 {%k1}
+vpopcntq    (%rdi){1to8}, %zmm0 {%k1}
+
+vpopcntq    %zmm1, %zmm0 {%k1} {z}
+vpopcntq    (%rdi), %zmm0 {%k1} {z}
+vpopcntq    (%rdi){1to8}, %zmm0 {%k1} {z}
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  1      1     0.50                        vpopcntd	%zmm1, %zmm0
+# CHECK-NEXT:  2      8     0.50    *                   vpopcntd	(%rdi), %zmm0
+# CHECK-NEXT:  2      8     0.50    *                   vpopcntd	(%rdi){1to16}, %zmm0
+# CHECK-NEXT:  1      1     0.50                        vpopcntd	%zmm1, %zmm0 {%k1}
+# CHECK-NEXT:  2      8     0.50    *                   vpopcntd	(%rdi), %zmm0 {%k1}
+# CHECK-NEXT:  2      8     0.50    *                   vpopcntd	(%rdi){1to16}, %zmm0 {%k1}
+# CHECK-NEXT:  1      1     0.50                        vpopcntd	%zmm1, %zmm0 {%k1} {z}
+# CHECK-NEXT:  2      8     0.50    *                   vpopcntd	(%rdi), %zmm0 {%k1} {z}
+# CHECK-NEXT:  2      8     0.50    *                   vpopcntd	(%rdi){1to16}, %zmm0 {%k1} {z}
+# CHECK-NEXT:  1      1     0.50                        vpopcntq	%zmm1, %zmm0
+# CHECK-NEXT:  2      8     0.50    *                   vpopcntq	(%rdi), %zmm0
+# CHECK-NEXT:  2      8     0.50    *                   vpopcntq	(%rdi){1to8}, %zmm0
+# CHECK-NEXT:  1      1     0.50                        vpopcntq	%zmm1, %zmm0 {%k1}
+# CHECK-NEXT:  2      8     0.50    *                   vpopcntq	(%rdi), %zmm0 {%k1}
+# CHECK-NEXT:  2      8     0.50    *                   vpopcntq	(%rdi){1to8}, %zmm0 {%k1}
+# CHECK-NEXT:  1      1     0.50                        vpopcntq	%zmm1, %zmm0 {%k1} {z}
+# CHECK-NEXT:  2      8     0.50    *                   vpopcntq	(%rdi), %zmm0 {%k1} {z}
+# CHECK-NEXT:  2      8     0.50    *                   vpopcntq	(%rdi){1to8}, %zmm0 {%k1} {z}
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0]   - SBDivider
+# CHECK-NEXT: [1]   - SBFPDivider
+# CHECK-NEXT: [2]   - SBPort0
+# CHECK-NEXT: [3]   - SBPort1
+# CHECK-NEXT: [4]   - SBPort4
+# CHECK-NEXT: [5]   - SBPort5
+# CHECK-NEXT: [6.0] - SBPort23
+# CHECK-NEXT: [6.1] - SBPort23
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6.0]  [6.1]
+# CHECK-NEXT:  -      -      -     9.00    -     9.00   6.00   6.00
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6.0]  [6.1]  Instructions:
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpopcntd	%zmm1, %zmm0
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpopcntd	(%rdi), %zmm0
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpopcntd	(%rdi){1to16}, %zmm0
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpopcntd	%zmm1, %zmm0 {%k1}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpopcntd	(%rdi), %zmm0 {%k1}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpopcntd	(%rdi){1to16}, %zmm0 {%k1}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpopcntd	%zmm1, %zmm0 {%k1} {z}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpopcntd	(%rdi), %zmm0 {%k1} {z}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpopcntd	(%rdi){1to16}, %zmm0 {%k1} {z}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpopcntq	%zmm1, %zmm0
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpopcntq	(%rdi), %zmm0
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpopcntq	(%rdi){1to8}, %zmm0
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpopcntq	%zmm1, %zmm0 {%k1}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpopcntq	(%rdi), %zmm0 {%k1}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpopcntq	(%rdi){1to8}, %zmm0 {%k1}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpopcntq	%zmm1, %zmm0 {%k1} {z}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpopcntq	(%rdi), %zmm0 {%k1} {z}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpopcntq	(%rdi){1to8}, %zmm0 {%k1} {z}
diff --git a/test/tools/llvm-mca/X86/Generic/resources-avx512vpopcntdqvl.s b/test/tools/llvm-mca/X86/Generic/resources-avx512vpopcntdqvl.s
new file mode 100644
index 0000000000000000000000000000000000000000..852f297634e9d10b7a9025409a1bf7a4a5aa002a
--- /dev/null
+++ b/test/tools/llvm-mca/X86/Generic/resources-avx512vpopcntdqvl.s
@@ -0,0 +1,149 @@
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -instruction-tables < %s | FileCheck %s
+
+vpopcntd    %xmm1, %xmm0
+vpopcntd    (%rdi), %xmm0
+vpopcntd    (%rdi){1to4}, %xmm0
+
+vpopcntd    %xmm1, %xmm0 {%k1}
+vpopcntd    (%rdi), %xmm0 {%k1}
+vpopcntd    (%rdi){1to4}, %xmm0 {%k1}
+
+vpopcntd    %xmm1, %xmm0 {%k1} {z}
+vpopcntd    (%rdi), %xmm0 {%k1} {z}
+vpopcntd    (%rdi){1to4}, %xmm0 {%k1} {z}
+
+vpopcntd    %ymm1, %ymm0
+vpopcntd    (%rdi), %ymm0
+vpopcntd    (%rdi){1to8}, %ymm0
+
+vpopcntd    %ymm1, %ymm0 {%k1}
+vpopcntd    (%rdi), %ymm0 {%k1}
+vpopcntd    (%rdi){1to8}, %ymm0 {%k1}
+
+vpopcntd    %ymm1, %ymm0 {%k1} {z}
+vpopcntd    (%rdi), %ymm0 {%k1} {z}
+vpopcntd    (%rdi){1to8}, %ymm0 {%k1} {z}
+
+vpopcntq    %xmm1, %xmm0
+vpopcntq    (%rdi), %xmm0
+vpopcntq    (%rdi){1to2}, %xmm0
+
+vpopcntq    %xmm1, %xmm0 {%k1}
+vpopcntq    (%rdi), %xmm0 {%k1}
+vpopcntq    (%rdi){1to2}, %xmm0 {%k1}
+
+vpopcntq    %xmm1, %xmm0 {%k1} {z}
+vpopcntq    (%rdi), %xmm0 {%k1} {z}
+vpopcntq    (%rdi){1to2}, %xmm0 {%k1} {z}
+
+vpopcntq    %ymm1, %ymm0
+vpopcntq    (%rdi), %ymm0
+vpopcntq    (%rdi){1to4}, %ymm0
+
+vpopcntq    %ymm1, %ymm0 {%k1}
+vpopcntq    (%rdi), %ymm0 {%k1}
+vpopcntq    (%rdi){1to4}, %ymm0 {%k1}
+
+vpopcntq    %ymm1, %ymm0 {%k1} {z}
+vpopcntq    (%rdi), %ymm0 {%k1} {z}
+vpopcntq    (%rdi){1to4}, %ymm0 {%k1} {z}
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  1      1     0.50                        vpopcntd	%xmm1, %xmm0
+# CHECK-NEXT:  2      7     0.50    *                   vpopcntd	(%rdi), %xmm0
+# CHECK-NEXT:  2      7     0.50    *                   vpopcntd	(%rdi){1to4}, %xmm0
+# CHECK-NEXT:  1      1     0.50                        vpopcntd	%xmm1, %xmm0 {%k1}
+# CHECK-NEXT:  2      7     0.50    *                   vpopcntd	(%rdi), %xmm0 {%k1}
+# CHECK-NEXT:  2      7     0.50    *                   vpopcntd	(%rdi){1to4}, %xmm0 {%k1}
+# CHECK-NEXT:  1      1     0.50                        vpopcntd	%xmm1, %xmm0 {%k1} {z}
+# CHECK-NEXT:  2      7     0.50    *                   vpopcntd	(%rdi), %xmm0 {%k1} {z}
+# CHECK-NEXT:  2      7     0.50    *                   vpopcntd	(%rdi){1to4}, %xmm0 {%k1} {z}
+# CHECK-NEXT:  1      1     0.50                        vpopcntd	%ymm1, %ymm0
+# CHECK-NEXT:  2      8     0.50    *                   vpopcntd	(%rdi), %ymm0
+# CHECK-NEXT:  2      8     0.50    *                   vpopcntd	(%rdi){1to8}, %ymm0
+# CHECK-NEXT:  1      1     0.50                        vpopcntd	%ymm1, %ymm0 {%k1}
+# CHECK-NEXT:  2      8     0.50    *                   vpopcntd	(%rdi), %ymm0 {%k1}
+# CHECK-NEXT:  2      8     0.50    *                   vpopcntd	(%rdi){1to8}, %ymm0 {%k1}
+# CHECK-NEXT:  1      1     0.50                        vpopcntd	%ymm1, %ymm0 {%k1} {z}
+# CHECK-NEXT:  2      8     0.50    *                   vpopcntd	(%rdi), %ymm0 {%k1} {z}
+# CHECK-NEXT:  2      8     0.50    *                   vpopcntd	(%rdi){1to8}, %ymm0 {%k1} {z}
+# CHECK-NEXT:  1      1     0.50                        vpopcntq	%xmm1, %xmm0
+# CHECK-NEXT:  2      7     0.50    *                   vpopcntq	(%rdi), %xmm0
+# CHECK-NEXT:  2      7     0.50    *                   vpopcntq	(%rdi){1to2}, %xmm0
+# CHECK-NEXT:  1      1     0.50                        vpopcntq	%xmm1, %xmm0 {%k1}
+# CHECK-NEXT:  2      7     0.50    *                   vpopcntq	(%rdi), %xmm0 {%k1}
+# CHECK-NEXT:  2      7     0.50    *                   vpopcntq	(%rdi){1to2}, %xmm0 {%k1}
+# CHECK-NEXT:  1      1     0.50                        vpopcntq	%xmm1, %xmm0 {%k1} {z}
+# CHECK-NEXT:  2      7     0.50    *                   vpopcntq	(%rdi), %xmm0 {%k1} {z}
+# CHECK-NEXT:  2      7     0.50    *                   vpopcntq	(%rdi){1to2}, %xmm0 {%k1} {z}
+# CHECK-NEXT:  1      1     0.50                        vpopcntq	%ymm1, %ymm0
+# CHECK-NEXT:  2      8     0.50    *                   vpopcntq	(%rdi), %ymm0
+# CHECK-NEXT:  2      8     0.50    *                   vpopcntq	(%rdi){1to4}, %ymm0
+# CHECK-NEXT:  1      1     0.50                        vpopcntq	%ymm1, %ymm0 {%k1}
+# CHECK-NEXT:  2      8     0.50    *                   vpopcntq	(%rdi), %ymm0 {%k1}
+# CHECK-NEXT:  2      8     0.50    *                   vpopcntq	(%rdi){1to4}, %ymm0 {%k1}
+# CHECK-NEXT:  1      1     0.50                        vpopcntq	%ymm1, %ymm0 {%k1} {z}
+# CHECK-NEXT:  2      8     0.50    *                   vpopcntq	(%rdi), %ymm0 {%k1} {z}
+# CHECK-NEXT:  2      8     0.50    *                   vpopcntq	(%rdi){1to4}, %ymm0 {%k1} {z}
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0]   - SBDivider
+# CHECK-NEXT: [1]   - SBFPDivider
+# CHECK-NEXT: [2]   - SBPort0
+# CHECK-NEXT: [3]   - SBPort1
+# CHECK-NEXT: [4]   - SBPort4
+# CHECK-NEXT: [5]   - SBPort5
+# CHECK-NEXT: [6.0] - SBPort23
+# CHECK-NEXT: [6.1] - SBPort23
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6.0]  [6.1]
+# CHECK-NEXT:  -      -      -     18.00   -     18.00  12.00  12.00
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6.0]  [6.1]  Instructions:
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpopcntd	%xmm1, %xmm0
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpopcntd	(%rdi), %xmm0
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpopcntd	(%rdi){1to4}, %xmm0
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpopcntd	%xmm1, %xmm0 {%k1}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpopcntd	(%rdi), %xmm0 {%k1}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpopcntd	(%rdi){1to4}, %xmm0 {%k1}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpopcntd	%xmm1, %xmm0 {%k1} {z}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpopcntd	(%rdi), %xmm0 {%k1} {z}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpopcntd	(%rdi){1to4}, %xmm0 {%k1} {z}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpopcntd	%ymm1, %ymm0
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpopcntd	(%rdi), %ymm0
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpopcntd	(%rdi){1to8}, %ymm0
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpopcntd	%ymm1, %ymm0 {%k1}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpopcntd	(%rdi), %ymm0 {%k1}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpopcntd	(%rdi){1to8}, %ymm0 {%k1}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpopcntd	%ymm1, %ymm0 {%k1} {z}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpopcntd	(%rdi), %ymm0 {%k1} {z}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpopcntd	(%rdi){1to8}, %ymm0 {%k1} {z}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpopcntq	%xmm1, %xmm0
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpopcntq	(%rdi), %xmm0
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpopcntq	(%rdi){1to2}, %xmm0
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpopcntq	%xmm1, %xmm0 {%k1}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpopcntq	(%rdi), %xmm0 {%k1}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpopcntq	(%rdi){1to2}, %xmm0 {%k1}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpopcntq	%xmm1, %xmm0 {%k1} {z}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpopcntq	(%rdi), %xmm0 {%k1} {z}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpopcntq	(%rdi){1to2}, %xmm0 {%k1} {z}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpopcntq	%ymm1, %ymm0
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpopcntq	(%rdi), %ymm0
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpopcntq	(%rdi){1to4}, %ymm0
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpopcntq	%ymm1, %ymm0 {%k1}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpopcntq	(%rdi), %ymm0 {%k1}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpopcntq	(%rdi){1to4}, %ymm0 {%k1}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     vpopcntq	%ymm1, %ymm0 {%k1} {z}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpopcntq	(%rdi), %ymm0 {%k1} {z}
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   vpopcntq	(%rdi){1to4}, %ymm0 {%k1} {z}
diff --git a/test/tools/llvm-mca/X86/Generic/resources-bmi1.s b/test/tools/llvm-mca/X86/Generic/resources-bmi1.s
index 193f5537c7cf10b113f235a5e884084e19381276..87b748414d3548315aa79e82a04742942308390f 100644
--- a/test/tools/llvm-mca/X86/Generic/resources-bmi1.s
+++ b/test/tools/llvm-mca/X86/Generic/resources-bmi1.s
@@ -31,6 +31,9 @@ blsr        (%rax), %ecx
 blsr        %rax, %rcx
 blsr        (%rax), %rcx
 
+tzcnt       %ax, %cx
+tzcnt       (%rax), %cx
+
 tzcnt       %eax, %ecx
 tzcnt       (%rax), %ecx
 
@@ -66,6 +69,8 @@ tzcnt       (%rax), %rcx
 # CHECK-NEXT:  2      6     0.50    *                   blsrl	(%rax), %ecx
 # CHECK-NEXT:  1      1     0.33                        blsrq	%rax, %rcx
 # CHECK-NEXT:  2      6     0.50    *                   blsrq	(%rax), %rcx
+# CHECK-NEXT:  1      3     1.00                        tzcntw	%ax, %cx
+# CHECK-NEXT:  2      8     1.00    *                   tzcntw	(%rax), %cx
 # CHECK-NEXT:  1      3     1.00                        tzcntl	%eax, %ecx
 # CHECK-NEXT:  2      8     1.00    *                   tzcntl	(%rax), %ecx
 # CHECK-NEXT:  1      3     1.00                        tzcntq	%rax, %rcx
@@ -83,7 +88,7 @@ tzcnt       (%rax), %rcx
 
 # CHECK:      Resource pressure per iteration:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6.0]  [6.1]
-# CHECK-NEXT:  -      -     7.33   13.33   -     7.33   6.00   6.00
+# CHECK-NEXT:  -      -     7.33   15.33   -     7.33   6.50   6.50
 
 # CHECK:      Resource pressure by instruction:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6.0]  [6.1]  Instructions:
@@ -107,6 +112,8 @@ tzcnt       (%rax), %rcx
 # CHECK-NEXT:  -      -     0.33   0.33    -     0.33   0.50   0.50   blsrl	(%rax), %ecx
 # CHECK-NEXT:  -      -     0.33   0.33    -     0.33    -      -     blsrq	%rax, %rcx
 # CHECK-NEXT:  -      -     0.33   0.33    -     0.33   0.50   0.50   blsrq	(%rax), %rcx
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -     tzcntw	%ax, %cx
+# CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   tzcntw	(%rax), %cx
 # CHECK-NEXT:  -      -      -     1.00    -      -      -      -     tzcntl	%eax, %ecx
 # CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   tzcntl	(%rax), %ecx
 # CHECK-NEXT:  -      -      -     1.00    -      -      -      -     tzcntq	%rax, %rcx
diff --git a/test/tools/llvm-mca/X86/Generic/resources-clwb.s b/test/tools/llvm-mca/X86/Generic/resources-clwb.s
new file mode 100644
index 0000000000000000000000000000000000000000..099edcda15aed3b81a4ad4bd2f6f6bab5df85391
--- /dev/null
+++ b/test/tools/llvm-mca/X86/Generic/resources-clwb.s
@@ -0,0 +1,33 @@
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -instruction-tables < %s | FileCheck %s
+
+clwb (%rax)
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  1      5     0.50    *      *      U     clwb	(%rax)
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0]   - SBDivider
+# CHECK-NEXT: [1]   - SBFPDivider
+# CHECK-NEXT: [2]   - SBPort0
+# CHECK-NEXT: [3]   - SBPort1
+# CHECK-NEXT: [4]   - SBPort4
+# CHECK-NEXT: [5]   - SBPort5
+# CHECK-NEXT: [6.0] - SBPort23
+# CHECK-NEXT: [6.1] - SBPort23
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6.0]  [6.1]
+# CHECK-NEXT:  -      -      -      -      -      -     0.50   0.50
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6.0]  [6.1]  Instructions:
+# CHECK-NEXT:  -      -      -      -      -      -     0.50   0.50   clwb	(%rax)
diff --git a/test/tools/llvm-mca/X86/Generic/resources-clzero.s b/test/tools/llvm-mca/X86/Generic/resources-clzero.s
new file mode 100644
index 0000000000000000000000000000000000000000..a13c25300b375250fce5aff82a21bc0886ed38b1
--- /dev/null
+++ b/test/tools/llvm-mca/X86/Generic/resources-clzero.s
@@ -0,0 +1,33 @@
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -instruction-tables < %s | FileCheck %s
+
+clzero
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  1      100   0.33                  U     clzero
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0]   - SBDivider
+# CHECK-NEXT: [1]   - SBFPDivider
+# CHECK-NEXT: [2]   - SBPort0
+# CHECK-NEXT: [3]   - SBPort1
+# CHECK-NEXT: [4]   - SBPort4
+# CHECK-NEXT: [5]   - SBPort5
+# CHECK-NEXT: [6.0] - SBPort23
+# CHECK-NEXT: [6.1] - SBPort23
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6.0]  [6.1]
+# CHECK-NEXT:  -      -     0.33   0.33    -     0.33    -      -
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6.0]  [6.1]  Instructions:
+# CHECK-NEXT:  -      -     0.33   0.33    -     0.33    -      -     clzero
diff --git a/test/tools/llvm-mca/X86/Generic/resources-fsgsbase.s b/test/tools/llvm-mca/X86/Generic/resources-fsgsbase.s
new file mode 100644
index 0000000000000000000000000000000000000000..7b0642214bae6defa1e8701d47658abc34e89a83
--- /dev/null
+++ b/test/tools/llvm-mca/X86/Generic/resources-fsgsbase.s
@@ -0,0 +1,57 @@
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -instruction-tables < %s | FileCheck %s
+
+rdfsbase %eax
+rdfsbase %rax
+
+rdgsbase %eax
+rdgsbase %rax
+
+wrfsbase %edi
+wrfsbase %rdi
+
+wrgsbase %edi
+wrgsbase %rdi
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  1      100   0.33    *      *      U     rdfsbasel	%eax
+# CHECK-NEXT:  1      100   0.33    *      *      U     rdfsbaseq	%rax
+# CHECK-NEXT:  1      100   0.33    *      *      U     rdgsbasel	%eax
+# CHECK-NEXT:  1      100   0.33    *      *      U     rdgsbaseq	%rax
+# CHECK-NEXT:  1      100   0.33    *      *      U     wrfsbasel	%edi
+# CHECK-NEXT:  1      100   0.33    *      *      U     wrfsbaseq	%rdi
+# CHECK-NEXT:  1      100   0.33    *      *      U     wrgsbasel	%edi
+# CHECK-NEXT:  1      100   0.33    *      *      U     wrgsbaseq	%rdi
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0]   - SBDivider
+# CHECK-NEXT: [1]   - SBFPDivider
+# CHECK-NEXT: [2]   - SBPort0
+# CHECK-NEXT: [3]   - SBPort1
+# CHECK-NEXT: [4]   - SBPort4
+# CHECK-NEXT: [5]   - SBPort5
+# CHECK-NEXT: [6.0] - SBPort23
+# CHECK-NEXT: [6.1] - SBPort23
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6.0]  [6.1]
+# CHECK-NEXT:  -      -     2.67   2.67    -     2.67    -      -
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6.0]  [6.1]  Instructions:
+# CHECK-NEXT:  -      -     0.33   0.33    -     0.33    -      -     rdfsbasel	%eax
+# CHECK-NEXT:  -      -     0.33   0.33    -     0.33    -      -     rdfsbaseq	%rax
+# CHECK-NEXT:  -      -     0.33   0.33    -     0.33    -      -     rdgsbasel	%eax
+# CHECK-NEXT:  -      -     0.33   0.33    -     0.33    -      -     rdgsbaseq	%rax
+# CHECK-NEXT:  -      -     0.33   0.33    -     0.33    -      -     wrfsbasel	%edi
+# CHECK-NEXT:  -      -     0.33   0.33    -     0.33    -      -     wrfsbaseq	%rdi
+# CHECK-NEXT:  -      -     0.33   0.33    -     0.33    -      -     wrgsbasel	%edi
+# CHECK-NEXT:  -      -     0.33   0.33    -     0.33    -      -     wrgsbaseq	%rdi
diff --git a/test/tools/llvm-mca/X86/Generic/resources-lwp.s b/test/tools/llvm-mca/X86/Generic/resources-lwp.s
new file mode 100644
index 0000000000000000000000000000000000000000..0de376544a405804284facd057fb859f2880c5c2
--- /dev/null
+++ b/test/tools/llvm-mca/X86/Generic/resources-lwp.s
@@ -0,0 +1,71 @@
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -instruction-tables < %s | FileCheck %s
+
+llwpcb %edi
+llwpcb %rdi
+
+lwpins $-1985229329, %esi, %edi
+lwpins $-1985229329, (%rsi), %edi
+
+lwpins $-1985229329, %esi, %rdi
+lwpins $-1985229329, (%rsi), %rdi
+
+lwpval $-1985229329, %esi, %edi
+lwpval $-1985229329, (%rsi), %edi
+
+lwpval $-1985229329, %esi, %rdi
+lwpval $-1985229329, (%rsi), %rdi
+
+slwpcb %edi
+slwpcb %rdi
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  1      100   0.33    *      *      U     llwpcb	%edi
+# CHECK-NEXT:  1      100   0.33    *      *      U     llwpcb	%rdi
+# CHECK-NEXT:  1      100   0.33    *      *      U     lwpins	$-1985229329, %esi, %edi
+# CHECK-NEXT:  1      100   0.33    *      *      U     lwpins	$-1985229329, (%rsi), %edi
+# CHECK-NEXT:  1      100   0.33    *      *      U     lwpins	$-1985229329, %esi, %rdi
+# CHECK-NEXT:  1      100   0.33    *      *      U     lwpins	$-1985229329, (%rsi), %rdi
+# CHECK-NEXT:  1      100   0.33    *      *      U     lwpval	$-1985229329, %esi, %edi
+# CHECK-NEXT:  1      100   0.33    *      *      U     lwpval	$-1985229329, (%rsi), %edi
+# CHECK-NEXT:  1      100   0.33    *      *      U     lwpval	$-1985229329, %esi, %rdi
+# CHECK-NEXT:  1      100   0.33    *      *      U     lwpval	$-1985229329, (%rsi), %rdi
+# CHECK-NEXT:  1      100   0.33    *      *      U     slwpcb	%edi
+# CHECK-NEXT:  1      100   0.33    *      *      U     slwpcb	%rdi
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0]   - SBDivider
+# CHECK-NEXT: [1]   - SBFPDivider
+# CHECK-NEXT: [2]   - SBPort0
+# CHECK-NEXT: [3]   - SBPort1
+# CHECK-NEXT: [4]   - SBPort4
+# CHECK-NEXT: [5]   - SBPort5
+# CHECK-NEXT: [6.0] - SBPort23
+# CHECK-NEXT: [6.1] - SBPort23
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6.0]  [6.1]
+# CHECK-NEXT:  -      -     4.00   4.00    -     4.00    -      -
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6.0]  [6.1]  Instructions:
+# CHECK-NEXT:  -      -     0.33   0.33    -     0.33    -      -     llwpcb	%edi
+# CHECK-NEXT:  -      -     0.33   0.33    -     0.33    -      -     llwpcb	%rdi
+# CHECK-NEXT:  -      -     0.33   0.33    -     0.33    -      -     lwpins	$-1985229329, %esi, %edi
+# CHECK-NEXT:  -      -     0.33   0.33    -     0.33    -      -     lwpins	$-1985229329, (%rsi), %edi
+# CHECK-NEXT:  -      -     0.33   0.33    -     0.33    -      -     lwpins	$-1985229329, %esi, %rdi
+# CHECK-NEXT:  -      -     0.33   0.33    -     0.33    -      -     lwpins	$-1985229329, (%rsi), %rdi
+# CHECK-NEXT:  -      -     0.33   0.33    -     0.33    -      -     lwpval	$-1985229329, %esi, %edi
+# CHECK-NEXT:  -      -     0.33   0.33    -     0.33    -      -     lwpval	$-1985229329, (%rsi), %edi
+# CHECK-NEXT:  -      -     0.33   0.33    -     0.33    -      -     lwpval	$-1985229329, %esi, %rdi
+# CHECK-NEXT:  -      -     0.33   0.33    -     0.33    -      -     lwpval	$-1985229329, (%rsi), %rdi
+# CHECK-NEXT:  -      -     0.33   0.33    -     0.33    -      -     slwpcb	%edi
+# CHECK-NEXT:  -      -     0.33   0.33    -     0.33    -      -     slwpcb	%rdi
diff --git a/test/tools/llvm-mca/X86/Generic/resources-mwaitx.s b/test/tools/llvm-mca/X86/Generic/resources-mwaitx.s
new file mode 100644
index 0000000000000000000000000000000000000000..517b283dafaca66c576bca23d629b26264550023
--- /dev/null
+++ b/test/tools/llvm-mca/X86/Generic/resources-mwaitx.s
@@ -0,0 +1,36 @@
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -instruction-tables < %s | FileCheck %s
+
+monitorx
+mwaitx
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  1      100   0.33                  U     monitorx
+# CHECK-NEXT:  1      100   0.33    *      *      U     mwaitx
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0]   - SBDivider
+# CHECK-NEXT: [1]   - SBFPDivider
+# CHECK-NEXT: [2]   - SBPort0
+# CHECK-NEXT: [3]   - SBPort1
+# CHECK-NEXT: [4]   - SBPort4
+# CHECK-NEXT: [5]   - SBPort5
+# CHECK-NEXT: [6.0] - SBPort23
+# CHECK-NEXT: [6.1] - SBPort23
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6.0]  [6.1]
+# CHECK-NEXT:  -      -     0.67   0.67    -     0.67    -      -
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6.0]  [6.1]  Instructions:
+# CHECK-NEXT:  -      -     0.33   0.33    -     0.33    -      -     monitorx
+# CHECK-NEXT:  -      -     0.33   0.33    -     0.33    -      -     mwaitx
diff --git a/test/tools/llvm-mca/X86/Generic/resources-rdpid.s b/test/tools/llvm-mca/X86/Generic/resources-rdpid.s
new file mode 100644
index 0000000000000000000000000000000000000000..ee7021dd2139fb108f86341591075316037c3216
--- /dev/null
+++ b/test/tools/llvm-mca/X86/Generic/resources-rdpid.s
@@ -0,0 +1,33 @@
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -instruction-tables < %s | FileCheck %s
+
+rdpid %rax
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  1      100   0.33                  U     rdpid	%rax
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0]   - SBDivider
+# CHECK-NEXT: [1]   - SBFPDivider
+# CHECK-NEXT: [2]   - SBPort0
+# CHECK-NEXT: [3]   - SBPort1
+# CHECK-NEXT: [4]   - SBPort4
+# CHECK-NEXT: [5]   - SBPort5
+# CHECK-NEXT: [6.0] - SBPort23
+# CHECK-NEXT: [6.1] - SBPort23
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6.0]  [6.1]
+# CHECK-NEXT:  -      -     0.33   0.33    -     0.33    -      -
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6.0]  [6.1]  Instructions:
+# CHECK-NEXT:  -      -     0.33   0.33    -     0.33    -      -     rdpid	%rax
diff --git a/test/tools/llvm-mca/X86/Generic/resources-sha.s b/test/tools/llvm-mca/X86/Generic/resources-sha.s
new file mode 100644
index 0000000000000000000000000000000000000000..71b37298fd611d7946647e979769fd1ec20859d2
--- /dev/null
+++ b/test/tools/llvm-mca/X86/Generic/resources-sha.s
@@ -0,0 +1,78 @@
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -instruction-tables < %s | FileCheck %s
+
+sha1msg1    %xmm0, %xmm2
+sha1msg1    (%rax), %xmm2
+
+sha1msg2    %xmm0, %xmm2
+sha1msg2    (%rax), %xmm2
+
+sha1nexte   %xmm0, %xmm2
+sha1nexte   (%rax), %xmm2
+
+sha1rnds4   $3, %xmm0, %xmm2
+sha1rnds4   $3, (%rax), %xmm2
+
+sha256msg1  %xmm0, %xmm2
+sha256msg1  (%rax), %xmm2
+
+sha256msg2  %xmm0, %xmm2
+sha256msg2  (%rax), %xmm2
+
+sha256rnds2 %xmm0, %xmm2
+sha256rnds2 (%rax), %xmm2
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  1      5     1.00                        sha1msg1	%xmm0, %xmm2
+# CHECK-NEXT:  2      11    1.00    *                   sha1msg1	(%rax), %xmm2
+# CHECK-NEXT:  1      5     1.00                        sha1msg2	%xmm0, %xmm2
+# CHECK-NEXT:  2      11    1.00    *                   sha1msg2	(%rax), %xmm2
+# CHECK-NEXT:  1      5     1.00                        sha1nexte	%xmm0, %xmm2
+# CHECK-NEXT:  2      11    1.00    *                   sha1nexte	(%rax), %xmm2
+# CHECK-NEXT:  1      5     1.00                        sha1rnds4	$3, %xmm0, %xmm2
+# CHECK-NEXT:  2      11    1.00    *                   sha1rnds4	$3, (%rax), %xmm2
+# CHECK-NEXT:  1      5     1.00                        sha256msg1	%xmm0, %xmm2
+# CHECK-NEXT:  2      11    1.00    *                   sha256msg1	(%rax), %xmm2
+# CHECK-NEXT:  1      5     1.00                        sha256msg2	%xmm0, %xmm2
+# CHECK-NEXT:  2      11    1.00    *                   sha256msg2	(%rax), %xmm2
+# CHECK-NEXT:  1      5     1.00                        sha256rnds2	%xmm0, %xmm0, %xmm2
+# CHECK-NEXT:  2      11    1.00    *                   sha256rnds2	%xmm0, (%rax), %xmm2
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0]   - SBDivider
+# CHECK-NEXT: [1]   - SBFPDivider
+# CHECK-NEXT: [2]   - SBPort0
+# CHECK-NEXT: [3]   - SBPort1
+# CHECK-NEXT: [4]   - SBPort4
+# CHECK-NEXT: [5]   - SBPort5
+# CHECK-NEXT: [6.0] - SBPort23
+# CHECK-NEXT: [6.1] - SBPort23
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6.0]  [6.1]
+# CHECK-NEXT:  -      -     14.00   -      -      -     3.50   3.50
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6.0]  [6.1]  Instructions:
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     sha1msg1	%xmm0, %xmm2
+# CHECK-NEXT:  -      -     1.00    -      -      -     0.50   0.50   sha1msg1	(%rax), %xmm2
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     sha1msg2	%xmm0, %xmm2
+# CHECK-NEXT:  -      -     1.00    -      -      -     0.50   0.50   sha1msg2	(%rax), %xmm2
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     sha1nexte	%xmm0, %xmm2
+# CHECK-NEXT:  -      -     1.00    -      -      -     0.50   0.50   sha1nexte	(%rax), %xmm2
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     sha1rnds4	$3, %xmm0, %xmm2
+# CHECK-NEXT:  -      -     1.00    -      -      -     0.50   0.50   sha1rnds4	$3, (%rax), %xmm2
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     sha256msg1	%xmm0, %xmm2
+# CHECK-NEXT:  -      -     1.00    -      -      -     0.50   0.50   sha256msg1	(%rax), %xmm2
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     sha256msg2	%xmm0, %xmm2
+# CHECK-NEXT:  -      -     1.00    -      -      -     0.50   0.50   sha256msg2	(%rax), %xmm2
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     sha256rnds2	%xmm0, %xmm0, %xmm2
+# CHECK-NEXT:  -      -     1.00    -      -      -     0.50   0.50   sha256rnds2	%xmm0, (%rax), %xmm2
diff --git a/test/tools/llvm-mca/X86/Generic/resources-sse2.s b/test/tools/llvm-mca/X86/Generic/resources-sse2.s
index 30534807cb31f3b3ca5e63c6051e58d7a32922c6..63a40d5c717145a87f97e73f9837bcb9f41b2b7e 100644
--- a/test/tools/llvm-mca/X86/Generic/resources-sse2.s
+++ b/test/tools/llvm-mca/X86/Generic/resources-sse2.s
@@ -94,6 +94,8 @@ maxpd       (%rax), %xmm2
 maxsd       %xmm0, %xmm2
 maxsd       (%rax), %xmm2
 
+mfence
+
 minpd       %xmm0, %xmm2
 minpd       (%rax), %xmm2
 
@@ -229,6 +231,9 @@ pcmpgtw     (%rax), %xmm2
 
 pextrw      $1, %xmm0, %rcx
 
+pinsrw      $1, %rax, %xmm0
+pinsrw      $1, (%rax), %xmm0
+
 pmaddwd     %xmm0, %xmm2
 pmaddwd     (%rax), %xmm2
 
@@ -465,6 +470,7 @@ xorpd       (%rax), %xmm2
 # CHECK-NEXT:  2      9     1.00    *                   maxpd	(%rax), %xmm2
 # CHECK-NEXT:  1      3     1.00                        maxsd	%xmm0, %xmm2
 # CHECK-NEXT:  2      9     1.00    *                   maxsd	(%rax), %xmm2
+# CHECK-NEXT:  1      1     1.00    *      *      U     mfence
 # CHECK-NEXT:  1      3     1.00                        minpd	%xmm0, %xmm2
 # CHECK-NEXT:  2      9     1.00    *                   minpd	(%rax), %xmm2
 # CHECK-NEXT:  1      3     1.00                        minsd	%xmm0, %xmm2
@@ -555,6 +561,8 @@ xorpd       (%rax), %xmm2
 # CHECK-NEXT:  1      1     0.50                        pcmpgtw	%xmm0, %xmm2
 # CHECK-NEXT:  2      7     0.50    *                   pcmpgtw	(%rax), %xmm2
 # CHECK-NEXT:  2      3     1.00                        pextrw	$1, %xmm0, %ecx
+# CHECK-NEXT:  2      2     1.00                        pinsrw	$1, %eax, %xmm0
+# CHECK-NEXT:  2      7     0.50    *                   pinsrw	$1, (%rax), %xmm0
 # CHECK-NEXT:  1      5     1.00                        pmaddwd	%xmm0, %xmm2
 # CHECK-NEXT:  2      11    1.00    *                   pmaddwd	(%rax), %xmm2
 # CHECK-NEXT:  1      1     0.50                        pmaxsw	%xmm0, %xmm2
@@ -679,7 +687,7 @@ xorpd       (%rax), %xmm2
 
 # CHECK:      Resource pressure per iteration:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6.0]  [6.1]
-# CHECK-NEXT:  -     172.00 75.83  117.33 16.00  98.83  66.00  66.00
+# CHECK-NEXT:  -     172.00 75.83  118.33 17.00  100.83 67.00  67.00
 
 # CHECK:      Resource pressure by instruction:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6.0]  [6.1]  Instructions:
@@ -746,6 +754,7 @@ xorpd       (%rax), %xmm2
 # CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   maxpd	(%rax), %xmm2
 # CHECK-NEXT:  -      -      -     1.00    -      -      -      -     maxsd	%xmm0, %xmm2
 # CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   maxsd	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -     1.00    -     0.50   0.50   mfence
 # CHECK-NEXT:  -      -      -     1.00    -      -      -      -     minpd	%xmm0, %xmm2
 # CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   minpd	(%rax), %xmm2
 # CHECK-NEXT:  -      -      -     1.00    -      -      -      -     minsd	%xmm0, %xmm2
@@ -836,6 +845,8 @@ xorpd       (%rax), %xmm2
 # CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     pcmpgtw	%xmm0, %xmm2
 # CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   pcmpgtw	(%rax), %xmm2
 # CHECK-NEXT:  -      -     1.00   0.50    -     0.50    -      -     pextrw	$1, %xmm0, %ecx
+# CHECK-NEXT:  -      -      -     0.50    -     1.50    -      -     pinsrw	$1, %eax, %xmm0
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   pinsrw	$1, (%rax), %xmm0
 # CHECK-NEXT:  -      -     1.00    -      -      -      -      -     pmaddwd	%xmm0, %xmm2
 # CHECK-NEXT:  -      -     1.00    -      -      -     0.50   0.50   pmaddwd	(%rax), %xmm2
 # CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     pmaxsw	%xmm0, %xmm2
diff --git a/test/tools/llvm-mca/X86/Generic/resources-sse3.s b/test/tools/llvm-mca/X86/Generic/resources-sse3.s
index 8438e1a7a8433f88b30e4447eb8c153400fae53b..7b4c0a34f3157176c30b8aaf06e9e46775ae40cc 100644
--- a/test/tools/llvm-mca/X86/Generic/resources-sse3.s
+++ b/test/tools/llvm-mca/X86/Generic/resources-sse3.s
@@ -21,6 +21,8 @@ hsubps    (%rax), %xmm2
 
 lddqu     (%rax), %xmm2
 
+monitor
+
 movddup   %xmm0, %xmm2
 movddup   (%rax), %xmm2
 
@@ -30,6 +32,8 @@ movshdup  (%rax), %xmm2
 movsldup  %xmm0, %xmm2
 movsldup  (%rax), %xmm2
 
+mwait
+
 # CHECK:      Instruction Info:
 # CHECK-NEXT: [1]: #uOps
 # CHECK-NEXT: [2]: Latency
@@ -52,12 +56,14 @@ movsldup  (%rax), %xmm2
 # CHECK-NEXT:  3      5     2.00                        hsubps	%xmm0, %xmm2
 # CHECK-NEXT:  4      11    2.00    *                   hsubps	(%rax), %xmm2
 # CHECK-NEXT:  1      6     0.50    *                   lddqu	(%rax), %xmm2
+# CHECK-NEXT:  1      100   0.33                  U     monitor
 # CHECK-NEXT:  1      1     1.00                        movddup	%xmm0, %xmm2
 # CHECK-NEXT:  1      6     0.50    *                   movddup	(%rax), %xmm2
 # CHECK-NEXT:  1      1     1.00                        movshdup	%xmm0, %xmm2
 # CHECK-NEXT:  1      6     0.50    *                   movshdup	(%rax), %xmm2
 # CHECK-NEXT:  1      1     1.00                        movsldup	%xmm0, %xmm2
 # CHECK-NEXT:  1      6     0.50    *                   movsldup	(%rax), %xmm2
+# CHECK-NEXT:  1      100   0.33    *      *      U     mwait
 
 # CHECK:      Resources:
 # CHECK-NEXT: [0]   - SBDivider
@@ -71,7 +77,7 @@ movsldup  (%rax), %xmm2
 
 # CHECK:      Resource pressure per iteration:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6.0]  [6.1]
-# CHECK-NEXT:  -      -      -     12.00   -     19.00  5.00   5.00
+# CHECK-NEXT:  -      -     0.67   12.67   -     19.67  5.00   5.00
 
 # CHECK:      Resource pressure by instruction:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6.0]  [6.1]  Instructions:
@@ -88,9 +94,11 @@ movsldup  (%rax), %xmm2
 # CHECK-NEXT:  -      -      -     1.00    -     2.00    -      -     hsubps	%xmm0, %xmm2
 # CHECK-NEXT:  -      -      -     1.00    -     2.00   0.50   0.50   hsubps	(%rax), %xmm2
 # CHECK-NEXT:  -      -      -      -      -      -     0.50   0.50   lddqu	(%rax), %xmm2
+# CHECK-NEXT:  -      -     0.33   0.33    -     0.33    -      -     monitor
 # CHECK-NEXT:  -      -      -      -      -     1.00    -      -     movddup	%xmm0, %xmm2
 # CHECK-NEXT:  -      -      -      -      -      -     0.50   0.50   movddup	(%rax), %xmm2
 # CHECK-NEXT:  -      -      -      -      -     1.00    -      -     movshdup	%xmm0, %xmm2
 # CHECK-NEXT:  -      -      -      -      -      -     0.50   0.50   movshdup	(%rax), %xmm2
 # CHECK-NEXT:  -      -      -      -      -     1.00    -      -     movsldup	%xmm0, %xmm2
 # CHECK-NEXT:  -      -      -      -      -      -     0.50   0.50   movsldup	(%rax), %xmm2
+# CHECK-NEXT:  -      -     0.33   0.33    -     0.33    -      -     mwait
diff --git a/test/tools/llvm-mca/X86/Generic/resources-x86_64.s b/test/tools/llvm-mca/X86/Generic/resources-x86_64.s
index 2ab041c3de6847c6aa7d905dfb49bd43620087a8..1a3e1d6b75d566e2213bee7e96b8de724bfb0680 100644
--- a/test/tools/llvm-mca/X86/Generic/resources-x86_64.s
+++ b/test/tools/llvm-mca/X86/Generic/resources-x86_64.s
@@ -254,6 +254,8 @@ divl (%rax)
 divq %rcx
 divq (%rax)
 
+enter $7, $4095
+
 idivb %dil
 idivb (%rax)
 idivw %si
@@ -315,8 +317,13 @@ insl
 
 int $7
 
+invlpg  (%rax)
+invlpga %rax, %ecx
+
 lahf
 
+leave
+
 lodsb
 lodsw
 lodsl
@@ -486,6 +493,11 @@ rcrq %cl, %rdi
 rclq %cl, (%rax)
 rcrq %cl, (%rax)
 
+rdmsr
+rdpmc
+rdtsc
+rdtscp
+
 rolb %dil
 rorb %dil
 rolb (%rax)
@@ -789,6 +801,8 @@ testq %rsi, (%rax)
 
 ud2
 
+wrmsr
+
 xaddb %bl, %cl
 xaddb %bl, (%rcx)
 
@@ -1081,6 +1095,7 @@ xorq (%rax), %rdi
 # CHECK-NEXT:  2      30    10.00   *             U     divl	(%rax)
 # CHECK-NEXT:  1      25    10.00                 U     divq	%rcx
 # CHECK-NEXT:  2      30    10.00   *             U     divq	(%rax)
+# CHECK-NEXT:  1      100   0.33                  U     enter	$7, $4095
 # CHECK-NEXT:  1      25    10.00                 U     idivb	%dil
 # CHECK-NEXT:  2      30    10.00   *             U     idivb	(%rax)
 # CHECK-NEXT:  1      25    10.00                 U     idivw	%si
@@ -1133,7 +1148,10 @@ xorq (%rax), %rdi
 # CHECK-NEXT:  1      100   0.33                  U     insw	%dx, %es:(%rdi)
 # CHECK-NEXT:  1      100   0.33                  U     insl	%dx, %es:(%rdi)
 # CHECK-NEXT:  1      100   0.33    *      *      U     int	$7
+# CHECK-NEXT:  1      100   0.33                  U     invlpg	(%rax)
+# CHECK-NEXT:  1      100   0.33                  U     invlpga	%rax, %ecx
 # CHECK-NEXT:  1      1     0.50                        lahf
+# CHECK-NEXT:  3      7     0.67    *                   leave
 # CHECK-NEXT:  3      7     0.67                  U     lodsb	(%rsi), %al
 # CHECK-NEXT:  3      7     0.67                  U     lodsw	(%rsi), %ax
 # CHECK-NEXT:  2      6     0.50                  U     lodsl	(%rsi), %eax
@@ -1283,6 +1301,10 @@ xorq (%rax), %rdi
 # CHECK-NEXT:  8      5     4.00                        rcrq	%cl, %rdi
 # CHECK-NEXT:  11     11    3.50           *            rclq	%cl, (%rax)
 # CHECK-NEXT:  11     11    3.50           *            rcrq	%cl, (%rax)
+# CHECK-NEXT:  1      100   0.33                  U     rdmsr
+# CHECK-NEXT:  1      100   0.33                  U     rdpmc
+# CHECK-NEXT:  1      100   0.33                  U     rdtsc
+# CHECK-NEXT:  1      100   0.33                  U     rdtscp
 # CHECK-NEXT:  2      2     1.00                        rolb	%dil
 # CHECK-NEXT:  2      2     1.00                        rorb	%dil
 # CHECK-NEXT:  5      8     1.00    *      *            rolb	(%rax)
@@ -1557,6 +1579,7 @@ xorq (%rax), %rdi
 # CHECK-NEXT:  1      1     0.33                        testq	%rsi, %rdi
 # CHECK-NEXT:  2      6     0.50    *                   testq	%rsi, (%rax)
 # CHECK-NEXT:  1      100   0.33    *             U     ud2
+# CHECK-NEXT:  1      100   0.33                  U     wrmsr
 # CHECK-NEXT:  3      2     1.00                        xaddb	%bl, %cl
 # CHECK-NEXT:  5      8     1.00    *      *            xaddb	%bl, (%rcx)
 # CHECK-NEXT:  3      2     1.00                        xaddw	%bx, %cx
@@ -1620,7 +1643,7 @@ xorq (%rax), %rdi
 
 # CHECK:      Resource pressure per iteration:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6.0]  [6.1]
-# CHECK-NEXT: 160.00  -     571.83 221.33 222.00 571.83 316.00 316.00
+# CHECK-NEXT: 160.00  -     575.17 224.67 222.00 575.17 316.50 316.50
 
 # CHECK:      Resource pressure by instruction:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6.0]  [6.1]  Instructions:
@@ -1844,6 +1867,7 @@ xorq (%rax), %rdi
 # CHECK-NEXT: 10.00   -     1.00    -      -      -     0.50   0.50   divl	(%rax)
 # CHECK-NEXT: 10.00   -     1.00    -      -      -      -      -     divq	%rcx
 # CHECK-NEXT: 10.00   -     1.00    -      -      -     0.50   0.50   divq	(%rax)
+# CHECK-NEXT:  -      -     0.33   0.33    -     0.33    -      -     enter	$7, $4095
 # CHECK-NEXT: 10.00   -     1.00    -      -      -      -      -     idivb	%dil
 # CHECK-NEXT: 10.00   -     1.00    -      -      -     0.50   0.50   idivb	(%rax)
 # CHECK-NEXT: 10.00   -     1.00    -      -      -      -      -     idivw	%si
@@ -1896,7 +1920,10 @@ xorq (%rax), %rdi
 # CHECK-NEXT:  -      -     0.33   0.33    -     0.33    -      -     insw	%dx, %es:(%rdi)
 # CHECK-NEXT:  -      -     0.33   0.33    -     0.33    -      -     insl	%dx, %es:(%rdi)
 # CHECK-NEXT:  -      -     0.33   0.33    -     0.33    -      -     int	$7
+# CHECK-NEXT:  -      -     0.33   0.33    -     0.33    -      -     invlpg	(%rax)
+# CHECK-NEXT:  -      -     0.33   0.33    -     0.33    -      -     invlpga	%rax, %ecx
 # CHECK-NEXT:  -      -     0.50    -      -     0.50    -      -     lahf
+# CHECK-NEXT:  -      -     0.67   0.67    -     0.67   0.50   0.50   leave
 # CHECK-NEXT:  -      -     0.67   0.67    -     0.67   0.50   0.50   lodsb	(%rsi), %al
 # CHECK-NEXT:  -      -     0.67   0.67    -     0.67   0.50   0.50   lodsw	(%rsi), %ax
 # CHECK-NEXT:  -      -     0.33   0.33    -     0.33   0.50   0.50   lodsl	(%rsi), %eax
@@ -2046,6 +2073,10 @@ xorq (%rax), %rdi
 # CHECK-NEXT:  -      -     4.00    -      -     4.00    -      -     rcrq	%cl, %rdi
 # CHECK-NEXT:  -      -     3.50    -      -     3.50   2.00   2.00   rclq	%cl, (%rax)
 # CHECK-NEXT:  -      -     3.50    -      -     3.50   2.00   2.00   rcrq	%cl, (%rax)
+# CHECK-NEXT:  -      -     0.33   0.33    -     0.33    -      -     rdmsr
+# CHECK-NEXT:  -      -     0.33   0.33    -     0.33    -      -     rdpmc
+# CHECK-NEXT:  -      -     0.33   0.33    -     0.33    -      -     rdtsc
+# CHECK-NEXT:  -      -     0.33   0.33    -     0.33    -      -     rdtscp
 # CHECK-NEXT:  -      -     1.00    -      -     1.00    -      -     rolb	%dil
 # CHECK-NEXT:  -      -     1.00    -      -     1.00    -      -     rorb	%dil
 # CHECK-NEXT:  -      -     1.00    -     1.00   1.00   1.00   1.00   rolb	(%rax)
@@ -2320,6 +2351,7 @@ xorq (%rax), %rdi
 # CHECK-NEXT:  -      -     0.33   0.33    -     0.33    -      -     testq	%rsi, %rdi
 # CHECK-NEXT:  -      -     0.33   0.33    -     0.33   0.50   0.50   testq	%rsi, (%rax)
 # CHECK-NEXT:  -      -     0.33   0.33    -     0.33    -      -     ud2
+# CHECK-NEXT:  -      -     0.33   0.33    -     0.33    -      -     wrmsr
 # CHECK-NEXT:  -      -     1.00   1.00    -     1.00    -      -     xaddb	%bl, %cl
 # CHECK-NEXT:  -      -     0.67   0.67   1.00   0.67   1.00   1.00   xaddb	%bl, (%rcx)
 # CHECK-NEXT:  -      -     1.00   1.00    -     1.00    -      -     xaddw	%bx, %cx
diff --git a/test/tools/llvm-mca/X86/Generic/resources-x87.s b/test/tools/llvm-mca/X86/Generic/resources-x87.s
index 1cba9a7d77fc288392e56eb9376396ff62db39b4..1f3e51e58b33c07aba509d8e807883c055d77b77 100644
--- a/test/tools/llvm-mca/X86/Generic/resources-x87.s
+++ b/test/tools/llvm-mca/X86/Generic/resources-x87.s
@@ -5,7 +5,7 @@ f2xm1
 
 fabs
 
-fadd %st(0), %st(1)
+fadd %st, %st(1)
 fadd %st(2)
 fadds (%ecx)
 faddl (%ecx)
@@ -21,14 +21,14 @@ fchs
 
 fnclex
 
-fcmovb %st(1), %st(0)
-fcmovbe %st(1), %st(0)
-fcmove %st(1), %st(0)
-fcmovnb %st(1), %st(0)
-fcmovnbe %st(1), %st(0)
-fcmovne %st(1), %st(0)
-fcmovnu %st(1), %st(0)
-fcmovu %st(1), %st(0)
+fcmovb %st(1), %st
+fcmovbe %st(1), %st
+fcmove %st(1), %st
+fcmovnb %st(1), %st
+fcmovnbe %st(1), %st
+fcmovne %st(1), %st
+fcmovnu %st(1), %st
+fcmovu %st(1), %st
 
 fcom %st(1)
 fcom %st(3)
@@ -47,7 +47,7 @@ fcos
 
 fdecstp
 
-fdiv %st(0), %st(1)
+fdiv %st, %st(1)
 fdiv %st(2)
 fdivs (%ecx)
 fdivl (%eax)
@@ -56,7 +56,7 @@ fdivp %st(2)
 fidivs (%ecx)
 fidivl (%eax)
 
-fdivr %st(0), %st(1)
+fdivr %st, %st(1)
 fdivr %st(2)
 fdivrs (%ecx)
 fdivrl (%eax)
@@ -106,7 +106,7 @@ fldln2
 fldpi
 fldz
 
-fmul %st(0), %st(1)
+fmul %st, %st(1)
 fmul %st(2)
 fmuls (%ecx)
 fmull (%eax)
@@ -153,7 +153,7 @@ fnstsw (%eax)
 frstor (%eax)
 fsave (%eax)
 
-fsub %st(0), %st(1)
+fsub %st, %st(1)
 fsub %st(2)
 fsubs (%ecx)
 fsubl (%eax)
@@ -162,7 +162,7 @@ fsubp %st(2)
 fisubs (%ecx)
 fisubl (%eax)
 
-fsubr %st(0), %st(1)
+fsubr %st, %st(1)
 fsubr %st(2)
 fsubrs (%ecx)
 fsubrl (%eax)
@@ -208,26 +208,26 @@ fyl2xp1
 # CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
 # CHECK-NEXT:  1      100   0.33                  U     f2xm1
 # CHECK-NEXT:  1      1     1.00                  U     fabs
-# CHECK-NEXT:  1      3     1.00                  U     fadd	%st(0), %st(1)
-# CHECK-NEXT:  1      3     1.00                  U     fadd	%st(2)
+# CHECK-NEXT:  1      3     1.00                  U     fadd	%st, %st(1)
+# CHECK-NEXT:  1      3     1.00                  U     fadd	%st(2), %st
 # CHECK-NEXT:  2      10    1.00    *             U     fadds	(%ecx)
 # CHECK-NEXT:  2      10    1.00    *             U     faddl	(%ecx)
-# CHECK-NEXT:  1      3     1.00                  U     faddp	%st(1)
-# CHECK-NEXT:  1      3     1.00                  U     faddp	%st(2)
+# CHECK-NEXT:  1      3     1.00                  U     faddp	%st, %st(1)
+# CHECK-NEXT:  1      3     1.00                  U     faddp	%st, %st(2)
 # CHECK-NEXT:  3      13    2.00    *             U     fiadds	(%ecx)
 # CHECK-NEXT:  3      13    2.00    *             U     fiaddl	(%ecx)
 # CHECK-NEXT:  1      100   0.33                  U     fbld	(%ecx)
 # CHECK-NEXT:  1      100   0.33                  U     fbstp	(%eax)
 # CHECK-NEXT:  1      1     1.00                  U     fchs
 # CHECK-NEXT:  1      100   0.33                  U     fnclex
-# CHECK-NEXT:  3      3     2.00                  U     fcmovb	%st(1), %st(0)
-# CHECK-NEXT:  3      3     2.00                  U     fcmovbe	%st(1), %st(0)
-# CHECK-NEXT:  3      3     2.00                  U     fcmove	%st(1), %st(0)
-# CHECK-NEXT:  3      3     2.00                  U     fcmovnb	%st(1), %st(0)
-# CHECK-NEXT:  3      3     2.00                  U     fcmovnbe	%st(1), %st(0)
-# CHECK-NEXT:  3      3     2.00                  U     fcmovne	%st(1), %st(0)
-# CHECK-NEXT:  3      3     2.00                  U     fcmovnu	%st(1), %st(0)
-# CHECK-NEXT:  3      3     2.00                  U     fcmovu	%st(1), %st(0)
+# CHECK-NEXT:  3      3     2.00                  U     fcmovb	%st(1), %st
+# CHECK-NEXT:  3      3     2.00                  U     fcmovbe	%st(1), %st
+# CHECK-NEXT:  3      3     2.00                  U     fcmove	%st(1), %st
+# CHECK-NEXT:  3      3     2.00                  U     fcmovnb	%st(1), %st
+# CHECK-NEXT:  3      3     2.00                  U     fcmovnbe	%st(1), %st
+# CHECK-NEXT:  3      3     2.00                  U     fcmovne	%st(1), %st
+# CHECK-NEXT:  3      3     2.00                  U     fcmovnu	%st(1), %st
+# CHECK-NEXT:  3      3     2.00                  U     fcmovu	%st(1), %st
 # CHECK-NEXT:  1      1     1.00                  U     fcom	%st(1)
 # CHECK-NEXT:  1      1     1.00                  U     fcom	%st(3)
 # CHECK-NEXT:  2      8     1.00                  U     fcoms	(%ecx)
@@ -237,24 +237,24 @@ fyl2xp1
 # CHECK-NEXT:  2      8     1.00                  U     fcomps	(%ecx)
 # CHECK-NEXT:  2      8     1.00                  U     fcompl	(%eax)
 # CHECK-NEXT:  1      100   0.33                  U     fcompp
-# CHECK-NEXT:  3      3     1.00                  U     fcomi	%st(3)
-# CHECK-NEXT:  3      3     1.00                  U     fcompi	%st(3)
+# CHECK-NEXT:  3      3     1.00                  U     fcomi	%st(3), %st
+# CHECK-NEXT:  3      3     1.00                  U     fcompi	%st(3), %st
 # CHECK-NEXT:  1      100   0.33                  U     fcos
 # CHECK-NEXT:  1      1     1.00                  U     fdecstp
-# CHECK-NEXT:  1      14    14.00                 U     fdiv	%st(0), %st(1)
-# CHECK-NEXT:  1      14    14.00                 U     fdiv	%st(2)
+# CHECK-NEXT:  1      14    14.00                 U     fdiv	%st, %st(1)
+# CHECK-NEXT:  1      14    14.00                 U     fdiv	%st(2), %st
 # CHECK-NEXT:  2      31    1.00    *             U     fdivs	(%ecx)
 # CHECK-NEXT:  2      31    1.00    *             U     fdivl	(%eax)
-# CHECK-NEXT:  1      14    14.00                 U     fdivp	%st(1)
-# CHECK-NEXT:  1      14    14.00                 U     fdivp	%st(2)
+# CHECK-NEXT:  1      14    14.00                 U     fdivp	%st, %st(1)
+# CHECK-NEXT:  1      14    14.00                 U     fdivp	%st, %st(2)
 # CHECK-NEXT:  3      34    1.00    *             U     fidivs	(%ecx)
 # CHECK-NEXT:  3      34    1.00    *             U     fidivl	(%eax)
-# CHECK-NEXT:  1      14    14.00                 U     fdivr	%st(0), %st(1)
-# CHECK-NEXT:  1      14    14.00                 U     fdivr	%st(2)
+# CHECK-NEXT:  1      14    14.00                 U     fdivr	%st, %st(1)
+# CHECK-NEXT:  1      14    14.00                 U     fdivr	%st(2), %st
 # CHECK-NEXT:  2      31    1.00    *             U     fdivrs	(%ecx)
 # CHECK-NEXT:  2      31    1.00    *             U     fdivrl	(%eax)
-# CHECK-NEXT:  1      14    14.00                 U     fdivrp	%st(1)
-# CHECK-NEXT:  1      14    14.00                 U     fdivrp	%st(2)
+# CHECK-NEXT:  1      14    14.00                 U     fdivrp	%st, %st(1)
+# CHECK-NEXT:  1      14    14.00                 U     fdivrp	%st, %st(2)
 # CHECK-NEXT:  3      34    1.00    *             U     fidivrs	(%ecx)
 # CHECK-NEXT:  3      34    1.00    *             U     fidivrl	(%eax)
 # CHECK-NEXT:  1      1     1.00                  U     ffree	%st(0)
@@ -288,12 +288,12 @@ fyl2xp1
 # CHECK-NEXT:  2      1     1.00                  U     fldln2
 # CHECK-NEXT:  2      1     1.00                  U     fldpi
 # CHECK-NEXT:  1      1     1.00                  U     fldz
-# CHECK-NEXT:  1      5     1.00                  U     fmul	%st(0), %st(1)
-# CHECK-NEXT:  1      5     1.00                  U     fmul	%st(2)
+# CHECK-NEXT:  1      5     1.00                  U     fmul	%st, %st(1)
+# CHECK-NEXT:  1      5     1.00                  U     fmul	%st(2), %st
 # CHECK-NEXT:  2      12    1.00    *             U     fmuls	(%ecx)
 # CHECK-NEXT:  2      12    1.00    *             U     fmull	(%eax)
-# CHECK-NEXT:  1      5     1.00                  U     fmulp	%st(1)
-# CHECK-NEXT:  1      5     1.00                  U     fmulp	%st(2)
+# CHECK-NEXT:  1      5     1.00                  U     fmulp	%st, %st(1)
+# CHECK-NEXT:  1      5     1.00                  U     fmulp	%st, %st(2)
 # CHECK-NEXT:  3      15    1.00    *             U     fimuls	(%ecx)
 # CHECK-NEXT:  3      15    1.00    *             U     fimull	(%eax)
 # CHECK-NEXT:  1      1     1.00                  U     fnop
@@ -321,20 +321,20 @@ fyl2xp1
 # CHECK-NEXT:  1      100   0.33                  U     frstor	(%eax)
 # CHECK-NEXT:  1      100   0.33                  U     wait
 # CHECK-NEXT:  1      100   0.33                  U     fnsave	(%eax)
-# CHECK-NEXT:  1      3     1.00                  U     fsub	%st(0), %st(1)
-# CHECK-NEXT:  1      3     1.00                  U     fsub	%st(2)
+# CHECK-NEXT:  1      3     1.00                  U     fsub	%st, %st(1)
+# CHECK-NEXT:  1      3     1.00                  U     fsub	%st(2), %st
 # CHECK-NEXT:  2      10    1.00    *             U     fsubs	(%ecx)
 # CHECK-NEXT:  2      10    1.00    *             U     fsubl	(%eax)
-# CHECK-NEXT:  1      3     1.00                  U     fsubp	%st(1)
-# CHECK-NEXT:  1      3     1.00                  U     fsubp	%st(2)
+# CHECK-NEXT:  1      3     1.00                  U     fsubp	%st, %st(1)
+# CHECK-NEXT:  1      3     1.00                  U     fsubp	%st, %st(2)
 # CHECK-NEXT:  3      13    2.00    *             U     fisubs	(%ecx)
 # CHECK-NEXT:  3      13    2.00    *             U     fisubl	(%eax)
-# CHECK-NEXT:  1      3     1.00                  U     fsubr	%st(0), %st(1)
-# CHECK-NEXT:  1      3     1.00                  U     fsubr	%st(2)
+# CHECK-NEXT:  1      3     1.00                  U     fsubr	%st, %st(1)
+# CHECK-NEXT:  1      3     1.00                  U     fsubr	%st(2), %st
 # CHECK-NEXT:  2      10    1.00    *             U     fsubrs	(%ecx)
 # CHECK-NEXT:  2      10    1.00    *             U     fsubrl	(%eax)
-# CHECK-NEXT:  1      3     1.00                  U     fsubrp	%st(1)
-# CHECK-NEXT:  1      3     1.00                  U     fsubrp	%st(2)
+# CHECK-NEXT:  1      3     1.00                  U     fsubrp	%st, %st(1)
+# CHECK-NEXT:  1      3     1.00                  U     fsubrp	%st, %st(2)
 # CHECK-NEXT:  3      13    2.00    *             U     fisubrs	(%ecx)
 # CHECK-NEXT:  3      13    2.00    *             U     fisubrl	(%eax)
 # CHECK-NEXT:  1      3     1.00                  U     ftst
@@ -343,8 +343,8 @@ fyl2xp1
 # CHECK-NEXT:  1      1     1.00                  U     fucomp	%st(1)
 # CHECK-NEXT:  1      1     1.00                  U     fucomp	%st(3)
 # CHECK-NEXT:  1      3     1.00                  U     fucompp
-# CHECK-NEXT:  3      3     1.00                  U     fucomi	%st(3)
-# CHECK-NEXT:  3      3     1.00                  U     fucompi	%st(3)
+# CHECK-NEXT:  3      3     1.00                  U     fucomi	%st(3), %st
+# CHECK-NEXT:  3      3     1.00                  U     fucompi	%st(3), %st
 # CHECK-NEXT:  1      100   0.33                  U     wait
 # CHECK-NEXT:  1      100   0.33                  U     fxam
 # CHECK-NEXT:  1      1     0.33                  U     fxch	%st(1)
@@ -373,26 +373,26 @@ fyl2xp1
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6.0]  [6.1]  Instructions:
 # CHECK-NEXT:  -      -     0.33   0.33    -     0.33    -      -     f2xm1
 # CHECK-NEXT:  -      -      -      -      -     1.00    -      -     fabs
-# CHECK-NEXT:  -      -      -     1.00    -      -      -      -     fadd	%st(0), %st(1)
-# CHECK-NEXT:  -      -      -     1.00    -      -      -      -     fadd	%st(2)
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -     fadd	%st, %st(1)
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -     fadd	%st(2), %st
 # CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   fadds	(%ecx)
 # CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   faddl	(%ecx)
-# CHECK-NEXT:  -      -      -     1.00    -      -      -      -     faddp	%st(1)
-# CHECK-NEXT:  -      -      -     1.00    -      -      -      -     faddp	%st(2)
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -     faddp	%st, %st(1)
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -     faddp	%st, %st(2)
 # CHECK-NEXT:  -      -      -     2.00    -      -     0.50   0.50   fiadds	(%ecx)
 # CHECK-NEXT:  -      -      -     2.00    -      -     0.50   0.50   fiaddl	(%ecx)
 # CHECK-NEXT:  -      -     0.33   0.33    -     0.33    -      -     fbld	(%ecx)
 # CHECK-NEXT:  -      -     0.33   0.33    -     0.33    -      -     fbstp	(%eax)
 # CHECK-NEXT:  -      -      -      -      -     1.00    -      -     fchs
 # CHECK-NEXT:  -      -     0.33   0.33    -     0.33    -      -     fnclex
-# CHECK-NEXT:  -      -     0.50    -      -     2.50    -      -     fcmovb	%st(1), %st(0)
-# CHECK-NEXT:  -      -     0.50    -      -     2.50    -      -     fcmovbe	%st(1), %st(0)
-# CHECK-NEXT:  -      -     0.50    -      -     2.50    -      -     fcmove	%st(1), %st(0)
-# CHECK-NEXT:  -      -     0.50    -      -     2.50    -      -     fcmovnb	%st(1), %st(0)
-# CHECK-NEXT:  -      -     0.50    -      -     2.50    -      -     fcmovnbe	%st(1), %st(0)
-# CHECK-NEXT:  -      -     0.50    -      -     2.50    -      -     fcmovne	%st(1), %st(0)
-# CHECK-NEXT:  -      -     0.50    -      -     2.50    -      -     fcmovnu	%st(1), %st(0)
-# CHECK-NEXT:  -      -     0.50    -      -     2.50    -      -     fcmovu	%st(1), %st(0)
+# CHECK-NEXT:  -      -     0.50    -      -     2.50    -      -     fcmovb	%st(1), %st
+# CHECK-NEXT:  -      -     0.50    -      -     2.50    -      -     fcmovbe	%st(1), %st
+# CHECK-NEXT:  -      -     0.50    -      -     2.50    -      -     fcmove	%st(1), %st
+# CHECK-NEXT:  -      -     0.50    -      -     2.50    -      -     fcmovnb	%st(1), %st
+# CHECK-NEXT:  -      -     0.50    -      -     2.50    -      -     fcmovnbe	%st(1), %st
+# CHECK-NEXT:  -      -     0.50    -      -     2.50    -      -     fcmovne	%st(1), %st
+# CHECK-NEXT:  -      -     0.50    -      -     2.50    -      -     fcmovnu	%st(1), %st
+# CHECK-NEXT:  -      -     0.50    -      -     2.50    -      -     fcmovu	%st(1), %st
 # CHECK-NEXT:  -      -      -     1.00    -      -      -      -     fcom	%st(1)
 # CHECK-NEXT:  -      -      -     1.00    -      -      -      -     fcom	%st(3)
 # CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   fcoms	(%ecx)
@@ -402,24 +402,24 @@ fyl2xp1
 # CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   fcomps	(%ecx)
 # CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   fcompl	(%eax)
 # CHECK-NEXT:  -      -     0.33   0.33    -     0.33    -      -     fcompp
-# CHECK-NEXT:  -      -     1.00   1.00    -     1.00    -      -     fcomi	%st(3)
-# CHECK-NEXT:  -      -     1.00   1.00    -     1.00    -      -     fcompi	%st(3)
+# CHECK-NEXT:  -      -     1.00   1.00    -     1.00    -      -     fcomi	%st(3), %st
+# CHECK-NEXT:  -      -     1.00   1.00    -     1.00    -      -     fcompi	%st(3), %st
 # CHECK-NEXT:  -      -     0.33   0.33    -     0.33    -      -     fcos
 # CHECK-NEXT:  -      -      -      -      -     1.00    -      -     fdecstp
-# CHECK-NEXT:  -     14.00  1.00    -      -      -      -      -     fdiv	%st(0), %st(1)
-# CHECK-NEXT:  -     14.00  1.00    -      -      -      -      -     fdiv	%st(2)
+# CHECK-NEXT:  -     14.00  1.00    -      -      -      -      -     fdiv	%st, %st(1)
+# CHECK-NEXT:  -     14.00  1.00    -      -      -      -      -     fdiv	%st(2), %st
 # CHECK-NEXT:  -      -     1.00    -      -      -     0.50   0.50   fdivs	(%ecx)
 # CHECK-NEXT:  -      -     1.00    -      -      -     0.50   0.50   fdivl	(%eax)
-# CHECK-NEXT:  -     14.00  1.00    -      -      -      -      -     fdivp	%st(1)
-# CHECK-NEXT:  -     14.00  1.00    -      -      -      -      -     fdivp	%st(2)
+# CHECK-NEXT:  -     14.00  1.00    -      -      -      -      -     fdivp	%st, %st(1)
+# CHECK-NEXT:  -     14.00  1.00    -      -      -      -      -     fdivp	%st, %st(2)
 # CHECK-NEXT:  -      -     1.00   1.00    -      -     0.50   0.50   fidivs	(%ecx)
 # CHECK-NEXT:  -      -     1.00   1.00    -      -     0.50   0.50   fidivl	(%eax)
-# CHECK-NEXT:  -     14.00  1.00    -      -      -      -      -     fdivr	%st(0), %st(1)
-# CHECK-NEXT:  -     14.00  1.00    -      -      -      -      -     fdivr	%st(2)
+# CHECK-NEXT:  -     14.00  1.00    -      -      -      -      -     fdivr	%st, %st(1)
+# CHECK-NEXT:  -     14.00  1.00    -      -      -      -      -     fdivr	%st(2), %st
 # CHECK-NEXT:  -      -     1.00    -      -      -     0.50   0.50   fdivrs	(%ecx)
 # CHECK-NEXT:  -      -     1.00    -      -      -     0.50   0.50   fdivrl	(%eax)
-# CHECK-NEXT:  -     14.00  1.00    -      -      -      -      -     fdivrp	%st(1)
-# CHECK-NEXT:  -     14.00  1.00    -      -      -      -      -     fdivrp	%st(2)
+# CHECK-NEXT:  -     14.00  1.00    -      -      -      -      -     fdivrp	%st, %st(1)
+# CHECK-NEXT:  -     14.00  1.00    -      -      -      -      -     fdivrp	%st, %st(2)
 # CHECK-NEXT:  -      -     1.00   1.00    -      -     0.50   0.50   fidivrs	(%ecx)
 # CHECK-NEXT:  -      -     1.00   1.00    -      -     0.50   0.50   fidivrl	(%eax)
 # CHECK-NEXT:  -      -      -      -      -     1.00    -      -     ffree	%st(0)
@@ -453,12 +453,12 @@ fyl2xp1
 # CHECK-NEXT:  -      -     1.00   1.00    -      -      -      -     fldln2
 # CHECK-NEXT:  -      -     1.00   1.00    -      -      -      -     fldpi
 # CHECK-NEXT:  -      -      -      -      -     1.00    -      -     fldz
-# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     fmul	%st(0), %st(1)
-# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     fmul	%st(2)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     fmul	%st, %st(1)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     fmul	%st(2), %st
 # CHECK-NEXT:  -      -     1.00    -      -      -     0.50   0.50   fmuls	(%ecx)
 # CHECK-NEXT:  -      -     1.00    -      -      -     0.50   0.50   fmull	(%eax)
-# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     fmulp	%st(1)
-# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     fmulp	%st(2)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     fmulp	%st, %st(1)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     fmulp	%st, %st(2)
 # CHECK-NEXT:  -      -     1.00   1.00    -      -     0.50   0.50   fimuls	(%ecx)
 # CHECK-NEXT:  -      -     1.00   1.00    -      -     0.50   0.50   fimull	(%eax)
 # CHECK-NEXT:  -      -      -      -      -     1.00    -      -     fnop
@@ -486,20 +486,20 @@ fyl2xp1
 # CHECK-NEXT:  -      -     0.33   0.33    -     0.33    -      -     frstor	(%eax)
 # CHECK-NEXT:  -      -     0.33   0.33    -     0.33    -      -     wait
 # CHECK-NEXT:  -      -     0.33   0.33    -     0.33    -      -     fnsave	(%eax)
-# CHECK-NEXT:  -      -      -     1.00    -      -      -      -     fsub	%st(0), %st(1)
-# CHECK-NEXT:  -      -      -     1.00    -      -      -      -     fsub	%st(2)
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -     fsub	%st, %st(1)
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -     fsub	%st(2), %st
 # CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   fsubs	(%ecx)
 # CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   fsubl	(%eax)
-# CHECK-NEXT:  -      -      -     1.00    -      -      -      -     fsubp	%st(1)
-# CHECK-NEXT:  -      -      -     1.00    -      -      -      -     fsubp	%st(2)
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -     fsubp	%st, %st(1)
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -     fsubp	%st, %st(2)
 # CHECK-NEXT:  -      -      -     2.00    -      -     0.50   0.50   fisubs	(%ecx)
 # CHECK-NEXT:  -      -      -     2.00    -      -     0.50   0.50   fisubl	(%eax)
-# CHECK-NEXT:  -      -      -     1.00    -      -      -      -     fsubr	%st(0), %st(1)
-# CHECK-NEXT:  -      -      -     1.00    -      -      -      -     fsubr	%st(2)
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -     fsubr	%st, %st(1)
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -     fsubr	%st(2), %st
 # CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   fsubrs	(%ecx)
 # CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   fsubrl	(%eax)
-# CHECK-NEXT:  -      -      -     1.00    -      -      -      -     fsubrp	%st(1)
-# CHECK-NEXT:  -      -      -     1.00    -      -      -      -     fsubrp	%st(2)
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -     fsubrp	%st, %st(1)
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -     fsubrp	%st, %st(2)
 # CHECK-NEXT:  -      -      -     2.00    -      -     0.50   0.50   fisubrs	(%ecx)
 # CHECK-NEXT:  -      -      -     2.00    -      -     0.50   0.50   fisubrl	(%eax)
 # CHECK-NEXT:  -      -      -     1.00    -      -      -      -     ftst
@@ -508,8 +508,8 @@ fyl2xp1
 # CHECK-NEXT:  -      -      -     1.00    -      -      -      -     fucomp	%st(1)
 # CHECK-NEXT:  -      -      -     1.00    -      -      -      -     fucomp	%st(3)
 # CHECK-NEXT:  -      -      -     1.00    -      -      -      -     fucompp
-# CHECK-NEXT:  -      -     1.00   1.00    -     1.00    -      -     fucomi	%st(3)
-# CHECK-NEXT:  -      -     1.00   1.00    -     1.00    -      -     fucompi	%st(3)
+# CHECK-NEXT:  -      -     1.00   1.00    -     1.00    -      -     fucomi	%st(3), %st
+# CHECK-NEXT:  -      -     1.00   1.00    -     1.00    -      -     fucompi	%st(3), %st
 # CHECK-NEXT:  -      -     0.33   0.33    -     0.33    -      -     wait
 # CHECK-NEXT:  -      -     0.33   0.33    -     0.33    -      -     fxam
 # CHECK-NEXT:  -      -     0.33   0.33    -     0.33    -      -     fxch	%st(1)
diff --git a/test/tools/llvm-mca/X86/Haswell/resources-avx2.s b/test/tools/llvm-mca/X86/Haswell/resources-avx2.s
index 7f76dba9320ce38ceaf667239a923f50c0a165be..b7223d31c877b91befd46b5ac8299f46f5347d17 100644
--- a/test/tools/llvm-mca/X86/Haswell/resources-avx2.s
+++ b/test/tools/llvm-mca/X86/Haswell/resources-avx2.s
@@ -149,6 +149,9 @@ vpcmpgtq        (%rax), %ymm1, %ymm2
 vpcmpgtw        %ymm0, %ymm1, %ymm2
 vpcmpgtw        (%rax), %ymm1, %ymm2
 
+vperm2i128      $1, %ymm0, %ymm1, %ymm2
+vperm2i128      $1, (%rax), %ymm1, %ymm2
+
 vpermd          %ymm0, %ymm1, %ymm2
 vpermd          (%rax), %ymm1, %ymm2
 
@@ -555,6 +558,8 @@ vpxor           (%rax), %ymm1, %ymm2
 # CHECK-NEXT:  2      12    1.00    *                   vpcmpgtq	(%rax), %ymm1, %ymm2
 # CHECK-NEXT:  1      1     0.50                        vpcmpgtw	%ymm0, %ymm1, %ymm2
 # CHECK-NEXT:  2      8     0.50    *                   vpcmpgtw	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      3     1.00                        vperm2i128	$1, %ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  2      10    1.00    *                   vperm2i128	$1, (%rax), %ymm1, %ymm2
 # CHECK-NEXT:  1      3     1.00                        vpermd	%ymm0, %ymm1, %ymm2
 # CHECK-NEXT:  2      10    1.00    *                   vpermd	(%rax), %ymm1, %ymm2
 # CHECK-NEXT:  1      3     1.00                        vpermpd	$1, %ymm0, %ymm2
@@ -771,7 +776,7 @@ vpxor           (%rax), %ymm1, %ymm2
 
 # CHECK:      Resource pressure per iteration:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]
-# CHECK-NEXT:  -      -     202.67 89.67  97.17  97.17  5.00   280.67 28.00  1.67
+# CHECK-NEXT:  -      -     202.67 89.67  97.67  97.67  5.00   282.67 28.00  1.67
 
 # CHECK:      Resource pressure by instruction:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    Instructions:
@@ -873,6 +878,8 @@ vpxor           (%rax), %ymm1, %ymm2
 # CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -      -      -      -     vpcmpgtq	(%rax), %ymm1, %ymm2
 # CHECK-NEXT:  -      -      -     0.50    -      -      -     0.50    -      -     vpcmpgtw	%ymm0, %ymm1, %ymm2
 # CHECK-NEXT:  -      -      -     0.50   0.50   0.50    -     0.50    -      -     vpcmpgtw	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vperm2i128	$1, %ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vperm2i128	$1, (%rax), %ymm1, %ymm2
 # CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpermd	%ymm0, %ymm1, %ymm2
 # CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermd	(%rax), %ymm1, %ymm2
 # CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpermpd	$1, %ymm0, %ymm2
diff --git a/test/tools/llvm-mca/X86/Haswell/resources-bmi1.s b/test/tools/llvm-mca/X86/Haswell/resources-bmi1.s
index 042d043bd23b0acc59c2668e09b19dbb5b2c580f..9e3655bfb00e198bfcf6e71b013bb30f63fd385f 100644
--- a/test/tools/llvm-mca/X86/Haswell/resources-bmi1.s
+++ b/test/tools/llvm-mca/X86/Haswell/resources-bmi1.s
@@ -31,6 +31,9 @@ blsr        (%rax), %ecx
 blsr        %rax, %rcx
 blsr        (%rax), %rcx
 
+tzcnt       %ax, %cx
+tzcnt       (%rax), %cx
+
 tzcnt       %eax, %ecx
 tzcnt       (%rax), %ecx
 
@@ -66,6 +69,8 @@ tzcnt       (%rax), %rcx
 # CHECK-NEXT:  2      6     0.50    *                   blsrl	(%rax), %ecx
 # CHECK-NEXT:  1      1     0.50                        blsrq	%rax, %rcx
 # CHECK-NEXT:  2      6     0.50    *                   blsrq	(%rax), %rcx
+# CHECK-NEXT:  1      3     1.00                        tzcntw	%ax, %cx
+# CHECK-NEXT:  2      8     1.00    *                   tzcntw	(%rax), %cx
 # CHECK-NEXT:  1      3     1.00                        tzcntl	%eax, %ecx
 # CHECK-NEXT:  2      8     1.00    *                   tzcntl	(%rax), %ecx
 # CHECK-NEXT:  1      3     1.00                        tzcntq	%rax, %rcx
@@ -85,7 +90,7 @@ tzcnt       (%rax), %rcx
 
 # CHECK:      Resource pressure per iteration:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]
-# CHECK-NEXT:  -      -     2.00   14.00  6.00   6.00    -     10.00  2.00    -
+# CHECK-NEXT:  -      -     2.00   16.00  6.50   6.50    -     10.00  2.00    -
 
 # CHECK:      Resource pressure by instruction:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    Instructions:
@@ -109,6 +114,8 @@ tzcnt       (%rax), %rcx
 # CHECK-NEXT:  -      -      -     0.50   0.50   0.50    -     0.50    -      -     blsrl	(%rax), %ecx
 # CHECK-NEXT:  -      -      -     0.50    -      -      -     0.50    -      -     blsrq	%rax, %rcx
 # CHECK-NEXT:  -      -      -     0.50   0.50   0.50    -     0.50    -      -     blsrq	(%rax), %rcx
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     tzcntw	%ax, %cx
+# CHECK-NEXT:  -      -      -     1.00   0.50   0.50    -      -      -      -     tzcntw	(%rax), %cx
 # CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     tzcntl	%eax, %ecx
 # CHECK-NEXT:  -      -      -     1.00   0.50   0.50    -      -      -      -     tzcntl	(%rax), %ecx
 # CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     tzcntq	%rax, %rcx
diff --git a/test/tools/llvm-mca/X86/Haswell/resources-fsgsbase.s b/test/tools/llvm-mca/X86/Haswell/resources-fsgsbase.s
new file mode 100644
index 0000000000000000000000000000000000000000..9c19a6869a03d6faf4132258b529d64563d7a6e7
--- /dev/null
+++ b/test/tools/llvm-mca/X86/Haswell/resources-fsgsbase.s
@@ -0,0 +1,59 @@
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=x86_64-unknown-unknown -mcpu=haswell -instruction-tables < %s | FileCheck %s
+
+rdfsbase %eax
+rdfsbase %rax
+
+rdgsbase %eax
+rdgsbase %rax
+
+wrfsbase %edi
+wrfsbase %rdi
+
+wrgsbase %edi
+wrgsbase %rdi
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  1      100   0.25    *      *      U     rdfsbasel	%eax
+# CHECK-NEXT:  1      100   0.25    *      *      U     rdfsbaseq	%rax
+# CHECK-NEXT:  1      100   0.25    *      *      U     rdgsbasel	%eax
+# CHECK-NEXT:  1      100   0.25    *      *      U     rdgsbaseq	%rax
+# CHECK-NEXT:  1      100   0.25    *      *      U     wrfsbasel	%edi
+# CHECK-NEXT:  1      100   0.25    *      *      U     wrfsbaseq	%rdi
+# CHECK-NEXT:  1      100   0.25    *      *      U     wrgsbasel	%edi
+# CHECK-NEXT:  1      100   0.25    *      *      U     wrgsbaseq	%rdi
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0]   - HWDivider
+# CHECK-NEXT: [1]   - HWFPDivider
+# CHECK-NEXT: [2]   - HWPort0
+# CHECK-NEXT: [3]   - HWPort1
+# CHECK-NEXT: [4]   - HWPort2
+# CHECK-NEXT: [5]   - HWPort3
+# CHECK-NEXT: [6]   - HWPort4
+# CHECK-NEXT: [7]   - HWPort5
+# CHECK-NEXT: [8]   - HWPort6
+# CHECK-NEXT: [9]   - HWPort7
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]
+# CHECK-NEXT:  -      -     2.00   2.00    -      -      -     2.00   2.00    -
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    Instructions:
+# CHECK-NEXT:  -      -     0.25   0.25    -      -      -     0.25   0.25    -     rdfsbasel	%eax
+# CHECK-NEXT:  -      -     0.25   0.25    -      -      -     0.25   0.25    -     rdfsbaseq	%rax
+# CHECK-NEXT:  -      -     0.25   0.25    -      -      -     0.25   0.25    -     rdgsbasel	%eax
+# CHECK-NEXT:  -      -     0.25   0.25    -      -      -     0.25   0.25    -     rdgsbaseq	%rax
+# CHECK-NEXT:  -      -     0.25   0.25    -      -      -     0.25   0.25    -     wrfsbasel	%edi
+# CHECK-NEXT:  -      -     0.25   0.25    -      -      -     0.25   0.25    -     wrfsbaseq	%rdi
+# CHECK-NEXT:  -      -     0.25   0.25    -      -      -     0.25   0.25    -     wrgsbasel	%edi
+# CHECK-NEXT:  -      -     0.25   0.25    -      -      -     0.25   0.25    -     wrgsbaseq	%rdi
diff --git a/test/tools/llvm-mca/X86/Haswell/resources-sse2.s b/test/tools/llvm-mca/X86/Haswell/resources-sse2.s
index 5f7881377103553d6b3fcb55b1c945fe89cb4f25..1a61240252d619233e0e5d4cb4d3c840eae4e6b2 100644
--- a/test/tools/llvm-mca/X86/Haswell/resources-sse2.s
+++ b/test/tools/llvm-mca/X86/Haswell/resources-sse2.s
@@ -94,6 +94,8 @@ maxpd       (%rax), %xmm2
 maxsd       %xmm0, %xmm2
 maxsd       (%rax), %xmm2
 
+mfence
+
 minpd       %xmm0, %xmm2
 minpd       (%rax), %xmm2
 
@@ -229,6 +231,9 @@ pcmpgtw     (%rax), %xmm2
 
 pextrw      $1, %xmm0, %rcx
 
+pinsrw      $1, %rax, %xmm0
+pinsrw      $1, (%rax), %xmm0
+
 pmaddwd     %xmm0, %xmm2
 pmaddwd     (%rax), %xmm2
 
@@ -465,6 +470,7 @@ xorpd       (%rax), %xmm2
 # CHECK-NEXT:  2      9     1.00    *                   maxpd	(%rax), %xmm2
 # CHECK-NEXT:  1      3     1.00                        maxsd	%xmm0, %xmm2
 # CHECK-NEXT:  2      8     1.00    *                   maxsd	(%rax), %xmm2
+# CHECK-NEXT:  2      2     0.50    *      *      U     mfence
 # CHECK-NEXT:  1      3     1.00                        minpd	%xmm0, %xmm2
 # CHECK-NEXT:  2      9     1.00    *                   minpd	(%rax), %xmm2
 # CHECK-NEXT:  1      3     1.00                        minsd	%xmm0, %xmm2
@@ -555,6 +561,8 @@ xorpd       (%rax), %xmm2
 # CHECK-NEXT:  1      1     0.50                        pcmpgtw	%xmm0, %xmm2
 # CHECK-NEXT:  2      7     0.50    *                   pcmpgtw	(%rax), %xmm2
 # CHECK-NEXT:  2      2     1.00                        pextrw	$1, %xmm0, %ecx
+# CHECK-NEXT:  2      2     2.00                        pinsrw	$1, %eax, %xmm0
+# CHECK-NEXT:  2      6     1.00    *                   pinsrw	$1, (%rax), %xmm0
 # CHECK-NEXT:  1      5     1.00                        pmaddwd	%xmm0, %xmm2
 # CHECK-NEXT:  2      11    1.00    *                   pmaddwd	(%rax), %xmm2
 # CHECK-NEXT:  1      1     0.50                        pmaxsw	%xmm0, %xmm2
@@ -681,7 +689,7 @@ xorpd       (%rax), %xmm2
 
 # CHECK:      Resource pressure per iteration:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]
-# CHECK-NEXT:  -     112.00 70.25  95.25  62.67  62.67  14.00  123.75 1.75   4.67
+# CHECK-NEXT:  -     112.00 70.75  95.75  63.17  63.17  14.00  127.25 2.25   4.67
 
 # CHECK:      Resource pressure by instruction:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    Instructions:
@@ -748,6 +756,7 @@ xorpd       (%rax), %xmm2
 # CHECK-NEXT:  -      -      -     1.00   0.50   0.50    -      -      -      -     maxpd	(%rax), %xmm2
 # CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     maxsd	%xmm0, %xmm2
 # CHECK-NEXT:  -      -      -     1.00   0.50   0.50    -      -      -      -     maxsd	(%rax), %xmm2
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -     0.50   0.50    -     mfence
 # CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     minpd	%xmm0, %xmm2
 # CHECK-NEXT:  -      -      -     1.00   0.50   0.50    -      -      -      -     minpd	(%rax), %xmm2
 # CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     minsd	%xmm0, %xmm2
@@ -838,6 +847,8 @@ xorpd       (%rax), %xmm2
 # CHECK-NEXT:  -      -      -     0.50    -      -      -     0.50    -      -     pcmpgtw	%xmm0, %xmm2
 # CHECK-NEXT:  -      -      -     0.50   0.50   0.50    -     0.50    -      -     pcmpgtw	(%rax), %xmm2
 # CHECK-NEXT:  -      -     1.00    -      -      -      -     1.00    -      -     pextrw	$1, %xmm0, %ecx
+# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -     pinsrw	$1, %eax, %xmm0
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     pinsrw	$1, (%rax), %xmm0
 # CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     pmaddwd	%xmm0, %xmm2
 # CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -      -      -      -     pmaddwd	(%rax), %xmm2
 # CHECK-NEXT:  -      -      -     0.50    -      -      -     0.50    -      -     pmaxsw	%xmm0, %xmm2
diff --git a/test/tools/llvm-mca/X86/Haswell/resources-sse3.s b/test/tools/llvm-mca/X86/Haswell/resources-sse3.s
index 776ff79bec5bb30fe466adba080784f8d1d81863..6832defc50e5921e106dc0d87ce2e53501e2df4b 100644
--- a/test/tools/llvm-mca/X86/Haswell/resources-sse3.s
+++ b/test/tools/llvm-mca/X86/Haswell/resources-sse3.s
@@ -21,6 +21,8 @@ hsubps    (%rax), %xmm2
 
 lddqu     (%rax), %xmm2
 
+monitor
+
 movddup   %xmm0, %xmm2
 movddup   (%rax), %xmm2
 
@@ -30,6 +32,8 @@ movshdup  (%rax), %xmm2
 movsldup  %xmm0, %xmm2
 movsldup  (%rax), %xmm2
 
+mwait
+
 # CHECK:      Instruction Info:
 # CHECK-NEXT: [1]: #uOps
 # CHECK-NEXT: [2]: Latency
@@ -52,12 +56,14 @@ movsldup  (%rax), %xmm2
 # CHECK-NEXT:  3      5     2.00                        hsubps	%xmm0, %xmm2
 # CHECK-NEXT:  4      11    2.00    *                   hsubps	(%rax), %xmm2
 # CHECK-NEXT:  1      6     0.50    *                   lddqu	(%rax), %xmm2
+# CHECK-NEXT:  1      100   0.25                  U     monitor
 # CHECK-NEXT:  1      1     1.00                        movddup	%xmm0, %xmm2
 # CHECK-NEXT:  1      5     0.50    *                   movddup	(%rax), %xmm2
 # CHECK-NEXT:  1      1     1.00                        movshdup	%xmm0, %xmm2
 # CHECK-NEXT:  1      6     0.50    *                   movshdup	(%rax), %xmm2
 # CHECK-NEXT:  1      1     1.00                        movsldup	%xmm0, %xmm2
 # CHECK-NEXT:  1      6     0.50    *                   movsldup	(%rax), %xmm2
+# CHECK-NEXT:  10     20    2.50    *      *      U     mwait
 
 # CHECK:      Resources:
 # CHECK-NEXT: [0]   - HWDivider
@@ -73,7 +79,7 @@ movsldup  (%rax), %xmm2
 
 # CHECK:      Resource pressure per iteration:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]
-# CHECK-NEXT:  -      -      -     12.00  5.00   5.00    -     19.00   -      -
+# CHECK-NEXT:  -      -     2.00   14.00  5.00   5.00    -     22.00  4.00    -
 
 # CHECK:      Resource pressure by instruction:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    Instructions:
@@ -90,9 +96,11 @@ movsldup  (%rax), %xmm2
 # CHECK-NEXT:  -      -      -     1.00    -      -      -     2.00    -      -     hsubps	%xmm0, %xmm2
 # CHECK-NEXT:  -      -      -     1.00   0.50   0.50    -     2.00    -      -     hsubps	(%rax), %xmm2
 # CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -     lddqu	(%rax), %xmm2
+# CHECK-NEXT:  -      -     0.25   0.25    -      -      -     0.25   0.25    -     monitor
 # CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     movddup	%xmm0, %xmm2
 # CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -     movddup	(%rax), %xmm2
 # CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     movshdup	%xmm0, %xmm2
 # CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -     movshdup	(%rax), %xmm2
 # CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     movsldup	%xmm0, %xmm2
 # CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -     movsldup	(%rax), %xmm2
+# CHECK-NEXT:  -      -     1.75   1.75    -      -      -     2.75   3.75    -     mwait
diff --git a/test/tools/llvm-mca/X86/Haswell/resources-x86_64.s b/test/tools/llvm-mca/X86/Haswell/resources-x86_64.s
index fd76bd1f53df16d8887bdb3a8c2f93672671cb27..2f6f48ba84c238eed8ba7b865c42af62380dd7e2 100644
--- a/test/tools/llvm-mca/X86/Haswell/resources-x86_64.s
+++ b/test/tools/llvm-mca/X86/Haswell/resources-x86_64.s
@@ -254,6 +254,8 @@ divl (%rax)
 divq %rcx
 divq (%rax)
 
+enter $7, $4095
+
 idivb %dil
 idivb (%rax)
 idivw %si
@@ -315,8 +317,13 @@ insl
 
 int $7
 
+invlpg  (%rax)
+invlpga %rax, %ecx
+
 lahf
 
+leave
+
 lodsb
 lodsw
 lodsl
@@ -486,6 +493,11 @@ rcrq %cl, %rdi
 rclq %cl, (%rax)
 rcrq %cl, (%rax)
 
+rdmsr
+rdpmc
+rdtsc
+rdtscp
+
 rolb %dil
 rorb %dil
 rolb (%rax)
@@ -789,6 +801,8 @@ testq %rsi, (%rax)
 
 ud2
 
+wrmsr
+
 xaddb %bl, %cl
 xaddb %bl, (%rcx)
 
@@ -1081,6 +1095,7 @@ xorq (%rax), %rdi
 # CHECK-NEXT:  2      29    10.00   *             U     divl	(%rax)
 # CHECK-NEXT:  32     98    8.00                  U     divq	%rcx
 # CHECK-NEXT:  2      29    10.00   *             U     divq	(%rax)
+# CHECK-NEXT:  1      100   0.25                  U     enter	$7, $4095
 # CHECK-NEXT:  9      23    1.00                  U     idivb	%dil
 # CHECK-NEXT:  2      29    10.00   *             U     idivb	(%rax)
 # CHECK-NEXT:  66     112   16.50                 U     idivw	%si
@@ -1133,7 +1148,10 @@ xorq (%rax), %rdi
 # CHECK-NEXT:  8      21    1.25                  U     insw	%dx, %es:(%rdi)
 # CHECK-NEXT:  8      21    1.25                  U     insl	%dx, %es:(%rdi)
 # CHECK-NEXT:  1      100   0.25    *      *      U     int	$7
+# CHECK-NEXT:  1      100   0.25                  U     invlpg	(%rax)
+# CHECK-NEXT:  1      100   0.25                  U     invlpga	%rax, %ecx
 # CHECK-NEXT:  1      1     0.50                        lahf
+# CHECK-NEXT:  3      7     0.50    *                   leave
 # CHECK-NEXT:  3      1     0.50                  U     lodsb	(%rsi), %al
 # CHECK-NEXT:  3      1     0.50                  U     lodsw	(%rsi), %ax
 # CHECK-NEXT:  2      1     0.50                  U     lodsl	(%rsi), %eax
@@ -1283,6 +1301,10 @@ xorq (%rax), %rdi
 # CHECK-NEXT:  7      11    2.00                        rcrq	%cl, %rdi
 # CHECK-NEXT:  10     16    2.00           *            rclq	%cl, (%rax)
 # CHECK-NEXT:  11     19    2.25           *            rcrq	%cl, (%rax)
+# CHECK-NEXT:  1      100   0.25                  U     rdmsr
+# CHECK-NEXT:  34     1     8.50                  U     rdpmc
+# CHECK-NEXT:  8      18    2.00                  U     rdtsc
+# CHECK-NEXT:  22     42    5.50                  U     rdtscp
 # CHECK-NEXT:  2      2     1.00                        rolb	%dil
 # CHECK-NEXT:  2      2     1.00                        rorb	%dil
 # CHECK-NEXT:  5      8     1.00    *      *            rolb	(%rax)
@@ -1557,6 +1579,7 @@ xorq (%rax), %rdi
 # CHECK-NEXT:  1      1     0.25                        testq	%rsi, %rdi
 # CHECK-NEXT:  2      6     0.50    *                   testq	%rsi, (%rax)
 # CHECK-NEXT:  1      100   0.25    *             U     ud2
+# CHECK-NEXT:  1      100   0.25                  U     wrmsr
 # CHECK-NEXT:  3      2     0.75                        xaddb	%bl, %cl
 # CHECK-NEXT:  5      8     1.00    *      *            xaddb	%bl, (%rcx)
 # CHECK-NEXT:  3      2     0.75                        xaddw	%bx, %cx
@@ -1622,7 +1645,7 @@ xorq (%rax), %rdi
 
 # CHECK:      Resource pressure per iteration:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]
-# CHECK-NEXT: 80.00   -     548.75 359.25 262.67 262.67 201.00 270.75 610.25 79.67
+# CHECK-NEXT: 80.00   -     557.25 367.25 263.17 263.17 201.00 281.75 619.75 79.67
 
 # CHECK:      Resource pressure by instruction:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    Instructions:
@@ -1846,6 +1869,7 @@ xorq (%rax), %rdi
 # CHECK-NEXT: 10.00   -     1.00    -     0.50   0.50    -      -      -      -     divl	(%rax)
 # CHECK-NEXT:  -      -     10.25  10.25   -      -      -     5.75   5.75    -     divq	%rcx
 # CHECK-NEXT: 10.00   -     1.00    -     0.50   0.50    -      -      -      -     divq	(%rax)
+# CHECK-NEXT:  -      -     0.25   0.25    -      -      -     0.25   0.25    -     enter	$7, $4095
 # CHECK-NEXT:  -      -     1.00   1.00    -      -      -     1.00   1.00    -     idivb	%dil
 # CHECK-NEXT: 10.00   -     1.00    -     0.50   0.50    -      -      -      -     idivb	(%rax)
 # CHECK-NEXT:  -      -     19.50  10.50   -      -      -     12.50  23.50   -     idivw	%si
@@ -1898,7 +1922,10 @@ xorq (%rax), %rdi
 # CHECK-NEXT:  -      -     1.00   0.50   0.83   0.83   1.00   1.50   2.00   0.33   insw	%dx, %es:(%rdi)
 # CHECK-NEXT:  -      -     1.00   0.50   0.83   0.83   1.00   1.50   2.00   0.33   insl	%dx, %es:(%rdi)
 # CHECK-NEXT:  -      -     0.25   0.25    -      -      -     0.25   0.25    -     int	$7
+# CHECK-NEXT:  -      -     0.25   0.25    -      -      -     0.25   0.25    -     invlpg	(%rax)
+# CHECK-NEXT:  -      -     0.25   0.25    -      -      -     0.25   0.25    -     invlpga	%rax, %ecx
 # CHECK-NEXT:  -      -     0.50    -      -      -      -      -     0.50    -     lahf
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -     0.50   0.50    -     leave
 # CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -     0.50   0.50    -     lodsb	(%rsi), %al
 # CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -     0.50   0.50    -     lodsw	(%rsi), %ax
 # CHECK-NEXT:  -      -     0.25   0.25   0.50   0.50    -     0.25   0.25    -     lodsl	(%rsi), %eax
@@ -2048,6 +2075,10 @@ xorq (%rax), %rdi
 # CHECK-NEXT:  -      -     1.75   2.75    -      -      -     0.75   1.75    -     rcrq	%cl, %rdi
 # CHECK-NEXT:  -      -     2.50   2.00   0.83   0.83    -     1.00   2.50   0.33   rclq	%cl, (%rax)
 # CHECK-NEXT:  -      -     2.25   3.25   0.83   0.83    -     1.25   2.25   0.33   rcrq	%cl, (%rax)
+# CHECK-NEXT:  -      -     0.25   0.25    -      -      -     0.25   0.25    -     rdmsr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     rdpmc
+# CHECK-NEXT:  -      -     1.75   1.25    -      -      -     2.25   2.75    -     rdtsc
+# CHECK-NEXT:  -      -     5.00   5.00    -      -      -     7.00   5.00    -     rdtscp
 # CHECK-NEXT:  -      -     1.00    -      -      -      -      -     1.00    -     rolb	%dil
 # CHECK-NEXT:  -      -     1.00    -      -      -      -      -     1.00    -     rorb	%dil
 # CHECK-NEXT:  -      -     1.00    -     0.83   0.83   1.00    -     1.00   0.33   rolb	(%rax)
@@ -2322,6 +2353,7 @@ xorq (%rax), %rdi
 # CHECK-NEXT:  -      -     0.25   0.25    -      -      -     0.25   0.25    -     testq	%rsi, %rdi
 # CHECK-NEXT:  -      -     0.25   0.25   0.50   0.50    -     0.25   0.25    -     testq	%rsi, (%rax)
 # CHECK-NEXT:  -      -     0.25   0.25    -      -      -     0.25   0.25    -     ud2
+# CHECK-NEXT:  -      -     0.25   0.25    -      -      -     0.25   0.25    -     wrmsr
 # CHECK-NEXT:  -      -     0.75   0.75    -      -      -     0.75   0.75    -     xaddb	%bl, %cl
 # CHECK-NEXT:  -      -     0.50   0.50   0.83   0.83   1.00   0.50   0.50   0.33   xaddb	%bl, (%rcx)
 # CHECK-NEXT:  -      -     0.75   0.75    -      -      -     0.75   0.75    -     xaddw	%bx, %cx
diff --git a/test/tools/llvm-mca/X86/Haswell/resources-x87.s b/test/tools/llvm-mca/X86/Haswell/resources-x87.s
index 53006bbc3296e1766c0ac5fb01785952debc5cd8..7da8b2802a9a16aa3bb733af80a0051ca98f306f 100644
--- a/test/tools/llvm-mca/X86/Haswell/resources-x87.s
+++ b/test/tools/llvm-mca/X86/Haswell/resources-x87.s
@@ -5,7 +5,7 @@ f2xm1
 
 fabs
 
-fadd %st(0), %st(1)
+fadd %st, %st(1)
 fadd %st(2)
 fadds (%ecx)
 faddl (%ecx)
@@ -21,14 +21,14 @@ fchs
 
 fnclex
 
-fcmovb %st(1), %st(0)
-fcmovbe %st(1), %st(0)
-fcmove %st(1), %st(0)
-fcmovnb %st(1), %st(0)
-fcmovnbe %st(1), %st(0)
-fcmovne %st(1), %st(0)
-fcmovnu %st(1), %st(0)
-fcmovu %st(1), %st(0)
+fcmovb %st(1), %st
+fcmovbe %st(1), %st
+fcmove %st(1), %st
+fcmovnb %st(1), %st
+fcmovnbe %st(1), %st
+fcmovne %st(1), %st
+fcmovnu %st(1), %st
+fcmovu %st(1), %st
 
 fcom %st(1)
 fcom %st(3)
@@ -47,7 +47,7 @@ fcos
 
 fdecstp
 
-fdiv %st(0), %st(1)
+fdiv %st, %st(1)
 fdiv %st(2)
 fdivs (%ecx)
 fdivl (%eax)
@@ -56,7 +56,7 @@ fdivp %st(2)
 fidivs (%ecx)
 fidivl (%eax)
 
-fdivr %st(0), %st(1)
+fdivr %st, %st(1)
 fdivr %st(2)
 fdivrs (%ecx)
 fdivrl (%eax)
@@ -106,7 +106,7 @@ fldln2
 fldpi
 fldz
 
-fmul %st(0), %st(1)
+fmul %st, %st(1)
 fmul %st(2)
 fmuls (%ecx)
 fmull (%eax)
@@ -153,7 +153,7 @@ fnstsw (%eax)
 frstor (%eax)
 fsave (%eax)
 
-fsub %st(0), %st(1)
+fsub %st, %st(1)
 fsub %st(2)
 fsubs (%ecx)
 fsubl (%eax)
@@ -162,7 +162,7 @@ fsubp %st(2)
 fisubs (%ecx)
 fisubl (%eax)
 
-fsubr %st(0), %st(1)
+fsubr %st, %st(1)
 fsubr %st(2)
 fsubrs (%ecx)
 fsubrl (%eax)
@@ -208,26 +208,26 @@ fyl2xp1
 # CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
 # CHECK-NEXT:  1      100   0.25                  U     f2xm1
 # CHECK-NEXT:  1      1     1.00                  U     fabs
-# CHECK-NEXT:  1      3     1.00                  U     fadd	%st(0), %st(1)
-# CHECK-NEXT:  1      3     1.00                  U     fadd	%st(2)
+# CHECK-NEXT:  1      3     1.00                  U     fadd	%st, %st(1)
+# CHECK-NEXT:  1      3     1.00                  U     fadd	%st(2), %st
 # CHECK-NEXT:  2      10    1.00    *             U     fadds	(%ecx)
 # CHECK-NEXT:  2      10    1.00    *             U     faddl	(%ecx)
-# CHECK-NEXT:  1      3     1.00                  U     faddp	%st(1)
-# CHECK-NEXT:  1      3     1.00                  U     faddp	%st(2)
+# CHECK-NEXT:  1      3     1.00                  U     faddp	%st, %st(1)
+# CHECK-NEXT:  1      3     1.00                  U     faddp	%st, %st(2)
 # CHECK-NEXT:  3      13    2.00    *             U     fiadds	(%ecx)
 # CHECK-NEXT:  3      13    2.00    *             U     fiaddl	(%ecx)
 # CHECK-NEXT:  43     47    10.75                 U     fbld	(%ecx)
 # CHECK-NEXT:  2      1     1.00                  U     fbstp	(%eax)
 # CHECK-NEXT:  1      1     1.00                  U     fchs
 # CHECK-NEXT:  4      4     1.00                  U     fnclex
-# CHECK-NEXT:  1      3     1.00                  U     fcmovb	%st(1), %st(0)
-# CHECK-NEXT:  1      3     1.00                  U     fcmovbe	%st(1), %st(0)
-# CHECK-NEXT:  1      3     1.00                  U     fcmove	%st(1), %st(0)
-# CHECK-NEXT:  1      3     1.00                  U     fcmovnb	%st(1), %st(0)
-# CHECK-NEXT:  1      3     1.00                  U     fcmovnbe	%st(1), %st(0)
-# CHECK-NEXT:  1      3     1.00                  U     fcmovne	%st(1), %st(0)
-# CHECK-NEXT:  1      3     1.00                  U     fcmovnu	%st(1), %st(0)
-# CHECK-NEXT:  1      3     1.00                  U     fcmovu	%st(1), %st(0)
+# CHECK-NEXT:  1      3     1.00                  U     fcmovb	%st(1), %st
+# CHECK-NEXT:  1      3     1.00                  U     fcmovbe	%st(1), %st
+# CHECK-NEXT:  1      3     1.00                  U     fcmove	%st(1), %st
+# CHECK-NEXT:  1      3     1.00                  U     fcmovnb	%st(1), %st
+# CHECK-NEXT:  1      3     1.00                  U     fcmovnbe	%st(1), %st
+# CHECK-NEXT:  1      3     1.00                  U     fcmovne	%st(1), %st
+# CHECK-NEXT:  1      3     1.00                  U     fcmovnu	%st(1), %st
+# CHECK-NEXT:  1      3     1.00                  U     fcmovu	%st(1), %st
 # CHECK-NEXT:  1      1     1.00                  U     fcom	%st(1)
 # CHECK-NEXT:  1      1     1.00                  U     fcom	%st(3)
 # CHECK-NEXT:  2      8     1.00                  U     fcoms	(%ecx)
@@ -237,24 +237,24 @@ fyl2xp1
 # CHECK-NEXT:  2      8     1.00                  U     fcomps	(%ecx)
 # CHECK-NEXT:  2      8     1.00                  U     fcompl	(%eax)
 # CHECK-NEXT:  2      1     0.50                  U     fcompp
-# CHECK-NEXT:  3      1     0.50                  U     fcomi	%st(3)
-# CHECK-NEXT:  3      1     0.50                  U     fcompi	%st(3)
+# CHECK-NEXT:  3      1     0.50                  U     fcomi	%st(3), %st
+# CHECK-NEXT:  3      1     0.50                  U     fcompi	%st(3), %st
 # CHECK-NEXT:  1      100   0.25                  U     fcos
 # CHECK-NEXT:  2      2     1.00                  U     fdecstp
-# CHECK-NEXT:  1      24    1.00                  U     fdiv	%st(0), %st(1)
-# CHECK-NEXT:  1      20    1.00                  U     fdiv	%st(2)
+# CHECK-NEXT:  1      24    1.00                  U     fdiv	%st, %st(1)
+# CHECK-NEXT:  1      20    1.00                  U     fdiv	%st(2), %st
 # CHECK-NEXT:  2      31    1.00    *             U     fdivs	(%ecx)
 # CHECK-NEXT:  2      31    1.00    *             U     fdivl	(%eax)
-# CHECK-NEXT:  1      24    1.00                  U     fdivp	%st(1)
-# CHECK-NEXT:  1      24    1.00                  U     fdivp	%st(2)
+# CHECK-NEXT:  1      24    1.00                  U     fdivp	%st, %st(1)
+# CHECK-NEXT:  1      24    1.00                  U     fdivp	%st, %st(2)
 # CHECK-NEXT:  3      34    1.00    *             U     fidivs	(%ecx)
 # CHECK-NEXT:  3      34    1.00    *             U     fidivl	(%eax)
-# CHECK-NEXT:  1      20    1.00                  U     fdivr	%st(0), %st(1)
-# CHECK-NEXT:  1      24    1.00                  U     fdivr	%st(2)
+# CHECK-NEXT:  1      20    1.00                  U     fdivr	%st, %st(1)
+# CHECK-NEXT:  1      24    1.00                  U     fdivr	%st(2), %st
 # CHECK-NEXT:  2      27    1.00    *             U     fdivrs	(%ecx)
 # CHECK-NEXT:  2      27    1.00    *             U     fdivrl	(%eax)
-# CHECK-NEXT:  1      20    1.00                  U     fdivrp	%st(1)
-# CHECK-NEXT:  1      20    1.00                  U     fdivrp	%st(2)
+# CHECK-NEXT:  1      20    1.00                  U     fdivrp	%st, %st(1)
+# CHECK-NEXT:  1      20    1.00                  U     fdivrp	%st, %st(2)
 # CHECK-NEXT:  3      30    1.00    *             U     fidivrs	(%ecx)
 # CHECK-NEXT:  3      30    1.00    *             U     fidivrl	(%eax)
 # CHECK-NEXT:  1      1     0.50                  U     ffree	%st(0)
@@ -288,12 +288,12 @@ fyl2xp1
 # CHECK-NEXT:  2      1     1.00                  U     fldln2
 # CHECK-NEXT:  2      1     1.00                  U     fldpi
 # CHECK-NEXT:  1      1     0.50                  U     fldz
-# CHECK-NEXT:  1      5     1.00                  U     fmul	%st(0), %st(1)
-# CHECK-NEXT:  1      5     1.00                  U     fmul	%st(2)
+# CHECK-NEXT:  1      5     1.00                  U     fmul	%st, %st(1)
+# CHECK-NEXT:  1      5     1.00                  U     fmul	%st(2), %st
 # CHECK-NEXT:  2      12    1.00    *             U     fmuls	(%ecx)
 # CHECK-NEXT:  2      12    1.00    *             U     fmull	(%eax)
-# CHECK-NEXT:  1      5     1.00                  U     fmulp	%st(1)
-# CHECK-NEXT:  1      5     1.00                  U     fmulp	%st(2)
+# CHECK-NEXT:  1      5     1.00                  U     fmulp	%st, %st(1)
+# CHECK-NEXT:  1      5     1.00                  U     fmulp	%st, %st(2)
 # CHECK-NEXT:  3      15    1.00    *             U     fimuls	(%ecx)
 # CHECK-NEXT:  3      15    1.00    *             U     fimull	(%eax)
 # CHECK-NEXT:  1      1     0.50                  U     fnop
@@ -321,20 +321,20 @@ fyl2xp1
 # CHECK-NEXT:  90     1     22.50                 U     frstor	(%eax)
 # CHECK-NEXT:  2      2     0.50                  U     wait
 # CHECK-NEXT:  147    1     36.75                 U     fnsave	(%eax)
-# CHECK-NEXT:  1      3     1.00                  U     fsub	%st(0), %st(1)
-# CHECK-NEXT:  1      3     1.00                  U     fsub	%st(2)
+# CHECK-NEXT:  1      3     1.00                  U     fsub	%st, %st(1)
+# CHECK-NEXT:  1      3     1.00                  U     fsub	%st(2), %st
 # CHECK-NEXT:  2      10    1.00    *             U     fsubs	(%ecx)
 # CHECK-NEXT:  2      10    1.00    *             U     fsubl	(%eax)
-# CHECK-NEXT:  1      3     1.00                  U     fsubp	%st(1)
-# CHECK-NEXT:  1      3     1.00                  U     fsubp	%st(2)
+# CHECK-NEXT:  1      3     1.00                  U     fsubp	%st, %st(1)
+# CHECK-NEXT:  1      3     1.00                  U     fsubp	%st, %st(2)
 # CHECK-NEXT:  3      13    2.00    *             U     fisubs	(%ecx)
 # CHECK-NEXT:  3      13    2.00    *             U     fisubl	(%eax)
-# CHECK-NEXT:  1      3     1.00                  U     fsubr	%st(0), %st(1)
-# CHECK-NEXT:  1      3     1.00                  U     fsubr	%st(2)
+# CHECK-NEXT:  1      3     1.00                  U     fsubr	%st, %st(1)
+# CHECK-NEXT:  1      3     1.00                  U     fsubr	%st(2), %st
 # CHECK-NEXT:  2      10    1.00    *             U     fsubrs	(%ecx)
 # CHECK-NEXT:  2      10    1.00    *             U     fsubrl	(%eax)
-# CHECK-NEXT:  1      3     1.00                  U     fsubrp	%st(1)
-# CHECK-NEXT:  1      3     1.00                  U     fsubrp	%st(2)
+# CHECK-NEXT:  1      3     1.00                  U     fsubrp	%st, %st(1)
+# CHECK-NEXT:  1      3     1.00                  U     fsubrp	%st, %st(2)
 # CHECK-NEXT:  3      13    2.00    *             U     fisubrs	(%ecx)
 # CHECK-NEXT:  3      13    2.00    *             U     fisubrl	(%eax)
 # CHECK-NEXT:  1      1     1.00                  U     ftst
@@ -343,8 +343,8 @@ fyl2xp1
 # CHECK-NEXT:  1      1     1.00                  U     fucomp	%st(1)
 # CHECK-NEXT:  1      1     1.00                  U     fucomp	%st(3)
 # CHECK-NEXT:  2      1     0.50                  U     fucompp
-# CHECK-NEXT:  3      1     0.50                  U     fucomi	%st(3)
-# CHECK-NEXT:  3      1     0.50                  U     fucompi	%st(3)
+# CHECK-NEXT:  3      1     0.50                  U     fucomi	%st(3), %st
+# CHECK-NEXT:  3      1     0.50                  U     fucompi	%st(3), %st
 # CHECK-NEXT:  2      2     0.50                  U     wait
 # CHECK-NEXT:  2      1     2.00                  U     fxam
 # CHECK-NEXT:  15     17    4.00                  U     fxch	%st(1)
@@ -375,26 +375,26 @@ fyl2xp1
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    Instructions:
 # CHECK-NEXT:  -      -     0.25   0.25    -      -      -     0.25   0.25    -     f2xm1
 # CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     fabs
-# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     fadd	%st(0), %st(1)
-# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     fadd	%st(2)
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     fadd	%st, %st(1)
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     fadd	%st(2), %st
 # CHECK-NEXT:  -      -      -     1.00   0.50   0.50    -      -      -      -     fadds	(%ecx)
 # CHECK-NEXT:  -      -      -     1.00   0.50   0.50    -      -      -      -     faddl	(%ecx)
-# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     faddp	%st(1)
-# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     faddp	%st(2)
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     faddp	%st, %st(1)
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     faddp	%st, %st(2)
 # CHECK-NEXT:  -      -      -     2.00   0.50   0.50    -      -      -      -     fiadds	(%ecx)
 # CHECK-NEXT:  -      -      -     2.00   0.50   0.50    -      -      -      -     fiaddl	(%ecx)
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     fbld	(%ecx)
 # CHECK-NEXT:  -      -      -      -     0.33   0.33   1.00    -      -     0.33   fbstp	(%eax)
 # CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     fchs
 # CHECK-NEXT:  -      -     1.00   1.00    -      -      -     1.00   1.00    -     fnclex
-# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     fcmovb	%st(1), %st(0)
-# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     fcmovbe	%st(1), %st(0)
-# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     fcmove	%st(1), %st(0)
-# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     fcmovnb	%st(1), %st(0)
-# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     fcmovnbe	%st(1), %st(0)
-# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     fcmovne	%st(1), %st(0)
-# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     fcmovnu	%st(1), %st(0)
-# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     fcmovu	%st(1), %st(0)
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     fcmovb	%st(1), %st
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     fcmovbe	%st(1), %st
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     fcmove	%st(1), %st
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     fcmovnb	%st(1), %st
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     fcmovnbe	%st(1), %st
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     fcmovne	%st(1), %st
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     fcmovnu	%st(1), %st
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     fcmovu	%st(1), %st
 # CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     fcom	%st(1)
 # CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     fcom	%st(3)
 # CHECK-NEXT:  -      -      -     1.00   0.50   0.50    -      -      -      -     fcoms	(%ecx)
@@ -404,24 +404,24 @@ fyl2xp1
 # CHECK-NEXT:  -      -      -     1.00   0.50   0.50    -      -      -      -     fcomps	(%ecx)
 # CHECK-NEXT:  -      -      -     1.00   0.50   0.50    -      -      -      -     fcompl	(%eax)
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -     fcompp
-# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -     fcomi	%st(3)
-# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -     fcompi	%st(3)
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -     fcomi	%st(3), %st
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -     fcompi	%st(3), %st
 # CHECK-NEXT:  -      -     0.25   0.25    -      -      -     0.25   0.25    -     fcos
 # CHECK-NEXT:  -      -     1.00   1.00    -      -      -      -      -      -     fdecstp
-# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     fdiv	%st(0), %st(1)
-# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     fdiv	%st(2)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     fdiv	%st, %st(1)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     fdiv	%st(2), %st
 # CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -      -      -      -     fdivs	(%ecx)
 # CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -      -      -      -     fdivl	(%eax)
-# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     fdivp	%st(1)
-# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     fdivp	%st(2)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     fdivp	%st, %st(1)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     fdivp	%st, %st(2)
 # CHECK-NEXT:  -      -     1.00   1.00   0.50   0.50    -      -      -      -     fidivs	(%ecx)
 # CHECK-NEXT:  -      -     1.00   1.00   0.50   0.50    -      -      -      -     fidivl	(%eax)
-# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     fdivr	%st(0), %st(1)
-# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     fdivr	%st(2)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     fdivr	%st, %st(1)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     fdivr	%st(2), %st
 # CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -      -      -      -     fdivrs	(%ecx)
 # CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -      -      -      -     fdivrl	(%eax)
-# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     fdivrp	%st(1)
-# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     fdivrp	%st(2)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     fdivrp	%st, %st(1)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     fdivrp	%st, %st(2)
 # CHECK-NEXT:  -      -     1.00   1.00   0.50   0.50    -      -      -      -     fidivrs	(%ecx)
 # CHECK-NEXT:  -      -     1.00   1.00   0.50   0.50    -      -      -      -     fidivrl	(%eax)
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -     ffree	%st(0)
@@ -455,12 +455,12 @@ fyl2xp1
 # CHECK-NEXT:  -      -     1.00   1.00    -      -      -      -      -      -     fldln2
 # CHECK-NEXT:  -      -     1.00   1.00    -      -      -      -      -      -     fldpi
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -     fldz
-# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     fmul	%st(0), %st(1)
-# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     fmul	%st(2)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     fmul	%st, %st(1)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     fmul	%st(2), %st
 # CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -      -      -      -     fmuls	(%ecx)
 # CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -      -      -      -     fmull	(%eax)
-# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     fmulp	%st(1)
-# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     fmulp	%st(2)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     fmulp	%st, %st(1)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     fmulp	%st, %st(2)
 # CHECK-NEXT:  -      -     1.00   1.00   0.50   0.50    -      -      -      -     fimuls	(%ecx)
 # CHECK-NEXT:  -      -     1.00   1.00   0.50   0.50    -      -      -      -     fimull	(%eax)
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -     fnop
@@ -488,20 +488,20 @@ fyl2xp1
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     frstor	(%eax)
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -     0.50   0.50    -     wait
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     fnsave	(%eax)
-# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     fsub	%st(0), %st(1)
-# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     fsub	%st(2)
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     fsub	%st, %st(1)
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     fsub	%st(2), %st
 # CHECK-NEXT:  -      -      -     1.00   0.50   0.50    -      -      -      -     fsubs	(%ecx)
 # CHECK-NEXT:  -      -      -     1.00   0.50   0.50    -      -      -      -     fsubl	(%eax)
-# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     fsubp	%st(1)
-# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     fsubp	%st(2)
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     fsubp	%st, %st(1)
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     fsubp	%st, %st(2)
 # CHECK-NEXT:  -      -      -     2.00   0.50   0.50    -      -      -      -     fisubs	(%ecx)
 # CHECK-NEXT:  -      -      -     2.00   0.50   0.50    -      -      -      -     fisubl	(%eax)
-# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     fsubr	%st(0), %st(1)
-# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     fsubr	%st(2)
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     fsubr	%st, %st(1)
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     fsubr	%st(2), %st
 # CHECK-NEXT:  -      -      -     1.00   0.50   0.50    -      -      -      -     fsubrs	(%ecx)
 # CHECK-NEXT:  -      -      -     1.00   0.50   0.50    -      -      -      -     fsubrl	(%eax)
-# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     fsubrp	%st(1)
-# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     fsubrp	%st(2)
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     fsubrp	%st, %st(1)
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     fsubrp	%st, %st(2)
 # CHECK-NEXT:  -      -      -     2.00   0.50   0.50    -      -      -      -     fisubrs	(%ecx)
 # CHECK-NEXT:  -      -      -     2.00   0.50   0.50    -      -      -      -     fisubrl	(%eax)
 # CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     ftst
@@ -510,8 +510,8 @@ fyl2xp1
 # CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     fucomp	%st(1)
 # CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     fucomp	%st(3)
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -     fucompp
-# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -     fucomi	%st(3)
-# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -     fucompi	%st(3)
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -     fucomi	%st(3), %st
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -     fucompi	%st(3), %st
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -     0.50   0.50    -     wait
 # CHECK-NEXT:  -      -      -     2.00    -      -      -      -      -      -     fxam
 # CHECK-NEXT:  -      -     4.00   3.00    -      -      -     3.00   5.00    -     fxch	%st(1)
diff --git a/test/tools/llvm-mca/X86/SLM/resources-sse2.s b/test/tools/llvm-mca/X86/SLM/resources-sse2.s
index 9ab05ee1083aff06e3c92f446a05420f5162b707..12b17629b8dfe1c84d884e13beb5ed86517196a2 100644
--- a/test/tools/llvm-mca/X86/SLM/resources-sse2.s
+++ b/test/tools/llvm-mca/X86/SLM/resources-sse2.s
@@ -94,6 +94,8 @@ maxpd       (%rax), %xmm2
 maxsd       %xmm0, %xmm2
 maxsd       (%rax), %xmm2
 
+mfence
+
 minpd       %xmm0, %xmm2
 minpd       (%rax), %xmm2
 
@@ -229,6 +231,9 @@ pcmpgtw     (%rax), %xmm2
 
 pextrw      $1, %xmm0, %rcx
 
+pinsrw      $1, %rax, %xmm0
+pinsrw      $1, (%rax), %xmm0
+
 pmaddwd     %xmm0, %xmm2
 pmaddwd     (%rax), %xmm2
 
@@ -465,6 +470,7 @@ xorpd       (%rax), %xmm2
 # CHECK-NEXT:  1      6     1.00    *                   maxpd	(%rax), %xmm2
 # CHECK-NEXT:  1      3     1.00                        maxsd	%xmm0, %xmm2
 # CHECK-NEXT:  1      6     1.00    *                   maxsd	(%rax), %xmm2
+# CHECK-NEXT:  1      1     1.00    *      *      U     mfence
 # CHECK-NEXT:  1      3     1.00                        minpd	%xmm0, %xmm2
 # CHECK-NEXT:  1      6     1.00    *                   minpd	(%rax), %xmm2
 # CHECK-NEXT:  1      3     1.00                        minsd	%xmm0, %xmm2
@@ -555,6 +561,8 @@ xorpd       (%rax), %xmm2
 # CHECK-NEXT:  1      1     0.50                        pcmpgtw	%xmm0, %xmm2
 # CHECK-NEXT:  1      4     1.00    *                   pcmpgtw	(%rax), %xmm2
 # CHECK-NEXT:  1      1     1.00                        pextrw	$1, %xmm0, %ecx
+# CHECK-NEXT:  1      1     1.00                        pinsrw	$1, %eax, %xmm0
+# CHECK-NEXT:  1      4     1.00    *                   pinsrw	$1, (%rax), %xmm0
 # CHECK-NEXT:  1      4     1.00                        pmaddwd	%xmm0, %xmm2
 # CHECK-NEXT:  1      7     1.00    *                   pmaddwd	(%rax), %xmm2
 # CHECK-NEXT:  1      1     0.50                        pmaxsw	%xmm0, %xmm2
@@ -679,7 +687,7 @@ xorpd       (%rax), %xmm2
 
 # CHECK:      Resource pressure per iteration:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]
-# CHECK-NEXT:  -     412.00 8.00   150.50 86.50  3.00   3.00   132.00
+# CHECK-NEXT:  -     412.00 8.00   152.50 86.50  3.00   3.00   134.00
 
 # CHECK:      Resource pressure by instruction:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    Instructions:
@@ -746,6 +754,7 @@ xorpd       (%rax), %xmm2
 # CHECK-NEXT:  -      -      -      -     1.00    -      -     1.00   maxpd	(%rax), %xmm2
 # CHECK-NEXT:  -      -      -      -     1.00    -      -      -     maxsd	%xmm0, %xmm2
 # CHECK-NEXT:  -      -      -      -     1.00    -      -     1.00   maxsd	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00   mfence
 # CHECK-NEXT:  -      -      -      -     1.00    -      -      -     minpd	%xmm0, %xmm2
 # CHECK-NEXT:  -      -      -      -     1.00    -      -     1.00   minpd	(%rax), %xmm2
 # CHECK-NEXT:  -      -      -      -     1.00    -      -      -     minsd	%xmm0, %xmm2
@@ -836,6 +845,8 @@ xorpd       (%rax), %xmm2
 # CHECK-NEXT:  -      -      -     0.50   0.50    -      -      -     pcmpgtw	%xmm0, %xmm2
 # CHECK-NEXT:  -      -      -     0.50   0.50    -      -     1.00   pcmpgtw	(%rax), %xmm2
 # CHECK-NEXT:  -      -      -     1.00    -      -      -      -     pextrw	$1, %xmm0, %ecx
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -     pinsrw	$1, %eax, %xmm0
+# CHECK-NEXT:  -      -      -     1.00    -      -      -     1.00   pinsrw	$1, (%rax), %xmm0
 # CHECK-NEXT:  -      -      -     1.00    -      -      -      -     pmaddwd	%xmm0, %xmm2
 # CHECK-NEXT:  -      -      -     1.00    -      -      -     1.00   pmaddwd	(%rax), %xmm2
 # CHECK-NEXT:  -      -      -     0.50   0.50    -      -      -     pmaxsw	%xmm0, %xmm2
diff --git a/test/tools/llvm-mca/X86/SLM/resources-sse3.s b/test/tools/llvm-mca/X86/SLM/resources-sse3.s
index ba140f216ed74da4b3fd695ca3e984ff3f07a9f1..484c353b07bf82e012c63cd5269a469c5b5a8011 100644
--- a/test/tools/llvm-mca/X86/SLM/resources-sse3.s
+++ b/test/tools/llvm-mca/X86/SLM/resources-sse3.s
@@ -21,6 +21,8 @@ hsubps    (%rax), %xmm2
 
 lddqu     (%rax), %xmm2
 
+monitor
+
 movddup   %xmm0, %xmm2
 movddup   (%rax), %xmm2
 
@@ -30,6 +32,8 @@ movshdup  (%rax), %xmm2
 movsldup  %xmm0, %xmm2
 movsldup  (%rax), %xmm2
 
+mwait
+
 # CHECK:      Instruction Info:
 # CHECK-NEXT: [1]: #uOps
 # CHECK-NEXT: [2]: Latency
@@ -52,12 +56,14 @@ movsldup  (%rax), %xmm2
 # CHECK-NEXT:  1      3     1.00                        hsubps	%xmm0, %xmm2
 # CHECK-NEXT:  1      6     1.00    *                   hsubps	(%rax), %xmm2
 # CHECK-NEXT:  1      3     1.00    *                   lddqu	(%rax), %xmm2
+# CHECK-NEXT:  1      100   1.00                  U     monitor
 # CHECK-NEXT:  1      1     1.00                        movddup	%xmm0, %xmm2
 # CHECK-NEXT:  1      4     1.00    *                   movddup	(%rax), %xmm2
 # CHECK-NEXT:  1      1     1.00                        movshdup	%xmm0, %xmm2
 # CHECK-NEXT:  1      4     1.00    *                   movshdup	(%rax), %xmm2
 # CHECK-NEXT:  1      1     1.00                        movsldup	%xmm0, %xmm2
 # CHECK-NEXT:  1      4     1.00    *                   movsldup	(%rax), %xmm2
+# CHECK-NEXT:  1      100   1.00    *      *      U     mwait
 
 # CHECK:      Resources:
 # CHECK-NEXT: [0]   - SLMDivider
@@ -71,7 +77,7 @@ movsldup  (%rax), %xmm2
 
 # CHECK:      Resource pressure per iteration:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]
-# CHECK-NEXT:  -      -      -     14.00  12.00   -      -     10.00
+# CHECK-NEXT:  -      -      -     16.00  12.00   -      -     10.00
 
 # CHECK:      Resource pressure by instruction:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    Instructions:
@@ -88,9 +94,11 @@ movsldup  (%rax), %xmm2
 # CHECK-NEXT:  -      -      -     1.00   1.00    -      -      -     hsubps	%xmm0, %xmm2
 # CHECK-NEXT:  -      -      -     1.00   1.00    -      -     1.00   hsubps	(%rax), %xmm2
 # CHECK-NEXT:  -      -      -      -      -      -      -     1.00   lddqu	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -     monitor
 # CHECK-NEXT:  -      -      -     1.00    -      -      -      -     movddup	%xmm0, %xmm2
 # CHECK-NEXT:  -      -      -     1.00    -      -      -     1.00   movddup	(%rax), %xmm2
 # CHECK-NEXT:  -      -      -     1.00    -      -      -      -     movshdup	%xmm0, %xmm2
 # CHECK-NEXT:  -      -      -     1.00    -      -      -     1.00   movshdup	(%rax), %xmm2
 # CHECK-NEXT:  -      -      -     1.00    -      -      -      -     movsldup	%xmm0, %xmm2
 # CHECK-NEXT:  -      -      -     1.00    -      -      -     1.00   movsldup	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -     mwait
diff --git a/test/tools/llvm-mca/X86/SLM/resources-x86_64.s b/test/tools/llvm-mca/X86/SLM/resources-x86_64.s
index 6c6c4d06c706ad25163bf1d9a88e6602fa0a7d55..9add7a7d63e9bc489d91959fada50e1b5600fe95 100644
--- a/test/tools/llvm-mca/X86/SLM/resources-x86_64.s
+++ b/test/tools/llvm-mca/X86/SLM/resources-x86_64.s
@@ -254,6 +254,8 @@ divl (%rax)
 divq %rcx
 divq (%rax)
 
+enter $7, $4095
+
 idivb %dil
 idivb (%rax)
 idivw %si
@@ -315,8 +317,13 @@ insl
 
 int $7
 
+invlpg  (%rax)
+invlpga %rax, %ecx
+
 lahf
 
+leave
+
 lodsb
 lodsw
 lodsl
@@ -486,6 +493,11 @@ rcrq %cl, %rdi
 rclq %cl, (%rax)
 rcrq %cl, (%rax)
 
+rdmsr
+rdpmc
+rdtsc
+rdtscp
+
 rolb %dil
 rorb %dil
 rolb (%rax)
@@ -789,6 +801,8 @@ testq %rsi, (%rax)
 
 ud2
 
+wrmsr
+
 xaddb %bl, %cl
 xaddb %bl, (%rcx)
 
@@ -1081,6 +1095,7 @@ xorq (%rax), %rdi
 # CHECK-NEXT:  1      29    25.00   *             U     divl	(%rax)
 # CHECK-NEXT:  1      25    25.00                 U     divq	%rcx
 # CHECK-NEXT:  1      29    25.00   *             U     divq	(%rax)
+# CHECK-NEXT:  1      100   1.00                  U     enter	$7, $4095
 # CHECK-NEXT:  1      25    25.00                 U     idivb	%dil
 # CHECK-NEXT:  1      29    25.00   *             U     idivb	(%rax)
 # CHECK-NEXT:  1      25    25.00                 U     idivw	%si
@@ -1133,7 +1148,10 @@ xorq (%rax), %rdi
 # CHECK-NEXT:  1      100   1.00                  U     insw	%dx, %es:(%rdi)
 # CHECK-NEXT:  1      100   1.00                  U     insl	%dx, %es:(%rdi)
 # CHECK-NEXT:  1      100   1.00    *      *      U     int	$7
+# CHECK-NEXT:  1      100   1.00                  U     invlpg	(%rax)
+# CHECK-NEXT:  1      100   1.00                  U     invlpga	%rax, %ecx
 # CHECK-NEXT:  1      1     0.50                        lahf
+# CHECK-NEXT:  1      1     0.50    *                   leave
 # CHECK-NEXT:  1      100   1.00                  U     lodsb	(%rsi), %al
 # CHECK-NEXT:  1      100   1.00                  U     lodsw	(%rsi), %ax
 # CHECK-NEXT:  1      100   1.00                  U     lodsl	(%rsi), %eax
@@ -1283,6 +1301,10 @@ xorq (%rax), %rdi
 # CHECK-NEXT:  1      1     1.00                        rcrq	%cl, %rdi
 # CHECK-NEXT:  2      4     2.00           *            rclq	%cl, (%rax)
 # CHECK-NEXT:  2      4     2.00           *            rcrq	%cl, (%rax)
+# CHECK-NEXT:  1      100   1.00                  U     rdmsr
+# CHECK-NEXT:  1      100   1.00                  U     rdpmc
+# CHECK-NEXT:  1      100   1.00                  U     rdtsc
+# CHECK-NEXT:  1      100   1.00                  U     rdtscp
 # CHECK-NEXT:  1      1     1.00                        rolb	%dil
 # CHECK-NEXT:  1      1     1.00                        rorb	%dil
 # CHECK-NEXT:  2      4     2.00    *      *            rolb	(%rax)
@@ -1557,6 +1579,7 @@ xorq (%rax), %rdi
 # CHECK-NEXT:  1      1     0.50                        testq	%rsi, %rdi
 # CHECK-NEXT:  1      4     1.00    *                   testq	%rsi, (%rax)
 # CHECK-NEXT:  1      100   1.00    *             U     ud2
+# CHECK-NEXT:  1      100   1.00                  U     wrmsr
 # CHECK-NEXT:  1      1     0.50                        xaddb	%bl, %cl
 # CHECK-NEXT:  2      4     2.00    *      *            xaddb	%bl, (%rcx)
 # CHECK-NEXT:  1      1     0.50                        xaddw	%bx, %cx
@@ -1620,7 +1643,7 @@ xorq (%rax), %rdi
 
 # CHECK:      Resource pressure per iteration:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]
-# CHECK-NEXT: 400.00  -      -     41.00   -     549.00 391.00 557.00
+# CHECK-NEXT: 400.00  -      -     49.00   -     549.50 391.50 557.00
 
 # CHECK:      Resource pressure by instruction:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    Instructions:
@@ -1844,6 +1867,7 @@ xorq (%rax), %rdi
 # CHECK-NEXT: 25.00   -      -      -      -     0.50   0.50   1.00   divl	(%rax)
 # CHECK-NEXT: 25.00   -      -      -      -     0.50   0.50    -     divq	%rcx
 # CHECK-NEXT: 25.00   -      -      -      -     0.50   0.50   1.00   divq	(%rax)
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -     enter	$7, $4095
 # CHECK-NEXT: 25.00   -      -      -      -     0.50   0.50    -     idivb	%dil
 # CHECK-NEXT: 25.00   -      -      -      -     0.50   0.50   1.00   idivb	(%rax)
 # CHECK-NEXT: 25.00   -      -      -      -     0.50   0.50    -     idivw	%si
@@ -1896,7 +1920,10 @@ xorq (%rax), %rdi
 # CHECK-NEXT:  -      -      -     1.00    -      -      -      -     insw	%dx, %es:(%rdi)
 # CHECK-NEXT:  -      -      -     1.00    -      -      -      -     insl	%dx, %es:(%rdi)
 # CHECK-NEXT:  -      -      -     1.00    -      -      -      -     int	$7
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -     invlpg	(%rax)
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -     invlpga	%rax, %ecx
 # CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -     lahf
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -     leave
 # CHECK-NEXT:  -      -      -     1.00    -      -      -      -     lodsb	(%rsi), %al
 # CHECK-NEXT:  -      -      -     1.00    -      -      -      -     lodsw	(%rsi), %ax
 # CHECK-NEXT:  -      -      -     1.00    -      -      -      -     lodsl	(%rsi), %eax
@@ -2046,6 +2073,10 @@ xorq (%rax), %rdi
 # CHECK-NEXT:  -      -      -      -      -     1.00    -      -     rcrq	%cl, %rdi
 # CHECK-NEXT:  -      -      -      -      -     1.00    -     2.00   rclq	%cl, (%rax)
 # CHECK-NEXT:  -      -      -      -      -     1.00    -     2.00   rcrq	%cl, (%rax)
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -     rdmsr
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -     rdpmc
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -     rdtsc
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -     rdtscp
 # CHECK-NEXT:  -      -      -      -      -     1.00    -      -     rolb	%dil
 # CHECK-NEXT:  -      -      -      -      -     1.00    -      -     rorb	%dil
 # CHECK-NEXT:  -      -      -      -      -     1.00    -     2.00   rolb	(%rax)
@@ -2320,6 +2351,7 @@ xorq (%rax), %rdi
 # CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -     testq	%rsi, %rdi
 # CHECK-NEXT:  -      -      -      -      -     0.50   0.50   1.00   testq	%rsi, (%rax)
 # CHECK-NEXT:  -      -      -     1.00    -      -      -      -     ud2
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -     wrmsr
 # CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -     xaddb	%bl, %cl
 # CHECK-NEXT:  -      -      -      -      -     0.50   0.50   2.00   xaddb	%bl, (%rcx)
 # CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -     xaddw	%bx, %cx
diff --git a/test/tools/llvm-mca/X86/SLM/resources-x87.s b/test/tools/llvm-mca/X86/SLM/resources-x87.s
index fe5de61296f56953a3bd2f88d5d1e858139c7238..d6d42e957bfe349eb2bf8ecc5c38548dbc0958de 100644
--- a/test/tools/llvm-mca/X86/SLM/resources-x87.s
+++ b/test/tools/llvm-mca/X86/SLM/resources-x87.s
@@ -5,7 +5,7 @@ f2xm1
 
 fabs
 
-fadd %st(0), %st(1)
+fadd %st, %st(1)
 fadd %st(2)
 fadds (%ecx)
 faddl (%ecx)
@@ -21,14 +21,14 @@ fchs
 
 fnclex
 
-fcmovb %st(1), %st(0)
-fcmovbe %st(1), %st(0)
-fcmove %st(1), %st(0)
-fcmovnb %st(1), %st(0)
-fcmovnbe %st(1), %st(0)
-fcmovne %st(1), %st(0)
-fcmovnu %st(1), %st(0)
-fcmovu %st(1), %st(0)
+fcmovb %st(1), %st
+fcmovbe %st(1), %st
+fcmove %st(1), %st
+fcmovnb %st(1), %st
+fcmovnbe %st(1), %st
+fcmovne %st(1), %st
+fcmovnu %st(1), %st
+fcmovu %st(1), %st
 
 fcom %st(1)
 fcom %st(3)
@@ -47,7 +47,7 @@ fcos
 
 fdecstp
 
-fdiv %st(0), %st(1)
+fdiv %st, %st(1)
 fdiv %st(2)
 fdivs (%ecx)
 fdivl (%eax)
@@ -56,7 +56,7 @@ fdivp %st(2)
 fidivs (%ecx)
 fidivl (%eax)
 
-fdivr %st(0), %st(1)
+fdivr %st, %st(1)
 fdivr %st(2)
 fdivrs (%ecx)
 fdivrl (%eax)
@@ -106,7 +106,7 @@ fldln2
 fldpi
 fldz
 
-fmul %st(0), %st(1)
+fmul %st, %st(1)
 fmul %st(2)
 fmuls (%ecx)
 fmull (%eax)
@@ -153,7 +153,7 @@ fnstsw (%eax)
 frstor (%eax)
 fsave (%eax)
 
-fsub %st(0), %st(1)
+fsub %st, %st(1)
 fsub %st(2)
 fsubs (%ecx)
 fsubl (%eax)
@@ -162,7 +162,7 @@ fsubp %st(2)
 fisubs (%ecx)
 fisubl (%eax)
 
-fsubr %st(0), %st(1)
+fsubr %st, %st(1)
 fsubr %st(2)
 fsubrs (%ecx)
 fsubrl (%eax)
@@ -208,26 +208,26 @@ fyl2xp1
 # CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
 # CHECK-NEXT:  1      100   1.00                  U     f2xm1
 # CHECK-NEXT:  1      1     0.50                  U     fabs
-# CHECK-NEXT:  1      3     1.00                  U     fadd	%st(0), %st(1)
-# CHECK-NEXT:  1      3     1.00                  U     fadd	%st(2)
+# CHECK-NEXT:  1      3     1.00                  U     fadd	%st, %st(1)
+# CHECK-NEXT:  1      3     1.00                  U     fadd	%st(2), %st
 # CHECK-NEXT:  1      6     1.00    *             U     fadds	(%ecx)
 # CHECK-NEXT:  1      6     1.00    *             U     faddl	(%ecx)
-# CHECK-NEXT:  1      3     1.00                  U     faddp	%st(1)
-# CHECK-NEXT:  1      3     1.00                  U     faddp	%st(2)
+# CHECK-NEXT:  1      3     1.00                  U     faddp	%st, %st(1)
+# CHECK-NEXT:  1      3     1.00                  U     faddp	%st, %st(2)
 # CHECK-NEXT:  1      6     1.00    *             U     fiadds	(%ecx)
 # CHECK-NEXT:  1      6     1.00    *             U     fiaddl	(%ecx)
 # CHECK-NEXT:  1      100   1.00                  U     fbld	(%ecx)
 # CHECK-NEXT:  1      100   1.00                  U     fbstp	(%eax)
 # CHECK-NEXT:  1      1     0.50                  U     fchs
 # CHECK-NEXT:  1      100   1.00                  U     fnclex
-# CHECK-NEXT:  1      3     1.00                  U     fcmovb	%st(1), %st(0)
-# CHECK-NEXT:  1      3     1.00                  U     fcmovbe	%st(1), %st(0)
-# CHECK-NEXT:  1      3     1.00                  U     fcmove	%st(1), %st(0)
-# CHECK-NEXT:  1      3     1.00                  U     fcmovnb	%st(1), %st(0)
-# CHECK-NEXT:  1      3     1.00                  U     fcmovnbe	%st(1), %st(0)
-# CHECK-NEXT:  1      3     1.00                  U     fcmovne	%st(1), %st(0)
-# CHECK-NEXT:  1      3     1.00                  U     fcmovnu	%st(1), %st(0)
-# CHECK-NEXT:  1      3     1.00                  U     fcmovu	%st(1), %st(0)
+# CHECK-NEXT:  1      3     1.00                  U     fcmovb	%st(1), %st
+# CHECK-NEXT:  1      3     1.00                  U     fcmovbe	%st(1), %st
+# CHECK-NEXT:  1      3     1.00                  U     fcmove	%st(1), %st
+# CHECK-NEXT:  1      3     1.00                  U     fcmovnb	%st(1), %st
+# CHECK-NEXT:  1      3     1.00                  U     fcmovnbe	%st(1), %st
+# CHECK-NEXT:  1      3     1.00                  U     fcmovne	%st(1), %st
+# CHECK-NEXT:  1      3     1.00                  U     fcmovnu	%st(1), %st
+# CHECK-NEXT:  1      3     1.00                  U     fcmovu	%st(1), %st
 # CHECK-NEXT:  1      3     1.00                  U     fcom	%st(1)
 # CHECK-NEXT:  1      3     1.00                  U     fcom	%st(3)
 # CHECK-NEXT:  1      6     1.00                  U     fcoms	(%ecx)
@@ -237,24 +237,24 @@ fyl2xp1
 # CHECK-NEXT:  1      6     1.00                  U     fcomps	(%ecx)
 # CHECK-NEXT:  1      6     1.00                  U     fcompl	(%eax)
 # CHECK-NEXT:  1      100   1.00                  U     fcompp
-# CHECK-NEXT:  1      3     1.00                  U     fcomi	%st(3)
-# CHECK-NEXT:  1      3     1.00                  U     fcompi	%st(3)
+# CHECK-NEXT:  1      3     1.00                  U     fcomi	%st(3), %st
+# CHECK-NEXT:  1      3     1.00                  U     fcompi	%st(3), %st
 # CHECK-NEXT:  1      100   1.00                  U     fcos
 # CHECK-NEXT:  1      100   1.00                  U     fdecstp
-# CHECK-NEXT:  1      19    17.00                 U     fdiv	%st(0), %st(1)
-# CHECK-NEXT:  1      19    17.00                 U     fdiv	%st(2)
+# CHECK-NEXT:  1      19    17.00                 U     fdiv	%st, %st(1)
+# CHECK-NEXT:  1      19    17.00                 U     fdiv	%st(2), %st
 # CHECK-NEXT:  1      22    17.00   *             U     fdivs	(%ecx)
 # CHECK-NEXT:  1      22    17.00   *             U     fdivl	(%eax)
-# CHECK-NEXT:  1      19    17.00                 U     fdivp	%st(1)
-# CHECK-NEXT:  1      19    17.00                 U     fdivp	%st(2)
+# CHECK-NEXT:  1      19    17.00                 U     fdivp	%st, %st(1)
+# CHECK-NEXT:  1      19    17.00                 U     fdivp	%st, %st(2)
 # CHECK-NEXT:  1      22    17.00   *             U     fidivs	(%ecx)
 # CHECK-NEXT:  1      22    17.00   *             U     fidivl	(%eax)
-# CHECK-NEXT:  1      19    17.00                 U     fdivr	%st(0), %st(1)
-# CHECK-NEXT:  1      19    17.00                 U     fdivr	%st(2)
+# CHECK-NEXT:  1      19    17.00                 U     fdivr	%st, %st(1)
+# CHECK-NEXT:  1      19    17.00                 U     fdivr	%st(2), %st
 # CHECK-NEXT:  1      22    17.00   *             U     fdivrs	(%ecx)
 # CHECK-NEXT:  1      22    17.00   *             U     fdivrl	(%eax)
-# CHECK-NEXT:  1      19    17.00                 U     fdivrp	%st(1)
-# CHECK-NEXT:  1      19    17.00                 U     fdivrp	%st(2)
+# CHECK-NEXT:  1      19    17.00                 U     fdivrp	%st, %st(1)
+# CHECK-NEXT:  1      19    17.00                 U     fdivrp	%st, %st(2)
 # CHECK-NEXT:  1      22    17.00   *             U     fidivrs	(%ecx)
 # CHECK-NEXT:  1      22    17.00   *             U     fidivrl	(%eax)
 # CHECK-NEXT:  1      100   1.00                  U     ffree	%st(0)
@@ -288,12 +288,12 @@ fyl2xp1
 # CHECK-NEXT:  2      1     1.00                  U     fldln2
 # CHECK-NEXT:  2      1     1.00                  U     fldpi
 # CHECK-NEXT:  1      1     0.50                  U     fldz
-# CHECK-NEXT:  1      5     2.00                  U     fmul	%st(0), %st(1)
-# CHECK-NEXT:  1      5     2.00                  U     fmul	%st(2)
+# CHECK-NEXT:  1      5     2.00                  U     fmul	%st, %st(1)
+# CHECK-NEXT:  1      5     2.00                  U     fmul	%st(2), %st
 # CHECK-NEXT:  1      8     2.00    *             U     fmuls	(%ecx)
 # CHECK-NEXT:  1      8     2.00    *             U     fmull	(%eax)
-# CHECK-NEXT:  1      5     2.00                  U     fmulp	%st(1)
-# CHECK-NEXT:  1      5     2.00                  U     fmulp	%st(2)
+# CHECK-NEXT:  1      5     2.00                  U     fmulp	%st, %st(1)
+# CHECK-NEXT:  1      5     2.00                  U     fmulp	%st, %st(2)
 # CHECK-NEXT:  1      8     2.00    *             U     fimuls	(%ecx)
 # CHECK-NEXT:  1      8     2.00    *             U     fimull	(%eax)
 # CHECK-NEXT:  1      1     0.50                  U     fnop
@@ -321,20 +321,20 @@ fyl2xp1
 # CHECK-NEXT:  1      100   1.00                  U     frstor	(%eax)
 # CHECK-NEXT:  1      100   1.00                  U     wait
 # CHECK-NEXT:  1      100   1.00                  U     fnsave	(%eax)
-# CHECK-NEXT:  1      3     1.00                  U     fsub	%st(0), %st(1)
-# CHECK-NEXT:  1      3     1.00                  U     fsub	%st(2)
+# CHECK-NEXT:  1      3     1.00                  U     fsub	%st, %st(1)
+# CHECK-NEXT:  1      3     1.00                  U     fsub	%st(2), %st
 # CHECK-NEXT:  1      6     1.00    *             U     fsubs	(%ecx)
 # CHECK-NEXT:  1      6     1.00    *             U     fsubl	(%eax)
-# CHECK-NEXT:  1      3     1.00                  U     fsubp	%st(1)
-# CHECK-NEXT:  1      3     1.00                  U     fsubp	%st(2)
+# CHECK-NEXT:  1      3     1.00                  U     fsubp	%st, %st(1)
+# CHECK-NEXT:  1      3     1.00                  U     fsubp	%st, %st(2)
 # CHECK-NEXT:  1      6     1.00    *             U     fisubs	(%ecx)
 # CHECK-NEXT:  1      6     1.00    *             U     fisubl	(%eax)
-# CHECK-NEXT:  1      3     1.00                  U     fsubr	%st(0), %st(1)
-# CHECK-NEXT:  1      3     1.00                  U     fsubr	%st(2)
+# CHECK-NEXT:  1      3     1.00                  U     fsubr	%st, %st(1)
+# CHECK-NEXT:  1      3     1.00                  U     fsubr	%st(2), %st
 # CHECK-NEXT:  1      6     1.00    *             U     fsubrs	(%ecx)
 # CHECK-NEXT:  1      6     1.00    *             U     fsubrl	(%eax)
-# CHECK-NEXT:  1      3     1.00                  U     fsubrp	%st(1)
-# CHECK-NEXT:  1      3     1.00                  U     fsubrp	%st(2)
+# CHECK-NEXT:  1      3     1.00                  U     fsubrp	%st, %st(1)
+# CHECK-NEXT:  1      3     1.00                  U     fsubrp	%st, %st(2)
 # CHECK-NEXT:  1      6     1.00    *             U     fisubrs	(%ecx)
 # CHECK-NEXT:  1      6     1.00    *             U     fisubrl	(%eax)
 # CHECK-NEXT:  1      3     1.00                  U     ftst
@@ -343,8 +343,8 @@ fyl2xp1
 # CHECK-NEXT:  1      3     1.00                  U     fucomp	%st(1)
 # CHECK-NEXT:  1      3     1.00                  U     fucomp	%st(3)
 # CHECK-NEXT:  1      3     1.00                  U     fucompp
-# CHECK-NEXT:  1      3     1.00                  U     fucomi	%st(3)
-# CHECK-NEXT:  1      3     1.00                  U     fucompi	%st(3)
+# CHECK-NEXT:  1      3     1.00                  U     fucomi	%st(3), %st
+# CHECK-NEXT:  1      3     1.00                  U     fucompi	%st(3), %st
 # CHECK-NEXT:  1      100   1.00                  U     wait
 # CHECK-NEXT:  1      100   1.00                  U     fxam
 # CHECK-NEXT:  1      1     0.50                  U     fxch	%st(1)
@@ -373,26 +373,26 @@ fyl2xp1
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    Instructions:
 # CHECK-NEXT:  -      -      -     1.00    -      -      -      -     f2xm1
 # CHECK-NEXT:  -      -      -     0.50   0.50    -      -      -     fabs
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -     fadd	%st(0), %st(1)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -     fadd	%st(2)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -     fadd	%st, %st(1)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -     fadd	%st(2), %st
 # CHECK-NEXT:  -      -      -      -     1.00    -      -     1.00   fadds	(%ecx)
 # CHECK-NEXT:  -      -      -      -     1.00    -      -     1.00   faddl	(%ecx)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -     faddp	%st(1)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -     faddp	%st(2)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -     faddp	%st, %st(1)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -     faddp	%st, %st(2)
 # CHECK-NEXT:  -      -      -      -     1.00    -      -     1.00   fiadds	(%ecx)
 # CHECK-NEXT:  -      -      -      -     1.00    -      -     1.00   fiaddl	(%ecx)
 # CHECK-NEXT:  -      -      -     1.00    -      -      -      -     fbld	(%ecx)
 # CHECK-NEXT:  -      -      -     1.00    -      -      -      -     fbstp	(%eax)
 # CHECK-NEXT:  -      -      -     0.50   0.50    -      -      -     fchs
 # CHECK-NEXT:  -      -      -     1.00    -      -      -      -     fnclex
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -     fcmovb	%st(1), %st(0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -     fcmovbe	%st(1), %st(0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -     fcmove	%st(1), %st(0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -     fcmovnb	%st(1), %st(0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -     fcmovnbe	%st(1), %st(0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -     fcmovne	%st(1), %st(0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -     fcmovnu	%st(1), %st(0)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -     fcmovu	%st(1), %st(0)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -     fcmovb	%st(1), %st
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -     fcmovbe	%st(1), %st
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -     fcmove	%st(1), %st
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -     fcmovnb	%st(1), %st
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -     fcmovnbe	%st(1), %st
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -     fcmovne	%st(1), %st
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -     fcmovnu	%st(1), %st
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -     fcmovu	%st(1), %st
 # CHECK-NEXT:  -      -      -      -     1.00    -      -      -     fcom	%st(1)
 # CHECK-NEXT:  -      -      -      -     1.00    -      -      -     fcom	%st(3)
 # CHECK-NEXT:  -      -      -      -     1.00    -      -     1.00   fcoms	(%ecx)
@@ -402,24 +402,24 @@ fyl2xp1
 # CHECK-NEXT:  -      -      -      -     1.00    -      -     1.00   fcomps	(%ecx)
 # CHECK-NEXT:  -      -      -      -     1.00    -      -     1.00   fcompl	(%eax)
 # CHECK-NEXT:  -      -      -     1.00    -      -      -      -     fcompp
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -     fcomi	%st(3)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -     fcompi	%st(3)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -     fcomi	%st(3), %st
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -     fcompi	%st(3), %st
 # CHECK-NEXT:  -      -      -     1.00    -      -      -      -     fcos
 # CHECK-NEXT:  -      -      -     1.00    -      -      -      -     fdecstp
-# CHECK-NEXT:  -     17.00   -     1.00    -      -      -      -     fdiv	%st(0), %st(1)
-# CHECK-NEXT:  -     17.00   -     1.00    -      -      -      -     fdiv	%st(2)
+# CHECK-NEXT:  -     17.00   -     1.00    -      -      -      -     fdiv	%st, %st(1)
+# CHECK-NEXT:  -     17.00   -     1.00    -      -      -      -     fdiv	%st(2), %st
 # CHECK-NEXT:  -     17.00   -     1.00    -      -      -     1.00   fdivs	(%ecx)
 # CHECK-NEXT:  -     17.00   -     1.00    -      -      -     1.00   fdivl	(%eax)
-# CHECK-NEXT:  -     17.00   -     1.00    -      -      -      -     fdivp	%st(1)
-# CHECK-NEXT:  -     17.00   -     1.00    -      -      -      -     fdivp	%st(2)
+# CHECK-NEXT:  -     17.00   -     1.00    -      -      -      -     fdivp	%st, %st(1)
+# CHECK-NEXT:  -     17.00   -     1.00    -      -      -      -     fdivp	%st, %st(2)
 # CHECK-NEXT:  -     17.00   -     1.00    -      -      -     1.00   fidivs	(%ecx)
 # CHECK-NEXT:  -     17.00   -     1.00    -      -      -     1.00   fidivl	(%eax)
-# CHECK-NEXT:  -     17.00   -     1.00    -      -      -      -     fdivr	%st(0), %st(1)
-# CHECK-NEXT:  -     17.00   -     1.00    -      -      -      -     fdivr	%st(2)
+# CHECK-NEXT:  -     17.00   -     1.00    -      -      -      -     fdivr	%st, %st(1)
+# CHECK-NEXT:  -     17.00   -     1.00    -      -      -      -     fdivr	%st(2), %st
 # CHECK-NEXT:  -     17.00   -     1.00    -      -      -     1.00   fdivrs	(%ecx)
 # CHECK-NEXT:  -     17.00   -     1.00    -      -      -     1.00   fdivrl	(%eax)
-# CHECK-NEXT:  -     17.00   -     1.00    -      -      -      -     fdivrp	%st(1)
-# CHECK-NEXT:  -     17.00   -     1.00    -      -      -      -     fdivrp	%st(2)
+# CHECK-NEXT:  -     17.00   -     1.00    -      -      -      -     fdivrp	%st, %st(1)
+# CHECK-NEXT:  -     17.00   -     1.00    -      -      -      -     fdivrp	%st, %st(2)
 # CHECK-NEXT:  -     17.00   -     1.00    -      -      -     1.00   fidivrs	(%ecx)
 # CHECK-NEXT:  -     17.00   -     1.00    -      -      -     1.00   fidivrl	(%eax)
 # CHECK-NEXT:  -      -      -     1.00    -      -      -      -     ffree	%st(0)
@@ -453,12 +453,12 @@ fyl2xp1
 # CHECK-NEXT:  -      -      -     1.00   1.00    -      -      -     fldln2
 # CHECK-NEXT:  -      -      -     1.00   1.00    -      -      -     fldpi
 # CHECK-NEXT:  -      -      -     0.50   0.50    -      -      -     fldz
-# CHECK-NEXT:  -      -     2.00   1.00    -      -      -      -     fmul	%st(0), %st(1)
-# CHECK-NEXT:  -      -     2.00   1.00    -      -      -      -     fmul	%st(2)
+# CHECK-NEXT:  -      -     2.00   1.00    -      -      -      -     fmul	%st, %st(1)
+# CHECK-NEXT:  -      -     2.00   1.00    -      -      -      -     fmul	%st(2), %st
 # CHECK-NEXT:  -      -     2.00   1.00    -      -      -     1.00   fmuls	(%ecx)
 # CHECK-NEXT:  -      -     2.00   1.00    -      -      -     1.00   fmull	(%eax)
-# CHECK-NEXT:  -      -     2.00   1.00    -      -      -      -     fmulp	%st(1)
-# CHECK-NEXT:  -      -     2.00   1.00    -      -      -      -     fmulp	%st(2)
+# CHECK-NEXT:  -      -     2.00   1.00    -      -      -      -     fmulp	%st, %st(1)
+# CHECK-NEXT:  -      -     2.00   1.00    -      -      -      -     fmulp	%st, %st(2)
 # CHECK-NEXT:  -      -     2.00   1.00    -      -      -     1.00   fimuls	(%ecx)
 # CHECK-NEXT:  -      -     2.00   1.00    -      -      -     1.00   fimull	(%eax)
 # CHECK-NEXT:  -      -      -      -      -      -      -      -     fnop
@@ -486,20 +486,20 @@ fyl2xp1
 # CHECK-NEXT:  -      -      -     1.00    -      -      -      -     frstor	(%eax)
 # CHECK-NEXT:  -      -      -     1.00    -      -      -      -     wait
 # CHECK-NEXT:  -      -      -     1.00    -      -      -      -     fnsave	(%eax)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -     fsub	%st(0), %st(1)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -     fsub	%st(2)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -     fsub	%st, %st(1)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -     fsub	%st(2), %st
 # CHECK-NEXT:  -      -      -      -     1.00    -      -     1.00   fsubs	(%ecx)
 # CHECK-NEXT:  -      -      -      -     1.00    -      -     1.00   fsubl	(%eax)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -     fsubp	%st(1)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -     fsubp	%st(2)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -     fsubp	%st, %st(1)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -     fsubp	%st, %st(2)
 # CHECK-NEXT:  -      -      -      -     1.00    -      -     1.00   fisubs	(%ecx)
 # CHECK-NEXT:  -      -      -      -     1.00    -      -     1.00   fisubl	(%eax)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -     fsubr	%st(0), %st(1)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -     fsubr	%st(2)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -     fsubr	%st, %st(1)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -     fsubr	%st(2), %st
 # CHECK-NEXT:  -      -      -      -     1.00    -      -     1.00   fsubrs	(%ecx)
 # CHECK-NEXT:  -      -      -      -     1.00    -      -     1.00   fsubrl	(%eax)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -     fsubrp	%st(1)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -     fsubrp	%st(2)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -     fsubrp	%st, %st(1)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -     fsubrp	%st, %st(2)
 # CHECK-NEXT:  -      -      -      -     1.00    -      -     1.00   fisubrs	(%ecx)
 # CHECK-NEXT:  -      -      -      -     1.00    -      -     1.00   fisubrl	(%eax)
 # CHECK-NEXT:  -      -      -      -     1.00    -      -      -     ftst
@@ -508,8 +508,8 @@ fyl2xp1
 # CHECK-NEXT:  -      -      -      -     1.00    -      -      -     fucomp	%st(1)
 # CHECK-NEXT:  -      -      -      -     1.00    -      -      -     fucomp	%st(3)
 # CHECK-NEXT:  -      -      -      -     1.00    -      -      -     fucompp
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -     fucomi	%st(3)
-# CHECK-NEXT:  -      -      -      -     1.00    -      -      -     fucompi	%st(3)
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -     fucomi	%st(3), %st
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -     fucompi	%st(3), %st
 # CHECK-NEXT:  -      -      -     1.00    -      -      -      -     wait
 # CHECK-NEXT:  -      -      -     1.00    -      -      -      -     fxam
 # CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -     fxch	%st(1)
diff --git a/test/tools/llvm-mca/X86/SandyBridge/resources-sse2.s b/test/tools/llvm-mca/X86/SandyBridge/resources-sse2.s
index d0e9437f5686f41d4c2aaa4e2dd925000c0e35e6..781ed65cd3d361d77d9bbf927820435ced0e1d61 100644
--- a/test/tools/llvm-mca/X86/SandyBridge/resources-sse2.s
+++ b/test/tools/llvm-mca/X86/SandyBridge/resources-sse2.s
@@ -94,6 +94,8 @@ maxpd       (%rax), %xmm2
 maxsd       %xmm0, %xmm2
 maxsd       (%rax), %xmm2
 
+mfence
+
 minpd       %xmm0, %xmm2
 minpd       (%rax), %xmm2
 
@@ -229,6 +231,9 @@ pcmpgtw     (%rax), %xmm2
 
 pextrw      $1, %xmm0, %rcx
 
+pinsrw      $1, %rax, %xmm0
+pinsrw      $1, (%rax), %xmm0
+
 pmaddwd     %xmm0, %xmm2
 pmaddwd     (%rax), %xmm2
 
@@ -465,6 +470,7 @@ xorpd       (%rax), %xmm2
 # CHECK-NEXT:  2      9     1.00    *                   maxpd	(%rax), %xmm2
 # CHECK-NEXT:  1      3     1.00                        maxsd	%xmm0, %xmm2
 # CHECK-NEXT:  2      9     1.00    *                   maxsd	(%rax), %xmm2
+# CHECK-NEXT:  1      1     1.00    *      *      U     mfence
 # CHECK-NEXT:  1      3     1.00                        minpd	%xmm0, %xmm2
 # CHECK-NEXT:  2      9     1.00    *                   minpd	(%rax), %xmm2
 # CHECK-NEXT:  1      3     1.00                        minsd	%xmm0, %xmm2
@@ -555,6 +561,8 @@ xorpd       (%rax), %xmm2
 # CHECK-NEXT:  1      1     0.50                        pcmpgtw	%xmm0, %xmm2
 # CHECK-NEXT:  2      7     0.50    *                   pcmpgtw	(%rax), %xmm2
 # CHECK-NEXT:  2      3     1.00                        pextrw	$1, %xmm0, %ecx
+# CHECK-NEXT:  2      2     1.00                        pinsrw	$1, %eax, %xmm0
+# CHECK-NEXT:  2      7     0.50    *                   pinsrw	$1, (%rax), %xmm0
 # CHECK-NEXT:  1      5     1.00                        pmaddwd	%xmm0, %xmm2
 # CHECK-NEXT:  2      11    1.00    *                   pmaddwd	(%rax), %xmm2
 # CHECK-NEXT:  1      1     0.50                        pmaxsw	%xmm0, %xmm2
@@ -679,7 +687,7 @@ xorpd       (%rax), %xmm2
 
 # CHECK:      Resource pressure per iteration:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6.0]  [6.1]
-# CHECK-NEXT:  -     172.00 75.83  117.33 16.00  98.83  66.00  66.00
+# CHECK-NEXT:  -     172.00 75.83  118.33 17.00  100.83 67.00  67.00
 
 # CHECK:      Resource pressure by instruction:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6.0]  [6.1]  Instructions:
@@ -746,6 +754,7 @@ xorpd       (%rax), %xmm2
 # CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   maxpd	(%rax), %xmm2
 # CHECK-NEXT:  -      -      -     1.00    -      -      -      -     maxsd	%xmm0, %xmm2
 # CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   maxsd	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -     1.00    -     0.50   0.50   mfence
 # CHECK-NEXT:  -      -      -     1.00    -      -      -      -     minpd	%xmm0, %xmm2
 # CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   minpd	(%rax), %xmm2
 # CHECK-NEXT:  -      -      -     1.00    -      -      -      -     minsd	%xmm0, %xmm2
@@ -836,6 +845,8 @@ xorpd       (%rax), %xmm2
 # CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     pcmpgtw	%xmm0, %xmm2
 # CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   pcmpgtw	(%rax), %xmm2
 # CHECK-NEXT:  -      -     1.00   0.50    -     0.50    -      -     pextrw	$1, %xmm0, %ecx
+# CHECK-NEXT:  -      -      -     0.50    -     1.50    -      -     pinsrw	$1, %eax, %xmm0
+# CHECK-NEXT:  -      -      -     0.50    -     0.50   0.50   0.50   pinsrw	$1, (%rax), %xmm0
 # CHECK-NEXT:  -      -     1.00    -      -      -      -      -     pmaddwd	%xmm0, %xmm2
 # CHECK-NEXT:  -      -     1.00    -      -      -     0.50   0.50   pmaddwd	(%rax), %xmm2
 # CHECK-NEXT:  -      -      -     0.50    -     0.50    -      -     pmaxsw	%xmm0, %xmm2
diff --git a/test/tools/llvm-mca/X86/SandyBridge/resources-sse3.s b/test/tools/llvm-mca/X86/SandyBridge/resources-sse3.s
index 70e3f5b31f44c637ae33105eb742ebb314721fd1..1f1f58674fe3eee98c36cdd1e525a87333d7843c 100644
--- a/test/tools/llvm-mca/X86/SandyBridge/resources-sse3.s
+++ b/test/tools/llvm-mca/X86/SandyBridge/resources-sse3.s
@@ -21,6 +21,8 @@ hsubps    (%rax), %xmm2
 
 lddqu     (%rax), %xmm2
 
+monitor
+
 movddup   %xmm0, %xmm2
 movddup   (%rax), %xmm2
 
@@ -30,6 +32,8 @@ movshdup  (%rax), %xmm2
 movsldup  %xmm0, %xmm2
 movsldup  (%rax), %xmm2
 
+mwait
+
 # CHECK:      Instruction Info:
 # CHECK-NEXT: [1]: #uOps
 # CHECK-NEXT: [2]: Latency
@@ -52,12 +56,14 @@ movsldup  (%rax), %xmm2
 # CHECK-NEXT:  3      5     2.00                        hsubps	%xmm0, %xmm2
 # CHECK-NEXT:  4      11    2.00    *                   hsubps	(%rax), %xmm2
 # CHECK-NEXT:  1      6     0.50    *                   lddqu	(%rax), %xmm2
+# CHECK-NEXT:  1      100   0.33                  U     monitor
 # CHECK-NEXT:  1      1     1.00                        movddup	%xmm0, %xmm2
 # CHECK-NEXT:  1      6     0.50    *                   movddup	(%rax), %xmm2
 # CHECK-NEXT:  1      1     1.00                        movshdup	%xmm0, %xmm2
 # CHECK-NEXT:  1      6     0.50    *                   movshdup	(%rax), %xmm2
 # CHECK-NEXT:  1      1     1.00                        movsldup	%xmm0, %xmm2
 # CHECK-NEXT:  1      6     0.50    *                   movsldup	(%rax), %xmm2
+# CHECK-NEXT:  1      100   0.33    *      *      U     mwait
 
 # CHECK:      Resources:
 # CHECK-NEXT: [0]   - SBDivider
@@ -71,7 +77,7 @@ movsldup  (%rax), %xmm2
 
 # CHECK:      Resource pressure per iteration:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6.0]  [6.1]
-# CHECK-NEXT:  -      -      -     12.00   -     19.00  5.00   5.00
+# CHECK-NEXT:  -      -     0.67   12.67   -     19.67  5.00   5.00
 
 # CHECK:      Resource pressure by instruction:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6.0]  [6.1]  Instructions:
@@ -88,9 +94,11 @@ movsldup  (%rax), %xmm2
 # CHECK-NEXT:  -      -      -     1.00    -     2.00    -      -     hsubps	%xmm0, %xmm2
 # CHECK-NEXT:  -      -      -     1.00    -     2.00   0.50   0.50   hsubps	(%rax), %xmm2
 # CHECK-NEXT:  -      -      -      -      -      -     0.50   0.50   lddqu	(%rax), %xmm2
+# CHECK-NEXT:  -      -     0.33   0.33    -     0.33    -      -     monitor
 # CHECK-NEXT:  -      -      -      -      -     1.00    -      -     movddup	%xmm0, %xmm2
 # CHECK-NEXT:  -      -      -      -      -      -     0.50   0.50   movddup	(%rax), %xmm2
 # CHECK-NEXT:  -      -      -      -      -     1.00    -      -     movshdup	%xmm0, %xmm2
 # CHECK-NEXT:  -      -      -      -      -      -     0.50   0.50   movshdup	(%rax), %xmm2
 # CHECK-NEXT:  -      -      -      -      -     1.00    -      -     movsldup	%xmm0, %xmm2
 # CHECK-NEXT:  -      -      -      -      -      -     0.50   0.50   movsldup	(%rax), %xmm2
+# CHECK-NEXT:  -      -     0.33   0.33    -     0.33    -      -     mwait
diff --git a/test/tools/llvm-mca/X86/SandyBridge/resources-x86_64.s b/test/tools/llvm-mca/X86/SandyBridge/resources-x86_64.s
index f255cbd5c851c5b96e197a945754fcc9a933b8a1..861bfaca0f27b4be2d1ac012f181ecb9b8cadb35 100644
--- a/test/tools/llvm-mca/X86/SandyBridge/resources-x86_64.s
+++ b/test/tools/llvm-mca/X86/SandyBridge/resources-x86_64.s
@@ -254,6 +254,8 @@ divl (%rax)
 divq %rcx
 divq (%rax)
 
+enter $7, $4095
+
 idivb %dil
 idivb (%rax)
 idivw %si
@@ -315,8 +317,13 @@ insl
 
 int $7
 
+invlpg  (%rax)
+invlpga %rax, %ecx
+
 lahf
 
+leave
+
 lodsb
 lodsw
 lodsl
@@ -486,6 +493,11 @@ rcrq %cl, %rdi
 rclq %cl, (%rax)
 rcrq %cl, (%rax)
 
+rdmsr
+rdpmc
+rdtsc
+rdtscp
+
 rolb %dil
 rorb %dil
 rolb (%rax)
@@ -789,6 +801,8 @@ testq %rsi, (%rax)
 
 ud2
 
+wrmsr
+
 xaddb %bl, %cl
 xaddb %bl, (%rcx)
 
@@ -852,8 +866,6 @@ xorq %rsi, %rdi
 xorq %rsi, (%rax)
 xorq (%rax), %rdi
 
-# CCHECK-NEXT: 160.00  -     439.83 221.33 250.00 431.83 294.00 294.00
-
 # CHECK:      Instruction Info:
 # CHECK-NEXT: [1]: #uOps
 # CHECK-NEXT: [2]: Latency
@@ -1083,6 +1095,7 @@ xorq (%rax), %rdi
 # CHECK-NEXT:  2      30    10.00   *             U     divl	(%rax)
 # CHECK-NEXT:  1      25    10.00                 U     divq	%rcx
 # CHECK-NEXT:  2      30    10.00   *             U     divq	(%rax)
+# CHECK-NEXT:  1      100   0.33                  U     enter	$7, $4095
 # CHECK-NEXT:  1      25    10.00                 U     idivb	%dil
 # CHECK-NEXT:  2      30    10.00   *             U     idivb	(%rax)
 # CHECK-NEXT:  1      25    10.00                 U     idivw	%si
@@ -1135,7 +1148,10 @@ xorq (%rax), %rdi
 # CHECK-NEXT:  1      100   0.33                  U     insw	%dx, %es:(%rdi)
 # CHECK-NEXT:  1      100   0.33                  U     insl	%dx, %es:(%rdi)
 # CHECK-NEXT:  1      100   0.33    *      *      U     int	$7
+# CHECK-NEXT:  1      100   0.33                  U     invlpg	(%rax)
+# CHECK-NEXT:  1      100   0.33                  U     invlpga	%rax, %ecx
 # CHECK-NEXT:  1      1     0.50                        lahf
+# CHECK-NEXT:  3      7     0.67    *                   leave
 # CHECK-NEXT:  3      7     0.67                  U     lodsb	(%rsi), %al
 # CHECK-NEXT:  3      7     0.67                  U     lodsw	(%rsi), %ax
 # CHECK-NEXT:  2      6     0.50                  U     lodsl	(%rsi), %eax
@@ -1285,6 +1301,10 @@ xorq (%rax), %rdi
 # CHECK-NEXT:  8      5     4.00                        rcrq	%cl, %rdi
 # CHECK-NEXT:  11     11    3.50           *            rclq	%cl, (%rax)
 # CHECK-NEXT:  11     11    3.50           *            rcrq	%cl, (%rax)
+# CHECK-NEXT:  1      100   0.33                  U     rdmsr
+# CHECK-NEXT:  1      100   0.33                  U     rdpmc
+# CHECK-NEXT:  1      100   0.33                  U     rdtsc
+# CHECK-NEXT:  1      100   0.33                  U     rdtscp
 # CHECK-NEXT:  2      2     1.00                        rolb	%dil
 # CHECK-NEXT:  2      2     1.00                        rorb	%dil
 # CHECK-NEXT:  5      8     1.00    *      *            rolb	(%rax)
@@ -1559,6 +1579,7 @@ xorq (%rax), %rdi
 # CHECK-NEXT:  1      1     0.33                        testq	%rsi, %rdi
 # CHECK-NEXT:  2      6     0.50    *                   testq	%rsi, (%rax)
 # CHECK-NEXT:  1      100   0.33    *             U     ud2
+# CHECK-NEXT:  1      100   0.33                  U     wrmsr
 # CHECK-NEXT:  3      2     1.00                        xaddb	%bl, %cl
 # CHECK-NEXT:  5      8     1.00    *      *            xaddb	%bl, (%rcx)
 # CHECK-NEXT:  3      2     1.00                        xaddw	%bx, %cx
@@ -1622,7 +1643,7 @@ xorq (%rax), %rdi
 
 # CHECK:      Resource pressure per iteration:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6.0]  [6.1]
-# CHECK-NEXT: 160.00  -     571.83 221.33 222.00 571.83 316.00 316.00
+# CHECK-NEXT: 160.00  -     575.17 224.67 222.00 575.17 316.50 316.50
 
 # CHECK:      Resource pressure by instruction:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6.0]  [6.1]  Instructions:
@@ -1846,6 +1867,7 @@ xorq (%rax), %rdi
 # CHECK-NEXT: 10.00   -     1.00    -      -      -     0.50   0.50   divl	(%rax)
 # CHECK-NEXT: 10.00   -     1.00    -      -      -      -      -     divq	%rcx
 # CHECK-NEXT: 10.00   -     1.00    -      -      -     0.50   0.50   divq	(%rax)
+# CHECK-NEXT:  -      -     0.33   0.33    -     0.33    -      -     enter	$7, $4095
 # CHECK-NEXT: 10.00   -     1.00    -      -      -      -      -     idivb	%dil
 # CHECK-NEXT: 10.00   -     1.00    -      -      -     0.50   0.50   idivb	(%rax)
 # CHECK-NEXT: 10.00   -     1.00    -      -      -      -      -     idivw	%si
@@ -1898,7 +1920,10 @@ xorq (%rax), %rdi
 # CHECK-NEXT:  -      -     0.33   0.33    -     0.33    -      -     insw	%dx, %es:(%rdi)
 # CHECK-NEXT:  -      -     0.33   0.33    -     0.33    -      -     insl	%dx, %es:(%rdi)
 # CHECK-NEXT:  -      -     0.33   0.33    -     0.33    -      -     int	$7
+# CHECK-NEXT:  -      -     0.33   0.33    -     0.33    -      -     invlpg	(%rax)
+# CHECK-NEXT:  -      -     0.33   0.33    -     0.33    -      -     invlpga	%rax, %ecx
 # CHECK-NEXT:  -      -     0.50    -      -     0.50    -      -     lahf
+# CHECK-NEXT:  -      -     0.67   0.67    -     0.67   0.50   0.50   leave
 # CHECK-NEXT:  -      -     0.67   0.67    -     0.67   0.50   0.50   lodsb	(%rsi), %al
 # CHECK-NEXT:  -      -     0.67   0.67    -     0.67   0.50   0.50   lodsw	(%rsi), %ax
 # CHECK-NEXT:  -      -     0.33   0.33    -     0.33   0.50   0.50   lodsl	(%rsi), %eax
@@ -2048,6 +2073,10 @@ xorq (%rax), %rdi
 # CHECK-NEXT:  -      -     4.00    -      -     4.00    -      -     rcrq	%cl, %rdi
 # CHECK-NEXT:  -      -     3.50    -      -     3.50   2.00   2.00   rclq	%cl, (%rax)
 # CHECK-NEXT:  -      -     3.50    -      -     3.50   2.00   2.00   rcrq	%cl, (%rax)
+# CHECK-NEXT:  -      -     0.33   0.33    -     0.33    -      -     rdmsr
+# CHECK-NEXT:  -      -     0.33   0.33    -     0.33    -      -     rdpmc
+# CHECK-NEXT:  -      -     0.33   0.33    -     0.33    -      -     rdtsc
+# CHECK-NEXT:  -      -     0.33   0.33    -     0.33    -      -     rdtscp
 # CHECK-NEXT:  -      -     1.00    -      -     1.00    -      -     rolb	%dil
 # CHECK-NEXT:  -      -     1.00    -      -     1.00    -      -     rorb	%dil
 # CHECK-NEXT:  -      -     1.00    -     1.00   1.00   1.00   1.00   rolb	(%rax)
@@ -2322,6 +2351,7 @@ xorq (%rax), %rdi
 # CHECK-NEXT:  -      -     0.33   0.33    -     0.33    -      -     testq	%rsi, %rdi
 # CHECK-NEXT:  -      -     0.33   0.33    -     0.33   0.50   0.50   testq	%rsi, (%rax)
 # CHECK-NEXT:  -      -     0.33   0.33    -     0.33    -      -     ud2
+# CHECK-NEXT:  -      -     0.33   0.33    -     0.33    -      -     wrmsr
 # CHECK-NEXT:  -      -     1.00   1.00    -     1.00    -      -     xaddb	%bl, %cl
 # CHECK-NEXT:  -      -     0.67   0.67   1.00   0.67   1.00   1.00   xaddb	%bl, (%rcx)
 # CHECK-NEXT:  -      -     1.00   1.00    -     1.00    -      -     xaddw	%bx, %cx
diff --git a/test/tools/llvm-mca/X86/SandyBridge/resources-x87.s b/test/tools/llvm-mca/X86/SandyBridge/resources-x87.s
index 332f365f1fdd3a971dbedfb0be0e2c715d3db1b9..1bed53326ced6863aa7ef332a9b96c70b1ea4742 100644
--- a/test/tools/llvm-mca/X86/SandyBridge/resources-x87.s
+++ b/test/tools/llvm-mca/X86/SandyBridge/resources-x87.s
@@ -5,7 +5,7 @@ f2xm1
 
 fabs
 
-fadd %st(0), %st(1)
+fadd %st, %st(1)
 fadd %st(2)
 fadds (%ecx)
 faddl (%ecx)
@@ -21,14 +21,14 @@ fchs
 
 fnclex
 
-fcmovb %st(1), %st(0)
-fcmovbe %st(1), %st(0)
-fcmove %st(1), %st(0)
-fcmovnb %st(1), %st(0)
-fcmovnbe %st(1), %st(0)
-fcmovne %st(1), %st(0)
-fcmovnu %st(1), %st(0)
-fcmovu %st(1), %st(0)
+fcmovb %st(1), %st
+fcmovbe %st(1), %st
+fcmove %st(1), %st
+fcmovnb %st(1), %st
+fcmovnbe %st(1), %st
+fcmovne %st(1), %st
+fcmovnu %st(1), %st
+fcmovu %st(1), %st
 
 fcom %st(1)
 fcom %st(3)
@@ -47,7 +47,7 @@ fcos
 
 fdecstp
 
-fdiv %st(0), %st(1)
+fdiv %st, %st(1)
 fdiv %st(2)
 fdivs (%ecx)
 fdivl (%eax)
@@ -56,7 +56,7 @@ fdivp %st(2)
 fidivs (%ecx)
 fidivl (%eax)
 
-fdivr %st(0), %st(1)
+fdivr %st, %st(1)
 fdivr %st(2)
 fdivrs (%ecx)
 fdivrl (%eax)
@@ -106,7 +106,7 @@ fldln2
 fldpi
 fldz
 
-fmul %st(0), %st(1)
+fmul %st, %st(1)
 fmul %st(2)
 fmuls (%ecx)
 fmull (%eax)
@@ -153,7 +153,7 @@ fnstsw (%eax)
 frstor (%eax)
 fsave (%eax)
 
-fsub %st(0), %st(1)
+fsub %st, %st(1)
 fsub %st(2)
 fsubs (%ecx)
 fsubl (%eax)
@@ -162,7 +162,7 @@ fsubp %st(2)
 fisubs (%ecx)
 fisubl (%eax)
 
-fsubr %st(0), %st(1)
+fsubr %st, %st(1)
 fsubr %st(2)
 fsubrs (%ecx)
 fsubrl (%eax)
@@ -208,26 +208,26 @@ fyl2xp1
 # CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
 # CHECK-NEXT:  1      100   0.33                  U     f2xm1
 # CHECK-NEXT:  1      1     1.00                  U     fabs
-# CHECK-NEXT:  1      3     1.00                  U     fadd	%st(0), %st(1)
-# CHECK-NEXT:  1      3     1.00                  U     fadd	%st(2)
+# CHECK-NEXT:  1      3     1.00                  U     fadd	%st, %st(1)
+# CHECK-NEXT:  1      3     1.00                  U     fadd	%st(2), %st
 # CHECK-NEXT:  2      10    1.00    *             U     fadds	(%ecx)
 # CHECK-NEXT:  2      10    1.00    *             U     faddl	(%ecx)
-# CHECK-NEXT:  1      3     1.00                  U     faddp	%st(1)
-# CHECK-NEXT:  1      3     1.00                  U     faddp	%st(2)
+# CHECK-NEXT:  1      3     1.00                  U     faddp	%st, %st(1)
+# CHECK-NEXT:  1      3     1.00                  U     faddp	%st, %st(2)
 # CHECK-NEXT:  3      13    2.00    *             U     fiadds	(%ecx)
 # CHECK-NEXT:  3      13    2.00    *             U     fiaddl	(%ecx)
 # CHECK-NEXT:  1      100   0.33                  U     fbld	(%ecx)
 # CHECK-NEXT:  1      100   0.33                  U     fbstp	(%eax)
 # CHECK-NEXT:  1      1     1.00                  U     fchs
 # CHECK-NEXT:  1      100   0.33                  U     fnclex
-# CHECK-NEXT:  3      3     2.00                  U     fcmovb	%st(1), %st(0)
-# CHECK-NEXT:  3      3     2.00                  U     fcmovbe	%st(1), %st(0)
-# CHECK-NEXT:  3      3     2.00                  U     fcmove	%st(1), %st(0)
-# CHECK-NEXT:  3      3     2.00                  U     fcmovnb	%st(1), %st(0)
-# CHECK-NEXT:  3      3     2.00                  U     fcmovnbe	%st(1), %st(0)
-# CHECK-NEXT:  3      3     2.00                  U     fcmovne	%st(1), %st(0)
-# CHECK-NEXT:  3      3     2.00                  U     fcmovnu	%st(1), %st(0)
-# CHECK-NEXT:  3      3     2.00                  U     fcmovu	%st(1), %st(0)
+# CHECK-NEXT:  3      3     2.00                  U     fcmovb	%st(1), %st
+# CHECK-NEXT:  3      3     2.00                  U     fcmovbe	%st(1), %st
+# CHECK-NEXT:  3      3     2.00                  U     fcmove	%st(1), %st
+# CHECK-NEXT:  3      3     2.00                  U     fcmovnb	%st(1), %st
+# CHECK-NEXT:  3      3     2.00                  U     fcmovnbe	%st(1), %st
+# CHECK-NEXT:  3      3     2.00                  U     fcmovne	%st(1), %st
+# CHECK-NEXT:  3      3     2.00                  U     fcmovnu	%st(1), %st
+# CHECK-NEXT:  3      3     2.00                  U     fcmovu	%st(1), %st
 # CHECK-NEXT:  1      1     1.00                  U     fcom	%st(1)
 # CHECK-NEXT:  1      1     1.00                  U     fcom	%st(3)
 # CHECK-NEXT:  2      8     1.00                  U     fcoms	(%ecx)
@@ -237,24 +237,24 @@ fyl2xp1
 # CHECK-NEXT:  2      8     1.00                  U     fcomps	(%ecx)
 # CHECK-NEXT:  2      8     1.00                  U     fcompl	(%eax)
 # CHECK-NEXT:  1      100   0.33                  U     fcompp
-# CHECK-NEXT:  3      3     1.00                  U     fcomi	%st(3)
-# CHECK-NEXT:  3      3     1.00                  U     fcompi	%st(3)
+# CHECK-NEXT:  3      3     1.00                  U     fcomi	%st(3), %st
+# CHECK-NEXT:  3      3     1.00                  U     fcompi	%st(3), %st
 # CHECK-NEXT:  1      100   0.33                  U     fcos
 # CHECK-NEXT:  1      1     1.00                  U     fdecstp
-# CHECK-NEXT:  1      14    14.00                 U     fdiv	%st(0), %st(1)
-# CHECK-NEXT:  1      14    14.00                 U     fdiv	%st(2)
+# CHECK-NEXT:  1      14    14.00                 U     fdiv	%st, %st(1)
+# CHECK-NEXT:  1      14    14.00                 U     fdiv	%st(2), %st
 # CHECK-NEXT:  2      31    1.00    *             U     fdivs	(%ecx)
 # CHECK-NEXT:  2      31    1.00    *             U     fdivl	(%eax)
-# CHECK-NEXT:  1      14    14.00                 U     fdivp	%st(1)
-# CHECK-NEXT:  1      14    14.00                 U     fdivp	%st(2)
+# CHECK-NEXT:  1      14    14.00                 U     fdivp	%st, %st(1)
+# CHECK-NEXT:  1      14    14.00                 U     fdivp	%st, %st(2)
 # CHECK-NEXT:  3      34    1.00    *             U     fidivs	(%ecx)
 # CHECK-NEXT:  3      34    1.00    *             U     fidivl	(%eax)
-# CHECK-NEXT:  1      14    14.00                 U     fdivr	%st(0), %st(1)
-# CHECK-NEXT:  1      14    14.00                 U     fdivr	%st(2)
+# CHECK-NEXT:  1      14    14.00                 U     fdivr	%st, %st(1)
+# CHECK-NEXT:  1      14    14.00                 U     fdivr	%st(2), %st
 # CHECK-NEXT:  2      31    1.00    *             U     fdivrs	(%ecx)
 # CHECK-NEXT:  2      31    1.00    *             U     fdivrl	(%eax)
-# CHECK-NEXT:  1      14    14.00                 U     fdivrp	%st(1)
-# CHECK-NEXT:  1      14    14.00                 U     fdivrp	%st(2)
+# CHECK-NEXT:  1      14    14.00                 U     fdivrp	%st, %st(1)
+# CHECK-NEXT:  1      14    14.00                 U     fdivrp	%st, %st(2)
 # CHECK-NEXT:  3      34    1.00    *             U     fidivrs	(%ecx)
 # CHECK-NEXT:  3      34    1.00    *             U     fidivrl	(%eax)
 # CHECK-NEXT:  1      1     1.00                  U     ffree	%st(0)
@@ -288,12 +288,12 @@ fyl2xp1
 # CHECK-NEXT:  2      1     1.00                  U     fldln2
 # CHECK-NEXT:  2      1     1.00                  U     fldpi
 # CHECK-NEXT:  1      1     1.00                  U     fldz
-# CHECK-NEXT:  1      5     1.00                  U     fmul	%st(0), %st(1)
-# CHECK-NEXT:  1      5     1.00                  U     fmul	%st(2)
+# CHECK-NEXT:  1      5     1.00                  U     fmul	%st, %st(1)
+# CHECK-NEXT:  1      5     1.00                  U     fmul	%st(2), %st
 # CHECK-NEXT:  2      12    1.00    *             U     fmuls	(%ecx)
 # CHECK-NEXT:  2      12    1.00    *             U     fmull	(%eax)
-# CHECK-NEXT:  1      5     1.00                  U     fmulp	%st(1)
-# CHECK-NEXT:  1      5     1.00                  U     fmulp	%st(2)
+# CHECK-NEXT:  1      5     1.00                  U     fmulp	%st, %st(1)
+# CHECK-NEXT:  1      5     1.00                  U     fmulp	%st, %st(2)
 # CHECK-NEXT:  3      15    1.00    *             U     fimuls	(%ecx)
 # CHECK-NEXT:  3      15    1.00    *             U     fimull	(%eax)
 # CHECK-NEXT:  1      1     1.00                  U     fnop
@@ -321,20 +321,20 @@ fyl2xp1
 # CHECK-NEXT:  1      100   0.33                  U     frstor	(%eax)
 # CHECK-NEXT:  1      100   0.33                  U     wait
 # CHECK-NEXT:  1      100   0.33                  U     fnsave	(%eax)
-# CHECK-NEXT:  1      3     1.00                  U     fsub	%st(0), %st(1)
-# CHECK-NEXT:  1      3     1.00                  U     fsub	%st(2)
+# CHECK-NEXT:  1      3     1.00                  U     fsub	%st, %st(1)
+# CHECK-NEXT:  1      3     1.00                  U     fsub	%st(2), %st
 # CHECK-NEXT:  2      10    1.00    *             U     fsubs	(%ecx)
 # CHECK-NEXT:  2      10    1.00    *             U     fsubl	(%eax)
-# CHECK-NEXT:  1      3     1.00                  U     fsubp	%st(1)
-# CHECK-NEXT:  1      3     1.00                  U     fsubp	%st(2)
+# CHECK-NEXT:  1      3     1.00                  U     fsubp	%st, %st(1)
+# CHECK-NEXT:  1      3     1.00                  U     fsubp	%st, %st(2)
 # CHECK-NEXT:  3      13    2.00    *             U     fisubs	(%ecx)
 # CHECK-NEXT:  3      13    2.00    *             U     fisubl	(%eax)
-# CHECK-NEXT:  1      3     1.00                  U     fsubr	%st(0), %st(1)
-# CHECK-NEXT:  1      3     1.00                  U     fsubr	%st(2)
+# CHECK-NEXT:  1      3     1.00                  U     fsubr	%st, %st(1)
+# CHECK-NEXT:  1      3     1.00                  U     fsubr	%st(2), %st
 # CHECK-NEXT:  2      10    1.00    *             U     fsubrs	(%ecx)
 # CHECK-NEXT:  2      10    1.00    *             U     fsubrl	(%eax)
-# CHECK-NEXT:  1      3     1.00                  U     fsubrp	%st(1)
-# CHECK-NEXT:  1      3     1.00                  U     fsubrp	%st(2)
+# CHECK-NEXT:  1      3     1.00                  U     fsubrp	%st, %st(1)
+# CHECK-NEXT:  1      3     1.00                  U     fsubrp	%st, %st(2)
 # CHECK-NEXT:  3      13    2.00    *             U     fisubrs	(%ecx)
 # CHECK-NEXT:  3      13    2.00    *             U     fisubrl	(%eax)
 # CHECK-NEXT:  1      3     1.00                  U     ftst
@@ -343,8 +343,8 @@ fyl2xp1
 # CHECK-NEXT:  1      1     1.00                  U     fucomp	%st(1)
 # CHECK-NEXT:  1      1     1.00                  U     fucomp	%st(3)
 # CHECK-NEXT:  1      3     1.00                  U     fucompp
-# CHECK-NEXT:  3      3     1.00                  U     fucomi	%st(3)
-# CHECK-NEXT:  3      3     1.00                  U     fucompi	%st(3)
+# CHECK-NEXT:  3      3     1.00                  U     fucomi	%st(3), %st
+# CHECK-NEXT:  3      3     1.00                  U     fucompi	%st(3), %st
 # CHECK-NEXT:  1      100   0.33                  U     wait
 # CHECK-NEXT:  1      100   0.33                  U     fxam
 # CHECK-NEXT:  1      1     0.33                  U     fxch	%st(1)
@@ -373,26 +373,26 @@ fyl2xp1
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6.0]  [6.1]  Instructions:
 # CHECK-NEXT:  -      -     0.33   0.33    -     0.33    -      -     f2xm1
 # CHECK-NEXT:  -      -      -      -      -     1.00    -      -     fabs
-# CHECK-NEXT:  -      -      -     1.00    -      -      -      -     fadd	%st(0), %st(1)
-# CHECK-NEXT:  -      -      -     1.00    -      -      -      -     fadd	%st(2)
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -     fadd	%st, %st(1)
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -     fadd	%st(2), %st
 # CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   fadds	(%ecx)
 # CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   faddl	(%ecx)
-# CHECK-NEXT:  -      -      -     1.00    -      -      -      -     faddp	%st(1)
-# CHECK-NEXT:  -      -      -     1.00    -      -      -      -     faddp	%st(2)
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -     faddp	%st, %st(1)
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -     faddp	%st, %st(2)
 # CHECK-NEXT:  -      -      -     2.00    -      -     0.50   0.50   fiadds	(%ecx)
 # CHECK-NEXT:  -      -      -     2.00    -      -     0.50   0.50   fiaddl	(%ecx)
 # CHECK-NEXT:  -      -     0.33   0.33    -     0.33    -      -     fbld	(%ecx)
 # CHECK-NEXT:  -      -     0.33   0.33    -     0.33    -      -     fbstp	(%eax)
 # CHECK-NEXT:  -      -      -      -      -     1.00    -      -     fchs
 # CHECK-NEXT:  -      -     0.33   0.33    -     0.33    -      -     fnclex
-# CHECK-NEXT:  -      -     0.50    -      -     2.50    -      -     fcmovb	%st(1), %st(0)
-# CHECK-NEXT:  -      -     0.50    -      -     2.50    -      -     fcmovbe	%st(1), %st(0)
-# CHECK-NEXT:  -      -     0.50    -      -     2.50    -      -     fcmove	%st(1), %st(0)
-# CHECK-NEXT:  -      -     0.50    -      -     2.50    -      -     fcmovnb	%st(1), %st(0)
-# CHECK-NEXT:  -      -     0.50    -      -     2.50    -      -     fcmovnbe	%st(1), %st(0)
-# CHECK-NEXT:  -      -     0.50    -      -     2.50    -      -     fcmovne	%st(1), %st(0)
-# CHECK-NEXT:  -      -     0.50    -      -     2.50    -      -     fcmovnu	%st(1), %st(0)
-# CHECK-NEXT:  -      -     0.50    -      -     2.50    -      -     fcmovu	%st(1), %st(0)
+# CHECK-NEXT:  -      -     0.50    -      -     2.50    -      -     fcmovb	%st(1), %st
+# CHECK-NEXT:  -      -     0.50    -      -     2.50    -      -     fcmovbe	%st(1), %st
+# CHECK-NEXT:  -      -     0.50    -      -     2.50    -      -     fcmove	%st(1), %st
+# CHECK-NEXT:  -      -     0.50    -      -     2.50    -      -     fcmovnb	%st(1), %st
+# CHECK-NEXT:  -      -     0.50    -      -     2.50    -      -     fcmovnbe	%st(1), %st
+# CHECK-NEXT:  -      -     0.50    -      -     2.50    -      -     fcmovne	%st(1), %st
+# CHECK-NEXT:  -      -     0.50    -      -     2.50    -      -     fcmovnu	%st(1), %st
+# CHECK-NEXT:  -      -     0.50    -      -     2.50    -      -     fcmovu	%st(1), %st
 # CHECK-NEXT:  -      -      -     1.00    -      -      -      -     fcom	%st(1)
 # CHECK-NEXT:  -      -      -     1.00    -      -      -      -     fcom	%st(3)
 # CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   fcoms	(%ecx)
@@ -402,24 +402,24 @@ fyl2xp1
 # CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   fcomps	(%ecx)
 # CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   fcompl	(%eax)
 # CHECK-NEXT:  -      -     0.33   0.33    -     0.33    -      -     fcompp
-# CHECK-NEXT:  -      -     1.00   1.00    -     1.00    -      -     fcomi	%st(3)
-# CHECK-NEXT:  -      -     1.00   1.00    -     1.00    -      -     fcompi	%st(3)
+# CHECK-NEXT:  -      -     1.00   1.00    -     1.00    -      -     fcomi	%st(3), %st
+# CHECK-NEXT:  -      -     1.00   1.00    -     1.00    -      -     fcompi	%st(3), %st
 # CHECK-NEXT:  -      -     0.33   0.33    -     0.33    -      -     fcos
 # CHECK-NEXT:  -      -      -      -      -     1.00    -      -     fdecstp
-# CHECK-NEXT:  -     14.00  1.00    -      -      -      -      -     fdiv	%st(0), %st(1)
-# CHECK-NEXT:  -     14.00  1.00    -      -      -      -      -     fdiv	%st(2)
+# CHECK-NEXT:  -     14.00  1.00    -      -      -      -      -     fdiv	%st, %st(1)
+# CHECK-NEXT:  -     14.00  1.00    -      -      -      -      -     fdiv	%st(2), %st
 # CHECK-NEXT:  -      -     1.00    -      -      -     0.50   0.50   fdivs	(%ecx)
 # CHECK-NEXT:  -      -     1.00    -      -      -     0.50   0.50   fdivl	(%eax)
-# CHECK-NEXT:  -     14.00  1.00    -      -      -      -      -     fdivp	%st(1)
-# CHECK-NEXT:  -     14.00  1.00    -      -      -      -      -     fdivp	%st(2)
+# CHECK-NEXT:  -     14.00  1.00    -      -      -      -      -     fdivp	%st, %st(1)
+# CHECK-NEXT:  -     14.00  1.00    -      -      -      -      -     fdivp	%st, %st(2)
 # CHECK-NEXT:  -      -     1.00   1.00    -      -     0.50   0.50   fidivs	(%ecx)
 # CHECK-NEXT:  -      -     1.00   1.00    -      -     0.50   0.50   fidivl	(%eax)
-# CHECK-NEXT:  -     14.00  1.00    -      -      -      -      -     fdivr	%st(0), %st(1)
-# CHECK-NEXT:  -     14.00  1.00    -      -      -      -      -     fdivr	%st(2)
+# CHECK-NEXT:  -     14.00  1.00    -      -      -      -      -     fdivr	%st, %st(1)
+# CHECK-NEXT:  -     14.00  1.00    -      -      -      -      -     fdivr	%st(2), %st
 # CHECK-NEXT:  -      -     1.00    -      -      -     0.50   0.50   fdivrs	(%ecx)
 # CHECK-NEXT:  -      -     1.00    -      -      -     0.50   0.50   fdivrl	(%eax)
-# CHECK-NEXT:  -     14.00  1.00    -      -      -      -      -     fdivrp	%st(1)
-# CHECK-NEXT:  -     14.00  1.00    -      -      -      -      -     fdivrp	%st(2)
+# CHECK-NEXT:  -     14.00  1.00    -      -      -      -      -     fdivrp	%st, %st(1)
+# CHECK-NEXT:  -     14.00  1.00    -      -      -      -      -     fdivrp	%st, %st(2)
 # CHECK-NEXT:  -      -     1.00   1.00    -      -     0.50   0.50   fidivrs	(%ecx)
 # CHECK-NEXT:  -      -     1.00   1.00    -      -     0.50   0.50   fidivrl	(%eax)
 # CHECK-NEXT:  -      -      -      -      -     1.00    -      -     ffree	%st(0)
@@ -453,12 +453,12 @@ fyl2xp1
 # CHECK-NEXT:  -      -     1.00   1.00    -      -      -      -     fldln2
 # CHECK-NEXT:  -      -     1.00   1.00    -      -      -      -     fldpi
 # CHECK-NEXT:  -      -      -      -      -     1.00    -      -     fldz
-# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     fmul	%st(0), %st(1)
-# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     fmul	%st(2)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     fmul	%st, %st(1)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     fmul	%st(2), %st
 # CHECK-NEXT:  -      -     1.00    -      -      -     0.50   0.50   fmuls	(%ecx)
 # CHECK-NEXT:  -      -     1.00    -      -      -     0.50   0.50   fmull	(%eax)
-# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     fmulp	%st(1)
-# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     fmulp	%st(2)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     fmulp	%st, %st(1)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -     fmulp	%st, %st(2)
 # CHECK-NEXT:  -      -     1.00   1.00    -      -     0.50   0.50   fimuls	(%ecx)
 # CHECK-NEXT:  -      -     1.00   1.00    -      -     0.50   0.50   fimull	(%eax)
 # CHECK-NEXT:  -      -      -      -      -     1.00    -      -     fnop
@@ -486,20 +486,20 @@ fyl2xp1
 # CHECK-NEXT:  -      -     0.33   0.33    -     0.33    -      -     frstor	(%eax)
 # CHECK-NEXT:  -      -     0.33   0.33    -     0.33    -      -     wait
 # CHECK-NEXT:  -      -     0.33   0.33    -     0.33    -      -     fnsave	(%eax)
-# CHECK-NEXT:  -      -      -     1.00    -      -      -      -     fsub	%st(0), %st(1)
-# CHECK-NEXT:  -      -      -     1.00    -      -      -      -     fsub	%st(2)
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -     fsub	%st, %st(1)
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -     fsub	%st(2), %st
 # CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   fsubs	(%ecx)
 # CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   fsubl	(%eax)
-# CHECK-NEXT:  -      -      -     1.00    -      -      -      -     fsubp	%st(1)
-# CHECK-NEXT:  -      -      -     1.00    -      -      -      -     fsubp	%st(2)
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -     fsubp	%st, %st(1)
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -     fsubp	%st, %st(2)
 # CHECK-NEXT:  -      -      -     2.00    -      -     0.50   0.50   fisubs	(%ecx)
 # CHECK-NEXT:  -      -      -     2.00    -      -     0.50   0.50   fisubl	(%eax)
-# CHECK-NEXT:  -      -      -     1.00    -      -      -      -     fsubr	%st(0), %st(1)
-# CHECK-NEXT:  -      -      -     1.00    -      -      -      -     fsubr	%st(2)
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -     fsubr	%st, %st(1)
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -     fsubr	%st(2), %st
 # CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   fsubrs	(%ecx)
 # CHECK-NEXT:  -      -      -     1.00    -      -     0.50   0.50   fsubrl	(%eax)
-# CHECK-NEXT:  -      -      -     1.00    -      -      -      -     fsubrp	%st(1)
-# CHECK-NEXT:  -      -      -     1.00    -      -      -      -     fsubrp	%st(2)
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -     fsubrp	%st, %st(1)
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -     fsubrp	%st, %st(2)
 # CHECK-NEXT:  -      -      -     2.00    -      -     0.50   0.50   fisubrs	(%ecx)
 # CHECK-NEXT:  -      -      -     2.00    -      -     0.50   0.50   fisubrl	(%eax)
 # CHECK-NEXT:  -      -      -     1.00    -      -      -      -     ftst
@@ -508,8 +508,8 @@ fyl2xp1
 # CHECK-NEXT:  -      -      -     1.00    -      -      -      -     fucomp	%st(1)
 # CHECK-NEXT:  -      -      -     1.00    -      -      -      -     fucomp	%st(3)
 # CHECK-NEXT:  -      -      -     1.00    -      -      -      -     fucompp
-# CHECK-NEXT:  -      -     1.00   1.00    -     1.00    -      -     fucomi	%st(3)
-# CHECK-NEXT:  -      -     1.00   1.00    -     1.00    -      -     fucompi	%st(3)
+# CHECK-NEXT:  -      -     1.00   1.00    -     1.00    -      -     fucomi	%st(3), %st
+# CHECK-NEXT:  -      -     1.00   1.00    -     1.00    -      -     fucompi	%st(3), %st
 # CHECK-NEXT:  -      -     0.33   0.33    -     0.33    -      -     wait
 # CHECK-NEXT:  -      -     0.33   0.33    -     0.33    -      -     fxam
 # CHECK-NEXT:  -      -     0.33   0.33    -     0.33    -      -     fxch	%st(1)
diff --git a/test/tools/llvm-mca/X86/SkylakeClient/resources-avx2.s b/test/tools/llvm-mca/X86/SkylakeClient/resources-avx2.s
index 6804bcc13e42a3f11b43f120b7b003bbcc67974a..26b5ef97dc0fea7d7470f6b821ac66de50ee3665 100644
--- a/test/tools/llvm-mca/X86/SkylakeClient/resources-avx2.s
+++ b/test/tools/llvm-mca/X86/SkylakeClient/resources-avx2.s
@@ -149,6 +149,9 @@ vpcmpgtq        (%rax), %ymm1, %ymm2
 vpcmpgtw        %ymm0, %ymm1, %ymm2
 vpcmpgtw        (%rax), %ymm1, %ymm2
 
+vperm2i128      $1, %ymm0, %ymm1, %ymm2
+vperm2i128      $1, (%rax), %ymm1, %ymm2
+
 vpermd          %ymm0, %ymm1, %ymm2
 vpermd          (%rax), %ymm1, %ymm2
 
@@ -555,6 +558,8 @@ vpxor           (%rax), %ymm1, %ymm2
 # CHECK-NEXT:  2      10    1.00    *                   vpcmpgtq	(%rax), %ymm1, %ymm2
 # CHECK-NEXT:  1      1     0.50                        vpcmpgtw	%ymm0, %ymm1, %ymm2
 # CHECK-NEXT:  2      8     0.50    *                   vpcmpgtw	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      3     1.00                        vperm2i128	$1, %ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  2      10    1.00    *                   vperm2i128	$1, (%rax), %ymm1, %ymm2
 # CHECK-NEXT:  1      3     1.00                        vpermd	%ymm0, %ymm1, %ymm2
 # CHECK-NEXT:  2      10    1.00    *                   vpermd	(%rax), %ymm1, %ymm2
 # CHECK-NEXT:  1      3     1.00                        vpermpd	$1, %ymm0, %ymm2
@@ -771,7 +776,7 @@ vpxor           (%rax), %ymm1, %ymm2
 
 # CHECK:      Resource pressure per iteration:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]
-# CHECK-NEXT:  -      -     110.33 89.33  85.17  85.17  1.00   162.33  -     1.67
+# CHECK-NEXT:  -      -     110.33 89.33  85.67  85.67  1.00   164.33  -     1.67
 
 # CHECK:      Resource pressure by instruction:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    Instructions:
@@ -873,6 +878,8 @@ vpxor           (%rax), %ymm1, %ymm2
 # CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpcmpgtq	(%rax), %ymm1, %ymm2
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -     vpcmpgtw	%ymm0, %ymm1, %ymm2
 # CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vpcmpgtw	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vperm2i128	$1, %ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vperm2i128	$1, (%rax), %ymm1, %ymm2
 # CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpermd	%ymm0, %ymm1, %ymm2
 # CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermd	(%rax), %ymm1, %ymm2
 # CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpermpd	$1, %ymm0, %ymm2
diff --git a/test/tools/llvm-mca/X86/SkylakeClient/resources-bmi1.s b/test/tools/llvm-mca/X86/SkylakeClient/resources-bmi1.s
index 396e7a2a2a102dfe9017af1b3d88d708c7629f74..5be6f752f089f6a4e221f5f940afb9064cb73315 100644
--- a/test/tools/llvm-mca/X86/SkylakeClient/resources-bmi1.s
+++ b/test/tools/llvm-mca/X86/SkylakeClient/resources-bmi1.s
@@ -31,6 +31,9 @@ blsr        (%rax), %ecx
 blsr        %rax, %rcx
 blsr        (%rax), %rcx
 
+tzcnt       %ax, %cx
+tzcnt       (%rax), %cx
+
 tzcnt       %eax, %ecx
 tzcnt       (%rax), %ecx
 
@@ -66,6 +69,8 @@ tzcnt       (%rax), %rcx
 # CHECK-NEXT:  2      6     0.50    *                   blsrl	(%rax), %ecx
 # CHECK-NEXT:  1      1     0.50                        blsrq	%rax, %rcx
 # CHECK-NEXT:  2      6     0.50    *                   blsrq	(%rax), %rcx
+# CHECK-NEXT:  1      3     1.00                        tzcntw	%ax, %cx
+# CHECK-NEXT:  2      8     1.00    *                   tzcntw	(%rax), %cx
 # CHECK-NEXT:  1      3     1.00                        tzcntl	%eax, %ecx
 # CHECK-NEXT:  2      8     1.00    *                   tzcntl	(%rax), %ecx
 # CHECK-NEXT:  1      3     1.00                        tzcntq	%rax, %rcx
@@ -85,7 +90,7 @@ tzcnt       (%rax), %rcx
 
 # CHECK:      Resource pressure per iteration:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]
-# CHECK-NEXT:  -      -     2.00   14.00  6.00   6.00    -     10.00  2.00    -
+# CHECK-NEXT:  -      -     2.00   16.00  6.50   6.50    -     10.00  2.00    -
 
 # CHECK:      Resource pressure by instruction:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    Instructions:
@@ -109,6 +114,8 @@ tzcnt       (%rax), %rcx
 # CHECK-NEXT:  -      -      -     0.50   0.50   0.50    -     0.50    -      -     blsrl	(%rax), %ecx
 # CHECK-NEXT:  -      -      -     0.50    -      -      -     0.50    -      -     blsrq	%rax, %rcx
 # CHECK-NEXT:  -      -      -     0.50   0.50   0.50    -     0.50    -      -     blsrq	(%rax), %rcx
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     tzcntw	%ax, %cx
+# CHECK-NEXT:  -      -      -     1.00   0.50   0.50    -      -      -      -     tzcntw	(%rax), %cx
 # CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     tzcntl	%eax, %ecx
 # CHECK-NEXT:  -      -      -     1.00   0.50   0.50    -      -      -      -     tzcntl	(%rax), %ecx
 # CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     tzcntq	%rax, %rcx
diff --git a/test/tools/llvm-mca/X86/SkylakeClient/resources-fsgsbase.s b/test/tools/llvm-mca/X86/SkylakeClient/resources-fsgsbase.s
new file mode 100644
index 0000000000000000000000000000000000000000..97819bafdc58e8e6323bf711840097f5def7be15
--- /dev/null
+++ b/test/tools/llvm-mca/X86/SkylakeClient/resources-fsgsbase.s
@@ -0,0 +1,59 @@
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=x86_64-unknown-unknown -mcpu=skylake -instruction-tables < %s | FileCheck %s
+
+rdfsbase %eax
+rdfsbase %rax
+
+rdgsbase %eax
+rdgsbase %rax
+
+wrfsbase %edi
+wrfsbase %rdi
+
+wrgsbase %edi
+wrgsbase %rdi
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  1      100   0.25    *      *      U     rdfsbasel	%eax
+# CHECK-NEXT:  1      100   0.25    *      *      U     rdfsbaseq	%rax
+# CHECK-NEXT:  1      100   0.25    *      *      U     rdgsbasel	%eax
+# CHECK-NEXT:  1      100   0.25    *      *      U     rdgsbaseq	%rax
+# CHECK-NEXT:  1      100   0.25    *      *      U     wrfsbasel	%edi
+# CHECK-NEXT:  1      100   0.25    *      *      U     wrfsbaseq	%rdi
+# CHECK-NEXT:  1      100   0.25    *      *      U     wrgsbasel	%edi
+# CHECK-NEXT:  1      100   0.25    *      *      U     wrgsbaseq	%rdi
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0]   - SKLDivider
+# CHECK-NEXT: [1]   - SKLFPDivider
+# CHECK-NEXT: [2]   - SKLPort0
+# CHECK-NEXT: [3]   - SKLPort1
+# CHECK-NEXT: [4]   - SKLPort2
+# CHECK-NEXT: [5]   - SKLPort3
+# CHECK-NEXT: [6]   - SKLPort4
+# CHECK-NEXT: [7]   - SKLPort5
+# CHECK-NEXT: [8]   - SKLPort6
+# CHECK-NEXT: [9]   - SKLPort7
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]
+# CHECK-NEXT:  -      -     2.00   2.00    -      -      -     2.00   2.00    -
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    Instructions:
+# CHECK-NEXT:  -      -     0.25   0.25    -      -      -     0.25   0.25    -     rdfsbasel	%eax
+# CHECK-NEXT:  -      -     0.25   0.25    -      -      -     0.25   0.25    -     rdfsbaseq	%rax
+# CHECK-NEXT:  -      -     0.25   0.25    -      -      -     0.25   0.25    -     rdgsbasel	%eax
+# CHECK-NEXT:  -      -     0.25   0.25    -      -      -     0.25   0.25    -     rdgsbaseq	%rax
+# CHECK-NEXT:  -      -     0.25   0.25    -      -      -     0.25   0.25    -     wrfsbasel	%edi
+# CHECK-NEXT:  -      -     0.25   0.25    -      -      -     0.25   0.25    -     wrfsbaseq	%rdi
+# CHECK-NEXT:  -      -     0.25   0.25    -      -      -     0.25   0.25    -     wrgsbasel	%edi
+# CHECK-NEXT:  -      -     0.25   0.25    -      -      -     0.25   0.25    -     wrgsbaseq	%rdi
diff --git a/test/tools/llvm-mca/X86/SkylakeClient/resources-sse2.s b/test/tools/llvm-mca/X86/SkylakeClient/resources-sse2.s
index f9cee78654bfc76a01cd9b6080c41ac1872ef8f8..8c30e781b3931e1351635f225ed51021104a967c 100644
--- a/test/tools/llvm-mca/X86/SkylakeClient/resources-sse2.s
+++ b/test/tools/llvm-mca/X86/SkylakeClient/resources-sse2.s
@@ -94,6 +94,8 @@ maxpd       (%rax), %xmm2
 maxsd       %xmm0, %xmm2
 maxsd       (%rax), %xmm2
 
+mfence
+
 minpd       %xmm0, %xmm2
 minpd       (%rax), %xmm2
 
@@ -229,6 +231,9 @@ pcmpgtw     (%rax), %xmm2
 
 pextrw      $1, %xmm0, %rcx
 
+pinsrw      $1, %rax, %xmm0
+pinsrw      $1, (%rax), %xmm0
+
 pmaddwd     %xmm0, %xmm2
 pmaddwd     (%rax), %xmm2
 
@@ -465,6 +470,7 @@ xorpd       (%rax), %xmm2
 # CHECK-NEXT:  2      10    0.50    *                   maxpd	(%rax), %xmm2
 # CHECK-NEXT:  1      4     0.50                        maxsd	%xmm0, %xmm2
 # CHECK-NEXT:  2      9     0.50    *                   maxsd	(%rax), %xmm2
+# CHECK-NEXT:  3      3     0.50    *      *      U     mfence
 # CHECK-NEXT:  1      4     0.50                        minpd	%xmm0, %xmm2
 # CHECK-NEXT:  2      10    0.50    *                   minpd	(%rax), %xmm2
 # CHECK-NEXT:  1      4     0.50                        minsd	%xmm0, %xmm2
@@ -555,6 +561,8 @@ xorpd       (%rax), %xmm2
 # CHECK-NEXT:  1      1     0.50                        pcmpgtw	%xmm0, %xmm2
 # CHECK-NEXT:  2      7     0.50    *                   pcmpgtw	(%rax), %xmm2
 # CHECK-NEXT:  2      3     1.00                        pextrw	$1, %xmm0, %ecx
+# CHECK-NEXT:  2      2     2.00                        pinsrw	$1, %eax, %xmm0
+# CHECK-NEXT:  2      6     1.00    *                   pinsrw	$1, (%rax), %xmm0
 # CHECK-NEXT:  1      4     0.50                        pmaddwd	%xmm0, %xmm2
 # CHECK-NEXT:  2      10    0.50    *                   pmaddwd	(%rax), %xmm2
 # CHECK-NEXT:  1      1     0.50                        pmaxsw	%xmm0, %xmm2
@@ -681,7 +689,7 @@ xorpd       (%rax), %xmm2
 
 # CHECK:      Resource pressure per iteration:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]
-# CHECK-NEXT:  -     38.00  110.42 78.42  62.67  62.67  14.00  93.42  1.75   4.67
+# CHECK-NEXT:  -     38.00  110.92 78.92  63.50  63.50  14.00  96.92  2.25   5.00
 
 # CHECK:      Resource pressure by instruction:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    Instructions:
@@ -748,6 +756,7 @@ xorpd       (%rax), %xmm2
 # CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     maxpd	(%rax), %xmm2
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -     maxsd	%xmm0, %xmm2
 # CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     maxsd	(%rax), %xmm2
+# CHECK-NEXT:  -      -     0.50   0.50   0.33   0.33    -     0.50   0.50   0.33   mfence
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -     minpd	%xmm0, %xmm2
 # CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     minpd	(%rax), %xmm2
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -     minsd	%xmm0, %xmm2
@@ -838,6 +847,8 @@ xorpd       (%rax), %xmm2
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -     pcmpgtw	%xmm0, %xmm2
 # CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     pcmpgtw	(%rax), %xmm2
 # CHECK-NEXT:  -      -     1.00    -      -      -      -     1.00    -      -     pextrw	$1, %xmm0, %ecx
+# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -     pinsrw	$1, %eax, %xmm0
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     pinsrw	$1, (%rax), %xmm0
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -     pmaddwd	%xmm0, %xmm2
 # CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     pmaddwd	(%rax), %xmm2
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -     pmaxsw	%xmm0, %xmm2
diff --git a/test/tools/llvm-mca/X86/SkylakeClient/resources-sse3.s b/test/tools/llvm-mca/X86/SkylakeClient/resources-sse3.s
index d7a9e5aa44fafd56beeba80fbe526d516a7bd401..df495ed45be95f8ab21514608a77675aa387a342 100644
--- a/test/tools/llvm-mca/X86/SkylakeClient/resources-sse3.s
+++ b/test/tools/llvm-mca/X86/SkylakeClient/resources-sse3.s
@@ -21,6 +21,8 @@ hsubps    (%rax), %xmm2
 
 lddqu     (%rax), %xmm2
 
+monitor
+
 movddup   %xmm0, %xmm2
 movddup   (%rax), %xmm2
 
@@ -30,6 +32,8 @@ movshdup  (%rax), %xmm2
 movsldup  %xmm0, %xmm2
 movsldup  (%rax), %xmm2
 
+mwait
+
 # CHECK:      Instruction Info:
 # CHECK-NEXT: [1]: #uOps
 # CHECK-NEXT: [2]: Latency
@@ -52,12 +56,14 @@ movsldup  (%rax), %xmm2
 # CHECK-NEXT:  3      6     2.00                        hsubps	%xmm0, %xmm2
 # CHECK-NEXT:  4      12    2.00    *                   hsubps	(%rax), %xmm2
 # CHECK-NEXT:  1      6     0.50    *                   lddqu	(%rax), %xmm2
+# CHECK-NEXT:  1      100   0.25                  U     monitor
 # CHECK-NEXT:  1      1     1.00                        movddup	%xmm0, %xmm2
 # CHECK-NEXT:  1      5     0.50    *                   movddup	(%rax), %xmm2
 # CHECK-NEXT:  1      1     1.00                        movshdup	%xmm0, %xmm2
 # CHECK-NEXT:  1      6     0.50    *                   movshdup	(%rax), %xmm2
 # CHECK-NEXT:  1      1     1.00                        movsldup	%xmm0, %xmm2
 # CHECK-NEXT:  1      6     0.50    *                   movsldup	(%rax), %xmm2
+# CHECK-NEXT:  10     20    2.50    *      *      U     mwait
 
 # CHECK:      Resources:
 # CHECK-NEXT: [0]   - SKLDivider
@@ -73,7 +79,7 @@ movsldup  (%rax), %xmm2
 
 # CHECK:      Resource pressure per iteration:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]
-# CHECK-NEXT:  -      -     6.00   6.00   5.00   5.00    -     19.00   -      -
+# CHECK-NEXT:  -      -     8.00   8.00   5.00   5.00    -     22.00  4.00    -
 
 # CHECK:      Resource pressure by instruction:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    Instructions:
@@ -90,9 +96,11 @@ movsldup  (%rax), %xmm2
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -     2.00    -      -     hsubps	%xmm0, %xmm2
 # CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -     2.00    -      -     hsubps	(%rax), %xmm2
 # CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -     lddqu	(%rax), %xmm2
+# CHECK-NEXT:  -      -     0.25   0.25    -      -      -     0.25   0.25    -     monitor
 # CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     movddup	%xmm0, %xmm2
 # CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -     movddup	(%rax), %xmm2
 # CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     movshdup	%xmm0, %xmm2
 # CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -     movshdup	(%rax), %xmm2
 # CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     movsldup	%xmm0, %xmm2
 # CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -     movsldup	(%rax), %xmm2
+# CHECK-NEXT:  -      -     1.75   1.75    -      -      -     2.75   3.75    -     mwait
diff --git a/test/tools/llvm-mca/X86/SkylakeClient/resources-x86_64.s b/test/tools/llvm-mca/X86/SkylakeClient/resources-x86_64.s
index c008e3d5f65d36a0398133bffae848d54e7c7992..963b8c90e9c2f866be9717b029b2cf601c78a3b7 100644
--- a/test/tools/llvm-mca/X86/SkylakeClient/resources-x86_64.s
+++ b/test/tools/llvm-mca/X86/SkylakeClient/resources-x86_64.s
@@ -254,6 +254,8 @@ divl (%rax)
 divq %rcx
 divq (%rax)
 
+enter $7, $4095
+
 idivb %dil
 idivb (%rax)
 idivw %si
@@ -315,8 +317,13 @@ insl
 
 int $7
 
+invlpg  (%rax)
+invlpga %rax, %ecx
+
 lahf
 
+leave
+
 lodsb
 lodsw
 lodsl
@@ -486,6 +493,11 @@ rcrq %cl, %rdi
 rclq %cl, (%rax)
 rcrq %cl, (%rax)
 
+rdmsr
+rdpmc
+rdtsc
+rdtscp
+
 rolb %dil
 rorb %dil
 rolb (%rax)
@@ -789,6 +801,8 @@ testq %rsi, (%rax)
 
 ud2
 
+wrmsr
+
 xaddb %bl, %cl
 xaddb %bl, (%rcx)
 
@@ -1081,6 +1095,7 @@ xorq (%rax), %rdi
 # CHECK-NEXT:  2      29    10.00   *             U     divl	(%rax)
 # CHECK-NEXT:  32     76    8.00                  U     divq	%rcx
 # CHECK-NEXT:  2      29    10.00   *             U     divq	(%rax)
+# CHECK-NEXT:  1      100   0.25                  U     enter	$7, $4095
 # CHECK-NEXT:  1      25    10.00                 U     idivb	%dil
 # CHECK-NEXT:  8      28    4.00    *             U     idivb	(%rax)
 # CHECK-NEXT:  66     102   16.50                 U     idivw	%si
@@ -1133,7 +1148,10 @@ xorq (%rax), %rdi
 # CHECK-NEXT:  8      20    1.25                  U     insw	%dx, %es:(%rdi)
 # CHECK-NEXT:  8      20    1.25                  U     insl	%dx, %es:(%rdi)
 # CHECK-NEXT:  1      100   0.25    *      *      U     int	$7
+# CHECK-NEXT:  1      100   0.25                  U     invlpg	(%rax)
+# CHECK-NEXT:  1      100   0.25                  U     invlpga	%rax, %ecx
 # CHECK-NEXT:  1      1     0.50                        lahf
+# CHECK-NEXT:  3      7     0.50    *                   leave
 # CHECK-NEXT:  1      100   0.25                  U     lodsb	(%rsi), %al
 # CHECK-NEXT:  1      100   0.25                  U     lodsw	(%rsi), %ax
 # CHECK-NEXT:  1      100   0.25                  U     lodsl	(%rsi), %eax
@@ -1283,6 +1301,10 @@ xorq (%rax), %rdi
 # CHECK-NEXT:  7      11    2.00                        rcrq	%cl, %rdi
 # CHECK-NEXT:  10     15    2.50           *            rclq	%cl, (%rax)
 # CHECK-NEXT:  11     18    2.25           *            rcrq	%cl, (%rax)
+# CHECK-NEXT:  1      100   0.25                  U     rdmsr
+# CHECK-NEXT:  1      100   0.25                  U     rdpmc
+# CHECK-NEXT:  8      18    2.00                  U     rdtsc
+# CHECK-NEXT:  22     42    5.50                  U     rdtscp
 # CHECK-NEXT:  2      2     1.00                        rolb	%dil
 # CHECK-NEXT:  2      2     1.00                        rorb	%dil
 # CHECK-NEXT:  5      7     1.00    *      *            rolb	(%rax)
@@ -1557,6 +1579,7 @@ xorq (%rax), %rdi
 # CHECK-NEXT:  1      1     0.25                        testq	%rsi, %rdi
 # CHECK-NEXT:  2      6     0.50    *                   testq	%rsi, (%rax)
 # CHECK-NEXT:  1      100   0.25    *             U     ud2
+# CHECK-NEXT:  1      100   0.25                  U     wrmsr
 # CHECK-NEXT:  3      2     0.75                        xaddb	%bl, %cl
 # CHECK-NEXT:  5      7     1.00    *      *            xaddb	%bl, (%rcx)
 # CHECK-NEXT:  3      2     0.75                        xaddw	%bx, %cx
@@ -1622,7 +1645,7 @@ xorq (%rax), %rdi
 
 # CHECK:      Resource pressure per iteration:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]
-# CHECK-NEXT: 60.00   -     564.50 314.50 262.17 262.17 206.00 279.00 620.00 82.67
+# CHECK-NEXT: 60.00   -     573.25 322.75 262.67 262.67 206.00 290.25 629.75 82.67
 
 # CHECK:      Resource pressure by instruction:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    Instructions:
@@ -1846,6 +1869,7 @@ xorq (%rax), %rdi
 # CHECK-NEXT: 10.00   -     1.00    -     0.50   0.50    -      -      -      -     divl	(%rax)
 # CHECK-NEXT:  -      -     10.25  4.75    -      -      -     11.25  5.75    -     divq	%rcx
 # CHECK-NEXT: 10.00   -     1.00    -     0.50   0.50    -      -      -      -     divq	(%rax)
+# CHECK-NEXT:  -      -     0.25   0.25    -      -      -     0.25   0.25    -     enter	$7, $4095
 # CHECK-NEXT: 10.00   -     1.00    -      -      -      -      -      -      -     idivb	%dil
 # CHECK-NEXT:  -      -     2.25   0.25   0.50   0.50    -     4.25   0.25    -     idivb	(%rax)
 # CHECK-NEXT:  -      -     19.50  10.50   -      -      -     12.50  23.50   -     idivw	%si
@@ -1898,7 +1922,10 @@ xorq (%rax), %rdi
 # CHECK-NEXT:  -      -     1.00   0.50   0.83   0.83   1.00   1.50   2.00   0.33   insw	%dx, %es:(%rdi)
 # CHECK-NEXT:  -      -     1.00   0.50   0.83   0.83   1.00   1.50   2.00   0.33   insl	%dx, %es:(%rdi)
 # CHECK-NEXT:  -      -     0.25   0.25    -      -      -     0.25   0.25    -     int	$7
+# CHECK-NEXT:  -      -     0.25   0.25    -      -      -     0.25   0.25    -     invlpg	(%rax)
+# CHECK-NEXT:  -      -     0.25   0.25    -      -      -     0.25   0.25    -     invlpga	%rax, %ecx
 # CHECK-NEXT:  -      -     0.50    -      -      -      -      -     0.50    -     lahf
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -     0.50   0.50    -     leave
 # CHECK-NEXT:  -      -     0.25   0.25    -      -      -     0.25   0.25    -     lodsb	(%rsi), %al
 # CHECK-NEXT:  -      -     0.25   0.25    -      -      -     0.25   0.25    -     lodsw	(%rsi), %ax
 # CHECK-NEXT:  -      -     0.25   0.25    -      -      -     0.25   0.25    -     lodsl	(%rsi), %eax
@@ -2048,6 +2075,10 @@ xorq (%rax), %rdi
 # CHECK-NEXT:  -      -     2.00   2.50    -      -      -     0.50   2.00    -     rcrq	%cl, %rdi
 # CHECK-NEXT:  -      -     2.75   1.75   0.83   0.83    -     0.75   2.75   0.33   rclq	%cl, (%rax)
 # CHECK-NEXT:  -      -     2.50   3.00   0.83   0.83    -     1.00   2.50   0.33   rcrq	%cl, (%rax)
+# CHECK-NEXT:  -      -     0.25   0.25    -      -      -     0.25   0.25    -     rdmsr
+# CHECK-NEXT:  -      -     0.25   0.25    -      -      -     0.25   0.25    -     rdpmc
+# CHECK-NEXT:  -      -     1.75   1.25    -      -      -     2.25   2.75    -     rdtsc
+# CHECK-NEXT:  -      -     5.00   5.00    -      -      -     7.00   5.00    -     rdtscp
 # CHECK-NEXT:  -      -     1.00    -      -      -      -      -     1.00    -     rolb	%dil
 # CHECK-NEXT:  -      -     1.00    -      -      -      -      -     1.00    -     rorb	%dil
 # CHECK-NEXT:  -      -     1.00    -     0.83   0.83   1.00    -     1.00   0.33   rolb	(%rax)
@@ -2322,6 +2353,7 @@ xorq (%rax), %rdi
 # CHECK-NEXT:  -      -     0.25   0.25    -      -      -     0.25   0.25    -     testq	%rsi, %rdi
 # CHECK-NEXT:  -      -     0.25   0.25   0.50   0.50    -     0.25   0.25    -     testq	%rsi, (%rax)
 # CHECK-NEXT:  -      -     0.25   0.25    -      -      -     0.25   0.25    -     ud2
+# CHECK-NEXT:  -      -     0.25   0.25    -      -      -     0.25   0.25    -     wrmsr
 # CHECK-NEXT:  -      -     0.75   0.75    -      -      -     0.75   0.75    -     xaddb	%bl, %cl
 # CHECK-NEXT:  -      -     0.50   0.50   0.83   0.83   1.00   0.50   0.50   0.33   xaddb	%bl, (%rcx)
 # CHECK-NEXT:  -      -     0.75   0.75    -      -      -     0.75   0.75    -     xaddw	%bx, %cx
diff --git a/test/tools/llvm-mca/X86/SkylakeClient/resources-x87.s b/test/tools/llvm-mca/X86/SkylakeClient/resources-x87.s
index 7be9d699573ac66cb1e59cd64dd15dbeef646b1a..6cd4439a25c1f368d376a507a2be896154ca71a8 100644
--- a/test/tools/llvm-mca/X86/SkylakeClient/resources-x87.s
+++ b/test/tools/llvm-mca/X86/SkylakeClient/resources-x87.s
@@ -5,7 +5,7 @@ f2xm1
 
 fabs
 
-fadd %st(0), %st(1)
+fadd %st, %st(1)
 fadd %st(2)
 fadds (%ecx)
 faddl (%ecx)
@@ -21,14 +21,14 @@ fchs
 
 fnclex
 
-fcmovb %st(1), %st(0)
-fcmovbe %st(1), %st(0)
-fcmove %st(1), %st(0)
-fcmovnb %st(1), %st(0)
-fcmovnbe %st(1), %st(0)
-fcmovne %st(1), %st(0)
-fcmovnu %st(1), %st(0)
-fcmovu %st(1), %st(0)
+fcmovb %st(1), %st
+fcmovbe %st(1), %st
+fcmove %st(1), %st
+fcmovnb %st(1), %st
+fcmovnbe %st(1), %st
+fcmovne %st(1), %st
+fcmovnu %st(1), %st
+fcmovu %st(1), %st
 
 fcom %st(1)
 fcom %st(3)
@@ -47,7 +47,7 @@ fcos
 
 fdecstp
 
-fdiv %st(0), %st(1)
+fdiv %st, %st(1)
 fdiv %st(2)
 fdivs (%ecx)
 fdivl (%eax)
@@ -56,7 +56,7 @@ fdivp %st(2)
 fidivs (%ecx)
 fidivl (%eax)
 
-fdivr %st(0), %st(1)
+fdivr %st, %st(1)
 fdivr %st(2)
 fdivrs (%ecx)
 fdivrl (%eax)
@@ -106,7 +106,7 @@ fldln2
 fldpi
 fldz
 
-fmul %st(0), %st(1)
+fmul %st, %st(1)
 fmul %st(2)
 fmuls (%ecx)
 fmull (%eax)
@@ -153,7 +153,7 @@ fnstsw (%eax)
 frstor (%eax)
 fsave (%eax)
 
-fsub %st(0), %st(1)
+fsub %st, %st(1)
 fsub %st(2)
 fsubs (%ecx)
 fsubl (%eax)
@@ -162,7 +162,7 @@ fsubp %st(2)
 fisubs (%ecx)
 fisubl (%eax)
 
-fsubr %st(0), %st(1)
+fsubr %st, %st(1)
 fsubr %st(2)
 fsubrs (%ecx)
 fsubrl (%eax)
@@ -208,26 +208,26 @@ fyl2xp1
 # CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
 # CHECK-NEXT:  1      100   0.25                  U     f2xm1
 # CHECK-NEXT:  1      1     1.00                  U     fabs
-# CHECK-NEXT:  1      3     1.00                  U     fadd	%st(0), %st(1)
-# CHECK-NEXT:  1      3     1.00                  U     fadd	%st(2)
+# CHECK-NEXT:  1      3     1.00                  U     fadd	%st, %st(1)
+# CHECK-NEXT:  1      3     1.00                  U     fadd	%st(2), %st
 # CHECK-NEXT:  2      10    1.00    *             U     fadds	(%ecx)
 # CHECK-NEXT:  2      10    1.00    *             U     faddl	(%ecx)
-# CHECK-NEXT:  1      3     1.00                  U     faddp	%st(1)
-# CHECK-NEXT:  1      3     1.00                  U     faddp	%st(2)
+# CHECK-NEXT:  1      3     1.00                  U     faddp	%st, %st(1)
+# CHECK-NEXT:  1      3     1.00                  U     faddp	%st, %st(2)
 # CHECK-NEXT:  3      13    2.00    *             U     fiadds	(%ecx)
 # CHECK-NEXT:  3      13    2.00    *             U     fiaddl	(%ecx)
 # CHECK-NEXT:  1      100   0.25                  U     fbld	(%ecx)
 # CHECK-NEXT:  2      1     1.00                  U     fbstp	(%eax)
 # CHECK-NEXT:  1      1     1.00                  U     fchs
 # CHECK-NEXT:  4      4     1.00                  U     fnclex
-# CHECK-NEXT:  1      3     1.00                  U     fcmovb	%st(1), %st(0)
-# CHECK-NEXT:  1      3     1.00                  U     fcmovbe	%st(1), %st(0)
-# CHECK-NEXT:  1      3     1.00                  U     fcmove	%st(1), %st(0)
-# CHECK-NEXT:  1      3     1.00                  U     fcmovnb	%st(1), %st(0)
-# CHECK-NEXT:  1      3     1.00                  U     fcmovnbe	%st(1), %st(0)
-# CHECK-NEXT:  1      3     1.00                  U     fcmovne	%st(1), %st(0)
-# CHECK-NEXT:  1      3     1.00                  U     fcmovnu	%st(1), %st(0)
-# CHECK-NEXT:  1      3     1.00                  U     fcmovu	%st(1), %st(0)
+# CHECK-NEXT:  1      3     1.00                  U     fcmovb	%st(1), %st
+# CHECK-NEXT:  1      3     1.00                  U     fcmovbe	%st(1), %st
+# CHECK-NEXT:  1      3     1.00                  U     fcmove	%st(1), %st
+# CHECK-NEXT:  1      3     1.00                  U     fcmovnb	%st(1), %st
+# CHECK-NEXT:  1      3     1.00                  U     fcmovnbe	%st(1), %st
+# CHECK-NEXT:  1      3     1.00                  U     fcmovne	%st(1), %st
+# CHECK-NEXT:  1      3     1.00                  U     fcmovnu	%st(1), %st
+# CHECK-NEXT:  1      3     1.00                  U     fcmovu	%st(1), %st
 # CHECK-NEXT:  1      1     1.00                  U     fcom	%st(1)
 # CHECK-NEXT:  1      1     1.00                  U     fcom	%st(3)
 # CHECK-NEXT:  2      8     1.00                  U     fcoms	(%ecx)
@@ -237,24 +237,24 @@ fyl2xp1
 # CHECK-NEXT:  2      8     1.00                  U     fcomps	(%ecx)
 # CHECK-NEXT:  2      8     1.00                  U     fcompl	(%eax)
 # CHECK-NEXT:  1      100   0.25                  U     fcompp
-# CHECK-NEXT:  1      2     1.00                  U     fcomi	%st(3)
-# CHECK-NEXT:  1      2     1.00                  U     fcompi	%st(3)
+# CHECK-NEXT:  1      2     1.00                  U     fcomi	%st(3), %st
+# CHECK-NEXT:  1      2     1.00                  U     fcompi	%st(3), %st
 # CHECK-NEXT:  1      100   0.25                  U     fcos
 # CHECK-NEXT:  2      2     1.00                  U     fdecstp
-# CHECK-NEXT:  1      15    1.00                  U     fdiv	%st(0), %st(1)
-# CHECK-NEXT:  1      20    1.00                  U     fdiv	%st(2)
+# CHECK-NEXT:  1      15    1.00                  U     fdiv	%st, %st(1)
+# CHECK-NEXT:  1      20    1.00                  U     fdiv	%st(2), %st
 # CHECK-NEXT:  2      22    1.00    *             U     fdivs	(%ecx)
 # CHECK-NEXT:  2      22    1.00    *             U     fdivl	(%eax)
-# CHECK-NEXT:  1      15    1.00                  U     fdivp	%st(1)
-# CHECK-NEXT:  1      15    1.00                  U     fdivp	%st(2)
+# CHECK-NEXT:  1      15    1.00                  U     fdivp	%st, %st(1)
+# CHECK-NEXT:  1      15    1.00                  U     fdivp	%st, %st(2)
 # CHECK-NEXT:  3      25    1.00    *             U     fidivs	(%ecx)
 # CHECK-NEXT:  3      25    1.00    *             U     fidivl	(%eax)
-# CHECK-NEXT:  1      20    1.00                  U     fdivr	%st(0), %st(1)
-# CHECK-NEXT:  1      15    1.00                  U     fdivr	%st(2)
+# CHECK-NEXT:  1      20    1.00                  U     fdivr	%st, %st(1)
+# CHECK-NEXT:  1      15    1.00                  U     fdivr	%st(2), %st
 # CHECK-NEXT:  2      27    1.00    *             U     fdivrs	(%ecx)
 # CHECK-NEXT:  2      27    1.00    *             U     fdivrl	(%eax)
-# CHECK-NEXT:  1      20    1.00                  U     fdivrp	%st(1)
-# CHECK-NEXT:  1      20    1.00                  U     fdivrp	%st(2)
+# CHECK-NEXT:  1      20    1.00                  U     fdivrp	%st, %st(1)
+# CHECK-NEXT:  1      20    1.00                  U     fdivrp	%st, %st(2)
 # CHECK-NEXT:  3      30    1.00    *             U     fidivrs	(%ecx)
 # CHECK-NEXT:  3      30    1.00    *             U     fidivrl	(%eax)
 # CHECK-NEXT:  1      100   0.25                  U     ffree	%st(0)
@@ -288,12 +288,12 @@ fyl2xp1
 # CHECK-NEXT:  2      1     1.00                  U     fldln2
 # CHECK-NEXT:  2      1     1.00                  U     fldpi
 # CHECK-NEXT:  1      1     0.50                  U     fldz
-# CHECK-NEXT:  1      4     1.00                  U     fmul	%st(0), %st(1)
-# CHECK-NEXT:  1      4     1.00                  U     fmul	%st(2)
+# CHECK-NEXT:  1      4     1.00                  U     fmul	%st, %st(1)
+# CHECK-NEXT:  1      4     1.00                  U     fmul	%st(2), %st
 # CHECK-NEXT:  2      11    1.00    *             U     fmuls	(%ecx)
 # CHECK-NEXT:  2      11    1.00    *             U     fmull	(%eax)
-# CHECK-NEXT:  1      4     1.00                  U     fmulp	%st(1)
-# CHECK-NEXT:  1      4     1.00                  U     fmulp	%st(2)
+# CHECK-NEXT:  1      4     1.00                  U     fmulp	%st, %st(1)
+# CHECK-NEXT:  1      4     1.00                  U     fmulp	%st, %st(2)
 # CHECK-NEXT:  3      14    1.00    *             U     fimuls	(%ecx)
 # CHECK-NEXT:  3      14    1.00    *             U     fimull	(%eax)
 # CHECK-NEXT:  1      1     0.50                  U     fnop
@@ -321,20 +321,20 @@ fyl2xp1
 # CHECK-NEXT:  1      100   0.25                  U     frstor	(%eax)
 # CHECK-NEXT:  2      2     0.50                  U     wait
 # CHECK-NEXT:  1      100   0.25                  U     fnsave	(%eax)
-# CHECK-NEXT:  1      3     1.00                  U     fsub	%st(0), %st(1)
-# CHECK-NEXT:  1      3     1.00                  U     fsub	%st(2)
+# CHECK-NEXT:  1      3     1.00                  U     fsub	%st, %st(1)
+# CHECK-NEXT:  1      3     1.00                  U     fsub	%st(2), %st
 # CHECK-NEXT:  2      10    1.00    *             U     fsubs	(%ecx)
 # CHECK-NEXT:  2      10    1.00    *             U     fsubl	(%eax)
-# CHECK-NEXT:  1      3     1.00                  U     fsubp	%st(1)
-# CHECK-NEXT:  1      3     1.00                  U     fsubp	%st(2)
+# CHECK-NEXT:  1      3     1.00                  U     fsubp	%st, %st(1)
+# CHECK-NEXT:  1      3     1.00                  U     fsubp	%st, %st(2)
 # CHECK-NEXT:  3      13    2.00    *             U     fisubs	(%ecx)
 # CHECK-NEXT:  3      13    2.00    *             U     fisubl	(%eax)
-# CHECK-NEXT:  1      3     1.00                  U     fsubr	%st(0), %st(1)
-# CHECK-NEXT:  1      3     1.00                  U     fsubr	%st(2)
+# CHECK-NEXT:  1      3     1.00                  U     fsubr	%st, %st(1)
+# CHECK-NEXT:  1      3     1.00                  U     fsubr	%st(2), %st
 # CHECK-NEXT:  2      10    1.00    *             U     fsubrs	(%ecx)
 # CHECK-NEXT:  2      10    1.00    *             U     fsubrl	(%eax)
-# CHECK-NEXT:  1      3     1.00                  U     fsubrp	%st(1)
-# CHECK-NEXT:  1      3     1.00                  U     fsubrp	%st(2)
+# CHECK-NEXT:  1      3     1.00                  U     fsubrp	%st, %st(1)
+# CHECK-NEXT:  1      3     1.00                  U     fsubrp	%st, %st(2)
 # CHECK-NEXT:  3      13    2.00    *             U     fisubrs	(%ecx)
 # CHECK-NEXT:  3      13    2.00    *             U     fisubrl	(%eax)
 # CHECK-NEXT:  1      2     1.00                  U     ftst
@@ -343,8 +343,8 @@ fyl2xp1
 # CHECK-NEXT:  1      1     1.00                  U     fucomp	%st(1)
 # CHECK-NEXT:  1      1     1.00                  U     fucomp	%st(3)
 # CHECK-NEXT:  1      2     1.00                  U     fucompp
-# CHECK-NEXT:  1      2     1.00                  U     fucomi	%st(3)
-# CHECK-NEXT:  1      2     1.00                  U     fucompi	%st(3)
+# CHECK-NEXT:  1      2     1.00                  U     fucomi	%st(3), %st
+# CHECK-NEXT:  1      2     1.00                  U     fucompi	%st(3), %st
 # CHECK-NEXT:  2      2     0.50                  U     wait
 # CHECK-NEXT:  1      100   0.25                  U     fxam
 # CHECK-NEXT:  15     17    4.00                  U     fxch	%st(1)
@@ -375,26 +375,26 @@ fyl2xp1
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    Instructions:
 # CHECK-NEXT:  -      -     0.25   0.25    -      -      -     0.25   0.25    -     f2xm1
 # CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     fabs
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     fadd	%st(0), %st(1)
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     fadd	%st(2)
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     fadd	%st, %st(1)
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     fadd	%st(2), %st
 # CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     fadds	(%ecx)
 # CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     faddl	(%ecx)
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     faddp	%st(1)
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     faddp	%st(2)
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     faddp	%st, %st(1)
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     faddp	%st, %st(2)
 # CHECK-NEXT:  -      -      -      -     0.50   0.50    -     2.00    -      -     fiadds	(%ecx)
 # CHECK-NEXT:  -      -      -      -     0.50   0.50    -     2.00    -      -     fiaddl	(%ecx)
 # CHECK-NEXT:  -      -     0.25   0.25    -      -      -     0.25   0.25    -     fbld	(%ecx)
 # CHECK-NEXT:  -      -      -      -     0.33   0.33   1.00    -      -     0.33   fbstp	(%eax)
 # CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     fchs
 # CHECK-NEXT:  -      -     1.00   1.00    -      -      -     1.00   1.00    -     fnclex
-# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     fcmovb	%st(1), %st(0)
-# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     fcmovbe	%st(1), %st(0)
-# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     fcmove	%st(1), %st(0)
-# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     fcmovnb	%st(1), %st(0)
-# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     fcmovnbe	%st(1), %st(0)
-# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     fcmovne	%st(1), %st(0)
-# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     fcmovnu	%st(1), %st(0)
-# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     fcmovu	%st(1), %st(0)
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     fcmovb	%st(1), %st
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     fcmovbe	%st(1), %st
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     fcmove	%st(1), %st
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     fcmovnb	%st(1), %st
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     fcmovnbe	%st(1), %st
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     fcmovne	%st(1), %st
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     fcmovnu	%st(1), %st
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     fcmovu	%st(1), %st
 # CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     fcom	%st(1)
 # CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     fcom	%st(3)
 # CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     fcoms	(%ecx)
@@ -404,24 +404,24 @@ fyl2xp1
 # CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     fcomps	(%ecx)
 # CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     fcompl	(%eax)
 # CHECK-NEXT:  -      -     0.25   0.25    -      -      -     0.25   0.25    -     fcompp
-# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     fcomi	%st(3)
-# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     fcompi	%st(3)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     fcomi	%st(3), %st
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     fcompi	%st(3), %st
 # CHECK-NEXT:  -      -     0.25   0.25    -      -      -     0.25   0.25    -     fcos
 # CHECK-NEXT:  -      -     1.00    -      -      -      -     1.00    -      -     fdecstp
-# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     fdiv	%st(0), %st(1)
-# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     fdiv	%st(2)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     fdiv	%st, %st(1)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     fdiv	%st(2), %st
 # CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -      -      -      -     fdivs	(%ecx)
 # CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -      -      -      -     fdivl	(%eax)
-# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     fdivp	%st(1)
-# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     fdivp	%st(2)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     fdivp	%st, %st(1)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     fdivp	%st, %st(2)
 # CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -     1.00    -      -     fidivs	(%ecx)
 # CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -     1.00    -      -     fidivl	(%eax)
-# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     fdivr	%st(0), %st(1)
-# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     fdivr	%st(2)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     fdivr	%st, %st(1)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     fdivr	%st(2), %st
 # CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -      -      -      -     fdivrs	(%ecx)
 # CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -      -      -      -     fdivrl	(%eax)
-# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     fdivrp	%st(1)
-# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     fdivrp	%st(2)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     fdivrp	%st, %st(1)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     fdivrp	%st, %st(2)
 # CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -     1.00    -      -     fidivrs	(%ecx)
 # CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -     1.00    -      -     fidivrl	(%eax)
 # CHECK-NEXT:  -      -     0.25   0.25    -      -      -     0.25   0.25    -     ffree	%st(0)
@@ -455,12 +455,12 @@ fyl2xp1
 # CHECK-NEXT:  -      -     1.00    -      -      -      -     1.00    -      -     fldln2
 # CHECK-NEXT:  -      -     1.00    -      -      -      -     1.00    -      -     fldpi
 # CHECK-NEXT:  -      -     0.50    -      -      -      -     0.50    -      -     fldz
-# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     fmul	%st(0), %st(1)
-# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     fmul	%st(2)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     fmul	%st, %st(1)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     fmul	%st(2), %st
 # CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -      -      -      -     fmuls	(%ecx)
 # CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -      -      -      -     fmull	(%eax)
-# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     fmulp	%st(1)
-# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     fmulp	%st(2)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     fmulp	%st, %st(1)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     fmulp	%st, %st(2)
 # CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -     1.00    -      -     fimuls	(%ecx)
 # CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -     1.00    -      -     fimull	(%eax)
 # CHECK-NEXT:  -      -     0.50    -      -      -      -     0.50    -      -     fnop
@@ -488,20 +488,20 @@ fyl2xp1
 # CHECK-NEXT:  -      -     0.25   0.25    -      -      -     0.25   0.25    -     frstor	(%eax)
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -     0.50   0.50    -     wait
 # CHECK-NEXT:  -      -     0.25   0.25    -      -      -     0.25   0.25    -     fnsave	(%eax)
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     fsub	%st(0), %st(1)
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     fsub	%st(2)
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     fsub	%st, %st(1)
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     fsub	%st(2), %st
 # CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     fsubs	(%ecx)
 # CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     fsubl	(%eax)
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     fsubp	%st(1)
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     fsubp	%st(2)
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     fsubp	%st, %st(1)
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     fsubp	%st, %st(2)
 # CHECK-NEXT:  -      -      -      -     0.50   0.50    -     2.00    -      -     fisubs	(%ecx)
 # CHECK-NEXT:  -      -      -      -     0.50   0.50    -     2.00    -      -     fisubl	(%eax)
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     fsubr	%st(0), %st(1)
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     fsubr	%st(2)
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     fsubr	%st, %st(1)
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     fsubr	%st(2), %st
 # CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     fsubrs	(%ecx)
 # CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     fsubrl	(%eax)
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     fsubrp	%st(1)
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     fsubrp	%st(2)
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     fsubrp	%st, %st(1)
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     fsubrp	%st, %st(2)
 # CHECK-NEXT:  -      -      -      -     0.50   0.50    -     2.00    -      -     fisubrs	(%ecx)
 # CHECK-NEXT:  -      -      -      -     0.50   0.50    -     2.00    -      -     fisubrl	(%eax)
 # CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     ftst
@@ -510,8 +510,8 @@ fyl2xp1
 # CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     fucomp	%st(1)
 # CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     fucomp	%st(3)
 # CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     fucompp
-# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     fucomi	%st(3)
-# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     fucompi	%st(3)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     fucomi	%st(3), %st
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     fucompi	%st(3), %st
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -     0.50   0.50    -     wait
 # CHECK-NEXT:  -      -     0.25   0.25    -      -      -     0.25   0.25    -     fxam
 # CHECK-NEXT:  -      -     4.00   2.00    -      -      -     4.00   5.00    -     fxch	%st(1)
diff --git a/test/tools/llvm-mca/X86/SkylakeServer/resources-avx2.s b/test/tools/llvm-mca/X86/SkylakeServer/resources-avx2.s
index 7842bcac9db097828368453278f1b30b6e571736..823ed8e7e093ca0df8ccb9739c48b570e78056c2 100644
--- a/test/tools/llvm-mca/X86/SkylakeServer/resources-avx2.s
+++ b/test/tools/llvm-mca/X86/SkylakeServer/resources-avx2.s
@@ -149,6 +149,9 @@ vpcmpgtq        (%rax), %ymm1, %ymm2
 vpcmpgtw        %ymm0, %ymm1, %ymm2
 vpcmpgtw        (%rax), %ymm1, %ymm2
 
+vperm2i128      $1, %ymm0, %ymm1, %ymm2
+vperm2i128      $1, (%rax), %ymm1, %ymm2
+
 vpermd          %ymm0, %ymm1, %ymm2
 vpermd          (%rax), %ymm1, %ymm2
 
@@ -555,6 +558,8 @@ vpxor           (%rax), %ymm1, %ymm2
 # CHECK-NEXT:  2      10    1.00    *                   vpcmpgtq	(%rax), %ymm1, %ymm2
 # CHECK-NEXT:  1      1     0.50                        vpcmpgtw	%ymm0, %ymm1, %ymm2
 # CHECK-NEXT:  2      8     0.50    *                   vpcmpgtw	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      3     1.00                        vperm2i128	$1, %ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  2      10    1.00    *                   vperm2i128	$1, (%rax), %ymm1, %ymm2
 # CHECK-NEXT:  1      3     1.00                        vpermd	%ymm0, %ymm1, %ymm2
 # CHECK-NEXT:  2      10    1.00    *                   vpermd	(%rax), %ymm1, %ymm2
 # CHECK-NEXT:  1      3     1.00                        vpermpd	$1, %ymm0, %ymm2
@@ -771,7 +776,7 @@ vpxor           (%rax), %ymm1, %ymm2
 
 # CHECK:      Resource pressure per iteration:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]
-# CHECK-NEXT:  -      -     110.33 89.33  85.17  85.17  1.00   162.33  -     1.67
+# CHECK-NEXT:  -      -     110.33 89.33  85.67  85.67  1.00   164.33  -     1.67
 
 # CHECK:      Resource pressure by instruction:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    Instructions:
@@ -873,6 +878,8 @@ vpxor           (%rax), %ymm1, %ymm2
 # CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpcmpgtq	(%rax), %ymm1, %ymm2
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -     vpcmpgtw	%ymm0, %ymm1, %ymm2
 # CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vpcmpgtw	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vperm2i128	$1, %ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vperm2i128	$1, (%rax), %ymm1, %ymm2
 # CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpermd	%ymm0, %ymm1, %ymm2
 # CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermd	(%rax), %ymm1, %ymm2
 # CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpermpd	$1, %ymm0, %ymm2
diff --git a/test/tools/llvm-mca/X86/SkylakeServer/resources-avx512.s b/test/tools/llvm-mca/X86/SkylakeServer/resources-avx512.s
index a941c0a3a669e87eee6d09e005ce297f9a783f74..957fa6677c1afc02783e8c75589dcac83beae22a 100644
--- a/test/tools/llvm-mca/X86/SkylakeServer/resources-avx512.s
+++ b/test/tools/llvm-mca/X86/SkylakeServer/resources-avx512.s
@@ -3,129 +3,517 @@
 
 vaddpd            %zmm16, %zmm17, %zmm19
 vaddpd            (%rax), %zmm17, %zmm19
+vaddpd            (%rax){1to8}, %zmm17, %zmm19
 vaddpd            %zmm16, %zmm17, %zmm19 {k1}
 vaddpd            (%rax), %zmm17, %zmm19 {k1}
+vaddpd            (%rax){1to8}, %zmm17, %zmm19 {k1}
 vaddpd            %zmm16, %zmm17, %zmm19 {z}{k1}
 vaddpd            (%rax), %zmm17, %zmm19 {z}{k1}
+vaddpd            (%rax){1to8}, %zmm17, %zmm19 {z}{k1}
 
 vaddps            %zmm16, %zmm17, %zmm19
 vaddps            (%rax), %zmm17, %zmm19
+vaddps            (%rax){1to16}, %zmm17, %zmm19
 vaddps            %zmm16, %zmm17, %zmm19 {k1}
 vaddps            (%rax), %zmm17, %zmm19 {k1}
+vaddps            (%rax){1to16}, %zmm17, %zmm19 {k1}
 vaddps            %zmm16, %zmm17, %zmm19 {z}{k1}
 vaddps            (%rax), %zmm17, %zmm19 {z}{k1}
+vaddps            (%rax){1to16}, %zmm17, %zmm19 {z}{k1}
+
+vcvtdq2pd         %ymm16, %zmm19
+vcvtdq2pd         (%rax), %zmm19
+vcvtdq2pd         (%rax){1to8}, %zmm19
+vcvtdq2pd         %ymm16, %zmm19 {k1}
+vcvtdq2pd         (%rax), %zmm19 {k1}
+vcvtdq2pd         (%rax){1to8}, %zmm19 {k1}
+vcvtdq2pd         %ymm16, %zmm19 {z}{k1}
+vcvtdq2pd         (%rax), %zmm19 {z}{k1}
+vcvtdq2pd         (%rax){1to8}, %zmm19 {z}{k1}
+
+vcvtdq2ps         %zmm16, %zmm19
+vcvtdq2ps         (%rax), %zmm19
+vcvtdq2ps         (%rax){1to16}, %zmm19
+vcvtdq2ps         %zmm16, %zmm19 {k1}
+vcvtdq2ps         (%rax), %zmm19 {k1}
+vcvtdq2ps         (%rax){1to16}, %zmm19 {k1}
+vcvtdq2ps         %zmm16, %zmm19 {z}{k1}
+vcvtdq2ps         (%rax), %zmm19 {z}{k1}
+vcvtdq2ps         (%rax){1to16}, %zmm19 {z}{k1}
+
+vcvtps2dq         %zmm16, %zmm19
+vcvtps2dq         (%rax), %zmm19
+vcvtps2dq         (%rax){1to16}, %zmm19
+vcvtps2dq         %zmm16, %zmm19 {k1}
+vcvtps2dq         (%rax), %zmm19 {k1}
+vcvtps2dq         (%rax){1to16}, %zmm19 {k1}
+vcvtps2dq         %zmm16, %zmm19 {z}{k1}
+vcvtps2dq         (%rax), %zmm19 {z}{k1}
+vcvtps2dq         (%rax){1to16}, %zmm19 {z}{k1}
+
+vcvttps2dq        %zmm16, %zmm19
+vcvttps2dq        (%rax), %zmm19
+vcvttps2dq        (%rax){1to16}, %zmm19
+vcvttps2dq        %zmm16, %zmm19 {k1}
+vcvttps2dq        (%rax), %zmm19 {k1}
+vcvttps2dq        (%rax){1to16}, %zmm19 {k1}
+vcvttps2dq        %zmm16, %zmm19 {z}{k1}
+vcvttps2dq        (%rax), %zmm19 {z}{k1}
+vcvttps2dq        (%rax){1to16}, %zmm19 {z}{k1}
 
 vdivpd            %zmm16, %zmm17, %zmm19
 vdivpd            (%rax), %zmm17, %zmm19
+vdivpd            (%rax){1to8}, %zmm17, %zmm19
 vdivpd            %zmm16, %zmm17, %zmm19 {k1}
 vdivpd            (%rax), %zmm17, %zmm19 {k1}
+vdivpd            (%rax){1to8}, %zmm17, %zmm19 {k1}
 vdivpd            %zmm16, %zmm17, %zmm19 {z}{k1}
 vdivpd            (%rax), %zmm17, %zmm19 {z}{k1}
+vdivpd            (%rax){1to8}, %zmm17, %zmm19 {z}{k1}
 
 vdivps            %zmm16, %zmm17, %zmm19
 vdivps            (%rax), %zmm17, %zmm19
+vdivps            (%rax){1to16}, %zmm17, %zmm19
 vdivps            %zmm16, %zmm17, %zmm19 {k1}
 vdivps            (%rax), %zmm17, %zmm19 {k1}
+vdivps            (%rax){1to16}, %zmm17, %zmm19 {k1}
 vdivps            %zmm16, %zmm17, %zmm19 {z}{k1}
 vdivps            (%rax), %zmm17, %zmm19 {z}{k1}
+vdivps            (%rax){1to16}, %zmm17, %zmm19 {z}{k1}
+
+vmaxpd            %zmm16, %zmm17, %zmm19
+vmaxpd            (%rax), %zmm17, %zmm19
+vmaxpd            (%rax){1to8}, %zmm17, %zmm19
+vmaxpd            %zmm16, %zmm17, %zmm19 {k1}
+vmaxpd            (%rax), %zmm17, %zmm19 {k1}
+vmaxpd            (%rax){1to8}, %zmm17, %zmm19 {k1}
+vmaxpd            %zmm16, %zmm17, %zmm19 {z}{k1}
+vmaxpd            (%rax), %zmm17, %zmm19 {z}{k1}
+vmaxpd            (%rax){1to8}, %zmm17, %zmm19 {z}{k1}
+
+vmaxps            %zmm16, %zmm17, %zmm19
+vmaxps            (%rax), %zmm17, %zmm19
+vmaxps            (%rax){1to16}, %zmm17, %zmm19
+vmaxps            %zmm16, %zmm17, %zmm19 {k1}
+vmaxps            (%rax), %zmm17, %zmm19 {k1}
+vmaxps            (%rax){1to16}, %zmm17, %zmm19 {k1}
+vmaxps            %zmm16, %zmm17, %zmm19 {z}{k1}
+vmaxps            (%rax), %zmm17, %zmm19 {z}{k1}
+vmaxps            (%rax){1to16}, %zmm17, %zmm19 {z}{k1}
+
+vminpd            %zmm16, %zmm17, %zmm19
+vminpd            (%rax), %zmm17, %zmm19
+vminpd            (%rax){1to8}, %zmm17, %zmm19
+vminpd            %zmm16, %zmm17, %zmm19 {k1}
+vminpd            (%rax), %zmm17, %zmm19 {k1}
+vminpd            (%rax){1to8}, %zmm17, %zmm19 {k1}
+vminpd            %zmm16, %zmm17, %zmm19 {z}{k1}
+vminpd            (%rax), %zmm17, %zmm19 {z}{k1}
+vminpd            (%rax){1to8}, %zmm17, %zmm19 {z}{k1}
+
+vminps            %zmm16, %zmm17, %zmm19
+vminps            (%rax), %zmm17, %zmm19
+vminps            (%rax){1to16}, %zmm17, %zmm19
+vminps            %zmm16, %zmm17, %zmm19 {k1}
+vminps            (%rax), %zmm17, %zmm19 {k1}
+vminps            (%rax){1to16}, %zmm17, %zmm19 {k1}
+vminps            %zmm16, %zmm17, %zmm19 {z}{k1}
+vminps            (%rax), %zmm17, %zmm19 {z}{k1}
+vminps            (%rax){1to16}, %zmm17, %zmm19 {z}{k1}
 
 vmulpd            %zmm16, %zmm17, %zmm19
 vmulpd            (%rax), %zmm17, %zmm19
+vmulpd            (%rax){1to8}, %zmm17, %zmm19
 vmulpd            %zmm16, %zmm17, %zmm19 {k1}
 vmulpd            (%rax), %zmm17, %zmm19 {k1}
+vmulpd            (%rax){1to8}, %zmm17, %zmm19 {k1}
 vmulpd            %zmm16, %zmm17, %zmm19 {z}{k1}
 vmulpd            (%rax), %zmm17, %zmm19 {z}{k1}
+vmulpd            (%rax){1to8}, %zmm17, %zmm19 {z}{k1}
 
 vmulps            %zmm16, %zmm17, %zmm19
 vmulps            (%rax), %zmm17, %zmm19
+vmulps            (%rax){1to16}, %zmm17, %zmm19
 vmulps            %zmm16, %zmm17, %zmm19 {k1}
 vmulps            (%rax), %zmm17, %zmm19 {k1}
+vmulps            (%rax){1to16}, %zmm17, %zmm19 {k1}
 vmulps            %zmm16, %zmm17, %zmm19 {z}{k1}
 vmulps            (%rax), %zmm17, %zmm19 {z}{k1}
+vmulps            (%rax){1to16}, %zmm17, %zmm19 {z}{k1}
 
 vpabsd            %zmm16, %zmm19
 vpabsd            (%rax), %zmm19
+vpabsd            (%rax){1to16}, %zmm19
 vpabsd            %zmm16, %zmm19 {k1}
 vpabsd            (%rax), %zmm19 {k1}
+vpabsd            (%rax){1to16}, %zmm19 {k1}
 vpabsd            %zmm16, %zmm19 {z}{k1}
 vpabsd            (%rax), %zmm19 {z}{k1}
+vpabsd            (%rax){1to16}, %zmm19 {z}{k1}
 
 vpabsq            %zmm16, %zmm19
 vpabsq            (%rax), %zmm19
+vpabsq            (%rax){1to8}, %zmm19
 vpabsq            %zmm16, %zmm19 {k1}
 vpabsq            (%rax), %zmm19 {k1}
+vpabsq            (%rax){1to8}, %zmm19 {k1}
 vpabsq            %zmm16, %zmm19 {z}{k1}
 vpabsq            (%rax), %zmm19 {z}{k1}
-
-vpaddb            %zmm16, %zmm17, %zmm19
-vpaddb            (%rax), %zmm17, %zmm19
-vpaddb            %zmm16, %zmm17, %zmm19 {k1}
-vpaddb            (%rax), %zmm17, %zmm19 {k1}
-vpaddb            %zmm16, %zmm17, %zmm19 {z}{k1}
-vpaddb            (%rax), %zmm17, %zmm19 {z}{k1}
+vpabsq            (%rax){1to8}, %zmm19 {z}{k1}
 
 vpaddd            %zmm16, %zmm17, %zmm19
 vpaddd            (%rax), %zmm17, %zmm19
+vpaddd            (%rax){1to16}, %zmm17, %zmm19
 vpaddd            %zmm16, %zmm17, %zmm19 {k1}
 vpaddd            (%rax), %zmm17, %zmm19 {k1}
+vpaddd            (%rax){1to16}, %zmm17, %zmm19 {k1}
 vpaddd            %zmm16, %zmm17, %zmm19 {z}{k1}
 vpaddd            (%rax), %zmm17, %zmm19 {z}{k1}
+vpaddd            (%rax){1to16}, %zmm17, %zmm19 {z}{k1}
 
 vpaddq            %zmm16, %zmm17, %zmm19
 vpaddq            (%rax), %zmm17, %zmm19
+vpaddq            (%rax){1to8}, %zmm17, %zmm19
 vpaddq            %zmm16, %zmm17, %zmm19 {k1}
 vpaddq            (%rax), %zmm17, %zmm19 {k1}
+vpaddq            (%rax){1to8}, %zmm17, %zmm19 {k1}
 vpaddq            %zmm16, %zmm17, %zmm19 {z}{k1}
 vpaddq            (%rax), %zmm17, %zmm19 {z}{k1}
+vpaddq            (%rax){1to8}, %zmm17, %zmm19 {z}{k1}
+
+vpmulld           %zmm16, %zmm17, %zmm19
+vpmulld           (%rax), %zmm17, %zmm19
+vpmulld           (%rax){1to16}, %zmm17, %zmm19
+vpmulld           %zmm16, %zmm17, %zmm19 {k1}
+vpmulld           (%rax), %zmm17, %zmm19 {k1}
+vpmulld           (%rax){1to16}, %zmm17, %zmm19 {k1}
+vpmulld           %zmm16, %zmm17, %zmm19 {z}{k1}
+vpmulld           (%rax), %zmm17, %zmm19 {z}{k1}
+vpmulld           (%rax){1to16}, %zmm17, %zmm19 {z}{k1}
+
+vpermd            %zmm16, %zmm17, %zmm19
+vpermd            (%rax), %zmm17, %zmm19
+vpermd            (%rax){1to16}, %zmm17, %zmm19
+vpermd            %zmm16, %zmm17, %zmm19 {k1}
+vpermd            (%rax), %zmm17, %zmm19 {k1}
+vpermd            (%rax){1to16}, %zmm17, %zmm19 {k1}
+vpermd            %zmm16, %zmm17, %zmm19 {z}{k1}
+vpermd            (%rax), %zmm17, %zmm19 {z}{k1}
+vpermd            (%rax){1to16}, %zmm17, %zmm19 {z}{k1}
+
+vpermilpd         $0, %zmm16, %zmm19
+vpermilpd         $0, (%rax), %zmm19
+vpermilpd         $0, (%rax){1to8}, %zmm19
+vpermilpd         $0, %zmm16, %zmm19 {k1}
+vpermilpd         $0, (%rax), %zmm19 {k1}
+vpermilpd         $0, (%rax){1to8}, %zmm19 {k1}
+vpermilpd         $0, %zmm16, %zmm19 {z}{k1}
+vpermilpd         $0, (%rax), %zmm19 {z}{k1}
+vpermilpd         $0, (%rax){1to8}, %zmm19 {z}{k1}
+
+vpermilpd         %zmm16, %zmm17, %zmm19
+vpermilpd         (%rax), %zmm17, %zmm19
+vpermilpd         (%rax){1to8}, %zmm17, %zmm19
+vpermilpd         %zmm16, %zmm17, %zmm19 {k1}
+vpermilpd         (%rax), %zmm17, %zmm19 {k1}
+vpermilpd         (%rax){1to8}, %zmm17, %zmm19 {k1}
+vpermilpd         %zmm16, %zmm17, %zmm19 {z}{k1}
+vpermilpd         (%rax), %zmm17, %zmm19 {z}{k1}
+vpermilpd         (%rax){1to8}, %zmm17, %zmm19 {z}{k1}
+
+vpermilps         $0, %zmm16, %zmm19
+vpermilps         $0, (%rax), %zmm19
+vpermilps         $0, (%rax){1to16}, %zmm19
+vpermilps         $0, %zmm16, %zmm19 {k1}
+vpermilps         $0, (%rax), %zmm19 {k1}
+vpermilps         $0, (%rax){1to16}, %zmm19 {k1}
+vpermilps         $0, %zmm16, %zmm19 {z}{k1}
+vpermilps         $0, (%rax), %zmm19 {z}{k1}
+vpermilps         $0, (%rax){1to16}, %zmm19 {z}{k1}
+
+vpermilps         %zmm16, %zmm17, %zmm19
+vpermilps         (%rax), %zmm17, %zmm19
+vpermilps         (%rax){1to16}, %zmm17, %zmm19
+vpermilps         %zmm16, %zmm17, %zmm19 {k1}
+vpermilps         (%rax), %zmm17, %zmm19 {k1}
+vpermilps         (%rax){1to16}, %zmm17, %zmm19 {k1}
+vpermilps         %zmm16, %zmm17, %zmm19 {z}{k1}
+vpermilps         (%rax), %zmm17, %zmm19 {z}{k1}
+vpermilps         (%rax){1to16}, %zmm17, %zmm19 {z}{k1}
+
+vpermpd           $0, %zmm16, %zmm19
+vpermpd           $0, (%rax), %zmm19
+vpermpd           $0, (%rax){1to8}, %zmm19
+vpermpd           $0, %zmm16, %zmm19 {k1}
+vpermpd           $0, (%rax), %zmm19 {k1}
+vpermpd           $0, (%rax){1to8}, %zmm19 {k1}
+vpermpd           $0, %zmm16, %zmm19 {z}{k1}
+vpermpd           $0, (%rax), %zmm19 {z}{k1}
+vpermpd           $0, (%rax){1to8}, %zmm19 {z}{k1}
+
+vpermpd           %zmm16, %zmm17, %zmm19
+vpermpd           (%rax), %zmm17, %zmm19
+vpermpd           (%rax){1to8}, %zmm17, %zmm19
+vpermpd           %zmm16, %zmm17, %zmm19 {k1}
+vpermpd           (%rax), %zmm17, %zmm19 {k1}
+vpermpd           (%rax){1to8}, %zmm17, %zmm19 {k1}
+vpermpd           %zmm16, %zmm17, %zmm19 {z}{k1}
+vpermpd           (%rax), %zmm17, %zmm19 {z}{k1}
+vpermpd           (%rax){1to8}, %zmm17, %zmm19 {z}{k1}
+
+vpermps           %zmm16, %zmm17, %zmm19
+vpermps           (%rax), %zmm17, %zmm19
+vpermps           (%rax){1to16}, %zmm17, %zmm19
+vpermps           %zmm16, %zmm17, %zmm19 {k1}
+vpermps           (%rax), %zmm17, %zmm19 {k1}
+vpermps           (%rax){1to16}, %zmm17, %zmm19 {k1}
+vpermps           %zmm16, %zmm17, %zmm19 {z}{k1}
+vpermps           (%rax), %zmm17, %zmm19 {z}{k1}
+vpermps           (%rax){1to16}, %zmm17, %zmm19 {z}{k1}
 
-vpaddw            %zmm16, %zmm17, %zmm19
-vpaddw            (%rax), %zmm17, %zmm19
-vpaddw            %zmm16, %zmm17, %zmm19 {k1}
-vpaddw            (%rax), %zmm17, %zmm19 {k1}
-vpaddw            %zmm16, %zmm17, %zmm19 {z}{k1}
-vpaddw            (%rax), %zmm17, %zmm19 {z}{k1}
+vpermq            $0, %zmm16, %zmm19
+vpermq            $0, (%rax), %zmm19
+vpermq            $0, (%rax){1to8}, %zmm19
+vpermq            $0, %zmm16, %zmm19 {k1}
+vpermq            $0, (%rax), %zmm19 {k1}
+vpermq            $0, (%rax){1to8}, %zmm19 {k1}
+vpermq            $0, %zmm16, %zmm19 {z}{k1}
+vpermq            $0, (%rax), %zmm19 {z}{k1}
+vpermq            $0, (%rax){1to8}, %zmm19 {z}{k1}
 
-vpsubb            %zmm16, %zmm17, %zmm19
-vpsubb            (%rax), %zmm17, %zmm19
-vpsubb            %zmm16, %zmm17, %zmm19 {k1}
-vpsubb            (%rax), %zmm17, %zmm19 {k1}
-vpsubb            %zmm16, %zmm17, %zmm19 {z}{k1}
-vpsubb            (%rax), %zmm17, %zmm19 {z}{k1}
+vpermq            %zmm16, %zmm17, %zmm19
+vpermq            (%rax), %zmm17, %zmm19
+vpermq            (%rax){1to8}, %zmm17, %zmm19
+vpermq            %zmm16, %zmm17, %zmm19 {k1}
+vpermq            (%rax), %zmm17, %zmm19 {k1}
+vpermq            (%rax){1to8}, %zmm17, %zmm19 {k1}
+vpermq            %zmm16, %zmm17, %zmm19 {z}{k1}
+vpermq            (%rax), %zmm17, %zmm19 {z}{k1}
+vpermq            (%rax){1to8}, %zmm17, %zmm19 {z}{k1}
+
+vpshufd           $0, %zmm16, %zmm19
+vpshufd           $0, (%rax), %zmm19
+vpshufd           $0, (%rax){1to16}, %zmm19
+vpshufd           $0, %zmm16, %zmm19 {k1}
+vpshufd           $0, (%rax), %zmm19 {k1}
+vpshufd           $0, (%rax){1to16}, %zmm19 {k1}
+vpshufd           $0, %zmm16, %zmm19 {z}{k1}
+vpshufd           $0, (%rax), %zmm19 {z}{k1}
+vpshufd           $0, (%rax){1to16}, %zmm19 {z}{k1}
 
 vpsubd            %zmm16, %zmm17, %zmm19
 vpsubd            (%rax), %zmm17, %zmm19
+vpsubd            (%rax){1to16}, %zmm17, %zmm19
 vpsubd            %zmm16, %zmm17, %zmm19 {k1}
 vpsubd            (%rax), %zmm17, %zmm19 {k1}
+vpsubd            (%rax){1to16}, %zmm17, %zmm19 {k1}
 vpsubd            %zmm16, %zmm17, %zmm19 {z}{k1}
 vpsubd            (%rax), %zmm17, %zmm19 {z}{k1}
+vpsubd            (%rax){1to16}, %zmm17, %zmm19 {z}{k1}
 
 vpsubq            %zmm16, %zmm17, %zmm19
 vpsubq            (%rax), %zmm17, %zmm19
+vpsubq            (%rax){1to8}, %zmm17, %zmm19
 vpsubq            %zmm16, %zmm17, %zmm19 {k1}
 vpsubq            (%rax), %zmm17, %zmm19 {k1}
+vpsubq            (%rax){1to8}, %zmm17, %zmm19 {k1}
 vpsubq            %zmm16, %zmm17, %zmm19 {z}{k1}
 vpsubq            (%rax), %zmm17, %zmm19 {z}{k1}
+vpsubq            (%rax){1to8}, %zmm17, %zmm19 {z}{k1}
+
+vpunpckhdq        %zmm16, %zmm17, %zmm19
+vpunpckhdq        (%rax), %zmm17, %zmm19
+vpunpckhdq        (%rax){1to16}, %zmm17, %zmm19
+vpunpckhdq        %zmm16, %zmm17, %zmm19 {k1}
+vpunpckhdq        (%rax), %zmm17, %zmm19 {k1}
+vpunpckhdq        (%rax){1to16}, %zmm17, %zmm19 {k1}
+vpunpckhdq        %zmm16, %zmm17, %zmm19 {z}{k1}
+vpunpckhdq        (%rax), %zmm17, %zmm19 {z}{k1}
+vpunpckhdq        (%rax){1to16}, %zmm17, %zmm19 {z}{k1}
+
+vpunpckhqdq       %zmm16, %zmm17, %zmm19
+vpunpckhqdq       (%rax), %zmm17, %zmm19
+vpunpckhqdq       (%rax){1to8}, %zmm17, %zmm19
+vpunpckhqdq       %zmm16, %zmm17, %zmm19 {k1}
+vpunpckhqdq       (%rax), %zmm17, %zmm19 {k1}
+vpunpckhqdq       (%rax){1to8}, %zmm17, %zmm19 {k1}
+vpunpckhqdq       %zmm16, %zmm17, %zmm19 {z}{k1}
+vpunpckhqdq       (%rax), %zmm17, %zmm19 {z}{k1}
+vpunpckhqdq       (%rax){1to8}, %zmm17, %zmm19 {z}{k1}
+
+vpunpckldq        %zmm16, %zmm17, %zmm19
+vpunpckldq        (%rax), %zmm17, %zmm19
+vpunpckldq        (%rax){1to16}, %zmm17, %zmm19
+vpunpckldq        %zmm16, %zmm17, %zmm19 {k1}
+vpunpckldq        (%rax), %zmm17, %zmm19 {k1}
+vpunpckldq        (%rax){1to16}, %zmm17, %zmm19 {k1}
+vpunpckldq        %zmm16, %zmm17, %zmm19 {z}{k1}
+vpunpckldq        (%rax), %zmm17, %zmm19 {z}{k1}
+vpunpckldq        (%rax){1to16}, %zmm17, %zmm19 {z}{k1}
+
+vpunpcklqdq       %zmm16, %zmm17, %zmm19
+vpunpcklqdq       (%rax), %zmm17, %zmm19
+vpunpcklqdq       (%rax){1to8}, %zmm17, %zmm19
+vpunpcklqdq       %zmm16, %zmm17, %zmm19 {k1}
+vpunpcklqdq       (%rax), %zmm17, %zmm19 {k1}
+vpunpcklqdq       (%rax){1to8}, %zmm17, %zmm19 {k1}
+vpunpcklqdq       %zmm16, %zmm17, %zmm19 {z}{k1}
+vpunpcklqdq       (%rax), %zmm17, %zmm19 {z}{k1}
+vpunpcklqdq       (%rax){1to8}, %zmm17, %zmm19 {z}{k1}
+
+vshuff32x4        $0, %zmm16, %zmm17, %zmm19
+vshuff32x4        $0, (%rax), %zmm17, %zmm19
+vshuff32x4        $0, (%rax){1to16}, %zmm17, %zmm19
+vshuff32x4        $0, %zmm16, %zmm17, %zmm19 {k1}
+vshuff32x4        $0, (%rax), %zmm17, %zmm19 {k1}
+vshuff32x4        $0, (%rax){1to16}, %zmm17, %zmm19 {k1}
+vshuff32x4        $0, %zmm16, %zmm17, %zmm19 {z}{k1}
+vshuff32x4        $0, (%rax), %zmm17, %zmm19 {z}{k1}
+vshuff32x4        $0, (%rax){1to16}, %zmm17, %zmm19 {z}{k1}
+
+vshuff64x2        $0, %zmm16, %zmm17, %zmm19
+vshuff64x2        $0, (%rax), %zmm17, %zmm19
+vshuff64x2        $0, (%rax){1to8}, %zmm17, %zmm19
+vshuff64x2        $0, %zmm16, %zmm17, %zmm19 {k1}
+vshuff64x2        $0, (%rax), %zmm17, %zmm19 {k1}
+vshuff64x2        $0, (%rax){1to8}, %zmm17, %zmm19 {k1}
+vshuff64x2        $0, %zmm16, %zmm17, %zmm19 {z}{k1}
+vshuff64x2        $0, (%rax), %zmm17, %zmm19 {z}{k1}
+vshuff64x2        $0, (%rax){1to8}, %zmm17, %zmm19 {z}{k1}
+
+vshufi32x4        $0, %zmm16, %zmm17, %zmm19
+vshufi32x4        $0, (%rax), %zmm17, %zmm19
+vshufi32x4        $0, (%rax){1to16}, %zmm17, %zmm19
+vshufi32x4        $0, %zmm16, %zmm17, %zmm19 {k1}
+vshufi32x4        $0, (%rax), %zmm17, %zmm19 {k1}
+vshufi32x4        $0, (%rax){1to16}, %zmm17, %zmm19 {k1}
+vshufi32x4        $0, %zmm16, %zmm17, %zmm19 {z}{k1}
+vshufi32x4        $0, (%rax), %zmm17, %zmm19 {z}{k1}
+vshufi32x4        $0, (%rax){1to16}, %zmm17, %zmm19 {z}{k1}
+
+vshufi64x2        $0, %zmm16, %zmm17, %zmm19
+vshufi64x2        $0, (%rax), %zmm17, %zmm19
+vshufi64x2        $0, (%rax){1to8}, %zmm17, %zmm19
+vshufi64x2        $0, %zmm16, %zmm17, %zmm19 {k1}
+vshufi64x2        $0, (%rax), %zmm17, %zmm19 {k1}
+vshufi64x2        $0, (%rax){1to8}, %zmm17, %zmm19 {k1}
+vshufi64x2        $0, %zmm16, %zmm17, %zmm19 {z}{k1}
+vshufi64x2        $0, (%rax), %zmm17, %zmm19 {z}{k1}
+vshufi64x2        $0, (%rax){1to8}, %zmm17, %zmm19 {z}{k1}
+
+vsqrtpd           %zmm16, %zmm19
+vsqrtpd           (%rax), %zmm19
+vsqrtpd           (%rax){1to8}, %zmm19
+vsqrtpd           %zmm16, %zmm19 {k1}
+vsqrtpd           (%rax), %zmm19 {k1}
+vsqrtpd           (%rax){1to8}, %zmm19 {k1}
+vsqrtpd           %zmm16, %zmm19 {z}{k1}
+vsqrtpd           (%rax), %zmm19 {z}{k1}
+vsqrtpd           (%rax){1to8}, %zmm19 {z}{k1}
+
+vsqrtps           %zmm16, %zmm19
+vsqrtps           (%rax), %zmm19
+vsqrtps           (%rax){1to16}, %zmm19
+vsqrtps           %zmm16, %zmm19 {k1}
+vsqrtps           (%rax), %zmm19 {k1}
+vsqrtps           (%rax){1to16}, %zmm19 {k1}
+vsqrtps           %zmm16, %zmm19 {z}{k1}
+vsqrtps           (%rax), %zmm19 {z}{k1}
+vsqrtps           (%rax){1to16}, %zmm19 {z}{k1}
+
+vsqrtsd           %xmm16, %xmm17, %xmm19
+vsqrtsd           (%rax), %xmm17, %xmm19
+vsqrtsd           %xmm16, %xmm17, %xmm19 {k1}
+vsqrtsd           (%rax), %xmm17, %xmm19 {k1}
+vsqrtsd           %xmm16, %xmm17, %xmm19 {z}{k1}
+vsqrtsd           (%rax), %xmm17, %xmm19 {z}{k1}
+
+vsqrtss           %xmm16, %xmm17, %xmm19
+vsqrtss           (%rax), %xmm17, %xmm19
+vsqrtss           %xmm16, %xmm17, %xmm19 {k1}
+vsqrtss           (%rax), %xmm17, %xmm19 {k1}
+vsqrtss           %xmm16, %xmm17, %xmm19 {z}{k1}
+vsqrtss           (%rax), %xmm17, %xmm19 {z}{k1}
+
+vpsubd            %zmm16, %zmm17, %zmm19
+vpsubd            (%rax), %zmm17, %zmm19
+vpsubd            (%rax){1to16}, %zmm17, %zmm19
+vpsubd            %zmm16, %zmm17, %zmm19 {k1}
+vpsubd            (%rax), %zmm17, %zmm19 {k1}
+vpsubd            (%rax){1to16}, %zmm17, %zmm19 {k1}
+vpsubd            %zmm16, %zmm17, %zmm19 {z}{k1}
+vpsubd            (%rax), %zmm17, %zmm19 {z}{k1}
+vpsubd            (%rax){1to16}, %zmm17, %zmm19 {z}{k1}
 
-vpsubw            %zmm16, %zmm17, %zmm19
-vpsubw            (%rax), %zmm17, %zmm19
-vpsubw            %zmm16, %zmm17, %zmm19 {k1}
-vpsubw            (%rax), %zmm17, %zmm19 {k1}
-vpsubw            %zmm16, %zmm17, %zmm19 {z}{k1}
-vpsubw            (%rax), %zmm17, %zmm19 {z}{k1}
+vpsubq            %zmm16, %zmm17, %zmm19
+vpsubq            (%rax), %zmm17, %zmm19
+vpsubq            (%rax){1to8}, %zmm17, %zmm19
+vpsubq            %zmm16, %zmm17, %zmm19 {k1}
+vpsubq            (%rax), %zmm17, %zmm19 {k1}
+vpsubq            (%rax){1to8}, %zmm17, %zmm19 {k1}
+vpsubq            %zmm16, %zmm17, %zmm19 {z}{k1}
+vpsubq            (%rax), %zmm17, %zmm19 {z}{k1}
+vpsubq            (%rax){1to8}, %zmm17, %zmm19 {z}{k1}
 
 vsubpd            %zmm16, %zmm17, %zmm19
 vsubpd            (%rax), %zmm17, %zmm19
+vsubpd            (%rax){1to8}, %zmm17, %zmm19
 vsubpd            %zmm16, %zmm17, %zmm19 {k1}
 vsubpd            (%rax), %zmm17, %zmm19 {k1}
+vsubpd            (%rax){1to8}, %zmm17, %zmm19 {k1}
 vsubpd            %zmm16, %zmm17, %zmm19 {z}{k1}
 vsubpd            (%rax), %zmm17, %zmm19 {z}{k1}
+vsubpd            (%rax){1to8}, %zmm17, %zmm19 {z}{k1}
 
 vsubps            %zmm16, %zmm17, %zmm19
 vsubps            (%rax), %zmm17, %zmm19
+vsubps            (%rax){1to16}, %zmm17, %zmm19
 vsubps            %zmm16, %zmm17, %zmm19 {k1}
 vsubps            (%rax), %zmm17, %zmm19 {k1}
+vsubps            (%rax){1to16}, %zmm17, %zmm19 {k1}
 vsubps            %zmm16, %zmm17, %zmm19 {z}{k1}
 vsubps            (%rax), %zmm17, %zmm19 {z}{k1}
+vsubps            (%rax){1to16}, %zmm17, %zmm19 {z}{k1}
+
+vunpckhpd         %zmm16, %zmm17, %zmm19
+vunpckhpd         (%rax), %zmm17, %zmm19
+vunpckhpd         (%rax){1to8}, %zmm17, %zmm19
+vunpckhpd         %zmm16, %zmm17, %zmm19 {k1}
+vunpckhpd         (%rax), %zmm17, %zmm19 {k1}
+vunpckhpd         (%rax){1to8}, %zmm17, %zmm19 {k1}
+vunpckhpd         %zmm16, %zmm17, %zmm19 {z}{k1}
+vunpckhpd         (%rax), %zmm17, %zmm19 {z}{k1}
+vunpckhpd         (%rax){1to8}, %zmm17, %zmm19 {z}{k1}
+
+vunpckhps         %zmm16, %zmm17, %zmm19
+vunpckhps         (%rax), %zmm17, %zmm19
+vunpckhps         (%rax){1to16}, %zmm17, %zmm19
+vunpckhps         %zmm16, %zmm17, %zmm19 {k1}
+vunpckhps         (%rax), %zmm17, %zmm19 {k1}
+vunpckhps         (%rax){1to16}, %zmm17, %zmm19 {k1}
+vunpckhps         %zmm16, %zmm17, %zmm19 {z}{k1}
+vunpckhps         (%rax), %zmm17, %zmm19 {z}{k1}
+vunpckhps         (%rax){1to16}, %zmm17, %zmm19 {z}{k1}
+
+vunpcklpd         %zmm16, %zmm17, %zmm19
+vunpcklpd         (%rax), %zmm17, %zmm19
+vunpcklpd         (%rax){1to8}, %zmm17, %zmm19
+vunpcklpd         %zmm16, %zmm17, %zmm19 {k1}
+vunpcklpd         (%rax), %zmm17, %zmm19 {k1}
+vunpcklpd         (%rax){1to8}, %zmm17, %zmm19 {k1}
+vunpcklpd         %zmm16, %zmm17, %zmm19 {z}{k1}
+vunpcklpd         (%rax), %zmm17, %zmm19 {z}{k1}
+vunpcklpd         (%rax){1to8}, %zmm17, %zmm19 {z}{k1}
+
+vunpcklps         %zmm16, %zmm17, %zmm19
+vunpcklps         (%rax), %zmm17, %zmm19
+vunpcklps         (%rax){1to16}, %zmm17, %zmm19
+vunpcklps         %zmm16, %zmm17, %zmm19 {k1}
+vunpcklps         (%rax), %zmm17, %zmm19 {k1}
+vunpcklps         (%rax){1to16}, %zmm17, %zmm19 {k1}
+vunpcklps         %zmm16, %zmm17, %zmm19 {z}{k1}
+vunpcklps         (%rax), %zmm17, %zmm19 {z}{k1}
+vunpcklps         (%rax){1to16}, %zmm17, %zmm19 {z}{k1}
 
 # CHECK:      Instruction Info:
 # CHECK-NEXT: [1]: #uOps
@@ -138,112 +526,466 @@ vsubps            (%rax), %zmm17, %zmm19 {z}{k1}
 # CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
 # CHECK-NEXT:  1      4     0.50                        vaddpd	%zmm16, %zmm17, %zmm19
 # CHECK-NEXT:  2      11    0.50    *                   vaddpd	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  2      11    0.50    *                   vaddpd	(%rax){1to8}, %zmm17, %zmm19
 # CHECK-NEXT:  1      4     0.50                        vaddpd	%zmm16, %zmm17, %zmm19 {%k1}
 # CHECK-NEXT:  2      11    0.50    *                   vaddpd	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      11    0.50    *                   vaddpd	(%rax){1to8}, %zmm17, %zmm19 {%k1}
 # CHECK-NEXT:  1      4     0.50                        vaddpd	%zmm16, %zmm17, %zmm19 {%k1} {z}
 # CHECK-NEXT:  2      11    0.50    *                   vaddpd	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      11    0.50    *                   vaddpd	(%rax){1to8}, %zmm17, %zmm19 {%k1} {z}
 # CHECK-NEXT:  1      4     0.50                        vaddps	%zmm16, %zmm17, %zmm19
 # CHECK-NEXT:  2      11    0.50    *                   vaddps	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  2      11    0.50    *                   vaddps	(%rax){1to16}, %zmm17, %zmm19
 # CHECK-NEXT:  1      4     0.50                        vaddps	%zmm16, %zmm17, %zmm19 {%k1}
 # CHECK-NEXT:  2      11    0.50    *                   vaddps	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      11    0.50    *                   vaddps	(%rax){1to16}, %zmm17, %zmm19 {%k1}
 # CHECK-NEXT:  1      4     0.50                        vaddps	%zmm16, %zmm17, %zmm19 {%k1} {z}
 # CHECK-NEXT:  2      11    0.50    *                   vaddps	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      11    0.50    *                   vaddps	(%rax){1to16}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      7     1.00                        vcvtdq2pd	%ymm16, %zmm19
+# CHECK-NEXT:  2      11    0.50    *                   vcvtdq2pd	(%rax), %zmm19
+# CHECK-NEXT:  2      11    0.50    *                   vcvtdq2pd	(%rax){1to8}, %zmm19
+# CHECK-NEXT:  1      4     0.50                        vcvtdq2pd	%ymm16, %zmm19 {%k1}
+# CHECK-NEXT:  2      11    0.50    *                   vcvtdq2pd	(%rax), %zmm19 {%k1}
+# CHECK-NEXT:  2      11    0.50    *                   vcvtdq2pd	(%rax){1to8}, %zmm19 {%k1}
+# CHECK-NEXT:  1      4     0.50                        vcvtdq2pd	%ymm16, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      11    0.50    *                   vcvtdq2pd	(%rax), %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      11    0.50    *                   vcvtdq2pd	(%rax){1to8}, %zmm19 {%k1} {z}
+# CHECK-NEXT:  1      4     0.50                        vcvtdq2ps	%zmm16, %zmm19
+# CHECK-NEXT:  2      11    0.50    *                   vcvtdq2ps	(%rax), %zmm19
+# CHECK-NEXT:  2      11    0.50    *                   vcvtdq2ps	(%rax){1to16}, %zmm19
+# CHECK-NEXT:  1      4     0.50                        vcvtdq2ps	%zmm16, %zmm19 {%k1}
+# CHECK-NEXT:  2      11    0.50    *                   vcvtdq2ps	(%rax), %zmm19 {%k1}
+# CHECK-NEXT:  2      11    0.50    *                   vcvtdq2ps	(%rax){1to16}, %zmm19 {%k1}
+# CHECK-NEXT:  1      4     0.50                        vcvtdq2ps	%zmm16, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      11    0.50    *                   vcvtdq2ps	(%rax), %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      11    0.50    *                   vcvtdq2ps	(%rax){1to16}, %zmm19 {%k1} {z}
+# CHECK-NEXT:  1      4     0.50                        vcvtps2dq	%zmm16, %zmm19
+# CHECK-NEXT:  2      11    0.50    *                   vcvtps2dq	(%rax), %zmm19
+# CHECK-NEXT:  2      11    0.50    *                   vcvtps2dq	(%rax){1to16}, %zmm19
+# CHECK-NEXT:  1      3     0.50                        vcvtps2dq	%zmm16, %zmm19 {%k1}
+# CHECK-NEXT:  2      11    0.50    *                   vcvtps2dq	(%rax), %zmm19 {%k1}
+# CHECK-NEXT:  2      11    0.50    *                   vcvtps2dq	(%rax){1to16}, %zmm19 {%k1}
+# CHECK-NEXT:  1      3     0.50                        vcvtps2dq	%zmm16, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      11    0.50    *                   vcvtps2dq	(%rax), %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      11    0.50    *                   vcvtps2dq	(%rax){1to16}, %zmm19 {%k1} {z}
+# CHECK-NEXT:  1      4     0.50                        vcvttps2dq	%zmm16, %zmm19
+# CHECK-NEXT:  2      11    0.50    *                   vcvttps2dq	(%rax), %zmm19
+# CHECK-NEXT:  2      11    0.50    *                   vcvttps2dq	(%rax){1to16}, %zmm19
+# CHECK-NEXT:  1      3     0.50                        vcvttps2dq	%zmm16, %zmm19 {%k1}
+# CHECK-NEXT:  2      11    0.50    *                   vcvttps2dq	(%rax), %zmm19 {%k1}
+# CHECK-NEXT:  2      11    0.50    *                   vcvttps2dq	(%rax){1to16}, %zmm19 {%k1}
+# CHECK-NEXT:  1      3     0.50                        vcvttps2dq	%zmm16, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      11    0.50    *                   vcvttps2dq	(%rax), %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      11    0.50    *                   vcvttps2dq	(%rax){1to16}, %zmm19 {%k1} {z}
 # CHECK-NEXT:  3      23    16.00                       vdivpd	%zmm16, %zmm17, %zmm19
 # CHECK-NEXT:  4      30    16.00   *                   vdivpd	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  4      30    16.00   *                   vdivpd	(%rax){1to8}, %zmm17, %zmm19
 # CHECK-NEXT:  3      23    16.00                       vdivpd	%zmm16, %zmm17, %zmm19 {%k1}
 # CHECK-NEXT:  4      30    16.00   *                   vdivpd	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  4      30    16.00   *                   vdivpd	(%rax){1to8}, %zmm17, %zmm19 {%k1}
 # CHECK-NEXT:  3      23    16.00                       vdivpd	%zmm16, %zmm17, %zmm19 {%k1} {z}
 # CHECK-NEXT:  4      30    16.00   *                   vdivpd	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  4      30    16.00   *                   vdivpd	(%rax){1to8}, %zmm17, %zmm19 {%k1} {z}
 # CHECK-NEXT:  3      18    10.00                       vdivps	%zmm16, %zmm17, %zmm19
 # CHECK-NEXT:  4      25    10.00   *                   vdivps	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  4      25    10.00   *                   vdivps	(%rax){1to16}, %zmm17, %zmm19
 # CHECK-NEXT:  3      18    10.00                       vdivps	%zmm16, %zmm17, %zmm19 {%k1}
 # CHECK-NEXT:  4      25    10.00   *                   vdivps	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  4      25    10.00   *                   vdivps	(%rax){1to16}, %zmm17, %zmm19 {%k1}
 # CHECK-NEXT:  3      18    10.00                       vdivps	%zmm16, %zmm17, %zmm19 {%k1} {z}
 # CHECK-NEXT:  4      25    10.00   *                   vdivps	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  4      25    10.00   *                   vdivps	(%rax){1to16}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  1      4     0.50                        vmaxpd	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  2      11    0.50    *                   vmaxpd	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  2      11    0.50    *                   vmaxpd	(%rax){1to8}, %zmm17, %zmm19
+# CHECK-NEXT:  1      4     0.50                        vmaxpd	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      11    0.50    *                   vmaxpd	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      11    0.50    *                   vmaxpd	(%rax){1to8}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  1      4     0.50                        vmaxpd	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      11    0.50    *                   vmaxpd	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      11    0.50    *                   vmaxpd	(%rax){1to8}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  1      4     0.50                        vmaxps	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  2      11    0.50    *                   vmaxps	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  2      11    0.50    *                   vmaxps	(%rax){1to16}, %zmm17, %zmm19
+# CHECK-NEXT:  1      4     0.50                        vmaxps	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      11    0.50    *                   vmaxps	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      11    0.50    *                   vmaxps	(%rax){1to16}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  1      4     0.50                        vmaxps	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      11    0.50    *                   vmaxps	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      11    0.50    *                   vmaxps	(%rax){1to16}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  1      4     0.50                        vminpd	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  2      11    0.50    *                   vminpd	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  2      11    0.50    *                   vminpd	(%rax){1to8}, %zmm17, %zmm19
+# CHECK-NEXT:  1      4     0.50                        vminpd	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      11    0.50    *                   vminpd	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      11    0.50    *                   vminpd	(%rax){1to8}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  1      4     0.50                        vminpd	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      11    0.50    *                   vminpd	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      11    0.50    *                   vminpd	(%rax){1to8}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  1      4     0.50                        vminps	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  2      11    0.50    *                   vminps	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  2      11    0.50    *                   vminps	(%rax){1to16}, %zmm17, %zmm19
+# CHECK-NEXT:  1      4     0.50                        vminps	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      11    0.50    *                   vminps	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      11    0.50    *                   vminps	(%rax){1to16}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  1      4     0.50                        vminps	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      11    0.50    *                   vminps	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      11    0.50    *                   vminps	(%rax){1to16}, %zmm17, %zmm19 {%k1} {z}
 # CHECK-NEXT:  1      4     0.50                        vmulpd	%zmm16, %zmm17, %zmm19
 # CHECK-NEXT:  2      11    0.50    *                   vmulpd	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  2      11    0.50    *                   vmulpd	(%rax){1to8}, %zmm17, %zmm19
 # CHECK-NEXT:  1      4     0.50                        vmulpd	%zmm16, %zmm17, %zmm19 {%k1}
 # CHECK-NEXT:  2      11    0.50    *                   vmulpd	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      11    0.50    *                   vmulpd	(%rax){1to8}, %zmm17, %zmm19 {%k1}
 # CHECK-NEXT:  1      4     0.50                        vmulpd	%zmm16, %zmm17, %zmm19 {%k1} {z}
 # CHECK-NEXT:  2      11    0.50    *                   vmulpd	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      11    0.50    *                   vmulpd	(%rax){1to8}, %zmm17, %zmm19 {%k1} {z}
 # CHECK-NEXT:  1      4     0.50                        vmulps	%zmm16, %zmm17, %zmm19
 # CHECK-NEXT:  2      11    0.50    *                   vmulps	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  2      11    0.50    *                   vmulps	(%rax){1to16}, %zmm17, %zmm19
 # CHECK-NEXT:  1      4     0.50                        vmulps	%zmm16, %zmm17, %zmm19 {%k1}
 # CHECK-NEXT:  2      11    0.50    *                   vmulps	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      11    0.50    *                   vmulps	(%rax){1to16}, %zmm17, %zmm19 {%k1}
 # CHECK-NEXT:  1      4     0.50                        vmulps	%zmm16, %zmm17, %zmm19 {%k1} {z}
 # CHECK-NEXT:  2      11    0.50    *                   vmulps	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      11    0.50    *                   vmulps	(%rax){1to16}, %zmm17, %zmm19 {%k1} {z}
 # CHECK-NEXT:  1      1     1.00                        vpabsd	%zmm16, %zmm19
 # CHECK-NEXT:  2      8     1.00    *                   vpabsd	(%rax), %zmm19
+# CHECK-NEXT:  2      8     1.00    *                   vpabsd	(%rax){1to16}, %zmm19
 # CHECK-NEXT:  1      1     1.00                        vpabsd	%zmm16, %zmm19 {%k1}
 # CHECK-NEXT:  2      8     1.00    *                   vpabsd	(%rax), %zmm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vpabsd	(%rax){1to16}, %zmm19 {%k1}
 # CHECK-NEXT:  1      1     1.00                        vpabsd	%zmm16, %zmm19 {%k1} {z}
 # CHECK-NEXT:  2      8     1.00    *                   vpabsd	(%rax), %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vpabsd	(%rax){1to16}, %zmm19 {%k1} {z}
 # CHECK-NEXT:  1      1     1.00                        vpabsq	%zmm16, %zmm19
 # CHECK-NEXT:  2      8     1.00    *                   vpabsq	(%rax), %zmm19
+# CHECK-NEXT:  2      8     1.00    *                   vpabsq	(%rax){1to8}, %zmm19
 # CHECK-NEXT:  1      1     1.00                        vpabsq	%zmm16, %zmm19 {%k1}
 # CHECK-NEXT:  2      8     1.00    *                   vpabsq	(%rax), %zmm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vpabsq	(%rax){1to8}, %zmm19 {%k1}
 # CHECK-NEXT:  1      1     1.00                        vpabsq	%zmm16, %zmm19 {%k1} {z}
 # CHECK-NEXT:  2      8     1.00    *                   vpabsq	(%rax), %zmm19 {%k1} {z}
-# CHECK-NEXT:  1      1     0.33                        vpaddb	%zmm16, %zmm17, %zmm19
-# CHECK-NEXT:  2      8     0.50    *                   vpaddb	(%rax), %zmm17, %zmm19
-# CHECK-NEXT:  1      1     0.33                        vpaddb	%zmm16, %zmm17, %zmm19 {%k1}
-# CHECK-NEXT:  2      8     0.50    *                   vpaddb	(%rax), %zmm17, %zmm19 {%k1}
-# CHECK-NEXT:  1      1     0.33                        vpaddb	%zmm16, %zmm17, %zmm19 {%k1} {z}
-# CHECK-NEXT:  2      8     0.50    *                   vpaddb	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vpabsq	(%rax){1to8}, %zmm19 {%k1} {z}
 # CHECK-NEXT:  1      1     0.33                        vpaddd	%zmm16, %zmm17, %zmm19
 # CHECK-NEXT:  2      8     0.50    *                   vpaddd	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  2      8     0.50    *                   vpaddd	(%rax){1to16}, %zmm17, %zmm19
 # CHECK-NEXT:  1      1     0.33                        vpaddd	%zmm16, %zmm17, %zmm19 {%k1}
 # CHECK-NEXT:  2      8     0.50    *                   vpaddd	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     0.50    *                   vpaddd	(%rax){1to16}, %zmm17, %zmm19 {%k1}
 # CHECK-NEXT:  1      1     0.33                        vpaddd	%zmm16, %zmm17, %zmm19 {%k1} {z}
 # CHECK-NEXT:  2      8     0.50    *                   vpaddd	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     0.50    *                   vpaddd	(%rax){1to16}, %zmm17, %zmm19 {%k1} {z}
 # CHECK-NEXT:  1      1     0.33                        vpaddq	%zmm16, %zmm17, %zmm19
 # CHECK-NEXT:  2      8     0.50    *                   vpaddq	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  2      8     0.50    *                   vpaddq	(%rax){1to8}, %zmm17, %zmm19
 # CHECK-NEXT:  1      1     0.33                        vpaddq	%zmm16, %zmm17, %zmm19 {%k1}
 # CHECK-NEXT:  2      8     0.50    *                   vpaddq	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     0.50    *                   vpaddq	(%rax){1to8}, %zmm17, %zmm19 {%k1}
 # CHECK-NEXT:  1      1     0.33                        vpaddq	%zmm16, %zmm17, %zmm19 {%k1} {z}
 # CHECK-NEXT:  2      8     0.50    *                   vpaddq	(%rax), %zmm17, %zmm19 {%k1} {z}
-# CHECK-NEXT:  1      1     0.33                        vpaddw	%zmm16, %zmm17, %zmm19
-# CHECK-NEXT:  2      8     0.50    *                   vpaddw	(%rax), %zmm17, %zmm19
-# CHECK-NEXT:  1      1     0.33                        vpaddw	%zmm16, %zmm17, %zmm19 {%k1}
-# CHECK-NEXT:  2      8     0.50    *                   vpaddw	(%rax), %zmm17, %zmm19 {%k1}
-# CHECK-NEXT:  1      1     0.33                        vpaddw	%zmm16, %zmm17, %zmm19 {%k1} {z}
-# CHECK-NEXT:  2      8     0.50    *                   vpaddw	(%rax), %zmm17, %zmm19 {%k1} {z}
-# CHECK-NEXT:  1      1     0.33                        vpsubb	%zmm16, %zmm17, %zmm19
-# CHECK-NEXT:  2      8     0.50    *                   vpsubb	(%rax), %zmm17, %zmm19
-# CHECK-NEXT:  1      1     0.33                        vpsubb	%zmm16, %zmm17, %zmm19 {%k1}
-# CHECK-NEXT:  2      8     0.50    *                   vpsubb	(%rax), %zmm17, %zmm19 {%k1}
-# CHECK-NEXT:  1      1     0.33                        vpsubb	%zmm16, %zmm17, %zmm19 {%k1} {z}
-# CHECK-NEXT:  2      8     0.50    *                   vpsubb	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     0.50    *                   vpaddq	(%rax){1to8}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      10    1.00                        vpmulld	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  3      17    1.00    *                   vpmulld	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  3      17    1.00    *                   vpmulld	(%rax){1to16}, %zmm17, %zmm19
+# CHECK-NEXT:  2      10    1.00                        vpmulld	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  3      17    1.00    *                   vpmulld	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  3      17    1.00    *                   vpmulld	(%rax){1to16}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      10    1.00                        vpmulld	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  3      17    1.00    *                   vpmulld	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  3      17    1.00    *                   vpmulld	(%rax){1to16}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  1      3     1.00                        vpermd	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  2      10    1.00    *                   vpermd	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  2      10    1.00    *                   vpermd	(%rax){1to16}, %zmm17, %zmm19
+# CHECK-NEXT:  1      3     1.00                        vpermd	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      10    1.00    *                   vpermd	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      10    1.00    *                   vpermd	(%rax){1to16}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  1      3     1.00                        vpermd	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      10    1.00    *                   vpermd	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      10    1.00    *                   vpermd	(%rax){1to16}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vpermilpd	$0, %zmm16, %zmm19
+# CHECK-NEXT:  2      8     1.00    *                   vpermilpd	$0, (%rax), %zmm19
+# CHECK-NEXT:  2      8     1.00    *                   vpermilpd	$0, (%rax){1to8}, %zmm19
+# CHECK-NEXT:  1      1     1.00                        vpermilpd	$0, %zmm16, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vpermilpd	$0, (%rax), %zmm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vpermilpd	$0, (%rax){1to8}, %zmm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vpermilpd	$0, %zmm16, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vpermilpd	$0, (%rax), %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vpermilpd	$0, (%rax){1to8}, %zmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vpermilpd	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  2      8     1.00    *                   vpermilpd	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  2      8     1.00    *                   vpermilpd	(%rax){1to8}, %zmm17, %zmm19
+# CHECK-NEXT:  1      1     1.00                        vpermilpd	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vpermilpd	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vpermilpd	(%rax){1to8}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vpermilpd	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vpermilpd	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vpermilpd	(%rax){1to8}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vpermilps	$0, %zmm16, %zmm19
+# CHECK-NEXT:  2      8     1.00    *                   vpermilps	$0, (%rax), %zmm19
+# CHECK-NEXT:  2      8     1.00    *                   vpermilps	$0, (%rax){1to16}, %zmm19
+# CHECK-NEXT:  1      1     1.00                        vpermilps	$0, %zmm16, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vpermilps	$0, (%rax), %zmm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vpermilps	$0, (%rax){1to16}, %zmm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vpermilps	$0, %zmm16, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vpermilps	$0, (%rax), %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vpermilps	$0, (%rax){1to16}, %zmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vpermilps	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  2      8     1.00    *                   vpermilps	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  2      8     1.00    *                   vpermilps	(%rax){1to16}, %zmm17, %zmm19
+# CHECK-NEXT:  1      1     1.00                        vpermilps	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vpermilps	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vpermilps	(%rax){1to16}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vpermilps	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vpermilps	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vpermilps	(%rax){1to16}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  1      3     1.00                        vpermpd	$0, %zmm16, %zmm19
+# CHECK-NEXT:  2      10    1.00    *                   vpermpd	$0, (%rax), %zmm19
+# CHECK-NEXT:  2      10    1.00    *                   vpermpd	$0, (%rax){1to8}, %zmm19
+# CHECK-NEXT:  1      3     1.00                        vpermpd	$0, %zmm16, %zmm19 {%k1}
+# CHECK-NEXT:  2      10    1.00    *                   vpermpd	$0, (%rax), %zmm19 {%k1}
+# CHECK-NEXT:  2      10    1.00    *                   vpermpd	$0, (%rax){1to8}, %zmm19 {%k1}
+# CHECK-NEXT:  1      3     1.00                        vpermpd	$0, %zmm16, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      10    1.00    *                   vpermpd	$0, (%rax), %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      10    1.00    *                   vpermpd	$0, (%rax){1to8}, %zmm19 {%k1} {z}
+# CHECK-NEXT:  1      3     1.00                        vpermpd	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  2      10    1.00    *                   vpermpd	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  2      10    1.00    *                   vpermpd	(%rax){1to8}, %zmm17, %zmm19
+# CHECK-NEXT:  1      3     1.00                        vpermpd	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      10    1.00    *                   vpermpd	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      10    1.00    *                   vpermpd	(%rax){1to8}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  1      3     1.00                        vpermpd	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      10    1.00    *                   vpermpd	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      10    1.00    *                   vpermpd	(%rax){1to8}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  1      3     1.00                        vpermps	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  2      10    1.00    *                   vpermps	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  2      10    1.00    *                   vpermps	(%rax){1to16}, %zmm17, %zmm19
+# CHECK-NEXT:  1      3     1.00                        vpermps	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      10    1.00    *                   vpermps	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      10    1.00    *                   vpermps	(%rax){1to16}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  1      3     1.00                        vpermps	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      10    1.00    *                   vpermps	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      10    1.00    *                   vpermps	(%rax){1to16}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  1      3     1.00                        vpermq	$0, %zmm16, %zmm19
+# CHECK-NEXT:  2      10    1.00    *                   vpermq	$0, (%rax), %zmm19
+# CHECK-NEXT:  2      10    1.00    *                   vpermq	$0, (%rax){1to8}, %zmm19
+# CHECK-NEXT:  1      3     1.00                        vpermq	$0, %zmm16, %zmm19 {%k1}
+# CHECK-NEXT:  2      10    1.00    *                   vpermq	$0, (%rax), %zmm19 {%k1}
+# CHECK-NEXT:  2      10    1.00    *                   vpermq	$0, (%rax){1to8}, %zmm19 {%k1}
+# CHECK-NEXT:  1      3     1.00                        vpermq	$0, %zmm16, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      10    1.00    *                   vpermq	$0, (%rax), %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      10    1.00    *                   vpermq	$0, (%rax){1to8}, %zmm19 {%k1} {z}
+# CHECK-NEXT:  1      3     1.00                        vpermq	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  2      10    1.00    *                   vpermq	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  2      10    1.00    *                   vpermq	(%rax){1to8}, %zmm17, %zmm19
+# CHECK-NEXT:  1      3     1.00                        vpermq	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      10    1.00    *                   vpermq	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      10    1.00    *                   vpermq	(%rax){1to8}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  1      3     1.00                        vpermq	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      10    1.00    *                   vpermq	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      10    1.00    *                   vpermq	(%rax){1to8}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vpshufd	$0, %zmm16, %zmm19
+# CHECK-NEXT:  2      8     1.00    *                   vpshufd	$0, (%rax), %zmm19
+# CHECK-NEXT:  2      8     1.00    *                   vpshufd	$0, (%rax){1to16}, %zmm19
+# CHECK-NEXT:  1      1     1.00                        vpshufd	$0, %zmm16, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vpshufd	$0, (%rax), %zmm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vpshufd	$0, (%rax){1to16}, %zmm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vpshufd	$0, %zmm16, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vpshufd	$0, (%rax), %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vpshufd	$0, (%rax){1to16}, %zmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     0.33                        vpsubd	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  2      8     0.50    *                   vpsubd	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  2      8     0.50    *                   vpsubd	(%rax){1to16}, %zmm17, %zmm19
+# CHECK-NEXT:  1      1     0.33                        vpsubd	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     0.50    *                   vpsubd	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     0.50    *                   vpsubd	(%rax){1to16}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  1      1     0.33                        vpsubd	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     0.50    *                   vpsubd	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     0.50    *                   vpsubd	(%rax){1to16}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     0.33                        vpsubq	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  2      8     0.50    *                   vpsubq	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  2      8     0.50    *                   vpsubq	(%rax){1to8}, %zmm17, %zmm19
+# CHECK-NEXT:  1      1     0.33                        vpsubq	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     0.50    *                   vpsubq	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     0.50    *                   vpsubq	(%rax){1to8}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  1      1     0.33                        vpsubq	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     0.50    *                   vpsubq	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     0.50    *                   vpsubq	(%rax){1to8}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vpunpckhdq	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  2      8     1.00    *                   vpunpckhdq	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  2      8     1.00    *                   vpunpckhdq	(%rax){1to16}, %zmm17, %zmm19
+# CHECK-NEXT:  1      1     1.00                        vpunpckhdq	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vpunpckhdq	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vpunpckhdq	(%rax){1to16}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vpunpckhdq	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vpunpckhdq	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vpunpckhdq	(%rax){1to16}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vpunpckhqdq	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  2      8     1.00    *                   vpunpckhqdq	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  2      8     1.00    *                   vpunpckhqdq	(%rax){1to8}, %zmm17, %zmm19
+# CHECK-NEXT:  1      1     1.00                        vpunpckhqdq	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vpunpckhqdq	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vpunpckhqdq	(%rax){1to8}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vpunpckhqdq	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vpunpckhqdq	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vpunpckhqdq	(%rax){1to8}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vpunpckldq	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  2      8     1.00    *                   vpunpckldq	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  2      8     1.00    *                   vpunpckldq	(%rax){1to16}, %zmm17, %zmm19
+# CHECK-NEXT:  1      1     1.00                        vpunpckldq	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vpunpckldq	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vpunpckldq	(%rax){1to16}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vpunpckldq	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vpunpckldq	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vpunpckldq	(%rax){1to16}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vpunpcklqdq	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  2      8     1.00    *                   vpunpcklqdq	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  2      8     1.00    *                   vpunpcklqdq	(%rax){1to8}, %zmm17, %zmm19
+# CHECK-NEXT:  1      1     1.00                        vpunpcklqdq	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vpunpcklqdq	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vpunpcklqdq	(%rax){1to8}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vpunpcklqdq	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vpunpcklqdq	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vpunpcklqdq	(%rax){1to8}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  1      3     1.00                        vshuff32x4	$0, %zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  2      10    1.00    *                   vshuff32x4	$0, (%rax), %zmm17, %zmm19
+# CHECK-NEXT:  2      10    1.00    *                   vshuff32x4	$0, (%rax){1to16}, %zmm17, %zmm19
+# CHECK-NEXT:  1      3     1.00                        vshuff32x4	$0, %zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      10    1.00    *                   vshuff32x4	$0, (%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      10    1.00    *                   vshuff32x4	$0, (%rax){1to16}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  1      3     1.00                        vshuff32x4	$0, %zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      10    1.00    *                   vshuff32x4	$0, (%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      10    1.00    *                   vshuff32x4	$0, (%rax){1to16}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  1      3     1.00                        vshuff64x2	$0, %zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  2      10    1.00    *                   vshuff64x2	$0, (%rax), %zmm17, %zmm19
+# CHECK-NEXT:  2      10    1.00    *                   vshuff64x2	$0, (%rax){1to8}, %zmm17, %zmm19
+# CHECK-NEXT:  1      3     1.00                        vshuff64x2	$0, %zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      10    1.00    *                   vshuff64x2	$0, (%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      10    1.00    *                   vshuff64x2	$0, (%rax){1to8}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  1      3     1.00                        vshuff64x2	$0, %zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      10    1.00    *                   vshuff64x2	$0, (%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      10    1.00    *                   vshuff64x2	$0, (%rax){1to8}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  1      3     1.00                        vshufi32x4	$0, %zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  2      10    1.00    *                   vshufi32x4	$0, (%rax), %zmm17, %zmm19
+# CHECK-NEXT:  2      10    1.00    *                   vshufi32x4	$0, (%rax){1to16}, %zmm17, %zmm19
+# CHECK-NEXT:  1      3     1.00                        vshufi32x4	$0, %zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      10    1.00    *                   vshufi32x4	$0, (%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      10    1.00    *                   vshufi32x4	$0, (%rax){1to16}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  1      3     1.00                        vshufi32x4	$0, %zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      10    1.00    *                   vshufi32x4	$0, (%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      10    1.00    *                   vshufi32x4	$0, (%rax){1to16}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  1      3     1.00                        vshufi64x2	$0, %zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  2      10    1.00    *                   vshufi64x2	$0, (%rax), %zmm17, %zmm19
+# CHECK-NEXT:  2      10    1.00    *                   vshufi64x2	$0, (%rax){1to8}, %zmm17, %zmm19
+# CHECK-NEXT:  1      3     1.00                        vshufi64x2	$0, %zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      10    1.00    *                   vshufi64x2	$0, (%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      10    1.00    *                   vshufi64x2	$0, (%rax){1to8}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  1      3     1.00                        vshufi64x2	$0, %zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      10    1.00    *                   vshufi64x2	$0, (%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      10    1.00    *                   vshufi64x2	$0, (%rax){1to8}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  3      32    24.00                       vsqrtpd	%zmm16, %zmm19
+# CHECK-NEXT:  4      39    24.00   *                   vsqrtpd	(%rax), %zmm19
+# CHECK-NEXT:  4      39    24.00   *                   vsqrtpd	(%rax){1to8}, %zmm19
+# CHECK-NEXT:  3      32    24.00                       vsqrtpd	%zmm16, %zmm19 {%k1}
+# CHECK-NEXT:  4      39    24.00   *                   vsqrtpd	(%rax), %zmm19 {%k1}
+# CHECK-NEXT:  4      39    24.00   *                   vsqrtpd	(%rax){1to8}, %zmm19 {%k1}
+# CHECK-NEXT:  3      32    24.00                       vsqrtpd	%zmm16, %zmm19 {%k1} {z}
+# CHECK-NEXT:  4      39    24.00   *                   vsqrtpd	(%rax), %zmm19 {%k1} {z}
+# CHECK-NEXT:  4      39    24.00   *                   vsqrtpd	(%rax){1to8}, %zmm19 {%k1} {z}
+# CHECK-NEXT:  3      20    12.00                       vsqrtps	%zmm16, %zmm19
+# CHECK-NEXT:  4      27    12.00   *                   vsqrtps	(%rax), %zmm19
+# CHECK-NEXT:  4      27    12.00   *                   vsqrtps	(%rax){1to16}, %zmm19
+# CHECK-NEXT:  3      20    12.00                       vsqrtps	%zmm16, %zmm19 {%k1}
+# CHECK-NEXT:  4      27    12.00   *                   vsqrtps	(%rax), %zmm19 {%k1}
+# CHECK-NEXT:  4      27    12.00   *                   vsqrtps	(%rax){1to16}, %zmm19 {%k1}
+# CHECK-NEXT:  3      20    12.00                       vsqrtps	%zmm16, %zmm19 {%k1} {z}
+# CHECK-NEXT:  4      27    12.00   *                   vsqrtps	(%rax), %zmm19 {%k1} {z}
+# CHECK-NEXT:  4      27    12.00   *                   vsqrtps	(%rax){1to16}, %zmm19 {%k1} {z}
+# CHECK-NEXT:  1      18    6.00                        vsqrtsd	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  2      23    6.00    *                   vsqrtsd	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  1      18    6.00                        vsqrtsd	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      23    6.00    *                   vsqrtsd	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  1      18    6.00                        vsqrtsd	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      23    6.00    *                   vsqrtsd	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  1      12    3.00                        vsqrtss	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  2      17    3.00    *                   vsqrtss	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  1      12    3.00                        vsqrtss	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      17    3.00    *                   vsqrtss	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  1      12    3.00                        vsqrtss	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      17    3.00    *                   vsqrtss	(%rax), %xmm17, %xmm19 {%k1} {z}
 # CHECK-NEXT:  1      1     0.33                        vpsubd	%zmm16, %zmm17, %zmm19
 # CHECK-NEXT:  2      8     0.50    *                   vpsubd	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  2      8     0.50    *                   vpsubd	(%rax){1to16}, %zmm17, %zmm19
 # CHECK-NEXT:  1      1     0.33                        vpsubd	%zmm16, %zmm17, %zmm19 {%k1}
 # CHECK-NEXT:  2      8     0.50    *                   vpsubd	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     0.50    *                   vpsubd	(%rax){1to16}, %zmm17, %zmm19 {%k1}
 # CHECK-NEXT:  1      1     0.33                        vpsubd	%zmm16, %zmm17, %zmm19 {%k1} {z}
 # CHECK-NEXT:  2      8     0.50    *                   vpsubd	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     0.50    *                   vpsubd	(%rax){1to16}, %zmm17, %zmm19 {%k1} {z}
 # CHECK-NEXT:  1      1     0.33                        vpsubq	%zmm16, %zmm17, %zmm19
 # CHECK-NEXT:  2      8     0.50    *                   vpsubq	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  2      8     0.50    *                   vpsubq	(%rax){1to8}, %zmm17, %zmm19
 # CHECK-NEXT:  1      1     0.33                        vpsubq	%zmm16, %zmm17, %zmm19 {%k1}
 # CHECK-NEXT:  2      8     0.50    *                   vpsubq	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     0.50    *                   vpsubq	(%rax){1to8}, %zmm17, %zmm19 {%k1}
 # CHECK-NEXT:  1      1     0.33                        vpsubq	%zmm16, %zmm17, %zmm19 {%k1} {z}
 # CHECK-NEXT:  2      8     0.50    *                   vpsubq	(%rax), %zmm17, %zmm19 {%k1} {z}
-# CHECK-NEXT:  1      1     0.33                        vpsubw	%zmm16, %zmm17, %zmm19
-# CHECK-NEXT:  2      8     0.50    *                   vpsubw	(%rax), %zmm17, %zmm19
-# CHECK-NEXT:  1      1     0.33                        vpsubw	%zmm16, %zmm17, %zmm19 {%k1}
-# CHECK-NEXT:  2      8     0.50    *                   vpsubw	(%rax), %zmm17, %zmm19 {%k1}
-# CHECK-NEXT:  1      1     0.33                        vpsubw	%zmm16, %zmm17, %zmm19 {%k1} {z}
-# CHECK-NEXT:  2      8     0.50    *                   vpsubw	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     0.50    *                   vpsubq	(%rax){1to8}, %zmm17, %zmm19 {%k1} {z}
 # CHECK-NEXT:  1      4     0.50                        vsubpd	%zmm16, %zmm17, %zmm19
 # CHECK-NEXT:  2      11    0.50    *                   vsubpd	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  2      11    0.50    *                   vsubpd	(%rax){1to8}, %zmm17, %zmm19
 # CHECK-NEXT:  1      4     0.50                        vsubpd	%zmm16, %zmm17, %zmm19 {%k1}
 # CHECK-NEXT:  2      11    0.50    *                   vsubpd	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      11    0.50    *                   vsubpd	(%rax){1to8}, %zmm17, %zmm19 {%k1}
 # CHECK-NEXT:  1      4     0.50                        vsubpd	%zmm16, %zmm17, %zmm19 {%k1} {z}
 # CHECK-NEXT:  2      11    0.50    *                   vsubpd	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      11    0.50    *                   vsubpd	(%rax){1to8}, %zmm17, %zmm19 {%k1} {z}
 # CHECK-NEXT:  1      4     0.50                        vsubps	%zmm16, %zmm17, %zmm19
 # CHECK-NEXT:  2      11    0.50    *                   vsubps	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  2      11    0.50    *                   vsubps	(%rax){1to16}, %zmm17, %zmm19
 # CHECK-NEXT:  1      4     0.50                        vsubps	%zmm16, %zmm17, %zmm19 {%k1}
 # CHECK-NEXT:  2      11    0.50    *                   vsubps	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      11    0.50    *                   vsubps	(%rax){1to16}, %zmm17, %zmm19 {%k1}
 # CHECK-NEXT:  1      4     0.50                        vsubps	%zmm16, %zmm17, %zmm19 {%k1} {z}
 # CHECK-NEXT:  2      11    0.50    *                   vsubps	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      11    0.50    *                   vsubps	(%rax){1to16}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vunpckhpd	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  2      8     1.00    *                   vunpckhpd	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  2      8     1.00    *                   vunpckhpd	(%rax){1to8}, %zmm17, %zmm19
+# CHECK-NEXT:  1      1     1.00                        vunpckhpd	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vunpckhpd	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vunpckhpd	(%rax){1to8}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vunpckhpd	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vunpckhpd	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vunpckhpd	(%rax){1to8}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vunpckhps	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  2      8     1.00    *                   vunpckhps	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  2      8     1.00    *                   vunpckhps	(%rax){1to16}, %zmm17, %zmm19
+# CHECK-NEXT:  1      1     1.00                        vunpckhps	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vunpckhps	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vunpckhps	(%rax){1to16}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vunpckhps	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vunpckhps	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vunpckhps	(%rax){1to16}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vunpcklpd	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  2      8     1.00    *                   vunpcklpd	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  2      8     1.00    *                   vunpcklpd	(%rax){1to8}, %zmm17, %zmm19
+# CHECK-NEXT:  1      1     1.00                        vunpcklpd	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vunpcklpd	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vunpcklpd	(%rax){1to8}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vunpcklpd	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vunpcklpd	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vunpcklpd	(%rax){1to8}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vunpcklps	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  2      8     1.00    *                   vunpcklps	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  2      8     1.00    *                   vunpcklps	(%rax){1to16}, %zmm17, %zmm19
+# CHECK-NEXT:  1      1     1.00                        vunpcklps	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vunpcklps	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vunpcklps	(%rax){1to16}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vunpcklps	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vunpcklps	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vunpcklps	(%rax){1to16}, %zmm17, %zmm19 {%k1} {z}
 
 # CHECK:      Resources:
 # CHECK-NEXT: [0]   - SKXDivider
@@ -259,115 +1001,469 @@ vsubps            (%rax), %zmm17, %zmm19 {z}{k1}
 
 # CHECK:      Resource pressure per iteration:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]
-# CHECK-NEXT:  -     156.00 70.00  16.00  27.00  27.00   -     46.00   -      -
+# CHECK-NEXT:  -     612.00 188.00 26.00  153.00 153.00  -     330.00  -      -
 
 # CHECK:      Resource pressure by instruction:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    Instructions:
 # CHECK-NEXT:  -      -     0.50    -      -      -      -     0.50    -      -     vaddpd	%zmm16, %zmm17, %zmm19
 # CHECK-NEXT:  -      -     0.50    -     0.50   0.50    -     0.50    -      -     vaddpd	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  -      -     0.50    -     0.50   0.50    -     0.50    -      -     vaddpd	(%rax){1to8}, %zmm17, %zmm19
 # CHECK-NEXT:  -      -     0.50    -      -      -      -     0.50    -      -     vaddpd	%zmm16, %zmm17, %zmm19 {%k1}
 # CHECK-NEXT:  -      -     0.50    -     0.50   0.50    -     0.50    -      -     vaddpd	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -     0.50    -     0.50   0.50    -     0.50    -      -     vaddpd	(%rax){1to8}, %zmm17, %zmm19 {%k1}
 # CHECK-NEXT:  -      -     0.50    -      -      -      -     0.50    -      -     vaddpd	%zmm16, %zmm17, %zmm19 {%k1} {z}
 # CHECK-NEXT:  -      -     0.50    -     0.50   0.50    -     0.50    -      -     vaddpd	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.50    -     0.50   0.50    -     0.50    -      -     vaddpd	(%rax){1to8}, %zmm17, %zmm19 {%k1} {z}
 # CHECK-NEXT:  -      -     0.50    -      -      -      -     0.50    -      -     vaddps	%zmm16, %zmm17, %zmm19
 # CHECK-NEXT:  -      -     0.50    -     0.50   0.50    -     0.50    -      -     vaddps	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  -      -     0.50    -     0.50   0.50    -     0.50    -      -     vaddps	(%rax){1to16}, %zmm17, %zmm19
 # CHECK-NEXT:  -      -     0.50    -      -      -      -     0.50    -      -     vaddps	%zmm16, %zmm17, %zmm19 {%k1}
 # CHECK-NEXT:  -      -     0.50    -     0.50   0.50    -     0.50    -      -     vaddps	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -     0.50    -     0.50   0.50    -     0.50    -      -     vaddps	(%rax){1to16}, %zmm17, %zmm19 {%k1}
 # CHECK-NEXT:  -      -     0.50    -      -      -      -     0.50    -      -     vaddps	%zmm16, %zmm17, %zmm19 {%k1} {z}
 # CHECK-NEXT:  -      -     0.50    -     0.50   0.50    -     0.50    -      -     vaddps	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.50    -     0.50   0.50    -     0.50    -      -     vaddps	(%rax){1to16}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.50    -      -      -      -     1.50    -      -     vcvtdq2pd	%ymm16, %zmm19
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vcvtdq2pd	(%rax), %zmm19
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vcvtdq2pd	(%rax){1to8}, %zmm19
+# CHECK-NEXT:  -      -     0.50    -      -      -      -     0.50    -      -     vcvtdq2pd	%ymm16, %zmm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vcvtdq2pd	(%rax), %zmm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vcvtdq2pd	(%rax){1to8}, %zmm19 {%k1}
+# CHECK-NEXT:  -      -     0.50    -      -      -      -     0.50    -      -     vcvtdq2pd	%ymm16, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vcvtdq2pd	(%rax), %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vcvtdq2pd	(%rax){1to8}, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.50    -      -      -      -     0.50    -      -     vcvtdq2ps	%zmm16, %zmm19
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vcvtdq2ps	(%rax), %zmm19
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vcvtdq2ps	(%rax){1to16}, %zmm19
+# CHECK-NEXT:  -      -     0.50    -      -      -      -     0.50    -      -     vcvtdq2ps	%zmm16, %zmm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vcvtdq2ps	(%rax), %zmm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vcvtdq2ps	(%rax){1to16}, %zmm19 {%k1}
+# CHECK-NEXT:  -      -     0.50    -      -      -      -     0.50    -      -     vcvtdq2ps	%zmm16, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vcvtdq2ps	(%rax), %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vcvtdq2ps	(%rax){1to16}, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.50    -      -      -      -     0.50    -      -     vcvtps2dq	%zmm16, %zmm19
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vcvtps2dq	(%rax), %zmm19
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vcvtps2dq	(%rax){1to16}, %zmm19
+# CHECK-NEXT:  -      -     0.50    -      -      -      -     0.50    -      -     vcvtps2dq	%zmm16, %zmm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vcvtps2dq	(%rax), %zmm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vcvtps2dq	(%rax){1to16}, %zmm19 {%k1}
+# CHECK-NEXT:  -      -     0.50    -      -      -      -     0.50    -      -     vcvtps2dq	%zmm16, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vcvtps2dq	(%rax), %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vcvtps2dq	(%rax){1to16}, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.50    -      -      -      -     0.50    -      -     vcvttps2dq	%zmm16, %zmm19
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vcvttps2dq	(%rax), %zmm19
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vcvttps2dq	(%rax){1to16}, %zmm19
+# CHECK-NEXT:  -      -     0.50    -      -      -      -     0.50    -      -     vcvttps2dq	%zmm16, %zmm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vcvttps2dq	(%rax), %zmm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vcvttps2dq	(%rax){1to16}, %zmm19 {%k1}
+# CHECK-NEXT:  -      -     0.50    -      -      -      -     0.50    -      -     vcvttps2dq	%zmm16, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vcvttps2dq	(%rax), %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vcvttps2dq	(%rax){1to16}, %zmm19 {%k1} {z}
 # CHECK-NEXT:  -     16.00  2.00    -      -      -      -     1.00    -      -     vdivpd	%zmm16, %zmm17, %zmm19
 # CHECK-NEXT:  -     16.00  2.00    -     0.50   0.50    -     1.00    -      -     vdivpd	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  -     16.00  2.00    -     0.50   0.50    -     1.00    -      -     vdivpd	(%rax){1to8}, %zmm17, %zmm19
 # CHECK-NEXT:  -     16.00  2.00    -      -      -      -     1.00    -      -     vdivpd	%zmm16, %zmm17, %zmm19 {%k1}
 # CHECK-NEXT:  -     16.00  2.00    -     0.50   0.50    -     1.00    -      -     vdivpd	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -     16.00  2.00    -     0.50   0.50    -     1.00    -      -     vdivpd	(%rax){1to8}, %zmm17, %zmm19 {%k1}
 # CHECK-NEXT:  -     16.00  2.00    -      -      -      -     1.00    -      -     vdivpd	%zmm16, %zmm17, %zmm19 {%k1} {z}
 # CHECK-NEXT:  -     16.00  2.00    -     0.50   0.50    -     1.00    -      -     vdivpd	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -     16.00  2.00    -     0.50   0.50    -     1.00    -      -     vdivpd	(%rax){1to8}, %zmm17, %zmm19 {%k1} {z}
 # CHECK-NEXT:  -     10.00  2.00    -      -      -      -     1.00    -      -     vdivps	%zmm16, %zmm17, %zmm19
 # CHECK-NEXT:  -     10.00  2.00    -     0.50   0.50    -     1.00    -      -     vdivps	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  -     10.00  2.00    -     0.50   0.50    -     1.00    -      -     vdivps	(%rax){1to16}, %zmm17, %zmm19
 # CHECK-NEXT:  -     10.00  2.00    -      -      -      -     1.00    -      -     vdivps	%zmm16, %zmm17, %zmm19 {%k1}
 # CHECK-NEXT:  -     10.00  2.00    -     0.50   0.50    -     1.00    -      -     vdivps	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -     10.00  2.00    -     0.50   0.50    -     1.00    -      -     vdivps	(%rax){1to16}, %zmm17, %zmm19 {%k1}
 # CHECK-NEXT:  -     10.00  2.00    -      -      -      -     1.00    -      -     vdivps	%zmm16, %zmm17, %zmm19 {%k1} {z}
 # CHECK-NEXT:  -     10.00  2.00    -     0.50   0.50    -     1.00    -      -     vdivps	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -     10.00  2.00    -     0.50   0.50    -     1.00    -      -     vdivps	(%rax){1to16}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.50    -      -      -      -     0.50    -      -     vmaxpd	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  -      -     0.50    -     0.50   0.50    -     0.50    -      -     vmaxpd	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  -      -     0.50    -     0.50   0.50    -     0.50    -      -     vmaxpd	(%rax){1to8}, %zmm17, %zmm19
+# CHECK-NEXT:  -      -     0.50    -      -      -      -     0.50    -      -     vmaxpd	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -     0.50    -     0.50   0.50    -     0.50    -      -     vmaxpd	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -     0.50    -     0.50   0.50    -     0.50    -      -     vmaxpd	(%rax){1to8}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -     0.50    -      -      -      -     0.50    -      -     vmaxpd	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.50    -     0.50   0.50    -     0.50    -      -     vmaxpd	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.50    -     0.50   0.50    -     0.50    -      -     vmaxpd	(%rax){1to8}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.50    -      -      -      -     0.50    -      -     vmaxps	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  -      -     0.50    -     0.50   0.50    -     0.50    -      -     vmaxps	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  -      -     0.50    -     0.50   0.50    -     0.50    -      -     vmaxps	(%rax){1to16}, %zmm17, %zmm19
+# CHECK-NEXT:  -      -     0.50    -      -      -      -     0.50    -      -     vmaxps	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -     0.50    -     0.50   0.50    -     0.50    -      -     vmaxps	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -     0.50    -     0.50   0.50    -     0.50    -      -     vmaxps	(%rax){1to16}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -     0.50    -      -      -      -     0.50    -      -     vmaxps	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.50    -     0.50   0.50    -     0.50    -      -     vmaxps	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.50    -     0.50   0.50    -     0.50    -      -     vmaxps	(%rax){1to16}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.50    -      -      -      -     0.50    -      -     vminpd	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  -      -     0.50    -     0.50   0.50    -     0.50    -      -     vminpd	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  -      -     0.50    -     0.50   0.50    -     0.50    -      -     vminpd	(%rax){1to8}, %zmm17, %zmm19
+# CHECK-NEXT:  -      -     0.50    -      -      -      -     0.50    -      -     vminpd	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -     0.50    -     0.50   0.50    -     0.50    -      -     vminpd	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -     0.50    -     0.50   0.50    -     0.50    -      -     vminpd	(%rax){1to8}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -     0.50    -      -      -      -     0.50    -      -     vminpd	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.50    -     0.50   0.50    -     0.50    -      -     vminpd	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.50    -     0.50   0.50    -     0.50    -      -     vminpd	(%rax){1to8}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.50    -      -      -      -     0.50    -      -     vminps	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  -      -     0.50    -     0.50   0.50    -     0.50    -      -     vminps	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  -      -     0.50    -     0.50   0.50    -     0.50    -      -     vminps	(%rax){1to16}, %zmm17, %zmm19
+# CHECK-NEXT:  -      -     0.50    -      -      -      -     0.50    -      -     vminps	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -     0.50    -     0.50   0.50    -     0.50    -      -     vminps	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -     0.50    -     0.50   0.50    -     0.50    -      -     vminps	(%rax){1to16}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -     0.50    -      -      -      -     0.50    -      -     vminps	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.50    -     0.50   0.50    -     0.50    -      -     vminps	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.50    -     0.50   0.50    -     0.50    -      -     vminps	(%rax){1to16}, %zmm17, %zmm19 {%k1} {z}
 # CHECK-NEXT:  -      -     0.50    -      -      -      -     0.50    -      -     vmulpd	%zmm16, %zmm17, %zmm19
 # CHECK-NEXT:  -      -     0.50    -     0.50   0.50    -     0.50    -      -     vmulpd	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  -      -     0.50    -     0.50   0.50    -     0.50    -      -     vmulpd	(%rax){1to8}, %zmm17, %zmm19
 # CHECK-NEXT:  -      -     0.50    -      -      -      -     0.50    -      -     vmulpd	%zmm16, %zmm17, %zmm19 {%k1}
 # CHECK-NEXT:  -      -     0.50    -     0.50   0.50    -     0.50    -      -     vmulpd	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -     0.50    -     0.50   0.50    -     0.50    -      -     vmulpd	(%rax){1to8}, %zmm17, %zmm19 {%k1}
 # CHECK-NEXT:  -      -     0.50    -      -      -      -     0.50    -      -     vmulpd	%zmm16, %zmm17, %zmm19 {%k1} {z}
 # CHECK-NEXT:  -      -     0.50    -     0.50   0.50    -     0.50    -      -     vmulpd	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.50    -     0.50   0.50    -     0.50    -      -     vmulpd	(%rax){1to8}, %zmm17, %zmm19 {%k1} {z}
 # CHECK-NEXT:  -      -     0.50    -      -      -      -     0.50    -      -     vmulps	%zmm16, %zmm17, %zmm19
 # CHECK-NEXT:  -      -     0.50    -     0.50   0.50    -     0.50    -      -     vmulps	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  -      -     0.50    -     0.50   0.50    -     0.50    -      -     vmulps	(%rax){1to16}, %zmm17, %zmm19
 # CHECK-NEXT:  -      -     0.50    -      -      -      -     0.50    -      -     vmulps	%zmm16, %zmm17, %zmm19 {%k1}
 # CHECK-NEXT:  -      -     0.50    -     0.50   0.50    -     0.50    -      -     vmulps	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -     0.50    -     0.50   0.50    -     0.50    -      -     vmulps	(%rax){1to16}, %zmm17, %zmm19 {%k1}
 # CHECK-NEXT:  -      -     0.50    -      -      -      -     0.50    -      -     vmulps	%zmm16, %zmm17, %zmm19 {%k1} {z}
 # CHECK-NEXT:  -      -     0.50    -     0.50   0.50    -     0.50    -      -     vmulps	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.50    -     0.50   0.50    -     0.50    -      -     vmulps	(%rax){1to16}, %zmm17, %zmm19 {%k1} {z}
 # CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     vpabsd	%zmm16, %zmm19
 # CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -      -      -      -     vpabsd	(%rax), %zmm19
+# CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -      -      -      -     vpabsd	(%rax){1to16}, %zmm19
 # CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     vpabsd	%zmm16, %zmm19 {%k1}
 # CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -      -      -      -     vpabsd	(%rax), %zmm19 {%k1}
+# CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -      -      -      -     vpabsd	(%rax){1to16}, %zmm19 {%k1}
 # CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     vpabsd	%zmm16, %zmm19 {%k1} {z}
 # CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -      -      -      -     vpabsd	(%rax), %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -      -      -      -     vpabsd	(%rax){1to16}, %zmm19 {%k1} {z}
 # CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     vpabsq	%zmm16, %zmm19
 # CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -      -      -      -     vpabsq	(%rax), %zmm19
+# CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -      -      -      -     vpabsq	(%rax){1to8}, %zmm19
 # CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     vpabsq	%zmm16, %zmm19 {%k1}
 # CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -      -      -      -     vpabsq	(%rax), %zmm19 {%k1}
+# CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -      -      -      -     vpabsq	(%rax){1to8}, %zmm19 {%k1}
 # CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     vpabsq	%zmm16, %zmm19 {%k1} {z}
 # CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -      -      -      -     vpabsq	(%rax), %zmm19 {%k1} {z}
-# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vpaddb	%zmm16, %zmm17, %zmm19
-# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpaddb	(%rax), %zmm17, %zmm19
-# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vpaddb	%zmm16, %zmm17, %zmm19 {%k1}
-# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpaddb	(%rax), %zmm17, %zmm19 {%k1}
-# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vpaddb	%zmm16, %zmm17, %zmm19 {%k1} {z}
-# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpaddb	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -      -      -      -     vpabsq	(%rax){1to8}, %zmm19 {%k1} {z}
 # CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vpaddd	%zmm16, %zmm17, %zmm19
 # CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpaddd	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpaddd	(%rax){1to16}, %zmm17, %zmm19
 # CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vpaddd	%zmm16, %zmm17, %zmm19 {%k1}
 # CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpaddd	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpaddd	(%rax){1to16}, %zmm17, %zmm19 {%k1}
 # CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vpaddd	%zmm16, %zmm17, %zmm19 {%k1} {z}
 # CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpaddd	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpaddd	(%rax){1to16}, %zmm17, %zmm19 {%k1} {z}
 # CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vpaddq	%zmm16, %zmm17, %zmm19
 # CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpaddq	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpaddq	(%rax){1to8}, %zmm17, %zmm19
 # CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vpaddq	%zmm16, %zmm17, %zmm19 {%k1}
 # CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpaddq	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpaddq	(%rax){1to8}, %zmm17, %zmm19 {%k1}
 # CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vpaddq	%zmm16, %zmm17, %zmm19 {%k1} {z}
 # CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpaddq	(%rax), %zmm17, %zmm19 {%k1} {z}
-# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vpaddw	%zmm16, %zmm17, %zmm19
-# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpaddw	(%rax), %zmm17, %zmm19
-# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vpaddw	%zmm16, %zmm17, %zmm19 {%k1}
-# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpaddw	(%rax), %zmm17, %zmm19 {%k1}
-# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vpaddw	%zmm16, %zmm17, %zmm19 {%k1} {z}
-# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpaddw	(%rax), %zmm17, %zmm19 {%k1} {z}
-# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vpsubb	%zmm16, %zmm17, %zmm19
-# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpsubb	(%rax), %zmm17, %zmm19
-# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vpsubb	%zmm16, %zmm17, %zmm19 {%k1}
-# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpsubb	(%rax), %zmm17, %zmm19 {%k1}
-# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vpsubb	%zmm16, %zmm17, %zmm19 {%k1} {z}
-# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpsubb	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpaddq	(%rax){1to8}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     1.00    -      -      -      -     1.00    -      -     vpmulld	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -     1.00    -      -     vpmulld	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -     1.00    -      -     vpmulld	(%rax){1to16}, %zmm17, %zmm19
+# CHECK-NEXT:  -      -     1.00    -      -      -      -     1.00    -      -     vpmulld	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -     1.00    -      -     vpmulld	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -     1.00    -      -     vpmulld	(%rax){1to16}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -     1.00    -      -      -      -     1.00    -      -     vpmulld	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -     1.00    -      -     vpmulld	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -     1.00    -      -     vpmulld	(%rax){1to16}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpermd	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermd	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermd	(%rax){1to16}, %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpermd	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermd	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermd	(%rax){1to16}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpermd	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermd	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermd	(%rax){1to16}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpermilpd	$0, %zmm16, %zmm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermilpd	$0, (%rax), %zmm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermilpd	$0, (%rax){1to8}, %zmm19
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpermilpd	$0, %zmm16, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermilpd	$0, (%rax), %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermilpd	$0, (%rax){1to8}, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpermilpd	$0, %zmm16, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermilpd	$0, (%rax), %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermilpd	$0, (%rax){1to8}, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpermilpd	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermilpd	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermilpd	(%rax){1to8}, %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpermilpd	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermilpd	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermilpd	(%rax){1to8}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpermilpd	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermilpd	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermilpd	(%rax){1to8}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpermilps	$0, %zmm16, %zmm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermilps	$0, (%rax), %zmm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermilps	$0, (%rax){1to16}, %zmm19
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpermilps	$0, %zmm16, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermilps	$0, (%rax), %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermilps	$0, (%rax){1to16}, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpermilps	$0, %zmm16, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermilps	$0, (%rax), %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermilps	$0, (%rax){1to16}, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpermilps	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermilps	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermilps	(%rax){1to16}, %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpermilps	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermilps	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermilps	(%rax){1to16}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpermilps	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermilps	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermilps	(%rax){1to16}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpermpd	$0, %zmm16, %zmm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermpd	$0, (%rax), %zmm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermpd	$0, (%rax){1to8}, %zmm19
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpermpd	$0, %zmm16, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermpd	$0, (%rax), %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermpd	$0, (%rax){1to8}, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpermpd	$0, %zmm16, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermpd	$0, (%rax), %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermpd	$0, (%rax){1to8}, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpermpd	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermpd	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermpd	(%rax){1to8}, %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpermpd	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermpd	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermpd	(%rax){1to8}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpermpd	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermpd	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermpd	(%rax){1to8}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpermps	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermps	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermps	(%rax){1to16}, %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpermps	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermps	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermps	(%rax){1to16}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpermps	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermps	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermps	(%rax){1to16}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpermq	$0, %zmm16, %zmm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermq	$0, (%rax), %zmm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermq	$0, (%rax){1to8}, %zmm19
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpermq	$0, %zmm16, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermq	$0, (%rax), %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermq	$0, (%rax){1to8}, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpermq	$0, %zmm16, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermq	$0, (%rax), %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermq	$0, (%rax){1to8}, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpermq	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermq	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermq	(%rax){1to8}, %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpermq	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermq	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermq	(%rax){1to8}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpermq	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermq	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermq	(%rax){1to8}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpshufd	$0, %zmm16, %zmm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpshufd	$0, (%rax), %zmm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpshufd	$0, (%rax){1to16}, %zmm19
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpshufd	$0, %zmm16, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpshufd	$0, (%rax), %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpshufd	$0, (%rax){1to16}, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpshufd	$0, %zmm16, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpshufd	$0, (%rax), %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpshufd	$0, (%rax){1to16}, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vpsubd	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpsubd	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpsubd	(%rax){1to16}, %zmm17, %zmm19
+# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vpsubd	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpsubd	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpsubd	(%rax){1to16}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vpsubd	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpsubd	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpsubd	(%rax){1to16}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vpsubq	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpsubq	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpsubq	(%rax){1to8}, %zmm17, %zmm19
+# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vpsubq	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpsubq	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpsubq	(%rax){1to8}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vpsubq	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpsubq	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpsubq	(%rax){1to8}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpunpckhdq	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpunpckhdq	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpunpckhdq	(%rax){1to16}, %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpunpckhdq	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpunpckhdq	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpunpckhdq	(%rax){1to16}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpunpckhdq	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpunpckhdq	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpunpckhdq	(%rax){1to16}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpunpckhqdq	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpunpckhqdq	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpunpckhqdq	(%rax){1to8}, %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpunpckhqdq	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpunpckhqdq	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpunpckhqdq	(%rax){1to8}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpunpckhqdq	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpunpckhqdq	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpunpckhqdq	(%rax){1to8}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpunpckldq	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpunpckldq	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpunpckldq	(%rax){1to16}, %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpunpckldq	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpunpckldq	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpunpckldq	(%rax){1to16}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpunpckldq	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpunpckldq	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpunpckldq	(%rax){1to16}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpunpcklqdq	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpunpcklqdq	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpunpcklqdq	(%rax){1to8}, %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpunpcklqdq	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpunpcklqdq	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpunpcklqdq	(%rax){1to8}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpunpcklqdq	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpunpcklqdq	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpunpcklqdq	(%rax){1to8}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vshuff32x4	$0, %zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vshuff32x4	$0, (%rax), %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vshuff32x4	$0, (%rax){1to16}, %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vshuff32x4	$0, %zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vshuff32x4	$0, (%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vshuff32x4	$0, (%rax){1to16}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vshuff32x4	$0, %zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vshuff32x4	$0, (%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vshuff32x4	$0, (%rax){1to16}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vshuff64x2	$0, %zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vshuff64x2	$0, (%rax), %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vshuff64x2	$0, (%rax){1to8}, %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vshuff64x2	$0, %zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vshuff64x2	$0, (%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vshuff64x2	$0, (%rax){1to8}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vshuff64x2	$0, %zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vshuff64x2	$0, (%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vshuff64x2	$0, (%rax){1to8}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vshufi32x4	$0, %zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vshufi32x4	$0, (%rax), %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vshufi32x4	$0, (%rax){1to16}, %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vshufi32x4	$0, %zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vshufi32x4	$0, (%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vshufi32x4	$0, (%rax){1to16}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vshufi32x4	$0, %zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vshufi32x4	$0, (%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vshufi32x4	$0, (%rax){1to16}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vshufi64x2	$0, %zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vshufi64x2	$0, (%rax), %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vshufi64x2	$0, (%rax){1to8}, %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vshufi64x2	$0, %zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vshufi64x2	$0, (%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vshufi64x2	$0, (%rax){1to8}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vshufi64x2	$0, %zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vshufi64x2	$0, (%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vshufi64x2	$0, (%rax){1to8}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -     24.00  2.00    -      -      -      -     1.00    -      -     vsqrtpd	%zmm16, %zmm19
+# CHECK-NEXT:  -     24.00  2.00    -     0.50   0.50    -     1.00    -      -     vsqrtpd	(%rax), %zmm19
+# CHECK-NEXT:  -     24.00  2.00    -     0.50   0.50    -     1.00    -      -     vsqrtpd	(%rax){1to8}, %zmm19
+# CHECK-NEXT:  -     24.00  2.00    -      -      -      -     1.00    -      -     vsqrtpd	%zmm16, %zmm19 {%k1}
+# CHECK-NEXT:  -     24.00  2.00    -     0.50   0.50    -     1.00    -      -     vsqrtpd	(%rax), %zmm19 {%k1}
+# CHECK-NEXT:  -     24.00  2.00    -     0.50   0.50    -     1.00    -      -     vsqrtpd	(%rax){1to8}, %zmm19 {%k1}
+# CHECK-NEXT:  -     24.00  2.00    -      -      -      -     1.00    -      -     vsqrtpd	%zmm16, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -     24.00  2.00    -     0.50   0.50    -     1.00    -      -     vsqrtpd	(%rax), %zmm19 {%k1} {z}
+# CHECK-NEXT:  -     24.00  2.00    -     0.50   0.50    -     1.00    -      -     vsqrtpd	(%rax){1to8}, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -     12.00  2.00    -      -      -      -     1.00    -      -     vsqrtps	%zmm16, %zmm19
+# CHECK-NEXT:  -     12.00  2.00    -     0.50   0.50    -     1.00    -      -     vsqrtps	(%rax), %zmm19
+# CHECK-NEXT:  -     12.00  2.00    -     0.50   0.50    -     1.00    -      -     vsqrtps	(%rax){1to16}, %zmm19
+# CHECK-NEXT:  -     12.00  2.00    -      -      -      -     1.00    -      -     vsqrtps	%zmm16, %zmm19 {%k1}
+# CHECK-NEXT:  -     12.00  2.00    -     0.50   0.50    -     1.00    -      -     vsqrtps	(%rax), %zmm19 {%k1}
+# CHECK-NEXT:  -     12.00  2.00    -     0.50   0.50    -     1.00    -      -     vsqrtps	(%rax){1to16}, %zmm19 {%k1}
+# CHECK-NEXT:  -     12.00  2.00    -      -      -      -     1.00    -      -     vsqrtps	%zmm16, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -     12.00  2.00    -     0.50   0.50    -     1.00    -      -     vsqrtps	(%rax), %zmm19 {%k1} {z}
+# CHECK-NEXT:  -     12.00  2.00    -     0.50   0.50    -     1.00    -      -     vsqrtps	(%rax){1to16}, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -     6.00   1.00    -      -      -      -      -      -      -     vsqrtsd	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  -     6.00   1.00    -     0.50   0.50    -      -      -      -     vsqrtsd	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  -     6.00   1.00    -      -      -      -      -      -      -     vsqrtsd	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -     6.00   1.00    -     0.50   0.50    -      -      -      -     vsqrtsd	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -     6.00   1.00    -      -      -      -      -      -      -     vsqrtsd	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -     6.00   1.00    -     0.50   0.50    -      -      -      -     vsqrtsd	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -     3.00   1.00    -      -      -      -      -      -      -     vsqrtss	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  -     3.00   1.00    -     0.50   0.50    -      -      -      -     vsqrtss	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  -     3.00   1.00    -      -      -      -      -      -      -     vsqrtss	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -     3.00   1.00    -     0.50   0.50    -      -      -      -     vsqrtss	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -     3.00   1.00    -      -      -      -      -      -      -     vsqrtss	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -     3.00   1.00    -     0.50   0.50    -      -      -      -     vsqrtss	(%rax), %xmm17, %xmm19 {%k1} {z}
 # CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vpsubd	%zmm16, %zmm17, %zmm19
 # CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpsubd	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpsubd	(%rax){1to16}, %zmm17, %zmm19
 # CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vpsubd	%zmm16, %zmm17, %zmm19 {%k1}
 # CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpsubd	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpsubd	(%rax){1to16}, %zmm17, %zmm19 {%k1}
 # CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vpsubd	%zmm16, %zmm17, %zmm19 {%k1} {z}
 # CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpsubd	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpsubd	(%rax){1to16}, %zmm17, %zmm19 {%k1} {z}
 # CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vpsubq	%zmm16, %zmm17, %zmm19
 # CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpsubq	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpsubq	(%rax){1to8}, %zmm17, %zmm19
 # CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vpsubq	%zmm16, %zmm17, %zmm19 {%k1}
 # CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpsubq	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpsubq	(%rax){1to8}, %zmm17, %zmm19 {%k1}
 # CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vpsubq	%zmm16, %zmm17, %zmm19 {%k1} {z}
 # CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpsubq	(%rax), %zmm17, %zmm19 {%k1} {z}
-# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vpsubw	%zmm16, %zmm17, %zmm19
-# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpsubw	(%rax), %zmm17, %zmm19
-# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vpsubw	%zmm16, %zmm17, %zmm19 {%k1}
-# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpsubw	(%rax), %zmm17, %zmm19 {%k1}
-# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vpsubw	%zmm16, %zmm17, %zmm19 {%k1} {z}
-# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpsubw	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpsubq	(%rax){1to8}, %zmm17, %zmm19 {%k1} {z}
 # CHECK-NEXT:  -      -     0.50    -      -      -      -     0.50    -      -     vsubpd	%zmm16, %zmm17, %zmm19
 # CHECK-NEXT:  -      -     0.50    -     0.50   0.50    -     0.50    -      -     vsubpd	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  -      -     0.50    -     0.50   0.50    -     0.50    -      -     vsubpd	(%rax){1to8}, %zmm17, %zmm19
 # CHECK-NEXT:  -      -     0.50    -      -      -      -     0.50    -      -     vsubpd	%zmm16, %zmm17, %zmm19 {%k1}
 # CHECK-NEXT:  -      -     0.50    -     0.50   0.50    -     0.50    -      -     vsubpd	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -     0.50    -     0.50   0.50    -     0.50    -      -     vsubpd	(%rax){1to8}, %zmm17, %zmm19 {%k1}
 # CHECK-NEXT:  -      -     0.50    -      -      -      -     0.50    -      -     vsubpd	%zmm16, %zmm17, %zmm19 {%k1} {z}
 # CHECK-NEXT:  -      -     0.50    -     0.50   0.50    -     0.50    -      -     vsubpd	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.50    -     0.50   0.50    -     0.50    -      -     vsubpd	(%rax){1to8}, %zmm17, %zmm19 {%k1} {z}
 # CHECK-NEXT:  -      -     0.50    -      -      -      -     0.50    -      -     vsubps	%zmm16, %zmm17, %zmm19
 # CHECK-NEXT:  -      -     0.50    -     0.50   0.50    -     0.50    -      -     vsubps	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  -      -     0.50    -     0.50   0.50    -     0.50    -      -     vsubps	(%rax){1to16}, %zmm17, %zmm19
 # CHECK-NEXT:  -      -     0.50    -      -      -      -     0.50    -      -     vsubps	%zmm16, %zmm17, %zmm19 {%k1}
 # CHECK-NEXT:  -      -     0.50    -     0.50   0.50    -     0.50    -      -     vsubps	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -     0.50    -     0.50   0.50    -     0.50    -      -     vsubps	(%rax){1to16}, %zmm17, %zmm19 {%k1}
 # CHECK-NEXT:  -      -     0.50    -      -      -      -     0.50    -      -     vsubps	%zmm16, %zmm17, %zmm19 {%k1} {z}
 # CHECK-NEXT:  -      -     0.50    -     0.50   0.50    -     0.50    -      -     vsubps	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.50    -     0.50   0.50    -     0.50    -      -     vsubps	(%rax){1to16}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vunpckhpd	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vunpckhpd	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vunpckhpd	(%rax){1to8}, %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vunpckhpd	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vunpckhpd	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vunpckhpd	(%rax){1to8}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vunpckhpd	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vunpckhpd	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vunpckhpd	(%rax){1to8}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vunpckhps	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vunpckhps	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vunpckhps	(%rax){1to16}, %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vunpckhps	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vunpckhps	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vunpckhps	(%rax){1to16}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vunpckhps	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vunpckhps	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vunpckhps	(%rax){1to16}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vunpcklpd	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vunpcklpd	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vunpcklpd	(%rax){1to8}, %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vunpcklpd	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vunpcklpd	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vunpcklpd	(%rax){1to8}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vunpcklpd	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vunpcklpd	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vunpcklpd	(%rax){1to8}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vunpcklps	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vunpcklps	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vunpcklps	(%rax){1to16}, %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vunpcklps	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vunpcklps	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vunpcklps	(%rax){1to16}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vunpcklps	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vunpcklps	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vunpcklps	(%rax){1to16}, %zmm17, %zmm19 {%k1} {z}
diff --git a/test/tools/llvm-mca/X86/SkylakeServer/resources-avx512bw.s b/test/tools/llvm-mca/X86/SkylakeServer/resources-avx512bw.s
new file mode 100644
index 0000000000000000000000000000000000000000..a73e63c846f7264ea7214cae62b247687dcbbeee
--- /dev/null
+++ b/test/tools/llvm-mca/X86/SkylakeServer/resources-avx512bw.s
@@ -0,0 +1,297 @@
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=x86_64-unknown-unknown -mcpu=skylake-avx512 -instruction-tables < %s | FileCheck %s
+
+vpabsb            %zmm16, %zmm19
+vpabsb            (%rax), %zmm19
+vpabsb            %zmm16, %zmm19 {k1}
+vpabsb            (%rax), %zmm19 {k1}
+vpabsb            %zmm16, %zmm19 {z}{k1}
+vpabsb            (%rax), %zmm19 {z}{k1}
+
+vpabsw            %zmm16, %zmm19
+vpabsw            (%rax), %zmm19
+vpabsw            %zmm16, %zmm19 {k1}
+vpabsw            (%rax), %zmm19 {k1}
+vpabsw            %zmm16, %zmm19 {z}{k1}
+vpabsw            (%rax), %zmm19 {z}{k1}
+
+vpaddb            %zmm16, %zmm17, %zmm19
+vpaddb            (%rax), %zmm17, %zmm19
+vpaddb            %zmm16, %zmm17, %zmm19 {k1}
+vpaddb            (%rax), %zmm17, %zmm19 {k1}
+vpaddb            %zmm16, %zmm17, %zmm19 {z}{k1}
+vpaddb            (%rax), %zmm17, %zmm19 {z}{k1}
+
+vpaddw            %zmm16, %zmm17, %zmm19
+vpaddw            (%rax), %zmm17, %zmm19
+vpaddw            %zmm16, %zmm17, %zmm19 {k1}
+vpaddw            (%rax), %zmm17, %zmm19 {k1}
+vpaddw            %zmm16, %zmm17, %zmm19 {z}{k1}
+vpaddw            (%rax), %zmm17, %zmm19 {z}{k1}
+
+vpermw            %zmm16, %zmm17, %zmm19
+vpermw            (%rax), %zmm17, %zmm19
+vpermw            %zmm16, %zmm17, %zmm19 {k1}
+vpermw            (%rax), %zmm17, %zmm19 {k1}
+vpermw            %zmm16, %zmm17, %zmm19 {z}{k1}
+vpermw            (%rax), %zmm17, %zmm19 {z}{k1}
+
+vpshufb           %zmm16, %zmm17, %zmm19
+vpshufb           (%rax), %zmm17, %zmm19
+vpshufb           %zmm16, %zmm17, %zmm19 {k1}
+vpshufb           (%rax), %zmm17, %zmm19 {k1}
+vpshufb           %zmm16, %zmm17, %zmm19 {z}{k1}
+vpshufb           (%rax), %zmm17, %zmm19 {z}{k1}
+
+vpshufhw          $0, %zmm16, %zmm19
+vpshufhw          $0, (%rax), %zmm19
+vpshufhw          $0, %zmm16, %zmm19 {k1}
+vpshufhw          $0, (%rax), %zmm19 {k1}
+vpshufhw          $0, %zmm16, %zmm19 {z}{k1}
+vpshufhw          $0, (%rax), %zmm19 {z}{k1}
+
+vpshuflw          $0, %zmm16, %zmm19
+vpshuflw          $0, (%rax), %zmm19
+vpshuflw          $0, %zmm16, %zmm19 {k1}
+vpshuflw          $0, (%rax), %zmm19 {k1}
+vpshuflw          $0, %zmm16, %zmm19 {z}{k1}
+vpshuflw          $0, (%rax), %zmm19 {z}{k1}
+
+vpsubb            %zmm16, %zmm17, %zmm19
+vpsubb            (%rax), %zmm17, %zmm19
+vpsubb            %zmm16, %zmm17, %zmm19 {k1}
+vpsubb            (%rax), %zmm17, %zmm19 {k1}
+vpsubb            %zmm16, %zmm17, %zmm19 {z}{k1}
+vpsubb            (%rax), %zmm17, %zmm19 {z}{k1}
+
+vpsubw            %zmm16, %zmm17, %zmm19
+vpsubw            (%rax), %zmm17, %zmm19
+vpsubw            %zmm16, %zmm17, %zmm19 {k1}
+vpsubw            (%rax), %zmm17, %zmm19 {k1}
+vpsubw            %zmm16, %zmm17, %zmm19 {z}{k1}
+vpsubw            (%rax), %zmm17, %zmm19 {z}{k1}
+
+vpunpckhbw        %zmm16, %zmm17, %zmm19
+vpunpckhbw        (%rax), %zmm17, %zmm19
+vpunpckhbw        %zmm16, %zmm17, %zmm19 {k1}
+vpunpckhbw        (%rax), %zmm17, %zmm19 {k1}
+vpunpckhbw        %zmm16, %zmm17, %zmm19 {z}{k1}
+vpunpckhbw        (%rax), %zmm17, %zmm19 {z}{k1}
+
+vpunpckhwd        %zmm16, %zmm17, %zmm19
+vpunpckhwd        (%rax), %zmm17, %zmm19
+vpunpckhwd        %zmm16, %zmm17, %zmm19 {k1}
+vpunpckhwd        (%rax), %zmm17, %zmm19 {k1}
+vpunpckhwd        %zmm16, %zmm17, %zmm19 {z}{k1}
+vpunpckhwd        (%rax), %zmm17, %zmm19 {z}{k1}
+
+vpunpcklbw        %zmm16, %zmm17, %zmm19
+vpunpcklbw        (%rax), %zmm17, %zmm19
+vpunpcklbw        %zmm16, %zmm17, %zmm19 {k1}
+vpunpcklbw        (%rax), %zmm17, %zmm19 {k1}
+vpunpcklbw        %zmm16, %zmm17, %zmm19 {z}{k1}
+vpunpcklbw        (%rax), %zmm17, %zmm19 {z}{k1}
+
+vpunpcklwd        %zmm16, %zmm17, %zmm19
+vpunpcklwd        (%rax), %zmm17, %zmm19
+vpunpcklwd        %zmm16, %zmm17, %zmm19 {k1}
+vpunpcklwd        (%rax), %zmm17, %zmm19 {k1}
+vpunpcklwd        %zmm16, %zmm17, %zmm19 {z}{k1}
+vpunpcklwd        (%rax), %zmm17, %zmm19 {z}{k1}
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  1      1     1.00                        vpabsb	%zmm16, %zmm19
+# CHECK-NEXT:  2      8     1.00    *                   vpabsb	(%rax), %zmm19
+# CHECK-NEXT:  1      1     1.00                        vpabsb	%zmm16, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vpabsb	(%rax), %zmm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vpabsb	%zmm16, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vpabsb	(%rax), %zmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vpabsw	%zmm16, %zmm19
+# CHECK-NEXT:  2      8     1.00    *                   vpabsw	(%rax), %zmm19
+# CHECK-NEXT:  1      1     1.00                        vpabsw	%zmm16, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vpabsw	(%rax), %zmm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vpabsw	%zmm16, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vpabsw	(%rax), %zmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     0.33                        vpaddb	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  2      8     0.50    *                   vpaddb	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  1      1     0.33                        vpaddb	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     0.50    *                   vpaddb	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  1      1     0.33                        vpaddb	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     0.50    *                   vpaddb	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     0.33                        vpaddw	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  2      8     0.50    *                   vpaddw	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  1      1     0.33                        vpaddw	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     0.50    *                   vpaddw	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  1      1     0.33                        vpaddw	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     0.50    *                   vpaddw	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      6     2.00                        vpermw	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  3      13    2.00    *                   vpermw	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  2      6     2.00                        vpermw	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  3      13    2.00    *                   vpermw	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      6     2.00                        vpermw	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  3      13    2.00    *                   vpermw	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vpshufb	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  2      8     1.00    *                   vpshufb	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  1      1     1.00                        vpshufb	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vpshufb	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vpshufb	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vpshufb	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vpshufhw	$0, %zmm16, %zmm19
+# CHECK-NEXT:  2      8     1.00    *                   vpshufhw	$0, (%rax), %zmm19
+# CHECK-NEXT:  1      1     1.00                        vpshufhw	$0, %zmm16, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vpshufhw	$0, (%rax), %zmm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vpshufhw	$0, %zmm16, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vpshufhw	$0, (%rax), %zmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vpshuflw	$0, %zmm16, %zmm19
+# CHECK-NEXT:  2      8     1.00    *                   vpshuflw	$0, (%rax), %zmm19
+# CHECK-NEXT:  1      1     1.00                        vpshuflw	$0, %zmm16, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vpshuflw	$0, (%rax), %zmm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vpshuflw	$0, %zmm16, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vpshuflw	$0, (%rax), %zmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     0.33                        vpsubb	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  2      8     0.50    *                   vpsubb	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  1      1     0.33                        vpsubb	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     0.50    *                   vpsubb	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  1      1     0.33                        vpsubb	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     0.50    *                   vpsubb	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     0.33                        vpsubw	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  2      8     0.50    *                   vpsubw	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  1      1     0.33                        vpsubw	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     0.50    *                   vpsubw	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  1      1     0.33                        vpsubw	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     0.50    *                   vpsubw	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vpunpckhbw	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  2      8     1.00    *                   vpunpckhbw	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  1      1     1.00                        vpunpckhbw	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vpunpckhbw	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vpunpckhbw	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vpunpckhbw	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vpunpckhwd	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  2      8     1.00    *                   vpunpckhwd	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  1      1     1.00                        vpunpckhwd	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vpunpckhwd	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vpunpckhwd	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vpunpckhwd	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vpunpcklbw	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  2      8     1.00    *                   vpunpcklbw	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  1      1     1.00                        vpunpcklbw	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vpunpcklbw	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vpunpcklbw	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vpunpcklbw	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vpunpcklwd	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  2      8     1.00    *                   vpunpcklwd	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  1      1     1.00                        vpunpcklwd	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vpunpcklwd	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vpunpcklwd	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vpunpcklwd	(%rax), %zmm17, %zmm19 {%k1} {z}
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0]   - SKXDivider
+# CHECK-NEXT: [1]   - SKXFPDivider
+# CHECK-NEXT: [2]   - SKXPort0
+# CHECK-NEXT: [3]   - SKXPort1
+# CHECK-NEXT: [4]   - SKXPort2
+# CHECK-NEXT: [5]   - SKXPort3
+# CHECK-NEXT: [6]   - SKXPort4
+# CHECK-NEXT: [7]   - SKXPort5
+# CHECK-NEXT: [8]   - SKXPort6
+# CHECK-NEXT: [9]   - SKXPort7
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]
+# CHECK-NEXT:  -      -     20.00  8.00   21.00  21.00   -     62.00   -      -
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    Instructions:
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     vpabsb	%zmm16, %zmm19
+# CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -      -      -      -     vpabsb	(%rax), %zmm19
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     vpabsb	%zmm16, %zmm19 {%k1}
+# CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -      -      -      -     vpabsb	(%rax), %zmm19 {%k1}
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     vpabsb	%zmm16, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -      -      -      -     vpabsb	(%rax), %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     vpabsw	%zmm16, %zmm19
+# CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -      -      -      -     vpabsw	(%rax), %zmm19
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     vpabsw	%zmm16, %zmm19 {%k1}
+# CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -      -      -      -     vpabsw	(%rax), %zmm19 {%k1}
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     vpabsw	%zmm16, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -      -      -      -     vpabsw	(%rax), %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vpaddb	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpaddb	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vpaddb	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpaddb	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vpaddb	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpaddb	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vpaddw	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpaddw	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vpaddw	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpaddw	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vpaddw	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpaddw	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -     vpermw	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     2.00    -      -     vpermw	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -     vpermw	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     2.00    -      -     vpermw	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -     vpermw	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     2.00    -      -     vpermw	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpshufb	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpshufb	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpshufb	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpshufb	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpshufb	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpshufb	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpshufhw	$0, %zmm16, %zmm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpshufhw	$0, (%rax), %zmm19
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpshufhw	$0, %zmm16, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpshufhw	$0, (%rax), %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpshufhw	$0, %zmm16, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpshufhw	$0, (%rax), %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpshuflw	$0, %zmm16, %zmm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpshuflw	$0, (%rax), %zmm19
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpshuflw	$0, %zmm16, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpshuflw	$0, (%rax), %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpshuflw	$0, %zmm16, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpshuflw	$0, (%rax), %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vpsubb	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpsubb	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vpsubb	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpsubb	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vpsubb	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpsubb	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vpsubw	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpsubw	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vpsubw	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpsubw	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vpsubw	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpsubw	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpunpckhbw	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpunpckhbw	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpunpckhbw	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpunpckhbw	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpunpckhbw	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpunpckhbw	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpunpckhwd	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpunpckhwd	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpunpckhwd	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpunpckhwd	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpunpckhwd	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpunpckhwd	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpunpcklbw	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpunpcklbw	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpunpcklbw	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpunpcklbw	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpunpcklbw	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpunpcklbw	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpunpcklwd	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpunpcklwd	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpunpcklwd	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpunpcklwd	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpunpcklwd	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpunpcklwd	(%rax), %zmm17, %zmm19 {%k1} {z}
diff --git a/test/tools/llvm-mca/X86/SkylakeServer/resources-avx512bwvl.s b/test/tools/llvm-mca/X86/SkylakeServer/resources-avx512bwvl.s
new file mode 100644
index 0000000000000000000000000000000000000000..2939f1cbe4964173890a61594d9f4b2b726d0d05
--- /dev/null
+++ b/test/tools/llvm-mca/X86/SkylakeServer/resources-avx512bwvl.s
@@ -0,0 +1,525 @@
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=x86_64-unknown-unknown -mcpu=skylake-avx512 -instruction-tables < %s | FileCheck %s
+
+vpabsb            %xmm16, %xmm19
+vpabsb            (%rax), %xmm19
+vpabsb            %xmm16, %xmm19 {k1}
+vpabsb            (%rax), %xmm19 {k1}
+vpabsb            %xmm16, %xmm19 {z}{k1}
+vpabsb            (%rax), %xmm19 {z}{k1}
+
+vpabsb            %ymm16, %ymm19
+vpabsb            (%rax), %ymm19
+vpabsb            %ymm16, %ymm19 {k1}
+vpabsb            (%rax), %ymm19 {k1}
+vpabsb            %ymm16, %ymm19 {z}{k1}
+vpabsb            (%rax), %ymm19 {z}{k1}
+
+vpabsw            %xmm16, %xmm19
+vpabsw            (%rax), %xmm19
+vpabsw            %xmm16, %xmm19 {k1}
+vpabsw            (%rax), %xmm19 {k1}
+vpabsw            %xmm16, %xmm19 {z}{k1}
+vpabsw            (%rax), %xmm19 {z}{k1}
+
+vpabsw            %ymm16, %ymm19
+vpabsw            (%rax), %ymm19
+vpabsw            %ymm16, %ymm19 {k1}
+vpabsw            (%rax), %ymm19 {k1}
+vpabsw            %ymm16, %ymm19 {z}{k1}
+vpabsw            (%rax), %ymm19 {z}{k1}
+
+vpaddb            %xmm16, %xmm17, %xmm19
+vpaddb            (%rax), %xmm17, %xmm19
+vpaddb            %xmm16, %xmm17, %xmm19 {k1}
+vpaddb            (%rax), %xmm17, %xmm19 {k1}
+vpaddb            %xmm16, %xmm17, %xmm19 {z}{k1}
+vpaddb            (%rax), %xmm17, %xmm19 {z}{k1}
+
+vpaddb            %ymm16, %ymm17, %ymm19
+vpaddb            (%rax), %ymm17, %ymm19
+vpaddb            %ymm16, %ymm17, %ymm19 {k1}
+vpaddb            (%rax), %ymm17, %ymm19 {k1}
+vpaddb            %ymm16, %ymm17, %ymm19 {z}{k1}
+vpaddb            (%rax), %ymm17, %ymm19 {z}{k1}
+
+vpaddw            %xmm16, %xmm17, %xmm19
+vpaddw            (%rax), %xmm17, %xmm19
+vpaddw            %xmm16, %xmm17, %xmm19 {k1}
+vpaddw            (%rax), %xmm17, %xmm19 {k1}
+vpaddw            %xmm16, %xmm17, %xmm19 {z}{k1}
+vpaddw            (%rax), %xmm17, %xmm19 {z}{k1}
+
+vpaddw            %ymm16, %ymm17, %ymm19
+vpaddw            (%rax), %ymm17, %ymm19
+vpaddw            %ymm16, %ymm17, %ymm19 {k1}
+vpaddw            (%rax), %ymm17, %ymm19 {k1}
+vpaddw            %ymm16, %ymm17, %ymm19 {z}{k1}
+vpaddw            (%rax), %ymm17, %ymm19 {z}{k1}
+
+vpermw            %xmm16, %xmm17, %xmm19
+vpermw            (%rax), %xmm17, %xmm19
+vpermw            %xmm16, %xmm17, %xmm19 {k1}
+vpermw            (%rax), %xmm17, %xmm19 {k1}
+vpermw            %xmm16, %xmm17, %xmm19 {z}{k1}
+vpermw            (%rax), %xmm17, %xmm19 {z}{k1}
+
+vpermw            %ymm16, %ymm17, %ymm19
+vpermw            (%rax), %ymm17, %ymm19
+vpermw            %ymm16, %ymm17, %ymm19 {k1}
+vpermw            (%rax), %ymm17, %ymm19 {k1}
+vpermw            %ymm16, %ymm17, %ymm19 {z}{k1}
+vpermw            (%rax), %ymm17, %ymm19 {z}{k1}
+
+vpshufb           %xmm16, %xmm17, %xmm19
+vpshufb           (%rax), %xmm17, %xmm19
+vpshufb           %xmm16, %xmm17, %xmm19 {k1}
+vpshufb           (%rax), %xmm17, %xmm19 {k1}
+vpshufb           %xmm16, %xmm17, %xmm19 {z}{k1}
+vpshufb           (%rax), %xmm17, %xmm19 {z}{k1}
+
+vpshufb           %ymm16, %ymm17, %ymm19
+vpshufb           (%rax), %ymm17, %ymm19
+vpshufb           %ymm16, %ymm17, %ymm19 {k1}
+vpshufb           (%rax), %ymm17, %ymm19 {k1}
+vpshufb           %ymm16, %ymm17, %ymm19 {z}{k1}
+vpshufb           (%rax), %ymm17, %ymm19 {z}{k1}
+
+vpshufhw          $0, %xmm16, %xmm19
+vpshufhw          $0, (%rax), %xmm19
+vpshufhw          $0, %xmm16, %xmm19 {k1}
+vpshufhw          $0, (%rax), %xmm19 {k1}
+vpshufhw          $0, %xmm16, %xmm19 {z}{k1}
+vpshufhw          $0, (%rax), %xmm19 {z}{k1}
+
+vpshufhw          $0, %ymm16, %ymm19
+vpshufhw          $0, (%rax), %ymm19
+vpshufhw          $0, %ymm16, %ymm19 {k1}
+vpshufhw          $0, (%rax), %ymm19 {k1}
+vpshufhw          $0, %ymm16, %ymm19 {z}{k1}
+vpshufhw          $0, (%rax), %ymm19 {z}{k1}
+
+vpsubb            %xmm16, %xmm17, %xmm19
+vpsubb            (%rax), %xmm17, %xmm19
+vpsubb            %xmm16, %xmm17, %xmm19 {k1}
+vpsubb            (%rax), %xmm17, %xmm19 {k1}
+vpsubb            %xmm16, %xmm17, %xmm19 {z}{k1}
+vpsubb            (%rax), %xmm17, %xmm19 {z}{k1}
+
+vpsubb            %ymm16, %ymm17, %ymm19
+vpsubb            (%rax), %ymm17, %ymm19
+vpsubb            %ymm16, %ymm17, %ymm19 {k1}
+vpsubb            (%rax), %ymm17, %ymm19 {k1}
+vpsubb            %ymm16, %ymm17, %ymm19 {z}{k1}
+vpsubb            (%rax), %ymm17, %ymm19 {z}{k1}
+
+vpsubw            %xmm16, %xmm17, %xmm19
+vpsubw            (%rax), %xmm17, %xmm19
+vpsubw            %xmm16, %xmm17, %xmm19 {k1}
+vpsubw            (%rax), %xmm17, %xmm19 {k1}
+vpsubw            %xmm16, %xmm17, %xmm19 {z}{k1}
+vpsubw            (%rax), %xmm17, %xmm19 {z}{k1}
+
+vpsubw            %ymm16, %ymm17, %ymm19
+vpsubw            (%rax), %ymm17, %ymm19
+vpsubw            %ymm16, %ymm17, %ymm19 {k1}
+vpsubw            (%rax), %ymm17, %ymm19 {k1}
+vpsubw            %ymm16, %ymm17, %ymm19 {z}{k1}
+vpsubw            (%rax), %ymm17, %ymm19 {z}{k1}
+
+vpunpckhbw         %xmm16, %xmm17, %xmm19
+vpunpckhbw         (%rax), %xmm17, %xmm19
+vpunpckhbw         %xmm16, %xmm17, %xmm19 {k1}
+vpunpckhbw         (%rax), %xmm17, %xmm19 {k1}
+vpunpckhbw         %xmm16, %xmm17, %xmm19 {z}{k1}
+vpunpckhbw         (%rax), %xmm17, %xmm19 {z}{k1}
+
+vpunpckhbw         %ymm16, %ymm17, %ymm19
+vpunpckhbw         (%rax), %ymm17, %ymm19
+vpunpckhbw         %ymm16, %ymm17, %ymm19 {k1}
+vpunpckhbw         (%rax), %ymm17, %ymm19 {k1}
+vpunpckhbw         %ymm16, %ymm17, %ymm19 {z}{k1}
+vpunpckhbw         (%rax), %ymm17, %ymm19 {z}{k1}
+
+vpunpckhwd         %xmm16, %xmm17, %xmm19
+vpunpckhwd         (%rax), %xmm17, %xmm19
+vpunpckhwd         %xmm16, %xmm17, %xmm19 {k1}
+vpunpckhwd         (%rax), %xmm17, %xmm19 {k1}
+vpunpckhwd         %xmm16, %xmm17, %xmm19 {z}{k1}
+vpunpckhwd         (%rax), %xmm17, %xmm19 {z}{k1}
+
+vpunpckhwd         %ymm16, %ymm17, %ymm19
+vpunpckhwd         (%rax), %ymm17, %ymm19
+vpunpckhwd         %ymm16, %ymm17, %ymm19 {k1}
+vpunpckhwd         (%rax), %ymm17, %ymm19 {k1}
+vpunpckhwd         %ymm16, %ymm17, %ymm19 {z}{k1}
+vpunpckhwd         (%rax), %ymm17, %ymm19 {z}{k1}
+
+vpunpcklbw         %xmm16, %xmm17, %xmm19
+vpunpcklbw         (%rax), %xmm17, %xmm19
+vpunpcklbw         %xmm16, %xmm17, %xmm19 {k1}
+vpunpcklbw         (%rax), %xmm17, %xmm19 {k1}
+vpunpcklbw         %xmm16, %xmm17, %xmm19 {z}{k1}
+vpunpcklbw         (%rax), %xmm17, %xmm19 {z}{k1}
+
+vpunpcklbw         %ymm16, %ymm17, %ymm19
+vpunpcklbw         (%rax), %ymm17, %ymm19
+vpunpcklbw         %ymm16, %ymm17, %ymm19 {k1}
+vpunpcklbw         (%rax), %ymm17, %ymm19 {k1}
+vpunpcklbw         %ymm16, %ymm17, %ymm19 {z}{k1}
+vpunpcklbw         (%rax), %ymm17, %ymm19 {z}{k1}
+
+vpunpcklwd         %xmm16, %xmm17, %xmm19
+vpunpcklwd         (%rax), %xmm17, %xmm19
+vpunpcklwd         %xmm16, %xmm17, %xmm19 {k1}
+vpunpcklwd         (%rax), %xmm17, %xmm19 {k1}
+vpunpcklwd         %xmm16, %xmm17, %xmm19 {z}{k1}
+vpunpcklwd         (%rax), %xmm17, %xmm19 {z}{k1}
+
+vpunpcklwd         %ymm16, %ymm17, %ymm19
+vpunpcklwd         (%rax), %ymm17, %ymm19
+vpunpcklwd         %ymm16, %ymm17, %ymm19 {k1}
+vpunpcklwd         (%rax), %ymm17, %ymm19 {k1}
+vpunpcklwd         %ymm16, %ymm17, %ymm19 {z}{k1}
+vpunpcklwd         (%rax), %ymm17, %ymm19 {z}{k1}
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  1      1     0.50                        vpabsb	%xmm16, %xmm19
+# CHECK-NEXT:  2      7     0.50    *                   vpabsb	(%rax), %xmm19
+# CHECK-NEXT:  1      1     0.50                        vpabsb	%xmm16, %xmm19 {%k1}
+# CHECK-NEXT:  2      7     0.50    *                   vpabsb	(%rax), %xmm19 {%k1}
+# CHECK-NEXT:  1      1     0.50                        vpabsb	%xmm16, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      7     0.50    *                   vpabsb	(%rax), %xmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     0.50                        vpabsb	%ymm16, %ymm19
+# CHECK-NEXT:  2      8     0.50    *                   vpabsb	(%rax), %ymm19
+# CHECK-NEXT:  1      1     0.50                        vpabsb	%ymm16, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     0.50    *                   vpabsb	(%rax), %ymm19 {%k1}
+# CHECK-NEXT:  1      1     0.50                        vpabsb	%ymm16, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     0.50    *                   vpabsb	(%rax), %ymm19 {%k1} {z}
+# CHECK-NEXT:  1      1     0.50                        vpabsw	%xmm16, %xmm19
+# CHECK-NEXT:  2      7     0.50    *                   vpabsw	(%rax), %xmm19
+# CHECK-NEXT:  1      1     0.50                        vpabsw	%xmm16, %xmm19 {%k1}
+# CHECK-NEXT:  2      7     0.50    *                   vpabsw	(%rax), %xmm19 {%k1}
+# CHECK-NEXT:  1      1     0.50                        vpabsw	%xmm16, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      7     0.50    *                   vpabsw	(%rax), %xmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     0.50                        vpabsw	%ymm16, %ymm19
+# CHECK-NEXT:  2      8     0.50    *                   vpabsw	(%rax), %ymm19
+# CHECK-NEXT:  1      1     0.50                        vpabsw	%ymm16, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     0.50    *                   vpabsw	(%rax), %ymm19 {%k1}
+# CHECK-NEXT:  1      1     0.50                        vpabsw	%ymm16, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     0.50    *                   vpabsw	(%rax), %ymm19 {%k1} {z}
+# CHECK-NEXT:  1      1     0.33                        vpaddb	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  2      7     0.50    *                   vpaddb	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  1      1     0.33                        vpaddb	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      7     0.50    *                   vpaddb	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  1      1     0.33                        vpaddb	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      7     0.50    *                   vpaddb	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     0.33                        vpaddb	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  2      8     0.50    *                   vpaddb	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  1      1     0.33                        vpaddb	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     0.50    *                   vpaddb	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  1      1     0.33                        vpaddb	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     0.50    *                   vpaddb	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  1      1     0.33                        vpaddw	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  2      7     0.50    *                   vpaddw	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  1      1     0.33                        vpaddw	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      7     0.50    *                   vpaddw	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  1      1     0.33                        vpaddw	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      7     0.50    *                   vpaddw	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     0.33                        vpaddw	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  2      8     0.50    *                   vpaddw	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  1      1     0.33                        vpaddw	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     0.50    *                   vpaddw	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  1      1     0.33                        vpaddw	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     0.50    *                   vpaddw	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      6     2.00                        vpermw	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  3      12    2.00    *                   vpermw	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  2      6     2.00                        vpermw	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  3      12    2.00    *                   vpermw	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      6     2.00                        vpermw	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  3      12    2.00    *                   vpermw	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      6     2.00                        vpermw	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  3      13    2.00    *                   vpermw	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  2      6     2.00                        vpermw	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  3      13    2.00    *                   vpermw	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      6     2.00                        vpermw	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  3      13    2.00    *                   vpermw	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vpshufb	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  2      7     1.00    *                   vpshufb	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  1      1     1.00                        vpshufb	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      7     1.00    *                   vpshufb	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vpshufb	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      7     1.00    *                   vpshufb	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vpshufb	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  2      8     1.00    *                   vpshufb	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  1      1     1.00                        vpshufb	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vpshufb	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vpshufb	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vpshufb	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vpshufhw	$0, %xmm16, %xmm19
+# CHECK-NEXT:  2      7     1.00    *                   vpshufhw	$0, (%rax), %xmm19
+# CHECK-NEXT:  1      1     1.00                        vpshufhw	$0, %xmm16, %xmm19 {%k1}
+# CHECK-NEXT:  2      7     1.00    *                   vpshufhw	$0, (%rax), %xmm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vpshufhw	$0, %xmm16, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      7     1.00    *                   vpshufhw	$0, (%rax), %xmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vpshufhw	$0, %ymm16, %ymm19
+# CHECK-NEXT:  2      8     1.00    *                   vpshufhw	$0, (%rax), %ymm19
+# CHECK-NEXT:  1      1     1.00                        vpshufhw	$0, %ymm16, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vpshufhw	$0, (%rax), %ymm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vpshufhw	$0, %ymm16, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vpshufhw	$0, (%rax), %ymm19 {%k1} {z}
+# CHECK-NEXT:  1      1     0.33                        vpsubb	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  2      7     0.50    *                   vpsubb	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  1      1     0.33                        vpsubb	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      7     0.50    *                   vpsubb	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  1      1     0.33                        vpsubb	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      7     0.50    *                   vpsubb	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     0.33                        vpsubb	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  2      8     0.50    *                   vpsubb	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  1      1     0.33                        vpsubb	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     0.50    *                   vpsubb	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  1      1     0.33                        vpsubb	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     0.50    *                   vpsubb	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  1      1     0.33                        vpsubw	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  2      7     0.50    *                   vpsubw	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  1      1     0.33                        vpsubw	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      7     0.50    *                   vpsubw	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  1      1     0.33                        vpsubw	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      7     0.50    *                   vpsubw	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     0.33                        vpsubw	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  2      8     0.50    *                   vpsubw	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  1      1     0.33                        vpsubw	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     0.50    *                   vpsubw	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  1      1     0.33                        vpsubw	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     0.50    *                   vpsubw	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vpunpckhbw	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  2      7     1.00    *                   vpunpckhbw	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  1      1     1.00                        vpunpckhbw	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      7     1.00    *                   vpunpckhbw	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vpunpckhbw	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      7     1.00    *                   vpunpckhbw	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vpunpckhbw	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  2      8     1.00    *                   vpunpckhbw	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  1      1     1.00                        vpunpckhbw	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vpunpckhbw	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vpunpckhbw	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vpunpckhbw	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vpunpckhwd	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  2      7     1.00    *                   vpunpckhwd	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  1      1     1.00                        vpunpckhwd	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      7     1.00    *                   vpunpckhwd	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vpunpckhwd	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      7     1.00    *                   vpunpckhwd	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vpunpckhwd	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  2      8     1.00    *                   vpunpckhwd	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  1      1     1.00                        vpunpckhwd	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vpunpckhwd	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vpunpckhwd	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vpunpckhwd	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vpunpcklbw	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  2      7     1.00    *                   vpunpcklbw	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  1      1     1.00                        vpunpcklbw	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      7     1.00    *                   vpunpcklbw	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vpunpcklbw	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      7     1.00    *                   vpunpcklbw	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vpunpcklbw	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  2      8     1.00    *                   vpunpcklbw	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  1      1     1.00                        vpunpcklbw	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vpunpcklbw	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vpunpcklbw	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vpunpcklbw	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vpunpcklwd	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  2      7     1.00    *                   vpunpcklwd	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  1      1     1.00                        vpunpcklwd	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      7     1.00    *                   vpunpcklwd	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vpunpcklwd	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      7     1.00    *                   vpunpcklwd	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vpunpcklwd	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  2      8     1.00    *                   vpunpcklwd	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  1      1     1.00                        vpunpcklwd	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vpunpcklwd	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vpunpcklwd	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vpunpcklwd	(%rax), %ymm17, %ymm19 {%k1} {z}
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0]   - SKXDivider
+# CHECK-NEXT: [1]   - SKXFPDivider
+# CHECK-NEXT: [2]   - SKXPort0
+# CHECK-NEXT: [3]   - SKXPort1
+# CHECK-NEXT: [4]   - SKXPort2
+# CHECK-NEXT: [5]   - SKXPort3
+# CHECK-NEXT: [6]   - SKXPort4
+# CHECK-NEXT: [7]   - SKXPort5
+# CHECK-NEXT: [8]   - SKXPort6
+# CHECK-NEXT: [9]   - SKXPort7
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]
+# CHECK-NEXT:  -      -     28.00  28.00  39.00  39.00   -     112.00  -      -
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    Instructions:
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -     vpabsb	%xmm16, %xmm19
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vpabsb	(%rax), %xmm19
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -     vpabsb	%xmm16, %xmm19 {%k1}
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vpabsb	(%rax), %xmm19 {%k1}
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -     vpabsb	%xmm16, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vpabsb	(%rax), %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -     vpabsb	%ymm16, %ymm19
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vpabsb	(%rax), %ymm19
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -     vpabsb	%ymm16, %ymm19 {%k1}
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vpabsb	(%rax), %ymm19 {%k1}
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -     vpabsb	%ymm16, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vpabsb	(%rax), %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -     vpabsw	%xmm16, %xmm19
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vpabsw	(%rax), %xmm19
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -     vpabsw	%xmm16, %xmm19 {%k1}
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vpabsw	(%rax), %xmm19 {%k1}
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -     vpabsw	%xmm16, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vpabsw	(%rax), %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -     vpabsw	%ymm16, %ymm19
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vpabsw	(%rax), %ymm19
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -     vpabsw	%ymm16, %ymm19 {%k1}
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vpabsw	(%rax), %ymm19 {%k1}
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -     vpabsw	%ymm16, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vpabsw	(%rax), %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vpaddb	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpaddb	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vpaddb	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpaddb	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vpaddb	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpaddb	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vpaddb	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpaddb	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vpaddb	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpaddb	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vpaddb	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpaddb	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vpaddw	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpaddw	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vpaddw	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpaddw	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vpaddw	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpaddw	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vpaddw	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpaddw	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vpaddw	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpaddw	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vpaddw	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpaddw	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -     vpermw	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     2.00    -      -     vpermw	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -     vpermw	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     2.00    -      -     vpermw	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -     vpermw	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     2.00    -      -     vpermw	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -     vpermw	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     2.00    -      -     vpermw	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -     vpermw	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     2.00    -      -     vpermw	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -     vpermw	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     2.00    -      -     vpermw	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpshufb	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpshufb	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpshufb	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpshufb	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpshufb	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpshufb	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpshufb	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpshufb	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpshufb	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpshufb	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpshufb	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpshufb	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpshufhw	$0, %xmm16, %xmm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpshufhw	$0, (%rax), %xmm19
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpshufhw	$0, %xmm16, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpshufhw	$0, (%rax), %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpshufhw	$0, %xmm16, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpshufhw	$0, (%rax), %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpshufhw	$0, %ymm16, %ymm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpshufhw	$0, (%rax), %ymm19
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpshufhw	$0, %ymm16, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpshufhw	$0, (%rax), %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpshufhw	$0, %ymm16, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpshufhw	$0, (%rax), %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vpsubb	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpsubb	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vpsubb	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpsubb	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vpsubb	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpsubb	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vpsubb	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpsubb	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vpsubb	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpsubb	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vpsubb	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpsubb	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vpsubw	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpsubw	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vpsubw	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpsubw	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vpsubw	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpsubw	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vpsubw	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpsubw	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vpsubw	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpsubw	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vpsubw	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpsubw	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpunpckhbw	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpunpckhbw	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpunpckhbw	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpunpckhbw	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpunpckhbw	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpunpckhbw	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpunpckhbw	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpunpckhbw	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpunpckhbw	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpunpckhbw	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpunpckhbw	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpunpckhbw	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpunpckhwd	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpunpckhwd	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpunpckhwd	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpunpckhwd	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpunpckhwd	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpunpckhwd	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpunpckhwd	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpunpckhwd	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpunpckhwd	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpunpckhwd	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpunpckhwd	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpunpckhwd	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpunpcklbw	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpunpcklbw	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpunpcklbw	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpunpcklbw	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpunpcklbw	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpunpcklbw	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpunpcklbw	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpunpcklbw	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpunpcklbw	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpunpcklbw	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpunpcklbw	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpunpcklbw	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpunpcklwd	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpunpcklwd	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpunpcklwd	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpunpcklwd	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpunpcklwd	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpunpcklwd	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpunpcklwd	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpunpcklwd	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpunpcklwd	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpunpcklwd	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpunpcklwd	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpunpcklwd	(%rax), %ymm17, %ymm19 {%k1} {z}
diff --git a/test/tools/llvm-mca/X86/SkylakeServer/resources-avx512dq.s b/test/tools/llvm-mca/X86/SkylakeServer/resources-avx512dq.s
new file mode 100644
index 0000000000000000000000000000000000000000..1612ede05fb501bc03ff42e0e8b90d5b2a65c20b
--- /dev/null
+++ b/test/tools/llvm-mca/X86/SkylakeServer/resources-avx512dq.s
@@ -0,0 +1,330 @@
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=x86_64-unknown-unknown -mcpu=skylake-avx512 -instruction-tables < %s | FileCheck %s
+
+vandnpd           %zmm16, %zmm17, %zmm19
+vandnpd           (%rax), %zmm17, %zmm19
+vandnpd           (%rax){1to8}, %zmm17, %zmm19
+vandnpd           %zmm16, %zmm17, %zmm19 {k1}
+vandnpd           (%rax), %zmm17, %zmm19 {k1}
+vandnpd           (%rax){1to8}, %zmm17, %zmm19 {k1}
+vandnpd           %zmm16, %zmm17, %zmm19 {z}{k1}
+vandnpd           (%rax), %zmm17, %zmm19 {z}{k1}
+vandnpd           (%rax){1to8}, %zmm17, %zmm19 {z}{k1}
+
+vandnps           %zmm16, %zmm17, %zmm19
+vandnps           (%rax), %zmm17, %zmm19
+vandnps           (%rax){1to16}, %zmm17, %zmm19
+vandnps           %zmm16, %zmm17, %zmm19 {k1}
+vandnps           (%rax), %zmm17, %zmm19 {k1}
+vandnps           (%rax){1to16}, %zmm17, %zmm19 {k1}
+vandnps           %zmm16, %zmm17, %zmm19 {z}{k1}
+vandnps           (%rax), %zmm17, %zmm19 {z}{k1}
+vandnps           (%rax){1to16}, %zmm17, %zmm19 {z}{k1}
+
+vandpd            %zmm16, %zmm17, %zmm19
+vandpd            (%rax), %zmm17, %zmm19
+vandpd            (%rax){1to8}, %zmm17, %zmm19
+vandpd            %zmm16, %zmm17, %zmm19 {k1}
+vandpd            (%rax), %zmm17, %zmm19 {k1}
+vandpd            (%rax){1to8}, %zmm17, %zmm19 {k1}
+vandpd            %zmm16, %zmm17, %zmm19 {z}{k1}
+vandpd            (%rax), %zmm17, %zmm19 {z}{k1}
+vandpd            (%rax){1to8}, %zmm17, %zmm19 {z}{k1}
+
+vandps            %zmm16, %zmm17, %zmm19
+vandps            (%rax), %zmm17, %zmm19
+vandps            (%rax){1to16}, %zmm17, %zmm19
+vandps            %zmm16, %zmm17, %zmm19 {k1}
+vandps            (%rax), %zmm17, %zmm19 {k1}
+vandps            (%rax){1to16}, %zmm17, %zmm19 {k1}
+vandps            %zmm16, %zmm17, %zmm19 {z}{k1}
+vandps            (%rax), %zmm17, %zmm19 {z}{k1}
+vandps            (%rax){1to16}, %zmm17, %zmm19 {z}{k1}
+
+vcvtqq2pd         %zmm16, %zmm19
+vcvtqq2pd         (%rax), %zmm19
+vcvtqq2pd         (%rax){1to8}, %zmm19
+vcvtqq2pd         %zmm16, %zmm19 {k1}
+vcvtqq2pd         (%rax), %zmm19 {k1}
+vcvtqq2pd         (%rax){1to8}, %zmm19 {k1}
+vcvtqq2pd         %zmm16, %zmm19 {z}{k1}
+vcvtqq2pd         (%rax), %zmm19 {z}{k1}
+vcvtqq2pd         (%rax){1to8}, %zmm19 {z}{k1}
+
+vcvtqq2ps         %zmm16, %ymm19
+vcvtqq2ps         (%rax), %ymm19
+vcvtqq2ps         (%rax){1to8}, %ymm19
+vcvtqq2ps         %zmm16, %ymm19 {k1}
+vcvtqq2ps         (%rax), %ymm19 {k1}
+vcvtqq2ps         (%rax){1to8}, %ymm19 {k1}
+vcvtqq2ps         %zmm16, %ymm19 {z}{k1}
+vcvtqq2ps         (%rax), %ymm19 {z}{k1}
+vcvtqq2ps         (%rax){1to8}, %ymm19 {z}{k1}
+
+vorpd             %zmm16, %zmm17, %zmm19
+vorpd             (%rax), %zmm17, %zmm19
+vorpd             (%rax){1to8}, %zmm17, %zmm19
+vorpd             %zmm16, %zmm17, %zmm19 {k1}
+vorpd             (%rax), %zmm17, %zmm19 {k1}
+vorpd             (%rax){1to8}, %zmm17, %zmm19 {k1}
+vorpd             %zmm16, %zmm17, %zmm19 {z}{k1}
+vorpd             (%rax), %zmm17, %zmm19 {z}{k1}
+vorpd             (%rax){1to8}, %zmm17, %zmm19 {z}{k1}
+
+vorps             %zmm16, %zmm17, %zmm19
+vorps             (%rax), %zmm17, %zmm19
+vorps             (%rax){1to16}, %zmm17, %zmm19
+vorps             %zmm16, %zmm17, %zmm19 {k1}
+vorps             (%rax), %zmm17, %zmm19 {k1}
+vorps             (%rax){1to16}, %zmm17, %zmm19 {k1}
+vorps             %zmm16, %zmm17, %zmm19 {z}{k1}
+vorps             (%rax), %zmm17, %zmm19 {z}{k1}
+vorps             (%rax){1to16}, %zmm17, %zmm19 {z}{k1}
+
+vpmullq           %zmm16, %zmm17, %zmm19
+vpmullq           (%rax), %zmm17, %zmm19
+vpmullq           %zmm16, %zmm17, %zmm19 {k1}
+vpmullq           (%rax), %zmm17, %zmm19 {k1}
+vpmullq           %zmm16, %zmm17, %zmm19 {z}{k1}
+vpmullq           (%rax), %zmm17, %zmm19 {z}{k1}
+
+vxorpd            %zmm16, %zmm17, %zmm19
+vxorpd            (%rax), %zmm17, %zmm19
+vxorpd            (%rax){1to8}, %zmm17, %zmm19
+vxorpd            %zmm16, %zmm17, %zmm19 {k1}
+vxorpd            (%rax), %zmm17, %zmm19 {k1}
+vxorpd            (%rax){1to8}, %zmm17, %zmm19 {k1}
+vxorpd            %zmm16, %zmm17, %zmm19 {z}{k1}
+vxorpd            (%rax), %zmm17, %zmm19 {z}{k1}
+vxorpd            (%rax){1to8}, %zmm17, %zmm19 {z}{k1}
+
+vxorps            %zmm16, %zmm17, %zmm19
+vxorps            (%rax), %zmm17, %zmm19
+vxorps            (%rax){1to16}, %zmm17, %zmm19
+vxorps            %zmm16, %zmm17, %zmm19 {k1}
+vxorps            (%rax), %zmm17, %zmm19 {k1}
+vxorps            (%rax){1to16}, %zmm17, %zmm19 {k1}
+vxorps            %zmm16, %zmm17, %zmm19 {z}{k1}
+vxorps            (%rax), %zmm17, %zmm19 {z}{k1}
+vxorps            (%rax){1to16}, %zmm17, %zmm19 {z}{k1}
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  1      1     0.50                        vandnpd	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  2      8     0.50    *                   vandnpd	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  2      8     0.50    *                   vandnpd	(%rax){1to8}, %zmm17, %zmm19
+# CHECK-NEXT:  1      1     0.50                        vandnpd	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     0.50    *                   vandnpd	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     0.50    *                   vandnpd	(%rax){1to8}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  1      1     0.50                        vandnpd	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     0.50    *                   vandnpd	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     0.50    *                   vandnpd	(%rax){1to8}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     0.50                        vandnps	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  2      8     0.50    *                   vandnps	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  2      8     0.50    *                   vandnps	(%rax){1to16}, %zmm17, %zmm19
+# CHECK-NEXT:  1      1     0.50                        vandnps	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     0.50    *                   vandnps	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     0.50    *                   vandnps	(%rax){1to16}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  1      1     0.50                        vandnps	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     0.50    *                   vandnps	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     0.50    *                   vandnps	(%rax){1to16}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     0.50                        vandpd	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  2      8     0.50    *                   vandpd	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  2      8     0.50    *                   vandpd	(%rax){1to8}, %zmm17, %zmm19
+# CHECK-NEXT:  1      1     0.50                        vandpd	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     0.50    *                   vandpd	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     0.50    *                   vandpd	(%rax){1to8}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  1      1     0.50                        vandpd	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     0.50    *                   vandpd	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     0.50    *                   vandpd	(%rax){1to8}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     0.50                        vandps	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  2      8     0.50    *                   vandps	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  2      8     0.50    *                   vandps	(%rax){1to16}, %zmm17, %zmm19
+# CHECK-NEXT:  1      1     0.50                        vandps	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     0.50    *                   vandps	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     0.50    *                   vandps	(%rax){1to16}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  1      1     0.50                        vandps	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     0.50    *                   vandps	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     0.50    *                   vandps	(%rax){1to16}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  1      4     0.50                        vcvtqq2pd	%zmm16, %zmm19
+# CHECK-NEXT:  2      11    0.50    *                   vcvtqq2pd	(%rax), %zmm19
+# CHECK-NEXT:  2      11    0.50    *                   vcvtqq2pd	(%rax){1to8}, %zmm19
+# CHECK-NEXT:  1      4     0.50                        vcvtqq2pd	%zmm16, %zmm19 {%k1}
+# CHECK-NEXT:  2      11    0.50    *                   vcvtqq2pd	(%rax), %zmm19 {%k1}
+# CHECK-NEXT:  2      11    0.50    *                   vcvtqq2pd	(%rax){1to8}, %zmm19 {%k1}
+# CHECK-NEXT:  1      4     0.50                        vcvtqq2pd	%zmm16, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      11    0.50    *                   vcvtqq2pd	(%rax), %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      11    0.50    *                   vcvtqq2pd	(%rax){1to8}, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      7     1.00                        vcvtqq2ps	%zmm16, %ymm19
+# CHECK-NEXT:  3      14    1.00    *                   vcvtqq2ps	(%rax), %ymm19
+# CHECK-NEXT:  3      14    1.00    *                   vcvtqq2ps	(%rax){1to8}, %ymm19
+# CHECK-NEXT:  1      4     0.50                        vcvtqq2ps	%zmm16, %ymm19 {%k1}
+# CHECK-NEXT:  3      14    1.00    *                   vcvtqq2ps	(%rax), %ymm19 {%k1}
+# CHECK-NEXT:  3      14    1.00    *                   vcvtqq2ps	(%rax){1to8}, %ymm19 {%k1}
+# CHECK-NEXT:  1      4     0.50                        vcvtqq2ps	%zmm16, %ymm19 {%k1} {z}
+# CHECK-NEXT:  3      14    1.00    *                   vcvtqq2ps	(%rax), %ymm19 {%k1} {z}
+# CHECK-NEXT:  3      14    1.00    *                   vcvtqq2ps	(%rax){1to8}, %ymm19 {%k1} {z}
+# CHECK-NEXT:  1      1     0.50                        vorpd	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  2      8     0.50    *                   vorpd	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  2      8     0.50    *                   vorpd	(%rax){1to8}, %zmm17, %zmm19
+# CHECK-NEXT:  1      1     0.50                        vorpd	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     0.50    *                   vorpd	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     0.50    *                   vorpd	(%rax){1to8}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  1      1     0.50                        vorpd	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     0.50    *                   vorpd	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     0.50    *                   vorpd	(%rax){1to8}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     0.50                        vorps	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  2      8     0.50    *                   vorps	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  2      8     0.50    *                   vorps	(%rax){1to16}, %zmm17, %zmm19
+# CHECK-NEXT:  1      1     0.50                        vorps	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     0.50    *                   vorps	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     0.50    *                   vorps	(%rax){1to16}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  1      1     0.50                        vorps	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     0.50    *                   vorps	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     0.50    *                   vorps	(%rax){1to16}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  3      12    1.50                        vpmullq	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  4      19    1.00    *                   vpmullq	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  3      12    1.50                        vpmullq	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  4      19    1.00    *                   vpmullq	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  3      12    1.50                        vpmullq	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  4      19    1.00    *                   vpmullq	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     0.50                        vxorpd	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  2      8     0.50    *                   vxorpd	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  2      8     0.50    *                   vxorpd	(%rax){1to8}, %zmm17, %zmm19
+# CHECK-NEXT:  1      1     0.50                        vxorpd	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     0.50    *                   vxorpd	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     0.50    *                   vxorpd	(%rax){1to8}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  1      1     0.50                        vxorpd	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     0.50    *                   vxorpd	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     0.50    *                   vxorpd	(%rax){1to8}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     0.50                        vxorps	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  2      8     0.50    *                   vxorps	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  2      8     0.50    *                   vxorps	(%rax){1to16}, %zmm17, %zmm19
+# CHECK-NEXT:  1      1     0.50                        vxorps	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     0.50    *                   vxorps	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  2      8     0.50    *                   vxorps	(%rax){1to16}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  1      1     0.50                        vxorps	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     0.50    *                   vxorps	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  2      8     0.50    *                   vxorps	(%rax){1to16}, %zmm17, %zmm19 {%k1} {z}
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0]   - SKXDivider
+# CHECK-NEXT: [1]   - SKXFPDivider
+# CHECK-NEXT: [2]   - SKXPort0
+# CHECK-NEXT: [3]   - SKXPort1
+# CHECK-NEXT: [4]   - SKXPort2
+# CHECK-NEXT: [5]   - SKXPort3
+# CHECK-NEXT: [6]   - SKXPort4
+# CHECK-NEXT: [7]   - SKXPort5
+# CHECK-NEXT: [8]   - SKXPort6
+# CHECK-NEXT: [9]   - SKXPort7
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]
+# CHECK-NEXT:  -      -     50.50  7.00   31.50  31.50   -     57.50   -      -
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    Instructions:
+# CHECK-NEXT:  -      -     0.50    -      -      -      -     0.50    -      -     vandnpd	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  -      -     0.50    -     0.50   0.50    -     0.50    -      -     vandnpd	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  -      -     0.50    -     0.50   0.50    -     0.50    -      -     vandnpd	(%rax){1to8}, %zmm17, %zmm19
+# CHECK-NEXT:  -      -     0.50    -      -      -      -     0.50    -      -     vandnpd	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -     0.50    -     0.50   0.50    -     0.50    -      -     vandnpd	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -     0.50    -     0.50   0.50    -     0.50    -      -     vandnpd	(%rax){1to8}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -     0.50    -      -      -      -     0.50    -      -     vandnpd	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.50    -     0.50   0.50    -     0.50    -      -     vandnpd	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.50    -     0.50   0.50    -     0.50    -      -     vandnpd	(%rax){1to8}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.50    -      -      -      -     0.50    -      -     vandnps	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  -      -     0.50    -     0.50   0.50    -     0.50    -      -     vandnps	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  -      -     0.50    -     0.50   0.50    -     0.50    -      -     vandnps	(%rax){1to16}, %zmm17, %zmm19
+# CHECK-NEXT:  -      -     0.50    -      -      -      -     0.50    -      -     vandnps	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -     0.50    -     0.50   0.50    -     0.50    -      -     vandnps	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -     0.50    -     0.50   0.50    -     0.50    -      -     vandnps	(%rax){1to16}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -     0.50    -      -      -      -     0.50    -      -     vandnps	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.50    -     0.50   0.50    -     0.50    -      -     vandnps	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.50    -     0.50   0.50    -     0.50    -      -     vandnps	(%rax){1to16}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.50    -      -      -      -     0.50    -      -     vandpd	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  -      -     0.50    -     0.50   0.50    -     0.50    -      -     vandpd	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  -      -     0.50    -     0.50   0.50    -     0.50    -      -     vandpd	(%rax){1to8}, %zmm17, %zmm19
+# CHECK-NEXT:  -      -     0.50    -      -      -      -     0.50    -      -     vandpd	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -     0.50    -     0.50   0.50    -     0.50    -      -     vandpd	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -     0.50    -     0.50   0.50    -     0.50    -      -     vandpd	(%rax){1to8}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -     0.50    -      -      -      -     0.50    -      -     vandpd	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.50    -     0.50   0.50    -     0.50    -      -     vandpd	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.50    -     0.50   0.50    -     0.50    -      -     vandpd	(%rax){1to8}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.50    -      -      -      -     0.50    -      -     vandps	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  -      -     0.50    -     0.50   0.50    -     0.50    -      -     vandps	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  -      -     0.50    -     0.50   0.50    -     0.50    -      -     vandps	(%rax){1to16}, %zmm17, %zmm19
+# CHECK-NEXT:  -      -     0.50    -      -      -      -     0.50    -      -     vandps	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -     0.50    -     0.50   0.50    -     0.50    -      -     vandps	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -     0.50    -     0.50   0.50    -     0.50    -      -     vandps	(%rax){1to16}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -     0.50    -      -      -      -     0.50    -      -     vandps	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.50    -     0.50   0.50    -     0.50    -      -     vandps	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.50    -     0.50   0.50    -     0.50    -      -     vandps	(%rax){1to16}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.50    -      -      -      -     0.50    -      -     vcvtqq2pd	%zmm16, %zmm19
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vcvtqq2pd	(%rax), %zmm19
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vcvtqq2pd	(%rax){1to8}, %zmm19
+# CHECK-NEXT:  -      -     0.50    -      -      -      -     0.50    -      -     vcvtqq2pd	%zmm16, %zmm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vcvtqq2pd	(%rax), %zmm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vcvtqq2pd	(%rax){1to8}, %zmm19 {%k1}
+# CHECK-NEXT:  -      -     0.50    -      -      -      -     0.50    -      -     vcvtqq2pd	%zmm16, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vcvtqq2pd	(%rax), %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vcvtqq2pd	(%rax){1to8}, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.50    -      -      -      -     1.50    -      -     vcvtqq2ps	%zmm16, %ymm19
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     1.33    -      -     vcvtqq2ps	(%rax), %ymm19
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     1.33    -      -     vcvtqq2ps	(%rax){1to8}, %ymm19
+# CHECK-NEXT:  -      -     0.50    -      -      -      -     0.50    -      -     vcvtqq2ps	%zmm16, %ymm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     1.33    -      -     vcvtqq2ps	(%rax), %ymm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     1.33    -      -     vcvtqq2ps	(%rax){1to8}, %ymm19 {%k1}
+# CHECK-NEXT:  -      -     0.50    -      -      -      -     0.50    -      -     vcvtqq2ps	%zmm16, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     1.33    -      -     vcvtqq2ps	(%rax), %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     1.33    -      -     vcvtqq2ps	(%rax){1to8}, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.50    -      -      -      -     0.50    -      -     vorpd	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  -      -     0.50    -     0.50   0.50    -     0.50    -      -     vorpd	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  -      -     0.50    -     0.50   0.50    -     0.50    -      -     vorpd	(%rax){1to8}, %zmm17, %zmm19
+# CHECK-NEXT:  -      -     0.50    -      -      -      -     0.50    -      -     vorpd	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -     0.50    -     0.50   0.50    -     0.50    -      -     vorpd	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -     0.50    -     0.50   0.50    -     0.50    -      -     vorpd	(%rax){1to8}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -     0.50    -      -      -      -     0.50    -      -     vorpd	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.50    -     0.50   0.50    -     0.50    -      -     vorpd	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.50    -     0.50   0.50    -     0.50    -      -     vorpd	(%rax){1to8}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.50    -      -      -      -     0.50    -      -     vorps	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  -      -     0.50    -     0.50   0.50    -     0.50    -      -     vorps	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  -      -     0.50    -     0.50   0.50    -     0.50    -      -     vorps	(%rax){1to16}, %zmm17, %zmm19
+# CHECK-NEXT:  -      -     0.50    -      -      -      -     0.50    -      -     vorps	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -     0.50    -     0.50   0.50    -     0.50    -      -     vorps	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -     0.50    -     0.50   0.50    -     0.50    -      -     vorps	(%rax){1to16}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -     0.50    -      -      -      -     0.50    -      -     vorps	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.50    -     0.50   0.50    -     0.50    -      -     vorps	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.50    -     0.50   0.50    -     0.50    -      -     vorps	(%rax){1to16}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     1.50    -      -      -      -     1.50    -      -     vpmullq	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  -      -     1.00   1.00   0.50   0.50    -     1.00    -      -     vpmullq	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  -      -     1.50    -      -      -      -     1.50    -      -     vpmullq	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -     1.00   1.00   0.50   0.50    -     1.00    -      -     vpmullq	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -     1.50    -      -      -      -     1.50    -      -     vpmullq	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     1.00   1.00   0.50   0.50    -     1.00    -      -     vpmullq	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.50    -      -      -      -     0.50    -      -     vxorpd	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  -      -     0.50    -     0.50   0.50    -     0.50    -      -     vxorpd	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  -      -     0.50    -     0.50   0.50    -     0.50    -      -     vxorpd	(%rax){1to8}, %zmm17, %zmm19
+# CHECK-NEXT:  -      -     0.50    -      -      -      -     0.50    -      -     vxorpd	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -     0.50    -     0.50   0.50    -     0.50    -      -     vxorpd	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -     0.50    -     0.50   0.50    -     0.50    -      -     vxorpd	(%rax){1to8}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -     0.50    -      -      -      -     0.50    -      -     vxorpd	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.50    -     0.50   0.50    -     0.50    -      -     vxorpd	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.50    -     0.50   0.50    -     0.50    -      -     vxorpd	(%rax){1to8}, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.50    -      -      -      -     0.50    -      -     vxorps	%zmm16, %zmm17, %zmm19
+# CHECK-NEXT:  -      -     0.50    -     0.50   0.50    -     0.50    -      -     vxorps	(%rax), %zmm17, %zmm19
+# CHECK-NEXT:  -      -     0.50    -     0.50   0.50    -     0.50    -      -     vxorps	(%rax){1to16}, %zmm17, %zmm19
+# CHECK-NEXT:  -      -     0.50    -      -      -      -     0.50    -      -     vxorps	%zmm16, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -     0.50    -     0.50   0.50    -     0.50    -      -     vxorps	(%rax), %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -     0.50    -     0.50   0.50    -     0.50    -      -     vxorps	(%rax){1to16}, %zmm17, %zmm19 {%k1}
+# CHECK-NEXT:  -      -     0.50    -      -      -      -     0.50    -      -     vxorps	%zmm16, %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.50    -     0.50   0.50    -     0.50    -      -     vxorps	(%rax), %zmm17, %zmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.50    -     0.50   0.50    -     0.50    -      -     vxorps	(%rax){1to16}, %zmm17, %zmm19 {%k1} {z}
diff --git a/test/tools/llvm-mca/X86/SkylakeServer/resources-avx512dqvl.s b/test/tools/llvm-mca/X86/SkylakeServer/resources-avx512dqvl.s
new file mode 100644
index 0000000000000000000000000000000000000000..f6b3cd63833ee72b877c6cb4a1a2f623426bc918
--- /dev/null
+++ b/test/tools/llvm-mca/X86/SkylakeServer/resources-avx512dqvl.s
@@ -0,0 +1,517 @@
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=x86_64-unknown-unknown -mcpu=skylake-avx512 -instruction-tables < %s | FileCheck %s
+
+vandnpd           %xmm16, %xmm17, %xmm19
+vandnpd           (%rax), %xmm17, %xmm19
+vandnpd           (%rax){1to2}, %xmm17, %xmm19
+vandnpd           %xmm16, %xmm17, %xmm19 {k1}
+vandnpd           (%rax), %xmm17, %xmm19 {k1}
+vandnpd           (%rax){1to2}, %xmm17, %xmm19 {k1}
+vandnpd           %xmm16, %xmm17, %xmm19 {z}{k1}
+vandnpd           (%rax), %xmm17, %xmm19 {z}{k1}
+vandnpd           (%rax){1to2}, %xmm17, %xmm19 {z}{k1}
+
+vandnpd           %ymm16, %ymm17, %ymm19
+vandnpd           (%rax), %ymm17, %ymm19
+vandnpd           (%rax){1to4}, %ymm17, %ymm19
+vandnpd           %ymm16, %ymm17, %ymm19 {k1}
+vandnpd           (%rax), %ymm17, %ymm19 {k1}
+vandnpd           (%rax){1to4}, %ymm17, %ymm19 {k1}
+vandnpd           %ymm16, %ymm17, %ymm19 {z}{k1}
+vandnpd           (%rax), %ymm17, %ymm19 {z}{k1}
+vandnpd           (%rax){1to4}, %ymm17, %ymm19 {z}{k1}
+
+vandpd            %xmm16, %xmm17, %xmm19
+vandpd            (%rax), %xmm17, %xmm19
+vandpd            (%rax){1to2}, %xmm17, %xmm19
+vandpd            %xmm16, %xmm17, %xmm19 {k1}
+vandpd            (%rax), %xmm17, %xmm19 {k1}
+vandpd            (%rax){1to2}, %xmm17, %xmm19 {k1}
+vandpd            %xmm16, %xmm17, %xmm19 {z}{k1}
+vandpd            (%rax), %xmm17, %xmm19 {z}{k1}
+vandpd            (%rax){1to2}, %xmm17, %xmm19 {z}{k1}
+
+vandpd            %ymm16, %ymm17, %ymm19
+vandpd            (%rax), %ymm17, %ymm19
+vandpd            (%rax){1to4}, %ymm17, %ymm19
+vandpd            %ymm16, %ymm17, %ymm19 {k1}
+vandpd            (%rax), %ymm17, %ymm19 {k1}
+vandpd            (%rax){1to4}, %ymm17, %ymm19 {k1}
+vandpd            %ymm16, %ymm17, %ymm19 {z}{k1}
+vandpd            (%rax), %ymm17, %ymm19 {z}{k1}
+vandpd            (%rax){1to4}, %ymm17, %ymm19 {z}{k1}
+
+vandps            %xmm16, %xmm17, %xmm19
+vandps            (%rax), %xmm17, %xmm19
+vandps            (%rax){1to4}, %xmm17, %xmm19
+vandps            %xmm16, %xmm17, %xmm19 {k1}
+vandps            (%rax), %xmm17, %xmm19 {k1}
+vandps            (%rax){1to4}, %xmm17, %xmm19 {k1}
+vandps            %xmm16, %xmm17, %xmm19 {z}{k1}
+vandps            (%rax), %xmm17, %xmm19 {z}{k1}
+vandps            (%rax){1to4}, %xmm17, %xmm19 {z}{k1}
+
+vandps            %ymm16, %ymm17, %ymm19
+vandps            (%rax), %ymm17, %ymm19
+vandps            (%rax){1to8}, %ymm17, %ymm19
+vandps            %ymm16, %ymm17, %ymm19 {k1}
+vandps            (%rax), %ymm17, %ymm19 {k1}
+vandps            (%rax){1to8}, %ymm17, %ymm19 {k1}
+vandps            %ymm16, %ymm17, %ymm19 {z}{k1}
+vandps            (%rax), %ymm17, %ymm19 {z}{k1}
+vandps            (%rax){1to8}, %ymm17, %ymm19 {z}{k1}
+
+vcvtqq2pd         %xmm16, %xmm19
+vcvtqq2pd         (%rax), %xmm19
+vcvtqq2pd         (%rax){1to2}, %xmm19
+vcvtqq2pd         %xmm16, %xmm19 {k1}
+vcvtqq2pd         (%rax), %xmm19 {k1}
+vcvtqq2pd         (%rax){1to2}, %xmm19 {k1}
+vcvtqq2pd         %xmm16, %xmm19 {z}{k1}
+vcvtqq2pd         (%rax), %xmm19 {z}{k1}
+vcvtqq2pd         (%rax){1to2}, %xmm19 {z}{k1}
+
+vcvtqq2pd         %ymm16, %ymm19
+vcvtqq2pd         (%rax), %ymm19
+vcvtqq2pd         (%rax){1to4}, %ymm19
+vcvtqq2pd         %ymm16, %ymm19 {k1}
+vcvtqq2pd         (%rax), %ymm19 {k1}
+vcvtqq2pd         (%rax){1to4}, %ymm19 {k1}
+vcvtqq2pd         %ymm16, %ymm19 {z}{k1}
+vcvtqq2pd         (%rax), %ymm19 {z}{k1}
+vcvtqq2pd         (%rax){1to4}, %ymm19 {z}{k1}
+
+vorpd             %xmm16, %xmm17, %xmm19
+vorpd             (%rax), %xmm17, %xmm19
+vorpd             (%rax){1to2}, %xmm17, %xmm19
+vorpd             %xmm16, %xmm17, %xmm19 {k1}
+vorpd             (%rax), %xmm17, %xmm19 {k1}
+vorpd             (%rax){1to2}, %xmm17, %xmm19 {k1}
+vorpd             %xmm16, %xmm17, %xmm19 {z}{k1}
+vorpd             (%rax), %xmm17, %xmm19 {z}{k1}
+vorpd             (%rax){1to2}, %xmm17, %xmm19 {z}{k1}
+
+vorpd             %ymm16, %ymm17, %ymm19
+vorpd             (%rax), %ymm17, %ymm19
+vorpd             (%rax){1to4}, %ymm17, %ymm19
+vorpd             %ymm16, %ymm17, %ymm19 {k1}
+vorpd             (%rax), %ymm17, %ymm19 {k1}
+vorpd             (%rax){1to4}, %ymm17, %ymm19 {k1}
+vorpd             %ymm16, %ymm17, %ymm19 {z}{k1}
+vorpd             (%rax), %ymm17, %ymm19 {z}{k1}
+vorpd             (%rax){1to4}, %ymm17, %ymm19 {z}{k1}
+
+vorps             %xmm16, %xmm17, %xmm19
+vorps             (%rax), %xmm17, %xmm19
+vorps             (%rax){1to4}, %xmm17, %xmm19
+vorps             %xmm16, %xmm17, %xmm19 {k1}
+vorps             (%rax), %xmm17, %xmm19 {k1}
+vorps             (%rax){1to4}, %xmm17, %xmm19 {k1}
+vorps             %xmm16, %xmm17, %xmm19 {z}{k1}
+vorps             (%rax), %xmm17, %xmm19 {z}{k1}
+vorps             (%rax){1to4}, %xmm17, %xmm19 {z}{k1}
+
+vorps             %ymm16, %ymm17, %ymm19
+vorps             (%rax), %ymm17, %ymm19
+vorps             (%rax){1to8}, %ymm17, %ymm19
+vorps             %ymm16, %ymm17, %ymm19 {k1}
+vorps             (%rax), %ymm17, %ymm19 {k1}
+vorps             (%rax){1to8}, %ymm17, %ymm19 {k1}
+vorps             %ymm16, %ymm17, %ymm19 {z}{k1}
+vorps             (%rax), %ymm17, %ymm19 {z}{k1}
+vorps             (%rax){1to8}, %ymm17, %ymm19 {z}{k1}
+
+vpmullq           %xmm16, %xmm17, %xmm19
+vpmullq           (%rax), %xmm17, %xmm19
+vpmullq           %xmm16, %xmm17, %xmm19 {k1}
+vpmullq           (%rax), %xmm17, %xmm19 {k1}
+vpmullq           %xmm16, %xmm17, %xmm19 {z}{k1}
+vpmullq           (%rax), %xmm17, %xmm19 {z}{k1}
+
+vpmullq           %ymm16, %ymm17, %ymm19
+vpmullq           (%rax), %ymm17, %ymm19
+vpmullq           %ymm16, %ymm17, %ymm19 {k1}
+vpmullq           (%rax), %ymm17, %ymm19 {k1}
+vpmullq           %ymm16, %ymm17, %ymm19 {z}{k1}
+vpmullq           (%rax), %ymm17, %ymm19 {z}{k1}
+
+vxorpd            %xmm16, %xmm17, %xmm19
+vxorpd            (%rax), %xmm17, %xmm19
+vxorpd            (%rax){1to2}, %xmm17, %xmm19
+vxorpd            %xmm16, %xmm17, %xmm19 {k1}
+vxorpd            (%rax), %xmm17, %xmm19 {k1}
+vxorpd            (%rax){1to2}, %xmm17, %xmm19 {k1}
+vxorpd            %xmm16, %xmm17, %xmm19 {z}{k1}
+vxorpd            (%rax), %xmm17, %xmm19 {z}{k1}
+vxorpd            (%rax){1to2}, %xmm17, %xmm19 {z}{k1}
+
+vxorpd            %ymm16, %ymm17, %ymm19
+vxorpd            (%rax), %ymm17, %ymm19
+vxorpd            (%rax){1to4}, %ymm17, %ymm19
+vxorpd            %ymm16, %ymm17, %ymm19 {k1}
+vxorpd            (%rax), %ymm17, %ymm19 {k1}
+vxorpd            (%rax){1to4}, %ymm17, %ymm19 {k1}
+vxorpd            %ymm16, %ymm17, %ymm19 {z}{k1}
+vxorpd            (%rax), %ymm17, %ymm19 {z}{k1}
+vxorpd            (%rax){1to4}, %ymm17, %ymm19 {z}{k1}
+
+vxorps            %xmm16, %xmm17, %xmm19
+vxorps            (%rax), %xmm17, %xmm19
+vxorps            (%rax){1to4}, %xmm17, %xmm19
+vxorps            %xmm16, %xmm17, %xmm19 {k1}
+vxorps            (%rax), %xmm17, %xmm19 {k1}
+vxorps            (%rax){1to4}, %xmm17, %xmm19 {k1}
+vxorps            %xmm16, %xmm17, %xmm19 {z}{k1}
+vxorps            (%rax), %xmm17, %xmm19 {z}{k1}
+vxorps            (%rax){1to4}, %xmm17, %xmm19 {z}{k1}
+
+vxorps            %ymm16, %ymm17, %ymm19
+vxorps            (%rax), %ymm17, %ymm19
+vxorps            (%rax){1to8}, %ymm17, %ymm19
+vxorps            %ymm16, %ymm17, %ymm19 {k1}
+vxorps            (%rax), %ymm17, %ymm19 {k1}
+vxorps            (%rax){1to8}, %ymm17, %ymm19 {k1}
+vxorps            %ymm16, %ymm17, %ymm19 {z}{k1}
+vxorps            (%rax), %ymm17, %ymm19 {z}{k1}
+vxorps            (%rax){1to8}, %ymm17, %ymm19 {z}{k1}
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  1      1     0.33                        vandnpd	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  2      7     0.50    *                   vandnpd	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  2      7     0.50    *                   vandnpd	(%rax){1to2}, %xmm17, %xmm19
+# CHECK-NEXT:  1      1     0.33                        vandnpd	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      7     0.50    *                   vandnpd	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      7     0.50    *                   vandnpd	(%rax){1to2}, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  1      1     0.33                        vandnpd	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      7     0.50    *                   vandnpd	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      7     0.50    *                   vandnpd	(%rax){1to2}, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     0.33                        vandnpd	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  2      8     0.50    *                   vandnpd	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  2      8     0.50    *                   vandnpd	(%rax){1to4}, %ymm17, %ymm19
+# CHECK-NEXT:  1      1     0.33                        vandnpd	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     0.50    *                   vandnpd	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     0.50    *                   vandnpd	(%rax){1to4}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  1      1     0.33                        vandnpd	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     0.50    *                   vandnpd	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     0.50    *                   vandnpd	(%rax){1to4}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  1      1     0.33                        vandpd	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  2      7     0.50    *                   vandpd	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  2      7     0.50    *                   vandpd	(%rax){1to2}, %xmm17, %xmm19
+# CHECK-NEXT:  1      1     0.33                        vandpd	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      7     0.50    *                   vandpd	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      7     0.50    *                   vandpd	(%rax){1to2}, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  1      1     0.33                        vandpd	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      7     0.50    *                   vandpd	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      7     0.50    *                   vandpd	(%rax){1to2}, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     0.33                        vandpd	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  2      8     0.50    *                   vandpd	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  2      8     0.50    *                   vandpd	(%rax){1to4}, %ymm17, %ymm19
+# CHECK-NEXT:  1      1     0.33                        vandpd	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     0.50    *                   vandpd	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     0.50    *                   vandpd	(%rax){1to4}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  1      1     0.33                        vandpd	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     0.50    *                   vandpd	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     0.50    *                   vandpd	(%rax){1to4}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  1      1     0.33                        vandps	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  2      7     0.50    *                   vandps	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  2      7     0.50    *                   vandps	(%rax){1to4}, %xmm17, %xmm19
+# CHECK-NEXT:  1      1     0.33                        vandps	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      7     0.50    *                   vandps	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      7     0.50    *                   vandps	(%rax){1to4}, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  1      1     0.33                        vandps	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      7     0.50    *                   vandps	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      7     0.50    *                   vandps	(%rax){1to4}, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     0.33                        vandps	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  2      8     0.50    *                   vandps	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  2      8     0.50    *                   vandps	(%rax){1to8}, %ymm17, %ymm19
+# CHECK-NEXT:  1      1     0.33                        vandps	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     0.50    *                   vandps	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     0.50    *                   vandps	(%rax){1to8}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  1      1     0.33                        vandps	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     0.50    *                   vandps	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     0.50    *                   vandps	(%rax){1to8}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  1      4     0.50                        vcvtqq2pd	%xmm16, %xmm19
+# CHECK-NEXT:  2      10    0.50    *                   vcvtqq2pd	(%rax), %xmm19
+# CHECK-NEXT:  2      10    0.50    *                   vcvtqq2pd	(%rax){1to2}, %xmm19
+# CHECK-NEXT:  1      4     0.50                        vcvtqq2pd	%xmm16, %xmm19 {%k1}
+# CHECK-NEXT:  2      10    0.50    *                   vcvtqq2pd	(%rax), %xmm19 {%k1}
+# CHECK-NEXT:  2      10    0.50    *                   vcvtqq2pd	(%rax){1to2}, %xmm19 {%k1}
+# CHECK-NEXT:  1      4     0.50                        vcvtqq2pd	%xmm16, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      10    0.50    *                   vcvtqq2pd	(%rax), %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      10    0.50    *                   vcvtqq2pd	(%rax){1to2}, %xmm19 {%k1} {z}
+# CHECK-NEXT:  1      4     0.50                        vcvtqq2pd	%ymm16, %ymm19
+# CHECK-NEXT:  2      11    0.50    *                   vcvtqq2pd	(%rax), %ymm19
+# CHECK-NEXT:  2      11    0.50    *                   vcvtqq2pd	(%rax){1to4}, %ymm19
+# CHECK-NEXT:  1      4     0.50                        vcvtqq2pd	%ymm16, %ymm19 {%k1}
+# CHECK-NEXT:  2      11    0.50    *                   vcvtqq2pd	(%rax), %ymm19 {%k1}
+# CHECK-NEXT:  2      11    0.50    *                   vcvtqq2pd	(%rax){1to4}, %ymm19 {%k1}
+# CHECK-NEXT:  1      4     0.50                        vcvtqq2pd	%ymm16, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      11    0.50    *                   vcvtqq2pd	(%rax), %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      11    0.50    *                   vcvtqq2pd	(%rax){1to4}, %ymm19 {%k1} {z}
+# CHECK-NEXT:  1      1     0.33                        vorpd	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  2      7     0.50    *                   vorpd	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  2      7     0.50    *                   vorpd	(%rax){1to2}, %xmm17, %xmm19
+# CHECK-NEXT:  1      1     0.33                        vorpd	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      7     0.50    *                   vorpd	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      7     0.50    *                   vorpd	(%rax){1to2}, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  1      1     0.33                        vorpd	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      7     0.50    *                   vorpd	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      7     0.50    *                   vorpd	(%rax){1to2}, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     0.33                        vorpd	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  2      8     0.50    *                   vorpd	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  2      8     0.50    *                   vorpd	(%rax){1to4}, %ymm17, %ymm19
+# CHECK-NEXT:  1      1     0.33                        vorpd	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     0.50    *                   vorpd	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     0.50    *                   vorpd	(%rax){1to4}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  1      1     0.33                        vorpd	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     0.50    *                   vorpd	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     0.50    *                   vorpd	(%rax){1to4}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  1      1     0.33                        vorps	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  2      7     0.50    *                   vorps	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  2      7     0.50    *                   vorps	(%rax){1to4}, %xmm17, %xmm19
+# CHECK-NEXT:  1      1     0.33                        vorps	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      7     0.50    *                   vorps	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      7     0.50    *                   vorps	(%rax){1to4}, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  1      1     0.33                        vorps	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      7     0.50    *                   vorps	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      7     0.50    *                   vorps	(%rax){1to4}, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     0.33                        vorps	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  2      8     0.50    *                   vorps	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  2      8     0.50    *                   vorps	(%rax){1to8}, %ymm17, %ymm19
+# CHECK-NEXT:  1      1     0.33                        vorps	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     0.50    *                   vorps	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     0.50    *                   vorps	(%rax){1to8}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  1      1     0.33                        vorps	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     0.50    *                   vorps	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     0.50    *                   vorps	(%rax){1to8}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  3      12    1.50                        vpmullq	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  4      18    1.00    *                   vpmullq	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  3      12    1.50                        vpmullq	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  4      18    1.00    *                   vpmullq	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  3      12    1.50                        vpmullq	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  4      18    1.00    *                   vpmullq	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  3      12    1.50                        vpmullq	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  4      19    1.00    *                   vpmullq	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  3      12    1.50                        vpmullq	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  4      19    1.00    *                   vpmullq	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  3      12    1.50                        vpmullq	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  4      19    1.00    *                   vpmullq	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  1      1     0.33                        vxorpd	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  2      7     0.50    *                   vxorpd	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  2      7     0.50    *                   vxorpd	(%rax){1to2}, %xmm17, %xmm19
+# CHECK-NEXT:  1      1     0.33                        vxorpd	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      7     0.50    *                   vxorpd	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      7     0.50    *                   vxorpd	(%rax){1to2}, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  1      1     0.33                        vxorpd	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      7     0.50    *                   vxorpd	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      7     0.50    *                   vxorpd	(%rax){1to2}, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     0.33                        vxorpd	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  2      8     0.50    *                   vxorpd	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  2      8     0.50    *                   vxorpd	(%rax){1to4}, %ymm17, %ymm19
+# CHECK-NEXT:  1      1     0.33                        vxorpd	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     0.50    *                   vxorpd	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     0.50    *                   vxorpd	(%rax){1to4}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  1      1     0.33                        vxorpd	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     0.50    *                   vxorpd	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     0.50    *                   vxorpd	(%rax){1to4}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  1      1     0.33                        vxorps	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  2      7     0.50    *                   vxorps	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  2      7     0.50    *                   vxorps	(%rax){1to4}, %xmm17, %xmm19
+# CHECK-NEXT:  1      1     0.33                        vxorps	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      7     0.50    *                   vxorps	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      7     0.50    *                   vxorps	(%rax){1to4}, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  1      1     0.33                        vxorps	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      7     0.50    *                   vxorps	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      7     0.50    *                   vxorps	(%rax){1to4}, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     0.33                        vxorps	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  2      8     0.50    *                   vxorps	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  2      8     0.50    *                   vxorps	(%rax){1to8}, %ymm17, %ymm19
+# CHECK-NEXT:  1      1     0.33                        vxorps	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     0.50    *                   vxorps	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     0.50    *                   vxorps	(%rax){1to8}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  1      1     0.33                        vxorps	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     0.50    *                   vxorps	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     0.50    *                   vxorps	(%rax){1to8}, %ymm17, %ymm19 {%k1} {z}
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0]   - SKXDivider
+# CHECK-NEXT: [1]   - SKXFPDivider
+# CHECK-NEXT: [2]   - SKXPort0
+# CHECK-NEXT: [3]   - SKXPort1
+# CHECK-NEXT: [4]   - SKXPort2
+# CHECK-NEXT: [5]   - SKXPort3
+# CHECK-NEXT: [6]   - SKXPort4
+# CHECK-NEXT: [7]   - SKXPort5
+# CHECK-NEXT: [8]   - SKXPort6
+# CHECK-NEXT: [9]   - SKXPort7
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]
+# CHECK-NEXT:  -      -     64.00  64.00  51.00  51.00   -     52.00   -      -
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    Instructions:
+# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vandnpd	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vandnpd	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vandnpd	(%rax){1to2}, %xmm17, %xmm19
+# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vandnpd	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vandnpd	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vandnpd	(%rax){1to2}, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vandnpd	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vandnpd	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vandnpd	(%rax){1to2}, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vandnpd	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vandnpd	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vandnpd	(%rax){1to4}, %ymm17, %ymm19
+# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vandnpd	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vandnpd	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vandnpd	(%rax){1to4}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vandnpd	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vandnpd	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vandnpd	(%rax){1to4}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vandpd	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vandpd	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vandpd	(%rax){1to2}, %xmm17, %xmm19
+# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vandpd	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vandpd	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vandpd	(%rax){1to2}, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vandpd	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vandpd	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vandpd	(%rax){1to2}, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vandpd	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vandpd	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vandpd	(%rax){1to4}, %ymm17, %ymm19
+# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vandpd	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vandpd	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vandpd	(%rax){1to4}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vandpd	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vandpd	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vandpd	(%rax){1to4}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vandps	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vandps	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vandps	(%rax){1to4}, %xmm17, %xmm19
+# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vandps	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vandps	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vandps	(%rax){1to4}, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vandps	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vandps	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vandps	(%rax){1to4}, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vandps	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vandps	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vandps	(%rax){1to8}, %ymm17, %ymm19
+# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vandps	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vandps	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vandps	(%rax){1to8}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vandps	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vandps	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vandps	(%rax){1to8}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -     vcvtqq2pd	%xmm16, %xmm19
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vcvtqq2pd	(%rax), %xmm19
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vcvtqq2pd	(%rax){1to2}, %xmm19
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -     vcvtqq2pd	%xmm16, %xmm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vcvtqq2pd	(%rax), %xmm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vcvtqq2pd	(%rax){1to2}, %xmm19 {%k1}
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -     vcvtqq2pd	%xmm16, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vcvtqq2pd	(%rax), %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vcvtqq2pd	(%rax){1to2}, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -     vcvtqq2pd	%ymm16, %ymm19
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vcvtqq2pd	(%rax), %ymm19
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vcvtqq2pd	(%rax){1to4}, %ymm19
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -     vcvtqq2pd	%ymm16, %ymm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vcvtqq2pd	(%rax), %ymm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vcvtqq2pd	(%rax){1to4}, %ymm19 {%k1}
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -     vcvtqq2pd	%ymm16, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vcvtqq2pd	(%rax), %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vcvtqq2pd	(%rax){1to4}, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vorpd	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vorpd	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vorpd	(%rax){1to2}, %xmm17, %xmm19
+# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vorpd	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vorpd	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vorpd	(%rax){1to2}, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vorpd	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vorpd	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vorpd	(%rax){1to2}, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vorpd	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vorpd	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vorpd	(%rax){1to4}, %ymm17, %ymm19
+# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vorpd	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vorpd	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vorpd	(%rax){1to4}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vorpd	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vorpd	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vorpd	(%rax){1to4}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vorps	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vorps	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vorps	(%rax){1to4}, %xmm17, %xmm19
+# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vorps	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vorps	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vorps	(%rax){1to4}, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vorps	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vorps	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vorps	(%rax){1to4}, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vorps	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vorps	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vorps	(%rax){1to8}, %ymm17, %ymm19
+# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vorps	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vorps	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vorps	(%rax){1to8}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vorps	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vorps	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vorps	(%rax){1to8}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -     1.50   1.50    -      -      -      -      -      -     vpmullq	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  -      -     1.00   1.00   0.50   0.50    -     1.00    -      -     vpmullq	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  -      -     1.50   1.50    -      -      -      -      -      -     vpmullq	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -     1.00   1.00   0.50   0.50    -     1.00    -      -     vpmullq	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -     1.50   1.50    -      -      -      -      -      -     vpmullq	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     1.00   1.00   0.50   0.50    -     1.00    -      -     vpmullq	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     1.50   1.50    -      -      -      -      -      -     vpmullq	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  -      -     1.00   1.00   0.50   0.50    -     1.00    -      -     vpmullq	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  -      -     1.50   1.50    -      -      -      -      -      -     vpmullq	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -     1.00   1.00   0.50   0.50    -     1.00    -      -     vpmullq	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -     1.50   1.50    -      -      -      -      -      -     vpmullq	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -     1.00   1.00   0.50   0.50    -     1.00    -      -     vpmullq	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vxorpd	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vxorpd	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vxorpd	(%rax){1to2}, %xmm17, %xmm19
+# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vxorpd	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vxorpd	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vxorpd	(%rax){1to2}, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vxorpd	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vxorpd	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vxorpd	(%rax){1to2}, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vxorpd	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vxorpd	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vxorpd	(%rax){1to4}, %ymm17, %ymm19
+# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vxorpd	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vxorpd	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vxorpd	(%rax){1to4}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vxorpd	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vxorpd	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vxorpd	(%rax){1to4}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vxorps	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vxorps	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vxorps	(%rax){1to4}, %xmm17, %xmm19
+# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vxorps	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vxorps	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vxorps	(%rax){1to4}, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vxorps	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vxorps	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vxorps	(%rax){1to4}, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vxorps	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vxorps	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vxorps	(%rax){1to8}, %ymm17, %ymm19
+# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vxorps	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vxorps	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vxorps	(%rax){1to8}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vxorps	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vxorps	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vxorps	(%rax){1to8}, %ymm17, %ymm19 {%k1} {z}
diff --git a/test/tools/llvm-mca/X86/SkylakeServer/resources-avx512vl.s b/test/tools/llvm-mca/X86/SkylakeServer/resources-avx512vl.s
index 888601a04c0810ab874a76d21c26a2179cfc1418..e96d501c243dcb4ca8e60fb0641315fac67193b1 100644
--- a/test/tools/llvm-mca/X86/SkylakeServer/resources-avx512vl.s
+++ b/test/tools/llvm-mca/X86/SkylakeServer/resources-avx512vl.s
@@ -3,115 +3,743 @@
 
 vaddpd            %xmm16, %xmm17, %xmm19
 vaddpd            (%rax), %xmm17, %xmm19
+vaddpd            (%rax){1to2}, %xmm17, %xmm19
 vaddpd            %xmm16, %xmm17, %xmm19 {k1}
 vaddpd            (%rax), %xmm17, %xmm19 {k1}
+vaddpd            (%rax){1to2}, %xmm17, %xmm19 {k1}
 vaddpd            %xmm16, %xmm17, %xmm19 {z}{k1}
 vaddpd            (%rax), %xmm17, %xmm19 {z}{k1}
+vaddpd            (%rax){1to2}, %xmm17, %xmm19 {z}{k1}
 
 vaddpd            %ymm16, %ymm17, %ymm19
 vaddpd            (%rax), %ymm17, %ymm19
+vaddpd            (%rax){1to4}, %ymm17, %ymm19
 vaddpd            %ymm16, %ymm17, %ymm19 {k1}
 vaddpd            (%rax), %ymm17, %ymm19 {k1}
+vaddpd            (%rax){1to4}, %ymm17, %ymm19 {k1}
 vaddpd            %ymm16, %ymm17, %ymm19 {z}{k1}
 vaddpd            (%rax), %ymm17, %ymm19 {z}{k1}
+vaddpd            (%rax){1to4}, %ymm17, %ymm19 {z}{k1}
 
 vaddps            %xmm16, %xmm17, %xmm19
 vaddps            (%rax), %xmm17, %xmm19
+vaddps            (%rax){1to4}, %xmm17, %xmm19
 vaddps            %xmm16, %xmm17, %xmm19 {k1}
 vaddps            (%rax), %xmm17, %xmm19 {k1}
+vaddps            (%rax){1to4}, %xmm17, %xmm19 {k1}
 vaddps            %xmm16, %xmm17, %xmm19 {z}{k1}
 vaddps            (%rax), %xmm17, %xmm19 {z}{k1}
+vaddps            (%rax){1to4}, %xmm17, %xmm19 {z}{k1}
 
 vaddps            %ymm16, %ymm17, %ymm19
 vaddps            (%rax), %ymm17, %ymm19
+vaddps            (%rax){1to8}, %ymm17, %ymm19
 vaddps            %ymm16, %ymm17, %ymm19 {k1}
 vaddps            (%rax), %ymm17, %ymm19 {k1}
+vaddps            (%rax){1to8}, %ymm17, %ymm19 {k1}
 vaddps            %ymm16, %ymm17, %ymm19 {z}{k1}
 vaddps            (%rax), %ymm17, %ymm19 {z}{k1}
+vaddps            (%rax){1to8}, %ymm17, %ymm19 {z}{k1}
+
+vcvtdq2pd         %xmm16, %xmm19
+vcvtdq2pd         (%rax), %xmm19
+vcvtdq2pd         (%rax){1to2}, %xmm19
+vcvtdq2pd         %xmm16, %xmm19 {k1}
+vcvtdq2pd         (%rax), %xmm19 {k1}
+vcvtdq2pd         (%rax){1to2}, %xmm19 {k1}
+vcvtdq2pd         %xmm16, %xmm19 {z}{k1}
+vcvtdq2pd         (%rax), %xmm19 {z}{k1}
+vcvtdq2pd         (%rax){1to2}, %xmm19 {z}{k1}
+
+vcvtdq2pd         %xmm16, %ymm19
+vcvtdq2pd         (%rax), %ymm19
+vcvtdq2pd         (%rax){1to4}, %ymm19
+vcvtdq2pd         %xmm16, %ymm19 {k1}
+vcvtdq2pd         (%rax), %ymm19 {k1}
+vcvtdq2pd         (%rax){1to4}, %ymm19 {k1}
+vcvtdq2pd         %xmm16, %ymm19 {z}{k1}
+vcvtdq2pd         (%rax), %ymm19 {z}{k1}
+vcvtdq2pd         (%rax){1to4}, %ymm19 {z}{k1}
+
+vcvtdq2ps         %xmm16, %xmm19
+vcvtdq2ps         (%rax), %xmm19
+vcvtdq2ps         (%rax){1to4}, %xmm19
+vcvtdq2ps         %xmm16, %xmm19 {k1}
+vcvtdq2ps         (%rax), %xmm19 {k1}
+vcvtdq2ps         (%rax){1to4},%xmm19 {k1}
+vcvtdq2ps         %xmm16, %xmm19 {z}{k1}
+vcvtdq2ps         (%rax), %xmm19 {z}{k1}
+vcvtdq2ps         (%rax){1to4}, %xmm19 {z}{k1}
+
+vcvtdq2ps         %ymm16, %ymm19
+vcvtdq2ps         (%rax), %ymm19
+vcvtdq2ps         (%rax){1to8}, %ymm19
+vcvtdq2ps         %ymm16,%ymm19 {k1}
+vcvtdq2ps         (%rax),%ymm19 {k1}
+vcvtdq2ps         (%rax){1to8}, %ymm19 {k1}
+vcvtdq2ps         %ymm16, %ymm19 {z}{k1}
+vcvtdq2ps         (%rax), %ymm19 {z}{k1}
+vcvtdq2ps         (%rax){1to8}, %ymm19 {z}{k1}
 
 vdivpd            %xmm16, %xmm17, %xmm19
 vdivpd            (%rax), %xmm17, %xmm19
+vdivpd            (%rax){1to2}, %xmm17, %xmm19
 vdivpd            %xmm16, %xmm17, %xmm19 {k1}
 vdivpd            (%rax), %xmm17, %xmm19 {k1}
+vdivpd            (%rax){1to2}, %xmm17, %xmm19 {k1}
 vdivpd            %xmm16, %xmm17, %xmm19 {z}{k1}
 vdivpd            (%rax), %xmm17, %xmm19 {z}{k1}
+vdivpd            (%rax){1to2}, %xmm17, %xmm19 {z}{k1}
 
 vdivpd            %ymm16, %ymm17, %ymm19
 vdivpd            (%rax), %ymm17, %ymm19
+vdivpd            (%rax){1to4}, %ymm17, %ymm19
 vdivpd            %ymm16, %ymm17, %ymm19 {k1}
 vdivpd            (%rax), %ymm17, %ymm19 {k1}
+vdivpd            (%rax){1to4}, %ymm17, %ymm19 {k1}
 vdivpd            %ymm16, %ymm17, %ymm19 {z}{k1}
 vdivpd            (%rax), %ymm17, %ymm19 {z}{k1}
+vdivpd            (%rax){1to4}, %ymm17, %ymm19 {z}{k1}
 
 vdivps            %xmm16, %xmm17, %xmm19
 vdivps            (%rax), %xmm17, %xmm19
+vdivps            (%rax){1to4}, %xmm17, %xmm19
 vdivps            %xmm16, %xmm17, %xmm19 {k1}
 vdivps            (%rax), %xmm17, %xmm19 {k1}
+vdivps            (%rax){1to4}, %xmm17, %xmm19 {k1}
 vdivps            %xmm16, %xmm17, %xmm19 {z}{k1}
 vdivps            (%rax), %xmm17, %xmm19 {z}{k1}
+vdivps            (%rax){1to4}, %xmm17, %xmm19 {z}{k1}
 
 vdivps            %ymm16, %ymm17, %ymm19
 vdivps            (%rax), %ymm17, %ymm19
+vdivps            (%rax){1to8}, %ymm17, %ymm19
 vdivps            %ymm16, %ymm17, %ymm19 {k1}
 vdivps            (%rax), %ymm17, %ymm19 {k1}
+vdivps            (%rax){1to8}, %ymm17, %ymm19 {k1}
 vdivps            %ymm16, %ymm17, %ymm19 {z}{k1}
 vdivps            (%rax), %ymm17, %ymm19 {z}{k1}
+vdivps            (%rax){1to8}, %ymm17, %ymm19 {z}{k1}
+
+vmaxpd            %xmm16, %xmm17, %xmm19
+vmaxpd            (%rax), %xmm17, %xmm19
+vmaxpd            (%rax){1to2}, %xmm17, %xmm19
+vmaxpd            %xmm16, %xmm17, %xmm19 {k1}
+vmaxpd            (%rax), %xmm17, %xmm19 {k1}
+vmaxpd            (%rax){1to2}, %xmm17, %xmm19 {k1}
+vmaxpd            %xmm16, %xmm17, %xmm19 {z}{k1}
+vmaxpd            (%rax), %xmm17, %xmm19 {z}{k1}
+vmaxpd            (%rax){1to2}, %xmm17, %xmm19 {z}{k1}
+
+vmaxpd            %ymm16, %ymm17, %ymm19
+vmaxpd            (%rax), %ymm17, %ymm19
+vmaxpd            (%rax){1to4}, %ymm17, %ymm19
+vmaxpd            %ymm16, %ymm17, %ymm19 {k1}
+vmaxpd            (%rax), %ymm17, %ymm19 {k1}
+vmaxpd            (%rax){1to4}, %ymm17, %ymm19 {k1}
+vmaxpd            %ymm16, %ymm17, %ymm19 {z}{k1}
+vmaxpd            (%rax), %ymm17, %ymm19 {z}{k1}
+vmaxpd            (%rax){1to4}, %ymm17, %ymm19 {z}{k1}
+
+vmaxps            %xmm16, %xmm17, %xmm19
+vmaxps            (%rax), %xmm17, %xmm19
+vmaxps            (%rax){1to4}, %xmm17, %xmm19
+vmaxps            %xmm16, %xmm17, %xmm19 {k1}
+vmaxps            (%rax), %xmm17, %xmm19 {k1}
+vmaxps            (%rax){1to4}, %xmm17, %xmm19 {k1}
+vmaxps            %xmm16, %xmm17, %xmm19 {z}{k1}
+vmaxps            (%rax), %xmm17, %xmm19 {z}{k1}
+vmaxps            (%rax){1to4}, %xmm17, %xmm19 {z}{k1}
+
+vmaxps            %ymm16, %ymm17, %ymm19
+vmaxps            (%rax), %ymm17, %ymm19
+vmaxps            (%rax){1to8}, %ymm17, %ymm19
+vmaxps            %ymm16, %ymm17, %ymm19 {k1}
+vmaxps            (%rax), %ymm17, %ymm19 {k1}
+vmaxps            (%rax){1to8}, %ymm17, %ymm19 {k1}
+vmaxps            %ymm16, %ymm17, %ymm19 {z}{k1}
+vmaxps            (%rax), %ymm17, %ymm19 {z}{k1}
+vmaxps            (%rax){1to8}, %ymm17, %ymm19 {z}{k1}
+
+vminpd            %xmm16, %xmm17, %xmm19
+vminpd            (%rax), %xmm17, %xmm19
+vminpd            (%rax){1to2}, %xmm17, %xmm19
+vminpd            %xmm16, %xmm17, %xmm19 {k1}
+vminpd            (%rax), %xmm17, %xmm19 {k1}
+vminpd            (%rax){1to2}, %xmm17, %xmm19 {k1}
+vminpd            %xmm16, %xmm17, %xmm19 {z}{k1}
+vminpd            (%rax), %xmm17, %xmm19 {z}{k1}
+vminpd            (%rax){1to2}, %xmm17, %xmm19 {z}{k1}
+
+vminpd            %ymm16, %ymm17, %ymm19
+vminpd            (%rax), %ymm17, %ymm19
+vminpd            (%rax){1to4}, %ymm17, %ymm19
+vminpd            %ymm16, %ymm17, %ymm19 {k1}
+vminpd            (%rax), %ymm17, %ymm19 {k1}
+vminpd            (%rax){1to4}, %ymm17, %ymm19 {k1}
+vminpd            %ymm16, %ymm17, %ymm19 {z}{k1}
+vminpd            (%rax), %ymm17, %ymm19 {z}{k1}
+vminpd            (%rax){1to4}, %ymm17, %ymm19 {z}{k1}
+
+vminps            %xmm16, %xmm17, %xmm19
+vminps            (%rax), %xmm17, %xmm19
+vminps            (%rax){1to4}, %xmm17, %xmm19
+vminps            %xmm16, %xmm17, %xmm19 {k1}
+vminps            (%rax), %xmm17, %xmm19 {k1}
+vminps            (%rax){1to4}, %xmm17, %xmm19 {k1}
+vminps            %xmm16, %xmm17, %xmm19 {z}{k1}
+vminps            (%rax), %xmm17, %xmm19 {z}{k1}
+vminps            (%rax){1to4}, %xmm17, %xmm19 {z}{k1}
+
+vminps            %ymm16, %ymm17, %ymm19
+vminps            (%rax), %ymm17, %ymm19
+vminps            (%rax){1to8}, %ymm17, %ymm19
+vminps            %ymm16, %ymm17, %ymm19 {k1}
+vminps            (%rax), %ymm17, %ymm19 {k1}
+vminps            (%rax){1to8}, %ymm17, %ymm19 {k1}
+vminps            %ymm16, %ymm17, %ymm19 {z}{k1}
+vminps            (%rax), %ymm17, %ymm19 {z}{k1}
+vminps            (%rax){1to8}, %ymm17, %ymm19 {z}{k1}
 
 vmulpd            %xmm16, %xmm17, %xmm19
 vmulpd            (%rax), %xmm17, %xmm19
+vmulpd            (%rax){1to2}, %xmm17, %xmm19
 vmulpd            %xmm16, %xmm17, %xmm19 {k1}
 vmulpd            (%rax), %xmm17, %xmm19 {k1}
+vmulpd            (%rax){1to2}, %xmm17, %xmm19 {k1}
 vmulpd            %xmm16, %xmm17, %xmm19 {z}{k1}
 vmulpd            (%rax), %xmm17, %xmm19 {z}{k1}
+vmulpd            (%rax){1to2}, %xmm17, %xmm19 {z}{k1}
 
 vmulpd            %ymm16, %ymm17, %ymm19
 vmulpd            (%rax), %ymm17, %ymm19
+vmulpd            (%rax){1to4}, %ymm17, %ymm19
 vmulpd            %ymm16, %ymm17, %ymm19 {k1}
 vmulpd            (%rax), %ymm17, %ymm19 {k1}
+vmulpd            (%rax){1to4}, %ymm17, %ymm19 {k1}
 vmulpd            %ymm16, %ymm17, %ymm19 {z}{k1}
 vmulpd            (%rax), %ymm17, %ymm19 {z}{k1}
+vmulpd            (%rax){1to4}, %ymm17, %ymm19 {z}{k1}
 
 vmulps            %xmm16, %xmm17, %xmm19
 vmulps            (%rax), %xmm17, %xmm19
+vmulps            (%rax){1to4}, %xmm17, %xmm19
 vmulps            %xmm16, %xmm17, %xmm19 {k1}
 vmulps            (%rax), %xmm17, %xmm19 {k1}
+vmulps            (%rax){1to4}, %xmm17, %xmm19 {k1}
 vmulps            %xmm16, %xmm17, %xmm19 {z}{k1}
 vmulps            (%rax), %xmm17, %xmm19 {z}{k1}
+vmulps            (%rax){1to4}, %xmm17, %xmm19 {z}{k1}
 
 vmulps            %ymm16, %ymm17, %ymm19
 vmulps            (%rax), %ymm17, %ymm19
+vmulps            (%rax){1to8}, %ymm17, %ymm19
 vmulps            %ymm16, %ymm17, %ymm19 {k1}
 vmulps            (%rax), %ymm17, %ymm19 {k1}
+vmulps            (%rax){1to8}, %ymm17, %ymm19 {k1}
 vmulps            %ymm16, %ymm17, %ymm19 {z}{k1}
 vmulps            (%rax), %ymm17, %ymm19 {z}{k1}
+vmulps            (%rax){1to8}, %ymm17, %ymm19 {z}{k1}
+
+vpaddd            %xmm16, %xmm17, %xmm19
+vpaddd            (%rax), %xmm17, %xmm19
+vpaddd            (%rax){1to4}, %xmm17, %xmm19
+vpaddd            %xmm16, %xmm17, %xmm19 {k1}
+vpaddd            (%rax), %xmm17, %xmm19 {k1}
+vpaddd            (%rax){1to4}, %xmm17, %xmm19 {k1}
+vpaddd            %xmm16, %xmm17, %xmm19 {z}{k1}
+vpaddd            (%rax), %xmm17, %xmm19 {z}{k1}
+vpaddd            (%rax){1to4}, %xmm17, %xmm19 {z}{k1}
+
+vpaddd            %ymm16, %ymm17, %ymm19
+vpaddd            (%rax), %ymm17, %ymm19
+vpaddd            (%rax){1to8}, %ymm17, %ymm19
+vpaddd            %ymm16, %ymm17, %ymm19 {k1}
+vpaddd            (%rax), %ymm17, %ymm19 {k1}
+vpaddd            (%rax){1to8}, %ymm17, %ymm19 {k1}
+vpaddd            %ymm16, %ymm17, %ymm19 {z}{k1}
+vpaddd            (%rax), %ymm17, %ymm19 {z}{k1}
+vpaddd            (%rax){1to8}, %ymm17, %ymm19 {z}{k1}
+
+vpaddq            %xmm16, %xmm17, %xmm19
+vpaddq            (%rax), %xmm17, %xmm19
+vpaddq            (%rax){1to2}, %xmm17, %xmm19
+vpaddq            %xmm16, %xmm17, %xmm19 {k1}
+vpaddq            (%rax), %xmm17, %xmm19 {k1}
+vpaddq            (%rax){1to2}, %xmm17, %xmm19 {k1}
+vpaddq            %xmm16, %xmm17, %xmm19 {z}{k1}
+vpaddq            (%rax), %xmm17, %xmm19 {z}{k1}
+vpaddq            (%rax){1to2}, %xmm17, %xmm19 {z}{k1}
+
+vpaddq            %ymm16, %ymm17, %ymm19
+vpaddq            (%rax), %ymm17, %ymm19
+vpaddq            (%rax){1to4}, %ymm17, %ymm19
+vpaddq            %ymm16, %ymm17, %ymm19 {k1}
+vpaddq            (%rax), %ymm17, %ymm19 {k1}
+vpaddq            (%rax){1to4}, %ymm17, %ymm19 {k1}
+vpaddq            %ymm16, %ymm17, %ymm19 {z}{k1}
+vpaddq            (%rax), %ymm17, %ymm19 {z}{k1}
+vpaddq            (%rax){1to4}, %ymm17, %ymm19 {z}{k1}
+
+vpermd            %ymm16, %ymm17, %ymm19
+vpermd            (%rax), %ymm17, %ymm19
+vpermd            (%rax){1to8}, %ymm17, %ymm19
+vpermd            %ymm16, %ymm17, %ymm19 {k1}
+vpermd            (%rax), %ymm17, %ymm19 {k1}
+vpermd            (%rax){1to8}, %ymm17, %ymm19 {k1}
+vpermd            %ymm16, %ymm17, %ymm19 {z}{k1}
+vpermd            (%rax), %ymm17, %ymm19 {z}{k1}
+vpermd            (%rax){1to8}, %ymm17, %ymm19 {z}{k1}
+
+vpermilpd         $0, %xmm16, %xmm19
+vpermilpd         $0, (%rax), %xmm19
+vpermilpd         $0, (%rax){1to2}, %xmm19
+vpermilpd         $0, %xmm16, %xmm19 {k1}
+vpermilpd         $0, (%rax), %xmm19 {k1}
+vpermilpd         $0, (%rax){1to2}, %xmm19 {k1}
+vpermilpd         $0, %xmm16, %xmm19 {z}{k1}
+vpermilpd         $0, (%rax), %xmm19 {z}{k1}
+vpermilpd         $0, (%rax){1to2}, %xmm19 {z}{k1}
+
+vpermilpd         $0, %ymm16, %ymm19
+vpermilpd         $0, (%rax), %ymm19
+vpermilpd         $0, (%rax){1to4}, %ymm19
+vpermilpd         $0, %ymm16, %ymm19 {k1}
+vpermilpd         $0, (%rax), %ymm19 {k1}
+vpermilpd         $0, (%rax){1to4}, %ymm19 {k1}
+vpermilpd         $0, %ymm16, %ymm19 {z}{k1}
+vpermilpd         $0, (%rax), %ymm19 {z}{k1}
+vpermilpd         $0, (%rax){1to4}, %ymm19 {z}{k1}
+
+vpermilpd         %xmm16, %xmm17, %xmm19
+vpermilpd         (%rax), %xmm17, %xmm19
+vpermilpd         (%rax){1to2}, %xmm17, %xmm19
+vpermilpd         %xmm16, %xmm17, %xmm19 {k1}
+vpermilpd         (%rax), %xmm17, %xmm19 {k1}
+vpermilpd         (%rax){1to2}, %xmm17, %xmm19 {k1}
+vpermilpd         %xmm16, %xmm17, %xmm19 {z}{k1}
+vpermilpd         (%rax), %xmm17, %xmm19 {z}{k1}
+vpermilpd         (%rax){1to2}, %xmm17, %xmm19 {z}{k1}
+
+vpermilpd         %ymm16, %ymm17, %ymm19
+vpermilpd         (%rax), %ymm17, %ymm19
+vpermilpd         (%rax){1to4}, %ymm17, %ymm19
+vpermilpd         %ymm16, %ymm17, %ymm19 {k1}
+vpermilpd         (%rax), %ymm17, %ymm19 {k1}
+vpermilpd         (%rax){1to4}, %ymm17, %ymm19 {k1}
+vpermilpd         %ymm16, %ymm17, %ymm19 {z}{k1}
+vpermilpd         (%rax), %ymm17, %ymm19 {z}{k1}
+vpermilpd         (%rax){1to4}, %ymm17, %ymm19 {z}{k1}
+
+vpermilps         $0, %xmm16, %xmm19
+vpermilps         $0, (%rax), %xmm19
+vpermilps         $0, (%rax){1to4}, %xmm19
+vpermilps         $0, %xmm16, %xmm19 {k1}
+vpermilps         $0, (%rax), %xmm19 {k1}
+vpermilps         $0, (%rax){1to4}, %xmm19 {k1}
+vpermilps         $0, %xmm16, %xmm19 {z}{k1}
+vpermilps         $0, (%rax), %xmm19 {z}{k1}
+vpermilps         $0, (%rax){1to4}, %xmm19 {z}{k1}
+
+vpermilps         $0, %ymm16, %ymm19
+vpermilps         $0, (%rax), %ymm19
+vpermilps         $0, (%rax){1to8}, %ymm19
+vpermilps         $0, %ymm16, %ymm19 {k1}
+vpermilps         $0, (%rax), %ymm19 {k1}
+vpermilps         $0, (%rax){1to8}, %ymm19 {k1}
+vpermilps         $0, %ymm16, %ymm19 {z}{k1}
+vpermilps         $0, (%rax), %ymm19 {z}{k1}
+vpermilps         $0, (%rax){1to8}, %ymm19 {z}{k1}
+
+vpermilps         %xmm16, %xmm17, %xmm19
+vpermilps         (%rax), %xmm17, %xmm19
+vpermilps         (%rax){1to4}, %xmm17, %xmm19
+vpermilps         %xmm16, %xmm17, %xmm19 {k1}
+vpermilps         (%rax), %xmm17, %xmm19 {k1}
+vpermilps         (%rax){1to4}, %xmm17, %xmm19 {k1}
+vpermilps         %xmm16, %xmm17, %xmm19 {z}{k1}
+vpermilps         (%rax), %xmm17, %xmm19 {z}{k1}
+vpermilps         (%rax){1to4}, %xmm17, %xmm19 {z}{k1}
+
+vpermilps         %ymm16, %ymm17, %ymm19
+vpermilps         (%rax), %ymm17, %ymm19
+vpermilps         (%rax){1to8}, %ymm17, %ymm19
+vpermilps         %ymm16, %ymm17, %ymm19 {k1}
+vpermilps         (%rax), %ymm17, %ymm19 {k1}
+vpermilps         (%rax){1to8}, %ymm17, %ymm19 {k1}
+vpermilps         %ymm16, %ymm17, %ymm19 {z}{k1}
+vpermilps         (%rax), %ymm17, %ymm19 {z}{k1}
+vpermilps         (%rax){1to8}, %ymm17, %ymm19 {z}{k1}
+
+vpermpd           $0, %ymm16, %ymm19
+vpermpd           $0, (%rax), %ymm19
+vpermpd           $0, (%rax){1to4}, %ymm19
+vpermpd           $0, %ymm16, %ymm19 {k1}
+vpermpd           $0, (%rax), %ymm19 {k1}
+vpermpd           $0, (%rax){1to4}, %ymm19 {k1}
+vpermpd           $0, %ymm16, %ymm19 {z}{k1}
+vpermpd           $0, (%rax), %ymm19 {z}{k1}
+vpermpd           $0, (%rax){1to4}, %ymm19 {z}{k1}
+
+vpermpd           %ymm16, %ymm17, %ymm19
+vpermpd           (%rax), %ymm17, %ymm19
+vpermpd           (%rax){1to4}, %ymm17, %ymm19
+vpermpd           %ymm16, %ymm17, %ymm19 {k1}
+vpermpd           (%rax), %ymm17, %ymm19 {k1}
+vpermpd           (%rax){1to4}, %ymm17, %ymm19 {k1}
+vpermpd           %ymm16, %ymm17, %ymm19 {z}{k1}
+vpermpd           (%rax), %ymm17, %ymm19 {z}{k1}
+vpermpd           (%rax){1to4}, %ymm17, %ymm19 {z}{k1}
+
+vpermps           %ymm16, %ymm17, %ymm19
+vpermps           (%rax), %ymm17, %ymm19
+vpermps           (%rax){1to8}, %ymm17, %ymm19
+vpermps           %ymm16, %ymm17, %ymm19 {k1}
+vpermps           (%rax), %ymm17, %ymm19 {k1}
+vpermps           (%rax){1to8}, %ymm17, %ymm19 {k1}
+vpermps           %ymm16, %ymm17, %ymm19 {z}{k1}
+vpermps           (%rax), %ymm17, %ymm19 {z}{k1}
+vpermps           (%rax){1to8}, %ymm17, %ymm19 {z}{k1}
+
+vpermq            $0, %ymm16, %ymm19
+vpermq            $0, (%rax), %ymm19
+vpermq            $0, (%rax){1to4}, %ymm19
+vpermq            $0, %ymm16, %ymm19 {k1}
+vpermq            $0, (%rax), %ymm19 {k1}
+vpermq            $0, (%rax){1to4}, %ymm19 {k1}
+vpermq            $0, %ymm16, %ymm19 {z}{k1}
+vpermq            $0, (%rax), %ymm19 {z}{k1}
+vpermq            $0, (%rax){1to4}, %ymm19 {z}{k1}
+
+vpermq            %ymm16, %ymm17, %ymm19
+vpermq            (%rax), %ymm17, %ymm19
+vpermq            (%rax){1to4}, %ymm17, %ymm19
+vpermq            %ymm16, %ymm17, %ymm19 {k1}
+vpermq            (%rax), %ymm17, %ymm19 {k1}
+vpermq            (%rax){1to4}, %ymm17, %ymm19 {k1}
+vpermq            %ymm16, %ymm17, %ymm19 {z}{k1}
+vpermq            (%rax), %ymm17, %ymm19 {z}{k1}
+vpermq            (%rax){1to4}, %ymm17, %ymm19 {z}{k1}
+
+vpmulld           %xmm16, %xmm17, %xmm19
+vpmulld           (%rax), %xmm17, %xmm19
+vpmulld           (%rax){1to4}, %xmm17, %xmm19
+vpmulld           %xmm16, %xmm17, %xmm19 {k1}
+vpmulld           (%rax), %xmm17, %xmm19 {k1}
+vpmulld           (%rax){1to4}, %xmm17, %xmm19 {k1}
+vpmulld           %xmm16, %xmm17, %xmm19 {z}{k1}
+vpmulld           (%rax), %xmm17, %xmm19 {z}{k1}
+vpmulld           (%rax){1to4}, %xmm17, %xmm19 {z}{k1}
+
+vpmulld           %ymm16, %ymm17, %ymm19
+vpmulld           (%rax), %ymm17, %ymm19
+vpmulld           (%rax){1to8}, %ymm17, %ymm19
+vpmulld           %ymm16, %ymm17, %ymm19 {k1}
+vpmulld           (%rax), %ymm17, %ymm19 {k1}
+vpmulld           (%rax){1to8}, %ymm17, %ymm19 {k1}
+vpmulld           %ymm16, %ymm17, %ymm19 {z}{k1}
+vpmulld           (%rax), %ymm17, %ymm19 {z}{k1}
+vpmulld           (%rax){1to8}, %ymm17, %ymm19 {z}{k1}
+
+vpshufd           $0, %xmm16, %xmm19
+vpshufd           $0, (%rax), %xmm19
+vpshufd           $0, (%rax){1to4}, %xmm19
+vpshufd           $0, %xmm16, %xmm19 {k1}
+vpshufd           $0, (%rax), %xmm19 {k1}
+vpshufd           $0, (%rax){1to4}, %xmm19 {k1}
+vpshufd           $0, %xmm16, %xmm19 {z}{k1}
+vpshufd           $0, (%rax), %xmm19 {z}{k1}
+vpshufd           $0, (%rax){1to4}, %xmm19 {z}{k1}
+
+vpshufd           $0, %ymm16, %ymm19
+vpshufd           $0, (%rax), %ymm19
+vpshufd           $0, (%rax){1to8}, %ymm19
+vpshufd           $0, %ymm16, %ymm19 {k1}
+vpshufd           $0, (%rax), %ymm19 {k1}
+vpshufd           $0, (%rax){1to8}, %ymm19 {k1}
+vpshufd           $0, %ymm16, %ymm19 {z}{k1}
+vpshufd           $0, (%rax), %ymm19 {z}{k1}
+vpshufd           $0, (%rax){1to8}, %ymm19 {z}{k1}
+
+vpsubd            %xmm16, %xmm17, %xmm19
+vpsubd            (%rax), %xmm17, %xmm19
+vpsubd            (%rax){1to4}, %xmm17, %xmm19
+vpsubd            %xmm16, %xmm17, %xmm19 {k1}
+vpsubd            (%rax), %xmm17, %xmm19 {k1}
+vpsubd            (%rax){1to4}, %xmm17, %xmm19 {k1}
+vpsubd            %xmm16, %xmm17, %xmm19 {z}{k1}
+vpsubd            (%rax), %xmm17, %xmm19 {z}{k1}
+vpsubd            (%rax){1to4}, %xmm17, %xmm19 {z}{k1}
+
+vpsubd            %ymm16, %ymm17, %ymm19
+vpsubd            (%rax), %ymm17, %ymm19
+vpsubd            (%rax){1to8}, %ymm17, %ymm19
+vpsubd            %ymm16, %ymm17, %ymm19 {k1}
+vpsubd            (%rax), %ymm17, %ymm19 {k1}
+vpsubd            (%rax){1to8}, %ymm17, %ymm19 {k1}
+vpsubd            %ymm16, %ymm17, %ymm19 {z}{k1}
+vpsubd            (%rax), %ymm17, %ymm19 {z}{k1}
+vpsubd            (%rax){1to8}, %ymm17, %ymm19 {z}{k1}
+
+vpsubq            %xmm16, %xmm17, %xmm19
+vpsubq            (%rax), %xmm17, %xmm19
+vpsubq            (%rax){1to2}, %xmm17, %xmm19
+vpsubq            %xmm16, %xmm17, %xmm19 {k1}
+vpsubq            (%rax), %xmm17, %xmm19 {k1}
+vpsubq            (%rax){1to2}, %xmm17, %xmm19 {k1}
+vpsubq            %xmm16, %xmm17, %xmm19 {z}{k1}
+vpsubq            (%rax), %xmm17, %xmm19 {z}{k1}
+vpsubq            (%rax){1to2}, %xmm17, %xmm19 {z}{k1}
+
+vpsubq            %ymm16, %ymm17, %ymm19
+vpsubq            (%rax), %ymm17, %ymm19
+vpsubq            (%rax){1to4}, %ymm17, %ymm19
+vpsubq            %ymm16, %ymm17, %ymm19 {k1}
+vpsubq            (%rax), %ymm17, %ymm19 {k1}
+vpsubq            (%rax){1to4}, %ymm17, %ymm19 {k1}
+vpsubq            %ymm16, %ymm17, %ymm19 {z}{k1}
+vpsubq            (%rax), %ymm17, %ymm19 {z}{k1}
+vpsubq            (%rax){1to4}, %ymm17, %ymm19 {z}{k1}
+
+vpunpckhdq        %xmm16, %xmm17, %xmm19
+vpunpckhdq        (%rax), %xmm17, %xmm19
+vpunpckhdq        (%rax){1to4}, %xmm17, %xmm19
+vpunpckhdq        %xmm16, %xmm17, %xmm19 {k1}
+vpunpckhdq        (%rax), %xmm17, %xmm19 {k1}
+vpunpckhdq        (%rax){1to4}, %xmm17, %xmm19 {k1}
+vpunpckhdq        %xmm16, %xmm17, %xmm19 {z}{k1}
+vpunpckhdq        (%rax), %xmm17, %xmm19 {z}{k1}
+vpunpckhdq        (%rax){1to4}, %xmm17, %xmm19 {z}{k1}
+
+vpunpckhdq        %ymm16, %ymm17, %ymm19
+vpunpckhdq        (%rax), %ymm17, %ymm19
+vpunpckhdq        (%rax){1to8}, %ymm17, %ymm19
+vpunpckhdq        %ymm16, %ymm17, %ymm19 {k1}
+vpunpckhdq        (%rax), %ymm17, %ymm19 {k1}
+vpunpckhdq        (%rax){1to8}, %ymm17, %ymm19 {k1}
+vpunpckhdq        %ymm16, %ymm17, %ymm19 {z}{k1}
+vpunpckhdq        (%rax), %ymm17, %ymm19 {z}{k1}
+vpunpckhdq        (%rax){1to8}, %ymm17, %ymm19 {z}{k1}
+
+vpunpckldq        %xmm16, %xmm17, %xmm19
+vpunpckldq        (%rax), %xmm17, %xmm19
+vpunpckldq        (%rax){1to4}, %xmm17, %xmm19
+vpunpckldq        %xmm16, %xmm17, %xmm19 {k1}
+vpunpckldq        (%rax), %xmm17, %xmm19 {k1}
+vpunpckldq        (%rax){1to4}, %xmm17, %xmm19 {k1}
+vpunpckldq        %xmm16, %xmm17, %xmm19 {z}{k1}
+vpunpckldq        (%rax), %xmm17, %xmm19 {z}{k1}
+vpunpckldq        (%rax){1to4}, %xmm17, %xmm19 {z}{k1}
+
+vpunpckldq        %ymm16, %ymm17, %ymm19
+vpunpckldq        (%rax), %ymm17, %ymm19
+vpunpckldq        (%rax){1to8}, %ymm17, %ymm19
+vpunpckldq        %ymm16, %ymm17, %ymm19 {k1}
+vpunpckldq        (%rax), %ymm17, %ymm19 {k1}
+vpunpckldq        (%rax){1to8}, %ymm17, %ymm19 {k1}
+vpunpckldq        %ymm16, %ymm17, %ymm19 {z}{k1}
+vpunpckldq        (%rax), %ymm17, %ymm19 {z}{k1}
+vpunpckldq        (%rax){1to8}, %ymm17, %ymm19 {z}{k1}
+
+vshuff32x4        $0, %ymm16, %ymm17, %ymm19
+vshuff32x4        $0, (%rax), %ymm17, %ymm19
+vshuff32x4        $0, (%rax){1to8}, %ymm17, %ymm19
+vshuff32x4        $0, %ymm16, %ymm17, %ymm19 {k1}
+vshuff32x4        $0, (%rax), %ymm17, %ymm19 {k1}
+vshuff32x4        $0, (%rax){1to8}, %ymm17, %ymm19 {k1}
+vshuff32x4        $0, %ymm16, %ymm17, %ymm19 {z}{k1}
+vshuff32x4        $0, (%rax), %ymm17, %ymm19 {z}{k1}
+vshuff32x4        $0, (%rax){1to8}, %ymm17, %ymm19 {z}{k1}
+
+vshuff64x2        $0, %ymm16, %ymm17, %ymm19
+vshuff64x2        $0, (%rax), %ymm17, %ymm19
+vshuff64x2        $0, (%rax){1to4}, %ymm17, %ymm19
+vshuff64x2        $0, %ymm16, %ymm17, %ymm19 {k1}
+vshuff64x2        $0, (%rax), %ymm17, %ymm19 {k1}
+vshuff64x2        $0, (%rax){1to4}, %ymm17, %ymm19 {k1}
+vshuff64x2        $0, %ymm16, %ymm17, %ymm19 {z}{k1}
+vshuff64x2        $0, (%rax), %ymm17, %ymm19 {z}{k1}
+vshuff64x2        $0, (%rax){1to4}, %ymm17, %ymm19 {z}{k1}
+
+vshufi32x4        $0, %ymm16, %ymm17, %ymm19
+vshufi32x4        $0, (%rax), %ymm17, %ymm19
+vshufi32x4        $0, (%rax){1to8}, %ymm17, %ymm19
+vshufi32x4        $0, %ymm16, %ymm17, %ymm19 {k1}
+vshufi32x4        $0, (%rax), %ymm17, %ymm19 {k1}
+vshufi32x4        $0, (%rax){1to8}, %ymm17, %ymm19 {k1}
+vshufi32x4        $0, %ymm16, %ymm17, %ymm19 {z}{k1}
+vshufi32x4        $0, (%rax), %ymm17, %ymm19 {z}{k1}
+vshufi32x4        $0, (%rax){1to8}, %ymm17, %ymm19 {z}{k1}
+
+vshufi64x2        $0, %ymm16, %ymm17, %ymm19
+vshufi64x2        $0, (%rax), %ymm17, %ymm19
+vshufi64x2        $0, (%rax){1to4}, %ymm17, %ymm19
+vshufi64x2        $0, %ymm16, %ymm17, %ymm19 {k1}
+vshufi64x2        $0, (%rax), %ymm17, %ymm19 {k1}
+vshufi64x2        $0, (%rax){1to4}, %ymm17, %ymm19 {k1}
+vshufi64x2        $0, %ymm16, %ymm17, %ymm19 {z}{k1}
+vshufi64x2        $0, (%rax), %ymm17, %ymm19 {z}{k1}
+vshufi64x2        $0, (%rax){1to4}, %ymm17, %ymm19 {z}{k1}
+
+vsqrtpd           %xmm16, %xmm19
+vsqrtpd           (%rax), %xmm19
+vsqrtpd           (%rax){1to2}, %xmm19
+vsqrtpd           %xmm16, %xmm19 {k1}
+vsqrtpd           (%rax), %xmm19 {k1}
+vsqrtpd           (%rax){1to2}, %xmm19 {k1}
+vsqrtpd           %xmm16, %xmm19 {z}{k1}
+vsqrtpd           (%rax), %xmm19 {z}{k1}
+vsqrtpd           (%rax){1to2}, %xmm19 {z}{k1}
+
+vsqrtpd           %ymm16, %ymm19
+vsqrtpd           (%rax), %ymm19
+vsqrtpd           (%rax){1to4}, %ymm19
+vsqrtpd           %ymm16, %ymm19 {k1}
+vsqrtpd           (%rax), %ymm19 {k1}
+vsqrtpd           (%rax){1to4}, %ymm19 {k1}
+vsqrtpd           %ymm16, %ymm19 {z}{k1}
+vsqrtpd           (%rax), %ymm19 {z}{k1}
+vsqrtpd           (%rax){1to4}, %ymm19 {z}{k1}
+
+vsqrtps           %xmm16, %xmm19
+vsqrtps           (%rax), %xmm19
+vsqrtps           (%rax){1to4}, %xmm19
+vsqrtps           %xmm16, %xmm19 {k1}
+vsqrtps           (%rax), %xmm19 {k1}
+vsqrtps           (%rax){1to4}, %xmm19 {k1}
+vsqrtps           %xmm16, %xmm19 {z}{k1}
+vsqrtps           (%rax), %xmm19 {z}{k1}
+vsqrtps           (%rax){1to4}, %xmm19 {z}{k1}
+
+vsqrtps           %ymm16, %ymm19
+vsqrtps           (%rax), %ymm19
+vsqrtps           (%rax){1to8}, %ymm19
+vsqrtps           %ymm16, %ymm19 {k1}
+vsqrtps           (%rax), %ymm19 {k1}
+vsqrtps           (%rax){1to8}, %ymm19 {k1}
+vsqrtps           %ymm16, %ymm19 {z}{k1}
+vsqrtps           (%rax), %ymm19 {z}{k1}
+vsqrtps           (%rax){1to8}, %ymm19 {z}{k1}
 
 vsubpd            %xmm16, %xmm17, %xmm19
 vsubpd            (%rax), %xmm17, %xmm19
+vsubpd            (%rax){1to2}, %xmm17, %xmm19
 vsubpd            %xmm16, %xmm17, %xmm19 {k1}
 vsubpd            (%rax), %xmm17, %xmm19 {k1}
+vsubpd            (%rax){1to2}, %xmm17, %xmm19 {k1}
 vsubpd            %xmm16, %xmm17, %xmm19 {z}{k1}
 vsubpd            (%rax), %xmm17, %xmm19 {z}{k1}
+vsubpd            (%rax){1to2}, %xmm17, %xmm19 {z}{k1}
 
 vsubpd            %ymm16, %ymm17, %ymm19
 vsubpd            (%rax), %ymm17, %ymm19
+vsubpd            (%rax){1to4}, %ymm17, %ymm19
 vsubpd            %ymm16, %ymm17, %ymm19 {k1}
 vsubpd            (%rax), %ymm17, %ymm19 {k1}
+vsubpd            (%rax){1to4}, %ymm17, %ymm19 {k1}
 vsubpd            %ymm16, %ymm17, %ymm19 {z}{k1}
 vsubpd            (%rax), %ymm17, %ymm19 {z}{k1}
+vsubpd            (%rax){1to4}, %ymm17, %ymm19 {z}{k1}
 
 vsubps            %xmm16, %xmm17, %xmm19
 vsubps            (%rax), %xmm17, %xmm19
+vsubps            (%rax){1to4}, %xmm17, %xmm19
 vsubps            %xmm16, %xmm17, %xmm19 {k1}
 vsubps            (%rax), %xmm17, %xmm19 {k1}
+vsubps            (%rax){1to4}, %xmm17, %xmm19 {k1}
 vsubps            %xmm16, %xmm17, %xmm19 {z}{k1}
 vsubps            (%rax), %xmm17, %xmm19 {z}{k1}
+vsubps            (%rax){1to4}, %xmm17, %xmm19 {z}{k1}
 
 vsubps            %ymm16, %ymm17, %ymm19
 vsubps            (%rax), %ymm17, %ymm19
+vsubps            (%rax){1to8}, %ymm17, %ymm19
 vsubps            %ymm16, %ymm17, %ymm19 {k1}
 vsubps            (%rax), %ymm17, %ymm19 {k1}
+vsubps            (%rax){1to8}, %ymm17, %ymm19 {k1}
 vsubps            %ymm16, %ymm17, %ymm19 {z}{k1}
 vsubps            (%rax), %ymm17, %ymm19 {z}{k1}
+vsubps            (%rax){1to8}, %ymm17, %ymm19 {z}{k1}
+
+vunpckhpd         %xmm16, %xmm17, %xmm19
+vunpckhpd         (%rax), %xmm17, %xmm19
+vunpckhpd         (%rax){1to2}, %xmm17, %xmm19
+vunpckhpd         %xmm16, %xmm17, %xmm19 {k1}
+vunpckhpd         (%rax), %xmm17, %xmm19 {k1}
+vunpckhpd         (%rax){1to2}, %xmm17, %xmm19 {k1}
+vunpckhpd         %xmm16, %xmm17, %xmm19 {z}{k1}
+vunpckhpd         (%rax), %xmm17, %xmm19 {z}{k1}
+vunpckhpd         (%rax){1to2}, %xmm17, %xmm19 {z}{k1}
+
+vunpckhpd         %ymm16, %ymm17, %ymm19
+vunpckhpd         (%rax), %ymm17, %ymm19
+vunpckhpd         (%rax){1to4}, %ymm17, %ymm19
+vunpckhpd         %ymm16, %ymm17, %ymm19 {k1}
+vunpckhpd         (%rax), %ymm17, %ymm19 {k1}
+vunpckhpd         (%rax){1to4}, %ymm17, %ymm19 {k1}
+vunpckhpd         %ymm16, %ymm17, %ymm19 {z}{k1}
+vunpckhpd         (%rax), %ymm17, %ymm19 {z}{k1}
+vunpckhpd         (%rax){1to4}, %ymm17, %ymm19 {z}{k1}
+
+vunpckhps         %xmm16, %xmm17, %xmm19
+vunpckhps         (%rax), %xmm17, %xmm19
+vunpckhps         (%rax){1to4}, %xmm17, %xmm19
+vunpckhps         %xmm16, %xmm17, %xmm19 {k1}
+vunpckhps         (%rax), %xmm17, %xmm19 {k1}
+vunpckhps         (%rax){1to4}, %xmm17, %xmm19 {k1}
+vunpckhps         %xmm16, %xmm17, %xmm19 {z}{k1}
+vunpckhps         (%rax), %xmm17, %xmm19 {z}{k1}
+vunpckhps         (%rax){1to4}, %xmm17, %xmm19 {z}{k1}
+
+vunpckhps         %ymm16, %ymm17, %ymm19
+vunpckhps         (%rax), %ymm17, %ymm19
+vunpckhps         (%rax){1to8}, %ymm17, %ymm19
+vunpckhps         %ymm16, %ymm17, %ymm19 {k1}
+vunpckhps         (%rax), %ymm17, %ymm19 {k1}
+vunpckhps         (%rax){1to8}, %ymm17, %ymm19 {k1}
+vunpckhps         %ymm16, %ymm17, %ymm19 {z}{k1}
+vunpckhps         (%rax), %ymm17, %ymm19 {z}{k1}
+vunpckhps         (%rax){1to8}, %ymm17, %ymm19 {z}{k1}
+
+vunpcklpd         %xmm16, %xmm17, %xmm19
+vunpcklpd         (%rax), %xmm17, %xmm19
+vunpcklpd         (%rax){1to2}, %xmm17, %xmm19
+vunpcklpd         %xmm16, %xmm17, %xmm19 {k1}
+vunpcklpd         (%rax), %xmm17, %xmm19 {k1}
+vunpcklpd         (%rax){1to2}, %xmm17, %xmm19 {k1}
+vunpcklpd         %xmm16, %xmm17, %xmm19 {z}{k1}
+vunpcklpd         (%rax), %xmm17, %xmm19 {z}{k1}
+vunpcklpd         (%rax){1to2}, %xmm17, %xmm19 {z}{k1}
+
+vunpcklpd         %ymm16, %ymm17, %ymm19
+vunpcklpd         (%rax), %ymm17, %ymm19
+vunpcklpd         (%rax){1to4}, %ymm17, %ymm19
+vunpcklpd         %ymm16, %ymm17, %ymm19 {k1}
+vunpcklpd         (%rax), %ymm17, %ymm19 {k1}
+vunpcklpd         (%rax){1to4}, %ymm17, %ymm19 {k1}
+vunpcklpd         %ymm16, %ymm17, %ymm19 {z}{k1}
+vunpcklpd         (%rax), %ymm17, %ymm19 {z}{k1}
+vunpcklpd         (%rax){1to4}, %ymm17, %ymm19 {z}{k1}
+
+vunpcklps         %xmm16, %xmm17, %xmm19
+vunpcklps         (%rax), %xmm17, %xmm19
+vunpcklps         (%rax){1to4}, %xmm17, %xmm19
+vunpcklps         %xmm16, %xmm17, %xmm19 {k1}
+vunpcklps         (%rax), %xmm17, %xmm19 {k1}
+vunpcklps         (%rax){1to4}, %xmm17, %xmm19 {k1}
+vunpcklps         %xmm16, %xmm17, %xmm19 {z}{k1}
+vunpcklps         (%rax), %xmm17, %xmm19 {z}{k1}
+vunpcklps         (%rax){1to4}, %xmm17, %xmm19 {z}{k1}
+
+vunpcklps         %ymm16, %ymm17, %ymm19
+vunpcklps         (%rax), %ymm17, %ymm19
+vunpcklps         (%rax){1to8}, %ymm17, %ymm19
+vunpcklps         %ymm16, %ymm17, %ymm19 {k1}
+vunpcklps         (%rax), %ymm17, %ymm19 {k1}
+vunpcklps         (%rax){1to8}, %ymm17, %ymm19 {k1}
+vunpcklps         %ymm16, %ymm17, %ymm19 {z}{k1}
+vunpcklps         (%rax), %ymm17, %ymm19 {z}{k1}
+vunpcklps         (%rax){1to8}, %ymm17, %ymm19 {z}{k1}
 
 # CHECK:      Instruction Info:
 # CHECK-NEXT: [1]: #uOps
@@ -124,100 +752,670 @@ vsubps            (%rax), %ymm17, %ymm19 {z}{k1}
 # CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
 # CHECK-NEXT:  1      4     0.50                        vaddpd	%xmm16, %xmm17, %xmm19
 # CHECK-NEXT:  2      10    0.50    *                   vaddpd	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  2      10    0.50    *                   vaddpd	(%rax){1to2}, %xmm17, %xmm19
 # CHECK-NEXT:  1      4     0.50                        vaddpd	%xmm16, %xmm17, %xmm19 {%k1}
 # CHECK-NEXT:  2      10    0.50    *                   vaddpd	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      10    0.50    *                   vaddpd	(%rax){1to2}, %xmm17, %xmm19 {%k1}
 # CHECK-NEXT:  1      4     0.50                        vaddpd	%xmm16, %xmm17, %xmm19 {%k1} {z}
 # CHECK-NEXT:  2      10    0.50    *                   vaddpd	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      10    0.50    *                   vaddpd	(%rax){1to2}, %xmm17, %xmm19 {%k1} {z}
 # CHECK-NEXT:  1      4     0.50                        vaddpd	%ymm16, %ymm17, %ymm19
 # CHECK-NEXT:  2      11    0.50    *                   vaddpd	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  2      11    0.50    *                   vaddpd	(%rax){1to4}, %ymm17, %ymm19
 # CHECK-NEXT:  1      4     0.50                        vaddpd	%ymm16, %ymm17, %ymm19 {%k1}
 # CHECK-NEXT:  2      11    0.50    *                   vaddpd	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      11    0.50    *                   vaddpd	(%rax){1to4}, %ymm17, %ymm19 {%k1}
 # CHECK-NEXT:  1      4     0.50                        vaddpd	%ymm16, %ymm17, %ymm19 {%k1} {z}
 # CHECK-NEXT:  2      11    0.50    *                   vaddpd	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      11    0.50    *                   vaddpd	(%rax){1to4}, %ymm17, %ymm19 {%k1} {z}
 # CHECK-NEXT:  1      4     0.50                        vaddps	%xmm16, %xmm17, %xmm19
 # CHECK-NEXT:  2      10    0.50    *                   vaddps	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  2      10    0.50    *                   vaddps	(%rax){1to4}, %xmm17, %xmm19
 # CHECK-NEXT:  1      4     0.50                        vaddps	%xmm16, %xmm17, %xmm19 {%k1}
 # CHECK-NEXT:  2      10    0.50    *                   vaddps	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      10    0.50    *                   vaddps	(%rax){1to4}, %xmm17, %xmm19 {%k1}
 # CHECK-NEXT:  1      4     0.50                        vaddps	%xmm16, %xmm17, %xmm19 {%k1} {z}
 # CHECK-NEXT:  2      10    0.50    *                   vaddps	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      10    0.50    *                   vaddps	(%rax){1to4}, %xmm17, %xmm19 {%k1} {z}
 # CHECK-NEXT:  1      4     0.50                        vaddps	%ymm16, %ymm17, %ymm19
 # CHECK-NEXT:  2      11    0.50    *                   vaddps	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  2      11    0.50    *                   vaddps	(%rax){1to8}, %ymm17, %ymm19
 # CHECK-NEXT:  1      4     0.50                        vaddps	%ymm16, %ymm17, %ymm19 {%k1}
 # CHECK-NEXT:  2      11    0.50    *                   vaddps	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      11    0.50    *                   vaddps	(%rax){1to8}, %ymm17, %ymm19 {%k1}
 # CHECK-NEXT:  1      4     0.50                        vaddps	%ymm16, %ymm17, %ymm19 {%k1} {z}
 # CHECK-NEXT:  2      11    0.50    *                   vaddps	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      11    0.50    *                   vaddps	(%rax){1to8}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      5     1.00                        vcvtdq2pd	%xmm16, %xmm19
+# CHECK-NEXT:  2      10    0.50    *                   vcvtdq2pd	(%rax), %xmm19
+# CHECK-NEXT:  2      10    0.50    *                   vcvtdq2pd	(%rax){1to2}, %xmm19
+# CHECK-NEXT:  2      5     1.00                        vcvtdq2pd	%xmm16, %xmm19 {%k1}
+# CHECK-NEXT:  2      10    0.50    *                   vcvtdq2pd	(%rax), %xmm19 {%k1}
+# CHECK-NEXT:  2      10    0.50    *                   vcvtdq2pd	(%rax){1to2}, %xmm19 {%k1}
+# CHECK-NEXT:  2      5     1.00                        vcvtdq2pd	%xmm16, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      10    0.50    *                   vcvtdq2pd	(%rax), %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      10    0.50    *                   vcvtdq2pd	(%rax){1to2}, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      7     1.00                        vcvtdq2pd	%xmm16, %ymm19
+# CHECK-NEXT:  2      11    0.50    *                   vcvtdq2pd	(%rax), %ymm19
+# CHECK-NEXT:  2      11    0.50    *                   vcvtdq2pd	(%rax){1to4}, %ymm19
+# CHECK-NEXT:  2      7     1.00                        vcvtdq2pd	%xmm16, %ymm19 {%k1}
+# CHECK-NEXT:  2      11    0.50    *                   vcvtdq2pd	(%rax), %ymm19 {%k1}
+# CHECK-NEXT:  2      11    0.50    *                   vcvtdq2pd	(%rax){1to4}, %ymm19 {%k1}
+# CHECK-NEXT:  2      7     1.00                        vcvtdq2pd	%xmm16, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      11    0.50    *                   vcvtdq2pd	(%rax), %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      11    0.50    *                   vcvtdq2pd	(%rax){1to4}, %ymm19 {%k1} {z}
+# CHECK-NEXT:  1      4     0.50                        vcvtdq2ps	%xmm16, %xmm19
+# CHECK-NEXT:  2      10    0.50    *                   vcvtdq2ps	(%rax), %xmm19
+# CHECK-NEXT:  2      10    0.50    *                   vcvtdq2ps	(%rax){1to4}, %xmm19
+# CHECK-NEXT:  1      4     0.50                        vcvtdq2ps	%xmm16, %xmm19 {%k1}
+# CHECK-NEXT:  2      10    0.50    *                   vcvtdq2ps	(%rax), %xmm19 {%k1}
+# CHECK-NEXT:  2      10    0.50    *                   vcvtdq2ps	(%rax){1to4}, %xmm19 {%k1}
+# CHECK-NEXT:  1      4     0.50                        vcvtdq2ps	%xmm16, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      10    0.50    *                   vcvtdq2ps	(%rax), %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      10    0.50    *                   vcvtdq2ps	(%rax){1to4}, %xmm19 {%k1} {z}
+# CHECK-NEXT:  1      4     0.50                        vcvtdq2ps	%ymm16, %ymm19
+# CHECK-NEXT:  2      11    0.50    *                   vcvtdq2ps	(%rax), %ymm19
+# CHECK-NEXT:  2      11    0.50    *                   vcvtdq2ps	(%rax){1to8}, %ymm19
+# CHECK-NEXT:  1      4     0.50                        vcvtdq2ps	%ymm16, %ymm19 {%k1}
+# CHECK-NEXT:  2      11    0.50    *                   vcvtdq2ps	(%rax), %ymm19 {%k1}
+# CHECK-NEXT:  2      11    0.50    *                   vcvtdq2ps	(%rax){1to8}, %ymm19 {%k1}
+# CHECK-NEXT:  1      4     0.50                        vcvtdq2ps	%ymm16, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      11    0.50    *                   vcvtdq2ps	(%rax), %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      11    0.50    *                   vcvtdq2ps	(%rax){1to8}, %ymm19 {%k1} {z}
 # CHECK-NEXT:  1      14    3.00                        vdivpd	%xmm16, %xmm17, %xmm19
 # CHECK-NEXT:  2      20    4.00    *                   vdivpd	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  2      20    4.00    *                   vdivpd	(%rax){1to2}, %xmm17, %xmm19
 # CHECK-NEXT:  1      14    3.00                        vdivpd	%xmm16, %xmm17, %xmm19 {%k1}
 # CHECK-NEXT:  2      20    4.00    *                   vdivpd	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      20    4.00    *                   vdivpd	(%rax){1to2}, %xmm17, %xmm19 {%k1}
 # CHECK-NEXT:  1      14    3.00                        vdivpd	%xmm16, %xmm17, %xmm19 {%k1} {z}
 # CHECK-NEXT:  2      20    4.00    *                   vdivpd	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      20    4.00    *                   vdivpd	(%rax){1to2}, %xmm17, %xmm19 {%k1} {z}
 # CHECK-NEXT:  1      14    5.00                        vdivpd	%ymm16, %ymm17, %ymm19
 # CHECK-NEXT:  2      21    8.00    *                   vdivpd	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  2      21    8.00    *                   vdivpd	(%rax){1to4}, %ymm17, %ymm19
 # CHECK-NEXT:  1      14    5.00                        vdivpd	%ymm16, %ymm17, %ymm19 {%k1}
 # CHECK-NEXT:  2      21    8.00    *                   vdivpd	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      21    8.00    *                   vdivpd	(%rax){1to4}, %ymm17, %ymm19 {%k1}
 # CHECK-NEXT:  1      14    5.00                        vdivpd	%ymm16, %ymm17, %ymm19 {%k1} {z}
 # CHECK-NEXT:  2      21    8.00    *                   vdivpd	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      21    8.00    *                   vdivpd	(%rax){1to4}, %ymm17, %ymm19 {%k1} {z}
 # CHECK-NEXT:  1      11    3.00                        vdivps	%xmm16, %xmm17, %xmm19
 # CHECK-NEXT:  2      17    5.00    *                   vdivps	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  2      17    5.00    *                   vdivps	(%rax){1to4}, %xmm17, %xmm19
 # CHECK-NEXT:  1      11    3.00                        vdivps	%xmm16, %xmm17, %xmm19 {%k1}
 # CHECK-NEXT:  2      17    5.00    *                   vdivps	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      17    5.00    *                   vdivps	(%rax){1to4}, %xmm17, %xmm19 {%k1}
 # CHECK-NEXT:  1      11    3.00                        vdivps	%xmm16, %xmm17, %xmm19 {%k1} {z}
 # CHECK-NEXT:  2      17    5.00    *                   vdivps	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      17    5.00    *                   vdivps	(%rax){1to4}, %xmm17, %xmm19 {%k1} {z}
 # CHECK-NEXT:  1      11    5.00                        vdivps	%ymm16, %ymm17, %ymm19
 # CHECK-NEXT:  2      18    5.00    *                   vdivps	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  2      18    5.00    *                   vdivps	(%rax){1to8}, %ymm17, %ymm19
 # CHECK-NEXT:  1      11    5.00                        vdivps	%ymm16, %ymm17, %ymm19 {%k1}
 # CHECK-NEXT:  2      18    5.00    *                   vdivps	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      18    5.00    *                   vdivps	(%rax){1to8}, %ymm17, %ymm19 {%k1}
 # CHECK-NEXT:  1      11    5.00                        vdivps	%ymm16, %ymm17, %ymm19 {%k1} {z}
 # CHECK-NEXT:  2      18    5.00    *                   vdivps	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      18    5.00    *                   vdivps	(%rax){1to8}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  1      4     0.50                        vmaxpd	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  2      10    0.50    *                   vmaxpd	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  2      10    0.50    *                   vmaxpd	(%rax){1to2}, %xmm17, %xmm19
+# CHECK-NEXT:  1      4     0.50                        vmaxpd	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      10    0.50    *                   vmaxpd	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      10    0.50    *                   vmaxpd	(%rax){1to2}, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  1      4     0.50                        vmaxpd	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      10    0.50    *                   vmaxpd	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      10    0.50    *                   vmaxpd	(%rax){1to2}, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  1      4     0.50                        vmaxpd	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  2      11    0.50    *                   vmaxpd	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  2      11    0.50    *                   vmaxpd	(%rax){1to4}, %ymm17, %ymm19
+# CHECK-NEXT:  1      4     0.50                        vmaxpd	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      11    0.50    *                   vmaxpd	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      11    0.50    *                   vmaxpd	(%rax){1to4}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  1      4     0.50                        vmaxpd	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      11    0.50    *                   vmaxpd	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      11    0.50    *                   vmaxpd	(%rax){1to4}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  1      4     0.50                        vmaxps	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  2      10    0.50    *                   vmaxps	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  2      10    0.50    *                   vmaxps	(%rax){1to4}, %xmm17, %xmm19
+# CHECK-NEXT:  1      4     0.50                        vmaxps	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      10    0.50    *                   vmaxps	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      10    0.50    *                   vmaxps	(%rax){1to4}, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  1      4     0.50                        vmaxps	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      10    0.50    *                   vmaxps	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      10    0.50    *                   vmaxps	(%rax){1to4}, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  1      4     0.50                        vmaxps	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  2      11    0.50    *                   vmaxps	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  2      11    0.50    *                   vmaxps	(%rax){1to8}, %ymm17, %ymm19
+# CHECK-NEXT:  1      4     0.50                        vmaxps	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      11    0.50    *                   vmaxps	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      11    0.50    *                   vmaxps	(%rax){1to8}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  1      4     0.50                        vmaxps	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      11    0.50    *                   vmaxps	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      11    0.50    *                   vmaxps	(%rax){1to8}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  1      4     0.50                        vminpd	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  2      10    0.50    *                   vminpd	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  2      10    0.50    *                   vminpd	(%rax){1to2}, %xmm17, %xmm19
+# CHECK-NEXT:  1      4     0.50                        vminpd	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      10    0.50    *                   vminpd	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      10    0.50    *                   vminpd	(%rax){1to2}, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  1      4     0.50                        vminpd	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      10    0.50    *                   vminpd	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      10    0.50    *                   vminpd	(%rax){1to2}, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  1      4     0.50                        vminpd	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  2      11    0.50    *                   vminpd	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  2      11    0.50    *                   vminpd	(%rax){1to4}, %ymm17, %ymm19
+# CHECK-NEXT:  1      4     0.50                        vminpd	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      11    0.50    *                   vminpd	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      11    0.50    *                   vminpd	(%rax){1to4}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  1      4     0.50                        vminpd	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      11    0.50    *                   vminpd	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      11    0.50    *                   vminpd	(%rax){1to4}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  1      4     0.50                        vminps	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  2      10    0.50    *                   vminps	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  2      10    0.50    *                   vminps	(%rax){1to4}, %xmm17, %xmm19
+# CHECK-NEXT:  1      4     0.50                        vminps	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      10    0.50    *                   vminps	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      10    0.50    *                   vminps	(%rax){1to4}, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  1      4     0.50                        vminps	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      10    0.50    *                   vminps	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      10    0.50    *                   vminps	(%rax){1to4}, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  1      4     0.50                        vminps	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  2      11    0.50    *                   vminps	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  2      11    0.50    *                   vminps	(%rax){1to8}, %ymm17, %ymm19
+# CHECK-NEXT:  1      4     0.50                        vminps	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      11    0.50    *                   vminps	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      11    0.50    *                   vminps	(%rax){1to8}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  1      4     0.50                        vminps	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      11    0.50    *                   vminps	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      11    0.50    *                   vminps	(%rax){1to8}, %ymm17, %ymm19 {%k1} {z}
 # CHECK-NEXT:  1      4     0.50                        vmulpd	%xmm16, %xmm17, %xmm19
 # CHECK-NEXT:  2      10    0.50    *                   vmulpd	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  2      10    0.50    *                   vmulpd	(%rax){1to2}, %xmm17, %xmm19
 # CHECK-NEXT:  1      4     0.50                        vmulpd	%xmm16, %xmm17, %xmm19 {%k1}
 # CHECK-NEXT:  2      10    0.50    *                   vmulpd	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      10    0.50    *                   vmulpd	(%rax){1to2}, %xmm17, %xmm19 {%k1}
 # CHECK-NEXT:  1      4     0.50                        vmulpd	%xmm16, %xmm17, %xmm19 {%k1} {z}
 # CHECK-NEXT:  2      10    0.50    *                   vmulpd	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      10    0.50    *                   vmulpd	(%rax){1to2}, %xmm17, %xmm19 {%k1} {z}
 # CHECK-NEXT:  1      4     0.50                        vmulpd	%ymm16, %ymm17, %ymm19
 # CHECK-NEXT:  2      11    0.50    *                   vmulpd	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  2      11    0.50    *                   vmulpd	(%rax){1to4}, %ymm17, %ymm19
 # CHECK-NEXT:  1      4     0.50                        vmulpd	%ymm16, %ymm17, %ymm19 {%k1}
 # CHECK-NEXT:  2      11    0.50    *                   vmulpd	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      11    0.50    *                   vmulpd	(%rax){1to4}, %ymm17, %ymm19 {%k1}
 # CHECK-NEXT:  1      4     0.50                        vmulpd	%ymm16, %ymm17, %ymm19 {%k1} {z}
 # CHECK-NEXT:  2      11    0.50    *                   vmulpd	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      11    0.50    *                   vmulpd	(%rax){1to4}, %ymm17, %ymm19 {%k1} {z}
 # CHECK-NEXT:  1      4     0.50                        vmulps	%xmm16, %xmm17, %xmm19
 # CHECK-NEXT:  2      10    0.50    *                   vmulps	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  2      10    0.50    *                   vmulps	(%rax){1to4}, %xmm17, %xmm19
 # CHECK-NEXT:  1      4     0.50                        vmulps	%xmm16, %xmm17, %xmm19 {%k1}
 # CHECK-NEXT:  2      10    0.50    *                   vmulps	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      10    0.50    *                   vmulps	(%rax){1to4}, %xmm17, %xmm19 {%k1}
 # CHECK-NEXT:  1      4     0.50                        vmulps	%xmm16, %xmm17, %xmm19 {%k1} {z}
 # CHECK-NEXT:  2      10    0.50    *                   vmulps	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      10    0.50    *                   vmulps	(%rax){1to4}, %xmm17, %xmm19 {%k1} {z}
 # CHECK-NEXT:  1      4     0.50                        vmulps	%ymm16, %ymm17, %ymm19
 # CHECK-NEXT:  2      11    0.50    *                   vmulps	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  2      11    0.50    *                   vmulps	(%rax){1to8}, %ymm17, %ymm19
 # CHECK-NEXT:  1      4     0.50                        vmulps	%ymm16, %ymm17, %ymm19 {%k1}
 # CHECK-NEXT:  2      11    0.50    *                   vmulps	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      11    0.50    *                   vmulps	(%rax){1to8}, %ymm17, %ymm19 {%k1}
 # CHECK-NEXT:  1      4     0.50                        vmulps	%ymm16, %ymm17, %ymm19 {%k1} {z}
 # CHECK-NEXT:  2      11    0.50    *                   vmulps	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      11    0.50    *                   vmulps	(%rax){1to8}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  1      1     0.33                        vpaddd	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  2      7     0.50    *                   vpaddd	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  2      7     0.50    *                   vpaddd	(%rax){1to4}, %xmm17, %xmm19
+# CHECK-NEXT:  1      1     0.33                        vpaddd	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      7     0.50    *                   vpaddd	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      7     0.50    *                   vpaddd	(%rax){1to4}, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  1      1     0.33                        vpaddd	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      7     0.50    *                   vpaddd	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      7     0.50    *                   vpaddd	(%rax){1to4}, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     0.33                        vpaddd	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  2      8     0.50    *                   vpaddd	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  2      8     0.50    *                   vpaddd	(%rax){1to8}, %ymm17, %ymm19
+# CHECK-NEXT:  1      1     0.33                        vpaddd	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     0.50    *                   vpaddd	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     0.50    *                   vpaddd	(%rax){1to8}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  1      1     0.33                        vpaddd	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     0.50    *                   vpaddd	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     0.50    *                   vpaddd	(%rax){1to8}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  1      1     0.33                        vpaddq	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  2      7     0.50    *                   vpaddq	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  2      7     0.50    *                   vpaddq	(%rax){1to2}, %xmm17, %xmm19
+# CHECK-NEXT:  1      1     0.33                        vpaddq	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      7     0.50    *                   vpaddq	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      7     0.50    *                   vpaddq	(%rax){1to2}, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  1      1     0.33                        vpaddq	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      7     0.50    *                   vpaddq	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      7     0.50    *                   vpaddq	(%rax){1to2}, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     0.33                        vpaddq	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  2      8     0.50    *                   vpaddq	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  2      8     0.50    *                   vpaddq	(%rax){1to4}, %ymm17, %ymm19
+# CHECK-NEXT:  1      1     0.33                        vpaddq	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     0.50    *                   vpaddq	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     0.50    *                   vpaddq	(%rax){1to4}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  1      1     0.33                        vpaddq	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     0.50    *                   vpaddq	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     0.50    *                   vpaddq	(%rax){1to4}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  1      3     1.00                        vpermd	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  2      10    1.00    *                   vpermd	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  2      10    1.00    *                   vpermd	(%rax){1to8}, %ymm17, %ymm19
+# CHECK-NEXT:  1      3     1.00                        vpermd	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      10    1.00    *                   vpermd	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      10    1.00    *                   vpermd	(%rax){1to8}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  1      3     1.00                        vpermd	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      10    1.00    *                   vpermd	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      10    1.00    *                   vpermd	(%rax){1to8}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vpermilpd	$0, %xmm16, %xmm19
+# CHECK-NEXT:  2      7     1.00    *                   vpermilpd	$0, (%rax), %xmm19
+# CHECK-NEXT:  2      7     1.00    *                   vpermilpd	$0, (%rax){1to2}, %xmm19
+# CHECK-NEXT:  1      1     1.00                        vpermilpd	$0, %xmm16, %xmm19 {%k1}
+# CHECK-NEXT:  2      7     1.00    *                   vpermilpd	$0, (%rax), %xmm19 {%k1}
+# CHECK-NEXT:  2      7     1.00    *                   vpermilpd	$0, (%rax){1to2}, %xmm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vpermilpd	$0, %xmm16, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      7     1.00    *                   vpermilpd	$0, (%rax), %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      7     1.00    *                   vpermilpd	$0, (%rax){1to2}, %xmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vpermilpd	$0, %ymm16, %ymm19
+# CHECK-NEXT:  2      8     1.00    *                   vpermilpd	$0, (%rax), %ymm19
+# CHECK-NEXT:  2      8     1.00    *                   vpermilpd	$0, (%rax){1to4}, %ymm19
+# CHECK-NEXT:  1      1     1.00                        vpermilpd	$0, %ymm16, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vpermilpd	$0, (%rax), %ymm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vpermilpd	$0, (%rax){1to4}, %ymm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vpermilpd	$0, %ymm16, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vpermilpd	$0, (%rax), %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vpermilpd	$0, (%rax){1to4}, %ymm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vpermilpd	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  2      7     1.00    *                   vpermilpd	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  2      7     1.00    *                   vpermilpd	(%rax){1to2}, %xmm17, %xmm19
+# CHECK-NEXT:  1      1     1.00                        vpermilpd	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      7     1.00    *                   vpermilpd	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      7     1.00    *                   vpermilpd	(%rax){1to2}, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vpermilpd	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      7     1.00    *                   vpermilpd	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      7     1.00    *                   vpermilpd	(%rax){1to2}, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vpermilpd	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  2      8     1.00    *                   vpermilpd	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  2      8     1.00    *                   vpermilpd	(%rax){1to4}, %ymm17, %ymm19
+# CHECK-NEXT:  1      1     1.00                        vpermilpd	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vpermilpd	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vpermilpd	(%rax){1to4}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vpermilpd	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vpermilpd	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vpermilpd	(%rax){1to4}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vpermilps	$0, %xmm16, %xmm19
+# CHECK-NEXT:  2      7     1.00    *                   vpermilps	$0, (%rax), %xmm19
+# CHECK-NEXT:  2      7     1.00    *                   vpermilps	$0, (%rax){1to4}, %xmm19
+# CHECK-NEXT:  1      1     1.00                        vpermilps	$0, %xmm16, %xmm19 {%k1}
+# CHECK-NEXT:  2      7     1.00    *                   vpermilps	$0, (%rax), %xmm19 {%k1}
+# CHECK-NEXT:  2      7     1.00    *                   vpermilps	$0, (%rax){1to4}, %xmm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vpermilps	$0, %xmm16, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      7     1.00    *                   vpermilps	$0, (%rax), %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      7     1.00    *                   vpermilps	$0, (%rax){1to4}, %xmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vpermilps	$0, %ymm16, %ymm19
+# CHECK-NEXT:  2      8     1.00    *                   vpermilps	$0, (%rax), %ymm19
+# CHECK-NEXT:  2      8     1.00    *                   vpermilps	$0, (%rax){1to8}, %ymm19
+# CHECK-NEXT:  1      1     1.00                        vpermilps	$0, %ymm16, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vpermilps	$0, (%rax), %ymm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vpermilps	$0, (%rax){1to8}, %ymm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vpermilps	$0, %ymm16, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vpermilps	$0, (%rax), %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vpermilps	$0, (%rax){1to8}, %ymm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vpermilps	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  2      7     1.00    *                   vpermilps	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  2      7     1.00    *                   vpermilps	(%rax){1to4}, %xmm17, %xmm19
+# CHECK-NEXT:  1      1     1.00                        vpermilps	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      7     1.00    *                   vpermilps	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      7     1.00    *                   vpermilps	(%rax){1to4}, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vpermilps	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      7     1.00    *                   vpermilps	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      7     1.00    *                   vpermilps	(%rax){1to4}, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vpermilps	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  2      8     1.00    *                   vpermilps	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  2      8     1.00    *                   vpermilps	(%rax){1to8}, %ymm17, %ymm19
+# CHECK-NEXT:  1      1     1.00                        vpermilps	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vpermilps	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vpermilps	(%rax){1to8}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vpermilps	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vpermilps	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vpermilps	(%rax){1to8}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  1      3     1.00                        vpermpd	$0, %ymm16, %ymm19
+# CHECK-NEXT:  2      10    1.00    *                   vpermpd	$0, (%rax), %ymm19
+# CHECK-NEXT:  2      10    1.00    *                   vpermpd	$0, (%rax){1to4}, %ymm19
+# CHECK-NEXT:  1      3     1.00                        vpermpd	$0, %ymm16, %ymm19 {%k1}
+# CHECK-NEXT:  2      10    1.00    *                   vpermpd	$0, (%rax), %ymm19 {%k1}
+# CHECK-NEXT:  2      10    1.00    *                   vpermpd	$0, (%rax){1to4}, %ymm19 {%k1}
+# CHECK-NEXT:  1      3     1.00                        vpermpd	$0, %ymm16, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      10    1.00    *                   vpermpd	$0, (%rax), %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      10    1.00    *                   vpermpd	$0, (%rax){1to4}, %ymm19 {%k1} {z}
+# CHECK-NEXT:  1      3     1.00                        vpermpd	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  2      10    1.00    *                   vpermpd	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  2      10    1.00    *                   vpermpd	(%rax){1to4}, %ymm17, %ymm19
+# CHECK-NEXT:  1      3     1.00                        vpermpd	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      10    1.00    *                   vpermpd	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      10    1.00    *                   vpermpd	(%rax){1to4}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  1      3     1.00                        vpermpd	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      10    1.00    *                   vpermpd	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      10    1.00    *                   vpermpd	(%rax){1to4}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  1      3     1.00                        vpermps	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  2      10    1.00    *                   vpermps	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  2      10    1.00    *                   vpermps	(%rax){1to8}, %ymm17, %ymm19
+# CHECK-NEXT:  1      3     1.00                        vpermps	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      10    1.00    *                   vpermps	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      10    1.00    *                   vpermps	(%rax){1to8}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  1      3     1.00                        vpermps	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      10    1.00    *                   vpermps	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      10    1.00    *                   vpermps	(%rax){1to8}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  1      3     1.00                        vpermq	$0, %ymm16, %ymm19
+# CHECK-NEXT:  2      10    1.00    *                   vpermq	$0, (%rax), %ymm19
+# CHECK-NEXT:  2      10    1.00    *                   vpermq	$0, (%rax){1to4}, %ymm19
+# CHECK-NEXT:  1      3     1.00                        vpermq	$0, %ymm16, %ymm19 {%k1}
+# CHECK-NEXT:  2      10    1.00    *                   vpermq	$0, (%rax), %ymm19 {%k1}
+# CHECK-NEXT:  2      10    1.00    *                   vpermq	$0, (%rax){1to4}, %ymm19 {%k1}
+# CHECK-NEXT:  1      3     1.00                        vpermq	$0, %ymm16, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      10    1.00    *                   vpermq	$0, (%rax), %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      10    1.00    *                   vpermq	$0, (%rax){1to4}, %ymm19 {%k1} {z}
+# CHECK-NEXT:  1      3     1.00                        vpermq	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  2      10    1.00    *                   vpermq	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  2      10    1.00    *                   vpermq	(%rax){1to4}, %ymm17, %ymm19
+# CHECK-NEXT:  1      3     1.00                        vpermq	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      10    1.00    *                   vpermq	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      10    1.00    *                   vpermq	(%rax){1to4}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  1      3     1.00                        vpermq	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      10    1.00    *                   vpermq	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      10    1.00    *                   vpermq	(%rax){1to4}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      10    1.00                        vpmulld	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  3      16    1.00    *                   vpmulld	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  3      16    1.00    *                   vpmulld	(%rax){1to4}, %xmm17, %xmm19
+# CHECK-NEXT:  2      10    1.00                        vpmulld	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  3      16    1.00    *                   vpmulld	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  3      16    1.00    *                   vpmulld	(%rax){1to4}, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      10    1.00                        vpmulld	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  3      16    1.00    *                   vpmulld	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  3      16    1.00    *                   vpmulld	(%rax){1to4}, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      10    1.00                        vpmulld	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  3      17    1.00    *                   vpmulld	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  3      17    1.00    *                   vpmulld	(%rax){1to8}, %ymm17, %ymm19
+# CHECK-NEXT:  2      10    1.00                        vpmulld	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  3      17    1.00    *                   vpmulld	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  3      17    1.00    *                   vpmulld	(%rax){1to8}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      10    1.00                        vpmulld	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  3      17    1.00    *                   vpmulld	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  3      17    1.00    *                   vpmulld	(%rax){1to8}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vpshufd	$0, %xmm16, %xmm19
+# CHECK-NEXT:  2      7     1.00    *                   vpshufd	$0, (%rax), %xmm19
+# CHECK-NEXT:  2      7     1.00    *                   vpshufd	$0, (%rax){1to4}, %xmm19
+# CHECK-NEXT:  1      1     1.00                        vpshufd	$0, %xmm16, %xmm19 {%k1}
+# CHECK-NEXT:  2      7     1.00    *                   vpshufd	$0, (%rax), %xmm19 {%k1}
+# CHECK-NEXT:  2      7     1.00    *                   vpshufd	$0, (%rax){1to4}, %xmm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vpshufd	$0, %xmm16, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      7     1.00    *                   vpshufd	$0, (%rax), %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      7     1.00    *                   vpshufd	$0, (%rax){1to4}, %xmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vpshufd	$0, %ymm16, %ymm19
+# CHECK-NEXT:  2      8     1.00    *                   vpshufd	$0, (%rax), %ymm19
+# CHECK-NEXT:  2      8     1.00    *                   vpshufd	$0, (%rax){1to8}, %ymm19
+# CHECK-NEXT:  1      1     1.00                        vpshufd	$0, %ymm16, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vpshufd	$0, (%rax), %ymm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vpshufd	$0, (%rax){1to8}, %ymm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vpshufd	$0, %ymm16, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vpshufd	$0, (%rax), %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vpshufd	$0, (%rax){1to8}, %ymm19 {%k1} {z}
+# CHECK-NEXT:  1      1     0.33                        vpsubd	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  2      7     0.50    *                   vpsubd	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  2      7     0.50    *                   vpsubd	(%rax){1to4}, %xmm17, %xmm19
+# CHECK-NEXT:  1      1     0.33                        vpsubd	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      7     0.50    *                   vpsubd	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      7     0.50    *                   vpsubd	(%rax){1to4}, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  1      1     0.33                        vpsubd	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      7     0.50    *                   vpsubd	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      7     0.50    *                   vpsubd	(%rax){1to4}, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     0.33                        vpsubd	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  2      8     0.50    *                   vpsubd	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  2      8     0.50    *                   vpsubd	(%rax){1to8}, %ymm17, %ymm19
+# CHECK-NEXT:  1      1     0.33                        vpsubd	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     0.50    *                   vpsubd	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     0.50    *                   vpsubd	(%rax){1to8}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  1      1     0.33                        vpsubd	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     0.50    *                   vpsubd	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     0.50    *                   vpsubd	(%rax){1to8}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  1      1     0.33                        vpsubq	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  2      7     0.50    *                   vpsubq	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  2      7     0.50    *                   vpsubq	(%rax){1to2}, %xmm17, %xmm19
+# CHECK-NEXT:  1      1     0.33                        vpsubq	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      7     0.50    *                   vpsubq	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      7     0.50    *                   vpsubq	(%rax){1to2}, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  1      1     0.33                        vpsubq	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      7     0.50    *                   vpsubq	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      7     0.50    *                   vpsubq	(%rax){1to2}, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     0.33                        vpsubq	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  2      8     0.50    *                   vpsubq	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  2      8     0.50    *                   vpsubq	(%rax){1to4}, %ymm17, %ymm19
+# CHECK-NEXT:  1      1     0.33                        vpsubq	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     0.50    *                   vpsubq	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     0.50    *                   vpsubq	(%rax){1to4}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  1      1     0.33                        vpsubq	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     0.50    *                   vpsubq	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     0.50    *                   vpsubq	(%rax){1to4}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vpunpckhdq	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  2      7     1.00    *                   vpunpckhdq	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  2      7     1.00    *                   vpunpckhdq	(%rax){1to4}, %xmm17, %xmm19
+# CHECK-NEXT:  1      1     1.00                        vpunpckhdq	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      7     1.00    *                   vpunpckhdq	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      7     1.00    *                   vpunpckhdq	(%rax){1to4}, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vpunpckhdq	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      7     1.00    *                   vpunpckhdq	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      7     1.00    *                   vpunpckhdq	(%rax){1to4}, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vpunpckhdq	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  2      8     1.00    *                   vpunpckhdq	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  2      8     1.00    *                   vpunpckhdq	(%rax){1to8}, %ymm17, %ymm19
+# CHECK-NEXT:  1      1     1.00                        vpunpckhdq	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vpunpckhdq	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vpunpckhdq	(%rax){1to8}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vpunpckhdq	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vpunpckhdq	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vpunpckhdq	(%rax){1to8}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vpunpckldq	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  2      7     1.00    *                   vpunpckldq	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  2      7     1.00    *                   vpunpckldq	(%rax){1to4}, %xmm17, %xmm19
+# CHECK-NEXT:  1      1     1.00                        vpunpckldq	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      7     1.00    *                   vpunpckldq	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      7     1.00    *                   vpunpckldq	(%rax){1to4}, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vpunpckldq	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      7     1.00    *                   vpunpckldq	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      7     1.00    *                   vpunpckldq	(%rax){1to4}, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vpunpckldq	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  2      8     1.00    *                   vpunpckldq	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  2      8     1.00    *                   vpunpckldq	(%rax){1to8}, %ymm17, %ymm19
+# CHECK-NEXT:  1      1     1.00                        vpunpckldq	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vpunpckldq	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vpunpckldq	(%rax){1to8}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vpunpckldq	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vpunpckldq	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vpunpckldq	(%rax){1to8}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  1      3     1.00                        vshuff32x4	$0, %ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  2      10    1.00    *                   vshuff32x4	$0, (%rax), %ymm17, %ymm19
+# CHECK-NEXT:  2      10    1.00    *                   vshuff32x4	$0, (%rax){1to8}, %ymm17, %ymm19
+# CHECK-NEXT:  1      3     1.00                        vshuff32x4	$0, %ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      10    1.00    *                   vshuff32x4	$0, (%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      10    1.00    *                   vshuff32x4	$0, (%rax){1to8}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  1      3     1.00                        vshuff32x4	$0, %ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      10    1.00    *                   vshuff32x4	$0, (%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      10    1.00    *                   vshuff32x4	$0, (%rax){1to8}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  1      3     1.00                        vshuff64x2	$0, %ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  2      10    1.00    *                   vshuff64x2	$0, (%rax), %ymm17, %ymm19
+# CHECK-NEXT:  2      10    1.00    *                   vshuff64x2	$0, (%rax){1to4}, %ymm17, %ymm19
+# CHECK-NEXT:  1      3     1.00                        vshuff64x2	$0, %ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      10    1.00    *                   vshuff64x2	$0, (%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      10    1.00    *                   vshuff64x2	$0, (%rax){1to4}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  1      3     1.00                        vshuff64x2	$0, %ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      10    1.00    *                   vshuff64x2	$0, (%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      10    1.00    *                   vshuff64x2	$0, (%rax){1to4}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  1      3     1.00                        vshufi32x4	$0, %ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  2      10    1.00    *                   vshufi32x4	$0, (%rax), %ymm17, %ymm19
+# CHECK-NEXT:  2      10    1.00    *                   vshufi32x4	$0, (%rax){1to8}, %ymm17, %ymm19
+# CHECK-NEXT:  1      3     1.00                        vshufi32x4	$0, %ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      10    1.00    *                   vshufi32x4	$0, (%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      10    1.00    *                   vshufi32x4	$0, (%rax){1to8}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  1      3     1.00                        vshufi32x4	$0, %ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      10    1.00    *                   vshufi32x4	$0, (%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      10    1.00    *                   vshufi32x4	$0, (%rax){1to8}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  1      3     1.00                        vshufi64x2	$0, %ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  2      10    1.00    *                   vshufi64x2	$0, (%rax), %ymm17, %ymm19
+# CHECK-NEXT:  2      10    1.00    *                   vshufi64x2	$0, (%rax){1to4}, %ymm17, %ymm19
+# CHECK-NEXT:  1      3     1.00                        vshufi64x2	$0, %ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      10    1.00    *                   vshufi64x2	$0, (%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      10    1.00    *                   vshufi64x2	$0, (%rax){1to4}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  1      3     1.00                        vshufi64x2	$0, %ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      10    1.00    *                   vshufi64x2	$0, (%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      10    1.00    *                   vshufi64x2	$0, (%rax){1to4}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  1      18    6.00                        vsqrtpd	%xmm16, %xmm19
+# CHECK-NEXT:  2      24    6.00    *                   vsqrtpd	(%rax), %xmm19
+# CHECK-NEXT:  2      24    6.00    *                   vsqrtpd	(%rax){1to2}, %xmm19
+# CHECK-NEXT:  1      18    6.00                        vsqrtpd	%xmm16, %xmm19 {%k1}
+# CHECK-NEXT:  2      24    6.00    *                   vsqrtpd	(%rax), %xmm19 {%k1}
+# CHECK-NEXT:  2      24    6.00    *                   vsqrtpd	(%rax){1to2}, %xmm19 {%k1}
+# CHECK-NEXT:  1      18    6.00                        vsqrtpd	%xmm16, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      24    6.00    *                   vsqrtpd	(%rax), %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      24    6.00    *                   vsqrtpd	(%rax){1to2}, %xmm19 {%k1} {z}
+# CHECK-NEXT:  1      18    12.00                       vsqrtpd	%ymm16, %ymm19
+# CHECK-NEXT:  2      25    12.00   *                   vsqrtpd	(%rax), %ymm19
+# CHECK-NEXT:  2      25    12.00   *                   vsqrtpd	(%rax){1to4}, %ymm19
+# CHECK-NEXT:  1      18    12.00                       vsqrtpd	%ymm16, %ymm19 {%k1}
+# CHECK-NEXT:  2      25    12.00   *                   vsqrtpd	(%rax), %ymm19 {%k1}
+# CHECK-NEXT:  2      25    12.00   *                   vsqrtpd	(%rax){1to4}, %ymm19 {%k1}
+# CHECK-NEXT:  1      18    12.00                       vsqrtpd	%ymm16, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      25    12.00   *                   vsqrtpd	(%rax), %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      25    12.00   *                   vsqrtpd	(%rax){1to4}, %ymm19 {%k1} {z}
+# CHECK-NEXT:  1      12    3.00                        vsqrtps	%xmm16, %xmm19
+# CHECK-NEXT:  2      18    3.00    *                   vsqrtps	(%rax), %xmm19
+# CHECK-NEXT:  2      18    3.00    *                   vsqrtps	(%rax){1to4}, %xmm19
+# CHECK-NEXT:  1      12    3.00                        vsqrtps	%xmm16, %xmm19 {%k1}
+# CHECK-NEXT:  2      18    3.00    *                   vsqrtps	(%rax), %xmm19 {%k1}
+# CHECK-NEXT:  2      18    3.00    *                   vsqrtps	(%rax){1to4}, %xmm19 {%k1}
+# CHECK-NEXT:  1      12    3.00                        vsqrtps	%xmm16, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      18    3.00    *                   vsqrtps	(%rax), %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      18    3.00    *                   vsqrtps	(%rax){1to4}, %xmm19 {%k1} {z}
+# CHECK-NEXT:  1      12    6.00                        vsqrtps	%ymm16, %ymm19
+# CHECK-NEXT:  2      19    6.00    *                   vsqrtps	(%rax), %ymm19
+# CHECK-NEXT:  2      19    6.00    *                   vsqrtps	(%rax){1to8}, %ymm19
+# CHECK-NEXT:  1      12    6.00                        vsqrtps	%ymm16, %ymm19 {%k1}
+# CHECK-NEXT:  2      19    6.00    *                   vsqrtps	(%rax), %ymm19 {%k1}
+# CHECK-NEXT:  2      19    6.00    *                   vsqrtps	(%rax){1to8}, %ymm19 {%k1}
+# CHECK-NEXT:  1      12    6.00                        vsqrtps	%ymm16, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      19    6.00    *                   vsqrtps	(%rax), %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      19    6.00    *                   vsqrtps	(%rax){1to8}, %ymm19 {%k1} {z}
 # CHECK-NEXT:  1      4     0.50                        vsubpd	%xmm16, %xmm17, %xmm19
 # CHECK-NEXT:  2      10    0.50    *                   vsubpd	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  2      10    0.50    *                   vsubpd	(%rax){1to2}, %xmm17, %xmm19
 # CHECK-NEXT:  1      4     0.50                        vsubpd	%xmm16, %xmm17, %xmm19 {%k1}
 # CHECK-NEXT:  2      10    0.50    *                   vsubpd	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      10    0.50    *                   vsubpd	(%rax){1to2}, %xmm17, %xmm19 {%k1}
 # CHECK-NEXT:  1      4     0.50                        vsubpd	%xmm16, %xmm17, %xmm19 {%k1} {z}
 # CHECK-NEXT:  2      10    0.50    *                   vsubpd	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      10    0.50    *                   vsubpd	(%rax){1to2}, %xmm17, %xmm19 {%k1} {z}
 # CHECK-NEXT:  1      4     0.50                        vsubpd	%ymm16, %ymm17, %ymm19
 # CHECK-NEXT:  2      11    0.50    *                   vsubpd	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  2      11    0.50    *                   vsubpd	(%rax){1to4}, %ymm17, %ymm19
 # CHECK-NEXT:  1      4     0.50                        vsubpd	%ymm16, %ymm17, %ymm19 {%k1}
 # CHECK-NEXT:  2      11    0.50    *                   vsubpd	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      11    0.50    *                   vsubpd	(%rax){1to4}, %ymm17, %ymm19 {%k1}
 # CHECK-NEXT:  1      4     0.50                        vsubpd	%ymm16, %ymm17, %ymm19 {%k1} {z}
 # CHECK-NEXT:  2      11    0.50    *                   vsubpd	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      11    0.50    *                   vsubpd	(%rax){1to4}, %ymm17, %ymm19 {%k1} {z}
 # CHECK-NEXT:  1      4     0.50                        vsubps	%xmm16, %xmm17, %xmm19
 # CHECK-NEXT:  2      10    0.50    *                   vsubps	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  2      10    0.50    *                   vsubps	(%rax){1to4}, %xmm17, %xmm19
 # CHECK-NEXT:  1      4     0.50                        vsubps	%xmm16, %xmm17, %xmm19 {%k1}
 # CHECK-NEXT:  2      10    0.50    *                   vsubps	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      10    0.50    *                   vsubps	(%rax){1to4}, %xmm17, %xmm19 {%k1}
 # CHECK-NEXT:  1      4     0.50                        vsubps	%xmm16, %xmm17, %xmm19 {%k1} {z}
 # CHECK-NEXT:  2      10    0.50    *                   vsubps	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      10    0.50    *                   vsubps	(%rax){1to4}, %xmm17, %xmm19 {%k1} {z}
 # CHECK-NEXT:  1      4     0.50                        vsubps	%ymm16, %ymm17, %ymm19
 # CHECK-NEXT:  2      11    0.50    *                   vsubps	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  2      11    0.50    *                   vsubps	(%rax){1to8}, %ymm17, %ymm19
 # CHECK-NEXT:  1      4     0.50                        vsubps	%ymm16, %ymm17, %ymm19 {%k1}
 # CHECK-NEXT:  2      11    0.50    *                   vsubps	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      11    0.50    *                   vsubps	(%rax){1to8}, %ymm17, %ymm19 {%k1}
 # CHECK-NEXT:  1      4     0.50                        vsubps	%ymm16, %ymm17, %ymm19 {%k1} {z}
 # CHECK-NEXT:  2      11    0.50    *                   vsubps	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      11    0.50    *                   vsubps	(%rax){1to8}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vunpckhpd	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  2      7     1.00    *                   vunpckhpd	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  2      7     1.00    *                   vunpckhpd	(%rax){1to2}, %xmm17, %xmm19
+# CHECK-NEXT:  1      1     1.00                        vunpckhpd	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      7     1.00    *                   vunpckhpd	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      7     1.00    *                   vunpckhpd	(%rax){1to2}, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vunpckhpd	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      7     1.00    *                   vunpckhpd	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      7     1.00    *                   vunpckhpd	(%rax){1to2}, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vunpckhpd	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  2      8     1.00    *                   vunpckhpd	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  2      8     1.00    *                   vunpckhpd	(%rax){1to4}, %ymm17, %ymm19
+# CHECK-NEXT:  1      1     1.00                        vunpckhpd	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vunpckhpd	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vunpckhpd	(%rax){1to4}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vunpckhpd	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vunpckhpd	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vunpckhpd	(%rax){1to4}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vunpckhps	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  2      7     1.00    *                   vunpckhps	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  2      7     1.00    *                   vunpckhps	(%rax){1to4}, %xmm17, %xmm19
+# CHECK-NEXT:  1      1     1.00                        vunpckhps	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      7     1.00    *                   vunpckhps	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      7     1.00    *                   vunpckhps	(%rax){1to4}, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vunpckhps	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      7     1.00    *                   vunpckhps	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      7     1.00    *                   vunpckhps	(%rax){1to4}, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vunpckhps	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  2      8     1.00    *                   vunpckhps	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  2      8     1.00    *                   vunpckhps	(%rax){1to8}, %ymm17, %ymm19
+# CHECK-NEXT:  1      1     1.00                        vunpckhps	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vunpckhps	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vunpckhps	(%rax){1to8}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vunpckhps	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vunpckhps	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vunpckhps	(%rax){1to8}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vunpcklpd	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  2      7     1.00    *                   vunpcklpd	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  2      7     1.00    *                   vunpcklpd	(%rax){1to2}, %xmm17, %xmm19
+# CHECK-NEXT:  1      1     1.00                        vunpcklpd	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      7     1.00    *                   vunpcklpd	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      7     1.00    *                   vunpcklpd	(%rax){1to2}, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vunpcklpd	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      7     1.00    *                   vunpcklpd	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      7     1.00    *                   vunpcklpd	(%rax){1to2}, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vunpcklpd	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  2      8     1.00    *                   vunpcklpd	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  2      8     1.00    *                   vunpcklpd	(%rax){1to4}, %ymm17, %ymm19
+# CHECK-NEXT:  1      1     1.00                        vunpcklpd	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vunpcklpd	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vunpcklpd	(%rax){1to4}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vunpcklpd	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vunpcklpd	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vunpcklpd	(%rax){1to4}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vunpcklps	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  2      7     1.00    *                   vunpcklps	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  2      7     1.00    *                   vunpcklps	(%rax){1to4}, %xmm17, %xmm19
+# CHECK-NEXT:  1      1     1.00                        vunpcklps	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      7     1.00    *                   vunpcklps	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  2      7     1.00    *                   vunpcklps	(%rax){1to4}, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vunpcklps	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      7     1.00    *                   vunpcklps	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  2      7     1.00    *                   vunpcklps	(%rax){1to4}, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  1      1     1.00                        vunpcklps	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  2      8     1.00    *                   vunpcklps	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  2      8     1.00    *                   vunpcklps	(%rax){1to8}, %ymm17, %ymm19
+# CHECK-NEXT:  1      1     1.00                        vunpcklps	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vunpcklps	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  2      8     1.00    *                   vunpcklps	(%rax){1to8}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  1      1     1.00                        vunpcklps	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vunpcklps	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  2      8     1.00    *                   vunpcklps	(%rax){1to8}, %ymm17, %ymm19 {%k1} {z}
 
 # CHECK:      Resources:
 # CHECK-NEXT: [0]   - SKXDivider
@@ -233,103 +1431,673 @@ vsubps            (%rax), %ymm17, %ymm19 {z}{k1}
 
 # CHECK:      Resource pressure per iteration:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]
-# CHECK-NEXT:  -     114.00 60.00  36.00  24.00  24.00   -      -      -      -
+# CHECK-NEXT:  -     423.00 217.00 145.00 222.00 222.00  -     328.00  -      -
 
 # CHECK:      Resource pressure by instruction:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    Instructions:
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -     vaddpd	%xmm16, %xmm17, %xmm19
 # CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vaddpd	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vaddpd	(%rax){1to2}, %xmm17, %xmm19
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -     vaddpd	%xmm16, %xmm17, %xmm19 {%k1}
 # CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vaddpd	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vaddpd	(%rax){1to2}, %xmm17, %xmm19 {%k1}
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -     vaddpd	%xmm16, %xmm17, %xmm19 {%k1} {z}
 # CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vaddpd	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vaddpd	(%rax){1to2}, %xmm17, %xmm19 {%k1} {z}
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -     vaddpd	%ymm16, %ymm17, %ymm19
 # CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vaddpd	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vaddpd	(%rax){1to4}, %ymm17, %ymm19
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -     vaddpd	%ymm16, %ymm17, %ymm19 {%k1}
 # CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vaddpd	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vaddpd	(%rax){1to4}, %ymm17, %ymm19 {%k1}
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -     vaddpd	%ymm16, %ymm17, %ymm19 {%k1} {z}
 # CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vaddpd	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vaddpd	(%rax){1to4}, %ymm17, %ymm19 {%k1} {z}
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -     vaddps	%xmm16, %xmm17, %xmm19
 # CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vaddps	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vaddps	(%rax){1to4}, %xmm17, %xmm19
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -     vaddps	%xmm16, %xmm17, %xmm19 {%k1}
 # CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vaddps	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vaddps	(%rax){1to4}, %xmm17, %xmm19 {%k1}
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -     vaddps	%xmm16, %xmm17, %xmm19 {%k1} {z}
 # CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vaddps	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vaddps	(%rax){1to4}, %xmm17, %xmm19 {%k1} {z}
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -     vaddps	%ymm16, %ymm17, %ymm19
 # CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vaddps	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vaddps	(%rax){1to8}, %ymm17, %ymm19
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -     vaddps	%ymm16, %ymm17, %ymm19 {%k1}
 # CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vaddps	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vaddps	(%rax){1to8}, %ymm17, %ymm19 {%k1}
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -     vaddps	%ymm16, %ymm17, %ymm19 {%k1} {z}
 # CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vaddps	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vaddps	(%rax){1to8}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     1.33    -      -     vcvtdq2pd	%xmm16, %xmm19
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vcvtdq2pd	(%rax), %xmm19
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vcvtdq2pd	(%rax){1to2}, %xmm19
+# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     1.33    -      -     vcvtdq2pd	%xmm16, %xmm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vcvtdq2pd	(%rax), %xmm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vcvtdq2pd	(%rax){1to2}, %xmm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     1.33    -      -     vcvtdq2pd	%xmm16, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vcvtdq2pd	(%rax), %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vcvtdq2pd	(%rax){1to2}, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     1.33    -      -     vcvtdq2pd	%xmm16, %ymm19
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vcvtdq2pd	(%rax), %ymm19
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vcvtdq2pd	(%rax){1to4}, %ymm19
+# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     1.33    -      -     vcvtdq2pd	%xmm16, %ymm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vcvtdq2pd	(%rax), %ymm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vcvtdq2pd	(%rax){1to4}, %ymm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     1.33    -      -     vcvtdq2pd	%xmm16, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vcvtdq2pd	(%rax), %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vcvtdq2pd	(%rax){1to4}, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -     vcvtdq2ps	%xmm16, %xmm19
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vcvtdq2ps	(%rax), %xmm19
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vcvtdq2ps	(%rax){1to4}, %xmm19
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -     vcvtdq2ps	%xmm16, %xmm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vcvtdq2ps	(%rax), %xmm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vcvtdq2ps	(%rax){1to4}, %xmm19 {%k1}
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -     vcvtdq2ps	%xmm16, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vcvtdq2ps	(%rax), %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vcvtdq2ps	(%rax){1to4}, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -     vcvtdq2ps	%ymm16, %ymm19
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vcvtdq2ps	(%rax), %ymm19
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vcvtdq2ps	(%rax){1to8}, %ymm19
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -     vcvtdq2ps	%ymm16, %ymm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vcvtdq2ps	(%rax), %ymm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vcvtdq2ps	(%rax){1to8}, %ymm19 {%k1}
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -     vcvtdq2ps	%ymm16, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vcvtdq2ps	(%rax), %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vcvtdq2ps	(%rax){1to8}, %ymm19 {%k1} {z}
 # CHECK-NEXT:  -     3.00   1.00    -      -      -      -      -      -      -     vdivpd	%xmm16, %xmm17, %xmm19
 # CHECK-NEXT:  -     4.00   1.00    -     0.50   0.50    -      -      -      -     vdivpd	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  -     4.00   1.00    -     0.50   0.50    -      -      -      -     vdivpd	(%rax){1to2}, %xmm17, %xmm19
 # CHECK-NEXT:  -     3.00   1.00    -      -      -      -      -      -      -     vdivpd	%xmm16, %xmm17, %xmm19 {%k1}
 # CHECK-NEXT:  -     4.00   1.00    -     0.50   0.50    -      -      -      -     vdivpd	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -     4.00   1.00    -     0.50   0.50    -      -      -      -     vdivpd	(%rax){1to2}, %xmm17, %xmm19 {%k1}
 # CHECK-NEXT:  -     3.00   1.00    -      -      -      -      -      -      -     vdivpd	%xmm16, %xmm17, %xmm19 {%k1} {z}
 # CHECK-NEXT:  -     4.00   1.00    -     0.50   0.50    -      -      -      -     vdivpd	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -     4.00   1.00    -     0.50   0.50    -      -      -      -     vdivpd	(%rax){1to2}, %xmm17, %xmm19 {%k1} {z}
 # CHECK-NEXT:  -     5.00   1.00    -      -      -      -      -      -      -     vdivpd	%ymm16, %ymm17, %ymm19
 # CHECK-NEXT:  -     8.00   1.00    -     0.50   0.50    -      -      -      -     vdivpd	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  -     8.00   1.00    -     0.50   0.50    -      -      -      -     vdivpd	(%rax){1to4}, %ymm17, %ymm19
 # CHECK-NEXT:  -     5.00   1.00    -      -      -      -      -      -      -     vdivpd	%ymm16, %ymm17, %ymm19 {%k1}
 # CHECK-NEXT:  -     8.00   1.00    -     0.50   0.50    -      -      -      -     vdivpd	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -     8.00   1.00    -     0.50   0.50    -      -      -      -     vdivpd	(%rax){1to4}, %ymm17, %ymm19 {%k1}
 # CHECK-NEXT:  -     5.00   1.00    -      -      -      -      -      -      -     vdivpd	%ymm16, %ymm17, %ymm19 {%k1} {z}
 # CHECK-NEXT:  -     8.00   1.00    -     0.50   0.50    -      -      -      -     vdivpd	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -     8.00   1.00    -     0.50   0.50    -      -      -      -     vdivpd	(%rax){1to4}, %ymm17, %ymm19 {%k1} {z}
 # CHECK-NEXT:  -     3.00   1.00    -      -      -      -      -      -      -     vdivps	%xmm16, %xmm17, %xmm19
 # CHECK-NEXT:  -     5.00   1.00    -     0.50   0.50    -      -      -      -     vdivps	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  -     5.00   1.00    -     0.50   0.50    -      -      -      -     vdivps	(%rax){1to4}, %xmm17, %xmm19
 # CHECK-NEXT:  -     3.00   1.00    -      -      -      -      -      -      -     vdivps	%xmm16, %xmm17, %xmm19 {%k1}
 # CHECK-NEXT:  -     5.00   1.00    -     0.50   0.50    -      -      -      -     vdivps	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -     5.00   1.00    -     0.50   0.50    -      -      -      -     vdivps	(%rax){1to4}, %xmm17, %xmm19 {%k1}
 # CHECK-NEXT:  -     3.00   1.00    -      -      -      -      -      -      -     vdivps	%xmm16, %xmm17, %xmm19 {%k1} {z}
 # CHECK-NEXT:  -     5.00   1.00    -     0.50   0.50    -      -      -      -     vdivps	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -     5.00   1.00    -     0.50   0.50    -      -      -      -     vdivps	(%rax){1to4}, %xmm17, %xmm19 {%k1} {z}
 # CHECK-NEXT:  -     5.00   1.00    -      -      -      -      -      -      -     vdivps	%ymm16, %ymm17, %ymm19
 # CHECK-NEXT:  -     5.00   1.00    -     0.50   0.50    -      -      -      -     vdivps	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  -     5.00   1.00    -     0.50   0.50    -      -      -      -     vdivps	(%rax){1to8}, %ymm17, %ymm19
 # CHECK-NEXT:  -     5.00   1.00    -      -      -      -      -      -      -     vdivps	%ymm16, %ymm17, %ymm19 {%k1}
 # CHECK-NEXT:  -     5.00   1.00    -     0.50   0.50    -      -      -      -     vdivps	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -     5.00   1.00    -     0.50   0.50    -      -      -      -     vdivps	(%rax){1to8}, %ymm17, %ymm19 {%k1}
 # CHECK-NEXT:  -     5.00   1.00    -      -      -      -      -      -      -     vdivps	%ymm16, %ymm17, %ymm19 {%k1} {z}
 # CHECK-NEXT:  -     5.00   1.00    -     0.50   0.50    -      -      -      -     vdivps	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -     5.00   1.00    -     0.50   0.50    -      -      -      -     vdivps	(%rax){1to8}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -     vmaxpd	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vmaxpd	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vmaxpd	(%rax){1to2}, %xmm17, %xmm19
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -     vmaxpd	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vmaxpd	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vmaxpd	(%rax){1to2}, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -     vmaxpd	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vmaxpd	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vmaxpd	(%rax){1to2}, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -     vmaxpd	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vmaxpd	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vmaxpd	(%rax){1to4}, %ymm17, %ymm19
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -     vmaxpd	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vmaxpd	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vmaxpd	(%rax){1to4}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -     vmaxpd	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vmaxpd	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vmaxpd	(%rax){1to4}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -     vmaxps	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vmaxps	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vmaxps	(%rax){1to4}, %xmm17, %xmm19
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -     vmaxps	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vmaxps	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vmaxps	(%rax){1to4}, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -     vmaxps	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vmaxps	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vmaxps	(%rax){1to4}, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -     vmaxps	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vmaxps	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vmaxps	(%rax){1to8}, %ymm17, %ymm19
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -     vmaxps	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vmaxps	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vmaxps	(%rax){1to8}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -     vmaxps	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vmaxps	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vmaxps	(%rax){1to8}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -     vminpd	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vminpd	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vminpd	(%rax){1to2}, %xmm17, %xmm19
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -     vminpd	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vminpd	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vminpd	(%rax){1to2}, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -     vminpd	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vminpd	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vminpd	(%rax){1to2}, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -     vminpd	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vminpd	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vminpd	(%rax){1to4}, %ymm17, %ymm19
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -     vminpd	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vminpd	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vminpd	(%rax){1to4}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -     vminpd	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vminpd	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vminpd	(%rax){1to4}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -     vminps	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vminps	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vminps	(%rax){1to4}, %xmm17, %xmm19
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -     vminps	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vminps	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vminps	(%rax){1to4}, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -     vminps	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vminps	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vminps	(%rax){1to4}, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -     vminps	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vminps	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vminps	(%rax){1to8}, %ymm17, %ymm19
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -     vminps	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vminps	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vminps	(%rax){1to8}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -     vminps	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vminps	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vminps	(%rax){1to8}, %ymm17, %ymm19 {%k1} {z}
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -     vmulpd	%xmm16, %xmm17, %xmm19
 # CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vmulpd	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vmulpd	(%rax){1to2}, %xmm17, %xmm19
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -     vmulpd	%xmm16, %xmm17, %xmm19 {%k1}
 # CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vmulpd	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vmulpd	(%rax){1to2}, %xmm17, %xmm19 {%k1}
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -     vmulpd	%xmm16, %xmm17, %xmm19 {%k1} {z}
 # CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vmulpd	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vmulpd	(%rax){1to2}, %xmm17, %xmm19 {%k1} {z}
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -     vmulpd	%ymm16, %ymm17, %ymm19
 # CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vmulpd	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vmulpd	(%rax){1to4}, %ymm17, %ymm19
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -     vmulpd	%ymm16, %ymm17, %ymm19 {%k1}
 # CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vmulpd	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vmulpd	(%rax){1to4}, %ymm17, %ymm19 {%k1}
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -     vmulpd	%ymm16, %ymm17, %ymm19 {%k1} {z}
 # CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vmulpd	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vmulpd	(%rax){1to4}, %ymm17, %ymm19 {%k1} {z}
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -     vmulps	%xmm16, %xmm17, %xmm19
 # CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vmulps	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vmulps	(%rax){1to4}, %xmm17, %xmm19
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -     vmulps	%xmm16, %xmm17, %xmm19 {%k1}
 # CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vmulps	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vmulps	(%rax){1to4}, %xmm17, %xmm19 {%k1}
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -     vmulps	%xmm16, %xmm17, %xmm19 {%k1} {z}
 # CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vmulps	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vmulps	(%rax){1to4}, %xmm17, %xmm19 {%k1} {z}
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -     vmulps	%ymm16, %ymm17, %ymm19
 # CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vmulps	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vmulps	(%rax){1to8}, %ymm17, %ymm19
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -     vmulps	%ymm16, %ymm17, %ymm19 {%k1}
 # CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vmulps	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vmulps	(%rax){1to8}, %ymm17, %ymm19 {%k1}
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -     vmulps	%ymm16, %ymm17, %ymm19 {%k1} {z}
 # CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vmulps	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vmulps	(%rax){1to8}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vpaddd	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpaddd	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpaddd	(%rax){1to4}, %xmm17, %xmm19
+# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vpaddd	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpaddd	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpaddd	(%rax){1to4}, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vpaddd	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpaddd	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpaddd	(%rax){1to4}, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vpaddd	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpaddd	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpaddd	(%rax){1to8}, %ymm17, %ymm19
+# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vpaddd	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpaddd	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpaddd	(%rax){1to8}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vpaddd	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpaddd	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpaddd	(%rax){1to8}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vpaddq	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpaddq	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpaddq	(%rax){1to2}, %xmm17, %xmm19
+# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vpaddq	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpaddq	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpaddq	(%rax){1to2}, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vpaddq	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpaddq	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpaddq	(%rax){1to2}, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vpaddq	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpaddq	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpaddq	(%rax){1to4}, %ymm17, %ymm19
+# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vpaddq	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpaddq	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpaddq	(%rax){1to4}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vpaddq	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpaddq	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpaddq	(%rax){1to4}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpermd	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermd	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermd	(%rax){1to8}, %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpermd	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermd	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermd	(%rax){1to8}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpermd	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermd	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermd	(%rax){1to8}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpermilpd	$0, %xmm16, %xmm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermilpd	$0, (%rax), %xmm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermilpd	$0, (%rax){1to2}, %xmm19
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpermilpd	$0, %xmm16, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermilpd	$0, (%rax), %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermilpd	$0, (%rax){1to2}, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpermilpd	$0, %xmm16, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermilpd	$0, (%rax), %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermilpd	$0, (%rax){1to2}, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpermilpd	$0, %ymm16, %ymm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermilpd	$0, (%rax), %ymm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermilpd	$0, (%rax){1to4}, %ymm19
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpermilpd	$0, %ymm16, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermilpd	$0, (%rax), %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermilpd	$0, (%rax){1to4}, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpermilpd	$0, %ymm16, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermilpd	$0, (%rax), %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermilpd	$0, (%rax){1to4}, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpermilpd	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermilpd	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermilpd	(%rax){1to2}, %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpermilpd	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermilpd	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermilpd	(%rax){1to2}, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpermilpd	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermilpd	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermilpd	(%rax){1to2}, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpermilpd	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermilpd	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermilpd	(%rax){1to4}, %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpermilpd	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermilpd	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermilpd	(%rax){1to4}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpermilpd	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermilpd	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermilpd	(%rax){1to4}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpermilps	$0, %xmm16, %xmm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermilps	$0, (%rax), %xmm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermilps	$0, (%rax){1to4}, %xmm19
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpermilps	$0, %xmm16, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermilps	$0, (%rax), %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermilps	$0, (%rax){1to4}, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpermilps	$0, %xmm16, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermilps	$0, (%rax), %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermilps	$0, (%rax){1to4}, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpermilps	$0, %ymm16, %ymm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermilps	$0, (%rax), %ymm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermilps	$0, (%rax){1to8}, %ymm19
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpermilps	$0, %ymm16, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermilps	$0, (%rax), %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermilps	$0, (%rax){1to8}, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpermilps	$0, %ymm16, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermilps	$0, (%rax), %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermilps	$0, (%rax){1to8}, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpermilps	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermilps	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermilps	(%rax){1to4}, %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpermilps	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermilps	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermilps	(%rax){1to4}, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpermilps	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermilps	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermilps	(%rax){1to4}, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpermilps	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermilps	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermilps	(%rax){1to8}, %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpermilps	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermilps	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermilps	(%rax){1to8}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpermilps	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermilps	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermilps	(%rax){1to8}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpermpd	$0, %ymm16, %ymm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermpd	$0, (%rax), %ymm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermpd	$0, (%rax){1to4}, %ymm19
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpermpd	$0, %ymm16, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermpd	$0, (%rax), %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermpd	$0, (%rax){1to4}, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpermpd	$0, %ymm16, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermpd	$0, (%rax), %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermpd	$0, (%rax){1to4}, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpermpd	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermpd	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermpd	(%rax){1to4}, %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpermpd	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermpd	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermpd	(%rax){1to4}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpermpd	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermpd	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermpd	(%rax){1to4}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpermps	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermps	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermps	(%rax){1to8}, %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpermps	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermps	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermps	(%rax){1to8}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpermps	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermps	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermps	(%rax){1to8}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpermq	$0, %ymm16, %ymm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermq	$0, (%rax), %ymm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermq	$0, (%rax){1to4}, %ymm19
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpermq	$0, %ymm16, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermq	$0, (%rax), %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermq	$0, (%rax){1to4}, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpermq	$0, %ymm16, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermq	$0, (%rax), %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermq	$0, (%rax){1to4}, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpermq	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermq	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermq	(%rax){1to4}, %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpermq	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermq	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermq	(%rax){1to4}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpermq	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermq	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpermq	(%rax){1to4}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -     1.00   1.00    -      -      -      -      -      -     vpmulld	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  -      -     1.00   1.00   0.50   0.50    -      -      -      -     vpmulld	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  -      -     1.00   1.00   0.50   0.50    -      -      -      -     vpmulld	(%rax){1to4}, %xmm17, %xmm19
+# CHECK-NEXT:  -      -     1.00   1.00    -      -      -      -      -      -     vpmulld	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -     1.00   1.00   0.50   0.50    -      -      -      -     vpmulld	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -     1.00   1.00   0.50   0.50    -      -      -      -     vpmulld	(%rax){1to4}, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -     1.00   1.00    -      -      -      -      -      -     vpmulld	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     1.00   1.00   0.50   0.50    -      -      -      -     vpmulld	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     1.00   1.00   0.50   0.50    -      -      -      -     vpmulld	(%rax){1to4}, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     1.00   1.00    -      -      -      -      -      -     vpmulld	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  -      -     1.00   1.00   0.50   0.50    -      -      -      -     vpmulld	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  -      -     1.00   1.00   0.50   0.50    -      -      -      -     vpmulld	(%rax){1to8}, %ymm17, %ymm19
+# CHECK-NEXT:  -      -     1.00   1.00    -      -      -      -      -      -     vpmulld	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -     1.00   1.00   0.50   0.50    -      -      -      -     vpmulld	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -     1.00   1.00   0.50   0.50    -      -      -      -     vpmulld	(%rax){1to8}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -     1.00   1.00    -      -      -      -      -      -     vpmulld	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -     1.00   1.00   0.50   0.50    -      -      -      -     vpmulld	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -     1.00   1.00   0.50   0.50    -      -      -      -     vpmulld	(%rax){1to8}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpshufd	$0, %xmm16, %xmm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpshufd	$0, (%rax), %xmm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpshufd	$0, (%rax){1to4}, %xmm19
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpshufd	$0, %xmm16, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpshufd	$0, (%rax), %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpshufd	$0, (%rax){1to4}, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpshufd	$0, %xmm16, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpshufd	$0, (%rax), %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpshufd	$0, (%rax){1to4}, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpshufd	$0, %ymm16, %ymm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpshufd	$0, (%rax), %ymm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpshufd	$0, (%rax){1to8}, %ymm19
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpshufd	$0, %ymm16, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpshufd	$0, (%rax), %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpshufd	$0, (%rax){1to8}, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpshufd	$0, %ymm16, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpshufd	$0, (%rax), %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpshufd	$0, (%rax){1to8}, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vpsubd	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpsubd	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpsubd	(%rax){1to4}, %xmm17, %xmm19
+# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vpsubd	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpsubd	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpsubd	(%rax){1to4}, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vpsubd	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpsubd	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpsubd	(%rax){1to4}, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vpsubd	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpsubd	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpsubd	(%rax){1to8}, %ymm17, %ymm19
+# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vpsubd	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpsubd	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpsubd	(%rax){1to8}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vpsubd	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpsubd	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpsubd	(%rax){1to8}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vpsubq	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpsubq	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpsubq	(%rax){1to2}, %xmm17, %xmm19
+# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vpsubq	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpsubq	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpsubq	(%rax){1to2}, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vpsubq	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpsubq	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpsubq	(%rax){1to2}, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vpsubq	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpsubq	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpsubq	(%rax){1to4}, %ymm17, %ymm19
+# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vpsubq	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpsubq	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpsubq	(%rax){1to4}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -     0.33   0.33    -      -      -     0.33    -      -     vpsubq	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpsubq	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     0.33    -      -     vpsubq	(%rax){1to4}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpunpckhdq	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpunpckhdq	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpunpckhdq	(%rax){1to4}, %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpunpckhdq	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpunpckhdq	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpunpckhdq	(%rax){1to4}, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpunpckhdq	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpunpckhdq	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpunpckhdq	(%rax){1to4}, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpunpckhdq	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpunpckhdq	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpunpckhdq	(%rax){1to8}, %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpunpckhdq	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpunpckhdq	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpunpckhdq	(%rax){1to8}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpunpckhdq	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpunpckhdq	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpunpckhdq	(%rax){1to8}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpunpckldq	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpunpckldq	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpunpckldq	(%rax){1to4}, %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpunpckldq	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpunpckldq	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpunpckldq	(%rax){1to4}, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpunpckldq	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpunpckldq	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpunpckldq	(%rax){1to4}, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpunpckldq	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpunpckldq	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpunpckldq	(%rax){1to8}, %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpunpckldq	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpunpckldq	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpunpckldq	(%rax){1to8}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vpunpckldq	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpunpckldq	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vpunpckldq	(%rax){1to8}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vshuff32x4	$0, %ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vshuff32x4	$0, (%rax), %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vshuff32x4	$0, (%rax){1to8}, %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vshuff32x4	$0, %ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vshuff32x4	$0, (%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vshuff32x4	$0, (%rax){1to8}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vshuff32x4	$0, %ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vshuff32x4	$0, (%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vshuff32x4	$0, (%rax){1to8}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vshuff64x2	$0, %ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vshuff64x2	$0, (%rax), %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vshuff64x2	$0, (%rax){1to4}, %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vshuff64x2	$0, %ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vshuff64x2	$0, (%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vshuff64x2	$0, (%rax){1to4}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vshuff64x2	$0, %ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vshuff64x2	$0, (%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vshuff64x2	$0, (%rax){1to4}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vshufi32x4	$0, %ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vshufi32x4	$0, (%rax), %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vshufi32x4	$0, (%rax){1to8}, %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vshufi32x4	$0, %ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vshufi32x4	$0, (%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vshufi32x4	$0, (%rax){1to8}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vshufi32x4	$0, %ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vshufi32x4	$0, (%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vshufi32x4	$0, (%rax){1to8}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vshufi64x2	$0, %ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vshufi64x2	$0, (%rax), %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vshufi64x2	$0, (%rax){1to4}, %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vshufi64x2	$0, %ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vshufi64x2	$0, (%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vshufi64x2	$0, (%rax){1to4}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vshufi64x2	$0, %ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vshufi64x2	$0, (%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vshufi64x2	$0, (%rax){1to4}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -     6.00   1.00    -      -      -      -      -      -      -     vsqrtpd	%xmm16, %xmm19
+# CHECK-NEXT:  -     6.00   1.00    -     0.50   0.50    -      -      -      -     vsqrtpd	(%rax), %xmm19
+# CHECK-NEXT:  -     6.00   1.00    -     0.50   0.50    -      -      -      -     vsqrtpd	(%rax){1to2}, %xmm19
+# CHECK-NEXT:  -     6.00   1.00    -      -      -      -      -      -      -     vsqrtpd	%xmm16, %xmm19 {%k1}
+# CHECK-NEXT:  -     6.00   1.00    -     0.50   0.50    -      -      -      -     vsqrtpd	(%rax), %xmm19 {%k1}
+# CHECK-NEXT:  -     6.00   1.00    -     0.50   0.50    -      -      -      -     vsqrtpd	(%rax){1to2}, %xmm19 {%k1}
+# CHECK-NEXT:  -     6.00   1.00    -      -      -      -      -      -      -     vsqrtpd	%xmm16, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -     6.00   1.00    -     0.50   0.50    -      -      -      -     vsqrtpd	(%rax), %xmm19 {%k1} {z}
+# CHECK-NEXT:  -     6.00   1.00    -     0.50   0.50    -      -      -      -     vsqrtpd	(%rax){1to2}, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -     12.00  1.00    -      -      -      -      -      -      -     vsqrtpd	%ymm16, %ymm19
+# CHECK-NEXT:  -     12.00  1.00    -     0.50   0.50    -      -      -      -     vsqrtpd	(%rax), %ymm19
+# CHECK-NEXT:  -     12.00  1.00    -     0.50   0.50    -      -      -      -     vsqrtpd	(%rax){1to4}, %ymm19
+# CHECK-NEXT:  -     12.00  1.00    -      -      -      -      -      -      -     vsqrtpd	%ymm16, %ymm19 {%k1}
+# CHECK-NEXT:  -     12.00  1.00    -     0.50   0.50    -      -      -      -     vsqrtpd	(%rax), %ymm19 {%k1}
+# CHECK-NEXT:  -     12.00  1.00    -     0.50   0.50    -      -      -      -     vsqrtpd	(%rax){1to4}, %ymm19 {%k1}
+# CHECK-NEXT:  -     12.00  1.00    -      -      -      -      -      -      -     vsqrtpd	%ymm16, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -     12.00  1.00    -     0.50   0.50    -      -      -      -     vsqrtpd	(%rax), %ymm19 {%k1} {z}
+# CHECK-NEXT:  -     12.00  1.00    -     0.50   0.50    -      -      -      -     vsqrtpd	(%rax){1to4}, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -     3.00   1.00    -      -      -      -      -      -      -     vsqrtps	%xmm16, %xmm19
+# CHECK-NEXT:  -     3.00   1.00    -     0.50   0.50    -      -      -      -     vsqrtps	(%rax), %xmm19
+# CHECK-NEXT:  -     3.00   1.00    -     0.50   0.50    -      -      -      -     vsqrtps	(%rax){1to4}, %xmm19
+# CHECK-NEXT:  -     3.00   1.00    -      -      -      -      -      -      -     vsqrtps	%xmm16, %xmm19 {%k1}
+# CHECK-NEXT:  -     3.00   1.00    -     0.50   0.50    -      -      -      -     vsqrtps	(%rax), %xmm19 {%k1}
+# CHECK-NEXT:  -     3.00   1.00    -     0.50   0.50    -      -      -      -     vsqrtps	(%rax){1to4}, %xmm19 {%k1}
+# CHECK-NEXT:  -     3.00   1.00    -      -      -      -      -      -      -     vsqrtps	%xmm16, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -     3.00   1.00    -     0.50   0.50    -      -      -      -     vsqrtps	(%rax), %xmm19 {%k1} {z}
+# CHECK-NEXT:  -     3.00   1.00    -     0.50   0.50    -      -      -      -     vsqrtps	(%rax){1to4}, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -     6.00   1.00    -      -      -      -      -      -      -     vsqrtps	%ymm16, %ymm19
+# CHECK-NEXT:  -     6.00   1.00    -     0.50   0.50    -      -      -      -     vsqrtps	(%rax), %ymm19
+# CHECK-NEXT:  -     6.00   1.00    -     0.50   0.50    -      -      -      -     vsqrtps	(%rax){1to8}, %ymm19
+# CHECK-NEXT:  -     6.00   1.00    -      -      -      -      -      -      -     vsqrtps	%ymm16, %ymm19 {%k1}
+# CHECK-NEXT:  -     6.00   1.00    -     0.50   0.50    -      -      -      -     vsqrtps	(%rax), %ymm19 {%k1}
+# CHECK-NEXT:  -     6.00   1.00    -     0.50   0.50    -      -      -      -     vsqrtps	(%rax){1to8}, %ymm19 {%k1}
+# CHECK-NEXT:  -     6.00   1.00    -      -      -      -      -      -      -     vsqrtps	%ymm16, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -     6.00   1.00    -     0.50   0.50    -      -      -      -     vsqrtps	(%rax), %ymm19 {%k1} {z}
+# CHECK-NEXT:  -     6.00   1.00    -     0.50   0.50    -      -      -      -     vsqrtps	(%rax){1to8}, %ymm19 {%k1} {z}
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -     vsubpd	%xmm16, %xmm17, %xmm19
 # CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vsubpd	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vsubpd	(%rax){1to2}, %xmm17, %xmm19
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -     vsubpd	%xmm16, %xmm17, %xmm19 {%k1}
 # CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vsubpd	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vsubpd	(%rax){1to2}, %xmm17, %xmm19 {%k1}
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -     vsubpd	%xmm16, %xmm17, %xmm19 {%k1} {z}
 # CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vsubpd	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vsubpd	(%rax){1to2}, %xmm17, %xmm19 {%k1} {z}
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -     vsubpd	%ymm16, %ymm17, %ymm19
 # CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vsubpd	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vsubpd	(%rax){1to4}, %ymm17, %ymm19
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -     vsubpd	%ymm16, %ymm17, %ymm19 {%k1}
 # CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vsubpd	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vsubpd	(%rax){1to4}, %ymm17, %ymm19 {%k1}
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -     vsubpd	%ymm16, %ymm17, %ymm19 {%k1} {z}
 # CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vsubpd	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vsubpd	(%rax){1to4}, %ymm17, %ymm19 {%k1} {z}
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -     vsubps	%xmm16, %xmm17, %xmm19
 # CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vsubps	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vsubps	(%rax){1to4}, %xmm17, %xmm19
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -     vsubps	%xmm16, %xmm17, %xmm19 {%k1}
 # CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vsubps	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vsubps	(%rax){1to4}, %xmm17, %xmm19 {%k1}
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -     vsubps	%xmm16, %xmm17, %xmm19 {%k1} {z}
 # CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vsubps	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vsubps	(%rax){1to4}, %xmm17, %xmm19 {%k1} {z}
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -     vsubps	%ymm16, %ymm17, %ymm19
 # CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vsubps	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vsubps	(%rax){1to8}, %ymm17, %ymm19
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -     vsubps	%ymm16, %ymm17, %ymm19 {%k1}
 # CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vsubps	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vsubps	(%rax){1to8}, %ymm17, %ymm19 {%k1}
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -     vsubps	%ymm16, %ymm17, %ymm19 {%k1} {z}
 # CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vsubps	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     vsubps	(%rax){1to8}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vunpckhpd	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vunpckhpd	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vunpckhpd	(%rax){1to2}, %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vunpckhpd	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vunpckhpd	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vunpckhpd	(%rax){1to2}, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vunpckhpd	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vunpckhpd	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vunpckhpd	(%rax){1to2}, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vunpckhpd	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vunpckhpd	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vunpckhpd	(%rax){1to4}, %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vunpckhpd	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vunpckhpd	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vunpckhpd	(%rax){1to4}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vunpckhpd	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vunpckhpd	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vunpckhpd	(%rax){1to4}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vunpckhps	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vunpckhps	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vunpckhps	(%rax){1to4}, %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vunpckhps	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vunpckhps	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vunpckhps	(%rax){1to4}, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vunpckhps	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vunpckhps	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vunpckhps	(%rax){1to4}, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vunpckhps	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vunpckhps	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vunpckhps	(%rax){1to8}, %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vunpckhps	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vunpckhps	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vunpckhps	(%rax){1to8}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vunpckhps	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vunpckhps	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vunpckhps	(%rax){1to8}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vunpcklpd	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vunpcklpd	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vunpcklpd	(%rax){1to2}, %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vunpcklpd	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vunpcklpd	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vunpcklpd	(%rax){1to2}, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vunpcklpd	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vunpcklpd	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vunpcklpd	(%rax){1to2}, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vunpcklpd	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vunpcklpd	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vunpcklpd	(%rax){1to4}, %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vunpcklpd	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vunpcklpd	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vunpcklpd	(%rax){1to4}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vunpcklpd	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vunpcklpd	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vunpcklpd	(%rax){1to4}, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vunpcklps	%xmm16, %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vunpcklps	(%rax), %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vunpcklps	(%rax){1to4}, %xmm17, %xmm19
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vunpcklps	%xmm16, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vunpcklps	(%rax), %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vunpcklps	(%rax){1to4}, %xmm17, %xmm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vunpcklps	%xmm16, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vunpcklps	(%rax), %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vunpcklps	(%rax){1to4}, %xmm17, %xmm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vunpcklps	%ymm16, %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vunpcklps	(%rax), %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vunpcklps	(%rax){1to8}, %ymm17, %ymm19
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vunpcklps	%ymm16, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vunpcklps	(%rax), %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vunpcklps	(%rax){1to8}, %ymm17, %ymm19 {%k1}
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     vunpcklps	%ymm16, %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vunpcklps	(%rax), %ymm17, %ymm19 {%k1} {z}
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     vunpcklps	(%rax){1to8}, %ymm17, %ymm19 {%k1} {z}
diff --git a/test/tools/llvm-mca/X86/SkylakeServer/resources-bmi1.s b/test/tools/llvm-mca/X86/SkylakeServer/resources-bmi1.s
index af42e2bf49915f7c29c485d0f2ac0c8da58a263f..0d1c4dbd8797bf3ce33141686bb278aaa0568114 100644
--- a/test/tools/llvm-mca/X86/SkylakeServer/resources-bmi1.s
+++ b/test/tools/llvm-mca/X86/SkylakeServer/resources-bmi1.s
@@ -31,6 +31,9 @@ blsr        (%rax), %ecx
 blsr        %rax, %rcx
 blsr        (%rax), %rcx
 
+tzcnt       %ax, %cx
+tzcnt       (%rax), %cx
+
 tzcnt       %eax, %ecx
 tzcnt       (%rax), %ecx
 
@@ -66,6 +69,8 @@ tzcnt       (%rax), %rcx
 # CHECK-NEXT:  2      6     0.50    *                   blsrl	(%rax), %ecx
 # CHECK-NEXT:  1      1     0.50                        blsrq	%rax, %rcx
 # CHECK-NEXT:  2      6     0.50    *                   blsrq	(%rax), %rcx
+# CHECK-NEXT:  1      3     1.00                        tzcntw	%ax, %cx
+# CHECK-NEXT:  2      8     1.00    *                   tzcntw	(%rax), %cx
 # CHECK-NEXT:  1      3     1.00                        tzcntl	%eax, %ecx
 # CHECK-NEXT:  2      8     1.00    *                   tzcntl	(%rax), %ecx
 # CHECK-NEXT:  1      3     1.00                        tzcntq	%rax, %rcx
@@ -85,7 +90,7 @@ tzcnt       (%rax), %rcx
 
 # CHECK:      Resource pressure per iteration:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]
-# CHECK-NEXT:  -      -     2.00   14.00  6.00   6.00    -     10.00  2.00    -
+# CHECK-NEXT:  -      -     2.00   16.00  6.50   6.50    -     10.00  2.00    -
 
 # CHECK:      Resource pressure by instruction:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    Instructions:
@@ -109,6 +114,8 @@ tzcnt       (%rax), %rcx
 # CHECK-NEXT:  -      -      -     0.50   0.50   0.50    -     0.50    -      -     blsrl	(%rax), %ecx
 # CHECK-NEXT:  -      -      -     0.50    -      -      -     0.50    -      -     blsrq	%rax, %rcx
 # CHECK-NEXT:  -      -      -     0.50   0.50   0.50    -     0.50    -      -     blsrq	(%rax), %rcx
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     tzcntw	%ax, %cx
+# CHECK-NEXT:  -      -      -     1.00   0.50   0.50    -      -      -      -     tzcntw	(%rax), %cx
 # CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     tzcntl	%eax, %ecx
 # CHECK-NEXT:  -      -      -     1.00   0.50   0.50    -      -      -      -     tzcntl	(%rax), %ecx
 # CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     tzcntq	%rax, %rcx
diff --git a/test/tools/llvm-mca/X86/SkylakeServer/resources-clwb.s b/test/tools/llvm-mca/X86/SkylakeServer/resources-clwb.s
new file mode 100644
index 0000000000000000000000000000000000000000..3c4b5dc3f4482ad8f0e810f689aacbf58bebaa13
--- /dev/null
+++ b/test/tools/llvm-mca/X86/SkylakeServer/resources-clwb.s
@@ -0,0 +1,35 @@
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=x86_64-unknown-unknown -mcpu=skylake-avx512 -instruction-tables < %s | FileCheck %s
+
+clwb (%rax)
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  1      5     0.50    *      *      U     clwb	(%rax)
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0]   - SKXDivider
+# CHECK-NEXT: [1]   - SKXFPDivider
+# CHECK-NEXT: [2]   - SKXPort0
+# CHECK-NEXT: [3]   - SKXPort1
+# CHECK-NEXT: [4]   - SKXPort2
+# CHECK-NEXT: [5]   - SKXPort3
+# CHECK-NEXT: [6]   - SKXPort4
+# CHECK-NEXT: [7]   - SKXPort5
+# CHECK-NEXT: [8]   - SKXPort6
+# CHECK-NEXT: [9]   - SKXPort7
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    Instructions:
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -     clwb	(%rax)
diff --git a/test/tools/llvm-mca/X86/SkylakeServer/resources-fsgsbase.s b/test/tools/llvm-mca/X86/SkylakeServer/resources-fsgsbase.s
new file mode 100644
index 0000000000000000000000000000000000000000..1d7d49a3189de2a5374cb71a8ad08ff134154a58
--- /dev/null
+++ b/test/tools/llvm-mca/X86/SkylakeServer/resources-fsgsbase.s
@@ -0,0 +1,59 @@
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=x86_64-unknown-unknown -mcpu=skylake-avx512 -instruction-tables < %s | FileCheck %s
+
+rdfsbase %eax
+rdfsbase %rax
+
+rdgsbase %eax
+rdgsbase %rax
+
+wrfsbase %edi
+wrfsbase %rdi
+
+wrgsbase %edi
+wrgsbase %rdi
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  1      100   0.25    *      *      U     rdfsbasel	%eax
+# CHECK-NEXT:  1      100   0.25    *      *      U     rdfsbaseq	%rax
+# CHECK-NEXT:  1      100   0.25    *      *      U     rdgsbasel	%eax
+# CHECK-NEXT:  1      100   0.25    *      *      U     rdgsbaseq	%rax
+# CHECK-NEXT:  1      100   0.25    *      *      U     wrfsbasel	%edi
+# CHECK-NEXT:  1      100   0.25    *      *      U     wrfsbaseq	%rdi
+# CHECK-NEXT:  1      100   0.25    *      *      U     wrgsbasel	%edi
+# CHECK-NEXT:  1      100   0.25    *      *      U     wrgsbaseq	%rdi
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0]   - SKXDivider
+# CHECK-NEXT: [1]   - SKXFPDivider
+# CHECK-NEXT: [2]   - SKXPort0
+# CHECK-NEXT: [3]   - SKXPort1
+# CHECK-NEXT: [4]   - SKXPort2
+# CHECK-NEXT: [5]   - SKXPort3
+# CHECK-NEXT: [6]   - SKXPort4
+# CHECK-NEXT: [7]   - SKXPort5
+# CHECK-NEXT: [8]   - SKXPort6
+# CHECK-NEXT: [9]   - SKXPort7
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]
+# CHECK-NEXT:  -      -     2.00   2.00    -      -      -     2.00   2.00    -
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    Instructions:
+# CHECK-NEXT:  -      -     0.25   0.25    -      -      -     0.25   0.25    -     rdfsbasel	%eax
+# CHECK-NEXT:  -      -     0.25   0.25    -      -      -     0.25   0.25    -     rdfsbaseq	%rax
+# CHECK-NEXT:  -      -     0.25   0.25    -      -      -     0.25   0.25    -     rdgsbasel	%eax
+# CHECK-NEXT:  -      -     0.25   0.25    -      -      -     0.25   0.25    -     rdgsbaseq	%rax
+# CHECK-NEXT:  -      -     0.25   0.25    -      -      -     0.25   0.25    -     wrfsbasel	%edi
+# CHECK-NEXT:  -      -     0.25   0.25    -      -      -     0.25   0.25    -     wrfsbaseq	%rdi
+# CHECK-NEXT:  -      -     0.25   0.25    -      -      -     0.25   0.25    -     wrgsbasel	%edi
+# CHECK-NEXT:  -      -     0.25   0.25    -      -      -     0.25   0.25    -     wrgsbaseq	%rdi
diff --git a/test/tools/llvm-mca/X86/SkylakeServer/resources-sse2.s b/test/tools/llvm-mca/X86/SkylakeServer/resources-sse2.s
index 89f468e51b006bc078185e0648ad9b3ff50a5990..90e9b69a233d3b9ae166c73110f9a12546443382 100644
--- a/test/tools/llvm-mca/X86/SkylakeServer/resources-sse2.s
+++ b/test/tools/llvm-mca/X86/SkylakeServer/resources-sse2.s
@@ -94,6 +94,8 @@ maxpd       (%rax), %xmm2
 maxsd       %xmm0, %xmm2
 maxsd       (%rax), %xmm2
 
+mfence
+
 minpd       %xmm0, %xmm2
 minpd       (%rax), %xmm2
 
@@ -229,6 +231,9 @@ pcmpgtw     (%rax), %xmm2
 
 pextrw      $1, %xmm0, %rcx
 
+pinsrw      $1, %rax, %xmm0
+pinsrw      $1, (%rax), %xmm0
+
 pmaddwd     %xmm0, %xmm2
 pmaddwd     (%rax), %xmm2
 
@@ -465,6 +470,7 @@ xorpd       (%rax), %xmm2
 # CHECK-NEXT:  2      10    0.50    *                   maxpd	(%rax), %xmm2
 # CHECK-NEXT:  1      4     0.50                        maxsd	%xmm0, %xmm2
 # CHECK-NEXT:  2      9     0.50    *                   maxsd	(%rax), %xmm2
+# CHECK-NEXT:  3      3     0.50    *      *      U     mfence
 # CHECK-NEXT:  1      4     0.50                        minpd	%xmm0, %xmm2
 # CHECK-NEXT:  2      10    0.50    *                   minpd	(%rax), %xmm2
 # CHECK-NEXT:  1      4     0.50                        minsd	%xmm0, %xmm2
@@ -555,6 +561,8 @@ xorpd       (%rax), %xmm2
 # CHECK-NEXT:  1      1     0.50                        pcmpgtw	%xmm0, %xmm2
 # CHECK-NEXT:  2      7     0.50    *                   pcmpgtw	(%rax), %xmm2
 # CHECK-NEXT:  2      3     1.00                        pextrw	$1, %xmm0, %ecx
+# CHECK-NEXT:  2      2     2.00                        pinsrw	$1, %eax, %xmm0
+# CHECK-NEXT:  2      6     1.00    *                   pinsrw	$1, (%rax), %xmm0
 # CHECK-NEXT:  1      4     0.50                        pmaddwd	%xmm0, %xmm2
 # CHECK-NEXT:  2      10    0.50    *                   pmaddwd	(%rax), %xmm2
 # CHECK-NEXT:  1      1     0.50                        pmaxsw	%xmm0, %xmm2
@@ -681,7 +689,7 @@ xorpd       (%rax), %xmm2
 
 # CHECK:      Resource pressure per iteration:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]
-# CHECK-NEXT:  -     38.00  103.08 82.08  62.67  62.67  14.00  94.08  1.75   4.67
+# CHECK-NEXT:  -     38.00  103.58 82.58  63.50  63.50  14.00  97.58  2.25   5.00
 
 # CHECK:      Resource pressure by instruction:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    Instructions:
@@ -748,6 +756,7 @@ xorpd       (%rax), %xmm2
 # CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     maxpd	(%rax), %xmm2
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -     maxsd	%xmm0, %xmm2
 # CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     maxsd	(%rax), %xmm2
+# CHECK-NEXT:  -      -     0.50   0.50   0.33   0.33    -     0.50   0.50   0.33   mfence
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -     minpd	%xmm0, %xmm2
 # CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     minpd	(%rax), %xmm2
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -     minsd	%xmm0, %xmm2
@@ -838,6 +847,8 @@ xorpd       (%rax), %xmm2
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -     pcmpgtw	%xmm0, %xmm2
 # CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     pcmpgtw	(%rax), %xmm2
 # CHECK-NEXT:  -      -     1.00    -      -      -      -     1.00    -      -     pextrw	$1, %xmm0, %ecx
+# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -     pinsrw	$1, %eax, %xmm0
+# CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     pinsrw	$1, (%rax), %xmm0
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -     pmaddwd	%xmm0, %xmm2
 # CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -      -      -      -     pmaddwd	(%rax), %xmm2
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -      -      -      -     pmaxsw	%xmm0, %xmm2
diff --git a/test/tools/llvm-mca/X86/SkylakeServer/resources-sse3.s b/test/tools/llvm-mca/X86/SkylakeServer/resources-sse3.s
index 0edccb9c78b6c44125ccc2939995e5a7586160c5..61cf0f1fdc393659c5680892501bd4052b209500 100644
--- a/test/tools/llvm-mca/X86/SkylakeServer/resources-sse3.s
+++ b/test/tools/llvm-mca/X86/SkylakeServer/resources-sse3.s
@@ -21,6 +21,8 @@ hsubps    (%rax), %xmm2
 
 lddqu     (%rax), %xmm2
 
+monitor
+
 movddup   %xmm0, %xmm2
 movddup   (%rax), %xmm2
 
@@ -30,6 +32,8 @@ movshdup  (%rax), %xmm2
 movsldup  %xmm0, %xmm2
 movsldup  (%rax), %xmm2
 
+mwait
+
 # CHECK:      Instruction Info:
 # CHECK-NEXT: [1]: #uOps
 # CHECK-NEXT: [2]: Latency
@@ -52,12 +56,14 @@ movsldup  (%rax), %xmm2
 # CHECK-NEXT:  3      6     2.00                        hsubps	%xmm0, %xmm2
 # CHECK-NEXT:  4      12    2.00    *                   hsubps	(%rax), %xmm2
 # CHECK-NEXT:  1      6     0.50    *                   lddqu	(%rax), %xmm2
+# CHECK-NEXT:  1      100   0.25                  U     monitor
 # CHECK-NEXT:  1      1     1.00                        movddup	%xmm0, %xmm2
 # CHECK-NEXT:  1      5     0.50    *                   movddup	(%rax), %xmm2
 # CHECK-NEXT:  1      1     1.00                        movshdup	%xmm0, %xmm2
 # CHECK-NEXT:  1      6     0.50    *                   movshdup	(%rax), %xmm2
 # CHECK-NEXT:  1      1     1.00                        movsldup	%xmm0, %xmm2
 # CHECK-NEXT:  1      6     0.50    *                   movsldup	(%rax), %xmm2
+# CHECK-NEXT:  10     20    2.50    *      *      U     mwait
 
 # CHECK:      Resources:
 # CHECK-NEXT: [0]   - SKXDivider
@@ -73,7 +79,7 @@ movsldup  (%rax), %xmm2
 
 # CHECK:      Resource pressure per iteration:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]
-# CHECK-NEXT:  -      -     4.67   4.67   5.00   5.00    -     21.67   -      -
+# CHECK-NEXT:  -      -     6.67   6.67   5.00   5.00    -     24.67  4.00    -
 
 # CHECK:      Resource pressure by instruction:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    Instructions:
@@ -90,9 +96,11 @@ movsldup  (%rax), %xmm2
 # CHECK-NEXT:  -      -     0.33   0.33    -      -      -     2.33    -      -     hsubps	%xmm0, %xmm2
 # CHECK-NEXT:  -      -     0.33   0.33   0.50   0.50    -     2.33    -      -     hsubps	(%rax), %xmm2
 # CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -     lddqu	(%rax), %xmm2
+# CHECK-NEXT:  -      -     0.25   0.25    -      -      -     0.25   0.25    -     monitor
 # CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     movddup	%xmm0, %xmm2
 # CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -     movddup	(%rax), %xmm2
 # CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     movshdup	%xmm0, %xmm2
 # CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -     movshdup	(%rax), %xmm2
 # CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     movsldup	%xmm0, %xmm2
 # CHECK-NEXT:  -      -      -      -     0.50   0.50    -      -      -      -     movsldup	(%rax), %xmm2
+# CHECK-NEXT:  -      -     1.75   1.75    -      -      -     2.75   3.75    -     mwait
diff --git a/test/tools/llvm-mca/X86/SkylakeServer/resources-x86_64.s b/test/tools/llvm-mca/X86/SkylakeServer/resources-x86_64.s
index e35da95896e803662c0a9cb7afbaea7db595ddef..e11f6a177c8359aa24ab8f06ac1264439fe35f3b 100644
--- a/test/tools/llvm-mca/X86/SkylakeServer/resources-x86_64.s
+++ b/test/tools/llvm-mca/X86/SkylakeServer/resources-x86_64.s
@@ -254,6 +254,8 @@ divl (%rax)
 divq %rcx
 divq (%rax)
 
+enter $7, $4095
+
 idivb %dil
 idivb (%rax)
 idivw %si
@@ -315,8 +317,13 @@ insl
 
 int $7
 
+invlpg  (%rax)
+invlpga %rax, %ecx
+
 lahf
 
+leave
+
 lodsb
 lodsw
 lodsl
@@ -486,6 +493,11 @@ rcrq %cl, %rdi
 rclq %cl, (%rax)
 rcrq %cl, (%rax)
 
+rdmsr
+rdpmc
+rdtsc
+rdtscp
+
 rolb %dil
 rorb %dil
 rolb (%rax)
@@ -789,6 +801,8 @@ testq %rsi, (%rax)
 
 ud2
 
+wrmsr
+
 xaddb %bl, %cl
 xaddb %bl, (%rcx)
 
@@ -1081,6 +1095,7 @@ xorq (%rax), %rdi
 # CHECK-NEXT:  2      29    10.00   *             U     divl	(%rax)
 # CHECK-NEXT:  32     76    8.00                  U     divq	%rcx
 # CHECK-NEXT:  2      29    10.00   *             U     divq	(%rax)
+# CHECK-NEXT:  1      100   0.25                  U     enter	$7, $4095
 # CHECK-NEXT:  1      25    10.00                 U     idivb	%dil
 # CHECK-NEXT:  8      28    4.00    *             U     idivb	(%rax)
 # CHECK-NEXT:  66     102   16.50                 U     idivw	%si
@@ -1133,7 +1148,10 @@ xorq (%rax), %rdi
 # CHECK-NEXT:  8      20    1.25                  U     insw	%dx, %es:(%rdi)
 # CHECK-NEXT:  8      20    1.25                  U     insl	%dx, %es:(%rdi)
 # CHECK-NEXT:  1      100   0.25    *      *      U     int	$7
+# CHECK-NEXT:  1      100   0.25                  U     invlpg	(%rax)
+# CHECK-NEXT:  1      100   0.25                  U     invlpga	%rax, %ecx
 # CHECK-NEXT:  1      1     0.50                        lahf
+# CHECK-NEXT:  3      7     0.50    *                   leave
 # CHECK-NEXT:  1      100   0.25                  U     lodsb	(%rsi), %al
 # CHECK-NEXT:  1      100   0.25                  U     lodsw	(%rsi), %ax
 # CHECK-NEXT:  1      100   0.25                  U     lodsl	(%rsi), %eax
@@ -1283,6 +1301,10 @@ xorq (%rax), %rdi
 # CHECK-NEXT:  7      11    2.00                        rcrq	%cl, %rdi
 # CHECK-NEXT:  10     15    2.50           *            rclq	%cl, (%rax)
 # CHECK-NEXT:  11     18    2.25           *            rcrq	%cl, (%rax)
+# CHECK-NEXT:  1      100   0.25                  U     rdmsr
+# CHECK-NEXT:  1      100   0.25                  U     rdpmc
+# CHECK-NEXT:  8      18    2.00                  U     rdtsc
+# CHECK-NEXT:  22     42    5.50                  U     rdtscp
 # CHECK-NEXT:  2      2     1.00                        rolb	%dil
 # CHECK-NEXT:  2      2     1.00                        rorb	%dil
 # CHECK-NEXT:  5      7     1.00    *      *            rolb	(%rax)
@@ -1557,6 +1579,7 @@ xorq (%rax), %rdi
 # CHECK-NEXT:  1      1     0.25                        testq	%rsi, %rdi
 # CHECK-NEXT:  2      6     0.50    *                   testq	%rsi, (%rax)
 # CHECK-NEXT:  1      100   0.25    *             U     ud2
+# CHECK-NEXT:  1      100   0.25                  U     wrmsr
 # CHECK-NEXT:  3      2     0.75                        xaddb	%bl, %cl
 # CHECK-NEXT:  5      7     1.00    *      *            xaddb	%bl, (%rcx)
 # CHECK-NEXT:  3      2     0.75                        xaddw	%bx, %cx
@@ -1622,7 +1645,7 @@ xorq (%rax), %rdi
 
 # CHECK:      Resource pressure per iteration:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]
-# CHECK-NEXT: 60.00   -     564.75 314.75 262.17 262.17 206.00 279.25 620.25 82.67
+# CHECK-NEXT: 60.00   -     573.50 323.00 262.67 262.67 206.00 290.50 630.00 82.67
 
 # CHECK:      Resource pressure by instruction:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    Instructions:
@@ -1846,6 +1869,7 @@ xorq (%rax), %rdi
 # CHECK-NEXT: 10.00   -     1.00    -     0.50   0.50    -      -      -      -     divl	(%rax)
 # CHECK-NEXT:  -      -     10.25  4.75    -      -      -     11.25  5.75    -     divq	%rcx
 # CHECK-NEXT: 10.00   -     1.00    -     0.50   0.50    -      -      -      -     divq	(%rax)
+# CHECK-NEXT:  -      -     0.25   0.25    -      -      -     0.25   0.25    -     enter	$7, $4095
 # CHECK-NEXT: 10.00   -     1.00    -      -      -      -      -      -      -     idivb	%dil
 # CHECK-NEXT:  -      -     2.25   0.25   0.50   0.50    -     4.25   0.25    -     idivb	(%rax)
 # CHECK-NEXT:  -      -     19.50  10.50   -      -      -     12.50  23.50   -     idivw	%si
@@ -1898,7 +1922,10 @@ xorq (%rax), %rdi
 # CHECK-NEXT:  -      -     1.00   0.50   0.83   0.83   1.00   1.50   2.00   0.33   insw	%dx, %es:(%rdi)
 # CHECK-NEXT:  -      -     1.00   0.50   0.83   0.83   1.00   1.50   2.00   0.33   insl	%dx, %es:(%rdi)
 # CHECK-NEXT:  -      -     0.25   0.25    -      -      -     0.25   0.25    -     int	$7
+# CHECK-NEXT:  -      -     0.25   0.25    -      -      -     0.25   0.25    -     invlpg	(%rax)
+# CHECK-NEXT:  -      -     0.25   0.25    -      -      -     0.25   0.25    -     invlpga	%rax, %ecx
 # CHECK-NEXT:  -      -     0.50    -      -      -      -      -     0.50    -     lahf
+# CHECK-NEXT:  -      -     0.50   0.50   0.50   0.50    -     0.50   0.50    -     leave
 # CHECK-NEXT:  -      -     0.25   0.25    -      -      -     0.25   0.25    -     lodsb	(%rsi), %al
 # CHECK-NEXT:  -      -     0.25   0.25    -      -      -     0.25   0.25    -     lodsw	(%rsi), %ax
 # CHECK-NEXT:  -      -     0.25   0.25    -      -      -     0.25   0.25    -     lodsl	(%rsi), %eax
@@ -2048,6 +2075,10 @@ xorq (%rax), %rdi
 # CHECK-NEXT:  -      -     2.00   2.50    -      -      -     0.50   2.00    -     rcrq	%cl, %rdi
 # CHECK-NEXT:  -      -     2.75   1.75   0.83   0.83    -     0.75   2.75   0.33   rclq	%cl, (%rax)
 # CHECK-NEXT:  -      -     2.50   3.00   0.83   0.83    -     1.00   2.50   0.33   rcrq	%cl, (%rax)
+# CHECK-NEXT:  -      -     0.25   0.25    -      -      -     0.25   0.25    -     rdmsr
+# CHECK-NEXT:  -      -     0.25   0.25    -      -      -     0.25   0.25    -     rdpmc
+# CHECK-NEXT:  -      -     1.75   1.25    -      -      -     2.25   2.75    -     rdtsc
+# CHECK-NEXT:  -      -     5.00   5.00    -      -      -     7.00   5.00    -     rdtscp
 # CHECK-NEXT:  -      -     1.00    -      -      -      -      -     1.00    -     rolb	%dil
 # CHECK-NEXT:  -      -     1.00    -      -      -      -      -     1.00    -     rorb	%dil
 # CHECK-NEXT:  -      -     1.00    -     0.83   0.83   1.00    -     1.00   0.33   rolb	(%rax)
@@ -2322,6 +2353,7 @@ xorq (%rax), %rdi
 # CHECK-NEXT:  -      -     0.25   0.25    -      -      -     0.25   0.25    -     testq	%rsi, %rdi
 # CHECK-NEXT:  -      -     0.25   0.25   0.50   0.50    -     0.25   0.25    -     testq	%rsi, (%rax)
 # CHECK-NEXT:  -      -     0.25   0.25    -      -      -     0.25   0.25    -     ud2
+# CHECK-NEXT:  -      -     0.25   0.25    -      -      -     0.25   0.25    -     wrmsr
 # CHECK-NEXT:  -      -     0.75   0.75    -      -      -     0.75   0.75    -     xaddb	%bl, %cl
 # CHECK-NEXT:  -      -     0.50   0.50   0.83   0.83   1.00   0.50   0.50   0.33   xaddb	%bl, (%rcx)
 # CHECK-NEXT:  -      -     0.75   0.75    -      -      -     0.75   0.75    -     xaddw	%bx, %cx
diff --git a/test/tools/llvm-mca/X86/SkylakeServer/resources-x87.s b/test/tools/llvm-mca/X86/SkylakeServer/resources-x87.s
index aecb4a7ab9211be98af8fdde68cc9cce63be3c4a..75cca5297704b2c6de4e6552734a352287009160 100644
--- a/test/tools/llvm-mca/X86/SkylakeServer/resources-x87.s
+++ b/test/tools/llvm-mca/X86/SkylakeServer/resources-x87.s
@@ -5,7 +5,7 @@ f2xm1
 
 fabs
 
-fadd %st(0), %st(1)
+fadd %st, %st(1)
 fadd %st(2)
 fadds (%ecx)
 faddl (%ecx)
@@ -21,14 +21,14 @@ fchs
 
 fnclex
 
-fcmovb %st(1), %st(0)
-fcmovbe %st(1), %st(0)
-fcmove %st(1), %st(0)
-fcmovnb %st(1), %st(0)
-fcmovnbe %st(1), %st(0)
-fcmovne %st(1), %st(0)
-fcmovnu %st(1), %st(0)
-fcmovu %st(1), %st(0)
+fcmovb %st(1), %st
+fcmovbe %st(1), %st
+fcmove %st(1), %st
+fcmovnb %st(1), %st
+fcmovnbe %st(1), %st
+fcmovne %st(1), %st
+fcmovnu %st(1), %st
+fcmovu %st(1), %st
 
 fcom %st(1)
 fcom %st(3)
@@ -47,7 +47,7 @@ fcos
 
 fdecstp
 
-fdiv %st(0), %st(1)
+fdiv %st, %st(1)
 fdiv %st(2)
 fdivs (%ecx)
 fdivl (%eax)
@@ -56,7 +56,7 @@ fdivp %st(2)
 fidivs (%ecx)
 fidivl (%eax)
 
-fdivr %st(0), %st(1)
+fdivr %st, %st(1)
 fdivr %st(2)
 fdivrs (%ecx)
 fdivrl (%eax)
@@ -106,7 +106,7 @@ fldln2
 fldpi
 fldz
 
-fmul %st(0), %st(1)
+fmul %st, %st(1)
 fmul %st(2)
 fmuls (%ecx)
 fmull (%eax)
@@ -153,7 +153,7 @@ fnstsw (%eax)
 frstor (%eax)
 fsave (%eax)
 
-fsub %st(0), %st(1)
+fsub %st, %st(1)
 fsub %st(2)
 fsubs (%ecx)
 fsubl (%eax)
@@ -162,7 +162,7 @@ fsubp %st(2)
 fisubs (%ecx)
 fisubl (%eax)
 
-fsubr %st(0), %st(1)
+fsubr %st, %st(1)
 fsubr %st(2)
 fsubrs (%ecx)
 fsubrl (%eax)
@@ -208,26 +208,26 @@ fyl2xp1
 # CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
 # CHECK-NEXT:  1      100   0.25                  U     f2xm1
 # CHECK-NEXT:  1      1     1.00                  U     fabs
-# CHECK-NEXT:  1      3     1.00                  U     fadd	%st(0), %st(1)
-# CHECK-NEXT:  1      3     1.00                  U     fadd	%st(2)
+# CHECK-NEXT:  1      3     1.00                  U     fadd	%st, %st(1)
+# CHECK-NEXT:  1      3     1.00                  U     fadd	%st(2), %st
 # CHECK-NEXT:  2      10    1.00    *             U     fadds	(%ecx)
 # CHECK-NEXT:  2      10    1.00    *             U     faddl	(%ecx)
-# CHECK-NEXT:  1      3     1.00                  U     faddp	%st(1)
-# CHECK-NEXT:  1      3     1.00                  U     faddp	%st(2)
+# CHECK-NEXT:  1      3     1.00                  U     faddp	%st, %st(1)
+# CHECK-NEXT:  1      3     1.00                  U     faddp	%st, %st(2)
 # CHECK-NEXT:  3      13    2.00    *             U     fiadds	(%ecx)
 # CHECK-NEXT:  3      13    2.00    *             U     fiaddl	(%ecx)
 # CHECK-NEXT:  1      100   0.25                  U     fbld	(%ecx)
 # CHECK-NEXT:  2      1     1.00                  U     fbstp	(%eax)
 # CHECK-NEXT:  1      1     1.00                  U     fchs
 # CHECK-NEXT:  4      4     1.00                  U     fnclex
-# CHECK-NEXT:  1      3     1.00                  U     fcmovb	%st(1), %st(0)
-# CHECK-NEXT:  1      3     1.00                  U     fcmovbe	%st(1), %st(0)
-# CHECK-NEXT:  1      3     1.00                  U     fcmove	%st(1), %st(0)
-# CHECK-NEXT:  1      3     1.00                  U     fcmovnb	%st(1), %st(0)
-# CHECK-NEXT:  1      3     1.00                  U     fcmovnbe	%st(1), %st(0)
-# CHECK-NEXT:  1      3     1.00                  U     fcmovne	%st(1), %st(0)
-# CHECK-NEXT:  1      3     1.00                  U     fcmovnu	%st(1), %st(0)
-# CHECK-NEXT:  1      3     1.00                  U     fcmovu	%st(1), %st(0)
+# CHECK-NEXT:  1      3     1.00                  U     fcmovb	%st(1), %st
+# CHECK-NEXT:  1      3     1.00                  U     fcmovbe	%st(1), %st
+# CHECK-NEXT:  1      3     1.00                  U     fcmove	%st(1), %st
+# CHECK-NEXT:  1      3     1.00                  U     fcmovnb	%st(1), %st
+# CHECK-NEXT:  1      3     1.00                  U     fcmovnbe	%st(1), %st
+# CHECK-NEXT:  1      3     1.00                  U     fcmovne	%st(1), %st
+# CHECK-NEXT:  1      3     1.00                  U     fcmovnu	%st(1), %st
+# CHECK-NEXT:  1      3     1.00                  U     fcmovu	%st(1), %st
 # CHECK-NEXT:  1      1     1.00                  U     fcom	%st(1)
 # CHECK-NEXT:  1      1     1.00                  U     fcom	%st(3)
 # CHECK-NEXT:  2      8     1.00                  U     fcoms	(%ecx)
@@ -237,24 +237,24 @@ fyl2xp1
 # CHECK-NEXT:  2      8     1.00                  U     fcomps	(%ecx)
 # CHECK-NEXT:  2      8     1.00                  U     fcompl	(%eax)
 # CHECK-NEXT:  1      100   0.25                  U     fcompp
-# CHECK-NEXT:  1      2     1.00                  U     fcomi	%st(3)
-# CHECK-NEXT:  1      2     1.00                  U     fcompi	%st(3)
+# CHECK-NEXT:  1      2     1.00                  U     fcomi	%st(3), %st
+# CHECK-NEXT:  1      2     1.00                  U     fcompi	%st(3), %st
 # CHECK-NEXT:  1      100   0.25                  U     fcos
 # CHECK-NEXT:  2      2     1.00                  U     fdecstp
-# CHECK-NEXT:  1      15    1.00                  U     fdiv	%st(0), %st(1)
-# CHECK-NEXT:  1      20    1.00                  U     fdiv	%st(2)
+# CHECK-NEXT:  1      15    1.00                  U     fdiv	%st, %st(1)
+# CHECK-NEXT:  1      20    1.00                  U     fdiv	%st(2), %st
 # CHECK-NEXT:  2      22    1.00    *             U     fdivs	(%ecx)
 # CHECK-NEXT:  2      22    1.00    *             U     fdivl	(%eax)
-# CHECK-NEXT:  1      15    1.00                  U     fdivp	%st(1)
-# CHECK-NEXT:  1      15    1.00                  U     fdivp	%st(2)
+# CHECK-NEXT:  1      15    1.00                  U     fdivp	%st, %st(1)
+# CHECK-NEXT:  1      15    1.00                  U     fdivp	%st, %st(2)
 # CHECK-NEXT:  3      25    1.00    *             U     fidivs	(%ecx)
 # CHECK-NEXT:  3      25    1.00    *             U     fidivl	(%eax)
-# CHECK-NEXT:  1      20    1.00                  U     fdivr	%st(0), %st(1)
-# CHECK-NEXT:  1      15    1.00                  U     fdivr	%st(2)
+# CHECK-NEXT:  1      20    1.00                  U     fdivr	%st, %st(1)
+# CHECK-NEXT:  1      15    1.00                  U     fdivr	%st(2), %st
 # CHECK-NEXT:  2      27    1.00    *             U     fdivrs	(%ecx)
 # CHECK-NEXT:  2      27    1.00    *             U     fdivrl	(%eax)
-# CHECK-NEXT:  1      20    1.00                  U     fdivrp	%st(1)
-# CHECK-NEXT:  1      20    1.00                  U     fdivrp	%st(2)
+# CHECK-NEXT:  1      20    1.00                  U     fdivrp	%st, %st(1)
+# CHECK-NEXT:  1      20    1.00                  U     fdivrp	%st, %st(2)
 # CHECK-NEXT:  3      30    1.00    *             U     fidivrs	(%ecx)
 # CHECK-NEXT:  3      30    1.00    *             U     fidivrl	(%eax)
 # CHECK-NEXT:  1      100   0.25                  U     ffree	%st(0)
@@ -288,12 +288,12 @@ fyl2xp1
 # CHECK-NEXT:  2      1     1.00                  U     fldln2
 # CHECK-NEXT:  2      1     1.00                  U     fldpi
 # CHECK-NEXT:  1      1     0.50                  U     fldz
-# CHECK-NEXT:  1      4     1.00                  U     fmul	%st(0), %st(1)
-# CHECK-NEXT:  1      4     1.00                  U     fmul	%st(2)
+# CHECK-NEXT:  1      4     1.00                  U     fmul	%st, %st(1)
+# CHECK-NEXT:  1      4     1.00                  U     fmul	%st(2), %st
 # CHECK-NEXT:  2      11    1.00    *             U     fmuls	(%ecx)
 # CHECK-NEXT:  2      11    1.00    *             U     fmull	(%eax)
-# CHECK-NEXT:  1      4     1.00                  U     fmulp	%st(1)
-# CHECK-NEXT:  1      4     1.00                  U     fmulp	%st(2)
+# CHECK-NEXT:  1      4     1.00                  U     fmulp	%st, %st(1)
+# CHECK-NEXT:  1      4     1.00                  U     fmulp	%st, %st(2)
 # CHECK-NEXT:  3      14    1.00    *             U     fimuls	(%ecx)
 # CHECK-NEXT:  3      14    1.00    *             U     fimull	(%eax)
 # CHECK-NEXT:  1      1     0.50                  U     fnop
@@ -321,20 +321,20 @@ fyl2xp1
 # CHECK-NEXT:  1      100   0.25                  U     frstor	(%eax)
 # CHECK-NEXT:  2      2     0.50                  U     wait
 # CHECK-NEXT:  1      100   0.25                  U     fnsave	(%eax)
-# CHECK-NEXT:  1      3     1.00                  U     fsub	%st(0), %st(1)
-# CHECK-NEXT:  1      3     1.00                  U     fsub	%st(2)
+# CHECK-NEXT:  1      3     1.00                  U     fsub	%st, %st(1)
+# CHECK-NEXT:  1      3     1.00                  U     fsub	%st(2), %st
 # CHECK-NEXT:  2      10    1.00    *             U     fsubs	(%ecx)
 # CHECK-NEXT:  2      10    1.00    *             U     fsubl	(%eax)
-# CHECK-NEXT:  1      3     1.00                  U     fsubp	%st(1)
-# CHECK-NEXT:  1      3     1.00                  U     fsubp	%st(2)
+# CHECK-NEXT:  1      3     1.00                  U     fsubp	%st, %st(1)
+# CHECK-NEXT:  1      3     1.00                  U     fsubp	%st, %st(2)
 # CHECK-NEXT:  3      13    2.00    *             U     fisubs	(%ecx)
 # CHECK-NEXT:  3      13    2.00    *             U     fisubl	(%eax)
-# CHECK-NEXT:  1      3     1.00                  U     fsubr	%st(0), %st(1)
-# CHECK-NEXT:  1      3     1.00                  U     fsubr	%st(2)
+# CHECK-NEXT:  1      3     1.00                  U     fsubr	%st, %st(1)
+# CHECK-NEXT:  1      3     1.00                  U     fsubr	%st(2), %st
 # CHECK-NEXT:  2      10    1.00    *             U     fsubrs	(%ecx)
 # CHECK-NEXT:  2      10    1.00    *             U     fsubrl	(%eax)
-# CHECK-NEXT:  1      3     1.00                  U     fsubrp	%st(1)
-# CHECK-NEXT:  1      3     1.00                  U     fsubrp	%st(2)
+# CHECK-NEXT:  1      3     1.00                  U     fsubrp	%st, %st(1)
+# CHECK-NEXT:  1      3     1.00                  U     fsubrp	%st, %st(2)
 # CHECK-NEXT:  3      13    2.00    *             U     fisubrs	(%ecx)
 # CHECK-NEXT:  3      13    2.00    *             U     fisubrl	(%eax)
 # CHECK-NEXT:  1      2     1.00                  U     ftst
@@ -343,8 +343,8 @@ fyl2xp1
 # CHECK-NEXT:  1      1     1.00                  U     fucomp	%st(1)
 # CHECK-NEXT:  1      1     1.00                  U     fucomp	%st(3)
 # CHECK-NEXT:  1      2     1.00                  U     fucompp
-# CHECK-NEXT:  1      2     1.00                  U     fucomi	%st(3)
-# CHECK-NEXT:  1      2     1.00                  U     fucompi	%st(3)
+# CHECK-NEXT:  1      2     1.00                  U     fucomi	%st(3), %st
+# CHECK-NEXT:  1      2     1.00                  U     fucompi	%st(3), %st
 # CHECK-NEXT:  2      2     0.50                  U     wait
 # CHECK-NEXT:  1      100   0.25                  U     fxam
 # CHECK-NEXT:  15     17    4.00                  U     fxch	%st(1)
@@ -375,26 +375,26 @@ fyl2xp1
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    Instructions:
 # CHECK-NEXT:  -      -     0.25   0.25    -      -      -     0.25   0.25    -     f2xm1
 # CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     fabs
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     fadd	%st(0), %st(1)
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     fadd	%st(2)
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     fadd	%st, %st(1)
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     fadd	%st(2), %st
 # CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     fadds	(%ecx)
 # CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     faddl	(%ecx)
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     faddp	%st(1)
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     faddp	%st(2)
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     faddp	%st, %st(1)
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     faddp	%st, %st(2)
 # CHECK-NEXT:  -      -      -      -     0.50   0.50    -     2.00    -      -     fiadds	(%ecx)
 # CHECK-NEXT:  -      -      -      -     0.50   0.50    -     2.00    -      -     fiaddl	(%ecx)
 # CHECK-NEXT:  -      -     0.25   0.25    -      -      -     0.25   0.25    -     fbld	(%ecx)
 # CHECK-NEXT:  -      -      -      -     0.33   0.33   1.00    -      -     0.33   fbstp	(%eax)
 # CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     fchs
 # CHECK-NEXT:  -      -     1.00   1.00    -      -      -     1.00   1.00    -     fnclex
-# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     fcmovb	%st(1), %st(0)
-# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     fcmovbe	%st(1), %st(0)
-# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     fcmove	%st(1), %st(0)
-# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     fcmovnb	%st(1), %st(0)
-# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     fcmovnbe	%st(1), %st(0)
-# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     fcmovne	%st(1), %st(0)
-# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     fcmovnu	%st(1), %st(0)
-# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     fcmovu	%st(1), %st(0)
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     fcmovb	%st(1), %st
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     fcmovbe	%st(1), %st
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     fcmove	%st(1), %st
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     fcmovnb	%st(1), %st
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     fcmovnbe	%st(1), %st
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     fcmovne	%st(1), %st
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     fcmovnu	%st(1), %st
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -     fcmovu	%st(1), %st
 # CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     fcom	%st(1)
 # CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     fcom	%st(3)
 # CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     fcoms	(%ecx)
@@ -404,24 +404,24 @@ fyl2xp1
 # CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     fcomps	(%ecx)
 # CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     fcompl	(%eax)
 # CHECK-NEXT:  -      -     0.25   0.25    -      -      -     0.25   0.25    -     fcompp
-# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     fcomi	%st(3)
-# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     fcompi	%st(3)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     fcomi	%st(3), %st
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     fcompi	%st(3), %st
 # CHECK-NEXT:  -      -     0.25   0.25    -      -      -     0.25   0.25    -     fcos
 # CHECK-NEXT:  -      -     1.00    -      -      -      -     1.00    -      -     fdecstp
-# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     fdiv	%st(0), %st(1)
-# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     fdiv	%st(2)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     fdiv	%st, %st(1)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     fdiv	%st(2), %st
 # CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -      -      -      -     fdivs	(%ecx)
 # CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -      -      -      -     fdivl	(%eax)
-# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     fdivp	%st(1)
-# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     fdivp	%st(2)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     fdivp	%st, %st(1)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     fdivp	%st, %st(2)
 # CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -     1.00    -      -     fidivs	(%ecx)
 # CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -     1.00    -      -     fidivl	(%eax)
-# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     fdivr	%st(0), %st(1)
-# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     fdivr	%st(2)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     fdivr	%st, %st(1)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     fdivr	%st(2), %st
 # CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -      -      -      -     fdivrs	(%ecx)
 # CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -      -      -      -     fdivrl	(%eax)
-# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     fdivrp	%st(1)
-# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     fdivrp	%st(2)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     fdivrp	%st, %st(1)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     fdivrp	%st, %st(2)
 # CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -     1.00    -      -     fidivrs	(%ecx)
 # CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -     1.00    -      -     fidivrl	(%eax)
 # CHECK-NEXT:  -      -     0.25   0.25    -      -      -     0.25   0.25    -     ffree	%st(0)
@@ -455,12 +455,12 @@ fyl2xp1
 # CHECK-NEXT:  -      -     1.00    -      -      -      -     1.00    -      -     fldln2
 # CHECK-NEXT:  -      -     1.00    -      -      -      -     1.00    -      -     fldpi
 # CHECK-NEXT:  -      -     0.50    -      -      -      -     0.50    -      -     fldz
-# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     fmul	%st(0), %st(1)
-# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     fmul	%st(2)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     fmul	%st, %st(1)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     fmul	%st(2), %st
 # CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -      -      -      -     fmuls	(%ecx)
 # CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -      -      -      -     fmull	(%eax)
-# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     fmulp	%st(1)
-# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     fmulp	%st(2)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     fmulp	%st, %st(1)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     fmulp	%st, %st(2)
 # CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -     1.00    -      -     fimuls	(%ecx)
 # CHECK-NEXT:  -      -     1.00    -     0.50   0.50    -     1.00    -      -     fimull	(%eax)
 # CHECK-NEXT:  -      -     0.50    -      -      -      -     0.50    -      -     fnop
@@ -488,20 +488,20 @@ fyl2xp1
 # CHECK-NEXT:  -      -     0.25   0.25    -      -      -     0.25   0.25    -     frstor	(%eax)
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -     0.50   0.50    -     wait
 # CHECK-NEXT:  -      -     0.25   0.25    -      -      -     0.25   0.25    -     fnsave	(%eax)
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     fsub	%st(0), %st(1)
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     fsub	%st(2)
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     fsub	%st, %st(1)
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     fsub	%st(2), %st
 # CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     fsubs	(%ecx)
 # CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     fsubl	(%eax)
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     fsubp	%st(1)
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     fsubp	%st(2)
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     fsubp	%st, %st(1)
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     fsubp	%st, %st(2)
 # CHECK-NEXT:  -      -      -      -     0.50   0.50    -     2.00    -      -     fisubs	(%ecx)
 # CHECK-NEXT:  -      -      -      -     0.50   0.50    -     2.00    -      -     fisubl	(%eax)
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     fsubr	%st(0), %st(1)
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     fsubr	%st(2)
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     fsubr	%st, %st(1)
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     fsubr	%st(2), %st
 # CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     fsubrs	(%ecx)
 # CHECK-NEXT:  -      -      -      -     0.50   0.50    -     1.00    -      -     fsubrl	(%eax)
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     fsubrp	%st(1)
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     fsubrp	%st(2)
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     fsubrp	%st, %st(1)
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     fsubrp	%st, %st(2)
 # CHECK-NEXT:  -      -      -      -     0.50   0.50    -     2.00    -      -     fisubrs	(%ecx)
 # CHECK-NEXT:  -      -      -      -     0.50   0.50    -     2.00    -      -     fisubrl	(%eax)
 # CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     ftst
@@ -510,8 +510,8 @@ fyl2xp1
 # CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     fucomp	%st(1)
 # CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -     fucomp	%st(3)
 # CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     fucompp
-# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     fucomi	%st(3)
-# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     fucompi	%st(3)
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     fucomi	%st(3), %st
+# CHECK-NEXT:  -      -     1.00    -      -      -      -      -      -      -     fucompi	%st(3), %st
 # CHECK-NEXT:  -      -     0.50   0.50    -      -      -     0.50   0.50    -     wait
 # CHECK-NEXT:  -      -     0.25   0.25    -      -      -     0.25   0.25    -     fxam
 # CHECK-NEXT:  -      -     4.00   2.00    -      -      -     4.00   5.00    -     fxch	%st(1)
diff --git a/test/tools/llvm-mca/X86/Znver1/resources-avx2.s b/test/tools/llvm-mca/X86/Znver1/resources-avx2.s
index 230244b2a52f867a9155b3bf338c6c032889563c..79eb2a44fcf6b122eb0747a6bd3c637024a1dec1 100644
--- a/test/tools/llvm-mca/X86/Znver1/resources-avx2.s
+++ b/test/tools/llvm-mca/X86/Znver1/resources-avx2.s
@@ -149,6 +149,9 @@ vpcmpgtq        (%rax), %ymm1, %ymm2
 vpcmpgtw        %ymm0, %ymm1, %ymm2
 vpcmpgtw        (%rax), %ymm1, %ymm2
 
+vperm2i128      $1, %ymm0, %ymm1, %ymm2
+vperm2i128      $1, (%rax), %ymm1, %ymm2
+
 vpermd          %ymm0, %ymm1, %ymm2
 vpermd          (%rax), %ymm1, %ymm2
 
@@ -555,6 +558,8 @@ vpxor           (%rax), %ymm1, %ymm2
 # CHECK-NEXT:  2      8     1.00    *                   vpcmpgtq	(%rax), %ymm1, %ymm2
 # CHECK-NEXT:  1      1     0.25                        vpcmpgtw	%ymm0, %ymm1, %ymm2
 # CHECK-NEXT:  1      8     0.50    *                   vpcmpgtw	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  1      2     0.25                        vperm2i128	$1, %ymm0, %ymm1, %ymm2
+# CHECK-NEXT:  1      9     0.50    *                   vperm2i128	$1, (%rax), %ymm1, %ymm2
 # CHECK-NEXT:  1      2     0.25                        vpermd	%ymm0, %ymm1, %ymm2
 # CHECK-NEXT:  1      9     0.50    *                   vpermd	(%rax), %ymm1, %ymm2
 # CHECK-NEXT:  1      100   0.25                        vpermpd	$1, %ymm0, %ymm2
@@ -773,7 +778,7 @@ vpxor           (%rax), %ymm1, %ymm2
 
 # CHECK:      Resource pressure per iteration:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]
-# CHECK-NEXT: 63.50  63.50   -      -      -      -      -     72.67  75.17  84.50  42.67   -
+# CHECK-NEXT: 64.00  64.00   -      -      -      -      -     73.17  75.67  85.00  43.17   -
 
 # CHECK:      Resource pressure by instruction:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   Instructions:
@@ -875,6 +880,8 @@ vpxor           (%rax), %ymm1, %ymm2
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -     1.00    -      -     1.00    -     vpcmpgtq	(%rax), %ymm1, %ymm2
 # CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -     vpcmpgtw	%ymm0, %ymm1, %ymm2
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -     0.25   0.25   0.25   0.25    -     vpcmpgtw	(%rax), %ymm1, %ymm2
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -     vperm2i128	$1, %ymm0, %ymm1, %ymm2
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -     0.25   0.25   0.25   0.25    -     vperm2i128	$1, (%rax), %ymm1, %ymm2
 # CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -     vpermd	%ymm0, %ymm1, %ymm2
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -     0.25   0.25   0.25   0.25    -     vpermd	(%rax), %ymm1, %ymm2
 # CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -     vpermpd	$1, %ymm0, %ymm2
diff --git a/test/tools/llvm-mca/X86/Znver1/resources-bmi1.s b/test/tools/llvm-mca/X86/Znver1/resources-bmi1.s
index 8b74a1d90f8e3580b06e0a2756255d884035c6d3..eebf53ea275ff17a65f244ee9bbde0901a6f47b9 100644
--- a/test/tools/llvm-mca/X86/Znver1/resources-bmi1.s
+++ b/test/tools/llvm-mca/X86/Znver1/resources-bmi1.s
@@ -31,6 +31,9 @@ blsr        (%rax), %ecx
 blsr        %rax, %rcx
 blsr        (%rax), %rcx
 
+tzcnt       %ax, %cx
+tzcnt       (%rax), %cx
+
 tzcnt       %eax, %ecx
 tzcnt       (%rax), %ecx
 
@@ -66,6 +69,8 @@ tzcnt       (%rax), %rcx
 # CHECK-NEXT:  1      6     0.50    *                   blsrl	(%rax), %ecx
 # CHECK-NEXT:  1      2     0.25                        blsrq	%rax, %rcx
 # CHECK-NEXT:  1      6     0.50    *                   blsrq	(%rax), %rcx
+# CHECK-NEXT:  1      2     0.25                        tzcntw	%ax, %cx
+# CHECK-NEXT:  2      6     0.50    *                   tzcntw	(%rax), %cx
 # CHECK-NEXT:  1      2     0.25                        tzcntl	%eax, %ecx
 # CHECK-NEXT:  2      6     0.50    *                   tzcntl	(%rax), %ecx
 # CHECK-NEXT:  1      2     0.25                        tzcntq	%rax, %rcx
@@ -87,7 +92,7 @@ tzcnt       (%rax), %rcx
 
 # CHECK:      Resource pressure per iteration:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]
-# CHECK-NEXT: 6.00   6.00   6.00   6.00   6.00   6.00    -      -      -      -      -      -
+# CHECK-NEXT: 6.50   6.50   6.50   6.50   6.50   6.50    -      -      -      -      -      -
 
 # CHECK:      Resource pressure by instruction:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   Instructions:
@@ -111,6 +116,8 @@ tzcnt       (%rax), %rcx
 # CHECK-NEXT: 0.50   0.50   0.25   0.25   0.25   0.25    -      -      -      -      -      -     blsrl	(%rax), %ecx
 # CHECK-NEXT:  -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     blsrq	%rax, %rcx
 # CHECK-NEXT: 0.50   0.50   0.25   0.25   0.25   0.25    -      -      -      -      -      -     blsrq	(%rax), %rcx
+# CHECK-NEXT:  -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     tzcntw	%ax, %cx
+# CHECK-NEXT: 0.50   0.50   0.25   0.25   0.25   0.25    -      -      -      -      -      -     tzcntw	(%rax), %cx
 # CHECK-NEXT:  -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     tzcntl	%eax, %ecx
 # CHECK-NEXT: 0.50   0.50   0.25   0.25   0.25   0.25    -      -      -      -      -      -     tzcntl	(%rax), %ecx
 # CHECK-NEXT:  -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     tzcntq	%rax, %rcx
diff --git a/test/tools/llvm-mca/X86/Znver1/resources-clzero.s b/test/tools/llvm-mca/X86/Znver1/resources-clzero.s
new file mode 100644
index 0000000000000000000000000000000000000000..29f04f18aeed388a67331501ceaf4eb55b28c648
--- /dev/null
+++ b/test/tools/llvm-mca/X86/Znver1/resources-clzero.s
@@ -0,0 +1,37 @@
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=x86_64-unknown-unknown -mcpu=znver1 -instruction-tables < %s | FileCheck %s
+
+clzero
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  1      100   0.25                  U     clzero
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0]   - ZnAGU0
+# CHECK-NEXT: [1]   - ZnAGU1
+# CHECK-NEXT: [2]   - ZnALU0
+# CHECK-NEXT: [3]   - ZnALU1
+# CHECK-NEXT: [4]   - ZnALU2
+# CHECK-NEXT: [5]   - ZnALU3
+# CHECK-NEXT: [6]   - ZnDivider
+# CHECK-NEXT: [7]   - ZnFPU0
+# CHECK-NEXT: [8]   - ZnFPU1
+# CHECK-NEXT: [9]   - ZnFPU2
+# CHECK-NEXT: [10]  - ZnFPU3
+# CHECK-NEXT: [11]  - ZnMultiplier
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   Instructions:
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     clzero
diff --git a/test/tools/llvm-mca/X86/Znver1/resources-fsgsbase.s b/test/tools/llvm-mca/X86/Znver1/resources-fsgsbase.s
new file mode 100644
index 0000000000000000000000000000000000000000..88bcd37b2d08b2dac7b687465b926e7de3844a1c
--- /dev/null
+++ b/test/tools/llvm-mca/X86/Znver1/resources-fsgsbase.s
@@ -0,0 +1,61 @@
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=x86_64-unknown-unknown -mcpu=znver1 -instruction-tables < %s | FileCheck %s
+
+rdfsbase %eax
+rdfsbase %rax
+
+rdgsbase %eax
+rdgsbase %rax
+
+wrfsbase %edi
+wrfsbase %rdi
+
+wrgsbase %edi
+wrgsbase %rdi
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  1      100   0.25    *      *      U     rdfsbasel	%eax
+# CHECK-NEXT:  1      100   0.25    *      *      U     rdfsbaseq	%rax
+# CHECK-NEXT:  1      100   0.25    *      *      U     rdgsbasel	%eax
+# CHECK-NEXT:  1      100   0.25    *      *      U     rdgsbaseq	%rax
+# CHECK-NEXT:  1      100   0.25    *      *      U     wrfsbasel	%edi
+# CHECK-NEXT:  1      100   0.25    *      *      U     wrfsbaseq	%rdi
+# CHECK-NEXT:  1      100   0.25    *      *      U     wrgsbasel	%edi
+# CHECK-NEXT:  1      100   0.25    *      *      U     wrgsbaseq	%rdi
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0]   - ZnAGU0
+# CHECK-NEXT: [1]   - ZnAGU1
+# CHECK-NEXT: [2]   - ZnALU0
+# CHECK-NEXT: [3]   - ZnALU1
+# CHECK-NEXT: [4]   - ZnALU2
+# CHECK-NEXT: [5]   - ZnALU3
+# CHECK-NEXT: [6]   - ZnDivider
+# CHECK-NEXT: [7]   - ZnFPU0
+# CHECK-NEXT: [8]   - ZnFPU1
+# CHECK-NEXT: [9]   - ZnFPU2
+# CHECK-NEXT: [10]  - ZnFPU3
+# CHECK-NEXT: [11]  - ZnMultiplier
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   Instructions:
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     rdfsbasel	%eax
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     rdfsbaseq	%rax
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     rdgsbasel	%eax
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     rdgsbaseq	%rax
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     wrfsbasel	%edi
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     wrfsbaseq	%rdi
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     wrgsbasel	%edi
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     wrgsbaseq	%rdi
diff --git a/test/tools/llvm-mca/X86/Znver1/resources-mwaitx.s b/test/tools/llvm-mca/X86/Znver1/resources-mwaitx.s
new file mode 100644
index 0000000000000000000000000000000000000000..c296b21d75418508371f61046e45dd7604477bc8
--- /dev/null
+++ b/test/tools/llvm-mca/X86/Znver1/resources-mwaitx.s
@@ -0,0 +1,40 @@
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=x86_64-unknown-unknown -mcpu=znver1 -instruction-tables < %s | FileCheck %s
+
+monitorx
+mwaitx
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  1      100   0.25                  U     monitorx
+# CHECK-NEXT:  1      100   0.25    *      *      U     mwaitx
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0]   - ZnAGU0
+# CHECK-NEXT: [1]   - ZnAGU1
+# CHECK-NEXT: [2]   - ZnALU0
+# CHECK-NEXT: [3]   - ZnALU1
+# CHECK-NEXT: [4]   - ZnALU2
+# CHECK-NEXT: [5]   - ZnALU3
+# CHECK-NEXT: [6]   - ZnDivider
+# CHECK-NEXT: [7]   - ZnFPU0
+# CHECK-NEXT: [8]   - ZnFPU1
+# CHECK-NEXT: [9]   - ZnFPU2
+# CHECK-NEXT: [10]  - ZnFPU3
+# CHECK-NEXT: [11]  - ZnMultiplier
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   Instructions:
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     monitorx
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     mwaitx
diff --git a/test/tools/llvm-mca/X86/Znver1/resources-sha.s b/test/tools/llvm-mca/X86/Znver1/resources-sha.s
new file mode 100644
index 0000000000000000000000000000000000000000..57839d03348152929e716c5f6f1a792a10b3deab
--- /dev/null
+++ b/test/tools/llvm-mca/X86/Znver1/resources-sha.s
@@ -0,0 +1,82 @@
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=x86_64-unknown-unknown -mcpu=znver1 -instruction-tables < %s | FileCheck %s
+
+sha1msg1    %xmm0, %xmm2
+sha1msg1    (%rax), %xmm2
+
+sha1msg2    %xmm0, %xmm2
+sha1msg2    (%rax), %xmm2
+
+sha1nexte   %xmm0, %xmm2
+sha1nexte   (%rax), %xmm2
+
+sha1rnds4   $3, %xmm0, %xmm2
+sha1rnds4   $3, (%rax), %xmm2
+
+sha256msg1  %xmm0, %xmm2
+sha256msg1  (%rax), %xmm2
+
+sha256msg2  %xmm0, %xmm2
+sha256msg2  (%rax), %xmm2
+
+sha256rnds2 %xmm0, %xmm2
+sha256rnds2 (%rax), %xmm2
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  1      2     1.00                        sha1msg1	%xmm0, %xmm2
+# CHECK-NEXT:  1      9     1.00    *                   sha1msg1	(%rax), %xmm2
+# CHECK-NEXT:  1      1     0.50                        sha1msg2	%xmm0, %xmm2
+# CHECK-NEXT:  1      8     0.50    *                   sha1msg2	(%rax), %xmm2
+# CHECK-NEXT:  1      1     1.00                        sha1nexte	%xmm0, %xmm2
+# CHECK-NEXT:  1      8     1.00    *                   sha1nexte	(%rax), %xmm2
+# CHECK-NEXT:  1      6     1.00                        sha1rnds4	$3, %xmm0, %xmm2
+# CHECK-NEXT:  1      13    1.00    *                   sha1rnds4	$3, (%rax), %xmm2
+# CHECK-NEXT:  1      2     1.00                        sha256msg1	%xmm0, %xmm2
+# CHECK-NEXT:  1      9     1.00    *                   sha256msg1	(%rax), %xmm2
+# CHECK-NEXT:  1      100   0.25                        sha256msg2	%xmm0, %xmm2
+# CHECK-NEXT:  1      100   0.25    *                   sha256msg2	(%rax), %xmm2
+# CHECK-NEXT:  1      4     1.00                        sha256rnds2	%xmm0, %xmm0, %xmm2
+# CHECK-NEXT:  1      11    1.00    *                   sha256rnds2	%xmm0, (%rax), %xmm2
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0]   - ZnAGU0
+# CHECK-NEXT: [1]   - ZnAGU1
+# CHECK-NEXT: [2]   - ZnALU0
+# CHECK-NEXT: [3]   - ZnALU1
+# CHECK-NEXT: [4]   - ZnALU2
+# CHECK-NEXT: [5]   - ZnALU3
+# CHECK-NEXT: [6]   - ZnDivider
+# CHECK-NEXT: [7]   - ZnFPU0
+# CHECK-NEXT: [8]   - ZnFPU1
+# CHECK-NEXT: [9]   - ZnFPU2
+# CHECK-NEXT: [10]  - ZnFPU3
+# CHECK-NEXT: [11]  - ZnMultiplier
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]
+# CHECK-NEXT: 3.00   3.00    -      -      -      -      -      -     11.00  5.00    -      -
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   Instructions:
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00    -      -     sha1msg1	%xmm0, %xmm2
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -     1.00   1.00    -      -     sha1msg1	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -     sha1msg2	%xmm0, %xmm2
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     sha1msg2	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00    -      -      -     sha1nexte	%xmm0, %xmm2
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -     1.00    -      -      -     sha1nexte	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00    -      -      -     sha1rnds4	$3, %xmm0, %xmm2
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -     1.00    -      -      -     sha1rnds4	$3, (%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   1.00    -      -     sha256msg1	%xmm0, %xmm2
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -     1.00   1.00    -      -     sha256msg1	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     sha256msg2	%xmm0, %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     sha256msg2	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00    -      -      -     sha256rnds2	%xmm0, %xmm0, %xmm2
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -     1.00    -      -      -     sha256rnds2	%xmm0, (%rax), %xmm2
diff --git a/test/tools/llvm-mca/X86/Znver1/resources-sse2.s b/test/tools/llvm-mca/X86/Znver1/resources-sse2.s
index a4ecff87ae196e3b24a025f9f0fc786aedd19485..d6b9c025766221460f07556b66607441dca715bb 100644
--- a/test/tools/llvm-mca/X86/Znver1/resources-sse2.s
+++ b/test/tools/llvm-mca/X86/Znver1/resources-sse2.s
@@ -94,6 +94,8 @@ maxpd       (%rax), %xmm2
 maxsd       %xmm0, %xmm2
 maxsd       (%rax), %xmm2
 
+mfence
+
 minpd       %xmm0, %xmm2
 minpd       (%rax), %xmm2
 
@@ -229,6 +231,9 @@ pcmpgtw     (%rax), %xmm2
 
 pextrw      $1, %xmm0, %rcx
 
+pinsrw      $1, %rax, %xmm0
+pinsrw      $1, (%rax), %xmm0
+
 pmaddwd     %xmm0, %xmm2
 pmaddwd     (%rax), %xmm2
 
@@ -465,6 +470,7 @@ xorpd       (%rax), %xmm2
 # CHECK-NEXT:  1      10    1.00    *                   maxpd	(%rax), %xmm2
 # CHECK-NEXT:  1      3     1.00                        maxsd	%xmm0, %xmm2
 # CHECK-NEXT:  1      10    1.00    *                   maxsd	(%rax), %xmm2
+# CHECK-NEXT:  1      1     0.50    *      *      U     mfence
 # CHECK-NEXT:  1      3     1.00                        minpd	%xmm0, %xmm2
 # CHECK-NEXT:  1      10    1.00    *                   minpd	(%rax), %xmm2
 # CHECK-NEXT:  1      3     1.00                        minsd	%xmm0, %xmm2
@@ -555,6 +561,8 @@ xorpd       (%rax), %xmm2
 # CHECK-NEXT:  1      1     0.25                        pcmpgtw	%xmm0, %xmm2
 # CHECK-NEXT:  1      8     0.50    *                   pcmpgtw	(%rax), %xmm2
 # CHECK-NEXT:  1      2     2.00                        pextrw	$1, %xmm0, %ecx
+# CHECK-NEXT:  1      1     0.25                        pinsrw	$1, %eax, %xmm0
+# CHECK-NEXT:  1      8     0.50    *                   pinsrw	$1, (%rax), %xmm0
 # CHECK-NEXT:  1      4     1.00                        pmaddwd	%xmm0, %xmm2
 # CHECK-NEXT:  1      11    1.00    *                   pmaddwd	(%rax), %xmm2
 # CHECK-NEXT:  1      1     0.25                        pmaxsw	%xmm0, %xmm2
@@ -683,7 +691,7 @@ xorpd       (%rax), %xmm2
 
 # CHECK:      Resource pressure per iteration:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]
-# CHECK-NEXT: 65.50  65.50   -      -      -      -      -     72.42  39.92  71.25  153.42  -
+# CHECK-NEXT: 66.50  66.50   -      -      -      -      -     72.92  40.42  71.75  153.92  -
 
 # CHECK:      Resource pressure by instruction:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   Instructions:
@@ -750,6 +758,7 @@ xorpd       (%rax), %xmm2
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -     1.00    -      -      -      -     maxpd	(%rax), %xmm2
 # CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -     maxsd	%xmm0, %xmm2
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -     1.00    -      -      -      -     maxsd	(%rax), %xmm2
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     mfence
 # CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -     minpd	%xmm0, %xmm2
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -     1.00    -      -      -      -     minpd	(%rax), %xmm2
 # CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -     minsd	%xmm0, %xmm2
@@ -840,6 +849,8 @@ xorpd       (%rax), %xmm2
 # CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -     pcmpgtw	%xmm0, %xmm2
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -     0.25   0.25   0.25   0.25    -     pcmpgtw	(%rax), %xmm2
 # CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   2.50    -      -     pextrw	$1, %xmm0, %ecx
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -     pinsrw	$1, %eax, %xmm0
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -     0.25   0.25   0.25   0.25    -     pinsrw	$1, (%rax), %xmm0
 # CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -     pmaddwd	%xmm0, %xmm2
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -     1.00    -      -      -      -     pmaddwd	(%rax), %xmm2
 # CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -     pmaxsw	%xmm0, %xmm2
diff --git a/test/tools/llvm-mca/X86/Znver1/resources-sse3.s b/test/tools/llvm-mca/X86/Znver1/resources-sse3.s
index 0e61c4d6c5ce43b031a07235e111b67c338cba47..8de91b63bbb3bce8b8ca297f1de660e41a7ac8ed 100644
--- a/test/tools/llvm-mca/X86/Znver1/resources-sse3.s
+++ b/test/tools/llvm-mca/X86/Znver1/resources-sse3.s
@@ -21,6 +21,8 @@ hsubps    (%rax), %xmm2
 
 lddqu     (%rax), %xmm2
 
+monitor
+
 movddup   %xmm0, %xmm2
 movddup   (%rax), %xmm2
 
@@ -30,6 +32,8 @@ movshdup  (%rax), %xmm2
 movsldup  %xmm0, %xmm2
 movsldup  (%rax), %xmm2
 
+mwait
+
 # CHECK:      Instruction Info:
 # CHECK-NEXT: [1]: #uOps
 # CHECK-NEXT: [2]: Latency
@@ -52,12 +56,14 @@ movsldup  (%rax), %xmm2
 # CHECK-NEXT:  1      100   0.25                        hsubps	%xmm0, %xmm2
 # CHECK-NEXT:  1      100   0.25    *                   hsubps	(%rax), %xmm2
 # CHECK-NEXT:  1      8     0.50    *                   lddqu	(%rax), %xmm2
+# CHECK-NEXT:  1      100   0.25                  U     monitor
 # CHECK-NEXT:  1      1     0.50                        movddup	%xmm0, %xmm2
 # CHECK-NEXT:  1      8     0.50    *                   movddup	(%rax), %xmm2
 # CHECK-NEXT:  1      1     0.50                        movshdup	%xmm0, %xmm2
 # CHECK-NEXT:  1      8     0.50    *                   movshdup	(%rax), %xmm2
 # CHECK-NEXT:  1      100   0.25                        movsldup	%xmm0, %xmm2
 # CHECK-NEXT:  1      100   0.25    *                   movsldup	(%rax), %xmm2
+# CHECK-NEXT:  1      100   0.25    *      *      U     mwait
 
 # CHECK:      Resources:
 # CHECK-NEXT: [0]   - ZnAGU0
@@ -92,9 +98,11 @@ movsldup  (%rax), %xmm2
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     hsubps	%xmm0, %xmm2
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     hsubps	(%rax), %xmm2
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -      -      -     lddqu	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     monitor
 # CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -     movddup	%xmm0, %xmm2
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     movddup	(%rax), %xmm2
 # CHECK-NEXT:  -      -      -      -      -      -      -      -     0.50   0.50    -      -     movshdup	%xmm0, %xmm2
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -     0.50   0.50    -      -     movshdup	(%rax), %xmm2
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     movsldup	%xmm0, %xmm2
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     movsldup	(%rax), %xmm2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     mwait
diff --git a/test/tools/llvm-mca/X86/Znver1/resources-x86_64.s b/test/tools/llvm-mca/X86/Znver1/resources-x86_64.s
index 5a6eed42f7e8dbfcbcd29648b2f379c55e298a39..7de329ec3bc3abb3c1a602e9ba269bdccd4f65ff 100644
--- a/test/tools/llvm-mca/X86/Znver1/resources-x86_64.s
+++ b/test/tools/llvm-mca/X86/Znver1/resources-x86_64.s
@@ -254,6 +254,8 @@ divl (%rax)
 divq %rcx
 divq (%rax)
 
+enter $7, $4095
+
 idivb %dil
 idivb (%rax)
 idivw %si
@@ -315,8 +317,13 @@ insl
 
 int $7
 
+invlpg  (%rax)
+invlpga %rax, %ecx
+
 lahf
 
+leave
+
 lodsb
 lodsw
 lodsl
@@ -486,6 +493,11 @@ rcrq %cl, %rdi
 rclq %cl, (%rax)
 rcrq %cl, (%rax)
 
+rdmsr
+rdpmc
+rdtsc
+rdtscp
+
 rolb %dil
 rorb %dil
 rolb (%rax)
@@ -789,6 +801,8 @@ testq %rsi, (%rax)
 
 ud2
 
+wrmsr
+
 xaddb %bl, %cl
 xaddb %bl, (%rcx)
 
@@ -1081,6 +1095,7 @@ xorq (%rax), %rdi
 # CHECK-NEXT:  3      29    25.00   *             U     divl	(%rax)
 # CHECK-NEXT:  2      41    41.00                 U     divq	%rcx
 # CHECK-NEXT:  3      45    41.00   *             U     divq	(%rax)
+# CHECK-NEXT:  1      100   0.25                  U     enter	$7, $4095
 # CHECK-NEXT:  1      15    15.00                 U     idivb	%dil
 # CHECK-NEXT:  2      19    15.00   *             U     idivb	(%rax)
 # CHECK-NEXT:  2      17    17.00                 U     idivw	%si
@@ -1133,7 +1148,10 @@ xorq (%rax), %rdi
 # CHECK-NEXT:  1      100   0.25                  U     insw	%dx, %es:(%rdi)
 # CHECK-NEXT:  1      100   0.25                  U     insl	%dx, %es:(%rdi)
 # CHECK-NEXT:  1      100   0.25    *      *      U     int	$7
+# CHECK-NEXT:  1      100   0.25                  U     invlpg	(%rax)
+# CHECK-NEXT:  1      100   0.25                  U     invlpga	%rax, %ecx
 # CHECK-NEXT:  1      100   0.25                        lahf
+# CHECK-NEXT:  2      8     0.50    *                   leave
 # CHECK-NEXT:  1      100   0.25                  U     lodsb	(%rsi), %al
 # CHECK-NEXT:  1      100   0.25                  U     lodsw	(%rsi), %ax
 # CHECK-NEXT:  1      100   0.25                  U     lodsl	(%rsi), %eax
@@ -1283,6 +1301,10 @@ xorq (%rax), %rdi
 # CHECK-NEXT:  1      1     0.25                        rcrq	%cl, %rdi
 # CHECK-NEXT:  1      100   0.25           *            rclq	%cl, (%rax)
 # CHECK-NEXT:  1      100   0.25           *            rcrq	%cl, (%rax)
+# CHECK-NEXT:  1      100   0.25                  U     rdmsr
+# CHECK-NEXT:  1      100   0.25                  U     rdpmc
+# CHECK-NEXT:  1      100   0.25                  U     rdtsc
+# CHECK-NEXT:  1      100   0.25                  U     rdtscp
 # CHECK-NEXT:  1      1     0.25                        rolb	%dil
 # CHECK-NEXT:  1      1     0.25                        rorb	%dil
 # CHECK-NEXT:  3      5     1.00    *      *            rolb	(%rax)
@@ -1557,6 +1579,7 @@ xorq (%rax), %rdi
 # CHECK-NEXT:  1      1     0.25                        testq	%rsi, %rdi
 # CHECK-NEXT:  2      5     0.50    *                   testq	%rsi, (%rax)
 # CHECK-NEXT:  1      100   0.25    *             U     ud2
+# CHECK-NEXT:  1      100   0.25                  U     wrmsr
 # CHECK-NEXT:  1      1     0.25                        xaddb	%bl, %cl
 # CHECK-NEXT:  1      100   0.25    *      *            xaddb	%bl, (%rcx)
 # CHECK-NEXT:  1      1     0.25                        xaddw	%bx, %cx
@@ -1624,7 +1647,7 @@ xorq (%rax), %rdi
 
 # CHECK:      Resource pressure per iteration:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]
-# CHECK-NEXT: 165.00 165.00 153.50 187.50 169.50 153.50 392.00  -      -      -      -     34.00
+# CHECK-NEXT: 165.50 165.50 153.75 187.75 169.75 153.75 392.00  -      -      -      -     34.00
 
 # CHECK:      Resource pressure by instruction:
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   Instructions:
@@ -1848,6 +1871,7 @@ xorq (%rax), %rdi
 # CHECK-NEXT: 0.50   0.50    -      -     1.00    -     25.00   -      -      -      -      -     divl	(%rax)
 # CHECK-NEXT:  -      -      -      -     1.00    -     41.00   -      -      -      -      -     divq	%rcx
 # CHECK-NEXT: 0.50   0.50    -      -     1.00    -     41.00   -      -      -      -      -     divq	(%rax)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     enter	$7, $4095
 # CHECK-NEXT:  -      -      -      -     1.00    -     15.00   -      -      -      -      -     idivb	%dil
 # CHECK-NEXT: 0.50   0.50    -      -     1.00    -     15.00   -      -      -      -      -     idivb	(%rax)
 # CHECK-NEXT:  -      -      -      -     1.00    -     17.00   -      -      -      -      -     idivw	%si
@@ -1900,7 +1924,10 @@ xorq (%rax), %rdi
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     insw	%dx, %es:(%rdi)
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     insl	%dx, %es:(%rdi)
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     int	$7
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     invlpg	(%rax)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     invlpga	%rax, %ecx
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     lahf
+# CHECK-NEXT: 0.50   0.50   0.25   0.25   0.25   0.25    -      -      -      -      -      -     leave
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     lodsb	(%rsi), %al
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     lodsw	(%rsi), %ax
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     lodsl	(%rsi), %eax
@@ -2050,6 +2077,10 @@ xorq (%rax), %rdi
 # CHECK-NEXT:  -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     rcrq	%cl, %rdi
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     rclq	%cl, (%rax)
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     rcrq	%cl, (%rax)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     rdmsr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     rdpmc
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     rdtsc
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     rdtscp
 # CHECK-NEXT:  -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     rolb	%dil
 # CHECK-NEXT:  -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     rorb	%dil
 # CHECK-NEXT: 1.00   1.00   0.25   0.25   0.25   0.25    -      -      -      -      -      -     rolb	(%rax)
@@ -2324,6 +2355,7 @@ xorq (%rax), %rdi
 # CHECK-NEXT:  -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     testq	%rsi, %rdi
 # CHECK-NEXT: 0.50   0.50   0.25   0.25   0.25   0.25    -      -      -      -      -      -     testq	%rsi, (%rax)
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     ud2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     wrmsr
 # CHECK-NEXT:  -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     xaddb	%bl, %cl
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     xaddb	%bl, (%rcx)
 # CHECK-NEXT:  -      -     0.25   0.25   0.25   0.25    -      -      -      -      -      -     xaddw	%bx, %cx
diff --git a/test/tools/llvm-mca/X86/Znver1/resources-x87.s b/test/tools/llvm-mca/X86/Znver1/resources-x87.s
index 2f5f6ef08f1c72d625e741f89fb721f9055e6ac4..030b71fb7b63f7f5dd21481aafcd214db7999ad9 100644
--- a/test/tools/llvm-mca/X86/Znver1/resources-x87.s
+++ b/test/tools/llvm-mca/X86/Znver1/resources-x87.s
@@ -5,7 +5,7 @@ f2xm1
 
 fabs
 
-fadd %st(0), %st(1)
+fadd %st, %st(1)
 fadd %st(2)
 fadds (%ecx)
 faddl (%ecx)
@@ -21,14 +21,14 @@ fchs
 
 fnclex
 
-fcmovb %st(1), %st(0)
-fcmovbe %st(1), %st(0)
-fcmove %st(1), %st(0)
-fcmovnb %st(1), %st(0)
-fcmovnbe %st(1), %st(0)
-fcmovne %st(1), %st(0)
-fcmovnu %st(1), %st(0)
-fcmovu %st(1), %st(0)
+fcmovb %st(1), %st
+fcmovbe %st(1), %st
+fcmove %st(1), %st
+fcmovnb %st(1), %st
+fcmovnbe %st(1), %st
+fcmovne %st(1), %st
+fcmovnu %st(1), %st
+fcmovu %st(1), %st
 
 fcom %st(1)
 fcom %st(3)
@@ -47,7 +47,7 @@ fcos
 
 fdecstp
 
-fdiv %st(0), %st(1)
+fdiv %st, %st(1)
 fdiv %st(2)
 fdivs (%ecx)
 fdivl (%eax)
@@ -56,7 +56,7 @@ fdivp %st(2)
 fidivs (%ecx)
 fidivl (%eax)
 
-fdivr %st(0), %st(1)
+fdivr %st, %st(1)
 fdivr %st(2)
 fdivrs (%ecx)
 fdivrl (%eax)
@@ -106,7 +106,7 @@ fldln2
 fldpi
 fldz
 
-fmul %st(0), %st(1)
+fmul %st, %st(1)
 fmul %st(2)
 fmuls (%ecx)
 fmull (%eax)
@@ -153,7 +153,7 @@ fnstsw (%eax)
 frstor (%eax)
 fsave (%eax)
 
-fsub %st(0), %st(1)
+fsub %st, %st(1)
 fsub %st(2)
 fsubs (%ecx)
 fsubl (%eax)
@@ -162,7 +162,7 @@ fsubp %st(2)
 fisubs (%ecx)
 fisubl (%eax)
 
-fsubr %st(0), %st(1)
+fsubr %st, %st(1)
 fsubr %st(2)
 fsubrs (%ecx)
 fsubrl (%eax)
@@ -208,26 +208,26 @@ fyl2xp1
 # CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
 # CHECK-NEXT:  1      100   0.25                  U     f2xm1
 # CHECK-NEXT:  1      2     1.00                  U     fabs
-# CHECK-NEXT:  1      3     1.00                  U     fadd	%st(0), %st(1)
-# CHECK-NEXT:  1      3     1.00                  U     fadd	%st(2)
+# CHECK-NEXT:  1      3     1.00                  U     fadd	%st, %st(1)
+# CHECK-NEXT:  1      3     1.00                  U     fadd	%st(2), %st
 # CHECK-NEXT:  1      10    1.00    *             U     fadds	(%ecx)
 # CHECK-NEXT:  1      10    1.00    *             U     faddl	(%ecx)
-# CHECK-NEXT:  1      3     1.00                  U     faddp	%st(1)
-# CHECK-NEXT:  1      3     1.00                  U     faddp	%st(2)
+# CHECK-NEXT:  1      3     1.00                  U     faddp	%st, %st(1)
+# CHECK-NEXT:  1      3     1.00                  U     faddp	%st, %st(2)
 # CHECK-NEXT:  1      10    1.00    *             U     fiadds	(%ecx)
 # CHECK-NEXT:  1      10    1.00    *             U     fiaddl	(%ecx)
 # CHECK-NEXT:  1      100   0.25                  U     fbld	(%ecx)
 # CHECK-NEXT:  1      100   0.25                  U     fbstp	(%eax)
 # CHECK-NEXT:  1      1     1.00                  U     fchs
 # CHECK-NEXT:  1      100   0.25                  U     fnclex
-# CHECK-NEXT:  1      100   0.25                  U     fcmovb	%st(1), %st(0)
-# CHECK-NEXT:  1      100   0.25                  U     fcmovbe	%st(1), %st(0)
-# CHECK-NEXT:  1      100   0.25                  U     fcmove	%st(1), %st(0)
-# CHECK-NEXT:  1      100   0.25                  U     fcmovnb	%st(1), %st(0)
-# CHECK-NEXT:  1      100   0.25                  U     fcmovnbe	%st(1), %st(0)
-# CHECK-NEXT:  1      100   0.25                  U     fcmovne	%st(1), %st(0)
-# CHECK-NEXT:  1      100   0.25                  U     fcmovnu	%st(1), %st(0)
-# CHECK-NEXT:  1      100   0.25                  U     fcmovu	%st(1), %st(0)
+# CHECK-NEXT:  1      100   0.25                  U     fcmovb	%st(1), %st
+# CHECK-NEXT:  1      100   0.25                  U     fcmovbe	%st(1), %st
+# CHECK-NEXT:  1      100   0.25                  U     fcmove	%st(1), %st
+# CHECK-NEXT:  1      100   0.25                  U     fcmovnb	%st(1), %st
+# CHECK-NEXT:  1      100   0.25                  U     fcmovnbe	%st(1), %st
+# CHECK-NEXT:  1      100   0.25                  U     fcmovne	%st(1), %st
+# CHECK-NEXT:  1      100   0.25                  U     fcmovnu	%st(1), %st
+# CHECK-NEXT:  1      100   0.25                  U     fcmovu	%st(1), %st
 # CHECK-NEXT:  1      1     1.00                  U     fcom	%st(1)
 # CHECK-NEXT:  1      1     1.00                  U     fcom	%st(3)
 # CHECK-NEXT:  1      8     1.00                  U     fcoms	(%ecx)
@@ -237,24 +237,24 @@ fyl2xp1
 # CHECK-NEXT:  1      8     1.00                  U     fcomps	(%ecx)
 # CHECK-NEXT:  1      8     1.00                  U     fcompl	(%eax)
 # CHECK-NEXT:  1      1     1.00                  U     fcompp
-# CHECK-NEXT:  1      9     0.50                  U     fcomi	%st(3)
-# CHECK-NEXT:  1      9     0.50                  U     fcompi	%st(3)
+# CHECK-NEXT:  1      9     0.50                  U     fcomi	%st(3), %st
+# CHECK-NEXT:  1      9     0.50                  U     fcompi	%st(3), %st
 # CHECK-NEXT:  1      100   0.25                  U     fcos
 # CHECK-NEXT:  1      11    1.00                  U     fdecstp
-# CHECK-NEXT:  1      15    1.00                  U     fdiv	%st(0), %st(1)
-# CHECK-NEXT:  1      15    1.00                  U     fdiv	%st(2)
+# CHECK-NEXT:  1      15    1.00                  U     fdiv	%st, %st(1)
+# CHECK-NEXT:  1      15    1.00                  U     fdiv	%st(2), %st
 # CHECK-NEXT:  1      22    1.00    *             U     fdivs	(%ecx)
 # CHECK-NEXT:  1      22    1.00    *             U     fdivl	(%eax)
-# CHECK-NEXT:  1      15    1.00                  U     fdivp	%st(1)
-# CHECK-NEXT:  1      15    1.00                  U     fdivp	%st(2)
+# CHECK-NEXT:  1      15    1.00                  U     fdivp	%st, %st(1)
+# CHECK-NEXT:  1      15    1.00                  U     fdivp	%st, %st(2)
 # CHECK-NEXT:  1      22    1.00    *             U     fidivs	(%ecx)
 # CHECK-NEXT:  1      22    1.00    *             U     fidivl	(%eax)
-# CHECK-NEXT:  1      15    1.00                  U     fdivr	%st(0), %st(1)
-# CHECK-NEXT:  1      15    1.00                  U     fdivr	%st(2)
+# CHECK-NEXT:  1      15    1.00                  U     fdivr	%st, %st(1)
+# CHECK-NEXT:  1      15    1.00                  U     fdivr	%st(2), %st
 # CHECK-NEXT:  1      22    1.00    *             U     fdivrs	(%ecx)
 # CHECK-NEXT:  1      22    1.00    *             U     fdivrl	(%eax)
-# CHECK-NEXT:  1      15    1.00                  U     fdivrp	%st(1)
-# CHECK-NEXT:  1      15    1.00                  U     fdivrp	%st(2)
+# CHECK-NEXT:  1      15    1.00                  U     fdivrp	%st, %st(1)
+# CHECK-NEXT:  1      15    1.00                  U     fdivrp	%st, %st(2)
 # CHECK-NEXT:  1      22    1.00    *             U     fidivrs	(%ecx)
 # CHECK-NEXT:  1      22    1.00    *             U     fidivrl	(%eax)
 # CHECK-NEXT:  1      11    1.00                  U     ffree	%st(0)
@@ -288,12 +288,12 @@ fyl2xp1
 # CHECK-NEXT:  1      11    1.00                  U     fldln2
 # CHECK-NEXT:  1      11    1.00                  U     fldpi
 # CHECK-NEXT:  1      8     0.50                  U     fldz
-# CHECK-NEXT:  1      3     0.50                  U     fmul	%st(0), %st(1)
-# CHECK-NEXT:  1      3     0.50                  U     fmul	%st(2)
+# CHECK-NEXT:  1      3     0.50                  U     fmul	%st, %st(1)
+# CHECK-NEXT:  1      3     0.50                  U     fmul	%st(2), %st
 # CHECK-NEXT:  2      10    0.50    *             U     fmuls	(%ecx)
 # CHECK-NEXT:  2      10    0.50    *             U     fmull	(%eax)
-# CHECK-NEXT:  1      3     0.50                  U     fmulp	%st(1)
-# CHECK-NEXT:  1      3     0.50                  U     fmulp	%st(2)
+# CHECK-NEXT:  1      3     0.50                  U     fmulp	%st, %st(1)
+# CHECK-NEXT:  1      3     0.50                  U     fmulp	%st, %st(2)
 # CHECK-NEXT:  2      10    0.50    *             U     fimuls	(%ecx)
 # CHECK-NEXT:  2      10    0.50    *             U     fimull	(%eax)
 # CHECK-NEXT:  1      1     1.00                  U     fnop
@@ -321,20 +321,20 @@ fyl2xp1
 # CHECK-NEXT:  1      100   0.25                  U     frstor	(%eax)
 # CHECK-NEXT:  1      1     1.00                  U     wait
 # CHECK-NEXT:  1      100   0.25                  U     fnsave	(%eax)
-# CHECK-NEXT:  1      3     1.00                  U     fsub	%st(0), %st(1)
-# CHECK-NEXT:  1      3     1.00                  U     fsub	%st(2)
+# CHECK-NEXT:  1      3     1.00                  U     fsub	%st, %st(1)
+# CHECK-NEXT:  1      3     1.00                  U     fsub	%st(2), %st
 # CHECK-NEXT:  1      10    1.00    *             U     fsubs	(%ecx)
 # CHECK-NEXT:  1      10    1.00    *             U     fsubl	(%eax)
-# CHECK-NEXT:  1      3     1.00                  U     fsubp	%st(1)
-# CHECK-NEXT:  1      3     1.00                  U     fsubp	%st(2)
+# CHECK-NEXT:  1      3     1.00                  U     fsubp	%st, %st(1)
+# CHECK-NEXT:  1      3     1.00                  U     fsubp	%st, %st(2)
 # CHECK-NEXT:  1      10    1.00    *             U     fisubs	(%ecx)
 # CHECK-NEXT:  1      10    1.00    *             U     fisubl	(%eax)
-# CHECK-NEXT:  1      3     1.00                  U     fsubr	%st(0), %st(1)
-# CHECK-NEXT:  1      3     1.00                  U     fsubr	%st(2)
+# CHECK-NEXT:  1      3     1.00                  U     fsubr	%st, %st(1)
+# CHECK-NEXT:  1      3     1.00                  U     fsubr	%st(2), %st
 # CHECK-NEXT:  1      10    1.00    *             U     fsubrs	(%ecx)
 # CHECK-NEXT:  1      10    1.00    *             U     fsubrl	(%eax)
-# CHECK-NEXT:  1      3     1.00                  U     fsubrp	%st(1)
-# CHECK-NEXT:  1      3     1.00                  U     fsubrp	%st(2)
+# CHECK-NEXT:  1      3     1.00                  U     fsubrp	%st, %st(1)
+# CHECK-NEXT:  1      3     1.00                  U     fsubrp	%st, %st(2)
 # CHECK-NEXT:  1      10    1.00    *             U     fisubrs	(%ecx)
 # CHECK-NEXT:  1      10    1.00    *             U     fisubrl	(%eax)
 # CHECK-NEXT:  1      1     1.00                  U     ftst
@@ -343,8 +343,8 @@ fyl2xp1
 # CHECK-NEXT:  1      1     1.00                  U     fucomp	%st(1)
 # CHECK-NEXT:  1      1     1.00                  U     fucomp	%st(3)
 # CHECK-NEXT:  1      1     1.00                  U     fucompp
-# CHECK-NEXT:  1      9     0.50                  U     fucomi	%st(3)
-# CHECK-NEXT:  1      9     0.50                  U     fucompi	%st(3)
+# CHECK-NEXT:  1      9     0.50                  U     fucomi	%st(3), %st
+# CHECK-NEXT:  1      9     0.50                  U     fucompi	%st(3), %st
 # CHECK-NEXT:  1      1     1.00                  U     wait
 # CHECK-NEXT:  1      1     1.00                  U     fxam
 # CHECK-NEXT:  1      1     0.25                  U     fxch	%st(1)
@@ -377,26 +377,26 @@ fyl2xp1
 # CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10]   [11]   Instructions:
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     f2xm1
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -     fabs
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -     fadd	%st(0), %st(1)
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -     fadd	%st(2)
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -     fadd	%st, %st(1)
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -     fadd	%st(2), %st
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -     1.00    -      -      -      -     fadds	(%ecx)
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -     1.00    -      -      -      -     faddl	(%ecx)
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -     faddp	%st(1)
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -     faddp	%st(2)
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -     faddp	%st, %st(1)
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -     faddp	%st, %st(2)
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -     1.00    -      -      -      -     fiadds	(%ecx)
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -     1.00    -      -      -      -     fiaddl	(%ecx)
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     fbld	(%ecx)
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     fbstp	(%eax)
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -     fchs
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     fnclex
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     fcmovb	%st(1), %st(0)
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     fcmovbe	%st(1), %st(0)
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     fcmove	%st(1), %st(0)
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     fcmovnb	%st(1), %st(0)
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     fcmovnbe	%st(1), %st(0)
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     fcmovne	%st(1), %st(0)
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     fcmovnu	%st(1), %st(0)
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     fcmovu	%st(1), %st(0)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     fcmovb	%st(1), %st
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     fcmovbe	%st(1), %st
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     fcmove	%st(1), %st
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     fcmovnb	%st(1), %st
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     fcmovnbe	%st(1), %st
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     fcmovne	%st(1), %st
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     fcmovnu	%st(1), %st
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     fcmovu	%st(1), %st
 # CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -     fcom	%st(1)
 # CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -     fcom	%st(3)
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -     1.00    -      -      -      -     fcoms	(%ecx)
@@ -406,24 +406,24 @@ fyl2xp1
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -     1.00    -      -      -      -     fcomps	(%ecx)
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -     1.00    -      -      -      -     fcompl	(%eax)
 # CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -     fcompp
-# CHECK-NEXT: 0.50   0.50    -      -      -      -      -     0.50    -     0.50    -      -     fcomi	%st(3)
-# CHECK-NEXT: 0.50   0.50    -      -      -      -      -     0.50    -     0.50    -      -     fcompi	%st(3)
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -     0.50    -     0.50    -      -     fcomi	%st(3), %st
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -     0.50    -     0.50    -      -     fcompi	%st(3), %st
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     fcos
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -     1.00    -     fdecstp
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -     fdiv	%st(0), %st(1)
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -     fdiv	%st(2)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -     fdiv	%st, %st(1)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -     fdiv	%st(2), %st
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -     1.00    -     fdivs	(%ecx)
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -     1.00    -     fdivl	(%eax)
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -     fdivp	%st(1)
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -     fdivp	%st(2)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -     fdivp	%st, %st(1)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -     fdivp	%st, %st(2)
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -     1.00    -     fidivs	(%ecx)
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -     1.00    -     fidivl	(%eax)
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -     fdivr	%st(0), %st(1)
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -     fdivr	%st(2)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -     fdivr	%st, %st(1)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -     fdivr	%st(2), %st
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -     1.00    -     fdivrs	(%ecx)
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -     1.00    -     fdivrl	(%eax)
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -     fdivrp	%st(1)
-# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -     fdivrp	%st(2)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -     fdivrp	%st, %st(1)
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -     fdivrp	%st, %st(2)
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -     1.00    -     fidivrs	(%ecx)
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -     1.00    -     fidivrl	(%eax)
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -     1.00    -     ffree	%st(0)
@@ -457,12 +457,12 @@ fyl2xp1
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -     1.00    -     fldln2
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -      -      -     1.00    -     fldpi
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -      -     0.50    -     0.50    -     fldz
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -     fmul	%st(0), %st(1)
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -     fmul	%st(2)
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -     fmul	%st, %st(1)
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -     fmul	%st(2), %st
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -     0.50   0.50    -      -      -     fmuls	(%ecx)
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -     0.50   0.50    -      -      -     fmull	(%eax)
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -     fmulp	%st(1)
-# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -     fmulp	%st(2)
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -     fmulp	%st, %st(1)
+# CHECK-NEXT:  -      -      -      -      -      -      -     0.50   0.50    -      -      -     fmulp	%st, %st(2)
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -     0.50   0.50    -      -      -     fimuls	(%ecx)
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -     0.50   0.50    -      -      -     fimull	(%eax)
 # CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -     fnop
@@ -490,20 +490,20 @@ fyl2xp1
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     frstor	(%eax)
 # CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -     wait
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -     fnsave	(%eax)
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -     fsub	%st(0), %st(1)
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -     fsub	%st(2)
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -     fsub	%st, %st(1)
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -     fsub	%st(2), %st
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -     1.00    -      -      -      -     fsubs	(%ecx)
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -     1.00    -      -      -      -     fsubl	(%eax)
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -     fsubp	%st(1)
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -     fsubp	%st(2)
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -     fsubp	%st, %st(1)
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -     fsubp	%st, %st(2)
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -     1.00    -      -      -      -     fisubs	(%ecx)
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -     1.00    -      -      -      -     fisubl	(%eax)
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -     fsubr	%st(0), %st(1)
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -     fsubr	%st(2)
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -     fsubr	%st, %st(1)
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -     fsubr	%st(2), %st
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -     1.00    -      -      -      -     fsubrs	(%ecx)
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -     1.00    -      -      -      -     fsubrl	(%eax)
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -     fsubrp	%st(1)
-# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -     fsubrp	%st(2)
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -     fsubrp	%st, %st(1)
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -     fsubrp	%st, %st(2)
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -     1.00    -      -      -      -     fisubrs	(%ecx)
 # CHECK-NEXT: 0.50   0.50    -      -      -      -      -     1.00    -      -      -      -     fisubrl	(%eax)
 # CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -     ftst
@@ -512,8 +512,8 @@ fyl2xp1
 # CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -     fucomp	%st(1)
 # CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -     fucomp	%st(3)
 # CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -     fucompp
-# CHECK-NEXT: 0.50   0.50    -      -      -      -      -     0.50    -     0.50    -      -     fucomi	%st(3)
-# CHECK-NEXT: 0.50   0.50    -      -      -      -      -     0.50    -     0.50    -      -     fucompi	%st(3)
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -     0.50    -     0.50    -      -     fucomi	%st(3), %st
+# CHECK-NEXT: 0.50   0.50    -      -      -      -      -     0.50    -     0.50    -      -     fucompi	%st(3), %st
 # CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -     wait
 # CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -     1.00    -     fxam
 # CHECK-NEXT:  -      -      -      -      -      -      -     0.25   0.25   0.25   0.25    -     fxch	%st(1)
diff --git a/test/tools/llvm-nm/AArch64/Inputs/cold-func.ll b/test/tools/llvm-nm/AArch64/Inputs/cold-func.ll
new file mode 100644
index 0000000000000000000000000000000000000000..686e67e8de8acf15b53c824b02297e886907d380
--- /dev/null
+++ b/test/tools/llvm-nm/AArch64/Inputs/cold-func.ll
@@ -0,0 +1,3 @@
+define void @cold_func() cold {
+  ret void
+}
diff --git a/test/tools/llvm-nm/AArch64/macho-cold.test b/test/tools/llvm-nm/AArch64/macho-cold.test
new file mode 100644
index 0000000000000000000000000000000000000000..590713f6a8d604c1cc638fa912d61409e0a3f722
--- /dev/null
+++ b/test/tools/llvm-nm/AArch64/macho-cold.test
@@ -0,0 +1,4 @@
+RUN: llc -O0 -mtriple=aarch64-apple-ios %p/Inputs/cold-func.ll -filetype=obj -o %t.aarch64.o
+RUN: llvm-nm -m %t.aarch64.o | FileCheck %s
+
+CHECK: [cold func] _cold_func
diff --git a/test/tools/llvm-nm/X86/posix-aliases.test b/test/tools/llvm-nm/X86/posix-aliases.test
new file mode 100644
index 0000000000000000000000000000000000000000..2e0999339a206b866de51a1a75fb25bce91abdf2
--- /dev/null
+++ b/test/tools/llvm-nm/X86/posix-aliases.test
@@ -0,0 +1,5 @@
+# RUN: llvm-nm -P %p/Inputs/hello.obj.elf-x86_64 > %t1
+# RUN: llvm-nm -format=posix %p/Inputs/hello.obj.elf-x86_64 > %t2
+# RUN: llvm-nm -portability %p/Inputs/hello.obj.elf-x86_64 > %t3
+# RUN: diff %t1 %t2
+# RUN: diff %t1 %t3
diff --git a/test/tools/llvm-nm/X86/posixELF.test b/test/tools/llvm-nm/X86/posixELF.test
index f66e1befc8fe509fe95fe7f029f5ba65e9e2cd59..810bc9705f97195d3e51455453ad561d112d4097 100644
--- a/test/tools/llvm-nm/X86/posixELF.test
+++ b/test/tools/llvm-nm/X86/posixELF.test
@@ -1,4 +1,4 @@
 # RUN: llvm-nm -P %p/Inputs/hello.obj.elf-x86_64 | FileCheck %s
 
-CHECK: main T 0 0
+CHECK: main T 0 15
 CHECK: puts U 0 0
diff --git a/test/tools/llvm-nm/X86/response-file.test b/test/tools/llvm-nm/X86/response-file.test
index 5c53960056c6d3e1c0fcf33afa7af98eb6ebe454..8f4e2a734b91efef1e2ff7ec2d9ce2b933dda375 100644
--- a/test/tools/llvm-nm/X86/response-file.test
+++ b/test/tools/llvm-nm/X86/response-file.test
@@ -1,5 +1,5 @@
 # RUN: echo "-P %p/Inputs/hello.obj.elf-x86_64" > %t-response
 # RUN: llvm-nm @%t-response | FileCheck %s
 
-CHECK: main T 0 0
+CHECK: main T 0 15
 CHECK: puts U 0 0
diff --git a/test/tools/llvm-nm/X86/size-sort.test b/test/tools/llvm-nm/X86/size-sort.test
new file mode 100644
index 0000000000000000000000000000000000000000..64ea99e35b1f7e4ed4fadfbbfea16e5145080dfd
--- /dev/null
+++ b/test/tools/llvm-nm/X86/size-sort.test
@@ -0,0 +1,12 @@
+# RUN: llvm-nm --size-sort %p/Inputs/hello.obj.elf-x86_64 | FileCheck --check-prefix=SIZE-SORT-NO-ADDR %s
+# RUN: llvm-nm --size-sort -S %p/Inputs/hello.obj.elf-x86_64 | FileCheck --check-prefix=SIZE-SORT-PRINT-ADDR %s
+# RUN: llvm-nm -S %p/Inputs/hello.obj.elf-x86_64 | FileCheck --check-prefix=NO-SIZE-SORT-PRINT-ADDR %s
+
+# SIZE-SORT-NO-ADDR:                  U puts
+# SIZE-SORT-NO-ADDR: 0000000000000015 T main
+
+# SIZE-SORT-PRINT-ADDR:                                   U puts
+# SIZE-SORT-PRINT-ADDR: 0000000000000000 0000000000000015 T main
+
+# NO-SIZE-SORT-PRINT-ADDR: 0000000000000000 0000000000000015 T main
+# NO-SIZE-SORT-PRINT-ADDR:                                   U puts
diff --git a/test/tools/llvm-nm/lit.local.cfg b/test/tools/llvm-nm/lit.local.cfg
index 447a7375519fdb897fdf81aa113255b3503f238e..1fc0bea084cad6faddd14cf0b9c866ee55c67036 100644
--- a/test/tools/llvm-nm/lit.local.cfg
+++ b/test/tools/llvm-nm/lit.local.cfg
@@ -1,4 +1,4 @@
 if not 'X86' in config.root.targets:
     config.unsupported = True
 
-config.suffixes = ['.s', '.test', '.yaml']
+config.suffixes = ['.ll', '.s', '.test', '.yaml']
diff --git a/test/tools/llvm-nm/wasm/exports.yaml b/test/tools/llvm-nm/wasm/exports.yaml
index 55d2b76470a236f2b096143a90adca0d0856db39..756e9c1e051940106dca37e2e2306da378c5946b 100644
--- a/test/tools/llvm-nm/wasm/exports.yaml
+++ b/test/tools/llvm-nm/wasm/exports.yaml
@@ -37,7 +37,7 @@ Sections:
         Content:         '616263'
   - Type:            CUSTOM
     Name:            linking
-    Version:         1
+    Version:         2
     SymbolTable:
        - Index:           0
          Kind:            FUNCTION
@@ -62,5 +62,5 @@ Sections:
         Flags:           [  ]
 
 # CHECK:      00000000 D dexport
-# CHECK-NEXT: 00000000 T fexport
+# CHECK-NEXT: 00000001 T fexport
 # CHECK-NEXT: 00000000 D gexport
diff --git a/test/tools/llvm-nm/wasm/extern-only.ll b/test/tools/llvm-nm/wasm/extern-only.ll
new file mode 100644
index 0000000000000000000000000000000000000000..46fecba047341ec87f2070ed7a4607b29f817665
--- /dev/null
+++ b/test/tools/llvm-nm/wasm/extern-only.ll
@@ -0,0 +1,23 @@
+; RUN: llc -filetype=obj -mtriple=wasm32-unknown-unknown -o %t.o %s
+; RUN: llvm-nm --extern-only %t.o | FileCheck %s
+
+; Verity that hidden symbols are listed even when --extern-only is passed
+
+define hidden i32 @foo() {
+entry:
+  ret i32 42
+}
+
+define i32 @bar() {
+entry:
+  ret i32 43
+}
+
+define internal i32 @baz() {
+entry:
+  ret i32 44
+}
+
+; CHECK: 00000006 T bar
+; CHECK-NOT: baz
+; CHECK: 00000001 T foo
diff --git a/test/tools/llvm-nm/wasm/imports.yaml b/test/tools/llvm-nm/wasm/imports.yaml
index 9696972de4dcbc2505b85f2548dcfb5b0a5ad844..2ea0d0f13fe34fd52fda8a9483a7ac8616762477 100644
--- a/test/tools/llvm-nm/wasm/imports.yaml
+++ b/test/tools/llvm-nm/wasm/imports.yaml
@@ -25,7 +25,7 @@ Sections:
         GlobalMutable:   false
   - Type:            CUSTOM
     Name:            linking
-    Version:         1
+    Version:         2
     SymbolTable:
        - Index:           0
          Kind:            FUNCTION
diff --git a/test/tools/llvm-nm/wasm/lit.local.cfg b/test/tools/llvm-nm/wasm/lit.local.cfg
new file mode 100644
index 0000000000000000000000000000000000000000..743473517cd0f94fe25819c806ad10f2cbc5a459
--- /dev/null
+++ b/test/tools/llvm-nm/wasm/lit.local.cfg
@@ -0,0 +1,2 @@
+if not 'WebAssembly' in config.root.targets:
+    config.unsupported = True
diff --git a/test/tools/llvm-nm/wasm/weak-symbols.yaml b/test/tools/llvm-nm/wasm/weak-symbols.yaml
index caa981df43692dbc80b53032736f77d1fd36170b..36711b17b981d783cc9f200804e895c27fce2229 100644
--- a/test/tools/llvm-nm/wasm/weak-symbols.yaml
+++ b/test/tools/llvm-nm/wasm/weak-symbols.yaml
@@ -43,7 +43,7 @@ Sections:
         Content:         '616263'
   - Type:            CUSTOM
     Name:            linking
-    Version:         1
+    Version:         2
     SymbolTable:
        - Index:           0
          Kind:            DATA
diff --git a/test/tools/llvm-objcopy/COFF/Inputs/bigobj.o.gz b/test/tools/llvm-objcopy/COFF/Inputs/bigobj.o.gz
new file mode 100644
index 0000000000000000000000000000000000000000..6435f4785ff76e0c6bca12f3e57bc6ad8888bece
Binary files /dev/null and b/test/tools/llvm-objcopy/COFF/Inputs/bigobj.o.gz differ
diff --git a/test/tools/llvm-objcopy/COFF/Inputs/discard-locals.yaml b/test/tools/llvm-objcopy/COFF/Inputs/discard-locals.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..8b34e8fffed02cfb30ac4195003eff95bc692e3f
--- /dev/null
+++ b/test/tools/llvm-objcopy/COFF/Inputs/discard-locals.yaml
@@ -0,0 +1,60 @@
+--- !COFF
+header:          
+  Machine:         IMAGE_FILE_MACHINE_AMD64
+  Characteristics: [  ]
+sections:        
+  - Name:            .text
+    Characteristics: [  ]
+    Alignment:       4
+    SectionData:     E800000000E800000000C3C3C3
+    Relocations:     
+      - VirtualAddress:  1
+        SymbolName:      local_referenced
+        Type:            IMAGE_REL_AMD64_REL32
+      - VirtualAddress:  5
+        SymbolName:      external_undefined
+        Type:            IMAGE_REL_AMD64_REL32
+symbols:         
+  - Name:            external
+    Value:           0
+    SectionNumber:   1
+    SimpleType:      IMAGE_SYM_TYPE_NULL
+    ComplexType:     IMAGE_SYM_DTYPE_NULL
+    StorageClass:    IMAGE_SYM_CLASS_EXTERNAL
+  - Name:            external_undefined
+    Value:           0
+    SectionNumber:   0
+    SimpleType:      IMAGE_SYM_TYPE_NULL
+    ComplexType:     IMAGE_SYM_DTYPE_NULL
+    StorageClass:    IMAGE_SYM_CLASS_EXTERNAL
+  - Name:            external_undefined_unreferenced
+    Value:           0
+    SectionNumber:   0
+    SimpleType:      IMAGE_SYM_TYPE_NULL
+    ComplexType:     IMAGE_SYM_DTYPE_NULL
+    StorageClass:    IMAGE_SYM_CLASS_EXTERNAL
+  - Name:            local_unreferenced
+    Value:           6
+    SectionNumber:   1
+    SimpleType:      IMAGE_SYM_TYPE_NULL
+    ComplexType:     IMAGE_SYM_DTYPE_NULL
+    StorageClass:    IMAGE_SYM_CLASS_STATIC
+  - Name:            local_referenced
+    Value:           7
+    SectionNumber:   1
+    SimpleType:      IMAGE_SYM_TYPE_NULL
+    ComplexType:     IMAGE_SYM_DTYPE_NULL
+    StorageClass:    IMAGE_SYM_CLASS_STATIC
+  - Name:            local_undefined_unreferenced
+    Value:           0
+    SectionNumber:   0
+    SimpleType:      IMAGE_SYM_TYPE_NULL
+    ComplexType:     IMAGE_SYM_DTYPE_NULL
+    StorageClass:    IMAGE_SYM_CLASS_STATIC
+  - Name:            '@feat.00'
+    Value:           0
+    SectionNumber:   -1
+    SimpleType:      IMAGE_SYM_TYPE_NULL
+    ComplexType:     IMAGE_SYM_DTYPE_NULL
+    StorageClass:    IMAGE_SYM_CLASS_STATIC
+...
diff --git a/test/tools/llvm-objcopy/COFF/Inputs/no-symbols.yaml b/test/tools/llvm-objcopy/COFF/Inputs/no-symbols.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..db8aeb63d5eb550b2f11e11de01c53f32726683b
--- /dev/null
+++ b/test/tools/llvm-objcopy/COFF/Inputs/no-symbols.yaml
@@ -0,0 +1,11 @@
+--- !COFF
+header:          
+  Machine:         IMAGE_FILE_MACHINE_AMD64
+  Characteristics: [  ]
+sections:        
+  - Name:            .text
+    Characteristics: [  ]
+    Alignment:       4
+    SectionData:     E800000000C3C3C3
+symbols:         
+...
diff --git a/test/tools/llvm-objcopy/COFF/Inputs/only-keep-sections.yaml b/test/tools/llvm-objcopy/COFF/Inputs/only-keep-sections.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..b5437e107634ff38af51e0e8a2b336fc552be154
--- /dev/null
+++ b/test/tools/llvm-objcopy/COFF/Inputs/only-keep-sections.yaml
@@ -0,0 +1,77 @@
+--- !COFF
+OptionalHeader:  
+  AddressOfEntryPoint: 4144
+  ImageBase:       1073741824
+  SectionAlignment: 4096
+  FileAlignment:   512
+  MajorOperatingSystemVersion: 6
+  MinorOperatingSystemVersion: 0
+  MajorImageVersion: 0
+  MinorImageVersion: 0
+  MajorSubsystemVersion: 6
+  MinorSubsystemVersion: 0
+  Subsystem:       IMAGE_SUBSYSTEM_WINDOWS_CUI
+  DLLCharacteristics: [  ]
+  SizeOfStackReserve: 1048576
+  SizeOfStackCommit: 4096
+  SizeOfHeapReserve: 1048576
+  SizeOfHeapCommit: 4096
+header:          
+  Machine:         IMAGE_FILE_MACHINE_AMD64
+  Characteristics: [  ]
+sections:        
+  - Name:            .text
+    Characteristics: [ IMAGE_SCN_CNT_CODE ]
+    VirtualAddress:  4096
+    VirtualSize:     4
+    SectionData:     C3C3C3C3
+  - Name:            .rdata
+    Characteristics: [ IMAGE_SCN_CNT_INITIALIZED_DATA ]
+    VirtualAddress:  8192
+    VirtualSize:     4
+    SectionData:     2A000000
+  - Name:            .buildid
+    Characteristics: [ IMAGE_SCN_CNT_INITIALIZED_DATA ]
+    VirtualAddress:  12288
+    VirtualSize:     4
+    SectionData:     2B000000
+  - Name:            .reloc
+    Characteristics: [ IMAGE_SCN_CNT_INITIALIZED_DATA, IMAGE_SCN_MEM_DISCARDABLE ]
+    VirtualAddress:  16384
+    VirtualSize:     4
+    SectionData:     2C000000
+  - Name:            .debug_discardable
+    Characteristics: [ IMAGE_SCN_CNT_INITIALIZED_DATA, IMAGE_SCN_MEM_DISCARDABLE ]
+    VirtualAddress:  20480
+    VirtualSize:     4
+    SectionData:     2D000000
+  - Name:            .debug_undiscardable
+    Characteristics: [ IMAGE_SCN_CNT_INITIALIZED_DATA ]
+    VirtualAddress:  24576
+    VirtualSize:     4
+    SectionData:     2E000000
+  - Name:            .unflagged
+    Characteristics: [  ]
+    VirtualAddress:  28672
+    VirtualSize:     4
+    SectionData:     2F000000
+symbols:         
+  - Name:            main
+    Value:           2
+    SectionNumber:   1
+    SimpleType:      IMAGE_SYM_TYPE_NULL
+    ComplexType:     IMAGE_SYM_DTYPE_FUNCTION
+    StorageClass:    IMAGE_SYM_CLASS_EXTERNAL
+  - Name:            debug_discardable_sym
+    Value:           0
+    SectionNumber:   5
+    SimpleType:      IMAGE_SYM_TYPE_NULL
+    ComplexType:     IMAGE_SYM_DTYPE_FUNCTION
+    StorageClass:    IMAGE_SYM_CLASS_EXTERNAL
+  - Name:            debug_undiscardable_sym
+    Value:           0
+    SectionNumber:   6
+    SimpleType:      IMAGE_SYM_TYPE_NULL
+    ComplexType:     IMAGE_SYM_DTYPE_FUNCTION
+    StorageClass:    IMAGE_SYM_CLASS_EXTERNAL
+...
diff --git a/test/tools/llvm-objcopy/COFF/Inputs/strip-symbols.yaml b/test/tools/llvm-objcopy/COFF/Inputs/strip-symbols.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..1d652691198c4ecf8a32c10466ee17b802dc1cf7
--- /dev/null
+++ b/test/tools/llvm-objcopy/COFF/Inputs/strip-symbols.yaml
@@ -0,0 +1,53 @@
+--- !COFF
+header:          
+  Machine:         IMAGE_FILE_MACHINE_AMD64
+  Characteristics: [  ]
+sections:        
+  - Name:            .text
+    Characteristics: [  ]
+    Alignment:       4
+    SectionData:     488B0500000000488B0500000000488B0500000000
+    Relocations:     
+      - VirtualAddress:  3
+        SymbolTableIndex: 0
+        Type:            IMAGE_REL_AMD64_REL32
+      - VirtualAddress:  10
+        SymbolTableIndex: 1
+        Type:            IMAGE_REL_AMD64_REL32
+      - VirtualAddress:  17
+        SymbolName:      foo
+        Type:            IMAGE_REL_AMD64_REL32
+  - Name:            .rdata
+    Characteristics: [  ]
+    Alignment:       1
+    SectionData:     '00'
+  - Name:            .rdata
+    Characteristics: [  ]
+    Alignment:       1
+    SectionData:     '01'
+symbols:         
+  - Name:            .rdata
+    Value:           0
+    SectionNumber:   2
+    SimpleType:      IMAGE_SYM_TYPE_NULL
+    ComplexType:     IMAGE_SYM_DTYPE_NULL
+    StorageClass:    IMAGE_SYM_CLASS_STATIC
+  - Name:            .rdata
+    Value:           0
+    SectionNumber:   3
+    SimpleType:      IMAGE_SYM_TYPE_NULL
+    ComplexType:     IMAGE_SYM_DTYPE_NULL
+    StorageClass:    IMAGE_SYM_CLASS_STATIC
+  - Name:            mainfunc
+    Value:           0
+    SectionNumber:   1
+    SimpleType:      IMAGE_SYM_TYPE_NULL
+    ComplexType:     IMAGE_SYM_DTYPE_NULL
+    StorageClass:    IMAGE_SYM_CLASS_EXTERNAL
+  - Name:            foo
+    Value:           0
+    SectionNumber:   3
+    SimpleType:      IMAGE_SYM_TYPE_NULL
+    ComplexType:     IMAGE_SYM_DTYPE_NULL
+    StorageClass:    IMAGE_SYM_CLASS_EXTERNAL
+...
diff --git a/test/tools/llvm-objcopy/COFF/add-gnu-debuglink.test b/test/tools/llvm-objcopy/COFF/add-gnu-debuglink.test
new file mode 100644
index 0000000000000000000000000000000000000000..fd47d79739e64914d66c0b22f3adc29f7def6f5d
--- /dev/null
+++ b/test/tools/llvm-objcopy/COFF/add-gnu-debuglink.test
@@ -0,0 +1,42 @@
+RUN: yaml2obj %p/Inputs/x86_64-exe.yaml > %t.in123.exe
+
+# Using a debuglink filename with a length that is a multiple of 4, to
+# showcase padding in CONTENTS below.
+
+RUN: llvm-objcopy --add-gnu-debuglink=%t.in123.exe %t.in123.exe %t.out.exe
+RUN: llvm-readobj -sections %t.out.exe | FileCheck %s --check-prefix=SECTIONS
+RUN: llvm-objdump -s %t.out.exe | FileCheck %s --check-prefix=CONTENTS
+
+# Show the last of the preexisting sections, which is used for choosing
+# a virtual address for the generated one.
+
+SECTIONS:        Section {
+SECTIONS:          Number: 4
+SECTIONS-NEXT:     Name: .pdata
+SECTIONS-NEXT:     VirtualSize: 0x18
+SECTIONS-NEXT:     VirtualAddress: 0x4000
+SECTIONS-NEXT:     RawDataSize: 512
+SECTIONS:        Section {
+SECTIONS-NEXT:     Number: 5
+SECTIONS-NEXT:     Name: .gnu_debuglink
+SECTIONS-NEXT:     VirtualSize: 0x2C
+SECTIONS-NEXT:     VirtualAddress: 0x5000
+SECTIONS-NEXT:     RawDataSize: 512
+SECTIONS-NEXT:     PointerToRawData:
+SECTIONS-NEXT:     PointerToRelocations:
+SECTIONS-NEXT:     PointerToLineNumbers:
+SECTIONS-NEXT:     RelocationCount:
+SECTIONS-NEXT:     LineNumberCount:
+SECTIONS-NEXT:     Characteristics [ (0x42000040)
+SECTIONS-NEXT:       IMAGE_SCN_CNT_INITIALIZED_DATA (0x40)
+SECTIONS-NEXT:       IMAGE_SCN_MEM_DISCARDABLE (0x2000000)
+SECTIONS-NEXT:       IMAGE_SCN_MEM_READ (0x40000000)
+SECTIONS-NEXT:     ]
+
+# Note: The last 4 bytes here are the crc of the referenced file - if the
+# yaml2obj generated file changes, this crc changes.
+
+CONTENTS: Contents of section .gnu_debuglink:
+CONTENTS:  40005000 6164642d 676e752d 64656275 676c696e  add-gnu-debuglin
+CONTENTS:  40005010 6b2e7465 73742e74 6d702e69 6e313233  k.test.tmp.in123
+CONTENTS:  40005020 2e657865 00000000 7929adc3           .exe
diff --git a/test/tools/llvm-objcopy/COFF/basic-copy.test b/test/tools/llvm-objcopy/COFF/basic-copy.test
index ddd12dcaf2fb8f3d702bfdedd14a76eb06b6df9c..ecdf430faf19ace4a1b4583b4489fd4e94d136a1 100644
--- a/test/tools/llvm-objcopy/COFF/basic-copy.test
+++ b/test/tools/llvm-objcopy/COFF/basic-copy.test
@@ -40,3 +40,9 @@ RUN: llvm-objcopy %t.x86_64.exe %t.x86_64-copy.exe
 RUN: obj2yaml %t.x86_64.exe > %t.x86_64.exe.yaml
 RUN: obj2yaml %t.x86_64-copy.exe > %t.x86_64-copy.exe.yaml
 RUN: cmp %t.x86_64.exe.yaml %t.x86_64-copy.exe.yaml
+
+RUN: yaml2obj %p/Inputs/no-symbols.yaml > %t.no-symbols.o
+RUN: llvm-objcopy %t.no-symbols.o %t.no-symbols-copy.o
+RUN: obj2yaml %t.no-symbols.o > %t.no-symbols.o.yaml
+RUN: obj2yaml %t.no-symbols-copy.o > %t.no-symbols-copy.o.yaml
+RUN: cmp %t.no-symbols.o.yaml %t.no-symbols-copy.o.yaml
diff --git a/test/tools/llvm-objcopy/COFF/bigobj.test b/test/tools/llvm-objcopy/COFF/bigobj.test
new file mode 100644
index 0000000000000000000000000000000000000000..17968f12b8aed58b14a2bcdc1463881d382c1ab9
--- /dev/null
+++ b/test/tools/llvm-objcopy/COFF/bigobj.test
@@ -0,0 +1,35 @@
+RUN: %python %p/../Inputs/ungzip.py %p/Inputs/bigobj.o.gz > %t.in.o
+
+RUN: llvm-objdump -t %t.in.o | FileCheck %s --check-prefixes=SYMBOLS,SYMBOLS-BIG,SYMBOLS-ORIG
+
+# Do a plain copy, to check that section numbers in symbols referring
+# to sections outside of the small object format are handled correctly.
+RUN: llvm-objcopy -R '.text$4' %t.in.o %t.small.o
+RUN: llvm-objdump -t %t.in.o | FileCheck %s --check-prefixes=SYMBOLS,SYMBOLS-BIG,SYMBOLS-ORIG
+
+# Remove a section, making the section count fit into a small object.
+RUN: llvm-objcopy -R '.text$4' %t.in.o %t.small.o
+RUN: llvm-objdump -t %t.small.o | FileCheck %s --check-prefixes=SYMBOLS,SYMBOLS-SMALL,SYMBOLS-REMOVED-SMALL
+
+# Add a .gnu_debuglink section, forcing the object back to big format.
+RUN: llvm-objcopy --add-gnu-debuglink=%t.in.o %t.small.o %t.big.o
+ llvm-objdump -t %t.big.o | FileCheck %s --check-prefixes=SYMBOLS,SYMBOLS-BIG,SYMBOLS-REMOVED-BIG
+
+# In big object format, the .file symbol occupies one symbol table entry for
+# the auxillary data, but needs two entries in the small format, forcing the
+# raw symbol indices of later symbols to change.
+SYMBOLS:            SYMBOL TABLE:
+SYMBOLS-NEXT:       [ 0]{{.*}} (nx 1) {{.*}} .text
+SYMBOLS-NEXT:       AUX scnlen
+SYMBOLS-SMALL-NEXT: [ 2]{{.*}} (nx 2) {{.*}} .file
+SYMBOLS-BIG-NEXT:   [ 2]{{.*}} (nx 1) {{.*}} .file
+SYMBOLS-NEXT:       AUX abcdefghijklmnopqrs
+SYMBOLS-SMALL-NEXT: [ 5]{{.*}} (nx 0) {{.*}} foo
+SYMBOLS-BIG-NEXT:   [ 4]{{.*}} (nx 0) {{.*}} foo
+
+# Check that the section numbers outside of signed 16 bit int range
+# are represented properly. After removing one section, the section
+# numbers decrease.
+SYMBOLS-ORIG:          [ 5](sec 65280){{.*}} symbol65280
+SYMBOLS-REMOVED-SMALL: [ 6](sec 65279){{.*}} symbol65280
+SYMBOLS-REMOVED-BIG:   [ 5](sec 65279){{.*}} symbol65280
diff --git a/test/tools/llvm-objcopy/COFF/discard-all.test b/test/tools/llvm-objcopy/COFF/discard-all.test
new file mode 100644
index 0000000000000000000000000000000000000000..5d7d5cef2f36094a54601de40d45fa2e502ff99e
--- /dev/null
+++ b/test/tools/llvm-objcopy/COFF/discard-all.test
@@ -0,0 +1,27 @@
+RUN: yaml2obj %p/Inputs/discard-locals.yaml > %t.in.o
+
+RUN: llvm-objdump -t %t.in.o | FileCheck %s --check-prefixes=SYMBOLS,SYMBOLS-PRE
+
+RUN: llvm-objcopy --discard-all %t.in.o %t.out.o
+RUN: llvm-objdump -t %t.out.o | FileCheck %s --check-prefixes=SYMBOLS
+
+RUN: llvm-objcopy -x %t.in.o %t.out-x.o
+RUN: cmp %t.out.o %t.out-x.o
+
+RUN: cp %t.in.o %t.strip-x.o
+RUN: llvm-strip -x %t.strip-x.o
+RUN: cmp %t.out.o %t.strip-x.o
+
+RUN: cp %t.in.o %t.strip-discard-all.o
+RUN: llvm-strip --discard-all %t.strip-discard-all.o
+RUN: cmp %t.out.o %t.strip-discard-all.o
+
+SYMBOLS: SYMBOL TABLE:
+SYMBOLS-NEXT: external
+SYMBOLS-NEXT: external_undefined
+SYMBOLS-NEXT: external_undefined_unreferenced
+SYMBOLS-PRE-NEXT: local_unreferenced
+SYMBOLS-NEXT: local_referenced
+SYMBOLS-NEXT: local_undefined_unreferenced
+SYMBOLS-PRE-NEXT: @feat.00
+SYMBOLS-EMPTY:
diff --git a/test/tools/llvm-objcopy/COFF/only-keep-debug.test b/test/tools/llvm-objcopy/COFF/only-keep-debug.test
new file mode 100644
index 0000000000000000000000000000000000000000..5518d4000fcfaf5a0990a9c6563b8aa86435465c
--- /dev/null
+++ b/test/tools/llvm-objcopy/COFF/only-keep-debug.test
@@ -0,0 +1,58 @@
+RUN: yaml2obj %p/Inputs/only-keep-sections.yaml > %t.in.exe
+
+RUN: llvm-objcopy --only-keep-debug %t.in.exe %t.out.exe
+RUN: llvm-readobj --sections %t.out.exe | FileCheck %s --check-prefix=SECTIONS
+RUN: llvm-objdump -t %t.out.exe | FileCheck %s --check-prefix=SYMBOLS
+
+Check that all non-debug/buildid sections with IMAGE_SCN_CNT_CODE
+or IMAGE_SCN_CNT_INITIALIZED_DATA are truncated, and no others.
+
+SECTIONS:      Sections [
+SECTIONS-NEXT:  Section {
+SECTIONS-NEXT:   Number: 1
+SECTIONS-NEXT:   Name: .text
+SECTIONS-NEXT:   VirtualSize: 0x4
+SECTIONS-NEXT:   VirtualAddress:
+SECTIONS-NEXT:   RawDataSize: 0
+SECTIONS:       Section {
+SECTIONS-NEXT:   Number: 2
+SECTIONS-NEXT:   Name: .rdata
+SECTIONS-NEXT:   VirtualSize: 0x4
+SECTIONS-NEXT:   VirtualAddress:
+SECTIONS-NEXT:   RawDataSize: 0
+SECTIONS:       Section {
+SECTIONS-NEXT:   Number: 3
+SECTIONS-NEXT:   Name: .buildid
+SECTIONS-NEXT:   VirtualSize: 0x4
+SECTIONS-NEXT:   VirtualAddress:
+SECTIONS-NEXT:   RawDataSize: 512
+SECTIONS:       Section {
+SECTIONS-NEXT:   Number: 4
+SECTIONS-NEXT:   Name: .reloc
+SECTIONS-NEXT:   VirtualSize: 0x4
+SECTIONS-NEXT:   VirtualAddress:
+SECTIONS-NEXT:   RawDataSize: 0
+SECTIONS:       Section {
+SECTIONS-NEXT:   Number: 5
+SECTIONS-NEXT:   Name: .debug_discardable
+SECTIONS-NEXT:   VirtualSize: 0x4
+SECTIONS-NEXT:   VirtualAddress:
+SECTIONS-NEXT:   RawDataSize: 512
+SECTIONS:       Section {
+SECTIONS-NEXT:   Number: 6
+SECTIONS-NEXT:   Name: .debug_undiscardable
+SECTIONS-NEXT:   VirtualSize: 0x4
+SECTIONS-NEXT:   VirtualAddress:
+SECTIONS-NEXT:   RawDataSize: 512
+SECTIONS:       Section {
+SECTIONS-NEXT:   Number: 7
+SECTIONS-NEXT:   Name: .unflagged
+SECTIONS-NEXT:   VirtualSize: 0x4
+SECTIONS-NEXT:   VirtualAddress:
+SECTIONS-NEXT:   RawDataSize: 512
+
+SYMBOLS:      SYMBOL TABLE:
+SYMBOLS-NEXT: main
+SYMBOLS-NEXT: debug_discardable_sym
+SYMBOLS-NEXT: debug_undiscardable_sym
+SYMBOLS-EMPTY:
diff --git a/test/tools/llvm-objcopy/COFF/only-section.test b/test/tools/llvm-objcopy/COFF/only-section.test
new file mode 100644
index 0000000000000000000000000000000000000000..42492ed80ff95760cae718f9cb12f6efb80a025f
--- /dev/null
+++ b/test/tools/llvm-objcopy/COFF/only-section.test
@@ -0,0 +1,21 @@
+RUN: yaml2obj %p/Inputs/only-keep-sections.yaml > %t.in.exe
+
+RUN: llvm-objcopy --only-section .debug_discardable %t.in.exe %t.out.exe
+RUN: llvm-objdump --section-headers -t %t.out.exe | FileCheck %s --check-prefixes=SECTIONS,SECTIONS-DEBUG,SYMBOLS,SYMBOLS-DEBUG
+
+Adding another section stripping option makes it return the intersection of
+kept sections - in this case keeping only .text.
+
+RUN: llvm-objcopy --only-section .debug_discardable --only-section .text --strip-debug %t.in.exe %t.combination.exe
+RUN: llvm-objdump --section-headers -t %t.combination.exe | FileCheck %s --check-prefixes=SECTIONS,SECTIONS-TEXT,SYMBOLS,SYMBOLS-TEXT
+
+SECTIONS:            Sections:
+SECTIONS-NEXT:       Idx Name
+SECTIONS-DEBUG-NEXT: .debug_discardable
+SECTIONS-TEXT-NEXT:  .text
+SECTIONS-EMPTY:
+
+SYMBOLS:            SYMBOL TABLE:
+SYMBOLS-DEBUG-NEXT: debug_discardable_sym
+SYMBOLS-TEXT-NEXT:  main
+SYMBOLS-EMPTY:
diff --git a/test/tools/llvm-objcopy/COFF/patch-debug-dir.test b/test/tools/llvm-objcopy/COFF/patch-debug-dir.test
new file mode 100644
index 0000000000000000000000000000000000000000..82af18311c8ae919e1b7a4a0aa9a795dbf0f027c
--- /dev/null
+++ b/test/tools/llvm-objcopy/COFF/patch-debug-dir.test
@@ -0,0 +1,64 @@
+# RUN: yaml2obj %s > %t.in.exe
+
+# RUN: llvm-objdump -s %t.in.exe | FileCheck %s --check-prefixes=CONTENTS,CONTENTS-PRE
+# RUN: llvm-readobj --sections %t.in.exe | FileCheck %s --check-prefixes=SECTIONS,SECTIONS-PRE
+# RUN: llvm-objcopy -R .rdata %t.in.exe %t.out.exe
+# RUN: llvm-objdump -s %t.out.exe | FileCheck %s --check-prefixes=CONTENTS,CONTENTS-POST
+# RUN: llvm-readobj --sections %t.out.exe | FileCheck %s --check-prefixes=SECTIONS,SECTIONS-POST
+
+# CONTENTS:           Contents of section .buildid:
+# CONTENTS-NEXT:       {{.*}} 00000000 42ee405c 00000000 02000000
+# CONTENTS-PRE-NEXT:   {{.*}} 19000000 1c300000 1c080000 52534453
+# CONTENTS-POST-NEXT:  {{.*}} 19000000 1c300000 1c040000 52534453
+# CONTENTS-NEXT:       {{.*}} c1330757 2839a337 4c4c4420 5044422e
+# CONTENTS-NEXT:       {{.*}} 01000000 00
+
+# SECTIONS:           Name: .buildid
+# SECTIONS-NEXT:      VirtualSize:
+# SECTIONS-NEXT:      VirtualAddress:
+# SECTIONS-NEXT:      RawDataSize:
+# SECTIONS-NEXT-PRE:  PointerToRawData: 0x800
+# SECTIONS-NEXT-POST: PointerToRawData: 0x400
+
+--- !COFF
+OptionalHeader:  
+  AddressOfEntryPoint: 4096
+  ImageBase:       1073741824
+  SectionAlignment: 4096
+  FileAlignment:   512
+  MajorOperatingSystemVersion: 6
+  MinorOperatingSystemVersion: 0
+  MajorImageVersion: 0
+  MinorImageVersion: 0
+  MajorSubsystemVersion: 6
+  MinorSubsystemVersion: 0
+  Subsystem:       IMAGE_SUBSYSTEM_WINDOWS_CUI
+  DLLCharacteristics: [  ]
+  SizeOfStackReserve: 1048576
+  SizeOfStackCommit: 4096
+  SizeOfHeapReserve: 1048576
+  SizeOfHeapCommit: 4096
+  Debug:           
+    RelativeVirtualAddress: 12288
+    Size:            28
+header:          
+  Machine:         IMAGE_FILE_MACHINE_AMD64
+  Characteristics: [  ]
+sections:        
+  - Name:            .text
+    Characteristics: [  ]
+    VirtualAddress:  4096
+    VirtualSize:     16
+    SectionData:     C3909090909090909090909090909090
+  - Name:            .rdata
+    Characteristics: [  ]
+    VirtualAddress:  8192
+    VirtualSize:     32
+    SectionData:     FFFFFFFF00000000FFFFFFFF00000000
+  - Name:            .buildid
+    Characteristics: [  ]
+    VirtualAddress:  12288
+    VirtualSize:     53
+    SectionData:     0000000042EE405C0000000002000000190000001C3000001C08000052534453C13307572839A3374C4C44205044422E0100000000
+symbols:         
+...
diff --git a/test/tools/llvm-objcopy/COFF/remove-section.test b/test/tools/llvm-objcopy/COFF/remove-section.test
new file mode 100644
index 0000000000000000000000000000000000000000..6dc8f6a6c2e36ec235e66c92a8dcf83ea60d61e5
--- /dev/null
+++ b/test/tools/llvm-objcopy/COFF/remove-section.test
@@ -0,0 +1,210 @@
+# RUN: yaml2obj %s > %t.in.o
+#
+# RUN: llvm-objdump -section-headers %t.in.o | FileCheck %s --check-prefixes=SECTIONS-PRE
+# RUN: llvm-objdump -t %t.in.o | FileCheck %s --check-prefixes=SYMBOLS-PRE
+#
+# RUN: llvm-objcopy -R .bss %t.in.o %t.remove-bss.o
+# RUN: llvm-objdump -section-headers %t.remove-bss.o | FileCheck %s --check-prefix=SECTIONS-REMOVE-BSS
+# RUN: llvm-objdump -t %t.remove-bss.o | FileCheck %s --check-prefix=SYMBOLS-REMOVE-BSS
+#
+# RUN: llvm-objcopy --remove-section .bss %t.in.o %t.cmp.o
+# RUN: cmp %t.remove-bss.o %t.cmp.o
+#
+# RUN: llvm-objcopy -R .text %t.in.o %t.remove-text.o
+# RUN: llvm-objdump -section-headers %t.remove-text.o | FileCheck %s --check-prefix=SECTIONS-REMOVE-TEXT
+# RUN: llvm-objdump -t %t.remove-text.o | FileCheck %s --check-prefix=SYMBOLS-REMOVE-TEXT
+#
+# RUN: not llvm-objcopy -R .comdat %t.in.o %t.remove-comdat.o 2>&1 | FileCheck %s --check-prefix=ERROR-RELOC
+#
+# RUN: llvm-objcopy -R .text -R .comdat %t.in.o %t.remove-text-comdat.o
+# RUN: llvm-objdump -section-headers %t.remove-text-comdat.o | FileCheck %s --check-prefix=SECTIONS-REMOVE-TEXT-COMDAT
+# RUN: llvm-objdump -t %t.remove-text-comdat.o | FileCheck %s --check-prefix=SYMBOLS-REMOVE-TEXT-COMDAT
+#
+#
+# SECTIONS-PRE: Sections:
+# SECTIONS-PRE-NEXT: Idx Name
+# SECTIONS-PRE-NEXT: 0 .text
+# SECTIONS-PRE-NEXT: 1 .bss
+# SECTIONS-PRE-NEXT: 2 .comdat
+# SECTIONS-PRE-NEXT: 3 .associative
+# SECTIONS-PRE-EMPTY:
+#
+# SYMBOLS-PRE: SYMBOL TABLE:
+# SYMBOLS-PRE-NEXT: {{.*}}(sec -1){{.*}} @feat.00
+# SYMBOLS-PRE-NEXT: {{.*}}(sec 1){{.*}} .text
+# SYMBOLS-PRE-NEXT: AUX scnlen {{.*}} assoc 1 comdat 0
+# SYMBOLS-PRE-NEXT: {{.*}}(sec 2){{.*}} .bss
+# SYMBOLS-PRE-NEXT: AUX scnlen {{.*}} assoc 2 comdat 0
+# SYMBOLS-PRE-NEXT: {{.*}}(sec 4){{.*}} .associative
+# SYMBOLS-PRE-NEXT: AUX scnlen {{.*}} assoc 3 comdat 5
+# SYMBOLS-PRE-NEXT: {{.*}}(sec 3){{.*}} .comdat
+# SYMBOLS-PRE-NEXT: AUX scnlen {{.*}} assoc 3 comdat 2
+# SYMBOLS-PRE-NEXT: {{.*}}(sec 3){{.*}} foo
+# SYMBOLS-PRE-NEXT: {{.*}}(sec 1){{.*}} main
+# SYMBOLS-PRE-EMPTY:
+#
+#
+# Removing the .bss section removes one symbol and its aux symbol,
+# and updates the section indices in symbols pointing to later
+# symbols, including the aux section defintitions.
+#
+# Testing that the absolute symbol @feat.00 survives the section number
+# mangling.
+#
+# SECTIONS-REMOVE-BSS: Sections:
+# SECTIONS-REMOVE-BSS-NEXT: Idx Name
+# SECTIONS-REMOVE-BSS-NEXT: 0 .text
+# SECTIONS-REMOVE-BSS-NEXT: 1 .comdat
+# SECTIONS-REMOVE-BSS-NEXT: 2 .associative
+# SECTIONS-REMOVE-BSS-EMPTY:
+#
+# SYMBOLS-REMOVE-BSS: SYMBOL TABLE:
+# SYMBOLS-REMOVE-BSS-NEXT: {{.*}}(sec -1){{.*}} @feat.00
+# SYMBOLS-REMOVE-BSS-NEXT: {{.*}}(sec 1){{.*}} .text
+# SYMBOLS-REMOVE-BSS-NEXT: AUX scnlen {{.*}} assoc 1 comdat 0
+# SYMBOLS-REMOVE-BSS-NEXT: {{.*}}(sec 3){{.*}} .associative
+# SYMBOLS-REMOVE-BSS-NEXT: AUX scnlen {{.*}} assoc 2 comdat 5
+# SYMBOLS-REMOVE-BSS-NEXT: {{.*}}(sec 2){{.*}} .comdat
+# SYMBOLS-REMOVE-BSS-NEXT: AUX scnlen {{.*}} assoc 2 comdat 2
+# SYMBOLS-REMOVE-BSS-NEXT: {{.*}}(sec 2){{.*}} foo
+# SYMBOLS-REMOVE-BSS-NEXT: {{.*}}(sec 1){{.*}} main
+# SYMBOLS-REMOVE-BSS-EMPTY:
+#
+#
+# Removing the .text section is ok and just removes the external symbol
+# referring to it.
+#
+# SECTIONS-REMOVE-TEXT: Sections:
+# SECTIONS-REMOVE-TEXT-NEXT: Idx Name
+# SECTIONS-REMOVE-TEXT-NEXT: 0 .bss
+# SECTIONS-REMOVE-TEXT-NEXT: 1 .comdat
+# SECTIONS-REMOVE-TEXT-NEXT: 2 .associative
+# SECTIONS-REMOVE-TEXT-EMPTY:
+#
+# SYMBOLS-REMOVE-TEXT: SYMBOL TABLE:
+# SYMBOLS-REMOVE-TEXT-NEXT: {{.*}}(sec -1){{.*}} @feat.00
+# SYMBOLS-REMOVE-TEXT-NEXT: {{.*}}(sec 1){{.*}} .bss
+# SYMBOLS-REMOVE-TEXT-NEXT: AUX scnlen {{.*}} assoc 1 comdat 0
+# SYMBOLS-REMOVE-TEXT-NEXT: {{.*}}(sec 3){{.*}} .associative
+# SYMBOLS-REMOVE-TEXT-NEXT: AUX scnlen {{.*}} assoc 2 comdat 5
+# SYMBOLS-REMOVE-TEXT-NEXT: {{.*}}(sec 2){{.*}} .comdat
+# SYMBOLS-REMOVE-TEXT-NEXT: AUX scnlen {{.*}} assoc 2 comdat 2
+# SYMBOLS-REMOVE-TEXT-NEXT: {{.*}}(sec 2){{.*}} foo
+# SYMBOLS-REMOVE-TEXT-EMPTY:
+#
+#
+# Removing the .comdat section fails, since the .text section has relocations
+# against it.
+#
+# ERROR-RELOC: Relocation target 'foo' ({{.*}}) not found
+#
+#
+# Removing the .comdat section and .text (with a relocation against .comdat)
+# works, as it also removes the .associative section transitively.
+#
+# SECTIONS-REMOVE-TEXT-COMDAT: Sections:
+# SECTIONS-REMOVE-TEXT-COMDAT-NEXT: Idx Name
+# SECTIONS-REMOVE-TEXT-COMDAT-NEXT: 0 .bss
+# SECTIONS-REMOVE-TEXT-COMDAT-EMPTY:
+#
+# SYMBOLS-REMOVE-TEXT-COMDAT: SYMBOL TABLE:
+# SYMBOLS-REMOVE-TEXT-COMDAT-NEXT: {{.*}}(sec -1){{.*}} @feat.00
+# SYMBOLS-REMOVE-TEXT-COMDAT-NEXT: {{.*}}(sec 1){{.*}} .bss
+# SYMBOLS-REMOVE-TEXT-COMDAT-NEXT: AUX scnlen {{.*}} assoc 1 comdat 0
+# SYMBOLS-REMOVE-TEXT-COMDAT-EMPTY:
+
+--- !COFF
+header:          
+  Machine:         IMAGE_FILE_MACHINE_AMD64
+  Characteristics: [  ]
+sections:        
+  - Name:            .text
+    Characteristics: [  ]
+    Alignment:       4
+    SectionData:     488B0500000000C3
+    Relocations:     
+      - VirtualAddress:  3
+        SymbolName:      foo
+        Type:            IMAGE_REL_AMD64_REL32
+  - Name:            .bss
+    Characteristics: [  ]
+    Alignment:       4
+    SectionData:     ''
+  - Name:            .comdat
+    Characteristics: [ IMAGE_SCN_LNK_COMDAT ]
+    Alignment:       1
+    SectionData:     '2A000000'
+  - Name:            .associative
+    Characteristics: [ IMAGE_SCN_LNK_COMDAT ]
+    Alignment:       1
+    SectionData:     '0000000000000000'
+symbols:         
+  - Name:            '@feat.00'
+    Value:           0
+    SectionNumber:   -1
+    SimpleType:      IMAGE_SYM_TYPE_NULL
+    ComplexType:     IMAGE_SYM_DTYPE_NULL
+    StorageClass:    IMAGE_SYM_CLASS_STATIC
+  - Name:            .text
+    Value:           0
+    SectionNumber:   1
+    SimpleType:      IMAGE_SYM_TYPE_NULL
+    ComplexType:     IMAGE_SYM_DTYPE_NULL
+    StorageClass:    IMAGE_SYM_CLASS_STATIC
+    SectionDefinition: 
+      Length:          8
+      NumberOfRelocations: 1
+      NumberOfLinenumbers: 0
+      CheckSum:        583624169
+      Number:          1
+  - Name:            .bss
+    Value:           0
+    SectionNumber:   2
+    SimpleType:      IMAGE_SYM_TYPE_NULL
+    ComplexType:     IMAGE_SYM_DTYPE_NULL
+    StorageClass:    IMAGE_SYM_CLASS_STATIC
+    SectionDefinition: 
+      Length:          0
+      NumberOfRelocations: 0
+      NumberOfLinenumbers: 0
+      CheckSum:        0
+      Number:          2
+  - Name:            .associative
+    Value:           0
+    SectionNumber:   4
+    SimpleType:      IMAGE_SYM_TYPE_NULL
+    ComplexType:     IMAGE_SYM_DTYPE_NULL
+    StorageClass:    IMAGE_SYM_CLASS_STATIC
+    SectionDefinition: 
+      Length:          8
+      NumberOfRelocations: 0
+      NumberOfLinenumbers: 0
+      CheckSum:        0
+      Number:          3
+      Selection:       IMAGE_COMDAT_SELECT_ASSOCIATIVE
+  - Name:            .comdat
+    Value:           0
+    SectionNumber:   3
+    SimpleType:      IMAGE_SYM_TYPE_NULL
+    ComplexType:     IMAGE_SYM_DTYPE_NULL
+    StorageClass:    IMAGE_SYM_CLASS_STATIC
+    SectionDefinition: 
+      Length:          4
+      NumberOfRelocations: 0
+      NumberOfLinenumbers: 0
+      CheckSum:        3482275674
+      Number:          3
+      Selection:       IMAGE_COMDAT_SELECT_ANY
+  - Name:            foo
+    Value:           0
+    SectionNumber:   3
+    SimpleType:      IMAGE_SYM_TYPE_NULL
+    ComplexType:     IMAGE_SYM_DTYPE_NULL
+    StorageClass:    IMAGE_SYM_CLASS_EXTERNAL
+  - Name:            main
+    Value:           0
+    SectionNumber:   1
+    SimpleType:      IMAGE_SYM_TYPE_NULL
+    ComplexType:     IMAGE_SYM_DTYPE_NULL
+    StorageClass:    IMAGE_SYM_CLASS_EXTERNAL
+...
diff --git a/test/tools/llvm-objcopy/COFF/strip-all.test b/test/tools/llvm-objcopy/COFF/strip-all.test
new file mode 100644
index 0000000000000000000000000000000000000000..8a92ac61509cd45981b923b4bfcdb868ba8e6e3b
--- /dev/null
+++ b/test/tools/llvm-objcopy/COFF/strip-all.test
@@ -0,0 +1,55 @@
+# RUN: yaml2obj %s > %t.in.o
+
+# RUN: llvm-objdump -t %t.in.o | FileCheck %s --check-prefixes=SYMBOLS,SYMBOLS-PRE
+
+# RUN: llvm-objcopy --strip-all %t.in.o %t.out.o
+# RUN: llvm-objdump -t %t.out.o | FileCheck %s --check-prefix=SYMBOLS
+# RUN: llvm-readobj -relocs %t.out.o | FileCheck %s --check-prefix=RELOCS
+
+# Test that -S, llvm-strip without arguments and --strip-all-gnu produces
+# output identical to --strip-all above.
+# RUN: llvm-objcopy -S %t.in.o %t.out-short.o
+# RUN: cmp %t.out.o %t.out-short.o
+
+# RUN: cp %t.in.o %t.out-strip.o
+# RUN: llvm-strip %t.out-strip.o
+# RUN: cmp %t.out.o %t.out-strip.o
+
+# RUN: llvm-objcopy --strip-all-gnu %t.in.o %t.out-gnu.o
+# RUN: cmp %t.out.o %t.out-gnu.o
+
+# SYMBOLS: SYMBOL TABLE:
+# SYMBOLS-PRE-NEXT: external
+# SYMBOLS-PRE-NEXT: external_undefined
+# SYMBOLS-EMPTY:
+
+# RELOCS:      Relocations [
+# RELOCS-NEXT: ]
+
+--- !COFF
+header:          
+  Machine:         IMAGE_FILE_MACHINE_AMD64
+  Characteristics: [  ]
+sections:        
+  - Name:            .text
+    Characteristics: [  ]
+    Alignment:       4
+    SectionData:     488B0500000000C3
+    Relocations:     
+      - VirtualAddress:  3
+        SymbolName:      external_undefined
+        Type:            IMAGE_REL_AMD64_REL32
+symbols:         
+  - Name:            external
+    Value:           0
+    SectionNumber:   1
+    SimpleType:      IMAGE_SYM_TYPE_NULL
+    ComplexType:     IMAGE_SYM_DTYPE_NULL
+    StorageClass:    IMAGE_SYM_CLASS_EXTERNAL
+  - Name:            external_undefined
+    Value:           0
+    SectionNumber:   0
+    SimpleType:      IMAGE_SYM_TYPE_NULL
+    ComplexType:     IMAGE_SYM_DTYPE_NULL
+    StorageClass:    IMAGE_SYM_CLASS_EXTERNAL
+...
diff --git a/test/tools/llvm-objcopy/COFF/strip-debug.test b/test/tools/llvm-objcopy/COFF/strip-debug.test
new file mode 100644
index 0000000000000000000000000000000000000000..97fa96aac70d2dd3c817bbab0bae34c1b743d392
--- /dev/null
+++ b/test/tools/llvm-objcopy/COFF/strip-debug.test
@@ -0,0 +1,109 @@
+# RUN: yaml2obj %s > %t.in.o
+#
+# RUN: llvm-objdump --section-headers %t.in.o | FileCheck %s --check-prefixes=SECTIONS,SECTIONS-PRE
+# RUN: llvm-objdump -t %t.in.o | FileCheck %s --check-prefixes=SYMBOLS,SYMBOLS-PRE
+#
+# RUN: llvm-objcopy --strip-debug %t.in.o %t.out.o
+# RUN: llvm-objdump --section-headers %t.out.o | FileCheck %s --check-prefixes=SECTIONS
+# RUN: llvm-objdump -t %t.out.o | FileCheck %s --check-prefixes=SYMBOLS
+#
+# Test that --strip-all, --strip-all-gnu, --discard-all and --strip-unneeded,
+# plus llvm-strip without arguments all produce a similiar set of sections
+# (while they remove symbols differently).
+#
+# RUN: llvm-objcopy --strip-all %t.in.o %t.strip-all.o
+# RUN: llvm-objdump --section-headers %t.strip-all.o | FileCheck %s --check-prefixes=SECTIONS
+#
+# RUN: llvm-objcopy --strip-all-gnu %t.in.o %t.strip-all-gnu.o
+# RUN: llvm-objdump --section-headers %t.strip-all-gnu.o | FileCheck %s --check-prefixes=SECTIONS
+#
+# RUN: llvm-objcopy --discard-all %t.in.o %t.discard-all.o
+# RUN: llvm-objdump --section-headers %t.discard-all.o | FileCheck %s --check-prefixes=SECTIONS
+#
+# RUN: llvm-objcopy --discard-all %t.in.o %t.strip-unneeded.o
+# RUN: llvm-objdump --section-headers %t.strip-unneeded.o | FileCheck %s --check-prefixes=SECTIONS
+#
+# SECTIONS:        Sections:
+# SECTIONS-NEXT:   Idx Name
+# SECTIONS-NEXT:     0      .text
+# SECTIONS-NEXT:     1      .data
+# SECTIONS-NEXT:     2      .bss
+# SECTIONS-NEXT:     3      .xdata
+# SECTIONS-NEXT:     4      .reloc
+# SECTIONS-PRE-NEXT: 5      .debug_discardable
+# SECTIONS-NEXT:     {{.*}} .debug_undiscardable
+# SECTIONS-NEXT:     {{.*}} .llvm_addrsig
+# SECTIONS-EMPTY:
+#
+# Test that --strip-debug doesn't remove e.g. unreferenced local symbols.
+#
+# SYMBOLS:          SYMBOL TABLE:
+# SYMBOLS-NEXT:     external
+# SYMBOLS-NEXT:     local_unreferenced
+# SYMBOLS-PRE-NEXT: debug_discardable_sym
+# SYMBOLS-NEXT:     debug_undiscardable_sym
+# SYMBOLS-EMPTY:
+
+--- !COFF
+header:          
+  Machine:         IMAGE_FILE_MACHINE_AMD64
+  Characteristics: [  ]
+sections:        
+  - Name:            .text
+    Characteristics: [  ]
+    Alignment:       4
+    SectionData:     00000000
+  - Name:            .data
+    Characteristics: [  ]
+    Alignment:       4
+    SectionData:     00000000
+  - Name:            .bss
+    Characteristics: [  ]
+    Alignment:       4
+    SectionData:     00000000
+  - Name:            .xdata
+    Characteristics: [  ]
+    Alignment:       4
+    SectionData:     00000000
+  - Name:            .reloc
+    Characteristics: [ IMAGE_SCN_MEM_DISCARDABLE ]
+    Alignment:       4
+    SectionData:     00000000
+  - Name:            .debug_discardable
+    Characteristics: [ IMAGE_SCN_MEM_DISCARDABLE ]
+    Alignment:       4
+    SectionData:     00000000
+  - Name:            .debug_undiscardable
+    Characteristics: [  ]
+    Alignment:       4
+    SectionData:     00000000
+  - Name:            .llvm_addrsig
+    Characteristics: [ IMAGE_SCN_LNK_REMOVE ]
+    Alignment:       4
+    SectionData:     00000000
+symbols:         
+  - Name:            external
+    Value:           0
+    SectionNumber:   1
+    SimpleType:      IMAGE_SYM_TYPE_NULL
+    ComplexType:     IMAGE_SYM_DTYPE_NULL
+    StorageClass:    IMAGE_SYM_CLASS_EXTERNAL
+  - Name:            local_unreferenced
+    Value:           0
+    SectionNumber:   1
+    SimpleType:      IMAGE_SYM_TYPE_NULL
+    ComplexType:     IMAGE_SYM_DTYPE_NULL
+    StorageClass:    IMAGE_SYM_CLASS_STATIC
+  - Name:            debug_discardable_sym
+    Value:           0
+    SectionNumber:   6
+    SimpleType:      IMAGE_SYM_TYPE_NULL
+    ComplexType:     IMAGE_SYM_DTYPE_NULL
+    StorageClass:    IMAGE_SYM_CLASS_EXTERNAL
+  - Name:            debug_undiscardable_sym
+    Value:           0
+    SectionNumber:   7
+    SimpleType:      IMAGE_SYM_TYPE_NULL
+    ComplexType:     IMAGE_SYM_DTYPE_NULL
+    StorageClass:    IMAGE_SYM_CLASS_EXTERNAL
+...
diff --git a/test/tools/llvm-objcopy/COFF/strip-reloc-symbol.test b/test/tools/llvm-objcopy/COFF/strip-reloc-symbol.test
new file mode 100644
index 0000000000000000000000000000000000000000..ba1dfb00efdf39ed6d91d8d7c0ac60efc016fd6e
--- /dev/null
+++ b/test/tools/llvm-objcopy/COFF/strip-reloc-symbol.test
@@ -0,0 +1,5 @@
+RUN: yaml2obj %p/Inputs/strip-symbols.yaml > %t.o
+RUN: not llvm-objcopy -N foo %t.o 2>&1 | FileCheck %s --check-prefix=ERROR
+RUN: not llvm-objcopy --strip-symbol foo %t.o 2>&1 | FileCheck %s --check-prefix=ERROR
+
+ERROR: error: '{{.*}}{{/|\\}}strip-reloc-symbol.test.tmp.o': not stripping symbol 'foo' because it is named in a relocation.
diff --git a/test/tools/llvm-objcopy/COFF/strip-symbol.test b/test/tools/llvm-objcopy/COFF/strip-symbol.test
new file mode 100644
index 0000000000000000000000000000000000000000..268b097255db73ddcbf67883ea8d80a8d497379c
--- /dev/null
+++ b/test/tools/llvm-objcopy/COFF/strip-symbol.test
@@ -0,0 +1,32 @@
+RUN: yaml2obj %p/Inputs/strip-symbols.yaml > %t.in.o
+
+RUN: llvm-readobj -relocations %t.in.o | FileCheck %s --check-prefixes=RELOCS,RELOCS-PRE
+RUN: llvm-objdump -t %t.in.o | FileCheck %s --check-prefixes=SYMBOLS,SYMBOLS-PRE
+
+RUN: llvm-objcopy -N mainfunc %t.in.o %t.out.o
+RUN: llvm-readobj -relocations %t.out.o | FileCheck %s --check-prefixes=RELOCS,RELOCS-POST
+RUN: llvm-objdump -t %t.out.o | FileCheck %s --check-prefix=SYMBOLS
+
+RUN: llvm-objcopy --strip-symbol mainfunc %t.in.o %t.out.o
+RUN: llvm-readobj -relocations %t.out.o | FileCheck %s --check-prefixes=RELOCS,RELOCS-POST
+RUN: llvm-objdump -t %t.out.o | FileCheck %s --check-prefix=SYMBOLS
+
+Explicitly listing the relocations for the input as well, to show
+that the symbol index of the symbol foo is updated in the relocations,
+while keeping relocations to two distinct .rdata symbols separate.
+
+RELOCS:      Relocations [
+RELOCS-NEXT:   Section (1) .text {
+RELOCS-NEXT:     0x3 IMAGE_REL_AMD64_REL32 .rdata (0)
+RELOCS-NEXT:     0xA IMAGE_REL_AMD64_REL32 .rdata (1)
+RELOCS-PRE-NEXT:  0x11 IMAGE_REL_AMD64_REL32 foo (3)
+RELOCS-POST-NEXT: 0x11 IMAGE_REL_AMD64_REL32 foo (2)
+RELOCS-NEXT:   }
+RELOCS-NEXT: ]
+
+SYMBOLS: SYMBOL TABLE:
+SYMBOLS-NEXT: .rdata
+SYMBOLS-NEXT: .rdata
+SYMBOLS-PRE-NEXT: mainfunc
+SYMBOLS-NEXT: foo
+SYMBOLS-EMPTY:
diff --git a/test/tools/llvm-objcopy/COFF/strip-unneeded.test b/test/tools/llvm-objcopy/COFF/strip-unneeded.test
new file mode 100644
index 0000000000000000000000000000000000000000..ca3efe4a3138a7b2e02948b159d0ff41ec65bbdd
--- /dev/null
+++ b/test/tools/llvm-objcopy/COFF/strip-unneeded.test
@@ -0,0 +1,23 @@
+RUN: yaml2obj %p/Inputs/discard-locals.yaml > %t.in.o
+
+RUN: llvm-objdump -t %t.in.o | FileCheck %s --check-prefixes=SYMBOLS,SYMBOLS-PRE
+
+RUN: llvm-objcopy --strip-unneeded %t.in.o %t.out.o
+RUN: llvm-objdump -t %t.out.o | FileCheck %s --check-prefix=SYMBOLS
+
+RUN: llvm-objcopy --strip-unneeded-symbol=external_undefined_unreferenced \
+RUN:              --strip-unneeded-symbol=local_unreferenced \
+RUN:              --strip-unneeded-symbol=local_undefined_unreferenced \
+RUN:              --strip-unneeded-symbol='@feat.00' \
+RUN:               %t.in.o %t.out2.o
+RUN: cmp %t.out.o %t.out2.o
+
+SYMBOLS: SYMBOL TABLE:
+SYMBOLS-NEXT: external
+SYMBOLS-NEXT: external_undefined
+SYMBOLS-PRE-NEXT: external_undefined_unreferenced
+SYMBOLS-PRE-NEXT: local_unreferenced
+SYMBOLS-NEXT: local_referenced
+SYMBOLS-PRE-NEXT: local_undefined_unreferenced
+SYMBOLS-PRE-NEXT: @feat.00
+SYMBOLS-EMPTY:
diff --git a/test/tools/llvm-objcopy/COFF/weak-external.test b/test/tools/llvm-objcopy/COFF/weak-external.test
new file mode 100644
index 0000000000000000000000000000000000000000..d36a53b4eb1b93348bca44134bb583eb7f64f57e
--- /dev/null
+++ b/test/tools/llvm-objcopy/COFF/weak-external.test
@@ -0,0 +1,49 @@
+# RUN: yaml2obj %s > %t.in.o
+
+# RUN: llvm-objdump -t %t.in.o | FileCheck %s --check-prefixes=SYMBOLS,SYMBOLS-PRE
+
+# RUN: llvm-objcopy -N func %t.in.o %t.out.o
+# RUN: llvm-objdump -t %t.out.o | FileCheck %s --check-prefixes=SYMBOLS,SYMBOLS-POST
+
+# RUN: not llvm-objcopy -N .weak.foobar.file1 %t.in.o %t.err.o 2>&1 | FileCheck %s --check-prefix=ERROR
+
+# SYMBOLS: SYMBOL TABLE:
+# SYMBOLS-PRE-NEXT: func
+# SYMBOLS-NEXT: .weak.foobar.file1
+# SYMBOLS-NEXT: foobar
+# SYMBOLS-PRE-NEXT: AUX indx 1
+# SYMBOLS-POST-NEXT: AUX indx 0
+# SYMBOLS-EMPTY:
+
+# ERROR: Symbol 'foobar' is missing its weak target
+
+--- !COFF
+header:          
+  Machine:         IMAGE_FILE_MACHINE_AMD64
+  Characteristics: [  ]
+sections:        
+  - Name:            .text
+    Characteristics: [  ]
+symbols:         
+  - Name:            func
+    Value:           0
+    SectionNumber:   1
+    SimpleType:      IMAGE_SYM_TYPE_NULL
+    ComplexType:     IMAGE_SYM_DTYPE_NULL
+    StorageClass:    IMAGE_SYM_CLASS_EXTERNAL
+  - Name:            .weak.foobar.file1
+    Value:           1
+    SectionNumber:   1
+    SimpleType:      IMAGE_SYM_TYPE_NULL
+    ComplexType:     IMAGE_SYM_DTYPE_NULL
+    StorageClass:    IMAGE_SYM_CLASS_EXTERNAL
+  - Name:            foobar
+    Value:           0
+    SectionNumber:   0
+    SimpleType:      IMAGE_SYM_TYPE_NULL
+    ComplexType:     IMAGE_SYM_DTYPE_FUNCTION
+    StorageClass:    IMAGE_SYM_CLASS_WEAK_EXTERNAL
+    WeakExternal:    
+      TagIndex:        1
+      Characteristics: IMAGE_WEAK_EXTERN_SEARCH_NOLIBRARY
+...
diff --git a/test/tools/llvm-objcopy/ELF/abs-symbol.test b/test/tools/llvm-objcopy/ELF/abs-symbol.test
index fb261ce341a1306ca2f7bab4d1411be250ba617d..5f2536d1df965e696942de9c01f6920ce6624740 100644
--- a/test/tools/llvm-objcopy/ELF/abs-symbol.test
+++ b/test/tools/llvm-objcopy/ELF/abs-symbol.test
@@ -1,6 +1,6 @@
 # RUN: yaml2obj %s > %t
 # RUN: llvm-objcopy %t %t2
-# RUN: llvm-readobj -symbols %t2 | FileCheck %s
+# RUN: llvm-readobj --symbols %t2 | FileCheck %s
 
 !ELF
 FileHeader:
diff --git a/test/tools/llvm-objcopy/ELF/add-gnu-debuglink.test b/test/tools/llvm-objcopy/ELF/add-gnu-debuglink.test
index 16f4637ffa9a0fdf1a8408c25b8044623316b417..330571c337776fdc8c87dafff8e69ae7672defbf 100644
--- a/test/tools/llvm-objcopy/ELF/add-gnu-debuglink.test
+++ b/test/tools/llvm-objcopy/ELF/add-gnu-debuglink.test
@@ -1,7 +1,7 @@
 # RUN: yaml2obj %s > %t
 # RUN: printf 0000 > %t.blob
-# RUN: llvm-objcopy -add-gnu-debuglink=%t.blob %t %t2
-# RUN: llvm-readobj -sections -section-data %t2 | FileCheck %s
+# RUN: llvm-objcopy --add-gnu-debuglink=%t.blob %t %t2
+# RUN: llvm-readobj --sections --section-data %t2 | FileCheck %s
 
 !ELF
 FileHeader:
diff --git a/test/tools/llvm-objcopy/ELF/add-note.test b/test/tools/llvm-objcopy/ELF/add-note.test
new file mode 100644
index 0000000000000000000000000000000000000000..b45623078463dac0b5e6550f88bb8165a6c47d58
--- /dev/null
+++ b/test/tools/llvm-objcopy/ELF/add-note.test
@@ -0,0 +1,36 @@
+# Verify that --add-section can be used to add a note section which is
+# successfully interpreted by tools that read notes.
+
+# Add [namesz, descsz, type, name, desc] for a build id.
+# RUN: echo -e -n "\x04\x00\x00\x00" >  %t-note.bin
+# RUN: echo -e -n "\x10\x00\x00\x00" >> %t-note.bin
+# RUN: echo -e -n "\x03\x00\x00\x00" >> %t-note.bin
+# RUN: echo -e -n "GNU\x00"          >> %t-note.bin
+# RUN: echo -e -n "\x00\x01\x02\x03" >> %t-note.bin
+# RUN: echo -e -n "\x04\x05\x06\x07" >> %t-note.bin
+# RUN: echo -e -n "\x08\x09\x0a\x0b" >> %t-note.bin
+# RUN: echo -e -n "\x0c\x0d\x0e\x0f" >> %t-note.bin
+
+# RUN: yaml2obj %s > %t.o
+# RUN: llvm-objcopy --add-section=.note.gnu.build-id=%t-note.bin %t.o %t-with-note.o
+# RUN: llvm-readobj --notes %t-with-note.o | FileCheck %s
+
+!ELF
+FileHeader:
+  Class:           ELFCLASS64
+  Data:            ELFDATA2LSB
+  Type:            ET_REL
+  Machine:         EM_X86_64
+
+# CHECK:      Notes [
+# CHECK-NEXT:   NoteSection {
+# CHECK-NEXT:     Offset:
+# CHECK-NEXT:     Size:
+# CHECK-NEXT:     Note {
+# CHECK-NEXT:       Owner: GNU
+# CHECK-NEXT:       Data size: 0x10
+# CHECK-NEXT:       Type: NT_GNU_BUILD_ID
+# CHECK-NEXT:       Build ID: 000102030405060708090a0b0c0d0e0f
+# CHECK-NEXT:     }
+# CHECK-NEXT:   }
+# CHECK-NEXT: ]
diff --git a/test/tools/llvm-objcopy/ELF/add-section-remove.test b/test/tools/llvm-objcopy/ELF/add-section-remove.test
index 9e7abc76aa3e59cb811fd00adb0a3c6382ed0ae1..fe462dba1e84f6d08cedb2f3726c1785db65da77 100644
--- a/test/tools/llvm-objcopy/ELF/add-section-remove.test
+++ b/test/tools/llvm-objcopy/ELF/add-section-remove.test
@@ -1,7 +1,7 @@
 # RUN: yaml2obj %s > %t
 # RUN: echo 0000 > %t.sec
-# RUN: llvm-objcopy -R .test2 -add-section=.test2=%t.sec %t %t2
-# RUN: llvm-readobj -file-headers -sections -section-data %t2 | FileCheck %s
+# RUN: llvm-objcopy -R .test2 --add-section=.test2=%t.sec %t %t2
+# RUN: llvm-readobj --file-headers --sections --section-data %t2 | FileCheck %s
 
 !ELF
 FileHeader:
diff --git a/test/tools/llvm-objcopy/ELF/add-section-special.test b/test/tools/llvm-objcopy/ELF/add-section-special.test
new file mode 100644
index 0000000000000000000000000000000000000000..e83cd51810f8e277d92d75503d2947e43f5843ae
--- /dev/null
+++ b/test/tools/llvm-objcopy/ELF/add-section-special.test
@@ -0,0 +1,22 @@
+# Check the properties of added sections.
+# By default, sections are SHT_PROGBITS, but .note sections (excluding
+# .note.GNU-stack) are SHT_NOTE sections.
+
+# RUN: yaml2obj %s > %t.o
+# RUN: llvm-objcopy --add-section=.foo=/dev/null %t.o %t-foo.o
+# RUN: llvm-objcopy --add-section=.note.foo=/dev/null %t.o %t-regular-note.o
+# RUN: llvm-objcopy --add-section=.note.GNU-stack=/dev/null %t.o %t-gnu-stack.o
+# RUN: llvm-readelf --sections %t-foo.o | FileCheck %s --check-prefix=NORMAL
+# RUN: llvm-readelf --sections %t-regular-note.o | FileCheck %s --check-prefix=NOTE
+# RUN: llvm-readelf --sections %t-gnu-stack.o | FileCheck %s --check-prefix=GNU-STACK
+
+!ELF
+FileHeader:
+  Class:           ELFCLASS64
+  Data:            ELFDATA2LSB
+  Type:            ET_REL
+  Machine:         EM_X86_64
+
+# NORMAL:    .foo            PROGBITS
+# NOTE:      .note.foo       NOTE
+# GNU-STACK: .note.GNU-stack PROGBITS
diff --git a/test/tools/llvm-objcopy/ELF/add-section.test b/test/tools/llvm-objcopy/ELF/add-section.test
index 369d1194efc2395b511fbac63ca24f4ec3c8652b..bf3ffdb090ef0ce04e3a32571bd742df58f8ea3a 100644
--- a/test/tools/llvm-objcopy/ELF/add-section.test
+++ b/test/tools/llvm-objcopy/ELF/add-section.test
@@ -1,8 +1,8 @@
 # RUN: yaml2obj %s > %t
 # RUN: llvm-objcopy -O binary -j .test2 %t %t.sec
 # RUN: llvm-objcopy -R .test2 %t %t2
-# RUN: llvm-objcopy -add-section=.test2=%t.sec %t2 %t3
-# RUN: llvm-readobj -file-headers -sections -section-data %t3 | FileCheck %s
+# RUN: llvm-objcopy --add-section=.test2=%t.sec %t2 %t3
+# RUN: llvm-readobj --file-headers --sections --section-data %t3 | FileCheck %s
 
 !ELF
 FileHeader:
diff --git a/test/tools/llvm-objcopy/ELF/armexidx-link.test b/test/tools/llvm-objcopy/ELF/armexidx-link.test
index ec942a1e6fc3f2a56ed01666df34d9ef9cca2411..596a7807ee7c9404593feba3446a6060a9e3a445 100644
--- a/test/tools/llvm-objcopy/ELF/armexidx-link.test
+++ b/test/tools/llvm-objcopy/ELF/armexidx-link.test
@@ -1,6 +1,6 @@
 # RUN: yaml2obj %s > %t
-# RUN: llvm-objcopy -remove-section=.text.bar %t %t2
-# RUN: llvm-readobj -sections %t2 | FileCheck %s
+# RUN: llvm-objcopy --remove-section=.text.bar %t %t2
+# RUN: llvm-readobj --sections %t2 | FileCheck %s
 
 # CHECK:          Index: 2
 # CHECK-NEXT:     Name: .ARM.exidx.text.foo (1)
diff --git a/test/tools/llvm-objcopy/ELF/auto-remove-shndx.test b/test/tools/llvm-objcopy/ELF/auto-remove-shndx.test
index 5452f34d96deb338b2d1c004a34c12a5208dae30..8e6c788bf4817da989d7b6df4271478c6007e8ad 100644
--- a/test/tools/llvm-objcopy/ELF/auto-remove-shndx.test
+++ b/test/tools/llvm-objcopy/ELF/auto-remove-shndx.test
@@ -1,5 +1,5 @@
-# RUN: %python %p/Inputs/ungzip.py %p/Inputs/many-sections.o.gz > %t
+# RUN: %python %p/../Inputs/ungzip.py %p/Inputs/many-sections.o.gz > %t
 # RUN: llvm-objcopy -R .text -R s0 -R s1 -R s2 -R s3 -R s4 -R s5 -R s6 %t %t2
-# RUN: llvm-readobj -sections %t2 | FileCheck --check-prefix=SECS %s
+# RUN: llvm-readobj --sections %t2 | FileCheck --check-prefix=SECS %s
 
 # SECS-NOT: Name: .symtab_shndx
diff --git a/test/tools/llvm-objcopy/ELF/bad-build-id.test b/test/tools/llvm-objcopy/ELF/bad-build-id.test
index 24b3cb697a059931ce34f818fd0177ffccce0a5e..e5e9b9556502546a17aabf1ec9388755bb070195 100644
--- a/test/tools/llvm-objcopy/ELF/bad-build-id.test
+++ b/test/tools/llvm-objcopy/ELF/bad-build-id.test
@@ -1,7 +1,7 @@
 # RUN: yaml2obj %s > %t
 # RUN: not llvm-objcopy --build-id-link-dir=%t-dir --build-id-link-input=.debug %t 2>&1 >/dev/null | FileCheck %s
 
-# CHECK: build ID in file {{.*}} is smaller than two bytes.
+# CHECK: build ID is smaller than two bytes.
 
 --- !ELF
 FileHeader:
diff --git a/test/tools/llvm-objcopy/ELF/bad-output-format.test b/test/tools/llvm-objcopy/ELF/bad-output-format.test
new file mode 100644
index 0000000000000000000000000000000000000000..e01d955111b8a5bf76a9026607bbc97eff6084a8
--- /dev/null
+++ b/test/tools/llvm-objcopy/ELF/bad-output-format.test
@@ -0,0 +1,13 @@
+# RUN: yaml2obj %s > %t.o
+
+# RUN: not llvm-objcopy -O xyz %t.o %t.2.o 2>&1 \
+# RUN:   | FileCheck %s --check-prefix=BAD-OUTPUT-FORMAT
+
+!ELF
+FileHeader:
+  Class:           ELFCLASS32
+  Data:            ELFDATA2LSB
+  Type:            ET_EXEC
+  Machine:         EM_386
+
+# BAD-OUTPUT-FORMAT: Invalid output format: 'xyz'.
diff --git a/test/tools/llvm-objcopy/ELF/basic-archive-copy.test b/test/tools/llvm-objcopy/ELF/basic-archive-copy.test
index bc050c3c70ddda9387814374144bb9bc9f5a2a9e..cf973d6cd3f66f5b32fc38fff5ad880384ca4746 100644
--- a/test/tools/llvm-objcopy/ELF/basic-archive-copy.test
+++ b/test/tools/llvm-objcopy/ELF/basic-archive-copy.test
@@ -8,9 +8,9 @@
 # RUN: llvm-ar p %t2.a > %t3
 # RUN: cmp %t2 %t3
 
-# RUN: llvm-readobj -sections %t2 | FileCheck %s
-# RUN: llvm-nm -print-armap %t.a | FileCheck --check-prefix=INDEX-TABLE %s
-# RUN: llvm-nm -print-armap %t2.a | FileCheck --check-prefix=INDEX-TABLE %s
+# RUN: llvm-readobj --sections %t2 | FileCheck %s
+# RUN: llvm-nm --print-armap %t.a | FileCheck --check-prefix=INDEX-TABLE %s
+# RUN: llvm-nm --print-armap %t2.a | FileCheck --check-prefix=INDEX-TABLE %s
 # Verify that llvm-objcopy has not modifed the input.
 # RUN: cmp %t.copy.a %t.a
 
@@ -22,8 +22,8 @@
 # RUN: llvm-objcopy %t.no.index.a %t2.no.index.a
 # RUN: llvm-ar p %t2.no.index.a > %t4
 
-# RUN: llvm-nm -print-armap %t.no.index.a | FileCheck --check-prefix=NO-INDEX-TABLE %s
-# RUN: llvm-nm -print-armap %t2.no.index.a | FileCheck --check-prefix=NO-INDEX-TABLE %s
+# RUN: llvm-nm --print-armap %t.no.index.a | FileCheck --check-prefix=NO-INDEX-TABLE %s
+# RUN: llvm-nm --print-armap %t2.no.index.a | FileCheck --check-prefix=NO-INDEX-TABLE %s
 # RUN: cmp %t2 %t4
 
 # NO-INDEX-TABLE-NOT: Archive map
diff --git a/test/tools/llvm-objcopy/ELF/basic-copy.test b/test/tools/llvm-objcopy/ELF/basic-copy.test
index 266d48b495e0ad74ee378c64e7e4f37bf5510560..b6b0bcc612be4502f3398c08f4ea030c3ff2c03e 100644
--- a/test/tools/llvm-objcopy/ELF/basic-copy.test
+++ b/test/tools/llvm-objcopy/ELF/basic-copy.test
@@ -1,6 +1,6 @@
 # RUN: yaml2obj %s > %t
 # RUN: llvm-objcopy %t %t2
-# RUN: llvm-readobj -sections %t2 | FileCheck %s
+# RUN: llvm-readobj --sections %t2 | FileCheck %s
 
 !ELF
 FileHeader:
diff --git a/test/tools/llvm-objcopy/ELF/basic-keep.test b/test/tools/llvm-objcopy/ELF/basic-keep.test
index 79d7717d9c0957179da8835d68674973c3633e41..8488a2679b38d8794a38c06747429ff4d029ce9c 100644
--- a/test/tools/llvm-objcopy/ELF/basic-keep.test
+++ b/test/tools/llvm-objcopy/ELF/basic-keep.test
@@ -1,7 +1,7 @@
 # RUN: yaml2obj %s > %t
-# RUN: llvm-objcopy -strip-non-alloc -keep-section=.test %t %t2
-# RUN: llvm-strip --strip-all -keep-section=.test %t -o %t3
-# RUN: llvm-readobj -file-headers -sections %t2 | FileCheck %s
+# RUN: llvm-objcopy --strip-non-alloc --keep-section=.test %t %t2
+# RUN: llvm-strip --strip-all --keep-section=.test %t -o %t3
+# RUN: llvm-readobj --file-headers --sections %t2 | FileCheck %s
 # RUN: cmp %t2 %t3
 
 !ELF
diff --git a/test/tools/llvm-objcopy/ELF/basic-only-section.test b/test/tools/llvm-objcopy/ELF/basic-only-section.test
index 536a0fc4cca919af488ad009e78085b71696ed17..e47a97ac6d731ab971906beebc7879c22c743cc0 100644
--- a/test/tools/llvm-objcopy/ELF/basic-only-section.test
+++ b/test/tools/llvm-objcopy/ELF/basic-only-section.test
@@ -1,7 +1,7 @@
 # RUN: yaml2obj %s > %t
-# RUN: llvm-objcopy -only-section=.test %t %t2
+# RUN: llvm-objcopy --only-section=.test %t %t2
 # RUN: llvm-objcopy -j .test %t %t3
-# RUN: llvm-readobj -file-headers -sections %t2 | FileCheck %s
+# RUN: llvm-readobj --file-headers --sections %t2 | FileCheck %s
 # RUN: diff %t2 %t3
 
 !ELF
diff --git a/test/tools/llvm-objcopy/ELF/basic-relocations.test b/test/tools/llvm-objcopy/ELF/basic-relocations.test
index e9afc05044e19136760158e0683120763c94800d..e043041ba5d706e5c7670546aff7a889b60c42e3 100644
--- a/test/tools/llvm-objcopy/ELF/basic-relocations.test
+++ b/test/tools/llvm-objcopy/ELF/basic-relocations.test
@@ -1,6 +1,6 @@
 # RUN: yaml2obj %s > %t
 # RUN: llvm-objcopy %t %t2
-# RUN: llvm-readobj -relocations %t2 | FileCheck %s
+# RUN: llvm-readobj --relocations %t2 | FileCheck %s
 
 !ELF
 FileHeader:
diff --git a/test/tools/llvm-objcopy/ELF/binary-input-arch.test b/test/tools/llvm-objcopy/ELF/binary-input-arch.test
index 08f02da882b2fb93c48d7ac75447ea3a799b870b..4c04b6ee329ac4d56811df6d4f128726db975506 100644
--- a/test/tools/llvm-objcopy/ELF/binary-input-arch.test
+++ b/test/tools/llvm-objcopy/ELF/binary-input-arch.test
@@ -1,25 +1,25 @@
 # RUN: echo abcd > %t.txt
 
 # RUN: llvm-objcopy -I binary -B aarch64 %t.txt %t.aarch64.o
-# RUN: llvm-readobj -file-headers %t.aarch64.o | FileCheck %s --check-prefixes=CHECK,AARCH64,64
+# RUN: llvm-readobj --file-headers %t.aarch64.o | FileCheck %s --check-prefixes=CHECK,AARCH64,64
 
 # RUN: llvm-objcopy -I binary -B arm %t.txt %t.arm.o
-# RUN: llvm-readobj -file-headers %t.arm.o | FileCheck %s --check-prefixes=CHECK,ARM,32
+# RUN: llvm-readobj --file-headers %t.arm.o | FileCheck %s --check-prefixes=CHECK,ARM,32
 
 # RUN: llvm-objcopy -I binary -B i386 %t.txt %t.i386.o
-# RUN: llvm-readobj -file-headers %t.i386.o | FileCheck %s --check-prefixes=CHECK,I386,32
+# RUN: llvm-readobj --file-headers %t.i386.o | FileCheck %s --check-prefixes=CHECK,I386,32
 
 # RUN: llvm-objcopy -I binary -B i386:x86-64 %t.txt %t.i386_x86-64.o
-# RUN: llvm-readobj -file-headers %t.i386_x86-64.o | FileCheck %s --check-prefixes=CHECK,X86-64,64
+# RUN: llvm-readobj --file-headers %t.i386_x86-64.o | FileCheck %s --check-prefixes=CHECK,X86-64,64
 
 # RUN: llvm-objcopy -I binary -B powerpc:common64 %t.txt %t.powerpc_common64.o
-# RUN: llvm-readobj -file-headers %t.powerpc_common64.o | FileCheck %s --check-prefixes=CHECK,PPC,64
+# RUN: llvm-readobj --file-headers %t.powerpc_common64.o | FileCheck %s --check-prefixes=CHECK,PPC,64
 
 # RUN: llvm-objcopy -I binary -B sparc %t.txt %t.sparc.o
-# RUN: llvm-readobj -file-headers %t.sparc.o | FileCheck %s --check-prefixes=CHECK,SPARC,32
+# RUN: llvm-readobj --file-headers %t.sparc.o | FileCheck %s --check-prefixes=CHECK,SPARC,32
 
 # RUN: llvm-objcopy -I binary -B x86-64 %t.txt %t.x86-64.o
-# RUN: llvm-readobj -file-headers %t.x86-64.o | FileCheck %s --check-prefixes=CHECK,X86-64,64
+# RUN: llvm-readobj --file-headers %t.x86-64.o | FileCheck %s --check-prefixes=CHECK,X86-64,64
 
 # CHECK: Format:
 # AARCH64-SAME: ELF64-aarch64-little
diff --git a/test/tools/llvm-objcopy/ELF/binary-input-with-arch.test b/test/tools/llvm-objcopy/ELF/binary-input-with-arch.test
new file mode 100644
index 0000000000000000000000000000000000000000..e8ae841e454596e9734bb745ed10dcfeb4669266
--- /dev/null
+++ b/test/tools/llvm-objcopy/ELF/binary-input-with-arch.test
@@ -0,0 +1,20 @@
+# RUN: echo -n abcd > %t.x-txt
+# Preserve input to verify it is not modified.
+# RUN: cp %t.x-txt %t-copy.txt
+# RUN: llvm-objcopy -I binary -B i386 -O elf64-x86-64 %t.x-txt %t.o
+# RUN: llvm-readobj --file-headers %t.o | FileCheck %s
+# RUN: cmp %t.x-txt %t-copy.txt
+
+# Many uses of objcopy use no spaces in the flags, make sure that also works.
+# RUN: llvm-objcopy -Ibinary -Bi386 -Oelf64-x86-64 %t.x-txt %t-no-spaces.o
+# RUN: cmp %t.o %t-no-spaces.o
+
+# CHECK:      Format: ELF64-x86-64
+# CHECK-NEXT: Arch: x86_64
+# CHECK-NEXT: AddressSize: 64bit
+
+# CHECK: Class: 64-bit
+# CHECK: DataEncoding: LittleEndian
+# CHECK: Machine: EM_X86_64
+# CHECK: HeaderSize: 64
+# CHECK: SectionHeaderEntrySize: 64
diff --git a/test/tools/llvm-objcopy/ELF/binary-input.test b/test/tools/llvm-objcopy/ELF/binary-input.test
index ff1d381c3fa558d6dd53f002f3ecd48225ce6676..d7a708dcea537d3b49520e05b77d06b98e28716b 100644
--- a/test/tools/llvm-objcopy/ELF/binary-input.test
+++ b/test/tools/llvm-objcopy/ELF/binary-input.test
@@ -2,7 +2,7 @@
 # Preserve input to verify it is not modified
 # RUN: cp %t.x-txt %t-copy.txt
 # RUN: llvm-objcopy -I binary -B i386:x86-64 %t.x-txt %t.o
-# RUN: llvm-readobj -sections -symbols %t.o | FileCheck %s
+# RUN: llvm-readobj --sections --symbols %t.o | FileCheck %s
 # RUN: cmp %t.x-txt %t-copy.txt
 
 # CHECK:      Sections [
@@ -45,7 +45,7 @@
 # CHECK-NEXT:     Size:
 # CHECK-NEXT:     Link: 1
 # CHECK-NEXT:     Info: 1
-# CHECK-NEXT:     AddressAlignment: 1
+# CHECK-NEXT:     AddressAlignment: 8
 # CHECK-NEXT:     EntrySize: 24
 # CHECK-NEXT:   }
 # CHECK-NEXT:   Section {
diff --git a/test/tools/llvm-objcopy/ELF/binary-output-empty.test b/test/tools/llvm-objcopy/ELF/binary-output-empty.test
new file mode 100644
index 0000000000000000000000000000000000000000..d234b58a4a2c6488ccf8cce009759a50bf30a843
--- /dev/null
+++ b/test/tools/llvm-objcopy/ELF/binary-output-empty.test
@@ -0,0 +1,27 @@
+# RUN: yaml2obj %s -o %t.o
+
+# Writing an empty output to a non-existent file will still create it.
+# RUN: rm -f %t-new.txt
+# RUN: llvm-objcopy -R .text -O binary %t.o %t-new.txt
+# RUN: wc -c %t-new.txt | FileCheck %s
+
+# Writing an empty output to an existing file will truncate it.
+# RUN: echo abcd > %t-existing.txt
+# RUN: llvm-objcopy -R .text -O binary %t.o %t-existing.txt
+# RUN: wc -c %t-existing.txt | FileCheck %s
+
+# In both cases, the file should be empty.
+# CHECK: 0
+
+!ELF
+FileHeader:
+  Class:           ELFCLASS64
+  Data:            ELFDATA2LSB
+  Type:            ET_EXEC
+  Machine:         EM_X86_64
+Sections:
+  - Name:            .text
+    Type:            SHT_PROGBITS
+    Flags:           [ SHF_ALLOC, SHF_EXECINSTR ]
+    Content:         "c3c3c3c3"
+    Size:            0x1000
diff --git a/test/tools/llvm-objcopy/ELF/binary-remove-all-but-one.test b/test/tools/llvm-objcopy/ELF/binary-remove-all-but-one.test
index e1ef7d6b0a4b5fa236ccf7588f19f0af0bae2e8f..857b76d9a2f343381c9d147f73fbcd3084098e5f 100644
--- a/test/tools/llvm-objcopy/ELF/binary-remove-all-but-one.test
+++ b/test/tools/llvm-objcopy/ELF/binary-remove-all-but-one.test
@@ -1,7 +1,7 @@
 # RUN: yaml2obj %s -o %t
 # RUN: llvm-objcopy -R .text -R .text3 -O binary %t %t2
 # RUN: od -Ax -t x1 %t2 | FileCheck %s
-# RUN: wc -c %t2 | FileCheck %s -check-prefix=SIZE
+# RUN: wc -c %t2 | FileCheck %s --check-prefix=SIZE
 
 !ELF
 FileHeader:
diff --git a/test/tools/llvm-objcopy/ELF/binary-remove-end.test b/test/tools/llvm-objcopy/ELF/binary-remove-end.test
index 1c2994bf852b13b09ca094cf53bc816e06c0a0e1..eddeaf167ec392246674a3f5f248bd02357f6a9d 100644
--- a/test/tools/llvm-objcopy/ELF/binary-remove-end.test
+++ b/test/tools/llvm-objcopy/ELF/binary-remove-end.test
@@ -1,7 +1,7 @@
 # RUN: yaml2obj %s -o %t
 # RUN: llvm-objcopy -R .text3 -O binary %t %t2
 # RUN: od -Ax -v -t x1 %t2 | FileCheck %s
-# RUN: wc -c %t2 | FileCheck %s -check-prefix=SIZE
+# RUN: wc -c %t2 | FileCheck %s --check-prefix=SIZE
 
 !ELF
 FileHeader:
diff --git a/test/tools/llvm-objcopy/ELF/binary-remove-middle.test b/test/tools/llvm-objcopy/ELF/binary-remove-middle.test
index 6d69907a4a73607baf401f333c8a24fe36ef49ee..fbc2f6a9ee37c82ee6148f2572c34ad5fedf92fb 100644
--- a/test/tools/llvm-objcopy/ELF/binary-remove-middle.test
+++ b/test/tools/llvm-objcopy/ELF/binary-remove-middle.test
@@ -1,7 +1,7 @@
 # RUN: yaml2obj %s -o %t
 # RUN: llvm-objcopy -R .text2 -O binary %t %t2
 # RUN: od -Ax -v -t x1 %t2 | FileCheck %s
-# RUN: wc -c %t2 | FileCheck %s -check-prefix=SIZE
+# RUN: wc -c %t2 | FileCheck %s --check-prefix=SIZE
 
 !ELF
 FileHeader:
diff --git a/test/tools/llvm-objcopy/ELF/check-addr-offset-align.test b/test/tools/llvm-objcopy/ELF/check-addr-offset-align.test
index ca2367ba4341453e008b8989b6e00fb42e43d608..bc219fff7839ba7b13208eb4d364fe7bf76cafad 100644
--- a/test/tools/llvm-objcopy/ELF/check-addr-offset-align.test
+++ b/test/tools/llvm-objcopy/ELF/check-addr-offset-align.test
@@ -1,6 +1,6 @@
 # RUN: yaml2obj %s -o %t
 # RUN: llvm-objcopy %t %t2
-# RUN: llvm-readobj -program-headers %t2 | FileCheck %s
+# RUN: llvm-readobj --program-headers %t2 | FileCheck %s
 
 !ELF
 FileHeader:
diff --git a/test/tools/llvm-objcopy/ELF/common-symbol.test b/test/tools/llvm-objcopy/ELF/common-symbol.test
index b659356eab0e238d68f58ab47ac421429ff9cccb..9ec49c711656c418bbcb18304cd1511595015151 100644
--- a/test/tools/llvm-objcopy/ELF/common-symbol.test
+++ b/test/tools/llvm-objcopy/ELF/common-symbol.test
@@ -1,6 +1,6 @@
 # RUN: yaml2obj %s > %t
 # RUN: llvm-objcopy %t %t2
-# RUN: llvm-readobj -symbols %t2 | FileCheck %s
+# RUN: llvm-readobj --symbols %t2 | FileCheck %s
 
 !ELF
 FileHeader:
diff --git a/test/tools/llvm-objcopy/ELF/compress-debug-sections-zlib-gnu.test b/test/tools/llvm-objcopy/ELF/compress-debug-sections-zlib-gnu.test
index 7199b2e5e11e018b192ec320fbf22315ee51aaec..0a360d0da1574af9273036515e0dfc9a0dfaa086 100644
--- a/test/tools/llvm-objcopy/ELF/compress-debug-sections-zlib-gnu.test
+++ b/test/tools/llvm-objcopy/ELF/compress-debug-sections-zlib-gnu.test
@@ -4,12 +4,12 @@
 # RUN: llvm-objcopy --compress-debug-sections=zlib-gnu %t.o %t-compressed.o
 # RUN: llvm-objcopy --decompress-debug-sections %t-compressed.o %t-decompressed.o
 
-# RUN: llvm-objdump -s %t.o -section=.debug_foo | FileCheck %s
+# RUN: llvm-objdump -s %t.o --section=.debug_foo | FileCheck %s
 # RUN: llvm-objdump -s %t-compressed.o | FileCheck %s --check-prefix=CHECK-COMPRESSED
-# RUN: llvm-readobj -relocations -s %t-compressed.o | FileCheck %s --check-prefix=CHECK-FLAGS
-# RUN: llvm-readobj -relocations -s %t-decompressed.o | FileCheck %s --check-prefix=CHECK-HEADER
-# RUN: llvm-readobj -relocations -s %t.o | FileCheck %s --check-prefix=CHECK-HEADER
-# RUN: llvm-objdump -s %t-decompressed.o -section=.debug_foo | FileCheck %s
+# RUN: llvm-readobj --relocations -s %t-compressed.o | FileCheck %s --check-prefix=CHECK-FLAGS
+# RUN: llvm-readobj --relocations -s %t-decompressed.o | FileCheck %s --check-prefix=CHECK-HEADER
+# RUN: llvm-readobj --relocations -s %t.o | FileCheck %s --check-prefix=CHECK-HEADER
+# RUN: llvm-objdump -s %t-decompressed.o --section=.debug_foo | FileCheck %s
 
 # CHECK: .debug_foo:
 # CHECK-NEXT: 0000 00000000 00000000
diff --git a/test/tools/llvm-objcopy/ELF/compress-debug-sections-zlib.test b/test/tools/llvm-objcopy/ELF/compress-debug-sections-zlib.test
index d2ca6ef249bf26060dcc80354a0ef6a29050d26d..1e7dfc9728cf9a8b8578796cd190b1c2aaac6b61 100644
--- a/test/tools/llvm-objcopy/ELF/compress-debug-sections-zlib.test
+++ b/test/tools/llvm-objcopy/ELF/compress-debug-sections-zlib.test
@@ -4,12 +4,12 @@
 # RUN: llvm-objcopy --compress-debug-sections=zlib %t.o %t-compressed.o
 # RUN: llvm-objcopy --decompress-debug-sections %t-compressed.o %t-decompressed.o
 
-# RUN: llvm-objdump -s %t.o -section=.debug_foo | FileCheck %s
+# RUN: llvm-objdump -s %t.o --section=.debug_foo | FileCheck %s
 # RUN: llvm-objdump -s %t-compressed.o | FileCheck %s --check-prefix=CHECK-COMPRESSED
-# RUN: llvm-readobj -relocations -s %t-compressed.o | FileCheck %s --check-prefix=CHECK-FLAGS
-# RUN: llvm-readobj -relocations -s %t-decompressed.o | FileCheck %s --check-prefix=CHECK-HEADER
-# RUN: llvm-readobj -relocations -s %t.o | FileCheck %s --check-prefix=CHECK-HEADER
-# RUN: llvm-objdump -s %t-decompressed.o -section=.debug_foo | FileCheck %s
+# RUN: llvm-readobj --relocations -s %t-compressed.o | FileCheck %s --check-prefix=CHECK-FLAGS
+# RUN: llvm-readobj --relocations -s %t-decompressed.o | FileCheck %s --check-prefix=CHECK-HEADER
+# RUN: llvm-readobj --relocations -s %t.o | FileCheck %s --check-prefix=CHECK-HEADER
+# RUN: llvm-objdump -s %t-decompressed.o --section=.debug_foo | FileCheck %s
 
 # CHECK: .debug_foo:
 # CHECK-NEXT: 0000 00000000 00000000
diff --git a/test/tools/llvm-objcopy/ELF/compress-debug-sections.test b/test/tools/llvm-objcopy/ELF/compress-debug-sections.test
index 6fadad3f38385d1b279510681eaacbb8c9583b53..014303fe638f99baa17a9e6601a1d35da3450711 100644
--- a/test/tools/llvm-objcopy/ELF/compress-debug-sections.test
+++ b/test/tools/llvm-objcopy/ELF/compress-debug-sections.test
@@ -12,9 +12,9 @@
 # RUN: llvm-objcopy --decompress-debug-sections %tzg.o %t3.o
 
 # Using redirects to avoid llvm-objdump from printing the filename.
-# RUN: llvm-objdump -s -section=.debug_str - < %t.o  > %t.txt
-# RUN: llvm-objdump -s -section=.debug_str - < %t2.o > %t2.txt
-# RUN: llvm-objdump -s -section=.debug_str - < %t3.o > %t3.txt
+# RUN: llvm-objdump -s --section=.debug_str - < %t.o  > %t.txt
+# RUN: llvm-objdump -s --section=.debug_str - < %t2.o > %t2.txt
+# RUN: llvm-objdump -s --section=.debug_str - < %t3.o > %t3.txt
 
 # RUN: diff %t.txt %t2.txt
 # RUN: diff %t.txt %t3.txt
diff --git a/test/tools/llvm-objcopy/ELF/copy-osabi.test b/test/tools/llvm-objcopy/ELF/copy-osabi.test
new file mode 100644
index 0000000000000000000000000000000000000000..75a22bc9bdfaf2e11cea5fba528f8425c23405de
--- /dev/null
+++ b/test/tools/llvm-objcopy/ELF/copy-osabi.test
@@ -0,0 +1,16 @@
+# RUN: yaml2obj %s > %t
+# RUN: llvm-objcopy %t %t2
+# RUN: llvm-readobj --file-headers %t2 | FileCheck %s
+
+## Check that llvm-objcopy preserves the OSABI and ABIVersion fields.
+# CHECK: OS/ABI: FreeBSD (0x9)
+# CHECK: ABIVersion: 1
+
+!ELF
+FileHeader:
+  Class:           ELFCLASS64
+  Data:            ELFDATA2LSB
+  OSABI:           ELFOSABI_FREEBSD
+  ABIVersion:      1
+  Type:            ET_REL
+  Machine:         EM_AARCH64
diff --git a/test/tools/llvm-objcopy/ELF/cross-arch-headers.test b/test/tools/llvm-objcopy/ELF/cross-arch-headers.test
new file mode 100644
index 0000000000000000000000000000000000000000..ec76fa655bef6c4384e0162b0417ccafb266c1b7
--- /dev/null
+++ b/test/tools/llvm-objcopy/ELF/cross-arch-headers.test
@@ -0,0 +1,71 @@
+# RUN: yaml2obj %s > %t.o
+
+# RUN: llvm-objcopy %t.o -O elf32-i386 %t.elf32_i386.o
+# RUN: llvm-readobj --file-headers %t.elf32_i386.o | FileCheck %s --check-prefixes=CHECK,I386,32
+
+# RUN: llvm-objcopy %t.o -O elf32-powerpcle %t.elf32_ppcle.o
+# RUN: llvm-readobj --file-headers %t.elf32_ppcle.o | FileCheck %s --check-prefixes=CHECK,PPC,32
+
+# RUN: llvm-objcopy %t.o -O elf32-x86-64 %t.elf32_x86_64.o
+# RUN: llvm-readobj --file-headers %t.elf32_x86_64.o | FileCheck %s --check-prefixes=CHECK,X86-64,32
+
+# RUN: llvm-objcopy %t.o -O elf64-powerpcle %t.elf64_ppcle.o
+# RUN: llvm-readobj --file-headers %t.elf64_ppcle.o | FileCheck %s --check-prefixes=CHECK,PPC64,64
+
+# RUN: llvm-objcopy %t.o -O elf64-x86-64 %t.elf64_x86_64.o
+# RUN: llvm-readobj --file-headers %t.elf64_x86_64.o | FileCheck %s --check-prefixes=CHECK,X86-64,64
+
+!ELF
+FileHeader:
+  Class:           ELFCLASS32
+  Data:            ELFDATA2LSB
+  Type:            ET_EXEC
+  Machine:         EM_386
+Sections:
+  - Name:            .text
+    Type:            SHT_PROGBITS
+    Flags:           [ SHF_ALLOC, SHF_EXECINSTR ]
+  - Name:            .data
+    Type:            SHT_PROGBITS
+    Flags:           [ SHF_ALLOC ]
+Symbols:
+  Global:
+    - Name:     foo
+      Type:     STT_FUNC
+      Section:  .text
+      Value:    0x1234
+    - Name:     bar
+      Type:     STT_OBJECT
+      Section:  .data
+      Value:    0xabcd
+
+# CHECK: Format:
+# 32-SAME:      ELF32-
+# 64-SAME:      ELF64-
+# I386-SAME:    i386
+# PPC-SAME:     ppc
+# PPC64-SAME:   ppc64
+# X86-64-SAME:  x86-64
+
+# I386-NEXT:    Arch: i386
+# PPC-NEXT:     Arch: powerpc
+# PPC64-NEXT:   Arch: powerpc64le
+# X86-64-NEXT:  Arch: x86_64
+
+# 32-NEXT:      AddressSize: 32bit
+# 64-NEXT:      AddressSize: 64bit
+
+# 32:     Class: 32-bit
+# 64:     Class: 64-bit
+# CHECK:  DataEncoding: LittleEndian
+
+# I386:   Machine: EM_386
+# PPC:    Machine: EM_PPC
+# PPC64:  Machine: EM_PPC64
+# X86-64: Machine: EM_X86_64
+
+# 32: HeaderSize: 52
+# 64: HeaderSize: 64
+
+# 32: SectionHeaderEntrySize: 40
+# 64: SectionHeaderEntrySize: 64
diff --git a/test/tools/llvm-objcopy/ELF/cross-arch-sections-symbols.test b/test/tools/llvm-objcopy/ELF/cross-arch-sections-symbols.test
new file mode 100644
index 0000000000000000000000000000000000000000..1d959a976350e0a5f7ac6797fffc5586f0ec77ca
--- /dev/null
+++ b/test/tools/llvm-objcopy/ELF/cross-arch-sections-symbols.test
@@ -0,0 +1,153 @@
+# RUN: yaml2obj %s > %t.o
+# Preserve input to verify it is not modified.
+# RUN: cp %t.o %t-copy.o
+# RUN: llvm-objcopy %t.o -O elf64-x86-64 %t.2.o
+# RUN: llvm-readobj --sections --symbols %t.2.o | FileCheck %s
+# RUN: cmp %t.o %t-copy.o
+
+!ELF
+FileHeader:
+  Class:           ELFCLASS32
+  Data:            ELFDATA2LSB
+  Type:            ET_EXEC
+  Machine:         EM_386
+Sections:
+  - Name:            .text
+    Type:            SHT_PROGBITS
+    Flags:           [ SHF_ALLOC, SHF_EXECINSTR ]
+    Size:            32
+  - Name:            .data
+    Type:            SHT_PROGBITS
+    Flags:           [ SHF_ALLOC ]
+    Content:         DEADBEEF
+    Size:            16
+Symbols:
+  Global:
+    - Name:     foo
+      Type:     STT_FUNC
+      Section:  .text
+      Value:    16
+      Size:     8
+    - Name:     bar
+      Type:     STT_OBJECT
+      Section:  .data
+      Size:     16
+
+# CHECK:      Sections [
+# CHECK-NEXT:   Section {
+# CHECK-NEXT:     Index: 0
+# CHECK-NEXT:     Name:  (0)
+# CHECK-NEXT:     Type: SHT_NULL (0x0)
+# CHECK-NEXT:     Flags [ (0x0)
+# CHECK-NEXT:     ]
+# CHECK-NEXT:     Address:
+# CHECK-NEXT:     Offset:
+# CHECK-NEXT:     Size: 0
+# CHECK-NEXT:     Link: 0
+# CHECK-NEXT:     Info: 0
+# CHECK-NEXT:     AddressAlignment: 0
+# CHECK-NEXT:     EntrySize: 0
+# CHECK-NEXT:   }
+# CHECK-NEXT:   Section {
+# CHECK-NEXT:     Index: 1
+# CHECK-NEXT:     Name: .text
+# CHECK-NEXT:     Type: SHT_PROGBITS (0x1)
+# CHECK-NEXT:     Flags [ (0x6)
+# CHECK-NEXT:       SHF_ALLOC (0x2)
+# CHECK-NEXT:       SHF_EXECINSTR (0x4)
+# CHECK-NEXT:     ]
+# CHECK-NEXT:     Address:
+# CHECK-NEXT:     Offset:
+# CHECK-NEXT:     Size: 32
+# CHECK-NEXT:     Link: 0
+# CHECK-NEXT:     Info: 0
+# CHECK-NEXT:     AddressAlignment: 0
+# CHECK-NEXT:     EntrySize: 0
+# CHECK-NEXT:   }
+# CHECK-NEXT:   Section {
+# CHECK-NEXT:     Index: 2
+# CHECK-NEXT:     Name: .data
+# CHECK-NEXT:     Type: SHT_PROGBITS (0x1)
+# CHECK-NEXT:     Flags [ (0x2)
+# CHECK-NEXT:       SHF_ALLOC (0x2)
+# CHECK-NEXT:     ]
+# CHECK-NEXT:     Address:
+# CHECK-NEXT:     Offset:
+# CHECK-NEXT:     Size: 16
+# CHECK-NEXT:     Link: 0
+# CHECK-NEXT:     Info: 0
+# CHECK-NEXT:     AddressAlignment: 0
+# CHECK-NEXT:     EntrySize: 0
+# CHECK-NEXT:   }
+# CHECK-NEXT:   Section {
+# CHECK-NEXT:     Index: 3
+# CHECK-NEXT:     Name: .symtab
+# CHECK-NEXT:     Type: SHT_SYMTAB (0x2)
+# CHECK-NEXT:     Flags [ (0x0)
+# CHECK-NEXT:     ]
+# CHECK-NEXT:     Address:
+# CHECK-NEXT:     Offset:
+# CHECK-NEXT:     Size: 72
+# CHECK-NEXT:     Link: 4
+# CHECK-NEXT:     Info: 1
+# CHECK-NEXT:     AddressAlignment: 8
+# CHECK-NEXT:     EntrySize: 24
+# CHECK-NEXT:   }
+# CHECK-NEXT:   Section {
+# CHECK-NEXT:     Index: 4
+# CHECK-NEXT:     Name: .strtab
+# CHECK-NEXT:     Type: SHT_STRTAB (0x3)
+# CHECK-NEXT:     Flags [ (0x0)
+# CHECK-NEXT:     ]
+# CHECK-NEXT:     Address:
+# CHECK-NEXT:     Offset:
+# CHECK-NEXT:     Size: 10
+# CHECK-NEXT:     Link: 0
+# CHECK-NEXT:     Info: 0
+# CHECK-NEXT:     AddressAlignment: 1
+# CHECK-NEXT:     EntrySize: 0
+# CHECK-NEXT:   }
+# CHECK-NEXT:   Section {
+# CHECK-NEXT:     Index: 5
+# CHECK-NEXT:     Name: .shstrtab
+# CHECK-NEXT:     Type: SHT_STRTAB (0x3)
+# CHECK-NEXT:     Flags [ (0x0)
+# CHECK-NEXT:     ]
+# CHECK-NEXT:     Address:
+# CHECK-NEXT:     Offset:
+# CHECK-NEXT:     Size: 39
+# CHECK-NEXT:     Link: 0
+# CHECK-NEXT:     Info: 0
+# CHECK-NEXT:     AddressAlignment: 1
+# CHECK-NEXT:     EntrySize: 0
+# CHECK-NEXT:   }
+# CHECK-NEXT: ]
+# CHECK-NEXT: Symbols [
+# CHECK-NEXT:   Symbol {
+# CHECK-NEXT:     Name:
+# CHECK-NEXT:     Value: 0x0
+# CHECK-NEXT:     Size: 0
+# CHECK-NEXT:     Binding: Local (0x0)
+# CHECK-NEXT:     Type: None (0x0)
+# CHECK-NEXT:     Other: 0
+# CHECK-NEXT:     Section: Undefined
+# CHECK-NEXT:   }
+# CHECK-NEXT:   Symbol {
+# CHECK-NEXT:     Name: foo
+# CHECK-NEXT:     Value: 0x10
+# CHECK-NEXT:     Size: 8
+# CHECK-NEXT:     Binding: Global (0x1)
+# CHECK-NEXT:     Type: Function (0x2)
+# CHECK-NEXT:     Other: 0
+# CHECK-NEXT:     Section: .text
+# CHECK-NEXT:   }
+# CHECK-NEXT:   Symbol {
+# CHECK-NEXT:     Name: bar
+# CHECK-NEXT:     Value: 0x0
+# CHECK-NEXT:     Size: 16
+# CHECK-NEXT:     Binding: Global (0x1)
+# CHECK-NEXT:     Type: Object (0x1)
+# CHECK-NEXT:     Other: 0
+# CHECK-NEXT:     Section: .data
+# CHECK-NEXT:   }
+# CHECK-NEXT: ]
diff --git a/test/tools/llvm-objcopy/ELF/discard-all.test b/test/tools/llvm-objcopy/ELF/discard-all.test
index db0686972620dba9a05afc609d67dc99e54716bf..cc676b59f5971c2e7544e97bccb84326fd282422 100644
--- a/test/tools/llvm-objcopy/ELF/discard-all.test
+++ b/test/tools/llvm-objcopy/ELF/discard-all.test
@@ -3,7 +3,7 @@
 # RUN: llvm-objcopy --discard-all %t %t2
 # Verify that llvm-objcopy has not modified the input.
 # RUN: cmp %t %t1
-# RUN: llvm-readobj -symbols %t2 | FileCheck %s
+# RUN: llvm-readobj --symbols %t2 | FileCheck %s
 
 # RUN: llvm-objcopy -x %t %t3
 # Verify that llvm-objcopy has not modified the input.
diff --git a/test/tools/llvm-objcopy/ELF/discard-locals-rel.test b/test/tools/llvm-objcopy/ELF/discard-locals-rel.test
new file mode 100644
index 0000000000000000000000000000000000000000..05e1acf6b1d8cb1fd5a16c158365ef51213def1a
--- /dev/null
+++ b/test/tools/llvm-objcopy/ELF/discard-locals-rel.test
@@ -0,0 +1,27 @@
+# RUN: yaml2obj %s > %t
+# RUN: not llvm-objcopy --discard-locals %t %t2 2>&1 | FileCheck %s
+
+!ELF
+FileHeader:
+  Class:           ELFCLASS64
+  Data:            ELFDATA2LSB
+  Type:            ET_REL
+  Machine:         EM_X86_64
+Sections:
+  - Name:            .text
+    Type:            SHT_PROGBITS
+  - Name:            .rel.text
+    Type:            SHT_REL
+    Link:            .symtab
+    Info:            .text
+    Relocations:
+      - Offset: 0x1000
+        Symbol: .L.rel
+        Type:   R_X86_64_PC32
+Symbols:
+  Local:
+    - Name:     .L.rel
+      Type:     STT_FUNC
+      Section:  .text
+
+# CHECK: not stripping symbol '.L.rel' because it is named in a relocation.
diff --git a/test/tools/llvm-objcopy/ELF/discard-locals.test b/test/tools/llvm-objcopy/ELF/discard-locals.test
new file mode 100644
index 0000000000000000000000000000000000000000..be489a2f0de93ec05176500061b72bb982981ca5
--- /dev/null
+++ b/test/tools/llvm-objcopy/ELF/discard-locals.test
@@ -0,0 +1,121 @@
+# RUN: yaml2obj %s > %t
+# RUN: cp %t %t1
+# RUN: llvm-objcopy --discard-locals %t %t2
+# Verify that llvm-objcopy has not modified the input.
+# RUN: cmp %t %t1
+# RUN: llvm-readobj --symbols %t2 | FileCheck %s
+
+# RUN: llvm-objcopy -X %t %t3
+# Verify that llvm-objcopy has not modified the input.
+# RUN: cmp %t %t1
+# RUN: cmp %t2 %t3
+
+# Verify that llvm-strip modifies the symbol table the same way.
+
+# RUN: cp %t %t4
+# RUN: llvm-strip --discard-locals %t4
+# RUN: cmp %t2 %t4
+
+# RUN: cp %t %t5
+# RUN: llvm-strip -X %t5
+# RUN: cmp %t2 %t5
+
+!ELF
+FileHeader:
+  Class:           ELFCLASS64
+  Data:            ELFDATA2LSB
+  Type:            ET_REL
+  Machine:         EM_X86_64
+Sections:
+  - Name:            .text
+    Type:            SHT_PROGBITS
+  - Name:            .LLVM.Custom.Section
+    Type:            SHT_PROGBITS
+Symbols:
+  Local:
+    - Name:     Local
+      Type:     STT_FUNC
+      Section:  .text
+    - Name:     .L.LocalSection
+      Type:     STT_SECTION
+      Section:  .text
+    - Type:     STT_SECTION
+      Section:  .LLVM.Custom.Section
+    - Name:     .L.LocalFile
+      Type:     STT_FILE
+    - Name:     .L.str
+      Type:     STT_OBJECT
+      Section:  .text
+    - Name:     .L.undefined
+    - Name:     .L.abs
+      Index:    SHN_ABS
+  Global:
+    - Name:     .L.Global
+      Type:     STT_FUNC
+      Section:  .text
+
+# CHECK:      Symbols [
+# CHECK-NEXT:   Symbol {
+# CHECK-NEXT:     Name:
+# CHECK-NEXT:     Value: 0x0
+# CHECK-NEXT:     Size: 0
+# CHECK-NEXT:     Binding: Local
+# CHECK-NEXT:     Type: None
+# CHECK-NEXT:     Other: 0
+# CHECK-NEXT:     Section: Undefined
+# CHECK-NEXT:   }
+# CHECK-NEXT:   Symbol {
+# CHECK-NEXT:     Name: Local
+# CHECK-NEXT:     Value:
+# CHECK-NEXT:     Size:
+# CHECK-NEXT:     Binding: Local
+# CHECK-NEXT:     Type: Function
+# CHECK-NEXT:     Other:
+# CHECK-NEXT:     Section: .text
+# CHECK-NEXT:   }
+# CHECK-NEXT:   Symbol {
+# CHECK-NEXT:     Name: .L.LocalSection
+# CHECK-NEXT:     Value:
+# CHECK-NEXT:     Size:
+# CHECK-NEXT:     Binding: Local
+# CHECK-NEXT:     Type: Section
+# CHECK-NEXT:     Other:
+# CHECK-NEXT:     Section: .text
+# CHECK-NEXT:   }
+# CHECK-NEXT:   Symbol {
+# CHECK-NEXT:     Name:
+# CHECK-NEXT:     Value:
+# CHECK-NEXT:     Size:
+# CHECK-NEXT:     Binding: Local
+# CHECK-NEXT:     Type: Section
+# CHECK-NEXT:     Other:
+# CHECK-NEXT:     Section: .LLVM.Custom.Section
+# CHECK-NEXT:   }
+# CHECK-NEXT:   Symbol {
+# CHECK-NEXT:     Name: .L.LocalFile
+# CHECK-NEXT:     Value:
+# CHECK-NEXT:     Size:
+# CHECK-NEXT:     Binding: Local
+# CHECK-NEXT:     Type: File
+# CHECK-NEXT:     Other:
+# CHECK-NEXT:     Section: Undefined
+# CHECK-NEXT:   }
+# CHECK-NEXT:   Symbol {
+# CHECK-NEXT:     Name: .L.undefined
+# CHECK-NEXT:     Value:
+# CHECK-NEXT:     Size:
+# CHECK-NEXT:     Binding: Local
+# CHECK-NEXT:     Type: None
+# CHECK-NEXT:     Other:
+# CHECK-NEXT:     Section: Undefined
+# CHECK-NEXT:   }
+# CHECK-NEXT:   Symbol {
+# CHECK-NEXT:     Name: .L.Global
+# CHECK-NEXT:     Value:
+# CHECK-NEXT:     Size:
+# CHECK-NEXT:     Binding: Global
+# CHECK-NEXT:     Type: Function
+# CHECK-NEXT:     Other:
+# CHECK-NEXT:     Section: .text
+# CHECK-NEXT:   }
+# CHECK-NEXT: ]
diff --git a/test/tools/llvm-objcopy/ELF/discard-mix-local-and-all.test b/test/tools/llvm-objcopy/ELF/discard-mix-local-and-all.test
new file mode 100644
index 0000000000000000000000000000000000000000..f9e0613eecfae5021bf9cbb959c449c67d8b25a5
--- /dev/null
+++ b/test/tools/llvm-objcopy/ELF/discard-mix-local-and-all.test
@@ -0,0 +1,107 @@
+# RUN: yaml2obj %s > %t
+# Establish baseline objects for further checks. --discard-locals only discards
+# compiler-generated local symbols (starting with .L), --discard-all discards
+# all regular local symbols.
+# RUN: llvm-objcopy %t %t-discard-none
+# RUN: llvm-readobj --symbols %t-discard-none | FileCheck %s --check-prefixes=CHECK,LOCAL,COMPILER-LOCAL
+# RUN: llvm-objcopy --discard-all %t %t-discard-all
+# RUN: llvm-readobj --symbols %t-discard-all | FileCheck %s
+# RUN: llvm-objcopy --discard-locals %t %t-discard-locals
+# RUN: llvm-readobj --symbols %t-discard-locals | FileCheck %s --check-prefixes=CHECK,LOCAL
+
+# When mixing --discard-all and --discard-locals, the last one wins.
+# RUN: llvm-objcopy --discard-all --discard-locals %t %t.1.o
+# RUN: cmp %t.1.o %t-discard-locals
+# RUN: llvm-objcopy --discard-locals --discard-all %t %t.2.o
+# RUN: cmp %t.2.o %t-discard-all
+# RUN: llvm-objcopy -x -X %t %t.3.o
+# RUN: cmp %t.3.o %t-discard-locals
+# RUN: llvm-objcopy -X -x %t %t.4.o
+# RUN: cmp %t.4.o %t-discard-all
+# RUN: llvm-objcopy -x -X -x -X %t %t.5.o
+# RUN: cmp %t.5.o %t-discard-locals
+# RUN: llvm-objcopy -X -x -X -x %t %t.6.o
+# RUN: cmp %t.6.o %t-discard-all
+# RUN: llvm-objcopy -X -x -X -x --discard-locals %t %t.7.o
+# RUN: cmp %t.7.o %t-discard-locals
+# RUN: llvm-objcopy -X -x -X -x --discard-all %t %t.8.o
+# RUN: cmp %t.8.o %t-discard-all
+
+# llvm-strip works in the same way.
+# RUN: llvm-strip --discard-all --discard-locals %t -o %t.9.o
+# RUN: cmp %t.9.o %t-discard-locals
+# RUN: llvm-strip --discard-locals --discard-all %t -o %t.10.o
+# RUN: cmp %t.10.o %t-discard-all
+# RUN: llvm-strip -x -X %t -o %t.11.o
+# RUN: cmp %t.11.o %t-discard-locals
+# RUN: llvm-strip -X -x %t -o %t.12.o
+# RUN: cmp %t.12.o %t-discard-all
+# RUN: llvm-strip -x -X -x -X %t -o %t.13.o
+# RUN: cmp %t.13.o %t-discard-locals
+# RUN: llvm-strip -X -x -X -x %t -o %t.14.o
+# RUN: cmp %t.14.o %t-discard-all
+# RUN: llvm-strip -X -x -X -x --discard-locals %t -o %t.15.o
+# RUN: cmp %t.15.o %t-discard-locals
+# RUN: llvm-strip -X -x -X -x --discard-all %t -o %t.16.o
+# RUN: cmp %t.16.o %t-discard-all
+
+!ELF
+FileHeader:
+  Class:           ELFCLASS64
+  Data:            ELFDATA2LSB
+  Type:            ET_REL
+  Machine:         EM_X86_64
+Sections:
+  - Name:            .text
+    Type:            SHT_PROGBITS
+Symbols:
+  Local:
+    - Name:     Local
+      Type:     STT_FUNC
+      Section:  .text
+    - Name:     .L.str
+      Type:     STT_OBJECT
+      Section:  .text
+  Global:
+    - Name:     Global
+      Type:     STT_FUNC
+      Section:  .text
+
+# CHECK:               Symbols [
+# CHECK-NEXT:            Symbol {
+# CHECK-NEXT:              Name:
+# CHECK-NEXT:              Value: 0x0
+# CHECK-NEXT:              Size: 0
+# CHECK-NEXT:              Binding: Local
+# CHECK-NEXT:              Type: None
+# CHECK-NEXT:              Other: 0
+# CHECK-NEXT:              Section: Undefined
+# CHECK-NEXT:            }
+# LOCAL-NEXT:            Symbol {
+# LOCAL-NEXT:              Name: Local
+# LOCAL-NEXT:              Value:
+# LOCAL-NEXT:              Size:
+# LOCAL-NEXT:              Binding: Local
+# LOCAL-NEXT:              Type: Function
+# LOCAL-NEXT:              Other:
+# LOCAL-NEXT:              Section: .text
+# LOCAL-NEXT:            }
+# COMPILER-LOCAL-NEXT:   Symbol {
+# COMPILER-LOCAL-NEXT:     Name: .L.str
+# COMPILER-LOCAL-NEXT:     Value:
+# COMPILER-LOCAL-NEXT:     Size:
+# COMPILER-LOCAL-NEXT:     Binding: Local
+# COMPILER-LOCAL-NEXT:     Type: Object
+# COMPILER-LOCAL-NEXT:     Other:
+# COMPILER-LOCAL-NEXT:     Section: .text
+# COMPILER-LOCAL-NEXT:   }
+# CHECK-NEXT:            Symbol {
+# CHECK-NEXT:              Name: Global
+# CHECK-NEXT:              Value:
+# CHECK-NEXT:              Size:
+# CHECK-NEXT:              Binding: Global
+# CHECK-NEXT:              Type: Function
+# CHECK-NEXT:              Other:
+# CHECK-NEXT:              Section: .text
+# CHECK-NEXT:            }
+# CHECK-NEXT:          ]
diff --git a/test/tools/llvm-objcopy/ELF/drawf-fission.test b/test/tools/llvm-objcopy/ELF/drawf-fission.test
index 112bffbc891bef0d9681d33e4ba1807de7a331d7..76f74ce7e80519991dd6c846f64f13afada5a079 100644
--- a/test/tools/llvm-objcopy/ELF/drawf-fission.test
+++ b/test/tools/llvm-objcopy/ELF/drawf-fission.test
@@ -1,8 +1,8 @@
-# RUN: llvm-objcopy -extract-dwo %p/Inputs/dwarf.dwo %t
-# RUN: llvm-objcopy -strip-dwo %p/Inputs/dwarf.dwo %t2
-# RUN: llvm-objcopy -split-dwo=%t3 %p/Inputs/dwarf.dwo %t4
-# RUN: llvm-readobj -file-headers -sections %t | FileCheck %s -check-prefix=DWARF
-# RUN: llvm-readobj -file-headers -sections %t2 | FileCheck %s -check-prefix=STRIP
+# RUN: llvm-objcopy --extract-dwo %p/Inputs/dwarf.dwo %t
+# RUN: llvm-objcopy --strip-dwo %p/Inputs/dwarf.dwo %t2
+# RUN: llvm-objcopy --split-dwo=%t3 %p/Inputs/dwarf.dwo %t4
+# RUN: llvm-readobj --file-headers --sections %t | FileCheck %s --check-prefix=DWARF
+# RUN: llvm-readobj --file-headers --sections %t2 | FileCheck %s --check-prefix=STRIP
 # RUN: diff %t %t3
 # RUN: diff %t2 %t4
 
diff --git a/test/tools/llvm-objcopy/ELF/dump-section.test b/test/tools/llvm-objcopy/ELF/dump-section.test
index 8e66f65e04a7b452f57bca2016f74af0fe450f50..7a25f9cf926f32142c9e79d384760f527384db50 100644
--- a/test/tools/llvm-objcopy/ELF/dump-section.test
+++ b/test/tools/llvm-objcopy/ELF/dump-section.test
@@ -1,6 +1,6 @@
 # RUN: yaml2obj %s > %t
 # RUN: llvm-objcopy -O binary -j .text %t %t2
-# RUN: llvm-objcopy -O binary -only-section .text %t %t3
+# RUN: llvm-objcopy -O binary --only-section .text %t %t3
 # RUN: llvm-objcopy --dump-section .text=%t4 %t %t5
 # RUN: llvm-objcopy --dump-section .foo=%t6 %t %t7
 # RUN: not llvm-objcopy --dump-section .bar=%t8 %t %t9 2>&1 | FileCheck %s --check-prefix=NOBITS
diff --git a/test/tools/llvm-objcopy/ELF/dynamic-relocations.test b/test/tools/llvm-objcopy/ELF/dynamic-relocations.test
index e544e2dd8ef6960a9983bc3d8865c5b8906563e0..1c0fabc7806a01eaeebdcb15c6188d209ff2ab32 100644
--- a/test/tools/llvm-objcopy/ELF/dynamic-relocations.test
+++ b/test/tools/llvm-objcopy/ELF/dynamic-relocations.test
@@ -1,5 +1,5 @@
 # RUN: llvm-objcopy %p/Inputs/dynrel.elf %t
-# RUN: llvm-readobj -sections -section-data %t | FileCheck %s
+# RUN: llvm-readobj --sections --section-data %t | FileCheck %s
 
 #CHECK:         Name: .rela.plt
 #CHECK-NEXT:    Type: SHT_RELA
diff --git a/test/tools/llvm-objcopy/ELF/dynamic.test b/test/tools/llvm-objcopy/ELF/dynamic.test
index 3e9ea2069f94279eaf6e24fff6b5b34c52faec48..5eb8809b41e0ad631657c2e4f7571ac3acb5d4db 100644
--- a/test/tools/llvm-objcopy/ELF/dynamic.test
+++ b/test/tools/llvm-objcopy/ELF/dynamic.test
@@ -1,6 +1,6 @@
 # RUN: llvm-objcopy %p/Inputs/dynamic.so %t
-# RUN: llvm-readobj -dynamic-table %t | FileCheck %s
-# RUN: llvm-readobj -sections %t | FileCheck -check-prefix=LINK %s
+# RUN: llvm-readobj --dynamic-table %t | FileCheck %s
+# RUN: llvm-readobj --sections %t | FileCheck --check-prefix=LINK %s
 
 #CHECK: DynamicSection [
 #CHECK-NEXT:  Tag                Type                 Name/Value
diff --git a/test/tools/llvm-objcopy/ELF/dynstr.test b/test/tools/llvm-objcopy/ELF/dynstr.test
index 6e193065a77765c419fb4daa0dec8aba3fa8ab01..68ec586649eb34ba96056b346a100fed4f84ffe6 100644
--- a/test/tools/llvm-objcopy/ELF/dynstr.test
+++ b/test/tools/llvm-objcopy/ELF/dynstr.test
@@ -1,6 +1,6 @@
 # RUN: yaml2obj %s > %t
 # RUN: llvm-objcopy %t %t2
-# RUN: llvm-readobj -sections -section-data %t2 | FileCheck %s
+# RUN: llvm-readobj --sections --section-data %t2 | FileCheck %s
 
 !ELF
 FileHeader:
diff --git a/test/tools/llvm-objcopy/ELF/dynsym.test b/test/tools/llvm-objcopy/ELF/dynsym.test
index b7d09536f6904e0848708c66a99e35e0adc6cfae..6a7eef38cd9ec5358a4f8b38e1678da8fdb4d41a 100644
--- a/test/tools/llvm-objcopy/ELF/dynsym.test
+++ b/test/tools/llvm-objcopy/ELF/dynsym.test
@@ -1,6 +1,6 @@
 # RUN: llvm-objcopy %p/Inputs/dynsym.so %t
-# RUN: llvm-readobj -dyn-symbols %t | FileCheck %s
-# RUN: llvm-readobj -sections %t | FileCheck -check-prefix=LINK %s
+# RUN: llvm-readobj --dyn-symbols %t | FileCheck %s
+# RUN: llvm-readobj --sections %t | FileCheck --check-prefix=LINK %s
 
 #LINK:         Name: .dynsym
 #LINK-NEXT:    Type: SHT_DYNSYM
@@ -17,7 +17,7 @@
 
 #CHECK: DynamicSymbols [
 #CHECK-NEXT:  Symbol {
-#CHECK-NEXT:    Name: @ (0)
+#CHECK-NEXT:    Name: (0)
 #CHECK-NEXT:    Value: 0x0
 #CHECK-NEXT:    Size: 0
 #CHECK-NEXT:    Binding: Local
@@ -26,7 +26,7 @@
 #CHECK-NEXT:    Section: Undefined (0x0)
 #CHECK-NEXT:  }
 #CHECK-NEXT:  Symbol {
-#CHECK-NEXT:    Name: bang@ (1)
+#CHECK-NEXT:    Name: bang (1)
 #CHECK-NEXT:    Value: 0x4000
 #CHECK-NEXT:    Size: 8
 #CHECK-NEXT:    Binding: Global (0x1)
@@ -35,7 +35,7 @@
 #CHECK-NEXT:    Section: .bss (0x7)
 #CHECK-NEXT:  }
 #CHECK-NEXT:  Symbol {
-#CHECK-NEXT:    Name: bar@ (6)
+#CHECK-NEXT:    Name: bar (6)
 #CHECK-NEXT:    Value: 0x1001
 #CHECK-NEXT:    Size: 0
 #CHECK-NEXT:    Binding: Global (0x1)
@@ -44,7 +44,7 @@
 #CHECK-NEXT:    Section: .text (0x4)
 #CHECK-NEXT:  }
 #CHECK-NEXT:  Symbol {
-#CHECK-NEXT:    Name: baz@ (10)
+#CHECK-NEXT:    Name: baz (10)
 #CHECK-NEXT:    Value: 0x2000
 #CHECK-NEXT:    Size: 0
 #CHECK-NEXT:    Binding: Global (0x1)
@@ -53,7 +53,7 @@
 #CHECK-NEXT:    Section: .data (0x5)
 #CHECK-NEXT:  }
 #CHECK-NEXT:  Symbol {
-#CHECK-NEXT:    Name: foo@ (14)
+#CHECK-NEXT:    Name: foo (14)
 #CHECK-NEXT:    Value: 0x1000
 #CHECK-NEXT:    Size: 0
 #CHECK-NEXT:    Binding: Global (0x1)
diff --git a/test/tools/llvm-objcopy/ELF/elf32be.test b/test/tools/llvm-objcopy/ELF/elf32be.test
index 786c862807e39350fd00c6c0306fd982f97803a7..374b7994ae33db78fdb0edcf232f11dba78e5e75 100644
--- a/test/tools/llvm-objcopy/ELF/elf32be.test
+++ b/test/tools/llvm-objcopy/ELF/elf32be.test
@@ -1,6 +1,6 @@
 # RUN: yaml2obj %s > %t
 # RUN: llvm-objcopy %t %t2
-# RUN: llvm-readobj -file-headers -sections %t2 | FileCheck %s
+# RUN: llvm-readobj --file-headers --sections %t2 | FileCheck %s
 
 !ELF
 FileHeader:
diff --git a/test/tools/llvm-objcopy/ELF/elf32le.test b/test/tools/llvm-objcopy/ELF/elf32le.test
index ed7d5ff5dbbf4fe6db6155f9c578bb94871af348..b2cb40ce14119e908a4a185f5965dfece0874e35 100644
--- a/test/tools/llvm-objcopy/ELF/elf32le.test
+++ b/test/tools/llvm-objcopy/ELF/elf32le.test
@@ -1,6 +1,6 @@
 # RUN: yaml2obj %s > %t
 # RUN: llvm-objcopy %t %t2
-# RUN: llvm-readobj -file-headers -sections %t2 | FileCheck %s
+# RUN: llvm-readobj --file-headers --sections %t2 | FileCheck %s
 
 !ELF
 FileHeader:
diff --git a/test/tools/llvm-objcopy/ELF/elf64be.test b/test/tools/llvm-objcopy/ELF/elf64be.test
index 49e707ec59dff7e6b97e534f766710d379c6a2a3..97977b0c195e8b5a5f89c8fbbdfe40b82acb32fb 100644
--- a/test/tools/llvm-objcopy/ELF/elf64be.test
+++ b/test/tools/llvm-objcopy/ELF/elf64be.test
@@ -1,6 +1,6 @@
 # RUN: yaml2obj %s > %t
 # RUN: llvm-objcopy %t %t2
-# RUN: llvm-readobj -file-headers -sections %t2 | FileCheck %s
+# RUN: llvm-readobj --file-headers --sections %t2 | FileCheck %s
 
 !ELF
 FileHeader:
diff --git a/test/tools/llvm-objcopy/ELF/empty-section.test b/test/tools/llvm-objcopy/ELF/empty-section.test
index 8847327a06957cde6dbce098208035c0607cc4fd..5c55383ac09f8128cfd09f9e4f4ce0fbc768ff7a 100644
--- a/test/tools/llvm-objcopy/ELF/empty-section.test
+++ b/test/tools/llvm-objcopy/ELF/empty-section.test
@@ -1,6 +1,6 @@
 # RUN: yaml2obj %s > %t
 # RUN: llvm-objcopy %t %t2
-# RUN: llvm-readobj -sections %t2 | FileCheck %s
+# RUN: llvm-readobj --sections %t2 | FileCheck %s
 
 !ELF
 FileHeader:
diff --git a/test/tools/llvm-objcopy/ELF/explicit-keep-remove.test b/test/tools/llvm-objcopy/ELF/explicit-keep-remove.test
index fea708edfd4936a8ce968f2cff56a733d5d1b3bd..98ad3ae7592de7a899e9ff329a5bf505aa96805d 100644
--- a/test/tools/llvm-objcopy/ELF/explicit-keep-remove.test
+++ b/test/tools/llvm-objcopy/ELF/explicit-keep-remove.test
@@ -1,6 +1,6 @@
 # RUN: yaml2obj %s > %t
-# RUN: llvm-objcopy -R=.test -keep-section=.test %t %t2
-# RUN: llvm-readobj -file-headers -sections %t2 | FileCheck %s
+# RUN: llvm-objcopy -R=.test --keep-section=.test %t %t2
+# RUN: llvm-readobj --file-headers --sections %t2 | FileCheck %s
 
 !ELF
 FileHeader:
diff --git a/test/tools/llvm-objcopy/ELF/explicit-only-section-remove.test b/test/tools/llvm-objcopy/ELF/explicit-only-section-remove.test
index 15a315906abf80079fed5fba7f08c7f5359da482..8a152e57a5d73461903758b78df272df21dbe178 100644
--- a/test/tools/llvm-objcopy/ELF/explicit-only-section-remove.test
+++ b/test/tools/llvm-objcopy/ELF/explicit-only-section-remove.test
@@ -1,6 +1,6 @@
 # RUN: yaml2obj %s > %t
-# RUN: llvm-objcopy -R=.test -only-section=.test %t %t2
-# RUN: llvm-readobj -file-headers -sections %t2 | FileCheck %s
+# RUN: llvm-objcopy -R=.test --only-section=.test %t %t2
+# RUN: llvm-readobj --file-headers --sections %t2 | FileCheck %s
 
 !ELF
 FileHeader:
diff --git a/test/tools/llvm-objcopy/ELF/fail-no-output-directory.test b/test/tools/llvm-objcopy/ELF/fail-no-output-directory.test
index f66b2e09fceba34219c653ed574cd60a5b62299c..0eb39b9022b47381d75308a1e84f7e6affe569d5 100644
--- a/test/tools/llvm-objcopy/ELF/fail-no-output-directory.test
+++ b/test/tools/llvm-objcopy/ELF/fail-no-output-directory.test
@@ -1,6 +1,7 @@
 # RUN: yaml2obj %s > %t
 # RUN: not llvm-objcopy %t no/such/dir 2>&1 | FileCheck %s
-# CHECK: failed to open no/such/dir:
+# Don't check "No such file" which is OS dependent.
+# CHECK: 'no/such/dir':
 
 !ELF
 FileHeader:
diff --git a/test/tools/llvm-objcopy/ELF/globalize.test b/test/tools/llvm-objcopy/ELF/globalize.test
index 4941cf12e921431fe626e44b95348b9666fe709f..bd7f38ee52dded3e2d84c5bd1f1df53c6d896fe5 100644
--- a/test/tools/llvm-objcopy/ELF/globalize.test
+++ b/test/tools/llvm-objcopy/ELF/globalize.test
@@ -3,7 +3,19 @@
 # RUN:   --globalize-symbol Local \
 # RUN:   --globalize-symbol Weak \
 # RUN:   --globalize-symbol WeakUndef %t %t2
-# RUN: llvm-readobj -symbols %t2 | FileCheck %s
+# RUN: llvm-readobj --symbols %t2 | FileCheck %s
+# RUN: llvm-objcopy --regex --globalize-symbol='.*' %t %t3
+# RUN: cmp %t2 %t3
+# RUN: echo " Global # global" > %t-list.txt
+# RUN: echo "Local" >> %t-list.txt
+# RUN: echo "Weak" >> %t-list.txt
+# RUN: echo "WeakUndef" >> %t-list.txt
+# RUN: echo " # comment " >> %t-list.txt
+# RUN: llvm-objcopy --globalize-symbols %t-list.txt %t %t4
+# RUN: cmp %t2 %t4
+# RUN: echo ".+ # .+ " > %t-list2.txt
+# RUN: llvm-objcopy --regex --globalize-symbols %t-list2.txt %t %t5
+# RUN: cmp %t2 %t5
 
 !ELF
 FileHeader:
diff --git a/test/tools/llvm-objcopy/ELF/group-big-endian.test b/test/tools/llvm-objcopy/ELF/group-big-endian.test
index d5a0260711328aea09b96e240b638ca89d5acb93..dd5ca88144baf3563390564d8a9a5d23ce1769f1 100644
--- a/test/tools/llvm-objcopy/ELF/group-big-endian.test
+++ b/test/tools/llvm-objcopy/ELF/group-big-endian.test
@@ -1,6 +1,6 @@
 # RUN: yaml2obj %s > %t
-# RUN: llvm-objcopy -remove-section=.text.bar %t %t2
-# RUN: llvm-readobj -elf-section-groups %t2 | FileCheck %s
+# RUN: llvm-objcopy --remove-section=.text.bar %t %t2
+# RUN: llvm-readobj --elf-section-groups %t2 | FileCheck %s
 
 # In this test the section .text.bar is getting removed, as a result, 
 # the indices of the sections which go after .text.bar will change, 
diff --git a/test/tools/llvm-objcopy/ELF/group-unchanged.test b/test/tools/llvm-objcopy/ELF/group-unchanged.test
index 9e86172bad54d136d0a4f9a970708fd48592245e..31fba11adca8dabb2c906de00f864ba99f9a7545 100644
--- a/test/tools/llvm-objcopy/ELF/group-unchanged.test
+++ b/test/tools/llvm-objcopy/ELF/group-unchanged.test
@@ -1,6 +1,6 @@
 # RUN: yaml2obj %s > %t
-# RUN: llvm-objcopy -remove-section=.text.bar %t %t2
-# RUN: llvm-readobj -elf-section-groups %t2 | FileCheck %s
+# RUN: llvm-objcopy --remove-section=.text.bar %t %t2
+# RUN: llvm-readobj --elf-section-groups %t2 | FileCheck %s
 
 # In this test the section .text.bar is getting removed, since this section
 # goes after all the sections comprising a group, the content of the
diff --git a/test/tools/llvm-objcopy/ELF/group.test b/test/tools/llvm-objcopy/ELF/group.test
index 4023f1228f015744c5e57f065d4414ef94030c06..2e058f5d6a5ce2f1e11ed4a44022efc0cb1c973e 100644
--- a/test/tools/llvm-objcopy/ELF/group.test
+++ b/test/tools/llvm-objcopy/ELF/group.test
@@ -1,6 +1,6 @@
 # RUN: yaml2obj %s > %t
-# RUN: llvm-objcopy -remove-section=.text.bar %t %t2
-# RUN: llvm-readobj -elf-section-groups %t2 | FileCheck %s
+# RUN: llvm-objcopy --remove-section=.text.bar %t %t2
+# RUN: llvm-readobj --elf-section-groups %t2 | FileCheck %s
 
 # In this test the section .text.bar is getting removed, as a result, 
 # the indices of the sections which go after .text.bar will change, 
diff --git a/test/tools/llvm-objcopy/ELF/help-message.test b/test/tools/llvm-objcopy/ELF/help-message.test
index 504ce215c533de194e559880298f99a82ca2d866..2dc92f673108f4d6007db5fd744df7a69f97a0d1 100644
--- a/test/tools/llvm-objcopy/ELF/help-message.test
+++ b/test/tools/llvm-objcopy/ELF/help-message.test
@@ -1,15 +1,19 @@
+# RUN: llvm-objcopy -help | FileCheck --check-prefix=OBJCOPY-USAGE %s
 # RUN: llvm-objcopy --help | FileCheck --check-prefix=OBJCOPY-USAGE %s
 # RUN: not llvm-objcopy 2>&1 | FileCheck --check-prefix=OBJCOPY-USAGE %s
 # RUN: not llvm-objcopy -abcabc 2>&1 | FileCheck --check-prefix=UNKNOWN-ARG %s
-# RUN: not llvm-objcopy -strip-debug 2>&1 | FileCheck %s --check-prefix=NO-INPUT-FILES
+# RUN: not llvm-objcopy --abcabc 2>&1 | FileCheck --check-prefix=UNKNOWN-ARG %s
+# RUN: not llvm-objcopy --strip-debug 2>&1 | FileCheck %s --check-prefix=NO-INPUT-FILES
 
+# RUN: llvm-strip -help | FileCheck --check-prefix=STRIP-USAGE %s
 # RUN: llvm-strip --help | FileCheck --check-prefix=STRIP-USAGE %s
 # RUN: not llvm-strip 2>&1 | FileCheck --check-prefix=STRIP-USAGE %s
 # RUN: not llvm-strip -abcabc 2>&1 | FileCheck --check-prefix=UNKNOWN-ARG %s
-# RUN: not llvm-strip -strip-debug 2>&1 | FileCheck %s --check-prefix=NO-INPUT-FILES
+# RUN: not llvm-strip --abcabc 2>&1 | FileCheck --check-prefix=UNKNOWN-ARG %s
+# RUN: not llvm-strip --strip-debug 2>&1 | FileCheck %s --check-prefix=NO-INPUT-FILES
 
 
 # OBJCOPY-USAGE:  USAGE: llvm-objcopy
 # STRIP-USAGE:    USAGE: llvm-strip
-# UNKNOWN-ARG:    unknown argument '-abcabc'
+# UNKNOWN-ARG:    unknown argument '{{-+}}abcabc'
 # NO-INPUT-FILES: No input file specified
diff --git a/test/tools/llvm-objcopy/ELF/keep-file-symbols.test b/test/tools/llvm-objcopy/ELF/keep-file-symbols.test
index 85546887ae582c05cd9a3c4c744e17f1815b6445..3655f39512d603d93cd8888fa9c189c645aaa696 100644
--- a/test/tools/llvm-objcopy/ELF/keep-file-symbols.test
+++ b/test/tools/llvm-objcopy/ELF/keep-file-symbols.test
@@ -1,8 +1,13 @@
 # RUN: yaml2obj %s > %t
 # RUN: llvm-objcopy --strip-all --keep-file-symbols %t %t2
-# RUN: llvm-readobj -symbols %t2 | FileCheck %s --check-prefix=STRIPALL
+# RUN: llvm-readobj --symbols %t2 | FileCheck %s --check-prefix=STRIPALL
 # RUN: llvm-objcopy --keep-file-symbols --strip-symbol foo %t %t2
-# RUN: llvm-readobj -symbols %t2 | FileCheck %s --check-prefix=STRIP
+# RUN: llvm-readobj --symbols %t2 | FileCheck %s --check-prefix=STRIP
+
+# RUN: llvm-strip --keep-file-symbols %t -o %t3
+# RUN: llvm-readobj --symbols %t3 | FileCheck %s --check-prefix=STRIPALL
+# RUN: llvm-strip --keep-file-symbols --strip-symbol foo %t -o %t4
+# RUN: llvm-readobj --symbols %t4 | FileCheck %s --check-prefix=STRIP
 
 !ELF
 FileHeader:
diff --git a/test/tools/llvm-objcopy/ELF/keep-global-symbols-mix-globalize.test b/test/tools/llvm-objcopy/ELF/keep-global-symbols-mix-globalize.test
index 2de5807173cc73241e8a33098ca2032846a8ee6c..bc5cba6f921d8a034c26772aa86cbd725a1526a5 100644
--- a/test/tools/llvm-objcopy/ELF/keep-global-symbols-mix-globalize.test
+++ b/test/tools/llvm-objcopy/ELF/keep-global-symbols-mix-globalize.test
@@ -11,7 +11,7 @@
 # RUN:   --globalize-symbol Global1 \
 # RUN:   --keep-global-symbol Global2 \
 # RUN:   %t.o %t.2.o
-# RUN: llvm-readobj -elf-output-style=GNU -symbols %t.2.o | FileCheck %s
+# RUN: llvm-readobj --elf-output-style=GNU --symbols %t.2.o | FileCheck %s
 
 !ELF
 FileHeader:
diff --git a/test/tools/llvm-objcopy/ELF/keep-global-symbols.test b/test/tools/llvm-objcopy/ELF/keep-global-symbols.test
index 8ce1d7f3a2afa8afd90326e7d28e635125bc776e..e0b36bad40269076f3a2d2511d3373712cf0b757 100644
--- a/test/tools/llvm-objcopy/ELF/keep-global-symbols.test
+++ b/test/tools/llvm-objcopy/ELF/keep-global-symbols.test
@@ -28,6 +28,7 @@
 # RUN: echo "Unknown" >> %t-globals2.txt
 # RUN: echo "   " >> %t-globals2.txt
 # RUN: echo "# File with no symbols" > %t-globals3.txt
+# RUN: echo "^Global[0-9]+([[:space:]]Global6)*$" > %t-globals-regexp.txt
 
 # RUN: llvm-objcopy \
 # RUN:   -G Global1 \
@@ -39,7 +40,10 @@
 # RUN:   -G Weak1 \
 # RUN:   --globalize-symbol Weak2 \
 # RUN:   %t.o %t.2.o
-# RUN: llvm-readobj -elf-output-style=GNU -symbols %t.2.o | FileCheck %s
+# RUN: llvm-readobj --elf-output-style=GNU --symbols %t.2.o | FileCheck %s
+
+# RUN: llvm-objcopy  --regex --keep-global-symbols %t-globals-regexp.txt %t.o %t.3.o
+# RUN: llvm-readobj --elf-output-style=GNU --symbols %t.3.o | FileCheck %s --check-prefix=REGEXP
 
 !ELF
 FileHeader:
@@ -97,3 +101,12 @@ Symbols:
 # CHECK-NEXT:     11: {{.*}}  GLOBAL {{.*}} UND Global7
 # CHECK-NEXT:     12: {{.*}}  WEAK   {{.*}} Weak1
 # CHECK-NEXT:     13: {{.*}}  GLOBAL {{.*}} Weak2
+
+# REGEXP:           6: {{.*}}  GLOBAL {{.*}} Global1
+# REGEXP-NEXT:      7: {{.*}}  GLOBAL {{.*}} Global2
+# REGEXP-NEXT:      8: {{.*}}  GLOBAL {{.*}} Global3
+# REGEXP-NEXT:      9: {{.*}}  GLOBAL {{.*}} Global4
+# REGEXP-NEXT:     10: {{.*}}  GLOBAL {{.*}} Global5
+# REGEXP-NEXT:     11: {{.*}}  GLOBAL {{.*}} Global6
+# REGEXP-NEXT:     12: {{.*}}  GLOBAL {{.*}} Global5 Global6
+# REGEXP-NEXT:     13: {{.*}}  GLOBAL {{.*}} UND Global7
diff --git a/test/tools/llvm-objcopy/ELF/keep-many.test b/test/tools/llvm-objcopy/ELF/keep-many.test
index 2abb19dd92b8bd2c972fdc1f8de626eca2984ddc..1abcb16dfca64fbcfcb80f37831aefd4b202d1be 100644
--- a/test/tools/llvm-objcopy/ELF/keep-many.test
+++ b/test/tools/llvm-objcopy/ELF/keep-many.test
@@ -1,6 +1,8 @@
 # RUN: yaml2obj %s > %t
-# RUN: llvm-objcopy -strip-non-alloc -keep-section=.test -keep-section=.test3 %t %t2
-# RUN: llvm-readobj -file-headers -sections %t2 | FileCheck %s
+# RUN: llvm-objcopy --strip-non-alloc --keep-section=.test --keep-section=.test3 %t %t2
+# RUN: llvm-objcopy --strip-non-alloc --regex --keep-section='^.test[0-9]+$' %t %t3
+# RUN: llvm-readobj --file-headers --sections %t2 | FileCheck %s
+# RUN: llvm-readobj --file-headers --sections %t3 | FileCheck %s --check-prefix=REGEX
 
 !ELF
 FileHeader:
@@ -25,3 +27,8 @@ Sections:
 # CHECK:     Name: .test
 # CHECK:     Name: .test3
 # CHECK:     Name: .shstrtab
+
+# REGEX: SectionHeaderCount: 4
+# REGEX:     Name: .test2
+# REGEX:     Name: .test3
+# REGEX:     Name: .shstrtab
diff --git a/test/tools/llvm-objcopy/ELF/keep-only-section.test b/test/tools/llvm-objcopy/ELF/keep-only-section.test
index 38aa52cc70f35d9e39abde8345122eda42ac2a1c..7ea8468fe5f991f1aeeae8a000c7d96ebfc038db 100644
--- a/test/tools/llvm-objcopy/ELF/keep-only-section.test
+++ b/test/tools/llvm-objcopy/ELF/keep-only-section.test
@@ -1,8 +1,10 @@
 # RUN: yaml2obj %s > %t
-# RUN: llvm-objcopy -keep-section=.test2 -only-section=.test %t %t2
-# RUN: llvm-objcopy -j .test -keep-section=.test2 %t %t3
-# RUN: llvm-readobj -file-headers -sections %t2 | FileCheck %s
+# RUN: llvm-objcopy --keep-section=.test2 --only-section=.test %t %t2
+# RUN: llvm-objcopy -j .test --keep-section=.test2 %t %t3
+# RUN: llvm-readobj --file-headers --sections %t2 | FileCheck %s
 # RUN: diff %t2 %t3
+# RUN: llvm-objcopy --regex --keep-section='^.test$'  --only-section='^.test[2-3]+$' %t %t4
+# RUN: llvm-readobj --file-headers --sections %t4 | FileCheck %s --check-prefix=REGEX
 
 !ELF
 FileHeader:
@@ -25,3 +27,12 @@ Sections:
 # CHECK:     Name: .symtab
 # CHECK:     Name: .strtab
 # CHECK:     Name: .shstrtab
+
+# REGEX: SectionHeaderCount: 7
+
+# REGEX:     Name: .test
+# REGEX:     Name: .test2
+# REGEX:     Name: .test3
+# REGEX:     Name: .symtab
+# REGEX:     Name: .strtab
+# REGEX:     Name: .shstrtab
diff --git a/test/tools/llvm-objcopy/ELF/keep-symbol-remove-section.test b/test/tools/llvm-objcopy/ELF/keep-symbol-remove-section.test
index 6b02caa876e9f74f8edad895ef250442afa1d5de..2dbfb2c8c861c4238cff826ee00d985d210e46cc 100644
--- a/test/tools/llvm-objcopy/ELF/keep-symbol-remove-section.test
+++ b/test/tools/llvm-objcopy/ELF/keep-symbol-remove-section.test
@@ -1,6 +1,6 @@
 # RUN: yaml2obj %s > %t
 # RUN: llvm-objcopy --remove-section .text --keep-symbol foo %t %t2
-# RUN: llvm-readobj -symbols %t2 | FileCheck %s
+# RUN: llvm-readobj --symbols %t2 | FileCheck %s
 
 !ELF
 FileHeader:
diff --git a/test/tools/llvm-objcopy/ELF/keep-symbol.test b/test/tools/llvm-objcopy/ELF/keep-symbol.test
index 8c6415f3cb6c840a9b78f8b07b79b42c849d5074..e4e1888dfa2b642bec627775d2e4ccda97662f32 100644
--- a/test/tools/llvm-objcopy/ELF/keep-symbol.test
+++ b/test/tools/llvm-objcopy/ELF/keep-symbol.test
@@ -1,8 +1,10 @@
 # RUN: yaml2obj %s > %t
 # RUN: llvm-objcopy --discard-all -K foo --keep-symbol bar %t %t2
 # RUN: llvm-objcopy -K foo -N foo -N bar --keep-symbol bar -N baz %t %t3
-# RUN: llvm-readobj -symbols %t2 | FileCheck %s
-# RUN: llvm-readobj -symbols %t3 | FileCheck %s
+# RUN: llvm-objcopy --discard-all --regex -K '^ba.*' %t %t4
+# RUN: llvm-readobj --symbols %t2 | FileCheck %s
+# RUN: llvm-readobj --symbols %t3 | FileCheck %s
+# RUN: llvm-readobj --symbols %t4 | FileCheck %s --check-prefix=REGEX
 
 !ELF
 FileHeader:
@@ -64,3 +66,9 @@ Symbols:
 #CHECK-NEXT:    Section: .text
 #CHECK-NEXT:  }
 #CHECK-NEXT:]
+
+#REGEX: Symbols [
+#REGEX-NOT: foo
+#REGEX: Name: bar
+#REGEX: Name: baz
+#REGEX: ]
diff --git a/test/tools/llvm-objcopy/ELF/localize-hidden.test b/test/tools/llvm-objcopy/ELF/localize-hidden.test
index 05d747b800bc6d587aefc224adfe4dcc18df6567..41662003fe8dac9680fcb5279ab1f88702160eef 100644
--- a/test/tools/llvm-objcopy/ELF/localize-hidden.test
+++ b/test/tools/llvm-objcopy/ELF/localize-hidden.test
@@ -1,6 +1,6 @@
 # RUN: yaml2obj %s > %t
-# RUN: llvm-objcopy -localize-hidden %t %t2
-# RUN: llvm-readobj -relocations -symbols %t2 | FileCheck %s
+# RUN: llvm-objcopy --localize-hidden %t %t2
+# RUN: llvm-readobj --relocations --symbols %t2 | FileCheck %s
 
 !ELF
 FileHeader:
diff --git a/test/tools/llvm-objcopy/ELF/localize.test b/test/tools/llvm-objcopy/ELF/localize.test
index 2e2d6ccd6bf5be45c126d9878d194d0b8ef134f3..fb812317c8fdcdeb6b186ac8841ff9b1b5089249 100644
--- a/test/tools/llvm-objcopy/ELF/localize.test
+++ b/test/tools/llvm-objcopy/ELF/localize.test
@@ -1,11 +1,26 @@
 # RUN: yaml2obj %s > %t
 # RUN: llvm-objcopy \
 # RUN:     --localize-symbol Global \
+# RUN:     -L GlobalUndef \
 # RUN:     -L Local \
 # RUN:     -L Weak \
 # RUN:     -L GlobalCommon \
 # RUN:     %t %t2
-# RUN: llvm-readobj -symbols %t2 | FileCheck %s
+# RUN: llvm-readobj --symbols %t2 | FileCheck %s
+# RUN: llvm-objcopy --regex --localize-symbol='.*' %t %t3
+# RUN: cmp %t2 %t3
+# RUN: echo "  Global # comment  " > %t-list.txt
+# RUN: echo "GlobalUndef" >> %t-list.txt
+# RUN: echo "Local" >> %t-list.txt
+# RUN: echo "Weak" >> %t-list.txt
+# RUN: echo "GlobalCommon" >> %t-list.txt
+# RUN: echo " # comment " >> %t-list.txt
+# RUN: llvm-objcopy --localize-symbols %t-list.txt %t %t4
+# RUN: cmp %t2 %t4
+# RUN: echo "  .*al.* # Global + Local + GlobalCommon " > %t-list2.txt
+# RUN: echo "Weak" >> %t-list2.txt
+# RUN: llvm-objcopy --regex --localize-symbols %t-list2.txt %t %t5
+# RUN: cmp %t2 %t5
 
 !ELF
 FileHeader:
@@ -45,6 +60,8 @@ Symbols:
       Size:     8
       Section:  .text
       Value:    0x1010
+    - Name:     GlobalUndef
+      Type:     STT_FUNC
     - Name:     GlobalCommon
       Type:     STT_OBJECT
       Index:    SHN_COMMON
@@ -89,6 +106,15 @@ Symbols:
 #CHECK-NEXT:    Section: .text
 #CHECK-NEXT:  }
 #CHECK-NEXT:  Symbol {
+#CHECK-NEXT:    Name: GlobalUndef
+#CHECK-NEXT:    Value:
+#CHECK-NEXT:    Size:
+#CHECK-NEXT:    Binding: Global
+#CHECK-NEXT:    Type: Function
+#CHECK-NEXT:    Other:
+#CHECK-NEXT:    Section: Undefined
+#CHECK-NEXT:  }
+#CHECK-NEXT:  Symbol {
 #CHECK-NEXT:    Name: GlobalCommon
 #CHECK-NEXT:    Value: 0x2006
 #CHECK-NEXT:    Size: 2
diff --git a/test/tools/llvm-objcopy/ELF/many-sections.test b/test/tools/llvm-objcopy/ELF/many-sections.test
index e9e12cf913f7ed0699ebded578cd68652a7b33e3..1dd41cfb10c3744b15f1b2ba490cd3dceaf09154 100644
--- a/test/tools/llvm-objcopy/ELF/many-sections.test
+++ b/test/tools/llvm-objcopy/ELF/many-sections.test
@@ -1,8 +1,8 @@
-RUN: %python %p/Inputs/ungzip.py %p/Inputs/many-sections.o.gz > %t
+RUN: %python %p/../Inputs/ungzip.py %p/Inputs/many-sections.o.gz > %t
 RUN: llvm-objcopy %t %t2
-RUN: llvm-readobj -file-headers %t2 | FileCheck --check-prefix=EHDR %s
-RUN: llvm-readobj -sections %t2 | FileCheck --check-prefix=SECS %s
-RUN: llvm-readobj -symbols %t2 | grep "Symbol {" | wc -l | FileCheck --check-prefix=SYMS %s
+RUN: llvm-readobj --file-headers %t2 | FileCheck --check-prefix=EHDR %s
+RUN: llvm-readobj --sections %t2 | FileCheck --check-prefix=SECS %s
+RUN: llvm-readobj --symbols %t2 | grep "Symbol {" | wc -l | FileCheck --check-prefix=SYMS %s
 
 EHDR:      Format: ELF64-x86-64
 EHDR-NEXT: Arch: x86_64
diff --git a/test/tools/llvm-objcopy/ELF/marker-segment.test b/test/tools/llvm-objcopy/ELF/marker-segment.test
index 01b55ebc32c11c27ded9d16878c10d9d2aed571c..226d21ccba971e406d502c2ee8b04ecff7aa1319 100644
--- a/test/tools/llvm-objcopy/ELF/marker-segment.test
+++ b/test/tools/llvm-objcopy/ELF/marker-segment.test
@@ -1,6 +1,6 @@
 # RUN: yaml2obj %s -o %t
 # RUN: llvm-objcopy %t %t2
-# RUN: llvm-readobj -program-headers %t2 | FileCheck %s
+# RUN: llvm-readobj --program-headers %t2 | FileCheck %s
 
 !ELF
 FileHeader:
diff --git a/test/tools/llvm-objcopy/ELF/no-symbol-relocation.test b/test/tools/llvm-objcopy/ELF/no-symbol-relocation.test
index 5e60ec856e7caf8faf753d4857831438ef37550f..9def536c239c1d537fdc61a7ed1ae576612055f3 100644
--- a/test/tools/llvm-objcopy/ELF/no-symbol-relocation.test
+++ b/test/tools/llvm-objcopy/ELF/no-symbol-relocation.test
@@ -1,6 +1,6 @@
 # RUN: yaml2obj %s > %t
 # RUN: llvm-objcopy %t %t2
-# RUN: llvm-readobj -relocations %t2 | FileCheck %s
+# RUN: llvm-readobj --relocations %t2 | FileCheck %s
 
 !ELF
 FileHeader:
diff --git a/test/tools/llvm-objcopy/ELF/null-symbol.test b/test/tools/llvm-objcopy/ELF/null-symbol.test
index 61ed41078391f0ec5afb4c5ed339336f11c18f7d..94f5ab5d8c2e5aa599493f0c8581937803fef6fd 100644
--- a/test/tools/llvm-objcopy/ELF/null-symbol.test
+++ b/test/tools/llvm-objcopy/ELF/null-symbol.test
@@ -1,6 +1,6 @@
 # RUN: yaml2obj %s > %t
 # RUN: llvm-objcopy %t %t2
-# RUN: llvm-readobj -symbols %t2 | FileCheck %s
+# RUN: llvm-readobj --symbols %t2 | FileCheck %s
 
 !ELF
 FileHeader:
diff --git a/test/tools/llvm-objcopy/ELF/only-section-many.test b/test/tools/llvm-objcopy/ELF/only-section-many.test
index 2f956238dc91546d8bf13a2aca845c6b94a411df..43ccff1c2c2d85cb6d3b652454201942cc86bc86 100644
--- a/test/tools/llvm-objcopy/ELF/only-section-many.test
+++ b/test/tools/llvm-objcopy/ELF/only-section-many.test
@@ -1,6 +1,6 @@
 # RUN: yaml2obj %s > %t
 # RUN: llvm-objcopy -j .test1 -j .test2 %t %t2
-# RUN: llvm-readobj -file-headers -sections %t2 | FileCheck %s
+# RUN: llvm-readobj --file-headers --sections %t2 | FileCheck %s
 
 !ELF
 FileHeader:
diff --git a/test/tools/llvm-objcopy/ELF/only-section-remove-strtab.test b/test/tools/llvm-objcopy/ELF/only-section-remove-strtab.test
index e336181d9fa43cdca30386f402d212495c3abd0a..c88fef2fd0cdd7fdf48e6023815e7534a13523c1 100644
--- a/test/tools/llvm-objcopy/ELF/only-section-remove-strtab.test
+++ b/test/tools/llvm-objcopy/ELF/only-section-remove-strtab.test
@@ -1,7 +1,7 @@
 # RUN: yaml2obj %s > %t
-# RUN: llvm-objcopy -R .symtab -R .strtab -only-section=.test %t %t2
+# RUN: llvm-objcopy -R .symtab -R .strtab --only-section=.test %t %t2
 # RUN: llvm-objcopy -j .test -R .strtab -R .symtab %t %t3
-# RUN: llvm-readobj -file-headers -sections %t2 | FileCheck %s
+# RUN: llvm-readobj --file-headers --sections %t2 | FileCheck %s
 # RUN: diff %t2 %t3
 
 !ELF
diff --git a/test/tools/llvm-objcopy/ELF/only-section-strip-non-alloc.test b/test/tools/llvm-objcopy/ELF/only-section-strip-non-alloc.test
index f61b4ae664c6586b2285e6fa9f3b820c2debd95e..ae39b51226b790cfe6f572b513dc1b01de7a4b9d 100644
--- a/test/tools/llvm-objcopy/ELF/only-section-strip-non-alloc.test
+++ b/test/tools/llvm-objcopy/ELF/only-section-strip-non-alloc.test
@@ -1,6 +1,6 @@
 # RUN: yaml2obj %s > %t
-# RUN: llvm-objcopy -strip-non-alloc -only-section=.test %t %t2
-# RUN: llvm-readobj -file-headers -sections %t2 | FileCheck %s
+# RUN: llvm-objcopy --strip-non-alloc --only-section=.test %t %t2
+# RUN: llvm-readobj --file-headers --sections %t2 | FileCheck %s
 
 !ELF
 FileHeader:
diff --git a/test/tools/llvm-objcopy/ELF/overlap-chain.test b/test/tools/llvm-objcopy/ELF/overlap-chain.test
index a084abbb5ba7afe72e8e34779876c4886ea5358a..6b54e73b66a1fc684d6111b42a3a66ed4b082456 100644
--- a/test/tools/llvm-objcopy/ELF/overlap-chain.test
+++ b/test/tools/llvm-objcopy/ELF/overlap-chain.test
@@ -5,7 +5,7 @@
 
 # RUN: yaml2obj %s -o %t
 # RUN: llvm-objcopy %t %t2
-# RUN: llvm-readobj -program-headers %t2 | FileCheck %s
+# RUN: llvm-readobj --program-headers %t2 | FileCheck %s
 
 !ELF
 FileHeader:
diff --git a/test/tools/llvm-objcopy/ELF/prefix-symbols.test b/test/tools/llvm-objcopy/ELF/prefix-symbols.test
index 8761ba9fada195e171ec8e1b3f5872aae6e542cb..1e6a54a8cf845ba79d680c3713913e41cded0c80 100644
--- a/test/tools/llvm-objcopy/ELF/prefix-symbols.test
+++ b/test/tools/llvm-objcopy/ELF/prefix-symbols.test
@@ -1,8 +1,8 @@
 # RUN: yaml2obj %s > %t
 # RUN: llvm-objcopy --prefix-symbols prefix %t %t2
-# RUN: llvm-readobj -symbols %t2 | FileCheck %s --check-prefix=COMMON --check-prefix=BASIC
+# RUN: llvm-readobj --symbols %t2 | FileCheck %s --check-prefix=COMMON --check-prefix=BASIC
 # RUN: llvm-objcopy --redefine-sym bar=baz --prefix-symbols prefix %t %t3
-# RUN: llvm-readobj -symbols %t3 | FileCheck %s --check-prefix=COMMON --check-prefix=REDEF
+# RUN: llvm-readobj --symbols %t3 | FileCheck %s --check-prefix=COMMON --check-prefix=REDEF
 
 !ELF
 FileHeader:
diff --git a/test/tools/llvm-objcopy/ELF/program-headers.test b/test/tools/llvm-objcopy/ELF/program-headers.test
index dc2551030ecc5d820f03f47f8e9c70432d4fe3f7..9c123d215470af0b72c32248df93a5e07215119b 100644
--- a/test/tools/llvm-objcopy/ELF/program-headers.test
+++ b/test/tools/llvm-objcopy/ELF/program-headers.test
@@ -1,6 +1,6 @@
 # RUN: yaml2obj %s -o %t
 # RUN: llvm-objcopy %t %t2
-# RUN: llvm-readobj -program-headers %t2 | FileCheck %s
+# RUN: llvm-readobj --program-headers %t2 | FileCheck %s
 
 !ELF
 FileHeader:
diff --git a/test/tools/llvm-objcopy/ELF/pt-phdr.test b/test/tools/llvm-objcopy/ELF/pt-phdr.test
index 336ce769770588520f89e0b8ff24ce08cff250cc..dba40efc640c262189b1a7c5cdbada341263b7ac 100644
--- a/test/tools/llvm-objcopy/ELF/pt-phdr.test
+++ b/test/tools/llvm-objcopy/ELF/pt-phdr.test
@@ -2,7 +2,7 @@
 # segments.
 
 # RUN: llvm-objcopy %p/Inputs/pt-phdr.elf %t
-# RUN: llvm-readobj -program-headers %t | FileCheck %s
+# RUN: llvm-readobj --program-headers %t | FileCheck %s
 
 #CHECK: ProgramHeaders [
 #CHECK-NEXT:  ProgramHeader {
diff --git a/test/tools/llvm-objcopy/ELF/redefine-symbol.test b/test/tools/llvm-objcopy/ELF/redefine-symbol.test
index a0600fac83b87d006f0d627d6081a6a2eb463a42..8ed8c85e93d384e181eea07abdb219203ef1b6b2 100644
--- a/test/tools/llvm-objcopy/ELF/redefine-symbol.test
+++ b/test/tools/llvm-objcopy/ELF/redefine-symbol.test
@@ -1,8 +1,17 @@
 # RUN: yaml2obj %s > %t
 # RUN: llvm-objcopy --redefine-sym foo=oof --redefine-sym empty= %t %t2
-# RUN: llvm-readobj -symbols %t2 | FileCheck %s
+# RUN: llvm-readobj --symbols %t2 | FileCheck %s
 # RUN: not llvm-objcopy --redefine-sym barbar %t %t2 2>&1 | FileCheck %s --check-prefix=BAD-FORMAT
 # RUN: not llvm-objcopy --redefine-sym foo=f1 --redefine-sym foo=f2 %t %t2 2>&1 | FileCheck %s --check-prefix=MULTIPLE-REDEFINITION
+# RUN: echo "  foo   oof #rename foo  " > %t.rename.txt
+# RUN: echo "empty" >> %t.rename.txt
+# RUN: not llvm-objcopy --redefine-syms %t.rename.txt %t %t3 2>&1 | FileCheck %s --check-prefix=MISSING-SYM-NAME
+# RUN: not llvm-objcopy --redefine-syms %t.rename-none.txt %t %t-none 2>&1 | FileCheck %s --check-prefix=NO-FILE
+# RUN: echo "  bar   rab #rename bar  " > %t.rename2.txt
+# RUN: echo "  foo   oof #rename foo  " > %t.rename3.txt
+# RUN: echo "  empty   ytpme #rename empty  " >> %t.rename3.txt
+# RUN: llvm-objcopy --redefine-syms %t.rename2.txt --redefine-syms %t.rename3.txt %t %t4
+# RUN: llvm-readobj --symbols %t4 | FileCheck %s --check-prefix=MULTIPLE-FILES
 
 !ELF
 FileHeader:
@@ -79,3 +88,12 @@ Symbols:
 
 #BAD-FORMAT: Bad format for --redefine-sym
 #MULTIPLE-REDEFINITION: Multiple redefinition of symbol foo
+#MISSING-SYM-NAME: error: {{.*}}.rename.txt:2: missing new symbol name
+#NO-FILE: error: '{{.*}}.rename-none.txt': {{[Nn]}}o such file or directory
+
+#MULTIPLE-FILES:        Name: oof
+#MULTIPLE-FILES-NEXT:   Value: 0x1004
+#MULTIPLE-FILES:        Name: rab
+#MULTIPLE-FILES-NEXT:   Value: 0x2000
+#MULTIPLE-FILES:        Name: ytpme
+#MULTIPLE-FILES-NEXT:   Value: 0x1008
diff --git a/test/tools/llvm-objcopy/ELF/regex.test b/test/tools/llvm-objcopy/ELF/regex.test
new file mode 100644
index 0000000000000000000000000000000000000000..6fa590b5dc576ae57d51f992067776a099dbe606
--- /dev/null
+++ b/test/tools/llvm-objcopy/ELF/regex.test
@@ -0,0 +1,60 @@
+# RUN: yaml2obj %s > %t
+
+# No symbol matches, because pattern is implicitly enveloped in '^$'
+# RUN: llvm-objcopy --discard-all --regex -K 'ba' %t %t2
+# RUN: llvm-readobj --symbols %t2 | FileCheck %s --check-prefix=REGEX1
+
+# Symbol 'bar' matches
+# RUN: llvm-objcopy --discard-all --regex -K 'ba.*' %t %t3
+# RUN: llvm-readobj --symbols %t3 | FileCheck %s --check-prefix=REGEX2
+
+# All symbols match
+# RUN: llvm-objcopy --discard-all --regex -K '.*ba.*' %t %t4
+# RUN: llvm-readobj --symbols %t4 | FileCheck %s --check-prefix=REGEX3
+
+# All symbols match
+# RUN: llvm-objcopy --discard-all --regex -K '^([a-z]+)*ba.*$' %t %t5
+# RUN: cmp %t4 %t5
+ 
+!ELF
+FileHeader:
+  Class:           ELFCLASS64
+  Data:            ELFDATA2LSB
+  Type:            ET_REL
+  Machine:         EM_X86_64
+Sections:
+  - Name:            .text
+    Type:            SHT_PROGBITS
+    Flags:           [ SHF_ALLOC, SHF_EXECINSTR ]
+    Address:         0x1000
+    AddressAlign:    0x0000000000000010
+    Size:            64
+Symbols:
+  Local:
+    - Name:     foobaz
+      Type:     STT_FUNC
+      Section:  .text
+      Value:    0x1000
+      Size:     8
+    - Name:     bar
+      Type:     STT_FUNC
+      Section:  .text
+      Value:    0x1008
+      Size:     8
+    - Name:     rebar
+      Type:     STT_FUNC
+      Section:  .text
+      Value:    0x1010
+      Size:     8
+
+#REGEX1-NOT: foobaz
+#REGEX1-NOT: bar
+#REGEX1-NOT: rebar
+
+#REGEX2-NOT: foobaz
+#REGEX2: bar
+#REGEX2-NOT: rebar
+
+#REGEX3: foobaz
+#REGEX3: bar
+#REGEX3: rebar
diff --git a/test/tools/llvm-objcopy/ELF/reloc-no-symtab.test b/test/tools/llvm-objcopy/ELF/reloc-no-symtab.test
index ea593613df0b48f9949a5d95d73af4d538d95e7b..96bf4d3f77dc57ce7ad9944b35e2fca0792235a6 100644
--- a/test/tools/llvm-objcopy/ELF/reloc-no-symtab.test
+++ b/test/tools/llvm-objcopy/ELF/reloc-no-symtab.test
@@ -7,8 +7,8 @@
 
 # RUN: yaml2obj %s > %t.original
 # RUN: llvm-strip %t.original -o %t.stripped
-# RUN: llvm-readobj -sections %t.original | FileCheck %s
-# RUN: llvm-readobj -sections %t.stripped | FileCheck %s
+# RUN: llvm-readobj --sections %t.original | FileCheck %s
+# RUN: llvm-readobj --sections %t.stripped | FileCheck %s
 
 !ELF
 FileHeader:
diff --git a/test/tools/llvm-objcopy/ELF/relocatable-phdr.test b/test/tools/llvm-objcopy/ELF/relocatable-phdr.test
index b6218956aff2cad5a06c6f1b7f55a6cc13693b5d..40216467a2439a9de66d2101f6b68b54ae90906c 100644
--- a/test/tools/llvm-objcopy/ELF/relocatable-phdr.test
+++ b/test/tools/llvm-objcopy/ELF/relocatable-phdr.test
@@ -2,7 +2,7 @@
 # for offset and header size when copied with llvm-objcopy.
 # RUN: yaml2obj %s > %t
 # RUN: llvm-objcopy %t %t2
-# RUN: llvm-readobj -file-headers %t2 | FileCheck %s
+# RUN: llvm-readobj --file-headers %t2 | FileCheck %s
 
 !ELF
 FileHeader:
diff --git a/test/tools/llvm-objcopy/ELF/remove-multiple-sections.test b/test/tools/llvm-objcopy/ELF/remove-multiple-sections.test
index 5e5de97cd31ce4ba9a7d554be60905b8fda9ade7..1f1bc4ce1910e3befd8d64a2ce2d0f44767f1edc 100644
--- a/test/tools/llvm-objcopy/ELF/remove-multiple-sections.test
+++ b/test/tools/llvm-objcopy/ELF/remove-multiple-sections.test
@@ -1,6 +1,6 @@
 # RUN: yaml2obj %s > %t
 # RUN: llvm-objcopy -R .test2 -R .test3 -R .test5 %t %t2
-# RUN: llvm-readobj -sections %t2 | FileCheck %s
+# RUN: llvm-readobj --sections %t2 | FileCheck %s
 
 !ELF
 FileHeader:
diff --git a/test/tools/llvm-objcopy/ELF/remove-section-with-symbol.test b/test/tools/llvm-objcopy/ELF/remove-section-with-symbol.test
index 067c0b25ea748793b1e97670bf8247bce525a0be..949d34cd5e848dd39d31425cff496b0d41615079 100644
--- a/test/tools/llvm-objcopy/ELF/remove-section-with-symbol.test
+++ b/test/tools/llvm-objcopy/ELF/remove-section-with-symbol.test
@@ -1,6 +1,6 @@
 # RUN: yaml2obj %s > %t
 # RUN: llvm-objcopy -R .test %t %t2
-# RUN: llvm-readobj -file-headers -symbols %t2 | FileCheck %s
+# RUN: llvm-readobj --file-headers --symbols %t2 | FileCheck %s
 
 !ELF
 FileHeader:
diff --git a/test/tools/llvm-objcopy/ELF/remove-section.test b/test/tools/llvm-objcopy/ELF/remove-section.test
index f19e7b7684ec9fe531199627a1a3635491e0a3f8..42097210055e66de9a0ca4c7e5d74dc4e49a2bf8 100644
--- a/test/tools/llvm-objcopy/ELF/remove-section.test
+++ b/test/tools/llvm-objcopy/ELF/remove-section.test
@@ -2,15 +2,19 @@
 # RUN: cp %t %t1
 
 # RUN: llvm-objcopy -R .test2 %t %t2
-# RUN: llvm-readobj -file-headers -sections %t2 | FileCheck %s
+# RUN: llvm-readobj --file-headers --sections %t2 | FileCheck %s
 
 # Verify that the first run of llvm-objcopy
 # has not modified the input binary.
 # RUN: cmp %t %t1
 
-# RUN: llvm-objcopy -remove-section=.test2 %t1 %t3
+# RUN: llvm-objcopy --remove-section=.test2 %t1 %t3
 # RUN: cmp %t2 %t3
 
+# RUN: llvm-objcopy --regex --remove-section='.test[0-9]+' %t %t4
+# RUN: llvm-readobj --file-headers --sections %t4 | FileCheck %s --check-prefix=REGEX
+
+
 !ELF
 FileHeader:
   Class:           ELFCLASS64
@@ -116,3 +120,7 @@ Sections:
 # CHECK:     EntrySize: 0
 # CHECK:   }
 # CHECK: ]
+
+# REGEX: SectionHeaderCount: 4
+# REGEX: Sections [
+# REGEX-NOT: test  
diff --git a/test/tools/llvm-objcopy/ELF/remove-shndx.test b/test/tools/llvm-objcopy/ELF/remove-shndx.test
index b8ea94cbc1da902143424ac255d0c9e677a6aac6..53ca8e7f220193095fdf6f7b5c74b8a481be6008 100644
--- a/test/tools/llvm-objcopy/ELF/remove-shndx.test
+++ b/test/tools/llvm-objcopy/ELF/remove-shndx.test
@@ -1,7 +1,7 @@
 # This test checks to see that a .symtab_shndx section is added to any binary
 # that needs it, even if the original was removed.
-RUN: %python %p/Inputs/ungzip.py %p/Inputs/many-sections.o.gz > %t
+RUN: %python %p/../Inputs/ungzip.py %p/Inputs/many-sections.o.gz > %t
 RUN: llvm-objcopy -R .symtab_shndx %t %t2
-RUN: llvm-readobj -sections %t2 | FileCheck %s
+RUN: llvm-readobj --sections %t2 | FileCheck %s
 
 CHECK: Name: .symtab_shndx (
diff --git a/test/tools/llvm-objcopy/ELF/remove-symtab.test b/test/tools/llvm-objcopy/ELF/remove-symtab.test
index 0e3f82b3a0dea75630c11d09236ed3e77acc23d9..5c53962b14b38f7f40d68a08e22fdff10b99b6bb 100644
--- a/test/tools/llvm-objcopy/ELF/remove-symtab.test
+++ b/test/tools/llvm-objcopy/ELF/remove-symtab.test
@@ -1,6 +1,6 @@
 # RUN: yaml2obj %s > %t
 # RUN: llvm-objcopy -R .symtab %t %t2
-# RUN: llvm-readobj -file-headers -sections %t2 | FileCheck %s
+# RUN: llvm-readobj --file-headers --sections %t2 | FileCheck %s
 
 !ELF
 FileHeader:
diff --git a/test/tools/llvm-objcopy/ELF/rename-section-flag-osproc-mask.test b/test/tools/llvm-objcopy/ELF/rename-section-flag-osproc-mask.test
index 58a5ffa18885fec8036fc69411b0fa91354b9898..a76929051060c49fd8e6fda459b7667cfdbe2f84 100644
--- a/test/tools/llvm-objcopy/ELF/rename-section-flag-osproc-mask.test
+++ b/test/tools/llvm-objcopy/ELF/rename-section-flag-osproc-mask.test
@@ -2,9 +2,9 @@
 
 # ===== x86_64 =====
 
-# RUN: yaml2obj -docnum 1 %s > %t-x86_64.o
+# RUN: yaml2obj --docnum 1 %s > %t-x86_64.o
 # RUN: llvm-objcopy --rename-section=.foo=.bar,alloc %t-x86_64.o
-# RUN: llvm-readobj -sections %t-x86_64.o | FileCheck %s --check-prefix=X86_64
+# RUN: llvm-readobj --sections %t-x86_64.o | FileCheck %s --check-prefix=X86_64
 
 --- !ELF
 FileHeader:
@@ -27,9 +27,9 @@ Sections:
 
 # ===== hex =====
 
-# RUN: yaml2obj -docnum 2 %s > %t-hex.o
+# RUN: yaml2obj --docnum 2 %s > %t-hex.o
 # RUN: llvm-objcopy --rename-section=.foo=.bar,alloc %t-hex.o
-# RUN: llvm-readobj -sections %t-hex.o | FileCheck %s --check-prefix=HEX
+# RUN: llvm-readobj --sections %t-hex.o | FileCheck %s --check-prefix=HEX
 
 --- !ELF
 FileHeader:
@@ -52,9 +52,9 @@ Sections:
 
 # ===== mips =====
 
-# RUN: yaml2obj -docnum 3 %s > %t-mips.o
+# RUN: yaml2obj --docnum 3 %s > %t-mips.o
 # RUN: llvm-objcopy --rename-section=.foo=.bar,alloc %t-mips.o
-# RUN: llvm-readobj -sections %t-mips.o | FileCheck %s --check-prefix=MIPS
+# RUN: llvm-readobj --sections %t-mips.o | FileCheck %s --check-prefix=MIPS
 
 --- !ELF
 FileHeader:
@@ -91,9 +91,9 @@ Sections:
 
 # ===== arm =====
 
-# RUN: yaml2obj -docnum 4 %s > %t-arm.o
+# RUN: yaml2obj --docnum 4 %s > %t-arm.o
 # RUN: llvm-objcopy --rename-section=.foo=.bar,alloc %t-arm.o
-# RUN: llvm-readobj -sections %t-arm.o | FileCheck %s --check-prefix=ARM
+# RUN: llvm-readobj --sections %t-arm.o | FileCheck %s --check-prefix=ARM
 
 --- !ELF
 FileHeader:
diff --git a/test/tools/llvm-objcopy/ELF/rename-section-flag-preserved.test b/test/tools/llvm-objcopy/ELF/rename-section-flag-preserved.test
index d944e40094c421a040e94be540d9db2b76b470df..6c22eb892bfe3bc832663b7d931390f8ddc86b4e 100644
--- a/test/tools/llvm-objcopy/ELF/rename-section-flag-preserved.test
+++ b/test/tools/llvm-objcopy/ELF/rename-section-flag-preserved.test
@@ -2,29 +2,29 @@
 
 # Single flags on a section with all flags:
 # RUN: llvm-objcopy --rename-section=.foo=.bar,alloc %t %t.alloc
-# RUN: llvm-readobj -sections %t.alloc | FileCheck %s --check-prefixes=CHECK,ALLOC,WRITE
+# RUN: llvm-readobj --sections %t.alloc | FileCheck %s --check-prefixes=CHECK,ALLOC,WRITE
 # RUN: llvm-objcopy --rename-section=.foo=.bar,load %t %t.load
-# RUN: llvm-readobj -sections %t.load | FileCheck %s --check-prefixes=CHECK,WRITE
+# RUN: llvm-readobj --sections %t.load | FileCheck %s --check-prefixes=CHECK,WRITE
 # RUN: llvm-objcopy --rename-section=.foo=.bar,noload %t %t.noload
-# RUN: llvm-readobj -sections %t.noload | FileCheck %s --check-prefixes=CHECK,WRITE
+# RUN: llvm-readobj --sections %t.noload | FileCheck %s --check-prefixes=CHECK,WRITE
 # RUN: llvm-objcopy --rename-section=.foo=.bar,readonly %t %t.readonly
-# RUN: llvm-readobj -sections %t.readonly | FileCheck %s --check-prefixes=CHECK
+# RUN: llvm-readobj --sections %t.readonly | FileCheck %s --check-prefixes=CHECK
 # RUN: llvm-objcopy --rename-section=.foo=.bar,debug %t %t.debug
-# RUN: llvm-readobj -sections %t.debug | FileCheck %s --check-prefixes=CHECK,WRITE
+# RUN: llvm-readobj --sections %t.debug | FileCheck %s --check-prefixes=CHECK,WRITE
 # RUN: llvm-objcopy --rename-section=.foo=.bar,code %t %t.code
-# RUN: llvm-readobj -sections %t.code | FileCheck %s --check-prefixes=CHECK,EXEC,WRITE
+# RUN: llvm-readobj --sections %t.code | FileCheck %s --check-prefixes=CHECK,EXEC,WRITE
 # RUN: llvm-objcopy --rename-section=.foo=.bar,data %t %t.data
-# RUN: llvm-readobj -sections %t.data | FileCheck %s --check-prefixes=CHECK,WRITE
+# RUN: llvm-readobj --sections %t.data | FileCheck %s --check-prefixes=CHECK,WRITE
 # RUN: llvm-objcopy --rename-section=.foo=.bar,rom %t %t.rom
-# RUN: llvm-readobj -sections %t.rom | FileCheck %s --check-prefixes=CHECK,WRITE
+# RUN: llvm-readobj --sections %t.rom | FileCheck %s --check-prefixes=CHECK,WRITE
 # RUN: llvm-objcopy --rename-section=.foo=.bar,contents %t %t.contents
-# RUN: llvm-readobj -sections %t.contents | FileCheck %s --check-prefixes=CHECK,WRITE
+# RUN: llvm-readobj --sections %t.contents | FileCheck %s --check-prefixes=CHECK,WRITE
 # RUN: llvm-objcopy --rename-section=.foo=.bar,merge %t %t.merge
-# RUN: llvm-readobj -sections %t.merge | FileCheck %s --check-prefixes=CHECK,MERGE,WRITE
+# RUN: llvm-readobj --sections %t.merge | FileCheck %s --check-prefixes=CHECK,MERGE,WRITE
 # RUN: llvm-objcopy --rename-section=.foo=.bar,strings %t %t.strings
-# RUN: llvm-readobj -sections %t.strings | FileCheck %s --check-prefixes=CHECK,STRINGS,WRITE
+# RUN: llvm-readobj --sections %t.strings | FileCheck %s --check-prefixes=CHECK,STRINGS,WRITE
 # RUN: llvm-objcopy --rename-section=.foo=.bar,share %t %t.share
-# RUN: llvm-readobj -sections %t.share | FileCheck %s --check-prefixes=CHECK,WRITE
+# RUN: llvm-readobj --sections %t.share | FileCheck %s --check-prefixes=CHECK,WRITE
 
 !ELF
 FileHeader:
diff --git a/test/tools/llvm-objcopy/ELF/rename-section-flag.test b/test/tools/llvm-objcopy/ELF/rename-section-flag.test
index f31452dd436ede90b9810c2fc71b1e4dc24b9560..75f35af4b3015df0c11a53bf68acafcdc52289e5 100644
--- a/test/tools/llvm-objcopy/ELF/rename-section-flag.test
+++ b/test/tools/llvm-objcopy/ELF/rename-section-flag.test
@@ -2,35 +2,35 @@
 
 # Single flags on a section with no flags:
 # RUN: llvm-objcopy --rename-section=.foo=.bar,alloc %t %t.alloc
-# RUN: llvm-readobj -sections %t.alloc | FileCheck %s --check-prefixes=CHECK,ALLOC,WRITE
+# RUN: llvm-readobj --sections %t.alloc | FileCheck %s --check-prefixes=CHECK,ALLOC,WRITE
 # RUN: llvm-objcopy --rename-section=.foo=.bar,load %t %t.load
-# RUN: llvm-readobj -sections %t.load | FileCheck %s --check-prefixes=CHECK,WRITE
+# RUN: llvm-readobj --sections %t.load | FileCheck %s --check-prefixes=CHECK,WRITE
 # RUN: llvm-objcopy --rename-section=.foo=.bar,noload %t %t.noload
-# RUN: llvm-readobj -sections %t.noload | FileCheck %s --check-prefixes=CHECK,WRITE
+# RUN: llvm-readobj --sections %t.noload | FileCheck %s --check-prefixes=CHECK,WRITE
 # RUN: llvm-objcopy --rename-section=.foo=.bar,readonly %t %t.readonly
-# RUN: llvm-readobj -sections %t.readonly | FileCheck %s --check-prefixes=CHECK
+# RUN: llvm-readobj --sections %t.readonly | FileCheck %s --check-prefixes=CHECK
 # RUN: llvm-objcopy --rename-section=.foo=.bar,debug %t %t.debug
-# RUN: llvm-readobj -sections %t.debug | FileCheck %s --check-prefixes=CHECK,WRITE
+# RUN: llvm-readobj --sections %t.debug | FileCheck %s --check-prefixes=CHECK,WRITE
 # RUN: llvm-objcopy --rename-section=.foo=.bar,code %t %t.code
-# RUN: llvm-readobj -sections %t.code | FileCheck %s --check-prefixes=CHECK,EXEC,WRITE
+# RUN: llvm-readobj --sections %t.code | FileCheck %s --check-prefixes=CHECK,EXEC,WRITE
 # RUN: llvm-objcopy --rename-section=.foo=.bar,data %t %t.data
-# RUN: llvm-readobj -sections %t.data | FileCheck %s --check-prefixes=CHECK,WRITE
+# RUN: llvm-readobj --sections %t.data | FileCheck %s --check-prefixes=CHECK,WRITE
 # RUN: llvm-objcopy --rename-section=.foo=.bar,rom %t %t.rom
-# RUN: llvm-readobj -sections %t.rom | FileCheck %s --check-prefixes=CHECK,WRITE
+# RUN: llvm-readobj --sections %t.rom | FileCheck %s --check-prefixes=CHECK,WRITE
 # RUN: llvm-objcopy --rename-section=.foo=.bar,contents %t %t.contents
-# RUN: llvm-readobj -sections %t.contents | FileCheck %s --check-prefixes=CHECK,WRITE
+# RUN: llvm-readobj --sections %t.contents | FileCheck %s --check-prefixes=CHECK,WRITE
 # RUN: llvm-objcopy --rename-section=.foo=.bar,merge %t %t.merge
-# RUN: llvm-readobj -sections %t.merge | FileCheck %s --check-prefixes=CHECK,MERGE,WRITE
+# RUN: llvm-readobj --sections %t.merge | FileCheck %s --check-prefixes=CHECK,MERGE,WRITE
 # RUN: llvm-objcopy --rename-section=.foo=.bar,strings %t %t.strings
-# RUN: llvm-readobj -sections %t.strings | FileCheck %s --check-prefixes=CHECK,STRINGS,WRITE
+# RUN: llvm-readobj --sections %t.strings | FileCheck %s --check-prefixes=CHECK,STRINGS,WRITE
 # RUN: llvm-objcopy --rename-section=.foo=.bar,share %t %t.share
-# RUN: llvm-readobj -sections %t.share | FileCheck %s --check-prefixes=CHECK,WRITE
+# RUN: llvm-readobj --sections %t.share | FileCheck %s --check-prefixes=CHECK,WRITE
 
 # Multiple flags:
 # RUN: llvm-objcopy --rename-section=.foo=.bar,alloc,readonly,strings %t %t.alloc_ro_strings
-# RUN: llvm-readobj -sections %t.alloc_ro_strings | FileCheck %s --check-prefixes=CHECK,ALLOC,STRINGS
+# RUN: llvm-readobj --sections %t.alloc_ro_strings | FileCheck %s --check-prefixes=CHECK,ALLOC,STRINGS
 # RUN: llvm-objcopy --rename-section=.foo=.bar,alloc,code %t %t.alloc_code
-# RUN: llvm-readobj -sections %t.alloc_code | FileCheck %s --check-prefixes=CHECK,ALLOC,EXEC,WRITE
+# RUN: llvm-readobj --sections %t.alloc_code | FileCheck %s --check-prefixes=CHECK,ALLOC,EXEC,WRITE
 
 # Invalid flags:
 # RUN: not llvm-objcopy --rename-section=.foo=.bar,xyzzy %t %t.xyzzy 2>&1 | FileCheck %s --check-prefix=BAD-FLAG
diff --git a/test/tools/llvm-objcopy/ELF/rename-section-multiple.test b/test/tools/llvm-objcopy/ELF/rename-section-multiple.test
index 8feff9fda74cf5e1a18bebca1dc7a388855b2a41..e614f656053c78e4d27ad006336a63b88262362a 100644
--- a/test/tools/llvm-objcopy/ELF/rename-section-multiple.test
+++ b/test/tools/llvm-objcopy/ELF/rename-section-multiple.test
@@ -1,6 +1,6 @@
 # RUN: yaml2obj %s > %t
 # RUN: llvm-objcopy --rename-section=.test1=.test2 --rename-section=.test3=.test4 --rename-section=.test5=.test6 %t %t2
-# RUN: llvm-readobj -file-headers -sections -section-data %t2 | FileCheck %s
+# RUN: llvm-readobj --file-headers --sections --section-data %t2 | FileCheck %s
 
 !ELF
 FileHeader:
diff --git a/test/tools/llvm-objcopy/ELF/rename-section.test b/test/tools/llvm-objcopy/ELF/rename-section.test
index 505cf7770d7c1b08850ab252c4afeeb98a327958..feaec44823b885cb0e921deb6a1cf33a17159cfc 100644
--- a/test/tools/llvm-objcopy/ELF/rename-section.test
+++ b/test/tools/llvm-objcopy/ELF/rename-section.test
@@ -1,6 +1,6 @@
 # RUN: yaml2obj %s > %t
 # RUN: llvm-objcopy --rename-section=.foo=.bar %t %t2
-# RUN: llvm-readobj -file-headers -sections -section-data %t2 | FileCheck %s
+# RUN: llvm-readobj --file-headers --sections --section-data %t2 | FileCheck %s
 # RUN: not llvm-objcopy --rename-section=.foo.bar --rename-section=.foo=.other %t %t2 2>&1 | FileCheck %s --check-prefix=BAD-FORMAT
 # RUN: not llvm-objcopy --rename-section=.foo=.bar --rename-section=.foo=.other %t %t2 2>&1 | FileCheck %s --check-prefix=MULTIPLE-RENAMES
 
diff --git a/test/tools/llvm-objcopy/ELF/segment-shift-section-remove.test b/test/tools/llvm-objcopy/ELF/segment-shift-section-remove.test
index caeb5596ef7775daa6f18fa48fab0cb9da42ce42..563a9e3f836ba8ce3473a50a788c5a2c90f3ac48 100644
--- a/test/tools/llvm-objcopy/ELF/segment-shift-section-remove.test
+++ b/test/tools/llvm-objcopy/ELF/segment-shift-section-remove.test
@@ -1,6 +1,6 @@
 # RUN: yaml2obj %s -o %t
 # RUN: llvm-objcopy -R .text2 %t %t2
-# RUN: llvm-readobj -file-headers -sections -program-headers %t2 | FileCheck %s
+# RUN: llvm-readobj --file-headers --sections --program-headers %t2 | FileCheck %s
 
 !ELF
 FileHeader:
diff --git a/test/tools/llvm-objcopy/ELF/segment-shift.test b/test/tools/llvm-objcopy/ELF/segment-shift.test
index 635fdccf75e6a14ce16ee2d82186d722fc85c002..b4caa197aec40ea52b745264389bbac7ebcb2900 100644
--- a/test/tools/llvm-objcopy/ELF/segment-shift.test
+++ b/test/tools/llvm-objcopy/ELF/segment-shift.test
@@ -1,6 +1,6 @@
 # RUN: yaml2obj %s -o %t
 # RUN: llvm-objcopy %t %t2
-# RUN: llvm-readobj -program-headers %t2 | FileCheck %s
+# RUN: llvm-readobj --program-headers %t2 | FileCheck %s
 
 !ELF
 FileHeader:
diff --git a/test/tools/llvm-objcopy/ELF/segment-test-remove-section.test b/test/tools/llvm-objcopy/ELF/segment-test-remove-section.test
index 9b98dc8611faeba4059abeaef2e4112c57c81960..49a9003cb15e40f234ab24061679068b2eddce6c 100644
--- a/test/tools/llvm-objcopy/ELF/segment-test-remove-section.test
+++ b/test/tools/llvm-objcopy/ELF/segment-test-remove-section.test
@@ -6,7 +6,7 @@
 
 # RUN: yaml2obj %s -o %t
 # RUN: llvm-objcopy -R .text2 %t %t2
-# RUN: llvm-readobj -file-headers -program-headers -sections %t2 | FileCheck %s
+# RUN: llvm-readobj --file-headers --program-headers --sections %t2 | FileCheck %s
 # RUN: od -t x1 -j 8192 %t2 | FileCheck %s --check-prefix=DATA
 
 !ELF
diff --git a/test/tools/llvm-objcopy/ELF/set-section-flags-and-rename.test b/test/tools/llvm-objcopy/ELF/set-section-flags-and-rename.test
new file mode 100644
index 0000000000000000000000000000000000000000..88f7243f4ca7ed7c8077f7fbf7ebde275d26501e
--- /dev/null
+++ b/test/tools/llvm-objcopy/ELF/set-section-flags-and-rename.test
@@ -0,0 +1,14 @@
+# RUN: yaml2obj %s > %t
+
+# RUN: not llvm-objcopy --rename-section=.foo=.bar --set-section-flags=.foo=alloc %t %t.2 2>&1 | FileCheck %s --check-prefix=SET-FOO
+# RUN: not llvm-objcopy --rename-section=.foo=.bar --set-section-flags=.bar=alloc %t %t.2 2>&1 | FileCheck %s --check-prefix=SET-BAR
+
+!ELF
+FileHeader:
+  Class:           ELFCLASS64
+  Data:            ELFDATA2LSB
+  Type:            ET_REL
+  Machine:         EM_X86_64
+
+# SET-FOO: --set-section-flags=.foo conflicts with --rename-section=.foo=.bar.
+# SET-BAR: --set-section-flags=.bar conflicts with --rename-section=.foo=.bar.
diff --git a/test/tools/llvm-objcopy/ELF/set-section-flags-multiple.test b/test/tools/llvm-objcopy/ELF/set-section-flags-multiple.test
new file mode 100644
index 0000000000000000000000000000000000000000..41732f3807da3536976623f060493cd44f4081c8
--- /dev/null
+++ b/test/tools/llvm-objcopy/ELF/set-section-flags-multiple.test
@@ -0,0 +1,32 @@
+# RUN: yaml2obj %s > %t
+
+# RUN: llvm-objcopy --set-section-flags=.foo=alloc --set-section-flags=.bar=code %t %t.2
+# RUN: llvm-readobj --sections %t.2 | FileCheck %s --check-prefixes=CHECK,ALLOC,WRITE
+
+!ELF
+FileHeader:
+  Class:           ELFCLASS64
+  Data:            ELFDATA2LSB
+  Type:            ET_REL
+  Machine:         EM_X86_64
+Sections:
+  - Name:            .foo
+    Type:            SHT_PROGBITS
+    Flags:           [ ]
+  - Name:            .bar
+    Type:            SHT_PROGBITS
+    Flags:           [ ]
+
+# CHECK:        Name: .foo
+# CHECK-NEXT:   Type: SHT_PROGBITS
+# CHECK-NEXT:   Flags [
+# CHECK-NEXT:     SHF_ALLOC (0x2)
+# CHECK-NEXT:     SHF_WRITE (0x1)
+# CHECK-NEXT:   ]
+
+# CHECK:        Name: .bar
+# CHECK-NEXT:   Type: SHT_PROGBITS
+# CHECK-NEXT:   Flags [
+# CHECK-NEXT:     SHF_EXECINSTR (0x4)
+# CHECK-NEXT:     SHF_WRITE (0x1)
+# CHECK-NEXT:   ]
diff --git a/test/tools/llvm-objcopy/ELF/set-section-flags.test b/test/tools/llvm-objcopy/ELF/set-section-flags.test
new file mode 100644
index 0000000000000000000000000000000000000000..6fde33e0749f79e52f720decd2b91d8660b04abe
--- /dev/null
+++ b/test/tools/llvm-objcopy/ELF/set-section-flags.test
@@ -0,0 +1,68 @@
+# RUN: yaml2obj %s > %t
+
+# Single flags on a section with no flags:
+# RUN: llvm-objcopy --set-section-flags=.foo=alloc %t %t.alloc
+# RUN: llvm-readobj --sections %t.alloc | FileCheck %s --check-prefixes=CHECK,ALLOC,WRITE
+# RUN: llvm-objcopy --set-section-flags=.foo=load %t %t.load
+# RUN: llvm-readobj --sections %t.load | FileCheck %s --check-prefixes=CHECK,WRITE
+# RUN: llvm-objcopy --set-section-flags=.foo=noload %t %t.noload
+# RUN: llvm-readobj --sections %t.noload | FileCheck %s --check-prefixes=CHECK,WRITE
+# RUN: llvm-objcopy --set-section-flags=.foo=readonly %t %t.readonly
+# RUN: llvm-readobj --sections %t.readonly | FileCheck %s --check-prefixes=CHECK
+# RUN: llvm-objcopy --set-section-flags=.foo=debug %t %t.debug
+# RUN: llvm-readobj --sections %t.debug | FileCheck %s --check-prefixes=CHECK,WRITE
+# RUN: llvm-objcopy --set-section-flags=.foo=code %t %t.code
+# RUN: llvm-readobj --sections %t.code | FileCheck %s --check-prefixes=CHECK,EXEC,WRITE
+# RUN: llvm-objcopy --set-section-flags=.foo=data %t %t.data
+# RUN: llvm-readobj --sections %t.data | FileCheck %s --check-prefixes=CHECK,WRITE
+# RUN: llvm-objcopy --set-section-flags=.foo=rom %t %t.rom
+# RUN: llvm-readobj --sections %t.rom | FileCheck %s --check-prefixes=CHECK,WRITE
+# RUN: llvm-objcopy --set-section-flags=.foo=contents %t %t.contents
+# RUN: llvm-readobj --sections %t.contents | FileCheck %s --check-prefixes=CHECK,WRITE
+# RUN: llvm-objcopy --set-section-flags=.foo=merge %t %t.merge
+# RUN: llvm-readobj --sections %t.merge | FileCheck %s --check-prefixes=CHECK,MERGE,WRITE
+# RUN: llvm-objcopy --set-section-flags=.foo=strings %t %t.strings
+# RUN: llvm-readobj --sections %t.strings | FileCheck %s --check-prefixes=CHECK,STRINGS,WRITE
+# RUN: llvm-objcopy --set-section-flags=.foo=share %t %t.share
+# RUN: llvm-readobj --sections %t.share | FileCheck %s --check-prefixes=CHECK,WRITE
+
+# Multiple flags:
+# RUN: llvm-objcopy --set-section-flags=.foo=alloc,readonly,strings %t %t.alloc_ro_strings
+# RUN: llvm-readobj --sections %t.alloc_ro_strings | FileCheck %s --check-prefixes=CHECK,ALLOC,STRINGS
+# RUN: llvm-objcopy --set-section-flags=.foo=alloc,code %t %t.alloc_code
+# RUN: llvm-readobj --sections %t.alloc_code | FileCheck %s --check-prefixes=CHECK,ALLOC,EXEC,WRITE
+
+# Invalid flags:
+# RUN: not llvm-objcopy --set-section-flags=.foo=xyzzy %t %t.xyzzy 2>&1 | FileCheck %s --check-prefix=BAD-FLAG
+
+# Bad flag format:
+# RUN: not llvm-objcopy --set-section-flags=.foo %t %t2 2>&1 | FileCheck %s --check-prefix=BAD-FORMAT
+
+# Setting flags for the same section multiple times:
+# RUN: not llvm-objcopy --set-section-flags=.foo=alloc --set-section-flags=.foo=load %t %t2 2>&1 | FileCheck %s --check-prefix=MULTIPLE-SETS
+
+!ELF
+FileHeader:
+  Class:           ELFCLASS64
+  Data:            ELFDATA2LSB
+  Type:            ET_REL
+  Machine:         EM_X86_64
+Sections:
+  - Name:            .foo
+    Type:            SHT_PROGBITS
+    Flags:           [ ]
+
+# CHECK:        Name: .foo
+# CHECK-NEXT:   Type: SHT_PROGBITS
+# CHECK-NEXT:   Flags [
+# ALLOC-NEXT:     SHF_ALLOC (0x2)
+# EXEC-NEXT:      SHF_EXECINSTR (0x4)
+# MERGE-NEXT:     SHF_MERGE (0x10)
+# STRINGS-NEXT:   SHF_STRINGS (0x20)
+# WRITE-NEXT:     SHF_WRITE (0x1)
+# CHECK-NEXT:   ]
+
+# BAD-FORMAT: Bad format for --set-section-flags: missing '='
+# MULTIPLE-SETS: --set-section-flags set multiple times for section .foo
+
+# BAD-FLAG: Unrecognized section flag 'xyzzy'. Flags supported for GNU compatibility: alloc, load, noload, readonly, debug, code, data, rom, share, contents, merge, strings.
diff --git a/test/tools/llvm-objcopy/ELF/strict-no-add.test b/test/tools/llvm-objcopy/ELF/strict-no-add.test
index 15f5251db34347884a26916f126d896bf3264dd6..348ab7c4fbd7fef200ab77349fb503bfa5dc4236 100644
--- a/test/tools/llvm-objcopy/ELF/strict-no-add.test
+++ b/test/tools/llvm-objcopy/ELF/strict-no-add.test
@@ -1,10 +1,10 @@
 # This test makes sure that sections added at the end that don't have symbols
 # defined in them don't trigger the creation of a large index table.
 
-RUN: %python %p/Inputs/ungzip.py %p/Inputs/many-sections.o.gz > %t.0
+RUN: %python %p/../Inputs/ungzip.py %p/Inputs/many-sections.o.gz > %t.0
 RUN: cat %p/Inputs/alloc-symtab.o > %t
 RUN: llvm-objcopy -R .text -R s0 -R s1 -R s2 -R s3 -R s4 -R s5 -R s6 %t.0 %t2
-RUN: llvm-objcopy -add-section=.s0=%t -add-section=.s1=%t -add-section=.s2=%t %t2 %t2
-RUN: llvm-readobj -sections %t2 | FileCheck --check-prefix=SECS %s
+RUN: llvm-objcopy --add-section=.s0=%t --add-section=.s1=%t --add-section=.s2=%t %t2 %t2
+RUN: llvm-readobj --sections %t2 | FileCheck --check-prefix=SECS %s
 
 SECS-NOT: Name: .symtab_shndx
diff --git a/test/tools/llvm-objcopy/ELF/strip-all-and-keep-symbol.test b/test/tools/llvm-objcopy/ELF/strip-all-and-keep-symbol.test
index 72e17a7d224160b42f4ea15dc986bbee6f9cf39b..9892a12650781f8187d1be8e923667868b2e66f0 100644
--- a/test/tools/llvm-objcopy/ELF/strip-all-and-keep-symbol.test
+++ b/test/tools/llvm-objcopy/ELF/strip-all-and-keep-symbol.test
@@ -1,13 +1,13 @@
 # RUN: yaml2obj %s > %t
 # RUN: cp %t %t1
 # RUN: llvm-objcopy --strip-all --keep-symbol foo %t %t2
-# RUN: llvm-readobj -sections -symbols %t2 | FileCheck %s
+# RUN: llvm-readobj --sections --symbols %t2 | FileCheck %s
 
 # Verify that the first run of llvm-objcopy
 # has not modified the input binary.
 # RUN: cmp %t %t1
 
-# RUN: llvm-strip -keep-symbol=foo %t1
+# RUN: llvm-strip --keep-symbol=foo %t1
 # RUN: cmp %t1 %t2
 
 # RUN: cp %t %t3
diff --git a/test/tools/llvm-objcopy/ELF/strip-all-and-remove.test b/test/tools/llvm-objcopy/ELF/strip-all-and-remove.test
index 9ee738aca245208d26d54e312795f2134713b688..5455c6bb7bca2a63db2a3eb85002fc108b3e6f98 100644
--- a/test/tools/llvm-objcopy/ELF/strip-all-and-remove.test
+++ b/test/tools/llvm-objcopy/ELF/strip-all-and-remove.test
@@ -1,10 +1,10 @@
 # RUN: yaml2obj %s > %t
 # RUN: cp %t %t1
 
-# RUN: llvm-strip -remove-section=.text.bar %t
-# RUN: llvm-readobj -file-headers -sections -symbols %t | FileCheck %s
+# RUN: llvm-strip --remove-section=.text.bar %t
+# RUN: llvm-readobj --file-headers --sections --symbols %t | FileCheck %s
 
-# RUN: llvm-objcopy -strip-all -remove-section=.text.bar %t1 %t1
+# RUN: llvm-objcopy --strip-all --remove-section=.text.bar %t1 %t1
 # RUN: cmp %t %t1
 
 !ELF
diff --git a/test/tools/llvm-objcopy/ELF/strip-all-gnu.test b/test/tools/llvm-objcopy/ELF/strip-all-gnu.test
index f6dbcc70cf409dc596f61553789b89926ff42268..8253526d23c0021b7e681e0e4660803d23120c62 100644
--- a/test/tools/llvm-objcopy/ELF/strip-all-gnu.test
+++ b/test/tools/llvm-objcopy/ELF/strip-all-gnu.test
@@ -2,7 +2,7 @@
 # RUN: cp %t %t1
 # RUN: llvm-objcopy --strip-all-gnu %t %t2
 # RUN: llvm-strip --strip-all-gnu %t -o %t3
-# RUN: llvm-readobj -file-headers -sections %t2 | FileCheck %s
+# RUN: llvm-readobj --file-headers --sections %t2 | FileCheck %s
 # RUN: cmp %t2 %t3
 
 !ELF
diff --git a/test/tools/llvm-objcopy/ELF/strip-all.test b/test/tools/llvm-objcopy/ELF/strip-all.test
index 5c5b6fd374ff74bb3d3df87de7da2ae09b2833ac..87388e0974769e2a7dfd5bb1d31aae2889d589a7 100644
--- a/test/tools/llvm-objcopy/ELF/strip-all.test
+++ b/test/tools/llvm-objcopy/ELF/strip-all.test
@@ -1,7 +1,7 @@
 # RUN: yaml2obj %s > %t
 # RUN: cp %t %t3
 # RUN: llvm-objcopy --strip-all %t %t2
-# RUN: llvm-readobj -file-headers -sections %t2 | FileCheck %s
+# RUN: llvm-readobj --file-headers --sections %t2 | FileCheck %s
 
 # Verify that the previous llvm-objcopy's run has not modified the input.
 # RUN: cmp %t %t3
diff --git a/test/tools/llvm-objcopy/ELF/strip-debug-and-remove.test b/test/tools/llvm-objcopy/ELF/strip-debug-and-remove.test
index 5d2b0c7af950ca0cf3309098a2f2cdbe50ba1d18..d2d0b34d9761727cc60a787d66ce472af108c9e3 100644
--- a/test/tools/llvm-objcopy/ELF/strip-debug-and-remove.test
+++ b/test/tools/llvm-objcopy/ELF/strip-debug-and-remove.test
@@ -1,10 +1,10 @@
 # RUN: yaml2obj %s > %t
 # RUN: cp %t %t1
 
-# RUN: llvm-strip -strip-debug -remove-section=.text.bar %t
-# RUN: llvm-readobj -file-headers -sections -symbols %t | FileCheck %s
+# RUN: llvm-strip --strip-debug --remove-section=.text.bar %t
+# RUN: llvm-readobj --file-headers --sections --symbols %t | FileCheck %s
 
-# RUN: llvm-objcopy -strip-debug -remove-section=.text.bar %t1 %t1
+# RUN: llvm-objcopy --strip-debug --remove-section=.text.bar %t1 %t1
 # RUN: cmp %t %t1
 
 !ELF
diff --git a/test/tools/llvm-objcopy/ELF/strip-debug.test b/test/tools/llvm-objcopy/ELF/strip-debug.test
index 6c833f301c7fca947d68f12ed7e4b78e664e56e2..d7079f6abcb2a0db24b149c96531bf1dc8b884f3 100644
--- a/test/tools/llvm-objcopy/ELF/strip-debug.test
+++ b/test/tools/llvm-objcopy/ELF/strip-debug.test
@@ -1,12 +1,16 @@
 # RUN: yaml2obj %s > %t
 # RUN: cp %t %t3
-# RUN: llvm-objcopy -strip-debug %t %t2
-# RUN: llvm-readobj -file-headers -sections -symbols %t2 | FileCheck %s
+# RUN: llvm-objcopy --strip-debug %t %t2
+# RUN: llvm-readobj --file-headers --sections --symbols %t2 | FileCheck %s
 
 # Verify that the previous run of llvm-objcopy has not modified the input.
 # RUN: cmp %t %t3
 
-# RUN: llvm-strip -strip-debug %t3
+# RUN: llvm-objcopy -g %t %t2g
+# Verify that --strip-debug and -g produce the same output
+# RUN: cmp %t2 %t2g
+
+# RUN: llvm-strip --strip-debug %t3
 # RUN: cmp %t2 %t3
 
 # RUN: cp %t %t4
diff --git a/test/tools/llvm-objcopy/ELF/strip-dwo-groups.test b/test/tools/llvm-objcopy/ELF/strip-dwo-groups.test
index 957bb553b725c7080a557f601f49fdbd688eca2d..724b0635b91b7d14e024979ac0507e98ea591c15 100644
--- a/test/tools/llvm-objcopy/ELF/strip-dwo-groups.test
+++ b/test/tools/llvm-objcopy/ELF/strip-dwo-groups.test
@@ -1,6 +1,6 @@
 # RUN: cp %p/Inputs/groups.o %t
-# RUN: llvm-objcopy -strip-dwo %t
-# RUN: llvm-readobj -elf-section-groups %t | FileCheck %s
+# RUN: llvm-objcopy --strip-dwo %t
+# RUN: llvm-readobj --elf-section-groups %t | FileCheck %s
 
 // Source code of groups.o:
 //
@@ -18,11 +18,11 @@
 //
 // clang -g -gsplit-dwarf -std=c++11 -c groups.cpp -o groups.o
 
-// `llvm-objcopy -strip-dwo` strips out dwo sections, as a result, the index of 
+// `llvm-objcopy --strip-dwo` strips out dwo sections, as a result, the index of 
 // the symbol table, the indices of the symbols and the indices of the sections 
 // which go after the removed ones will change. Consequently, the fields 
 // Link, Info and the content of .group need to be updated. In the past 
-// `llvm-objcopy -strip-dwo` used to produce invalid binaries with 
+// `llvm-objcopy --strip-dwo` used to produce invalid binaries with 
 // broken .group section, this test verifies the correctness of 
 // Link, Info and the content of this section.
 
diff --git a/test/tools/llvm-objcopy/ELF/strip-dwo-inplace.test b/test/tools/llvm-objcopy/ELF/strip-dwo-inplace.test
index 31bbd36a31a0b5304525ec18a55e4861c7436119..fc2d6c34280afc816614d0d036fda47442117228 100644
--- a/test/tools/llvm-objcopy/ELF/strip-dwo-inplace.test
+++ b/test/tools/llvm-objcopy/ELF/strip-dwo-inplace.test
@@ -1,6 +1,6 @@
 # RUN: cp %p/Inputs/dwarf.dwo %t
-# RUN: llvm-objcopy -strip-dwo %t
-# RUN: llvm-readobj -file-headers -sections %t | FileCheck %s
+# RUN: llvm-objcopy --strip-dwo %t
+# RUN: llvm-readobj --file-headers --sections %t | FileCheck %s
 
 CHECK:     SectionHeaderCount: 24
 
diff --git a/test/tools/llvm-objcopy/ELF/strip-multiple-files.test b/test/tools/llvm-objcopy/ELF/strip-multiple-files.test
index d1496eb52209002e00f6f7bef2a61e23b5f20a9d..9aef87284436e5c7b364c91f7175ddd115e39976 100644
--- a/test/tools/llvm-objcopy/ELF/strip-multiple-files.test
+++ b/test/tools/llvm-objcopy/ELF/strip-multiple-files.test
@@ -4,7 +4,7 @@
 # tests so we only have to run FileCheck on it once.
 # RUN: cp %t.o %t.1.o
 # RUN: llvm-strip --keep-symbol=foo %t.1.o -o %t.stripped.o
-# RUN: llvm-readobj -symbols %t.stripped.o | FileCheck %s
+# RUN: llvm-readobj --symbols %t.stripped.o | FileCheck %s
 
 # llvm-strip on two files:
 # RUN: cp %t.o %t.1.o
diff --git a/test/tools/llvm-objcopy/ELF/strip-non-alloc.test b/test/tools/llvm-objcopy/ELF/strip-non-alloc.test
index 270670cffd3c3b6c4549d6ae112fa9dbd8344ee3..8db732147d3632e116644a6f3a760614ba4c6ddd 100644
--- a/test/tools/llvm-objcopy/ELF/strip-non-alloc.test
+++ b/test/tools/llvm-objcopy/ELF/strip-non-alloc.test
@@ -1,6 +1,6 @@
 # RUN: yaml2obj %s > %t
 # RUN: llvm-objcopy --strip-non-alloc %t %t2
-# RUN: llvm-readobj -file-headers -sections %t2 | FileCheck %s
+# RUN: llvm-readobj --file-headers --sections %t2 | FileCheck %s
 
 !ELF
 FileHeader:
diff --git a/test/tools/llvm-objcopy/ELF/strip-preserve-atime.test b/test/tools/llvm-objcopy/ELF/strip-preserve-atime.test
index 5cee21fb96c47ea2c81bfeb4a47037cb3b322325..aad537cf48eb464ce8e1893b94e42f0f4f5eb535 100644
--- a/test/tools/llvm-objcopy/ELF/strip-preserve-atime.test
+++ b/test/tools/llvm-objcopy/ELF/strip-preserve-atime.test
@@ -43,7 +43,7 @@
 # Preserve dates in split DWO files.
 # RUN: cp %p/Inputs/dwarf.dwo %t-input.dwo
 # RUN: touch -a -t 199505050555.55 %t-input.dwo
-# RUN: llvm-objcopy -p -split-dwo=%t-dwo %t-input.dwo %t-nondwo
+# RUN: llvm-objcopy -p --split-dwo=%t-dwo %t-input.dwo %t-nondwo
 # RUN: ls -lu %t-dwo | FileCheck %s --check-prefix=CHECK-PRESERVE-ATIME
 # RUN: llvm-readobj %t-dwo
 # RUN: ls -lu %t-nondwo | FileCheck %s --check-prefix=CHECK-PRESERVE-ATIME
diff --git a/test/tools/llvm-objcopy/ELF/strip-preserve-mtime.test b/test/tools/llvm-objcopy/ELF/strip-preserve-mtime.test
index ac430b4ecbd81abe8a6ccf5711b0f6c705f3f3f8..5f247aefdbf8f1546ece64ea7a692fa625a9d20f 100644
--- a/test/tools/llvm-objcopy/ELF/strip-preserve-mtime.test
+++ b/test/tools/llvm-objcopy/ELF/strip-preserve-mtime.test
@@ -41,7 +41,7 @@
 # Preserve dates in split DWO files.
 # RUN: cp %p/Inputs/dwarf.dwo %t-input.dwo
 # RUN: touch -m -t 199705050555.55 %t-input.dwo
-# RUN: llvm-objcopy -p -split-dwo=%t-dwo %t-input.dwo %t-nondwo
+# RUN: llvm-objcopy -p --split-dwo=%t-dwo %t-input.dwo %t-nondwo
 # RUN: ls -l %t-dwo | FileCheck %s --check-prefix=CHECK-PRESERVE-MTIME
 # RUN: llvm-readobj %t-dwo
 # RUN: ls -l %t-nondwo | FileCheck %s --check-prefix=CHECK-PRESERVE-MTIME
diff --git a/test/tools/llvm-objcopy/ELF/strip-sections-keep.test b/test/tools/llvm-objcopy/ELF/strip-sections-keep.test
index f0031faef6e778e18e5ffcc35571bf76487303cc..711687f9253befe91c928c4858ce4578ffd7d30f 100644
--- a/test/tools/llvm-objcopy/ELF/strip-sections-keep.test
+++ b/test/tools/llvm-objcopy/ELF/strip-sections-keep.test
@@ -1,5 +1,5 @@
 # RUN: yaml2obj %s > %t
-# RUN: llvm-objcopy -strip-sections -keep-section=.shstrtab %t %t2
+# RUN: llvm-objcopy --strip-sections --keep-section=.shstrtab %t %t2
 # RUN: od -Ax -t c %t2 | FileCheck %s
 
 !ELF
diff --git a/test/tools/llvm-objcopy/ELF/strip-sections-only-section.test b/test/tools/llvm-objcopy/ELF/strip-sections-only-section.test
index 2c14a7272e3ee1ff68001c92ba81683bcfbd0f94..e2236a9fdd96160954c3fbf7a0c833fd142e2bd0 100644
--- a/test/tools/llvm-objcopy/ELF/strip-sections-only-section.test
+++ b/test/tools/llvm-objcopy/ELF/strip-sections-only-section.test
@@ -1,7 +1,7 @@
 # RUN: yaml2obj %s > %t
-# RUN: llvm-objcopy -strip-sections -only-section=.test %t %t2
+# RUN: llvm-objcopy --strip-sections --only-section=.test %t %t2
 # RUN: od -Ax -t x1 %t2 | FileCheck %s
-# RUN: od -Ax -t c  %t2 | FileCheck %s -check-prefix=TEXT
+# RUN: od -Ax -t c  %t2 | FileCheck %s --check-prefix=TEXT
 
 !ELF
 FileHeader:
diff --git a/test/tools/llvm-objcopy/ELF/strip-sections.test b/test/tools/llvm-objcopy/ELF/strip-sections.test
index fd8e224d27c6688f9b23f5571979d43958be7fdb..2dee6e2808d452aab8a045e365a9a5aa259b7f92 100644
--- a/test/tools/llvm-objcopy/ELF/strip-sections.test
+++ b/test/tools/llvm-objcopy/ELF/strip-sections.test
@@ -1,6 +1,6 @@
 # RUN: yaml2obj %s > %t
 # RUN: llvm-objcopy --strip-sections %t %t2
-# RUN: llvm-readobj -file-headers -program-headers %t2 | FileCheck %s
+# RUN: llvm-readobj --file-headers --program-headers %t2 | FileCheck %s
 # RUN: od -t x1 -j 4096 %t2 | FileCheck %s --check-prefix=DATA
 
 !ELF
diff --git a/test/tools/llvm-objcopy/ELF/strip-symbol.test b/test/tools/llvm-objcopy/ELF/strip-symbol.test
index 1b5a681989501a648818e538aca02457737f8caa..73d3c1cb39208a6e542571d88b7fd2959624e7de 100644
--- a/test/tools/llvm-objcopy/ELF/strip-symbol.test
+++ b/test/tools/llvm-objcopy/ELF/strip-symbol.test
@@ -1,6 +1,18 @@
 # RUN: yaml2obj %s > %t
 # RUN: llvm-objcopy --strip-symbol baz -N bar %t %t2
-# RUN: llvm-readobj -symbols -sections %t2 | FileCheck %s
+# RUN: llvm-readobj --symbols --sections %t2 | FileCheck %s
+# RUN: llvm-strip --strip-symbol baz -N bar %t -o %t3
+# RUN: cmp %t2 %t3
+# RUN: llvm-strip --regex --strip-symbol '^b.*' -N bar %t -o %t4
+# RUN: cmp %t3 %t4
+# RUN: echo " bar # bar" > %t-list.txt
+# RUN: echo " baz # baz" >> %t-list.txt
+# RUN: echo " # no symbol" >> %t-list.txt
+# RUN: llvm-objcopy --strip-symbols %t-list.txt %t %t5
+# RUN: cmp %t3 %t5
+# RUN: echo "b.* # bar & baz" > %t-list2.txt
+# RUN: llvm-objcopy --regex --strip-symbols %t-list2.txt %t %t6
+# RUN: cmp %t3 %t6
 
 !ELF
 FileHeader:
diff --git a/test/tools/llvm-objcopy/ELF/strip-unneeded.test b/test/tools/llvm-objcopy/ELF/strip-unneeded.test
index 8f96035e0959d943375edadcc8f9ce7f0f1134b0..60d83884715358608f30b8e9355b51a55a438b70 100644
--- a/test/tools/llvm-objcopy/ELF/strip-unneeded.test
+++ b/test/tools/llvm-objcopy/ELF/strip-unneeded.test
@@ -3,7 +3,25 @@
 # RUN: llvm-objcopy --strip-unneeded %t %t2
 # Verify that llvm-objcopy has not modified the input.
 # RUN: cmp %t %t1
-# RUN: llvm-readobj -symbols %t2 | FileCheck %s
+# RUN: llvm-readobj --symbols %t2 | FileCheck %s
+
+# RUN: llvm-objcopy --strip-unneeded-symbol=bar \
+# RUN:              %t %t3
+# RUN: llvm-readobj --symbols %t3 | FileCheck %s --check-prefixes=STRIP-BAR,REMAIN
+
+# RUN: llvm-objcopy --regex --strip-unneeded-symbol='.*' %t %t4
+# RUN: cmp %t2 %t4
+
+# RUN: echo " bar # bar " > %t.list.txt
+# RUN: echo "foobar" >> %t.list.txt
+# RUN: echo "foobaz" >> %t.list.txt
+# RUN: echo " # comment " >> %t.list.txt
+# RUN: llvm-objcopy --strip-unneeded-symbols %t.list.txt %t %t5
+# RUN: cmp %t2 %t5
+
+# RUN: echo " .* # * - remove all " > %t.list2.txt
+# RUN: llvm-objcopy --regex --strip-unneeded-symbols %t.list2.txt %t %t6
+# RUN: cmp %t2 %t6
 
 # Verify that llvm-strip modifies the symbol table the same way.
 # RUN: llvm-strip --strip-unneeded %t
@@ -141,3 +159,7 @@ Symbols:
 #CHECK-NEXT:    Section: .text
 #CHECK-NEXT:  }
 #CHECK-NEXT:]
+
+#STRIP-BAR-NOT: Name: bar ({{.*}})
+#REMAIN: Name: foobar
+#REMAIN: Name: foobaz
diff --git a/test/tools/llvm-objcopy/ELF/symbol-copy.test b/test/tools/llvm-objcopy/ELF/symbol-copy.test
index 3e346701fb93d79ece804420516460f6c7e3ccb2..43328680adf84bcf8dd6ddb1f2a33fc515748005 100644
--- a/test/tools/llvm-objcopy/ELF/symbol-copy.test
+++ b/test/tools/llvm-objcopy/ELF/symbol-copy.test
@@ -1,6 +1,6 @@
 # RUN: yaml2obj %s > %t
 # RUN: llvm-objcopy %t %t2
-# RUN: llvm-readobj -symbols %t2 | FileCheck %s
+# RUN: llvm-readobj --symbols %t2 | FileCheck %s
 
 !ELF
 FileHeader:
diff --git a/test/tools/llvm-objcopy/ELF/symtab-link.test b/test/tools/llvm-objcopy/ELF/symtab-link.test
index 7ed895e1c627a9df7b69f8dd7498d52cf0415ef4..7693bfc37ece24bc5bd24929a11bfbb8a6671864 100644
--- a/test/tools/llvm-objcopy/ELF/symtab-link.test
+++ b/test/tools/llvm-objcopy/ELF/symtab-link.test
@@ -1,9 +1,9 @@
 # RUN: yaml2obj %s > %t
 # RUN: llvm-objcopy %t %t2
-# RUN: llvm-readobj -sections %t2 | FileCheck %s
+# RUN: llvm-readobj --sections %t2 | FileCheck %s
 # RUN: cp %t %t3
 # RUN: llvm-strip --strip-debug %t3
-# RUN: llvm-readobj -sections %t3 | FileCheck %s
+# RUN: llvm-readobj --sections %t3 | FileCheck %s
 
 !ELF
 FileHeader:
diff --git a/test/tools/llvm-objcopy/ELF/two-seg-remove-end.test b/test/tools/llvm-objcopy/ELF/two-seg-remove-end.test
index f78a96410c154fe6cbe86c1de77ba297ee646e07..20c2689ec88a23da47afbee41f8c1cd008f9f858 100644
--- a/test/tools/llvm-objcopy/ELF/two-seg-remove-end.test
+++ b/test/tools/llvm-objcopy/ELF/two-seg-remove-end.test
@@ -1,7 +1,7 @@
 # RUN: yaml2obj %s -o %t
 # RUN: llvm-objcopy -R .text4 -O binary %t %t2
 # RUN: od -Ax -v -t x1 %t2 | FileCheck %s
-# RUN: wc -c %t2 | FileCheck %s -check-prefix=SIZE
+# RUN: wc -c %t2 | FileCheck %s --check-prefix=SIZE
 
 !ELF
 FileHeader:
diff --git a/test/tools/llvm-objcopy/ELF/two-seg-remove-first.test b/test/tools/llvm-objcopy/ELF/two-seg-remove-first.test
index 7d0ffefbb819a487990ba54e8b75f83703233047..a541fc691555beb68e53da837858f009afe8c090 100644
--- a/test/tools/llvm-objcopy/ELF/two-seg-remove-first.test
+++ b/test/tools/llvm-objcopy/ELF/two-seg-remove-first.test
@@ -1,7 +1,7 @@
 # RUN: yaml2obj %s -o %t
 # RUN: llvm-objcopy -R .text -O binary %t %t2
 # RUN: od -Ax -v -t x1 %t2 | FileCheck %s
-# RUN: wc -c %t2 | FileCheck %s -check-prefix=SIZE
+# RUN: wc -c %t2 | FileCheck %s --check-prefix=SIZE
 
 !ELF
 FileHeader:
diff --git a/test/tools/llvm-objcopy/ELF/two-seg-remove-third-sec.test b/test/tools/llvm-objcopy/ELF/two-seg-remove-third-sec.test
index bedd4aac6ae3b3067421a2598f589bd6a97be95c..254273d55e7ebbd0b1ebc827692aee379f7d512c 100644
--- a/test/tools/llvm-objcopy/ELF/two-seg-remove-third-sec.test
+++ b/test/tools/llvm-objcopy/ELF/two-seg-remove-third-sec.test
@@ -1,7 +1,7 @@
 # RUN: yaml2obj %s -o %t
 # RUN: llvm-objcopy -R .text3 -O binary %t %t2
 # RUN: od -Ax -v -t x1 %t2 | FileCheck %s
-# RUN: wc -c %t2 | FileCheck %s -check-prefix=SIZE
+# RUN: wc -c %t2 | FileCheck %s --check-prefix=SIZE
 
 !ELF
 FileHeader:
diff --git a/test/tools/llvm-objcopy/ELF/weaken-all.test b/test/tools/llvm-objcopy/ELF/weaken-all.test
index c9a7e5a258fe04a7c13dc0c0cfde6c85b85fbe07..6107d370adcba410ae0b1e5ee27a05f2749a63d2 100644
--- a/test/tools/llvm-objcopy/ELF/weaken-all.test
+++ b/test/tools/llvm-objcopy/ELF/weaken-all.test
@@ -1,6 +1,6 @@
 # RUN: yaml2obj %s > %t
 # RUN: llvm-objcopy --weaken %t %t2
-# RUN: llvm-readobj -symbols %t2 | FileCheck %s
+# RUN: llvm-readobj --symbols %t2 | FileCheck %s
 
 !ELF
 FileHeader:
diff --git a/test/tools/llvm-objcopy/ELF/weaken.test b/test/tools/llvm-objcopy/ELF/weaken.test
index b833ff62a214eb6d0f1b1af30188e4083dd2f711..01de61ab2758a0bd1f17a02f199f8607ec5806c7 100644
--- a/test/tools/llvm-objcopy/ELF/weaken.test
+++ b/test/tools/llvm-objcopy/ELF/weaken.test
@@ -1,6 +1,18 @@
 # RUN: yaml2obj %s > %t
 # RUN: llvm-objcopy --weaken-symbol Global -W Local -W Weak %t %t2
-# RUN: llvm-readobj -symbols %t2 | FileCheck %s
+# RUN: llvm-readobj --symbols %t2 | FileCheck %s
+# RUN: llvm-objcopy --regex --weaken-symbol='.*' %t %t3
+# RUN: cmp %t2 %t3
+# RUN: echo "  Global  # comment" > %t-list.txt
+# RUN: echo "Local" >> %t-list.txt
+# RUN: echo "Weak" >> %t-list.txt
+# RUN: echo " # comment" >> %t-list.txt
+# RUN: llvm-objcopy --weaken-symbols %t-list.txt %t %t4
+# RUN: cmp %t2 %t4
+# RUN: echo ".*al # Global + Local == .*al" > %t-list2.txt
+# RUN: echo "Weak # weak" >> %t-list2.txt
+# RUN: llvm-objcopy --regex --weaken-symbols %t-list2.txt %t %t5
+# RUN: cmp %t2 %t5
 
 !ELF
 FileHeader:
diff --git a/test/tools/llvm-objcopy/ELF/Inputs/ungzip.py b/test/tools/llvm-objcopy/Inputs/ungzip.py
similarity index 100%
rename from test/tools/llvm-objcopy/ELF/Inputs/ungzip.py
rename to test/tools/llvm-objcopy/Inputs/ungzip.py
diff --git a/test/tools/llvm-objcopy/MachO/Inputs/macho.32.s b/test/tools/llvm-objcopy/MachO/Inputs/macho.32.s
new file mode 100644
index 0000000000000000000000000000000000000000..66bcda8a9387c519ed4a0417064524472544c367
--- /dev/null
+++ b/test/tools/llvm-objcopy/MachO/Inputs/macho.32.s
@@ -0,0 +1,54 @@
+	.section	__TEXT,__text,regular,pure_instructions
+	.build_version macos, 10, 14
+	.globl	__Z1fi                  ## -- Begin function _Z1fi
+	.p2align	4, 0x90
+__Z1fi:                                 ## @_Z1fi
+	.cfi_startproc
+## %bb.0:
+	pushl	%ebp
+	.cfi_def_cfa_offset 8
+	.cfi_offset %ebp, -8
+	movl	%esp, %ebp
+	.cfi_def_cfa_register %ebp
+	pushl	%eax
+	calll	L0$pb
+L0$pb:
+	popl	%eax
+	movl	8(%ebp), %ecx
+	movl	_x-L0$pb(%eax), %eax
+	addl	8(%ebp), %eax
+	movl	%ecx, -4(%ebp)          ## 4-byte Spill
+	addl	$4, %esp
+	popl	%ebp
+	retl
+	.cfi_endproc
+                                        ## -- End function
+	.globl	_main                   ## -- Begin function main
+	.p2align	4, 0x90
+_main:                                  ## @main
+	.cfi_startproc
+## %bb.0:
+	pushl	%ebp
+	.cfi_def_cfa_offset 8
+	.cfi_offset %ebp, -8
+	movl	%esp, %ebp
+	.cfi_def_cfa_register %ebp
+	subl	$24, %esp
+	movl	$2, %eax
+	movl	$0, -4(%ebp)
+	movl	$2, (%esp)
+	movl	%eax, -8(%ebp)          ## 4-byte Spill
+	calll	__Z1fi
+	addl	$24, %esp
+	popl	%ebp
+	retl
+	.cfi_endproc
+                                        ## -- End function
+	.section	__DATA,__data
+	.globl	_x                      ## @x
+	.p2align	2
+_x:
+	.long	1                       ## 0x1
+
+
+.subsections_via_symbols
diff --git a/test/tools/llvm-objcopy/MachO/Inputs/macho.64.s b/test/tools/llvm-objcopy/MachO/Inputs/macho.64.s
new file mode 100644
index 0000000000000000000000000000000000000000..9d7e964972cd7053bcfbc56bcc992890b16afe05
--- /dev/null
+++ b/test/tools/llvm-objcopy/MachO/Inputs/macho.64.s
@@ -0,0 +1,47 @@
+	.section	__TEXT,__text,regular,pure_instructions
+	.build_version macos, 10, 14
+	.globl	__Z1fi                  ## -- Begin function _Z1fi
+	.p2align	4, 0x90
+__Z1fi:                                 ## @_Z1fi
+	.cfi_startproc
+## %bb.0:
+	pushq	%rbp
+	.cfi_def_cfa_offset 16
+	.cfi_offset %rbp, -16
+	movq	%rsp, %rbp
+	.cfi_def_cfa_register %rbp
+	movl	%edi, -4(%rbp)
+	movl	_x(%rip), %edi
+	addl	-4(%rbp), %edi
+	movl	%edi, %eax
+	popq	%rbp
+	retq
+	.cfi_endproc
+                                        ## -- End function
+	.globl	_main                   ## -- Begin function main
+	.p2align	4, 0x90
+_main:                                  ## @main
+	.cfi_startproc
+## %bb.0:
+	pushq	%rbp
+	.cfi_def_cfa_offset 16
+	.cfi_offset %rbp, -16
+	movq	%rsp, %rbp
+	.cfi_def_cfa_register %rbp
+	subq	$16, %rsp
+	movl	$2, %edi
+	movl	$0, -4(%rbp)
+	callq	__Z1fi
+	addq	$16, %rsp
+	popq	%rbp
+	retq
+	.cfi_endproc
+                                        ## -- End function
+	.section	__DATA,__data
+	.globl	_x                      ## @x
+	.p2align	2
+_x:
+	.long	1                       ## 0x1
+
+
+.subsections_via_symbols
diff --git a/test/tools/llvm-objcopy/MachO/basic-big-endian-32-copy.test b/test/tools/llvm-objcopy/MachO/basic-big-endian-32-copy.test
new file mode 100644
index 0000000000000000000000000000000000000000..692fb50f05816915ee5363444c36a3aef896629c
--- /dev/null
+++ b/test/tools/llvm-objcopy/MachO/basic-big-endian-32-copy.test
@@ -0,0 +1,112 @@
+# RUN: yaml2obj %s > %t
+# RUN: llvm-objcopy %t %t2
+# RUN: cmp %t %t2
+
+--- !mach-o
+IsLittleEndian:  false
+FileHeader:      
+  magic:           0xFEEDFACE
+  cputype:         0x00000007
+  cpusubtype:      0x00000003
+  filetype:        0x00000001
+  ncmds:           4
+  sizeofcmds:      380
+  flags:           0x00002000
+LoadCommands:    
+  - cmd:             LC_SEGMENT
+    cmdsize:         260
+    segname:         ''
+    vmaddr:          0
+    vmsize:          184
+    fileoff:         408
+    filesize:        184
+    maxprot:         7
+    initprot:        7
+    nsects:          3
+    flags:           0
+    Sections:        
+      - sectname:        __text
+        segname:         __TEXT
+        addr:            0x0000000000000000
+        size:            61
+        offset:          0x00000198
+        align:           4
+        reloff:          0x00000250
+        nreloc:          1
+        flags:           0x80000400
+        reserved1:       0x00000000
+        reserved2:       0x00000000
+        reserved3:       0x00000000
+      - sectname:        __compact_unwind
+        segname:         __LD
+        addr:            0x0000000000000040
+        size:            40
+        offset:          0x000001D8
+        align:           2
+        reloff:          0x00000258
+        nreloc:          2
+        flags:           0x02000000
+        reserved1:       0x00000000
+        reserved2:       0x00000000
+        reserved3:       0x00000000
+      - sectname:        __eh_frame
+        segname:         __TEXT
+        addr:            0x0000000000000068
+        size:            80
+        offset:          0x00000200
+        align:           2
+        reloff:          0x00000000
+        nreloc:          0
+        flags:           0x6800000B
+        reserved1:       0x00000000
+        reserved2:       0x00000000
+        reserved3:       0x00000000
+  - cmd:             LC_VERSION_MIN_MACOSX
+    cmdsize:         16
+    version:         658688
+    sdk:             0
+  - cmd:             LC_SYMTAB
+    cmdsize:         24
+    symoff:          616
+    nsyms:           2
+    stroff:          640
+    strsize:         16
+  - cmd:             LC_DYSYMTAB
+    cmdsize:         80
+    ilocalsym:       0
+    nlocalsym:       0
+    iextdefsym:      0
+    nextdefsym:      2
+    iundefsym:       2
+    nundefsym:       0
+    tocoff:          0
+    ntoc:            0
+    modtaboff:       0
+    nmodtab:         0
+    extrefsymoff:    0
+    nextrefsyms:     0
+    indirectsymoff:  0
+    nindirectsyms:   0
+    extreloff:       0
+    nextrel:         0
+    locreloff:       0
+    nlocrel:         0
+LinkEditData:    
+  NameList:        
+    - n_strx:          1
+      n_type:          0x0F
+      n_sect:          1
+      n_desc:          0
+      n_value:         0
+    - n_strx:          8
+      n_type:          0x0F
+      n_sect:          1
+      n_desc:          0
+      n_value:         32
+  StringTable:     
+    - ''
+    - __Z1fv
+    - _main
+    - ''
+    - ''
+...
diff --git a/test/tools/llvm-objcopy/MachO/basic-big-endian-64-copy.test b/test/tools/llvm-objcopy/MachO/basic-big-endian-64-copy.test
new file mode 100644
index 0000000000000000000000000000000000000000..5294f4e5be134c4d408b1f0905f6b7de17634373
--- /dev/null
+++ b/test/tools/llvm-objcopy/MachO/basic-big-endian-64-copy.test
@@ -0,0 +1,124 @@
+# RUN: yaml2obj %s > %t
+# RUN: llvm-objcopy %t %t2
+# RUN: cmp %t %t2
+
+--- !mach-o
+IsLittleEndian:  false
+FileHeader:      
+  magic:           0xFEEDFACF
+  cputype:         0x01000007
+  cpusubtype:      0x00000003
+  filetype:        0x00000001
+  ncmds:           4
+  sizeofcmds:      512
+  flags:           0x00002000
+  reserved:        0x00000000
+LoadCommands:    
+  - cmd:             LC_SEGMENT_64
+    cmdsize:         392
+    segname:         ''
+    vmaddr:          0
+    vmsize:          240
+    fileoff:         544
+    filesize:        240
+    maxprot:         7
+    initprot:        7
+    nsects:          4
+    flags:           0
+    Sections:        
+      - sectname:        __text
+        segname:         __TEXT
+        addr:            0x0000000000000000
+        size:            93
+        offset:          0x00000220
+        align:           4
+        reloff:          0x00000310
+        nreloc:          3
+        flags:           0x80000400
+        reserved1:       0x00000000
+        reserved2:       0x00000000
+        reserved3:       0x00000000
+      - sectname:        __cstring
+        segname:         __TEXT
+        addr:            0x000000000000005D
+        size:            48
+        offset:          0x0000027D
+        align:           0
+        reloff:          0x00000000
+        nreloc:          0
+        flags:           0x00000002
+        reserved1:       0x00000000
+        reserved2:       0x00000000
+        reserved3:       0x00000000
+      - sectname:        __compact_unwind
+        segname:         __LD
+        addr:            0x0000000000000090
+        size:            32
+        offset:          0x000002B0
+        align:           3
+        reloff:          0x00000328
+        nreloc:          1
+        flags:           0x02000000
+        reserved1:       0x00000000
+        reserved2:       0x00000000
+        reserved3:       0x00000000
+      - sectname:        __eh_frame
+        segname:         __TEXT
+        addr:            0x00000000000000B0
+        size:            64
+        offset:          0x000002D0
+        align:           3
+        reloff:          0x00000000
+        nreloc:          0
+        flags:           0x6800000B
+        reserved1:       0x00000000
+        reserved2:       0x00000000
+        reserved3:       0x00000000
+  - cmd:             LC_VERSION_MIN_MACOSX
+    cmdsize:         16
+    version:         658432
+    sdk:             0
+  - cmd:             LC_SYMTAB
+    cmdsize:         24
+    symoff:          816
+    nsyms:           2
+    stroff:          848
+    strsize:         36
+  - cmd:             LC_DYSYMTAB
+    cmdsize:         80
+    ilocalsym:       0
+    nlocalsym:       0
+    iextdefsym:      0
+    nextdefsym:      1
+    iundefsym:       1
+    nundefsym:       1
+    tocoff:          0
+    ntoc:            0
+    modtaboff:       0
+    nmodtab:         0
+    extrefsymoff:    0
+    nextrefsyms:     0
+    indirectsymoff:  0
+    nindirectsyms:   0
+    extreloff:       0
+    nextrel:         0
+    locreloff:       0
+    nlocrel:         0
+LinkEditData:    
+  NameList:        
+    - n_strx:          24
+      n_type:          0x0F
+      n_sect:          1
+      n_desc:          0
+      n_value:         0
+    - n_strx:          1
+      n_type:          0x01
+      n_sect:          0
+      n_desc:          0
+      n_value:         0
+  StringTable:     
+    - ''
+    - _compilerrt_abort_impl
+    - ___absvdi2
+    - ''
+...
diff --git a/test/tools/llvm-objcopy/MachO/basic-little-endian-32-copy.test b/test/tools/llvm-objcopy/MachO/basic-little-endian-32-copy.test
new file mode 100644
index 0000000000000000000000000000000000000000..c43aed7004bab4a27ee1b5415232e96a964ac332
--- /dev/null
+++ b/test/tools/llvm-objcopy/MachO/basic-little-endian-32-copy.test
@@ -0,0 +1,112 @@
+# RUN: yaml2obj %s > %t
+# RUN: llvm-objcopy %t %t2
+# RUN: cmp %t %t2
+
+--- !mach-o
+IsLittleEndian:  true
+FileHeader:      
+  magic:           0xFEEDFACE
+  cputype:         0x00000007
+  cpusubtype:      0x00000003
+  filetype:        0x00000001
+  ncmds:           4
+  sizeofcmds:      380
+  flags:           0x00002000
+LoadCommands:    
+  - cmd:             LC_SEGMENT
+    cmdsize:         260
+    segname:         ''
+    vmaddr:          0
+    vmsize:          184
+    fileoff:         408
+    filesize:        184
+    maxprot:         7
+    initprot:        7
+    nsects:          3
+    flags:           0
+    Sections:        
+      - sectname:        __text
+        segname:         __TEXT
+        addr:            0x0000000000000000
+        size:            61
+        offset:          0x00000198
+        align:           4
+        reloff:          0x00000250
+        nreloc:          1
+        flags:           0x80000400
+        reserved1:       0x00000000
+        reserved2:       0x00000000
+        reserved3:       0x00000000
+      - sectname:        __compact_unwind
+        segname:         __LD
+        addr:            0x0000000000000040
+        size:            40
+        offset:          0x000001D8
+        align:           2
+        reloff:          0x00000258
+        nreloc:          2
+        flags:           0x02000000
+        reserved1:       0x00000000
+        reserved2:       0x00000000
+        reserved3:       0x00000000
+      - sectname:        __eh_frame
+        segname:         __TEXT
+        addr:            0x0000000000000068
+        size:            80
+        offset:          0x00000200
+        align:           2
+        reloff:          0x00000000
+        nreloc:          0
+        flags:           0x6800000B
+        reserved1:       0x00000000
+        reserved2:       0x00000000
+        reserved3:       0x00000000
+  - cmd:             LC_VERSION_MIN_MACOSX
+    cmdsize:         16
+    version:         658688
+    sdk:             0
+  - cmd:             LC_SYMTAB
+    cmdsize:         24
+    symoff:          616
+    nsyms:           2
+    stroff:          640
+    strsize:         16
+  - cmd:             LC_DYSYMTAB
+    cmdsize:         80
+    ilocalsym:       0
+    nlocalsym:       0
+    iextdefsym:      0
+    nextdefsym:      2
+    iundefsym:       2
+    nundefsym:       0
+    tocoff:          0
+    ntoc:            0
+    modtaboff:       0
+    nmodtab:         0
+    extrefsymoff:    0
+    nextrefsyms:     0
+    indirectsymoff:  0
+    nindirectsyms:   0
+    extreloff:       0
+    nextrel:         0
+    locreloff:       0
+    nlocrel:         0
+LinkEditData:    
+  NameList:        
+    - n_strx:          1
+      n_type:          0x0F
+      n_sect:          1
+      n_desc:          0
+      n_value:         0
+    - n_strx:          8
+      n_type:          0x0F
+      n_sect:          1
+      n_desc:          0
+      n_value:         32
+  StringTable:     
+    - ''
+    - __Z1fv
+    - _main
+    - ''
+    - ''
+...
diff --git a/test/tools/llvm-objcopy/MachO/basic-little-endian-64-copy.test b/test/tools/llvm-objcopy/MachO/basic-little-endian-64-copy.test
new file mode 100644
index 0000000000000000000000000000000000000000..b395020b4bc414dd9ea5d9a32f397dd031571256
--- /dev/null
+++ b/test/tools/llvm-objcopy/MachO/basic-little-endian-64-copy.test
@@ -0,0 +1,124 @@
+# RUN: yaml2obj %s > %t
+# RUN: llvm-objcopy %t %t2
+# RUN: cmp %t %t2
+
+--- !mach-o
+IsLittleEndian:  true
+FileHeader:      
+  magic:           0xFEEDFACF
+  cputype:         0x01000007
+  cpusubtype:      0x00000003
+  filetype:        0x00000001
+  ncmds:           4
+  sizeofcmds:      512
+  flags:           0x00002000
+  reserved:        0x00000000
+LoadCommands:    
+  - cmd:             LC_SEGMENT_64
+    cmdsize:         392
+    segname:         ''
+    vmaddr:          0
+    vmsize:          240
+    fileoff:         544
+    filesize:        240
+    maxprot:         7
+    initprot:        7
+    nsects:          4
+    flags:           0
+    Sections:        
+      - sectname:        __text
+        segname:         __TEXT
+        addr:            0x0000000000000000
+        size:            93
+        offset:          0x00000220
+        align:           4
+        reloff:          0x00000310
+        nreloc:          3
+        flags:           0x80000400
+        reserved1:       0x00000000
+        reserved2:       0x00000000
+        reserved3:       0x00000000
+      - sectname:        __cstring
+        segname:         __TEXT
+        addr:            0x000000000000005D
+        size:            48
+        offset:          0x0000027D
+        align:           0
+        reloff:          0x00000000
+        nreloc:          0
+        flags:           0x00000002
+        reserved1:       0x00000000
+        reserved2:       0x00000000
+        reserved3:       0x00000000
+      - sectname:        __compact_unwind
+        segname:         __LD
+        addr:            0x0000000000000090
+        size:            32
+        offset:          0x000002B0
+        align:           3
+        reloff:          0x00000328
+        nreloc:          1
+        flags:           0x02000000
+        reserved1:       0x00000000
+        reserved2:       0x00000000
+        reserved3:       0x00000000
+      - sectname:        __eh_frame
+        segname:         __TEXT
+        addr:            0x00000000000000B0
+        size:            64
+        offset:          0x000002D0
+        align:           3
+        reloff:          0x00000000
+        nreloc:          0
+        flags:           0x6800000B
+        reserved1:       0x00000000
+        reserved2:       0x00000000
+        reserved3:       0x00000000
+  - cmd:             LC_VERSION_MIN_MACOSX
+    cmdsize:         16
+    version:         658432
+    sdk:             0
+  - cmd:             LC_SYMTAB
+    cmdsize:         24
+    symoff:          816
+    nsyms:           2
+    stroff:          848
+    strsize:         36
+  - cmd:             LC_DYSYMTAB
+    cmdsize:         80
+    ilocalsym:       0
+    nlocalsym:       0
+    iextdefsym:      0
+    nextdefsym:      1
+    iundefsym:       1
+    nundefsym:       1
+    tocoff:          0
+    ntoc:            0
+    modtaboff:       0
+    nmodtab:         0
+    extrefsymoff:    0
+    nextrefsyms:     0
+    indirectsymoff:  0
+    nindirectsyms:   0
+    extreloff:       0
+    nextrel:         0
+    locreloff:       0
+    nlocrel:         0
+LinkEditData:    
+  NameList:        
+    - n_strx:          24
+      n_type:          0x0F
+      n_sect:          1
+      n_desc:          0
+      n_value:         0
+    - n_strx:          1
+      n_type:          0x01
+      n_sect:          0
+      n_desc:          0
+      n_value:         0
+  StringTable:     
+    - ''
+    - _compilerrt_abort_impl
+    - ___absvdi2
+    - ''
+...
diff --git a/test/tools/llvm-objcopy/MachO/real-world-input-copy.test b/test/tools/llvm-objcopy/MachO/real-world-input-copy.test
new file mode 100644
index 0000000000000000000000000000000000000000..d8e68a2e1410646b2ef6127a4e5e12cc8e43f26e
--- /dev/null
+++ b/test/tools/llvm-objcopy/MachO/real-world-input-copy.test
@@ -0,0 +1,9 @@
+# RUN: llvm-mc -assemble -triple i386-apple-darwin9 -filetype=obj %p/Inputs/macho.32.s -o %t.32.o
+# RUN: llvm-objcopy %t.32.o %t.32.copy.o
+# RUN: cmp %t.32.o %t.32.copy.o
+
+# RUN: llvm-mc -assemble -triple x86_64-apple-darwin9 -filetype=obj %p/Inputs/macho.64.s -o %t.64.o
+# RUN: llvm-objcopy %t.64.o %t.64.copy.o
+# RUN: cmp %t.64.o %t.64.copy.o
+
+# REQUIRES: x86-registered-target
diff --git a/test/tools/llvm-objdump/Hexagon/source-interleave-hexagon.ll b/test/tools/llvm-objdump/Hexagon/source-interleave-hexagon.ll
index bfa18e00ad9035a8d701f5264690ca2965946569..02cef67643d261b3a13d2fd9ccc58c0ed24247f3 100644
--- a/test/tools/llvm-objdump/Hexagon/source-interleave-hexagon.ll
+++ b/test/tools/llvm-objdump/Hexagon/source-interleave-hexagon.ll
@@ -41,7 +41,7 @@ attributes #1 = { nounwind readnone }
 !llvm.module.flags = !{!6, !7}
 !llvm.ident = !{!8}
 
-!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 4.0.0 (http://llvm.org/git/clang d19a95e94dc57c5a72fd25d64f26134aa7d25fa0) (http://llvm.org/git/llvm.git 313924e6ff8a332063f61d3fda03812c220762f6)", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, globals: !3)
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 4.0.0", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, globals: !3)
 !1 = !DIFile(filename: "source-interleave-hexagon.c", directory: "SRC_COMPDIR")
 !2 = !{}
 !3 = !{!4}
@@ -49,7 +49,7 @@ attributes #1 = { nounwind readnone }
 !5 = !DIBasicType(name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
 !6 = !{i32 2, !"Dwarf Version", i32 4}
 !7 = !{i32 2, !"Debug Info Version", i32 3}
-!8 = !{!"clang version 4.0.0 (http://llvm.org/git/clang d19a95e94dc57c5a72fd25d64f26134aa7d25fa0) (http://llvm.org/git/llvm.git 313924e6ff8a332063f61d3fda03812c220762f6)"}
+!8 = !{!"clang version 4.0.0"}
 !9 = distinct !DISubprogram(name: "foo", scope: !1, file: !1, line: 2, type: !10, isLocal: false, isDefinition: true, scopeLine: 2, isOptimized: false, unit: !0, retainedNodes: !2)
 !10 = !DISubroutineType(types: !11)
 !11 = !{!5}
diff --git a/test/tools/llvm-objdump/Inputs/trivial.obj.wasm b/test/tools/llvm-objdump/Inputs/trivial.obj.wasm
index 8652d67f69222ab57610df1b13d3972e031460a1..a894522fa152ba86d59c6c585788e1467809f416 100644
Binary files a/test/tools/llvm-objdump/Inputs/trivial.obj.wasm and b/test/tools/llvm-objdump/Inputs/trivial.obj.wasm differ
diff --git a/test/tools/llvm-objdump/PowerPC/branch-offset.s b/test/tools/llvm-objdump/PowerPC/branch-offset.s
new file mode 100644
index 0000000000000000000000000000000000000000..b0b3f05f9cdb5e7a3a8c0bd6c16a7e59d5a9ca74
--- /dev/null
+++ b/test/tools/llvm-objdump/PowerPC/branch-offset.s
@@ -0,0 +1,43 @@
+# RUN: llvm-mc -triple=powerpc64le-unknown-linux -filetype=obj %s -o %t.o
+# RUN: llvm-objdump -d %t.o | FileCheck %s
+
+# RUN: llvm-mc -triple=powerpc64-unknown-linux -filetype=obj %s -o %t.o
+# RUN: llvm-objdump -d %t.o | FileCheck %s
+
+# RUN: llvm-mc -triple=powerpc-unknown-linux -filetype=obj %s -o %t.o
+# RUN: llvm-objdump -d %t.o | FileCheck %s
+
+# CHECK: 0000000000000000 callee_back:
+# CHECK: 18: {{.*}} bl .-24
+# CHECK: 20: {{.*}} bl .+16
+# CHECK: 0000000000000030 callee_forward:
+
+        .text
+        .global caller
+        .type caller,@function
+        .type callee_forward,@function
+        .type callee_back,@function
+
+        .p2align 4
+callee_back:
+        li 3, 55
+        blr
+
+        .p2align 4
+caller:
+.Lgep:
+        addis 2, 12, .TOC.-.Lgep@ha
+        addi 2, 2, .TOC.-.Lgep@l
+.Llep:
+        .localentry caller, .Llep-.Lgep
+        bl callee_back
+        mr 31, 3
+        bl callee_forward
+        add 3, 3, 31
+        blr
+
+        .p2align 4
+callee_forward:
+        li 3, 66
+        blr
+
diff --git a/test/tools/llvm-objdump/PowerPC/lit.local.cfg b/test/tools/llvm-objdump/PowerPC/lit.local.cfg
new file mode 100644
index 0000000000000000000000000000000000000000..b77510721e100b59db080fdb4c58018cc083f8f3
--- /dev/null
+++ b/test/tools/llvm-objdump/PowerPC/lit.local.cfg
@@ -0,0 +1,2 @@
+if not 'PowerPC' in config.root.targets:
+      config.unsupported = True
diff --git a/test/tools/llvm-objdump/WebAssembly/relocations.test b/test/tools/llvm-objdump/WebAssembly/relocations.test
index 1384dd6bf1aef4ae9a9c162f00d2be3924fcbce2..acf276e7ba76aad28f039334f0bf7086bb51e5f8 100644
--- a/test/tools/llvm-objdump/WebAssembly/relocations.test
+++ b/test/tools/llvm-objdump/WebAssembly/relocations.test
@@ -10,7 +10,7 @@ entry:
 }
 
 ; CHECK:      RELOCATION RECORDS FOR [CODE]:
-; CHECK-NEXT: 00000006 R_WEBASSEMBLY_TYPE_INDEX_LEB 1+0
+; CHECK-NEXT: 00000006 R_WASM_TYPE_INDEX_LEB 1+0
 
 ; CHECK:      RELOCATION RECORDS FOR [DATA]:
-; CHECK-NEXT: 00000006 R_WEBASSEMBLY_MEMORY_ADDR_I32 foo+0
+; CHECK-NEXT: 00000006 R_WASM_MEMORY_ADDR_I32 foo+0
diff --git a/test/tools/llvm-objdump/WebAssembly/symbol-table.test b/test/tools/llvm-objdump/WebAssembly/symbol-table.test
index b8455aaf5aea9239d703302ca052c6ac13dba59a..b67b0a366b1bd964a7989241a54e907b84e51894 100644
--- a/test/tools/llvm-objdump/WebAssembly/symbol-table.test
+++ b/test/tools/llvm-objdump/WebAssembly/symbol-table.test
@@ -1,9 +1,9 @@
 RUN: llvm-objdump -t %p/../Inputs/trivial.obj.wasm | FileCheck %s
 
 CHECK:      SYMBOL TABLE:
-CHECK-NEXT: 00000002 g     F CODE	main
+CHECK-NEXT: 00000001 g     F CODE	main
 CHECK-NEXT: 00000000 l     O DATA	.L.str
 CHECK-NEXT: 00000000 g     F *UND*	puts
-CHECK-NEXT: 00000003 l     F CODE	.LSomeOtherFunction_bitcast
+CHECK-NEXT: 00000019 l     F CODE	.LSomeOtherFunction_bitcast
 CHECK-NEXT: 00000000 g     F *UND*	SomeOtherFunction
 CHECK-NEXT: 00000010 g     O DATA	var
diff --git a/test/tools/llvm-objdump/X86/Inputs/macho-multiple-text b/test/tools/llvm-objdump/X86/Inputs/macho-multiple-text
new file mode 100755
index 0000000000000000000000000000000000000000..7b04830ff27e3cd05ba4970126c691ef14b868a4
Binary files /dev/null and b/test/tools/llvm-objdump/X86/Inputs/macho-multiple-text differ
diff --git a/test/tools/llvm-objdump/X86/adjust-vma.test b/test/tools/llvm-objdump/X86/adjust-vma.test
new file mode 100644
index 0000000000000000000000000000000000000000..6b1a7c65c1a51589b5c4d05efbab25b6965647b2
--- /dev/null
+++ b/test/tools/llvm-objdump/X86/adjust-vma.test
@@ -0,0 +1,131 @@
+# RUN: yaml2obj %s > %t
+# RUN: llvm-objdump --all-headers -D -z %t | FileCheck %s --check-prefixes=COMMON,NOADJUST
+# RUN: llvm-objdump --all-headers -D -z --adjust-vma=0x0 %t | FileCheck %s --check-prefixes=COMMON,NOADJUST
+# RUN: llvm-objdump --all-headers -D -z --adjust-vma=0x123000 %t | FileCheck %s --check-prefixes=COMMON,ADJUST
+
+# NOADJUST:      Disassembly of section .text:
+# NOADJUST-NEXT: 0000000000000000 sym:
+# NOADJUST-NEXT:   0: {{.*}}  nop
+# NOADJUST:      0000000000000001 func:
+# NOADJUST-NEXT:   1: {{.*}}  retq
+
+# ADJUST:        Disassembly of section .text:
+# ADJUST-NEXT:   0000000000123000 sym:
+# ADJUST-NEXT:     123000: {{.*}} nop
+# ADJUST:        0000000000123001 func:
+# ADJUST-NEXT:     123001: {{.*}} retq
+
+# NOADJUST:      Disassembly of section .debug_str:
+# NOADJUST-NEXT: 0000000000000000 .debug_str:
+# NOADJUST-NEXT:   0: {{.*}}  %al, (%rax)
+# NOADJUST-NEXT:      0000000000000001:  R_X86_64_32  .text
+# NOADJUST-NEXT:   2: {{.*}}  addb    %al, (%rax)
+
+# ADJUST:        Disassembly of section .debug_str:
+# ADJUST-NEXT:   0000000000000000 .debug_str:
+# ADJUST-NEXT:     0: {{.*}}  %al, (%rax)
+# ADJUST-NEXT:        0000000000123001:  R_X86_64_32  .text
+# ADJUST-NEXT:     2: {{.*}}  addb    %al, (%rax)
+
+# COMMON:        Disassembly of section .rela.debug_str:
+# COMMON-NEXT:   0000000000000000    .rela.debug_str:
+# COMMON-NEXT:     0: {{.*}}  addl %eax, (%rax)
+## ... There are more lines here. We do not care.
+
+# NOADJUST:      Disassembly of section .data:
+# NOADJUST-NEXT: 0000000000000000  .data:
+# NOADJUST-NEXT:  0: {{.*}} addb    %al, (%rax)
+# NOADJUST-NEXT: 0000000000000000: R_X86_64_32  .text
+# NOADJUST-NEXT:  2: {{.*}} addb    %al, (%rax)
+
+# ADJUST:        Disassembly of section .data:
+# ADJUST-NEXT:   0000000000123000  .data:
+# ADJUST-NEXT:    123000: {{.*}}  addb    %al, (%rax)
+# ADJUST-NEXT:   0000000000123000: R_X86_64_32  .text
+# ADJUST-NEXT:    123002: {{.*}}  addb    %al, (%rax)
+
+# COMMON:        Disassembly of section .rela.data:
+# COMMON-NEXT:   0000000000000000   .rela.data:
+# COMMON-NEXT:    0: {{.*}} addb %al, (%rax)
+## ... There are more lines here. We do not care.
+
+# NOADJUST:      Sections:
+# NOADJUST-NEXT: Idx Name           Size     VMA              Type
+# NOADJUST-NEXT:  0                 00000000 0000000000000000
+# NOADJUST-NEXT:  1 .text           00000002 0000000000000000 TEXT
+# NOADJUST-NEXT:  2 .debug_str      00000004 0000000000000000
+# NOADJUST-NEXT:  3 .rela.debug_str 00000018 0000000000000000
+# NOADJUST-NEXT:  4 .data           00000004 0000000000000000 DATA
+# NOADJUST-NEXT:  5 .rela.data      00000018 0000000000000000
+# NOADJUST-NEXT:  6 .symtab         00000060 0000000000000000
+# NOADJUST-NEXT:  7 .strtab         00000010 0000000000000000
+# NOADJUST-NEXT:  8 .shstrtab       0000003c 0000000000000000
+
+# ADJUST:        Sections:
+# ADJUST-NEXT:   Idx Name           Size     VMA              Type
+# ADJUST-NEXT:    0                 00000000 0000000000000000
+# ADJUST-NEXT:    1 .text           00000002 0000000000123000 TEXT
+# ADJUST-NEXT:    2 .debug_str      00000004 0000000000000000
+# ADJUST-NEXT:    3 .rela.debug_str 00000018 0000000000000000
+# ADJUST-NEXT:    4 .data           00000004 0000000000123000 DATA
+# ADJUST-NEXT:    5 .rela.data      00000018 0000000000000000
+# ADJUST-NEXT:    6 .symtab         00000060 0000000000000000
+# ADJUST-NEXT:    7 .strtab         00000010 0000000000000000
+# ADJUST-NEXT:    8 .shstrtab       0000003c 0000000000000000
+
+# COMMON:      SYMBOL TABLE:
+# COMMON-NEXT:  0000000000000001  l F .text 00000000 func
+# COMMON-NEXT:  0000000000000000      .text 00000000 sym
+# COMMON-NEXT:  0000000000000000  l d .text 00000000 .text
+
+--- !ELF
+FileHeader:      
+  Class:           ELFCLASS64
+  Data:            ELFDATA2LSB
+  Type:            ET_REL
+  Machine:         EM_X86_64
+Sections:        
+  - Name:            .text
+    Type:            SHT_PROGBITS
+    Flags:           [ SHF_ALLOC, SHF_EXECINSTR ]
+    AddressAlign:    0x0000000000000004
+    Content:         90C3
+  - Name:            .debug_str
+    Type:            SHT_PROGBITS
+    Flags:           [ SHF_MERGE, SHF_STRINGS ]
+    AddressAlign:    0x0000000000000001
+    Content:         '00000000'
+  - Name:            .rela.debug_str
+    Type:            SHT_RELA
+    Link:            .symtab
+    AddressAlign:    0x0000000000000008
+    Info:            .debug_str
+    Relocations:     
+      - Offset:          0x0000000000000001
+        Symbol:          .text
+        Type:            R_X86_64_32
+  - Name:            .data
+    Type:            SHT_PROGBITS
+    Flags:           [ SHF_WRITE, SHF_ALLOC ]
+    AddressAlign:    0x0000000000000001
+    Content:         '00000000'
+  - Name:            .rela.data
+    Type:            SHT_RELA
+    Link:            .symtab
+    AddressAlign:    0x0000000000000008
+    Info:            .data
+    Relocations:     
+      - Offset:      0x0000000000000000
+        Symbol:      .text
+        Type:        R_X86_64_32
+Symbols:
+  Local:
+    - Name:            func
+      Type:            STT_FUNC
+      Section:         .text
+      Value:           0x0000000000000001
+    - Name:            sym
+      Section:         .text
+    - Name:            .text
+      Type:            STT_SECTION
+      Section:         .text
diff --git a/test/tools/llvm-objdump/X86/demangle.s b/test/tools/llvm-objdump/X86/demangle.s
new file mode 100644
index 0000000000000000000000000000000000000000..e20fa3de5c2c48799c97603cfca342609dc387eb
--- /dev/null
+++ b/test/tools/llvm-objdump/X86/demangle.s
@@ -0,0 +1,22 @@
+# RUN: llvm-mc %s -filetype=obj -triple=x86_64-pc-linux -o %t
+# RUN: llvm-objdump -syms -reloc -demangle %t | FileCheck %s
+
+## Check we demangle symbols when printing relocations.
+# CHECK:      000000000000001 R_X86_64_PLT32 foo()-4
+
+## Check we demangle symbols when printing symbol table.
+# CHECK:      SYMBOL TABLE:
+# CHECK-NEXT: 0000000000000000 g     F .text           00000000 foo()
+
+## Check the case when relocations are inlined into disassembly.
+# RUN: llvm-objdump -d -r -demangle %t | FileCheck %s --check-prefix=INLINE
+# INLINE:      Disassembly of section .text:
+# INLINE-NEXT: foo():
+# INLINE-NEXT:  0: {{.*}}  callq   0 <_Z3foov+0x5>
+# INLINE-NEXT:  0000000000000001:  R_X86_64_PLT32 foo()-4
+
+.text
+.globl _Z3foov
+.type _Z3foov,@function
+_Z3foov:
+ callq _Z3foov@PLT
diff --git a/test/tools/llvm-objdump/X86/disasm-text.test b/test/tools/llvm-objdump/X86/disasm-text.test
new file mode 100644
index 0000000000000000000000000000000000000000..38422d81935a0c20369e8b89a34deca891d89511
--- /dev/null
+++ b/test/tools/llvm-objdump/X86/disasm-text.test
@@ -0,0 +1,11 @@
+# RUN: llvm-mc -filetype=obj -triple=x86_64-unknown-linux %s -o -| \
+# RUN: llvm-objdump -d - | FileCheck %s
+
+.globl foo
+.type foo, @object
+foo:
+.ascii "this is a test"
+
+# CHECK: foo:
+# CHECK: 0:{{.*}}this is
+# CHECK: 8:{{.*}}a test
diff --git a/test/tools/llvm-objdump/X86/macho-disassemble-all.test b/test/tools/llvm-objdump/X86/macho-disassemble-all.test
new file mode 100644
index 0000000000000000000000000000000000000000..ce22d740087d37206f490bc4d71e0e74acf5889e
--- /dev/null
+++ b/test/tools/llvm-objdump/X86/macho-disassemble-all.test
@@ -0,0 +1,39 @@
+// RUN: llvm-objdump -m -d -full-leading-addr -print-imm-hex -no-show-raw-insn %p/Inputs/macho-multiple-text | FileCheck %s -check-prefix=TEXT
+
+TEXT:      (__TEXT,__text) section
+TEXT_NEXT: _main:
+TEXT_NEXT: 0000000100000f60	pushq	%rbp
+TEXT_NEXT: 0000000100000f61	movq	%rsp, %rbp
+TEXT_NEXT: 0000000100000f64	subq	$0x10, %rsp
+TEXT_NEXT: 0000000100000f68	movl	$0x0, -0x4(%rbp)
+TEXT_NEXT: 0000000100000f6f	callq	_hello
+TEXT_NEXT: 0000000100000f74	xorl	%eax, %eax
+TEXT_NEXT: 0000000100000f76	addq	$0x10, %rsp
+TEXT_NEXT: 0000000100000f7a	popq	%rbp
+TEXT_NEXT: 0000000100000f7b	retq
+
+// RUN: llvm-objdump -m -D -full-leading-addr -print-imm-hex -no-show-raw-insn %p/Inputs/macho-multiple-text | FileCheck %s -check-prefix=ALL
+
+ALL:      (__TEXT,__text) section
+ALL_NEXT: _main:
+ALL_NEXT: 0000000100000f60	pushq	%rbp
+ALL_NEXT: 0000000100000f61	movq	%rsp, %rbp
+ALL_NEXT: 0000000100000f64	subq	$0x10, %rsp
+ALL_NEXT: 0000000100000f68	movl	$0x0, -0x4(%rbp)
+ALL_NEXT: 0000000100000f6f	callq	_hello
+ALL_NEXT: 0000000100000f74	xorl	%eax, %eax
+ALL_NEXT: 0000000100000f76	addq	$0x10, %rsp
+ALL_NEXT: 0000000100000f7a	popq	%rbp
+ALL_NEXT: 0000000100000f7b	retq
+ALL:      (__TEXT_EXEC,__text) section
+ALL_NEXT: _hello:
+ALL_NEXT: 0000000100001000	pushq	%rbp
+ALL_NEXT: 0000000100001001	movq	%rsp, %rbp
+ALL_NEXT: 0000000100001004	subq	$0x10, %rsp
+ALL_NEXT: 0000000100001008	leaq	-0x71(%rip), %rdi ## literal pool for: "hello, world!\n"
+ALL_NEXT: 000000010000100f	movb	$0x0, %al
+ALL_NEXT: 0000000100001011	callq	0x100000f7c ## symbol stub for: _printf
+ALL_NEXT: 0000000100001016	movl	%eax, -0x4(%rbp)
+ALL_NEXT: 0000000100001019	addq	$0x10, %rsp
+ALL_NEXT: 000000010000101d	popq	%rbp
+ALL_NEXT: 000000010000101e	retq
diff --git a/test/tools/llvm-objdump/X86/macho-section-headers.test b/test/tools/llvm-objdump/X86/macho-section-headers.test
index 5159d1808228afc1335337cb36a41a9cb384566b..770b25b22ba520901384006848c420accb180c8a 100644
--- a/test/tools/llvm-objdump/X86/macho-section-headers.test
+++ b/test/tools/llvm-objdump/X86/macho-section-headers.test
@@ -1,7 +1,7 @@
 RUN: llvm-objdump -macho -h %p/Inputs/hello.obj.macho-x86_64 | FileCheck %s
 
 CHECK: Sections:
-CHECK: Idx Name          Size      Address          Type
+CHECK: Idx Name          Size     VMA              Type
 CHECK:   0 __text        0000003b 0000000000000000 TEXT 
 CHECK:   1 __cstring     0000000d 000000000000003b DATA 
 CHECK:   2 __compact_unwind 00000020 0000000000000048 DATA 
diff --git a/test/tools/llvm-objdump/X86/out-of-section-sym.test b/test/tools/llvm-objdump/X86/out-of-section-sym.test
index f70dce6a8ed81e46e520680e3b97c08d7ebdf6d7..55de107a60b31757889fa2adf157569f0e0e613f 100644
--- a/test/tools/llvm-objdump/X86/out-of-section-sym.test
+++ b/test/tools/llvm-objdump/X86/out-of-section-sym.test
@@ -6,7 +6,6 @@ CHECK: Disassembly of section .text:
 CHECK-NEXT: _start:
 CHECK-NEXT:   10:  c3  retl
 CHECK-NEXT: SYMBOL TABLE:
-CHECK-NEXT: 00000000         *UND*  00000000
 CHECK-NEXT: 00000010 l    d  .text  00000000 .text
 CHECK-NEXT: 00000010         .text  00000000 _start
 CHECK-NEXT: 00000020         .text  00000000 _fdata
diff --git a/test/tools/llvm-objdump/X86/phdrs-lma.test b/test/tools/llvm-objdump/X86/phdrs-lma.test
new file mode 100644
index 0000000000000000000000000000000000000000..de1b982c0de781db981549ceb98e90024da8a1cc
--- /dev/null
+++ b/test/tools/llvm-objdump/X86/phdrs-lma.test
@@ -0,0 +1,48 @@
+# RUN: yaml2obj %s > %t
+
+## Check we print both VMA and LMA correctly when dumping section headers.
+# RUN: llvm-objdump --section-headers %t | FileCheck %s
+
+# CHECK:      Sections:
+# CHECK-NEXT: Idx Name  Size     VMA              LMA              Type
+# CHECK-NEXT:   0       00000000 0000000000000000 0000000000000000
+# CHECK-NEXT:   1 .text 00000004 0000000000001000 0000000000002000 TEXT
+# CHECK-NEXT:   2 .init 00000004 0000000000001010 0000000000001010 TEXT
+# CHECK-NEXT:   3 .data 00000004 0000000000002000 0000000000003000 DATA
+
+!ELF
+FileHeader:
+  Class:           ELFCLASS64
+  Data:            ELFDATA2LSB
+  Type:            ET_EXEC
+  Machine:         EM_X86_64
+Sections:
+  - Name:            .text
+    Type:            SHT_PROGBITS
+    Flags:           [ SHF_ALLOC, SHF_EXECINSTR ]
+    Content:         "00000000"
+    Address:         0x00001000
+  - Name:            .init
+    Type:            SHT_PROGBITS
+    Flags:           [ SHF_ALLOC, SHF_EXECINSTR ]
+    Content:         "00000000"
+    Address:         0x00001010
+  - Name:            .data
+    Type:            SHT_PROGBITS
+    Flags:           [ SHF_ALLOC ]
+    Content:         "00000000"
+    Address:         0x00002000
+ProgramHeaders:
+  - Type: PT_LOAD
+    Flags: [ PF_X, PF_R ]
+    VAddr: 0x00001000
+    PAddr: 0x00002000
+    Sections:
+      - Section: .text
+      - Section: .init
+  - Type: PT_LOAD
+    Flags: [ PF_R ]
+    VAddr: 0x00002000
+    PAddr: 0x00003000
+    Sections:
+      - Section: .data
diff --git a/test/tools/llvm-objdump/X86/phdrs-lma2.test b/test/tools/llvm-objdump/X86/phdrs-lma2.test
new file mode 100644
index 0000000000000000000000000000000000000000..2dd155f246ca4c09205b4e0bfe76f5ad814392b0
--- /dev/null
+++ b/test/tools/llvm-objdump/X86/phdrs-lma2.test
@@ -0,0 +1,60 @@
+# RUN: yaml2obj %s > %t
+
+## If there are no sections with different LMA to VMA,
+## we do not display LMA column.
+# RUN: llvm-objdump --section-headers %t | FileCheck %s
+
+# CHECK:      Sections:
+# CHECK-NEXT: Idx Name  Size     VMA              Type
+# CHECK-NEXT:   0       00000000 0000000000000000
+# CHECK-NEXT:   1 .text 00000004 0000000000001000 TEXT
+# CHECK-NEXT:   2 .init 00000004 0000000000001010 TEXT
+# CHECK-NEXT:   3 .data 00000004 0000000000002000 DATA
+
+## Check we can trigger displaying the LMA column with --show-lma.
+# RUN: llvm-objdump --section-headers --show-lma %t |\
+# RUN:   FileCheck %s --check-prefix=LMA
+
+# LMA:      Sections:
+# LMA-NEXT: Idx Name  Size     VMA              LMA              Type
+# LMA-NEXT:   0       00000000 0000000000000000 0000000000000000
+# LMA-NEXT:   1 .text 00000004 0000000000001000 0000000000001000 TEXT
+# LMA-NEXT:   2 .init 00000004 0000000000001010 0000000000001010 TEXT
+# LMA-NEXT:   3 .data 00000004 0000000000002000 0000000000002000 DATA
+
+!ELF
+FileHeader:
+  Class:           ELFCLASS64
+  Data:            ELFDATA2LSB
+  Type:            ET_EXEC
+  Machine:         EM_X86_64
+Sections:
+  - Name:            .text
+    Type:            SHT_PROGBITS
+    Flags:           [ SHF_ALLOC, SHF_EXECINSTR ]
+    Content:         "00000000"
+    Address:         0x00001000
+  - Name:            .init
+    Type:            SHT_PROGBITS
+    Flags:           [ SHF_ALLOC, SHF_EXECINSTR ]
+    Content:         "00000000"
+    Address:         0x00001010
+  - Name:            .data
+    Type:            SHT_PROGBITS
+    Flags:           [ SHF_ALLOC ]
+    Content:         "00000000"
+    Address:         0x00002000
+ProgramHeaders:
+  - Type: PT_LOAD
+    Flags: [ PF_X, PF_R ]
+    VAddr: 0x00001000
+    PAddr: 0x00001000
+    Sections:
+      - Section: .text
+      - Section: .init
+  - Type: PT_LOAD
+    Flags: [ PF_R ]
+    VAddr: 0x00002000
+    PAddr: 0x00002000
+    Sections:
+      - Section: .data
diff --git a/test/tools/llvm-objdump/X86/print-symbol-addr.s b/test/tools/llvm-objdump/X86/print-symbol-addr.s
new file mode 100644
index 0000000000000000000000000000000000000000..9c5b23ecc0b98a0dd75a214a7d0dd516723d44d0
--- /dev/null
+++ b/test/tools/llvm-objdump/X86/print-symbol-addr.s
@@ -0,0 +1,29 @@
+// RUN: llvm-mc %s -filetype=obj -triple=x86_64-pc-linux -o %t.o
+
+// Check we print the address of `foo` and `bar`.
+// RUN: llvm-objdump -d %t.o | FileCheck %s
+// CHECK:      Disassembly of section .text:
+// CHECK-NEXT: 0000000000000000 foo:
+// CHECK-NEXT:   0: {{.*}}  nop
+// CHECK-NEXT:   1: {{.*}}  nop
+// CHECK:      0000000000000002 bar:
+// CHECK-NEXT:   2: {{.*}}  nop
+
+// Check we do not print the addresses with -no-leading-addr.
+// RUN: llvm-objdump -d -no-leading-addr %t.o | FileCheck %s --check-prefix=NOADDR
+// NOADDR:      Disassembly of section .text:
+// NOADDR-NEXT: {{^}}foo:
+// NOADDR-NEXT:   {{.*}} nop
+// NOADDR-NEXT:   {{.*}} nop
+// NOADDR:      {{^}}bar:
+// NOADDR-NEXT:   {{.*}} nop
+
+.text
+.globl  foo
+.type   foo, @function
+foo:
+ nop
+ nop
+
+bar:
+ nop
diff --git a/test/tools/llvm-objdump/X86/source-interleave-x86_64.ll b/test/tools/llvm-objdump/X86/source-interleave-x86_64.ll
index f9b905443709cca1e73f08728a9dd64a8526cd72..2a4772627e2ce7fec578e22a09c14f8a8c232ac9 100644
--- a/test/tools/llvm-objdump/X86/source-interleave-x86_64.ll
+++ b/test/tools/llvm-objdump/X86/source-interleave-x86_64.ll
@@ -42,7 +42,7 @@ attributes #1 = { nounwind readnone }
 !llvm.module.flags = !{!6, !7}
 !llvm.ident = !{!8}
 
-!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 4.0.0 (http://llvm.org/git/clang d19a95e94dc57c5a72fd25d64f26134aa7d25fa0) (http://llvm.org/git/llvm.git 313924e6ff8a332063f61d3fda03812c220762f6)", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, globals: !3)
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 4.0.0", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, globals: !3)
 !1 = !DIFile(filename: "source-interleave-x86_64.c", directory: "SRC_COMPDIR")
 !2 = !{}
 !3 = !{!4}
@@ -50,7 +50,7 @@ attributes #1 = { nounwind readnone }
 !5 = !DIBasicType(name: "int", size: 32, align: 32, encoding: DW_ATE_signed)
 !6 = !{i32 2, !"Dwarf Version", i32 4}
 !7 = !{i32 2, !"Debug Info Version", i32 3}
-!8 = !{!"clang version 4.0.0 (http://llvm.org/git/clang d19a95e94dc57c5a72fd25d64f26134aa7d25fa0) (http://llvm.org/git/llvm.git 313924e6ff8a332063f61d3fda03812c220762f6)"}
+!8 = !{!"clang version 4.0.0"}
 !9 = distinct !DISubprogram(name: "foo", scope: !1, file: !1, line: 2, type: !10, isLocal: false, isDefinition: true, scopeLine: 2, isOptimized: false, unit: !0, retainedNodes: !2)
 !10 = !DISubroutineType(types: !11)
 !11 = !{!5}
diff --git a/test/tools/llvm-objdump/all-headers.test b/test/tools/llvm-objdump/all-headers.test
new file mode 100644
index 0000000000000000000000000000000000000000..5cb78c7e3ecc7cf4c64e1391c2cf65522a4cfe7d
--- /dev/null
+++ b/test/tools/llvm-objdump/all-headers.test
@@ -0,0 +1,45 @@
+# RUN: yaml2obj %s > %t
+# RUN: llvm-objdump --all-headers %t | FileCheck %s
+
+## Check we print file format, architecture and start address before
+## anything else when using --all-headers.
+
+# CHECK:       file format ELF64-x86-64
+# CHECK-EMPTY:
+# CHECK-NEXT:  architecture: x86_64
+# CHECK-NEXT:  start address: 0x0000000000000000
+# CHECK-EMPTY:
+# CHECK-NEXT:  Sections:
+
+## Check how we dump the archives.
+## Check we dump the appropriate headers for each file in the archive.
+
+# RUN: cp %t %t2
+# RUN: rm -f %t.a
+# RUN: llvm-ar rcs %t.a %t %t2
+# RUN: llvm-objdump --all-headers %t.a | FileCheck %s --check-prefix=ARCHIVE
+
+# ARCHIVE: {{.*}}.a({{.*}}):   file format ELF64-x86-64
+# ARCHIVE: architecture: x86_64
+# ARCHIVE: start address: 0x0000000000000000
+# ARCHIVE: rw-r--r--
+# ARCHIVE: Sections:
+# ARCHIVE: SYMBOL TABLE:
+# ARCHIVE: Program Header:
+# ARCHIVE: Dynamic Section:
+# ARCHIVE: {{.*}}.a({{.*}}2):   file format ELF64-x86-64
+# ARCHIVE: architecture: x86_64
+# ARCHIVE: start address: 0x0000000000000000
+# ARCHIVE: rw-r--r--
+# ARCHIVE: Sections:
+# ARCHIVE: SYMBOL TABLE:
+# ARCHIVE: Program Header:
+# ARCHIVE: Dynamic Section:
+
+!ELF
+FileHeader:
+  Class:           ELFCLASS64
+  Data:            ELFDATA2LSB
+  Type:            ET_EXEC
+  Machine:         EM_X86_64
+Sections:
diff --git a/test/tools/llvm-objdump/private-headers-no-dynamic-segment.test b/test/tools/llvm-objdump/private-headers-no-dynamic-segment.test
index 0aec1fedfeceba1ee2368728a4a42418f6313e3c..c3f592532b7e87e4fce21e09aab785d465d6ff31 100644
--- a/test/tools/llvm-objdump/private-headers-no-dynamic-segment.test
+++ b/test/tools/llvm-objdump/private-headers-no-dynamic-segment.test
@@ -10,8 +10,65 @@ FileHeader:
 Sections:
   - Name:            .dynamic
     Type:            SHT_DYNAMIC
-    Flags:           [ SHF_ALLOC, SHF_WRITE ]
-    Content: 0c00000000000000a0060000000000000d0000000000000024090000000000001900000000000000a80d2000000000001b0000000000000010000000000000001a00000000000000b80d2000000000001c000000000000000800000000000000f5feff6f0000000098020000000000000500000000000000c8030000000000000600000000000000c0020000000000000a000000000000002f010000000000000b0000000000000018000000000000001500000000000000000000000000000003000000000000000010200000000000020000000000000048000000000000001400000000000000070000000000000017000000000000005806000000000000070000000000000050050000000000000800000000000000080100000000000009000000000000001800000000000000fbffff6f000000000000000800000000feffff6f000000001005000000000000ffffff6f000000000200000000000000f0ffff6f00000000f804000000000000f9ffff6f0000000004000000000000002300000000000000140000000000000024000000000000002143658700000000250000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000
+    Flags:           [ SHF_WRITE, SHF_ALLOC ]
+    EntSize:         0x0000000000000010
+    Entries:
+      - Tag:             DT_INIT
+        Value:           0x00000000000006A0
+      - Tag:             DT_FINI
+        Value:           0x0000000000000924
+      - Tag:             DT_INIT_ARRAY
+        Value:           0x0000000000200DA8
+      - Tag:             DT_INIT_ARRAYSZ
+        Value:           0x0000000000000010
+      - Tag:             DT_FINI_ARRAY
+        Value:           0x0000000000200DB8
+      - Tag:             DT_FINI_ARRAYSZ
+        Value:           0x0000000000000008
+      - Tag:             DT_GNU_HASH
+        Value:           0x0000000000000298
+      - Tag:             DT_STRTAB
+        Value:           0x00000000000003C8
+      - Tag:             DT_SYMTAB
+        Value:           0x00000000000002C0
+      - Tag:             DT_STRSZ
+        Value:           0x000000000000012F
+      - Tag:             DT_SYMENT
+        Value:           0x0000000000000018
+      - Tag:             DT_DEBUG
+        Value:           0x0000000000000000
+      - Tag:             DT_PLTGOT
+        Value:           0x0000000000201000
+      - Tag:             DT_PLTRELSZ
+        Value:           0x0000000000000048
+      - Tag:             DT_PLTREL
+        Value:           0x0000000000000007
+      - Tag:             DT_JMPREL
+        Value:           0x0000000000000658
+      - Tag:             DT_RELA
+        Value:           0x0000000000000550
+      - Tag:             DT_RELASZ
+        Value:           0x0000000000000108
+      - Tag:             DT_RELAENT
+        Value:           0x0000000000000018
+      - Tag:             DT_FLAGS_1
+        Value:           0x0000000008000000
+      - Tag:             DT_VERNEED
+        Value:           0x0000000000000510
+      - Tag:             DT_VERNEEDNUM
+        Value:           0x0000000000000002
+      - Tag:             DT_VERSYM
+        Value:           0x00000000000004F8
+      - Tag:             DT_RELACOUNT
+        Value:           0x0000000000000004
+      - Tag:             DT_RELRSZ
+        Value:           0x0000000000000014
+      - Tag:             DT_RELR
+        Value:           0x0000000087654321
+      - Tag:             DT_RELRENT
+        Value:           0x0000000000000010
+      - Tag:             DT_NULL
+        Value:           0x0000000000000000
 
 # CHECK:  INIT                 0x00000000000006a0
 # CHECK:  FINI                 0x0000000000000924
diff --git a/test/tools/llvm-objdump/symbol-table-elf.test b/test/tools/llvm-objdump/symbol-table-elf.test
index abe339d7d536bd3fd7c3951003af297c26aa93fc..649e54c99bf85a3f40172fcc8a8d67c3cae10274 100644
--- a/test/tools/llvm-objdump/symbol-table-elf.test
+++ b/test/tools/llvm-objdump/symbol-table-elf.test
@@ -5,7 +5,6 @@
 # RUN: FileCheck %s --input-file=%t1
 
 # CHECK:      SYMBOL TABLE:
-# CHECK-NEXT: 0000000000000000         *UND*     00000000
 # CHECK-NEXT: 0000000000001004 l     F .text     00000000 lfoo
 # CHECK-NEXT: 0000000000001008 l     O .text     00000000 lbar
 # CHECK-NEXT: 0000000000001004 g     F .text     00000000 foo
diff --git a/test/tools/llvm-objdump/wasm.txt b/test/tools/llvm-objdump/wasm.txt
index 93517fed6d418f7b04608fac70a20ec035bfdd40..4a0cfd06582aee91eebad03f825704d258950b61 100644
--- a/test/tools/llvm-objdump/wasm.txt
+++ b/test/tools/llvm-objdump/wasm.txt
@@ -1,7 +1,7 @@
 # RUN: llvm-objdump -h %p/Inputs/trivial.obj.wasm | FileCheck %s
 
 # CHECK:      Sections:
-# CHECK-NEXT: Idx Name          Size      Address          Type
+# CHECK-NEXT: Idx Name          Size    VMA              Type
 # CHECK-NEXT:  0 TYPE          00000011 0000000000000000
 # CHECK-NEXT:  1 IMPORT        0000005d 0000000000000000
 # CHECK-NEXT:  2 FUNCTION      00000003 0000000000000000
diff --git a/test/tools/llvm-pdbdump/Inputs/FPOTest.pdb b/test/tools/llvm-pdbdump/Inputs/FPOTest.pdb
new file mode 100644
index 0000000000000000000000000000000000000000..7096328b39e6fd8dc889f045914a0a5bf27665bb
Binary files /dev/null and b/test/tools/llvm-pdbdump/Inputs/FPOTest.pdb differ
diff --git a/test/tools/llvm-pdbdump/fpo-data.test b/test/tools/llvm-pdbdump/fpo-data.test
new file mode 100644
index 0000000000000000000000000000000000000000..ac909cf5e7c28cad2d0a5c14b45789ca61917270
--- /dev/null
+++ b/test/tools/llvm-pdbdump/fpo-data.test
@@ -0,0 +1,14 @@
+; RUN: llvm-pdbutil dump -fpo %p/Inputs/FPOTest.pdb \
+; RUN:     | FileCheck %s
+
+CHECK:                        Old FPO Data                        
+CHECK-NEXT: ============================================================
+CHECK-NEXT:   RVA    | Code | Locals | Params | Prolog | Saved Regs | Use BP | Has SEH | Frame Type
+CHECK-NEXT: 0000004E |   19 |      0 |      0 |      0 |          0 |  false |   false |       FPO
+
+CHECK:                        New FPO Data                        
+CHECK-NEXT: ============================================================
+CHECK-NEXT:   RVA    | Code | Locals | Params | Stack | Prolog | Saved Regs | Has SEH | Has C++EH | Start | Program
+CHECK-NEXT: 00001010 |   18 |      0 |      0 |     0 |      4 |          0 |   false |     false |  true | $T0 .raSearch = $eip $T0 ^ = $esp $T0 4 + = 
+CHECK-NEXT: 00001011 |   17 |      0 |      0 |     0 |      3 |          4 |   false |     false | false | $T0 .raSearch = $eip $T0 ^ = $esp $T0 4 + = $ebp $T0 4 - ^ = 
+CHECK-NEXT: 00001013 |   15 |      0 |      0 |     0 |      1 |          4 |   false |     false | false | $T0 $ebp 4 + = $eip $T0 ^ = $esp $T0 4 + = $ebp $T0 4 - ^ = 
\ No newline at end of file
diff --git a/test/tools/llvm-profdata/Inputs/cutoff.proftext b/test/tools/llvm-profdata/Inputs/cutoff.proftext
new file mode 100644
index 0000000000000000000000000000000000000000..1ce4843f54fab46d0dbbbf2ea05d1b5df75f528c
--- /dev/null
+++ b/test/tools/llvm-profdata/Inputs/cutoff.proftext
@@ -0,0 +1,21 @@
+# IR level Instrumentation Flag
+:ir
+bar
+10
+2
+0
+0
+
+main
+16650
+4
+1
+1000
+1000000
+499500
+
+foo
+10
+2
+999
+1
diff --git a/test/tools/llvm-profdata/cutoff.test b/test/tools/llvm-profdata/cutoff.test
new file mode 100644
index 0000000000000000000000000000000000000000..f04ea16f324e4480fa2f6607b61038f367c5a7d9
--- /dev/null
+++ b/test/tools/llvm-profdata/cutoff.test
@@ -0,0 +1,23 @@
+Basic tests for cutoff options in show command.
+
+RUN: llvm-profdata show -value-cutoff=1 %p/Inputs/cutoff.proftext | FileCheck %s -check-prefix=CUTOFF1 -check-prefix=CHECK
+RUN: llvm-profdata show -value-cutoff=1000 %p/Inputs/cutoff.proftext | FileCheck %s -check-prefix=CUTOFF1000 -check-prefix=CHECK
+RUN: llvm-profdata show -all-functions -value-cutoff=1 %p/Inputs/cutoff.proftext | FileCheck %s -check-prefix=CUTOFF1FUNC -check-prefix=CUTOFF1 -check-prefix=CHECK
+RUN: llvm-profdata show -all-functions -value-cutoff=1000 %p/Inputs/cutoff.proftext | FileCheck %s -check-prefix=CUTOFF1000FUNC -check-prefix=CUTOFF1000 -check-prefix=CHECK
+RUN: llvm-profdata show -value-cutoff=1 -list-below-cutoff %p/Inputs/cutoff.proftext | FileCheck %s -check-prefix=BELOW1 -check-prefix=CUTOFF1 -check-prefix=CHECK
+RUN: llvm-profdata show -value-cutoff=1000 -list-below-cutoff %p/Inputs/cutoff.proftext | FileCheck %s -check-prefix=BELOW1000 -check-prefix=CUTOFF1000 -check-prefix=CHECK
+CUTOFF1FUNC-NOT: bar
+CUTOFF1FUNC: Functions shown: 2
+CUTOFF1000FUNC-NOT: bar
+CUTOFF1000FUNC-NOT: foo
+CUTOFF1000FUNC: Functions shown: 1
+BELOW1: The list of functions with the maximum counter less than 1:
+BELOW1:  bar: (Max = 0 Sum = 0)
+BELOW1000:The list of functions with the maximum counter less than 1000:
+BELOW1000:  bar: (Max = 0 Sum = 0)
+BELOW1000:  foo: (Max = 999 Sum = 1000)
+CHECK: Total functions: 3
+CUTOFF1: Number of functions with maximum count (< 1): 1
+CUTOFF1: Number of functions with maximum count (>= 1): 2
+CUTOFF1000: Number of functions with maximum count (< 1000): 2
+CUTOFF1000: Number of functions with maximum count (>= 1000): 1
diff --git a/test/tools/llvm-profdata/value-prof.proftext b/test/tools/llvm-profdata/value-prof.proftext
index 31a7698895ddbc69d60f008f4d6b2b5a46182846..a854a51afa6b50a9a99dc1d61d772271751ee995 100644
--- a/test/tools/llvm-profdata/value-prof.proftext
+++ b/test/tools/llvm-profdata/value-prof.proftext
@@ -47,9 +47,9 @@ foo2:20000
 
 #ICTXT: Indirect Call Site Count: 3
 #ICTXT-NEXT:    Indirect Target Results:
-#ICTXT-NEXT:	[ 1, foo, 100 ]
-#ICTXT-NEXT:	[ 1, foo2, 1000 ]
-#ICTXT-NEXT:	[ 2, foo2, 20000 ]
+#ICTXT-NEXT:	[ 1, foo, 100 ] (9.09%)
+#ICTXT-NEXT:	[ 1, foo2, 1000 ] (90.91%)
+#ICTXT-NEXT:	[ 2, foo2, 20000 ] (100.00%)
 
 #IC: Indirect Call Site Count: 3
 #IC-NEXT:    Indirect Target Results:
diff --git a/test/tools/llvm-rc/absolute.test b/test/tools/llvm-rc/absolute.test
index 5e8a6c8c33663f4043c972017c8b733623c4c274..95aff3e424407f33bb013d888faba29387ed87f9 100644
--- a/test/tools/llvm-rc/absolute.test
+++ b/test/tools/llvm-rc/absolute.test
@@ -1,3 +1,3 @@
 ; RUN: touch %t.manifest
 ; RUN: echo "1 24 \"%t.manifest\"" > %t.rc
-; RUN: llvm-rc %t.rc
+; RUN: llvm-rc -- %t.rc
diff --git a/test/tools/llvm-rc/codepage.test b/test/tools/llvm-rc/codepage.test
index ce17e0a6b2b51216f62212e182793ef7b303c5bc..05e3181b58289fdca90ba8240363bc14963d2be3 100644
--- a/test/tools/llvm-rc/codepage.test
+++ b/test/tools/llvm-rc/codepage.test
@@ -1,4 +1,4 @@
-; RUN: llvm-rc /C 65001 /FO %t.utf8.res %p/Inputs/utf8.rc
+; RUN: llvm-rc /C 65001 /FO %t.utf8.res -- %p/Inputs/utf8.rc
 ; RUN: llvm-readobj %t.utf8.res | FileCheck %s --check-prefix=UTF8
 
 ; UTF8:      Resource type (int): 6
diff --git a/test/tools/llvm-rc/cpp-output.test b/test/tools/llvm-rc/cpp-output.test
index 456a59da7a70f733943f6d2c4314b17c2d29f86c..984e9fd7da4b99e0ef244ded276f21664a098422 100644
--- a/test/tools/llvm-rc/cpp-output.test
+++ b/test/tools/llvm-rc/cpp-output.test
@@ -1,4 +1,4 @@
-; RUN: llvm-rc /FO %t %p/Inputs/cpp-output.rc
+; RUN: llvm-rc /FO %t -- %p/Inputs/cpp-output.rc
 ; RUN: llvm-readobj %t | FileCheck %s
 
 ; CHECK:      Resource type (int): 6
diff --git a/test/tools/llvm-rc/flags.test b/test/tools/llvm-rc/flags.test
index 452e90a406bbab5c78d1029a3846c462bfc7c0c8..5b71481af2df9987a2b3d6a3105b319dbfb80a93 100644
--- a/test/tools/llvm-rc/flags.test
+++ b/test/tools/llvm-rc/flags.test
@@ -1,4 +1,4 @@
-; RUN: llvm-rc /dry-run /FO %t %p/Inputs/empty.rc 2>&1 | FileCheck %s --allow-empty --check-prefix=FO
-; RUN: llvm-rc /dry-run /FO%t %p/Inputs/empty.rc 2>&1 | FileCheck %s --allow-empty --check-prefix=FO
+; RUN: llvm-rc /dry-run /FO %t -- %p/Inputs/empty.rc 2>&1 | FileCheck %s --allow-empty --check-prefix=FO
+; RUN: llvm-rc /dry-run /FO%t -- %p/Inputs/empty.rc 2>&1 | FileCheck %s --allow-empty --check-prefix=FO
 
 ; FO-NOT: Exactly one input file should be provided.
diff --git a/test/tools/llvm-rc/include-paths.test b/test/tools/llvm-rc/include-paths.test
index e6c52b12d6d6d86c62fb056055618169d67619f5..10a77e6de167d05de2284d8b842c8e8f43794493 100644
--- a/test/tools/llvm-rc/include-paths.test
+++ b/test/tools/llvm-rc/include-paths.test
@@ -1,21 +1,21 @@
 ; Should find the bitmap if it is in the same folder as the rc file.
 ; RUN: rm -f %t.include.res
-; RUN: llvm-rc /FO %t.include.res %p/Inputs/include.rc
+; RUN: llvm-rc /FO %t.include.res -- %p/Inputs/include.rc
 ; RUN: llvm-readobj %t.include.res | FileCheck --check-prefix=FOUND %s
 
 ; Try including files without quotes.
 ; RUN: rm -f %t.noquotes.res
-; RUN: llvm-rc /FO %t.noquotes.res %p/Inputs/include-noquotes.rc
+; RUN: llvm-rc /FO %t.noquotes.res -- %p/Inputs/include-noquotes.rc
 ; RUN: llvm-readobj %t.noquotes.res | FileCheck --check-prefix=FOUND %s
 
 ; Should find the bitmap if the folder is explicitly specified.
 ; RUN: rm -f %t.nested-include.res
-; RUN: llvm-rc /FO %t.nested-include.res /I %p/Inputs/nested %p/Inputs/deep-include.rc
+; RUN: llvm-rc /FO %t.nested-include.res /I %p/Inputs/nested -- %p/Inputs/deep-include.rc
 ; RUN: llvm-readobj %t.nested-include.res | FileCheck --check-prefix=FOUND %s
 
 ; Otherwise, it should not find the bitmap.
 ; RUN: rm -f %t.nested-include.res
-; RUN: not llvm-rc /FO %t.nested-include.res %p/Inputs/deep-include.rc 2>&1 \
+; RUN: not llvm-rc /FO %t.nested-include.res -- %p/Inputs/deep-include.rc 2>&1 \
 ; RUN:   | FileCheck --check-prefix=MISSING %s
 
 ; Should find the bitmap if the process's current working directory
@@ -24,7 +24,7 @@
 ; failure of other tests if run first.
 ; RUN: rm -f %t.nested-include.res
 ; RUN: cd %p/Inputs/nested
-; RUN: llvm-rc /FO %t.nested-include.res %p/Inputs/include.rc
+; RUN: llvm-rc /FO %t.nested-include.res -- %p/Inputs/include.rc
 ; RUN: llvm-readobj %t.nested-include.res | FileCheck --check-prefix=FOUND %s
 
 FOUND:      Resource type (int): 2
diff --git a/test/tools/llvm-rc/memoryflags-stringtable.test b/test/tools/llvm-rc/memoryflags-stringtable.test
index f168beda7b976f152ad9299eb07919dc8e9b16ac..e8a2d40e8fe8d675148657f492d4e7e371e6975d 100644
--- a/test/tools/llvm-rc/memoryflags-stringtable.test
+++ b/test/tools/llvm-rc/memoryflags-stringtable.test
@@ -1,4 +1,4 @@
-; RUN: llvm-rc /FO %t %p/Inputs/memoryflags-stringtable.rc
+; RUN: llvm-rc /FO %t -- %p/Inputs/memoryflags-stringtable.rc
 ; RUN: llvm-readobj %t | FileCheck %s
 
 ; CHECK:      Resource type (int): 6
diff --git a/test/tools/llvm-rc/memoryflags.test b/test/tools/llvm-rc/memoryflags.test
index c36fb9bda8e15a4c703d4bcb772b155f3cb9bdba..14e1193d1f6c4f610a13b9ddfbb1d7b0d55a511b 100644
--- a/test/tools/llvm-rc/memoryflags.test
+++ b/test/tools/llvm-rc/memoryflags.test
@@ -1,4 +1,4 @@
-; RUN: llvm-rc /FO %t %p/Inputs/memoryflags.rc
+; RUN: llvm-rc /FO %t -- %p/Inputs/memoryflags.rc
 ; RUN: llvm-readobj %t | FileCheck %s
 
 ; CHECK:      Resource type (int): 1
diff --git a/test/tools/llvm-rc/not-expr.test b/test/tools/llvm-rc/not-expr.test
index c60223425c7157e7d1199e2702dc914c8f1683d7..43725ed4f4362e94ad1c04e1cd852a38215b2a25 100644
--- a/test/tools/llvm-rc/not-expr.test
+++ b/test/tools/llvm-rc/not-expr.test
@@ -1,4 +1,4 @@
-; RUN: llvm-rc /FO %t %p/Inputs/not-expr.rc
+; RUN: llvm-rc /FO %t -- %p/Inputs/not-expr.rc
 ; RUN: llvm-readobj %t | FileCheck %s --check-prefix=NOTEXPR
 
 ; NOTEXPR: Resource type (int): 5
diff --git a/test/tools/llvm-rc/parser-expr.test b/test/tools/llvm-rc/parser-expr.test
index 9558f9305f32dded15b5278b87f970767e02c771..1fb6aa62fde1cfcdbadfe75eeb36283d879da3a0 100644
--- a/test/tools/llvm-rc/parser-expr.test
+++ b/test/tools/llvm-rc/parser-expr.test
@@ -1,4 +1,4 @@
-; RUN: llvm-rc /dry-run /V %p/Inputs/parser-expr.rc | FileCheck %s
+; RUN: llvm-rc /dry-run /V -- %p/Inputs/parser-expr.rc | FileCheck %s
 
 ; CHECK:  Language: 5, Sublanguage: 1
 ; CHECK-NEXT:  Language: 3, Sublanguage: 2
diff --git a/test/tools/llvm-rc/parser.test b/test/tools/llvm-rc/parser.test
index 66ed7386e821e1d2cd016a35014206b9d68d22a6..4f10d7886a7fbcb39918c1696dceb14d0b7fe76a 100644
--- a/test/tools/llvm-rc/parser.test
+++ b/test/tools/llvm-rc/parser.test
@@ -1,4 +1,4 @@
-; RUN: llvm-rc /dry-run /V %p/Inputs/parser-correct-everything.rc | FileCheck %s --check-prefix PGOOD
+; RUN: llvm-rc /dry-run /V -- %p/Inputs/parser-correct-everything.rc | FileCheck %s --check-prefix PGOOD
 
 ; PGOOD:  Icon (meh): "hello.bmp"
 ; PGOOD-NEXT:  Icon (Icon): "Icon"
diff --git a/test/tools/llvm-rc/tag-accelerators.test b/test/tools/llvm-rc/tag-accelerators.test
index 093a0bddbb64854a9ca8bed37e06e15412aa6f11..910bf218633b9edf427ffd4823f29e1edb4518bb 100644
--- a/test/tools/llvm-rc/tag-accelerators.test
+++ b/test/tools/llvm-rc/tag-accelerators.test
@@ -1,4 +1,4 @@
-; RUN: llvm-rc /FO %t %p/Inputs/tag-accelerators.rc
+; RUN: llvm-rc /FO %t -- %p/Inputs/tag-accelerators.rc
 ; RUN: llvm-readobj %t | FileCheck %s --check-prefix=ACCELERATORS
 
 ; ACCELERATORS: Resource type (int): 9
diff --git a/test/tools/llvm-rc/tag-dialog.test b/test/tools/llvm-rc/tag-dialog.test
index 85a8c20edc7d647dd497e2f7eeb9b5fc0bbd4b38..d44326e80a16418fc662054803771f33d46f9f61 100644
--- a/test/tools/llvm-rc/tag-dialog.test
+++ b/test/tools/llvm-rc/tag-dialog.test
@@ -1,4 +1,4 @@
-; RUN: llvm-rc /FO %t %p/Inputs/tag-dialog.rc
+; RUN: llvm-rc /FO %t -- %p/Inputs/tag-dialog.rc
 ; RUN: llvm-readobj %t | FileCheck %s --check-prefix=DIALOG
 
 ; DIALOG: Resource type (int): 5
diff --git a/test/tools/llvm-rc/tag-escape.test b/test/tools/llvm-rc/tag-escape.test
index 7c58e9953e4e24675f34e7aeed6f852994b36e6f..51c3d928e79cf4d499efbe2932fa0cf191d2ffdd 100644
--- a/test/tools/llvm-rc/tag-escape.test
+++ b/test/tools/llvm-rc/tag-escape.test
@@ -1,4 +1,4 @@
-; RUN: llvm-rc /FO %t %p/Inputs/tag-escape.rc
+; RUN: llvm-rc /FO %t -- %p/Inputs/tag-escape.rc
 ; RUN: llvm-readobj %t | FileCheck %s
 
 ; CHECK:      Resource type (int): 4
diff --git a/test/tools/llvm-rc/tag-html.test b/test/tools/llvm-rc/tag-html.test
index 571e1bcb46c38faf236e8d881435027c654ca68c..efeb04b0f2fe48f03f9b37e5319a2c31fe97bbed 100644
--- a/test/tools/llvm-rc/tag-html.test
+++ b/test/tools/llvm-rc/tag-html.test
@@ -1,6 +1,6 @@
 ; RUN: rm -rf %t && mkdir %t && cd %t
 ; RUN: cp %p/Inputs/webpage*.html .
-; RUN: llvm-rc /FO %t/tag-html.res %p/Inputs/tag-html.rc
+; RUN: llvm-rc /FO %t/tag-html.res -- %p/Inputs/tag-html.rc
 ; RUN: llvm-readobj %t/tag-html.res | FileCheck %s --check-prefix HTML
 
 ; HTML: Resource type (int): 23
diff --git a/test/tools/llvm-rc/tag-icon-cursor.test b/test/tools/llvm-rc/tag-icon-cursor.test
index 44ef6841165d98821d614b42404c1835cf86016e..b4301f5d593748c93cd5b7d8794befc8bd7f24c9 100644
--- a/test/tools/llvm-rc/tag-icon-cursor.test
+++ b/test/tools/llvm-rc/tag-icon-cursor.test
@@ -1,7 +1,7 @@
 ; RUN: rm -rf %t
 ; RUN: mkdir %t
 
-; RUN: llvm-rc /FO %t/tag-icon-cursor.res %p/Inputs/tag-icon-cursor.rc
+; RUN: llvm-rc /FO %t/tag-icon-cursor.res -- %p/Inputs/tag-icon-cursor.rc
 ; RUN: llvm-readobj %t/tag-icon-cursor.res | FileCheck %s
 
 ; CHECK: Resource type (int): 1
diff --git a/test/tools/llvm-rc/tag-menu.test b/test/tools/llvm-rc/tag-menu.test
index 25dc395a1d97a4c4fa6d51aaa3886e7e345f3a32..91e24457956cba3881c224abab7c215f251f59f6 100644
--- a/test/tools/llvm-rc/tag-menu.test
+++ b/test/tools/llvm-rc/tag-menu.test
@@ -1,9 +1,9 @@
-; RUN: llvm-rc /FO %t %p/Inputs/tag-menu.rc
+; RUN: llvm-rc /FO %t -- %p/Inputs/tag-menu.rc
 ; RUN: llvm-readobj %t | FileCheck %s --check-prefix=MENU
 
 ; Test running llvm-rc without an explicit output file.
 ; RUN: cp %p/Inputs/tag-menu.rc %t.implicit.rc
-; RUN: llvm-rc %t.implicit.rc
+; RUN: llvm-rc -- %t.implicit.rc
 ; RUN: llvm-readobj %t.implicit.res | FileCheck --check-prefix=MENU %s
 
 ; MENU: Resource type (int): 4
diff --git a/test/tools/llvm-rc/tag-stringtable.test b/test/tools/llvm-rc/tag-stringtable.test
index 43b5f5c965804b159f83c821a3751376f57e0ef9..8dc4e57c6fd6edce69aceb10d7bc78edba3189b4 100644
--- a/test/tools/llvm-rc/tag-stringtable.test
+++ b/test/tools/llvm-rc/tag-stringtable.test
@@ -1,4 +1,4 @@
-; RUN: llvm-rc /FO %t %p/Inputs/tag-stringtable-basic.rc
+; RUN: llvm-rc /FO %t -- %p/Inputs/tag-stringtable-basic.rc
 ; RUN: llvm-readobj %t | FileCheck %s
 
 ; CHECK:      Resource type (int): 6
diff --git a/test/tools/llvm-rc/tag-user.test b/test/tools/llvm-rc/tag-user.test
index c0c1cfd1823c52113f038d369019c1a5ed11d30d..e93899cdd8aad7a84f27d0a71a7cd7b9c735920e 100644
--- a/test/tools/llvm-rc/tag-user.test
+++ b/test/tools/llvm-rc/tag-user.test
@@ -2,7 +2,7 @@
 ; RUN: mkdir %t
 ; RUN: cd %t
 ; RUN: cp %p/Inputs/bitmap.bmp .
-; RUN: llvm-rc /FO %t/tag-user.res %p/Inputs/tag-user.rc
+; RUN: llvm-rc /FO %t/tag-user.res -- %p/Inputs/tag-user.rc
 ; RUN: llvm-readobj %t/tag-user.res | FileCheck %s
 
 ; CHECK:      Resource type (int): 500
diff --git a/test/tools/llvm-rc/tag-versioninfo.test b/test/tools/llvm-rc/tag-versioninfo.test
index 4c30346f6b28e83154809f6ccfb2fc2706100f10..0118f3c109d1cf303c2422ee3fb178184674d511 100644
--- a/test/tools/llvm-rc/tag-versioninfo.test
+++ b/test/tools/llvm-rc/tag-versioninfo.test
@@ -1,4 +1,4 @@
-; RUN: llvm-rc /FO %t %p/Inputs/tag-versioninfo.rc
+; RUN: llvm-rc /FO %t -- %p/Inputs/tag-versioninfo.rc
 ; RUN: llvm-readobj %t | FileCheck %s
 
 ; CHECK:      Resource type (int): 16
diff --git a/test/tools/llvm-rc/tokenizer.test b/test/tools/llvm-rc/tokenizer.test
index 5103b190c310fa07fa3384c176ed81584c547171..59164677bac8ff03e74fb72e9d80f6954820acf5 100644
--- a/test/tools/llvm-rc/tokenizer.test
+++ b/test/tools/llvm-rc/tokenizer.test
@@ -1,4 +1,4 @@
-; RUN: not llvm-rc /V /FO %t.res %p/Inputs/tokens.rc | FileCheck %s
+; RUN: not llvm-rc /V /FO %t.res -- %p/Inputs/tokens.rc | FileCheck %s
 ; llvm-rc fails now on this sample because it is an invalid resource file
 ; script. We silence the error message and just analyze the output.
 
diff --git a/test/tools/llvm-rc/versioninfo-padding.test b/test/tools/llvm-rc/versioninfo-padding.test
index 7cfb537b3ff55c16eabebd4a29c4ffab4f836afc..fab18a6dd92ea58f9026eb66995c03e593a7506c 100644
--- a/test/tools/llvm-rc/versioninfo-padding.test
+++ b/test/tools/llvm-rc/versioninfo-padding.test
@@ -1,4 +1,4 @@
-; RUN: llvm-rc /FO %t %p/Inputs/versioninfo-padding.rc
+; RUN: llvm-rc /FO %t -- %p/Inputs/versioninfo-padding.rc
 ; RUN: llvm-readobj %t | FileCheck %s
 
 ; CHECK:      Resource type (int): 16
diff --git a/test/tools/llvm-readobj/Inputs/relocs.py b/test/tools/llvm-readobj/Inputs/relocs.py
index 3d0cae567e55684332dcd49e2f15c63321c360cd..8e621cef5cd98ba37cb51a096f53fe915d3d22a0 100644
--- a/test/tools/llvm-readobj/Inputs/relocs.py
+++ b/test/tools/llvm-readobj/Inputs/relocs.py
@@ -1,5 +1,7 @@
 #!/usr/bin/env python
 
+from __future__ import print_function
+
 # Generates ELF, COFF and MachO object files for different architectures
 # containing all relocations:
 #
@@ -14,7 +16,6 @@
 
 import operator
 import shutil
-import StringIO
 import struct
 import subprocess
 import sys
@@ -43,17 +44,17 @@ class EnumType(type):
 
   # Not supported (Enums are immutable).
   def __setattr__(self, name, value):
-    raise NotSupportedException, self.__setattr__
+    raise NotSupportedException(self.__setattr__)
 
   # Not supported (Enums are immutable).
   def __delattr__(self, name):
-    raise NotSupportedException, self.__delattr__
+    raise NotSupportedException(self.__delattr__)
 
   # Gets the enum symbol for the specified value.
   def __getitem__(self, value):
     symbol = self._map.get(value)
     if symbol is None:
-      raise KeyError, value
+      raise KeyError(value)
     return symbol
 
   # Gets the enum symbol for the specified value or none.
@@ -63,18 +64,18 @@ class EnumType(type):
 
   # Not supported (Enums are immutable).
   def __setitem__(self, value, symbol):
-    raise NotSupportedException, self.__setitem__
+    raise NotSupportedException(self.__setitem__)
 
   # Not supported (Enums are immutable).
   def __delitem__(self, value):
-    raise NotSupportedException, self.__delitem__
+    raise NotSupportedException(self.__delitem__)
 
   def entries(self):
     # sort by (value, name)
     def makeKey(item):
       return (item[1], item[0])
     e = []
-    for pair in sorted(self._nameMap.iteritems(), key=makeKey):
+    for pair in sorted(self._nameMap.items(), key=makeKey):
       e.append(pair)
     return e
 
@@ -100,7 +101,7 @@ class BinaryReader:
   def read(self, N):
     data = self.file.read(N)
     if len(data) != N:
-      raise ValueError, "Out of data!"
+      raise ValueError("Out of data!")
     return data
 
   def int8(self):
@@ -266,7 +267,7 @@ def patchElf(path, relocs):
   elif fileclass == 2:
     f.is64Bit = True
   else:
-    raise ValueError, "Unknown file class %x" % fileclass
+    raise ValueError("Unknown file class %x" % fileclass)
 
   byteordering = f.uint8()
   if byteordering == 1:
@@ -274,7 +275,7 @@ def patchElf(path, relocs):
   elif byteordering == 2:
       f.isLSB = False
   else:
-      raise ValueError, "Unknown byte ordering %x" % byteordering
+      raise ValueError("Unknown byte ordering %x" % byteordering)
 
   f.seek(18)
   e_machine = f.uint16()
@@ -375,7 +376,7 @@ def patchMacho(filename, relocs):
   elif magic == '\xCF\xFA\xED\xFE':
     f.isLSB, f.is64Bit = True, True
   else:
-    raise ValueError,"Not a Mach-O object file: %r (bad magic)" % path
+    raise ValueError("Not a Mach-O object file: %r (bad magic)" % path)
 
   cputype = f.uint32()
   cpusubtype = f.uint32()
@@ -392,8 +393,8 @@ def patchMacho(filename, relocs):
     patchMachoLoadCommand(f, relocs)
 
   if f.tell() - start != loadCommandsSize:
-    raise ValueError,"%s: warning: invalid load commands size: %r" % (
-      sys.argv[0], loadCommandsSize)
+    raise ValueError("%s: warning: invalid load commands size: %r" % (
+      sys.argv[0], loadCommandsSize))
 
 def patchMachoLoadCommand(f, relocs):
   start = f.tell()
@@ -408,8 +409,8 @@ def patchMachoLoadCommand(f, relocs):
     f.read(cmdSize - 8)
 
   if f.tell() - start != cmdSize:
-    raise ValueError,"%s: warning: invalid load command size: %r" % (
-      sys.argv[0], cmdSize)
+    raise ValueError("%s: warning: invalid load command size: %r" % (
+      sys.argv[0], cmdSize))
 
 def patchMachoSegmentLoadCommand(f, relocs):
   segment_name = f.read(16)
diff --git a/test/tools/llvm-readobj/Inputs/trivial.obj.wasm b/test/tools/llvm-readobj/Inputs/trivial.obj.wasm
index 2f99d3446123f06927724b5d2c733d32cdef044d..10ebbeed6acbfc35ce460b83c61218b7e64ec1fa 100644
Binary files a/test/tools/llvm-readobj/Inputs/trivial.obj.wasm and b/test/tools/llvm-readobj/Inputs/trivial.obj.wasm differ
diff --git a/test/tools/llvm-readobj/codeview-merging-ghash.test b/test/tools/llvm-readobj/codeview-merging-ghash.test
new file mode 100644
index 0000000000000000000000000000000000000000..04b30ffa7cd79db68d614eb4f811eafcbf790076
--- /dev/null
+++ b/test/tools/llvm-readobj/codeview-merging-ghash.test
@@ -0,0 +1,132 @@
+# RUN: yaml2obj %s -o=%t.obj
+# RUN: llvm-readobj -codeview-merged-types %t.obj | FileCheck %s --check-prefix=MERGED
+# RUN: llvm-readobj -codeview-merged-types -codeview-ghash %t.obj | FileCheck %s --check-prefix=MERGED
+
+# MERGED: Format: COFF-x86-64
+# MERGED-NEXT: Arch: x86_64
+# MERGED-NEXT: AddressSize: 64bit
+# MERGED-NEXT: MergedTypeStream [
+# MERGED-NEXT:   ArgList (0x1000) {
+# MERGED-NEXT:     TypeLeafKind: LF_ARGLIST (0x1201)
+# MERGED-NEXT:     NumArgs: 0
+# MERGED-NEXT:     Arguments [
+# MERGED-NEXT:     ]
+# MERGED-NEXT:   }
+# MERGED-NEXT:   Modifier (0x1001) {
+# MERGED-NEXT:     TypeLeafKind: LF_MODIFIER (0x1001)
+# MERGED-NEXT:     ModifiedType: void (0x3)
+# MERGED-NEXT:     Modifiers [ (0x3)
+# MERGED-NEXT:       Const (0x1)
+# MERGED-NEXT:       Volatile (0x2)
+# MERGED-NEXT:     ]
+# MERGED-NEXT:   }
+# MERGED-NEXT:   Procedure (0x1002) {
+# MERGED-NEXT:     TypeLeafKind: LF_PROCEDURE (0x1008)
+# MERGED-NEXT:     ReturnType: void (0x3)
+# MERGED-NEXT:     CallingConvention: NearC (0x0)
+# MERGED-NEXT:     FunctionOptions [ (0x0)
+# MERGED-NEXT:     ]
+# MERGED-NEXT:     NumParameters: 0
+# MERGED-NEXT:     ArgListType: () (0x1000)
+# MERGED-NEXT:   }
+# MERGED-NEXT:   Pointer (0x1003) {
+# MERGED-NEXT:     TypeLeafKind: LF_POINTER (0x1002)
+# MERGED-NEXT:     PointeeType: const volatile void (0x1001)
+# MERGED-NEXT:     PtrType: Near64 (0xC)
+# MERGED-NEXT:     PtrMode: Pointer (0x0)
+# MERGED-NEXT:     IsFlat: 0
+# MERGED-NEXT:     IsConst: 1
+# MERGED-NEXT:     IsVolatile: 0
+# MERGED-NEXT:     IsUnaligned: 0
+# MERGED-NEXT:     IsRestrict: 0
+# MERGED-NEXT:     IsThisPtr&: 0
+# MERGED-NEXT:     IsThisPtr&&: 0
+# MERGED-NEXT:     SizeOf: 8
+# MERGED-NEXT:   }
+# MERGED-NEXT:   Pointer (0x1004) {
+# MERGED-NEXT:     TypeLeafKind: LF_POINTER (0x1002)
+# MERGED-NEXT:     PointeeType: const volatile void* const (0x1003)
+# MERGED-NEXT:     PtrType: Near64 (0xC)
+# MERGED-NEXT:     PtrMode: Pointer (0x0)
+# MERGED-NEXT:     IsFlat: 0
+# MERGED-NEXT:     IsConst: 1
+# MERGED-NEXT:     IsVolatile: 0
+# MERGED-NEXT:     IsUnaligned: 0
+# MERGED-NEXT:     IsRestrict: 0
+# MERGED-NEXT:     IsThisPtr&: 0
+# MERGED-NEXT:     IsThisPtr&&: 0
+# MERGED-NEXT:     SizeOf: 8
+# MERGED-NEXT:   }
+# MERGED-NEXT:   Pointer (0x1005) {
+# MERGED-NEXT:     TypeLeafKind: LF_POINTER (0x1002)
+# MERGED-NEXT:     PointeeType: const volatile void* const* const (0x1004)
+# MERGED-NEXT:     PtrType: Near64 (0xC)
+# MERGED-NEXT:     PtrMode: Pointer (0x0)
+# MERGED-NEXT:     IsFlat: 0
+# MERGED-NEXT:     IsConst: 1
+# MERGED-NEXT:     IsVolatile: 0
+# MERGED-NEXT:     IsUnaligned: 0
+# MERGED-NEXT:     IsRestrict: 0
+# MERGED-NEXT:     IsThisPtr&: 0
+# MERGED-NEXT:     IsThisPtr&&: 0
+# MERGED-NEXT:     SizeOf: 8
+# MERGED-NEXT:   }
+# MERGED-NEXT:   Pointer (0x1006) {
+# MERGED-NEXT:     TypeLeafKind: LF_POINTER (0x1002)
+# MERGED-NEXT:     PointeeType: const volatile void* const* const* const (0x1005)
+# MERGED-NEXT:     PtrType: Near64 (0xC)
+# MERGED-NEXT:     PtrMode: Pointer (0x0)
+# MERGED-NEXT:     IsFlat: 0
+# MERGED-NEXT:     IsConst: 1
+# MERGED-NEXT:     IsVolatile: 0
+# MERGED-NEXT:     IsUnaligned: 0
+# MERGED-NEXT:     IsRestrict: 0
+# MERGED-NEXT:     IsThisPtr&: 0
+# MERGED-NEXT:     IsThisPtr&&: 0
+# MERGED-NEXT:     SizeOf: 8
+# MERGED-NEXT:   }
+# MERGED-NEXT: ]
+# MERGED-NEXT: MergedIDStream [
+# MERGED-NEXT: ]
+
+--- !COFF
+header:          
+  Machine:         IMAGE_FILE_MACHINE_AMD64
+  Characteristics: [  ]
+sections:        
+  - Name:            '.debug$T'
+    Characteristics: [ IMAGE_SCN_CNT_INITIALIZED_DATA, IMAGE_SCN_MEM_DISCARDABLE, IMAGE_SCN_MEM_READ ]
+    Alignment:       1
+    Types:           
+      - Kind:            LF_PROCEDURE
+        Procedure:       
+          ReturnType:      3
+          CallConv:        NearC
+          Options:         [ None ]
+          ParameterCount:  0
+          ArgumentList:    4097
+      - Kind:            LF_ARGLIST
+        ArgList:         
+          ArgIndices:      [  ]
+      - Kind:            LF_POINTER
+        Pointer:         
+          ReferentType:    4099
+          Attrs:           66572
+      - Kind:            LF_POINTER
+        Pointer:         
+          ReferentType:    4100
+          Attrs:           66572
+      - Kind:            LF_POINTER
+        Pointer:         
+          ReferentType:    4101
+          Attrs:           66572
+      - Kind:            LF_MODIFIER
+        Modifier:        
+          ModifiedType:    3
+          Modifiers:       [ None, Const, Volatile ]
+      - Kind:            LF_POINTER
+        Pointer:         
+          ReferentType:    4098
+          Attrs:           66572
+symbols:         
+...
diff --git a/test/tools/llvm-readobj/demangle.test b/test/tools/llvm-readobj/demangle.test
new file mode 100644
index 0000000000000000000000000000000000000000..809a7243edf34fb5868f69c78e14af26520b01d6
--- /dev/null
+++ b/test/tools/llvm-readobj/demangle.test
@@ -0,0 +1,236 @@
+## Show that llvm-readelf + llvm-readobj demangle symbol names in symbol tables
+## (including dynamic symbols), relocations (including dynamic relocations), and groups.
+
+# RUN: yaml2obj %s > %t.so
+
+## Check LLVM output style.
+# RUN: llvm-readobj --symbols --relocations --dyn-symbols --dyn-relocations \
+# RUN:              --elf-section-groups --elf-cg-profile --addrsig         \
+# RUN:              --demangle %t.so > %t.llvm.long
+# RUN: llvm-readobj --symbols --relocations --dyn-symbols --dyn-relocations \
+# RUN:              --elf-section-groups --elf-cg-profile --addrsig         \
+# RUN:              -C %t.so > %t.llvm.short
+# RUN: FileCheck %s --input-file %t.llvm.long --check-prefixes=LLVM-COMMON,LLVM-DEMANGLE
+# RUN: diff %t.llvm.long %t.llvm.short
+
+## Check that default is no demangling.
+# RUN: llvm-readobj --symbols --relocations --dyn-symbols --dyn-relocations \
+# RUN:              --elf-section-groups --elf-cg-profile --addrsig         \
+# RUN:              %t.so > %t.llvm.default
+# RUN: llvm-readobj --symbols --relocations --dyn-symbols --dyn-relocations \
+# RUN:              --elf-section-groups --elf-cg-profile --addrsig         \
+# RUN:              --demangle=false %t.so > %t.llvm.nodemangle
+# RUN: FileCheck %s --input-file %t.llvm.default --check-prefixes=LLVM-COMMON,LLVM-MANGLED
+# RUN: diff %t.llvm.default %t.llvm.nodemangle
+
+# LLVM-COMMON:        Relocations [
+# LLVM-COMMON:          Section {{.*}} .rela.text.foo {
+# LLVM-DEMANGLE-NEXT:     {{ }}foo(char){{ }}
+# LLVM-MANGLED-NEXT:      {{ }}_Z3fooc{{ }}
+# LLVM-COMMON-NEXT:     }
+# LLVM-COMMON:        ]
+
+# LLVM-COMMON:        Dynamic Relocations {
+# LLVM-DEMANGLE-NEXT:   {{ }}foo(int){{ }}
+# LLVM-MANGLED-NEXT:    {{ }}_Z3fooi{{ }}
+# LLVM-COMMON-NEXT:   }
+
+# LLVM-COMMON:   Symbols [
+# LLVM-DEMANGLE:   Name: foo(char){{ }}
+# LLVM-DEMANGLE:   Name: blah(float){{ }}
+# LLVM-MANGLED:    Name: _Z3fooc{{ }}
+# LLVM-MANGLED:    Name: _Z4blahf{{ }}
+# LLVM-COMMON:   ]
+
+# LLVM-COMMON:   DynamicSymbols [
+# LLVM-DEMANGLE:   Name: foo(int){{ }}
+# LLVM-MANGLED:    Name: _Z3fooi{{ }}
+# LLVM-COMMON:   ]
+
+# LLVM-COMMON:   Groups {
+# LLVM-DEMANGLE:   Signature: foo(char){{$}}
+# LLVM-MANGLED:    Signature: _Z3fooc{{$}}
+# LLVM-COMMON:   }
+
+# LLVM-COMMON:   CGProfile [
+# LLVM-DEMANGLE:   From: foo(char){{ }}
+# LLVM-DEMANGLE:   To:   blah(float){{ }}
+# LLVM-MANGLED:    From: _Z3fooc{{ }}
+# LLVM-MANGLED:    To:   _Z4blahf{{ }}
+# LLVM-COMMON:   ]
+
+# LLVM-COMMON:        Addrsig [
+# LLVM-DEMANGLE-NEXT:   Sym: foo(char){{ }}
+# LLVM-DEMANGLE-NEXT:   Sym: blah(float){{ }}
+# LLVM-MANGLED-NEXT:    Sym: _Z3fooc{{ }}
+# LLVM-MANGLED-NEXT:    Sym: _Z4blahf{{ }}
+# LLVM-COMMON-NEXT:   ]
+
+## Check GNU output style.
+# RUN: llvm-readelf --symbols --relocations --dyn-symbols --dyn-relocations \
+# RUN:              --elf-section-groups --demangle %t.so > %t.gnu.long
+# RUN: llvm-readelf --symbols --relocations --dyn-symbols --dyn-relocations \
+# RUN:              --elf-section-groups -C %t.so > %t.gnu.short
+# RUN: FileCheck %s --input-file %t.gnu.long --check-prefixes=GNU-COMMON,GNU-DEMANGLE
+# RUN: diff %t.gnu.long %t.gnu.short
+
+## Check that default is no demangling.
+# RUN: llvm-readelf --symbols --relocations --dyn-symbols --dyn-relocations \
+# RUN:              --elf-section-groups %t.so > %t.gnu.default
+# RUN: llvm-readelf --symbols --relocations --dyn-symbols --dyn-relocations \
+# RUN:              --elf-section-groups --demangle=false %t.so > %t.gnu.nodemangle
+# RUN: FileCheck %s --input-file %t.gnu.default --check-prefixes=GNU-COMMON,GNU-MANGLED
+# RUN: diff %t.gnu.default %t.gnu.nodemangle
+
+# GNU-COMMON:        Relocation section '.rela.text.foo' at offset {{.*}} contains 1 entries:
+# GNU-COMMON-NEXT:     Offset Info Type Symbol's Value Symbol's Name + Addend
+# GNU-DEMANGLE-NEXT:     foo(char){{ }}
+# GNU-MANGLED-NEXT:      _Z3fooc{{ }}
+
+# GNU-COMMON:        'RELA' relocation section at offset {{.*}} contains 24 bytes:
+# GNU-COMMON-NEXT:     Offset Info Type Symbol's Value Symbol's Name + Addend
+# GNU-DEMANGLE-NEXT:     foo(int){{ }}
+# GNU-MANGLED-NEXT:      _Z3fooi{{ }}
+
+# GNU-COMMON:        Symbol table '.dynsym' contains 2 entries:
+# GNU-COMMON-NEXT:     Num: Value            Size Type   Bind  Vis     Ndx Name
+# GNU-COMMON-NEXT:       0: 0000000000000000    0 NOTYPE LOCAL DEFAULT UND
+# GNU-DEMANGLE-NEXT:       foo(int){{$}}
+# GNU-MANGLED-NEXT:        _Z3fooi{{$}}
+
+# GNU-COMMON:        Symbol table '.symtab' contains 3 entries:
+# GNU-COMMON-NEXT:     Num: Value            Size Type   Bind  Vis     Ndx Name
+# GNU-COMMON-NEXT:       0: 0000000000000000    0 NOTYPE LOCAL DEFAULT UND
+# GNU-DEMANGLE-NEXT:       foo(char){{$}}
+# GNU-DEMANGLE-NEXT:       blah(float){{$}}
+# GNU-MANGLED-NEXT:        _Z3fooc{{$}}
+# GNU-MANGLED-NEXT:        _Z4blahf{{$}}
+
+# GNU-COMMON:        COMDAT group section [{{.*}}] `.group'
+# GNU-DEMANGLE-SAME: [foo(char)]
+# GNU-MANGLED-SAME:  [_Z3fooc]
+
+!ELF
+FileHeader:
+  Class:           ELFCLASS64
+  Data:            ELFDATA2LSB
+  Type:            ET_DYN
+  Machine:         EM_X86_64
+Sections:
+  ## TODO:  Replace the raw section contents with more meaningful dynamic
+  ## tags/symbols/etc, once yaml2obj supports it.
+  ## FIXME: yaml2obj does not currently allow custom addresses for .dynstr and
+  ## .dynsym if DynamicSymbols are specified.
+  ## See https://bugs.llvm.org/show_bug.cgi?id=40339
+  - Name:         .dynstr
+    Type:         SHT_STRTAB
+    Flags:        [ SHF_ALLOC ]
+    AddressAlign: 0x100
+    EntSize:      0x1
+    ## "\0_Z3fooi\0"
+    Content: "005f5a33666f6f6900"
+  - Name:         .dynsym
+    Type:         SHT_DYNSYM
+    Flags:        [ SHF_ALLOC ]
+    Link:         .dynstr
+    Address:      0x100
+    AddressAlign: 0x100
+    EntSize:      0x18
+    ## Null symbol;
+    ## st_name: 1; st_info: Global | Func; st_other: 0;
+    ##   st_shndx: .text.foo; st_value: 0x2000; st_size: 0
+    Content: "000000000000000000000000000000000000000000000000010000001200050000200000000000000000000000000000"
+  - Name:         .rela.dyn
+    Type:         SHT_RELA
+    Flags:        [ SHF_ALLOC ]
+    Link:         .dynsym
+    Info:         .text.foo
+    Address:      0x200
+    AddressAlign: 0x100
+    EntSize:      0x18
+    Relocations:
+      - Offset: 0x10
+        ## FIXME: This should be a lookup in the corresponding symbol table, not necessarily the static symbol table.
+        ## See https://bugs.llvm.org/show_bug.cgi?id=40337.
+        Symbol: _Z3fooc
+        Type:   R_X86_64_PC32
+        Addend: 0x4
+  - Name:            .dynamic
+    Type:            SHT_DYNAMIC
+    Flags:           [ SHF_ALLOC ]
+    Address:         0x0000000000001000
+    Link:            .dynstr
+    AddressAlign:    0x0000000000001000
+    EntSize:         0x0000000000000010
+    Entries:
+      - Tag:             DT_STRTAB
+        Value:           0x0000000000000000
+      - Tag:             DT_STRSZ
+        Value:           0x0000000000000009
+      - Tag:             DT_SYMTAB
+        Value:           0x0000000000000100
+      - Tag:             DT_SYMENT
+        Value:           0x0000000000000018
+      - Tag:             DT_RELA
+        Value:           0x0000000000000200
+      - Tag:             DT_RELASZ
+        Value:           0x0000000000000018
+      - Tag:             DT_RELAENT
+        Value:           0x0000000000000018
+      - Tag:             DT_NULL
+        Value:           0x0000000000000000
+  - Name:  .text.foo
+    Type:  SHT_PROGBITS
+    Flags: [ SHF_ALLOC, SHF_EXECINSTR, SHF_GROUP ]
+    Size:  0x40
+    Address: 0x2000
+    AddressAlign: 0x2000
+  - Name:  .group
+    Type:  SHT_GROUP
+    Link:  .symtab
+    Info:  _Z3fooc
+    Members:
+      - SectionOrType: GRP_COMDAT
+      - SectionOrType: .text.foo
+  - Name: .rela.text.foo
+    Type: SHT_RELA
+    Link: .symtab
+    Info: .text.foo
+    Relocations:
+      - Offset: 0x10
+        Symbol: _Z3fooc
+        Type:   R_X86_64_PC32
+  - Name: .llvm.call-graph-profile
+    Type: SHT_LLVM_CALL_GRAPH_PROFILE
+    Link: .symtab
+    EntSize: 16
+    Content: "01000000020000002000000000000000"
+  - Name: .llvm_addrsig
+    Type: SHT_LLVM_ADDRSIG
+    Link: .symtab
+    Content: "0102"
+Symbols:
+  Global:
+    - Name:    _Z3fooc
+      Type:    STT_FUNC
+      Section: .text.foo
+    - Name:    _Z4blahf
+      Type:    STT_FUNC
+      Section: .text.foo
+ProgramHeaders:
+  - Type:  PT_LOAD
+    Flags: [ PF_R, PF_X ]
+    VAddr: 0x0
+    PAddr: 0x0
+    Sections:
+      - Section: .dynsym
+      - Section: .dynstr
+      - Section: .rela.dyn
+      - Section: .dynamic
+      - Section: .text.foo
+  - Type:  PT_DYNAMIC
+    Flags: [ PF_R ]
+    VAddr: 0x1000
+    PAddr: 0x1000
+    Sections:
+      - Section: .dynamic
diff --git a/test/tools/llvm-readobj/dyn-symbols.test b/test/tools/llvm-readobj/dyn-symbols.test
new file mode 100644
index 0000000000000000000000000000000000000000..16a2aba99e1dbf25a98f25b674b2f4f8325e8154
--- /dev/null
+++ b/test/tools/llvm-readobj/dyn-symbols.test
@@ -0,0 +1,163 @@
+RUN: llvm-readobj --dyn-symbols %p/Inputs/dynamic-table-so.x86 | FileCheck %s
+
+# Check the two-letter alias -dt is equivalent to the --dyn-symbols full flag
+# name.
+RUN: llvm-readobj -dt %p/Inputs/dynamic-table-so.x86 > %t.readobj-dt-alias
+RUN: llvm-readobj --dyn-symbols %p/Inputs/dynamic-table-so.x86 > %t.readobj-dt-no-alias
+RUN: diff %t.readobj-dt-alias %t.readobj-dt-no-alias
+
+# CHECK:      DynamicSymbols [
+# CHECK-NEXT:   Symbol {
+# CHECK-NEXT:     Name:
+# CHECK-NEXT:     Value: 0x0
+# CHECK-NEXT:     Size: 0
+# CHECK-NEXT:     Binding: Local
+# CHECK-NEXT:     Type: None
+# CHECK-NEXT:     Other: 0
+# CHECK-NEXT:     Section: Undefined
+# CHECK-NEXT:   }
+# CHECK-NEXT:   Symbol {
+# CHECK-NEXT:     Name:
+# CHECK-NEXT:     Value: 0x618
+# CHECK-NEXT:     Size: 0
+# CHECK-NEXT:     Binding: Local
+# CHECK-NEXT:     Type: Section
+# CHECK-NEXT:     Other: 0
+# CHECK-NEXT:     Section: .init
+# CHECK-NEXT:   }
+# CHECK-NEXT:   Symbol {
+# CHECK-NEXT:     Name:
+# CHECK-NEXT:     Value: 0x200DC0
+# CHECK-NEXT:     Size: 0
+# CHECK-NEXT:     Binding: Local
+# CHECK-NEXT:     Type: Section
+# CHECK-NEXT:     Other: 0
+# CHECK-NEXT:     Section: .tbss
+# CHECK-NEXT:   }
+# CHECK-NEXT:   Symbol {
+# CHECK-NEXT:     Name: _ITM_deregisterTMCloneTable{{ }}
+# CHECK-NEXT:     Value: 0x0
+# CHECK-NEXT:     Size: 0
+# CHECK-NEXT:     Binding: Weak
+# CHECK-NEXT:     Type: None
+# CHECK-NEXT:     Other: 0
+# CHECK-NEXT:     Section: Undefined
+# CHECK-NEXT:   }
+# CHECK-NEXT:   Symbol {
+# CHECK-NEXT:     Name: puts@GLIBC_2.2.5{{ }}
+# CHECK-NEXT:     Value: 0x0
+# CHECK-NEXT:     Size: 0
+# CHECK-NEXT:     Binding: Global
+# CHECK-NEXT:     Type: Function
+# CHECK-NEXT:     Other: 0
+# CHECK-NEXT:     Section: Undefined
+# CHECK-NEXT:   }
+# CHECK-NEXT:   Symbol {
+# CHECK-NEXT:     Name: __tls_get_addr@GLIBC_2.3{{ }}
+# CHECK-NEXT:     Value: 0x0
+# CHECK-NEXT:     Size: 0
+# CHECK-NEXT:     Binding: Global
+# CHECK-NEXT:     Type: Function
+# CHECK-NEXT:     Other: 0
+# CHECK-NEXT:     Section: Undefined
+# CHECK-NEXT:   }
+# CHECK-NEXT:   Symbol {
+# CHECK-NEXT:     Name: __gmon_start__{{ }}
+# CHECK-NEXT:     Value: 0x0
+# CHECK-NEXT:     Size: 0
+# CHECK-NEXT:     Binding: Weak
+# CHECK-NEXT:     Type: None
+# CHECK-NEXT:     Other: 0
+# CHECK-NEXT:     Section: Undefined
+# CHECK-NEXT:   }
+# CHECK-NEXT:   Symbol {
+# CHECK-NEXT:     Name: _Jv_RegisterClasses{{ }}
+# CHECK-NEXT:     Value: 0x0
+# CHECK-NEXT:     Size: 0
+# CHECK-NEXT:     Binding: Weak
+# CHECK-NEXT:     Type: None
+# CHECK-NEXT:     Other: 0
+# CHECK-NEXT:     Section: Undefined
+# CHECK-NEXT:   }
+# CHECK-NEXT:   Symbol {
+# CHECK-NEXT:     Name: _ITM_registerTMCloneTable{{ }}
+# CHECK-NEXT:     Value: 0x0
+# CHECK-NEXT:     Size: 0
+# CHECK-NEXT:     Binding: Weak
+# CHECK-NEXT:     Type: None
+# CHECK-NEXT:     Other: 0
+# CHECK-NEXT:     Section: Undefined
+# CHECK-NEXT:   }
+# CHECK-NEXT:   Symbol {
+# CHECK-NEXT:     Name: __cxa_finalize@GLIBC_2.2.5{{ }}
+# CHECK-NEXT:     Value: 0x0
+# CHECK-NEXT:     Size: 0
+# CHECK-NEXT:     Binding: Weak
+# CHECK-NEXT:     Type: Function
+# CHECK-NEXT:     Other: 0
+# CHECK-NEXT:     Section: Undefined
+# CHECK-NEXT:   }
+# CHECK-NEXT:   Symbol {
+# CHECK-NEXT:     Name: _edata{{ }}
+# CHECK-NEXT:     Value: 0x201030
+# CHECK-NEXT:     Size: 0
+# CHECK-NEXT:     Binding: Global
+# CHECK-NEXT:     Type: None
+# CHECK-NEXT:     Other: 0
+# CHECK-NEXT:     Section: .data
+# CHECK-NEXT:   }
+# CHECK-NEXT:   Symbol {
+# CHECK-NEXT:     Name: foo{{ }}
+# CHECK-NEXT:     Value: 0x0
+# CHECK-NEXT:     Size: 4
+# CHECK-NEXT:     Binding: Global
+# CHECK-NEXT:     Type: TLS
+# CHECK-NEXT:     Other: 0
+# CHECK-NEXT:     Section: .tbss
+# CHECK-NEXT:   }
+# CHECK-NEXT:   Symbol {
+# CHECK-NEXT:     Name: _end{{ }}
+# CHECK-NEXT:     Value: 0x201038
+# CHECK-NEXT:     Size: 0
+# CHECK-NEXT:     Binding: Global
+# CHECK-NEXT:     Type: None
+# CHECK-NEXT:     Other: 0
+# CHECK-NEXT:     Section: .bss
+# CHECK-NEXT:   }
+# CHECK-NEXT:   Symbol {
+# CHECK-NEXT:     Name: __bss_start{{ }}
+# CHECK-NEXT:     Value: 0x201030
+# CHECK-NEXT:     Size: 0
+# CHECK-NEXT:     Binding: Global
+# CHECK-NEXT:     Type: None
+# CHECK-NEXT:     Other: 0
+# CHECK-NEXT:     Section: .bss
+# CHECK-NEXT:   }
+# CHECK-NEXT:   Symbol {
+# CHECK-NEXT:     Name: main{{ }}
+# CHECK-NEXT:     Value: 0x780
+# CHECK-NEXT:     Size: 59
+# CHECK-NEXT:     Binding: Global
+# CHECK-NEXT:     Type: Function
+# CHECK-NEXT:     Other: 0
+# CHECK-NEXT:     Section: .text
+# CHECK-NEXT:   }
+# CHECK-NEXT:   Symbol {
+# CHECK-NEXT:     Name: _init{{ }}
+# CHECK-NEXT:     Value: 0x618
+# CHECK-NEXT:     Size: 0
+# CHECK-NEXT:     Binding: Global
+# CHECK-NEXT:     Type: Function
+# CHECK-NEXT:     Other: 0
+# CHECK-NEXT:     Section: .init
+# CHECK-NEXT:   }
+# CHECK-NEXT:   Symbol {
+# CHECK-NEXT:     Name: _fini{{ }}
+# CHECK-NEXT:     Value: 0x7BC
+# CHECK-NEXT:     Size: 0
+# CHECK-NEXT:     Binding: Global
+# CHECK-NEXT:     Type: Function
+# CHECK-NEXT:     Other: 0
+# CHECK-NEXT:     Section: .fini
+# CHECK-NEXT:   }
+# CHECK-NEXT: ]
diff --git a/test/tools/llvm-readobj/elf-versioninfo.test b/test/tools/llvm-readobj/elf-versioninfo.test
index 919120e2caa936c7fef8c21e71ffc19411d4e4cc..82029cf82fd3e3691c9bd3e11c6543511cd3bf94 100644
--- a/test/tools/llvm-readobj/elf-versioninfo.test
+++ b/test/tools/llvm-readobj/elf-versioninfo.test
@@ -14,15 +14,15 @@ CHECK-NEXT:   Link: 1
 CHECK-NEXT:   Symbols [
 CHECK-NEXT:     Symbol {
 CHECK-NEXT:       Version: 0
-CHECK-NEXT:       Name: @
+CHECK-NEXT:       Name: {{$}}
 CHECK-NEXT:     }
 CHECK-NEXT:     Symbol {
 CHECK-NEXT:       Version: 1
-CHECK-NEXT:       Name: _end@
+CHECK-NEXT:       Name: _end{{$}}
 CHECK-NEXT:     }
 CHECK-NEXT:     Symbol {
 CHECK-NEXT:       Version: 1
-CHECK-NEXT:       Name: _edata@
+CHECK-NEXT:       Name: _edata{{$}}
 CHECK-NEXT:     }
 CHECK-NEXT:     Symbol {
 CHECK-NEXT:       Version: 3
@@ -30,7 +30,7 @@ CHECK-NEXT:       Name: goo@@VERSION2
 CHECK-NEXT:     }
 CHECK-NEXT:     Symbol {
 CHECK-NEXT:       Version: 1
-CHECK-NEXT:       Name: __bss_start@
+CHECK-NEXT:       Name: __bss_start{{$}}
 CHECK-NEXT:     }
 CHECK-NEXT:     Symbol {
 CHECK-NEXT:       Version: 2
diff --git a/test/tools/llvm-readobj/gnu-hash-symbols.test b/test/tools/llvm-readobj/gnu-hash-symbols.test
new file mode 100644
index 0000000000000000000000000000000000000000..8babac599c36006daf338bee77c3a0ca4b50a054
--- /dev/null
+++ b/test/tools/llvm-readobj/gnu-hash-symbols.test
@@ -0,0 +1,112 @@
+# RUN: llvm-readelf --hash-symbols %p/Inputs/dynamic-table-exe.x86 \
+# RUN:   | FileCheck %s --check-prefix HASH
+
+# HASH:     Symbol table of .hash for image:
+# HASH-NEXT:   Num Buc:    Value  Size   Type   Bind Vis      Ndx Name
+# HASH-NEXT:     9   0: 00000000     0 FUNC    GLOBAL DEFAULT UND __gxx_personality_v0@CXXABI_1.3
+# HASH-NEXT:    13   0: 00001b64     0 NOTYPE  GLOBAL DEFAULT ABS _edata{{$}}
+# HASH-NEXT:     7   0: 00000000     0 FUNC    GLOBAL DEFAULT UND _ZNSt14basic_ifstreamIcSt11char_traitsIcEEC1EPKcSt13_Ios_Openmode@GLIBCXX_3.4
+# HASH-NEXT:     2   0: 00000000     0 NOTYPE  WEAK   DEFAULT UND _Jv_RegisterClasses{{$}}
+# HASH-NEXT:     1   0: 00000000     0 NOTYPE  WEAK   DEFAULT UND __gmon_start__{{$}}
+# HASH-NEXT:    16   1: 00000850    81 FUNC    GLOBAL DEFAULT  14 main{{$}}
+# HASH-NEXT:    10   1: 00000000     0 FUNC    GLOBAL DEFAULT UND _Unwind_Resume@GCC_3.0
+# HASH-NEXT:     8   1: 00000000     0 FUNC    GLOBAL DEFAULT UND puts@GLIBC_2.0
+# HASH-NEXT:    12   1: 00001b68     0 NOTYPE  GLOBAL DEFAULT ABS _end{{$}}
+# HASH-NEXT:     6   1: 00000000     0 FUNC    GLOBAL DEFAULT UND _ZNSt14basic_ifstreamIcSt11char_traitsIcEED1Ev@GLIBCXX_3.4
+# HASH-NEXT:     5   1: 00000000     0 NOTYPE  WEAK   DEFAULT UND _ITM_registerTMCloneTable{{$}}
+# HASH-NEXT:     4   1: 00000000     0 NOTYPE  WEAK   DEFAULT UND _ITM_deregisterTMCloneTable{{$}}
+# HASH-NEXT:     3   1: 00000000     0 FUNC    GLOBAL DEFAULT UND __libc_start_main@GLIBC_2.0
+# HASH-NEXT:    11   2: 00000000     0 FUNC    WEAK   DEFAULT UND __cxa_finalize@GLIBC_2.1.3
+# HASH-NEXT:    15   2: 00001b64     0 NOTYPE  GLOBAL DEFAULT ABS __bss_start{{$}}
+# HASH-NEXT:    14   2: 0000093c     4 OBJECT  GLOBAL DEFAULT  16 _IO_stdin_used{{$}}
+# HASH:     Symbol table of .gnu.hash for image:
+# HASH-NEXT:   Num Buc:    Value  Size   Type   Bind Vis      Ndx Name
+# HASH-NEXT:    12   0: 00001b68     0 NOTYPE  GLOBAL DEFAULT ABS _end{{$}}
+# HASH-NEXT:    13   0: 00001b64     0 NOTYPE  GLOBAL DEFAULT ABS _edata{{$}}
+# HASH-NEXT:    14   1: 0000093c     4 OBJECT  GLOBAL DEFAULT  16 _IO_stdin_used{{$}}
+# HASH-NEXT:    15   1: 00001b64     0 NOTYPE  GLOBAL DEFAULT ABS __bss_start{{$}}
+# HASH-NEXT:    16   1: 00000850    81 FUNC    GLOBAL DEFAULT  14 main{{$}}
+
+## Show that if there are no hash sections, we do not print anything.
+# RUN: yaml2obj %s -o %t.o
+# RUN: llvm-readelf --hash-symbols %t.o \
+# RUN:   | FileCheck %s --check-prefix NO-HASH --allow-empty
+
+# NO-HASH-NOT: {{.}}
+
+## Sanity check that we can still find the dynamic symbols (i.e. the above test
+## doesn't pass due to a mistake in the dynamic section).
+# RUN: llvm-readelf --dyn-symbols %t.o | FileCheck %s --check-prefix DYNSYMS
+
+# DYNSYMS: Symbol table '.dynsym' contains 2 entries:
+
+!ELF
+FileHeader:
+  Class:           ELFCLASS64
+  Data:            ELFDATA2LSB
+  Type:            ET_DYN
+  Machine:         EM_X86_64
+Sections:
+  ## TODO:  Replace the raw section contents with more meaningful dynamic
+  ## tags/symbols/etc, once yaml2obj supports it.
+  ## FIXME: yaml2obj does not currently allow custom addresses for .dynstr and
+  ## .dynsym if DynamicSymbols are specified.
+  ## See https://bugs.llvm.org/show_bug.cgi?id=40339
+  - Name:         .dynstr
+    Type:         SHT_STRTAB
+    Flags:        [ SHF_ALLOC ]
+    AddressAlign: 0x100
+    EntSize:      0x1
+    ## "\0_Z3fooi\0"
+    Content: "005f5a33666f6f6900"
+  - Name:         .dynsym
+    Type:         SHT_DYNSYM
+    Flags:        [ SHF_ALLOC ]
+    Link:         .dynstr
+    Address:      0x100
+    AddressAlign: 0x100
+    EntSize:      0x18
+    ## Null symbol;
+    ## st_name: 1; st_info: Global | Func; st_other: 0;
+    ##   st_shndx: .text.foo; st_value: 0x2000; st_size: 0
+    Content: "000000000000000000000000000000000000000000000000010000001200040000200000000000000000000000000000"
+  - Name:            .dynamic
+    Type:            SHT_DYNAMIC
+    Flags:           [ SHF_ALLOC ]
+    Address:         0x0000000000001000
+    Link:            .dynstr
+    AddressAlign:    0x0000000000001000
+    EntSize:         0x0000000000000010
+    Entries:
+      - Tag:             DT_STRTAB
+        Value:           0x0000000000000000
+      - Tag:             DT_STRSZ
+        Value:           0x0000000000000009
+      - Tag:             DT_SYMTAB
+        Value:           0x0000000000000100
+      - Tag:             DT_SYMENT
+        Value:           0x0000000000000018
+      - Tag:             DT_NULL
+        Value:           0x0000000000000000
+  - Name:  .text.foo
+    Type:  SHT_PROGBITS
+    Flags: [ SHF_ALLOC, SHF_EXECINSTR, SHF_GROUP ]
+    Size:  0x40
+    Address: 0x2000
+    AddressAlign: 0x2000
+ProgramHeaders:
+  - Type:  PT_LOAD
+    Flags: [ PF_R, PF_X ]
+    VAddr: 0x0
+    PAddr: 0x0
+    Sections:
+      - Section: .dynsym
+      - Section: .dynstr
+      - Section: .dynamic
+      - Section: .text.foo
+  - Type:  PT_DYNAMIC
+    Flags: [ PF_R ]
+    VAddr: 0x1000
+    PAddr: 0x1000
+    Sections:
+      - Section: .dynamic
diff --git a/test/tools/llvm-readobj/gnu-phdrs.test b/test/tools/llvm-readobj/gnu-phdrs.test
index 3054a5a70a2aa6043d22621ab6ffdc89652742e3..f6840e5039c857e8f17e1af2c149c5f546c3790c 100644
--- a/test/tools/llvm-readobj/gnu-phdrs.test
+++ b/test/tools/llvm-readobj/gnu-phdrs.test
@@ -12,8 +12,22 @@
 
 RUN: llvm-readobj -program-headers %p/Inputs/phdrs-elf.exe-i386 --elf-output-style=GNU \
 RUN:   | FileCheck %s -check-prefix ELF32
-RUN: llvm-readobj -program-headers %p/Inputs/phdrs-elf.exe-x86_64 \
-RUN:  --elf-output-style=GNU | FileCheck %s -check-prefix ELF64
+RUN: llvm-readobj -program-headers %p/Inputs/phdrs-elf.exe-x86_64 --elf-output-style=GNU \
+RUN:   | FileCheck %s -check-prefixes ELF64-PHDRS,ELF64-MAPPING
+RUN: llvm-readelf -program-headers %p/Inputs/phdrs-elf.exe-x86_64 \
+RUN:   | FileCheck %s -check-prefixes ELF64-PHDRS,ELF64-MAPPING
+
+# Check that -section-mapping produces a mapping and not the program headers.
+RUN: llvm-readelf -section-mapping %p/Inputs/phdrs-elf.exe-x86_64 \
+RUN:   | FileCheck %s -check-prefix ELF64-MAPPING -implicit-check-not="Program Headers:"
+
+# Check that -section-mapping=false -program-headers produces just program headers.
+RUN: llvm-readelf -section-mapping=false -program-headers %p/Inputs/phdrs-elf.exe-x86_64 \
+RUN:   | FileCheck %s -check-prefix ELF64-PHDRS -implicit-check-not="Section to Segment mapping:"
+
+# Check that only one copy of the section/segment mapping table is produced.
+RUN: llvm-readelf -section-mapping -program-headers %p/Inputs/phdrs-elf.exe-x86_64 \
+RUN:   | FileCheck %s -check-prefix ELF64-ONEMAPPING
 
 ELF32: Elf file type is EXEC (Executable file)
 ELF32-NEXT: Entry point 0x8048460
@@ -45,34 +59,39 @@ ELF32-NEXT:    06     .tdata .tbss
 ELF32-NEXT:    07     .eh_frame_hdr
 ELF32-NEXT:    08
 ELF32-NEXT:    09     .tdata .ctors .dtors .jcr .dynamic .got
+ELF32-NEXT:    None   .comment .shstrtab .symtab .strtab
+
+ELF64-PHDRS: Elf file type is EXEC (Executable file)
+ELF64-PHDRS-NEXT: Entry point 0x400610
+ELF64-PHDRS-NEXT: There are 10 program headers, starting at offset 64
 
-ELF64: Elf file type is EXEC (Executable file)
-ELF64-NEXT: Entry point 0x400610
-ELF64-NEXT: There are 10 program headers, starting at offset 64
+ELF64-PHDRS: Program Headers:
+ELF64-PHDRS-NEXT:   Type           Offset   VirtAddr           PhysAddr           FileSiz  MemSiz   Flg Align
+ELF64-PHDRS-NEXT:   PHDR           0x000040 0x0000000000400040 0x0000000000400040 0x000230 0x000230 R E 0x8
+ELF64-PHDRS-NEXT:   INTERP         0x000270 0x0000000000400270 0x0000000000400270 0x00001c 0x00001c R   0x1
+ELF64-PHDRS-NEXT:       [Requesting program interpreter: /lib64/ld-linux-x86-64.so.2]
+ELF64-PHDRS-NEXT:   LOAD           0x000000 0x0000000000400000 0x0000000000400000 0x000924 0x000924 R E 0x200000
+ELF64-PHDRS-NEXT:   LOAD           0x000db4 0x0000000000600db4 0x0000000000600db4 0x000274 0x0002a4 RW  0x200000
+ELF64-PHDRS-NEXT:   DYNAMIC        0x000dd0 0x0000000000600dd0 0x0000000000600dd0 0x000210 0x000210 RW  0x8
+ELF64-PHDRS-NEXT:   NOTE           0x00028c 0x000000000040028c 0x000000000040028c 0x000044 0x000044 R   0x4
+ELF64-PHDRS-NEXT:   TLS            0x000db4 0x0000000000600db4 0x0000000000600db4 0x000004 0x000008 R   0x4
+ELF64-PHDRS-NEXT:   GNU_EH_FRAME   0x00083c 0x000000000040083c 0x000000000040083c 0x00002c 0x00002c R   0x4
+ELF64-PHDRS-NEXT:   GNU_STACK      0x000000 0x0000000000000000 0x0000000000000000 0x000000 0x000000 RW  0x8
+ELF64-PHDRS-NEXT:   GNU_RELRO      0x000db4 0x0000000000600db4 0x0000000000600db4 0x00024c 0x00024c R   0x1
 
-ELF64: Program Headers:
-ELF64-NEXT:   Type           Offset   VirtAddr           PhysAddr           FileSiz  MemSiz   Flg Align
-ELF64-NEXT:   PHDR           0x000040 0x0000000000400040 0x0000000000400040 0x000230 0x000230 R E 0x8
-ELF64-NEXT:   INTERP         0x000270 0x0000000000400270 0x0000000000400270 0x00001c 0x00001c R   0x1
-ELF64-NEXT:       [Requesting program interpreter: /lib64/ld-linux-x86-64.so.2]
-ELF64-NEXT:   LOAD           0x000000 0x0000000000400000 0x0000000000400000 0x000924 0x000924 R E 0x200000
-ELF64-NEXT:   LOAD           0x000db4 0x0000000000600db4 0x0000000000600db4 0x000274 0x0002a4 RW  0x200000
-ELF64-NEXT:   DYNAMIC        0x000dd0 0x0000000000600dd0 0x0000000000600dd0 0x000210 0x000210 RW  0x8
-ELF64-NEXT:   NOTE           0x00028c 0x000000000040028c 0x000000000040028c 0x000044 0x000044 R   0x4
-ELF64-NEXT:   TLS            0x000db4 0x0000000000600db4 0x0000000000600db4 0x000004 0x000008 R   0x4
-ELF64-NEXT:   GNU_EH_FRAME   0x00083c 0x000000000040083c 0x000000000040083c 0x00002c 0x00002c R   0x4
-ELF64-NEXT:   GNU_STACK      0x000000 0x0000000000000000 0x0000000000000000 0x000000 0x000000 RW  0x8
-ELF64-NEXT:   GNU_RELRO      0x000db4 0x0000000000600db4 0x0000000000600db4 0x00024c 0x00024c R   0x1
+ELF64-MAPPING:  Section to Segment mapping:
+ELF64-MAPPING-NEXT:   Segment Sections...
+ELF64-MAPPING-NEXT:    00
+ELF64-MAPPING-NEXT:    01     .interp
+ELF64-MAPPING-NEXT:    02     .interp .note.ABI-tag .note.gnu.build-id .hash .gnu.hash .dynsym .dynstr .gnu.version .gnu.version_r .rela.dyn .rela.plt .init .plt .text .fini .rodata .eh_frame_hdr .eh_frame
+ELF64-MAPPING-NEXT:    03     .tdata .init_array .fini_array .jcr .dynamic .got .got.plt .data .bss
+ELF64-MAPPING-NEXT:    04     .dynamic
+ELF64-MAPPING-NEXT:    05     .note.ABI-tag .note.gnu.build-id
+ELF64-MAPPING-NEXT:    06     .tdata .tbss
+ELF64-MAPPING-NEXT:    07     .eh_frame_hdr
+ELF64-MAPPING-NEXT:    08
+ELF64-MAPPING-NEXT:    09     .tdata .init_array .fini_array .jcr .dynamic .got
+ELF64-MAPPING-NEXT:    None   .comment .shstrtab .symtab .strtab
 
-ELF64:  Section to Segment mapping:
-ELF64-NEXT:   Segment Sections...
-ELF64-NEXT:    00
-ELF64-NEXT:    01     .interp
-ELF64-NEXT:    02     .interp .note.ABI-tag .note.gnu.build-id .hash .gnu.hash .dynsym .dynstr .gnu.version .gnu.version_r .rela.dyn .rela.plt .init .plt .text .fini .rodata .eh_frame_hdr .eh_frame
-ELF64-NEXT:    03     .tdata .init_array .fini_array .jcr .dynamic .got .got.plt .data .bss
-ELF64-NEXT:    04     .dynamic
-ELF64-NEXT:    05     .note.ABI-tag .note.gnu.build-id
-ELF64-NEXT:    06     .tdata .tbss
-ELF64-NEXT:    07     .eh_frame_hdr
-ELF64-NEXT:    08
-ELF64-NEXT:    09     .tdata .init_array .fini_array .jcr .dynamic .got
+ELF64-ONEMAPPING: Section to Segment mapping:
+ELF64-ONEMAPPING-NOT: Section to Segment mapping:
diff --git a/test/tools/llvm-readobj/gnu-symbols.test b/test/tools/llvm-readobj/gnu-symbols.test
index 0d8dcc8c5bb051bd33f58bc1631473766fd17a1b..18d43f6290d6ba6a8bddd77f284c4ccd9244f263 100644
--- a/test/tools/llvm-readobj/gnu-symbols.test
+++ b/test/tools/llvm-readobj/gnu-symbols.test
@@ -1,11 +1,13 @@
-RUN: llvm-readobj -symbols %p/Inputs/symbols-proc-specific.elf-hexagon \
-RUN:  --elf-output-style=GNU | FileCheck %s -check-prefix ELF32
-RUN: llvm-readobj -symbols %p/Inputs/relocs.obj.elf-x86_64 --elf-output-style=GNU \
-RUN:   | FileCheck %s -check-prefix ELF64
-RUN: llvm-readobj -symbols %p/Inputs/gnuhash.so.elf-x86_64 --elf-output-style=GNU \
-RUN:   | FileCheck %s -check-prefix DYN
-RUN: llvm-readobj -dyn-symbols %p/Inputs/dynamic-table-exe.x86 --elf-output-style=GNU \
-RUN:   | FileCheck %s -check-prefix HASH
+RUN: llvm-readobj --symbols %p/Inputs/symbols-proc-specific.elf-hexagon \
+RUN:  --elf-output-style=GNU | FileCheck %s --check-prefix ELF32
+RUN: llvm-readobj --symbols %p/Inputs/relocs.obj.elf-x86_64 --elf-output-style=GNU \
+RUN:   | FileCheck %s --check-prefix ELF64
+RUN: llvm-readobj --symbols %p/Inputs/gnuhash.so.elf-x86_64 --elf-output-style=GNU \
+RUN:   | FileCheck %s --check-prefixes=SYMTAB,DYN
+RUN: llvm-readobj --dyn-symbols %p/Inputs/gnuhash.so.elf-x86_64 --elf-output-style=GNU \
+RUN:   | FileCheck %s --check-prefixes=NO-SYMTAB,DYN
+RUN: llvm-readobj --symbols --dyn-symbols %p/Inputs/gnuhash.so.elf-x86_64 --elf-output-style=GNU \
+RUN:   | FileCheck %s --check-prefixes=SYMTAB,DYN
 
 ELF32: Symbol table '.symtab' contains 5 entries:
 ELF32-NEXT:    Num:    Value  Size Type    Bind   Vis      Ndx Name
@@ -24,7 +26,9 @@ ELF64-NEXT:     3:  0000000000000000     0 SECTION LOCAL  DEFAULT    4
 ELF64-NEXT:     4:  0000000000000000     0 NOTYPE  GLOBAL DEFAULT  UND _GLOBAL_OFFSET_TABLE_
 ELF64-NEXT:     5:  0000000000000000     0 TLS     GLOBAL DEFAULT  UND sym
 
-DYN:Symbol table '.dynsym' contains 5 entries:
+NO-SYMTAB-NOT: Symbol table '.symtab'
+
+DYN: Symbol table '.dynsym' contains 5 entries:
 DYN-NEXT:   Num:    Value          Size Type    Bind   Vis      Ndx Name
 DYN-NEXT:     0: 0000000000000000     0 NOTYPE  LOCAL  DEFAULT  UND
 DYN-NEXT:     1: 00000000000001b8     0 NOTYPE  GLOBAL DEFAULT    4 foo
@@ -32,43 +36,22 @@ DYN-NEXT:     2: 0000000000200268     0 NOTYPE  GLOBAL DEFAULT    5 _edata
 DYN-NEXT:     3: 0000000000200268     0 NOTYPE  GLOBAL DEFAULT    5 _end
 DYN-NEXT:     4: 0000000000200268     0 NOTYPE  GLOBAL DEFAULT    5 __bss_start
 
-DYN: Symbol table '.symtab' contains 12 entries:
-DYN-NEXT:   Num:    Value          Size Type    Bind   Vis      Ndx Name
-DYN-NEXT:     0: 0000000000000000     0 NOTYPE  LOCAL  DEFAULT  UND
-DYN-NEXT:     1: 00000000000000e8     0 SECTION LOCAL  DEFAULT    1
-DYN-NEXT:     2: 0000000000000120     0 SECTION LOCAL  DEFAULT    2
-DYN-NEXT:     3: 0000000000000198     0 SECTION LOCAL  DEFAULT    3
-DYN-NEXT:     4: 00000000000001b8     0 SECTION LOCAL  DEFAULT    4
-DYN-NEXT:     5: 00000000002001b8     0 SECTION LOCAL  DEFAULT    5
-DYN-NEXT:     6: 00000000002001b8     0 OBJECT  LOCAL  DEFAULT    5 _DYNAMIC
-DYN-NEXT:     7: 0000000000200268     0 OBJECT  LOCAL  DEFAULT    5 _GLOBAL_OFFSET_TABLE_
-DYN-NEXT:     8: 0000000000200268     0 NOTYPE  GLOBAL DEFAULT    5 __bss_start
-DYN-NEXT:     9: 00000000000001b8     0 NOTYPE  GLOBAL DEFAULT    4 foo
-DYN-NEXT:    10: 0000000000200268     0 NOTYPE  GLOBAL DEFAULT    5 _edata
-DYN-NEXT:    11: 0000000000200268     0 NOTYPE  GLOBAL DEFAULT    5 _end
+DYN-NOT: .dynsym
+NO-SYMTAB-NOT: Symbol table '.symtab'
+
+SYMTAB: Symbol table '.symtab' contains 12 entries:
+SYMTAB-NEXT:   Num:    Value          Size Type    Bind   Vis      Ndx Name
+SYMTAB-NEXT:     0: 0000000000000000     0 NOTYPE  LOCAL  DEFAULT  UND
+SYMTAB-NEXT:     1: 00000000000000e8     0 SECTION LOCAL  DEFAULT    1
+SYMTAB-NEXT:     2: 0000000000000120     0 SECTION LOCAL  DEFAULT    2
+SYMTAB-NEXT:     3: 0000000000000198     0 SECTION LOCAL  DEFAULT    3
+SYMTAB-NEXT:     4: 00000000000001b8     0 SECTION LOCAL  DEFAULT    4
+SYMTAB-NEXT:     5: 00000000002001b8     0 SECTION LOCAL  DEFAULT    5
+SYMTAB-NEXT:     6: 00000000002001b8     0 OBJECT  LOCAL  DEFAULT    5 _DYNAMIC
+SYMTAB-NEXT:     7: 0000000000200268     0 OBJECT  LOCAL  DEFAULT    5 _GLOBAL_OFFSET_TABLE_
+SYMTAB-NEXT:     8: 0000000000200268     0 NOTYPE  GLOBAL DEFAULT    5 __bss_start
+SYMTAB-NEXT:     9: 00000000000001b8     0 NOTYPE  GLOBAL DEFAULT    4 foo
+SYMTAB-NEXT:    10: 0000000000200268     0 NOTYPE  GLOBAL DEFAULT    5 _edata
+SYMTAB-NEXT:    11: 0000000000200268     0 NOTYPE  GLOBAL DEFAULT    5 _end
 
-HASH:     Symbol table of .hash for image:
-HASH-NEXT:   Num Buc:    Value  Size   Type   Bind Vis      Ndx Name
-HASH-NEXT:     9   0: 00000000     0 FUNC    GLOBAL DEFAULT UND __gxx_personality_v0@CXXABI_1.3
-HASH-NEXT:    13   0: 00001b64     0 NOTYPE  GLOBAL DEFAULT ABS _edata@
-HASH-NEXT:     7   0: 00000000     0 FUNC    GLOBAL DEFAULT UND _ZNSt14basic_ifstreamIcSt11char_traitsIcEEC1EPKcSt13_Ios_Openmode@GLIBCXX_3.4
-HASH-NEXT:     2   0: 00000000     0 NOTYPE  WEAK   DEFAULT UND _Jv_RegisterClasses@
-HASH-NEXT:     1   0: 00000000     0 NOTYPE  WEAK   DEFAULT UND __gmon_start__@
-HASH-NEXT:    16   1: 00000850    81 FUNC    GLOBAL DEFAULT  14 main@
-HASH-NEXT:    10   1: 00000000     0 FUNC    GLOBAL DEFAULT UND _Unwind_Resume@GCC_3.0
-HASH-NEXT:     8   1: 00000000     0 FUNC    GLOBAL DEFAULT UND puts@GLIBC_2.0
-HASH-NEXT:    12   1: 00001b68     0 NOTYPE  GLOBAL DEFAULT ABS _end@
-HASH-NEXT:     6   1: 00000000     0 FUNC    GLOBAL DEFAULT UND _ZNSt14basic_ifstreamIcSt11char_traitsIcEED1Ev@GLIBCXX_3.4
-HASH-NEXT:     5   1: 00000000     0 NOTYPE  WEAK   DEFAULT UND _ITM_registerTMCloneTable@
-HASH-NEXT:     4   1: 00000000     0 NOTYPE  WEAK   DEFAULT UND _ITM_deregisterTMCloneTable@
-HASH-NEXT:     3   1: 00000000     0 FUNC    GLOBAL DEFAULT UND __libc_start_main@GLIBC_2.0
-HASH-NEXT:    11   2: 00000000     0 FUNC    WEAK   DEFAULT UND __cxa_finalize@GLIBC_2.1.3
-HASH-NEXT:    15   2: 00001b64     0 NOTYPE  GLOBAL DEFAULT ABS __bss_start@
-HASH-NEXT:    14   2: 0000093c     4 OBJECT  GLOBAL DEFAULT  16 _IO_stdin_used@
-HASH:     Symbol table of .gnu.hash for image:
-HASH-NEXT:   Num Buc:    Value  Size   Type   Bind Vis      Ndx Name
-HASH-NEXT:    12   0: 00001b68     0 NOTYPE  GLOBAL DEFAULT ABS _end@
-HASH-NEXT:    13   0: 00001b64     0 NOTYPE  GLOBAL DEFAULT ABS _edata@
-HASH-NEXT:    14   1: 0000093c     4 OBJECT  GLOBAL DEFAULT  16 _IO_stdin_used@
-HASH-NEXT:    15   1: 00001b64     0 NOTYPE  GLOBAL DEFAULT ABS __bss_start@
-HASH-NEXT:    16   1: 00000850    81 FUNC    GLOBAL DEFAULT  14 main@
+DYN-NOT: .dynsym
diff --git a/test/tools/llvm-readobj/hexdump.test b/test/tools/llvm-readobj/hexdump.test
new file mode 100644
index 0000000000000000000000000000000000000000..cadb7225596d0dcc825d675107c9aae54cedfeee
--- /dev/null
+++ b/test/tools/llvm-readobj/hexdump.test
@@ -0,0 +1,5 @@
+// Check dumping of the hexadecimal bytes of a section.
+RUN: llvm-readobj -x .text %p/../../Object/Inputs/hello-world.elf-x86-64 | FileCheck %s
+
+CHECK: Hex dump of section '.text':
+CHECK-NEXT: {{^}}0x00400460
diff --git a/test/tools/llvm-readobj/merged.test b/test/tools/llvm-readobj/merged.test
new file mode 100644
index 0000000000000000000000000000000000000000..0fcc14f1f9533d7be1a277194726c3f19687fac1
--- /dev/null
+++ b/test/tools/llvm-readobj/merged.test
@@ -0,0 +1,13 @@
+# Check merged args produce identical output to when not merged.
+RUN: llvm-readelf -aeWhSrnudlVgIs %p/Inputs/trivial.obj.elf-i386 > %t.merged
+RUN: llvm-readelf -a -e -W -h -S -r -n -u -d -l -V -g -I -s %p/Inputs/trivial.obj.elf-i386 > %t.not-merged
+RUN: cmp %t.merged %t.not-merged
+RUN: FileCheck %s --input-file %t.merged
+
+# llvm-readobj does not support merged args, because it also supports some old
+# flags (-st, -sd, etc.), and it would be confusing if only some merged args
+# were supported.
+RUN: not llvm-readobj -aeWhSrnudlVgIs %p/Inputs/trivial.obj.elf-i386 2>&1 | FileCheck %s --check-prefix=UNKNOWN
+
+CHECK-NOT: Unknown command line argument
+UNKNOWN:   Unknown command line argument
diff --git a/test/tools/llvm-readobj/mips-got.test b/test/tools/llvm-readobj/mips-got.test
index a5c15fdfe230abda5d7184546d1f03f7a72fb72f..7d8d2669bd27488716b7f70e96d54fc0515f9a34 100644
--- a/test/tools/llvm-readobj/mips-got.test
+++ b/test/tools/llvm-readobj/mips-got.test
@@ -73,7 +73,7 @@ GOT-EXE-NEXT:       Initial: 0x0
 GOT-EXE-NEXT:       Value: 0x0
 GOT-EXE-NEXT:       Type: Function (0x2)
 GOT-EXE-NEXT:       Section: Undefined (0x0)
-GOT-EXE-NEXT:       Name: __gmon_start__@ (1)
+GOT-EXE-NEXT:       Name: __gmon_start__ (1)
 GOT-EXE-NEXT:     }
 GOT-EXE-NEXT:   ]
 GOT-EXE-NEXT:   Number of TLS and multi-GOT entries: 0
@@ -145,7 +145,7 @@ GOT-SO-NEXT:       Initial: 0x0
 GOT-SO-NEXT:       Value: 0x0
 GOT-SO-NEXT:       Type: None (0x0)
 GOT-SO-NEXT:       Section: Undefined (0x0)
-GOT-SO-NEXT:       Name: _ITM_registerTMCloneTable@ (87)
+GOT-SO-NEXT:       Name: _ITM_registerTMCloneTable (87)
 GOT-SO-NEXT:     }
 GOT-SO-NEXT:     Entry {
 GOT-SO-NEXT:       Address: 0x1090C
@@ -154,7 +154,7 @@ GOT-SO-NEXT:       Initial: 0x0
 GOT-SO-NEXT:       Value: 0x0
 GOT-SO-NEXT:       Type: None (0x0)
 GOT-SO-NEXT:       Section: Undefined (0x0)
-GOT-SO-NEXT:       Name: _Jv_RegisterClasses@ (128)
+GOT-SO-NEXT:       Name: _Jv_RegisterClasses (128)
 GOT-SO-NEXT:     }
 GOT-SO-NEXT:     Entry {
 GOT-SO-NEXT:       Address: 0x10910
@@ -163,7 +163,7 @@ GOT-SO-NEXT:       Initial: 0x0
 GOT-SO-NEXT:       Value: 0x0
 GOT-SO-NEXT:       Type: Function (0x2)
 GOT-SO-NEXT:       Section: Undefined (0x0)
-GOT-SO-NEXT:       Name: __gmon_start__@ (23)
+GOT-SO-NEXT:       Name: __gmon_start__ (23)
 GOT-SO-NEXT:     }
 GOT-SO-NEXT:     Entry {
 GOT-SO-NEXT:       Address: 0x10914
@@ -181,7 +181,7 @@ GOT-SO-NEXT:       Initial: 0x0
 GOT-SO-NEXT:       Value: 0x0
 GOT-SO-NEXT:       Type: None (0x0)
 GOT-SO-NEXT:       Section: Undefined (0x0)
-GOT-SO-NEXT:       Name: _ITM_deregisterTMCloneTable@ (59)
+GOT-SO-NEXT:       Name: _ITM_deregisterTMCloneTable (59)
 GOT-SO-NEXT:     }
 GOT-SO-NEXT:     Entry {
 GOT-SO-NEXT:       Address: 0x1091C
@@ -277,7 +277,7 @@ GOT-TLS-NEXT:       Initial: 0x0
 GOT-TLS-NEXT:       Value: 0x0
 GOT-TLS-NEXT:       Type: None (0x0)
 GOT-TLS-NEXT:       Section: Undefined (0x0)
-GOT-TLS-NEXT:       Name: _ITM_registerTMCloneTable@ (78)
+GOT-TLS-NEXT:       Name: _ITM_registerTMCloneTable (78)
 GOT-TLS-NEXT:     }
 GOT-TLS-NEXT:     Entry {
 GOT-TLS-NEXT:       Address: 0x10C70
@@ -286,7 +286,7 @@ GOT-TLS-NEXT:       Initial: 0x0
 GOT-TLS-NEXT:       Value: 0x0
 GOT-TLS-NEXT:       Type: None (0x0)
 GOT-TLS-NEXT:       Section: Undefined (0x0)
-GOT-TLS-NEXT:       Name: _Jv_RegisterClasses@ (119)
+GOT-TLS-NEXT:       Name: _Jv_RegisterClasses (119)
 GOT-TLS-NEXT:     }
 GOT-TLS-NEXT:     Entry {
 GOT-TLS-NEXT:       Address: 0x10C78
@@ -295,7 +295,7 @@ GOT-TLS-NEXT:       Initial: 0x0
 GOT-TLS-NEXT:       Value: 0x0
 GOT-TLS-NEXT:       Type: Function (0x2)
 GOT-TLS-NEXT:       Section: Undefined (0x0)
-GOT-TLS-NEXT:       Name: __gmon_start__@ (23)
+GOT-TLS-NEXT:       Name: __gmon_start__ (23)
 GOT-TLS-NEXT:     }
 GOT-TLS-NEXT:     Entry {
 GOT-TLS-NEXT:       Address: 0x10C80
@@ -313,7 +313,7 @@ GOT-TLS-NEXT:       Initial: 0x0
 GOT-TLS-NEXT:       Value: 0x0
 GOT-TLS-NEXT:       Type: None (0x0)
 GOT-TLS-NEXT:       Section: Undefined (0x0)
-GOT-TLS-NEXT:       Name: _ITM_deregisterTMCloneTable@ (50)
+GOT-TLS-NEXT:       Name: _ITM_deregisterTMCloneTable (50)
 GOT-TLS-NEXT:     }
 GOT-TLS-NEXT:     Entry {
 GOT-TLS-NEXT:       Address: 0x10C90
diff --git a/test/tools/llvm-readobj/note-gnu-property.s b/test/tools/llvm-readobj/note-gnu-property.s
index d513a3e460c69414c4fd31bd0a63f8bf852b3340..1d35274bf2e563584491160cdc9c7fa95943bb7c 100644
--- a/test/tools/llvm-readobj/note-gnu-property.s
+++ b/test/tools/llvm-readobj/note-gnu-property.s
@@ -3,44 +3,48 @@
 // RUN: llvm-readobj -elf-output-style GNU --notes %t | FileCheck %s --check-prefix=GNU
 // RUN: llvm-readobj -elf-output-style LLVM --notes %t | FileCheck %s --check-prefix=LLVM
 
-// GNU:      Displaying notes found at file offset 0x00000040 with length 0x000000b8:
+// GNU:      Displaying notes found at file offset 0x00000040 with length 0x000000d8:
 // GNU-NEXT:   Owner                 Data size       Description
-// GNU-NEXT:   GNU                   0x000000a8      NT_GNU_PROPERTY_TYPE_0 (property note)
+// GNU-NEXT:   GNU                   0x000000c8      NT_GNU_PROPERTY_TYPE_0 (property note)
 // GNU-NEXT:     Properties:  stack size: 0x100
 // GNU-NEXT:     stack size: 0x100
 // GNU-NEXT:     no copy on protected
-// GNU-NEXT:     X86 features: SHSTK
-// GNU-NEXT:     X86 features: IBT, SHSTK
-// GNU-NEXT:     X86 features: none
+// GNU-NEXT:     x86 feature: SHSTK
+// GNU-NEXT:     x86 feature: IBT, SHSTK
+// GNU-NEXT:     x86 feature: <None>
+// GNU-NEXT:     x86 feature needed: x86, x87
+// GNU-NEXT:     x86 feature used: XSAVEOPT, XSAVEC
 // GNU-NEXT:     <application-specific type 0xfefefefe>
 // GNU-NEXT:     stack size: <corrupt length: 0x0>
 // GNU-NEXT:     stack size: <corrupt length: 0x4>
 // GNU-NEXT:     no copy on protected <corrupt length: 0x1>
-// GNU-NEXT:     X86 features: <corrupt length: 0x0>
-// GNU-NEXT:     X86 features: IBT, <unknown flags: 0xf000f000f000f000>
+// GNU-NEXT:     x86 feature: <corrupt length: 0x0>
+// GNU-NEXT:     x86 feature: IBT, <unknown flags: 0xf000f000>
 // GNU-NEXT:     <corrupt type (0x2) datasz: 0x1>
 
 // LLVM:      Notes [
 // LLVM-NEXT:   NoteSection {
 // LLVM-NEXT:     Offset: 0x40
-// LLVM-NEXT:     Size: 0xB8
+// LLVM-NEXT:     Size: 0xD8
 // LLVM-NEXT:     Note {
 // LLVM-NEXT:       Owner: GNU
-// LLVM-NEXT:       Data size: 0xA8
+// LLVM-NEXT:       Data size: 0xC8
 // LLVM-NEXT:       Type: NT_GNU_PROPERTY_TYPE_0 (property note)
 // LLVM-NEXT:       Property [
 // LLVM-NEXT:         stack size: 0x100
 // LLVM-NEXT:         stack size: 0x100
 // LLVM-NEXT:         no copy on protected
-// LLVM-NEXT:         X86 features: SHSTK
-// LLVM-NEXT:         X86 features: IBT, SHSTK
-// LLVM-NEXT:         X86 features: none
+// LLVM-NEXT:         x86 feature: SHSTK
+// LLVM-NEXT:         x86 feature: IBT, SHSTK
+// LLVM-NEXT:         x86 feature: <None>
+// LLVM-NEXT:         x86 feature needed: x86, x87
+// LLVM-NEXT:         x86 feature used: XSAVEOPT, XSAVEC
 // LLVM-NEXT:         <application-specific type 0xfefefefe>
 // LLVM-NEXT:         stack size: <corrupt length: 0x0>
 // LLVM-NEXT:         stack size: <corrupt length: 0x4>
 // LLVM-NEXT:         no copy on protected <corrupt length: 0x1>
-// LLVM-NEXT:         X86 features: <corrupt length: 0x0>
-// LLVM-NEXT:         X86 features: IBT, <unknown flags: 0xf000f000f000f000>
+// LLVM-NEXT:         x86 feature: <corrupt length: 0x0>
+// LLVM-NEXT:         x86 feature: IBT, <unknown flags: 0xf000f000>
 // LLVM-NEXT:         <corrupt type (0x2) datasz: 0x1>
 // LLVM-NEXT:       ]
 // LLVM-NEXT:     }
@@ -72,8 +76,8 @@ begin:
 
   /* CET property note */
   .long 0xc0000002  /* Type: GNU_PROPERTY_X86_FEATURE_1_AND */
-  .long 8           /* Data size */
-  .quad 2           /* GNU_PROPERTY_X86_FEATURE_1_SHSTK */
+  .long 4           /* Data size */
+  .long 2           /* GNU_PROPERTY_X86_FEATURE_1_SHSTK */
   .p2align 3        /* Align to 8 byte for 64 bit */
 
   /* CET property note with padding */
@@ -83,9 +87,19 @@ begin:
   .p2align 3        /* Align to 8 byte for 64 bit */
 
   .long 0xc0000002  /* Type: GNU_PROPERTY_X86_FEATURE_1_AND */
-  .long 8           /* Data size */
-  .quad 0           /* Empty flags, not an error */
+  .long 4           /* Data size */
+  .long 0           /* Empty flags, not an error */
   .p2align 3        /* Align to 8 byte for 64 bit */
+
+  .long 0xc0008001         /* Type: GNU_PROPERTY_X86_FEATURE_2_NEEDED */
+  .long 4                  /* Data size */
+  .long 0x00000003         /* X86 X87 */
+  .p2align 3               /* Align to 8 byte for 64 bit */
+
+  .long 0xc0010001         /* Type: GNU_PROPERTY_X86_FEATURE_2_USED */
+  .long 4                  /* Data size */
+  .long 0x00000300         /* XSAVEOPT XSAVEC */
+  .p2align 3               /* Align to 8 byte for 64 bit */
   
   /* All notes below are broken. Test we are able to report them. */
   
@@ -118,10 +132,10 @@ begin:
 
   /* CET note with bad flags */
   .long 0xc0000002         /* Type: GNU_PROPERTY_X86_FEATURE_1_AND */
-  .long 8                  /* Data size */
-  .quad 0xf000f000f000f001 /* GNU_PROPERTY_X86_FEATURE_1_IBT and bad bits */
+  .long 4                  /* Data size */
+  .long 0xf000f001         /* GNU_PROPERTY_X86_FEATURE_1_IBT and bad bits */
   .p2align 3               /* Align to 8 byte for 64 bit */
-  
+
   /* GNU_PROPERTY_NO_COPY_ON_PROTECTED with pr_datasz and without data */
   .long 2           /* Type: GNU_PROPERTY_NO_COPY_ON_PROTECTED */
   .long 1           /* Data size (corrupted) */
diff --git a/test/tools/llvm-readobj/relocations.test b/test/tools/llvm-readobj/relocations.test
index 4a7dfa5eba0e389e2a5b44f0120c11ec6e420198..9ca79e3c78d5b89a023b9fea79ebc78b8322a2d3 100644
--- a/test/tools/llvm-readobj/relocations.test
+++ b/test/tools/llvm-readobj/relocations.test
@@ -21,9 +21,9 @@ RUN:   | FileCheck %s -check-prefix WASM
 
 COFF:      Relocations [
 COFF-NEXT:   Section (1) .text {
-COFF-NEXT:     0x4 IMAGE_REL_I386_DIR32 .data
-COFF-NEXT:     0x9 IMAGE_REL_I386_REL32 _puts
-COFF-NEXT:     0xE IMAGE_REL_I386_REL32 _SomeOtherFunction
+COFF-NEXT:     0x4 IMAGE_REL_I386_DIR32 .data (4)
+COFF-NEXT:     0x9 IMAGE_REL_I386_REL32 _puts (7)
+COFF-NEXT:     0xE IMAGE_REL_I386_REL32 _SomeOtherFunction (8)
 COFF-NEXT:   }
 COFF-NEXT: ]
 
@@ -289,23 +289,23 @@ MACHO-ARM-NEXT:  ]
 WASM:      Relocations [
 WASM-NEXT:   Section (4) CODE {
 WASM-NEXT:     Relocation {
-WASM-NEXT:       Type: R_WEBASSEMBLY_MEMORY_ADDR_SLEB (4)
+WASM-NEXT:       Type: R_WASM_MEMORY_ADDR_SLEB (4)
 WASM-NEXT:       Offset: 0x4
 WASM-NEXT:       Symbol: .L.str
 WASM-NEXT:       Addend: 0
 WASM-NEXT:     }
 WASM-NEXT:     Relocation {
-WASM-NEXT:       Type: R_WEBASSEMBLY_FUNCTION_INDEX_LEB (0)
+WASM-NEXT:       Type: R_WASM_FUNCTION_INDEX_LEB (0)
 WASM-NEXT:       Offset: 0xA
 WASM-NEXT:       Symbol: puts
 WASM-NEXT:     }
 WASM-NEXT:     Relocation {
-WASM-NEXT:       Type: R_WEBASSEMBLY_FUNCTION_INDEX_LEB (0)
+WASM-NEXT:       Type: R_WASM_FUNCTION_INDEX_LEB (0)
 WASM-NEXT:       Offset: 0x11
 WASM-NEXT:       Symbol: .LSomeOtherFunction_bitcast
 WASM-NEXT:     }
 WASM-NEXT:     Relocation {
-WASM-NEXT:       Type: R_WEBASSEMBLY_FUNCTION_INDEX_LEB (0)
+WASM-NEXT:       Type: R_WASM_FUNCTION_INDEX_LEB (0)
 WASM-NEXT:       Offset: 0x1E
 WASM-NEXT:       Symbol: SomeOtherFunction
 WASM-NEXT:     }
diff --git a/test/tools/llvm-readobj/sections-ext.test b/test/tools/llvm-readobj/sections-ext.test
index 70ae0f22a0e40cc25a71a5d9f86aab77bbdf618b..085693829c394eb8388570adcb3777106081f718 100644
--- a/test/tools/llvm-readobj/sections-ext.test
+++ b/test/tools/llvm-readobj/sections-ext.test
@@ -13,6 +13,20 @@ RUN:   | FileCheck %s -check-prefix MACHO-PPC64
 RUN: llvm-readobj -expand-relocs -s -st -sr -sd %p/Inputs/trivial.obj.macho-arm \
 RUN:   | FileCheck %s -check-prefix MACHO-ARM
 
+# Check the two-letter aliases above (-st, -sr, -sd) are equivalent to their
+# full flag names.
+RUN: llvm-readobj -s -st %p/Inputs/trivial.obj.elf-i386 > %t.readobj-st-alias
+RUN: llvm-readobj -s --section-symbols %p/Inputs/trivial.obj.elf-i386 > %t.readobj-st-no-alias
+RUN: diff %t.readobj-st-alias %t.readobj-st-no-alias
+
+RUN: llvm-readobj -s -sr %p/Inputs/trivial.obj.elf-i386 > %t.readobj-sr-alias
+RUN: llvm-readobj -s --section-relocations %p/Inputs/trivial.obj.elf-i386 > %t.readobj-sr-no-alias
+RUN: diff %t.readobj-sr-alias %t.readobj-sr-no-alias
+
+RUN: llvm-readobj -s -sd %p/Inputs/trivial.obj.elf-i386 > %t.readobj-sd-alias
+RUN: llvm-readobj -s --section-data %p/Inputs/trivial.obj.elf-i386 > %t.readobj-sd-no-alias
+RUN: diff %t.readobj-sd-alias %t.readobj-sd-no-alias
+
 COFF:      Sections [
 COFF-NEXT:   Section {
 COFF-NEXT:     Number: 1
diff --git a/test/tools/llvm-readobj/symbols.test b/test/tools/llvm-readobj/symbols.test
index e68d654bd65d849bd0d9536333177a643d0c3c66..1931cefdea41143e631ce709868bcd9609af510c 100644
--- a/test/tools/llvm-readobj/symbols.test
+++ b/test/tools/llvm-readobj/symbols.test
@@ -84,26 +84,46 @@ WASM:      Symbols [
 WASM-NEXT:   Symbol {
 WASM-NEXT:     Name: main
 WASM-NEXT:     Type: FUNCTION (0x0)
-WASM-NEXT:     Flags: 0x0
+WASM-NEXT:     Flags [ (0x0)
+WASM-NEXT:     ]
+WASM-NEXT:     ElementIndex: 0x2
 WASM-NEXT:   }
 WASM-NEXT:   Symbol {
 WASM-NEXT:     Name: .L.str
 WASM-NEXT:     Type: DATA (0x1)
-WASM-NEXT:     Flags: 0x2
+WASM-NEXT:     Flags [ (0x2)
+WASM-NEXT:       BINDING_LOCAL (0x2)
+WASM-NEXT:     ]
+WASM-NEXT:     Offset: 0x0
+WASM-NEXT:     Segment: 0x0
+WASM-NEXT:     Size: 0xD
 WASM-NEXT:   }
 WASM-NEXT:   Symbol {
 WASM-NEXT:     Name: puts
 WASM-NEXT:     Type: FUNCTION (0x0)
-WASM-NEXT:     Flags: 0x10
+WASM-NEXT:     Flags [ (0x10)
+WASM-NEXT:       UNDEFINED (0x10)
+WASM-NEXT:     ]
+WASM-NEXT:     ImportName: puts
+WASM-NEXT:     ImportModule: env
+WASM-NEXT:     ElementIndex: 0x0
 WASM-NEXT:   }
 WASM-NEXT:   Symbol {
 WASM-NEXT:     Name: .LSomeOtherFunction_bitcast
 WASM-NEXT:     Type: FUNCTION (0x0)
-WASM-NEXT:     Flags: 0x2
+WASM-NEXT:     Flags [ (0x2)
+WASM-NEXT:       BINDING_LOCAL (0x2)
+WASM-NEXT:     ]
+WASM-NEXT:     ElementIndex: 0x3
 WASM-NEXT:   }
 WASM-NEXT:   Symbol {
 WASM-NEXT:     Name: SomeOtherFunction
 WASM-NEXT:     Type: FUNCTION (0x0)
-WASM-NEXT:     Flags: 0x10
+WASM-NEXT:     Flags [ (0x10)
+WASM-NEXT:       UNDEFINED (0x10)
+WASM-NEXT:     ]
+WASM-NEXT:     ImportName: SomeOtherFunction
+WASM-NEXT:     ImportModule: env
+WASM-NEXT:     ElementIndex: 0x1
 WASM-NEXT:   }
 WASM-NEXT: ]
diff --git a/test/DebugInfo/Inputs/llvm-symbolizer-dwo-test b/test/tools/llvm-symbolizer/Inputs/dwo-test
similarity index 100%
rename from test/DebugInfo/Inputs/llvm-symbolizer-dwo-test
rename to test/tools/llvm-symbolizer/Inputs/dwo-test
diff --git a/test/DebugInfo/Inputs/fission-ranges.cc b/test/tools/llvm-symbolizer/Inputs/fission-ranges.cc
similarity index 100%
rename from test/DebugInfo/Inputs/fission-ranges.cc
rename to test/tools/llvm-symbolizer/Inputs/fission-ranges.cc
diff --git a/test/DebugInfo/Inputs/fission-ranges.elf-x86_64 b/test/tools/llvm-symbolizer/Inputs/fission-ranges.elf-x86_64
similarity index 100%
rename from test/DebugInfo/Inputs/fission-ranges.elf-x86_64
rename to test/tools/llvm-symbolizer/Inputs/fission-ranges.elf-x86_64
diff --git a/test/DebugInfo/Inputs/split-dwarf-addr-object-relocation.cpp b/test/tools/llvm-symbolizer/Inputs/split-dwarf-addr-object-relocation.cpp
similarity index 100%
rename from test/DebugInfo/Inputs/split-dwarf-addr-object-relocation.cpp
rename to test/tools/llvm-symbolizer/Inputs/split-dwarf-addr-object-relocation.cpp
diff --git a/test/DebugInfo/Inputs/split-dwarf-addr-object-relocation.dwo b/test/tools/llvm-symbolizer/Inputs/split-dwarf-addr-object-relocation.dwo
similarity index 100%
rename from test/DebugInfo/Inputs/split-dwarf-addr-object-relocation.dwo
rename to test/tools/llvm-symbolizer/Inputs/split-dwarf-addr-object-relocation.dwo
diff --git a/test/DebugInfo/Inputs/split-dwarf-addr-object-relocation.o b/test/tools/llvm-symbolizer/Inputs/split-dwarf-addr-object-relocation.o
similarity index 100%
rename from test/DebugInfo/Inputs/split-dwarf-addr-object-relocation.o
rename to test/tools/llvm-symbolizer/Inputs/split-dwarf-addr-object-relocation.o
diff --git a/test/DebugInfo/Inputs/split-dwarf-dwp.cpp b/test/tools/llvm-symbolizer/Inputs/split-dwarf-dwp.cpp
similarity index 100%
rename from test/DebugInfo/Inputs/split-dwarf-dwp.cpp
rename to test/tools/llvm-symbolizer/Inputs/split-dwarf-dwp.cpp
diff --git a/test/DebugInfo/Inputs/split-dwarf-dwp.o b/test/tools/llvm-symbolizer/Inputs/split-dwarf-dwp.o
similarity index 100%
rename from test/DebugInfo/Inputs/split-dwarf-dwp.o
rename to test/tools/llvm-symbolizer/Inputs/split-dwarf-dwp.o
diff --git a/test/DebugInfo/Inputs/split-dwarf-dwp.o.dwp b/test/tools/llvm-symbolizer/Inputs/split-dwarf-dwp.o.dwp
similarity index 100%
rename from test/DebugInfo/Inputs/split-dwarf-dwp.o.dwp
rename to test/tools/llvm-symbolizer/Inputs/split-dwarf-dwp.o.dwp
diff --git a/test/DebugInfo/Inputs/split-dwarf-multiple-cu.cpp b/test/tools/llvm-symbolizer/Inputs/split-dwarf-multiple-cu.cpp
similarity index 100%
rename from test/DebugInfo/Inputs/split-dwarf-multiple-cu.cpp
rename to test/tools/llvm-symbolizer/Inputs/split-dwarf-multiple-cu.cpp
diff --git a/test/DebugInfo/Inputs/split-dwarf-multiple-cu.dwo b/test/tools/llvm-symbolizer/Inputs/split-dwarf-multiple-cu.dwo
similarity index 100%
rename from test/DebugInfo/Inputs/split-dwarf-multiple-cu.dwo
rename to test/tools/llvm-symbolizer/Inputs/split-dwarf-multiple-cu.dwo
diff --git a/test/DebugInfo/Inputs/split-dwarf-multiple-cu.o b/test/tools/llvm-symbolizer/Inputs/split-dwarf-multiple-cu.o
similarity index 100%
rename from test/DebugInfo/Inputs/split-dwarf-multiple-cu.o
rename to test/tools/llvm-symbolizer/Inputs/split-dwarf-multiple-cu.o
diff --git a/test/DebugInfo/Inputs/split-dwarf-test b/test/tools/llvm-symbolizer/Inputs/split-dwarf-test
similarity index 100%
rename from test/DebugInfo/Inputs/split-dwarf-test
rename to test/tools/llvm-symbolizer/Inputs/split-dwarf-test
diff --git a/test/DebugInfo/Inputs/split-dwarf-test-nogmlt b/test/tools/llvm-symbolizer/Inputs/split-dwarf-test-nogmlt
similarity index 100%
rename from test/DebugInfo/Inputs/split-dwarf-test-nogmlt
rename to test/tools/llvm-symbolizer/Inputs/split-dwarf-test-nogmlt
diff --git a/test/DebugInfo/Inputs/split-dwarf-test.cc b/test/tools/llvm-symbolizer/Inputs/split-dwarf-test.cc
similarity index 100%
rename from test/DebugInfo/Inputs/split-dwarf-test.cc
rename to test/tools/llvm-symbolizer/Inputs/split-dwarf-test.cc
diff --git a/test/DebugInfo/Inputs/split-dwarf-test.cpp b/test/tools/llvm-symbolizer/Inputs/split-dwarf-test.cpp
similarity index 100%
rename from test/DebugInfo/Inputs/split-dwarf-test.cpp
rename to test/tools/llvm-symbolizer/Inputs/split-dwarf-test.cpp
diff --git a/test/DebugInfo/Inputs/split-dwarf-test.dwo b/test/tools/llvm-symbolizer/Inputs/split-dwarf-test.dwo
similarity index 100%
rename from test/DebugInfo/Inputs/split-dwarf-test.dwo
rename to test/tools/llvm-symbolizer/Inputs/split-dwarf-test.dwo
diff --git a/test/tools/llvm-symbolizer/adjust-vma.s b/test/tools/llvm-symbolizer/adjust-vma.s
new file mode 100644
index 0000000000000000000000000000000000000000..5bb5a336a52c5a5304ac7fcd6fe4924ef47e4d82
--- /dev/null
+++ b/test/tools/llvm-symbolizer/adjust-vma.s
@@ -0,0 +1,39 @@
+# REQUIRES: x86-registered-target
+
+.type foo,@function
+.size foo,12
+foo:
+    .space 10
+    nop
+    nop
+
+# RUN: llvm-mc -filetype=obj -triple=x86_64-pc-linux %s -o %t.o -g
+
+# RUN: llvm-symbolizer 0xa 0xb --print-address --obj=%t.o \
+# RUN:   | FileCheck %s --check-prefix=NORMAL
+# RUN: llvm-symbolizer 0x10a 0x10b --print-address --adjust-vma 0x100 --obj=%t.o \
+# RUN:   | FileCheck %s --check-prefix=ADJUST
+
+# Show that we can handle addresses less than the offset.
+# RUN: llvm-symbolizer 0xa 0xb --print-address --adjust-vma 0xc --obj=%t.o \
+# RUN:   | FileCheck %s --check-prefix=OVERFLOW
+
+# NORMAL:      0xa
+# NORMAL-NEXT: foo
+# NORMAL-NEXT: adjust-vma.s:7:0
+# NORMAL-EMPTY:
+# NORMAL-NEXT: 0xb
+# NORMAL-NEXT: foo
+# NORMAL-NEXT: adjust-vma.s:8:0
+
+# ADJUST:      0x10a
+# ADJUST-NEXT: foo
+# ADJUST-NEXT: adjust-vma.s:7:0
+# ADJUST-EMPTY:
+# ADJUST-NEXT: 0x10b
+# ADJUST-NEXT: foo
+# ADJUST-NEXT: adjust-vma.s:8:0
+
+# OVERFLOW:      0xa
+# OVERFLOW-NEXT: ??
+# OVERFLOW-NEXT: ??
diff --git a/test/tools/llvm-symbolizer/basenames.s b/test/tools/llvm-symbolizer/basenames.s
new file mode 100644
index 0000000000000000000000000000000000000000..b95817c6522d6d3b876405ee2042b7e4382eafb5
--- /dev/null
+++ b/test/tools/llvm-symbolizer/basenames.s
@@ -0,0 +1,12 @@
+# REQUIRES: x86-registered-target
+
+foo:
+    nop
+
+# RUN: llvm-mc --filetype=obj --triple=x86_64-pc-linux %s -o %t.o -g
+# RUN: llvm-symbolizer 0 --basenames --obj=%t.o | FileCheck %s
+# RUN: llvm-symbolizer 0 -s --obj=%t.o | FileCheck %s
+# RUN: llvm-symbolizer 0 --obj=%t.o | FileCheck %s -DDIR=%p --check-prefix=DEFAULT
+
+# CHECK: {{^}}basenames.s:4
+# DEFAULT: [[DIR]]{{\\|/}}basenames.s:4
diff --git a/test/tools/llvm-symbolizer/basic.s b/test/tools/llvm-symbolizer/basic.s
new file mode 100644
index 0000000000000000000000000000000000000000..c966f985a5034943411bdff3e5242d67b2d0f09c
--- /dev/null
+++ b/test/tools/llvm-symbolizer/basic.s
@@ -0,0 +1,23 @@
+# REQUIRES: x86-registered-target
+
+foo:
+    .space 10
+    nop
+    nop
+
+# RUN: llvm-mc -filetype=obj -triple=x86_64-pc-linux %s -o %t.o -g
+
+# Check input addresses specified on stdin.
+# RUN: echo -e "0xa\n0xb" | llvm-symbolizer --obj=%t.o | FileCheck %s
+# RUN: echo -e "10\n11" | llvm-symbolizer --obj=%t.o | FileCheck %s
+
+# Check input addresses specified on the command-line.
+# RUN: llvm-symbolizer 0xa 0xb --obj=%t.o | FileCheck %s
+# RUN: llvm-symbolizer 10 11 --obj=%t.o | FileCheck %s
+
+# Check --obj aliases --exe, -e
+# RUN: llvm-symbolizer 0xa 0xb --exe=%t.o | FileCheck %s
+# RUN: llvm-symbolizer 0xa 0xb -e=%t.o | FileCheck %s
+
+# CHECK: basic.s:5:0
+# CHECK: basic.s:6:0
diff --git a/test/tools/llvm-symbolizer/coff-dwarf.test b/test/tools/llvm-symbolizer/coff-dwarf.test
index 50763a6bde64f557796d31b6b022192a211a6ad0..790763a2ddf0734ec7bb177c8d96741b1f4ca0d2 100644
--- a/test/tools/llvm-symbolizer/coff-dwarf.test
+++ b/test/tools/llvm-symbolizer/coff-dwarf.test
@@ -1,13 +1,13 @@
-RUN: grep '^ADDR:' %s | sed -s 's/ADDR: //' \
-RUN:     | llvm-symbolizer --inlining --relative-address -obj="%p/Inputs/coff-dwarf.exe" \
+RUN: llvm-symbolizer 0x5009 0x5038 --inlining --relative-address -obj="%p/Inputs/coff-dwarf.exe" \
+RUN:     | FileCheck %s
+RUN: llvm-symbolizer 0x5009 0x5038 --inlines --relative-address -obj="%p/Inputs/coff-dwarf.exe" \
+RUN:     | FileCheck %s
+RUN: llvm-symbolizer 0x5009 0x5038 -i --relative-address -obj="%p/Inputs/coff-dwarf.exe" \
 RUN:     | FileCheck %s
 
 This test relies on UnDecorateSymbolName, which is Windows-only.
 REQUIRES: target-windows, system-windows
 
-ADDR: 0x5009
-ADDR: 0x5038
-
 CHECK: foo(void)
 CHECK: coff-dwarf.cpp:7
 CHECK: bar(void)
diff --git a/test/tools/llvm-symbolizer/coff-exports.test b/test/tools/llvm-symbolizer/coff-exports.test
index dde90fb4273d85ecdf3ceb5776143309be2296e8..8678aae2960e945c5b5c415b18c26ff44c171b58 100644
--- a/test/tools/llvm-symbolizer/coff-exports.test
+++ b/test/tools/llvm-symbolizer/coff-exports.test
@@ -1,16 +1,15 @@
-RUN: grep '^ADDR:' %s | sed -s 's/ADDR: //' \
-RUN: 	 | llvm-symbolizer --inlining --relative-address -obj="%p/Inputs/coff-exports.exe" \
-RUN:	 | FileCheck %s
+RUN: llvm-symbolizer 0x500A 0x5038 0x504B --inlining --relative-address -obj="%p/Inputs/coff-exports.exe" \
+RUN:   | FileCheck %s
+RUN: llvm-symbolizer 0x500A 0x5038 0x504B --inlines --relative-address -obj="%p/Inputs/coff-exports.exe" \
+RUN:   | FileCheck %s
+RUN: llvm-symbolizer 0x500A 0x5038 0x504B -i --relative-address -obj="%p/Inputs/coff-exports.exe" \
+RUN:   | FileCheck %s
 
 This test relies on UnDecorateSymbolName, which is Win32-only.
 REQUIRES: system-windows
 REQUIRES: target-windows
 FIXME: This test depends on host, not target.
 
-ADDR: 0x500A
-ADDR: 0x5038
-ADDR: 0x504B
-
 We get the expected stack trace, except 'foo' appears for the 'bar' frame
 because 'bar' isn't in the export table.
 
diff --git a/test/tools/llvm-symbolizer/demangle.s b/test/tools/llvm-symbolizer/demangle.s
new file mode 100644
index 0000000000000000000000000000000000000000..744e78eb8aa1f52fe96e8ed00e3178aeb5a4d950
--- /dev/null
+++ b/test/tools/llvm-symbolizer/demangle.s
@@ -0,0 +1,29 @@
+# REQUIRES: x86-registered-target
+
+# RUN: llvm-mc -filetype=obj -triple=x86_64-pc-linux %s -o %t.o -g
+
+.type _Z1cv,@function
+_Z1cv:
+    nop
+
+# Check the default is to demangle and that the last of --demangle and
+# --no-demangle wins.
+# RUN: llvm-symbolizer --obj %t.o 0 \
+# RUN:    | FileCheck %s --check-prefix=DEMANGLED_FUNCTION_NAME
+# RUN: llvm-symbolizer --demangle --obj %t.o 0 \
+# RUN:    | FileCheck %s --check-prefix=DEMANGLED_FUNCTION_NAME
+# RUN: llvm-symbolizer -C --obj %t.o 0 \
+# RUN:    | FileCheck %s --check-prefix=DEMANGLED_FUNCTION_NAME
+# RUN: llvm-symbolizer --no-demangle --obj %t.o 0 \
+# RUN:    | FileCheck %s --check-prefix=MANGLED_FUNCTION_NAME
+# RUN: llvm-symbolizer --demangle --no-demangle --obj %t.o 0 \
+# RUN:    | FileCheck %s --check-prefix=MANGLED_FUNCTION_NAME
+# RUN: llvm-symbolizer -C --no-demangle --obj %t.o 0 \
+# RUN:    | FileCheck %s --check-prefix=MANGLED_FUNCTION_NAME
+# RUN: llvm-symbolizer --no-demangle --demangle --obj %t.o 0 \
+# RUN:    | FileCheck %s --check-prefix=DEMANGLED_FUNCTION_NAME
+# RUN: llvm-symbolizer --no-demangle -C --obj %t.o 0 \
+# RUN:    | FileCheck %s --check-prefix=DEMANGLED_FUNCTION_NAME
+
+# MANGLED_FUNCTION_NAME: _Z1cv
+# DEMANGLED_FUNCTION_NAME: c()
diff --git a/test/tools/llvm-symbolizer/dwo.test b/test/tools/llvm-symbolizer/dwo.test
new file mode 100644
index 0000000000000000000000000000000000000000..c2c72f50f86133ff5c20441ccf6a25e3df6f7688
--- /dev/null
+++ b/test/tools/llvm-symbolizer/dwo.test
@@ -0,0 +1,4 @@
+RUN: llvm-symbolizer --obj=%p/Inputs/dwo-test 0x400514 | FileCheck %s
+
+CHECK: main
+CHECK-NEXT: llvm-symbolizer-dwo-test.cc:11
diff --git a/test/tools/llvm-symbolizer/fat.test b/test/tools/llvm-symbolizer/fat.test
index 1ecd1abb356f48ba75b6fddcdd29c4376715a370..4596b4e079dbefcbf22dab30e6c6ea0443a7ee90 100644
--- a/test/tools/llvm-symbolizer/fat.test
+++ b/test/tools/llvm-symbolizer/fat.test
@@ -1,8 +1,8 @@
-RUN: echo 0 | llvm-symbolizer -obj=%p/Inputs/fat.o -default-arch=x86_64 | FileCheck --check-prefix=X86_64 %s
-RUN: echo 0 | llvm-symbolizer -obj=%p/Inputs/fat.o -default-arch=x86_64h | FileCheck --check-prefix=X86_64H %s
-RUN: echo 0 | llvm-symbolizer -obj=%p/Inputs/fat.o -default-arch=armv7 | FileCheck --check-prefix=ARMV7 %s
-RUN: echo 0 | llvm-symbolizer -obj=%p/Inputs/fat.o -default-arch=armv7em | FileCheck --check-prefix=ARMV7EM %s
-RUN: echo 0 | llvm-symbolizer -obj=%p/Inputs/fat.o -default-arch=armv7m | FileCheck --check-prefix=ARMV7M %s
+RUN: llvm-symbolizer 0 -obj=%p/Inputs/fat.o -default-arch=x86_64 | FileCheck --check-prefix=X86_64 %s
+RUN: llvm-symbolizer 0 -obj=%p/Inputs/fat.o -default-arch=x86_64h | FileCheck --check-prefix=X86_64H %s
+RUN: llvm-symbolizer 0 -obj=%p/Inputs/fat.o -default-arch=armv7 | FileCheck --check-prefix=ARMV7 %s
+RUN: llvm-symbolizer 0 -obj=%p/Inputs/fat.o -default-arch=armv7em | FileCheck --check-prefix=ARMV7EM %s
+RUN: llvm-symbolizer 0 -obj=%p/Inputs/fat.o -default-arch=armv7m | FileCheck --check-prefix=ARMV7M %s
 
 X86_64: x86_64_function
 X86_64H: x86_64h_function
diff --git a/test/tools/llvm-symbolizer/fission-ranges.test b/test/tools/llvm-symbolizer/fission-ranges.test
new file mode 100644
index 0000000000000000000000000000000000000000..4d35ed4f29383a437c6e673899ff466a724fda65
--- /dev/null
+++ b/test/tools/llvm-symbolizer/fission-ranges.test
@@ -0,0 +1,4 @@
+RUN: llvm-symbolizer --obj=%p/Inputs/fission-ranges.elf-x86_64 0x720 | FileCheck %s
+
+CHECK: main
+CHECK-NEXT: {{.*}}fission-ranges.cc:6
diff --git a/test/tools/llvm-symbolizer/flag-grouping.test b/test/tools/llvm-symbolizer/flag-grouping.test
new file mode 100644
index 0000000000000000000000000000000000000000..9341eca8bdc64537705c6141d41fa8463167173c
--- /dev/null
+++ b/test/tools/llvm-symbolizer/flag-grouping.test
@@ -0,0 +1,7 @@
+RUN: llvm-symbolizer -apCi -obj=%p/Inputs/addr.exe < %p/Inputs/addr.inp | FileCheck %s
+
+CHECK: some text
+CHECK: 0x40054d: inctwo
+CHECK: (inlined by) inc
+CHECK  (inlined by) main
+CHECK: some text2
diff --git a/test/tools/llvm-symbolizer/functions.s b/test/tools/llvm-symbolizer/functions.s
new file mode 100644
index 0000000000000000000000000000000000000000..93cca51a0c17d630cd6ae26a0157432aad022015
--- /dev/null
+++ b/test/tools/llvm-symbolizer/functions.s
@@ -0,0 +1,109 @@
+# REQUIRES: x86-registered-target
+
+# RUN: llvm-mc -filetype=obj -triple=x86_64-pc-linux %s -o %t.o
+
+# RUN: llvm-symbolizer 0 --obj=%t.o | FileCheck %s --check-prefix=LINKAGE
+# RUN: llvm-symbolizer 0 --functions --obj=%t.o | FileCheck %s --check-prefix=LINKAGE
+# RUN: llvm-symbolizer 0 --functions=linkage --obj=%t.o | FileCheck %s --check-prefix=LINKAGE
+# RUN: llvm-symbolizer 0 --functions=short --obj=%t.o | FileCheck %s --check-prefix=SHORT
+# RUN: llvm-symbolizer 0 --functions=none --obj=%t.o | FileCheck %s --check-prefix=NONE
+
+## Characterise behaviour for no '=' sign. llvm-symbolizer treats the next option as an
+## input address, and just prints it.
+# RUN: llvm-symbolizer 0 --functions none --obj=%t.o | FileCheck %s --check-prefixes=LINKAGE,ERR
+
+# LINKAGE:      {{^}}foo(int){{$}}
+# LINKAGE-NEXT: functions.cpp:2:0
+
+# SHORT:      {{^}}foo{{$}}
+# SHORT-NEXT: functions.cpp:2:0
+
+# NONE-NOT: foo
+# NONE:     functions.cpp:2:0
+
+# ERR: none
+
+# The assembly below is a stripped down version of the output of:
+#   clang -S -g --target=x86_64-pc-linux
+# for the following C++ source:
+#   void foo(int bar) {}
+  .type _Z3fooi,@function
+_Z3fooi:
+.Lfunc_begin0:
+  .file   1 "/llvm-symbolizer/Inputs" "functions.cpp"
+  .loc    1 2 0                 # functions.cpp:2:0
+  nop
+  .loc  1 2 20 prologue_end     # functions.cpp:2:20
+.Lfunc_end0:
+
+  .section    .debug_str,"MS",@progbits,1
+.Linfo_string1:
+  .asciz  "functions.cpp"
+.Linfo_string2:
+  .asciz  "/llvm-symbolizer/Inputs"
+.Linfo_string3:
+  .asciz  "_Z3fooi"
+.Linfo_string4:
+  .asciz  "foo"
+
+  .section  .debug_abbrev,"",@progbits
+  .byte 1                       # Abbreviation Code
+  .byte 17                      # DW_TAG_compile_unit
+  .byte 1                       # DW_CHILDREN_yes
+  .byte 3                       # DW_AT_name
+  .byte 14                      # DW_FORM_strp
+  .byte 16                      # DW_AT_stmt_list
+  .byte 23                      # DW_FORM_sec_offset
+  .byte 27                      # DW_AT_comp_dir
+  .byte 14                      # DW_FORM_strp
+  .byte 17                      # DW_AT_low_pc
+  .byte 1                       # DW_FORM_addr
+  .byte 18                      # DW_AT_high_pc
+  .byte 6                       # DW_FORM_data4
+  .byte 0                       # EOM(1)
+  .byte 0                       # EOM(2)
+  .byte 2                       # Abbreviation Code
+  .byte 46                      # DW_TAG_subprogram
+  .byte 0                       # DW_CHILDREN_no
+  .byte 17                      # DW_AT_low_pc
+  .byte 1                       # DW_FORM_addr
+  .byte 18                      # DW_AT_high_pc
+  .byte 6                       # DW_FORM_data4
+  .byte 110                     # DW_AT_linkage_name
+  .byte 14                      # DW_FORM_strp
+  .byte 3                       # DW_AT_name
+  .byte 14                      # DW_FORM_strp
+  .byte 58                      # DW_AT_decl_file
+  .byte 11                      # DW_FORM_data1
+  .byte 59                      # DW_AT_decl_line
+  .byte 11                      # DW_FORM_data1
+  .byte 0                       # EOM(1)
+  .byte 0                       # EOM(2)
+  .byte 0                       # EOM(3)
+
+  .section  .debug_info,"",@progbits
+.Lcu_begin0:
+  .long .Ldebug_info_end0-.Ldebug_info_start0 # Length of Unit
+.Ldebug_info_start0:
+  .short  4                     # DWARF version number
+  .long .debug_abbrev           # Offset Into Abbrev. Section
+  .byte 8                       # Address Size (in bytes)
+  .byte 1                       # Abbrev [1] 0xb:0x4f DW_TAG_compile_unit
+  .long .Linfo_string1          # DW_AT_name
+  .long .Lline_table_start0     # DW_AT_stmt_list
+  .long .Linfo_string2          # DW_AT_comp_dir
+  .quad .Lfunc_begin0           # DW_AT_low_pc
+  .long .Lfunc_end0-.Lfunc_begin0 # DW_AT_high_pc
+  .byte 2                       # Abbrev [2] 0x2a:0x28 DW_TAG_subprogram
+  .quad .Lfunc_begin0           # DW_AT_low_pc
+  .long .Lfunc_end0-.Lfunc_begin0 # DW_AT_high_pc
+  .long .Linfo_string3          # DW_AT_linkage_name
+  .long .Linfo_string4          # DW_AT_name
+  .byte 1                       # DW_AT_decl_file
+  .byte 2                       # DW_AT_decl_line
+  .byte 0                       # End Of Children Mark
+  .byte 0                       # End Of Children Mark
+.Ldebug_info_end0:
+
+  .section  .debug_line,"",@progbits
+.Lline_table_start0:
diff --git a/test/tools/llvm-symbolizer/invalid-input-address.test b/test/tools/llvm-symbolizer/invalid-input-address.test
new file mode 100644
index 0000000000000000000000000000000000000000..2cd3d8f604c130ec2b07e5cb01eb78284f5cd64e
--- /dev/null
+++ b/test/tools/llvm-symbolizer/invalid-input-address.test
@@ -0,0 +1,7 @@
+# Use address that can't fit in a 64-bit number. Show that llvm-symbolizer
+# simply echoes it as per other malformed input addresses.
+RUN: llvm-symbolizer --obj=addr.exe 0x10000000000000000 | FileCheck %s
+
+CHECK-NOT: {{.}}
+CHECK:     0x10000000000000000
+CHECK-NOT: {{.}}
diff --git a/test/tools/llvm-symbolizer/padding-x86_64.ll b/test/tools/llvm-symbolizer/padding-x86_64.ll
index 1d3d6d8b689477cb445319fbb2529bcd1c0629e5..7b2cff8b276d30f8c73978405feede740616abbd 100644
--- a/test/tools/llvm-symbolizer/padding-x86_64.ll
+++ b/test/tools/llvm-symbolizer/padding-x86_64.ll
@@ -2,9 +2,9 @@
 ; Checks if symbolizer can correctly symbolize address in the padding between
 ; functions.
 ; RUN: llc  -o %t.o -filetype=obj -mtriple=x86_64-pc-linux  %s
-; RUN: echo 0x5 | llvm-symbolizer -obj=%t.o | FileCheck %s --check-prefix=FOO
-; RUN: echo 0xd | llvm-symbolizer -obj=%t.o | FileCheck %s --check-prefix=PADDING
-; RUN: echo 0x10 | llvm-symbolizer -obj=%t.o | FileCheck %s --check-prefix=MAIN
+; RUN: llvm-symbolizer 0x5 -obj=%t.o | FileCheck %s --check-prefix=FOO
+; RUN: llvm-symbolizer 0xd -obj=%t.o | FileCheck %s --check-prefix=PADDING
+; RUN: llvm-symbolizer 0x10 -obj=%t.o | FileCheck %s --check-prefix=MAIN
 
 ;FOO: foo
 ;PADDING: ??
diff --git a/test/tools/llvm-symbolizer/pdb/missing_pdb.test b/test/tools/llvm-symbolizer/pdb/missing_pdb.test
index e5282f30fa6f82de445783cfd22ca836120defb7..b83b46a752133c51bd7b192054062be70a48e6e2 100644
--- a/test/tools/llvm-symbolizer/pdb/missing_pdb.test
+++ b/test/tools/llvm-symbolizer/pdb/missing_pdb.test
@@ -1,11 +1,7 @@
-RUN: grep '^ADDR:' %s | sed -s 's/ADDR: //' \
-RUN:    | llvm-symbolizer -obj="%p/Inputs/missing_pdb.exe" 2>%t.err \
+RUN: llvm-symbolizer 0x401000 0x401001 -obj="%p/Inputs/missing_pdb.exe" 2>%t.err \
 RUN:    | FileCheck %s
 RUN: FileCheck --check-prefix=ERROR %s < %t.err
 
-ADDR: 0x401000
-ADDR: 0x401001
-
 llvm-symbolizer should print one error and two unknown line info records.
 
 ERROR: LLVMSymbolizer: error reading file: {{.*}}: {{N|n}}o such file or directory
diff --git a/test/tools/llvm-symbolizer/pdb/pdb.test b/test/tools/llvm-symbolizer/pdb/pdb.test
index a97b35eab9c405b8556663c2f2bd82c3ec416d28..062c0409895f832b283c9d93e060fb72fabffbf1 100644
--- a/test/tools/llvm-symbolizer/pdb/pdb.test
+++ b/test/tools/llvm-symbolizer/pdb/pdb.test
@@ -1,27 +1,23 @@
-RUN: grep '^ADDR:' %s | sed -s 's/ADDR: //' \
-RUN: 	| llvm-symbolizer -obj="%p/Inputs/test.exe" \
+RUN: echo 0x401380 > %t.input
+RUN: echo 0x401390 >> %t.input
+RUN: echo 0x4013A0 >> %t.input
+RUN: echo 0x4013C0 >> %t.input
+RUN: echo 0x4013D0 >> %t.input
+RUN: echo 0x4013E0 >> %t.input
+RUN: echo 0x4013F0 >> %t.input
+RUN: echo 0x401420 >> %t.input
+RUN: llvm-symbolizer -obj="%p/Inputs/test.exe" < %t.input \
 RUN:    | FileCheck %s
-RUN: grep '^ADDR:' %s | sed -s 's/ADDR: //' \
-RUN: 	| llvm-symbolizer -obj="%p/Inputs/test.exe" -demangle=false \
-RUN: 	| FileCheck %s --check-prefix=CHECK-NO-DEMANGLE
+RUN: llvm-symbolizer -obj="%p/Inputs/test.exe" -demangle=false < %t.input \
+RUN:    | FileCheck %s --check-prefix=CHECK-NO-DEMANGLE
 
 Subtract ImageBase from all the offsets and run the test again with
 --relative-address.
 
-RUN: grep '^ADDR:' %s | sed -s 's/ADDR: //' \
-RUN: 	| %python -c 'import sys;print("\n".join([hex(int(x, 16) - 0x400000) for x in sys.stdin]))' \
-RUN:	| llvm-symbolizer -obj="%p/Inputs/test.exe" -demangle=false --relative-address \
+RUN: %python -c 'import sys;print("\n".join([hex(int(x, 16) - 0x400000) for x in sys.stdin]))' < %t.input \
+RUN:    | llvm-symbolizer -obj="%p/Inputs/test.exe" -demangle=false --relative-address \
 RUN:    | FileCheck %s --check-prefix=CHECK-NO-DEMANGLE
 
-ADDR: 0x401380
-ADDR: 0x401390
-ADDR: 0x4013A0
-ADDR: 0x4013C0
-ADDR: 0x4013D0
-ADDR: 0x4013E0
-ADDR: 0x4013F0
-ADDR: 0x401420
-
 CHECK: foo(void)
 CHECK-NEXT: test.cpp:10
 CHECK: {{^private_symbol$}}
diff --git a/test/tools/llvm-symbolizer/ppc64.test b/test/tools/llvm-symbolizer/ppc64.test
index fc8e4ffc7e2e92d5c7ddf3d5e1781ebeecc6f1d2..681557cd2e54d7d9941f80f5a6edff10ab693f8d 100644
--- a/test/tools/llvm-symbolizer/ppc64.test
+++ b/test/tools/llvm-symbolizer/ppc64.test
@@ -4,7 +4,7 @@ int foo() { return 0; }
 int bar() { return foo(); }
 int _start() { return bar(); }
 
-RUN: %python -c "print('0x1000014c\n0x1000018c\n0x100001cc')" | llvm-symbolizer -obj=%p/Inputs/ppc64 | FileCheck %s
+RUN: llvm-symbolizer 0x1000014c 0x1000018c 0x100001cc -obj=%p/Inputs/ppc64 | FileCheck %s
 
 CHECK: foo
 CHECK: bar
diff --git a/test/tools/llvm-symbolizer/print_context.c b/test/tools/llvm-symbolizer/print_context.c
index fe8747c8dfdb6c5c0d3d0c463b13519c374280ee..5b80054416d688d557ba5f75d51f8198f99466f0 100644
--- a/test/tools/llvm-symbolizer/print_context.c
+++ b/test/tools/llvm-symbolizer/print_context.c
@@ -13,7 +13,7 @@ int main() {
 // RUN: cp %s %t/
 // RUN: cp %p/Inputs/print_context.o %t
 // RUN: cd %t
-// RUN: echo "%t/print_context.o 0x0" | llvm-symbolizer -print-source-context-lines=5 | FileCheck %s
+// RUN: llvm-symbolizer -obj=%t/print_context.o 0x0 -print-source-context-lines=5 | FileCheck %s
 
 // Inputs/print_context.o built with plain -g -c from this source file
 // Specifying -Xclang -fdebug-compilation-dir -Xclang . to make the debug info
diff --git a/test/tools/llvm-symbolizer/space-in-path.s b/test/tools/llvm-symbolizer/space-in-path.s
new file mode 100644
index 0000000000000000000000000000000000000000..b54f0aa1fe05b227e2628e1d0d25b72248b19e9d
--- /dev/null
+++ b/test/tools/llvm-symbolizer/space-in-path.s
@@ -0,0 +1,15 @@
+# REQUIRES: x86-registered-target
+
+.type foo,@function
+foo:
+    nop
+
+# RUN: llvm-mc -filetype=obj -triple=x86_64-pc-linux %s -o "%t space.o" -g
+
+# Test both passing via stdin and via --obj.
+# RUN: echo "\"%t space.o\" 0" > %t.input
+# RUN: llvm-symbolizer < %t.input | FileCheck %s
+# RUN: llvm-symbolizer --obj="%t space.o" 0 | FileCheck %s
+
+# CHECK: foo
+# CHECK-NEXT: space-in-path.s:5
diff --git a/test/tools/llvm-symbolizer/split-debug.test b/test/tools/llvm-symbolizer/split-debug.test
index 7474cd6e6f248cd7731e91df7f9b027127417406..8904683228df66acfc20ebc395b867c2a94a808a 100644
--- a/test/tools/llvm-symbolizer/split-debug.test
+++ b/test/tools/llvm-symbolizer/split-debug.test
@@ -20,7 +20,7 @@
 RUN: mkdir -p %t/.debug
 RUN: llvm-objcopy --keep-section=.debug_info %p/Inputs/addr.exe %t/.debug/addr
 RUN: llvm-objcopy --strip-debug --add-gnu-debuglink=%t/.debug/addr %p/Inputs/addr.exe %t/addr.exe
-RUN: llvm-symbolizer -print-address -obj=%t/addr.exe < %p/Inputs/addr.inp | FileCheck %s
+RUN: llvm-symbolizer -print-address -obj=%t/addr.exe 0x40054d | FileCheck %s
 
 #CHECK: 0x40054d
 #CHECK: {{[/\]+}}tmp{{[/\]+}}x.c:14:0
diff --git a/test/tools/llvm-symbolizer/split-dwarf-addr-object-relocation.test b/test/tools/llvm-symbolizer/split-dwarf-addr-object-relocation.test
new file mode 100644
index 0000000000000000000000000000000000000000..d98756dd7964b372fde7d8062509b037ecffe7a7
--- /dev/null
+++ b/test/tools/llvm-symbolizer/split-dwarf-addr-object-relocation.test
@@ -0,0 +1,11 @@
+RUN: rm -rf %t
+RUN: mkdir -p %t
+RUN: cp %p/Inputs/split-dwarf-addr-object-relocation.dwo %t
+
+RUN: cd %t
+RUN: llvm-symbolizer --obj=%p/Inputs/split-dwarf-addr-object-relocation.o 0x14 | FileCheck %s
+
+CHECK:      f2
+CHECK-NEXT: split-dwarf-addr-object-relocation.cpp:3:3
+CHECK-NEXT: f3
+CHECK-NEXT: split-dwarf-addr-object-relocation.cpp:6:0
diff --git a/test/tools/llvm-symbolizer/split-dwarf-dwp.test b/test/tools/llvm-symbolizer/split-dwarf-dwp.test
new file mode 100644
index 0000000000000000000000000000000000000000..71ecca436ebefc87f5576b6ac6f99fba58ceb68e
--- /dev/null
+++ b/test/tools/llvm-symbolizer/split-dwarf-dwp.test
@@ -0,0 +1,12 @@
+RUN: rm -rf %t
+RUN: mkdir -p %t
+
+RUN: cp %p/Inputs/split-dwarf-dwp.o %t/split-dwarf-dwp-different-name.o
+
+RUN: llvm-symbolizer --dwp=%p/Inputs/split-dwarf-dwp.o.dwp \
+RUN:     --obj=%t/split-dwarf-dwp-different-name.o 0x54 | FileCheck %s
+
+CHECK:      f2
+CHECK-NEXT: split-dwarf-dwp.cpp:3:3
+CHECK-NEXT: main
+CHECK-NEXT: split-dwarf-dwp.cpp:10:0
diff --git a/test/tools/llvm-symbolizer/split-dwarf-multiple-cu.test b/test/tools/llvm-symbolizer/split-dwarf-multiple-cu.test
new file mode 100644
index 0000000000000000000000000000000000000000..465816f69589ee2f85da48163c72653604b1d5e0
--- /dev/null
+++ b/test/tools/llvm-symbolizer/split-dwarf-multiple-cu.test
@@ -0,0 +1,12 @@
+RUN: rm -rf %t
+RUN: mkdir -p %t
+RUN: cp %p/Inputs/split-dwarf-multiple-cu.dwo %t
+
+RUN: cd %t
+RUN: llvm-symbolizer --obj=%p/Inputs/split-dwarf-multiple-cu.o 0x4 | FileCheck %s
+
+CHECK:      f2
+CHECK-NEXT: split-dwarf-multiple-cu.cpp:3:3
+CHECK-NEXT: f3
+CHECK-NEXT: split-dwarf-multiple-cu.cpp:6:0
+
diff --git a/test/tools/llvm-symbolizer/split-dwarf.test b/test/tools/llvm-symbolizer/split-dwarf.test
new file mode 100644
index 0000000000000000000000000000000000000000..e8185cee205cabe4fef19bc404b0cfe8ee7514cd
--- /dev/null
+++ b/test/tools/llvm-symbolizer/split-dwarf.test
@@ -0,0 +1,32 @@
+RUN: rm -rf %t
+RUN: mkdir -p %t
+
+RUN: cp %p/Inputs/split-dwarf-test.dwo %t
+
+RUN: cd %t
+RUN: llvm-symbolizer --functions=linkage --inlining --demangle=false \
+RUN:    --obj=%p/Inputs/split-dwarf-test 0x400504 0x4004f4 | FileCheck --check-prefix=CHECK --check-prefix=SPLIT --check-prefix=DWO %s
+
+Ensure we get the same results in the absence of gmlt-like data in the executable but the presence of a .dwo file
+
+RUN: echo "%p/Inputs/split-dwarf-test-nogmlt 0x400504" >> %t.input
+RUN: echo "%p/Inputs/split-dwarf-test-nogmlt 0x4004f4" >> %t.input
+RUN: llvm-symbolizer --functions=linkage --inlining --demangle=false \
+RUN:    --default-arch=i386 --obj=%p/Inputs/split-dwarf-test-nogmlt 0x400504 0x4004f4 | FileCheck --check-prefix=SPLIT --check-prefix=DWO %s
+
+Ensure we get gmlt like results in the absence of a .dwo file but the presence of gmlt-like data in the executable
+
+RUN: rm %t/split-dwarf-test.dwo
+RUN: echo "%p/Inputs/split-dwarf-test 0x400504" >> %t.input
+RUN: echo "%p/Inputs/split-dwarf-test 0x4004f4" >> %t.input
+RUN: llvm-symbolizer --functions=linkage --inlining --demangle=false \
+RUN:    --default-arch=i386 --obj=%p/Inputs/split-dwarf-test 0x400504 0x4004f4 | FileCheck --check-prefix=SPLIT --check-prefix=NODWO %s
+
+DWO: _Z2f2v
+NODWO: {{^f2$}}
+SPLIT-NEXT: {{.*}}split-dwarf-test.cpp
+SPLIT-NEXT: main
+SPLIT-NEXT: {{.*}}split-dwarf-test.cpp
+
+SPLIT: _Z2f1v
+SPLIT-NEXT: {{.*}}split-dwarf-test.cpp
diff --git a/test/tools/llvm-symbolizer/sym.test b/test/tools/llvm-symbolizer/sym.test
index 871c16a494c3b03d6a7a9189734d7faf934dc7eb..50c710ad179ba47c39a061c30f829020cef2d1d8 100644
--- a/test/tools/llvm-symbolizer/sym.test
+++ b/test/tools/llvm-symbolizer/sym.test
@@ -18,9 +18,16 @@
 #Build as : clang -g -O2 addr.c
 
 RUN: llvm-symbolizer -print-address -obj=%p/Inputs/addr.exe < %p/Inputs/addr.inp | FileCheck %s
-RUN: llvm-symbolizer -inlining -print-address -pretty-print -obj=%p/Inputs/addr.exe < %p/Inputs/addr.inp | FileCheck --check-prefix="PRETTY" %s 
+RUN: llvm-symbolizer -addresses -obj=%p/Inputs/addr.exe < %p/Inputs/addr.inp | FileCheck %s
+RUN: llvm-symbolizer -a -obj=%p/Inputs/addr.exe < %p/Inputs/addr.inp | FileCheck %s
+RUN: llvm-symbolizer -inlining -print-address -pretty-print -obj=%p/Inputs/addr.exe < %p/Inputs/addr.inp | FileCheck -check-prefix="PRETTY" %s 
+RUN: llvm-symbolizer -inlining -print-address -p -obj=%p/Inputs/addr.exe < %p/Inputs/addr.inp | FileCheck -check-prefix="PRETTY" %s
+RUN: llvm-symbolizer -inlines -print-address -pretty-print -obj=%p/Inputs/addr.exe < %p/Inputs/addr.inp | FileCheck -check-prefix="PRETTY" %s
+RUN: llvm-symbolizer -inlines -print-address -p -obj=%p/Inputs/addr.exe < %p/Inputs/addr.inp | FileCheck -check-prefix="PRETTY" %s
+RUN: llvm-symbolizer -i -print-address -pretty-print -obj=%p/Inputs/addr.exe < %p/Inputs/addr.inp | FileCheck -check-prefix="PRETTY" %s
+RUN: llvm-symbolizer -i -print-address -p -obj=%p/Inputs/addr.exe < %p/Inputs/addr.inp | FileCheck -check-prefix="PRETTY" %s
 RUN: echo "0x1" > %t.input
-RUN: llvm-symbolizer -obj=%p/Inputs/zero < %t.input | FileCheck --check-prefix="ZERO" %s
+RUN: llvm-symbolizer -obj=%p/Inputs/zero < %t.input | FileCheck -check-prefix="ZERO" %s
 
 #CHECK: some text
 #CHECK: 0x40054d
diff --git a/test/tools/lto/no-bitcode.s b/test/tools/lto/no-bitcode.s
index 2672d2fa1284b538762665c81fcec290381e1189..8fb8e3b64585111aa1fe070324801bcc0ee0292b 100644
--- a/test/tools/lto/no-bitcode.s
+++ b/test/tools/lto/no-bitcode.s
@@ -1,4 +1,4 @@
-; REQUIRES: arm-registered-target
+; REQUIRES: aarch64-registered-target
 ; libLTO.dylib shouldn't assert on invalid inputs.
 ; RUN: llvm-mc -triple=arm64-apple-ios7.0.0 -filetype=obj -o %t.o
 ; RUN: llvm-ar r %t.a %t.o
diff --git a/test/tools/obj2yaml/dynamic-section.test b/test/tools/obj2yaml/dynamic-section.test
new file mode 100644
index 0000000000000000000000000000000000000000..d8fb7e9928c4dc8280bf24736dde34bf7e1cef9a
--- /dev/null
+++ b/test/tools/obj2yaml/dynamic-section.test
@@ -0,0 +1,248 @@
+# RUN: yaml2obj %s -o %t
+# RUN: obj2yaml %t | FileCheck %s
+
+## Check we can use obj2yaml to yamalize the object containing
+## .dynamic section. Check that resulting section has the
+## proper attributes and dynamic tags.
+
+# CHECK:       Sections:
+# CHECK-NEXT:  - Name:            .dynamic
+# CHECK-NEXT:    Type:            SHT_DYNAMIC
+# CHECK-NEXT:    Address:         0x0000000000001000
+# CHECK-NEXT:    AddressAlign:    0x0000000000002000
+# CHECK-NEXT:    EntSize:         0x0000000000000010
+# CHECK-NEXT:    Entries:
+# CHECK-NEXT:      - Tag:             DT_NULL
+# CHECK-NEXT:        Value:           0x0000000000000000
+# CHECK-NEXT:      - Tag:             DT_NEEDED
+# CHECK-NEXT:        Value:           0x0000000000000001
+# CHECK-NEXT:      - Tag:             DT_PLTRELSZ
+# CHECK-NEXT:        Value:           0x0000000000000002
+# CHECK-NEXT:      - Tag:             DT_PLTGOT
+# CHECK-NEXT:        Value:           0x0000000000000003
+# CHECK-NEXT:      - Tag:             DT_HASH
+# CHECK-NEXT:        Value:           0x0000000000000004
+# CHECK-NEXT:      - Tag:             DT_STRTAB
+# CHECK-NEXT:        Value:           0x0000000000000005
+# CHECK-NEXT:      - Tag:             DT_SYMTAB
+# CHECK-NEXT:        Value:           0x0000000000000006
+# CHECK-NEXT:      - Tag:             DT_RELA
+# CHECK-NEXT:        Value:           0x0000000000000007
+# CHECK-NEXT:      - Tag:             DT_RELASZ
+# CHECK-NEXT:        Value:           0x0000000000000008
+# CHECK-NEXT:      - Tag:             DT_RELAENT
+# CHECK-NEXT:        Value:           0x0000000000000009
+# CHECK-NEXT:      - Tag:             DT_STRSZ
+# CHECK-NEXT:        Value:           0x000000000000000A
+# CHECK-NEXT:      - Tag:             DT_SYMENT
+# CHECK-NEXT:        Value:           0x000000000000000B
+# CHECK-NEXT:      - Tag:             DT_INIT
+# CHECK-NEXT:        Value:           0x000000000000000C
+# CHECK-NEXT:      - Tag:             DT_FINI
+# CHECK-NEXT:        Value:           0x000000000000000D
+# CHECK-NEXT:      - Tag:             DT_SONAME
+# CHECK-NEXT:        Value:           0x000000000000000E
+# CHECK-NEXT:      - Tag:             DT_RPATH
+# CHECK-NEXT:        Value:           0x000000000000000F
+# CHECK-NEXT:      - Tag:             DT_SYMBOLIC
+# CHECK-NEXT:        Value:           0x0000000000000010
+# CHECK-NEXT:      - Tag:             DT_REL
+# CHECK-NEXT:        Value:           0x0000000000000011
+# CHECK-NEXT:      - Tag:             DT_RELSZ
+# CHECK-NEXT:        Value:           0x0000000000000012
+# CHECK-NEXT:      - Tag:             DT_RELENT
+# CHECK-NEXT:        Value:           0x0000000000000013
+# CHECK-NEXT:      - Tag:             DT_PLTREL
+# CHECK-NEXT:        Value:           0x0000000000000014
+# CHECK-NEXT:      - Tag:             DT_DEBUG
+# CHECK-NEXT:        Value:           0x0000000000000015
+# CHECK-NEXT:      - Tag:             DT_TEXTREL
+# CHECK-NEXT:        Value:           0x0000000000000016
+# CHECK-NEXT:      - Tag:             DT_JMPREL
+# CHECK-NEXT:        Value:           0x0000000000000017
+# CHECK-NEXT:      - Tag:             DT_BIND_NOW
+# CHECK-NEXT:        Value:           0x0000000000000018
+# CHECK-NEXT:      - Tag:             DT_INIT_ARRAY
+# CHECK-NEXT:        Value:           0x0000000000000019
+# CHECK-NEXT:      - Tag:             DT_FINI_ARRAY
+# CHECK-NEXT:        Value:           0x000000000000001A
+# CHECK-NEXT:      - Tag:             DT_INIT_ARRAYSZ
+# CHECK-NEXT:        Value:           0x000000000000001B
+# CHECK-NEXT:      - Tag:             DT_FINI_ARRAYSZ
+# CHECK-NEXT:        Value:           0x000000000000001C
+# CHECK-NEXT:      - Tag:             DT_RUNPATH
+# CHECK-NEXT:        Value:           0x000000000000001D
+# CHECK-NEXT:      - Tag:             DT_FLAGS
+# CHECK-NEXT:        Value:           0x000000000000001E
+# CHECK-NEXT:      - Tag:             DT_PREINIT_ARRAY
+# CHECK-NEXT:        Value:           0x000000000000001F
+# CHECK-NEXT:      - Tag:             DT_PREINIT_ARRAYSZ
+# CHECK-NEXT:        Value:           0x0000000000000020
+# CHECK-NEXT:      - Tag:             DT_SYMTAB_SHNDX
+# CHECK-NEXT:        Value:           0x0000000000000021
+# CHECK-NEXT:      - Tag:             DT_RELRSZ
+# CHECK-NEXT:        Value:           0x0000000000000022
+# CHECK-NEXT:      - Tag:             DT_RELR
+# CHECK-NEXT:        Value:           0x0000000000000023
+# CHECK-NEXT:      - Tag:             DT_RELRENT
+# CHECK-NEXT:        Value:           0x0000000000000024
+# CHECK-NEXT:      - Tag:             DT_ANDROID_REL
+# CHECK-NEXT:        Value:           0x0000000000000025
+# CHECK-NEXT:      - Tag:             DT_ANDROID_RELSZ
+# CHECK-NEXT:        Value:           0x0000000000000026
+# CHECK-NEXT:      - Tag:             DT_ANDROID_RELA
+# CHECK-NEXT:        Value:           0x0000000000000027
+# CHECK-NEXT:      - Tag:             DT_ANDROID_RELASZ
+# CHECK-NEXT:        Value:           0x0000000000000028
+# CHECK-NEXT:      - Tag:             DT_ANDROID_RELR
+# CHECK-NEXT:        Value:           0x0000000000000029
+# CHECK-NEXT:      - Tag:             DT_ANDROID_RELRSZ
+# CHECK-NEXT:        Value:           0x000000000000002A
+# CHECK-NEXT:      - Tag:             DT_ANDROID_RELRENT
+# CHECK-NEXT:        Value:           0x000000000000002B
+# CHECK-NEXT:      - Tag:             DT_GNU_HASH
+# CHECK-NEXT:        Value:           0x000000000000002C
+# CHECK-NEXT:      - Tag:             DT_TLSDESC_PLT
+# CHECK-NEXT:        Value:           0x000000000000002D
+# CHECK-NEXT:      - Tag:             DT_TLSDESC_GOT
+# CHECK-NEXT:        Value:           0x000000000000002E
+# CHECK-NEXT:      - Tag:             DT_RELACOUNT
+# CHECK-NEXT:        Value:           0x000000000000002F
+# CHECK-NEXT:      - Tag:             DT_RELCOUNT
+# CHECK-NEXT:        Value:           0x0000000000000030
+# CHECK-NEXT:      - Tag:             DT_FLAGS_1
+# CHECK-NEXT:        Value:           0x0000000000000031
+# CHECK-NEXT:      - Tag:             DT_VERSYM
+# CHECK-NEXT:        Value:           0x0000000000000032
+# CHECK-NEXT:      - Tag:             DT_VERDEF
+# CHECK-NEXT:        Value:           0x0000000000000033
+# CHECK-NEXT:      - Tag:             DT_VERDEFNUM
+# CHECK-NEXT:        Value:           0x0000000000000034
+# CHECK-NEXT:      - Tag:             DT_VERNEED
+# CHECK-NEXT:        Value:           0x0000000000000035
+# CHECK-NEXT:      - Tag:             DT_VERNEEDNUM
+# CHECK-NEXT:        Value:           0x0000000000000036
+
+!ELF
+FileHeader:
+  Class:           ELFCLASS64
+  Data:            ELFDATA2LSB
+  Type:            ET_DYN
+  Machine:         EM_X86_64
+Sections:
+  - Name:            .dynamic
+    Type:            SHT_DYNAMIC
+    Address:         0x0000000000001000
+    AddressAlign:    0x0000000000002000
+    EntSize:         0x0000000000000010
+    Entries:
+      - Tag:             DT_NULL
+        Value:           0x0000000000000000
+      - Tag:             DT_NEEDED
+        Value:           0x0000000000000001
+      - Tag:             DT_PLTRELSZ
+        Value:           0x0000000000000002
+      - Tag:             DT_PLTGOT
+        Value:           0x0000000000000003
+      - Tag:             DT_HASH
+        Value:           0x0000000000000004
+      - Tag:             DT_STRTAB
+        Value:           0x0000000000000005
+      - Tag:             DT_SYMTAB
+        Value:           0x0000000000000006
+      - Tag:             DT_RELA
+        Value:           0x0000000000000007
+      - Tag:             DT_RELASZ
+        Value:           0x0000000000000008
+      - Tag:             DT_RELAENT
+        Value:           0x0000000000000009
+      - Tag:             DT_STRSZ
+        Value:           0x000000000000000A
+      - Tag:             DT_SYMENT
+        Value:           0x000000000000000B
+      - Tag:             DT_INIT
+        Value:           0x000000000000000C
+      - Tag:             DT_FINI
+        Value:           0x000000000000000D
+      - Tag:             DT_SONAME
+        Value:           0x000000000000000E
+      - Tag:             DT_RPATH
+        Value:           0x000000000000000F
+      - Tag:             DT_SYMBOLIC
+        Value:           0x0000000000000010
+      - Tag:             DT_REL
+        Value:           0x0000000000000011
+      - Tag:             DT_RELSZ
+        Value:           0x0000000000000012
+      - Tag:             DT_RELENT
+        Value:           0x0000000000000013
+      - Tag:             DT_PLTREL
+        Value:           0x0000000000000014
+      - Tag:             DT_DEBUG
+        Value:           0x0000000000000015
+      - Tag:             DT_TEXTREL
+        Value:           0x0000000000000016
+      - Tag:             DT_JMPREL
+        Value:           0x0000000000000017
+      - Tag:             DT_BIND_NOW
+        Value:           0x0000000000000018
+      - Tag:             DT_INIT_ARRAY
+        Value:           0x0000000000000019
+      - Tag:             DT_FINI_ARRAY
+        Value:           0x000000000000001A
+      - Tag:             DT_INIT_ARRAYSZ
+        Value:           0x000000000000001B
+      - Tag:             DT_FINI_ARRAYSZ
+        Value:           0x000000000000001C
+      - Tag:             DT_RUNPATH
+        Value:           0x000000000000001D
+      - Tag:             DT_FLAGS
+        Value:           0x000000000000001E
+      - Tag:             DT_PREINIT_ARRAY
+        Value:           0x000000000000001F
+      - Tag:             DT_PREINIT_ARRAYSZ
+        Value:           0x0000000000000020
+      - Tag:             DT_SYMTAB_SHNDX
+        Value:           0x0000000000000021
+      - Tag:             DT_RELRSZ
+        Value:           0x0000000000000022
+      - Tag:             DT_RELR
+        Value:           0x0000000000000023
+      - Tag:             DT_RELRENT
+        Value:           0x0000000000000024
+      - Tag:             DT_ANDROID_REL
+        Value:           0x0000000000000025
+      - Tag:             DT_ANDROID_RELSZ
+        Value:           0x0000000000000026
+      - Tag:             DT_ANDROID_RELA
+        Value:           0x0000000000000027
+      - Tag:             DT_ANDROID_RELASZ
+        Value:           0x0000000000000028
+      - Tag:             DT_ANDROID_RELR
+        Value:           0x0000000000000029
+      - Tag:             DT_ANDROID_RELRSZ
+        Value:           0x000000000000002A
+      - Tag:             DT_ANDROID_RELRENT
+        Value:           0x000000000000002B
+      - Tag:             DT_GNU_HASH
+        Value:           0x000000000000002C
+      - Tag:             DT_TLSDESC_PLT
+        Value:           0x000000000000002D
+      - Tag:             DT_TLSDESC_GOT
+        Value:           0x000000000000002E
+      - Tag:             DT_RELACOUNT
+        Value:           0x000000000000002F
+      - Tag:             DT_RELCOUNT
+        Value:           0x0000000000000030
+      - Tag:             DT_FLAGS_1
+        Value:           0x0000000000000031
+      - Tag:             DT_VERSYM
+        Value:           0x0000000000000032
+      - Tag:             DT_VERDEF
+        Value:           0x0000000000000033
+      - Tag:             DT_VERDEFNUM
+        Value:           0x0000000000000034
+      - Tag:             DT_VERNEED
+        Value:           0x0000000000000035
+      - Tag:             DT_VERNEEDNUM
+        Value:           0x0000000000000036
diff --git a/test/tools/obj2yaml/elf-entsize.yaml b/test/tools/obj2yaml/elf-entsize.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..7d8b6c84558fa9ee2823e1cb6d81aedc1707aa5a
--- /dev/null
+++ b/test/tools/obj2yaml/elf-entsize.yaml
@@ -0,0 +1,21 @@
+# RUN: yaml2obj %s -o %t
+# RUN: obj2yaml %t | FileCheck %s
+
+## Check obj2yaml is able to dump sh_entsize field of a section.
+
+# CHECK:      - Name:            .rodata.cst4
+# CHECK-NEXT:   Type:            SHT_PROGBITS
+# CHECK-NEXT:   Flags:           [ SHF_ALLOC, SHF_MERGE ]
+# CHECK-NEXT:   EntSize:         0x0000000000000004
+
+--- !ELF
+FileHeader:
+  Class:           ELFCLASS64
+  Data:            ELFDATA2LSB
+  Type:            ET_REL
+  Machine:         EM_X86_64
+Sections:
+  - Name:            .rodata.cst4
+    Type:            SHT_PROGBITS
+    Flags:           [ SHF_ALLOC, SHF_MERGE ]
+    EntSize:         0x0000000000000004
diff --git a/test/tools/obj2yaml/symbol-type.yaml b/test/tools/obj2yaml/symbol-type.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..8231990960deef2fbf98a54061cbf0f17aa0c093
--- /dev/null
+++ b/test/tools/obj2yaml/symbol-type.yaml
@@ -0,0 +1,22 @@
+# RUN: yaml2obj %s > %t
+# RUN: obj2yaml %t | FileCheck %s
+
+# CHECK:      Symbols:
+# CHECK-NEXT:   Global:
+# CHECK-NEXT:     - Name: a_known_type
+# CHECK-NEXT:       Type: STT_OBJECT
+# CHECK-NEXT:     - Name: an_unknown_type
+# CHECK-NEXT:       Type: 0x07
+
+!ELF
+FileHeader:
+  Class:           ELFCLASS64
+  Data:            ELFDATA2LSB
+  Type:            ET_REL
+  Machine:         EM_X86_64
+Symbols:
+  Global:
+    - Name: a_known_type
+      Type: STT_OBJECT
+    - Name: an_unknown_type
+      Type: 0x7
diff --git a/test/tools/opt-viewer/Inputs/basic/or.yaml b/test/tools/opt-viewer/Inputs/basic/or.yaml
index 11c5528b43fc1b4484cde42dac3dd466d141567b..80de83455a53643684e409857c332c95ff723c45 100644
--- a/test/tools/opt-viewer/Inputs/basic/or.yaml
+++ b/test/tools/opt-viewer/Inputs/basic/or.yaml
@@ -225,3 +225,14 @@ Args:
   - InterleaveCount: '2'
   - String:          ')'
 ...
+--- !Passed
+Pass:            multiline
+Name:            ShowMultiLine
+DebugLoc:        { File: basic/or.c, Line: 10, Column: 3 }
+Function:        Test
+Args:
+  - String:   |
+      This is a remark with
+      multiple
+        lines
+...
diff --git a/test/tools/opt-viewer/Outputs/basic/basic_or.c.html b/test/tools/opt-viewer/Outputs/basic/basic_or.c.html
index 225ebabdbde3bc263ff1e5534eef79ebdb6bdc3c..2b0baf942b507bfd66dec987b71226e6ad59dfc0 100644
--- a/test/tools/opt-viewer/Outputs/basic/basic_or.c.html
+++ b/test/tools/opt-viewer/Outputs/basic/basic_or.c.html
@@ -4,6 +4,23 @@
 <meta charset="utf-8" />
 <head>
 <link rel='stylesheet' type='text/css' href='style.css'>
+<script type="text/javascript">
+/* Simple helper to show/hide the expanded message of a remark. */
+function toggleExpandedMessage(e) {
+  var FullTextElems = e.parentElement.parentElement.getElementsByClassName("full-info");
+  if (!FullTextElems || FullTextElems.length < 1) {
+      return false;
+  }
+  var FullText = FullTextElems[0];
+  if (FullText.style.display == 'none') {
+    e.innerHTML = '-';
+    FullText.style.display = 'block';
+  } else {
+    e.innerHTML = '+';
+    FullText.style.display = 'none';
+  }
+}
+</script>
 </head>
 <body>
 <div class="centered">
@@ -105,6 +122,20 @@
 <td class="column-entry-yellow">Test</td>
 </tr>
 
+<tr>
+<td></td>
+<td></td>
+<td class="column-entry-green">multiline</td>
+<td><pre style="display:inline">  </pre><span class="column-entry-yellow"><a style="text-decoration: none;" href="" onclick="toggleExpandedMessage(this); return false;">+</a> This is a remark with&nbsp;</span>
+<div class="full-info" style="display:none;">
+  <div class="col-left"><pre style="display:inline">  </pre></div>
+  <div class="expanded col-left"><pre>multiple
+  lines
+</pre></div>
+</div></td>
+<td class="column-entry-yellow">Test</td>
+</tr>
+
 <tr>
 <td><a name="L11">11</a></td>
 <td></td>
diff --git a/test/tools/opt-viewer/Outputs/basic/basic_or.h.html b/test/tools/opt-viewer/Outputs/basic/basic_or.h.html
index 7cbda8a66c76df473fea668215bc3d715cb9b361..b6cc2f37fab317bc916cefe54def03d2ee115368 100644
--- a/test/tools/opt-viewer/Outputs/basic/basic_or.h.html
+++ b/test/tools/opt-viewer/Outputs/basic/basic_or.h.html
@@ -4,6 +4,23 @@
 <meta charset="utf-8" />
 <head>
 <link rel='stylesheet' type='text/css' href='style.css'>
+<script type="text/javascript">
+/* Simple helper to show/hide the expanded message of a remark. */
+function toggleExpandedMessage(e) {
+  var FullTextElems = e.parentElement.parentElement.getElementsByClassName("full-info");
+  if (!FullTextElems || FullTextElems.length < 1) {
+      return false;
+  }
+  var FullText = FullTextElems[0];
+  if (FullText.style.display == 'none') {
+    e.innerHTML = '-';
+    FullText.style.display = 'block';
+  } else {
+    e.innerHTML = '+';
+    FullText.style.display = 'none';
+  }
+}
+</script>
 </head>
 <body>
 <div class="centered">
diff --git a/test/tools/opt-viewer/Outputs/basic/index.html b/test/tools/opt-viewer/Outputs/basic/index.html
index 3424a9a1f0e6a54988c20a6fe8181b19abd3ff9c..9bdacd2641577458d5369bb81258b58e6f95735c 100644
--- a/test/tools/opt-viewer/Outputs/basic/index.html
+++ b/test/tools/opt-viewer/Outputs/basic/index.html
@@ -29,121 +29,128 @@
 </tr>
 
 <tr>
-<td class="column-entry-0"><a href="basic_or.c.html#L14">basic/or.c:14:3</a></td>
+<td class="column-entry-0"><a href="basic_or.c.html#L10">basic/or.c:10:3</a></td>
 <td class="column-entry-0"></td>
 <td class="column-entry-0">Test</td>
-<td class="column-entry-green">loop-unroll</td>
+<td class="column-entry-green">multiline</td>
 </tr>
 
 <tr>
-<td class="column-entry-1"><a href="basic_or.c.html#L18">basic/or.c:18:3</a></td>
+<td class="column-entry-1"><a href="basic_or.c.html#L14">basic/or.c:14:3</a></td>
 <td class="column-entry-1"></td>
 <td class="column-entry-1">Test</td>
-<td class="column-entry-white">inline</td>
+<td class="column-entry-green">loop-unroll</td>
 </tr>
 
 <tr>
 <td class="column-entry-0"><a href="basic_or.c.html#L18">basic/or.c:18:3</a></td>
 <td class="column-entry-0"></td>
 <td class="column-entry-0">Test</td>
-<td class="column-entry-green">inline</td>
+<td class="column-entry-white">inline</td>
 </tr>
 
 <tr>
-<td class="column-entry-1"><a href="basic_or.c.html#L20">basic/or.c:20:3</a></td>
+<td class="column-entry-1"><a href="basic_or.c.html#L18">basic/or.c:18:3</a></td>
 <td class="column-entry-1"></td>
 <td class="column-entry-1">Test</td>
-<td class="column-entry-white">inline</td>
+<td class="column-entry-green">inline</td>
 </tr>
 
 <tr>
 <td class="column-entry-0"><a href="basic_or.c.html#L20">basic/or.c:20:3</a></td>
 <td class="column-entry-0"></td>
 <td class="column-entry-0">Test</td>
-<td class="column-entry-green">inline</td>
+<td class="column-entry-white">inline</td>
 </tr>
 
 <tr>
-<td class="column-entry-1"><a href="basic_or.c.html#L20">basic/or.c:20:10</a></td>
+<td class="column-entry-1"><a href="basic_or.c.html#L20">basic/or.c:20:3</a></td>
 <td class="column-entry-1"></td>
 <td class="column-entry-1">Test</td>
-<td class="column-entry-red">inline</td>
+<td class="column-entry-green">inline</td>
 </tr>
 
 <tr>
-<td class="column-entry-0"><a href="basic_or.c.html#L20">basic/or.c:20:17</a></td>
+<td class="column-entry-0"><a href="basic_or.c.html#L20">basic/or.c:20:10</a></td>
 <td class="column-entry-0"></td>
 <td class="column-entry-0">Test</td>
-<td class="column-entry-white">inline</td>
+<td class="column-entry-red">inline</td>
 </tr>
 
 <tr>
 <td class="column-entry-1"><a href="basic_or.c.html#L20">basic/or.c:20:17</a></td>
 <td class="column-entry-1"></td>
 <td class="column-entry-1">Test</td>
-<td class="column-entry-green">inline</td>
+<td class="column-entry-white">inline</td>
 </tr>
 
 <tr>
-<td class="column-entry-0"><a href="basic_or.h.html#L5">basic/or.h:5:3</a></td>
+<td class="column-entry-0"><a href="basic_or.c.html#L20">basic/or.c:20:17</a></td>
 <td class="column-entry-0"></td>
-<td class="column-entry-0">TestH</td>
-<td class="column-entry-green">loop-vectorize</td>
+<td class="column-entry-0">Test</td>
+<td class="column-entry-green">inline</td>
 </tr>
 
 <tr>
-<td class="column-entry-1"><a href="basic_or.h.html#L9">basic/or.h:9:3</a></td>
+<td class="column-entry-1"><a href="basic_or.h.html#L5">basic/or.h:5:3</a></td>
 <td class="column-entry-1"></td>
 <td class="column-entry-1">TestH</td>
-<td class="column-entry-green">loop-unroll</td>
+<td class="column-entry-green">loop-vectorize</td>
 </tr>
 
 <tr>
-<td class="column-entry-0"><a href="basic_or.h.html#L13">basic/or.h:13:3</a></td>
+<td class="column-entry-0"><a href="basic_or.h.html#L9">basic/or.h:9:3</a></td>
 <td class="column-entry-0"></td>
 <td class="column-entry-0">TestH</td>
-<td class="column-entry-white">inline</td>
+<td class="column-entry-green">loop-unroll</td>
 </tr>
 
 <tr>
 <td class="column-entry-1"><a href="basic_or.h.html#L13">basic/or.h:13:3</a></td>
 <td class="column-entry-1"></td>
 <td class="column-entry-1">TestH</td>
-<td class="column-entry-green">inline</td>
+<td class="column-entry-white">inline</td>
 </tr>
 
 <tr>
-<td class="column-entry-0"><a href="basic_or.h.html#L15">basic/or.h:15:3</a></td>
+<td class="column-entry-0"><a href="basic_or.h.html#L13">basic/or.h:13:3</a></td>
 <td class="column-entry-0"></td>
 <td class="column-entry-0">TestH</td>
-<td class="column-entry-white">inline</td>
+<td class="column-entry-green">inline</td>
 </tr>
 
 <tr>
 <td class="column-entry-1"><a href="basic_or.h.html#L15">basic/or.h:15:3</a></td>
 <td class="column-entry-1"></td>
 <td class="column-entry-1">TestH</td>
-<td class="column-entry-green">inline</td>
+<td class="column-entry-white">inline</td>
 </tr>
 
 <tr>
-<td class="column-entry-0"><a href="basic_or.h.html#L15">basic/or.h:15:10</a></td>
+<td class="column-entry-0"><a href="basic_or.h.html#L15">basic/or.h:15:3</a></td>
 <td class="column-entry-0"></td>
 <td class="column-entry-0">TestH</td>
-<td class="column-entry-red">inline</td>
+<td class="column-entry-green">inline</td>
 </tr>
 
 <tr>
-<td class="column-entry-1"><a href="basic_or.h.html#L15">basic/or.h:15:17</a></td>
+<td class="column-entry-1"><a href="basic_or.h.html#L15">basic/or.h:15:10</a></td>
 <td class="column-entry-1"></td>
 <td class="column-entry-1">TestH</td>
-<td class="column-entry-white">inline</td>
+<td class="column-entry-red">inline</td>
 </tr>
 
 <tr>
 <td class="column-entry-0"><a href="basic_or.h.html#L15">basic/or.h:15:17</a></td>
 <td class="column-entry-0"></td>
 <td class="column-entry-0">TestH</td>
+<td class="column-entry-white">inline</td>
+</tr>
+
+<tr>
+<td class="column-entry-1"><a href="basic_or.h.html#L15">basic/or.h:15:17</a></td>
+<td class="column-entry-1"></td>
+<td class="column-entry-1">TestH</td>
 <td class="column-entry-green">inline</td>
 </tr>
 
diff --git a/test/tools/opt-viewer/Outputs/filter/basic_or.c.html b/test/tools/opt-viewer/Outputs/filter/basic_or.c.html
new file mode 100644
index 0000000000000000000000000000000000000000..005cc2e63cde7edca7ddefddac83cede547f4546
--- /dev/null
+++ b/test/tools/opt-viewer/Outputs/filter/basic_or.c.html
@@ -0,0 +1,260 @@
+
+<html>
+<title>or.c</title>
+<meta charset="utf-8" />
+<head>
+<link rel='stylesheet' type='text/css' href='style.css'>
+<script type="text/javascript">
+/* Simple helper to show/hide the expanded message of a remark. */
+function toggleExpandedMessage(e) {
+  var FullTextElems = e.parentElement.parentElement.getElementsByClassName("full-info");
+  if (!FullTextElems || FullTextElems.length < 1) {
+      return false;
+  }
+  var FullText = FullTextElems[0];
+  if (FullText.style.display == 'none') {
+    e.innerHTML = '-';
+    FullText.style.display = 'block';
+  } else {
+    e.innerHTML = '+';
+    FullText.style.display = 'none';
+  }
+}
+</script>
+</head>
+<body>
+<div class="centered">
+<table class="source">
+<thead>
+<tr>
+<th style="width: 2%">Line</td>
+<th style="width: 3%">Hotness</td>
+<th style="width: 10%">Optimization</td>
+<th style="width: 70%">Source</td>
+<th style="width: 15%">Inline Context</td>
+</tr>
+</thead>
+<tbody>
+
+<tr>
+<td><a name="L1">1</a></td>
+<td></td>
+<td></td>
+<td><div class="highlight"><pre>void bar();</pre></div></td>
+</tr>
+
+<tr>
+<td><a name="L2">2</a></td>
+<td></td>
+<td></td>
+<td><div class="highlight"><pre>void foo() { bar(); }</pre></div></td>
+</tr>
+
+<tr>
+<td></td>
+<td></td>
+<td class="column-entry-red">inline</td>
+<td><pre style="display:inline">             </pre><span class="column-entry-yellow"> bar will not be inlined into foo because its definition is unavailable&nbsp;</span></td>
+<td class="column-entry-yellow">foo</td>
+</tr>
+
+<tr>
+<td><a name="L3">3</a></td>
+<td></td>
+<td></td>
+<td><div class="highlight"><pre></pre></div></td>
+</tr>
+
+<tr>
+<td><a name="L4">4</a></td>
+<td></td>
+<td></td>
+<td><div class="highlight"><pre>#include "or.h"</pre></div></td>
+</tr>
+
+<tr>
+<td><a name="L5">5</a></td>
+<td></td>
+<td></td>
+<td><div class="highlight"><pre></pre></div></td>
+</tr>
+
+<tr>
+<td><a name="L6">6</a></td>
+<td></td>
+<td></td>
+<td><div class="highlight"><pre>void Test(int *res, int *c, int *d, int *p, int n) {</pre></div></td>
+</tr>
+
+<tr>
+<td><a name="L7">7</a></td>
+<td></td>
+<td></td>
+<td><div class="highlight"><pre>  int i;</pre></div></td>
+</tr>
+
+<tr>
+<td><a name="L8">8</a></td>
+<td></td>
+<td></td>
+<td><div class="highlight"><pre></pre></div></td>
+</tr>
+
+<tr>
+<td><a name="L9">9</a></td>
+<td></td>
+<td></td>
+<td><div class="highlight"><pre>#pragma clang loop vectorize(assume_safety)</pre></div></td>
+</tr>
+
+<tr>
+<td><a name="L10">10</a></td>
+<td></td>
+<td></td>
+<td><div class="highlight"><pre>  for (i = 0; i < 1600; i++) {</pre></div></td>
+</tr>
+
+<tr>
+<td><a name="L11">11</a></td>
+<td></td>
+<td></td>
+<td><div class="highlight"><pre>    res[i] = (p[i] == 0) ? res[i] : res[i] + d[i];</pre></div></td>
+</tr>
+
+<tr>
+<td><a name="L12">12</a></td>
+<td></td>
+<td></td>
+<td><div class="highlight"><pre>  }</pre></div></td>
+</tr>
+
+<tr>
+<td><a name="L13">13</a></td>
+<td></td>
+<td></td>
+<td><div class="highlight"><pre></pre></div></td>
+</tr>
+
+<tr>
+<td><a name="L14">14</a></td>
+<td></td>
+<td></td>
+<td><div class="highlight"><pre>  for (i = 0; i < 16; i++) {</pre></div></td>
+</tr>
+
+<tr>
+<td><a name="L15">15</a></td>
+<td></td>
+<td></td>
+<td><div class="highlight"><pre>    res[i] = (p[i] == 0) ? res[i] : res[i] + d[i];</pre></div></td>
+</tr>
+
+<tr>
+<td><a name="L16">16</a></td>
+<td></td>
+<td></td>
+<td><div class="highlight"><pre>  }</pre></div></td>
+</tr>
+
+<tr>
+<td><a name="L17">17</a></td>
+<td></td>
+<td></td>
+<td><div class="highlight"><pre></pre></div></td>
+</tr>
+
+<tr>
+<td><a name="L18">18</a></td>
+<td></td>
+<td></td>
+<td><div class="highlight"><pre>  foo();</pre></div></td>
+</tr>
+
+<tr>
+<td></td>
+<td></td>
+<td class="column-entry-white">inline</td>
+<td><pre style="display:inline">  </pre><span class="column-entry-yellow"> foo can be inlined into Test with cost=30 (threshold=412)&nbsp;</span></td>
+<td class="column-entry-yellow">Test</td>
+</tr>
+
+<tr>
+<td></td>
+<td></td>
+<td class="column-entry-green">inline</td>
+<td><pre style="display:inline">  </pre><span class="column-entry-yellow"> foo inlined into Test&nbsp;</span></td>
+<td class="column-entry-yellow">Test</td>
+</tr>
+
+<tr>
+<td><a name="L19">19</a></td>
+<td></td>
+<td></td>
+<td><div class="highlight"><pre></pre></div></td>
+</tr>
+
+<tr>
+<td><a name="L20">20</a></td>
+<td></td>
+<td></td>
+<td><div class="highlight"><pre>  foo(); bar(); foo();</pre></div></td>
+</tr>
+
+<tr>
+<td></td>
+<td></td>
+<td class="column-entry-red">inline</td>
+<td><pre style="display:inline">         </pre><span class="column-entry-yellow"> bar will not be inlined into Test because its definition is unavailable&nbsp;</span></td>
+<td class="column-entry-yellow">Test</td>
+</tr>
+
+<tr>
+<td></td>
+<td></td>
+<td class="column-entry-white">inline</td>
+<td><pre style="display:inline">  </pre><span class="column-entry-yellow"> foo can be inlined into Test with cost=30 (threshold=412)&nbsp;</span></td>
+<td class="column-entry-yellow">Test</td>
+</tr>
+
+<tr>
+<td></td>
+<td></td>
+<td class="column-entry-green">inline</td>
+<td><pre style="display:inline">  </pre><span class="column-entry-yellow"> foo inlined into Test&nbsp;</span></td>
+<td class="column-entry-yellow">Test</td>
+</tr>
+
+<tr>
+<td></td>
+<td></td>
+<td class="column-entry-white">inline</td>
+<td><pre style="display:inline">                </pre><span class="column-entry-yellow"> foo can be inlined into Test with cost=30 (threshold=412)&nbsp;</span></td>
+<td class="column-entry-yellow">Test</td>
+</tr>
+
+<tr>
+<td></td>
+<td></td>
+<td class="column-entry-green">inline</td>
+<td><pre style="display:inline">                </pre><span class="column-entry-yellow"> foo inlined into Test&nbsp;</span></td>
+<td class="column-entry-yellow">Test</td>
+</tr>
+
+<tr>
+<td><a name="L21">21</a></td>
+<td></td>
+<td></td>
+<td><div class="highlight"><pre>}</pre></div></td>
+</tr>
+
+<tr>
+<td><a name="L22">22</a></td>
+<td></td>
+<td></td>
+<td><div class="highlight"><pre></pre></div></td>
+</tr>
+
+</tbody>
+</table>
+</body>
+</html>
diff --git a/test/tools/opt-viewer/Outputs/filter/basic_or.h.html b/test/tools/opt-viewer/Outputs/filter/basic_or.h.html
new file mode 100644
index 0000000000000000000000000000000000000000..19b0f1ed6c4902cbf568364648e74b3564eb97cf
--- /dev/null
+++ b/test/tools/opt-viewer/Outputs/filter/basic_or.h.html
@@ -0,0 +1,217 @@
+
+<html>
+<title>or.h</title>
+<meta charset="utf-8" />
+<head>
+<link rel='stylesheet' type='text/css' href='style.css'>
+<script type="text/javascript">
+/* Simple helper to show/hide the expanded message of a remark. */
+function toggleExpandedMessage(e) {
+  var FullTextElems = e.parentElement.parentElement.getElementsByClassName("full-info");
+  if (!FullTextElems || FullTextElems.length < 1) {
+      return false;
+  }
+  var FullText = FullTextElems[0];
+  if (FullText.style.display == 'none') {
+    e.innerHTML = '-';
+    FullText.style.display = 'block';
+  } else {
+    e.innerHTML = '+';
+    FullText.style.display = 'none';
+  }
+}
+</script>
+</head>
+<body>
+<div class="centered">
+<table class="source">
+<thead>
+<tr>
+<th style="width: 2%">Line</td>
+<th style="width: 3%">Hotness</td>
+<th style="width: 10%">Optimization</td>
+<th style="width: 70%">Source</td>
+<th style="width: 15%">Inline Context</td>
+</tr>
+</thead>
+<tbody>
+
+<tr>
+<td><a name="L1">1</a></td>
+<td></td>
+<td></td>
+<td><div class="highlight"><pre>void TestH(int *res, int *c, int *d, int *p, int n) {</pre></div></td>
+</tr>
+
+<tr>
+<td><a name="L2">2</a></td>
+<td></td>
+<td></td>
+<td><div class="highlight"><pre>  int i;</pre></div></td>
+</tr>
+
+<tr>
+<td><a name="L3">3</a></td>
+<td></td>
+<td></td>
+<td><div class="highlight"><pre></pre></div></td>
+</tr>
+
+<tr>
+<td><a name="L4">4</a></td>
+<td></td>
+<td></td>
+<td><div class="highlight"><pre>#pragma clang loop vectorize(assume_safety)</pre></div></td>
+</tr>
+
+<tr>
+<td><a name="L5">5</a></td>
+<td></td>
+<td></td>
+<td><div class="highlight"><pre>  for (i = 0; i < 1600; i++) {</pre></div></td>
+</tr>
+
+<tr>
+<td><a name="L6">6</a></td>
+<td></td>
+<td></td>
+<td><div class="highlight"><pre>    res[i] = (p[i] == 0) ? res[i] : res[i] + d[i];</pre></div></td>
+</tr>
+
+<tr>
+<td><a name="L7">7</a></td>
+<td></td>
+<td></td>
+<td><div class="highlight"><pre>  }</pre></div></td>
+</tr>
+
+<tr>
+<td><a name="L8">8</a></td>
+<td></td>
+<td></td>
+<td><div class="highlight"><pre></pre></div></td>
+</tr>
+
+<tr>
+<td><a name="L9">9</a></td>
+<td></td>
+<td></td>
+<td><div class="highlight"><pre>  for (i = 0; i < 16; i++) {</pre></div></td>
+</tr>
+
+<tr>
+<td><a name="L10">10</a></td>
+<td></td>
+<td></td>
+<td><div class="highlight"><pre>    res[i] = (p[i] == 0) ? res[i] : res[i] + d[i];</pre></div></td>
+</tr>
+
+<tr>
+<td><a name="L11">11</a></td>
+<td></td>
+<td></td>
+<td><div class="highlight"><pre>  }</pre></div></td>
+</tr>
+
+<tr>
+<td><a name="L12">12</a></td>
+<td></td>
+<td></td>
+<td><div class="highlight"><pre></pre></div></td>
+</tr>
+
+<tr>
+<td><a name="L13">13</a></td>
+<td></td>
+<td></td>
+<td><div class="highlight"><pre>  foo();</pre></div></td>
+</tr>
+
+<tr>
+<td></td>
+<td></td>
+<td class="column-entry-white">inline</td>
+<td><pre style="display:inline">  </pre><span class="column-entry-yellow"> foo can be inlined into TestH with cost=30 (threshold=412)&nbsp;</span></td>
+<td class="column-entry-yellow">TestH</td>
+</tr>
+
+<tr>
+<td></td>
+<td></td>
+<td class="column-entry-green">inline</td>
+<td><pre style="display:inline">  </pre><span class="column-entry-yellow"> foo inlined into TestH&nbsp;</span></td>
+<td class="column-entry-yellow">TestH</td>
+</tr>
+
+<tr>
+<td><a name="L14">14</a></td>
+<td></td>
+<td></td>
+<td><div class="highlight"><pre></pre></div></td>
+</tr>
+
+<tr>
+<td><a name="L15">15</a></td>
+<td></td>
+<td></td>
+<td><div class="highlight"><pre>  foo(); bar(); foo();</pre></div></td>
+</tr>
+
+<tr>
+<td></td>
+<td></td>
+<td class="column-entry-red">inline</td>
+<td><pre style="display:inline">         </pre><span class="column-entry-yellow"> bar will not be inlined into TestH because its definition is unavailable&nbsp;</span></td>
+<td class="column-entry-yellow">TestH</td>
+</tr>
+
+<tr>
+<td></td>
+<td></td>
+<td class="column-entry-white">inline</td>
+<td><pre style="display:inline">  </pre><span class="column-entry-yellow"> foo can be inlined into TestH with cost=30 (threshold=412)&nbsp;</span></td>
+<td class="column-entry-yellow">TestH</td>
+</tr>
+
+<tr>
+<td></td>
+<td></td>
+<td class="column-entry-green">inline</td>
+<td><pre style="display:inline">  </pre><span class="column-entry-yellow"> foo inlined into TestH&nbsp;</span></td>
+<td class="column-entry-yellow">TestH</td>
+</tr>
+
+<tr>
+<td></td>
+<td></td>
+<td class="column-entry-white">inline</td>
+<td><pre style="display:inline">                </pre><span class="column-entry-yellow"> foo can be inlined into TestH with cost=30 (threshold=412)&nbsp;</span></td>
+<td class="column-entry-yellow">TestH</td>
+</tr>
+
+<tr>
+<td></td>
+<td></td>
+<td class="column-entry-green">inline</td>
+<td><pre style="display:inline">                </pre><span class="column-entry-yellow"> foo inlined into TestH&nbsp;</span></td>
+<td class="column-entry-yellow">TestH</td>
+</tr>
+
+<tr>
+<td><a name="L16">16</a></td>
+<td></td>
+<td></td>
+<td><div class="highlight"><pre>}</pre></div></td>
+</tr>
+
+<tr>
+<td><a name="L17">17</a></td>
+<td></td>
+<td></td>
+<td><div class="highlight"><pre></pre></div></td>
+</tr>
+
+</tbody>
+</table>
+</body>
+</html>
diff --git a/test/tools/opt-viewer/Outputs/filter/index.html b/test/tools/opt-viewer/Outputs/filter/index.html
new file mode 100644
index 0000000000000000000000000000000000000000..57aa9c8629832c38c8b0f6a8090a097bc91d35e1
--- /dev/null
+++ b/test/tools/opt-viewer/Outputs/filter/index.html
@@ -0,0 +1,124 @@
+
+<html>
+<meta charset="utf-8" />
+<head>
+<link rel='stylesheet' type='text/css' href='style.css'>
+</head>
+<body>
+<div class="centered">
+<table>
+<tr>
+<td>Source Location</td>
+<td>Hotness</td>
+<td>Function</td>
+<td>Pass</td>
+</tr>
+
+<tr>
+<td class="column-entry-0"><a href="basic_or.c.html#L2">basic/or.c:2:14</a></td>
+<td class="column-entry-0"></td>
+<td class="column-entry-0">foo</td>
+<td class="column-entry-red">inline</td>
+</tr>
+
+<tr>
+<td class="column-entry-1"><a href="basic_or.c.html#L18">basic/or.c:18:3</a></td>
+<td class="column-entry-1"></td>
+<td class="column-entry-1">Test</td>
+<td class="column-entry-white">inline</td>
+</tr>
+
+<tr>
+<td class="column-entry-0"><a href="basic_or.c.html#L18">basic/or.c:18:3</a></td>
+<td class="column-entry-0"></td>
+<td class="column-entry-0">Test</td>
+<td class="column-entry-green">inline</td>
+</tr>
+
+<tr>
+<td class="column-entry-1"><a href="basic_or.c.html#L20">basic/or.c:20:3</a></td>
+<td class="column-entry-1"></td>
+<td class="column-entry-1">Test</td>
+<td class="column-entry-white">inline</td>
+</tr>
+
+<tr>
+<td class="column-entry-0"><a href="basic_or.c.html#L20">basic/or.c:20:3</a></td>
+<td class="column-entry-0"></td>
+<td class="column-entry-0">Test</td>
+<td class="column-entry-green">inline</td>
+</tr>
+
+<tr>
+<td class="column-entry-1"><a href="basic_or.c.html#L20">basic/or.c:20:10</a></td>
+<td class="column-entry-1"></td>
+<td class="column-entry-1">Test</td>
+<td class="column-entry-red">inline</td>
+</tr>
+
+<tr>
+<td class="column-entry-0"><a href="basic_or.c.html#L20">basic/or.c:20:17</a></td>
+<td class="column-entry-0"></td>
+<td class="column-entry-0">Test</td>
+<td class="column-entry-white">inline</td>
+</tr>
+
+<tr>
+<td class="column-entry-1"><a href="basic_or.c.html#L20">basic/or.c:20:17</a></td>
+<td class="column-entry-1"></td>
+<td class="column-entry-1">Test</td>
+<td class="column-entry-green">inline</td>
+</tr>
+
+<tr>
+<td class="column-entry-0"><a href="basic_or.h.html#L13">basic/or.h:13:3</a></td>
+<td class="column-entry-0"></td>
+<td class="column-entry-0">TestH</td>
+<td class="column-entry-white">inline</td>
+</tr>
+
+<tr>
+<td class="column-entry-1"><a href="basic_or.h.html#L13">basic/or.h:13:3</a></td>
+<td class="column-entry-1"></td>
+<td class="column-entry-1">TestH</td>
+<td class="column-entry-green">inline</td>
+</tr>
+
+<tr>
+<td class="column-entry-0"><a href="basic_or.h.html#L15">basic/or.h:15:3</a></td>
+<td class="column-entry-0"></td>
+<td class="column-entry-0">TestH</td>
+<td class="column-entry-white">inline</td>
+</tr>
+
+<tr>
+<td class="column-entry-1"><a href="basic_or.h.html#L15">basic/or.h:15:3</a></td>
+<td class="column-entry-1"></td>
+<td class="column-entry-1">TestH</td>
+<td class="column-entry-green">inline</td>
+</tr>
+
+<tr>
+<td class="column-entry-0"><a href="basic_or.h.html#L15">basic/or.h:15:10</a></td>
+<td class="column-entry-0"></td>
+<td class="column-entry-0">TestH</td>
+<td class="column-entry-red">inline</td>
+</tr>
+
+<tr>
+<td class="column-entry-1"><a href="basic_or.h.html#L15">basic/or.h:15:17</a></td>
+<td class="column-entry-1"></td>
+<td class="column-entry-1">TestH</td>
+<td class="column-entry-white">inline</td>
+</tr>
+
+<tr>
+<td class="column-entry-0"><a href="basic_or.h.html#L15">basic/or.h:15:17</a></td>
+<td class="column-entry-0"></td>
+<td class="column-entry-0">TestH</td>
+<td class="column-entry-green">inline</td>
+</tr>
+
+</table>
+</body>
+</html>
diff --git a/test/tools/opt-viewer/Outputs/suppress/s.swift.html b/test/tools/opt-viewer/Outputs/suppress/s.swift.html
index 813db9f98205b0fe17c7589402d9de87890ba1d8..f67352079ed6ba6f81849eecb5799575c46c4f56 100644
--- a/test/tools/opt-viewer/Outputs/suppress/s.swift.html
+++ b/test/tools/opt-viewer/Outputs/suppress/s.swift.html
@@ -4,6 +4,23 @@
 <meta charset="utf-8" />
 <head>
 <link rel='stylesheet' type='text/css' href='style.css'>
+<script type="text/javascript">
+/* Simple helper to show/hide the expanded message of a remark. */
+function toggleExpandedMessage(e) {
+  var FullTextElems = e.parentElement.parentElement.getElementsByClassName("full-info");
+  if (!FullTextElems || FullTextElems.length < 1) {
+      return false;
+  }
+  var FullText = FullTextElems[0];
+  if (FullText.style.display == 'none') {
+    e.innerHTML = '-';
+    FullText.style.display = 'block';
+  } else {
+    e.innerHTML = '+';
+    FullText.style.display = 'none';
+  }
+}
+</script>
 </head>
 <body>
 <div class="centered">
diff --git a/test/tools/opt-viewer/Outputs/unicode-function-name/s.swift.html b/test/tools/opt-viewer/Outputs/unicode-function-name/s.swift.html
index 67514d556c389558744b4dc392ac9919d76128ec..744c5b7579e886304bb7ac210c7e6e83542dda7d 100644
--- a/test/tools/opt-viewer/Outputs/unicode-function-name/s.swift.html
+++ b/test/tools/opt-viewer/Outputs/unicode-function-name/s.swift.html
@@ -4,6 +4,23 @@
 <meta charset="utf-8" />
 <head>
 <link rel='stylesheet' type='text/css' href='style.css'>
+<script type="text/javascript">
+/* Simple helper to show/hide the expanded message of a remark. */
+function toggleExpandedMessage(e) {
+  var FullTextElems = e.parentElement.parentElement.getElementsByClassName("full-info");
+  if (!FullTextElems || FullTextElems.length < 1) {
+      return false;
+  }
+  var FullText = FullTextElems[0];
+  if (FullText.style.display == 'none') {
+    e.innerHTML = '-';
+    FullText.style.display = 'block';
+  } else {
+    e.innerHTML = '+';
+    FullText.style.display = 'none';
+  }
+}
+</script>
 </head>
 <body>
 <div class="centered">
diff --git a/test/tools/opt-viewer/filter.test b/test/tools/opt-viewer/filter.test
new file mode 100644
index 0000000000000000000000000000000000000000..a6913ce0c5ae00d185bfbd6c7e8468940b621486
--- /dev/null
+++ b/test/tools/opt-viewer/filter.test
@@ -0,0 +1,8 @@
+# Since we're performing a full compare of the generate HTML files disable
+# syntax highlighting; pygments generates slightly different code with
+# different versions.
+
+RUN: %opt-viewer --filter inline -s %p/Inputs -o %t %p/Inputs/basic/or.yaml --no-highlight --demangler=llvm-cxxfilt
+RUN: diff %p/Outputs/filter/index.html %t/index.html
+RUN: diff %p/Outputs/filter/basic_or.h.html %t/basic_or.h.html
+RUN: diff %p/Outputs/filter/basic_or.c.html %t/basic_or.c.html
diff --git a/test/tools/yaml2obj/coff-symbol-index.yaml b/test/tools/yaml2obj/coff-symbol-index.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..592ecb38b1d8d51ddbd8d8da41e53e63ec92b2d0
--- /dev/null
+++ b/test/tools/yaml2obj/coff-symbol-index.yaml
@@ -0,0 +1,74 @@
+# RUN: yaml2obj %s -o %t
+# RUN: llvm-readobj -relocations %t | FileCheck %s --check-prefix=RELOCS
+# RUN: obj2yaml %t | FileCheck %s --check-prefix=YAML
+
+# RELOCS:      Relocations [
+# RELOCS-NEXT:   Section (1) .text {
+# RELOCS-NEXT:     0x3 IMAGE_REL_AMD64_REL32 .rdata (0)
+# RELOCS-NEXT:     0xA IMAGE_REL_AMD64_REL32 .rdata (1)
+# RELOCS-NEXT:     0x11 IMAGE_REL_AMD64_REL32 foo (2)
+# RELOCS-NEXT:   }
+# RELOCS-NEXT: ]
+
+# Check that we usually output relocations with SymbolName.
+# For relocations with a non-unique symbol name, output
+# SymbolTableIndex instead.
+
+# YAML:          Relocations:     
+# YAML-NEXT:       - VirtualAddress:  3
+# YAML-NEXT:         SymbolTableIndex: 0
+# YAML-NEXT:         Type:            IMAGE_REL_AMD64_REL32
+# YAML-NEXT:       - VirtualAddress:  10
+# YAML-NEXT:         SymbolTableIndex: 1
+# YAML-NEXT:         Type:            IMAGE_REL_AMD64_REL32
+# YAML-NEXT:       - VirtualAddress:  17
+# YAML-NEXT:         SymbolName:      foo
+# YAML-NEXT:         Type:            IMAGE_REL_AMD64_REL32
+
+--- !COFF
+header:          
+  Machine:         IMAGE_FILE_MACHINE_AMD64
+  Characteristics: [  ]
+sections:        
+  - Name:            .text
+    Characteristics: [  ]
+    Alignment:       4
+    SectionData:     488B0500000000488B0500000000488B0500000000
+    Relocations:     
+      - VirtualAddress:  3
+        SymbolTableIndex: 0
+        Type:            IMAGE_REL_AMD64_REL32
+      - VirtualAddress:  10
+        SymbolTableIndex: 1
+        Type:            IMAGE_REL_AMD64_REL32
+      - VirtualAddress:  17
+        SymbolName:      foo
+        Type:            IMAGE_REL_AMD64_REL32
+  - Name:            .rdata
+    Characteristics: [  ]
+    Alignment:       1
+    SectionData:     '00'
+  - Name:            .rdata
+    Characteristics: [  ]
+    Alignment:       1
+    SectionData:     '01'
+symbols:         
+  - Name:            .rdata
+    Value:           0
+    SectionNumber:   2
+    SimpleType:      IMAGE_SYM_TYPE_NULL
+    ComplexType:     IMAGE_SYM_DTYPE_NULL
+    StorageClass:    IMAGE_SYM_CLASS_STATIC
+  - Name:            .rdata
+    Value:           0
+    SectionNumber:   3
+    SimpleType:      IMAGE_SYM_TYPE_NULL
+    ComplexType:     IMAGE_SYM_DTYPE_NULL
+    StorageClass:    IMAGE_SYM_CLASS_STATIC
+  - Name:            foo
+    Value:           0
+    SectionNumber:   3
+    SimpleType:      IMAGE_SYM_TYPE_NULL
+    ComplexType:     IMAGE_SYM_DTYPE_NULL
+    StorageClass:    IMAGE_SYM_CLASS_EXTERNAL
+...
diff --git a/test/tools/yaml2obj/dynamic-section-i386.test b/test/tools/yaml2obj/dynamic-section-i386.test
new file mode 100644
index 0000000000000000000000000000000000000000..89f7de6fd7651ee93d3c2a1a43bf3f1f513c1aab
--- /dev/null
+++ b/test/tools/yaml2obj/dynamic-section-i386.test
@@ -0,0 +1,43 @@
+# RUN: yaml2obj %s -o %t
+# RUN: llvm-readobj %t --dynamic-table | FileCheck %s
+
+## Check we write a valid .dynamic section values when emiting a 32-bit object.
+
+# CHECK:      DynamicSection [ (2 entries)
+# CHECK-NEXT:   Tag        Type                 Name/Value
+# CHECK-NEXT:   0x0000000A STRSZ                1 (bytes)
+# CHECK-NEXT:   0x00000000 NULL                 0x0
+# CHECK-NEXT: ]
+
+!ELF
+FileHeader:
+  Class:           ELFCLASS32
+  Data:            ELFDATA2LSB
+  Type:            ET_DYN
+  Machine:         EM_386
+Sections:
+  - Name:            .dynamic
+    Type:            SHT_DYNAMIC
+    Flags:           [ SHF_ALLOC ]
+    Address:         0x0000000000000008
+    AddressAlign:    0x0000000000000008
+    EntSize:         0x0000000000000010
+    Entries:
+      - Tag:             DT_STRSZ
+        Value:           0x0000000000000001
+      - Tag:             DT_NULL
+        Value:           0x0000000000000000
+ProgramHeaders:
+  - Type: PT_LOAD
+    Flags: [ PF_R ]
+    VAddr: 0x0000
+    PAddr: 0x0000
+    Align: 8
+    Sections:
+      - Section: .dynamic
+  - Type: PT_DYNAMIC
+    Flags: [ PF_X, PF_R ]
+    VAddr: 0x0008
+    PAddr: 0x0008
+    Sections:
+      - Section: .dynamic
diff --git a/test/tools/yaml2obj/elf-abiversion.yaml b/test/tools/yaml2obj/elf-abiversion.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..0f43e69828fbb694b2789a6d931bfaa1df574b82
--- /dev/null
+++ b/test/tools/yaml2obj/elf-abiversion.yaml
@@ -0,0 +1,16 @@
+# RUN: yaml2obj %s -o %t
+# RUN: llvm-readobj -file-headers %t | FileCheck %s --check-prefix=FROMYAML
+# RUN: obj2yaml %t | FileCheck %s --check-prefix=TOYAML
+
+## Check we are able to parse/emit the ABI version with yaml2obj/obj2yaml.
+
+# FROMYAML: ABIVersion: 5
+# TOYAML:   ABIVersion: 0x05
+
+--- !ELF
+FileHeader:
+  Class:           ELFCLASS64
+  Data:            ELFDATA2LSB
+  Type:            ET_REL
+  Machine:         EM_X86_64
+  ABIVersion:      0x05
diff --git a/test/tools/yaml2obj/symbol-type.yaml b/test/tools/yaml2obj/symbol-type.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..1bca0e4101525dfa9b8355a2a460ac0148f7797c
--- /dev/null
+++ b/test/tools/yaml2obj/symbol-type.yaml
@@ -0,0 +1,43 @@
+# RUN: yaml2obj %s > %t
+# RUN: llvm-readobj --symbols %t | FileCheck %s
+
+# CHECK: Name: notype
+# CHECK: Type: None
+# CHECK: Name: normal_type
+# CHECK: Type: Object
+# CHECK: Name: .text
+# CHECK: Type: Section
+# CHECK: Name: known_hex
+# CHECK: Type: Object
+# CHECK: Name: unknown_hex
+# CHECK: Type: 0xB
+# CHECK: Name: known_int
+# CHECK: Type: Object
+# CHECK: Name: unknown_int
+# CHECK: Type: 0xB
+
+!ELF
+FileHeader:
+  Class:           ELFCLASS64
+  Data:            ELFDATA2LSB
+  Type:            ET_REL
+  Machine:         EM_X86_64
+Sections:
+  - Name: .text
+    Type: SHT_PROGBITS
+Symbols:
+  Global:
+    - Name: notype
+      Type: STT_NOTYPE
+    - Name: normal_type
+      Type: STT_OBJECT
+    - Name: .text
+      Type: STT_SECTION
+    - Name: known_hex
+      Type: 0x1
+    - Name: unknown_hex
+      Type: 0xb
+    - Name: known_int
+      Type: 1
+    - Name: unknown_int
+      Type: 11
diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt
index ddafc98d63bbcaedb058d186c52be87a0adca9b8..6084b3befa44439de2e1f2d0b4c4cc2c820b83f3 100644
--- a/tools/CMakeLists.txt
+++ b/tools/CMakeLists.txt
@@ -27,7 +27,6 @@ endif()
 
 if(CYGWIN OR NOT LLVM_ENABLE_PIC)
   set(LLVM_TOOL_LTO_BUILD Off)
-  set(LLVM_TOOL_LLVM_LTO_BUILD Off)
 endif()
 
 # Add LTO, llvm-ar, llvm-config, and llvm-profdata before clang, ExternalProject
diff --git a/tools/LLVMBuild.txt b/tools/LLVMBuild.txt
index 1732ea0cb8a0b3faff7293067a53182adb985580..78db3ed133b28073208a5160ad2228d413fbe430 100644
--- a/tools/LLVMBuild.txt
+++ b/tools/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./tools/LLVMBuild.txt ------------------------------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
@@ -32,6 +31,7 @@ subdirectories =
  llvm-dis
  llvm-dwarfdump
  llvm-dwp
+ llvm-elfabi
  llvm-exegesis
  llvm-extract
  llvm-jitlistener
diff --git a/tools/bugpoint-passes/CMakeLists.txt b/tools/bugpoint-passes/CMakeLists.txt
index e32b0a3aa34f3be8917523c4fd010ccb7e7b9c26..eea3e239b80830d1d8914d9e59829f5394962349 100644
--- a/tools/bugpoint-passes/CMakeLists.txt
+++ b/tools/bugpoint-passes/CMakeLists.txt
@@ -14,7 +14,7 @@ if(WIN32 OR CYGWIN)
   set(LLVM_LINK_COMPONENTS Core)
 endif()
 
-add_llvm_loadable_module( BugpointPasses
+add_llvm_library( BugpointPasses MODULE BUILDTREE_ONLY
   TestPasses.cpp
 
   DEPENDS
diff --git a/tools/bugpoint-passes/TestPasses.cpp b/tools/bugpoint-passes/TestPasses.cpp
index 6b1463685b2ca8d3fea995998906366e63e90600..6340a3ec5c6ca9dbaf0ea92224414f4baebf721e 100644
--- a/tools/bugpoint-passes/TestPasses.cpp
+++ b/tools/bugpoint-passes/TestPasses.cpp
@@ -1,9 +1,8 @@
 //===- TestPasses.cpp - "buggy" passes used to test bugpoint --------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/tools/bugpoint/BugDriver.cpp b/tools/bugpoint/BugDriver.cpp
index 3832e075a693c05702bd7f098667593044b25da1..942028cad80b1ceb7f7394dfee640fde69a4fd53 100644
--- a/tools/bugpoint/BugDriver.cpp
+++ b/tools/bugpoint/BugDriver.cpp
@@ -1,9 +1,8 @@
 //===- BugDriver.cpp - Top-Level BugPoint class implementation ------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/tools/bugpoint/BugDriver.h b/tools/bugpoint/BugDriver.h
index bc60ae753548bd9d8072ead83010962498e3dec4..75f166b21b2c008ab3c02f54612f089606f62236 100644
--- a/tools/bugpoint/BugDriver.h
+++ b/tools/bugpoint/BugDriver.h
@@ -1,9 +1,8 @@
 //===- BugDriver.h - Top-Level BugPoint class -------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/tools/bugpoint/CrashDebugger.cpp b/tools/bugpoint/CrashDebugger.cpp
index ef6a214fde2054a5a84138c39beb4f9dad92d1b5..aab9debf9b597a4b4201f4f4286dc3b510a0a4ff 100644
--- a/tools/bugpoint/CrashDebugger.cpp
+++ b/tools/bugpoint/CrashDebugger.cpp
@@ -1,9 +1,8 @@
 //===- CrashDebugger.cpp - Debug compilation crashes ----------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/tools/bugpoint/ExecutionDriver.cpp b/tools/bugpoint/ExecutionDriver.cpp
index 1b86b103d83546d08e361cdb0223f6737dbe4449..40f198b88d1a0418407774ffadf80ab86812799d 100644
--- a/tools/bugpoint/ExecutionDriver.cpp
+++ b/tools/bugpoint/ExecutionDriver.cpp
@@ -1,9 +1,8 @@
 //===- ExecutionDriver.cpp - Allow execution of LLVM program --------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/tools/bugpoint/ExtractFunction.cpp b/tools/bugpoint/ExtractFunction.cpp
index 48f1575c25eb863ee073bc4d8569154d13fd9800..105702de3f1d88a3c9d0ef04932654cf9c726ae1 100644
--- a/tools/bugpoint/ExtractFunction.cpp
+++ b/tools/bugpoint/ExtractFunction.cpp
@@ -1,9 +1,8 @@
 //===- ExtractFunction.cpp - Extract a function from Program --------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/tools/bugpoint/FindBugs.cpp b/tools/bugpoint/FindBugs.cpp
index a695e875b787f03279eb84dafe8ca6eec609a5f2..2b1146da9680d9e76b382b28449af6843fed85f5 100644
--- a/tools/bugpoint/FindBugs.cpp
+++ b/tools/bugpoint/FindBugs.cpp
@@ -1,9 +1,8 @@
 //===-- FindBugs.cpp - Run Many Different Optimizations -------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/tools/bugpoint/LLVMBuild.txt b/tools/bugpoint/LLVMBuild.txt
index 68ecb8c8f4f9124155b4b4671372509adb5429a8..a0be9ab7ea033b18738ef01db52c4ef9040d61ca 100644
--- a/tools/bugpoint/LLVMBuild.txt
+++ b/tools/bugpoint/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./tools/bugpoint/LLVMBuild.txt ---------------------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/tools/bugpoint/ListReducer.h b/tools/bugpoint/ListReducer.h
index 0f9db022d555e34ae6ba536648015e47efbe7dc8..04f2207a31ed793e50858bb8097fbf1a0add918e 100644
--- a/tools/bugpoint/ListReducer.h
+++ b/tools/bugpoint/ListReducer.h
@@ -1,9 +1,8 @@
 //===- ListReducer.h - Trim down list while retaining property --*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/tools/bugpoint/Miscompilation.cpp b/tools/bugpoint/Miscompilation.cpp
index 375bee7a0d5080cc8c56fe0ecde8adb07992d78c..b7477149c61f424334f8447c6966bead889cb844 100644
--- a/tools/bugpoint/Miscompilation.cpp
+++ b/tools/bugpoint/Miscompilation.cpp
@@ -1,9 +1,8 @@
 //===- Miscompilation.cpp - Debug program miscompilations -----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -827,13 +826,14 @@ CleanupAndPrepareModules(BugDriver &BD, std::unique_ptr<Module> Test,
 
   // Add the resolver to the Safe module.
   // Prototype: void *getPointerToNamedFunction(const char* Name)
-  Constant *resolverFunc = Safe->getOrInsertFunction(
+  FunctionCallee resolverFunc = Safe->getOrInsertFunction(
       "getPointerToNamedFunction", Type::getInt8PtrTy(Safe->getContext()),
       Type::getInt8PtrTy(Safe->getContext()));
 
   // Use the function we just added to get addresses of functions we need.
   for (Module::iterator F = Safe->begin(), E = Safe->end(); F != E; ++F) {
-    if (F->isDeclaration() && !F->use_empty() && &*F != resolverFunc &&
+    if (F->isDeclaration() && !F->use_empty() &&
+        &*F != resolverFunc.getCallee() &&
         !F->isIntrinsic() /* ignore intrinsics */) {
       Function *TestFn = Test->getFunction(F->getName());
 
@@ -879,7 +879,8 @@ CleanupAndPrepareModules(BugDriver &BD, std::unique_ptr<Module> Test,
               BasicBlock::Create(F->getContext(), "lookupfp", FuncWrapper);
 
           // Check to see if we already looked up the value.
-          Value *CachedVal = new LoadInst(Cache, "fpcache", EntryBB);
+          Value *CachedVal =
+              new LoadInst(F->getType(), Cache, "fpcache", EntryBB);
           Value *IsNull = new ICmpInst(*EntryBB, ICmpInst::ICMP_EQ, CachedVal,
                                        NullPtr, "isNull");
           BranchInst::Create(LookupBB, DoCallBB, IsNull, EntryBB);
@@ -911,11 +912,11 @@ CleanupAndPrepareModules(BugDriver &BD, std::unique_ptr<Module> Test,
 
           // Pass on the arguments to the real function, return its result
           if (F->getReturnType()->isVoidTy()) {
-            CallInst::Create(FuncPtr, Args, "", DoCallBB);
+            CallInst::Create(FuncTy, FuncPtr, Args, "", DoCallBB);
             ReturnInst::Create(F->getContext(), DoCallBB);
           } else {
             CallInst *Call =
-                CallInst::Create(FuncPtr, Args, "retval", DoCallBB);
+                CallInst::Create(FuncTy, FuncPtr, Args, "retval", DoCallBB);
             ReturnInst::Create(F->getContext(), Call, DoCallBB);
           }
 
diff --git a/tools/bugpoint/OptimizerDriver.cpp b/tools/bugpoint/OptimizerDriver.cpp
index 64fe675de20ccaf7bb1cd9e44c237f111dfd3b60..562de795238889fefc9e6b096eb238fa1af8dc19 100644
--- a/tools/bugpoint/OptimizerDriver.cpp
+++ b/tools/bugpoint/OptimizerDriver.cpp
@@ -1,9 +1,8 @@
 //===- OptimizerDriver.cpp - Allow BugPoint to run passes safely ----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/tools/bugpoint/ToolRunner.cpp b/tools/bugpoint/ToolRunner.cpp
index 7ba8ea1f16c572bccf9b99c1d419f9ccd007743c..da4244345e3bc6c209097f3d2ba09bf6775e15d8 100644
--- a/tools/bugpoint/ToolRunner.cpp
+++ b/tools/bugpoint/ToolRunner.cpp
@@ -1,9 +1,8 @@
 //===-- ToolRunner.cpp ----------------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/tools/bugpoint/ToolRunner.h b/tools/bugpoint/ToolRunner.h
index ef8551cc669b44942dbdb5c42a3a78d70fa32818..dde4ec539cfb0ed526583821ce8afddf02274034 100644
--- a/tools/bugpoint/ToolRunner.h
+++ b/tools/bugpoint/ToolRunner.h
@@ -1,9 +1,8 @@
 //===-- tools/bugpoint/ToolRunner.h -----------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/tools/bugpoint/bugpoint.cpp b/tools/bugpoint/bugpoint.cpp
index f6b7d08455d438f2eadf35aec37123afe6dddc76..2d5322a351ad1f5a930e6152e759338107ac3d0b 100644
--- a/tools/bugpoint/bugpoint.cpp
+++ b/tools/bugpoint/bugpoint.cpp
@@ -1,9 +1,8 @@
 //===- bugpoint.cpp - The LLVM Bugpoint utility ---------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/tools/dsymutil/BinaryHolder.cpp b/tools/dsymutil/BinaryHolder.cpp
index e78871c487790b318e122ff51d583af0ad327d6e..9c9672af045c698d0aeb5e8bc5d46ded80ad0b13 100644
--- a/tools/dsymutil/BinaryHolder.cpp
+++ b/tools/dsymutil/BinaryHolder.cpp
@@ -1,9 +1,8 @@
 //===-- BinaryHolder.cpp --------------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/tools/dsymutil/BinaryHolder.h b/tools/dsymutil/BinaryHolder.h
index c78bff2f8f3132c878ef00def4292568cfc1ec33..0376d1e786433bd6da52c437f409a050c984976d 100644
--- a/tools/dsymutil/BinaryHolder.h
+++ b/tools/dsymutil/BinaryHolder.h
@@ -1,9 +1,8 @@
 //===-- BinaryHolder.h - Utility class for accessing binaries -------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/tools/dsymutil/CFBundle.cpp b/tools/dsymutil/CFBundle.cpp
index 15ee8011a4a4815b79f5b9204c5d54e0502906f2..eb07287b63c693ac2e0348d0a724159ee079b56c 100644
--- a/tools/dsymutil/CFBundle.cpp
+++ b/tools/dsymutil/CFBundle.cpp
@@ -1,9 +1,8 @@
 //===- tools/dsymutil/CFBundle.cpp - CFBundle helper ------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/tools/dsymutil/CFBundle.h b/tools/dsymutil/CFBundle.h
index 5f241849910df2e1b481aa2bce52c477ac6a1932..103db035164323ef20f6f8b61e4dc979d8fc51b6 100644
--- a/tools/dsymutil/CFBundle.h
+++ b/tools/dsymutil/CFBundle.h
@@ -1,9 +1,8 @@
 //===- tools/dsymutil/CFBundle.h - CFBundle helper --------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/tools/dsymutil/CMakeLists.txt b/tools/dsymutil/CMakeLists.txt
index f41a6fd28504649491eae15e2ce1da535465ff1b..480f78fb1888cce4ca1a8794667e285c1a759bf4 100644
--- a/tools/dsymutil/CMakeLists.txt
+++ b/tools/dsymutil/CMakeLists.txt
@@ -20,6 +20,7 @@ add_llvm_tool(dsymutil
   MachODebugMapParser.cpp
   MachOUtils.cpp
   NonRelocatableStringpool.cpp
+  SymbolMap.cpp
 
   DEPENDS
   intrinsics_gen
diff --git a/tools/dsymutil/CompileUnit.cpp b/tools/dsymutil/CompileUnit.cpp
index 67e1739ae1084d6105f0144918fa9624c7cb9d94..670591a9436f95b4a1df7bf75746f9c2bb886d7f 100644
--- a/tools/dsymutil/CompileUnit.cpp
+++ b/tools/dsymutil/CompileUnit.cpp
@@ -1,9 +1,8 @@
 //===- tools/dsymutil/CompileUnit.h - Dwarf compile unit ------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -56,12 +55,11 @@ void CompileUnit::markEverythingAsKept() {
 }
 
 uint64_t CompileUnit::computeNextUnitOffset() {
-  NextUnitOffset = StartOffset + 11 /* Header size */;
-  // The root DIE might be null, meaning that the Unit had nothing to
-  // contribute to the linked output. In that case, we will emit the
-  // unit header without any actual DIE.
-  if (NewUnit)
+  NextUnitOffset = StartOffset;
+  if (NewUnit) {
+    NextUnitOffset += 11 /* Header size */;
     NextUnitOffset += NewUnit->getUnitDie().getSize();
+  }
   return NextUnitOffset;
 }
 
@@ -92,7 +90,11 @@ void CompileUnit::addLabelLowPc(uint64_t LabelLowPc, int64_t PcOffset) {
 
 void CompileUnit::addFunctionRange(uint64_t FuncLowPc, uint64_t FuncHighPc,
                                    int64_t PcOffset) {
-  Ranges.insert(FuncLowPc, FuncHighPc, PcOffset);
+  //  Don't add empty ranges to the interval map.  They are a problem because
+  //  the interval map expects half open intervals. This is safe because they
+  //  are empty anyway.
+  if (FuncHighPc != FuncLowPc)
+    Ranges.insert(FuncLowPc, FuncHighPc, PcOffset);
   this->LowPc = std::min(LowPc, FuncLowPc + PcOffset);
   this->HighPc = std::max(HighPc, FuncHighPc + PcOffset);
 }
diff --git a/tools/dsymutil/CompileUnit.h b/tools/dsymutil/CompileUnit.h
index 79b88fd4d6bb0fe659992887a1883943ea3462f0..6e3c2c669e09dedf6db7fffab5b720d25146c14a 100644
--- a/tools/dsymutil/CompileUnit.h
+++ b/tools/dsymutil/CompileUnit.h
@@ -1,9 +1,8 @@
 //===- tools/dsymutil/CompileUnit.h - Dwarf debug info linker ---*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/tools/dsymutil/DebugMap.cpp b/tools/dsymutil/DebugMap.cpp
index 509afaf56f4e86658dc5ad7d7e246bfdfe7f9fe6..83dddfe9e5218735fffad637a3c3d53b7614a85c 100644
--- a/tools/dsymutil/DebugMap.cpp
+++ b/tools/dsymutil/DebugMap.cpp
@@ -1,9 +1,8 @@
 //===- tools/dsymutil/DebugMap.cpp - Generic debug map representation -----===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/tools/dsymutil/DebugMap.h b/tools/dsymutil/DebugMap.h
index c9883773d3dc9d9cfec4a380a9847aff45a4692c..6c0cb8baff6a7a0e93696c097673fb43d3d5a980 100644
--- a/tools/dsymutil/DebugMap.h
+++ b/tools/dsymutil/DebugMap.h
@@ -1,9 +1,8 @@
 //=- tools/dsymutil/DebugMap.h - Generic debug map representation -*- C++ -*-=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -75,7 +74,7 @@ class DebugMapObject;
 class DebugMap {
   Triple BinaryTriple;
   std::string BinaryPath;
-
+  std::vector<uint8_t> BinaryUUID;
   using ObjectContainer = std::vector<std::unique_ptr<DebugMapObject>>;
 
   ObjectContainer Objects;
@@ -89,8 +88,10 @@ class DebugMap {
   ///@}
 
 public:
-  DebugMap(const Triple &BinaryTriple, StringRef BinaryPath)
-      : BinaryTriple(BinaryTriple), BinaryPath(BinaryPath) {}
+  DebugMap(const Triple &BinaryTriple, StringRef BinaryPath,
+           ArrayRef<uint8_t> BinaryUUID = ArrayRef<uint8_t>())
+      : BinaryTriple(BinaryTriple), BinaryPath(BinaryPath),
+        BinaryUUID(BinaryUUID.begin(), BinaryUUID.end()) {}
 
   using const_iterator = ObjectContainer::const_iterator;
 
@@ -113,6 +114,10 @@ public:
 
   const Triple &getTriple() const { return BinaryTriple; }
 
+  const ArrayRef<uint8_t> getUUID() const {
+    return ArrayRef<uint8_t>(BinaryUUID);
+  }
+
   StringRef getBinaryPath() const { return BinaryPath; }
 
   void print(raw_ostream &OS) const;
diff --git a/tools/dsymutil/DeclContext.cpp b/tools/dsymutil/DeclContext.cpp
index 3e6dcaf51a2cd119af6cfcd76af6fec1ffcf8665..1c33b672c26ba6e84da5df2284658eb15c092590 100644
--- a/tools/dsymutil/DeclContext.cpp
+++ b/tools/dsymutil/DeclContext.cpp
@@ -1,9 +1,8 @@
 //===- tools/dsymutil/DeclContext.cpp - Declaration context ---------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/tools/dsymutil/DeclContext.h b/tools/dsymutil/DeclContext.h
index 425c73671e12759f050ee1c61f9184b029d1bc74..e88678c6613f8383bf055ef411c753c34049304e 100644
--- a/tools/dsymutil/DeclContext.h
+++ b/tools/dsymutil/DeclContext.h
@@ -1,9 +1,8 @@
 //===- tools/dsymutil/DeclContext.h - Dwarf debug info linker ---*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/tools/dsymutil/DwarfLinker.cpp b/tools/dsymutil/DwarfLinker.cpp
index 2862739fd73ac86a0ea97cbe7dfdfc69b7ce56a1..3fe018a6f2619fd5d072ccf31886c56ba619268b 100644
--- a/tools/dsymutil/DwarfLinker.cpp
+++ b/tools/dsymutil/DwarfLinker.cpp
@@ -1,9 +1,8 @@
 //===- tools/dsymutil/DwarfLinker.cpp - Dwarf debug info linker -----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -1701,6 +1700,8 @@ void DwarfLinker::patchLineTableForUnit(CompileUnit &Unit,
   DWARFDataExtractor LineExtractor(
       OrigDwarf.getDWARFObj(), OrigDwarf.getDWARFObj().getLineSection(),
       OrigDwarf.isLittleEndian(), Unit.getOrigUnit().getAddressByteSize());
+  if (Options.Translator)
+    return Streamer->translateLineTable(LineExtractor, StmtOffset, Options);
 
   Error Err = LineTable.parse(LineExtractor, &StmtOffset, OrigDwarf,
                               &Unit.getOrigUnit(), DWARFContext::dumpWarning);
@@ -2241,21 +2242,25 @@ void DwarfLinker::DIECloner::cloneAllCompileUnits(
       cloneDIE(InputDIE, DMO, *CurrentUnit, StringPool, 0 /* PC offset */,
                11 /* Unit Header size */, 0, CurrentUnit->getOutputUnitDIE());
     }
+
     Linker.OutputDebugInfoSize = CurrentUnit->computeNextUnitOffset();
+
     if (Linker.Options.NoOutput)
       continue;
 
-    if (LLVM_LIKELY(!Linker.Options.Update)) {
-      // FIXME: for compatibility with the classic dsymutil, we emit an empty
-      // line table for the unit, even if the unit doesn't actually exist in
-      // the DIE tree.
+    // FIXME: for compatibility with the classic dsymutil, we emit
+    // an empty line table for the unit, even if the unit doesn't
+    // actually exist in the DIE tree.
+    if (LLVM_LIKELY(!Linker.Options.Update) || Linker.Options.Translator)
       Linker.patchLineTableForUnit(*CurrentUnit, DwarfContext, Ranges, DMO);
-      Linker.emitAcceleratorEntriesForUnit(*CurrentUnit);
-      Linker.patchRangesForUnit(*CurrentUnit, DwarfContext, DMO);
-      Linker.Streamer->emitLocationsForUnit(*CurrentUnit, DwarfContext);
-    } else {
-      Linker.emitAcceleratorEntriesForUnit(*CurrentUnit);
-    }
+
+    Linker.emitAcceleratorEntriesForUnit(*CurrentUnit);
+
+    if (LLVM_UNLIKELY(Linker.Options.Update))
+      continue;
+
+    Linker.patchRangesForUnit(*CurrentUnit, DwarfContext, DMO);
+    Linker.Streamer->emitLocationsForUnit(*CurrentUnit, DwarfContext);
   }
 
   if (Linker.Options.NoOutput)
@@ -2265,10 +2270,13 @@ void DwarfLinker::DIECloner::cloneAllCompileUnits(
   for (auto &CurrentUnit : CompileUnits) {
     if (LLVM_LIKELY(!Linker.Options.Update))
       Linker.generateUnitRanges(*CurrentUnit);
+
     CurrentUnit->fixupForwardReferences();
-    Linker.Streamer->emitCompileUnitHeader(*CurrentUnit);
+
     if (!CurrentUnit->getOutputUnitDIE())
       continue;
+
+    Linker.Streamer->emitCompileUnitHeader(*CurrentUnit);
     Linker.Streamer->emitDIE(*CurrentUnit->getOutputUnitDIE());
   }
 }
@@ -2380,7 +2388,7 @@ bool DwarfLinker::link(const DebugMap &Map) {
   // This Dwarf string pool which is used for emission. It must be used
   // serially as the order of calling getStringOffset matters for
   // reproducibility.
-  OffsetsStringPool OffsetsStringPool;
+  OffsetsStringPool OffsetsStringPool(Options.Translator);
 
   // ODR Contexts for the link.
   DeclContextTree ODRContexts;
@@ -2649,7 +2657,7 @@ bool DwarfLinker::link(const DebugMap &Map) {
     pool.wait();
   }
 
-  return Options.NoOutput ? true : Streamer->finish(Map);
+  return Options.NoOutput ? true : Streamer->finish(Map, Options.Translator);
 } // namespace dsymutil
 
 bool linkDwarf(raw_fd_ostream &OutFile, BinaryHolder &BinHolder,
diff --git a/tools/dsymutil/DwarfLinker.h b/tools/dsymutil/DwarfLinker.h
index 64fa8d2107d5ba5b7bca93ac9c5cdc13912e7234..159572743b97d2bd22b378e9c89997f28f5e8c41 100644
--- a/tools/dsymutil/DwarfLinker.h
+++ b/tools/dsymutil/DwarfLinker.h
@@ -1,9 +1,8 @@
 //===- tools/dsymutil/DwarfLinker.h - Dwarf debug info linker ---*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/tools/dsymutil/DwarfStreamer.cpp b/tools/dsymutil/DwarfStreamer.cpp
index ef798be7bdfa9202bc341fe68b72d5a454c42089..2814167400152f96fffc6844afe98333963a083b 100644
--- a/tools/dsymutil/DwarfStreamer.cpp
+++ b/tools/dsymutil/DwarfStreamer.cpp
@@ -1,9 +1,8 @@
 //===- tools/dsymutil/DwarfStreamer.cpp - Dwarf Streamer ------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -124,11 +123,11 @@ bool DwarfStreamer::init(Triple TheTriple) {
   return true;
 }
 
-bool DwarfStreamer::finish(const DebugMap &DM) {
+bool DwarfStreamer::finish(const DebugMap &DM, SymbolMapTranslator &T) {
   bool Result = true;
   if (DM.getTriple().isOSDarwin() && !DM.getBinaryPath().empty() &&
       Options.FileType == OutputFileType::Object)
-    Result = MachOUtils::generateDsymCompanion(DM, *MS, OutFile);
+    Result = MachOUtils::generateDsymCompanion(DM, T, *MS, OutFile);
   else
     MS->Finish();
   return Result;
@@ -577,6 +576,89 @@ void DwarfStreamer::emitLineTableForUnit(MCDwarfLineTableParams Params,
   MS->EmitLabel(LineEndSym);
 }
 
+/// Copy the debug_line over to the updated binary while unobfuscating the file
+/// names and directories.
+void DwarfStreamer::translateLineTable(DataExtractor Data, uint32_t Offset,
+                                       LinkOptions &Options) {
+  MS->SwitchSection(MC->getObjectFileInfo()->getDwarfLineSection());
+  StringRef Contents = Data.getData();
+
+  // We have to deconstruct the line table header, because it contains to
+  // length fields that will need to be updated when we change the length of
+  // the files and directories in there.
+  unsigned UnitLength = Data.getU32(&Offset);
+  unsigned UnitEnd = Offset + UnitLength;
+  MCSymbol *BeginLabel = MC->createTempSymbol();
+  MCSymbol *EndLabel = MC->createTempSymbol();
+  unsigned Version = Data.getU16(&Offset);
+
+  if (Version > 5) {
+    warn("Unsupported line table version: dropping contents and not "
+         "unobfsucating line table.");
+    return;
+  }
+
+  Asm->EmitLabelDifference(EndLabel, BeginLabel, 4);
+  Asm->OutStreamer->EmitLabel(BeginLabel);
+  Asm->emitInt16(Version);
+  LineSectionSize += 6;
+
+  MCSymbol *HeaderBeginLabel = MC->createTempSymbol();
+  MCSymbol *HeaderEndLabel = MC->createTempSymbol();
+  Asm->EmitLabelDifference(HeaderEndLabel, HeaderBeginLabel, 4);
+  Asm->OutStreamer->EmitLabel(HeaderBeginLabel);
+  Offset += 4;
+  LineSectionSize += 4;
+
+  uint32_t AfterHeaderLengthOffset = Offset;
+  // Skip to the directories.
+  Offset += (Version >= 4) ? 5 : 4;
+  unsigned OpcodeBase = Data.getU8(&Offset);
+  Offset += OpcodeBase - 1;
+  Asm->OutStreamer->EmitBytes(Contents.slice(AfterHeaderLengthOffset, Offset));
+  LineSectionSize += Offset - AfterHeaderLengthOffset;
+
+  // Offset points to the first directory.
+  while (const char *Dir = Data.getCStr(&Offset)) {
+    if (Dir[0] == 0)
+      break;
+
+    StringRef Translated = Options.Translator(Dir);
+    Asm->OutStreamer->EmitBytes(Translated);
+    Asm->emitInt8(0);
+    LineSectionSize += Translated.size() + 1;
+  }
+  Asm->emitInt8(0);
+  LineSectionSize += 1;
+
+  while (const char *File = Data.getCStr(&Offset)) {
+    if (File[0] == 0)
+      break;
+
+    StringRef Translated = Options.Translator(File);
+    Asm->OutStreamer->EmitBytes(Translated);
+    Asm->emitInt8(0);
+    LineSectionSize += Translated.size() + 1;
+
+    uint32_t OffsetBeforeLEBs = Offset;
+    Asm->EmitULEB128(Data.getULEB128(&Offset));
+    Asm->EmitULEB128(Data.getULEB128(&Offset));
+    Asm->EmitULEB128(Data.getULEB128(&Offset));
+    LineSectionSize += Offset - OffsetBeforeLEBs;
+  }
+  Asm->emitInt8(0);
+  LineSectionSize += 1;
+
+  Asm->OutStreamer->EmitLabel(HeaderEndLabel);
+
+  // Copy the actual line table program over.
+  Asm->OutStreamer->EmitBytes(Contents.slice(Offset, UnitEnd));
+  LineSectionSize += UnitEnd - Offset;
+
+  Asm->OutStreamer->EmitLabel(EndLabel);
+  Offset = UnitEnd;
+}
+
 static void emitSectionContents(const object::ObjectFile &Obj,
                                 StringRef SecName, MCStreamer *MS) {
   StringRef Contents;
@@ -586,8 +668,10 @@ static void emitSectionContents(const object::ObjectFile &Obj,
 }
 
 void DwarfStreamer::copyInvariantDebugSection(const object::ObjectFile &Obj) {
-  MS->SwitchSection(MC->getObjectFileInfo()->getDwarfLineSection());
-  emitSectionContents(Obj, "debug_line", MS);
+  if (!Options.Translator) {
+    MS->SwitchSection(MC->getObjectFileInfo()->getDwarfLineSection());
+    emitSectionContents(Obj, "debug_line", MS);
+  }
 
   MS->SwitchSection(MC->getObjectFileInfo()->getDwarfLocSection());
   emitSectionContents(Obj, "debug_loc", MS);
diff --git a/tools/dsymutil/DwarfStreamer.h b/tools/dsymutil/DwarfStreamer.h
index 679d124f4cb7fca9b2f56a73b45568351b36a53a..7477004eecfca9df2c01b327385252e8ed088a24 100644
--- a/tools/dsymutil/DwarfStreamer.h
+++ b/tools/dsymutil/DwarfStreamer.h
@@ -1,9 +1,8 @@
 //===- tools/dsymutil/DwarfStreamer.h - Dwarf Streamer ----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -50,7 +49,7 @@ public:
   bool init(Triple TheTriple);
 
   /// Dump the file to the disk.
-  bool finish(const DebugMap &);
+  bool finish(const DebugMap &, SymbolMapTranslator &T);
 
   AsmPrinter &getAsmPrinter() const { return *Asm; }
 
@@ -104,6 +103,11 @@ public:
                             std::vector<DWARFDebugLine::Row> &Rows,
                             unsigned AdddressSize);
 
+  /// Copy the debug_line over to the updated binary while unobfuscating the
+  /// file names and directories.
+  void translateLineTable(DataExtractor LineData, uint32_t Offset,
+                          LinkOptions &Options);
+
   /// Copy over the debug sections that are not modified when updating.
   void copyInvariantDebugSection(const object::ObjectFile &Obj);
 
diff --git a/tools/dsymutil/LLVMBuild.txt b/tools/dsymutil/LLVMBuild.txt
index 3769fbe33f21644fc0c7e4dc1c1b2f5e3d3a4e05..6eed917880301387859433c3bf4636ed90ca7d5b 100644
--- a/tools/dsymutil/LLVMBuild.txt
+++ b/tools/dsymutil/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./tools/dsymutil/LLVMBuild.txt ---------------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/tools/dsymutil/LinkUtils.h b/tools/dsymutil/LinkUtils.h
index f0abd888b52972033c580e3dd93b8dab31c330d1..2b3ccb29088f308b98e9f892c423cb8aa6884446 100644
--- a/tools/dsymutil/LinkUtils.h
+++ b/tools/dsymutil/LinkUtils.h
@@ -1,17 +1,19 @@
 //===- tools/dsymutil/LinkUtils.h - Dwarf linker utilities ------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #ifndef LLVM_TOOLS_DSYMUTIL_LINKOPTIONS_H
 #define LLVM_TOOLS_DSYMUTIL_LINKOPTIONS_H
 
+#include "SymbolMap.h"
+
 #include "llvm/ADT/Twine.h"
 #include "llvm/Support/WithColor.h"
+
 #include <string>
 
 namespace llvm {
@@ -60,6 +62,9 @@ struct LinkOptions {
   /// -oso-prepend-path
   std::string PrependPath;
 
+  /// Symbol map translator.
+  SymbolMapTranslator Translator;
+
   LinkOptions() = default;
 };
 
diff --git a/tools/dsymutil/MachODebugMapParser.cpp b/tools/dsymutil/MachODebugMapParser.cpp
index d696e1d4cc84e9f15ed3eeba6f17543a60e99877..8c7e686a304ecaf386f35e5cf6ad083aac61888a 100644
--- a/tools/dsymutil/MachODebugMapParser.cpp
+++ b/tools/dsymutil/MachODebugMapParser.cpp
@@ -1,9 +1,8 @@
 //===- tools/dsymutil/MachODebugMapParser.cpp - Parse STABS debug maps ----===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -163,7 +162,8 @@ std::unique_ptr<DebugMap>
 MachODebugMapParser::parseOneBinary(const MachOObjectFile &MainBinary,
                                     StringRef BinaryPath) {
   loadMainBinarySymbols(MainBinary);
-  Result = make_unique<DebugMap>(MainBinary.getArchTriple(), BinaryPath);
+  ArrayRef<uint8_t> UUID = MainBinary.getUuid();
+  Result = make_unique<DebugMap>(MainBinary.getArchTriple(), BinaryPath, UUID);
   MainBinaryStrings = MainBinary.getStringTableData();
   for (const SymbolRef &Symbol : MainBinary.symbols()) {
     const DataRefImpl &DRI = Symbol.getRawDataRefImpl();
diff --git a/tools/dsymutil/MachOUtils.cpp b/tools/dsymutil/MachOUtils.cpp
index cac4ad89eb20c792e42b1eee862b189be748ca32..b7ab35274f9e86b09aaef85d6bb30eace14b31f8 100644
--- a/tools/dsymutil/MachOUtils.cpp
+++ b/tools/dsymutil/MachOUtils.cpp
@@ -1,9 +1,8 @@
 //===-- MachOUtils.cpp - Mach-o specific helpers for dsymutil  ------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -333,8 +332,8 @@ static unsigned segmentLoadCommandSize(bool Is64Bit, unsigned NumSections) {
 // Stream a dSYM companion binary file corresponding to the binary referenced
 // by \a DM to \a OutFile. The passed \a MS MCStreamer is setup to write to
 // \a OutFile and it must be using a MachObjectWriter object to do so.
-bool generateDsymCompanion(const DebugMap &DM, MCStreamer &MS,
-                           raw_fd_ostream &OutFile) {
+bool generateDsymCompanion(const DebugMap &DM, SymbolMapTranslator &Translator,
+                           MCStreamer &MS, raw_fd_ostream &OutFile) {
   auto &ObjectStreamer = static_cast<MCObjectStreamer &>(MS);
   MCAssembler &MCAsm = ObjectStreamer.getAssembler();
   auto &Writer = static_cast<MachObjectWriter &>(MCAsm.getWriter());
@@ -443,7 +442,7 @@ bool generateDsymCompanion(const DebugMap &DM, MCStreamer &MS,
   }
 
   SmallString<0> NewSymtab;
-  NonRelocatableStringpool NewStrings;
+  NonRelocatableStringpool NewStrings(Translator);
   unsigned NListSize = Is64Bit ? sizeof(MachO::nlist_64) : sizeof(MachO::nlist);
   unsigned NumSyms = 0;
   uint64_t NewStringsSize = 0;
diff --git a/tools/dsymutil/MachOUtils.h b/tools/dsymutil/MachOUtils.h
index a8be89e906b5d530071cfc91785eeea84bd0f2c3..83b648122b78bb8ccc18a908f706c66058235b29 100644
--- a/tools/dsymutil/MachOUtils.h
+++ b/tools/dsymutil/MachOUtils.h
@@ -1,16 +1,18 @@
 //===-- MachOUtils.h - Mach-o specific helpers for dsymutil  --------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 #ifndef LLVM_TOOLS_DSYMUTIL_MACHOUTILS_H
 #define LLVM_TOOLS_DSYMUTIL_MACHOUTILS_H
 
+#include "SymbolMap.h"
+
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/FileSystem.h"
+
 #include <string>
 
 namespace llvm {
@@ -38,8 +40,8 @@ bool generateUniversalBinary(SmallVectorImpl<ArchAndFile> &ArchFiles,
                              StringRef OutputFileName, const LinkOptions &,
                              StringRef SDKPath);
 
-bool generateDsymCompanion(const DebugMap &DM, MCStreamer &MS,
-                           raw_fd_ostream &OutFile);
+bool generateDsymCompanion(const DebugMap &DM, SymbolMapTranslator &Translator,
+                           MCStreamer &MS, raw_fd_ostream &OutFile);
 
 std::string getArchName(StringRef Arch);
 } // namespace MachOUtils
diff --git a/tools/dsymutil/NonRelocatableStringpool.cpp b/tools/dsymutil/NonRelocatableStringpool.cpp
index d82ff84589d4b6bdacf2a50d67c8a661c2dbf53d..8cb1590cfb64394a4c20d79756705a9a49a15330 100644
--- a/tools/dsymutil/NonRelocatableStringpool.cpp
+++ b/tools/dsymutil/NonRelocatableStringpool.cpp
@@ -1,9 +1,8 @@
 //===- NonRelocatableStringpool.cpp - A simple stringpool  ----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -16,6 +15,8 @@ DwarfStringPoolEntryRef NonRelocatableStringpool::getEntry(StringRef S) {
   if (S.empty() && !Strings.empty())
     return EmptyString;
 
+  if (Translator)
+    S = Translator(S);
   auto I = Strings.insert({S, DwarfStringPoolEntry()});
   auto &Entry = I.first->second;
   if (I.second || !Entry.isIndexed()) {
@@ -29,6 +30,10 @@ DwarfStringPoolEntryRef NonRelocatableStringpool::getEntry(StringRef S) {
 
 StringRef NonRelocatableStringpool::internString(StringRef S) {
   DwarfStringPoolEntry Entry{nullptr, 0, DwarfStringPoolEntry::NotIndexed};
+
+  if (Translator)
+    S = Translator(S);
+
   auto InsertResult = Strings.insert({S, Entry});
   return InsertResult.first->getKey();
 }
diff --git a/tools/dsymutil/NonRelocatableStringpool.h b/tools/dsymutil/NonRelocatableStringpool.h
index e339e51f0c694dbbb69e46e7938979cc72e8ccb5..ef73242d14e2125191a828b03634c0543fa82e3d 100644
--- a/tools/dsymutil/NonRelocatableStringpool.h
+++ b/tools/dsymutil/NonRelocatableStringpool.h
@@ -1,15 +1,16 @@
 //===- NonRelocatableStringpool.h - A simple stringpool  --------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #ifndef LLVM_TOOLS_DSYMUTIL_NONRELOCATABLESTRINGPOOL_H
 #define LLVM_TOOLS_DSYMUTIL_NONRELOCATABLESTRINGPOOL_H
 
+#include "SymbolMap.h"
+
 #include "llvm/ADT/StringMap.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/CodeGen/DwarfStringPoolEntry.h"
@@ -32,7 +33,9 @@ public:
   /// order.
   using MapTy = StringMap<DwarfStringPoolEntry, BumpPtrAllocator>;
 
-  NonRelocatableStringpool() {
+  NonRelocatableStringpool(
+      SymbolMapTranslator Translator = SymbolMapTranslator())
+      : Translator(Translator) {
     // Legacy dsymutil puts an empty string at the start of the line table.
     EmptyString = getEntry("");
   }
@@ -62,6 +65,7 @@ private:
   uint32_t CurrentEndOffset = 0;
   unsigned NumEntries = 0;
   DwarfStringPoolEntryRef EmptyString;
+  SymbolMapTranslator Translator;
 };
 
 /// Helper for making strong types.
diff --git a/tools/dsymutil/SymbolMap.cpp b/tools/dsymutil/SymbolMap.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..7ee078cd9f934a7e162eb8109a103d1970f6c390
--- /dev/null
+++ b/tools/dsymutil/SymbolMap.cpp
@@ -0,0 +1,161 @@
+//===- tools/dsymutil/SymbolMap.cpp ---------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "SymbolMap.h"
+#include "DebugMap.h"
+#include "MachOUtils.h"
+
+#include "llvm/Support/FileSystem.h"
+#include "llvm/Support/Path.h"
+#include "llvm/Support/WithColor.h"
+
+#ifdef __APPLE__
+#include <CoreFoundation/CoreFoundation.h>
+#include <uuid/uuid.h>
+#endif
+
+namespace llvm {
+namespace dsymutil {
+
+StringRef SymbolMapTranslator::operator()(StringRef Input) {
+  if (!Input.startswith("__hidden#") && !Input.startswith("___hidden#"))
+    return Input;
+
+  bool MightNeedUnderscore = false;
+  StringRef Line = Input.drop_front(sizeof("__hidden#") - 1);
+  if (Line[0] == '#') {
+    Line = Line.drop_front();
+    MightNeedUnderscore = true;
+  }
+
+  std::size_t LineNumber = std::numeric_limits<std::size_t>::max();
+  Line.split('_').first.getAsInteger(10, LineNumber);
+  if (LineNumber >= UnobfuscatedStrings.size()) {
+    WithColor::warning() << "reference to a unexisting unobfuscated string "
+                         << Input << ": symbol map mismatch?\n"
+                         << Line << '\n';
+    return Input;
+  }
+
+  const std::string &Translation = UnobfuscatedStrings[LineNumber];
+  if (!MightNeedUnderscore || !MangleNames)
+    return Translation;
+
+  // Objective-C symbols for the MachO symbol table start with a \1. Please see
+  // `CGObjCCommonMac::GetNameForMethod` in clang.
+  if (Translation[0] == 1)
+    return StringRef(Translation).drop_front();
+
+  // We need permanent storage for the string we are about to create. Just
+  // append it to the vector containing translations. This should only happen
+  // during MachO symbol table translation, thus there should be no risk on
+  // exponential growth.
+  UnobfuscatedStrings.emplace_back("_" + Translation);
+  return UnobfuscatedStrings.back();
+}
+
+SymbolMapTranslator SymbolMapLoader::Load(StringRef InputFile,
+                                          const DebugMap &Map) const {
+  if (SymbolMap.empty())
+    return {};
+
+  std::string SymbolMapPath = SymbolMap;
+
+#if __APPLE__
+  // Look through the UUID Map.
+  if (sys::fs::is_directory(SymbolMapPath) && !Map.getUUID().empty()) {
+    uuid_string_t UUIDString;
+    uuid_unparse_upper((const uint8_t *)Map.getUUID().data(), UUIDString);
+
+    SmallString<256> PlistPath(
+        sys::path::parent_path(sys::path::parent_path(InputFile)));
+    sys::path::append(PlistPath, StringRef(UUIDString).str() + ".plist");
+
+    CFStringRef plistFile = CFStringCreateWithCString(
+        kCFAllocatorDefault, PlistPath.c_str(), kCFStringEncodingUTF8);
+    CFURLRef fileURL = CFURLCreateWithFileSystemPath(
+        kCFAllocatorDefault, plistFile, kCFURLPOSIXPathStyle, false);
+    CFReadStreamRef resourceData =
+        CFReadStreamCreateWithFile(kCFAllocatorDefault, fileURL);
+    if (resourceData) {
+      CFReadStreamOpen(resourceData);
+      CFDictionaryRef plist = (CFDictionaryRef)CFPropertyListCreateWithStream(
+          kCFAllocatorDefault, resourceData, 0, kCFPropertyListImmutable,
+          nullptr, nullptr);
+
+      if (plist) {
+        if (CFDictionaryContainsKey(plist, CFSTR("DBGOriginalUUID"))) {
+          CFStringRef OldUUID = (CFStringRef)CFDictionaryGetValue(
+              plist, CFSTR("DBGOriginalUUID"));
+
+          StringRef UUID(CFStringGetCStringPtr(OldUUID, kCFStringEncodingUTF8));
+          SmallString<256> BCSymbolMapPath(SymbolMapPath);
+          sys::path::append(BCSymbolMapPath, UUID.str() + ".bcsymbolmap");
+          SymbolMapPath = BCSymbolMapPath.str();
+        }
+        CFRelease(plist);
+      }
+      CFReadStreamClose(resourceData);
+      CFRelease(resourceData);
+    }
+    CFRelease(fileURL);
+    CFRelease(plistFile);
+  }
+#endif
+
+  if (sys::fs::is_directory(SymbolMapPath)) {
+    SymbolMapPath += (Twine("/") + sys::path::filename(InputFile) + "-" +
+                      MachOUtils::getArchName(Map.getTriple().getArchName()) +
+                      ".bcsymbolmap")
+                         .str();
+  }
+
+  auto ErrOrMemBuffer = MemoryBuffer::getFile(SymbolMapPath);
+  if (auto EC = ErrOrMemBuffer.getError()) {
+    WithColor::warning() << SymbolMapPath << ": " << EC.message()
+                         << ": not unobfuscating.\n";
+    return {};
+  }
+
+  std::vector<std::string> UnobfuscatedStrings;
+  auto &MemBuf = **ErrOrMemBuffer;
+  StringRef Data(MemBuf.getBufferStart(),
+                 MemBuf.getBufferEnd() - MemBuf.getBufferStart());
+  StringRef LHS;
+  std::tie(LHS, Data) = Data.split('\n');
+  bool MangleNames = false;
+
+  // Check version string first.
+  if (!LHS.startswith("BCSymbolMap Version:")) {
+    // Version string not present, warns but try to parse it.
+    WithColor::warning() << SymbolMapPath
+                         << " is missing version string: assuming 1.0.\n";
+    UnobfuscatedStrings.emplace_back(LHS);
+  } else if (LHS.equals("BCSymbolMap Version: 1.0")) {
+    MangleNames = true;
+  } else if (LHS.equals("BCSymbolMap Version: 2.0")) {
+    MangleNames = false;
+  } else {
+    StringRef VersionNum;
+    std::tie(LHS, VersionNum) = LHS.split(':');
+    WithColor::warning() << SymbolMapPath
+                         << " has unsupported symbol map version" << VersionNum
+                         << ": not unobfuscating.\n";
+    return {};
+  }
+
+  while (!Data.empty()) {
+    std::tie(LHS, Data) = Data.split('\n');
+    UnobfuscatedStrings.emplace_back(LHS);
+  }
+
+  return SymbolMapTranslator(std::move(UnobfuscatedStrings), MangleNames);
+}
+
+} // namespace dsymutil
+} // namespace llvm
diff --git a/tools/dsymutil/SymbolMap.h b/tools/dsymutil/SymbolMap.h
new file mode 100644
index 0000000000000000000000000000000000000000..977de31a5a17ace277ff4a51784cdc3049d23e3e
--- /dev/null
+++ b/tools/dsymutil/SymbolMap.h
@@ -0,0 +1,53 @@
+//=- tools/dsymutil/SymbolMap.h -----------------------------------*- C++ -*-=//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_TOOLS_DSYMUTIL_SYMBOLMAP_H
+#define LLVM_TOOLS_DSYMUTIL_SYMBOLMAP_H
+
+#include "llvm/ADT/StringRef.h"
+
+#include <string>
+#include <vector>
+
+namespace llvm {
+namespace dsymutil {
+class DebugMap;
+
+/// Callable class to unobfuscate strings based on a BCSymbolMap.
+class SymbolMapTranslator {
+public:
+  SymbolMapTranslator() : MangleNames(false) {}
+
+  SymbolMapTranslator(std::vector<std::string> UnobfuscatedStrings,
+                      bool MangleNames)
+      : UnobfuscatedStrings(std::move(UnobfuscatedStrings)),
+        MangleNames(MangleNames) {}
+
+  StringRef operator()(StringRef Input);
+
+  operator bool() const { return !UnobfuscatedStrings.empty(); }
+
+private:
+  std::vector<std::string> UnobfuscatedStrings;
+  bool MangleNames;
+};
+
+/// Class to initialize SymbolMapTranslators from a BCSymbolMap.
+class SymbolMapLoader {
+public:
+  SymbolMapLoader(std::string SymbolMap) : SymbolMap(std::move(SymbolMap)) {}
+
+  SymbolMapTranslator Load(StringRef InputFile, const DebugMap &Map) const;
+
+private:
+  const std::string SymbolMap;
+};
+} // namespace dsymutil
+} // namespace llvm
+
+#endif // LLVM_TOOLS_DSYMUTIL_SYMBOLMAP_H
diff --git a/tools/dsymutil/dsymutil.cpp b/tools/dsymutil/dsymutil.cpp
index 5fe40678ca9acd20af8d17905f51d47491461ed7..14f9a0e5137f66f2a5169c744710e47daa6a2970 100644
--- a/tools/dsymutil/dsymutil.cpp
+++ b/tools/dsymutil/dsymutil.cpp
@@ -1,9 +1,8 @@
 //===- dsymutil.cpp - Debug info dumping utility for llvm -----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -59,6 +58,8 @@ static opt<std::string>
     OutputFileOpt("o",
                   desc("Specify the output file. default: <input file>.dwarf"),
                   value_desc("filename"), cat(DsymCategory));
+static alias OutputFileOptA("out", desc("Alias for -o"),
+                            aliasopt(OutputFileOpt));
 
 static opt<std::string> OsoPrependPath(
     "oso-prepend-path",
@@ -100,6 +101,11 @@ static opt<bool> Update(
     init(false), cat(DsymCategory));
 static alias UpdateA("u", desc("Alias for --update"), aliasopt(Update));
 
+static opt<std::string> SymbolMap(
+    "symbol-map",
+    desc("Updates the existing dSYMs inplace using symbol map specified."),
+    value_desc("bcsymbolmap"), cat(DsymCategory));
+
 static cl::opt<AccelTableKind> AcceleratorTable(
     "accelerator", cl::desc("Output accelerator tables."),
     cl::values(clEnumValN(AccelTableKind::Default, "Default",
@@ -273,8 +279,11 @@ static bool verify(llvm::StringRef OutputFile, llvm::StringRef Arch) {
 }
 
 static Expected<std::string> getOutputFileName(llvm::StringRef InputFile) {
+  if (OutputFileOpt == "-")
+    return OutputFileOpt;
+
   // When updating, do in place replacement.
-  if (OutputFileOpt.empty() && Update)
+  if (OutputFileOpt.empty() && (Update || !SymbolMap.empty()))
     return InputFile;
 
   // If a flat dSYM has been requested, things are pretty simple.
@@ -325,6 +334,9 @@ static Expected<LinkOptions> getOptions() {
   Options.PrependPath = OsoPrependPath;
   Options.TheAccelTableKind = AcceleratorTable;
 
+  if (!SymbolMap.empty())
+    Options.Update = true;
+
   if (Assembly)
     Options.FileType = OutputFileType::Assembly;
 
@@ -443,6 +455,13 @@ int main(int argc, char **argv) {
     return 1;
   }
 
+  if (InputFiles.size() > 1 && !SymbolMap.empty() &&
+      !llvm::sys::fs::is_directory(SymbolMap)) {
+    WithColor::error() << "when unobfuscating multiple files, --symbol-map "
+                       << "needs to point to a directory.\n";
+    return 1;
+  }
+
   if (getenv("RC_DEBUG_OPTIONS"))
     PaperTrailWarnings = true;
 
@@ -457,6 +476,8 @@ int main(int argc, char **argv) {
       return 1;
     }
 
+  SymbolMapLoader SymMapLoader(SymbolMap);
+
   for (auto &InputFile : *InputsOrErr) {
     // Dump the symbol table for each input file and requested arch
     if (DumpStab) {
@@ -511,6 +532,9 @@ int main(int argc, char **argv) {
       if (DumpDebugMap)
         continue;
 
+      if (!SymbolMap.empty())
+        OptionsOrErr->Translator = SymMapLoader.Load(InputFile, *Map);
+
       if (Map->begin() == Map->end())
         WithColor::warning()
             << "no debug symbols in executable (-arch "
diff --git a/tools/dsymutil/dsymutil.h b/tools/dsymutil/dsymutil.h
index 556b2d65e1a72ab931f608a0dac20c7727788c16..ac70bdaa82b71d9161ba661ca1eb5f3eba22569d 100644
--- a/tools/dsymutil/dsymutil.h
+++ b/tools/dsymutil/dsymutil.h
@@ -1,9 +1,8 @@
 //===- tools/dsymutil/dsymutil.h - dsymutil high-level functionality ------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/tools/gold/CMakeLists.txt b/tools/gold/CMakeLists.txt
index d2580329acabd23ae17d9224a37c16fa01eeb35a..72f76558c088c4d9925ddafba38156994a61b46e 100644
--- a/tools/gold/CMakeLists.txt
+++ b/tools/gold/CMakeLists.txt
@@ -11,7 +11,7 @@ if( LLVM_ENABLE_PIC AND LLVM_BINUTILS_INCDIR )
      IPO
      )
 
-  add_llvm_loadable_module(LLVMgold
+  add_llvm_library(LLVMgold MODULE
     gold-plugin.cpp
     )
 
diff --git a/tools/gold/gold-plugin.cpp b/tools/gold/gold-plugin.cpp
index 8ffa5877ba665c7dbfe9c9f0bdc7737a3c4b8a84..626fb5f636347eaaebe44ae587541d35a45ca797 100644
--- a/tools/gold/gold-plugin.cpp
+++ b/tools/gold/gold-plugin.cpp
@@ -1,9 +1,8 @@
 //===-- gold-plugin.cpp - Plugin to gold for Link Time Optimization  ------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -128,6 +127,7 @@ namespace options {
     OT_NORMAL,
     OT_DISABLE,
     OT_BC_ONLY,
+    OT_ASM_ONLY,
     OT_SAVE_TEMPS
   };
   static OutputType TheOutputType = OT_NORMAL;
@@ -229,6 +229,8 @@ namespace options {
       TheOutputType = OT_SAVE_TEMPS;
     } else if (opt == "disable-output") {
       TheOutputType = OT_DISABLE;
+    } else if (opt == "emit-asm") {
+      TheOutputType = OT_ASM_ONLY;
     } else if (opt == "thinlto") {
       thinlto = true;
     } else if (opt == "thinlto-index-only") {
@@ -882,6 +884,9 @@ static std::unique_ptr<LTO> createLTO(IndexWriteCallback OnIndexWrite,
     check(Conf.addSaveTemps(output_name + ".",
                             /* UseInputModulePath */ true));
     break;
+  case options::OT_ASM_ONLY:
+    Conf.CGFileType = TargetMachine::CGFT_AssemblyFile;
+    break;
   }
 
   if (!options::sample_profile.empty())
@@ -1009,6 +1014,8 @@ static std::vector<std::pair<SmallString<128>, bool>> runLTO() {
     Filename = options::obj_path;
   else if (options::TheOutputType == options::OT_SAVE_TEMPS)
     Filename = output_name + ".o";
+  else if (options::TheOutputType == options::OT_ASM_ONLY)
+    Filename = output_name;
   bool SaveTemps = !Filename.empty();
 
   size_t MaxTasks = Lto->getMaxTasks();
@@ -1057,7 +1064,8 @@ static ld_plugin_status allSymbolsReadHook() {
   std::vector<std::pair<SmallString<128>, bool>> Files = runLTO();
 
   if (options::TheOutputType == options::OT_DISABLE ||
-      options::TheOutputType == options::OT_BC_ONLY)
+      options::TheOutputType == options::OT_BC_ONLY ||
+      options::TheOutputType == options::OT_ASM_ONLY)
     return LDPS_OK;
 
   if (options::thinlto_index_only) {
@@ -1082,6 +1090,7 @@ static ld_plugin_status all_symbols_read_hook(void) {
   llvm_shutdown();
 
   if (options::TheOutputType == options::OT_BC_ONLY ||
+      options::TheOutputType == options::OT_ASM_ONLY ||
       options::TheOutputType == options::OT_DISABLE) {
     if (options::TheOutputType == options::OT_DISABLE) {
       // Remove the output file here since ld.bfd creates the output file
diff --git a/tools/llc/LLVMBuild.txt b/tools/llc/LLVMBuild.txt
index 014dcac176106ad3d5fbc07a28b1926435e8de88..53fdeaf8199196de2afde3fdf370bd37e346cbdf 100644
--- a/tools/llc/LLVMBuild.txt
+++ b/tools/llc/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./tools/llc/LLVMBuild.txt --------------------------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/tools/llc/llc.cpp b/tools/llc/llc.cpp
index 2329fb3e87c973558766b2bb6e538d2d359bc2f4..ae51eef9e9ccfa570237e313be55474872386e0a 100644
--- a/tools/llc/llc.cpp
+++ b/tools/llc/llc.cpp
@@ -1,9 +1,8 @@
 //===-- llc.cpp - Implement the LLVM Native Code Generator ----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/tools/lli/ChildTarget/LLVMBuild.txt b/tools/lli/ChildTarget/LLVMBuild.txt
index daf6df113242de26d57f2e86b5fdac4f30c4f82b..7c3d813aa71eaa57cb192c98b897bbe0e8fce736 100644
--- a/tools/lli/ChildTarget/LLVMBuild.txt
+++ b/tools/lli/ChildTarget/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./tools/lli/ChildTarget/LLVMBuild.txt --------------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/tools/lli/LLVMBuild.txt b/tools/lli/LLVMBuild.txt
index 9d889bf4c2e671249e01ac8ba1cee70610064ab9..133543a26fce4cf1d369e2ccebccdbfc8682dbb6 100644
--- a/tools/lli/LLVMBuild.txt
+++ b/tools/lli/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./tools/lli/LLVMBuild.txt --------------------------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/tools/lli/RemoteJITUtils.h b/tools/lli/RemoteJITUtils.h
index 944881070c7034e2267a802d4ca82bef45a450a7..de7e5ed3b0bfd7dfbfa8a7d726da38118b854bf7 100644
--- a/tools/lli/RemoteJITUtils.h
+++ b/tools/lli/RemoteJITUtils.h
@@ -1,9 +1,8 @@
 //===-- RemoteJITUtils.h - Utilities for remote-JITing with LLI -*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/tools/lli/lli.cpp b/tools/lli/lli.cpp
index e4a7462a6352ad488a7717fd0167d52550307238..36d7d98265e5b3ff510e6008be9acbe78be6c961 100644
--- a/tools/lli/lli.cpp
+++ b/tools/lli/lli.cpp
@@ -1,9 +1,8 @@
 //===- lli.cpp - LLVM Interpreter / Dynamic compiler ----------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -35,7 +34,6 @@
 #include "llvm/IR/LLVMContext.h"
 #include "llvm/IR/Module.h"
 #include "llvm/IR/Type.h"
-#include "llvm/IR/TypeBuilder.h"
 #include "llvm/IR/Verifier.h"
 #include "llvm/IRReader/IRReader.h"
 #include "llvm/Object/Archive.h"
@@ -317,23 +315,18 @@ static void addCygMingExtraModule(ExecutionEngine &EE, LLVMContext &Context,
   M->setTargetTriple(TargetTripleStr);
 
   // Create an empty function named "__main".
-  Function *Result;
-  if (TargetTriple.isArch64Bit()) {
-    Result = Function::Create(
-      TypeBuilder<int64_t(void), false>::get(Context),
-      GlobalValue::ExternalLinkage, "__main", M.get());
-  } else {
-    Result = Function::Create(
-      TypeBuilder<int32_t(void), false>::get(Context),
-      GlobalValue::ExternalLinkage, "__main", M.get());
-  }
-  BasicBlock *BB = BasicBlock::Create(Context, "__main", Result);
-  Builder.SetInsertPoint(BB);
-  Value *ReturnVal;
+  Type *ReturnTy;
   if (TargetTriple.isArch64Bit())
-    ReturnVal = ConstantInt::get(Context, APInt(64, 0));
+    ReturnTy = Type::getInt64Ty(Context);
   else
-    ReturnVal = ConstantInt::get(Context, APInt(32, 0));
+    ReturnTy = Type::getInt32Ty(Context);
+  Function *Result =
+      Function::Create(FunctionType::get(ReturnTy, {}, false),
+                       GlobalValue::ExternalLinkage, "__main", M.get());
+
+  BasicBlock *BB = BasicBlock::Create(Context, "__main", Result);
+  Builder.SetInsertPoint(BB);
+  Value *ReturnVal = ConstantInt::get(ReturnTy, 0);
   Builder.CreateRet(ReturnVal);
 
   // Add this new module to the ExecutionEngine.
@@ -602,8 +595,8 @@ int main(int argc, char **argv, char * const *envp) {
   if (!RemoteMCJIT) {
     // If the program doesn't explicitly call exit, we will need the Exit
     // function later on to make an explicit call, so get the function now.
-    Constant *Exit = Mod->getOrInsertFunction("exit", Type::getVoidTy(Context),
-                                                      Type::getInt32Ty(Context));
+    FunctionCallee Exit = Mod->getOrInsertFunction(
+        "exit", Type::getVoidTy(Context), Type::getInt32Ty(Context));
 
     // Run static constructors.
     if (!ForceInterpreter) {
@@ -627,19 +620,21 @@ int main(int argc, char **argv, char * const *envp) {
 
     // If the program didn't call exit explicitly, we should call it now.
     // This ensures that any atexit handlers get called correctly.
-    if (Function *ExitF = dyn_cast<Function>(Exit)) {
-      std::vector<GenericValue> Args;
-      GenericValue ResultGV;
-      ResultGV.IntVal = APInt(32, Result);
-      Args.push_back(ResultGV);
-      EE->runFunction(ExitF, Args);
-      WithColor::error(errs(), argv[0]) << "exit(" << Result << ") returned!\n";
-      abort();
-    } else {
-      WithColor::error(errs(), argv[0])
-          << "exit defined with wrong prototype!\n";
-      abort();
+    if (Function *ExitF =
+            dyn_cast<Function>(Exit.getCallee()->stripPointerCasts())) {
+      if (ExitF->getFunctionType() == Exit.getFunctionType()) {
+        std::vector<GenericValue> Args;
+        GenericValue ResultGV;
+        ResultGV.IntVal = APInt(32, Result);
+        Args.push_back(ResultGV);
+        EE->runFunction(ExitF, Args);
+        WithColor::error(errs(), argv[0])
+            << "exit(" << Result << ") returned!\n";
+        abort();
+      }
     }
+    WithColor::error(errs(), argv[0]) << "exit defined with wrong prototype!\n";
+    abort();
   } else {
     // else == "if (RemoteMCJIT)"
 
diff --git a/tools/llvm-ar/LLVMBuild.txt b/tools/llvm-ar/LLVMBuild.txt
index 236b4659f7b9c4566def0f04d423f237f8115ff2..7247851829c3f0f8b66b8ae37eed9390b97f1d61 100644
--- a/tools/llvm-ar/LLVMBuild.txt
+++ b/tools/llvm-ar/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./tools/llvm-ar/LLVMBuild.txt ----------------------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/tools/llvm-ar/llvm-ar.cpp b/tools/llvm-ar/llvm-ar.cpp
index 5ab8ae13d3e0eeb5d70fec29272ffc7600beef35..0119ec502e55d2aaab9ab56632f2d560bb9fba9b 100644
--- a/tools/llvm-ar/llvm-ar.cpp
+++ b/tools/llvm-ar/llvm-ar.cpp
@@ -1,9 +1,8 @@
 //===-- llvm-ar.cpp - LLVM archive librarian utility ----------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -33,6 +32,7 @@
 #include "llvm/Support/StringSaver.h"
 #include "llvm/Support/TargetSelect.h"
 #include "llvm/Support/ToolOutputFile.h"
+#include "llvm/Support/WithColor.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/ToolDrivers/llvm-dlltool/DlltoolDriver.h"
 #include "llvm/ToolDrivers/llvm-lib/LibDriver.h"
@@ -115,7 +115,7 @@ void printHelpMessage() {
 
 // Show the error message and exit.
 LLVM_ATTRIBUTE_NORETURN static void fail(Twine Error) {
-  errs() << ToolName << ": " << Error << ".\n";
+  WithColor::error(errs(), ToolName) << Error << ".\n";
   printHelpMessage();
   exit(1);
 }
@@ -125,7 +125,7 @@ static void failIfError(std::error_code EC, Twine Context = "") {
     return;
 
   std::string ContextStr = Context.str();
-  if (ContextStr == "")
+  if (ContextStr.empty())
     fail(EC.message());
   fail(Context + ": " + EC.message());
 }
@@ -136,7 +136,7 @@ static void failIfError(Error E, Twine Context = "") {
 
   handleAllErrors(std::move(E), [&](const llvm::ErrorInfoBase &EIB) {
     std::string ContextStr = Context.str();
-    if (ContextStr == "")
+    if (ContextStr.empty())
       fail(EIB.message());
     fail(Context + ": " + EIB.message());
   });
@@ -196,7 +196,7 @@ static std::vector<StringRef> Members;
 // Extract the member filename from the command line for the [relpos] argument
 // associated with a, b, and i modifiers
 static void getRelPos() {
-  if (PositionalArgs.size() == 0)
+  if (PositionalArgs.empty())
     fail("Expected [relpos] for a, b, or i modifier");
   RelPos = PositionalArgs[0];
   PositionalArgs.erase(PositionalArgs.begin());
@@ -204,7 +204,7 @@ static void getRelPos() {
 
 // Get the archive file name from the command line
 static void getArchive() {
-  if (PositionalArgs.size() == 0)
+  if (PositionalArgs.empty())
     fail("An archive name must be specified");
   ArchiveName = PositionalArgs[0];
   PositionalArgs.erase(PositionalArgs.begin());
@@ -221,7 +221,7 @@ std::vector<std::unique_ptr<object::Archive>> Archives;
 
 static object::Archive &readLibrary(const Twine &Library) {
   auto BufOrErr = MemoryBuffer::getFile(Library, -1, false);
-  failIfError(BufOrErr.getError(), "Could not open library");
+  failIfError(BufOrErr.getError(), "Could not open library " + Library);
   ArchiveBuffers.push_back(std::move(*BufOrErr));
   auto LibOrErr =
       object::Archive::create(ArchiveBuffers.back()->getMemBufferRef());
@@ -429,8 +429,9 @@ static void doDisplayTable(StringRef Name, const object::Archive::Child &C) {
   }
 
   if (C.getParent()->isThin()) {
-    outs() << sys::path::parent_path(ArchiveName);
-    outs() << '/';
+    StringRef ParentDir = sys::path::parent_path(ArchiveName);
+    if (!ParentDir.empty())
+      outs() << ParentDir << '/';
   }
   outs() << Name << "\n";
 }
@@ -532,56 +533,57 @@ static void performReadOperation(ArchiveOperation Operation,
   if (Members.empty())
     return;
   for (StringRef Name : Members)
-    errs() << Name << " was not found\n";
+    WithColor::error(errs(), ToolName) << "'" << Name << "' was not found\n";
   exit(1);
 }
 
-static void addMember(std::vector<NewArchiveMember> &Members,
-                      StringRef FileName, int Pos = -1) {
-  Expected<NewArchiveMember> NMOrErr =
-      NewArchiveMember::getFile(FileName, Deterministic);
-  failIfError(NMOrErr.takeError(), FileName);
-
-  // Use the basename of the object path for the member name.
-  NMOrErr->MemberName = sys::path::filename(NMOrErr->MemberName);
-
-  if (Pos == -1)
-    Members.push_back(std::move(*NMOrErr));
-  else
-    Members[Pos] = std::move(*NMOrErr);
-}
-
-static void addMember(std::vector<NewArchiveMember> &Members,
-                      const object::Archive::Child &M, int Pos = -1) {
+static void addChildMember(std::vector<NewArchiveMember> &Members,
+                           const object::Archive::Child &M,
+                           bool FlattenArchive = false) {
   if (Thin && !M.getParent()->isThin())
     fail("Cannot convert a regular archive to a thin one");
   Expected<NewArchiveMember> NMOrErr =
       NewArchiveMember::getOldMember(M, Deterministic);
   failIfError(NMOrErr.takeError());
-  if (Pos == -1)
-    Members.push_back(std::move(*NMOrErr));
-  else
-    Members[Pos] = std::move(*NMOrErr);
+  if (FlattenArchive &&
+      identify_magic(NMOrErr->Buf->getBuffer()) == file_magic::archive) {
+    Expected<std::string> FileNameOrErr = M.getFullName();
+    failIfError(FileNameOrErr.takeError());
+    object::Archive &Lib = readLibrary(*FileNameOrErr);
+    // When creating thin archives, only flatten if the member is also thin.
+    if (!Thin || Lib.isThin()) {
+      Error Err = Error::success();
+      // Only Thin archives are recursively flattened.
+      for (auto &Child : Lib.children(Err))
+        addChildMember(Members, Child, /*FlattenArchive=*/Thin);
+      failIfError(std::move(Err));
+      return;
+    }
+  }
+  Members.push_back(std::move(*NMOrErr));
 }
 
-static void addLibMember(std::vector<NewArchiveMember> &Members,
-                         StringRef FileName) {
+static void addMember(std::vector<NewArchiveMember> &Members,
+                      StringRef FileName, bool FlattenArchive = false) {
   Expected<NewArchiveMember> NMOrErr =
       NewArchiveMember::getFile(FileName, Deterministic);
   failIfError(NMOrErr.takeError(), FileName);
-  if (identify_magic(NMOrErr->Buf->getBuffer()) == file_magic::archive) {
+  if (FlattenArchive &&
+      identify_magic(NMOrErr->Buf->getBuffer()) == file_magic::archive) {
     object::Archive &Lib = readLibrary(FileName);
-    Error Err = Error::success();
-
-    for (auto &Child : Lib.children(Err))
-      addMember(Members, Child);
-
-    failIfError(std::move(Err));
-  } else {
-    // Use the basename of the object path for the member name.
-    NMOrErr->MemberName = sys::path::filename(NMOrErr->MemberName);
-    Members.push_back(std::move(*NMOrErr));
+    // When creating thin archives, only flatten if the member is also thin.
+    if (!Thin || Lib.isThin()) {
+      Error Err = Error::success();
+      // Only Thin archives are recursively flattened.
+      for (auto &Child : Lib.children(Err))
+        addChildMember(Members, Child, /*FlattenArchive=*/Thin);
+      failIfError(std::move(Err));
+      return;
+    }
   }
+  // Use the basename of the object path for the member name.
+  NMOrErr->MemberName = sys::path::filename(NMOrErr->MemberName);
+  Members.push_back(std::move(*NMOrErr));
 }
 
 enum InsertAction {
@@ -670,7 +672,7 @@ computeNewArchiveMembers(ArchiveOperation Operation,
           computeInsertAction(Operation, Child, Name, MemberI);
       switch (Action) {
       case IA_AddOldMember:
-        addMember(Ret, Child);
+        addChildMember(Ret, Child);
         break;
       case IA_AddNewMember:
         addMember(Ret, *MemberI);
@@ -678,7 +680,7 @@ computeNewArchiveMembers(ArchiveOperation Operation,
       case IA_Delete:
         break;
       case IA_MoveOldMember:
-        addMember(Moved, Child);
+        addChildMember(Moved, Child);
         break;
       case IA_MoveNewMember:
         addMember(Moved, *MemberI);
@@ -709,17 +711,16 @@ computeNewArchiveMembers(ArchiveOperation Operation,
   if (AddLibrary) {
     assert(Operation == QuickAppend);
     for (auto &Member : Members)
-      addLibMember(Ret, Member);
+      addMember(Ret, Member, /*FlattenArchive=*/true);
     return Ret;
   }
 
-  for (unsigned I = 0; I != Members.size(); ++I)
-    Ret.insert(Ret.begin() + InsertPos, NewArchiveMember());
-  Pos = InsertPos;
-  for (auto &Member : Members) {
-    addMember(Ret, Member, Pos);
-    ++Pos;
-  }
+  std::vector<NewArchiveMember> NewMembers;
+  for (auto &Member : Members)
+    addMember(NewMembers, Member, /*FlattenArchive=*/Thin);
+  Ret.reserve(Ret.size() + NewMembers.size());
+  std::move(NewMembers.begin(), NewMembers.end(),
+            std::inserter(Ret, std::next(Ret.begin(), InsertPos)));
 
   return Ret;
 }
@@ -760,11 +761,11 @@ static void performWriteOperation(ArchiveOperation Operation,
     else if (OldArchive)
       Kind = OldArchive->kind();
     else if (NewMembersP)
-      Kind = NewMembersP->size() ? getKindFromMember(NewMembersP->front())
-                                 : getDefaultForHost();
+      Kind = !NewMembersP->empty() ? getKindFromMember(NewMembersP->front())
+                                   : getDefaultForHost();
     else
-      Kind = NewMembers.size() ? getKindFromMember(NewMembers.front())
-                               : getDefaultForHost();
+      Kind = !NewMembers.empty() ? getKindFromMember(NewMembers.front())
+                                 : getDefaultForHost();
     break;
   case GNU:
     Kind = object::Archive::K_GNU;
@@ -853,7 +854,8 @@ static int performOperation(ArchiveOperation Operation,
   } else {
     if (!Create) {
       // Produce a warning if we should and we're creating the archive
-      errs() << ToolName << ": creating " << ArchiveName << "\n";
+      WithColor::warning(errs(), ToolName)
+          << "creating " << ArchiveName << "\n";
     }
   }
 
@@ -897,7 +899,7 @@ static void runMRIScript() {
       {
         Error Err = Error::success();
         for (auto &Member : Lib.children(Err))
-          addMember(NewMembers, Member);
+          addChildMember(NewMembers, Member);
         failIfError(std::move(Err));
       }
       break;
diff --git a/tools/llvm-as-fuzzer/llvm-as-fuzzer.cpp b/tools/llvm-as-fuzzer/llvm-as-fuzzer.cpp
index 52deca6e9cd779581213ea65fe08dce2a7ac5071..8a13da55747c8e07a058a9fd2f90f8d0c4fade35 100644
--- a/tools/llvm-as-fuzzer/llvm-as-fuzzer.cpp
+++ b/tools/llvm-as-fuzzer/llvm-as-fuzzer.cpp
@@ -1,9 +1,8 @@
 //===--- fuzz-llvm-as.cpp - Fuzzer for llvm-as using lib/Fuzzer -----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/tools/llvm-as/LLVMBuild.txt b/tools/llvm-as/LLVMBuild.txt
index 542470bbdd8b91f085c5a879f89cdc4ad9a24289..2e037e860a65534d73b68e5b9b249c6a242985d4 100644
--- a/tools/llvm-as/LLVMBuild.txt
+++ b/tools/llvm-as/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./tools/llvm-as/LLVMBuild.txt ----------------------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/tools/llvm-as/llvm-as.cpp b/tools/llvm-as/llvm-as.cpp
index bb4233aa9ba0f831a2a28ac8d868c70bef729790..8f8d10ff572807db9b88ad77eebf4f6580f85e4b 100644
--- a/tools/llvm-as/llvm-as.cpp
+++ b/tools/llvm-as/llvm-as.cpp
@@ -1,9 +1,8 @@
 //===--- llvm-as.cpp - The low-level LLVM assembler -----------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/tools/llvm-bcanalyzer/LLVMBuild.txt b/tools/llvm-bcanalyzer/LLVMBuild.txt
index ee77a7d33ab69749662ede20c0c2e32292dae44d..281d60246e9d23b5cafceb045f1ac09ba709b741 100644
--- a/tools/llvm-bcanalyzer/LLVMBuild.txt
+++ b/tools/llvm-bcanalyzer/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./tools/llvm-bcanalyzer/LLVMBuild.txt --------------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/tools/llvm-bcanalyzer/llvm-bcanalyzer.cpp b/tools/llvm-bcanalyzer/llvm-bcanalyzer.cpp
index 789a666cb41a2530c494af443d9843d29ee4d622..092dbd13675aa5c43949fedda70297d223115a96 100644
--- a/tools/llvm-bcanalyzer/llvm-bcanalyzer.cpp
+++ b/tools/llvm-bcanalyzer/llvm-bcanalyzer.cpp
@@ -1,9 +1,8 @@
 //===-- llvm-bcanalyzer.cpp - Bitcode Analyzer --------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -292,6 +291,7 @@ static const char *GetCodeName(unsigned CodeID, unsigned BlockID,
       STRINGIFY_CODE(FUNC_CODE, INST_LOADATOMIC)
       STRINGIFY_CODE(FUNC_CODE, INST_STOREATOMIC)
       STRINGIFY_CODE(FUNC_CODE, INST_CMPXCHG)
+      STRINGIFY_CODE(FUNC_CODE, INST_CALLBR)
     }
   case bitc::VALUE_SYMTAB_BLOCK_ID:
     switch (CodeID) {
diff --git a/tools/llvm-c-test/attributes.c b/tools/llvm-c-test/attributes.c
index c7bc0d3c87b258a41f876a5284ccebd9e5f6099b..826cb094e226f64356ea197b64b2ebdbb476ac7b 100644
--- a/tools/llvm-c-test/attributes.c
+++ b/tools/llvm-c-test/attributes.c
@@ -1,9 +1,9 @@
 /*===-- attributes.c - tool for testing libLLVM and llvm-c API ------------===*\
 |*                                                                            *|
-|*                     The LLVM Compiler Infrastructure                       *|
-|*                                                                            *|
-|* This file is distributed under the University of Illinois Open Source      *|
-|* License. See LICENSE.TXT for details.                                      *|
+|* Part of the LLVM Project, under the Apache License v2.0 with LLVM          *|
+|* Exceptions.                                                                *|
+|* See https://llvm.org/LICENSE.txt for license information.                  *|
+|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception                    *|
 |*                                                                            *|
 |*===----------------------------------------------------------------------===*|
 |*                                                                            *|
diff --git a/tools/llvm-c-test/calc.c b/tools/llvm-c-test/calc.c
index 4c273cbf70a26857fd8a1ef09197fd5826d4069e..bd7aff41ba4fcab655cfe3f362db16265b0dc817 100644
--- a/tools/llvm-c-test/calc.c
+++ b/tools/llvm-c-test/calc.c
@@ -1,9 +1,9 @@
 /*===-- calc.c - tool for testing libLLVM and llvm-c API ------------------===*\
 |*                                                                            *|
-|*                     The LLVM Compiler Infrastructure                       *|
-|*                                                                            *|
-|* This file is distributed under the University of Illinois Open Source      *|
-|* License. See LICENSE.TXT for details.                                      *|
+|* Part of the LLVM Project, under the Apache License v2.0 with LLVM          *|
+|* Exceptions.                                                                *|
+|* See https://llvm.org/LICENSE.txt for license information.                  *|
+|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception                    *|
 |*                                                                            *|
 |*===----------------------------------------------------------------------===*|
 |*                                                                            *|
diff --git a/tools/llvm-c-test/debuginfo.c b/tools/llvm-c-test/debuginfo.c
index fbd41e97d7a1f7e5004d761f388f8cd583bdb086..ff96037d4af10460460f80b569f68ae77a81e311 100644
--- a/tools/llvm-c-test/debuginfo.c
+++ b/tools/llvm-c-test/debuginfo.c
@@ -1,9 +1,9 @@
 /*===-- debuginfo.c - tool for testing libLLVM and llvm-c API -------------===*\
 |*                                                                            *|
-|*                     The LLVM Compiler Infrastructure                       *|
-|*                                                                            *|
-|* This file is distributed under the University of Illinois Open Source      *|
-|* License. See LICENSE.TXT for details.                                      *|
+|* Part of the LLVM Project, under the Apache License v2.0 with LLVM          *|
+|* Exceptions.                                                                *|
+|* See https://llvm.org/LICENSE.txt for license information.                  *|
+|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception                    *|
 |*                                                                            *|
 |*===----------------------------------------------------------------------===*|
 |*                                                                            *|
diff --git a/tools/llvm-c-test/diagnostic.c b/tools/llvm-c-test/diagnostic.c
index 16d5174732360c12db34560de73efa13fa376788..d759097e8e962806af628785a243b7c23341b8ac 100644
--- a/tools/llvm-c-test/diagnostic.c
+++ b/tools/llvm-c-test/diagnostic.c
@@ -1,9 +1,8 @@
 //===-- diagnostic.cpp - tool for testing libLLVM and llvm-c API ----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/tools/llvm-c-test/disassemble.c b/tools/llvm-c-test/disassemble.c
index 7d6f1783aed9eb69bb70988ab7b1656d93f0bd5a..43706c266782e89d7f5fc0a99c73e17e0c06a301 100644
--- a/tools/llvm-c-test/disassemble.c
+++ b/tools/llvm-c-test/disassemble.c
@@ -1,9 +1,9 @@
 /*===-- disassemble.c - tool for testing libLLVM and llvm-c API -----------===*\
 |*                                                                            *|
-|*                     The LLVM Compiler Infrastructure                       *|
-|*                                                                            *|
-|* This file is distributed under the University of Illinois Open Source      *|
-|* License. See LICENSE.TXT for details.                                      *|
+|* Part of the LLVM Project, under the Apache License v2.0 with LLVM          *|
+|* Exceptions.                                                                *|
+|* See https://llvm.org/LICENSE.txt for license information.                  *|
+|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception                    *|
 |*                                                                            *|
 |*===----------------------------------------------------------------------===*|
 |*                                                                            *|
diff --git a/tools/llvm-c-test/echo.cpp b/tools/llvm-c-test/echo.cpp
index db926e8aceaac9c18ee6277bbc7c14a6e3cef687..8341b11aa45fc68eff93534028f70c07b7ae08f4 100644
--- a/tools/llvm-c-test/echo.cpp
+++ b/tools/llvm-c-test/echo.cpp
@@ -1,9 +1,8 @@
 //===-- echo.cpp - tool for testing libLLVM and llvm-c API ----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -940,7 +939,7 @@ AliasDecl:
   if (!Begin) {
     if (End != nullptr)
       report_fatal_error("Range has an end but no beginning");
-    goto NamedMDDecl;
+    goto GlobalIFuncDecl;
   }
 
   Cur = Begin;
@@ -968,6 +967,41 @@ AliasDecl:
     Cur = Next;
   }
 
+GlobalIFuncDecl:
+  Begin = LLVMGetFirstGlobalIFunc(Src);
+  End = LLVMGetLastGlobalIFunc(Src);
+  if (!Begin) {
+    if (End != nullptr)
+      report_fatal_error("Range has an end but no beginning");
+    goto NamedMDDecl;
+  }
+
+  Cur = Begin;
+  Next = nullptr;
+  while (true) {
+    size_t NameLen;
+    const char *Name = LLVMGetValueName2(Cur, &NameLen);
+    if (LLVMGetNamedGlobalIFunc(M, Name, NameLen))
+      report_fatal_error("Global ifunc already cloned");
+    LLVMTypeRef CurType = TypeCloner(M).Clone(LLVMGlobalGetValueType(Cur));
+    // FIXME: Allow NULL resolver.
+    LLVMAddGlobalIFunc(M, Name, NameLen,
+                       CurType, /*addressSpace*/ 0, LLVMGetUndef(CurType));
+
+    Next = LLVMGetNextGlobalIFunc(Cur);
+    if (Next == nullptr) {
+      if (Cur != End)
+        report_fatal_error("");
+      break;
+    }
+
+    LLVMValueRef Prev = LLVMGetPreviousGlobalIFunc(Next);
+    if (Prev != Cur)
+      report_fatal_error("Next.Previous global is not Current");
+
+    Cur = Next;
+  }
+
 NamedMDDecl:
   LLVMNamedMDNodeRef BeginMD = LLVMGetFirstNamedMetadata(Src);
   LLVMNamedMDNodeRef EndMD = LLVMGetLastNamedMetadata(Src);
@@ -1115,7 +1149,7 @@ AliasClone:
   if (!Begin) {
     if (End != nullptr)
       report_fatal_error("Range has an end but no beginning");
-    goto NamedMDClone;
+    goto GlobalIFuncClone;
   }
 
   Cur = Begin;
@@ -1148,6 +1182,45 @@ AliasClone:
     Cur = Next;
   }
 
+GlobalIFuncClone:
+  Begin = LLVMGetFirstGlobalIFunc(Src);
+  End = LLVMGetLastGlobalIFunc(Src);
+  if (!Begin) {
+    if (End != nullptr)
+      report_fatal_error("Range has an end but no beginning");
+    goto NamedMDClone;
+  }
+
+  Cur = Begin;
+  Next = nullptr;
+  while (true) {
+    size_t NameLen;
+    const char *Name = LLVMGetValueName2(Cur, &NameLen);
+    LLVMValueRef IFunc = LLVMGetNamedGlobalIFunc(M, Name, NameLen);
+    if (!IFunc)
+      report_fatal_error("Global ifunc must have been declared already");
+
+    if (LLVMValueRef Resolver = LLVMGetGlobalIFuncResolver(Cur)) {
+      LLVMSetGlobalIFuncResolver(IFunc, clone_constant(Resolver, M));
+    }
+
+    LLVMSetLinkage(IFunc, LLVMGetLinkage(Cur));
+    LLVMSetUnnamedAddress(IFunc, LLVMGetUnnamedAddress(Cur));
+
+    Next = LLVMGetNextGlobalIFunc(Cur);
+    if (Next == nullptr) {
+      if (Cur != End)
+        report_fatal_error("Last global alias does not match End");
+      break;
+    }
+
+    LLVMValueRef Prev = LLVMGetPreviousGlobalIFunc(Next);
+    if (Prev != Cur)
+      report_fatal_error("Next.Previous global alias is not Current");
+
+    Cur = Next;
+  }
+
 NamedMDClone:
   LLVMNamedMDNodeRef BeginMD = LLVMGetFirstNamedMetadata(Src);
   LLVMNamedMDNodeRef EndMD = LLVMGetLastNamedMetadata(Src);
diff --git a/tools/llvm-c-test/helpers.c b/tools/llvm-c-test/helpers.c
index 9af88bd8be90cc1ec59fb1160694880e01653fef..56537656d4320ecc0af244132f8defb2057d6a53 100644
--- a/tools/llvm-c-test/helpers.c
+++ b/tools/llvm-c-test/helpers.c
@@ -1,9 +1,9 @@
 /*===-- helpers.c - tool for testing libLLVM and llvm-c API ---------------===*\
 |*                                                                            *|
-|*                     The LLVM Compiler Infrastructure                       *|
-|*                                                                            *|
-|* This file is distributed under the University of Illinois Open Source      *|
-|* License. See LICENSE.TXT for details.                                      *|
+|* Part of the LLVM Project, under the Apache License v2.0 with LLVM          *|
+|* Exceptions.                                                                *|
+|* See https://llvm.org/LICENSE.txt for license information.                  *|
+|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception                    *|
 |*                                                                            *|
 |*===----------------------------------------------------------------------===*|
 |*                                                                            *|
diff --git a/tools/llvm-c-test/include-all.c b/tools/llvm-c-test/include-all.c
index 17b9917a8761219b76d6fe35e3acd62ccb79e71e..b6e80a098d1054a1633b9501abb2363d9387fbf8 100644
--- a/tools/llvm-c-test/include-all.c
+++ b/tools/llvm-c-test/include-all.c
@@ -1,9 +1,9 @@
 /*===-- include-all.c - tool for testing libLLVM and llvm-c API -----------===*\
 |*                                                                            *|
-|*                     The LLVM Compiler Infrastructure                       *|
-|*                                                                            *|
-|* This file is distributed under the University of Illinois Open Source      *|
-|* License. See LICENSE.TXT for details.                                      *|
+|* Part of the LLVM Project, under the Apache License v2.0 with LLVM          *|
+|* Exceptions.                                                                *|
+|* See https://llvm.org/LICENSE.txt for license information.                  *|
+|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception                    *|
 |*                                                                            *|
 |*===----------------------------------------------------------------------===*|
 |*                                                                            *|
diff --git a/tools/llvm-c-test/llvm-c-test.h b/tools/llvm-c-test/llvm-c-test.h
index cf9a0f99de64e1e57e4aa9eb85e1984e38c5bf80..b828a827c55a2568f4cc1cc2525eb29ca01a7665 100644
--- a/tools/llvm-c-test/llvm-c-test.h
+++ b/tools/llvm-c-test/llvm-c-test.h
@@ -1,9 +1,9 @@
 /*===-- llvm-c-test.h - tool for testing libLLVM and llvm-c API -----------===*\
 |*                                                                            *|
-|*                     The LLVM Compiler Infrastructure                       *|
-|*                                                                            *|
-|* This file is distributed under the University of Illinois Open Source      *|
-|* License. See LICENSE.TXT for details.                                      *|
+|* Part of the LLVM Project, under the Apache License v2.0 with LLVM          *|
+|* Exceptions.                                                                *|
+|* See https://llvm.org/LICENSE.txt for license information.                  *|
+|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception                    *|
 |*                                                                            *|
 |*===----------------------------------------------------------------------===*|
 |*                                                                            *|
diff --git a/tools/llvm-c-test/main.c b/tools/llvm-c-test/main.c
index 60ab7f0f90099b3f0f62205f01058b942f08ed67..0be48930f0b6562f9b4c281f3761b13a6f4e67ef 100644
--- a/tools/llvm-c-test/main.c
+++ b/tools/llvm-c-test/main.c
@@ -1,9 +1,9 @@
 /*===-- main.c - tool for testing libLLVM and llvm-c API ------------------===*\
 |*                                                                            *|
-|*                     The LLVM Compiler Infrastructure                       *|
-|*                                                                            *|
-|* This file is distributed under the University of Illinois Open Source      *|
-|* License. See LICENSE.TXT for details.                                      *|
+|* Part of the LLVM Project, under the Apache License v2.0 with LLVM          *|
+|* Exceptions.                                                                *|
+|* See https://llvm.org/LICENSE.txt for license information.                  *|
+|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception                    *|
 |*                                                                            *|
 |*===----------------------------------------------------------------------===*|
 |*                                                                            *|
diff --git a/tools/llvm-c-test/metadata.c b/tools/llvm-c-test/metadata.c
index 89215b8ebcbd84c7b490945979b5ea17af9f688b..4e1aa9cd99d881c8dcc3a9240a644a37b55a20ef 100644
--- a/tools/llvm-c-test/metadata.c
+++ b/tools/llvm-c-test/metadata.c
@@ -1,9 +1,9 @@
 /*===-- object.c - tool for testing libLLVM and llvm-c API ----------------===*\
 |*                                                                            *|
-|*                     The LLVM Compiler Infrastructure                       *|
-|*                                                                            *|
-|* This file is distributed under the University of Illinois Open Source      *|
-|* License. See LICENSE.TXT for details.                                      *|
+|* Part of the LLVM Project, under the Apache License v2.0 with LLVM          *|
+|* Exceptions.                                                                *|
+|* See https://llvm.org/LICENSE.txt for license information.                  *|
+|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception                    *|
 |*                                                                            *|
 |*===----------------------------------------------------------------------===*|
 |*                                                                            *|
diff --git a/tools/llvm-c-test/module.c b/tools/llvm-c-test/module.c
index cbb44d0bd15e4dad8469ab2b06824b5c01f33cb7..9fc86cfe5404b31372a9716bf74b5cecb3923e19 100644
--- a/tools/llvm-c-test/module.c
+++ b/tools/llvm-c-test/module.c
@@ -1,9 +1,9 @@
 /*===-- module.c - tool for testing libLLVM and llvm-c API ----------------===*\
 |*                                                                            *|
-|*                     The LLVM Compiler Infrastructure                       *|
-|*                                                                            *|
-|* This file is distributed under the University of Illinois Open Source      *|
-|* License. See LICENSE.TXT for details.                                      *|
+|* Part of the LLVM Project, under the Apache License v2.0 with LLVM          *|
+|* Exceptions.                                                                *|
+|* See https://llvm.org/LICENSE.txt for license information.                  *|
+|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception                    *|
 |*                                                                            *|
 |*===----------------------------------------------------------------------===*|
 |*                                                                            *|
diff --git a/tools/llvm-c-test/object.c b/tools/llvm-c-test/object.c
index 809ad54f8721c82dc7df69f5e88099f59e892d77..278911b0bc6f1b68d0b32c8d5eb535944eee29de 100644
--- a/tools/llvm-c-test/object.c
+++ b/tools/llvm-c-test/object.c
@@ -1,9 +1,9 @@
 /*===-- object.c - tool for testing libLLVM and llvm-c API ----------------===*\
 |*                                                                            *|
-|*                     The LLVM Compiler Infrastructure                       *|
-|*                                                                            *|
-|* This file is distributed under the University of Illinois Open Source      *|
-|* License. See LICENSE.TXT for details.                                      *|
+|* Part of the LLVM Project, under the Apache License v2.0 with LLVM          *|
+|* Exceptions.                                                                *|
+|* See https://llvm.org/LICENSE.txt for license information.                  *|
+|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception                    *|
 |*                                                                            *|
 |*===----------------------------------------------------------------------===*|
 |*                                                                            *|
diff --git a/tools/llvm-c-test/targets.c b/tools/llvm-c-test/targets.c
index f2a9e924a74a3e3169b8dcc47d7368edf0dbb9c5..685598e1a5eaa4b79c594c5bb0996fc134859e0c 100644
--- a/tools/llvm-c-test/targets.c
+++ b/tools/llvm-c-test/targets.c
@@ -1,9 +1,9 @@
 /*===-- targets.c - tool for testing libLLVM and llvm-c API ---------------===*\
 |*                                                                            *|
-|*                     The LLVM Compiler Infrastructure                       *|
-|*                                                                            *|
-|* This file is distributed under the University of Illinois Open Source      *|
-|* License. See LICENSE.TXT for details.                                      *|
+|* Part of the LLVM Project, under the Apache License v2.0 with LLVM          *|
+|* Exceptions.                                                                *|
+|* See https://llvm.org/LICENSE.txt for license information.                  *|
+|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception                    *|
 |*                                                                            *|
 |*===----------------------------------------------------------------------===*|
 |*                                                                            *|
diff --git a/tools/llvm-cat/LLVMBuild.txt b/tools/llvm-cat/LLVMBuild.txt
index 3b2cc635ecc73c63f7a26423d4669f813c9f7055..1a85099674eda2024166b56e09d8232abda8d1a7 100644
--- a/tools/llvm-cat/LLVMBuild.txt
+++ b/tools/llvm-cat/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./tools/llvm-cat/LLVMBuild.txt ---------------------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/tools/llvm-cat/llvm-cat.cpp b/tools/llvm-cat/llvm-cat.cpp
index 99973a22eb9666f08f758fbc40a2f07beac8aefa..5ae2799b60c87840cb4dec68978965a1fd4fb38c 100644
--- a/tools/llvm-cat/llvm-cat.cpp
+++ b/tools/llvm-cat/llvm-cat.cpp
@@ -1,9 +1,8 @@
 //===- llvm-cat.cpp - LLVM module concatenation utility -------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/tools/llvm-cfi-verify/LLVMBuild.txt b/tools/llvm-cfi-verify/LLVMBuild.txt
index d5e9323027289fcdc8c803ef29d0c7c25f83970f..63a5dc562959623f22f1e9eb4675757ba2f3f6f9 100644
--- a/tools/llvm-cfi-verify/LLVMBuild.txt
+++ b/tools/llvm-cfi-verify/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./tools/llvm-cfi-verify/LLVMBuild.txt --------------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/tools/llvm-cfi-verify/lib/FileAnalysis.cpp b/tools/llvm-cfi-verify/lib/FileAnalysis.cpp
index 8dc0802c46524cbc027f84225516371199ac2947..a3750a2735d53d6e9d4a0fa813badb1fccaeb509 100644
--- a/tools/llvm-cfi-verify/lib/FileAnalysis.cpp
+++ b/tools/llvm-cfi-verify/lib/FileAnalysis.cpp
@@ -1,9 +1,8 @@
 //===- FileAnalysis.cpp -----------------------------------------*- C++ -*-===//
 //
-//                      The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/tools/llvm-cfi-verify/lib/FileAnalysis.h b/tools/llvm-cfi-verify/lib/FileAnalysis.h
index 67421e80da37f3916c8077b1fe054b09be276bd7..d8031ed5f7da68b3637b7f0e5449f24594aa650c 100644
--- a/tools/llvm-cfi-verify/lib/FileAnalysis.h
+++ b/tools/llvm-cfi-verify/lib/FileAnalysis.h
@@ -1,9 +1,8 @@
 //===- FileAnalysis.h -------------------------------------------*- C++ -*-===//
 //
-//                      The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/tools/llvm-cfi-verify/lib/GraphBuilder.cpp b/tools/llvm-cfi-verify/lib/GraphBuilder.cpp
index 5b2bc6f0c3bb0cb7c33f305e46df90c9f0bba68d..1f8fccb32ff95d528aac8da31b2e8bc01636dcee 100644
--- a/tools/llvm-cfi-verify/lib/GraphBuilder.cpp
+++ b/tools/llvm-cfi-verify/lib/GraphBuilder.cpp
@@ -1,9 +1,8 @@
 //===- GraphBuilder.cpp -----------------------------------------*- C++ -*-===//
 //
-//                      The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/tools/llvm-cfi-verify/lib/GraphBuilder.h b/tools/llvm-cfi-verify/lib/GraphBuilder.h
index 2bf07b53bf6d6dc25acc4597d48367f5f7a4ada8..c76e89a37ecc928d379579b19facd093c8a4bb1b 100644
--- a/tools/llvm-cfi-verify/lib/GraphBuilder.h
+++ b/tools/llvm-cfi-verify/lib/GraphBuilder.h
@@ -1,9 +1,8 @@
 //===- GraphBuilder.h -------------------------------------------*- C++ -*-===//
 //
-//                      The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/tools/llvm-cfi-verify/lib/LLVMBuild.txt b/tools/llvm-cfi-verify/lib/LLVMBuild.txt
index c0ae1905521a294c950b15567236b6e3a094700e..7083e35d99e74fcb1c751443b758b7c475423d1e 100644
--- a/tools/llvm-cfi-verify/lib/LLVMBuild.txt
+++ b/tools/llvm-cfi-verify/lib/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./tools/llvm-cfi-verify/lib/LLVMBuild.txt ----------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/tools/llvm-cfi-verify/llvm-cfi-verify.cpp b/tools/llvm-cfi-verify/llvm-cfi-verify.cpp
index 2fde74bd24c9beabb4073a771e85c2578d605982..9fff1a5ec38fc1accf9e6f7fb90f35b4c9a915d3 100644
--- a/tools/llvm-cfi-verify/llvm-cfi-verify.cpp
+++ b/tools/llvm-cfi-verify/llvm-cfi-verify.cpp
@@ -1,9 +1,8 @@
 //===-- llvm-cfi-verify.cpp - CFI Verification tool for LLVM --------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/tools/llvm-config/BuildVariables.inc.in b/tools/llvm-config/BuildVariables.inc.in
index f201e1f7bff0e8cb79e8b4d3c9ad9d41c697bea3..3a24d3e974e1fb7c97335b8622181305b494aa87 100644
--- a/tools/llvm-config/BuildVariables.inc.in
+++ b/tools/llvm-config/BuildVariables.inc.in
@@ -1,9 +1,8 @@
 //===-- BuildVariables.inc.in - llvm-config build variables -*- C++ -*-----===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/tools/llvm-config/llvm-config.cpp b/tools/llvm-config/llvm-config.cpp
index bec89fef98caa09bf72494dc2609203cf6c5df25..7ef7c46a262703220fa52875f1be422d45b2c724 100644
--- a/tools/llvm-config/llvm-config.cpp
+++ b/tools/llvm-config/llvm-config.cpp
@@ -1,9 +1,8 @@
 //===-- llvm-config.cpp - LLVM project configuration utility --------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/tools/llvm-cov/CodeCoverage.cpp b/tools/llvm-cov/CodeCoverage.cpp
index 1dc6eedecc11c48e4b2d2d022de8a30ad4ae84cb..43311a41c426ee0e7ee885bc045f6b039fee6347 100644
--- a/tools/llvm-cov/CodeCoverage.cpp
+++ b/tools/llvm-cov/CodeCoverage.cpp
@@ -1,9 +1,8 @@
 //===- CodeCoverage.cpp - Coverage tool based on profiling instrumentation-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -691,7 +690,7 @@ int CodeCoverageTool::run(Command Cmd, int argc, const char **argv) {
       PathRemapping = EquivPair;
 
     // If a demangler is supplied, check if it exists and register it.
-    if (DemanglerOpts.size()) {
+    if (!DemanglerOpts.empty()) {
       auto DemanglerPathOrErr = sys::findProgramByName(DemanglerOpts[0]);
       if (!DemanglerPathOrErr) {
         error("Could not find the demangler!",
diff --git a/tools/llvm-cov/CoverageExporter.h b/tools/llvm-cov/CoverageExporter.h
index b226d68813d9cc731da43874e8aff60baa07f45f..751e55dc091618df8458e88a194742f714342008 100644
--- a/tools/llvm-cov/CoverageExporter.h
+++ b/tools/llvm-cov/CoverageExporter.h
@@ -1,9 +1,8 @@
 //===- CoverageExporter.h - Code coverage exporter ------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/tools/llvm-cov/CoverageExporterJson.cpp b/tools/llvm-cov/CoverageExporterJson.cpp
index 22243f8e2c3e839f4de5ebf9f9c40ff2d26b3b61..e6361d48b8e1a47eb4fd6bce6c6b9584517e5ebe 100644
--- a/tools/llvm-cov/CoverageExporterJson.cpp
+++ b/tools/llvm-cov/CoverageExporterJson.cpp
@@ -1,9 +1,8 @@
 //===- CoverageExporterJson.cpp - Code coverage export --------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/tools/llvm-cov/CoverageExporterJson.h b/tools/llvm-cov/CoverageExporterJson.h
index c37c86b42be9859a0b74f0d6337b4a245360ffba..c19475005552f0fdca481523092222f8eaefece3 100644
--- a/tools/llvm-cov/CoverageExporterJson.h
+++ b/tools/llvm-cov/CoverageExporterJson.h
@@ -1,9 +1,8 @@
 //===- CoverageExporterJson.h - Code coverage JSON exporter ---------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/tools/llvm-cov/CoverageExporterLcov.cpp b/tools/llvm-cov/CoverageExporterLcov.cpp
index d149ba1a4c87423834885dc0fc5cc04b4ac21667..6e017743eec8b57d3f49568aaea30c983a295093 100644
--- a/tools/llvm-cov/CoverageExporterLcov.cpp
+++ b/tools/llvm-cov/CoverageExporterLcov.cpp
@@ -1,9 +1,8 @@
 //===- CoverageExporterLcov.cpp - Code coverage export --------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/tools/llvm-cov/CoverageExporterLcov.h b/tools/llvm-cov/CoverageExporterLcov.h
index 539b2dacd38424d86e9a0da745d0a3e0f12624c5..e8a260bf493707a4af69bd179b5f89f39f0f72a0 100644
--- a/tools/llvm-cov/CoverageExporterLcov.h
+++ b/tools/llvm-cov/CoverageExporterLcov.h
@@ -1,9 +1,8 @@
 //===- CoverageExporterLcov.h - Code coverage lcov exporter ---------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/tools/llvm-cov/CoverageFilters.cpp b/tools/llvm-cov/CoverageFilters.cpp
index 4dd0f552c7e0f8a38bbd2a0793e1057005bb512a..ca241e386e87e309e91bbc3280a62a1ab27ec487 100644
--- a/tools/llvm-cov/CoverageFilters.cpp
+++ b/tools/llvm-cov/CoverageFilters.cpp
@@ -1,9 +1,8 @@
 //===- CoverageFilters.cpp - Function coverage mapping filters ------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/tools/llvm-cov/CoverageFilters.h b/tools/llvm-cov/CoverageFilters.h
index 6424ca5a8081d283570f9ac4f86a329323945261..ce56e1607111733227328f75df647bdaebe768b0 100644
--- a/tools/llvm-cov/CoverageFilters.h
+++ b/tools/llvm-cov/CoverageFilters.h
@@ -1,9 +1,8 @@
 //===- CoverageFilters.h - Function coverage mapping filters --------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/tools/llvm-cov/CoverageReport.cpp b/tools/llvm-cov/CoverageReport.cpp
index 607a3ceb30cb820ae5e6c4381adbc51e66dbac02..82259542c59706e5efce2f7125457fd052eb0b46 100644
--- a/tools/llvm-cov/CoverageReport.cpp
+++ b/tools/llvm-cov/CoverageReport.cpp
@@ -1,9 +1,8 @@
 //===- CoverageReport.cpp - Code coverage report -------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/tools/llvm-cov/CoverageReport.h b/tools/llvm-cov/CoverageReport.h
index 4a6527e9fe5d4e04ef23e69ddce6c11e78fd0d11..f9a092f510b504616c03c5a6030fad8582696be8 100644
--- a/tools/llvm-cov/CoverageReport.h
+++ b/tools/llvm-cov/CoverageReport.h
@@ -1,9 +1,8 @@
 //===- CoverageReport.h - Code coverage report ----------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/tools/llvm-cov/CoverageSummaryInfo.cpp b/tools/llvm-cov/CoverageSummaryInfo.cpp
index 7847a2abf48c73ccded1307ce592521892dd0645..1029f7784040da3e98504257134ce9aae8877714 100644
--- a/tools/llvm-cov/CoverageSummaryInfo.cpp
+++ b/tools/llvm-cov/CoverageSummaryInfo.cpp
@@ -1,9 +1,8 @@
 //===- CoverageSummaryInfo.cpp - Coverage summary for function/file -------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/tools/llvm-cov/CoverageSummaryInfo.h b/tools/llvm-cov/CoverageSummaryInfo.h
index 0845e2ce2e776fe2e6f686d91cf64d900f4d700e..97beacb26d07c7ecbda23e7dae5ea0c585b5bb91 100644
--- a/tools/llvm-cov/CoverageSummaryInfo.h
+++ b/tools/llvm-cov/CoverageSummaryInfo.h
@@ -1,9 +1,8 @@
 //===- CoverageSummaryInfo.h - Coverage summary for function/file ---------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/tools/llvm-cov/CoverageViewOptions.h b/tools/llvm-cov/CoverageViewOptions.h
index c8a47286002794aa509582031c2fdc1e1708b937..0a3e211cabc5d73021c2fbd6f324fef42f7503c5 100644
--- a/tools/llvm-cov/CoverageViewOptions.h
+++ b/tools/llvm-cov/CoverageViewOptions.h
@@ -1,9 +1,8 @@
 //===- CoverageViewOptions.h - Code coverage display options -------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/tools/llvm-cov/LLVMBuild.txt b/tools/llvm-cov/LLVMBuild.txt
index 33f51fb2ed5a9ff0c92762720363517ee8c70441..e6aff83ac55522ddeda6bbc80b4c81ebd3ef275e 100644
--- a/tools/llvm-cov/LLVMBuild.txt
+++ b/tools/llvm-cov/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./tools/llvm-cov/LLVMBuild.txt ---------------------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/tools/llvm-cov/RenderingSupport.h b/tools/llvm-cov/RenderingSupport.h
index 2cfe24919142119cd00e52aa7d3fd4af9cfa5c80..0674fbac9a3cab8d7f8934c75fc4784663584a5d 100644
--- a/tools/llvm-cov/RenderingSupport.h
+++ b/tools/llvm-cov/RenderingSupport.h
@@ -1,9 +1,8 @@
 //===- RenderingSupport.h - output stream rendering support functions  ----===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/tools/llvm-cov/SourceCoverageView.cpp b/tools/llvm-cov/SourceCoverageView.cpp
index 775322b2de2fef7bab9100130c754f20db7edf08..4d4795fbe402a42343614fccac3af93830b99137 100644
--- a/tools/llvm-cov/SourceCoverageView.cpp
+++ b/tools/llvm-cov/SourceCoverageView.cpp
@@ -1,9 +1,8 @@
 //===- SourceCoverageView.cpp - Code coverage view for source code --------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
@@ -31,7 +30,7 @@ void CoveragePrinter::StreamDestructor::operator()(raw_ostream *OS) const {
 std::string CoveragePrinter::getOutputPath(StringRef Path, StringRef Extension,
                                            bool InToplevel,
                                            bool Relative) const {
-  assert(Extension.size() && "The file extension may not be empty");
+  assert(!Extension.empty() && "The file extension may not be empty");
 
   SmallString<256> FullPath;
 
diff --git a/tools/llvm-cov/SourceCoverageView.h b/tools/llvm-cov/SourceCoverageView.h
index e3a2f9e5c0b43fe864f90c52d7e3dda429e33f1e..9ae92844365133d9954fd34fc102eac0186e126f 100644
--- a/tools/llvm-cov/SourceCoverageView.h
+++ b/tools/llvm-cov/SourceCoverageView.h
@@ -1,9 +1,8 @@
 //===- SourceCoverageView.h - Code coverage view for source code ----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
diff --git a/tools/llvm-cov/SourceCoverageViewHTML.cpp b/tools/llvm-cov/SourceCoverageViewHTML.cpp
index acb67aa5cfc7ff8e6ee63e8f6c98fb64e2386954..e3332245f9c8e8a3703f71360d90b26d4437d67f 100644
--- a/tools/llvm-cov/SourceCoverageViewHTML.cpp
+++ b/tools/llvm-cov/SourceCoverageViewHTML.cpp
@@ -1,9 +1,8 @@
 //===- SourceCoverageViewHTML.cpp - A html code coverage view -------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
@@ -54,7 +53,7 @@ std::string escape(StringRef Str, const CoverageViewOptions &Opts) {
 std::string tag(const std::string &Name, const std::string &Str,
                 const std::string &ClassName = "") {
   std::string Tag = "<" + Name;
-  if (ClassName != "")
+  if (!ClassName.empty())
     Tag += " class='" + ClassName + "'";
   return Tag + ">" + Str + "</" + Name + ">";
 }
diff --git a/tools/llvm-cov/SourceCoverageViewHTML.h b/tools/llvm-cov/SourceCoverageViewHTML.h
index cb41fcaf37b934df92e78d3adf4a8838029546fc..9834040008a650dc814da8737d235429999dcd9a 100644
--- a/tools/llvm-cov/SourceCoverageViewHTML.h
+++ b/tools/llvm-cov/SourceCoverageViewHTML.h
@@ -1,9 +1,8 @@
 //===- SourceCoverageViewHTML.h - A html code coverage view ---------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
diff --git a/tools/llvm-cov/SourceCoverageViewText.cpp b/tools/llvm-cov/SourceCoverageViewText.cpp
index aac70baed6132d0d9be82d4cc3cf3320dc7ce0a5..fcabee2ee69d2e526ebd629135307bb15d5275d3 100644
--- a/tools/llvm-cov/SourceCoverageViewText.cpp
+++ b/tools/llvm-cov/SourceCoverageViewText.cpp
@@ -1,9 +1,8 @@
 //===- SourceCoverageViewText.cpp - A text-based code coverage view -------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
diff --git a/tools/llvm-cov/SourceCoverageViewText.h b/tools/llvm-cov/SourceCoverageViewText.h
index a46f35cc6495e68244930c39710d97bc07936a89..c8c4632c3b9d80ba5f1fa5279c18e9ee41da5b30 100644
--- a/tools/llvm-cov/SourceCoverageViewText.h
+++ b/tools/llvm-cov/SourceCoverageViewText.h
@@ -1,9 +1,8 @@
 //===- SourceCoverageViewText.h - A text-based code coverage view ---------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
diff --git a/tools/llvm-cov/TestingSupport.cpp b/tools/llvm-cov/TestingSupport.cpp
index 16a1c266529968ab0cdf5648a13a246b125df528..e2b86c0b79c5e70d80947b57e62aaed118b0ed9b 100644
--- a/tools/llvm-cov/TestingSupport.cpp
+++ b/tools/llvm-cov/TestingSupport.cpp
@@ -1,9 +1,8 @@
 //===- TestingSupport.cpp - Convert objects files into test files --------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/tools/llvm-cov/gcov.cpp b/tools/llvm-cov/gcov.cpp
index 7776f2aa9a6844efe0ae52112a53bad704d60ce3..58ef2cf0219a12042c9a949bc06cfd874d23d2e2 100644
--- a/tools/llvm-cov/gcov.cpp
+++ b/tools/llvm-cov/gcov.cpp
@@ -1,9 +1,8 @@
 //===- gcov.cpp - GCOV compatible LLVM coverage tool ----------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/tools/llvm-cov/llvm-cov.cpp b/tools/llvm-cov/llvm-cov.cpp
index 4c3b574451c35058f9bb935f1c087b184695456d..172ec9f3cedffffdfa49bc1c7906651726fc3937 100644
--- a/tools/llvm-cov/llvm-cov.cpp
+++ b/tools/llvm-cov/llvm-cov.cpp
@@ -1,9 +1,8 @@
 //===- llvm-cov.cpp - LLVM coverage tool ----------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/tools/llvm-cvtres/LLVMBuild.txt b/tools/llvm-cvtres/LLVMBuild.txt
index 78c598b75e3142fc590baf83b3f2d662695e7d55..d24c947e4e777aee379b85263cdcfb500d8a7693 100644
--- a/tools/llvm-cvtres/LLVMBuild.txt
+++ b/tools/llvm-cvtres/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./tools/llvm-cvtres/LLVMBuild.txt ------------------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/tools/llvm-cvtres/llvm-cvtres.cpp b/tools/llvm-cvtres/llvm-cvtres.cpp
index 53989c8191faaf710f19841e87d22a54e044fde0..bb00d49cf617806917588922cc607df34a02e26d 100644
--- a/tools/llvm-cvtres/llvm-cvtres.cpp
+++ b/tools/llvm-cvtres/llvm-cvtres.cpp
@@ -1,9 +1,8 @@
 //===- llvm-cvtres.cpp - Serialize .res files into .obj ---------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/tools/llvm-cxxdump/Error.cpp b/tools/llvm-cxxdump/Error.cpp
index 54207fad32af235237fdeb8a1f302d5d00fd2aa0..25317820409ce9531b89e99769b02c8d97bbde9b 100644
--- a/tools/llvm-cxxdump/Error.cpp
+++ b/tools/llvm-cxxdump/Error.cpp
@@ -1,9 +1,8 @@
 //===- Error.cpp - system_error extensions for llvm-cxxdump -----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/tools/llvm-cxxdump/Error.h b/tools/llvm-cxxdump/Error.h
index 7caf6d6447c9e204b3f8860e37b60916db14bbe6..439902fa3803fcfc2c9f919c00dd84d57966ce3f 100644
--- a/tools/llvm-cxxdump/Error.h
+++ b/tools/llvm-cxxdump/Error.h
@@ -1,9 +1,8 @@
 //===- Error.h - system_error extensions for llvm-cxxdump -------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/tools/llvm-cxxdump/LLVMBuild.txt b/tools/llvm-cxxdump/LLVMBuild.txt
index 4e088157799f3f5d50edb54601620eb8bf6c57f6..a46c78bd3efc9c60a0a75ec48fd6fb50fe791690 100644
--- a/tools/llvm-cxxdump/LLVMBuild.txt
+++ b/tools/llvm-cxxdump/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./tools/llvm-cxxdump/LLVMBuild.txt -----------------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/tools/llvm-cxxdump/llvm-cxxdump.cpp b/tools/llvm-cxxdump/llvm-cxxdump.cpp
index 7594066a395dd79c0d77cda047f463ef8f5eb0c7..c850815cd2f1b7106c2f0086d7d8052c47fca999 100644
--- a/tools/llvm-cxxdump/llvm-cxxdump.cpp
+++ b/tools/llvm-cxxdump/llvm-cxxdump.cpp
@@ -1,9 +1,8 @@
 //===- llvm-cxxdump.cpp - Dump C++ data in an Object File -------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/tools/llvm-cxxdump/llvm-cxxdump.h b/tools/llvm-cxxdump/llvm-cxxdump.h
index daa05cb2ca0a6d37daf974961defaee797a958d7..739cfe481a4b39ef9c3165523cac0002bbb20b51 100644
--- a/tools/llvm-cxxdump/llvm-cxxdump.h
+++ b/tools/llvm-cxxdump/llvm-cxxdump.h
@@ -1,9 +1,8 @@
 //===-- llvm-cxxdump.h ------------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/tools/llvm-cxxfilt/llvm-cxxfilt.cpp b/tools/llvm-cxxfilt/llvm-cxxfilt.cpp
index afc1e4a8d128b2f2c3d9968209f3dbb5e997efe3..821c705b95beb50c277dff794ebf3df8f0c39146 100644
--- a/tools/llvm-cxxfilt/llvm-cxxfilt.cpp
+++ b/tools/llvm-cxxfilt/llvm-cxxfilt.cpp
@@ -1,12 +1,12 @@
 //===-- llvm-c++filt.cpp --------------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
+#include "llvm/ADT/StringExtras.h"
 #include "llvm/Demangle/Demangle.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/InitLLVM.h"
@@ -52,7 +52,7 @@ static cl::alias TypesShort("t", cl::desc("alias for --types"),
 static cl::list<std::string>
 Decorated(cl::Positional, cl::desc("<mangled>"), cl::ZeroOrMore);
 
-static void demangle(llvm::raw_ostream &OS, const std::string &Mangled) {
+static std::string demangle(llvm::raw_ostream &OS, const std::string &Mangled) {
   int Status;
 
   const char *Decorated = Mangled.c_str();
@@ -73,10 +73,28 @@ static void demangle(llvm::raw_ostream &OS, const std::string &Mangled) {
     Undecorated = itaniumDemangle(Decorated + 6, nullptr, nullptr, &Status);
   }
 
-  OS << (Undecorated ? Undecorated : Mangled) << '\n';
-  OS.flush();
-
+  std::string Result(Undecorated ? Undecorated : Mangled);
   free(Undecorated);
+  return Result;
+}
+
+// If 'Split' is true, then 'Mangled' is broken into individual words and each
+// word is demangled.  Otherwise, the entire string is treated as a single
+// mangled item.  The result is output to 'OS'.
+static void demangleLine(llvm::raw_ostream &OS, StringRef Mangled, bool Split) {
+  std::string Result;
+  if (Split) {
+    SmallVector<StringRef, 16> Words;
+    SplitString(Mangled, Words);
+    for (auto Word : Words)
+      Result += demangle(OS, Word) + ' ';
+    // Remove the trailing space character.
+    if (Result.back() == ' ')
+      Result.pop_back();
+  } else
+    Result = demangle(OS, Mangled);
+  OS << Result << '\n';
+  OS.flush();
 }
 
 int main(int argc, char **argv) {
@@ -86,10 +104,10 @@ int main(int argc, char **argv) {
 
   if (Decorated.empty())
     for (std::string Mangled; std::getline(std::cin, Mangled);)
-      demangle(llvm::outs(), Mangled);
+      demangleLine(llvm::outs(), Mangled, true);
   else
     for (const auto &Symbol : Decorated)
-      demangle(llvm::outs(), Symbol);
+      demangleLine(llvm::outs(), Symbol, false);
 
   return EXIT_SUCCESS;
 }
diff --git a/tools/llvm-cxxmap/LLVMBuild.txt b/tools/llvm-cxxmap/LLVMBuild.txt
index 2876c4a2d1f29fe32a590dbf13c17d676028f81a..92c4242da0f2d1b4da910f54bc5d935769847c2f 100644
--- a/tools/llvm-cxxmap/LLVMBuild.txt
+++ b/tools/llvm-cxxmap/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./tools/llvm-cxxmap/LLVMBuild.txt ------------------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/tools/llvm-cxxmap/llvm-cxxmap.cpp b/tools/llvm-cxxmap/llvm-cxxmap.cpp
index 39028cc8672323be1a7104dba19aa1d93799dba5..87d4d06bbc96df54f9f49f3b497fbd5e35aa869a 100644
--- a/tools/llvm-cxxmap/llvm-cxxmap.cpp
+++ b/tools/llvm-cxxmap/llvm-cxxmap.cpp
@@ -1,9 +1,8 @@
 //===- llvm-cxxmap.cpp ----------------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/tools/llvm-diff/DiffConsumer.cpp b/tools/llvm-diff/DiffConsumer.cpp
index ec189df275216f112856465acc99ad33d0ff86cb..f2a2e68b2a2bb2bdedcc452b47324408e37d06ce 100644
--- a/tools/llvm-diff/DiffConsumer.cpp
+++ b/tools/llvm-diff/DiffConsumer.cpp
@@ -1,9 +1,8 @@
 //===-- DiffConsumer.cpp - Difference Consumer ------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/tools/llvm-diff/DiffConsumer.h b/tools/llvm-diff/DiffConsumer.h
index 82f5ce598b44374b675158db260dc19876a5bde7..6cb8f2eb7eeb4fcb495a67ccfa2f149b6ab5eec5 100644
--- a/tools/llvm-diff/DiffConsumer.h
+++ b/tools/llvm-diff/DiffConsumer.h
@@ -1,9 +1,8 @@
 //===-- DiffConsumer.h - Difference Consumer --------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/tools/llvm-diff/DiffLog.cpp b/tools/llvm-diff/DiffLog.cpp
index 50c0c4cff2fce5e1338c87beba75579bcaa230dc..6484197521f23f69fc5a83cac849857c331cab92 100644
--- a/tools/llvm-diff/DiffLog.cpp
+++ b/tools/llvm-diff/DiffLog.cpp
@@ -1,9 +1,8 @@
 //===-- DiffLog.h - Difference Log Builder and accessories ------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/tools/llvm-diff/DiffLog.h b/tools/llvm-diff/DiffLog.h
index 8f28461afddea1db1d8b44f537dec90c01bdedc4..0c8952496155046efaac8b2a0f65a593a5a89626 100644
--- a/tools/llvm-diff/DiffLog.h
+++ b/tools/llvm-diff/DiffLog.h
@@ -1,9 +1,8 @@
 //===-- DiffLog.h - Difference Log Builder and accessories ------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/tools/llvm-diff/DifferenceEngine.cpp b/tools/llvm-diff/DifferenceEngine.cpp
index acff8bb3e89bef92e9126194f5efe162a8d02ac7..bc93ece864900bb8105a8c811b6f1f5a0f38b956 100644
--- a/tools/llvm-diff/DifferenceEngine.cpp
+++ b/tools/llvm-diff/DifferenceEngine.cpp
@@ -1,9 +1,8 @@
 //===-- DifferenceEngine.cpp - Structural function/module comparison ------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -68,7 +67,7 @@ public:
     unsigned NewSize = Storage.size() - 1;
     if (NewSize) {
       // Move the slot at the end to the beginning.
-      if (isPodLike<T>::value)
+      if (is_trivially_copyable<T>::value)
         Storage[0] = Storage[NewSize];
       else
         std::swap(Storage[0], Storage[NewSize]);
diff --git a/tools/llvm-diff/DifferenceEngine.h b/tools/llvm-diff/DifferenceEngine.h
index 7f084a377f0c7a1f4d127e19b20b73d5064d6bc5..da1b6526a6e2ddb073c035a4aa143f073c192c5e 100644
--- a/tools/llvm-diff/DifferenceEngine.h
+++ b/tools/llvm-diff/DifferenceEngine.h
@@ -1,9 +1,8 @@
 //===-- DifferenceEngine.h - Module comparator ------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/tools/llvm-diff/LLVMBuild.txt b/tools/llvm-diff/LLVMBuild.txt
index 5adfdc2bd6e9dc260552dfb55001764854dd0d0b..f7bba4773546ebda24a239e952be18970159946d 100644
--- a/tools/llvm-diff/LLVMBuild.txt
+++ b/tools/llvm-diff/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./tools/llvm-diff/LLVMBuild.txt --------------------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/tools/llvm-diff/llvm-diff.cpp b/tools/llvm-diff/llvm-diff.cpp
index e449d6994784e7694bf07f1b8697fcfa2f932558..aaf7989e2e3d1d9379679b524ff48a1b50728854 100644
--- a/tools/llvm-diff/llvm-diff.cpp
+++ b/tools/llvm-diff/llvm-diff.cpp
@@ -1,9 +1,8 @@
 //===-- llvm-diff.cpp - Module comparator command-line driver ---*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/tools/llvm-dis/LLVMBuild.txt b/tools/llvm-dis/LLVMBuild.txt
index 4525010c1fc92c1b37d826763a29090eba0d616c..ca8e11ee4304b04b919be51e27545d6b6e5a1c37 100644
--- a/tools/llvm-dis/LLVMBuild.txt
+++ b/tools/llvm-dis/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./tools/llvm-dis/LLVMBuild.txt ---------------------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/tools/llvm-dis/llvm-dis.cpp b/tools/llvm-dis/llvm-dis.cpp
index 8143a2a5a9343e6b5bd4740484d7c585d381da1c..3f337b874b166cb94f7ff64514625abfcb5bf9da 100644
--- a/tools/llvm-dis/llvm-dis.cpp
+++ b/tools/llvm-dis/llvm-dis.cpp
@@ -1,9 +1,8 @@
 //===-- llvm-dis.cpp - The low-level LLVM disassembler --------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/tools/llvm-dwarfdump/LLVMBuild.txt b/tools/llvm-dwarfdump/LLVMBuild.txt
index a9dca3e0cfff0677b9c423739b75303c101345b1..4eda51443213ab70dc5ca1454cd81a4c1cb85adc 100644
--- a/tools/llvm-dwarfdump/LLVMBuild.txt
+++ b/tools/llvm-dwarfdump/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./tools/llvm-dwarfdump/LLVMBuild.txt ---------------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/tools/llvm-dwarfdump/Statistics.cpp b/tools/llvm-dwarfdump/Statistics.cpp
index 5fe7e8b4615b8f5b3659ada76310353ea3c64a79..a01a5017c59f7994db487d7958407ee60f9cad48 100644
--- a/tools/llvm-dwarfdump/Statistics.cpp
+++ b/tools/llvm-dwarfdump/Statistics.cpp
@@ -23,6 +23,9 @@ struct PerFunctionStats {
   StringSet<> VarsInFunction;
   /// Compile units also cover a PC range, but have this flag set to false.
   bool IsFunction = false;
+  /// Verify function definition has PC addresses (for detecting when
+  /// a function has been inlined everywhere).
+  bool HasPCAddresses = false;
 };
 
 /// Holds accumulated global statistics about DIEs.
@@ -136,8 +139,10 @@ static void collectStatsForDie(DWARFDie Die, std::string FnPrefix,
     GlobalStats.ScopeBytesFromFirstDefinition += BytesInScope;
     assert(GlobalStats.ScopeBytesCovered <=
            GlobalStats.ScopeBytesFromFirstDefinition);
-  } else {
+  } else if (Die.getTag() == dwarf::DW_TAG_member) {
     FnStats.ConstantMembers++;
+  } else {
+    FnStats.TotalVarWithLoc += (unsigned)HasLoc;
   }
 }
 
@@ -164,6 +169,19 @@ static void collectStatsRecursive(DWARFDie Die, std::string FnPrefix,
     if (Die.find(dwarf::DW_AT_declaration))
       return;
 
+    // PC Ranges.
+    auto RangesOrError = Die.getAddressRanges();
+    if (!RangesOrError) {
+      llvm::consumeError(RangesOrError.takeError());
+      return;
+    }
+
+    auto Ranges = RangesOrError.get();
+    uint64_t BytesInThisScope = 0;
+    for (auto Range : Ranges)
+      BytesInThisScope += Range.HighPC - Range.LowPC;
+    ScopeLowPC = getLowPC(Die);
+
     // Count the function.
     if (!IsBlock) {
       StringRef Name = Die.getName(DINameKind::LinkageName);
@@ -175,23 +193,13 @@ static void collectStatsRecursive(DWARFDie Die, std::string FnPrefix,
         return;
       // We've seen an (inlined) instance of this function.
       auto &FnStats = FnStatMap[Name];
-      FnStats.NumFnInlined++;
+      if (IsInlinedFunction)
+        FnStats.NumFnInlined++;
       FnStats.IsFunction = true;
+      if (BytesInThisScope && !IsInlinedFunction)
+        FnStats.HasPCAddresses = true;
     }
 
-    // PC Ranges.
-    auto RangesOrError = Die.getAddressRanges();
-    if (!RangesOrError) {
-      llvm::consumeError(RangesOrError.takeError());
-      return;
-    }
-
-    auto Ranges = RangesOrError.get();
-    uint64_t BytesInThisScope = 0;
-    for (auto Range : Ranges)
-      BytesInThisScope += Range.HighPC - Range.LowPC;
-    ScopeLowPC = getLowPC(Die);
-
     if (BytesInThisScope) {
       BytesInScope = BytesInThisScope;
       if (IsFunction)
@@ -258,7 +266,7 @@ bool collectStatsForObjectFile(ObjectFile &Obj, DWARFContext &DICtx,
   /// The version number should be increased every time the algorithm is changed
   /// (including bug fixes). New metrics may be added without increasing the
   /// version.
-  unsigned Version = 1;
+  unsigned Version = 2;
   unsigned VarTotal = 0;
   unsigned VarUnique = 0;
   unsigned VarWithLoc = 0;
@@ -267,9 +275,12 @@ bool collectStatsForObjectFile(ObjectFile &Obj, DWARFContext &DICtx,
   for (auto &Entry : Statistics) {
     PerFunctionStats &Stats = Entry.getValue();
     unsigned TotalVars = Stats.VarsInFunction.size() * Stats.NumFnInlined;
+    // Count variables in concrete out-of-line functions and in global scope.
+    if (Stats.HasPCAddresses || !Stats.IsFunction)
+      TotalVars += Stats.VarsInFunction.size();
     unsigned Constants = Stats.ConstantMembers;
     VarWithLoc += Stats.TotalVarWithLoc + Constants;
-    VarTotal += TotalVars + Constants;
+    VarTotal += TotalVars;
     VarUnique += Stats.VarsInFunction.size();
     LLVM_DEBUG(for (auto &V : Stats.VarsInFunction) llvm::dbgs()
                << Entry.getKey() << ": " << V.getKey() << "\n");
diff --git a/tools/llvm-dwarfdump/fuzzer/llvm-dwarfdump-fuzzer.cpp b/tools/llvm-dwarfdump/fuzzer/llvm-dwarfdump-fuzzer.cpp
index 2caccaa0fb68b8610447069ca23d582cb8e548e1..1d74856c0fb8a619d3b51204432bc7a833ee928a 100644
--- a/tools/llvm-dwarfdump/fuzzer/llvm-dwarfdump-fuzzer.cpp
+++ b/tools/llvm-dwarfdump/fuzzer/llvm-dwarfdump-fuzzer.cpp
@@ -1,9 +1,8 @@
 //===-- llvm-dwarfdump-fuzzer.cpp - Fuzz the llvm-dwarfdump tool ----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
diff --git a/tools/llvm-dwarfdump/llvm-dwarfdump.cpp b/tools/llvm-dwarfdump/llvm-dwarfdump.cpp
index d9e8e36efe5c57d602b0b74c3cf360f84b7fef55..de2221b1392421aa4119cb8ac678a2cd677d7432 100644
--- a/tools/llvm-dwarfdump/llvm-dwarfdump.cpp
+++ b/tools/llvm-dwarfdump/llvm-dwarfdump.cpp
@@ -1,9 +1,8 @@
 //===-- llvm-dwarfdump.cpp - Debug info dumping utility for llvm ----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/tools/llvm-dwp/LLVMBuild.txt b/tools/llvm-dwp/LLVMBuild.txt
index 345a73757255252ba2867017830ebfa2a4a1177e..f83e5581e6b5f0096d96175e3812bb6183ea7ac0 100644
--- a/tools/llvm-dwp/LLVMBuild.txt
+++ b/tools/llvm-dwp/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./tools/llvm-dwp/LLVMBuild.txt ---------------------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/tools/llvm-dwp/llvm-dwp.cpp b/tools/llvm-dwp/llvm-dwp.cpp
index 8a0744cf1e016b7866480c3157afd963469f9bb0..2d6bd7fdd23adb74919ce7f52f0bd30200a351db 100644
--- a/tools/llvm-dwp/llvm-dwp.cpp
+++ b/tools/llvm-dwp/llvm-dwp.cpp
@@ -1,9 +1,8 @@
 //===-- llvm-dwp.cpp - Split DWARF merging tool for llvm ------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -14,6 +13,7 @@
 #include "DWPError.h"
 #include "DWPStringPool.h"
 #include "llvm/ADT/MapVector.h"
+#include "llvm/ADT/Optional.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/DebugInfo/DWARF/DWARFContext.h"
 #include "llvm/DebugInfo/DWARF/DWARFFormValue.h"
@@ -39,6 +39,8 @@
 #include "llvm/Support/Path.h"
 #include "llvm/Support/TargetRegistry.h"
 #include "llvm/Support/TargetSelect.h"
+#include "llvm/Support/ToolOutputFile.h"
+#include "llvm/Support/WithColor.h"
 #include "llvm/Support/raw_ostream.h"
 
 using namespace llvm;
@@ -160,6 +162,7 @@ static Expected<CompileUnitIdentifiers> getCUIdentifiers(StringRef Abbrev,
   uint32_t Name;
   dwarf::Form Form;
   CompileUnitIdentifiers ID;
+  Optional<uint64_t> Signature = None;
   while ((Name = AbbrevData.getULEB128(&AbbrevOffset)) |
          (Form = static_cast<dwarf::Form>(AbbrevData.getULEB128(&AbbrevOffset))) &&
          (Name != 0 || Form != 0)) {
@@ -181,13 +184,16 @@ static Expected<CompileUnitIdentifiers> getCUIdentifiers(StringRef Abbrev,
       break;
     }
     case dwarf::DW_AT_GNU_dwo_id:
-      ID.Signature = InfoData.getU64(&Offset);
+      Signature = InfoData.getU64(&Offset);
       break;
     default:
       DWARFFormValue::skipValue(Form, InfoData, &Offset,
                                 dwarf::FormParams({Version, AddrSize, Format}));
     }
   }
+  if (!Signature)
+    return make_error<DWPError>("compile unit missing dwo_id");
+  ID.Signature = *Signature;
   return ID;
 }
 
@@ -561,7 +567,7 @@ static Error write(MCStreamer &Out, ArrayRef<std::string> Inputs) {
       Expected<CompileUnitIdentifiers> EID = getCUIdentifiers(
           AbbrevSection, InfoSection, CurStrOffsetSection, CurStrSection);
       if (!EID)
-        return EID.takeError();
+        return createFileError(Input, EID.takeError());
       const auto &ID = *EID;
       auto P = IndexEntries.insert(std::make_pair(ID.Signature, CurEntry));
       if (!P.second)
@@ -589,7 +595,7 @@ static Error write(MCStreamer &Out, ArrayRef<std::string> Inputs) {
           getSubsection(CurStrOffsetSection, E, DW_SECT_STR_OFFSETS),
           CurStrSection);
       if (!EID)
-        return EID.takeError();
+        return createFileError(Input, EID.takeError());
       const auto &ID = *EID;
       if (!P.second)
         return buildDuplicateError(*P.first, ID, Input);
@@ -696,23 +702,23 @@ int main(int argc, char **argv) {
 
   // Create the output file.
   std::error_code EC;
-  raw_fd_ostream OutFile(OutputFilename, EC, sys::fs::F_None);
+  ToolOutputFile OutFile(OutputFilename, EC, sys::fs::F_None);
   Optional<buffer_ostream> BOS;
   raw_pwrite_stream *OS;
   if (EC)
     return error(Twine(OutputFilename) + ": " + EC.message(), Context);
-  if (OutFile.supportsSeeking()) {
-    OS = &OutFile;
+  if (OutFile.os().supportsSeeking()) {
+    OS = &OutFile.os();
   } else {
-    BOS.emplace(OutFile);
+    BOS.emplace(OutFile.os());
     OS = BOS.getPointer();
   }
 
   MCTargetOptions MCOptions = InitMCTargetOptionsFromFlags();
   std::unique_ptr<MCStreamer> MS(TheTarget->createMCObjectStreamer(
       TheTriple, MC, std::unique_ptr<MCAsmBackend>(MAB),
-      MAB->createObjectWriter(*OS), std::unique_ptr<MCCodeEmitter>(MCE),
-      *MSTI, MCOptions.MCRelaxAll, MCOptions.MCIncrementalLinkerCompatible,
+      MAB->createObjectWriter(*OS), std::unique_ptr<MCCodeEmitter>(MCE), *MSTI,
+      MCOptions.MCRelaxAll, MCOptions.MCIncrementalLinkerCompatible,
       /*DWARFMustBeAtTheEnd*/ false));
   if (!MS)
     return error("no object streamer for target " + TripleName, Context);
@@ -721,7 +727,7 @@ int main(int argc, char **argv) {
   for (const auto &ExecFilename : ExecFilenames) {
     auto DWOs = getDWOFilenames(ExecFilename);
     if (!DWOs) {
-      logAllUnhandledErrors(DWOs.takeError(), errs(), "error: ");
+      logAllUnhandledErrors(DWOs.takeError(), WithColor::error());
       return 1;
     }
     DWOFilenames.insert(DWOFilenames.end(),
@@ -730,9 +736,11 @@ int main(int argc, char **argv) {
   }
 
   if (auto Err = write(*MS, DWOFilenames)) {
-    logAllUnhandledErrors(std::move(Err), errs(), "error: ");
+    logAllUnhandledErrors(std::move(Err), WithColor::error());
     return 1;
   }
 
   MS->Finish();
+  OutFile.keep();
+  return 0;
 }
diff --git a/tools/llvm-elfabi/CMakeLists.txt b/tools/llvm-elfabi/CMakeLists.txt
new file mode 100644
index 0000000000000000000000000000000000000000..bd3ec851887aba1571a950f564e37b7459a86434
--- /dev/null
+++ b/tools/llvm-elfabi/CMakeLists.txt
@@ -0,0 +1,11 @@
+set(LLVM_LINK_COMPONENTS
+  Object
+  Support
+  TextAPI
+  )
+
+add_llvm_tool(llvm-elfabi
+  ELFObjHandler.cpp
+  ErrorCollector.cpp
+  llvm-elfabi.cpp
+  )
diff --git a/tools/llvm-elfabi/ELFObjHandler.cpp b/tools/llvm-elfabi/ELFObjHandler.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..a41fc19f56c5df3aa2e827fd66c834e4c6bdc578
--- /dev/null
+++ b/tools/llvm-elfabi/ELFObjHandler.cpp
@@ -0,0 +1,391 @@
+//===- ELFObjHandler.cpp --------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===-----------------------------------------------------------------------===/
+
+#include "ELFObjHandler.h"
+#include "llvm/Object/Binary.h"
+#include "llvm/Object/ELFObjectFile.h"
+#include "llvm/Object/ELFTypes.h"
+#include "llvm/Support/Errc.h"
+#include "llvm/Support/Error.h"
+#include "llvm/Support/MemoryBuffer.h"
+#include "llvm/TextAPI/ELF/ELFStub.h"
+
+using llvm::MemoryBufferRef;
+using llvm::object::ELFObjectFile;
+
+using namespace llvm;
+using namespace llvm::object;
+using namespace llvm::ELF;
+
+namespace llvm {
+namespace elfabi {
+
+// Simple struct to hold relevant .dynamic entries.
+struct DynamicEntries {
+  uint64_t StrTabAddr = 0;
+  uint64_t StrSize = 0;
+  Optional<uint64_t> SONameOffset;
+  std::vector<uint64_t> NeededLibNames;
+  // Symbol table:
+  uint64_t DynSymAddr = 0;
+  // Hash tables:
+  Optional<uint64_t> ElfHash;
+  Optional<uint64_t> GnuHash;
+};
+
+/// This function behaves similarly to StringRef::substr(), but attempts to
+/// terminate the returned StringRef at the first null terminator. If no null
+/// terminator is found, an error is returned.
+///
+/// @param Str Source string to create a substring from.
+/// @param Offset The start index of the desired substring.
+static Expected<StringRef> terminatedSubstr(StringRef Str, size_t Offset) {
+  size_t StrEnd = Str.find('\0', Offset);
+  if (StrEnd == StringLiteral::npos) {
+    return createError(
+        "String overran bounds of string table (no null terminator)");
+  }
+
+  size_t StrLen = StrEnd - Offset;
+  return Str.substr(Offset, StrLen);
+}
+
+/// This function takes an error, and appends a string of text to the end of
+/// that error. Since "appending" to an Error isn't supported behavior of an
+/// Error, this function technically creates a new error with the combined
+/// message and consumes the old error.
+///
+/// @param Err Source error.
+/// @param After Text to append at the end of Err's error message.
+Error appendToError(Error Err, StringRef After) {
+  std::string Message;
+  raw_string_ostream Stream(Message);
+  Stream << Err;
+  Stream << " " << After;
+  consumeError(std::move(Err));
+  return createError(Stream.str().c_str());
+}
+
+/// This function populates a DynamicEntries struct using an ELFT::DynRange.
+/// After populating the struct, the members are validated with
+/// some basic sanity checks.
+///
+/// @param Dyn Target DynamicEntries struct to populate.
+/// @param DynTable Source dynamic table.
+template <class ELFT>
+static Error populateDynamic(DynamicEntries &Dyn,
+                             typename ELFT::DynRange DynTable) {
+  if (DynTable.empty())
+    return createError("No .dynamic section found");
+
+  // Search .dynamic for relevant entries.
+  bool FoundDynStr = false;
+  bool FoundDynStrSz = false;
+  bool FoundDynSym = false;
+  for (auto &Entry : DynTable) {
+    switch (Entry.d_tag) {
+    case DT_SONAME:
+      Dyn.SONameOffset = Entry.d_un.d_val;
+      break;
+    case DT_STRTAB:
+      Dyn.StrTabAddr = Entry.d_un.d_ptr;
+      FoundDynStr = true;
+      break;
+    case DT_STRSZ:
+      Dyn.StrSize = Entry.d_un.d_val;
+      FoundDynStrSz = true;
+      break;
+    case DT_NEEDED:
+      Dyn.NeededLibNames.push_back(Entry.d_un.d_val);
+      break;
+    case DT_SYMTAB:
+      Dyn.DynSymAddr = Entry.d_un.d_ptr;
+      FoundDynSym = true;
+      break;
+    case DT_HASH:
+      Dyn.ElfHash = Entry.d_un.d_ptr;
+      break;
+    case DT_GNU_HASH:
+      Dyn.GnuHash = Entry.d_un.d_ptr;
+    }
+  }
+
+  if (!FoundDynStr) {
+    return createError(
+        "Couldn't locate dynamic string table (no DT_STRTAB entry)");
+  }
+  if (!FoundDynStrSz) {
+    return createError(
+        "Couldn't determine dynamic string table size (no DT_STRSZ entry)");
+  }
+  if (!FoundDynSym) {
+    return createError(
+        "Couldn't locate dynamic symbol table (no DT_SYMTAB entry)");
+  }
+  if (Dyn.SONameOffset.hasValue() && *Dyn.SONameOffset >= Dyn.StrSize) {
+    return createStringError(
+        object_error::parse_failed,
+        "DT_SONAME string offset (0x%016" PRIx64
+        ") outside of dynamic string table",
+        *Dyn.SONameOffset);
+  }
+  for (uint64_t Offset : Dyn.NeededLibNames) {
+    if (Offset >= Dyn.StrSize) {
+      return createStringError(
+          object_error::parse_failed,
+          "DT_NEEDED string offset (0x%016" PRIx64
+          ") outside of dynamic string table",
+          Offset);
+    }
+  }
+
+  return Error::success();
+}
+
+/// This function finds the number of dynamic symbols using a GNU hash table.
+///
+/// @param Table The GNU hash table for .dynsym.
+template <class ELFT>
+static uint64_t getDynSymtabSize(const typename ELFT::GnuHash &Table) {
+  using Elf_Word = typename ELFT::Word;
+  if (Table.nbuckets == 0)
+    return Table.symndx + 1;
+  uint64_t LastSymIdx = 0;
+  uint64_t BucketVal = 0;
+  // Find the index of the first symbol in the last chain.
+  for (Elf_Word Val : Table.buckets()) {
+    BucketVal = std::max(BucketVal, (uint64_t)Val);
+  }
+  LastSymIdx += BucketVal;
+  const Elf_Word *It =
+      reinterpret_cast<const Elf_Word *>(Table.values(BucketVal).end());
+  // Locate the end of the chain to find the last symbol index.
+  while ((*It & 1) == 0) {
+    LastSymIdx++;
+    It++;
+  }
+  return LastSymIdx + 1;
+}
+
+/// This function determines the number of dynamic symbols.
+/// Without access to section headers, the number of symbols must be determined
+/// by parsing dynamic hash tables.
+///
+/// @param Dyn Entries with the locations of hash tables.
+/// @param ElfFile The ElfFile that the section contents reside in.
+template <class ELFT>
+static Expected<uint64_t> getNumSyms(DynamicEntries &Dyn,
+                                     const ELFFile<ELFT> &ElfFile) {
+  using Elf_Hash = typename ELFT::Hash;
+  using Elf_GnuHash = typename ELFT::GnuHash;
+  // Search GNU hash table to try to find the upper bound of dynsym.
+  if (Dyn.GnuHash.hasValue()) {
+    Expected<const uint8_t *> TablePtr = ElfFile.toMappedAddr(*Dyn.GnuHash);
+    if (!TablePtr)
+      return TablePtr.takeError();
+    const Elf_GnuHash *Table =
+        reinterpret_cast<const Elf_GnuHash *>(TablePtr.get());
+    return getDynSymtabSize<ELFT>(*Table);
+  }
+  // Search SYSV hash table to try to find the upper bound of dynsym.
+  if (Dyn.ElfHash.hasValue()) {
+    Expected<const uint8_t *> TablePtr = ElfFile.toMappedAddr(*Dyn.ElfHash);
+    if (!TablePtr)
+      return TablePtr.takeError();
+    const Elf_Hash *Table = reinterpret_cast<const Elf_Hash *>(TablePtr.get());
+    return Table->nchain;
+  }
+  return 0;
+}
+
+/// This function extracts symbol type from a symbol's st_info member and
+/// maps it to an ELFSymbolType enum.
+/// Currently, STT_NOTYPE, STT_OBJECT, STT_FUNC, and STT_TLS are supported.
+/// Other symbol types are mapped to ELFSymbolType::Unknown.
+///
+/// @param Info Binary symbol st_info to extract symbol type from.
+static ELFSymbolType convertInfoToType(uint8_t Info) {
+  Info = Info & 0xf;
+  switch (Info) {
+    case ELF::STT_NOTYPE:
+      return ELFSymbolType::NoType;
+    case ELF::STT_OBJECT:
+      return ELFSymbolType::Object;
+    case ELF::STT_FUNC:
+      return ELFSymbolType::Func;
+    case ELF::STT_TLS:
+      return ELFSymbolType::TLS;
+    default:
+      return ELFSymbolType::Unknown;
+  }
+}
+
+/// This function creates an ELFSymbol and populates all members using
+/// information from a binary ELFT::Sym.
+///
+/// @param SymName The desired name of the ELFSymbol.
+/// @param RawSym ELFT::Sym to extract symbol information from.
+template <class ELFT>
+static ELFSymbol createELFSym(StringRef SymName,
+                              const typename ELFT::Sym &RawSym) {
+  ELFSymbol TargetSym(SymName);
+  uint8_t Binding = RawSym.getBinding();
+  if (Binding == STB_WEAK)
+    TargetSym.Weak = true;
+  else
+    TargetSym.Weak = false;
+
+  TargetSym.Undefined = RawSym.isUndefined();
+  TargetSym.Type = convertInfoToType(RawSym.st_info);
+
+  if (TargetSym.Type == ELFSymbolType::Func) {
+    TargetSym.Size = 0;
+  } else {
+    TargetSym.Size = RawSym.st_size;
+  }
+  return TargetSym;
+}
+
+/// This function populates an ELFStub with symbols using information read
+/// from an ELF binary.
+///
+/// @param TargetStub ELFStub to add symbols to.
+/// @param DynSym Range of dynamic symbols to add to TargetStub.
+/// @param DynStr StringRef to the dynamic string table.
+template <class ELFT>
+static Error populateSymbols(ELFStub &TargetStub,
+                            const typename ELFT::SymRange DynSym,
+                            StringRef DynStr) {
+  // Skips the first symbol since it's the NULL symbol.
+  for (auto RawSym : DynSym.drop_front(1)) {
+    // If a symbol does not have global or weak binding, ignore it.
+    uint8_t Binding = RawSym.getBinding();
+    if (!(Binding == STB_GLOBAL || Binding == STB_WEAK))
+      continue;
+    // If a symbol doesn't have default or protected visibility, ignore it.
+    uint8_t Visibility = RawSym.getVisibility();
+    if (!(Visibility == STV_DEFAULT || Visibility == STV_PROTECTED))
+      continue;
+    // Create an ELFSymbol and populate it with information from the symbol
+    // table entry.
+    Expected<StringRef> SymName = terminatedSubstr(DynStr, RawSym.st_name);
+    if (!SymName)
+      return SymName.takeError();
+    ELFSymbol Sym = createELFSym<ELFT>(*SymName, RawSym);
+    TargetStub.Symbols.insert(std::move(Sym));
+    // TODO: Populate symbol warning.
+  }
+  return Error::success();
+}
+
+/// Returns a new ELFStub with all members populated from an ELFObjectFile.
+/// @param ElfObj Source ELFObjectFile.
+template <class ELFT>
+static Expected<std::unique_ptr<ELFStub>>
+buildStub(const ELFObjectFile<ELFT> &ElfObj) {
+  using Elf_Dyn_Range = typename ELFT::DynRange;
+  using Elf_Phdr_Range = typename ELFT::PhdrRange;
+  using Elf_Sym_Range = typename ELFT::SymRange;
+  using Elf_Sym = typename ELFT::Sym;
+  std::unique_ptr<ELFStub> DestStub = make_unique<ELFStub>();
+  const ELFFile<ELFT> *ElfFile = ElfObj.getELFFile();
+  // Fetch .dynamic table.
+  Expected<Elf_Dyn_Range> DynTable = ElfFile->dynamicEntries();
+  if (!DynTable) {
+    return DynTable.takeError();
+  }
+
+  // Fetch program headers.
+  Expected<Elf_Phdr_Range> PHdrs = ElfFile->program_headers();
+  if (!PHdrs) {
+    return PHdrs.takeError();
+  }
+
+  // Collect relevant .dynamic entries.
+  DynamicEntries DynEnt;
+  if (Error Err = populateDynamic<ELFT>(DynEnt, *DynTable))
+    return std::move(Err);
+
+    // Get pointer to in-memory location of .dynstr section.
+  Expected<const uint8_t *> DynStrPtr =
+      ElfFile->toMappedAddr(DynEnt.StrTabAddr);
+  if (!DynStrPtr)
+    return appendToError(DynStrPtr.takeError(),
+                         "when locating .dynstr section contents");
+
+  StringRef DynStr(reinterpret_cast<const char *>(DynStrPtr.get()),
+                   DynEnt.StrSize);
+
+  // Populate Arch from ELF header.
+  DestStub->Arch = ElfFile->getHeader()->e_machine;
+
+  // Populate SoName from .dynamic entries and dynamic string table.
+  if (DynEnt.SONameOffset.hasValue()) {
+    Expected<StringRef> NameOrErr =
+        terminatedSubstr(DynStr, *DynEnt.SONameOffset);
+    if (!NameOrErr) {
+      return appendToError(NameOrErr.takeError(), "when reading DT_SONAME");
+    }
+    DestStub->SoName = *NameOrErr;
+  }
+
+  // Populate NeededLibs from .dynamic entries and dynamic string table.
+  for (uint64_t NeededStrOffset : DynEnt.NeededLibNames) {
+    Expected<StringRef> LibNameOrErr =
+        terminatedSubstr(DynStr, NeededStrOffset);
+    if (!LibNameOrErr) {
+      return appendToError(LibNameOrErr.takeError(), "when reading DT_NEEDED");
+    }
+    DestStub->NeededLibs.push_back(*LibNameOrErr);
+  }
+
+  // Populate Symbols from .dynsym table and dynamic string table.
+  Expected<uint64_t> SymCount = getNumSyms(DynEnt, *ElfFile);
+  if (!SymCount)
+    return SymCount.takeError();
+  if (*SymCount > 0) {
+    // Get pointer to in-memory location of .dynsym section.
+    Expected<const uint8_t *> DynSymPtr =
+        ElfFile->toMappedAddr(DynEnt.DynSymAddr);
+    if (!DynSymPtr)
+      return appendToError(DynSymPtr.takeError(),
+                           "when locating .dynsym section contents");
+    Elf_Sym_Range DynSyms =
+        ArrayRef<Elf_Sym>(reinterpret_cast<const Elf_Sym *>(*DynSymPtr),
+                          *SymCount);
+    Error SymReadError = populateSymbols<ELFT>(*DestStub, DynSyms, DynStr);
+    if (SymReadError)
+      return appendToError(std::move(SymReadError),
+                           "when reading dynamic symbols");
+  }
+
+  return std::move(DestStub);
+}
+
+Expected<std::unique_ptr<ELFStub>> readELFFile(MemoryBufferRef Buf) {
+  Expected<std::unique_ptr<Binary>> BinOrErr = createBinary(Buf);
+  if (!BinOrErr) {
+    return BinOrErr.takeError();
+  }
+
+  Binary *Bin = BinOrErr->get();
+  if (auto Obj = dyn_cast<ELFObjectFile<ELF32LE>>(Bin)) {
+    return buildStub(*Obj);
+  } else if (auto Obj = dyn_cast<ELFObjectFile<ELF64LE>>(Bin)) {
+    return buildStub(*Obj);
+  } else if (auto Obj = dyn_cast<ELFObjectFile<ELF32BE>>(Bin)) {
+    return buildStub(*Obj);
+  } else if (auto Obj = dyn_cast<ELFObjectFile<ELF64BE>>(Bin)) {
+    return buildStub(*Obj);
+  }
+
+  return createStringError(errc::not_supported, "Unsupported binary format");
+}
+
+} // end namespace elfabi
+} // end namespace llvm
diff --git a/tools/llvm-elfabi/ELFObjHandler.h b/tools/llvm-elfabi/ELFObjHandler.h
new file mode 100644
index 0000000000000000000000000000000000000000..97f0d68f4d4fbcad76511d2847e0058dc7e97a7c
--- /dev/null
+++ b/tools/llvm-elfabi/ELFObjHandler.h
@@ -0,0 +1,32 @@
+//===- ELFObjHandler.h ------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===-----------------------------------------------------------------------===/
+///
+/// This supports reading and writing of elf dynamic shared objects.
+///
+//===-----------------------------------------------------------------------===/
+
+#ifndef LLVM_TOOLS_ELFABI_ELFOBJHANDLER_H
+#define LLVM_TOOLS_ELFABI_ELFOBJHANDLER_H
+
+#include "llvm/Object/ELFObjectFile.h"
+#include "llvm/Object/ELFTypes.h"
+#include "llvm/TextAPI/ELF/ELFStub.h"
+
+namespace llvm {
+
+class MemoryBuffer;
+
+namespace elfabi {
+
+/// Attempt to read a binary ELF file from a MemoryBuffer.
+Expected<std::unique_ptr<ELFStub>> readELFFile(MemoryBufferRef Buf);
+
+} // end namespace elfabi
+} // end namespace llvm
+
+#endif // LLVM_TOOLS_ELFABI_ELFOBJHANDLER_H
diff --git a/tools/llvm-elfabi/ErrorCollector.cpp b/tools/llvm-elfabi/ErrorCollector.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..f7ec9e4bf93556d1c756083f1cf64f08b4bdcd55
--- /dev/null
+++ b/tools/llvm-elfabi/ErrorCollector.cpp
@@ -0,0 +1,69 @@
+//===- ErrorCollector.cpp -------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===-----------------------------------------------------------------------===/
+
+#include "ErrorCollector.h"
+#include "llvm/Support/Errc.h"
+#include "llvm/Support/Error.h"
+#include "llvm/Support/raw_ostream.h"
+#include "llvm/Support/WithColor.h"
+#include <vector>
+
+using namespace llvm;
+using namespace llvm::elfabi;
+
+void ErrorCollector::escalateToFatal() {
+  ErrorsAreFatal = true;
+}
+
+void ErrorCollector::addError(Error &&Err, StringRef Tag) {
+  if (Err) {
+    Errors.push_back(std::move(Err));
+    Tags.push_back(Tag.str());
+  }
+}
+
+Error ErrorCollector::makeError() {
+  // TODO: Make this return something (an AggregateError?) that gives more
+  // individual control over each error and which might be of interest.
+  Error JoinedErrors = Error::success();
+  for (Error &E : Errors) {
+    JoinedErrors = joinErrors(std::move(JoinedErrors), std::move(E));
+  }
+  Errors.clear();
+  Tags.clear();
+  return JoinedErrors;
+}
+
+void ErrorCollector::log(raw_ostream &OS) {
+  OS << "Encountered multiple errors:\n";
+  for (size_t i = 0; i < Errors.size(); ++i) {
+    WithColor::error(OS) << "(" << Tags[i] << ") " << Errors[i];
+    if (i != Errors.size() - 1)
+      OS << "\n";
+  }
+}
+
+bool ErrorCollector::allErrorsHandled() const {
+  return Errors.empty();
+}
+
+ErrorCollector::~ErrorCollector() {
+  if (ErrorsAreFatal && !allErrorsHandled())
+    fatalUnhandledError();
+
+  for (Error &E : Errors) {
+    consumeError(std::move(E));
+  }
+}
+
+LLVM_ATTRIBUTE_NORETURN void ErrorCollector::fatalUnhandledError() {
+  errs() << "Program aborted due to unhandled Error(s):\n";
+  log(errs());
+  errs() << "\n";
+  abort();
+}
diff --git a/tools/llvm-elfabi/ErrorCollector.h b/tools/llvm-elfabi/ErrorCollector.h
new file mode 100644
index 0000000000000000000000000000000000000000..e2eb1c5ed9f3ba18f42a2d3f5b954a4aa93712ac
--- /dev/null
+++ b/tools/llvm-elfabi/ErrorCollector.h
@@ -0,0 +1,74 @@
+//===- ErrorCollector.h -----------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===-----------------------------------------------------------------------===/
+///
+/// This class collects errors that should be reported or ignored in aggregate.
+///
+/// Like llvm::Error, an ErrorCollector cannot be copied. Unlike llvm::Error,
+/// an ErrorCollector may be destroyed if it was originally constructed to treat
+/// errors as non-fatal. In this case, all Errors are consumed upon destruction.
+/// An ErrorCollector may be initially constructed (or escalated) such that
+/// errors are treated as fatal. This causes a crash if an attempt is made to
+/// delete the ErrorCollector when some Errors have not been retrieved via
+/// makeError().
+///
+//===-----------------------------------------------------------------------===/
+
+#ifndef LLVM_TOOLS_ELFABI_ERRORCOLLECTOR_H
+#define LLVM_TOOLS_ELFABI_ERRORCOLLECTOR_H
+
+#include "llvm/Support/Error.h"
+#include <vector>
+
+namespace llvm {
+namespace elfabi {
+
+class ErrorCollector {
+public:
+  /// Upon destruction, an ErrorCollector will crash if UseFatalErrors=true and
+  /// there are remaining errors that haven't been fetched by makeError().
+  ErrorCollector(bool UseFatalErrors = true) : ErrorsAreFatal(UseFatalErrors) {}
+  // Don't allow copying.
+  ErrorCollector(const ErrorCollector &Stub) = delete;
+  ErrorCollector &operator=(const ErrorCollector &Other) = delete;
+  ~ErrorCollector();
+
+  // TODO: Add move constructor and operator= when a testable situation arises.
+
+  /// Returns a single error that contains messages for all stored Errors.
+  Error makeError();
+
+  /// Adds an error with a descriptive tag that helps with identification.
+  /// If the error is an Error::success(), it is checked and discarded.
+  void addError(Error &&E, StringRef Tag);
+
+  /// This ensures an ErrorCollector will treat unhandled errors as fatal.
+  /// This function should be called if errors that usually can be ignored
+  /// are suddenly of concern (i.e. attempt multiple things that return Error,
+  /// but only care about the Errors if no attempt succeeds).
+  void escalateToFatal();
+
+private:
+  /// Logs all errors to a raw_ostream.
+  void log(raw_ostream &OS);
+
+  /// Returns true if all errors have been retrieved through makeError(), or
+  /// false if errors have been added since the last makeError() call.
+  bool allErrorsHandled() const;
+
+  /// Dump output and crash.
+  LLVM_ATTRIBUTE_NORETURN void fatalUnhandledError();
+
+  bool ErrorsAreFatal;
+  std::vector<Error> Errors;
+  std::vector<std::string> Tags;
+};
+
+} // end namespace elfabi
+} // end namespace llvm
+
+#endif // LLVM_TOOLS_ELFABI_ERRORCOLLECTOR_H
diff --git a/lib/Target/Nios2/InstPrinter/LLVMBuild.txt b/tools/llvm-elfabi/LLVMBuild.txt
similarity index 52%
rename from lib/Target/Nios2/InstPrinter/LLVMBuild.txt
rename to tools/llvm-elfabi/LLVMBuild.txt
index bc7882dd35776f78deedbcfa31e78c0f04cbb29b..b1a80e9e4a15f96ad732062ad019ada5b78dac34 100644
--- a/lib/Target/Nios2/InstPrinter/LLVMBuild.txt
+++ b/tools/llvm-elfabi/LLVMBuild.txt
@@ -1,9 +1,8 @@
-;===- ./lib/Target/Nios2/InstPrinter/LLVMBuild.txt -------------*- Conf -*--===;
+;===- ./tools/llvm-elfabi/LLVMBuild.txt ------------------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
@@ -16,8 +15,7 @@
 ;===------------------------------------------------------------------------===;
 
 [component_0]
-type = Library
-name = Nios2AsmPrinter
-parent = Nios2
-required_libraries = MC Support
-add_to_library_groups = Nios2
+type = Tool
+name = llvm-elfabi
+parent = Tools
+required_libraries = Object Support TextAPI
diff --git a/tools/llvm-elfabi/llvm-elfabi.cpp b/tools/llvm-elfabi/llvm-elfabi.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..044b5f77c6d71d3a1a7e3282b6d7b707d8a21e52
--- /dev/null
+++ b/tools/llvm-elfabi/llvm-elfabi.cpp
@@ -0,0 +1,142 @@
+//===- llvm-elfabi.cpp ----------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===-----------------------------------------------------------------------===/
+
+#include "ELFObjHandler.h"
+#include "ErrorCollector.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Errc.h"
+#include "llvm/Support/FileOutputBuffer.h"
+#include "llvm/Support/MemoryBuffer.h"
+#include "llvm/Support/Path.h"
+#include "llvm/Support/raw_ostream.h"
+#include "llvm/Support/WithColor.h"
+#include "llvm/TextAPI/ELF/TBEHandler.h"
+#include <string>
+
+namespace llvm {
+namespace elfabi {
+
+enum class FileFormat {
+  TBE,
+  ELF
+};
+
+} // end namespace elfabi
+} // end namespace llvm
+
+using namespace llvm;
+using namespace llvm::elfabi;
+
+// Command line flags:
+cl::opt<FileFormat> InputFileFormat(
+    cl::desc("Force input file format:"),
+    cl::values(clEnumValN(FileFormat::TBE,
+                          "tbe", "Read `input` as text-based ELF stub"),
+               clEnumValN(FileFormat::ELF,
+                          "elf", "Read `input` as ELF binary")));
+cl::opt<std::string> InputFilePath(cl::Positional, cl::desc("input"),
+                                   cl::Required);
+cl::opt<std::string>
+    EmitTBE("emit-tbe",
+            cl::desc("Emit a text-based ELF stub (.tbe) from the input file"),
+            cl::value_desc("path"));
+cl::opt<std::string> SOName(
+    "soname",
+    cl::desc("Manually set the DT_SONAME entry of any emitted files"),
+    cl::value_desc("name"));
+
+/// writeTBE() writes a Text-Based ELF stub to a file using the latest version
+/// of the YAML parser.
+static Error writeTBE(StringRef FilePath, ELFStub &Stub) {
+  std::error_code SysErr;
+
+  // Open file for writing.
+  raw_fd_ostream Out(FilePath, SysErr);
+  if (SysErr)
+    return createStringError(SysErr, "Couldn't open `%s` for writing",
+                             FilePath.data());
+  // Write file.
+  Error YAMLErr = writeTBEToOutputStream(Out, Stub);
+  if (YAMLErr)
+    return YAMLErr;
+
+  return Error::success();
+}
+
+/// readInputFile populates an ELFStub by attempting to read the
+/// input file using both the TBE and binary ELF parsers.
+static Expected<std::unique_ptr<ELFStub>> readInputFile(StringRef FilePath) {
+  // Read in file.
+  ErrorOr<std::unique_ptr<MemoryBuffer>> BufOrError =
+      MemoryBuffer::getFile(FilePath);
+  if (!BufOrError) {
+    return createStringError(BufOrError.getError(), "Could not open `%s`",
+                             FilePath.data());
+  }
+
+  std::unique_ptr<MemoryBuffer> FileReadBuffer = std::move(*BufOrError);
+  ErrorCollector EC(/*UseFatalErrors=*/false);
+
+  // First try to read as a binary (fails fast if not binary).
+  if (InputFileFormat.getNumOccurrences() == 0 ||
+      InputFileFormat == FileFormat::ELF) {
+    Expected<std::unique_ptr<ELFStub>> StubFromELF =
+        readELFFile(FileReadBuffer->getMemBufferRef());
+    if (StubFromELF) {
+      return std::move(*StubFromELF);
+    }
+    EC.addError(StubFromELF.takeError(), "BinaryRead");
+  }
+
+  // Fall back to reading as a tbe.
+  if (InputFileFormat.getNumOccurrences() == 0 ||
+      InputFileFormat == FileFormat::TBE) {
+    Expected<std::unique_ptr<ELFStub>> StubFromTBE =
+        readTBEFromBuffer(FileReadBuffer->getBuffer());
+    if (StubFromTBE) {
+      return std::move(*StubFromTBE);
+    }
+    EC.addError(StubFromTBE.takeError(), "YamlParse");
+  }
+
+  // If both readers fail, build a new error that includes all information.
+  EC.addError(createStringError(errc::not_supported,
+                                "No file readers succeeded reading `%s` "
+                                "(unsupported/malformed file?)",
+                                FilePath.data()),
+              "ReadInputFile");
+  EC.escalateToFatal();
+  return EC.makeError();
+}
+
+int main(int argc, char *argv[]) {
+  // Parse arguments.
+  cl::ParseCommandLineOptions(argc, argv);
+
+  Expected<std::unique_ptr<ELFStub>> StubOrErr = readInputFile(InputFilePath);
+  if (!StubOrErr) {
+    Error ReadError = StubOrErr.takeError();
+    WithColor::error() << ReadError << "\n";
+    exit(1);
+  }
+
+  std::unique_ptr<ELFStub> TargetStub = std::move(StubOrErr.get());
+
+  // Write out .tbe file.
+  if (EmitTBE.getNumOccurrences() == 1) {
+    TargetStub->TbeVersion = TBEVersionCurrent;
+    if (SOName.getNumOccurrences() == 1) {
+      TargetStub->SoName = SOName;
+    }
+    Error TBEWriteError = writeTBE(EmitTBE, *TargetStub);
+    if (TBEWriteError) {
+      WithColor::error() << TBEWriteError << "\n";
+      exit(1);
+    }
+  }
+}
diff --git a/tools/llvm-exegesis/LLVMBuild.txt b/tools/llvm-exegesis/LLVMBuild.txt
index 2955dc5c8801fcbb000794ec3d5da0c796afe352..d13f2abea979a36c153aebe5e77679df6aff5796 100644
--- a/tools/llvm-exegesis/LLVMBuild.txt
+++ b/tools/llvm-exegesis/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./tools/llvm-exegesis/LLVMBuild.txt ----------------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/tools/llvm-exegesis/lib/AArch64/LLVMBuild.txt b/tools/llvm-exegesis/lib/AArch64/LLVMBuild.txt
index 886fcb22749e0ee8a23caf3eba0b3d65dd8817b7..dcbf72efd3c050721185900e88779478dfede2f9 100644
--- a/tools/llvm-exegesis/lib/AArch64/LLVMBuild.txt
+++ b/tools/llvm-exegesis/lib/AArch64/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./tools/llvm-exegesis/lib/AArch64/LLVMBuild.txt ----------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/tools/llvm-exegesis/lib/AArch64/Target.cpp b/tools/llvm-exegesis/lib/AArch64/Target.cpp
index d6b6cff54727dd952eca7a69231f3af4e365c2e2..a659278c031354ca6a9160b2cebf747787249b23 100644
--- a/tools/llvm-exegesis/lib/AArch64/Target.cpp
+++ b/tools/llvm-exegesis/lib/AArch64/Target.cpp
@@ -1,9 +1,8 @@
 //===-- Target.cpp ----------------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 #include "../Target.h"
diff --git a/tools/llvm-exegesis/lib/Analysis.cpp b/tools/llvm-exegesis/lib/Analysis.cpp
index 0a91679fe1d1c94a2f6b952bfc393f8b08385391..158fbfe3b0ac3dbc70d7be482e6c2568db398484 100644
--- a/tools/llvm-exegesis/lib/Analysis.cpp
+++ b/tools/llvm-exegesis/lib/Analysis.cpp
@@ -1,9 +1,8 @@
 //===-- Analysis.cpp --------------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -317,6 +316,7 @@ void Analysis::printSchedClassClustersHtml(
         writeLatencySnippetHtml(OS, Point.Key.Instructions, *InstrInfo_);
         break;
       case InstructionBenchmark::Uops:
+      case InstructionBenchmark::InverseThroughput:
         writeUopsSnippetHtml(OS, Point.Key.Instructions, *InstrInfo_);
         break;
       default:
@@ -508,9 +508,14 @@ bool Analysis::SchedClassCluster::measurementsMatch(
       }
       ClusterCenterPoint[I].PerInstructionValue = Representative[I].avg();
     }
+  } else if (Mode == InstructionBenchmark::InverseThroughput) {
+    for (int I = 0, E = Representative.size(); I < E; ++I) {
+      SchedClassPoint[I].PerInstructionValue =
+          MCSchedModel::getReciprocalThroughput(STI, *RSC.SCDesc);
+      ClusterCenterPoint[I].PerInstructionValue = Representative[I].min();
+    }
   } else {
-    llvm::errs() << "unimplemented measurement matching for mode " << Mode
-                 << "\n";
+    llvm_unreachable("unimplemented measurement matching mode");
     return false;
   }
   return Clustering.isNeighbour(ClusterCenterPoint, SchedClassPoint);
@@ -520,9 +525,9 @@ void Analysis::printSchedClassDescHtml(const ResolvedSchedClass &RSC,
                                        llvm::raw_ostream &OS) const {
   OS << "<table class=\"sched-class-desc\">";
   OS << "<tr><th>Valid</th><th>Variant</th><th>NumMicroOps</th><th>Latency</"
-        "th><th>WriteProcRes</th><th title=\"This is the idealized unit "
-        "resource (port) pressure assuming ideal distribution\">Idealized "
-        "Resource Pressure</th></tr>";
+        "th><th>RThroughput</th><th>WriteProcRes</th><th title=\"This is the "
+        "idealized unit resource (port) pressure assuming ideal "
+        "distribution\">Idealized Resource Pressure</th></tr>";
   if (RSC.SCDesc->isValid()) {
     const auto &SM = SubtargetInfo_->getSchedModel();
     OS << "<tr><td>&#10004;</td>";
@@ -541,6 +546,12 @@ void Analysis::printSchedClassDescHtml(const ResolvedSchedClass &RSC,
       OS << "</li>";
     }
     OS << "</ul></td>";
+    // inverse throughput.
+    OS << "<td>";
+    writeMeasurementValue<kEscapeHtml>(
+        OS,
+        MCSchedModel::getReciprocalThroughput(*SubtargetInfo_, *RSC.SCDesc));
+    OS << "</td>";
     // WriteProcRes.
     OS << "<td><ul>";
     for (const auto &WPR : RSC.NonRedundantWriteProcRes) {
diff --git a/tools/llvm-exegesis/lib/Analysis.h b/tools/llvm-exegesis/lib/Analysis.h
index 9ee1493f4e051939aa0e846c5527da2d13f7f774..852aaae0012f0e7c0d47ea9813a4a86633c72e98 100644
--- a/tools/llvm-exegesis/lib/Analysis.h
+++ b/tools/llvm-exegesis/lib/Analysis.h
@@ -1,9 +1,8 @@
 //===-- Analysis.h ----------------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
diff --git a/tools/llvm-exegesis/lib/Assembler.cpp b/tools/llvm-exegesis/lib/Assembler.cpp
index 2e3712ce7dc7d859e27f0bc746c5758f4ae9ec14..437fe892603e1ab09968ab7a33a299cab954b9db 100644
--- a/tools/llvm-exegesis/lib/Assembler.cpp
+++ b/tools/llvm-exegesis/lib/Assembler.cpp
@@ -1,9 +1,8 @@
 //===-- Assembler.cpp -------------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/tools/llvm-exegesis/lib/Assembler.h b/tools/llvm-exegesis/lib/Assembler.h
index ee6bc86f3788487f12c70ca8ae17536dd5eaf889..ff1e8db1a0c353761733c22742a0900ffca65f83 100644
--- a/tools/llvm-exegesis/lib/Assembler.h
+++ b/tools/llvm-exegesis/lib/Assembler.h
@@ -1,9 +1,8 @@
 //===-- Assembler.h ---------------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
diff --git a/tools/llvm-exegesis/lib/BenchmarkCode.h b/tools/llvm-exegesis/lib/BenchmarkCode.h
index 38bea2519a64ecad80c765a4607f7462629a54b4..1976004c251a32b4e1546313edd1b7d966a35547 100644
--- a/tools/llvm-exegesis/lib/BenchmarkCode.h
+++ b/tools/llvm-exegesis/lib/BenchmarkCode.h
@@ -1,9 +1,8 @@
 //===-- BenchmarkCode.h -----------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/tools/llvm-exegesis/lib/BenchmarkResult.cpp b/tools/llvm-exegesis/lib/BenchmarkResult.cpp
index 0507ae8959d0bfea2d754a54e49b429c2f23d6ae..c7a0c8470bbd2523d2fc7be5852bed65df80840d 100644
--- a/tools/llvm-exegesis/lib/BenchmarkResult.cpp
+++ b/tools/llvm-exegesis/lib/BenchmarkResult.cpp
@@ -1,17 +1,17 @@
 //===-- BenchmarkResult.cpp -------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #include "BenchmarkResult.h"
 #include "BenchmarkRunner.h"
 #include "llvm/ADT/STLExtras.h"
-#include "llvm/ADT/bit.h"
+#include "llvm/ADT/StringMap.h"
 #include "llvm/ADT/StringRef.h"
+#include "llvm/ADT/bit.h"
 #include "llvm/ObjectYAML/YAML.h"
 #include "llvm/Support/FileOutputBuffer.h"
 #include "llvm/Support/FileSystem.h"
@@ -30,7 +30,28 @@ namespace {
 // serialization process to encode/decode registers and instructions.
 struct YamlContext {
   YamlContext(const exegesis::LLVMState &State)
-      : State(&State), ErrorStream(LastError) {}
+      : State(&State), ErrorStream(LastError),
+        OpcodeNameToOpcodeIdx(
+            generateOpcodeNameToOpcodeIdxMapping(State.getInstrInfo())),
+        RegNameToRegNo(generateRegNameToRegNoMapping(State.getRegInfo())) {}
+
+  static llvm::StringMap<unsigned>
+  generateOpcodeNameToOpcodeIdxMapping(const llvm::MCInstrInfo &InstrInfo) {
+    llvm::StringMap<unsigned> Map(InstrInfo.getNumOpcodes());
+    for (unsigned I = 0, E = InstrInfo.getNumOpcodes(); I < E; ++I)
+      Map[InstrInfo.getName(I)] = I;
+    assert(Map.size() == InstrInfo.getNumOpcodes() && "Size prediction failed");
+    return Map;
+  };
+
+  llvm::StringMap<unsigned>
+  generateRegNameToRegNoMapping(const llvm::MCRegisterInfo &RegInfo) {
+    llvm::StringMap<unsigned> Map(RegInfo.getNumRegs());
+    for (unsigned I = 0, E = RegInfo.getNumRegs(); I < E; ++I)
+      Map[RegInfo.getName(I)] = I;
+    assert(Map.size() == RegInfo.getNumRegs() && "Size prediction failed");
+    return Map;
+  };
 
   void serializeMCInst(const llvm::MCInst &MCInst, llvm::raw_ostream &OS) {
     OS << getInstrName(MCInst.getOpcode());
@@ -41,7 +62,7 @@ struct YamlContext {
   }
 
   void deserializeMCInst(llvm::StringRef String, llvm::MCInst &Value) {
-    llvm::SmallVector<llvm::StringRef, 8> Pieces;
+    llvm::SmallVector<llvm::StringRef, 16> Pieces;
     String.split(Pieces, " ", /* MaxSplit */ -1, /* KeepEmpty */ false);
     if (Pieces.empty()) {
       ErrorStream << "Unknown Instruction: '" << String << "'";
@@ -69,10 +90,9 @@ struct YamlContext {
   }
 
   unsigned getRegNo(llvm::StringRef RegName) {
-    const llvm::MCRegisterInfo &RegInfo = State->getRegInfo();
-    for (unsigned E = RegInfo.getNumRegs(), I = 0; I < E; ++I)
-      if (RegInfo.getName(I) == RegName)
-        return I;
+    auto Iter = RegNameToRegNo.find(RegName);
+    if (Iter != RegNameToRegNo.end())
+      return Iter->second;
     ErrorStream << "No register with name " << RegName;
     return 0;
   }
@@ -137,10 +157,9 @@ private:
   }
 
   unsigned getInstrOpcode(llvm::StringRef InstrName) {
-    const llvm::MCInstrInfo &InstrInfo = State->getInstrInfo();
-    for (unsigned E = InstrInfo.getNumOpcodes(), I = 0; I < E; ++I)
-      if (InstrInfo.getName(I) == InstrName)
-        return I;
+    auto Iter = OpcodeNameToOpcodeIdx.find(InstrName);
+    if (Iter != OpcodeNameToOpcodeIdx.end())
+      return Iter->second;
     ErrorStream << "No opcode with name " << InstrName;
     return 0;
   }
@@ -148,6 +167,8 @@ private:
   const llvm::exegesis::LLVMState *State;
   std::string LastError;
   llvm::raw_string_ostream ErrorStream;
+  const llvm::StringMap<unsigned> OpcodeNameToOpcodeIdx;
+  const llvm::StringMap<unsigned> RegNameToRegNo;
 };
 } // namespace
 
@@ -210,6 +231,8 @@ struct ScalarEnumerationTraits<exegesis::InstructionBenchmark::ModeE> {
     Io.enumCase(Value, "", exegesis::InstructionBenchmark::Unknown);
     Io.enumCase(Value, "latency", exegesis::InstructionBenchmark::Latency);
     Io.enumCase(Value, "uops", exegesis::InstructionBenchmark::Uops);
+    Io.enumCase(Value, "inverse_throughput",
+                exegesis::InstructionBenchmark::InverseThroughput);
   }
 };
 
diff --git a/tools/llvm-exegesis/lib/BenchmarkResult.h b/tools/llvm-exegesis/lib/BenchmarkResult.h
index 773a2e50abc4d70053db917658120d236bf5edbc..0ef4fb3caa95cc0b865f9cc17fabcff0ade20b46 100644
--- a/tools/llvm-exegesis/lib/BenchmarkResult.h
+++ b/tools/llvm-exegesis/lib/BenchmarkResult.h
@@ -1,9 +1,8 @@
 //===-- BenchmarkResult.h ---------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
@@ -58,7 +57,7 @@ struct BenchmarkMeasure {
 // The result of an instruction benchmark.
 struct InstructionBenchmark {
   InstructionBenchmarkKey Key;
-  enum ModeE { Unknown, Latency, Uops };
+  enum ModeE { Unknown, Latency, Uops, InverseThroughput };
   ModeE Mode;
   std::string CpuName;
   std::string LLVMTriple;
diff --git a/tools/llvm-exegesis/lib/BenchmarkRunner.cpp b/tools/llvm-exegesis/lib/BenchmarkRunner.cpp
index 437503f848659dc6377253e28d56ea90bcb53475..32b68a5c23d00b503329c54e414196cf3c4ee82c 100644
--- a/tools/llvm-exegesis/lib/BenchmarkRunner.cpp
+++ b/tools/llvm-exegesis/lib/BenchmarkRunner.cpp
@@ -1,9 +1,8 @@
 //===-- BenchmarkRunner.cpp -------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/tools/llvm-exegesis/lib/BenchmarkRunner.h b/tools/llvm-exegesis/lib/BenchmarkRunner.h
index 4f77f492ab4b09e6b64df77f276b836d63e3ccee..4387bc8456e858ec03388e6a685beb8a577deb30 100644
--- a/tools/llvm-exegesis/lib/BenchmarkRunner.h
+++ b/tools/llvm-exegesis/lib/BenchmarkRunner.h
@@ -1,9 +1,8 @@
 //===-- BenchmarkRunner.h ---------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
@@ -76,6 +75,7 @@ public:
 
 protected:
   const LLVMState &State;
+  const InstructionBenchmark::ModeE Mode;
 
 private:
   virtual llvm::Expected<std::vector<BenchmarkMeasure>>
@@ -85,7 +85,6 @@ private:
   writeObjectFile(const BenchmarkCode &Configuration,
                   llvm::ArrayRef<llvm::MCInst> Code) const;
 
-  const InstructionBenchmark::ModeE Mode;
 
   const std::unique_ptr<ScratchSpace> Scratch;
 };
diff --git a/tools/llvm-exegesis/lib/Clustering.cpp b/tools/llvm-exegesis/lib/Clustering.cpp
index b2cd97c12eb0c82b9d9e81c68b249f7c00192923..36e6c5b110ffe0a69ba97a01ccb554af649d8bc1 100644
--- a/tools/llvm-exegesis/lib/Clustering.cpp
+++ b/tools/llvm-exegesis/lib/Clustering.cpp
@@ -1,9 +1,8 @@
 //===-- Clustering.cpp ------------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/tools/llvm-exegesis/lib/Clustering.h b/tools/llvm-exegesis/lib/Clustering.h
index 932cbba2038e4cfeb3196a67e44dfcb764921f09..dccc28d9518c932a9375241e1bbbf3109ba3167c 100644
--- a/tools/llvm-exegesis/lib/Clustering.h
+++ b/tools/llvm-exegesis/lib/Clustering.h
@@ -1,9 +1,8 @@
 //===-- Clustering.h --------------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
diff --git a/tools/llvm-exegesis/lib/CodeTemplate.cpp b/tools/llvm-exegesis/lib/CodeTemplate.cpp
index e159b000755acbec6a11516c5b406fee29d0296f..f538384f593b52619e473e75d678d58cfa4f6b95 100644
--- a/tools/llvm-exegesis/lib/CodeTemplate.cpp
+++ b/tools/llvm-exegesis/lib/CodeTemplate.cpp
@@ -1,9 +1,8 @@
 //===-- CodeTemplate.cpp ----------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/tools/llvm-exegesis/lib/CodeTemplate.h b/tools/llvm-exegesis/lib/CodeTemplate.h
index 4c55487f3d12161c977c5097c2b5de16967ddb23..2edd51481695765a724bbcb93e89207f45581449 100644
--- a/tools/llvm-exegesis/lib/CodeTemplate.h
+++ b/tools/llvm-exegesis/lib/CodeTemplate.h
@@ -1,9 +1,8 @@
 //===-- CodeTemplate.h ------------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
diff --git a/tools/llvm-exegesis/lib/LLVMBuild.txt b/tools/llvm-exegesis/lib/LLVMBuild.txt
index e02ea7151dfe2108e9f7a9739e8e0005eb2c7f29..85a18f2cf4c9f5cfdf297ad221fb0e7ed65aeb0f 100644
--- a/tools/llvm-exegesis/lib/LLVMBuild.txt
+++ b/tools/llvm-exegesis/lib/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./tools/llvm-exegesis/lib/LLVMBuild.txt ------------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/tools/llvm-exegesis/lib/Latency.cpp b/tools/llvm-exegesis/lib/Latency.cpp
index 9a56b275088bc7c20dc2c4d3024443edf2c13930..2f3fbaef4f4e5e59491bb00dc0af28eab6de0a5e 100644
--- a/tools/llvm-exegesis/lib/Latency.cpp
+++ b/tools/llvm-exegesis/lib/Latency.cpp
@@ -1,9 +1,8 @@
 //===-- Latency.cpp ---------------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -166,6 +165,14 @@ LatencySnippetGenerator::generateCodeTemplates(const Instruction &Instr) const {
   return std::move(Results);
 }
 
+LatencyBenchmarkRunner::LatencyBenchmarkRunner(const LLVMState &State,
+                                               InstructionBenchmark::ModeE Mode)
+    : BenchmarkRunner(State, Mode) {
+  assert((Mode == InstructionBenchmark::Latency ||
+          Mode == InstructionBenchmark::InverseThroughput) &&
+         "invalid mode");
+}
+
 LatencyBenchmarkRunner::~LatencyBenchmarkRunner() = default;
 
 llvm::Expected<std::vector<BenchmarkMeasure>>
@@ -185,8 +192,17 @@ LatencyBenchmarkRunner::runMeasurements(
     if (*ExpectedCounterValue < MinValue)
       MinValue = *ExpectedCounterValue;
   }
-  std::vector<BenchmarkMeasure> Result = {
-      BenchmarkMeasure::Create("latency", MinValue)};
+  std::vector<BenchmarkMeasure> Result;
+  switch (Mode) {
+  case InstructionBenchmark::Latency:
+    Result = {BenchmarkMeasure::Create("latency", MinValue)};
+    break;
+  case InstructionBenchmark::InverseThroughput:
+    Result = {BenchmarkMeasure::Create("inverse_throughput", MinValue)};
+    break;
+  default:
+    break;
+  }
   return std::move(Result);
 }
 
diff --git a/tools/llvm-exegesis/lib/Latency.h b/tools/llvm-exegesis/lib/Latency.h
index 513c5f3ce780d37ffde485efecae156d14c938c9..7d6d96a195efb338015e704d73ebf2c63f54afa8 100644
--- a/tools/llvm-exegesis/lib/Latency.h
+++ b/tools/llvm-exegesis/lib/Latency.h
@@ -1,9 +1,8 @@
 //===-- Latency.h -----------------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
@@ -33,8 +32,8 @@ public:
 
 class LatencyBenchmarkRunner : public BenchmarkRunner {
 public:
-  LatencyBenchmarkRunner(const LLVMState &State)
-      : BenchmarkRunner(State, InstructionBenchmark::Latency) {}
+  LatencyBenchmarkRunner(const LLVMState &State,
+                         InstructionBenchmark::ModeE Mode);
   ~LatencyBenchmarkRunner() override;
 
 private:
diff --git a/tools/llvm-exegesis/lib/LlvmState.cpp b/tools/llvm-exegesis/lib/LlvmState.cpp
index cc01f1fa4ca5777c2dd2860ea20bb8b7cc71b04d..ab487fd3897b13550d729e615888b614f6255819 100644
--- a/tools/llvm-exegesis/lib/LlvmState.cpp
+++ b/tools/llvm-exegesis/lib/LlvmState.cpp
@@ -1,9 +1,8 @@
 //===-- LlvmState.cpp -------------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/tools/llvm-exegesis/lib/LlvmState.h b/tools/llvm-exegesis/lib/LlvmState.h
index e4f12a3f858e6ca02b00b12b6b29c26f9709ce2e..f52a521c4dd5f68db1092588cf4ccaebc532b90c 100644
--- a/tools/llvm-exegesis/lib/LlvmState.h
+++ b/tools/llvm-exegesis/lib/LlvmState.h
@@ -1,9 +1,8 @@
 //===-- LlvmState.h ---------------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
diff --git a/tools/llvm-exegesis/lib/MCInstrDescView.cpp b/tools/llvm-exegesis/lib/MCInstrDescView.cpp
index e0521af4d19e11bc8899c9ed0720447633f819d9..de36d5cda0c0d70f9ccdae114064596f6a863014 100644
--- a/tools/llvm-exegesis/lib/MCInstrDescView.cpp
+++ b/tools/llvm-exegesis/lib/MCInstrDescView.cpp
@@ -1,9 +1,8 @@
 //===-- MCInstrDescView.cpp -------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/tools/llvm-exegesis/lib/MCInstrDescView.h b/tools/llvm-exegesis/lib/MCInstrDescView.h
index 58efd2a4e41c1af7a447a43cc57e77a7ea8356da..b333b92bb0e627c48d26bc2d6098c1aa66780a20 100644
--- a/tools/llvm-exegesis/lib/MCInstrDescView.h
+++ b/tools/llvm-exegesis/lib/MCInstrDescView.h
@@ -1,9 +1,8 @@
 //===-- MCInstrDescView.h ---------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
diff --git a/tools/llvm-exegesis/lib/PerfHelper.cpp b/tools/llvm-exegesis/lib/PerfHelper.cpp
index c1c242ca88fa55b1afe0b06bee696c097272a56f..e10b4e4a6803007abb621e6da4bd708b7aa40dc0 100644
--- a/tools/llvm-exegesis/lib/PerfHelper.cpp
+++ b/tools/llvm-exegesis/lib/PerfHelper.cpp
@@ -1,9 +1,8 @@
 //===-- PerfHelper.cpp ------------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/tools/llvm-exegesis/lib/PerfHelper.h b/tools/llvm-exegesis/lib/PerfHelper.h
index 2d0810846606a7ce798b648c3fe82a2e3168f628..6b800e0d245fda0079b226e378da3465e0b89387 100644
--- a/tools/llvm-exegesis/lib/PerfHelper.h
+++ b/tools/llvm-exegesis/lib/PerfHelper.h
@@ -1,9 +1,8 @@
 //===-- PerfHelper.h ------------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
diff --git a/tools/llvm-exegesis/lib/PowerPC/LLVMBuild.txt b/tools/llvm-exegesis/lib/PowerPC/LLVMBuild.txt
index c535562065eba6f2b3d667087934d3a719fdc65c..0e1b94a39a6e4f1833780215aa51e4f08fe8bd5b 100644
--- a/tools/llvm-exegesis/lib/PowerPC/LLVMBuild.txt
+++ b/tools/llvm-exegesis/lib/PowerPC/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./tools/llvm-exegesis/lib/PowerPC/LLVMBuild.txt ---------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/tools/llvm-exegesis/lib/PowerPC/Target.cpp b/tools/llvm-exegesis/lib/PowerPC/Target.cpp
index b4bdb5dc4adc33a65563c493c2ccae75a9748d95..0f8162d2296ef4f3be13c1d3c78fe430dd0629ee 100644
--- a/tools/llvm-exegesis/lib/PowerPC/Target.cpp
+++ b/tools/llvm-exegesis/lib/PowerPC/Target.cpp
@@ -1,9 +1,8 @@
 //===-- Target.cpp ----------------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 // The PowerPC ExegesisTarget.
 //===----------------------------------------------------------------------===//
diff --git a/tools/llvm-exegesis/lib/RegisterAliasing.cpp b/tools/llvm-exegesis/lib/RegisterAliasing.cpp
index 54041ca30aa02c47706b8214a310ae84d45b8e85..9c2653707ca04585d44b6358c72ec88b45b4d43c 100644
--- a/tools/llvm-exegesis/lib/RegisterAliasing.cpp
+++ b/tools/llvm-exegesis/lib/RegisterAliasing.cpp
@@ -1,9 +1,8 @@
 //===-- RegisterAliasingTracker.cpp -----------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/tools/llvm-exegesis/lib/RegisterAliasing.h b/tools/llvm-exegesis/lib/RegisterAliasing.h
index 94a2eb07f4954f911605a2ea63413ab778e76683..0ddccdd6e52a8e93f18164a57e2cb9548bc7c998 100644
--- a/tools/llvm-exegesis/lib/RegisterAliasing.h
+++ b/tools/llvm-exegesis/lib/RegisterAliasing.h
@@ -1,9 +1,8 @@
 //===-- RegisterAliasingTracker.h -------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
diff --git a/tools/llvm-exegesis/lib/RegisterValue.cpp b/tools/llvm-exegesis/lib/RegisterValue.cpp
index 2bf996cead48dbf91a422cb318ca3edfb57a647d..a8286c8d856a60109019120027896c88c23af8b1 100644
--- a/tools/llvm-exegesis/lib/RegisterValue.cpp
+++ b/tools/llvm-exegesis/lib/RegisterValue.cpp
@@ -1,9 +1,8 @@
 //===-- RegisterValue.cpp ---------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/tools/llvm-exegesis/lib/RegisterValue.h b/tools/llvm-exegesis/lib/RegisterValue.h
index 689e354e2416fe74d0b8861ed4e517f44aeacc19..da581275461604adce0706193d9ec186037ec679 100644
--- a/tools/llvm-exegesis/lib/RegisterValue.h
+++ b/tools/llvm-exegesis/lib/RegisterValue.h
@@ -1,9 +1,8 @@
 //===-- RegisterValue.h -----------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
diff --git a/tools/llvm-exegesis/lib/SnippetGenerator.cpp b/tools/llvm-exegesis/lib/SnippetGenerator.cpp
index 88ba315548d3570866869b04e5f61090180623b8..44592fd89a3f4c97d0b24fe33960e4d317d13460 100644
--- a/tools/llvm-exegesis/lib/SnippetGenerator.cpp
+++ b/tools/llvm-exegesis/lib/SnippetGenerator.cpp
@@ -1,9 +1,8 @@
 //===-- SnippetGenerator.cpp ------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/tools/llvm-exegesis/lib/SnippetGenerator.h b/tools/llvm-exegesis/lib/SnippetGenerator.h
index 967b273182b74b3f2e00fd83b56bf1ce169512f7..a07bc6981aeb69e39ae64bbe7e9258c673994a71 100644
--- a/tools/llvm-exegesis/lib/SnippetGenerator.h
+++ b/tools/llvm-exegesis/lib/SnippetGenerator.h
@@ -1,9 +1,8 @@
 //===-- SnippetGenerator.h --------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
diff --git a/tools/llvm-exegesis/lib/Target.cpp b/tools/llvm-exegesis/lib/Target.cpp
index 2c89d27b4ab601e98581ef8159c20f1ed931202a..c662f1f5566fe18803813c5e932232434757bf2c 100644
--- a/tools/llvm-exegesis/lib/Target.cpp
+++ b/tools/llvm-exegesis/lib/Target.cpp
@@ -1,9 +1,8 @@
 //===-- Target.cpp ----------------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 #include "Target.h"
@@ -46,6 +45,7 @@ ExegesisTarget::createSnippetGenerator(InstructionBenchmark::ModeE Mode,
   case InstructionBenchmark::Latency:
     return createLatencySnippetGenerator(State);
   case InstructionBenchmark::Uops:
+  case InstructionBenchmark::InverseThroughput:
     return createUopsSnippetGenerator(State);
   }
   return nullptr;
@@ -58,7 +58,8 @@ ExegesisTarget::createBenchmarkRunner(InstructionBenchmark::ModeE Mode,
   case InstructionBenchmark::Unknown:
     return nullptr;
   case InstructionBenchmark::Latency:
-    return createLatencyBenchmarkRunner(State);
+  case InstructionBenchmark::InverseThroughput:
+    return createLatencyBenchmarkRunner(State, Mode);
   case InstructionBenchmark::Uops:
     return createUopsBenchmarkRunner(State);
   }
@@ -75,9 +76,9 @@ ExegesisTarget::createUopsSnippetGenerator(const LLVMState &State) const {
   return llvm::make_unique<UopsSnippetGenerator>(State);
 }
 
-std::unique_ptr<BenchmarkRunner>
-ExegesisTarget::createLatencyBenchmarkRunner(const LLVMState &State) const {
-  return llvm::make_unique<LatencyBenchmarkRunner>(State);
+std::unique_ptr<BenchmarkRunner> ExegesisTarget::createLatencyBenchmarkRunner(
+    const LLVMState &State, InstructionBenchmark::ModeE Mode) const {
+  return llvm::make_unique<LatencyBenchmarkRunner>(State, Mode);
 }
 
 std::unique_ptr<BenchmarkRunner>
diff --git a/tools/llvm-exegesis/lib/Target.h b/tools/llvm-exegesis/lib/Target.h
index b0f0e996173634ea493edb540e4858d48d3c1899..f3429b79a34beb9a874c02eebbefd465e3abe35e 100644
--- a/tools/llvm-exegesis/lib/Target.h
+++ b/tools/llvm-exegesis/lib/Target.h
@@ -1,9 +1,8 @@
 //===-- Target.h ------------------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
@@ -131,7 +130,7 @@ private:
   std::unique_ptr<SnippetGenerator> virtual createUopsSnippetGenerator(
       const LLVMState &State) const;
   std::unique_ptr<BenchmarkRunner> virtual createLatencyBenchmarkRunner(
-      const LLVMState &State) const;
+      const LLVMState &State, InstructionBenchmark::ModeE Mode) const;
   std::unique_ptr<BenchmarkRunner> virtual createUopsBenchmarkRunner(
       const LLVMState &State) const;
 
diff --git a/tools/llvm-exegesis/lib/Uops.cpp b/tools/llvm-exegesis/lib/Uops.cpp
index 9768f4533f782e3f1831fb7002fd765bd8eb9cfd..055bcdf06a546a8407d321c72dc300d75f204015 100644
--- a/tools/llvm-exegesis/lib/Uops.cpp
+++ b/tools/llvm-exegesis/lib/Uops.cpp
@@ -1,9 +1,8 @@
 //===-- Uops.cpp ------------------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/tools/llvm-exegesis/lib/Uops.h b/tools/llvm-exegesis/lib/Uops.h
index b2a5ea177f44a33e031cba3ba08a3c66b6eb97ab..077c9a85bd51a64c10f34991c445b9277ee9b25b 100644
--- a/tools/llvm-exegesis/lib/Uops.h
+++ b/tools/llvm-exegesis/lib/Uops.h
@@ -1,9 +1,8 @@
 //===-- Uops.h --------------------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
diff --git a/tools/llvm-exegesis/lib/X86/LLVMBuild.txt b/tools/llvm-exegesis/lib/X86/LLVMBuild.txt
index 2a7ddca81e7c6a8e224835c09a0da0d6767ceebd..58a6596e3da3a4bfba63097b43f5806b09851cfa 100644
--- a/tools/llvm-exegesis/lib/X86/LLVMBuild.txt
+++ b/tools/llvm-exegesis/lib/X86/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./tools/llvm-exegesis/lib/X86LLVMBuild.txt ---------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/tools/llvm-exegesis/lib/X86/Target.cpp b/tools/llvm-exegesis/lib/X86/Target.cpp
index 43bf9b750580cfee92f05beb55daeac1c4301813..2d705c9d18af3ee9c20fc5c650e54bc23ea3cbbd 100644
--- a/tools/llvm-exegesis/lib/X86/Target.cpp
+++ b/tools/llvm-exegesis/lib/X86/Target.cpp
@@ -1,9 +1,8 @@
 //===-- Target.cpp ----------------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 #include "../Target.h"
diff --git a/tools/llvm-exegesis/llvm-exegesis.cpp b/tools/llvm-exegesis/llvm-exegesis.cpp
index a28e68ec006d3d7bbadd0f866fddc6500697c928..d9976cfb1554509ecd6a23fa11861d1291e66600 100644
--- a/tools/llvm-exegesis/llvm-exegesis.cpp
+++ b/tools/llvm-exegesis/llvm-exegesis.cpp
@@ -1,9 +1,8 @@
 //===-- llvm-exegesis.cpp ---------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
@@ -57,16 +56,19 @@ static cl::opt<std::string> SnippetsFile("snippets-file",
 static cl::opt<std::string> BenchmarkFile("benchmarks-file", cl::desc(""),
                                           cl::init(""));
 
-static cl::opt<exegesis::InstructionBenchmark::ModeE>
-    BenchmarkMode("mode", cl::desc("the mode to run"),
-                  cl::values(clEnumValN(exegesis::InstructionBenchmark::Latency,
-                                        "latency", "Instruction Latency"),
-                             clEnumValN(exegesis::InstructionBenchmark::Uops,
-                                        "uops", "Uop Decomposition"),
-                             // When not asking for a specific benchmark mode,
-                             // we'll analyse the results.
-                             clEnumValN(exegesis::InstructionBenchmark::Unknown,
-                                        "analysis", "Analysis")));
+static cl::opt<exegesis::InstructionBenchmark::ModeE> BenchmarkMode(
+    "mode", cl::desc("the mode to run"),
+    cl::values(clEnumValN(exegesis::InstructionBenchmark::Latency, "latency",
+                          "Instruction Latency"),
+               clEnumValN(exegesis::InstructionBenchmark::InverseThroughput,
+                          "inverse_throughput",
+                          "Instruction Inverse Throughput"),
+               clEnumValN(exegesis::InstructionBenchmark::Uops, "uops",
+                          "Uop Decomposition"),
+               // When not asking for a specific benchmark mode,
+               // we'll analyse the results.
+               clEnumValN(exegesis::InstructionBenchmark::Unknown, "analysis",
+                          "Analysis")));
 
 static cl::opt<unsigned>
     NumRepetitions("num-repetitions",
@@ -89,10 +91,10 @@ static cl::opt<float>
 
 static cl::opt<std::string>
     AnalysisClustersOutputFile("analysis-clusters-output-file", cl::desc(""),
-                               cl::init("-"));
+                               cl::init(""));
 static cl::opt<std::string>
     AnalysisInconsistenciesOutputFile("analysis-inconsistencies-output-file",
-                                      cl::desc(""), cl::init("-"));
+                                      cl::desc(""), cl::init(""));
 
 static cl::opt<std::string>
     CpuName("mcpu",
@@ -189,8 +191,7 @@ public:
   // Implementation of the llvm::MCStreamer interface. We only care about
   // instructions.
   void EmitInstruction(const llvm::MCInst &Instruction,
-                       const llvm::MCSubtargetInfo &STI,
-                       bool PrintSchedInfo) override {
+                       const llvm::MCSubtargetInfo &STI) override {
     Result->Instructions.push_back(Instruction);
   }
 
@@ -402,6 +403,13 @@ static void analysisMain() {
   if (BenchmarkFile.empty())
     llvm::report_fatal_error("--benchmarks-file must be set.");
 
+  if (AnalysisClustersOutputFile.empty() &&
+      AnalysisInconsistenciesOutputFile.empty()) {
+    llvm::report_fatal_error(
+        "At least one of --analysis-clusters-output-file and "
+        "--analysis-inconsistencies-output-file must be specified.");
+  }
+
   llvm::InitializeNativeTarget();
   llvm::InitializeNativeTargetAsmPrinter();
   llvm::InitializeNativeTargetDisassembler();
diff --git a/tools/llvm-extract/LLVMBuild.txt b/tools/llvm-extract/LLVMBuild.txt
index 70e3507a73eb41efa31961f23c3cf759200aec76..77072b4e899a65bd1c47760c7847b2b7998ccd35 100644
--- a/tools/llvm-extract/LLVMBuild.txt
+++ b/tools/llvm-extract/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./tools/llvm-extract/LLVMBuild.txt -----------------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/tools/llvm-extract/llvm-extract.cpp b/tools/llvm-extract/llvm-extract.cpp
index 94aaa2f52eb58738591d51ac6ce40f694ccbf3fc..30c89a874398428855eb61001d04bd963fc37bc3 100644
--- a/tools/llvm-extract/llvm-extract.cpp
+++ b/tools/llvm-extract/llvm-extract.cpp
@@ -1,9 +1,8 @@
 //===- llvm-extract.cpp - LLVM function extraction utility ----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/tools/llvm-go/llvm-go.go b/tools/llvm-go/llvm-go.go
index 8215483b83329d87573ad83d6cb65deb4094dda6..cdcdf044381cede2cbd510aa79f6374a1487eece 100644
--- a/tools/llvm-go/llvm-go.go
+++ b/tools/llvm-go/llvm-go.go
@@ -1,9 +1,8 @@
 //===-- llvm-go.go - go tool wrapper for LLVM -----------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/tools/llvm-isel-fuzzer/DummyISelFuzzer.cpp b/tools/llvm-isel-fuzzer/DummyISelFuzzer.cpp
index ec9c74e3040dd830b0f096904063672fc3700c06..82fbd73e3872004e4bbd05315b42d5fa014b766b 100644
--- a/tools/llvm-isel-fuzzer/DummyISelFuzzer.cpp
+++ b/tools/llvm-isel-fuzzer/DummyISelFuzzer.cpp
@@ -1,9 +1,8 @@
 //===--- DummyFuzzerMain.cpp - Entry point to sanity check the fuzzer -----===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/tools/llvm-isel-fuzzer/llvm-isel-fuzzer.cpp b/tools/llvm-isel-fuzzer/llvm-isel-fuzzer.cpp
index 0cab6799f22c9d1ad89236e564cf8344930f86ad..a348ab300d19d121f589aa74ef1c97fdd3a1b07b 100644
--- a/tools/llvm-isel-fuzzer/llvm-isel-fuzzer.cpp
+++ b/tools/llvm-isel-fuzzer/llvm-isel-fuzzer.cpp
@@ -1,9 +1,8 @@
 //===--- llvm-isel-fuzzer.cpp - Fuzzer for instruction selection ----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/tools/llvm-demangle-fuzzer/CMakeLists.txt b/tools/llvm-itanium-demangle-fuzzer/CMakeLists.txt
similarity index 56%
rename from tools/llvm-demangle-fuzzer/CMakeLists.txt
rename to tools/llvm-itanium-demangle-fuzzer/CMakeLists.txt
index 0fe711cdb16c75db2b7da4513f9296802a5b026c..07f02a35b20313dbf406d09e36282e479790bdc3 100644
--- a/tools/llvm-demangle-fuzzer/CMakeLists.txt
+++ b/tools/llvm-itanium-demangle-fuzzer/CMakeLists.txt
@@ -4,7 +4,7 @@ set(LLVM_LINK_COMPONENTS
   Support
 )
 
-add_llvm_fuzzer(llvm-demangle-fuzzer
-  llvm-demangle-fuzzer.cpp
+add_llvm_fuzzer(llvm-itanium-demangle-fuzzer
+  llvm-itanium-demangle-fuzzer.cpp
   DUMMY_MAIN DummyDemanglerFuzzer.cpp
   )
diff --git a/tools/llvm-demangle-fuzzer/DummyDemanglerFuzzer.cpp b/tools/llvm-itanium-demangle-fuzzer/DummyDemanglerFuzzer.cpp
similarity index 73%
rename from tools/llvm-demangle-fuzzer/DummyDemanglerFuzzer.cpp
rename to tools/llvm-itanium-demangle-fuzzer/DummyDemanglerFuzzer.cpp
index a2bf9f1b807e5c20b61842e71791829ebd4b4561..f305ca5a800f79f45fc5768a7a5f747f7c309c16 100644
--- a/tools/llvm-demangle-fuzzer/DummyDemanglerFuzzer.cpp
+++ b/tools/llvm-itanium-demangle-fuzzer/DummyDemanglerFuzzer.cpp
@@ -1,9 +1,8 @@
 //===--- DummyDemanglerMain.cpp - Entry point to sanity check the fuzzer --===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/tools/llvm-demangle-fuzzer/llvm-demangle-fuzzer.cpp b/tools/llvm-itanium-demangle-fuzzer/llvm-itanium-demangle-fuzzer.cpp
similarity index 74%
rename from tools/llvm-demangle-fuzzer/llvm-demangle-fuzzer.cpp
rename to tools/llvm-itanium-demangle-fuzzer/llvm-itanium-demangle-fuzzer.cpp
index 07c290a0be5c800e08cfb1ea60e8f422c9c800dc..dcc2bd7ef97b8203bf503eb9968a3129133ad12c 100644
--- a/tools/llvm-demangle-fuzzer/llvm-demangle-fuzzer.cpp
+++ b/tools/llvm-itanium-demangle-fuzzer/llvm-itanium-demangle-fuzzer.cpp
@@ -1,9 +1,8 @@
 //===--- llvm-demangle-fuzzer.cpp - Fuzzer for the Itanium Demangler ------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/tools/llvm-jitlistener/LLVMBuild.txt b/tools/llvm-jitlistener/LLVMBuild.txt
index 8861cc742f46ae002ef58891612fcd44b32cb6e5..4a1390d00571554ddd0e89fba641e0aff28a6b09 100644
--- a/tools/llvm-jitlistener/LLVMBuild.txt
+++ b/tools/llvm-jitlistener/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./tools/llvm-jitlistener/LLVMBuild.txt -------------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/tools/llvm-jitlistener/llvm-jitlistener.cpp b/tools/llvm-jitlistener/llvm-jitlistener.cpp
index 693c69d8fa7b29b0c53b9be5ea802f1bfbbcb8d7..82a798d3f74d3c5de0c12cf4ca95e6a678659dbc 100644
--- a/tools/llvm-jitlistener/llvm-jitlistener.cpp
+++ b/tools/llvm-jitlistener/llvm-jitlistener.cpp
@@ -1,9 +1,8 @@
 //===-- llvm-jitlistener.cpp - Utility for testing MCJIT event listener ---===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/tools/llvm-link/LLVMBuild.txt b/tools/llvm-link/LLVMBuild.txt
index c7476043164a7100a8633dd7543d6c10521bb111..adc6a25c8b5fc52de42fa25321740c4a2483aa73 100644
--- a/tools/llvm-link/LLVMBuild.txt
+++ b/tools/llvm-link/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./tools/llvm-link/LLVMBuild.txt --------------------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/tools/llvm-link/llvm-link.cpp b/tools/llvm-link/llvm-link.cpp
index b7a888375b3d27f9ff593aafff149cc8a2acf9be..50ba57178d02f97897573931f81d8452ae1e35b7 100644
--- a/tools/llvm-link/llvm-link.cpp
+++ b/tools/llvm-link/llvm-link.cpp
@@ -1,9 +1,8 @@
 //===- llvm-link.cpp - Low-level LLVM linker ------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/tools/llvm-lto/LLVMBuild.txt b/tools/llvm-lto/LLVMBuild.txt
index 7a86797899dea7318d64c984ff1d189ef0f997b6..bc7cdcd2db940888ea89d65f99b1eb7afb3111a4 100644
--- a/tools/llvm-lto/LLVMBuild.txt
+++ b/tools/llvm-lto/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./tools/llvm-lto/LLVMBuild.txt ----------------------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/tools/llvm-lto/llvm-lto.cpp b/tools/llvm-lto/llvm-lto.cpp
index b6facc919b5148bd0a5d9a0a6d345590f617baa8..a71c3dd445cbcf59ed6bc3a03dd46619f81a636e 100644
--- a/tools/llvm-lto/llvm-lto.cpp
+++ b/tools/llvm-lto/llvm-lto.cpp
@@ -1,9 +1,8 @@
 //===- llvm-lto: a simple command-line program to link modules with LTO ---===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/tools/llvm-lto2/LLVMBuild.txt b/tools/llvm-lto2/LLVMBuild.txt
index d51aa22906751dda8214513b8c7fd250cedd4dd3..1665b331202c0db1dc0fcef5e86dbebba266e55f 100644
--- a/tools/llvm-lto2/LLVMBuild.txt
+++ b/tools/llvm-lto2/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./tools/llvm-lto2/LLVMBuild.txt --------------------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/tools/llvm-lto2/llvm-lto2.cpp b/tools/llvm-lto2/llvm-lto2.cpp
index 26426367e252ed4a8650bada41ed90109a098e3e..d9950815a61ad332deb53968201cf0bddf225298 100644
--- a/tools/llvm-lto2/llvm-lto2.cpp
+++ b/tools/llvm-lto2/llvm-lto2.cpp
@@ -1,9 +1,8 @@
 //===-- llvm-lto2: test harness for the resolution-based LTO interface ----===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/tools/llvm-mc-assemble-fuzzer/llvm-mc-assemble-fuzzer.cpp b/tools/llvm-mc-assemble-fuzzer/llvm-mc-assemble-fuzzer.cpp
index 8731cf60fca8b8004c0c824a8dadd8002809578d..9d58f6b1f4a6bc904932817670c96ca32ea460ba 100644
--- a/tools/llvm-mc-assemble-fuzzer/llvm-mc-assemble-fuzzer.cpp
+++ b/tools/llvm-mc-assemble-fuzzer/llvm-mc-assemble-fuzzer.cpp
@@ -1,9 +1,8 @@
 //===--- llvm-mc-fuzzer.cpp - Fuzzer for the MC layer ---------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/tools/llvm-mc-disassemble-fuzzer/llvm-mc-disassemble-fuzzer.cpp b/tools/llvm-mc-disassemble-fuzzer/llvm-mc-disassemble-fuzzer.cpp
index 36d1f7ca994879165a41839133ffa461eb9f84a2..ecc415bf8aded949a97740d58d2dc5f09fd25f28 100644
--- a/tools/llvm-mc-disassemble-fuzzer/llvm-mc-disassemble-fuzzer.cpp
+++ b/tools/llvm-mc-disassemble-fuzzer/llvm-mc-disassemble-fuzzer.cpp
@@ -1,9 +1,8 @@
 //===--- llvm-mc-fuzzer.cpp - Fuzzer for the MC layer ---------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/tools/llvm-mc/Disassembler.cpp b/tools/llvm-mc/Disassembler.cpp
index acc5a5f4cab2dfd58c8bc46b03d55d3f90af6040..e2af2e7f2e324dccdabdce79a5352f63c7f113b3 100644
--- a/tools/llvm-mc/Disassembler.cpp
+++ b/tools/llvm-mc/Disassembler.cpp
@@ -1,9 +1,8 @@
 //===- Disassembler.cpp - Disassembler for hex strings --------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/tools/llvm-mc/Disassembler.h b/tools/llvm-mc/Disassembler.h
index 1f18ac075f854be6a2fe2e56edc8cfbebd7da45b..11b685233abcc3755925a0896c443194224b5c7b 100644
--- a/tools/llvm-mc/Disassembler.h
+++ b/tools/llvm-mc/Disassembler.h
@@ -1,9 +1,8 @@
 //===- Disassembler.h - Text File Disassembler ----------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/tools/llvm-mc/LLVMBuild.txt b/tools/llvm-mc/LLVMBuild.txt
index dff5358f3858da1f30918e692ac6f7cbeac24c53..64f8ed0a9896c428be27736759f3bf848d533240 100644
--- a/tools/llvm-mc/LLVMBuild.txt
+++ b/tools/llvm-mc/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./tools/llvm-mc/LLVMBuild.txt ----------------------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/tools/llvm-mc/llvm-mc.cpp b/tools/llvm-mc/llvm-mc.cpp
index c0976502f54533ffa1e7e93263f0c48fd07498d3..af19a64d37d10d4d121f032c9d658886a82987b3 100644
--- a/tools/llvm-mc/llvm-mc.cpp
+++ b/tools/llvm-mc/llvm-mc.cpp
@@ -1,9 +1,8 @@
 //===-- llvm-mc.cpp - Machine Code Hacking Driver ---------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/tools/llvm-mca/CodeRegion.cpp b/tools/llvm-mca/CodeRegion.cpp
index 29a27c50c171f3ff5f99fedf53ffc4a1c51af298..83541675774b345af3b0c925af419d93dd6a2856 100644
--- a/tools/llvm-mca/CodeRegion.cpp
+++ b/tools/llvm-mca/CodeRegion.cpp
@@ -1,9 +1,8 @@
 //===-------------------------- CodeRegion.cpp -----------------*- C++ -* -===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/tools/llvm-mca/CodeRegion.h b/tools/llvm-mca/CodeRegion.h
index 867aa18bb4fe0d3f70c24767b5bb347b7b4983d8..1cdfeabfef4d3d2534f07a8ba17b9a396eedf036 100644
--- a/tools/llvm-mca/CodeRegion.h
+++ b/tools/llvm-mca/CodeRegion.h
@@ -1,9 +1,8 @@
 //===-------------------------- CodeRegion.h -------------------*- C++ -* -===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/tools/llvm-mca/CodeRegionGenerator.cpp b/tools/llvm-mca/CodeRegionGenerator.cpp
index 5bd37adeeae999a911f208ba9d6eb2a1e7915e7b..4660af2c40aca37fca05172259029d6171867d05 100644
--- a/tools/llvm-mca/CodeRegionGenerator.cpp
+++ b/tools/llvm-mca/CodeRegionGenerator.cpp
@@ -1,9 +1,8 @@
 //===----------------------- CodeRegionGenerator.cpp ------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
@@ -49,8 +48,7 @@ public:
 
   // We only want to intercept the emission of new instructions.
   virtual void EmitInstruction(const MCInst &Inst,
-                               const MCSubtargetInfo & /* unused */,
-                               bool /* unused */) override {
+                               const MCSubtargetInfo &/* unused */) override {
     Regions.addInstruction(Inst);
   }
 
diff --git a/tools/llvm-mca/CodeRegionGenerator.h b/tools/llvm-mca/CodeRegionGenerator.h
index 892cafb926861a44fb5f5d56efddc06444aa62a7..9a10aa2c148b730e6e3d092409ab98dcf040a8e5 100644
--- a/tools/llvm-mca/CodeRegionGenerator.h
+++ b/tools/llvm-mca/CodeRegionGenerator.h
@@ -1,9 +1,8 @@
 //===----------------------- CodeRegionGenerator.h --------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/tools/llvm-mca/LLVMBuild.txt b/tools/llvm-mca/LLVMBuild.txt
index a704612934ff11dd490bc780db47932c797a7372..d1f9fc81060c2624e44fdfb9850bc84969f05b98 100644
--- a/tools/llvm-mca/LLVMBuild.txt
+++ b/tools/llvm-mca/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./tools/llvm-mc/LLVMBuild.txt ----------------------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/tools/llvm-mca/PipelinePrinter.cpp b/tools/llvm-mca/PipelinePrinter.cpp
index 18ef45fc2a6553752c45340f42bb438ae8be4102..90d46807599693707066fe68ecc3e0d060cbc4a2 100644
--- a/tools/llvm-mca/PipelinePrinter.cpp
+++ b/tools/llvm-mca/PipelinePrinter.cpp
@@ -1,9 +1,8 @@
 //===--------------------- PipelinePrinter.cpp ------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/tools/llvm-mca/PipelinePrinter.h b/tools/llvm-mca/PipelinePrinter.h
index 456026e12df39fe029619e07d15794f52bd5ad68..004309cd7b8e2d01caa1ae617a2380376f46d56c 100644
--- a/tools/llvm-mca/PipelinePrinter.h
+++ b/tools/llvm-mca/PipelinePrinter.h
@@ -1,9 +1,8 @@
 //===--------------------- PipelinePrinter.h --------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/tools/llvm-mca/Views/DispatchStatistics.cpp b/tools/llvm-mca/Views/DispatchStatistics.cpp
index 2562c82407bf52423603d015ce95d69e3c909f19..557b8ba17b17f796912571a87522f4a7df5eda6a 100644
--- a/tools/llvm-mca/Views/DispatchStatistics.cpp
+++ b/tools/llvm-mca/Views/DispatchStatistics.cpp
@@ -1,10 +1,9 @@
 //===--------------------- DispatchStatistics.cpp ---------------------*- C++
 //-*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/tools/llvm-mca/Views/DispatchStatistics.h b/tools/llvm-mca/Views/DispatchStatistics.h
index 6679c81efe954ada2db98196bab5d851828ee206..07c0f5a4c68f978d37ad0361397bd83751c779ee 100644
--- a/tools/llvm-mca/Views/DispatchStatistics.h
+++ b/tools/llvm-mca/Views/DispatchStatistics.h
@@ -1,9 +1,8 @@
 //===--------------------- DispatchStatistics.h -----------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/tools/llvm-mca/Views/InstructionInfoView.cpp b/tools/llvm-mca/Views/InstructionInfoView.cpp
index 5016afb49e442af600a4d8f2161bae973aecc793..1fbffa3e5b69798f0bc694de641e015a1abb1788 100644
--- a/tools/llvm-mca/Views/InstructionInfoView.cpp
+++ b/tools/llvm-mca/Views/InstructionInfoView.cpp
@@ -1,9 +1,8 @@
 //===--------------------- InstructionInfoView.cpp --------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
@@ -44,6 +43,9 @@ void InstructionInfoView::printView(raw_ostream &OS) const {
     const MCSchedClassDesc &SCDesc = *SM.getSchedClassDesc(SchedClassID);
     unsigned NumMicroOpcodes = SCDesc.NumMicroOps;
     unsigned Latency = MCSchedModel::computeInstrLatency(STI, SCDesc);
+    // Add extra latency due to delays in the forwarding data paths.
+    Latency += MCSchedModel::getForwardingDelayCycles(
+        STI.getReadAdvanceEntries(SCDesc));
     Optional<double> RThroughput =
         MCSchedModel::getReciprocalThroughput(STI, SCDesc);
 
diff --git a/tools/llvm-mca/Views/InstructionInfoView.h b/tools/llvm-mca/Views/InstructionInfoView.h
index 3ef95d474490b0e9c7dc0b210460f792b70c5974..640d87383436ef10baf3f776f4a6f057c0a326de 100644
--- a/tools/llvm-mca/Views/InstructionInfoView.h
+++ b/tools/llvm-mca/Views/InstructionInfoView.h
@@ -1,9 +1,8 @@
 //===--------------------- InstructionInfoView.h ----------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/tools/llvm-mca/Views/RegisterFileStatistics.cpp b/tools/llvm-mca/Views/RegisterFileStatistics.cpp
index 06202bc414212af9fe369d931154567509a23b9b..58736ee0d18c30e6f43e4cc1ab40e753892749e8 100644
--- a/tools/llvm-mca/Views/RegisterFileStatistics.cpp
+++ b/tools/llvm-mca/Views/RegisterFileStatistics.cpp
@@ -1,9 +1,8 @@
 //===--------------------- RegisterFileStatistics.cpp -----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/tools/llvm-mca/Views/RegisterFileStatistics.h b/tools/llvm-mca/Views/RegisterFileStatistics.h
index a2c52a668daef74ea0388751ec78a653ed5d47d5..a2273dd48b221ccb7c6adc412e41094ae6969dd6 100644
--- a/tools/llvm-mca/Views/RegisterFileStatistics.h
+++ b/tools/llvm-mca/Views/RegisterFileStatistics.h
@@ -1,9 +1,8 @@
 //===--------------------- RegisterFileStatistics.h -------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/tools/llvm-mca/Views/ResourcePressureView.cpp b/tools/llvm-mca/Views/ResourcePressureView.cpp
index 6df61840437df225ff41bdb3419e469dee5cb4b6..38a2478cf4fecebd9caf104ac71d411c0716ef30 100644
--- a/tools/llvm-mca/Views/ResourcePressureView.cpp
+++ b/tools/llvm-mca/Views/ResourcePressureView.cpp
@@ -1,9 +1,8 @@
 //===--------------------- ResourcePressureView.cpp -------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/tools/llvm-mca/Views/ResourcePressureView.h b/tools/llvm-mca/Views/ResourcePressureView.h
index 572ce6fe6b7007eed7dc655d2c748b4892e74b80..0fa0b9a36aa381e987906e61d55c4d0e908d2290 100644
--- a/tools/llvm-mca/Views/ResourcePressureView.h
+++ b/tools/llvm-mca/Views/ResourcePressureView.h
@@ -1,9 +1,8 @@
 //===--------------------- ResourcePressureView.h ---------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/tools/llvm-mca/Views/RetireControlUnitStatistics.cpp b/tools/llvm-mca/Views/RetireControlUnitStatistics.cpp
index 54eb28f1add9da0dafe9f4e4aa0a6fb55a0b9184..cb4fbae78039b4c4da5fe0f7aaea90cc13a63925 100644
--- a/tools/llvm-mca/Views/RetireControlUnitStatistics.cpp
+++ b/tools/llvm-mca/Views/RetireControlUnitStatistics.cpp
@@ -1,9 +1,8 @@
 //===--------------------- RetireControlUnitStatistics.cpp ------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/tools/llvm-mca/Views/RetireControlUnitStatistics.h b/tools/llvm-mca/Views/RetireControlUnitStatistics.h
index 02aa13bc444a24b246bfbe7b2b3a4a029ab24a84..1a4d3dec5c56f1ac566d95ad39061fbb8ea6bad5 100644
--- a/tools/llvm-mca/Views/RetireControlUnitStatistics.h
+++ b/tools/llvm-mca/Views/RetireControlUnitStatistics.h
@@ -1,9 +1,8 @@
 //===--------------------- RetireControlUnitStatistics.h --------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/tools/llvm-mca/Views/SchedulerStatistics.cpp b/tools/llvm-mca/Views/SchedulerStatistics.cpp
index 670f90127f18b51e83530e00ccc686da39f4fa48..a6143494d48601c5ef68ad85701ed0cb1d314fef 100644
--- a/tools/llvm-mca/Views/SchedulerStatistics.cpp
+++ b/tools/llvm-mca/Views/SchedulerStatistics.cpp
@@ -1,9 +1,8 @@
 //===--------------------- SchedulerStatistics.cpp --------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/tools/llvm-mca/Views/SchedulerStatistics.h b/tools/llvm-mca/Views/SchedulerStatistics.h
index d99a395a726d0d2de72832ceceaba097d7f5df88..8e4127c131d9fd32a2291f13a9c203f8b6681aca 100644
--- a/tools/llvm-mca/Views/SchedulerStatistics.h
+++ b/tools/llvm-mca/Views/SchedulerStatistics.h
@@ -1,9 +1,8 @@
 //===--------------------- SchedulerStatistics.h ----------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/tools/llvm-mca/Views/SummaryView.cpp b/tools/llvm-mca/Views/SummaryView.cpp
index 3a7eb827c478e1b37152a3c3328cbc03113e2a2e..55d73c6fcb17bc3b047ce63750e6d226accfa9bd 100644
--- a/tools/llvm-mca/Views/SummaryView.cpp
+++ b/tools/llvm-mca/Views/SummaryView.cpp
@@ -1,9 +1,8 @@
 //===--------------------- SummaryView.cpp -------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
@@ -27,7 +26,8 @@ SummaryView::SummaryView(const MCSchedModel &Model, ArrayRef<MCInst> S,
                          unsigned Width)
     : SM(Model), Source(S), DispatchWidth(Width), LastInstructionIdx(0),
       TotalCycles(0), NumMicroOps(0),
-      ProcResourceUsage(Model.getNumProcResourceKinds(), 0) {
+      ProcResourceUsage(Model.getNumProcResourceKinds(), 0),
+      ProcResourceMasks(Model.getNumProcResourceKinds()) {
   computeProcResourceMasks(SM, ProcResourceMasks);
 }
 
diff --git a/tools/llvm-mca/Views/SummaryView.h b/tools/llvm-mca/Views/SummaryView.h
index f59fd4233fbecc03bf7f88a036659e8b7bff1cb5..d1d43d6818a84b0a015483694a17b8b1789643dd 100644
--- a/tools/llvm-mca/Views/SummaryView.h
+++ b/tools/llvm-mca/Views/SummaryView.h
@@ -1,9 +1,8 @@
 //===--------------------- SummaryView.h ---------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/tools/llvm-mca/Views/TimelineView.cpp b/tools/llvm-mca/Views/TimelineView.cpp
index 7d55bbc99c73d7666319a40063e99a5689df0924..fe3f16ba344cb4781d6b53d649dfcc3fd7144ced 100644
--- a/tools/llvm-mca/Views/TimelineView.cpp
+++ b/tools/llvm-mca/Views/TimelineView.cpp
@@ -1,9 +1,8 @@
 //===--------------------- TimelineView.cpp ---------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \brief
diff --git a/tools/llvm-mca/Views/TimelineView.h b/tools/llvm-mca/Views/TimelineView.h
index ee981800161c90c0112fc7c3f432ce2e8b26e34c..b63b234293cda03f9a83a9f873112a71eb5541cf 100644
--- a/tools/llvm-mca/Views/TimelineView.h
+++ b/tools/llvm-mca/Views/TimelineView.h
@@ -1,9 +1,8 @@
 //===--------------------- TimelineView.h -----------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \brief
diff --git a/tools/llvm-mca/Views/View.cpp b/tools/llvm-mca/Views/View.cpp
index 6cfb9dd9f3948b5c24e046d64edda1f2c777933f..8e5c34d2d5c247df0a1d91ea6244e42796bbe631 100644
--- a/tools/llvm-mca/Views/View.cpp
+++ b/tools/llvm-mca/Views/View.cpp
@@ -1,9 +1,8 @@
 //===----------------------- View.cpp ---------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/tools/llvm-mca/Views/View.h b/tools/llvm-mca/Views/View.h
index 4b82b0da0d27d6e4ace69cd3f2a9f11594a76753..3b52511b4d2922264a896cb9024a66a419b06da2 100644
--- a/tools/llvm-mca/Views/View.h
+++ b/tools/llvm-mca/Views/View.h
@@ -1,9 +1,8 @@
 //===----------------------- View.h -----------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/tools/llvm-mca/llvm-mca.cpp b/tools/llvm-mca/llvm-mca.cpp
index d0da3b85c28e9023b514ae6ccf1d713436859768..704a7b40efd7cac569f8149fc1eb15194fa844b1 100644
--- a/tools/llvm-mca/llvm-mca.cpp
+++ b/tools/llvm-mca/llvm-mca.cpp
@@ -1,9 +1,8 @@
 //===-- llvm-mca.cpp - Machine Code Analyzer -------------------*- C++ -* -===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -149,15 +148,13 @@ static cl::opt<bool>
                   cl::desc("If set, assume that loads and stores do not alias"),
                   cl::cat(ToolOptions), cl::init(true));
 
-static cl::opt<unsigned>
-    LoadQueueSize("lqueue",
-                  cl::desc("Size of the load queue"),
-                  cl::cat(ToolOptions), cl::init(0));
+static cl::opt<unsigned> LoadQueueSize("lqueue",
+                                       cl::desc("Size of the load queue"),
+                                       cl::cat(ToolOptions), cl::init(0));
 
-static cl::opt<unsigned>
-    StoreQueueSize("squeue",
-                   cl::desc("Size of the store queue"),
-                   cl::cat(ToolOptions), cl::init(0));
+static cl::opt<unsigned> StoreQueueSize("squeue",
+                                        cl::desc("Size of the store queue"),
+                                        cl::cat(ToolOptions), cl::init(0));
 
 static cl::opt<bool>
     PrintInstructionTables("instruction-tables",
@@ -339,8 +336,14 @@ int main(int argc, char **argv) {
   // Parse the input and create CodeRegions that llvm-mca can analyze.
   mca::AsmCodeRegionGenerator CRG(*TheTarget, SrcMgr, Ctx, *MAI, *STI, *MCII);
   Expected<const mca::CodeRegions &> RegionsOrErr = CRG.parseCodeRegions();
-  if (auto Err = RegionsOrErr.takeError()) {
-    WithColor::error() << Err << "\n";
+  if (!RegionsOrErr) {
+    if (auto Err =
+            handleErrors(RegionsOrErr.takeError(), [](const StringError &E) {
+              WithColor::error() << E.getMessage() << '\n';
+            })) {
+      // Default case.
+      WithColor::error() << toString(std::move(Err)) << '\n';
+    }
     return 1;
   }
   const mca::CodeRegions &Regions = *RegionsOrErr;
diff --git a/tools/llvm-microsoft-demangle-fuzzer/CMakeLists.txt b/tools/llvm-microsoft-demangle-fuzzer/CMakeLists.txt
new file mode 100644
index 0000000000000000000000000000000000000000..d3db43a686bbd38a598950f7192b38f1253a2896
--- /dev/null
+++ b/tools/llvm-microsoft-demangle-fuzzer/CMakeLists.txt
@@ -0,0 +1,10 @@
+set(LLVM_LINK_COMPONENTS
+  Demangle
+  FuzzMutate
+  Support
+)
+
+add_llvm_fuzzer(llvm-microsoft-demangle-fuzzer
+  llvm-microsoft-demangle-fuzzer.cpp
+  DUMMY_MAIN DummyDemanglerFuzzer.cpp
+  )
diff --git a/tools/llvm-microsoft-demangle-fuzzer/DummyDemanglerFuzzer.cpp b/tools/llvm-microsoft-demangle-fuzzer/DummyDemanglerFuzzer.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..f305ca5a800f79f45fc5768a7a5f747f7c309c16
--- /dev/null
+++ b/tools/llvm-microsoft-demangle-fuzzer/DummyDemanglerFuzzer.cpp
@@ -0,0 +1,18 @@
+//===--- DummyDemanglerMain.cpp - Entry point to sanity check the fuzzer --===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Implementation of main so we can build and test without linking libFuzzer.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/FuzzMutate/FuzzerCLI.h"
+
+extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size);
+int main(int argc, char *argv[]) {
+  return llvm::runFuzzerOnInputs(argc, argv, LLVMFuzzerTestOneInput);
+}
diff --git a/tools/llvm-microsoft-demangle-fuzzer/llvm-microsoft-demangle-fuzzer.cpp b/tools/llvm-microsoft-demangle-fuzzer/llvm-microsoft-demangle-fuzzer.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..2aa4d8dca27bc7837e59e9492aa367a7a7ef08d5
--- /dev/null
+++ b/tools/llvm-microsoft-demangle-fuzzer/llvm-microsoft-demangle-fuzzer.cpp
@@ -0,0 +1,20 @@
+//===--- llvm-demangle-fuzzer.cpp - Fuzzer for the Itanium Demangler ------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Demangle/Demangle.h"
+
+#include <cstdint>
+#include <cstdlib>
+#include <string>
+
+extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
+  std::string NullTerminatedString((const char *)Data, Size);
+  free(llvm::microsoftDemangle(NullTerminatedString.c_str(), nullptr, nullptr,
+                               nullptr));
+  return 0;
+}
diff --git a/tools/llvm-modextract/LLVMBuild.txt b/tools/llvm-modextract/LLVMBuild.txt
index 6ae5a678f3209251b648149dd15f6587d4b877a5..24cb8806d1c92cfb126dc59423e9838a1cc70fe8 100644
--- a/tools/llvm-modextract/LLVMBuild.txt
+++ b/tools/llvm-modextract/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./tools/llvm-modextract/LLVMBuild.txt --------------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/tools/llvm-modextract/llvm-modextract.cpp b/tools/llvm-modextract/llvm-modextract.cpp
index 9fd8340505aa0b37425ccb03563198d28505d680..3adefc5f0d3ed513d96b232c4fffc43f558ee92b 100644
--- a/tools/llvm-modextract/llvm-modextract.cpp
+++ b/tools/llvm-modextract/llvm-modextract.cpp
@@ -1,9 +1,8 @@
 //===-- llvm-modextract.cpp - LLVM module extractor utility ---------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/tools/llvm-mt/LLVMBuild.txt b/tools/llvm-mt/LLVMBuild.txt
index 02fc9ace7552165b017e2f2dbebeb5d8abcdb357..322477023041130ee9432b602234bdaad4bc9d88 100644
--- a/tools/llvm-mt/LLVMBuild.txt
+++ b/tools/llvm-mt/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./tools/llvm-mt/LLVMBuild.txt ---------------------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/tools/llvm-mt/llvm-mt.cpp b/tools/llvm-mt/llvm-mt.cpp
index ea75c012eac89b11ef233aad118e22303baca012..cac9b878abbad8406eddd4c9eb31b3bba154f9ab 100644
--- a/tools/llvm-mt/llvm-mt.cpp
+++ b/tools/llvm-mt/llvm-mt.cpp
@@ -1,9 +1,8 @@
 //===- llvm-mt.cpp - Merge .manifest files ---------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===---------------------------------------------------------------------===//
 //
diff --git a/tools/llvm-nm/LLVMBuild.txt b/tools/llvm-nm/LLVMBuild.txt
index 3e64577d6d241d256cf9b520994d1d788ac007bc..1486786c2c74a60c0cda5a30ce2c8e53df8613c6 100644
--- a/tools/llvm-nm/LLVMBuild.txt
+++ b/tools/llvm-nm/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./tools/llvm-nm/LLVMBuild.txt ----------------------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/tools/llvm-nm/llvm-nm.cpp b/tools/llvm-nm/llvm-nm.cpp
index 88217a5403fd68cffc65324f05702234044c5674..7c009297139638a18fa62245fbb6a94a4e7eccfb 100644
--- a/tools/llvm-nm/llvm-nm.cpp
+++ b/tools/llvm-nm/llvm-nm.cpp
@@ -1,9 +1,8 @@
 //===-- llvm-nm.cpp - Symbol table dumping utility for llvm ---------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -91,6 +90,8 @@ cl::opt<bool> BSDFormat("B", cl::desc("Alias for --format=bsd"),
                         cl::Grouping);
 cl::opt<bool> POSIXFormat("P", cl::desc("Alias for --format=posix"),
                           cl::Grouping);
+cl::alias Portability("portability", cl::desc("Alias for --format=posix"),
+                      cl::aliasopt(POSIXFormat), cl::NotHidden);
 cl::opt<bool> DarwinFormat("m", cl::desc("Alias for --format=darwin"),
                            cl::Grouping);
 
@@ -553,6 +554,11 @@ static void darwinPrintSymbol(SymbolicFile &Obj, SymbolListT::iterator I,
       (NDesc & MachO::N_ALT_ENTRY) == MachO::N_ALT_ENTRY)
     outs() << "[alt entry] ";
 
+  if (Filetype == MachO::MH_OBJECT &&
+      ((NType & MachO::N_TYPE) != MachO::N_UNDF) &&
+      (NDesc & MachO::N_COLD_FUNC) == MachO::N_COLD_FUNC)
+    outs() << "[cold func] ";
+
   if ((NDesc & MachO::N_ARM_THUMB_DEF) == MachO::N_ARM_THUMB_DEF)
     outs() << "[Thumb] ";
 
@@ -677,7 +683,7 @@ static void darwinPrintStab(MachOObjectFile *MachO, SymbolListT::iterator I) {
 }
 
 static Optional<std::string> demangle(StringRef Name, bool StripUnderscore) {
-  if (StripUnderscore && Name.size() > 0 && Name[0] == '_')
+  if (StripUnderscore && !Name.empty() && Name[0] == '_')
     Name = Name.substr(1);
 
   if (!Name.startswith("_Z"))
@@ -796,8 +802,7 @@ static void sortAndPrintSymbolList(SymbolicFile &Obj, bool printName,
     bool Global = SymFlags & SymbolRef::SF_Global;
     bool Weak = SymFlags & SymbolRef::SF_Weak;
     if ((!Undefined && UndefinedOnly) || (Undefined && DefinedOnly) ||
-        (!Global && ExternalOnly) || (SizeSort && !PrintAddress) ||
-        (Weak && NoWeakSymbols))
+        (!Global && ExternalOnly) || (Weak && NoWeakSymbols))
       continue;
     if (PrintFileName)
       writeFileName(outs());
@@ -930,6 +935,7 @@ static char getSymbolNMTypeChar(ELFObjectFileBase &Obj,
     return StringSwitch<char>(*Name)
         .StartsWith(".debug", 'N')
         .StartsWith(".note", 'n')
+        .StartsWith(".comment", 'n')
         .Default('?');
   }
 
@@ -1163,7 +1169,7 @@ dumpSymbolNamesFromObject(SymbolicFile &Obj, bool printName,
   // file get the section number for that section in this object file.
   unsigned int Nsect = 0;
   MachOObjectFile *MachO = dyn_cast<MachOObjectFile>(&Obj);
-  if (SegSect.size() != 0 && MachO) {
+  if (!SegSect.empty() && MachO) {
     Nsect = getNsectForSegSect(MachO);
     // If this section is not in the object file no symbols are printed.
     if (Nsect == 0)
@@ -1184,10 +1190,8 @@ dumpSymbolNamesFromObject(SymbolicFile &Obj, bool printName,
       NMSymbol S = {};
       S.Size = 0;
       S.Address = 0;
-      if (PrintSize) {
-        if (isa<ELFObjectFileBase>(&Obj))
-          S.Size = ELFSymbolRef(Sym).getSize();
-      }
+      if (isa<ELFObjectFileBase>(&Obj))
+        S.Size = ELFSymbolRef(Sym).getSize();
       if (PrintAddress && isa<ObjectFile>(Obj)) {
         SymbolRef SymRef(Sym);
         Expected<uint64_t> AddressOrErr = SymRef.getAddress();
@@ -1800,7 +1804,7 @@ static void dumpSymbolNamesFromFile(std::string &Filename) {
   }
   if (MachOUniversalBinary *UB = dyn_cast<MachOUniversalBinary>(&Bin)) {
     // If we have a list of architecture flags specified dump only those.
-    if (!ArchAll && ArchFlags.size() != 0) {
+    if (!ArchAll && !ArchFlags.empty()) {
       // Look for a slice in the universal binary that matches each ArchFlag.
       bool ArchFound;
       for (unsigned i = 0; i < ArchFlags.size(); ++i) {
@@ -2071,7 +2075,7 @@ int main(int argc, char **argv) {
     }
   }
 
-  if (SegSect.size() != 0 && SegSect.size() != 2)
+  if (!SegSect.empty() && SegSect.size() != 2)
     error("bad number of arguments (must be two arguments)",
           "for the -s option");
 
diff --git a/tools/llvm-objcopy/Buffer.cpp b/tools/llvm-objcopy/Buffer.cpp
index 8044b023aaad0f1809981ef17dcdf40185067c84..7af9d912edaef65475a6a84a2c206a2637992345 100644
--- a/tools/llvm-objcopy/Buffer.cpp
+++ b/tools/llvm-objcopy/Buffer.cpp
@@ -1,16 +1,17 @@
 //===- Buffer.cpp ---------------------------------------------------------===//
 //
-//                      The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #include "Buffer.h"
 #include "llvm-objcopy.h"
 #include "llvm/Support/FileOutputBuffer.h"
+#include "llvm/Support/FileSystem.h"
 #include "llvm/Support/MemoryBuffer.h"
+#include "llvm/Support/Process.h"
 #include <memory>
 
 namespace llvm {
@@ -18,23 +19,51 @@ namespace objcopy {
 
 Buffer::~Buffer() {}
 
-void FileBuffer::allocate(size_t Size) {
+static Error createEmptyFile(StringRef FileName) {
+  // Create an empty tempfile and atomically swap it in place with the desired
+  // output file.
+  Expected<sys::fs::TempFile> Temp =
+      sys::fs::TempFile::create(FileName + ".temp-empty-%%%%%%%");
+  return Temp ? Temp->keep(FileName) : Temp.takeError();
+}
+
+Error FileBuffer::allocate(size_t Size) {
+  // When a 0-sized file is requested, skip allocation but defer file
+  // creation/truncation until commit() to avoid side effects if something
+  // happens between allocate() and commit().
+  if (Size == 0) {
+    EmptyFile = true;
+    return Error::success();
+  }
+
   Expected<std::unique_ptr<FileOutputBuffer>> BufferOrErr =
       FileOutputBuffer::create(getName(), Size, FileOutputBuffer::F_executable);
-  handleAllErrors(BufferOrErr.takeError(), [this](const ErrorInfoBase &E) {
-    error("failed to open " + getName() + ": " + E.message());
-  });
+  // FileOutputBuffer::create() returns an Error that is just a wrapper around
+  // std::error_code. Wrap it in FileError to include the actual filename.
+  if (!BufferOrErr)
+    return createFileError(getName(), BufferOrErr.takeError());
   Buf = std::move(*BufferOrErr);
+  return Error::success();
 }
 
-Error FileBuffer::commit() { return Buf->commit(); }
+Error FileBuffer::commit() {
+  if (EmptyFile)
+    return createEmptyFile(getName());
+
+  assert(Buf && "allocate() not called before commit()!");
+  Error Err = Buf->commit();
+  // FileOutputBuffer::commit() returns an Error that is just a wrapper around
+  // std::error_code. Wrap it in FileError to include the actual filename.
+  return Err ? createFileError(getName(), std::move(Err)) : std::move(Err);
+}
 
 uint8_t *FileBuffer::getBufferStart() {
   return reinterpret_cast<uint8_t *>(Buf->getBufferStart());
 }
 
-void MemBuffer::allocate(size_t Size) {
+Error MemBuffer::allocate(size_t Size) {
   Buf = WritableMemoryBuffer::getNewMemBuffer(Size, getName());
+  return Error::success();
 }
 
 Error MemBuffer::commit() { return Error::success(); }
diff --git a/tools/llvm-objcopy/Buffer.h b/tools/llvm-objcopy/Buffer.h
index e5b9c5b2d22b1d38379b61b9193a6aefbf05a424..487d5585c3643e226ddae93719a5990e80cf2e4f 100644
--- a/tools/llvm-objcopy/Buffer.h
+++ b/tools/llvm-objcopy/Buffer.h
@@ -1,9 +1,8 @@
 //===- Buffer.h -------------------------------------------------*- C++ -*-===//
 //
-//                      The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -28,7 +27,7 @@ class Buffer {
 
 public:
   virtual ~Buffer();
-  virtual void allocate(size_t Size) = 0;
+  virtual Error allocate(size_t Size) = 0;
   virtual uint8_t *getBufferStart() = 0;
   virtual Error commit() = 0;
 
@@ -38,9 +37,12 @@ public:
 
 class FileBuffer : public Buffer {
   std::unique_ptr<FileOutputBuffer> Buf;
+  // Indicates that allocate(0) was called, and commit() should create or
+  // truncate a file instead of using a FileOutputBuffer.
+  bool EmptyFile = false;
 
 public:
-  void allocate(size_t Size) override;
+  Error allocate(size_t Size) override;
   uint8_t *getBufferStart() override;
   Error commit() override;
 
@@ -51,7 +53,7 @@ class MemBuffer : public Buffer {
   std::unique_ptr<WritableMemoryBuffer> Buf;
 
 public:
-  void allocate(size_t Size) override;
+  Error allocate(size_t Size) override;
   uint8_t *getBufferStart() override;
   Error commit() override;
 
diff --git a/tools/llvm-objcopy/CMakeLists.txt b/tools/llvm-objcopy/CMakeLists.txt
index 3b6c345b292b6fbe0b31aaea810af3b90ede36d3..8406786e9e4eb58c81a5580421ccdc376fabe0d2 100644
--- a/tools/llvm-objcopy/CMakeLists.txt
+++ b/tools/llvm-objcopy/CMakeLists.txt
@@ -18,10 +18,14 @@ add_llvm_tool(llvm-objcopy
   CopyConfig.cpp
   llvm-objcopy.cpp
   COFF/COFFObjcopy.cpp
+  COFF/Object.cpp
   COFF/Reader.cpp
   COFF/Writer.cpp
   ELF/ELFObjcopy.cpp
   ELF/Object.cpp
+  MachO/MachOObjcopy.cpp
+  MachO/MachOReader.cpp
+  MachO/MachOWriter.cpp
   DEPENDS
   ObjcopyOptsTableGen
   StripOptsTableGen
diff --git a/tools/llvm-objcopy/COFF/COFFObjcopy.cpp b/tools/llvm-objcopy/COFF/COFFObjcopy.cpp
index 89e926af567e93181fbd99a33ac1d1ec9f4cd2c0..7923ef22fbe31644ead1ee6581574f0f4445c1bf 100644
--- a/tools/llvm-objcopy/COFF/COFFObjcopy.cpp
+++ b/tools/llvm-objcopy/COFF/COFFObjcopy.cpp
@@ -1,9 +1,8 @@
 //===- COFFObjcopy.cpp ----------------------------------------------------===//
 //
-//                      The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -17,6 +16,9 @@
 
 #include "llvm/Object/Binary.h"
 #include "llvm/Object/COFF.h"
+#include "llvm/Support/Errc.h"
+#include "llvm/Support/JamCRC.h"
+#include "llvm/Support/Path.h"
 #include <cassert>
 
 namespace llvm {
@@ -26,13 +28,185 @@ namespace coff {
 using namespace object;
 using namespace COFF;
 
-void executeObjcopyOnBinary(const CopyConfig &Config,
-                            object::COFFObjectFile &In, Buffer &Out) {
+static bool isDebugSection(const Section &Sec) {
+  return Sec.Name.startswith(".debug");
+}
+
+static uint64_t getNextRVA(const Object &Obj) {
+  if (Obj.getSections().empty())
+    return 0;
+  const Section &Last = Obj.getSections().back();
+  return alignTo(Last.Header.VirtualAddress + Last.Header.VirtualSize,
+                 Obj.IsPE ? Obj.PeHeader.SectionAlignment : 1);
+}
+
+static uint32_t getCRC32(StringRef Data) {
+  JamCRC CRC;
+  CRC.update(ArrayRef<char>(Data.data(), Data.size()));
+  // The CRC32 value needs to be complemented because the JamCRC dosn't
+  // finalize the CRC32 value. It also dosn't negate the initial CRC32 value
+  // but it starts by default at 0xFFFFFFFF which is the complement of zero.
+  return ~CRC.getCRC();
+}
+
+static std::vector<uint8_t> createGnuDebugLinkSectionContents(StringRef File) {
+  ErrorOr<std::unique_ptr<MemoryBuffer>> LinkTargetOrErr =
+      MemoryBuffer::getFile(File);
+  if (!LinkTargetOrErr)
+    error("'" + File + "': " + LinkTargetOrErr.getError().message());
+  auto LinkTarget = std::move(*LinkTargetOrErr);
+  uint32_t CRC32 = getCRC32(LinkTarget->getBuffer());
+
+  StringRef FileName = sys::path::filename(File);
+  size_t CRCPos = alignTo(FileName.size() + 1, 4);
+  std::vector<uint8_t> Data(CRCPos + 4);
+  memcpy(Data.data(), FileName.data(), FileName.size());
+  support::endian::write32le(Data.data() + CRCPos, CRC32);
+  return Data;
+}
+
+static void addGnuDebugLink(Object &Obj, StringRef DebugLinkFile) {
+  uint32_t StartRVA = getNextRVA(Obj);
+
+  std::vector<Section> Sections;
+  Section Sec;
+  Sec.setOwnedContents(createGnuDebugLinkSectionContents(DebugLinkFile));
+  Sec.Name = ".gnu_debuglink";
+  Sec.Header.VirtualSize = Sec.getContents().size();
+  Sec.Header.VirtualAddress = StartRVA;
+  Sec.Header.SizeOfRawData = alignTo(Sec.Header.VirtualSize,
+                                     Obj.IsPE ? Obj.PeHeader.FileAlignment : 1);
+  // Sec.Header.PointerToRawData is filled in by the writer.
+  Sec.Header.PointerToRelocations = 0;
+  Sec.Header.PointerToLinenumbers = 0;
+  // Sec.Header.NumberOfRelocations is filled in by the writer.
+  Sec.Header.NumberOfLinenumbers = 0;
+  Sec.Header.Characteristics = IMAGE_SCN_CNT_INITIALIZED_DATA |
+                               IMAGE_SCN_MEM_READ | IMAGE_SCN_MEM_DISCARDABLE;
+  Sections.push_back(Sec);
+  Obj.addSections(Sections);
+}
+
+static Error handleArgs(const CopyConfig &Config, Object &Obj) {
+  // Perform the actual section removals.
+  Obj.removeSections([&Config](const Section &Sec) {
+    // Contrary to --only-keep-debug, --only-section fully removes sections that
+    // aren't mentioned.
+    if (!Config.OnlySection.empty() &&
+        !is_contained(Config.OnlySection, Sec.Name))
+      return true;
+
+    if (Config.StripDebug || Config.StripAll || Config.StripAllGNU ||
+        Config.DiscardMode == DiscardType::All || Config.StripUnneeded) {
+      if (isDebugSection(Sec) &&
+          (Sec.Header.Characteristics & IMAGE_SCN_MEM_DISCARDABLE) != 0)
+        return true;
+    }
+
+    if (is_contained(Config.ToRemove, Sec.Name))
+      return true;
+
+    return false;
+  });
+
+  if (Config.OnlyKeepDebug) {
+    // For --only-keep-debug, we keep all other sections, but remove their
+    // content. The VirtualSize field in the section header is kept intact.
+    Obj.truncateSections([](const Section &Sec) {
+      return !isDebugSection(Sec) && Sec.Name != ".buildid" &&
+             ((Sec.Header.Characteristics &
+               (IMAGE_SCN_CNT_CODE | IMAGE_SCN_CNT_INITIALIZED_DATA)) != 0);
+    });
+  }
+
+  // StripAll removes all symbols and thus also removes all relocations.
+  if (Config.StripAll || Config.StripAllGNU)
+    for (Section &Sec : Obj.getMutableSections())
+      Sec.Relocs.clear();
+
+  // If we need to do per-symbol removals, initialize the Referenced field.
+  if (Config.StripUnneeded || Config.DiscardMode == DiscardType::All ||
+      !Config.SymbolsToRemove.empty())
+    if (Error E = Obj.markSymbols())
+      return E;
+
+  // Actually do removals of symbols.
+  Obj.removeSymbols([&](const Symbol &Sym) {
+    // For StripAll, all relocations have been stripped and we remove all
+    // symbols.
+    if (Config.StripAll || Config.StripAllGNU)
+      return true;
+
+    if (is_contained(Config.SymbolsToRemove, Sym.Name)) {
+      // Explicitly removing a referenced symbol is an error.
+      if (Sym.Referenced)
+        reportError(Config.OutputFilename,
+                    createStringError(llvm::errc::invalid_argument,
+                                      "not stripping symbol '%s' because it is "
+                                      "named in a relocation.",
+                                      Sym.Name.str().c_str()));
+      return true;
+    }
+
+    if (!Sym.Referenced) {
+      // With --strip-unneeded, GNU objcopy removes all unreferenced local
+      // symbols, and any unreferenced undefined external.
+      // With --strip-unneeded-symbol we strip only specific unreferenced
+      // local symbol instead of removing all of such.
+      if (Sym.Sym.StorageClass == IMAGE_SYM_CLASS_STATIC ||
+          Sym.Sym.SectionNumber == 0)
+        if (Config.StripUnneeded ||
+            is_contained(Config.UnneededSymbolsToRemove, Sym.Name))
+          return true;
+
+      // GNU objcopy keeps referenced local symbols and external symbols
+      // if --discard-all is set, similar to what --strip-unneeded does,
+      // but undefined local symbols are kept when --discard-all is set.
+      if (Config.DiscardMode == DiscardType::All &&
+          Sym.Sym.StorageClass == IMAGE_SYM_CLASS_STATIC &&
+          Sym.Sym.SectionNumber != 0)
+        return true;
+    }
+
+    return false;
+  });
+
+  if (!Config.AddGnuDebugLink.empty())
+    addGnuDebugLink(Obj, Config.AddGnuDebugLink);
+
+  if (!Config.BuildIdLinkDir.empty() || Config.BuildIdLinkInput ||
+      Config.BuildIdLinkOutput || !Config.SplitDWO.empty() ||
+      !Config.SymbolsPrefix.empty() || !Config.AddSection.empty() ||
+      !Config.DumpSection.empty() || !Config.KeepSection.empty() ||
+      !Config.SymbolsToGlobalize.empty() || !Config.SymbolsToKeep.empty() ||
+      !Config.SymbolsToLocalize.empty() || !Config.SymbolsToWeaken.empty() ||
+      !Config.SymbolsToKeepGlobal.empty() || !Config.SectionsToRename.empty() ||
+      !Config.SetSectionFlags.empty() || !Config.SymbolsToRename.empty() ||
+      Config.ExtractDWO || Config.KeepFileSymbols || Config.LocalizeHidden ||
+      Config.PreserveDates || Config.StripDWO || Config.StripNonAlloc ||
+      Config.StripSections || Config.Weaken || Config.DecompressDebugSections ||
+      Config.DiscardMode == DiscardType::Locals) {
+    return createStringError(llvm::errc::invalid_argument,
+                             "Option not supported by llvm-objcopy for COFF");
+  }
+
+  return Error::success();
+}
+
+Error executeObjcopyOnBinary(const CopyConfig &Config, COFFObjectFile &In,
+                             Buffer &Out) {
   COFFReader Reader(In);
-  std::unique_ptr<Object> Obj = Reader.create();
+  Expected<std::unique_ptr<Object>> ObjOrErr = Reader.create();
+  if (!ObjOrErr)
+    return createFileError(Config.InputFilename, ObjOrErr.takeError());
+  Object *Obj = ObjOrErr->get();
   assert(Obj && "Unable to deserialize COFF object");
+  if (Error E = handleArgs(Config, *Obj))
+    return createFileError(Config.InputFilename, std::move(E));
   COFFWriter Writer(*Obj, Out);
-  Writer.write();
+  if (Error E = Writer.write())
+    return createFileError(Config.OutputFilename, std::move(E));
+  return Error::success();
 }
 
 } // end namespace coff
diff --git a/tools/llvm-objcopy/COFF/COFFObjcopy.h b/tools/llvm-objcopy/COFF/COFFObjcopy.h
index bf70bd9b4d844a1b751f391c82e69a82326e7cab..858759e52c4afabf33130fabbfc13cf140ac1df9 100644
--- a/tools/llvm-objcopy/COFF/COFFObjcopy.h
+++ b/tools/llvm-objcopy/COFF/COFFObjcopy.h
@@ -1,9 +1,8 @@
 //===- COFFObjcopy.h --------------------------------------------*- C++ -*-===//
 //
-//                      The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -11,6 +10,7 @@
 #define LLVM_TOOLS_OBJCOPY_COFFOBJCOPY_H
 
 namespace llvm {
+class Error;
 
 namespace object {
 class COFFObjectFile;
@@ -21,8 +21,8 @@ struct CopyConfig;
 class Buffer;
 
 namespace coff {
-void executeObjcopyOnBinary(const CopyConfig &Config,
-                            object::COFFObjectFile &In, Buffer &Out);
+Error executeObjcopyOnBinary(const CopyConfig &Config,
+                             object::COFFObjectFile &In, Buffer &Out);
 
 } // end namespace coff
 } // end namespace objcopy
diff --git a/tools/llvm-objcopy/COFF/Object.cpp b/tools/llvm-objcopy/COFF/Object.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..0ad5a05a14441f3aea0beb089648c25fca1830e1
--- /dev/null
+++ b/tools/llvm-objcopy/COFF/Object.cpp
@@ -0,0 +1,137 @@
+//===- Object.cpp ---------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "Object.h"
+#include "llvm/ADT/DenseSet.h"
+#include <algorithm>
+
+namespace llvm {
+namespace objcopy {
+namespace coff {
+
+using namespace object;
+
+void Object::addSymbols(ArrayRef<Symbol> NewSymbols) {
+  for (Symbol S : NewSymbols) {
+    S.UniqueId = NextSymbolUniqueId++;
+    Symbols.emplace_back(S);
+  }
+  updateSymbols();
+}
+
+void Object::updateSymbols() {
+  SymbolMap = DenseMap<size_t, Symbol *>(Symbols.size());
+  for (Symbol &Sym : Symbols)
+    SymbolMap[Sym.UniqueId] = &Sym;
+}
+
+const Symbol *Object::findSymbol(size_t UniqueId) const {
+  auto It = SymbolMap.find(UniqueId);
+  if (It == SymbolMap.end())
+    return nullptr;
+  return It->second;
+}
+
+void Object::removeSymbols(function_ref<bool(const Symbol &)> ToRemove) {
+  Symbols.erase(
+      std::remove_if(std::begin(Symbols), std::end(Symbols),
+                     [ToRemove](const Symbol &Sym) { return ToRemove(Sym); }),
+      std::end(Symbols));
+  updateSymbols();
+}
+
+Error Object::markSymbols() {
+  for (Symbol &Sym : Symbols)
+    Sym.Referenced = false;
+  for (const Section &Sec : Sections) {
+    for (const Relocation &R : Sec.Relocs) {
+      auto It = SymbolMap.find(R.Target);
+      if (It == SymbolMap.end())
+        return createStringError(object_error::invalid_symbol_index,
+                                 "Relocation target %zu not found", R.Target);
+      It->second->Referenced = true;
+    }
+  }
+  return Error::success();
+}
+
+void Object::addSections(ArrayRef<Section> NewSections) {
+  for (Section S : NewSections) {
+    S.UniqueId = NextSectionUniqueId++;
+    Sections.emplace_back(S);
+  }
+  updateSections();
+}
+
+void Object::updateSections() {
+  SectionMap = DenseMap<ssize_t, Section *>(Sections.size());
+  size_t Index = 1;
+  for (Section &S : Sections) {
+    SectionMap[S.UniqueId] = &S;
+    S.Index = Index++;
+  }
+}
+
+const Section *Object::findSection(ssize_t UniqueId) const {
+  auto It = SectionMap.find(UniqueId);
+  if (It == SectionMap.end())
+    return nullptr;
+  return It->second;
+}
+
+void Object::removeSections(function_ref<bool(const Section &)> ToRemove) {
+  DenseSet<ssize_t> AssociatedSections;
+  auto RemoveAssociated = [&AssociatedSections](const Section &Sec) {
+    return AssociatedSections.count(Sec.UniqueId) == 1;
+  };
+  do {
+    DenseSet<ssize_t> RemovedSections;
+    Sections.erase(
+        std::remove_if(std::begin(Sections), std::end(Sections),
+                       [ToRemove, &RemovedSections](const Section &Sec) {
+                         bool Remove = ToRemove(Sec);
+                         if (Remove)
+                           RemovedSections.insert(Sec.UniqueId);
+                         return Remove;
+                       }),
+        std::end(Sections));
+    // Remove all symbols referring to the removed sections.
+    AssociatedSections.clear();
+    Symbols.erase(
+        std::remove_if(
+            std::begin(Symbols), std::end(Symbols),
+            [&RemovedSections, &AssociatedSections](const Symbol &Sym) {
+              // If there are sections that are associative to a removed
+              // section,
+              // remove those as well as nothing will include them (and we can't
+              // leave them dangling).
+              if (RemovedSections.count(Sym.AssociativeComdatTargetSectionId) ==
+                  1)
+                AssociatedSections.insert(Sym.TargetSectionId);
+              return RemovedSections.count(Sym.TargetSectionId) == 1;
+            }),
+        std::end(Symbols));
+    ToRemove = RemoveAssociated;
+  } while (!AssociatedSections.empty());
+  updateSections();
+  updateSymbols();
+}
+
+void Object::truncateSections(function_ref<bool(const Section &)> ToTruncate) {
+  for (Section &Sec : Sections) {
+    if (ToTruncate(Sec)) {
+      Sec.clearContents();
+      Sec.Relocs.clear();
+      Sec.Header.SizeOfRawData = 0;
+    }
+  }
+}
+
+} // end namespace coff
+} // end namespace objcopy
+} // end namespace llvm
diff --git a/tools/llvm-objcopy/COFF/Object.h b/tools/llvm-objcopy/COFF/Object.h
index 89f9903c0fd2e6698b07d0a8d0c89316897d3fa6..21475b068629c24705d6a3150b13e1b41bec57f4 100644
--- a/tools/llvm-objcopy/COFF/Object.h
+++ b/tools/llvm-objcopy/COFF/Object.h
@@ -1,9 +1,8 @@
 //===- Object.h -------------------------------------------------*- C++ -*-===//
 //
-//                      The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -11,7 +10,10 @@
 #define LLVM_TOOLS_OBJCOPY_COFF_OBJECT_H
 
 #include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/Optional.h"
 #include "llvm/ADT/StringRef.h"
+#include "llvm/ADT/iterator_range.h"
 #include "llvm/BinaryFormat/COFF.h"
 #include "llvm/Object/COFF.h"
 #include <cstddef>
@@ -22,17 +24,72 @@ namespace llvm {
 namespace objcopy {
 namespace coff {
 
+struct Relocation {
+  Relocation() {}
+  Relocation(const object::coff_relocation& R) : Reloc(R) {}
+
+  object::coff_relocation Reloc;
+  size_t Target;
+  StringRef TargetName; // Used for diagnostics only
+};
+
 struct Section {
   object::coff_section Header;
-  ArrayRef<uint8_t> Contents;
-  std::vector<object::coff_relocation> Relocs;
+  std::vector<Relocation> Relocs;
   StringRef Name;
+  ssize_t UniqueId;
+  size_t Index;
+
+  ArrayRef<uint8_t> getContents() const {
+    if (!OwnedContents.empty())
+      return OwnedContents;
+    return ContentsRef;
+  }
+
+  void setContentsRef(ArrayRef<uint8_t> Data) {
+    OwnedContents.clear();
+    ContentsRef = Data;
+  }
+
+  void setOwnedContents(std::vector<uint8_t> &&Data) {
+    ContentsRef = ArrayRef<uint8_t>();
+    OwnedContents = std::move(Data);
+  }
+
+  void clearContents() {
+    ContentsRef = ArrayRef<uint8_t>();
+    OwnedContents.clear();
+  }
+
+private:
+  ArrayRef<uint8_t> ContentsRef;
+  std::vector<uint8_t> OwnedContents;
+};
+
+struct AuxSymbol {
+  AuxSymbol(ArrayRef<uint8_t> In) {
+    assert(In.size() == sizeof(Opaque));
+    std::copy(In.begin(), In.end(), Opaque);
+  }
+
+  ArrayRef<uint8_t> getRef() const {
+    return ArrayRef<uint8_t>(Opaque, sizeof(Opaque));
+  }
+
+  uint8_t Opaque[sizeof(object::coff_symbol16)];
 };
 
 struct Symbol {
   object::coff_symbol32 Sym;
   StringRef Name;
-  ArrayRef<uint8_t> AuxData;
+  std::vector<AuxSymbol> AuxData;
+  StringRef AuxFile;
+  ssize_t TargetSectionId;
+  ssize_t AssociativeComdatTargetSectionId = 0;
+  Optional<size_t> WeakTargetSymbolId;
+  size_t UniqueId;
+  size_t RawIndex;
+  bool Referenced;
 };
 
 struct Object {
@@ -48,8 +105,52 @@ struct Object {
   uint32_t BaseOfData = 0; // pe32plus_header lacks this field.
 
   std::vector<object::data_directory> DataDirectories;
-  std::vector<Section> Sections;
+
+  ArrayRef<Symbol> getSymbols() const { return Symbols; }
+  // This allows mutating individual Symbols, but not mutating the list
+  // of symbols itself.
+  iterator_range<std::vector<Symbol>::iterator> getMutableSymbols() {
+    return make_range(Symbols.begin(), Symbols.end());
+  }
+
+  const Symbol *findSymbol(size_t UniqueId) const;
+
+  void addSymbols(ArrayRef<Symbol> NewSymbols);
+  void removeSymbols(function_ref<bool(const Symbol &)> ToRemove);
+
+  // Set the Referenced field on all Symbols, based on relocations in
+  // all sections.
+  Error markSymbols();
+
+  ArrayRef<Section> getSections() const { return Sections; }
+  // This allows mutating individual Sections, but not mutating the list
+  // of symbols itself.
+  iterator_range<std::vector<Section>::iterator> getMutableSections() {
+    return make_range(Sections.begin(), Sections.end());
+  }
+
+  const Section *findSection(ssize_t UniqueId) const;
+
+  void addSections(ArrayRef<Section> NewSections);
+  void removeSections(function_ref<bool(const Section &)> ToRemove);
+  void truncateSections(function_ref<bool(const Section &)> ToTruncate);
+
+private:
   std::vector<Symbol> Symbols;
+  DenseMap<size_t, Symbol *> SymbolMap;
+
+  size_t NextSymbolUniqueId = 0;
+
+  std::vector<Section> Sections;
+  DenseMap<ssize_t, Section *> SectionMap;
+
+  ssize_t NextSectionUniqueId = 1; // Allow a UniqueId 0 to mean undefined.
+
+  // Update SymbolMap.
+  void updateSymbols();
+
+  // Update SectionMap and Index in each Section.
+  void updateSections();
 };
 
 // Copy between coff_symbol16 and coff_symbol32.
diff --git a/tools/llvm-objcopy/COFF/Reader.cpp b/tools/llvm-objcopy/COFF/Reader.cpp
index a8a3576084cd1fa74ef82a5e3bf02d6593d78775..7270bbf94de6f9d58707a13fae222d6859b82382 100644
--- a/tools/llvm-objcopy/COFF/Reader.cpp
+++ b/tools/llvm-objcopy/COFF/Reader.cpp
@@ -1,9 +1,8 @@
 //===- Reader.cpp ---------------------------------------------------------===//
 //
-//                      The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -12,6 +11,7 @@
 #include "llvm-objcopy.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/StringRef.h"
+#include "llvm/BinaryFormat/COFF.h"
 #include "llvm/Object/COFF.h"
 #include "llvm/Support/ErrorHandling.h"
 #include <cstddef>
@@ -22,14 +22,13 @@ namespace objcopy {
 namespace coff {
 
 using namespace object;
+using namespace COFF;
 
-Reader::~Reader() {}
-
-void COFFReader::readExecutableHeaders(Object &Obj) const {
+Error COFFReader::readExecutableHeaders(Object &Obj) const {
   const dos_header *DH = COFFObj.getDOSHeader();
   Obj.Is64 = COFFObj.is64();
   if (!DH)
-    return;
+    return Error::success();
 
   Obj.IsPE = true;
   Obj.DosHeader = *DH;
@@ -40,12 +39,12 @@ void COFFReader::readExecutableHeaders(Object &Obj) const {
   if (COFFObj.is64()) {
     const pe32plus_header *PE32Plus = nullptr;
     if (auto EC = COFFObj.getPE32PlusHeader(PE32Plus))
-      reportError(COFFObj.getFileName(), std::move(EC));
+      return errorCodeToError(EC);
     Obj.PeHeader = *PE32Plus;
   } else {
     const pe32_header *PE32 = nullptr;
     if (auto EC = COFFObj.getPE32Header(PE32))
-      reportError(COFFObj.getFileName(), std::move(EC));
+      return errorCodeToError(EC);
     copyPeHeader(Obj.PeHeader, *PE32);
     // The pe32plus_header (stored in Object) lacks the BaseOfData field.
     Obj.BaseOfData = PE32->BaseOfData;
@@ -54,43 +53,51 @@ void COFFReader::readExecutableHeaders(Object &Obj) const {
   for (size_t I = 0; I < Obj.PeHeader.NumberOfRvaAndSize; I++) {
     const data_directory *Dir;
     if (auto EC = COFFObj.getDataDirectory(I, Dir))
-      reportError(COFFObj.getFileName(), std::move(EC));
+      return errorCodeToError(EC);
     Obj.DataDirectories.emplace_back(*Dir);
   }
+  return Error::success();
 }
 
-void COFFReader::readSections(Object &Obj) const {
+Error COFFReader::readSections(Object &Obj) const {
+  std::vector<Section> Sections;
   // Section indexing starts from 1.
   for (size_t I = 1, E = COFFObj.getNumberOfSections(); I <= E; I++) {
     const coff_section *Sec;
     if (auto EC = COFFObj.getSection(I, Sec))
-      reportError(COFFObj.getFileName(), std::move(EC));
-    Obj.Sections.push_back(Section());
-    Section &S = Obj.Sections.back();
+      return errorCodeToError(EC);
+    Sections.push_back(Section());
+    Section &S = Sections.back();
     S.Header = *Sec;
-    if (auto EC = COFFObj.getSectionContents(Sec, S.Contents))
-      reportError(COFFObj.getFileName(), std::move(EC));
+    ArrayRef<uint8_t> Contents;
+    if (auto EC = COFFObj.getSectionContents(Sec, Contents))
+      return errorCodeToError(EC);
+    S.setContentsRef(Contents);
     ArrayRef<coff_relocation> Relocs = COFFObj.getRelocations(Sec);
-    S.Relocs.insert(S.Relocs.end(), Relocs.begin(), Relocs.end());
+    for (const coff_relocation &R : Relocs)
+      S.Relocs.push_back(R);
     if (auto EC = COFFObj.getSectionName(Sec, S.Name))
-      reportError(COFFObj.getFileName(), std::move(EC));
+      return errorCodeToError(EC);
     if (Sec->hasExtendedRelocations())
-      reportError(
-          COFFObj.getFileName(),
-          make_error<StringError>("Extended relocations not supported yet",
-                                  object_error::parse_failed));
+      return createStringError(object_error::parse_failed,
+                               "Extended relocations not supported yet");
   }
+  Obj.addSections(Sections);
+  return Error::success();
 }
 
-void COFFReader::readSymbols(Object &Obj, bool IsBigObj) const {
+Error COFFReader::readSymbols(Object &Obj, bool IsBigObj) const {
+  std::vector<Symbol> Symbols;
+  Symbols.reserve(COFFObj.getRawNumberOfSymbols());
+  ArrayRef<Section> Sections = Obj.getSections();
   for (uint32_t I = 0, E = COFFObj.getRawNumberOfSymbols(); I < E;) {
     Expected<COFFSymbolRef> SymOrErr = COFFObj.getSymbol(I);
     if (!SymOrErr)
-      reportError(COFFObj.getFileName(), SymOrErr.takeError());
+      return SymOrErr.takeError();
     COFFSymbolRef SymRef = *SymOrErr;
 
-    Obj.Symbols.push_back(Symbol());
-    Symbol &Sym = Obj.Symbols.back();
+    Symbols.push_back(Symbol());
+    Symbol &Sym = Symbols.back();
     // Copy symbols from the original form into an intermediate coff_symbol32.
     if (IsBigObj)
       copySymbol(Sym.Sym,
@@ -99,15 +106,95 @@ void COFFReader::readSymbols(Object &Obj, bool IsBigObj) const {
       copySymbol(Sym.Sym,
                  *reinterpret_cast<const coff_symbol16 *>(SymRef.getRawPtr()));
     if (auto EC = COFFObj.getSymbolName(SymRef, Sym.Name))
-      reportError(COFFObj.getFileName(), std::move(EC));
-    Sym.AuxData = COFFObj.getSymbolAuxData(SymRef);
-    assert((Sym.AuxData.size() %
-            (IsBigObj ? sizeof(coff_symbol32) : sizeof(coff_symbol16))) == 0);
+      return errorCodeToError(EC);
+
+    ArrayRef<uint8_t> AuxData = COFFObj.getSymbolAuxData(SymRef);
+    size_t SymSize = IsBigObj ? sizeof(coff_symbol32) : sizeof(coff_symbol16);
+    assert(AuxData.size() == SymSize * SymRef.getNumberOfAuxSymbols());
+    // The auxillary symbols are structs of sizeof(coff_symbol16) each.
+    // In the big object format (where symbols are coff_symbol32), each
+    // auxillary symbol is padded with 2 bytes at the end. Copy each
+    // auxillary symbol to the Sym.AuxData vector. For file symbols,
+    // the whole range of aux symbols are interpreted as one null padded
+    // string instead.
+    if (SymRef.isFileRecord())
+      Sym.AuxFile = StringRef(reinterpret_cast<const char *>(AuxData.data()),
+                              AuxData.size())
+                        .rtrim('\0');
+    else
+      for (size_t I = 0; I < SymRef.getNumberOfAuxSymbols(); I++)
+        Sym.AuxData.push_back(AuxData.slice(I * SymSize, sizeof(AuxSymbol)));
+
+    // Find the unique id of the section
+    if (SymRef.getSectionNumber() <=
+        0) // Special symbol (undefined/absolute/debug)
+      Sym.TargetSectionId = SymRef.getSectionNumber();
+    else if (static_cast<uint32_t>(SymRef.getSectionNumber() - 1) <
+             Sections.size())
+      Sym.TargetSectionId = Sections[SymRef.getSectionNumber() - 1].UniqueId;
+    else
+      return createStringError(object_error::parse_failed,
+                               "Section number out of range");
+    // For section definitions, check if it is comdat associative, and if
+    // it is, find the target section unique id.
+    const coff_aux_section_definition *SD = SymRef.getSectionDefinition();
+    const coff_aux_weak_external *WE = SymRef.getWeakExternal();
+    if (SD && SD->Selection == IMAGE_COMDAT_SELECT_ASSOCIATIVE) {
+      int32_t Index = SD->getNumber(IsBigObj);
+      if (Index <= 0 || static_cast<uint32_t>(Index - 1) >= Sections.size())
+        return createStringError(object_error::parse_failed,
+                                 "Unexpected associative section index");
+      Sym.AssociativeComdatTargetSectionId = Sections[Index - 1].UniqueId;
+    } else if (WE) {
+      // This is a raw symbol index for now, but store it in the Symbol
+      // until we've added them to the Object, which assigns the final
+      // unique ids.
+      Sym.WeakTargetSymbolId = WE->TagIndex;
+    }
     I += 1 + SymRef.getNumberOfAuxSymbols();
   }
+  Obj.addSymbols(Symbols);
+  return Error::success();
+}
+
+Error COFFReader::setSymbolTargets(Object &Obj) const {
+  std::vector<const Symbol *> RawSymbolTable;
+  for (const Symbol &Sym : Obj.getSymbols()) {
+    RawSymbolTable.push_back(&Sym);
+    for (size_t I = 0; I < Sym.Sym.NumberOfAuxSymbols; I++)
+      RawSymbolTable.push_back(nullptr);
+  }
+  for (Symbol &Sym : Obj.getMutableSymbols()) {
+    // Convert WeakTargetSymbolId from the original raw symbol index to
+    // a proper unique id.
+    if (Sym.WeakTargetSymbolId) {
+      if (*Sym.WeakTargetSymbolId >= RawSymbolTable.size())
+        return createStringError(object_error::parse_failed,
+                                 "Weak external reference out of range");
+      const Symbol *Target = RawSymbolTable[*Sym.WeakTargetSymbolId];
+      if (Target == nullptr)
+        return createStringError(object_error::parse_failed,
+                                 "Invalid SymbolTableIndex");
+      Sym.WeakTargetSymbolId = Target->UniqueId;
+    }
+  }
+  for (Section &Sec : Obj.getMutableSections()) {
+    for (Relocation &R : Sec.Relocs) {
+      if (R.Reloc.SymbolTableIndex >= RawSymbolTable.size())
+        return createStringError(object_error::parse_failed,
+                                 "SymbolTableIndex out of range");
+      const Symbol *Sym = RawSymbolTable[R.Reloc.SymbolTableIndex];
+      if (Sym == nullptr)
+        return createStringError(object_error::parse_failed,
+                                 "Invalid SymbolTableIndex");
+      R.Target = Sym->UniqueId;
+      R.TargetName = Sym->Name;
+    }
+  }
+  return Error::success();
 }
 
-std::unique_ptr<Object> COFFReader::create() const {
+Expected<std::unique_ptr<Object>> COFFReader::create() const {
   auto Obj = llvm::make_unique<Object>();
 
   const coff_file_header *CFH = nullptr;
@@ -119,9 +206,8 @@ std::unique_ptr<Object> COFFReader::create() const {
     Obj->CoffFileHeader = *CFH;
   } else {
     if (!CBFH)
-      reportError(COFFObj.getFileName(),
-                  make_error<StringError>("No COFF file header returned",
-                                          object_error::parse_failed));
+      return createStringError(object_error::parse_failed,
+                               "No COFF file header returned");
     // Only copying the few fields from the bigobj header that we need
     // and won't recreate in the end.
     Obj->CoffFileHeader.Machine = CBFH->Machine;
@@ -129,11 +215,16 @@ std::unique_ptr<Object> COFFReader::create() const {
     IsBigObj = true;
   }
 
-  readExecutableHeaders(*Obj);
-  readSections(*Obj);
-  readSymbols(*Obj, IsBigObj);
+  if (Error E = readExecutableHeaders(*Obj))
+    return std::move(E);
+  if (Error E = readSections(*Obj))
+    return std::move(E);
+  if (Error E = readSymbols(*Obj, IsBigObj))
+    return std::move(E);
+  if (Error E = setSymbolTargets(*Obj))
+    return std::move(E);
 
-  return Obj;
+  return std::move(Obj);
 }
 
 } // end namespace coff
diff --git a/tools/llvm-objcopy/COFF/Reader.h b/tools/llvm-objcopy/COFF/Reader.h
index c81a9751f1804cd83f60f2ab93bdcca46a191b8a..ec15369db0b80d867fc4b252c9bb796d404ab3bc 100644
--- a/tools/llvm-objcopy/COFF/Reader.h
+++ b/tools/llvm-objcopy/COFF/Reader.h
@@ -1,9 +1,8 @@
 //===- Reader.h -------------------------------------------------*- C++ -*-===//
 //
-//                      The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -13,6 +12,7 @@
 #include "Buffer.h"
 #include "llvm/BinaryFormat/COFF.h"
 #include "llvm/Object/COFF.h"
+#include "llvm/Support/Error.h"
 
 namespace llvm {
 namespace objcopy {
@@ -22,22 +22,17 @@ struct Object;
 
 using object::COFFObjectFile;
 
-class Reader {
-public:
-  virtual ~Reader();
-  virtual std::unique_ptr<Object> create() const = 0;
-};
-
-class COFFReader : public Reader {
+class COFFReader {
   const COFFObjectFile &COFFObj;
 
-  void readExecutableHeaders(Object &Obj) const;
-  void readSections(Object &Obj) const;
-  void readSymbols(Object &Obj, bool IsBigObj) const;
+  Error readExecutableHeaders(Object &Obj) const;
+  Error readSections(Object &Obj) const;
+  Error readSymbols(Object &Obj, bool IsBigObj) const;
+  Error setSymbolTargets(Object &Obj) const;
 
 public:
   explicit COFFReader(const COFFObjectFile &O) : COFFObj(O) {}
-  std::unique_ptr<Object> create() const override;
+  Expected<std::unique_ptr<Object>> create() const;
 };
 
 } // end namespace coff
diff --git a/tools/llvm-objcopy/COFF/Writer.cpp b/tools/llvm-objcopy/COFF/Writer.cpp
index 48afe7889e872148c766c00cc7d23817696a3cf7..6e69c597217970215fe40b046fed3bbc18b93bdd 100644
--- a/tools/llvm-objcopy/COFF/Writer.cpp
+++ b/tools/llvm-objcopy/COFF/Writer.cpp
@@ -1,9 +1,8 @@
 //===- Writer.cpp ---------------------------------------------------------===//
 //
-//                      The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -25,16 +24,83 @@ namespace coff {
 using namespace object;
 using namespace COFF;
 
-Writer::~Writer() {}
+Error COFFWriter::finalizeRelocTargets() {
+  for (Section &Sec : Obj.getMutableSections()) {
+    for (Relocation &R : Sec.Relocs) {
+      const Symbol *Sym = Obj.findSymbol(R.Target);
+      if (Sym == nullptr)
+        return createStringError(object_error::invalid_symbol_index,
+                                 "Relocation target '%s' (%zu) not found",
+                                 R.TargetName.str().c_str(), R.Target);
+      R.Reloc.SymbolTableIndex = Sym->RawIndex;
+    }
+  }
+  return Error::success();
+}
+
+Error COFFWriter::finalizeSymbolContents() {
+  for (Symbol &Sym : Obj.getMutableSymbols()) {
+    if (Sym.TargetSectionId <= 0) {
+      // Undefined, or a special kind of symbol. These negative values
+      // are stored in the SectionNumber field which is unsigned.
+      Sym.Sym.SectionNumber = static_cast<uint32_t>(Sym.TargetSectionId);
+    } else {
+      const Section *Sec = Obj.findSection(Sym.TargetSectionId);
+      if (Sec == nullptr)
+        return createStringError(object_error::invalid_symbol_index,
+                                 "Symbol '%s' points to a removed section",
+                                 Sym.Name.str().c_str());
+      Sym.Sym.SectionNumber = Sec->Index;
+
+      if (Sym.Sym.NumberOfAuxSymbols == 1 &&
+          Sym.Sym.StorageClass == IMAGE_SYM_CLASS_STATIC) {
+        coff_aux_section_definition *SD =
+            reinterpret_cast<coff_aux_section_definition *>(
+                Sym.AuxData[0].Opaque);
+        uint32_t SDSectionNumber;
+        if (Sym.AssociativeComdatTargetSectionId == 0) {
+          // Not a comdat associative section; just set the Number field to
+          // the number of the section itself.
+          SDSectionNumber = Sec->Index;
+        } else {
+          Sec = Obj.findSection(Sym.AssociativeComdatTargetSectionId);
+          if (Sec == nullptr)
+            return createStringError(
+                object_error::invalid_symbol_index,
+                "Symbol '%s' is associative to a removed section",
+                Sym.Name.str().c_str());
+          SDSectionNumber = Sec->Index;
+        }
+        // Update the section definition with the new section number.
+        SD->NumberLowPart = static_cast<uint16_t>(SDSectionNumber);
+        SD->NumberHighPart = static_cast<uint16_t>(SDSectionNumber >> 16);
+      }
+    }
+    // Check that we actually have got AuxData to match the weak symbol target
+    // we want to set. Only >= 1 would be required, but only == 1 makes sense.
+    if (Sym.WeakTargetSymbolId && Sym.Sym.NumberOfAuxSymbols == 1) {
+      coff_aux_weak_external *WE =
+          reinterpret_cast<coff_aux_weak_external *>(Sym.AuxData[0].Opaque);
+      const Symbol *Target = Obj.findSymbol(*Sym.WeakTargetSymbolId);
+      if (Target == nullptr)
+        return createStringError(object_error::invalid_symbol_index,
+                                 "Symbol '%s' is missing its weak target",
+                                 Sym.Name.str().c_str());
+      WE->TagIndex = Target->RawIndex;
+    }
+  }
+  return Error::success();
+}
 
 void COFFWriter::layoutSections() {
-  for (auto &S : Obj.Sections) {
+  for (auto &S : Obj.getMutableSections()) {
     if (S.Header.SizeOfRawData > 0)
       S.Header.PointerToRawData = FileSize;
     FileSize += S.Header.SizeOfRawData; // For executables, this is already
                                         // aligned to FileAlignment.
-    if (S.Header.NumberOfRelocations > 0)
-      S.Header.PointerToRelocations = FileSize;
+    S.Header.NumberOfRelocations = S.Relocs.size();
+    S.Header.PointerToRelocations =
+        S.Header.NumberOfRelocations > 0 ? FileSize : 0;
     FileSize += S.Relocs.size() * sizeof(coff_relocation);
     FileSize = alignTo(FileSize, FileAlignment);
 
@@ -44,25 +110,26 @@ void COFFWriter::layoutSections() {
 }
 
 size_t COFFWriter::finalizeStringTable() {
-  for (auto &S : Obj.Sections)
+  for (const auto &S : Obj.getSections())
     if (S.Name.size() > COFF::NameSize)
       StrTabBuilder.add(S.Name);
 
-  for (const auto &S : Obj.Symbols)
+  for (const auto &S : Obj.getSymbols())
     if (S.Name.size() > COFF::NameSize)
       StrTabBuilder.add(S.Name);
 
   StrTabBuilder.finalize();
 
-  for (auto &S : Obj.Sections) {
+  for (auto &S : Obj.getMutableSections()) {
     if (S.Name.size() > COFF::NameSize) {
+      memset(S.Header.Name, 0, sizeof(S.Header.Name));
       snprintf(S.Header.Name, sizeof(S.Header.Name), "/%d",
                (int)StrTabBuilder.getOffset(S.Name));
     } else {
       strncpy(S.Header.Name, S.Name.data(), COFF::NameSize);
     }
   }
-  for (auto &S : Obj.Symbols) {
+  for (auto &S : Obj.getMutableSymbols()) {
     if (S.Name.size() > COFF::NameSize) {
       S.Sym.Name.Offset.Zeroes = 0;
       S.Sym.Name.Offset.Offset = StrTabBuilder.getOffset(S.Name);
@@ -75,13 +142,31 @@ size_t COFFWriter::finalizeStringTable() {
 
 template <class SymbolTy>
 std::pair<size_t, size_t> COFFWriter::finalizeSymbolTable() {
-  size_t SymTabSize = Obj.Symbols.size() * sizeof(SymbolTy);
-  for (const auto &S : Obj.Symbols)
-    SymTabSize += S.AuxData.size();
-  return std::make_pair(SymTabSize, sizeof(SymbolTy));
+  size_t RawSymIndex = 0;
+  for (auto &S : Obj.getMutableSymbols()) {
+    // Symbols normally have NumberOfAuxSymbols set correctly all the time.
+    // For file symbols, we need to know the output file's symbol size to be
+    // able to calculate the number of slots it occupies.
+    if (!S.AuxFile.empty())
+      S.Sym.NumberOfAuxSymbols =
+          alignTo(S.AuxFile.size(), sizeof(SymbolTy)) / sizeof(SymbolTy);
+    S.RawIndex = RawSymIndex;
+    RawSymIndex += 1 + S.Sym.NumberOfAuxSymbols;
+  }
+  return std::make_pair(RawSymIndex * sizeof(SymbolTy), sizeof(SymbolTy));
 }
 
-void COFFWriter::finalize(bool IsBigObj) {
+Error COFFWriter::finalize(bool IsBigObj) {
+  size_t SymTabSize, SymbolSize;
+  std::tie(SymTabSize, SymbolSize) = IsBigObj
+                                         ? finalizeSymbolTable<coff_symbol32>()
+                                         : finalizeSymbolTable<coff_symbol16>();
+
+  if (Error E = finalizeRelocTargets())
+    return E;
+  if (Error E = finalizeSymbolContents())
+    return E;
+
   size_t SizeOfHeaders = 0;
   FileAlignment = 1;
   size_t PeHeaderSize = 0;
@@ -97,10 +182,10 @@ void COFFWriter::finalize(bool IsBigObj) {
     SizeOfHeaders +=
         PeHeaderSize + sizeof(data_directory) * Obj.DataDirectories.size();
   }
-  Obj.CoffFileHeader.NumberOfSections = Obj.Sections.size();
+  Obj.CoffFileHeader.NumberOfSections = Obj.getSections().size();
   SizeOfHeaders +=
       IsBigObj ? sizeof(coff_bigobj_file_header) : sizeof(coff_file_header);
-  SizeOfHeaders += sizeof(coff_section) * Obj.Sections.size();
+  SizeOfHeaders += sizeof(coff_section) * Obj.getSections().size();
   SizeOfHeaders = alignTo(SizeOfHeaders, FileAlignment);
 
   Obj.CoffFileHeader.SizeOfOptionalHeader =
@@ -115,8 +200,8 @@ void COFFWriter::finalize(bool IsBigObj) {
     Obj.PeHeader.SizeOfHeaders = SizeOfHeaders;
     Obj.PeHeader.SizeOfInitializedData = SizeOfInitializedData;
 
-    if (!Obj.Sections.empty()) {
-      const Section &S = Obj.Sections.back();
+    if (!Obj.getSections().empty()) {
+      const Section &S = Obj.getSections().back();
       Obj.PeHeader.SizeOfImage =
           alignTo(S.Header.VirtualAddress + S.Header.VirtualSize,
                   Obj.PeHeader.SectionAlignment);
@@ -128,20 +213,15 @@ void COFFWriter::finalize(bool IsBigObj) {
   }
 
   size_t StrTabSize = finalizeStringTable();
-  size_t SymTabSize, SymbolSize;
-  std::tie(SymTabSize, SymbolSize) = IsBigObj
-                                         ? finalizeSymbolTable<coff_symbol32>()
-                                         : finalizeSymbolTable<coff_symbol16>();
 
   size_t PointerToSymbolTable = FileSize;
   // StrTabSize <= 4 is the size of an empty string table, only consisting
   // of the length field.
-  if (SymTabSize == 0 && StrTabSize <= 4) {
-    // Don't point to the symbol table if empty.
+  if (SymTabSize == 0 && StrTabSize <= 4 && Obj.IsPE) {
+    // For executables, don't point to the symbol table and skip writing
+    // the length field, if both the symbol and string tables are empty.
     PointerToSymbolTable = 0;
-    // For executables, skip the length field of an empty string table.
-    if (Obj.IsPE)
-      StrTabSize = 0;
+    StrTabSize = 0;
   }
 
   size_t NumRawSymbols = SymTabSize / SymbolSize;
@@ -149,6 +229,8 @@ void COFFWriter::finalize(bool IsBigObj) {
   Obj.CoffFileHeader.NumberOfSymbols = NumRawSymbols;
   FileSize += SymTabSize + StrTabSize;
   FileSize = alignTo(FileSize, FileAlignment);
+
+  return Error::success();
 }
 
 void COFFWriter::writeHeaders(bool IsBigObj) {
@@ -181,7 +263,7 @@ void COFFWriter::writeHeaders(bool IsBigObj) {
     BigObjHeader.unused4 = 0;
     // The value in Obj.CoffFileHeader.NumberOfSections is truncated, thus
     // get the original one instead.
-    BigObjHeader.NumberOfSections = Obj.Sections.size();
+    BigObjHeader.NumberOfSections = Obj.getSections().size();
     BigObjHeader.PointerToSymbolTable = Obj.CoffFileHeader.PointerToSymbolTable;
     BigObjHeader.NumberOfSymbols = Obj.CoffFileHeader.NumberOfSymbols;
 
@@ -206,38 +288,57 @@ void COFFWriter::writeHeaders(bool IsBigObj) {
       Ptr += sizeof(DD);
     }
   }
-  for (const auto &S : Obj.Sections) {
+  for (const auto &S : Obj.getSections()) {
     memcpy(Ptr, &S.Header, sizeof(S.Header));
     Ptr += sizeof(S.Header);
   }
 }
 
 void COFFWriter::writeSections() {
-  for (const auto &S : Obj.Sections) {
+  for (const auto &S : Obj.getSections()) {
     uint8_t *Ptr = Buf.getBufferStart() + S.Header.PointerToRawData;
-    memcpy(Ptr, S.Contents.data(), S.Contents.size());
+    ArrayRef<uint8_t> Contents = S.getContents();
+    std::copy(Contents.begin(), Contents.end(), Ptr);
 
     // For executable sections, pad the remainder of the raw data size with
     // 0xcc, which is int3 on x86.
     if ((S.Header.Characteristics & IMAGE_SCN_CNT_CODE) &&
-        S.Header.SizeOfRawData > S.Contents.size())
-      memset(Ptr + S.Contents.size(), 0xcc,
-             S.Header.SizeOfRawData - S.Contents.size());
+        S.Header.SizeOfRawData > Contents.size())
+      memset(Ptr + Contents.size(), 0xcc,
+             S.Header.SizeOfRawData - Contents.size());
 
     Ptr += S.Header.SizeOfRawData;
-    memcpy(Ptr, S.Relocs.data(), S.Relocs.size() * sizeof(coff_relocation));
+    for (const auto &R : S.Relocs) {
+      memcpy(Ptr, &R.Reloc, sizeof(R.Reloc));
+      Ptr += sizeof(R.Reloc);
+    }
   }
 }
 
 template <class SymbolTy> void COFFWriter::writeSymbolStringTables() {
   uint8_t *Ptr = Buf.getBufferStart() + Obj.CoffFileHeader.PointerToSymbolTable;
-  for (const auto &S : Obj.Symbols) {
+  for (const auto &S : Obj.getSymbols()) {
     // Convert symbols back to the right size, from coff_symbol32.
     copySymbol<SymbolTy, coff_symbol32>(*reinterpret_cast<SymbolTy *>(Ptr),
                                         S.Sym);
     Ptr += sizeof(SymbolTy);
-    memcpy(Ptr, S.AuxData.data(), S.AuxData.size());
-    Ptr += S.AuxData.size();
+    if (!S.AuxFile.empty()) {
+      // For file symbols, just write the string into the aux symbol slots,
+      // assuming that the unwritten parts are initialized to zero in the memory
+      // mapped file.
+      std::copy(S.AuxFile.begin(), S.AuxFile.end(), Ptr);
+      Ptr += S.Sym.NumberOfAuxSymbols * sizeof(SymbolTy);
+    } else {
+      // For other auxillary symbols, write their opaque payload into one symbol
+      // table slot each. For big object files, the symbols are larger than the
+      // opaque auxillary symbol struct and we leave padding at the end of each
+      // entry.
+      for (const AuxSymbol &AuxSym : S.AuxData) {
+        ArrayRef<uint8_t> Ref = AuxSym.getRef();
+        std::copy(Ref.begin(), Ref.end(), Ptr);
+        Ptr += sizeof(SymbolTy);
+      }
+    }
   }
   if (StrTabBuilder.getSize() > 4 || !Obj.IsPE) {
     // Always write a string table in object files, even an empty one.
@@ -246,10 +347,12 @@ template <class SymbolTy> void COFFWriter::writeSymbolStringTables() {
   }
 }
 
-void COFFWriter::write(bool IsBigObj) {
-  finalize(IsBigObj);
+Error COFFWriter::write(bool IsBigObj) {
+  if (Error E = finalize(IsBigObj))
+    return E;
 
-  Buf.allocate(FileSize);
+  if (Error E = Buf.allocate(FileSize))
+    return E;
 
   writeHeaders(IsBigObj);
   writeSections();
@@ -259,31 +362,29 @@ void COFFWriter::write(bool IsBigObj) {
     writeSymbolStringTables<coff_symbol16>();
 
   if (Obj.IsPE)
-    patchDebugDirectory();
+    if (Error E = patchDebugDirectory())
+      return E;
 
-  if (auto E = Buf.commit())
-    reportError(Buf.getName(), errorToErrorCode(std::move(E)));
+  return Buf.commit();
 }
 
 // Locate which sections contain the debug directories, iterate over all
 // the debug_directory structs in there, and set the PointerToRawData field
 // in all of them, according to their new physical location in the file.
-void COFFWriter::patchDebugDirectory() {
+Error COFFWriter::patchDebugDirectory() {
   if (Obj.DataDirectories.size() < DEBUG_DIRECTORY)
-    return;
+    return Error::success();
   const data_directory *Dir = &Obj.DataDirectories[DEBUG_DIRECTORY];
   if (Dir->Size <= 0)
-    return;
-  for (const auto &S : Obj.Sections) {
+    return Error::success();
+  for (const auto &S : Obj.getSections()) {
     if (Dir->RelativeVirtualAddress >= S.Header.VirtualAddress &&
         Dir->RelativeVirtualAddress <
             S.Header.VirtualAddress + S.Header.SizeOfRawData) {
       if (Dir->RelativeVirtualAddress + Dir->Size >
           S.Header.VirtualAddress + S.Header.SizeOfRawData)
-        reportError(Buf.getName(),
-                    make_error<StringError>(
-                        "Debug directory extends past end of section",
-                        object_error::parse_failed));
+        return createStringError(object_error::parse_failed,
+                                 "Debug directory extends past end of section");
 
       size_t Offset = Dir->RelativeVirtualAddress - S.Header.VirtualAddress;
       uint8_t *Ptr = Buf.getBufferStart() + S.Header.PointerToRawData + Offset;
@@ -296,21 +397,19 @@ void COFFWriter::patchDebugDirectory() {
         Offset += sizeof(debug_directory) + Debug->SizeOfData;
       }
       // Debug directory found and patched, all done.
-      return;
+      return Error::success();
     }
   }
-  reportError(Buf.getName(),
-              make_error<StringError>("Debug directory not found",
-                                      object_error::parse_failed));
+  return createStringError(object_error::parse_failed,
+                           "Debug directory not found");
 }
 
-void COFFWriter::write() {
-  bool IsBigObj = Obj.Sections.size() > MaxNumberOfSections16;
+Error COFFWriter::write() {
+  bool IsBigObj = Obj.getSections().size() > MaxNumberOfSections16;
   if (IsBigObj && Obj.IsPE)
-    reportError(Buf.getName(),
-                make_error<StringError>("Too many sections for executable",
-                                        object_error::parse_failed));
-  write(IsBigObj);
+    return createStringError(object_error::parse_failed,
+                             "Too many sections for executable");
+  return write(IsBigObj);
 }
 
 } // end namespace coff
diff --git a/tools/llvm-objcopy/COFF/Writer.h b/tools/llvm-objcopy/COFF/Writer.h
index 397fda02ee980761333e12d6105c64a6c38579ea..681a8d5e4a66cb7a2774a368504c5bed11502364 100644
--- a/tools/llvm-objcopy/COFF/Writer.h
+++ b/tools/llvm-objcopy/COFF/Writer.h
@@ -1,9 +1,8 @@
 //===- Writer.h -------------------------------------------------*- C++ -*-===//
 //
-//                      The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -12,6 +11,7 @@
 
 #include "Buffer.h"
 #include "llvm/MC/StringTableBuilder.h"
+#include "llvm/Support/Error.h"
 #include <cstddef>
 #include <utility>
 
@@ -21,44 +21,37 @@ namespace coff {
 
 struct Object;
 
-class Writer {
-protected:
+class COFFWriter {
   Object &Obj;
   Buffer &Buf;
 
-public:
-  virtual ~Writer();
-  virtual void write() = 0;
-
-  Writer(Object &O, Buffer &B) : Obj(O), Buf(B) {}
-};
-
-class COFFWriter : public Writer {
   size_t FileSize;
   size_t FileAlignment;
   size_t SizeOfInitializedData;
   StringTableBuilder StrTabBuilder;
 
+  template <class SymbolTy> std::pair<size_t, size_t> finalizeSymbolTable();
+  Error finalizeRelocTargets();
+  Error finalizeSymbolContents();
   void layoutSections();
   size_t finalizeStringTable();
-  template <class SymbolTy> std::pair<size_t, size_t> finalizeSymbolTable();
 
-  void finalize(bool IsBigObj);
+  Error finalize(bool IsBigObj);
 
   void writeHeaders(bool IsBigObj);
   void writeSections();
   template <class SymbolTy> void writeSymbolStringTables();
 
-  void write(bool IsBigObj);
+  Error write(bool IsBigObj);
 
-  void patchDebugDirectory();
+  Error patchDebugDirectory();
 
 public:
   virtual ~COFFWriter() {}
-  void write() override;
+  Error write();
 
   COFFWriter(Object &Obj, Buffer &Buf)
-      : Writer(Obj, Buf), StrTabBuilder(StringTableBuilder::WinCOFF) {}
+      : Obj(Obj), Buf(Buf), StrTabBuilder(StringTableBuilder::WinCOFF) {}
 };
 
 } // end namespace coff
diff --git a/tools/llvm-objcopy/CopyConfig.cpp b/tools/llvm-objcopy/CopyConfig.cpp
index 24f0e29d4926f37f07a8b6ba1816d2a6d8f54292..811f8b753e07b8c37c455ee130cf0471130f8320 100644
--- a/tools/llvm-objcopy/CopyConfig.cpp
+++ b/tools/llvm-objcopy/CopyConfig.cpp
@@ -1,9 +1,8 @@
 //===- CopyConfig.cpp -----------------------------------------------------===//
 //
-//                      The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -19,9 +18,10 @@
 #include "llvm/Option/ArgList.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Compression.h"
+#include "llvm/Support/Errc.h"
 #include "llvm/Support/MemoryBuffer.h"
+#include "llvm/Support/StringSaver.h"
 #include <memory>
-#include <string>
 
 namespace llvm {
 namespace objcopy {
@@ -129,6 +129,32 @@ static SectionFlag parseSectionRenameFlag(StringRef SectionName) {
       .Default(SectionFlag::SecNone);
 }
 
+static uint64_t parseSectionFlagSet(ArrayRef<StringRef> SectionFlags) {
+  SectionFlag ParsedFlags = SectionFlag::SecNone;
+  for (StringRef Flag : SectionFlags) {
+    SectionFlag ParsedFlag = parseSectionRenameFlag(Flag);
+    if (ParsedFlag == SectionFlag::SecNone)
+      error("Unrecognized section flag '" + Flag +
+            "'. Flags supported for GNU compatibility: alloc, load, noload, "
+            "readonly, debug, code, data, rom, share, contents, merge, "
+            "strings.");
+    ParsedFlags |= ParsedFlag;
+  }
+
+  uint64_t NewFlags = 0;
+  if (ParsedFlags & SectionFlag::SecAlloc)
+    NewFlags |= ELF::SHF_ALLOC;
+  if (!(ParsedFlags & SectionFlag::SecReadonly))
+    NewFlags |= ELF::SHF_WRITE;
+  if (ParsedFlags & SectionFlag::SecCode)
+    NewFlags |= ELF::SHF_EXECINSTR;
+  if (ParsedFlags & SectionFlag::SecMerge)
+    NewFlags |= ELF::SHF_MERGE;
+  if (ParsedFlags & SectionFlag::SecStrings)
+    NewFlags |= ELF::SHF_STRINGS;
+  return NewFlags;
+}
+
 static SectionRename parseRenameSectionValue(StringRef FlagValue) {
   if (!FlagValue.contains('='))
     error("Bad format for --rename-section: missing '='");
@@ -143,34 +169,29 @@ static SectionRename parseRenameSectionValue(StringRef FlagValue) {
   Old2New.second.split(NameAndFlags, ',');
   SR.NewName = NameAndFlags[0];
 
-  if (NameAndFlags.size() > 1) {
-    SectionFlag Flags = SectionFlag::SecNone;
-    for (size_t I = 1, Size = NameAndFlags.size(); I < Size; ++I) {
-      SectionFlag Flag = parseSectionRenameFlag(NameAndFlags[I]);
-      if (Flag == SectionFlag::SecNone)
-        error("Unrecognized section flag '" + NameAndFlags[I] +
-              "'. Flags supported for GNU compatibility: alloc, load, noload, "
-              "readonly, debug, code, data, rom, share, contents, merge, "
-              "strings.");
-      Flags |= Flag;
-    }
-
-    SR.NewFlags = 0;
-    if (Flags & SectionFlag::SecAlloc)
-      *SR.NewFlags |= ELF::SHF_ALLOC;
-    if (!(Flags & SectionFlag::SecReadonly))
-      *SR.NewFlags |= ELF::SHF_WRITE;
-    if (Flags & SectionFlag::SecCode)
-      *SR.NewFlags |= ELF::SHF_EXECINSTR;
-    if (Flags & SectionFlag::SecMerge)
-      *SR.NewFlags |= ELF::SHF_MERGE;
-    if (Flags & SectionFlag::SecStrings)
-      *SR.NewFlags |= ELF::SHF_STRINGS;
-  }
+  if (NameAndFlags.size() > 1)
+    SR.NewFlags = parseSectionFlagSet(makeArrayRef(NameAndFlags).drop_front());
 
   return SR;
 }
 
+static SectionFlagsUpdate parseSetSectionFlagValue(StringRef FlagValue) {
+  if (!StringRef(FlagValue).contains('='))
+    error("Bad format for --set-section-flags: missing '='");
+
+  // Initial split: ".foo" = "f1,f2,..."
+  auto Section2Flags = StringRef(FlagValue).split('=');
+  SectionFlagsUpdate SFU;
+  SFU.Name = Section2Flags.first;
+
+  // Flags split: "f1" "f2" ...
+  SmallVector<StringRef, 6> SectionFlags;
+  Section2Flags.second.split(SectionFlags, ',');
+  SFU.NewFlags = parseSectionFlagSet(SectionFlags);
+
+  return SFU;
+}
+
 static const StringMap<MachineInfo> ArchMap{
     // Name, {EMachine, 64bit, LittleEndian}
     {"aarch64", {ELF::EM_AARCH64, true, true}},
@@ -189,8 +210,26 @@ static const MachineInfo &getMachineInfo(StringRef Arch) {
   return Iter->getValue();
 }
 
-static void addGlobalSymbolsFromFile(std::vector<std::string> &Symbols,
-                                     StringRef Filename) {
+static const StringMap<MachineInfo> OutputFormatMap{
+    // Name, {EMachine, 64bit, LittleEndian}
+    {"elf32-i386", {ELF::EM_386, false, true}},
+    {"elf32-powerpcle", {ELF::EM_PPC, false, true}},
+    {"elf32-x86-64", {ELF::EM_X86_64, false, true}},
+    {"elf64-powerpcle", {ELF::EM_PPC64, true, true}},
+    {"elf64-x86-64", {ELF::EM_X86_64, true, true}},
+};
+
+static const MachineInfo &getOutputFormatMachineInfo(StringRef Format) {
+  auto Iter = OutputFormatMap.find(Format);
+  if (Iter == std::end(OutputFormatMap))
+    error("Invalid output format: '" + Format + "'");
+  return Iter->getValue();
+}
+
+static void addSymbolsFromFile(std::vector<NameOrRegex> &Symbols,
+                               BumpPtrAllocator &Alloc, StringRef Filename,
+                               bool UseRegex) {
+  StringSaver Saver(Alloc);
   SmallVector<StringRef, 16> Lines;
   auto BufOrErr = MemoryBuffer::getFile(Filename);
   if (!BufOrErr)
@@ -202,14 +241,52 @@ static void addGlobalSymbolsFromFile(std::vector<std::string> &Symbols,
     // it's not empty.
     auto TrimmedLine = Line.split('#').first.trim();
     if (!TrimmedLine.empty())
-      Symbols.push_back(TrimmedLine.str());
+      Symbols.emplace_back(Saver.save(TrimmedLine), UseRegex);
+  }
+}
+
+NameOrRegex::NameOrRegex(StringRef Pattern, bool IsRegex) {
+  if (!IsRegex) {
+    Name = Pattern;
+    return;
   }
+
+  SmallVector<char, 32> Data;
+  R = std::make_shared<Regex>(
+      ("^" + Pattern.ltrim('^').rtrim('$') + "$").toStringRef(Data));
 }
 
+static Error addSymbolsToRenameFromFile(StringMap<StringRef> &SymbolsToRename,
+                                        BumpPtrAllocator &Alloc,
+                                        StringRef Filename) {
+  StringSaver Saver(Alloc);
+  SmallVector<StringRef, 16> Lines;
+  auto BufOrErr = MemoryBuffer::getFile(Filename);
+  if (!BufOrErr)
+    return createFileError(Filename, BufOrErr.getError());
+
+  BufOrErr.get()->getBuffer().split(Lines, '\n');
+  size_t NumLines = Lines.size();
+  for (size_t LineNo = 0; LineNo < NumLines; ++LineNo) {
+    StringRef TrimmedLine = Lines[LineNo].split('#').first.trim();
+    if (TrimmedLine.empty())
+      continue;
+
+    std::pair<StringRef, StringRef> Pair = Saver.save(TrimmedLine).split(' ');
+    StringRef NewName = Pair.second.trim();
+    if (NewName.empty())
+      return createStringError(errc::invalid_argument,
+                               "%s:%zu: missing new symbol name",
+                               Filename.str().c_str(), LineNo + 1);
+    SymbolsToRename.insert({Pair.first, NewName});
+  }
+  return Error::success();
+}
 // ParseObjcopyOptions returns the config and sets the input arguments. If a
 // help flag is set then ParseObjcopyOptions will print the help messege and
 // exit.
 DriverConfig parseObjcopyOptions(ArrayRef<const char *> ArgsArr) {
+  DriverConfig DC;
   ObjcopyOptTable T;
   unsigned MissingArgumentIndex, MissingArgumentCount;
   llvm::opt::InputArgList InputArgs =
@@ -253,6 +330,7 @@ DriverConfig parseObjcopyOptions(ArrayRef<const char *> ArgsArr) {
        InputArgs.hasArg(OBJCOPY_output_target)))
     error("--target cannot be used with --input-target or --output-target");
 
+  bool UseRegex = InputArgs.hasArg(OBJCOPY_regex);
   if (InputArgs.hasArg(OBJCOPY_target)) {
     Config.InputFormat = InputArgs.getLastArgValue(OBJCOPY_target);
     Config.OutputFormat = InputArgs.getLastArgValue(OBJCOPY_target);
@@ -266,6 +344,8 @@ DriverConfig parseObjcopyOptions(ArrayRef<const char *> ArgsArr) {
       error("Specified binary input without specifiying an architecture");
     Config.BinaryArch = getMachineInfo(BinaryArch);
   }
+  if (!Config.OutputFormat.empty() && Config.OutputFormat != "binary")
+    Config.OutputArch = getOutputFormatMachineInfo(Config.OutputFormat);
 
   if (auto Arg = InputArgs.getLastArg(OBJCOPY_compress_debug_sections,
                                       OBJCOPY_compress_debug_sections_eq)) {
@@ -305,18 +385,41 @@ DriverConfig parseObjcopyOptions(ArrayRef<const char *> ArgsArr) {
       error("Multiple redefinition of symbol " + Old2New.first);
   }
 
+  for (auto Arg : InputArgs.filtered(OBJCOPY_redefine_symbols))
+    if (Error E = addSymbolsToRenameFromFile(Config.SymbolsToRename, DC.Alloc,
+                                             Arg->getValue()))
+      error(std::move(E));
+
   for (auto Arg : InputArgs.filtered(OBJCOPY_rename_section)) {
     SectionRename SR = parseRenameSectionValue(StringRef(Arg->getValue()));
     if (!Config.SectionsToRename.try_emplace(SR.OriginalName, SR).second)
       error("Multiple renames of section " + SR.OriginalName);
   }
+  for (auto Arg : InputArgs.filtered(OBJCOPY_set_section_flags)) {
+    SectionFlagsUpdate SFU = parseSetSectionFlagValue(Arg->getValue());
+    if (!Config.SetSectionFlags.try_emplace(SFU.Name, SFU).second)
+      error("--set-section-flags set multiple times for section " + SFU.Name);
+  }
+  // Prohibit combinations of --set-section-flags when the section name is used
+  // by --rename-section, either as a source or a destination.
+  for (const auto &E : Config.SectionsToRename) {
+    const SectionRename &SR = E.second;
+    if (Config.SetSectionFlags.count(SR.OriginalName))
+      error("--set-section-flags=" + SR.OriginalName +
+            " conflicts with --rename-section=" + SR.OriginalName + "=" +
+            SR.NewName);
+    if (Config.SetSectionFlags.count(SR.NewName))
+      error("--set-section-flags=" + SR.NewName +
+            " conflicts with --rename-section=" + SR.OriginalName + "=" +
+            SR.NewName);
+  }
 
   for (auto Arg : InputArgs.filtered(OBJCOPY_remove_section))
-    Config.ToRemove.push_back(Arg->getValue());
+    Config.ToRemove.emplace_back(Arg->getValue(), UseRegex);
   for (auto Arg : InputArgs.filtered(OBJCOPY_keep_section))
-    Config.KeepSection.push_back(Arg->getValue());
+    Config.KeepSection.emplace_back(Arg->getValue(), UseRegex);
   for (auto Arg : InputArgs.filtered(OBJCOPY_only_section))
-    Config.OnlySection.push_back(Arg->getValue());
+    Config.OnlySection.emplace_back(Arg->getValue(), UseRegex);
   for (auto Arg : InputArgs.filtered(OBJCOPY_add_section))
     Config.AddSection.push_back(Arg->getValue());
   for (auto Arg : InputArgs.filtered(OBJCOPY_dump_section))
@@ -331,25 +434,47 @@ DriverConfig parseObjcopyOptions(ArrayRef<const char *> ArgsArr) {
   Config.ExtractDWO = InputArgs.hasArg(OBJCOPY_extract_dwo);
   Config.LocalizeHidden = InputArgs.hasArg(OBJCOPY_localize_hidden);
   Config.Weaken = InputArgs.hasArg(OBJCOPY_weaken);
-  Config.DiscardAll = InputArgs.hasArg(OBJCOPY_discard_all);
+  if (InputArgs.hasArg(OBJCOPY_discard_all, OBJCOPY_discard_locals))
+    Config.DiscardMode =
+        InputArgs.hasFlag(OBJCOPY_discard_all, OBJCOPY_discard_locals)
+            ? DiscardType::All
+            : DiscardType::Locals;
   Config.OnlyKeepDebug = InputArgs.hasArg(OBJCOPY_only_keep_debug);
   Config.KeepFileSymbols = InputArgs.hasArg(OBJCOPY_keep_file_symbols);
   Config.DecompressDebugSections =
       InputArgs.hasArg(OBJCOPY_decompress_debug_sections);
   for (auto Arg : InputArgs.filtered(OBJCOPY_localize_symbol))
-    Config.SymbolsToLocalize.push_back(Arg->getValue());
+    Config.SymbolsToLocalize.emplace_back(Arg->getValue(), UseRegex);
+  for (auto Arg : InputArgs.filtered(OBJCOPY_localize_symbols))
+    addSymbolsFromFile(Config.SymbolsToLocalize, DC.Alloc, Arg->getValue(),
+                       UseRegex);
   for (auto Arg : InputArgs.filtered(OBJCOPY_keep_global_symbol))
-    Config.SymbolsToKeepGlobal.push_back(Arg->getValue());
+    Config.SymbolsToKeepGlobal.emplace_back(Arg->getValue(), UseRegex);
   for (auto Arg : InputArgs.filtered(OBJCOPY_keep_global_symbols))
-    addGlobalSymbolsFromFile(Config.SymbolsToKeepGlobal, Arg->getValue());
+    addSymbolsFromFile(Config.SymbolsToKeepGlobal, DC.Alloc, Arg->getValue(),
+                       UseRegex);
   for (auto Arg : InputArgs.filtered(OBJCOPY_globalize_symbol))
-    Config.SymbolsToGlobalize.push_back(Arg->getValue());
+    Config.SymbolsToGlobalize.emplace_back(Arg->getValue(), UseRegex);
+  for (auto Arg : InputArgs.filtered(OBJCOPY_globalize_symbols))
+    addSymbolsFromFile(Config.SymbolsToGlobalize, DC.Alloc, Arg->getValue(),
+                       UseRegex);
   for (auto Arg : InputArgs.filtered(OBJCOPY_weaken_symbol))
-    Config.SymbolsToWeaken.push_back(Arg->getValue());
+    Config.SymbolsToWeaken.emplace_back(Arg->getValue(), UseRegex);
+  for (auto Arg : InputArgs.filtered(OBJCOPY_weaken_symbols))
+    addSymbolsFromFile(Config.SymbolsToWeaken, DC.Alloc, Arg->getValue(),
+                       UseRegex);
   for (auto Arg : InputArgs.filtered(OBJCOPY_strip_symbol))
-    Config.SymbolsToRemove.push_back(Arg->getValue());
+    Config.SymbolsToRemove.emplace_back(Arg->getValue(), UseRegex);
+  for (auto Arg : InputArgs.filtered(OBJCOPY_strip_symbols))
+    addSymbolsFromFile(Config.SymbolsToRemove, DC.Alloc, Arg->getValue(),
+                       UseRegex);
+  for (auto Arg : InputArgs.filtered(OBJCOPY_strip_unneeded_symbol))
+    Config.UnneededSymbolsToRemove.emplace_back(Arg->getValue(), UseRegex);
+  for (auto Arg : InputArgs.filtered(OBJCOPY_strip_unneeded_symbols))
+    addSymbolsFromFile(Config.UnneededSymbolsToRemove, DC.Alloc,
+                       Arg->getValue(), UseRegex);
   for (auto Arg : InputArgs.filtered(OBJCOPY_keep_symbol))
-    Config.SymbolsToKeep.push_back(Arg->getValue());
+    Config.SymbolsToKeep.emplace_back(Arg->getValue(), UseRegex);
 
   Config.DeterministicArchives = InputArgs.hasFlag(
       OBJCOPY_enable_deterministic_archives,
@@ -366,7 +491,6 @@ DriverConfig parseObjcopyOptions(ArrayRef<const char *> ArgsArr) {
   if (Config.DecompressDebugSections && !zlib::isAvailable())
     error("LLVM was not compiled with LLVM_ENABLE_ZLIB: cannot decompress.");
 
-  DriverConfig DC;
   DC.CopyConfigs.push_back(std::move(Config));
   return DC;
 }
@@ -409,25 +533,34 @@ DriverConfig parseStripOptions(ArrayRef<const char *> ArgsArr) {
     error("Multiple input files cannot be used in combination with -o");
 
   CopyConfig Config;
+  bool UseRegexp = InputArgs.hasArg(STRIP_regex);
   Config.StripDebug = InputArgs.hasArg(STRIP_strip_debug);
 
-  Config.DiscardAll = InputArgs.hasArg(STRIP_discard_all);
+  if (InputArgs.hasArg(STRIP_discard_all, STRIP_discard_locals))
+    Config.DiscardMode =
+        InputArgs.hasFlag(STRIP_discard_all, STRIP_discard_locals)
+            ? DiscardType::All
+            : DiscardType::Locals;
   Config.StripUnneeded = InputArgs.hasArg(STRIP_strip_unneeded);
   Config.StripAll = InputArgs.hasArg(STRIP_strip_all);
   Config.StripAllGNU = InputArgs.hasArg(STRIP_strip_all_gnu);
-
-  if (!Config.StripDebug && !Config.StripUnneeded && !Config.DiscardAll &&
-      !Config.StripAllGNU)
-    Config.StripAll = true;
+  Config.KeepFileSymbols = InputArgs.hasArg(STRIP_keep_file_symbols);
 
   for (auto Arg : InputArgs.filtered(STRIP_keep_section))
-    Config.KeepSection.push_back(Arg->getValue());
+    Config.KeepSection.emplace_back(Arg->getValue(), UseRegexp);
 
   for (auto Arg : InputArgs.filtered(STRIP_remove_section))
-    Config.ToRemove.push_back(Arg->getValue());
+    Config.ToRemove.emplace_back(Arg->getValue(), UseRegexp);
+
+  for (auto Arg : InputArgs.filtered(STRIP_strip_symbol))
+    Config.SymbolsToRemove.emplace_back(Arg->getValue(), UseRegexp);
 
   for (auto Arg : InputArgs.filtered(STRIP_keep_symbol))
-    Config.SymbolsToKeep.push_back(Arg->getValue());
+    Config.SymbolsToKeep.emplace_back(Arg->getValue(), UseRegexp);
+
+  if (!Config.StripDebug && !Config.StripUnneeded &&
+      Config.DiscardMode == DiscardType::None && !Config.StripAllGNU && Config.SymbolsToRemove.empty())
+    Config.StripAll = true;
 
   Config.DeterministicArchives =
       InputArgs.hasFlag(STRIP_enable_deterministic_archives,
diff --git a/tools/llvm-objcopy/CopyConfig.h b/tools/llvm-objcopy/CopyConfig.h
index ce6ead80b7c5e08eec3197211b14f6383973a505..f1d5cf2c10cbf011a28827d7eab5b1c3711d4388 100644
--- a/tools/llvm-objcopy/CopyConfig.h
+++ b/tools/llvm-objcopy/CopyConfig.h
@@ -1,9 +1,8 @@
 //===- CopyConfig.h -------------------------------------------------------===//
 //
-//                      The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -15,9 +14,10 @@
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringMap.h"
 #include "llvm/ADT/StringRef.h"
+#include "llvm/Support/Allocator.h"
+#include "llvm/Support/Regex.h"
 // Necessary for llvm::DebugCompressionType::None
 #include "llvm/Target/TargetOptions.h"
-#include <string>
 #include <vector>
 
 namespace llvm {
@@ -38,6 +38,28 @@ struct SectionRename {
   Optional<uint64_t> NewFlags;
 };
 
+struct SectionFlagsUpdate {
+  StringRef Name;
+  uint64_t NewFlags;
+};
+
+enum class DiscardType {
+  None,   // Default
+  All,    // --discard-all (-x)
+  Locals, // --discard-locals (-X)
+};
+
+class NameOrRegex {
+  StringRef Name;
+  // Regex is shared between multiple CopyConfig instances.
+  std::shared_ptr<Regex> R;
+
+public:
+  NameOrRegex(StringRef Pattern, bool IsRegex);
+  bool operator==(StringRef S) const { return R ? R->match(S) : Name == S; }
+  bool operator!=(StringRef S) const { return !operator==(S); }
+};
+
 // Configuration for copying/stripping a single file.
 struct CopyConfig {
   // Main input/output options
@@ -46,8 +68,10 @@ struct CopyConfig {
   StringRef OutputFilename;
   StringRef OutputFormat;
 
-  // Only applicable for --input-format=Binary
+  // Only applicable for --input-format=binary
   MachineInfo BinaryArch;
+  // Only applicable when --output-format!=binary (e.g. elf64-x86-64).
+  Optional<MachineInfo> OutputArch;
 
   // Advanced options
   StringRef AddGnuDebugLink;
@@ -56,27 +80,29 @@ struct CopyConfig {
   Optional<StringRef> BuildIdLinkOutput;
   StringRef SplitDWO;
   StringRef SymbolsPrefix;
+  DiscardType DiscardMode = DiscardType::None;
 
   // Repeated options
   std::vector<StringRef> AddSection;
   std::vector<StringRef> DumpSection;
-  std::vector<StringRef> KeepSection;
-  std::vector<StringRef> OnlySection;
-  std::vector<StringRef> SymbolsToGlobalize;
-  std::vector<StringRef> SymbolsToKeep;
-  std::vector<StringRef> SymbolsToLocalize;
-  std::vector<StringRef> SymbolsToRemove;
-  std::vector<StringRef> SymbolsToWeaken;
-  std::vector<StringRef> ToRemove;
-  std::vector<std::string> SymbolsToKeepGlobal;
+  std::vector<NameOrRegex> KeepSection;
+  std::vector<NameOrRegex> OnlySection;
+  std::vector<NameOrRegex> SymbolsToGlobalize;
+  std::vector<NameOrRegex> SymbolsToKeep;
+  std::vector<NameOrRegex> SymbolsToLocalize;
+  std::vector<NameOrRegex> SymbolsToRemove;
+  std::vector<NameOrRegex> UnneededSymbolsToRemove;
+  std::vector<NameOrRegex> SymbolsToWeaken;
+  std::vector<NameOrRegex> ToRemove;
+  std::vector<NameOrRegex> SymbolsToKeepGlobal;
 
   // Map options
   StringMap<SectionRename> SectionsToRename;
+  StringMap<SectionFlagsUpdate> SetSectionFlags;
   StringMap<StringRef> SymbolsToRename;
 
   // Boolean options
   bool DeterministicArchives = true;
-  bool DiscardAll = false;
   bool ExtractDWO = false;
   bool KeepFileSymbols = false;
   bool LocalizeHidden = false;
@@ -99,6 +125,7 @@ struct CopyConfig {
 // will contain one or more CopyConfigs.
 struct DriverConfig {
   SmallVector<CopyConfig, 1> CopyConfigs;
+  BumpPtrAllocator Alloc;
 };
 
 // ParseObjcopyOptions returns the config and sets the input arguments. If a
diff --git a/tools/llvm-objcopy/ELF/ELFObjcopy.cpp b/tools/llvm-objcopy/ELF/ELFObjcopy.cpp
index 8a136de24a87bce94a80e5493e13e82cace4e30b..4d2e78ade999effad250866695ed44adbd6efbc1 100644
--- a/tools/llvm-objcopy/ELF/ELFObjcopy.cpp
+++ b/tools/llvm-objcopy/ELF/ELFObjcopy.cpp
@@ -1,9 +1,8 @@
 //===- ELFObjcopy.cpp -----------------------------------------------------===//
 //
-//                      The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -71,6 +70,17 @@ static bool onlyKeepDWOPred(const Object &Obj, const SectionBase &Sec) {
   return !isDWOSection(Sec);
 }
 
+static uint64_t setSectionFlagsPreserveMask(uint64_t OldFlags,
+                                            uint64_t NewFlags) {
+  // Preserve some flags which should not be dropped when setting flags.
+  // Also, preserve anything OS/processor dependant.
+  const uint64_t PreserveMask = ELF::SHF_COMPRESSED | ELF::SHF_EXCLUDE |
+                                ELF::SHF_GROUP | ELF::SHF_LINK_ORDER |
+                                ELF::SHF_MASKOS | ELF::SHF_MASKPROC |
+                                ELF::SHF_TLS | ELF::SHF_INFO_LINK;
+  return (OldFlags & PreserveMask) | (NewFlags & ~PreserveMask);
+}
+
 static ElfType getOutputElfType(const Binary &Bin) {
   // Infer output ELF type from the input ELF object
   if (isa<ELFObjectFile<ELF32LE>>(Bin))
@@ -147,13 +157,15 @@ findBuildID(const object::ELFObjectFileBase &In) {
   llvm_unreachable("Bad file format");
 }
 
-static void linkToBuildIdDir(const CopyConfig &Config, StringRef ToLink,
-                             StringRef Suffix, ArrayRef<uint8_t> BuildIdBytes) {
+static Error linkToBuildIdDir(const CopyConfig &Config, StringRef ToLink,
+                              StringRef Suffix,
+                              ArrayRef<uint8_t> BuildIdBytes) {
   SmallString<128> Path = Config.BuildIdLinkDir;
   sys::path::append(Path, llvm::toHex(BuildIdBytes[0], /*LowerCase*/ true));
   if (auto EC = sys::fs::create_directories(Path))
-    error("cannot create build ID link directory " + Path + ": " +
-          EC.message());
+    return createFileError(
+        Path.str(),
+        createStringError(EC, "cannot create build ID link directory"));
 
   sys::path::append(Path,
                     llvm::toHex(BuildIdBytes.slice(1), /*LowerCase*/ true));
@@ -164,29 +176,38 @@ static void linkToBuildIdDir(const CopyConfig &Config, StringRef ToLink,
       sys::fs::remove(Path);
     EC = sys::fs::create_hard_link(ToLink, Path);
     if (EC)
-      error("cannot link " + ToLink + " to " + Path + ": " + EC.message());
+      return createStringError(EC, "cannot link %s to %s", ToLink.data(),
+                               Path.data());
   }
+  return Error::success();
 }
 
-static void splitDWOToFile(const CopyConfig &Config, const Reader &Reader,
-                           StringRef File, ElfType OutputElfType) {
+static Error splitDWOToFile(const CopyConfig &Config, const Reader &Reader,
+                            StringRef File, ElfType OutputElfType) {
   auto DWOFile = Reader.create();
-  DWOFile->removeSections(
-      [&](const SectionBase &Sec) { return onlyKeepDWOPred(*DWOFile, Sec); });
+  auto OnlyKeepDWOPred = [&DWOFile](const SectionBase &Sec) {
+    return onlyKeepDWOPred(*DWOFile, Sec);
+  };
+  if (Error E = DWOFile->removeSections(OnlyKeepDWOPred))
+    return E;
+  if (Config.OutputArch)
+    DWOFile->Machine = Config.OutputArch.getValue().EMachine;
   FileBuffer FB(File);
   auto Writer = createWriter(Config, *DWOFile, FB, OutputElfType);
-  Writer->finalize();
-  Writer->write();
+  if (Error E = Writer->finalize())
+    return E;
+  return Writer->write();
 }
 
 static Error dumpSectionToFile(StringRef SecName, StringRef Filename,
                                Object &Obj) {
   for (auto &Sec : Obj.sections()) {
     if (Sec.Name == SecName) {
-      if (Sec.OriginalData.size() == 0)
-        return make_error<StringError>("Can't dump section \"" + SecName +
-                                           "\": it has no contents",
-                                       object_error::parse_failed);
+      if (Sec.OriginalData.empty())
+        return createStringError(
+            object_error::parse_failed,
+            "Can't dump section \"%s\": it has no contents",
+            SecName.str().c_str());
       Expected<std::unique_ptr<FileOutputBuffer>> BufferOrErr =
           FileOutputBuffer::create(Filename, Sec.OriginalData.size());
       if (!BufferOrErr)
@@ -199,8 +220,7 @@ static Error dumpSectionToFile(StringRef SecName, StringRef Filename,
       return Error::success();
     }
   }
-  return make_error<StringError>("Section not found",
-                                 object_error::parse_failed);
+  return createStringError(object_error::parse_failed, "Section not found");
 }
 
 static bool isCompressed(const SectionBase &Section) {
@@ -248,6 +268,12 @@ static void replaceDebugSections(
   };
 }
 
+static bool isUnneededSymbol(const Symbol &Sym) {
+  return !Sym.Referenced &&
+         (Sym.Binding == STB_LOCAL || Sym.getShndx() == SHN_UNDEF) &&
+         Sym.Type != STT_FILE && Sym.Type != STT_SECTION;
+}
+
 // This function handles the high level operations of GNU objcopy including
 // handling command line options. It's important to outline certain properties
 // we expect to hold of the command line operations. Any operation that "keeps"
@@ -255,18 +281,24 @@ static void replaceDebugSections(
 // any previous removals. Lastly whether or not something is removed shouldn't
 // depend a) on the order the options occur in or b) on some opaque priority
 // system. The only priority is that keeps/copies overrule removes.
-static void handleArgs(const CopyConfig &Config, Object &Obj,
-                       const Reader &Reader, ElfType OutputElfType) {
+static Error handleArgs(const CopyConfig &Config, Object &Obj,
+                        const Reader &Reader, ElfType OutputElfType) {
 
-  if (!Config.SplitDWO.empty()) {
-    splitDWOToFile(Config, Reader, Config.SplitDWO, OutputElfType);
-  }
+  if (!Config.SplitDWO.empty())
+    if (Error E =
+            splitDWOToFile(Config, Reader, Config.SplitDWO, OutputElfType))
+      return E;
+
+  if (Config.OutputArch)
+    Obj.Machine = Config.OutputArch.getValue().EMachine;
 
   // TODO: update or remove symbols only if there is an option that affects
   // them.
   if (Obj.SymbolTable) {
     Obj.SymbolTable->updateSymbols([&](Symbol &Sym) {
-      if (!Sym.isCommon() &&
+      // Common and undefined symbols don't make sense as local symbols, and can
+      // even cause crashes if we localize those, so skip them.
+      if (!Sym.isCommon() && Sym.getShndx() != SHN_UNDEF &&
           ((Config.LocalizeHidden &&
             (Sym.Visibility == STV_HIDDEN || Sym.Visibility == STV_INTERNAL)) ||
            is_contained(Config.SymbolsToLocalize, Sym.Name)))
@@ -310,19 +342,21 @@ static void handleArgs(const CopyConfig &Config, Object &Obj,
     // The purpose of this loop is to mark symbols referenced by sections
     // (like GroupSection or RelocationSection). This way, we know which
     // symbols are still 'needed' and which are not.
-    if (Config.StripUnneeded) {
+    if (Config.StripUnneeded || !Config.UnneededSymbolsToRemove.empty()) {
       for (auto &Section : Obj.sections())
         Section.markSymbols();
     }
 
-    Obj.removeSymbols([&](const Symbol &Sym) {
+    auto RemoveSymbolsPred = [&](const Symbol &Sym) {
       if (is_contained(Config.SymbolsToKeep, Sym.Name) ||
           (Config.KeepFileSymbols && Sym.Type == STT_FILE))
         return false;
 
-      if (Config.DiscardAll && Sym.Binding == STB_LOCAL &&
-          Sym.getShndx() != SHN_UNDEF && Sym.Type != STT_FILE &&
-          Sym.Type != STT_SECTION)
+      if ((Config.DiscardMode == DiscardType::All ||
+           (Config.DiscardMode == DiscardType::Locals &&
+            StringRef(Sym.Name).startswith(".L"))) &&
+          Sym.Binding == STB_LOCAL && Sym.getShndx() != SHN_UNDEF &&
+          Sym.Type != STT_FILE && Sym.Type != STT_SECTION)
         return true;
 
       if (Config.StripAll || Config.StripAllGNU)
@@ -331,13 +365,15 @@ static void handleArgs(const CopyConfig &Config, Object &Obj,
       if (is_contained(Config.SymbolsToRemove, Sym.Name))
         return true;
 
-      if (Config.StripUnneeded && !Sym.Referenced &&
-          (Sym.Binding == STB_LOCAL || Sym.getShndx() == SHN_UNDEF) &&
-          Sym.Type != STT_FILE && Sym.Type != STT_SECTION)
+      if ((Config.StripUnneeded ||
+           is_contained(Config.UnneededSymbolsToRemove, Sym.Name)) &&
+          isUnneededSymbol(Sym))
         return true;
 
       return false;
-    });
+    };
+    if (Error E = Obj.removeSymbols(RemoveSymbolsPred))
+      return E;
   }
 
   SectionPred RemovePred = [](const SectionBase &) { return false; };
@@ -471,7 +507,8 @@ static void handleArgs(const CopyConfig &Config, Object &Obj,
           return &Obj.addSection<DecompressedSection>(*CS);
         });
 
-  Obj.removeSections(RemovePred);
+  if (Error E = Obj.removeSections(RemovePred))
+    return E;
 
   if (!Config.SectionsToRename.empty()) {
     for (auto &Sec : Obj.sections()) {
@@ -479,33 +516,40 @@ static void handleArgs(const CopyConfig &Config, Object &Obj,
       if (Iter != Config.SectionsToRename.end()) {
         const SectionRename &SR = Iter->second;
         Sec.Name = SR.NewName;
-        if (SR.NewFlags.hasValue()) {
-          // Preserve some flags which should not be dropped when setting flags.
-          // Also, preserve anything OS/processor dependant.
-          const uint64_t PreserveMask = ELF::SHF_COMPRESSED | ELF::SHF_EXCLUDE |
-                                        ELF::SHF_GROUP | ELF::SHF_LINK_ORDER |
-                                        ELF::SHF_MASKOS | ELF::SHF_MASKPROC |
-                                        ELF::SHF_TLS | ELF::SHF_INFO_LINK;
-          Sec.Flags = (Sec.Flags & PreserveMask) |
-                      (SR.NewFlags.getValue() & ~PreserveMask);
-        }
+        if (SR.NewFlags.hasValue())
+          Sec.Flags =
+              setSectionFlagsPreserveMask(Sec.Flags, SR.NewFlags.getValue());
+      }
+    }
+  }
+
+  if (!Config.SetSectionFlags.empty()) {
+    for (auto &Sec : Obj.sections()) {
+      const auto Iter = Config.SetSectionFlags.find(Sec.Name);
+      if (Iter != Config.SetSectionFlags.end()) {
+        const SectionFlagsUpdate &SFU = Iter->second;
+        Sec.Flags = setSectionFlagsPreserveMask(Sec.Flags, SFU.NewFlags);
       }
     }
   }
 
   if (!Config.AddSection.empty()) {
     for (const auto &Flag : Config.AddSection) {
-      auto SecPair = Flag.split("=");
-      auto SecName = SecPair.first;
-      auto File = SecPair.second;
-      auto BufOrErr = MemoryBuffer::getFile(File);
+      std::pair<StringRef, StringRef> SecPair = Flag.split("=");
+      StringRef SecName = SecPair.first;
+      StringRef File = SecPair.second;
+      ErrorOr<std::unique_ptr<MemoryBuffer>> BufOrErr =
+          MemoryBuffer::getFile(File);
       if (!BufOrErr)
-        reportError(File, BufOrErr.getError());
-      auto Buf = std::move(*BufOrErr);
-      auto BufPtr = reinterpret_cast<const uint8_t *>(Buf->getBufferStart());
-      auto BufSize = Buf->getBufferSize();
-      Obj.addSection<OwnedDataSection>(SecName,
-                                       ArrayRef<uint8_t>(BufPtr, BufSize));
+        return createFileError(File, errorCodeToError(BufOrErr.getError()));
+      std::unique_ptr<MemoryBuffer> Buf = std::move(*BufOrErr);
+      ArrayRef<uint8_t> Data(
+          reinterpret_cast<const uint8_t *>(Buf->getBufferStart()),
+          Buf->getBufferSize());
+      OwnedDataSection &NewSection =
+          Obj.addSection<OwnedDataSection>(SecName, Data);
+      if (SecName.startswith(".note") && SecName != ".note.GNU-stack")
+        NewSection.Type = SHT_NOTE;
     }
   }
 
@@ -515,54 +559,74 @@ static void handleArgs(const CopyConfig &Config, Object &Obj,
       StringRef SecName = SecPair.first;
       StringRef File = SecPair.second;
       if (Error E = dumpSectionToFile(SecName, File, Obj))
-        reportError(Config.InputFilename, std::move(E));
+        return createFileError(Config.InputFilename, std::move(E));
     }
   }
 
   if (!Config.AddGnuDebugLink.empty())
     Obj.addSection<GnuDebugLinkSection>(Config.AddGnuDebugLink);
+
+  return Error::success();
 }
 
-void executeObjcopyOnRawBinary(const CopyConfig &Config, MemoryBuffer &In,
-                               Buffer &Out) {
+Error executeObjcopyOnRawBinary(const CopyConfig &Config, MemoryBuffer &In,
+                                Buffer &Out) {
   BinaryReader Reader(Config.BinaryArch, &In);
   std::unique_ptr<Object> Obj = Reader.create();
 
-  const ElfType OutputElfType = getOutputElfType(Config.BinaryArch);
-  handleArgs(Config, *Obj, Reader, OutputElfType);
+  // Prefer OutputArch (-O<format>) if set, otherwise fallback to BinaryArch
+  // (-B<arch>).
+  const ElfType OutputElfType = getOutputElfType(
+      Config.OutputArch ? Config.OutputArch.getValue() : Config.BinaryArch);
+  if (Error E = handleArgs(Config, *Obj, Reader, OutputElfType))
+    return E;
   std::unique_ptr<Writer> Writer =
       createWriter(Config, *Obj, Out, OutputElfType);
-  Writer->finalize();
-  Writer->write();
+  if (Error E = Writer->finalize())
+    return E;
+  return Writer->write();
 }
 
-void executeObjcopyOnBinary(const CopyConfig &Config,
-                            object::ELFObjectFileBase &In, Buffer &Out) {
+Error executeObjcopyOnBinary(const CopyConfig &Config,
+                             object::ELFObjectFileBase &In, Buffer &Out) {
   ELFReader Reader(&In);
   std::unique_ptr<Object> Obj = Reader.create();
-  const ElfType OutputElfType = getOutputElfType(In);
+  // Prefer OutputArch (-O<format>) if set, otherwise infer it from the input.
+  const ElfType OutputElfType =
+      Config.OutputArch ? getOutputElfType(Config.OutputArch.getValue())
+                        : getOutputElfType(In);
   ArrayRef<uint8_t> BuildIdBytes;
 
   if (!Config.BuildIdLinkDir.empty()) {
     BuildIdBytes = unwrapOrError(findBuildID(In));
     if (BuildIdBytes.size() < 2)
-      error("build ID in file '" + Config.InputFilename +
-            "' is smaller than two bytes");
+      return createFileError(
+          Config.InputFilename,
+          createStringError(object_error::parse_failed,
+                            "build ID is smaller than two bytes."));
   }
 
-  if (!Config.BuildIdLinkDir.empty() && Config.BuildIdLinkInput) {
-    linkToBuildIdDir(Config, Config.InputFilename,
-                     Config.BuildIdLinkInput.getValue(), BuildIdBytes);
-  }
-  handleArgs(Config, *Obj, Reader, OutputElfType);
+  if (!Config.BuildIdLinkDir.empty() && Config.BuildIdLinkInput)
+    if (Error E =
+            linkToBuildIdDir(Config, Config.InputFilename,
+                             Config.BuildIdLinkInput.getValue(), BuildIdBytes))
+      return E;
+
+  if (Error E = handleArgs(Config, *Obj, Reader, OutputElfType))
+    return E;
   std::unique_ptr<Writer> Writer =
       createWriter(Config, *Obj, Out, OutputElfType);
-  Writer->finalize();
-  Writer->write();
-  if (!Config.BuildIdLinkDir.empty() && Config.BuildIdLinkOutput) {
-    linkToBuildIdDir(Config, Config.OutputFilename,
-                     Config.BuildIdLinkOutput.getValue(), BuildIdBytes);
-  }
+  if (Error E = Writer->finalize())
+    return E;
+  if (Error E = Writer->write())
+    return E;
+  if (!Config.BuildIdLinkDir.empty() && Config.BuildIdLinkOutput)
+    if (Error E =
+            linkToBuildIdDir(Config, Config.OutputFilename,
+                             Config.BuildIdLinkOutput.getValue(), BuildIdBytes))
+      return E;
+
+  return Error::success();
 }
 
 } // end namespace elf
diff --git a/tools/llvm-objcopy/ELF/ELFObjcopy.h b/tools/llvm-objcopy/ELF/ELFObjcopy.h
index 43f41c00ce5b12a3498d59db1481c5f2c95974ea..a7fe37d00505998bfe25052a727641249e37c931 100644
--- a/tools/llvm-objcopy/ELF/ELFObjcopy.h
+++ b/tools/llvm-objcopy/ELF/ELFObjcopy.h
@@ -1,9 +1,8 @@
 //===- ELFObjcopy.h ---------------------------------------------*- C++ -*-===//
 //
-//                      The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -11,6 +10,7 @@
 #define LLVM_TOOLS_OBJCOPY_ELFOBJCOPY_H
 
 namespace llvm {
+class Error;
 class MemoryBuffer;
 
 namespace object {
@@ -22,10 +22,10 @@ struct CopyConfig;
 class Buffer;
 
 namespace elf {
-void executeObjcopyOnRawBinary(const CopyConfig &Config, MemoryBuffer &In,
-                               Buffer &Out);
-void executeObjcopyOnBinary(const CopyConfig &Config,
-                            object::ELFObjectFileBase &In, Buffer &Out);
+Error executeObjcopyOnRawBinary(const CopyConfig &Config, MemoryBuffer &In,
+                                Buffer &Out);
+Error executeObjcopyOnBinary(const CopyConfig &Config,
+                             object::ELFObjectFileBase &In, Buffer &Out);
 
 } // end namespace elf
 } // end namespace objcopy
diff --git a/tools/llvm-objcopy/ELF/Object.cpp b/tools/llvm-objcopy/ELF/Object.cpp
index ae02966b730841acf5161d969085c0f462f4a521..8a065d5c109b54873ab04d8fd0a4f6de38eb3864 100644
--- a/tools/llvm-objcopy/ELF/Object.cpp
+++ b/tools/llvm-objcopy/ELF/Object.cpp
@@ -1,9 +1,8 @@
 //===- Object.cpp ---------------------------------------------------------===//
 //
-//                      The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -18,6 +17,7 @@
 #include "llvm/MC/MCTargetOptions.h"
 #include "llvm/Object/ELFObjectFile.h"
 #include "llvm/Support/Compression.h"
+#include "llvm/Support/Errc.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/FileOutputBuffer.h"
 #include "llvm/Support/Path.h"
@@ -49,8 +49,14 @@ template <class ELFT> void ELFWriter<ELFT>::writePhdr(const Segment &Seg) {
   Phdr.p_align = Seg.Align;
 }
 
-void SectionBase::removeSectionReferences(const SectionBase *Sec) {}
-void SectionBase::removeSymbols(function_ref<bool(const Symbol &)> ToRemove) {}
+Error SectionBase::removeSectionReferences(const SectionBase *Sec) {
+  return Error::success();
+}
+
+Error SectionBase::removeSymbols(function_ref<bool(const Symbol &)> ToRemove) {
+  return Error::success();
+}
+
 void SectionBase::initialize(SectionTableRef SecTable) {}
 void SectionBase::finalize() {}
 void SectionBase::markSymbols() {}
@@ -71,7 +77,46 @@ template <class ELFT> void ELFWriter<ELFT>::writeShdr(const SectionBase &Sec) {
   Shdr.sh_entsize = Sec.EntrySize;
 }
 
-SectionVisitor::~SectionVisitor() {}
+template <class ELFT> void ELFSectionSizer<ELFT>::visit(Section &Sec) {}
+
+template <class ELFT>
+void ELFSectionSizer<ELFT>::visit(OwnedDataSection &Sec) {}
+
+template <class ELFT>
+void ELFSectionSizer<ELFT>::visit(StringTableSection &Sec) {}
+
+template <class ELFT>
+void ELFSectionSizer<ELFT>::visit(DynamicRelocationSection &Sec) {}
+
+template <class ELFT>
+void ELFSectionSizer<ELFT>::visit(SymbolTableSection &Sec) {
+  Sec.EntrySize = sizeof(Elf_Sym);
+  Sec.Size = Sec.Symbols.size() * Sec.EntrySize;
+  // Align to the largest field in Elf_Sym.
+  Sec.Align = ELFT::Is64Bits ? sizeof(Elf_Xword) : sizeof(Elf_Word);
+}
+
+template <class ELFT>
+void ELFSectionSizer<ELFT>::visit(RelocationSection &Sec) {
+  Sec.EntrySize = Sec.Type == SHT_REL ? sizeof(Elf_Rel) : sizeof(Elf_Rela);
+  Sec.Size = Sec.Relocations.size() * Sec.EntrySize;
+  // Align to the largest field in Elf_Rel(a).
+  Sec.Align = ELFT::Is64Bits ? sizeof(Elf_Xword) : sizeof(Elf_Word);
+}
+
+template <class ELFT>
+void ELFSectionSizer<ELFT>::visit(GnuDebugLinkSection &Sec) {}
+
+template <class ELFT> void ELFSectionSizer<ELFT>::visit(GroupSection &Sec) {}
+
+template <class ELFT>
+void ELFSectionSizer<ELFT>::visit(SectionIndexSection &Sec) {}
+
+template <class ELFT>
+void ELFSectionSizer<ELFT>::visit(CompressedSection &Sec) {}
+
+template <class ELFT>
+void ELFSectionSizer<ELFT>::visit(DecompressedSection &Sec) {}
 
 void BinarySectionWriter::visit(const SectionIndexSection &Sec) {
   error("Cannot write symbol section index table '" + Sec.Name + "' ");
@@ -102,6 +147,8 @@ void SectionWriter::visit(const Section &Sec) {
 
 void Section::accept(SectionVisitor &Visitor) const { Visitor.visit(*this); }
 
+void Section::accept(MutableSectionVisitor &Visitor) { Visitor.visit(*this); }
+
 void SectionWriter::visit(const OwnedDataSection &Sec) {
   uint8_t *Buf = Out.getBufferStart() + Sec.Offset;
   llvm::copy(Sec.Data, Buf);
@@ -164,10 +211,18 @@ void DecompressedSection::accept(SectionVisitor &Visitor) const {
   Visitor.visit(*this);
 }
 
+void DecompressedSection::accept(MutableSectionVisitor &Visitor) {
+  Visitor.visit(*this);
+}
+
 void OwnedDataSection::accept(SectionVisitor &Visitor) const {
   Visitor.visit(*this);
 }
 
+void OwnedDataSection::accept(MutableSectionVisitor &Visitor) {
+  Visitor.visit(*this);
+}
+
 void BinarySectionWriter::visit(const CompressedSection &Sec) {
   error("Cannot write compressed section '" + Sec.Name + "' ");
 }
@@ -246,6 +301,10 @@ void CompressedSection::accept(SectionVisitor &Visitor) const {
   Visitor.visit(*this);
 }
 
+void CompressedSection::accept(MutableSectionVisitor &Visitor) {
+  Visitor.visit(*this);
+}
+
 void StringTableSection::addString(StringRef Name) {
   StrTabBuilder.add(Name);
   Size = StrTabBuilder.getSize();
@@ -265,6 +324,10 @@ void StringTableSection::accept(SectionVisitor &Visitor) const {
   Visitor.visit(*this);
 }
 
+void StringTableSection::accept(MutableSectionVisitor &Visitor) {
+  Visitor.visit(*this);
+}
+
 template <class ELFT>
 void ELFSectionWriter<ELFT>::visit(const SectionIndexSection &Sec) {
   uint8_t *Buf = Out.getBufferStart() + Sec.Offset;
@@ -288,6 +351,10 @@ void SectionIndexSection::accept(SectionVisitor &Visitor) const {
   Visitor.visit(*this);
 }
 
+void SectionIndexSection::accept(MutableSectionVisitor &Visitor) {
+  Visitor.visit(*this);
+}
+
 static bool isValidReservedSectionIndex(uint16_t Index, uint16_t Machine) {
   switch (Index) {
   case SHN_ABS:
@@ -365,15 +432,17 @@ void SymbolTableSection::addSymbol(Twine Name, uint8_t Bind, uint8_t Type,
   Size += this->EntrySize;
 }
 
-void SymbolTableSection::removeSectionReferences(const SectionBase *Sec) {
+Error SymbolTableSection::removeSectionReferences(const SectionBase *Sec) {
   if (SectionIndexTable == Sec)
     SectionIndexTable = nullptr;
   if (SymbolNames == Sec) {
-    error("String table " + SymbolNames->Name +
-          " cannot be removed because it is referenced by the symbol table " +
-          this->Name);
+    return createStringError(llvm::errc::invalid_argument,
+                             "String table %s cannot be removed because it is "
+                             "referenced by the symbol table %s",
+                             SymbolNames->Name.data(), this->Name.data());
   }
-  removeSymbols([Sec](const Symbol &Sym) { return Sym.DefinedIn == Sec; });
+  return removeSymbols(
+      [Sec](const Symbol &Sym) { return Sym.DefinedIn == Sec; });
 }
 
 void SymbolTableSection::updateSymbols(function_ref<void(Symbol &)> Callable) {
@@ -385,7 +454,7 @@ void SymbolTableSection::updateSymbols(function_ref<void(Symbol &)> Callable) {
   assignIndices();
 }
 
-void SymbolTableSection::removeSymbols(
+Error SymbolTableSection::removeSymbols(
     function_ref<bool(const Symbol &)> ToRemove) {
   Symbols.erase(
       std::remove_if(std::begin(Symbols) + 1, std::end(Symbols),
@@ -393,6 +462,7 @@ void SymbolTableSection::removeSymbols(
       std::end(Symbols));
   Size = Symbols.size() * EntrySize;
   assignIndices();
+  return Error::success();
 }
 
 void SymbolTableSection::initialize(SectionTableRef SecTable) {
@@ -470,16 +540,19 @@ void SymbolTableSection::accept(SectionVisitor &Visitor) const {
   Visitor.visit(*this);
 }
 
+void SymbolTableSection::accept(MutableSectionVisitor &Visitor) {
+  Visitor.visit(*this);
+}
+
 template <class SymTabType>
-void RelocSectionWithSymtabBase<SymTabType>::removeSectionReferences(
+Error RelocSectionWithSymtabBase<SymTabType>::removeSectionReferences(
     const SectionBase *Sec) {
-  if (Symbols == Sec) {
-    error("Symbol table " + Symbols->Name +
-          " cannot be removed because it is "
-          "referenced by the relocation "
-          "section " +
-          this->Name);
-  }
+  if (Symbols == Sec)
+    return createStringError(llvm::errc::invalid_argument,
+                             "Symbol table %s cannot be removed because it is "
+                             "referenced by the relocation section %s.",
+                             Symbols->Name.data(), this->Name.data());
+  return Error::success();
 }
 
 template <class SymTabType>
@@ -540,12 +613,19 @@ void RelocationSection::accept(SectionVisitor &Visitor) const {
   Visitor.visit(*this);
 }
 
-void RelocationSection::removeSymbols(
+void RelocationSection::accept(MutableSectionVisitor &Visitor) {
+  Visitor.visit(*this);
+}
+
+Error RelocationSection::removeSymbols(
     function_ref<bool(const Symbol &)> ToRemove) {
   for (const Relocation &Reloc : Relocations)
     if (ToRemove(*Reloc.RelocSymbol))
-      error("not stripping symbol '" + Reloc.RelocSymbol->Name +
-            "' because it is named in a relocation");
+      return createStringError(
+          llvm::errc::invalid_argument,
+          "not stripping symbol '%s' because it is named in a relocation.",
+          Reloc.RelocSymbol->Name.data());
+  return Error::success();
 }
 
 void RelocationSection::markSymbols() {
@@ -562,13 +642,17 @@ void DynamicRelocationSection::accept(SectionVisitor &Visitor) const {
   Visitor.visit(*this);
 }
 
-void Section::removeSectionReferences(const SectionBase *Sec) {
-  if (LinkSection == Sec) {
-    error("Section " + LinkSection->Name +
-          " cannot be removed because it is "
-          "referenced by the section " +
-          this->Name);
-  }
+void DynamicRelocationSection::accept(MutableSectionVisitor &Visitor) {
+  Visitor.visit(*this);
+}
+
+Error Section::removeSectionReferences(const SectionBase *Sec) {
+  if (LinkSection == Sec)
+    return createStringError(llvm::errc::invalid_argument,
+                             "Section %s cannot be removed because it is "
+                             "referenced by the section %s",
+                             LinkSection->Name.data(), this->Name.data());
+  return Error::success();
 }
 
 void GroupSection::finalize() {
@@ -576,13 +660,13 @@ void GroupSection::finalize() {
   this->Link = SymTab->Index;
 }
 
-void GroupSection::removeSymbols(function_ref<bool(const Symbol &)> ToRemove) {
-  if (ToRemove(*Sym)) {
-    error("Symbol " + Sym->Name +
-          " cannot be removed because it is "
-          "referenced by the section " +
-          this->Name + "[" + Twine(this->Index) + "]");
-  }
+Error GroupSection::removeSymbols(function_ref<bool(const Symbol &)> ToRemove) {
+  if (ToRemove(*Sym))
+    return createStringError(llvm::errc::invalid_argument,
+                             "Symbol %s cannot be removed because it is "
+                             "referenced by the section %s[%d].",
+                             Sym->Name.data(), this->Name.data(), this->Index);
+  return Error::success();
 }
 
 void GroupSection::markSymbols() {
@@ -648,6 +732,10 @@ void GnuDebugLinkSection::accept(SectionVisitor &Visitor) const {
   Visitor.visit(*this);
 }
 
+void GnuDebugLinkSection::accept(MutableSectionVisitor &Visitor) {
+  Visitor.visit(*this);
+}
+
 template <class ELFT>
 void ELFSectionWriter<ELFT>::visit(const GroupSection &Sec) {
   ELF::Elf32_Word *Buf =
@@ -661,6 +749,10 @@ void GroupSection::accept(SectionVisitor &Visitor) const {
   Visitor.visit(*this);
 }
 
+void GroupSection::accept(MutableSectionVisitor &Visitor) {
+  Visitor.visit(*this);
+}
+
 // Returns true IFF a section is wholly inside the range of a segment
 static bool sectionWithinSegment(const SectionBase &Section,
                                  const Segment &Segment) {
@@ -700,19 +792,19 @@ static bool compareSegmentsByPAddr(const Segment *A, const Segment *B) {
   return A->Index < B->Index;
 }
 
-template <class ELFT> void BinaryELFBuilder<ELFT>::initFileHeader() {
+void BinaryELFBuilder::initFileHeader() {
   Obj->Flags = 0x0;
   Obj->Type = ET_REL;
+  Obj->OSABI = ELFOSABI_NONE;
+  Obj->ABIVersion = 0;
   Obj->Entry = 0x0;
   Obj->Machine = EMachine;
   Obj->Version = 1;
 }
 
-template <class ELFT> void BinaryELFBuilder<ELFT>::initHeaderSegment() {
-  Obj->ElfHdrSegment.Index = 0;
-}
+void BinaryELFBuilder::initHeaderSegment() { Obj->ElfHdrSegment.Index = 0; }
 
-template <class ELFT> StringTableSection *BinaryELFBuilder<ELFT>::addStrTab() {
+StringTableSection *BinaryELFBuilder::addStrTab() {
   auto &StrTab = Obj->addSection<StringTableSection>();
   StrTab.Name = ".strtab";
 
@@ -720,15 +812,11 @@ template <class ELFT> StringTableSection *BinaryELFBuilder<ELFT>::addStrTab() {
   return &StrTab;
 }
 
-template <class ELFT>
-SymbolTableSection *
-BinaryELFBuilder<ELFT>::addSymTab(StringTableSection *StrTab) {
+SymbolTableSection *BinaryELFBuilder::addSymTab(StringTableSection *StrTab) {
   auto &SymTab = Obj->addSection<SymbolTableSection>();
 
   SymTab.Name = ".symtab";
   SymTab.Link = StrTab->Index;
-  // TODO: Factor out dependence on ElfType here.
-  SymTab.EntrySize = sizeof(Elf_Sym);
 
   // The symbol table always needs a null symbol
   SymTab.addSymbol("", 0, 0, nullptr, 0, 0, 0, 0);
@@ -737,8 +825,7 @@ BinaryELFBuilder<ELFT>::addSymTab(StringTableSection *StrTab) {
   return &SymTab;
 }
 
-template <class ELFT>
-void BinaryELFBuilder<ELFT>::addData(SymbolTableSection *SymTab) {
+void BinaryELFBuilder::addData(SymbolTableSection *SymTab) {
   auto Data = ArrayRef<uint8_t>(
       reinterpret_cast<const uint8_t *>(MemBuf->getBufferStart()),
       MemBuf->getBufferSize());
@@ -761,13 +848,13 @@ void BinaryELFBuilder<ELFT>::addData(SymbolTableSection *SymTab) {
                     /*Value=*/DataSection.Size, STV_DEFAULT, SHN_ABS, 0);
 }
 
-template <class ELFT> void BinaryELFBuilder<ELFT>::initSections() {
+void BinaryELFBuilder::initSections() {
   for (auto &Section : Obj->sections()) {
     Section.initialize(Obj->sections());
   }
 }
 
-template <class ELFT> std::unique_ptr<Object> BinaryELFBuilder<ELFT>::build() {
+std::unique_ptr<Object> BinaryELFBuilder::build() {
   initFileHeader();
   initHeaderSegment();
   StringTableSection *StrTab = addStrTab();
@@ -1086,6 +1173,8 @@ template <class ELFT> void ELFBuilder<ELFT>::readSectionHeaders() {
 template <class ELFT> void ELFBuilder<ELFT>::build() {
   const auto &Ehdr = *ElfFile.getHeader();
 
+  Obj.OSABI = Ehdr.e_ident[EI_OSABI];
+  Obj.ABIVersion = Ehdr.e_ident[EI_ABIVERSION];
   Obj.Type = Ehdr.e_type;
   Obj.Machine = Ehdr.e_machine;
   Obj.Version = Ehdr.e_version;
@@ -1118,14 +1207,7 @@ Writer::~Writer() {}
 Reader::~Reader() {}
 
 std::unique_ptr<Object> BinaryReader::create() const {
-  if (MInfo.Is64Bit)
-    return MInfo.IsLittleEndian
-               ? BinaryELFBuilder<ELF64LE>(MInfo.EMachine, MemBuf).build()
-               : BinaryELFBuilder<ELF64BE>(MInfo.EMachine, MemBuf).build();
-  else
-    return MInfo.IsLittleEndian
-               ? BinaryELFBuilder<ELF32LE>(MInfo.EMachine, MemBuf).build()
-               : BinaryELFBuilder<ELF32BE>(MInfo.EMachine, MemBuf).build();
+  return BinaryELFBuilder(MInfo.EMachine, MemBuf).build();
 }
 
 std::unique_ptr<Object> ELFReader::create() const {
@@ -1162,8 +1244,8 @@ template <class ELFT> void ELFWriter<ELFT>::writeEhdr() {
   Ehdr.e_ident[EI_DATA] =
       ELFT::TargetEndianness == support::big ? ELFDATA2MSB : ELFDATA2LSB;
   Ehdr.e_ident[EI_VERSION] = EV_CURRENT;
-  Ehdr.e_ident[EI_OSABI] = ELFOSABI_NONE;
-  Ehdr.e_ident[EI_ABIVERSION] = 0;
+  Ehdr.e_ident[EI_OSABI] = Obj.OSABI;
+  Ehdr.e_ident[EI_ABIVERSION] = Obj.ABIVersion;
 
   Ehdr.e_type = Obj.Type;
   Ehdr.e_machine = Obj.Machine;
@@ -1247,7 +1329,8 @@ template <class ELFT> void ELFWriter<ELFT>::writeSectionData() {
     Sec.accept(*SecWriter);
 }
 
-void Object::removeSections(std::function<bool(const SectionBase &)> ToRemove) {
+Error Object::removeSections(
+    std::function<bool(const SectionBase &)> ToRemove) {
 
   auto Iter = std::stable_partition(
       std::begin(Sections), std::end(Sections), [=](const SecPtr &Sec) {
@@ -1272,18 +1355,20 @@ void Object::removeSections(std::function<bool(const SectionBase &)> ToRemove) {
     for (auto &Segment : Segments)
       Segment->removeSection(RemoveSec.get());
     for (auto &KeepSec : make_range(std::begin(Sections), Iter))
-      KeepSec->removeSectionReferences(RemoveSec.get());
+      if (Error E = KeepSec->removeSectionReferences(RemoveSec.get()))
+        return E;
   }
   // Now finally get rid of them all togethor.
   Sections.erase(Iter, std::end(Sections));
+  return Error::success();
 }
 
-void Object::removeSymbols(function_ref<bool(const Symbol &)> ToRemove) {
-  if (!SymbolTable)
-    return;
-
-  for (const SecPtr &Sec : Sections)
-    Sec->removeSymbols(ToRemove);
+Error Object::removeSymbols(function_ref<bool(const Symbol &)> ToRemove) {
+  if (SymbolTable)
+    for (const SecPtr &Sec : Sections)
+      if (Error E = Sec->removeSymbols(ToRemove))
+        return E;
+  return Error::success();
 }
 
 void Object::sortSections() {
@@ -1418,23 +1503,23 @@ template <class ELFT> size_t ELFWriter<ELFT>::totalSize() const {
          NullSectionSize;
 }
 
-template <class ELFT> void ELFWriter<ELFT>::write() {
+template <class ELFT> Error ELFWriter<ELFT>::write() {
   writeEhdr();
   writePhdrs();
   writeSectionData();
   if (WriteSectionHeaders)
     writeShdrs();
-  if (auto E = Buf.commit())
-    reportError(Buf.getName(), errorToErrorCode(std::move(E)));
+  return Buf.commit();
 }
 
-template <class ELFT> void ELFWriter<ELFT>::finalize() {
+template <class ELFT> Error ELFWriter<ELFT>::finalize() {
   // It could happen that SectionNames has been removed and yet the user wants
   // a section header table output. We need to throw an error if a user tries
   // to do that.
   if (Obj.SectionNames == nullptr && WriteSectionHeaders)
-    error("Cannot write section header table because section header string "
-          "table was removed.");
+    return createStringError(llvm::errc::invalid_argument,
+                             "Cannot write section header table because "
+                             "section header string table was removed.");
 
   Obj.sortSections();
 
@@ -1465,9 +1550,10 @@ template <class ELFT> void ELFWriter<ELFT>::finalize() {
     // Since we don't need SectionIndexTable we should remove it and all
     // references to it.
     if (Obj.SectionIndexTable != nullptr) {
-      Obj.removeSections([this](const SectionBase &Sec) {
-        return &Sec == Obj.SectionIndexTable;
-      });
+      if (Error E = Obj.removeSections([this](const SectionBase &Sec) {
+            return &Sec == Obj.SectionIndexTable;
+          }))
+        return E;
     }
   }
 
@@ -1479,10 +1565,16 @@ template <class ELFT> void ELFWriter<ELFT>::finalize() {
     }
 
   initEhdrSegment();
+
   // Before we can prepare for layout the indexes need to be finalized.
+  // Also, the output arch may not be the same as the input arch, so fix up
+  // size-related fields before doing layout calculations.
   uint64_t Index = 0;
-  for (auto &Sec : Obj.sections())
+  auto SecSizer = llvm::make_unique<ELFSectionSizer<ELFT>>();
+  for (auto &Sec : Obj.sections()) {
     Sec.Index = Index++;
+    Sec.accept(*SecSizer);
+  }
 
   // The symbol table does not update all other sections on update. For
   // instance, symbol names are not added as new symbols are added. This means
@@ -1506,21 +1598,22 @@ template <class ELFT> void ELFWriter<ELFT>::finalize() {
     Section.finalize();
   }
 
-  Buf.allocate(totalSize());
+  if (Error E = Buf.allocate(totalSize()))
+    return E;
   SecWriter = llvm::make_unique<ELFSectionWriter<ELFT>>(Buf);
+  return Error::success();
 }
 
-void BinaryWriter::write() {
+Error BinaryWriter::write() {
   for (auto &Section : Obj.sections()) {
     if ((Section.Flags & SHF_ALLOC) == 0)
       continue;
     Section.accept(*SecWriter);
   }
-  if (auto E = Buf.commit())
-    reportError(Buf.getName(), errorToErrorCode(std::move(E)));
+  return Buf.commit();
 }
 
-void BinaryWriter::finalize() {
+Error BinaryWriter::finalize() {
   // TODO: Create a filter range to construct OrderedSegments from so that this
   // code can be deduped with assignOffsets above. This should also solve the
   // todo below for LayoutSections.
@@ -1599,15 +1692,12 @@ void BinaryWriter::finalize() {
       TotalSize = std::max(TotalSize, Section->Offset + Section->Size);
   }
 
-  Buf.allocate(TotalSize);
+  if (Error E = Buf.allocate(TotalSize))
+    return E;
   SecWriter = llvm::make_unique<BinarySectionWriter>(Buf);
+  return Error::success();
 }
 
-template class BinaryELFBuilder<ELF64LE>;
-template class BinaryELFBuilder<ELF64BE>;
-template class BinaryELFBuilder<ELF32LE>;
-template class BinaryELFBuilder<ELF32BE>;
-
 template class ELFBuilder<ELF64LE>;
 template class ELFBuilder<ELF64BE>;
 template class ELFBuilder<ELF32LE>;
diff --git a/tools/llvm-objcopy/ELF/Object.h b/tools/llvm-objcopy/ELF/Object.h
index 4b8406511066958158699836816ff729c98f3fa1..0953eb7e184c76a7d4cd44b5ffdb89be6d806b9b 100644
--- a/tools/llvm-objcopy/ELF/Object.h
+++ b/tools/llvm-objcopy/ELF/Object.h
@@ -1,9 +1,8 @@
 //===- Object.h -------------------------------------------------*- C++ -*-===//
 //
-//                      The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -71,7 +70,7 @@ enum ElfType { ELFT_ELF32LE, ELFT_ELF64LE, ELFT_ELF32BE, ELFT_ELF64BE };
 
 class SectionVisitor {
 public:
-  virtual ~SectionVisitor();
+  virtual ~SectionVisitor() = default;
 
   virtual void visit(const Section &Sec) = 0;
   virtual void visit(const OwnedDataSection &Sec) = 0;
@@ -86,6 +85,23 @@ public:
   virtual void visit(const DecompressedSection &Sec) = 0;
 };
 
+class MutableSectionVisitor {
+public:
+  virtual ~MutableSectionVisitor() = default;
+
+  virtual void visit(Section &Sec) = 0;
+  virtual void visit(OwnedDataSection &Sec) = 0;
+  virtual void visit(StringTableSection &Sec) = 0;
+  virtual void visit(SymbolTableSection &Sec) = 0;
+  virtual void visit(RelocationSection &Sec) = 0;
+  virtual void visit(DynamicRelocationSection &Sec) = 0;
+  virtual void visit(GnuDebugLinkSection &Sec) = 0;
+  virtual void visit(GroupSection &Sec) = 0;
+  virtual void visit(SectionIndexSection &Sec) = 0;
+  virtual void visit(CompressedSection &Sec) = 0;
+  virtual void visit(DecompressedSection &Sec) = 0;
+};
+
 class SectionWriter : public SectionVisitor {
 protected:
   Buffer &Out;
@@ -128,9 +144,32 @@ public:
   explicit ELFSectionWriter(Buffer &Buf) : SectionWriter(Buf) {}
 };
 
+template <class ELFT> class ELFSectionSizer : public MutableSectionVisitor {
+private:
+  using Elf_Rel = typename ELFT::Rel;
+  using Elf_Rela = typename ELFT::Rela;
+  using Elf_Sym = typename ELFT::Sym;
+  using Elf_Word = typename ELFT::Word;
+  using Elf_Xword = typename ELFT::Xword;
+
+public:
+  void visit(Section &Sec) override;
+  void visit(OwnedDataSection &Sec) override;
+  void visit(StringTableSection &Sec) override;
+  void visit(DynamicRelocationSection &Sec) override;
+  void visit(SymbolTableSection &Sec) override;
+  void visit(RelocationSection &Sec) override;
+  void visit(GnuDebugLinkSection &Sec) override;
+  void visit(GroupSection &Sec) override;
+  void visit(SectionIndexSection &Sec) override;
+  void visit(CompressedSection &Sec) override;
+  void visit(DecompressedSection &Sec) override;
+};
+
 #define MAKE_SEC_WRITER_FRIEND                                                 \
   friend class SectionWriter;                                                  \
-  template <class ELFT> friend class ELFSectionWriter;
+  template <class ELFT> friend class ELFSectionWriter;                         \
+  template <class ELFT> friend class ELFSectionSizer;
 
 class BinarySectionWriter : public SectionWriter {
 public:
@@ -154,8 +193,8 @@ protected:
 
 public:
   virtual ~Writer();
-  virtual void finalize() = 0;
-  virtual void write() = 0;
+  virtual Error finalize() = 0;
+  virtual Error write() = 0;
 
   Writer(Object &O, Buffer &B) : Obj(O), Buf(B) {}
 };
@@ -187,8 +226,8 @@ public:
   virtual ~ELFWriter() {}
   bool WriteSectionHeaders = true;
 
-  void finalize() override;
-  void write() override;
+  Error finalize() override;
+  Error write() override;
   ELFWriter(Object &Obj, Buffer &Buf, bool WSH)
       : Writer(Obj, Buf), WriteSectionHeaders(WSH) {}
 };
@@ -201,8 +240,8 @@ private:
 
 public:
   ~BinaryWriter() {}
-  void finalize() override;
-  void write() override;
+  Error finalize() override;
+  Error write() override;
   BinaryWriter(Object &Obj, Buffer &Buf) : Writer(Obj, Buf) {}
 };
 
@@ -234,9 +273,10 @@ public:
 
   virtual void initialize(SectionTableRef SecTable);
   virtual void finalize();
-  virtual void removeSectionReferences(const SectionBase *Sec);
-  virtual void removeSymbols(function_ref<bool(const Symbol &)> ToRemove);
+  virtual Error removeSectionReferences(const SectionBase *Sec);
+  virtual Error removeSymbols(function_ref<bool(const Symbol &)> ToRemove);
   virtual void accept(SectionVisitor &Visitor) const = 0;
+  virtual void accept(MutableSectionVisitor &Visitor) = 0;
   virtual void markSymbols();
 };
 
@@ -293,7 +333,8 @@ public:
   explicit Section(ArrayRef<uint8_t> Data) : Contents(Data) {}
 
   void accept(SectionVisitor &Visitor) const override;
-  void removeSectionReferences(const SectionBase *Sec) override;
+  void accept(MutableSectionVisitor &Visitor) override;
+  Error removeSectionReferences(const SectionBase *Sec) override;
   void initialize(SectionTableRef SecTable) override;
   void finalize() override;
 };
@@ -313,6 +354,7 @@ public:
   }
 
   void accept(SectionVisitor &Sec) const override;
+  void accept(MutableSectionVisitor &Visitor) override;
 };
 
 class CompressedSection : public SectionBase {
@@ -333,6 +375,7 @@ public:
   uint64_t getDecompressedAlign() const { return DecompressedAlign; }
 
   void accept(SectionVisitor &Visitor) const override;
+  void accept(MutableSectionVisitor &Visitor) override;
 
   static bool classof(const SectionBase *S) {
     return (S->Flags & ELF::SHF_COMPRESSED) ||
@@ -354,6 +397,7 @@ public:
   }
 
   void accept(SectionVisitor &Visitor) const override;
+  void accept(MutableSectionVisitor &Visitor) override;
 };
 
 // There are two types of string tables that can exist, dynamic and not dynamic.
@@ -378,6 +422,7 @@ public:
   uint32_t findIndex(StringRef Name) const;
   void finalize() override;
   void accept(SectionVisitor &Visitor) const override;
+  void accept(MutableSectionVisitor &Visitor) override;
 
   static bool classof(const SectionBase *S) {
     if (S->Flags & ELF::SHF_ALLOC)
@@ -435,6 +480,7 @@ public:
   void initialize(SectionTableRef SecTable) override;
   void finalize() override;
   void accept(SectionVisitor &Visitor) const override;
+  void accept(MutableSectionVisitor &Visitor) override;
 
   SectionIndexSection() {
     Name = ".symtab_shndx";
@@ -475,11 +521,12 @@ public:
   Symbol *getSymbolByIndex(uint32_t Index);
   void updateSymbols(function_ref<void(Symbol &)> Callable);
 
-  void removeSectionReferences(const SectionBase *Sec) override;
+  Error removeSectionReferences(const SectionBase *Sec) override;
   void initialize(SectionTableRef SecTable) override;
   void finalize() override;
   void accept(SectionVisitor &Visitor) const override;
-  void removeSymbols(function_ref<bool(const Symbol &)> ToRemove) override;
+  void accept(MutableSectionVisitor &Visitor) override;
+  Error removeSymbols(function_ref<bool(const Symbol &)> ToRemove) override;
 
   static bool classof(const SectionBase *S) {
     return S->Type == ELF::SHT_SYMTAB;
@@ -526,7 +573,7 @@ protected:
   RelocSectionWithSymtabBase() = default;
 
 public:
-  void removeSectionReferences(const SectionBase *Sec) override;
+  Error removeSectionReferences(const SectionBase *Sec) override;
   void initialize(SectionTableRef SecTable) override;
   void finalize() override;
 };
@@ -540,7 +587,8 @@ class RelocationSection
 public:
   void addRelocation(Relocation Rel) { Relocations.push_back(Rel); }
   void accept(SectionVisitor &Visitor) const override;
-  void removeSymbols(function_ref<bool(const Symbol &)> ToRemove) override;
+  void accept(MutableSectionVisitor &Visitor) override;
+  Error removeSymbols(function_ref<bool(const Symbol &)> ToRemove) override;
   void markSymbols() override;
 
   static bool classof(const SectionBase *S) {
@@ -573,8 +621,9 @@ public:
   void addMember(SectionBase *Sec) { GroupMembers.push_back(Sec); }
 
   void accept(SectionVisitor &) const override;
+  void accept(MutableSectionVisitor &Visitor) override;
   void finalize() override;
-  void removeSymbols(function_ref<bool(const Symbol &)> ToRemove) override;
+  Error removeSymbols(function_ref<bool(const Symbol &)> ToRemove) override;
   void markSymbols() override;
 
   static bool classof(const SectionBase *S) {
@@ -611,6 +660,7 @@ public:
   explicit DynamicRelocationSection(ArrayRef<uint8_t> Data) : Contents(Data) {}
 
   void accept(SectionVisitor &) const override;
+  void accept(MutableSectionVisitor &Visitor) override;
 
   static bool classof(const SectionBase *S) {
     if (!(S->Flags & ELF::SHF_ALLOC))
@@ -632,6 +682,7 @@ public:
   // If we add this section from an external source we can use this ctor.
   explicit GnuDebugLinkSection(StringRef File);
   void accept(SectionVisitor &Visitor) const override;
+  void accept(MutableSectionVisitor &Visitor) override;
 };
 
 class Reader {
@@ -645,9 +696,7 @@ using object::ELFFile;
 using object::ELFObjectFile;
 using object::OwningBinary;
 
-template <class ELFT> class BinaryELFBuilder {
-  using Elf_Sym = typename ELFT::Sym;
-
+class BinaryELFBuilder {
   uint16_t EMachine;
   MemoryBuffer *MemBuf;
   std::unique_ptr<Object> Obj;
@@ -733,6 +782,8 @@ public:
   Segment ElfHdrSegment;
   Segment ProgramHdrSegment;
 
+  uint8_t OSABI;
+  uint8_t ABIVersion;
   uint64_t Entry;
   uint64_t SHOffset;
   uint32_t Type;
@@ -752,8 +803,8 @@ public:
   Range<Segment> segments() { return make_pointee_range(Segments); }
   ConstRange<Segment> segments() const { return make_pointee_range(Segments); }
 
-  void removeSections(std::function<bool(const SectionBase &)> ToRemove);
-  void removeSymbols(function_ref<bool(const Symbol &)> ToRemove);
+  Error removeSections(std::function<bool(const SectionBase &)> ToRemove);
+  Error removeSymbols(function_ref<bool(const Symbol &)> ToRemove);
   template <class T, class... Ts> T &addSection(Ts &&... Args) {
     auto Sec = llvm::make_unique<T>(std::forward<Ts>(Args)...);
     auto Ptr = Sec.get();
diff --git a/tools/llvm-objcopy/LLVMBuild.txt b/tools/llvm-objcopy/LLVMBuild.txt
index a26e13932b2103638b3ef21a47b89ce9dd1b6b54..8951715c3647e67aeb97f7b31a1bdcb1ea072af6 100644
--- a/tools/llvm-objcopy/LLVMBuild.txt
+++ b/tools/llvm-objcopy/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./tools/llvm-objcopy/LLVMBuild.txt -----------------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/tools/llvm-objcopy/MachO/MachOObjcopy.cpp b/tools/llvm-objcopy/MachO/MachOObjcopy.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..8cf1d10943a6c2c32f6e362a6209bdf214faaf71
--- /dev/null
+++ b/tools/llvm-objcopy/MachO/MachOObjcopy.cpp
@@ -0,0 +1,31 @@
+//===- MachOObjcopy.cpp -----------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "MachOObjcopy.h"
+#include "../CopyConfig.h"
+#include "../llvm-objcopy.h"
+#include "MachOReader.h"
+#include "MachOWriter.h"
+#include "llvm/Support/Error.h"
+
+namespace llvm {
+namespace objcopy {
+namespace macho {
+
+Error executeObjcopyOnBinary(const CopyConfig &Config,
+                             object::MachOObjectFile &In, Buffer &Out) {
+  MachOReader Reader(In);
+  std::unique_ptr<Object> O = Reader.create();
+  assert(O && "Unable to deserialize MachO object");
+  MachOWriter Writer(*O, In.is64Bit(), In.isLittleEndian(), Out);
+  return Writer.write();
+}
+
+} // end namespace macho
+} // end namespace objcopy
+} // end namespace llvm
diff --git a/tools/llvm-objcopy/MachO/MachOObjcopy.h b/tools/llvm-objcopy/MachO/MachOObjcopy.h
new file mode 100644
index 0000000000000000000000000000000000000000..f34e361db7ea94195a8dd1ad2b1555b90e1c086d
--- /dev/null
+++ b/tools/llvm-objcopy/MachO/MachOObjcopy.h
@@ -0,0 +1,31 @@
+//===- MachOObjcopy.h -------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_TOOLS_OBJCOPY_MACHOOBJCOPY_H
+#define LLVM_TOOLS_OBJCOPY_MACHOOBJCOPY_H
+
+namespace llvm {
+class Error;
+
+namespace object {
+class MachOObjectFile;
+class MachOUniversalBinary;
+} // end namespace object
+
+namespace objcopy {
+struct CopyConfig;
+class Buffer;
+
+namespace macho {
+Error executeObjcopyOnBinary(const CopyConfig &Config,
+                             object::MachOObjectFile &In, Buffer &Out);
+} // end namespace macho
+} // end namespace objcopy
+} // end namespace llvm
+
+#endif // LLVM_TOOLS_OBJCOPY_MACHOOBJCOPY_H
diff --git a/tools/llvm-objcopy/MachO/MachOReader.cpp b/tools/llvm-objcopy/MachO/MachOReader.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..39702cda4ebc8cdabffd40c4a81ddb91819521a9
--- /dev/null
+++ b/tools/llvm-objcopy/MachO/MachOReader.cpp
@@ -0,0 +1,224 @@
+//===- MachOReader.cpp ------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "MachOReader.h"
+#include "../llvm-objcopy.h"
+#include "Object.h"
+#include "llvm/BinaryFormat/MachO.h"
+#include "llvm/Object/MachO.h"
+#include <memory>
+
+namespace llvm {
+namespace objcopy {
+namespace macho {
+
+void MachOReader::readHeader(Object &O) const {
+  O.Header.Magic = MachOObj.getHeader().magic;
+  O.Header.CPUType = MachOObj.getHeader().cputype;
+  O.Header.CPUSubType = MachOObj.getHeader().cpusubtype;
+  O.Header.FileType = MachOObj.getHeader().filetype;
+  O.Header.NCmds = MachOObj.getHeader().ncmds;
+  O.Header.SizeOfCmds = MachOObj.getHeader().sizeofcmds;
+  O.Header.Flags = MachOObj.getHeader().flags;
+}
+
+template <typename SectionType>
+Section constructSectionCommon(SectionType Sec) {
+  Section S;
+  memcpy(S.Sectname, Sec.sectname, sizeof(Sec.sectname));
+  memcpy(S.Segname, Sec.segname, sizeof(Sec.segname));
+  S.Addr = Sec.addr;
+  S.Size = Sec.size;
+  S.Offset = Sec.offset;
+  S.Align = Sec.align;
+  S.RelOff = Sec.reloff;
+  S.NReloc = Sec.nreloc;
+  S.Flags = Sec.flags;
+  S.Reserved1 = Sec.reserved1;
+  S.Reserved2 = Sec.reserved2;
+  S.Reserved3 = 0;
+  return S;
+}
+
+template <typename SectionType> Section constructSection(SectionType Sec);
+
+template <> Section constructSection(MachO::section Sec) {
+  return constructSectionCommon(Sec);
+}
+
+template <> Section constructSection(MachO::section_64 Sec) {
+  Section S = constructSectionCommon(Sec);
+  S.Reserved3 = Sec.reserved3;
+  return S;
+}
+
+// TODO: get rid of reportError and make MachOReader return Expected<> instead.
+template <typename SectionType, typename SegmentType>
+std::vector<Section>
+extractSections(const object::MachOObjectFile::LoadCommandInfo &LoadCmd,
+                const object::MachOObjectFile &MachOObj,
+                size_t &NextSectionIndex) {
+  auto End = LoadCmd.Ptr + LoadCmd.C.cmdsize;
+  const SectionType *Curr =
+      reinterpret_cast<const SectionType *>(LoadCmd.Ptr + sizeof(SegmentType));
+  std::vector<Section> Sections;
+  for (; reinterpret_cast<const void *>(Curr) < End; Curr++) {
+    if (MachOObj.isLittleEndian() != sys::IsLittleEndianHost) {
+      SectionType Sec;
+      memcpy((void *)&Sec, Curr, sizeof(SectionType));
+      MachO::swapStruct(Sec);
+      Sections.push_back(constructSection(Sec));
+    } else {
+      Sections.push_back(constructSection(*Curr));
+    }
+
+    Section &S = Sections.back();
+
+    StringRef SectName(S.Sectname);
+    Expected<object::SectionRef> SecRef =
+        MachOObj.getSection(NextSectionIndex++);
+    if (!SecRef)
+      reportError(MachOObj.getFileName(), SecRef.takeError());
+
+    StringRef Content;
+    if (auto EC =
+            MachOObj.getSectionContents(SecRef->getRawDataRefImpl(), Content))
+      reportError(MachOObj.getFileName(), std::move(EC));
+    S.Content = Content;
+
+    S.Relocations.reserve(S.NReloc);
+    for (auto RI = MachOObj.section_rel_begin(SecRef->getRawDataRefImpl()),
+              RE = MachOObj.section_rel_end(SecRef->getRawDataRefImpl());
+         RI != RE; ++RI)
+      S.Relocations.push_back(MachOObj.getRelocation(RI->getRawDataRefImpl()));
+    assert(S.NReloc == S.Relocations.size() &&
+           "Incorrect number of relocations");
+  }
+  return Sections;
+}
+
+void MachOReader::readLoadCommands(Object &O) const {
+  // For MachO sections indices start from 1.
+  size_t NextSectionIndex = 1;
+  for (auto LoadCmd : MachOObj.load_commands()) {
+    LoadCommand LC;
+    switch (LoadCmd.C.cmd) {
+    case MachO::LC_SEGMENT:
+      LC.Sections = extractSections<MachO::section, MachO::segment_command>(
+          LoadCmd, MachOObj, NextSectionIndex);
+      break;
+    case MachO::LC_SEGMENT_64:
+      LC.Sections =
+          extractSections<MachO::section_64, MachO::segment_command_64>(
+              LoadCmd, MachOObj, NextSectionIndex);
+      break;
+    case MachO::LC_SYMTAB:
+      O.SymTabCommandIndex = O.LoadCommands.size();
+      break;
+    case MachO::LC_DYLD_INFO:
+    case MachO::LC_DYLD_INFO_ONLY:
+      O.DyLdInfoCommandIndex = O.LoadCommands.size();
+      break;
+    }
+#define HANDLE_LOAD_COMMAND(LCName, LCValue, LCStruct)                         \
+  case MachO::LCName:                                                          \
+    memcpy((void *)&(LC.MachOLoadCommand.LCStruct##_data), LoadCmd.Ptr,        \
+           sizeof(MachO::LCStruct));                                           \
+    if (MachOObj.isLittleEndian() != sys::IsLittleEndianHost)                  \
+      MachO::swapStruct(LC.MachOLoadCommand.LCStruct##_data);                  \
+    LC.Payload = ArrayRef<uint8_t>(                                            \
+        reinterpret_cast<uint8_t *>(const_cast<char *>(LoadCmd.Ptr)) +         \
+            sizeof(MachO::LCStruct),                                           \
+        LoadCmd.C.cmdsize - sizeof(MachO::LCStruct));                          \
+    break;
+
+    switch (LoadCmd.C.cmd) {
+    default:
+      memcpy((void *)&(LC.MachOLoadCommand.load_command_data), LoadCmd.Ptr,
+             sizeof(MachO::load_command));
+      if (MachOObj.isLittleEndian() != sys::IsLittleEndianHost)
+        MachO::swapStruct(LC.MachOLoadCommand.load_command_data);
+      LC.Payload = ArrayRef<uint8_t>(
+          reinterpret_cast<uint8_t *>(const_cast<char *>(LoadCmd.Ptr)) +
+              sizeof(MachO::load_command),
+          LoadCmd.C.cmdsize - sizeof(MachO::load_command));
+      break;
+#include "llvm/BinaryFormat/MachO.def"
+    }
+    O.LoadCommands.push_back(std::move(LC));
+  }
+}
+
+template <typename nlist_t> NListEntry constructNameList(const nlist_t &nlist) {
+  NListEntry NL;
+  NL.n_strx = nlist.n_strx;
+  NL.n_type = nlist.n_type;
+  NL.n_sect = nlist.n_sect;
+  NL.n_desc = nlist.n_desc;
+  NL.n_value = nlist.n_value;
+  return NL;
+}
+
+void MachOReader::readSymbolTable(Object &O) const {
+  for (auto Symbol : MachOObj.symbols()) {
+    NListEntry NLE =
+        MachOObj.is64Bit()
+            ? constructNameList<MachO::nlist_64>(
+                  MachOObj.getSymbol64TableEntry(Symbol.getRawDataRefImpl()))
+            : constructNameList<MachO::nlist>(
+                  MachOObj.getSymbolTableEntry(Symbol.getRawDataRefImpl()));
+    O.SymTable.NameList.push_back(NLE);
+  }
+}
+
+void MachOReader::readStringTable(Object &O) const {
+  StringRef Data = MachOObj.getStringTableData();
+  SmallVector<StringRef, 10> Strs;
+  Data.split(Strs, '\0');
+  O.StrTable.Strings.reserve(Strs.size());
+  for (auto S : Strs)
+    O.StrTable.Strings.push_back(S.str());
+}
+
+void MachOReader::readRebaseInfo(Object &O) const {
+  O.Rebases.Opcodes = MachOObj.getDyldInfoRebaseOpcodes();
+}
+
+void MachOReader::readBindInfo(Object &O) const {
+  O.Binds.Opcodes = MachOObj.getDyldInfoBindOpcodes();
+}
+
+void MachOReader::readWeakBindInfo(Object &O) const {
+  O.WeakBinds.Opcodes = MachOObj.getDyldInfoWeakBindOpcodes();
+}
+
+void MachOReader::readLazyBindInfo(Object &O) const {
+  O.LazyBinds.Opcodes = MachOObj.getDyldInfoLazyBindOpcodes();
+}
+
+void MachOReader::readExportInfo(Object &O) const {
+  O.Exports.Trie = MachOObj.getDyldInfoExportsTrie();
+}
+
+std::unique_ptr<Object> MachOReader::create() const {
+  auto Obj = llvm::make_unique<Object>();
+  readHeader(*Obj);
+  readLoadCommands(*Obj);
+  readSymbolTable(*Obj);
+  readStringTable(*Obj);
+  readRebaseInfo(*Obj);
+  readBindInfo(*Obj);
+  readWeakBindInfo(*Obj);
+  readLazyBindInfo(*Obj);
+  readExportInfo(*Obj);
+  return Obj;
+}
+
+} // end namespace macho
+} // end namespace objcopy
+} // end namespace llvm
diff --git a/tools/llvm-objcopy/MachO/MachOReader.h b/tools/llvm-objcopy/MachO/MachOReader.h
new file mode 100644
index 0000000000000000000000000000000000000000..cad3cdf8e6220889d647eeb17b65cafa852c32f0
--- /dev/null
+++ b/tools/llvm-objcopy/MachO/MachOReader.h
@@ -0,0 +1,48 @@
+//===- MachOReader.h --------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "MachOObjcopy.h"
+#include "Object.h"
+#include "llvm/BinaryFormat/MachO.h"
+#include "llvm/Object/MachO.h"
+#include <memory>
+
+namespace llvm {
+namespace objcopy {
+namespace macho {
+
+// The hierarchy of readers is responsible for parsing different inputs:
+// raw binaries and regular MachO object files.
+class Reader {
+public:
+  virtual ~Reader(){};
+  virtual std::unique_ptr<Object> create() const = 0;
+};
+
+class MachOReader : public Reader {
+  const object::MachOObjectFile &MachOObj;
+
+  void readHeader(Object &O) const;
+  void readLoadCommands(Object &O) const;
+  void readSymbolTable(Object &O) const;
+  void readStringTable(Object &O) const;
+  void readRebaseInfo(Object &O) const;
+  void readBindInfo(Object &O) const;
+  void readWeakBindInfo(Object &O) const;
+  void readLazyBindInfo(Object &O) const;
+  void readExportInfo(Object &O) const;
+
+public:
+  explicit MachOReader(const object::MachOObjectFile &Obj) : MachOObj(Obj) {}
+
+  std::unique_ptr<Object> create() const override;
+};
+
+} // end namespace macho
+} // end namespace objcopy
+} // end namespace llvm
diff --git a/tools/llvm-objcopy/MachO/MachOWriter.cpp b/tools/llvm-objcopy/MachO/MachOWriter.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..a72a9609454e6aadb2dd8d215ddfa071cf3f02d8
--- /dev/null
+++ b/tools/llvm-objcopy/MachO/MachOWriter.cpp
@@ -0,0 +1,350 @@
+//===- MachOWriter.cpp ------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "MachOWriter.h"
+#include "../llvm-objcopy.h"
+#include "Object.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/BinaryFormat/MachO.h"
+#include "llvm/Object/MachO.h"
+#include <memory>
+
+namespace llvm {
+namespace objcopy {
+namespace macho {
+
+size_t MachOWriter::headerSize() const {
+  return Is64Bit ? sizeof(MachO::mach_header_64) : sizeof(MachO::mach_header);
+}
+
+size_t MachOWriter::loadCommandsSize() const { return O.Header.SizeOfCmds; }
+
+size_t MachOWriter::symTableSize() const {
+  return O.SymTable.NameList.size() *
+         (Is64Bit ? sizeof(MachO::nlist_64) : sizeof(MachO::nlist));
+}
+
+size_t MachOWriter::strTableSize() const {
+  size_t S = 0;
+  for (const auto &Str : O.StrTable.Strings)
+    S += Str.size();
+  S += (O.StrTable.Strings.empty() ? 0 : O.StrTable.Strings.size() - 1);
+  return S;
+}
+
+size_t MachOWriter::totalSize() const {
+  // Going from tail to head and looking for an appropriate "anchor" to
+  // calculate the total size assuming that all the offsets are either valid
+  // ("true") or 0 (0 indicates that the corresponding part is missing).
+
+  SmallVector<size_t, 7> Ends;
+  if (O.SymTabCommandIndex) {
+    const MachO::symtab_command &SymTabCommand =
+        O.LoadCommands[*O.SymTabCommandIndex]
+            .MachOLoadCommand.symtab_command_data;
+    if (SymTabCommand.symoff) {
+      assert((SymTabCommand.nsyms == O.SymTable.NameList.size()) &&
+             "Incorrect number of symbols");
+      Ends.push_back(SymTabCommand.symoff + symTableSize());
+    }
+    if (SymTabCommand.stroff) {
+      assert((SymTabCommand.strsize == strTableSize()) &&
+             "Incorrect string table size");
+      Ends.push_back(SymTabCommand.stroff + SymTabCommand.strsize);
+    }
+  }
+  if (O.DyLdInfoCommandIndex) {
+    const MachO::dyld_info_command &DyLdInfoCommand =
+        O.LoadCommands[*O.DyLdInfoCommandIndex]
+            .MachOLoadCommand.dyld_info_command_data;
+    if (DyLdInfoCommand.rebase_off) {
+      assert((DyLdInfoCommand.rebase_size == O.Rebases.Opcodes.size()) &&
+             "Incorrect rebase opcodes size");
+      Ends.push_back(DyLdInfoCommand.rebase_off + DyLdInfoCommand.rebase_size);
+    }
+    if (DyLdInfoCommand.bind_off) {
+      assert((DyLdInfoCommand.bind_size == O.Binds.Opcodes.size()) &&
+             "Incorrect bind opcodes size");
+      Ends.push_back(DyLdInfoCommand.bind_off + DyLdInfoCommand.bind_size);
+    }
+    if (DyLdInfoCommand.weak_bind_off) {
+      assert((DyLdInfoCommand.weak_bind_size == O.WeakBinds.Opcodes.size()) &&
+             "Incorrect weak bind opcodes size");
+      Ends.push_back(DyLdInfoCommand.weak_bind_off +
+                     DyLdInfoCommand.weak_bind_size);
+    }
+    if (DyLdInfoCommand.lazy_bind_off) {
+      assert((DyLdInfoCommand.lazy_bind_size == O.LazyBinds.Opcodes.size()) &&
+             "Incorrect lazy bind opcodes size");
+      Ends.push_back(DyLdInfoCommand.lazy_bind_off +
+                     DyLdInfoCommand.lazy_bind_size);
+    }
+    if (DyLdInfoCommand.export_off) {
+      assert((DyLdInfoCommand.export_size == O.Exports.Trie.size()) &&
+             "Incorrect trie size");
+      Ends.push_back(DyLdInfoCommand.export_off + DyLdInfoCommand.export_size);
+    }
+  }
+
+  // Otherwise, use the last section / reloction.
+  for (const auto &LC : O.LoadCommands)
+    for (const auto &S : LC.Sections) {
+      Ends.push_back(S.Offset + S.Size);
+      if (S.RelOff)
+        Ends.push_back(S.RelOff +
+                       S.NReloc * sizeof(MachO::any_relocation_info));
+    }
+
+  if (!Ends.empty())
+    return *std::max_element(Ends.begin(), Ends.end());
+
+  // Otherwise, we have only Mach header and load commands.
+  return headerSize() + loadCommandsSize();
+}
+
+void MachOWriter::writeHeader() {
+  MachO::mach_header_64 Header;
+
+  Header.magic = O.Header.Magic;
+  Header.cputype = O.Header.CPUType;
+  Header.cpusubtype = O.Header.CPUSubType;
+  Header.filetype = O.Header.FileType;
+  Header.ncmds = O.Header.NCmds;
+  Header.sizeofcmds = O.Header.SizeOfCmds;
+  Header.flags = O.Header.Flags;
+  Header.reserved = O.Header.Reserved;
+
+  if (IsLittleEndian != sys::IsLittleEndianHost)
+    MachO::swapStruct(Header);
+
+  auto HeaderSize =
+      Is64Bit ? sizeof(MachO::mach_header_64) : sizeof(MachO::mach_header);
+  memcpy(B.getBufferStart(), &Header, HeaderSize);
+}
+
+void MachOWriter::writeLoadCommands() {
+  uint8_t *Begin = B.getBufferStart() + headerSize();
+  MachO::macho_load_command MLC;
+  for (const auto &LC : O.LoadCommands) {
+#define HANDLE_LOAD_COMMAND(LCName, LCValue, LCStruct)                         \
+  case MachO::LCName:                                                          \
+    assert(sizeof(MachO::LCStruct) + LC.Payload.size() ==                      \
+           LC.MachOLoadCommand.load_command_data.cmdsize);                     \
+    MLC = LC.MachOLoadCommand;                                                 \
+    if (IsLittleEndian != sys::IsLittleEndianHost)                             \
+      MachO::swapStruct(MLC.LCStruct##_data);                                  \
+    memcpy(Begin, &MLC.LCStruct##_data, sizeof(MachO::LCStruct));              \
+    Begin += sizeof(MachO::LCStruct);                                          \
+    memcpy(Begin, LC.Payload.data(), LC.Payload.size());                       \
+    Begin += LC.Payload.size();                                                \
+    break;
+
+    switch (LC.MachOLoadCommand.load_command_data.cmd) {
+    default:
+      assert(sizeof(MachO::load_command) + LC.Payload.size() ==
+             LC.MachOLoadCommand.load_command_data.cmdsize);
+      MLC = LC.MachOLoadCommand;
+      if (IsLittleEndian != sys::IsLittleEndianHost)
+        MachO::swapStruct(MLC.load_command_data);
+      memcpy(Begin, &MLC.load_command_data, sizeof(MachO::load_command));
+      Begin += sizeof(MachO::load_command);
+      memcpy(Begin, LC.Payload.data(), LC.Payload.size());
+      Begin += LC.Payload.size();
+      break;
+#include "llvm/BinaryFormat/MachO.def"
+    }
+  }
+}
+
+void MachOWriter::writeSections() {
+  for (const auto &LC : O.LoadCommands)
+    for (const auto &Sec : LC.Sections) {
+      assert(Sec.Offset && "Section offset can not be zero");
+      assert((Sec.Size == Sec.Content.size()) && "Incorrect section size");
+      memcpy(B.getBufferStart() + Sec.Offset, Sec.Content.data(),
+             Sec.Content.size());
+      for (size_t Index = 0; Index < Sec.Relocations.size(); ++Index) {
+        MachO::any_relocation_info R = Sec.Relocations[Index];
+        if (IsLittleEndian != sys::IsLittleEndianHost)
+          MachO::swapStruct(R);
+        memcpy(B.getBufferStart() + Sec.RelOff +
+                   Index * sizeof(MachO::any_relocation_info),
+               &R, sizeof(R));
+      }
+    }
+}
+
+template <typename NListType>
+void writeNListEntry(const NListEntry &NLE, bool IsLittleEndian, char *&Out) {
+  NListType ListEntry;
+  ListEntry.n_strx = NLE.n_strx;
+  ListEntry.n_type = NLE.n_type;
+  ListEntry.n_sect = NLE.n_sect;
+  ListEntry.n_desc = NLE.n_desc;
+  ListEntry.n_value = NLE.n_value;
+
+  if (IsLittleEndian != sys::IsLittleEndianHost)
+    MachO::swapStruct(ListEntry);
+  memcpy(Out, reinterpret_cast<const char *>(&ListEntry), sizeof(NListType));
+  Out += sizeof(NListType);
+}
+
+void MachOWriter::writeSymbolTable() {
+  if (!O.SymTabCommandIndex)
+    return;
+  const MachO::symtab_command &SymTabCommand =
+      O.LoadCommands[*O.SymTabCommandIndex]
+          .MachOLoadCommand.symtab_command_data;
+  assert((SymTabCommand.nsyms == O.SymTable.NameList.size()) &&
+         "Incorrect number of symbols");
+  char *Out = (char *)B.getBufferStart() + SymTabCommand.symoff;
+  for (auto NLE : O.SymTable.NameList) {
+    if (Is64Bit)
+      writeNListEntry<MachO::nlist_64>(NLE, IsLittleEndian, Out);
+    else
+      writeNListEntry<MachO::nlist>(NLE, IsLittleEndian, Out);
+  }
+}
+
+void MachOWriter::writeStringTable() {
+  if (!O.SymTabCommandIndex)
+    return;
+  const MachO::symtab_command &SymTabCommand =
+      O.LoadCommands[*O.SymTabCommandIndex]
+          .MachOLoadCommand.symtab_command_data;
+  char *Out = (char *)B.getBufferStart() + SymTabCommand.stroff;
+  assert((SymTabCommand.strsize == strTableSize()) &&
+         "Incorrect string table size");
+  for (size_t Index = 0; Index < O.StrTable.Strings.size(); ++Index) {
+    memcpy(Out, O.StrTable.Strings[Index].data(),
+           O.StrTable.Strings[Index].size());
+    Out += O.StrTable.Strings[Index].size();
+    if (Index + 1 != O.StrTable.Strings.size()) {
+      memcpy(Out, "\0", 1);
+      Out += 1;
+    }
+  }
+}
+
+void MachOWriter::writeRebaseInfo() {
+  if (!O.DyLdInfoCommandIndex)
+    return;
+  const MachO::dyld_info_command &DyLdInfoCommand =
+      O.LoadCommands[*O.DyLdInfoCommandIndex]
+          .MachOLoadCommand.dyld_info_command_data;
+  char *Out = (char *)B.getBufferStart() + DyLdInfoCommand.rebase_off;
+  assert((DyLdInfoCommand.rebase_size == O.Rebases.Opcodes.size()) &&
+         "Incorrect rebase opcodes size");
+  memcpy(Out, O.Rebases.Opcodes.data(), O.Rebases.Opcodes.size());
+}
+
+void MachOWriter::writeBindInfo() {
+  if (!O.DyLdInfoCommandIndex)
+    return;
+  const MachO::dyld_info_command &DyLdInfoCommand =
+      O.LoadCommands[*O.DyLdInfoCommandIndex]
+          .MachOLoadCommand.dyld_info_command_data;
+  char *Out = (char *)B.getBufferStart() + DyLdInfoCommand.bind_off;
+  assert((DyLdInfoCommand.bind_size == O.Binds.Opcodes.size()) &&
+         "Incorrect bind opcodes size");
+  memcpy(Out, O.Binds.Opcodes.data(), O.Binds.Opcodes.size());
+}
+
+void MachOWriter::writeWeakBindInfo() {
+  if (!O.DyLdInfoCommandIndex)
+    return;
+  const MachO::dyld_info_command &DyLdInfoCommand =
+      O.LoadCommands[*O.DyLdInfoCommandIndex]
+          .MachOLoadCommand.dyld_info_command_data;
+  char *Out = (char *)B.getBufferStart() + DyLdInfoCommand.weak_bind_off;
+  assert((DyLdInfoCommand.weak_bind_size == O.WeakBinds.Opcodes.size()) &&
+         "Incorrect weak bind opcodes size");
+  memcpy(Out, O.WeakBinds.Opcodes.data(), O.WeakBinds.Opcodes.size());
+}
+
+void MachOWriter::writeLazyBindInfo() {
+  if (!O.DyLdInfoCommandIndex)
+    return;
+  const MachO::dyld_info_command &DyLdInfoCommand =
+      O.LoadCommands[*O.DyLdInfoCommandIndex]
+          .MachOLoadCommand.dyld_info_command_data;
+  char *Out = (char *)B.getBufferStart() + DyLdInfoCommand.lazy_bind_off;
+  assert((DyLdInfoCommand.lazy_bind_size == O.LazyBinds.Opcodes.size()) &&
+         "Incorrect lazy bind opcodes size");
+  memcpy(Out, O.LazyBinds.Opcodes.data(), O.LazyBinds.Opcodes.size());
+}
+
+void MachOWriter::writeExportInfo() {
+  if (!O.DyLdInfoCommandIndex)
+    return;
+  const MachO::dyld_info_command &DyLdInfoCommand =
+      O.LoadCommands[*O.DyLdInfoCommandIndex]
+          .MachOLoadCommand.dyld_info_command_data;
+  char *Out = (char *)B.getBufferStart() + DyLdInfoCommand.export_off;
+  assert((DyLdInfoCommand.export_size == O.Exports.Trie.size()) &&
+         "Incorrect export trie size");
+  memcpy(Out, O.Exports.Trie.data(), O.Exports.Trie.size());
+}
+
+void MachOWriter::writeTail() {
+  typedef void (MachOWriter::*WriteHandlerType)(void);
+  typedef std::pair<uint64_t, WriteHandlerType> WriteOperation;
+  SmallVector<WriteOperation, 7> Queue;
+
+  if (O.SymTabCommandIndex) {
+    const MachO::symtab_command &SymTabCommand =
+        O.LoadCommands[*O.SymTabCommandIndex]
+            .MachOLoadCommand.symtab_command_data;
+    if (SymTabCommand.symoff)
+      Queue.push_back({SymTabCommand.symoff, &MachOWriter::writeSymbolTable});
+    if (SymTabCommand.stroff)
+      Queue.push_back({SymTabCommand.stroff, &MachOWriter::writeStringTable});
+  }
+
+  if (O.DyLdInfoCommandIndex) {
+    const MachO::dyld_info_command &DyLdInfoCommand =
+        O.LoadCommands[*O.DyLdInfoCommandIndex]
+            .MachOLoadCommand.dyld_info_command_data;
+    if (DyLdInfoCommand.rebase_off)
+      Queue.push_back(
+          {DyLdInfoCommand.rebase_off, &MachOWriter::writeRebaseInfo});
+    if (DyLdInfoCommand.bind_off)
+      Queue.push_back({DyLdInfoCommand.bind_off, &MachOWriter::writeBindInfo});
+    if (DyLdInfoCommand.weak_bind_off)
+      Queue.push_back(
+          {DyLdInfoCommand.weak_bind_off, &MachOWriter::writeWeakBindInfo});
+    if (DyLdInfoCommand.lazy_bind_off)
+      Queue.push_back(
+          {DyLdInfoCommand.lazy_bind_off, &MachOWriter::writeLazyBindInfo});
+    if (DyLdInfoCommand.export_off)
+      Queue.push_back(
+          {DyLdInfoCommand.export_off, &MachOWriter::writeExportInfo});
+  }
+
+  llvm::sort(Queue, [](const WriteOperation &LHS, const WriteOperation &RHS) {
+    return LHS.first < RHS.first;
+  });
+
+  for (auto WriteOp : Queue)
+    (this->*WriteOp.second)();
+}
+
+Error MachOWriter::write() {
+  if (Error E = B.allocate(totalSize()))
+    return E;
+  memset(B.getBufferStart(), 0, totalSize());
+  writeHeader();
+  writeLoadCommands();
+  writeSections();
+  writeTail();
+  return B.commit();
+}
+
+} // end namespace macho
+} // end namespace objcopy
+} // end namespace llvm
diff --git a/tools/llvm-objcopy/MachO/MachOWriter.h b/tools/llvm-objcopy/MachO/MachOWriter.h
new file mode 100644
index 0000000000000000000000000000000000000000..9c91452a2add29d2b448e6ec23c0291574207c3a
--- /dev/null
+++ b/tools/llvm-objcopy/MachO/MachOWriter.h
@@ -0,0 +1,54 @@
+//===- MachOWriter.h --------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "../Buffer.h"
+#include "MachOObjcopy.h"
+#include "Object.h"
+#include "llvm/BinaryFormat/MachO.h"
+#include "llvm/Object/MachO.h"
+
+namespace llvm {
+class Error;
+
+namespace objcopy {
+namespace macho {
+
+class MachOWriter {
+  Object &O;
+  bool Is64Bit;
+  bool IsLittleEndian;
+  Buffer &B;
+
+  size_t headerSize() const;
+  size_t loadCommandsSize() const;
+  size_t symTableSize() const;
+  size_t strTableSize() const;
+
+  void writeHeader();
+  void writeLoadCommands();
+  void writeSections();
+  void writeSymbolTable();
+  void writeStringTable();
+  void writeRebaseInfo();
+  void writeBindInfo();
+  void writeWeakBindInfo();
+  void writeLazyBindInfo();
+  void writeExportInfo();
+  void writeTail();
+
+public:
+  MachOWriter(Object &O, bool Is64Bit, bool IsLittleEndian, Buffer &B)
+      : O(O), Is64Bit(Is64Bit), IsLittleEndian(IsLittleEndian), B(B) {}
+
+  size_t totalSize() const;
+  Error write();
+};
+
+} // end namespace macho
+} // end namespace objcopy
+} // end namespace llvm
diff --git a/tools/llvm-objcopy/MachO/Object.h b/tools/llvm-objcopy/MachO/Object.h
new file mode 100644
index 0000000000000000000000000000000000000000..61cfbf18bdd9b5311216e0227d5de9c5d289657a
--- /dev/null
+++ b/tools/llvm-objcopy/MachO/Object.h
@@ -0,0 +1,208 @@
+//===- Object.h - Mach-O object file model ----------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_OBJCOPY_MACHO_OBJECT_H
+#define LLVM_OBJCOPY_MACHO_OBJECT_H
+
+#include "llvm/ADT/Optional.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/BinaryFormat/MachO.h"
+#include "llvm/ObjectYAML/DWARFYAML.h"
+#include "llvm/Support/YAMLTraits.h"
+#include <cstdint>
+#include <string>
+#include <vector>
+
+namespace llvm {
+namespace objcopy {
+namespace macho {
+
+struct MachHeader {
+  uint32_t Magic;
+  uint32_t CPUType;
+  uint32_t CPUSubType;
+  uint32_t FileType;
+  uint32_t NCmds;
+  uint32_t SizeOfCmds;
+  uint32_t Flags;
+  uint32_t Reserved = 0;
+};
+
+struct Section {
+  char Sectname[16];
+  char Segname[16];
+  uint64_t Addr;
+  uint64_t Size;
+  uint32_t Offset;
+  uint32_t Align;
+  uint32_t RelOff;
+  uint32_t NReloc;
+  uint32_t Flags;
+  uint32_t Reserved1;
+  uint32_t Reserved2;
+  uint32_t Reserved3;
+
+  StringRef Content;
+  std::vector<MachO::any_relocation_info> Relocations;
+};
+
+struct LoadCommand {
+  // The type MachO::macho_load_command is defined in llvm/BinaryFormat/MachO.h
+  // and it is a union of all the structs corresponding to various load
+  // commands.
+  MachO::macho_load_command MachOLoadCommand;
+
+  // The raw content of the payload of the load command (located right after the
+  // corresponding struct). In some cases it is either empty or can be
+  // copied-over without digging into its structure.
+  ArrayRef<uint8_t> Payload;
+
+  // Some load commands can contain (inside the payload) an array of sections,
+  // though the contents of the sections are stored separately. The struct
+  // Section describes only sections' metadata and where to find the
+  // corresponding content inside the binary.
+  std::vector<Section> Sections;
+};
+
+struct NListEntry {
+  uint32_t n_strx;
+  uint8_t n_type;
+  uint8_t n_sect;
+  uint16_t n_desc;
+  uint64_t n_value;
+};
+
+/// The location of the symbol table inside the binary is described by LC_SYMTAB
+/// load command.
+struct SymbolTable {
+  std::vector<NListEntry> NameList;
+};
+
+/// The location of the string table inside the binary is described by LC_SYMTAB
+/// load command.
+struct StringTable {
+  std::vector<std::string> Strings;
+};
+
+/// The location of the rebase info inside the binary is described by
+/// LC_DYLD_INFO load command. Dyld rebases an image whenever dyld loads it at
+/// an address different from its preferred address.  The rebase information is
+/// a stream of byte sized opcodes whose symbolic names start with
+/// REBASE_OPCODE_. Conceptually the rebase information is a table of tuples:
+///   <seg-index, seg-offset, type>
+/// The opcodes are a compressed way to encode the table by only
+/// encoding when a column changes.  In addition simple patterns
+/// like "every n'th offset for m times" can be encoded in a few
+/// bytes.
+struct RebaseInfo {
+  // At the moment we do not parse this info (and it is simply copied over),
+  // but the proper support will be added later.
+  ArrayRef<uint8_t> Opcodes;
+};
+
+/// The location of the bind info inside the binary is described by
+/// LC_DYLD_INFO load command. Dyld binds an image during the loading process,
+/// if the image requires any pointers to be initialized to symbols in other
+/// images. The bind information is a stream of byte sized opcodes whose
+/// symbolic names start with BIND_OPCODE_. Conceptually the bind information is
+/// a table of tuples: <seg-index, seg-offset, type, symbol-library-ordinal,
+/// symbol-name, addend> The opcodes are a compressed way to encode the table by
+/// only encoding when a column changes.  In addition simple patterns like for
+/// runs of pointers initialized to the same value can be encoded in a few
+/// bytes.
+struct BindInfo {
+  // At the moment we do not parse this info (and it is simply copied over),
+  // but the proper support will be added later.
+  ArrayRef<uint8_t> Opcodes;
+};
+
+/// The location of the weak bind info inside the binary is described by
+/// LC_DYLD_INFO load command. Some C++ programs require dyld to unique symbols
+/// so that all images in the process use the same copy of some code/data. This
+/// step is done after binding. The content of the weak_bind info is an opcode
+/// stream like the bind_info.  But it is sorted alphabetically by symbol name.
+/// This enable dyld to walk all images with weak binding information in order
+/// and look for collisions.  If there are no collisions, dyld does no updating.
+/// That means that some fixups are also encoded in the bind_info.  For
+/// instance, all calls to "operator new" are first bound to libstdc++.dylib
+/// using the information in bind_info.  Then if some image overrides operator
+/// new that is detected when the weak_bind information is processed and the
+/// call to operator new is then rebound.
+struct WeakBindInfo {
+  // At the moment we do not parse this info (and it is simply copied over),
+  // but the proper support will be added later.
+  ArrayRef<uint8_t> Opcodes;
+};
+
+/// The location of the lazy bind info inside the binary is described by
+/// LC_DYLD_INFO load command. Some uses of external symbols do not need to be
+/// bound immediately. Instead they can be lazily bound on first use.  The
+/// lazy_bind contains a stream of BIND opcodes to bind all lazy symbols. Normal
+/// use is that dyld ignores the lazy_bind section when loading an image.
+/// Instead the static linker arranged for the lazy pointer to initially point
+/// to a helper function which pushes the offset into the lazy_bind area for the
+/// symbol needing to be bound, then jumps to dyld which simply adds the offset
+/// to lazy_bind_off to get the information on what to bind.
+struct LazyBindInfo {
+  ArrayRef<uint8_t> Opcodes;
+};
+
+/// The location of the export info inside the binary is described by
+/// LC_DYLD_INFO load command. The symbols exported by a dylib are encoded in a
+/// trie.  This is a compact representation that factors out common prefixes. It
+/// also reduces LINKEDIT pages in RAM because it encodes all information (name,
+/// address, flags) in one small, contiguous range. The export area is a stream
+/// of nodes.  The first node sequentially is the start node for the trie. Nodes
+/// for a symbol start with a uleb128 that is the length of the exported symbol
+/// information for the string so far. If there is no exported symbol, the node
+/// starts with a zero byte. If there is exported info, it follows the length.
+/// First is a uleb128 containing flags. Normally, it is followed by
+/// a uleb128 encoded offset which is location of the content named
+/// by the symbol from the mach_header for the image.  If the flags
+/// is EXPORT_SYMBOL_FLAGS_REEXPORT, then following the flags is
+/// a uleb128 encoded library ordinal, then a zero terminated
+/// UTF8 string.  If the string is zero length, then the symbol
+/// is re-export from the specified dylib with the same name.
+/// If the flags is EXPORT_SYMBOL_FLAGS_STUB_AND_RESOLVER, then following
+/// the flags is two uleb128s: the stub offset and the resolver offset.
+/// The stub is used by non-lazy pointers.  The resolver is used
+/// by lazy pointers and must be called to get the actual address to use.
+/// After the optional exported symbol information is a byte of
+/// how many edges (0-255) that this node has leaving it,
+/// followed by each edge.
+/// Each edge is a zero terminated UTF8 of the addition chars
+/// in the symbol, followed by a uleb128 offset for the node that
+/// edge points to.
+struct ExportInfo {
+  ArrayRef<uint8_t> Trie;
+};
+
+struct Object {
+  MachHeader Header;
+  std::vector<LoadCommand> LoadCommands;
+
+  SymbolTable SymTable;
+  StringTable StrTable;
+
+  RebaseInfo Rebases;
+  BindInfo Binds;
+  WeakBindInfo WeakBinds;
+  LazyBindInfo LazyBinds;
+  ExportInfo Exports;
+
+  /// The index of LC_SYMTAB load command if present.
+  Optional<size_t> SymTabCommandIndex;
+  /// The index of LC_DYLD_INFO or LC_DYLD_INFO_ONLY load command if present.
+  Optional<size_t> DyLdInfoCommandIndex;
+};
+
+} // end namespace macho
+} // end namespace objcopy
+} // end namespace llvm
+
+#endif // LLVM_OBJCOPY_MACHO_OBJECT_H
diff --git a/tools/llvm-objcopy/ObjcopyOpts.td b/tools/llvm-objcopy/ObjcopyOpts.td
index 1f7e64e4091c811f23a3fce36148288d624adef6..b3172190b04b86e4827a4c60807dde258f7b21c3 100644
--- a/tools/llvm-objcopy/ObjcopyOpts.td
+++ b/tools/llvm-objcopy/ObjcopyOpts.td
@@ -76,6 +76,16 @@ defm rename_section
 defm redefine_symbol
     : Eq<"redefine-sym", "Change the name of a symbol old to new">,
       MetaVarName<"old=new">;
+defm redefine_symbols
+    : Eq<"redefine-syms",
+         "Reads a list of symbol pairs from <filename> and runs as if "
+         "--redefine-sym=<old>=<new> is set for each one. <filename> "
+         "contains two symbols per line separated with whitespace and may "
+         "contain comments beginning with '#'. Leading and trailing "
+         "whitespace is stripped from each line. May be repeated to read "
+         "symbols from many files.">,         
+      MetaVarName<"filename">;
+
 defm keep_section : Eq<"keep-section", "Keep <section>">,
                     MetaVarName<"section">;
 defm only_section : Eq<"only-section", "Remove all but <section>">,
@@ -86,6 +96,13 @@ defm add_section
          "Make a section named <section> with the contents of <file>.">,
       MetaVarName<"section=file">;
 
+defm set_section_flags
+    : Eq<"set-section-flags",
+         "Set section flags for a given section. Flags supported for GNU "
+         "compatibility: alloc, load, noload, readonly, debug, code, data, "
+         "rom, share, contents, merge, strings.">,
+      MetaVarName<"section=flag1[,flag2,...]">;
+
 def strip_all
     : Flag<["-", "--"], "strip-all">,
       HelpText<
@@ -95,6 +112,8 @@ def strip_all_gnu : Flag<["-", "--"], "strip-all-gnu">,
                     HelpText<"Compatible with GNU objcopy's --strip-all">;
 def strip_debug : Flag<["-", "--"], "strip-debug">,
                   HelpText<"Remove all debug information">;
+def g : Flag<["-"], "g">, Alias<strip_debug>,
+        HelpText<"Alias for --strip-debug">;
 def strip_dwo : Flag<["-", "--"], "strip-dwo">,
                 HelpText<"Remove all DWARF .dwo sections from file">;
 def strip_sections : Flag<["-", "--"], "strip-sections">,
@@ -103,6 +122,15 @@ def strip_non_alloc : Flag<["-", "--"], "strip-non-alloc">,
                       HelpText<"Remove all non-allocated sections">;
 def strip_unneeded : Flag<["-", "--"], "strip-unneeded">,
                      HelpText<"Remove all symbols not needed by relocations">;
+defm strip_unneeded_symbol 
+    : Eq<"strip-unneeded-symbol",
+         "Remove symbol <symbol> if it is not needed by relocations">,
+      MetaVarName<"symbol">;
+defm strip_unneeded_symbols
+    : Eq<"strip-unneeded-symbols",
+         "Reads a list of symbols from <filename> and removes them "
+         "if they are not needed by relocations">,
+      MetaVarName<"filename">;
 
 def extract_dwo
     : Flag<["-", "--"], "extract-dwo">,
@@ -115,10 +143,21 @@ def localize_hidden
           "Mark all symbols that have hidden or internal visibility as local">;
 defm localize_symbol : Eq<"localize-symbol", "Mark <symbol> as local">,
                        MetaVarName<"symbol">;
+defm localize_symbols
+    : Eq<"localize-symbols",
+         "Reads a list of symbols from <filename> and marks them local.">,
+      MetaVarName<"filename">;
+
 def L : JoinedOrSeparate<["-"], "L">, Alias<localize_symbol>;
 
 defm globalize_symbol : Eq<"globalize-symbol", "Mark <symbol> as global">,
                         MetaVarName<"symbol">;
+
+defm globalize_symbols
+    : Eq<"globalize-symbols",
+         "Reads a list of symbols from <filename> and marks them global.">,
+      MetaVarName<"filename">;
+
 defm keep_global_symbol
     : Eq<"keep-global-symbol",
          "Convert all symbols except <symbol> to local. May be repeated to "
@@ -137,15 +176,31 @@ defm keep_global_symbols
 
 defm weaken_symbol : Eq<"weaken-symbol", "Mark <symbol> as weak">,
                      MetaVarName<"symbol">;
+defm weaken_symbols
+    : Eq<"weaken-symbols",
+         "Reads a list of symbols from <filename> and marks them weak.">,
+      MetaVarName<"filename">;
+
 def W : JoinedOrSeparate<["-"], "W">, Alias<weaken_symbol>;
 def weaken : Flag<["-", "--"], "weaken">,
              HelpText<"Mark all global symbols as weak">;
+
+def discard_locals : Flag<["-", "--"], "discard-locals">,
+                     HelpText<"Remove compiler-generated local symbols, (e.g. "
+                              "symbols starting with .L)">;
+def X : Flag<["-"], "X">, Alias<discard_locals>;
+
 def discard_all
     : Flag<["-", "--"], "discard-all">,
       HelpText<"Remove all local symbols except file and section symbols">;
 def x : Flag<["-"], "x">, Alias<discard_all>;
 defm strip_symbol : Eq<"strip-symbol", "Remove symbol <symbol>">,
                     MetaVarName<"symbol">;
+defm strip_symbols
+    : Eq<"strip-symbols",
+         "Reads a list of symbols from <filename> and removes them.">,
+      MetaVarName<"filename">;
+
 def N : JoinedOrSeparate<["-"], "N">, Alias<strip_symbol>;
 defm keep_symbol : Eq<"keep-symbol", "Do not remove symbol <symbol>">,
                    MetaVarName<"symbol">;
@@ -178,3 +233,7 @@ defm build_id_link_output
     : Eq<"build-id-link-output", "Hard-link the output to <dir>/xx/xxx<suffix> "
                                  "name derived from hex build ID">,
       MetaVarName<"suffix">;
+
+def regex
+    : Flag<["-", "--"], "regex">,
+      HelpText<"Permit regular expressions in name comparison">;
diff --git a/tools/llvm-objcopy/StripOpts.td b/tools/llvm-objcopy/StripOpts.td
index fa98e27e9321b41a244b6e5ec5a14ef70dad7b9e..c3f351992a064ce8d15598b65559de93bf6aed42 100644
--- a/tools/llvm-objcopy/StripOpts.td
+++ b/tools/llvm-objcopy/StripOpts.td
@@ -51,17 +51,33 @@ defm remove_section : Eq<"remove-section", "Remove <section>">,
                       MetaVarName<"section">;
 def R : JoinedOrSeparate<["-"], "R">, Alias<remove_section>;
 
+defm strip_symbol : Eq<"strip-symbol", "Strip <symbol>">,
+                    MetaVarName<"symbol">;
+def N : JoinedOrSeparate<["-"], "N">, Alias<strip_symbol>;
+
 defm keep_section : Eq<"keep-section", "Keep <section>">,
                     MetaVarName<"section">;
 defm keep_symbol : Eq<"keep-symbol", "Do not remove symbol <symbol>">,
                    MetaVarName<"symbol">;
+def keep_file_symbols : Flag<["-", "--"], "keep-file-symbols">,
+                        HelpText<"Do not remove file symbols">;
+
 def K : JoinedOrSeparate<["-"], "K">, Alias<keep_symbol>;
 
+def discard_locals : Flag<["-", "--"], "discard-locals">,
+                     HelpText<"Remove compiler-generated local symbols, (e.g. "
+                              "symbols starting with .L)">;
+def X : Flag<["-"], "X">, Alias<discard_locals>;
+
 def discard_all
     : Flag<["-", "--"], "discard-all">,
       HelpText<"Remove all local symbols except file and section symbols">;
 def x : Flag<["-"], "x">, Alias<discard_all>;
 
+def regex
+    : Flag<["-", "--"], "regex">,
+      HelpText<"Permit regular expressions in name comparison">;
+
 def version : Flag<["-", "--"], "version">,
               HelpText<"Print the version and exit.">;
 def V : Flag<["-"], "V">, Alias<version>;
diff --git a/tools/llvm-objcopy/llvm-objcopy.cpp b/tools/llvm-objcopy/llvm-objcopy.cpp
index fb1ff18b015b153e8c9c72f5b898654e4edcf118..fdf1c305b61e0313cb43885b2c2bd749a91936c9 100644
--- a/tools/llvm-objcopy/llvm-objcopy.cpp
+++ b/tools/llvm-objcopy/llvm-objcopy.cpp
@@ -1,9 +1,8 @@
 //===- llvm-objcopy.cpp ---------------------------------------------------===//
 //
-//                      The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -12,6 +11,7 @@
 #include "COFF/COFFObjcopy.h"
 #include "CopyConfig.h"
 #include "ELF/ELFObjcopy.h"
+#include "MachO/MachOObjcopy.h"
 
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SmallVector.h"
@@ -24,6 +24,7 @@
 #include "llvm/Object/ELFObjectFile.h"
 #include "llvm/Object/ELFTypes.h"
 #include "llvm/Object/Error.h"
+#include "llvm/Object/MachO.h"
 #include "llvm/Option/Arg.h"
 #include "llvm/Option/ArgList.h"
 #include "llvm/Option/Option.h"
@@ -57,11 +58,19 @@ LLVM_ATTRIBUTE_NORETURN void error(Twine Message) {
   exit(1);
 }
 
+LLVM_ATTRIBUTE_NORETURN void error(Error E) {
+  assert(E);
+  std::string Buf;
+  raw_string_ostream OS(Buf);
+  logAllUnhandledErrors(std::move(E), OS);
+  OS.flush();
+  WithColor::error(errs(), ToolName) << Buf;
+  exit(1);
+}
+
 LLVM_ATTRIBUTE_NORETURN void reportError(StringRef File, std::error_code EC) {
   assert(EC);
-  WithColor::error(errs(), ToolName)
-      << "'" << File << "': " << EC.message() << ".\n";
-  exit(1);
+  error(createFileError(File, EC));
 }
 
 LLVM_ATTRIBUTE_NORETURN void reportError(StringRef File, Error E) {
@@ -87,10 +96,13 @@ static Error deepWriteArchive(StringRef ArcName,
                               ArrayRef<NewArchiveMember> NewMembers,
                               bool WriteSymtab, object::Archive::Kind Kind,
                               bool Deterministic, bool Thin) {
-  Error E =
-      writeArchive(ArcName, NewMembers, WriteSymtab, Kind, Deterministic, Thin);
-  if (!Thin || E)
-    return E;
+  if (Error E = writeArchive(ArcName, NewMembers, WriteSymtab, Kind,
+                             Deterministic, Thin))
+    return createFileError(ArcName, std::move(E));
+
+  if (!Thin)
+    return Error::success();
+
   for (const NewArchiveMember &Member : NewMembers) {
     // Internally, FileBuffer will use the buffer created by
     // FileOutputBuffer::create, for regular files (that is the case for
@@ -101,10 +113,11 @@ static Error deepWriteArchive(StringRef ArcName,
     // NewArchiveMember still requires them even though writeArchive does not
     // write them on disk.
     FileBuffer FB(Member.MemberName);
-    FB.allocate(Member.Buf->getBufferSize());
+    if (Error E = FB.allocate(Member.Buf->getBufferSize()))
+      return E;
     std::copy(Member.Buf->getBufferStart(), Member.Buf->getBufferEnd(),
               FB.getBufferStart());
-    if (auto E = FB.commit())
+    if (Error E = FB.commit())
       return E;
   }
   return Error::success();
@@ -112,8 +125,8 @@ static Error deepWriteArchive(StringRef ArcName,
 
 /// The function executeObjcopyOnRawBinary does the dispatch based on the format
 /// of the output specified by the command line options.
-static void executeObjcopyOnRawBinary(const CopyConfig &Config,
-                                      MemoryBuffer &In, Buffer &Out) {
+static Error executeObjcopyOnRawBinary(const CopyConfig &Config,
+                                       MemoryBuffer &In, Buffer &Out) {
   // TODO: llvm-objcopy should parse CopyConfig.OutputFormat to recognize
   // formats other than ELF / "binary" and invoke
   // elf::executeObjcopyOnRawBinary, macho::executeObjcopyOnRawBinary or
@@ -123,48 +136,51 @@ static void executeObjcopyOnRawBinary(const CopyConfig &Config,
 
 /// The function executeObjcopyOnBinary does the dispatch based on the format
 /// of the input binary (ELF, MachO or COFF).
-static void executeObjcopyOnBinary(const CopyConfig &Config, object::Binary &In,
-                                   Buffer &Out) {
+static Error executeObjcopyOnBinary(const CopyConfig &Config,
+                                    object::Binary &In, Buffer &Out) {
   if (auto *ELFBinary = dyn_cast<object::ELFObjectFileBase>(&In))
     return elf::executeObjcopyOnBinary(Config, *ELFBinary, Out);
   else if (auto *COFFBinary = dyn_cast<object::COFFObjectFile>(&In))
     return coff::executeObjcopyOnBinary(Config, *COFFBinary, Out);
+  else if (auto *MachOBinary = dyn_cast<object::MachOObjectFile>(&In))
+    return macho::executeObjcopyOnBinary(Config, *MachOBinary, Out);
   else
-    error("Unsupported object file format");
+    return createStringError(object_error::invalid_file_type,
+                             "Unsupported object file format");
 }
 
-static void executeObjcopyOnArchive(const CopyConfig &Config,
-                                    const Archive &Ar) {
+static Error executeObjcopyOnArchive(const CopyConfig &Config,
+                                     const Archive &Ar) {
   std::vector<NewArchiveMember> NewArchiveMembers;
   Error Err = Error::success();
   for (const Archive::Child &Child : Ar.children(Err)) {
     Expected<std::unique_ptr<Binary>> ChildOrErr = Child.getAsBinary();
     if (!ChildOrErr)
-      reportError(Ar.getFileName(), ChildOrErr.takeError());
+      return createFileError(Ar.getFileName(), ChildOrErr.takeError());
     Binary *Bin = ChildOrErr->get();
 
     Expected<StringRef> ChildNameOrErr = Child.getName();
     if (!ChildNameOrErr)
-      reportError(Ar.getFileName(), ChildNameOrErr.takeError());
+      return createFileError(Ar.getFileName(), ChildNameOrErr.takeError());
 
     MemBuffer MB(ChildNameOrErr.get());
-    executeObjcopyOnBinary(Config, *Bin, MB);
+    if (Error E = executeObjcopyOnBinary(Config, *Bin, MB))
+      return E;
 
     Expected<NewArchiveMember> Member =
         NewArchiveMember::getOldMember(Child, Config.DeterministicArchives);
     if (!Member)
-      reportError(Ar.getFileName(), Member.takeError());
+      return createFileError(Ar.getFileName(), Member.takeError());
     Member->Buf = MB.releaseMemoryBuffer();
     Member->MemberName = Member->Buf->getBufferIdentifier();
     NewArchiveMembers.push_back(std::move(*Member));
   }
-
   if (Err)
-    reportError(Config.InputFilename, std::move(Err));
-  if (Error E = deepWriteArchive(Config.OutputFilename, NewArchiveMembers,
-                                 Ar.hasSymbolTable(), Ar.kind(),
-                                 Config.DeterministicArchives, Ar.isThin()))
-    reportError(Config.OutputFilename, std::move(E));
+    return createFileError(Config.InputFilename, std::move(Err));
+
+  return deepWriteArchive(Config.OutputFilename, NewArchiveMembers,
+                          Ar.hasSymbolTable(), Ar.kind(),
+                          Config.DeterministicArchives, Ar.isThin());
 }
 
 static void restoreDateOnFile(StringRef Filename,
@@ -197,7 +213,8 @@ static void executeObjcopy(const CopyConfig &Config) {
     if (!BufOrErr)
       reportError(Config.InputFilename, BufOrErr.getError());
     FileBuffer FB(Config.OutputFilename);
-    executeObjcopyOnRawBinary(Config, *BufOrErr->get(), FB);
+    if (Error E = executeObjcopyOnRawBinary(Config, *BufOrErr->get(), FB))
+      error(std::move(E));
   } else {
     Expected<OwningBinary<llvm::object::Binary>> BinaryOrErr =
         createBinary(Config.InputFilename);
@@ -205,10 +222,13 @@ static void executeObjcopy(const CopyConfig &Config) {
       reportError(Config.InputFilename, BinaryOrErr.takeError());
 
     if (Archive *Ar = dyn_cast<Archive>(BinaryOrErr.get().getBinary())) {
-      executeObjcopyOnArchive(Config, *Ar);
+      if (Error E = executeObjcopyOnArchive(Config, *Ar))
+        error(std::move(E));
     } else {
       FileBuffer FB(Config.OutputFilename);
-      executeObjcopyOnBinary(Config, *BinaryOrErr.get().getBinary(), FB);
+      if (Error E = executeObjcopyOnBinary(Config,
+                                           *BinaryOrErr.get().getBinary(), FB))
+        error(std::move(E));
     }
   }
 
diff --git a/tools/llvm-objcopy/llvm-objcopy.h b/tools/llvm-objcopy/llvm-objcopy.h
index d8edf3e29ee097484e6f360e391287134d5548cf..18a789ca1f83bbe030fd05953366d43f79ab1dae 100644
--- a/tools/llvm-objcopy/llvm-objcopy.h
+++ b/tools/llvm-objcopy/llvm-objcopy.h
@@ -1,9 +1,8 @@
 //===- llvm-objcopy.h -------------------------------------------*- C++ -*-===//
 //
-//                      The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -20,6 +19,7 @@ namespace llvm {
 namespace objcopy {
 
 LLVM_ATTRIBUTE_NORETURN extern void error(Twine Message);
+LLVM_ATTRIBUTE_NORETURN extern void error(Error E);
 LLVM_ATTRIBUTE_NORETURN extern void reportError(StringRef File, Error E);
 LLVM_ATTRIBUTE_NORETURN extern void reportError(StringRef File,
                                                 std::error_code EC);
diff --git a/tools/llvm-objdump/COFFDump.cpp b/tools/llvm-objdump/COFFDump.cpp
index a7ee49535d6544982146a7ac20322b99db732794..2d129043c26043a2d075f6fefb91132888e13dea 100644
--- a/tools/llvm-objdump/COFFDump.cpp
+++ b/tools/llvm-objdump/COFFDump.cpp
@@ -1,9 +1,8 @@
 //===-- COFFDump.cpp - COFF-specific dumper ---------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
@@ -455,7 +454,7 @@ static bool getPDataSection(const COFFObjectFile *Obj,
       Rels.push_back(Reloc);
 
     // Sort relocations by address.
-    llvm::sort(Rels, RelocAddressLess);
+    llvm::sort(Rels, isRelocAddressLess);
 
     ArrayRef<uint8_t> Contents;
     error(Obj->getSectionContents(Pdata, Contents));
@@ -469,6 +468,19 @@ static bool getPDataSection(const COFFObjectFile *Obj,
   return false;
 }
 
+std::error_code
+llvm::getCOFFRelocationValueString(const COFFObjectFile *Obj,
+                                   const RelocationRef &Rel,
+                                   SmallVectorImpl<char> &Result) {
+  symbol_iterator SymI = Rel.getSymbol();
+  Expected<StringRef> SymNameOrErr = SymI->getName();
+  if (!SymNameOrErr)
+    return errorToErrorCode(SymNameOrErr.takeError());
+  StringRef SymName = *SymNameOrErr;
+  Result.append(SymName.begin(), SymName.end());
+  return std::error_code();
+}
+
 static void printWin64EHUnwindInfo(const Win64EH::UnwindInfo *UI) {
   // The casts to int are required in order to output the value as number.
   // Without the casts the value would be interpreted as char data (which
diff --git a/tools/llvm-objdump/ELFDump.cpp b/tools/llvm-objdump/ELFDump.cpp
index b17a15a0d8fc9bd5066aee16935441558738f285..7157469d13f600980ae0d1561201716c639df68d 100644
--- a/tools/llvm-objdump/ELFDump.cpp
+++ b/tools/llvm-objdump/ELFDump.cpp
@@ -1,9 +1,8 @@
 //===-- ELFDump.cpp - ELF-specific dumper -----------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
@@ -13,6 +12,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm-objdump.h"
+#include "llvm/Demangle/Demangle.h"
 #include "llvm/Object/ELFObjectFile.h"
 #include "llvm/Support/Format.h"
 #include "llvm/Support/MathExtras.h"
@@ -51,6 +51,116 @@ Expected<StringRef> getDynamicStrTab(const ELFFile<ELFT> *Elf) {
   return createError("dynamic string table not found");
 }
 
+template <class ELFT>
+static std::error_code getRelocationValueString(const ELFObjectFile<ELFT> *Obj,
+                                                const RelocationRef &RelRef,
+                                                SmallVectorImpl<char> &Result) {
+  typedef typename ELFObjectFile<ELFT>::Elf_Sym Elf_Sym;
+  typedef typename ELFObjectFile<ELFT>::Elf_Shdr Elf_Shdr;
+  typedef typename ELFObjectFile<ELFT>::Elf_Rela Elf_Rela;
+
+  const ELFFile<ELFT> &EF = *Obj->getELFFile();
+  DataRefImpl Rel = RelRef.getRawDataRefImpl();
+  auto SecOrErr = EF.getSection(Rel.d.a);
+  if (!SecOrErr)
+    return errorToErrorCode(SecOrErr.takeError());
+
+  int64_t Addend = 0;
+  // If there is no Symbol associated with the relocation, we set the undef
+  // boolean value to 'true'. This will prevent us from calling functions that
+  // requires the relocation to be associated with a symbol.
+  //
+  // In SHT_REL case we would need to read the addend from section data.
+  // GNU objdump does not do that and we just follow for simplicity atm.
+  bool Undef = false;
+  if ((*SecOrErr)->sh_type == ELF::SHT_RELA) {
+    const Elf_Rela *ERela = Obj->getRela(Rel);
+    Addend = ERela->r_addend;
+    Undef = ERela->getSymbol(false) == 0;
+  } else if ((*SecOrErr)->sh_type != ELF::SHT_REL) {
+    return object_error::parse_failed;
+  }
+
+  // Default scheme is to print Target, as well as "+ <addend>" for nonzero
+  // addend. Should be acceptable for all normal purposes.
+  std::string FmtBuf;
+  raw_string_ostream Fmt(FmtBuf);
+
+  if (!Undef) {
+    symbol_iterator SI = RelRef.getSymbol();
+    const Elf_Sym *Sym = Obj->getSymbol(SI->getRawDataRefImpl());
+    if (Sym->getType() == ELF::STT_SECTION) {
+      Expected<section_iterator> SymSI = SI->getSection();
+      if (!SymSI)
+        return errorToErrorCode(SymSI.takeError());
+      const Elf_Shdr *SymSec = Obj->getSection((*SymSI)->getRawDataRefImpl());
+      auto SecName = EF.getSectionName(SymSec);
+      if (!SecName)
+        return errorToErrorCode(SecName.takeError());
+      Fmt << *SecName;
+    } else {
+      Expected<StringRef> SymName = SI->getName();
+      if (!SymName)
+        return errorToErrorCode(SymName.takeError());
+      if (Demangle)
+        Fmt << demangle(*SymName);
+      else
+        Fmt << *SymName;
+    }
+  } else {
+    Fmt << "*ABS*";
+  }
+
+  if (Addend != 0)
+    Fmt << (Addend < 0 ? "" : "+") << Addend;
+  Fmt.flush();
+  Result.append(FmtBuf.begin(), FmtBuf.end());
+  return std::error_code();
+}
+
+std::error_code
+llvm::getELFRelocationValueString(const ELFObjectFileBase *Obj,
+                                  const RelocationRef &Rel,
+                                  SmallVectorImpl<char> &Result) {
+  if (auto *ELF32LE = dyn_cast<ELF32LEObjectFile>(Obj))
+    return getRelocationValueString(ELF32LE, Rel, Result);
+  if (auto *ELF64LE = dyn_cast<ELF64LEObjectFile>(Obj))
+    return getRelocationValueString(ELF64LE, Rel, Result);
+  if (auto *ELF32BE = dyn_cast<ELF32BEObjectFile>(Obj))
+    return getRelocationValueString(ELF32BE, Rel, Result);
+  auto *ELF64BE = cast<ELF64BEObjectFile>(Obj);
+  return getRelocationValueString(ELF64BE, Rel, Result);
+}
+
+template <class ELFT>
+static uint64_t getSectionLMA(const ELFFile<ELFT> *Obj,
+                              const object::ELFSectionRef &Sec) {
+  auto PhdrRangeOrErr = Obj->program_headers();
+  if (!PhdrRangeOrErr)
+    report_fatal_error(errorToErrorCode(PhdrRangeOrErr.takeError()).message());
+
+  // Search for a PT_LOAD segment containing the requested section. Use this
+  // segment's p_addr to calculate the section's LMA.
+  for (const typename ELFFile<ELFT>::Elf_Phdr &Phdr : *PhdrRangeOrErr)
+    if ((Phdr.p_type == ELF::PT_LOAD) && (Phdr.p_vaddr <= Sec.getAddress()) &&
+        (Phdr.p_vaddr + Phdr.p_memsz > Sec.getAddress()))
+      return Sec.getAddress() - Phdr.p_vaddr + Phdr.p_paddr;
+
+  // Return section's VMA if it isn't in a PT_LOAD segment.
+  return Sec.getAddress();
+}
+
+uint64_t llvm::getELFSectionLMA(const object::ELFSectionRef &Sec) {
+  if (const auto *ELFObj = dyn_cast<ELF32LEObjectFile>(Sec.getObject()))
+    return getSectionLMA(ELFObj->getELFFile(), Sec);
+  else if (const auto *ELFObj = dyn_cast<ELF32BEObjectFile>(Sec.getObject()))
+    return getSectionLMA(ELFObj->getELFFile(), Sec);
+  else if (const auto *ELFObj = dyn_cast<ELF64LEObjectFile>(Sec.getObject()))
+    return getSectionLMA(ELFObj->getELFFile(), Sec);
+  const auto *ELFObj = cast<ELF64BEObjectFile>(Sec.getObject());
+  return getSectionLMA(ELFObj->getELFFile(), Sec);
+}
+
 template <class ELFT>
 void printDynamicSection(const ELFFile<ELFT> *Elf, StringRef Filename) {
   auto ProgramHeaderOrError = Elf->program_headers();
diff --git a/tools/llvm-objdump/LLVMBuild.txt b/tools/llvm-objdump/LLVMBuild.txt
index 3a966916c35d0db533c7f3a3e1a2ce81afb97d71..3aea3b7432cee807dc62ab644050c140e1ca017d 100644
--- a/tools/llvm-objdump/LLVMBuild.txt
+++ b/tools/llvm-objdump/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./tools/llvm-objdump/LLVMBuild.txt -----------------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/tools/llvm-objdump/MachODump.cpp b/tools/llvm-objdump/MachODump.cpp
index b5c0439c24c2916c4583d60275b5acd7b66cfd5d..549a20311df58998bb52f40b682c088aa923d7d0 100644
--- a/tools/llvm-objdump/MachODump.cpp
+++ b/tools/llvm-objdump/MachODump.cpp
@@ -1,9 +1,8 @@
 //===-- MachODump.cpp - Object file dumping utility for llvm --------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -342,6 +341,264 @@ static void getSectionsAndSymbols(MachOObjectFile *MachOObj,
   }
 }
 
+static void printRelocationTargetName(const MachOObjectFile *O,
+                                      const MachO::any_relocation_info &RE,
+                                      raw_string_ostream &Fmt) {
+  // Target of a scattered relocation is an address.  In the interest of
+  // generating pretty output, scan through the symbol table looking for a
+  // symbol that aligns with that address.  If we find one, print it.
+  // Otherwise, we just print the hex address of the target.
+  if (O->isRelocationScattered(RE)) {
+    uint32_t Val = O->getPlainRelocationSymbolNum(RE);
+
+    for (const SymbolRef &Symbol : O->symbols()) {
+      Expected<uint64_t> Addr = Symbol.getAddress();
+      if (!Addr)
+        report_error(O->getFileName(), Addr.takeError());
+      if (*Addr != Val)
+        continue;
+      Expected<StringRef> Name = Symbol.getName();
+      if (!Name)
+        report_error(O->getFileName(), Name.takeError());
+      Fmt << *Name;
+      return;
+    }
+
+    // If we couldn't find a symbol that this relocation refers to, try
+    // to find a section beginning instead.
+    for (const SectionRef &Section : ToolSectionFilter(*O)) {
+      std::error_code ec;
+
+      StringRef Name;
+      uint64_t Addr = Section.getAddress();
+      if (Addr != Val)
+        continue;
+      if ((ec = Section.getName(Name)))
+        report_error(O->getFileName(), ec);
+      Fmt << Name;
+      return;
+    }
+
+    Fmt << format("0x%x", Val);
+    return;
+  }
+
+  StringRef S;
+  bool isExtern = O->getPlainRelocationExternal(RE);
+  uint64_t Val = O->getPlainRelocationSymbolNum(RE);
+
+  if (O->getAnyRelocationType(RE) == MachO::ARM64_RELOC_ADDEND) {
+    Fmt << format("0x%0" PRIx64, Val);
+    return;
+  }
+
+  if (isExtern) {
+    symbol_iterator SI = O->symbol_begin();
+    advance(SI, Val);
+    Expected<StringRef> SOrErr = SI->getName();
+    if (!SOrErr)
+      report_error(O->getFileName(), SOrErr.takeError());
+    S = *SOrErr;
+  } else {
+    section_iterator SI = O->section_begin();
+    // Adjust for the fact that sections are 1-indexed.
+    if (Val == 0) {
+      Fmt << "0 (?,?)";
+      return;
+    }
+    uint32_t I = Val - 1;
+    while (I != 0 && SI != O->section_end()) {
+      --I;
+      advance(SI, 1);
+    }
+    if (SI == O->section_end())
+      Fmt << Val << " (?,?)";
+    else
+      SI->getName(S);
+  }
+
+  Fmt << S;
+}
+
+std::error_code
+llvm::getMachORelocationValueString(const MachOObjectFile *Obj,
+                                    const RelocationRef &RelRef,
+                                    SmallVectorImpl<char> &Result) {
+  DataRefImpl Rel = RelRef.getRawDataRefImpl();
+  MachO::any_relocation_info RE = Obj->getRelocation(Rel);
+
+  unsigned Arch = Obj->getArch();
+
+  std::string FmtBuf;
+  raw_string_ostream Fmt(FmtBuf);
+  unsigned Type = Obj->getAnyRelocationType(RE);
+  bool IsPCRel = Obj->getAnyRelocationPCRel(RE);
+
+  // Determine any addends that should be displayed with the relocation.
+  // These require decoding the relocation type, which is triple-specific.
+
+  // X86_64 has entirely custom relocation types.
+  if (Arch == Triple::x86_64) {
+    switch (Type) {
+    case MachO::X86_64_RELOC_GOT_LOAD:
+    case MachO::X86_64_RELOC_GOT: {
+      printRelocationTargetName(Obj, RE, Fmt);
+      Fmt << "@GOT";
+      if (IsPCRel)
+        Fmt << "PCREL";
+      break;
+    }
+    case MachO::X86_64_RELOC_SUBTRACTOR: {
+      DataRefImpl RelNext = Rel;
+      Obj->moveRelocationNext(RelNext);
+      MachO::any_relocation_info RENext = Obj->getRelocation(RelNext);
+
+      // X86_64_RELOC_SUBTRACTOR must be followed by a relocation of type
+      // X86_64_RELOC_UNSIGNED.
+      // NOTE: Scattered relocations don't exist on x86_64.
+      unsigned RType = Obj->getAnyRelocationType(RENext);
+      if (RType != MachO::X86_64_RELOC_UNSIGNED)
+        report_error(Obj->getFileName(), "Expected X86_64_RELOC_UNSIGNED after "
+                                         "X86_64_RELOC_SUBTRACTOR.");
+
+      // The X86_64_RELOC_UNSIGNED contains the minuend symbol;
+      // X86_64_RELOC_SUBTRACTOR contains the subtrahend.
+      printRelocationTargetName(Obj, RENext, Fmt);
+      Fmt << "-";
+      printRelocationTargetName(Obj, RE, Fmt);
+      break;
+    }
+    case MachO::X86_64_RELOC_TLV:
+      printRelocationTargetName(Obj, RE, Fmt);
+      Fmt << "@TLV";
+      if (IsPCRel)
+        Fmt << "P";
+      break;
+    case MachO::X86_64_RELOC_SIGNED_1:
+      printRelocationTargetName(Obj, RE, Fmt);
+      Fmt << "-1";
+      break;
+    case MachO::X86_64_RELOC_SIGNED_2:
+      printRelocationTargetName(Obj, RE, Fmt);
+      Fmt << "-2";
+      break;
+    case MachO::X86_64_RELOC_SIGNED_4:
+      printRelocationTargetName(Obj, RE, Fmt);
+      Fmt << "-4";
+      break;
+    default:
+      printRelocationTargetName(Obj, RE, Fmt);
+      break;
+    }
+    // X86 and ARM share some relocation types in common.
+  } else if (Arch == Triple::x86 || Arch == Triple::arm ||
+             Arch == Triple::ppc) {
+    // Generic relocation types...
+    switch (Type) {
+    case MachO::GENERIC_RELOC_PAIR: // prints no info
+      return std::error_code();
+    case MachO::GENERIC_RELOC_SECTDIFF: {
+      DataRefImpl RelNext = Rel;
+      Obj->moveRelocationNext(RelNext);
+      MachO::any_relocation_info RENext = Obj->getRelocation(RelNext);
+
+      // X86 sect diff's must be followed by a relocation of type
+      // GENERIC_RELOC_PAIR.
+      unsigned RType = Obj->getAnyRelocationType(RENext);
+
+      if (RType != MachO::GENERIC_RELOC_PAIR)
+        report_error(Obj->getFileName(), "Expected GENERIC_RELOC_PAIR after "
+                                         "GENERIC_RELOC_SECTDIFF.");
+
+      printRelocationTargetName(Obj, RE, Fmt);
+      Fmt << "-";
+      printRelocationTargetName(Obj, RENext, Fmt);
+      break;
+    }
+    }
+
+    if (Arch == Triple::x86 || Arch == Triple::ppc) {
+      switch (Type) {
+      case MachO::GENERIC_RELOC_LOCAL_SECTDIFF: {
+        DataRefImpl RelNext = Rel;
+        Obj->moveRelocationNext(RelNext);
+        MachO::any_relocation_info RENext = Obj->getRelocation(RelNext);
+
+        // X86 sect diff's must be followed by a relocation of type
+        // GENERIC_RELOC_PAIR.
+        unsigned RType = Obj->getAnyRelocationType(RENext);
+        if (RType != MachO::GENERIC_RELOC_PAIR)
+          report_error(Obj->getFileName(), "Expected GENERIC_RELOC_PAIR after "
+                                           "GENERIC_RELOC_LOCAL_SECTDIFF.");
+
+        printRelocationTargetName(Obj, RE, Fmt);
+        Fmt << "-";
+        printRelocationTargetName(Obj, RENext, Fmt);
+        break;
+      }
+      case MachO::GENERIC_RELOC_TLV: {
+        printRelocationTargetName(Obj, RE, Fmt);
+        Fmt << "@TLV";
+        if (IsPCRel)
+          Fmt << "P";
+        break;
+      }
+      default:
+        printRelocationTargetName(Obj, RE, Fmt);
+      }
+    } else { // ARM-specific relocations
+      switch (Type) {
+      case MachO::ARM_RELOC_HALF:
+      case MachO::ARM_RELOC_HALF_SECTDIFF: {
+        // Half relocations steal a bit from the length field to encode
+        // whether this is an upper16 or a lower16 relocation.
+        bool isUpper = (Obj->getAnyRelocationLength(RE) & 0x1) == 1;
+
+        if (isUpper)
+          Fmt << ":upper16:(";
+        else
+          Fmt << ":lower16:(";
+        printRelocationTargetName(Obj, RE, Fmt);
+
+        DataRefImpl RelNext = Rel;
+        Obj->moveRelocationNext(RelNext);
+        MachO::any_relocation_info RENext = Obj->getRelocation(RelNext);
+
+        // ARM half relocs must be followed by a relocation of type
+        // ARM_RELOC_PAIR.
+        unsigned RType = Obj->getAnyRelocationType(RENext);
+        if (RType != MachO::ARM_RELOC_PAIR)
+          report_error(Obj->getFileName(), "Expected ARM_RELOC_PAIR after "
+                                           "ARM_RELOC_HALF");
+
+        // NOTE: The half of the target virtual address is stashed in the
+        // address field of the secondary relocation, but we can't reverse
+        // engineer the constant offset from it without decoding the movw/movt
+        // instruction to find the other half in its immediate field.
+
+        // ARM_RELOC_HALF_SECTDIFF encodes the second section in the
+        // symbol/section pointer of the follow-on relocation.
+        if (Type == MachO::ARM_RELOC_HALF_SECTDIFF) {
+          Fmt << "-";
+          printRelocationTargetName(Obj, RENext, Fmt);
+        }
+
+        Fmt << ")";
+        break;
+      }
+      default: {
+        printRelocationTargetName(Obj, RE, Fmt);
+      }
+      }
+    }
+  } else
+    printRelocationTargetName(Obj, RE, Fmt);
+
+  Fmt.flush();
+  Result.append(FmtBuf.begin(), FmtBuf.end());
+  return std::error_code();
+}
+
 static void PrintIndirectSymbolTable(MachOObjectFile *O, bool verbose,
                                      uint32_t n, uint32_t count,
                                      uint32_t stride, uint64_t addr) {
@@ -1412,7 +1669,7 @@ static void DumpSectionContents(StringRef Filename, MachOObjectFile *O,
     std::pair<StringRef, StringRef> DumpSegSectName;
     DumpSegSectName = DumpSection.split(',');
     StringRef DumpSegName, DumpSectName;
-    if (DumpSegSectName.second.size()) {
+    if (!DumpSegSectName.second.empty()) {
       DumpSegName = DumpSegSectName.first;
       DumpSectName = DumpSegSectName.second;
     } else {
@@ -1582,7 +1839,7 @@ static void ProcessMachO(StringRef Name, MachOObjectFile *MachOOF,
   if (Disassemble || Relocations || PrivateHeaders || ExportsTrie || Rebase ||
       Bind || SymbolTable || LazyBind || WeakBind || IndirectSymbols ||
       DataInCode || LinkOptHints || DylibsUsed || DylibId || ObjcMetaData ||
-      (FilterSections.size() != 0)) {
+      (!FilterSections.empty())) {
     if (!NoLeadingHeaders) {
       outs() << Name;
       if (!ArchiveMemberName.empty())
@@ -1607,12 +1864,22 @@ static void ProcessMachO(StringRef Name, MachOObjectFile *MachOOF,
   // If we need the symbol table to do the operation then check it here to
   // produce a good error message as to where the Mach-O file comes from in
   // the error message.
-  if (Disassemble || IndirectSymbols || FilterSections.size() != 0 ||
-      UnwindInfo)
+  if (Disassemble || IndirectSymbols || !FilterSections.empty() || UnwindInfo)
     if (Error Err = MachOOF->checkSymbolTable())
       report_error(ArchiveName, FileName, std::move(Err), ArchitectureName);
-
-  if (Disassemble) {
+  
+  if (DisassembleAll) {
+    for (const SectionRef &Section : MachOOF->sections()) {
+      StringRef SectName;
+      Section.getName(SectName);
+      if (SectName.equals("__text")) {
+        DataRefImpl Ref = Section.getRawDataRefImpl();
+        StringRef SegName = MachOOF->getSectionFinalSegmentName(Ref);
+        DisassembleMachO(FileName, MachOOF, SegName, SectName);
+      }
+    }
+  }
+  else if (Disassemble) {
     if (MachOOF->getHeader().filetype == MachO::MH_KEXT_BUNDLE &&
         MachOOF->getHeader().cputype == MachO::CPU_TYPE_ARM64)
       DisassembleMachO(FileName, MachOOF, "__TEXT_EXEC", "__text");
@@ -1628,10 +1895,10 @@ static void ProcessMachO(StringRef Name, MachOObjectFile *MachOOF,
   if (Relocations)
     PrintRelocations(MachOOF, !NonVerbose);
   if (SectionHeaders)
-    PrintSectionHeaders(MachOOF);
+    printSectionHeaders(MachOOF);
   if (SectionContents)
-    PrintSectionContents(MachOOF);
-  if (FilterSections.size() != 0)
+    printSectionContents(MachOOF);
+  if (!FilterSections.empty())
     DumpSectionContents(FileName, MachOOF, !NonVerbose);
   if (InfoPlist)
     DumpInfoPlistSectionContents(FileName, MachOOF);
@@ -1640,7 +1907,7 @@ static void ProcessMachO(StringRef Name, MachOObjectFile *MachOOF,
   if (DylibId)
     PrintDylibs(MachOOF, true);
   if (SymbolTable)
-    PrintSymbolTable(MachOOF, ArchiveName, ArchitectureName);
+    printSymbolTable(MachOOF, ArchiveName, ArchitectureName);
   if (UnwindInfo)
     printMachOUnwindInfo(MachOOF);
   if (PrivateHeaders) {
@@ -1960,7 +2227,7 @@ static bool ValidateArchFlags() {
 // -arch flags selecting just those slices as specified by them and also parses
 // archive files.  Then for each individual Mach-O file ProcessMachO() is
 // called to process the file based on the command line options.
-void llvm::ParseInputMachO(StringRef Filename) {
+void llvm::parseInputMachO(StringRef Filename) {
   if (!ValidateArchFlags())
     return;
 
@@ -1999,7 +2266,7 @@ void llvm::ParseInputMachO(StringRef Filename) {
     return;
   }
   if (MachOUniversalBinary *UB = dyn_cast<MachOUniversalBinary>(&Bin)) {
-    ParseInputMachO(UB);
+    parseInputMachO(UB);
     return;
   }
   if (ObjectFile *O = dyn_cast<ObjectFile>(&Bin)) {
@@ -2016,7 +2283,7 @@ void llvm::ParseInputMachO(StringRef Filename) {
   llvm_unreachable("Input object can't be invalid at this point");
 }
 
-void llvm::ParseInputMachO(MachOUniversalBinary *UB) {
+void llvm::parseInputMachO(MachOUniversalBinary *UB) {
   if (!ValidateArchFlags())
     return;
 
@@ -2026,7 +2293,7 @@ void llvm::ParseInputMachO(MachOUniversalBinary *UB) {
     printMachOUniversalHeaders(UB, !NonVerbose);
 
   // If we have a list of architecture flags specified dump only those.
-  if (!ArchAll && ArchFlags.size() != 0) {
+  if (!ArchAll && !ArchFlags.empty()) {
     // Look for a slice in the universal binary that matches each ArchFlag.
     bool ArchFound;
     for (unsigned i = 0; i < ArchFlags.size(); ++i) {
@@ -6831,7 +7098,7 @@ static void DisassembleMachO(StringRef Filename, MachOObjectFile *MachOOF,
 
   // Package up features to be passed to target/subtarget
   std::string FeaturesStr;
-  if (MAttrs.size()) {
+  if (!MAttrs.empty()) {
     SubtargetFeatures Features;
     for (unsigned i = 0; i != MAttrs.size(); ++i)
       Features.AddFeature(MAttrs[i]);
@@ -6971,15 +7238,18 @@ static void DisassembleMachO(StringRef Filename, MachOObjectFile *MachOOF,
       ErrorOr<std::unique_ptr<MemoryBuffer>> BufOrErr =
           MemoryBuffer::getFileOrSTDIN(DSYMFile);
       if (std::error_code EC = BufOrErr.getError()) {
-        WithColor::error(errs(), "llvm-objdump")
-            << Filename << ": " << EC.message() << '\n';
+        report_error(DSYMFile, errorCodeToError(EC));
         return;
       }
+
       Expected<std::unique_ptr<MachOObjectFile>> DbgObjCheck =
           ObjectFile::createMachOObjectFile(BufOrErr.get()->getMemBufferRef());
 
-      if (DbgObjCheck.takeError())
-        report_error(MachOOF->getFileName(), DbgObjCheck.takeError());
+      if (Error E = DbgObjCheck.takeError()) {
+        report_error(DSYMFile, std::move(E));
+        return;
+      }
+
       DbgObj = DbgObjCheck.get().release();
       // We need to keep the file alive, because we're replacing DbgObj with it.
       DSYMBuf = std::move(BufOrErr.get());
@@ -6989,7 +7259,7 @@ static void DisassembleMachO(StringRef Filename, MachOObjectFile *MachOOF,
     diContext = DWARFContext::create(*DbgObj);
   }
 
-  if (FilterSections.size() == 0)
+  if (FilterSections.empty())
     outs() << "(" << DisSegName << "," << DisSectName << ") section\n";
 
   for (unsigned SectIdx = 0; SectIdx != Sections.size(); SectIdx++) {
@@ -7052,7 +7322,7 @@ static void DisassembleMachO(StringRef Filename, MachOObjectFile *MachOOF,
     unsigned int Arch = MachOOF->getArch();
 
     // Skip all symbols if this is a stubs file.
-    if (Bytes.size() == 0)
+    if (Bytes.empty())
       return;
 
     // If the section has symbols but no symbol at the start of the section
diff --git a/tools/llvm-objdump/WasmDump.cpp b/tools/llvm-objdump/WasmDump.cpp
index 045002cd4b34a0326f96b4648952055d0752b388..45eb8fceb4b3fc8fccee0d733a2307c5c5165acf 100644
--- a/tools/llvm-objdump/WasmDump.cpp
+++ b/tools/llvm-objdump/WasmDump.cpp
@@ -1,9 +1,8 @@
 //===-- WasmDump.cpp - wasm-specific dumper ---------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
@@ -19,10 +18,35 @@ using namespace llvm;
 using namespace object;
 
 void llvm::printWasmFileHeader(const object::ObjectFile *Obj) {
-  const WasmObjectFile *File = dyn_cast<const WasmObjectFile>(Obj);
+  const auto *File = dyn_cast<const WasmObjectFile>(Obj);
 
   outs() << "Program Header:\n";
   outs() << "Version: 0x";
   outs().write_hex(File->getHeader().Version);
   outs() << "\n";
 }
+
+std::error_code
+llvm::getWasmRelocationValueString(const WasmObjectFile *Obj,
+                                   const RelocationRef &RelRef,
+                                   SmallVectorImpl<char> &Result) {
+  const wasm::WasmRelocation &Rel = Obj->getWasmRelocation(RelRef);
+  symbol_iterator SI = RelRef.getSymbol();
+  std::string FmtBuf;
+  raw_string_ostream Fmt(FmtBuf);
+  if (SI == Obj->symbol_end()) {
+    // Not all wasm relocations have symbols associated with them.
+    // In particular R_WASM_TYPE_INDEX_LEB.
+    Fmt << Rel.Index;
+  } else {
+    Expected<StringRef> SymNameOrErr = SI->getName();
+    if (!SymNameOrErr)
+      return errorToErrorCode(SymNameOrErr.takeError());
+    StringRef SymName = *SymNameOrErr;
+    Result.append(SymName.begin(), SymName.end());
+  }
+  Fmt << (Rel.Addend < 0 ? "" : "+") << Rel.Addend;
+  Fmt.flush();
+  Result.append(FmtBuf.begin(), FmtBuf.end());
+  return std::error_code();
+}
diff --git a/tools/llvm-objdump/llvm-objdump.cpp b/tools/llvm-objdump/llvm-objdump.cpp
index 9a405c6b4665b78d8a22b632e2acd0d52a783e7a..735025bac63740a0ebebfaa845eed09a91f968ed 100644
--- a/tools/llvm-objdump/llvm-objdump.cpp
+++ b/tools/llvm-objdump/llvm-objdump.cpp
@@ -1,9 +1,8 @@
 //===-- llvm-objdump.cpp - Object file dumping utility for llvm -----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -71,11 +70,16 @@
 using namespace llvm;
 using namespace object;
 
+cl::opt<unsigned long long> AdjustVMA(
+    "adjust-vma",
+    cl::desc("Increase the displayed address by the specified offset"),
+    cl::value_desc("offset"), cl::init(0));
+
 cl::opt<bool>
     llvm::AllHeaders("all-headers",
                      cl::desc("Display all available header information"));
 static cl::alias AllHeadersShort("x", cl::desc("Alias for --all-headers"),
-                                 cl::aliasopt(AllHeaders));
+                                 cl::NotHidden, cl::aliasopt(AllHeaders));
 
 static cl::list<std::string>
 InputFilenames(cl::Positional, cl::desc("<input object files>"),cl::ZeroOrMore);
@@ -83,22 +87,20 @@ InputFilenames(cl::Positional, cl::desc("<input object files>"),cl::ZeroOrMore);
 cl::opt<bool>
 llvm::Disassemble("disassemble",
   cl::desc("Display assembler mnemonics for the machine instructions"));
-static cl::alias
-Disassembled("d", cl::desc("Alias for --disassemble"),
-             cl::aliasopt(Disassemble));
+static cl::alias Disassembled("d", cl::desc("Alias for --disassemble"),
+                              cl::NotHidden, cl::aliasopt(Disassemble));
 
 cl::opt<bool>
 llvm::DisassembleAll("disassemble-all",
   cl::desc("Display assembler mnemonics for the machine instructions"));
-static cl::alias
-DisassembleAlld("D", cl::desc("Alias for --disassemble-all"),
-             cl::aliasopt(DisassembleAll));
+static cl::alias DisassembleAlld("D", cl::desc("Alias for --disassemble-all"),
+                                 cl::NotHidden, cl::aliasopt(DisassembleAll));
 
 cl::opt<bool> llvm::Demangle("demangle", cl::desc("Demangle symbols names"),
                              cl::init(false));
 
 static cl::alias DemangleShort("C", cl::desc("Alias for --demangle"),
-                               cl::aliasopt(llvm::Demangle));
+                               cl::NotHidden, cl::aliasopt(llvm::Demangle));
 
 static cl::list<std::string>
 DisassembleFunctions("df",
@@ -116,15 +118,16 @@ static cl::alias RelocationsShort("r", cl::desc("Alias for --reloc"),
 cl::opt<bool>
 llvm::DynamicRelocations("dynamic-reloc",
   cl::desc("Display the dynamic relocation entries in the file"));
-static cl::alias
-DynamicRelocationsd("R", cl::desc("Alias for --dynamic-reloc"),
-             cl::aliasopt(DynamicRelocations));
+static cl::alias DynamicRelocationsd("R", cl::desc("Alias for --dynamic-reloc"),
+                                     cl::NotHidden,
+                                     cl::aliasopt(DynamicRelocations));
 
 cl::opt<bool>
     llvm::SectionContents("full-contents",
                           cl::desc("Display the content of each section"));
 static cl::alias SectionContentsShort("s",
                                       cl::desc("Alias for --full-contents"),
+                                      cl::NotHidden,
                                       cl::aliasopt(SectionContents));
 
 cl::opt<bool> llvm::SymbolTable("syms", cl::desc("Display the symbol table"));
@@ -153,8 +156,8 @@ llvm::RawClangAST("raw-clang-ast",
 
 static cl::opt<bool>
 MachOOpt("macho", cl::desc("Use MachO specific object file parser"));
-static cl::alias
-MachOm("m", cl::desc("Alias for --macho"), cl::aliasopt(MachOOpt));
+static cl::alias MachOm("m", cl::desc("Alias for --macho"), cl::NotHidden,
+                        cl::aliasopt(MachOOpt));
 
 cl::opt<std::string>
 llvm::TripleName("triple", cl::desc("Target triple to disassemble for, "
@@ -173,19 +176,25 @@ llvm::ArchName("arch-name", cl::desc("Target arch to disassemble for, "
 cl::opt<bool>
 llvm::SectionHeaders("section-headers", cl::desc("Display summaries of the "
                                                  "headers for each section."));
-static cl::alias
-SectionHeadersShort("headers", cl::desc("Alias for --section-headers"),
-                    cl::aliasopt(SectionHeaders));
-static cl::alias
-SectionHeadersShorter("h", cl::desc("Alias for --section-headers"),
-                      cl::aliasopt(SectionHeaders));
+static cl::alias SectionHeadersShort("headers",
+                                     cl::desc("Alias for --section-headers"),
+                                     cl::NotHidden,
+                                     cl::aliasopt(SectionHeaders));
+static cl::alias SectionHeadersShorter("h",
+                                       cl::desc("Alias for --section-headers"),
+                                       cl::NotHidden,
+                                       cl::aliasopt(SectionHeaders));
+
+static cl::opt<bool>
+    ShowLMA("show-lma",
+            cl::desc("Display LMA column when dumping ELF section headers"));
 
 cl::list<std::string>
 llvm::FilterSections("section", cl::desc("Operate on the specified sections only. "
                                          "With -macho dump segment,section"));
-cl::alias
-static FilterSectionsj("j", cl::desc("Alias for --section"),
-                 cl::aliasopt(llvm::FilterSections));
+cl::alias static FilterSectionsj("j", cl::desc("Alias for --section"),
+                                 cl::NotHidden,
+                                 cl::aliasopt(llvm::FilterSections));
 
 cl::list<std::string>
 llvm::MAttrs("mattr",
@@ -203,9 +212,8 @@ llvm::NoLeadingAddr("no-leading-addr", cl::desc("Print no leading address"));
 cl::opt<bool>
 llvm::UnwindInfo("unwind-info", cl::desc("Display unwind information"));
 
-static cl::alias
-UnwindInfoShort("u", cl::desc("Alias for --unwind-info"),
-                cl::aliasopt(UnwindInfo));
+static cl::alias UnwindInfoShort("u", cl::desc("Alias for --unwind-info"),
+                                 cl::NotHidden, cl::aliasopt(UnwindInfo));
 
 cl::opt<bool>
 llvm::PrivateHeaders("private-headers",
@@ -216,24 +224,24 @@ llvm::FirstPrivateHeader("private-header",
                          cl::desc("Display only the first format specific file "
                                   "header"));
 
-static cl::alias
-PrivateHeadersShort("p", cl::desc("Alias for --private-headers"),
-                    cl::aliasopt(PrivateHeaders));
+static cl::alias PrivateHeadersShort("p",
+                                     cl::desc("Alias for --private-headers"),
+                                     cl::NotHidden,
+                                     cl::aliasopt(PrivateHeaders));
 
 cl::opt<bool> llvm::FileHeaders(
     "file-headers",
     cl::desc("Display the contents of the overall file header"));
 
 static cl::alias FileHeadersShort("f", cl::desc("Alias for --file-headers"),
-                                  cl::aliasopt(FileHeaders));
+                                  cl::NotHidden, cl::aliasopt(FileHeaders));
 
 cl::opt<bool>
     llvm::ArchiveHeaders("archive-headers",
                          cl::desc("Display archive header information"));
 
-cl::alias
-ArchiveHeadersShort("a", cl::desc("Alias for --archive-headers"),
-                    cl::aliasopt(ArchiveHeaders));
+cl::alias ArchiveHeadersShort("a", cl::desc("Alias for --archive-headers"),
+                              cl::NotHidden, cl::aliasopt(ArchiveHeaders));
 
 cl::opt<bool>
     llvm::PrintImmHex("print-imm-hex",
@@ -251,7 +259,7 @@ cl::opt<bool> PrintSource(
     cl::desc(
         "Display source inlined with disassembly. Implies disassemble object"));
 
-cl::alias PrintSourceShort("S", cl::desc("Alias for -source"),
+cl::alias PrintSourceShort("S", cl::desc("Alias for -source"), cl::NotHidden,
                            cl::aliasopt(PrintSource));
 
 cl::opt<bool> PrintLines("line-numbers",
@@ -259,68 +267,29 @@ cl::opt<bool> PrintLines("line-numbers",
                                   "disassembly. Implies disassemble object"));
 
 cl::alias PrintLinesShort("l", cl::desc("Alias for -line-numbers"),
-                          cl::aliasopt(PrintLines));
+                          cl::NotHidden, cl::aliasopt(PrintLines));
 
 cl::opt<unsigned long long>
     StartAddress("start-address", cl::desc("Disassemble beginning at address"),
                  cl::value_desc("address"), cl::init(0));
 cl::opt<unsigned long long>
-    StopAddress("stop-address", cl::desc("Stop disassembly at address"),
+    StopAddress("stop-address",
+                cl::desc("Stop disassembly at address"),
                 cl::value_desc("address"), cl::init(UINT64_MAX));
-static StringRef ToolName;
-
-typedef std::vector<std::tuple<uint64_t, StringRef, uint8_t>> SectionSymbolsTy;
-
-namespace {
-typedef std::function<bool(llvm::object::SectionRef const &)> FilterPredicate;
 
-class SectionFilterIterator {
-public:
-  SectionFilterIterator(FilterPredicate P,
-                        llvm::object::section_iterator const &I,
-                        llvm::object::section_iterator const &E)
-      : Predicate(std::move(P)), Iterator(I), End(E) {
-    ScanPredicate();
-  }
-  const llvm::object::SectionRef &operator*() const { return *Iterator; }
-  SectionFilterIterator &operator++() {
-    ++Iterator;
-    ScanPredicate();
-    return *this;
-  }
-  bool operator!=(SectionFilterIterator const &Other) const {
-    return Iterator != Other.Iterator;
-  }
+cl::opt<bool> DisassembleZeroes(
+                "disassemble-zeroes",
+                cl::desc("Do not skip blocks of zeroes when disassembling"));
+cl::alias DisassembleZeroesShort("z",
+                                 cl::desc("Alias for --disassemble-zeroes"),
+                                 cl::NotHidden,
+                                 cl::aliasopt(DisassembleZeroes));
 
-private:
-  void ScanPredicate() {
-    while (Iterator != End && !Predicate(*Iterator)) {
-      ++Iterator;
-    }
-  }
-  FilterPredicate Predicate;
-  llvm::object::section_iterator Iterator;
-  llvm::object::section_iterator End;
-};
+static StringRef ToolName;
 
-class SectionFilter {
-public:
-  SectionFilter(FilterPredicate P, llvm::object::ObjectFile const &O)
-      : Predicate(std::move(P)), Object(O) {}
-  SectionFilterIterator begin() {
-    return SectionFilterIterator(Predicate, Object.section_begin(),
-                                 Object.section_end());
-  }
-  SectionFilterIterator end() {
-    return SectionFilterIterator(Predicate, Object.section_end(),
-                                 Object.section_end());
-  }
+typedef std::vector<std::tuple<uint64_t, StringRef, uint8_t>> SectionSymbolsTy;
 
-private:
-  FilterPredicate Predicate;
-  llvm::object::ObjectFile const &Object;
-};
-SectionFilter ToolSectionFilter(llvm::object::ObjectFile const &O) {
+SectionFilter llvm::ToolSectionFilter(llvm::object::ObjectFile const &O) {
   return SectionFilter(
       [](llvm::object::SectionRef const &S) {
         if (FilterSections.empty())
@@ -333,7 +302,6 @@ SectionFilter ToolSectionFilter(llvm::object::ObjectFile const &O) {
       },
       O);
 }
-}
 
 void llvm::error(std::error_code EC) {
   if (!EC)
@@ -421,18 +389,16 @@ static const Target *getTarget(const ObjectFile *Obj = nullptr) {
   // Figure out the target triple.
   llvm::Triple TheTriple("unknown-unknown-unknown");
   if (TripleName.empty()) {
-    if (Obj) {
+    if (Obj)
       TheTriple = Obj->makeTriple();
-    }
   } else {
     TheTriple.setTriple(Triple::normalize(TripleName));
 
     // Use the triple, but also try to combine with ARM build attributes.
     if (Obj) {
       auto Arch = Obj->getArch();
-      if (Arch == Triple::arm || Arch == Triple::armeb) {
+      if (Arch == Triple::arm || Arch == Triple::armeb)
         Obj->setARMSubArch(TheTriple);
-      }
     }
   }
 
@@ -452,410 +418,21 @@ static const Target *getTarget(const ObjectFile *Obj = nullptr) {
   return TheTarget;
 }
 
-bool llvm::RelocAddressLess(RelocationRef a, RelocationRef b) {
-  return a.getOffset() < b.getOffset();
-}
-
-template <class ELFT>
-static std::error_code getRelocationValueString(const ELFObjectFile<ELFT> *Obj,
-                                                const RelocationRef &RelRef,
-                                                SmallVectorImpl<char> &Result) {
-  DataRefImpl Rel = RelRef.getRawDataRefImpl();
-
-  typedef typename ELFObjectFile<ELFT>::Elf_Sym Elf_Sym;
-  typedef typename ELFObjectFile<ELFT>::Elf_Shdr Elf_Shdr;
-  typedef typename ELFObjectFile<ELFT>::Elf_Rela Elf_Rela;
-
-  const ELFFile<ELFT> &EF = *Obj->getELFFile();
-
-  auto SecOrErr = EF.getSection(Rel.d.a);
-  if (!SecOrErr)
-    return errorToErrorCode(SecOrErr.takeError());
-  const Elf_Shdr *Sec = *SecOrErr;
-  auto SymTabOrErr = EF.getSection(Sec->sh_link);
-  if (!SymTabOrErr)
-    return errorToErrorCode(SymTabOrErr.takeError());
-  const Elf_Shdr *SymTab = *SymTabOrErr;
-  assert(SymTab->sh_type == ELF::SHT_SYMTAB ||
-         SymTab->sh_type == ELF::SHT_DYNSYM);
-  auto StrTabSec = EF.getSection(SymTab->sh_link);
-  if (!StrTabSec)
-    return errorToErrorCode(StrTabSec.takeError());
-  auto StrTabOrErr = EF.getStringTable(*StrTabSec);
-  if (!StrTabOrErr)
-    return errorToErrorCode(StrTabOrErr.takeError());
-  StringRef StrTab = *StrTabOrErr;
-  int64_t addend = 0;
-  // If there is no Symbol associated with the relocation, we set the undef
-  // boolean value to 'true'. This will prevent us from calling functions that
-  // requires the relocation to be associated with a symbol.
-  bool undef = false;
-  switch (Sec->sh_type) {
-  default:
-    return object_error::parse_failed;
-  case ELF::SHT_REL: {
-    // TODO: Read implicit addend from section data.
-    break;
-  }
-  case ELF::SHT_RELA: {
-    const Elf_Rela *ERela = Obj->getRela(Rel);
-    addend = ERela->r_addend;
-    undef = ERela->getSymbol(false) == 0;
-    break;
-  }
-  }
-  StringRef Target;
-  if (!undef) {
-    symbol_iterator SI = RelRef.getSymbol();
-    const Elf_Sym *symb = Obj->getSymbol(SI->getRawDataRefImpl());
-    if (symb->getType() == ELF::STT_SECTION) {
-      Expected<section_iterator> SymSI = SI->getSection();
-      if (!SymSI)
-        return errorToErrorCode(SymSI.takeError());
-      const Elf_Shdr *SymSec = Obj->getSection((*SymSI)->getRawDataRefImpl());
-      auto SecName = EF.getSectionName(SymSec);
-      if (!SecName)
-        return errorToErrorCode(SecName.takeError());
-      Target = *SecName;
-    } else {
-      Expected<StringRef> SymName = symb->getName(StrTab);
-      if (!SymName)
-        return errorToErrorCode(SymName.takeError());
-      Target = *SymName;
-    }
-  } else
-    Target = "*ABS*";
-
-  // Default scheme is to print Target, as well as "+ <addend>" for nonzero
-  // addend. Should be acceptable for all normal purposes.
-  std::string fmtbuf;
-  raw_string_ostream fmt(fmtbuf);
-  fmt << Target;
-  if (addend != 0)
-    fmt << (addend < 0 ? "" : "+") << addend;
-  fmt.flush();
-  Result.append(fmtbuf.begin(), fmtbuf.end());
-  return std::error_code();
-}
-
-static std::error_code getRelocationValueString(const ELFObjectFileBase *Obj,
-                                                const RelocationRef &Rel,
-                                                SmallVectorImpl<char> &Result) {
-  if (auto *ELF32LE = dyn_cast<ELF32LEObjectFile>(Obj))
-    return getRelocationValueString(ELF32LE, Rel, Result);
-  if (auto *ELF64LE = dyn_cast<ELF64LEObjectFile>(Obj))
-    return getRelocationValueString(ELF64LE, Rel, Result);
-  if (auto *ELF32BE = dyn_cast<ELF32BEObjectFile>(Obj))
-    return getRelocationValueString(ELF32BE, Rel, Result);
-  auto *ELF64BE = cast<ELF64BEObjectFile>(Obj);
-  return getRelocationValueString(ELF64BE, Rel, Result);
-}
-
-static std::error_code getRelocationValueString(const COFFObjectFile *Obj,
-                                                const RelocationRef &Rel,
-                                                SmallVectorImpl<char> &Result) {
-  symbol_iterator SymI = Rel.getSymbol();
-  Expected<StringRef> SymNameOrErr = SymI->getName();
-  if (!SymNameOrErr)
-    return errorToErrorCode(SymNameOrErr.takeError());
-  StringRef SymName = *SymNameOrErr;
-  Result.append(SymName.begin(), SymName.end());
-  return std::error_code();
-}
-
-static void printRelocationTargetName(const MachOObjectFile *O,
-                                      const MachO::any_relocation_info &RE,
-                                      raw_string_ostream &fmt) {
-  bool IsScattered = O->isRelocationScattered(RE);
-
-  // Target of a scattered relocation is an address.  In the interest of
-  // generating pretty output, scan through the symbol table looking for a
-  // symbol that aligns with that address.  If we find one, print it.
-  // Otherwise, we just print the hex address of the target.
-  if (IsScattered) {
-    uint32_t Val = O->getPlainRelocationSymbolNum(RE);
-
-    for (const SymbolRef &Symbol : O->symbols()) {
-      std::error_code ec;
-      Expected<uint64_t> Addr = Symbol.getAddress();
-      if (!Addr)
-        report_error(O->getFileName(), Addr.takeError());
-      if (*Addr != Val)
-        continue;
-      Expected<StringRef> Name = Symbol.getName();
-      if (!Name)
-        report_error(O->getFileName(), Name.takeError());
-      fmt << *Name;
-      return;
-    }
-
-    // If we couldn't find a symbol that this relocation refers to, try
-    // to find a section beginning instead.
-    for (const SectionRef &Section : ToolSectionFilter(*O)) {
-      std::error_code ec;
-
-      StringRef Name;
-      uint64_t Addr = Section.getAddress();
-      if (Addr != Val)
-        continue;
-      if ((ec = Section.getName(Name)))
-        report_error(O->getFileName(), ec);
-      fmt << Name;
-      return;
-    }
-
-    fmt << format("0x%x", Val);
-    return;
-  }
-
-  StringRef S;
-  bool isExtern = O->getPlainRelocationExternal(RE);
-  uint64_t Val = O->getPlainRelocationSymbolNum(RE);
-
-  if (O->getAnyRelocationType(RE) == MachO::ARM64_RELOC_ADDEND) {
-    fmt << format("0x%0" PRIx64, Val);
-    return;
-  } else if (isExtern) {
-    symbol_iterator SI = O->symbol_begin();
-    advance(SI, Val);
-    Expected<StringRef> SOrErr = SI->getName();
-    if (!SOrErr)
-      report_error(O->getFileName(), SOrErr.takeError());
-    S = *SOrErr;
-  } else {
-    section_iterator SI = O->section_begin();
-    // Adjust for the fact that sections are 1-indexed.
-    if (Val == 0) {
-      fmt << "0 (?,?)";
-      return;
-    }
-    uint32_t i = Val - 1;
-    while (i != 0 && SI != O->section_end()) {
-      i--;
-      advance(SI, 1);
-    }
-    if (SI == O->section_end())
-      fmt << Val << " (?,?)";
-    else
-      SI->getName(S);
-  }
-
-  fmt << S;
-}
-
-static std::error_code getRelocationValueString(const WasmObjectFile *Obj,
-                                                const RelocationRef &RelRef,
-                                                SmallVectorImpl<char> &Result) {
-  const wasm::WasmRelocation& Rel = Obj->getWasmRelocation(RelRef);
-  symbol_iterator SI = RelRef.getSymbol();
-  std::string fmtbuf;
-  raw_string_ostream fmt(fmtbuf);
-  if (SI == Obj->symbol_end()) {
-    // Not all wasm relocations have symbols associated with them.
-    // In particular R_WEBASSEMBLY_TYPE_INDEX_LEB.
-    fmt << Rel.Index;
-  } else {
-    Expected<StringRef> SymNameOrErr = SI->getName();
-    if (!SymNameOrErr)
-      return errorToErrorCode(SymNameOrErr.takeError());
-    StringRef SymName = *SymNameOrErr;
-    Result.append(SymName.begin(), SymName.end());
-  }
-  fmt << (Rel.Addend < 0 ? "" : "+") << Rel.Addend;
-  fmt.flush();
-  Result.append(fmtbuf.begin(), fmtbuf.end());
-  return std::error_code();
-}
-
-static std::error_code getRelocationValueString(const MachOObjectFile *Obj,
-                                                const RelocationRef &RelRef,
-                                                SmallVectorImpl<char> &Result) {
-  DataRefImpl Rel = RelRef.getRawDataRefImpl();
-  MachO::any_relocation_info RE = Obj->getRelocation(Rel);
-
-  unsigned Arch = Obj->getArch();
-
-  std::string fmtbuf;
-  raw_string_ostream fmt(fmtbuf);
-  unsigned Type = Obj->getAnyRelocationType(RE);
-  bool IsPCRel = Obj->getAnyRelocationPCRel(RE);
-
-  // Determine any addends that should be displayed with the relocation.
-  // These require decoding the relocation type, which is triple-specific.
-
-  // X86_64 has entirely custom relocation types.
-  if (Arch == Triple::x86_64) {
-    bool isPCRel = Obj->getAnyRelocationPCRel(RE);
-
-    switch (Type) {
-    case MachO::X86_64_RELOC_GOT_LOAD:
-    case MachO::X86_64_RELOC_GOT: {
-      printRelocationTargetName(Obj, RE, fmt);
-      fmt << "@GOT";
-      if (isPCRel)
-        fmt << "PCREL";
-      break;
-    }
-    case MachO::X86_64_RELOC_SUBTRACTOR: {
-      DataRefImpl RelNext = Rel;
-      Obj->moveRelocationNext(RelNext);
-      MachO::any_relocation_info RENext = Obj->getRelocation(RelNext);
-
-      // X86_64_RELOC_SUBTRACTOR must be followed by a relocation of type
-      // X86_64_RELOC_UNSIGNED.
-      // NOTE: Scattered relocations don't exist on x86_64.
-      unsigned RType = Obj->getAnyRelocationType(RENext);
-      if (RType != MachO::X86_64_RELOC_UNSIGNED)
-        report_error(Obj->getFileName(), "Expected X86_64_RELOC_UNSIGNED after "
-                     "X86_64_RELOC_SUBTRACTOR.");
-
-      // The X86_64_RELOC_UNSIGNED contains the minuend symbol;
-      // X86_64_RELOC_SUBTRACTOR contains the subtrahend.
-      printRelocationTargetName(Obj, RENext, fmt);
-      fmt << "-";
-      printRelocationTargetName(Obj, RE, fmt);
-      break;
-    }
-    case MachO::X86_64_RELOC_TLV:
-      printRelocationTargetName(Obj, RE, fmt);
-      fmt << "@TLV";
-      if (isPCRel)
-        fmt << "P";
-      break;
-    case MachO::X86_64_RELOC_SIGNED_1:
-      printRelocationTargetName(Obj, RE, fmt);
-      fmt << "-1";
-      break;
-    case MachO::X86_64_RELOC_SIGNED_2:
-      printRelocationTargetName(Obj, RE, fmt);
-      fmt << "-2";
-      break;
-    case MachO::X86_64_RELOC_SIGNED_4:
-      printRelocationTargetName(Obj, RE, fmt);
-      fmt << "-4";
-      break;
-    default:
-      printRelocationTargetName(Obj, RE, fmt);
-      break;
-    }
-    // X86 and ARM share some relocation types in common.
-  } else if (Arch == Triple::x86 || Arch == Triple::arm ||
-             Arch == Triple::ppc) {
-    // Generic relocation types...
-    switch (Type) {
-    case MachO::GENERIC_RELOC_PAIR: // prints no info
-      return std::error_code();
-    case MachO::GENERIC_RELOC_SECTDIFF: {
-      DataRefImpl RelNext = Rel;
-      Obj->moveRelocationNext(RelNext);
-      MachO::any_relocation_info RENext = Obj->getRelocation(RelNext);
-
-      // X86 sect diff's must be followed by a relocation of type
-      // GENERIC_RELOC_PAIR.
-      unsigned RType = Obj->getAnyRelocationType(RENext);
-
-      if (RType != MachO::GENERIC_RELOC_PAIR)
-        report_error(Obj->getFileName(), "Expected GENERIC_RELOC_PAIR after "
-                     "GENERIC_RELOC_SECTDIFF.");
-
-      printRelocationTargetName(Obj, RE, fmt);
-      fmt << "-";
-      printRelocationTargetName(Obj, RENext, fmt);
-      break;
-    }
-    }
-
-    if (Arch == Triple::x86 || Arch == Triple::ppc) {
-      switch (Type) {
-      case MachO::GENERIC_RELOC_LOCAL_SECTDIFF: {
-        DataRefImpl RelNext = Rel;
-        Obj->moveRelocationNext(RelNext);
-        MachO::any_relocation_info RENext = Obj->getRelocation(RelNext);
-
-        // X86 sect diff's must be followed by a relocation of type
-        // GENERIC_RELOC_PAIR.
-        unsigned RType = Obj->getAnyRelocationType(RENext);
-        if (RType != MachO::GENERIC_RELOC_PAIR)
-          report_error(Obj->getFileName(), "Expected GENERIC_RELOC_PAIR after "
-                       "GENERIC_RELOC_LOCAL_SECTDIFF.");
-
-        printRelocationTargetName(Obj, RE, fmt);
-        fmt << "-";
-        printRelocationTargetName(Obj, RENext, fmt);
-        break;
-      }
-      case MachO::GENERIC_RELOC_TLV: {
-        printRelocationTargetName(Obj, RE, fmt);
-        fmt << "@TLV";
-        if (IsPCRel)
-          fmt << "P";
-        break;
-      }
-      default:
-        printRelocationTargetName(Obj, RE, fmt);
-      }
-    } else { // ARM-specific relocations
-      switch (Type) {
-      case MachO::ARM_RELOC_HALF:
-      case MachO::ARM_RELOC_HALF_SECTDIFF: {
-        // Half relocations steal a bit from the length field to encode
-        // whether this is an upper16 or a lower16 relocation.
-        bool isUpper = (Obj->getAnyRelocationLength(RE) & 0x1) == 1;
-
-        if (isUpper)
-          fmt << ":upper16:(";
-        else
-          fmt << ":lower16:(";
-        printRelocationTargetName(Obj, RE, fmt);
-
-        DataRefImpl RelNext = Rel;
-        Obj->moveRelocationNext(RelNext);
-        MachO::any_relocation_info RENext = Obj->getRelocation(RelNext);
-
-        // ARM half relocs must be followed by a relocation of type
-        // ARM_RELOC_PAIR.
-        unsigned RType = Obj->getAnyRelocationType(RENext);
-        if (RType != MachO::ARM_RELOC_PAIR)
-          report_error(Obj->getFileName(), "Expected ARM_RELOC_PAIR after "
-                       "ARM_RELOC_HALF");
-
-        // NOTE: The half of the target virtual address is stashed in the
-        // address field of the secondary relocation, but we can't reverse
-        // engineer the constant offset from it without decoding the movw/movt
-        // instruction to find the other half in its immediate field.
-
-        // ARM_RELOC_HALF_SECTDIFF encodes the second section in the
-        // symbol/section pointer of the follow-on relocation.
-        if (Type == MachO::ARM_RELOC_HALF_SECTDIFF) {
-          fmt << "-";
-          printRelocationTargetName(Obj, RENext, fmt);
-        }
-
-        fmt << ")";
-        break;
-      }
-      default: { printRelocationTargetName(Obj, RE, fmt); }
-      }
-    }
-  } else
-    printRelocationTargetName(Obj, RE, fmt);
-
-  fmt.flush();
-  Result.append(fmtbuf.begin(), fmtbuf.end());
-  return std::error_code();
+bool llvm::isRelocAddressLess(RelocationRef A, RelocationRef B) {
+  return A.getOffset() < B.getOffset();
 }
 
 static std::error_code getRelocationValueString(const RelocationRef &Rel,
                                                 SmallVectorImpl<char> &Result) {
   const ObjectFile *Obj = Rel.getObject();
   if (auto *ELF = dyn_cast<ELFObjectFileBase>(Obj))
-    return getRelocationValueString(ELF, Rel, Result);
+    return getELFRelocationValueString(ELF, Rel, Result);
   if (auto *COFF = dyn_cast<COFFObjectFile>(Obj))
-    return getRelocationValueString(COFF, Rel, Result);
+    return getCOFFRelocationValueString(COFF, Rel, Result);
   if (auto *Wasm = dyn_cast<WasmObjectFile>(Obj))
-    return getRelocationValueString(Wasm, Rel, Result);
+    return getWasmRelocationValueString(Wasm, Rel, Result);
   if (auto *MachO = dyn_cast<MachOObjectFile>(Obj))
-    return getRelocationValueString(MachO, Rel, Result);
+    return getMachORelocationValueString(MachO, Rel, Result);
   llvm_unreachable("unknown object file format");
 }
 
@@ -863,8 +440,7 @@ static std::error_code getRelocationValueString(const RelocationRef &Rel,
 /// relocations, usually because it is the trailing part of a multipart
 /// relocation that will be printed as part of the leading relocation.
 static bool getHidden(RelocationRef RelRef) {
-  const ObjectFile *Obj = RelRef.getObject();
-  auto *MachO = dyn_cast<MachOObjectFile>(Obj);
+  auto *MachO = dyn_cast<MachOObjectFile>(RelRef.getObject());
   if (!MachO)
     return false;
 
@@ -874,10 +450,10 @@ static bool getHidden(RelocationRef RelRef) {
 
   // On arches that use the generic relocations, GENERIC_RELOC_PAIR
   // is always hidden.
-  if (Arch == Triple::x86 || Arch == Triple::arm || Arch == Triple::ppc) {
-    if (Type == MachO::GENERIC_RELOC_PAIR)
-      return true;
-  } else if (Arch == Triple::x86_64) {
+  if (Arch == Triple::x86 || Arch == Triple::arm || Arch == Triple::ppc)
+    return Type == MachO::GENERIC_RELOC_PAIR;
+
+  if (Arch == Triple::x86_64) {
     // On x86_64, X86_64_RELOC_UNSIGNED is hidden only when it follows
     // an X86_64_RELOC_SUBTRACTOR.
     if (Type == MachO::X86_64_RELOC_UNSIGNED && Rel.d.a > 0) {
@@ -991,6 +567,17 @@ static bool isArmElf(const ObjectFile *Obj) {
            Obj->getArch() == Triple::thumbeb));
 }
 
+static void printRelocation(const RelocationRef &Rel, uint64_t Address,
+                            uint8_t AddrSize) {
+  StringRef Fmt =
+      AddrSize > 4 ? "\t\t%016" PRIx64 ":  " : "\t\t\t%08" PRIx64 ":  ";
+  SmallString<16> Name;
+  SmallString<32> Val;
+  Rel.getTypeName(Name);
+  error(getRelocationValueString(Rel, Val));
+  outs() << format(Fmt.data(), Address) << Name << "\t" << Val << "\n";
+}
+
 class PrettyPrinter {
 public:
   virtual ~PrettyPrinter() = default;
@@ -1051,28 +638,22 @@ public:
     auto HeadTail = PacketBundle.first.split('\n');
     auto Preamble = " { ";
     auto Separator = "";
-    StringRef Fmt = "\t\t\t%08" PRIx64 ":  ";
-    std::vector<RelocationRef>::const_iterator rel_cur = Rels->begin();
-    std::vector<RelocationRef>::const_iterator rel_end = Rels->end();
 
     // Hexagon's packets require relocations to be inline rather than
     // clustered at the end of the packet.
+    std::vector<RelocationRef>::const_iterator RelCur = Rels->begin();
+    std::vector<RelocationRef>::const_iterator RelEnd = Rels->end();
     auto PrintReloc = [&]() -> void {
-      while ((rel_cur != rel_end) && (rel_cur->getOffset() <= Address)) {
-        if (rel_cur->getOffset() == Address) {
-          SmallString<16> name;
-          SmallString<32> val;
-          rel_cur->getTypeName(name);
-          error(getRelocationValueString(*rel_cur, val));
-          OS << Separator << format(Fmt.data(), Address) << name << "\t" << val
-                << "\n";
+      while ((RelCur != RelEnd) && (RelCur->getOffset() <= Address)) {
+        if (RelCur->getOffset() == Address) {
+          printRelocation(*RelCur, Address, 4);
           return;
         }
-        rel_cur++;
+        ++RelCur;
       }
     };
 
-    while(!HeadTail.first.empty()) {
+    while (!HeadTail.first.empty()) {
       OS << Separator;
       Separator = "\n";
       if (SP && (PrintSource || PrintLines))
@@ -1082,7 +663,7 @@ public:
       Preamble = "   ";
       StringRef Inst;
       auto Duplex = HeadTail.first.split('\v');
-      if(!Duplex.second.empty()){
+      if (!Duplex.second.empty()) {
         OS << Duplex.first;
         OS << "; ";
         Inst = Duplex.second;
@@ -1214,7 +795,6 @@ addDynamicElfSymbols(const ELFObjectFile<ELFT> *Obj,
     Expected<uint64_t> AddressOrErr = Symbol.getAddress();
     if (!AddressOrErr)
       report_error(Obj->getFileName(), AddressOrErr.takeError());
-    uint64_t Address = *AddressOrErr;
 
     Expected<StringRef> Name = Symbol.getName();
     if (!Name)
@@ -1229,7 +809,7 @@ addDynamicElfSymbols(const ELFObjectFile<ELFT> *Obj,
     if (SecI == Obj->section_end())
       continue;
 
-    AllSymbols[*SecI].emplace_back(Address, *Name, SymbolType);
+    AllSymbols[*SecI].emplace_back(*AddressOrErr, *Name, SymbolType);
   }
 }
 
@@ -1265,7 +845,6 @@ static void addPltEntries(const ObjectFile *Obj,
   if (auto *ElfObj = dyn_cast<ELFObjectFileBase>(Obj)) {
     for (auto PltEntry : ElfObj->getPltAddresses()) {
       SymbolRef Symbol(PltEntry.first, ElfObj);
-
       uint8_t SymbolType = getElfSymbolType(Obj, Symbol);
 
       Expected<StringRef> NameOrErr = Symbol.getName();
@@ -1280,77 +859,66 @@ static void addPltEntries(const ObjectFile *Obj,
   }
 }
 
-static void DisassembleObject(const ObjectFile *Obj, bool InlineRelocs) {
-  if (StartAddress > StopAddress)
-    error("Start address should be less than stop address");
-
-  const Target *TheTarget = getTarget(Obj);
+// Normally the disassembly output will skip blocks of zeroes. This function
+// returns the number of zero bytes that can be skipped when dumping the
+// disassembly of the instructions in Buf.
+static size_t countSkippableZeroBytes(ArrayRef<uint8_t> Buf) {
+  // When -z or --disassemble-zeroes are given we always dissasemble them.
+  if (DisassembleZeroes)
+    return 0;
+
+  // Find the number of leading zeroes.
+  size_t N = 0;
+  while (N < Buf.size() && !Buf[N])
+    ++N;
+
+  // We may want to skip blocks of zero bytes, but unless we see
+  // at least 8 of them in a row.
+  if (N < 8)
+    return 0;
+
+  // We skip zeroes in multiples of 4 because do not want to truncate an
+  // instruction if it starts with a zero byte.
+  return N & ~0x3;
+}
 
-  // Package up features to be passed to target/subtarget
-  SubtargetFeatures Features = Obj->getFeatures();
-  if (MAttrs.size()) {
-    for (unsigned i = 0; i != MAttrs.size(); ++i)
-      Features.AddFeature(MAttrs[i]);
+// Returns a map from sections to their relocations.
+static std::map<SectionRef, std::vector<RelocationRef>>
+getRelocsMap(llvm::object::ObjectFile const &Obj) {
+  std::map<SectionRef, std::vector<RelocationRef>> Ret;
+  for (const SectionRef &Section : ToolSectionFilter(Obj)) {
+    section_iterator RelSec = Section.getRelocatedSection();
+    if (RelSec == Obj.section_end())
+      continue;
+    std::vector<RelocationRef> &V = Ret[*RelSec];
+    for (const RelocationRef &R : Section.relocations())
+      V.push_back(R);
+    // Sort relocations by address.
+    llvm::sort(V, isRelocAddressLess);
   }
+  return Ret;
+}
 
-  std::unique_ptr<const MCRegisterInfo> MRI(
-      TheTarget->createMCRegInfo(TripleName));
-  if (!MRI)
-    report_error(Obj->getFileName(), "no register info for target " +
-                 TripleName);
-
-  // Set up disassembler.
-  std::unique_ptr<const MCAsmInfo> AsmInfo(
-      TheTarget->createMCAsmInfo(*MRI, TripleName));
-  if (!AsmInfo)
-    report_error(Obj->getFileName(), "no assembly info for target " +
-                 TripleName);
-  std::unique_ptr<const MCSubtargetInfo> STI(
-      TheTarget->createMCSubtargetInfo(TripleName, MCPU, Features.getString()));
-  if (!STI)
-    report_error(Obj->getFileName(), "no subtarget info for target " +
-                 TripleName);
-  std::unique_ptr<const MCInstrInfo> MII(TheTarget->createMCInstrInfo());
-  if (!MII)
-    report_error(Obj->getFileName(), "no instruction info for target " +
-                 TripleName);
-  MCObjectFileInfo MOFI;
-  MCContext Ctx(AsmInfo.get(), MRI.get(), &MOFI);
-  // FIXME: for now initialize MCObjectFileInfo with default values
-  MOFI.InitMCObjectFileInfo(Triple(TripleName), false, Ctx);
-
-  std::unique_ptr<MCDisassembler> DisAsm(
-    TheTarget->createMCDisassembler(*STI, Ctx));
-  if (!DisAsm)
-    report_error(Obj->getFileName(), "no disassembler for target " +
-                 TripleName);
-
-  std::unique_ptr<const MCInstrAnalysis> MIA(
-      TheTarget->createMCInstrAnalysis(MII.get()));
-
-  int AsmPrinterVariant = AsmInfo->getAssemblerDialect();
-  std::unique_ptr<MCInstPrinter> IP(TheTarget->createMCInstPrinter(
-      Triple(TripleName), AsmPrinterVariant, *AsmInfo, *MII, *MRI));
-  if (!IP)
-    report_error(Obj->getFileName(), "no instruction printer for target " +
-                 TripleName);
-  IP->setPrintImmHex(PrintImmHex);
-  PrettyPrinter &PIP = selectPrettyPrinter(Triple(TripleName));
-
-  StringRef Fmt = Obj->getBytesInAddress() > 4 ? "\t\t%016" PRIx64 ":  " :
-                                                 "\t\t\t%08" PRIx64 ":  ";
-
-  SourcePrinter SP(Obj, TheTarget->getName());
+// Used for --adjust-vma to check if address should be adjusted by the
+// specified value for a given section.
+// For ELF we do not adjust non-allocatable sections like debug ones,
+// because they are not loadable.
+// TODO: implement for other file formats.
+static bool shouldAdjustVA(const SectionRef &Section) {
+  const ObjectFile *Obj = Section.getObject();
+  if (isa<object::ELFObjectFileBase>(Obj))
+    return ELFSectionRef(Section).getFlags() & ELF::SHF_ALLOC;
+  return false;
+}
 
-  // Create a mapping, RelocSecs = SectionRelocMap[S], where sections
-  // in RelocSecs contain the relocations for section S.
-  std::error_code EC;
-  std::map<SectionRef, SmallVector<SectionRef, 1>> SectionRelocMap;
-  for (const SectionRef &Section : ToolSectionFilter(*Obj)) {
-    section_iterator Sec2 = Section.getRelocatedSection();
-    if (Sec2 != Obj->section_end())
-      SectionRelocMap[*Sec2].push_back(Section);
-  }
+static void disassembleObject(const Target *TheTarget, const ObjectFile *Obj,
+                              MCContext &Ctx, MCDisassembler *DisAsm,
+                              const MCInstrAnalysis *MIA, MCInstPrinter *IP,
+                              const MCSubtargetInfo *STI, PrettyPrinter &PIP,
+                              SourcePrinter &SP, bool InlineRelocs) {
+  std::map<SectionRef, std::vector<RelocationRef>> RelocMap;
+  if (InlineRelocs)
+    RelocMap = getRelocsMap(*Obj);
 
   // Create a mapping from virtual address to symbol name.  This is used to
   // pretty print the symbols while disassembling.
@@ -1373,8 +941,11 @@ static void DisassembleObject(const ObjectFile *Obj, bool InlineRelocs) {
       report_error(Obj->getFileName(), SectionOrErr.takeError());
 
     uint8_t SymbolType = ELF::STT_NOTYPE;
-    if (Obj->isELF())
+    if (Obj->isELF()) {
       SymbolType = getElfSymbolType(Obj, Symbol);
+      if (SymbolType == ELF::STT_SECTION)
+        continue;
+    }
 
     section_iterator SecI = *SectionOrErr;
     if (SecI != Obj->section_end())
@@ -1475,19 +1046,6 @@ static void DisassembleObject(const ObjectFile *Obj, bool InlineRelocs) {
       }
     }
 
-    // Make a list of all the relocations for this section.
-    std::vector<RelocationRef> Rels;
-    if (InlineRelocs) {
-      for (const SectionRef &RelocSec : SectionRelocMap[Section]) {
-        for (const RelocationRef &Reloc : RelocSec.relocations()) {
-          Rels.push_back(Reloc);
-        }
-      }
-    }
-
-    // Sort relocations by address.
-    llvm::sort(Rels, RelocAddressLess);
-
     StringRef SegmentName = "";
     if (const MachOObjectFile *MachO = dyn_cast<const MachOObjectFile>(Obj)) {
       DataRefImpl DR = Section.getRawDataRefImpl();
@@ -1512,19 +1070,24 @@ static void DisassembleObject(const ObjectFile *Obj, bool InlineRelocs) {
     ArrayRef<uint8_t> Bytes(reinterpret_cast<const uint8_t *>(BytesStr.data()),
                             BytesStr.size());
 
+    uint64_t VMAAdjustment = 0;
+    if (shouldAdjustVA(Section))
+      VMAAdjustment = AdjustVMA;
+
     uint64_t Size;
     uint64_t Index;
     bool PrintedSection = false;
-
-    std::vector<RelocationRef>::const_iterator rel_cur = Rels.begin();
-    std::vector<RelocationRef>::const_iterator rel_end = Rels.end();
+    std::vector<RelocationRef> Rels = RelocMap[Section];
+    std::vector<RelocationRef>::const_iterator RelCur = Rels.begin();
+    std::vector<RelocationRef>::const_iterator RelEnd = Rels.end();
     // Disassemble symbol by symbol.
-    for (unsigned si = 0, se = Symbols.size(); si != se; ++si) {
-      uint64_t Start = std::get<0>(Symbols[si]) - SectionAddr;
+    for (unsigned SI = 0, SE = Symbols.size(); SI != SE; ++SI) {
+      uint64_t Start = std::get<0>(Symbols[SI]) - SectionAddr;
       // The end is either the section end or the beginning of the next
       // symbol.
-      uint64_t End =
-          (si == se - 1) ? SectSize : std::get<0>(Symbols[si + 1]) - SectionAddr;
+      uint64_t End = (SI == SE - 1)
+                         ? SectSize
+                         : std::get<0>(Symbols[SI + 1]) - SectionAddr;
       // Don't try to disassemble beyond the end of section contents.
       if (End > SectSize)
         End = SectSize;
@@ -1541,7 +1104,7 @@ static void DisassembleObject(const ObjectFile *Obj, bool InlineRelocs) {
 
       /// Skip if user requested specific symbols and this is not in the list
       if (!DisasmFuncsSet.empty() &&
-          !DisasmFuncsSet.count(std::get<1>(Symbols[si])))
+          !DisasmFuncsSet.count(std::get<1>(Symbols[SI])))
         continue;
 
       if (!PrintedSection) {
@@ -1557,12 +1120,12 @@ static void DisassembleObject(const ObjectFile *Obj, bool InlineRelocs) {
         End = StopAddress - SectionAddr;
 
       if (Obj->isELF() && Obj->getArch() == Triple::amdgcn) {
-        if (std::get<2>(Symbols[si]) == ELF::STT_AMDGPU_HSA_KERNEL) {
+        if (std::get<2>(Symbols[SI]) == ELF::STT_AMDGPU_HSA_KERNEL) {
           // skip amd_kernel_code_t at the begining of kernel symbol (256 bytes)
           Start += 256;
         }
-        if (si == se - 1 ||
-            std::get<2>(Symbols[si + 1]) == ELF::STT_AMDGPU_HSA_KERNEL) {
+        if (SI == SE - 1 ||
+            std::get<2>(Symbols[SI + 1]) == ELF::STT_AMDGPU_HSA_KERNEL) {
           // cut trailing zeroes at the end of kernel
           // cut up to 256 bytes
           const uint64_t EndAlign = 256;
@@ -1573,30 +1136,16 @@ static void DisassembleObject(const ObjectFile *Obj, bool InlineRelocs) {
         }
       }
 
-      auto PrintSymbol = [](StringRef Name) {
-        outs() << '\n' << Name << ":\n";
-      };
-      StringRef SymbolName = std::get<1>(Symbols[si]);
-      if (Demangle) {
-        char *DemangledSymbol = nullptr;
-        size_t Size = 0;
-        int Status = -1;
-        if (SymbolName.startswith("_Z"))
-          DemangledSymbol = itaniumDemangle(SymbolName.data(), DemangledSymbol,
-                                            &Size, &Status);
-        else if (SymbolName.startswith("?"))
-          DemangledSymbol = microsoftDemangle(SymbolName.data(),
-                                              DemangledSymbol, &Size, &Status);
-
-        if (Status == 0 && DemangledSymbol)
-          PrintSymbol(StringRef(DemangledSymbol));
-        else
-          PrintSymbol(SymbolName);
+      outs() << '\n';
+      if (!NoLeadingAddr)
+        outs() << format("%016" PRIx64 " ",
+                         SectionAddr + Start + VMAAdjustment);
 
-        if (DemangledSymbol)
-          free(DemangledSymbol);
-      } else
-        PrintSymbol(SymbolName);
+      StringRef SymbolName = std::get<1>(Symbols[SI]);
+      if (Demangle)
+        outs() << demangle(SymbolName) << ":\n";
+      else
+        outs() << SymbolName << ":\n";
 
       // Don't print raw contents of a virtual section. A virtual section
       // doesn't have any contents in the file.
@@ -1611,6 +1160,12 @@ static void DisassembleObject(const ObjectFile *Obj, bool InlineRelocs) {
       raw_ostream &DebugOut = nulls();
 #endif
 
+      // Some targets (like WebAssembly) have a special prelude at the start
+      // of each symbol.
+      DisAsm->onSymbolStart(SymbolName, Size, Bytes.slice(Start, End - Start),
+                            SectionAddr + Start, DebugOut, CommentStream);
+      Start += Size;
+
       for (Index = Start; Index < End; Index += Size) {
         MCInst Inst;
 
@@ -1624,7 +1179,7 @@ static void DisassembleObject(const ObjectFile *Obj, bool InlineRelocs) {
         // same section. We rely on the markers introduced to
         // understand what we need to dump. If the data marker is within a
         // function, it is denoted as a word/short etc
-        if (isArmElf(Obj) && std::get<2>(Symbols[si]) != ELF::STT_OBJECT &&
+        if (isArmElf(Obj) && std::get<2>(Symbols[SI]) != ELF::STT_OBJECT &&
             !DisassembleAll) {
           uint64_t Stride = 0;
 
@@ -1688,7 +1243,7 @@ static void DisassembleObject(const ObjectFile *Obj, bool InlineRelocs) {
         // disassembling text (applicable all architectures),
         // we are in a situation where we must print the data and not
         // disassemble it.
-        if (Obj->isELF() && std::get<2>(Symbols[si]) == ELF::STT_OBJECT &&
+        if (Obj->isELF() && std::get<2>(Symbols[SI]) == ELF::STT_OBJECT &&
             !DisassembleAll && Section.isText()) {
           // print out data up to 8 bytes at a time in hex and ascii
           uint8_t AsciiData[9] = {'\0'};
@@ -1713,7 +1268,7 @@ static void DisassembleObject(const ObjectFile *Obj, bool InlineRelocs) {
               // Indent the space for less than 8 bytes data.
               // 2 spaces for byte and one for space between bytes
               IndentOffset = 3 * (8 - NumBytes);
-              for (int Excess = 8 - NumBytes; Excess < 8; Excess++)
+              for (int Excess = NumBytes; Excess < 8; Excess++)
                 AsciiData[Excess] = '\0';
               NumBytes = 8;
             }
@@ -1729,6 +1284,14 @@ static void DisassembleObject(const ObjectFile *Obj, bool InlineRelocs) {
         if (Index >= End)
           break;
 
+        if (size_t N =
+                countSkippableZeroBytes(Bytes.slice(Index, End - Index))) {
+          outs() << "\t\t..." << '\n';
+          Index += N;
+          if (Index >= End)
+            break;
+        }
+
         // Disassemble a real instruction or a data when disassemble all is
         // provided
         bool Disassembled = DisAsm->getInstruction(Inst, Size, Bytes.slice(Index),
@@ -1737,9 +1300,9 @@ static void DisassembleObject(const ObjectFile *Obj, bool InlineRelocs) {
         if (Size == 0)
           Size = 1;
 
-        PIP.printInst(*IP, Disassembled ? &Inst : nullptr,
-                      Bytes.slice(Index, Size), SectionAddr + Index, outs(), "",
-                      *STI, &SP, &Rels);
+        PIP.printInst(
+            *IP, Disassembled ? &Inst : nullptr, Bytes.slice(Index, Size),
+            SectionAddr + Index + VMAAdjustment, outs(), "", *STI, &SP, &Rels);
         outs() << CommentStream.str();
         Comments.clear();
 
@@ -1805,34 +1368,102 @@ static void DisassembleObject(const ObjectFile *Obj, bool InlineRelocs) {
         outs() << "\n";
 
         // Hexagon does this in pretty printer
-        if (Obj->getArch() != Triple::hexagon)
+        if (Obj->getArch() != Triple::hexagon) {
           // Print relocation for instruction.
-          while (rel_cur != rel_end) {
-            bool hidden = getHidden(*rel_cur);
-            uint64_t addr = rel_cur->getOffset();
-            SmallString<16> name;
-            SmallString<32> val;
-
+          while (RelCur != RelEnd) {
+            uint64_t Offset = RelCur->getOffset();
             // If this relocation is hidden, skip it.
-            if (hidden || ((SectionAddr + addr) < StartAddress)) {
-              ++rel_cur;
+            if (getHidden(*RelCur) || ((SectionAddr + Offset) < StartAddress)) {
+              ++RelCur;
               continue;
             }
 
-            // Stop when rel_cur's address is past the current instruction.
-            if (addr >= Index + Size) break;
-            rel_cur->getTypeName(name);
-            error(getRelocationValueString(*rel_cur, val));
-            outs() << format(Fmt.data(), SectionAddr + addr) << name
-                   << "\t" << val << "\n";
-            ++rel_cur;
+            // Stop when RelCur's offset is past the current instruction.
+            if (Offset >= Index + Size)
+              break;
+
+            // When --adjust-vma is used, update the address printed.
+            if (RelCur->getSymbol() != Obj->symbol_end()) {
+              Expected<section_iterator> SymSI =
+                  RelCur->getSymbol()->getSection();
+              if (SymSI && *SymSI != Obj->section_end() &&
+                  (shouldAdjustVA(**SymSI)))
+                Offset += AdjustVMA;
+            }
+
+            printRelocation(*RelCur, SectionAddr + Offset,
+                            Obj->getBytesInAddress());
+            ++RelCur;
           }
+        }
       }
     }
   }
 }
 
-void llvm::PrintRelocations(const ObjectFile *Obj) {
+static void disassembleObject(const ObjectFile *Obj, bool InlineRelocs) {
+  if (StartAddress > StopAddress)
+    error("Start address should be less than stop address");
+
+  const Target *TheTarget = getTarget(Obj);
+
+  // Package up features to be passed to target/subtarget
+  SubtargetFeatures Features = Obj->getFeatures();
+  if (!MAttrs.empty())
+    for (unsigned I = 0; I != MAttrs.size(); ++I)
+      Features.AddFeature(MAttrs[I]);
+
+  std::unique_ptr<const MCRegisterInfo> MRI(
+      TheTarget->createMCRegInfo(TripleName));
+  if (!MRI)
+    report_error(Obj->getFileName(),
+                 "no register info for target " + TripleName);
+
+  // Set up disassembler.
+  std::unique_ptr<const MCAsmInfo> AsmInfo(
+      TheTarget->createMCAsmInfo(*MRI, TripleName));
+  if (!AsmInfo)
+    report_error(Obj->getFileName(),
+                 "no assembly info for target " + TripleName);
+  std::unique_ptr<const MCSubtargetInfo> STI(
+      TheTarget->createMCSubtargetInfo(TripleName, MCPU, Features.getString()));
+  if (!STI)
+    report_error(Obj->getFileName(),
+                 "no subtarget info for target " + TripleName);
+  std::unique_ptr<const MCInstrInfo> MII(TheTarget->createMCInstrInfo());
+  if (!MII)
+    report_error(Obj->getFileName(),
+                 "no instruction info for target " + TripleName);
+  MCObjectFileInfo MOFI;
+  MCContext Ctx(AsmInfo.get(), MRI.get(), &MOFI);
+  // FIXME: for now initialize MCObjectFileInfo with default values
+  MOFI.InitMCObjectFileInfo(Triple(TripleName), false, Ctx);
+
+  std::unique_ptr<MCDisassembler> DisAsm(
+      TheTarget->createMCDisassembler(*STI, Ctx));
+  if (!DisAsm)
+    report_error(Obj->getFileName(),
+                 "no disassembler for target " + TripleName);
+
+  std::unique_ptr<const MCInstrAnalysis> MIA(
+      TheTarget->createMCInstrAnalysis(MII.get()));
+
+  int AsmPrinterVariant = AsmInfo->getAssemblerDialect();
+  std::unique_ptr<MCInstPrinter> IP(TheTarget->createMCInstPrinter(
+      Triple(TripleName), AsmPrinterVariant, *AsmInfo, *MII, *MRI));
+  if (!IP)
+    report_error(Obj->getFileName(),
+                 "no instruction printer for target " + TripleName);
+  IP->setPrintImmHex(PrintImmHex);
+
+  PrettyPrinter &PIP = selectPrettyPrinter(Triple(TripleName));
+  SourcePrinter SP(Obj, TheTarget->getName());
+
+  disassembleObject(TheTarget, Obj, Ctx, DisAsm.get(), MIA.get(), IP.get(),
+                    STI.get(), PIP, SP, InlineRelocs);
+}
+
+void llvm::printRelocations(const ObjectFile *Obj) {
   StringRef Fmt = Obj->getBytesInAddress() > 4 ? "%016" PRIx64 :
                                                  "%08" PRIx64;
   // Regular objdump doesn't print relocations in non-relocatable object
@@ -1843,81 +1474,106 @@ void llvm::PrintRelocations(const ObjectFile *Obj) {
   for (const SectionRef &Section : ToolSectionFilter(*Obj)) {
     if (Section.relocation_begin() == Section.relocation_end())
       continue;
-    StringRef secname;
-    error(Section.getName(secname));
-    outs() << "RELOCATION RECORDS FOR [" << secname << "]:\n";
+    StringRef SecName;
+    error(Section.getName(SecName));
+    outs() << "RELOCATION RECORDS FOR [" << SecName << "]:\n";
     for (const RelocationRef &Reloc : Section.relocations()) {
-      bool hidden = getHidden(Reloc);
-      uint64_t address = Reloc.getOffset();
-      SmallString<32> relocname;
-      SmallString<32> valuestr;
-      if (address < StartAddress || address > StopAddress || hidden)
+      uint64_t Address = Reloc.getOffset();
+      SmallString<32> RelocName;
+      SmallString<32> ValueStr;
+      if (Address < StartAddress || Address > StopAddress || getHidden(Reloc))
         continue;
-      Reloc.getTypeName(relocname);
-      error(getRelocationValueString(Reloc, valuestr));
-      outs() << format(Fmt.data(), address) << " " << relocname << " "
-             << valuestr << "\n";
+      Reloc.getTypeName(RelocName);
+      error(getRelocationValueString(Reloc, ValueStr));
+      outs() << format(Fmt.data(), Address) << " " << RelocName << " "
+             << ValueStr << "\n";
     }
     outs() << "\n";
   }
 }
 
-void llvm::PrintDynamicRelocations(const ObjectFile *Obj) {
-
+void llvm::printDynamicRelocations(const ObjectFile *Obj) {
   // For the moment, this option is for ELF only
   if (!Obj->isELF())
     return;
 
   const auto *Elf = dyn_cast<ELFObjectFileBase>(Obj);
-
   if (!Elf || Elf->getEType() != ELF::ET_DYN) {
     error("not a dynamic object");
     return;
   }
 
-  StringRef Fmt = Obj->getBytesInAddress() > 4 ? "%016" PRIx64 : "%08" PRIx64;
-
   std::vector<SectionRef> DynRelSec = Obj->dynamic_relocation_sections();
   if (DynRelSec.empty())
     return;
 
   outs() << "DYNAMIC RELOCATION RECORDS\n";
+  StringRef Fmt = Obj->getBytesInAddress() > 4 ? "%016" PRIx64 : "%08" PRIx64;
   for (const SectionRef &Section : DynRelSec) {
     if (Section.relocation_begin() == Section.relocation_end())
       continue;
     for (const RelocationRef &Reloc : Section.relocations()) {
-      uint64_t address = Reloc.getOffset();
-      SmallString<32> relocname;
-      SmallString<32> valuestr;
-      Reloc.getTypeName(relocname);
-      error(getRelocationValueString(Reloc, valuestr));
-      outs() << format(Fmt.data(), address) << " " << relocname << " "
-             << valuestr << "\n";
+      uint64_t Address = Reloc.getOffset();
+      SmallString<32> RelocName;
+      SmallString<32> ValueStr;
+      Reloc.getTypeName(RelocName);
+      error(getRelocationValueString(Reloc, ValueStr));
+      outs() << format(Fmt.data(), Address) << " " << RelocName << " "
+             << ValueStr << "\n";
     }
   }
 }
 
-void llvm::PrintSectionHeaders(const ObjectFile *Obj) {
-  outs() << "Sections:\n"
-            "Idx Name          Size      Address          Type\n";
+// Returns true if we need to show LMA column when dumping section headers. We
+// show it only when the platform is ELF and either we have at least one section
+// whose VMA and LMA are different and/or when --show-lma flag is used.
+static bool shouldDisplayLMA(const ObjectFile *Obj) {
+  if (!Obj->isELF())
+    return false;
+  for (const SectionRef &S : ToolSectionFilter(*Obj))
+    if (S.getAddress() != getELFSectionLMA(S))
+      return true;
+  return ShowLMA;
+}
+
+void llvm::printSectionHeaders(const ObjectFile *Obj) {
+  bool HasLMAColumn = shouldDisplayLMA(Obj);
+  if (HasLMAColumn)
+    outs() << "Sections:\n"
+              "Idx Name          Size     VMA              LMA              "
+              "Type\n";
+  else
+    outs() << "Sections:\n"
+              "Idx Name          Size     VMA          Type\n";
+
   for (const SectionRef &Section : ToolSectionFilter(*Obj)) {
     StringRef Name;
     error(Section.getName(Name));
-    uint64_t Address = Section.getAddress();
+    uint64_t VMA = Section.getAddress();
+    if (shouldAdjustVA(Section))
+      VMA += AdjustVMA;
+
     uint64_t Size = Section.getSize();
     bool Text = Section.isText();
     bool Data = Section.isData();
     bool BSS = Section.isBSS();
     std::string Type = (std::string(Text ? "TEXT " : "") +
                         (Data ? "DATA " : "") + (BSS ? "BSS" : ""));
-    outs() << format("%3d %-13s %08" PRIx64 " %016" PRIx64 " %s\n",
-                     (unsigned)Section.getIndex(), Name.str().c_str(), Size,
-                     Address, Type.c_str());
+
+    if (HasLMAColumn)
+      outs() << format("%3d %-13s %08" PRIx64 " %016" PRIx64 " %016" PRIx64
+                       " %s\n",
+                       (unsigned)Section.getIndex(), Name.str().c_str(), Size,
+                       VMA, getELFSectionLMA(Section), Type.c_str());
+    else
+      outs() << format("%3d %-13s %08" PRIx64 " %016" PRIx64 " %s\n",
+                       (unsigned)Section.getIndex(), Name.str().c_str(), Size,
+                       VMA, Type.c_str());
   }
   outs() << "\n";
 }
 
-void llvm::PrintSectionContents(const ObjectFile *Obj) {
+void llvm::printSectionContents(const ObjectFile *Obj) {
   std::error_code EC;
   for (const SectionRef &Section : ToolSectionFilter(*Obj)) {
     StringRef Name;
@@ -1939,23 +1595,23 @@ void llvm::PrintSectionContents(const ObjectFile *Obj) {
     error(Section.getContents(Contents));
 
     // Dump out the content as hex and printable ascii characters.
-    for (std::size_t addr = 0, end = Contents.size(); addr < end; addr += 16) {
-      outs() << format(" %04" PRIx64 " ", BaseAddr + addr);
+    for (std::size_t Addr = 0, End = Contents.size(); Addr < End; Addr += 16) {
+      outs() << format(" %04" PRIx64 " ", BaseAddr + Addr);
       // Dump line of hex.
-      for (std::size_t i = 0; i < 16; ++i) {
-        if (i != 0 && i % 4 == 0)
+      for (std::size_t I = 0; I < 16; ++I) {
+        if (I != 0 && I % 4 == 0)
           outs() << ' ';
-        if (addr + i < end)
-          outs() << hexdigit((Contents[addr + i] >> 4) & 0xF, true)
-                 << hexdigit(Contents[addr + i] & 0xF, true);
+        if (Addr + I < End)
+          outs() << hexdigit((Contents[Addr + I] >> 4) & 0xF, true)
+                 << hexdigit(Contents[Addr + I] & 0xF, true);
         else
           outs() << "  ";
       }
       // Print ascii.
       outs() << "  ";
-      for (std::size_t i = 0; i < 16 && addr + i < end; ++i) {
-        if (isPrint(static_cast<unsigned char>(Contents[addr + i]) & 0xFF))
-          outs() << Contents[addr + i];
+      for (std::size_t I = 0; I < 16 && Addr + I < End; ++I) {
+        if (isPrint(static_cast<unsigned char>(Contents[Addr + I]) & 0xFF))
+          outs() << Contents[Addr + I];
         else
           outs() << ".";
       }
@@ -1964,40 +1620,47 @@ void llvm::PrintSectionContents(const ObjectFile *Obj) {
   }
 }
 
-void llvm::PrintSymbolTable(const ObjectFile *o, StringRef ArchiveName,
+void llvm::printSymbolTable(const ObjectFile *O, StringRef ArchiveName,
                             StringRef ArchitectureName) {
   outs() << "SYMBOL TABLE:\n";
 
-  if (const COFFObjectFile *coff = dyn_cast<const COFFObjectFile>(o)) {
-    printCOFFSymbolTable(coff);
+  if (const COFFObjectFile *Coff = dyn_cast<const COFFObjectFile>(O)) {
+    printCOFFSymbolTable(Coff);
     return;
   }
-  for (const SymbolRef &Symbol : o->symbols()) {
+
+  for (auto I = O->symbol_begin(), E = O->symbol_end(); I != E; ++I) {
+    // Skip printing the special zero symbol when dumping an ELF file.
+    // This makes the output consistent with the GNU objdump.
+    if (I == O->symbol_begin() && isa<ELFObjectFileBase>(O))
+      continue;
+
+    const SymbolRef &Symbol = *I;
     Expected<uint64_t> AddressOrError = Symbol.getAddress();
     if (!AddressOrError)
-      report_error(ArchiveName, o->getFileName(), AddressOrError.takeError(),
+      report_error(ArchiveName, O->getFileName(), AddressOrError.takeError(),
                    ArchitectureName);
     uint64_t Address = *AddressOrError;
     if ((Address < StartAddress) || (Address > StopAddress))
       continue;
     Expected<SymbolRef::Type> TypeOrError = Symbol.getType();
     if (!TypeOrError)
-      report_error(ArchiveName, o->getFileName(), TypeOrError.takeError(),
+      report_error(ArchiveName, O->getFileName(), TypeOrError.takeError(),
                    ArchitectureName);
     SymbolRef::Type Type = *TypeOrError;
     uint32_t Flags = Symbol.getFlags();
     Expected<section_iterator> SectionOrErr = Symbol.getSection();
     if (!SectionOrErr)
-      report_error(ArchiveName, o->getFileName(), SectionOrErr.takeError(),
+      report_error(ArchiveName, O->getFileName(), SectionOrErr.takeError(),
                    ArchitectureName);
     section_iterator Section = *SectionOrErr;
     StringRef Name;
-    if (Type == SymbolRef::ST_Debug && Section != o->section_end()) {
+    if (Type == SymbolRef::ST_Debug && Section != O->section_end()) {
       Section->getName(Name);
     } else {
       Expected<StringRef> NameOrErr = Symbol.getName();
       if (!NameOrErr)
-        report_error(ArchiveName, o->getFileName(), NameOrErr.takeError(),
+        report_error(ArchiveName, O->getFileName(), NameOrErr.takeError(),
                      ArchitectureName);
       Name = *NameOrErr;
     }
@@ -2021,7 +1684,7 @@ void llvm::PrintSymbolTable(const ObjectFile *o, StringRef ArchiveName,
     else if (Type == SymbolRef::ST_Data)
       FileFunc = 'O';
 
-    const char *Fmt = o->getBytesInAddress() > 4 ? "%016" PRIx64 :
+    const char *Fmt = O->getBytesInAddress() > 4 ? "%016" PRIx64 :
                                                    "%08" PRIx64;
 
     outs() << format(Fmt, Address) << " "
@@ -2037,11 +1700,11 @@ void llvm::PrintSymbolTable(const ObjectFile *o, StringRef ArchiveName,
       outs() << "*ABS*";
     } else if (Common) {
       outs() << "*COM*";
-    } else if (Section == o->section_end()) {
+    } else if (Section == O->section_end()) {
       outs() << "*UND*";
     } else {
       if (const MachOObjectFile *MachO =
-          dyn_cast<const MachOObjectFile>(o)) {
+          dyn_cast<const MachOObjectFile>(O)) {
         DataRefImpl DR = Section->getRawDataRefImpl();
         StringRef SegmentName = MachO->getSectionFinalSegmentName(DR);
         outs() << SegmentName << ",";
@@ -2052,94 +1715,84 @@ void llvm::PrintSymbolTable(const ObjectFile *o, StringRef ArchiveName,
     }
 
     outs() << '\t';
-    if (Common || isa<ELFObjectFileBase>(o)) {
+    if (Common || isa<ELFObjectFileBase>(O)) {
       uint64_t Val =
           Common ? Symbol.getAlignment() : ELFSymbolRef(Symbol).getSize();
       outs() << format("\t %08" PRIx64 " ", Val);
     }
 
-    if (Hidden) {
+    if (Hidden)
       outs() << ".hidden ";
-    }
-    outs() << Name
-           << '\n';
+
+    if (Demangle)
+      outs() << demangle(Name) << '\n';
+    else
+      outs() << Name << '\n';
   }
 }
 
-static void PrintUnwindInfo(const ObjectFile *o) {
+static void printUnwindInfo(const ObjectFile *O) {
   outs() << "Unwind info:\n\n";
 
-  if (const COFFObjectFile *coff = dyn_cast<COFFObjectFile>(o)) {
-    printCOFFUnwindInfo(coff);
-  } else if (const MachOObjectFile *MachO = dyn_cast<MachOObjectFile>(o))
+  if (const COFFObjectFile *Coff = dyn_cast<COFFObjectFile>(O))
+    printCOFFUnwindInfo(Coff);
+  else if (const MachOObjectFile *MachO = dyn_cast<MachOObjectFile>(O))
     printMachOUnwindInfo(MachO);
-  else {
+  else
     // TODO: Extract DWARF dump tool to objdump.
     WithColor::error(errs(), ToolName)
         << "This operation is only currently supported "
            "for COFF and MachO object files.\n";
-    return;
-  }
 }
 
 void llvm::printExportsTrie(const ObjectFile *o) {
   outs() << "Exports trie:\n";
   if (const MachOObjectFile *MachO = dyn_cast<MachOObjectFile>(o))
     printMachOExportsTrie(MachO);
-  else {
+  else
     WithColor::error(errs(), ToolName)
         << "This operation is only currently supported "
            "for Mach-O executable files.\n";
-    return;
-  }
 }
 
 void llvm::printRebaseTable(ObjectFile *o) {
   outs() << "Rebase table:\n";
   if (MachOObjectFile *MachO = dyn_cast<MachOObjectFile>(o))
     printMachORebaseTable(MachO);
-  else {
+  else
     WithColor::error(errs(), ToolName)
         << "This operation is only currently supported "
            "for Mach-O executable files.\n";
-    return;
-  }
 }
 
 void llvm::printBindTable(ObjectFile *o) {
   outs() << "Bind table:\n";
   if (MachOObjectFile *MachO = dyn_cast<MachOObjectFile>(o))
     printMachOBindTable(MachO);
-  else {
+  else
     WithColor::error(errs(), ToolName)
         << "This operation is only currently supported "
            "for Mach-O executable files.\n";
-    return;
-  }
 }
 
 void llvm::printLazyBindTable(ObjectFile *o) {
   outs() << "Lazy bind table:\n";
   if (MachOObjectFile *MachO = dyn_cast<MachOObjectFile>(o))
     printMachOLazyBindTable(MachO);
-  else {
+  else
     WithColor::error(errs(), ToolName)
         << "This operation is only currently supported "
            "for Mach-O executable files.\n";
-    return;
-  }
 }
 
 void llvm::printWeakBindTable(ObjectFile *o) {
   outs() << "Weak bind table:\n";
   if (MachOObjectFile *MachO = dyn_cast<MachOObjectFile>(o))
     printMachOWeakBindTable(MachO);
-  else {
+  else
     WithColor::error(errs(), ToolName)
         << "This operation is only currently supported "
            "for Mach-O executable files.\n";
-    return;
-  }
 }
 
 /// Dump the raw contents of the __clangast section so the output can be piped
@@ -2177,7 +1830,7 @@ void llvm::printRawClangAST(const ObjectFile *Obj) {
 }
 
 static void printFaultMaps(const ObjectFile *Obj) {
-  const char *FaultMapSectionName = nullptr;
+  StringRef FaultMapSectionName;
 
   if (isa<ELFObjectFileBase>(Obj)) {
     FaultMapSectionName = ".llvm_faultmaps";
@@ -2217,39 +1870,38 @@ static void printFaultMaps(const ObjectFile *Obj) {
   outs() << FMP;
 }
 
-static void printPrivateFileHeaders(const ObjectFile *o, bool onlyFirst) {
-  if (o->isELF()) {
-    printELFFileHeader(o);
-    return printELFDynamicSection(o);
+static void printPrivateFileHeaders(const ObjectFile *O, bool OnlyFirst) {
+  if (O->isELF()) {
+    printELFFileHeader(O);
+    return printELFDynamicSection(O);
   }
-  if (o->isCOFF())
-    return printCOFFFileHeader(o);
-  if (o->isWasm())
-    return printWasmFileHeader(o);
-  if (o->isMachO()) {
-    printMachOFileHeader(o);
-    if (!onlyFirst)
-      printMachOLoadCommands(o);
+  if (O->isCOFF())
+    return printCOFFFileHeader(O);
+  if (O->isWasm())
+    return printWasmFileHeader(O);
+  if (O->isMachO()) {
+    printMachOFileHeader(O);
+    if (!OnlyFirst)
+      printMachOLoadCommands(O);
     return;
   }
-  report_error(o->getFileName(), "Invalid/Unsupported object file format");
+  report_error(O->getFileName(), "Invalid/Unsupported object file format");
 }
 
-static void printFileHeaders(const ObjectFile *o) {
-  if (!o->isELF() && !o->isCOFF())
-    report_error(o->getFileName(), "Invalid/Unsupported object file format");
+static void printFileHeaders(const ObjectFile *O) {
+  if (!O->isELF() && !O->isCOFF())
+    report_error(O->getFileName(), "Invalid/Unsupported object file format");
 
-  Triple::ArchType AT = o->getArch();
+  Triple::ArchType AT = O->getArch();
   outs() << "architecture: " << Triple::getArchTypeName(AT) << "\n";
-  Expected<uint64_t> StartAddrOrErr = o->getStartAddress();
+  Expected<uint64_t> StartAddrOrErr = O->getStartAddress();
   if (!StartAddrOrErr)
-    report_error(o->getFileName(), StartAddrOrErr.takeError());
+    report_error(O->getFileName(), StartAddrOrErr.takeError());
 
-  StringRef Fmt = o->getBytesInAddress() > 4 ? "%016" PRIx64 : "%08" PRIx64;
+  StringRef Fmt = O->getBytesInAddress() > 4 ? "%016" PRIx64 : "%08" PRIx64;
   uint64_t Address = StartAddrOrErr.get();
   outs() << "start address: "
-         << "0x" << format(Fmt.data(), Address)
-         << "\n";
+         << "0x" << format(Fmt.data(), Address) << "\n\n";
 }
 
 static void printArchiveChild(StringRef Filename, const Archive::Child &C) {
@@ -2316,55 +1968,55 @@ static void printArchiveChild(StringRef Filename, const Archive::Child &C) {
   outs() << Name << "\n";
 }
 
-static void DumpObject(ObjectFile *o, const Archive *a = nullptr,
-                       const Archive::Child *c = nullptr) {
-  StringRef ArchiveName = a != nullptr ? a->getFileName() : "";
+static void dumpObject(ObjectFile *O, const Archive *A = nullptr,
+                       const Archive::Child *C = nullptr) {
   // Avoid other output when using a raw option.
   if (!RawClangAST) {
     outs() << '\n';
-    if (a)
-      outs() << a->getFileName() << "(" << o->getFileName() << ")";
+    if (A)
+      outs() << A->getFileName() << "(" << O->getFileName() << ")";
     else
-      outs() << o->getFileName();
-    outs() << ":\tfile format " << o->getFileFormatName() << "\n\n";
+      outs() << O->getFileName();
+    outs() << ":\tfile format " << O->getFileFormatName() << "\n\n";
   }
 
-  if (ArchiveHeaders && !MachOOpt && c)
-    printArchiveChild(ArchiveName, *c);
+  StringRef ArchiveName = A ? A->getFileName() : "";
+  if (FileHeaders)
+    printFileHeaders(O);
+  if (ArchiveHeaders && !MachOOpt && C)
+    printArchiveChild(ArchiveName, *C);
   if (Disassemble)
-    DisassembleObject(o, Relocations);
+    disassembleObject(O, Relocations);
   if (Relocations && !Disassemble)
-    PrintRelocations(o);
+    printRelocations(O);
   if (DynamicRelocations)
-    PrintDynamicRelocations(o);
+    printDynamicRelocations(O);
   if (SectionHeaders)
-    PrintSectionHeaders(o);
+    printSectionHeaders(O);
   if (SectionContents)
-    PrintSectionContents(o);
+    printSectionContents(O);
   if (SymbolTable)
-    PrintSymbolTable(o, ArchiveName);
+    printSymbolTable(O, ArchiveName);
   if (UnwindInfo)
-    PrintUnwindInfo(o);
+    printUnwindInfo(O);
   if (PrivateHeaders || FirstPrivateHeader)
-    printPrivateFileHeaders(o, FirstPrivateHeader);
-  if (FileHeaders)
-    printFileHeaders(o);
+    printPrivateFileHeaders(O, FirstPrivateHeader);
   if (ExportsTrie)
-    printExportsTrie(o);
+    printExportsTrie(O);
   if (Rebase)
-    printRebaseTable(o);
+    printRebaseTable(O);
   if (Bind)
-    printBindTable(o);
+    printBindTable(O);
   if (LazyBind)
-    printLazyBindTable(o);
+    printLazyBindTable(O);
   if (WeakBind)
-    printWeakBindTable(o);
+    printWeakBindTable(O);
   if (RawClangAST)
-    printRawClangAST(o);
+    printRawClangAST(O);
   if (PrintFaultMaps)
-    printFaultMaps(o);
+    printFaultMaps(O);
   if (DwarfDumpType != DIDT_Null) {
-    std::unique_ptr<DIContext> DICtx = DWARFContext::create(*o);
+    std::unique_ptr<DIContext> DICtx = DWARFContext::create(*O);
     // Dump the complete DWARF structure.
     DIDumpOptions DumpOpts;
     DumpOpts.DumpType = DwarfDumpType;
@@ -2372,7 +2024,7 @@ static void DumpObject(ObjectFile *o, const Archive *a = nullptr,
   }
 }
 
-static void DumpObject(const COFFImportFile *I, const Archive *A,
+static void dumpObject(const COFFImportFile *I, const Archive *A,
                        const Archive::Child *C = nullptr) {
   StringRef ArchiveName = A ? A->getFileName() : "";
 
@@ -2390,34 +2042,33 @@ static void DumpObject(const COFFImportFile *I, const Archive *A,
 }
 
 /// Dump each object file in \a a;
-static void DumpArchive(const Archive *a) {
+static void dumpArchive(const Archive *A) {
   Error Err = Error::success();
-  for (auto &C : a->children(Err)) {
+  for (auto &C : A->children(Err)) {
     Expected<std::unique_ptr<Binary>> ChildOrErr = C.getAsBinary();
     if (!ChildOrErr) {
       if (auto E = isNotObjectErrorInvalidFileType(ChildOrErr.takeError()))
-        report_error(a->getFileName(), C, std::move(E));
+        report_error(A->getFileName(), C, std::move(E));
       continue;
     }
-    if (ObjectFile *o = dyn_cast<ObjectFile>(&*ChildOrErr.get()))
-      DumpObject(o, a, &C);
+    if (ObjectFile *O = dyn_cast<ObjectFile>(&*ChildOrErr.get()))
+      dumpObject(O, A, &C);
     else if (COFFImportFile *I = dyn_cast<COFFImportFile>(&*ChildOrErr.get()))
-      DumpObject(I, a, &C);
+      dumpObject(I, A, &C);
     else
-      report_error(a->getFileName(), object_error::invalid_file_type);
+      report_error(A->getFileName(), object_error::invalid_file_type);
   }
   if (Err)
-    report_error(a->getFileName(), std::move(Err));
+    report_error(A->getFileName(), std::move(Err));
 }
 
 /// Open file and figure out how to dump it.
-static void DumpInput(StringRef file) {
-
+static void dumpInput(StringRef file) {
   // If we are using the Mach-O specific object file parser, then let it parse
   // the file and process the command line options.  So the -arch flags can
   // be used to select specific slices, etc.
   if (MachOOpt) {
-    ParseInputMachO(file);
+    parseInputMachO(file);
     return;
   }
 
@@ -2427,12 +2078,12 @@ static void DumpInput(StringRef file) {
     report_error(file, BinaryOrErr.takeError());
   Binary &Binary = *BinaryOrErr.get().getBinary();
 
-  if (Archive *a = dyn_cast<Archive>(&Binary))
-    DumpArchive(a);
-  else if (ObjectFile *o = dyn_cast<ObjectFile>(&Binary))
-    DumpObject(o);
+  if (Archive *A = dyn_cast<Archive>(&Binary))
+    dumpArchive(A);
+  else if (ObjectFile *O = dyn_cast<ObjectFile>(&Binary))
+    dumpObject(O);
   else if (MachOUniversalBinary *UB = dyn_cast<MachOUniversalBinary>(&Binary))
-    ParseInputMachO(UB);
+    parseInputMachO(UB);
   else
     report_error(file, object_error::invalid_file_type);
 }
@@ -2453,11 +2104,12 @@ int main(int argc, char **argv) {
   ToolName = argv[0];
 
   // Defaults to a.out if no filenames specified.
-  if (InputFilenames.size() == 0)
+  if (InputFilenames.empty())
     InputFilenames.push_back("a.out");
 
   if (AllHeaders)
-    PrivateHeaders = Relocations = SectionHeaders = SymbolTable = true;
+    ArchiveHeaders = FileHeaders = PrivateHeaders = Relocations =
+        SectionHeaders = SymbolTable = true;
 
   if (DisassembleAll || PrintSource || PrintLines)
     Disassemble = true;
@@ -2487,7 +2139,7 @@ int main(int argc, char **argv) {
       && !(DylibsUsed && MachOOpt)
       && !(DylibId && MachOOpt)
       && !(ObjcMetaData && MachOOpt)
-      && !(FilterSections.size() != 0 && MachOOpt)
+      && !(!FilterSections.empty() && MachOOpt)
       && !PrintFaultMaps
       && DwarfDumpType == DIDT_Null) {
     cl::PrintHelpMessage();
@@ -2497,7 +2149,7 @@ int main(int argc, char **argv) {
   DisasmFuncsSet.insert(DisassembleFunctions.begin(),
                         DisassembleFunctions.end());
 
-  llvm::for_each(InputFilenames, DumpInput);
+  llvm::for_each(InputFilenames, dumpInput);
 
   return EXIT_SUCCESS;
 }
diff --git a/tools/llvm-objdump/llvm-objdump.h b/tools/llvm-objdump/llvm-objdump.h
index 44ef14d7f698a165d6ff5d400057f2279cf5df57..e559b72aeb7ac23477bf5eac51afa5569614d438 100644
--- a/tools/llvm-objdump/llvm-objdump.h
+++ b/tools/llvm-objdump/llvm-objdump.h
@@ -1,8 +1,7 @@
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -19,13 +18,15 @@ namespace llvm {
 class StringRef;
 
 namespace object {
-  class COFFObjectFile;
-  class COFFImportFile;
-  class MachOObjectFile;
-  class MachOUniversalBinary;
-  class ObjectFile;
-  class Archive;
-  class RelocationRef;
+class COFFObjectFile;
+class COFFImportFile;
+class ELFObjectFileBase;
+class ELFSectionRef;
+class MachOObjectFile;
+class MachOUniversalBinary;
+class ObjectFile;
+class Archive;
+class RelocationRef;
 }
 
 extern cl::opt<std::string> TripleName;
@@ -68,37 +69,107 @@ extern cl::opt<bool> UnwindInfo;
 extern cl::opt<bool> PrintImmHex;
 extern cl::opt<DIDumpType> DwarfDumpType;
 
+typedef std::function<bool(llvm::object::SectionRef const &)> FilterPredicate;
+
+class SectionFilterIterator {
+public:
+  SectionFilterIterator(FilterPredicate P,
+                        llvm::object::section_iterator const &I,
+                        llvm::object::section_iterator const &E)
+      : Predicate(std::move(P)), Iterator(I), End(E) {
+    ScanPredicate();
+  }
+  const llvm::object::SectionRef &operator*() const { return *Iterator; }
+  SectionFilterIterator &operator++() {
+    ++Iterator;
+    ScanPredicate();
+    return *this;
+  }
+  bool operator!=(SectionFilterIterator const &Other) const {
+    return Iterator != Other.Iterator;
+  }
+
+private:
+  void ScanPredicate() {
+    while (Iterator != End && !Predicate(*Iterator)) {
+      ++Iterator;
+    }
+  }
+  FilterPredicate Predicate;
+  llvm::object::section_iterator Iterator;
+  llvm::object::section_iterator End;
+};
+
+class SectionFilter {
+public:
+  SectionFilter(FilterPredicate P, llvm::object::ObjectFile const &O)
+      : Predicate(std::move(P)), Object(O) {}
+  SectionFilterIterator begin() {
+    return SectionFilterIterator(Predicate, Object.section_begin(),
+                                 Object.section_end());
+  }
+  SectionFilterIterator end() {
+    return SectionFilterIterator(Predicate, Object.section_end(),
+                                 Object.section_end());
+  }
+
+private:
+  FilterPredicate Predicate;
+  llvm::object::ObjectFile const &Object;
+};
+
 // Various helper functions.
+SectionFilter ToolSectionFilter(llvm::object::ObjectFile const &O);
+
+std::error_code
+getELFRelocationValueString(const object::ELFObjectFileBase *Obj,
+                            const object::RelocationRef &Rel,
+                            llvm::SmallVectorImpl<char> &Result);
+std::error_code
+getCOFFRelocationValueString(const object::COFFObjectFile *Obj,
+                             const object::RelocationRef &Rel,
+                             llvm::SmallVectorImpl<char> &Result);
+std::error_code
+getWasmRelocationValueString(const object::WasmObjectFile *Obj,
+                             const object::RelocationRef &RelRef,
+                             llvm::SmallVectorImpl<char> &Result);
+std::error_code
+getMachORelocationValueString(const object::MachOObjectFile *Obj,
+                              const object::RelocationRef &RelRef,
+                              llvm::SmallVectorImpl<char> &Result);
+
+uint64_t getELFSectionLMA(const object::ELFSectionRef& Sec);
+
 void error(std::error_code ec);
-bool RelocAddressLess(object::RelocationRef a, object::RelocationRef b);
-void ParseInputMachO(StringRef Filename);
-void ParseInputMachO(object::MachOUniversalBinary *UB);
-void printCOFFUnwindInfo(const object::COFFObjectFile* o);
-void printMachOUnwindInfo(const object::MachOObjectFile* o);
-void printMachOExportsTrie(const object::MachOObjectFile* o);
-void printMachORebaseTable(object::MachOObjectFile* o);
-void printMachOBindTable(object::MachOObjectFile* o);
-void printMachOLazyBindTable(object::MachOObjectFile* o);
-void printMachOWeakBindTable(object::MachOObjectFile* o);
-void printELFFileHeader(const object::ObjectFile *o);
+bool isRelocAddressLess(object::RelocationRef A, object::RelocationRef B);
+void parseInputMachO(StringRef Filename);
+void parseInputMachO(object::MachOUniversalBinary *UB);
+void printCOFFUnwindInfo(const object::COFFObjectFile *O);
+void printMachOUnwindInfo(const object::MachOObjectFile *O);
+void printMachOExportsTrie(const object::MachOObjectFile *O);
+void printMachORebaseTable(object::MachOObjectFile *O);
+void printMachOBindTable(object::MachOObjectFile *O);
+void printMachOLazyBindTable(object::MachOObjectFile *O);
+void printMachOWeakBindTable(object::MachOObjectFile *O);
+void printELFFileHeader(const object::ObjectFile *O);
 void printELFDynamicSection(const object::ObjectFile *Obj);
-void printCOFFFileHeader(const object::ObjectFile *o);
-void printCOFFSymbolTable(const object::COFFImportFile *i);
-void printCOFFSymbolTable(const object::COFFObjectFile *o);
-void printMachOFileHeader(const object::ObjectFile *o);
-void printMachOLoadCommands(const object::ObjectFile *o);
-void printWasmFileHeader(const object::ObjectFile *o);
-void printExportsTrie(const object::ObjectFile *o);
-void printRebaseTable(object::ObjectFile *o);
-void printBindTable(object::ObjectFile *o);
-void printLazyBindTable(object::ObjectFile *o);
-void printWeakBindTable(object::ObjectFile *o);
-void printRawClangAST(const object::ObjectFile *o);
-void PrintRelocations(const object::ObjectFile *o);
-void PrintDynamicRelocations(const object::ObjectFile *o);
-void PrintSectionHeaders(const object::ObjectFile *o);
-void PrintSectionContents(const object::ObjectFile *o);
-void PrintSymbolTable(const object::ObjectFile *o, StringRef ArchiveName,
+void printCOFFFileHeader(const object::ObjectFile *O);
+void printCOFFSymbolTable(const object::COFFImportFile *I);
+void printCOFFSymbolTable(const object::COFFObjectFile *O);
+void printMachOFileHeader(const object::ObjectFile *O);
+void printMachOLoadCommands(const object::ObjectFile *O);
+void printWasmFileHeader(const object::ObjectFile *O);
+void printExportsTrie(const object::ObjectFile *O);
+void printRebaseTable(object::ObjectFile *O);
+void printBindTable(object::ObjectFile *O);
+void printLazyBindTable(object::ObjectFile *O);
+void printWeakBindTable(object::ObjectFile *O);
+void printRawClangAST(const object::ObjectFile *O);
+void printRelocations(const object::ObjectFile *O);
+void printDynamicRelocations(const object::ObjectFile *O);
+void printSectionHeaders(const object::ObjectFile *O);
+void printSectionContents(const object::ObjectFile *O);
+void printSymbolTable(const object::ObjectFile *O, StringRef ArchiveName,
                       StringRef ArchitectureName = StringRef());
 void warn(StringRef Message);
 LLVM_ATTRIBUTE_NORETURN void error(Twine Message);
diff --git a/tools/llvm-opt-fuzzer/DummyOptFuzzer.cpp b/tools/llvm-opt-fuzzer/DummyOptFuzzer.cpp
index 115923fe33c2ca6402927a4dd3c63fad004ec858..8d6b07b00890e22b00d25937f92eb4bd17ff0e1d 100644
--- a/tools/llvm-opt-fuzzer/DummyOptFuzzer.cpp
+++ b/tools/llvm-opt-fuzzer/DummyOptFuzzer.cpp
@@ -1,9 +1,8 @@
 //===--- DummyOptFuzzer.cpp - Entry point to sanity check the fuzzer ------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/tools/llvm-opt-fuzzer/llvm-opt-fuzzer.cpp b/tools/llvm-opt-fuzzer/llvm-opt-fuzzer.cpp
index 57e75b1db9ece9e9e3a447bf1187d9bc0b493fae..6567aaebd6c3913bb5267719e6273e9db0761d5b 100644
--- a/tools/llvm-opt-fuzzer/llvm-opt-fuzzer.cpp
+++ b/tools/llvm-opt-fuzzer/llvm-opt-fuzzer.cpp
@@ -1,9 +1,8 @@
 //===--- llvm-opt-fuzzer.cpp - Fuzzer for instruction selection ----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/tools/llvm-opt-report/OptReport.cpp b/tools/llvm-opt-report/OptReport.cpp
index 071f779a9e62c1ca97c83c867c639380424a994e..a06f15447fb6e719f717d2e80cab2b9e77c920db 100644
--- a/tools/llvm-opt-report/OptReport.cpp
+++ b/tools/llvm-opt-report/OptReport.cpp
@@ -1,9 +1,8 @@
 //===------------------ llvm-opt-report/OptReport.cpp ---------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
@@ -231,13 +230,8 @@ static bool writeReport(LocationInfoTy &LocationInfo) {
   bool FirstFile = true;
   for (auto &FI : LocationInfo) {
     SmallString<128> FileName(FI.first);
-    if (!InputRelDir.empty()) {
-      if (std::error_code EC = sys::fs::make_absolute(InputRelDir, FileName)) {
-        WithColor::error() << "Can't resolve file path to " << FileName << ": "
-                           << EC.message() << "\n";
-        return false;
-      }
-    }
+    if (!InputRelDir.empty())
+      sys::fs::make_absolute(InputRelDir, FileName);
 
     const auto &FileInfo = FI.second;
 
diff --git a/tools/llvm-pdbutil/BytesOutputStyle.cpp b/tools/llvm-pdbutil/BytesOutputStyle.cpp
index 2b96c8f986aa0ef701735938eebeb79a5cc3a224..a8ea5ba897cba9af0af903731a199a366c1054a0 100644
--- a/tools/llvm-pdbutil/BytesOutputStyle.cpp
+++ b/tools/llvm-pdbutil/BytesOutputStyle.cpp
@@ -1,9 +1,8 @@
 //===- BytesOutputStyle.cpp ----------------------------------- *- C++ --*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/tools/llvm-pdbutil/BytesOutputStyle.h b/tools/llvm-pdbutil/BytesOutputStyle.h
index aa5342998e56c9c1df7f481ab753ff2ca65c3c90..d3aceb47679e89ab0312d72a27e17265a838df54 100644
--- a/tools/llvm-pdbutil/BytesOutputStyle.h
+++ b/tools/llvm-pdbutil/BytesOutputStyle.h
@@ -1,9 +1,8 @@
 //===- BytesOutputStyle.h ------------------------------------- *- C++ --*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/tools/llvm-pdbutil/DumpOutputStyle.cpp b/tools/llvm-pdbutil/DumpOutputStyle.cpp
index e4f6aa7f6ec583d7582ea9e8e48b8b7f3e357e48..23f9af6ab10c9ff01c1014411be967cc798e011a 100644
--- a/tools/llvm-pdbutil/DumpOutputStyle.cpp
+++ b/tools/llvm-pdbutil/DumpOutputStyle.cpp
@@ -1,9 +1,8 @@
 //===- DumpOutputStyle.cpp ------------------------------------ *- C++ --*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -1011,17 +1010,12 @@ Error DumpOutputStyle::dumpOldFpo(PDBFile &File) {
   ExitOnError Err("Error dumping old fpo data:");
   auto &Dbi = Err(File.getPDBDbiStream());
 
-  uint32_t Index = Dbi.getDebugStreamIndex(DbgHeaderType::FPO);
-  if (Index == kInvalidStreamIndex) {
+  if (!Dbi.hasOldFpoRecords()) {
     printStreamNotPresent("FPO");
     return Error::success();
   }
 
-  std::unique_ptr<MappedBlockStream> OldFpo = File.createIndexedStream(Index);
-  BinaryStreamReader Reader(*OldFpo);
-  FixedStreamArray<object::FpoData> Records;
-  Err(Reader.readArray(Records,
-                       Reader.bytesRemaining() / sizeof(object::FpoData)));
+  const FixedStreamArray<object::FpoData>& Records = Dbi.getOldFpoRecords();
 
   P.printLine("  RVA    | Code | Locals | Params | Prolog | Saved Regs | Use "
               "BP | Has SEH | Frame Type");
@@ -1043,18 +1037,12 @@ Error DumpOutputStyle::dumpNewFpo(PDBFile &File) {
   ExitOnError Err("Error dumping new fpo data:");
   auto &Dbi = Err(File.getPDBDbiStream());
 
-  uint32_t Index = Dbi.getDebugStreamIndex(DbgHeaderType::NewFPO);
-  if (Index == kInvalidStreamIndex) {
+  if (!Dbi.hasNewFpoRecords()) {
     printStreamNotPresent("New FPO");
     return Error::success();
   }
 
-  std::unique_ptr<MappedBlockStream> NewFpo = File.createIndexedStream(Index);
-
-  DebugFrameDataSubsectionRef FDS;
-  if (auto EC = FDS.initialize(*NewFpo))
-    return make_error<RawError>(raw_error_code::corrupt_file,
-                                "Invalid new fpo stream");
+  const DebugFrameDataSubsectionRef& FDS = Dbi.getNewFpoRecords();
 
   P.printLine("  RVA    | Code | Locals | Params | Stack | Prolog | Saved Regs "
               "| Has SEH | Has C++EH | Start | Program");
@@ -1413,6 +1401,15 @@ Error DumpOutputStyle::dumpTpiStream(uint32_t StreamIdx) {
 
   if (DumpExtras) {
     P.NewLine();
+
+    P.formatLine("Header Version: {0}",
+                 static_cast<uint32_t>(Stream.getTpiVersion()));
+    P.formatLine("Hash Stream Index: {0}", Stream.getTypeHashStreamIndex());
+    P.formatLine("Aux Hash Stream Index: {0}",
+                 Stream.getTypeHashStreamAuxIndex());
+    P.formatLine("Hash Key Size: {0}", Stream.getHashKeySize());
+    P.formatLine("Num Hash Buckets: {0}", Stream.getNumHashBuckets());
+
     auto IndexOffsets = Stream.getTypeIndexOffsets();
     P.formatLine("Type Index Offsets:");
     for (const auto &IO : IndexOffsets) {
diff --git a/tools/llvm-pdbutil/DumpOutputStyle.h b/tools/llvm-pdbutil/DumpOutputStyle.h
index 9b3a85587bde3b00bce4673fa3c9ff2bbbd44bfc..c95490e95df43823b14bcdf26421875d2554aa73 100644
--- a/tools/llvm-pdbutil/DumpOutputStyle.h
+++ b/tools/llvm-pdbutil/DumpOutputStyle.h
@@ -1,9 +1,8 @@
 //===- DumpOutputStyle.h -------------------------------------- *- C++ --*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/tools/llvm-pdbutil/ExplainOutputStyle.cpp b/tools/llvm-pdbutil/ExplainOutputStyle.cpp
index d16bfa480e1d84dc3ebd3ae155763fe354a6c6f6..94faa04639817f91864ca84f4d5d0c2006522809 100644
--- a/tools/llvm-pdbutil/ExplainOutputStyle.cpp
+++ b/tools/llvm-pdbutil/ExplainOutputStyle.cpp
@@ -1,9 +1,8 @@
 //===- ExplainOutputStyle.cpp --------------------------------- *- C++ --*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/tools/llvm-pdbutil/ExplainOutputStyle.h b/tools/llvm-pdbutil/ExplainOutputStyle.h
index 9a497accb812d4009a995f72fb3bf985f8320b68..f405cf615e924580432a14e4cc12a2fe4bbd38c2 100644
--- a/tools/llvm-pdbutil/ExplainOutputStyle.h
+++ b/tools/llvm-pdbutil/ExplainOutputStyle.h
@@ -1,9 +1,8 @@
 //===- ExplainOutputStyle.h ----------------------------------- *- C++ --*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/tools/llvm-pdbutil/FormatUtil.cpp b/tools/llvm-pdbutil/FormatUtil.cpp
index f55d478127d6bdc0b675d514e6dcc01929cafdda..1a13f383e53c86913ac4fcfd50632d58acf873a0 100644
--- a/tools/llvm-pdbutil/FormatUtil.cpp
+++ b/tools/llvm-pdbutil/FormatUtil.cpp
@@ -1,9 +1,8 @@
 //===- FormatUtil.cpp ----------------------------------------- *- C++ --*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/tools/llvm-pdbutil/FormatUtil.h b/tools/llvm-pdbutil/FormatUtil.h
index 9a003c9285c9484d6ab06aa508db891e16ead709..19ce248f9a6f0b3e560655ec8344693a8c585127 100644
--- a/tools/llvm-pdbutil/FormatUtil.h
+++ b/tools/llvm-pdbutil/FormatUtil.h
@@ -1,9 +1,8 @@
 //===- FormatUtil.h ------------------------------------------- *- C++ --*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/tools/llvm-pdbutil/InputFile.cpp b/tools/llvm-pdbutil/InputFile.cpp
index 8eb116cf0d80c01f2e6de58b79dc7f313e62c2d5..f61d89bd70c395147a48bb85e7a5b8dde812cc7e 100644
--- a/tools/llvm-pdbutil/InputFile.cpp
+++ b/tools/llvm-pdbutil/InputFile.cpp
@@ -1,9 +1,8 @@
 //===- InputFile.cpp ------------------------------------------ *- C++ --*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/tools/llvm-pdbutil/InputFile.h b/tools/llvm-pdbutil/InputFile.h
index ee4e651c1e99893f3d092d203c6e577dde0d4549..f25390c971d061c164e85584c7d5c831dc26bb2c 100644
--- a/tools/llvm-pdbutil/InputFile.h
+++ b/tools/llvm-pdbutil/InputFile.h
@@ -1,9 +1,8 @@
 //===- InputFile.h -------------------------------------------- *- C++ --*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/tools/llvm-pdbutil/LLVMBuild.txt b/tools/llvm-pdbutil/LLVMBuild.txt
index adbe4993e3fa81b2e1aff8d7734f0794272bb722..c694058dabd2f9e0eb289ce9faf446f743607d1f 100644
--- a/tools/llvm-pdbutil/LLVMBuild.txt
+++ b/tools/llvm-pdbutil/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./tools/llvm-pdbutil/LLVMBuild.txt -----------------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/tools/llvm-pdbutil/LinePrinter.cpp b/tools/llvm-pdbutil/LinePrinter.cpp
index e80a1762450b9d85c34660dddc1d9a304f5c9b29..aae8809be631b9aec7cfd8a3b593ea2fe5a69059 100644
--- a/tools/llvm-pdbutil/LinePrinter.cpp
+++ b/tools/llvm-pdbutil/LinePrinter.cpp
@@ -1,9 +1,8 @@
 //===- LinePrinter.cpp ------------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/tools/llvm-pdbutil/LinePrinter.h b/tools/llvm-pdbutil/LinePrinter.h
index 09bde28f516aa9888a33264fbbbf9b5c67dc3e86..74e341267accda4b921ed8e3ebb1ff2b663d888a 100644
--- a/tools/llvm-pdbutil/LinePrinter.h
+++ b/tools/llvm-pdbutil/LinePrinter.h
@@ -1,9 +1,8 @@
 //===- LinePrinter.h ------------------------------------------ *- C++ --*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/tools/llvm-pdbutil/MinimalSymbolDumper.cpp b/tools/llvm-pdbutil/MinimalSymbolDumper.cpp
index 2c7b213b0a9fcb88e7663e75e64b4bc6b9ac5a4c..6093ab9f2f2c883f7606a6a064ecff5fdf0355e6 100644
--- a/tools/llvm-pdbutil/MinimalSymbolDumper.cpp
+++ b/tools/llvm-pdbutil/MinimalSymbolDumper.cpp
@@ -1,9 +1,8 @@
 //===- MinimalSymbolDumper.cpp -------------------------------- *- C++ --*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/tools/llvm-pdbutil/MinimalSymbolDumper.h b/tools/llvm-pdbutil/MinimalSymbolDumper.h
index 033e193cee6c1c3e58743b08e197d85674d8516e..cdc75c1cfba0ea78ec94548985ea5fb9e1839303 100644
--- a/tools/llvm-pdbutil/MinimalSymbolDumper.h
+++ b/tools/llvm-pdbutil/MinimalSymbolDumper.h
@@ -1,9 +1,8 @@
 //===- MinimalSymbolDumper.h ---------------------------------- *- C++ --*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/tools/llvm-pdbutil/MinimalTypeDumper.cpp b/tools/llvm-pdbutil/MinimalTypeDumper.cpp
index 3f10e8ab8a1e6a8a669cf737c16191586aa904eb..a6622480c9ff2116d165729a3fb97f672d1d42c2 100644
--- a/tools/llvm-pdbutil/MinimalTypeDumper.cpp
+++ b/tools/llvm-pdbutil/MinimalTypeDumper.cpp
@@ -1,9 +1,8 @@
 //===- MinimalTypeDumper.cpp ---------------------------------- *- C++ --*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/tools/llvm-pdbutil/MinimalTypeDumper.h b/tools/llvm-pdbutil/MinimalTypeDumper.h
index 8f6bdc6110aeaf80e4064d86d49e76019e8f25e6..0374e042e423269ddec0dc989a58b55d9424e2c1 100644
--- a/tools/llvm-pdbutil/MinimalTypeDumper.h
+++ b/tools/llvm-pdbutil/MinimalTypeDumper.h
@@ -1,9 +1,8 @@
 //===- MinimalTypeDumper.h ------------------------------------ *- C++ --*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/tools/llvm-pdbutil/OutputStyle.h b/tools/llvm-pdbutil/OutputStyle.h
index dfefc25a215e88f08d1302fd84559c3aa57eec32..40b0de8bdf726708cebde4891855b894d7041732 100644
--- a/tools/llvm-pdbutil/OutputStyle.h
+++ b/tools/llvm-pdbutil/OutputStyle.h
@@ -1,9 +1,8 @@
 //===- OutputStyle.h ------------------------------------------ *- C++ --*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/tools/llvm-pdbutil/PdbYaml.cpp b/tools/llvm-pdbutil/PdbYaml.cpp
index 3ea333608314e470d8c5af0bcafa377cd6dff493..d1c404237d9c6912b08b2f8349ed9b1fc6cc55ef 100644
--- a/tools/llvm-pdbutil/PdbYaml.cpp
+++ b/tools/llvm-pdbutil/PdbYaml.cpp
@@ -1,9 +1,8 @@
 //===- PdbYAML.cpp -------------------------------------------- *- C++ --*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/tools/llvm-pdbutil/PdbYaml.h b/tools/llvm-pdbutil/PdbYaml.h
index 97ba87266cc67af85655a6fba5349dbec9c2b73a..ed6346c2c4db0230dd2920b3b0295e2bb6155d41 100644
--- a/tools/llvm-pdbutil/PdbYaml.h
+++ b/tools/llvm-pdbutil/PdbYaml.h
@@ -1,9 +1,8 @@
 //===- PdbYAML.h ---------------------------------------------- *- C++ --*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/tools/llvm-pdbutil/PrettyBuiltinDumper.cpp b/tools/llvm-pdbutil/PrettyBuiltinDumper.cpp
index bcdecca81aecad57c897292ab851e4d2d03cfc69..cd01a4004819157bf3796c7b89e5acc87ee16f9c 100644
--- a/tools/llvm-pdbutil/PrettyBuiltinDumper.cpp
+++ b/tools/llvm-pdbutil/PrettyBuiltinDumper.cpp
@@ -1,9 +1,8 @@
 //===- PrettyBuiltinDumper.cpp ---------------------------------- *- C++ *-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/tools/llvm-pdbutil/PrettyBuiltinDumper.h b/tools/llvm-pdbutil/PrettyBuiltinDumper.h
index fb6b0b172e6ebd96bdfceec72bc3a7203660b603..3bdef34c48f819861d20fe6159ab14ef3147950b 100644
--- a/tools/llvm-pdbutil/PrettyBuiltinDumper.h
+++ b/tools/llvm-pdbutil/PrettyBuiltinDumper.h
@@ -1,9 +1,8 @@
 //===- PrettyBuiltinDumper.h ---------------------------------- *- C++ --*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/tools/llvm-pdbutil/PrettyClassDefinitionDumper.cpp b/tools/llvm-pdbutil/PrettyClassDefinitionDumper.cpp
index f009f53a39325494136506c5480792ecd0b3db48..b7eccac5988caa4aded952d16427ae35a2f466c3 100644
--- a/tools/llvm-pdbutil/PrettyClassDefinitionDumper.cpp
+++ b/tools/llvm-pdbutil/PrettyClassDefinitionDumper.cpp
@@ -1,9 +1,8 @@
 //===- PrettyClassDefinitionDumper.cpp --------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/tools/llvm-pdbutil/PrettyClassDefinitionDumper.h b/tools/llvm-pdbutil/PrettyClassDefinitionDumper.h
index 6569a1d304f6c68f26d9148c144b26249d1fe570..f43c5c11bdfd5173316a37b45d50899e0cc39c9e 100644
--- a/tools/llvm-pdbutil/PrettyClassDefinitionDumper.h
+++ b/tools/llvm-pdbutil/PrettyClassDefinitionDumper.h
@@ -1,9 +1,8 @@
 //===- PrettyClassDefinitionDumper.h ----------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/tools/llvm-pdbutil/PrettyClassLayoutGraphicalDumper.cpp b/tools/llvm-pdbutil/PrettyClassLayoutGraphicalDumper.cpp
index a572522c8cd704b1857f44250e8944fb93b69889..a522935e34f14331804f32468545b858bf4254f2 100644
--- a/tools/llvm-pdbutil/PrettyClassLayoutGraphicalDumper.cpp
+++ b/tools/llvm-pdbutil/PrettyClassLayoutGraphicalDumper.cpp
@@ -1,9 +1,8 @@
 //===- PrettyClassLayoutGraphicalDumper.h -----------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/tools/llvm-pdbutil/PrettyClassLayoutGraphicalDumper.h b/tools/llvm-pdbutil/PrettyClassLayoutGraphicalDumper.h
index f83f1a6c1b34fae6ce7e17633e98c725ee0f103a..8f78b3b503d0dc378eb2cdd90feca0a974100f32 100644
--- a/tools/llvm-pdbutil/PrettyClassLayoutGraphicalDumper.h
+++ b/tools/llvm-pdbutil/PrettyClassLayoutGraphicalDumper.h
@@ -1,9 +1,8 @@
 //===- PrettyClassLayoutGraphicalDumper.h -----------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/tools/llvm-pdbutil/PrettyCompilandDumper.cpp b/tools/llvm-pdbutil/PrettyCompilandDumper.cpp
index 94a0b2d5e780af3abdc617134ffa5ceaf44002ff..cf769ff66472bfe6a548c9d25dcef3a8d561b94d 100644
--- a/tools/llvm-pdbutil/PrettyCompilandDumper.cpp
+++ b/tools/llvm-pdbutil/PrettyCompilandDumper.cpp
@@ -1,9 +1,8 @@
 //===- PrettyCompilandDumper.cpp - llvm-pdbutil compiland dumper -*- C++ *-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/tools/llvm-pdbutil/PrettyCompilandDumper.h b/tools/llvm-pdbutil/PrettyCompilandDumper.h
index 1a840e49607c12a7e7689d211ba98f1696beacae..c83a58672d1adbd5a2c9be29b6a1ef99f47b4eeb 100644
--- a/tools/llvm-pdbutil/PrettyCompilandDumper.h
+++ b/tools/llvm-pdbutil/PrettyCompilandDumper.h
@@ -1,9 +1,8 @@
 //===- PrettyCompilandDumper.h - llvm-pdbutil compiland dumper -*- C++ --*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/tools/llvm-pdbutil/PrettyEnumDumper.cpp b/tools/llvm-pdbutil/PrettyEnumDumper.cpp
index f4cbd3f8fa144c5a077b0a4d43770793c3c452ec..9ed5893f252e8cf9f71552928574d055cfda0ca9 100644
--- a/tools/llvm-pdbutil/PrettyEnumDumper.cpp
+++ b/tools/llvm-pdbutil/PrettyEnumDumper.cpp
@@ -1,9 +1,8 @@
 //===- PrettyEnumDumper.cpp -------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/tools/llvm-pdbutil/PrettyEnumDumper.h b/tools/llvm-pdbutil/PrettyEnumDumper.h
index c6e65a6d1772a126d0754294189d507abdb2ba4a..e7c5c1aeb018526b55d9e722f413e3ed1cb17752 100644
--- a/tools/llvm-pdbutil/PrettyEnumDumper.h
+++ b/tools/llvm-pdbutil/PrettyEnumDumper.h
@@ -1,9 +1,8 @@
 //===- PrettyEnumDumper.h ---------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/tools/llvm-pdbutil/PrettyExternalSymbolDumper.cpp b/tools/llvm-pdbutil/PrettyExternalSymbolDumper.cpp
index 1270223b1c78b1ce39e396b103d7fe6bc8e96b4b..fede031ec0c09484fcb520cbc4056cad5b70c5d5 100644
--- a/tools/llvm-pdbutil/PrettyExternalSymbolDumper.cpp
+++ b/tools/llvm-pdbutil/PrettyExternalSymbolDumper.cpp
@@ -1,9 +1,8 @@
 //===- PrettyExternalSymbolDumper.cpp -------------------------- *- C++ *-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/tools/llvm-pdbutil/PrettyExternalSymbolDumper.h b/tools/llvm-pdbutil/PrettyExternalSymbolDumper.h
index 6a009862ddd4609fe5692e982c79f01f79c1bb6b..58fafe943315f87854ad880fca3a96c5fac7257a 100644
--- a/tools/llvm-pdbutil/PrettyExternalSymbolDumper.h
+++ b/tools/llvm-pdbutil/PrettyExternalSymbolDumper.h
@@ -1,9 +1,8 @@
 //===- PrettyExternalSymbolDumper.h --------------------------- *- C++ --*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/tools/llvm-pdbutil/PrettyFunctionDumper.cpp b/tools/llvm-pdbutil/PrettyFunctionDumper.cpp
index 836ede41054ec8b8e74f9d816346727fde91275f..126ee97eca1133b6590704d1098f1f400b743f36 100644
--- a/tools/llvm-pdbutil/PrettyFunctionDumper.cpp
+++ b/tools/llvm-pdbutil/PrettyFunctionDumper.cpp
@@ -1,9 +1,8 @@
 //===- PrettyFunctionDumper.cpp --------------------------------- *- C++ *-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/tools/llvm-pdbutil/PrettyFunctionDumper.h b/tools/llvm-pdbutil/PrettyFunctionDumper.h
index 1a6f5430ec5abb0de1d552af45ab43bc52f2b921..df62604ac8815e8f3cb1367152370e6f013585dd 100644
--- a/tools/llvm-pdbutil/PrettyFunctionDumper.h
+++ b/tools/llvm-pdbutil/PrettyFunctionDumper.h
@@ -1,9 +1,8 @@
 //===- PrettyFunctionDumper.h --------------------------------- *- C++ --*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/tools/llvm-pdbutil/PrettyTypeDumper.cpp b/tools/llvm-pdbutil/PrettyTypeDumper.cpp
index daf3cd45b3272cd18cd273574506d84a7b746108..e8f8e5aa62c9f49e0d3eb29782040500a27f2c3b 100644
--- a/tools/llvm-pdbutil/PrettyTypeDumper.cpp
+++ b/tools/llvm-pdbutil/PrettyTypeDumper.cpp
@@ -1,9 +1,8 @@
 //===- PrettyTypeDumper.cpp - PDBSymDumper type dumper *------------ C++ *-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/tools/llvm-pdbutil/PrettyTypeDumper.h b/tools/llvm-pdbutil/PrettyTypeDumper.h
index 36e586fea7e3166e014d5579990392497cab3331..b6539d95bf31bebb6a0bc786e7433dc8e709468d 100644
--- a/tools/llvm-pdbutil/PrettyTypeDumper.h
+++ b/tools/llvm-pdbutil/PrettyTypeDumper.h
@@ -1,9 +1,8 @@
 //===- PrettyTypeDumper.h - PDBSymDumper implementation for types *- C++ *-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/tools/llvm-pdbutil/PrettyTypedefDumper.cpp b/tools/llvm-pdbutil/PrettyTypedefDumper.cpp
index 2b3f3691ed98e78945c27a78bba4cc850a57b619..ef73a8cdf9c48e5ab39ee35cb67149e6e90b83c7 100644
--- a/tools/llvm-pdbutil/PrettyTypedefDumper.cpp
+++ b/tools/llvm-pdbutil/PrettyTypedefDumper.cpp
@@ -1,9 +1,8 @@
 //===- PrettyTypedefDumper.cpp - PDBSymDumper impl for typedefs -- * C++ *-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/tools/llvm-pdbutil/PrettyTypedefDumper.h b/tools/llvm-pdbutil/PrettyTypedefDumper.h
index 133bbfb7db0eb7b2bdf01dd35da61c6db74eb9ce..ad8b3f37dcfd40d11f8d1d7eb536837d0c8bdec9 100644
--- a/tools/llvm-pdbutil/PrettyTypedefDumper.h
+++ b/tools/llvm-pdbutil/PrettyTypedefDumper.h
@@ -1,9 +1,8 @@
 //===- PrettyTypedefDumper.h - llvm-pdbutil typedef dumper ---*- C++ ----*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/tools/llvm-pdbutil/PrettyVariableDumper.cpp b/tools/llvm-pdbutil/PrettyVariableDumper.cpp
index ddac8cf0da4a89c846a394a6f72ad4a2764947e5..6dd7cc384cc9e724d7886d6a9d5a065a4d8fe5f7 100644
--- a/tools/llvm-pdbutil/PrettyVariableDumper.cpp
+++ b/tools/llvm-pdbutil/PrettyVariableDumper.cpp
@@ -1,9 +1,8 @@
 //===- PrettyVariableDumper.cpp ---------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/tools/llvm-pdbutil/PrettyVariableDumper.h b/tools/llvm-pdbutil/PrettyVariableDumper.h
index cacf1ce9577b03eda8d53525c6527a5904d0ea3a..65cf5cd2cf5522286f6c9b4dd3186d87d315aacd 100644
--- a/tools/llvm-pdbutil/PrettyVariableDumper.h
+++ b/tools/llvm-pdbutil/PrettyVariableDumper.h
@@ -1,9 +1,8 @@
 //===- PrettyVariableDumper.h - PDBSymDumper variable dumper ----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/tools/llvm-pdbutil/StreamUtil.cpp b/tools/llvm-pdbutil/StreamUtil.cpp
index 367d947d25eea5b8344e3b5d9d18b0c59b973d5e..7dfc2beefe78df76f151f5f45117dadb63f052b1 100644
--- a/tools/llvm-pdbutil/StreamUtil.cpp
+++ b/tools/llvm-pdbutil/StreamUtil.cpp
@@ -1,9 +1,8 @@
 //===- StreamUtil.cpp - PDB stream utilities --------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/tools/llvm-pdbutil/StreamUtil.h b/tools/llvm-pdbutil/StreamUtil.h
index 0e2e80707361c0ed8f699b08b888100aea566e38..f810f7dc15b4c8399201e7a2617d1b69bf3b5312 100644
--- a/tools/llvm-pdbutil/StreamUtil.h
+++ b/tools/llvm-pdbutil/StreamUtil.h
@@ -1,9 +1,8 @@
 //===- Streamutil.h - PDB stream utilities ----------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/tools/llvm-pdbutil/YAMLOutputStyle.cpp b/tools/llvm-pdbutil/YAMLOutputStyle.cpp
index 62b5c428d41026e0bd1ac6d71377756a2ad796a4..f81381e17276dc75bcb073b76ddd9cec48d1d30c 100644
--- a/tools/llvm-pdbutil/YAMLOutputStyle.cpp
+++ b/tools/llvm-pdbutil/YAMLOutputStyle.cpp
@@ -1,9 +1,8 @@
 //===- YAMLOutputStyle.cpp ------------------------------------ *- C++ --*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/tools/llvm-pdbutil/YAMLOutputStyle.h b/tools/llvm-pdbutil/YAMLOutputStyle.h
index a5ad3355d2ab4d8b7996ba0bd5f9c23693ea72c2..7a50af1abe3f55fa23ae05240f2cf8ab13f5d9f6 100644
--- a/tools/llvm-pdbutil/YAMLOutputStyle.h
+++ b/tools/llvm-pdbutil/YAMLOutputStyle.h
@@ -1,9 +1,8 @@
 //===- YAMLOutputStyle.h -------------------------------------- *- C++ --*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/tools/llvm-pdbutil/llvm-pdbutil.cpp b/tools/llvm-pdbutil/llvm-pdbutil.cpp
index 76f61a2a95a79ae7a2513075856dfdea1ebc40fe..418ba135dc79d515012d79a1c9690da0c24a81f0 100644
--- a/tools/llvm-pdbutil/llvm-pdbutil.cpp
+++ b/tools/llvm-pdbutil/llvm-pdbutil.cpp
@@ -1,9 +1,8 @@
 //===- llvm-pdbutil.cpp - Dump debug info from a PDB file -------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/tools/llvm-pdbutil/llvm-pdbutil.h b/tools/llvm-pdbutil/llvm-pdbutil.h
index a57cc51d7fd79b028bc93b214601a2dbc9928c30..a54bfcdb72226455044b0c8158a25a032c28eef8 100644
--- a/tools/llvm-pdbutil/llvm-pdbutil.h
+++ b/tools/llvm-pdbutil/llvm-pdbutil.h
@@ -1,9 +1,8 @@
 //===- llvm-pdbutil.h ----------------------------------------- *- C++ --*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/tools/llvm-profdata/LLVMBuild.txt b/tools/llvm-profdata/LLVMBuild.txt
index bbc1ea6fceb9cfcabed8a6c7e62500e5698397ba..6378fafd99b0484ed9d2e0a61d270212a5b5d5fd 100644
--- a/tools/llvm-profdata/LLVMBuild.txt
+++ b/tools/llvm-profdata/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./tools/llvm-profdata/LLVMBuild.txt ----------------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/tools/llvm-profdata/llvm-profdata.cpp b/tools/llvm-profdata/llvm-profdata.cpp
index db3c3042ed790a4243ad4f71a217f89d2f9119db..a8d94e838e890ecd6e7971833e43e1e3a65c68d8 100644
--- a/tools/llvm-profdata/llvm-profdata.cpp
+++ b/tools/llvm-profdata/llvm-profdata.cpp
@@ -1,9 +1,8 @@
 //===- llvm-profdata.cpp - LLVM profile data tool -------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -633,13 +632,21 @@ static void traverseAllValueSites(const InstrProfRecord &Func, uint32_t VK,
         Stats.ValueSitesHistogram.resize(NV, 0);
       Stats.ValueSitesHistogram[NV - 1]++;
     }
+
+    uint64_t SiteSum = 0;
+    for (uint32_t V = 0; V < NV; V++)
+      SiteSum += VD[V].Count;
+    if (SiteSum == 0)
+      SiteSum = 1;
+
     for (uint32_t V = 0; V < NV; V++) {
-      OS << "\t[ " << I << ", ";
+      OS << "\t[ " << format("%2u", I) << ", ";
       if (Symtab == nullptr)
-        OS << VD[V].Value;
+        OS << format("%4" PRIu64, VD[V].Value);
       else
         OS << Symtab->getFuncName(VD[V].Value);
-      OS << ", " << VD[V].Count << " ]\n";
+      OS << ", " << format("%10" PRId64, VD[V].Count) << " ] ("
+         << format("%.2f%%", (VD[V].Count * 100.0 / SiteSum)) << ")\n";
     }
   }
 }
@@ -662,9 +669,9 @@ static int showInstrProfile(const std::string &Filename, bool ShowCounts,
                             uint32_t TopN, bool ShowIndirectCallTargets,
                             bool ShowMemOPSizes, bool ShowDetailedSummary,
                             std::vector<uint32_t> DetailedSummaryCutoffs,
-                            bool ShowAllFunctions,
-                            const std::string &ShowFunction, bool TextFormat,
-                            raw_fd_ostream &OS) {
+                            bool ShowAllFunctions, uint64_t ValueCutoff,
+                            bool OnlyListBelow, const std::string &ShowFunction,
+                            bool TextFormat, raw_fd_ostream &OS) {
   auto ReaderOrErr = InstrProfReader::create(Filename);
   std::vector<uint32_t> Cutoffs = std::move(DetailedSummaryCutoffs);
   if (ShowDetailedSummary && Cutoffs.empty()) {
@@ -677,6 +684,7 @@ static int showInstrProfile(const std::string &Filename, bool ShowCounts,
   auto Reader = std::move(ReaderOrErr.get());
   bool IsIRInstr = Reader->isIRLevelProfile();
   size_t ShownFunctions = 0;
+  size_t BelowCutoffFunctions = 0;
   int NumVPKind = IPVK_Last - IPVK_First + 1;
   std::vector<ValueSitesStats> VPStats(NumVPKind);
 
@@ -690,6 +698,11 @@ static int showInstrProfile(const std::string &Filename, bool ShowCounts,
                       decltype(MinCmp)>
       HottestFuncs(MinCmp);
 
+  if (!TextFormat && OnlyListBelow) {
+    OS << "The list of functions with the maximum counter less than "
+       << ValueCutoff << ":\n";
+  }
+
   // Add marker so that IR-level instrumentation round-trips properly.
   if (TextFormat && IsIRInstr)
     OS << ":ir\n";
@@ -711,11 +724,24 @@ static int showInstrProfile(const std::string &Filename, bool ShowCounts,
     assert(Func.Counts.size() > 0 && "function missing entry counter");
     Builder.addRecord(Func);
 
-    if (TopN) {
-      uint64_t FuncMax = 0;
-      for (size_t I = 0, E = Func.Counts.size(); I < E; ++I)
-        FuncMax = std::max(FuncMax, Func.Counts[I]);
+    uint64_t FuncMax = 0;
+    uint64_t FuncSum = 0;
+    for (size_t I = 0, E = Func.Counts.size(); I < E; ++I) {
+      FuncMax = std::max(FuncMax, Func.Counts[I]);
+      FuncSum += Func.Counts[I];
+    }
 
+    if (FuncMax < ValueCutoff) {
+      ++BelowCutoffFunctions;
+      if (OnlyListBelow) {
+        OS << "  " << Func.Name << ": (Max = " << FuncMax
+           << " Sum = " << FuncSum << ")\n";
+      }
+      continue;
+    } else if (OnlyListBelow)
+      continue;
+
+    if (TopN) {
       if (HottestFuncs.size() == TopN) {
         if (HottestFuncs.top().second < FuncMax) {
           HottestFuncs.pop();
@@ -726,7 +752,6 @@ static int showInstrProfile(const std::string &Filename, bool ShowCounts,
     }
 
     if (Show) {
-
       if (!ShownFunctions)
         OS << "Counters:\n";
 
@@ -781,6 +806,12 @@ static int showInstrProfile(const std::string &Filename, bool ShowCounts,
   if (ShowAllFunctions || !ShowFunction.empty())
     OS << "Functions shown: " << ShownFunctions << "\n";
   OS << "Total functions: " << PS->getNumFunctions() << "\n";
+  if (ValueCutoff > 0) {
+    OS << "Number of functions with maximum count (< " << ValueCutoff
+       << "): " << BelowCutoffFunctions << "\n";
+    OS << "Number of functions with maximum count (>= " << ValueCutoff
+       << "): " << PS->getNumFunctions() - BelowCutoffFunctions << "\n";
+  }
   OS << "Maximum function count: " << PS->getMaxFunctionCount() << "\n";
   OS << "Maximum internal block count: " << PS->getMaxInternalCount() << "\n";
 
@@ -882,7 +913,14 @@ static int show_main(int argc, const char *argv[]) {
   cl::opt<uint32_t> TopNFunctions(
       "topn", cl::init(0),
       cl::desc("Show the list of functions with the largest internal counts"));
-
+  cl::opt<uint32_t> ValueCutoff(
+      "value-cutoff", cl::init(0),
+      cl::desc("Set the count value cutoff. Functions with the maximum count "
+               "less than this value will not be printed out. (Default is 0)"));
+  cl::opt<bool> OnlyListBelow(
+      "list-below-cutoff", cl::init(false),
+      cl::desc("Only output names of functions whose max count values are "
+               "below the cutoff value"));
   cl::ParseCommandLineOptions(argc, argv, "LLVM profile data summary\n");
 
   if (OutputFilename.empty())
@@ -902,7 +940,8 @@ static int show_main(int argc, const char *argv[]) {
     return showInstrProfile(Filename, ShowCounts, TopNFunctions,
                             ShowIndirectCallTargets, ShowMemOPSizes,
                             ShowDetailedSummary, DetailedSummaryCutoffs,
-                            ShowAllFunctions, ShowFunction, TextFormat, OS);
+                            ShowAllFunctions, ValueCutoff, OnlyListBelow,
+                            ShowFunction, TextFormat, OS);
   else
     return showSampleProfile(Filename, ShowCounts, ShowAllFunctions,
                              ShowFunction, OS);
diff --git a/tools/llvm-rc/LLVMBuild.txt b/tools/llvm-rc/LLVMBuild.txt
index fec4d33698830b8b95d9723eedf49253348e5bcc..1a96eaa0540139e4b04776d9849084e7f7c4f7f3 100644
--- a/tools/llvm-rc/LLVMBuild.txt
+++ b/tools/llvm-rc/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./tools/llvm-rc/LLVMBuild.txt ----------------------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/tools/llvm-rc/ResourceFileWriter.cpp b/tools/llvm-rc/ResourceFileWriter.cpp
index 7fe95669083bf65386c68eef013eedc0553c8f2a..78d99e7ebcb4073f4b698bb6c69fa7f535f061e7 100644
--- a/tools/llvm-rc/ResourceFileWriter.cpp
+++ b/tools/llvm-rc/ResourceFileWriter.cpp
@@ -1,9 +1,8 @@
 //===-- ResourceFileWriter.cpp --------------------------------*- C++-*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===---------------------------------------------------------------------===//
 //
diff --git a/tools/llvm-rc/ResourceFileWriter.h b/tools/llvm-rc/ResourceFileWriter.h
index 36d57da11eb6105acbf8b76d4047734bb3d1cf95..1d7583db1149e856768a880bd9e656c56f4f4bbc 100644
--- a/tools/llvm-rc/ResourceFileWriter.h
+++ b/tools/llvm-rc/ResourceFileWriter.h
@@ -1,9 +1,8 @@
 //===-- ResourceSerializator.h ----------------------------------*- C++-*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===---------------------------------------------------------------------===//
 //
diff --git a/tools/llvm-rc/ResourceScriptCppFilter.cpp b/tools/llvm-rc/ResourceScriptCppFilter.cpp
index 07740f9e8bf02dbf6deffa8b517c7efcf40a275b..e610be99dfb4d2476d105743cff62e2e2749a104 100644
--- a/tools/llvm-rc/ResourceScriptCppFilter.cpp
+++ b/tools/llvm-rc/ResourceScriptCppFilter.cpp
@@ -1,9 +1,8 @@
 //===-- ResourceScriptCppFilter.cpp ----------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===---------------------------------------------------------------------===//
 //
diff --git a/tools/llvm-rc/ResourceScriptCppFilter.h b/tools/llvm-rc/ResourceScriptCppFilter.h
index 2e61a075fb8dcd265ad470cb6a95e736b436b306..780db317c9f03a823ed51826e36241010af2cd32 100644
--- a/tools/llvm-rc/ResourceScriptCppFilter.h
+++ b/tools/llvm-rc/ResourceScriptCppFilter.h
@@ -1,9 +1,8 @@
 //===-- ResourceScriptCppFilter.h ------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===---------------------------------------------------------------------===//
 //
diff --git a/tools/llvm-rc/ResourceScriptParser.cpp b/tools/llvm-rc/ResourceScriptParser.cpp
index c66fc4fc2e70c3516f2fea4c2dd1c11448edc171..56534e7308772653fbe72d8f96759ca384af36c8 100644
--- a/tools/llvm-rc/ResourceScriptParser.cpp
+++ b/tools/llvm-rc/ResourceScriptParser.cpp
@@ -1,9 +1,8 @@
 //===-- ResourceScriptParser.cpp --------------------------------*- C++-*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===---------------------------------------------------------------------===//
 //
diff --git a/tools/llvm-rc/ResourceScriptParser.h b/tools/llvm-rc/ResourceScriptParser.h
index 20d8ff9624d9f21f6c18130bab9e26818d800c46..fe202073b6bb0ccf385357fd96f74bb2c206d825 100644
--- a/tools/llvm-rc/ResourceScriptParser.h
+++ b/tools/llvm-rc/ResourceScriptParser.h
@@ -1,9 +1,8 @@
 //===-- ResourceScriptParser.h ----------------------------------*- C++-*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===---------------------------------------------------------------------===//
 //
diff --git a/tools/llvm-rc/ResourceScriptStmt.cpp b/tools/llvm-rc/ResourceScriptStmt.cpp
index 7b8b0def4c9a955ae95abb1dac3046870e203261..a0d4adbe641890007d0aa6d0c023580886228aaa 100644
--- a/tools/llvm-rc/ResourceScriptStmt.cpp
+++ b/tools/llvm-rc/ResourceScriptStmt.cpp
@@ -1,8 +1,7 @@
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===---------------------------------------------------------------------===//
 //
diff --git a/tools/llvm-rc/ResourceScriptStmt.h b/tools/llvm-rc/ResourceScriptStmt.h
index 3ba1f6619eb8d2a7c0329034bb3b78a5e665d43c..fcb55aa76d6dce9a2bfd1635dbd68ea9684a317c 100644
--- a/tools/llvm-rc/ResourceScriptStmt.h
+++ b/tools/llvm-rc/ResourceScriptStmt.h
@@ -1,9 +1,8 @@
 //===-- ResourceScriptStmt.h ------------------------------------*- C++-*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===---------------------------------------------------------------------===//
 //
diff --git a/tools/llvm-rc/ResourceScriptToken.cpp b/tools/llvm-rc/ResourceScriptToken.cpp
index fc0a0e91845076e672074e31ba2586d6cee7eacd..bb0b3bdab03cff23de5cefe6c0af230a6fea1737 100644
--- a/tools/llvm-rc/ResourceScriptToken.cpp
+++ b/tools/llvm-rc/ResourceScriptToken.cpp
@@ -1,9 +1,8 @@
 //===-- ResourceScriptToken.cpp ---------------------------------*- C++-*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===---------------------------------------------------------------------===//
 //
diff --git a/tools/llvm-rc/ResourceScriptToken.h b/tools/llvm-rc/ResourceScriptToken.h
index d1a2ae7a41bbfeb83600d119d9b840922cea974a..cc8ca48b4907ea84e9883038d585932a4420afc5 100644
--- a/tools/llvm-rc/ResourceScriptToken.h
+++ b/tools/llvm-rc/ResourceScriptToken.h
@@ -1,9 +1,8 @@
 //===-- ResourceScriptToken.h -----------------------------------*- C++-*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===---------------------------------------------------------------------===//
 //
diff --git a/tools/llvm-rc/ResourceScriptTokenList.def b/tools/llvm-rc/ResourceScriptTokenList.def
index 085b728ff5208c9f415c4844da6dc7841c859b27..a61a96461f0fb66f3c1a0eb5d1d3d6d87741fed6 100644
--- a/tools/llvm-rc/ResourceScriptTokenList.def
+++ b/tools/llvm-rc/ResourceScriptTokenList.def
@@ -1,9 +1,8 @@
 //===-- ResourceScriptTokenList.h -------------------------------*- C++-*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===---------------------------------------------------------------------===//
 //
diff --git a/tools/llvm-rc/ResourceVisitor.h b/tools/llvm-rc/ResourceVisitor.h
index 2117b12d8f5f102527e2447d2013f8f7acb9c4ee..843c8d898a211225a4e85fb23f39723d302e995d 100644
--- a/tools/llvm-rc/ResourceVisitor.h
+++ b/tools/llvm-rc/ResourceVisitor.h
@@ -1,9 +1,8 @@
 //===-- ResourceVisitor.h ---------------------------------------*- C++-*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===---------------------------------------------------------------------===//
 //
diff --git a/tools/llvm-rc/llvm-rc.cpp b/tools/llvm-rc/llvm-rc.cpp
index 4511c5c3128385788a3da2504448471be5bbc0f0..4c115b7f06f8a786955e42ca5a0ad6097b6d7894 100644
--- a/tools/llvm-rc/llvm-rc.cpp
+++ b/tools/llvm-rc/llvm-rc.cpp
@@ -1,9 +1,8 @@
 //===-- llvm-rc.cpp - Compile .rc scripts into .res -------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -31,6 +30,7 @@
 #include "llvm/Support/Signals.h"
 #include "llvm/Support/raw_ostream.h"
 
+#include <algorithm>
 #include <system_error>
 
 using namespace llvm;
@@ -85,7 +85,10 @@ int main(int Argc, const char **Argv) {
 
   RcOptTable T;
   unsigned MAI, MAC;
-  ArrayRef<const char *> ArgsArr = makeArrayRef(Argv + 1, Argc - 1);
+  const char **DashDash = std::find_if(
+      Argv + 1, Argv + Argc, [](StringRef Str) { return Str == "--"; });
+  ArrayRef<const char *> ArgsArr = makeArrayRef(Argv + 1, DashDash);
+
   opt::InputArgList InputArgs = T.ParseArgs(ArgsArr, MAI, MAC);
 
   // The tool prints nothing when invoked with no command-line arguments.
@@ -97,6 +100,8 @@ int main(int Argc, const char **Argv) {
   const bool BeVerbose = InputArgs.hasArg(OPT_VERBOSE);
 
   std::vector<std::string> InArgsInfo = InputArgs.getAllArgValues(OPT_INPUT);
+  if (DashDash != Argv + Argc)
+    InArgsInfo.insert(InArgsInfo.end(), DashDash + 1, Argv + Argc);
   if (InArgsInfo.size() != 1) {
     fatalError("Exactly one input file should be provided.");
   }
diff --git a/tools/llvm-readobj/ARMEHABIPrinter.h b/tools/llvm-readobj/ARMEHABIPrinter.h
index 51128f113c4c212797d3336c88c92989714cb4dc..f847041a6ae6f42a8460eab46519ad7dbf936eec 100644
--- a/tools/llvm-readobj/ARMEHABIPrinter.h
+++ b/tools/llvm-readobj/ARMEHABIPrinter.h
@@ -1,9 +1,8 @@
 //===--- ARMEHABIPrinter.h - ARM EHABI Unwind Information Printer ----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/tools/llvm-readobj/ARMWinEHPrinter.cpp b/tools/llvm-readobj/ARMWinEHPrinter.cpp
index 4b823b816c35a1d67f35ab74905d1b10bc9ceab7..770062aa73b8521a96c854d99996f9195b66877a 100644
--- a/tools/llvm-readobj/ARMWinEHPrinter.cpp
+++ b/tools/llvm-readobj/ARMWinEHPrinter.cpp
@@ -1,9 +1,8 @@
 //===-- ARMWinEHPrinter.cpp - Windows on ARM EH Data Printer ----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/tools/llvm-readobj/ARMWinEHPrinter.h b/tools/llvm-readobj/ARMWinEHPrinter.h
index e271a1e6fe77206962b5509cd45484411061470d..cb3c482f95e2f98e4355b230cdbd0ca618131c37 100644
--- a/tools/llvm-readobj/ARMWinEHPrinter.h
+++ b/tools/llvm-readobj/ARMWinEHPrinter.h
@@ -1,9 +1,8 @@
 //===--- ARMWinEHPrinter.h - Windows on ARM Unwind Information Printer ----===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License.  See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/tools/llvm-readobj/CMakeLists.txt b/tools/llvm-readobj/CMakeLists.txt
index b0550f3401232ff6da60483dd98f74159fb00647..389e348bc4d2f9f75d77cdf469f32fd237f671a3 100644
--- a/tools/llvm-readobj/CMakeLists.txt
+++ b/tools/llvm-readobj/CMakeLists.txt
@@ -1,6 +1,7 @@
 set(LLVM_LINK_COMPONENTS
   DebugInfoCodeView
   DebugInfoDWARF
+  Demangle
   Object
   BinaryFormat
   Support
diff --git a/tools/llvm-readobj/COFFDumper.cpp b/tools/llvm-readobj/COFFDumper.cpp
index 36654919201b53555111c58d64d085ac36eb4847..85bbe8d7b84f26d5a464a8155a00343e56b2b846 100644
--- a/tools/llvm-readobj/COFFDumper.cpp
+++ b/tools/llvm-readobj/COFFDumper.cpp
@@ -1,9 +1,8 @@
 //===-- COFFDumper.cpp - COFF-specific dumper -------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
@@ -81,8 +80,6 @@ public:
   void printFileHeaders() override;
   void printSectionHeaders() override;
   void printRelocations() override;
-  void printSymbols() override;
-  void printDynamicSymbols() override;
   void printUnwindInfo() override;
 
   void printNeededLibraries() override;
@@ -95,12 +92,16 @@ public:
   void printCOFFResources() override;
   void printCOFFLoadConfig() override;
   void printCodeViewDebugInfo() override;
-  void
-  mergeCodeViewTypes(llvm::codeview::MergingTypeTableBuilder &CVIDs,
-                     llvm::codeview::MergingTypeTableBuilder &CVTypes) override;
+  void mergeCodeViewTypes(llvm::codeview::MergingTypeTableBuilder &CVIDs,
+                          llvm::codeview::MergingTypeTableBuilder &CVTypes,
+                          llvm::codeview::GlobalTypeTableBuilder &GlobalCVIDs,
+                          llvm::codeview::GlobalTypeTableBuilder &GlobalCVTypes,
+                          bool GHash) override;
   void printStackMap() const override;
   void printAddrsig() override;
 private:
+  void printSymbols() override;
+  void printDynamicSymbols() override;
   void printSymbol(const SymbolRef &Sym);
   void printRelocation(const SectionRef &Section, const RelocationRef &Reloc,
                        uint64_t Bias = 0);
@@ -1228,7 +1229,10 @@ void COFFDumper::printFileNameForOffset(StringRef Label, uint32_t FileOffset) {
 }
 
 void COFFDumper::mergeCodeViewTypes(MergingTypeTableBuilder &CVIDs,
-                                    MergingTypeTableBuilder &CVTypes) {
+                                    MergingTypeTableBuilder &CVTypes,
+                                    GlobalTypeTableBuilder &GlobalCVIDs,
+                                    GlobalTypeTableBuilder &GlobalCVTypes,
+                                    bool GHash) {
   for (const SectionRef &S : Obj->sections()) {
     StringRef SectionName;
     error(S.getName(SectionName));
@@ -1248,10 +1252,18 @@ void COFFDumper::mergeCodeViewTypes(MergingTypeTableBuilder &CVIDs,
         error(object_error::parse_failed);
       }
       SmallVector<TypeIndex, 128> SourceToDest;
-      Optional<EndPrecompRecord> EndPrecomp;
-      if (auto EC = mergeTypeAndIdRecords(CVIDs, CVTypes, SourceToDest, Types,
-                                          EndPrecomp))
-        return error(std::move(EC));
+      Optional<uint32_t> PCHSignature;
+      if (GHash) {
+        std::vector<GloballyHashedType> Hashes =
+            GloballyHashedType::hashTypes(Types);
+        if (auto EC = mergeTypeAndIdRecords(GlobalCVIDs, GlobalCVTypes, SourceToDest, Types,
+                                            Hashes, PCHSignature))
+          return error(std::move(EC));
+      } else {
+        if (auto EC = mergeTypeAndIdRecords(CVIDs, CVTypes, SourceToDest, Types,
+                                            PCHSignature))
+          return error(std::move(EC));
+      }
     }
   }
 }
@@ -1365,10 +1377,12 @@ void COFFDumper::printRelocation(const SectionRef &Section,
   StringRef SymbolName;
   Reloc.getTypeName(RelocName);
   symbol_iterator Symbol = Reloc.getSymbol();
+  int64_t SymbolIndex = -1;
   if (Symbol != Obj->symbol_end()) {
     Expected<StringRef> SymbolNameOrErr = Symbol->getName();
     error(errorToErrorCode(SymbolNameOrErr.takeError()));
     SymbolName = *SymbolNameOrErr;
+    SymbolIndex = Obj->getSymbolIndex(Obj->getCOFFSymbol(*Symbol));
   }
 
   if (opts::ExpandRelocs) {
@@ -1376,11 +1390,13 @@ void COFFDumper::printRelocation(const SectionRef &Section,
     W.printHex("Offset", Offset);
     W.printNumber("Type", RelocName, RelocType);
     W.printString("Symbol", SymbolName.empty() ? "-" : SymbolName);
+    W.printNumber("SymbolIndex", SymbolIndex);
   } else {
     raw_ostream& OS = W.startLine();
     OS << W.hex(Offset)
        << " " << RelocName
        << " " << (SymbolName.empty() ? "-" : SymbolName)
+       << " (" << SymbolIndex << ")"
        << "\n";
   }
 }
@@ -1901,16 +1917,10 @@ void COFFDumper::printAddrsig() {
   }
 }
 
-void llvm::dumpCodeViewMergedTypes(
-    ScopedPrinter &Writer, llvm::codeview::MergingTypeTableBuilder &IDTable,
-    llvm::codeview::MergingTypeTableBuilder &CVTypes) {
-  // Flatten it first, then run our dumper on it.
-  SmallString<0> TypeBuf;
-  CVTypes.ForEachRecord([&](TypeIndex TI, const CVType &Record) {
-    TypeBuf.append(Record.RecordData.begin(), Record.RecordData.end());
-  });
-
-  TypeTableCollection TpiTypes(CVTypes.records());
+void llvm::dumpCodeViewMergedTypes(ScopedPrinter &Writer,
+                                   ArrayRef<ArrayRef<uint8_t>> IpiRecords,
+                                   ArrayRef<ArrayRef<uint8_t>> TpiRecords) {
+  TypeTableCollection TpiTypes(TpiRecords);
   {
     ListScope S(Writer, "MergedTypeStream");
     TypeDumpVisitor TDV(TpiTypes, &Writer, opts::CodeViewSubsectionBytes);
@@ -1920,7 +1930,7 @@ void llvm::dumpCodeViewMergedTypes(
 
   // Flatten the id stream and print it next. The ID stream refers to names from
   // the type stream.
-  TypeTableCollection IpiTypes(IDTable.records());
+  TypeTableCollection IpiTypes(IpiRecords);
   {
     ListScope S(Writer, "MergedIDStream");
     TypeDumpVisitor TDV(TpiTypes, &Writer, opts::CodeViewSubsectionBytes);
diff --git a/tools/llvm-readobj/COFFImportDumper.cpp b/tools/llvm-readobj/COFFImportDumper.cpp
index 18010c34f0f39fa863c688ad943fdfd8ce64584c..1d9e7c8ff0e8ef74e9cb636984e23898d1fa14ab 100644
--- a/tools/llvm-readobj/COFFImportDumper.cpp
+++ b/tools/llvm-readobj/COFFImportDumper.cpp
@@ -1,9 +1,8 @@
 //===-- COFFImportDumper.cpp - COFF import library dumper -------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
diff --git a/tools/llvm-readobj/DwarfCFIEHPrinter.h b/tools/llvm-readobj/DwarfCFIEHPrinter.h
index d91d764c4d0a674b050192b2007e6c24023e7029..7055510ef2f2e1321c4caa339711e97ce614088d 100644
--- a/tools/llvm-readobj/DwarfCFIEHPrinter.h
+++ b/tools/llvm-readobj/DwarfCFIEHPrinter.h
@@ -1,9 +1,8 @@
 //===--- DwarfCFIEHPrinter.h - DWARF-based Unwind Information Printer -----===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/tools/llvm-readobj/ELFDumper.cpp b/tools/llvm-readobj/ELFDumper.cpp
index b1b0b6f3a35847e7be9fb1c131fe8d371a867a26..aaac9a56547e05c5d3677716352d2d8eca879e12 100644
--- a/tools/llvm-readobj/ELFDumper.cpp
+++ b/tools/llvm-readobj/ELFDumper.cpp
@@ -1,9 +1,8 @@
 //===- ELFDumper.cpp - ELF-specific dumper --------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
@@ -20,6 +19,7 @@
 #include "llvm-readobj.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/Optional.h"
 #include "llvm/ADT/PointerIntPair.h"
 #include "llvm/ADT/STLExtras.h"
@@ -30,6 +30,7 @@
 #include "llvm/ADT/Twine.h"
 #include "llvm/BinaryFormat/AMDGPUMetadataVerifier.h"
 #include "llvm/BinaryFormat/ELF.h"
+#include "llvm/Demangle/Demangle.h"
 #include "llvm/Object/ELF.h"
 #include "llvm/Object/ELFObjectFile.h"
 #include "llvm/Object/ELFTypes.h"
@@ -147,13 +148,14 @@ public:
   void printSectionHeaders() override;
   void printRelocations() override;
   void printDynamicRelocations() override;
-  void printSymbols() override;
-  void printDynamicSymbols() override;
+  void printSymbols(bool PrintSymbols, bool PrintDynamicSymbols) override;
+  void printHashSymbols() override;
   void printUnwindInfo() override;
 
   void printDynamicTable() override;
   void printNeededLibraries() override;
-  void printProgramHeaders() override;
+  void printProgramHeaders(bool PrintProgramHeaders,
+                           cl::boolOrDefault PrintSectionMapping) override;
   void printHashTable() override;
   void printGnuHashTable() override;
   void printLoadName() override;
@@ -271,7 +273,7 @@ public:
   void getSectionNameIndex(const Elf_Sym *Symbol, const Elf_Sym *FirstSym,
                            StringRef &SectionName,
                            unsigned &SectionIndex) const;
-  StringRef getStaticSymbolName(uint32_t Index) const;
+  std::string getStaticSymbolName(uint32_t Index) const;
 
   void printSymbolsHelper(bool IsDynamic) const;
   const Elf_Shdr *getDotSymtabSec() const { return DotSymtabSec; }
@@ -328,15 +330,18 @@ public:
   virtual void printGroupSections(const ELFFile<ELFT> *Obj) = 0;
   virtual void printRelocations(const ELFFile<ELFT> *Obj) = 0;
   virtual void printSectionHeaders(const ELFFile<ELFT> *Obj) = 0;
-  virtual void printSymbols(const ELFFile<ELFT> *Obj) = 0;
-  virtual void printDynamicSymbols(const ELFFile<ELFT> *Obj) = 0;
+  virtual void printSymbols(const ELFFile<ELFT> *Obj, bool PrintSymbols,
+                            bool PrintDynamicSymbols) = 0;
+  virtual void printHashSymbols(const ELFFile<ELFT> *Obj) {}
   virtual void printDynamicRelocations(const ELFFile<ELFT> *Obj) = 0;
-  virtual void printSymtabMessage(const ELFFile<ELFT> *obj, StringRef Name,
+  virtual void printSymtabMessage(const ELFFile<ELFT> *Obj, StringRef Name,
                                   size_t Offset) {}
   virtual void printSymbol(const ELFFile<ELFT> *Obj, const Elf_Sym *Symbol,
                            const Elf_Sym *FirstSym, StringRef StrTable,
                            bool IsDynamic) = 0;
-  virtual void printProgramHeaders(const ELFFile<ELFT> *Obj) = 0;
+  virtual void printProgramHeaders(const ELFFile<ELFT> *Obj,
+                                   bool PrintProgramHeaders,
+                                   cl::boolOrDefault PrintSectionMapping) = 0;
   virtual void printHashHistogram(const ELFFile<ELFT> *Obj) = 0;
   virtual void printCGProfile(const ELFFile<ELFT> *Obj) = 0;
   virtual void printAddrsig(const ELFFile<ELFT> *Obj) = 0;
@@ -363,12 +368,14 @@ public:
   void printGroupSections(const ELFFile<ELFT> *Obj) override;
   void printRelocations(const ELFO *Obj) override;
   void printSectionHeaders(const ELFO *Obj) override;
-  void printSymbols(const ELFO *Obj) override;
-  void printDynamicSymbols(const ELFO *Obj) override;
+  void printSymbols(const ELFO *Obj, bool PrintSymbols,
+                    bool PrintDynamicSymbols) override;
+  void printHashSymbols(const ELFO *Obj) override;
   void printDynamicRelocations(const ELFO *Obj) override;
   void printSymtabMessage(const ELFO *Obj, StringRef Name,
                           size_t Offset) override;
-  void printProgramHeaders(const ELFO *Obj) override;
+  void printProgramHeaders(const ELFO *Obj, bool PrintProgramHeaders,
+                           cl::boolOrDefault PrintSectionMapping) override;
   void printHashHistogram(const ELFFile<ELFT> *Obj) override;
   void printCGProfile(const ELFFile<ELFT> *Obj) override;
   void printAddrsig(const ELFFile<ELFT> *Obj) override;
@@ -379,11 +386,11 @@ public:
 
 private:
   struct Field {
-    StringRef Str;
+    std::string Str;
     unsigned Column;
 
     Field(StringRef S, unsigned Col) : Str(S), Column(Col) {}
-    Field(unsigned Col) : Str(""), Column(Col) {}
+    Field(unsigned Col) : Column(Col) {}
   };
 
   template <typename T, typename TEnum>
@@ -442,6 +449,8 @@ private:
   bool checkoffsets(const Elf_Phdr &Phdr, const Elf_Shdr &Sec);
   bool checkVMA(const Elf_Phdr &Phdr, const Elf_Shdr &Sec);
   bool checkPTDynamic(const Elf_Phdr &Phdr, const Elf_Shdr &Sec);
+  void printProgramHeaders(const ELFO *Obj);
+  void printSectionMapping(const ELFO *Obj);
 };
 
 template <typename ELFT> class LLVMStyle : public DumpStyle<ELFT> {
@@ -456,10 +465,11 @@ public:
   void printRelocations(const ELFO *Obj) override;
   void printRelocations(const Elf_Shdr *Sec, const ELFO *Obj);
   void printSectionHeaders(const ELFO *Obj) override;
-  void printSymbols(const ELFO *Obj) override;
-  void printDynamicSymbols(const ELFO *Obj) override;
+  void printSymbols(const ELFO *Obj, bool PrintSymbols,
+                    bool PrintDynamicSymbols) override;
   void printDynamicRelocations(const ELFO *Obj) override;
-  void printProgramHeaders(const ELFO *Obj) override;
+  void printProgramHeaders(const ELFO *Obj, bool PrintProgramHeaders,
+                           cl::boolOrDefault PrintSectionMapping) override;
   void printHashHistogram(const ELFFile<ELFT> *Obj) override;
   void printCGProfile(const ELFFile<ELFT> *Obj) override;
   void printAddrsig(const ELFFile<ELFT> *Obj) override;
@@ -471,8 +481,12 @@ public:
 private:
   void printRelocation(const ELFO *Obj, Elf_Rela Rel, const Elf_Shdr *SymTab);
   void printDynamicRelocation(const ELFO *Obj, Elf_Rela Rel);
+  void printSymbols(const ELFO *Obj);
+  void printDynamicSymbols(const ELFO *Obj);
   void printSymbol(const ELFO *Obj, const Elf_Sym *Symbol, const Elf_Sym *First,
                    StringRef StrTable, bool IsDynamic) override;
+  void printProgramHeaders(const ELFO *Obj);
+  void printSectionMapping(const ELFO *Obj) {}
 
   ScopedPrinter &W;
 };
@@ -578,7 +592,7 @@ template <class ELFT> void ELFDumper<ELFT>::LoadVersionMap() const {
     return;
 
   // Has the VersionMap already been loaded?
-  if (VersionMap.size() > 0)
+  if (!VersionMap.empty())
     return;
 
   // The first two version indexes are reserved.
@@ -697,19 +711,12 @@ static void printVersionDependencySection(ELFDumper<ELFT> *Dumper,
   if (!Sec)
     return;
 
-  unsigned VerNeedNum = 0;
-  for (const typename ELFO::Elf_Dyn &Dyn : Dumper->dynamic_table()) {
-    if (Dyn.d_tag == DT_VERNEEDNUM) {
-      VerNeedNum = Dyn.d_un.d_val;
-      break;
-    }
-  }
-
   const uint8_t *SecData = (const uint8_t *)Obj->base() + Sec->sh_offset;
   const typename ELFO::Elf_Shdr *StrTab =
       unwrapOrError(Obj->getSection(Sec->sh_link));
 
   const uint8_t *P = SecData;
+  unsigned VerNeedNum = Sec->sh_info;
   for (unsigned I = 0; I < VerNeedNum; ++I) {
     const VerNeed *Need = reinterpret_cast<const VerNeed *>(P);
     DictScope Entry(W, "Dependency");
@@ -795,32 +802,37 @@ StringRef ELFDumper<ELFT>::getSymbolVersion(StringRef StrTab,
   return StringRef(StrTab.data() + name_offset);
 }
 
+static std::string maybeDemangle(StringRef Name) {
+  return opts::Demangle ? demangle(Name) : Name.str();
+}
+
 template <typename ELFT>
-StringRef ELFDumper<ELFT>::getStaticSymbolName(uint32_t Index) const {
+std::string ELFDumper<ELFT>::getStaticSymbolName(uint32_t Index) const {
   const ELFFile<ELFT> *Obj = ObjF->getELFFile();
   StringRef StrTable = unwrapOrError(Obj->getStringTableForSymtab(*DotSymtabSec));
   Elf_Sym_Range Syms = unwrapOrError(Obj->symbols(DotSymtabSec));
   if (Index >= Syms.size())
     reportError("Invalid symbol index");
   const Elf_Sym *Sym = &Syms[Index];
-  return unwrapOrError(Sym->getName(StrTable));
+  return maybeDemangle(unwrapOrError(Sym->getName(StrTable)));
 }
 
 template <typename ELFT>
 std::string ELFDumper<ELFT>::getFullSymbolName(const Elf_Sym *Symbol,
                                                StringRef StrTable,
                                                bool IsDynamic) const {
-  StringRef SymbolName = unwrapOrError(Symbol->getName(StrTable));
+  std::string SymbolName =
+      maybeDemangle(unwrapOrError(Symbol->getName(StrTable)));
   if (!IsDynamic)
     return SymbolName;
 
-  std::string FullSymbolName(SymbolName);
-
   bool IsDefault;
   StringRef Version = getSymbolVersion(StrTable, &*Symbol, IsDefault);
-  FullSymbolName += (IsDefault ? "@@" : "@");
-  FullSymbolName += Version;
-  return FullSymbolName;
+  if (!Version.empty()) {
+    SymbolName += (IsDefault ? "@@" : "@");
+    SymbolName += Version;
+  }
+  return SymbolName;
 }
 
 template <typename ELFT>
@@ -1606,22 +1618,27 @@ void ELFDumper<ELFT>::printRelocations() {
   ELFDumperStyle->printRelocations(ObjF->getELFFile());
 }
 
-template <class ELFT> void ELFDumper<ELFT>::printProgramHeaders() {
-  ELFDumperStyle->printProgramHeaders(ObjF->getELFFile());
+template <class ELFT>
+void ELFDumper<ELFT>::printProgramHeaders(
+    bool PrintProgramHeaders, cl::boolOrDefault PrintSectionMapping) {
+  ELFDumperStyle->printProgramHeaders(ObjF->getELFFile(), PrintProgramHeaders,
+                                      PrintSectionMapping);
 }
 
 template <class ELFT> void ELFDumper<ELFT>::printDynamicRelocations() {
   ELFDumperStyle->printDynamicRelocations(ObjF->getELFFile());
 }
 
-template<class ELFT>
-void ELFDumper<ELFT>::printSymbols() {
-  ELFDumperStyle->printSymbols(ObjF->getELFFile());
+template <class ELFT>
+void ELFDumper<ELFT>::printSymbols(bool PrintSymbols,
+                                   bool PrintDynamicSymbols) {
+  ELFDumperStyle->printSymbols(ObjF->getELFFile(), PrintSymbols,
+                               PrintDynamicSymbols);
 }
 
 template<class ELFT>
-void ELFDumper<ELFT>::printDynamicSymbols() {
-  ELFDumperStyle->printDynamicSymbols(ObjF->getELFFile());
+void ELFDumper<ELFT>::printHashSymbols() {
+  ELFDumperStyle->printHashSymbols(ObjF->getELFFile());
 }
 
 template <class ELFT> void ELFDumper<ELFT>::printHashHistogram() {
@@ -2597,7 +2614,7 @@ struct GroupMember {
 
 struct GroupSection {
   StringRef Name;
-  StringRef Signature;
+  std::string Signature;
   uint64_t ShName;
   uint64_t Index;
   uint32_t Link;
@@ -2628,13 +2645,13 @@ std::vector<GroupSection> getGroups(const ELFFile<ELFT> *Obj) {
 
     StringRef Name = unwrapOrError(Obj->getSectionName(&Sec));
     StringRef Signature = StrTable.data() + Sym->st_name;
-    Ret.push_back({Name, 
-                   Signature, 
-                   Sec.sh_name, 
+    Ret.push_back({Name,
+                   maybeDemangle(Signature),
+                   Sec.sh_name,
                    I - 1,
                    Sec.sh_link,
                    Sec.sh_info,
-                   Data[0], 
+                   Data[0],
                    {}});
 
     std::vector<GroupMember> &GM = Ret.back().Members;
@@ -2689,27 +2706,34 @@ template <class ELFT> void GNUStyle<ELFT>::printGroupSections(const ELFO *Obj) {
 template <class ELFT>
 void GNUStyle<ELFT>::printRelocation(const ELFO *Obj, const Elf_Shdr *SymTab,
                                      const Elf_Rela &R, bool IsRela) {
-  std::string Offset, Info, Addend, Value;
-  SmallString<32> RelocName;
-  StringRef TargetName;
-  const Elf_Sym *Sym = nullptr;
-  unsigned Width = ELFT::Is64Bits ? 16 : 8;
-  unsigned Bias = ELFT::Is64Bits ? 8 : 0;
-
   // First two fields are bit width dependent. The rest of them are after are
   // fixed width.
+  unsigned Bias = ELFT::Is64Bits ? 8 : 0;
   Field Fields[5] = {0, 10 + Bias, 19 + 2 * Bias, 42 + 2 * Bias, 53 + 2 * Bias};
+  SmallString<32> RelocName;
   Obj->getRelocationTypeName(R.getType(Obj->isMips64EL()), RelocName);
-  Sym = unwrapOrError(Obj->getRelocationSymbol(&R, SymTab));
+  const Elf_Sym *Sym = unwrapOrError(Obj->getRelocationSymbol(&R, SymTab));
+  std::string TargetName;
   if (Sym && Sym->getType() == ELF::STT_SECTION) {
     const Elf_Shdr *Sec = unwrapOrError(
         Obj->getSection(Sym, SymTab, this->dumper()->getShndxTable()));
     TargetName = unwrapOrError(Obj->getSectionName(Sec));
   } else if (Sym) {
     StringRef StrTable = unwrapOrError(Obj->getStringTableForSymtab(*SymTab));
-    TargetName = unwrapOrError(Sym->getName(StrTable));
+    TargetName = maybeDemangle(unwrapOrError(Sym->getName(StrTable)));
   }
 
+  unsigned Width = ELFT::Is64Bits ? 16 : 8;
+  Fields[0].Str = to_string(format_hex_no_prefix(R.r_offset, Width));
+  Fields[1].Str = to_string(format_hex_no_prefix(R.r_info, Width));
+  Fields[2].Str = RelocName.str();
+  if (Sym)
+    Fields[3].Str = to_string(format_hex_no_prefix(Sym->getValue(), Width));
+  Fields[4].Str = TargetName;
+  for (auto &F : Fields)
+    printField(F);
+
+  std::string Addend;
   if (Sym && IsRela) {
     if (R.r_addend < 0)
       Addend = " - ";
@@ -2717,25 +2741,9 @@ void GNUStyle<ELFT>::printRelocation(const ELFO *Obj, const Elf_Shdr *SymTab,
       Addend = " + ";
   }
 
-  Offset = to_string(format_hex_no_prefix(R.r_offset, Width));
-  Info = to_string(format_hex_no_prefix(R.r_info, Width));
-
-  int64_t RelAddend = R.r_addend;
   if (IsRela)
-    Addend += to_hexString(std::abs(RelAddend), false);
-
-  if (Sym)
-    Value = to_string(format_hex_no_prefix(Sym->getValue(), Width));
-
-  Fields[0].Str = Offset;
-  Fields[1].Str = Info;
-  Fields[2].Str = RelocName;
-  Fields[3].Str = Value;
-  Fields[4].Str = TargetName;
-  for (auto &field : Fields)
-    printField(field);
-  OS << Addend;
-  OS << "\n";
+    Addend += to_hexString(std::abs(R.r_addend), false);
+  OS << Addend << "\n";
 }
 
 template <class ELFT> void GNUStyle<ELFT>::printRelocHeader(unsigned SType) {
@@ -2933,62 +2941,37 @@ std::string getSectionTypeString(unsigned Arch, unsigned Type) {
 
 template <class ELFT>
 void GNUStyle<ELFT>::printSectionHeaders(const ELFO *Obj) {
-  size_t SectionIndex = 0;
-  std::string Number, Type, Size, Address, Offset, Flags, Link, Info, EntrySize,
-      Alignment;
-  unsigned Bias;
-  unsigned Width;
-
-  if (ELFT::Is64Bits) {
-    Bias = 0;
-    Width = 16;
-  } else {
-    Bias = 8;
-    Width = 8;
-  }
-
+  unsigned Bias = ELFT::Is64Bits ? 0 : 8;
   ArrayRef<Elf_Shdr> Sections = unwrapOrError(Obj->sections());
   OS << "There are " << to_string(Sections.size())
      << " section headers, starting at offset "
      << "0x" << to_hexString(Obj->getHeader()->e_shoff, false) << ":\n\n";
   OS << "Section Headers:\n";
-  Field Fields[11] = {{"[Nr]", 2},
-                      {"Name", 7},
-                      {"Type", 25},
-                      {"Address", 41},
-                      {"Off", 58 - Bias},
-                      {"Size", 65 - Bias},
-                      {"ES", 72 - Bias},
-                      {"Flg", 75 - Bias},
-                      {"Lk", 79 - Bias},
-                      {"Inf", 82 - Bias},
-                      {"Al", 86 - Bias}};
-  for (auto &f : Fields)
-    printField(f);
+  Field Fields[11] = {
+      {"[Nr]", 2},        {"Name", 7},        {"Type", 25},
+      {"Address", 41},    {"Off", 58 - Bias}, {"Size", 65 - Bias},
+      {"ES", 72 - Bias},  {"Flg", 75 - Bias}, {"Lk", 79 - Bias},
+      {"Inf", 82 - Bias}, {"Al", 86 - Bias}};
+  for (auto &F : Fields)
+    printField(F);
   OS << "\n";
 
+  size_t SectionIndex = 0;
   for (const Elf_Shdr &Sec : Sections) {
-    Number = to_string(SectionIndex);
-    Fields[0].Str = Number;
+    Fields[0].Str = to_string(SectionIndex);
     Fields[1].Str = unwrapOrError(Obj->getSectionName(&Sec));
-    Type = getSectionTypeString(Obj->getHeader()->e_machine, Sec.sh_type);
-    Fields[2].Str = Type;
-    Address = to_string(format_hex_no_prefix(Sec.sh_addr, Width));
-    Fields[3].Str = Address;
-    Offset = to_string(format_hex_no_prefix(Sec.sh_offset, 6));
-    Fields[4].Str = Offset;
-    Size = to_string(format_hex_no_prefix(Sec.sh_size, 6));
-    Fields[5].Str = Size;
-    EntrySize = to_string(format_hex_no_prefix(Sec.sh_entsize, 2));
-    Fields[6].Str = EntrySize;
-    Flags = getGNUFlags(Sec.sh_flags);
-    Fields[7].Str = Flags;
-    Link = to_string(Sec.sh_link);
-    Fields[8].Str = Link;
-    Info = to_string(Sec.sh_info);
-    Fields[9].Str = Info;
-    Alignment = to_string(Sec.sh_addralign);
-    Fields[10].Str = Alignment;
+    Fields[2].Str =
+        getSectionTypeString(Obj->getHeader()->e_machine, Sec.sh_type);
+    Fields[3].Str =
+        to_string(format_hex_no_prefix(Sec.sh_addr, ELFT::Is64Bits ? 16 : 8));
+    Fields[4].Str = to_string(format_hex_no_prefix(Sec.sh_offset, 6));
+    Fields[5].Str = to_string(format_hex_no_prefix(Sec.sh_size, 6));
+    Fields[6].Str = to_string(format_hex_no_prefix(Sec.sh_entsize, 2));
+    Fields[7].Str = getGNUFlags(Sec.sh_flags);
+    Fields[8].Str = to_string(Sec.sh_link);
+    Fields[9].Str = to_string(Sec.sh_info);
+    Fields[10].Str = to_string(Sec.sh_addralign);
+
     OS.PadToColumn(Fields[0].Column);
     OS << "[" << right_justify(Fields[0].Str, 2) << "]";
     for (int i = 1; i < 7; i++)
@@ -3070,7 +3053,6 @@ void GNUStyle<ELFT>::printSymbol(const ELFO *Obj, const Elf_Sym *Symbol,
                                  bool IsDynamic) {
   static int Idx = 0;
   static bool Dynamic = true;
-  size_t Width;
 
   // If this function was called with a different value from IsDynamic
   // from last call, happens when we move from dynamic to static symbol
@@ -3079,111 +3061,87 @@ void GNUStyle<ELFT>::printSymbol(const ELFO *Obj, const Elf_Sym *Symbol,
     Idx = 0;
     Dynamic = false;
   }
-  std::string Num, Name, Value, Size, Binding, Type, Visibility, Section;
-  unsigned Bias = 0;
-  if (ELFT::Is64Bits) {
-    Bias = 8;
-    Width = 16;
-  } else {
-    Bias = 0;
-    Width = 8;
-  }
+
+  unsigned Bias = ELFT::Is64Bits ? 8 : 0;
   Field Fields[8] = {0,         8,         17 + Bias, 23 + Bias,
                      31 + Bias, 38 + Bias, 47 + Bias, 51 + Bias};
-  Num = to_string(format_decimal(Idx++, 6)) + ":";
-  Value = to_string(format_hex_no_prefix(Symbol->st_value, Width));
-  Size = to_string(format_decimal(Symbol->st_size, 5));
+  Fields[0].Str = to_string(format_decimal(Idx++, 6)) + ":";
+  Fields[1].Str = to_string(
+      format_hex_no_prefix(Symbol->st_value, ELFT::Is64Bits ? 16 : 8));
+  Fields[2].Str = to_string(format_decimal(Symbol->st_size, 5));
+
   unsigned char SymbolType = Symbol->getType();
   if (Obj->getHeader()->e_machine == ELF::EM_AMDGPU &&
       SymbolType >= ELF::STT_LOOS && SymbolType < ELF::STT_HIOS)
-    Type = printEnum(SymbolType, makeArrayRef(AMDGPUSymbolTypes));
+    Fields[3].Str = printEnum(SymbolType, makeArrayRef(AMDGPUSymbolTypes));
   else
-    Type = printEnum(SymbolType, makeArrayRef(ElfSymbolTypes));
-  unsigned Vis = Symbol->getVisibility();
-  Binding = printEnum(Symbol->getBinding(), makeArrayRef(ElfSymbolBindings));
-  Visibility = printEnum(Vis, makeArrayRef(ElfSymbolVisibilities));
-  Section = getSymbolSectionNdx(Obj, Symbol, FirstSym);
-  Name = this->dumper()->getFullSymbolName(Symbol, StrTable, IsDynamic);
-  Fields[0].Str = Num;
-  Fields[1].Str = Value;
-  Fields[2].Str = Size;
-  Fields[3].Str = Type;
-  Fields[4].Str = Binding;
-  Fields[5].Str = Visibility;
-  Fields[6].Str = Section;
-  Fields[7].Str = Name;
+    Fields[3].Str = printEnum(SymbolType, makeArrayRef(ElfSymbolTypes));
+
+  Fields[4].Str =
+      printEnum(Symbol->getBinding(), makeArrayRef(ElfSymbolBindings));
+  Fields[5].Str =
+      printEnum(Symbol->getVisibility(), makeArrayRef(ElfSymbolVisibilities));
+  Fields[6].Str = getSymbolSectionNdx(Obj, Symbol, FirstSym);
+  Fields[7].Str =
+      this->dumper()->getFullSymbolName(Symbol, StrTable, IsDynamic);
   for (auto &Entry : Fields)
     printField(Entry);
   OS << "\n";
 }
+
 template <class ELFT>
 void GNUStyle<ELFT>::printHashedSymbol(const ELFO *Obj, const Elf_Sym *FirstSym,
                                        uint32_t Sym, StringRef StrTable,
                                        uint32_t Bucket) {
-  std::string Num, Buc, Name, Value, Size, Binding, Type, Visibility, Section;
-  unsigned Width, Bias = 0;
-  if (ELFT::Is64Bits) {
-    Bias = 8;
-    Width = 16;
-  } else {
-    Bias = 0;
-    Width = 8;
-  }
+  unsigned Bias = ELFT::Is64Bits ? 8 : 0;
   Field Fields[9] = {0,         6,         11,        20 + Bias, 25 + Bias,
                      34 + Bias, 41 + Bias, 49 + Bias, 53 + Bias};
-  Num = to_string(format_decimal(Sym, 5));
-  Buc = to_string(format_decimal(Bucket, 3)) + ":";
+  Fields[0].Str = to_string(format_decimal(Sym, 5));
+  Fields[1].Str = to_string(format_decimal(Bucket, 3)) + ":";
 
   const auto Symbol = FirstSym + Sym;
-  Value = to_string(format_hex_no_prefix(Symbol->st_value, Width));
-  Size = to_string(format_decimal(Symbol->st_size, 5));
+  Fields[2].Str = to_string(
+      format_hex_no_prefix(Symbol->st_value, ELFT::Is64Bits ? 18 : 8));
+  Fields[3].Str = to_string(format_decimal(Symbol->st_size, 5));
+
   unsigned char SymbolType = Symbol->getType();
   if (Obj->getHeader()->e_machine == ELF::EM_AMDGPU &&
       SymbolType >= ELF::STT_LOOS && SymbolType < ELF::STT_HIOS)
-    Type = printEnum(SymbolType, makeArrayRef(AMDGPUSymbolTypes));
+    Fields[4].Str = printEnum(SymbolType, makeArrayRef(AMDGPUSymbolTypes));
   else
-    Type = printEnum(SymbolType, makeArrayRef(ElfSymbolTypes));
-  unsigned Vis = Symbol->getVisibility();
-  Binding = printEnum(Symbol->getBinding(), makeArrayRef(ElfSymbolBindings));
-  Visibility = printEnum(Vis, makeArrayRef(ElfSymbolVisibilities));
-  Section = getSymbolSectionNdx(Obj, Symbol, FirstSym);
-  Name = this->dumper()->getFullSymbolName(Symbol, StrTable, true);
-  Fields[0].Str = Num;
-  Fields[1].Str = Buc;
-  Fields[2].Str = Value;
-  Fields[3].Str = Size;
-  Fields[4].Str = Type;
-  Fields[5].Str = Binding;
-  Fields[6].Str = Visibility;
-  Fields[7].Str = Section;
-  Fields[8].Str = Name;
+    Fields[4].Str = printEnum(SymbolType, makeArrayRef(ElfSymbolTypes));
+
+  Fields[5].Str =
+      printEnum(Symbol->getBinding(), makeArrayRef(ElfSymbolBindings));
+  Fields[6].Str =
+      printEnum(Symbol->getVisibility(), makeArrayRef(ElfSymbolVisibilities));
+  Fields[7].Str = getSymbolSectionNdx(Obj, Symbol, FirstSym);
+  Fields[8].Str = this->dumper()->getFullSymbolName(Symbol, StrTable, true);
+
   for (auto &Entry : Fields)
     printField(Entry);
   OS << "\n";
 }
 
-template <class ELFT> void GNUStyle<ELFT>::printSymbols(const ELFO *Obj) {
-  if (opts::DynamicSymbols)
+template <class ELFT>
+void GNUStyle<ELFT>::printSymbols(const ELFO *Obj, bool PrintSymbols,
+                                  bool PrintDynamicSymbols) {
+  if (!PrintSymbols && !PrintDynamicSymbols)
     return;
+  // GNU readelf prints both the .dynsym and .symtab with --symbols.
   this->dumper()->printSymbolsHelper(true);
-  this->dumper()->printSymbolsHelper(false);
+  if (PrintSymbols)
+    this->dumper()->printSymbolsHelper(false);
 }
 
-template <class ELFT>
-void GNUStyle<ELFT>::printDynamicSymbols(const ELFO *Obj) {
+template <class ELFT> void GNUStyle<ELFT>::printHashSymbols(const ELFO *Obj) {
   if (this->dumper()->getDynamicStringTable().empty())
     return;
   auto StringTable = this->dumper()->getDynamicStringTable();
   auto DynSyms = this->dumper()->dynamic_symbols();
-  auto GnuHash = this->dumper()->getGnuHashTable();
-  auto SysVHash = this->dumper()->getHashTable();
-
-  // If no hash or .gnu.hash found, try using symbol table
-  if (GnuHash == nullptr && SysVHash == nullptr)
-    this->dumper()->printSymbolsHelper(true);
 
   // Try printing .hash
-  if (this->dumper()->getHashTable()) {
+  if (auto SysVHash = this->dumper()->getHashTable()) {
     OS << "\n Symbol table of .hash for image:\n";
     if (ELFT::Is64Bits)
       OS << "  Num Buc:    Value          Size   Type   Bind Vis      Ndx Name";
@@ -3191,14 +3149,12 @@ void GNUStyle<ELFT>::printDynamicSymbols(const ELFO *Obj) {
       OS << "  Num Buc:    Value  Size   Type   Bind Vis      Ndx Name";
     OS << "\n";
 
-    uint32_t NBuckets = SysVHash->nbucket;
-    uint32_t NChains = SysVHash->nchain;
     auto Buckets = SysVHash->buckets();
     auto Chains = SysVHash->chains();
-    for (uint32_t Buc = 0; Buc < NBuckets; Buc++) {
+    for (uint32_t Buc = 0; Buc < SysVHash->nbucket; Buc++) {
       if (Buckets[Buc] == ELF::STN_UNDEF)
         continue;
-      for (uint32_t Ch = Buckets[Buc]; Ch < NChains; Ch = Chains[Ch]) {
+      for (uint32_t Ch = Buckets[Buc]; Ch < SysVHash->nchain; Ch = Chains[Ch]) {
         if (Ch == ELF::STN_UNDEF)
           break;
         printHashedSymbol(Obj, &DynSyms[0], Ch, StringTable, Buc);
@@ -3207,16 +3163,15 @@ void GNUStyle<ELFT>::printDynamicSymbols(const ELFO *Obj) {
   }
 
   // Try printing .gnu.hash
-  if (GnuHash) {
+  if (auto GnuHash = this->dumper()->getGnuHashTable()) {
     OS << "\n Symbol table of .gnu.hash for image:\n";
     if (ELFT::Is64Bits)
       OS << "  Num Buc:    Value          Size   Type   Bind Vis      Ndx Name";
     else
       OS << "  Num Buc:    Value  Size   Type   Bind Vis      Ndx Name";
     OS << "\n";
-    uint32_t NBuckets = GnuHash->nbuckets;
     auto Buckets = GnuHash->buckets();
-    for (uint32_t Buc = 0; Buc < NBuckets; Buc++) {
+    for (uint32_t Buc = 0; Buc < GnuHash->nbuckets; Buc++) {
       if (Buckets[Buc] == ELF::STN_UNDEF)
         continue;
       uint32_t Index = Buckets[Buc];
@@ -3299,13 +3254,22 @@ bool GNUStyle<ELFT>::checkPTDynamic(const Elf_Phdr &Phdr, const Elf_Shdr &Sec) {
           (Sec.sh_addr > Phdr.p_vaddr && Sec.sh_addr < Phdr.p_memsz));
 }
 
+template <class ELFT>
+void GNUStyle<ELFT>::printProgramHeaders(
+    const ELFO *Obj, bool PrintProgramHeaders,
+    cl::boolOrDefault PrintSectionMapping) {
+  if (PrintProgramHeaders)
+    printProgramHeaders(Obj);
+
+  // Display the section mapping along with the program headers, unless
+  // -section-mapping is explicitly set to false.
+  if (PrintSectionMapping != cl::BOU_FALSE)
+    printSectionMapping(Obj);
+}
+
 template <class ELFT>
 void GNUStyle<ELFT>::printProgramHeaders(const ELFO *Obj) {
   unsigned Bias = ELFT::Is64Bits ? 8 : 0;
-  unsigned Width = ELFT::Is64Bits ? 18 : 10;
-  unsigned SizeWidth = ELFT::Is64Bits ? 8 : 7;
-  std::string Type, Offset, VMA, LMA, FileSz, MemSz, Flag, Align;
-
   const Elf_Ehdr *Header = Obj->getHeader();
   Field Fields[8] = {2,         17,        26,        37 + Bias,
                      48 + Bias, 56 + Bias, 64 + Bias, 68 + Bias};
@@ -3321,23 +3285,18 @@ void GNUStyle<ELFT>::printProgramHeaders(const ELFO *Obj) {
   else
     OS << "  Type           Offset   VirtAddr   PhysAddr   FileSiz "
        << "MemSiz  Flg Align\n";
+
+  unsigned Width = ELFT::Is64Bits ? 18 : 10;
+  unsigned SizeWidth = ELFT::Is64Bits ? 8 : 7;
   for (const auto &Phdr : unwrapOrError(Obj->program_headers())) {
-    Type = getElfPtType(Header->e_machine, Phdr.p_type);
-    Offset = to_string(format_hex(Phdr.p_offset, 8));
-    VMA = to_string(format_hex(Phdr.p_vaddr, Width));
-    LMA = to_string(format_hex(Phdr.p_paddr, Width));
-    FileSz = to_string(format_hex(Phdr.p_filesz, SizeWidth));
-    MemSz = to_string(format_hex(Phdr.p_memsz, SizeWidth));
-    Flag = printPhdrFlags(Phdr.p_flags);
-    Align = to_string(format_hex(Phdr.p_align, 1));
-    Fields[0].Str = Type;
-    Fields[1].Str = Offset;
-    Fields[2].Str = VMA;
-    Fields[3].Str = LMA;
-    Fields[4].Str = FileSz;
-    Fields[5].Str = MemSz;
-    Fields[6].Str = Flag;
-    Fields[7].Str = Align;
+    Fields[0].Str = getElfPtType(Header->e_machine, Phdr.p_type);
+    Fields[1].Str = to_string(format_hex(Phdr.p_offset, 8));
+    Fields[2].Str = to_string(format_hex(Phdr.p_vaddr, Width));
+    Fields[3].Str = to_string(format_hex(Phdr.p_paddr, Width));
+    Fields[4].Str = to_string(format_hex(Phdr.p_filesz, SizeWidth));
+    Fields[5].Str = to_string(format_hex(Phdr.p_memsz, SizeWidth));
+    Fields[6].Str = printPhdrFlags(Phdr.p_flags);
+    Fields[7].Str = to_string(format_hex(Phdr.p_align, 1));
     for (auto Field : Fields)
       printField(Field);
     if (Phdr.p_type == ELF::PT_INTERP) {
@@ -3346,7 +3305,12 @@ void GNUStyle<ELFT>::printProgramHeaders(const ELFO *Obj) {
     }
     OS << "\n";
   }
+}
+
+template <class ELFT>
+void GNUStyle<ELFT>::printSectionMapping(const ELFO *Obj) {
   OS << "\n Section to Segment mapping:\n  Segment Sections...\n";
+  DenseSet<const Elf_Shdr *> BelongsToSegment;
   int Phnum = 0;
   for (const Elf_Phdr &Phdr : unwrapOrError(Obj->program_headers())) {
     std::string Sections;
@@ -3361,35 +3325,57 @@ void GNUStyle<ELFT>::printProgramHeaders(const ELFO *Obj) {
                           Phdr.p_type != ELF::PT_TLS;
       if (!TbssInNonTLS && checkTLSSections(Phdr, Sec) &&
           checkoffsets(Phdr, Sec) && checkVMA(Phdr, Sec) &&
-          checkPTDynamic(Phdr, Sec) && (Sec.sh_type != ELF::SHT_NULL))
+          checkPTDynamic(Phdr, Sec) && (Sec.sh_type != ELF::SHT_NULL)) {
         Sections += unwrapOrError(Obj->getSectionName(&Sec)).str() + " ";
+        BelongsToSegment.insert(&Sec);
+      }
     }
     OS << Sections << "\n";
     OS.flush();
   }
+
+  // Display sections that do not belong to a segment.
+  std::string Sections;
+  for (const Elf_Shdr &Sec : unwrapOrError(Obj->sections())) {
+    if (BelongsToSegment.find(&Sec) == BelongsToSegment.end())
+      Sections += unwrapOrError(Obj->getSectionName(&Sec)).str() + ' ';
+  }
+  if (!Sections.empty()) {
+    OS << "   None  " << Sections << '\n';
+    OS.flush();
+  }
 }
 
 template <class ELFT>
 void GNUStyle<ELFT>::printDynamicRelocation(const ELFO *Obj, Elf_Rela R,
                                             bool IsRela) {
-  SmallString<32> RelocName;
-  StringRef SymbolName;
-  unsigned Width = ELFT::Is64Bits ? 16 : 8;
   unsigned Bias = ELFT::Is64Bits ? 8 : 0;
   // First two fields are bit width dependent. The rest of them are after are
   // fixed width.
   Field Fields[5] = {0, 10 + Bias, 19 + 2 * Bias, 42 + 2 * Bias, 53 + 2 * Bias};
 
+  unsigned Width = ELFT::Is64Bits ? 16 : 8;
+  Fields[0].Str = to_string(format_hex_no_prefix(R.r_offset, Width));
+  Fields[1].Str = to_string(format_hex_no_prefix(R.r_info, Width));
+
   uint32_t SymIndex = R.getSymbol(Obj->isMips64EL());
   const Elf_Sym *Sym = this->dumper()->dynamic_symbols().begin() + SymIndex;
+  SmallString<32> RelocName;
   Obj->getRelocationTypeName(R.getType(Obj->isMips64EL()), RelocName);
-  SymbolName =
-      unwrapOrError(Sym->getName(this->dumper()->getDynamicStringTable()));
-  std::string Addend, Info, Offset, Value;
-  Offset = to_string(format_hex_no_prefix(R.r_offset, Width));
-  Info = to_string(format_hex_no_prefix(R.r_info, Width));
-  Value = to_string(format_hex_no_prefix(Sym->getValue(), Width));
+  Fields[2].Str = RelocName.c_str();
+
+  std::string SymbolName = maybeDemangle(
+      unwrapOrError(Sym->getName(this->dumper()->getDynamicStringTable())));
+
+  if (!SymbolName.empty() || Sym->getValue() != 0)
+    Fields[3].Str = to_string(format_hex_no_prefix(Sym->getValue(), Width));
+
+  Fields[4].Str = SymbolName;
+  for (auto &Field : Fields)
+    printField(Field);
+
   int64_t RelAddend = R.r_addend;
+  std::string Addend;
   if (!SymbolName.empty() && IsRela) {
     if (R.r_addend < 0)
       Addend = " - ";
@@ -3397,22 +3383,9 @@ void GNUStyle<ELFT>::printDynamicRelocation(const ELFO *Obj, Elf_Rela R,
       Addend = " + ";
   }
 
-  if (SymbolName.empty() && Sym->getValue() == 0)
-    Value = "";
-
   if (IsRela)
     Addend += to_string(format_hex_no_prefix(std::abs(RelAddend), 1));
-
-
-  Fields[0].Str = Offset;
-  Fields[1].Str = Info;
-  Fields[2].Str = RelocName.c_str();
-  Fields[3].Str = Value;
-  Fields[4].Str = SymbolName;
-  for (auto &Field : Fields)
-    printField(Field);
-  OS << Addend;
-  OS << "\n";
+  OS << Addend << "\n";
 }
 
 template <class ELFT>
@@ -3484,12 +3457,8 @@ void GNUStyle<ELFT>::printDynamicRelocations(const ELFO *Obj) {
 // Additionally cumulative coverage of symbols for each set of buckets.
 template <class ELFT>
 void GNUStyle<ELFT>::printHashHistogram(const ELFFile<ELFT> *Obj) {
-
-  const Elf_Hash *HashTable = this->dumper()->getHashTable();
-  const Elf_GnuHash *GnuHashTable = this->dumper()->getGnuHashTable();
-
   // Print histogram for .hash section
-  if (HashTable) {
+  if (const Elf_Hash *HashTable = this->dumper()->getHashTable()) {
     size_t NBucket = HashTable->nbucket;
     size_t NChain = HashTable->nchain;
     ArrayRef<Elf_Word> Buckets = HashTable->buckets();
@@ -3533,7 +3502,7 @@ void GNUStyle<ELFT>::printHashHistogram(const ELFFile<ELFT> *Obj) {
   }
 
   // Print histogram for .gnu.hash section
-  if (GnuHashTable) {
+  if (const Elf_GnuHash *GnuHashTable = this->dumper()->getGnuHashTable()) {
     size_t NBucket = GnuHashTable->nbuckets;
     ArrayRef<Elf_Word> Buckets = GnuHashTable->buckets();
     unsigned NumSyms = this->dumper()->dynamic_symbols().size();
@@ -3681,6 +3650,16 @@ static std::string getGNUProperty(uint32_t Type, uint32_t DataSize,
                                   ArrayRef<uint8_t> Data) {
   std::string str;
   raw_string_ostream OS(str);
+  uint32_t PrData;
+  auto DumpBit = [&](uint32_t Flag, StringRef Name) {
+    if (PrData & Flag) {
+      PrData &= ~Flag;
+      OS << Name;
+      if (PrData)
+        OS << ", ";
+    }
+  };
+
   switch (Type) {
   default:
     OS << format("<application-specific type 0x%x>", Type);
@@ -3700,33 +3679,46 @@ static std::string getGNUProperty(uint32_t Type, uint32_t DataSize,
       OS << format(" <corrupt length: 0x%x>", DataSize);
     return OS.str();
   case GNU_PROPERTY_X86_FEATURE_1_AND:
-    OS << "X86 features: ";
-    if (DataSize != 4 && DataSize != 8) {
+    OS << "x86 feature: ";
+    if (DataSize != 4) {
       OS << format("<corrupt length: 0x%x>", DataSize);
       return OS.str();
     }
-    uint64_t CFProtection =
-        (DataSize == 4)
-            ? support::endian::read32<ELFT::TargetEndianness>(Data.data())
-            : support::endian::read64<ELFT::TargetEndianness>(Data.data());
-    if (CFProtection == 0) {
-      OS << "none";
+    PrData = support::endian::read32<ELFT::TargetEndianness>(Data.data());
+    if (PrData == 0) {
+      OS << "<None>";
       return OS.str();
     }
-    if (CFProtection & GNU_PROPERTY_X86_FEATURE_1_IBT) {
-      OS << "IBT";
-      CFProtection &= ~GNU_PROPERTY_X86_FEATURE_1_IBT;
-      if (CFProtection)
-        OS << ", ";
+    DumpBit(GNU_PROPERTY_X86_FEATURE_1_IBT, "IBT");
+    DumpBit(GNU_PROPERTY_X86_FEATURE_1_SHSTK, "SHSTK");
+    if (PrData)
+      OS << format("<unknown flags: 0x%x>", PrData);
+    return OS.str();
+  case GNU_PROPERTY_X86_FEATURE_2_NEEDED:
+  case GNU_PROPERTY_X86_FEATURE_2_USED:
+    OS << "x86 feature "
+       << (Type == GNU_PROPERTY_X86_FEATURE_2_NEEDED ? "needed: " : "used: ");
+    if (DataSize != 4) {
+      OS << format("<corrupt length: 0x%x>", DataSize);
+      return OS.str();
     }
-    if (CFProtection & GNU_PROPERTY_X86_FEATURE_1_SHSTK) {
-      OS << "SHSTK";
-      CFProtection &= ~GNU_PROPERTY_X86_FEATURE_1_SHSTK;
-      if (CFProtection)
-        OS << ", ";
+    PrData = support::endian::read32<ELFT::TargetEndianness>(Data.data());
+    if (PrData == 0) {
+      OS << "<None>";
+      return OS.str();
     }
-    if (CFProtection)
-      OS << format("<unknown flags: 0x%llx>", CFProtection);
+    DumpBit(GNU_PROPERTY_X86_FEATURE_2_X86, "x86");
+    DumpBit(GNU_PROPERTY_X86_FEATURE_2_X87, "x87");
+    DumpBit(GNU_PROPERTY_X86_FEATURE_2_MMX, "MMX");
+    DumpBit(GNU_PROPERTY_X86_FEATURE_2_XMM, "XMM");
+    DumpBit(GNU_PROPERTY_X86_FEATURE_2_YMM, "YMM");
+    DumpBit(GNU_PROPERTY_X86_FEATURE_2_ZMM, "ZMM");
+    DumpBit(GNU_PROPERTY_X86_FEATURE_2_FXSR, "FXSR");
+    DumpBit(GNU_PROPERTY_X86_FEATURE_2_XSAVE, "XSAVE");
+    DumpBit(GNU_PROPERTY_X86_FEATURE_2_XSAVEOPT, "XSAVEOPT");
+    DumpBit(GNU_PROPERTY_X86_FEATURE_2_XSAVEC, "XSAVEC");
+    if (PrData)
+      OS << format("<unknown flags: 0x%x>", PrData);
     return OS.str();
   }
 }
@@ -3902,9 +3894,6 @@ static AMDGPUNote getAMDGPUNote(uint32_t NoteType, ArrayRef<uint8_t> Desc) {
 
 template <class ELFT>
 void GNUStyle<ELFT>::printNotes(const ELFFile<ELFT> *Obj) {
-  const Elf_Ehdr *e = Obj->getHeader();
-  bool IsCore = e->e_type == ELF::ET_CORE;
-
   auto PrintHeader = [&](const typename ELFT::Off Offset,
                          const typename ELFT::Addr Size) {
     OS << "Displaying notes found at file offset " << format_hex(Offset, 10)
@@ -3941,7 +3930,7 @@ void GNUStyle<ELFT>::printNotes(const ELFFile<ELFT> *Obj) {
     OS << '\n';
   };
 
-  if (IsCore) {
+  if (Obj->getHeader()->e_type == ELF::ET_CORE) {
     for (const auto &P : unwrapOrError(Obj->program_headers())) {
       if (P.p_type != PT_NOTE)
         continue;
@@ -4085,21 +4074,21 @@ void GNUStyle<ELFT>::printMipsPLT(const MipsGOTParser<ELFT> &Parser) {
 }
 
 template <class ELFT> void LLVMStyle<ELFT>::printFileHeaders(const ELFO *Obj) {
-  const Elf_Ehdr *e = Obj->getHeader();
+  const Elf_Ehdr *E = Obj->getHeader();
   {
     DictScope D(W, "ElfHeader");
     {
       DictScope D(W, "Ident");
-      W.printBinary("Magic", makeArrayRef(e->e_ident).slice(ELF::EI_MAG0, 4));
-      W.printEnum("Class", e->e_ident[ELF::EI_CLASS], makeArrayRef(ElfClass));
-      W.printEnum("DataEncoding", e->e_ident[ELF::EI_DATA],
+      W.printBinary("Magic", makeArrayRef(E->e_ident).slice(ELF::EI_MAG0, 4));
+      W.printEnum("Class", E->e_ident[ELF::EI_CLASS], makeArrayRef(ElfClass));
+      W.printEnum("DataEncoding", E->e_ident[ELF::EI_DATA],
                   makeArrayRef(ElfDataEncoding));
-      W.printNumber("FileVersion", e->e_ident[ELF::EI_VERSION]);
+      W.printNumber("FileVersion", E->e_ident[ELF::EI_VERSION]);
 
       auto OSABI = makeArrayRef(ElfOSABI);
-      if (e->e_ident[ELF::EI_OSABI] >= ELF::ELFOSABI_FIRST_ARCH &&
-          e->e_ident[ELF::EI_OSABI] <= ELF::ELFOSABI_LAST_ARCH) {
-        switch (e->e_machine) {
+      if (E->e_ident[ELF::EI_OSABI] >= ELF::ELFOSABI_FIRST_ARCH &&
+          E->e_ident[ELF::EI_OSABI] <= ELF::ELFOSABI_LAST_ARCH) {
+        switch (E->e_machine) {
         case ELF::EM_AMDGPU:
           OSABI = makeArrayRef(AMDGPUElfOSABI);
           break;
@@ -4111,32 +4100,32 @@ template <class ELFT> void LLVMStyle<ELFT>::printFileHeaders(const ELFO *Obj) {
           break;
         }
       }
-      W.printEnum("OS/ABI", e->e_ident[ELF::EI_OSABI], OSABI);
-      W.printNumber("ABIVersion", e->e_ident[ELF::EI_ABIVERSION]);
-      W.printBinary("Unused", makeArrayRef(e->e_ident).slice(ELF::EI_PAD));
+      W.printEnum("OS/ABI", E->e_ident[ELF::EI_OSABI], OSABI);
+      W.printNumber("ABIVersion", E->e_ident[ELF::EI_ABIVERSION]);
+      W.printBinary("Unused", makeArrayRef(E->e_ident).slice(ELF::EI_PAD));
     }
 
-    W.printEnum("Type", e->e_type, makeArrayRef(ElfObjectFileType));
-    W.printEnum("Machine", e->e_machine, makeArrayRef(ElfMachineType));
-    W.printNumber("Version", e->e_version);
-    W.printHex("Entry", e->e_entry);
-    W.printHex("ProgramHeaderOffset", e->e_phoff);
-    W.printHex("SectionHeaderOffset", e->e_shoff);
-    if (e->e_machine == EM_MIPS)
-      W.printFlags("Flags", e->e_flags, makeArrayRef(ElfHeaderMipsFlags),
+    W.printEnum("Type", E->e_type, makeArrayRef(ElfObjectFileType));
+    W.printEnum("Machine", E->e_machine, makeArrayRef(ElfMachineType));
+    W.printNumber("Version", E->e_version);
+    W.printHex("Entry", E->e_entry);
+    W.printHex("ProgramHeaderOffset", E->e_phoff);
+    W.printHex("SectionHeaderOffset", E->e_shoff);
+    if (E->e_machine == EM_MIPS)
+      W.printFlags("Flags", E->e_flags, makeArrayRef(ElfHeaderMipsFlags),
                    unsigned(ELF::EF_MIPS_ARCH), unsigned(ELF::EF_MIPS_ABI),
                    unsigned(ELF::EF_MIPS_MACH));
-    else if (e->e_machine == EM_AMDGPU)
-      W.printFlags("Flags", e->e_flags, makeArrayRef(ElfHeaderAMDGPUFlags),
+    else if (E->e_machine == EM_AMDGPU)
+      W.printFlags("Flags", E->e_flags, makeArrayRef(ElfHeaderAMDGPUFlags),
                    unsigned(ELF::EF_AMDGPU_MACH));
-    else if (e->e_machine == EM_RISCV)
-      W.printFlags("Flags", e->e_flags, makeArrayRef(ElfHeaderRISCVFlags));
+    else if (E->e_machine == EM_RISCV)
+      W.printFlags("Flags", E->e_flags, makeArrayRef(ElfHeaderRISCVFlags));
     else
-      W.printFlags("Flags", e->e_flags);
-    W.printNumber("HeaderSize", e->e_ehsize);
-    W.printNumber("ProgramHeaderEntrySize", e->e_phentsize);
-    W.printNumber("ProgramHeaderCount", e->e_phnum);
-    W.printNumber("SectionHeaderEntrySize", e->e_shentsize);
+      W.printFlags("Flags", E->e_flags);
+    W.printNumber("HeaderSize", E->e_ehsize);
+    W.printNumber("ProgramHeaderEntrySize", E->e_phentsize);
+    W.printNumber("ProgramHeaderCount", E->e_phnum);
+    W.printNumber("SectionHeaderEntrySize", E->e_shentsize);
     W.printString("SectionHeaderCount", getSectionHeadersNumString(Obj));
     W.printString("StringTableSectionIndex", getSectionHeaderTableIndexString(Obj));
   }
@@ -4247,7 +4236,7 @@ void LLVMStyle<ELFT>::printRelocation(const ELFO *Obj, Elf_Rela Rel,
                                       const Elf_Shdr *SymTab) {
   SmallString<32> RelocName;
   Obj->getRelocationTypeName(Rel.getType(Obj->isMips64EL()), RelocName);
-  StringRef TargetName;
+  std::string TargetName;
   const Elf_Sym *Sym = unwrapOrError(Obj->getRelocationSymbol(&Rel, SymTab));
   if (Sym && Sym->getType() == ELF::STT_SECTION) {
     const Elf_Shdr *Sec = unwrapOrError(
@@ -4255,7 +4244,7 @@ void LLVMStyle<ELFT>::printRelocation(const ELFO *Obj, Elf_Rela Rel,
     TargetName = unwrapOrError(Obj->getSectionName(Sec));
   } else if (Sym) {
     StringRef StrTable = unwrapOrError(Obj->getStringTableForSymtab(*SymTab));
-    TargetName = unwrapOrError(Sym->getName(StrTable));
+    TargetName = maybeDemangle(unwrapOrError(Sym->getName(StrTable)));
   }
 
   if (opts::ExpandRelocs) {
@@ -4400,6 +4389,15 @@ void LLVMStyle<ELFT>::printSymbol(const ELFO *Obj, const Elf_Sym *Symbol,
   W.printHex("Section", SectionName, SectionIndex);
 }
 
+template <class ELFT>
+void LLVMStyle<ELFT>::printSymbols(const ELFO *Obj, bool PrintSymbols,
+                                   bool PrintDynamicSymbols) {
+  if (PrintSymbols)
+    printSymbols(Obj);
+  if (PrintDynamicSymbols)
+    printDynamicSymbols(Obj);
+}
+
 template <class ELFT> void LLVMStyle<ELFT>::printSymbols(const ELFO *Obj) {
   ListScope Group(W, "Symbols");
   this->dumper()->printSymbolsHelper(false);
@@ -4457,11 +4455,11 @@ template <class ELFT>
 void LLVMStyle<ELFT>::printDynamicRelocation(const ELFO *Obj, Elf_Rela Rel) {
   SmallString<32> RelocName;
   Obj->getRelocationTypeName(Rel.getType(Obj->isMips64EL()), RelocName);
-  StringRef SymbolName;
+  std::string SymbolName;
   uint32_t SymIndex = Rel.getSymbol(Obj->isMips64EL());
   const Elf_Sym *Sym = this->dumper()->dynamic_symbols().begin() + SymIndex;
-  SymbolName =
-      unwrapOrError(Sym->getName(this->dumper()->getDynamicStringTable()));
+  SymbolName = maybeDemangle(
+      unwrapOrError(Sym->getName(this->dumper()->getDynamicStringTable())));
   if (opts::ExpandRelocs) {
     DictScope Group(W, "Relocation");
     W.printHex("Offset", Rel.r_offset);
@@ -4476,6 +4474,16 @@ void LLVMStyle<ELFT>::printDynamicRelocation(const ELFO *Obj, Elf_Rela Rel) {
   }
 }
 
+template <class ELFT>
+void LLVMStyle<ELFT>::printProgramHeaders(
+    const ELFO *Obj, bool PrintProgramHeaders,
+    cl::boolOrDefault PrintSectionMapping) {
+  if (PrintProgramHeaders)
+    printProgramHeaders(Obj);
+  if (PrintSectionMapping == cl::BOU_TRUE)
+    printSectionMapping(Obj);
+}
+
 template <class ELFT>
 void LLVMStyle<ELFT>::printProgramHeaders(const ELFO *Obj) {
   ListScope L(W, "ProgramHeaders");
@@ -4574,8 +4582,6 @@ static void printGNUNoteLLVMStyle(uint32_t NoteType,
 template <class ELFT>
 void LLVMStyle<ELFT>::printNotes(const ELFFile<ELFT> *Obj) {
   ListScope L(W, "Notes");
-  const Elf_Ehdr *e = Obj->getHeader();
-  bool IsCore = e->e_type == ELF::ET_CORE;
 
   auto PrintHeader = [&](const typename ELFT::Off Offset,
                          const typename ELFT::Addr Size) {
@@ -4611,7 +4617,7 @@ void LLVMStyle<ELFT>::printNotes(const ELFFile<ELFT> *Obj) {
     }
   };
 
-  if (IsCore) {
+  if (Obj->getHeader()->e_type == ELF::ET_CORE) {
     for (const auto &P : unwrapOrError(Obj->program_headers())) {
       if (P.p_type != PT_NOTE)
         continue;
diff --git a/tools/llvm-readobj/Error.cpp b/tools/llvm-readobj/Error.cpp
index 03d349440e6b62a1750aa5f0a8404d9d27cfd4d0..1010f18a58c8421347b3161a36952f4e0d81f2a4 100644
--- a/tools/llvm-readobj/Error.cpp
+++ b/tools/llvm-readobj/Error.cpp
@@ -1,9 +1,8 @@
 //===- Error.cpp - system_error extensions for llvm-readobj -----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/tools/llvm-readobj/Error.h b/tools/llvm-readobj/Error.h
index f3e24bbe5dbfd79890fc66806b532e3a7f4cb735..f390e1b96f8a213e313d975d56251823c1fb69ad 100644
--- a/tools/llvm-readobj/Error.h
+++ b/tools/llvm-readobj/Error.h
@@ -1,9 +1,8 @@
 //===- Error.h - system_error extensions for llvm-readobj -------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/tools/llvm-readobj/LLVMBuild.txt b/tools/llvm-readobj/LLVMBuild.txt
index 87ede1008cb349050eedf7858a0001e16a6182e7..a6963658e51b69980c15b356bf7ebdd5ce264536 100644
--- a/tools/llvm-readobj/LLVMBuild.txt
+++ b/tools/llvm-readobj/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./tools/llvm-readobj/LLVMBuild.txt ---------------------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/tools/llvm-readobj/MachODumper.cpp b/tools/llvm-readobj/MachODumper.cpp
index 35e4cfcb6b10db95099d9be2d2866f9ee8bec00b..259f30de0bebaa3c4c176bb98213502a168fb697 100644
--- a/tools/llvm-readobj/MachODumper.cpp
+++ b/tools/llvm-readobj/MachODumper.cpp
@@ -1,9 +1,8 @@
 //===-- MachODump.cpp - Object file dumping utility for llvm --------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -34,8 +33,6 @@ public:
   void printFileHeaders() override;
   void printSectionHeaders() override;
   void printRelocations() override;
-  void printSymbols() override;
-  void printDynamicSymbols() override;
   void printUnwindInfo() override;
   void printStackMap() const override;
 
@@ -53,6 +50,8 @@ private:
   template<class MachHeader>
   void printFileHeaders(const MachHeader &Header);
 
+  void printSymbols() override;
+  void printDynamicSymbols() override;
   void printSymbol(const SymbolRef &Symbol);
 
   void printRelocation(const RelocationRef &Reloc);
diff --git a/tools/llvm-readobj/ObjDumper.cpp b/tools/llvm-readobj/ObjDumper.cpp
index a725140c9d33f11e2249bbc4e2204f567d8e54f8..e6d84a124fce8059d76fdc540bc31d531dbfcef9 100644
--- a/tools/llvm-readobj/ObjDumper.cpp
+++ b/tools/llvm-readobj/ObjDumper.cpp
@@ -1,9 +1,8 @@
 //===-- ObjDumper.cpp - Base dumper class -----------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
@@ -117,7 +116,8 @@ void ObjDumper::printSectionAsHex(const object::ObjectFile *Obj,
     uint8_t i;
     uint8_t k;
 
-    W.startLine() << format_hex(SecPtr - SecContent, 10);
+    W.startLine() << format_hex(Section.getAddress() + (SecPtr - SecContent),
+                                10);
     W.startLine() << ' ';
     for (i = 0; TmpSecPtr < SecEnd && i < 4; ++i) {
       for (k = 0; TmpSecPtr < SecEnd && k < 4; k++, TmpSecPtr++) {
diff --git a/tools/llvm-readobj/ObjDumper.h b/tools/llvm-readobj/ObjDumper.h
index 13de563469abd51a3fe03998aff0949f5a9e8400..5767485a19290714f05a338324e2bc92334dce0f 100644
--- a/tools/llvm-readobj/ObjDumper.h
+++ b/tools/llvm-readobj/ObjDumper.h
@@ -1,9 +1,8 @@
 //===-- ObjDumper.h ---------------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -15,6 +14,7 @@
 
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Object/ObjectFile.h"
+#include "llvm/Support/CommandLine.h"
 
 namespace llvm {
 namespace object {
@@ -22,8 +22,9 @@ class COFFImportFile;
 class ObjectFile;
 }
 namespace codeview {
+class GlobalTypeTableBuilder;
 class MergingTypeTableBuilder;
-}
+} // namespace codeview
 
 class ScopedPrinter;
 
@@ -35,18 +36,30 @@ public:
   virtual void printFileHeaders() = 0;
   virtual void printSectionHeaders() = 0;
   virtual void printRelocations() = 0;
-  virtual void printSymbols() = 0;
-  virtual void printDynamicSymbols() = 0;
+  virtual void printSymbols(bool PrintSymbols, bool PrintDynamicSymbols) {
+    if (PrintSymbols)
+      printSymbols();
+    if (PrintDynamicSymbols)
+      printDynamicSymbols();
+  }
+  virtual void printProgramHeaders(bool PrintProgramHeaders,
+                                   cl::boolOrDefault PrintSectionMapping) {
+    if (PrintProgramHeaders)
+      printProgramHeaders();
+    if (PrintSectionMapping == cl::BOU_TRUE)
+      printSectionMapping();
+  }
+
   virtual void printUnwindInfo() = 0;
 
   // Only implemented for ELF at this time.
   virtual void printDynamicRelocations() { }
   virtual void printDynamicTable() { }
   virtual void printNeededLibraries() { }
-  virtual void printProgramHeaders() { }
   virtual void printSectionAsHex(StringRef SectionName) {}
   virtual void printHashTable() { }
   virtual void printGnuHashTable() { }
+  virtual void printHashSymbols() {}
   virtual void printLoadName() {}
   virtual void printVersionInfo() {}
   virtual void printGroupSections() {}
@@ -76,7 +89,10 @@ public:
   virtual void printCodeViewDebugInfo() { }
   virtual void
   mergeCodeViewTypes(llvm::codeview::MergingTypeTableBuilder &CVIDs,
-                     llvm::codeview::MergingTypeTableBuilder &CVTypes) {}
+                     llvm::codeview::MergingTypeTableBuilder &CVTypes,
+                     llvm::codeview::GlobalTypeTableBuilder &GlobalCVIDs,
+                     llvm::codeview::GlobalTypeTableBuilder &GlobalCVTypes,
+                     bool GHash) {}
 
   // Only implemented for MachO.
   virtual void printMachODataInCode() { }
@@ -93,6 +109,12 @@ public:
 
 protected:
   ScopedPrinter &W;
+
+private:
+  virtual void printSymbols() {}
+  virtual void printDynamicSymbols() {}
+  virtual void printProgramHeaders() {}
+  virtual void printSectionMapping() {}
 };
 
 std::error_code createCOFFDumper(const object::ObjectFile *Obj,
@@ -114,9 +136,9 @@ std::error_code createWasmDumper(const object::ObjectFile *Obj,
 void dumpCOFFImportFile(const object::COFFImportFile *File,
                         ScopedPrinter &Writer);
 
-void dumpCodeViewMergedTypes(
-    ScopedPrinter &Writer, llvm::codeview::MergingTypeTableBuilder &IDTable,
-    llvm::codeview::MergingTypeTableBuilder &TypeTable);
+void dumpCodeViewMergedTypes(ScopedPrinter &Writer,
+                             ArrayRef<ArrayRef<uint8_t>> IpiRecords,
+                             ArrayRef<ArrayRef<uint8_t>> TpiRecords);
 
 } // namespace llvm
 
diff --git a/tools/llvm-readobj/StackMapPrinter.h b/tools/llvm-readobj/StackMapPrinter.h
index 77a054b178a523dca428885bcf68ded05f3d7258..c40e65a53a57e692e00d1d84943617120852d05d 100644
--- a/tools/llvm-readobj/StackMapPrinter.h
+++ b/tools/llvm-readobj/StackMapPrinter.h
@@ -1,9 +1,8 @@
 //===-------- StackMapPrinter.h - Pretty-print stackmaps --------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/tools/llvm-readobj/WasmDumper.cpp b/tools/llvm-readobj/WasmDumper.cpp
index 79d3db4e2d291659bc7c92529118062c5492ed3c..edcd0731553d10e20fccc5758d3bad4efeed1eb5 100644
--- a/tools/llvm-readobj/WasmDumper.cpp
+++ b/tools/llvm-readobj/WasmDumper.cpp
@@ -1,9 +1,8 @@
 //===-- WasmDumper.cpp - Wasm-specific object file dumper -----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -41,6 +40,19 @@ static const EnumEntry<uint32_t> WasmSectionTypes[] = {
 #undef ENUM_ENTRY
 };
 
+static const EnumEntry<unsigned> WasmSymbolFlags[] = {
+#define ENUM_ENTRY(X)                                                          \
+  { #X, wasm::WASM_SYMBOL_##X }
+  ENUM_ENTRY(BINDING_GLOBAL),
+  ENUM_ENTRY(BINDING_WEAK),
+  ENUM_ENTRY(BINDING_LOCAL),
+  ENUM_ENTRY(VISIBILITY_DEFAULT),
+  ENUM_ENTRY(VISIBILITY_HIDDEN),
+  ENUM_ENTRY(UNDEFINED),
+  ENUM_ENTRY(EXPORTED),
+#undef ENUM_ENTRY
+};
+
 class WasmDumper : public ObjDumper {
 public:
   WasmDumper(const WasmObjectFile *Obj, ScopedPrinter &Writer)
@@ -49,8 +61,6 @@ public:
   void printFileHeaders() override;
   void printSectionHeaders() override;
   void printRelocations() override;
-  void printSymbols() override;
-  void printDynamicSymbols() override { llvm_unreachable("unimplemented"); }
   void printUnwindInfo() override { llvm_unreachable("unimplemented"); }
   void printStackMap() const override { llvm_unreachable("unimplemented"); }
 
@@ -59,6 +69,9 @@ protected:
   void printRelocation(const SectionRef &Section, const RelocationRef &Reloc);
 
 private:
+  void printSymbols() override;
+  void printDynamicSymbols() override { llvm_unreachable("unimplemented"); }
+
   const WasmObjectFile *Obj;
 };
 
@@ -80,11 +93,11 @@ void WasmDumper::printRelocation(const SectionRef &Section,
 
   bool HasAddend = false;
   switch (RelocType) {
-  case wasm::R_WEBASSEMBLY_MEMORY_ADDR_LEB:
-  case wasm::R_WEBASSEMBLY_MEMORY_ADDR_SLEB:
-  case wasm::R_WEBASSEMBLY_MEMORY_ADDR_I32:
-  case wasm::R_WEBASSEMBLY_FUNCTION_OFFSET_I32:
-  case wasm::R_WEBASSEMBLY_SECTION_OFFSET_I32:
+  case wasm::R_WASM_MEMORY_ADDR_LEB:
+  case wasm::R_WASM_MEMORY_ADDR_SLEB:
+  case wasm::R_WASM_MEMORY_ADDR_I32:
+  case wasm::R_WASM_FUNCTION_OFFSET_I32:
+  case wasm::R_WASM_SECTION_OFFSET_I32:
     HasAddend = true;
     break;
   default:
@@ -209,7 +222,19 @@ void WasmDumper::printSymbol(const SymbolRef &Sym) {
   WasmSymbol Symbol = Obj->getWasmSymbol(Sym.getRawDataRefImpl());
   W.printString("Name", Symbol.Info.Name);
   W.printEnum("Type", Symbol.Info.Kind, makeArrayRef(WasmSymbolTypes));
-  W.printHex("Flags", Symbol.Info.Flags);
+  W.printFlags("Flags", Symbol.Info.Flags, makeArrayRef(WasmSymbolFlags));
+
+  if (Symbol.Info.Flags & wasm::WASM_SYMBOL_UNDEFINED) {
+    W.printString("ImportName", Symbol.Info.ImportName);
+    W.printString("ImportModule", Symbol.Info.ImportModule);
+  }
+  if (Symbol.Info.Kind != wasm::WASM_SYMBOL_TYPE_DATA) {
+    W.printHex("ElementIndex", Symbol.Info.ElementIndex);
+  } else if (!(Symbol.Info.Flags & wasm::WASM_SYMBOL_UNDEFINED)) {
+    W.printHex("Offset", Symbol.Info.DataRef.Offset);
+    W.printHex("Segment", Symbol.Info.DataRef.Segment);
+    W.printHex("Size", Symbol.Info.DataRef.Size);
+  }
 }
 
 } // namespace
@@ -219,7 +244,7 @@ namespace llvm {
 std::error_code createWasmDumper(const object::ObjectFile *Obj,
                                  ScopedPrinter &Writer,
                                  std::unique_ptr<ObjDumper> &Result) {
-  const WasmObjectFile *WasmObj = dyn_cast<WasmObjectFile>(Obj);
+  const auto *WasmObj = dyn_cast<WasmObjectFile>(Obj);
   assert(WasmObj && "createWasmDumper called with non-wasm object");
 
   Result.reset(new WasmDumper(WasmObj, Writer));
diff --git a/tools/llvm-readobj/Win64EHDumper.cpp b/tools/llvm-readobj/Win64EHDumper.cpp
index f7e56b3615425c8abe6b685fe7bf388f79ff9e9b..e64b8f157180bc29285e0fc3f2f04df451e3ab45 100644
--- a/tools/llvm-readobj/Win64EHDumper.cpp
+++ b/tools/llvm-readobj/Win64EHDumper.cpp
@@ -1,9 +1,8 @@
 //===- Win64EHDumper.cpp - Win64 EH Printer ---------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/tools/llvm-readobj/Win64EHDumper.h b/tools/llvm-readobj/Win64EHDumper.h
index 772f68bf283f722bfb191ada9e1d8f4505d05c47..97458c916bec6f589bebaef3805b72549ca89f92 100644
--- a/tools/llvm-readobj/Win64EHDumper.h
+++ b/tools/llvm-readobj/Win64EHDumper.h
@@ -1,9 +1,8 @@
 //===- Win64EHDumper.h - Win64 EH Printing ----------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/tools/llvm-readobj/WindowsResourceDumper.cpp b/tools/llvm-readobj/WindowsResourceDumper.cpp
index 1f568a9636712efcfb053c278312b91033b4ab68..13989f696d9de2c93964af4bc51d8e2e3bec4ae3 100644
--- a/tools/llvm-readobj/WindowsResourceDumper.cpp
+++ b/tools/llvm-readobj/WindowsResourceDumper.cpp
@@ -1,9 +1,8 @@
 //===-- WindowsResourceDumper.cpp - Windows Resource printer --------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/tools/llvm-readobj/WindowsResourceDumper.h b/tools/llvm-readobj/WindowsResourceDumper.h
index ca6da404660524aefe5cc88c7421345aee107e89..6a5878804eb1ef139338c56638bdd1c16344f3a3 100644
--- a/tools/llvm-readobj/WindowsResourceDumper.h
+++ b/tools/llvm-readobj/WindowsResourceDumper.h
@@ -1,9 +1,8 @@
 //===- WindowsResourceDumper.h - Windows Resource printer -------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/tools/llvm-readobj/llvm-readobj.cpp b/tools/llvm-readobj/llvm-readobj.cpp
index ffc58450192198c14daa5c795df2a42bd62949e5..2c1674e064eb108908ae601ee11da523452580b5 100644
--- a/tools/llvm-readobj/llvm-readobj.cpp
+++ b/tools/llvm-readobj/llvm-readobj.cpp
@@ -1,9 +1,8 @@
 //===- llvm-readobj.cpp - Dump contents of an Object File -----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -23,6 +22,7 @@
 #include "Error.h"
 #include "ObjDumper.h"
 #include "WindowsResourceDumper.h"
+#include "llvm/DebugInfo/CodeView/GlobalTypeTableBuilder.h"
 #include "llvm/DebugInfo/CodeView/MergingTypeTableBuilder.h"
 #include "llvm/Object/Archive.h"
 #include "llvm/Object/COFFImportFile.h"
@@ -92,26 +92,25 @@ namespace opts {
                                 cl::desc("Alias for --section-headers"),
                                 cl::aliasopt(SectionHeaders), cl::NotHidden);
 
-  // -section-relocations, -sr
+  // -section-relocations
+  // Also -sr in llvm-readobj mode.
   cl::opt<bool> SectionRelocations("section-relocations",
     cl::desc("Display relocations for each section shown."));
-  cl::alias SectionRelocationsShort("sr",
-    cl::desc("Alias for --section-relocations"),
-    cl::aliasopt(SectionRelocations));
 
-  // -section-symbols, -st
+  // -section-symbols
+  // Also -st in llvm-readobj mode.
   cl::opt<bool> SectionSymbols("section-symbols",
     cl::desc("Display symbols for each section shown."));
-  cl::alias SectionSymbolsShort("st",
-    cl::desc("Alias for --section-symbols"),
-    cl::aliasopt(SectionSymbols));
 
-  // -section-data, -sd
+  // -section-data
+  // Also -sd in llvm-readobj mode.
   cl::opt<bool> SectionData("section-data",
     cl::desc("Display section data for each section shown."));
-  cl::alias SectionDataShort("sd",
-    cl::desc("Alias for --section-data"),
-    cl::aliasopt(SectionData));
+
+  // -section-mapping
+  cl::opt<cl::boolOrDefault>
+      SectionMapping("section-mapping",
+                     cl::desc("Display the section to segment mapping."));
 
   // -relocations, -relocs, -r
   cl::opt<bool> Relocations("relocations",
@@ -131,20 +130,25 @@ namespace opts {
 
   // -symbols
   // Also -s in llvm-readelf mode, or -t in llvm-readobj mode.
-  cl::opt<bool> Symbols("symbols",
-    cl::desc("Display the symbol table"));
+  cl::opt<bool>
+      Symbols("symbols",
+              cl::desc("Display the symbol table. Also display the dynamic "
+                       "symbol table when using GNU output style for ELF"));
   cl::alias SymbolsGNU("syms", cl::desc("Alias for --symbols"),
                        cl::aliasopt(Symbols));
 
-  // -dyn-symbols, -dyn-syms, -dt
+  // -dyn-symbols, -dyn-syms
+  // Also -dt in llvm-readobj mode.
   cl::opt<bool> DynamicSymbols("dyn-symbols",
     cl::desc("Display the dynamic symbol table"));
-  cl::alias DynamicSymbolsShort("dt",
-    cl::desc("Alias for --dyn-symbols"),
-    cl::aliasopt(DynamicSymbols));
   cl::alias DynSymsGNU("dyn-syms", cl::desc("Alias for --dyn-symbols"),
                        cl::aliasopt(DynamicSymbols));
 
+  // -hash-symbols
+  cl::opt<bool> HashSymbols(
+      "hash-symbols",
+      cl::desc("Display the dynamic symbols derived from the hash section"));
+
   // -unwind, -u
   cl::opt<bool> UnwindInfo("unwind",
     cl::desc("Display unwind information"));
@@ -184,6 +188,12 @@ namespace opts {
   cl::alias HexDumpShort("x", cl::desc("Alias for --hex-dump"),
                          cl::aliasopt(HexDump));
 
+  // -demangle, -C
+  cl::opt<bool> Demangle("demangle",
+                         cl::desc("Demangle symbol names in output"));
+  cl::alias DemangleShort("C", cl::desc("Alias for --demangle"),
+                          cl::aliasopt(Demangle), cl::NotHidden);
+
   // -hash-table
   cl::opt<bool> HashTable("hash-table",
     cl::desc("Display ELF hash table"));
@@ -209,6 +219,12 @@ namespace opts {
       CodeViewMergedTypes("codeview-merged-types",
                           cl::desc("Display the merged CodeView type stream"));
 
+  // -codeview-ghash
+  cl::opt<bool> CodeViewEnableGHash(
+      "codeview-ghash",
+      cl::desc(
+          "Enable global hashing for CodeView type stream de-duplication"));
+
   // -codeview-subsection-bytes
   cl::opt<bool> CodeViewSubsectionBytes(
       "codeview-subsection-bytes",
@@ -407,13 +423,17 @@ static bool isMipsArch(unsigned Arch) {
 namespace {
 struct ReadObjTypeTableBuilder {
   ReadObjTypeTableBuilder()
-      : Allocator(), IDTable(Allocator), TypeTable(Allocator) {}
+      : Allocator(), IDTable(Allocator), TypeTable(Allocator),
+        GlobalIDTable(Allocator), GlobalTypeTable(Allocator) {}
 
   llvm::BumpPtrAllocator Allocator;
   llvm::codeview::MergingTypeTableBuilder IDTable;
   llvm::codeview::MergingTypeTableBuilder TypeTable;
+  llvm::codeview::GlobalTypeTableBuilder GlobalIDTable;
+  llvm::codeview::GlobalTypeTableBuilder GlobalTypeTable;
+  std::vector<OwningBinary<Binary>> Binaries;
 };
-}
+} // namespace
 static ReadObjTypeTableBuilder CVTypes;
 
 /// Creates an format-specific object file dumper.
@@ -460,18 +480,18 @@ static void dumpObject(const ObjectFile *Obj, ScopedPrinter &Writer) {
     Dumper->printRelocations();
   if (opts::DynRelocs)
     Dumper->printDynamicRelocations();
-  if (opts::Symbols)
-    Dumper->printSymbols();
-  if (opts::DynamicSymbols)
-    Dumper->printDynamicSymbols();
+  if (opts::Symbols || opts::DynamicSymbols)
+    Dumper->printSymbols(opts::Symbols, opts::DynamicSymbols);
+  if (opts::HashSymbols)
+    Dumper->printHashSymbols();
   if (opts::UnwindInfo)
     Dumper->printUnwindInfo();
   if (opts::DynamicTable)
     Dumper->printDynamicTable();
   if (opts::NeededLibraries)
     Dumper->printNeededLibraries();
-  if (opts::ProgramHeaders)
-    Dumper->printProgramHeaders();
+  if (opts::ProgramHeaders || opts::SectionMapping == cl::BOU_TRUE)
+    Dumper->printProgramHeaders(opts::ProgramHeaders, opts::SectionMapping);
   if (!opts::StringDump.empty())
     llvm::for_each(opts::StringDump, [&Dumper, Obj](StringRef SectionName) {
       Dumper->printSectionAsString(Obj, SectionName);
@@ -533,7 +553,9 @@ static void dumpObject(const ObjectFile *Obj, ScopedPrinter &Writer) {
     if (opts::CodeView)
       Dumper->printCodeViewDebugInfo();
     if (opts::CodeViewMergedTypes)
-      Dumper->mergeCodeViewTypes(CVTypes.IDTable, CVTypes.TypeTable);
+      Dumper->mergeCodeViewTypes(CVTypes.IDTable, CVTypes.TypeTable,
+                                 CVTypes.GlobalIDTable, CVTypes.GlobalTypeTable,
+                                 opts::CodeViewEnableGHash);
   }
   if (Obj->isMachO()) {
     if (opts::MachODataInCode)
@@ -622,6 +644,8 @@ static void dumpInput(StringRef File) {
     dumpWindowsResourceFile(WinRes);
   else
     reportError(File, readobj_error::unrecognized_file_format);
+
+  CVTypes.Binaries.push_back(std::move(*BinaryOrErr));
 }
 
 /// Registers aliases that should only be allowed by readobj.
@@ -636,13 +660,36 @@ static void registerReadobjAliases() {
   // --section-details (not implemented yet).
   static cl::alias SymbolsShort("t", cl::desc("Alias for --symbols"),
                                 cl::aliasopt(opts::Symbols), cl::NotHidden);
+
+  // The following two-letter aliases are only provided for readobj, as readelf
+  // allows single-letter args to be grouped together.
+  static cl::alias SectionRelocationsShort(
+      "sr", cl::desc("Alias for --section-relocations"),
+      cl::aliasopt(opts::SectionRelocations));
+  static cl::alias SectionDataShort("sd", cl::desc("Alias for --section-data"),
+                                    cl::aliasopt(opts::SectionData));
+  static cl::alias SectionSymbolsShort("st",
+                                       cl::desc("Alias for --section-symbols"),
+                                       cl::aliasopt(opts::SectionSymbols));
+  static cl::alias DynamicSymbolsShort("dt",
+                                       cl::desc("Alias for --dyn-symbols"),
+                                       cl::aliasopt(opts::DynamicSymbols));
 }
 
 /// Registers aliases that should only be allowed by readelf.
 static void registerReadelfAliases() {
   // -s is here because for readobj it means --sections.
   static cl::alias SymbolsShort("s", cl::desc("Alias for --symbols"),
-                                cl::aliasopt(opts::Symbols), cl::NotHidden);
+                                cl::aliasopt(opts::Symbols), cl::NotHidden,
+                                cl::Grouping);
+
+  // Allow all single letter flags to be grouped together.
+  for (auto &OptEntry : cl::getRegisteredOptions()) {
+    StringRef ArgName = OptEntry.getKey();
+    cl::Option *Option = OptEntry.getValue();
+    if (ArgName.size() == 1)
+      Option->setFormattingFlag(cl::Grouping);
+  }
 }
 
 int main(int argc, const char *argv[]) {
@@ -681,14 +728,19 @@ int main(int argc, const char *argv[]) {
   }
 
   // Default to stdin if no filename is specified.
-  if (opts::InputFilenames.size() == 0)
+  if (opts::InputFilenames.empty())
     opts::InputFilenames.push_back("-");
 
   llvm::for_each(opts::InputFilenames, dumpInput);
 
   if (opts::CodeViewMergedTypes) {
     ScopedPrinter W(outs());
-    dumpCodeViewMergedTypes(W, CVTypes.IDTable, CVTypes.TypeTable);
+    if (opts::CodeViewEnableGHash)
+      dumpCodeViewMergedTypes(W, CVTypes.GlobalIDTable.records(),
+                              CVTypes.GlobalTypeTable.records());
+    else
+      dumpCodeViewMergedTypes(W, CVTypes.IDTable.records(),
+                              CVTypes.TypeTable.records());
   }
 
   return 0;
diff --git a/tools/llvm-readobj/llvm-readobj.h b/tools/llvm-readobj/llvm-readobj.h
index 92ed098dc6421cfe5f1bd8999fe0a65584ef520a..500d7295e69468ae0e3fb3bc78d654da85560e5f 100644
--- a/tools/llvm-readobj/llvm-readobj.h
+++ b/tools/llvm-readobj/llvm-readobj.h
@@ -1,9 +1,8 @@
 //===-- llvm-readobj.h ----------------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -52,10 +51,10 @@ namespace opts {
   extern llvm::cl::opt<bool> SectionRelocations;
   extern llvm::cl::opt<bool> SectionSymbols;
   extern llvm::cl::opt<bool> SectionData;
-  extern llvm::cl::opt<bool> DynamicSymbols;
   extern llvm::cl::opt<bool> ExpandRelocs;
   extern llvm::cl::opt<bool> RawRelr;
   extern llvm::cl::opt<bool> CodeViewSubsectionBytes;
+  extern llvm::cl::opt<bool> Demangle;
   enum OutputStyleTy { LLVM, GNU };
   extern llvm::cl::opt<OutputStyleTy> Output;
 } // namespace opts
diff --git a/tools/llvm-rtdyld/LLVMBuild.txt b/tools/llvm-rtdyld/LLVMBuild.txt
index c4ed49bdff29ee42f3a9d7b196ae7cd22ee9efa7..38d29a14262fd8ba1ce4add15e67a73dfb64be8c 100644
--- a/tools/llvm-rtdyld/LLVMBuild.txt
+++ b/tools/llvm-rtdyld/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./tools/llvm-rtdyld/LLVMBuild.txt ------------------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/tools/llvm-rtdyld/llvm-rtdyld.cpp b/tools/llvm-rtdyld/llvm-rtdyld.cpp
index 975638ed82d1dc0dacc330378adfd2ebf7d1e845..e792b7257ea5e877b19c50f98a640c0ef668ef54 100644
--- a/tools/llvm-rtdyld/llvm-rtdyld.cpp
+++ b/tools/llvm-rtdyld/llvm-rtdyld.cpp
@@ -1,9 +1,8 @@
 //===-- llvm-rtdyld.cpp - MCJIT Testing Tool ------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/tools/llvm-shlib/gen-msvc-exports.py b/tools/llvm-shlib/gen-msvc-exports.py
index 6f0a6786d34fb35492d3fab060dd8d560923f2be..2b7291de28b272ab55bb6f9fd0ddd67ec13b804f 100644
--- a/tools/llvm-shlib/gen-msvc-exports.py
+++ b/tools/llvm-shlib/gen-msvc-exports.py
@@ -1,9 +1,8 @@
 #===- gen-msvc-exports.py - Generate C API export file -------*- python -*--===#
 #
-#                     The LLVM Compiler Infrastructure
-#
-# This file is distributed under the University of Illinois Open Source
-# License. See LICENSE.TXT for details.
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 #
 #===------------------------------------------------------------------------===#
 #
diff --git a/tools/llvm-shlib/libllvm.cpp b/tools/llvm-shlib/libllvm.cpp
index 40b4f66b0733135a9d0ca726acb390b3f0eb2533..51abeb79ad96e6febc729538cc23289adcd8744d 100644
--- a/tools/llvm-shlib/libllvm.cpp
+++ b/tools/llvm-shlib/libllvm.cpp
@@ -1,9 +1,8 @@
 //===-libllvm.cpp - LLVM Shared Library -----------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/tools/llvm-size/LLVMBuild.txt b/tools/llvm-size/LLVMBuild.txt
index b4c538a406be0f4d36dfaf9bb467bba914f1c5de..5ae7fc39aec864dd11aeaa6640e58995a603b34d 100644
--- a/tools/llvm-size/LLVMBuild.txt
+++ b/tools/llvm-size/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./tools/llvm-size/LLVMBuild.txt --------------------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/tools/llvm-size/llvm-size.cpp b/tools/llvm-size/llvm-size.cpp
index be1e5bb72681d7e9878eeba85d37742a1cdf8563..32d10abe773f293728d4ab30046cabd05129ce2d 100644
--- a/tools/llvm-size/llvm-size.cpp
+++ b/tools/llvm-size/llvm-size.cpp
@@ -1,9 +1,8 @@
 //===-- llvm-size.cpp - Print the size of each object section ---*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -578,7 +577,7 @@ static void printFileSectionSizes(StringRef file) {
   } else if (MachOUniversalBinary *UB =
                  dyn_cast<MachOUniversalBinary>(&Bin)) {
     // If we have a list of architecture flags specified dump only those.
-    if (!ArchAll && ArchFlags.size() != 0) {
+    if (!ArchAll && !ArchFlags.empty()) {
       // Look for a slice in the universal binary that matches each ArchFlag.
       bool ArchFound;
       for (unsigned i = 0; i < ArchFlags.size(); ++i) {
diff --git a/tools/llvm-special-case-list-fuzzer/DummySpecialCaseListFuzzer.cpp b/tools/llvm-special-case-list-fuzzer/DummySpecialCaseListFuzzer.cpp
index e447419113b9b6f519c251aa3265beb84cd99c13..86bce4abd9d5207b72d28f550f147d80e594da25 100644
--- a/tools/llvm-special-case-list-fuzzer/DummySpecialCaseListFuzzer.cpp
+++ b/tools/llvm-special-case-list-fuzzer/DummySpecialCaseListFuzzer.cpp
@@ -1,9 +1,8 @@
 //===--- DummySpecialCaseListFuzzer.cpp -----------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/tools/llvm-special-case-list-fuzzer/special-case-list-fuzzer.cpp b/tools/llvm-special-case-list-fuzzer/special-case-list-fuzzer.cpp
index e7e310b3c7f2576cfa23661d180822bed2880d0b..aaab5f8470c9d075f8d348b29b533120d29eb21e 100644
--- a/tools/llvm-special-case-list-fuzzer/special-case-list-fuzzer.cpp
+++ b/tools/llvm-special-case-list-fuzzer/special-case-list-fuzzer.cpp
@@ -1,9 +1,8 @@
 //===--- special-case-list-fuzzer.cpp - Fuzzer for special case lists -----===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/tools/llvm-split/LLVMBuild.txt b/tools/llvm-split/LLVMBuild.txt
index 780c76afa41c3c8d761becc8232f9d7046a67263..546f455a891d1ecf935e5b227a925a7edffa872d 100644
--- a/tools/llvm-split/LLVMBuild.txt
+++ b/tools/llvm-split/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./tools/llvm-split/LLVMBuild.txt -------------------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/tools/llvm-split/llvm-split.cpp b/tools/llvm-split/llvm-split.cpp
index a6204dc156d1a4ea794a3f6bb527b63b92f51754..879e69ccec534fc8bedecdf2d88d8ec36707b919 100644
--- a/tools/llvm-split/llvm-split.cpp
+++ b/tools/llvm-split/llvm-split.cpp
@@ -1,9 +1,8 @@
 //===-- llvm-split: command line tool for testing module splitter ---------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/tools/llvm-stress/LLVMBuild.txt b/tools/llvm-stress/LLVMBuild.txt
index f383d351dd36d4ea95cde67118500899e08c0481..abea0b1b0f73605b21dbdf5670dc32fb6763df84 100644
--- a/tools/llvm-stress/LLVMBuild.txt
+++ b/tools/llvm-stress/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./tools/llvm-stress/LLVMBuild.txt -------------------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/tools/llvm-stress/llvm-stress.cpp b/tools/llvm-stress/llvm-stress.cpp
index d8ec11251ff65a1d4b035f8785f987a75dd4e06e..64624d70a2cf866bc419cdb35292b78591458633 100644
--- a/tools/llvm-stress/llvm-stress.cpp
+++ b/tools/llvm-stress/llvm-stress.cpp
@@ -1,9 +1,8 @@
 //===- llvm-stress.cpp - Generate random LL files to stress-test LLVM -----===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -356,8 +355,8 @@ struct StoreModifier: public Modifier {
   void Act() override {
     // Try to use predefined pointers. If non-exist, use undef pointer value;
     Value *Ptr = getRandomPointerValue();
-    Type  *Tp = Ptr->getType();
-    Value *Val = getRandomValue(Tp->getContainedType(0));
+    PointerType *Tp = cast<PointerType>(Ptr->getType());
+    Value *Val = getRandomValue(Tp->getElementType());
     Type  *ValTy = Val->getType();
 
     // Do not store vectors of i1s because they are unsupported
diff --git a/tools/llvm-strings/LLVMBuild.txt b/tools/llvm-strings/LLVMBuild.txt
index 9cd41d0eebf5c6d5f577b6e8e21cc8a6ea0126fd..ec336934811213ae807c848917646f1158489013 100644
--- a/tools/llvm-strings/LLVMBuild.txt
+++ b/tools/llvm-strings/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./tools/llvm-strings/LLVMBuild.txt -----------------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/tools/llvm-strings/llvm-strings.cpp b/tools/llvm-strings/llvm-strings.cpp
index cdc2a6ef033b8e1e2fed7988aad92a88036430b5..996610d26b26fa20af5a6dcab6c106c80033b56a 100644
--- a/tools/llvm-strings/llvm-strings.cpp
+++ b/tools/llvm-strings/llvm-strings.cpp
@@ -1,9 +1,8 @@
 //===-- llvm-strings.cpp - Printable String dumping utility ---------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/tools/llvm-symbolizer/llvm-symbolizer.cpp b/tools/llvm-symbolizer/llvm-symbolizer.cpp
index 6d40a5403504691cd673f2fd6b62a10645130f30..8ff7a22a71aa6102f53c7773c8fc57c89b6aecf8 100644
--- a/tools/llvm-symbolizer/llvm-symbolizer.cpp
+++ b/tools/llvm-symbolizer/llvm-symbolizer.cpp
@@ -1,9 +1,8 @@
 //===-- llvm-symbolizer.cpp - Simple addr2line-like symbolizer ------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -39,12 +38,17 @@ ClUseSymbolTable("use-symbol-table", cl::init(true),
 
 static cl::opt<FunctionNameKind> ClPrintFunctions(
     "functions", cl::init(FunctionNameKind::LinkageName),
-    cl::desc("Print function name for a given address:"),
+    cl::desc("Print function name for a given address"), cl::ValueOptional,
     cl::values(clEnumValN(FunctionNameKind::None, "none", "omit function name"),
                clEnumValN(FunctionNameKind::ShortName, "short",
                           "print short function name"),
                clEnumValN(FunctionNameKind::LinkageName, "linkage",
-                          "print function linkage name")));
+                          "print function linkage name (default)"),
+               // Sentinel value for unspecified value.
+               clEnumValN(FunctionNameKind::LinkageName, "", "")));
+static cl::alias ClPrintFunctionsShort("f", cl::desc("Alias for -functions"),
+                                       cl::NotHidden, cl::Grouping,
+                                       cl::aliasopt(ClPrintFunctions));
 
 static cl::opt<bool>
     ClUseRelativeAddress("relative-address", cl::init(false),
@@ -54,18 +58,46 @@ static cl::opt<bool>
 static cl::opt<bool>
     ClPrintInlining("inlining", cl::init(true),
                     cl::desc("Print all inlined frames for a given address"));
+static cl::alias
+    ClPrintInliningAliasI("i", cl::desc("Alias for -inlining"),
+                          cl::NotHidden, cl::aliasopt(ClPrintInlining),
+                          cl::Grouping);
+static cl::alias
+    ClPrintInliningAliasInlines("inlines", cl::desc("Alias for -inlining"),
+                                cl::NotHidden, cl::aliasopt(ClPrintInlining));
+
+// -basenames, -s
+static cl::opt<bool> ClBasenames("basenames", cl::init(false),
+                                 cl::desc("Strip directory names from paths"));
+static cl::alias ClBasenamesShort("s", cl::desc("Alias for -basenames"),
+                                  cl::NotHidden, cl::aliasopt(ClBasenames));
 
+// -demangle, -C, -no-demangle
 static cl::opt<bool>
 ClDemangle("demangle", cl::init(true), cl::desc("Demangle function names"));
+static cl::alias
+ClDemangleShort("C", cl::desc("Alias for -demangle"),
+                cl::NotHidden, cl::aliasopt(ClDemangle), cl::Grouping);
+static cl::opt<bool>
+ClNoDemangle("no-demangle", cl::init(false),
+             cl::desc("Don't demangle function names"));
 
 static cl::opt<std::string> ClDefaultArch("default-arch", cl::init(""),
                                           cl::desc("Default architecture "
                                                    "(for multi-arch objects)"));
 
+// -obj, -exe, -e
 static cl::opt<std::string>
 ClBinaryName("obj", cl::init(""),
              cl::desc("Path to object file to be symbolized (if not provided, "
                       "object file should be specified for each input line)"));
+static cl::alias
+ClBinaryNameAliasExe("exe", cl::desc("Alias for -obj"),
+                     cl::NotHidden, cl::aliasopt(ClBinaryName));
+static cl::alias
+ClBinaryNameAliasE("e", cl::desc("Alias for -obj"),
+                   cl::NotHidden, cl::aliasopt(ClBinaryName));
+
 
 static cl::opt<std::string>
     ClDwpName("dwp", cl::init(""),
@@ -75,13 +107,25 @@ static cl::list<std::string>
 ClDsymHint("dsym-hint", cl::ZeroOrMore,
            cl::desc("Path to .dSYM bundles to search for debug info for the "
                     "object files"));
+
+// -print-address, -addresses, -a
 static cl::opt<bool>
-    ClPrintAddress("print-address", cl::init(false),
-                   cl::desc("Show address before line information"));
+ClPrintAddress("print-address", cl::init(false),
+               cl::desc("Show address before line information"));
+static cl::alias
+ClPrintAddressAliasAddresses("addresses", cl::desc("Alias for -print-address"),
+                             cl::NotHidden, cl::aliasopt(ClPrintAddress));
+static cl::alias
+ClPrintAddressAliasA("a", cl::desc("Alias for -print-address"),
+                     cl::NotHidden, cl::aliasopt(ClPrintAddress), cl::Grouping);
 
+// -pretty-print, -p
 static cl::opt<bool>
     ClPrettyPrint("pretty-print", cl::init(false),
                   cl::desc("Make the output more human friendly"));
+static cl::alias ClPrettyPrintShort("p", cl::desc("Alias for -pretty-print"),
+                                    cl::NotHidden,
+                                    cl::aliasopt(ClPrettyPrint), cl::Grouping);
 
 static cl::opt<int> ClPrintSourceContextLines(
     "print-source-context-lines", cl::init(0),
@@ -90,6 +134,19 @@ static cl::opt<int> ClPrintSourceContextLines(
 static cl::opt<bool> ClVerbose("verbose", cl::init(false),
                                cl::desc("Print verbose line info"));
 
+// -adjust-vma
+static cl::opt<unsigned long long>
+    ClAdjustVMA("adjust-vma", cl::init(0), cl::value_desc("offset"),
+                cl::desc("Add specified offset to object file addresses"));
+
+static cl::list<std::string> ClInputAddresses(cl::Positional,
+                                              cl::desc("<input addresses>..."),
+                                              cl::ZeroOrMore);
+
+static cl::opt<std::string>
+    ClFallbackDebugPath("fallback-debug-path", cl::init(""),
+                        cl::desc("Fallback path for debug binaries."));
+
 template<typename T>
 static bool error(Expected<T> &ResOrErr) {
   if (ResOrErr)
@@ -137,14 +194,52 @@ static bool parseCommand(StringRef InputString, bool &IsData,
   return !StringRef(pos, offset_length).getAsInteger(0, ModuleOffset);
 }
 
+static void symbolizeInput(StringRef InputString, LLVMSymbolizer &Symbolizer,
+                           DIPrinter &Printer) {
+  bool IsData = false;
+  std::string ModuleName;
+  uint64_t ModuleOffset = 0;
+  if (!parseCommand(StringRef(InputString), IsData, ModuleName, ModuleOffset)) {
+    outs() << InputString;
+    return;
+  }
+
+  if (ClPrintAddress) {
+    outs() << "0x";
+    outs().write_hex(ModuleOffset);
+    StringRef Delimiter = ClPrettyPrint ? ": " : "\n";
+    outs() << Delimiter;
+  }
+  ModuleOffset -= ClAdjustVMA;
+  if (IsData) {
+    auto ResOrErr = Symbolizer.symbolizeData(ModuleName, ModuleOffset);
+    Printer << (error(ResOrErr) ? DIGlobal() : ResOrErr.get());
+  } else if (ClPrintInlining) {
+    auto ResOrErr =
+        Symbolizer.symbolizeInlinedCode(ModuleName, ModuleOffset, ClDwpName);
+    Printer << (error(ResOrErr) ? DIInliningInfo() : ResOrErr.get());
+  } else {
+    auto ResOrErr =
+        Symbolizer.symbolizeCode(ModuleName, ModuleOffset, ClDwpName);
+    Printer << (error(ResOrErr) ? DILineInfo() : ResOrErr.get());
+  }
+  outs() << "\n";
+  outs().flush();
+}
+
 int main(int argc, char **argv) {
   InitLLVM X(argc, argv);
 
   llvm::sys::InitializeCOMRAII COM(llvm::sys::COMThreadingMode::MultiThreaded);
-
   cl::ParseCommandLineOptions(argc, argv, "llvm-symbolizer\n");
+
+  // If both --demangle and --no-demangle are specified then pick the last one.
+  if (ClNoDemangle.getPosition() > ClDemangle.getPosition())
+    ClDemangle = !ClNoDemangle;
+
   LLVMSymbolizer::Options Opts(ClPrintFunctions, ClUseSymbolTable, ClDemangle,
-                               ClUseRelativeAddress, ClDefaultArch);
+                               ClUseRelativeAddress, ClDefaultArch,
+                               ClFallbackDebugPath);
 
   for (const auto &hint : ClDsymHint) {
     if (sys::path::extension(hint) == ".dSYM") {
@@ -157,45 +252,18 @@ int main(int argc, char **argv) {
   LLVMSymbolizer Symbolizer(Opts);
 
   DIPrinter Printer(outs(), ClPrintFunctions != FunctionNameKind::None,
-                    ClPrettyPrint, ClPrintSourceContextLines, ClVerbose);
-
-  const int kMaxInputStringLength = 1024;
-  char InputString[kMaxInputStringLength];
-
-  while (true) {
-    if (!fgets(InputString, sizeof(InputString), stdin))
-      break;
-
-    bool IsData = false;
-    std::string ModuleName;
-    uint64_t ModuleOffset = 0;
-    if (!parseCommand(StringRef(InputString), IsData, ModuleName,
-                      ModuleOffset)) {
-      outs() << InputString;
-      continue;
-    }
+                    ClPrettyPrint, ClPrintSourceContextLines, ClVerbose,
+                    ClBasenames);
 
-    if (ClPrintAddress) {
-      outs() << "0x";
-      outs().write_hex(ModuleOffset);
-      StringRef Delimiter = ClPrettyPrint ? ": " : "\n";
-      outs() << Delimiter;
-    }
-    if (IsData) {
-      auto ResOrErr = Symbolizer.symbolizeData(ModuleName, ModuleOffset);
-      Printer << (error(ResOrErr) ? DIGlobal() : ResOrErr.get());
-    } else if (ClPrintInlining) {
-      auto ResOrErr =
-          Symbolizer.symbolizeInlinedCode(ModuleName, ModuleOffset, ClDwpName);
-      Printer << (error(ResOrErr) ? DIInliningInfo()
-                                             : ResOrErr.get());
-    } else {
-      auto ResOrErr =
-          Symbolizer.symbolizeCode(ModuleName, ModuleOffset, ClDwpName);
-      Printer << (error(ResOrErr) ? DILineInfo() : ResOrErr.get());
-    }
-    outs() << "\n";
-    outs().flush();
+  if (ClInputAddresses.empty()) {
+    const int kMaxInputStringLength = 1024;
+    char InputString[kMaxInputStringLength];
+
+    while (fgets(InputString, sizeof(InputString), stdin))
+      symbolizeInput(InputString, Symbolizer, Printer);
+  } else {
+    for (StringRef Address : ClInputAddresses)
+      symbolizeInput(Address, Symbolizer, Printer);
   }
 
   return 0;
diff --git a/tools/llvm-undname/LLVMBuild.txt b/tools/llvm-undname/LLVMBuild.txt
index 6e598f888ff1d834c638b9bad604aeb20265141c..a931bfa32677d0b44e61ea07c6b1e3f34c227e8e 100644
--- a/tools/llvm-undname/LLVMBuild.txt
+++ b/tools/llvm-undname/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./tools/llvm-undname/LLVMBuild.txt -----------------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/tools/llvm-undname/llvm-undname.cpp b/tools/llvm-undname/llvm-undname.cpp
index 60520c8f7be0c521bb4764432807a423e24e6568..bbc7b7334211859b8fef08a5cf6ff2a586a31699 100644
--- a/tools/llvm-undname/llvm-undname.cpp
+++ b/tools/llvm-undname/llvm-undname.cpp
@@ -1,10 +1,9 @@
 //===-- llvm-undname.cpp - Microsoft ABI name undecorator
 //------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -33,7 +32,7 @@ cl::opt<bool> DumpBackReferences("backrefs", cl::Optional,
 cl::list<std::string> Symbols(cl::Positional, cl::desc("<input symbols>"),
                               cl::ZeroOrMore);
 
-static void demangle(const std::string &S) {
+static void msDemangle(const std::string &S) {
   int Status;
   MSDemangleFlags Flags = MSDF_None;
   if (DumpBackReferences)
@@ -75,14 +74,14 @@ int main(int argc, char **argv) {
         outs() << Line << "\n";
         outs().flush();
       }
-      demangle(Line);
+      msDemangle(Line);
       outs() << "\n";
     }
   } else {
     for (StringRef S : Symbols) {
       outs() << S << "\n";
       outs().flush();
-      demangle(S);
+      msDemangle(S);
       outs() << "\n";
     }
   }
diff --git a/tools/llvm-xray/func-id-helper.cpp b/tools/llvm-xray/func-id-helper.cpp
index c2bef6ddfb3966c3f35c41b39f75313699bb1aa1..987eb43b73242a76fb8ea0b20a537017bf101c5f 100644
--- a/tools/llvm-xray/func-id-helper.cpp
+++ b/tools/llvm-xray/func-id-helper.cpp
@@ -1,9 +1,8 @@
 //===- xray-fc-account.cpp: XRay Function Call Accounting Tool ------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/tools/llvm-xray/func-id-helper.h b/tools/llvm-xray/func-id-helper.h
index 3e0780d54f9007ccd9cfd9c1a2a25557cd50d26e..c6ce198170d51a04f64d0f768d845815c0eefef3 100644
--- a/tools/llvm-xray/func-id-helper.h
+++ b/tools/llvm-xray/func-id-helper.h
@@ -1,9 +1,8 @@
 //===- func-id-helper.h - XRay Function ID Conversion Helpers -------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/tools/llvm-xray/llvm-xray.cpp b/tools/llvm-xray/llvm-xray.cpp
index e74628f5025f669fbf15d8ca73a1f3bc2d40a7f8..9ee653e97b2ddfa58e2673333c92af6a823abf36 100644
--- a/tools/llvm-xray/llvm-xray.cpp
+++ b/tools/llvm-xray/llvm-xray.cpp
@@ -1,9 +1,8 @@
 //===- llvm-xray.cpp: XRay Tool Main Program ------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/tools/llvm-xray/trie-node.h b/tools/llvm-xray/trie-node.h
index e6ba4e215b91f1b93d342df07cfcb6010d4b72a3..47d4b8f1e78c84cff110bd1be291423f482f1d53 100644
--- a/tools/llvm-xray/trie-node.h
+++ b/tools/llvm-xray/trie-node.h
@@ -1,9 +1,8 @@
 //===- trie-node.h - XRay Call Stack Data Structure -----------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/tools/llvm-xray/xray-account.cpp b/tools/llvm-xray/xray-account.cpp
index 9985c9adcf6c1073ad1e1d2f2f28d98b89d7ffc2..bc5354a849d60bcaef55ac2aff20fafd68b0a557 100644
--- a/tools/llvm-xray/xray-account.cpp
+++ b/tools/llvm-xray/xray-account.cpp
@@ -1,9 +1,8 @@
 //===- xray-account.h - XRay Function Call Accounting ---------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/tools/llvm-xray/xray-account.h b/tools/llvm-xray/xray-account.h
index 5c457f178166567daf7ad994498e15ef3ba4a383..9c9f86c8fcf7f9da739fa859b28a923fa06ef057 100644
--- a/tools/llvm-xray/xray-account.h
+++ b/tools/llvm-xray/xray-account.h
@@ -1,9 +1,8 @@
 //===- xray-account.h - XRay Function Call Accounting ---------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/tools/llvm-xray/xray-color-helper.cpp b/tools/llvm-xray/xray-color-helper.cpp
index 78a264b73d8f35e5f1661e7d62e41921e50ddbc4..c09cad3ba7d248c0552dc34c6d06f28acbabfb2c 100644
--- a/tools/llvm-xray/xray-color-helper.cpp
+++ b/tools/llvm-xray/xray-color-helper.cpp
@@ -1,9 +1,8 @@
 //===-- xray-graph.cpp: XRay Function Call Graph Renderer -----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/tools/llvm-xray/xray-color-helper.h b/tools/llvm-xray/xray-color-helper.h
index b2dcf626a65f66509d2b20d8527b4cddba26cb8b..0940fc2113433f5d0c27079078a14f3abfab4b3d 100644
--- a/tools/llvm-xray/xray-color-helper.h
+++ b/tools/llvm-xray/xray-color-helper.h
@@ -1,9 +1,8 @@
 //===-- xray-graph.h - XRay Function Call Graph Renderer --------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/tools/llvm-xray/xray-converter.cpp b/tools/llvm-xray/xray-converter.cpp
index 3f153b99bc9366adfc710f3cc518bbcd678602f7..efc067cfc28678577945047854f80641ad79cbdd 100644
--- a/tools/llvm-xray/xray-converter.cpp
+++ b/tools/llvm-xray/xray-converter.cpp
@@ -1,9 +1,8 @@
 //===- xray-converter.cpp: XRay Trace Conversion --------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/tools/llvm-xray/xray-converter.h b/tools/llvm-xray/xray-converter.h
index 5f0a3ee298eb203a2095b17300bd2121992645db..db6d2b1614ee2394f16b9be61514bf92c0014085 100644
--- a/tools/llvm-xray/xray-converter.h
+++ b/tools/llvm-xray/xray-converter.h
@@ -1,9 +1,8 @@
 //===- xray-converter.h - XRay Trace Conversion ---------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/tools/llvm-xray/xray-extract.cpp b/tools/llvm-xray/xray-extract.cpp
index 10fe7d8d62094e74d6298e0d072652a887925cfc..b54bc143ed6ad01534eb7a9fdadf8a875ead4c4f 100644
--- a/tools/llvm-xray/xray-extract.cpp
+++ b/tools/llvm-xray/xray-extract.cpp
@@ -1,9 +1,8 @@
 //===- xray-extract.cpp: XRay Instrumentation Map Extraction --------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/tools/llvm-xray/xray-fdr-dump.cpp b/tools/llvm-xray/xray-fdr-dump.cpp
index 389825605b621280037412f470ae0167e9367c47..e0c07039a8914bf215f7d5e7a668436f1ef957d6 100644
--- a/tools/llvm-xray/xray-fdr-dump.cpp
+++ b/tools/llvm-xray/xray-fdr-dump.cpp
@@ -1,9 +1,8 @@
 //===- xray-fdr-dump.cpp: XRay FDR Trace Dump Tool ------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/tools/llvm-xray/xray-graph-diff.cpp b/tools/llvm-xray/xray-graph-diff.cpp
index a22f2a99811d1963fc9788ded2f0a32ff1b57818..a514be97f40bee566e771ad4fc604679517c064c 100644
--- a/tools/llvm-xray/xray-graph-diff.cpp
+++ b/tools/llvm-xray/xray-graph-diff.cpp
@@ -1,9 +1,8 @@
 //===-- xray-graph-diff.cpp: XRay Function Call Graph Renderer ------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/tools/llvm-xray/xray-graph-diff.h b/tools/llvm-xray/xray-graph-diff.h
index 5abec91d858258c976f0f9b9244a7484103d071a..5d12c563f47c497a415ad3c2be55ed10192e57a1 100644
--- a/tools/llvm-xray/xray-graph-diff.h
+++ b/tools/llvm-xray/xray-graph-diff.h
@@ -1,9 +1,8 @@
 //===-- xray-graph-diff.h - XRay Graph Diff Renderer ------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/tools/llvm-xray/xray-graph.cpp b/tools/llvm-xray/xray-graph.cpp
index fe49cca20d577305ddea4f5dee57bfb49befc5cc..2365ce56f921946f17bb5646738feea8baaf066e 100644
--- a/tools/llvm-xray/xray-graph.cpp
+++ b/tools/llvm-xray/xray-graph.cpp
@@ -1,9 +1,8 @@
 //===-- xray-graph.cpp: XRay Function Call Graph Renderer -----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/tools/llvm-xray/xray-graph.h b/tools/llvm-xray/xray-graph.h
index fc7f8bb470f24319848c31eb703661eafc8d2062..79e5d8fa422bcd45e13ce7cef6576a8504d42002 100644
--- a/tools/llvm-xray/xray-graph.h
+++ b/tools/llvm-xray/xray-graph.h
@@ -1,9 +1,8 @@
 //===-- xray-graph.h - XRay Function Call Graph Renderer --------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/tools/llvm-xray/xray-registry.cpp b/tools/llvm-xray/xray-registry.cpp
index fe58e4deaa1eb57e862184ba0dd9ae7e5f33c8fe..e5c253d2e8f10ac8bcd65dac01d159d9026f1dca 100644
--- a/tools/llvm-xray/xray-registry.cpp
+++ b/tools/llvm-xray/xray-registry.cpp
@@ -1,9 +1,8 @@
 //===- xray-registry.cpp: Implement a command registry. -------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/tools/llvm-xray/xray-registry.h b/tools/llvm-xray/xray-registry.h
index 6eab016273f52073b754d96bdbeff2d4c3ee3e7a..d6fae78ea53ccc2a3b007483bb7322048c227f7b 100644
--- a/tools/llvm-xray/xray-registry.h
+++ b/tools/llvm-xray/xray-registry.h
@@ -1,9 +1,8 @@
 //===- xray-registry.h - Define registry mechanism for commands. ----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/tools/llvm-xray/xray-stacks.cpp b/tools/llvm-xray/xray-stacks.cpp
index d3af9e25e6f28eafe3ec5447be8aaa11265aa947..1c5838339e1ca08ae4c48137d4676b7f256f04d6 100644
--- a/tools/llvm-xray/xray-stacks.cpp
+++ b/tools/llvm-xray/xray-stacks.cpp
@@ -1,9 +1,8 @@
 //===- xray-stacks.cpp: XRay Function Call Stack Accounting ---------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/tools/llvm-yaml-numeric-parser-fuzzer/DummyYAMLNumericParserFuzzer.cpp b/tools/llvm-yaml-numeric-parser-fuzzer/DummyYAMLNumericParserFuzzer.cpp
index 3396168acec02fdc20d2fc202e13adca86dc3e78..d0dc5beb4ac6b87923d3fa7327653c4dbcbae5e1 100644
--- a/tools/llvm-yaml-numeric-parser-fuzzer/DummyYAMLNumericParserFuzzer.cpp
+++ b/tools/llvm-yaml-numeric-parser-fuzzer/DummyYAMLNumericParserFuzzer.cpp
@@ -1,9 +1,8 @@
 //===--- DummyYAMLNumericParserFuzzer.cpp ---------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/tools/llvm-yaml-numeric-parser-fuzzer/yaml-numeric-parser-fuzzer.cpp b/tools/llvm-yaml-numeric-parser-fuzzer/yaml-numeric-parser-fuzzer.cpp
index 9134d7628ec9dbff2f2651943c76a00350375726..039c3dc599988cddcc2c31298049e694b44a0401 100644
--- a/tools/llvm-yaml-numeric-parser-fuzzer/yaml-numeric-parser-fuzzer.cpp
+++ b/tools/llvm-yaml-numeric-parser-fuzzer/yaml-numeric-parser-fuzzer.cpp
@@ -1,9 +1,8 @@
 //===--- special-case-list-fuzzer.cpp - Fuzzer for special case lists -----===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/tools/lto/LTODisassembler.cpp b/tools/lto/LTODisassembler.cpp
index 186acebc01bf71c6cbcabf153c3ee423890dc182..1c4cc6af29d14c0369c27fadc07c546517c1a9f8 100644
--- a/tools/lto/LTODisassembler.cpp
+++ b/tools/lto/LTODisassembler.cpp
@@ -1,9 +1,8 @@
 //===-- LTODisassembler.cpp - LTO Disassembler interface ------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/tools/lto/lto.cpp b/tools/lto/lto.cpp
index 0a3ba98957e7c5dde91294f4265e7011f545063e..379b5b897141af094f113e0e43e917119cd88560 100644
--- a/tools/lto/lto.cpp
+++ b/tools/lto/lto.cpp
@@ -1,9 +1,8 @@
 //===-lto.cpp - LLVM Link Time Optimizer ----------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/tools/msbuild/license.txt b/tools/msbuild/license.txt
index 76aa2afdf81e98c59fd20c9fb5bb5f814971c49c..63c17f148e7a99365704909fa3d4e4ba9d22c440 100644
--- a/tools/msbuild/license.txt
+++ b/tools/msbuild/license.txt
@@ -1,6 +1,241 @@
-====================
-LLVM Release License
-====================
+==============================================================================
+The LLVM Project is under the Apache License v2.0 with LLVM Exceptions:
+==============================================================================
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+    TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+    1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+    2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+    3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+    4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+    5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+    6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+    7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+    8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+    9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+    END OF TERMS AND CONDITIONS
+
+    APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+    Copyright [yyyy] [name of copyright owner]
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+
+
+---- LLVM Exceptions to the Apache 2.0 License ----
+
+As an exception, if, as a result of your compiling your source code, portions
+of this Software are embedded into an Object form of such source code, you
+may redistribute such embedded portions in such Object form without complying
+with the conditions of Sections 4(a), 4(b) and 4(d) of the License.
+
+In addition, if you combine or link compiled forms of this Software with
+software that is licensed under the GPLv2 ("Combined Software") and if a
+court of competent jurisdiction determines that the patent provision (Section
+3), the indemnity provision (Section 9) or other Section of the License
+conflicts with the conditions of the GPLv2, you may retroactively and
+prospectively choose to deem waived or otherwise exclude such Section(s) of
+the License, but only in their entirety and only with respect to the Combined
+Software.
+
+==============================================================================
+Software from third parties included in the LLVM Project:
+==============================================================================
+The LLVM Project contains third party software which is under different license
+terms. All such code will be identified clearly using at least one of two
+mechanisms:
+1) It will be in a separate directory tree with its own `LICENSE.txt` or
+   `LICENSE` file at the top containing the specific license and restrictions
+   which apply to that software, or
+2) It will contain specific license and restriction terms at the top of every
+   file.
+
+==============================================================================
+Legacy LLVM License (https://llvm.org/docs/DeveloperPolicy.html#legacy):
+==============================================================================
 University of Illinois/NCSA
 Open Source License
 
@@ -24,16 +259,3 @@ Permission is hereby granted, free of charge, to any person obtaining a copy of
     * Neither the names of the LLVM Team, University of Illinois at Urbana-Champaign, nor the names of its contributors may be used to endorse or promote products derived from this Software without specific prior written permission.
 
 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH THE SOFTWARE.
-
-====================
-The LLVM software contains code written by third parties.  Such software will have its own individual LICENSE.TXT file in the directory in which it appears.  This file will describe the copyrights, license, and restrictions which apply
-to that code.
-
-The disclaimer of warranty in the University of Illinois Open Source License applies to all code in the LLVM Distribution, and nothing in any of the other licenses gives permission to use the names of the LLVM Team or the University of Illinois to endorse or promote products derived from this Software.
-
-The following pieces of software have additional or alternate copyrights, licenses, and/or restrictions:
-
-Program             Directory
--------             ---------
-<none yet>
-
diff --git a/tools/obj2yaml/Error.cpp b/tools/obj2yaml/Error.cpp
index 2b6a815fe2f57df2a9525846efd415a498339ddc..d11d47b5b227d2af630e91830d7a1b0bbc92c4c5 100644
--- a/tools/obj2yaml/Error.cpp
+++ b/tools/obj2yaml/Error.cpp
@@ -1,9 +1,8 @@
 //===- Error.cpp - system_error extensions for obj2yaml ---------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/tools/obj2yaml/Error.h b/tools/obj2yaml/Error.h
index f5111f257ce36f257af540f56b8e6a9b7a7df8af..315d7f4bf49f263daa5aa2eaab5e61314aa86408 100644
--- a/tools/obj2yaml/Error.h
+++ b/tools/obj2yaml/Error.h
@@ -1,9 +1,8 @@
 //===- Error.h - system_error extensions for obj2yaml -----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/tools/obj2yaml/coff2yaml.cpp b/tools/obj2yaml/coff2yaml.cpp
index 6835dcf71ffec81939dab384a29b633662cd5bfb..44d7c272e1fa8b9332a5ea2317313d0223ebead1 100644
--- a/tools/obj2yaml/coff2yaml.cpp
+++ b/tools/obj2yaml/coff2yaml.cpp
@@ -1,13 +1,13 @@
 //===------ utils/obj2yaml.cpp - obj2yaml conversion tool -------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #include "obj2yaml.h"
+#include "llvm/ADT/StringMap.h"
 #include "llvm/DebugInfo/CodeView/DebugChecksumsSubsection.h"
 #include "llvm/DebugInfo/CodeView/DebugStringTableSubsection.h"
 #include "llvm/DebugInfo/CodeView/StringsAndChecksums.h"
@@ -142,6 +142,18 @@ void COFFDumper::dumpSections(unsigned NumSections) {
   codeview::StringsAndChecksumsRef SC;
   initializeFileAndStringTable(Obj, SC);
 
+  StringMap<bool> SymbolUnique;
+  for (const auto &S : Obj.symbols()) {
+    object::COFFSymbolRef Symbol = Obj.getCOFFSymbol(S);
+    StringRef Name;
+    Obj.getSymbolName(Symbol, Name);
+    StringMap<bool>::iterator It;
+    bool Inserted;
+    std::tie(It, Inserted) = SymbolUnique.insert(std::make_pair(Name, true));
+    if (!Inserted)
+      It->second = false;
+  }
+
   for (const auto &ObjSection : Obj.sections()) {
     const object::coff_section *COFFSection = Obj.getCOFFSection(ObjSection);
     COFFYAML::Section NewYAMLSection;
@@ -192,7 +204,10 @@ void COFFDumper::dumpSections(unsigned NumSections) {
        OS.flush();
        report_fatal_error(Buf);
       }
-      Rel.SymbolName = *SymbolNameOrErr;
+      if (SymbolUnique.lookup(*SymbolNameOrErr))
+        Rel.SymbolName = *SymbolNameOrErr;
+      else
+        Rel.SymbolTableIndex = reloc->SymbolTableIndex;
       Rel.VirtualAddress = reloc->VirtualAddress;
       Rel.Type = reloc->Type;
       Relocations.push_back(Rel);
diff --git a/tools/obj2yaml/dwarf2yaml.cpp b/tools/obj2yaml/dwarf2yaml.cpp
index 1ce6c036e6f5f5a5c7b8136f3731a248c1c2dc20..371cba85fa41a3846867964f3ffd0d2a3eafbfa5 100644
--- a/tools/obj2yaml/dwarf2yaml.cpp
+++ b/tools/obj2yaml/dwarf2yaml.cpp
@@ -1,9 +1,8 @@
 //===------ dwarf2yaml.cpp - obj2yaml conversion tool -----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/tools/obj2yaml/elf2yaml.cpp b/tools/obj2yaml/elf2yaml.cpp
index dea4d1b31827ef505354756252077fe22f68afb2..a3fe811b9d5c3b9738c8a99d9645c8d0bc76f6d2 100644
--- a/tools/obj2yaml/elf2yaml.cpp
+++ b/tools/obj2yaml/elf2yaml.cpp
@@ -1,9 +1,8 @@
 //===------ utils/elf2yaml.cpp - obj2yaml conversion tool -------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -22,6 +21,7 @@ namespace {
 template <class ELFT>
 class ELFDumper {
   typedef object::Elf_Sym_Impl<ELFT> Elf_Sym;
+  typedef typename ELFT::Dyn Elf_Dyn;
   typedef typename ELFT::Shdr Elf_Shdr;
   typedef typename ELFT::Word Elf_Word;
   typedef typename ELFT::Rel Elf_Rel;
@@ -51,9 +51,9 @@ class ELFDumper {
   template <class RelT>
   std::error_code dumpRelocation(const RelT *Rel, const Elf_Shdr *SymTab,
                                  ELFYAML::Relocation &R);
-
-  ErrorOr<ELFYAML::RelocationSection *> dumpRelSection(const Elf_Shdr *Shdr);
-  ErrorOr<ELFYAML::RelocationSection *> dumpRelaSection(const Elf_Shdr *Shdr);
+  
+  ErrorOr<ELFYAML::DynamicSection *> dumpDynamicSection(const Elf_Shdr *Shdr);
+  ErrorOr<ELFYAML::RelocationSection *> dumpRelocSection(const Elf_Shdr *Shdr);
   ErrorOr<ELFYAML::RawContentSection *>
   dumpContentSection(const Elf_Shdr *Shdr);
   ErrorOr<ELFYAML::NoBitsSection *> dumpNoBitsSection(const Elf_Shdr *Shdr);
@@ -114,6 +114,7 @@ template <class ELFT> ErrorOr<ELFYAML::Object *> ELFDumper<ELFT>::dump() {
   Y->Header.Class = ELFYAML::ELF_ELFCLASS(Obj.getHeader()->getFileClass());
   Y->Header.Data = ELFYAML::ELF_ELFDATA(Obj.getHeader()->getDataEncoding());
   Y->Header.OSABI = Obj.getHeader()->e_ident[ELF::EI_OSABI];
+  Y->Header.ABIVersion = Obj.getHeader()->e_ident[ELF::EI_ABIVERSION];
   Y->Header.Type = Obj.getHeader()->e_type;
   Y->Header.Machine = Obj.getHeader()->e_machine;
   Y->Header.Flags = Obj.getHeader()->e_flags;
@@ -130,6 +131,13 @@ template <class ELFT> ErrorOr<ELFYAML::Object *> ELFDumper<ELFT>::dump() {
   SectionNames.resize(Sections.size());
   for (const Elf_Shdr &Sec : Sections) {
     switch (Sec.sh_type) {
+    case ELF::SHT_DYNAMIC: {
+      ErrorOr<ELFYAML::DynamicSection *> S = dumpDynamicSection(&Sec);
+      if (std::error_code EC = S.getError())
+        return EC;
+      Y->Sections.push_back(std::unique_ptr<ELFYAML::Section>(S.get()));
+      break;
+    }
     case ELF::SHT_NULL:
     case ELF::SHT_STRTAB:
       // Do not dump these sections.
@@ -147,15 +155,9 @@ template <class ELFT> ErrorOr<ELFYAML::Object *> ELFDumper<ELFT>::dump() {
       ShndxTable = *TableOrErr;
       break;
     }
+    case ELF::SHT_REL:
     case ELF::SHT_RELA: {
-      ErrorOr<ELFYAML::RelocationSection *> S = dumpRelaSection(&Sec);
-      if (std::error_code EC = S.getError())
-        return EC;
-      Y->Sections.push_back(std::unique_ptr<ELFYAML::Section>(S.get()));
-      break;
-    }
-    case ELF::SHT_REL: {
-      ErrorOr<ELFYAML::RelocationSection *> S = dumpRelSection(&Sec);
+      ErrorOr<ELFYAML::RelocationSection *> S = dumpRelocSection(&Sec);
       if (std::error_code EC = S.getError())
         return EC;
       Y->Sections.push_back(std::unique_ptr<ELFYAML::Section>(S.get()));
@@ -317,6 +319,8 @@ std::error_code ELFDumper<ELFT>::dumpCommonSection(const Elf_Shdr *Shdr,
   S.Flags = Shdr->sh_flags;
   S.Address = Shdr->sh_addr;
   S.AddressAlign = Shdr->sh_addralign;
+  if (Shdr->sh_entsize)
+    S.EntSize = static_cast<llvm::yaml::Hex64>(Shdr->sh_entsize);
 
   auto NameOrErr = getUniquedSectionName(Shdr);
   if (!NameOrErr)
@@ -350,44 +354,32 @@ ELFDumper<ELFT>::dumpCommonRelocationSection(const Elf_Shdr *Shdr,
   auto NameOrErr = getUniquedSectionName(*InfoSection);
   if (!NameOrErr)
     return errorToErrorCode(NameOrErr.takeError());
-  S.Info = NameOrErr.get();
+  S.RelocatableSec = NameOrErr.get();
 
   return obj2yaml_error::success;
 }
 
 template <class ELFT>
-ErrorOr<ELFYAML::RelocationSection *>
-ELFDumper<ELFT>::dumpRelSection(const Elf_Shdr *Shdr) {
-  assert(Shdr->sh_type == ELF::SHT_REL && "Section type is not SHT_REL");
-  auto S = make_unique<ELFYAML::RelocationSection>();
-
-  if (std::error_code EC = dumpCommonRelocationSection(Shdr, *S))
+ErrorOr<ELFYAML::DynamicSection *>
+ELFDumper<ELFT>::dumpDynamicSection(const Elf_Shdr *Shdr) {
+  auto S = make_unique<ELFYAML::DynamicSection>();
+  if (std::error_code EC = dumpCommonSection(Shdr, *S))
     return EC;
 
-  auto SymTabOrErr = Obj.getSection(Shdr->sh_link);
-  if (!SymTabOrErr)
-    return errorToErrorCode(SymTabOrErr.takeError());
-  const Elf_Shdr *SymTab = *SymTabOrErr;
+  auto DynTagsOrErr = Obj.template getSectionContentsAsArray<Elf_Dyn>(Shdr);
+  if (!DynTagsOrErr)
+    return errorToErrorCode(DynTagsOrErr.takeError());
 
-  auto Rels = Obj.rels(Shdr);
-  if (!Rels)
-    return errorToErrorCode(Rels.takeError());
-  for (const Elf_Rel &Rel : *Rels) {
-    ELFYAML::Relocation R;
-    if (std::error_code EC = dumpRelocation(&Rel, SymTab, R))
-      return EC;
-    S->Relocations.push_back(R);
-  }
+  for (const Elf_Dyn &Dyn : *DynTagsOrErr)
+    S->Entries.push_back({(ELFYAML::ELF_DYNTAG)Dyn.getTag(), Dyn.getVal()});
 
   return S.release();
 }
 
 template <class ELFT>
 ErrorOr<ELFYAML::RelocationSection *>
-ELFDumper<ELFT>::dumpRelaSection(const Elf_Shdr *Shdr) {
-  assert(Shdr->sh_type == ELF::SHT_RELA && "Section type is not SHT_RELA");
+ELFDumper<ELFT>::dumpRelocSection(const Elf_Shdr *Shdr) {
   auto S = make_unique<ELFYAML::RelocationSection>();
-
   if (std::error_code EC = dumpCommonRelocationSection(Shdr, *S))
     return EC;
 
@@ -396,15 +388,27 @@ ELFDumper<ELFT>::dumpRelaSection(const Elf_Shdr *Shdr) {
     return errorToErrorCode(SymTabOrErr.takeError());
   const Elf_Shdr *SymTab = *SymTabOrErr;
 
-  auto Rels = Obj.relas(Shdr);
-  if (!Rels)
-    return errorToErrorCode(Rels.takeError());
-  for (const Elf_Rela &Rel : *Rels) {
-    ELFYAML::Relocation R;
-    if (std::error_code EC = dumpRelocation(&Rel, SymTab, R))
-      return EC;
-    R.Addend = Rel.r_addend;
-    S->Relocations.push_back(R);
+  if (Shdr->sh_type == ELF::SHT_REL) {
+    auto Rels = Obj.rels(Shdr);
+    if (!Rels)
+      return errorToErrorCode(Rels.takeError());
+    for (const Elf_Rel &Rel : *Rels) {
+      ELFYAML::Relocation R;
+      if (std::error_code EC = dumpRelocation(&Rel, SymTab, R))
+        return EC;
+      S->Relocations.push_back(R);
+    }
+  } else {
+    auto Rels = Obj.relas(Shdr);
+    if (!Rels)
+      return errorToErrorCode(Rels.takeError());
+    for (const Elf_Rela &Rel : *Rels) {
+      ELFYAML::Relocation R;
+      if (std::error_code EC = dumpRelocation(&Rel, SymTab, R))
+        return EC;
+      R.Addend = Rel.r_addend;
+      S->Relocations.push_back(R);
+    }
   }
 
   return S.release();
@@ -464,7 +468,7 @@ ErrorOr<ELFYAML::Group *> ELFDumper<ELFT>::dumpGroup(const Elf_Shdr *Shdr) {
   Expected<StringRef> symbolName = getSymbolName(symbol, StrTab, Symtab);
   if (!symbolName)
     return errorToErrorCode(symbolName.takeError());
-  S->Info = *symbolName;
+  S->Signature = *symbolName;
   const Elf_Word *groupMembers =
       reinterpret_cast<const Elf_Word *>(sectionContents->data());
   const long count = (Shdr->sh_size) / sizeof(Elf_Word);
diff --git a/tools/obj2yaml/macho2yaml.cpp b/tools/obj2yaml/macho2yaml.cpp
index fa81ce974ecf6598578738ca335cd1e38c7d3c46..63e81686632489a8dd23fafccadf3e95ff1aa1ef 100644
--- a/tools/obj2yaml/macho2yaml.cpp
+++ b/tools/obj2yaml/macho2yaml.cpp
@@ -1,9 +1,8 @@
 //===------ macho2yaml.cpp - obj2yaml conversion tool -----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/tools/obj2yaml/obj2yaml.cpp b/tools/obj2yaml/obj2yaml.cpp
index 459572a57344db7368c0c6f05cb8743e9989ec22..1350970096f24c8eef2e8e6dc46302254dc0a2c2 100644
--- a/tools/obj2yaml/obj2yaml.cpp
+++ b/tools/obj2yaml/obj2yaml.cpp
@@ -1,9 +1,8 @@
 //===------ utils/obj2yaml.cpp - obj2yaml conversion tool -------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/tools/obj2yaml/obj2yaml.h b/tools/obj2yaml/obj2yaml.h
index 3e96659462b09040f1d79a69c70cc2de3300634f..c7c4956d6bf9c3ba32e1ff77f01711b80cfbb171 100644
--- a/tools/obj2yaml/obj2yaml.h
+++ b/tools/obj2yaml/obj2yaml.h
@@ -1,9 +1,8 @@
 //===------ utils/obj2yaml.hpp - obj2yaml conversion tool -------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 // This file declares some helper routines, and also the format-specific
 // writers. To add a new format, add the declaration here, and, in a separate
diff --git a/tools/obj2yaml/wasm2yaml.cpp b/tools/obj2yaml/wasm2yaml.cpp
index 7581bbef50ad602b31a5c60d1eadc38dac1b541f..8e7f2fa641fb34bd3c7d0381495ca114fe22a5e1 100644
--- a/tools/obj2yaml/wasm2yaml.cpp
+++ b/tools/obj2yaml/wasm2yaml.cpp
@@ -1,9 +1,8 @@
 //===------ utils/wasm2yaml.cpp - obj2yaml conversion tool ------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -32,7 +31,7 @@ public:
 
 } // namespace
 
-static WasmYAML::Table make_table(const wasm::WasmTable &Table) {
+static WasmYAML::Table makeTable(const wasm::WasmTable &Table) {
   WasmYAML::Table T;
   T.ElemType = Table.ElemType;
   T.TableLimits.Flags = Table.Limits.Flags;
@@ -41,7 +40,7 @@ static WasmYAML::Table make_table(const wasm::WasmTable &Table) {
   return T;
 }
 
-static WasmYAML::Limits make_limits(const wasm::WasmLimits &Limits) {
+static WasmYAML::Limits makeLimits(const wasm::WasmLimits &Limits) {
   WasmYAML::Limits L;
   L.Flags = Limits.Flags;
   L.Initial = Limits.Initial;
@@ -133,6 +132,29 @@ WasmDumper::dumpCustomSection(const WasmSection &WasmSec) {
     }
 
     CustomSec = std::move(LinkingSec);
+  } else if (WasmSec.Name == "producers") {
+    std::unique_ptr<WasmYAML::ProducersSection> ProducersSec =
+        make_unique<WasmYAML::ProducersSection>();
+    const llvm::wasm::WasmProducerInfo &Info = Obj.getProducerInfo();
+    for (auto &E : Info.Languages) {
+      WasmYAML::ProducerEntry Producer;
+      Producer.Name = E.first;
+      Producer.Version = E.second;
+      ProducersSec->Languages.push_back(Producer);
+    }
+    for (auto &E : Info.Tools) {
+      WasmYAML::ProducerEntry Producer;
+      Producer.Name = E.first;
+      Producer.Version = E.second;
+      ProducersSec->Tools.push_back(Producer);
+    }
+    for (auto &E : Info.SDKs) {
+      WasmYAML::ProducerEntry Producer;
+      Producer.Name = E.first;
+      Producer.Version = E.second;
+      ProducersSec->SDKs.push_back(Producer);
+    }
+    CustomSec = std::move(ProducersSec);
   } else {
     CustomSec = make_unique<WasmYAML::CustomSection>(WasmSec.Name);
   }
@@ -172,7 +194,7 @@ ErrorOr<WasmYAML::Object *> WasmDumper::dump() {
         if (FunctionSig.Returns.size())
           Sig.ReturnType = static_cast<uint32_t>(FunctionSig.Returns[0]);
         for (const auto &ParamType : FunctionSig.Params)
-          Sig.ParamTypes.push_back(static_cast<uint32_t>(ParamType));
+          Sig.ParamTypes.emplace_back(static_cast<uint32_t>(ParamType));
         TypeSec->Signatures.push_back(Sig);
       }
       S = std::move(TypeSec);
@@ -198,10 +220,10 @@ ErrorOr<WasmYAML::Object *> WasmDumper::dump() {
           Im.EventImport.SigIndex = Import.Event.SigIndex;
           break;
         case wasm::WASM_EXTERNAL_TABLE:
-          Im.TableImport = make_table(Import.Table);
+          Im.TableImport = makeTable(Import.Table);
           break;
         case wasm::WASM_EXTERNAL_MEMORY:
-          Im.Memory = make_limits(Import.Memory);
+          Im.Memory = makeLimits(Import.Memory);
           break;
         }
         ImportSec->Imports.push_back(Im);
@@ -220,7 +242,7 @@ ErrorOr<WasmYAML::Object *> WasmDumper::dump() {
     case wasm::WASM_SEC_TABLE: {
       auto TableSec = make_unique<WasmYAML::TableSection>();
       for (const wasm::WasmTable &Table : Obj.tables()) {
-        TableSec->Tables.push_back(make_table(Table));
+        TableSec->Tables.push_back(makeTable(Table));
       }
       S = std::move(TableSec);
       break;
@@ -228,7 +250,7 @@ ErrorOr<WasmYAML::Object *> WasmDumper::dump() {
     case wasm::WASM_SEC_MEMORY: {
       auto MemorySec = make_unique<WasmYAML::MemorySection>();
       for (const wasm::WasmLimits &Memory : Obj.memories()) {
-        MemorySec->Memories.push_back(make_limits(Memory));
+        MemorySec->Memories.push_back(makeLimits(Memory));
       }
       S = std::move(MemorySec);
       break;
diff --git a/tools/opt-remarks/liboptremarks.cpp b/tools/opt-remarks/liboptremarks.cpp
index 13acada06ac2c0015a7a3d7ee200c98120a268ab..699d33a8e62764d01d3f72b7e3c552e522472105 100644
--- a/tools/opt-remarks/liboptremarks.cpp
+++ b/tools/opt-remarks/liboptremarks.cpp
@@ -1,9 +1,8 @@
 //===-liboptremarks.cpp - LLVM Opt-Remarks Shared Library -----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/tools/opt-viewer/opt-diff.py b/tools/opt-viewer/opt-diff.py
index f3bfd1860b911d8beb11be885073d5184dabbc8d..36e81a5d569a746567776da4f1aa57cbd5fc8570 100755
--- a/tools/opt-viewer/opt-diff.py
+++ b/tools/opt-viewer/opt-diff.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2.7
+#!/usr/bin/env python
 
 from __future__ import print_function
 
diff --git a/tools/opt-viewer/opt-stats.py b/tools/opt-viewer/opt-stats.py
index 03de23bdb2757c9a256cd87df125c1bd59fce261..f4ee3a7d44e63958dffc6ed7add97e5797a66f7c 100755
--- a/tools/opt-viewer/opt-stats.py
+++ b/tools/opt-viewer/opt-stats.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2.7
+#!/usr/bin/env python
 
 from __future__ import print_function
 
diff --git a/tools/opt-viewer/opt-viewer.py b/tools/opt-viewer/opt-viewer.py
index 4887043e0f9650c7a83f0bc6e7edd13615e5c682..4c105886cfdf274b38f5ed3ce6539516971585e9 100755
--- a/tools/opt-viewer/opt-viewer.py
+++ b/tools/opt-viewer/opt-viewer.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2.7
+#!/usr/bin/env python
 
 from __future__ import print_function
 
@@ -72,7 +72,10 @@ class SourceFileRenderer:
         file_text = stream.read()
 
         if self.no_highlight:
-            html_highlighted = file_text.decode('utf-8')
+            if sys.version_info.major >= 3:
+                html_highlighted = file_text
+            else:
+                html_highlighted = file_text.decode('utf-8')
         else:
             html_highlighted = highlight(
             file_text,
@@ -117,12 +120,26 @@ class SourceFileRenderer:
         indent = line[:max(r.Column, 1) - 1]
         indent = re.sub('\S', ' ', indent)
 
+        # Create expanded message and link if we have a multiline message.
+        lines = r.message.split('\n')
+        if len(lines) > 1:
+            expand_link = '<a style="text-decoration: none;" href="" onclick="toggleExpandedMessage(this); return false;">+</a>'
+            message = lines[0]
+            expand_message = u'''
+<div class="full-info" style="display:none;">
+  <div class="col-left"><pre style="display:inline">{}</pre></div>
+  <div class="expanded col-left"><pre>{}</pre></div>
+</div>'''.format(indent, '\n'.join(lines[1:]))
+        else:
+            expand_link = ''
+            expand_message = ''
+            message = r.message
         print(u'''
 <tr>
 <td></td>
 <td>{r.RelativeHotness}</td>
 <td class=\"column-entry-{r.color}\">{r.PassWithDiffPrefix}</td>
-<td><pre style="display:inline">{indent}</pre><span class=\"column-entry-yellow\"> {r.message}&nbsp;</span></td>
+<td><pre style="display:inline">{indent}</pre><span class=\"column-entry-yellow\">{expand_link} {message}&nbsp;</span>{expand_message}</td>
 <td class=\"column-entry-yellow\">{inlining_context}</td>
 </tr>'''.format(**locals()), file=self.stream)
 
@@ -136,6 +153,23 @@ class SourceFileRenderer:
 <meta charset="utf-8" />
 <head>
 <link rel='stylesheet' type='text/css' href='style.css'>
+<script type="text/javascript">
+/* Simple helper to show/hide the expanded message of a remark. */
+function toggleExpandedMessage(e) {{
+  var FullTextElems = e.parentElement.parentElement.getElementsByClassName("full-info");
+  if (!FullTextElems || FullTextElems.length < 1) {{
+      return false;
+  }}
+  var FullText = FullTextElems[0];
+  if (FullText.style.display == 'none') {{
+    e.innerHTML = '-';
+    FullText.style.display = 'block';
+  }} else {{
+    e.innerHTML = '+';
+    FullText.style.display = 'none';
+  }}
+}}
+</script>
 </head>
 <body>
 <div class="centered">
@@ -205,7 +239,7 @@ class IndexRenderer:
 </html>''', file=self.stream)
 
 
-def _render_file(source_dir, output_dir, ctx, no_highlight, entry):
+def _render_file(source_dir, output_dir, ctx, no_highlight, entry, filter_):
     global context
     context = ctx
     filename, remarks = entry
@@ -310,6 +344,11 @@ def main():
         '--demangler',
         help='Set the demangler to be used (defaults to %s)' % optrecord.Remark.default_demangler)
 
+    parser.add_argument(
+        '--filter',
+        default='',
+        help='Only display remarks from passes matching filter expression')
+
     # Do not make this a global variable.  Values needed to be propagated through
     # to individual classes and functions to be portable with multiprocessing across
     # Windows and non-Windows.
@@ -325,7 +364,7 @@ def main():
         sys.exit(1)
 
     all_remarks, file_remarks, should_display_hotness = \
-        optrecord.gather_results(files, args.jobs, print_progress)
+        optrecord.gather_results(files, args.jobs, print_progress, args.filter)
 
     map_remarks(all_remarks)
 
diff --git a/tools/opt-viewer/optpmap.py b/tools/opt-viewer/optpmap.py
index db6b079b3a6d712e388ff465ad346125cc5401b9..8124c8c9036eb87bbba989c311cd0b6efbd63595 100644
--- a/tools/opt-viewer/optpmap.py
+++ b/tools/opt-viewer/optpmap.py
@@ -14,7 +14,7 @@ def _init(current, total):
 
 
 def _wrapped_func(func_and_args):
-    func, argument, should_print_progress = func_and_args
+    func, argument, should_print_progress, filter_ = func_and_args
 
     if should_print_progress:
         with _current.get_lock():
@@ -22,10 +22,10 @@ def _wrapped_func(func_and_args):
         sys.stdout.write('\r\t{} of {}'.format(_current.value, _total.value))
         sys.stdout.flush()
 
-    return func(argument)
+    return func(argument, filter_)
 
 
-def pmap(func, iterable, processes, should_print_progress, *args, **kwargs):
+def pmap(func, iterable, processes, should_print_progress, filter_=None, *args, **kwargs):
     """
     A parallel map function that reports on its progress.
 
@@ -40,9 +40,9 @@ def pmap(func, iterable, processes, should_print_progress, *args, **kwargs):
     _current = multiprocessing.Value('i', 0)
     _total = multiprocessing.Value('i', len(iterable))
 
-    func_and_args = [(func, arg, should_print_progress,) for arg in iterable]
+    func_and_args = [(func, arg, should_print_progress, filter_) for arg in iterable]
     if processes == 1:
-        result = map(_wrapped_func, func_and_args, *args, **kwargs)
+        result = list(map(_wrapped_func, func_and_args, *args, **kwargs))
     else:
         pool = multiprocessing.Pool(initializer=_init,
                                     initargs=(_current, _total,),
diff --git a/tools/opt-viewer/optrecord.py b/tools/opt-viewer/optrecord.py
index 8cf22ee3f05c38163e597e6fa091dc4a484fb27b..e952d1befd65e6c4167b66cf09904df7c51fdc03 100644
--- a/tools/opt-viewer/optrecord.py
+++ b/tools/opt-viewer/optrecord.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2.7
+#!/usr/bin/env python
 
 from __future__ import print_function
 
@@ -24,6 +24,8 @@ try:
 except:
     pass
 
+import re
+
 import optpmap
 
 try:
@@ -263,18 +265,24 @@ class Missed(Remark):
         return "red"
 
 
-def get_remarks(input_file):
+def get_remarks(input_file, filter_):
     max_hotness = 0
     all_remarks = dict()
     file_remarks = defaultdict(functools.partial(defaultdict, list))
 
     with open(input_file) as f:
         docs = yaml.load_all(f, Loader=Loader)
+
+        filter_e = re.compile(filter_)
         for remark in docs:
             remark.canonicalize()
             # Avoid remarks withoug debug location or if they are duplicated
             if not hasattr(remark, 'DebugLoc') or remark.key in all_remarks:
                 continue
+
+            if filter_ and not filter_e.search(remark.Pass):
+                continue
+
             all_remarks[remark.key] = remark
 
             file_remarks[remark.File][remark.Line].append(remark)
@@ -289,13 +297,13 @@ def get_remarks(input_file):
     return max_hotness, all_remarks, file_remarks
 
 
-def gather_results(filenames, num_jobs, should_print_progress):
+def gather_results(filenames, num_jobs, should_print_progress, filter_):
     if should_print_progress:
         print('Reading YAML files...')
     if not Remark.demangler_proc:
         Remark.set_demangler(Remark.default_demangler)
     remarks = optpmap.pmap(
-        get_remarks, filenames, num_jobs, should_print_progress)
+        get_remarks, filenames, num_jobs, should_print_progress, filter_)
     max_hotness = max(entry[0] for entry in remarks)
 
     def merge_file_remarks(file_remarks_job, all_remarks, merged):
diff --git a/tools/opt-viewer/style.css b/tools/opt-viewer/style.css
index 0d3347c1578c828e2d34ac0f7f9f0831a0e599c2..550c7e1cdff31e1bda8744abd3a1f99bfc8d868f 100644
--- a/tools/opt-viewer/style.css
+++ b/tools/opt-viewer/style.css
@@ -139,6 +139,16 @@ td:first-child {
 td:last-child {
   border-right: none;
 }
+.expanded {
+  background-color: #f2f2f2;
+  padding-top: 5px;
+  padding-left: 5px;
+}
+.col-left {
+  float: left;
+  margin-bottom: -99999px;
+  padding-bottom: 99999px;
+}
 
 /* Generated with pygmentize -S colorful -f html >> style.css */
 
diff --git a/tools/opt/AnalysisWrappers.cpp b/tools/opt/AnalysisWrappers.cpp
index cfdd2cf1582b0c553855217d96fb49263aa227c7..b888605a516c753eb2df38960b2b1aa6834b36c7 100644
--- a/tools/opt/AnalysisWrappers.cpp
+++ b/tools/opt/AnalysisWrappers.cpp
@@ -1,9 +1,8 @@
 //===- AnalysisWrappers.cpp - Wrappers around non-pass analyses -----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/tools/opt/BreakpointPrinter.cpp b/tools/opt/BreakpointPrinter.cpp
index d3f54c034f55ecd5389927b1fe79780acc8ebd37..db9c676175ce76f96e8f9066a46bcdaaea7c379c 100644
--- a/tools/opt/BreakpointPrinter.cpp
+++ b/tools/opt/BreakpointPrinter.cpp
@@ -1,9 +1,8 @@
 //===- BreakpointPrinter.cpp - Breakpoint location printer ----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
diff --git a/tools/opt/BreakpointPrinter.h b/tools/opt/BreakpointPrinter.h
index 57670e5ee8d8e71dd710a4ac809b604a669e4059..2877555f852ce56648424a85b3bda5d038655d66 100644
--- a/tools/opt/BreakpointPrinter.h
+++ b/tools/opt/BreakpointPrinter.h
@@ -1,9 +1,8 @@
 //===- BreakpointPrinter.h - Breakpoint location printer ------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
diff --git a/tools/opt/Debugify.cpp b/tools/opt/Debugify.cpp
index 3b1effba1592226f2500ff806e03c05be06fd515..222cc702bc1f5da195fa85322b505067b8edc51a 100644
--- a/tools/opt/Debugify.cpp
+++ b/tools/opt/Debugify.cpp
@@ -1,9 +1,8 @@
 //===- Debugify.cpp - Attach synthetic debug info to everything -----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
diff --git a/tools/opt/Debugify.h b/tools/opt/Debugify.h
index d1a60c73e7236081102696735cc7e62512320566..266f577951aedd3291b875cb5511c28cfe45cc8c 100644
--- a/tools/opt/Debugify.h
+++ b/tools/opt/Debugify.h
@@ -1,9 +1,8 @@
 //===- Debugify.h - Attach synthetic debug info to everything -------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
diff --git a/tools/opt/GraphPrinters.cpp b/tools/opt/GraphPrinters.cpp
index a8bb12f3e0184a1c74655c90bcb8b584e096b0ad..611fb20513c96ce486a7cbb965be8ee7807974c9 100644
--- a/tools/opt/GraphPrinters.cpp
+++ b/tools/opt/GraphPrinters.cpp
@@ -1,9 +1,8 @@
 //===- GraphPrinters.cpp - DOT printers for various graph types -----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/tools/opt/LLVMBuild.txt b/tools/opt/LLVMBuild.txt
index 047719042de96484d1fc436e3676e8d55ee1b6fb..70d44f29867540361d3e0a8c31c73f7fd04a10fc 100644
--- a/tools/opt/LLVMBuild.txt
+++ b/tools/opt/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./tools/opt/LLVMBuild.txt --------------------------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/tools/opt/NewPMDriver.cpp b/tools/opt/NewPMDriver.cpp
index 211a3b151fe1ee4830a9bb35b67c62a35820ac1b..e032c40a67ed2660cb62c014314021a83aec0224 100644
--- a/tools/opt/NewPMDriver.cpp
+++ b/tools/opt/NewPMDriver.cpp
@@ -1,9 +1,8 @@
 //===- NewPMDriver.cpp - Driver for opt with new PM -----------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
@@ -101,19 +100,8 @@ static cl::opt<std::string> OptimizerLastEPPipeline(
              "the OptimizerLast extension point into default pipelines"),
     cl::Hidden);
 
-enum PGOKind { NoPGO, InstrGen, InstrUse, SampleUse };
-static cl::opt<PGOKind> PGOKindFlag(
-    "pgo-kind", cl::init(NoPGO), cl::Hidden,
-    cl::desc("The kind of profile guided optimization"),
-    cl::values(clEnumValN(NoPGO, "nopgo", "Do not use PGO."),
-               clEnumValN(InstrGen, "new-pm-pgo-instr-gen-pipeline",
-                          "Instrument the IR to generate profile."),
-               clEnumValN(InstrUse, "new-pm-pgo-instr-use-pipeline",
-                          "Use instrumented profile to guide PGO."),
-               clEnumValN(SampleUse, "new-pm-pgo-sample-use-pipeline",
-                          "Use sampled profile to guide PGO.")));
-static cl::opt<std::string> ProfileFile(
-    "profile-file", cl::desc("Path to the profile."), cl::Hidden);
+extern cl::opt<PGOKind> PGOKindFlag;
+extern cl::opt<std::string> ProfileFile;
 static cl::opt<std::string>
     ProfileRemappingFile("profile-remapping-file",
                          cl::desc("Path to the profile remapping file."),
diff --git a/tools/opt/NewPMDriver.h b/tools/opt/NewPMDriver.h
index 7d74a5777d11566ae22903e6975ff9730501f6cc..35c2886fecbfcaaccbf254e4e82eab3c0fafd7ad 100644
--- a/tools/opt/NewPMDriver.h
+++ b/tools/opt/NewPMDriver.h
@@ -1,9 +1,8 @@
 //===- NewPMDriver.h - Function to drive opt with the new PM ----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
@@ -40,6 +39,12 @@ enum VerifierKind {
   VK_VerifyInAndOut,
   VK_VerifyEachPass
 };
+enum PGOKind {
+  NoPGO,
+  InstrGen,
+  InstrUse,
+  SampleUse
+};
 }
 
 /// Driver function to run the new pass manager over a module.
diff --git a/tools/opt/PassPrinters.cpp b/tools/opt/PassPrinters.cpp
index 310d491c06a520e8a8b1a73665b5b15c33c77dce..70da6a43f8d997366473b12c3b6ba64f4d6b2fcb 100644
--- a/tools/opt/PassPrinters.cpp
+++ b/tools/opt/PassPrinters.cpp
@@ -1,9 +1,8 @@
 //===- PassPrinters.cpp - Utilities to print analysis info for passes -----===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
diff --git a/tools/opt/PassPrinters.h b/tools/opt/PassPrinters.h
index e66f3f457b7ace49b1fe2de30d896ccc2948f3cc..d4e7a4a97f31fe31730765cb317dcf9f82d05232 100644
--- a/tools/opt/PassPrinters.h
+++ b/tools/opt/PassPrinters.h
@@ -1,9 +1,8 @@
 //=- PassPrinters.h - Utilities to print analysis info for passes -*- C++ -*-=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
diff --git a/tools/opt/PrintSCC.cpp b/tools/opt/PrintSCC.cpp
index 78ede2b72f84905dee1cb8dea29a9faeec7f80f8..419886d6cc60aa52a720543de3c9a7630f442f0a 100644
--- a/tools/opt/PrintSCC.cpp
+++ b/tools/opt/PrintSCC.cpp
@@ -1,9 +1,8 @@
 //===- PrintSCC.cpp - Enumerate SCCs in some key graphs -------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/tools/opt/opt.cpp b/tools/opt/opt.cpp
index 266603992aa905ebbb961445d78e0cee6d635d5c..4afe7508fc252169c290d33e7113a5104d2989b8 100644
--- a/tools/opt/opt.cpp
+++ b/tools/opt/opt.cpp
@@ -1,9 +1,8 @@
 //===- opt.cpp - The LLVM Modular Optimizer -------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -103,6 +102,10 @@ static cl::opt<bool>
     OutputThinLTOBC("thinlto-bc",
                     cl::desc("Write output as ThinLTO-ready bitcode"));
 
+static cl::opt<bool>
+    SplitLTOUnit("thinlto-split-lto-unit",
+                 cl::desc("Enable splitting of a ThinLTO LTOUnit"));
+
 static cl::opt<std::string> ThinLinkBitcodeFile(
     "thin-link-bitcode-file", cl::value_desc("filename"),
     cl::desc(
@@ -271,6 +274,19 @@ static cl::opt<std::string>
                     cl::desc("YAML output filename for pass remarks"),
                     cl::value_desc("filename"));
 
+cl::opt<PGOKind>
+    PGOKindFlag("pgo-kind", cl::init(NoPGO), cl::Hidden,
+                cl::desc("The kind of profile guided optimization"),
+                cl::values(clEnumValN(NoPGO, "nopgo", "Do not use PGO."),
+                           clEnumValN(InstrGen, "pgo-instr-gen-pipeline",
+                                      "Instrument the IR to generate profile."),
+                           clEnumValN(InstrUse, "pgo-instr-use-pipeline",
+                                      "Use instrumented profile to guide PGO."),
+                           clEnumValN(SampleUse, "pgo-sample-use-pipeline",
+                                      "Use sampled profile to guide PGO.")));
+cl::opt<std::string> ProfileFile("profile-file",
+                                 cl::desc("Path to the profile."), cl::Hidden);
+
 class OptCustomPassManager : public legacy::PassManager {
   DebugifyStatsMap DIStatsMap;
 
@@ -365,6 +381,21 @@ static void AddOptimizationPasses(legacy::PassManagerBase &MPM,
   if (Coroutines)
     addCoroutinePassesToExtensionPoints(Builder);
 
+  switch (PGOKindFlag) {
+  case InstrGen:
+    Builder.EnablePGOInstrGen = true;
+    Builder.PGOInstrGen = ProfileFile;
+    break;
+  case InstrUse:
+    Builder.PGOInstrUse = ProfileFile;
+    break;
+  case SampleUse:
+    Builder.PGOSampleUse = ProfileFile;
+    break;
+  default:
+    break;
+  }
+
   Builder.populateFunctionPassManager(FPM);
   Builder.populateModulePassManager(MPM);
 }
@@ -596,6 +627,9 @@ int main(int argc, char **argv) {
     if (CheckBitcodeOutputToConsole(Out->os(), !Quiet))
       NoOutput = true;
 
+  if (OutputThinLTOBC)
+    M->addModuleFlag(Module::Error, "EnableSplitLTOUnit", SplitLTOUnit);
+
   if (PassPipeline.getNumOccurrences() > 0) {
     OutputKind OK = OK_NoOutput;
     if (!NoOutput)
diff --git a/tools/sancov/coverage-report-server.py b/tools/sancov/coverage-report-server.py
index a2e161d0de58b43a580e9609ef3e4de9aae2a164..251d8f1b77baca358c17d608b33f7d33ac5471c2 100755
--- a/tools/sancov/coverage-report-server.py
+++ b/tools/sancov/coverage-report-server.py
@@ -1,10 +1,9 @@
 #!/usr/bin/env python3
 #===- symcov-report-server.py - Coverage Reports HTTP Serve --*- python -*--===#
 #
-#                     The LLVM Compiler Infrastructure
-#
-# This file is distributed under the University of Illinois Open Source
-# License. See LICENSE.TXT for details.
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 #
 #===------------------------------------------------------------------------===#
 '''(EXPERIMENTAL) HTTP server to browse coverage reports from .symcov files.
@@ -22,6 +21,8 @@ Other options:
     --host host_name - host name to bind server to (127.0.0.1)
 '''
 
+from __future__ import print_function
+
 import argparse
 import http.server
 import json
diff --git a/tools/sancov/sancov.cpp b/tools/sancov/sancov.cpp
index e8935d1be21251da146d1a02de444665bb7e4876..ea8240fa3c0a6ed968339ea8d70704000381114f 100644
--- a/tools/sancov/sancov.cpp
+++ b/tools/sancov/sancov.cpp
@@ -1,9 +1,8 @@
 //===-- sancov.cpp --------------------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 // This file is a command-line tool for reading and analyzing sanitizer
diff --git a/tools/sanstats/sanstats.cpp b/tools/sanstats/sanstats.cpp
index a345e9f06bedd8b5e98b883424725f3ca320c0cc..0b4b2b8196bbad155ee4d9f65caa432c127e2d5c 100644
--- a/tools/sanstats/sanstats.cpp
+++ b/tools/sanstats/sanstats.cpp
@@ -1,9 +1,8 @@
 //===- sanstats.cpp - Sanitizer statistics dumper -------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/tools/verify-uselistorder/LLVMBuild.txt b/tools/verify-uselistorder/LLVMBuild.txt
index 23957c189c8a5bee545b42cc716d037bbccb1c50..7f802939081134e1284945610ab79fa41108649d 100644
--- a/tools/verify-uselistorder/LLVMBuild.txt
+++ b/tools/verify-uselistorder/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./tools/verify-uselistorder/LLVMBuild.txt ----------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/tools/verify-uselistorder/verify-uselistorder.cpp b/tools/verify-uselistorder/verify-uselistorder.cpp
index 99c7007e4e990cc75562b529fa376a6b39869fd2..527ba37bcc57738f47e9ff49778471c47ee894a8 100644
--- a/tools/verify-uselistorder/verify-uselistorder.cpp
+++ b/tools/verify-uselistorder/verify-uselistorder.cpp
@@ -1,9 +1,8 @@
 //===- verify-uselistorder.cpp - The LLVM Modular Optimizer ---------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/tools/yaml2obj/yaml2coff.cpp b/tools/yaml2obj/yaml2coff.cpp
index f9d255e1ad20ee09a9a9667c9a593e8ad1587e2e..223ade065e1c5fc5b9ce790c47b379996b6e46dd 100644
--- a/tools/yaml2obj/yaml2coff.cpp
+++ b/tools/yaml2obj/yaml2coff.cpp
@@ -1,9 +1,8 @@
 //===- yaml2coff - Convert YAML to a COFF object file ---------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
@@ -24,6 +23,7 @@
 #include "llvm/Support/Endian.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/SourceMgr.h"
+#include "llvm/Support/WithColor.h"
 #include "llvm/Support/raw_ostream.h"
 #include <vector>
 
@@ -520,7 +520,15 @@ static bool writeCOFF(COFFParser &CP, raw_ostream &OS) {
     assert(S.Header.SizeOfRawData >= S.SectionData.binary_size());
     OS << num_zeros(S.Header.SizeOfRawData - S.SectionData.binary_size());
     for (const COFFYAML::Relocation &R : S.Relocations) {
-      uint32_t SymbolTableIndex = SymbolTableIndexMap[R.SymbolName];
+      uint32_t SymbolTableIndex;
+      if (R.SymbolTableIndex) {
+        if (!R.SymbolName.empty())
+          WithColor::error()
+              << "Both SymbolName and SymbolTableIndex specified\n";
+        SymbolTableIndex = *R.SymbolTableIndex;
+      } else {
+        SymbolTableIndex = SymbolTableIndexMap[R.SymbolName];
+      }
       OS << binary_le(R.VirtualAddress)
          << binary_le(SymbolTableIndex)
          << binary_le(R.Type);
diff --git a/tools/yaml2obj/yaml2elf.cpp b/tools/yaml2obj/yaml2elf.cpp
index 672d1540daf058b8f07fbcadf40630d7d34a80df..5198a751cfbd4df54642661c17814606d14c924c 100644
--- a/tools/yaml2obj/yaml2elf.cpp
+++ b/tools/yaml2obj/yaml2elf.cpp
@@ -1,9 +1,8 @@
 //===- yaml2elf - Convert YAML to a ELF object file -----------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
@@ -160,6 +159,9 @@ class ELFState {
   bool writeSectionContent(Elf_Shdr &SHeader,
                            const ELFYAML::MipsABIFlags &Section,
                            ContiguousBlobAccumulator &CBA);
+  void writeSectionContent(Elf_Shdr &SHeader,
+                           const ELFYAML::DynamicSection &Section,
+                           ContiguousBlobAccumulator &CBA);
   bool hasDynamicSymbols() const;
   SmallVector<const char *, 5> implicitSectionNames() const;
 
@@ -196,7 +198,7 @@ void ELFState<ELFT>::initELFHeader(Elf_Ehdr &Header) {
   Header.e_ident[EI_DATA] = IsLittleEndian ? ELFDATA2LSB : ELFDATA2MSB;
   Header.e_ident[EI_VERSION] = EV_CURRENT;
   Header.e_ident[EI_OSABI] = Doc.Header.OSABI;
-  Header.e_ident[EI_ABIVERSION] = 0;
+  Header.e_ident[EI_ABIVERSION] = Doc.Header.ABIVersion;
   Header.e_type = Doc.Header.Type;
   Header.e_machine = Doc.Header.Machine;
   Header.e_version = EV_CURRENT;
@@ -268,15 +270,16 @@ bool ELFState<ELFT>::initSectionHeaders(std::vector<Elf_Shdr> &SHeaders,
         SHeader.sh_link = getDotSymTabSecNo();
 
       unsigned Index;
-      if (!convertSectionIndex(SN2I, S->Name, S->Info, Index))
+      if (!convertSectionIndex(SN2I, S->Name, S->RelocatableSec, Index))
         return false;
       SHeader.sh_info = Index;
       if (!writeSectionContent(SHeader, *S, CBA))
         return false;
     } else if (auto S = dyn_cast<ELFYAML::Group>(Sec.get())) {
       unsigned SymIdx;
-      if (SymN2I.lookup(S->Info, SymIdx) && !to_integer(S->Info, SymIdx)) {
-        WithColor::error() << "Unknown symbol referenced: '" << S->Info
+      if (SymN2I.lookup(S->Signature, SymIdx) &&
+          !to_integer(S->Signature, SymIdx)) {
+        WithColor::error() << "Unknown symbol referenced: '" << S->Signature
                            << "' at YAML section '" << S->Name << "'.\n";
         return false;
       }
@@ -292,6 +295,8 @@ bool ELFState<ELFT>::initSectionHeaders(std::vector<Elf_Shdr> &SHeaders,
       // SHT_NOBITS section does not have content
       // so just to setup the section offset.
       CBA.getOSAndAlignedOffset(SHeader.sh_offset, SHeader.sh_addralign);
+    } else if (auto S = dyn_cast<ELFYAML::DynamicSection>(Sec.get())) {
+      writeSectionContent(SHeader, *S, CBA);
     } else
       llvm_unreachable("Unknown section type");
 
@@ -466,8 +471,6 @@ ELFState<ELFT>::writeSectionContent(Elf_Shdr &SHeader,
     SHeader.sh_entsize = *Section.EntSize;
   else if (Section.Type == llvm::ELF::SHT_RELR)
     SHeader.sh_entsize = sizeof(Elf_Relr);
-  else if (Section.Type == llvm::ELF::SHT_DYNAMIC)
-    SHeader.sh_entsize = sizeof(Elf_Dyn);
   else
     SHeader.sh_entsize = 0;
   SHeader.sh_size = Section.Size;
@@ -576,6 +579,29 @@ bool ELFState<ELFT>::writeSectionContent(Elf_Shdr &SHeader,
   return true;
 }
 
+template <class ELFT>
+void ELFState<ELFT>::writeSectionContent(Elf_Shdr &SHeader,
+                                         const ELFYAML::DynamicSection &Section,
+                                         ContiguousBlobAccumulator &CBA) {
+  typedef typename ELFT::Addr Elf_Addr;
+  assert(Section.Type == llvm::ELF::SHT_DYNAMIC &&
+         "Section type is not SHT_DYNAMIC");
+
+  SHeader.sh_size = 2 * sizeof(Elf_Addr) * Section.Entries.size();
+  if (Section.EntSize)
+    SHeader.sh_entsize = *Section.EntSize;
+  else
+    SHeader.sh_entsize = sizeof(Elf_Dyn);
+
+  auto &OS = CBA.getOSAndAlignedOffset(SHeader.sh_offset, SHeader.sh_addralign);
+  for (const ELFYAML::DynamicEntry &DE : Section.Entries) {
+    Elf_Addr Tag = (Elf_Addr)DE.Tag;
+    OS.write((const char *)&Tag, sizeof(Elf_Addr));
+    Elf_Addr Val = (Elf_Addr)DE.Val;
+    OS.write((const char *)&Val, sizeof(Elf_Addr));
+  }
+}
+
 template <class ELFT> bool ELFState<ELFT>::buildSectionIndex() {
   for (unsigned i = 0, e = Doc.Sections.size(); i != e; ++i) {
     StringRef Name = Doc.Sections[i]->Name;
diff --git a/tools/yaml2obj/yaml2macho.cpp b/tools/yaml2obj/yaml2macho.cpp
index 1f36431c05de0421adbb0cf67195915392434ab5..2df03fd390128a950661adc31b27ce2c6a2e1644 100644
--- a/tools/yaml2obj/yaml2macho.cpp
+++ b/tools/yaml2obj/yaml2macho.cpp
@@ -1,9 +1,8 @@
 //===- yaml2macho - Convert YAML to a Mach object file --------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
diff --git a/tools/yaml2obj/yaml2obj.cpp b/tools/yaml2obj/yaml2obj.cpp
index 949daffa9e8c3c7209cb1e6573f9b84c2deb6018..d96f7a68c0d1b54a32689e5f78299c6b9e2b8ddf 100644
--- a/tools/yaml2obj/yaml2obj.cpp
+++ b/tools/yaml2obj/yaml2obj.cpp
@@ -1,9 +1,8 @@
 //===- yaml2obj - Convert YAML to a binary object file --------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/tools/yaml2obj/yaml2obj.h b/tools/yaml2obj/yaml2obj.h
index fc784067a9f0506ff8ff941b8954e6bdbddd38db..40ed9e7782db27501d558afdd88aa4899ef91659 100644
--- a/tools/yaml2obj/yaml2obj.h
+++ b/tools/yaml2obj/yaml2obj.h
@@ -1,9 +1,8 @@
 //===--- yaml2obj.h - -------------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/tools/yaml2obj/yaml2wasm.cpp b/tools/yaml2obj/yaml2wasm.cpp
index e8d998d991468e91661db0dca93bc88174aa5c38..3398ace3e1e5ed50388d026bb24a27629f8ec7eb 100644
--- a/tools/yaml2obj/yaml2wasm.cpp
+++ b/tools/yaml2obj/yaml2wasm.cpp
@@ -1,9 +1,8 @@
 //===- yaml2wasm - Convert YAML to a Wasm object file --------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
@@ -49,6 +48,7 @@ private:
   int writeSectionContent(raw_ostream &OS, WasmYAML::DylinkSection &Section);
   int writeSectionContent(raw_ostream &OS, WasmYAML::NameSection &Section);
   int writeSectionContent(raw_ostream &OS, WasmYAML::LinkingSection &Section);
+  int writeSectionContent(raw_ostream &OS, WasmYAML::ProducersSection &Section);
   WasmYAML::Object &Obj;
   uint32_t NumImportedFunctions = 0;
   uint32_t NumImportedGlobals = 0;
@@ -105,7 +105,7 @@ static int writeInitExpr(const wasm::WasmInitExpr &InitExpr, raw_ostream &OS) {
   case wasm::WASM_OPCODE_F64_CONST:
     writeUint64(OS, InitExpr.Value.Float64);
     break;
-  case wasm::WASM_OPCODE_GET_GLOBAL:
+  case wasm::WASM_OPCODE_GLOBAL_GET:
     encodeULEB128(InitExpr.Value.Global, OS);
     break;
   default:
@@ -124,14 +124,14 @@ class SubSectionWriter {
 public:
   SubSectionWriter(raw_ostream &OS) : OS(OS), StringStream(OutString) {}
 
-  void Done() {
+  void done() {
     StringStream.flush();
     encodeULEB128(OutString.size(), OS);
     OS << OutString;
     OutString.clear();
   }
 
-  raw_ostream &GetStream() { return StringStream; }
+  raw_ostream &getStream() { return StringStream; }
 };
 
 int WasmWriter::writeSectionContent(raw_ostream &OS,
@@ -159,78 +159,79 @@ int WasmWriter::writeSectionContent(raw_ostream &OS,
   if (Section.SymbolTable.size()) {
     writeUint8(OS, wasm::WASM_SYMBOL_TABLE);
 
-    encodeULEB128(Section.SymbolTable.size(), SubSection.GetStream());
+    encodeULEB128(Section.SymbolTable.size(), SubSection.getStream());
 #ifndef NDEBUG
     uint32_t SymbolIndex = 0;
 #endif
     for (const WasmYAML::SymbolInfo &Info : Section.SymbolTable) {
       assert(Info.Index == SymbolIndex++);
-      writeUint8(SubSection.GetStream(), Info.Kind);
-      encodeULEB128(Info.Flags, SubSection.GetStream());
+      writeUint8(SubSection.getStream(), Info.Kind);
+      encodeULEB128(Info.Flags, SubSection.getStream());
       switch (Info.Kind) {
       case wasm::WASM_SYMBOL_TYPE_FUNCTION:
       case wasm::WASM_SYMBOL_TYPE_GLOBAL:
       case wasm::WASM_SYMBOL_TYPE_EVENT:
-        encodeULEB128(Info.ElementIndex, SubSection.GetStream());
-        if ((Info.Flags & wasm::WASM_SYMBOL_UNDEFINED) == 0)
-          writeStringRef(Info.Name, SubSection.GetStream());
+        encodeULEB128(Info.ElementIndex, SubSection.getStream());
+        if ((Info.Flags & wasm::WASM_SYMBOL_UNDEFINED) == 0 ||
+            (Info.Flags & wasm::WASM_SYMBOL_EXPLICIT_NAME) != 0)
+          writeStringRef(Info.Name, SubSection.getStream());
         break;
       case wasm::WASM_SYMBOL_TYPE_DATA:
-        writeStringRef(Info.Name, SubSection.GetStream());
+        writeStringRef(Info.Name, SubSection.getStream());
         if ((Info.Flags & wasm::WASM_SYMBOL_UNDEFINED) == 0) {
-          encodeULEB128(Info.DataRef.Segment, SubSection.GetStream());
-          encodeULEB128(Info.DataRef.Offset, SubSection.GetStream());
-          encodeULEB128(Info.DataRef.Size, SubSection.GetStream());
+          encodeULEB128(Info.DataRef.Segment, SubSection.getStream());
+          encodeULEB128(Info.DataRef.Offset, SubSection.getStream());
+          encodeULEB128(Info.DataRef.Size, SubSection.getStream());
         }
         break;
       case wasm::WASM_SYMBOL_TYPE_SECTION:
-        encodeULEB128(Info.ElementIndex, SubSection.GetStream());
+        encodeULEB128(Info.ElementIndex, SubSection.getStream());
         break;
       default:
         llvm_unreachable("unexpected kind");
       }
     }
 
-    SubSection.Done();
+    SubSection.done();
   }
 
   // SEGMENT_NAMES subsection
   if (Section.SegmentInfos.size()) {
     writeUint8(OS, wasm::WASM_SEGMENT_INFO);
-    encodeULEB128(Section.SegmentInfos.size(), SubSection.GetStream());
+    encodeULEB128(Section.SegmentInfos.size(), SubSection.getStream());
     for (const WasmYAML::SegmentInfo &SegmentInfo : Section.SegmentInfos) {
-      writeStringRef(SegmentInfo.Name, SubSection.GetStream());
-      encodeULEB128(SegmentInfo.Alignment, SubSection.GetStream());
-      encodeULEB128(SegmentInfo.Flags, SubSection.GetStream());
+      writeStringRef(SegmentInfo.Name, SubSection.getStream());
+      encodeULEB128(SegmentInfo.Alignment, SubSection.getStream());
+      encodeULEB128(SegmentInfo.Flags, SubSection.getStream());
     }
-    SubSection.Done();
+    SubSection.done();
   }
 
   // INIT_FUNCS subsection
   if (Section.InitFunctions.size()) {
     writeUint8(OS, wasm::WASM_INIT_FUNCS);
-    encodeULEB128(Section.InitFunctions.size(), SubSection.GetStream());
+    encodeULEB128(Section.InitFunctions.size(), SubSection.getStream());
     for (const WasmYAML::InitFunction &Func : Section.InitFunctions) {
-      encodeULEB128(Func.Priority, SubSection.GetStream());
-      encodeULEB128(Func.Symbol, SubSection.GetStream());
+      encodeULEB128(Func.Priority, SubSection.getStream());
+      encodeULEB128(Func.Symbol, SubSection.getStream());
     }
-    SubSection.Done();
+    SubSection.done();
   }
 
   // COMDAT_INFO subsection
   if (Section.Comdats.size()) {
     writeUint8(OS, wasm::WASM_COMDAT_INFO);
-    encodeULEB128(Section.Comdats.size(), SubSection.GetStream());
+    encodeULEB128(Section.Comdats.size(), SubSection.getStream());
     for (const auto &C : Section.Comdats) {
-      writeStringRef(C.Name, SubSection.GetStream());
-      encodeULEB128(0, SubSection.GetStream()); // flags for future use
-      encodeULEB128(C.Entries.size(), SubSection.GetStream());
+      writeStringRef(C.Name, SubSection.getStream());
+      encodeULEB128(0, SubSection.getStream()); // flags for future use
+      encodeULEB128(C.Entries.size(), SubSection.getStream());
       for (const WasmYAML::ComdatEntry &Entry : C.Entries) {
-        writeUint8(SubSection.GetStream(), Entry.Kind);
-        encodeULEB128(Entry.Index, SubSection.GetStream());
+        writeUint8(SubSection.getStream(), Entry.Kind);
+        encodeULEB128(Entry.Index, SubSection.getStream());
       }
     }
-    SubSection.Done();
+    SubSection.done();
   }
 
   return 0;
@@ -244,13 +245,36 @@ int WasmWriter::writeSectionContent(raw_ostream &OS,
 
     SubSectionWriter SubSection(OS);
 
-    encodeULEB128(Section.FunctionNames.size(), SubSection.GetStream());
+    encodeULEB128(Section.FunctionNames.size(), SubSection.getStream());
     for (const WasmYAML::NameEntry &NameEntry : Section.FunctionNames) {
-      encodeULEB128(NameEntry.Index, SubSection.GetStream());
-      writeStringRef(NameEntry.Name, SubSection.GetStream());
+      encodeULEB128(NameEntry.Index, SubSection.getStream());
+      writeStringRef(NameEntry.Name, SubSection.getStream());
     }
 
-    SubSection.Done();
+    SubSection.done();
+  }
+  return 0;
+}
+
+int WasmWriter::writeSectionContent(raw_ostream &OS,
+                                    WasmYAML::ProducersSection &Section) {
+  writeStringRef(Section.Name, OS);
+  int Fields = int(!Section.Languages.empty()) + int(!Section.Tools.empty()) +
+               int(!Section.SDKs.empty());
+  if (Fields == 0)
+    return 0;
+  encodeULEB128(Fields, OS);
+  for (auto &Field : {std::make_pair(StringRef("language"), &Section.Languages),
+                      std::make_pair(StringRef("processed-by"), &Section.Tools),
+                      std::make_pair(StringRef("sdk"), &Section.SDKs)}) {
+    if (Field.second->empty())
+      continue;
+    writeStringRef(Field.first, OS);
+    encodeULEB128(Field.second->size(), OS);
+    for (auto &Entry : *Field.second) {
+      writeStringRef(Entry.Name, OS);
+      writeStringRef(Entry.Version, OS);
+    }
   }
   return 0;
 }
@@ -266,6 +290,9 @@ int WasmWriter::writeSectionContent(raw_ostream &OS,
   } else if (auto S = dyn_cast<WasmYAML::LinkingSection>(&Section)) {
     if (auto Err = writeSectionContent(OS, *S))
       return Err;
+  } else if (auto S = dyn_cast<WasmYAML::ProducersSection>(&Section)) {
+    if (auto Err = writeSectionContent(OS, *S))
+      return Err;
   } else {
     writeStringRef(Section.Name, OS);
     Section.Payload.writeAsBinary(OS);
@@ -500,11 +527,11 @@ int WasmWriter::writeRelocSection(raw_ostream &OS, WasmYAML::Section &Sec,
     encodeULEB128(Reloc.Offset, OS);
     encodeULEB128(Reloc.Index, OS);
     switch (Reloc.Type) {
-    case wasm::R_WEBASSEMBLY_MEMORY_ADDR_LEB:
-    case wasm::R_WEBASSEMBLY_MEMORY_ADDR_SLEB:
-    case wasm::R_WEBASSEMBLY_MEMORY_ADDR_I32:
-    case wasm::R_WEBASSEMBLY_FUNCTION_OFFSET_I32:
-    case wasm::R_WEBASSEMBLY_SECTION_OFFSET_I32:
+    case wasm::R_WASM_MEMORY_ADDR_LEB:
+    case wasm::R_WASM_MEMORY_ADDR_SLEB:
+    case wasm::R_WASM_MEMORY_ADDR_I32:
+    case wasm::R_WASM_FUNCTION_OFFSET_I32:
+    case wasm::R_WASM_SECTION_OFFSET_I32:
       encodeULEB128(Reloc.Addend, OS);
     }
   }
diff --git a/unittests/ADT/APFloatTest.cpp b/unittests/ADT/APFloatTest.cpp
index 64053a811a35f6ebee2897dbbfdfa38a7367d269..fb3607772f0e64983ebe6957d2392cba10cffbc5 100644
--- a/unittests/ADT/APFloatTest.cpp
+++ b/unittests/ADT/APFloatTest.cpp
@@ -1,9 +1,8 @@
 //===- llvm/unittest/ADT/APFloat.cpp - APFloat unit tests ---------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/ADT/APIntTest.cpp b/unittests/ADT/APIntTest.cpp
index 8b876f3369576dfd94749acb9cb1df6b3157a87b..6ef5b25f8d48ea6c0cd5c67957d558aabf646b59 100644
--- a/unittests/ADT/APIntTest.cpp
+++ b/unittests/ADT/APIntTest.cpp
@@ -1,9 +1,8 @@
 //===- llvm/unittest/ADT/APInt.cpp - APInt unit tests ---------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/ADT/APSIntTest.cpp b/unittests/ADT/APSIntTest.cpp
index a9b3071d13883775dc5b4bd78eafa5edcfd30175..c76a32051d87caa94eaee98e2a3c35a83074b301 100644
--- a/unittests/ADT/APSIntTest.cpp
+++ b/unittests/ADT/APSIntTest.cpp
@@ -1,9 +1,8 @@
 //===- llvm/unittest/ADT/APSIntTest.cpp - APSInt unit tests ---------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/ADT/AnyTest.cpp b/unittests/ADT/AnyTest.cpp
index 658f6b6ce3b7241af11dd14664b9563525a982b5..2ca81bd61d571580dab8949344b00586ca1b0345 100644
--- a/unittests/ADT/AnyTest.cpp
+++ b/unittests/ADT/AnyTest.cpp
@@ -1,9 +1,8 @@
 //===- llvm/unittest/Support/AnyTest.cpp - Any tests ---===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/ADT/ArrayRefTest.cpp b/unittests/ADT/ArrayRefTest.cpp
index e01d212f218bfba33fa7661d9db61923c4d98287..a5609f89ff95bd3ba647af3f9a6c5142dca6b205 100644
--- a/unittests/ADT/ArrayRefTest.cpp
+++ b/unittests/ADT/ArrayRefTest.cpp
@@ -1,9 +1,8 @@
 //===- llvm/unittest/ADT/ArrayRefTest.cpp - ArrayRef unit tests -----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -249,4 +248,7 @@ TEST(ArrayRefTest, makeArrayRef) {
   EXPECT_TRUE(AR2.equals(AR2Ref));
 }
 
+static_assert(is_trivially_copyable<ArrayRef<int>>::value,
+              "trivially copyable");
+
 } // end anonymous namespace
diff --git a/unittests/ADT/BitVectorTest.cpp b/unittests/ADT/BitVectorTest.cpp
index 84290b3b854d3e8694f0c87b1c46764433ac2aa3..559dcc579d1d83937cf1024f684daac0e40a6977 100644
--- a/unittests/ADT/BitVectorTest.cpp
+++ b/unittests/ADT/BitVectorTest.cpp
@@ -1,9 +1,8 @@
 //===- llvm/unittest/ADT/BitVectorTest.cpp - BitVector tests --------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/ADT/BitmaskEnumTest.cpp b/unittests/ADT/BitmaskEnumTest.cpp
index 77635c638e9360388cd6414550d9d772dda52057..f07e20330aec28b24ec79bf5f9c86aaee18ecd0d 100644
--- a/unittests/ADT/BitmaskEnumTest.cpp
+++ b/unittests/ADT/BitmaskEnumTest.cpp
@@ -1,9 +1,8 @@
 //===- llvm/unittest/ADT/BitmaskEnumTest.cpp - BitmaskEnum unit tests -----===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/ADT/BreadthFirstIteratorTest.cpp b/unittests/ADT/BreadthFirstIteratorTest.cpp
index 42a07bbe930ba380170a613fc3cd6e0a4573bcf2..5d034ad41fc7c997aa169237250d859ee8673c48 100644
--- a/unittests/ADT/BreadthFirstIteratorTest.cpp
+++ b/unittests/ADT/BreadthFirstIteratorTest.cpp
@@ -1,9 +1,8 @@
 //=== llvm/unittest/ADT/BreadthFirstIteratorTest.cpp - BFS iterator tests -===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/ADT/BumpPtrListTest.cpp b/unittests/ADT/BumpPtrListTest.cpp
index be34a71373ce2c31022fc08317180d2a8deb8310..ebd0ce2532f4fb98c95839bc2e1076955bd4703d 100644
--- a/unittests/ADT/BumpPtrListTest.cpp
+++ b/unittests/ADT/BumpPtrListTest.cpp
@@ -1,9 +1,8 @@
 //===- unittests/ADT/BumpPtrListTest.cpp - BumpPtrList unit tests ---------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/ADT/CMakeLists.txt b/unittests/ADT/CMakeLists.txt
index 098b6b67416b5080fd4a61bcd1bd300401796c93..d2a35273389fa18805635b34cb42063c00ebaf93 100644
--- a/unittests/ADT/CMakeLists.txt
+++ b/unittests/ADT/CMakeLists.txt
@@ -18,6 +18,7 @@ add_llvm_unittest(ADTTests
   DenseSetTest.cpp
   DepthFirstIteratorTest.cpp
   EquivalenceClassesTest.cpp
+  FallibleIteratorTest.cpp
   FoldingSet.cpp
   FunctionExtrasTest.cpp
   FunctionRefTest.cpp
@@ -71,4 +72,6 @@ add_llvm_unittest(ADTTests
   VariadicFunctionTest.cpp
   )
 
+target_link_libraries(ADTTests PRIVATE LLVMTestingSupport)
+
 add_dependencies(ADTTests intrinsics_gen)
diff --git a/unittests/ADT/DAGDeltaAlgorithmTest.cpp b/unittests/ADT/DAGDeltaAlgorithmTest.cpp
index 030fadbc9a3820201a19c9ea3699f4075eb47366..66a67d96d15323fd44a3334ba3106da7d3bee108 100644
--- a/unittests/ADT/DAGDeltaAlgorithmTest.cpp
+++ b/unittests/ADT/DAGDeltaAlgorithmTest.cpp
@@ -1,9 +1,8 @@
 //===- llvm/unittest/ADT/DAGDeltaAlgorithmTest.cpp ------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/ADT/DeltaAlgorithmTest.cpp b/unittests/ADT/DeltaAlgorithmTest.cpp
index 01dc1f36bd2ff1e69004fa999c8ef2b6c76ff814..5e284129180a060b5e65fc8cc8b7adb9ae01bb00 100644
--- a/unittests/ADT/DeltaAlgorithmTest.cpp
+++ b/unittests/ADT/DeltaAlgorithmTest.cpp
@@ -1,9 +1,8 @@
 //===- llvm/unittest/ADT/DeltaAlgorithmTest.cpp ---------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/ADT/DenseMapTest.cpp b/unittests/ADT/DenseMapTest.cpp
index ee9c5dd38000d0fa4759c188a39c1cede191a8e8..383c2a59a24b6e4c5c4179580fa360f7be7d7ec8 100644
--- a/unittests/ADT/DenseMapTest.cpp
+++ b/unittests/ADT/DenseMapTest.cpp
@@ -1,9 +1,8 @@
 //===- llvm/unittest/ADT/DenseMapMap.cpp - DenseMap unit tests --*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/ADT/DenseSetTest.cpp b/unittests/ADT/DenseSetTest.cpp
index 7368e2ed0e019d743fce2025deda00d722126bb3..e6fe9ca3b318f08c327250fc728a321e8ee17180 100644
--- a/unittests/ADT/DenseSetTest.cpp
+++ b/unittests/ADT/DenseSetTest.cpp
@@ -1,9 +1,8 @@
 //===- llvm/unittest/ADT/DenseSetTest.cpp - DenseSet unit tests --*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/ADT/DepthFirstIteratorTest.cpp b/unittests/ADT/DepthFirstIteratorTest.cpp
index 4169cd48fcbd13c6dc6a7c221da3d99595549eb0..0e3e0e66d71e0083103e27ad4d0e71d9d2127227 100644
--- a/unittests/ADT/DepthFirstIteratorTest.cpp
+++ b/unittests/ADT/DepthFirstIteratorTest.cpp
@@ -1,9 +1,8 @@
 //=== llvm/unittest/ADT/DepthFirstIteratorTest.cpp - DFS iterator tests ---===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/ADT/EquivalenceClassesTest.cpp b/unittests/ADT/EquivalenceClassesTest.cpp
index 57d588a4c516594d7a5090d492a012df5fff20aa..30eab6eab08c9cf203aafb4e2f7a06764a0a5ff5 100644
--- a/unittests/ADT/EquivalenceClassesTest.cpp
+++ b/unittests/ADT/EquivalenceClassesTest.cpp
@@ -1,9 +1,8 @@
 //=== llvm/unittest/ADT/EquivalenceClassesTest.cpp - the structure tests --===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/ADT/FallibleIteratorTest.cpp b/unittests/ADT/FallibleIteratorTest.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..d3389744ffbfe8851e670c1a669cc6237bd7a631
--- /dev/null
+++ b/unittests/ADT/FallibleIteratorTest.cpp
@@ -0,0 +1,291 @@
+//===- unittests/ADT/FallibleIteratorTest.cpp - fallible_iterator.h tests -===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ADT/fallible_iterator.h"
+#include "llvm/Testing/Support/Error.h"
+
+#include "gtest/gtest-spi.h"
+#include "gtest/gtest.h"
+
+#include <utility>
+#include <vector>
+
+using namespace llvm;
+
+namespace {
+
+using ItemValid = enum { ValidItem, InvalidItem };
+using LinkValid = enum { ValidLink, InvalidLink };
+
+class Item {
+public:
+  Item(ItemValid V) : V(V) {}
+  bool isValid() const { return V == ValidItem; }
+
+private:
+  ItemValid V;
+};
+
+// A utility to mock "bad collections". It supports both invalid items,
+// where the dereference operator may return an Error, and bad links
+// where the inc/dec operations may return an Error.
+// Each element of the mock collection contains a pair of a (possibly broken)
+// item and link.
+using FallibleCollection = std::vector<std::pair<Item, LinkValid>>;
+
+class FallibleCollectionWalker {
+public:
+  FallibleCollectionWalker(FallibleCollection &C, unsigned Idx)
+      : C(C), Idx(Idx) {}
+
+  Item &operator*() { return C[Idx].first; }
+
+  const Item &operator*() const { return C[Idx].first; }
+
+  Error inc() {
+    assert(Idx != C.size() && "Walking off end of (mock) collection");
+    if (C[Idx].second == ValidLink) {
+      ++Idx;
+      return Error::success();
+    }
+    return make_error<StringError>("cant get next object in (mock) collection",
+                                   inconvertibleErrorCode());
+  }
+
+  Error dec() {
+    assert(Idx != 0 && "Walking off start of (mock) collection");
+    --Idx;
+    if (C[Idx].second == ValidLink)
+      return Error::success();
+    return make_error<StringError>("cant get prev object in (mock) collection",
+                                   inconvertibleErrorCode());
+  }
+
+  friend bool operator==(const FallibleCollectionWalker &LHS,
+                         const FallibleCollectionWalker &RHS) {
+    assert(&LHS.C == &RHS.C && "Comparing iterators across collectionss.");
+    return LHS.Idx == RHS.Idx;
+  }
+
+private:
+  FallibleCollection &C;
+  unsigned Idx;
+};
+
+class FallibleCollectionWalkerWithStructDeref
+    : public FallibleCollectionWalker {
+public:
+  using FallibleCollectionWalker::FallibleCollectionWalker;
+
+  Item *operator->() { return &this->operator*(); }
+
+  const Item *operator->() const { return &this->operator*(); }
+};
+
+class FallibleCollectionWalkerWithFallibleDeref
+    : public FallibleCollectionWalker {
+public:
+  using FallibleCollectionWalker::FallibleCollectionWalker;
+
+  Expected<Item &> operator*() {
+    auto &I = FallibleCollectionWalker::operator*();
+    if (!I.isValid())
+      return make_error<StringError>("bad item", inconvertibleErrorCode());
+    return I;
+  }
+
+  Expected<const Item &> operator*() const {
+    const auto &I = FallibleCollectionWalker::operator*();
+    if (!I.isValid())
+      return make_error<StringError>("bad item", inconvertibleErrorCode());
+    return I;
+  }
+};
+
+TEST(FallibleIteratorTest, BasicSuccess) {
+
+  // Check that a basic use-case involing successful iteration over a
+  // "FallibleCollection" works.
+
+  FallibleCollection C({{ValidItem, ValidLink}, {ValidItem, ValidLink}});
+
+  FallibleCollectionWalker begin(C, 0);
+  FallibleCollectionWalker end(C, 2);
+
+  Error Err = Error::success();
+  for (auto &Elem :
+       make_fallible_range<FallibleCollectionWalker>(begin, end, Err))
+    EXPECT_TRUE(Elem.isValid());
+  cantFail(std::move(Err));
+}
+
+TEST(FallibleIteratorTest, BasicFailure) {
+
+  // Check that a iteration failure (due to the InvalidLink state on element one
+  // of the fallible collection) breaks out of the loop and raises an Error.
+
+  FallibleCollection C({{ValidItem, ValidLink}, {ValidItem, InvalidLink}});
+
+  FallibleCollectionWalker begin(C, 0);
+  FallibleCollectionWalker end(C, 2);
+
+  Error Err = Error::success();
+  for (auto &Elem :
+       make_fallible_range<FallibleCollectionWalker>(begin, end, Err))
+    EXPECT_TRUE(Elem.isValid());
+
+  EXPECT_THAT_ERROR(std::move(Err), Failed()) << "Expected failure value";
+}
+
+TEST(FallibleIteratorTest, NoRedundantErrorCheckOnEarlyExit) {
+
+  // Check that an early return from the loop body does not require a redundant
+  // check of Err.
+
+  FallibleCollection C({{ValidItem, ValidLink}, {ValidItem, ValidLink}});
+
+  FallibleCollectionWalker begin(C, 0);
+  FallibleCollectionWalker end(C, 2);
+
+  Error Err = Error::success();
+  for (auto &Elem :
+       make_fallible_range<FallibleCollectionWalker>(begin, end, Err)) {
+    (void)Elem;
+    return;
+  }
+  // Err not checked, but should be ok because we exit from the loop
+  // body.
+}
+
+#if LLVM_ENABLE_ABI_BREAKING_CHECKS
+TEST(FallibleIteratorTest, RegularLoopExitRequiresErrorCheck) {
+
+  // Check that Err must be checked after a normal (i.e. not early) loop exit
+  // by failing to check and expecting program death (due to the unchecked
+  // error).
+
+  EXPECT_DEATH(
+      {
+        FallibleCollection C({{ValidItem, ValidLink}, {ValidItem, ValidLink}});
+
+        FallibleCollectionWalker begin(C, 0);
+        FallibleCollectionWalker end(C, 2);
+
+        Error Err = Error::success();
+        for (auto &Elem :
+             make_fallible_range<FallibleCollectionWalker>(begin, end, Err))
+          (void)Elem;
+      },
+      "Program aborted due to an unhandled Error:")
+      << "Normal (i.e. not early) loop exit should require an error check";
+}
+#endif
+
+TEST(FallibleIteratorTest, RawIncrementAndDecrementBehavior) {
+
+  // Check the exact behavior of increment / decrement.
+
+  FallibleCollection C({{ValidItem, ValidLink},
+                        {ValidItem, InvalidLink},
+                        {ValidItem, ValidLink},
+                        {ValidItem, InvalidLink}});
+
+  {
+    // One increment from begin succeeds.
+    Error Err = Error::success();
+    auto I = make_fallible_itr(FallibleCollectionWalker(C, 0), Err);
+    ++I;
+    EXPECT_THAT_ERROR(std::move(Err), Succeeded());
+  }
+
+  {
+    // Two increments from begin fail.
+    Error Err = Error::success();
+    auto I = make_fallible_itr(FallibleCollectionWalker(C, 0), Err);
+    ++I;
+    EXPECT_THAT_ERROR(std::move(Err), Succeeded());
+    ++I;
+    EXPECT_THAT_ERROR(std::move(Err), Failed()) << "Expected failure value";
+  }
+
+  {
+    // One decement from element three succeeds.
+    Error Err = Error::success();
+    auto I = make_fallible_itr(FallibleCollectionWalker(C, 3), Err);
+    --I;
+    EXPECT_THAT_ERROR(std::move(Err), Succeeded());
+  }
+
+  {
+    // One decement from element three succeeds.
+    Error Err = Error::success();
+    auto I = make_fallible_itr(FallibleCollectionWalker(C, 3), Err);
+    --I;
+    EXPECT_THAT_ERROR(std::move(Err), Succeeded());
+    --I;
+    EXPECT_THAT_ERROR(std::move(Err), Failed());
+  }
+}
+
+TEST(FallibleIteratorTest, CheckStructDerefOperatorSupport) {
+  // Check that the fallible_iterator wrapper forwards through to the
+  // underlying iterator's structure dereference operator if present.
+
+  FallibleCollection C({{ValidItem, ValidLink},
+                        {ValidItem, ValidLink},
+                        {InvalidItem, InvalidLink}});
+
+  FallibleCollectionWalkerWithStructDeref begin(C, 0);
+
+  {
+    Error Err = Error::success();
+    auto I = make_fallible_itr(begin, Err);
+    EXPECT_TRUE(I->isValid());
+    cantFail(std::move(Err));
+  }
+
+  {
+    Error Err = Error::success();
+    const auto I = make_fallible_itr(begin, Err);
+    EXPECT_TRUE(I->isValid());
+    cantFail(std::move(Err));
+  }
+}
+
+TEST(FallibleIteratorTest, CheckDerefToExpectedSupport) {
+
+  // Check that the fallible_iterator wrapper forwards value types, in
+  // particular llvm::Expected, correctly.
+
+  FallibleCollection C({{ValidItem, ValidLink},
+                        {InvalidItem, ValidLink},
+                        {ValidItem, ValidLink}});
+
+  FallibleCollectionWalkerWithFallibleDeref begin(C, 0);
+  FallibleCollectionWalkerWithFallibleDeref end(C, 3);
+
+  Error Err = Error::success();
+  auto I = make_fallible_itr(begin, Err);
+  auto E = make_fallible_end(end);
+
+  Expected<Item> V1 = *I;
+  EXPECT_THAT_ERROR(V1.takeError(), Succeeded());
+  ++I;
+  EXPECT_NE(I, E); // Implicitly check error.
+  Expected<Item> V2 = *I;
+  EXPECT_THAT_ERROR(V2.takeError(), Failed());
+  ++I;
+  EXPECT_NE(I, E); // Implicitly check error.
+  Expected<Item> V3 = *I;
+  EXPECT_THAT_ERROR(V3.takeError(), Succeeded());
+  ++I;
+  EXPECT_EQ(I, E);
+  cantFail(std::move(Err));
+}
+
+} // namespace
diff --git a/unittests/ADT/FoldingSet.cpp b/unittests/ADT/FoldingSet.cpp
index f5b1b71beb63d44d56ec2700d61d232a0a29da2f..c03353f4e4b071b400ea705bed29a42ad0358e11 100644
--- a/unittests/ADT/FoldingSet.cpp
+++ b/unittests/ADT/FoldingSet.cpp
@@ -1,9 +1,8 @@
 //===- llvm/unittest/ADT/FoldingSetTest.cpp -------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/unittests/ADT/FunctionExtrasTest.cpp b/unittests/ADT/FunctionExtrasTest.cpp
index d85962e0f7731c6ceb62652ec08afc45f6a33fce..bbbb045cb14abf743fa27d80409542b33d26b9ca 100644
--- a/unittests/ADT/FunctionExtrasTest.cpp
+++ b/unittests/ADT/FunctionExtrasTest.cpp
@@ -1,9 +1,8 @@
 //===- FunctionExtrasTest.cpp - Unit tests for function type erasure ------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/ADT/FunctionRefTest.cpp b/unittests/ADT/FunctionRefTest.cpp
index b7ef7d79e5f99f8636ec60d55da8d0dcc0fbeddd..a599bee7e150c45f3faefefa7da7fbf322b0b526 100644
--- a/unittests/ADT/FunctionRefTest.cpp
+++ b/unittests/ADT/FunctionRefTest.cpp
@@ -1,9 +1,8 @@
 //===- llvm/unittest/ADT/MakeUniqueTest.cpp - make_unique unit tests ------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/ADT/HashingTest.cpp b/unittests/ADT/HashingTest.cpp
index d96dd7e2f071cd0fb91cac601def12c03c73f1fd..66d63d437a4661d4cdf04262f157ffa97fb5525b 100644
--- a/unittests/ADT/HashingTest.cpp
+++ b/unittests/ADT/HashingTest.cpp
@@ -1,9 +1,8 @@
 //===- llvm/unittest/ADT/HashingTest.cpp ----------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/unittests/ADT/IListBaseTest.cpp b/unittests/ADT/IListBaseTest.cpp
index 3b8ede801c56760359749e983c8f33589b0fd594..2983618f07e4afe8f095a8a723dacd5d501d45d2 100644
--- a/unittests/ADT/IListBaseTest.cpp
+++ b/unittests/ADT/IListBaseTest.cpp
@@ -1,9 +1,8 @@
 //===- unittests/ADT/IListBaseTest.cpp - ilist_base unit tests ------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/ADT/IListIteratorTest.cpp b/unittests/ADT/IListIteratorTest.cpp
index 8b2aa62f1f8b7d7e34c6d90aa03d0397761862c5..559e32de077c5f3bf165ee7e8d03f1cebc4bf559 100644
--- a/unittests/ADT/IListIteratorTest.cpp
+++ b/unittests/ADT/IListIteratorTest.cpp
@@ -1,9 +1,8 @@
 //===- unittests/ADT/IListIteratorTest.cpp - ilist_iterator unit tests ----===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/ADT/IListNodeBaseTest.cpp b/unittests/ADT/IListNodeBaseTest.cpp
index 8819ab118a599eb58489942720095e66e8b8dcb5..65f85fc3a237e197b26f312422efd2eef846605e 100644
--- a/unittests/ADT/IListNodeBaseTest.cpp
+++ b/unittests/ADT/IListNodeBaseTest.cpp
@@ -1,9 +1,8 @@
 //===- unittests/ADT/IListNodeBaseTest.cpp - ilist_node_base unit tests ---===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/ADT/IListNodeTest.cpp b/unittests/ADT/IListNodeTest.cpp
index 51bebc21ce6544e1b32880903d879e3cedd75a6c..cf775eb5d0699fabe2dc134b1e87227702ada07d 100644
--- a/unittests/ADT/IListNodeTest.cpp
+++ b/unittests/ADT/IListNodeTest.cpp
@@ -1,9 +1,8 @@
 //===- unittests/ADT/IListNodeTest.cpp - ilist_node unit tests ------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/ADT/IListSentinelTest.cpp b/unittests/ADT/IListSentinelTest.cpp
index bd60c909de78f957045a6afad86886372d550b1f..1f4a8311370a6d5beed7ddcf15df12ea756e48be 100644
--- a/unittests/ADT/IListSentinelTest.cpp
+++ b/unittests/ADT/IListSentinelTest.cpp
@@ -1,9 +1,8 @@
 //===- unittests/ADT/IListSentinelTest.cpp - ilist_sentinel unit tests ----===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/ADT/IListTest.cpp b/unittests/ADT/IListTest.cpp
index 0dee4c10f68f65d714dffab9def6a6cb57ce4e0f..18f6c41a64807636d32bd9613631db5dbd646583 100644
--- a/unittests/ADT/IListTest.cpp
+++ b/unittests/ADT/IListTest.cpp
@@ -1,9 +1,8 @@
 //===- unittests/ADT/IListTest.cpp - ilist unit tests ---------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -208,6 +207,12 @@ struct NodeWithCallback : ilist_node<NodeWithCallback> {
 } // end namespace
 
 namespace llvm {
+// These nodes are stack-allocated for testing purposes, so don't let the ilist
+// own or delete them.
+template <> struct ilist_alloc_traits<NodeWithCallback> {
+  static void deleteNode(NodeWithCallback *) {}
+};
+
 template <> struct ilist_callback_traits<NodeWithCallback> {
   void addNodeToList(NodeWithCallback *N) { N->IsInList = true; }
   void removeNodeFromList(NodeWithCallback *N) { N->IsInList = false; }
@@ -248,6 +253,30 @@ TEST(IListTest, addNodeToList) {
   ASSERT_TRUE(N.WasTransferred);
 }
 
+TEST(IListTest, sameListSplice) {
+  NodeWithCallback N1(1);
+  NodeWithCallback N2(2);
+  ASSERT_FALSE(N1.WasTransferred);
+  ASSERT_FALSE(N2.WasTransferred);
+
+  ilist<NodeWithCallback> L1;
+  L1.insert(L1.end(), &N1);
+  L1.insert(L1.end(), &N2);
+  ASSERT_EQ(2u, L1.size());
+  ASSERT_EQ(&N1, &L1.front());
+  ASSERT_FALSE(N1.WasTransferred);
+  ASSERT_FALSE(N2.WasTransferred);
+
+  // Swap the nodes with splice inside the same list. Check that we get the
+  // transfer callback.
+  L1.splice(L1.begin(), L1, std::next(L1.begin()), L1.end());
+  ASSERT_EQ(2u, L1.size());
+  ASSERT_EQ(&N1, &L1.back());
+  ASSERT_EQ(&N2, &L1.front());
+  ASSERT_FALSE(N1.WasTransferred);
+  ASSERT_TRUE(N2.WasTransferred);
+}
+
 struct PrivateNode : private ilist_node<PrivateNode> {
   friend struct llvm::ilist_detail::NodeAccess;
 
diff --git a/unittests/ADT/ImmutableListTest.cpp b/unittests/ADT/ImmutableListTest.cpp
index 280cb2ef4d6864f7b69fa6af9b2008636efd5c62..b0b1e0e6c29f41dab16949cc8e4a4c2d8d1e9354 100644
--- a/unittests/ADT/ImmutableListTest.cpp
+++ b/unittests/ADT/ImmutableListTest.cpp
@@ -1,9 +1,8 @@
 //===--------- ImmutableListTest.cpp - ImmutableList unit tests --*- C++ -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. Lee LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -268,4 +267,7 @@ TEST_F(ImmutableListTest, LongListOrderingTest) {
   ASSERT_EQ(6, i);
 }
 
+static_assert(is_trivially_copyable<ImmutableList<Wrapper<long>>>::value,
+              "trivially copyable");
+
 } // namespace
diff --git a/unittests/ADT/ImmutableMapTest.cpp b/unittests/ADT/ImmutableMapTest.cpp
index 23ca168a7f7ca419ef3e18fedda85c0a65b5cc75..fa61816d213cf64027f6edf184fedec75626cc4a 100644
--- a/unittests/ADT/ImmutableMapTest.cpp
+++ b/unittests/ADT/ImmutableMapTest.cpp
@@ -1,9 +1,8 @@
 //===----------- ImmutableMapTest.cpp - ImmutableMap unit tests ------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/ADT/ImmutableSetTest.cpp b/unittests/ADT/ImmutableSetTest.cpp
index 35ac2c19b2f50183182ca9bd64c68171dc9be9b9..9fe7a80d99001f6b0d9e11727dee35be6be5f074 100644
--- a/unittests/ADT/ImmutableSetTest.cpp
+++ b/unittests/ADT/ImmutableSetTest.cpp
@@ -1,9 +1,8 @@
 //===----------- ImmutableSetTest.cpp - ImmutableSet unit tests ------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/ADT/IntEqClassesTest.cpp b/unittests/ADT/IntEqClassesTest.cpp
index fc908c1e8bff3c1bf7af053d6cb4e9dba4db8074..1c15d2c6f3c50ba6245126a9de79f37686add375 100644
--- a/unittests/ADT/IntEqClassesTest.cpp
+++ b/unittests/ADT/IntEqClassesTest.cpp
@@ -1,9 +1,8 @@
 //===---- ADT/IntEqClassesTest.cpp - IntEqClasses unit tests ----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/ADT/IntervalMapTest.cpp b/unittests/ADT/IntervalMapTest.cpp
index 11f13752f310f18806e9ad8470c0d8264aa9e0f8..170d1523a5dea442e57b51c3b4459f927b5711e4 100644
--- a/unittests/ADT/IntervalMapTest.cpp
+++ b/unittests/ADT/IntervalMapTest.cpp
@@ -1,9 +1,8 @@
 //===---- ADT/IntervalMapTest.cpp - IntervalMap unit tests ------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -611,6 +610,50 @@ TEST(IntervalMapTest, RandomCoalescing) {
 
 }
 
+TEST(IntervalMapTest, Overlaps) {
+  UUMap::Allocator allocator;
+  UUMap map(allocator);
+  map.insert(10, 20, 0);
+  map.insert(30, 40, 0);
+  map.insert(50, 60, 0);
+
+  EXPECT_FALSE(map.overlaps(0, 9));
+  EXPECT_TRUE(map.overlaps(0, 10));
+  EXPECT_TRUE(map.overlaps(0, 15));
+  EXPECT_TRUE(map.overlaps(0, 25));
+  EXPECT_TRUE(map.overlaps(0, 45));
+  EXPECT_TRUE(map.overlaps(10, 45));
+  EXPECT_TRUE(map.overlaps(30, 45));
+  EXPECT_TRUE(map.overlaps(35, 36));
+  EXPECT_TRUE(map.overlaps(40, 45));
+  EXPECT_FALSE(map.overlaps(45, 45));
+  EXPECT_TRUE(map.overlaps(60, 60));
+  EXPECT_TRUE(map.overlaps(60, 66));
+  EXPECT_FALSE(map.overlaps(66, 66));
+}
+
+TEST(IntervalMapTest, OverlapsHalfOpen) {
+  UUHalfOpenMap::Allocator allocator;
+  UUHalfOpenMap map(allocator);
+  map.insert(10, 20, 0);
+  map.insert(30, 40, 0);
+  map.insert(50, 60, 0);
+
+  EXPECT_FALSE(map.overlaps(0, 9));
+  EXPECT_FALSE(map.overlaps(0, 10));
+  EXPECT_TRUE(map.overlaps(0, 15));
+  EXPECT_TRUE(map.overlaps(0, 25));
+  EXPECT_TRUE(map.overlaps(0, 45));
+  EXPECT_TRUE(map.overlaps(10, 45));
+  EXPECT_TRUE(map.overlaps(30, 45));
+  EXPECT_TRUE(map.overlaps(35, 36));
+  EXPECT_FALSE(map.overlaps(40, 45));
+  EXPECT_FALSE(map.overlaps(45, 46));
+  EXPECT_FALSE(map.overlaps(60, 61));
+  EXPECT_FALSE(map.overlaps(60, 66));
+  EXPECT_FALSE(map.overlaps(66, 67));
+}
+
 TEST(IntervalMapOverlapsTest, SmallMaps) {
   typedef IntervalMapOverlaps<UUMap,UUMap> UUOverlaps;
   UUMap::Allocator allocator;
diff --git a/unittests/ADT/IntrusiveRefCntPtrTest.cpp b/unittests/ADT/IntrusiveRefCntPtrTest.cpp
index 83dc137664977a90fc5747c09a31e8bccdbdb150..c248b04ed5a2c505e5e94689eea1754877c3fc85 100644
--- a/unittests/ADT/IntrusiveRefCntPtrTest.cpp
+++ b/unittests/ADT/IntrusiveRefCntPtrTest.cpp
@@ -1,9 +1,8 @@
 //===- unittest/ADT/IntrusiveRefCntPtrTest.cpp ----------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/ADT/IteratorTest.cpp b/unittests/ADT/IteratorTest.cpp
index 902ddfb49f22a2e21114fd5ff5d0fde37ea61652..f46f109d5722e6c4755904301fb4415f02816d95 100644
--- a/unittests/ADT/IteratorTest.cpp
+++ b/unittests/ADT/IteratorTest.cpp
@@ -1,9 +1,8 @@
 //===- IteratorTest.cpp - Unit tests for iterator utilities ---------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/ADT/MakeUniqueTest.cpp b/unittests/ADT/MakeUniqueTest.cpp
index 3b4938a2ef36d9cc7b9d2d34ebcd33d20edd91be..c0c1096994701cf38ad09601efbb25979dbca36c 100644
--- a/unittests/ADT/MakeUniqueTest.cpp
+++ b/unittests/ADT/MakeUniqueTest.cpp
@@ -1,9 +1,8 @@
 //===- llvm/unittest/ADT/MakeUniqueTest.cpp - make_unique unit tests ------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/ADT/MapVectorTest.cpp b/unittests/ADT/MapVectorTest.cpp
index 16e9b5a74f42aaeca97349d45454901e24147011..b8fdbdd0e0bae9abf1ca4f7ae38dd111d3fdd10f 100644
--- a/unittests/ADT/MapVectorTest.cpp
+++ b/unittests/ADT/MapVectorTest.cpp
@@ -1,9 +1,8 @@
 //===- unittest/ADT/MapVectorTest.cpp - MapVector unit tests ----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/ADT/MappedIteratorTest.cpp b/unittests/ADT/MappedIteratorTest.cpp
index d8c48a7d6257cf686f7d2ff7410d8f7901fdaef5..61e45e3a66eb105015f1440383cf09965cf835df 100644
--- a/unittests/ADT/MappedIteratorTest.cpp
+++ b/unittests/ADT/MappedIteratorTest.cpp
@@ -1,9 +1,8 @@
 //===------ MappedIteratorTest.cpp - Unit tests for mapped_iterator -------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/ADT/OptionalTest.cpp b/unittests/ADT/OptionalTest.cpp
index 20bc9da4d594f4d11fa51379949e8298e5db2b34..98adaccca961341bc95eac617f90545e09735c79 100644
--- a/unittests/ADT/OptionalTest.cpp
+++ b/unittests/ADT/OptionalTest.cpp
@@ -1,15 +1,19 @@
 //===- llvm/unittest/ADT/OptionalTest.cpp - Optional unit tests -----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #include "llvm/ADT/Optional.h"
+#include "llvm/ADT/SmallString.h"
+#include "llvm/Support/raw_ostream.h"
+#include "gtest/gtest-spi.h"
 #include "gtest/gtest.h"
 
+#include <array>
+
 using namespace llvm;
 
 namespace {
@@ -518,5 +522,52 @@ TEST_F(OptionalTest, OperatorGreaterEqual) {
   CheckRelation<GreaterEqual>(InequalityLhs, InequalityRhs, !IsLess);
 }
 
-} // end anonymous namespace
+struct ComparableAndStreamable {
+  friend bool operator==(ComparableAndStreamable,
+                         ComparableAndStreamable) LLVM_ATTRIBUTE_USED {
+    return true;
+  }
+
+  friend raw_ostream &operator<<(raw_ostream &OS, ComparableAndStreamable) {
+    return OS << "ComparableAndStreamable";
+  }
+
+  static Optional<ComparableAndStreamable> get() {
+    return ComparableAndStreamable();
+  }
+};
 
+TEST_F(OptionalTest, StreamOperator) {
+  auto to_string = [](Optional<ComparableAndStreamable> O) {
+    SmallString<16> S;
+    raw_svector_ostream OS(S);
+    OS << O;
+    return S;
+  };
+  EXPECT_EQ("ComparableAndStreamable",
+            to_string(ComparableAndStreamable::get()));
+  EXPECT_EQ("None", to_string(None));
+}
+
+struct Comparable {
+  friend bool operator==(Comparable, Comparable) LLVM_ATTRIBUTE_USED {
+    return true;
+  }
+  static Optional<Comparable> get() { return Comparable(); }
+};
+
+TEST_F(OptionalTest, UseInUnitTests) {
+  // Test that we invoke the streaming operators when pretty-printing values in
+  // EXPECT macros.
+  EXPECT_NONFATAL_FAILURE(EXPECT_EQ(llvm::None, ComparableAndStreamable::get()),
+                          "Expected: llvm::None\n"
+                          "      Which is: None\n"
+                          "To be equal to: ComparableAndStreamable::get()\n"
+                          "      Which is: ComparableAndStreamable");
+
+  // Test that it is still possible to compare objects which do not have a
+  // custom streaming operator.
+  EXPECT_NONFATAL_FAILURE(EXPECT_EQ(llvm::None, Comparable::get()), "object");
+}
+
+} // end anonymous namespace
diff --git a/unittests/ADT/PackedVectorTest.cpp b/unittests/ADT/PackedVectorTest.cpp
index 199a67043090fd3a246ac8152fe86ee52fdb1692..24df398934670b75f051881ef717cc5c4461888f 100644
--- a/unittests/ADT/PackedVectorTest.cpp
+++ b/unittests/ADT/PackedVectorTest.cpp
@@ -1,9 +1,8 @@
 //===- llvm/unittest/ADT/PackedVectorTest.cpp - PackedVector tests --------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/ADT/PointerEmbeddedIntTest.cpp b/unittests/ADT/PointerEmbeddedIntTest.cpp
index 695ea12e0b03a09864dcfd7e85e8cec5bb2a8a28..d24c8b8c75f903a9274c646d5736c5f2a7d332f5 100644
--- a/unittests/ADT/PointerEmbeddedIntTest.cpp
+++ b/unittests/ADT/PointerEmbeddedIntTest.cpp
@@ -1,9 +1,8 @@
 //===- llvm/unittest/ADT/PointerEmbeddedIntTest.cpp -----------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/ADT/PointerIntPairTest.cpp b/unittests/ADT/PointerIntPairTest.cpp
index 985fdba0247fea5175d49bb0a45d316a8c1cc9b2..6b3a4c055c9e7d4adcbd1d04116655afa80d2507 100644
--- a/unittests/ADT/PointerIntPairTest.cpp
+++ b/unittests/ADT/PointerIntPairTest.cpp
@@ -1,9 +1,8 @@
 //===- llvm/unittest/ADT/PointerIntPairTest.cpp - Unit tests --------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -62,6 +61,9 @@ TEST(PointerIntPairTest, GetSet) {
   Pair2.setPointerAndInt(&s, E::Case3);
   EXPECT_EQ(&s, Pair2.getPointer());
   EXPECT_EQ(E::Case3, Pair2.getInt());
+
+  static_assert(is_trivially_copyable<PointerIntPair<S *, 2, E>>::value,
+                "trivially copyable");
 }
 
 TEST(PointerIntPairTest, DefaultInitialize) {
@@ -97,6 +99,11 @@ TEST(PointerIntPairTest, ManyUnusedBits) {
 
   EXPECT_EQ(FixnumPointerTraits::NumLowBitsAvailable - 1,
             PointerLikeTypeTraits<decltype(pair)>::NumLowBitsAvailable);
+
+  static_assert(
+      is_trivially_copyable<
+          PointerIntPair<Fixnum31, 1, bool, FixnumPointerTraits>>::value,
+      "trivially copyable");
 }
 
 } // end anonymous namespace
diff --git a/unittests/ADT/PointerSumTypeTest.cpp b/unittests/ADT/PointerSumTypeTest.cpp
index a4faea624f1cf5fa85ee45cf0e131282ae7ece68..fbf59f3a2fda581ad03e828b567104acd55f9184 100644
--- a/unittests/ADT/PointerSumTypeTest.cpp
+++ b/unittests/ADT/PointerSumTypeTest.cpp
@@ -1,9 +1,8 @@
 //===- llvm/unittest/ADT/PointerSumTypeTest.cpp ---------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/ADT/PointerUnionTest.cpp b/unittests/ADT/PointerUnionTest.cpp
index 360c3714841bfae793af4d10ec7a04ec50378042..657767a240eea3c1423de1d5b765305f95b5dd5f 100644
--- a/unittests/ADT/PointerUnionTest.cpp
+++ b/unittests/ADT/PointerUnionTest.cpp
@@ -1,9 +1,8 @@
 //===- llvm/unittest/ADT/PointerUnionTest.cpp - Optional unit tests -------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/ADT/PostOrderIteratorTest.cpp b/unittests/ADT/PostOrderIteratorTest.cpp
index 20c938e893260db2ddec86ebcb1bec7a2b9bd95d..8e53247fc2f6f7403974bab514cecde01f8b4ecd 100644
--- a/unittests/ADT/PostOrderIteratorTest.cpp
+++ b/unittests/ADT/PostOrderIteratorTest.cpp
@@ -1,9 +1,8 @@
 //===- PostOrderIteratorTest.cpp - PostOrderIterator unit tests -----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 #include "llvm/ADT/PostOrderIterator.h"
diff --git a/unittests/ADT/PriorityWorklistTest.cpp b/unittests/ADT/PriorityWorklistTest.cpp
index 040a11f95f42f810fa69e2053ea4010ea4ca4e73..4dfdd5f9a2611685d484259457e04aaa681aff7c 100644
--- a/unittests/ADT/PriorityWorklistTest.cpp
+++ b/unittests/ADT/PriorityWorklistTest.cpp
@@ -1,9 +1,8 @@
 //===- llvm/unittest/ADT/PriorityWorklist.cpp -----------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/unittests/ADT/RangeAdapterTest.cpp b/unittests/ADT/RangeAdapterTest.cpp
index edc1ced72ba9f3377846b8cd15da92b2338984cf..eb1852639a3a6d906d202d299b0dcdf1b62e0f04 100644
--- a/unittests/ADT/RangeAdapterTest.cpp
+++ b/unittests/ADT/RangeAdapterTest.cpp
@@ -1,9 +1,8 @@
 //===- RangeAdapterTest.cpp - Unit tests for range adapters  --------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/ADT/SCCIteratorTest.cpp b/unittests/ADT/SCCIteratorTest.cpp
index 57a999bea9d56328bf70e493bc7f0fd8709fb486..48350959d046ba434b3b77f91893b4fc0ccd01a4 100644
--- a/unittests/ADT/SCCIteratorTest.cpp
+++ b/unittests/ADT/SCCIteratorTest.cpp
@@ -1,9 +1,8 @@
 //===----- llvm/unittest/ADT/SCCIteratorTest.cpp - SCCIterator tests ------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/ADT/STLExtrasTest.cpp b/unittests/ADT/STLExtrasTest.cpp
index e65e71fe4855dc83f509dbea2d7d4f17e60d2bb5..1db1d58870dd7caed94a32be064d13fa5d873d66 100644
--- a/unittests/ADT/STLExtrasTest.cpp
+++ b/unittests/ADT/STLExtrasTest.cpp
@@ -1,9 +1,8 @@
 //===- STLExtrasTest.cpp - Unit tests for STL extras ----------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/ADT/ScopeExitTest.cpp b/unittests/ADT/ScopeExitTest.cpp
index ab11dff8ce483ffafc4dc4c990cd07a8d85b0ebd..14376529766cf22b0e2fb26ec18a0f42520444b6 100644
--- a/unittests/ADT/ScopeExitTest.cpp
+++ b/unittests/ADT/ScopeExitTest.cpp
@@ -1,9 +1,8 @@
 //===- llvm/unittest/ADT/ScopeExit.cpp - Scope exit unit tests --*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/ADT/SequenceTest.cpp b/unittests/ADT/SequenceTest.cpp
index cc82f86c30d5601c60d2b8676a767cfe6488d17e..4356bb18a0cd923d749da89f5818c6d1664e5e1f 100644
--- a/unittests/ADT/SequenceTest.cpp
+++ b/unittests/ADT/SequenceTest.cpp
@@ -1,9 +1,8 @@
 //===- SequenceTest.cpp - Unit tests for a sequence abstraciton -----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/ADT/SetVectorTest.cpp b/unittests/ADT/SetVectorTest.cpp
index b8cac0dc311e50da9166457c743319289135caa4..b9fef78d2fdb3f56fcf1d76e56b54a0e62fcea30 100644
--- a/unittests/ADT/SetVectorTest.cpp
+++ b/unittests/ADT/SetVectorTest.cpp
@@ -1,9 +1,8 @@
 //===- llvm/unittest/ADT/SetVector.cpp ------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/unittests/ADT/SimpleIListTest.cpp b/unittests/ADT/SimpleIListTest.cpp
index a354f332006a11b1fbb051e242f71fdeb86dbf37..9b8ffac56354af74aafd2f86f1000e3e7a2518eb 100644
--- a/unittests/ADT/SimpleIListTest.cpp
+++ b/unittests/ADT/SimpleIListTest.cpp
@@ -1,9 +1,8 @@
 //===- unittests/ADT/SimpleIListTest.cpp - simple_ilist unit tests --------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/ADT/SmallPtrSetTest.cpp b/unittests/ADT/SmallPtrSetTest.cpp
index 76f9cf7fb9afd3ba99923f09b8ec2a55a2f06c02..cff1632586be433b2dd8878c8cc78459a5751dee 100644
--- a/unittests/ADT/SmallPtrSetTest.cpp
+++ b/unittests/ADT/SmallPtrSetTest.cpp
@@ -1,9 +1,8 @@
 //===- llvm/unittest/ADT/SmallPtrSetTest.cpp ------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/unittests/ADT/SmallSetTest.cpp b/unittests/ADT/SmallSetTest.cpp
index 3391a5c83f5a129b04481235c0631d296f520e07..8fb78b01f4464a4096da012765874799191bc11d 100644
--- a/unittests/ADT/SmallSetTest.cpp
+++ b/unittests/ADT/SmallSetTest.cpp
@@ -1,9 +1,8 @@
 //===- llvm/unittest/ADT/SmallSetTest.cpp ------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/unittests/ADT/SmallStringTest.cpp b/unittests/ADT/SmallStringTest.cpp
index 995ef8e812781f9c76a701dd2bf49fe086e61011..6868381c5cf11508f1a5b13a88b5523614f9a29b 100644
--- a/unittests/ADT/SmallStringTest.cpp
+++ b/unittests/ADT/SmallStringTest.cpp
@@ -1,9 +1,8 @@
 //===- llvm/unittest/ADT/SmallStringTest.cpp ------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/unittests/ADT/SmallVectorTest.cpp b/unittests/ADT/SmallVectorTest.cpp
index 9c501bbdf1a3ccc3d43217cbacb21915e654a278..7aaf82dde04bc9e0c8585c14e69325475a75820b 100644
--- a/unittests/ADT/SmallVectorTest.cpp
+++ b/unittests/ADT/SmallVectorTest.cpp
@@ -1,9 +1,8 @@
 //===- llvm/unittest/ADT/SmallVectorTest.cpp ------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/unittests/ADT/SparseBitVectorTest.cpp b/unittests/ADT/SparseBitVectorTest.cpp
index 7675ddac14b670b57330b9de30b315cd7dfcfb88..db8d58a11992466510a17d4b0e560c7551ab9bce 100644
--- a/unittests/ADT/SparseBitVectorTest.cpp
+++ b/unittests/ADT/SparseBitVectorTest.cpp
@@ -1,9 +1,8 @@
 //===- llvm/unittest/ADT/SparseBitVectorTest.cpp - SparseBitVector tests --===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/ADT/SparseMultiSetTest.cpp b/unittests/ADT/SparseMultiSetTest.cpp
index 032990e4bcd657538525c7e5b975b41cdb856a00..54f7bc99b52fad130fb157d91969981a510f225e 100644
--- a/unittests/ADT/SparseMultiSetTest.cpp
+++ b/unittests/ADT/SparseMultiSetTest.cpp
@@ -1,9 +1,8 @@
 //===------ ADT/SparseSetTest.cpp - SparseSet unit tests -  -----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/ADT/SparseSetTest.cpp b/unittests/ADT/SparseSetTest.cpp
index 4db7a7d61fb259775f4a54f3262aa1752db77157..2b065ea901f360a3d79ac11cba49ed22d224b474 100644
--- a/unittests/ADT/SparseSetTest.cpp
+++ b/unittests/ADT/SparseSetTest.cpp
@@ -1,9 +1,8 @@
 //===------ ADT/SparseSetTest.cpp - SparseSet unit tests -  -----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/ADT/StatisticTest.cpp b/unittests/ADT/StatisticTest.cpp
index 17a5c7fc204a1d64e715868a02ee51714211007b..1b5530fb7296f7e47b1dd157cc2f3a842f62182b 100644
--- a/unittests/ADT/StatisticTest.cpp
+++ b/unittests/ADT/StatisticTest.cpp
@@ -1,9 +1,8 @@
 //===- llvm/unittest/ADT/StatisticTest.cpp - Statistic unit tests ---------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/ADT/StringExtrasTest.cpp b/unittests/ADT/StringExtrasTest.cpp
index 02a45c3404f0c9d2a860f1505c1c95ba6e7d3488..97c91de1d3faf3844ca5654f03d5f98ff139f7c1 100644
--- a/unittests/ADT/StringExtrasTest.cpp
+++ b/unittests/ADT/StringExtrasTest.cpp
@@ -1,9 +1,8 @@
 //===- StringExtrasTest.cpp - Unit tests for String extras ----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/ADT/StringMapTest.cpp b/unittests/ADT/StringMapTest.cpp
index a44bbbfee0951e20f2a2570a947fc3b24dfc91fb..841b515393543b3d607aa1fc5a72430a3c705bfc 100644
--- a/unittests/ADT/StringMapTest.cpp
+++ b/unittests/ADT/StringMapTest.cpp
@@ -1,9 +1,8 @@
 //===- llvm/unittest/ADT/StringMapMap.cpp - StringMap unit tests ----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/ADT/StringRefTest.cpp b/unittests/ADT/StringRefTest.cpp
index 4087d6c99a93fb971ee697ac16b58c5b56c14c39..6f83dc3224a3d30f92533174f0178dfad34a6f29 100644
--- a/unittests/ADT/StringRefTest.cpp
+++ b/unittests/ADT/StringRefTest.cpp
@@ -1,9 +1,8 @@
 //===- llvm/unittest/ADT/StringRefTest.cpp - StringRef unit tests ---------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -1062,4 +1061,6 @@ TEST(StringRefTest, StringLiteral) {
   EXPECT_EQ(StringRef("Bar"), Strings[1]);
 }
 
+static_assert(is_trivially_copyable<StringRef>::value, "trivially copyable");
+
 } // end anonymous namespace
diff --git a/unittests/ADT/StringSwitchTest.cpp b/unittests/ADT/StringSwitchTest.cpp
index 62d3a319e7a4565f399818557fed3de3ef9d7ed7..2ce6cdca8d36a3117e6f8527c833c0627b9079a3 100644
--- a/unittests/ADT/StringSwitchTest.cpp
+++ b/unittests/ADT/StringSwitchTest.cpp
@@ -1,9 +1,8 @@
 //===- llvm/unittest/ADT/StringSwitchTest.cpp - StringSwitch unit tests ---===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/ADT/TestGraph.h b/unittests/ADT/TestGraph.h
index 4ddd14cf53351d888137a7bc1340a82213e5ded6..36d298255c1b7ecc75dcf6c7894d4f06bfedbd52 100644
--- a/unittests/ADT/TestGraph.h
+++ b/unittests/ADT/TestGraph.h
@@ -1,9 +1,8 @@
 //===- llvm/unittest/ADT/TestGraph.h - Graph for testing ------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/unittests/ADT/TinyPtrVectorTest.cpp b/unittests/ADT/TinyPtrVectorTest.cpp
index cc14ccc1e54ef538c3dca7f1776742140646db01..c8c00d94736d29e5925c2d88ba47db90aae20b00 100644
--- a/unittests/ADT/TinyPtrVectorTest.cpp
+++ b/unittests/ADT/TinyPtrVectorTest.cpp
@@ -1,9 +1,8 @@
 //===- llvm/unittest/ADT/TinyPtrVectorTest.cpp ----------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/unittests/ADT/TripleTest.cpp b/unittests/ADT/TripleTest.cpp
index efe859f158f26122b1a30cf68f268e1390d7d754..6ad6b62e90cdb3a626f641ff26c1d4cfbeda442a 100644
--- a/unittests/ADT/TripleTest.cpp
+++ b/unittests/ADT/TripleTest.cpp
@@ -1,9 +1,8 @@
 //===----------- Triple.cpp - Triple unit tests ---------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -253,12 +252,24 @@ TEST(TripleTest, ParsedIDs) {
   EXPECT_EQ(Triple::UnknownOS, T.getOS());
   EXPECT_EQ(Triple::UnknownEnvironment, T.getEnvironment());
 
+  T = Triple("wasm32-unknown-wasi-musl");
+  EXPECT_EQ(Triple::wasm32, T.getArch());
+  EXPECT_EQ(Triple::UnknownVendor, T.getVendor());
+  EXPECT_EQ(Triple::WASI, T.getOS());
+  EXPECT_EQ(Triple::Musl, T.getEnvironment());
+
   T = Triple("wasm64-unknown-unknown");
   EXPECT_EQ(Triple::wasm64, T.getArch());
   EXPECT_EQ(Triple::UnknownVendor, T.getVendor());
   EXPECT_EQ(Triple::UnknownOS, T.getOS());
   EXPECT_EQ(Triple::UnknownEnvironment, T.getEnvironment());
 
+  T = Triple("wasm64-unknown-wasi-musl");
+  EXPECT_EQ(Triple::wasm64, T.getArch());
+  EXPECT_EQ(Triple::UnknownVendor, T.getVendor());
+  EXPECT_EQ(Triple::WASI, T.getOS());
+  EXPECT_EQ(Triple::Musl, T.getEnvironment());
+
   T = Triple("avr-unknown-unknown");
   EXPECT_EQ(Triple::avr, T.getArch());
   EXPECT_EQ(Triple::UnknownVendor, T.getVendor());
@@ -1235,11 +1246,17 @@ TEST(TripleTest, FileFormat) {
 
   EXPECT_EQ(Triple::Wasm, Triple("wasm32-unknown-unknown").getObjectFormat());
   EXPECT_EQ(Triple::Wasm, Triple("wasm64-unknown-unknown").getObjectFormat());
+  EXPECT_EQ(Triple::Wasm, Triple("wasm32-unknown-wasi-musl").getObjectFormat());
+  EXPECT_EQ(Triple::Wasm, Triple("wasm64-unknown-wasi-musl").getObjectFormat());
 
   EXPECT_EQ(Triple::Wasm,
             Triple("wasm32-unknown-unknown-wasm").getObjectFormat());
   EXPECT_EQ(Triple::Wasm,
             Triple("wasm64-unknown-unknown-wasm").getObjectFormat());
+  EXPECT_EQ(Triple::Wasm,
+            Triple("wasm32-unknown-wasi-musl-wasm").getObjectFormat());
+  EXPECT_EQ(Triple::Wasm,
+            Triple("wasm64-unknown-wasi-musl-wasm").getObjectFormat());
 
   Triple MSVCNormalized(Triple::normalize("i686-pc-windows-msvc-elf"));
   EXPECT_EQ(Triple::ELF, MSVCNormalized.getObjectFormat());
diff --git a/unittests/ADT/TwineTest.cpp b/unittests/ADT/TwineTest.cpp
index 950eda2b058ad0b54e1f10133adc6e9090c806c4..a717036c136455cc944b6e3bd927987de880ee7b 100644
--- a/unittests/ADT/TwineTest.cpp
+++ b/unittests/ADT/TwineTest.cpp
@@ -1,9 +1,8 @@
 //===- TwineTest.cpp - Twine unit tests -----------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/ADT/VariadicFunctionTest.cpp b/unittests/ADT/VariadicFunctionTest.cpp
index 43db64836cc1009042f34470e9563119e42799bd..56cf85a9ca2ae39462606d062542d804b18459f0 100644
--- a/unittests/ADT/VariadicFunctionTest.cpp
+++ b/unittests/ADT/VariadicFunctionTest.cpp
@@ -1,9 +1,8 @@
 //===----------- VariadicFunctionTest.cpp - VariadicFunction unit tests ---===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/Analysis/AliasAnalysisTest.cpp b/unittests/Analysis/AliasAnalysisTest.cpp
index 42a4210feba26019aafac24cf398c4d50017fc76..7ebff43369d2d97c592a553c8ee2e620a8ca8e12 100644
--- a/unittests/Analysis/AliasAnalysisTest.cpp
+++ b/unittests/Analysis/AliasAnalysisTest.cpp
@@ -1,9 +1,8 @@
 //===--- AliasAnalysisTest.cpp - Mixed TBAA unit tests --------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -167,7 +166,7 @@ TEST_F(AliasAnalysisTest, getModRefInfo) {
   // Setup function.
   FunctionType *FTy =
       FunctionType::get(Type::getVoidTy(C), std::vector<Type *>(), false);
-  auto *F = cast<Function>(M.getOrInsertFunction("f", FTy));
+  auto *F = Function::Create(FTy, Function::ExternalLinkage, "f", M);
   auto *BB = BasicBlock::Create(C, "entry", F);
   auto IntType = Type::getInt32Ty(C);
   auto PtrType = Type::getInt32PtrTy(C);
@@ -175,7 +174,7 @@ TEST_F(AliasAnalysisTest, getModRefInfo) {
   auto *Addr = ConstantPointerNull::get(PtrType);
 
   auto *Store1 = new StoreInst(Value, Addr, BB);
-  auto *Load1 = new LoadInst(Addr, "load", BB);
+  auto *Load1 = new LoadInst(IntType, Addr, "load", BB);
   auto *Add1 = BinaryOperator::CreateAdd(Value, Value, "add", BB);
   auto *VAArg1 = new VAArgInst(Addr, PtrType, "vaarg", BB);
   auto *CmpXChg1 = new AtomicCmpXchgInst(
diff --git a/unittests/Analysis/AliasSetTrackerTest.cpp b/unittests/Analysis/AliasSetTrackerTest.cpp
index 57d21e2fcb86a236819550c272765456c914a1fd..77a5ac47af12e564e5f68d72a2cf7425b9afa6aa 100644
--- a/unittests/Analysis/AliasSetTrackerTest.cpp
+++ b/unittests/Analysis/AliasSetTrackerTest.cpp
@@ -1,9 +1,8 @@
 //=======- AliasSetTrackerTest.cpp - Unit test for the Alias Set Tracker  -===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/Analysis/BasicAliasAnalysisTest.cpp b/unittests/Analysis/BasicAliasAnalysisTest.cpp
index fc52aff8dcc95a584f26acc3f1415f970f982f2a..068774cdecea92ebbbf980a610c76fb9c386574a 100644
--- a/unittests/Analysis/BasicAliasAnalysisTest.cpp
+++ b/unittests/Analysis/BasicAliasAnalysisTest.cpp
@@ -1,9 +1,8 @@
 //===- BasicAliasAnalysisTest.cpp - Unit tests for BasicAA ----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -109,7 +108,7 @@ TEST_F(BasicAATest, AliasInstWithFullObjectOfImpreciseSize) {
   Value *ArbitraryI32 = F->arg_begin();
   AllocaInst *I8 = B.CreateAlloca(B.getInt8Ty(), B.getInt32(2));
   auto *I8AtUncertainOffset =
-      cast<GetElementPtrInst>(B.CreateGEP(I8, ArbitraryI32));
+      cast<GetElementPtrInst>(B.CreateGEP(B.getInt8Ty(), I8, ArbitraryI32));
 
   BasicAAResult &BasicAA = setupAnalyses();
   ASSERT_EQ(BasicAA.alias(
diff --git a/unittests/Analysis/BlockFrequencyInfoTest.cpp b/unittests/Analysis/BlockFrequencyInfoTest.cpp
index cc053066a26906abfb8c033422bfc693d818b970..2aeba947a745c4c283d84c30e95fabbfd36165ae 100644
--- a/unittests/Analysis/BlockFrequencyInfoTest.cpp
+++ b/unittests/Analysis/BlockFrequencyInfoTest.cpp
@@ -1,13 +1,13 @@
 //===- BlockFrequencyInfoTest.cpp - BlockFrequencyInfo unit tests ---------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #include "llvm/Analysis/BlockFrequencyInfo.h"
+#include "llvm/Analysis/BlockFrequencyInfoImpl.h"
 #include "llvm/Analysis/BranchProbabilityInfo.h"
 #include "llvm/Analysis/LoopInfo.h"
 #include "llvm/AsmParser/Parser.h"
@@ -91,5 +91,8 @@ TEST_F(BlockFrequencyInfoTest, Basic) {
   EXPECT_EQ(BFI.getBlockFreq(BB3).getFrequency(), BB3Freq);
 }
 
+static_assert(is_trivially_copyable<bfi_detail::BlockMass>::value,
+              "trivially copyable");
+
 } // end anonymous namespace
 } // end namespace llvm
diff --git a/unittests/Analysis/BranchProbabilityInfoTest.cpp b/unittests/Analysis/BranchProbabilityInfoTest.cpp
index 529af5c068da933845b3cb8e5bb65844f1f8e4f2..01661636cddac8361687491ec7615ae7edefbe45 100644
--- a/unittests/Analysis/BranchProbabilityInfoTest.cpp
+++ b/unittests/Analysis/BranchProbabilityInfoTest.cpp
@@ -1,9 +1,8 @@
 //===- BranchProbabilityInfoTest.cpp - BranchProbabilityInfo unit tests ---===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/Analysis/CFGTest.cpp b/unittests/Analysis/CFGTest.cpp
index d7f14c3e1c35e7ac1fcdc57ee36b470193628a35..d2f0d8e13df7db7fef9e8df371535f030c29c7fe 100644
--- a/unittests/Analysis/CFGTest.cpp
+++ b/unittests/Analysis/CFGTest.cpp
@@ -1,9 +1,8 @@
 //===- CFGTest.cpp - CFG tests --------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/Analysis/CGSCCPassManagerTest.cpp b/unittests/Analysis/CGSCCPassManagerTest.cpp
index 60da2bbc314aac32653804ddf7c4c0382075dc0a..7a1b585f53ef276071b733cf368589672667a35a 100644
--- a/unittests/Analysis/CGSCCPassManagerTest.cpp
+++ b/unittests/Analysis/CGSCCPassManagerTest.cpp
@@ -1,9 +1,8 @@
 //===- CGSCCPassManagerTest.cpp -------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/Analysis/CMakeLists.txt b/unittests/Analysis/CMakeLists.txt
index 7d4fd33716e03b933cfa5042448a454cd5404d57..45d31f889113835aad09d216c8d1700a1b9371b5 100644
--- a/unittests/Analysis/CMakeLists.txt
+++ b/unittests/Analysis/CMakeLists.txt
@@ -12,11 +12,12 @@ add_llvm_unittest(AnalysisTests
   BlockFrequencyInfoTest.cpp
   BranchProbabilityInfoTest.cpp
   CallGraphTest.cpp
+  CaptureTrackingTest.cpp
   CFGTest.cpp
   CGSCCPassManagerTest.cpp
   DivergenceAnalysisTest.cpp
+  DomTreeUpdaterTest.cpp
   GlobalsModRefTest.cpp
-  ValueLatticeTest.cpp
   LazyCallGraphTest.cpp
   LoopInfoTest.cpp
   MemoryBuiltinsTest.cpp
@@ -30,5 +31,6 @@ add_llvm_unittest(AnalysisTests
   TargetLibraryInfoTest.cpp
   TBAATest.cpp
   UnrollAnalyzerTest.cpp
+  ValueLatticeTest.cpp
   ValueTrackingTest.cpp
   )
diff --git a/unittests/Analysis/CallGraphTest.cpp b/unittests/Analysis/CallGraphTest.cpp
index 2d4e63facf3b54f7c070f6d19df1d3d4015b1bd3..0060d2c4b23969c62eeabdc19d3ff17b91b05e93 100644
--- a/unittests/Analysis/CallGraphTest.cpp
+++ b/unittests/Analysis/CallGraphTest.cpp
@@ -1,9 +1,8 @@
 //=======- CallGraphTest.cpp - Unit tests for the CG analysis -------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/Analysis/CaptureTrackingTest.cpp b/unittests/Analysis/CaptureTrackingTest.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..86a0aa16a066ab6e6fa2934f75033c95be66a4f5
--- /dev/null
+++ b/unittests/Analysis/CaptureTrackingTest.cpp
@@ -0,0 +1,77 @@
+//=======- CaptureTrackingTest.cpp - Unit test for the Capture Tracking ---===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Analysis/CaptureTracking.h"
+#include "llvm/Analysis/OrderedBasicBlock.h"
+#include "llvm/AsmParser/Parser.h"
+#include "llvm/IR/Dominators.h"
+#include "llvm/IR/Instructions.h"
+#include "llvm/IR/LLVMContext.h"
+#include "llvm/IR/Module.h"
+#include "llvm/Support/SourceMgr.h"
+#include "gtest/gtest.h"
+
+using namespace llvm;
+
+TEST(CaptureTracking, MaxUsesToExplore) {
+  StringRef Assembly = R"(
+    ; Function Attrs: nounwind ssp uwtable
+    declare void @doesnt_capture(i8* nocapture, i8* nocapture, i8* nocapture, 
+                                 i8* nocapture, i8* nocapture)
+
+    ; %arg has 5 uses
+    define void @test_few_uses(i8* %arg) {
+      call void @doesnt_capture(i8* %arg, i8* %arg, i8* %arg, i8* %arg, i8* %arg)
+      ret void
+    }
+
+    ; %arg has 50 uses
+    define void @test_many_uses(i8* %arg) {
+      call void @doesnt_capture(i8* %arg, i8* %arg, i8* %arg, i8* %arg, i8* %arg)
+      call void @doesnt_capture(i8* %arg, i8* %arg, i8* %arg, i8* %arg, i8* %arg)
+      call void @doesnt_capture(i8* %arg, i8* %arg, i8* %arg, i8* %arg, i8* %arg)
+      call void @doesnt_capture(i8* %arg, i8* %arg, i8* %arg, i8* %arg, i8* %arg)
+      call void @doesnt_capture(i8* %arg, i8* %arg, i8* %arg, i8* %arg, i8* %arg)
+      call void @doesnt_capture(i8* %arg, i8* %arg, i8* %arg, i8* %arg, i8* %arg)
+      call void @doesnt_capture(i8* %arg, i8* %arg, i8* %arg, i8* %arg, i8* %arg)
+      call void @doesnt_capture(i8* %arg, i8* %arg, i8* %arg, i8* %arg, i8* %arg)
+      call void @doesnt_capture(i8* %arg, i8* %arg, i8* %arg, i8* %arg, i8* %arg)
+      call void @doesnt_capture(i8* %arg, i8* %arg, i8* %arg, i8* %arg, i8* %arg)
+      ret void
+    }
+  )";
+
+  LLVMContext Context;
+  SMDiagnostic Error;
+  auto M = parseAssemblyString(Assembly, Error, Context);
+  ASSERT_TRUE(M) << "Bad assembly?";
+
+  auto Test = [&M](const char *FName, unsigned FalseMaxUsesLimit,
+                   unsigned TrueMaxUsesLimit) {
+    Function *F = M->getFunction(FName);
+    ASSERT_NE(F, nullptr);
+    Value *Arg = &*F->arg_begin();
+    ASSERT_NE(Arg, nullptr);
+    ASSERT_FALSE(PointerMayBeCaptured(Arg, true, true, FalseMaxUsesLimit));
+    ASSERT_TRUE(PointerMayBeCaptured(Arg, true, true, TrueMaxUsesLimit));
+
+    BasicBlock *EntryBB = &F->getEntryBlock();
+    DominatorTree DT(*F);
+    OrderedBasicBlock OBB(EntryBB);
+
+    Instruction *Ret = EntryBB->getTerminator();
+    ASSERT_TRUE(isa<ReturnInst>(Ret));
+    ASSERT_FALSE(PointerMayBeCapturedBefore(Arg, true, true, Ret, &DT, false, 
+                                            &OBB, FalseMaxUsesLimit));
+    ASSERT_TRUE(PointerMayBeCapturedBefore(Arg, true, true, Ret, &DT, false,
+                                           &OBB, TrueMaxUsesLimit));
+  };
+
+  Test("test_few_uses", 6, 4);
+  Test("test_many_uses", 50, 30);
+}
diff --git a/unittests/Analysis/DivergenceAnalysisTest.cpp b/unittests/Analysis/DivergenceAnalysisTest.cpp
index 97dbd18af51d52137ea153f2750eaa0d063e84a3..9416e592012de5fa244a6db1cb8be3d1e995099c 100644
--- a/unittests/Analysis/DivergenceAnalysisTest.cpp
+++ b/unittests/Analysis/DivergenceAnalysisTest.cpp
@@ -1,9 +1,8 @@
 //===- DivergenceAnalysisTest.cpp - DivergenceAnalysis unit tests ---------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -79,7 +78,7 @@ TEST_F(DivergenceAnalysisTest, DAInitialState) {
   IntegerType *IntTy = IntegerType::getInt32Ty(Context);
   FunctionType *FTy =
       FunctionType::get(Type::getVoidTy(Context), {IntTy}, false);
-  Function *F = cast<Function>(M.getOrInsertFunction("f", FTy));
+  Function *F = Function::Create(FTy, Function::ExternalLinkage, "f", M);
   BasicBlock *BB = BasicBlock::Create(Context, "entry", F);
   ReturnInst::Create(Context, nullptr, BB);
 
diff --git a/unittests/IR/DomTreeUpdaterTest.cpp b/unittests/Analysis/DomTreeUpdaterTest.cpp
similarity index 98%
rename from unittests/IR/DomTreeUpdaterTest.cpp
rename to unittests/Analysis/DomTreeUpdaterTest.cpp
index db2bb80a79ae331eb22ab5afc95dd6ec1fced201..0fe98237fc18c4cedca80be3c96721bd961567d1 100644
--- a/unittests/IR/DomTreeUpdaterTest.cpp
+++ b/unittests/Analysis/DomTreeUpdaterTest.cpp
@@ -1,13 +1,12 @@
-//==- llvm/unittests/IR/DomTreeUpdaterTest.cpp - DomTreeUpdater unit tests ===//
+//===- DomTreeUpdaterTest.cpp - DomTreeUpdater unit tests -----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
-#include "llvm/IR/DomTreeUpdater.h"
+#include "llvm/Analysis/DomTreeUpdater.h"
 #include "llvm/Analysis/PostDominators.h"
 #include "llvm/AsmParser/Parser.h"
 #include "llvm/IR/Constants.h"
diff --git a/unittests/Analysis/GlobalsModRefTest.cpp b/unittests/Analysis/GlobalsModRefTest.cpp
index 323edc2cc1759e4054222562028fdd281f4c0291..dfc8e1eea75402724f477e57e72b99c6d5b43b11 100644
--- a/unittests/Analysis/GlobalsModRefTest.cpp
+++ b/unittests/Analysis/GlobalsModRefTest.cpp
@@ -1,9 +1,8 @@
 //===--- GlobalsModRefTest.cpp - Mixed TBAA unit tests --------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/Analysis/LazyCallGraphTest.cpp b/unittests/Analysis/LazyCallGraphTest.cpp
index 5e6dd1aee5adb9abaf1512f4bf2d8618800cc4a7..28e70f2f02108c5e40a6d4662bd55a9402ac3ade 100644
--- a/unittests/Analysis/LazyCallGraphTest.cpp
+++ b/unittests/Analysis/LazyCallGraphTest.cpp
@@ -1,9 +1,8 @@
 //===- LazyCallGraphTest.cpp - Unit tests for the lazy CG analysis --------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/Analysis/LoopInfoTest.cpp b/unittests/Analysis/LoopInfoTest.cpp
index 9a59fc84142f478f5225dcc97c0ce193a11b5e8e..483532a187527346642ff3097dc93fc12e7491ba 100644
--- a/unittests/Analysis/LoopInfoTest.cpp
+++ b/unittests/Analysis/LoopInfoTest.cpp
@@ -1,9 +1,8 @@
 //===- LoopInfoTest.cpp - LoopInfo unit tests -----------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/Analysis/MemoryBuiltinsTest.cpp b/unittests/Analysis/MemoryBuiltinsTest.cpp
index 898ebeece4fbcc28a45f10e1cd58412ef573c043..575b7f32047372e4af1e1aab05023ce197039242 100644
--- a/unittests/Analysis/MemoryBuiltinsTest.cpp
+++ b/unittests/Analysis/MemoryBuiltinsTest.cpp
@@ -1,9 +1,8 @@
 //===- MemoryBuiltinsTest.cpp - Tests for utilities in MemoryBuiltins.h ---===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/Analysis/MemorySSATest.cpp b/unittests/Analysis/MemorySSATest.cpp
index 505e185190fde82d6d04d6e6a2d91950812f004a..c38124050f06b70592cd46c35ca063be6f9d81a4 100644
--- a/unittests/Analysis/MemorySSATest.cpp
+++ b/unittests/Analysis/MemorySSATest.cpp
@@ -1,9 +1,8 @@
 //===- MemorySSA.cpp - Unit tests for MemorySSA ---------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 #include "llvm/Analysis/MemorySSA.h"
@@ -93,7 +92,7 @@ TEST_F(MemorySSATest, CreateALoad) {
   MemorySSAUpdater Updater(&MSSA);
   // Add the load
   B.SetInsertPoint(Merge);
-  LoadInst *LoadInst = B.CreateLoad(PointerArg);
+  LoadInst *LoadInst = B.CreateLoad(B.getInt8Ty(), PointerArg);
 
   // MemoryPHI should already exist.
   MemoryPhi *MP = MSSA.getMemoryAccess(Merge);
@@ -139,7 +138,7 @@ TEST_F(MemorySSATest, CreateLoadsAndStoreUpdater) {
 
   // Add the load
   B.SetInsertPoint(Merge, Merge->begin());
-  LoadInst *FirstLoad = B.CreateLoad(PointerArg);
+  LoadInst *FirstLoad = B.CreateLoad(B.getInt8Ty(), PointerArg);
 
   // MemoryPHI should not already exist.
   MemoryPhi *MP = MSSA.getMemoryAccess(Merge);
@@ -163,7 +162,7 @@ TEST_F(MemorySSATest, CreateLoadsAndStoreUpdater) {
   // We don't touch existing loads, so we need to create a new one to get a phi
   // Add the second load
   B.SetInsertPoint(Merge, Merge->begin());
-  LoadInst *SecondLoad = B.CreateLoad(PointerArg);
+  LoadInst *SecondLoad = B.CreateLoad(B.getInt8Ty(), PointerArg);
 
   // MemoryPHI should not already exist.
   MP = MSSA.getMemoryAccess(Merge);
@@ -229,7 +228,7 @@ TEST_F(MemorySSATest, CreateALoadUpdater) {
 
   // Add the load
   B.SetInsertPoint(Merge, Merge->begin());
-  LoadInst *LoadInst = B.CreateLoad(PointerArg);
+  LoadInst *LoadInst = B.CreateLoad(B.getInt8Ty(), PointerArg);
 
   // MemoryPHI should not already exist.
   MemoryPhi *MP = MSSA.getMemoryAccess(Merge);
@@ -263,7 +262,7 @@ TEST_F(MemorySSATest, SinkLoad) {
 
   // Load in left block
   B.SetInsertPoint(Left, Left->begin());
-  LoadInst *LoadInst1 = B.CreateLoad(PointerArg);
+  LoadInst *LoadInst1 = B.CreateLoad(B.getInt8Ty(), PointerArg);
   // Store in merge block
   B.SetInsertPoint(Merge, Merge->begin());
   B.CreateStore(B.getInt8(16), PointerArg);
@@ -311,7 +310,7 @@ TEST_F(MemorySSATest, MoveAStore) {
   BranchInst::Create(Merge, Left);
   BranchInst::Create(Merge, Right);
   B.SetInsertPoint(Merge);
-  B.CreateLoad(PointerArg);
+  B.CreateLoad(B.getInt8Ty(), PointerArg);
   setupAnalyses();
   MemorySSA &MSSA = *Analyses->MSSA;
   MemorySSAUpdater Updater(&MSSA);
@@ -347,7 +346,7 @@ TEST_F(MemorySSATest, MoveAStoreUpdater) {
   BranchInst::Create(Merge, Left);
   BranchInst::Create(Merge, Right);
   B.SetInsertPoint(Merge);
-  auto *MergeLoad = B.CreateLoad(PointerArg);
+  auto *MergeLoad = B.CreateLoad(B.getInt8Ty(), PointerArg);
   setupAnalyses();
   MemorySSA &MSSA = *Analyses->MSSA;
   MemorySSAUpdater Updater(&MSSA);
@@ -393,7 +392,7 @@ TEST_F(MemorySSATest, MoveAStoreUpdaterMove) {
   BranchInst::Create(Merge, Left);
   BranchInst::Create(Merge, Right);
   B.SetInsertPoint(Merge);
-  auto *MergeLoad = B.CreateLoad(PointerArg);
+  auto *MergeLoad = B.CreateLoad(B.getInt8Ty(), PointerArg);
   setupAnalyses();
   MemorySSA &MSSA = *Analyses->MSSA;
   MemorySSAUpdater Updater(&MSSA);
@@ -437,7 +436,7 @@ TEST_F(MemorySSATest, MoveAStoreAllAround) {
   BranchInst::Create(Merge, Left);
   BranchInst::Create(Merge, Right);
   B.SetInsertPoint(Merge);
-  auto *MergeLoad = B.CreateLoad(PointerArg);
+  auto *MergeLoad = B.CreateLoad(B.getInt8Ty(), PointerArg);
   setupAnalyses();
   MemorySSA &MSSA = *Analyses->MSSA;
   MemorySSAUpdater Updater(&MSSA);
@@ -491,7 +490,7 @@ TEST_F(MemorySSATest, RemoveAPhi) {
   BranchInst::Create(Merge, Left);
   BranchInst::Create(Merge, Right);
   B.SetInsertPoint(Merge);
-  LoadInst *LoadInst = B.CreateLoad(PointerArg);
+  LoadInst *LoadInst = B.CreateLoad(B.getInt8Ty(), PointerArg);
 
   setupAnalyses();
   MemorySSA &MSSA = *Analyses->MSSA;
@@ -536,7 +535,7 @@ TEST_F(MemorySSATest, RemoveMemoryAccess) {
   BranchInst::Create(Merge, Left);
   BranchInst::Create(Merge, Right);
   B.SetInsertPoint(Merge);
-  LoadInst *LoadInst = B.CreateLoad(PointerArg);
+  LoadInst *LoadInst = B.CreateLoad(B.getInt8Ty(), PointerArg);
 
   setupAnalyses();
   MemorySSA &MSSA = *Analyses->MSSA;
@@ -632,7 +631,7 @@ TEST_F(MemorySSATest, TestStoreAndLoad) {
   Type *Int8 = Type::getInt8Ty(C);
   Value *Alloca = B.CreateAlloca(Int8, ConstantInt::get(Int8, 1), "A");
   Instruction *SI = B.CreateStore(ConstantInt::get(Int8, 0), Alloca);
-  Instruction *LI = B.CreateLoad(Alloca);
+  Instruction *LI = B.CreateLoad(Int8, Alloca);
 
   setupAnalyses();
   MemorySSA &MSSA = *Analyses->MSSA;
@@ -716,12 +715,12 @@ TEST_F(MemorySSATest, PartialWalkerCacheWithPhis) {
   B.SetInsertPoint(IfThen);
   Instruction *FirstStore = B.CreateStore(Zero, AllocA);
   B.CreateStore(Zero, AllocB);
-  Instruction *ALoad0 = B.CreateLoad(AllocA, "");
+  Instruction *ALoad0 = B.CreateLoad(Int8, AllocA, "");
   Instruction *BStore = B.CreateStore(Zero, AllocB);
   // Due to use optimization/etc. we make a store to A, which is removed after
   // we build MSSA. This helps keep the test case simple-ish.
   Instruction *KillStore = B.CreateStore(Zero, AllocA);
-  Instruction *ALoad = B.CreateLoad(AllocA, "");
+  Instruction *ALoad = B.CreateLoad(Int8, AllocA, "");
   B.CreateBr(IfEnd);
 
   B.SetInsertPoint(IfEnd);
@@ -772,7 +771,7 @@ TEST_F(MemorySSATest, WalkerInvariantLoadOpt) {
   Value *AllocA = B.CreateAlloca(Int8, One, "");
 
   Instruction *Store = B.CreateStore(One, AllocA);
-  Instruction *Load = B.CreateLoad(AllocA);
+  Instruction *Load = B.CreateLoad(Int8, AllocA);
 
   setupAnalyses();
   MemorySSA &MSSA = *Analyses->MSSA;
@@ -801,7 +800,7 @@ TEST_F(MemorySSATest, WalkerReopt) {
   Instruction *SIA = B.CreateStore(ConstantInt::get(Int8, 0), AllocaA);
   Value *AllocaB = B.CreateAlloca(Int8, ConstantInt::get(Int8, 1), "B");
   Instruction *SIB = B.CreateStore(ConstantInt::get(Int8, 0), AllocaB);
-  Instruction *LIA = B.CreateLoad(AllocaA);
+  Instruction *LIA = B.CreateLoad(Int8, AllocaA);
 
   setupAnalyses();
   MemorySSA &MSSA = *Analyses->MSSA;
@@ -835,11 +834,11 @@ TEST_F(MemorySSATest, MoveAboveMemoryDef) {
 
   StoreInst *StoreA0 = B.CreateStore(ConstantInt::get(Int8, 0), A);
   StoreInst *StoreB = B.CreateStore(ConstantInt::get(Int8, 0), B_);
-  LoadInst *LoadB = B.CreateLoad(B_);
+  LoadInst *LoadB = B.CreateLoad(Int8, B_);
   StoreInst *StoreA1 = B.CreateStore(ConstantInt::get(Int8, 4), A);
   StoreInst *StoreC = B.CreateStore(ConstantInt::get(Int8, 4), C);
   StoreInst *StoreA2 = B.CreateStore(ConstantInt::get(Int8, 4), A);
-  LoadInst *LoadC = B.CreateLoad(C);
+  LoadInst *LoadC = B.CreateLoad(Int8, C);
 
   setupAnalyses();
   MemorySSA &MSSA = *Analyses->MSSA;
@@ -903,7 +902,7 @@ TEST_F(MemorySSATest, Irreducible) {
   MemorySSA &MSSA = *Analyses->MSSA;
   MemorySSAUpdater Updater(&MSSA);
   // Create the load memory acccess
-  LoadInst *LoadInst = B.CreateLoad(FirstArg);
+  LoadInst *LoadInst = B.CreateLoad(B.getInt8Ty(), FirstArg);
   MemoryUse *LoadAccess = cast<MemoryUse>(Updater.createMemoryAccessInBB(
       LoadInst, nullptr, AfterLoopBB, MemorySSA::Beginning));
   Updater.insertUse(LoadAccess);
@@ -1011,15 +1010,15 @@ TEST_F(MemorySSATest, TestLoadMustAlias) {
 
   B.CreateStore(ConstantInt::get(Int8, 1), AllocaB);
   // Check load from LOE
-  LoadInst *LA1 = B.CreateLoad(AllocaA, "");
+  LoadInst *LA1 = B.CreateLoad(Int8, AllocaA, "");
   // Check load alias cached for second load
-  LoadInst *LA2 = B.CreateLoad(AllocaA, "");
+  LoadInst *LA2 = B.CreateLoad(Int8, AllocaA, "");
 
   B.CreateStore(ConstantInt::get(Int8, 1), AllocaA);
   // Check load from store/def
-  LoadInst *LA3 = B.CreateLoad(AllocaA, "");
+  LoadInst *LA3 = B.CreateLoad(Int8, AllocaA, "");
   // Check load alias cached for second load
-  LoadInst *LA4 = B.CreateLoad(AllocaA, "");
+  LoadInst *LA4 = B.CreateLoad(Int8, AllocaA, "");
 
   setupAnalyses();
   MemorySSA &MSSA = *Analyses->MSSA;
@@ -1104,13 +1103,13 @@ TEST_F(MemorySSATest, TestLoadMayAlias) {
   Argument *PointerA = &*ArgIt;
   Argument *PointerB = &*(++ArgIt);
   B.CreateStore(ConstantInt::get(Int8, 1), PointerB);
-  LoadInst *LA1 = B.CreateLoad(PointerA, "");
+  LoadInst *LA1 = B.CreateLoad(Int8, PointerA, "");
   B.CreateStore(ConstantInt::get(Int8, 0), PointerA);
-  LoadInst *LB1 = B.CreateLoad(PointerB, "");
+  LoadInst *LB1 = B.CreateLoad(Int8, PointerB, "");
   B.CreateStore(ConstantInt::get(Int8, 0), PointerA);
-  LoadInst *LA2 = B.CreateLoad(PointerA, "");
+  LoadInst *LA2 = B.CreateLoad(Int8, PointerA, "");
   B.CreateStore(ConstantInt::get(Int8, 0), PointerB);
-  LoadInst *LB2 = B.CreateLoad(PointerB, "");
+  LoadInst *LB2 = B.CreateLoad(Int8, PointerB, "");
 
   setupAnalyses();
   MemorySSA &MSSA = *Analyses->MSSA;
@@ -1228,7 +1227,7 @@ TEST_F(MemorySSATest, LifetimeMarkersAreClobbers) {
   B.SetInsertPoint(Entry);
   Value *Foo = &*F->arg_begin();
 
-  Value *Bar = B.CreateGEP(Foo, B.getInt64(1), "bar");
+  Value *Bar = B.CreateGEP(B.getInt8Ty(), Foo, B.getInt64(1), "bar");
 
   B.CreateStore(B.getInt8(0), Foo);
   B.CreateStore(B.getInt8(0), Bar);
diff --git a/unittests/Analysis/OrderedBasicBlockTest.cpp b/unittests/Analysis/OrderedBasicBlockTest.cpp
index b8b9ff04ce7c20584e4b31a2adb4a0c87b153349..4cccb38daff35c4a978714d7c24f8e6008ffbd03 100644
--- a/unittests/Analysis/OrderedBasicBlockTest.cpp
+++ b/unittests/Analysis/OrderedBasicBlockTest.cpp
@@ -1,9 +1,8 @@
 //===- OrderedBasicBlockTest.cpp - OrderedBasicBlock unit tests -----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/Analysis/OrderedInstructionsTest.cpp b/unittests/Analysis/OrderedInstructionsTest.cpp
index dc1b25b8c02ec548ef70adce480f7885719cd2bc..473fe7f50fc8baddb9b403a9cb31907289b8fb68 100644
--- a/unittests/Analysis/OrderedInstructionsTest.cpp
+++ b/unittests/Analysis/OrderedInstructionsTest.cpp
@@ -1,9 +1,8 @@
 //===- OrderedInstructions.cpp - Unit tests for OrderedInstructions  ------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -26,7 +25,7 @@ TEST(OrderedInstructionsTest, DominanceTest) {
   IRBuilder<> B(Ctx);
   FunctionType *FTy =
       FunctionType::get(Type::getVoidTy(Ctx), {B.getInt8PtrTy()}, false);
-  Function *F = cast<Function>(M.getOrInsertFunction("f", FTy));
+  Function *F = Function::Create(FTy, Function::ExternalLinkage, "f", M);
 
   // Create the function as follow and check for dominance relation.
   //
@@ -45,13 +44,13 @@ TEST(OrderedInstructionsTest, DominanceTest) {
   BasicBlock *BBX = BasicBlock::Create(Ctx, "bbx", F);
   B.SetInsertPoint(BBX);
   Argument *PointerArg = &*F->arg_begin();
-  LoadInst *LoadInstX = B.CreateLoad(PointerArg);
-  LoadInst *LoadInstY = B.CreateLoad(PointerArg);
+  LoadInst *LoadInstX = B.CreateLoad(B.getInt8Ty(), PointerArg);
+  LoadInst *LoadInstY = B.CreateLoad(B.getInt8Ty(), PointerArg);
 
   // Create BBY with 1 load.
   BasicBlock *BBY = BasicBlock::Create(Ctx, "bby", F);
   B.SetInsertPoint(BBY);
-  LoadInst *LoadInstZ = B.CreateLoad(PointerArg);
+  LoadInst *LoadInstZ = B.CreateLoad(B.getInt8Ty(), PointerArg);
   B.CreateRet(LoadInstZ);
   std::unique_ptr<DominatorTree> DT(new DominatorTree(*F));
   OrderedInstructions OI(&*DT);
diff --git a/unittests/Analysis/PhiValuesTest.cpp b/unittests/Analysis/PhiValuesTest.cpp
index 303b0df21f52d6d8118ded763dcd27c4c0d2a916..82c02337ef2c9cb8e8e16ab90a7755b2368ffffa 100644
--- a/unittests/Analysis/PhiValuesTest.cpp
+++ b/unittests/Analysis/PhiValuesTest.cpp
@@ -1,9 +1,8 @@
 //===- PhiValuesTest.cpp - PhiValues unit tests ---------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -27,7 +26,8 @@ TEST(PhiValuesTest, SimplePhi) {
   Type *I32PtrTy = Type::getInt32PtrTy(C);
 
   // Create a function with phis that do not have other phis as incoming values
-  Function *F = cast<Function>(M.getOrInsertFunction("f", FunctionType::get(VoidTy, false)));
+  Function *F = Function::Create(FunctionType::get(VoidTy, false),
+                                 Function::ExternalLinkage, "f", M);
 
   BasicBlock *Entry = BasicBlock::Create(C, "entry", F);
   BasicBlock *If = BasicBlock::Create(C, "if", F);
@@ -37,10 +37,10 @@ TEST(PhiValuesTest, SimplePhi) {
   BranchInst::Create(Then, If);
   BranchInst::Create(Then, Else);
 
-  Value *Val1 = new LoadInst(UndefValue::get(I32PtrTy), "val1", Entry);
-  Value *Val2 = new LoadInst(UndefValue::get(I32PtrTy), "val2", Entry);
-  Value *Val3 = new LoadInst(UndefValue::get(I32PtrTy), "val3", Entry);
-  Value *Val4 = new LoadInst(UndefValue::get(I32PtrTy), "val4", Entry);
+  Value *Val1 = new LoadInst(I32Ty, UndefValue::get(I32PtrTy), "val1", Entry);
+  Value *Val2 = new LoadInst(I32Ty, UndefValue::get(I32PtrTy), "val2", Entry);
+  Value *Val3 = new LoadInst(I32Ty, UndefValue::get(I32PtrTy), "val3", Entry);
+  Value *Val4 = new LoadInst(I32Ty, UndefValue::get(I32PtrTy), "val4", Entry);
 
   PHINode *Phi1 = PHINode::Create(I32Ty, 2, "phi1", Then);
   Phi1->addIncoming(Val1, If);
@@ -93,7 +93,8 @@ TEST(PhiValuesTest, DependentPhi) {
   Type *I32PtrTy = Type::getInt32PtrTy(C);
 
   // Create a function with a phi that has another phi as an incoming value
-  Function *F = cast<Function>(M.getOrInsertFunction("f", FunctionType::get(VoidTy, false)));
+  Function *F = Function::Create(FunctionType::get(VoidTy, false),
+                                 Function::ExternalLinkage, "f", M);
 
   BasicBlock *Entry = BasicBlock::Create(C, "entry", F);
   BasicBlock *If1 = BasicBlock::Create(C, "if1", F);
@@ -109,10 +110,10 @@ TEST(PhiValuesTest, DependentPhi) {
   BranchInst::Create(End, If2);
   BranchInst::Create(End, Else2);
 
-  Value *Val1 = new LoadInst(UndefValue::get(I32PtrTy), "val1", Entry);
-  Value *Val2 = new LoadInst(UndefValue::get(I32PtrTy), "val2", Entry);
-  Value *Val3 = new LoadInst(UndefValue::get(I32PtrTy), "val3", Entry);
-  Value *Val4 = new LoadInst(UndefValue::get(I32PtrTy), "val4", Entry);
+  Value *Val1 = new LoadInst(I32Ty, UndefValue::get(I32PtrTy), "val1", Entry);
+  Value *Val2 = new LoadInst(I32Ty, UndefValue::get(I32PtrTy), "val2", Entry);
+  Value *Val3 = new LoadInst(I32Ty, UndefValue::get(I32PtrTy), "val3", Entry);
+  Value *Val4 = new LoadInst(I32Ty, UndefValue::get(I32PtrTy), "val4", Entry);
 
   PHINode *Phi1 = PHINode::Create(I32Ty, 2, "phi1", Then);
   Phi1->addIncoming(Val1, If1);
diff --git a/unittests/Analysis/ProfileSummaryInfoTest.cpp b/unittests/Analysis/ProfileSummaryInfoTest.cpp
index 95d8426c3f213a1ec16aa576a66644f77d033cfa..002901d943b37db88556d9e8cd42a8724396b1ad 100644
--- a/unittests/Analysis/ProfileSummaryInfoTest.cpp
+++ b/unittests/Analysis/ProfileSummaryInfoTest.cpp
@@ -1,9 +1,8 @@
 //===- ProfileSummaryInfoTest.cpp - ProfileSummaryInfo unit tests ---------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/Analysis/ScalarEvolutionTest.cpp b/unittests/Analysis/ScalarEvolutionTest.cpp
index 3da0614bb71c54918f349c9126abc4ec974d99a3..93a61af9c37b0b34716ac807cbac31d9e13d0f47 100644
--- a/unittests/Analysis/ScalarEvolutionTest.cpp
+++ b/unittests/Analysis/ScalarEvolutionTest.cpp
@@ -1,9 +1,8 @@
 //===- ScalarEvolutionsTest.cpp - ScalarEvolution unit tests --------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -64,7 +63,7 @@ protected:
 TEST_F(ScalarEvolutionsTest, SCEVUnknownRAUW) {
   FunctionType *FTy = FunctionType::get(Type::getVoidTy(Context),
                                               std::vector<Type *>(), false);
-  Function *F = cast<Function>(M.getOrInsertFunction("f", FTy));
+  Function *F = Function::Create(FTy, Function::ExternalLinkage, "f", M);
   BasicBlock *BB = BasicBlock::Create(Context, "entry", F);
   ReturnInst::Create(Context, nullptr, BB);
 
@@ -113,7 +112,7 @@ TEST_F(ScalarEvolutionsTest, SCEVUnknownRAUW) {
 TEST_F(ScalarEvolutionsTest, SimplifiedPHI) {
   FunctionType *FTy = FunctionType::get(Type::getVoidTy(Context),
                                               std::vector<Type *>(), false);
-  Function *F = cast<Function>(M.getOrInsertFunction("f", FTy));
+  Function *F = Function::Create(FTy, Function::ExternalLinkage, "f", M);
   BasicBlock *EntryBB = BasicBlock::Create(Context, "entry", F);
   BasicBlock *LoopBB = BasicBlock::Create(Context, "loop", F);
   BasicBlock *ExitBB = BasicBlock::Create(Context, "exit", F);
@@ -147,7 +146,7 @@ TEST_F(ScalarEvolutionsTest, ExpandPtrTypeSCEV) {
   auto *I32PtrTy = Type::getInt32PtrTy(Context);
   FunctionType *FTy =
       FunctionType::get(Type::getVoidTy(Context), std::vector<Type *>(), false);
-  Function *F = cast<Function>(M.getOrInsertFunction("f", FTy));
+  Function *F = Function::Create(FTy, Function::ExternalLinkage, "f", M);
   BasicBlock *EntryBB = BasicBlock::Create(Context, "entry", F);
   BasicBlock *LoopBB = BasicBlock::Create(Context, "loop", F);
   BasicBlock *ExitBB = BasicBlock::Create(Context, "exit", F);
@@ -330,7 +329,7 @@ TEST_F(ScalarEvolutionsTest, CommutativeExprOperandOrder) {
 TEST_F(ScalarEvolutionsTest, CompareSCEVComplexity) {
   FunctionType *FTy =
       FunctionType::get(Type::getVoidTy(Context), std::vector<Type *>(), false);
-  Function *F = cast<Function>(M.getOrInsertFunction("f", FTy));
+  Function *F = Function::Create(FTy, Function::ExternalLinkage, "f", M);
   BasicBlock *EntryBB = BasicBlock::Create(Context, "entry", F);
   BasicBlock *LoopBB = BasicBlock::Create(Context, "bb1", F);
   BranchInst::Create(LoopBB, EntryBB);
@@ -400,7 +399,7 @@ TEST_F(ScalarEvolutionsTest, CompareValueComplexity) {
 
   FunctionType *FTy =
       FunctionType::get(Type::getVoidTy(Context), {IntPtrTy, IntPtrTy}, false);
-  Function *F = cast<Function>(M.getOrInsertFunction("f", FTy));
+  Function *F = Function::Create(FTy, Function::ExternalLinkage, "f", M);
   BasicBlock *EntryBB = BasicBlock::Create(Context, "entry", F);
 
   Value *X = &*F->arg_begin();
@@ -408,9 +407,11 @@ TEST_F(ScalarEvolutionsTest, CompareValueComplexity) {
 
   const int ValueDepth = 10;
   for (int i = 0; i < ValueDepth; i++) {
-    X = new LoadInst(new IntToPtrInst(X, IntPtrPtrTy, "", EntryBB), "",
+    X = new LoadInst(IntPtrTy, new IntToPtrInst(X, IntPtrPtrTy, "", EntryBB),
+                     "",
                      /*isVolatile*/ false, EntryBB);
-    Y = new LoadInst(new IntToPtrInst(Y, IntPtrPtrTy, "", EntryBB), "",
+    Y = new LoadInst(IntPtrTy, new IntToPtrInst(Y, IntPtrPtrTy, "", EntryBB),
+                     "",
                      /*isVolatile*/ false, EntryBB);
   }
 
@@ -436,7 +437,7 @@ TEST_F(ScalarEvolutionsTest, SCEVAddExpr) {
 
   FunctionType *FTy =
       FunctionType::get(Type::getVoidTy(Context), ArgTys, false);
-  Function *F = cast<Function>(M.getOrInsertFunction("f", FTy));
+  Function *F = Function::Create(FTy, Function::ExternalLinkage, "f", M);
 
   Argument *A1 = &*F->arg_begin();
   Argument *A2 = &*(std::next(F->arg_begin()));
@@ -670,7 +671,7 @@ TEST_F(ScalarEvolutionsTest, SCEVZeroExtendExpr) {
   //   ret void
   // }
   FunctionType *FTy = FunctionType::get(Type::getVoidTy(Context), {}, false);
-  Function *F = cast<Function>(M.getOrInsertFunction("foo", FTy));
+  Function *F = Function::Create(FTy, Function::ExternalLinkage, "foo", M);
 
   BasicBlock *EntryBB = BasicBlock::Create(Context, "entry", F);
   BasicBlock *CondBB = BasicBlock::Create(Context, "for.cond", F);
@@ -749,7 +750,7 @@ TEST_F(ScalarEvolutionsTest, SCEVZeroExtendExprNonIntegral) {
 
   FunctionType *FTy =
       FunctionType::get(Type::getVoidTy(Context), {T_pint64}, false);
-  Function *F = cast<Function>(NIM.getOrInsertFunction("foo", FTy));
+  Function *F = Function::Create(FTy, Function::ExternalLinkage, "foo", NIM);
 
   Argument *Arg = &*F->arg_begin();
 
@@ -772,7 +773,8 @@ TEST_F(ScalarEvolutionsTest, SCEVZeroExtendExprNonIntegral) {
   Phi->addIncoming(Add, L);
 
   Builder.SetInsertPoint(Post);
-  Value *GepBase = Builder.CreateGEP(Arg, ConstantInt::get(T_int64, 1));
+  Value *GepBase =
+      Builder.CreateGEP(T_int64, Arg, ConstantInt::get(T_int64, 1));
   Instruction *Ret = Builder.CreateRetVoid();
 
   ScalarEvolution SE = buildSE(*F);
@@ -822,7 +824,7 @@ TEST_F(ScalarEvolutionsTest, SCEVExitLimitForgetLoop) {
 
   FunctionType *FTy =
       FunctionType::get(Type::getVoidTy(Context), {T_pint64}, false);
-  Function *F = cast<Function>(NIM.getOrInsertFunction("foo", FTy));
+  Function *F = Function::Create(FTy, Function::ExternalLinkage, "foo", NIM);
 
   BasicBlock *Top = BasicBlock::Create(Context, "top", F);
   BasicBlock *LPh = BasicBlock::Create(Context, "L.ph", F);
@@ -920,7 +922,7 @@ TEST_F(ScalarEvolutionsTest, SCEVExitLimitForgetValue) {
 
   FunctionType *FTy =
       FunctionType::get(Type::getVoidTy(Context), {T_pint64}, false);
-  Function *F = cast<Function>(NIM.getOrInsertFunction("foo", FTy));
+  Function *F = Function::Create(FTy, Function::ExternalLinkage, "foo", NIM);
 
   Argument *Arg = &*F->arg_begin();
 
@@ -980,7 +982,8 @@ TEST_F(ScalarEvolutionsTest, SCEVAddRecFromPHIwithLargeConstants) {
   // ix.
   FunctionType *FTy =
       FunctionType::get(Type::getVoidTy(Context), std::vector<Type *>(), false);
-  Function *F = cast<Function>(M.getOrInsertFunction("addrecphitest", FTy));
+  Function *F =
+      Function::Create(FTy, Function::ExternalLinkage, "addrecphitest", M);
 
   /*
     Create IR:
@@ -1036,7 +1039,8 @@ TEST_F(ScalarEvolutionsTest, SCEVAddRecFromPHIwithLargeConstantAccum) {
   SmallVector<Type *, 1> Types;
   Types.push_back(Int32Ty);
   FunctionType *FTy = FunctionType::get(Type::getVoidTy(Context), Types, false);
-  Function *F = cast<Function>(M.getOrInsertFunction("addrecphitest", FTy));
+  Function *F =
+      Function::Create(FTy, Function::ExternalLinkage, "addrecphitest", M);
 
   /*
     Create IR:
@@ -1090,7 +1094,7 @@ TEST_F(ScalarEvolutionsTest, SCEVFoldSumOfTruncs) {
   SmallVector<Type *, 1> Types;
   Types.push_back(ArgTy);
   FunctionType *FTy = FunctionType::get(Type::getVoidTy(Context), Types, false);
-  Function *F = cast<Function>(M.getOrInsertFunction("f", FTy));
+  Function *F = Function::Create(FTy, Function::ExternalLinkage, "f", M);
   BasicBlock *BB = BasicBlock::Create(Context, "entry", F);
   ReturnInst::Create(Context, nullptr, BB);
 
@@ -1146,7 +1150,7 @@ TEST_F(ScalarEvolutionsTest, SCEVExpanderIsSafeToExpandAt) {
 
   FunctionType *FTy =
       FunctionType::get(Type::getVoidTy(Context), {T_pint64}, false);
-  Function *F = cast<Function>(NIM.getOrInsertFunction("foo", FTy));
+  Function *F = Function::Create(FTy, Function::ExternalLinkage, "foo", NIM);
 
   BasicBlock *Top = BasicBlock::Create(Context, "top", F);
   BasicBlock *LPh = BasicBlock::Create(Context, "L.ph", F);
@@ -1207,7 +1211,7 @@ TEST_F(ScalarEvolutionsTest, SCEVExpanderNUW) {
 
   FunctionType *FTy =
       FunctionType::get(Type::getVoidTy(Context), { T_int64 }, false);
-  Function *F = cast<Function>(M.getOrInsertFunction("func", FTy));
+  Function *F = Function::Create(FTy, Function::ExternalLinkage, "func", M);
   Argument *Arg = &*F->arg_begin();
   ConstantInt *C = ConstantInt::get(Context, APInt(64, -1));
 
@@ -1259,7 +1263,7 @@ TEST_F(ScalarEvolutionsTest, SCEVExpanderNSW) {
 
   FunctionType *FTy =
       FunctionType::get(Type::getVoidTy(Context), { T_int64 }, false);
-  Function *F = cast<Function>(M.getOrInsertFunction("func", FTy));
+  Function *F = Function::Create(FTy, Function::ExternalLinkage, "func", M);
   Argument *Arg = &*F->arg_begin();
   ConstantInt *C = ConstantInt::get(Context, APInt(64, -1));
 
@@ -1309,7 +1313,7 @@ TEST_F(ScalarEvolutionsTest, SCEVCacheNUW) {
 
   FunctionType *FTy =
       FunctionType::get(Type::getVoidTy(Context), { T_int64 }, false);
-  Function *F = cast<Function>(M.getOrInsertFunction("func", FTy));
+  Function *F = Function::Create(FTy, Function::ExternalLinkage, "func", M);
   Argument *Arg = &*F->arg_begin();
   ConstantInt *C = ConstantInt::get(Context, APInt(64, -1));
 
@@ -1360,7 +1364,7 @@ TEST_F(ScalarEvolutionsTest, SCEVCacheNSW) {
 
   FunctionType *FTy =
       FunctionType::get(Type::getVoidTy(Context), { T_int64 }, false);
-  Function *F = cast<Function>(M.getOrInsertFunction("func", FTy));
+  Function *F = Function::Create(FTy, Function::ExternalLinkage, "func", M);
   Argument *Arg = &*F->arg_begin();
   ConstantInt *C = ConstantInt::get(Context, APInt(64, -1));
 
@@ -1390,5 +1394,55 @@ TEST_F(ScalarEvolutionsTest, SCEVCacheNSW) {
   EXPECT_FALSE(I->hasNoSignedWrap());
 }
 
+// Check logic of SCEV expression size computation.
+TEST_F(ScalarEvolutionsTest, SCEVComputeExpressionSize) {
+  /*
+   * Create the following code:
+   * void func(i64 %a, i64 %b)
+   * entry:
+   *  %s1 = add i64 %a, 1
+   *  %s2 = udiv i64 %s1, %b
+   *  br label %exit
+   * exit:
+   *  ret
+   */
+
+  // Create a module.
+  Module M("SCEVComputeExpressionSize", Context);
+
+  Type *T_int64 = Type::getInt64Ty(Context);
+
+  FunctionType *FTy =
+      FunctionType::get(Type::getVoidTy(Context), { T_int64, T_int64 }, false);
+  Function *F = Function::Create(FTy, Function::ExternalLinkage, "func", M);
+  Argument *A = &*F->arg_begin();
+  Argument *B = &*std::next(F->arg_begin());
+  ConstantInt *C = ConstantInt::get(Context, APInt(64, 1));
+
+  BasicBlock *Entry = BasicBlock::Create(Context, "entry", F);
+  BasicBlock *Exit = BasicBlock::Create(Context, "exit", F);
+
+  IRBuilder<> Builder(Entry);
+  auto *S1 = cast<Instruction>(Builder.CreateAdd(A, C, "s1"));
+  auto *S2 = cast<Instruction>(Builder.CreateUDiv(S1, B, "s2"));
+  Builder.CreateBr(Exit);
+
+  Builder.SetInsertPoint(Exit);
+  Builder.CreateRetVoid();
+
+  ScalarEvolution SE = buildSE(*F);
+  // Get S2 first to move it to cache.
+  const SCEV *AS = SE.getSCEV(A);
+  const SCEV *BS = SE.getSCEV(B);
+  const SCEV *CS = SE.getSCEV(C);
+  const SCEV *S1S = SE.getSCEV(S1);
+  const SCEV *S2S = SE.getSCEV(S2);
+  EXPECT_EQ(AS->getExpressionSize(), 1u);
+  EXPECT_EQ(BS->getExpressionSize(), 1u);
+  EXPECT_EQ(CS->getExpressionSize(), 1u);
+  EXPECT_EQ(S1S->getExpressionSize(), 3u);
+  EXPECT_EQ(S2S->getExpressionSize(), 5u);
+}
+
 }  // end anonymous namespace
 }  // end namespace llvm
diff --git a/unittests/Analysis/SparsePropagation.cpp b/unittests/Analysis/SparsePropagation.cpp
index 298b1403eb5a61fd751d0c1ff1d282a1b08dbfad..bab14a2d055b9b8e6a1de539a492fe13657f4467 100644
--- a/unittests/Analysis/SparsePropagation.cpp
+++ b/unittests/Analysis/SparsePropagation.cpp
@@ -1,9 +1,8 @@
 //===- SparsePropagation.cpp - Unit tests for the generic solver ----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -382,7 +381,7 @@ TEST_F(SparsePropagationTest, FunctionDefined) {
   BasicBlock *Else = BasicBlock::Create(Context, "else", F);
   F->arg_begin()->setName("cond");
   Builder.SetInsertPoint(If);
-  LoadInst *Cond = Builder.CreateLoad(F->arg_begin());
+  LoadInst *Cond = Builder.CreateLoad(Type::getInt1Ty(Context), F->arg_begin());
   Builder.CreateCondBr(Cond, Then, Else);
   Builder.SetInsertPoint(Then);
   Builder.CreateRet(Builder.getInt64(1));
@@ -422,7 +421,7 @@ TEST_F(SparsePropagationTest, FunctionOverDefined) {
   BasicBlock *Else = BasicBlock::Create(Context, "else", F);
   F->arg_begin()->setName("cond");
   Builder.SetInsertPoint(If);
-  LoadInst *Cond = Builder.CreateLoad(F->arg_begin());
+  LoadInst *Cond = Builder.CreateLoad(Type::getInt1Ty(Context), F->arg_begin());
   Builder.CreateCondBr(Cond, Then, Else);
   Builder.SetInsertPoint(Then);
   Builder.CreateRet(Builder.getInt64(0));
diff --git a/unittests/Analysis/TBAATest.cpp b/unittests/Analysis/TBAATest.cpp
index f3f05d8fbd93dceb14f00ae98abb41e4e25d4e52..5a71c74a80b5920bffa2068ab69eba8d27a2a89d 100644
--- a/unittests/Analysis/TBAATest.cpp
+++ b/unittests/Analysis/TBAATest.cpp
@@ -1,9 +1,8 @@
 //===--- TBAATest.cpp - Mixed TBAA unit tests -----------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -34,7 +33,7 @@ protected:
 static StoreInst *getFunctionWithSingleStore(Module *M, StringRef Name) {
   auto &C = M->getContext();
   FunctionType *FTy = FunctionType::get(Type::getVoidTy(C), {});
-  auto *F = cast<Function>(M->getOrInsertFunction(Name, FTy));
+  auto *F = Function::Create(FTy, Function::ExternalLinkage, Name, M);
   auto *BB = BasicBlock::Create(C, "entry", F);
   auto *IntType = Type::getInt32Ty(C);
   auto *PtrType = Type::getInt32PtrTy(C);
diff --git a/unittests/Analysis/TargetLibraryInfoTest.cpp b/unittests/Analysis/TargetLibraryInfoTest.cpp
index 482d9d8d7c04f32750d680b8e517cbb4086c3aed..f3b72b624fc5cccb45e293fc0a67c3574c0bc8d9 100644
--- a/unittests/Analysis/TargetLibraryInfoTest.cpp
+++ b/unittests/Analysis/TargetLibraryInfoTest.cpp
@@ -1,9 +1,8 @@
 //===--- TargetLibraryInfoTest.cpp - TLI/LibFunc unit tests ---------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -68,7 +67,7 @@ TEST_F(TargetLibraryInfoTest, InvalidProto) {
   for (unsigned FI = 0; FI != LibFunc::NumLibFuncs; ++FI) {
     LibFunc LF = (LibFunc)FI;
     auto *F = cast<Function>(
-        M->getOrInsertFunction(TLI.getName(LF), InvalidFTy));
+        M->getOrInsertFunction(TLI.getName(LF), InvalidFTy).getCallee());
     EXPECT_FALSE(isLibFunc(F, LF));
   }
 }
diff --git a/unittests/Analysis/UnrollAnalyzerTest.cpp b/unittests/Analysis/UnrollAnalyzerTest.cpp
index 937e69f0c25796a1b4ead935f3c98a6ffd64587d..dfa42678ea8fe83924ab16ea2a97ffe69b94dff6 100644
--- a/unittests/Analysis/UnrollAnalyzerTest.cpp
+++ b/unittests/Analysis/UnrollAnalyzerTest.cpp
@@ -1,9 +1,8 @@
 //===- UnrollAnalyzerTest.cpp - UnrollAnalyzer unit tests -----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/Analysis/ValueLatticeTest.cpp b/unittests/Analysis/ValueLatticeTest.cpp
index b0b379760a5306cad0c97c154336214f56858d2a..844254ee32f2d85e4d9f902cec347168b91d35f1 100644
--- a/unittests/Analysis/ValueLatticeTest.cpp
+++ b/unittests/Analysis/ValueLatticeTest.cpp
@@ -1,9 +1,8 @@
 //===- ValueLatticeTest.cpp - ScalarEvolution unit tests --------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/Analysis/ValueTrackingTest.cpp b/unittests/Analysis/ValueTrackingTest.cpp
index 5b36d63b3906304f42f4d11b33e41ae6fa32f5ff..a0f5aed2b6d97b63d2141ccf0aab3d4775a676a5 100644
--- a/unittests/Analysis/ValueTrackingTest.cpp
+++ b/unittests/Analysis/ValueTrackingTest.cpp
@@ -1,9 +1,8 @@
 //===- ValueTrackingTest.cpp - ValueTracking tests ------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/AsmParser/AsmParserTest.cpp b/unittests/AsmParser/AsmParserTest.cpp
index ddbedd05c7baa565dff440f6ed42f8645035cf99..d5f734d390af93a8d397f537eab49195f256a31f 100644
--- a/unittests/AsmParser/AsmParserTest.cpp
+++ b/unittests/AsmParser/AsmParserTest.cpp
@@ -1,9 +1,8 @@
 //===- llvm/unittest/AsmParser/AsmParserTest.cpp - asm parser unittests ---===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/BinaryFormat/DwarfTest.cpp b/unittests/BinaryFormat/DwarfTest.cpp
index 08a731f6174dfb8eb5368ec49c508a9db89ec071..5509689f89e0878a0a8df0e28c28004b7cf49346 100644
--- a/unittests/BinaryFormat/DwarfTest.cpp
+++ b/unittests/BinaryFormat/DwarfTest.cpp
@@ -1,9 +1,8 @@
 //===- unittest/BinaryFormat/DwarfTest.cpp - Dwarf support tests ----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/BinaryFormat/MachOTest.cpp b/unittests/BinaryFormat/MachOTest.cpp
index da8e9e8c6901ff24a1255ffdff852129f7f65371..5acffc4e166eb11a55b6de2b9bf231c18b6b1abe 100644
--- a/unittests/BinaryFormat/MachOTest.cpp
+++ b/unittests/BinaryFormat/MachOTest.cpp
@@ -1,9 +1,8 @@
 //===- unittest/BinaryFormat/MachOTest.cpp - MachO support tests ----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/BinaryFormat/MsgPackReaderTest.cpp b/unittests/BinaryFormat/MsgPackReaderTest.cpp
index e3c1bbbaea394c1e197d13c0634c3d8a40b6f2ed..06ae8b080d3a4236a78e075f4a871f1588ca0fa4 100644
--- a/unittests/BinaryFormat/MsgPackReaderTest.cpp
+++ b/unittests/BinaryFormat/MsgPackReaderTest.cpp
@@ -1,9 +1,8 @@
 //===- MsgPackReaderTest.cpp ------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/BinaryFormat/MsgPackTypesTest.cpp b/unittests/BinaryFormat/MsgPackTypesTest.cpp
index 87bb6fa7e4da81f418e2063101206d688bac3594..84ca48a105c861b093f5ddf1d4a61f0c5f50a599 100644
--- a/unittests/BinaryFormat/MsgPackTypesTest.cpp
+++ b/unittests/BinaryFormat/MsgPackTypesTest.cpp
@@ -1,9 +1,8 @@
 //===- MsgPackTypesTest.cpp -------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/BinaryFormat/MsgPackWriterTest.cpp b/unittests/BinaryFormat/MsgPackWriterTest.cpp
index ecca3eee7734b3eb24a6b2539c8e6855ab01f881..3a5ba9af08f6c8728b048b508c6b0dca5816ecc2 100644
--- a/unittests/BinaryFormat/MsgPackWriterTest.cpp
+++ b/unittests/BinaryFormat/MsgPackWriterTest.cpp
@@ -1,9 +1,8 @@
 //===- MsgPackWriterTest.cpp ------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/BinaryFormat/TestFileMagic.cpp b/unittests/BinaryFormat/TestFileMagic.cpp
index 5f132ba4144da26331dcd2836e862062221aaeef..22c26df3080df7e0ff1015a00cccf1d9a96739ca 100644
--- a/unittests/BinaryFormat/TestFileMagic.cpp
+++ b/unittests/BinaryFormat/TestFileMagic.cpp
@@ -1,9 +1,8 @@
 //===- llvm/unittest/BinaryFormat/TestFileMagic.cpp - File magic tests ----===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/Bitcode/BitReaderTest.cpp b/unittests/Bitcode/BitReaderTest.cpp
index 3b4600718b531e493ddce879aaa3db6f592f55eb..e803677a6eee2a68e2aba55cf3453210f87cfa62 100644
--- a/unittests/Bitcode/BitReaderTest.cpp
+++ b/unittests/Bitcode/BitReaderTest.cpp
@@ -1,9 +1,8 @@
 //===- llvm/unittest/Bitcode/BitReaderTest.cpp - Tests for BitReader ------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/Bitcode/BitstreamReaderTest.cpp b/unittests/Bitcode/BitstreamReaderTest.cpp
index e7535f3e818eb0df90d35fa811d89473402fdc72..859548586099e551531374cb4d200bf36080fc33 100644
--- a/unittests/Bitcode/BitstreamReaderTest.cpp
+++ b/unittests/Bitcode/BitstreamReaderTest.cpp
@@ -1,9 +1,8 @@
 //===- BitstreamReaderTest.cpp - Tests for BitstreamReader ----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -148,4 +147,7 @@ TEST(BitstreamReaderTest, shortRead) {
   }
 }
 
+static_assert(is_trivially_copyable<BitCodeAbbrevOp>::value,
+              "trivially copyable");
+
 } // end anonymous namespace
diff --git a/unittests/Bitcode/BitstreamWriterTest.cpp b/unittests/Bitcode/BitstreamWriterTest.cpp
index 79143c8e7a63d184f640a24b30eff4f52bfd5de4..ef4696c8c173398cd786f67a65a1caa8c8d920b0 100644
--- a/unittests/Bitcode/BitstreamWriterTest.cpp
+++ b/unittests/Bitcode/BitstreamWriterTest.cpp
@@ -1,9 +1,8 @@
 //===- BitstreamWriterTest.cpp - Tests for BitstreamWriter ----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/CodeGen/AArch64SelectionDAGTest.cpp b/unittests/CodeGen/AArch64SelectionDAGTest.cpp
index 03bfdc2b5b29ef78dd2295f2837437b1d29bb6b0..ae4c76ecf152e18aa076797dd8fb8cd0ace7ec60 100644
--- a/unittests/CodeGen/AArch64SelectionDAGTest.cpp
+++ b/unittests/CodeGen/AArch64SelectionDAGTest.cpp
@@ -1,9 +1,8 @@
 //===- llvm/unittest/CodeGen/AArch64SelectionDAGTest.cpp -------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -89,8 +88,7 @@ TEST_F(AArch64SelectionDAGTest, computeKnownBits_ZERO_EXTEND_VECTOR_INREG) {
   auto InVec = DAG->getConstant(0, Loc, InVecVT);
   auto Op = DAG->getNode(ISD::ZERO_EXTEND_VECTOR_INREG, Loc, OutVecVT, InVec);
   auto DemandedElts = APInt(2, 3);
-  KnownBits Known;
-  DAG->computeKnownBits(Op, Known, DemandedElts);
+  KnownBits Known = DAG->computeKnownBits(Op, DemandedElts);
   EXPECT_TRUE(Known.isZero());
 }
 
@@ -105,8 +103,7 @@ TEST_F(AArch64SelectionDAGTest, computeKnownBits_EXTRACT_SUBVECTOR) {
   auto ZeroIdx = DAG->getConstant(0, Loc, IdxVT);
   auto Op = DAG->getNode(ISD::EXTRACT_SUBVECTOR, Loc, VecVT, Vec, ZeroIdx);
   auto DemandedElts = APInt(3, 7);
-  KnownBits Known;
-  DAG->computeKnownBits(Op, Known, DemandedElts);
+  KnownBits Known = DAG->computeKnownBits(Op, DemandedElts);
   EXPECT_TRUE(Known.isZero());
 }
 
diff --git a/unittests/CodeGen/CMakeLists.txt b/unittests/CodeGen/CMakeLists.txt
index 5ef8c2afb8c1ef2fe946ccee1beea1e7233f0768..772c075de123f5f57ae104438b20d53a8f5eb656 100644
--- a/unittests/CodeGen/CMakeLists.txt
+++ b/unittests/CodeGen/CMakeLists.txt
@@ -19,6 +19,7 @@ add_llvm_unittest(CodeGenTests
   MachineInstrTest.cpp
   MachineOperandTest.cpp
   ScalableVectorMVTsTest.cpp
+  TypeTraitsTest.cpp
   )
 
 add_subdirectory(GlobalISel)
diff --git a/unittests/CodeGen/DIEHashTest.cpp b/unittests/CodeGen/DIEHashTest.cpp
index c82876b8058428d643815f03e735b3c3284072aa..649e13208f0c103f7731bb3bdab166bbd4a1754b 100644
--- a/unittests/CodeGen/DIEHashTest.cpp
+++ b/unittests/CodeGen/DIEHashTest.cpp
@@ -1,9 +1,8 @@
 //===- llvm/unittest/CodeGen/DIEHashTest.cpp ------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/CodeGen/GlobalISel/CMakeLists.txt b/unittests/CodeGen/GlobalISel/CMakeLists.txt
index 60566cb2d59fd75d27deede674df898dd6938e86..2ffec276db6fdd071722e5880b668035bf42883f 100644
--- a/unittests/CodeGen/GlobalISel/CMakeLists.txt
+++ b/unittests/CodeGen/GlobalISel/CMakeLists.txt
@@ -10,7 +10,10 @@ set(LLVM_LINK_COMPONENTS
   )
 
 add_llvm_unittest(GlobalISelTests
-        LegalizerInfoTest.cpp
-        PatternMatchTest.cpp
-        LegalizerHelperTest.cpp
-        )
+  CSETest.cpp
+  LegalizerHelperTest.cpp
+  LegalizerInfoTest.cpp
+  MachineIRBuilderTest.cpp
+  GISelMITest.cpp
+  PatternMatchTest.cpp
+  )
diff --git a/unittests/CodeGen/GlobalISel/CSETest.cpp b/unittests/CodeGen/GlobalISel/CSETest.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..02627cae164fe0eb258ee2f127ebf06916b8abf5
--- /dev/null
+++ b/unittests/CodeGen/GlobalISel/CSETest.cpp
@@ -0,0 +1,99 @@
+//===- CSETest.cpp -----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "GISelMITest.h"
+#include "llvm/CodeGen/GlobalISel/CSEMIRBuilder.h"
+
+namespace {
+
+TEST_F(GISelMITest, TestCSE) {
+  if (!TM)
+    return;
+
+  LLT s16{LLT::scalar(16)};
+  LLT s32{LLT::scalar(32)};
+  auto MIBInput = B.buildInstr(TargetOpcode::G_TRUNC, {s16}, {Copies[0]});
+  auto MIBInput1 = B.buildInstr(TargetOpcode::G_TRUNC, {s16}, {Copies[1]});
+  auto MIBAdd = B.buildInstr(TargetOpcode::G_ADD, {s16}, {MIBInput, MIBInput});
+  GISelCSEInfo CSEInfo;
+  CSEInfo.setCSEConfig(make_unique<CSEConfig>());
+  CSEInfo.analyze(*MF);
+  B.setCSEInfo(&CSEInfo);
+  CSEMIRBuilder CSEB(B.getState());
+
+  CSEB.setInsertPt(*EntryMBB, EntryMBB->begin());
+  unsigned AddReg = MRI->createGenericVirtualRegister(s16);
+  auto MIBAddCopy =
+      CSEB.buildInstr(TargetOpcode::G_ADD, {AddReg}, {MIBInput, MIBInput});
+  EXPECT_EQ(MIBAddCopy->getOpcode(), TargetOpcode::COPY);
+  auto MIBAdd2 =
+      CSEB.buildInstr(TargetOpcode::G_ADD, {s16}, {MIBInput, MIBInput});
+  EXPECT_TRUE(&*MIBAdd == &*MIBAdd2);
+  auto MIBAdd4 =
+      CSEB.buildInstr(TargetOpcode::G_ADD, {s16}, {MIBInput, MIBInput});
+  EXPECT_TRUE(&*MIBAdd == &*MIBAdd4);
+  auto MIBAdd5 =
+      CSEB.buildInstr(TargetOpcode::G_ADD, {s16}, {MIBInput, MIBInput1});
+  EXPECT_TRUE(&*MIBAdd != &*MIBAdd5);
+
+  // Try building G_CONSTANTS.
+  auto MIBCst = CSEB.buildConstant(s32, 0);
+  auto MIBCst1 = CSEB.buildConstant(s32, 0);
+  EXPECT_TRUE(&*MIBCst == &*MIBCst1);
+  // Try the CFing of BinaryOps.
+  auto MIBCF1 = CSEB.buildInstr(TargetOpcode::G_ADD, {s32}, {MIBCst, MIBCst});
+  EXPECT_TRUE(&*MIBCF1 == &*MIBCst);
+
+  // Try out building FCONSTANTs.
+  auto MIBFP0 = CSEB.buildFConstant(s32, 1.0);
+  auto MIBFP0_1 = CSEB.buildFConstant(s32, 1.0);
+  EXPECT_TRUE(&*MIBFP0 == &*MIBFP0_1);
+  CSEInfo.print();
+
+  // Make sure buildConstant with a vector type doesn't crash, and the elements
+  // CSE.
+  auto Splat0 = CSEB.buildConstant(LLT::vector(2, s32), 0);
+  EXPECT_EQ(TargetOpcode::G_BUILD_VECTOR, Splat0->getOpcode());
+  EXPECT_EQ(Splat0->getOperand(1).getReg(), Splat0->getOperand(2).getReg());
+  EXPECT_EQ(&*MIBCst, MRI->getVRegDef(Splat0->getOperand(1).getReg()));
+
+  auto FSplat = CSEB.buildFConstant(LLT::vector(2, s32), 1.0);
+  EXPECT_EQ(TargetOpcode::G_BUILD_VECTOR, FSplat->getOpcode());
+  EXPECT_EQ(FSplat->getOperand(1).getReg(), FSplat->getOperand(2).getReg());
+  EXPECT_EQ(&*MIBFP0, MRI->getVRegDef(FSplat->getOperand(1).getReg()));
+
+  // Check G_UNMERGE_VALUES
+  auto MIBUnmerge = CSEB.buildUnmerge({s32, s32}, Copies[0]);
+  auto MIBUnmerge2 = CSEB.buildUnmerge({s32, s32}, Copies[0]);
+  EXPECT_TRUE(&*MIBUnmerge == &*MIBUnmerge2);
+}
+
+TEST_F(GISelMITest, TestCSEConstantConfig) {
+  if (!TM)
+    return;
+
+  LLT s16{LLT::scalar(16)};
+  auto MIBInput = B.buildInstr(TargetOpcode::G_TRUNC, {s16}, {Copies[0]});
+  auto MIBAdd = B.buildInstr(TargetOpcode::G_ADD, {s16}, {MIBInput, MIBInput});
+  auto MIBZero = B.buildConstant(s16, 0);
+  GISelCSEInfo CSEInfo;
+  CSEInfo.setCSEConfig(make_unique<CSEConfigConstantOnly>());
+  CSEInfo.analyze(*MF);
+  B.setCSEInfo(&CSEInfo);
+  CSEMIRBuilder CSEB(B.getState());
+  CSEB.setInsertPt(*EntryMBB, EntryMBB->begin());
+  auto MIBAdd1 =
+      CSEB.buildInstr(TargetOpcode::G_ADD, {s16}, {MIBInput, MIBInput});
+  // We should CSE constants only. Adds should not be CSEd.
+  EXPECT_TRUE(MIBAdd1->getOpcode() != TargetOpcode::COPY);
+  EXPECT_TRUE(&*MIBAdd1 != &*MIBAdd);
+  // We should CSE constant.
+  auto MIBZeroTmp = CSEB.buildConstant(s16, 0);
+  EXPECT_TRUE(&*MIBZero == &*MIBZeroTmp);
+}
+} // namespace
diff --git a/unittests/CodeGen/GlobalISel/GISelMITest.cpp b/unittests/CodeGen/GlobalISel/GISelMITest.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..0558c4121fdd51c3e6f73e503d749bf3e51399ce
--- /dev/null
+++ b/unittests/CodeGen/GlobalISel/GISelMITest.cpp
@@ -0,0 +1,30 @@
+//===------------------------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "GISelMITest.h"
+
+namespace llvm {
+std::ostream &
+operator<<(std::ostream &OS, const LLT Ty) {
+  std::string Repr;
+  raw_string_ostream SS{Repr};
+  Ty.print(SS);
+  OS << SS.str();
+  return OS;
+}
+
+std::ostream &
+operator<<(std::ostream &OS, const MachineFunction &MF) {
+  std::string Repr;
+  raw_string_ostream SS{Repr};
+  MF.print(SS);
+  OS << SS.str();
+  return OS;
+}
+
+}
diff --git a/unittests/CodeGen/GlobalISel/LegalizerHelperTest.h b/unittests/CodeGen/GlobalISel/GISelMITest.h
similarity index 78%
rename from unittests/CodeGen/GlobalISel/LegalizerHelperTest.h
rename to unittests/CodeGen/GlobalISel/GISelMITest.h
index 0a171a76b4787ca98cd20dd5a27d58cb973f71fa..5ac81e227cdc971708a53da827cf59368feb4330 100644
--- a/unittests/CodeGen/GlobalISel/LegalizerHelperTest.h
+++ b/unittests/CodeGen/GlobalISel/GISelMITest.h
@@ -1,12 +1,12 @@
-//===- LegalizerHelperTest.h
-//-----------------------------------------------===//
+//===- GISelMITest.h --------------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
+#ifndef LLVM_UNITTEST_CODEGEN_GLOBALISEL_GISELMI_H
+#define LLVM_UNITTEST_CODEGEN_GLOBALISEL_GISELMI_H
 
 #include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h"
 #include "llvm/CodeGen/GlobalISel/LegalizerHelper.h"
@@ -32,7 +32,7 @@
 using namespace llvm;
 using namespace MIPatternMatch;
 
-void initLLVM() {
+static inline void initLLVM() {
   InitializeAllTargets();
   InitializeAllTargetMCs();
   InitializeAllAsmPrinters();
@@ -43,9 +43,18 @@ void initLLVM() {
   initializeCodeGen(*Registry);
 }
 
+// Define a printers to help debugging when things go wrong.
+namespace llvm {
+std::ostream &
+operator<<(std::ostream &OS, const LLT Ty);
+
+std::ostream &
+operator<<(std::ostream &OS, const MachineFunction &MF);
+}
+
 /// Create a TargetMachine. As we lack a dedicated always available target for
 /// unittests, we go for "AArch64".
-std::unique_ptr<LLVMTargetMachine> createTargetMachine() {
+static std::unique_ptr<LLVMTargetMachine> createTargetMachine() {
   Triple TargetTriple("aarch64--");
   std::string Error;
   const Target *T = TargetRegistry::lookupTarget("", TargetTriple, Error);
@@ -53,15 +62,16 @@ std::unique_ptr<LLVMTargetMachine> createTargetMachine() {
     return nullptr;
 
   TargetOptions Options;
-  return std::unique_ptr<LLVMTargetMachine>(static_cast<LLVMTargetMachine*>(
-      T->createTargetMachine("AArch64", "", "", Options, None, None,
-                             CodeGenOpt::Aggressive)));
+  return std::unique_ptr<LLVMTargetMachine>(
+      static_cast<LLVMTargetMachine *>(T->createTargetMachine(
+          "AArch64", "", "", Options, None, None, CodeGenOpt::Aggressive)));
 }
 
-std::unique_ptr<Module> parseMIR(LLVMContext &Context,
-                                 std::unique_ptr<MIRParser> &MIR,
-                                 const TargetMachine &TM, StringRef MIRCode,
-                                 const char *FuncName, MachineModuleInfo &MMI) {
+static std::unique_ptr<Module> parseMIR(LLVMContext &Context,
+                                        std::unique_ptr<MIRParser> &MIR,
+                                        const TargetMachine &TM,
+                                        StringRef MIRCode, const char *FuncName,
+                                        MachineModuleInfo &MMI) {
   SMDiagnostic Diagnostic;
   std::unique_ptr<MemoryBuffer> MBuffer = MemoryBuffer::getMemBuffer(MIRCode);
   MIR = createMIRParser(std::move(MBuffer), Context);
@@ -80,7 +90,7 @@ std::unique_ptr<Module> parseMIR(LLVMContext &Context,
   return M;
 }
 
-std::pair<std::unique_ptr<Module>, std::unique_ptr<MachineModuleInfo>>
+static std::pair<std::unique_ptr<Module>, std::unique_ptr<MachineModuleInfo>>
 createDummyModule(LLVMContext &Context, const LLVMTargetMachine &TM,
                   StringRef MIRFunc) {
   SmallString<512> S;
@@ -123,9 +133,9 @@ static void collectCopies(SmallVectorImpl<unsigned> &Copies,
     }
 }
 
-class LegalizerHelperTest : public ::testing::Test {
+class GISelMITest : public ::testing::Test {
 protected:
-  LegalizerHelperTest() : ::testing::Test() {
+  GISelMITest() : ::testing::Test() {
     TM = createTargetMachine();
     if (!TM)
       return;
@@ -168,8 +178,8 @@ protected:
     }                                                                          \
   };
 
-static bool CheckMachineFunction(const MachineFunction &MF,
-                                 StringRef CheckStr) {
+static inline bool CheckMachineFunction(const MachineFunction &MF,
+                                        StringRef CheckStr) {
   SmallString<512> Msg;
   raw_svector_ostream OS(Msg);
   MF.print(OS);
@@ -185,8 +195,11 @@ static bool CheckMachineFunction(const MachineFunction &MF,
                         SMLoc());
   Regex PrefixRE = FC.buildCheckPrefixRegex();
   std::vector<FileCheckString> CheckStrings;
-  FC.ReadCheckFile(SM, CheckFileText, PrefixRE, CheckStrings);
+  if (FC.ReadCheckFile(SM, CheckFileText, PrefixRE, CheckStrings))
+    return false;
+
   auto OutBuffer = OutputBuf->getBuffer();
   SM.AddNewSourceBuffer(std::move(OutputBuf), SMLoc());
   return FC.CheckInput(SM, OutBuffer, CheckStrings);
 }
+#endif
diff --git a/unittests/CodeGen/GlobalISel/LegalizerHelperTest.cpp b/unittests/CodeGen/GlobalISel/LegalizerHelperTest.cpp
index ee84aeff994f09463025c9db838294536e28dbf3..a9c9a938db52e85339ea4b1c006f8ecd09e0c0a2 100644
--- a/unittests/CodeGen/GlobalISel/LegalizerHelperTest.cpp
+++ b/unittests/CodeGen/GlobalISel/LegalizerHelperTest.cpp
@@ -1,41 +1,42 @@
-//===- PatternMatchTest.cpp -----------------------------------------------===//
+//===- LegalizerHelperTest.cpp
+//-----------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
-#include "LegalizerHelperTest.h"
+#include "GISelMITest.h"
 
 namespace {
 
 class DummyGISelObserver : public GISelChangeObserver {
 public:
-  void changingInstr(const MachineInstr &MI) override {}
-  void changedInstr(const MachineInstr &MI) override {}
-  void createdInstr(const MachineInstr &MI) override {}
-  void erasingInstr(const MachineInstr &MI) override {}
+  void changingInstr(MachineInstr &MI) override {}
+  void changedInstr(MachineInstr &MI) override {}
+  void createdInstr(MachineInstr &MI) override {}
+  void erasingInstr(MachineInstr &MI) override {}
 };
 
 // Test CTTZ expansion when CTTZ_ZERO_UNDEF is legal or custom,
 // in which case it becomes CTTZ_ZERO_UNDEF with select.
-TEST_F(LegalizerHelperTest, LowerBitCountingCTTZ0) {
+TEST_F(GISelMITest, LowerBitCountingCTTZ0) {
   if (!TM)
     return;
 
   // Declare your legalization info
-  DefineLegalizerInfo(
-      A, { getActionDefinitionsBuilder(G_CTTZ_ZERO_UNDEF).legalFor({s64}); });
+  DefineLegalizerInfo(A, {
+    getActionDefinitionsBuilder(G_CTTZ_ZERO_UNDEF).legalFor({{s64, s64}});
+  });
   // Build Instr
   auto MIBCTTZ =
       B.buildInstr(TargetOpcode::G_CTTZ, {LLT::scalar(64)}, {Copies[0]});
   AInfo Info(MF->getSubtarget());
   DummyGISelObserver Observer;
-  LegalizerHelper Helper(*MF, Info, Observer);
+  LegalizerHelper Helper(*MF, Info, Observer, B);
   // Perform Legalization
-  ASSERT_TRUE(Helper.lower(*MIBCTTZ, 0, LLT::scalar(64)) ==
+  EXPECT_TRUE(Helper.lower(*MIBCTTZ, 0, LLT::scalar(64)) ==
               LegalizerHelper::LegalizeResult::Legalized);
 
   auto CheckStr = R"(
@@ -47,25 +48,26 @@ TEST_F(LegalizerHelperTest, LowerBitCountingCTTZ0) {
   )";
 
   // Check
-  ASSERT_TRUE(CheckMachineFunction(*MF, CheckStr));
+  EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
 }
 
 // CTTZ expansion in terms of CTLZ
-TEST_F(LegalizerHelperTest, LowerBitCountingCTTZ1) {
+TEST_F(GISelMITest, LowerBitCountingCTTZ1) {
   if (!TM)
     return;
 
   // Declare your legalization info
-  DefineLegalizerInfo(A,
-                      { getActionDefinitionsBuilder(G_CTLZ).legalFor({s64}); });
+  DefineLegalizerInfo(A, {
+    getActionDefinitionsBuilder(G_CTLZ).legalFor({{s64, s64}});
+  });
   // Build Instr
   auto MIBCTTZ =
       B.buildInstr(TargetOpcode::G_CTTZ, {LLT::scalar(64)}, {Copies[0]});
   AInfo Info(MF->getSubtarget());
   DummyGISelObserver Observer;
-  LegalizerHelper Helper(*MF, Info, Observer);
+  LegalizerHelper Helper(*MF, Info, Observer, B);
   // Perform Legalization
-  ASSERT_TRUE(Helper.lower(*MIBCTTZ, 0, LLT::scalar(64)) ==
+  EXPECT_TRUE(Helper.lower(*MIBCTTZ, 0, LLT::scalar(64)) ==
               LegalizerHelper::LegalizeResult::Legalized);
 
   auto CheckStr = R"(
@@ -79,24 +81,25 @@ TEST_F(LegalizerHelperTest, LowerBitCountingCTTZ1) {
   )";
 
   // Check
-  ASSERT_TRUE(CheckMachineFunction(*MF, CheckStr));
+  EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
 }
 
 // CTTZ expansion in terms of CTPOP
-TEST_F(LegalizerHelperTest, LowerBitCountingCTTZ2) {
+TEST_F(GISelMITest, LowerBitCountingCTTZ2) {
   if (!TM)
     return;
 
   // Declare your legalization info
-  DefineLegalizerInfo(
-      A, { getActionDefinitionsBuilder(G_CTPOP).legalFor({s64}); });
+  DefineLegalizerInfo(A, {
+    getActionDefinitionsBuilder(G_CTPOP).legalFor({{s64, s64}});
+  });
   // Build
   auto MIBCTTZ =
       B.buildInstr(TargetOpcode::G_CTTZ, {LLT::scalar(64)}, {Copies[0]});
   AInfo Info(MF->getSubtarget());
   DummyGISelObserver Observer;
-  LegalizerHelper Helper(*MF, Info, Observer);
-  ASSERT_TRUE(Helper.lower(*MIBCTTZ, 0, LLT::scalar(64)) ==
+  LegalizerHelper Helper(*MF, Info, Observer, B);
+  EXPECT_TRUE(Helper.lower(*MIBCTTZ, 0, LLT::scalar(64)) ==
               LegalizerHelper::LegalizeResult::Legalized);
 
   auto CheckStr = R"(
@@ -108,24 +111,90 @@ TEST_F(LegalizerHelperTest, LowerBitCountingCTTZ2) {
   )";
 
   // Check
-  ASSERT_TRUE(CheckMachineFunction(*MF, CheckStr));
+  EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
+}
+
+// CTPOP widening.
+TEST_F(GISelMITest, WidenBitCountingCTPOP1) {
+  if (!TM)
+    return;
+
+  // Declare your legalization info
+  DefineLegalizerInfo(A, {
+      getActionDefinitionsBuilder(G_CTPOP).legalFor({{s16, s16}});
+    });
+
+  // Build
+  // Trunc it to s8.
+  LLT s8{LLT::scalar(8)};
+  LLT s16{LLT::scalar(16)};
+  auto MIBTrunc = B.buildTrunc(s8, Copies[0]);
+  auto MIBCTPOP = B.buildInstr(TargetOpcode::G_CTPOP, {s16}, {MIBTrunc});
+  AInfo Info(MF->getSubtarget());
+  DummyGISelObserver Observer;
+  LegalizerHelper Helper(*MF, Info, Observer, B);
+  EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
+            Helper.widenScalar(*MIBCTPOP, 1, s16));
+
+  auto CheckStr = R"(
+  CHECK: [[TRUNC:%[0-9]+]]:_(s8) = G_TRUNC %0:_(s64)
+  CHECK: [[ZEXT:%[0-9]+]]:_(s16) = G_ZEXT [[TRUNC]]:_(s8)
+  CHECK: [[CTPOP:%[0-9]+]]:_(s16) = G_CTPOP [[ZEXT]]
+  CHECK: [[COPY:%[0-9]+]]:_(s16) = COPY [[CTPOP]]:_(s16)
+  )";
+
+  EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
+}
+
+// Test a strange case where the result is wider than the source
+TEST_F(GISelMITest, WidenBitCountingCTPOP2) {
+  if (!TM)
+    return;
+
+  // Declare your legalization info
+  DefineLegalizerInfo(A, {
+      getActionDefinitionsBuilder(G_CTPOP).legalFor({{s32, s16}});
+    });
+
+  // Build
+  // Trunc it to s8.
+  LLT s8{LLT::scalar(8)};
+  LLT s16{LLT::scalar(16)};
+  LLT s32{LLT::scalar(32)};
+  auto MIBTrunc = B.buildTrunc(s8, Copies[0]);
+  auto MIBCTPOP = B.buildInstr(TargetOpcode::G_CTPOP, {s32}, {MIBTrunc});
+  AInfo Info(MF->getSubtarget());
+  DummyGISelObserver Observer;
+  LegalizerHelper Helper(*MF, Info, Observer, B);
+  EXPECT_EQ(LegalizerHelper::LegalizeResult::Legalized,
+            Helper.widenScalar(*MIBCTPOP, 1, s16));
+
+  auto CheckStr = R"(
+  CHECK: [[TRUNC:%[0-9]+]]:_(s8) = G_TRUNC %0:_(s64)
+  CHECK: [[ZEXT:%[0-9]+]]:_(s16) = G_ZEXT [[TRUNC]]:_(s8)
+  CHECK: [[CTPOP:%[0-9]+]]:_(s16) = G_CTPOP [[ZEXT]]
+  CHECK: [[COPY:%[0-9]+]]:_(s32) = G_ZEXT [[CTPOP]]:_(s16)
+  )";
+
+  EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
 }
 
 // CTTZ_ZERO_UNDEF expansion in terms of CTTZ
-TEST_F(LegalizerHelperTest, LowerBitCountingCTTZ3) {
+TEST_F(GISelMITest, LowerBitCountingCTTZ3) {
   if (!TM)
     return;
 
   // Declare your legalization info
-  DefineLegalizerInfo(A,
-                      { getActionDefinitionsBuilder(G_CTTZ).legalFor({s64}); });
+  DefineLegalizerInfo(A, {
+    getActionDefinitionsBuilder(G_CTTZ).legalFor({{s64, s64}});
+  });
   // Build
   auto MIBCTTZ = B.buildInstr(TargetOpcode::G_CTTZ_ZERO_UNDEF,
                               {LLT::scalar(64)}, {Copies[0]});
   AInfo Info(MF->getSubtarget());
   DummyGISelObserver Observer;
-  LegalizerHelper Helper(*MF, Info, Observer);
-  ASSERT_TRUE(Helper.lower(*MIBCTTZ, 0, LLT::scalar(64)) ==
+  LegalizerHelper Helper(*MF, Info, Observer, B);
+  EXPECT_TRUE(Helper.lower(*MIBCTTZ, 0, LLT::scalar(64)) ==
               LegalizerHelper::LegalizeResult::Legalized);
 
   auto CheckStr = R"(
@@ -133,24 +202,25 @@ TEST_F(LegalizerHelperTest, LowerBitCountingCTTZ3) {
   )";
 
   // Check
-  ASSERT_TRUE(CheckMachineFunction(*MF, CheckStr));
+  EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
 }
 
 // CTLZ expansion in terms of CTLZ_ZERO_UNDEF
-TEST_F(LegalizerHelperTest, LowerBitCountingCTLZ0) {
+TEST_F(GISelMITest, LowerBitCountingCTLZ0) {
   if (!TM)
     return;
 
   // Declare your legalization info
-  DefineLegalizerInfo(
-      A, { getActionDefinitionsBuilder(G_CTLZ_ZERO_UNDEF).legalFor({s64}); });
+  DefineLegalizerInfo(A, {
+    getActionDefinitionsBuilder(G_CTLZ_ZERO_UNDEF).legalFor({{s64, s64}});
+  });
   // Build
   auto MIBCTLZ =
       B.buildInstr(TargetOpcode::G_CTLZ, {LLT::scalar(64)}, {Copies[0]});
   AInfo Info(MF->getSubtarget());
   DummyGISelObserver Observer;
-  LegalizerHelper Helper(*MF, Info, Observer);
-  ASSERT_TRUE(Helper.lower(*MIBCTLZ, 0, LLT::scalar(64)) ==
+  LegalizerHelper Helper(*MF, Info, Observer, B);
+  EXPECT_TRUE(Helper.lower(*MIBCTLZ, 0, LLT::scalar(64)) ==
               LegalizerHelper::LegalizeResult::Legalized);
 
   auto CheckStr = R"(
@@ -162,24 +232,25 @@ TEST_F(LegalizerHelperTest, LowerBitCountingCTLZ0) {
   )";
 
   // Check
-  ASSERT_TRUE(CheckMachineFunction(*MF, CheckStr));
+  EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
 }
 
 // CTLZ expansion in terms of CTLZ_ZERO_UNDEF if the latter is a libcall
-TEST_F(LegalizerHelperTest, LowerBitCountingCTLZLibcall) {
+TEST_F(GISelMITest, LowerBitCountingCTLZLibcall) {
   if (!TM)
     return;
 
   // Declare your legalization info
-  DefineLegalizerInfo(
-      A, { getActionDefinitionsBuilder(G_CTLZ_ZERO_UNDEF).libcallFor({s64}); });
+  DefineLegalizerInfo(A, {
+    getActionDefinitionsBuilder(G_CTLZ_ZERO_UNDEF).libcallFor({{s64, s64}});
+  });
   // Build
   auto MIBCTLZ =
       B.buildInstr(TargetOpcode::G_CTLZ, {LLT::scalar(64)}, {Copies[0]});
   AInfo Info(MF->getSubtarget());
   DummyGISelObserver Observer;
-  LegalizerHelper Helper(*MF, Info, Observer);
-  ASSERT_TRUE(Helper.lower(*MIBCTLZ, 0, LLT::scalar(64)) ==
+  LegalizerHelper Helper(*MF, Info, Observer, B);
+  EXPECT_TRUE(Helper.lower(*MIBCTLZ, 0, LLT::scalar(64)) ==
               LegalizerHelper::LegalizeResult::Legalized);
 
   auto CheckStr = R"(
@@ -191,17 +262,18 @@ TEST_F(LegalizerHelperTest, LowerBitCountingCTLZLibcall) {
   )";
 
   // Check
-  ASSERT_TRUE(CheckMachineFunction(*MF, CheckStr));
+  EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
 }
 
 // CTLZ expansion
-TEST_F(LegalizerHelperTest, LowerBitCountingCTLZ1) {
+TEST_F(GISelMITest, LowerBitCountingCTLZ1) {
   if (!TM)
     return;
 
   // Declare your legalization info
-  DefineLegalizerInfo(A,
-                      { getActionDefinitionsBuilder(G_CTPOP).legalFor({s8}); });
+  DefineLegalizerInfo(A, {
+    getActionDefinitionsBuilder(G_CTPOP).legalFor({{s8, s8}});
+  });
   // Build
   // Trunc it to s8.
   LLT s8{LLT::scalar(8)};
@@ -209,8 +281,8 @@ TEST_F(LegalizerHelperTest, LowerBitCountingCTLZ1) {
   auto MIBCTLZ = B.buildInstr(TargetOpcode::G_CTLZ, {s8}, {MIBTrunc});
   AInfo Info(MF->getSubtarget());
   DummyGISelObserver Observer;
-  LegalizerHelper Helper(*MF, Info, Observer);
-  ASSERT_TRUE(Helper.lower(*MIBCTLZ, 0, s8) ==
+  LegalizerHelper Helper(*MF, Info, Observer, B);
+  EXPECT_TRUE(Helper.lower(*MIBCTLZ, 0, s8) ==
               LegalizerHelper::LegalizeResult::Legalized);
 
   auto CheckStr = R"(
@@ -230,17 +302,18 @@ TEST_F(LegalizerHelperTest, LowerBitCountingCTLZ1) {
   )";
 
   // Check
-  ASSERT_TRUE(CheckMachineFunction(*MF, CheckStr));
+  EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
 }
 
 // CTLZ widening.
-TEST_F(LegalizerHelperTest, WidenBitCountingCTLZ) {
+TEST_F(GISelMITest, WidenBitCountingCTLZ) {
   if (!TM)
     return;
 
   // Declare your legalization info
-  DefineLegalizerInfo(A,
-                      { getActionDefinitionsBuilder(G_CTLZ).legalFor({s16}); });
+  DefineLegalizerInfo(A, {
+    getActionDefinitionsBuilder(G_CTLZ).legalFor({{s16, s16}});
+  });
   // Build
   // Trunc it to s8.
   LLT s8{LLT::scalar(8)};
@@ -249,8 +322,8 @@ TEST_F(LegalizerHelperTest, WidenBitCountingCTLZ) {
   auto MIBCTLZ = B.buildInstr(TargetOpcode::G_CTLZ, {s8}, {MIBTrunc});
   AInfo Info(MF->getSubtarget());
   DummyGISelObserver Observer;
-  LegalizerHelper Helper(*MF, Info, Observer);
-  ASSERT_TRUE(Helper.widenScalar(*MIBCTLZ, 0, s16) ==
+  LegalizerHelper Helper(*MF, Info, Observer, B);
+  EXPECT_TRUE(Helper.widenScalar(*MIBCTLZ, 1, s16) ==
               LegalizerHelper::LegalizeResult::Legalized);
 
   auto CheckStr = R"(
@@ -263,17 +336,18 @@ TEST_F(LegalizerHelperTest, WidenBitCountingCTLZ) {
   )";
 
   // Check
-  ASSERT_TRUE(CheckMachineFunction(*MF, CheckStr));
+  EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
 }
 
 // CTLZ_ZERO_UNDEF widening.
-TEST_F(LegalizerHelperTest, WidenBitCountingCTLZZeroUndef) {
+TEST_F(GISelMITest, WidenBitCountingCTLZZeroUndef) {
   if (!TM)
     return;
 
   // Declare your legalization info
-  DefineLegalizerInfo(
-      A, { getActionDefinitionsBuilder(G_CTLZ_ZERO_UNDEF).legalFor({s16}); });
+  DefineLegalizerInfo(A, {
+    getActionDefinitionsBuilder(G_CTLZ_ZERO_UNDEF).legalFor({{s16, s16}});
+  });
   // Build
   // Trunc it to s8.
   LLT s8{LLT::scalar(8)};
@@ -283,8 +357,8 @@ TEST_F(LegalizerHelperTest, WidenBitCountingCTLZZeroUndef) {
       B.buildInstr(TargetOpcode::G_CTLZ_ZERO_UNDEF, {s8}, {MIBTrunc});
   AInfo Info(MF->getSubtarget());
   DummyGISelObserver Observer;
-  LegalizerHelper Helper(*MF, Info, Observer);
-  ASSERT_TRUE(Helper.widenScalar(*MIBCTLZ_ZU, 0, s16) ==
+  LegalizerHelper Helper(*MF, Info, Observer, B);
+  EXPECT_TRUE(Helper.widenScalar(*MIBCTLZ_ZU, 1, s16) ==
               LegalizerHelper::LegalizeResult::Legalized);
 
   auto CheckStr = R"(
@@ -297,17 +371,18 @@ TEST_F(LegalizerHelperTest, WidenBitCountingCTLZZeroUndef) {
   )";
 
   // Check
-  ASSERT_TRUE(CheckMachineFunction(*MF, CheckStr));
+  EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
 }
 
 // CTPOP widening.
-TEST_F(LegalizerHelperTest, WidenBitCountingCTPOP) {
+TEST_F(GISelMITest, WidenBitCountingCTPOP) {
   if (!TM)
     return;
 
   // Declare your legalization info
-  DefineLegalizerInfo(
-      A, { getActionDefinitionsBuilder(G_CTPOP).legalFor({s16}); });
+  DefineLegalizerInfo(A, {
+    getActionDefinitionsBuilder(G_CTPOP).legalFor({{s16, s16}});
+  });
   // Build
   // Trunc it to s8.
   LLT s8{LLT::scalar(8)};
@@ -316,8 +391,8 @@ TEST_F(LegalizerHelperTest, WidenBitCountingCTPOP) {
   auto MIBCTPOP = B.buildInstr(TargetOpcode::G_CTPOP, {s8}, {MIBTrunc});
   AInfo Info(MF->getSubtarget());
   DummyGISelObserver Observer;
-  LegalizerHelper Helper(*MF, Info, Observer);
-  ASSERT_TRUE(Helper.widenScalar(*MIBCTPOP, 0, s16) ==
+  LegalizerHelper Helper(*MF, Info, Observer, B);
+  EXPECT_TRUE(Helper.widenScalar(*MIBCTPOP, 1, s16) ==
               LegalizerHelper::LegalizeResult::Legalized);
 
   auto CheckStr = R"(
@@ -328,17 +403,18 @@ TEST_F(LegalizerHelperTest, WidenBitCountingCTPOP) {
   )";
 
   // Check
-  ASSERT_TRUE(CheckMachineFunction(*MF, CheckStr));
+  EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
 }
 
 // CTTZ_ZERO_UNDEF widening.
-TEST_F(LegalizerHelperTest, WidenBitCountingCTTZ_ZERO_UNDEF) {
+TEST_F(GISelMITest, WidenBitCountingCTTZ_ZERO_UNDEF) {
   if (!TM)
     return;
 
   // Declare your legalization info
-  DefineLegalizerInfo(
-      A, { getActionDefinitionsBuilder(G_CTTZ_ZERO_UNDEF).legalFor({s16}); });
+  DefineLegalizerInfo(A, {
+    getActionDefinitionsBuilder(G_CTTZ_ZERO_UNDEF).legalFor({{s16, s16}});
+  });
   // Build
   // Trunc it to s8.
   LLT s8{LLT::scalar(8)};
@@ -348,8 +424,8 @@ TEST_F(LegalizerHelperTest, WidenBitCountingCTTZ_ZERO_UNDEF) {
       B.buildInstr(TargetOpcode::G_CTTZ_ZERO_UNDEF, {s8}, {MIBTrunc});
   AInfo Info(MF->getSubtarget());
   DummyGISelObserver Observer;
-  LegalizerHelper Helper(*MF, Info, Observer);
-  ASSERT_TRUE(Helper.widenScalar(*MIBCTTZ_ZERO_UNDEF, 0, s16) ==
+  LegalizerHelper Helper(*MF, Info, Observer, B);
+  EXPECT_TRUE(Helper.widenScalar(*MIBCTTZ_ZERO_UNDEF, 1, s16) ==
               LegalizerHelper::LegalizeResult::Legalized);
 
   auto CheckStr = R"(
@@ -360,17 +436,18 @@ TEST_F(LegalizerHelperTest, WidenBitCountingCTTZ_ZERO_UNDEF) {
   )";
 
   // Check
-  ASSERT_TRUE(CheckMachineFunction(*MF, CheckStr));
+  EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
 }
 
 // CTTZ widening.
-TEST_F(LegalizerHelperTest, WidenBitCountingCTTZ) {
+TEST_F(GISelMITest, WidenBitCountingCTTZ) {
   if (!TM)
     return;
 
   // Declare your legalization info
-  DefineLegalizerInfo(A,
-                      { getActionDefinitionsBuilder(G_CTTZ).legalFor({s16}); });
+  DefineLegalizerInfo(A, {
+    getActionDefinitionsBuilder(G_CTTZ).legalFor({{s16, s16}});
+  });
   // Build
   // Trunc it to s8.
   LLT s8{LLT::scalar(8)};
@@ -379,8 +456,8 @@ TEST_F(LegalizerHelperTest, WidenBitCountingCTTZ) {
   auto MIBCTTZ = B.buildInstr(TargetOpcode::G_CTTZ, {s8}, {MIBTrunc});
   AInfo Info(MF->getSubtarget());
   DummyGISelObserver Observer;
-  LegalizerHelper Helper(*MF, Info, Observer);
-  ASSERT_TRUE(Helper.widenScalar(*MIBCTTZ, 0, s16) ==
+  LegalizerHelper Helper(*MF, Info, Observer, B);
+  EXPECT_TRUE(Helper.widenScalar(*MIBCTTZ, 1, s16) ==
               LegalizerHelper::LegalizeResult::Legalized);
 
   auto CheckStr = R"(
@@ -393,16 +470,17 @@ TEST_F(LegalizerHelperTest, WidenBitCountingCTTZ) {
   )";
 
   // Check
-  ASSERT_TRUE(CheckMachineFunction(*MF, CheckStr));
+  EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
 }
 // UADDO widening.
-TEST_F(LegalizerHelperTest, WidenUADDO) {
+TEST_F(GISelMITest, WidenUADDO) {
   if (!TM)
     return;
 
   // Declare your legalization info
-  DefineLegalizerInfo(A,
-                      { getActionDefinitionsBuilder(G_ADD).legalFor({s16}); });
+  DefineLegalizerInfo(A, {
+    getActionDefinitionsBuilder(G_ADD).legalFor({{s16, s16}});
+  });
   // Build
   // Trunc it to s8.
   LLT s8{LLT::scalar(8)};
@@ -413,8 +491,8 @@ TEST_F(LegalizerHelperTest, WidenUADDO) {
       B.buildInstr(TargetOpcode::G_UADDO, {s8, CarryReg}, {MIBTrunc, MIBTrunc});
   AInfo Info(MF->getSubtarget());
   DummyGISelObserver Observer;
-  LegalizerHelper Helper(*MF, Info, Observer);
-  ASSERT_TRUE(Helper.widenScalar(*MIBUAddO, 0, s16) ==
+  LegalizerHelper Helper(*MF, Info, Observer, B);
+  EXPECT_TRUE(Helper.widenScalar(*MIBUAddO, 0, s16) ==
               LegalizerHelper::LegalizeResult::Legalized);
 
   auto CheckStr = R"(
@@ -429,17 +507,18 @@ TEST_F(LegalizerHelperTest, WidenUADDO) {
   )";
 
   // Check
-  ASSERT_TRUE(CheckMachineFunction(*MF, CheckStr));
+  EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
 }
 
 // USUBO widening.
-TEST_F(LegalizerHelperTest, WidenUSUBO) {
+TEST_F(GISelMITest, WidenUSUBO) {
   if (!TM)
     return;
 
   // Declare your legalization info
-  DefineLegalizerInfo(A,
-                      { getActionDefinitionsBuilder(G_SUB).legalFor({s16}); });
+  DefineLegalizerInfo(A, {
+    getActionDefinitionsBuilder(G_SUB).legalFor({{s16, s16}});
+  });
   // Build
   // Trunc it to s8.
   LLT s8{LLT::scalar(8)};
@@ -450,8 +529,8 @@ TEST_F(LegalizerHelperTest, WidenUSUBO) {
       B.buildInstr(TargetOpcode::G_USUBO, {s8, CarryReg}, {MIBTrunc, MIBTrunc});
   AInfo Info(MF->getSubtarget());
   DummyGISelObserver Observer;
-  LegalizerHelper Helper(*MF, Info, Observer);
-  ASSERT_TRUE(Helper.widenScalar(*MIBUSUBO, 0, s16) ==
+  LegalizerHelper Helper(*MF, Info, Observer, B);
+  EXPECT_TRUE(Helper.widenScalar(*MIBUSUBO, 0, s16) ==
               LegalizerHelper::LegalizeResult::Legalized);
 
   auto CheckStr = R"(
@@ -466,6 +545,6 @@ TEST_F(LegalizerHelperTest, WidenUSUBO) {
   )";
 
   // Check
-  ASSERT_TRUE(CheckMachineFunction(*MF, CheckStr));
+  EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
 }
 } // namespace
diff --git a/unittests/CodeGen/GlobalISel/LegalizerInfoTest.cpp b/unittests/CodeGen/GlobalISel/LegalizerInfoTest.cpp
index 42371e63da9d5bc1f281f299614ce1bc75e090dc..3617388c04e9de8656610f0ea46aa75e6d5d4448 100644
--- a/unittests/CodeGen/GlobalISel/LegalizerInfoTest.cpp
+++ b/unittests/CodeGen/GlobalISel/LegalizerInfoTest.cpp
@@ -1,14 +1,14 @@
 //===- llvm/unittest/CodeGen/GlobalISel/LegalizerInfoTest.cpp -------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #include "llvm/CodeGen/GlobalISel/LegalizerInfo.h"
 #include "llvm/CodeGen/TargetOpcodes.h"
+#include "GISelMITest.h"
 #include "gtest/gtest.h"
 
 using namespace llvm;
@@ -34,12 +34,9 @@ operator<<(std::ostream &OS, const LegalizeAction Act) {
   return OS;
 }
 
-std::ostream &
-operator<<(std::ostream &OS, const llvm::LLT Ty) {
-  std::string Repr;
-  raw_string_ostream SS{Repr};
-  Ty.print(SS);
-  OS << SS.str();
+std::ostream &operator<<(std::ostream &OS, const llvm::LegalizeActionStep Ty) {
+  OS << "LegalizeActionStep(" << Ty.Action << ", " << Ty.TypeIdx << ", "
+     << Ty.NewType << ')';
   return OS;
 }
 }
@@ -62,28 +59,28 @@ TEST(LegalizerInfoTest, ScalarRISC) {
 
   for (unsigned opcode : {G_ADD, G_SUB}) {
     // Check we infer the correct types and actually do what we're told.
-    ASSERT_EQ(L.getAction({opcode, {LLT::scalar(8)}}),
+    EXPECT_EQ(L.getAction({opcode, {LLT::scalar(8)}}),
               LegalizeActionStep(WidenScalar, 0, LLT::scalar(32)));
-    ASSERT_EQ(L.getAction({opcode, {LLT::scalar(16)}}),
+    EXPECT_EQ(L.getAction({opcode, {LLT::scalar(16)}}),
               LegalizeActionStep(WidenScalar, 0, LLT::scalar(32)));
-    ASSERT_EQ(L.getAction({opcode, {LLT::scalar(32)}}),
+    EXPECT_EQ(L.getAction({opcode, {LLT::scalar(32)}}),
               LegalizeActionStep(Legal, 0, LLT{}));
-    ASSERT_EQ(L.getAction({opcode, {LLT::scalar(64)}}),
+    EXPECT_EQ(L.getAction({opcode, {LLT::scalar(64)}}),
               LegalizeActionStep(Legal, 0, LLT{}));
 
     // Make sure the default for over-sized types applies.
-    ASSERT_EQ(L.getAction({opcode, {LLT::scalar(128)}}),
+    EXPECT_EQ(L.getAction({opcode, {LLT::scalar(128)}}),
               LegalizeActionStep(NarrowScalar, 0, LLT::scalar(64)));
     // Make sure we also handle unusual sizes
-    ASSERT_EQ(L.getAction({opcode, {LLT::scalar(1)}}),
+    EXPECT_EQ(L.getAction({opcode, {LLT::scalar(1)}}),
               LegalizeActionStep(WidenScalar, 0, LLT::scalar(32)));
-    ASSERT_EQ(L.getAction({opcode, {LLT::scalar(31)}}),
+    EXPECT_EQ(L.getAction({opcode, {LLT::scalar(31)}}),
               LegalizeActionStep(WidenScalar, 0, LLT::scalar(32)));
-    ASSERT_EQ(L.getAction({opcode, {LLT::scalar(33)}}),
+    EXPECT_EQ(L.getAction({opcode, {LLT::scalar(33)}}),
               LegalizeActionStep(WidenScalar, 0, LLT::scalar(64)));
-    ASSERT_EQ(L.getAction({opcode, {LLT::scalar(63)}}),
+    EXPECT_EQ(L.getAction({opcode, {LLT::scalar(63)}}),
               LegalizeActionStep(WidenScalar, 0, LLT::scalar(64)));
-    ASSERT_EQ(L.getAction({opcode, {LLT::scalar(65)}}),
+    EXPECT_EQ(L.getAction({opcode, {LLT::scalar(65)}}),
               LegalizeActionStep(NarrowScalar, 0, LLT::scalar(64)));
   }
 }
@@ -108,18 +105,18 @@ TEST(LegalizerInfoTest, VectorRISC) {
 
   // Check we infer the correct types and actually do what we're told for some
   // simple cases.
-  ASSERT_EQ(L.getAction({G_ADD, {LLT::vector(8, 8)}}),
+  EXPECT_EQ(L.getAction({G_ADD, {LLT::vector(8, 8)}}),
             LegalizeActionStep(Legal, 0, LLT{}));
-  ASSERT_EQ(L.getAction({G_ADD, {LLT::vector(8, 7)}}),
+  EXPECT_EQ(L.getAction({G_ADD, {LLT::vector(8, 7)}}),
             LegalizeActionStep(WidenScalar, 0, LLT::vector(8, 8)));
-  ASSERT_EQ(L.getAction({G_ADD, {LLT::vector(2, 8)}}),
+  EXPECT_EQ(L.getAction({G_ADD, {LLT::vector(2, 8)}}),
             LegalizeActionStep(MoreElements, 0, LLT::vector(8, 8)));
-  ASSERT_EQ(L.getAction({G_ADD, {LLT::vector(8, 32)}}),
+  EXPECT_EQ(L.getAction({G_ADD, {LLT::vector(8, 32)}}),
             LegalizeActionStep(FewerElements, 0, LLT::vector(4, 32)));
   // Check a few non-power-of-2 sizes:
-  ASSERT_EQ(L.getAction({G_ADD, {LLT::vector(3, 3)}}),
+  EXPECT_EQ(L.getAction({G_ADD, {LLT::vector(3, 3)}}),
             LegalizeActionStep(WidenScalar, 0, LLT::vector(3, 8)));
-  ASSERT_EQ(L.getAction({G_ADD, {LLT::vector(3, 8)}}),
+  EXPECT_EQ(L.getAction({G_ADD, {LLT::vector(3, 8)}}),
             LegalizeActionStep(MoreElements, 0, LLT::vector(8, 8)));
 }
 
@@ -139,14 +136,14 @@ TEST(LegalizerInfoTest, MultipleTypes) {
   L.computeTables();
 
   // Check we infer the correct types and actually do what we're told.
-  ASSERT_EQ(L.getAction({G_PTRTOINT, {s64, p0}}),
+  EXPECT_EQ(L.getAction({G_PTRTOINT, {s64, p0}}),
             LegalizeActionStep(Legal, 0, LLT{}));
 
   // Make sure we also handle unusual sizes
-  ASSERT_EQ(
+  EXPECT_EQ(
       L.getAction({G_PTRTOINT, {LLT::scalar(65), s64}}),
       LegalizeActionStep(NarrowScalar, 0, s64));
-  ASSERT_EQ(
+  EXPECT_EQ(
       L.getAction({G_PTRTOINT, {s64, LLT::pointer(0, 32)}}),
       LegalizeActionStep(Unsupported, 1, LLT::pointer(0, 32)));
 }
@@ -164,9 +161,9 @@ TEST(LegalizerInfoTest, MultipleSteps) {
 
   L.computeTables();
 
-  ASSERT_EQ(L.getAction({G_UREM, {LLT::scalar(16)}}),
+  EXPECT_EQ(L.getAction({G_UREM, {LLT::scalar(16)}}),
             LegalizeActionStep(WidenScalar, 0, LLT::scalar(32)));
-  ASSERT_EQ(L.getAction({G_UREM, {LLT::scalar(32)}}),
+  EXPECT_EQ(L.getAction({G_UREM, {LLT::scalar(32)}}),
             LegalizeActionStep(Lower, 0, LLT::scalar(32)));
 }
 
@@ -182,20 +179,180 @@ TEST(LegalizerInfoTest, SizeChangeStrategy) {
 
   // Check we infer the correct types and actually do what we're told.
   for (unsigned Size : {1, 8, 16, 32}) {
-    ASSERT_EQ(L.getAction({G_UREM, {LLT::scalar(Size)}}),
+    EXPECT_EQ(L.getAction({G_UREM, {LLT::scalar(Size)}}),
               LegalizeActionStep(Legal, 0, LLT{}));
   }
-  ASSERT_EQ(L.getAction({G_UREM, {LLT::scalar(2)}}),
+  EXPECT_EQ(L.getAction({G_UREM, {LLT::scalar(2)}}),
             LegalizeActionStep(WidenScalar, 0, LLT::scalar(8)));
-  ASSERT_EQ(L.getAction({G_UREM, {LLT::scalar(7)}}),
+  EXPECT_EQ(L.getAction({G_UREM, {LLT::scalar(7)}}),
             LegalizeActionStep(WidenScalar, 0, LLT::scalar(8)));
-  ASSERT_EQ(L.getAction({G_UREM, {LLT::scalar(9)}}),
+  EXPECT_EQ(L.getAction({G_UREM, {LLT::scalar(9)}}),
             LegalizeActionStep(WidenScalar, 0, LLT::scalar(16)));
-  ASSERT_EQ(L.getAction({G_UREM, {LLT::scalar(17)}}),
+  EXPECT_EQ(L.getAction({G_UREM, {LLT::scalar(17)}}),
             LegalizeActionStep(WidenScalar, 0, LLT::scalar(32)));
-  ASSERT_EQ(L.getAction({G_UREM, {LLT::scalar(31)}}),
+  EXPECT_EQ(L.getAction({G_UREM, {LLT::scalar(31)}}),
             LegalizeActionStep(WidenScalar, 0, LLT::scalar(32)));
-  ASSERT_EQ(L.getAction({G_UREM, {LLT::scalar(33)}}),
+  EXPECT_EQ(L.getAction({G_UREM, {LLT::scalar(33)}}),
             LegalizeActionStep(Unsupported, 0, LLT::scalar(33)));
 }
 }
+
+#define EXPECT_ACTION(Action, Index, Type, Query)                              \
+  do {                                                                         \
+    auto A = LI.getAction(Query);                                              \
+    EXPECT_EQ(LegalizeActionStep(Action, Index, Type), A) << A;                \
+  } while (0)
+
+TEST(LegalizerInfoTest, RuleSets) {
+  using namespace TargetOpcode;
+
+  const LLT s5 = LLT::scalar(5);
+  const LLT s8 = LLT::scalar(8);
+  const LLT s16 = LLT::scalar(16);
+  const LLT s32 = LLT::scalar(32);
+  const LLT s33 = LLT::scalar(33);
+  const LLT s64 = LLT::scalar(64);
+
+  const LLT v2s5 = LLT::vector(2, 5);
+  const LLT v2s8 = LLT::vector(2, 8);
+  const LLT v2s16 = LLT::vector(2, 16);
+  const LLT v2s32 = LLT::vector(2, 32);
+  const LLT v3s32 = LLT::vector(3, 32);
+  const LLT v4s32 = LLT::vector(4, 32);
+  const LLT v2s33 = LLT::vector(2, 33);
+  const LLT v2s64 = LLT::vector(2, 64);
+
+  const LLT p0 = LLT::pointer(0, 32);
+  const LLT v3p0 = LLT::vector(3, p0);
+  const LLT v4p0 = LLT::vector(4, p0);
+
+  {
+    LegalizerInfo LI;
+
+    LI.getActionDefinitionsBuilder(G_IMPLICIT_DEF)
+      .legalFor({v4s32, v4p0})
+      .moreElementsToNextPow2(0);
+    LI.computeTables();
+
+    EXPECT_ACTION(Unsupported, 0, LLT(), LegalityQuery(G_IMPLICIT_DEF, {s32}));
+    EXPECT_ACTION(Unsupported, 0, LLT(), LegalityQuery(G_IMPLICIT_DEF, {v2s32}));
+    EXPECT_ACTION(MoreElements, 0, v4p0, LegalityQuery(G_IMPLICIT_DEF, {v3p0}));
+    EXPECT_ACTION(MoreElements, 0, v4s32, LegalityQuery(G_IMPLICIT_DEF, {v3s32}));
+  }
+
+  // Test minScalarOrElt
+  {
+    LegalizerInfo LI;
+    LI.getActionDefinitionsBuilder(G_OR)
+      .legalFor({s32})
+      .minScalarOrElt(0, s32);
+    LI.computeTables();
+
+    EXPECT_ACTION(WidenScalar, 0, s32, LegalityQuery(G_OR, {s16}));
+    EXPECT_ACTION(WidenScalar, 0, v2s32, LegalityQuery(G_OR, {v2s16}));
+  }
+
+  // Test maxScalarOrELt
+  {
+    LegalizerInfo LI;
+    LI.getActionDefinitionsBuilder(G_AND)
+      .legalFor({s16})
+      .maxScalarOrElt(0, s16);
+    LI.computeTables();
+
+    EXPECT_ACTION(NarrowScalar, 0, s16, LegalityQuery(G_AND, {s32}));
+    EXPECT_ACTION(NarrowScalar, 0, v2s16, LegalityQuery(G_AND, {v2s32}));
+  }
+
+  // Test clampScalarOrElt
+  {
+    LegalizerInfo LI;
+    LI.getActionDefinitionsBuilder(G_XOR)
+      .legalFor({s16})
+      .clampScalarOrElt(0, s16, s32);
+    LI.computeTables();
+
+    EXPECT_ACTION(NarrowScalar, 0, s32, LegalityQuery(G_XOR, {s64}));
+    EXPECT_ACTION(WidenScalar, 0, s16, LegalityQuery(G_XOR, {s8}));
+
+    // Make sure the number of elements is preserved.
+    EXPECT_ACTION(NarrowScalar, 0, v2s32, LegalityQuery(G_XOR, {v2s64}));
+    EXPECT_ACTION(WidenScalar, 0, v2s16, LegalityQuery(G_XOR, {v2s8}));
+  }
+
+  // Test minScalar
+  {
+    LegalizerInfo LI;
+    LI.getActionDefinitionsBuilder(G_OR)
+      .legalFor({s32})
+      .minScalar(0, s32);
+    LI.computeTables();
+
+    // Only handle scalars, ignore vectors.
+    EXPECT_ACTION(WidenScalar, 0, s32, LegalityQuery(G_OR, {s16}));
+    EXPECT_ACTION(Unsupported, 0, LLT(), LegalityQuery(G_OR, {v2s16}));
+  }
+
+  // Test maxScalar
+  {
+    LegalizerInfo LI;
+    LI.getActionDefinitionsBuilder(G_AND)
+      .legalFor({s16})
+      .maxScalar(0, s16);
+    LI.computeTables();
+
+    // Only handle scalars, ignore vectors.
+    EXPECT_ACTION(NarrowScalar, 0, s16, LegalityQuery(G_AND, {s32}));
+    EXPECT_ACTION(Unsupported, 0, LLT(), LegalityQuery(G_AND, {v2s32}));
+  }
+
+  // Test clampScalar
+  {
+    LegalizerInfo LI;
+
+    LI.getActionDefinitionsBuilder(G_XOR)
+      .legalFor({s16})
+      .clampScalar(0, s16, s32);
+    LI.computeTables();
+
+    EXPECT_ACTION(NarrowScalar, 0, s32, LegalityQuery(G_XOR, {s64}));
+    EXPECT_ACTION(WidenScalar, 0, s16, LegalityQuery(G_XOR, {s8}));
+
+    // Only handle scalars, ignore vectors.
+    EXPECT_ACTION(Unsupported, 0, LLT(), LegalityQuery(G_XOR, {v2s64}));
+    EXPECT_ACTION(Unsupported, 0, LLT(), LegalityQuery(G_XOR, {v2s8}));
+  }
+
+  // Test widenScalarOrEltToNextPow2
+  {
+    LegalizerInfo LI;
+
+    LI.getActionDefinitionsBuilder(G_AND)
+      .legalFor({s32})
+      .widenScalarOrEltToNextPow2(0, 32);
+    LI.computeTables();
+
+    // Handle scalars and vectors
+    EXPECT_ACTION(WidenScalar, 0, s32, LegalityQuery(G_AND, {s5}));
+    EXPECT_ACTION(WidenScalar, 0, v2s32, LegalityQuery(G_AND, {v2s5}));
+    EXPECT_ACTION(WidenScalar, 0, s64, LegalityQuery(G_AND, {s33}));
+    EXPECT_ACTION(WidenScalar, 0, v2s64, LegalityQuery(G_AND, {v2s33}));
+  }
+
+  // Test widenScalarToNextPow2
+  {
+    LegalizerInfo LI;
+
+    LI.getActionDefinitionsBuilder(G_AND)
+      .legalFor({s32})
+      .widenScalarToNextPow2(0, 32);
+    LI.computeTables();
+
+    EXPECT_ACTION(WidenScalar, 0, s32, LegalityQuery(G_AND, {s5}));
+    EXPECT_ACTION(WidenScalar, 0, s64, LegalityQuery(G_AND, {s33}));
+
+    // Do nothing for vectors.
+    EXPECT_ACTION(Unsupported, 0, LLT(), LegalityQuery(G_AND, {v2s5}));
+    EXPECT_ACTION(Unsupported, 0, LLT(), LegalityQuery(G_AND, {v2s33}));
+  }
+}
diff --git a/unittests/CodeGen/GlobalISel/MachineIRBuilderTest.cpp b/unittests/CodeGen/GlobalISel/MachineIRBuilderTest.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..d64f5747e729e33c2a465580f306fb57e86a786b
--- /dev/null
+++ b/unittests/CodeGen/GlobalISel/MachineIRBuilderTest.cpp
@@ -0,0 +1,92 @@
+//===- MachineIRBuilderTest.cpp -------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "GISelMITest.h"
+#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
+
+TEST_F(GISelMITest, TestBuildConstantFConstant) {
+  if (!TM)
+    return;
+
+  B.buildConstant(LLT::scalar(32), 42);
+  B.buildFConstant(LLT::scalar(32), 1.0);
+
+  B.buildConstant(LLT::vector(2, 32), 99);
+  B.buildFConstant(LLT::vector(2, 32), 2.0);
+
+  auto CheckStr = R"(
+  CHECK: [[CONST0:%[0-9]+]]:_(s32) = G_CONSTANT i32 42
+  CHECK: [[FCONST0:%[0-9]+]]:_(s32) = G_FCONSTANT float 1.000000e+00
+  CHECK: [[CONST1:%[0-9]+]]:_(s32) = G_CONSTANT i32 99
+  CHECK: [[VEC0:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[CONST1]]:_(s32), [[CONST1]]:_(s32)
+  CHECK: [[FCONST1:%[0-9]+]]:_(s32) = G_FCONSTANT float 2.000000e+00
+  CHECK: [[VEC1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[FCONST1]]:_(s32), [[FCONST1]]:_(s32)
+  )";
+
+  EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
+}
+
+
+#ifdef GTEST_HAS_DEATH_TEST
+#ifndef NDEBUG
+
+TEST_F(GISelMITest, TestBuildConstantFConstantDeath) {
+  if (!TM)
+    return;
+
+  LLVMContext &Ctx = MF->getFunction().getContext();
+  APInt APV32(32, 12345);
+
+  // Test APInt version breaks
+  EXPECT_DEATH(B.buildConstant(LLT::scalar(16), APV32),
+               "creating constant with the wrong size");
+  EXPECT_DEATH(B.buildConstant(LLT::vector(2, 16), APV32),
+               "creating constant with the wrong size");
+
+  // Test ConstantInt version breaks
+  ConstantInt *CI = ConstantInt::get(Ctx, APV32);
+  EXPECT_DEATH(B.buildConstant(LLT::scalar(16), *CI),
+               "creating constant with the wrong size");
+  EXPECT_DEATH(B.buildConstant(LLT::vector(2, 16), *CI),
+               "creating constant with the wrong size");
+
+  APFloat DoubleVal(APFloat::IEEEdouble());
+  ConstantFP *CF = ConstantFP::get(Ctx, DoubleVal);
+  EXPECT_DEATH(B.buildFConstant(LLT::scalar(16), *CF),
+               "creating fconstant with the wrong size");
+  EXPECT_DEATH(B.buildFConstant(LLT::vector(2, 16), *CF),
+               "creating fconstant with the wrong size");
+}
+
+#endif
+#endif
+
+TEST_F(GISelMITest, DstOpSrcOp) {
+  if (!TM)
+    return;
+
+  SmallVector<unsigned, 4> Copies;
+  collectCopies(Copies, MF);
+
+  LLT s64 = LLT::scalar(64);
+  auto MIBAdd = B.buildAdd(s64, Copies[0], Copies[1]);
+
+  // Test SrcOp and DstOp can be constructed directly from MachineOperand by
+  // copying the instruction
+  B.buildAdd(MIBAdd->getOperand(0), MIBAdd->getOperand(1), MIBAdd->getOperand(2));
+
+
+  auto CheckStr = R"(
+  ; CHECK: [[COPY0:%[0-9]+]]:_(s64) = COPY $x0
+  ; CHECK: [[COPY1:%[0-9]+]]:_(s64) = COPY $x1
+  ; CHECK: [[ADD:%[0-9]+]]:_(s64) = G_ADD [[COPY0]]:_, [[COPY1]]:_
+  ; CHECK: [[ADD]]:_(s64) = G_ADD [[COPY0]]:_, [[COPY1]]:_
+  )";
+
+  EXPECT_TRUE(CheckMachineFunction(*MF, CheckStr)) << *MF;
+}
diff --git a/unittests/CodeGen/GlobalISel/PatternMatchTest.cpp b/unittests/CodeGen/GlobalISel/PatternMatchTest.cpp
index 466a2dd6b06c2794b3ad93f8c26186da7d60cfe8..1bb07a9fa1e6b383d4bfa6e48c903b18b010300a 100644
--- a/unittests/CodeGen/GlobalISel/PatternMatchTest.cpp
+++ b/unittests/CodeGen/GlobalISel/PatternMatchTest.cpp
@@ -1,9 +1,8 @@
 //===- PatternMatchTest.cpp -----------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -138,8 +137,8 @@ TEST(PatternMatchInstr, MatchIntConstant) {
   auto MIBCst = B.buildConstant(LLT::scalar(64), 42);
   int64_t Cst;
   bool match = mi_match(MIBCst->getOperand(0).getReg(), MRI, m_ICst(Cst));
-  ASSERT_TRUE(match);
-  ASSERT_EQ(Cst, 42);
+  EXPECT_TRUE(match);
+  EXPECT_EQ(Cst, 42);
 }
 
 TEST(PatternMatchInstr, MatchBinaryOp) {
@@ -161,13 +160,13 @@ TEST(PatternMatchInstr, MatchBinaryOp) {
   // Test case for no bind.
   bool match =
       mi_match(MIBAdd->getOperand(0).getReg(), MRI, m_GAdd(m_Reg(), m_Reg()));
-  ASSERT_TRUE(match);
+  EXPECT_TRUE(match);
   unsigned Src0, Src1, Src2;
   match = mi_match(MIBAdd->getOperand(0).getReg(), MRI,
                    m_GAdd(m_Reg(Src0), m_Reg(Src1)));
-  ASSERT_TRUE(match);
-  ASSERT_EQ(Src0, Copies[0]);
-  ASSERT_EQ(Src1, Copies[1]);
+  EXPECT_TRUE(match);
+  EXPECT_EQ(Src0, Copies[0]);
+  EXPECT_EQ(Src1, Copies[1]);
 
   // Build MUL(ADD %0, %1), %2
   auto MIBMul = B.buildMul(s64, MIBAdd, Copies[2]);
@@ -175,17 +174,17 @@ TEST(PatternMatchInstr, MatchBinaryOp) {
   // Try to match MUL.
   match = mi_match(MIBMul->getOperand(0).getReg(), MRI,
                    m_GMul(m_Reg(Src0), m_Reg(Src1)));
-  ASSERT_TRUE(match);
-  ASSERT_EQ(Src0, MIBAdd->getOperand(0).getReg());
-  ASSERT_EQ(Src1, Copies[2]);
+  EXPECT_TRUE(match);
+  EXPECT_EQ(Src0, MIBAdd->getOperand(0).getReg());
+  EXPECT_EQ(Src1, Copies[2]);
 
   // Try to match MUL(ADD)
   match = mi_match(MIBMul->getOperand(0).getReg(), MRI,
                    m_GMul(m_GAdd(m_Reg(Src0), m_Reg(Src1)), m_Reg(Src2)));
-  ASSERT_TRUE(match);
-  ASSERT_EQ(Src0, Copies[0]);
-  ASSERT_EQ(Src1, Copies[1]);
-  ASSERT_EQ(Src2, Copies[2]);
+  EXPECT_TRUE(match);
+  EXPECT_EQ(Src0, Copies[0]);
+  EXPECT_EQ(Src1, Copies[1]);
+  EXPECT_EQ(Src2, Copies[2]);
 
   // Test Commutativity.
   auto MIBMul2 = B.buildMul(s64, Copies[0], B.buildConstant(s64, 42));
@@ -194,50 +193,50 @@ TEST(PatternMatchInstr, MatchBinaryOp) {
   int64_t Cst;
   match = mi_match(MIBMul2->getOperand(0).getReg(), MRI,
                    m_GMul(m_ICst(Cst), m_Reg(Src0)));
-  ASSERT_TRUE(match);
-  ASSERT_EQ(Cst, 42);
-  ASSERT_EQ(Src0, Copies[0]);
+  EXPECT_TRUE(match);
+  EXPECT_EQ(Cst, 42);
+  EXPECT_EQ(Src0, Copies[0]);
 
   // Make sure commutative doesn't work with something like SUB.
   auto MIBSub = B.buildSub(s64, Copies[0], B.buildConstant(s64, 42));
   match = mi_match(MIBSub->getOperand(0).getReg(), MRI,
                    m_GSub(m_ICst(Cst), m_Reg(Src0)));
-  ASSERT_FALSE(match);
+  EXPECT_FALSE(match);
 
   auto MIBFMul = B.buildInstr(TargetOpcode::G_FMUL, {s64},
                               {Copies[0], B.buildConstant(s64, 42)});
   // Match and test commutativity for FMUL.
   match = mi_match(MIBFMul->getOperand(0).getReg(), MRI,
                    m_GFMul(m_ICst(Cst), m_Reg(Src0)));
-  ASSERT_TRUE(match);
-  ASSERT_EQ(Cst, 42);
-  ASSERT_EQ(Src0, Copies[0]);
+  EXPECT_TRUE(match);
+  EXPECT_EQ(Cst, 42);
+  EXPECT_EQ(Src0, Copies[0]);
 
   // FSUB
   auto MIBFSub = B.buildInstr(TargetOpcode::G_FSUB, {s64},
                               {Copies[0], B.buildConstant(s64, 42)});
   match = mi_match(MIBFSub->getOperand(0).getReg(), MRI,
                    m_GFSub(m_Reg(Src0), m_Reg()));
-  ASSERT_TRUE(match);
-  ASSERT_EQ(Src0, Copies[0]);
+  EXPECT_TRUE(match);
+  EXPECT_EQ(Src0, Copies[0]);
 
   // Build AND %0, %1
   auto MIBAnd = B.buildAnd(s64, Copies[0], Copies[1]);
   // Try to match AND.
   match = mi_match(MIBAnd->getOperand(0).getReg(), MRI,
                    m_GAnd(m_Reg(Src0), m_Reg(Src1)));
-  ASSERT_TRUE(match);
-  ASSERT_EQ(Src0, Copies[0]);
-  ASSERT_EQ(Src1, Copies[1]);
+  EXPECT_TRUE(match);
+  EXPECT_EQ(Src0, Copies[0]);
+  EXPECT_EQ(Src1, Copies[1]);
 
   // Build OR %0, %1
   auto MIBOr = B.buildOr(s64, Copies[0], Copies[1]);
   // Try to match OR.
   match = mi_match(MIBOr->getOperand(0).getReg(), MRI,
                    m_GOr(m_Reg(Src0), m_Reg(Src1)));
-  ASSERT_TRUE(match);
-  ASSERT_EQ(Src0, Copies[0]);
-  ASSERT_EQ(Src1, Copies[1]);
+  EXPECT_TRUE(match);
+  EXPECT_EQ(Src0, Copies[0]);
+  EXPECT_EQ(Src1, Copies[1]);
 
   // Try to use the FoldableInstructionsBuilder to build binary ops.
   ConstantFoldingMIRBuilder CFB(B.getState());
@@ -246,15 +245,15 @@ TEST(PatternMatchInstr, MatchBinaryOp) {
       CFB.buildAdd(s32, CFB.buildConstant(s32, 0), CFB.buildConstant(s32, 1));
   // This should be a constant now.
   match = mi_match(MIBCAdd->getOperand(0).getReg(), MRI, m_ICst(Cst));
-  ASSERT_TRUE(match);
-  ASSERT_EQ(Cst, 1);
+  EXPECT_TRUE(match);
+  EXPECT_EQ(Cst, 1);
   auto MIBCAdd1 =
       CFB.buildInstr(TargetOpcode::G_ADD, {s32},
                      {CFB.buildConstant(s32, 0), CFB.buildConstant(s32, 1)});
   // This should be a constant now.
   match = mi_match(MIBCAdd1->getOperand(0).getReg(), MRI, m_ICst(Cst));
-  ASSERT_TRUE(match);
-  ASSERT_EQ(Cst, 1);
+  EXPECT_TRUE(match);
+  EXPECT_EQ(Cst, 1);
 
   // Try one of the other constructors of MachineIRBuilder to make sure it's
   // compatible.
@@ -265,8 +264,8 @@ TEST(PatternMatchInstr, MatchBinaryOp) {
                       {CFB1.buildConstant(s32, 1), CFB1.buildConstant(s32, 1)});
   // This should be a constant now.
   match = mi_match(MIBCSub->getOperand(0).getReg(), MRI, m_ICst(Cst));
-  ASSERT_TRUE(match);
-  ASSERT_EQ(Cst, 0);
+  EXPECT_TRUE(match);
+  EXPECT_EQ(Cst, 0);
 }
 
 TEST(PatternMatchInstr, MatchFPUnaryOp) {
@@ -291,53 +290,53 @@ TEST(PatternMatchInstr, MatchFPUnaryOp) {
   // Match G_FABS.
   auto MIBFabs = B.buildInstr(TargetOpcode::G_FABS, {s32}, {Copy0s32});
   bool match = mi_match(MIBFabs->getOperand(0).getReg(), MRI, m_GFabs(m_Reg()));
-  ASSERT_TRUE(match);
+  EXPECT_TRUE(match);
 
   unsigned Src;
   auto MIBFNeg = B.buildInstr(TargetOpcode::G_FNEG, {s32}, {Copy0s32});
   match = mi_match(MIBFNeg->getOperand(0).getReg(), MRI, m_GFNeg(m_Reg(Src)));
-  ASSERT_TRUE(match);
-  ASSERT_EQ(Src, Copy0s32->getOperand(0).getReg());
+  EXPECT_TRUE(match);
+  EXPECT_EQ(Src, Copy0s32->getOperand(0).getReg());
 
   match = mi_match(MIBFabs->getOperand(0).getReg(), MRI, m_GFabs(m_Reg(Src)));
-  ASSERT_TRUE(match);
-  ASSERT_EQ(Src, Copy0s32->getOperand(0).getReg());
+  EXPECT_TRUE(match);
+  EXPECT_EQ(Src, Copy0s32->getOperand(0).getReg());
 
   // Build and match FConstant.
   auto MIBFCst = B.buildFConstant(s32, .5);
   const ConstantFP *TmpFP{};
   match = mi_match(MIBFCst->getOperand(0).getReg(), MRI, m_GFCst(TmpFP));
-  ASSERT_TRUE(match);
-  ASSERT_TRUE(TmpFP);
+  EXPECT_TRUE(match);
+  EXPECT_TRUE(TmpFP);
   APFloat APF((float).5);
   auto *CFP = ConstantFP::get(Context, APF);
-  ASSERT_EQ(CFP, TmpFP);
+  EXPECT_EQ(CFP, TmpFP);
 
   // Build double float.
   LLT s64 = LLT::scalar(64);
   auto MIBFCst64 = B.buildFConstant(s64, .5);
   const ConstantFP *TmpFP64{};
   match = mi_match(MIBFCst64->getOperand(0).getReg(), MRI, m_GFCst(TmpFP64));
-  ASSERT_TRUE(match);
-  ASSERT_TRUE(TmpFP64);
+  EXPECT_TRUE(match);
+  EXPECT_TRUE(TmpFP64);
   APFloat APF64(.5);
   auto CFP64 = ConstantFP::get(Context, APF64);
-  ASSERT_EQ(CFP64, TmpFP64);
-  ASSERT_NE(TmpFP64, TmpFP);
+  EXPECT_EQ(CFP64, TmpFP64);
+  EXPECT_NE(TmpFP64, TmpFP);
 
   // Build half float.
   LLT s16 = LLT::scalar(16);
   auto MIBFCst16 = B.buildFConstant(s16, .5);
   const ConstantFP *TmpFP16{};
   match = mi_match(MIBFCst16->getOperand(0).getReg(), MRI, m_GFCst(TmpFP16));
-  ASSERT_TRUE(match);
-  ASSERT_TRUE(TmpFP16);
+  EXPECT_TRUE(match);
+  EXPECT_TRUE(TmpFP16);
   bool Ignored;
   APFloat APF16(.5);
   APF16.convert(APFloat::IEEEhalf(), APFloat::rmNearestTiesToEven, &Ignored);
   auto CFP16 = ConstantFP::get(Context, APF16);
-  ASSERT_EQ(TmpFP16, CFP16);
-  ASSERT_NE(TmpFP16, TmpFP);
+  EXPECT_EQ(TmpFP16, CFP16);
+  EXPECT_NE(TmpFP16, TmpFP);
 }
 
 TEST(PatternMatchInstr, MatchExtendsTrunc) {
@@ -364,36 +363,36 @@ TEST(PatternMatchInstr, MatchExtendsTrunc) {
   unsigned Src0;
   bool match =
       mi_match(MIBTrunc->getOperand(0).getReg(), MRI, m_GTrunc(m_Reg(Src0)));
-  ASSERT_TRUE(match);
-  ASSERT_EQ(Src0, Copies[0]);
+  EXPECT_TRUE(match);
+  EXPECT_EQ(Src0, Copies[0]);
   match =
       mi_match(MIBAExt->getOperand(0).getReg(), MRI, m_GAnyExt(m_Reg(Src0)));
-  ASSERT_TRUE(match);
-  ASSERT_EQ(Src0, MIBTrunc->getOperand(0).getReg());
+  EXPECT_TRUE(match);
+  EXPECT_EQ(Src0, MIBTrunc->getOperand(0).getReg());
 
   match = mi_match(MIBSExt->getOperand(0).getReg(), MRI, m_GSExt(m_Reg(Src0)));
-  ASSERT_TRUE(match);
-  ASSERT_EQ(Src0, MIBTrunc->getOperand(0).getReg());
+  EXPECT_TRUE(match);
+  EXPECT_EQ(Src0, MIBTrunc->getOperand(0).getReg());
 
   match = mi_match(MIBZExt->getOperand(0).getReg(), MRI, m_GZExt(m_Reg(Src0)));
-  ASSERT_TRUE(match);
-  ASSERT_EQ(Src0, MIBTrunc->getOperand(0).getReg());
+  EXPECT_TRUE(match);
+  EXPECT_EQ(Src0, MIBTrunc->getOperand(0).getReg());
 
   // Match ext(trunc src)
   match = mi_match(MIBAExt->getOperand(0).getReg(), MRI,
                    m_GAnyExt(m_GTrunc(m_Reg(Src0))));
-  ASSERT_TRUE(match);
-  ASSERT_EQ(Src0, Copies[0]);
+  EXPECT_TRUE(match);
+  EXPECT_EQ(Src0, Copies[0]);
 
   match = mi_match(MIBSExt->getOperand(0).getReg(), MRI,
                    m_GSExt(m_GTrunc(m_Reg(Src0))));
-  ASSERT_TRUE(match);
-  ASSERT_EQ(Src0, Copies[0]);
+  EXPECT_TRUE(match);
+  EXPECT_EQ(Src0, Copies[0]);
 
   match = mi_match(MIBZExt->getOperand(0).getReg(), MRI,
                    m_GZExt(m_GTrunc(m_Reg(Src0))));
-  ASSERT_TRUE(match);
-  ASSERT_EQ(Src0, Copies[0]);
+  EXPECT_TRUE(match);
+  EXPECT_EQ(Src0, Copies[0]);
 }
 
 TEST(PatternMatchInstr, MatchSpecificType) {
@@ -415,19 +414,19 @@ TEST(PatternMatchInstr, MatchSpecificType) {
   LLT s64 = LLT::scalar(64);
   LLT s32 = LLT::scalar(32);
   auto MIBAdd = B.buildAdd(s64, Copies[0], Copies[1]);
-  ASSERT_FALSE(mi_match(MIBAdd->getOperand(0).getReg(), MRI,
+  EXPECT_FALSE(mi_match(MIBAdd->getOperand(0).getReg(), MRI,
                         m_GAdd(m_SpecificType(s32), m_Reg())));
-  ASSERT_TRUE(mi_match(MIBAdd->getOperand(0).getReg(), MRI,
+  EXPECT_TRUE(mi_match(MIBAdd->getOperand(0).getReg(), MRI,
                        m_GAdd(m_SpecificType(s64), m_Reg())));
 
   // Try to match the destination type of a bitcast.
   LLT v2s32 = LLT::vector(2, 32);
   auto MIBCast = B.buildCast(v2s32, Copies[0]);
-  ASSERT_TRUE(
+  EXPECT_TRUE(
       mi_match(MIBCast->getOperand(0).getReg(), MRI, m_GBitcast(m_Reg())));
-  ASSERT_TRUE(
+  EXPECT_TRUE(
       mi_match(MIBCast->getOperand(0).getReg(), MRI, m_SpecificType(v2s32)));
-  ASSERT_TRUE(
+  EXPECT_TRUE(
       mi_match(MIBCast->getOperand(1).getReg(), MRI, m_SpecificType(s64)));
 
   // Build a PTRToInt and INTTOPTR and match and test them.
@@ -439,8 +438,8 @@ TEST(PatternMatchInstr, MatchSpecificType) {
   // match the ptrtoint(inttoptr reg)
   bool match = mi_match(MIBPtrToInt->getOperand(0).getReg(), MRI,
                         m_GPtrToInt(m_GIntToPtr(m_Reg(Src0))));
-  ASSERT_TRUE(match);
-  ASSERT_EQ(Src0, Copies[0]);
+  EXPECT_TRUE(match);
+  EXPECT_EQ(Src0, Copies[0]);
 }
 
 TEST(PatternMatchInstr, MatchCombinators) {
@@ -464,26 +463,26 @@ TEST(PatternMatchInstr, MatchCombinators) {
   bool match =
       mi_match(MIBAdd->getOperand(0).getReg(), MRI,
                m_all_of(m_SpecificType(s64), m_GAdd(m_Reg(Src0), m_Reg(Src1))));
-  ASSERT_TRUE(match);
-  ASSERT_EQ(Src0, Copies[0]);
-  ASSERT_EQ(Src1, Copies[1]);
+  EXPECT_TRUE(match);
+  EXPECT_EQ(Src0, Copies[0]);
+  EXPECT_EQ(Src1, Copies[1]);
   // Check for s32 (which should fail).
   match =
       mi_match(MIBAdd->getOperand(0).getReg(), MRI,
                m_all_of(m_SpecificType(s32), m_GAdd(m_Reg(Src0), m_Reg(Src1))));
-  ASSERT_FALSE(match);
+  EXPECT_FALSE(match);
   match =
       mi_match(MIBAdd->getOperand(0).getReg(), MRI,
                m_any_of(m_SpecificType(s32), m_GAdd(m_Reg(Src0), m_Reg(Src1))));
-  ASSERT_TRUE(match);
-  ASSERT_EQ(Src0, Copies[0]);
-  ASSERT_EQ(Src1, Copies[1]);
+  EXPECT_TRUE(match);
+  EXPECT_EQ(Src0, Copies[0]);
+  EXPECT_EQ(Src1, Copies[1]);
 
   // Match a case where none of the predicates hold true.
   match = mi_match(
       MIBAdd->getOperand(0).getReg(), MRI,
       m_any_of(m_SpecificType(LLT::scalar(16)), m_GSub(m_Reg(), m_Reg())));
-  ASSERT_FALSE(match);
+  EXPECT_FALSE(match);
 }
 } // namespace
 
diff --git a/unittests/CodeGen/LowLevelTypeTest.cpp b/unittests/CodeGen/LowLevelTypeTest.cpp
index a4765d99856200c7e2f24c47efab1b1250b07e16..bf4277d82fd2ff03e3856e00933ed751d91d9813 100644
--- a/unittests/CodeGen/LowLevelTypeTest.cpp
+++ b/unittests/CodeGen/LowLevelTypeTest.cpp
@@ -1,9 +1,8 @@
 //===- llvm/unittest/CodeGen/GlobalISel/LowLevelTypeTest.cpp --------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -16,18 +15,6 @@
 
 using namespace llvm;
 
-// Define a pretty printer to help debugging when things go wrong.
-namespace llvm {
-std::ostream &
-operator<<(std::ostream &OS, const llvm::LLT Ty) {
-  std::string Repr;
-  raw_string_ostream SS{Repr};
-  Ty.print(SS);
-  OS << SS.str();
-  return OS;
-}
-}
-
 namespace {
 
 TEST(LowLevelTypeTest, Scalar) {
@@ -104,41 +91,118 @@ TEST(LowLevelTypeTest, Vector) {
   }
 }
 
+TEST(LowLevelTypeTest, ScalarOrVector) {
+  // Test version with number of bits for scalar type.
+  EXPECT_EQ(LLT::scalar(32), LLT::scalarOrVector(1, 32));
+  EXPECT_EQ(LLT::vector(2, 32), LLT::scalarOrVector(2, 32));
+
+  // Test version with LLT for scalar type.
+  EXPECT_EQ(LLT::scalar(32), LLT::scalarOrVector(1, LLT::scalar(32)));
+  EXPECT_EQ(LLT::vector(2, 32), LLT::scalarOrVector(2, LLT::scalar(32)));
+
+  // Test with pointer elements.
+  EXPECT_EQ(LLT::pointer(1, 32), LLT::scalarOrVector(1, LLT::pointer(1, 32)));
+  EXPECT_EQ(LLT::vector(2, LLT::pointer(1, 32)),
+            LLT::scalarOrVector(2, LLT::pointer(1, 32)));
+}
+
+TEST(LowLevelTypeTest, ChangeElementType) {
+  const LLT P0 = LLT::pointer(0, 32);
+  const LLT P1 = LLT::pointer(1, 64);
+
+  const LLT S32 = LLT::scalar(32);
+  const LLT S64 = LLT::scalar(64);
+
+  const LLT V2S32 = LLT::vector(2, 32);
+  const LLT V2S64 = LLT::vector(2, 64);
+
+  const LLT V2P0 = LLT::vector(2, P0);
+  const LLT V2P1 = LLT::vector(2, P1);
+
+  EXPECT_EQ(S64, S32.changeElementType(S64));
+  EXPECT_EQ(S32, S32.changeElementType(S32));
+
+  EXPECT_EQ(S32, S64.changeElementSize(32));
+  EXPECT_EQ(S32, S32.changeElementSize(32));
+
+  EXPECT_EQ(V2S64, V2S32.changeElementType(S64));
+  EXPECT_EQ(V2S32, V2S64.changeElementType(S32));
+
+  EXPECT_EQ(V2S64, V2S32.changeElementSize(64));
+  EXPECT_EQ(V2S32, V2S64.changeElementSize(32));
+
+  EXPECT_EQ(P0, S32.changeElementType(P0));
+  EXPECT_EQ(S32, P0.changeElementType(S32));
+
+  EXPECT_EQ(V2P1, V2P0.changeElementType(P1));
+  EXPECT_EQ(V2S32, V2P0.changeElementType(S32));
+}
+
+#ifdef GTEST_HAS_DEATH_TEST
+#ifndef NDEBUG
+
+// Invalid to directly change the element size for pointers.
+TEST(LowLevelTypeTest, ChangeElementTypeDeath) {
+  const LLT P0 = LLT::pointer(0, 32);
+  const LLT V2P0 = LLT::vector(2, P0);
+
+  EXPECT_DEATH(P0.changeElementSize(64),
+               "invalid to directly change element size for pointers");
+  EXPECT_DEATH(V2P0.changeElementSize(64),
+               "invalid to directly change element size for pointers");
+
+  // Make sure this still fails even without a change in size.
+  EXPECT_DEATH(P0.changeElementSize(32),
+               "invalid to directly change element size for pointers");
+  EXPECT_DEATH(V2P0.changeElementSize(32),
+               "invalid to directly change element size for pointers");
+}
+
+#endif
+#endif
+
 TEST(LowLevelTypeTest, Pointer) {
   LLVMContext C;
-  DataLayout DL("");
+  DataLayout DL("p64:64:64-p127:512:512:512-p16777215:65528:8");
 
-  for (unsigned AS : {0U, 1U, 127U, 0xffffU}) {
-    const LLT Ty = LLT::pointer(AS, DL.getPointerSizeInBits(AS));
-    const LLT VTy = LLT::vector(4, Ty);
+  for (unsigned AS : {0U, 1U, 127U, 0xffffU,
+        static_cast<unsigned>(maxUIntN(23)),
+        static_cast<unsigned>(maxUIntN(24))}) {
+    for (unsigned NumElts : {2, 3, 4, 256, 65535}) {
+      const LLT Ty = LLT::pointer(AS, DL.getPointerSizeInBits(AS));
+      const LLT VTy = LLT::vector(NumElts, Ty);
 
-    // Test kind.
-    ASSERT_TRUE(Ty.isValid());
-    ASSERT_TRUE(Ty.isPointer());
+      // Test kind.
+      ASSERT_TRUE(Ty.isValid());
+      ASSERT_TRUE(Ty.isPointer());
 
-    ASSERT_FALSE(Ty.isScalar());
-    ASSERT_FALSE(Ty.isVector());
+      ASSERT_FALSE(Ty.isScalar());
+      ASSERT_FALSE(Ty.isVector());
 
-    ASSERT_TRUE(VTy.isValid());
-    ASSERT_TRUE(VTy.isVector());
-    ASSERT_TRUE(VTy.getElementType().isPointer());
+      ASSERT_TRUE(VTy.isValid());
+      ASSERT_TRUE(VTy.isVector());
+      ASSERT_TRUE(VTy.getElementType().isPointer());
 
-    // Test addressspace.
-    EXPECT_EQ(AS, Ty.getAddressSpace());
-    EXPECT_EQ(AS, VTy.getElementType().getAddressSpace());
+      EXPECT_EQ(Ty, VTy.getElementType());
+      EXPECT_EQ(Ty.getSizeInBits(), VTy.getScalarSizeInBits());
 
-    // Test equality operators.
-    EXPECT_TRUE(Ty == Ty);
-    EXPECT_FALSE(Ty != Ty);
-    EXPECT_TRUE(VTy == VTy);
-    EXPECT_FALSE(VTy != VTy);
+      // Test address space.
+      EXPECT_EQ(AS, Ty.getAddressSpace());
+      EXPECT_EQ(AS, VTy.getElementType().getAddressSpace());
 
-    // Test Type->LLT conversion.
-    Type *IRTy = PointerType::get(IntegerType::get(C, 8), AS);
-    EXPECT_EQ(Ty, getLLTForType(*IRTy, DL));
-    Type *IRVTy =
-        VectorType::get(PointerType::get(IntegerType::get(C, 8), AS), 4);
-    EXPECT_EQ(VTy, getLLTForType(*IRVTy, DL));
+      // Test equality operators.
+      EXPECT_TRUE(Ty == Ty);
+      EXPECT_FALSE(Ty != Ty);
+      EXPECT_TRUE(VTy == VTy);
+      EXPECT_FALSE(VTy != VTy);
+
+      // Test Type->LLT conversion.
+      Type *IRTy = PointerType::get(IntegerType::get(C, 8), AS);
+      EXPECT_EQ(Ty, getLLTForType(*IRTy, DL));
+      Type *IRVTy =
+        VectorType::get(PointerType::get(IntegerType::get(C, 8), AS), NumElts);
+      EXPECT_EQ(VTy, getLLTForType(*IRVTy, DL));
+    }
   }
 }
 
diff --git a/unittests/CodeGen/MachineInstrBundleIteratorTest.cpp b/unittests/CodeGen/MachineInstrBundleIteratorTest.cpp
index 63365ab8fd4e116a2818ed587b4cd1d780b836fe..5ce0983ae3cdc33048a744dcaccd6606b738a9da 100644
--- a/unittests/CodeGen/MachineInstrBundleIteratorTest.cpp
+++ b/unittests/CodeGen/MachineInstrBundleIteratorTest.cpp
@@ -1,9 +1,8 @@
 //===- MachineInstrBundleIteratorTest.cpp ---------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/CodeGen/MachineInstrTest.cpp b/unittests/CodeGen/MachineInstrTest.cpp
index 73d3d25e8154df10ca1c92b7d42fe3b903a68785..2678a4589f9f9ba0fd28196395a8c4abec2269d6 100644
--- a/unittests/CodeGen/MachineInstrTest.cpp
+++ b/unittests/CodeGen/MachineInstrTest.cpp
@@ -1,9 +1,8 @@
 //===- MachineInstrTest.cpp -----------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -273,4 +272,6 @@ TEST(MachineInstrPrintingTest, DebugLocPrinting) {
       StringRef(OS.str()).endswith("filename:1:5"));
 }
 
+static_assert(is_trivially_copyable<MCOperand>::value, "trivially copyable");
+
 } // end namespace
diff --git a/unittests/CodeGen/MachineOperandTest.cpp b/unittests/CodeGen/MachineOperandTest.cpp
index 53ec5aed81890d5bbb2f4af44f025ca9a51fd04d..cfedfa9c8835b4494033b8cb48c0ba62493f4b4a 100644
--- a/unittests/CodeGen/MachineOperandTest.cpp
+++ b/unittests/CodeGen/MachineOperandTest.cpp
@@ -1,9 +1,8 @@
 //===- MachineOperandTest.cpp ---------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/CodeGen/ScalableVectorMVTsTest.cpp b/unittests/CodeGen/ScalableVectorMVTsTest.cpp
index 20b3fb1b9f481e84712f77a822c9b50368ac018a..3dd3bec8db79cf1eea4e4665832e7b7be18c36f8 100644
--- a/unittests/CodeGen/ScalableVectorMVTsTest.cpp
+++ b/unittests/CodeGen/ScalableVectorMVTsTest.cpp
@@ -1,9 +1,8 @@
 //===-------- llvm/unittest/CodeGen/ScalableVectorMVTsTest.cpp ------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/CodeGen/TypeTraitsTest.cpp b/unittests/CodeGen/TypeTraitsTest.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..840375bd4ab58ceeb319ff411502fef76d693665
--- /dev/null
+++ b/unittests/CodeGen/TypeTraitsTest.cpp
@@ -0,0 +1,25 @@
+//===- llvm/unittest/CodeGen/TypeTraitsTest.cpp --------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/CodeGen/RegisterPressure.h"
+#include "llvm/CodeGen/ScheduleDAG.h"
+#include "llvm/CodeGen/SelectionDAGNodes.h"
+#include "llvm/CodeGen/SlotIndexes.h"
+#include "llvm/CodeGen/TargetPassConfig.h"
+#include "gtest/gtest.h"
+
+using namespace llvm;
+
+#if __has_feature(is_trivially_copyable) || (defined(__GNUC__) && __GNUC__ >= 5)
+static_assert(is_trivially_copyable<PressureChange>::value, "trivially copyable");
+static_assert(is_trivially_copyable<SDep>::value, "trivially copyable");
+static_assert(is_trivially_copyable<SDValue>::value, "trivially copyable");
+static_assert(is_trivially_copyable<SlotIndex>::value, "trivially copyable");
+static_assert(is_trivially_copyable<IdentifyingPassPtr>::value, "trivially copyable");
+#endif
+
diff --git a/unittests/DebugInfo/CodeView/RandomAccessVisitorTest.cpp b/unittests/DebugInfo/CodeView/RandomAccessVisitorTest.cpp
index c84eae32face9fc8e0e87da56b00ff622a546695..4049f6165e246607ee58082935d0ccfe13d6aa44 100644
--- a/unittests/DebugInfo/CodeView/RandomAccessVisitorTest.cpp
+++ b/unittests/DebugInfo/CodeView/RandomAccessVisitorTest.cpp
@@ -1,9 +1,8 @@
 //===- llvm/unittest/DebugInfo/CodeView/RandomAccessVisitorTest.cpp -------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/DebugInfo/CodeView/TypeHashingTest.cpp b/unittests/DebugInfo/CodeView/TypeHashingTest.cpp
index 5b9dadfb33ff07e2e80e1eb82772a0176d21977b..8b9dc7ab285e953c8a89e8ef5d30256d7d9d13d6 100644
--- a/unittests/DebugInfo/CodeView/TypeHashingTest.cpp
+++ b/unittests/DebugInfo/CodeView/TypeHashingTest.cpp
@@ -1,9 +1,8 @@
 //===- llvm/unittest/DebugInfo/CodeView/TypeHashingTest.cpp ---------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/DebugInfo/CodeView/TypeIndexDiscoveryTest.cpp b/unittests/DebugInfo/CodeView/TypeIndexDiscoveryTest.cpp
index 1b89160a0f3cfb52dbfc307cd5772aede296a060..d011db9fcd47d6176eeb34b9097b126f8a45dbaf 100644
--- a/unittests/DebugInfo/CodeView/TypeIndexDiscoveryTest.cpp
+++ b/unittests/DebugInfo/CodeView/TypeIndexDiscoveryTest.cpp
@@ -1,9 +1,8 @@
 //===- llvm/unittest/DebugInfo/CodeView/TypeIndexDiscoveryTest.cpp --------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/DebugInfo/DWARF/DWARFDebugInfoTest.cpp b/unittests/DebugInfo/DWARF/DWARFDebugInfoTest.cpp
index ffbde2df2bc9f216707a092429c1e17ad6de560c..13119df005cbd43c558f32f612f8958b0f60417c 100644
--- a/unittests/DebugInfo/DWARF/DWARFDebugInfoTest.cpp
+++ b/unittests/DebugInfo/DWARF/DWARFDebugInfoTest.cpp
@@ -1,9 +1,8 @@
 //===- llvm/unittest/DebugInfo/DWARFDebugInfoTest.cpp ---------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/DebugInfo/DWARF/DWARFDebugLineTest.cpp b/unittests/DebugInfo/DWARF/DWARFDebugLineTest.cpp
index 03586ca60a48e614b4e8bf787baf8cc496294fc3..d016a946b1af9edd9431b7be509415af38a5e35a 100644
--- a/unittests/DebugInfo/DWARF/DWARFDebugLineTest.cpp
+++ b/unittests/DebugInfo/DWARF/DWARFDebugLineTest.cpp
@@ -1,9 +1,8 @@
 //===- DWARFDebugLineTest.cpp ---------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/DebugInfo/DWARF/DWARFFormValueTest.cpp b/unittests/DebugInfo/DWARF/DWARFFormValueTest.cpp
index 582b58062c4568942a2ddeb9006b2ef41675d67e..c40db0751ee1ebc816a62c1d22ac4fd4b4301b13 100644
--- a/unittests/DebugInfo/DWARF/DWARFFormValueTest.cpp
+++ b/unittests/DebugInfo/DWARF/DWARFFormValueTest.cpp
@@ -1,9 +1,8 @@
 //===- llvm/unittest/DebugInfo/DWARFFormValueTest.cpp ---------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/DebugInfo/DWARF/DwarfGenerator.cpp b/unittests/DebugInfo/DWARF/DwarfGenerator.cpp
index 973276616b8289bba408c78ab90bd7ccc553a5e5..f77de77a44425d535764889ff60fd7e38878d383 100644
--- a/unittests/DebugInfo/DWARF/DwarfGenerator.cpp
+++ b/unittests/DebugInfo/DWARF/DwarfGenerator.cpp
@@ -1,9 +1,8 @@
 //===--- unittests/DebugInfo/DWARF/DwarfGenerator.cpp -----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/DebugInfo/DWARF/DwarfGenerator.h b/unittests/DebugInfo/DWARF/DwarfGenerator.h
index 40ecaa98d05395e4c1140eb37055a263513587e4..2f95e29b84581840e773c1913889783572977251 100644
--- a/unittests/DebugInfo/DWARF/DwarfGenerator.h
+++ b/unittests/DebugInfo/DWARF/DwarfGenerator.h
@@ -1,9 +1,8 @@
 //===--- unittests/DebugInfo/DWARF/DwarfGenerator.h -------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/unittests/DebugInfo/DWARF/DwarfUtils.cpp b/unittests/DebugInfo/DWARF/DwarfUtils.cpp
index ef43c13db4ebb037f7d54ba3ccac6e3825a5e3dd..fe0cebd75b027c983a7f2530c6d88712674c33df 100644
--- a/unittests/DebugInfo/DWARF/DwarfUtils.cpp
+++ b/unittests/DebugInfo/DWARF/DwarfUtils.cpp
@@ -1,9 +1,8 @@
 //===--- unittests/DebugInfo/DWARF/DwarfUtils.cpp ---------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/DebugInfo/DWARF/DwarfUtils.h b/unittests/DebugInfo/DWARF/DwarfUtils.h
index 3d8045679d54afab58d762d987cdc274171f48b1..e96b2b944796475ec8cafaad2b2e5cf2045a329a 100644
--- a/unittests/DebugInfo/DWARF/DwarfUtils.h
+++ b/unittests/DebugInfo/DWARF/DwarfUtils.h
@@ -1,9 +1,8 @@
 //===--- unittests/DebugInfo/DWARF/DwarfUtils.h -----------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/DebugInfo/MSF/MSFBuilderTest.cpp b/unittests/DebugInfo/MSF/MSFBuilderTest.cpp
index 16247951804c498e729a1e5f51b27ec8b76b1e70..f327fef52bcf067d5e71aea3538314754eb0508d 100644
--- a/unittests/DebugInfo/MSF/MSFBuilderTest.cpp
+++ b/unittests/DebugInfo/MSF/MSFBuilderTest.cpp
@@ -1,9 +1,8 @@
 //===- MSFBuilderTest.cpp  Tests manipulation of MSF stream metadata ------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/DebugInfo/MSF/MSFCommonTest.cpp b/unittests/DebugInfo/MSF/MSFCommonTest.cpp
index ee9ac75d15cecf01917da0c31aed5b9e77afb1e5..68d087526cc2f3e1eeb5792b674506ced55c4b3d 100644
--- a/unittests/DebugInfo/MSF/MSFCommonTest.cpp
+++ b/unittests/DebugInfo/MSF/MSFCommonTest.cpp
@@ -1,9 +1,8 @@
 //===- MSFBuilderTest.cpp  Tests manipulation of MSF stream metadata ------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/DebugInfo/MSF/MappedBlockStreamTest.cpp b/unittests/DebugInfo/MSF/MappedBlockStreamTest.cpp
index 639536e94be988ae8f7cb96fe6394e1429932202..bc127213a3a41ee1e57295b3a005c0913d7ad837 100644
--- a/unittests/DebugInfo/MSF/MappedBlockStreamTest.cpp
+++ b/unittests/DebugInfo/MSF/MappedBlockStreamTest.cpp
@@ -1,9 +1,8 @@
 //===- llvm/unittest/DebugInfo/MSF/MappedBlockStreamTest.cpp --------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/DebugInfo/PDB/HashTableTest.cpp b/unittests/DebugInfo/PDB/HashTableTest.cpp
index 301b215badf5e634006bc1425eeccdf56672f3f8..4ebde45ff9a621a10ceb8a31137271ccffafbeec 100644
--- a/unittests/DebugInfo/PDB/HashTableTest.cpp
+++ b/unittests/DebugInfo/PDB/HashTableTest.cpp
@@ -1,9 +1,8 @@
 //===- llvm/unittest/DebugInfo/PDB/HashTableTest.cpp ----------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/DebugInfo/PDB/NativeSymbolReuseTest.cpp b/unittests/DebugInfo/PDB/NativeSymbolReuseTest.cpp
index 15afea6729295b9cd52bbae8e5c78d0168191ea6..91c7db4fc393fb253fe7bb98972d7f29d72617a8 100644
--- a/unittests/DebugInfo/PDB/NativeSymbolReuseTest.cpp
+++ b/unittests/DebugInfo/PDB/NativeSymbolReuseTest.cpp
@@ -1,9 +1,8 @@
 //===- NativeSymbolReuseTest.cpp ------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/DebugInfo/PDB/PDBApiTest.cpp b/unittests/DebugInfo/PDB/PDBApiTest.cpp
index 007ea9040856be9716595b23d82fcb45b3438c08..766f54d520635bf4636f9ae404c70e6f580d1ba5 100644
--- a/unittests/DebugInfo/PDB/PDBApiTest.cpp
+++ b/unittests/DebugInfo/PDB/PDBApiTest.cpp
@@ -1,9 +1,8 @@
 //===- llvm/unittest/DebugInfo/PDB/PDBApiTest.cpp -------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/DebugInfo/PDB/StringTableBuilderTest.cpp b/unittests/DebugInfo/PDB/StringTableBuilderTest.cpp
index bf08ab20923b1245f08ef0ea9f602b2ce4b8fd23..719cb807a2b85708b7bff5d6d57ee7fbea7e2a5b 100644
--- a/unittests/DebugInfo/PDB/StringTableBuilderTest.cpp
+++ b/unittests/DebugInfo/PDB/StringTableBuilderTest.cpp
@@ -1,9 +1,8 @@
 //===- StringTableBuilderTest.cpp -----------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/Demangle/CMakeLists.txt b/unittests/Demangle/CMakeLists.txt
index 954f3d05eacb3a2d329e8000d1d7c3ac0ede1cb6..c6291bb2a9803d1c12dae2a3c822d4fcf515304e 100644
--- a/unittests/Demangle/CMakeLists.txt
+++ b/unittests/Demangle/CMakeLists.txt
@@ -4,6 +4,7 @@ set(LLVM_LINK_COMPONENTS
 )
 
 add_llvm_unittest(DemangleTests
+  DemangleTest.cpp
   ItaniumDemangleTest.cpp
   PartialDemangleTest.cpp
 )
diff --git a/unittests/Demangle/DemangleTest.cpp b/unittests/Demangle/DemangleTest.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..cb70a720cad9d35a6c8b42120c89788cade0b1a2
--- /dev/null
+++ b/unittests/Demangle/DemangleTest.cpp
@@ -0,0 +1,24 @@
+//===-- DemangleTest.cpp --------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Demangle/Demangle.h"
+#include "gmock/gmock.h"
+
+using namespace llvm;
+
+TEST(Demangle, demangleTest) {
+  EXPECT_EQ(demangle("_"), "_");
+  EXPECT_EQ(demangle("_Z3fooi"), "foo(int)");
+  EXPECT_EQ(demangle("__Z3fooi"), "foo(int)");
+  EXPECT_EQ(demangle("___Z3fooi_block_invoke"),
+            "invocation function for block in foo(int)");
+  EXPECT_EQ(demangle("____Z3fooi_block_invoke"),
+            "invocation function for block in foo(int)");
+  EXPECT_EQ(demangle("?foo@@YAXH@Z"), "void __cdecl foo(int)");
+  EXPECT_EQ(demangle("foo"), "foo");
+}
diff --git a/unittests/Demangle/ItaniumDemangleTest.cpp b/unittests/Demangle/ItaniumDemangleTest.cpp
index abb690c626a4518219fd48fd05965de004a4c258..11bb78429a0a9fc3984a3305e4aa8e7ff04f3fbc 100644
--- a/unittests/Demangle/ItaniumDemangleTest.cpp
+++ b/unittests/Demangle/ItaniumDemangleTest.cpp
@@ -1,9 +1,8 @@
 //===------------------ ItaniumDemangleTest.cpp ---------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is dual licensed under the MIT and the University of Illinois Open
-// Source Licenses. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/Demangle/PartialDemangleTest.cpp b/unittests/Demangle/PartialDemangleTest.cpp
index 8bb4a3abcddadfc85c03a87567f18b489a70a497..0379d8372f76d495f650baa87bed80690e334aa0 100644
--- a/unittests/Demangle/PartialDemangleTest.cpp
+++ b/unittests/Demangle/PartialDemangleTest.cpp
@@ -1,9 +1,8 @@
 //===----------------------- PartialDemangleTest.cpp ----------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is dual licensed under the MIT and the University of Illinois Open
-// Source Licenses. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/ExecutionEngine/ExecutionEngineTest.cpp b/unittests/ExecutionEngine/ExecutionEngineTest.cpp
index ec5fab6c489335a1432739e58dba4def6d755129..77b1085a852f04eda48e0d396cc9da5705de0c99 100644
--- a/unittests/ExecutionEngine/ExecutionEngineTest.cpp
+++ b/unittests/ExecutionEngine/ExecutionEngineTest.cpp
@@ -1,9 +1,8 @@
 //===- ExecutionEngineTest.cpp - Unit tests for ExecutionEngine -----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/ExecutionEngine/MCJIT/MCJITCAPITest.cpp b/unittests/ExecutionEngine/MCJIT/MCJITCAPITest.cpp
index bc1ebc51b44be3f21b48c0e306980a7b925ee074..52b3b76e795fae9bcce3b1221afd5552663c584f 100644
--- a/unittests/ExecutionEngine/MCJIT/MCJITCAPITest.cpp
+++ b/unittests/ExecutionEngine/MCJIT/MCJITCAPITest.cpp
@@ -1,9 +1,8 @@
 //===- MCJITTest.cpp - Unit tests for the MCJIT -----------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/unittests/ExecutionEngine/MCJIT/MCJITMemoryManagerTest.cpp b/unittests/ExecutionEngine/MCJIT/MCJITMemoryManagerTest.cpp
index 0582c92b5f82ded306ed4b55db8dcfe971c084c1..7a756a707160adbd2a97ed5d0da08d6d72395018 100644
--- a/unittests/ExecutionEngine/MCJIT/MCJITMemoryManagerTest.cpp
+++ b/unittests/ExecutionEngine/MCJIT/MCJITMemoryManagerTest.cpp
@@ -1,9 +1,8 @@
 //===- MCJITMemoryManagerTest.cpp - Unit tests for the JIT memory manager -===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/ExecutionEngine/MCJIT/MCJITMultipleModuleTest.cpp b/unittests/ExecutionEngine/MCJIT/MCJITMultipleModuleTest.cpp
index 1226bba1c506117f5052b915d8c09fe95bb20686..710ce6d526605905a3a894a4c4b6d485db186b1f 100644
--- a/unittests/ExecutionEngine/MCJIT/MCJITMultipleModuleTest.cpp
+++ b/unittests/ExecutionEngine/MCJIT/MCJITMultipleModuleTest.cpp
@@ -1,9 +1,8 @@
 //===- MCJITMultipeModuleTest.cpp - Unit tests for the MCJIT ----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -175,7 +174,7 @@ TEST_F(MCJITMultipleModuleTest, two_module_consecutive_call_case) {
   std::unique_ptr<Module> A, B;
   Function *FA1, *FA2, *FB;
   createTwoModuleExternCase(A, FA1, B, FB);
-  FA2 = insertSimpleCallFunction<int32_t(int32_t, int32_t)>(A.get(), FA1);
+  FA2 = insertSimpleCallFunction(A.get(), FA1);
 
   createJIT(std::move(A));
   TheJIT->addModule(std::move(B));
@@ -203,16 +202,19 @@ TEST_F(MCJITMultipleModuleTest, two_module_global_variables_case) {
   std::unique_ptr<Module> A, B;
   Function *FA, *FB;
   GlobalVariable *GVA, *GVB, *GVC;
+
   A.reset(createEmptyModule("A"));
   B.reset(createEmptyModule("B"));
 
   int32_t initialNum = 7;
   GVA = insertGlobalInt32(A.get(), "GVA", initialNum);
   GVB = insertGlobalInt32(B.get(), "GVB", initialNum);
-  FA = startFunction<int32_t(void)>(A.get(), "FA");
-  endFunctionWithRet(FA, Builder.CreateLoad(GVA));
-  FB = startFunction<int32_t(void)>(B.get(), "FB");
-  endFunctionWithRet(FB, Builder.CreateLoad(GVB));
+  FA = startFunction(A.get(),
+                     FunctionType::get(Builder.getInt32Ty(), {}, false), "FA");
+  endFunctionWithRet(FA, Builder.CreateLoad(Builder.getInt32Ty(), GVA));
+  FB = startFunction(B.get(),
+                     FunctionType::get(Builder.getInt32Ty(), {}, false), "FB");
+  endFunctionWithRet(FB, Builder.CreateLoad(Builder.getInt32Ty(), GVB));
 
   GVC = insertGlobalInt32(B.get(), "GVC", initialNum);
   GVC->setLinkage(GlobalValue::InternalLinkage);
diff --git a/unittests/ExecutionEngine/MCJIT/MCJITObjectCacheTest.cpp b/unittests/ExecutionEngine/MCJIT/MCJITObjectCacheTest.cpp
index 2e3d2b6665a516e82bcac37492fbceec2539f320..4ec2e1aaec153ab44a3e6ced3f1f36ead6f0ff3a 100644
--- a/unittests/ExecutionEngine/MCJIT/MCJITObjectCacheTest.cpp
+++ b/unittests/ExecutionEngine/MCJIT/MCJITObjectCacheTest.cpp
@@ -1,9 +1,8 @@
 //===- MCJITObjectCacheTest.cpp - Unit tests for MCJIT object caching -----===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/ExecutionEngine/MCJIT/MCJITTest.cpp b/unittests/ExecutionEngine/MCJIT/MCJITTest.cpp
index e7da75a6d7e3fb19697e49e5e81758e6bfc8b8fd..14099074598a83c3513b48670aea5ebdb53b21a6 100644
--- a/unittests/ExecutionEngine/MCJIT/MCJITTest.cpp
+++ b/unittests/ExecutionEngine/MCJIT/MCJITTest.cpp
@@ -1,9 +1,8 @@
 //===- MCJITTest.cpp - Unit tests for the MCJIT -----------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -99,9 +98,10 @@ TEST_F(MCJITTest, return_global) {
   int32_t initialNum = 7;
   GlobalVariable *GV = insertGlobalInt32(M.get(), "myglob", initialNum);
 
-  Function *ReturnGlobal = startFunction<int32_t(void)>(M.get(),
-                                                        "ReturnGlobal");
-  Value *ReadGlobal = Builder.CreateLoad(GV);
+  Function *ReturnGlobal =
+      startFunction(M.get(), FunctionType::get(Builder.getInt32Ty(), {}, false),
+                    "ReturnGlobal");
+  Value *ReadGlobal = Builder.CreateLoad(Builder.getInt32Ty(), GV);
   endFunctionWithRet(ReturnGlobal, ReadGlobal);
 
   createJIT(std::move(M));
@@ -126,7 +126,10 @@ TEST_F(MCJITTest, increment_global) {
   SKIP_UNSUPPORTED_PLATFORM;
 
   int32_t initialNum = 5;
-  Function *IncrementGlobal = startFunction<int32_t(void)>(M.get(), "IncrementGlobal");
+  Function *IncrementGlobal = startFunction(
+      M.get(),
+      FunctionType::get(Builder.getInt32Ty(), {}, false),
+      "IncrementGlobal");
   GlobalVariable *GV = insertGlobalInt32(M.get(), "my_global", initialNum);
   Value *DerefGV = Builder.CreateLoad(GV);
   Value *AddResult = Builder.CreateAdd(DerefGV,
@@ -161,14 +164,17 @@ TEST_F(MCJITTest, multiple_functions) {
   unsigned int numLevels = 23;
   int32_t innerRetVal= 5;
 
-  Function *Inner = startFunction<int32_t(void)>(M.get(), "Inner");
+  Function *Inner = startFunction(
+      M.get(), FunctionType::get(Builder.getInt32Ty(), {}, false), "Inner");
   endFunctionWithRet(Inner, ConstantInt::get(Context, APInt(32, innerRetVal)));
 
   Function *Outer;
   for (unsigned int i = 0; i < numLevels; ++i) {
     std::stringstream funcName;
     funcName << "level_" << i;
-    Outer = startFunction<int32_t(void)>(M.get(), funcName.str());
+    Outer = startFunction(M.get(),
+                          FunctionType::get(Builder.getInt32Ty(), {}, false),
+                          funcName.str());
     Value *innerResult = Builder.CreateCall(Inner, {});
     endFunctionWithRet(Outer, innerResult);
 
@@ -190,7 +196,8 @@ TEST_F(MCJITTest, multiple_functions) {
 TEST_F(MCJITTest, multiple_decl_lookups) {
   SKIP_UNSUPPORTED_PLATFORM;
 
-  Function *Foo = insertExternalReferenceToFunction<void(void)>(M.get(), "_exit");
+  Function *Foo = insertExternalReferenceToFunction(
+      M.get(), FunctionType::get(Builder.getVoidTy(), {}, false), "_exit");
   createJIT(std::move(M));
   void *A = TheJIT->getPointerToFunction(Foo);
   void *B = TheJIT->getPointerToFunction(Foo);
@@ -203,10 +210,12 @@ typedef void * (*FunctionHandlerPtr)(const std::string &str);
 
 TEST_F(MCJITTest, lazy_function_creator_pointer) {
   SKIP_UNSUPPORTED_PLATFORM;
-  
-  Function *Foo = insertExternalReferenceToFunction<int32_t(void)>(M.get(),
-                                                                   "\1Foo");
-  startFunction<int32_t(void)>(M.get(), "Parent");
+
+  Function *Foo = insertExternalReferenceToFunction(
+      M.get(), FunctionType::get(Builder.getInt32Ty(), {}, false),
+      "\1Foo");
+  startFunction(M.get(), FunctionType::get(Builder.getInt32Ty(), {}, false),
+                "Parent");
   CallInst *Call = Builder.CreateCall(Foo, {});
   Builder.CreateRet(Call);
   
@@ -240,12 +249,14 @@ TEST_F(MCJITTest, lazy_function_creator_pointer) {
 
 TEST_F(MCJITTest, lazy_function_creator_lambda) {
   SKIP_UNSUPPORTED_PLATFORM;
-  
-  Function *Foo1 = insertExternalReferenceToFunction<int32_t(void)>(M.get(),
-                                                                   "\1Foo1");
-  Function *Foo2 = insertExternalReferenceToFunction<int32_t(void)>(M.get(),
-                                                                   "\1Foo2");
-  startFunction<int32_t(void)>(M.get(), "Parent");
+
+  FunctionType *Int32VoidFnTy =
+      FunctionType::get(Builder.getInt32Ty(), {}, false);
+  Function *Foo1 =
+      insertExternalReferenceToFunction(M.get(), Int32VoidFnTy, "\1Foo1");
+  Function *Foo2 =
+      insertExternalReferenceToFunction(M.get(), Int32VoidFnTy, "\1Foo2");
+  startFunction(M.get(), Int32VoidFnTy, "Parent");
   CallInst *Call1 = Builder.CreateCall(Foo1, {});
   CallInst *Call2 = Builder.CreateCall(Foo2, {});
   Value *Result = Builder.CreateAdd(Call1, Call2);
diff --git a/unittests/ExecutionEngine/MCJIT/MCJITTestAPICommon.h b/unittests/ExecutionEngine/MCJIT/MCJITTestAPICommon.h
index d3ebb95abc3a9ace263099eaa144f7a2a51e65fd..0f554439ad42a8f66fb16fab9153089100f0e057 100644
--- a/unittests/ExecutionEngine/MCJIT/MCJITTestAPICommon.h
+++ b/unittests/ExecutionEngine/MCJIT/MCJITTestAPICommon.h
@@ -1,9 +1,8 @@
 //===- MCJITTestBase.h - Common base class for MCJIT Unit tests  ----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/unittests/ExecutionEngine/MCJIT/MCJITTestBase.h b/unittests/ExecutionEngine/MCJIT/MCJITTestBase.h
index a768920ff479ce6d02437cb512125bd0008b4d5a..e0c4b07a71b08f5541d6cb4f75cd6f30e342a2b5 100644
--- a/unittests/ExecutionEngine/MCJIT/MCJITTestBase.h
+++ b/unittests/ExecutionEngine/MCJIT/MCJITTestBase.h
@@ -1,9 +1,8 @@
 //===- MCJITTestBase.h - Common base class for MCJIT Unit tests -*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -24,7 +23,7 @@
 #include "llvm/IR/IRBuilder.h"
 #include "llvm/IR/LLVMContext.h"
 #include "llvm/IR/Module.h"
-#include "llvm/IR/TypeBuilder.h"
+#include "llvm/IR/Type.h"
 #include "llvm/Support/CodeGen.h"
 
 namespace llvm {
@@ -45,11 +44,9 @@ protected:
     return M;
   }
 
-  template<typename FuncType>
-  Function *startFunction(Module *M, StringRef Name) {
-    Function *Result = Function::Create(
-      TypeBuilder<FuncType, false>::get(Context),
-      GlobalValue::ExternalLinkage, Name, M);
+  Function *startFunction(Module *M, FunctionType *FT, StringRef Name) {
+    Function *Result =
+        Function::Create(FT, GlobalValue::ExternalLinkage, Name, M);
 
     BasicBlock *BB = BasicBlock::Create(Context, Name, Result);
     Builder.SetInsertPoint(BB);
@@ -63,9 +60,8 @@ protected:
 
   // Inserts a simple function that invokes Callee and takes the same arguments:
   //    int Caller(...) { return Callee(...); }
-  template<typename Signature>
   Function *insertSimpleCallFunction(Module *M, Function *Callee) {
-    Function *Result = startFunction<Signature>(M, "caller");
+    Function *Result = startFunction(M, Callee->getFunctionType(), "caller");
 
     SmallVector<Value*, 1> CallArgs;
 
@@ -81,7 +77,8 @@ protected:
   //    int32_t main() { return X; }
   // where X is given by returnCode
   Function *insertMainFunction(Module *M, uint32_t returnCode) {
-    Function *Result = startFunction<int32_t(void)>(M, "main");
+    Function *Result = startFunction(
+        M, FunctionType::get(Type::getInt32Ty(Context), {}, false), "main");
 
     Value *ReturnVal = ConstantInt::get(Context, APInt(32, returnCode));
     endFunctionWithRet(Result, ReturnVal);
@@ -93,7 +90,12 @@ protected:
   //    int32_t add(int32_t a, int32_t b) { return a + b; }
   // in the current module and returns a pointer to it.
   Function *insertAddFunction(Module *M, StringRef Name = "add") {
-    Function *Result = startFunction<int32_t(int32_t, int32_t)>(M, Name);
+    Function *Result = startFunction(
+        M,
+        FunctionType::get(
+            Type::getInt32Ty(Context),
+            {Type::getInt32Ty(Context), Type::getInt32Ty(Context)}, false),
+        Name);
 
     Function::arg_iterator args = Result->arg_begin();
     Value *Arg1 = &*args;
@@ -106,20 +108,10 @@ protected:
   }
 
   // Inserts a declaration to a function defined elsewhere
-  template <typename FuncType>
-  Function *insertExternalReferenceToFunction(Module *M, StringRef Name) {
-    Function *Result = Function::Create(
-                         TypeBuilder<FuncType, false>::get(Context),
-                         GlobalValue::ExternalLinkage, Name, M);
-    return Result;
-  }
-
-  // Inserts an declaration to a function defined elsewhere
-  Function *insertExternalReferenceToFunction(Module *M, StringRef Name,
-                                              FunctionType *FuncTy) {
-    Function *Result = Function::Create(FuncTy,
-                                        GlobalValue::ExternalLinkage,
-                                        Name, M);
+  Function *insertExternalReferenceToFunction(Module *M, FunctionType *FTy,
+                                              StringRef Name) {
+    Function *Result =
+        Function::Create(FTy, GlobalValue::ExternalLinkage, Name, M);
     return Result;
   }
 
@@ -136,7 +128,7 @@ protected:
   GlobalVariable *insertGlobalInt32(Module *M,
                                     StringRef name,
                                     int32_t InitialValue) {
-    Type *GlobalTy = TypeBuilder<types::i<32>, true>::get(Context);
+    Type *GlobalTy = Type::getInt32Ty(Context);
     Constant *IV = ConstantInt::get(Context, APInt(32, InitialValue));
     GlobalVariable *Global = new GlobalVariable(*M,
                                                 GlobalTy,
@@ -160,7 +152,11 @@ protected:
   Function *insertAccumulateFunction(Module *M,
                                      Function *Helper = nullptr,
                                      StringRef Name = "accumulate") {
-    Function *Result = startFunction<int32_t(int32_t)>(M, Name);
+    Function *Result =
+        startFunction(M,
+                      FunctionType::get(Type::getInt32Ty(Context),
+                                        {Type::getInt32Ty(Context)}, false),
+                      Name);
     if (!Helper)
       Helper = Result;
 
@@ -225,11 +221,11 @@ protected:
 
     B.reset(createEmptyModule("B"));
     Function *FAExtern_in_B = insertExternalReferenceToFunction(B.get(), FA);
-    FB = insertSimpleCallFunction<int32_t(int32_t, int32_t)>(B.get(), FAExtern_in_B);
+    FB = insertSimpleCallFunction(B.get(), FAExtern_in_B);
 
     C.reset(createEmptyModule("C"));
     Function *FBExtern_in_C = insertExternalReferenceToFunction(C.get(), FB);
-    FC = insertSimpleCallFunction<int32_t(int32_t, int32_t)>(C.get(), FBExtern_in_C);
+    FC = insertSimpleCallFunction(C.get(), FBExtern_in_C);
   }
 
   // Module A { Function FA },
@@ -253,8 +249,7 @@ protected:
 
     B.reset(createEmptyModule("B"));
     Function *FAExtern_in_B = insertExternalReferenceToFunction(B.get(), FA);
-    FB = insertSimpleCallFunction<int32_t(int32_t, int32_t)>(B.get(),
-                                                             FAExtern_in_B);
+    FB = insertSimpleCallFunction(B.get(), FAExtern_in_B);
   }
 
   // Module A { Function FA },
@@ -268,11 +263,11 @@ protected:
 
     B.reset(createEmptyModule("B"));
     Function *FAExtern_in_B = insertExternalReferenceToFunction(B.get(), FA);
-    FB = insertSimpleCallFunction<int32_t(int32_t, int32_t)>(B.get(), FAExtern_in_B);
+    FB = insertSimpleCallFunction(B.get(), FAExtern_in_B);
 
     C.reset(createEmptyModule("C"));
     Function *FAExtern_in_C = insertExternalReferenceToFunction(C.get(), FA);
-    FC = insertSimpleCallFunction<int32_t(int32_t, int32_t)>(C.get(), FAExtern_in_C);
+    FC = insertSimpleCallFunction(C.get(), FAExtern_in_C);
   }
 };
 
diff --git a/unittests/ExecutionEngine/Orc/CoreAPIsTest.cpp b/unittests/ExecutionEngine/Orc/CoreAPIsTest.cpp
index 22be76a2eb6d27dabe2eb59aadad0ec30a364909..84b19ec2f9024a6d009a217be59032662569137a 100644
--- a/unittests/ExecutionEngine/Orc/CoreAPIsTest.cpp
+++ b/unittests/ExecutionEngine/Orc/CoreAPIsTest.cpp
@@ -1,9 +1,8 @@
 //===----------- CoreAPIsTest.cpp - Unit tests for Core ORC APIs ----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/ExecutionEngine/Orc/GlobalMappingLayerTest.cpp b/unittests/ExecutionEngine/Orc/GlobalMappingLayerTest.cpp
index d093b2a99bd7fbeac21c20e4df485b150d919cb7..d3853df80cfd2547c18864237b98a7aa4cfd3d43 100644
--- a/unittests/ExecutionEngine/Orc/GlobalMappingLayerTest.cpp
+++ b/unittests/ExecutionEngine/Orc/GlobalMappingLayerTest.cpp
@@ -1,9 +1,8 @@
 //===--- GlobalMappingLayerTest.cpp - Unit test the global mapping layer --===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/ExecutionEngine/Orc/IndirectionUtilsTest.cpp b/unittests/ExecutionEngine/Orc/IndirectionUtilsTest.cpp
index ed425449784cbff36d28336b83e40b82b5aba720..e9d39efd344f21bfa7a7a14a2b54ba3e819c01d1 100644
--- a/unittests/ExecutionEngine/Orc/IndirectionUtilsTest.cpp
+++ b/unittests/ExecutionEngine/Orc/IndirectionUtilsTest.cpp
@@ -1,9 +1,8 @@
 //===- LazyEmittingLayerTest.cpp - Unit tests for the lazy emitting layer -===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -19,7 +18,10 @@ namespace {
 TEST(IndirectionUtilsTest, MakeStub) {
   LLVMContext Context;
   ModuleBuilder MB(Context, "x86_64-apple-macosx10.10", "");
-  Function *F = MB.createFunctionDecl<void(DummyStruct, DummyStruct)>("");
+  FunctionType *FTy = FunctionType::get(
+      Type::getVoidTy(Context),
+      {getDummyStructTy(Context), getDummyStructTy(Context)}, false);
+  Function *F = MB.createFunctionDecl(FTy, "");
   AttributeSet FnAttrs = AttributeSet::get(
       Context, AttrBuilder().addAttribute(Attribute::NoUnwind));
   AttributeSet RetAttrs; // None
diff --git a/unittests/ExecutionEngine/Orc/JITTargetMachineBuilderTest.cpp b/unittests/ExecutionEngine/Orc/JITTargetMachineBuilderTest.cpp
index f978a90efb3d91bc3e7edbefbf587171d48941c0..1f3626e8af13168b3bcfe43eabbcde9ea364fab7 100644
--- a/unittests/ExecutionEngine/Orc/JITTargetMachineBuilderTest.cpp
+++ b/unittests/ExecutionEngine/Orc/JITTargetMachineBuilderTest.cpp
@@ -1,9 +1,8 @@
 //===----------- CoreAPIsTest.cpp - Unit tests for Core ORC APIs ----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/ExecutionEngine/Orc/LazyEmittingLayerTest.cpp b/unittests/ExecutionEngine/Orc/LazyEmittingLayerTest.cpp
index 3801fa918db7a61eb7f4c1a271dd0b9b993b846c..88d12cd103a9213f4dc0d0df65f384fcf3e8375d 100644
--- a/unittests/ExecutionEngine/Orc/LazyEmittingLayerTest.cpp
+++ b/unittests/ExecutionEngine/Orc/LazyEmittingLayerTest.cpp
@@ -1,9 +1,8 @@
 //===- LazyEmittingLayerTest.cpp - Unit tests for the lazy emitting layer -===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/ExecutionEngine/Orc/LegacyAPIInteropTest.cpp b/unittests/ExecutionEngine/Orc/LegacyAPIInteropTest.cpp
index 36cb7102623f7beb99423ac481a2a8ee10236c7f..ea7479ff1c21f2fc15cce231978c69d019c896fb 100644
--- a/unittests/ExecutionEngine/Orc/LegacyAPIInteropTest.cpp
+++ b/unittests/ExecutionEngine/Orc/LegacyAPIInteropTest.cpp
@@ -1,9 +1,8 @@
 //===----------- CoreAPIsTest.cpp - Unit tests for Core ORC APIs ----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/ExecutionEngine/Orc/LegacyCompileOnDemandLayerTest.cpp b/unittests/ExecutionEngine/Orc/LegacyCompileOnDemandLayerTest.cpp
index 38f7a654571ea98bfd870c839c0390d348429897..9a526071c84c42fdb33f88de08a333bb9503a92d 100644
--- a/unittests/ExecutionEngine/Orc/LegacyCompileOnDemandLayerTest.cpp
+++ b/unittests/ExecutionEngine/Orc/LegacyCompileOnDemandLayerTest.cpp
@@ -1,9 +1,8 @@
 //===----- CompileOnDemandLayerTest.cpp - Unit tests for the COD layer ----===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/ExecutionEngine/Orc/LegacyRTDyldObjectLinkingLayerTest.cpp b/unittests/ExecutionEngine/Orc/LegacyRTDyldObjectLinkingLayerTest.cpp
index 8c9c958cc42adb366d972054c3aab9d6381fe665..04b6f2101b256c81aa16e305aa9079dac3c12635 100644
--- a/unittests/ExecutionEngine/Orc/LegacyRTDyldObjectLinkingLayerTest.cpp
+++ b/unittests/ExecutionEngine/Orc/LegacyRTDyldObjectLinkingLayerTest.cpp
@@ -1,9 +1,8 @@
 //===- RTDyldObjectLinkingLayerTest.cpp - RTDyld linking layer unit tests -===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -123,6 +122,8 @@ TEST_F(LegacyRTDyldObjectLinkingLayerExecutionTest, NoDuplicateFinalization) {
   if (!SupportsJIT)
     return;
 
+  Type *Int32Ty = IntegerType::get(Context, 32);
+
   ExecutionSession ES;
 
   auto MM = std::make_shared<SectionMemoryManagerWrapper>();
@@ -153,7 +154,8 @@ TEST_F(LegacyRTDyldObjectLinkingLayerExecutionTest, NoDuplicateFinalization) {
   ModuleBuilder MB1(Context, "", "dummy");
   {
     MB1.getModule()->setDataLayout(TM->createDataLayout());
-    Function *BarImpl = MB1.createFunctionDecl<int32_t(void)>("bar");
+    Function *BarImpl =
+        MB1.createFunctionDecl(FunctionType::get(Int32Ty, {}, false), "bar");
     BasicBlock *BarEntry = BasicBlock::Create(Context, "entry", BarImpl);
     IRBuilder<> Builder(BarEntry);
     IntegerType *Int32Ty = IntegerType::get(Context, 32);
@@ -166,8 +168,10 @@ TEST_F(LegacyRTDyldObjectLinkingLayerExecutionTest, NoDuplicateFinalization) {
   ModuleBuilder MB2(Context, "", "dummy");
   {
     MB2.getModule()->setDataLayout(TM->createDataLayout());
-    Function *BarDecl = MB2.createFunctionDecl<int32_t(void)>("bar");
-    Function *FooImpl = MB2.createFunctionDecl<int32_t(void)>("foo");
+    Function *BarDecl =
+        MB2.createFunctionDecl(FunctionType::get(Int32Ty, {}, false), "bar");
+    Function *FooImpl =
+        MB2.createFunctionDecl(FunctionType::get(Int32Ty, {}, false), "foo");
     BasicBlock *FooEntry = BasicBlock::Create(Context, "entry", FooImpl);
     IRBuilder<> Builder(FooEntry);
     Builder.CreateRet(Builder.CreateCall(BarDecl));
@@ -207,6 +211,8 @@ TEST_F(LegacyRTDyldObjectLinkingLayerExecutionTest, NoPrematureAllocation) {
   if (!SupportsJIT)
     return;
 
+  Type *Int32Ty = IntegerType::get(Context, 32);
+
   ExecutionSession ES;
 
   auto MM = std::make_shared<SectionMemoryManagerWrapper>();
@@ -233,7 +239,8 @@ TEST_F(LegacyRTDyldObjectLinkingLayerExecutionTest, NoPrematureAllocation) {
   ModuleBuilder MB1(Context, "", "dummy");
   {
     MB1.getModule()->setDataLayout(TM->createDataLayout());
-    Function *BarImpl = MB1.createFunctionDecl<int32_t(void)>("foo");
+    Function *BarImpl =
+        MB1.createFunctionDecl(FunctionType::get(Int32Ty, {}, false), "foo");
     BasicBlock *BarEntry = BasicBlock::Create(Context, "entry", BarImpl);
     IRBuilder<> Builder(BarEntry);
     IntegerType *Int32Ty = IntegerType::get(Context, 32);
@@ -246,7 +253,8 @@ TEST_F(LegacyRTDyldObjectLinkingLayerExecutionTest, NoPrematureAllocation) {
   ModuleBuilder MB2(Context, "", "dummy");
   {
     MB2.getModule()->setDataLayout(TM->createDataLayout());
-    Function *BarImpl = MB2.createFunctionDecl<int32_t(void)>("bar");
+    Function *BarImpl =
+        MB2.createFunctionDecl(FunctionType::get(Int32Ty, {}, false), "bar");
     BasicBlock *BarEntry = BasicBlock::Create(Context, "entry", BarImpl);
     IRBuilder<> Builder(BarEntry);
     IntegerType *Int32Ty = IntegerType::get(Context, 32);
diff --git a/unittests/ExecutionEngine/Orc/ObjectTransformLayerTest.cpp b/unittests/ExecutionEngine/Orc/ObjectTransformLayerTest.cpp
index 1c530247a7c0dd903d0b2a474196e1e3b6f2e7df..2b6b5a7670d4b75765650c11cc134216f2f3a01c 100644
--- a/unittests/ExecutionEngine/Orc/ObjectTransformLayerTest.cpp
+++ b/unittests/ExecutionEngine/Orc/ObjectTransformLayerTest.cpp
@@ -1,9 +1,8 @@
 //===- ObjectTransformLayerTest.cpp - Unit tests for ObjectTransformLayer -===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/ExecutionEngine/Orc/OrcCAPITest.cpp b/unittests/ExecutionEngine/Orc/OrcCAPITest.cpp
index b288b6bab2c908f08bae9ddc48d926ca11032083..39fdc2e9ce9b6fef1ab69e2bbe5ecd64ced8d0dc 100644
--- a/unittests/ExecutionEngine/Orc/OrcCAPITest.cpp
+++ b/unittests/ExecutionEngine/Orc/OrcCAPITest.cpp
@@ -1,9 +1,8 @@
 //===--------------- OrcCAPITest.cpp - Unit tests Orc C API ---------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -27,8 +26,15 @@ class OrcCAPIExecutionTest : public testing::Test, public OrcExecutionTest {
 protected:
   std::unique_ptr<Module> createTestModule(const Triple &TT) {
     ModuleBuilder MB(Context, TT.str(), "");
-    Function *TestFunc = MB.createFunctionDecl<int()>("testFunc");
-    Function *Main = MB.createFunctionDecl<int(int, char*[])>("main");
+    Type *IntTy = Type::getScalarTy<int>(Context);
+    Function *TestFunc =
+        MB.createFunctionDecl(FunctionType::get(IntTy, {}, false), "testFunc");
+    Function *Main = MB.createFunctionDecl(
+        FunctionType::get(
+            IntTy,
+            {IntTy, Type::getInt8PtrTy(Context)->getPointerTo()},
+            false),
+        "main");
 
     Main->getBasicBlockList().push_back(BasicBlock::Create(Context));
     IRBuilder<> B(&Main->back());
diff --git a/unittests/ExecutionEngine/Orc/OrcTestCommon.cpp b/unittests/ExecutionEngine/Orc/OrcTestCommon.cpp
index a12b1876e29c26efd1bebd51a550b7f5573ff990..2a3223a97319345a2ffd0d2388a06f6b2f13e4a4 100644
--- a/unittests/ExecutionEngine/Orc/OrcTestCommon.cpp
+++ b/unittests/ExecutionEngine/Orc/OrcTestCommon.cpp
@@ -1,9 +1,8 @@
 //===--------- OrcTestCommon.cpp - Utilities for Orc Unit Tests -----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/unittests/ExecutionEngine/Orc/OrcTestCommon.h b/unittests/ExecutionEngine/Orc/OrcTestCommon.h
index e76d2fae5e319ab8c73c97050903e82e96d3996a..d82012b5a8531e02c8c51bdad274d569a00715a2 100644
--- a/unittests/ExecutionEngine/Orc/OrcTestCommon.h
+++ b/unittests/ExecutionEngine/Orc/OrcTestCommon.h
@@ -1,9 +1,8 @@
 //===------ OrcTestCommon.h - Utilities for Orc Unit Tests ------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -22,7 +21,6 @@
 #include "llvm/IR/IRBuilder.h"
 #include "llvm/IR/LLVMContext.h"
 #include "llvm/IR/Module.h"
-#include "llvm/IR/TypeBuilder.h"
 #include "llvm/Object/ObjectFile.h"
 #include "llvm/Support/TargetRegistry.h"
 #include "llvm/Support/TargetSelect.h"
@@ -169,11 +167,8 @@ public:
   ModuleBuilder(LLVMContext &Context, StringRef Triple,
                 StringRef Name);
 
-  template <typename FuncType>
-  Function* createFunctionDecl(StringRef Name) {
-    return Function::Create(
-             TypeBuilder<FuncType, false>::get(M->getContext()),
-             GlobalValue::ExternalLinkage, Name, M.get());
+  Function *createFunctionDecl(FunctionType *FTy, StringRef Name) {
+    return Function::Create(FTy, GlobalValue::ExternalLinkage, Name, M.get());
   }
 
   Module* getModule() { return M.get(); }
@@ -189,15 +184,9 @@ struct DummyStruct {
   int X[256];
 };
 
-// TypeBuilder specialization for DummyStruct.
-template <bool XCompile>
-class TypeBuilder<DummyStruct, XCompile> {
-public:
-  static StructType *get(LLVMContext &Context) {
-    return StructType::get(
-        TypeBuilder<types::i<32>[256], XCompile>::get(Context));
-  }
-};
+inline StructType *getDummyStructTy(LLVMContext &Context) {
+  return StructType::get(ArrayType::get(Type::getInt32Ty(Context), 256));
+}
 
 template <typename HandleT, typename ModuleT>
 class MockBaseLayer {
diff --git a/unittests/ExecutionEngine/Orc/QueueChannel.cpp b/unittests/ExecutionEngine/Orc/QueueChannel.cpp
index e309a7e428c0404ab682f0160af84ccf25735ce6..7556f8a15e7517b75de0bec6c1b54479136edaff 100644
--- a/unittests/ExecutionEngine/Orc/QueueChannel.cpp
+++ b/unittests/ExecutionEngine/Orc/QueueChannel.cpp
@@ -1,9 +1,8 @@
 //===-------- QueueChannel.cpp - Unit tests the remote executors ----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/ExecutionEngine/Orc/QueueChannel.h b/unittests/ExecutionEngine/Orc/QueueChannel.h
index d8c16811af11676835cbd796ab2d42e6d19e6d08..56a3926ee65bdf9ee3592091c5735dfec80184df 100644
--- a/unittests/ExecutionEngine/Orc/QueueChannel.h
+++ b/unittests/ExecutionEngine/Orc/QueueChannel.h
@@ -1,9 +1,8 @@
 //===----------------------- Queue.h - RPC Queue ------------------*-c++-*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/ExecutionEngine/Orc/RPCUtilsTest.cpp b/unittests/ExecutionEngine/Orc/RPCUtilsTest.cpp
index c884aaa718a925a4ba79a61e49107610038e00a9..1f7c88d93d068783d862a66eda42048a14a98a6e 100644
--- a/unittests/ExecutionEngine/Orc/RPCUtilsTest.cpp
+++ b/unittests/ExecutionEngine/Orc/RPCUtilsTest.cpp
@@ -1,9 +1,8 @@
 //===----------- RPCUtilsTest.cpp - Unit tests the Orc RPC utils ----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/ExecutionEngine/Orc/RTDyldObjectLinkingLayerTest.cpp b/unittests/ExecutionEngine/Orc/RTDyldObjectLinkingLayerTest.cpp
index 6b1dbe93d5e35b3adb1ddb0a84ea9b43a087eb0c..a5f529a37b615950c82016628f13a4de557d1750 100644
--- a/unittests/ExecutionEngine/Orc/RTDyldObjectLinkingLayerTest.cpp
+++ b/unittests/ExecutionEngine/Orc/RTDyldObjectLinkingLayerTest.cpp
@@ -1,9 +1,8 @@
 //===--- RTDyldObjectLinkingLayerTest.cpp - RTDyld linking layer tests ---===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -131,13 +130,17 @@ TEST(RTDyldObjectLinkingLayerTest, TestOverrideObjectFlags) {
     ModuleBuilder MB(*TSCtx.getContext(), TM->getTargetTriple().str(), "dummy");
     MB.getModule()->setDataLayout(TM->createDataLayout());
 
-    Function *FooImpl = MB.createFunctionDecl<void()>("foo");
+    Function *FooImpl = MB.createFunctionDecl(
+        FunctionType::get(Type::getVoidTy(*TSCtx.getContext()), {}, false),
+        "foo");
     BasicBlock *FooEntry =
         BasicBlock::Create(*TSCtx.getContext(), "entry", FooImpl);
     IRBuilder<> B1(FooEntry);
     B1.CreateRetVoid();
 
-    Function *BarImpl = MB.createFunctionDecl<void()>("bar");
+    Function *BarImpl = MB.createFunctionDecl(
+        FunctionType::get(Type::getVoidTy(*TSCtx.getContext()), {}, false),
+        "bar");
     BasicBlock *BarEntry =
         BasicBlock::Create(*TSCtx.getContext(), "entry", BarImpl);
     IRBuilder<> B2(BarEntry);
@@ -181,9 +184,9 @@ TEST(RTDyldObjectLinkingLayerTest, TestAutoClaimResponsibilityForSymbols) {
     FunkySimpleCompiler(TargetMachine &TM) : SimpleCompiler(TM) {}
 
     CompileResult operator()(Module &M) {
-      Function *BarImpl =
-          Function::Create(TypeBuilder<void(), false>::get(M.getContext()),
-                           GlobalValue::ExternalLinkage, "bar", &M);
+      Function *BarImpl = Function::Create(
+          FunctionType::get(Type::getVoidTy(M.getContext()), {}, false),
+          GlobalValue::ExternalLinkage, "bar", &M);
       BasicBlock *BarEntry =
           BasicBlock::Create(M.getContext(), "entry", BarImpl);
       IRBuilder<> B(BarEntry);
@@ -200,7 +203,9 @@ TEST(RTDyldObjectLinkingLayerTest, TestAutoClaimResponsibilityForSymbols) {
     ModuleBuilder MB(*TSCtx.getContext(), TM->getTargetTriple().str(), "dummy");
     MB.getModule()->setDataLayout(TM->createDataLayout());
 
-    Function *FooImpl = MB.createFunctionDecl<void()>("foo");
+    Function *FooImpl = MB.createFunctionDecl(
+        FunctionType::get(Type::getVoidTy(*TSCtx.getContext()), {}, false),
+        "foo");
     BasicBlock *FooEntry =
         BasicBlock::Create(*TSCtx.getContext(), "entry", FooImpl);
     IRBuilder<> B(FooEntry);
diff --git a/unittests/ExecutionEngine/Orc/RemoteObjectLayerTest.cpp b/unittests/ExecutionEngine/Orc/RemoteObjectLayerTest.cpp
index 09224c289b4d1f421fca6614a93e8f7d92ddd017..cb02a91cb6c2b73efa92fe7e5fdfe61a969767f0 100644
--- a/unittests/ExecutionEngine/Orc/RemoteObjectLayerTest.cpp
+++ b/unittests/ExecutionEngine/Orc/RemoteObjectLayerTest.cpp
@@ -1,9 +1,8 @@
 //===---------------------- RemoteObjectLayerTest.cpp ---------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -95,7 +94,12 @@ MockObjectLayer::ObjectPtr createTestObject() {
   LLVMContext Ctx;
   ModuleBuilder MB(Ctx, TM->getTargetTriple().str(), "TestModule");
   MB.getModule()->setDataLayout(TM->createDataLayout());
-  auto *Main = MB.createFunctionDecl<void(int, char**)>("main");
+  auto *Main = MB.createFunctionDecl(
+      FunctionType::get(Type::getInt32Ty(Ctx),
+                        {Type::getInt32Ty(Ctx),
+                         Type::getInt8PtrTy(Ctx)->getPointerTo()},
+                        false),
+      "main");
   Main->getBasicBlockList().push_back(BasicBlock::Create(Ctx));
   IRBuilder<> B(&Main->back());
   B.CreateRet(ConstantInt::getSigned(Type::getInt32Ty(Ctx), 42));
diff --git a/unittests/ExecutionEngine/Orc/SymbolStringPoolTest.cpp b/unittests/ExecutionEngine/Orc/SymbolStringPoolTest.cpp
index 861a9661223aa78dc688c29f89dfe8486dc4145d..a68da0be960656f1ff6d15dc610022dbe2868694 100644
--- a/unittests/ExecutionEngine/Orc/SymbolStringPoolTest.cpp
+++ b/unittests/ExecutionEngine/Orc/SymbolStringPoolTest.cpp
@@ -1,9 +1,8 @@
 //===----- SymbolStringPoolTest.cpp - Unit tests for SymbolStringPool -----===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/ExecutionEngine/Orc/ThreadSafeModuleTest.cpp b/unittests/ExecutionEngine/Orc/ThreadSafeModuleTest.cpp
index eb2cd41d84885021a853eddd14489bec70c984d2..6cd6c206ce082ec9d5fccebec64a7478c44eab2b 100644
--- a/unittests/ExecutionEngine/Orc/ThreadSafeModuleTest.cpp
+++ b/unittests/ExecutionEngine/Orc/ThreadSafeModuleTest.cpp
@@ -1,9 +1,8 @@
 //===--- ThreadSafeModuleTest.cpp - Test basic use of ThreadSafeModule ----===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/FuzzMutate/OperationsTest.cpp b/unittests/FuzzMutate/OperationsTest.cpp
index 0fc6b2c2aebd9833559674553f35959da4f98ea8..a077c5cd59e0426861b0127cda64218bce9f4471 100644
--- a/unittests/FuzzMutate/OperationsTest.cpp
+++ b/unittests/FuzzMutate/OperationsTest.cpp
@@ -1,9 +1,8 @@
 //===- OperationsTest.cpp - Tests for fuzzer operations -------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/FuzzMutate/RandomIRBuilderTest.cpp b/unittests/FuzzMutate/RandomIRBuilderTest.cpp
index d6ad07d00538515a5dd837bb138bf981cd7d94fb..1528be670fe85803789eb2691f2c97b49dad7b4c 100644
--- a/unittests/FuzzMutate/RandomIRBuilderTest.cpp
+++ b/unittests/FuzzMutate/RandomIRBuilderTest.cpp
@@ -1,9 +1,8 @@
 //===- RandomIRBuilderTest.cpp - Tests for injector strategy --------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/FuzzMutate/ReservoirSamplerTest.cpp b/unittests/FuzzMutate/ReservoirSamplerTest.cpp
index d246ba1428b3235a066205fea652bfb67acaa4ce..ac9270a265ec378336e32977a4177d0d9731a02a 100644
--- a/unittests/FuzzMutate/ReservoirSamplerTest.cpp
+++ b/unittests/FuzzMutate/ReservoirSamplerTest.cpp
@@ -1,9 +1,8 @@
 //===- ReservoirSampler.cpp - Tests for the ReservoirSampler --------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/FuzzMutate/StrategiesTest.cpp b/unittests/FuzzMutate/StrategiesTest.cpp
index 320a4e950f3d5e9571d2424ca4b6e5fb5bc24043..9ef88e226824edbdaa14a184866cb6ea990156a2 100644
--- a/unittests/FuzzMutate/StrategiesTest.cpp
+++ b/unittests/FuzzMutate/StrategiesTest.cpp
@@ -1,9 +1,8 @@
 //===- InjectorIRStrategyTest.cpp - Tests for injector strategy -----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/IR/AsmWriterTest.cpp b/unittests/IR/AsmWriterTest.cpp
index 9ad55237b5c0f7a19d0dffd665ef6e5a8667aa43..0e13ccae1c7aee7c2e4792a67cdb99bbde78ccb4 100644
--- a/unittests/IR/AsmWriterTest.cpp
+++ b/unittests/IR/AsmWriterTest.cpp
@@ -1,9 +1,8 @@
 //===- llvm/unittest/IR/AsmWriter.cpp - AsmWriter tests -------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 #include "llvm/BinaryFormat/Dwarf.h"
diff --git a/unittests/IR/AttributesTest.cpp b/unittests/IR/AttributesTest.cpp
index dd88cc359343b1fdb4e1ac1592f64b3610a8055b..e0be2343a14457b1275cbc64cf82461ee3196ff0 100644
--- a/unittests/IR/AttributesTest.cpp
+++ b/unittests/IR/AttributesTest.cpp
@@ -1,9 +1,8 @@
 //===- llvm/unittest/IR/AttributesTest.cpp - Attributes unit tests --------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/IR/BasicBlockTest.cpp b/unittests/IR/BasicBlockTest.cpp
index 07ed997f6381a975e2f2917dcf68faadb48f6f41..95bc242ec30a6a9ef843ac5543a495da02ffd427 100644
--- a/unittests/IR/BasicBlockTest.cpp
+++ b/unittests/IR/BasicBlockTest.cpp
@@ -1,9 +1,8 @@
 //===- llvm/unittest/IR/BasicBlockTest.cpp - BasicBlock unit tests --------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -101,10 +100,9 @@ TEST(BasicBlockTest, TestInstructionsWithoutDebug) {
   Argument *V = new Argument(Type::getInt32Ty(Ctx));
   Function *F = Function::Create(FT, Function::ExternalLinkage, "", M);
 
-  Value *DbgAddr = Intrinsic::getDeclaration(M, Intrinsic::dbg_addr);
-  Value *DbgDeclare =
-      Intrinsic::getDeclaration(M, Intrinsic::dbg_declare);
-  Value *DbgValue = Intrinsic::getDeclaration(M, Intrinsic::dbg_value);
+  Function *DbgAddr = Intrinsic::getDeclaration(M, Intrinsic::dbg_addr);
+  Function *DbgDeclare = Intrinsic::getDeclaration(M, Intrinsic::dbg_declare);
+  Function *DbgValue = Intrinsic::getDeclaration(M, Intrinsic::dbg_value);
   Value *DIV = MetadataAsValue::get(Ctx, (Metadata *)nullptr);
   SmallVector<Value *, 3> Args = {DIV, DIV, DIV};
 
diff --git a/unittests/IR/CFGBuilder.cpp b/unittests/IR/CFGBuilder.cpp
index 0f9fb8b18cc19e566bb8f11e90aa527e2a49f071..1ce598799cafa24e23b4e2c91a5fb8bf0fa668c4 100644
--- a/unittests/IR/CFGBuilder.cpp
+++ b/unittests/IR/CFGBuilder.cpp
@@ -1,17 +1,16 @@
 //===- llvm/Testing/Support/CFGBuilder.cpp --------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #include "CFGBuilder.h"
 
+#include "llvm/IR/CFG.h"
 #include "llvm/IR/IRBuilder.h"
 #include "llvm/IR/LLVMContext.h"
-#include "llvm/IR/TypeBuilder.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/raw_ostream.h"
 #include "gtest/gtest.h"
@@ -23,8 +22,8 @@ using namespace llvm;
 CFGHolder::CFGHolder(StringRef ModuleName, StringRef FunctionName)
     : Context(llvm::make_unique<LLVMContext>()),
       M(llvm::make_unique<Module>(ModuleName, *Context)) {
-  FunctionType *FTy = TypeBuilder<void(), false>::get(*Context);
-  F = cast<Function>(M->getOrInsertFunction(FunctionName, FTy));
+  FunctionType *FTy = FunctionType::get(Type::getVoidTy(*Context), {}, false);
+  F = Function::Create(FTy, Function::ExternalLinkage, FunctionName, M.get());
 }
 CFGHolder::~CFGHolder() = default;
 
@@ -267,3 +266,11 @@ TEST(CFGBuilder, Rebuild) {
   EXPECT_TRUE(isa<SwitchInst>(B.getOrAddBlock("c")->getTerminator()));
   EXPECT_TRUE(isa<SwitchInst>(B.getOrAddBlock("d")->getTerminator()));
 }
+
+static_assert(is_trivially_copyable<succ_iterator>::value,
+              "trivially copyable");
+static_assert(is_trivially_copyable<succ_const_iterator>::value,
+              "trivially copyable");
+static_assert(is_trivially_copyable<succ_range>::value, "trivially copyable");
+static_assert(is_trivially_copyable<succ_const_range>::value,
+              "trivially copyable");
diff --git a/unittests/IR/CFGBuilder.h b/unittests/IR/CFGBuilder.h
index d9d9c378e110324204c38b93ae09fc05fcd489f1..97be4a1641d666057f93aa77208496660fa83496 100644
--- a/unittests/IR/CFGBuilder.h
+++ b/unittests/IR/CFGBuilder.h
@@ -1,9 +1,8 @@
 //===- CFGBuilder.h - CFG building and updating utility ----------*- C++ -*-==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/unittests/IR/CMakeLists.txt b/unittests/IR/CMakeLists.txt
index 7498983b2609bb25b1f82ba2a892a090f844767d..a823407169f5ed31da75e1c7fc28e5d1ddbffdcf 100644
--- a/unittests/IR/CMakeLists.txt
+++ b/unittests/IR/CMakeLists.txt
@@ -17,7 +17,6 @@ add_llvm_unittest(IRTests
   DebugTypeODRUniquingTest.cpp
   DominatorTreeTest.cpp
   DominatorTreeBatchUpdatesTest.cpp
-  DomTreeUpdaterTest.cpp
   FunctionTest.cpp
   PassBuilderCallbacksTest.cpp
   IRBuilderTest.cpp
@@ -30,7 +29,6 @@ add_llvm_unittest(IRTests
   ModuleTest.cpp
   PassManagerTest.cpp
   PatternMatch.cpp
-  TypeBuilderTest.cpp
   TypesTest.cpp
   UseTest.cpp
   UserTest.cpp
diff --git a/unittests/IR/ConstantRangeTest.cpp b/unittests/IR/ConstantRangeTest.cpp
index c0a30f1b4d94d964c9588e2fe07b8394b9fb18b1..b3fe89451a26fa76e07080f2e2fbeec290e5442d 100644
--- a/unittests/IR/ConstantRangeTest.cpp
+++ b/unittests/IR/ConstantRangeTest.cpp
@@ -1,9 +1,8 @@
 //===- ConstantRangeTest.cpp - ConstantRange tests ------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/IR/ConstantsTest.cpp b/unittests/IR/ConstantsTest.cpp
index cf67e55b32a289a7e4aa76d2e7662af41428a5f6..6b26b38e35e9cae2d32395675f6b0cf24727a32c 100644
--- a/unittests/IR/ConstantsTest.cpp
+++ b/unittests/IR/ConstantsTest.cpp
@@ -1,9 +1,8 @@
 //===- llvm/unittest/IR/ConstantsTest.cpp - Constants unit tests ----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/IR/DebugInfoTest.cpp b/unittests/IR/DebugInfoTest.cpp
index a80df3c54fd4f25b8905cf4d79fe4f8120afbfec..93247908d8a77fd263e4c0a93e3771c22f09873a 100644
--- a/unittests/IR/DebugInfoTest.cpp
+++ b/unittests/IR/DebugInfoTest.cpp
@@ -1,9 +1,8 @@
 //===- llvm/unittest/IR/DebugInfo.cpp - DebugInfo tests -------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/IR/DebugTypeODRUniquingTest.cpp b/unittests/IR/DebugTypeODRUniquingTest.cpp
index af77091221c34e8f180a9b97b0b9702c03bd5e72..8031bf756aef83fba8aae0633edad2f08ecbb2a7 100644
--- a/unittests/IR/DebugTypeODRUniquingTest.cpp
+++ b/unittests/IR/DebugTypeODRUniquingTest.cpp
@@ -1,9 +1,8 @@
 //===- DebugTypeODRUniquingTest.cpp - Debug type ODR uniquing tests -------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/IR/DominatorTreeBatchUpdatesTest.cpp b/unittests/IR/DominatorTreeBatchUpdatesTest.cpp
index 6775fc1ab5c85ad621007f37153c346f73db91c8..a2d5805a9874e114b4a5810ec63b76ee76313957 100644
--- a/unittests/IR/DominatorTreeBatchUpdatesTest.cpp
+++ b/unittests/IR/DominatorTreeBatchUpdatesTest.cpp
@@ -1,9 +1,8 @@
 //===- llvm/unittests/IR/DominatorTreeBatchUpdatesTest.cpp ----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/IR/DominatorTreeTest.cpp b/unittests/IR/DominatorTreeTest.cpp
index 7539bbc860bd11329f00bbf0013b558bedd1196b..7773856b6bc3e88bf887e3cd90d32d3a56c7c190 100644
--- a/unittests/IR/DominatorTreeTest.cpp
+++ b/unittests/IR/DominatorTreeTest.cpp
@@ -1,9 +1,8 @@
 //===- llvm/unittests/IR/DominatorTreeTest.cpp - Constants unit tests -----===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/IR/FunctionTest.cpp b/unittests/IR/FunctionTest.cpp
index 6838d7e2527ff0c608c81d75b2e9da4ef91bfad8..10a359ff64bf5f59bca1eadf8c01f167432f34f0 100644
--- a/unittests/IR/FunctionTest.cpp
+++ b/unittests/IR/FunctionTest.cpp
@@ -1,9 +1,8 @@
 //===- FunctionTest.cpp - Function unit tests -----------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/IR/IRBuilderTest.cpp b/unittests/IR/IRBuilderTest.cpp
index db174fa2ff76585bcf6795816e0b1c00a0575e2e..71edb4bee6b50dac896139dd2bd63c1859f3d7a7 100644
--- a/unittests/IR/IRBuilderTest.cpp
+++ b/unittests/IR/IRBuilderTest.cpp
@@ -1,9 +1,8 @@
 //===- llvm/unittest/IR/IRBuilderTest.cpp - IRBuilder tests ---------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -55,7 +54,7 @@ TEST_F(IRBuilderTest, Intrinsics) {
   CallInst *Call;
   IntrinsicInst *II;
 
-  V = Builder.CreateLoad(GV);
+  V = Builder.CreateLoad(GV->getValueType(), GV);
   I = cast<Instruction>(Builder.CreateFAdd(V, V));
   I->setHasNoInfs(true);
   I->setHasNoNaNs(false);
@@ -208,7 +207,7 @@ TEST_F(IRBuilderTest, FastMathFlags) {
   Value *F, *FC;
   Instruction *FDiv, *FAdd, *FCmp, *FCall;
 
-  F = Builder.CreateLoad(GV);
+  F = Builder.CreateLoad(GV->getValueType(), GV);
   F = Builder.CreateFAdd(F, F);
 
   EXPECT_FALSE(Builder.getFastMathFlags().any());
@@ -354,7 +353,7 @@ TEST_F(IRBuilderTest, FastMathFlags) {
   FCall = Builder.CreateCall(Callee, None);
   EXPECT_FALSE(FCall->hasNoNaNs());
 
-  Value *V = 
+  Function *V =
       Function::Create(CalleeTy, Function::ExternalLinkage, "", M.get());
   FCall = Builder.CreateCall(V, None);
   EXPECT_FALSE(FCall->hasNoNaNs());
@@ -395,7 +394,7 @@ TEST_F(IRBuilderTest, WrapFlags) {
   // Test instructions.
   GlobalVariable *G = new GlobalVariable(*M, Builder.getInt32Ty(), true,
                                          GlobalValue::ExternalLinkage, nullptr);
-  Value *V = Builder.CreateLoad(G);
+  Value *V = Builder.CreateLoad(G->getValueType(), G);
   EXPECT_TRUE(
       cast<BinaryOperator>(Builder.CreateNSWAdd(V, V))->hasNoSignedWrap());
   EXPECT_TRUE(
@@ -462,7 +461,7 @@ TEST_F(IRBuilderTest, RAIIHelpersTest) {
   EXPECT_FALSE(Builder.getFastMathFlags().allowReciprocal());
   EXPECT_EQ(FPMathA, Builder.getDefaultFPMathTag());
 
-  Value *F = Builder.CreateLoad(GV);
+  Value *F = Builder.CreateLoad(GV->getValueType(), GV);
 
   {
     IRBuilder<>::InsertPointGuard Guard(Builder);
diff --git a/unittests/IR/InstructionsTest.cpp b/unittests/IR/InstructionsTest.cpp
index 080e35c743342f68595afcded98382dd6213d87e..94f37251e60bcfc6cae3c087298efe816e66f3ec 100644
--- a/unittests/IR/InstructionsTest.cpp
+++ b/unittests/IR/InstructionsTest.cpp
@@ -1,9 +1,8 @@
 //===- llvm/unittest/IR/InstructionsTest.cpp - Instructions unit tests ----===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -505,14 +504,15 @@ TEST(InstructionsTest, CloneCall) {
   LLVMContext C;
   Type *Int32Ty = Type::getInt32Ty(C);
   Type *ArgTys[] = {Int32Ty, Int32Ty, Int32Ty};
-  Type *FnTy = FunctionType::get(Int32Ty, ArgTys, /*isVarArg=*/false);
+  FunctionType *FnTy = FunctionType::get(Int32Ty, ArgTys, /*isVarArg=*/false);
   Value *Callee = Constant::getNullValue(FnTy->getPointerTo());
   Value *Args[] = {
     ConstantInt::get(Int32Ty, 1),
     ConstantInt::get(Int32Ty, 2),
     ConstantInt::get(Int32Ty, 3)
   };
-  std::unique_ptr<CallInst> Call(CallInst::Create(Callee, Args, "result"));
+  std::unique_ptr<CallInst> Call(
+      CallInst::Create(FnTy, Callee, Args, "result"));
 
   // Test cloning the tail call kind.
   CallInst::TailCallKind Kinds[] = {CallInst::TCK_None, CallInst::TCK_Tail,
@@ -538,12 +538,12 @@ TEST(InstructionsTest, CloneCall) {
 TEST(InstructionsTest, AlterCallBundles) {
   LLVMContext C;
   Type *Int32Ty = Type::getInt32Ty(C);
-  Type *FnTy = FunctionType::get(Int32Ty, Int32Ty, /*isVarArg=*/false);
+  FunctionType *FnTy = FunctionType::get(Int32Ty, Int32Ty, /*isVarArg=*/false);
   Value *Callee = Constant::getNullValue(FnTy->getPointerTo());
   Value *Args[] = {ConstantInt::get(Int32Ty, 42)};
   OperandBundleDef OldBundle("before", UndefValue::get(Int32Ty));
   std::unique_ptr<CallInst> Call(
-      CallInst::Create(Callee, Args, OldBundle, "result"));
+      CallInst::Create(FnTy, Callee, Args, OldBundle, "result"));
   Call->setTailCallKind(CallInst::TailCallKind::TCK_NoTail);
   AttrBuilder AB;
   AB.addAttribute(Attribute::Cold);
@@ -565,14 +565,15 @@ TEST(InstructionsTest, AlterCallBundles) {
 TEST(InstructionsTest, AlterInvokeBundles) {
   LLVMContext C;
   Type *Int32Ty = Type::getInt32Ty(C);
-  Type *FnTy = FunctionType::get(Int32Ty, Int32Ty, /*isVarArg=*/false);
+  FunctionType *FnTy = FunctionType::get(Int32Ty, Int32Ty, /*isVarArg=*/false);
   Value *Callee = Constant::getNullValue(FnTy->getPointerTo());
   Value *Args[] = {ConstantInt::get(Int32Ty, 42)};
   std::unique_ptr<BasicBlock> NormalDest(BasicBlock::Create(C));
   std::unique_ptr<BasicBlock> UnwindDest(BasicBlock::Create(C));
   OperandBundleDef OldBundle("before", UndefValue::get(Int32Ty));
-  std::unique_ptr<InvokeInst> Invoke(InvokeInst::Create(
-      Callee, NormalDest.get(), UnwindDest.get(), Args, OldBundle, "result"));
+  std::unique_ptr<InvokeInst> Invoke(
+      InvokeInst::Create(FnTy, Callee, NormalDest.get(), UnwindDest.get(), Args,
+                         OldBundle, "result"));
   AttrBuilder AB;
   AB.addAttribute(Attribute::Cold);
   Invoke->setAttributes(
@@ -646,7 +647,8 @@ TEST_F(ModuleWithFunctionTest, DropPoisonGeneratingFlags) {
 
   {
     Value *GEPBase = Constant::getNullValue(B.getInt8PtrTy());
-    auto *GI = cast<GetElementPtrInst>(B.CreateInBoundsGEP(GEPBase, {Arg0}));
+    auto *GI = cast<GetElementPtrInst>(
+        B.CreateInBoundsGEP(B.getInt8Ty(), GEPBase, Arg0));
     ASSERT_TRUE(GI->isInBounds());
     GI->dropPoisonGeneratingFlags();
     ASSERT_FALSE(GI->isInBounds());
diff --git a/unittests/IR/IntrinsicsTest.cpp b/unittests/IR/IntrinsicsTest.cpp
index 0d12126b1fcf2e07e34a57c49587d04c357c0fa5..a500346b66a5e40d895a8fc7191ddb42850e7975 100644
--- a/unittests/IR/IntrinsicsTest.cpp
+++ b/unittests/IR/IntrinsicsTest.cpp
@@ -1,9 +1,8 @@
 //===- llvm/unittest/IR/IntrinsicsTest.cpp - ------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/IR/LegacyPassManagerTest.cpp b/unittests/IR/LegacyPassManagerTest.cpp
index 9f5f431a55850ff710e7e080212b3545e43f4c4b..ea3b112f2a54939fe063ed4f72c4758597949712 100644
--- a/unittests/IR/LegacyPassManagerTest.cpp
+++ b/unittests/IR/LegacyPassManagerTest.cpp
@@ -1,9 +1,8 @@
 //===- llvm/unittest/IR/LegacyPassManager.cpp - Legacy PassManager tests --===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/unittests/IR/MDBuilderTest.cpp b/unittests/IR/MDBuilderTest.cpp
index ab2d34e89db6e7de342d8e4574d44dbab9f784cc..f8d08b5f3eae6cb437f3b9d6e1f1179ba2b0714d 100644
--- a/unittests/IR/MDBuilderTest.cpp
+++ b/unittests/IR/MDBuilderTest.cpp
@@ -1,9 +1,8 @@
 //===- llvm/unittests/MDBuilderTest.cpp - MDBuilder unit tests ------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/IR/ManglerTest.cpp b/unittests/IR/ManglerTest.cpp
index 04f1ca6cf9776fc206bf60bbe2daa614f8cc4d5f..f0ef15e5c5cd712061a7c3975b49e17ac2050a57 100644
--- a/unittests/IR/ManglerTest.cpp
+++ b/unittests/IR/ManglerTest.cpp
@@ -1,9 +1,8 @@
 //===- llvm/unittest/IR/ManglerTest.cpp - Mangler unit tests --------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/IR/MetadataTest.cpp b/unittests/IR/MetadataTest.cpp
index 3f744cd5e6d5fb0ee37276b1a3f17d6ab9ed70a6..7b9b7c4d89869af9a61090d7385909e58557f2d9 100644
--- a/unittests/IR/MetadataTest.cpp
+++ b/unittests/IR/MetadataTest.cpp
@@ -1,9 +1,8 @@
 //===- unittests/IR/MetadataTest.cpp - Metadata unit tests ----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -118,8 +117,9 @@ protected:
         32, 32, 0, DINode::FlagZero, nullptr, 0, nullptr, nullptr, "");
   }
   Function *getFunction(StringRef Name) {
-    return cast<Function>(M.getOrInsertFunction(
-        Name, FunctionType::get(Type::getVoidTy(Context), None, false)));
+    return Function::Create(
+        FunctionType::get(Type::getVoidTy(Context), None, false),
+        Function::ExternalLinkage, Name, M);
   }
 };
 typedef MetadataTest MDStringTest;
@@ -981,6 +981,113 @@ TEST_F(DILocationTest, cloneTemporary) {
   EXPECT_TRUE(L2->isTemporary());
 }
 
+TEST_F(DILocationTest, discriminatorEncoding) {
+  EXPECT_EQ(0U, DILocation::encodeDiscriminator(0, 0, 0).getValue());
+
+  // Encode base discriminator as a component: lsb is 0, then the value.
+  // The other components are all absent, so we leave all the other bits 0.
+  EXPECT_EQ(2U, DILocation::encodeDiscriminator(1, 0, 0).getValue());
+
+  // Base discriminator component is empty, so lsb is 1. Next component is not
+  // empty, so its lsb is 0, then its value (1). Next component is empty.
+  // So the bit pattern is 101.
+  EXPECT_EQ(5U, DILocation::encodeDiscriminator(0, 1, 0).getValue());
+
+  // First 2 components are empty, so the bit pattern is 11. Then the
+  // next component - ending up with 1011.
+  EXPECT_EQ(0xbU, DILocation::encodeDiscriminator(0, 0, 1).getValue());
+
+  // The bit pattern for the first 2 components is 11. The next bit is 0,
+  // because the last component is not empty. We have 29 bits usable for
+  // encoding, but we cap it at 12 bits uniformously for all components. We
+  // encode the last component over 14 bits.
+  EXPECT_EQ(0xfffbU, DILocation::encodeDiscriminator(0, 0, 0xfff).getValue());
+
+  EXPECT_EQ(0x102U, DILocation::encodeDiscriminator(1, 1, 0).getValue());
+
+  EXPECT_EQ(0x13eU, DILocation::encodeDiscriminator(0x1f, 1, 0).getValue());
+
+  EXPECT_EQ(0x87feU, DILocation::encodeDiscriminator(0x1ff, 1, 0).getValue());
+
+  EXPECT_EQ(0x1f3eU, DILocation::encodeDiscriminator(0x1f, 0x1f, 0).getValue());
+
+  EXPECT_EQ(0x3ff3eU,
+            DILocation::encodeDiscriminator(0x1f, 0x1ff, 0).getValue());
+
+  EXPECT_EQ(0x1ff87feU,
+            DILocation::encodeDiscriminator(0x1ff, 0x1ff, 0).getValue());
+
+  EXPECT_EQ(0xfff9f3eU,
+            DILocation::encodeDiscriminator(0x1f, 0x1f, 0xfff).getValue());
+
+  EXPECT_EQ(0xffc3ff3eU,
+            DILocation::encodeDiscriminator(0x1f, 0x1ff, 0x1ff).getValue());
+
+  EXPECT_EQ(0xffcf87feU,
+            DILocation::encodeDiscriminator(0x1ff, 0x1f, 0x1ff).getValue());
+
+  EXPECT_EQ(0xe1ff87feU,
+            DILocation::encodeDiscriminator(0x1ff, 0x1ff, 7).getValue());
+}
+
+TEST_F(DILocationTest, discriminatorEncodingNegativeTests) {
+  EXPECT_EQ(None, DILocation::encodeDiscriminator(0, 0, 0x1000));
+  EXPECT_EQ(None, DILocation::encodeDiscriminator(0x1000, 0, 0));
+  EXPECT_EQ(None, DILocation::encodeDiscriminator(0, 0x1000, 0));
+  EXPECT_EQ(None, DILocation::encodeDiscriminator(0, 0, 0x1000));
+  EXPECT_EQ(None, DILocation::encodeDiscriminator(0x1ff, 0x1ff, 8));
+  EXPECT_EQ(None,
+            DILocation::encodeDiscriminator(std::numeric_limits<uint32_t>::max(),
+                                            std::numeric_limits<uint32_t>::max(),
+                                            0));
+}
+
+TEST_F(DILocationTest, discriminatorSpecialCases) {
+  // We don't test getCopyIdentifier here because the only way
+  // to set it is by constructing an encoded discriminator using
+  // encodeDiscriminator, which is already tested.
+  auto L1 = DILocation::get(Context, 1, 2, getSubprogram());
+  EXPECT_EQ(0U, L1->getBaseDiscriminator());
+  EXPECT_EQ(1U, L1->getDuplicationFactor());
+
+  EXPECT_EQ(L1, L1->cloneWithBaseDiscriminator(0).getValue());
+  EXPECT_EQ(L1, L1->cloneByMultiplyingDuplicationFactor(0).getValue());
+  EXPECT_EQ(L1, L1->cloneByMultiplyingDuplicationFactor(1).getValue());
+
+  auto L2 = L1->cloneWithBaseDiscriminator(1).getValue();
+  EXPECT_EQ(0U, L1->getBaseDiscriminator());
+  EXPECT_EQ(1U, L1->getDuplicationFactor());
+
+  EXPECT_EQ(1U, L2->getBaseDiscriminator());
+  EXPECT_EQ(1U, L2->getDuplicationFactor());
+
+  auto L3 = L2->cloneByMultiplyingDuplicationFactor(2).getValue();
+  EXPECT_EQ(1U, L3->getBaseDiscriminator());
+  EXPECT_EQ(2U, L3->getDuplicationFactor());
+
+  EXPECT_EQ(L2, L2->cloneByMultiplyingDuplicationFactor(1).getValue());
+
+  auto L4 = L3->cloneByMultiplyingDuplicationFactor(4).getValue();
+  EXPECT_EQ(1U, L4->getBaseDiscriminator());
+  EXPECT_EQ(8U, L4->getDuplicationFactor());
+
+  auto L5 = L4->cloneWithBaseDiscriminator(2).getValue();
+  EXPECT_EQ(2U, L5->getBaseDiscriminator());
+  EXPECT_EQ(8U, L5->getDuplicationFactor());
+
+  // Check extreme cases
+  auto L6 = L1->cloneWithBaseDiscriminator(0xfff).getValue();
+  EXPECT_EQ(0xfffU, L6->getBaseDiscriminator());
+  EXPECT_EQ(0xfffU, L6->cloneByMultiplyingDuplicationFactor(0xfff)
+                        .getValue()
+                        ->getDuplicationFactor());
+
+  // Check we return None for unencodable cases.
+  EXPECT_EQ(None, L4->cloneWithBaseDiscriminator(0x1000));
+  EXPECT_EQ(None, L4->cloneByMultiplyingDuplicationFactor(0x1000));
+}
+
+
 typedef MetadataTest GenericDINodeTest;
 
 TEST_F(GenericDINodeTest, get) {
diff --git a/unittests/IR/ModuleTest.cpp b/unittests/IR/ModuleTest.cpp
index af55a098adda031788dd46e7f2277cf021687d89..ae420bb5406d5eee948b6540b699f4b95eae1891 100644
--- a/unittests/IR/ModuleTest.cpp
+++ b/unittests/IR/ModuleTest.cpp
@@ -1,9 +1,8 @@
 //===- unittests/IR/ModuleTest.cpp - Module unit tests --------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/IR/PassBuilderCallbacksTest.cpp b/unittests/IR/PassBuilderCallbacksTest.cpp
index 63fdc67b11524f53d97a3068e4102478a40ad1ed..7cd938fcddb9c47740e6d01878d137e1882a83d6 100644
--- a/unittests/IR/PassBuilderCallbacksTest.cpp
+++ b/unittests/IR/PassBuilderCallbacksTest.cpp
@@ -1,9 +1,8 @@
 //===- unittests/IR/PassBuilderCallbacksTest.cpp - PB Callback Tests --===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/IR/PassManagerTest.cpp b/unittests/IR/PassManagerTest.cpp
index 13b87f1022e5c853b8c6e8e28b527d89744988f7..304803d40814a38e5f19dc405a317faf5c548a82 100644
--- a/unittests/IR/PassManagerTest.cpp
+++ b/unittests/IR/PassManagerTest.cpp
@@ -1,9 +1,8 @@
 //===- llvm/unittest/IR/PassManager.cpp - PassManager tests ---------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/IR/PatternMatch.cpp b/unittests/IR/PatternMatch.cpp
index 976e42d0dbba1ce4305f785041b953d7aae6e43c..e380fb90ad781282e5cd97bd298bf9b3bd6d3006 100644
--- a/unittests/IR/PatternMatch.cpp
+++ b/unittests/IR/PatternMatch.cpp
@@ -1,9 +1,8 @@
 //===---- llvm/unittest/IR/PatternMatch.cpp - PatternMatch unit tests ----===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -398,7 +397,7 @@ TEST_F(PatternMatchTest, LoadStoreOps) {
   //  store i32 42, i32* %0
 
   Value *Alloca = IRB.CreateAlloca(IRB.getInt32Ty());
-  Value *LoadInst = IRB.CreateLoad(Alloca);
+  Value *LoadInst = IRB.CreateLoad(IRB.getInt32Ty(), Alloca);
   Value *FourtyTwo = IRB.getInt32(42);
   Value *StoreInst = IRB.CreateStore(FourtyTwo, Alloca);
   Value *MatchLoad, *MatchStoreVal, *MatchStorePointer;
diff --git a/unittests/IR/TypeBuilderTest.cpp b/unittests/IR/TypeBuilderTest.cpp
deleted file mode 100644
index 9ba776543d944eb8f08bd3697f6be8b77f27ac68..0000000000000000000000000000000000000000
--- a/unittests/IR/TypeBuilderTest.cpp
+++ /dev/null
@@ -1,284 +0,0 @@
-//===- llvm/unittest/TypeBuilderTest.cpp - TypeBuilder tests --------------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-
-#include "llvm/IR/TypeBuilder.h"
-#include "llvm/IR/LLVMContext.h"
-#include "gtest/gtest.h"
-
-using namespace llvm;
-
-namespace {
-
-TEST(TypeBuilderTest, Void) {
-  LLVMContext Context;
-  EXPECT_EQ(Type::getVoidTy(Context), (TypeBuilder<void, true>::get(Context)));
-  EXPECT_EQ(Type::getVoidTy(Context), (TypeBuilder<void, false>::get(Context)));
-  // Special cases for C compatibility:
-  EXPECT_EQ(Type::getInt8PtrTy(Context),
-            (TypeBuilder<void *, false>::get(Context)));
-  EXPECT_EQ(Type::getInt8PtrTy(Context),
-            (TypeBuilder<const void *, false>::get(Context)));
-  EXPECT_EQ(Type::getInt8PtrTy(Context),
-            (TypeBuilder<volatile void *, false>::get(Context)));
-  EXPECT_EQ(Type::getInt8PtrTy(Context),
-            (TypeBuilder<const volatile void *, false>::get(Context)));
-}
-
-TEST(TypeBuilderTest, HostIntegers) {
-  LLVMContext Context;
-  EXPECT_EQ(Type::getInt8Ty(Context),
-            (TypeBuilder<int8_t, false>::get(Context)));
-  EXPECT_EQ(Type::getInt8Ty(Context),
-            (TypeBuilder<uint8_t, false>::get(Context)));
-  EXPECT_EQ(Type::getInt16Ty(Context),
-            (TypeBuilder<int16_t, false>::get(Context)));
-  EXPECT_EQ(Type::getInt16Ty(Context),
-            (TypeBuilder<uint16_t, false>::get(Context)));
-  EXPECT_EQ(Type::getInt32Ty(Context),
-            (TypeBuilder<int32_t, false>::get(Context)));
-  EXPECT_EQ(Type::getInt32Ty(Context),
-            (TypeBuilder<uint32_t, false>::get(Context)));
-  EXPECT_EQ(Type::getInt64Ty(Context),
-            (TypeBuilder<int64_t, false>::get(Context)));
-  EXPECT_EQ(Type::getInt64Ty(Context),
-            (TypeBuilder<uint64_t, false>::get(Context)));
-
-  EXPECT_EQ(IntegerType::get(Context, sizeof(size_t) * CHAR_BIT),
-            (TypeBuilder<size_t, false>::get(Context)));
-  EXPECT_EQ(IntegerType::get(Context, sizeof(ptrdiff_t) * CHAR_BIT),
-            (TypeBuilder<ptrdiff_t, false>::get(Context)));
-}
-
-TEST(TypeBuilderTest, CrossCompilableIntegers) {
-  LLVMContext Context;
-  EXPECT_EQ(IntegerType::get(Context, 1),
-            (TypeBuilder<types::i<1>, true>::get(Context)));
-  EXPECT_EQ(IntegerType::get(Context, 1),
-            (TypeBuilder<types::i<1>, false>::get(Context)));
-  EXPECT_EQ(IntegerType::get(Context, 72),
-            (TypeBuilder<types::i<72>, true>::get(Context)));
-  EXPECT_EQ(IntegerType::get(Context, 72),
-            (TypeBuilder<types::i<72>, false>::get(Context)));
-}
-
-TEST(TypeBuilderTest, Float) {
-  LLVMContext Context;
-  EXPECT_EQ(Type::getFloatTy(Context),
-            (TypeBuilder<float, false>::get(Context)));
-  EXPECT_EQ(Type::getDoubleTy(Context),
-            (TypeBuilder<double, false>::get(Context)));
-  // long double isn't supported yet.
-  EXPECT_EQ(Type::getFloatTy(Context),
-            (TypeBuilder<types::ieee_float, true>::get(Context)));
-  EXPECT_EQ(Type::getFloatTy(Context),
-            (TypeBuilder<types::ieee_float, false>::get(Context)));
-  EXPECT_EQ(Type::getDoubleTy(Context),
-            (TypeBuilder<types::ieee_double, true>::get(Context)));
-  EXPECT_EQ(Type::getDoubleTy(Context),
-            (TypeBuilder<types::ieee_double, false>::get(Context)));
-  EXPECT_EQ(Type::getX86_FP80Ty(Context),
-            (TypeBuilder<types::x86_fp80, true>::get(Context)));
-  EXPECT_EQ(Type::getX86_FP80Ty(Context),
-            (TypeBuilder<types::x86_fp80, false>::get(Context)));
-  EXPECT_EQ(Type::getFP128Ty(Context),
-            (TypeBuilder<types::fp128, true>::get(Context)));
-  EXPECT_EQ(Type::getFP128Ty(Context),
-            (TypeBuilder<types::fp128, false>::get(Context)));
-  EXPECT_EQ(Type::getPPC_FP128Ty(Context),
-            (TypeBuilder<types::ppc_fp128, true>::get(Context)));
-  EXPECT_EQ(Type::getPPC_FP128Ty(Context),
-            (TypeBuilder<types::ppc_fp128, false>::get(Context)));
-}
-
-TEST(TypeBuilderTest, Derived) {
-  LLVMContext Context;
-  EXPECT_EQ(PointerType::getUnqual(Type::getInt8PtrTy(Context)),
-            (TypeBuilder<int8_t **, false>::get(Context)));
-  EXPECT_EQ(ArrayType::get(Type::getInt8Ty(Context), 7),
-            (TypeBuilder<int8_t[7], false>::get(Context)));
-  EXPECT_EQ(ArrayType::get(Type::getInt8Ty(Context), 0),
-            (TypeBuilder<int8_t[], false>::get(Context)));
-
-  EXPECT_EQ(PointerType::getUnqual(Type::getInt8PtrTy(Context)),
-            (TypeBuilder<types::i<8> **, false>::get(Context)));
-  EXPECT_EQ(ArrayType::get(Type::getInt8Ty(Context), 7),
-            (TypeBuilder<types::i<8>[7], false>::get(Context)));
-  EXPECT_EQ(ArrayType::get(Type::getInt8Ty(Context), 0),
-            (TypeBuilder<types::i<8>[], false>::get(Context)));
-
-  EXPECT_EQ(PointerType::getUnqual(Type::getInt8PtrTy(Context)),
-            (TypeBuilder<types::i<8> **, true>::get(Context)));
-  EXPECT_EQ(ArrayType::get(Type::getInt8Ty(Context), 7),
-            (TypeBuilder<types::i<8>[7], true>::get(Context)));
-  EXPECT_EQ(ArrayType::get(Type::getInt8Ty(Context), 0),
-            (TypeBuilder<types::i<8>[], true>::get(Context)));
-
-  EXPECT_EQ(Type::getInt8Ty(Context),
-            (TypeBuilder<const int8_t, false>::get(Context)));
-  EXPECT_EQ(Type::getInt8Ty(Context),
-            (TypeBuilder<volatile int8_t, false>::get(Context)));
-  EXPECT_EQ(Type::getInt8Ty(Context),
-            (TypeBuilder<const volatile int8_t, false>::get(Context)));
-
-  EXPECT_EQ(Type::getInt8Ty(Context),
-            (TypeBuilder<const types::i<8>, false>::get(Context)));
-  EXPECT_EQ(Type::getInt8Ty(Context),
-            (TypeBuilder<volatile types::i<8>, false>::get(Context)));
-  EXPECT_EQ(Type::getInt8Ty(Context),
-            (TypeBuilder<const volatile types::i<8>, false>::get(Context)));
-
-  EXPECT_EQ(Type::getInt8Ty(Context),
-            (TypeBuilder<const types::i<8>, true>::get(Context)));
-  EXPECT_EQ(Type::getInt8Ty(Context),
-            (TypeBuilder<volatile types::i<8>, true>::get(Context)));
-  EXPECT_EQ(Type::getInt8Ty(Context),
-            (TypeBuilder<const volatile types::i<8>, true>::get(Context)));
-
-  EXPECT_EQ(Type::getInt8PtrTy(Context),
-            (TypeBuilder<const volatile int8_t *const volatile, false>::get(
-                Context)));
-}
-
-TEST(TypeBuilderTest, Functions) {
-  LLVMContext Context;
-  std::vector<Type*> params;
-  EXPECT_EQ(FunctionType::get(Type::getVoidTy(Context), params, false),
-            (TypeBuilder<void(), true>::get(Context)));
-  EXPECT_EQ(FunctionType::get(Type::getInt8Ty(Context), params, true),
-            (TypeBuilder<int8_t(...), false>::get(Context)));
-  params.push_back(TypeBuilder<int32_t *, false>::get(Context));
-  EXPECT_EQ(FunctionType::get(Type::getInt8Ty(Context), params, false),
-            (TypeBuilder<int8_t(const int32_t *), false>::get(Context)));
-  EXPECT_EQ(FunctionType::get(Type::getInt8Ty(Context), params, true),
-            (TypeBuilder<int8_t(const int32_t *, ...), false>::get(Context)));
-  params.push_back(TypeBuilder<char *, false>::get(Context));
-  EXPECT_EQ(FunctionType::get(Type::getInt8Ty(Context), params, false),
-            (TypeBuilder<int8_t(int32_t *, void *), false>::get(Context)));
-  EXPECT_EQ(FunctionType::get(Type::getInt8Ty(Context), params, true),
-            (TypeBuilder<int8_t(int32_t *, char *, ...), false>::get(Context)));
-  params.push_back(TypeBuilder<char, false>::get(Context));
-  EXPECT_EQ(
-      FunctionType::get(Type::getInt8Ty(Context), params, false),
-      (TypeBuilder<int8_t(int32_t *, void *, char), false>::get(Context)));
-  EXPECT_EQ(
-      FunctionType::get(Type::getInt8Ty(Context), params, true),
-      (TypeBuilder<int8_t(int32_t *, char *, char, ...), false>::get(Context)));
-  params.push_back(TypeBuilder<char, false>::get(Context));
-  EXPECT_EQ(FunctionType::get(Type::getInt8Ty(Context), params, false),
-            (TypeBuilder<int8_t(int32_t *, void *, char, char), false>::get(
-                Context)));
-  EXPECT_EQ(
-      FunctionType::get(Type::getInt8Ty(Context), params, true),
-      (TypeBuilder<int8_t(int32_t *, char *, char, char, ...), false>::get(
-          Context)));
-  params.push_back(TypeBuilder<char, false>::get(Context));
-  EXPECT_EQ(
-      FunctionType::get(Type::getInt8Ty(Context), params, false),
-      (TypeBuilder<int8_t(int32_t *, void *, char, char, char), false>::get(
-          Context)));
-  EXPECT_EQ(FunctionType::get(Type::getInt8Ty(Context), params, true),
-            (TypeBuilder<int8_t(int32_t *, char *, char, char, char, ...),
-                         false>::get(Context)));
-}
-
-TEST(TypeBuilderTest, Context) {
-  // We used to cache TypeBuilder results in static local variables.  This
-  // produced the same type for different contexts, which of course broke
-  // things.
-  LLVMContext context1;
-  EXPECT_EQ(&context1,
-            &(TypeBuilder<types::i<1>, true>::get(context1))->getContext());
-  LLVMContext context2;
-  EXPECT_EQ(&context2,
-            &(TypeBuilder<types::i<1>, true>::get(context2))->getContext());
-}
-
-struct MyType {
-  int a;
-  int *b;
-  void *array[1];
-};
-
-struct MyPortableType {
-  int32_t a;
-  int32_t *b;
-  void *array[1];
-};
-
-}  // anonymous namespace
-
-namespace llvm {
-template<bool cross> class TypeBuilder<MyType, cross> {
-public:
-  static StructType *get(LLVMContext &Context) {
-    // Using the static result variable ensures that the type is
-    // only looked up once.
-    std::vector<Type*> st;
-    st.push_back(TypeBuilder<int, cross>::get(Context));
-    st.push_back(TypeBuilder<int*, cross>::get(Context));
-    st.push_back(TypeBuilder<void*[], cross>::get(Context));
-    static StructType *const result = StructType::get(Context, st);
-    return result;
-  }
-
-  // You may find this a convenient place to put some constants
-  // to help with getelementptr.  They don't have any effect on
-  // the operation of TypeBuilder.
-  enum Fields {
-    FIELD_A,
-    FIELD_B,
-    FIELD_ARRAY
-  };
-};
-
-template<bool cross> class TypeBuilder<MyPortableType, cross> {
-public:
-  static StructType *get(LLVMContext &Context) {
-    // Using the static result variable ensures that the type is
-    // only looked up once.
-    std::vector<Type*> st;
-    st.push_back(TypeBuilder<types::i<32>, cross>::get(Context));
-    st.push_back(TypeBuilder<types::i<32>*, cross>::get(Context));
-    st.push_back(TypeBuilder<types::i<8>*[], cross>::get(Context));
-    static StructType *const result = StructType::get(Context, st);
-    return result;
-  }
-
-  // You may find this a convenient place to put some constants
-  // to help with getelementptr.  They don't have any effect on
-  // the operation of TypeBuilder.
-  enum Fields {
-    FIELD_A,
-    FIELD_B,
-    FIELD_ARRAY
-  };
-};
-}  // namespace llvm
-namespace {
-
-TEST(TypeBuilderTest, Extensions) {
-  LLVMContext Context;
-  EXPECT_EQ(PointerType::getUnqual(
-                StructType::get(TypeBuilder<int, false>::get(Context),
-                                TypeBuilder<int *, false>::get(Context),
-                                TypeBuilder<void *[], false>::get(Context))),
-            (TypeBuilder<MyType *, false>::get(Context)));
-  EXPECT_EQ(PointerType::getUnqual(StructType::get(
-                TypeBuilder<types::i<32>, false>::get(Context),
-                TypeBuilder<types::i<32> *, false>::get(Context),
-                TypeBuilder<types::i<8> *[], false>::get(Context))),
-            (TypeBuilder<MyPortableType *, false>::get(Context)));
-  EXPECT_EQ(PointerType::getUnqual(StructType::get(
-                TypeBuilder<types::i<32>, false>::get(Context),
-                TypeBuilder<types::i<32> *, false>::get(Context),
-                TypeBuilder<types::i<8> *[], false>::get(Context))),
-            (TypeBuilder<MyPortableType *, true>::get(Context)));
-}
-
-}  // anonymous namespace
diff --git a/unittests/IR/TypesTest.cpp b/unittests/IR/TypesTest.cpp
index f006db51de56746f650dfe7925632879d96a4724..a147367934fa1fc874002f4b74dc6173cae47892 100644
--- a/unittests/IR/TypesTest.cpp
+++ b/unittests/IR/TypesTest.cpp
@@ -1,9 +1,8 @@
 //===- llvm/unittest/IR/TypesTest.cpp - Type unit tests -------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/IR/UseTest.cpp b/unittests/IR/UseTest.cpp
index 51ac2f48689dba46a351d8454b25930c581fcf29..a647ec721081e26635a40ef06aea8ae1012099dd 100644
--- a/unittests/IR/UseTest.cpp
+++ b/unittests/IR/UseTest.cpp
@@ -1,9 +1,8 @@
 //===- llvm/unittest/IR/UseTest.cpp - Use unit tests ----------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/IR/UserTest.cpp b/unittests/IR/UserTest.cpp
index 794dfc133bba5a7e5deb0c66be6dd5d54418a398..e495735aec6532b282e8614f01f6524c6d9e3d4d 100644
--- a/unittests/IR/UserTest.cpp
+++ b/unittests/IR/UserTest.cpp
@@ -1,9 +1,8 @@
 //===- llvm/unittest/IR/UserTest.cpp - User unit tests --------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/IR/ValueHandleTest.cpp b/unittests/IR/ValueHandleTest.cpp
index 9a4ce156dc3df1ec6ed682c9664e5e1335102cf2..b799ff0958065465e244b2062b549e381f54ff83 100644
--- a/unittests/IR/ValueHandleTest.cpp
+++ b/unittests/IR/ValueHandleTest.cpp
@@ -1,9 +1,8 @@
 //===- ValueHandleTest.cpp - ValueHandle tests ----------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/IR/ValueMapTest.cpp b/unittests/IR/ValueMapTest.cpp
index cbcd6f6e53f4a1d9d2c1fb59043f80fb0cfd8bb2..9dcb4fa7fba391b6992932dabfba96618b3d168f 100644
--- a/unittests/IR/ValueMapTest.cpp
+++ b/unittests/IR/ValueMapTest.cpp
@@ -1,9 +1,8 @@
 //===- llvm/unittest/ADT/ValueMapTest.cpp - ValueMap unit tests -*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/IR/ValueTest.cpp b/unittests/IR/ValueTest.cpp
index 90466b2505ecd40fa0fa83430c7f42218b162427..03523a74e1aa3216a005a3c51dcdb5fd97a50051 100644
--- a/unittests/IR/ValueTest.cpp
+++ b/unittests/IR/ValueTest.cpp
@@ -1,9 +1,8 @@
 //===- llvm/unittest/IR/ValueTest.cpp - Value unit tests ------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/IR/VerifierTest.cpp b/unittests/IR/VerifierTest.cpp
index 53e5d6fbbe4d5cde69772b5e36e2f5c8b0d07ff1..a85f0a25fc8c3dce5edfbf90ba5e9367fa054ada 100644
--- a/unittests/IR/VerifierTest.cpp
+++ b/unittests/IR/VerifierTest.cpp
@@ -1,9 +1,8 @@
 //===- llvm/unittest/IR/VerifierTest.cpp - Verifier unit tests --*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -27,7 +26,7 @@ TEST(VerifierTest, Branch_i1) {
   LLVMContext C;
   Module M("M", C);
   FunctionType *FTy = FunctionType::get(Type::getVoidTy(C), /*isVarArg=*/false);
-  Function *F = cast<Function>(M.getOrInsertFunction("foo", FTy));
+  Function *F = Function::Create(FTy, Function::ExternalLinkage, "foo", M);
   BasicBlock *Entry = BasicBlock::Create(C, "entry", F);
   BasicBlock *Exit = BasicBlock::Create(C, "exit", F);
   ReturnInst::Create(C, Exit);
@@ -50,7 +49,7 @@ TEST(VerifierTest, InvalidRetAttribute) {
   LLVMContext C;
   Module M("M", C);
   FunctionType *FTy = FunctionType::get(Type::getInt32Ty(C), /*isVarArg=*/false);
-  Function *F = cast<Function>(M.getOrInsertFunction("foo", FTy));
+  Function *F = Function::Create(FTy, Function::ExternalLinkage, "foo", M);
   AttributeList AS = F->getAttributes();
   F->setAttributes(
       AS.addAttribute(C, AttributeList::ReturnIndex, Attribute::UWTable));
@@ -68,9 +67,9 @@ TEST(VerifierTest, CrossModuleRef) {
   Module M2("M2", C);
   Module M3("M3", C);
   FunctionType *FTy = FunctionType::get(Type::getInt32Ty(C), /*isVarArg=*/false);
-  Function *F1 = cast<Function>(M1.getOrInsertFunction("foo1", FTy));
-  Function *F2 = cast<Function>(M2.getOrInsertFunction("foo2", FTy));
-  Function *F3 = cast<Function>(M3.getOrInsertFunction("foo3", FTy));
+  Function *F1 = Function::Create(FTy, Function::ExternalLinkage, "foo1", M1);
+  Function *F2 = Function::Create(FTy, Function::ExternalLinkage, "foo2", M2);
+  Function *F3 = Function::Create(FTy, Function::ExternalLinkage, "foo3", M3);
 
   BasicBlock *Entry1 = BasicBlock::Create(C, "entry", F1);
   BasicBlock *Entry3 = BasicBlock::Create(C, "entry", F3);
@@ -174,8 +173,8 @@ TEST(VerifierTest, DetectInvalidDebugInfo) {
     new GlobalVariable(M, Type::getInt8Ty(C), false,
                        GlobalValue::ExternalLinkage, nullptr, "g");
 
-    auto *F = cast<Function>(M.getOrInsertFunction(
-        "f", FunctionType::get(Type::getVoidTy(C), false)));
+    auto *F = Function::Create(FunctionType::get(Type::getVoidTy(C), false),
+                               Function::ExternalLinkage, "f", M);
     IRBuilder<> Builder(BasicBlock::Create(C, "", F));
     Builder.CreateUnreachable();
     F->setSubprogram(DIB.createFunction(
diff --git a/unittests/IR/WaymarkTest.cpp b/unittests/IR/WaymarkTest.cpp
index 4d2671c06c1d9ffa4d6a96cfe2927c82f9d1ac7e..2f64fe0ae99e73f7d3d22948d8f80d4587cbf090 100644
--- a/unittests/IR/WaymarkTest.cpp
+++ b/unittests/IR/WaymarkTest.cpp
@@ -1,9 +1,8 @@
 //===- llvm/unittest/IR/WaymarkTest.cpp - getUser() unit tests ------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/LineEditor/LineEditor.cpp b/unittests/LineEditor/LineEditor.cpp
index 4d9081fd5c201ee02ccefa53f16191b12ad7b407..42e5c6c3c01f5320327c39f130de25b256a87ba6 100644
--- a/unittests/LineEditor/LineEditor.cpp
+++ b/unittests/LineEditor/LineEditor.cpp
@@ -1,9 +1,8 @@
 //===-- LineEditor.cpp ----------------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/Linker/LinkModulesTest.cpp b/unittests/Linker/LinkModulesTest.cpp
index b8e2449437c0ecb64f92681adffaeb37a568b8ec..832e50c19ee1fc29547e2570bf7291fdae62f18d 100644
--- a/unittests/Linker/LinkModulesTest.cpp
+++ b/unittests/Linker/LinkModulesTest.cpp
@@ -1,9 +1,8 @@
 //===- llvm/unittest/Linker/LinkModulesTest.cpp - IRBuilder tests ---------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -84,7 +83,7 @@ TEST_F(LinkModuleTest, BlockAddress) {
   GEPIndices.push_back(&*F->arg_begin());
 
   Value *GEP = Builder.CreateGEP(AT, GV, GEPIndices, "switch.gep");
-  Value *Load = Builder.CreateLoad(GEP, "switch.load");
+  Value *Load = Builder.CreateLoad(AT->getElementType(), GEP, "switch.load");
 
   Builder.CreateRet(Load);
 
diff --git a/unittests/MC/Disassembler.cpp b/unittests/MC/Disassembler.cpp
index ca9581af83a01ee6a8c7fbcc9efaa74bd04126a5..cdb344474ca26d40a434e5c1dc9b62bf43052921 100644
--- a/unittests/MC/Disassembler.cpp
+++ b/unittests/MC/Disassembler.cpp
@@ -1,9 +1,8 @@
 //===- llvm/unittest/Object/Disassembler.cpp ------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/MC/DwarfLineTables.cpp b/unittests/MC/DwarfLineTables.cpp
index 1b1a4d647ce0f462e90ebd527a88bb5a176aaa60..95ae72bceac3555a798169f859048474d78a9abd 100644
--- a/unittests/MC/DwarfLineTables.cpp
+++ b/unittests/MC/DwarfLineTables.cpp
@@ -1,9 +1,8 @@
 //===- llvm/unittest/MC/DwarfLineTables.cpp ------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/MC/StringTableBuilderTest.cpp b/unittests/MC/StringTableBuilderTest.cpp
index b547f934b02c88e674d837fef06eccfcfa49ec09..81b7fa36beeb83cd07bb4e03e728ffdf397bd652 100644
--- a/unittests/MC/StringTableBuilderTest.cpp
+++ b/unittests/MC/StringTableBuilderTest.cpp
@@ -1,9 +1,8 @@
 //===----------- StringTableBuilderTest.cpp -------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/MC/TargetRegistry.cpp b/unittests/MC/TargetRegistry.cpp
index eb46b22f06c1f2939cfcc04b89f5d57f02938fed..587b2eeffdd7544274feb1bbca2c61ecd3be7968 100644
--- a/unittests/MC/TargetRegistry.cpp
+++ b/unittests/MC/TargetRegistry.cpp
@@ -1,9 +1,8 @@
 //===- unittests/MC/TargetRegistry.cpp ------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/Object/SymbolSizeTest.cpp b/unittests/Object/SymbolSizeTest.cpp
index ad9c40b986db56da5132f84c24d0e54f8c51909e..41269a6111615603c5ce0f81bfa0af0701d5ba31 100644
--- a/unittests/Object/SymbolSizeTest.cpp
+++ b/unittests/Object/SymbolSizeTest.cpp
@@ -1,9 +1,8 @@
 //===- SymbolSizeTest.cpp - Tests for SymbolSize.cpp ----------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/Object/SymbolicFileTest.cpp b/unittests/Object/SymbolicFileTest.cpp
index ec954e5e67d08abb90eaf720a83a84cb146c6ea4..3fc5d540e21adab77174683ee7890b2f96cbe3ba 100644
--- a/unittests/Object/SymbolicFileTest.cpp
+++ b/unittests/Object/SymbolicFileTest.cpp
@@ -1,9 +1,8 @@
 //===- SymbolicFileTest.cpp - Tests for SymbolicFile.cpp ------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/ObjectYAML/YAMLTest.cpp b/unittests/ObjectYAML/YAMLTest.cpp
index 606e160d19a5be12e432b5f00fbb67d17934d882..3d283a03b22567b2894169d227449446559cd6ac 100644
--- a/unittests/ObjectYAML/YAMLTest.cpp
+++ b/unittests/ObjectYAML/YAMLTest.cpp
@@ -1,9 +1,8 @@
 //===- YAMLTest.cpp - Tests for Object YAML -------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/OptRemarks/OptRemarksParsingTest.cpp b/unittests/OptRemarks/OptRemarksParsingTest.cpp
index a3b28f038b5033c87ca0431abee84d0e0c253b09..baadffefddea277ec68c10a5910b9e2913611367 100644
--- a/unittests/OptRemarks/OptRemarksParsingTest.cpp
+++ b/unittests/OptRemarks/OptRemarksParsingTest.cpp
@@ -1,9 +1,8 @@
 //===- unittest/Support/OptRemarksParsingTest.cpp - OptTable tests --------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/Option/OptionParsingTest.cpp b/unittests/Option/OptionParsingTest.cpp
index 230049c8bd03914d13bb4a0cb2a92974af7d6988..ae3f6522eda232306806d0ed71815189160ba76a 100644
--- a/unittests/Option/OptionParsingTest.cpp
+++ b/unittests/Option/OptionParsingTest.cpp
@@ -1,9 +1,8 @@
 //===- unittest/Support/OptionParsingTest.cpp - OptTable tests ------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/Passes/CMakeLists.txt b/unittests/Passes/CMakeLists.txt
index 415f3a71734b053e1c2215864b4773d94a52a7b8..3e83b527958a65f0289da572b0144b13443671ef 100644
--- a/unittests/Passes/CMakeLists.txt
+++ b/unittests/Passes/CMakeLists.txt
@@ -15,7 +15,7 @@ export_executable_symbols(PluginsTests)
 target_link_libraries(PluginsTests PRIVATE LLVMTestingSupport)
 
 set(LLVM_LINK_COMPONENTS)
-add_llvm_loadable_module(TestPlugin
+add_llvm_library(TestPlugin MODULE BUILDTREE_ONLY
   TestPlugin.cpp
   )
 
diff --git a/unittests/Passes/PluginsTest.cpp b/unittests/Passes/PluginsTest.cpp
index abb7b57ee0c60fdb972836e497db0d47b513e327..dd889e68492063811a90c9409039462f2516e973 100644
--- a/unittests/Passes/PluginsTest.cpp
+++ b/unittests/Passes/PluginsTest.cpp
@@ -1,9 +1,8 @@
 //===- unittests/Passes/Plugins/PluginsTest.cpp ---------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/Passes/TestPlugin.cpp b/unittests/Passes/TestPlugin.cpp
index 3dc5cdc84768f00d1512b3bc2294fc78632c834a..e0ae861f16fc153a9934b45eab12e055f5fffe47 100644
--- a/unittests/Passes/TestPlugin.cpp
+++ b/unittests/Passes/TestPlugin.cpp
@@ -1,9 +1,8 @@
 //===- unittests/Passes/Plugins/Plugin.cpp --------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/ProfileData/CoverageMappingTest.cpp b/unittests/ProfileData/CoverageMappingTest.cpp
index 987f6769958549bbf14205aac5bac3b7d3743414..6b94fa16bfa74b7ff17a331a56469e2fff3fc897 100644
--- a/unittests/ProfileData/CoverageMappingTest.cpp
+++ b/unittests/ProfileData/CoverageMappingTest.cpp
@@ -1,9 +1,8 @@
 //===- unittest/ProfileData/CoverageMappingTest.cpp -------------------------=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/ProfileData/InstrProfTest.cpp b/unittests/ProfileData/InstrProfTest.cpp
index 2d915d4459824227c36ba5c44f488f895629f496..19daf5cdb5fddc6fac8d2d0e80875898510fb283 100644
--- a/unittests/ProfileData/InstrProfTest.cpp
+++ b/unittests/ProfileData/InstrProfTest.cpp
@@ -1,9 +1,8 @@
 //===- unittest/ProfileData/InstrProfTest.cpp -------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/ProfileData/SampleProfTest.cpp b/unittests/ProfileData/SampleProfTest.cpp
index a31eccefb4d65cc0bad2b26a03fa744de7ccb75c..c25b3d2752b3cd8190269d76029e12ab2faf9593 100644
--- a/unittests/ProfileData/SampleProfTest.cpp
+++ b/unittests/ProfileData/SampleProfTest.cpp
@@ -1,9 +1,8 @@
 //===- unittest/ProfileData/SampleProfTest.cpp ------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -128,15 +127,17 @@ struct SampleProfTest : ::testing::Test {
 
     FunctionSamples *ReadFooSamples = Reader->getSamplesFor(FooName);
     ASSERT_TRUE(ReadFooSamples != nullptr);
-    if (Format != SampleProfileFormat::SPF_Compact_Binary)
+    if (Format != SampleProfileFormat::SPF_Compact_Binary) {
       ASSERT_EQ("_Z3fooi", ReadFooSamples->getName());
+    }
     ASSERT_EQ(7711u, ReadFooSamples->getTotalSamples());
     ASSERT_EQ(610u, ReadFooSamples->getHeadSamples());
 
     FunctionSamples *ReadBarSamples = Reader->getSamplesFor(BarName);
     ASSERT_TRUE(ReadBarSamples != nullptr);
-    if (Format != SampleProfileFormat::SPF_Compact_Binary)
+    if (Format != SampleProfileFormat::SPF_Compact_Binary) {
       ASSERT_EQ("_Z3bari", ReadBarSamples->getName());
+    }
     ASSERT_EQ(20301u, ReadBarSamples->getTotalSamples());
     ASSERT_EQ(1437u, ReadBarSamples->getHeadSamples());
     ErrorOr<SampleRecord::CallTargetMap> CTMap =
diff --git a/unittests/Support/AlignOfTest.cpp b/unittests/Support/AlignOfTest.cpp
index 388ca11b776936949b308307d16f188b6c6b7bb8..94c6f5a0f53f7d2d2eb77728e363ff4b7fbf4b16 100644
--- a/unittests/Support/AlignOfTest.cpp
+++ b/unittests/Support/AlignOfTest.cpp
@@ -1,9 +1,8 @@
 //=== - llvm/unittest/Support/AlignOfTest.cpp - Alignment utility tests ---===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/Support/AllocatorTest.cpp b/unittests/Support/AllocatorTest.cpp
index 74b394f1b176ba5972220316a54b0ba8c2ddf6d9..a0223ea932b0b83fd1e35516dd5bbce7d7870f38 100644
--- a/unittests/Support/AllocatorTest.cpp
+++ b/unittests/Support/AllocatorTest.cpp
@@ -1,9 +1,8 @@
 //===- llvm/unittest/Support/AllocatorTest.cpp - BumpPtrAllocator tests ---===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/Support/ArrayRecyclerTest.cpp b/unittests/Support/ArrayRecyclerTest.cpp
index 1ff97ba9e2b99a6628501f6431d675f9e6dc7e06..264cf9740c5f71fb9c784459017d9111767bed55 100644
--- a/unittests/Support/ArrayRecyclerTest.cpp
+++ b/unittests/Support/ArrayRecyclerTest.cpp
@@ -1,9 +1,8 @@
 //===--- unittest/Support/ArrayRecyclerTest.cpp ---------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/Support/BinaryStreamTest.cpp b/unittests/Support/BinaryStreamTest.cpp
index 35a010e73c7bb88e177134b12b67f14449a2f98b..fe574072a11d4a6ccd24cf551055b21714d5f164 100644
--- a/unittests/Support/BinaryStreamTest.cpp
+++ b/unittests/Support/BinaryStreamTest.cpp
@@ -1,9 +1,8 @@
 //===- llvm/unittest/Support/BinaryStreamTest.cpp -------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/Support/BlockFrequencyTest.cpp b/unittests/Support/BlockFrequencyTest.cpp
index c1f5671815bdfbcbdc80a56bbb0e91fec15f4e6a..cf5cf9213501c471a051f469cf4a4ca2aecde34a 100644
--- a/unittests/Support/BlockFrequencyTest.cpp
+++ b/unittests/Support/BlockFrequencyTest.cpp
@@ -1,9 +1,8 @@
 //===- unittests/Support/BlockFrequencyTest.cpp - BlockFrequency tests ----===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/Support/BranchProbabilityTest.cpp b/unittests/Support/BranchProbabilityTest.cpp
index 54948baea515aa087a91d7accb35d346aec5b3b6..94626e796e2ec0f362d58dedbd9436e4695fd707 100644
--- a/unittests/Support/BranchProbabilityTest.cpp
+++ b/unittests/Support/BranchProbabilityTest.cpp
@@ -1,9 +1,8 @@
 //===- unittest/Support/BranchProbabilityTest.cpp - BranchProbability tests -=//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/Support/CMakeLists.txt b/unittests/Support/CMakeLists.txt
index f9c877fc557773a985e7b7785b3f96f6c76f1b43..40bf939400249548d5522f1faaad7887a27af6c7 100644
--- a/unittests/Support/CMakeLists.txt
+++ b/unittests/Support/CMakeLists.txt
@@ -83,7 +83,21 @@ target_link_libraries(SupportTests PRIVATE LLVMTestingSupport)
 # Disable all warning for AlignOfTest.cpp,
 # as it does things intentionally, and there is no reliable way of
 # disabling all warnings for all the compilers by using pragmas.
-set_source_files_properties(AlignOfTest.cpp PROPERTIES COMPILE_FLAGS -w)
+# Don't disable on MSVC, because all incriminated warnings are already disabled
+# in source; and because we would otherwise see this warning:
+#   cl : Command line warning D9025: overriding '/W4' with '/w'
+if(NOT MSVC)
+  set_source_files_properties(AlignOfTest.cpp PROPERTIES COMPILE_FLAGS -w)
+endif()
+if(MSVC)
+  if( CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 19.14 )
+    # Since VS2017 15.8, the following snippet: Failed<CustomSubError>()
+    # generates a warning:
+    # \svn\llvm\utils\unittest\googlemock\include\gmock\gmock-matchers.h(186):
+    #   warning C5046: 'testing::MatcherInterface<T>::~MatcherInterface': Symbol involving type with internal linkage not defined
+    set_source_files_properties(ErrorTest.cpp PROPERTIES COMPILE_FLAGS -wd5046)
+  endif()
+endif()
 
 # ManagedStatic.cpp uses <pthread>.
 target_link_libraries(SupportTests PRIVATE LLVMTestingSupport ${LLVM_PTHREAD_LIB})
diff --git a/unittests/Support/CachePruningTest.cpp b/unittests/Support/CachePruningTest.cpp
index 4bc2ad19ba43700e9bf3c833db5793182bc1115f..eeffe218ae6cbeeb11d3c2ed0827b9179f5f1621 100644
--- a/unittests/Support/CachePruningTest.cpp
+++ b/unittests/Support/CachePruningTest.cpp
@@ -1,9 +1,8 @@
 //===- CachePruningTest.cpp -----------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/Support/Casting.cpp b/unittests/Support/Casting.cpp
index 9a818f6bdebde085b069a52eb346655df24244d4..368ca17ef7cb087de32fe612163e5f2688dc1d27 100644
--- a/unittests/Support/Casting.cpp
+++ b/unittests/Support/Casting.cpp
@@ -1,9 +1,8 @@
 //===---------- llvm/unittest/Support/Casting.cpp - Casting tests ---------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/Support/Chrono.cpp b/unittests/Support/Chrono.cpp
index a6b76c81a9c05bb4ca67f0af7f81de0c0dc651bc..9a08a5c1bfdff409c2240b7d15727d32d6339399 100644
--- a/unittests/Support/Chrono.cpp
+++ b/unittests/Support/Chrono.cpp
@@ -1,9 +1,8 @@
 //===- llvm/unittest/Support/Chrono.cpp - Time utilities tests ------------===//
 //
-//           The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/Support/CommandLineTest.cpp b/unittests/Support/CommandLineTest.cpp
index a296912a305e4f7c3042a2dc8bbb82e5046df034..f9841f580b1c1cd4210d7f357a2513ef093f01d1 100644
--- a/unittests/Support/CommandLineTest.cpp
+++ b/unittests/Support/CommandLineTest.cpp
@@ -1,9 +1,8 @@
 //===- llvm/unittest/Support/CommandLineTest.cpp - CommandLine tests ------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -14,6 +13,7 @@
 #include "llvm/Config/config.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/InitLLVM.h"
+#include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/Program.h"
 #include "llvm/Support/StringSaver.h"
@@ -840,4 +840,292 @@ TEST(CommandLineTest, GetCommandLineArguments) {
 }
 #endif
 
+class OutputRedirector {
+public:
+  OutputRedirector(int RedirectFD)
+      : RedirectFD(RedirectFD), OldFD(dup(RedirectFD)) {
+    if (OldFD == -1 ||
+        sys::fs::createTemporaryFile("unittest-redirect", "", NewFD,
+                                     FilePath) ||
+        dup2(NewFD, RedirectFD) == -1)
+      Valid = false;
+  }
+
+  ~OutputRedirector() {
+    dup2(OldFD, RedirectFD);
+    close(OldFD);
+    close(NewFD);
+  }
+
+  SmallVector<char, 128> FilePath;
+  bool Valid = true;
+
+private:
+  int RedirectFD;
+  int OldFD;
+  int NewFD;
+};
+
+struct AutoDeleteFile {
+  SmallVector<char, 128> FilePath;
+  ~AutoDeleteFile() {
+    if (!FilePath.empty())
+      sys::fs::remove(std::string(FilePath.data(), FilePath.size()));
+  }
+};
+
+class PrintOptionInfoTest : public ::testing::Test {
+public:
+  // Return std::string because the output of a failing EXPECT check is
+  // unreadable for StringRef. It also avoids any lifetime issues.
+  template <typename... Ts> std::string runTest(Ts... OptionAttributes) {
+    AutoDeleteFile File;
+    {
+      OutputRedirector Stdout(fileno(stdout));
+      if (!Stdout.Valid)
+        return "";
+      File.FilePath = Stdout.FilePath;
+
+      StackOption<OptionValue> TestOption(Opt, cl::desc(HelpText),
+                                          OptionAttributes...);
+      printOptionInfo(TestOption, 25);
+      outs().flush();
+    }
+    auto Buffer = MemoryBuffer::getFile(File.FilePath);
+    if (!Buffer)
+      return "";
+    return Buffer->get()->getBuffer().str();
+  }
+
+  enum class OptionValue { Val };
+  const StringRef Opt = "some-option";
+  const StringRef HelpText = "some help";
+
+private:
+  // This is a workaround for cl::Option sub-classes having their
+  // printOptionInfo functions private.
+  void printOptionInfo(const cl::Option &O, size_t Width) {
+    O.printOptionInfo(Width);
+  }
+};
+
+TEST_F(PrintOptionInfoTest, PrintOptionInfoValueOptionalWithoutSentinel) {
+  std::string Output =
+      runTest(cl::ValueOptional,
+              cl::values(clEnumValN(OptionValue::Val, "v1", "desc1")));
+
+  // clang-format off
+  EXPECT_EQ(Output, ("  -" + Opt + "=<value> - " + HelpText + "\n"
+                     "    =v1                -   desc1\n")
+                        .str());
+  // clang-format on
+}
+
+TEST_F(PrintOptionInfoTest, PrintOptionInfoValueOptionalWithSentinel) {
+  std::string Output = runTest(
+      cl::ValueOptional, cl::values(clEnumValN(OptionValue::Val, "v1", "desc1"),
+                                    clEnumValN(OptionValue::Val, "", "")));
+
+  // clang-format off
+  EXPECT_EQ(Output,
+            ("  -" + Opt + "         - " + HelpText + "\n"
+             "  -" + Opt + "=<value> - " + HelpText + "\n"
+             "    =v1                -   desc1\n")
+                .str());
+  // clang-format on
+}
+
+TEST_F(PrintOptionInfoTest, PrintOptionInfoValueOptionalWithSentinelWithHelp) {
+  std::string Output = runTest(
+      cl::ValueOptional, cl::values(clEnumValN(OptionValue::Val, "v1", "desc1"),
+                                    clEnumValN(OptionValue::Val, "", "desc2")));
+
+  // clang-format off
+  EXPECT_EQ(Output, ("  -" + Opt + "         - " + HelpText + "\n"
+                     "  -" + Opt + "=<value> - " + HelpText + "\n"
+                     "    =v1                -   desc1\n"
+                     "    =<empty>           -   desc2\n")
+                        .str());
+  // clang-format on
+}
+
+TEST_F(PrintOptionInfoTest, PrintOptionInfoValueRequiredWithEmptyValueName) {
+  std::string Output = runTest(
+      cl::ValueRequired, cl::values(clEnumValN(OptionValue::Val, "v1", "desc1"),
+                                    clEnumValN(OptionValue::Val, "", "")));
+
+  // clang-format off
+  EXPECT_EQ(Output, ("  -" + Opt + "=<value> - " + HelpText + "\n"
+                     "    =v1                -   desc1\n"
+                     "    =<empty>\n")
+                        .str());
+  // clang-format on
+}
+
+TEST_F(PrintOptionInfoTest, PrintOptionInfoEmptyValueDescription) {
+  std::string Output = runTest(
+      cl::ValueRequired, cl::values(clEnumValN(OptionValue::Val, "v1", "")));
+
+  // clang-format off
+  EXPECT_EQ(Output,
+            ("  -" + Opt + "=<value> - " + HelpText + "\n"
+             "    =v1\n").str());
+  // clang-format on
+}
+
+class GetOptionWidthTest : public ::testing::Test {
+public:
+  enum class OptionValue { Val };
+
+  template <typename... Ts>
+  size_t runTest(StringRef ArgName, Ts... OptionAttributes) {
+    StackOption<OptionValue> TestOption(ArgName, cl::desc("some help"),
+                                        OptionAttributes...);
+    return getOptionWidth(TestOption);
+  }
+
+private:
+  // This is a workaround for cl::Option sub-classes having their
+  // printOptionInfo
+  // functions private.
+  size_t getOptionWidth(const cl::Option &O) { return O.getOptionWidth(); }
+};
+
+TEST_F(GetOptionWidthTest, GetOptionWidthArgNameLonger) {
+  StringRef ArgName("a-long-argument-name");
+  size_t ExpectedStrSize = ("  -" + ArgName + "=<value> - ").str().size();
+  EXPECT_EQ(
+      runTest(ArgName, cl::values(clEnumValN(OptionValue::Val, "v", "help"))),
+      ExpectedStrSize);
+}
+
+TEST_F(GetOptionWidthTest, GetOptionWidthFirstOptionNameLonger) {
+  StringRef OptName("a-long-option-name");
+  size_t ExpectedStrSize = ("    =" + OptName + " - ").str().size();
+  EXPECT_EQ(
+      runTest("a", cl::values(clEnumValN(OptionValue::Val, OptName, "help"),
+                              clEnumValN(OptionValue::Val, "b", "help"))),
+      ExpectedStrSize);
+}
+
+TEST_F(GetOptionWidthTest, GetOptionWidthSecondOptionNameLonger) {
+  StringRef OptName("a-long-option-name");
+  size_t ExpectedStrSize = ("    =" + OptName + " - ").str().size();
+  EXPECT_EQ(
+      runTest("a", cl::values(clEnumValN(OptionValue::Val, "b", "help"),
+                              clEnumValN(OptionValue::Val, OptName, "help"))),
+      ExpectedStrSize);
+}
+
+TEST_F(GetOptionWidthTest, GetOptionWidthEmptyOptionNameLonger) {
+  size_t ExpectedStrSize = StringRef("    =<empty> - ").size();
+  // The length of a=<value> (including indentation) is actually the same as the
+  // =<empty> string, so it is impossible to distinguish via testing the case
+  // where the empty string is picked from where the option name is picked.
+  EXPECT_EQ(runTest("a", cl::values(clEnumValN(OptionValue::Val, "b", "help"),
+                                    clEnumValN(OptionValue::Val, "", "help"))),
+            ExpectedStrSize);
+}
+
+TEST_F(GetOptionWidthTest,
+       GetOptionWidthValueOptionalEmptyOptionWithNoDescription) {
+  StringRef ArgName("a");
+  // The length of a=<value> (including indentation) is actually the same as the
+  // =<empty> string, so it is impossible to distinguish via testing the case
+  // where the empty string is ignored from where it is not ignored.
+  // The dash will not actually be printed, but the space it would take up is
+  // included to ensure a consistent column width.
+  size_t ExpectedStrSize = ("  -" + ArgName + "=<value> - ").str().size();
+  EXPECT_EQ(runTest(ArgName, cl::ValueOptional,
+                    cl::values(clEnumValN(OptionValue::Val, "value", "help"),
+                               clEnumValN(OptionValue::Val, "", ""))),
+            ExpectedStrSize);
+}
+
+TEST_F(GetOptionWidthTest,
+       GetOptionWidthValueRequiredEmptyOptionWithNoDescription) {
+  // The length of a=<value> (including indentation) is actually the same as the
+  // =<empty> string, so it is impossible to distinguish via testing the case
+  // where the empty string is picked from where the option name is picked
+  size_t ExpectedStrSize = StringRef("    =<empty> - ").size();
+  EXPECT_EQ(runTest("a", cl::ValueRequired,
+                    cl::values(clEnumValN(OptionValue::Val, "value", "help"),
+                               clEnumValN(OptionValue::Val, "", ""))),
+            ExpectedStrSize);
+}
+
+TEST(CommandLineTest, PrefixOptions) {
+  cl::ResetCommandLineParser();
+
+  StackOption<std::string, cl::list<std::string>> IncludeDirs(
+      "I", cl::Prefix, cl::desc("Declare an include directory"));
+
+  // Test non-prefixed variant works with cl::Prefix options.
+  EXPECT_TRUE(IncludeDirs.empty());
+  const char *args[] = {"prog", "-I=/usr/include"};
+  EXPECT_TRUE(
+      cl::ParseCommandLineOptions(2, args, StringRef(), &llvm::nulls()));
+  EXPECT_TRUE(IncludeDirs.size() == 1);
+  EXPECT_TRUE(IncludeDirs.front().compare("/usr/include") == 0);
+
+  IncludeDirs.erase(IncludeDirs.begin());
+  cl::ResetAllOptionOccurrences();
+
+  // Test non-prefixed variant works with cl::Prefix options when value is
+  // passed in following argument.
+  EXPECT_TRUE(IncludeDirs.empty());
+  const char *args2[] = {"prog", "-I", "/usr/include"};
+  EXPECT_TRUE(
+      cl::ParseCommandLineOptions(3, args2, StringRef(), &llvm::nulls()));
+  EXPECT_TRUE(IncludeDirs.size() == 1);
+  EXPECT_TRUE(IncludeDirs.front().compare("/usr/include") == 0);
+
+  IncludeDirs.erase(IncludeDirs.begin());
+  cl::ResetAllOptionOccurrences();
+
+  // Test prefixed variant works with cl::Prefix options.
+  EXPECT_TRUE(IncludeDirs.empty());
+  const char *args3[] = {"prog", "-I/usr/include"};
+  EXPECT_TRUE(
+      cl::ParseCommandLineOptions(2, args3, StringRef(), &llvm::nulls()));
+  EXPECT_TRUE(IncludeDirs.size() == 1);
+  EXPECT_TRUE(IncludeDirs.front().compare("/usr/include") == 0);
+
+  StackOption<std::string, cl::list<std::string>> MacroDefs(
+      "D", cl::AlwaysPrefix, cl::desc("Define a macro"),
+      cl::value_desc("MACRO[=VALUE]"));
+
+  cl::ResetAllOptionOccurrences();
+
+  // Test non-prefixed variant does not work with cl::AlwaysPrefix options:
+  // equal sign is part of the value.
+  EXPECT_TRUE(MacroDefs.empty());
+  const char *args4[] = {"prog", "-D=HAVE_FOO"};
+  EXPECT_TRUE(
+      cl::ParseCommandLineOptions(2, args4, StringRef(), &llvm::nulls()));
+  EXPECT_TRUE(MacroDefs.size() == 1);
+  EXPECT_TRUE(MacroDefs.front().compare("=HAVE_FOO") == 0);
+
+  MacroDefs.erase(MacroDefs.begin());
+  cl::ResetAllOptionOccurrences();
+
+  // Test non-prefixed variant does not allow value to be passed in following
+  // argument with cl::AlwaysPrefix options.
+  EXPECT_TRUE(MacroDefs.empty());
+  const char *args5[] = {"prog", "-D", "HAVE_FOO"};
+  EXPECT_FALSE(
+      cl::ParseCommandLineOptions(3, args5, StringRef(), &llvm::nulls()));
+  EXPECT_TRUE(MacroDefs.empty());
+
+  cl::ResetAllOptionOccurrences();
+
+  // Test prefixed variant works with cl::AlwaysPrefix options.
+  EXPECT_TRUE(MacroDefs.empty());
+  const char *args6[] = {"prog", "-DHAVE_FOO"};
+  EXPECT_TRUE(
+      cl::ParseCommandLineOptions(2, args6, StringRef(), &llvm::nulls()));
+  EXPECT_TRUE(MacroDefs.size() == 1);
+  EXPECT_TRUE(MacroDefs.front().compare("HAVE_FOO") == 0);
+}
+
 }  // anonymous namespace
diff --git a/unittests/Support/CompressionTest.cpp b/unittests/Support/CompressionTest.cpp
index e9f36c5eebdd89b85f6823b432b2210a6621eadb..cc7be431b62bc392508fd342c7db0dbbde995823 100644
--- a/unittests/Support/CompressionTest.cpp
+++ b/unittests/Support/CompressionTest.cpp
@@ -1,9 +1,8 @@
 //===- llvm/unittest/Support/CompressionTest.cpp - Compression tests ------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/unittests/Support/ConvertUTFTest.cpp b/unittests/Support/ConvertUTFTest.cpp
index dd6e0df3688fd9dea6051f0758fd8cb3071cf248..83019722332d3c50fab2429835bddcc232aba060 100644
--- a/unittests/Support/ConvertUTFTest.cpp
+++ b/unittests/Support/ConvertUTFTest.cpp
@@ -1,9 +1,8 @@
 //===- llvm/unittest/Support/ConvertUTFTest.cpp - ConvertUTF tests --------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/Support/CrashRecoveryTest.cpp b/unittests/Support/CrashRecoveryTest.cpp
index ac531b246c6ed0dff49cce381cd86b5a1111190b..d863dd06f09189b71c48ff04a687ef4e932c08c0 100644
--- a/unittests/Support/CrashRecoveryTest.cpp
+++ b/unittests/Support/CrashRecoveryTest.cpp
@@ -1,9 +1,8 @@
 //===- llvm/unittest/Support/CrashRecoveryTest.cpp ------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/Support/DJBTest.cpp b/unittests/Support/DJBTest.cpp
index b157b96e6b11052f947d2551e0c7d1d897dccbd4..c01bbe1eaa67c20955a28326b774e25632522bb1 100644
--- a/unittests/Support/DJBTest.cpp
+++ b/unittests/Support/DJBTest.cpp
@@ -1,9 +1,8 @@
 //===---------- llvm/unittest/Support/DJBTest.cpp -------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/Support/DataExtractorTest.cpp b/unittests/Support/DataExtractorTest.cpp
index 8b645524ec6cc31db5202d329f7265ea70b08246..5663dad5cb6594914514b007e5569f6519f6f8b9 100644
--- a/unittests/Support/DataExtractorTest.cpp
+++ b/unittests/Support/DataExtractorTest.cpp
@@ -1,9 +1,8 @@
 //===- llvm/unittest/Support/DataExtractorTest.cpp - DataExtractor tests --===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/Support/DebugCounterTest.cpp b/unittests/Support/DebugCounterTest.cpp
index 32bf56c9f2fbade369ec596dd307a9481059d4df..e7345b13cc1729da7573141a772deed4bc15aa7b 100644
--- a/unittests/Support/DebugCounterTest.cpp
+++ b/unittests/Support/DebugCounterTest.cpp
@@ -1,9 +1,8 @@
 //===- llvm/unittest/Support/DebugCounterTest.cpp -------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/Support/DebugTest.cpp b/unittests/Support/DebugTest.cpp
index 7db91ff0d854fb83876b8e86861e1a1860ec39cd..e68e3764b4383954e7746e95b1355748d9d1e9e1 100644
--- a/unittests/Support/DebugTest.cpp
+++ b/unittests/Support/DebugTest.cpp
@@ -1,9 +1,8 @@
 //===- llvm/unittest/Support/DebugTest.cpp --------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/Support/DynamicLibrary/DynamicLibraryTest.cpp b/unittests/Support/DynamicLibrary/DynamicLibraryTest.cpp
index 50a0f1e621fb3a0e2d4b64c76e7fea6dec727cb2..d1afd50fb4c4625c98b055bbcdbf765f679e0d0c 100644
--- a/unittests/Support/DynamicLibrary/DynamicLibraryTest.cpp
+++ b/unittests/Support/DynamicLibrary/DynamicLibraryTest.cpp
@@ -1,9 +1,8 @@
 //===- llvm/unittest/Support/DynamicLibrary/DynamicLibraryTest.cpp --------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/Support/DynamicLibrary/ExportedFuncs.cpp b/unittests/Support/DynamicLibrary/ExportedFuncs.cpp
index 370c8cb471d1480be932343f6b4d7500b1cc3185..e1c6596261f0af985d7f5d12e32ac99f9a0555dd 100644
--- a/unittests/Support/DynamicLibrary/ExportedFuncs.cpp
+++ b/unittests/Support/DynamicLibrary/ExportedFuncs.cpp
@@ -1,9 +1,8 @@
 //===- llvm/unittest/Support/DynamicLibrary/ExportedFuncs.cpp -------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/Support/DynamicLibrary/PipSqueak.cpp b/unittests/Support/DynamicLibrary/PipSqueak.cpp
index e2f1cf7de082992fccf5696fc0c99b5f9ff934cc..8ff2c30f4925a71fc443b97cb504e969cf5ddaee 100644
--- a/unittests/Support/DynamicLibrary/PipSqueak.cpp
+++ b/unittests/Support/DynamicLibrary/PipSqueak.cpp
@@ -1,9 +1,8 @@
 //===- llvm/unittest/Support/DynamicLibrary/PipSqueak.cpp -----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/Support/DynamicLibrary/PipSqueak.h b/unittests/Support/DynamicLibrary/PipSqueak.h
index b44c61d64dfe47faf26200109da923e880a69541..a32c06cdf5f6c153f3374f4597de0ec0fca3c3ce 100644
--- a/unittests/Support/DynamicLibrary/PipSqueak.h
+++ b/unittests/Support/DynamicLibrary/PipSqueak.h
@@ -1,9 +1,8 @@
 //===- llvm/unittest/Support/DynamicLibrary/PipSqueak.h -------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/Support/EndianStreamTest.cpp b/unittests/Support/EndianStreamTest.cpp
index 9f938eef879149b7bc46e0041ca576a9e81f8dd4..1e800ff5570b9267929d593934a027325e6f71a4 100644
--- a/unittests/Support/EndianStreamTest.cpp
+++ b/unittests/Support/EndianStreamTest.cpp
@@ -1,9 +1,8 @@
 //===- unittests/Support/EndianStreamTest.cpp - EndianStream.h tests ------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/Support/EndianTest.cpp b/unittests/Support/EndianTest.cpp
index c2b5572e574e368085a82b07d3a0605791c1753a..c2b292885377c1a27d143375b603a12e96a18d2d 100644
--- a/unittests/Support/EndianTest.cpp
+++ b/unittests/Support/EndianTest.cpp
@@ -1,9 +1,8 @@
 //===- unittests/Support/EndianTest.cpp - Endian.h tests ------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/Support/ErrnoTest.cpp b/unittests/Support/ErrnoTest.cpp
index 701ac96082514b0a22b8058db9199c843277d17a..747d9dde0b73752e53c23e8fc750a3a0c64743fb 100644
--- a/unittests/Support/ErrnoTest.cpp
+++ b/unittests/Support/ErrnoTest.cpp
@@ -1,9 +1,8 @@
 //===- ErrnoTest.cpp - Error handling unit tests --------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/Support/ErrorOrTest.cpp b/unittests/Support/ErrorOrTest.cpp
index 2ffc6e5b7940a6e80abe216bc00d8cdbb495dfd4..6e5bd2edd3129e57958a6c62229afce790c265c9 100644
--- a/unittests/Support/ErrorOrTest.cpp
+++ b/unittests/Support/ErrorOrTest.cpp
@@ -1,9 +1,8 @@
 //===- unittests/ErrorOrTest.cpp - ErrorOr.h tests ------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/Support/ErrorTest.cpp b/unittests/Support/ErrorTest.cpp
index eee2fe2d649379b9b32cc7813358d05023567e78..c4a9f3e51684aab06ee7a8513b1fd849f32d4186 100644
--- a/unittests/Support/ErrorTest.cpp
+++ b/unittests/Support/ErrorTest.cpp
@@ -1,9 +1,8 @@
 //===----- unittests/ErrorTest.cpp - Error.h tests ------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/Support/FileOutputBufferTest.cpp b/unittests/Support/FileOutputBufferTest.cpp
index 554859587d4572fa8830e8d41eba2b3ef662bee9..8afc2125ef46e14c2c36575fc66029f7436c0ae8 100644
--- a/unittests/Support/FileOutputBufferTest.cpp
+++ b/unittests/Support/FileOutputBufferTest.cpp
@@ -1,9 +1,8 @@
 //===- llvm/unittest/Support/FileOutputBuffer.cpp - unit tests ------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -122,53 +121,4 @@ TEST(FileOutputBuffer, Test) {
   // Clean up.
   ASSERT_NO_ERROR(fs::remove(TestDirectory.str()));
 }
-
-TEST(FileOutputBuffer, TestModify) {
-  // Create unique temporary directory for these tests
-  SmallString<128> TestDirectory;
-  {
-    ASSERT_NO_ERROR(
-      fs::createUniqueDirectory("FileOutputBuffer-modify", TestDirectory));
-  }
-
-  SmallString<128> File1(TestDirectory);
-  File1.append("/file");
-  // First write some data.
-  {
-    Expected<std::unique_ptr<FileOutputBuffer>> BufferOrErr =
-      FileOutputBuffer::create(File1, 10);
-    ASSERT_NO_ERROR(errorToErrorCode(BufferOrErr.takeError()));
-    std::unique_ptr<FileOutputBuffer> &Buffer = *BufferOrErr;
-    memcpy(Buffer->getBufferStart(), "AAAAAAAAAA", 10);
-    ASSERT_NO_ERROR(errorToErrorCode(Buffer->commit()));
-  }
-
-  // Then re-open the file for modify and change only some bytes.
-  {
-    Expected<std::unique_ptr<FileOutputBuffer>> BufferOrErr =
-        FileOutputBuffer::create(File1, size_t(-1), FileOutputBuffer::F_modify);
-    ASSERT_NO_ERROR(errorToErrorCode(BufferOrErr.takeError()));
-    std::unique_ptr<FileOutputBuffer> &Buffer = *BufferOrErr;
-    ASSERT_EQ(10U, Buffer->getBufferSize());
-    uint8_t *Data = Buffer->getBufferStart();
-    Data[0] = 'X';
-    Data[9] = 'X';
-    ASSERT_NO_ERROR(errorToErrorCode(Buffer->commit()));
-  }
-
-  // Finally, re-open the file for read and verify that it has the modified
-  // contents.
-  {
-    ErrorOr<std::unique_ptr<MemoryBuffer>> BufferOrErr = MemoryBuffer::getFile(File1);
-    ASSERT_NO_ERROR(BufferOrErr.getError());
-    std::unique_ptr<MemoryBuffer> Buffer = std::move(*BufferOrErr);
-    ASSERT_EQ(10U, Buffer->getBufferSize());
-    EXPECT_EQ(StringRef("XAAAAAAAAX"), Buffer->getBuffer());
-  }
-
-  // Clean up.
-  ASSERT_NO_ERROR(fs::remove(File1));
-  ASSERT_NO_ERROR(fs::remove(TestDirectory));
-}
-
 } // anonymous namespace
diff --git a/unittests/Support/FormatVariadicTest.cpp b/unittests/Support/FormatVariadicTest.cpp
index 91a44bae3a9dad2fc55669d64905b60a1b992a5a..e9565c13dd372859aa5abf60922322f47e3405f8 100644
--- a/unittests/Support/FormatVariadicTest.cpp
+++ b/unittests/Support/FormatVariadicTest.cpp
@@ -1,9 +1,8 @@
 //===- FormatVariadicTest.cpp - Unit tests for string formatting ----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/Support/GlobPatternTest.cpp b/unittests/Support/GlobPatternTest.cpp
index d7016ab218d320508313c2b83cfde7ce57d43fe6..113b20e047eae67113c605e3a4ebcf1e6b16383a 100644
--- a/unittests/Support/GlobPatternTest.cpp
+++ b/unittests/Support/GlobPatternTest.cpp
@@ -1,9 +1,8 @@
 //===- llvm/unittest/Support/GlobPatternTest.cpp --------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/Support/Host.cpp b/unittests/Support/Host.cpp
index 0d257a705910d214c851ed0794013ac84366b6cc..b809317147587ce3bdbf1a8c5cdc079257a6863e 100644
--- a/unittests/Support/Host.cpp
+++ b/unittests/Support/Host.cpp
@@ -1,9 +1,8 @@
 //========- unittests/Support/Host.cpp - Host.cpp tests --------------========//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/Support/ItaniumManglingCanonicalizerTest.cpp b/unittests/Support/ItaniumManglingCanonicalizerTest.cpp
index 2fd6bd8c558026954993a5388111f38aa0dcb132..91432899d1457204968c0208d361a735c41ba930 100644
--- a/unittests/Support/ItaniumManglingCanonicalizerTest.cpp
+++ b/unittests/Support/ItaniumManglingCanonicalizerTest.cpp
@@ -1,9 +1,8 @@
 //===-------------- ItaniumManglingCanonicalizerTest.cpp ------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is dual licensed under the MIT and the University of Illinois Open
-// Source Licenses. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/Support/JSONTest.cpp b/unittests/Support/JSONTest.cpp
index 9f2d47b9aa980be765cb60254eb417654fefafaa..23645b35f4395ef722273fe47cfb0718d4876070 100644
--- a/unittests/Support/JSONTest.cpp
+++ b/unittests/Support/JSONTest.cpp
@@ -1,9 +1,8 @@
 //===-- JSONTest.cpp - JSON unit tests --------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/Support/LEB128Test.cpp b/unittests/Support/LEB128Test.cpp
index 1c9b5dbd2bd9aa06007be5f8ac46a98cf97471ef..e429279ff9fbe9bfb4a25cb0af5eb69f18f714e6 100644
--- a/unittests/Support/LEB128Test.cpp
+++ b/unittests/Support/LEB128Test.cpp
@@ -1,9 +1,8 @@
 //===- llvm/unittest/Support/LEB128Test.cpp - LEB128 function tests -------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/Support/LineIteratorTest.cpp b/unittests/Support/LineIteratorTest.cpp
index 67f9d9777363b9a1a4f3b7c02e7ea3efe1a0d1ee..96cf37a26d3cf509ffadf4f40e541dc58575ac22 100644
--- a/unittests/Support/LineIteratorTest.cpp
+++ b/unittests/Support/LineIteratorTest.cpp
@@ -1,9 +1,8 @@
 //===- LineIterator.cpp - Unit tests --------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/Support/LockFileManagerTest.cpp b/unittests/Support/LockFileManagerTest.cpp
index 1775d05e44db2356b2fc540c35a226412f293167..5f1fe0fd3aeacb2fbf6225b4ac29d57b04dacd7b 100644
--- a/unittests/Support/LockFileManagerTest.cpp
+++ b/unittests/Support/LockFileManagerTest.cpp
@@ -1,9 +1,8 @@
 //===- unittests/LockFileManagerTest.cpp - LockFileManager tests ----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/Support/MD5Test.cpp b/unittests/Support/MD5Test.cpp
index bac1ec2f2b9018c1db26bb6600a204e538ce9c20..3a24e1fa69796bb7ac6261f3a9296fa2e277e7c8 100644
--- a/unittests/Support/MD5Test.cpp
+++ b/unittests/Support/MD5Test.cpp
@@ -1,9 +1,8 @@
 //===- llvm/unittest/Support/MD5Test.cpp - MD5 tests ----------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/unittests/Support/ManagedStatic.cpp b/unittests/Support/ManagedStatic.cpp
index d3cc80cf5e92c8eb5c7c997c9a90f75bacf01df2..3e9e3f52e41c383621d91590b8ab4617ac1a83f7 100644
--- a/unittests/Support/ManagedStatic.cpp
+++ b/unittests/Support/ManagedStatic.cpp
@@ -1,9 +1,8 @@
 //===- llvm/unittest/Support/ManagedStatic.cpp - ManagedStatic tests ------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/Support/MathExtrasTest.cpp b/unittests/Support/MathExtrasTest.cpp
index 694a1f24d03204fb8e549900f265c3ffd9227e9e..de8771402fbbd1cde134d047be7615e46652197a 100644
--- a/unittests/Support/MathExtrasTest.cpp
+++ b/unittests/Support/MathExtrasTest.cpp
@@ -1,9 +1,8 @@
 //===- unittests/Support/MathExtrasTest.cpp - math utils tests ------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/Support/MemoryBufferTest.cpp b/unittests/Support/MemoryBufferTest.cpp
index 034b2ea9495dec560f355e8fe0bc84c3d58a0818..bba662931d68acad684edd75fd7193422001054e 100644
--- a/unittests/Support/MemoryBufferTest.cpp
+++ b/unittests/Support/MemoryBufferTest.cpp
@@ -1,9 +1,8 @@
 //===- llvm/unittest/Support/MemoryBufferTest.cpp - MemoryBuffer tests ----===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/unittests/Support/MemoryTest.cpp b/unittests/Support/MemoryTest.cpp
index bc2fba05f6ad653c8ac3679e710d4374b0be9183..473a0da97c8eca4c7fca13368a36dbffd4f27091 100644
--- a/unittests/Support/MemoryTest.cpp
+++ b/unittests/Support/MemoryTest.cpp
@@ -1,9 +1,8 @@
 //===- llvm/unittest/Support/AllocatorTest.cpp - BumpPtrAllocator tests ---===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/Support/NativeFormatTests.cpp b/unittests/Support/NativeFormatTests.cpp
index 52acb6a99337dd0755b9795ba53b8e98029a62e7..bbe9aa6f6dc46e854e8fa60deb5cc470d10b4dfe 100644
--- a/unittests/Support/NativeFormatTests.cpp
+++ b/unittests/Support/NativeFormatTests.cpp
@@ -1,9 +1,8 @@
 //===- llvm/unittest/Support/NativeFormatTests.cpp - formatting tests -----===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/Support/ParallelTest.cpp b/unittests/Support/ParallelTest.cpp
index 8779a61b185383fdead51c0ec3d979f47c47bf6e..63c71522308b52d1d26cd31da295727aead5a587 100644
--- a/unittests/Support/ParallelTest.cpp
+++ b/unittests/Support/ParallelTest.cpp
@@ -1,9 +1,8 @@
 //===- llvm/unittest/Support/ParallelTest.cpp -----------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
diff --git a/unittests/Support/Path.cpp b/unittests/Support/Path.cpp
index 40faa669f87dd9d0597ee9d2c8a27617e2b5afc8..7ce6d2d52db600659df40dc089db09a1243cb444 100644
--- a/unittests/Support/Path.cpp
+++ b/unittests/Support/Path.cpp
@@ -1,9 +1,8 @@
 //===- llvm/unittest/Support/Path.cpp - Path tests ------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -187,7 +186,7 @@ TEST(Support, Path) {
   }
 
   SmallString<32> Relative("foo.cpp");
-  ASSERT_NO_ERROR(sys::fs::make_absolute("/root", Relative));
+  sys::fs::make_absolute("/root", Relative);
   Relative[5] = '/'; // Fix up windows paths.
   ASSERT_EQ("/root/foo.cpp", Relative);
 }
@@ -1665,7 +1664,9 @@ TEST_F(FileSystemTest, permissions) {
   EXPECT_TRUE(CheckPermissions(fs::set_gid_on_exe));
 
   // Modern BSDs require root to set the sticky bit on files.
-#if !defined(__FreeBSD__) && !defined(__NetBSD__) && !defined(__OpenBSD__)
+  // AIX without root will mask off (i.e., lose) the sticky bit on files.
+#if !defined(__FreeBSD__) && !defined(__NetBSD__) && !defined(__OpenBSD__) &&  \
+    !defined(_AIX)
   EXPECT_EQ(fs::setPermissions(TempPath, fs::sticky_bit), NoError);
   EXPECT_TRUE(CheckPermissions(fs::sticky_bit));
 
@@ -1685,7 +1686,7 @@ TEST_F(FileSystemTest, permissions) {
 
   EXPECT_EQ(fs::setPermissions(TempPath, fs::all_perms), NoError);
   EXPECT_TRUE(CheckPermissions(fs::all_perms));
-#endif // !FreeBSD && !NetBSD && !OpenBSD
+#endif // !FreeBSD && !NetBSD && !OpenBSD && !AIX
 
   EXPECT_EQ(fs::setPermissions(TempPath, fs::all_perms & ~fs::sticky_bit),
                                NoError);
diff --git a/unittests/Support/ProcessTest.cpp b/unittests/Support/ProcessTest.cpp
index 85b1839ddbcbaa7fa028d373c5de89a5d8e513de..dbfaf8ed0f614cc60a84e7fd1c49c78558106160 100644
--- a/unittests/Support/ProcessTest.cpp
+++ b/unittests/Support/ProcessTest.cpp
@@ -1,9 +1,8 @@
 //===- unittest/Support/ProcessTest.cpp -----------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/Support/ProgramTest.cpp b/unittests/Support/ProgramTest.cpp
index ec1c85a9f25a59aa3008d4e8885c6b068c04e6f6..85a1d532c9a659dc6611b2935c699cf4b56bc5b6 100644
--- a/unittests/Support/ProgramTest.cpp
+++ b/unittests/Support/ProgramTest.cpp
@@ -1,9 +1,8 @@
 //===- unittest/Support/ProgramTest.cpp -----------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/Support/RegexTest.cpp b/unittests/Support/RegexTest.cpp
index 7e44a3c0614a184c241b741438c684e51df918d0..eb8160e46666570a80ea1c913551be5c013fcbf6 100644
--- a/unittests/Support/RegexTest.cpp
+++ b/unittests/Support/RegexTest.cpp
@@ -1,9 +1,8 @@
 //===- llvm/unittest/Support/RegexTest.cpp - Regex tests --===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/Support/ReplaceFileTest.cpp b/unittests/Support/ReplaceFileTest.cpp
index 15143be794fcab94873abbfb835b1ed0f791d638..1e498c0d2c48d742b107a1e5c4bb35531d146a6a 100644
--- a/unittests/Support/ReplaceFileTest.cpp
+++ b/unittests/Support/ReplaceFileTest.cpp
@@ -1,9 +1,8 @@
 //===- llvm/unittest/Support/ReplaceFileTest.cpp - unit tests -------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/Support/ReverseIterationTest.cpp b/unittests/Support/ReverseIterationTest.cpp
index 930bd43d11b27874e4efe885009cbaec6a54d4ce..05804e69c26026fbb416224b7a36a2c93f718bc5 100644
--- a/unittests/Support/ReverseIterationTest.cpp
+++ b/unittests/Support/ReverseIterationTest.cpp
@@ -1,9 +1,8 @@
 //===- llvm/unittest/Support/ReverseIterationTest.cpp ---------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===---------------------------------------------------------------------===//
 //
diff --git a/unittests/Support/ScaledNumberTest.cpp b/unittests/Support/ScaledNumberTest.cpp
index 9e3f6de6bd17c75b702559db19734b7ab065f73a..3fa63b7bf3c72b84d069de0adb2b393b56c0a23b 100644
--- a/unittests/Support/ScaledNumberTest.cpp
+++ b/unittests/Support/ScaledNumberTest.cpp
@@ -1,9 +1,8 @@
 //===- llvm/unittest/Support/ScaledNumberTest.cpp - ScaledPair tests -----==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -563,4 +562,7 @@ TEST(ScaledNumberHelpersTest, toIntBug) {
   EXPECT_EQ(1u, (n * n).toInt<uint32_t>());
 }
 
+static_assert(is_trivially_copyable<ScaledNumber<uint32_t>>::value,
+              "trivially copyable");
+
 } // end namespace
diff --git a/unittests/Support/SourceMgrTest.cpp b/unittests/Support/SourceMgrTest.cpp
index 7bb76f556f70b287b2e67dce0c1b5fc1b4178e8c..a3079a403a10acb413a6af5a3ee7a49812ef1b71 100644
--- a/unittests/Support/SourceMgrTest.cpp
+++ b/unittests/Support/SourceMgrTest.cpp
@@ -1,9 +1,8 @@
 //===- unittests/Support/SourceMgrTest.cpp - SourceMgr tests --------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/Support/SpecialCaseListTest.cpp b/unittests/Support/SpecialCaseListTest.cpp
index 060703e102fc8e6fdc13df66a01428af77305116..2245588d97cae6c82c9a3a72345e2a2e71992e8f 100644
--- a/unittests/Support/SpecialCaseListTest.cpp
+++ b/unittests/Support/SpecialCaseListTest.cpp
@@ -1,9 +1,8 @@
 //===- SpecialCaseListTest.cpp - Unit tests for SpecialCaseList -----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/Support/StringPool.cpp b/unittests/Support/StringPool.cpp
index ac39fec059e50429956f4a9390b50200bb8b2b90..0a540e95acfd5a60f102cd657999c5cb7b3629c6 100644
--- a/unittests/Support/StringPool.cpp
+++ b/unittests/Support/StringPool.cpp
@@ -1,9 +1,8 @@
 //===- llvm/unittest/Support/StringPoiil.cpp - StringPool tests -----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/Support/SwapByteOrderTest.cpp b/unittests/Support/SwapByteOrderTest.cpp
index 474eac6bbb388f16afa0777301d063097fe2beda..9581f070475e14d9cca07cce4d3b2af13eda86ff 100644
--- a/unittests/Support/SwapByteOrderTest.cpp
+++ b/unittests/Support/SwapByteOrderTest.cpp
@@ -1,9 +1,8 @@
 //===- unittests/Support/SwapByteOrderTest.cpp - swap byte order test -----===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/Support/SymbolRemappingReaderTest.cpp b/unittests/Support/SymbolRemappingReaderTest.cpp
index 717df62344cf8e6c0ede0fba1f7a8b925e85857d..6db4b2a6da11b79265f5efd2878ac07612f17c2b 100644
--- a/unittests/Support/SymbolRemappingReaderTest.cpp
+++ b/unittests/Support/SymbolRemappingReaderTest.cpp
@@ -1,9 +1,8 @@
 //===- unittests/Support/SymbolRemappingReaderTest.cpp --------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/Support/TarWriterTest.cpp b/unittests/Support/TarWriterTest.cpp
index 901dd906ca78ae1062d2f64e7646f800506459df..bd67e036518f28f5037013c6c8ad28bad78b1dfa 100644
--- a/unittests/Support/TarWriterTest.cpp
+++ b/unittests/Support/TarWriterTest.cpp
@@ -1,9 +1,8 @@
 //===- llvm/unittest/Support/TarWriterTest.cpp ----------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/Support/TargetParserTest.cpp b/unittests/Support/TargetParserTest.cpp
index a6a6150d2b550706d470242b97622cc69c9f1189..53ccc8dd2b80a4b5238461cc5129ff4876d06488 100644
--- a/unittests/Support/TargetParserTest.cpp
+++ b/unittests/Support/TargetParserTest.cpp
@@ -1,9 +1,8 @@
 //===----------- TargetParser.cpp - Target Parser -------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -265,11 +264,12 @@ TEST(TargetParserTest, testARMCPU) {
                          ARM::AEK_VIRT | ARM::AEK_HWDIVARM |
                          ARM::AEK_HWDIVTHUMB | ARM::AEK_DSP,
                          "8-A"));
-  EXPECT_TRUE(testARMCPU("exynos-m4", "armv8-a", "crypto-neon-fp-armv8",
+  EXPECT_TRUE(testARMCPU("exynos-m4", "armv8.2-a", "crypto-neon-fp-armv8",
                          ARM::AEK_CRC | ARM::AEK_SEC | ARM::AEK_MP |
                          ARM::AEK_VIRT | ARM::AEK_HWDIVARM |
-                         ARM::AEK_HWDIVTHUMB | ARM::AEK_DSP,
-                         "8-A"));
+                         ARM::AEK_HWDIVTHUMB | ARM::AEK_DSP | ARM::AEK_DOTPROD |
+                         ARM::AEK_FP16 | ARM::AEK_RAS,
+                         "8.2-A"));
   EXPECT_TRUE(testARMCPU("cortex-m23", "armv8-m.base", "none",
                          ARM::AEK_HWDIVTHUMB, "8-M.Baseline"));
   EXPECT_TRUE(testARMCPU("cortex-m33", "armv8-m.main", "fpv5-sp-d16",
@@ -584,7 +584,8 @@ TEST(TargetParserTest, ARMArchExtFeature) {
                               {"iwmmxt", "noiwmmxt", nullptr, nullptr},
                               {"iwmmxt2", "noiwmmxt2", nullptr, nullptr},
                               {"maverick", "maverick", nullptr, nullptr},
-                              {"xscale", "noxscale", nullptr, nullptr}};
+                              {"xscale", "noxscale", nullptr, nullptr},
+                              {"sb", "nosb", "+sb", "-sb"}};
 
   for (unsigned i = 0; i < array_lengthof(ArchExt); i++) {
     EXPECT_EQ(StringRef(ArchExt[i][2]), ARM::getArchExtFeature(ArchExt[i][0]));
@@ -758,9 +759,11 @@ TEST(TargetParserTest, testAArch64CPU) {
       AArch64::AEK_CRC | AArch64::AEK_CRYPTO | AArch64::AEK_FP |
       AArch64::AEK_SIMD, "8-A"));
   EXPECT_TRUE(testAArch64CPU(
-      "exynos-m4", "armv8-a", "crypto-neon-fp-armv8",
-      AArch64::AEK_CRC | AArch64::AEK_CRYPTO | AArch64::AEK_FP |
-      AArch64::AEK_SIMD, "8-A"));
+      "exynos-m4", "armv8.2-a", "crypto-neon-fp-armv8",
+      AArch64::AEK_CRC | AArch64::AEK_CRYPTO |
+      AArch64::AEK_DOTPROD | AArch64::AEK_FP | AArch64::AEK_FP16 |
+      AArch64::AEK_LSE | AArch64::AEK_RAS | AArch64::AEK_RDM |
+      AArch64::AEK_SIMD, "8.2-A"));
   EXPECT_TRUE(testAArch64CPU(
       "falkor", "armv8-a", "crypto-neon-fp-armv8",
       AArch64::AEK_CRC | AArch64::AEK_CRYPTO | AArch64::AEK_FP |
@@ -869,8 +872,16 @@ TEST(TargetParserTest, testAArch64Extension) {
                                     AArch64::ArchKind::INVALID, "ras"));
   EXPECT_FALSE(testAArch64Extension("exynos-m3",
                                     AArch64::ArchKind::INVALID, "ras"));
+  EXPECT_TRUE(testAArch64Extension("exynos-m4",
+                                   AArch64::ArchKind::INVALID, "lse"));
+  EXPECT_TRUE(testAArch64Extension("exynos-m4",
+                                   AArch64::ArchKind::INVALID, "rdm"));
+  EXPECT_TRUE(testAArch64Extension("exynos-m4",
+                                   AArch64::ArchKind::INVALID, "ras"));
   EXPECT_FALSE(testAArch64Extension("exynos-m4",
-                                    AArch64::ArchKind::INVALID, "ras"));
+                                    AArch64::ArchKind::INVALID, "fullfp16"));
+  EXPECT_TRUE(testAArch64Extension("exynos-m4",
+                                   AArch64::ArchKind::INVALID, "dotprod"));
   EXPECT_TRUE(testAArch64Extension("falkor",
                                    AArch64::ArchKind::INVALID, "rdm"));
   EXPECT_FALSE(testAArch64Extension("kryo",
@@ -988,7 +999,10 @@ TEST(TargetParserTest, AArch64ArchExtFeature) {
                               {"rcpc", "norcpc", "+rcpc", "-rcpc" },
                               {"rng", "norng", "+rand", "-rand"},
                               {"memtag", "nomemtag", "+mte", "-mte"},
-                              {"ssbs", "nossbs", "+ssbs", "-ssbs"}};
+                              {"ssbs", "nossbs", "+ssbs", "-ssbs"},
+                              {"sb", "nosb", "+sb", "-sb"},
+                              {"predres", "nopredres", "+predres", "-predres"}
+};
 
   for (unsigned i = 0; i < array_lengthof(ArchExt); i++) {
     EXPECT_EQ(StringRef(ArchExt[i][2]),
diff --git a/unittests/Support/TaskQueueTest.cpp b/unittests/Support/TaskQueueTest.cpp
index 14ca9c057ac83c56e3dde1ae5e7ce2d258ce64d8..0a8aeca4e2d6f74f83fd695214966585eeaf2d76 100644
--- a/unittests/Support/TaskQueueTest.cpp
+++ b/unittests/Support/TaskQueueTest.cpp
@@ -1,9 +1,8 @@
 //========- unittests/Support/TaskQueue.cpp - TaskQueue.h tests ------========//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/Support/ThreadLocalTest.cpp b/unittests/Support/ThreadLocalTest.cpp
index e71c7dba22fc98135fcc503a4816934736cef815..075d7d95b25b0270ad75a0e1b382147528743c5e 100644
--- a/unittests/Support/ThreadLocalTest.cpp
+++ b/unittests/Support/ThreadLocalTest.cpp
@@ -1,9 +1,8 @@
 //===- llvm/unittest/Support/ThreadLocalTest.cpp - ThreadLocal tests ------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/Support/ThreadPool.cpp b/unittests/Support/ThreadPool.cpp
index 0da33ad50c0789c044f6593696e5a92d243712b4..4755b52a21db4cca1316e2415e6281e7d24fe5f8 100644
--- a/unittests/Support/ThreadPool.cpp
+++ b/unittests/Support/ThreadPool.cpp
@@ -1,9 +1,8 @@
 //========- unittests/Support/ThreadPools.cpp - ThreadPools.h tests --========//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/Support/Threading.cpp b/unittests/Support/Threading.cpp
index be53026415d7f2818f9a4515b7d3432fddcaefc7..01f1c5134482fb5675f4fb8f920319447fd0ff2d 100644
--- a/unittests/Support/Threading.cpp
+++ b/unittests/Support/Threading.cpp
@@ -1,9 +1,8 @@
 //===- unittests/Threading.cpp - Thread tests -----------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/Support/TimerTest.cpp b/unittests/Support/TimerTest.cpp
index a92ecf1ac5102582fba2c91491e9ac51eca4a98c..9a9b67eaa9f783a015829df66e930c6bcee02ea5 100644
--- a/unittests/Support/TimerTest.cpp
+++ b/unittests/Support/TimerTest.cpp
@@ -1,9 +1,8 @@
 //===- unittests/TimerTest.cpp - Timer tests ------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/Support/TrailingObjectsTest.cpp b/unittests/Support/TrailingObjectsTest.cpp
index 23acc54d23761149e0a7af1bd8e7d38fb30e70d7..967f6f6a65875c3600265c17a7d140a16867f990 100644
--- a/unittests/Support/TrailingObjectsTest.cpp
+++ b/unittests/Support/TrailingObjectsTest.cpp
@@ -1,9 +1,8 @@
 //=== - llvm/unittest/Support/TrailingObjectsTest.cpp ---------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/Support/TrigramIndexTest.cpp b/unittests/Support/TrigramIndexTest.cpp
index df42c1efd8a0ab7347ad921ba2242ec253acf021..9b10daa86e8c28ee90224d5d1732055fcc8035ed 100644
--- a/unittests/Support/TrigramIndexTest.cpp
+++ b/unittests/Support/TrigramIndexTest.cpp
@@ -1,9 +1,8 @@
 //===- TrigramIndexTest.cpp - Unit tests for TrigramIndex -----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/Support/TypeNameTest.cpp b/unittests/Support/TypeNameTest.cpp
index 63381d46f02fe3057f1d538a4de8cefe1d4333ba..c8962933e9cee66a69de5536d0a40934a0a3537b 100644
--- a/unittests/Support/TypeNameTest.cpp
+++ b/unittests/Support/TypeNameTest.cpp
@@ -1,9 +1,8 @@
 //===- TypeNameTest.cpp ---------------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/Support/TypeTraitsTest.cpp b/unittests/Support/TypeTraitsTest.cpp
index 7a3ce537e17142ae22c8e888b080022c69e39737..d33dee97ecabaab748e23376f91d9a1ed57f664c 100644
--- a/unittests/Support/TypeTraitsTest.cpp
+++ b/unittests/Support/TypeTraitsTest.cpp
@@ -1,9 +1,8 @@
 //===- TypeTraitsTest.cpp -------------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/Support/UnicodeTest.cpp b/unittests/Support/UnicodeTest.cpp
index 0733397d45c66605bb5957776fad12818a965cde..376fbee4ae666c0201c91262fae10a947e300d97 100644
--- a/unittests/Support/UnicodeTest.cpp
+++ b/unittests/Support/UnicodeTest.cpp
@@ -1,9 +1,8 @@
 //===- unittests/Support/UnicodeTest.cpp - Unicode.h tests ----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/Support/VersionTupleTest.cpp b/unittests/Support/VersionTupleTest.cpp
index cd7ecda5fefd39b2b755025e95c9eea78d23b917..af6c0a7febad50f5a6e214e158bd2c2b35c0f5c5 100644
--- a/unittests/Support/VersionTupleTest.cpp
+++ b/unittests/Support/VersionTupleTest.cpp
@@ -1,9 +1,8 @@
 //===- VersionTupleTests.cpp - Version Number Handling Tests --------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/Support/VirtualFileSystemTest.cpp b/unittests/Support/VirtualFileSystemTest.cpp
index 7b42943f0fd969817ac8817eb572203c00528fc6..fc39f5004c26d856c6bd1b21d3ff7987eb774643 100644
--- a/unittests/Support/VirtualFileSystemTest.cpp
+++ b/unittests/Support/VirtualFileSystemTest.cpp
@@ -1,9 +1,8 @@
 //===- unittests/Support/VirtualFileSystem.cpp -------------- VFS tests ---===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -777,7 +776,7 @@ TEST(ProxyFileSystemTest, Basic) {
 
   bool Local = true;
   ASSERT_FALSE(PFS.isLocal("/a", Local));
-  ASSERT_EQ(false, Local);
+  EXPECT_FALSE(Local);
 }
 
 class InMemoryFileSystemTest : public ::testing::Test {
diff --git a/unittests/Support/YAMLIOTest.cpp b/unittests/Support/YAMLIOTest.cpp
index 17f38f9b86cf7e43b5f10e2df001bbd4c8bc933a..cf279ffd4c2d45542ce77dcd6912342a9d010a1c 100644
--- a/unittests/Support/YAMLIOTest.cpp
+++ b/unittests/Support/YAMLIOTest.cpp
@@ -1,9 +1,8 @@
 //===- unittest/Support/YAMLIOTest.cpp ------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/Support/YAMLParserTest.cpp b/unittests/Support/YAMLParserTest.cpp
index 7962f3ca1ad1cf3b02609d992773b47e1a31a33b..06d4b0e269b9b115c8c8f500ec56d1894825953f 100644
--- a/unittests/Support/YAMLParserTest.cpp
+++ b/unittests/Support/YAMLParserTest.cpp
@@ -1,9 +1,8 @@
 //===- unittest/Support/YAMLParserTest ------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/Support/formatted_raw_ostream_test.cpp b/unittests/Support/formatted_raw_ostream_test.cpp
index 2b8f06523beb0653a2c3590f10e0db349154167d..0fe0869922a147ad4be8e195fa406b830bb9b597 100644
--- a/unittests/Support/formatted_raw_ostream_test.cpp
+++ b/unittests/Support/formatted_raw_ostream_test.cpp
@@ -1,9 +1,8 @@
 //===- llvm/unittest/Support/formatted_raw_ostream_test.cpp ---------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/Support/raw_ostream_test.cpp b/unittests/Support/raw_ostream_test.cpp
index a75f446e4ba15d91901af6a89c86ad0adf2fad3e..4ce4917fcead078872f377d9e8cf6410fb10cf5f 100644
--- a/unittests/Support/raw_ostream_test.cpp
+++ b/unittests/Support/raw_ostream_test.cpp
@@ -1,9 +1,8 @@
 //===- llvm/unittest/Support/raw_ostream_test.cpp - raw_ostream tests -----===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/Support/raw_pwrite_stream_test.cpp b/unittests/Support/raw_pwrite_stream_test.cpp
index a528fd25b932e603aed76b027e142a7618334372..459eb940a43d573a4bf8362a49100db3f44f1b11 100644
--- a/unittests/Support/raw_pwrite_stream_test.cpp
+++ b/unittests/Support/raw_pwrite_stream_test.cpp
@@ -1,9 +1,8 @@
 //===- raw_pwrite_stream_test.cpp - raw_pwrite_stream tests ---------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/Support/raw_sha1_ostream_test.cpp b/unittests/Support/raw_sha1_ostream_test.cpp
index e176f3f7a19b5875c3ec4154dee89147eb3a66fb..be0353614836b4280ae7b5b93057bb044fba176c 100644
--- a/unittests/Support/raw_sha1_ostream_test.cpp
+++ b/unittests/Support/raw_sha1_ostream_test.cpp
@@ -1,9 +1,8 @@
 //===- llvm/unittest/Support/raw_ostream_test.cpp - raw_ostream tests -----===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/Support/xxhashTest.cpp b/unittests/Support/xxhashTest.cpp
index 6a88a2d3a5c5c0f85a2c3c3edf11a488fa8648aa..7dc7c40c69ecec064307e941162df8dfe469386f 100644
--- a/unittests/Support/xxhashTest.cpp
+++ b/unittests/Support/xxhashTest.cpp
@@ -1,9 +1,8 @@
 //===- llvm/unittest/Support/xxhashTest.cpp -------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/Target/WebAssembly/WebAssemblyExceptionInfoTest.cpp b/unittests/Target/WebAssembly/WebAssemblyExceptionInfoTest.cpp
index 095ee0665e1a1870fc6aea443faff5723a2815be..ec946379b1e9e52447875b1ba2b25aab3139fd77 100644
--- a/unittests/Target/WebAssembly/WebAssemblyExceptionInfoTest.cpp
+++ b/unittests/Target/WebAssembly/WebAssemblyExceptionInfoTest.cpp
@@ -1,9 +1,8 @@
 //=== WebAssemblyExceptionInfoTest.cpp - WebAssebmlyExceptionInfo unit tests =//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -75,7 +74,7 @@ TEST(WebAssemblyExceptionInfoTest, TEST0) {
 
   declare i32 @__gxx_wasm_personality_v0(...)
 
-  define hidden void @test0() personality i8* bitcast (i32 (...)* @__gxx_wasm_personality_v0 to i8*) {
+  define void @test0() personality i8* bitcast (i32 (...)* @__gxx_wasm_personality_v0 to i8*) {
     unreachable
   }
 
@@ -101,14 +100,14 @@ body: |
   ; predecessors: %bb.0
     successors: %bb.3, %bb.9
     liveins: $value_stack
-    CATCH_ALL implicit-def $arguments
+    %0:except_ref = CATCH implicit-def $arguments
     CLEANUPRET implicit-def dead $arguments
 
   bb.3 (landing-pad):
   ; predecessors: %bb.2
     successors: %bb.4, %bb.6
     liveins: $value_stack
-    CATCH_ALL implicit-def $arguments
+    %1:except_ref = CATCH implicit-def $arguments
     BR_IF %bb.4, %58:i32, implicit-def $arguments, implicit-def $value_stack, implicit $value_stack
     BR %bb.6, implicit-def $arguments
 
@@ -139,13 +138,13 @@ body: |
   ; predecessors: %bb.4
     successors: %bb.9
     liveins: $value_stack
-    CATCH_ALL implicit-def $arguments
+    %2:except_ref = CATCH implicit-def $arguments
     CLEANUPRET implicit-def dead $arguments
 
   bb.9 (landing-pad):
   ; predecessors: %bb.2, %bb.6, %bb.8
     liveins: $value_stack
-    CATCH_ALL implicit-def $arguments
+    %3:except_ref = CATCH implicit-def $arguments
     CLEANUPRET implicit-def dead $arguments
 
   bb.10:
@@ -238,7 +237,7 @@ TEST(WebAssemblyExceptionInfoTest, TEST1) {
 
   declare i32 @__gxx_wasm_personality_v0(...)
 
-  define hidden void @test1() personality i8* bitcast (i32 (...)* @__gxx_wasm_personality_v0 to i8*) {
+  define void @test1() personality i8* bitcast (i32 (...)* @__gxx_wasm_personality_v0 to i8*) {
     unreachable
   }
 
@@ -258,7 +257,7 @@ body: |
   ; predecessors: %bb.0
     successors: %bb.2, %bb.8
     liveins: $value_stack
-    %52:i32 = CATCH_I32 0, implicit-def dead $arguments
+    %0:except_ref = CATCH implicit-def $arguments
     BR_IF %bb.2, %32:i32, implicit-def $arguments, implicit-def $value_stack, implicit $value_stack
     BR %bb.8, implicit-def $arguments
 
@@ -272,7 +271,7 @@ body: |
   ; predecessors: %bb.2
     successors: %bb.4, %bb.6
     liveins: $value_stack
-    CATCH_ALL implicit-def $arguments
+    %1:except_ref = CATCH implicit-def $arguments
     BR_IF %bb.4, %43:i32, implicit-def $arguments, implicit-def $value_stack, implicit $value_stack
     BR %bb.6, implicit-def $arguments
 
@@ -314,13 +313,13 @@ body: |
   ; predecessors: %bb.4
     successors: %bb.11
     liveins: $value_stack
-    CATCH_ALL implicit-def $arguments
+    %2:except_ref = CATCH implicit-def $arguments
     CLEANUPRET implicit-def dead $arguments
 
   bb.11 (landing-pad):
   ; predecessors: %bb.2, %bb.6, %bb.10
     liveins: $value_stack
-    CATCH_ALL implicit-def $arguments
+    %3:except_ref = CATCH implicit-def $arguments
     CLEANUPRET implicit-def dead $arguments
 
   bb.12:
@@ -416,135 +415,3 @@ body: |
   EXPECT_EQ(WE0_1->getParentException(), WE0);
   EXPECT_EQ(WE0_1->getExceptionDepth(), (unsigned)2);
 }
-
-// Terminate pad test
-TEST(WebAssemblyExceptionInfoTest, TEST2) {
-  std::unique_ptr<LLVMTargetMachine> TM = createTargetMachine();
-  ASSERT_TRUE(TM);
-
-  StringRef MIRString = R"MIR(
---- |
-  target datalayout = "e-m:e-p:32:32-i64:64-n32:64-S128"
-  target triple = "wasm32-unknown-unknown"
-
-  declare i32 @__gxx_wasm_personality_v0(...)
-  declare void @_ZSt9terminatev()
-  declare void @__clang_call_terminate(i8*)
-
-  define hidden void @test2() personality i8* bitcast (i32 (...)* @__gxx_wasm_personality_v0 to i8*) {
-    unreachable
-  }
-
-...
----
-name: test2
-liveins:
-  - { reg: '$arguments' }
-  - { reg: '$value_stack' }
-body: |
-  bb.0:
-    successors: %bb.3, %bb.1
-    BR %bb.3, implicit-def dead $arguments
-
-  bb.1 (landing-pad):
-  ; predecessors: %bb.0
-    successors: %bb.2, %bb.4
-    %3:i32 = CATCH_I32 0, implicit-def dead $arguments
-    BR %bb.2, implicit-def dead $arguments
-
-  bb.2:
-  ; predecessors: %bb.1
-    successors: %bb.3(0x80000000); %bb.3(200.00%)
-    CATCHRET %bb.3, %bb.0, implicit-def dead $arguments
-
-  bb.3:
-  ; predecessors: %bb.0, %bb.2
-    RETURN_VOID implicit-def $arguments
-
-  bb.4 (landing-pad):
-  ; predecessors: %bb.1
-    successors: %bb.5, %bb.6
-    CATCH_ALL implicit-def $arguments
-    BR %bb.5, implicit-def dead $arguments
-
-  bb.5:
-  ; predecessors: %bb.4
-    CLEANUPRET implicit-def dead $arguments
-
-  bb.6 (landing-pad):
-  ; predecessors: %bb.4
-    successors: %bb.7(0x80000000); %bb.7(200.00%)
-    %6:i32 = CATCH_I32 0, implicit-def dead $arguments
-    CALL_VOID @__clang_call_terminate, %7:i32, implicit-def $arguments
-    UNREACHABLE implicit-def $arguments
-
-  bb.7 (landing-pad):
-  ; predecessors: %bb.6
-    CATCH_ALL implicit-def $arguments
-    CALL_VOID @_ZSt9terminatev, implicit-def $arguments
-    UNREACHABLE implicit-def $arguments
-)MIR";
-
-  LLVMContext Context;
-  std::unique_ptr<MIRParser> MIR;
-  MachineModuleInfo MMI(TM.get());
-  std::unique_ptr<Module> M =
-      parseMIR(Context, MIR, *TM, MIRString, "test2", MMI);
-  ASSERT_TRUE(M);
-
-  Function *F = M->getFunction("test2");
-  auto *MF = MMI.getMachineFunction(*F);
-  ASSERT_TRUE(MF);
-
-  WebAssemblyExceptionInfo WEI;
-  MachineDominatorTree MDT;
-  MachineDominanceFrontier MDF;
-  MDT.runOnMachineFunction(*MF);
-  MDF.getBase().analyze(MDT.getBase());
-  WEI.recalculate(MDT, MDF);
-
-  // Exception info structure:
-  // |- bb1 (ehpad), bb2, bb4, bb5, bb6, bb7
-  //   |- bb4 (ehpad), bb5, bb6, bb7
-  //     |- bb6 (ehpad), bb7
-  //
-  // Here, bb6 is a terminate pad with a 'catch' instruction, and bb7 is a
-  // terminate pad with a 'catch_all' instruction, In this case we put bb6 and
-  // bb7 into one exception.
-
-  auto *MBB1 = MF->getBlockNumbered(1);
-  auto *WE0 = WEI.getExceptionFor(MBB1);
-  ASSERT_TRUE(WE0);
-  EXPECT_EQ(WE0->getEHPad(), MBB1);
-  EXPECT_EQ(WE0->getParentException(), nullptr);
-  EXPECT_EQ(WE0->getExceptionDepth(), (unsigned)1);
-
-  auto *MBB2 = MF->getBlockNumbered(2);
-  WE0 = WEI.getExceptionFor(MBB2);
-  ASSERT_TRUE(WE0);
-  EXPECT_EQ(WE0->getEHPad(), MBB1);
-
-  auto *MBB4 = MF->getBlockNumbered(4);
-  auto *WE0_0 = WEI.getExceptionFor(MBB4);
-  ASSERT_TRUE(WE0_0);
-  EXPECT_EQ(WE0_0->getEHPad(), MBB4);
-  EXPECT_EQ(WE0_0->getParentException(), WE0);
-  EXPECT_EQ(WE0_0->getExceptionDepth(), (unsigned)2);
-
-  auto *MBB5 = MF->getBlockNumbered(5);
-  WE0_0 = WEI.getExceptionFor(MBB5);
-  ASSERT_TRUE(WE0_0);
-  EXPECT_EQ(WE0_0->getEHPad(), MBB4);
-
-  auto *MBB6 = MF->getBlockNumbered(6);
-  auto *WE0_0_0 = WEI.getExceptionFor(MBB6);
-  ASSERT_TRUE(WE0_0_0);
-  EXPECT_EQ(WE0_0_0->getEHPad(), MBB6);
-  EXPECT_EQ(WE0_0_0->getParentException(), WE0_0);
-  EXPECT_EQ(WE0_0_0->getExceptionDepth(), (unsigned)3);
-
-  auto *MBB7 = MF->getBlockNumbered(7);
-  WE0_0_0 = WEI.getExceptionFor(MBB7);
-  ASSERT_TRUE(WE0_0_0);
-  EXPECT_EQ(WE0_0_0->getEHPad(), MBB6);
-}
diff --git a/unittests/TextAPI/CMakeLists.txt b/unittests/TextAPI/CMakeLists.txt
index 5f64c8a59005747221b5947216945e6e17096100..d7208b2b060fa06fdff4387b9bb05e4d0e146e23 100644
--- a/unittests/TextAPI/CMakeLists.txt
+++ b/unittests/TextAPI/CMakeLists.txt
@@ -2,8 +2,8 @@ set(LLVM_LINK_COMPONENTS
   TextAPI
 )
 
-add_llvm_unittest(TapiTests
+add_llvm_unittest(TextAPITests
   ELFYAMLTest.cpp
 )
 
-target_link_libraries(TapiTests PRIVATE LLVMTestingSupport)
+target_link_libraries(TextAPITests PRIVATE LLVMTestingSupport)
diff --git a/unittests/TextAPI/ELFYAMLTest.cpp b/unittests/TextAPI/ELFYAMLTest.cpp
index 802a2d7127b25bdb6c24b8ee10313f4eda02fb0d..9c50a6bee5004da3aedd0a58fd643b4b0b957b00 100644
--- a/unittests/TextAPI/ELFYAMLTest.cpp
+++ b/unittests/TextAPI/ELFYAMLTest.cpp
@@ -1,9 +1,8 @@
 //===- llvm/unittests/TextAPI/YAMLTest.cpp --------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===-----------------------------------------------------------------------===/
 
@@ -65,7 +64,7 @@ TEST(ElfYamlTextAPI, YAMLReadsTBESymbols) {
                       "  foo: { Type: Func, Warning: \"Deprecated!\" }\n"
                       "  nor: { Type: NoType, Undefined: true }\n"
                       "  not: { Type: File, Undefined: true, Size: 111, "
-                      "Warning: \'All fields populated!\' }\n"
+                      "Weak: true, Warning: \'All fields populated!\' }\n"
                       "...\n";
   Expected<std::unique_ptr<ELFStub>> StubOrErr = readTBEFromBuffer(Data);
   ASSERT_THAT_ERROR(StubOrErr.takeError(), Succeeded());
@@ -81,6 +80,7 @@ TEST(ElfYamlTextAPI, YAMLReadsTBESymbols) {
   EXPECT_EQ(SymBar.Size, 42u);
   EXPECT_EQ(SymBar.Type, ELFSymbolType::Object);
   EXPECT_FALSE(SymBar.Undefined);
+  EXPECT_FALSE(SymBar.Weak);
   EXPECT_FALSE(SymBar.Warning.hasValue());
 
   ELFSymbol const &SymBaz = *Iterator++;
@@ -88,6 +88,7 @@ TEST(ElfYamlTextAPI, YAMLReadsTBESymbols) {
   EXPECT_EQ(SymBaz.Size, 3u);
   EXPECT_EQ(SymBaz.Type, ELFSymbolType::TLS);
   EXPECT_FALSE(SymBaz.Undefined);
+  EXPECT_FALSE(SymBaz.Weak);
   EXPECT_FALSE(SymBaz.Warning.hasValue());
 
   ELFSymbol const &SymFoo = *Iterator++;
@@ -95,6 +96,7 @@ TEST(ElfYamlTextAPI, YAMLReadsTBESymbols) {
   EXPECT_EQ(SymFoo.Size, 0u);
   EXPECT_EQ(SymFoo.Type, ELFSymbolType::Func);
   EXPECT_FALSE(SymFoo.Undefined);
+  EXPECT_FALSE(SymFoo.Weak);
   EXPECT_TRUE(SymFoo.Warning.hasValue());
   EXPECT_STREQ(SymFoo.Warning->c_str(), "Deprecated!");
 
@@ -103,6 +105,7 @@ TEST(ElfYamlTextAPI, YAMLReadsTBESymbols) {
   EXPECT_EQ(SymNor.Size, 0u);
   EXPECT_EQ(SymNor.Type, ELFSymbolType::NoType);
   EXPECT_TRUE(SymNor.Undefined);
+  EXPECT_FALSE(SymNor.Weak);
   EXPECT_FALSE(SymNor.Warning.hasValue());
 
   ELFSymbol const &SymNot = *Iterator++;
@@ -110,6 +113,7 @@ TEST(ElfYamlTextAPI, YAMLReadsTBESymbols) {
   EXPECT_EQ(SymNot.Size, 111u);
   EXPECT_EQ(SymNot.Type, ELFSymbolType::Unknown);
   EXPECT_TRUE(SymNot.Undefined);
+  EXPECT_TRUE(SymNot.Weak);
   EXPECT_TRUE(SymNot.Warning.hasValue());
   EXPECT_STREQ(SymNot.Warning->c_str(), "All fields populated!");
 }
@@ -146,6 +150,7 @@ TEST(ElfYamlTextAPI, YAMLWritesTBESymbols) {
       "TbeVersion:      1.0\n"
       "Arch:            AArch64\n"
       "Symbols:         \n"
+      "  bar:             { Type: Func, Weak: true }\n"
       "  foo:             { Type: NoType, Size: 99, Warning: Does nothing }\n"
       "  nor:             { Type: Func, Undefined: true }\n"
       "  not:             { Type: Unknown, Size: 12345678901234 }\n"
@@ -158,19 +163,30 @@ TEST(ElfYamlTextAPI, YAMLWritesTBESymbols) {
   SymFoo.Size = 99u;
   SymFoo.Type = ELFSymbolType::NoType;
   SymFoo.Undefined = false;
+  SymFoo.Weak = false;
   SymFoo.Warning = "Does nothing";
 
+  ELFSymbol SymBar("bar");
+  SymBar.Size = 128u;
+  SymBar.Type = ELFSymbolType::Func;
+  SymBar.Undefined = false;
+  SymBar.Weak = true;
+
   ELFSymbol SymNor("nor");
+  SymNor.Size = 1234u;
   SymNor.Type = ELFSymbolType::Func;
   SymNor.Undefined = true;
+  SymNor.Weak = false;
 
   ELFSymbol SymNot("not");
   SymNot.Size = 12345678901234u;
   SymNot.Type = ELFSymbolType::Unknown;
   SymNot.Undefined = false;
+  SymNot.Weak = false;
 
   // Deliberately not in order to check that result is sorted.
   Stub.Symbols.insert(SymNot);
+  Stub.Symbols.insert(SymBar);
   Stub.Symbols.insert(SymFoo);
   Stub.Symbols.insert(SymNor);
 
diff --git a/unittests/Transforms/IPO/LowerTypeTests.cpp b/unittests/Transforms/IPO/LowerTypeTests.cpp
index 66c9de6bd662015afc054d94f68f56a4344c712d..ba13378099ecb01b34e98672504a120ba24665e3 100644
--- a/unittests/Transforms/IPO/LowerTypeTests.cpp
+++ b/unittests/Transforms/IPO/LowerTypeTests.cpp
@@ -1,9 +1,8 @@
 //===- LowerTypeTests.cpp - Unit tests for type test lowering -------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/Transforms/IPO/WholeProgramDevirt.cpp b/unittests/Transforms/IPO/WholeProgramDevirt.cpp
index 7e7a6bf459641037a270f2b18cfb95fd41ccebe1..8fae05bef2ae1373e84d63172e430a333e27c762 100644
--- a/unittests/Transforms/IPO/WholeProgramDevirt.cpp
+++ b/unittests/Transforms/IPO/WholeProgramDevirt.cpp
@@ -1,9 +1,8 @@
 //===- WholeProgramDevirt.cpp - Unit tests for whole-program devirt -------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/Transforms/Scalar/LoopPassManagerTest.cpp b/unittests/Transforms/Scalar/LoopPassManagerTest.cpp
index 9f0b6f0ea03ec3af86c095f7e030c61a6dd7799e..2615e19d44f5deef632ef957bed932cab75e8590 100644
--- a/unittests/Transforms/Scalar/LoopPassManagerTest.cpp
+++ b/unittests/Transforms/Scalar/LoopPassManagerTest.cpp
@@ -1,9 +1,8 @@
 //===- llvm/unittest/Analysis/LoopPassManagerTest.cpp - LPM tests ---------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -304,6 +303,8 @@ public:
     // those.
     FAM.registerPass([&] { return AAManager(); });
     FAM.registerPass([&] { return AssumptionAnalysis(); });
+    if (EnableMSSALoopDependency)
+      FAM.registerPass([&] { return MemorySSAAnalysis(); });
     FAM.registerPass([&] { return ScalarEvolutionAnalysis(); });
     FAM.registerPass([&] { return TargetLibraryAnalysis(); });
     FAM.registerPass([&] { return TargetIRAnalysis(); });
@@ -909,7 +910,8 @@ TEST_F(LoopPassManagerTest, LoopChildInsertion) {
   ASSERT_THAT(BBI, F.end());
   auto CreateCondBr = [&](BasicBlock *TrueBB, BasicBlock *FalseBB,
                           const char *Name, BasicBlock *BB) {
-    auto *Cond = new LoadInst(&Ptr, Name, /*isVolatile*/ true, BB);
+    auto *Cond = new LoadInst(Type::getInt1Ty(Context), &Ptr, Name,
+                              /*isVolatile*/ true, BB);
     BranchInst::Create(TrueBB, FalseBB, Cond, BB);
   };
 
@@ -1111,7 +1113,8 @@ TEST_F(LoopPassManagerTest, LoopPeerInsertion) {
   ASSERT_THAT(BBI, F.end());
   auto CreateCondBr = [&](BasicBlock *TrueBB, BasicBlock *FalseBB,
                           const char *Name, BasicBlock *BB) {
-    auto *Cond = new LoadInst(&Ptr, Name, /*isVolatile*/ true, BB);
+    auto *Cond = new LoadInst(Type::getInt1Ty(Context), &Ptr, Name,
+                              /*isVolatile*/ true, BB);
     BranchInst::Create(TrueBB, FalseBB, Cond, BB);
   };
 
@@ -1504,8 +1507,9 @@ TEST_F(LoopPassManagerTest, LoopDeletion) {
             auto *NewLoop03BB =
                 BasicBlock::Create(Context, "loop.0.3", &F, &Loop0LatchBB);
             BranchInst::Create(NewLoop03BB, NewLoop03PHBB);
-            auto *Cond = new LoadInst(&Ptr, "cond.0.3", /*isVolatile*/ true,
-                                      NewLoop03BB);
+            auto *Cond =
+                new LoadInst(Type::getInt1Ty(Context), &Ptr, "cond.0.3",
+                             /*isVolatile*/ true, NewLoop03BB);
             BranchInst::Create(&Loop0LatchBB, NewLoop03BB, Cond, NewLoop03BB);
             Loop02PHBB.getTerminator()->replaceUsesOfWith(&Loop0LatchBB,
                                                           NewLoop03PHBB);
diff --git a/unittests/Transforms/Utils/ASanStackFrameLayoutTest.cpp b/unittests/Transforms/Utils/ASanStackFrameLayoutTest.cpp
index 777918e5cfdd720e0a501600814069a35c8e369e..8af49c6fb9f485e75e10ddd9d53186a5118faf3d 100644
--- a/unittests/Transforms/Utils/ASanStackFrameLayoutTest.cpp
+++ b/unittests/Transforms/Utils/ASanStackFrameLayoutTest.cpp
@@ -1,9 +1,8 @@
 //===- ASanStackFrameLayoutTest.cpp - Tests for ComputeASanStackFrameLayout===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 #include "llvm/Transforms/Utils/ASanStackFrameLayout.h"
diff --git a/unittests/Transforms/Utils/BasicBlockUtilsTest.cpp b/unittests/Transforms/Utils/BasicBlockUtilsTest.cpp
index 2d0a9302011f71d3b60083a9de74b2eff4460769..db30837507aab7d40c9ab1aded959369fecb2b98 100644
--- a/unittests/Transforms/Utils/BasicBlockUtilsTest.cpp
+++ b/unittests/Transforms/Utils/BasicBlockUtilsTest.cpp
@@ -1,9 +1,8 @@
 //===- BasicBlockUtils.cpp - Unit tests for BasicBlockUtils ---------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/Transforms/Utils/CloningTest.cpp b/unittests/Transforms/Utils/CloningTest.cpp
index 9d6bf0a5a966b5ad4afa4d63cd8ab610f40c6b12..abc18bc377c409c0b5ed5f7fae5b54cdb0f4a262 100644
--- a/unittests/Transforms/Utils/CloningTest.cpp
+++ b/unittests/Transforms/Utils/CloningTest.cpp
@@ -1,20 +1,19 @@
 //===- Cloning.cpp - Unit tests for the Cloner ----------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #include "llvm/Transforms/Utils/Cloning.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SmallPtrSet.h"
+#include "llvm/Analysis/DomTreeUpdater.h"
 #include "llvm/IR/Argument.h"
 #include "llvm/IR/Constant.h"
 #include "llvm/IR/DIBuilder.h"
 #include "llvm/IR/DebugInfo.h"
-#include "llvm/IR/DomTreeUpdater.h"
 #include "llvm/IR/Function.h"
 #include "llvm/IR/IRBuilder.h"
 #include "llvm/IR/InstIterator.h"
diff --git a/unittests/Transforms/Utils/CodeExtractorTest.cpp b/unittests/Transforms/Utils/CodeExtractorTest.cpp
index 7e3512f56f32ac0a66abcff754b9d995a7ad66b4..8b86951fa5e199ea8cf542074a22b1e07432e74f 100644
--- a/unittests/Transforms/Utils/CodeExtractorTest.cpp
+++ b/unittests/Transforms/Utils/CodeExtractorTest.cpp
@@ -1,9 +1,8 @@
 //===- CodeExtractor.cpp - Unit tests for CodeExtractor -------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -59,8 +58,7 @@ TEST(CodeExtractor, ExitStub) {
                                            getBlockByName(Func, "body1"),
                                            getBlockByName(Func, "body2") };
 
-  DominatorTree DT(*Func);
-  CodeExtractor CE(Candidates, &DT);
+  CodeExtractor CE(Candidates);
   EXPECT_TRUE(CE.isEligible());
 
   Function *Outlined = CE.extractCodeRegion();
@@ -110,8 +108,7 @@ TEST(CodeExtractor, ExitPHIOnePredFromRegion) {
     getBlockByName(Func, "extracted2")
   };
 
-  DominatorTree DT(*Func);
-  CodeExtractor CE(ExtractedBlocks, &DT);
+  CodeExtractor CE(ExtractedBlocks);
   EXPECT_TRUE(CE.isEligible());
 
   Function *Outlined = CE.extractCodeRegion();
@@ -185,8 +182,7 @@ TEST(CodeExtractor, StoreOutputInvokeResultAfterEHPad) {
     getBlockByName(Func, "lpad2")
   };
 
-  DominatorTree DT(*Func);
-  CodeExtractor CE(ExtractedBlocks, &DT);
+  CodeExtractor CE(ExtractedBlocks);
   EXPECT_TRUE(CE.isEligible());
 
   Function *Outlined = CE.extractCodeRegion();
@@ -195,4 +191,38 @@ TEST(CodeExtractor, StoreOutputInvokeResultAfterEHPad) {
   EXPECT_FALSE(verifyFunction(*Func, &errs()));
 }
 
+TEST(CodeExtractor, StoreOutputInvokeResultInExitStub) {
+  LLVMContext Ctx;
+  SMDiagnostic Err;
+  std::unique_ptr<Module> M(parseAssemblyString(R"invalid(
+    declare i32 @bar()
+
+    define i32 @foo() personality i8* null {
+    entry:
+      %0 = invoke i32 @bar() to label %exit unwind label %lpad
+
+    exit:
+      ret i32 %0
+
+    lpad:
+      %1 = landingpad { i8*, i32 }
+              cleanup
+      resume { i8*, i32 } %1
+    }
+  )invalid",
+                                                Err, Ctx));
+
+  Function *Func = M->getFunction("foo");
+  SmallVector<BasicBlock *, 1> Blocks{ getBlockByName(Func, "entry"),
+                                       getBlockByName(Func, "lpad") };
+
+  CodeExtractor CE(Blocks);
+  EXPECT_TRUE(CE.isEligible());
+
+  Function *Outlined = CE.extractCodeRegion();
+  EXPECT_TRUE(Outlined);
+  EXPECT_FALSE(verifyFunction(*Outlined));
+  EXPECT_FALSE(verifyFunction(*Func));
+}
+
 } // end anonymous namespace
diff --git a/unittests/Transforms/Utils/FunctionComparatorTest.cpp b/unittests/Transforms/Utils/FunctionComparatorTest.cpp
index 26e20cd9112c30e8a5b6679498afec3191ca2959..42f9fb39c2d15d754ce8904c5bd64ce4d17dc3da 100644
--- a/unittests/Transforms/Utils/FunctionComparatorTest.cpp
+++ b/unittests/Transforms/Utils/FunctionComparatorTest.cpp
@@ -1,9 +1,8 @@
 //===- FunctionComparator.cpp - Unit tests for FunctionComparator ---------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 #include "llvm/Transforms/Utils/FunctionComparator.h"
@@ -32,7 +31,7 @@ struct TestFunction {
     BB = BasicBlock::Create(Ctx, "", F);
     B.SetInsertPoint(BB);
     Argument *PointerArg = &*F->arg_begin();
-    LoadInst *LoadInst = B.CreateLoad(PointerArg);
+    LoadInst *LoadInst = B.CreateLoad(T, PointerArg);
     C = B.getInt8(addVal);
     I = cast<Instruction>(B.CreateAdd(LoadInst, C));
     B.CreateRet(I);
diff --git a/unittests/Transforms/Utils/IntegerDivisionTest.cpp b/unittests/Transforms/Utils/IntegerDivisionTest.cpp
index e337b9f547a89412cf6ae82f0f5edaf3cf9fd8ca..79946735f8546855534231d94b1dec571ccd3702 100644
--- a/unittests/Transforms/Utils/IntegerDivisionTest.cpp
+++ b/unittests/Transforms/Utils/IntegerDivisionTest.cpp
@@ -1,9 +1,8 @@
 //===- IntegerDivision.cpp - Unit tests for the integer division code -----===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/Transforms/Utils/LocalTest.cpp b/unittests/Transforms/Utils/LocalTest.cpp
index 3868bfb09a54f47e1e7a5203006fb957febddc0d..80f263d24a7b2451fb5ac64b76c7e345ec822200 100644
--- a/unittests/Transforms/Utils/LocalTest.cpp
+++ b/unittests/Transforms/Utils/LocalTest.cpp
@@ -1,18 +1,17 @@
 //===- Local.cpp - Unit tests for Local -----------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #include "llvm/Transforms/Utils/Local.h"
+#include "llvm/Analysis/DomTreeUpdater.h"
 #include "llvm/Analysis/PostDominators.h"
 #include "llvm/AsmParser/Parser.h"
 #include "llvm/IR/BasicBlock.h"
 #include "llvm/IR/DIBuilder.h"
-#include "llvm/IR/DomTreeUpdater.h"
 #include "llvm/IR/IRBuilder.h"
 #include "llvm/IR/Instructions.h"
 #include "llvm/IR/IntrinsicInst.h"
diff --git a/unittests/Transforms/Utils/SSAUpdaterBulkTest.cpp b/unittests/Transforms/Utils/SSAUpdaterBulkTest.cpp
index 61cbcb7b1a776e3f642c6701e7c4017e430d7937..b75a492c58bc408141d45467d2ca11f3b9cd6a06 100644
--- a/unittests/Transforms/Utils/SSAUpdaterBulkTest.cpp
+++ b/unittests/Transforms/Utils/SSAUpdaterBulkTest.cpp
@@ -1,9 +1,8 @@
 //===- SSAUpdaterBulk.cpp - Unit tests for SSAUpdaterBulk -----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/Transforms/Utils/UnrollLoopTest.cpp b/unittests/Transforms/Utils/UnrollLoopTest.cpp
index 5f7c2d62380d1fdd80c17e24bcfa285d8882cae0..319498479c35f8a7b1a5d64e3e9959979fd7f144 100644
--- a/unittests/Transforms/Utils/UnrollLoopTest.cpp
+++ b/unittests/Transforms/Utils/UnrollLoopTest.cpp
@@ -1,9 +1,8 @@
 //===- UnrollLoopTest.cpp - Unit tests for UnrollLoop ---------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/Transforms/Utils/ValueMapperTest.cpp b/unittests/Transforms/Utils/ValueMapperTest.cpp
index 94ac76bb076cdc9cc5daa5de791af0e8255d6d4c..690f4340068422a60ee337ea5f83cd7124f8d6d3 100644
--- a/unittests/Transforms/Utils/ValueMapperTest.cpp
+++ b/unittests/Transforms/Utils/ValueMapperTest.cpp
@@ -1,9 +1,8 @@
 //===- ValueMapper.cpp - Unit tests for ValueMapper -----------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/Transforms/Vectorize/CMakeLists.txt b/unittests/Transforms/Vectorize/CMakeLists.txt
index 6e886bbe25af71b539e0d5f9596cd4a868cdf659..74fc5ed43f36e53623f6a525deef9006a06c4501 100644
--- a/unittests/Transforms/Vectorize/CMakeLists.txt
+++ b/unittests/Transforms/Vectorize/CMakeLists.txt
@@ -8,6 +8,7 @@ set(LLVM_LINK_COMPONENTS
 add_llvm_unittest(VectorizeTests
   VPlanDominatorTreeTest.cpp
   VPlanLoopInfoTest.cpp
+  VPlanPredicatorTest.cpp
   VPlanTest.cpp
   VPlanHCFGTest.cpp
   VPlanSlpTest.cpp
diff --git a/unittests/Transforms/Vectorize/VPlanDominatorTreeTest.cpp b/unittests/Transforms/Vectorize/VPlanDominatorTreeTest.cpp
index 57f07392d8b832ada54773e16c06db200e08f828..6f397907e31d7a2a66a248eecae8251d792b88b2 100644
--- a/unittests/Transforms/Vectorize/VPlanDominatorTreeTest.cpp
+++ b/unittests/Transforms/Vectorize/VPlanDominatorTreeTest.cpp
@@ -1,9 +1,8 @@
 //===- llvm/unittests/Transforms/Vectorize/VPlanDominatorTreeTest.cpp -----===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/Transforms/Vectorize/VPlanHCFGTest.cpp b/unittests/Transforms/Vectorize/VPlanHCFGTest.cpp
index 215c3ef40daadfee736e98c50d589f8626805ff6..f5894a4f10f42738ec0cbb74871ee0e9333eb94f 100644
--- a/unittests/Transforms/Vectorize/VPlanHCFGTest.cpp
+++ b/unittests/Transforms/Vectorize/VPlanHCFGTest.cpp
@@ -1,9 +1,8 @@
 //===- llvm/unittest/Transforms/Vectorize/VPlanHCFGTest.cpp ---------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/Transforms/Vectorize/VPlanLoopInfoTest.cpp b/unittests/Transforms/Vectorize/VPlanLoopInfoTest.cpp
index 55486ba0f14e89d8b77bddd88be382749bc9adcb..0bb459968986fabcb53ccdc472d8659f4ea321a5 100644
--- a/unittests/Transforms/Vectorize/VPlanLoopInfoTest.cpp
+++ b/unittests/Transforms/Vectorize/VPlanLoopInfoTest.cpp
@@ -1,9 +1,8 @@
 //===- llvm/unittests/Transforms/Vectorize/VPlanLoopInfoTest.cpp -----===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/Transforms/Vectorize/VPlanPredicatorTest.cpp b/unittests/Transforms/Vectorize/VPlanPredicatorTest.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..81ed3cee3d2a81e90bb6227154d1b6d0c3f1e928
--- /dev/null
+++ b/unittests/Transforms/Vectorize/VPlanPredicatorTest.cpp
@@ -0,0 +1,229 @@
+//===- llvm/unittests/Transforms/Vectorize/VPlanPredicatorTest.cpp -----===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "../lib/Transforms/Vectorize/VPlanPredicator.h"
+#include "VPlanTestBase.h"
+#include "gtest/gtest.h"
+
+namespace llvm {
+namespace {
+
+class VPlanPredicatorTest : public VPlanTestBase {};
+
+TEST_F(VPlanPredicatorTest, BasicPredicatorTest) {
+  const char *ModuleString =
+      "@arr = common global [8 x [8 x i64]] "
+      "zeroinitializer, align 16\n"
+      "@arr2 = common global [8 x [8 x i64]] "
+      "zeroinitializer, align 16\n"
+      "@arr3 = common global [8 x [8 x i64]] "
+      "zeroinitializer, align 16\n"
+      "define void @f(i64 %n1) {\n"
+      "entry:\n"
+      "  br label %for.cond1.preheader\n"
+      "for.cond1.preheader:                              \n"
+      "  %i1.029 = phi i64 [ 0, %entry ], [ %inc14, %for.inc13 ]\n"
+      "  br label %for.body3\n"
+      "for.body3:                                        \n"
+      "  %i2.028 = phi i64 [ 0, %for.cond1.preheader ], [ %inc, %for.inc ]\n"
+      "  %arrayidx4 = getelementptr inbounds [8 x [8 x i64]], [8 x [8 x i64]]* "
+      "@arr, i64 0, i64 %i2.028, i64 %i1.029\n"
+      "  %0 = load i64, i64* %arrayidx4, align 8\n"
+      "  %cmp5 = icmp ugt i64 %0, 10\n"
+      "  br i1 %cmp5, label %if.then, label %for.inc\n"
+      "if.then:                                          \n"
+      "  %arrayidx7 = getelementptr inbounds [8 x [8 x i64]], [8 x [8 x i64]]* "
+      "@arr2, i64 0, i64 %i2.028, i64 %i1.029\n"
+      "  %1 = load i64, i64* %arrayidx7, align 8\n"
+      "  %cmp8 = icmp ugt i64 %1, 100\n"
+      "  br i1 %cmp8, label %if.then9, label %for.inc\n"
+      "if.then9:                                         \n"
+      "  %add = add nuw nsw i64 %i2.028, %i1.029\n"
+      "  %arrayidx11 = getelementptr inbounds [8 x [8 x i64]], [8 x [8 x "
+      "i64]]* @arr3, i64 0, i64 %i2.028, i64 %i1.029\n"
+      "  store i64 %add, i64* %arrayidx11, align 8\n"
+      "  br label %for.inc\n"
+      "for.inc:                                          \n"
+      "  %inc = add nuw nsw i64 %i2.028, 1\n"
+      "  %exitcond = icmp eq i64 %inc, 8\n"
+      "  br i1 %exitcond, label %for.inc13, label %for.body3\n"
+      "for.inc13:                                        \n"
+      "  %inc14 = add nuw nsw i64 %i1.029, 1\n"
+      "  %exitcond30 = icmp eq i64 %inc14, 8\n"
+      "  br i1 %exitcond30, label %for.end15, label %for.cond1.preheader\n"
+      "for.end15:                                        \n"
+      "  ret void\n"
+      "}\n";
+
+  Module &M = parseModule(ModuleString);
+
+  Function *F = M.getFunction("f");
+  BasicBlock *LoopHeader = F->getEntryBlock().getSingleSuccessor();
+  auto Plan = buildHCFG(LoopHeader);
+
+  VPRegionBlock *TopRegion = cast<VPRegionBlock>(Plan->getEntry());
+  VPBlockBase *PH = TopRegion->getEntry();
+  VPBlockBase *H = PH->getSingleSuccessor();
+  VPBlockBase *InnerLoopH = H->getSingleSuccessor();
+  VPBlockBase *OuterIf = InnerLoopH->getSuccessors()[0];
+  VPBlockBase *InnerLoopLatch = InnerLoopH->getSuccessors()[1];
+  VPBlockBase *InnerIf = OuterIf->getSuccessors()[0];
+  VPValue *CBV1 = InnerLoopH->getCondBit();
+  VPValue *CBV2 = OuterIf->getCondBit();
+
+  // Apply predication.
+  VPlanPredicator VPP(*Plan);
+  VPP.predicate();
+
+  VPBlockBase *InnerLoopLinSucc = InnerLoopH->getSingleSuccessor();
+  VPBlockBase *OuterIfLinSucc = OuterIf->getSingleSuccessor();
+  VPBlockBase *InnerIfLinSucc = InnerIf->getSingleSuccessor();
+  VPValue *OuterIfPred = OuterIf->getPredicate();
+  VPInstruction *InnerAnd =
+      cast<VPInstruction>(InnerIf->getEntryBasicBlock()->begin());
+  VPValue *InnerIfPred = InnerIf->getPredicate();
+
+  // Test block predicates
+  EXPECT_NE(nullptr, CBV1);
+  EXPECT_NE(nullptr, CBV2);
+  EXPECT_NE(nullptr, InnerAnd);
+  EXPECT_EQ(CBV1, OuterIfPred);
+  EXPECT_EQ(InnerAnd->getOpcode(), Instruction::And);
+  EXPECT_EQ(InnerAnd->getOperand(0), CBV1);
+  EXPECT_EQ(InnerAnd->getOperand(1), CBV2);
+  EXPECT_EQ(InnerIfPred, InnerAnd);
+
+  // Test Linearization
+  EXPECT_EQ(InnerLoopLinSucc, OuterIf);
+  EXPECT_EQ(OuterIfLinSucc, InnerIf);
+  EXPECT_EQ(InnerIfLinSucc, InnerLoopLatch);
+}
+
+// Test generation of Not and Or during predication.
+TEST_F(VPlanPredicatorTest, PredicatorNegOrTest) {
+  const char *ModuleString =
+      "@arr = common global [100 x [100 x i32]] zeroinitializer, align 16\n"
+      "@arr2 = common global [100 x [100 x i32]] zeroinitializer, align 16\n"
+      "@arr3 = common global [100 x [100 x i32]] zeroinitializer, align 16\n"
+      "define void @foo() {\n"
+      "entry:\n"
+      "  br label %for.cond1.preheader\n"
+      "for.cond1.preheader:                              \n"
+      "  %indvars.iv42 = phi i64 [ 0, %entry ], [ %indvars.iv.next43, "
+      "%for.inc22 ]\n"
+      "  br label %for.body3\n"
+      "for.body3:                                        \n"
+      "  %indvars.iv = phi i64 [ 0, %for.cond1.preheader ], [ "
+      "%indvars.iv.next, %if.end21 ]\n"
+      "  %arrayidx5 = getelementptr inbounds [100 x [100 x i32]], [100 x [100 "
+      "x i32]]* @arr, i64 0, i64 %indvars.iv, i64 %indvars.iv42\n"
+      "  %0 = load i32, i32* %arrayidx5, align 4\n"
+      "  %cmp6 = icmp slt i32 %0, 100\n"
+      "  br i1 %cmp6, label %if.then, label %if.end21\n"
+      "if.then:                                          \n"
+      "  %cmp7 = icmp sgt i32 %0, 10\n"
+      "  br i1 %cmp7, label %if.then8, label %if.else\n"
+      "if.then8:                                         \n"
+      "  %add = add nsw i32 %0, 10\n"
+      "  %arrayidx12 = getelementptr inbounds [100 x [100 x i32]], [100 x [100 "
+      "x i32]]* @arr2, i64 0, i64 %indvars.iv, i64 %indvars.iv42\n"
+      "  store i32 %add, i32* %arrayidx12, align 4\n"
+      "  br label %if.end\n"
+      "if.else:                                          \n"
+      "  %sub = add nsw i32 %0, -10\n"
+      "  %arrayidx16 = getelementptr inbounds [100 x [100 x i32]], [100 x [100 "
+      "x i32]]* @arr3, i64 0, i64 %indvars.iv, i64 %indvars.iv42\n"
+      "  store i32 %sub, i32* %arrayidx16, align 4\n"
+      "  br label %if.end\n"
+      "if.end:                                           \n"
+      "  store i32 222, i32* %arrayidx5, align 4\n"
+      "  br label %if.end21\n"
+      "if.end21:                                         \n"
+      "  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1\n"
+      "  %exitcond = icmp eq i64 %indvars.iv.next, 100\n"
+      "  br i1 %exitcond, label %for.inc22, label %for.body3\n"
+      "for.inc22:                                        \n"
+      "  %indvars.iv.next43 = add nuw nsw i64 %indvars.iv42, 1\n"
+      "  %exitcond44 = icmp eq i64 %indvars.iv.next43, 100\n"
+      "  br i1 %exitcond44, label %for.end24, label %for.cond1.preheader\n"
+      "for.end24:                                        \n"
+      "  ret void\n"
+      "}\n";
+
+  Module &M = parseModule(ModuleString);
+  Function *F = M.getFunction("foo");
+  BasicBlock *LoopHeader = F->getEntryBlock().getSingleSuccessor();
+  auto Plan = buildHCFG(LoopHeader);
+
+  VPRegionBlock *TopRegion = cast<VPRegionBlock>(Plan->getEntry());
+  VPBlockBase *PH = TopRegion->getEntry();
+  VPBlockBase *H = PH->getSingleSuccessor();
+  VPBlockBase *OuterIfCmpBlk = H->getSingleSuccessor();
+  VPBlockBase *InnerIfCmpBlk = OuterIfCmpBlk->getSuccessors()[0];
+  VPBlockBase *InnerIfTSucc = InnerIfCmpBlk->getSuccessors()[0];
+  VPBlockBase *InnerIfFSucc = InnerIfCmpBlk->getSuccessors()[1];
+  VPBlockBase *TSuccSucc = InnerIfTSucc->getSingleSuccessor();
+  VPBlockBase *FSuccSucc = InnerIfFSucc->getSingleSuccessor();
+
+  VPValue *OuterCBV = OuterIfCmpBlk->getCondBit();
+  VPValue *InnerCBV = InnerIfCmpBlk->getCondBit();
+
+  // Apply predication.
+  VPlanPredicator VPP(*Plan);
+  VPP.predicate();
+
+  VPInstruction *And =
+      cast<VPInstruction>(InnerIfTSucc->getEntryBasicBlock()->begin());
+  VPInstruction *Not =
+      cast<VPInstruction>(InnerIfFSucc->getEntryBasicBlock()->begin());
+  VPInstruction *NotAnd = cast<VPInstruction>(
+      &*std::next(InnerIfFSucc->getEntryBasicBlock()->begin(), 1));
+  VPInstruction *Or =
+      cast<VPInstruction>(TSuccSucc->getEntryBasicBlock()->begin());
+
+  // Test block predicates
+  EXPECT_NE(nullptr, OuterCBV);
+  EXPECT_NE(nullptr, InnerCBV);
+  EXPECT_NE(nullptr, And);
+  EXPECT_NE(nullptr, Not);
+  EXPECT_NE(nullptr, NotAnd);
+
+  EXPECT_EQ(And->getOpcode(), Instruction::And);
+  EXPECT_EQ(NotAnd->getOpcode(), Instruction::And);
+  EXPECT_EQ(Not->getOpcode(), VPInstruction::Not);
+
+  EXPECT_EQ(And->getOperand(0), OuterCBV);
+  EXPECT_EQ(And->getOperand(1), InnerCBV);
+
+  EXPECT_EQ(Not->getOperand(0), InnerCBV);
+
+  EXPECT_EQ(NotAnd->getOperand(0), OuterCBV);
+  EXPECT_EQ(NotAnd->getOperand(1), Not);
+
+  EXPECT_EQ(InnerIfTSucc->getPredicate(), And);
+  EXPECT_EQ(InnerIfFSucc->getPredicate(), NotAnd);
+
+  EXPECT_EQ(TSuccSucc, FSuccSucc);
+  EXPECT_EQ(Or->getOpcode(), Instruction::Or);
+  EXPECT_EQ(TSuccSucc->getPredicate(), Or);
+
+  // Test operands of the Or - account for differences in predecessor block
+  // ordering.
+  VPInstruction *OrOp0Inst = cast<VPInstruction>(Or->getOperand(0));
+  VPInstruction *OrOp1Inst = cast<VPInstruction>(Or->getOperand(1));
+
+  bool ValidOrOperands = false;
+  if (((OrOp0Inst == And) && (OrOp1Inst == NotAnd)) ||
+      ((OrOp0Inst == NotAnd) && (OrOp1Inst == And)))
+    ValidOrOperands = true;
+
+  EXPECT_TRUE(ValidOrOperands);
+}
+
+} // namespace
+} // namespace llvm
diff --git a/unittests/Transforms/Vectorize/VPlanSlpTest.cpp b/unittests/Transforms/Vectorize/VPlanSlpTest.cpp
index 0381925655e716730de88a97c9a705cc634ec04e..403ffb1d315227ae1b45d6dc76def91b206bf810 100644
--- a/unittests/Transforms/Vectorize/VPlanSlpTest.cpp
+++ b/unittests/Transforms/Vectorize/VPlanSlpTest.cpp
@@ -1,9 +1,8 @@
 //===- llvm/unittest/Transforms/Vectorize/VPlanSlpTest.cpp ---------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/Transforms/Vectorize/VPlanTest.cpp b/unittests/Transforms/Vectorize/VPlanTest.cpp
index 67712a7cae2e35ff9feb18868794ebb5d6badbed..90f79a3eaf435db5c76a4b9009a21417309f3d1b 100644
--- a/unittests/Transforms/Vectorize/VPlanTest.cpp
+++ b/unittests/Transforms/Vectorize/VPlanTest.cpp
@@ -1,10 +1,9 @@
 //===- llvm/unittests/Transforms/Vectorize/VPlanTest.cpp - VPlan tests ----===//
 //
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/Transforms/Vectorize/VPlanTestBase.h b/unittests/Transforms/Vectorize/VPlanTestBase.h
index 0fc4a9a21c8809a6c597118c24ddb6ae94afd2de..3ee811a68302f49a0b5a4b245acc383e89c9fbfc 100644
--- a/unittests/Transforms/Vectorize/VPlanTestBase.h
+++ b/unittests/Transforms/Vectorize/VPlanTestBase.h
@@ -1,9 +1,8 @@
 //===- llvm/unittest/Transforms/Vectorize/VPlanTestBase.h -----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/unittests/XRay/FDRBlockIndexerTest.cpp b/unittests/XRay/FDRBlockIndexerTest.cpp
index 558840b111a6b4ea589bc8651584075173600354..f2fab2f2ba1eaa9948dedbcc0f8c4286f8ccd969 100644
--- a/unittests/XRay/FDRBlockIndexerTest.cpp
+++ b/unittests/XRay/FDRBlockIndexerTest.cpp
@@ -1,9 +1,8 @@
 //===- llvm/unittest/XRay/FDRTraceWriterTest.cpp ----------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 #include "llvm/XRay/BlockIndexer.h"
diff --git a/unittests/XRay/FDRBlockVerifierTest.cpp b/unittests/XRay/FDRBlockVerifierTest.cpp
index a6e0a8cd38883e9f42ddedcce3e1e334cc0b3769..5af27450291abc55640d5b9ff5e8ed0f2d4121cf 100644
--- a/unittests/XRay/FDRBlockVerifierTest.cpp
+++ b/unittests/XRay/FDRBlockVerifierTest.cpp
@@ -1,9 +1,8 @@
 //===- llvm/unittest/XRay/FDRBlockVerifierTest.cpp --------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 #include "llvm/Testing/Support/Error.h"
diff --git a/unittests/XRay/FDRProducerConsumerTest.cpp b/unittests/XRay/FDRProducerConsumerTest.cpp
index 03fabdf2372da9ed05807faa6a5a75b6c8f4699a..bbc15be03027816baeb83d3cea8204ce1a493474 100644
--- a/unittests/XRay/FDRProducerConsumerTest.cpp
+++ b/unittests/XRay/FDRProducerConsumerTest.cpp
@@ -1,9 +1,8 @@
 //===- llvm/unittest/XRay/FDRProducerConsumerTest.cpp -----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/unittests/XRay/FDRRecordPrinterTest.cpp b/unittests/XRay/FDRRecordPrinterTest.cpp
index a0ec3f22bf5bdc8e4b896a66ba1a8669e776249b..39c1e866f04e74e3a9ab7e58cdd5489bb7955700 100644
--- a/unittests/XRay/FDRRecordPrinterTest.cpp
+++ b/unittests/XRay/FDRRecordPrinterTest.cpp
@@ -1,9 +1,8 @@
 //===- llvm/unittest/XRay/FDRRecordPrinterTest.cpp --------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 #include "llvm/Support/raw_ostream.h"
diff --git a/unittests/XRay/FDRRecordsTest.cpp b/unittests/XRay/FDRRecordsTest.cpp
index 86b478a5a4588eabca303272de206e2844284e4d..aac3c86e49e0f84272e14cf76587569d3056836e 100644
--- a/unittests/XRay/FDRRecordsTest.cpp
+++ b/unittests/XRay/FDRRecordsTest.cpp
@@ -1,9 +1,8 @@
 //===- FDRRecords.cpp - Unit Tests for XRay FDR Record Loading ------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 #include "gmock/gmock.h"
diff --git a/unittests/XRay/FDRTraceWriterTest.cpp b/unittests/XRay/FDRTraceWriterTest.cpp
index e78b71d6ff748d7b8f6502e62708628fdba20550..4f2f06e468c68f6ad817d2be2311de7e5eddc6ce 100644
--- a/unittests/XRay/FDRTraceWriterTest.cpp
+++ b/unittests/XRay/FDRTraceWriterTest.cpp
@@ -1,9 +1,8 @@
 //===- llvm/unittest/XRay/FDRTraceWriterTest.cpp ----------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/unittests/XRay/GraphTest.cpp b/unittests/XRay/GraphTest.cpp
index b17858f0c1c0b43b6fb8e80b6ba8c0812cdbef69..3c82d53492e8d2aa2c57759c5356e02fae6dfaa1 100644
--- a/unittests/XRay/GraphTest.cpp
+++ b/unittests/XRay/GraphTest.cpp
@@ -1,9 +1,8 @@
 //===- llvm/unittest/XRay/GraphTest.cpp - XRay Graph unit tests -*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/XRay/ProfileTest.cpp b/unittests/XRay/ProfileTest.cpp
index 8486b7b27e8bce0aa28caf658007cde433e357e8..7eb848f11163df43735dde39eaf2842e25e4f96d 100644
--- a/unittests/XRay/ProfileTest.cpp
+++ b/unittests/XRay/ProfileTest.cpp
@@ -1,9 +1,8 @@
 //===- ProfileTest.cpp - XRay Profile unit tests ----------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 #include "llvm/XRay/Profile.h"
diff --git a/unittests/tools/llvm-cfi-verify/FileAnalysis.cpp b/unittests/tools/llvm-cfi-verify/FileAnalysis.cpp
index 05880fcec805d8f6e0af745a977597efacd4cb2c..58fdbe5c7ad502c84934b5d13e85ae97f0fe5a05 100644
--- a/unittests/tools/llvm-cfi-verify/FileAnalysis.cpp
+++ b/unittests/tools/llvm-cfi-verify/FileAnalysis.cpp
@@ -1,9 +1,8 @@
 //===- llvm/unittests/tools/llvm-cfi-verify/FileAnalysis.cpp --------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/tools/llvm-cfi-verify/GraphBuilder.cpp b/unittests/tools/llvm-cfi-verify/GraphBuilder.cpp
index a7d09b547814a6c178938469ed2c8b431a6d0231..9775c5bd8ec3c542f44bf22af5ec3139e8ced6f2 100644
--- a/unittests/tools/llvm-cfi-verify/GraphBuilder.cpp
+++ b/unittests/tools/llvm-cfi-verify/GraphBuilder.cpp
@@ -1,9 +1,8 @@
 //===- llvm/unittests/llvm-cfi-verify/GraphBuilder.cpp --------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/tools/llvm-exegesis/AArch64/TargetTest.cpp b/unittests/tools/llvm-exegesis/AArch64/TargetTest.cpp
index 6dd5305d4dc6d0aa4e9bcc07c909ddf69aac7e9c..4a685410ca2eeb42ccc769a352449f3394c61d06 100644
--- a/unittests/tools/llvm-exegesis/AArch64/TargetTest.cpp
+++ b/unittests/tools/llvm-exegesis/AArch64/TargetTest.cpp
@@ -1,9 +1,8 @@
 //===-- TargetTest.cpp ------------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/tools/llvm-exegesis/ARM/AssemblerTest.cpp b/unittests/tools/llvm-exegesis/ARM/AssemblerTest.cpp
index a20fa5556bb501d7763126c5af62e21ac7f34f2f..70ad4875be26da46774ed28ad1a8827165c356d7 100644
--- a/unittests/tools/llvm-exegesis/ARM/AssemblerTest.cpp
+++ b/unittests/tools/llvm-exegesis/ARM/AssemblerTest.cpp
@@ -1,9 +1,8 @@
 //===-- AssemblerTest.cpp ---------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/tools/llvm-exegesis/BenchmarkRunnerTest.cpp b/unittests/tools/llvm-exegesis/BenchmarkRunnerTest.cpp
index c518491063afd1326a286a882dd0ddbefabfa594..9414a48c989e4e91a6ef6b3ef1bf41e97c02289b 100644
--- a/unittests/tools/llvm-exegesis/BenchmarkRunnerTest.cpp
+++ b/unittests/tools/llvm-exegesis/BenchmarkRunnerTest.cpp
@@ -1,9 +1,8 @@
 //===-- BenchmarkRunnerTest.cpp ---------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/tools/llvm-exegesis/ClusteringTest.cpp b/unittests/tools/llvm-exegesis/ClusteringTest.cpp
index 8ea77dcbddec14f1a469f13e4ac4330be2a1d808..6834206ff574caaeea8bc6179c72c2b13897191c 100644
--- a/unittests/tools/llvm-exegesis/ClusteringTest.cpp
+++ b/unittests/tools/llvm-exegesis/ClusteringTest.cpp
@@ -1,9 +1,8 @@
 //===-- ClusteringTest.cpp --------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/tools/llvm-exegesis/Common/AssemblerUtils.h b/unittests/tools/llvm-exegesis/Common/AssemblerUtils.h
index 8a144e5c26f0daf7be721931d9148693e3ef0142..fcbff940528381f35760aababdc3534f58efd005 100644
--- a/unittests/tools/llvm-exegesis/Common/AssemblerUtils.h
+++ b/unittests/tools/llvm-exegesis/Common/AssemblerUtils.h
@@ -1,9 +1,8 @@
 //===-- AssemblerUtils.h ----------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/tools/llvm-exegesis/PerfHelperTest.cpp b/unittests/tools/llvm-exegesis/PerfHelperTest.cpp
index 91ed4a609674071535f3db9dab509273de6e8308..39942afb85c8ccad9aca68ffc613493b83472595 100644
--- a/unittests/tools/llvm-exegesis/PerfHelperTest.cpp
+++ b/unittests/tools/llvm-exegesis/PerfHelperTest.cpp
@@ -1,9 +1,8 @@
 //===-- PerfHelperTest.cpp --------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/tools/llvm-exegesis/PowerPC/AnalysisTest.cpp b/unittests/tools/llvm-exegesis/PowerPC/AnalysisTest.cpp
index 15d81ca88dec3131fe146740aef2a1b5fdc35b1f..a7d2c6d95e649809f357b8c5c7486b3e0386cabf 100644
--- a/unittests/tools/llvm-exegesis/PowerPC/AnalysisTest.cpp
+++ b/unittests/tools/llvm-exegesis/PowerPC/AnalysisTest.cpp
@@ -1,9 +1,8 @@
 //===-- AnalysisTest.cpp ---------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/tools/llvm-exegesis/PowerPC/TargetTest.cpp b/unittests/tools/llvm-exegesis/PowerPC/TargetTest.cpp
index 0e34a15d5e32288d84a325f8fd70b4f19277f80e..3f6c5b6d9ea6c043d2ad9d2000ccbe050037f85d 100644
--- a/unittests/tools/llvm-exegesis/PowerPC/TargetTest.cpp
+++ b/unittests/tools/llvm-exegesis/PowerPC/TargetTest.cpp
@@ -1,9 +1,8 @@
 //===-- TargetTest.cpp ---------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/tools/llvm-exegesis/RegisterValueTest.cpp b/unittests/tools/llvm-exegesis/RegisterValueTest.cpp
index 8453720dc70846e3a2da82e7c7a13a933971d19b..a7c8ba6d571f14666abc2facdef197ded98e0110 100644
--- a/unittests/tools/llvm-exegesis/RegisterValueTest.cpp
+++ b/unittests/tools/llvm-exegesis/RegisterValueTest.cpp
@@ -1,9 +1,8 @@
 //===-- RegisterValueTest.cpp -----------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/tools/llvm-exegesis/X86/AnalysisTest.cpp b/unittests/tools/llvm-exegesis/X86/AnalysisTest.cpp
index c8939186eb1482bbdd5ff680e72e148e102895f1..76ca39138c4a91fb97485770b762c5d9168f7888 100644
--- a/unittests/tools/llvm-exegesis/X86/AnalysisTest.cpp
+++ b/unittests/tools/llvm-exegesis/X86/AnalysisTest.cpp
@@ -1,9 +1,8 @@
 //===-- AnalysisTest.cpp ---------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/tools/llvm-exegesis/X86/AssemblerTest.cpp b/unittests/tools/llvm-exegesis/X86/AssemblerTest.cpp
index 451c3f67e75ed0d5162f962a3a39cb4a707784b5..ce7d9be663c517259fda00d98cb160c5860170dd 100644
--- a/unittests/tools/llvm-exegesis/X86/AssemblerTest.cpp
+++ b/unittests/tools/llvm-exegesis/X86/AssemblerTest.cpp
@@ -1,9 +1,8 @@
 //===-- AssemblerTest.cpp ---------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/tools/llvm-exegesis/X86/BenchmarkResultTest.cpp b/unittests/tools/llvm-exegesis/X86/BenchmarkResultTest.cpp
index f069c21b364f00fbb89666a264c94cc3cb40eb7e..56cbe3183f1c39b18c93ea44457829e2b4a8466a 100644
--- a/unittests/tools/llvm-exegesis/X86/BenchmarkResultTest.cpp
+++ b/unittests/tools/llvm-exegesis/X86/BenchmarkResultTest.cpp
@@ -1,9 +1,8 @@
 //===-- BenchmarkResultTest.cpp ---------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/tools/llvm-exegesis/X86/RegisterAliasingTest.cpp b/unittests/tools/llvm-exegesis/X86/RegisterAliasingTest.cpp
index c85c99c25ecd719e302cdecbe2222b449d66877d..b65b2a9315462f45239f298dfc07568e47b354d6 100644
--- a/unittests/tools/llvm-exegesis/X86/RegisterAliasingTest.cpp
+++ b/unittests/tools/llvm-exegesis/X86/RegisterAliasingTest.cpp
@@ -1,9 +1,8 @@
 //===-- RegisterAliasingTest.cpp --------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/tools/llvm-exegesis/X86/SnippetGeneratorTest.cpp b/unittests/tools/llvm-exegesis/X86/SnippetGeneratorTest.cpp
index 1689defded838e13e330dbb9e0bb0ff36b33636e..0f8dc1f39c6babc8a20c997ac7e6fd7718f353dd 100644
--- a/unittests/tools/llvm-exegesis/X86/SnippetGeneratorTest.cpp
+++ b/unittests/tools/llvm-exegesis/X86/SnippetGeneratorTest.cpp
@@ -1,9 +1,8 @@
 //===-- SnippetGeneratorTest.cpp --------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/unittests/tools/llvm-exegesis/X86/TargetTest.cpp b/unittests/tools/llvm-exegesis/X86/TargetTest.cpp
index b1ccb486bfb335a890d919f142c8ef6dd32c78b5..15a1d8381fd34499027d970314a8cdf82f4c353e 100644
--- a/unittests/tools/llvm-exegesis/X86/TargetTest.cpp
+++ b/unittests/tools/llvm-exegesis/X86/TargetTest.cpp
@@ -1,9 +1,8 @@
 //===-- TargetTest.cpp -----------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/utils/DSAclean.py b/utils/DSAclean.py
index 6c43357019ef00592c53ecfb59d0ca93eb80c757..789a825a06712221ef5140798022d5bee7a1bd5b 100755
--- a/utils/DSAclean.py
+++ b/utils/DSAclean.py
@@ -8,10 +8,13 @@
 #the comments
 #10/12/2005: now it only removes nodes and edges for which the label is %tmp.# rather
 #than removing all lines for which the lable CONTAINS %tmp.#
+
+from __future__ import print_function
+
 import re
 import sys
 if( len(sys.argv) < 3 ):
-	print 'usage is: ./DSAclean <dot_file_to_be_cleaned> <out_put_file>'
+	print('usage is: ./DSAclean <dot_file_to_be_cleaned> <out_put_file>')
 	sys.exit(1)
 #get a file object
 input = open(sys.argv[1], 'r')
diff --git a/utils/DSAextract.py b/utils/DSAextract.py
index 89dece1f3339419048cdbf05bc1838c184c66f5d..258aac47e82d2dd7abc86d385e0e3441f4bd3964 100644
--- a/utils/DSAextract.py
+++ b/utils/DSAextract.py
@@ -25,14 +25,16 @@
 #currently the script prints the names it is searching for
 #to STDOUT, so you can check to see if they are what you intend
 
+from __future__ import print_function
+
 import re
 import string
 import sys
 
 
 if len(sys.argv) < 3:
-	print 'usage is ./DSAextract <dot_file_to_modify> \
-			<output_file> [list of nodes to extract]'
+	print('usage is ./DSAextract <dot_file_to_modify> \
+			<output_file> [list of nodes to extract]')
 
 #open the input file
 input = open(sys.argv[1], 'r')
@@ -73,7 +75,7 @@ while buffer != '':
 #test code
 #print '\n'
 
-print node_name_set
+print(node_name_set)
 
 #print node_set
 	
diff --git a/utils/FileCheck/FileCheck.cpp b/utils/FileCheck/FileCheck.cpp
index fdfd3e7cb7c56057940f4507077f419a86271510..0d5f38acfa7aab584ad0eeb4cca8fd4771bacdd5 100644
--- a/utils/FileCheck/FileCheck.cpp
+++ b/utils/FileCheck/FileCheck.cpp
@@ -1,9 +1,8 @@
 //===- FileCheck.cpp - Check that File's Contents match what is expected --===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -51,9 +50,10 @@ static cl::list<std::string> ImplicitCheckNot(
              "this pattern occur which are not matched by a positive pattern"),
     cl::value_desc("pattern"));
 
-static cl::list<std::string> GlobalDefines("D", cl::Prefix,
-    cl::desc("Define a variable to be used in capture patterns."),
-    cl::value_desc("VAR=VALUE"));
+static cl::list<std::string>
+    GlobalDefines("D", cl::AlwaysPrefix,
+                  cl::desc("Define a variable to be used in capture patterns."),
+                  cl::value_desc("VAR=VALUE"));
 
 static cl::opt<bool> AllowEmptyInput(
     "allow-empty", cl::init(false),
@@ -79,13 +79,16 @@ static cl::opt<bool> AllowDeprecatedDagOverlap(
              "provided for convenience as old tests are migrated to the new\n"
              "non-overlapping CHECK-DAG implementation.\n"));
 
-static cl::opt<bool> Verbose("v", cl::init(false),
-                             cl::desc("Print directive pattern matches.\n"));
+static cl::opt<bool> Verbose(
+    "v", cl::init(false),
+    cl::desc("Print directive pattern matches, or add them to the input dump\n"
+             "if enabled.\n"));
 
 static cl::opt<bool> VerboseVerbose(
     "vv", cl::init(false),
     cl::desc("Print information helpful in diagnosing internal FileCheck\n"
-             "issues.  Implies -v.\n"));
+             "issues, or add it to the input dump if enabled.  Implies\n"
+             "-v.\n"));
 static const char * DumpInputEnv = "FILECHECK_DUMP_INPUT_ON_FAILURE";
 
 static cl::opt<bool> DumpInputOnFailure(
@@ -523,8 +526,25 @@ int main(int argc, char **argv) {
   for (auto CheckNot : ImplicitCheckNot)
     Req.ImplicitCheckNot.push_back(CheckNot);
 
-  for (auto G : GlobalDefines)
+  bool GlobalDefineError = false;
+  for (auto G : GlobalDefines) {
+    size_t EqIdx = G.find('=');
+    if (EqIdx == std::string::npos) {
+      errs() << "Missing equal sign in command-line definition '-D" << G
+             << "'\n";
+      GlobalDefineError = true;
+      continue;
+    }
+    if (EqIdx == 0) {
+      errs() << "Missing pattern variable name in command-line definition '-D"
+             << G << "'\n";
+      GlobalDefineError = true;
+      continue;
+    }
     Req.GlobalDefines.push_back(G);
+  }
+  if (GlobalDefineError)
+    return 2;
 
   Req.AllowEmptyInput = AllowEmptyInput;
   Req.EnableVarScope = EnableVarScope;
diff --git a/utils/KillTheDoctor/KillTheDoctor.cpp b/utils/KillTheDoctor/KillTheDoctor.cpp
index c9def83309f6c3a4459a94d5e175e48bdbc3c8f8..358ef165cf63a58ea343b50b9a3ba41f6aa43dc5 100644
--- a/utils/KillTheDoctor/KillTheDoctor.cpp
+++ b/utils/KillTheDoctor/KillTheDoctor.cpp
@@ -1,9 +1,8 @@
 //===- KillTheDoctor - Prevent Dr. Watson from stopping tests ---*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/utils/LLVMBuild.txt b/utils/LLVMBuild.txt
index 382bfd31447b42ba55d8029965d1ad261eeb2769..2bf879755f538818eb8b128c4d62870d4bfb9ede 100644
--- a/utils/LLVMBuild.txt
+++ b/utils/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./utils/LLVMBuild.txt ------------------------------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/utils/LLVMVisualizers/llvm.natvis b/utils/LLVMVisualizers/llvm.natvis
index bbded5dcc305bf6244d94f70a83258d3451609b2..42d3a122bece5945d1428b02b18ad9f3d5815943 100644
--- a/utils/LLVMVisualizers/llvm.natvis
+++ b/utils/LLVMVisualizers/llvm.natvis
@@ -35,6 +35,7 @@ For later versions of Visual Studio, no setup is required.
     <DisplayString IncludeView ="elt4">, /* {Size - 4} more*/ </DisplayString>
     <DisplayString Condition="Size == 0">empty</DisplayString>
     <DisplayString Condition="Size != 0">{{{*this,view(elt0)}}}</DisplayString>
+    <DisplayString>Uninitialized</DisplayString>
     <Expand>
       <Item Name="[size]">Size</Item>
       <Item Name="[capacity]">Capacity</Item>
@@ -92,36 +93,87 @@ For later versions of Visual Studio, no setup is required.
     </Expand>
   </Type>
 
+  <!-- PointerUnion types - In addition to the regular view, which displays the pointer, there is a "deref" view that
+       displays the pointed to object, which is often needed by other visualizers -->
   <Type Name="llvm::PointerUnion&lt;*,*&gt;">
-    <DisplayString Condition="((Val.Value &gt;&gt; Val.IntShift) &amp; Val.IntMask) == 0">{"$T1", s8b}: {($T1)(Val.Value &amp; Val.PointerBitMask)}</DisplayString>
-    <DisplayString Condition="((Val.Value &gt;&gt; Val.IntShift) &amp; Val.IntMask) != 0">{"$T2", s8b}: {($T2)(Val.Value &amp; Val.PointerBitMask)}</DisplayString>
+    <DisplayString  Optional="true" IncludeView="deref" Condition="((Val.Value &gt;&gt; ValTy::InfoTy::IntShift) &amp; ValTy::InfoTy::IntMask) == 0">{*($T1)(Val.Value &amp; ValTy::InfoTy::PointerBitMask)}</DisplayString>
+    <DisplayString  Optional="true" IncludeView="deref" Condition="((Val.Value &gt;&gt; ValTy::InfoTy::IntShift) &amp; ValTy::InfoTy::IntMask) != 0">{*($T2)(Val.Value &amp; ValTy::InfoTy::PointerBitMask)}</DisplayString>
+    <DisplayString  Optional="true" Condition="((Val.Value &gt;&gt; ValTy::InfoTy::IntShift) &amp; ValTy::InfoTy::IntMask) == 0">{"$T1", s8b}: {($T1)(Val.Value &amp; ValTy::InfoTy::PointerBitMask)}</DisplayString>
+    <DisplayString  Optional="true" Condition="((Val.Value &gt;&gt; ValTy::InfoTy::IntShift) &amp; ValTy::InfoTy::IntMask) != 0">{"$T2", s8b}: {($T2)(Val.Value &amp; ValTy::InfoTy::PointerBitMask)}</DisplayString>
     <Expand>
-      <ExpandedItem Condition="((Val.Value &gt;&gt; Val.IntShift) &amp; Val.IntMask) == 0">($T1)(Val.Value &amp; Val.PointerBitMask)</ExpandedItem>
-      <ExpandedItem Condition="((Val.Value &gt;&gt; Val.IntShift) &amp; Val.IntMask) != 0">($T2)(Val.Value &amp; Val.PointerBitMask)</ExpandedItem>
+      <Item Name="[Holds]" Condition="((Val.Value &gt;&gt; ValTy::InfoTy::IntShift) &amp; ValTy::InfoTy::IntMask) == 0">"$T1", s8b</Item>
+      <Item Name="[Ptr]" Optional="true"  Condition="((Val.Value &gt;&gt; ValTy::InfoTy::IntShift) &amp; ValTy::InfoTy::IntMask) == 0">($T1)(Val.Value &amp; ValTy::InfoTy::PointerBitMask)</Item>
+      <Item Name="[Holds]" Condition="((Val.Value &gt;&gt; ValTy::InfoTy::IntShift) &amp; ValTy::InfoTy::IntMask) != 0">"$T2", s8b</Item>
+      <Item Name="[Ptr]" Optional="true" Condition="((Val.Value &gt;&gt; ValTy::InfoTy::IntShift) &amp; ValTy::InfoTy::IntMask) != 0">($T2)(Val.Value &amp; ValTy::InfoTy::PointerBitMask)</Item>
     </Expand>
   </Type>
 
   <Type Name="llvm::PointerUnion3&lt;*,*,*&gt;">
-    <DisplayString Condition="(Val.Val.Value &amp; 2) != 2 &amp;&amp; (Val.Val.Value &amp; 1) != 1">{"$T1", s8b}: {($T1)((Val.Val.Value &gt;&gt; 2) &lt;&lt; 2)}</DisplayString>
-    <DisplayString Condition="(Val.Val.Value &amp; 2) == 2">{"$T2", s8b}: {($T2)((Val.Val.Value &gt;&gt; 2) &lt;&lt; 2)}</DisplayString>
-    <DisplayString Condition="(Val.Val.Value &amp; 1) == 1">{"$T3", s8b}: {($T3)((Val.Val.Value &gt;&gt; 2) &lt;&lt; 2)}</DisplayString>
+    <DisplayString Optional="true" IncludeView="deref" Condition="(Val.Val.Value &gt;&gt; ValTy::ValTy::InfoTy::IntShift) &amp; 1">{Val,view(deref)}</DisplayString>
+    <DisplayString Optional="true" IncludeView="deref">{*(InnerUnion*)&amp;Val.Val.Value,view(deref)}</DisplayString>
+    <DisplayString Condition="(Val.Val.Value &gt;&gt; ValTy::ValTy::InfoTy::IntShift) &amp; 1">{Val}</DisplayString>
+    <DisplayString>{*(InnerUnion*)&amp;Val.Val.Value}</DisplayString>
     <Expand>
-      <ExpandedItem Condition="(Val.Val.Value &amp; 2) != 2 &amp;&amp; (Val.Val.Value &amp; 1) != 1">($T1)((Val.Val.Value &gt;&gt; 2) &lt;&lt; 2)</ExpandedItem>
-      <ExpandedItem Condition="(Val.Val.Value &amp; 2) == 2">($T2)((Val.Val.Value &gt;&gt; 2) &lt;&lt; 2)</ExpandedItem>
-      <ExpandedItem Condition="(Val.Val.Value &amp; 1) == 1">($T3)((Val.Val.Value &gt;&gt; 2) &lt;&lt; 2)</ExpandedItem>
+      <Item Name="[Holds]" Condition="(Val.Val.Value &gt;&gt; ValTy::ValTy::InfoTy::IntShift) &amp; 1">"$T3", s8b</Item>
+      <Item Name="[Ptr]" Condition="(Val.Val.Value &gt;&gt; ValTy::ValTy::InfoTy::IntShift) &amp; 1">($T3)(Val.Val.Value &amp; ValTy::ValTy::InfoTy::PointerBitMask)</Item>
+      <ExpandedItem Condition="!((Val.Val.Value &gt;&gt; ValTy::ValTy::InfoTy::IntShift) &amp; 1)">*(InnerUnion*)&amp;Val.Val.Value</ExpandedItem>
     </Expand>
   </Type>
 
   <Type Name="llvm::PointerUnion4&lt;*,*,*,*&gt;">
-    <DisplayString Condition="(Val.Val.Value &amp; 3) != 3 &amp;&amp; (Val.Val.Value &amp; 2) != 2 &amp;&amp; (Val.Val.Value &amp; 1) != 1">{"$T1", s8b}: {($T1)((Val.Val.Value &gt;&gt; 2) &lt;&lt; 2)}</DisplayString>
-    <DisplayString Condition="(Val.Val.Value &amp; 3) != 3 &amp;&amp; (Val.Val.Value &amp; 2) == 2">{"$T2", s8b}: {($T2)((Val.Val.Value &gt;&gt; 2) &lt;&lt; 2)}</DisplayString>
-    <DisplayString Condition="(Val.Val.Value &amp; 3) != 3 &amp;&amp; (Val.Val.Value &amp; 1) == 1">{"$T3", s8b}: {($T3)((Val.Val.Value &gt;&gt; 2) &lt;&lt; 2)}</DisplayString>
-    <DisplayString Condition="(Val.Val.Value &amp; 3) == 3">{"$T4", s8b}: {($T4)((Val.Val.Value &gt;&gt; 2) &lt;&lt; 2)}</DisplayString>
+    <DisplayString Optional="true" IncludeView="deref" 
+                   Condition="((Val.Val.Value &gt;&gt; ValTy::ValTy::InfoTy::IntShift) &amp; 1) == 0 &amp;&amp; ((Val.Val.Value &gt;&gt; InnerUnion1::ValTy::InfoTy::IntShift) &amp; 1) == 0">
+      {*($T1)(Val.Val.Value &amp; InnerUnion1::ValTy::InfoTy::PointerBitMask)}
+    </DisplayString>
+    <DisplayString Optional="true" IncludeView="deref" 
+                   Condition="((Val.Val.Value &gt;&gt; ValTy::ValTy::InfoTy::IntShift) &amp; 1) == 0 &amp;&amp; ((Val.Val.Value &gt;&gt; InnerUnion1::ValTy::InfoTy::IntShift) &amp; 1) == 1">
+      {*($T2)(Val.Val.Value &amp; InnerUnion1::ValTy::InfoTy::PointerBitMask)}
+    </DisplayString>
+    <DisplayString  Optional="true" IncludeView="deref" 
+                    Condition="((Val.Val.Value &gt;&gt; ValTy::ValTy::InfoTy::IntShift) &amp; 1) == 1 &amp;&amp; ((Val.Val.Value &gt;&gt; InnerUnion2::ValTy::InfoTy::IntShift) &amp; 1) == 0">
+      {*($T3)(Val.Val.Value &amp; InnerUnion2::ValTy::InfoTy::PointerBitMask)}
+    </DisplayString>
+    <DisplayString  Optional="true" IncludeView="deref" 
+                    Condition="((Val.Val.Value &gt;&gt; ValTy::ValTy::InfoTy::IntShift) &amp; 1) == 1 &amp;&amp; ((Val.Val.Value &gt;&gt; InnerUnion2::ValTy::InfoTy::IntShift) &amp; 1) == 1">
+      {*($T4)(Val.Val.Value &amp; InnerUnion2::ValTy::InfoTy::PointerBitMask)}
+    </DisplayString>
+    <DisplayString Condition="((Val.Val.Value &gt;&gt; ValTy::ValTy::InfoTy::IntShift) &amp; 1) == 0 &amp;&amp; ((Val.Val.Value &gt;&gt; InnerUnion1::ValTy::InfoTy::IntShift) &amp; 1) == 0">
+      {"$T1", s8b}: {($T1)(Val.Val.Value &amp; InnerUnion1::ValTy::InfoTy::PointerBitMask)}
+    </DisplayString>
+    <DisplayString Condition="((Val.Val.Value &gt;&gt; ValTy::ValTy::InfoTy::IntShift) &amp; 1) == 0 &amp;&amp; ((Val.Val.Value &gt;&gt; InnerUnion1::ValTy::InfoTy::IntShift) &amp; 1) == 1">
+      {"$T2", s8b}: {($T2)(Val.Val.Value &amp; InnerUnion1::ValTy::InfoTy::PointerBitMask)}
+    </DisplayString>
+    <DisplayString Condition="((Val.Val.Value &gt;&gt; ValTy::ValTy::InfoTy::IntShift) &amp; 1) == 1 &amp;&amp; ((Val.Val.Value &gt;&gt; InnerUnion2::ValTy::InfoTy::IntShift) &amp; 1) == 0">
+      {"$T3", s8b}: {($T3)(Val.Val.Value &amp; InnerUnion2::ValTy::InfoTy::PointerBitMask)}
+    </DisplayString>
+    <DisplayString Condition="((Val.Val.Value &gt;&gt; ValTy::ValTy::InfoTy::IntShift) &amp; 1) == 1 &amp;&amp; ((Val.Val.Value &gt;&gt; InnerUnion2::ValTy::InfoTy::IntShift) &amp; 1) == 1">
+      {"$T4", s8b}: {($T4)(Val.Val.Value &amp; InnerUnion2::ValTy::InfoTy::PointerBitMask)}
+    </DisplayString>
     <Expand>
-      <ExpandedItem Condition="(Val.Val.Value &amp; 3) != 3 &amp;&amp; (Val.Val.Value &amp; 2) != 2 &amp;&amp; (Val.Val.Value &amp; 1) != 1">($T1)((Val.Val.Value &gt;&gt; 2) &lt;&lt; 2)</ExpandedItem>
-      <ExpandedItem Condition="(Val.Val.Value &amp; 3) != 3 &amp;&amp; (Val.Val.Value &amp; 2) == 2">($T2)((Val.Val.Value &gt;&gt; 2) &lt;&lt; 2)</ExpandedItem>
-      <ExpandedItem Condition="(Val.Val.Value &amp; 3) != 3 &amp;&amp; (Val.Val.Value &amp; 1) == 1">($T3)((Val.Val.Value &gt;&gt; 2) &lt;&lt; 2)</ExpandedItem>
-      <ExpandedItem Condition="(Val.Val.Value &amp; 3) == 3">($T4)((Val.Val.Value &gt;&gt; 2) &lt;&lt; 2)</ExpandedItem>
+      <Item Name="[Holds]" Condition="((Val.Val.Value &gt;&gt; ValTy::ValTy::InfoTy::IntShift) &amp; 1) == 0 &amp;&amp; ((Val.Val.Value &gt;&gt; InnerUnion1::ValTy::InfoTy::IntShift) &amp; 1) == 0">
+        "$T1", s8b
+      </Item>
+      <Item Name="[Ptr]" Condition="((Val.Val.Value &gt;&gt; ValTy::ValTy::InfoTy::IntShift) &amp; 1) == 0 &amp;&amp; ((Val.Val.Value &gt;&gt; InnerUnion1::ValTy::InfoTy::IntShift) &amp; 1) == 0">
+        ($T1)(Val.Val.Value &amp; InnerUnion1::ValTy::InfoTy::PointerBitMask)
+      </Item>
+      <Item Name="[Holds]" Condition="((Val.Val.Value &gt;&gt; ValTy::ValTy::InfoTy::IntShift) &amp; 1) == 0 &amp;&amp; ((Val.Val.Value &gt;&gt; InnerUnion1::ValTy::InfoTy::IntShift) &amp; 1) == 1">
+        "$T2", s8b
+      </Item>
+      <Item Name="[Ptr]" Condition="((Val.Val.Value &gt;&gt; ValTy::ValTy::InfoTy::IntShift) &amp; 1) == 0 &amp;&amp; ((Val.Val.Value &gt;&gt; InnerUnion1::ValTy::InfoTy::IntShift) &amp; 1) == 1">
+        ($T2)(Val.Val.Value &amp; InnerUnion1::ValTy::InfoTy::PointerBitMask)
+      </Item>
+      <Item Name="[Holds]" Condition="((Val.Val.Value &gt;&gt; ValTy::ValTy::InfoTy::IntShift) &amp; 1) == 1 &amp;&amp; ((Val.Val.Value &gt;&gt; InnerUnion2::ValTy::InfoTy::IntShift) &amp; 1) == 0">
+        "$T3", s8b
+      </Item>
+      <Item Name="[Ptr]" Condition="((Val.Val.Value &gt;&gt; ValTy::ValTy::InfoTy::IntShift) &amp; 1) == 1 &amp;&amp; ((Val.Val.Value &gt;&gt; InnerUnion2::ValTy::InfoTy::IntShift) &amp; 1) == 0">
+        ($T3)(Val.Val.Value &amp; InnerUnion1::ValTy::InfoTy::PointerBitMask)
+      </Item>
+      <Item Name="[Holds]" Condition="((Val.Val.Value &gt;&gt; ValTy::ValTy::InfoTy::IntShift) &amp; 1) == 1 &amp;&amp; ((Val.Val.Value &gt;&gt; InnerUnion2::ValTy::InfoTy::IntShift) &amp; 1) == 1">
+        "$T4", s8b
+      </Item>
+      <Item Name="[Ptr]" Condition="((Val.Val.Value &gt;&gt; ValTy::ValTy::InfoTy::IntShift) &amp; 1) == 1 &amp;&amp; ((Val.Val.Value &gt;&gt; InnerUnion2::ValTy::InfoTy::IntShift) &amp; 1) == 1">
+        ($T4)(Val.Val.Value &amp; InnerUnion1::ValTy::InfoTy::PointerBitMask)
+      </Item>
     </Expand>
   </Type>
 
@@ -139,10 +191,10 @@ For later versions of Visual Studio, no setup is required.
 
   <Type Name="llvm::IntrusiveRefCntPtr&lt;*&gt;">
     <DisplayString Condition="Obj == 0">empty</DisplayString>
-    <DisplayString Condition="(Obj != 0) &amp;&amp; (Obj-&gt;ref_cnt == 1)">RefPtr [1 ref] {*Obj}</DisplayString>
-    <DisplayString Condition="(Obj != 0) &amp;&amp; (Obj-&gt;ref_cnt != 1)">RefPtr [{Obj-&gt;ref_cnt} refs] {*Obj}</DisplayString>
+    <DisplayString Condition="(Obj != 0) &amp;&amp; (Obj-&gt;RefCount == 1)">RefPtr [1 ref] {*Obj}</DisplayString>
+    <DisplayString Condition="(Obj != 0) &amp;&amp; (Obj-&gt;RefCount != 1)">RefPtr [{Obj-&gt;RefCount} refs] {*Obj}</DisplayString>
     <Expand>
-      <Item Condition="Obj != 0" Name="[refs]">Obj-&gt;ref_cnt</Item>
+      <Item Condition="Obj != 0" Name="[refs]">Obj-&gt;RefCount</Item>
       <ExpandedItem Condition="Obj != 0">Obj</ExpandedItem>
     </Expand>
   </Type>
diff --git a/utils/PerfectShuffle/PerfectShuffle.cpp b/utils/PerfectShuffle/PerfectShuffle.cpp
index 5c142a5f54e16f6541031ec2cc3e9ce633289e47..0f2e67298404b9fc6f5f652be2979cc342e3628c 100644
--- a/utils/PerfectShuffle/PerfectShuffle.cpp
+++ b/utils/PerfectShuffle/PerfectShuffle.cpp
@@ -1,9 +1,8 @@
 //===-- PerfectShuffle.cpp - Perfect Shuffle Generator --------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/utils/Reviewing/find_interesting_reviews.py b/utils/Reviewing/find_interesting_reviews.py
index 5af462b54a9fb7f237ea36e262946e6aa1331d93..7bfbec8cfde9e5459274fc25dc3140438ef921e5 100644
--- a/utils/Reviewing/find_interesting_reviews.py
+++ b/utils/Reviewing/find_interesting_reviews.py
@@ -1,5 +1,7 @@
 #!/usr/bin/env python
 
+from __future__ import print_function
+
 import argparse
 import email.mime.multipart
 import email.mime.text
@@ -50,10 +52,10 @@ class PhabObjectCache:
         return self.id2PhabObjects[id]
 
     def get_ids_in_cache(self):
-        return self.id2PhabObjects.keys()
+        return list(self.id2PhabObjects.keys())
 
     def get_objects(self):
-        return self.id2PhabObjects.values()
+        return list(self.id2PhabObjects.values())
 
     DEFAULT_DIRECTORY = "PhabObjectCache"
 
diff --git a/utils/TableGen/AsmMatcherEmitter.cpp b/utils/TableGen/AsmMatcherEmitter.cpp
index 5b4229e6468240335f49c45dfc8e9b59dfe11ebd..c14401699336de4fc68012cf12685852987a3136 100644
--- a/utils/TableGen/AsmMatcherEmitter.cpp
+++ b/utils/TableGen/AsmMatcherEmitter.cpp
@@ -1,9 +1,8 @@
 //===- AsmMatcherEmitter.cpp - Generate an assembly matcher ---------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -1160,8 +1159,9 @@ AsmMatcherInfo::getOperandClass(Record *Rec, int SubOpIdx) {
     // use it, else just fall back to the underlying register class.
     const RecordVal *R = Rec->getValue("ParserMatchClass");
     if (!R || !R->getValue())
-      PrintFatalError("Record `" + Rec->getName() +
-        "' does not have a ParserMatchClass!\n");
+      PrintFatalError(Rec->getLoc(),
+                      "Record `" + Rec->getName() +
+                          "' does not have a ParserMatchClass!\n");
 
     if (DefInit *DI= dyn_cast<DefInit>(R->getValue())) {
       Record *MatchClass = DI->getDef();
diff --git a/utils/TableGen/AsmWriterEmitter.cpp b/utils/TableGen/AsmWriterEmitter.cpp
index a8f191181766b35909fd0bb4d9708a8371c45a26..24e16f9c464266813d8b37341e9a6d088321c942 100644
--- a/utils/TableGen/AsmWriterEmitter.cpp
+++ b/utils/TableGen/AsmWriterEmitter.cpp
@@ -1,9 +1,8 @@
 //===- AsmWriterEmitter.cpp - Generate an assembly writer -----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/utils/TableGen/AsmWriterInst.cpp b/utils/TableGen/AsmWriterInst.cpp
index 2c19e5d663d66e81c0262018f9b74639a9a06257..c26e0e421183243dde204fb42ba665b103e8afe8 100644
--- a/utils/TableGen/AsmWriterInst.cpp
+++ b/utils/TableGen/AsmWriterInst.cpp
@@ -1,9 +1,8 @@
 //===- AsmWriterInst.h - Classes encapsulating a printable inst -----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -94,8 +93,10 @@ AsmWriterInst::AsmWriterInst(const CodeGenInstruction &CGI, unsigned CGIIndex,
                    != std::string::npos) {
           AddLiteralString(std::string(1, AsmString[DollarPos+1]));
         } else {
-          PrintFatalError("Non-supported escaped character found in instruction '" +
-            CGI.TheDef->getName() + "'!");
+          PrintFatalError(
+              CGI.TheDef->getLoc(),
+              "Non-supported escaped character found in instruction '" +
+                  CGI.TheDef->getName() + "'!");
         }
         LastEmitted = DollarPos+2;
         continue;
@@ -132,15 +133,19 @@ AsmWriterInst::AsmWriterInst(const CodeGenInstruction &CGI, unsigned CGIIndex,
       // brace.
       if (hasCurlyBraces) {
         if (VarEnd >= AsmString.size())
-          PrintFatalError("Reached end of string before terminating curly brace in '"
-            + CGI.TheDef->getName() + "'");
+          PrintFatalError(
+              CGI.TheDef->getLoc(),
+              "Reached end of string before terminating curly brace in '" +
+                  CGI.TheDef->getName() + "'");
 
         // Look for a modifier string.
         if (AsmString[VarEnd] == ':') {
           ++VarEnd;
           if (VarEnd >= AsmString.size())
-            PrintFatalError("Reached end of string before terminating curly brace in '"
-              + CGI.TheDef->getName() + "'");
+            PrintFatalError(
+                CGI.TheDef->getLoc(),
+                "Reached end of string before terminating curly brace in '" +
+                    CGI.TheDef->getName() + "'");
 
           std::string::size_type ModifierStart = VarEnd;
           while (VarEnd < AsmString.size() && isIdentChar(AsmString[VarEnd]))
@@ -148,17 +153,22 @@ AsmWriterInst::AsmWriterInst(const CodeGenInstruction &CGI, unsigned CGIIndex,
           Modifier = std::string(AsmString.begin()+ModifierStart,
                                  AsmString.begin()+VarEnd);
           if (Modifier.empty())
-            PrintFatalError("Bad operand modifier name in '"+ CGI.TheDef->getName() + "'");
+            PrintFatalError(CGI.TheDef->getLoc(),
+                            "Bad operand modifier name in '" +
+                                CGI.TheDef->getName() + "'");
         }
 
         if (AsmString[VarEnd] != '}')
-          PrintFatalError("Variable name beginning with '{' did not end with '}' in '"
-            + CGI.TheDef->getName() + "'");
+          PrintFatalError(
+              CGI.TheDef->getLoc(),
+              "Variable name beginning with '{' did not end with '}' in '" +
+                  CGI.TheDef->getName() + "'");
         ++VarEnd;
       }
       if (VarName.empty() && Modifier.empty())
-        PrintFatalError("Stray '$' in '" + CGI.TheDef->getName() +
-          "' asm string, maybe you want $$?");
+        PrintFatalError(CGI.TheDef->getLoc(),
+                        "Stray '$' in '" + CGI.TheDef->getName() +
+                            "' asm string, maybe you want $$?");
 
       if (VarName.empty()) {
         // Just a modifier, pass this into PrintSpecial.
diff --git a/utils/TableGen/AsmWriterInst.h b/utils/TableGen/AsmWriterInst.h
index 708f23cb5b0ef7b453be07ab053183071d46a550..7d88e5a9d0378a7d4f8ad2fe968ebef98892066d 100644
--- a/utils/TableGen/AsmWriterInst.h
+++ b/utils/TableGen/AsmWriterInst.h
@@ -1,9 +1,8 @@
 //===- AsmWriterInst.h - Classes encapsulating a printable inst -*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/utils/TableGen/Attributes.cpp b/utils/TableGen/Attributes.cpp
index 6bfc0ab896f9175b58ec6c4a4844d7d2a41043bf..6fbc595d730021e99255735bdcdbca5efac459bd 100644
--- a/utils/TableGen/Attributes.cpp
+++ b/utils/TableGen/Attributes.cpp
@@ -1,9 +1,8 @@
 //===- Attributes.cpp - Generate attributes -------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/utils/TableGen/CTagsEmitter.cpp b/utils/TableGen/CTagsEmitter.cpp
index bd596bcb47a8ba9c8ecafb58a32dcdb143fb939e..ccb7f3300dde4b86bd88cc633db588b513ea097a 100644
--- a/utils/TableGen/CTagsEmitter.cpp
+++ b/utils/TableGen/CTagsEmitter.cpp
@@ -1,9 +1,8 @@
 //===- CTagsEmitter.cpp - Generate ctags-compatible index ------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/utils/TableGen/CallingConvEmitter.cpp b/utils/TableGen/CallingConvEmitter.cpp
index d452031f8850764d231cbbf5821d53671ce31d8e..de5044e24d4998f0f754fd262414fae48672adb5 100644
--- a/utils/TableGen/CallingConvEmitter.cpp
+++ b/utils/TableGen/CallingConvEmitter.cpp
@@ -1,9 +1,8 @@
 //===- CallingConvEmitter.cpp - Generate calling conventions --------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -41,11 +40,17 @@ void CallingConvEmitter::run(raw_ostream &O) {
   // each other.
   for (Record *CC : CCs) {
     if (!CC->getValueAsBit("Custom")) {
-      O << "static bool " << CC->getName()
-        << "(unsigned ValNo, MVT ValVT,\n"
-        << std::string(CC->getName().size() + 13, ' ')
-        << "MVT LocVT, CCValAssign::LocInfo LocInfo,\n"
-        << std::string(CC->getName().size() + 13, ' ')
+      unsigned Pad = CC->getName().size();
+      if (CC->getValueAsBit("Entry")) {
+        O << "bool llvm::";
+        Pad += 12;
+      } else {
+        O << "static bool ";
+        Pad += 13;
+      }
+      O << CC->getName() << "(unsigned ValNo, MVT ValVT,\n"
+        << std::string(Pad, ' ') << "MVT LocVT, CCValAssign::LocInfo LocInfo,\n"
+        << std::string(Pad, ' ')
         << "ISD::ArgFlagsTy ArgFlags, CCState &State);\n";
     }
   }
@@ -62,12 +67,18 @@ void CallingConvEmitter::EmitCallingConv(Record *CC, raw_ostream &O) {
   ListInit *CCActions = CC->getValueAsListInit("Actions");
   Counter = 0;
 
-  O << "\n\nstatic bool " << CC->getName()
-    << "(unsigned ValNo, MVT ValVT,\n"
-    << std::string(CC->getName().size()+13, ' ')
-    << "MVT LocVT, CCValAssign::LocInfo LocInfo,\n"
-    << std::string(CC->getName().size()+13, ' ')
-    << "ISD::ArgFlagsTy ArgFlags, CCState &State) {\n";
+  O << "\n\n";
+  unsigned Pad = CC->getName().size();
+  if (CC->getValueAsBit("Entry")) {
+    O << "bool llvm::";
+    Pad += 12;
+  } else {
+    O << "static bool ";
+    Pad += 13;
+  }
+  O << CC->getName() << "(unsigned ValNo, MVT ValVT,\n"
+    << std::string(Pad, ' ') << "MVT LocVT, CCValAssign::LocInfo LocInfo,\n"
+    << std::string(Pad, ' ') << "ISD::ArgFlagsTy ArgFlags, CCState &State) {\n";
   // Emit all of the actions, in order.
   for (unsigned i = 0, e = CCActions->size(); i != e; ++i) {
     O << "\n";
@@ -97,7 +108,7 @@ void CallingConvEmitter::EmitAction(Record *Action,
       O << Action->getValueAsString("Predicate");
     } else {
       errs() << *Action;
-      PrintFatalError("Unknown CCPredicateAction!");
+      PrintFatalError(Action->getLoc(), "Unknown CCPredicateAction!");
     }
     
     O << ") {\n";
@@ -134,7 +145,8 @@ void CallingConvEmitter::EmitAction(Record *Action,
       ListInit *RegList = Action->getValueAsListInit("RegList");
       ListInit *ShadowRegList = Action->getValueAsListInit("ShadowRegList");
       if (!ShadowRegList->empty() && ShadowRegList->size() != RegList->size())
-        PrintFatalError("Invalid length of list of shadowed registers");
+        PrintFatalError(Action->getLoc(),
+                        "Invalid length of list of shadowed registers");
 
       if (RegList->size() == 1) {
         O << IndentStr << "if (unsigned Reg = State.AllocateReg(";
@@ -237,7 +249,8 @@ void CallingConvEmitter::EmitAction(Record *Action,
       MVT::SimpleValueType DestVT = getValueType(DestTy);
       O << IndentStr << "LocVT = " << getEnumName(DestVT) << ";\n";
       if (MVT(DestVT).isFloatingPoint()) {
-        PrintFatalError("CCPromoteToUpperBitsInType does not handle floating "
+        PrintFatalError(Action->getLoc(),
+                        "CCPromoteToUpperBitsInType does not handle floating "
                         "point");
       } else {
         O << IndentStr << "if (ArgFlags.isSExt())\n"
@@ -269,7 +282,7 @@ void CallingConvEmitter::EmitAction(Record *Action,
       O << IndentStr << IndentStr << "return false;\n";
     } else {
       errs() << *Action;
-      PrintFatalError("Unknown CCAction!");
+      PrintFatalError(Action->getLoc(), "Unknown CCAction!");
     }
   }
 }
diff --git a/utils/TableGen/CodeEmitterGen.cpp b/utils/TableGen/CodeEmitterGen.cpp
index 23751a2cbfba09cf0d26c24ba4ff8e34cbca91a7..ec16be14c6a739505e69f966dbc474581a2782cd 100644
--- a/utils/TableGen/CodeEmitterGen.cpp
+++ b/utils/TableGen/CodeEmitterGen.cpp
@@ -1,9 +1,8 @@
 //===- CodeEmitterGen.cpp - Code Emitter Generator ------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/utils/TableGen/CodeGenDAGPatterns.cpp b/utils/TableGen/CodeGenDAGPatterns.cpp
index 96c90c9cf6bd84a979d38d946af35a3492bddea3..f231c3a4aa78ebe4af30cd9fae5e23c368e4f79b 100644
--- a/utils/TableGen/CodeGenDAGPatterns.cpp
+++ b/utils/TableGen/CodeGenDAGPatterns.cpp
@@ -1,9 +1,8 @@
 //===- CodeGenDAGPatterns.cpp - Read DAG patterns from .td file -----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -1408,7 +1407,8 @@ SDTypeConstraint::SDTypeConstraint(Record *R, const CodeGenHwModes &CGH) {
     x.SDTCisSameSizeAs_Info.OtherOperandNum =
       R->getValueAsInt("OtherOperandNum");
   } else {
-    PrintFatalError("Unrecognized SDTypeConstraint '" + R->getName() + "'!\n");
+    PrintFatalError(R->getLoc(),
+                    "Unrecognized SDTypeConstraint '" + R->getName() + "'!\n");
   }
 }
 
diff --git a/utils/TableGen/CodeGenDAGPatterns.h b/utils/TableGen/CodeGenDAGPatterns.h
index 4be9afdcacd22587a39f22259aadbfd82e3fba76..24b58b1d9495784469cf908b6d0ace8b89ee8157 100644
--- a/utils/TableGen/CodeGenDAGPatterns.h
+++ b/utils/TableGen/CodeGenDAGPatterns.h
@@ -1,9 +1,8 @@
 //===- CodeGenDAGPatterns.h - Read DAG patterns from .td file ---*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/utils/TableGen/CodeGenHwModes.cpp b/utils/TableGen/CodeGenHwModes.cpp
index 9f88d95275b451a118e7f997f4272dd2f8f10b2f..9052cdd2bd3e4de952f6dba4b45e90c15ad3aa4e 100644
--- a/utils/TableGen/CodeGenHwModes.cpp
+++ b/utils/TableGen/CodeGenHwModes.cpp
@@ -1,9 +1,8 @@
 //===--- CodeGenHwModes.cpp -----------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 // Classes to parse and store HW mode information for instruction selection
diff --git a/utils/TableGen/CodeGenHwModes.h b/utils/TableGen/CodeGenHwModes.h
index 36df835d1933971f5a4ee32b62cca2a89633a51e..1ff2faaa0e52453fc23f6bbc0757e7a1a95bbec9 100644
--- a/utils/TableGen/CodeGenHwModes.h
+++ b/utils/TableGen/CodeGenHwModes.h
@@ -1,9 +1,8 @@
 //===--- CodeGenHwModes.h ---------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 // Classes to parse and store HW mode information for instruction selection.
diff --git a/utils/TableGen/CodeGenInstruction.cpp b/utils/TableGen/CodeGenInstruction.cpp
index 6d06ba2c8b6768e6a1b651d3dfa9964776916143..96c2c7d56b7b522e07f04e4805dd6b1a51b2de51 100644
--- a/utils/TableGen/CodeGenInstruction.cpp
+++ b/utils/TableGen/CodeGenInstruction.cpp
@@ -1,9 +1,8 @@
 //===- CodeGenInstruction.cpp - CodeGen Instruction Class Wrapper ---------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -34,18 +33,24 @@ CGIOperandList::CGIOperandList(Record *R) : TheDef(R) {
 
   if (DefInit *Init = dyn_cast<DefInit>(OutDI->getOperator())) {
     if (Init->getDef()->getName() != "outs")
-      PrintFatalError(R->getName() + ": invalid def name for output list: use 'outs'");
+      PrintFatalError(R->getLoc(),
+                      R->getName() +
+                          ": invalid def name for output list: use 'outs'");
   } else
-    PrintFatalError(R->getName() + ": invalid output list: use 'outs'");
+    PrintFatalError(R->getLoc(),
+                    R->getName() + ": invalid output list: use 'outs'");
 
   NumDefs = OutDI->getNumArgs();
 
   DagInit *InDI = R->getValueAsDag("InOperandList");
   if (DefInit *Init = dyn_cast<DefInit>(InDI->getOperator())) {
     if (Init->getDef()->getName() != "ins")
-      PrintFatalError(R->getName() + ": invalid def name for input list: use 'ins'");
+      PrintFatalError(R->getLoc(),
+                      R->getName() +
+                          ": invalid def name for input list: use 'ins'");
   } else
-    PrintFatalError(R->getName() + ": invalid input list: use 'ins'");
+    PrintFatalError(R->getLoc(),
+                    R->getName() + ": invalid input list: use 'ins'");
 
   unsigned MIOperandNo = 0;
   std::set<std::string> OperandNames;
@@ -64,7 +69,8 @@ CGIOperandList::CGIOperandList(Record *R) : TheDef(R) {
 
     DefInit *Arg = dyn_cast<DefInit>(ArgInit);
     if (!Arg)
-      PrintFatalError("Illegal operand for the '" + R->getName() + "' instruction!");
+      PrintFatalError(R->getLoc(), "Illegal operand for the '" + R->getName() +
+                                       "' instruction!");
 
     Record *Rec = Arg->getDef();
     std::string PrintMethod = "printOperand";
@@ -89,8 +95,9 @@ CGIOperandList::CGIOperandList(Record *R) : TheDef(R) {
       // Verify that MIOpInfo has an 'ops' root value.
       if (!isa<DefInit>(MIOpInfo->getOperator()) ||
           cast<DefInit>(MIOpInfo->getOperator())->getDef()->getName() != "ops")
-        PrintFatalError("Bad value for MIOperandInfo in operand '" + Rec->getName() +
-          "'\n");
+        PrintFatalError(R->getLoc(),
+                        "Bad value for MIOperandInfo in operand '" +
+                            Rec->getName() + "'\n");
 
       // If we have MIOpInfo, then we have #operands equal to number of entries
       // in MIOperandInfo.
@@ -108,16 +115,20 @@ CGIOperandList::CGIOperandList(Record *R) : TheDef(R) {
       OperandType = "OPERAND_REGISTER";
     } else if (!Rec->isSubClassOf("PointerLikeRegClass") &&
                !Rec->isSubClassOf("unknown_class"))
-      PrintFatalError("Unknown operand class '" + Rec->getName() +
-        "' in '" + R->getName() + "' instruction!");
+      PrintFatalError(R->getLoc(), "Unknown operand class '" + Rec->getName() +
+                                       "' in '" + R->getName() +
+                                       "' instruction!");
 
     // Check that the operand has a name and that it's unique.
     if (ArgName.empty())
-      PrintFatalError("In instruction '" + R->getName() + "', operand #" +
-                      Twine(i) + " has no name!");
+      PrintFatalError(R->getLoc(), "In instruction '" + R->getName() +
+                                       "', operand #" + Twine(i) +
+                                       " has no name!");
     if (!OperandNames.insert(ArgName).second)
-      PrintFatalError("In instruction '" + R->getName() + "', operand #" +
-                      Twine(i) + " has the same name as a previous operand!");
+      PrintFatalError(R->getLoc(),
+                      "In instruction '" + R->getName() + "', operand #" +
+                          Twine(i) +
+                          " has the same name as a previous operand!");
 
     OperandList.emplace_back(Rec, ArgName, PrintMethod, EncoderMethod,
                              OperandNamespace + "::" + OperandType, MIOperandNo,
@@ -139,9 +150,11 @@ CGIOperandList::CGIOperandList(Record *R) : TheDef(R) {
 ///
 unsigned CGIOperandList::getOperandNamed(StringRef Name) const {
   unsigned OpIdx;
-  if (hasOperandNamed(Name, OpIdx)) return OpIdx;
-  PrintFatalError("'" + TheDef->getName() +
-                  "' does not have an operand named '$" + Name + "'!");
+  if (hasOperandNamed(Name, OpIdx))
+    return OpIdx;
+  PrintFatalError(TheDef->getLoc(), "'" + TheDef->getName() +
+                                        "' does not have an operand named '$" +
+                                        Name + "'!");
 }
 
 /// hasOperandNamed - Query whether the instruction has an operand of the
@@ -160,7 +173,8 @@ bool CGIOperandList::hasOperandNamed(StringRef Name, unsigned &OpIdx) const {
 std::pair<unsigned,unsigned>
 CGIOperandList::ParseOperandName(const std::string &Op, bool AllowWholeOp) {
   if (Op.empty() || Op[0] != '$')
-    PrintFatalError(TheDef->getName() + ": Illegal operand name: '" + Op + "'");
+    PrintFatalError(TheDef->getLoc(),
+                    TheDef->getName() + ": Illegal operand name: '" + Op + "'");
 
   std::string OpName = Op.substr(1);
   std::string SubOpName;
@@ -170,7 +184,9 @@ CGIOperandList::ParseOperandName(const std::string &Op, bool AllowWholeOp) {
   if (DotIdx != std::string::npos) {
     SubOpName = OpName.substr(DotIdx+1);
     if (SubOpName.empty())
-      PrintFatalError(TheDef->getName() + ": illegal empty suboperand name in '" +Op +"'");
+      PrintFatalError(TheDef->getLoc(),
+                      TheDef->getName() +
+                          ": illegal empty suboperand name in '" + Op + "'");
     OpName = OpName.substr(0, DotIdx);
   }
 
@@ -180,8 +196,11 @@ CGIOperandList::ParseOperandName(const std::string &Op, bool AllowWholeOp) {
     // If one was needed, throw.
     if (OperandList[OpIdx].MINumOperands > 1 && !AllowWholeOp &&
         SubOpName.empty())
-      PrintFatalError(TheDef->getName() + ": Illegal to refer to"
-        " whole operand part of complex operand '" + Op + "'");
+      PrintFatalError(TheDef->getLoc(),
+                      TheDef->getName() +
+                          ": Illegal to refer to"
+                          " whole operand part of complex operand '" +
+                          Op + "'");
 
     // Otherwise, return the operand.
     return std::make_pair(OpIdx, 0U);
@@ -190,7 +209,9 @@ CGIOperandList::ParseOperandName(const std::string &Op, bool AllowWholeOp) {
   // Find the suboperand number involved.
   DagInit *MIOpInfo = OperandList[OpIdx].MIOperandInfo;
   if (!MIOpInfo)
-    PrintFatalError(TheDef->getName() + ": unknown suboperand name in '" + Op + "'");
+    PrintFatalError(TheDef->getLoc(), TheDef->getName() +
+                                          ": unknown suboperand name in '" +
+                                          Op + "'");
 
   // Find the operand with the right name.
   for (unsigned i = 0, e = MIOpInfo->getNumArgs(); i != e; ++i)
@@ -198,7 +219,9 @@ CGIOperandList::ParseOperandName(const std::string &Op, bool AllowWholeOp) {
       return std::make_pair(OpIdx, i);
 
   // Otherwise, didn't find it!
-  PrintFatalError(TheDef->getName() + ": unknown suboperand name in '" + Op + "'");
+  PrintFatalError(TheDef->getLoc(), TheDef->getName() +
+                                        ": unknown suboperand name in '" + Op +
+                                        "'");
   return std::make_pair(0U, 0U);
 }
 
diff --git a/utils/TableGen/CodeGenInstruction.h b/utils/TableGen/CodeGenInstruction.h
index 2e3d2f48a928f648fd27d5e847e8bb1bd78d6df3..28056a98056e3c2af0a82da2d6f4fc1b6e925513 100644
--- a/utils/TableGen/CodeGenInstruction.h
+++ b/utils/TableGen/CodeGenInstruction.h
@@ -1,9 +1,8 @@
 //===- CodeGenInstruction.h - Instruction Class Wrapper ---------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/utils/TableGen/CodeGenIntrinsics.h b/utils/TableGen/CodeGenIntrinsics.h
index 9487a79c143238d20853564d4f562b4eb78bb7b0..31bdc01959a62229ae34005f2084c9f9c2ce61a4 100644
--- a/utils/TableGen/CodeGenIntrinsics.h
+++ b/utils/TableGen/CodeGenIntrinsics.h
@@ -1,9 +1,8 @@
 //===- CodeGenIntrinsic.h - Intrinsic Class Wrapper ------------*- C++ -*--===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/utils/TableGen/CodeGenMapTable.cpp b/utils/TableGen/CodeGenMapTable.cpp
index e5b0426cdcc3dcbd2718d512b780d2d1e5f935c6..b1774b01ba8cba05d6e984f708ef545801ffcbb3 100644
--- a/utils/TableGen/CodeGenMapTable.cpp
+++ b/utils/TableGen/CodeGenMapTable.cpp
@@ -1,9 +1,8 @@
 //===- CodeGenMapTable.cpp - Instruction Mapping Table Generator ----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 // CodeGenMapTable provides functionality for the TabelGen to create
diff --git a/utils/TableGen/CodeGenRegisters.cpp b/utils/TableGen/CodeGenRegisters.cpp
index 74a2b078dfb321261f3fc89b267afe64b3e6a2bd..7d8efbf8b967fe2dc36412212cc92d2ea75aafc9 100644
--- a/utils/TableGen/CodeGenRegisters.cpp
+++ b/utils/TableGen/CodeGenRegisters.cpp
@@ -1,9 +1,8 @@
 //===- CodeGenRegisters.cpp - Register and RegisterClass Info -------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -740,8 +739,9 @@ CodeGenRegisterClass::CodeGenRegisterClass(CodeGenRegBank &RegBank, Record *R)
   for (unsigned i = 0, e = TypeList.size(); i != e; ++i) {
     Record *Type = TypeList[i];
     if (!Type->isSubClassOf("ValueType"))
-      PrintFatalError("RegTypes list member '" + Type->getName() +
-        "' does not derive from the ValueType class!");
+      PrintFatalError(R->getLoc(),
+                      "RegTypes list member '" + Type->getName() +
+                          "' does not derive from the ValueType class!");
     VTs.push_back(getValueTypeByHwMode(Type, RegBank.getHwModes()));
   }
   assert(!VTs.empty() && "RegisterClass must contain at least one ValueType!");
diff --git a/utils/TableGen/CodeGenRegisters.h b/utils/TableGen/CodeGenRegisters.h
index 0f7a025ded108cec57dc3c4f0fa42ad118053dd6..f04a90f8fde52309d39d5c67fb1e58d47dc5d3e1 100644
--- a/utils/TableGen/CodeGenRegisters.h
+++ b/utils/TableGen/CodeGenRegisters.h
@@ -1,9 +1,8 @@
 //===- CodeGenRegisters.h - Register and RegisterClass Info -----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/utils/TableGen/CodeGenSchedule.cpp b/utils/TableGen/CodeGenSchedule.cpp
index 6d259cbb33ee3e2908cfa612a881ccf6c5802c71..4abe12273004595c5c87d79f844f124b18cd3c7b 100644
--- a/utils/TableGen/CodeGenSchedule.cpp
+++ b/utils/TableGen/CodeGenSchedule.cpp
@@ -1,9 +1,8 @@
 //===- CodeGenSchedule.cpp - Scheduling MachineModels ---------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -1936,8 +1935,9 @@ void CodeGenSchedModels::checkCompleteness() {
       unsigned SCIdx = getSchedClassIdx(*Inst);
       if (!SCIdx) {
         if (Inst->TheDef->isValueUnset("SchedRW") && !HadCompleteModel) {
-          PrintError("No schedule information for instruction '"
-                     + Inst->TheDef->getName() + "'");
+          PrintError(Inst->TheDef->getLoc(),
+                     "No schedule information for instruction '" +
+                         Inst->TheDef->getName() + "'");
           Complete = false;
         }
         continue;
@@ -1955,8 +1955,9 @@ void CodeGenSchedModels::checkCompleteness() {
         return R->getValueAsDef("SchedModel") == ProcModel.ModelDef;
       });
       if (I == InstRWs.end()) {
-        PrintError("'" + ProcModel.ModelName + "' lacks information for '" +
-                   Inst->TheDef->getName() + "'");
+        PrintError(Inst->TheDef->getLoc(), "'" + ProcModel.ModelName +
+                                               "' lacks information for '" +
+                                               Inst->TheDef->getName() + "'");
         Complete = false;
       }
     }
diff --git a/utils/TableGen/CodeGenSchedule.h b/utils/TableGen/CodeGenSchedule.h
index 87a051b0c05ec6cc6bc77540e1820ddc98ce8ef6..c26fb1f978072136f830310f70f9632dcb63cae0 100644
--- a/utils/TableGen/CodeGenSchedule.h
+++ b/utils/TableGen/CodeGenSchedule.h
@@ -1,9 +1,8 @@
 //===- CodeGenSchedule.h - Scheduling Machine Models ------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/utils/TableGen/CodeGenTarget.cpp b/utils/TableGen/CodeGenTarget.cpp
index bcb653135551f7321aa561403a5686b5de092721..5a9bc60ad295bd1ff6bfb75c87961fbaa466ac59 100644
--- a/utils/TableGen/CodeGenTarget.cpp
+++ b/utils/TableGen/CodeGenTarget.cpp
@@ -1,9 +1,8 @@
 //===- CodeGenTarget.cpp - CodeGen Target Class Wrapper -------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -21,8 +20,10 @@
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Timer.h"
 #include "llvm/TableGen/Error.h"
 #include "llvm/TableGen/Record.h"
+#include "llvm/TableGen/TableGenBackend.h"
 #include <algorithm>
 using namespace llvm;
 
@@ -327,6 +328,8 @@ CodeGenSchedModels &CodeGenTarget::getSchedModels() const {
 }
 
 void CodeGenTarget::ReadInstructions() const {
+  NamedRegionTimer T("Read Instructions", "Time spent reading instructions",
+                     "CodeGenTarget", "CodeGenTarget", TimeRegions);
   std::vector<Record*> Insts = Records.getAllDerivedDefinitions("Instruction");
   if (Insts.size() <= 2)
     PrintFatalError("No 'Instruction' subclasses defined!");
@@ -492,9 +495,10 @@ ComplexPattern::ComplexPattern(Record *R) {
     } else if (PropList[i]->getName() == "SDNPWantParent") {
       Properties |= 1 << SDNPWantParent;
     } else {
-      PrintFatalError("Unsupported SD Node property '" +
-                      PropList[i]->getName() + "' on ComplexPattern '" +
-                      R->getName() + "'!");
+      PrintFatalError(R->getLoc(), "Unsupported SD Node property '" +
+                                       PropList[i]->getName() +
+                                       "' on ComplexPattern '" + R->getName() +
+                                       "'!");
     }
 }
 
@@ -530,6 +534,7 @@ CodeGenIntrinsicTable::CodeGenIntrinsicTable(const RecordKeeper &RC,
 CodeGenIntrinsic::CodeGenIntrinsic(Record *R) {
   TheDef = R;
   std::string DefName = R->getName();
+  ArrayRef<SMLoc> DefLoc = R->getLoc();
   ModRef = ReadWriteMem;
   Properties = 0;
   isOverloaded = false;
@@ -544,7 +549,8 @@ CodeGenIntrinsic::CodeGenIntrinsic(Record *R) {
 
   if (DefName.size() <= 4 ||
       std::string(DefName.begin(), DefName.begin() + 4) != "int_")
-    PrintFatalError("Intrinsic '" + DefName + "' does not start with 'int_'!");
+    PrintFatalError(DefLoc,
+                    "Intrinsic '" + DefName + "' does not start with 'int_'!");
 
   EnumName = std::string(DefName.begin()+4, DefName.end());
 
@@ -566,7 +572,8 @@ CodeGenIntrinsic::CodeGenIntrinsic(Record *R) {
     // Verify it starts with "llvm.".
     if (Name.size() <= 5 ||
         std::string(Name.begin(), Name.begin() + 5) != "llvm.")
-      PrintFatalError("Intrinsic '" + DefName + "'s name does not start with 'llvm.'!");
+      PrintFatalError(DefLoc, "Intrinsic '" + DefName +
+                                  "'s name does not start with 'llvm.'!");
   }
 
   // If TargetPrefix is specified, make sure that Name starts with
@@ -575,8 +582,9 @@ CodeGenIntrinsic::CodeGenIntrinsic(Record *R) {
     if (Name.size() < 6+TargetPrefix.size() ||
         std::string(Name.begin() + 5, Name.begin() + 6 + TargetPrefix.size())
         != (TargetPrefix + "."))
-      PrintFatalError("Intrinsic '" + DefName + "' does not start with 'llvm." +
-        TargetPrefix + ".'!");
+      PrintFatalError(DefLoc, "Intrinsic '" + DefName +
+                                  "' does not start with 'llvm." +
+                                  TargetPrefix + ".'!");
   }
 
   // Parse the list of return types.
@@ -608,7 +616,8 @@ CodeGenIntrinsic::CodeGenIntrinsic(Record *R) {
 
     // Reject invalid types.
     if (VT == MVT::isVoid)
-      PrintFatalError("Intrinsic '" + DefName + " has void in result type list!");
+      PrintFatalError(DefLoc, "Intrinsic '" + DefName +
+                                  " has void in result type list!");
 
     IS.RetVTs.push_back(VT);
     IS.RetTypeDefs.push_back(TyEl);
@@ -626,7 +635,8 @@ CodeGenIntrinsic::CodeGenIntrinsic(Record *R) {
         PrintError(R->getLoc(),
                    "Parameter #" + Twine(i) + " has out of bounds matching "
                    "number " + Twine(MatchTy));
-        PrintFatalError(Twine("ParamTypes is ") + TypeList->getAsString());
+        PrintFatalError(DefLoc,
+                        Twine("ParamTypes is ") + TypeList->getAsString());
       }
       VT = OverloadedVTs[MatchTy];
       // It only makes sense to use the extended and truncated vector element
@@ -634,7 +644,7 @@ CodeGenIntrinsic::CodeGenIntrinsic(Record *R) {
       // overloaded, all the types can be specified directly.
       assert(((!TyEl->isSubClassOf("LLVMExtendedType") &&
                !TyEl->isSubClassOf("LLVMTruncatedType") &&
-               !TyEl->isSubClassOf("LLVMVectorSameWidth")) ||
+               !TyEl->isSubClassOf("LLVMScalarOrSameVectorWidth")) ||
               VT == MVT::iAny || VT == MVT::vAny) &&
              "Expected iAny or vAny type");
     } else
@@ -647,7 +657,8 @@ CodeGenIntrinsic::CodeGenIntrinsic(Record *R) {
 
     // Reject invalid types.
     if (VT == MVT::isVoid && i != e-1 /*void at end means varargs*/)
-      PrintFatalError("Intrinsic '" + DefName + " has void in result type list!");
+      PrintFatalError(DefLoc, "Intrinsic '" + DefName +
+                                  " has void in result type list!");
 
     IS.ParamVTs.push_back(VT);
     IS.ParamTypeDefs.push_back(TyEl);
diff --git a/utils/TableGen/CodeGenTarget.h b/utils/TableGen/CodeGenTarget.h
index d2833d5b6a920251d42fcdaf7a37d8345e9b9c62..1ab2de269c761f7116ba9a8ff1cca802fc497b86 100644
--- a/utils/TableGen/CodeGenTarget.h
+++ b/utils/TableGen/CodeGenTarget.h
@@ -1,9 +1,8 @@
 //===- CodeGenTarget.h - Target Class Wrapper -------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/utils/TableGen/DAGISelEmitter.cpp b/utils/TableGen/DAGISelEmitter.cpp
index 62a0ff7007256c64f010f10a5af6a1f9f7d85950..fb0c6faa52957cc5173b9df22458b96b3d835ff8 100644
--- a/utils/TableGen/DAGISelEmitter.cpp
+++ b/utils/TableGen/DAGISelEmitter.cpp
@@ -1,9 +1,8 @@
 //===- DAGISelEmitter.cpp - Generate an instruction selector --------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/utils/TableGen/DAGISelMatcher.cpp b/utils/TableGen/DAGISelMatcher.cpp
index c8e005739460c56e74d7bab81a5d3fa24e1f9952..121618de916553451884d55b7f6d6b320cd6cabe 100644
--- a/utils/TableGen/DAGISelMatcher.cpp
+++ b/utils/TableGen/DAGISelMatcher.cpp
@@ -1,9 +1,8 @@
 //===- DAGISelMatcher.cpp - Representation of DAG pattern matcher ---------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/utils/TableGen/DAGISelMatcher.h b/utils/TableGen/DAGISelMatcher.h
index 9be7295c67d4a99fb6473706d7abd8e6a185c549..63efef485d84299c2613613eec825b069d09175e 100644
--- a/utils/TableGen/DAGISelMatcher.h
+++ b/utils/TableGen/DAGISelMatcher.h
@@ -1,9 +1,8 @@
 //===- DAGISelMatcher.h - Representation of DAG pattern matcher -*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/utils/TableGen/DAGISelMatcherEmitter.cpp b/utils/TableGen/DAGISelMatcherEmitter.cpp
index 90ca1bff53443f90529c44e27180ebf85edc1a2b..e83e773122924a56b4385bac30f168b3983e4dba 100644
--- a/utils/TableGen/DAGISelMatcherEmitter.cpp
+++ b/utils/TableGen/DAGISelMatcherEmitter.cpp
@@ -1,9 +1,8 @@
 //===- DAGISelMatcherEmitter.cpp - Matcher Emitter ------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -259,7 +258,7 @@ void MatcherTableEmitter::EmitPatternMatchTable(raw_ostream &OS) {
 
   OS << "\n};";
   OS << "\nreturn StringRef(PATTERN_MATCH_TABLE[Index]);";
-  OS << "\n}";
+  OS << "\n}\n";
   EndEmitFunction(OS);
 
   BeginEmitFunction(OS, "StringRef", "getIncludePathForIndex(unsigned Index)",
@@ -273,7 +272,7 @@ void MatcherTableEmitter::EmitPatternMatchTable(raw_ostream &OS) {
 
   OS << "\n};";
   OS << "\nreturn StringRef(INCLUDE_PATH_TABLE[Index]);";
-  OS << "\n}";
+  OS << "\n}\n";
   EndEmitFunction(OS);
 }
 
diff --git a/utils/TableGen/DAGISelMatcherGen.cpp b/utils/TableGen/DAGISelMatcherGen.cpp
index 612342ddcddf14b41ba7936efea972752fb8d6fa..1a0490507de343a485a1a4fca6cbc117b228ae64 100644
--- a/utils/TableGen/DAGISelMatcherGen.cpp
+++ b/utils/TableGen/DAGISelMatcherGen.cpp
@@ -1,9 +1,8 @@
 //===- DAGISelMatcherGen.cpp - Matcher generator --------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/utils/TableGen/DAGISelMatcherOpt.cpp b/utils/TableGen/DAGISelMatcherOpt.cpp
index 554c7438ce3d3abe7c14e09d440133e75fcbf851..b44f49c7ef621b26858af283239fa2b8860052fb 100644
--- a/utils/TableGen/DAGISelMatcherOpt.cpp
+++ b/utils/TableGen/DAGISelMatcherOpt.cpp
@@ -1,9 +1,8 @@
 //===- DAGISelMatcherOpt.cpp - Optimize a DAG Matcher ---------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/utils/TableGen/DFAPacketizerEmitter.cpp b/utils/TableGen/DFAPacketizerEmitter.cpp
index 0db0f55f5ed6d9c63e1e076d896b5a7546ffbc27..dabcc8f8ed552b302b68748374bde358588350f7 100644
--- a/utils/TableGen/DFAPacketizerEmitter.cpp
+++ b/utils/TableGen/DFAPacketizerEmitter.cpp
@@ -1,9 +1,8 @@
 //===- DFAPacketizerEmitter.cpp - Packetization DFA for a VLIW machine ----===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/utils/TableGen/DisassemblerEmitter.cpp b/utils/TableGen/DisassemblerEmitter.cpp
index b99a0a973a2ce02b219fc5c2a21aaa7586a6a1f1..9e75c7fba77b83df6ce767392fcbdff2a6cff789 100644
--- a/utils/TableGen/DisassemblerEmitter.cpp
+++ b/utils/TableGen/DisassemblerEmitter.cpp
@@ -1,9 +1,8 @@
 //===- DisassemblerEmitter.cpp - Generate a disassembler ------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/utils/TableGen/ExegesisEmitter.cpp b/utils/TableGen/ExegesisEmitter.cpp
index 208237aca20c8e36339cf4543700e60e7ee6e924..976d5f51776f64bf8db1e165a4e79edd85793f36 100644
--- a/utils/TableGen/ExegesisEmitter.cpp
+++ b/utils/TableGen/ExegesisEmitter.cpp
@@ -1,9 +1,8 @@
 //===- ExegesisEmitter.cpp - Generate exegesis target data ----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/utils/TableGen/FastISelEmitter.cpp b/utils/TableGen/FastISelEmitter.cpp
index 5134b684c6f9cdb5bfa5e458fa30b28d7bf3d685..b39956859fe8a6e28c72091bfedacf8f2176c59b 100644
--- a/utils/TableGen/FastISelEmitter.cpp
+++ b/utils/TableGen/FastISelEmitter.cpp
@@ -1,9 +1,8 @@
 ///===- FastISelEmitter.cpp - Generate an instruction selector -------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/utils/TableGen/FixedLenDecoderEmitter.cpp b/utils/TableGen/FixedLenDecoderEmitter.cpp
index 1206f7751a9ea203d9a030bc18a8262d8621bd37..ea28e06cc6e8c8b0560ab1406efdf84b1f8bffca 100644
--- a/utils/TableGen/FixedLenDecoderEmitter.cpp
+++ b/utils/TableGen/FixedLenDecoderEmitter.cpp
@@ -1,9 +1,8 @@
 //===------------ FixedLenDecoderEmitter.cpp - Decoder Generator ----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -1717,7 +1716,7 @@ void FilterChooser::emitTableEntries(DecoderTableInfo &TableInfo) const {
   dumpStack(errs(), "\t\t");
 
   for (unsigned i = 0; i < Opcodes.size(); ++i) {
-    errs() << '\t' << Opcodes[i] << " ";
+    errs() << '\t' << AllInstructions[Opcodes[i]] << " ";
     dumpBits(errs(),
              getBitsField(*AllInstructions[Opcodes[i]].EncodingDef, "Inst"));
     errs() << '\n';
diff --git a/utils/TableGen/GlobalISelEmitter.cpp b/utils/TableGen/GlobalISelEmitter.cpp
index 997ceb12becd053960fde0efcf912e03c69fde4d..8079e2989b6d9cb22322fc8f649d9bae43ecfc76 100644
--- a/utils/TableGen/GlobalISelEmitter.cpp
+++ b/utils/TableGen/GlobalISelEmitter.cpp
@@ -1,9 +1,8 @@
 //===- GlobalISelEmitter.cpp - Generate an instruction selector -----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -882,12 +881,19 @@ public:
 
   void defineOperand(StringRef SymbolicName, OperandMatcher &OM);
 
-  void defineComplexSubOperand(StringRef SymbolicName, Record *ComplexPattern,
-                               unsigned RendererID, unsigned SubOperandID) {
-    assert(ComplexSubOperands.count(SymbolicName) == 0 && "Already defined");
+  Error defineComplexSubOperand(StringRef SymbolicName, Record *ComplexPattern,
+                                unsigned RendererID, unsigned SubOperandID) {
+    if (ComplexSubOperands.count(SymbolicName))
+      return failedImport(
+          "Complex suboperand referenced more than once (Operand: " +
+          SymbolicName + ")");
+
     ComplexSubOperands[SymbolicName] =
         std::make_tuple(ComplexPattern, RendererID, SubOperandID);
+
+    return Error::success();
   }
+
   Optional<DefinedComplexPatternSubOperand>
   getComplexSubOperand(StringRef SymbolicName) const {
     const auto &I = ComplexSubOperands.find(SymbolicName);
@@ -3422,9 +3428,12 @@ Error GlobalISelEmitter::importChildMatcher(RuleMatcher &Rule,
 
       for (unsigned i = 0, e = SrcChild->getNumChildren(); i != e; ++i) {
         auto *SubOperand = SrcChild->getChild(i);
-        if (!SubOperand->getName().empty())
-          Rule.defineComplexSubOperand(SubOperand->getName(),
-                                       SrcChild->getOperator(), RendererID, i);
+        if (!SubOperand->getName().empty()) {
+          if (auto Error = Rule.defineComplexSubOperand(SubOperand->getName(),
+                                                        SrcChild->getOperator(),
+                                                        RendererID, i))
+            return Error;
+        }
       }
 
       return Error::success();
diff --git a/utils/TableGen/InfoByHwMode.cpp b/utils/TableGen/InfoByHwMode.cpp
index 086e12dafd74e73b303015aab194a3c28f21db07..ce1b78a8f17d94ab1cb11159eb1f0b22bee558f9 100644
--- a/utils/TableGen/InfoByHwMode.cpp
+++ b/utils/TableGen/InfoByHwMode.cpp
@@ -1,9 +1,8 @@
 //===--- InfoByHwMode.cpp -------------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 // Classes that implement data parameterized by HW modes for instruction
diff --git a/utils/TableGen/InfoByHwMode.h b/utils/TableGen/InfoByHwMode.h
index 7be4678f271bdc62d1d0ede6be46276da68bfccb..6871d61028c351fc072800c31b4a78bd8edea85b 100644
--- a/utils/TableGen/InfoByHwMode.h
+++ b/utils/TableGen/InfoByHwMode.h
@@ -1,9 +1,8 @@
 //===--- InfoByHwMode.h -----------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 // Classes that implement data parameterized by HW modes for instruction
diff --git a/utils/TableGen/InstrDocsEmitter.cpp b/utils/TableGen/InstrDocsEmitter.cpp
index 9d50351854ec46fe36bba6b2c1925b80ebf3cbf8..91c457ba08fd6d3ea6ffcdb5171592c5b01811b8 100644
--- a/utils/TableGen/InstrDocsEmitter.cpp
+++ b/utils/TableGen/InstrDocsEmitter.cpp
@@ -1,9 +1,8 @@
 //===- InstrDocsEmitter.cpp - Opcode Documentation Generator --------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/utils/TableGen/InstrInfoEmitter.cpp b/utils/TableGen/InstrInfoEmitter.cpp
index 39d9e8526386e0977117918f5d70f1a247a1618d..4c0b53dc49a3c5705d47c2a8de5e2374f7f4890e 100644
--- a/utils/TableGen/InstrInfoEmitter.cpp
+++ b/utils/TableGen/InstrInfoEmitter.cpp
@@ -1,9 +1,8 @@
 //===- InstrInfoEmitter.cpp - Generate a Instruction Set Desc. --*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -629,13 +628,14 @@ void InstrInfoEmitter::emitRecord(const CodeGenInstruction &Inst, unsigned Num,
   // Emit all of the target-specific flags...
   BitsInit *TSF = Inst.TheDef->getValueAsBitsInit("TSFlags");
   if (!TSF)
-    PrintFatalError("no TSFlags?");
+    PrintFatalError(Inst.TheDef->getLoc(), "no TSFlags?");
   uint64_t Value = 0;
   for (unsigned i = 0, e = TSF->getNumBits(); i != e; ++i) {
     if (const auto *Bit = dyn_cast<BitInit>(TSF->getBit(i)))
       Value |= uint64_t(Bit->getValue()) << i;
     else
-      PrintFatalError("Invalid TSFlags bit in " + Inst.TheDef->getName());
+      PrintFatalError(Inst.TheDef->getLoc(),
+                      "Invalid TSFlags bit in " + Inst.TheDef->getName());
   }
   OS << ", 0x";
   OS.write_hex(Value);
diff --git a/utils/TableGen/IntrinsicEmitter.cpp b/utils/TableGen/IntrinsicEmitter.cpp
index 049282e5ebfeed8d44beeb3e888703a83c24f18a..19c0266cc3869ea110d2c51359d967362b73e2e1 100644
--- a/utils/TableGen/IntrinsicEmitter.cpp
+++ b/utils/TableGen/IntrinsicEmitter.cpp
@@ -1,9 +1,8 @@
 //===- IntrinsicEmitter.cpp - Generate intrinsic information --------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -270,7 +269,7 @@ static void EncodeFixedType(Record *R, std::vector<unsigned char> &ArgCodes,
       Sig.push_back(IIT_TRUNC_ARG);
     else if (R->isSubClassOf("LLVMHalfElementsVectorType"))
       Sig.push_back(IIT_HALF_VEC_ARG);
-    else if (R->isSubClassOf("LLVMVectorSameWidth")) {
+    else if (R->isSubClassOf("LLVMScalarOrSameVectorWidth")) {
       Sig.push_back(IIT_SAME_VEC_WIDTH_ARG);
       Sig.push_back((Number << 3) | ArgCodes[Number]);
       MVT::SimpleValueType VT = getValueType(R->getValueAsDef("ElTy"));
@@ -771,8 +770,9 @@ void IntrinsicEmitter::EmitIntrinsicToBuiltinMap(
           BuiltinMap[Ints[i].TargetPrefix];
 
       if (!BIM.insert(std::make_pair(BuiltinName, Ints[i].EnumName)).second)
-        PrintFatalError("Intrinsic '" + Ints[i].TheDef->getName() +
-                        "': duplicate " + CompilerName + " builtin name!");
+        PrintFatalError(Ints[i].TheDef->getLoc(),
+                        "Intrinsic '" + Ints[i].TheDef->getName() +
+                            "': duplicate " + CompilerName + " builtin name!");
       Table.GetOrAddStringOffset(BuiltinName);
     }
   }
diff --git a/utils/TableGen/LLVMBuild.txt b/utils/TableGen/LLVMBuild.txt
index 66387cfa0d1a829a2735cca725bf702bcbc12850..5eec4e060be58401fa2a73865afda0fab5039fef 100644
--- a/utils/TableGen/LLVMBuild.txt
+++ b/utils/TableGen/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./utils/TableGen/LLVMBuild.txt ---------------------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/utils/TableGen/OptParserEmitter.cpp b/utils/TableGen/OptParserEmitter.cpp
index 0358cf26509b83b0145c7d56e91c88764a4b04f9..51b1cb093b213b8799fc6b7a6feec6473b31fc6e 100644
--- a/utils/TableGen/OptParserEmitter.cpp
+++ b/utils/TableGen/OptParserEmitter.cpp
@@ -1,9 +1,8 @@
 //===- OptParserEmitter.cpp - Table Driven Command Line Parsing -----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/utils/TableGen/PredicateExpander.cpp b/utils/TableGen/PredicateExpander.cpp
index 2e01b7c3138e534a0ee2d99b99dec0170aacfa35..9f7f40db2626cd7fc0eafbb24ccbab2eeca1486b 100644
--- a/utils/TableGen/PredicateExpander.cpp
+++ b/utils/TableGen/PredicateExpander.cpp
@@ -1,9 +1,8 @@
 //===--------------------- PredicateExpander.cpp --------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/utils/TableGen/PredicateExpander.h b/utils/TableGen/PredicateExpander.h
index 0f3ee6867e658880e07ec28c2fc4a3f3fa3d15c3..115a81cf123bcbc42581f431d0e95313699f6642 100644
--- a/utils/TableGen/PredicateExpander.h
+++ b/utils/TableGen/PredicateExpander.h
@@ -1,9 +1,8 @@
 //===--------------------- PredicateExpander.h ----------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
diff --git a/utils/TableGen/PseudoLoweringEmitter.cpp b/utils/TableGen/PseudoLoweringEmitter.cpp
index a363015730f315edc65cb231f30dcdf0df6457c1..3a80d8e5d1c47d9b0951ab4e8582925206992d4b 100644
--- a/utils/TableGen/PseudoLoweringEmitter.cpp
+++ b/utils/TableGen/PseudoLoweringEmitter.cpp
@@ -1,9 +1,8 @@
 //===- PseudoLoweringEmitter.cpp - PseudoLowering Generator -----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/utils/TableGen/RISCVCompressInstEmitter.cpp b/utils/TableGen/RISCVCompressInstEmitter.cpp
index e03663b40f8a421965b98c7fd379a3079655516f..f95b3bf9cb48b59f705bdd855718b28904d982fb 100644
--- a/utils/TableGen/RISCVCompressInstEmitter.cpp
+++ b/utils/TableGen/RISCVCompressInstEmitter.cpp
@@ -1,9 +1,8 @@
 //===- RISCVCompressInstEmitter.cpp - Generator for RISCV Compression -===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 // RISCVCompressInstEmitter implements a tablegen-driven CompressPat based
 // RISCV Instruction Compression mechanism.
@@ -253,12 +252,14 @@ static bool verifyDagOpCount(CodeGenInstruction &Inst, DagInit *Dag,
   // Source instructions are non compressed instructions and don't have tied
   // operands.
   if (IsSource)
-    PrintFatalError("Input operands for Inst '" + Inst.TheDef->getName() +
-                    "' and input Dag operand count mismatch");
+    PrintFatalError(Inst.TheDef->getLoc(),
+                    "Input operands for Inst '" + Inst.TheDef->getName() +
+                        "' and input Dag operand count mismatch");
   // The Dag can't have more arguments than the Instruction.
   if (Dag->getNumArgs() > Inst.Operands.size())
-    PrintFatalError("Inst '" + Inst.TheDef->getName() +
-                    "' and Dag operand count mismatch");
+    PrintFatalError(Inst.TheDef->getLoc(),
+                    "Inst '" + Inst.TheDef->getName() +
+                        "' and Dag operand count mismatch");
 
   // The Instruction might have tied operands so the Dag might have
   //  a fewer operand count.
@@ -268,8 +269,9 @@ static bool verifyDagOpCount(CodeGenInstruction &Inst, DagInit *Dag,
       --RealCount;
 
   if (Dag->getNumArgs() != RealCount)
-    PrintFatalError("Inst '" + Inst.TheDef->getName() +
-                    "' and Dag operand count mismatch");
+    PrintFatalError(Inst.TheDef->getLoc(),
+                    "Inst '" + Inst.TheDef->getName() +
+                        "' and Dag operand count mismatch");
   return true;
 }
 
@@ -530,7 +532,8 @@ void RISCVCompressInstEmitter::emitCompressInstEmitter(raw_ostream &o,
                                                        bool Compress) {
   Record *AsmWriter = Target.getAsmWriter();
   if (!AsmWriter->getValueAsInt("PassSubtarget"))
-    PrintFatalError("'PassSubtarget' is false. SubTargetInfo object is needed "
+    PrintFatalError(AsmWriter->getLoc(),
+                    "'PassSubtarget' is false. SubTargetInfo object is needed "
                     "for target features.\n");
 
   std::string Namespace = Target.getName();
diff --git a/utils/TableGen/RegisterBankEmitter.cpp b/utils/TableGen/RegisterBankEmitter.cpp
index 879b4162d629ead352026fc8f35834e20025a2ba..7f6b3931d3dea850dfbd7e41c05f4be91f0fa279 100644
--- a/utils/TableGen/RegisterBankEmitter.cpp
+++ b/utils/TableGen/RegisterBankEmitter.cpp
@@ -1,9 +1,8 @@
 //===- RegisterBankEmitter.cpp - Generate a Register Bank Desc. -*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/utils/TableGen/RegisterInfoEmitter.cpp b/utils/TableGen/RegisterInfoEmitter.cpp
index ded54c828bcd8a062c54bb9ff35ca92bbed664f8..1b619072c8141ccd55372906183b76d004cd1716 100644
--- a/utils/TableGen/RegisterInfoEmitter.cpp
+++ b/utils/TableGen/RegisterInfoEmitter.cpp
@@ -1,9 +1,8 @@
 //===- RegisterInfoEmitter.cpp - Generate a Register File Desc. -*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/utils/TableGen/SDNodeProperties.cpp b/utils/TableGen/SDNodeProperties.cpp
index 343febc99d1e684602553493bd5216e221f5187a..1843a78aa3cf0b7254540d27b0eb439a8831443b 100644
--- a/utils/TableGen/SDNodeProperties.cpp
+++ b/utils/TableGen/SDNodeProperties.cpp
@@ -1,9 +1,8 @@
 //===- SDNodeProperties.cpp -----------------------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
@@ -39,9 +38,9 @@ unsigned llvm::parseSDPatternOperatorProperties(Record *R) {
     } else if (Property->getName() == "SDNPVariadic") {
       Properties |= 1 << SDNPVariadic;
     } else {
-      PrintFatalError("Unknown SD Node property '" +
-                      Property->getName() + "' on node '" +
-                      R->getName() + "'!");
+      PrintFatalError(R->getLoc(), "Unknown SD Node property '" +
+                                       Property->getName() + "' on node '" +
+                                       R->getName() + "'!");
     }
   }
 
diff --git a/utils/TableGen/SDNodeProperties.h b/utils/TableGen/SDNodeProperties.h
index a8d4efb5dab049ee60a579ce2c4663b1e03d23ab..66a04e63150c40e0d41f7be9b0277eefe5b69fcb 100644
--- a/utils/TableGen/SDNodeProperties.h
+++ b/utils/TableGen/SDNodeProperties.h
@@ -1,9 +1,8 @@
 //===- SDNodeProperties.h ---------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/utils/TableGen/SearchableTableEmitter.cpp b/utils/TableGen/SearchableTableEmitter.cpp
index f98a7c74bf0c28720ac186cdc39e82471c8b0b24..b847ef30958a8cf3a2950b21eb9f22a86ab00b53 100644
--- a/utils/TableGen/SearchableTableEmitter.cpp
+++ b/utils/TableGen/SearchableTableEmitter.cpp
@@ -1,9 +1,8 @@
 //===- SearchableTableEmitter.cpp - Generate efficiently searchable tables -==//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -600,9 +599,10 @@ void SearchableTableEmitter::collectTableEntries(
     for (auto &Field : Table.Fields) {
       auto TI = dyn_cast<TypedInit>(EntryRec->getValueInit(Field.Name));
       if (!TI) {
-        PrintFatalError(Twine("Record '") + EntryRec->getName() +
-                        "' in table '" + Table.Name + "' is missing field '" +
-                        Field.Name + "'");
+        PrintFatalError(EntryRec->getLoc(),
+                        Twine("Record '") + EntryRec->getName() +
+                            "' in table '" + Table.Name +
+                            "' is missing field '" + Field.Name + "'");
       }
       if (!Field.RecType) {
         Field.RecType = TI->getType();
@@ -654,8 +654,8 @@ void SearchableTableEmitter::run(raw_ostream &OS) {
     StringRef FilterClass = EnumRec->getValueAsString("FilterClass");
     Enum->Class = Records.getClass(FilterClass);
     if (!Enum->Class)
-      PrintFatalError(Twine("Enum FilterClass '") + FilterClass +
-                      "' does not exist");
+      PrintFatalError(EnumRec->getLoc(), Twine("Enum FilterClass '") +
+                                             FilterClass + "' does not exist");
 
     collectEnumEntries(*Enum, NameField, ValueField,
                        Records.getAllDerivedDefinitions(FilterClass));
@@ -675,9 +675,10 @@ void SearchableTableEmitter::run(raw_ostream &OS) {
 
       if (auto TypeOfVal = TableRec->getValue(("TypeOf_" + FieldName).str())) {
         if (!parseFieldType(Table->Fields.back(), TypeOfVal->getValue())) {
-          PrintFatalError(Twine("Table '") + Table->Name +
-                          "' has bad 'TypeOf_" + FieldName + "': " +
-                          TypeOfVal->getValue()->getAsString());
+          PrintFatalError(TableRec->getLoc(),
+                          Twine("Table '") + Table->Name +
+                              "' has bad 'TypeOf_" + FieldName +
+                              "': " + TypeOfVal->getValue()->getAsString());
         }
       }
     }
@@ -705,8 +706,10 @@ void SearchableTableEmitter::run(raw_ostream &OS) {
     Record *TableRec = IndexRec->getValueAsDef("Table");
     auto It = TableMap.find(TableRec);
     if (It == TableMap.end())
-      PrintFatalError(Twine("SearchIndex '") + IndexRec->getName() +
-                      "' refers to non-existing table '" + TableRec->getName());
+      PrintFatalError(IndexRec->getLoc(),
+                      Twine("SearchIndex '") + IndexRec->getName() +
+                          "' refers to non-existing table '" +
+                          TableRec->getName());
 
     GenericTable &Table = *It->second;
     Table.Indices.push_back(parseSearchIndex(
diff --git a/utils/TableGen/SequenceToOffsetTable.h b/utils/TableGen/SequenceToOffsetTable.h
index 2b8f66a3bf3e81e72df1869815f18ec5ed396825..8a826eff311d4360605fe8b20233e3b2f69da2fd 100644
--- a/utils/TableGen/SequenceToOffsetTable.h
+++ b/utils/TableGen/SequenceToOffsetTable.h
@@ -1,9 +1,8 @@
 //===-- SequenceToOffsetTable.h - Compress similar sequences ----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/utils/TableGen/SubtargetEmitter.cpp b/utils/TableGen/SubtargetEmitter.cpp
index 731c14bdb9a0d1445c4566d07148d4b39401f69f..792c957ea19c2fb4a82ee71c13b1396f0a148823 100644
--- a/utils/TableGen/SubtargetEmitter.cpp
+++ b/utils/TableGen/SubtargetEmitter.cpp
@@ -1,9 +1,8 @@
 //===- SubtargetEmitter.cpp - Generate subtarget enumerations -------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/utils/TableGen/SubtargetFeatureInfo.cpp b/utils/TableGen/SubtargetFeatureInfo.cpp
index f9b8853cc117dacbeefaf264e10485a2f9821114..b7ed23fff877de4160a2df1eda72e63ea5b769b2 100644
--- a/utils/TableGen/SubtargetFeatureInfo.cpp
+++ b/utils/TableGen/SubtargetFeatureInfo.cpp
@@ -1,9 +1,8 @@
 //===- SubtargetFeatureInfo.cpp - Helpers for subtarget features ----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/utils/TableGen/SubtargetFeatureInfo.h b/utils/TableGen/SubtargetFeatureInfo.h
index 71e6748c863f4a5989c019a6090895ba33239346..17e142e8264fbb65288f2afe08bd3153620449e6 100644
--- a/utils/TableGen/SubtargetFeatureInfo.h
+++ b/utils/TableGen/SubtargetFeatureInfo.h
@@ -1,9 +1,8 @@
 //===- SubtargetFeatureInfo.h - Helpers for subtarget features ------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/utils/TableGen/TableGen.cpp b/utils/TableGen/TableGen.cpp
index d5b6a3c12647e49e2562796702310eb91ef91c4e..c485ed2feb7afd7eba66038558294f04ad37e3af 100644
--- a/utils/TableGen/TableGen.cpp
+++ b/utils/TableGen/TableGen.cpp
@@ -1,9 +1,8 @@
 //===- TableGen.cpp - Top-Level TableGen implementation for LLVM ----------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -56,6 +55,12 @@ enum ActionType {
   GenExegesis,
 };
 
+namespace llvm {
+/// Storage for TimeRegionsOpt as a global so that backends aren't required to
+/// include CommandLine.h
+bool TimeRegions = false;
+} // end namespace llvm
+
 namespace {
   cl::opt<ActionType>
   Action(cl::desc("Action to perform:"),
@@ -127,6 +132,11 @@ namespace {
   Class("class", cl::desc("Print Enum list for this class"),
         cl::value_desc("class name"), cl::cat(PrintEnumsCat));
 
+cl::opt<bool, true>
+    TimeRegionsOpt("time-regions",
+                   cl::desc("Time regions of tablegens execution"),
+                   cl::location(TimeRegions));
+
 bool LLVMTableGenMain(raw_ostream &OS, RecordKeeper &Records) {
   switch (Action) {
   case PrintRecords:
diff --git a/utils/TableGen/TableGenBackends.h b/utils/TableGen/TableGenBackends.h
index f4f2909f8e888d89eae2c2f0a5a620e1651c52cf..135ec65c0f954c740d69bc73679c36d886a66e2a 100644
--- a/utils/TableGen/TableGenBackends.h
+++ b/utils/TableGen/TableGenBackends.h
@@ -1,9 +1,8 @@
 //===- TableGenBackends.h - Declarations for LLVM TableGen Backends -------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/utils/TableGen/Types.cpp b/utils/TableGen/Types.cpp
index 04d9e40f6743dabfe7cf5a0625ec86ca20832f0d..a6682da90e6b3afe17a101eba560494bbe914eef 100644
--- a/utils/TableGen/Types.cpp
+++ b/utils/TableGen/Types.cpp
@@ -1,9 +1,8 @@
 //===- Types.cpp - Helper for the selection of C++ data types. ------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/utils/TableGen/Types.h b/utils/TableGen/Types.h
index d511b7eae6e8101c40044f8ebb4860e99da5b8f0..17c7742ccaacd677e97cd7b2a92cf50315b673f8 100644
--- a/utils/TableGen/Types.h
+++ b/utils/TableGen/Types.h
@@ -1,9 +1,8 @@
 //===- Types.h - Helper for the selection of C++ types. ---------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/utils/TableGen/WebAssemblyDisassemblerEmitter.cpp b/utils/TableGen/WebAssemblyDisassemblerEmitter.cpp
index ab785715d974ea3b5bc5559421291dd3192ba5d8..365cba5a60caba3e9cebdb066af2184bad620de4 100644
--- a/utils/TableGen/WebAssemblyDisassemblerEmitter.cpp
+++ b/utils/TableGen/WebAssemblyDisassemblerEmitter.cpp
@@ -1,9 +1,8 @@
 //===- WebAssemblyDisassemblerEmitter.cpp - Disassembler tables -*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -51,12 +50,33 @@ void emitWebAssemblyDisassemblerTables(
     auto IsStackBased =
         StackString &&
         reinterpret_cast<const StringInit *>(StackString)->getValue() == "true";
-    if (IsStackBased && !CGIP.second) {
-      // this picks the first of many typed variants, which is
-      // currently the except_ref one, though this shouldn't matter for
-      // disassembly purposes.
-      CGIP = std::make_pair(I, &CGI);
+    if (!IsStackBased)
+      continue;
+    if (CGIP.second) {
+      // We already have an instruction for this slot, so decide which one
+      // should be the canonical one. This determines which variant gets
+      // printed in a disassembly. We want e.g. "call" not "i32.call", and
+      // "end" when we don't know if its "end_loop" or "end_block" etc.
+      auto IsCanonicalExisting = CGIP.second->TheDef->getValue("IsCanonical")
+                                     ->getValue()
+                                     ->getAsString() == "1";
+      // We already have one marked explicitly as canonical, so keep it.
+      if (IsCanonicalExisting)
+        continue;
+      auto IsCanonicalNew =
+          Def.getValue("IsCanonical")->getValue()->getAsString() == "1";
+      // If the new one is explicitly marked as canonical, take it.
+      if (!IsCanonicalNew) {
+        // Neither the existing or new instruction is canonical.
+        // Pick the one with the shortest name as heuristic.
+        // Though ideally IsCanonical is always defined for at least one
+        // variant so this never has to apply.
+        if (CGIP.second->AsmString.size() <= CGI.AsmString.size())
+          continue;
+      }
     }
+    // Set this instruction as the one to use.
+    CGIP = std::make_pair(I, &CGI);
   }
   OS << "#include \"MCTargetDesc/WebAssemblyMCTargetDesc.h\"\n";
   OS << "\n";
@@ -92,6 +112,7 @@ void emitWebAssemblyDisassemblerTables(
         // Collect operand types for storage in a shared list.
         CurOperandList.clear();
         for (auto &Op : CGI.Operands.OperandList) {
+          assert(Op.OperandType != "MCOI::OPERAND_UNKNOWN");
           CurOperandList.push_back(Op.OperandType);
         }
         // See if we already have stored this sequence before. This is not
diff --git a/utils/TableGen/WebAssemblyDisassemblerEmitter.h b/utils/TableGen/WebAssemblyDisassemblerEmitter.h
index 91f820f120a2510fe565a9c62cbdf2a7230cbca8..60d3d9433eca14ff1a4b591149d65fe422d83e84 100644
--- a/utils/TableGen/WebAssemblyDisassemblerEmitter.h
+++ b/utils/TableGen/WebAssemblyDisassemblerEmitter.h
@@ -1,9 +1,8 @@
 //===- WebAssemblyDisassemblerEmitter.h - Disassembler tables ---*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/utils/TableGen/X86DisassemblerShared.h b/utils/TableGen/X86DisassemblerShared.h
index 220765f7241026ea14a632d4a166e49e80fafb79..093f220fda5e64198713eb81610eea6abf3036aa 100644
--- a/utils/TableGen/X86DisassemblerShared.h
+++ b/utils/TableGen/X86DisassemblerShared.h
@@ -1,9 +1,8 @@
 //===- X86DisassemblerShared.h - Emitter shared header ----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/utils/TableGen/X86DisassemblerTables.cpp b/utils/TableGen/X86DisassemblerTables.cpp
index 2b5cc127960595bd7edce31ed71be36327f95718..db8d623d0e630e4d6ac0c27d1e71fd0f1a7e28b1 100644
--- a/utils/TableGen/X86DisassemblerTables.cpp
+++ b/utils/TableGen/X86DisassemblerTables.cpp
@@ -1,9 +1,8 @@
 //===- X86DisassemblerTables.cpp - Disassembler tables ----------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/utils/TableGen/X86DisassemblerTables.h b/utils/TableGen/X86DisassemblerTables.h
index b0ea9c2e86252d07c7d964835ef774c2f3152b31..63af68b6fbfade1a3fc04c46413714a2a1d90783 100644
--- a/utils/TableGen/X86DisassemblerTables.h
+++ b/utils/TableGen/X86DisassemblerTables.h
@@ -1,9 +1,8 @@
 //===- X86DisassemblerTables.h - Disassembler tables ------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/utils/TableGen/X86EVEX2VEXTablesEmitter.cpp b/utils/TableGen/X86EVEX2VEXTablesEmitter.cpp
index d5dc10ecad25ade27941c9e00a3a5084453843c4..65c9aa0dd1ad8d549cbab1f548c9997193c1d584 100644
--- a/utils/TableGen/X86EVEX2VEXTablesEmitter.cpp
+++ b/utils/TableGen/X86EVEX2VEXTablesEmitter.cpp
@@ -1,9 +1,8 @@
 //===- utils/TableGen/X86EVEX2VEXTablesEmitter.cpp - X86 backend-*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 ///
diff --git a/utils/TableGen/X86FoldTablesEmitter.cpp b/utils/TableGen/X86FoldTablesEmitter.cpp
index 1ea6686435754d5037385b824bda167a821710dc..ab9ddc23967664b50b92426b0384727dc91c7bbd 100644
--- a/utils/TableGen/X86FoldTablesEmitter.cpp
+++ b/utils/TableGen/X86FoldTablesEmitter.cpp
@@ -1,9 +1,8 @@
 //===- utils/TableGen/X86FoldTablesEmitter.cpp - X86 backend-*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/utils/TableGen/X86ModRMFilters.cpp b/utils/TableGen/X86ModRMFilters.cpp
index 1641613aa32d579b693690002c04c3f5564f6811..98e6fb6104d77ba2e06dfc2d9596b5300abeea21 100644
--- a/utils/TableGen/X86ModRMFilters.cpp
+++ b/utils/TableGen/X86ModRMFilters.cpp
@@ -1,9 +1,8 @@
 //===- X86ModRMFilters.cpp - Disassembler ModR/M filterss -------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/utils/TableGen/X86ModRMFilters.h b/utils/TableGen/X86ModRMFilters.h
index b0248e878d07faec2098d8d2648b3b9c8b0e04e1..c77b4c21aec41e03c789403f056cbd97dfeb5b71 100644
--- a/utils/TableGen/X86ModRMFilters.h
+++ b/utils/TableGen/X86ModRMFilters.h
@@ -1,9 +1,8 @@
 //===- X86ModRMFilters.h - Disassembler ModR/M filterss ---------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/utils/TableGen/X86RecognizableInstr.cpp b/utils/TableGen/X86RecognizableInstr.cpp
index 2f9b428b8cfe8bddee7845811798fbd735f29fda..f7f7966075a7c4d9ea5828021ee098bd1c189080 100644
--- a/utils/TableGen/X86RecognizableInstr.cpp
+++ b/utils/TableGen/X86RecognizableInstr.cpp
@@ -1,9 +1,8 @@
 //===- X86RecognizableInstr.cpp - Disassembler instruction spec --*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
@@ -842,6 +841,7 @@ OperandType RecognizableInstr::typeFromString(const std::string &s,
   TYPE("f32mem",              TYPE_M)
   TYPE("ssmem",               TYPE_M)
   TYPE("RST",                 TYPE_ST)
+  TYPE("RSTi",                TYPE_ST)
   TYPE("i128mem",             TYPE_M)
   TYPE("i256mem",             TYPE_M)
   TYPE("i512mem",             TYPE_M)
@@ -964,6 +964,7 @@ OperandEncoding
 RecognizableInstr::rmRegisterEncodingFromString(const std::string &s,
                                                 uint8_t OpSize) {
   ENCODING("RST",             ENCODING_FP)
+  ENCODING("RSTi",            ENCODING_FP)
   ENCODING("GR16",            ENCODING_RM)
   ENCODING("GR32",            ENCODING_RM)
   ENCODING("GR32orGR64",      ENCODING_RM)
diff --git a/utils/TableGen/X86RecognizableInstr.h b/utils/TableGen/X86RecognizableInstr.h
index c4d34ee6c80c9b6b7567cb9a40f581d39c153ba3..7b703fcfca0b4685ab60fc29ee378c2d878cdbb0 100644
--- a/utils/TableGen/X86RecognizableInstr.h
+++ b/utils/TableGen/X86RecognizableInstr.h
@@ -1,9 +1,8 @@
 //===- X86RecognizableInstr.h - Disassembler instruction spec ----*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/utils/TableGen/tdtags b/utils/TableGen/tdtags
index 5214485f4ec1dae254baf203735398fbb7b6d381..a3fb0805e1818c5eca9feb6b49f6f0178205b349 100644
--- a/utils/TableGen/tdtags
+++ b/utils/TableGen/tdtags
@@ -3,10 +3,9 @@
 # vim:set sts=2 sw=2 et:
 #===----------------------------------------------------------------------===#
 #
-#                     The LLVM Compiler Infrastructure
-#
-# This file is distributed under the University of Illinois Open Source
-# License. See LICENSE.TXT for details.
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 #
 #===----------------------------------------------------------------------===#
 #
diff --git a/utils/Target/ARM/analyze-match-table.py b/utils/Target/ARM/analyze-match-table.py
index aa952d40085aa810e4772d66169221cdd39db437..d4e158d9e1d80347b47db044bcb7dbe017c88cd6 100644
--- a/utils/Target/ARM/analyze-match-table.py
+++ b/utils/Target/ARM/analyze-match-table.py
@@ -1,5 +1,7 @@
 #!/usr/bin/env python
 
+from __future__ import print_function
+
 def analyze_match_table(path):
     # Extract the instruction table.
     data = open(path).read()
@@ -37,10 +39,10 @@ def analyze_match_table(path):
     condcode_mnemonics = set(m for m in mnemonics
                              if 'MCK_CondCode' in mnemonic_flags[m])
     noncondcode_mnemonics = mnemonics - condcode_mnemonics
-    print ' || '.join('Mnemonic == "%s"' % m
-                      for m in ccout_mnemonics)
-    print ' || '.join('Mnemonic == "%s"' % m
-                      for m in noncondcode_mnemonics)
+    print(' || '.join('Mnemonic == "%s"' % m
+                      for m in ccout_mnemonics))
+    print(' || '.join('Mnemonic == "%s"' % m
+                      for m in noncondcode_mnemonics))
 
 def main():
     import sys
diff --git a/utils/count/count.c b/utils/count/count.c
index ae96791ae497361dd4059c507ad9e037b1001ba3..0aacafe2fe78e1fe8a503217a4ab3ec5dbc28d18 100644
--- a/utils/count/count.c
+++ b/utils/count/count.c
@@ -1,9 +1,8 @@
 /*===- count.c - The 'count' testing tool ---------------------------------===*\
  *
- *                     The LLVM Compiler Infrastructure
- *
- * This file is distributed under the University of Illinois Open Source
- * License. See LICENSE.TXT for details.
+ * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+ * See https://llvm.org/LICENSE.txt for license information.
+ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
  *
 \*===----------------------------------------------------------------------===*/
 
diff --git a/utils/countloc.sh b/utils/countloc.sh
index 4d1b775d74aad808ceb69d0542f12c58bed68b2a..497fab0a722ac2e610c731b18d1ea628f2a068cd 100755
--- a/utils/countloc.sh
+++ b/utils/countloc.sh
@@ -1,10 +1,9 @@
 #!/bin/sh
 ##===- utils/countloc.sh - Counts Lines Of Code --------------*- Script -*-===##
 # 
-#                     The LLVM Compiler Infrastructure
-#
-# This file is distributed under the University of Illinois Open Source
-# License. See LICENSE.TXT for details.
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 # 
 ##===----------------------------------------------------------------------===##
 #
diff --git a/utils/create_ladder_graph.py b/utils/create_ladder_graph.py
index d29e3ad3a10d1d64477b5d236562942eb5438c99..a5946ff24af5ad850ced4270ae7a731ab2ed4e3a 100644
--- a/utils/create_ladder_graph.py
+++ b/utils/create_ladder_graph.py
@@ -10,6 +10,8 @@ One good use of this program is to test whether your linear time algorithm is
 really behaving linearly.
 """
 
+from __future__ import print_function
+
 import argparse
 def main():
   parser = argparse.ArgumentParser(description=__doc__)
@@ -17,27 +19,27 @@ def main():
                       help="Number of ladder rungs. Must be a multiple of 2")
   args = parser.parse_args()
   if (args.rungs % 2) != 0:
-    print "Rungs must be a multiple of 2"
+    print("Rungs must be a multiple of 2")
     return
-  print "int ladder(int *foo, int *bar, int x) {"
-  rung1 = xrange(0, args.rungs, 2)
-  rung2 = xrange(1, args.rungs, 2)
+  print("int ladder(int *foo, int *bar, int x) {")
+  rung1 = range(0, args.rungs, 2)
+  rung2 = range(1, args.rungs, 2)
   for i in rung1:
-    print "rung1%d:" % i
-    print "*foo = x++;"
+    print("rung1%d:" % i)
+    print("*foo = x++;")
     if i != rung1[-1]:
-      print "if (*bar) goto rung1%d;" % (i+2)
-      print "else goto rung2%d;" % (i+1)
+      print("if (*bar) goto rung1%d;" % (i+2))
+      print("else goto rung2%d;" % (i+1))
     else:
-      print "goto rung2%d;" % (i+1)
+      print("goto rung2%d;" % (i+1))
   for i in rung2:
-    print "rung2%d:" % i
-    print "*foo = x++;"
+    print("rung2%d:" % i)
+    print("*foo = x++;")
     if i != rung2[-1]:
-      print "goto rung2%d;" % (i+2)
+      print("goto rung2%d;" % (i+2))
     else:
-      print "return *foo;"
-  print "}"
+      print("return *foo;")
+  print("}")
 
 if __name__ == '__main__':
   main()
diff --git a/utils/demangle_tree.py b/utils/demangle_tree.py
index 1185a233a419fa378df8f411bdcf1f94558c5be1..00de72b2a18d7158969c21bdf7d5ca62de1a7890 100644
--- a/utils/demangle_tree.py
+++ b/utils/demangle_tree.py
@@ -4,6 +4,8 @@
 # demanglings.  Useful for stress testing the demangler against a large corpus
 # of inputs.
 
+from __future__ import print_function
+
 import argparse
 import functools
 import os
diff --git a/utils/docker/build_docker_image.sh b/utils/docker/build_docker_image.sh
index 820a5f75ad132d6ea1bc1141ac7ccab55aeceab6..0911b2a0f4d35513f6a8cf17f760b78f32a2f1e7 100755
--- a/utils/docker/build_docker_image.sh
+++ b/utils/docker/build_docker_image.sh
@@ -1,10 +1,9 @@
 #!/bin/bash
 #===- llvm/utils/docker/build_docker_image.sh ----------------------------===//
 #
-#                     The LLVM Compiler Infrastructure
-#
-# This file is distributed under the University of Illinois Open Source
-# License. See LICENSE.TXT for details.
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 #
 #===----------------------------------------------------------------------===//
 set -e
diff --git a/utils/docker/debian8/Dockerfile b/utils/docker/debian8/Dockerfile
index fe12ec55e52ab4272478d784fb03512bdf34ceec..9d0e2b8d795e55dfa8214094789bbe6a4025f5f2 100644
--- a/utils/docker/debian8/Dockerfile
+++ b/utils/docker/debian8/Dockerfile
@@ -1,9 +1,8 @@
 #===- llvm/utils/docker/debian8/build/Dockerfile -------------------------===//
 #
-#                     The LLVM Compiler Infrastructure
-#
-# This file is distributed under the University of Illinois Open Source
-# License. See LICENSE.TXT for details.
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 #
 #===----------------------------------------------------------------------===//
 # Stage 1. Check out LLVM source code and run the build.
diff --git a/utils/docker/example/Dockerfile b/utils/docker/example/Dockerfile
index d875ed96d9070f17dd628a4e6cfaad2e42521adb..ebfb0e49c82c86e0f82b8c46b6083a4e4e338f61 100644
--- a/utils/docker/example/Dockerfile
+++ b/utils/docker/example/Dockerfile
@@ -1,9 +1,8 @@
 #===- llvm/utils/docker/example/build/Dockerfile -------------------------===//
 #
-#                     The LLVM Compiler Infrastructure
-#
-# This file is distributed under the University of Illinois Open Source
-# License. See LICENSE.TXT for details.
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 #
 #===----------------------------------------------------------------------===//
 # This is an example Dockerfile to build an image that compiles clang.
diff --git a/utils/docker/nvidia-cuda/Dockerfile b/utils/docker/nvidia-cuda/Dockerfile
index 32804d77ee9a8ba5204277e822fd655e7a046011..5878259c27f4e69085b43c6cbca1fa39919eeeb2 100644
--- a/utils/docker/nvidia-cuda/Dockerfile
+++ b/utils/docker/nvidia-cuda/Dockerfile
@@ -1,9 +1,8 @@
 #===- llvm/utils/docker/nvidia-cuda/build/Dockerfile ---------------------===//
 #
-#                     The LLVM Compiler Infrastructure
-#
-# This file is distributed under the University of Illinois Open Source
-# License. See LICENSE.TXT for details.
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 #
 #===----------------------------------------------------------------------===//
 # Stage 1. Check out LLVM source code and run the build.
diff --git a/utils/docker/scripts/build_install_llvm.sh b/utils/docker/scripts/build_install_llvm.sh
index 0b9c0837203965eab1a6112079a081cf61b89a58..3a52e41fa655e40e148a13952e1a268835de712d 100755
--- a/utils/docker/scripts/build_install_llvm.sh
+++ b/utils/docker/scripts/build_install_llvm.sh
@@ -1,10 +1,9 @@
 #!/usr/bin/env bash
 #===- llvm/utils/docker/scripts/build_install_llvm.sh ---------------------===//
 #
-#                     The LLVM Compiler Infrastructure
-#
-# This file is distributed under the University of Illinois Open Source
-# License. See LICENSE.TXT for details.
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 #
 #===-----------------------------------------------------------------------===//
 
diff --git a/utils/docker/scripts/checkout.sh b/utils/docker/scripts/checkout.sh
index 1bd034d184404a77695e231250f1bed94f5936cb..c5e7cc784fdc564a34ceb0b8ba3f00a758810da8 100755
--- a/utils/docker/scripts/checkout.sh
+++ b/utils/docker/scripts/checkout.sh
@@ -1,10 +1,9 @@
 #!/usr/bin/env bash
 #===- llvm/utils/docker/scripts/checkout.sh ---------------------===//
 #
-#                     The LLVM Compiler Infrastructure
-#
-# This file is distributed under the University of Illinois Open Source
-# License. See LICENSE.TXT for details.
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 #
 #===-----------------------------------------------------------------------===//
 
diff --git a/utils/docker/scripts/llvm_checksum/llvm_checksum.py b/utils/docker/scripts/llvm_checksum/llvm_checksum.py
index 584efa2598bfdab1e1af768940a0f923d66c11f6..f554d920104f0a92218f213e8b2c718b35b7cdff 100755
--- a/utils/docker/scripts/llvm_checksum/llvm_checksum.py
+++ b/utils/docker/scripts/llvm_checksum/llvm_checksum.py
@@ -183,7 +183,7 @@ def ValidateChecksums(reference_checksums,
     if len(new_checksums) != len(reference_checksums):
       return False
 
-  for proj, checksum in new_checksums.iteritems():
+  for proj, checksum in new_checksums.items():
     # We never computed a checksum for this project.
     if proj not in reference_checksums:
       return False
diff --git a/utils/extract_vplan.py b/utils/extract_vplan.py
index ac0055d2e798ad0e57b0298d8eb25c0206fc03f1..b4e34fc76ba5e7967ef9af27051974bcf79e6355 100755
--- a/utils/extract_vplan.py
+++ b/utils/extract_vplan.py
@@ -4,6 +4,8 @@
 # and saves them in individual dot files (one for each plan). Optionally, and
 # providing 'dot' is installed, it can also render the dot into a PNG file.
 
+from __future__ import print_function
+
 import sys
 import re
 import argparse
diff --git a/utils/fpcmp/fpcmp.cpp b/utils/fpcmp/fpcmp.cpp
index bbe7276e13a0d428895e7d381758c864afdc81d6..52bddeedc7597068b08f762fd01b7dd3aad89b4b 100644
--- a/utils/fpcmp/fpcmp.cpp
+++ b/utils/fpcmp/fpcmp.cpp
@@ -1,9 +1,8 @@
 //===- fpcmp.cpp - A fuzzy "cmp" that permits floating point noise --------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
diff --git a/utils/gdb-scripts/prettyprinters.py b/utils/gdb-scripts/prettyprinters.py
index 918411db42fbb703e023649dfd2cd2f11584ce04..7ddc33acb20c8f57cf6db8d89e7337502cdd4259 100644
--- a/utils/gdb-scripts/prettyprinters.py
+++ b/utils/gdb-scripts/prettyprinters.py
@@ -1,12 +1,15 @@
+from __future__ import print_function
+import sys
+
 import gdb.printing
 
 class Iterator:
   def __iter__(self):
     return self
 
-  # Python 2 compatibility
-  def next(self):
-    return self.__next__()
+  if sys.version_info.major == 2:
+      def next(self):
+        return self.__next__()
 
   def children(self):
     return self
@@ -68,7 +71,7 @@ class ArrayRefPrinter:
     def __iter__(self):
       return self
 
-    def next(self):
+    def __next__(self):
       if self.cur == self.end:
         raise StopIteration
       count = self.count
@@ -77,13 +80,12 @@ class ArrayRefPrinter:
       self.cur = self.cur + 1
       return '[%d]' % count, cur.dereference()
 
-    __next__ = next
+    if sys.version_info.major == 2:
+        next = __next__
 
   def __init__(self, val):
     self.val = val
 
-    __next__ = next
-
   def children(self):
     data = self.val['Data']
     return self._iterator(data, data + self.val['Length'])
@@ -167,7 +169,7 @@ class DenseMapPrinter:
       while self.cur != self.end and (is_equal(self.cur.dereference()['first'], empty) or is_equal(self.cur.dereference()['first'], tombstone)):
         self.cur = self.cur + 1
 
-    def next(self):
+    def __next__(self):
       if self.cur == self.end:
         raise StopIteration
       cur = self.cur
@@ -180,7 +182,8 @@ class DenseMapPrinter:
         self.first = False
       return 'x', v
 
-    __next__ = next
+    if sys.version_info.major == 2:
+        next = __next__
 
   def __init__(self, val):
     self.val = val
diff --git a/utils/getsrcs.sh b/utils/getsrcs.sh
index c8bff8cf833d8b8dcf29af6360b535674d724aea..ef0ad726774064b5d08475ee1a5b3e4f9a19571a 100755
--- a/utils/getsrcs.sh
+++ b/utils/getsrcs.sh
@@ -1,10 +1,9 @@
 #!/bin/sh
 ##===- utils/getsrcs.sh - Counts Lines Of Code ---------------*- Script -*-===##
 # 
-#                     The LLVM Compiler Infrastructure
-#
-# This file is distributed under the University of Illinois Open Source
-# License. See LICENSE.TXT for details.
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 # details.
 # 
 ##===----------------------------------------------------------------------===##
diff --git a/utils/git-svn/git-llvm b/utils/git-svn/git-llvm
index 53c0b24ae2c8568f68c0d5a227cf04badd436305..4b17aca6e95be515fe13403450b6447d76a60249 100755
--- a/utils/git-svn/git-llvm
+++ b/utils/git-svn/git-llvm
@@ -2,10 +2,9 @@
 #
 # ======- git-llvm - LLVM Git Help Integration ---------*- python -*--========#
 #
-#                     The LLVM Compiler Infrastructure
-#
-# This file is distributed under the University of Illinois Open Source
-# License. See LICENSE.TXT for details.
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 #
 # ==------------------------------------------------------------------------==#
 
diff --git a/utils/gn/.gitignore b/utils/gn/.gitignore
new file mode 100644
index 0000000000000000000000000000000000000000..ba077a4031add5b3a04384f8b9cfc414efbf47dd
--- /dev/null
+++ b/utils/gn/.gitignore
@@ -0,0 +1 @@
+bin
diff --git a/utils/gn/.gn b/utils/gn/.gn
index 86d2da97565b37e798b7f7f7b160fee81742c02c..6c77157731e91ee8030dd445ce515629e6dce55b 100644
--- a/utils/gn/.gn
+++ b/utils/gn/.gn
@@ -1,6 +1,6 @@
-# FIXME: Once it's possible to add files to the root directory of the
-# monorepo, move this file to there.  Until then, you need to pass
-# `--dotfile=llvm/utils/gn/.gn --root=.` to the `gn gen` command.
+# Since this can't be at the toplevel, you either need to pass
+# `--dotfile=llvm/utils/gn/.gn --root=.` to the `gn gen` command
+# or use llvm/utils/gn/gn.py which calls gn with these two flags added.
 
 buildconfig = "//llvm/utils/gn/build/BUILDCONFIG.gn"
 
diff --git a/utils/gn/README.rst b/utils/gn/README.rst
index fd811e916e2cbfb2ed3ab319075485c9f314618b..3c40846689c2d86e138ad9e2f5a30e026c4cbf58 100644
--- a/utils/gn/README.rst
+++ b/utils/gn/README.rst
@@ -18,11 +18,7 @@ to update GN build files. Reviewers should not ask authors to update GN build
 files. Keeping the GN build files up-to-date is on the people who use the GN
 build.
 
-*Another Warning* Right now, we're in the process of getting the GN build
-checked in. As of this writing, it's not yet functional at all. Check back
-in a few weeks!
-
-`GN <https://gn.googlesource.com/gn/>`_ is another metabuild system. It always
+`GN <https://gn.googlesource.com/gn/>`_ is a metabuild system. It always
 creates ninja files, but it can create some IDE projects (MSVC, Xcode, ...)
 which then shell out to ninja for the actual build.
 
@@ -32,11 +28,6 @@ a 2000x difference), and since it's so fast it doesn't aggressively cache,
 making it possible to switch e.g. between release and debug builds in one build
 directory.
 
-It is arguable easier to configure than the CMake build, and has native support
-for building with multiple toolchains in one build directory. The build
-description is declarative-ish, allowing GN to print it in a json format that
-can fairly easily be converted to other metabuild system inputs.
-
 The main motivation behind the GN build is that some people find it more
 convenient for day-to-day hacking on LLVM than CMake. Distribution, building
 just parts of LLVM, and embedding the LLVM GN build from other builds are a
@@ -49,27 +40,30 @@ This is a `good overview of GN <https://docs.google.com/presentation/d/15Zwb53Jc
 Quick start
 ===========
 
-*Warning* Right now, we're in the process of getting the GN build checked in.
-As of this writing, it's not yet functional at all.
-
 GN only works in the monorepo layout.
 
-#. Obtain a `gn binary <https://gn.googlesource.com/gn/#getting-started>`_.
+#. Obtain a gn binary. If gn is not already on your PATH, run
+   `llvm/utils/gn/get.py` to download a prebuilt gn binary if you're on a 64-bit
+   X86 system running Linux, macOS, or Windows, or `build gn yourself
+   <https://gn.googlesource.com/gn/#getting-started>`_ if you're on a different
+   platform or don't want to trust prebuilt binaries.
 
-#. In the root of the monorepo, run
-   `gn gen --dotfile=$PWD/llvm/utils/gn/.gn --root=. out/gn` (`out/gn` is the
-   build directory, it can have any name, and you can have as many as you want,
-   each with different build settings).
+#. In the root of the monorepo, run `llvm/utils/gn/gn.py gen out/gn`.
+   `out/gn` is the build directory, it can have any name, and you can have as
+   many as you want, each with different build settings.  (The `gn.py` script
+   adds `--dotfile=llvm/utils/gn/.gn --root=.` and just runs regular `gn`;
+   you can manually pass these parameters and not use the wrapper if you
+   prefer.)
 
-#. Run e.g. `ninja -C out/gn llvm-undname` to build all prerequisites for and
-   including the Microsoft symbol name pretty printing tool llvm-undname.
+#. Run e.g. `ninja -C out/gn check-lld` to build all prerequisites for and
+   run the LLD tests.
 
 By default, you get a release build with assertions enabled that targets
 the host arch. You can set various build options by editing `out/gn/args.gn`,
 for example putting `is_debug = true` in there gives you a debug build. Run
-`gn args --list out/gn` to see a list of all possible options. After touching
-`out/gn/args.gn`, just run ninja, it will re-invoke gn before starting the
-build.
+`llvm/utils/gn/gn.py args --list out/gn` to see a list of all possible
+options. After touching `out/gn/args.gn`, just run ninja, it will re-invoke gn
+before starting the build.
 
 GN has extensive built-in help; try e.g. `gn help gen` to see the help
 for the `gen` command. The full GN reference is also `available online
diff --git a/utils/gn/TODO.txt b/utils/gn/TODO.txt
new file mode 100644
index 0000000000000000000000000000000000000000..032fe95e6ef2518c17210ca231984d17faab616b
--- /dev/null
+++ b/utils/gn/TODO.txt
@@ -0,0 +1,26 @@
+Ideas for things to do:
+
+- more projects (compiler-rt libcxx libcxxabi libunwind clang-tools-extra lldb)
+  - phosek expressed interest in compiler-rt libcxx libcxxabi libunwind
+  - once there are more projects, have an llvm_enable_projects arg, modeled
+    after llvm_targets_to_build in the GN build
+  - a check-all build target that runs test of all projects
+- more targets (AVR MIPS RISCV SystemZ etc)
+  - example: https://reviews.llvm.org/D56416
+- investigate feasibility of working `gn check`
+
+- "optimized tblgen" mode
+  - either just always build tablegen and support with opt config
+  - or use opt toolchain and build tablegen twice in debug builds, like cmake
+
+- cross builds using GN's toolchain feature
+- one-build-dir bootstrap builds using GN's toolchain feature
+
+- move clang_tablegen into lib/ for private files
+- add dead code stripping
+- move run_tablegen.py from build to tablegen folder
+- figure out why -Iclang/Support gets added so often
+- plugin() template with working rpath, exports thingers
+  - then port clang_build_examples and enable by default so that clang
+    plugin tests run by default
+- plugin_host() template
diff --git a/utils/gn/build/BUILD.gn b/utils/gn/build/BUILD.gn
index 37aa4311651eea51493aeec8013b7aa8749a244d..91ea275b3fe97b7a301d271e9e7ec39fa98f2787 100644
--- a/utils/gn/build/BUILD.gn
+++ b/utils/gn/build/BUILD.gn
@@ -1,20 +1,17 @@
 import("//llvm/utils/gn/build/buildflags.gni")
 import("//llvm/utils/gn/build/mac_sdk.gni")
 import("//llvm/utils/gn/build/toolchain/compiler.gni")
+import("//llvm/utils/gn/build/toolchain/target_flags.gni")
 
 config("compiler_defaults") {
   defines = []
 
-  # FIXME: Don't define this globally here.
-  if (host_os != "win") {
-    defines += [ "LLVM_ON_UNIX" ]
-  }
-
   if (!llvm_enable_assertions) {
     defines += [ "NDEBUG" ]
   }
 
-  cflags = []
+  cflags = target_flags
+  ldflags = target_flags + target_ldflags
 
   if (host_os == "mac" && clang_base_path != "") {
     cflags += [
@@ -26,24 +23,33 @@ config("compiler_defaults") {
   if (host_os != "win") {
     if (is_debug) {
       cflags += [ "-g" ]
-    } else {
+    }
+    if (is_optimized) {
       cflags += [ "-O3" ]
     }
     cflags += [ "-fdiagnostics-color" ]
     cflags_cc = [
       "-std=c++11",
       "-fno-exceptions",
-      "-fno-rtti",
       "-fvisibility-inlines-hidden",
     ]
   } else {
     if (is_debug) {
-      cflags += [ "/Zi" ]
-    } else {
+      cflags += [
+        "/Zi",
+        "/FS",
+      ]
+      ldflags += [ "/DEBUG" ]
+    }
+    if (is_optimized) {
       cflags += [
         "/O2",
         "/Zc:inline",
       ]
+      ldflags += [
+        "/OPT:REF",
+        "/OPT:ICF",
+      ]
     }
     defines += [
       "_CRT_SECURE_NO_DEPRECATE",
@@ -57,14 +63,11 @@ config("compiler_defaults") {
       "_UNICODE",
       "UNICODE",
     ]
-    cflags += [
-      "/EHs-c-",
-      "/GR-",
-    ]
+    cflags += [ "/EHs-c-" ]
 
     # The MSVC default value (1 MB) is not enough for parsing recursive C++
     # templates in Clang.
-    ldflags = [ "/STACK:10000000" ]
+    ldflags += [ "/STACK:10000000" ]
   }
 
   # Warning setup.
@@ -100,12 +103,27 @@ config("compiler_defaults") {
         "-Wextra",
       ]
     }
-    cflags += [
-      "-Wno-unused-parameter",
-      "-Wstring-conversion",
-    ]
+    cflags += [ "-Wno-unused-parameter" ]
     if (is_clang) {
-      cflags += [ "-Wdelete-non-virtual-dtor" ]
+      cflags += [
+        "-Wdelete-non-virtual-dtor",
+        "-Wstring-conversion",
+      ]
+    } else {
+      cflags += [
+        # GCC's -Wcomment complains about // comments ending with '\' if the
+        # next line is also a // comment.
+        "-Wno-comment",
+
+        # Disable gcc's potentially uninitialized use analysis as it presents
+        # lots of false positives.
+        "-Wno-maybe-uninitialized",
+      ]
+      cflags_cc += [
+        # The LLVM libraries have no stable C++ API, so -Wnoexcept-type is not
+        # useful.
+        "-Wno-noexcept-type",
+      ]
     }
     if (is_clang && use_goma) {
       # goma converts all paths to lowercase on the server, breaking this
@@ -113,6 +131,27 @@ config("compiler_defaults") {
       cflags += [ "-Wno-nonportable-include-path" ]
     }
   }
+
+  # On Windows, the linker is not invoked through the compiler driver.
+  if (use_lld && host_os != "win") {
+    ldflags += [ "-fuse-ld=lld" ]
+  }
+}
+
+config("no_rtti") {
+  if (current_os == "win") {
+    cflags_cc = [ "/GR-" ]
+  } else {
+    cflags_cc = [ "-fno-rtti" ]
+  }
+}
+
+# To make an archive that can be distributed, you need to remove this config and
+# set complete_static_lib.
+config("thin_archive") {
+  if (current_os != "win" && current_os != "mac") {
+    arflags = [ "-T" ]
+  }
 }
 
 config("llvm_code") {
@@ -130,12 +169,24 @@ config("lld_code") {
 }
 
 config("clang_code") {
+  if (host_os != "win") {
+    cflags = [ "-fno-strict-aliasing" ]
+  }
   include_dirs = [
     "//clang/include",
     "$root_gen_dir/clang/include",
   ]
 }
 
+config("crt_code") {
+  include_dirs = [ "//compiler-rt/lib" ]
+  cflags = [
+    "-fPIC",
+    "-funwind-tables",
+    "-gline-tables-only",
+  ]
+}
+
 config("warn_covered_switch_default") {
   if (is_clang) {
     cflags = [ "-Wcovered-switch-default" ]
diff --git a/utils/gn/build/BUILDCONFIG.gn b/utils/gn/build/BUILDCONFIG.gn
index c4403657e5df3881f452805d0b528b519a9662d7..7de638ff3df3862a4094af44cdc5aa0aec224802 100644
--- a/utils/gn/build/BUILDCONFIG.gn
+++ b/utils/gn/build/BUILDCONFIG.gn
@@ -2,31 +2,49 @@
 # Targets can opt out of a config by removing it from their local configs list.
 # If you're adding global flags and don't need targets to be able to opt out,
 # add the flags to compiler_defaults, not to a new config.
-_shared_binary_target_configs = [
+shared_binary_target_configs = [
   "//llvm/utils/gn/build:compiler_defaults",
   "//llvm/utils/gn/build:llvm_code",
+  "//llvm/utils/gn/build:no_rtti",
+  "//llvm/utils/gn/build:thin_archive",
   "//llvm/utils/gn/build:warn_covered_switch_default",
 ]
 
 # Apply that default list to the binary target types.
 set_defaults("executable") {
-  configs = _shared_binary_target_configs
+  configs = shared_binary_target_configs
 }
 set_defaults("loadable_module") {
-  configs = _shared_binary_target_configs
+  configs = shared_binary_target_configs
 }
 set_defaults("static_library") {
-  configs = _shared_binary_target_configs
+  configs = shared_binary_target_configs
 }
 set_defaults("shared_library") {
-  configs = _shared_binary_target_configs
+  configs = shared_binary_target_configs
 }
 set_defaults("source_set") {
-  configs = _shared_binary_target_configs
+  configs = shared_binary_target_configs
+}
+
+if (target_os == "") {
+  target_os = host_os
+}
+if (current_os == "") {
+  current_os = target_os
+}
+
+if (target_cpu == "") {
+  target_cpu = host_cpu
+}
+if (current_cpu == "") {
+  current_cpu = target_cpu
 }
 
 if (host_os == "win") {
-  set_default_toolchain("//llvm/utils/gn/build/toolchain:win")
+  host_toolchain = "//llvm/utils/gn/build/toolchain:win"
 } else {
-  set_default_toolchain("//llvm/utils/gn/build/toolchain:unix")
+  host_toolchain = "//llvm/utils/gn/build/toolchain:unix"
 }
+
+set_default_toolchain(host_toolchain)
diff --git a/utils/gn/build/buildflags.gni b/utils/gn/build/buildflags.gni
index a28b788418dac70ea16fe475be805c0d8c9ab772..4dcdc962b7d1164bd212b3ecea94061163747225 100644
--- a/utils/gn/build/buildflags.gni
+++ b/utils/gn/build/buildflags.gni
@@ -1,10 +1,13 @@
 declare_args() {
-  # Whether to build with debug information and without optimizations.
+  # Whether to build with debug information.
   is_debug = false
 }
 
 # args that depend on other args must live in a later declare_args() block.
 declare_args() {
+  # Whether to build with optimizations.
+  is_optimized = !is_debug
+
   # Whether to enable assertions.
   llvm_enable_assertions = true
 }
diff --git a/utils/gn/build/fuzzer.gni b/utils/gn/build/fuzzer.gni
new file mode 100644
index 0000000000000000000000000000000000000000..b5aaf9425a4048e6bd6f9e4dad69432630f0955c
--- /dev/null
+++ b/utils/gn/build/fuzzer.gni
@@ -0,0 +1,49 @@
+# This file introduces a templates for defining fuzzers.
+#
+# All parameters valid for executable() targets are valid (cflags, defines,
+# deps, include_dirs, sources, ...). In addition to that:
+#
+#   dummy_main (required)
+#       Path to a cpp file containing main(), used when neither
+#       llvm_use_sanitize_coverage nor llvm_use_sanitize_coverage are set.
+#
+# Example of usage:
+#
+#   fuzzer("llvm-opt-fuzzer") {
+#     deps = [ ... ]
+#     dummy_main = "DummyOptFuzzer.cpp"
+#     sources = [ "llvm-opt-fuzzer.cpp" ]
+#   }
+
+declare_args() {
+  # Set to the path of a static library containing a fuzzing engine, e.g.
+  # oss-fuzz's $LIB_FUZZING_ENGINE.
+  llvm_lib_fuzzing_engine = ""
+
+  # If true, pass -fsanitize=fuzzer to the compiler for fuzzer() targets.
+  # Likely only makes sense to set if you know that the host compiler is clang.
+  llvm_use_sanitize_coverage = false
+}
+
+template("fuzzer") {
+  assert(defined(invoker.dummy_main), "must set 'dummy_main' in $target_name")
+  assert(defined(invoker.sources), "must set 'sources' for $target_name")
+  executable(target_name) {
+    forward_variables_from(invoker, "*", [ "dummy_main" ])
+    if (llvm_lib_fuzzing_engine != "") {
+      if (!defined(libs)) {
+        libs = []
+      }
+      libs += [ llvm_lib_fuzzing_engine ]
+      not_needed(invoker, [ "dummy_main" ])
+    } else if (llvm_use_sanitize_coverage) {
+      if (!defined(cflags)) {
+        cflags = []
+      }
+      cflags += [ "-fsanitize=fuzzer" ]
+      not_needed(invoker, [ "dummy_main" ])
+    } else {
+      sources += [ invoker.dummy_main ]
+    }
+  }
+}
diff --git a/utils/gn/build/libs/edit/BUILD.gn b/utils/gn/build/libs/edit/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..c22a7e60052fe8d4755c3c220ad1fcada394fc4d
--- /dev/null
+++ b/utils/gn/build/libs/edit/BUILD.gn
@@ -0,0 +1,12 @@
+import("//llvm/utils/gn/build/libs/edit/enable.gni")
+
+config("edit_config") {
+  visibility = [ ":edit" ]
+  libs = [ "edit" ]
+}
+
+group("edit") {
+  if (llvm_enable_libedit) {
+    public_configs = [ ":edit_config" ]
+  }
+}
diff --git a/utils/gn/build/libs/edit/enable.gni b/utils/gn/build/libs/edit/enable.gni
new file mode 100644
index 0000000000000000000000000000000000000000..b973b0e032cf09f49d792a855f95ab636d48ba09
--- /dev/null
+++ b/utils/gn/build/libs/edit/enable.gni
@@ -0,0 +1,3 @@
+declare_args() {
+  llvm_enable_libedit = host_os == "mac"
+}
diff --git a/utils/gn/build/libs/pthread/BUILD.gn b/utils/gn/build/libs/pthread/BUILD.gn
index 51e7f1f31fa8f9ac8b194da3e88e6f40179cb68f..7708d3146033696fc00c433e7449d03f654f7c6a 100644
--- a/utils/gn/build/libs/pthread/BUILD.gn
+++ b/utils/gn/build/libs/pthread/BUILD.gn
@@ -6,7 +6,8 @@ config("pthread_config") {
 }
 
 group("pthread") {
-  if (llvm_enable_threads && host_os != "win") {
+  # On Android, bionic has built-in support for pthreads.
+  if (llvm_enable_threads && current_os != "win" && current_os != "android") {
     public_configs = [ ":pthread_config" ]
   }
 }
diff --git a/utils/gn/build/libs/xml/BUILD.gn b/utils/gn/build/libs/xml/BUILD.gn
index f3590f4601cc8dc725298b0ce91b2b2e0f57f349..3af3c851698c8c0deff8c127bbd86465da55ece8 100644
--- a/utils/gn/build/libs/xml/BUILD.gn
+++ b/utils/gn/build/libs/xml/BUILD.gn
@@ -3,7 +3,6 @@ import("//llvm/utils/gn/build/mac_sdk.gni")
 
 config("xml_config") {
   visibility = [ ":xml" ]
-  defines = [ "LLVM_LIBXML2_ENABLED" ]
   libs = [ "xml2" ]
   if (host_os == "mac") {
     include_dirs = [ "$mac_sdk_path/usr/include/libxml2" ]
diff --git a/utils/gn/build/sync_source_lists_from_cmake.py b/utils/gn/build/sync_source_lists_from_cmake.py
index 77539672e33ad3c805ec87c629dea980400693eb..e580f0dfce76e01e8f0d79ebf710ab5a970acc8b 100755
--- a/utils/gn/build/sync_source_lists_from_cmake.py
+++ b/utils/gn/build/sync_source_lists_from_cmake.py
@@ -4,7 +4,11 @@
 
 For each BUILD.gn file in the tree, checks if the list of cpp files in
 it is identical to the list of cpp files in the corresponding CMakeLists.txt
-file, and prints the difference if not."""
+file, and prints the difference if not.
+
+Also checks that each CMakeLists.txt file below unittests/ folders that define
+binaries have corresponding BUILD.gn files.
+"""
 
 from __future__ import print_function
 
@@ -12,13 +16,15 @@ import os
 import re
 import subprocess
 
-def main():
-    gn_files = subprocess.check_output(
-            ['git', 'ls-files', '*BUILD.gn']).splitlines()
 
-    # Matches e.g. |   "foo.cpp",|.
+def sync_source_lists():
+    # Use shell=True on Windows in case git is a bat file.
+    gn_files = subprocess.check_output(['git', 'ls-files', '*BUILD.gn'],
+                                       shell=os.name == 'nt').splitlines()
+
+    # Matches e.g. |   "foo.cpp",|, captures |foo| in group 1.
     gn_cpp_re = re.compile(r'^\s*"([^"]+\.(?:cpp|h))",$', re.MULTILINE)
-    # Matches e.g. |   "foo.cpp"|.
+    # Matches e.g. |   foo.cpp|, captures |foo| in group 1.
     cmake_cpp_re = re.compile(r'^\s*([A-Za-z_0-9/-]+\.(?:cpp|h))$',
                               re.MULTILINE)
 
@@ -50,5 +56,29 @@ def main():
             print('remove:\n' + '\n'.join(remove))
         print()
 
+
+def sync_unittests():
+    # Matches e.g. |add_llvm_unittest_with_input_files|.
+    unittest_re = re.compile(r'^add_\S+_unittest', re.MULTILINE)
+
+    checked = [ 'clang', 'lld', 'llvm' ]
+    for c in checked:
+        for root, _, _ in os.walk(os.path.join(c, 'unittests')):
+            cmake_file = os.path.join(root, 'CMakeLists.txt')
+            if not os.path.exists(cmake_file):
+                continue
+            if not unittest_re.search(open(cmake_file).read()):
+                continue  # Skip CMake files that just add subdirectories.
+            gn_file = os.path.join('llvm/utils/gn/secondary', root, 'BUILD.gn')
+            if not os.path.exists(gn_file):
+                print('missing GN file %s for unittest CMake file %s' %
+                      (gn_file, cmake_file))
+
+
+def main():
+    sync_source_lists()
+    sync_unittests()
+
+
 if __name__ == '__main__':
     main()
diff --git a/utils/gn/build/toolchain/BUILD.gn b/utils/gn/build/toolchain/BUILD.gn
index 927a53d9d63153e7b242068a8f4e723f7d867c04..3588a4ec90d09513c18de61b64a8a07d0f35c9ff 100644
--- a/utils/gn/build/toolchain/BUILD.gn
+++ b/utils/gn/build/toolchain/BUILD.gn
@@ -9,7 +9,123 @@ declare_args() {
   }
 }
 
-toolchain("unix") {
+template("unix_toolchain") {
+  toolchain(target_name) {
+    forward_variables_from(invoker, "*")
+
+    tool("cc") {
+      depfile = "{{output}}.d"
+      command = "$cc -MMD -MF $depfile -o {{output}} -c {{source}} {{defines}} {{include_dirs}} {{cflags}} {{cflags_c}}"
+      depsformat = "gcc"
+      description = "CC {{output}}"
+      outputs = [
+        "{{source_out_dir}}/{{target_output_name}}.{{source_name_part}}.o",
+      ]
+    }
+
+    tool("cxx") {
+      depfile = "{{output}}.d"
+      command = "$cxx -MMD -MF $depfile -o {{output}} -c {{source}} {{defines}} {{include_dirs}} {{cflags}} {{cflags_cc}}"
+      depsformat = "gcc"
+      description = "CXX {{output}}"
+      outputs = [
+        "{{source_out_dir}}/{{target_output_name}}.{{source_name_part}}.o",
+      ]
+    }
+
+    tool("asm") {
+      depfile = "{{output}}.d"
+      command = "$cc -MMD -MF $depfile -o {{output}} -c {{source}} {{defines}} {{include_dirs}} {{asmflags}}"
+      depsformat = "gcc"
+      description = "ASM {{output}}"
+      outputs = [
+        "{{source_out_dir}}/{{target_output_name}}.{{source_name_part}}.o",
+      ]
+    }
+
+    tool("alink") {
+      if (current_os == "mac") {
+        command = "libtool -static -no_warning_for_no_symbols {{arflags}} -o {{output}} {{inputs}}"
+      } else {
+        # Remove the output file first so that ar doesn't try to modify the
+        # existing file.
+        command =
+            "rm -f {{output}} && $ar rcsD {{arflags}} {{output}} {{inputs}}"
+      }
+      description = "AR {{output}}"
+      outputs = [
+        "{{output_dir}}/{{target_output_name}}.a",
+      ]
+      output_prefix = "lib"
+      default_output_dir = "{{root_out_dir}}/lib"
+    }
+
+    tool("solink") {
+      outfile = "{{output_dir}}/{{target_output_name}}{{output_extension}}"
+      if (current_os == "mac") {
+        command = "$ld -shared {{ldflags}} -o $outfile {{libs}} {{inputs}}"
+        default_output_extension = ".dylib"
+      } else {
+        command = "$ld -shared {{ldflags}} -Wl,-z,defs -Wl,-soname,{{target_output_name}}{{output_extension}} -o $outfile {{libs}} {{inputs}}"
+        default_output_extension = ".so"
+      }
+      description = "SOLINK $outfile"
+      outputs = [
+        outfile,
+      ]
+      lib_switch = "-l"
+      output_prefix = "lib"
+      default_output_dir = "{{root_out_dir}}/lib"
+    }
+
+    tool("solink_module") {
+      outfile = "{{output_dir}}/{{target_output_name}}{{output_extension}}"
+      if (current_os == "mac") {
+        command = "$ld -shared {{ldflags}} -Wl,-flat_namespace -Wl,-undefined,suppress -o $outfile {{libs}} {{inputs}}"
+        default_output_extension = ".dylib"
+      } else {
+        command = "$ld -shared {{ldflags}} -Wl,-soname,{{target_output_name}}{{output_extension}} -o $outfile {{libs}} {{inputs}}"
+        default_output_extension = ".so"
+      }
+      description = "SOLINK $outfile"
+      outputs = [
+        outfile,
+      ]
+      lib_switch = "-l"
+      default_output_dir = "{{root_out_dir}}/lib"
+    }
+
+    tool("link") {
+      outfile = "{{output_dir}}/{{target_output_name}}{{output_extension}}"
+      if (current_os == "mac") {
+        command = "$ld {{ldflags}} -o $outfile {{libs}} {{inputs}}"
+      } else {
+        command = "$ld {{ldflags}} -o $outfile {{libs}} -Wl,--start-group {{inputs}} -Wl,--end-group"
+      }
+      description = "LINK $outfile"
+      outputs = [
+        outfile,
+      ]
+      lib_switch = "-l"
+
+      # Setting this allows targets to override the default executable output by
+      # setting output_dir.
+      default_output_dir = "{{root_out_dir}}/bin"
+    }
+
+    tool("copy") {
+      command = "ln -f {{source}} {{output}} 2>/dev/null || (rm -rf {{output}} && cp -af {{source}} {{output}})"
+      description = "COPY {{source}} {{output}}"
+    }
+
+    tool("stamp") {
+      command = "touch {{output}}"
+      description = "STAMP {{output}}"
+    }
+  }
+}
+
+unix_toolchain("unix") {
   cc = "cc"
   cxx = "c++"
 
@@ -24,105 +140,54 @@ toolchain("unix") {
     cxx = "$goma_dir/gomacc $cxx"
   }
 
-  tool("cc") {
-    depfile = "{{output}}.d"
-    command = "$cc -MMD -MF $depfile -o {{output}} -c {{source}} {{defines}} {{include_dirs}} {{cflags}} {{cflags_c}}"
-    depsformat = "gcc"
-    description = "CC {{output}}"
-    outputs = [
-      "{{source_out_dir}}/{{target_output_name}}.{{source_name_part}}.o",
-    ]
+  if (current_os != "mac") {
+    ar = "ar"
   }
 
-  tool("cxx") {
-    depfile = "{{output}}.d"
-    command = "$cxx -MMD -MF $depfile -o {{output}} -c {{source}} {{defines}} {{include_dirs}} {{cflags}} {{cflags_cc}}"
-    depsformat = "gcc"
-    description = "CXX {{output}}"
-    outputs = [
-      "{{source_out_dir}}/{{target_output_name}}.{{source_name_part}}.o",
-    ]
+  toolchain_args = {
+    current_os = host_os
+    current_cpu = host_cpu
   }
+}
 
-  tool("alink") {
-    if (host_os == "mac") {
-      command = "libtool -static -no_warning_for_no_symbols {{arflags}} -o {{output}} {{inputs}}"
-    } else {
-      # Remove the output file first so that ar doesn't try to modify the
-      # existing file.
-      command =
-          "rm -f {{output}} && ar rcsDT {{arflags}} -o {{output}} {{inputs}}"
-    }
-    description = "AR {{output}}"
-    outputs = [
-      "{{output_dir}}/{{target_output_name}}.a",
-    ]
-    output_prefix = "lib"
-    default_output_dir = "{{root_out_dir}}/lib"
-  }
+template("stage2_unix_toolchain") {
+  unix_toolchain(target_name) {
+    forward_variables_from(invoker, "*")
 
-  tool("solink") {
-    outfile = "{{output_dir}}/{{target_output_name}}{{output_extension}}"
-    if (host_os == "mac") {
-      command = "$ld -shared {{ldflags}} -o $outfile {{libs}} {{inputs}}"
-      default_output_extension = ".dylib"
-    } else {
-      command =
-          "$ld -shared {{ldflags}} -Wl,-z,defs -o $outfile {{libs}} {{inputs}}"
-      default_output_extension = ".so"
+    cc = "bin/clang"
+    cxx = "bin/clang++"
+    ld = cxx
+    if (current_os != "mac") {
+      ar = "bin/llvm-ar"
     }
-    description = "SOLINK $outfile"
-    outputs = [
-      outfile,
-    ]
-    lib_switch = "-l"
-    output_prefix = "lib"
-    default_output_dir = "{{root_out_dir}}/lib"
-  }
 
-  tool("solink_module") {
-    outfile = "{{output_dir}}/{{target_output_name}}{{output_extension}}"
-    if (host_os == "mac") {
-      command = "$ld -shared {{ldflags}} -Wl,-flat_namespace -Wl,-undefined,suppress -o $outfile {{libs}} {{inputs}}"
-      default_output_extension = ".dylib"
-    } else {
-      command = "$ld -shared {{ldflags}} -o $outfile {{libs}} {{inputs}}"
-      default_output_extension = ".so"
-    }
-    description = "SOLINK $outfile"
-    outputs = [
-      outfile,
+    deps = [
+      "//:clang($host_toolchain)",
+      "//:lld($host_toolchain)",
     ]
-    lib_switch = "-l"
-    default_output_dir = "{{root_out_dir}}/lib"
-  }
-
-  tool("link") {
-    outfile = "{{output_dir}}/{{target_output_name}}{{output_extension}}"
-    if (host_os == "mac") {
-      command = "$ld {{ldflags}} -o $outfile {{libs}} {{inputs}}"
-    } else {
-      command = "$ld {{ldflags}} -o $outfile {{libs}} -Wl,--start-group {{inputs}} -Wl,--end-group"
+    if (current_os != "mac") {
+      deps += [ "//:llvm-ar($host_toolchain)" ]
     }
-    description = "LINK $outfile"
-    outputs = [
-      outfile,
-    ]
-    lib_switch = "-l"
-
-    # Setting this allows targets to override the default executable output by
-    # setting output_dir.
-    default_output_dir = "{{root_out_dir}}/bin"
   }
+}
 
-  tool("copy") {
-    command = "ln -f {{source}} {{output}} 2>/dev/null || (rm -rf {{output}} && cp -af {{source}} {{output}})"
-    description = "COPY {{source}} {{output}}"
+stage2_unix_toolchain("stage2_unix") {
+  toolchain_args = {
+    current_os = host_os
+    current_cpu = host_cpu
+    is_clang = true
+    use_lld = host_os != "mac"
   }
+}
 
-  tool("stamp") {
-    command = "touch {{output}}"
-    description = "STAMP {{output}}"
+if (android_ndk_path != "") {
+  stage2_unix_toolchain("stage2_android_aarch64") {
+    toolchain_args = {
+      current_os = "android"
+      current_cpu = "arm64"
+      is_clang = true
+      use_lld = true
+    }
   }
 }
 
@@ -132,7 +197,9 @@ toolchain("win") {
 
   if (clang_base_path != "") {
     cl = "$clang_base_path/bin/clang-cl"
-    link = "$clang_base_path/bin/lld-link"
+    if (use_lld) {
+      link = "$clang_base_path/bin/lld-link"
+    }
   }
 
   if (use_goma) {
@@ -167,9 +234,11 @@ toolchain("win") {
   }
 
   tool("solink") {
-    dllfile = "{{output_dir}}/{{target_output_name}}{{output_extension}}"
-    libfile = "$dllfile.lib"
-    command = "$link /nologo /dll {{ldflags}} /out:$dllfile /implib:$libfile {{libs}} /pdb:$dllfile.pdb {{inputs}}"
+    outprefix = "{{output_dir}}/{{target_output_name}}"
+    dllfile = "$outprefix{{output_extension}}"
+    libfile = "$outprefix.lib"
+    pdbfile = "$outprefix.pdb"
+    command = "$link /nologo /dll {{ldflags}} /out:$dllfile /implib:$libfile /pdb:$pdbfile {{libs}} {{inputs}}"
     description = "LINK $dllfile"
     link_output = libfile
     depend_output = libfile
@@ -192,8 +261,10 @@ toolchain("win") {
   # standalone loadable modules used for unit-testing LLVM's dynamic library
   # loading code.
   tool("solink_module") {
-    dllfile = "{{output_dir}}/{{target_output_name}}{{output_extension}}"
-    command = "$link /nologo /dll {{ldflags}} /out:$dllfile {{libs}} /pdb:$dllfile.pdb {{inputs}}"
+    outprefix = "{{output_dir}}/{{target_output_name}}"
+    dllfile = "$outprefix{{output_extension}}"
+    pdbfile = "$outprefix.pdb"
+    command = "$link /nologo /dll {{ldflags}} /out:$dllfile /pdb:$pdbfile {{libs}} {{inputs}}"
     description = "LINK_MODULE $dllfile"
     outputs = [
       dllfile,
@@ -205,8 +276,10 @@ toolchain("win") {
   }
 
   tool("link") {
-    outfile = "{{output_dir}}/{{target_output_name}}{{output_extension}}"
-    command = "$link /nologo {{ldflags}} /out:$outfile {{libs}} /pdb:$outfile.pdb {{inputs}}"
+    outprefix = "{{output_dir}}/{{target_output_name}}"
+    outfile = "$outprefix{{output_extension}}"
+    pdbfile = "$outprefix.pdb"
+    command = "$link /nologo {{ldflags}} /out:$outfile /pdb:$pdbfile {{libs}} {{inputs}}"
     description = "LINK $outfile"
     outputs = [
       outfile,
@@ -229,4 +302,9 @@ toolchain("win") {
     command = "cmd /c type nul > {{output}}"
     description = "STAMP {{output}}"
   }
+
+  toolchain_args = {
+    current_os = "win"
+    current_cpu = host_cpu
+  }
 }
diff --git a/utils/gn/build/toolchain/compiler.gni b/utils/gn/build/toolchain/compiler.gni
index 92d965ef96738098c953d5b5e872c6158e288467..37ad1f5bc31351d333852bdb58813b762d4d1d30 100644
--- a/utils/gn/build/toolchain/compiler.gni
+++ b/utils/gn/build/toolchain/compiler.gni
@@ -9,10 +9,17 @@ declare_args() {
   # On Windows, setting this also causes lld-link to be used as linker.
   # Example value: getenv("HOME") + "/src/llvm-build/Release+Asserts"
   clang_base_path = ""
+
+  # Set this to the path to Android NDK r19. If set, cross compilation targeting
+  # Android will be enabled.
+  android_ndk_path = ""
 }
 
 declare_args() {
   # Set if the host compiler is clang.  On by default on Mac or if
   # clang_base_path is set.
   is_clang = host_os == "mac" || clang_base_path != ""
+
+  # Set this to true to link with LLD instead of the default linker.
+  use_lld = clang_base_path != "" && host_os != "mac"
 }
diff --git a/utils/gn/build/toolchain/target_flags.gni b/utils/gn/build/toolchain/target_flags.gni
new file mode 100644
index 0000000000000000000000000000000000000000..3ad02a969fd446fb4ebaa286b3e17b5049be9083
--- /dev/null
+++ b/utils/gn/build/toolchain/target_flags.gni
@@ -0,0 +1,14 @@
+import("//llvm/triples.gni")
+import("//llvm/utils/gn/build/toolchain/compiler.gni")
+
+target_flags = []
+target_ldflags = []
+
+if (current_os == "android") {
+  target_flags += [
+    "--target=$llvm_current_triple",
+    "--sysroot=$android_ndk_path/toolchains/llvm/prebuilt/linux-x86_64/sysroot",
+    "-B$android_ndk_path/toolchains/llvm/prebuilt/linux-x86_64",
+  ]
+  target_ldflags += [ "-static-libstdc++" ]
+}
diff --git a/utils/gn/build/write_cmake_config.gni b/utils/gn/build/write_cmake_config.gni
new file mode 100644
index 0000000000000000000000000000000000000000..6dca02608e157c9912a23f3b26ebb9311259d76e
--- /dev/null
+++ b/utils/gn/build/write_cmake_config.gni
@@ -0,0 +1,54 @@
+# This file introduces a templates for calling write_cmake_config.py.
+#
+# write_cmake_config behaves like CMake's configure_file(), but runs at build
+# time, not at generator time.  See write_cmake_config.py for details.
+#
+# Parameters:
+#
+#   input (required) [string]
+#
+#   output (required) [string]
+#
+#   values (required) [list of strings]
+#       Each entry is a '='-separated key-value pair used for substitution.
+#
+# Example use:
+#
+#   write_cmake_config("attributes_compat_func_gen") {
+#     input = "Version.inc.in"
+#     output = "$root_gen_dir/clang/include/clang/Basic/Version.inc",
+#     values = [
+#       "CLANG_VERSION=$llvm_version",
+#     ]
+#   }
+
+template("write_cmake_config") {
+  assert(defined(invoker.input), "must set 'input' in $target_name")
+  assert(defined(invoker.output), "must set 'output' in $target_name")
+  assert(defined(invoker.values), "must set 'values' in $target_name")
+
+  action(target_name) {
+    script = "//llvm/utils/gn/build/write_cmake_config.py"
+
+    sources = [
+      invoker.input,
+    ]
+    outputs = [
+      invoker.output,
+    ]
+    args = [
+             "-o",
+             rebase_path(outputs[0], root_build_dir),
+             rebase_path(sources[0], root_build_dir),
+           ] + invoker.values
+
+    forward_variables_from(invoker,
+                           [
+                             "configs",
+                             "deps",
+                             "public_configs",
+                             "public_deps",
+                             "visibility",
+                           ])
+  }
+}
diff --git a/utils/gn/build/write_cmake_config.py b/utils/gn/build/write_cmake_config.py
index 6a54073d85b596a40de4feda13d929ab7f46549f..eba6b4ec521a51de65bfc0d44e8127c45a76559b 100755
--- a/utils/gn/build/write_cmake_config.py
+++ b/utils/gn/build/write_cmake_config.py
@@ -53,6 +53,9 @@ def main():
     values = {}
     for value in args.values:
         key, val = value.split('=', 1)
+        if key in values:
+            print('duplicate key "%s" in args' % key, file=sys.stderr)
+            return 1
         values[key] = val.replace('\\n', '\n')
     unused_values = set(values.keys())
 
@@ -86,7 +89,7 @@ def main():
 
     if unused_values:
         print('unused values args:', file=sys.stderr)
-        print('    ', '\n    '.join(unused_values), file=sys.stderr)
+        print('    ' + '\n    '.join(unused_values), file=sys.stderr)
         return 1
 
     output = ''.join(out_lines)
@@ -98,6 +101,7 @@ def main():
 
     if not os.path.exists(args.output) or open(args.output).read() != output:
         open(args.output, 'w').write(output)
+        os.chmod(args.output, os.stat(args.input).st_mode & 0o777)
 
 
 if __name__ == '__main__':
diff --git a/utils/gn/build/write_vcsrevision.py b/utils/gn/build/write_vcsrevision.py
index ed7c05f1d0ef8730bcadfbaa1c752e0eeb081a70..5bdca23e76485c156cd3cbba78464fe98ce9cdd5 100755
--- a/utils/gn/build/write_vcsrevision.py
+++ b/utils/gn/build/write_vcsrevision.py
@@ -12,7 +12,6 @@ import sys
 
 THIS_DIR = os.path.abspath(os.path.dirname(__file__))
 LLVM_DIR = os.path.dirname(os.path.dirname(os.path.dirname(THIS_DIR)))
-MONO_DIR = os.path.dirname(LLVM_DIR)
 
 
 def which(program):
@@ -36,25 +35,27 @@ def main():
     if os.path.isdir(os.path.join(LLVM_DIR, '.svn')):
         print('SVN support not implemented', file=sys.stderr)
         return 1
-    if os.path.isdir(os.path.join(LLVM_DIR, '.git')):
+    if os.path.exists(os.path.join(LLVM_DIR, '.git')):
         print('non-mono-repo git support not implemented', file=sys.stderr)
         return 1
 
-    git_dir = os.path.join(MONO_DIR, '.git')
-    if not os.path.isdir(git_dir):
-        print('.git dir not found at "%s"' % git_dir, file=sys.stderr)
-        return 1
-
     git, use_shell = which('git'), False
     if not git:
         git = which('git.exe')
     if not git:
         git = which('git.bat')
         use_shell = True
+
+    git_dir = subprocess.check_output([git, 'rev-parse', '--git-dir'],
+                                      cwd=LLVM_DIR, shell=use_shell).decode().strip()
+    if not os.path.isdir(git_dir):
+        print('.git dir not found at "%s"' % git_dir, file=sys.stderr)
+        return 1
+
     rev = subprocess.check_output([git, 'rev-parse', '--short', 'HEAD'],
-                                  cwd=git_dir, shell=use_shell)
+                                  cwd=git_dir, shell=use_shell).decode().strip()
     # FIXME: add pizzas such as the svn revision read off a git note?
-    vcsrevision_contents = '#define LLVM_REVISION "git-%s"\n' % rev.strip()
+    vcsrevision_contents = '#define LLVM_REVISION "git-%s"\n' % rev
 
     # If the output already exists and is identical to what we'd write,
     # return to not perturb the existing file's timestamp.
diff --git a/utils/gn/get.py b/utils/gn/get.py
new file mode 100755
index 0000000000000000000000000000000000000000..defeac98f2c1f3ce349ccc74212d1032db596478
--- /dev/null
+++ b/utils/gn/get.py
@@ -0,0 +1,63 @@
+#!/usr/bin/env python
+"""Downloads a prebuilt gn binary to a place where gn.py can find it."""
+
+from __future__ import print_function
+
+import os
+import urllib2
+import sys
+import tempfile
+import zipfile
+
+
+def download_url(url, output_file):
+    """Download url into output_file."""
+    print('downloading %s ...' % url, end='')
+    sys.stdout.flush()
+    output_file.write(urllib2.urlopen(url).read())
+    print(' done')
+
+
+def download_and_unpack(url, output_dir, gn):
+    """Download an archive from url and extract gn from it into output_dir."""
+    with tempfile.TemporaryFile() as f:
+        download_url(url, f)
+        f.seek(0)
+        zipfile.ZipFile(f).extract(gn, path=output_dir)
+
+
+def set_executable_bit(path):
+    mode = os.stat(path).st_mode
+    mode |= (mode & 0o444) >> 2 # Copy R bits to X.
+    os.chmod(path, mode) # No-op on Windows.
+
+
+def get_platform():
+    if os.uname()[4] != 'x86_64':
+        return None
+    if sys.platform.startswith('linux'):
+        return 'linux-amd64'
+    if sys.platform == 'darwin':
+        return 'mac-amd64'
+    if sys.platform == 'win32':
+        return 'windows-amd64'
+
+
+def main():
+    platform = get_platform()
+    if not platform:
+        print('no prebuilt binary for', sys.platform)
+        return 1
+
+    dirname = os.path.join(os.path.dirname(__file__), 'bin', platform)
+    if not os.path.exists(dirname):
+        os.makedirs(dirname)
+
+    url = 'https://chrome-infra-packages.appspot.com/dl/gn/gn/%s/+/latest'
+    gn = 'gn' + ('.exe' if sys.platform == 'win32' else '')
+    download_and_unpack(url % platform, dirname, gn)
+    set_executable_bit(os.path.join(dirname, gn))
+
+
+if __name__ == '__main__':
+    sys.exit(main())
diff --git a/utils/gn/gn.py b/utils/gn/gn.py
new file mode 100755
index 0000000000000000000000000000000000000000..f523e6443ea78d4e1fc07240f1f7ff4d9988fe51
--- /dev/null
+++ b/utils/gn/gn.py
@@ -0,0 +1,66 @@
+#!/usr/bin/env python
+"""Calls `gn` with the right --dotfile= and --root= arguments for LLVM."""
+
+# GN normally expects a file called '.gn' at the root of the repository.
+# Since LLVM's GN build isn't supported, putting that file at the root
+# is deemed inappropriate, which requires passing --dotfile= and -root= to GN.
+# Since that gets old fast, this script automatically passes these arguments.
+
+import os
+import subprocess
+import sys
+
+
+THIS_DIR = os.path.dirname(__file__)
+ROOT_DIR = os.path.join(THIS_DIR, '..', '..', '..')
+
+
+def get_platform():
+    if os.uname()[4] != 'x86_64':
+        return None
+    if sys.platform.startswith('linux'):
+        return 'linux-amd64'
+    if sys.platform == 'darwin':
+        return 'mac-amd64'
+    if sys.platform == 'win32':
+        return 'windows-amd64'
+
+
+def print_no_gn(mention_get):
+    print('gn binary not found in PATH')
+    if mention_get:
+        print('run llvm/utils/gn/get.py to download a binary and try again, or')
+    print('follow https://gn.googlesource.com/gn/#getting-started')
+    return 1
+
+
+def main():
+    # Find real gn executable.
+    gn = 'gn'
+    if subprocess.call([gn, '--version'], stdout=open(os.devnull, 'w'),
+                                          stderr=subprocess.STDOUT) != 0:
+        # Not on path. See if get.py downloaded a prebuilt binary and run that
+        # if it's there, or suggest to run get.py if it isn't.
+        platform = get_platform()
+        if not platform:
+            return print_no_gn(mention_get=False)
+        gn = os.path.join(os.path.dirname(__file__), 'bin', platform, 'gn')
+        if not os.path.exists(gn + ('.exe' if platform == 'windows' else '')):
+            return print_no_gn(mention_get=True)
+
+    # Compute --dotfile= and --root= args to add.
+    extra_args = []
+    gn_main_arg = next((x for x in sys.argv[1:] if not x.startswith('-')), None)
+    if gn_main_arg != 'help':  # `gn help` gets confused by the switches.
+        cwd = os.getcwd()
+        dotfile = os.path.relpath(os.path.join(THIS_DIR, '.gn'), cwd)
+        root = os.path.relpath(ROOT_DIR, cwd)
+        extra_args = [ '--dotfile=' + dotfile, '--root=' + root ]
+
+    # Run GN command with --dotfile= and --root= added.
+    cmd = [gn] + extra_args + sys.argv[1:]
+    sys.exit(subprocess.call(cmd))
+
+
+if __name__ == '__main__':
+    main()
diff --git a/utils/gn/secondary/BUILD.gn b/utils/gn/secondary/BUILD.gn
index 38716d569a92a206944e66d02fbf23ab20a7b269..7509f590186d7faef9ee6b6f7991db81419a070e 100644
--- a/utils/gn/secondary/BUILD.gn
+++ b/utils/gn/secondary/BUILD.gn
@@ -1,26 +1,20 @@
+import("//clang/lib/ARCMigrate/enable.gni")
+import("//clang/lib/StaticAnalyzer/Frontend/enable.gni")
+import("//llvm/utils/gn/build/toolchain/compiler.gni")
+
 group("default") {
   deps = [
-    ":lld",
-    "//llvm/tools/llc",
-    "//llvm/tools/llvm-ar:symlinks",
-    "//llvm/tools/llvm-as",
-    "//llvm/tools/llvm-bcanalyzer",
-    "//llvm/tools/llvm-dis",
-    "//llvm/tools/llvm-dwarfdump",
-    "//llvm/tools/llvm-mc",
-    "//llvm/tools/llvm-nm:symlinks",
-    "//llvm/tools/llvm-objcopy:symlinks",
-    "//llvm/tools/llvm-objdump:symlinks",
-    "//llvm/tools/llvm-pdbutil",
-    "//llvm/tools/llvm-readobj:symlinks",
-    "//llvm/tools/llvm-undname",
-    "//llvm/tools/obj2yaml",
-    "//llvm/tools/opt",
-    "//llvm/tools/yaml2obj",
-    "//llvm/utils/FileCheck",
-    "//llvm/utils/count",
-    "//llvm/utils/not",
+    "//clang-tools-extra/clang-apply-replacements/tool:clang-apply-replacements",
+    "//clang-tools-extra/clang-tidy/tool:clang-tidy",
+    "//clang/test",
+    "//lld/test",
+    "//llvm/test",
   ]
+  if (current_os == "linux" || current_os == "android") {
+    deps += [ "//compiler-rt/test/hwasan" ]
+  }
+
+  testonly = true
 }
 
 # Symlink handling.
@@ -45,6 +39,11 @@ group("default") {
 # update symlinks (in particular, on Windows it doesn't copy the new lld to its
 # new locations).
 # That seems simpler, more explicit, and good enough.
+group("clang") {
+  deps = [
+    "//clang/tools/driver:symlinks",
+  ]
+}
 group("lld") {
   deps = [
     "//lld/tools/lld:symlinks",
@@ -55,11 +54,21 @@ group("llvm-ar") {
     "//llvm/tools/llvm-ar:symlinks",
   ]
 }
+group("llvm-dwp") {
+  deps = [
+    "//llvm/tools/llvm-dwp:symlinks",
+  ]
+}
 group("llvm-nm") {
   deps = [
     "//llvm/tools/llvm-nm:symlinks",
   ]
 }
+group("llvm-cxxfilt") {
+  deps = [
+    "//llvm/tools/llvm-cxxfilt:symlinks",
+  ]
+}
 group("llvm-objcopy") {
   deps = [
     "//llvm/tools/llvm-objcopy:symlinks",
@@ -75,6 +84,21 @@ group("llvm-readobj") {
     "//llvm/tools/llvm-readobj:symlinks",
   ]
 }
+group("llvm-size") {
+  deps = [
+    "//llvm/tools/llvm-size:symlinks",
+  ]
+}
+group("llvm-strings") {
+  deps = [
+    "//llvm/tools/llvm-strings:symlinks",
+  ]
+}
+group("llvm-symbolizer") {
+  deps = [
+    "//llvm/tools/llvm-symbolizer:symlinks",
+  ]
+}
 
 # A pool called "console" in the root BUILD.gn is magic and represents ninja's
 # built-in console pool. (Requires a GN with `gn --version` >= 552353.)
diff --git a/utils/gn/secondary/clang-tools-extra/clang-apply-replacements/BUILD.gn b/utils/gn/secondary/clang-tools-extra/clang-apply-replacements/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..dae99784a71939ad68fe7093b23049995ae27433
--- /dev/null
+++ b/utils/gn/secondary/clang-tools-extra/clang-apply-replacements/BUILD.gn
@@ -0,0 +1,15 @@
+static_library("clang-apply-replacements") {
+  configs += [ "//llvm/utils/gn/build:clang_code" ]
+  deps = [
+    "//clang/lib/AST",
+    "//clang/lib/Basic",
+    "//clang/lib/Rewrite",
+    "//clang/lib/Tooling/Core",
+    "//clang/lib/Tooling/Refactoring",
+    "//llvm/lib/Support",
+  ]
+  include_dirs = [ "include" ]
+  sources = [
+    "lib/Tooling/ApplyReplacements.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/clang-tools-extra/clang-apply-replacements/tool/BUILD.gn b/utils/gn/secondary/clang-tools-extra/clang-apply-replacements/tool/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..06d2f392a45f28548e9dacdb00ea334cef50af70
--- /dev/null
+++ b/utils/gn/secondary/clang-tools-extra/clang-apply-replacements/tool/BUILD.gn
@@ -0,0 +1,16 @@
+executable("clang-apply-replacements") {
+  configs += [ "//llvm/utils/gn/build:clang_code" ]
+  deps = [
+    "//clang-tools-extra/clang-apply-replacements",
+    "//clang/lib/Basic",
+    "//clang/lib/Format",
+    "//clang/lib/Rewrite",
+    "//clang/lib/Tooling/Core",
+    "//clang/lib/Tooling/Refactoring",
+    "//llvm/lib/Support",
+  ]
+  include_dirs = [ "../include" ]
+  sources = [
+    "ClangApplyReplacementsMain.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/clang-tools-extra/clang-tidy/BUILD.gn b/utils/gn/secondary/clang-tools-extra/clang-tidy/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..344cce3002d91e28dc75e294d922841d0a88f9c2
--- /dev/null
+++ b/utils/gn/secondary/clang-tools-extra/clang-tidy/BUILD.gn
@@ -0,0 +1,36 @@
+import("//clang/lib/StaticAnalyzer/Frontend/enable.gni")
+
+static_library("clang-tidy") {
+  output_name = "clangTidy"
+  configs += [ "//llvm/utils/gn/build:clang_code" ]
+  deps = [
+    "//clang/include/clang/StaticAnalyzer/Checkers",
+    "//clang/lib/AST",
+    "//clang/lib/ASTMatchers",
+    "//clang/lib/Basic",
+    "//clang/lib/Format",
+    "//clang/lib/Frontend",
+    "//clang/lib/Lex",
+    "//clang/lib/Rewrite",
+    "//clang/lib/Sema",
+    "//clang/lib/Serialization",
+    "//clang/lib/Tooling",
+    "//clang/lib/Tooling/Core",
+    "//llvm/lib/Support",
+  ]
+
+  if (clang_enable_static_analyzer) {
+    deps += [
+      "//clang/lib/StaticAnalyzer/Core",
+      "//clang/lib/StaticAnalyzer/Frontend",
+    ]
+  }
+
+  sources = [
+    "ClangTidy.cpp",
+    "ClangTidyDiagnosticConsumer.cpp",
+    "ClangTidyModule.cpp",
+    "ClangTidyOptions.cpp",
+    "ClangTidyProfiling.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/clang-tools-extra/clang-tidy/abseil/BUILD.gn b/utils/gn/secondary/clang-tools-extra/clang-tidy/abseil/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..e60149352bd6372a271c98a65ce2e9b573cb77cb
--- /dev/null
+++ b/utils/gn/secondary/clang-tools-extra/clang-tidy/abseil/BUILD.gn
@@ -0,0 +1,33 @@
+static_library("abseil") {
+  output_name = "clangTidyAbseilModule"
+  configs += [ "//llvm/utils/gn/build:clang_code" ]
+  deps = [
+    "//clang-tools-extra/clang-tidy",
+    "//clang-tools-extra/clang-tidy/utils",
+    "//clang/lib/AST",
+    "//clang/lib/ASTMatchers",
+    "//clang/lib/Basic",
+    "//clang/lib/Lex",
+    "//clang/lib/Tooling",
+    "//llvm/lib/Support",
+  ]
+  sources = [
+    "AbseilTidyModule.cpp",
+    "DurationAdditionCheck.cpp",
+    "DurationComparisonCheck.cpp",
+    "DurationConversionCastCheck.cpp",
+    "DurationDivisionCheck.cpp",
+    "DurationFactoryFloatCheck.cpp",
+    "DurationFactoryScaleCheck.cpp",
+    "DurationRewriter.cpp",
+    "DurationSubtractionCheck.cpp",
+    "DurationUnnecessaryConversionCheck.cpp",
+    "FasterStrsplitDelimiterCheck.cpp",
+    "NoInternalDependenciesCheck.cpp",
+    "NoNamespaceCheck.cpp",
+    "RedundantStrcatCallsCheck.cpp",
+    "StrCatAppendCheck.cpp",
+    "StringFindStartswithCheck.cpp",
+    "UpgradeDurationConversionsCheck.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/clang-tools-extra/clang-tidy/android/BUILD.gn b/utils/gn/secondary/clang-tools-extra/clang-tidy/android/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..4a7444e4715103a37daf1bd1d31d9f2f22dae769
--- /dev/null
+++ b/utils/gn/secondary/clang-tools-extra/clang-tidy/android/BUILD.gn
@@ -0,0 +1,30 @@
+static_library("android") {
+  output_name = "clangTidyAndroidModule"
+  configs += [ "//llvm/utils/gn/build:clang_code" ]
+  deps = [
+    "//clang-tools-extra/clang-tidy",
+    "//clang-tools-extra/clang-tidy/utils",
+    "//clang/lib/AST",
+    "//clang/lib/ASTMatchers",
+    "//clang/lib/Basic",
+    "//clang/lib/Lex",
+    "//llvm/lib/Support",
+  ]
+  sources = [
+    "AndroidTidyModule.cpp",
+    "CloexecAccept4Check.cpp",
+    "CloexecAcceptCheck.cpp",
+    "CloexecCheck.cpp",
+    "CloexecCreatCheck.cpp",
+    "CloexecDupCheck.cpp",
+    "CloexecEpollCreate1Check.cpp",
+    "CloexecEpollCreateCheck.cpp",
+    "CloexecFopenCheck.cpp",
+    "CloexecInotifyInit1Check.cpp",
+    "CloexecInotifyInitCheck.cpp",
+    "CloexecMemfdCreateCheck.cpp",
+    "CloexecOpenCheck.cpp",
+    "CloexecSocketCheck.cpp",
+    "ComparisonInTempFailureRetryCheck.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/clang-tools-extra/clang-tidy/boost/BUILD.gn b/utils/gn/secondary/clang-tools-extra/clang-tidy/boost/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..c7f9e0466b04eaea562d2b4007dcb9ecf76e2b6a
--- /dev/null
+++ b/utils/gn/secondary/clang-tools-extra/clang-tidy/boost/BUILD.gn
@@ -0,0 +1,17 @@
+static_library("boost") {
+  output_name = "clangTidyBoostModule"
+  configs += [ "//llvm/utils/gn/build:clang_code" ]
+  deps = [
+    "//clang-tools-extra/clang-tidy",
+    "//clang-tools-extra/clang-tidy/utils",
+    "//clang/lib/AST",
+    "//clang/lib/ASTMatchers",
+    "//clang/lib/Basic",
+    "//clang/lib/Lex",
+    "//llvm/lib/Support",
+  ]
+  sources = [
+    "BoostTidyModule.cpp",
+    "UseToStringCheck.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/clang-tools-extra/clang-tidy/bugprone/BUILD.gn b/utils/gn/secondary/clang-tools-extra/clang-tidy/bugprone/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..7d3767c2b867fddbaa8748c08f7be6f92dddd789
--- /dev/null
+++ b/utils/gn/secondary/clang-tools-extra/clang-tidy/bugprone/BUILD.gn
@@ -0,0 +1,59 @@
+static_library("bugprone") {
+  output_name = "clangTidyBugproneModule"
+  configs += [ "//llvm/utils/gn/build:clang_code" ]
+  deps = [
+    "//clang-tools-extra/clang-tidy",
+    "//clang-tools-extra/clang-tidy/cppcoreguidelines",
+    "//clang-tools-extra/clang-tidy/utils",
+    "//clang/lib/AST",
+    "//clang/lib/ASTMatchers",
+    "//clang/lib/Analysis",
+    "//clang/lib/Basic",
+    "//clang/lib/Lex",
+    "//clang/lib/Tooling",
+    "//llvm/lib/Support",
+  ]
+  sources = [
+    "ArgumentCommentCheck.cpp",
+    "AssertSideEffectCheck.cpp",
+    "BoolPointerImplicitConversionCheck.cpp",
+    "BugproneTidyModule.cpp",
+    "CopyConstructorInitCheck.cpp",
+    "DanglingHandleCheck.cpp",
+    "ExceptionEscapeCheck.cpp",
+    "FoldInitTypeCheck.cpp",
+    "ForwardDeclarationNamespaceCheck.cpp",
+    "ForwardingReferenceOverloadCheck.cpp",
+    "InaccurateEraseCheck.cpp",
+    "IncorrectRoundingsCheck.cpp",
+    "IntegerDivisionCheck.cpp",
+    "LambdaFunctionNameCheck.cpp",
+    "MacroParenthesesCheck.cpp",
+    "MacroRepeatedSideEffectsCheck.cpp",
+    "MisplacedOperatorInStrlenInAllocCheck.cpp",
+    "MisplacedWideningCastCheck.cpp",
+    "MoveForwardingReferenceCheck.cpp",
+    "MultipleStatementMacroCheck.cpp",
+    "ParentVirtualCallCheck.cpp",
+    "SizeofContainerCheck.cpp",
+    "SizeofExpressionCheck.cpp",
+    "StringConstructorCheck.cpp",
+    "StringIntegerAssignmentCheck.cpp",
+    "StringLiteralWithEmbeddedNulCheck.cpp",
+    "SuspiciousEnumUsageCheck.cpp",
+    "SuspiciousMemsetUsageCheck.cpp",
+    "SuspiciousMissingCommaCheck.cpp",
+    "SuspiciousSemicolonCheck.cpp",
+    "SuspiciousStringCompareCheck.cpp",
+    "SwappedArgumentsCheck.cpp",
+    "TerminatingContinueCheck.cpp",
+    "ThrowKeywordMissingCheck.cpp",
+    "TooSmallLoopVariableCheck.cpp",
+    "UndefinedMemoryManipulationCheck.cpp",
+    "UndelegatedConstructorCheck.cpp",
+    "UnusedRaiiCheck.cpp",
+    "UnusedReturnValueCheck.cpp",
+    "UseAfterMoveCheck.cpp",
+    "VirtualNearMissCheck.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/clang-tools-extra/clang-tidy/cert/BUILD.gn b/utils/gn/secondary/clang-tools-extra/clang-tidy/cert/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..bd2ac4896616487dd4f01594c9233664164474d7
--- /dev/null
+++ b/utils/gn/secondary/clang-tools-extra/clang-tidy/cert/BUILD.gn
@@ -0,0 +1,31 @@
+static_library("cert") {
+  output_name = "clangTidyCERTModule"
+  configs += [ "//llvm/utils/gn/build:clang_code" ]
+  deps = [
+    "//clang-tools-extra/clang-tidy",
+    "//clang-tools-extra/clang-tidy/google",
+    "//clang-tools-extra/clang-tidy/misc",
+    "//clang-tools-extra/clang-tidy/performance",
+    "//clang-tools-extra/clang-tidy/readability",
+    "//clang-tools-extra/clang-tidy/utils",
+    "//clang/lib/AST",
+    "//clang/lib/ASTMatchers",
+    "//clang/lib/Basic",
+    "//clang/lib/Lex",
+    "//llvm/lib/Support",
+  ]
+  sources = [
+    "CERTTidyModule.cpp",
+    "CommandProcessorCheck.cpp",
+    "DontModifyStdNamespaceCheck.cpp",
+    "FloatLoopCounter.cpp",
+    "LimitedRandomnessCheck.cpp",
+    "PostfixOperatorCheck.cpp",
+    "ProperlySeededRandomGeneratorCheck.cpp",
+    "SetLongJmpCheck.cpp",
+    "StaticObjectExceptionCheck.cpp",
+    "StrToNumCheck.cpp",
+    "ThrownExceptionTypeCheck.cpp",
+    "VariadicFunctionDefCheck.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/clang-tools-extra/clang-tidy/cppcoreguidelines/BUILD.gn b/utils/gn/secondary/clang-tools-extra/clang-tidy/cppcoreguidelines/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..a467d4947bffbed4b66555aa4a78ad659535e937
--- /dev/null
+++ b/utils/gn/secondary/clang-tools-extra/clang-tidy/cppcoreguidelines/BUILD.gn
@@ -0,0 +1,39 @@
+static_library("cppcoreguidelines") {
+  output_name = "clangTidyCppCoreGuidelinesModule"
+  configs += [ "//llvm/utils/gn/build:clang_code" ]
+  deps = [
+    "//clang-tools-extra/clang-tidy",
+    "//clang-tools-extra/clang-tidy/misc",
+    "//clang-tools-extra/clang-tidy/modernize",
+    "//clang-tools-extra/clang-tidy/readability",
+    "//clang-tools-extra/clang-tidy/utils",
+    "//clang/lib/AST",
+    "//clang/lib/ASTMatchers",
+    "//clang/lib/Basic",
+    "//clang/lib/Lex",
+    "//clang/lib/Serialization",
+    "//clang/lib/Tooling",
+    "//llvm/lib/Support",
+  ]
+  sources = [
+    "AvoidGotoCheck.cpp",
+    "CppCoreGuidelinesTidyModule.cpp",
+    "InterfacesGlobalInitCheck.cpp",
+    "MacroUsageCheck.cpp",
+    "NarrowingConversionsCheck.cpp",
+    "NoMallocCheck.cpp",
+    "OwningMemoryCheck.cpp",
+    "ProBoundsArrayToPointerDecayCheck.cpp",
+    "ProBoundsConstantArrayIndexCheck.cpp",
+    "ProBoundsPointerArithmeticCheck.cpp",
+    "ProTypeConstCastCheck.cpp",
+    "ProTypeCstyleCastCheck.cpp",
+    "ProTypeMemberInitCheck.cpp",
+    "ProTypeReinterpretCastCheck.cpp",
+    "ProTypeStaticCastDowncastCheck.cpp",
+    "ProTypeUnionAccessCheck.cpp",
+    "ProTypeVarargCheck.cpp",
+    "SlicingCheck.cpp",
+    "SpecialMemberFunctionsCheck.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/clang-tools-extra/clang-tidy/fuchsia/BUILD.gn b/utils/gn/secondary/clang-tools-extra/clang-tidy/fuchsia/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..ae38a7521b45ae125aec43f8f7c17f6c34afaa1f
--- /dev/null
+++ b/utils/gn/secondary/clang-tools-extra/clang-tidy/fuchsia/BUILD.gn
@@ -0,0 +1,24 @@
+static_library("fuchsia") {
+  output_name = "clangTidyFuchsiaModule"
+  configs += [ "//llvm/utils/gn/build:clang_code" ]
+  deps = [
+    "//clang-tools-extra/clang-tidy",
+    "//clang-tools-extra/clang-tidy/google",
+    "//clang-tools-extra/clang-tidy/utils",
+    "//clang/lib/AST",
+    "//clang/lib/ASTMatchers",
+    "//clang/lib/Basic",
+    "//clang/lib/Lex",
+    "//llvm/lib/Support",
+  ]
+  sources = [
+    "DefaultArgumentsCheck.cpp",
+    "FuchsiaTidyModule.cpp",
+    "MultipleInheritanceCheck.cpp",
+    "OverloadedOperatorCheck.cpp",
+    "RestrictSystemIncludesCheck.cpp",
+    "StaticallyConstructedObjectsCheck.cpp",
+    "TrailingReturnCheck.cpp",
+    "VirtualInheritanceCheck.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/clang-tools-extra/clang-tidy/google/BUILD.gn b/utils/gn/secondary/clang-tools-extra/clang-tidy/google/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..1dcccc71d9a67d77483bb0cc53a35bf0c050401e
--- /dev/null
+++ b/utils/gn/secondary/clang-tools-extra/clang-tidy/google/BUILD.gn
@@ -0,0 +1,32 @@
+static_library("google") {
+  output_name = "clangTidyGoogleModule"
+  configs += [ "//llvm/utils/gn/build:clang_code" ]
+  deps = [
+    "//clang-tools-extra/clang-tidy",
+    "//clang-tools-extra/clang-tidy/readability",
+    "//clang-tools-extra/clang-tidy/utils",
+    "//clang/lib/AST",
+    "//clang/lib/ASTMatchers",
+    "//clang/lib/Basic",
+    "//clang/lib/Lex",
+    "//llvm/lib/Support",
+  ]
+  sources = [
+    "AvoidCStyleCastsCheck.cpp",
+    "AvoidThrowingObjCExceptionCheck.cpp",
+    "AvoidUnderscoreInGoogletestNameCheck.cpp",
+    "DefaultArgumentsCheck.cpp",
+    "ExplicitConstructorCheck.cpp",
+    "ExplicitMakePairCheck.cpp",
+    "FunctionNamingCheck.cpp",
+    "GlobalNamesInHeadersCheck.cpp",
+    "GlobalVariableDeclarationCheck.cpp",
+    "GoogleTidyModule.cpp",
+    "IntegerTypesCheck.cpp",
+    "NonConstReferences.cpp",
+    "OverloadedUnaryAndCheck.cpp",
+    "TodoCommentCheck.cpp",
+    "UnnamedNamespaceInHeaderCheck.cpp",
+    "UsingNamespaceDirectiveCheck.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/clang-tools-extra/clang-tidy/hicpp/BUILD.gn b/utils/gn/secondary/clang-tools-extra/clang-tidy/hicpp/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..cd886d89afcf75a3855c3827e7228238fdc67803
--- /dev/null
+++ b/utils/gn/secondary/clang-tools-extra/clang-tidy/hicpp/BUILD.gn
@@ -0,0 +1,28 @@
+static_library("hicpp") {
+  output_name = "clangTidyHICPPModule"
+  configs += [ "//llvm/utils/gn/build:clang_code" ]
+  deps = [
+    "//clang-tools-extra/clang-tidy",
+    "//clang-tools-extra/clang-tidy/bugprone",
+    "//clang-tools-extra/clang-tidy/cppcoreguidelines",
+    "//clang-tools-extra/clang-tidy/google",
+    "//clang-tools-extra/clang-tidy/misc",
+    "//clang-tools-extra/clang-tidy/modernize",
+    "//clang-tools-extra/clang-tidy/performance",
+    "//clang-tools-extra/clang-tidy/readability",
+    "//clang-tools-extra/clang-tidy/utils",
+    "//clang/lib/AST",
+    "//clang/lib/ASTMatchers",
+    "//clang/lib/Basic",
+    "//clang/lib/Lex",
+    "//clang/lib/Serialization",
+    "//llvm/lib/Support",
+  ]
+  sources = [
+    "ExceptionBaseclassCheck.cpp",
+    "HICPPTidyModule.cpp",
+    "MultiwayPathsCoveredCheck.cpp",
+    "NoAssemblerCheck.cpp",
+    "SignedBitwiseCheck.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/clang-tools-extra/clang-tidy/llvm/BUILD.gn b/utils/gn/secondary/clang-tools-extra/clang-tidy/llvm/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..8f5026a995c723d2cc9d5284dfdbd1b8bc59fa33
--- /dev/null
+++ b/utils/gn/secondary/clang-tools-extra/clang-tidy/llvm/BUILD.gn
@@ -0,0 +1,21 @@
+static_library("llvm") {
+  output_name = "clangTidyLLVMModule"
+  configs += [ "//llvm/utils/gn/build:clang_code" ]
+  deps = [
+    "//clang-tools-extra/clang-tidy",
+    "//clang-tools-extra/clang-tidy/readability",
+    "//clang-tools-extra/clang-tidy/utils",
+    "//clang/lib/AST",
+    "//clang/lib/ASTMatchers",
+    "//clang/lib/Basic",
+    "//clang/lib/Lex",
+    "//clang/lib/Tooling",
+    "//llvm/lib/Support",
+  ]
+  sources = [
+    "HeaderGuardCheck.cpp",
+    "IncludeOrderCheck.cpp",
+    "LLVMTidyModule.cpp",
+    "TwineLocalCheck.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/clang-tools-extra/clang-tidy/misc/BUILD.gn b/utils/gn/secondary/clang-tools-extra/clang-tidy/misc/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..a7a1abd0d41a40854794acfe5bf29ad67bcdc8ca
--- /dev/null
+++ b/utils/gn/secondary/clang-tools-extra/clang-tidy/misc/BUILD.gn
@@ -0,0 +1,32 @@
+static_library("misc") {
+  output_name = "clangTidyMiscModule"
+  configs += [ "//llvm/utils/gn/build:clang_code" ]
+  deps = [
+    "//clang-tools-extra/clang-tidy",
+    "//clang-tools-extra/clang-tidy/utils",
+    "//clang/lib/AST",
+    "//clang/lib/ASTMatchers",
+    "//clang/lib/Analysis",
+    "//clang/lib/Basic",
+    "//clang/lib/Lex",
+    "//clang/lib/Serialization",
+    "//clang/lib/Tooling",
+    "//llvm/lib/Support",
+  ]
+  sources = [
+    "DefinitionsInHeadersCheck.cpp",
+    "MiscTidyModule.cpp",
+    "MisplacedConstCheck.cpp",
+    "NewDeleteOverloadsCheck.cpp",
+    "NonCopyableObjects.cpp",
+    "NonPrivateMemberVariablesInClassesCheck.cpp",
+    "RedundantExpressionCheck.cpp",
+    "StaticAssertCheck.cpp",
+    "ThrowByValueCatchByReferenceCheck.cpp",
+    "UnconventionalAssignOperatorCheck.cpp",
+    "UniqueptrResetReleaseCheck.cpp",
+    "UnusedAliasDeclsCheck.cpp",
+    "UnusedParametersCheck.cpp",
+    "UnusedUsingDeclsCheck.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/clang-tools-extra/clang-tidy/modernize/BUILD.gn b/utils/gn/secondary/clang-tools-extra/clang-tidy/modernize/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..ff23cf28bb8af4861254573495d2ca6db54b86ea
--- /dev/null
+++ b/utils/gn/secondary/clang-tools-extra/clang-tidy/modernize/BUILD.gn
@@ -0,0 +1,49 @@
+static_library("modernize") {
+  output_name = "clangTidyModernizeModule"
+  configs += [ "//llvm/utils/gn/build:clang_code" ]
+  deps = [
+    "//clang-tools-extra/clang-tidy",
+    "//clang-tools-extra/clang-tidy/readability",
+    "//clang-tools-extra/clang-tidy/utils",
+    "//clang/lib/AST",
+    "//clang/lib/ASTMatchers",
+    "//clang/lib/Basic",
+    "//clang/lib/Lex",
+    "//clang/lib/Tooling",
+    "//llvm/lib/Support",
+  ]
+  sources = [
+    "AvoidBindCheck.cpp",
+    "AvoidCArraysCheck.cpp",
+    "ConcatNestedNamespacesCheck.cpp",
+    "DeprecatedHeadersCheck.cpp",
+    "DeprecatedIosBaseAliasesCheck.cpp",
+    "LoopConvertCheck.cpp",
+    "LoopConvertUtils.cpp",
+    "MakeSharedCheck.cpp",
+    "MakeSmartPtrCheck.cpp",
+    "MakeUniqueCheck.cpp",
+    "ModernizeTidyModule.cpp",
+    "PassByValueCheck.cpp",
+    "RawStringLiteralCheck.cpp",
+    "RedundantVoidArgCheck.cpp",
+    "ReplaceAutoPtrCheck.cpp",
+    "ReplaceRandomShuffleCheck.cpp",
+    "ReturnBracedInitListCheck.cpp",
+    "ShrinkToFitCheck.cpp",
+    "UnaryStaticAssertCheck.cpp",
+    "UseAutoCheck.cpp",
+    "UseBoolLiteralsCheck.cpp",
+    "UseDefaultMemberInitCheck.cpp",
+    "UseEmplaceCheck.cpp",
+    "UseEqualsDefaultCheck.cpp",
+    "UseEqualsDeleteCheck.cpp",
+    "UseNodiscardCheck.cpp",
+    "UseNoexceptCheck.cpp",
+    "UseNullptrCheck.cpp",
+    "UseOverrideCheck.cpp",
+    "UseTransparentFunctorsCheck.cpp",
+    "UseUncaughtExceptionsCheck.cpp",
+    "UseUsingCheck.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/clang-tools-extra/clang-tidy/mpi/BUILD.gn b/utils/gn/secondary/clang-tools-extra/clang-tidy/mpi/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..876a53794693243db65378abee1877881b5d8ec3
--- /dev/null
+++ b/utils/gn/secondary/clang-tools-extra/clang-tidy/mpi/BUILD.gn
@@ -0,0 +1,21 @@
+static_library("mpi") {
+  output_name = "clangTidyMPIModule"
+  configs += [ "//llvm/utils/gn/build:clang_code" ]
+  deps = [
+    "//clang-tools-extra/clang-tidy",
+    "//clang-tools-extra/clang-tidy/utils",
+    "//clang/lib/AST",
+    "//clang/lib/ASTMatchers",
+    "//clang/lib/Analysis",
+    "//clang/lib/Basic",
+    "//clang/lib/Lex",
+    "//clang/lib/StaticAnalyzer/Checkers",
+    "//clang/lib/Tooling",
+    "//llvm/lib/Support",
+  ]
+  sources = [
+    "BufferDerefCheck.cpp",
+    "MPITidyModule.cpp",
+    "TypeMismatchCheck.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/clang-tools-extra/clang-tidy/objc/BUILD.gn b/utils/gn/secondary/clang-tools-extra/clang-tidy/objc/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..1697baa2588892d7f03db34138ac7b6d098f18a7
--- /dev/null
+++ b/utils/gn/secondary/clang-tools-extra/clang-tidy/objc/BUILD.gn
@@ -0,0 +1,20 @@
+static_library("objc") {
+  output_name = "clangTidyObjCModule"
+  configs += [ "//llvm/utils/gn/build:clang_code" ]
+  deps = [
+    "//clang-tools-extra/clang-tidy",
+    "//clang-tools-extra/clang-tidy/utils",
+    "//clang/lib/AST",
+    "//clang/lib/ASTMatchers",
+    "//clang/lib/Basic",
+    "//clang/lib/Lex",
+    "//llvm/lib/Support",
+  ]
+  sources = [
+    "AvoidNSErrorInitCheck.cpp",
+    "AvoidSpinlockCheck.cpp",
+    "ForbiddenSubclassingCheck.cpp",
+    "ObjCTidyModule.cpp",
+    "PropertyDeclarationCheck.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/clang-tools-extra/clang-tidy/performance/BUILD.gn b/utils/gn/secondary/clang-tools-extra/clang-tidy/performance/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..a8c7475b23c2fd007911a7c89df50427c8dc1fd7
--- /dev/null
+++ b/utils/gn/secondary/clang-tools-extra/clang-tidy/performance/BUILD.gn
@@ -0,0 +1,29 @@
+static_library("performance") {
+  output_name = "clangTidyPerformanceModule"
+  configs += [ "//llvm/utils/gn/build:clang_code" ]
+  deps = [
+    "//clang-tools-extra/clang-tidy",
+    "//clang-tools-extra/clang-tidy/utils",
+    "//clang/lib/AST",
+    "//clang/lib/ASTMatchers",
+    "//clang/lib/Analysis",
+    "//clang/lib/Basic",
+    "//clang/lib/Lex",
+    "//llvm/lib/Support",
+  ]
+  sources = [
+    "FasterStringFindCheck.cpp",
+    "ForRangeCopyCheck.cpp",
+    "ImplicitConversionInLoopCheck.cpp",
+    "InefficientAlgorithmCheck.cpp",
+    "InefficientStringConcatenationCheck.cpp",
+    "InefficientVectorOperationCheck.cpp",
+    "MoveConstArgCheck.cpp",
+    "MoveConstructorInitCheck.cpp",
+    "NoexceptMoveConstructorCheck.cpp",
+    "PerformanceTidyModule.cpp",
+    "TypePromotionInMathFnCheck.cpp",
+    "UnnecessaryCopyInitialization.cpp",
+    "UnnecessaryValueParamCheck.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/clang-tools-extra/clang-tidy/plugin/BUILD.gn b/utils/gn/secondary/clang-tools-extra/clang-tidy/plugin/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..1d033e87ee52878fae21fd9f17bf3adb87674cfc
--- /dev/null
+++ b/utils/gn/secondary/clang-tools-extra/clang-tidy/plugin/BUILD.gn
@@ -0,0 +1,38 @@
+static_library("plugin") {
+  output_name = "clangTidyPlugin"
+  configs += [ "//llvm/utils/gn/build:clang_code" ]
+  deps = [
+    "//clang-tools-extra/clang-tidy",
+    "//clang-tools-extra/clang-tidy/abseil",
+    "//clang-tools-extra/clang-tidy/android",
+    "//clang-tools-extra/clang-tidy/boost",
+    "//clang-tools-extra/clang-tidy/bugprone",
+    "//clang-tools-extra/clang-tidy/cert",
+    "//clang-tools-extra/clang-tidy/cppcoreguidelines",
+    "//clang-tools-extra/clang-tidy/fuchsia",
+    "//clang-tools-extra/clang-tidy/google",
+    "//clang-tools-extra/clang-tidy/hicpp",
+    "//clang-tools-extra/clang-tidy/llvm",
+    "//clang-tools-extra/clang-tidy/misc",
+    "//clang-tools-extra/clang-tidy/modernize",
+    "//clang-tools-extra/clang-tidy/objc",
+    "//clang-tools-extra/clang-tidy/performance",
+    "//clang-tools-extra/clang-tidy/portability",
+    "//clang-tools-extra/clang-tidy/readability",
+    "//clang-tools-extra/clang-tidy/zircon",
+    "//clang/lib/AST",
+    "//clang/lib/ASTMatchers",
+    "//clang/lib/Basic",
+    "//clang/lib/Frontend",
+    "//clang/lib/Sema",
+    "//clang/lib/Tooling",
+  ]
+
+  if (clang_enable_static_analyzer) {
+    deps += [ "//clang-tools-extra/clang-tidy/mpi" ]
+  }
+
+  sources = [
+    "ClangTidyPlugin.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/clang-tools-extra/clang-tidy/portability/BUILD.gn b/utils/gn/secondary/clang-tools-extra/clang-tidy/portability/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..480b7190f4e559789ab497cd90dd17c335dab597
--- /dev/null
+++ b/utils/gn/secondary/clang-tools-extra/clang-tidy/portability/BUILD.gn
@@ -0,0 +1,18 @@
+static_library("portability") {
+  output_name = "clangTidyPortabilityModule"
+  configs += [ "//llvm/utils/gn/build:clang_code" ]
+  deps = [
+    "//clang-tools-extra/clang-tidy",
+    "//clang-tools-extra/clang-tidy/utils",
+    "//clang/lib/AST",
+    "//clang/lib/ASTMatchers",
+    "//clang/lib/Basic",
+    "//clang/lib/Lex",
+    "//clang/lib/Tooling",
+    "//llvm/lib/Support",
+  ]
+  sources = [
+    "PortabilityTidyModule.cpp",
+    "SIMDIntrinsicsCheck.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/clang-tools-extra/clang-tidy/readability/BUILD.gn b/utils/gn/secondary/clang-tools-extra/clang-tidy/readability/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..2bd2a69b4e6aed4f84e74566718842b991bc1554
--- /dev/null
+++ b/utils/gn/secondary/clang-tools-extra/clang-tidy/readability/BUILD.gn
@@ -0,0 +1,50 @@
+static_library("readability") {
+  output_name = "clangTidyReadabilityModule"
+  configs += [ "//llvm/utils/gn/build:clang_code" ]
+  deps = [
+    "//clang-tools-extra/clang-tidy",
+    "//clang-tools-extra/clang-tidy/utils",
+    "//clang/lib/AST",
+    "//clang/lib/ASTMatchers",
+    "//clang/lib/Basic",
+    "//clang/lib/Lex",
+    "//clang/lib/Tooling",
+    "//llvm/lib/Support",
+  ]
+  sources = [
+    "AvoidConstParamsInDecls.cpp",
+    "BracesAroundStatementsCheck.cpp",
+    "ConstReturnTypeCheck.cpp",
+    "ContainerSizeEmptyCheck.cpp",
+    "DeleteNullPointerCheck.cpp",
+    "DeletedDefaultCheck.cpp",
+    "ElseAfterReturnCheck.cpp",
+    "FunctionSizeCheck.cpp",
+    "IdentifierNamingCheck.cpp",
+    "ImplicitBoolConversionCheck.cpp",
+    "InconsistentDeclarationParameterNameCheck.cpp",
+    "IsolateDeclarationCheck.cpp",
+    "MagicNumbersCheck.cpp",
+    "MisleadingIndentationCheck.cpp",
+    "MisplacedArrayIndexCheck.cpp",
+    "NamedParameterCheck.cpp",
+    "NamespaceCommentCheck.cpp",
+    "NonConstParameterCheck.cpp",
+    "ReadabilityTidyModule.cpp",
+    "RedundantControlFlowCheck.cpp",
+    "RedundantDeclarationCheck.cpp",
+    "RedundantFunctionPtrDereferenceCheck.cpp",
+    "RedundantMemberInitCheck.cpp",
+    "RedundantPreprocessorCheck.cpp",
+    "RedundantSmartptrGetCheck.cpp",
+    "RedundantStringCStrCheck.cpp",
+    "RedundantStringInitCheck.cpp",
+    "SimplifyBooleanExprCheck.cpp",
+    "SimplifySubscriptExprCheck.cpp",
+    "StaticAccessedThroughInstanceCheck.cpp",
+    "StaticDefinitionInAnonymousNamespaceCheck.cpp",
+    "StringCompareCheck.cpp",
+    "UniqueptrDeleteReleaseCheck.cpp",
+    "UppercaseLiteralSuffixCheck.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/clang-tools-extra/clang-tidy/tool/BUILD.gn b/utils/gn/secondary/clang-tools-extra/clang-tidy/tool/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..c8b378d3b66f32ef845ae347faf41573afcd2590
--- /dev/null
+++ b/utils/gn/secondary/clang-tools-extra/clang-tidy/tool/BUILD.gn
@@ -0,0 +1,40 @@
+import("//clang/lib/StaticAnalyzer/Frontend/enable.gni")
+
+executable("clang-tidy") {
+  configs += [ "//llvm/utils/gn/build:clang_code" ]
+  deps = [
+    "//clang-tools-extra/clang-tidy",
+    "//clang-tools-extra/clang-tidy/abseil",
+    "//clang-tools-extra/clang-tidy/android",
+    "//clang-tools-extra/clang-tidy/boost",
+    "//clang-tools-extra/clang-tidy/bugprone",
+    "//clang-tools-extra/clang-tidy/cert",
+    "//clang-tools-extra/clang-tidy/cppcoreguidelines",
+    "//clang-tools-extra/clang-tidy/fuchsia",
+    "//clang-tools-extra/clang-tidy/google",
+    "//clang-tools-extra/clang-tidy/hicpp",
+    "//clang-tools-extra/clang-tidy/llvm",
+    "//clang-tools-extra/clang-tidy/misc",
+    "//clang-tools-extra/clang-tidy/modernize",
+    "//clang-tools-extra/clang-tidy/objc",
+    "//clang-tools-extra/clang-tidy/performance",
+    "//clang-tools-extra/clang-tidy/portability",
+    "//clang-tools-extra/clang-tidy/readability",
+    "//clang-tools-extra/clang-tidy/zircon",
+    "//clang/lib/AST",
+    "//clang/lib/ASTMatchers",
+    "//clang/lib/Basic",
+    "//clang/lib/Tooling",
+    "//clang/lib/Tooling/Core",
+    "//llvm/lib/Support",
+    "//llvm/lib/Target:AllTargetsAsmParsers",
+    "//llvm/lib/Target:AllTargetsDescs",
+    "//llvm/lib/Target:AllTargetsInfos",
+  ]
+  if (clang_enable_static_analyzer) {
+    deps += [ "//clang-tools-extra/clang-tidy/mpi" ]
+  }
+  sources = [
+    "ClangTidyMain.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/clang-tools-extra/clang-tidy/utils/BUILD.gn b/utils/gn/secondary/clang-tools-extra/clang-tidy/utils/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..abb98917b333f10b6c4322370e9630b333d82f97
--- /dev/null
+++ b/utils/gn/secondary/clang-tools-extra/clang-tidy/utils/BUILD.gn
@@ -0,0 +1,28 @@
+static_library("utils") {
+  output_name = "clangTidyUtils"
+  configs += [ "//llvm/utils/gn/build:clang_code" ]
+  deps = [
+    "//clang-tools-extra/clang-tidy",
+    "//clang/lib/AST",
+    "//clang/lib/ASTMatchers",
+    "//clang/lib/Basic",
+    "//clang/lib/Lex",
+    "//llvm/lib/Support",
+  ]
+  sources = [
+    "ASTUtils.cpp",
+    "DeclRefExprUtils.cpp",
+    "ExceptionAnalyzer.cpp",
+    "ExprSequence.cpp",
+    "FixItHintUtils.cpp",
+    "HeaderFileExtensionsUtils.cpp",
+    "HeaderGuard.cpp",
+    "IncludeInserter.cpp",
+    "IncludeSorter.cpp",
+    "LexerUtils.cpp",
+    "NamespaceAliaser.cpp",
+    "OptionsUtils.cpp",
+    "TypeTraits.cpp",
+    "UsingInserter.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/clang-tools-extra/clang-tidy/zircon/BUILD.gn b/utils/gn/secondary/clang-tools-extra/clang-tidy/zircon/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..c349414afc32780829a63396169ae1f4ff66c50c
--- /dev/null
+++ b/utils/gn/secondary/clang-tools-extra/clang-tidy/zircon/BUILD.gn
@@ -0,0 +1,17 @@
+static_library("zircon") {
+  output_name = "clangTidyZirconModule"
+  configs += [ "//llvm/utils/gn/build:clang_code" ]
+  deps = [
+    "//clang-tools-extra/clang-tidy",
+    "//clang-tools-extra/clang-tidy/utils",
+    "//clang/lib/AST",
+    "//clang/lib/ASTMatchers",
+    "//clang/lib/Basic",
+    "//clang/lib/Lex",
+    "//llvm/lib/Support",
+  ]
+  sources = [
+    "TemporaryObjectsCheck.cpp",
+    "ZirconTidyModule.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/clang/include/clang/AST/BUILD.gn b/utils/gn/secondary/clang/include/clang/AST/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..6ac4a72890604b6c6ac3b5be6071eadfdb6b74a6
--- /dev/null
+++ b/utils/gn/secondary/clang/include/clang/AST/BUILD.gn
@@ -0,0 +1,88 @@
+import("//clang/utils/TableGen/clang_tablegen.gni")
+
+clang_tablegen("Attrs") {
+  args = [
+    "-gen-clang-attr-classes",
+    "-I",
+    rebase_path("../..", root_out_dir),
+  ]
+  td_file = "../Basic/Attr.td"
+}
+
+clang_tablegen("AttrImpl") {
+  args = [
+    "-gen-clang-attr-impl",
+    "-I",
+    rebase_path("../..", root_out_dir),
+  ]
+  td_file = "../Basic/Attr.td"
+}
+
+clang_tablegen("AttrTextNodeDump") {
+  args = [
+    "-gen-clang-attr-text-node-dump",
+    "-I",
+    rebase_path("../..", root_out_dir),
+  ]
+  td_file = "../Basic/Attr.td"
+}
+
+clang_tablegen("AttrNodeTraverse") {
+  args = [
+    "-gen-clang-attr-node-traverse",
+    "-I",
+    rebase_path("../..", root_out_dir),
+  ]
+  td_file = "../Basic/Attr.td"
+}
+
+clang_tablegen("AttrVisitor") {
+  args = [
+    "-gen-clang-attr-ast-visitor",
+    "-I",
+    rebase_path("../..", root_out_dir),
+  ]
+  td_file = "../Basic/Attr.td"
+}
+
+clang_tablegen("StmtNodes") {
+  args = [ "-gen-clang-stmt-nodes" ]
+  td_file = "../Basic/StmtNodes.td"
+}
+
+clang_tablegen("DeclNodes") {
+  args = [ "-gen-clang-decl-nodes" ]
+  td_file = "../Basic/DeclNodes.td"
+}
+
+clang_tablegen("CommentNodes") {
+  args = [ "-gen-clang-comment-nodes" ]
+  td_file = "../Basic/CommentNodes.td"
+}
+
+clang_tablegen("CommentHTMLTags") {
+  args = [ "-gen-clang-comment-html-tags" ]
+}
+
+clang_tablegen("CommentHTMLTagsProperties") {
+  args = [ "-gen-clang-comment-html-tags-properties" ]
+  td_file = "CommentHTMLTags.td"
+}
+
+clang_tablegen("CommentHTMLNamedCharacterReferences") {
+  args = [ "-gen-clang-comment-html-named-character-references" ]
+}
+
+clang_tablegen("CommentCommandInfo") {
+  args = [ "-gen-clang-comment-command-info" ]
+  td_file = "CommentCommands.td"
+}
+
+clang_tablegen("CommentCommandList") {
+  args = [ "-gen-clang-comment-command-list" ]
+  td_file = "CommentCommands.td"
+}
+
+clang_tablegen("StmtDataCollectors") {
+  args = [ "-gen-clang-data-collectors" ]
+}
diff --git a/utils/gn/secondary/clang/include/clang/Basic/BUILD.gn b/utils/gn/secondary/clang/include/clang/Basic/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..7d67a2f96ce8e71ea17d22c08b3b5d3f7ca5989d
--- /dev/null
+++ b/utils/gn/secondary/clang/include/clang/Basic/BUILD.gn
@@ -0,0 +1,100 @@
+import("//clang/utils/TableGen/clang_tablegen.gni")
+import("//llvm/utils/gn/build/write_cmake_config.gni")
+import("//llvm/version.gni")
+
+# Version header.
+
+write_cmake_config("version") {
+  input = "Version.inc.in"
+  output = "$target_gen_dir/Version.inc"
+  values = [
+    "CLANG_VERSION=$llvm_version",
+    "CLANG_VERSION_MAJOR=$llvm_version_major",
+    "CLANG_VERSION_MINOR=$llvm_version_minor",
+    "CLANG_VERSION_PATCHLEVEL=$llvm_version_patch",
+  ]
+}
+
+# Diagnostics.
+
+diag_groups = [
+  "Analysis",
+  "AST",
+  "Comment",
+  "Common",
+  "CrossTU",
+  "Driver",
+  "Frontend",
+  "Lex",
+  "Parse",
+  "Refactoring",
+  "Sema",
+  "Serialization",
+]
+foreach(diag_group, diag_groups) {
+  clang_tablegen("Diagnostic${diag_group}Kinds") {
+    args = [
+      "-gen-clang-diags-defs",
+      "-clang-component=${diag_group}",
+    ]
+    td_file = "Diagnostic.td"
+  }
+}
+group("diags_tablegen") {
+  # DiagnosticGroups and DiagnosticIndexName are intentionally not part of this
+  # group.  Much of clang depends on the DiagKinds.inc files transitively,
+  # but almost nothing needs DiagnosticGroups.inc or DiagnosticIndexName.inc.
+  public_deps = []
+  foreach(diag_group, diag_groups) {
+    public_deps += [ ":Diagnostic${diag_group}Kinds" ]
+  }
+}
+
+clang_tablegen("DiagnosticGroups") {
+  args = [ "-gen-clang-diag-groups" ]
+  td_file = "Diagnostic.td"
+}
+
+clang_tablegen("DiagnosticIndexName") {
+  args = [ "-gen-clang-diags-index-name" ]
+  td_file = "Diagnostic.td"
+}
+
+# Attributes
+
+clang_tablegen("AttrList") {
+  args = [
+    "-gen-clang-attr-list",
+    "-I",
+    rebase_path("../..", root_out_dir),
+  ]
+  td_file = "Attr.td"
+}
+
+clang_tablegen("AttrSubMatchRulesList") {
+  args = [
+    "-gen-clang-attr-subject-match-rule-list",
+    "-I",
+    rebase_path("../..", root_out_dir),
+  ]
+  td_file = "Attr.td"
+}
+
+clang_tablegen("AttrHasAttributeImpl") {
+  args = [
+    "-gen-clang-attr-has-attribute-impl",
+    "-I",
+    rebase_path("../..", root_out_dir),
+  ]
+  td_file = "Attr.td"
+}
+
+# Misc
+
+clang_tablegen("arm_neon") {
+  args = [ "-gen-arm-neon-sema" ]
+}
+
+clang_tablegen("arm_fp16") {
+  args = [ "-gen-arm-neon-sema" ]
+}
diff --git a/utils/gn/secondary/clang/include/clang/Config/BUILD.gn b/utils/gn/secondary/clang/include/clang/Config/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..a3c0d5a05242869af7e6abd9a97b97bcd01288e4
--- /dev/null
+++ b/utils/gn/secondary/clang/include/clang/Config/BUILD.gn
@@ -0,0 +1,75 @@
+import("//clang/lib/ARCMigrate/enable.gni")
+import("//clang/lib/StaticAnalyzer/Frontend/enable.gni")
+import("//llvm/utils/gn/build/libs/xml/enable.gni")
+import("//llvm/utils/gn/build/write_cmake_config.gni")
+import("//llvm/version.gni")
+
+config("Config_config") {
+  visibility = [ ":Config" ]
+  include_dirs = [ "$target_gen_dir/clang/include" ]
+}
+
+write_cmake_config("Config") {
+  input = "config.h.cmake"
+  output = "$target_gen_dir/config.h"
+  values = [
+    "BUG_REPORT_URL=https://bugs.llvm.org/",
+    "CLANG_DEFAULT_LINKER=",
+    "CLANG_DEFAULT_STD_C=",
+    "CLANG_DEFAULT_STD_CXX=",
+    "CLANG_DEFAULT_CXX_STDLIB=",
+    "CLANG_DEFAULT_RTLIB=",
+    "CLANG_DEFAULT_OBJCOPY=objcopy",
+    "CLANG_DEFAULT_OPENMP_RUNTIME=libomp",
+    "CLANG_OPENMP_NVPTX_DEFAULT_ARCH=sm_35",
+    "CLANG_LIBDIR_SUFFIX=",
+    "CLANG_RESOURCE_DIR=",
+    "C_INCLUDE_DIRS=",
+    "CLANG_CONFIG_FILE_SYSTEM_DIR=",
+    "CLANG_CONFIG_FILE_USER_DIR=",
+    "DEFAULT_SYSROOT=",
+    "GCC_INSTALL_PREFIX=",
+    "CLANG_ANALYZER_WITH_Z3=",
+    "BACKEND_PACKAGE_STRING=LLVM ${llvm_version}svn",
+    "ENABLE_LINKER_BUILD_ID=",
+    "ENABLE_X86_RELAX_RELOCATIONS=",
+    "ENABLE_EXPERIMENTAL_NEW_PASS_MANAGER=",
+    "CLANG_ENABLE_OBJC_REWRITER=1",  # FIXME: flag?
+  ]
+
+  if (clang_enable_arcmt) {
+    values += [ "CLANG_ENABLE_ARCMT=1" ]
+  } else {
+    values += [ "CLANG_ENABLE_ARCMT=" ]
+  }
+
+  if (clang_enable_static_analyzer) {
+    values += [ "CLANG_ENABLE_STATIC_ANALYZER=1" ]
+  } else {
+    values += [ "CLANG_ENABLE_STATIC_ANALYZER=" ]
+  }
+
+  if (host_os != "win") {
+    values += [ "CLANG_HAVE_RLIMITS=1" ]
+  } else {
+    values += [ "CLANG_HAVE_RLIMITS=" ]
+  }
+
+  if (llvm_enable_libxml2) {
+    values += [ "CLANG_HAVE_LIBXML=1" ]
+  } else {
+    values += [ "CLANG_HAVE_LIBXML=" ]
+  }
+
+  if (host_os == "mac") {
+    # FIXME: Hardcoding this isn't great, but assuming that the host ld version
+    # has anything to do with the ld version where the built clang will run
+    # isn't either. Probably want to make this a declare_args.
+    values += [ "HOST_LINK_VERSION=305" ]
+  } else {
+    values += [ "HOST_LINK_VERSION=" ]
+  }
+
+  # Let targets depending on this find the generated file.
+  public_configs = [ ":Config_config" ]
+}
diff --git a/utils/gn/secondary/clang/include/clang/Driver/BUILD.gn b/utils/gn/secondary/clang/include/clang/Driver/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..955e786cfe35b8efd9afc4928f30cade67cfcb79
--- /dev/null
+++ b/utils/gn/secondary/clang/include/clang/Driver/BUILD.gn
@@ -0,0 +1,5 @@
+import("//llvm/utils/TableGen/tablegen.gni")
+
+tablegen("Options") {
+  args = [ "-gen-opt-parser-defs" ]
+}
diff --git a/utils/gn/secondary/clang/include/clang/Parse/BUILD.gn b/utils/gn/secondary/clang/include/clang/Parse/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..24aaaeae80c8cd90fbe019f03456b57649ac2b86
--- /dev/null
+++ b/utils/gn/secondary/clang/include/clang/Parse/BUILD.gn
@@ -0,0 +1,19 @@
+import("//clang/utils/TableGen/clang_tablegen.gni")
+
+clang_tablegen("AttrParserStringSwitches") {
+  args = [
+    "-gen-clang-attr-parser-string-switches",
+    "-I",
+    rebase_path("../..", root_out_dir),
+  ]
+  td_file = "../Basic/Attr.td"
+}
+
+clang_tablegen("AttrSubMatchRulesParserStringSwitches") {
+  args = [
+    "-gen-clang-attr-subject-match-rules-parser-string-switches",
+    "-I",
+    rebase_path("../..", root_out_dir),
+  ]
+  td_file = "../Basic/Attr.td"
+}
diff --git a/utils/gn/secondary/clang/include/clang/Sema/BUILD.gn b/utils/gn/secondary/clang/include/clang/Sema/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..1f41189e8abb5fbaf11cf23782f585841a20e7f2
--- /dev/null
+++ b/utils/gn/secondary/clang/include/clang/Sema/BUILD.gn
@@ -0,0 +1,46 @@
+import("//clang/utils/TableGen/clang_tablegen.gni")
+
+clang_tablegen("AttrTemplateInstantiate") {
+  args = [
+    "-gen-clang-attr-template-instantiate",
+    "-I",
+    rebase_path("../..", root_out_dir),
+  ]
+  td_file = "../Basic/Attr.td"
+}
+
+clang_tablegen("AttrParsedAttrList") {
+  args = [
+    "-gen-clang-attr-parsed-attr-list",
+    "-I",
+    rebase_path("../..", root_out_dir),
+  ]
+  td_file = "../Basic/Attr.td"
+}
+
+clang_tablegen("AttrParsedAttrKinds") {
+  args = [
+    "-gen-clang-attr-parsed-attr-kinds",
+    "-I",
+    rebase_path("../..", root_out_dir),
+  ]
+  td_file = "../Basic/Attr.td"
+}
+
+clang_tablegen("AttrSpellingListIndex") {
+  args = [
+    "-gen-clang-attr-spelling-index",
+    "-I",
+    rebase_path("../..", root_out_dir),
+  ]
+  td_file = "../Basic/Attr.td"
+}
+
+clang_tablegen("AttrParsedAttrImpl") {
+  args = [
+    "-gen-clang-attr-parsed-attr-impl",
+    "-I",
+    rebase_path("../..", root_out_dir),
+  ]
+  td_file = "../Basic/Attr.td"
+}
diff --git a/utils/gn/secondary/clang/include/clang/Serialization/BUILD.gn b/utils/gn/secondary/clang/include/clang/Serialization/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..bf90e0667c926e28d62dc8768ce1ae4f04caed1c
--- /dev/null
+++ b/utils/gn/secondary/clang/include/clang/Serialization/BUILD.gn
@@ -0,0 +1,19 @@
+import("//clang/utils/TableGen/clang_tablegen.gni")
+
+clang_tablegen("AttrPCHRead") {
+  args = [
+    "-gen-clang-attr-pch-read",
+    "-I",
+    rebase_path("../..", root_out_dir),
+  ]
+  td_file = "../Basic/Attr.td"
+}
+
+clang_tablegen("AttrPCHWrite") {
+  args = [
+    "-gen-clang-attr-pch-write",
+    "-I",
+    rebase_path("../..", root_out_dir),
+  ]
+  td_file = "../Basic/Attr.td"
+}
diff --git a/utils/gn/secondary/clang/include/clang/StaticAnalyzer/Checkers/BUILD.gn b/utils/gn/secondary/clang/include/clang/StaticAnalyzer/Checkers/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..1921a57d82536fab0e124141125bc6f7e9f7f0f0
--- /dev/null
+++ b/utils/gn/secondary/clang/include/clang/StaticAnalyzer/Checkers/BUILD.gn
@@ -0,0 +1,5 @@
+import("//clang/utils/TableGen/clang_tablegen.gni")
+
+clang_tablegen("Checkers") {
+  args = [ "-gen-clang-sa-checkers" ]
+}
diff --git a/utils/gn/secondary/clang/lib/ARCMigrate/BUILD.gn b/utils/gn/secondary/clang/lib/ARCMigrate/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..923ff66b1d2c721e3b4c642f1e980906f2afe713
--- /dev/null
+++ b/utils/gn/secondary/clang/lib/ARCMigrate/BUILD.gn
@@ -0,0 +1,38 @@
+static_library("ARCMigrate") {
+  output_name = "clangARCMigrate"
+  configs += [ "//llvm/utils/gn/build:clang_code" ]
+  deps = [
+    "//clang/lib/AST",
+    "//clang/lib/Analysis",
+    "//clang/lib/Basic",
+    "//clang/lib/Edit",
+    "//clang/lib/Frontend",
+    "//clang/lib/Lex",
+    "//clang/lib/Rewrite",
+    "//clang/lib/Sema",
+    "//clang/lib/Serialization",
+    "//llvm/lib/Support",
+  ]
+  sources = [
+    "ARCMT.cpp",
+    "ARCMTActions.cpp",
+    "FileRemapper.cpp",
+    "ObjCMT.cpp",
+    "PlistReporter.cpp",
+    "TransAPIUses.cpp",
+    "TransARCAssign.cpp",
+    "TransAutoreleasePool.cpp",
+    "TransBlockObjCVariable.cpp",
+    "TransEmptyStatementsAndDealloc.cpp",
+    "TransGCAttrs.cpp",
+    "TransGCCalls.cpp",
+    "TransProperties.cpp",
+    "TransProtectedScope.cpp",
+    "TransRetainReleaseDealloc.cpp",
+    "TransUnbridgedCasts.cpp",
+    "TransUnusedInitDelegate.cpp",
+    "TransZeroOutPropsInDealloc.cpp",
+    "TransformActions.cpp",
+    "Transforms.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/clang/lib/ARCMigrate/enable.gni b/utils/gn/secondary/clang/lib/ARCMigrate/enable.gni
new file mode 100644
index 0000000000000000000000000000000000000000..7a7a42edbfd42bc916887e6657880f6bb7597f01
--- /dev/null
+++ b/utils/gn/secondary/clang/lib/ARCMigrate/enable.gni
@@ -0,0 +1,4 @@
+declare_args() {
+  # Whether to include the arc migrate tool in the clang binary.
+  clang_enable_arcmt = true
+}
diff --git a/utils/gn/secondary/clang/lib/AST/BUILD.gn b/utils/gn/secondary/clang/lib/AST/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..fc3b0b9b0f86c9c4b30d15052b939fdab0c7403a
--- /dev/null
+++ b/utils/gn/secondary/clang/lib/AST/BUILD.gn
@@ -0,0 +1,102 @@
+static_library("AST") {
+  output_name = "clangAST"
+  configs += [ "//llvm/utils/gn/build:clang_code" ]
+  deps = [
+    "//clang/include/clang/AST:AttrImpl",
+    "//clang/include/clang/AST:AttrNodeTraverse",
+    "//clang/include/clang/AST:AttrTextNodeDump",
+    "//clang/include/clang/AST:CommentCommandInfo",
+    "//clang/include/clang/AST:CommentHTMLNamedCharacterReferences",
+    "//clang/include/clang/AST:CommentHTMLTags",
+    "//clang/include/clang/AST:CommentHTMLTagsProperties",
+    "//clang/include/clang/AST:DeclNodes",
+    "//clang/lib/Basic",
+    "//clang/lib/Lex",
+    "//llvm/lib/BinaryFormat",
+    "//llvm/lib/Support",
+  ]
+
+  # Generated files used in public headers should be in public_deps, the rest
+  # in regular deps.
+  public_deps = [
+    "//clang/include/clang/AST:AttrVisitor",
+    "//clang/include/clang/AST:Attrs",
+    "//clang/include/clang/AST:CommentCommandList",
+    "//clang/include/clang/AST:CommentNodes",
+    "//clang/include/clang/AST:StmtNodes",
+  ]
+  sources = [
+    "APValue.cpp",
+    "ASTConsumer.cpp",
+    "ASTContext.cpp",
+    "ASTDiagnostic.cpp",
+    "ASTDumper.cpp",
+    "ASTImporter.cpp",
+    "ASTImporterLookupTable.cpp",
+    "ASTStructuralEquivalence.cpp",
+    "ASTTypeTraits.cpp",
+    "AttrImpl.cpp",
+    "CXXInheritance.cpp",
+    "Comment.cpp",
+    "CommentBriefParser.cpp",
+    "CommentCommandTraits.cpp",
+    "CommentLexer.cpp",
+    "CommentParser.cpp",
+    "CommentSema.cpp",
+    "ComparisonCategories.cpp",
+    "DataCollection.cpp",
+    "Decl.cpp",
+    "DeclBase.cpp",
+    "DeclCXX.cpp",
+    "DeclFriend.cpp",
+    "DeclGroup.cpp",
+    "DeclObjC.cpp",
+    "DeclOpenMP.cpp",
+    "DeclPrinter.cpp",
+    "DeclTemplate.cpp",
+    "DeclarationName.cpp",
+    "Expr.cpp",
+    "ExprCXX.cpp",
+    "ExprClassification.cpp",
+    "ExprConstant.cpp",
+    "ExprObjC.cpp",
+    "ExternalASTMerger.cpp",
+    "ExternalASTSource.cpp",
+    "FormatString.cpp",
+    "InheritViz.cpp",
+    "ItaniumCXXABI.cpp",
+    "ItaniumMangle.cpp",
+    "Mangle.cpp",
+    "MicrosoftCXXABI.cpp",
+    "MicrosoftMangle.cpp",
+    "NSAPI.cpp",
+    "NestedNameSpecifier.cpp",
+    "ODRHash.cpp",
+    "OSLog.cpp",
+    "OpenMPClause.cpp",
+    "ParentMap.cpp",
+    "PrintfFormatString.cpp",
+    "QualTypeNames.cpp",
+    "RawCommentList.cpp",
+    "RecordLayout.cpp",
+    "RecordLayoutBuilder.cpp",
+    "ScanfFormatString.cpp",
+    "SelectorLocationsKind.cpp",
+    "Stmt.cpp",
+    "StmtCXX.cpp",
+    "StmtIterator.cpp",
+    "StmtObjC.cpp",
+    "StmtOpenMP.cpp",
+    "StmtPrinter.cpp",
+    "StmtProfile.cpp",
+    "StmtViz.cpp",
+    "TemplateBase.cpp",
+    "TemplateName.cpp",
+    "TextNodeDumper.cpp",
+    "Type.cpp",
+    "TypeLoc.cpp",
+    "TypePrinter.cpp",
+    "VTTBuilder.cpp",
+    "VTableBuilder.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/clang/lib/ASTMatchers/BUILD.gn b/utils/gn/secondary/clang/lib/ASTMatchers/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..8dbb132c7e6e5775b36fe7aeb17dc6d64c075dcb
--- /dev/null
+++ b/utils/gn/secondary/clang/lib/ASTMatchers/BUILD.gn
@@ -0,0 +1,13 @@
+static_library("ASTMatchers") {
+  output_name = "clangASTMatchers"
+  configs += [ "//llvm/utils/gn/build:clang_code" ]
+  deps = [
+    "//clang/lib/AST",
+    "//clang/lib/Basic",
+    "//llvm/lib/Support",
+  ]
+  sources = [
+    "ASTMatchFinder.cpp",
+    "ASTMatchersInternal.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/clang/lib/ASTMatchers/Dynamic/BUILD.gn b/utils/gn/secondary/clang/lib/ASTMatchers/Dynamic/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..31d27de0491d7cae242c2dfe2654ca86733025ef
--- /dev/null
+++ b/utils/gn/secondary/clang/lib/ASTMatchers/Dynamic/BUILD.gn
@@ -0,0 +1,16 @@
+static_library("Dynamic") {
+  output_name = "clangDynamicASTMatchers"
+  configs += [ "//llvm/utils/gn/build:clang_code" ]
+  deps = [
+    "//clang/lib/AST",
+    "//clang/lib/ASTMatchers",
+    "//clang/lib/Basic",
+    "//llvm/lib/Support",
+  ]
+  sources = [
+    "Diagnostics.cpp",
+    "Parser.cpp",
+    "Registry.cpp",
+    "VariantValue.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/clang/lib/Analysis/BUILD.gn b/utils/gn/secondary/clang/lib/Analysis/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..004a4edb57f857469cd95304c26b8cbdc9939ae3
--- /dev/null
+++ b/utils/gn/secondary/clang/lib/Analysis/BUILD.gn
@@ -0,0 +1,37 @@
+static_library("Analysis") {
+  output_name = "clangAnalysis"
+  configs += [ "//llvm/utils/gn/build:clang_code" ]
+  deps = [
+    "//clang/include/clang/AST:StmtDataCollectors",
+    "//clang/lib/AST",
+    "//clang/lib/Basic",
+    "//clang/lib/Lex",
+    "//llvm/lib/Support",
+  ]
+  sources = [
+    "AnalysisDeclContext.cpp",
+    "BodyFarm.cpp",
+    "CFG.cpp",
+    "CFGReachabilityAnalysis.cpp",
+    "CFGStmtMap.cpp",
+    "CallGraph.cpp",
+    "CloneDetection.cpp",
+    "CocoaConventions.cpp",
+    "CodeInjector.cpp",
+    "ConstructionContext.cpp",
+    "Consumed.cpp",
+    "Dominators.cpp",
+    "ExprMutationAnalyzer.cpp",
+    "LiveVariables.cpp",
+    "ObjCNoReturn.cpp",
+    "PostOrderCFGView.cpp",
+    "ProgramPoint.cpp",
+    "ReachableCode.cpp",
+    "RetainSummaryManager.cpp",
+    "ThreadSafety.cpp",
+    "ThreadSafetyCommon.cpp",
+    "ThreadSafetyLogical.cpp",
+    "ThreadSafetyTIL.cpp",
+    "UninitializedValues.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/clang/lib/Basic/BUILD.gn b/utils/gn/secondary/clang/lib/Basic/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..4d0578b0867ab9164fc01b47affa7542aaaa4d4d
--- /dev/null
+++ b/utils/gn/secondary/clang/lib/Basic/BUILD.gn
@@ -0,0 +1,81 @@
+static_library("Basic") {
+  output_name = "clangBasic"
+  configs += [ "//llvm/utils/gn/build:clang_code" ]
+  public_deps = [
+    # public_dep because public header Version.h includes generated Version.inc.
+    "//clang/include/clang/Basic:AttrList",
+    "//clang/include/clang/Basic:AttrSubMatchRulesList",
+    "//clang/include/clang/Basic:DiagnosticGroups",
+    "//clang/include/clang/Basic:diags_tablegen",
+    "//clang/include/clang/Basic:version",
+  ]
+  deps = [
+    "//clang/include/clang/Basic:AttrHasAttributeImpl",
+    "//clang/include/clang/Basic:arm_fp16",
+    "//clang/include/clang/Basic:arm_neon",
+    "//clang/include/clang/Config",
+    "//llvm/include/llvm/Config:llvm-config",
+    "//llvm/lib/IR",
+    "//llvm/lib/MC",
+    "//llvm/lib/Support",
+  ]
+  include_dirs = [ "." ]
+  sources = [
+    "Attributes.cpp",
+    "Builtins.cpp",
+    "CharInfo.cpp",
+    "CodeGenOptions.cpp",
+    "Cuda.cpp",
+    "Diagnostic.cpp",
+    "DiagnosticIDs.cpp",
+    "DiagnosticOptions.cpp",
+    "FileManager.cpp",
+    "FileSystemStatCache.cpp",
+    "FixedPoint.cpp",
+    "IdentifierTable.cpp",
+    "LangOptions.cpp",
+    "MemoryBufferCache.cpp",
+    "Module.cpp",
+    "ObjCRuntime.cpp",
+    "OpenMPKinds.cpp",
+    "OperatorPrecedence.cpp",
+    "SanitizerBlacklist.cpp",
+    "SanitizerSpecialCaseList.cpp",
+    "Sanitizers.cpp",
+    "SourceLocation.cpp",
+    "SourceManager.cpp",
+    "TargetInfo.cpp",
+    "Targets.cpp",
+    "Targets/AArch64.cpp",
+    "Targets/AMDGPU.cpp",
+    "Targets/ARC.cpp",
+    "Targets/ARM.cpp",
+    "Targets/AVR.cpp",
+    "Targets/BPF.cpp",
+    "Targets/Hexagon.cpp",
+    "Targets/Lanai.cpp",
+    "Targets/Le64.cpp",
+    "Targets/MSP430.cpp",
+    "Targets/Mips.cpp",
+    "Targets/NVPTX.cpp",
+    "Targets/OSTargets.cpp",
+    "Targets/PNaCl.cpp",
+    "Targets/PPC.cpp",
+    "Targets/RISCV.cpp",
+    "Targets/SPIR.cpp",
+    "Targets/Sparc.cpp",
+    "Targets/SystemZ.cpp",
+    "Targets/TCE.cpp",
+    "Targets/WebAssembly.cpp",
+    "Targets/X86.cpp",
+    "Targets/XCore.cpp",
+    "TokenKinds.cpp",
+
+    # FIXME: This should be in its own target that passes -DHAVE_SVN_VERSION_INC
+    # and that also depends on a target generating SVNVersion.inc.
+    "Version.cpp",
+    "Warnings.cpp",
+    "XRayInstr.cpp",
+    "XRayLists.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/clang/lib/CodeGen/BUILD.gn b/utils/gn/secondary/clang/lib/CodeGen/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..c24548901e793a6be85cb0e6d66dec3d264dde43
--- /dev/null
+++ b/utils/gn/secondary/clang/lib/CodeGen/BUILD.gn
@@ -0,0 +1,88 @@
+static_library("CodeGen") {
+  output_name = "clangCodeGen"
+  configs += [ "//llvm/utils/gn/build:clang_code" ]
+  deps = [
+    "//clang/lib/AST",
+    "//clang/lib/Analysis",
+    "//clang/lib/Basic",
+    "//clang/lib/Frontend",
+    "//clang/lib/Lex",
+    "//llvm/lib/Analysis",
+    "//llvm/lib/Bitcode/Reader",
+    "//llvm/lib/IR",
+    "//llvm/lib/IRReader",
+    "//llvm/lib/LTO",
+    "//llvm/lib/Linker",
+    "//llvm/lib/MC",
+    "//llvm/lib/Object",
+    "//llvm/lib/Passes",
+    "//llvm/lib/ProfileData",
+    "//llvm/lib/ProfileData/Coverage",
+    "//llvm/lib/Support",
+    "//llvm/lib/Target",
+    "//llvm/lib/Transforms/Coroutines",
+    "//llvm/lib/Transforms/IPO",
+    "//llvm/lib/Transforms/InstCombine",
+    "//llvm/lib/Transforms/Instrumentation",
+    "//llvm/lib/Transforms/ObjCARC",
+    "//llvm/lib/Transforms/Scalar",
+    "//llvm/lib/Transforms/Utils",
+  ]
+  sources = [
+    "BackendUtil.cpp",
+    "CGAtomic.cpp",
+    "CGBlocks.cpp",
+    "CGBuiltin.cpp",
+    "CGCUDANV.cpp",
+    "CGCUDARuntime.cpp",
+    "CGCXX.cpp",
+    "CGCXXABI.cpp",
+    "CGCall.cpp",
+    "CGClass.cpp",
+    "CGCleanup.cpp",
+    "CGCoroutine.cpp",
+    "CGDebugInfo.cpp",
+    "CGDecl.cpp",
+    "CGDeclCXX.cpp",
+    "CGException.cpp",
+    "CGExpr.cpp",
+    "CGExprAgg.cpp",
+    "CGExprCXX.cpp",
+    "CGExprComplex.cpp",
+    "CGExprConstant.cpp",
+    "CGExprScalar.cpp",
+    "CGGPUBuiltin.cpp",
+    "CGLoopInfo.cpp",
+    "CGNonTrivialStruct.cpp",
+    "CGObjC.cpp",
+    "CGObjCGNU.cpp",
+    "CGObjCMac.cpp",
+    "CGObjCRuntime.cpp",
+    "CGOpenCLRuntime.cpp",
+    "CGOpenMPRuntime.cpp",
+    "CGOpenMPRuntimeNVPTX.cpp",
+    "CGRecordLayoutBuilder.cpp",
+    "CGStmt.cpp",
+    "CGStmtOpenMP.cpp",
+    "CGVTT.cpp",
+    "CGVTables.cpp",
+    "CodeGenABITypes.cpp",
+    "CodeGenAction.cpp",
+    "CodeGenFunction.cpp",
+    "CodeGenModule.cpp",
+    "CodeGenPGO.cpp",
+    "CodeGenTBAA.cpp",
+    "CodeGenTypes.cpp",
+    "ConstantInitBuilder.cpp",
+    "CoverageMappingGen.cpp",
+    "ItaniumCXXABI.cpp",
+    "MacroPPCallbacks.cpp",
+    "MicrosoftCXXABI.cpp",
+    "ModuleBuilder.cpp",
+    "ObjectFilePCHContainerOperations.cpp",
+    "SanitizerMetadata.cpp",
+    "SwiftCallingConv.cpp",
+    "TargetInfo.cpp",
+    "VarBypassDetector.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/clang/lib/CrossTU/BUILD.gn b/utils/gn/secondary/clang/lib/CrossTU/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..81fb0b688911a4f4eed77966a31127e060ce3afa
--- /dev/null
+++ b/utils/gn/secondary/clang/lib/CrossTU/BUILD.gn
@@ -0,0 +1,14 @@
+static_library("CrossTU") {
+  output_name = "clangCrossTU"
+  configs += [ "//llvm/utils/gn/build:clang_code" ]
+  deps = [
+    "//clang/lib/AST",
+    "//clang/lib/Basic",
+    "//clang/lib/Frontend",
+    "//clang/lib/Index",
+    "//llvm/lib/Support",
+  ]
+  sources = [
+    "CrossTranslationUnit.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/clang/lib/Driver/BUILD.gn b/utils/gn/secondary/clang/lib/Driver/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..1f07a6272d83bf0e7e332104155724e19ed77ecd
--- /dev/null
+++ b/utils/gn/secondary/clang/lib/Driver/BUILD.gn
@@ -0,0 +1,89 @@
+static_library("Driver") {
+  output_name = "clangDriver"
+  configs += [ "//llvm/utils/gn/build:clang_code" ]
+  include_dirs = [ "." ]
+  deps = [
+    "//clang/include/clang/Config",
+
+    # Driver doesn't depend on StaticAnalyzer and the other way round, but
+    # as of clang r311958 Driver does depend on StaticAnalyzer/Checkers's
+    # tablegen'd Checkers.inc.  The CMake build runs all clang tablegen steps
+    # before all lib compilations via the clang-tablegen-targets target; the
+    # GN build has this dependency instead.
+    # FIXME: Move Checkers.td somewhere else to clean up this layering mess.
+    # See the review thread of r311958 for details.
+    "//clang/include/clang/StaticAnalyzer/Checkers",
+    "//clang/lib/Basic",
+    "//llvm/include/llvm/Config:llvm-config",
+    "//llvm/lib/BinaryFormat",
+    "//llvm/lib/Option",
+    "//llvm/lib/Support",
+  ]
+  public_deps = [
+    # public_dep because public header Options.h includes generated Options.inc.
+    "//clang/include/clang/Driver:Options",
+  ]
+  if (host_os == "win") {
+    # MSVCToolChain.cpp uses version.dll.
+    libs = [ "version.lib" ]
+  }
+  sources = [
+    "Action.cpp",
+    "Compilation.cpp",
+    "DarwinSDKInfo.cpp",
+    "Distro.cpp",
+    "Driver.cpp",
+    "DriverOptions.cpp",
+    "Job.cpp",
+    "Multilib.cpp",
+    "Phases.cpp",
+    "SanitizerArgs.cpp",
+    "Tool.cpp",
+    "ToolChain.cpp",
+    "ToolChains/AMDGPU.cpp",
+    "ToolChains/AVR.cpp",
+    "ToolChains/Ananas.cpp",
+    "ToolChains/Arch/AArch64.cpp",
+    "ToolChains/Arch/ARM.cpp",
+    "ToolChains/Arch/Mips.cpp",
+    "ToolChains/Arch/PPC.cpp",
+    "ToolChains/Arch/RISCV.cpp",
+    "ToolChains/Arch/Sparc.cpp",
+    "ToolChains/Arch/SystemZ.cpp",
+    "ToolChains/Arch/X86.cpp",
+    "ToolChains/BareMetal.cpp",
+    "ToolChains/Clang.cpp",
+    "ToolChains/CloudABI.cpp",
+    "ToolChains/CommonArgs.cpp",
+    "ToolChains/Contiki.cpp",
+    "ToolChains/CrossWindows.cpp",
+    "ToolChains/Cuda.cpp",
+    "ToolChains/Darwin.cpp",
+    "ToolChains/DragonFly.cpp",
+    "ToolChains/FreeBSD.cpp",
+    "ToolChains/Fuchsia.cpp",
+    "ToolChains/Gnu.cpp",
+    "ToolChains/HIP.cpp",
+    "ToolChains/Haiku.cpp",
+    "ToolChains/Hexagon.cpp",
+    "ToolChains/Hurd.cpp",
+    "ToolChains/Linux.cpp",
+    "ToolChains/MSP430.cpp",
+    "ToolChains/MSVC.cpp",
+    "ToolChains/MinGW.cpp",
+    "ToolChains/Minix.cpp",
+    "ToolChains/MipsLinux.cpp",
+    "ToolChains/Myriad.cpp",
+    "ToolChains/NaCl.cpp",
+    "ToolChains/NetBSD.cpp",
+    "ToolChains/OpenBSD.cpp",
+    "ToolChains/PS4CPU.cpp",
+    "ToolChains/RISCVToolchain.cpp",
+    "ToolChains/Solaris.cpp",
+    "ToolChains/TCE.cpp",
+    "ToolChains/WebAssembly.cpp",
+    "ToolChains/XCore.cpp",
+    "Types.cpp",
+    "XRayArgs.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/clang/lib/Edit/BUILD.gn b/utils/gn/secondary/clang/lib/Edit/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..e32e207443f78cb6bb8483e0776576bbe785b39e
--- /dev/null
+++ b/utils/gn/secondary/clang/lib/Edit/BUILD.gn
@@ -0,0 +1,15 @@
+static_library("Edit") {
+  output_name = "clangEdit"
+  configs += [ "//llvm/utils/gn/build:clang_code" ]
+  deps = [
+    "//clang/lib/AST",
+    "//clang/lib/Basic",
+    "//clang/lib/Lex",
+    "//llvm/lib/Support",
+  ]
+  sources = [
+    "Commit.cpp",
+    "EditedSource.cpp",
+    "RewriteObjCFoundationAPI.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/clang/lib/Format/BUILD.gn b/utils/gn/secondary/clang/lib/Format/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..e08c92312df01f7ad51ca548a65353bd5e021dec
--- /dev/null
+++ b/utils/gn/secondary/clang/lib/Format/BUILD.gn
@@ -0,0 +1,27 @@
+static_library("Format") {
+  output_name = "clangFormat"
+  configs += [ "//llvm/utils/gn/build:clang_code" ]
+  deps = [
+    "//clang/lib/Basic",
+    "//clang/lib/Lex",
+    "//clang/lib/Tooling/Core",
+    "//clang/lib/Tooling/Inclusions",
+    "//llvm/lib/Support",
+  ]
+  sources = [
+    "AffectedRangeManager.cpp",
+    "BreakableToken.cpp",
+    "ContinuationIndenter.cpp",
+    "Format.cpp",
+    "FormatToken.cpp",
+    "FormatTokenLexer.cpp",
+    "NamespaceEndCommentsFixer.cpp",
+    "SortJavaScriptImports.cpp",
+    "TokenAnalyzer.cpp",
+    "TokenAnnotator.cpp",
+    "UnwrappedLineFormatter.cpp",
+    "UnwrappedLineParser.cpp",
+    "UsingDeclarationsSorter.cpp",
+    "WhitespaceManager.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/clang/lib/Frontend/BUILD.gn b/utils/gn/secondary/clang/lib/Frontend/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..0f80aa64d15f7c93a207155944c7ae96825985d0
--- /dev/null
+++ b/utils/gn/secondary/clang/lib/Frontend/BUILD.gn
@@ -0,0 +1,54 @@
+static_library("Frontend") {
+  output_name = "clangFrontend"
+  configs += [ "//llvm/utils/gn/build:clang_code" ]
+  deps = [
+    "//clang/include/clang/Config",
+    "//clang/lib/AST",
+    "//clang/lib/Basic",
+    "//clang/lib/Driver",
+    "//clang/lib/Edit",
+    "//clang/lib/Lex",
+    "//clang/lib/Parse",
+    "//clang/lib/Sema",
+    "//clang/lib/Serialization",
+    "//llvm/include/llvm/Config:llvm-config",
+    "//llvm/lib/Bitcode/Reader",
+    "//llvm/lib/Option",
+    "//llvm/lib/ProfileData",
+    "//llvm/lib/Support",
+  ]
+  sources = [
+    "ASTConsumers.cpp",
+    "ASTMerge.cpp",
+    "ASTUnit.cpp",
+    "ChainedDiagnosticConsumer.cpp",
+    "ChainedIncludesSource.cpp",
+    "CompilerInstance.cpp",
+    "CompilerInvocation.cpp",
+    "CreateInvocationFromCommandLine.cpp",
+    "DependencyFile.cpp",
+    "DependencyGraph.cpp",
+    "DiagnosticRenderer.cpp",
+    "FrontendAction.cpp",
+    "FrontendActions.cpp",
+    "FrontendOptions.cpp",
+    "FrontendTiming.cpp",
+    "HeaderIncludeGen.cpp",
+    "InitHeaderSearch.cpp",
+    "InitPreprocessor.cpp",
+    "LangStandards.cpp",
+    "LayoutOverrideSource.cpp",
+    "LogDiagnosticPrinter.cpp",
+    "ModuleDependencyCollector.cpp",
+    "MultiplexConsumer.cpp",
+    "PrecompiledPreamble.cpp",
+    "PrintPreprocessedOutput.cpp",
+    "SerializedDiagnosticPrinter.cpp",
+    "SerializedDiagnosticReader.cpp",
+    "TestModuleFileExtension.cpp",
+    "TextDiagnostic.cpp",
+    "TextDiagnosticBuffer.cpp",
+    "TextDiagnosticPrinter.cpp",
+    "VerifyDiagnosticConsumer.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/clang/lib/Frontend/Rewrite/BUILD.gn b/utils/gn/secondary/clang/lib/Frontend/Rewrite/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..1f185cf8834a4619b83abff68f72429c31d6cb28
--- /dev/null
+++ b/utils/gn/secondary/clang/lib/Frontend/Rewrite/BUILD.gn
@@ -0,0 +1,24 @@
+static_library("Rewrite") {
+  output_name = "clangRewriteFrontend"
+  configs += [ "//llvm/utils/gn/build:clang_code" ]
+  deps = [
+    "//clang/lib/AST",
+    "//clang/lib/Basic",
+    "//clang/lib/Edit",
+    "//clang/lib/Frontend",
+    "//clang/lib/Lex",
+    "//clang/lib/Rewrite",
+    "//clang/lib/Serialization",
+    "//llvm/lib/Support",
+  ]
+  sources = [
+    "FixItRewriter.cpp",
+    "FrontendActions.cpp",
+    "HTMLPrint.cpp",
+    "InclusionRewriter.cpp",
+    "RewriteMacros.cpp",
+    "RewriteModernObjC.cpp",
+    "RewriteObjC.cpp",
+    "RewriteTest.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/clang/lib/FrontendTool/BUILD.gn b/utils/gn/secondary/clang/lib/FrontendTool/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..6bcb4c320097a5e790e7f8430bded77e57637193
--- /dev/null
+++ b/utils/gn/secondary/clang/lib/FrontendTool/BUILD.gn
@@ -0,0 +1,29 @@
+import("//clang/lib/ARCMigrate/enable.gni")
+import("//clang/lib/StaticAnalyzer/Frontend/enable.gni")
+
+assert(clang_enable_static_analyzer || !clang_enable_arcmt,
+       "Cannot disable static analyzer while enabling ARCMT")
+
+static_library("FrontendTool") {
+  output_name = "clangFrontendTool"
+  configs += [ "//llvm/utils/gn/build:clang_code" ]
+  deps = [
+    "//clang/include/clang/Config",
+    "//clang/lib/Basic",
+    "//clang/lib/CodeGen",
+    "//clang/lib/Driver",
+    "//clang/lib/Frontend",
+    "//clang/lib/Frontend/Rewrite",
+    "//llvm/lib/Option",
+    "//llvm/lib/Support",
+  ]
+  if (clang_enable_arcmt) {
+    deps += [ "//clang/lib/ARCMigrate" ]
+  }
+  if (clang_enable_static_analyzer) {
+    deps += [ "//clang/lib/StaticAnalyzer/Frontend" ]
+  }
+  sources = [
+    "ExecuteCompilerInvocation.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/clang/lib/Headers/BUILD.gn b/utils/gn/secondary/clang/lib/Headers/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..2c472ce7839e3641af2c2cbebdd99b6fc3274790
--- /dev/null
+++ b/utils/gn/secondary/clang/lib/Headers/BUILD.gn
@@ -0,0 +1,161 @@
+import("//clang/resource_dir.gni")
+import("//clang/utils/TableGen/clang_tablegen.gni")
+
+# Generate arm_neon.h
+clang_tablegen("arm_neon") {
+  args = [ "-gen-arm-neon" ]
+  td_file = "//clang/include/clang/Basic/arm_neon.td"
+  output_name = "arm_neon.h"
+}
+
+# Generate arm_fp16.h
+clang_tablegen("arm_fp16") {
+  args = [ "-gen-arm-fp16" ]
+  td_file = "//clang/include/clang/Basic/arm_fp16.td"
+  output_name = "arm_fp16.h"
+}
+
+copy("arm_headers") {
+  visibility = [ ":Headers" ]
+  deps = [
+    ":arm_fp16",
+    ":arm_neon",
+  ]
+  sources = get_target_outputs(":arm_neon") + get_target_outputs(":arm_fp16")
+  outputs = [
+    "$clang_resource_dir/include/{{source_file_part}}",
+  ]
+}
+
+copy("Headers") {
+  deps = [
+    ":arm_headers",
+  ]
+
+  # NOSORT
+  # Tell `gn format` to not reorder the sources list: Its order matches CMake,
+  # and the ordering is alphabetical but ignores leading underscores.
+  sources = [
+    "adxintrin.h",
+    "altivec.h",
+    "ammintrin.h",
+    "arm_acle.h",
+    "armintr.h",
+    "arm64intr.h",
+    "avx2intrin.h",
+    "avx512bwintrin.h",
+    "avx512bitalgintrin.h",
+    "avx512vlbitalgintrin.h",
+    "avx512cdintrin.h",
+    "avx512vpopcntdqintrin.h",
+    "avx512dqintrin.h",
+    "avx512erintrin.h",
+    "avx512fintrin.h",
+    "avx512ifmaintrin.h",
+    "avx512ifmavlintrin.h",
+    "avx512pfintrin.h",
+    "avx512vbmiintrin.h",
+    "avx512vbmivlintrin.h",
+    "avx512vbmi2intrin.h",
+    "avx512vlvbmi2intrin.h",
+    "avx512vlbwintrin.h",
+    "avx512vlcdintrin.h",
+    "avx512vldqintrin.h",
+    "avx512vlintrin.h",
+    "avx512vpopcntdqvlintrin.h",
+    "avx512vnniintrin.h",
+    "avx512vlvnniintrin.h",
+    "avxintrin.h",
+    "bmi2intrin.h",
+    "bmiintrin.h",
+    "__clang_cuda_builtin_vars.h",
+    "__clang_cuda_cmath.h",
+    "__clang_cuda_complex_builtins.h",
+    "__clang_cuda_device_functions.h",
+    "__clang_cuda_intrinsics.h",
+    "__clang_cuda_libdevice_declares.h",
+    "__clang_cuda_math_forward_declares.h",
+    "__clang_cuda_runtime_wrapper.h",
+    "cetintrin.h",
+    "cldemoteintrin.h",
+    "clzerointrin.h",
+    "cpuid.h",
+    "clflushoptintrin.h",
+    "clwbintrin.h",
+    "emmintrin.h",
+    "f16cintrin.h",
+    "float.h",
+    "fma4intrin.h",
+    "fmaintrin.h",
+    "fxsrintrin.h",
+    "gfniintrin.h",
+    "htmintrin.h",
+    "htmxlintrin.h",
+    "ia32intrin.h",
+    "immintrin.h",
+    "intrin.h",
+    "inttypes.h",
+    "invpcidintrin.h",
+    "iso646.h",
+    "limits.h",
+    "lwpintrin.h",
+    "lzcntintrin.h",
+    "mm3dnow.h",
+    "mmintrin.h",
+    "mm_malloc.h",
+    "module.modulemap",
+    "movdirintrin.h",
+    "msa.h",
+    "mwaitxintrin.h",
+    "nmmintrin.h",
+    "opencl-c.h",
+    "pconfigintrin.h",
+    "pkuintrin.h",
+    "pmmintrin.h",
+    "popcntintrin.h",
+    "prfchwintrin.h",
+    "ptwriteintrin.h",
+    "rdseedintrin.h",
+    "rtmintrin.h",
+    "s390intrin.h",
+    "sgxintrin.h",
+    "shaintrin.h",
+    "smmintrin.h",
+    "stdalign.h",
+    "stdarg.h",
+    "stdatomic.h",
+    "stdbool.h",
+    "stddef.h",
+    "__stddef_max_align_t.h",
+    "stdint.h",
+    "stdnoreturn.h",
+    "tbmintrin.h",
+    "tgmath.h",
+    "tmmintrin.h",
+    "unwind.h",
+    "vadefs.h",
+    "vaesintrin.h",
+    "varargs.h",
+    "vecintrin.h",
+    "vpclmulqdqintrin.h",
+    "waitpkgintrin.h",
+    "wbnoinvdintrin.h",
+    "wmmintrin.h",
+    "__wmmintrin_aes.h",
+    "__wmmintrin_pclmul.h",
+    "x86intrin.h",
+    "xmmintrin.h",
+    "xopintrin.h",
+    "xsavecintrin.h",
+    "xsaveintrin.h",
+    "xsaveoptintrin.h",
+    "xsavesintrin.h",
+    "xtestintrin.h",
+    "cuda_wrappers/algorithm",
+    "cuda_wrappers/complex",
+    "cuda_wrappers/new",
+  ]
+  outputs = [
+    "$clang_resource_dir/include/{{source_target_relative}}",
+  ]
+}
diff --git a/utils/gn/secondary/clang/lib/Index/BUILD.gn b/utils/gn/secondary/clang/lib/Index/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..8c7190f4bc4087e666ac217d188463b3c4da8bff
--- /dev/null
+++ b/utils/gn/secondary/clang/lib/Index/BUILD.gn
@@ -0,0 +1,28 @@
+static_library("Index") {
+  output_name = "clangIndex"
+  configs += [ "//llvm/utils/gn/build:clang_code" ]
+  deps = [
+    "//clang/lib/AST",
+    "//clang/lib/Basic",
+    "//clang/lib/Format",
+    "//clang/lib/Frontend",
+    "//clang/lib/Rewrite",
+    "//clang/lib/Serialization",
+    "//clang/lib/Tooling/Core",
+    "//llvm/lib/IR",
+    "//llvm/lib/Support",
+  ]
+  sources = [
+    "CodegenNameGenerator.cpp",
+    "CommentToXML.cpp",
+    "IndexBody.cpp",
+    "IndexDecl.cpp",
+    "IndexSymbol.cpp",
+    "IndexTypeSourceInfo.cpp",
+    "IndexingAction.cpp",
+    "IndexingContext.cpp",
+    "IndexingContext.h",
+    "SimpleFormatContext.h",
+    "USRGeneration.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/clang/lib/Lex/BUILD.gn b/utils/gn/secondary/clang/lib/Lex/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..2934e9cfcdffbc59821910793619f50f15982450
--- /dev/null
+++ b/utils/gn/secondary/clang/lib/Lex/BUILD.gn
@@ -0,0 +1,31 @@
+static_library("Lex") {
+  output_name = "clangLex"
+  configs += [ "//llvm/utils/gn/build:clang_code" ]
+  deps = [
+    "//clang/lib/Basic",
+    "//llvm/lib/Support",
+  ]
+  sources = [
+    "HeaderMap.cpp",
+    "HeaderSearch.cpp",
+    "Lexer.cpp",
+    "LiteralSupport.cpp",
+    "MacroArgs.cpp",
+    "MacroInfo.cpp",
+    "ModuleMap.cpp",
+    "PPCaching.cpp",
+    "PPCallbacks.cpp",
+    "PPConditionalDirectiveRecord.cpp",
+    "PPDirectives.cpp",
+    "PPExpressions.cpp",
+    "PPLexerChange.cpp",
+    "PPMacroExpansion.cpp",
+    "Pragma.cpp",
+    "PreprocessingRecord.cpp",
+    "Preprocessor.cpp",
+    "PreprocessorLexer.cpp",
+    "ScratchBuffer.cpp",
+    "TokenConcatenation.cpp",
+    "TokenLexer.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/clang/lib/Parse/BUILD.gn b/utils/gn/secondary/clang/lib/Parse/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..341881e365c353343dd5bbb26214e41dc851259c
--- /dev/null
+++ b/utils/gn/secondary/clang/lib/Parse/BUILD.gn
@@ -0,0 +1,32 @@
+static_library("Parse") {
+  output_name = "clangParse"
+  configs += [ "//llvm/utils/gn/build:clang_code" ]
+  deps = [
+    "//clang/include/clang/Parse:AttrParserStringSwitches",
+    "//clang/include/clang/Parse:AttrSubMatchRulesParserStringSwitches",
+    "//clang/lib/AST",
+    "//clang/lib/Basic",
+    "//clang/lib/Lex",
+    "//clang/lib/Sema",
+    "//llvm/lib/MC",
+    "//llvm/lib/MC/MCParser",
+    "//llvm/lib/Support",
+  ]
+  sources = [
+    "ParseAST.cpp",
+    "ParseCXXInlineMethods.cpp",
+    "ParseDecl.cpp",
+    "ParseDeclCXX.cpp",
+    "ParseExpr.cpp",
+    "ParseExprCXX.cpp",
+    "ParseInit.cpp",
+    "ParseObjc.cpp",
+    "ParseOpenMP.cpp",
+    "ParsePragma.cpp",
+    "ParseStmt.cpp",
+    "ParseStmtAsm.cpp",
+    "ParseTemplate.cpp",
+    "ParseTentative.cpp",
+    "Parser.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/clang/lib/Rewrite/BUILD.gn b/utils/gn/secondary/clang/lib/Rewrite/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..2ea3cdd393bda502f3f7e0ac75c045c83f9d37d9
--- /dev/null
+++ b/utils/gn/secondary/clang/lib/Rewrite/BUILD.gn
@@ -0,0 +1,16 @@
+static_library("Rewrite") {
+  output_name = "clangRewrite"
+  configs += [ "//llvm/utils/gn/build:clang_code" ]
+  deps = [
+    "//clang/lib/Basic",
+    "//clang/lib/Lex",
+    "//llvm/lib/Support",
+  ]
+  sources = [
+    "DeltaTree.cpp",
+    "HTMLRewrite.cpp",
+    "RewriteRope.cpp",
+    "Rewriter.cpp",
+    "TokenRewriter.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/clang/lib/Sema/BUILD.gn b/utils/gn/secondary/clang/lib/Sema/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..716f4d555b972d0d5b2c1b80e06258cb61030c68
--- /dev/null
+++ b/utils/gn/secondary/clang/lib/Sema/BUILD.gn
@@ -0,0 +1,66 @@
+static_library("Sema") {
+  output_name = "clangSema"
+  configs += [ "//llvm/utils/gn/build:clang_code" ]
+  deps = [
+    "//clang/include/clang/Sema:AttrParsedAttrImpl",
+    "//clang/include/clang/Sema:AttrParsedAttrKinds",
+    "//clang/include/clang/Sema:AttrParsedAttrList",
+    "//clang/include/clang/Sema:AttrSpellingListIndex",
+    "//clang/include/clang/Sema:AttrTemplateInstantiate",
+    "//clang/lib/AST",
+    "//clang/lib/Analysis",
+    "//clang/lib/Basic",
+    "//clang/lib/Edit",
+    "//clang/lib/Lex",
+    "//llvm/lib/Support",
+  ]
+  sources = [
+    "AnalysisBasedWarnings.cpp",
+    "CodeCompleteConsumer.cpp",
+    "DeclSpec.cpp",
+    "DelayedDiagnostic.cpp",
+    "IdentifierResolver.cpp",
+    "JumpDiagnostics.cpp",
+    "MultiplexExternalSemaSource.cpp",
+    "ParsedAttr.cpp",
+    "Scope.cpp",
+    "ScopeInfo.cpp",
+    "Sema.cpp",
+    "SemaAccess.cpp",
+    "SemaAttr.cpp",
+    "SemaCUDA.cpp",
+    "SemaCXXScopeSpec.cpp",
+    "SemaCast.cpp",
+    "SemaChecking.cpp",
+    "SemaCodeComplete.cpp",
+    "SemaConsumer.cpp",
+    "SemaCoroutine.cpp",
+    "SemaDecl.cpp",
+    "SemaDeclAttr.cpp",
+    "SemaDeclCXX.cpp",
+    "SemaDeclObjC.cpp",
+    "SemaExceptionSpec.cpp",
+    "SemaExpr.cpp",
+    "SemaExprCXX.cpp",
+    "SemaExprMember.cpp",
+    "SemaExprObjC.cpp",
+    "SemaFixItUtils.cpp",
+    "SemaInit.cpp",
+    "SemaLambda.cpp",
+    "SemaLookup.cpp",
+    "SemaObjCProperty.cpp",
+    "SemaOpenMP.cpp",
+    "SemaOverload.cpp",
+    "SemaPseudoObject.cpp",
+    "SemaStmt.cpp",
+    "SemaStmtAsm.cpp",
+    "SemaStmtAttr.cpp",
+    "SemaTemplate.cpp",
+    "SemaTemplateDeduction.cpp",
+    "SemaTemplateInstantiate.cpp",
+    "SemaTemplateInstantiateDecl.cpp",
+    "SemaTemplateVariadic.cpp",
+    "SemaType.cpp",
+    "TypeLocBuilder.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/clang/lib/Serialization/BUILD.gn b/utils/gn/secondary/clang/lib/Serialization/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..394b1edf2f8bdfb030297de3326925b7647f3644
--- /dev/null
+++ b/utils/gn/secondary/clang/lib/Serialization/BUILD.gn
@@ -0,0 +1,31 @@
+static_library("Serialization") {
+  output_name = "clangSerialization"
+  configs += [ "//llvm/utils/gn/build:clang_code" ]
+  deps = [
+    "//clang/include/clang/Serialization:AttrPCHRead",
+    "//clang/include/clang/Serialization:AttrPCHWrite",
+    "//clang/lib/AST",
+    "//clang/lib/Basic",
+    "//clang/lib/Lex",
+    "//clang/lib/Sema",
+    "//llvm/lib/Bitcode/Reader",
+    "//llvm/lib/Support",
+  ]
+  sources = [
+    "ASTCommon.cpp",
+    "ASTCommon.h",
+    "ASTReader.cpp",
+    "ASTReaderDecl.cpp",
+    "ASTReaderInternals.h",
+    "ASTReaderStmt.cpp",
+    "ASTWriter.cpp",
+    "ASTWriterDecl.cpp",
+    "ASTWriterStmt.cpp",
+    "GeneratePCH.cpp",
+    "GlobalModuleIndex.cpp",
+    "Module.cpp",
+    "ModuleFileExtension.cpp",
+    "ModuleManager.cpp",
+    "PCHContainerOperations.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/clang/lib/StaticAnalyzer/Checkers/BUILD.gn b/utils/gn/secondary/clang/lib/StaticAnalyzer/Checkers/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..785ff82badf493462d5742d33a27fc054fb04d78
--- /dev/null
+++ b/utils/gn/secondary/clang/lib/StaticAnalyzer/Checkers/BUILD.gn
@@ -0,0 +1,115 @@
+static_library("Checkers") {
+  output_name = "clangStaticAnalyzerCheckers"
+  configs += [ "//llvm/utils/gn/build:clang_code" ]
+  deps = [
+    "//clang/include/clang/StaticAnalyzer/Checkers",
+    "//clang/lib/AST",
+    "//clang/lib/ASTMatchers",
+    "//clang/lib/Analysis",
+    "//clang/lib/Basic",
+    "//clang/lib/Lex",
+    "//clang/lib/StaticAnalyzer/Core",
+    "//llvm/lib/Support",
+  ]
+  sources = [
+    "AnalysisOrderChecker.cpp",
+    "AnalyzerStatsChecker.cpp",
+    "ArrayBoundChecker.cpp",
+    "ArrayBoundCheckerV2.cpp",
+    "BasicObjCFoundationChecks.cpp",
+    "BlockInCriticalSectionChecker.cpp",
+    "BoolAssignmentChecker.cpp",
+    "BuiltinFunctionChecker.cpp",
+    "CStringChecker.cpp",
+    "CStringSyntaxChecker.cpp",
+    "CXXSelfAssignmentChecker.cpp",
+    "CallAndMessageChecker.cpp",
+    "CastSizeChecker.cpp",
+    "CastToStructChecker.cpp",
+    "CheckObjCDealloc.cpp",
+    "CheckObjCInstMethSignature.cpp",
+    "CheckSecuritySyntaxOnly.cpp",
+    "CheckSizeofPointer.cpp",
+    "CheckerDocumentation.cpp",
+    "ChrootChecker.cpp",
+    "CloneChecker.cpp",
+    "ConversionChecker.cpp",
+    "DeadStoresChecker.cpp",
+    "DebugCheckers.cpp",
+    "DeleteWithNonVirtualDtorChecker.cpp",
+    "DereferenceChecker.cpp",
+    "DirectIvarAssignment.cpp",
+    "DivZeroChecker.cpp",
+    "DynamicTypeChecker.cpp",
+    "DynamicTypePropagation.cpp",
+    "EnumCastOutOfRangeChecker.cpp",
+    "ExprInspectionChecker.cpp",
+    "FixedAddressChecker.cpp",
+    "GCDAntipatternChecker.cpp",
+    "GTestChecker.cpp",
+    "GenericTaintChecker.cpp",
+    "IdenticalExprChecker.cpp",
+    "InnerPointerChecker.cpp",
+    "IteratorChecker.cpp",
+    "IvarInvalidationChecker.cpp",
+    "LLVMConventionsChecker.cpp",
+    "LocalizationChecker.cpp",
+    "MPI-Checker/MPIBugReporter.cpp",
+    "MPI-Checker/MPIChecker.cpp",
+    "MPI-Checker/MPIFunctionClassifier.cpp",
+    "MacOSKeychainAPIChecker.cpp",
+    "MacOSXAPIChecker.cpp",
+    "MallocChecker.cpp",
+    "MallocOverflowSecurityChecker.cpp",
+    "MallocSizeofChecker.cpp",
+    "MmapWriteExecChecker.cpp",
+    "MoveChecker.cpp",
+    "NSAutoreleasePoolChecker.cpp",
+    "NSErrorChecker.cpp",
+    "NoReturnFunctionChecker.cpp",
+    "NonNullParamChecker.cpp",
+    "NonnullGlobalConstantsChecker.cpp",
+    "NullabilityChecker.cpp",
+    "NumberObjectConversionChecker.cpp",
+    "OSObjectCStyleCast.cpp",
+    "ObjCAtSyncChecker.cpp",
+    "ObjCAutoreleaseWriteChecker.cpp",
+    "ObjCContainersASTChecker.cpp",
+    "ObjCContainersChecker.cpp",
+    "ObjCMissingSuperCallChecker.cpp",
+    "ObjCPropertyChecker.cpp",
+    "ObjCSelfInitChecker.cpp",
+    "ObjCSuperDeallocChecker.cpp",
+    "ObjCUnusedIVarsChecker.cpp",
+    "PaddingChecker.cpp",
+    "PointerArithChecker.cpp",
+    "PointerSubChecker.cpp",
+    "PthreadLockChecker.cpp",
+    "RetainCountChecker/RetainCountChecker.cpp",
+    "RetainCountChecker/RetainCountDiagnostics.cpp",
+    "ReturnPointerRangeChecker.cpp",
+    "ReturnUndefChecker.cpp",
+    "RunLoopAutoreleaseLeakChecker.cpp",
+    "SimpleStreamChecker.cpp",
+    "StackAddrEscapeChecker.cpp",
+    "StdLibraryFunctionsChecker.cpp",
+    "StreamChecker.cpp",
+    "TaintTesterChecker.cpp",
+    "TestAfterDivZeroChecker.cpp",
+    "TraversalChecker.cpp",
+    "TrustNonnullChecker.cpp",
+    "UndefBranchChecker.cpp",
+    "UndefCapturedBlockVarChecker.cpp",
+    "UndefResultChecker.cpp",
+    "UndefinedArraySubscriptChecker.cpp",
+    "UndefinedAssignmentChecker.cpp",
+    "UninitializedObject/UninitializedObjectChecker.cpp",
+    "UninitializedObject/UninitializedPointee.cpp",
+    "UnixAPIChecker.cpp",
+    "UnreachableCodeChecker.cpp",
+    "VLASizeChecker.cpp",
+    "ValistChecker.cpp",
+    "VforkChecker.cpp",
+    "VirtualCallChecker.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/clang/lib/StaticAnalyzer/Core/BUILD.gn b/utils/gn/secondary/clang/lib/StaticAnalyzer/Core/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..745bc236935adca3cf0df2eda8423fb17599f6f9
--- /dev/null
+++ b/utils/gn/secondary/clang/lib/StaticAnalyzer/Core/BUILD.gn
@@ -0,0 +1,66 @@
+static_library("Core") {
+  output_name = "clangStaticAnalyzerCore"
+  configs += [ "//llvm/utils/gn/build:clang_code" ]
+  deps = [
+    "//clang/include/clang/Config",
+    "//clang/lib/AST",
+    "//clang/lib/ASTMatchers",
+    "//clang/lib/Analysis",
+    "//clang/lib/Basic",
+    "//clang/lib/CrossTU",
+    "//clang/lib/Lex",
+    "//clang/lib/Rewrite",
+    "//llvm/lib/Support",
+  ]
+  sources = [
+    "APSIntType.cpp",
+    "AnalysisManager.cpp",
+    "AnalyzerOptions.cpp",
+    "BasicValueFactory.cpp",
+    "BlockCounter.cpp",
+    "BugReporter.cpp",
+    "BugReporterVisitors.cpp",
+    "CallEvent.cpp",
+    "Checker.cpp",
+    "CheckerContext.cpp",
+    "CheckerHelpers.cpp",
+    "CheckerManager.cpp",
+    "CommonBugCategories.cpp",
+    "ConstraintManager.cpp",
+    "CoreEngine.cpp",
+    "DynamicTypeMap.cpp",
+    "Environment.cpp",
+    "ExplodedGraph.cpp",
+    "ExprEngine.cpp",
+    "ExprEngineC.cpp",
+    "ExprEngineCXX.cpp",
+    "ExprEngineCallAndReturn.cpp",
+    "ExprEngineObjC.cpp",
+    "FunctionSummary.cpp",
+    "HTMLDiagnostics.cpp",
+    "IssueHash.cpp",
+    "LoopUnrolling.cpp",
+    "LoopWidening.cpp",
+    "MemRegion.cpp",
+    "PathDiagnostic.cpp",
+    "PlistDiagnostics.cpp",
+    "ProgramState.cpp",
+    "RangeConstraintManager.cpp",
+    "RangedConstraintManager.cpp",
+    "RegionStore.cpp",
+    "SValBuilder.cpp",
+    "SVals.cpp",
+    "SarifDiagnostics.cpp",
+    "SimpleConstraintManager.cpp",
+    "SimpleSValBuilder.cpp",
+    "Store.cpp",
+    "SubEngine.cpp",
+    "SymbolManager.cpp",
+    "TaintManager.cpp",
+    "WorkList.cpp",
+    "Z3ConstraintManager.cpp",
+  ]
+
+  # FIXME: clang/Config/BUILD.gn currently always sets CLANG_ANALYZER_WITH_Z3
+  # to false. If that changes we need to link to Z3 libs here.
+}
diff --git a/utils/gn/secondary/clang/lib/StaticAnalyzer/Frontend/BUILD.gn b/utils/gn/secondary/clang/lib/StaticAnalyzer/Frontend/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..e4654bf14944bc88211e2a6f9b783e4f60b34e34
--- /dev/null
+++ b/utils/gn/secondary/clang/lib/StaticAnalyzer/Frontend/BUILD.gn
@@ -0,0 +1,23 @@
+static_library("Frontend") {
+  output_name = "clangStaticAnalyzerFrontend"
+  configs += [ "//llvm/utils/gn/build:clang_code" ]
+  deps = [
+    "//clang/lib/AST",
+    "//clang/lib/Analysis",
+    "//clang/lib/Basic",
+    "//clang/lib/CrossTU",
+    "//clang/lib/Frontend",
+    "//clang/lib/Lex",
+    "//clang/lib/StaticAnalyzer/Checkers",
+    "//clang/lib/StaticAnalyzer/Core",
+    "//llvm/lib/Support",
+  ]
+  sources = [
+    "AnalysisConsumer.cpp",
+    "CheckerRegistration.cpp",
+    "CheckerRegistry.cpp",
+    "FrontendActions.cpp",
+    "ModelConsumer.cpp",
+    "ModelInjector.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/clang/lib/StaticAnalyzer/Frontend/enable.gni b/utils/gn/secondary/clang/lib/StaticAnalyzer/Frontend/enable.gni
new file mode 100644
index 0000000000000000000000000000000000000000..75653a96dd0b6e355ccdda2d0d5088ffb5861c4e
--- /dev/null
+++ b/utils/gn/secondary/clang/lib/StaticAnalyzer/Frontend/enable.gni
@@ -0,0 +1,4 @@
+declare_args() {
+  # Whether to include the static analyzer in the clang binary.
+  clang_enable_static_analyzer = true
+}
diff --git a/utils/gn/secondary/clang/lib/Tooling/ASTDiff/BUILD.gn b/utils/gn/secondary/clang/lib/Tooling/ASTDiff/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..0058a26ca0881d0d1c8728a8b625a78a452c0c96
--- /dev/null
+++ b/utils/gn/secondary/clang/lib/Tooling/ASTDiff/BUILD.gn
@@ -0,0 +1,13 @@
+static_library("ASTDiff") {
+  output_name = "clangToolingASTDiff"
+  configs += [ "//llvm/utils/gn/build:clang_code" ]
+  deps = [
+    "//clang/lib/AST",
+    "//clang/lib/Basic",
+    "//clang/lib/Lex",
+    "//llvm/lib/Support",
+  ]
+  sources = [
+    "ASTDiff.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/clang/lib/Tooling/BUILD.gn b/utils/gn/secondary/clang/lib/Tooling/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..a298d5289a5cc506e826fb83f440a646bca1e758
--- /dev/null
+++ b/utils/gn/secondary/clang/lib/Tooling/BUILD.gn
@@ -0,0 +1,31 @@
+static_library("Tooling") {
+  output_name = "clangTooling"
+  configs += [ "//llvm/utils/gn/build:clang_code" ]
+  deps = [
+    "//clang/include/clang/Driver:Options",
+    "//clang/lib/AST",
+    "//clang/lib/ASTMatchers",
+    "//clang/lib/Basic",
+    "//clang/lib/Driver",
+    "//clang/lib/Format",
+    "//clang/lib/Frontend",
+    "//clang/lib/Lex",
+    "//clang/lib/Rewrite",
+    "//clang/lib/Tooling/Core",
+  ]
+  sources = [
+    "AllTUsExecution.cpp",
+    "ArgumentsAdjusters.cpp",
+    "CommonOptionsParser.cpp",
+    "CompilationDatabase.cpp",
+    "Execution.cpp",
+    "FileMatchTrie.cpp",
+    "FixIt.cpp",
+    "InterpolatingCompilationDatabase.cpp",
+    "JSONCompilationDatabase.cpp",
+    "Refactoring.cpp",
+    "RefactoringCallbacks.cpp",
+    "StandaloneExecution.cpp",
+    "Tooling.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/clang/lib/Tooling/Core/BUILD.gn b/utils/gn/secondary/clang/lib/Tooling/Core/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..1cc513e544950b6c52bcaf866c30e413a4e229b8
--- /dev/null
+++ b/utils/gn/secondary/clang/lib/Tooling/Core/BUILD.gn
@@ -0,0 +1,16 @@
+static_library("Core") {
+  output_name = "clangToolingCore"
+  configs += [ "//llvm/utils/gn/build:clang_code" ]
+  deps = [
+    "//clang/lib/AST",
+    "//clang/lib/Basic",
+    "//clang/lib/Lex",
+    "//clang/lib/Rewrite",
+    "//llvm/lib/Support",
+  ]
+  sources = [
+    "Diagnostic.cpp",
+    "Lookup.cpp",
+    "Replacement.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/clang/lib/Tooling/Inclusions/BUILD.gn b/utils/gn/secondary/clang/lib/Tooling/Inclusions/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..419c615da2ee2f89549a6f9d915dd04b7dd63f44
--- /dev/null
+++ b/utils/gn/secondary/clang/lib/Tooling/Inclusions/BUILD.gn
@@ -0,0 +1,15 @@
+static_library("Inclusions") {
+  output_name = "clangToolingInclusions"
+  configs += [ "//llvm/utils/gn/build:clang_code" ]
+  deps = [
+    "//clang/lib/Basic",
+    "//clang/lib/Lex",
+    "//clang/lib/Rewrite",
+    "//clang/lib/Tooling/Core",
+    "//llvm/lib/Support",
+  ]
+  sources = [
+    "HeaderIncludes.cpp",
+    "IncludeStyle.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/clang/lib/Tooling/Refactoring/BUILD.gn b/utils/gn/secondary/clang/lib/Tooling/Refactoring/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..bf57e3e1fc2cd3ce5ba6fdc591d444d89e159f02
--- /dev/null
+++ b/utils/gn/secondary/clang/lib/Tooling/Refactoring/BUILD.gn
@@ -0,0 +1,29 @@
+static_library("Refactoring") {
+  output_name = "clangToolingRefactor"
+  configs += [ "//llvm/utils/gn/build:clang_code" ]
+  deps = [
+    "//clang/lib/AST",
+    "//clang/lib/ASTMatchers",
+    "//clang/lib/Basic",
+    "//clang/lib/Format",
+    "//clang/lib/Index",
+    "//clang/lib/Lex",
+    "//clang/lib/Rewrite",
+    "//clang/lib/Tooling/Core",
+    "//llvm/lib/Support",
+  ]
+  include_dirs = [ "." ]
+  sources = [
+    "ASTSelection.cpp",
+    "ASTSelectionRequirements.cpp",
+    "AtomicChange.cpp",
+    "Extract/Extract.cpp",
+    "Extract/SourceExtraction.cpp",
+    "RefactoringActions.cpp",
+    "Rename/RenamingAction.cpp",
+    "Rename/SymbolOccurrences.cpp",
+    "Rename/USRFinder.cpp",
+    "Rename/USRFindingAction.cpp",
+    "Rename/USRLocFinder.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/clang/resource_dir.gni b/utils/gn/secondary/clang/resource_dir.gni
new file mode 100644
index 0000000000000000000000000000000000000000..b5a877a8d9cd28feca5be4f9b16c70676a72cf28
--- /dev/null
+++ b/utils/gn/secondary/clang/resource_dir.gni
@@ -0,0 +1,3 @@
+import("//llvm/version.gni")
+
+clang_resource_dir = "$root_build_dir/lib/clang/$llvm_version"
diff --git a/utils/gn/secondary/clang/test/BUILD.gn b/utils/gn/secondary/clang/test/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..681b07cb860a15596cdd24fa3e0eac8280c0afc5
--- /dev/null
+++ b/utils/gn/secondary/clang/test/BUILD.gn
@@ -0,0 +1,186 @@
+import("//clang/lib/ARCMigrate/enable.gni")
+import("//clang/lib/StaticAnalyzer/Frontend/enable.gni")
+import("//llvm/lib/Target/targets.gni")
+import("//llvm/triples.gni")
+import("//llvm/utils/gn/build/libs/zlib/enable.gni")
+import("//llvm/utils/gn/build/write_cmake_config.gni")
+import("clang_lit_site_cfg_files.gni")
+
+template("write_lit_config") {
+  write_cmake_config(target_name) {
+    input = invoker.input
+    output = invoker.output
+    values = [
+      "LIT_SITE_CFG_IN_HEADER=## Autogenerated from $input, do not edit",
+      "CLANG_BINARY_DIR=" +
+          rebase_path(get_label_info("//clang", "target_out_dir")),
+      "CLANG_SOURCE_DIR=" + rebase_path("//clang"),
+      "ENABLE_SHARED=0",
+      "LLVM_BINARY_DIR=" +
+          rebase_path(get_label_info("//llvm", "target_out_dir")),
+      "LLVM_LIBS_DIR=",  # needed only for shared builds
+      "LLVM_SOURCE_DIR=" + rebase_path("//llvm"),
+      "LLVM_TOOLS_DIR=" + rebase_path("$root_out_dir/bin"),
+      "TARGET_TRIPLE=$llvm_target_triple",
+    ]
+    if (host_os == "win") {
+      # See comment for Windows solink in llvm/utils/gn/build/toolchain/BUILD.gn
+      values += [ "SHLIBDIR=" + rebase_path("$root_out_dir/bin") ]
+    } else {
+      values += [ "SHLIBDIR=" + rebase_path("$root_out_dir/lib") ]
+    }
+    values += invoker.extra_values
+  }
+}
+
+write_lit_config("lit_site_cfg") {
+  # Fully-qualified instead of relative for LIT_SITE_CFG_IN_HEADER.
+  input = "//clang/test/lit.site.cfg.py.in"
+  output = clang_lit_site_cfg_file
+
+  extra_values = [
+    "CLANG_ANALYZER_WITH_Z3=",  # Must be empty, not 0.
+    "CLANG_BUILD_EXAMPLES=0",
+    "CLANG_DEFAULT_CXX_STDLIB=",  # Empty string means "default value" here.
+    "CLANG_TOOLS_DIR=" + rebase_path("$root_out_dir/bin"),
+
+    # This is only used if LLVM_USE_SANITIZER includes lsan and the host
+    # OS is macOS. Since the GN build currently never uses LLVM_USE_SANITIZER,
+    # this is never read.  If it's ever needed,
+    # utils/gn/build/toolchain/BUILD.gn should get the compiler from a variable
+    # that's also read here -- but that should happen after multi-toolchain
+    # builds exist, to make sure it's a toolchain var.
+    "CMAKE_CXX_COMPILER=c++",
+    "ENABLE_BACKTRACES=1",
+    "LLVM_HOST_TRIPLE=$llvm_current_triple",
+    "LLVM_LIT_TOOLS_DIR=",  # Intentionally empty, matches cmake build.
+    "LLVM_USE_SANITIZER=",
+    "PYTHON_EXECUTABLE=$python_path",
+    "USE_Z3_SOLVER=",
+  ]
+
+  if (clang_enable_arcmt) {
+    extra_values += [ "CLANG_ENABLE_ARCMT=1" ]
+  } else {
+    extra_values += [ "CLANG_ENABLE_ARCMT=0" ]
+  }
+
+  if (clang_enable_static_analyzer) {
+    extra_values += [ "CLANG_ENABLE_STATIC_ANALYZER=1" ]
+  } else {
+    extra_values += [ "CLANG_ENABLE_STATIC_ANALYZER=0" ]
+  }
+
+  if (llvm_enable_zlib) {
+    extra_values += [ "HAVE_LIBZ=1" ]
+  } else {
+    extra_values += [ "HAVE_LIBZ=0" ]  # Must be 0.
+  }
+
+  if (host_cpu == "x64") {
+    extra_values += [ "HOST_ARCH=x86_64" ]
+  } else {
+    assert(false, "unimplemented host_cpu " + host_cpu)
+  }
+
+  if (host_os == "mac") {
+    extra_values += [ "LLVM_PLUGIN_EXT=.dylib" ]
+  } else if (host_os == "win") {
+    extra_values += [ "LLVM_PLUGIN_EXT=.dll" ]
+  } else {
+    extra_values += [ "LLVM_PLUGIN_EXT=.so" ]
+  }
+}
+
+write_lit_config("lit_unit_site_cfg") {
+  # Fully-qualified instead of relative for LIT_SITE_CFG_IN_HEADER.
+  input = "//clang/test/Unit/lit.site.cfg.py.in"
+  output = clang_lit_unit_site_cfg_file
+  extra_values = [ "LLVM_BUILD_MODE=." ]
+}
+
+# This target should contain all dependencies of check-clang.
+# //:default depends on it, so that ninja's default target builds all
+# prerequisites for check-clang but doesn't run check-clang itself.
+group("test") {
+  deps = [
+    ":lit_site_cfg",
+    ":lit_unit_site_cfg",
+    "//clang/lib/Headers",
+    "//clang/tools/c-index-test",
+    "//clang/tools/clang-diff",
+    "//clang/tools/clang-format",
+    "//clang/tools/clang-import-test",
+    "//clang/tools/clang-offload-bundler",
+    "//clang/tools/clang-refactor",
+    "//clang/tools/clang-rename",
+    "//clang/tools/diagtool",
+    "//clang/tools/driver:symlinks",
+    "//clang/unittests",
+    "//clang/utils/TableGen:clang-tblgen",
+    "//clang/utils/hmaptool",
+    "//llvm/tools/llc",
+    "//llvm/tools/llvm-bcanalyzer",
+    "//llvm/tools/llvm-cat",
+    "//llvm/tools/llvm-config",
+    "//llvm/tools/llvm-dis",
+    "//llvm/tools/llvm-lto",
+    "//llvm/tools/llvm-lto2",
+    "//llvm/tools/llvm-modextract",
+    "//llvm/tools/llvm-nm:symlinks",
+    "//llvm/tools/llvm-objdump:symlinks",
+    "//llvm/tools/llvm-profdata",
+    "//llvm/tools/llvm-readobj:symlinks",
+    "//llvm/tools/llvm-symbolizer:symlinks",
+    "//llvm/tools/opt",
+    "//llvm/utils/FileCheck",
+    "//llvm/utils/count",
+    "//llvm/utils/llvm-lit",
+    "//llvm/utils/not",
+  ]
+  if (clang_enable_arcmt) {
+    deps += [
+      "//clang/tools/arcmt-test",
+      "//clang/tools/c-arcmt-test",
+    ]
+  }
+  if (clang_enable_static_analyzer) {
+    deps += [
+      "//clang/tools/clang-check",
+      "//clang/tools/clang-extdef-mapping",
+    ]
+  }
+
+  # FIXME: clang_build_examples
+  testonly = true
+}
+
+action("check-clang") {
+  script = "$root_out_dir/bin/llvm-lit"
+  if (host_os == "win") {
+    script += ".py"
+  }
+  args = [
+    "-sv",
+    "--param",
+    "clang_site_config=" + rebase_path(clang_lit_site_cfg_file, root_out_dir),
+    "--param",
+    "clang_unit_site_config=" +
+        rebase_path(clang_lit_unit_site_cfg_file, root_out_dir),
+    rebase_path(".", root_out_dir),
+  ]
+  outputs = [
+    "$target_gen_dir/run-lit",  # Non-existing, so that ninja runs it each time.
+  ]
+
+  # Since check-clang is always dirty, //:default doesn't depend on it so that
+  # it's not part of the default ninja target.  Hence, check-clang shouldn't
+  # have any deps except :test. so that the default target is sure to build
+  # all the deps.
+  deps = [
+    ":test",
+  ]
+  testonly = true
+
+  pool = "//:console"
+}
diff --git a/utils/gn/secondary/clang/test/clang_lit_site_cfg_files.gni b/utils/gn/secondary/clang/test/clang_lit_site_cfg_files.gni
new file mode 100644
index 0000000000000000000000000000000000000000..f4079a697010b9d571beccd8b4a634bc841c05cc
--- /dev/null
+++ b/utils/gn/secondary/clang/test/clang_lit_site_cfg_files.gni
@@ -0,0 +1,2 @@
+clang_lit_site_cfg_file = "$root_gen_dir/clang/test/lit.site.cfg.py"
+clang_lit_unit_site_cfg_file = "$root_gen_dir/clang/test/Unit/lit.site.cfg.py"
diff --git a/utils/gn/secondary/clang/tools/arcmt-test/BUILD.gn b/utils/gn/secondary/clang/tools/arcmt-test/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..c16ce2a6b7a0465a97d575edb4b1451a2f27b4b3
--- /dev/null
+++ b/utils/gn/secondary/clang/tools/arcmt-test/BUILD.gn
@@ -0,0 +1,13 @@
+executable("arcmt-test") {
+  configs += [ "//llvm/utils/gn/build:clang_code" ]
+  deps = [
+    "//clang/lib/ARCMigrate",
+    "//clang/lib/Basic",
+    "//clang/lib/Frontend",
+    "//clang/lib/Lex",
+    "//llvm/lib/Support",
+  ]
+  sources = [
+    "arcmt-test.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/clang/tools/c-arcmt-test/BUILD.gn b/utils/gn/secondary/clang/tools/c-arcmt-test/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..2c77f7aac917c30c30358fdf5304a5e94969d0fa
--- /dev/null
+++ b/utils/gn/secondary/clang/tools/c-arcmt-test/BUILD.gn
@@ -0,0 +1,15 @@
+executable("c-arcmt-test") {
+  configs += [ "//llvm/utils/gn/build:clang_code" ]
+  deps = [
+    "//clang/tools/libclang",
+  ]
+  sources = [
+    "c-arcmt-test.c",
+  ]
+
+  # See comment at top of clang/tools/libclang/BUILD.gn for why this isn't
+  # needed on Linux.
+  if (host_os == "mac") {
+    ldflags = [ "-Wl,-rpath,@loader_path/../lib" ]
+  }
+}
diff --git a/utils/gn/secondary/clang/tools/c-index-test/BUILD.gn b/utils/gn/secondary/clang/tools/c-index-test/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..c168b6dc9cb39158ebfeef80d7efd7c686ab2c0e
--- /dev/null
+++ b/utils/gn/secondary/clang/tools/c-index-test/BUILD.gn
@@ -0,0 +1,28 @@
+executable("c-index-test") {
+  configs += [ "//llvm/utils/gn/build:clang_code" ]
+  deps = [
+    "//clang/include/clang/Config",
+    "//clang/lib/AST",
+    "//clang/lib/Basic",
+    "//clang/lib/CodeGen",
+    "//clang/lib/Frontend",
+    "//clang/lib/Index",
+    "//clang/lib/Serialization",
+    "//clang/tools/libclang",
+    "//llvm/lib/Support",
+    "//llvm/utils/gn/build/libs/xml",
+  ]
+  if (host_os != "win") {
+    cflags_c = [ "-std=gnu89" ]
+  }
+  sources = [
+    "c-index-test.c",
+    "core_main.cpp",
+  ]
+
+  # See comment at top of clang/tools/libclang/BUILD.gn for why this isn't
+  # needed on Linux.
+  if (host_os == "mac") {
+    ldflags = [ "-Wl,-rpath,@loader_path/../lib" ]
+  }
+}
diff --git a/utils/gn/secondary/clang/tools/clang-check/BUILD.gn b/utils/gn/secondary/clang/tools/clang-check/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..3d18b14949e5003078b1707037c301d4aa077bd1
--- /dev/null
+++ b/utils/gn/secondary/clang/tools/clang-check/BUILD.gn
@@ -0,0 +1,18 @@
+executable("clang-check") {
+  configs += [ "//llvm/utils/gn/build:clang_code" ]
+  deps = [
+    "//clang/lib/AST",
+    "//clang/lib/Basic",
+    "//clang/lib/Driver",
+    "//clang/lib/Frontend",
+    "//clang/lib/Frontend/Rewrite",
+    "//clang/lib/StaticAnalyzer/Frontend",
+    "//clang/lib/Tooling",
+    "//llvm/lib/Option",
+    "//llvm/lib/Support",
+    "//llvm/lib/Target:TargetsToBuild",
+  ]
+  sources = [
+    "ClangCheck.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/clang/tools/clang-diff/BUILD.gn b/utils/gn/secondary/clang/tools/clang-diff/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..c68cd29b43d1f71a83b5dd1d9f328014778e2196
--- /dev/null
+++ b/utils/gn/secondary/clang/tools/clang-diff/BUILD.gn
@@ -0,0 +1,13 @@
+executable("clang-diff") {
+  configs += [ "//llvm/utils/gn/build:clang_code" ]
+  deps = [
+    "//clang/lib/Basic",
+    "//clang/lib/Frontend",
+    "//clang/lib/Tooling",
+    "//clang/lib/Tooling/ASTDiff",
+    "//llvm/lib/Support",
+  ]
+  sources = [
+    "ClangDiff.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/clang/tools/clang-extdef-mapping/BUILD.gn b/utils/gn/secondary/clang/tools/clang-extdef-mapping/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..68b3af406e5dbc496f4b7e487cf725de6f27c4bb
--- /dev/null
+++ b/utils/gn/secondary/clang/tools/clang-extdef-mapping/BUILD.gn
@@ -0,0 +1,17 @@
+executable("clang-extdef-mapping") {
+  configs += [ "//llvm/utils/gn/build:clang_code" ]
+  deps = [
+    "//clang/lib/AST",
+    "//clang/lib/Basic",
+    "//clang/lib/CrossTU",
+    "//clang/lib/Frontend",
+    "//clang/lib/Index",
+    "//clang/lib/Tooling",
+    "//llvm/lib/AsmParser",
+    "//llvm/lib/MC",
+    "//llvm/lib/Support",
+  ]
+  sources = [
+    "ClangExtDefMapGen.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/clang/tools/clang-format/BUILD.gn b/utils/gn/secondary/clang/tools/clang-format/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..7352ae1433b552f260ccebd7b29dda29aabea15c
--- /dev/null
+++ b/utils/gn/secondary/clang/tools/clang-format/BUILD.gn
@@ -0,0 +1,13 @@
+executable("clang-format") {
+  configs += [ "//llvm/utils/gn/build:clang_code" ]
+  deps = [
+    "//clang/lib/Basic",
+    "//clang/lib/Format",
+    "//clang/lib/Rewrite",
+    "//clang/lib/Tooling/Core",
+    "//llvm/lib/Support",
+  ]
+  sources = [
+    "ClangFormat.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/clang/tools/clang-import-test/BUILD.gn b/utils/gn/secondary/clang/tools/clang-import-test/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..60a8a66dda00afe1a4b1a8fa83143d3b2808c57a
--- /dev/null
+++ b/utils/gn/secondary/clang/tools/clang-import-test/BUILD.gn
@@ -0,0 +1,17 @@
+executable("clang-import-test") {
+  configs += [ "//llvm/utils/gn/build:clang_code" ]
+  deps = [
+    "//clang/lib/AST",
+    "//clang/lib/Basic",
+    "//clang/lib/CodeGen",
+    "//clang/lib/Driver",
+    "//clang/lib/Frontend",
+    "//clang/lib/Lex",
+    "//clang/lib/Parse",
+    "//llvm/lib/IR",
+    "//llvm/lib/Support",
+  ]
+  sources = [
+    "clang-import-test.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/clang/tools/clang-offload-bundler/BUILD.gn b/utils/gn/secondary/clang/tools/clang-offload-bundler/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..4335087b53d160789620de4aba2145c77a8a445b
--- /dev/null
+++ b/utils/gn/secondary/clang/tools/clang-offload-bundler/BUILD.gn
@@ -0,0 +1,13 @@
+executable("clang-offload-bundler") {
+  configs += [ "//llvm/utils/gn/build:clang_code" ]
+  deps = [
+    "//clang/lib/Basic",
+    "//llvm/lib/Bitcode/Writer",
+    "//llvm/lib/IR",
+    "//llvm/lib/Object",
+    "//llvm/lib/Support",
+  ]
+  sources = [
+    "ClangOffloadBundler.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/clang/tools/clang-refactor/BUILD.gn b/utils/gn/secondary/clang/tools/clang-refactor/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..4942aeb4bad5b5313708acd471bbec6954f77e5d
--- /dev/null
+++ b/utils/gn/secondary/clang/tools/clang-refactor/BUILD.gn
@@ -0,0 +1,19 @@
+executable("clang-refactor") {
+  configs += [ "//llvm/utils/gn/build:clang_code" ]
+  deps = [
+    "//clang/lib/AST",
+    "//clang/lib/Basic",
+    "//clang/lib/Format",
+    "//clang/lib/Frontend",
+    "//clang/lib/Lex",
+    "//clang/lib/Rewrite",
+    "//clang/lib/Tooling",
+    "//clang/lib/Tooling/Refactoring",
+    "//llvm/lib/Option",
+    "//llvm/lib/Support",
+  ]
+  sources = [
+    "ClangRefactor.cpp",
+    "TestSupport.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/clang/tools/clang-rename/BUILD.gn b/utils/gn/secondary/clang/tools/clang-rename/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..ad08ec1daf47dd31983a6365aa4c3cf6c62d27e2
--- /dev/null
+++ b/utils/gn/secondary/clang/tools/clang-rename/BUILD.gn
@@ -0,0 +1,16 @@
+executable("clang-rename") {
+  configs += [ "//llvm/utils/gn/build:clang_code" ]
+  deps = [
+    "//clang/lib/Basic",
+    "//clang/lib/Frontend",
+    "//clang/lib/Rewrite",
+    "//clang/lib/Tooling",
+    "//clang/lib/Tooling/Core",
+    "//clang/lib/Tooling/Refactoring",
+    "//llvm/lib/Option",
+    "//llvm/lib/Support",
+  ]
+  sources = [
+    "ClangRename.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/clang/tools/diagtool/BUILD.gn b/utils/gn/secondary/clang/tools/diagtool/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..52924d70e9a3e172b430e171630992e2f10a8f8d
--- /dev/null
+++ b/utils/gn/secondary/clang/tools/diagtool/BUILD.gn
@@ -0,0 +1,18 @@
+executable("diagtool") {
+  configs += [ "//llvm/utils/gn/build:clang_code" ]
+  deps = [
+    "//clang/include/clang/Basic:DiagnosticIndexName",
+    "//clang/lib/Basic",
+    "//clang/lib/Frontend",
+    "//llvm/lib/Support",
+  ]
+  sources = [
+    "DiagTool.cpp",
+    "DiagnosticNames.cpp",
+    "FindDiagnosticID.cpp",
+    "ListWarnings.cpp",
+    "ShowEnabledWarnings.cpp",
+    "TreeView.cpp",
+    "diagtool_main.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/clang/tools/driver/BUILD.gn b/utils/gn/secondary/clang/tools/driver/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..7324e047ade6b830547a2136275dcff69cd8898e
--- /dev/null
+++ b/utils/gn/secondary/clang/tools/driver/BUILD.gn
@@ -0,0 +1,91 @@
+import("//llvm/utils/gn/build/symlink_or_copy.gni")
+import("//llvm/utils/gn/build/write_cmake_config.gni")
+import("//llvm/version.gni")
+
+symlinks = [
+  # target_name, symlink_target pairs: GN doesn't support '+' in rule names.
+  [
+    "clangxx",
+    "clang++",
+  ],
+  [
+    "clang-cl",
+    "clang-cl",
+  ],
+  [
+    "clang-cpp",
+    "clang-cpp",
+  ],
+]
+foreach(target, symlinks) {
+  symlink_or_copy(target[0]) {
+    deps = [
+      ":clang",
+    ]
+    source = "clang"
+    output = "$root_out_dir/bin/${target[1]}"
+  }
+}
+
+# //:clang depends on this symlink target, see comment in //BUILD.gn.
+group("symlinks") {
+  deps = []
+  foreach(target, symlinks) {
+    deps += [ ":${target[0]}" ]
+  }
+}
+
+if (host_os == "mac") {
+  write_cmake_config("write_info_plist") {
+    input = "Info.plist.in"
+    output = "$target_gen_dir/Info.plist"
+    values = [
+      "TOOL_INFO_BUILD_VERSION=$llvm_version_major.$llvm_version_minor",
+      "TOOL_INFO_NAME=clang",
+      "TOOL_INFO_UTI=org.llvm.clang",
+      "TOOL_INFO_VERSION=$llvm_version",
+    ]
+  }
+}
+
+executable("clang") {
+  configs += [ "//llvm/utils/gn/build:clang_code" ]
+  deps = [
+    "//clang/include/clang/Config",
+    "//clang/lib/Basic",
+    "//clang/lib/CodeGen",
+    "//clang/lib/Driver",
+    "//clang/lib/Frontend",
+    "//clang/lib/FrontendTool",
+    "//clang/lib/Headers",
+    "//clang/tools/clang-offload-bundler",
+    "//llvm/include/llvm/Config:llvm-config",
+    "//llvm/lib/Analysis",
+    "//llvm/lib/CodeGen",
+    "//llvm/lib/IR",
+    "//llvm/lib/MC",
+    "//llvm/lib/MC/MCParser",
+    "//llvm/lib/Option",
+    "//llvm/lib/Support",
+    "//llvm/lib/Target:TargetsToBuild",
+    "//llvm/lib/Transforms/IPO",
+    "//llvm/lib/Transforms/InstCombine",
+    "//llvm/lib/Transforms/Instrumentation",
+    "//llvm/lib/Transforms/ObjCARC",
+    "//llvm/lib/Transforms/Scalar",
+    "//llvm/lib/Transforms/Utils",
+    "//llvm/lib/Transforms/Vectorize",
+  ]
+  if (host_os == "mac") {
+    deps += [ ":write_info_plist" ]
+    plist = get_target_outputs(":write_info_plist")
+    ldflags = [ "-Wl,-sectcreate,__TEXT,__info_plist," +
+                rebase_path(plist[0], root_out_dir) ]
+  }
+  sources = [
+    "cc1_main.cpp",
+    "cc1as_main.cpp",
+    "cc1gen_reproducer_main.cpp",
+    "driver.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/clang/tools/libclang/BUILD.gn b/utils/gn/secondary/clang/tools/libclang/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..7417e5982ee031c00fcbd1e25fa95fe99a89af9c
--- /dev/null
+++ b/utils/gn/secondary/clang/tools/libclang/BUILD.gn
@@ -0,0 +1,89 @@
+import("//clang/lib/ARCMigrate/enable.gni")
+import("//llvm/version.gni")
+
+# This build file is just enough to get check-clang to pass, it's missing
+# several things from the CMake build:
+# - linking in clangTidyPlugin and clangIncludeFixerPlugin from
+#   clang-tools-extra (which doesn't have any GN build files yet)
+# - using libclang.exports
+# - a build target copying the Python bindings
+# - the GN linux build always builds without -fPIC (as if LLVM_ENABLE_PIC=OFF
+#   in the CMake build), so libclang is always a static library on linux
+# - the GN build doesn't have LIBCLANG_BUILD_STATIC
+
+libclang_target_type = "shared_library"
+if (host_os != "win" && host_os != "mac") {
+  # ELF targets need -fPIC to build shared libs but they aren't on by default.
+  # For now, make libclang a static lib there.
+  libclang_target_type = "static_library"
+}
+
+target(libclang_target_type, "libclang") {
+  configs += [ "//llvm/utils/gn/build:clang_code" ]
+  deps = [
+    "//clang/include/clang/Config",
+    "//clang/lib/AST",
+    "//clang/lib/Basic",
+    "//clang/lib/Frontend",
+    "//clang/lib/Headers",
+    "//clang/lib/Index",
+    "//clang/lib/Lex",
+    "//clang/lib/Sema",
+    "//clang/lib/Tooling",
+    "//llvm/include/llvm/Config:llvm-config",
+    "//llvm/lib/IR",
+    "//llvm/lib/Support",
+    "//llvm/lib/Target:TargetsToBuild",
+  ]
+  if (clang_enable_arcmt) {
+    deps += [ "//clang/lib/ARCMigrate" ]
+  }
+
+  if (host_os == "win") {
+    defines = [ "_CINDEX_LIB_" ]
+  }
+
+  sources = [
+    "ARCMigrate.cpp",
+    "BuildSystem.cpp",
+    "CIndex.cpp",
+    "CIndexCXX.cpp",
+    "CIndexCodeCompletion.cpp",
+    "CIndexDiagnostic.cpp",
+    "CIndexDiagnostic.h",
+    "CIndexHigh.cpp",
+    "CIndexInclusionStack.cpp",
+    "CIndexUSRs.cpp",
+    "CIndexer.cpp",
+    "CIndexer.h",
+    "CXComment.cpp",
+    "CXCompilationDatabase.cpp",
+    "CXCursor.cpp",
+    "CXCursor.h",
+    "CXIndexDataConsumer.cpp",
+    "CXLoadedDiagnostic.cpp",
+    "CXLoadedDiagnostic.h",
+    "CXSourceLocation.cpp",
+    "CXSourceLocation.h",
+    "CXStoredDiagnostic.cpp",
+    "CXString.cpp",
+    "CXString.h",
+    "CXTranslationUnit.h",
+    "CXType.cpp",
+    "CXType.h",
+    "Index_Internal.h",
+    "Indexing.cpp",
+  ]
+  if (host_os == "mac") {
+    ldflags = [
+      "-Wl,-compatibility_version,1",
+      "-Wl,-current_version,$llvm_version",
+
+      # See llvm_setup_rpath() in CMake.
+      "-Wl,-install_name,@rpath/libclang.dylib",
+      "-Wl,-rpath,@loader_path/../lib",
+    ]
+  }
+
+  # FIXME: Use libclang.exports
+}
diff --git a/utils/gn/secondary/clang/unittests/AST/BUILD.gn b/utils/gn/secondary/clang/unittests/AST/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..d5b00af3ae57eba12717419bef2c141b22c34a35
--- /dev/null
+++ b/utils/gn/secondary/clang/unittests/AST/BUILD.gn
@@ -0,0 +1,33 @@
+import("//llvm/utils/unittest/unittest.gni")
+
+unittest("ASTTests") {
+  configs += [ "//llvm/utils/gn/build:clang_code" ]
+  deps = [
+    "//clang/lib/AST",
+    "//clang/lib/ASTMatchers",
+    "//clang/lib/Analysis",
+    "//clang/lib/Basic",
+    "//clang/lib/Frontend",
+    "//clang/lib/Tooling",
+    "//llvm/lib/Support",
+  ]
+  sources = [
+    "ASTContextParentMapTest.cpp",
+    "ASTImporterTest.cpp",
+    "ASTTypeTraitsTest.cpp",
+    "ASTVectorTest.cpp",
+    "CommentLexer.cpp",
+    "CommentParser.cpp",
+    "CommentTextTest.cpp",
+    "DataCollectionTest.cpp",
+    "DeclPrinterTest.cpp",
+    "DeclTest.cpp",
+    "EvaluateAsRValueTest.cpp",
+    "ExternalASTSourceTest.cpp",
+    "Language.cpp",
+    "NamedDeclPrinterTest.cpp",
+    "SourceLocationTest.cpp",
+    "StmtPrinterTest.cpp",
+    "StructuralEquivalenceTest.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/clang/unittests/ASTMatchers/BUILD.gn b/utils/gn/secondary/clang/unittests/ASTMatchers/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..241d03dd61c4f3c1474bb70baa34a514072e865c
--- /dev/null
+++ b/utils/gn/secondary/clang/unittests/ASTMatchers/BUILD.gn
@@ -0,0 +1,19 @@
+import("//llvm/utils/unittest/unittest.gni")
+
+unittest("ASTMatchersTests") {
+  configs += [ "//llvm/utils/gn/build:clang_code" ]
+  deps = [
+    "//clang/lib/AST",
+    "//clang/lib/ASTMatchers",
+    "//clang/lib/Basic",
+    "//clang/lib/Frontend",
+    "//clang/lib/Tooling",
+    "//llvm/lib/Support",
+  ]
+  sources = [
+    "ASTMatchersInternalTest.cpp",
+    "ASTMatchersNarrowingTest.cpp",
+    "ASTMatchersNodeTest.cpp",
+    "ASTMatchersTraversalTest.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/clang/unittests/ASTMatchers/Dynamic/BUILD.gn b/utils/gn/secondary/clang/unittests/ASTMatchers/Dynamic/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..de89f0fc809435f7f7d370b6f06a731b69d53ad8
--- /dev/null
+++ b/utils/gn/secondary/clang/unittests/ASTMatchers/Dynamic/BUILD.gn
@@ -0,0 +1,19 @@
+import("//llvm/utils/unittest/unittest.gni")
+
+unittest("DynamicASTMatchersTests") {
+  configs += [ "//llvm/utils/gn/build:clang_code" ]
+  deps = [
+    "//clang/lib/AST",
+    "//clang/lib/ASTMatchers",
+    "//clang/lib/ASTMatchers/Dynamic",
+    "//clang/lib/Basic",
+    "//clang/lib/Frontend",
+    "//clang/lib/Tooling",
+    "//llvm/lib/Support",
+  ]
+  sources = [
+    "ParserTest.cpp",
+    "RegistryTest.cpp",
+    "VariantValueTest.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/clang/unittests/Analysis/BUILD.gn b/utils/gn/secondary/clang/unittests/Analysis/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..c6c6fbe58a96b09297a73e55b49411586797b030
--- /dev/null
+++ b/utils/gn/secondary/clang/unittests/Analysis/BUILD.gn
@@ -0,0 +1,19 @@
+import("//llvm/utils/unittest/unittest.gni")
+
+unittest("ClangAnalysisTests") {
+  configs += [ "//llvm/utils/gn/build:clang_code" ]
+  deps = [
+    "//clang/lib/AST",
+    "//clang/lib/ASTMatchers",
+    "//clang/lib/Analysis",
+    "//clang/lib/Basic",
+    "//clang/lib/Frontend",
+    "//clang/lib/Tooling",
+    "//llvm/lib/Support",
+  ]
+  sources = [
+    "CFGTest.cpp",
+    "CloneDetectionTest.cpp",
+    "ExprMutationAnalyzerTest.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/clang/unittests/BUILD.gn b/utils/gn/secondary/clang/unittests/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..a8293ba926bbf9737543a657d8002cd8f1b972b6
--- /dev/null
+++ b/utils/gn/secondary/clang/unittests/BUILD.gn
@@ -0,0 +1,35 @@
+import("//clang/lib/StaticAnalyzer/Frontend/enable.gni")
+
+group("unittests") {
+  deps = [
+    "AST:ASTTests",
+    "ASTMatchers:ASTMatchersTests",
+    "ASTMatchers/Dynamic:DynamicASTMatchersTests",
+    "Basic:BasicTests",
+    "CodeGen:ClangCodeGenTests",
+    "CrossTU:CrossTUTests",
+    "Driver:ClangDriverTests",
+    "Format:FormatTests",
+    "Index:IndexTests",
+    "Lex:LexTests",
+    "Rename:ClangRenameTests",
+    "Rewrite:RewriteTests",
+    "Sema:SemaTests",
+    "Tooling:ToolingTests",
+  ]
+  if (clang_enable_static_analyzer) {
+    deps += [
+      "Analysis:ClangAnalysisTests",
+      "Frontend:FrontendTests",
+      "StaticAnalyzer:StaticAnalysisTests",
+    ]
+  }
+  if (host_os != "win") {
+    # FIXME: libclang unit tests are disabled on Windows due
+    # to failures, mostly in libclang.VirtualFileOverlay_*.
+    # FIXME: Also, the executable can't find libclang.dll since that's
+    # in a different directory.
+    deps += [ "libclang:libclangTests" ]
+  }
+  testonly = true
+}
diff --git a/utils/gn/secondary/clang/unittests/Basic/BUILD.gn b/utils/gn/secondary/clang/unittests/Basic/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..31277aea8d2097facfc6e93d8a0317f8422344fd
--- /dev/null
+++ b/utils/gn/secondary/clang/unittests/Basic/BUILD.gn
@@ -0,0 +1,18 @@
+import("//llvm/utils/unittest/unittest.gni")
+
+unittest("BasicTests") {
+  configs += [ "//llvm/utils/gn/build:clang_code" ]
+  deps = [
+    "//clang/lib/Basic",
+    "//clang/lib/Lex",
+    "//llvm/lib/Support",
+  ]
+  sources = [
+    "CharInfoTest.cpp",
+    "DiagnosticTest.cpp",
+    "FileManagerTest.cpp",
+    "FixedPointTest.cpp",
+    "MemoryBufferCacheTest.cpp",
+    "SourceManagerTest.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/clang/unittests/CodeGen/BUILD.gn b/utils/gn/secondary/clang/unittests/CodeGen/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..37b514eab496c9f4469755b6e9b23e6d38f48445
--- /dev/null
+++ b/utils/gn/secondary/clang/unittests/CodeGen/BUILD.gn
@@ -0,0 +1,21 @@
+import("//llvm/utils/unittest/unittest.gni")
+
+unittest("ClangCodeGenTests") {
+  configs += [ "//llvm/utils/gn/build:clang_code" ]
+  deps = [
+    "//clang/lib/AST",
+    "//clang/lib/Basic",
+    "//clang/lib/CodeGen",
+    "//clang/lib/Frontend",
+    "//clang/lib/Lex",
+    "//clang/lib/Parse",
+    "//llvm/lib/IR",
+    "//llvm/lib/Support",
+  ]
+  sources = [
+    "BufferSourceTest.cpp",
+    "CodeGenExternalTest.cpp",
+    "IncrementalProcessingTest.cpp",
+    "TBAAMetadataTest.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/clang/unittests/CrossTU/BUILD.gn b/utils/gn/secondary/clang/unittests/CrossTU/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..6330f30057952efd6f9960d95a30913b3d6a5fa5
--- /dev/null
+++ b/utils/gn/secondary/clang/unittests/CrossTU/BUILD.gn
@@ -0,0 +1,17 @@
+import("//llvm/utils/unittest/unittest.gni")
+
+unittest("CrossTUTests") {
+  configs += [ "//llvm/utils/gn/build:clang_code" ]
+  deps = [
+    "//clang/lib/AST",
+    "//clang/lib/Basic",
+    "//clang/lib/CrossTU",
+    "//clang/lib/Frontend",
+    "//clang/lib/Tooling",
+    "//llvm/lib/Support",
+    "//llvm/lib/Target:TargetsToBuild",
+  ]
+  sources = [
+    "CrossTranslationUnitTest.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/clang/unittests/Driver/BUILD.gn b/utils/gn/secondary/clang/unittests/Driver/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..6a91b0364416108f67aef18a49f96440081f5655
--- /dev/null
+++ b/utils/gn/secondary/clang/unittests/Driver/BUILD.gn
@@ -0,0 +1,18 @@
+import("//llvm/utils/unittest/unittest.gni")
+
+unittest("ClangDriverTests") {
+  configs += [ "//llvm/utils/gn/build:clang_code" ]
+  deps = [
+    "//clang/lib/Basic",
+    "//clang/lib/Driver",
+    "//llvm/lib/Option",
+    "//llvm/lib/Support",
+    "//llvm/lib/Target:TargetsToBuild",
+  ]
+  sources = [
+    "DistroTest.cpp",
+    "ModuleCacheTest.cpp",
+    "MultilibTest.cpp",
+    "ToolChainTest.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/clang/unittests/Format/BUILD.gn b/utils/gn/secondary/clang/unittests/Format/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..aaf990b485ee87d1aa85ae24ddb15b8aad897b9c
--- /dev/null
+++ b/utils/gn/secondary/clang/unittests/Format/BUILD.gn
@@ -0,0 +1,31 @@
+import("//llvm/utils/unittest/unittest.gni")
+
+unittest("FormatTests") {
+  configs += [ "//llvm/utils/gn/build:clang_code" ]
+  deps = [
+    "//clang/lib/Basic",
+    "//clang/lib/Format",
+    "//clang/lib/Frontend",
+    "//clang/lib/Rewrite",
+    "//clang/lib/Tooling/Core",
+    "//llvm/lib/Support",
+  ]
+  sources = [
+    "CleanupTest.cpp",
+    "FormatTest.cpp",
+    "FormatTestComments.cpp",
+    "FormatTestJS.cpp",
+    "FormatTestJava.cpp",
+    "FormatTestObjC.cpp",
+    "FormatTestProto.cpp",
+    "FormatTestRawStrings.cpp",
+    "FormatTestSelective.cpp",
+    "FormatTestTableGen.cpp",
+    "FormatTestTextProto.cpp",
+    "NamespaceEndCommentsFixerTest.cpp",
+    "SortImportsTestJS.cpp",
+    "SortImportsTestJava.cpp",
+    "SortIncludesTest.cpp",
+    "UsingDeclarationsSorterTest.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/clang/unittests/Frontend/BUILD.gn b/utils/gn/secondary/clang/unittests/Frontend/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..f3b535cf486c231f04619cd250f055d02401a9f1
--- /dev/null
+++ b/utils/gn/secondary/clang/unittests/Frontend/BUILD.gn
@@ -0,0 +1,25 @@
+import("//llvm/utils/unittest/unittest.gni")
+
+unittest("FrontendTests") {
+  configs += [ "//llvm/utils/gn/build:clang_code" ]
+  deps = [
+    "//clang/lib/AST",
+    "//clang/lib/Basic",
+    "//clang/lib/CodeGen",
+    "//clang/lib/Frontend",
+    "//clang/lib/FrontendTool",
+    "//clang/lib/Lex",
+    "//clang/lib/Sema",
+    "//llvm/lib/Support",
+  ]
+  sources = [
+    "ASTUnitTest.cpp",
+    "CodeGenActionTest.cpp",
+    "CompilerInstanceTest.cpp",
+    "FixedPointString.cpp",
+    "FrontendActionTest.cpp",
+    "OutputStreamTest.cpp",
+    "PCHPreambleTest.cpp",
+    "ParsedSourceLocationTest.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/clang/unittests/Index/BUILD.gn b/utils/gn/secondary/clang/unittests/Index/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..c2e6395f3772e2411d521d8134a49d0e3ce374e5
--- /dev/null
+++ b/utils/gn/secondary/clang/unittests/Index/BUILD.gn
@@ -0,0 +1,18 @@
+import("//llvm/utils/unittest/unittest.gni")
+
+unittest("IndexTests") {
+  configs += [ "//llvm/utils/gn/build:clang_code" ]
+  deps = [
+    "//clang/lib/AST",
+    "//clang/lib/Basic",
+    "//clang/lib/Frontend",
+    "//clang/lib/Index",
+    "//clang/lib/Lex",
+    "//clang/lib/Serialization",
+    "//clang/lib/Tooling",
+    "//llvm/lib/Support",
+  ]
+  sources = [
+    "IndexTests.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/clang/unittests/Lex/BUILD.gn b/utils/gn/secondary/clang/unittests/Lex/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..63180cb99537ad46235f2a7476fc46c659d49a8b
--- /dev/null
+++ b/utils/gn/secondary/clang/unittests/Lex/BUILD.gn
@@ -0,0 +1,20 @@
+import("//llvm/utils/unittest/unittest.gni")
+
+unittest("LexTests") {
+  configs += [ "//llvm/utils/gn/build:clang_code" ]
+  deps = [
+    "//clang/lib/AST",
+    "//clang/lib/Basic",
+    "//clang/lib/Lex",
+    "//clang/lib/Parse",
+    "//clang/lib/Sema",
+    "//llvm/lib/Support",
+  ]
+  sources = [
+    "HeaderMapTest.cpp",
+    "HeaderSearchTest.cpp",
+    "LexerTest.cpp",
+    "PPCallbacksTest.cpp",
+    "PPConditionalDirectiveRecordTest.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/clang/unittests/Rename/BUILD.gn b/utils/gn/secondary/clang/unittests/Rename/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..54c0dba085957b1abfce838998304bc97eb00dc4
--- /dev/null
+++ b/utils/gn/secondary/clang/unittests/Rename/BUILD.gn
@@ -0,0 +1,28 @@
+import("//llvm/utils/unittest/unittest.gni")
+
+unittest("ClangRenameTests") {
+  configs += [ "//llvm/utils/gn/build:clang_code" ]
+
+  # We'd like clang/unittests/Tooling/RewriterTestContext.h in the test.
+  include_dirs = [ "../.." ]
+
+  deps = [
+    "//clang/lib/AST",
+    "//clang/lib/ASTMatchers",
+    "//clang/lib/Basic",
+    "//clang/lib/Format",
+    "//clang/lib/Frontend",
+    "//clang/lib/Rewrite",
+    "//clang/lib/Tooling",
+    "//clang/lib/Tooling/Core",
+    "//clang/lib/Tooling/Refactoring",
+    "//llvm/lib/Support",
+  ]
+  sources = [
+    "RenameAliasTest.cpp",
+    "RenameClassTest.cpp",
+    "RenameEnumTest.cpp",
+    "RenameFunctionTest.cpp",
+    "RenameMemberTest.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/clang/unittests/Rewrite/BUILD.gn b/utils/gn/secondary/clang/unittests/Rewrite/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..350f67d2e08d07e388900cf0e349ff0742ed66ce
--- /dev/null
+++ b/utils/gn/secondary/clang/unittests/Rewrite/BUILD.gn
@@ -0,0 +1,12 @@
+import("//llvm/utils/unittest/unittest.gni")
+
+unittest("RewriteTests") {
+  configs += [ "//llvm/utils/gn/build:clang_code" ]
+  deps = [
+    "//clang/lib/Rewrite",
+    "//llvm/lib/Support",
+  ]
+  sources = [
+    "RewriteBufferTest.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/clang/unittests/Sema/BUILD.gn b/utils/gn/secondary/clang/unittests/Sema/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..b65865d37e688638bf5b99e5b1c1ef0da3ef31f1
--- /dev/null
+++ b/utils/gn/secondary/clang/unittests/Sema/BUILD.gn
@@ -0,0 +1,18 @@
+import("//llvm/utils/unittest/unittest.gni")
+
+unittest("SemaTests") {
+  configs += [ "//llvm/utils/gn/build:clang_code" ]
+  deps = [
+    "//clang/lib/AST",
+    "//clang/lib/Basic",
+    "//clang/lib/Frontend",
+    "//clang/lib/Parse",
+    "//clang/lib/Sema",
+    "//clang/lib/Tooling",
+    "//llvm/lib/Support",
+  ]
+  sources = [
+    "CodeCompleteTest.cpp",
+    "ExternalSemaSourceTest.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/clang/unittests/StaticAnalyzer/BUILD.gn b/utils/gn/secondary/clang/unittests/StaticAnalyzer/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..fae8f7e83707a7da95d2d6994db635bd21cbb9e1
--- /dev/null
+++ b/utils/gn/secondary/clang/unittests/StaticAnalyzer/BUILD.gn
@@ -0,0 +1,18 @@
+import("//llvm/utils/unittest/unittest.gni")
+
+unittest("StaticAnalysisTests") {
+  configs += [ "//llvm/utils/gn/build:clang_code" ]
+  deps = [
+    "//clang/lib/Analysis",
+    "//clang/lib/Basic",
+    "//clang/lib/StaticAnalyzer/Core",
+    "//clang/lib/StaticAnalyzer/Frontend",
+    "//clang/lib/Tooling",
+    "//llvm/lib/Support",
+  ]
+  sources = [
+    "AnalyzerOptionsTest.cpp",
+    "RegisterCustomCheckersTest.cpp",
+    "SymbolReaperTest.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/clang/unittests/Tooling/BUILD.gn b/utils/gn/secondary/clang/unittests/Tooling/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..5b027dc1b843ab71a68614b447f0fd8478cda687
--- /dev/null
+++ b/utils/gn/secondary/clang/unittests/Tooling/BUILD.gn
@@ -0,0 +1,61 @@
+import("//llvm/utils/unittest/unittest.gni")
+
+unittest("ToolingTests") {
+  configs += [ "//llvm/utils/gn/build:clang_code" ]
+  include_dirs = [ "." ]
+  deps = [
+    "//clang/lib/AST",
+    "//clang/lib/ASTMatchers",
+    "//clang/lib/Basic",
+    "//clang/lib/Format",
+    "//clang/lib/Frontend",
+    "//clang/lib/Lex",
+    "//clang/lib/Rewrite",
+    "//clang/lib/Tooling",
+    "//clang/lib/Tooling/Core",
+    "//clang/lib/Tooling/Refactoring",
+    "//llvm/lib/Support",
+    "//llvm/lib/Target:TargetsToBuild",
+  ]
+  sources = [
+    "ASTSelectionTest.cpp",
+    "CastExprTest.cpp",
+    "CommentHandlerTest.cpp",
+    "CompilationDatabaseTest.cpp",
+    "DiagnosticsYamlTest.cpp",
+    "ExecutionTest.cpp",
+    "FixItTest.cpp",
+    "HeaderIncludesTest.cpp",
+    "LexicallyOrderedRecursiveASTVisitorTest.cpp",
+    "LookupTest.cpp",
+    "QualTypeNamesTest.cpp",
+    "RecursiveASTVisitorTestDeclVisitor.cpp",
+    "RecursiveASTVisitorTestPostOrderVisitor.cpp",
+    "RecursiveASTVisitorTestTypeLocVisitor.cpp",
+    "RecursiveASTVisitorTests/Attr.cpp",
+    "RecursiveASTVisitorTests/CXXBoolLiteralExpr.cpp",
+    "RecursiveASTVisitorTests/CXXMemberCall.cpp",
+    "RecursiveASTVisitorTests/CXXOperatorCallExprTraverser.cpp",
+    "RecursiveASTVisitorTests/Class.cpp",
+    "RecursiveASTVisitorTests/ConstructExpr.cpp",
+    "RecursiveASTVisitorTests/DeclRefExpr.cpp",
+    "RecursiveASTVisitorTests/ImplicitCtor.cpp",
+    "RecursiveASTVisitorTests/InitListExprPostOrder.cpp",
+    "RecursiveASTVisitorTests/InitListExprPostOrderNoQueue.cpp",
+    "RecursiveASTVisitorTests/InitListExprPreOrder.cpp",
+    "RecursiveASTVisitorTests/InitListExprPreOrderNoQueue.cpp",
+    "RecursiveASTVisitorTests/IntegerLiteral.cpp",
+    "RecursiveASTVisitorTests/LambdaDefaultCapture.cpp",
+    "RecursiveASTVisitorTests/LambdaExpr.cpp",
+    "RecursiveASTVisitorTests/NestedNameSpecifiers.cpp",
+    "RecursiveASTVisitorTests/ParenExpr.cpp",
+    "RecursiveASTVisitorTests/TemplateArgumentLocTraverser.cpp",
+    "RecursiveASTVisitorTests/TraversalScope.cpp",
+    "RefactoringActionRulesTest.cpp",
+    "RefactoringCallbacksTest.cpp",
+    "RefactoringTest.cpp",
+    "ReplacementsYamlTest.cpp",
+    "RewriterTest.cpp",
+    "ToolingTest.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/clang/unittests/libclang/BUILD.gn b/utils/gn/secondary/clang/unittests/libclang/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..cd99640423f97325690dca48249919638e2a9116
--- /dev/null
+++ b/utils/gn/secondary/clang/unittests/libclang/BUILD.gn
@@ -0,0 +1,14 @@
+import("//llvm/utils/unittest/unittest.gni")
+
+unittest("libclangTests") {
+  configs += [ "//llvm/utils/gn/build:clang_code" ]
+  deps = [
+    "//clang/tools/libclang",
+  ]
+  sources = [
+    "LibclangTest.cpp",
+  ]
+  if (host_os == "mac") {
+    ldflags = [ "-Wl,-rpath," + rebase_path("$root_out_dir/lib") ]
+  }
+}
diff --git a/utils/gn/secondary/clang/utils/TableGen/BUILD.gn b/utils/gn/secondary/clang/utils/TableGen/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..acfcccab6b1566e92d72a30c468ac64d348b1588
--- /dev/null
+++ b/utils/gn/secondary/clang/utils/TableGen/BUILD.gn
@@ -0,0 +1,19 @@
+executable("clang-tblgen") {
+  deps = [
+    "//llvm/lib/Support",
+    "//llvm/lib/TableGen",
+  ]
+  sources = [
+    "ClangASTNodesEmitter.cpp",
+    "ClangAttrEmitter.cpp",
+    "ClangCommentCommandInfoEmitter.cpp",
+    "ClangCommentHTMLNamedCharacterReferenceEmitter.cpp",
+    "ClangCommentHTMLTagsEmitter.cpp",
+    "ClangDataCollectorsEmitter.cpp",
+    "ClangDiagnosticsEmitter.cpp",
+    "ClangOptionDocEmitter.cpp",
+    "ClangSACheckersEmitter.cpp",
+    "NeonEmitter.cpp",
+    "TableGen.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/clang/utils/TableGen/clang_tablegen.gni b/utils/gn/secondary/clang/utils/TableGen/clang_tablegen.gni
new file mode 100644
index 0000000000000000000000000000000000000000..efc1e3c5d5427ae0474e5d3d4499117c87b20634
--- /dev/null
+++ b/utils/gn/secondary/clang/utils/TableGen/clang_tablegen.gni
@@ -0,0 +1,40 @@
+# This file introduces a templates for running clang-tblgen.
+#
+# Parameters:
+#
+#   args (required)
+#       [list of strings] Flags to pass to llvm-tblgen.
+#
+#   output_name (optional)
+#       Basename of the generated output file.
+#       Defaults to target name with ".inc" appended.
+#
+#   td_file (optional)
+#       The .td file to pass to llvm-tblgen.
+#       Defaults to target name with ".td" appended.
+#
+#   visibility (optional)
+#       GN's regular visibility attribute, see `gn help visibility`.
+#
+# Example of usage:
+#
+#   clang_tablegen("DiagnosticGroups") {
+#     args = [ "-gen-clang-diag-groups" ]
+#     td_file = "Diagnostic.td"
+#   }
+
+import("//llvm/utils/TableGen/tablegen.gni")
+
+template("clang_tablegen") {
+  tablegen(target_name) {
+    forward_variables_from(invoker,
+                           [
+                             "args",
+                             "output_name",
+                             "td_file",
+                             "visibility",
+                           ])
+
+    tblgen_target = "//clang/utils/TableGen:clang-tblgen"
+  }
+}
diff --git a/utils/gn/secondary/clang/utils/hmaptool/BUILD.gn b/utils/gn/secondary/clang/utils/hmaptool/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..f76db221b9933958259f4320f607d30f1438f833
--- /dev/null
+++ b/utils/gn/secondary/clang/utils/hmaptool/BUILD.gn
@@ -0,0 +1,8 @@
+copy("hmaptool") {
+  sources = [
+    "hmaptool",
+  ]
+  outputs = [
+    "$root_out_dir/bin/{{source_file_part}}",
+  ]
+}
diff --git a/utils/gn/secondary/compiler-rt/include/BUILD.gn b/utils/gn/secondary/compiler-rt/include/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..39aac9fd933986d134372d4f093bdf320b7d362b
--- /dev/null
+++ b/utils/gn/secondary/compiler-rt/include/BUILD.gn
@@ -0,0 +1,26 @@
+import("//clang/resource_dir.gni")
+
+copy("include") {
+  sources = [
+    "sanitizer/allocator_interface.h",
+    "sanitizer/asan_interface.h",
+    "sanitizer/common_interface_defs.h",
+    "sanitizer/coverage_interface.h",
+    "sanitizer/dfsan_interface.h",
+    "sanitizer/esan_interface.h",
+    "sanitizer/hwasan_interface.h",
+    "sanitizer/linux_syscall_hooks.h",
+    "sanitizer/lsan_interface.h",
+    "sanitizer/msan_interface.h",
+    "sanitizer/netbsd_syscall_hooks.h",
+    "sanitizer/scudo_interface.h",
+    "sanitizer/tsan_interface.h",
+    "sanitizer/tsan_interface_atomic.h",
+    "xray/xray_interface.h",
+    "xray/xray_log_interface.h",
+    "xray/xray_records.h",
+  ]
+  outputs = [
+    "$clang_resource_dir/include/{{source_target_relative}}",
+  ]
+}
diff --git a/utils/gn/secondary/compiler-rt/lib/cfi/BUILD.gn b/utils/gn/secondary/compiler-rt/lib/cfi/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..dd24b5b987b5277e77b8474c2136039911a3dfbe
--- /dev/null
+++ b/utils/gn/secondary/compiler-rt/lib/cfi/BUILD.gn
@@ -0,0 +1,10 @@
+import("//clang/resource_dir.gni")
+
+copy("blacklist") {
+  sources = [
+    "cfi_blacklist.txt",
+  ]
+  outputs = [
+    "$clang_resource_dir/share/{{source_target_relative}}",
+  ]
+}
diff --git a/utils/gn/secondary/compiler-rt/lib/hwasan/BUILD.gn b/utils/gn/secondary/compiler-rt/lib/hwasan/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..fe5870ad7824e789234e5ec11244a9b71b393f6a
--- /dev/null
+++ b/utils/gn/secondary/compiler-rt/lib/hwasan/BUILD.gn
@@ -0,0 +1,118 @@
+import("//compiler-rt/target.gni")
+
+action("version_script") {
+  script = "//compiler-rt/lib/sanitizer_common/scripts/gen_dynamic_list.py"
+  sources = [
+    "hwasan.syms.extra",
+  ]
+  deps = [
+    ":hwasan",
+    ":hwasan_cxx",
+  ]
+  outputs = [
+    "$target_gen_dir/hwasan.vers",
+  ]
+  args = [
+    "--version-list",
+    "--extra",
+    rebase_path(sources[0], root_build_dir),
+    rebase_path("$crt_current_out_dir/libclang_rt.hwasan-$crt_current_target.a",
+                root_build_dir),
+    rebase_path(
+        "$crt_current_out_dir/libclang_rt.hwasan_cxx-$crt_current_target.a",
+        root_build_dir),
+    "-o",
+    rebase_path(outputs[0], root_build_dir),
+  ]
+}
+
+source_set("sources") {
+  configs -= [ "//llvm/utils/gn/build:llvm_code" ]
+  configs += [ "//llvm/utils/gn/build:crt_code" ]
+  defines = [ "HWASAN_WITH_INTERCEPTORS=1" ]
+  deps = [
+    "//compiler-rt/lib/interception:sources",
+    "//compiler-rt/lib/sanitizer_common:sources",
+    "//compiler-rt/lib/ubsan:sources",
+  ]
+  sources = [
+    "hwasan.cc",
+    "hwasan.h",
+    "hwasan_allocator.cc",
+    "hwasan_allocator.h",
+    "hwasan_dynamic_shadow.cc",
+    "hwasan_dynamic_shadow.h",
+    "hwasan_flags.h",
+    "hwasan_interceptors.cc",
+    "hwasan_interface_internal.h",
+    "hwasan_linux.cc",
+    "hwasan_mapping.h",
+    "hwasan_memintrinsics.cc",
+    "hwasan_poisoning.cc",
+    "hwasan_poisoning.h",
+    "hwasan_report.cc",
+    "hwasan_report.h",
+    "hwasan_thread.cc",
+    "hwasan_thread.h",
+    "hwasan_thread_list.cc",
+    "hwasan_thread_list.h",
+  ]
+}
+
+source_set("cxx_sources") {
+  configs -= [ "//llvm/utils/gn/build:llvm_code" ]
+  configs += [ "//llvm/utils/gn/build:crt_code" ]
+  deps = [
+    "//compiler-rt/lib/ubsan:cxx_sources",
+  ]
+  sources = [
+    "hwasan_new_delete.cc",
+  ]
+}
+
+static_library("hwasan") {
+  output_dir = crt_current_out_dir
+  output_name = "clang_rt.hwasan-$crt_current_target"
+  complete_static_lib = true
+  configs -= [
+    "//llvm/utils/gn/build:llvm_code",
+    "//llvm/utils/gn/build:thin_archive",
+  ]
+  configs += [ "//llvm/utils/gn/build:crt_code" ]
+  deps = [
+    ":sources",
+  ]
+}
+
+static_library("hwasan_cxx") {
+  output_dir = crt_current_out_dir
+  output_name = "clang_rt.hwasan_cxx-$crt_current_target"
+  complete_static_lib = true
+  configs -= [
+    "//llvm/utils/gn/build:llvm_code",
+    "//llvm/utils/gn/build:thin_archive",
+  ]
+  configs += [ "//llvm/utils/gn/build:crt_code" ]
+  deps = [
+    ":cxx_sources",
+  ]
+}
+
+shared_library("hwasan_shared") {
+  output_dir = crt_current_out_dir
+  output_name = "clang_rt.hwasan-$crt_current_target"
+  configs -= [ "//llvm/utils/gn/build:llvm_code" ]
+  configs += [ "//llvm/utils/gn/build:crt_code" ]
+  deps = [
+    ":cxx_sources",
+    ":sources",
+    ":version_script",
+  ]
+  inputs = [
+    "$target_gen_dir/hwasan.vers",
+  ]
+  ldflags = [
+    "-Wl,--version-script," + rebase_path(inputs[0], root_build_dir),
+    "-Wl,-z,global",
+  ]
+}
diff --git a/utils/gn/secondary/compiler-rt/lib/interception/BUILD.gn b/utils/gn/secondary/compiler-rt/lib/interception/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..77a6c284031455cb48403708b78ddd759fe362bc
--- /dev/null
+++ b/utils/gn/secondary/compiler-rt/lib/interception/BUILD.gn
@@ -0,0 +1,17 @@
+source_set("sources") {
+  configs -= [ "//llvm/utils/gn/build:llvm_code" ]
+  configs += [ "//llvm/utils/gn/build:crt_code" ]
+  deps = [
+    "//compiler-rt/lib/sanitizer_common:sources",
+  ]
+  sources = [
+    "interception.h",
+    "interception_linux.cc",
+    "interception_linux.h",
+    "interception_mac.cc",
+    "interception_mac.h",
+    "interception_type_test.cc",
+    "interception_win.cc",
+    "interception_win.h",
+  ]
+}
diff --git a/utils/gn/secondary/compiler-rt/lib/sanitizer_common/BUILD.gn b/utils/gn/secondary/compiler-rt/lib/sanitizer_common/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..a712adad7e1d007aed2e29474a04f1cba914182b
--- /dev/null
+++ b/utils/gn/secondary/compiler-rt/lib/sanitizer_common/BUILD.gn
@@ -0,0 +1,162 @@
+source_set("sources") {
+  configs -= [ "//llvm/utils/gn/build:llvm_code" ]
+  configs += [ "//llvm/utils/gn/build:crt_code" ]
+  deps = [
+    "//llvm/utils/gn/build/libs/pthread",
+  ]
+  libs = []
+  if (current_os == "linux" || current_os == "android") {
+    libs += [ "dl" ]
+  }
+  if (current_os == "linux") {
+    libs += [ "rt" ]
+  }
+  sources = [
+    "sancov_flags.cc",
+    "sancov_flags.h",
+    "sanitizer_addrhashmap.h",
+    "sanitizer_allocator.cc",
+    "sanitizer_allocator.h",
+    "sanitizer_allocator_bytemap.h",
+    "sanitizer_allocator_checks.cc",
+    "sanitizer_allocator_checks.h",
+    "sanitizer_allocator_combined.h",
+    "sanitizer_allocator_interface.h",
+    "sanitizer_allocator_internal.h",
+    "sanitizer_allocator_local_cache.h",
+    "sanitizer_allocator_primary32.h",
+    "sanitizer_allocator_primary64.h",
+    "sanitizer_allocator_report.cc",
+    "sanitizer_allocator_report.h",
+    "sanitizer_allocator_secondary.h",
+    "sanitizer_allocator_size_class_map.h",
+    "sanitizer_allocator_stats.h",
+    "sanitizer_asm.h",
+    "sanitizer_atomic.h",
+    "sanitizer_atomic_clang.h",
+    "sanitizer_atomic_clang_mips.h",
+    "sanitizer_atomic_clang_other.h",
+    "sanitizer_atomic_clang_x86.h",
+    "sanitizer_atomic_msvc.h",
+    "sanitizer_bitvector.h",
+    "sanitizer_bvgraph.h",
+    "sanitizer_common.cc",
+    "sanitizer_common.h",
+    "sanitizer_common_libcdep.cc",
+    "sanitizer_coverage_fuchsia.cc",
+    "sanitizer_coverage_libcdep_new.cc",
+    "sanitizer_coverage_win_sections.cc",
+    "sanitizer_dbghelp.h",
+    "sanitizer_deadlock_detector.h",
+    "sanitizer_deadlock_detector1.cc",
+    "sanitizer_deadlock_detector2.cc",
+    "sanitizer_deadlock_detector_interface.h",
+    "sanitizer_errno.cc",
+    "sanitizer_errno.h",
+    "sanitizer_errno_codes.h",
+    "sanitizer_file.cc",
+    "sanitizer_file.h",
+    "sanitizer_flag_parser.cc",
+    "sanitizer_flag_parser.h",
+    "sanitizer_flags.cc",
+    "sanitizer_flags.h",
+    "sanitizer_freebsd.h",
+    "sanitizer_fuchsia.cc",
+    "sanitizer_fuchsia.h",
+    "sanitizer_getauxval.h",
+    "sanitizer_interface_internal.h",
+    "sanitizer_internal_defs.h",
+    "sanitizer_lfstack.h",
+    "sanitizer_libc.cc",
+    "sanitizer_libc.h",
+    "sanitizer_libignore.cc",
+    "sanitizer_libignore.h",
+    "sanitizer_linux.cc",
+    "sanitizer_linux.h",
+    "sanitizer_linux_libcdep.cc",
+    "sanitizer_linux_s390.cc",
+    "sanitizer_list.h",
+    "sanitizer_local_address_space_view.h",
+    "sanitizer_mac.cc",
+    "sanitizer_mac.h",
+    "sanitizer_mac_libcdep.cc",
+    "sanitizer_mutex.h",
+    "sanitizer_netbsd.cc",
+    "sanitizer_openbsd.cc",
+    "sanitizer_persistent_allocator.cc",
+    "sanitizer_persistent_allocator.h",
+    "sanitizer_placement_new.h",
+    "sanitizer_platform.h",
+    "sanitizer_platform_interceptors.h",
+    "sanitizer_platform_limits_freebsd.cc",
+    "sanitizer_platform_limits_linux.cc",
+    "sanitizer_platform_limits_netbsd.cc",
+    "sanitizer_platform_limits_netbsd.h",
+    "sanitizer_platform_limits_openbsd.cc",
+    "sanitizer_platform_limits_openbsd.h",
+    "sanitizer_platform_limits_posix.cc",
+    "sanitizer_platform_limits_posix.h",
+    "sanitizer_platform_limits_solaris.cc",
+    "sanitizer_platform_limits_solaris.h",
+    "sanitizer_posix.cc",
+    "sanitizer_posix.h",
+    "sanitizer_posix_libcdep.cc",
+    "sanitizer_printf.cc",
+    "sanitizer_procmaps.h",
+    "sanitizer_procmaps_bsd.cc",
+    "sanitizer_procmaps_common.cc",
+    "sanitizer_procmaps_linux.cc",
+    "sanitizer_procmaps_mac.cc",
+    "sanitizer_procmaps_solaris.cc",
+    "sanitizer_quarantine.h",
+    "sanitizer_report_decorator.h",
+    "sanitizer_ring_buffer.h",
+    "sanitizer_rtems.cc",
+    "sanitizer_rtems.h",
+    "sanitizer_solaris.cc",
+    "sanitizer_stackdepot.cc",
+    "sanitizer_stackdepot.h",
+    "sanitizer_stackdepotbase.h",
+    "sanitizer_stacktrace.cc",
+    "sanitizer_stacktrace.h",
+    "sanitizer_stacktrace_libcdep.cc",
+    "sanitizer_stacktrace_printer.cc",
+    "sanitizer_stacktrace_printer.h",
+    "sanitizer_stacktrace_sparc.cc",
+    "sanitizer_stoptheworld.h",
+    "sanitizer_stoptheworld_linux_libcdep.cc",
+    "sanitizer_stoptheworld_mac.cc",
+    "sanitizer_suppressions.cc",
+    "sanitizer_suppressions.h",
+    "sanitizer_symbolizer.cc",
+    "sanitizer_symbolizer.h",
+    "sanitizer_symbolizer_fuchsia.h",
+    "sanitizer_symbolizer_internal.h",
+    "sanitizer_symbolizer_libbacktrace.cc",
+    "sanitizer_symbolizer_libbacktrace.h",
+    "sanitizer_symbolizer_libcdep.cc",
+    "sanitizer_symbolizer_mac.cc",
+    "sanitizer_symbolizer_mac.h",
+    "sanitizer_symbolizer_markup.cc",
+    "sanitizer_symbolizer_posix_libcdep.cc",
+    "sanitizer_symbolizer_report.cc",
+    "sanitizer_symbolizer_rtems.h",
+    "sanitizer_symbolizer_win.cc",
+    "sanitizer_termination.cc",
+    "sanitizer_thread_registry.cc",
+    "sanitizer_thread_registry.h",
+    "sanitizer_tls_get_addr.cc",
+    "sanitizer_tls_get_addr.h",
+    "sanitizer_unwind_linux_libcdep.cc",
+    "sanitizer_unwind_win.cc",
+    "sanitizer_vector.h",
+    "sanitizer_win.cc",
+    "sanitizer_win.h",
+    "sanitizer_win_defs.h",
+    "sanitizer_win_dll_thunk.h",
+    "sanitizer_win_weak_interception.h",
+  ]
+  if (current_cpu == "x64") {
+    sources += [ "sanitizer_linux_x86_64.S" ]
+  }
+}
diff --git a/utils/gn/secondary/compiler-rt/lib/ubsan/BUILD.gn b/utils/gn/secondary/compiler-rt/lib/ubsan/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..6fa8663523cbc0f43b2ff90c3a8045e28c099c04
--- /dev/null
+++ b/utils/gn/secondary/compiler-rt/lib/ubsan/BUILD.gn
@@ -0,0 +1,40 @@
+source_set("sources") {
+  configs -= [ "//llvm/utils/gn/build:llvm_code" ]
+  configs += [ "//llvm/utils/gn/build:crt_code" ]
+  defines = [ "UBSAN_CAN_USE_CXXABI" ]
+  deps = [
+    "//compiler-rt/lib/interception:sources",
+    "//compiler-rt/lib/sanitizer_common:sources",
+  ]
+  sources = [
+    "ubsan_diag.cc",
+    "ubsan_diag.h",
+    "ubsan_flags.cc",
+    "ubsan_flags.h",
+    "ubsan_handlers.cc",
+    "ubsan_handlers.h",
+    "ubsan_init.cc",
+    "ubsan_init.h",
+    "ubsan_monitor.cc",
+    "ubsan_monitor.h",
+    "ubsan_platform.h",
+    "ubsan_signals_standalone.h",
+    "ubsan_type_hash.h",
+    "ubsan_value.cc",
+    "ubsan_value.h",
+  ]
+}
+
+source_set("cxx_sources") {
+  configs -= [ "//llvm/utils/gn/build:llvm_code" ]
+  configs -= [ "//llvm/utils/gn/build:no_rtti" ]
+  configs += [ "//llvm/utils/gn/build:crt_code" ]
+  defines = [ "UBSAN_CAN_USE_CXXABI" ]
+  sources = [
+    "ubsan_handlers_cxx.cc",
+    "ubsan_handlers_cxx.h",
+    "ubsan_type_hash.cc",
+    "ubsan_type_hash_itanium.cc",
+    "ubsan_type_hash_win.cc",
+  ]
+}
diff --git a/utils/gn/secondary/compiler-rt/target.gni b/utils/gn/secondary/compiler-rt/target.gni
new file mode 100644
index 0000000000000000000000000000000000000000..7ea73b12ea1ebc092409788f2c2050dce0ea5d0a
--- /dev/null
+++ b/utils/gn/secondary/compiler-rt/target.gni
@@ -0,0 +1,20 @@
+import("//clang/resource_dir.gni")
+
+if (current_os == "linux" || current_os == "android") {
+  crt_current_out_dir = "$clang_resource_dir/lib/linux"
+} else {
+  assert(false, "unimplemented current_os " + current_os)
+}
+
+if (current_cpu == "x64") {
+  crt_current_target_arch = "x86_64"
+} else if (current_cpu == "arm64") {
+  crt_current_target_arch = "aarch64"
+} else {
+  assert(false, "unimplemented current_cpu " + current_cpu)
+}
+
+crt_current_target = crt_current_target_arch
+if (current_os == "android") {
+  crt_current_target += "-android"
+}
diff --git a/utils/gn/secondary/compiler-rt/test/BUILD.gn b/utils/gn/secondary/compiler-rt/test/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..a5fa1a3f9fb05babb3c2e4b615a44d15e9aff90e
--- /dev/null
+++ b/utils/gn/secondary/compiler-rt/test/BUILD.gn
@@ -0,0 +1,87 @@
+import("//compiler-rt/target.gni")
+import("//compiler-rt/test/test.gni")
+import("//llvm/triples.gni")
+import("//llvm/utils/gn/build/libs/zlib/enable.gni")
+import("//llvm/utils/gn/build/toolchain/compiler.gni")
+import("//llvm/utils/gn/build/write_cmake_config.gni")
+import("//llvm/version.gni")
+
+write_cmake_config("lit_common_configured") {
+  input = "lit.common.configured.in"
+  output = "$target_gen_dir/lit.common.configured"
+
+  values = [
+    "LIT_SITE_CFG_IN_HEADER=## Autogenerated from $input, do not edit",
+
+    "COMPILER_RT_DEFAULT_TARGET_TRIPLE=$llvm_current_triple",
+    "COMPILER_RT_DEFAULT_TARGET_ARCH=$crt_current_target_arch",
+
+    "COMPILER_RT_TEST_COMPILER_CFLAGS=$target_flags_string",
+    "LLVM_BUILD_MODE=.",
+    "LLVM_MAIN_SRC_DIR=" + rebase_path("//llvm"),
+    "LLVM_BINARY_DIR=" + rebase_path(root_build_dir),
+    "COMPILER_RT_SOURCE_DIR=" + rebase_path("//compiler-rt"),
+
+    # This is only used by tsan to find the path to an instrumented libc++.
+    # Since we don't currently support running the tsan tests, leave it empty
+    # for now. Eventually it should probably be replaced with some mechanism
+    # where the libraries are found in a toolchain dir.
+    "COMPILER_RT_BINARY_DIR=",
+
+    # We don't currently support the multiarch runtime layout.
+    "LLVM_ENABLE_PER_TARGET_RUNTIME_DIR_PYBOOL=False",
+
+    "LLVM_TOOLS_DIR=" + rebase_path("$root_build_dir/bin"),
+    "LLVM_LIBRARY_OUTPUT_INTDIR=" + rebase_path("$root_build_dir/lib"),
+
+    "GOLD_EXECUTABLE=ld",
+    "COMPILER_RT_RESOLVED_TEST_COMPILER=" +
+        rebase_path("$root_build_dir/bin/clang"),
+    "COMPILER_RT_TEST_COMPILER_ID=Clang",
+    "PYTHON_EXECUTABLE=$python_path",
+    "COMPILER_RT_DEBUG_PYBOOL=False",
+    "COMPILER_RT_RESOLVED_LIBRARY_OUTPUT_DIR=" +
+        rebase_path(crt_current_out_dir),
+    "COMPILER_RT_EMULATOR=",
+    "COMPILER_RT_ASAN_SHADOW_SCALE=",
+    "SANITIZER_CAN_USE_CXXABI_PYBOOL=True",
+    "COMPILER_RT_HAS_LLD_PYBOOL=True",
+    "HAVE_RPC_XDR_H=0",
+    "ANDROID_NDK_VERSION=19",
+    "ANDROID_SERIAL_FOR_TESTING=$android_serial_for_testing",
+  ]
+
+  if (host_cpu == "x64") {
+    values += [ "HOST_ARCH=x86_64" ]
+  } else {
+    assert(false, "unimplemented host_cpu " + host_cpu)
+  }
+
+  if (host_os == "mac") {
+    values += [ "HOST_OS=Darwin" ]
+  } else if (host_os == "linux") {
+    values += [ "HOST_OS=Linux" ]
+  } else if (host_os == "win") {
+    values += [ "HOST_OS=Windows" ]
+  } else {
+    assert(false, "unsupported host_os " + host_os)
+  }
+
+  if (current_os != "win" || llvm_enable_dia_sdk) {
+    values += [ "CAN_SYMBOLIZE=1" ]
+  } else {
+    values += [ "CAN_SYMBOLIZE=0" ]
+  }
+
+  if (current_os == "android") {
+    values += [ "ANDROID_PYBOOL=True" ]
+  } else {
+    values += [ "ANDROID_PYBOOL=False" ]
+  }
+
+  if (llvm_enable_zlib) {
+    values += [ "HAVE_LIBZ=1" ]
+  } else {
+    values += [ "HAVE_LIBZ=0" ]
+  }
+}
diff --git a/utils/gn/secondary/compiler-rt/test/hwasan/BUILD.gn b/utils/gn/secondary/compiler-rt/test/hwasan/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..ee658381dbd49c2b9a6b14b7f0ca932f50be4d23
--- /dev/null
+++ b/utils/gn/secondary/compiler-rt/test/hwasan/BUILD.gn
@@ -0,0 +1,99 @@
+import("//compiler-rt/target.gni")
+import("//compiler-rt/test/test.gni")
+import("//llvm/utils/gn/build/toolchain/compiler.gni")
+import("//llvm/utils/gn/build/write_cmake_config.gni")
+import("//llvm/version.gni")
+
+write_cmake_config("lit_site_cfg") {
+  input = "lit.site.cfg.in"
+  output = "$target_gen_dir/lit.site.cfg"
+
+  values = [
+    "LIT_SITE_CFG_IN_HEADER=## Autogenerated from $input, do not edit",
+
+    "HWASAN_TEST_CONFIG_SUFFIX=-$crt_current_target",
+    "HWASAN_TEST_TARGET_CFLAGS=$target_flags_string",
+    "HWASAN_TEST_TARGET_ARCH=$crt_current_target_arch",
+
+    "COMPILER_RT_BINARY_DIR=" + rebase_path("$root_gen_dir/compiler-rt"),
+    "HWASAN_LIT_SOURCE_DIR=" + rebase_path("."),
+  ]
+
+  if (current_os == "android") {
+    values += [ "HWASAN_ANDROID_FILES_TO_PUSH=[\"" + rebase_path(
+                    "$crt_current_out_dir/libclang_rt.hwasan-$crt_current_target.so") + "\", \"" + rebase_path(
+                    "$root_out_dir/bin/llvm-symbolizer") + "\"]" ]
+  } else {
+    values += [ "HWASAN_ANDROID_FILES_TO_PUSH=[]" ]
+  }
+}
+
+if (current_toolchain != host_toolchain) {
+  group("hwasan_toolchain") {
+    deps = [
+      ":lit_site_cfg",
+      "//compiler-rt/include($host_toolchain)",
+      "//compiler-rt/lib/cfi:blacklist($host_toolchain)",
+      "//compiler-rt/lib/hwasan:hwasan_shared",
+      "//compiler-rt/test:lit_common_configured",
+      "//llvm/utils/FileCheck($host_toolchain)",
+      "//llvm/utils/llvm-lit($host_toolchain)",
+      "//llvm/utils/not($host_toolchain)",
+    ]
+
+    # FIXME: Make the host use the stage2 llvm-symbolizer as well, for
+    # consistency. Currently lit.common.cfg sets up the sanitizer runtime to
+    # look for llvm-symbolizer in llvm_tools_dir, and also looks there for
+    # other tools which are built with the host toolchain.
+    if (current_os == host_os && current_cpu == host_cpu) {
+      deps += [ "//llvm/tools/llvm-symbolizer($host_toolchain)" ]
+    } else {
+      deps += [ "//llvm/tools/llvm-symbolizer" ]
+    }
+  }
+}
+
+supported_toolchains = []
+if (host_os == "linux" && host_cpu == "x64") {
+  supported_toolchains += [ "//llvm/utils/gn/build/toolchain:stage2_unix" ]
+}
+if (android_ndk_path != "") {
+  supported_toolchains +=
+      [ "//llvm/utils/gn/build/toolchain:stage2_android_aarch64" ]
+}
+
+group("hwasan") {
+  deps = []
+  foreach(toolchain, supported_toolchains) {
+    deps += [ ":hwasan_toolchain($toolchain)" ]
+  }
+}
+
+if (supported_toolchains != []) {
+  action("check-hwasan") {
+    script = "$root_build_dir/bin/llvm-lit"
+    if (host_os == "win") {
+      script += ".py"
+    }
+    args = [ "-sv" ]
+    foreach(toolchain, supported_toolchains) {
+      args += [ rebase_path(
+              get_label_info(":lit_site_cfg($toolchain)", "target_gen_dir"),
+              root_build_dir) ]
+    }
+    outputs = [
+      "$target_gen_dir/run-lit",  # Non-existing, so that ninja runs it each time.
+    ]
+
+    # Since check-hwasan is always dirty, //:default doesn't depend on it so that
+    # it's not part of the default ninja target.  Hence, check-hwasan shouldn't
+    # have any deps except :hwasan. so that the default target is sure to build
+    # all the deps.
+    deps = [
+      ":hwasan",
+    ]
+    testonly = true
+
+    pool = "//:console"
+  }
+}
diff --git a/utils/gn/secondary/compiler-rt/test/test.gni b/utils/gn/secondary/compiler-rt/test/test.gni
new file mode 100644
index 0000000000000000000000000000000000000000..e2335933659fd060621480b366bf20af22a8058d
--- /dev/null
+++ b/utils/gn/secondary/compiler-rt/test/test.gni
@@ -0,0 +1,15 @@
+import("//llvm/utils/gn/build/toolchain/target_flags.gni")
+
+declare_args() {
+  # Specifies the serial number of the Android device to be used for testing.
+  android_serial_for_testing = ""
+}
+
+target_flags_string = ""
+
+foreach(flag, target_flags + target_ldflags + [ "-fuse-ld=lld" ]) {
+  if (target_flags_string != "") {
+    target_flags_string += " "
+  }
+  target_flags_string += flag
+}
diff --git a/utils/gn/secondary/lld/ELF/BUILD.gn b/utils/gn/secondary/lld/ELF/BUILD.gn
index d5f8a8999956d531bc75c48b45164a77be7ee949..8006ef17c2f501310f760d15a0ea3e5216cd381e 100644
--- a/utils/gn/secondary/lld/ELF/BUILD.gn
+++ b/utils/gn/secondary/lld/ELF/BUILD.gn
@@ -29,6 +29,7 @@ static_library("ELF") {
     "Arch/ARM.cpp",
     "Arch/AVR.cpp",
     "Arch/Hexagon.cpp",
+    "Arch/MSP430.cpp",
     "Arch/Mips.cpp",
     "Arch/MipsArchTree.cpp",
     "Arch/PPC.cpp",
diff --git a/utils/gn/secondary/lld/include/lld/Common/BUILD.gn b/utils/gn/secondary/lld/include/lld/Common/BUILD.gn
index b45c04ea67ad44f6c74618aeb8854962af986f0e..82a0923e930f015b923c67c4e886a81ce31e2eb7 100644
--- a/utils/gn/secondary/lld/include/lld/Common/BUILD.gn
+++ b/utils/gn/secondary/lld/include/lld/Common/BUILD.gn
@@ -1,20 +1,10 @@
+import("//llvm/utils/gn/build/write_cmake_config.gni")
 import("//llvm/version.gni")
 
-action("version") {
-  script = "//llvm/utils/gn/build/write_cmake_config.py"
-
-  sources = [
-    "Version.inc.in",
-  ]
-  outputs = [
-    "$target_gen_dir/Version.inc",
-  ]
-  args = [
-    "-o",
-    rebase_path(outputs[0], root_out_dir),
-
-    rebase_path(sources[0], root_out_dir),
-
+write_cmake_config("version") {
+  input = "Version.inc.in"
+  output = "$target_gen_dir/Version.inc"
+  values = [
     "LLD_VERSION=$llvm_version",
     "LLD_VERSION_MAJOR=$llvm_version_major",
     "LLD_VERSION_MINOR=$llvm_version_minor",
diff --git a/utils/gn/secondary/lld/test/BUILD.gn b/utils/gn/secondary/lld/test/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..b62fbc1555ef3f6b0c9ae2c29bb89920c13f8e4a
--- /dev/null
+++ b/utils/gn/secondary/lld/test/BUILD.gn
@@ -0,0 +1,126 @@
+import("//llvm/lib/DebugInfo/PDB/enable_dia.gni")
+import("//llvm/triples.gni")
+import("//llvm/utils/gn/build/libs/xml/enable.gni")
+import("//llvm/utils/gn/build/libs/zlib/enable.gni")
+import("//llvm/utils/gn/build/write_cmake_config.gni")
+import("lld_lit_site_cfg_files.gni")
+
+# The bits common to writing lit.site.cfg.py.in and Unit/lit.site.cfg.py.in.
+template("write_lit_cfg") {
+  write_cmake_config(target_name) {
+    input = invoker.input
+    output = invoker.output
+    values = [
+      "LIT_SITE_CFG_IN_HEADER=## Autogenerated from $input, do not edit",
+      "LLD_BINARY_DIR=" +
+          rebase_path(get_label_info("//lld", "target_out_dir")),
+      "LLD_SOURCE_DIR=" + rebase_path("//lld"),
+      "LLVM_BINARY_DIR=" +
+          rebase_path(get_label_info("//llvm", "target_out_dir")),
+      "LLVM_LIBRARY_OUTPUT_INTDIR=",  # FIXME: for shared builds only (?)
+      "LLVM_LIBS_DIR=",  # needed only for shared builds
+      "LLVM_LIT_TOOLS_DIR=",  # Intentionally empty, matches cmake build.
+      "LLVM_RUNTIME_OUTPUT_INTDIR=" + rebase_path("$root_out_dir/bin"),
+      "LLVM_SOURCE_DIR=" + rebase_path("//llvm"),
+      "LLVM_TOOLS_DIR=" + rebase_path("$root_out_dir/bin"),
+      "PYTHON_EXECUTABLE=$python_path",
+      "TARGET_TRIPLE=$llvm_target_triple",
+    ]
+    values += invoker.extra_values
+  }
+}
+
+write_lit_cfg("lit_site_cfg") {
+  # Fully-qualified instead of relative for LIT_SITE_CFG_IN_HEADER.
+  input = "//lld/test/lit.site.cfg.py.in"
+  output = lld_lit_site_cfg_file
+
+  extra_values = []
+  if (llvm_enable_dia_sdk) {
+    extra_values += [ "LLVM_ENABLE_DIA_SDK=1" ]
+  } else {
+    extra_values += [ "LLVM_ENABLE_DIA_SDK=0" ]  # Must be 0.
+  }
+
+  if (llvm_enable_libxml2) {
+    extra_values += [ "LLVM_LIBXML2_ENABLED=1" ]
+  } else {
+    extra_values += [ "LLVM_LIBXML2_ENABLED=0" ]  # Must be 0.
+  }
+
+  if (llvm_enable_zlib) {
+    extra_values += [ "HAVE_LIBZ=1" ]
+  } else {
+    extra_values += [ "HAVE_LIBZ=0" ]  # Must be 0.
+  }
+}
+
+write_lit_cfg("lit_unit_site_cfg") {
+  # Fully-qualified instead of relative for LIT_SITE_CFG_IN_HEADER.
+  input = "//lld/test/Unit/lit.site.cfg.py.in"
+  output = lld_lit_unit_site_cfg_file
+  extra_values = [ "LLVM_BUILD_MODE=." ]
+}
+
+# This target should contain all dependencies of check-lld.
+# //:default depends on it, so that ninja's default target builds all
+# prerequisites for check-lld but doesn't run check-lld itself.
+group("test") {
+  deps = [
+    ":lit_site_cfg",
+    ":lit_unit_site_cfg",
+    "//lld/tools/lld:symlinks",
+    "//lld/unittests",
+    "//llvm/tools/llc",
+    "//llvm/tools/llvm-ar:symlinks",
+    "//llvm/tools/llvm-as",
+    "//llvm/tools/llvm-bcanalyzer",
+    "//llvm/tools/llvm-dis",
+    "//llvm/tools/llvm-dwarfdump",
+    "//llvm/tools/llvm-mc",
+    "//llvm/tools/llvm-nm:symlinks",
+    "//llvm/tools/llvm-objcopy:symlinks",
+    "//llvm/tools/llvm-objdump:symlinks",
+    "//llvm/tools/llvm-pdbutil",
+    "//llvm/tools/llvm-readobj:symlinks",
+    "//llvm/tools/obj2yaml",
+    "//llvm/tools/opt",
+    "//llvm/tools/yaml2obj",
+    "//llvm/utils/FileCheck",
+    "//llvm/utils/count",
+    "//llvm/utils/llvm-lit",
+    "//llvm/utils/not",
+  ]
+  testonly = true
+}
+
+# This is the action that runs all of lld's tests, check-lld.
+action("check-lld") {
+  script = "$root_out_dir/bin/llvm-lit"
+  if (host_os == "win") {
+    script += ".py"
+  }
+  args = [
+    "-sv",
+    "--param",
+    "lld_site_config=" + rebase_path(lld_lit_site_cfg_file, root_out_dir),
+    "--param",
+    "lld_unit_site_config=" +
+        rebase_path(lld_lit_unit_site_cfg_file, root_out_dir),
+    rebase_path(".", root_out_dir),
+  ]
+  outputs = [
+    "$target_gen_dir/run-lit",  # Non-existing, so that ninja runs it each time.
+  ]
+
+  # Since check-lld is always dirty, //:default doesn't depend on it so that
+  # it's not part of the default ninja target.  Hence, check-lld shouldn't
+  # have any deps except :test, so that the default target is sure to build
+  # all the deps.
+  deps = [
+    ":test",
+  ]
+  testonly = true
+
+  pool = "//:console"
+}
diff --git a/utils/gn/secondary/lld/test/lld_lit_site_cfg_files.gni b/utils/gn/secondary/lld/test/lld_lit_site_cfg_files.gni
new file mode 100644
index 0000000000000000000000000000000000000000..87c00b634756819296616babc53246a30d53f110
--- /dev/null
+++ b/utils/gn/secondary/lld/test/lld_lit_site_cfg_files.gni
@@ -0,0 +1,2 @@
+lld_lit_site_cfg_file = "$root_gen_dir/lld/test/lit.site.cfg.py"
+lld_lit_unit_site_cfg_file = "$root_gen_dir/lld/test/Unit/lit.site.cfg.py"
diff --git a/utils/gn/secondary/lld/unittests/BUILD.gn b/utils/gn/secondary/lld/unittests/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..275637c1ef89677ef7bcdc1b629c43fc5dbbec7f
--- /dev/null
+++ b/utils/gn/secondary/lld/unittests/BUILD.gn
@@ -0,0 +1,7 @@
+group("unittests") {
+  deps = [
+    "DriverTests",
+    "MachOTests",
+  ]
+  testonly = true
+}
diff --git a/utils/gn/secondary/lld/unittests/DriverTests/BUILD.gn b/utils/gn/secondary/lld/unittests/DriverTests/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..b683c3507a2d1d190b297c9d7663e68ff48de273
--- /dev/null
+++ b/utils/gn/secondary/lld/unittests/DriverTests/BUILD.gn
@@ -0,0 +1,12 @@
+import("//llvm/utils/unittest/unittest.gni")
+
+unittest("DriverTests") {
+  configs += [ "//llvm/utils/gn/build:lld_code" ]
+  deps = [
+    "//lld/lib/Driver",
+    "//lld/lib/ReaderWriter/MachO",
+  ]
+  sources = [
+    "DarwinLdDriverTest.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/lld/unittests/MachOTests/BUILD.gn b/utils/gn/secondary/lld/unittests/MachOTests/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..90d0a618c309c0f159eed136f40372af9c7ced4e
--- /dev/null
+++ b/utils/gn/secondary/lld/unittests/MachOTests/BUILD.gn
@@ -0,0 +1,16 @@
+import("//llvm/utils/unittest/unittest.gni")
+
+unittest("MachOTests") {
+  configs += [ "//llvm/utils/gn/build:lld_code" ]
+  deps = [
+    "//lld/lib/Driver",
+    "//lld/lib/ReaderWriter/MachO",
+    "//lld/lib/ReaderWriter/YAML",
+  ]
+  sources = [
+    "MachONormalizedFileBinaryReaderTests.cpp",
+    "MachONormalizedFileBinaryWriterTests.cpp",
+    "MachONormalizedFileToAtomsTests.cpp",
+    "MachONormalizedFileYAMLTests.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/llvm/include/llvm/Config/BUILD.gn b/utils/gn/secondary/llvm/include/llvm/Config/BUILD.gn
index 08924e36dab9126a92dbfab864f4acaf82e50d30..4d0a3a8144ce32e70c3af1fed0d09b000c724043 100644
--- a/utils/gn/secondary/llvm/include/llvm/Config/BUILD.gn
+++ b/utils/gn/secondary/llvm/include/llvm/Config/BUILD.gn
@@ -1,11 +1,13 @@
 import("//llvm/lib/Target/targets.gni")
 import("//llvm/triples.gni")
 import("//llvm/utils/gn/build/buildflags.gni")
+import("//llvm/utils/gn/build/libs/edit/enable.gni")
 import("//llvm/utils/gn/build/libs/pthread/enable.gni")
 import("//llvm/utils/gn/build/libs/terminfo/enable.gni")
 import("//llvm/utils/gn/build/libs/xar/enable.gni")
 import("//llvm/utils/gn/build/libs/xml/enable.gni")
 import("//llvm/utils/gn/build/libs/zlib/enable.gni")
+import("//llvm/utils/gn/build/write_cmake_config.gni")
 import("//llvm/version.gni")
 
 # Contains actions to create config.h, llvm-config.h, abi-breaking.h,
@@ -42,53 +44,32 @@ declare_args() {
   llvm_enable_reverse_iteration = false
 }
 
-action("abi-breaking") {
-  script = "//llvm/utils/gn/build/write_cmake_config.py"
-
-  sources = [
-    "abi-breaking.h.cmake",
-  ]
-  outputs = [
-    "$target_gen_dir/abi-breaking.h",
-  ]
-  args = [
-    "-o",
-    rebase_path(outputs[0], root_out_dir),
-
-    rebase_path(sources[0], root_out_dir),
-  ]
+write_cmake_config("abi-breaking") {
+  input = "abi-breaking.h.cmake"
+  output = "$target_gen_dir/abi-breaking.h"
+  values = []
 
   if (llvm_enable_abi_breaking_checks) {
-    args += [ "LLVM_ENABLE_ABI_BREAKING_CHECKS=1" ]
+    values += [ "LLVM_ENABLE_ABI_BREAKING_CHECKS=1" ]
   } else {
-    args += [ "LLVM_ENABLE_ABI_BREAKING_CHECKS=" ]
+    values += [ "LLVM_ENABLE_ABI_BREAKING_CHECKS=" ]
   }
 
   if (llvm_enable_reverse_iteration) {
-    args += [ "LLVM_ENABLE_REVERSE_ITERATION=1" ]
+    values += [ "LLVM_ENABLE_REVERSE_ITERATION=1" ]
   } else {
-    args += [ "LLVM_ENABLE_REVERSE_ITERATION=" ]
+    values += [ "LLVM_ENABLE_REVERSE_ITERATION=" ]
   }
 }
 
-action("config") {
-  script = "//llvm/utils/gn/build/write_cmake_config.py"
-
+write_cmake_config("config") {
   public_deps = [
     ":llvm-config",
   ]
 
-  sources = [
-    "config.h.cmake",
-  ]
-  outputs = [
-    "$target_gen_dir/config.h",
-  ]
-  args = [
-    "-o",
-    rebase_path(outputs[0], root_out_dir),
-    rebase_path(sources[0], root_out_dir),
-
+  input = "config.h.cmake"
+  output = "$target_gen_dir/config.h"
+  values = [
     "BUG_REPORT_URL=https://bugs.llvm.org/",
     "ENABLE_BACKTRACES=1",
     "ENABLE_CRASH_OVERRIDES=1",
@@ -108,9 +89,11 @@ action("config") {
     "HAVE_LIBPSAPI=",
     "HAVE_MALLCTL=",
     "HAVE_SIGNAL_H=1",
+    "HAVE_STD_IS_TRIVIALLY_COPYABLE=1",
     "HAVE_STRERROR=1",
     "HAVE_SYS_STAT_H=1",
     "HAVE_SYS_TYPES_H=1",
+    "HAVE_VALGRIND_VALGRIND_H=",
     "HAVE__ALLOCA=",
     "HAVE___ALLOCA=",
     "HAVE___ASHLDI3=",
@@ -147,8 +130,8 @@ action("config") {
     "LLVM_DEFAULT_TARGET_TRIPLE=$llvm_target_triple",
   ]
 
-  if (host_os == "linux") {
-    args += [
+  if (current_os == "linux" || current_os == "android") {
+    values += [
       "HAVE_FUTIMENS=1",
       "HAVE_LINK_H=1",
       "HAVE_LSEEK64=1",
@@ -157,10 +140,9 @@ action("config") {
       "HAVE_SCHED_GETAFFINITY=1",
       "HAVE_CPU_COUNT=1",
       "HAVE_STRUCT_STAT_ST_MTIM_TV_NSEC=1",
-      "HAVE_VALGRIND_VALGRIND_H=1",
     ]
   } else {
-    args += [
+    values += [
       "HAVE_FUTIMENS=",
       "HAVE_LINK_H=",
       "HAVE_LSEEK64=",
@@ -169,39 +151,47 @@ action("config") {
       "HAVE_SCHED_GETAFFINITY=",
       "HAVE_CPU_COUNT=",
       "HAVE_STRUCT_STAT_ST_MTIM_TV_NSEC=",
-      "HAVE_VALGRIND_VALGRIND_H=",
     ]
   }
 
-  if (host_os == "mac") {
-    args += [
+  if (current_os == "mac") {
+    values += [
       "HAVE_CRASHREPORTER_INFO=1",
       "HAVE_DECL_ARC4RANDOM=1",
       "HAVE_DLADDR=1",
-      "HAVE_LIBEDIT=1",
-      "HAVE_MALLOC_H=",
       "HAVE_MACH_MACH_H=1",
       "HAVE_MALLOC_MALLOC_H=1",
       "HAVE_MALLOC_ZONE_STATISTICS=1",
       "HAVE_STRUCT_STAT_ST_MTIMESPEC_TV_NSEC=1",
     ]
   } else {
-    args += [
+    values += [
       "HAVE_CRASHREPORTER_INFO=",
       "HAVE_DECL_ARC4RANDOM=",
       "HAVE_DLADDR=",
-      "HAVE_LIBEDIT=",
       "HAVE_MACH_MACH_H=",
-      "HAVE_MALLOC_H=1",
       "HAVE_MALLOC_MALLOC_H=",
       "HAVE_MALLOC_ZONE_STATISTICS=",
       "HAVE_STRUCT_STAT_ST_MTIMESPEC_TV_NSEC=",
     ]
   }
 
-  if (host_os == "win") {
-    args += [
+  if (current_os == "linux" || current_os == "mac") {
+    values += [
+      "HAVE_BACKTRACE=1",
+      "HAVE_POSIX_SPAWN=1",
+      "HAVE_PTHREAD_GETNAME_NP=1",
+    ]
+  } else {
+    values += [
       "HAVE_BACKTRACE=",
+      "HAVE_POSIX_SPAWN=",
+      "HAVE_PTHREAD_GETNAME_NP=",
+    ]
+  }
+
+  if (current_os == "win") {
+    values += [
       "HAVE_DECL_STRERROR_S=1",
       "HAVE_DLFCN_H=",
       "HAVE_DLOPEN=",
@@ -211,10 +201,8 @@ action("config") {
       "HAVE_GETRUSAGE=",
       "HAVE_ISATTY=",
       "HAVE_LIBPTHREAD=",
-      "HAVE_PTHREAD_GETNAME_NP=",
       "HAVE_PTHREAD_SETNAME_NP=",
       "HAVE_LIBZ=",
-      "HAVE_POSIX_SPAWN=",
       "HAVE_PREAD=",
       "HAVE_PTHREAD_GETSPECIFIC=",
       "HAVE_PTHREAD_H=",
@@ -242,8 +230,7 @@ action("config") {
     ]
   } else {
     # POSIX-y system defaults.
-    args += [
-      "HAVE_BACKTRACE=1",
+    values += [
       "HAVE_DECL_STRERROR_S=",
       "HAVE_DLFCN_H=1",
       "HAVE_DLOPEN=1",
@@ -253,10 +240,8 @@ action("config") {
       "HAVE_GETRUSAGE=1",
       "HAVE_ISATTY=1",
       "HAVE_LIBPTHREAD=1",
-      "HAVE_PTHREAD_GETNAME_NP=1",
       "HAVE_PTHREAD_SETNAME_NP=1",
       "HAVE_LIBZ=1",
-      "HAVE_POSIX_SPAWN=1",
       "HAVE_PREAD=1",
       "HAVE_PTHREAD_GETSPECIFIC=1",
       "HAVE_PTHREAD_H=1",
@@ -284,58 +269,54 @@ action("config") {
     ]
   }
 
-  if (host_os == "linux") {
-    args += [ "LTDL_SHLIB_EXT=.so" ]
-  } else if (host_os == "mac") {
-    args += [ "LTDL_SHLIB_EXT=.dylib" ]
-  } else if (host_os == "win") {
-    args += [ "LTDL_SHLIB_EXT=.dll" ]
+  if (current_os == "mac") {
+    values += [ "LTDL_SHLIB_EXT=.dylib" ]
+  } else if (current_os == "win") {
+    values += [ "LTDL_SHLIB_EXT=.dll" ]
+  } else {
+    values += [ "LTDL_SHLIB_EXT=.so" ]
+  }
+
+  if (llvm_enable_libedit) {
+    values += [ "HAVE_LIBEDIT=1" ]
+  } else {
+    values += [ "HAVE_LIBEDIT=" ]
   }
 
   if (llvm_enable_libxar) {
-    args += [ "HAVE_LIBXAR=1" ]
+    values += [ "HAVE_LIBXAR=1" ]
   } else {
-    args += [ "HAVE_LIBXAR=" ]
+    values += [ "HAVE_LIBXAR=" ]
   }
 
   if (llvm_enable_terminfo) {
-    args += [ "HAVE_TERMINFO=1" ]
+    values += [ "HAVE_TERMINFO=1" ]
   } else {
-    args += [ "HAVE_TERMINFO=" ]
+    values += [ "HAVE_TERMINFO=" ]
   }
 
   if (llvm_enable_zlib) {
-    args += [ "LLVM_ENABLE_ZLIB=1" ]
+    values += [ "LLVM_ENABLE_ZLIB=1" ]
   } else {
-    args += [ "LLVM_ENABLE_ZLIB=" ]
+    values += [ "LLVM_ENABLE_ZLIB=" ]
   }
 
   if (llvm_enable_libxml2) {
-    args += [ "LLVM_LIBXML2_ENABLED=1" ]
+    values += [ "LLVM_LIBXML2_ENABLED=1" ]
   } else {
-    args += [ "LLVM_LIBXML2_ENABLED=" ]
+    values += [ "LLVM_LIBXML2_ENABLED=" ]
   }
 }
 
-action("llvm-config") {
-  script = "//llvm/utils/gn/build/write_cmake_config.py"
-
-  sources = [
-    "llvm-config.h.cmake",
-  ]
-  outputs = [
-    "$target_gen_dir/llvm-config.h",
-  ]
-  args = [
-    "-o",
-    rebase_path(outputs[0], root_out_dir),
-    rebase_path(sources[0], root_out_dir),
-
+write_cmake_config("llvm-config") {
+  input = "llvm-config.h.cmake"
+  output = "$target_gen_dir/llvm-config.h"
+  values = [
     "LLVM_ENABLE_DUMP=",
     "LINK_POLLY_INTO_TOOLS=",
     "LLVM_DEFAULT_TARGET_TRIPLE=$llvm_target_triple",
     "LLVM_HAS_ATOMICS=1",
-    "LLVM_HOST_TRIPLE=$llvm_host_triple",
+    "LLVM_HOST_TRIPLE=$llvm_current_triple",
     "LLVM_NATIVE_ARCH=$native_target",
     "LLVM_NATIVE_ASMPARSER=1",
     "LLVM_NATIVE_ASMPRINTER=1",
@@ -353,16 +334,16 @@ action("llvm-config") {
     "LLVM_FORCE_ENABLE_STATS=",
   ]
 
-  if (host_os == "win") {
-    args += [ "LLVM_ON_UNIX=" ]
+  if (current_os == "win") {
+    values += [ "LLVM_ON_UNIX=" ]
   } else {
-    args += [ "LLVM_ON_UNIX=1" ]
+    values += [ "LLVM_ON_UNIX=1" ]
   }
 
   if (llvm_enable_threads) {
-    args += [ "LLVM_ENABLE_THREADS=1" ]
+    values += [ "LLVM_ENABLE_THREADS=1" ]
   } else {
-    args += [ "LLVM_ENABLE_THREADS=" ]
+    values += [ "LLVM_ENABLE_THREADS=" ]
   }
 }
 
@@ -370,19 +351,13 @@ action("llvm-config") {
 # .def files used by llvm/lib/Target
 
 template("write_target_def_file") {
-  assert(defined(invoker.key), "callers must set key")
-  assert(defined(invoker.value), "callers must set value")
+  assert(defined(invoker.key), "must set 'key' in $target_name")
+  assert(defined(invoker.value), "must set 'value' in $target_name")
 
-  action(target_name) {
+  write_cmake_config(target_name) {
     visibility = [ ":write_target_def_files" ]
-    script = "//llvm/utils/gn/build/write_cmake_config.py"
-
-    sources = [
-      "$target_name.in",
-    ]
-    outputs = [
-      "$target_gen_dir/$target_name",
-    ]
+    input = "$target_name.in"
+    output = "$target_gen_dir/$target_name"
 
     # Build something like
     # `LLVM_ENUM_ASM_PARSERS=LLVM_ASM_PARSER(ARM)\nLLVM_ASM_PARSER(X86)\n`. Note
@@ -392,12 +367,7 @@ template("write_target_def_file") {
     foreach(target, llvm_targets_to_build) {
       value = "$value${invoker.value}($target)\n"
     }
-    args = [
-      "-o",
-      rebase_path(outputs[0], root_out_dir),
-      rebase_path(sources[0], root_out_dir),
-      "${invoker.key}=$value",
-    ]
+    values = [ "${invoker.key}=$value" ]
   }
 }
 
diff --git a/utils/gn/secondary/llvm/include/llvm/IR/BUILD.gn b/utils/gn/secondary/llvm/include/llvm/IR/BUILD.gn
index cc73446a582d0a11edc8b94c1f07adf59cde7cb1..c9165c38a790c04f3931cff44ac490d633cc6a63 100644
--- a/utils/gn/secondary/llvm/include/llvm/IR/BUILD.gn
+++ b/utils/gn/secondary/llvm/include/llvm/IR/BUILD.gn
@@ -1,7 +1,7 @@
 import("//llvm/utils/TableGen/tablegen.gni")
 
 tablegen("IntrinsicEnums") {
-  visibility = [ "//llvm/lib/IR" ]
+  visibility = [ ":public_tablegen" ]
   args = [ "-gen-intrinsic-enums" ]
   td_file = "Intrinsics.td"
 }
@@ -13,6 +13,20 @@ tablegen("IntrinsicImpl") {
 }
 
 tablegen("Attributes") {
-  visibility = [ "//llvm/lib/IR" ]
+  visibility = [ ":public_tablegen" ]
   args = [ "-gen-attrs" ]
 }
+
+# Groups all tablegen() calls that create .inc files that are included in
+# IR's public headers.  //llvm/lib/Target has this as a public_dep, so targets
+# dependign on //llvm/lib/IR don't need to depend on this.  This exists
+# solely for targets that use IR's public headers but don't link against IR.
+group("public_tablegen") {
+  public_deps = [
+    # IR's public headers include Attributes.inc.
+    ":Attributes",
+
+    # IR's public headers include IntrinsicEnums.inc.
+    ":IntrinsicEnums",
+  ]
+}
diff --git a/utils/gn/secondary/llvm/lib/Analysis/BUILD.gn b/utils/gn/secondary/llvm/lib/Analysis/BUILD.gn
index 5b203d4d7035543c4418c19ee8046f46af414b29..0b439ce450c49f4f961601c66c77bd93a4190801 100644
--- a/utils/gn/secondary/llvm/lib/Analysis/BUILD.gn
+++ b/utils/gn/secondary/llvm/lib/Analysis/BUILD.gn
@@ -41,6 +41,7 @@ static_library("Analysis") {
     "DependenceAnalysis.cpp",
     "DivergenceAnalysis.cpp",
     "DomPrinter.cpp",
+    "DomTreeUpdater.cpp",
     "DominanceFrontier.cpp",
     "EHPersonalities.cpp",
     "GlobalsModRef.cpp",
diff --git a/utils/gn/secondary/llvm/lib/CodeGen/GlobalISel/BUILD.gn b/utils/gn/secondary/llvm/lib/CodeGen/GlobalISel/BUILD.gn
index df8a11035873b46d9da11ffd8c9b154fab765872..7e6ecad0e116cfbd4ee62fbf459a217455d9a2e9 100644
--- a/utils/gn/secondary/llvm/lib/CodeGen/GlobalISel/BUILD.gn
+++ b/utils/gn/secondary/llvm/lib/CodeGen/GlobalISel/BUILD.gn
@@ -12,6 +12,8 @@ static_library("GlobalISel") {
     "//llvm/lib/Transforms/Utils",
   ]
   sources = [
+    "CSEInfo.cpp",
+    "CSEMIRBuilder.cpp",
     "CallLowering.cpp",
     "Combiner.cpp",
     "CombinerHelper.cpp",
diff --git a/utils/gn/secondary/llvm/lib/Demangle/BUILD.gn b/utils/gn/secondary/llvm/lib/Demangle/BUILD.gn
index 8e97dfea30f77ab335527ddef683f2f4a66db614..81ddb6c76c41608ccd9225121d602af0a76d3c67 100644
--- a/utils/gn/secondary/llvm/lib/Demangle/BUILD.gn
+++ b/utils/gn/secondary/llvm/lib/Demangle/BUILD.gn
@@ -2,6 +2,7 @@ static_library("Demangle") {
   output_name = "LLVMDemangle"
 
   sources = [
+    "Demangle.cpp",
     "ItaniumDemangle.cpp",
     "MicrosoftDemangle.cpp",
     "MicrosoftDemangleNodes.cpp",
diff --git a/utils/gn/secondary/llvm/lib/ExecutionEngine/BUILD.gn b/utils/gn/secondary/llvm/lib/ExecutionEngine/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..ac73dd4f7317c4fef3e30ca8c68b480642359717
--- /dev/null
+++ b/utils/gn/secondary/llvm/lib/ExecutionEngine/BUILD.gn
@@ -0,0 +1,21 @@
+static_library("ExecutionEngine") {
+  output_name = "LLVMExecutionEngine"
+  public_deps = [
+    # Must be a public_dep because ExecutionEngine's headers include
+    # llvm-config.h.
+    "//llvm/include/llvm/Config:llvm-config",
+  ]
+  deps = [
+    "//llvm/lib/IR",
+    "//llvm/lib/MC",
+    "//llvm/lib/Object",
+    "//llvm/lib/Target",
+  ]
+  sources = [
+    "ExecutionEngine.cpp",
+    "ExecutionEngineBindings.cpp",
+    "GDBRegistrationListener.cpp",
+    "SectionMemoryManager.cpp",
+    "TargetSelect.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/llvm/lib/ExecutionEngine/Interpreter/BUILD.gn b/utils/gn/secondary/llvm/lib/ExecutionEngine/Interpreter/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..81192ca3fadb5f383f22472d880b0866947f2f3d
--- /dev/null
+++ b/utils/gn/secondary/llvm/lib/ExecutionEngine/Interpreter/BUILD.gn
@@ -0,0 +1,15 @@
+static_library("Interpreter") {
+  output_name = "LLVMInterpreter"
+  deps = [
+    "//llvm/include/llvm/Config:config",
+    "//llvm/lib/CodeGen",
+    "//llvm/lib/ExecutionEngine",
+    "//llvm/lib/IR",
+    "//llvm/lib/Support",
+  ]
+  sources = [
+    "Execution.cpp",
+    "ExternalFunctions.cpp",
+    "Interpreter.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/llvm/lib/ExecutionEngine/MCJIT/BUILD.gn b/utils/gn/secondary/llvm/lib/ExecutionEngine/MCJIT/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..9d686cebdad536d80329c249c6863d52eda83af8
--- /dev/null
+++ b/utils/gn/secondary/llvm/lib/ExecutionEngine/MCJIT/BUILD.gn
@@ -0,0 +1,14 @@
+static_library("MCJIT") {
+  output_name = "LLVMMCJIT"
+  deps = [
+    "//llvm/lib/ExecutionEngine",
+    "//llvm/lib/ExecutionEngine/RuntimeDyld",
+    "//llvm/lib/IR",
+    "//llvm/lib/Object",
+    "//llvm/lib/Support",
+    "//llvm/lib/Target",
+  ]
+  sources = [
+    "MCJIT.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/llvm/lib/ExecutionEngine/Orc/BUILD.gn b/utils/gn/secondary/llvm/lib/ExecutionEngine/Orc/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..19c68f090cf05c6bfacbbcfbba6d110b464f8633
--- /dev/null
+++ b/utils/gn/secondary/llvm/lib/ExecutionEngine/Orc/BUILD.gn
@@ -0,0 +1,33 @@
+static_library("Orc") {
+  output_name = "LLVMOrcJIT"
+  deps = [
+    "//llvm/lib/ExecutionEngine",
+    "//llvm/lib/ExecutionEngine/RuntimeDyld",
+    "//llvm/lib/IR",
+    "//llvm/lib/Object",
+    "//llvm/lib/Support",
+    "//llvm/lib/Transforms/Utils",
+  ]
+  sources = [
+    "CompileOnDemandLayer.cpp",
+    "Core.cpp",
+    "ExecutionUtils.cpp",
+    "IRCompileLayer.cpp",
+    "IRTransformLayer.cpp",
+    "IndirectionUtils.cpp",
+    "JITTargetMachineBuilder.cpp",
+    "LLJIT.cpp",
+    "Layer.cpp",
+    "LazyReexports.cpp",
+    "Legacy.cpp",
+    "NullResolver.cpp",
+    "ObjectTransformLayer.cpp",
+    "OrcABISupport.cpp",
+    "OrcCBindings.cpp",
+    "OrcError.cpp",
+    "OrcMCJITReplacement.cpp",
+    "RPCUtils.cpp",
+    "RTDyldObjectLinkingLayer.cpp",
+    "ThreadSafeModule.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/llvm/lib/ExecutionEngine/RuntimeDyld/BUILD.gn b/utils/gn/secondary/llvm/lib/ExecutionEngine/RuntimeDyld/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..75deb0609e2ffb4c7c553620137d98cae15e63af
--- /dev/null
+++ b/utils/gn/secondary/llvm/lib/ExecutionEngine/RuntimeDyld/BUILD.gn
@@ -0,0 +1,19 @@
+static_library("RuntimeDyld") {
+  output_name = "LLVMRuntimeDyld"
+  deps = [
+    "//llvm/include/llvm/Config:config",
+    "//llvm/lib/MC",
+    "//llvm/lib/Object",
+    "//llvm/lib/Support",
+  ]
+  sources = [
+    "JITSymbol.cpp",
+    "RTDyldMemoryManager.cpp",
+    "RuntimeDyld.cpp",
+    "RuntimeDyldCOFF.cpp",
+    "RuntimeDyldChecker.cpp",
+    "RuntimeDyldELF.cpp",
+    "RuntimeDyldMachO.cpp",
+    "Targets/RuntimeDyldELFMips.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/llvm/lib/FuzzMutate/BUILD.gn b/utils/gn/secondary/llvm/lib/FuzzMutate/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..57a6b5702e868151300eab0c1b84a08207e6ca7e
--- /dev/null
+++ b/utils/gn/secondary/llvm/lib/FuzzMutate/BUILD.gn
@@ -0,0 +1,19 @@
+static_library("FuzzMutate") {
+  output_name = "LLVMFuzzMutate"
+  deps = [
+    "//llvm/lib/Analysis",
+    "//llvm/lib/Bitcode/Reader",
+    "//llvm/lib/Bitcode/Writer",
+    "//llvm/lib/IR",
+    "//llvm/lib/Support",
+    "//llvm/lib/Target",
+    "//llvm/lib/Transforms/Scalar",
+  ]
+  sources = [
+    "FuzzerCLI.cpp",
+    "IRMutator.cpp",
+    "OpDescriptor.cpp",
+    "Operations.cpp",
+    "RandomIRBuilder.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/llvm/lib/IR/BUILD.gn b/utils/gn/secondary/llvm/lib/IR/BUILD.gn
index 35e3db91c93ccd91a3f03da024bde03266334143..4613e0a15704961a7a723b50b7a8df187d7b0727 100644
--- a/utils/gn/secondary/llvm/lib/IR/BUILD.gn
+++ b/utils/gn/secondary/llvm/lib/IR/BUILD.gn
@@ -10,12 +10,7 @@ static_library("IR") {
   public_deps = [
     # Must be public_dep because IR's public headers include llvm-config.h.
     "//llvm/include/llvm/Config:llvm-config",
-
-    # Must be public_dep because IR's public headers include Attributes.inc.
-    "//llvm/include/llvm/IR:Attributes",
-
-    # Must be public_dep because IR's public headers include IntrinsicEnums.inc.
-    "//llvm/include/llvm/IR:IntrinsicEnums",
+    "//llvm/include/llvm/IR:public_tablegen",
   ]
   deps = [
     ":AttributesCompatFunc",
@@ -24,6 +19,7 @@ static_library("IR") {
     "//llvm/lib/Support",
   ]
   sources = [
+    "AbstractCallSite.cpp",
     "AsmWriter.cpp",
     "Attributes.cpp",
     "AutoUpgrade.cpp",
@@ -41,7 +37,6 @@ static_library("IR") {
     "DiagnosticHandler.cpp",
     "DiagnosticInfo.cpp",
     "DiagnosticPrinter.cpp",
-    "DomTreeUpdater.cpp",
     "Dominators.cpp",
     "Function.cpp",
     "GVMaterializer.cpp",
diff --git a/utils/gn/secondary/llvm/lib/LineEditor/BUILD.gn b/utils/gn/secondary/llvm/lib/LineEditor/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..b8f618533bc123e4131e2f7e304dbbd9e6796e17
--- /dev/null
+++ b/utils/gn/secondary/llvm/lib/LineEditor/BUILD.gn
@@ -0,0 +1,12 @@
+static_library("LineEditor") {
+  output_name = "LLVMLineEditor"
+  deps = [
+    "//llvm/include/llvm/Config:config",
+    "//llvm/lib/Support",
+    "//llvm/utils/gn/build/libs/edit",
+  ]
+
+  sources = [
+    "LineEditor.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/llvm/lib/MCA/BUILD.gn b/utils/gn/secondary/llvm/lib/MCA/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..9752ca828d68c886d3bec475820e6b0d67d6f9f9
--- /dev/null
+++ b/utils/gn/secondary/llvm/lib/MCA/BUILD.gn
@@ -0,0 +1,29 @@
+static_library("MCA") {
+  output_name = "LLVMMCA"
+  deps = [
+    "//llvm/lib/CodeGen",
+    "//llvm/lib/MC",
+    "//llvm/lib/Support",
+  ]
+  include_dirs = [ "../include" ]
+  sources = [
+    "Context.cpp",
+    "HWEventListener.cpp",
+    "HardwareUnits/HardwareUnit.cpp",
+    "HardwareUnits/LSUnit.cpp",
+    "HardwareUnits/RegisterFile.cpp",
+    "HardwareUnits/ResourceManager.cpp",
+    "HardwareUnits/RetireControlUnit.cpp",
+    "HardwareUnits/Scheduler.cpp",
+    "InstrBuilder.cpp",
+    "Instruction.cpp",
+    "Pipeline.cpp",
+    "Stages/DispatchStage.cpp",
+    "Stages/EntryStage.cpp",
+    "Stages/ExecuteStage.cpp",
+    "Stages/InstructionTables.cpp",
+    "Stages/RetireStage.cpp",
+    "Stages/Stage.cpp",
+    "Support.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/llvm/lib/OptRemarks/BUILD.gn b/utils/gn/secondary/llvm/lib/OptRemarks/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..cd80b85ae15a291976d3a328925216ab892a949b
--- /dev/null
+++ b/utils/gn/secondary/llvm/lib/OptRemarks/BUILD.gn
@@ -0,0 +1,10 @@
+static_library("OptRemarks") {
+  output_name = "LLVMOptRemarks"
+  deps = [
+    "//llvm/lib/Support",
+  ]
+
+  sources = [
+    "OptRemarksParser.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/llvm/lib/ProfileData/Coverage/BUILD.gn b/utils/gn/secondary/llvm/lib/ProfileData/Coverage/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..bd72861b5e5847a4d968b811fa3752846d435e44
--- /dev/null
+++ b/utils/gn/secondary/llvm/lib/ProfileData/Coverage/BUILD.gn
@@ -0,0 +1,14 @@
+static_library("Coverage") {
+  output_name = "LLVMCoverage"
+  deps = [
+    "//llvm/lib/IR",
+    "//llvm/lib/Object",
+    "//llvm/lib/ProfileData",
+    "//llvm/lib/Support",
+  ]
+  sources = [
+    "CoverageMapping.cpp",
+    "CoverageMappingReader.cpp",
+    "CoverageMappingWriter.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/llvm/lib/Support/BUILD.gn b/utils/gn/secondary/llvm/lib/Support/BUILD.gn
index 84ae80fbcfa26856e2523af2413eda2a58603663..9a5fbd9b6073f69dc0e7d75073fceac564302758 100644
--- a/utils/gn/secondary/llvm/lib/Support/BUILD.gn
+++ b/utils/gn/secondary/llvm/lib/Support/BUILD.gn
@@ -88,6 +88,7 @@ static_library("Support") {
     "MathExtras.cpp",
     "MemoryBuffer.cpp",
     "NativeFormatting.cpp",
+    "Optional.cpp",
     "Options.cpp",
     "Parallel.cpp",
     "PluginLoader.cpp",
@@ -155,7 +156,7 @@ static_library("Support") {
 
   libs = []
 
-  if (host_os == "linux") {
+  if (current_os == "linux" || current_os == "android") {
     libs += [ "dl" ]
   }
 }
diff --git a/utils/gn/secondary/llvm/lib/Target/AArch64/AsmParser/BUILD.gn b/utils/gn/secondary/llvm/lib/Target/AArch64/AsmParser/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..54a12d20c70185dfb41caa3719fb83b25650f527
--- /dev/null
+++ b/utils/gn/secondary/llvm/lib/Target/AArch64/AsmParser/BUILD.gn
@@ -0,0 +1,24 @@
+import("//llvm/utils/TableGen/tablegen.gni")
+
+tablegen("AArch64GenAsmMatcher") {
+  visibility = [ ":AsmParser" ]
+  args = [ "-gen-asm-matcher" ]
+  td_file = "../AArch64.td"
+}
+
+static_library("AsmParser") {
+  output_name = "LLVMAArch64AsmParser"
+  deps = [
+    ":AArch64GenAsmMatcher",
+    "//llvm/lib/MC",
+    "//llvm/lib/MC/MCParser",
+    "//llvm/lib/Support",
+    "//llvm/lib/Target/AArch64/MCTargetDesc",
+    "//llvm/lib/Target/AArch64/TargetInfo",
+    "//llvm/lib/Target/AArch64/Utils",
+  ]
+  include_dirs = [ ".." ]
+  sources = [
+    "AArch64AsmParser.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/llvm/lib/Target/AArch64/BUILD.gn b/utils/gn/secondary/llvm/lib/Target/AArch64/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..2a3fa89fbe39ac877de52834240318c3bcf92673
--- /dev/null
+++ b/utils/gn/secondary/llvm/lib/Target/AArch64/BUILD.gn
@@ -0,0 +1,130 @@
+import("//llvm/utils/TableGen/tablegen.gni")
+
+tablegen("AArch64GenAsmMatcher") {
+  visibility = [ ":LLVMAArch64CodeGen" ]
+  args = [ "-gen-asm-matcher" ]
+  td_file = "AArch64.td"
+}
+
+tablegen("AArch64GenCallingConv") {
+  visibility = [ ":LLVMAArch64CodeGen" ]
+  args = [ "-gen-callingconv" ]
+  td_file = "AArch64.td"
+}
+
+tablegen("AArch64GenDAGISel") {
+  visibility = [ ":LLVMAArch64CodeGen" ]
+  args = [ "-gen-dag-isel" ]
+  td_file = "AArch64.td"
+}
+
+tablegen("AArch64GenFastISel") {
+  visibility = [ ":LLVMAArch64CodeGen" ]
+  args = [ "-gen-fast-isel" ]
+  td_file = "AArch64.td"
+}
+
+tablegen("AArch64GenGlobalISel") {
+  visibility = [ ":LLVMAArch64CodeGen" ]
+  args = [ "-gen-global-isel" ]
+  td_file = "AArch64.td"
+}
+
+tablegen("AArch64GenMCPseudoLowering") {
+  visibility = [ ":LLVMAArch64CodeGen" ]
+  args = [ "-gen-pseudo-lowering" ]
+  td_file = "AArch64.td"
+}
+
+tablegen("AArch64GenRegisterBank") {
+  visibility = [ ":LLVMAArch64CodeGen" ]
+  args = [ "-gen-register-bank" ]
+  td_file = "AArch64.td"
+}
+
+static_library("LLVMAArch64CodeGen") {
+  deps = [
+    ":AArch64GenAsmMatcher",
+    ":AArch64GenCallingConv",
+    ":AArch64GenDAGISel",
+    ":AArch64GenFastISel",
+    ":AArch64GenGlobalISel",
+    ":AArch64GenMCPseudoLowering",
+    ":AArch64GenRegisterBank",
+    "InstPrinter",
+    "MCTargetDesc",
+    "TargetInfo",
+    "Utils",
+    "//llvm/lib/Analysis",
+    "//llvm/lib/CodeGen",
+    "//llvm/lib/CodeGen/AsmPrinter",
+    "//llvm/lib/CodeGen/GlobalISel",
+    "//llvm/lib/CodeGen/SelectionDAG",
+    "//llvm/lib/IR",
+    "//llvm/lib/MC",
+    "//llvm/lib/Support",
+    "//llvm/lib/Target",
+    "//llvm/lib/Transforms/Scalar",
+  ]
+  include_dirs = [ "." ]
+  sources = [
+    "AArch64A53Fix835769.cpp",
+    "AArch64A57FPLoadBalancing.cpp",
+    "AArch64AdvSIMDScalarPass.cpp",
+    "AArch64AsmPrinter.cpp",
+    "AArch64BranchTargets.cpp",
+    "AArch64CallLowering.cpp",
+    "AArch64CallingConvention.cpp",
+    "AArch64CleanupLocalDynamicTLSPass.cpp",
+    "AArch64CollectLOH.cpp",
+    "AArch64CompressJumpTables.cpp",
+    "AArch64CondBrTuning.cpp",
+    "AArch64ConditionOptimizer.cpp",
+    "AArch64ConditionalCompares.cpp",
+    "AArch64DeadRegisterDefinitionsPass.cpp",
+    "AArch64ExpandPseudoInsts.cpp",
+    "AArch64FalkorHWPFFix.cpp",
+    "AArch64FastISel.cpp",
+    "AArch64FrameLowering.cpp",
+    "AArch64ISelDAGToDAG.cpp",
+    "AArch64ISelLowering.cpp",
+    "AArch64InstrInfo.cpp",
+    "AArch64InstructionSelector.cpp",
+    "AArch64LegalizerInfo.cpp",
+    "AArch64LoadStoreOptimizer.cpp",
+    "AArch64MCInstLower.cpp",
+    "AArch64MacroFusion.cpp",
+    "AArch64PBQPRegAlloc.cpp",
+    "AArch64PreLegalizerCombiner.cpp",
+    "AArch64PromoteConstant.cpp",
+    "AArch64RedundantCopyElimination.cpp",
+    "AArch64RegisterBankInfo.cpp",
+    "AArch64RegisterInfo.cpp",
+    "AArch64SIMDInstrOpt.cpp",
+    "AArch64SelectionDAGInfo.cpp",
+    "AArch64SpeculationHardening.cpp",
+    "AArch64StorePairSuppress.cpp",
+    "AArch64Subtarget.cpp",
+    "AArch64TargetMachine.cpp",
+    "AArch64TargetObjectFile.cpp",
+    "AArch64TargetTransformInfo.cpp",
+  ]
+}
+
+# This is a bit different from most build files: Due to this group
+# having the directory's name, "//llvm/lib/Target/AArch64" will refer to this
+# target, which pulls in the code in this directory *and all subdirectories*.
+# For most other directories, "//llvm/lib/Foo" only pulls in the code directly
+# in "llvm/lib/Foo". The forwarding targets in //llvm/lib/Target expect this
+# different behavior.
+group("AArch64") {
+  deps = [
+    ":LLVMAArch64CodeGen",
+    "AsmParser",
+    "Disassembler",
+    "InstPrinter",
+    "MCTargetDesc",
+    "TargetInfo",
+    "Utils",
+  ]
+}
diff --git a/utils/gn/secondary/llvm/lib/Target/AArch64/Disassembler/BUILD.gn b/utils/gn/secondary/llvm/lib/Target/AArch64/Disassembler/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..eda8422db3afaf60adeec6ae0ab075d01ae8b13a
--- /dev/null
+++ b/utils/gn/secondary/llvm/lib/Target/AArch64/Disassembler/BUILD.gn
@@ -0,0 +1,25 @@
+import("//llvm/utils/TableGen/tablegen.gni")
+
+tablegen("AArch64GenDisassemblerTables") {
+  visibility = [ ":Disassembler" ]
+  args = [ "-gen-disassembler" ]
+  td_file = "../AArch64.td"
+}
+
+static_library("Disassembler") {
+  output_name = "LLVMAArch64Disassembler"
+  deps = [
+    ":AArch64GenDisassemblerTables",
+    "//llvm/lib/MC",
+    "//llvm/lib/MC/MCDisassembler",
+    "//llvm/lib/Support",
+    "//llvm/lib/Target/AArch64/MCTargetDesc",
+    "//llvm/lib/Target/AArch64/TargetInfo",
+    "//llvm/lib/Target/AArch64/Utils",
+  ]
+  include_dirs = [ ".." ]
+  sources = [
+    "AArch64Disassembler.cpp",
+    "AArch64ExternalSymbolizer.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/llvm/lib/Target/AArch64/InstPrinter/BUILD.gn b/utils/gn/secondary/llvm/lib/Target/AArch64/InstPrinter/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..6c1749545d3507453415af04c0ea20eced6fb087
--- /dev/null
+++ b/utils/gn/secondary/llvm/lib/Target/AArch64/InstPrinter/BUILD.gn
@@ -0,0 +1,35 @@
+import("//llvm/utils/TableGen/tablegen.gni")
+
+tablegen("AArch64GenAsmWriter") {
+  visibility = [ ":InstPrinter" ]
+  args = [ "-gen-asm-writer" ]
+  td_file = "../AArch64.td"
+}
+
+tablegen("AArch64GenAsmWriter1") {
+  visibility = [ ":InstPrinter" ]
+  args = [
+    "-gen-asm-writer",
+    "-asmwriternum=1",
+  ]
+  td_file = "../AArch64.td"
+}
+
+static_library("InstPrinter") {
+  output_name = "LLVMAArch64AsmPrinter"
+  deps = [
+    ":AArch64GenAsmWriter",
+    ":AArch64GenAsmWriter1",
+    "//llvm/lib/MC",
+    "//llvm/lib/Support",
+
+    # MCTargetDesc depends on InstPrinter, so we can't depend on the full
+    # MCTargetDesc target here: it would form a cycle.
+    "//llvm/lib/Target/AArch64/MCTargetDesc:tablegen",
+    "//llvm/lib/Target/AArch64/Utils",
+  ]
+  include_dirs = [ ".." ]
+  sources = [
+    "AArch64InstPrinter.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/llvm/lib/Target/AArch64/MCTargetDesc/BUILD.gn b/utils/gn/secondary/llvm/lib/Target/AArch64/MCTargetDesc/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..6303fbef17a1c16ea0d3f47a14e5ca27589b643b
--- /dev/null
+++ b/utils/gn/secondary/llvm/lib/Target/AArch64/MCTargetDesc/BUILD.gn
@@ -0,0 +1,68 @@
+import("//llvm/utils/TableGen/tablegen.gni")
+
+tablegen("AArch64GenInstrInfo") {
+  visibility = [ ":tablegen" ]
+  args = [ "-gen-instr-info" ]
+  td_file = "../AArch64.td"
+}
+
+tablegen("AArch64GenMCCodeEmitter") {
+  visibility = [ ":tablegen" ]
+  args = [ "-gen-emitter" ]
+  td_file = "../AArch64.td"
+}
+
+tablegen("AArch64GenRegisterInfo") {
+  visibility = [ ":tablegen" ]
+  args = [ "-gen-register-info" ]
+  td_file = "../AArch64.td"
+}
+
+tablegen("AArch64GenSubtargetInfo") {
+  visibility = [ ":tablegen" ]
+  args = [ "-gen-subtarget" ]
+  td_file = "../AArch64.td"
+}
+
+group("tablegen") {
+  visibility = [
+    ":MCTargetDesc",
+    "../InstPrinter",
+    "../TargetInfo",
+    "../Utils",
+  ]
+  public_deps = [
+    ":AArch64GenInstrInfo",
+    ":AArch64GenMCCodeEmitter",
+    ":AArch64GenRegisterInfo",
+    ":AArch64GenSubtargetInfo",
+  ]
+}
+
+static_library("MCTargetDesc") {
+  output_name = "LLVMAArch64Desc"
+  public_deps = [
+    ":tablegen",
+  ]
+  deps = [
+    "//llvm/lib/MC",
+    "//llvm/lib/Support",
+    "//llvm/lib/Target/AArch64/InstPrinter",
+    "//llvm/lib/Target/AArch64/TargetInfo",
+    "//llvm/lib/Target/AArch64/Utils",
+  ]
+  include_dirs = [ ".." ]
+  sources = [
+    "AArch64AsmBackend.cpp",
+    "AArch64ELFObjectWriter.cpp",
+    "AArch64ELFStreamer.cpp",
+    "AArch64MCAsmInfo.cpp",
+    "AArch64MCCodeEmitter.cpp",
+    "AArch64MCExpr.cpp",
+    "AArch64MCTargetDesc.cpp",
+    "AArch64MachObjectWriter.cpp",
+    "AArch64TargetStreamer.cpp",
+    "AArch64WinCOFFObjectWriter.cpp",
+    "AArch64WinCOFFStreamer.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/llvm/lib/Target/AArch64/TargetInfo/BUILD.gn b/utils/gn/secondary/llvm/lib/Target/AArch64/TargetInfo/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..114d93f15e764ac35e1409822001ec7056551b5a
--- /dev/null
+++ b/utils/gn/secondary/llvm/lib/Target/AArch64/TargetInfo/BUILD.gn
@@ -0,0 +1,14 @@
+static_library("TargetInfo") {
+  output_name = "LLVMAArch64Info"
+  deps = [
+    "//llvm/lib/Support",
+
+    # MCTargetDesc depends on TargetInfo, so we can't depend on the full
+    # MCTargetDesc target here: it would form a cycle.
+    "//llvm/lib/Target/AArch64/MCTargetDesc:tablegen",
+  ]
+  include_dirs = [ ".." ]
+  sources = [
+    "AArch64TargetInfo.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/llvm/lib/Target/AArch64/Utils/BUILD.gn b/utils/gn/secondary/llvm/lib/Target/AArch64/Utils/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..144bd5cafef178512e689cce6bee19ca134b8520
--- /dev/null
+++ b/utils/gn/secondary/llvm/lib/Target/AArch64/Utils/BUILD.gn
@@ -0,0 +1,24 @@
+import("//llvm/utils/TableGen/tablegen.gni")
+
+tablegen("AArch64GenSystemOperands") {
+  visibility = [ ":Utils" ]
+  args = [ "-gen-searchable-tables" ]
+  td_file = "../AArch64.td"
+}
+
+static_library("Utils") {
+  output_name = "LLVMAArch64Utils"
+  public_deps = [
+    ":AArch64GenSystemOperands",
+  ]
+  deps = [
+    "//llvm/lib/Support",
+    "//llvm/lib/Target/AArch64/MCTargetDesc:tablegen",
+  ]
+
+  # AArch64BaseInfo.h includes a header from MCTargetDesc :-/
+  include_dirs = [ ".." ]
+  sources = [
+    "AArch64BaseInfo.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/llvm/lib/Target/ARM/AsmParser/BUILD.gn b/utils/gn/secondary/llvm/lib/Target/ARM/AsmParser/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..9b6dd6401ddc656e7900e6085581d7543eb0c96a
--- /dev/null
+++ b/utils/gn/secondary/llvm/lib/Target/ARM/AsmParser/BUILD.gn
@@ -0,0 +1,24 @@
+import("//llvm/utils/TableGen/tablegen.gni")
+
+tablegen("ARMGenAsmMatcher") {
+  visibility = [ ":AsmParser" ]
+  args = [ "-gen-asm-matcher" ]
+  td_file = "../ARM.td"
+}
+
+static_library("AsmParser") {
+  output_name = "LLVMARMAsmParser"
+  deps = [
+    ":ARMGenAsmMatcher",
+    "//llvm/lib/MC",
+    "//llvm/lib/MC/MCParser",
+    "//llvm/lib/Support",
+    "//llvm/lib/Target/ARM/MCTargetDesc",
+    "//llvm/lib/Target/ARM/TargetInfo",
+    "//llvm/lib/Target/ARM/Utils",
+  ]
+  include_dirs = [ ".." ]
+  sources = [
+    "ARMAsmParser.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/llvm/lib/Target/ARM/BUILD.gn b/utils/gn/secondary/llvm/lib/Target/ARM/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..f4bc1ed0c485ab38c2e30eccc27d23df865c874d
--- /dev/null
+++ b/utils/gn/secondary/llvm/lib/Target/ARM/BUILD.gn
@@ -0,0 +1,122 @@
+import("//llvm/utils/TableGen/tablegen.gni")
+
+tablegen("ARMGenCallingConv") {
+  visibility = [ ":LLVMARMCodeGen" ]
+  args = [ "-gen-callingconv" ]
+  td_file = "ARM.td"
+}
+
+tablegen("ARMGenDAGISel") {
+  visibility = [ ":LLVMARMCodeGen" ]
+  args = [ "-gen-dag-isel" ]
+  td_file = "ARM.td"
+}
+
+tablegen("ARMGenFastISel") {
+  visibility = [ ":LLVMARMCodeGen" ]
+  args = [ "-gen-fast-isel" ]
+  td_file = "ARM.td"
+}
+
+tablegen("ARMGenGlobalISel") {
+  visibility = [ ":LLVMARMCodeGen" ]
+  args = [ "-gen-global-isel" ]
+  td_file = "ARM.td"
+}
+
+tablegen("ARMGenMCPseudoLowering") {
+  visibility = [ ":LLVMARMCodeGen" ]
+  args = [ "-gen-pseudo-lowering" ]
+  td_file = "ARM.td"
+}
+
+tablegen("ARMGenRegisterBank") {
+  visibility = [ ":LLVMARMCodeGen" ]
+  args = [ "-gen-register-bank" ]
+  td_file = "ARM.td"
+}
+
+static_library("LLVMARMCodeGen") {
+  deps = [
+    ":ARMGenCallingConv",
+    ":ARMGenDAGISel",
+    ":ARMGenFastISel",
+    ":ARMGenGlobalISel",
+    ":ARMGenMCPseudoLowering",
+    ":ARMGenRegisterBank",
+    "InstPrinter",
+    "MCTargetDesc",
+    "TargetInfo",
+    "Utils",
+    "//llvm/include/llvm/Config:llvm-config",
+    "//llvm/lib/Analysis",
+    "//llvm/lib/CodeGen",
+    "//llvm/lib/CodeGen/AsmPrinter",
+    "//llvm/lib/CodeGen/GlobalISel",
+    "//llvm/lib/CodeGen/SelectionDAG",
+    "//llvm/lib/IR",
+    "//llvm/lib/MC",
+    "//llvm/lib/Support",
+    "//llvm/lib/Target",
+  ]
+  include_dirs = [ "." ]
+  sources = [
+    "A15SDOptimizer.cpp",
+    "ARMAsmPrinter.cpp",
+    "ARMBaseInstrInfo.cpp",
+    "ARMBaseRegisterInfo.cpp",
+    "ARMCallLowering.cpp",
+    "ARMCallingConv.cpp",
+    "ARMCodeGenPrepare.cpp",
+    "ARMComputeBlockSize.cpp",
+    "ARMConstantIslandPass.cpp",
+    "ARMConstantPoolValue.cpp",
+    "ARMExpandPseudoInsts.cpp",
+    "ARMFastISel.cpp",
+    "ARMFrameLowering.cpp",
+    "ARMHazardRecognizer.cpp",
+    "ARMISelDAGToDAG.cpp",
+    "ARMISelLowering.cpp",
+    "ARMInstrInfo.cpp",
+    "ARMInstructionSelector.cpp",
+    "ARMLegalizerInfo.cpp",
+    "ARMLoadStoreOptimizer.cpp",
+    "ARMMCInstLower.cpp",
+    "ARMMachineFunctionInfo.cpp",
+    "ARMMacroFusion.cpp",
+    "ARMOptimizeBarriersPass.cpp",
+    "ARMParallelDSP.cpp",
+    "ARMRegisterBankInfo.cpp",
+    "ARMRegisterInfo.cpp",
+    "ARMSelectionDAGInfo.cpp",
+    "ARMSubtarget.cpp",
+    "ARMTargetMachine.cpp",
+    "ARMTargetObjectFile.cpp",
+    "ARMTargetTransformInfo.cpp",
+    "MLxExpansionPass.cpp",
+    "Thumb1FrameLowering.cpp",
+    "Thumb1InstrInfo.cpp",
+    "Thumb2ITBlockPass.cpp",
+    "Thumb2InstrInfo.cpp",
+    "Thumb2SizeReduction.cpp",
+    "ThumbRegisterInfo.cpp",
+  ]
+}
+
+# This is a bit different from most build files: Due to this group
+# having the directory's name, "//llvm/lib/Target/ARM" will refer to this
+# target, which pulls in the code in this directory *and all subdirectories*.
+# For most other directories, "//llvm/lib/Foo" only pulls in the code directly
+# in "llvm/lib/Foo". The forwarding targets in //llvm/lib/Target expect this
+# different behavior.
+group("ARM") {
+  deps = [
+    ":LLVMARMCodeGen",
+    "AsmParser",
+    "Disassembler",
+    "InstPrinter",
+    "MCTargetDesc",
+    "TargetInfo",
+    "Utils",
+  ]
+}
diff --git a/utils/gn/secondary/llvm/lib/Target/ARM/Disassembler/BUILD.gn b/utils/gn/secondary/llvm/lib/Target/ARM/Disassembler/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..bfe5e50c5943f97e08898195ea3bbc49ddd77c0d
--- /dev/null
+++ b/utils/gn/secondary/llvm/lib/Target/ARM/Disassembler/BUILD.gn
@@ -0,0 +1,23 @@
+import("//llvm/utils/TableGen/tablegen.gni")
+
+tablegen("ARMGenDisassemblerTables") {
+  visibility = [ ":Disassembler" ]
+  args = [ "-gen-disassembler" ]
+  td_file = "../ARM.td"
+}
+
+static_library("Disassembler") {
+  output_name = "LLVMARMDisassembler"
+  deps = [
+    ":ARMGenDisassemblerTables",
+    "//llvm/lib/MC",
+    "//llvm/lib/MC/MCDisassembler",
+    "//llvm/lib/Support",
+    "//llvm/lib/Target/ARM/MCTargetDesc",
+    "//llvm/lib/Target/ARM/Utils",
+  ]
+  include_dirs = [ ".." ]
+  sources = [
+    "ARMDisassembler.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/llvm/lib/Target/ARM/InstPrinter/BUILD.gn b/utils/gn/secondary/llvm/lib/Target/ARM/InstPrinter/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..6cac44291fdf236a7dc5d378217f446a11aa6497
--- /dev/null
+++ b/utils/gn/secondary/llvm/lib/Target/ARM/InstPrinter/BUILD.gn
@@ -0,0 +1,25 @@
+import("//llvm/utils/TableGen/tablegen.gni")
+
+tablegen("ARMGenAsmWriter") {
+  visibility = [ ":InstPrinter" ]
+  args = [ "-gen-asm-writer" ]
+  td_file = "../ARM.td"
+}
+
+static_library("InstPrinter") {
+  output_name = "LLVMARMAsmPrinter"
+  deps = [
+    ":ARMGenAsmWriter",
+    "//llvm/lib/MC",
+    "//llvm/lib/Support",
+
+    # MCTargetDesc depends on InstPrinter, so we can't depend on the full
+    # MCTargetDesc target here: it would form a cycle.
+    "//llvm/lib/Target/ARM/MCTargetDesc:tablegen",
+    "//llvm/lib/Target/ARM/Utils",
+  ]
+  include_dirs = [ ".." ]
+  sources = [
+    "ARMInstPrinter.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/llvm/lib/Target/ARM/MCTargetDesc/BUILD.gn b/utils/gn/secondary/llvm/lib/Target/ARM/MCTargetDesc/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..3ef2f435df5fc9e2e1870910ed393249728d2e07
--- /dev/null
+++ b/utils/gn/secondary/llvm/lib/Target/ARM/MCTargetDesc/BUILD.gn
@@ -0,0 +1,70 @@
+import("//llvm/utils/TableGen/tablegen.gni")
+
+tablegen("ARMGenInstrInfo") {
+  visibility = [ ":tablegen" ]
+  args = [ "-gen-instr-info" ]
+  td_file = "../ARM.td"
+}
+
+tablegen("ARMGenMCCodeEmitter") {
+  visibility = [ ":tablegen" ]
+  args = [ "-gen-emitter" ]
+  td_file = "../ARM.td"
+}
+
+tablegen("ARMGenRegisterInfo") {
+  visibility = [ ":tablegen" ]
+  args = [ "-gen-register-info" ]
+  td_file = "../ARM.td"
+}
+
+tablegen("ARMGenSubtargetInfo") {
+  visibility = [ ":tablegen" ]
+  args = [ "-gen-subtarget" ]
+  td_file = "../ARM.td"
+}
+
+group("tablegen") {
+  visibility = [
+    ":MCTargetDesc",
+    "../InstPrinter",
+    "../TargetInfo",
+    "../Utils",
+  ]
+  public_deps = [
+    ":ARMGenInstrInfo",
+    ":ARMGenMCCodeEmitter",
+    ":ARMGenRegisterInfo",
+    ":ARMGenSubtargetInfo",
+  ]
+}
+static_library("MCTargetDesc") {
+  output_name = "LLVMARMDesc"
+  public_deps = [
+    ":tablegen",
+  ]
+  deps = [
+    "//llvm/lib/MC",
+    "//llvm/lib/MC/MCDisassembler",
+    "//llvm/lib/Support",
+    "//llvm/lib/Target/ARM/InstPrinter",
+    "//llvm/lib/Target/ARM/TargetInfo",
+    "//llvm/lib/Target/ARM/Utils",
+  ]
+  include_dirs = [ ".." ]
+  sources = [
+    "ARMAsmBackend.cpp",
+    "ARMELFObjectWriter.cpp",
+    "ARMELFStreamer.cpp",
+    "ARMMCAsmInfo.cpp",
+    "ARMMCCodeEmitter.cpp",
+    "ARMMCExpr.cpp",
+    "ARMMCTargetDesc.cpp",
+    "ARMMachORelocationInfo.cpp",
+    "ARMMachObjectWriter.cpp",
+    "ARMTargetStreamer.cpp",
+    "ARMUnwindOpAsm.cpp",
+    "ARMWinCOFFObjectWriter.cpp",
+    "ARMWinCOFFStreamer.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/llvm/lib/Target/ARM/TargetInfo/BUILD.gn b/utils/gn/secondary/llvm/lib/Target/ARM/TargetInfo/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..521367c4f42707055c859923618aa5e6d754078a
--- /dev/null
+++ b/utils/gn/secondary/llvm/lib/Target/ARM/TargetInfo/BUILD.gn
@@ -0,0 +1,14 @@
+static_library("TargetInfo") {
+  output_name = "LLVMARMInfo"
+  deps = [
+    "//llvm/lib/Support",
+
+    # MCTargetDesc depends on TargetInfo, so we can't depend on the full
+    # MCTargetDesc target here: it would form a cycle.
+    "//llvm/lib/Target/ARM/MCTargetDesc:tablegen",
+  ]
+  include_dirs = [ ".." ]
+  sources = [
+    "ARMTargetInfo.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/llvm/lib/Target/ARM/Utils/BUILD.gn b/utils/gn/secondary/llvm/lib/Target/ARM/Utils/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..e0ddc22f90bd6e7d37ffd90d11bb293b1a216f02
--- /dev/null
+++ b/utils/gn/secondary/llvm/lib/Target/ARM/Utils/BUILD.gn
@@ -0,0 +1,25 @@
+import("//llvm/utils/TableGen/tablegen.gni")
+
+tablegen("ARMGenSystemRegister") {
+  visibility = [ ":Utils" ]
+  args = [ "-gen-searchable-tables" ]
+  td_file = "../ARM.td"
+}
+
+static_library("Utils") {
+  output_name = "LLVMARMUtils"
+  public_deps = [
+    ":ARMGenSystemRegister",
+  ]
+  deps = [
+    "//llvm/lib/Support",
+    "//llvm/lib/Target/ARM/MCTargetDesc:tablegen",
+  ]
+
+  # ARMBaseInfo.h includes a header from MCTargetDesc,
+  # https://reviews.llvm.org/D35209#1075113 :-/
+  include_dirs = [ ".." ]
+  sources = [
+    "ARMBaseInfo.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/llvm/lib/Target/BPF/AsmParser/BUILD.gn b/utils/gn/secondary/llvm/lib/Target/BPF/AsmParser/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..bc6e7ab9bf01a87319a4d3b6220555ea01323ed7
--- /dev/null
+++ b/utils/gn/secondary/llvm/lib/Target/BPF/AsmParser/BUILD.gn
@@ -0,0 +1,23 @@
+import("//llvm/utils/TableGen/tablegen.gni")
+
+tablegen("BPFGenAsmMatcher") {
+  visibility = [ ":AsmParser" ]
+  args = [ "-gen-asm-matcher" ]
+  td_file = "../BPF.td"
+}
+
+static_library("AsmParser") {
+  output_name = "LLVMBPFAsmParser"
+  deps = [
+    ":BPFGenAsmMatcher",
+    "//llvm/lib/MC",
+    "//llvm/lib/MC/MCParser",
+    "//llvm/lib/Support",
+    "//llvm/lib/Target/BPF/MCTargetDesc",
+    "//llvm/lib/Target/BPF/TargetInfo",
+  ]
+  include_dirs = [ ".." ]
+  sources = [
+    "BPFAsmParser.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/llvm/lib/Target/BPF/BUILD.gn b/utils/gn/secondary/llvm/lib/Target/BPF/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..180ab94927a5b17839f11661f91119b3865b5dc2
--- /dev/null
+++ b/utils/gn/secondary/llvm/lib/Target/BPF/BUILD.gn
@@ -0,0 +1,94 @@
+import("//llvm/utils/TableGen/tablegen.gni")
+
+tablegen("BPFGenCallingConv") {
+  visibility = [ ":LLVMBPFCodeGen" ]
+  args = [ "-gen-callingconv" ]
+  td_file = "BPF.td"
+}
+
+tablegen("BPFGenDAGISel") {
+  visibility = [ ":LLVMBPFCodeGen" ]
+  args = [ "-gen-dag-isel" ]
+  td_file = "BPF.td"
+}
+
+tablegen("BPFGenFastISel") {
+  visibility = [ ":LLVMBPFCodeGen" ]
+  args = [ "-gen-fast-isel" ]
+  td_file = "BPF.td"
+}
+
+tablegen("BPFGenGlobalISel") {
+  visibility = [ ":LLVMBPFCodeGen" ]
+  args = [ "-gen-global-isel" ]
+  td_file = "BPF.td"
+}
+
+tablegen("BPFGenMCPseudoLowering") {
+  visibility = [ ":LLVMBPFCodeGen" ]
+  args = [ "-gen-pseudo-lowering" ]
+  td_file = "BPF.td"
+}
+
+tablegen("BPFGenRegisterBank") {
+  visibility = [ ":LLVMBPFCodeGen" ]
+  args = [ "-gen-register-bank" ]
+  td_file = "BPF.td"
+}
+
+static_library("LLVMBPFCodeGen") {
+  deps = [
+    ":BPFGenCallingConv",
+    ":BPFGenDAGISel",
+    ":BPFGenFastISel",
+    ":BPFGenGlobalISel",
+    ":BPFGenMCPseudoLowering",
+    ":BPFGenRegisterBank",
+    "InstPrinter",
+    "MCTargetDesc",
+    "TargetInfo",
+    "//llvm/include/llvm/Config:llvm-config",
+    "//llvm/lib/Analysis",
+    "//llvm/lib/CodeGen",
+    "//llvm/lib/CodeGen/AsmPrinter",
+    "//llvm/lib/CodeGen/GlobalISel",
+    "//llvm/lib/CodeGen/SelectionDAG",
+    "//llvm/lib/IR",
+    "//llvm/lib/MC",
+    "//llvm/lib/Support",
+    "//llvm/lib/Target",
+  ]
+  include_dirs = [ "." ]
+  sources = [
+    "BPFAsmPrinter.cpp",
+    "BPFFrameLowering.cpp",
+    "BPFISelDAGToDAG.cpp",
+    "BPFISelLowering.cpp",
+    "BPFInstrInfo.cpp",
+    "BPFMCInstLower.cpp",
+    "BPFMIChecking.cpp",
+    "BPFMIPeephole.cpp",
+    "BPFRegisterInfo.cpp",
+    "BPFSelectionDAGInfo.cpp",
+    "BPFSubtarget.cpp",
+    "BPFTargetMachine.cpp",
+    "BTFDebug.cpp",
+  ]
+}
+
+# This is a bit different from most build files: Due to this group
+# having the directory's name, "//llvm/lib/Target/BPF" will refer to this
+# target, which pulls in the code in this directory *and all subdirectories*.
+# For most other directories, "//llvm/lib/Foo" only pulls in the code directly
+# in "llvm/lib/Foo". The forwarding targets in //llvm/lib/Target expect this
+# different behavior.
+group("BPF") {
+  deps = [
+    ":LLVMBPFCodeGen",
+    "AsmParser",
+    "Disassembler",
+    "InstPrinter",
+    "MCTargetDesc",
+    "TargetInfo",
+  ]
+}
diff --git a/utils/gn/secondary/llvm/lib/Target/BPF/Disassembler/BUILD.gn b/utils/gn/secondary/llvm/lib/Target/BPF/Disassembler/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..1ee3bc70ba75a8abad5cff4a048bd99793642ec7
--- /dev/null
+++ b/utils/gn/secondary/llvm/lib/Target/BPF/Disassembler/BUILD.gn
@@ -0,0 +1,22 @@
+import("//llvm/utils/TableGen/tablegen.gni")
+
+tablegen("BPFGenDisassemblerTables") {
+  visibility = [ ":Disassembler" ]
+  args = [ "-gen-disassembler" ]
+  td_file = "../BPF.td"
+}
+
+static_library("Disassembler") {
+  output_name = "LLVMBPFDisassembler"
+  deps = [
+    ":BPFGenDisassemblerTables",
+    "//llvm/lib/MC",
+    "//llvm/lib/MC/MCDisassembler",
+    "//llvm/lib/Support",
+    "//llvm/lib/Target/BPF/MCTargetDesc",
+  ]
+  include_dirs = [ ".." ]
+  sources = [
+    "BPFDisassembler.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/llvm/lib/Target/BPF/InstPrinter/BUILD.gn b/utils/gn/secondary/llvm/lib/Target/BPF/InstPrinter/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..30e2004c2820e1afb5bd1ec797bdad91b7eab26c
--- /dev/null
+++ b/utils/gn/secondary/llvm/lib/Target/BPF/InstPrinter/BUILD.gn
@@ -0,0 +1,24 @@
+import("//llvm/utils/TableGen/tablegen.gni")
+
+tablegen("BPFGenAsmWriter") {
+  visibility = [ ":InstPrinter" ]
+  args = [ "-gen-asm-writer" ]
+  td_file = "../BPF.td"
+}
+
+static_library("InstPrinter") {
+  output_name = "LLVMBPFAsmPrinter"
+  deps = [
+    ":BPFGenAsmWriter",
+    "//llvm/lib/MC",
+    "//llvm/lib/Support",
+
+    # MCTargetDesc depends on InstPrinter, so we can't depend on the full
+    # MCTargetDesc target here: it would form a cycle.
+    "//llvm/lib/Target/BPF/MCTargetDesc:tablegen",
+  ]
+  include_dirs = [ ".." ]
+  sources = [
+    "BPFInstPrinter.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/llvm/lib/Target/BPF/MCTargetDesc/BUILD.gn b/utils/gn/secondary/llvm/lib/Target/BPF/MCTargetDesc/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..843e23053d918ce2a84db065dd58e8bc57ef1559
--- /dev/null
+++ b/utils/gn/secondary/llvm/lib/Target/BPF/MCTargetDesc/BUILD.gn
@@ -0,0 +1,60 @@
+import("//llvm/utils/TableGen/tablegen.gni")
+
+tablegen("BPFGenInstrInfo") {
+  visibility = [ ":tablegen" ]
+  args = [ "-gen-instr-info" ]
+  td_file = "../BPF.td"
+}
+
+tablegen("BPFGenMCCodeEmitter") {
+  visibility = [ ":tablegen" ]
+  args = [ "-gen-emitter" ]
+  td_file = "../BPF.td"
+}
+
+tablegen("BPFGenRegisterInfo") {
+  visibility = [ ":tablegen" ]
+  args = [ "-gen-register-info" ]
+  td_file = "../BPF.td"
+}
+
+tablegen("BPFGenSubtargetInfo") {
+  visibility = [ ":tablegen" ]
+  args = [ "-gen-subtarget" ]
+  td_file = "../BPF.td"
+}
+
+group("tablegen") {
+  visibility = [
+    ":MCTargetDesc",
+    "../InstPrinter",
+    "../TargetInfo",
+  ]
+  public_deps = [
+    ":BPFGenInstrInfo",
+    ":BPFGenMCCodeEmitter",
+    ":BPFGenRegisterInfo",
+    ":BPFGenSubtargetInfo",
+  ]
+}
+
+static_library("MCTargetDesc") {
+  output_name = "LLVMBPFDesc"
+  public_deps = [
+    ":tablegen",
+  ]
+  deps = [
+    "//llvm/lib/MC",
+    "//llvm/lib/MC/MCDisassembler",
+    "//llvm/lib/Support",
+    "//llvm/lib/Target/BPF/InstPrinter",
+    "//llvm/lib/Target/BPF/TargetInfo",
+  ]
+  include_dirs = [ ".." ]
+  sources = [
+    "BPFAsmBackend.cpp",
+    "BPFELFObjectWriter.cpp",
+    "BPFMCCodeEmitter.cpp",
+    "BPFMCTargetDesc.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/llvm/lib/Target/BPF/TargetInfo/BUILD.gn b/utils/gn/secondary/llvm/lib/Target/BPF/TargetInfo/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..1fd264d70f8af24c311b56960e27355112fe2bc4
--- /dev/null
+++ b/utils/gn/secondary/llvm/lib/Target/BPF/TargetInfo/BUILD.gn
@@ -0,0 +1,14 @@
+static_library("TargetInfo") {
+  output_name = "LLVMBPFInfo"
+  deps = [
+    "//llvm/lib/Support",
+
+    # MCTargetDesc depends on TargetInfo, so we can't depend on the full
+    # MCTargetDesc target here: it would form a cycle.
+    "//llvm/lib/Target/BPF/MCTargetDesc:tablegen",
+  ]
+  include_dirs = [ ".." ]
+  sources = [
+    "BPFTargetInfo.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/llvm/lib/Target/PowerPC/AsmParser/BUILD.gn b/utils/gn/secondary/llvm/lib/Target/PowerPC/AsmParser/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..28c8945ca0fe1699de35185b47060e9240c42612
--- /dev/null
+++ b/utils/gn/secondary/llvm/lib/Target/PowerPC/AsmParser/BUILD.gn
@@ -0,0 +1,23 @@
+import("//llvm/utils/TableGen/tablegen.gni")
+
+tablegen("PPCGenAsmMatcher") {
+  visibility = [ ":AsmParser" ]
+  args = [ "-gen-asm-matcher" ]
+  td_file = "../PPC.td"
+}
+
+static_library("AsmParser") {
+  output_name = "LLVMPowerPCAsmParser"
+  deps = [
+    ":PPCGenAsmMatcher",
+    "//llvm/lib/MC",
+    "//llvm/lib/MC/MCParser",
+    "//llvm/lib/Support",
+    "//llvm/lib/Target/PowerPC/MCTargetDesc",
+    "//llvm/lib/Target/PowerPC/TargetInfo",
+  ]
+  include_dirs = [ ".." ]
+  sources = [
+    "PPCAsmParser.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/llvm/lib/Target/PowerPC/BUILD.gn b/utils/gn/secondary/llvm/lib/Target/PowerPC/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..a6d30273e5b5f7e9bdc6d969c2c111812c861d3c
--- /dev/null
+++ b/utils/gn/secondary/llvm/lib/Target/PowerPC/BUILD.gn
@@ -0,0 +1,92 @@
+import("//llvm/utils/TableGen/tablegen.gni")
+
+tablegen("PPCGenCallingConv") {
+  visibility = [ ":LLVMPowerPCCodeGen" ]
+  args = [ "-gen-callingconv" ]
+  td_file = "PPC.td"
+}
+
+tablegen("PPCGenDAGISel") {
+  visibility = [ ":LLVMPowerPCCodeGen" ]
+  args = [ "-gen-dag-isel" ]
+  td_file = "PPC.td"
+}
+
+tablegen("PPCGenFastISel") {
+  visibility = [ ":LLVMPowerPCCodeGen" ]
+  args = [ "-gen-fast-isel" ]
+  td_file = "PPC.td"
+}
+
+static_library("LLVMPowerPCCodeGen") {
+  deps = [
+    ":PPCGenCallingConv",
+    ":PPCGenDAGISel",
+    ":PPCGenFastISel",
+    "InstPrinter",
+    "MCTargetDesc",
+    "TargetInfo",
+    "//llvm/include/llvm/Config:llvm-config",
+    "//llvm/lib/Analysis",
+    "//llvm/lib/CodeGen",
+    "//llvm/lib/CodeGen/AsmPrinter",
+    "//llvm/lib/CodeGen/SelectionDAG",
+    "//llvm/lib/IR",
+    "//llvm/lib/MC",
+    "//llvm/lib/Support",
+    "//llvm/lib/Target",
+    "//llvm/lib/Transforms/Utils",
+  ]
+  include_dirs = [ "." ]
+  sources = [
+    "PPCAsmPrinter.cpp",
+    "PPCBoolRetToInt.cpp",
+    "PPCBranchCoalescing.cpp",
+    "PPCBranchSelector.cpp",
+    "PPCCCState.cpp",
+    "PPCCTRLoops.cpp",
+    "PPCCallingConv.cpp",
+    "PPCEarlyReturn.cpp",
+    "PPCExpandISEL.cpp",
+    "PPCFastISel.cpp",
+    "PPCFrameLowering.cpp",
+    "PPCHazardRecognizers.cpp",
+    "PPCISelDAGToDAG.cpp",
+    "PPCISelLowering.cpp",
+    "PPCInstrInfo.cpp",
+    "PPCLoopPreIncPrep.cpp",
+    "PPCMCInstLower.cpp",
+    "PPCMIPeephole.cpp",
+    "PPCMachineFunctionInfo.cpp",
+    "PPCPreEmitPeephole.cpp",
+    "PPCQPXLoadSplat.cpp",
+    "PPCReduceCRLogicals.cpp",
+    "PPCRegisterInfo.cpp",
+    "PPCSubtarget.cpp",
+    "PPCTLSDynamicCall.cpp",
+    "PPCTOCRegDeps.cpp",
+    "PPCTargetMachine.cpp",
+    "PPCTargetObjectFile.cpp",
+    "PPCTargetTransformInfo.cpp",
+    "PPCVSXCopy.cpp",
+    "PPCVSXFMAMutate.cpp",
+    "PPCVSXSwapRemoval.cpp",
+  ]
+}
+
+# This is a bit different from most build files: Due to this group
+# having the directory's name, "//llvm/lib/Target/PowerPC" will refer to this
+# target, which pulls in the code in this directory *and all subdirectories*.
+# For most other directories, "//llvm/lib/Foo" only pulls in the code directly
+# in "llvm/lib/Foo". The forwarding targets in //llvm/lib/Target expect this
+# different behavior.
+group("PowerPC") {
+  deps = [
+    ":LLVMPowerPCCodeGen",
+    "AsmParser",
+    "Disassembler",
+    "InstPrinter",
+    "MCTargetDesc",
+    "TargetInfo",
+  ]
+}
diff --git a/utils/gn/secondary/llvm/lib/Target/PowerPC/Disassembler/BUILD.gn b/utils/gn/secondary/llvm/lib/Target/PowerPC/Disassembler/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..3b5febf9d8e6c011084ac93599475234bb56a6df
--- /dev/null
+++ b/utils/gn/secondary/llvm/lib/Target/PowerPC/Disassembler/BUILD.gn
@@ -0,0 +1,22 @@
+import("//llvm/utils/TableGen/tablegen.gni")
+
+tablegen("PPCGenDisassemblerTables") {
+  visibility = [ ":Disassembler" ]
+  args = [ "-gen-disassembler" ]
+  td_file = "../PPC.td"
+}
+
+static_library("Disassembler") {
+  output_name = "LLVMPowerPCDisassembler"
+  deps = [
+    ":PPCGenDisassemblerTables",
+    "//llvm/lib/MC/MCDisassembler",
+    "//llvm/lib/Support",
+    "//llvm/lib/Target/PowerPC/MCTargetDesc",
+    "//llvm/lib/Target/PowerPC/TargetInfo",
+  ]
+  include_dirs = [ ".." ]
+  sources = [
+    "PPCDisassembler.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/llvm/lib/Target/PowerPC/InstPrinter/BUILD.gn b/utils/gn/secondary/llvm/lib/Target/PowerPC/InstPrinter/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..8d885d6b365325b068121b68749e8ebff5f0dd49
--- /dev/null
+++ b/utils/gn/secondary/llvm/lib/Target/PowerPC/InstPrinter/BUILD.gn
@@ -0,0 +1,24 @@
+import("//llvm/utils/TableGen/tablegen.gni")
+
+tablegen("PPCGenAsmWriter") {
+  visibility = [ ":InstPrinter" ]
+  args = [ "-gen-asm-writer" ]
+  td_file = "../PPC.td"
+}
+
+static_library("InstPrinter") {
+  output_name = "LLVMPowerPCAsmPrinter"
+  deps = [
+    ":PPCGenAsmWriter",
+    "//llvm/lib/MC",
+    "//llvm/lib/Support",
+
+    # MCTargetDesc depends on InstPrinter, so we can't depend on the full
+    # MCTargetDesc target here: it would form a cycle.
+    "//llvm/lib/Target/PowerPC/MCTargetDesc:tablegen",
+  ]
+  include_dirs = [ ".." ]
+  sources = [
+    "PPCInstPrinter.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/llvm/lib/Target/PowerPC/MCTargetDesc/BUILD.gn b/utils/gn/secondary/llvm/lib/Target/PowerPC/MCTargetDesc/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..4425612a3f1f1f59d7ddfaed008eec8b4ffabfef
--- /dev/null
+++ b/utils/gn/secondary/llvm/lib/Target/PowerPC/MCTargetDesc/BUILD.gn
@@ -0,0 +1,63 @@
+import("//llvm/utils/TableGen/tablegen.gni")
+
+tablegen("PPCGenInstrInfo") {
+  visibility = [ ":tablegen" ]
+  args = [ "-gen-instr-info" ]
+  td_file = "../PPC.td"
+}
+
+tablegen("PPCGenMCCodeEmitter") {
+  visibility = [ ":tablegen" ]
+  args = [ "-gen-emitter" ]
+  td_file = "../PPC.td"
+}
+
+tablegen("PPCGenRegisterInfo") {
+  visibility = [ ":tablegen" ]
+  args = [ "-gen-register-info" ]
+  td_file = "../PPC.td"
+}
+
+tablegen("PPCGenSubtargetInfo") {
+  visibility = [ ":tablegen" ]
+  args = [ "-gen-subtarget" ]
+  td_file = "../PPC.td"
+}
+
+group("tablegen") {
+  visibility = [
+    ":MCTargetDesc",
+    "../InstPrinter",
+    "../TargetInfo",
+  ]
+  public_deps = [
+    ":PPCGenInstrInfo",
+    ":PPCGenMCCodeEmitter",
+    ":PPCGenRegisterInfo",
+    ":PPCGenSubtargetInfo",
+  ]
+}
+
+static_library("MCTargetDesc") {
+  output_name = "LLVMPowerPCDesc"
+  public_deps = [
+    ":tablegen",
+  ]
+  deps = [
+    "//llvm/lib/MC",
+    "//llvm/lib/Support",
+    "//llvm/lib/Target/PowerPC/InstPrinter",
+    "//llvm/lib/Target/PowerPC/TargetInfo",
+  ]
+  include_dirs = [ ".." ]
+  sources = [
+    "PPCAsmBackend.cpp",
+    "PPCELFObjectWriter.cpp",
+    "PPCMCAsmInfo.cpp",
+    "PPCMCCodeEmitter.cpp",
+    "PPCMCExpr.cpp",
+    "PPCMCTargetDesc.cpp",
+    "PPCMachObjectWriter.cpp",
+    "PPCPredicates.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/llvm/lib/Target/PowerPC/TargetInfo/BUILD.gn b/utils/gn/secondary/llvm/lib/Target/PowerPC/TargetInfo/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..c706924563d7c7088af27d4d9273796318a4312c
--- /dev/null
+++ b/utils/gn/secondary/llvm/lib/Target/PowerPC/TargetInfo/BUILD.gn
@@ -0,0 +1,14 @@
+static_library("TargetInfo") {
+  output_name = "LLVMPowerPCInfo"
+  deps = [
+    "//llvm/lib/Support",
+
+    # MCTargetDesc depends on TargetInfo, so we can't depend on the full
+    # MCTargetDesc target here: it would form a cycle.
+    "//llvm/lib/Target/PowerPC/MCTargetDesc:tablegen",
+  ]
+  include_dirs = [ ".." ]
+  sources = [
+    "PowerPCTargetInfo.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/llvm/lib/Target/WebAssembly/AsmParser/BUILD.gn b/utils/gn/secondary/llvm/lib/Target/WebAssembly/AsmParser/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..158f9e97fd847e099d55831fee5d0082cf8031b1
--- /dev/null
+++ b/utils/gn/secondary/llvm/lib/Target/WebAssembly/AsmParser/BUILD.gn
@@ -0,0 +1,23 @@
+import("//llvm/utils/TableGen/tablegen.gni")
+
+tablegen("WebAssemblyGenAsmMatcher") {
+  visibility = [ ":AsmParser" ]
+  args = [ "-gen-asm-matcher" ]
+  td_file = "../WebAssembly.td"
+}
+
+static_library("AsmParser") {
+  output_name = "LLVMWebAssemblyAsmParser"
+  deps = [
+    ":WebAssemblyGenAsmMatcher",
+    "//llvm/lib/MC",
+    "//llvm/lib/MC/MCParser",
+    "//llvm/lib/Support",
+    "//llvm/lib/Target/WebAssembly/MCTargetDesc",
+    "//llvm/lib/Target/WebAssembly/TargetInfo",
+  ]
+  include_dirs = [ ".." ]
+  sources = [
+    "WebAssemblyAsmParser.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/llvm/lib/Target/WebAssembly/BUILD.gn b/utils/gn/secondary/llvm/lib/Target/WebAssembly/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..3c8803c19c6b50f5e7c778312eeca52391b18d8b
--- /dev/null
+++ b/utils/gn/secondary/llvm/lib/Target/WebAssembly/BUILD.gn
@@ -0,0 +1,121 @@
+import("//llvm/utils/TableGen/tablegen.gni")
+
+tablegen("WebAssemblyGenCallingConv") {
+  visibility = [ ":LLVMWebAssemblyCodeGen" ]
+  args = [ "-gen-callingconv" ]
+  td_file = "WebAssembly.td"
+}
+
+tablegen("WebAssemblyGenDAGISel") {
+  visibility = [ ":LLVMWebAssemblyCodeGen" ]
+  args = [ "-gen-dag-isel" ]
+  td_file = "WebAssembly.td"
+}
+
+tablegen("WebAssemblyGenFastISel") {
+  visibility = [ ":LLVMWebAssemblyCodeGen" ]
+  args = [ "-gen-fast-isel" ]
+  td_file = "WebAssembly.td"
+}
+
+tablegen("WebAssemblyGenGlobalISel") {
+  visibility = [ ":LLVMWebAssemblyCodeGen" ]
+  args = [ "-gen-global-isel" ]
+  td_file = "WebAssembly.td"
+}
+
+tablegen("WebAssemblyGenMCPseudoLowering") {
+  visibility = [ ":LLVMWebAssemblyCodeGen" ]
+  args = [ "-gen-pseudo-lowering" ]
+  td_file = "WebAssembly.td"
+}
+
+tablegen("WebAssemblyGenRegisterBank") {
+  visibility = [ ":LLVMWebAssemblyCodeGen" ]
+  args = [ "-gen-register-bank" ]
+  td_file = "WebAssembly.td"
+}
+
+static_library("LLVMWebAssemblyCodeGen") {
+  deps = [
+    ":WebAssemblyGenCallingConv",
+    ":WebAssemblyGenDAGISel",
+    ":WebAssemblyGenFastISel",
+    ":WebAssemblyGenGlobalISel",
+    ":WebAssemblyGenMCPseudoLowering",
+    ":WebAssemblyGenRegisterBank",
+    "InstPrinter",
+    "MCTargetDesc",
+    "TargetInfo",
+    "//llvm/include/llvm/Config:llvm-config",
+    "//llvm/lib/Analysis",
+    "//llvm/lib/CodeGen",
+    "//llvm/lib/CodeGen/AsmPrinter",
+    "//llvm/lib/CodeGen/GlobalISel",
+    "//llvm/lib/CodeGen/SelectionDAG",
+    "//llvm/lib/IR",
+    "//llvm/lib/MC",
+    "//llvm/lib/Support",
+    "//llvm/lib/Target",
+  ]
+  include_dirs = [ "." ]
+  sources = [
+    "WebAssemblyAddMissingPrototypes.cpp",
+    "WebAssemblyArgumentMove.cpp",
+    "WebAssemblyAsmPrinter.cpp",
+    "WebAssemblyCFGSort.cpp",
+    "WebAssemblyCFGStackify.cpp",
+    "WebAssemblyCallIndirectFixup.cpp",
+    "WebAssemblyDebugValueManager.cpp",
+    "WebAssemblyExceptionInfo.cpp",
+    "WebAssemblyExplicitLocals.cpp",
+    "WebAssemblyFastISel.cpp",
+    "WebAssemblyFixFunctionBitcasts.cpp",
+    "WebAssemblyFixIrreducibleControlFlow.cpp",
+    "WebAssemblyFrameLowering.cpp",
+    "WebAssemblyISelDAGToDAG.cpp",
+    "WebAssemblyISelLowering.cpp",
+    "WebAssemblyInstrInfo.cpp",
+    "WebAssemblyLateEHPrepare.cpp",
+    "WebAssemblyLowerBrUnless.cpp",
+    "WebAssemblyLowerEmscriptenEHSjLj.cpp",
+    "WebAssemblyLowerGlobalDtors.cpp",
+    "WebAssemblyMCInstLower.cpp",
+    "WebAssemblyMachineFunctionInfo.cpp",
+    "WebAssemblyMemIntrinsicResults.cpp",
+    "WebAssemblyOptimizeLiveIntervals.cpp",
+    "WebAssemblyOptimizeReturned.cpp",
+    "WebAssemblyPeephole.cpp",
+    "WebAssemblyPrepareForLiveIntervals.cpp",
+    "WebAssemblyRegColoring.cpp",
+    "WebAssemblyRegNumbering.cpp",
+    "WebAssemblyRegStackify.cpp",
+    "WebAssemblyRegisterInfo.cpp",
+    "WebAssemblyReplacePhysRegs.cpp",
+    "WebAssemblyRuntimeLibcallSignatures.cpp",
+    "WebAssemblySelectionDAGInfo.cpp",
+    "WebAssemblySetP2AlignOperands.cpp",
+    "WebAssemblySubtarget.cpp",
+    "WebAssemblyTargetMachine.cpp",
+    "WebAssemblyTargetObjectFile.cpp",
+    "WebAssemblyTargetTransformInfo.cpp",
+    "WebAssemblyUtilities.cpp",
+  ]
+}
+
+# This is a bit different from most build files: Due to this group
+# having the directory's name, "//llvm/lib/Target/AArch64" will refer to this
+# target, which pulls in the code in this directory *and all subdirectories*.
+# For most other directories, "//llvm/lib/Foo" only pulls in the code directly
+# in "llvm/lib/Foo". The forwarding targets in //llvm/lib/Target expect this
+# different behavior.
+group("WebAssembly") {
+  deps = [
+    ":LLVMWebAssemblyCodeGen",
+    "AsmParser",
+    "Disassembler",
+    "InstPrinter",
+    "MCTargetDesc",
+    "TargetInfo",
+  ]
+}
diff --git a/utils/gn/secondary/llvm/lib/Target/WebAssembly/Disassembler/BUILD.gn b/utils/gn/secondary/llvm/lib/Target/WebAssembly/Disassembler/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..44e586dbdd79f4fbf1e0e3909a17ba2a7a7c80aa
--- /dev/null
+++ b/utils/gn/secondary/llvm/lib/Target/WebAssembly/Disassembler/BUILD.gn
@@ -0,0 +1,23 @@
+import("//llvm/utils/TableGen/tablegen.gni")
+
+tablegen("WebAssemblyGenDisassemblerTables") {
+  visibility = [ ":Disassembler" ]
+  args = [ "-gen-disassembler" ]
+  td_file = "../WebAssembly.td"
+}
+
+static_library("Disassembler") {
+  output_name = "LLVMWebAssemblyDisassembler"
+  deps = [
+    ":WebAssemblyGenDisassemblerTables",
+    "//llvm/lib/MC",
+    "//llvm/lib/MC/MCDisassembler",
+    "//llvm/lib/Support",
+    "//llvm/lib/Target/WebAssembly/MCTargetDesc",
+    "//llvm/lib/Target/WebAssembly/TargetInfo",
+  ]
+  include_dirs = [ ".." ]
+  sources = [
+    "WebAssemblyDisassembler.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/llvm/lib/Target/WebAssembly/InstPrinter/BUILD.gn b/utils/gn/secondary/llvm/lib/Target/WebAssembly/InstPrinter/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..e63ea79174e96217e897832bc0764d0483ba39e3
--- /dev/null
+++ b/utils/gn/secondary/llvm/lib/Target/WebAssembly/InstPrinter/BUILD.gn
@@ -0,0 +1,24 @@
+import("//llvm/utils/TableGen/tablegen.gni")
+
+tablegen("WebAssemblyGenAsmWriter") {
+  visibility = [ ":InstPrinter" ]
+  args = [ "-gen-asm-writer" ]
+  td_file = "../WebAssembly.td"
+}
+
+static_library("InstPrinter") {
+  output_name = "LLVMWebAssemblyAsmPrinter"
+  deps = [
+    ":WebAssemblyGenAsmWriter",
+    "//llvm/lib/MC",
+    "//llvm/lib/Support",
+
+    # MCTargetDesc depends on InstPrinter, so we can't depend on the full
+    # MCTargetDesc target here: it would form a cycle.
+    "//llvm/lib/Target/WebAssembly/MCTargetDesc:tablegen",
+  ]
+  include_dirs = [ ".." ]
+  sources = [
+    "WebAssemblyInstPrinter.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/llvm/lib/Target/WebAssembly/MCTargetDesc/BUILD.gn b/utils/gn/secondary/llvm/lib/Target/WebAssembly/MCTargetDesc/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..8d8e6c0a0e908a8ab607617b654ce6c13c7eaf81
--- /dev/null
+++ b/utils/gn/secondary/llvm/lib/Target/WebAssembly/MCTargetDesc/BUILD.gn
@@ -0,0 +1,61 @@
+import("//llvm/utils/TableGen/tablegen.gni")
+
+tablegen("WebAssemblyGenInstrInfo") {
+  visibility = [ ":tablegen" ]
+  args = [ "-gen-instr-info" ]
+  td_file = "../WebAssembly.td"
+}
+
+tablegen("WebAssemblyGenMCCodeEmitter") {
+  visibility = [ ":tablegen" ]
+  args = [ "-gen-emitter" ]
+  td_file = "../WebAssembly.td"
+}
+
+tablegen("WebAssemblyGenRegisterInfo") {
+  visibility = [ ":tablegen" ]
+  args = [ "-gen-register-info" ]
+  td_file = "../WebAssembly.td"
+}
+
+tablegen("WebAssemblyGenSubtargetInfo") {
+  visibility = [ ":tablegen" ]
+  args = [ "-gen-subtarget" ]
+  td_file = "../WebAssembly.td"
+}
+
+group("tablegen") {
+  visibility = [
+    ":MCTargetDesc",
+    "../InstPrinter",
+    "../TargetInfo",
+    "../Utils",
+  ]
+  public_deps = [
+    ":WebAssemblyGenInstrInfo",
+    ":WebAssemblyGenMCCodeEmitter",
+    ":WebAssemblyGenRegisterInfo",
+    ":WebAssemblyGenSubtargetInfo",
+  ]
+}
+static_library("MCTargetDesc") {
+  output_name = "LLVMWebAssemblyDesc"
+  public_deps = [
+    ":tablegen",
+  ]
+  deps = [
+    "//llvm/lib/MC",
+    "//llvm/lib/Support",
+    "//llvm/lib/Target/WebAssembly/InstPrinter",
+    "//llvm/lib/Target/WebAssembly/TargetInfo",
+  ]
+  include_dirs = [ ".." ]
+  sources = [
+    "WebAssemblyAsmBackend.cpp",
+    "WebAssemblyMCAsmInfo.cpp",
+    "WebAssemblyMCCodeEmitter.cpp",
+    "WebAssemblyMCTargetDesc.cpp",
+    "WebAssemblyTargetStreamer.cpp",
+    "WebAssemblyWasmObjectWriter.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/llvm/lib/Target/WebAssembly/TargetInfo/BUILD.gn b/utils/gn/secondary/llvm/lib/Target/WebAssembly/TargetInfo/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..334183bb5d6f0f46ab35aed48cfcd29cdfe86cdd
--- /dev/null
+++ b/utils/gn/secondary/llvm/lib/Target/WebAssembly/TargetInfo/BUILD.gn
@@ -0,0 +1,14 @@
+static_library("TargetInfo") {
+  output_name = "LLVMWebAssemblyInfo"
+  deps = [
+    "//llvm/lib/Support",
+
+    # MCTargetDesc depends on TargetInfo, so we can't depend on the full
+    # MCTargetDesc target here: it would form a cycle.
+    "//llvm/lib/Target/WebAssembly/MCTargetDesc:tablegen",
+  ]
+  include_dirs = [ ".." ]
+  sources = [
+    "WebAssemblyTargetInfo.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/llvm/lib/Target/targets.gni b/utils/gn/secondary/llvm/lib/Target/targets.gni
index 76a3d57210cf96b8c17106a06829e27c5f90596e..dc17b505a8eeb54b3e181cf7a4bfd165df590e99 100644
--- a/utils/gn/secondary/llvm/lib/Target/targets.gni
+++ b/utils/gn/secondary/llvm/lib/Target/targets.gni
@@ -6,30 +6,64 @@ declare_args() {
 }
 
 if (llvm_targets_to_build == "host") {
-  if (host_cpu == "x86" || host_cpu == "x64") {
+  if (host_cpu == "arm64") {
+    llvm_targets_to_build = [ "AArch64" ]
+  } else if (host_cpu == "arm") {
+    llvm_targets_to_build = [ "ARM" ]
+  } else if (host_cpu == "pcc" || host_cpu == "pcc64") {
+    llvm_targets_to_build = [ "PowerPC" ]
+  } else if (host_cpu == "x86" || host_cpu == "x64") {
     llvm_targets_to_build = [ "X86" ]
   } else {
     assert(false, "add your host_cpu above")
   }
 } else if (llvm_targets_to_build == "all") {
   # FIXME: Port the remaining targets.
-  llvm_targets_to_build = [ "X86" ]
+  llvm_targets_to_build = [
+    "AArch64",
+    "ARM",
+    "BPF",
+    "PowerPC",
+    "WebAssembly",
+    "X86",
+  ]
 }
 
 # Validate that llvm_targets_to_build is set to a list of valid targets,
 # and remember which targets are built.
+llvm_build_AArch64 = false
+llvm_build_ARM = false
+llvm_build_BPF = false
+llvm_build_PowerPC = false
+llvm_build_WebAssembly = false
 llvm_build_X86 = false
 foreach(target, llvm_targets_to_build) {
-  if (target == "X86") {
+  if (target == "AArch64") {
+    llvm_build_AArch64 = true
+  } else if (target == "ARM") {
+    llvm_build_ARM = true
+  } else if (target == "BPF") {
+    llvm_build_BPF = true
+  } else if (target == "PowerPC") {
+    llvm_build_PowerPC = true
+  } else if (target == "WebAssembly") {
+    llvm_build_WebAssembly = true
+  } else if (target == "X86") {
     llvm_build_X86 = true
   } else {
-    #FIXME : Port the remaining targets.
+    # FIXME: Port the remaining targets.
     assert(false, "Unknown target '$target'.")
   }
 }
 
 # FIXME: This should be based off target_cpu once cross compiles work.
-if (host_cpu == "x86" || host_cpu == "x64") {
+if (host_cpu == "arm64") {
+  native_target = "AArch64"
+} else if (host_cpu == "arm") {
+  native_target = "ARM"
+} else if (host_cpu == "pcc" || host_cpu == "pcc64") {
+  native_target = [ "PowerPC" ]
+} else if (host_cpu == "x86" || host_cpu == "x64") {
   native_target = "X86"
 } else {
   assert(false, "Unsuppored host_cpu '$host_cpu'.")
diff --git a/utils/gn/secondary/llvm/lib/Target/targets_string.gni b/utils/gn/secondary/llvm/lib/Target/targets_string.gni
new file mode 100644
index 0000000000000000000000000000000000000000..dcf8c68d62bc09ec1243289ba776a39fdbd8aa19
--- /dev/null
+++ b/utils/gn/secondary/llvm/lib/Target/targets_string.gni
@@ -0,0 +1,10 @@
+import("//llvm/lib/Target/targets.gni")
+
+# A single string with all built targets, separated by spaces.
+llvm_targets_to_build_string = ""
+foreach(target, llvm_targets_to_build) {
+  if (llvm_targets_to_build_string != "") {
+    llvm_targets_to_build_string += " "
+  }
+  llvm_targets_to_build_string += target
+}
diff --git a/utils/gn/secondary/llvm/lib/Testing/Support/BUILD.gn b/utils/gn/secondary/llvm/lib/Testing/Support/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..064d5ec2c86df5888e7a63252262346bdc6b5bc6
--- /dev/null
+++ b/utils/gn/secondary/llvm/lib/Testing/Support/BUILD.gn
@@ -0,0 +1,12 @@
+static_library("Support") {
+  output_name = "LLVMTestingSupport"
+  deps = [
+    "//llvm/lib/Support",
+    "//llvm/utils/unittest:gtest",
+  ]
+  sources = [
+    "Error.cpp",
+    "SupportHelpers.cpp",
+  ]
+  testonly = true
+}
diff --git a/utils/gn/secondary/llvm/lib/TextAPI/BUILD.gn b/utils/gn/secondary/llvm/lib/TextAPI/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..0daf58b598cb73c14433f24c8d0585d51cfb3a34
--- /dev/null
+++ b/utils/gn/secondary/llvm/lib/TextAPI/BUILD.gn
@@ -0,0 +1,12 @@
+static_library("TextAPI") {
+  output_name = "LLVMTextAPI"
+  deps = [
+    "//llvm/lib/BinaryFormat",
+    "//llvm/lib/Support",
+  ]
+  include_dirs = [ "." ]
+  sources = [
+    "ELF/ELFStub.cpp",
+    "ELF/TBEHandler.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/llvm/lib/Transforms/Hello/BUILD.gn b/utils/gn/secondary/llvm/lib/Transforms/Hello/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..9e5557840c080f4afc52e264d72fc9d46ba8f346
--- /dev/null
+++ b/utils/gn/secondary/llvm/lib/Transforms/Hello/BUILD.gn
@@ -0,0 +1,22 @@
+assert(host_os != "win", "loadable modules not supported on win")
+
+loadable_module("Hello") {
+  output_name = "LLVMHello"
+  deps = [
+    # LLVMHello doesn't want to link in any LLVM code, it just
+    # needs its headers.
+    "//llvm/include/llvm/IR:public_tablegen",
+  ]
+  sources = [
+    "Hello.cpp",
+  ]
+
+  if (host_os != "mac" && host_os != "win") {
+    # The GN build currently doesn't globally pass -fPIC, but that's
+    # needed for building .so files on ELF.  Just pass it manually
+    # for loadable_modules for now.
+    cflags = [ "-fPIC" ]
+  }
+
+  # FIXME: Use Hello.exports to remove all exports.
+}
diff --git a/utils/gn/secondary/llvm/lib/Transforms/InstCombine/BUILD.gn b/utils/gn/secondary/llvm/lib/Transforms/InstCombine/BUILD.gn
index 0082528307b4939e83b93c5b2e2f6163f2231d41..d68a00712720ac8699d6c7d6d030b7f33398368a 100644
--- a/utils/gn/secondary/llvm/lib/Transforms/InstCombine/BUILD.gn
+++ b/utils/gn/secondary/llvm/lib/Transforms/InstCombine/BUILD.gn
@@ -17,6 +17,7 @@ static_library("InstCombine") {
   sources = [
     "InstCombineAddSub.cpp",
     "InstCombineAndOrXor.cpp",
+    "InstCombineAtomicRMW.cpp",
     "InstCombineCalls.cpp",
     "InstCombineCasts.cpp",
     "InstCombineCompares.cpp",
diff --git a/utils/gn/secondary/llvm/lib/Transforms/Scalar/BUILD.gn b/utils/gn/secondary/llvm/lib/Transforms/Scalar/BUILD.gn
index 3d000fa846b1c3a00855413fdf00b4fe6d8d7768..d0e79ca94bf42cbdf130c3bbfff18075688d49bd 100644
--- a/utils/gn/secondary/llvm/lib/Transforms/Scalar/BUILD.gn
+++ b/utils/gn/secondary/llvm/lib/Transforms/Scalar/BUILD.gn
@@ -56,6 +56,7 @@ static_library("Scalar") {
     "LowerAtomic.cpp",
     "LowerExpectIntrinsic.cpp",
     "LowerGuardIntrinsic.cpp",
+    "LowerWidenableCondition.cpp",
     "MakeGuardsExplicit.cpp",
     "MemCpyOptimizer.cpp",
     "MergeICmps.cpp",
diff --git a/utils/gn/secondary/llvm/lib/Transforms/Utils/BUILD.gn b/utils/gn/secondary/llvm/lib/Transforms/Utils/BUILD.gn
index 843dd78e8d7f1b9ed8270313f3afbe96d790130b..eee9b8071bddcfd55f42b31ef8f94d4c4bfbfb2b 100644
--- a/utils/gn/secondary/llvm/lib/Transforms/Utils/BUILD.gn
+++ b/utils/gn/secondary/llvm/lib/Transforms/Utils/BUILD.gn
@@ -13,6 +13,7 @@ static_library("Utils") {
     "BuildLibCalls.cpp",
     "BypassSlowDivision.cpp",
     "CallPromotionUtils.cpp",
+    "CanonicalizeAliases.cpp",
     "CloneFunction.cpp",
     "CloneModule.cpp",
     "CodeExtractor.cpp",
diff --git a/utils/gn/secondary/llvm/lib/Transforms/Vectorize/BUILD.gn b/utils/gn/secondary/llvm/lib/Transforms/Vectorize/BUILD.gn
index 24d52bac1cf8bb1524b36c1f08c2d48020cd9bef..a308fce046987091098922fcb75778271716cb07 100644
--- a/utils/gn/secondary/llvm/lib/Transforms/Vectorize/BUILD.gn
+++ b/utils/gn/secondary/llvm/lib/Transforms/Vectorize/BUILD.gn
@@ -14,6 +14,7 @@ static_library("Vectorize") {
     "VPlan.cpp",
     "VPlanHCFGBuilder.cpp",
     "VPlanHCFGTransforms.cpp",
+    "VPlanPredicator.cpp",
     "VPlanSLP.cpp",
     "VPlanVerifier.cpp",
     "Vectorize.cpp",
diff --git a/utils/gn/secondary/llvm/lib/XRay/BUILD.gn b/utils/gn/secondary/llvm/lib/XRay/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..c7907e3ff74aad9d3cf1441afe3c6a04324bb6f3
--- /dev/null
+++ b/utils/gn/secondary/llvm/lib/XRay/BUILD.gn
@@ -0,0 +1,23 @@
+static_library("XRay") {
+  output_name = "LLVMXRay"
+  deps = [
+    "//llvm/lib/Object",
+    "//llvm/lib/Support",
+  ]
+  sources = [
+    "BlockIndexer.cpp",
+    "BlockPrinter.cpp",
+    "BlockVerifier.cpp",
+    "FDRRecordProducer.cpp",
+    "FDRRecords.cpp",
+    "FDRTraceExpander.cpp",
+    "FDRTraceWriter.cpp",
+    "FileHeaderReader.cpp",
+    "InstrumentationMap.cpp",
+    "LogBuilderConsumer.cpp",
+    "Profile.cpp",
+    "RecordInitializer.cpp",
+    "RecordPrinter.cpp",
+    "Trace.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/llvm/test/BUILD.gn b/utils/gn/secondary/llvm/test/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..0dc292e4ecd4afa61e0fc3cc26b8b978b234e126
--- /dev/null
+++ b/utils/gn/secondary/llvm/test/BUILD.gn
@@ -0,0 +1,292 @@
+import("//llvm/lib/DebugInfo/PDB/enable_dia.gni")
+import("//llvm/lib/Target/targets_string.gni")
+import("//llvm/triples.gni")
+import("//llvm/utils/gn/build/buildflags.gni")
+import("//llvm/utils/gn/build/libs/pthread/enable.gni")
+import("//llvm/utils/gn/build/libs/xar/enable.gni")
+import("//llvm/utils/gn/build/libs/xml/enable.gni")
+import("//llvm/utils/gn/build/libs/zlib/enable.gni")
+import("//llvm/utils/gn/build/write_cmake_config.gni")
+import("llvm_lit_site_cfg_files.gni")
+
+template("write_lit_config") {
+  write_cmake_config(target_name) {
+    input = invoker.input
+    output = invoker.output
+    values = [
+      "LIT_SITE_CFG_IN_HEADER=## Autogenerated from $input, do not edit",
+      "ENABLE_SHARED=0",
+      "LLVM_BINARY_DIR=" +
+          rebase_path(get_label_info("//llvm", "target_out_dir")),
+      "LLVM_SOURCE_DIR=" + rebase_path("//llvm"),
+      "LLVM_TOOLS_DIR=" + rebase_path("$root_out_dir/bin"),
+    ]
+    if (host_os == "win") {
+      # See comment for Windows solink in llvm/utils/gn/build/toolchain/BUILD.gn
+      values += [ "SHLIBDIR=" + rebase_path("$root_out_dir/bin") ]
+    } else {
+      values += [ "SHLIBDIR=" + rebase_path("$root_out_dir/lib") ]
+    }
+    values += invoker.extra_values
+  }
+}
+
+write_lit_config("lit_site_cfg") {
+  input = "//llvm/test/lit.site.cfg.py.in"
+  output = llvm_lit_site_cfg_file
+  extra_values = [
+    "BUILD_SHARED_LIBS=0",
+
+    # Only used by the Go bindings tests, or if LLVM_USE_SANITIZER includes
+    # asan and the host OS is macOS. The GN build currently never uses
+    # LLVM_USE_SANITIZER.  (See also CMAKE_CXX_COMPILER in clang/test/BUILD.gn.)
+    "HOST_CC=cc",
+
+    # Only used by the Go bindings tests, or if LLVM_USE_SANITIZER includes
+    # lsan and the host OS is macOS. The GN build currently never uses
+    # LLVM_USE_SANITIZER.  (See also CMAKE_CXX_COMPILER in clang/test/BUILD.gn.)
+    "HOST_CXX=c++",
+
+    # Only used by the Go bindings tests, and for detecting a 32-bit build
+    # and in a belt-and-suspenders check for detecting 32-bit host builds.
+    # (That check also checks LLVM_HOST_TRIPLE.)
+    "HOST_LDFLAGS=",
+
+    "LLVM_ENABLE_FFI=0",
+    "LLVM_HAVE_OPT_VIEWER_MODULES=0",
+    "LLVM_HOST_TRIPLE=$llvm_current_triple",
+    "LLVM_LIBRARY_DIR=" + rebase_path("$root_out_dir/lib"),
+    "LLVM_LINK_LLVM_DYLIB=0",
+    "LLVM_LIT_TOOLS_DIR=",  # Intentionally empty, matches cmake build.
+    "LLVM_NATIVE_ARCH=$native_target",
+    "LLVM_USE_INTEL_JITEVENTS=0",
+    "LLVM_USE_SANITIZER=",
+    "PYTHON_EXECUTABLE=$python_path",
+    "TARGETS_TO_BUILD=$llvm_targets_to_build_string",
+    "TARGET_TRIPLE=$llvm_target_triple",
+
+    # No bindings are implemented in the GN build.
+    "LLVM_BINDINGS=",
+
+    "GO_EXECUTABLE=",
+    "LLVM_INCLUDE_GO_TESTS=0",
+
+    "HAVE_OCAMLOPT=0",
+    "HAVE_OCAML_OUNIT=0",
+    "OCAMLFIND=OCAMLFIND-NOTFOUND",
+    "OCAMLFLAGS=",
+  ]
+
+  if (host_cpu == "x64") {
+    extra_values += [ "HOST_ARCH=x86_64" ]
+  } else {
+    assert(false, "unimplemented host_cpu " + host_cpu)
+  }
+
+  if (host_os == "win") {
+    extra_values += [ "EXEEXT=.exe" ]
+  } else {
+    extra_values += [ "EXEEXT=" ]
+  }
+
+  if (host_os == "mac") {
+    extra_values += [ "SHLIBEXT=.dylib" ]
+  } else if (host_os == "win") {
+    extra_values += [ "SHLIBEXT=.dll" ]
+  } else {
+    extra_values += [ "SHLIBEXT=.so" ]
+  }
+
+  if (host_os == "freebsd") {
+    extra_values += [ "HOST_OS=FreeBSD" ]
+  } else if (host_os == "mac") {
+    extra_values += [ "HOST_OS=Darwin" ]
+  } else if (host_os == "linux") {
+    extra_values += [ "HOST_OS=Linux" ]
+  } else if (host_os == "win") {
+    extra_values += [ "HOST_OS=Windows" ]
+  } else {
+    assert(false, "unsupported host_os " + host_os)
+  }
+  if (host_os == "linux") {
+    # lit.cfg.py's have_ld_plugin_support() checks for "gold" in --version,
+    # so just claim that ld is gold on Linux.  The function also checks if
+    # LLVMgold.so exists, but since that target isn't hooked up yet in the GN
+    # build the LLVMgold.so tests currently don't run anywhere in the GN build.
+    extra_values += [ "GOLD_EXECUTABLE=ld" ]
+  } else {
+    extra_values += [ "GOLD_EXECUTABLE=" ]
+  }
+  if (host_os == "mac") {
+    extra_values += [ "LD64_EXECUTABLE=ld" ]
+  } else {
+    extra_values += [ "LD64_EXECUTABLE=" ]
+  }
+
+  if (llvm_enable_assertions) {
+    extra_values += [ "ENABLE_ASSERTIONS=1" ]
+  } else {
+    extra_values += [ "ENABLE_ASSERTIONS=0" ]  # Must be 0.
+  }
+
+  if (llvm_enable_libxar) {
+    extra_values += [ "HAVE_LIBXAR=1" ]
+  } else {
+    extra_values += [ "HAVE_LIBXAR=0" ]  # Must be 0.
+  }
+
+  if (llvm_enable_dia_sdk) {
+    extra_values += [ "LLVM_ENABLE_DIA_SDK=1" ]
+  } else {
+    extra_values += [ "LLVM_ENABLE_DIA_SDK=0" ]  # Must be 0.
+  }
+
+  if (llvm_enable_libxml2) {
+    extra_values += [ "LLVM_LIBXML2_ENABLED=1" ]
+  } else {
+    extra_values += [ "LLVM_LIBXML2_ENABLED=0" ]  # Must be 0.
+  }
+
+  if (llvm_enable_threads) {
+    extra_values += [ "LLVM_ENABLE_THREADS=1" ]
+  } else {
+    extra_values += [ "LLVM_ENABLE_THREADS=0" ]  # Must be 0.
+  }
+
+  if (llvm_enable_zlib) {
+    extra_values += [ "HAVE_LIBZ=1" ]
+  } else {
+    extra_values += [ "HAVE_LIBZ=0" ]  # Must be 0.
+  }
+}
+
+write_lit_config("lit_unit_site_cfg") {
+  input = "//llvm/test/Unit/lit.site.cfg.py.in"
+  output = llvm_lit_unit_site_cfg_file
+  extra_values = [ "LLVM_BUILD_MODE=." ]
+}
+
+# This target should contain all dependencies of check-llvm.
+# //:default depends on it, so that ninja's default target builds all
+# prerequisites for check-llvm but doesn't run check-llvm itself.
+group("test") {
+  deps = [
+    ":lit_site_cfg",
+    ":lit_unit_site_cfg",
+
+    # Because llvm/tools/llvm-config/BUILD.gn calls llvm-build to generate
+    # LibraryDependencies.inc, llvm-config expects these libraries to exist
+    # even though nothing but unittests depends on them.  Add explicit
+    # dependencies to make sure the libaries exist on disk when llvm-config's
+    # lit tests run.
+    "//llvm/lib/LineEditor",
+    "//llvm/lib/Testing/Support",
+    "//llvm/tools/bugpoint",
+    "//llvm/tools/dsymutil",
+    "//llvm/tools/llc",
+    "//llvm/tools/lli",
+    "//llvm/tools/lli/ChildTarget:lli-child-target",
+    "//llvm/tools/llvm-ar:symlinks",
+    "//llvm/tools/llvm-as",
+    "//llvm/tools/llvm-bcanalyzer",
+    "//llvm/tools/llvm-c-test",
+    "//llvm/tools/llvm-cat",
+    "//llvm/tools/llvm-cfi-verify",
+    "//llvm/tools/llvm-cov",
+    "//llvm/tools/llvm-cvtres",
+    "//llvm/tools/llvm-cxxdump",
+    "//llvm/tools/llvm-cxxfilt",
+    "//llvm/tools/llvm-cxxmap",
+    "//llvm/tools/llvm-diff",
+    "//llvm/tools/llvm-dis",
+    "//llvm/tools/llvm-dwarfdump",
+    "//llvm/tools/llvm-dwp",
+    "//llvm/tools/llvm-elfabi",
+    "//llvm/tools/llvm-exegesis",
+    "//llvm/tools/llvm-extract",
+    "//llvm/tools/llvm-isel-fuzzer",
+    "//llvm/tools/llvm-link",
+    "//llvm/tools/llvm-lto",
+    "//llvm/tools/llvm-lto2",
+    "//llvm/tools/llvm-mc",
+    "//llvm/tools/llvm-mca",
+    "//llvm/tools/llvm-modextract",
+    "//llvm/tools/llvm-mt",
+    "//llvm/tools/llvm-nm",
+    "//llvm/tools/llvm-objcopy:symlinks",
+    "//llvm/tools/llvm-objdump",
+    "//llvm/tools/llvm-opt-fuzzer",
+    "//llvm/tools/llvm-opt-report",
+    "//llvm/tools/llvm-pdbutil",
+    "//llvm/tools/llvm-profdata",
+    "//llvm/tools/llvm-rc",
+    "//llvm/tools/llvm-readobj:symlinks",
+    "//llvm/tools/llvm-rtdyld",
+    "//llvm/tools/llvm-size",
+    "//llvm/tools/llvm-split",
+    "//llvm/tools/llvm-strings",
+    "//llvm/tools/llvm-symbolizer",
+    "//llvm/tools/llvm-undname",
+    "//llvm/tools/llvm-xray",
+    "//llvm/tools/lto",
+    "//llvm/tools/obj2yaml",
+    "//llvm/tools/opt",
+    "//llvm/tools/sancov",
+    "//llvm/tools/sanstats",
+    "//llvm/tools/verify-uselistorder",
+    "//llvm/tools/yaml2obj",
+    "//llvm/unittests",
+    "//llvm/utils/FileCheck",
+    "//llvm/utils/TableGen:llvm-tblgen",
+    "//llvm/utils/count",
+    "//llvm/utils/llvm-lit",
+    "//llvm/utils/not",
+
+    # llvm-config wants libgtest_main.a to exist at runtime when run as in
+    # its tests, but nothing in the tree depends on them.
+    "//llvm/utils/unittest/UnitTestMain:gtest_main",
+    "//llvm/utils/yaml-bench",
+  ]
+  if (host_os != "win") {
+    # loadable_modules don't work on Windows.
+    # FIXME: In the CMake build, ENABLE_SHARED makes them work somehow
+    # (but they're off by default there too).
+    deps += [
+      "//llvm/lib/Transforms/Hello",
+      "//llvm/tools/bugpoint-passes",
+    ]
+  }
+
+  # FIXME: llvm_build_examples
+  testonly = true
+}
+
+action("check-llvm") {
+  script = "$root_out_dir/bin/llvm-lit"
+  if (host_os == "win") {
+    script += ".py"
+  }
+  args = [
+    "-sv",
+    "--param",
+    "llvm_site_config=" + rebase_path(llvm_lit_site_cfg_file, root_out_dir),
+    "--param",
+    "llvm_unit_site_config=" +
+        rebase_path(llvm_lit_unit_site_cfg_file, root_out_dir),
+    rebase_path(".", root_out_dir),
+  ]
+  outputs = [
+    "$target_gen_dir/run-lit",  # Non-existing, so that ninja runs it each time.
+  ]
+
+  # Since check-llvm is always dirty, //:default doesn't depend on it so that
+  # it's not part of the default ninja target.  Hence, check-llvm shouldn't
+  # have any deps except :test. so that the default target is sure to build
+  # all the deps.
+  deps = [
+    ":test",
+  ]
+  testonly = true
+
+  pool = "//:console"
+}
diff --git a/utils/gn/secondary/llvm/test/llvm_lit_site_cfg_files.gni b/utils/gn/secondary/llvm/test/llvm_lit_site_cfg_files.gni
new file mode 100644
index 0000000000000000000000000000000000000000..b0f5aca7f7fcda83f373c59db38c711f79249c2e
--- /dev/null
+++ b/utils/gn/secondary/llvm/test/llvm_lit_site_cfg_files.gni
@@ -0,0 +1,2 @@
+llvm_lit_site_cfg_file = "$root_gen_dir/llvm/test/lit.site.cfg.py"
+llvm_lit_unit_site_cfg_file = "$root_gen_dir/llvm/test/Unit/lit.site.cfg.py"
diff --git a/utils/gn/secondary/llvm/tools/bugpoint-passes/BUILD.gn b/utils/gn/secondary/llvm/tools/bugpoint-passes/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..260c3909b600603410769ace3ee4972adc8d66f8
--- /dev/null
+++ b/utils/gn/secondary/llvm/tools/bugpoint-passes/BUILD.gn
@@ -0,0 +1,22 @@
+assert(host_os != "win", "loadable modules not supported on win")
+
+loadable_module("bugpoint-passes") {
+  output_name = "BugpointPasses"
+  deps = [
+    # BugpointPasses doesn't want to link in any LLVM code, it just
+    # needs its headers.
+    "//llvm/include/llvm/IR:public_tablegen",
+  ]
+  sources = [
+    "TestPasses.cpp",
+  ]
+
+  if (host_os != "mac" && host_os != "win") {
+    # The GN build currently doesn't globally pass -fPIC, but that's
+    # needed for building .so files on ELF.  Just pass it manually
+    # for loadable_modules for now.
+    cflags = [ "-fPIC" ]
+  }
+
+  # FIXME: Use bugpoint.exports to remove all exports.
+}
diff --git a/utils/gn/secondary/llvm/tools/bugpoint/BUILD.gn b/utils/gn/secondary/llvm/tools/bugpoint/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..d14aec2f72aa9e5cc24c22721673e7cb9a505410
--- /dev/null
+++ b/utils/gn/secondary/llvm/tools/bugpoint/BUILD.gn
@@ -0,0 +1,41 @@
+executable("bugpoint") {
+  deps = [
+    "//llvm/include/llvm/Config:config",
+    "//llvm/include/llvm/Config:llvm-config",
+    "//llvm/lib/Analysis",
+    "//llvm/lib/Bitcode/Writer",
+    "//llvm/lib/CodeGen",
+    "//llvm/lib/IR",
+    "//llvm/lib/IRReader",
+    "//llvm/lib/Linker",
+    "//llvm/lib/Support",
+    "//llvm/lib/Target",
+    "//llvm/lib/Target:TargetsToBuild",
+    "//llvm/lib/Transforms/AggressiveInstCombine",
+    "//llvm/lib/Transforms/IPO",
+    "//llvm/lib/Transforms/Instrumentation",
+    "//llvm/lib/Transforms/ObjCARC",
+    "//llvm/lib/Transforms/Scalar",
+    "//llvm/lib/Transforms/Utils",
+    "//llvm/lib/Transforms/Vectorize",
+  ]
+  sources = [
+    "BugDriver.cpp",
+    "CrashDebugger.cpp",
+    "ExecutionDriver.cpp",
+    "ExtractFunction.cpp",
+    "FindBugs.cpp",
+    "Miscompilation.cpp",
+    "OptimizerDriver.cpp",
+    "ToolRunner.cpp",
+    "bugpoint.cpp",
+  ]
+
+  # Support plugins.
+  # FIXME: Disable dead stripping once other binaries are dead-stripped.
+  if (host_os != "mac" && host_os != "win") {
+    # Make sure bugpoint plugins can access bugpoint's symbols.
+    # Corresponds to export_executable_symbols() in cmake.
+    ldflags = [ "-rdynamic" ]
+  }
+}
diff --git a/utils/gn/secondary/llvm/tools/dsymutil/BUILD.gn b/utils/gn/secondary/llvm/tools/dsymutil/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..721b4c4be11508dad6b039cdd7d636a2911f0fc6
--- /dev/null
+++ b/utils/gn/secondary/llvm/tools/dsymutil/BUILD.gn
@@ -0,0 +1,28 @@
+executable("dsymutil") {
+  deps = [
+    "//llvm/lib/CodeGen/AsmPrinter",
+    "//llvm/lib/DebugInfo/DWARF",
+    "//llvm/lib/MC",
+    "//llvm/lib/Object",
+    "//llvm/lib/Support",
+    "//llvm/lib/Target",
+    "//llvm/lib/Target:TargetsToBuild",
+  ]
+  sources = [
+    "BinaryHolder.cpp",
+    "CFBundle.cpp",
+    "CompileUnit.cpp",
+    "DebugMap.cpp",
+    "DeclContext.cpp",
+    "DwarfLinker.cpp",
+    "DwarfStreamer.cpp",
+    "MachODebugMapParser.cpp",
+    "MachOUtils.cpp",
+    "NonRelocatableStringpool.cpp",
+    "SymbolMap.cpp",
+    "dsymutil.cpp",
+  ]
+  if (host_os == "mac") {
+    libs = [ "CoreFoundation.framework" ]
+  }
+}
diff --git a/utils/gn/secondary/llvm/tools/llc/BUILD.gn b/utils/gn/secondary/llvm/tools/llc/BUILD.gn
index 45cde4a59b285610f0b1f611291cface142a797f..5f7aff10d993e2ddd62832620d82f0442c45c9db 100644
--- a/utils/gn/secondary/llvm/tools/llc/BUILD.gn
+++ b/utils/gn/secondary/llvm/tools/llc/BUILD.gn
@@ -21,7 +21,7 @@ executable("llc") {
 
   # Support plugins.
   # FIXME: Disable dead stripping once other binaries are dead-stripped.
-  if (host_os == "linux") {
+  if (host_os != "mac" && host_os != "win") {
     # Corresponds to export_executable_symbols() in cmake.
     ldflags = [ "-rdynamic" ]
   }
diff --git a/utils/gn/secondary/llvm/tools/lli/BUILD.gn b/utils/gn/secondary/llvm/tools/lli/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..171e0dd87c93dbc3053334b317a52480d9b3a1c3
--- /dev/null
+++ b/utils/gn/secondary/llvm/tools/lli/BUILD.gn
@@ -0,0 +1,29 @@
+executable("lli") {
+  deps = [
+    "//llvm/include/llvm/Config:llvm-config",
+    "//llvm/lib/AsmParser",
+    "//llvm/lib/Bitcode/Reader",
+    "//llvm/lib/CodeGen",
+    "//llvm/lib/CodeGen/SelectionDAG",
+    "//llvm/lib/ExecutionEngine",
+    "//llvm/lib/ExecutionEngine/Interpreter",
+    "//llvm/lib/ExecutionEngine/MCJIT",
+    "//llvm/lib/ExecutionEngine/Orc",
+    "//llvm/lib/ExecutionEngine/RuntimeDyld",
+    "//llvm/lib/IR",
+    "//llvm/lib/IRReader",
+    "//llvm/lib/MC",
+    "//llvm/lib/Support",
+    "//llvm/lib/Target",
+    "//llvm/lib/Target:NativeTarget",
+    "//llvm/lib/Transforms/Instrumentation",
+    "//llvm/lib/Transforms/Utils",
+  ]
+  sources = [
+    "lli.cpp",
+  ]
+  if (host_os != "mac" && host_os != "win") {
+    # Corresponds to export_executable_symbols() in cmake.
+    ldflags = [ "-rdynamic" ]
+  }
+}
diff --git a/utils/gn/secondary/llvm/tools/lli/ChildTarget/BUILD.gn b/utils/gn/secondary/llvm/tools/lli/ChildTarget/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..6203b2a79453bcf8a9d0ce1cfb7d5ac5fa2b1d3b
--- /dev/null
+++ b/utils/gn/secondary/llvm/tools/lli/ChildTarget/BUILD.gn
@@ -0,0 +1,10 @@
+executable("lli-child-target") {
+  deps = [
+    "//llvm/lib/ExecutionEngine/Orc",
+    "//llvm/lib/ExecutionEngine/RuntimeDyld",
+    "//llvm/lib/Support",
+  ]
+  sources = [
+    "ChildTarget.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/llvm/tools/llvm-c-test/BUILD.gn b/utils/gn/secondary/llvm/tools/llvm-c-test/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..1bbe64dd8e783827877f36c6ba2f73cd454fc9d1
--- /dev/null
+++ b/utils/gn/secondary/llvm/tools/llvm-c-test/BUILD.gn
@@ -0,0 +1,32 @@
+executable("llvm-c-test") {
+  deps = [
+    "//llvm/lib/Bitcode/Reader",
+    "//llvm/lib/IR",
+    "//llvm/lib/MC/MCDisassembler",
+    "//llvm/lib/Object",
+    "//llvm/lib/Support",
+    "//llvm/lib/Target",
+    "//llvm/lib/Target:TargetsToBuild",
+  ]
+  if (host_os != "win") {
+    cflags_c = [
+      "-std=gnu99",
+      "-Wstrict-prototypes",
+    ]
+  }
+  sources = [
+    "attributes.c",
+    "calc.c",
+    "debuginfo.c",
+    "diagnostic.c",
+    "disassemble.c",
+    "echo.cpp",
+    "helpers.c",
+    "include-all.c",
+    "main.c",
+    "metadata.c",
+    "module.c",
+    "object.c",
+    "targets.c",
+  ]
+}
diff --git a/utils/gn/secondary/llvm/tools/llvm-cat/BUILD.gn b/utils/gn/secondary/llvm/tools/llvm-cat/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..29c717647b347870f4f297a0860b797a963f8fe3
--- /dev/null
+++ b/utils/gn/secondary/llvm/tools/llvm-cat/BUILD.gn
@@ -0,0 +1,12 @@
+executable("llvm-cat") {
+  deps = [
+    "//llvm/lib/Bitcode/Reader",
+    "//llvm/lib/Bitcode/Writer",
+    "//llvm/lib/IR",
+    "//llvm/lib/IRReader",
+    "//llvm/lib/Support",
+  ]
+  sources = [
+    "llvm-cat.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/llvm/tools/llvm-cfi-verify/BUILD.gn b/utils/gn/secondary/llvm/tools/llvm-cfi-verify/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..e5779aeb29f94afa4f83f5d96b928d384927db3e
--- /dev/null
+++ b/utils/gn/secondary/llvm/tools/llvm-cfi-verify/BUILD.gn
@@ -0,0 +1,18 @@
+executable("llvm-cfi-verify") {
+  deps = [
+    "lib",
+    "//llvm/lib/DebugInfo/Symbolize",
+    "//llvm/lib/MC",
+    "//llvm/lib/MC/MCParser",
+    "//llvm/lib/Object",
+    "//llvm/lib/Support",
+    "//llvm/lib/Target:AllTargetsAsmParsers",
+    "//llvm/lib/Target:AllTargetsAsmPrinters",
+    "//llvm/lib/Target:AllTargetsDescs",
+    "//llvm/lib/Target:AllTargetsDisassemblers",
+    "//llvm/lib/Target:AllTargetsInfos",
+  ]
+  sources = [
+    "llvm-cfi-verify.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/llvm/tools/llvm-cfi-verify/lib/BUILD.gn b/utils/gn/secondary/llvm/tools/llvm-cfi-verify/lib/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..4a1d61c4fccc89d5b72893dfef4705ec7c521b3a
--- /dev/null
+++ b/utils/gn/secondary/llvm/tools/llvm-cfi-verify/lib/BUILD.gn
@@ -0,0 +1,18 @@
+static_library("lib") {
+  output_name = "LLVMCFIVerify"
+  deps = [
+    "//llvm/lib/DebugInfo/DWARF",
+    "//llvm/lib/DebugInfo/Symbolize",
+    "//llvm/lib/MC",
+    "//llvm/lib/MC/MCDisassembler",
+    "//llvm/lib/MC/MCParser",
+    "//llvm/lib/Object",
+    "//llvm/lib/Support",
+  ]
+  sources = [
+    "FileAnalysis.cpp",
+    "FileAnalysis.h",
+    "GraphBuilder.cpp",
+    "GraphBuilder.h",
+  ]
+}
diff --git a/utils/gn/secondary/llvm/tools/llvm-config/BUILD.gn b/utils/gn/secondary/llvm/tools/llvm-config/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..2f662e0ebd022a7e4bd4e2207d97bdbfa3ade50d
--- /dev/null
+++ b/utils/gn/secondary/llvm/tools/llvm-config/BUILD.gn
@@ -0,0 +1,123 @@
+import("//llvm/lib/Target/targets_string.gni")
+import("//llvm/utils/gn/build/buildflags.gni")
+import("//llvm/utils/gn/build/libs/pthread/enable.gni")
+import("//llvm/utils/gn/build/libs/terminfo/enable.gni")
+import("//llvm/utils/gn/build/libs/xml/enable.gni")
+import("//llvm/utils/gn/build/libs/zlib/enable.gni")
+import("//llvm/utils/gn/build/write_cmake_config.gni")
+import("//llvm/version.gni")
+
+write_cmake_config("BuildVariables.inc") {
+  input = "BuildVariables.inc.in"
+  output = "$target_gen_dir/BuildVariables.inc"
+
+  if (is_debug) {
+    build_mode = "debug"
+  } else {
+    build_mode = "release"
+  }
+
+  # FIXME: Why doesn't llvm-config do this, why is this done in
+  # llvm-config/CMakeLists.txt?
+  if (host_os == "win") {
+    l = ""
+    lib = ".lib"
+  } else {
+    l = "-l"
+    lib = ""
+  }
+
+  # Windows doesn't use any of libxml2,terminfo, zlib by default.
+  # Make GN not warn about these variables being unused.
+  not_needed([
+               "l",
+               "lib",
+             ])
+
+  system_libs = ""
+  if (host_os == "win") {
+    # libuuid required for FOLDERID_Profile usage in
+    # lib/Support/Windows/Path.inc.
+    # advapi32 required for CryptAcquireContextW in
+    # lib/Support/Windows/Path.inc
+    system_libs = "psapi.lib shell32.lib ole32.lib uuid.lib advapi32"
+  } else {
+    system_libs += "-lm"
+    if (host_os == "linux") {
+      system_libs += " -lrt -ldl"
+    }
+    if (llvm_enable_threads) {
+      system_libs += " -llibpthreads"
+      if (host_os == "linux") {
+        system_libs += " -latomic"
+      }
+    }
+  }
+  if (llvm_enable_libxml2) {
+    system_libs += " ${l}xml2${lib}"
+  }
+  if (llvm_enable_terminfo) {
+    system_libs += " ${l}ncurses${lib}"
+  }
+  if (llvm_enable_zlib) {
+    system_libs += " ${l}z${lib}"
+  }
+
+  values = [
+    "LLVM_SRC_ROOT=" + rebase_path("//llvm"),
+    "LLVM_OBJ_ROOT=" + rebase_path(root_out_dir),
+
+    # FIXME: Only the bits needed to run LLVM's test are implemented.
+    "LLVM_CPPFLAGS=.",  # FIXME
+    "LLVM_CFLAGS=.",  # FIXME
+    "LLVM_LDFLAGS=.",  # FIXME
+    "LLVM_CXXFLAGS=.",  # FIXME
+    "LLVM_BUILDMODE=$build_mode",
+    "LLVM_LIBDIR_SUFFIX=",
+    "LLVM_TARGETS_BUILT=$llvm_targets_to_build_string",
+    "LLVM_SYSTEM_LIBS=$system_libs",
+    "LLVM_BUILD_SYSTEM=gn",
+    "LLVM_HAS_RTTI=0",
+    "LLVM_BUILD_LLVM_DYLIB=0",
+    "LLVM_LINK_LLVM_DYLIB=0",
+    "BUILD_SHARED_LIBS=0",
+    "LLVM_DYLIB_COMPONENTS=all",
+    "LLVM_DYLIB_VERSION=${llvm_version_major}svn",
+    "LLVM_HAS_GLOBAL_ISEL=1",
+    "LLVM_TOOLS_INSTALL_DIR=",
+  ]
+}
+
+# FIXME: It'd be nice to not depend on llvm-build on this, Depending on all the
+# LLVMBuild.txt files just for this seems a bit overkill.  `gn desc` should
+# have all this information too and could be called at build time.
+# When this is removed, update llvm/test/BUILD.gn to no longer have unnecessary
+# deps on a couple llvm/lib/ targets.
+action("LibraryDependencies.inc") {
+  script = "//llvm/utils/llvm-build/llvm-build"
+  output = "$target_gen_dir/LibraryDependencies.inc"
+  args = [
+    "--native-target=$native_target",
+    "--enable-targets=$llvm_targets_to_build_string",
+    "--write-library-table=" + rebase_path(output, root_out_dir),
+  ]
+  outputs = [
+    output,
+  ]
+}
+
+executable("llvm-config") {
+  deps = [
+    ":BuildVariables.inc",
+    ":LibraryDependencies.inc",
+    "//llvm/include/llvm/Config:config",
+    "//llvm/include/llvm/Config:llvm-config",
+    "//llvm/lib/Support",
+  ]
+
+  # To pick up the generated inc files.
+  include_dirs = [ "$target_gen_dir" ]
+  sources = [
+    "llvm-config.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/llvm/tools/llvm-cov/BUILD.gn b/utils/gn/secondary/llvm/tools/llvm-cov/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..0d4e7669cd1d69bc8c268edf248ce06f3c5b59e3
--- /dev/null
+++ b/utils/gn/secondary/llvm/tools/llvm-cov/BUILD.gn
@@ -0,0 +1,24 @@
+executable("llvm-cov") {
+  deps = [
+    "//llvm/include/llvm/Config:llvm-config",
+    "//llvm/lib/IR",
+    "//llvm/lib/Object",
+    "//llvm/lib/ProfileData",
+    "//llvm/lib/ProfileData/Coverage",
+    "//llvm/lib/Support",
+  ]
+  sources = [
+    "CodeCoverage.cpp",
+    "CoverageExporterJson.cpp",
+    "CoverageExporterLcov.cpp",
+    "CoverageFilters.cpp",
+    "CoverageReport.cpp",
+    "CoverageSummaryInfo.cpp",
+    "SourceCoverageView.cpp",
+    "SourceCoverageViewHTML.cpp",
+    "SourceCoverageViewText.cpp",
+    "TestingSupport.cpp",
+    "gcov.cpp",
+    "llvm-cov.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/llvm/tools/llvm-cvtres/BUILD.gn b/utils/gn/secondary/llvm/tools/llvm-cvtres/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..77d410d2df8dbef6a379de08d8ef89e0604a6ee0
--- /dev/null
+++ b/utils/gn/secondary/llvm/tools/llvm-cvtres/BUILD.gn
@@ -0,0 +1,18 @@
+import("//llvm/utils/TableGen/tablegen.gni")
+
+tablegen("Opts") {
+  visibility = [ ":llvm-cvtres" ]
+  args = [ "-gen-opt-parser-defs" ]
+}
+
+executable("llvm-cvtres") {
+  deps = [
+    ":Opts",
+    "//llvm/lib/Object",
+    "//llvm/lib/Option",
+    "//llvm/lib/Support",
+  ]
+  sources = [
+    "llvm-cvtres.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/llvm/tools/llvm-cxxdump/BUILD.gn b/utils/gn/secondary/llvm/tools/llvm-cxxdump/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..c146958d2d6c011e082170125b17cbda1aee1b55
--- /dev/null
+++ b/utils/gn/secondary/llvm/tools/llvm-cxxdump/BUILD.gn
@@ -0,0 +1,11 @@
+executable("llvm-cxxdump") {
+  deps = [
+    "//llvm/lib/Object",
+    "//llvm/lib/Support",
+    "//llvm/lib/Target:TargetsToBuild",
+  ]
+  sources = [
+    "Error.cpp",
+    "llvm-cxxdump.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/llvm/tools/llvm-cxxfilt/BUILD.gn b/utils/gn/secondary/llvm/tools/llvm-cxxfilt/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..37e7ebd8846d1c904b13c53d79a7828a74fe2152
--- /dev/null
+++ b/utils/gn/secondary/llvm/tools/llvm-cxxfilt/BUILD.gn
@@ -0,0 +1,32 @@
+import("//llvm/tools/binutils_symlinks.gni")
+import("//llvm/utils/gn/build/symlink_or_copy.gni")
+
+if (llvm_install_binutils_symlinks) {
+  symlink_or_copy("cxxfilt") {  # Can't have '+' in target name.
+    deps = [
+      ":llvm-cxxfilt",
+    ]
+    source = "llvm-cxxfilt"
+    output = "$root_out_dir/bin/c++filt"  # Note: c++filt, not cxxfilt
+  }
+}
+
+# //:llvm-cxxfilt depends on this symlink target, see comment in //BUILD.gn.
+group("symlinks") {
+  deps = [
+    ":llvm-cxxfilt",
+  ]
+  if (llvm_install_binutils_symlinks) {
+    deps += [ ":cxxfilt" ]
+  }
+}
+
+executable("llvm-cxxfilt") {
+  deps = [
+    "//llvm/lib/Demangle",
+    "//llvm/lib/Support",
+  ]
+  sources = [
+    "llvm-cxxfilt.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/llvm/tools/llvm-cxxmap/BUILD.gn b/utils/gn/secondary/llvm/tools/llvm-cxxmap/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..7cee4185badef3c1d1e7ec0b5d12159de092ca6c
--- /dev/null
+++ b/utils/gn/secondary/llvm/tools/llvm-cxxmap/BUILD.gn
@@ -0,0 +1,10 @@
+executable("llvm-cxxmap") {
+  deps = [
+    "//llvm/lib/IR",
+    "//llvm/lib/Support",
+    "//llvm/lib/Target:TargetsToBuild",
+  ]
+  sources = [
+    "llvm-cxxmap.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/llvm/tools/llvm-diff/BUILD.gn b/utils/gn/secondary/llvm/tools/llvm-diff/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..5f46d8c61112ff8566c65c8df107d81070ab7dd6
--- /dev/null
+++ b/utils/gn/secondary/llvm/tools/llvm-diff/BUILD.gn
@@ -0,0 +1,13 @@
+executable("llvm-diff") {
+  deps = [
+    "//llvm/lib/IR",
+    "//llvm/lib/IRReader",
+    "//llvm/lib/Support",
+  ]
+  sources = [
+    "DiffConsumer.cpp",
+    "DiffLog.cpp",
+    "DifferenceEngine.cpp",
+    "llvm-diff.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/llvm/tools/llvm-dwp/BUILD.gn b/utils/gn/secondary/llvm/tools/llvm-dwp/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..efb5b827224a0d1324ccfb8fcbd9422f998024e1
--- /dev/null
+++ b/utils/gn/secondary/llvm/tools/llvm-dwp/BUILD.gn
@@ -0,0 +1,38 @@
+import("//llvm/tools/binutils_symlinks.gni")
+import("//llvm/utils/gn/build/symlink_or_copy.gni")
+
+if (llvm_install_binutils_symlinks) {
+  symlink_or_copy("dwp") {
+    deps = [
+      ":llvm-dwp",
+    ]
+    source = "llvm-dwp"
+    output = "$root_out_dir/bin/dwp"
+  }
+}
+
+# //:llvm-dwp depends on this symlink target, see comment in //BUILD.gn.
+group("symlinks") {
+  deps = [
+    ":llvm-dwp",
+  ]
+  if (llvm_install_binutils_symlinks) {
+    deps += [ ":dwp" ]
+  }
+}
+
+executable("llvm-dwp") {
+  deps = [
+    "//llvm/lib/CodeGen/AsmPrinter",
+    "//llvm/lib/DebugInfo/DWARF",
+    "//llvm/lib/MC",
+    "//llvm/lib/Object",
+    "//llvm/lib/Support",
+    "//llvm/lib/Target",
+    "//llvm/lib/Target:TargetsToBuild",
+  ]
+  sources = [
+    "DWPError.cpp",
+    "llvm-dwp.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/llvm/tools/llvm-elfabi/BUILD.gn b/utils/gn/secondary/llvm/tools/llvm-elfabi/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..dd12e20a924e301d24843155e30656313278f29e
--- /dev/null
+++ b/utils/gn/secondary/llvm/tools/llvm-elfabi/BUILD.gn
@@ -0,0 +1,12 @@
+executable("llvm-elfabi") {
+  deps = [
+    "//llvm/lib/Object",
+    "//llvm/lib/Support",
+    "//llvm/lib/TextAPI",
+  ]
+  sources = [
+    "ELFObjHandler.cpp",
+    "ErrorCollector.cpp",
+    "llvm-elfabi.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/llvm/tools/llvm-exegesis/BUILD.gn b/utils/gn/secondary/llvm/tools/llvm-exegesis/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..7b81b7e5fb87810a8cb6ca7c602af7f948d0faed
--- /dev/null
+++ b/utils/gn/secondary/llvm/tools/llvm-exegesis/BUILD.gn
@@ -0,0 +1,15 @@
+executable("llvm-exegesis") {
+  deps = [
+    "lib",
+    "//llvm/lib/CodeGen",
+    "//llvm/lib/ExecutionEngine",
+    "//llvm/lib/ExecutionEngine/MCJIT",
+    "//llvm/lib/MC",
+    "//llvm/lib/Object",
+    "//llvm/lib/Support",
+    "//llvm/lib/Target:NativeTarget",
+  ]
+  sources = [
+    "llvm-exegesis.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/llvm/tools/llvm-exegesis/lib/AArch64/BUILD.gn b/utils/gn/secondary/llvm/tools/llvm-exegesis/lib/AArch64/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..ef280c67ff31f8d06a0d7926238e30a5ed1fafd5
--- /dev/null
+++ b/utils/gn/secondary/llvm/tools/llvm-exegesis/lib/AArch64/BUILD.gn
@@ -0,0 +1,22 @@
+import("//llvm/utils/TableGen/tablegen.gni")
+
+tablegen("AArch64GenExegesis") {
+  args = [ "-gen-exegesis" ]
+  td_file = "//llvm/lib/Target/AArch64/AArch64.td"
+}
+
+static_library("AArch64") {
+  output_name = "LLVMExegesisAArch64"
+  deps = [
+    ":AArch64GenExegesis",
+
+    # Exegesis reaches inside the Target/AArch64 tablegen internals and must
+    # depend on these Target/AArch64-internal build targets.
+    "//llvm/lib/Target/AArch64/MCTargetDesc",
+    "//llvm/lib/Target/AArch64/Utils",
+  ]
+  sources = [
+    "Target.cpp",
+  ]
+  include_dirs = [ "//llvm/lib/Target/AArch64" ]
+}
diff --git a/utils/gn/secondary/llvm/tools/llvm-exegesis/lib/BUILD.gn b/utils/gn/secondary/llvm/tools/llvm-exegesis/lib/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..ef4eea6d12f09a5f8ad5361a6c57c3561499035a
--- /dev/null
+++ b/utils/gn/secondary/llvm/tools/llvm-exegesis/lib/BUILD.gn
@@ -0,0 +1,44 @@
+import("//llvm/lib/Target/targets.gni")
+
+static_library("lib") {
+  output_name = "LLVMExegesis"
+  deps = [
+    "//llvm/lib/Analysis",
+    "//llvm/lib/CodeGen",
+    "//llvm/lib/CodeGen/GlobalISel",
+    "//llvm/lib/ExecutionEngine",
+    "//llvm/lib/ExecutionEngine/MCJIT",
+    "//llvm/lib/IR",
+    "//llvm/lib/MC",
+    "//llvm/lib/Object",
+    "//llvm/lib/ObjectYAML",
+    "//llvm/lib/Support",
+  ]
+  sources = [
+    "Analysis.cpp",
+    "Assembler.cpp",
+    "BenchmarkResult.cpp",
+    "BenchmarkRunner.cpp",
+    "Clustering.cpp",
+    "CodeTemplate.cpp",
+    "Latency.cpp",
+    "LlvmState.cpp",
+    "MCInstrDescView.cpp",
+    "PerfHelper.cpp",
+    "RegisterAliasing.cpp",
+    "RegisterValue.cpp",
+    "SnippetGenerator.cpp",
+    "Target.cpp",
+    "Uops.cpp",
+  ]
+
+  if (llvm_build_AArch64) {
+    deps += [ "AArch64" ]
+  }
+  if (llvm_build_PowerPC) {
+    deps += [ "PowerPC" ]
+  }
+  if (llvm_build_X86) {
+    deps += [ "X86" ]
+  }
+}
diff --git a/utils/gn/secondary/llvm/tools/llvm-exegesis/lib/PowerPC/BUILD.gn b/utils/gn/secondary/llvm/tools/llvm-exegesis/lib/PowerPC/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..4c4af42166f50ee02ed752cf65a4a5b7fc3fb8de
--- /dev/null
+++ b/utils/gn/secondary/llvm/tools/llvm-exegesis/lib/PowerPC/BUILD.gn
@@ -0,0 +1,21 @@
+import("//llvm/utils/TableGen/tablegen.gni")
+
+tablegen("PPCGenExegesis") {
+  args = [ "-gen-exegesis" ]
+  td_file = "//llvm/lib/Target/PowerPC/PPC.td"
+}
+
+static_library("PowerPC") {
+  output_name = "LLVMExegesisPowerPC"
+  deps = [
+    ":PPCGenExegesis",
+
+    # Exegesis reaches inside the Target/PowerPC tablegen internals and must
+    # depend on these Target/PowerPC-internal build targets.
+    "//llvm/lib/Target/PowerPC/MCTargetDesc",
+  ]
+  sources = [
+    "Target.cpp",
+  ]
+  include_dirs = [ "//llvm/lib/Target/PowerPC" ]
+}
diff --git a/utils/gn/secondary/llvm/tools/llvm-exegesis/lib/X86/BUILD.gn b/utils/gn/secondary/llvm/tools/llvm-exegesis/lib/X86/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..a1cb577b7961e21198a2f6daaea40800ad417c50
--- /dev/null
+++ b/utils/gn/secondary/llvm/tools/llvm-exegesis/lib/X86/BUILD.gn
@@ -0,0 +1,21 @@
+import("//llvm/utils/TableGen/tablegen.gni")
+
+tablegen("X86GenExegesis") {
+  args = [ "-gen-exegesis" ]
+  td_file = "//llvm/lib/Target/X86/X86.td"
+}
+
+static_library("X86") {
+  output_name = "LLVMExegesisX86"
+  deps = [
+    ":X86GenExegesis",
+
+    # Exegesis reaches inside the Target/X86 tablegen internals and must
+    # depend on this Target/X86-internal build target.
+    "//llvm/lib/Target/X86/MCTargetDesc",
+  ]
+  sources = [
+    "Target.cpp",
+  ]
+  include_dirs = [ "//llvm/lib/Target/X86" ]
+}
diff --git a/utils/gn/secondary/llvm/tools/llvm-extract/BUILD.gn b/utils/gn/secondary/llvm/tools/llvm-extract/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..c8699a7bd5f2e14eb375a3f82468bb9d91a61452
--- /dev/null
+++ b/utils/gn/secondary/llvm/tools/llvm-extract/BUILD.gn
@@ -0,0 +1,12 @@
+executable("llvm-extract") {
+  deps = [
+    "//llvm/lib/Bitcode/Writer",
+    "//llvm/lib/IR",
+    "//llvm/lib/IRReader",
+    "//llvm/lib/Support",
+    "//llvm/lib/Transforms/IPO",
+  ]
+  sources = [
+    "llvm-extract.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/llvm/tools/llvm-isel-fuzzer/BUILD.gn b/utils/gn/secondary/llvm/tools/llvm-isel-fuzzer/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..d9ef7fcc3155aa61dee1d8a23ed387a6498111d0
--- /dev/null
+++ b/utils/gn/secondary/llvm/tools/llvm-isel-fuzzer/BUILD.gn
@@ -0,0 +1,24 @@
+import("//llvm/utils/gn/build/fuzzer.gni")
+
+fuzzer("llvm-isel-fuzzer") {
+  deps = [
+    "//llvm/lib/Analysis",
+    "//llvm/lib/Bitcode/Reader",
+    "//llvm/lib/Bitcode/Writer",
+    "//llvm/lib/CodeGen",
+    "//llvm/lib/CodeGen/AsmPrinter",
+    "//llvm/lib/CodeGen/SelectionDAG",
+    "//llvm/lib/FuzzMutate",
+    "//llvm/lib/IR",
+    "//llvm/lib/IRReader",
+    "//llvm/lib/MC",
+    "//llvm/lib/Support",
+    "//llvm/lib/Target",
+    "//llvm/lib/Target:TargetsToBuild",
+    "//llvm/lib/Transforms/Scalar",
+  ]
+  dummy_main = "DummyISelFuzzer.cpp"
+  sources = [
+    "llvm-isel-fuzzer.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/llvm/tools/llvm-link/BUILD.gn b/utils/gn/secondary/llvm/tools/llvm-link/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..10d6783798c87e5130b716652519f8751e3f8989
--- /dev/null
+++ b/utils/gn/secondary/llvm/tools/llvm-link/BUILD.gn
@@ -0,0 +1,16 @@
+executable("llvm-link") {
+  deps = [
+    "//llvm/lib/Bitcode/Reader",
+    "//llvm/lib/Bitcode/Writer",
+    "//llvm/lib/IR",
+    "//llvm/lib/IRReader",
+    "//llvm/lib/Linker",
+    "//llvm/lib/Object",
+    "//llvm/lib/Support",
+    "//llvm/lib/Transforms/IPO",
+    "//llvm/lib/Transforms/Utils",
+  ]
+  sources = [
+    "llvm-link.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/llvm/tools/llvm-lto/BUILD.gn b/utils/gn/secondary/llvm/tools/llvm-lto/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..a569fd96db6fb5e7c5c0297c4b6724956382cecf
--- /dev/null
+++ b/utils/gn/secondary/llvm/tools/llvm-lto/BUILD.gn
@@ -0,0 +1,17 @@
+executable("llvm-lto") {
+  deps = [
+    "//llvm/lib/Bitcode/Reader",
+    "//llvm/lib/Bitcode/Writer",
+    "//llvm/lib/IR",
+    "//llvm/lib/IRReader",
+    "//llvm/lib/LTO",
+    "//llvm/lib/MC",
+    "//llvm/lib/Object",
+    "//llvm/lib/Support",
+    "//llvm/lib/Target",
+    "//llvm/lib/Target:TargetsToBuild",
+  ]
+  sources = [
+    "llvm-lto.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/llvm/tools/llvm-lto2/BUILD.gn b/utils/gn/secondary/llvm/tools/llvm-lto2/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..9dfb23143408273e9d2e7d2eaa80070617451d93
--- /dev/null
+++ b/utils/gn/secondary/llvm/tools/llvm-lto2/BUILD.gn
@@ -0,0 +1,16 @@
+executable("llvm-lto2") {
+  deps = [
+    "//llvm/lib/Bitcode/Reader",
+    "//llvm/lib/IR",
+    "//llvm/lib/LTO",
+    "//llvm/lib/Linker",
+    "//llvm/lib/MC",
+    "//llvm/lib/Object",
+    "//llvm/lib/Support",
+    "//llvm/lib/Target",
+    "//llvm/lib/Target:TargetsToBuild",
+  ]
+  sources = [
+    "llvm-lto2.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/llvm/tools/llvm-mca/BUILD.gn b/utils/gn/secondary/llvm/tools/llvm-mca/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..f4b66ebb11a012016e4f40e7a40460389757dff2
--- /dev/null
+++ b/utils/gn/secondary/llvm/tools/llvm-mca/BUILD.gn
@@ -0,0 +1,32 @@
+executable("llvm-mca") {
+  deps = [
+    "//llvm/lib/MC",
+    "//llvm/lib/MC/MCParser",
+    "//llvm/lib/MCA",
+    "//llvm/lib/Support",
+    "//llvm/lib/Target:AllTargetsAsmParsers",
+    "//llvm/lib/Target:AllTargetsAsmPrinters",
+    "//llvm/lib/Target:AllTargetsDescs",
+    "//llvm/lib/Target:AllTargetsDisassemblers",
+    "//llvm/lib/Target:AllTargetsInfos",
+  ]
+  include_dirs = [
+    ".",
+    "include",
+  ]
+  sources = [
+    "CodeRegion.cpp",
+    "CodeRegionGenerator.cpp",
+    "PipelinePrinter.cpp",
+    "Views/DispatchStatistics.cpp",
+    "Views/InstructionInfoView.cpp",
+    "Views/RegisterFileStatistics.cpp",
+    "Views/ResourcePressureView.cpp",
+    "Views/RetireControlUnitStatistics.cpp",
+    "Views/SchedulerStatistics.cpp",
+    "Views/SummaryView.cpp",
+    "Views/TimelineView.cpp",
+    "Views/View.cpp",
+    "llvm-mca.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/llvm/tools/llvm-modextract/BUILD.gn b/utils/gn/secondary/llvm/tools/llvm-modextract/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..a8bf2d307c4910b479e5ffb39e22849fb74f6e0a
--- /dev/null
+++ b/utils/gn/secondary/llvm/tools/llvm-modextract/BUILD.gn
@@ -0,0 +1,12 @@
+executable("llvm-modextract") {
+  deps = [
+    "//llvm/lib/Bitcode/Reader",
+    "//llvm/lib/Bitcode/Writer",
+    "//llvm/lib/IR",
+    "//llvm/lib/IRReader",
+    "//llvm/lib/Support",
+  ]
+  sources = [
+    "llvm-modextract.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/llvm/tools/llvm-mt/BUILD.gn b/utils/gn/secondary/llvm/tools/llvm-mt/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..7f3ea4809eb0b94dc928f016330afd77f68f800f
--- /dev/null
+++ b/utils/gn/secondary/llvm/tools/llvm-mt/BUILD.gn
@@ -0,0 +1,18 @@
+import("//llvm/utils/TableGen/tablegen.gni")
+
+tablegen("Opts") {
+  visibility = [ ":llvm-mt" ]
+  args = [ "-gen-opt-parser-defs" ]
+}
+
+executable("llvm-mt") {
+  deps = [
+    ":Opts",
+    "//llvm/lib/Option",
+    "//llvm/lib/Support",
+    "//llvm/lib/WindowsManifest",
+  ]
+  sources = [
+    "llvm-mt.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/llvm/tools/llvm-nm/BUILD.gn b/utils/gn/secondary/llvm/tools/llvm-nm/BUILD.gn
index b433ef602471a39183c7fa377c2f5f24528e6f18..51279d997191b0d5d5088073513e2c59468c4a77 100644
--- a/utils/gn/secondary/llvm/tools/llvm-nm/BUILD.gn
+++ b/utils/gn/secondary/llvm/tools/llvm-nm/BUILD.gn
@@ -13,10 +13,11 @@ if (llvm_install_binutils_symlinks) {
 
 # //:llvm-nm depends on this symlink target, see comment in //BUILD.gn.
 group("symlinks") {
+  deps = [
+    ":llvm-nm",
+  ]
   if (llvm_install_binutils_symlinks) {
-    deps = [
-      ":nm",
-    ]
+    deps += [ ":nm" ]
   }
 }
 
diff --git a/utils/gn/secondary/llvm/tools/llvm-objcopy/BUILD.gn b/utils/gn/secondary/llvm/tools/llvm-objcopy/BUILD.gn
index 6dd48d6ef8b055bab6415347e7e5f97c5b5551d8..1cbb129fb58a0cdead3487b151af9aec99c49746 100644
--- a/utils/gn/secondary/llvm/tools/llvm-objcopy/BUILD.gn
+++ b/utils/gn/secondary/llvm/tools/llvm-objcopy/BUILD.gn
@@ -49,9 +49,16 @@ executable("llvm-objcopy") {
   include_dirs = [ "." ]
   sources = [
     "Buffer.cpp",
+    "COFF/COFFObjcopy.cpp",
+    "COFF/Object.cpp",
+    "COFF/Reader.cpp",
+    "COFF/Writer.cpp",
     "CopyConfig.cpp",
     "ELF/ELFObjcopy.cpp",
     "ELF/Object.cpp",
+    "MachO/MachOObjcopy.cpp",
+    "MachO/MachOReader.cpp",
+    "MachO/MachOWriter.cpp",
     "llvm-objcopy.cpp",
   ]
 }
diff --git a/utils/gn/secondary/llvm/tools/llvm-objdump/BUILD.gn b/utils/gn/secondary/llvm/tools/llvm-objdump/BUILD.gn
index e8ee2e760c7925b5375300753d6f81cadcbbbbfd..e567a70d0a8b801bba1ea0a437066d461cbdcf7c 100644
--- a/utils/gn/secondary/llvm/tools/llvm-objdump/BUILD.gn
+++ b/utils/gn/secondary/llvm/tools/llvm-objdump/BUILD.gn
@@ -13,10 +13,11 @@ if (llvm_install_binutils_symlinks) {
 
 # //:llvm-nm depends on this symlink target, see comment in //BUILD.gn.
 group("symlinks") {
+  deps = [
+    ":llvm-objdump",
+  ]
   if (llvm_install_binutils_symlinks) {
-    deps = [
-      ":objdump",
-    ]
+    deps += [ ":objdump" ]
   }
 }
 
diff --git a/utils/gn/secondary/llvm/tools/llvm-opt-fuzzer/BUILD.gn b/utils/gn/secondary/llvm/tools/llvm-opt-fuzzer/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..cb95dda81e8174a7e171d0cd66187a3284dca1ec
--- /dev/null
+++ b/utils/gn/secondary/llvm/tools/llvm-opt-fuzzer/BUILD.gn
@@ -0,0 +1,31 @@
+import("//llvm/utils/gn/build/fuzzer.gni")
+
+fuzzer("llvm-opt-fuzzer") {
+  deps = [
+    "//llvm/lib/Analysis",
+    "//llvm/lib/Bitcode/Reader",
+    "//llvm/lib/Bitcode/Writer",
+    "//llvm/lib/CodeGen",
+    "//llvm/lib/FuzzMutate",
+    "//llvm/lib/IR",
+    "//llvm/lib/IRReader",
+    "//llvm/lib/MC",
+    "//llvm/lib/Passes",
+    "//llvm/lib/Support",
+    "//llvm/lib/Target",
+    "//llvm/lib/Target:TargetsToBuild",
+    "//llvm/lib/Transforms/AggressiveInstCombine",
+    "//llvm/lib/Transforms/Coroutines",
+    "//llvm/lib/Transforms/IPO",
+    "//llvm/lib/Transforms/InstCombine",
+    "//llvm/lib/Transforms/Instrumentation",
+    "//llvm/lib/Transforms/ObjCARC",
+    "//llvm/lib/Transforms/Scalar",
+    "//llvm/lib/Transforms/Utils",
+    "//llvm/lib/Transforms/Vectorize",
+  ]
+  dummy_main = "DummyOptFuzzer.cpp"
+  sources = [
+    "llvm-opt-fuzzer.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/llvm/tools/llvm-opt-report/BUILD.gn b/utils/gn/secondary/llvm/tools/llvm-opt-report/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..70b2e1be872f1d906efd1690030dc60d496b6752
--- /dev/null
+++ b/utils/gn/secondary/llvm/tools/llvm-opt-report/BUILD.gn
@@ -0,0 +1,12 @@
+executable("llvm-opt-report") {
+  deps = [
+    "//llvm/lib/Demangle",
+    "//llvm/lib/IR",
+    "//llvm/lib/Object",
+    "//llvm/lib/OptRemarks",
+    "//llvm/lib/Support",
+  ]
+  sources = [
+    "OptReport.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/llvm/tools/llvm-profdata/BUILD.gn b/utils/gn/secondary/llvm/tools/llvm-profdata/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..748acfc8d03c21d6ac92350c0126b75ac469f84f
--- /dev/null
+++ b/utils/gn/secondary/llvm/tools/llvm-profdata/BUILD.gn
@@ -0,0 +1,10 @@
+executable("llvm-profdata") {
+  deps = [
+    "//llvm/lib/IR",
+    "//llvm/lib/ProfileData",
+    "//llvm/lib/Support",
+  ]
+  sources = [
+    "llvm-profdata.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/llvm/tools/llvm-rc/BUILD.gn b/utils/gn/secondary/llvm/tools/llvm-rc/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..d47148a1459aa166ad7f793e46d0aefe9fe8a8b3
--- /dev/null
+++ b/utils/gn/secondary/llvm/tools/llvm-rc/BUILD.gn
@@ -0,0 +1,22 @@
+import("//llvm/utils/TableGen/tablegen.gni")
+
+tablegen("Opts") {
+  visibility = [ ":llvm-rc" ]
+  args = [ "-gen-opt-parser-defs" ]
+}
+
+executable("llvm-rc") {
+  deps = [
+    ":Opts",
+    "//llvm/lib/Option",
+    "//llvm/lib/Support",
+  ]
+  sources = [
+    "ResourceFileWriter.cpp",
+    "ResourceScriptCppFilter.cpp",
+    "ResourceScriptParser.cpp",
+    "ResourceScriptStmt.cpp",
+    "ResourceScriptToken.cpp",
+    "llvm-rc.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/llvm/tools/llvm-rtdyld/BUILD.gn b/utils/gn/secondary/llvm/tools/llvm-rtdyld/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..2c4d8f2a9dc24cc737b2f26d5ed53bf26397768b
--- /dev/null
+++ b/utils/gn/secondary/llvm/tools/llvm-rtdyld/BUILD.gn
@@ -0,0 +1,14 @@
+executable("llvm-rtdyld") {
+  deps = [
+    "//llvm/lib/DebugInfo/DWARF",
+    "//llvm/lib/ExecutionEngine",
+    "//llvm/lib/ExecutionEngine/RuntimeDyld",
+    "//llvm/lib/MC",
+    "//llvm/lib/Object",
+    "//llvm/lib/Support",
+    "//llvm/lib/Target:TargetsToBuild",
+  ]
+  sources = [
+    "llvm-rtdyld.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/llvm/tools/llvm-size/BUILD.gn b/utils/gn/secondary/llvm/tools/llvm-size/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..03bc5447b0779c2484f33a941ed31b3a1e623527
--- /dev/null
+++ b/utils/gn/secondary/llvm/tools/llvm-size/BUILD.gn
@@ -0,0 +1,32 @@
+import("//llvm/tools/binutils_symlinks.gni")
+import("//llvm/utils/gn/build/symlink_or_copy.gni")
+
+if (llvm_install_binutils_symlinks) {
+  symlink_or_copy("size") {
+    deps = [
+      ":llvm-size",
+    ]
+    source = "llvm-size"
+    output = "$root_out_dir/bin/size"
+  }
+}
+
+# //:llvm-size depends on this symlink target, see comment in //BUILD.gn.
+group("symlinks") {
+  deps = [
+    ":llvm-size",
+  ]
+  if (llvm_install_binutils_symlinks) {
+    deps += [ ":size" ]
+  }
+}
+
+executable("llvm-size") {
+  deps = [
+    "//llvm/lib/Object",
+    "//llvm/lib/Support",
+  ]
+  sources = [
+    "llvm-size.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/llvm/tools/llvm-split/BUILD.gn b/utils/gn/secondary/llvm/tools/llvm-split/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..375dce3b682b596876b0b4d02774536cc472e01b
--- /dev/null
+++ b/utils/gn/secondary/llvm/tools/llvm-split/BUILD.gn
@@ -0,0 +1,12 @@
+executable("llvm-split") {
+  deps = [
+    "//llvm/lib/Bitcode/Writer",
+    "//llvm/lib/IR",
+    "//llvm/lib/IRReader",
+    "//llvm/lib/Support",
+    "//llvm/lib/Transforms/Utils",
+  ]
+  sources = [
+    "llvm-split.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/llvm/tools/llvm-strings/BUILD.gn b/utils/gn/secondary/llvm/tools/llvm-strings/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..dddf3a2397be1c52090a2bae31cbb22bb67f06d2
--- /dev/null
+++ b/utils/gn/secondary/llvm/tools/llvm-strings/BUILD.gn
@@ -0,0 +1,33 @@
+import("//llvm/tools/binutils_symlinks.gni")
+import("//llvm/utils/gn/build/symlink_or_copy.gni")
+
+if (llvm_install_binutils_symlinks) {
+  symlink_or_copy("strings") {
+    deps = [
+      ":llvm-strings",
+    ]
+    source = "llvm-strings"
+    output = "$root_out_dir/bin/strings"
+  }
+}
+
+# //:llvm-strings depends on this symlink target, see comment in //BUILD.gn.
+group("symlinks") {
+  deps = [
+    ":llvm-strings",
+  ]
+  if (llvm_install_binutils_symlinks) {
+    deps += [ ":strings" ]
+  }
+}
+
+executable("llvm-strings") {
+  deps = [
+    "//llvm/lib/IR",
+    "//llvm/lib/Object",
+    "//llvm/lib/Support",
+  ]
+  sources = [
+    "llvm-strings.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/llvm/tools/llvm-symbolizer/BUILD.gn b/utils/gn/secondary/llvm/tools/llvm-symbolizer/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..497d2c8284bed06af325d6ad101ca4967d04c9f9
--- /dev/null
+++ b/utils/gn/secondary/llvm/tools/llvm-symbolizer/BUILD.gn
@@ -0,0 +1,36 @@
+import("//llvm/tools/binutils_symlinks.gni")
+import("//llvm/utils/gn/build/symlink_or_copy.gni")
+
+if (llvm_install_binutils_symlinks) {
+  symlink_or_copy("addr2line") {
+    deps = [
+      ":llvm-symbolizer",
+    ]
+    source = "llvm-symbolizer"
+    output = "$root_out_dir/bin/addr2line"
+  }
+}
+
+# //:llvm-symbolizer depends on this symlink target, see comment in //BUILD.gn.
+group("symlinks") {
+  deps = [
+    ":llvm-symbolizer",
+  ]
+  if (llvm_install_binutils_symlinks) {
+    deps += [ ":addr2line" ]
+  }
+}
+
+executable("llvm-symbolizer") {
+  deps = [
+    "//llvm/lib/DebugInfo/DWARF",
+    "//llvm/lib/DebugInfo/PDB",
+    "//llvm/lib/DebugInfo/Symbolize",
+    "//llvm/lib/Demangle",
+    "//llvm/lib/Object",
+    "//llvm/lib/Support",
+  ]
+  sources = [
+    "llvm-symbolizer.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/llvm/tools/llvm-xray/BUILD.gn b/utils/gn/secondary/llvm/tools/llvm-xray/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..89128bd95bb2496ecb0b210e4b9e26652f63ec2a
--- /dev/null
+++ b/utils/gn/secondary/llvm/tools/llvm-xray/BUILD.gn
@@ -0,0 +1,23 @@
+executable("llvm-xray") {
+  deps = [
+    "//llvm/lib/DebugInfo/DWARF",
+    "//llvm/lib/DebugInfo/Symbolize",
+    "//llvm/lib/Object",
+    "//llvm/lib/Support",
+    "//llvm/lib/Target:TargetsToBuild",
+    "//llvm/lib/XRay",
+  ]
+  sources = [
+    "func-id-helper.cpp",
+    "llvm-xray.cpp",
+    "xray-account.cpp",
+    "xray-color-helper.cpp",
+    "xray-converter.cpp",
+    "xray-extract.cpp",
+    "xray-fdr-dump.cpp",
+    "xray-graph-diff.cpp",
+    "xray-graph.cpp",
+    "xray-registry.cpp",
+    "xray-stacks.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/llvm/tools/lto/BUILD.gn b/utils/gn/secondary/llvm/tools/lto/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..135b952c1535298e5bdd01264017c0ac87e39b5d
--- /dev/null
+++ b/utils/gn/secondary/llvm/tools/lto/BUILD.gn
@@ -0,0 +1,39 @@
+import("//llvm/version.gni")
+
+lto_target_type = "shared_library"
+if (host_os != "mac" && host_os != "win") {
+  # ELF targets need -fPIC to build shared libs but they aren't on by default.
+  # For now, make libclang a static lib there.
+  lto_target_type = "static_library"
+}
+
+target(lto_target_type, "lto") {
+  output_name = "LTO"
+  deps = [
+    "//llvm/lib/Bitcode/Reader",
+    "//llvm/lib/IR",
+    "//llvm/lib/LTO",
+    "//llvm/lib/MC",
+    "//llvm/lib/MC/MCDisassembler",
+    "//llvm/lib/Support",
+    "//llvm/lib/Target",
+    "//llvm/lib/Target:TargetsToBuild",
+  ]
+  sources = [
+    "LTODisassembler.cpp",
+    "lto.cpp",
+  ]
+
+  if (host_os == "mac") {
+    ldflags = [
+      "-Wl,-compatibility_version,1",
+      "-Wl,-current_version,$llvm_version",
+
+      # See llvm_setup_rpath() in CMake.
+      "-Wl,-install_name,@rpath/libLTO.dylib",
+      "-Wl,-rpath,@loader_path/../lib",
+    ]
+  }
+
+  # FIXME: Use lto.exports
+}
diff --git a/utils/gn/secondary/llvm/tools/opt/BUILD.gn b/utils/gn/secondary/llvm/tools/opt/BUILD.gn
index 7deae1c79c315df9c8e3242b4c9bd1e1e0c551c5..2bf062c4dc2387d2f2ef43987524db0a42ebd089 100644
--- a/utils/gn/secondary/llvm/tools/opt/BUILD.gn
+++ b/utils/gn/secondary/llvm/tools/opt/BUILD.gn
@@ -32,7 +32,7 @@ executable("opt") {
 
   # Support plugins.
   # FIXME: Disable dead stripping once other binaries are dead-stripped.
-  if (host_os == "linux") {
+  if (host_os != "mac" && host_os != "win") {
     # Corresponds to export_executable_symbols() in cmake.
     ldflags = [ "-rdynamic" ]
   }
diff --git a/utils/gn/secondary/llvm/tools/sancov/BUILD.gn b/utils/gn/secondary/llvm/tools/sancov/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..c07af7b6ca1da3e133d830337ffb9fe30f176959
--- /dev/null
+++ b/utils/gn/secondary/llvm/tools/sancov/BUILD.gn
@@ -0,0 +1,18 @@
+executable("sancov") {
+  deps = [
+    "//llvm/lib/DebugInfo/DWARF",
+    "//llvm/lib/DebugInfo/PDB",
+    "//llvm/lib/DebugInfo/Symbolize",
+    "//llvm/lib/MC",
+    "//llvm/lib/MC/MCDisassembler",
+    "//llvm/lib/Object",
+    "//llvm/lib/Support",
+    "//llvm/lib/Target:AllTargetsAsmPrinters",
+    "//llvm/lib/Target:AllTargetsDescs",
+    "//llvm/lib/Target:AllTargetsDisassemblers",
+    "//llvm/lib/Target:AllTargetsInfos",
+  ]
+  sources = [
+    "sancov.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/llvm/tools/sanstats/BUILD.gn b/utils/gn/secondary/llvm/tools/sanstats/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..306775d6f80d49e62b0416b2742a15e39a7bba1b
--- /dev/null
+++ b/utils/gn/secondary/llvm/tools/sanstats/BUILD.gn
@@ -0,0 +1,9 @@
+executable("sanstats") {
+  deps = [
+    "//llvm/lib/DebugInfo/Symbolize",
+    "//llvm/lib/Support",
+  ]
+  sources = [
+    "sanstats.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/llvm/tools/verify-uselistorder/BUILD.gn b/utils/gn/secondary/llvm/tools/verify-uselistorder/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..09fd4899b799991fdea0a0fa8867c796ba162aaa
--- /dev/null
+++ b/utils/gn/secondary/llvm/tools/verify-uselistorder/BUILD.gn
@@ -0,0 +1,13 @@
+executable("verify-uselistorder") {
+  deps = [
+    "//llvm/lib/AsmParser",
+    "//llvm/lib/Bitcode/Reader",
+    "//llvm/lib/Bitcode/Writer",
+    "//llvm/lib/IR",
+    "//llvm/lib/IRReader",
+    "//llvm/lib/Support",
+  ]
+  sources = [
+    "verify-uselistorder.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/llvm/triples.gni b/utils/gn/secondary/llvm/triples.gni
index b5ff76237218c3f16049f74618852adfa2bdb904..a82bd9b921bf7c5e8d18faf5eeda9f18fe829384 100644
--- a/utils/gn/secondary/llvm/triples.gni
+++ b/utils/gn/secondary/llvm/triples.gni
@@ -1,12 +1,24 @@
-if (host_os == "linux") {
-  llvm_host_triple = "x86_64-unknown-linux-gnu"
-} else if (host_os == "mac") {
-  llvm_host_triple = "x86_64-apple-darwin"
-} else if (host_os == "win") {
-  llvm_host_triple = "x86_64-pc-windows"
+if (current_cpu == "x64") {
+  if (current_os == "freebsd") {
+    llvm_current_triple = "x86_64-unknown-freebsd"
+  } else if (current_os == "linux") {
+    llvm_current_triple = "x86_64-unknown-linux-gnu"
+  } else if (current_os == "mac") {
+    llvm_current_triple = "x86_64-apple-darwin"
+  } else if (current_os == "win") {
+    llvm_current_triple = "x86_64-pc-windows"
+  }
+} else if (current_cpu == "arm64") {
+  if (current_os == "android") {
+    llvm_current_triple = "aarch64-linux-android21"
+  }
+}
+
+if (!defined(llvm_current_triple)) {
+  assert(false, "unimplemented cpu/os " + current_cpu + "/" + current_os)
 }
 
 declare_args() {
   # The default target triple.
-  llvm_target_triple = llvm_host_triple
+  llvm_target_triple = llvm_current_triple
 }
diff --git a/utils/gn/secondary/llvm/unittests/ADT/BUILD.gn b/utils/gn/secondary/llvm/unittests/ADT/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..0762180ad9281bb7ddcba473a31829c0d58c2f95
--- /dev/null
+++ b/utils/gn/secondary/llvm/unittests/ADT/BUILD.gn
@@ -0,0 +1,81 @@
+import("//llvm/utils/unittest/unittest.gni")
+
+unittest("ADTTests") {
+  # ADT is a headers-only library so there's no //llvm/lib/ADT to depend on.
+  # Also see note in //llvm/lib/Support/BUILD.gn.
+  deps = [
+    # Some tests include files from IR, but there's no library dependency.
+    "//llvm/include/llvm/IR:public_tablegen",
+    "//llvm/lib/Support",
+    "//llvm/lib/Testing/Support",
+  ]
+  sources = [
+    "APFloatTest.cpp",
+    "APIntTest.cpp",
+    "APSIntTest.cpp",
+    "AnyTest.cpp",
+    "ArrayRefTest.cpp",
+    "BitVectorTest.cpp",
+    "BitmaskEnumTest.cpp",
+    "BreadthFirstIteratorTest.cpp",
+    "BumpPtrListTest.cpp",
+    "DAGDeltaAlgorithmTest.cpp",
+    "DeltaAlgorithmTest.cpp",
+    "DenseMapTest.cpp",
+    "DenseSetTest.cpp",
+    "DepthFirstIteratorTest.cpp",
+    "EquivalenceClassesTest.cpp",
+    "FallibleIteratorTest.cpp",
+    "FoldingSet.cpp",
+    "FunctionExtrasTest.cpp",
+    "FunctionRefTest.cpp",
+    "HashingTest.cpp",
+    "IListBaseTest.cpp",
+    "IListIteratorTest.cpp",
+    "IListNodeBaseTest.cpp",
+    "IListNodeTest.cpp",
+    "IListSentinelTest.cpp",
+    "IListTest.cpp",
+    "ImmutableListTest.cpp",
+    "ImmutableMapTest.cpp",
+    "ImmutableSetTest.cpp",
+    "IntEqClassesTest.cpp",
+    "IntervalMapTest.cpp",
+    "IntrusiveRefCntPtrTest.cpp",
+    "IteratorTest.cpp",
+    "MakeUniqueTest.cpp",
+    "MapVectorTest.cpp",
+    "MappedIteratorTest.cpp",
+    "OptionalTest.cpp",
+    "PackedVectorTest.cpp",
+    "PointerEmbeddedIntTest.cpp",
+    "PointerIntPairTest.cpp",
+    "PointerSumTypeTest.cpp",
+    "PointerUnionTest.cpp",
+    "PostOrderIteratorTest.cpp",
+    "PriorityWorklistTest.cpp",
+    "RangeAdapterTest.cpp",
+    "SCCIteratorTest.cpp",
+    "STLExtrasTest.cpp",
+    "ScopeExitTest.cpp",
+    "SequenceTest.cpp",
+    "SetVectorTest.cpp",
+    "SimpleIListTest.cpp",
+    "SmallPtrSetTest.cpp",
+    "SmallSetTest.cpp",
+    "SmallStringTest.cpp",
+    "SmallVectorTest.cpp",
+    "SparseBitVectorTest.cpp",
+    "SparseMultiSetTest.cpp",
+    "SparseSetTest.cpp",
+    "StatisticTest.cpp",
+    "StringExtrasTest.cpp",
+    "StringMapTest.cpp",
+    "StringRefTest.cpp",
+    "StringSwitchTest.cpp",
+    "TinyPtrVectorTest.cpp",
+    "TripleTest.cpp",
+    "TwineTest.cpp",
+    "VariadicFunctionTest.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/llvm/unittests/Analysis/BUILD.gn b/utils/gn/secondary/llvm/unittests/Analysis/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..fb9b2ab82d05714856e4c206b15d4d01b99964ff
--- /dev/null
+++ b/utils/gn/secondary/llvm/unittests/Analysis/BUILD.gn
@@ -0,0 +1,39 @@
+import("//llvm/utils/unittest/unittest.gni")
+
+unittest("AnalysisTests") {
+  deps = [
+    "//llvm/lib/Analysis",
+    "//llvm/lib/AsmParser",
+    "//llvm/lib/IR",
+    "//llvm/lib/Support",
+  ]
+  sources = [
+    "AliasAnalysisTest.cpp",
+    "AliasSetTrackerTest.cpp",
+    "BasicAliasAnalysisTest.cpp",
+    "BlockFrequencyInfoTest.cpp",
+    "BranchProbabilityInfoTest.cpp",
+    "CFGTest.cpp",
+    "CGSCCPassManagerTest.cpp",
+    "CallGraphTest.cpp",
+    "CaptureTrackingTest.cpp",
+    "DivergenceAnalysisTest.cpp",
+    "DomTreeUpdaterTest.cpp",
+    "GlobalsModRefTest.cpp",
+    "LazyCallGraphTest.cpp",
+    "LoopInfoTest.cpp",
+    "MemoryBuiltinsTest.cpp",
+    "MemorySSATest.cpp",
+    "OrderedBasicBlockTest.cpp",
+    "OrderedInstructionsTest.cpp",
+    "PhiValuesTest.cpp",
+    "ProfileSummaryInfoTest.cpp",
+    "ScalarEvolutionTest.cpp",
+    "SparsePropagation.cpp",
+    "TBAATest.cpp",
+    "TargetLibraryInfoTest.cpp",
+    "UnrollAnalyzerTest.cpp",
+    "ValueLatticeTest.cpp",
+    "ValueTrackingTest.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/llvm/unittests/AsmParser/BUILD.gn b/utils/gn/secondary/llvm/unittests/AsmParser/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..7cc4a546ca16753da18513b9dc6c123dc03012cf
--- /dev/null
+++ b/utils/gn/secondary/llvm/unittests/AsmParser/BUILD.gn
@@ -0,0 +1,12 @@
+import("//llvm/utils/unittest/unittest.gni")
+
+unittest("AsmParserTests") {
+  deps = [
+    "//llvm/lib/AsmParser",
+    "//llvm/lib/IR",
+    "//llvm/lib/Support",
+  ]
+  sources = [
+    "AsmParserTest.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/llvm/unittests/BUILD.gn b/utils/gn/secondary/llvm/unittests/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..b46d4027db1d929308cd123db63e9a216705960b
--- /dev/null
+++ b/utils/gn/secondary/llvm/unittests/BUILD.gn
@@ -0,0 +1,68 @@
+import("//llvm/lib/Target/targets.gni")
+
+group("unittests") {
+  deps = [
+    "ADT:ADTTests",
+    "Analysis:AnalysisTests",
+    "AsmParser:AsmParserTests",
+    "BinaryFormat:BinaryFormatTests",
+    "Bitcode:BitcodeTests",
+    "CodeGen:CodeGenTests",
+    "CodeGen/GlobalISel:GlobalISelTests",
+    "DebugInfo/CodeView:DebugInfoCodeViewTests",
+    "DebugInfo/DWARF:DebugInfoDWARFTests",
+    "DebugInfo/MSF:DebugInfoMSFTests",
+    "DebugInfo/PDB:DebugInfoPDBTests",
+    "Demangle:DemangleTests",
+    "ExecutionEngine:ExecutionEngineTests",
+    "ExecutionEngine/MCJIT:MCJITTests",
+    "ExecutionEngine/Orc:OrcJITTests",
+    "FuzzMutate:FuzzMutateTests",
+    "IR:IRTests",
+    "LineEditor:LineEditorTests",
+    "Linker:LinkerTests",
+    "MC:MCTests",
+    "MI:MITests",
+    "Object:ObjectTests",
+    "ObjectYAML:ObjectYAMLTests",
+    "OptRemarks:OptRemarksTests",
+    "Option:OptionTests",
+    "Passes:PluginsTests",
+    "ProfileData:ProfileDataTests",
+    "Support:SupportTests",
+    "Support/DynamicLibrary:DynamicLibraryTests",
+    "TextAPI:TextAPITests",
+    "Transforms/IPO:IPOTests",
+    "Transforms/Scalar:ScalarTests",
+    "Transforms/Utils:UtilsTests",
+    "Transforms/Vectorize:VectorizeTests",
+    "XRay:XRayTests",
+    "tools/llvm-cfi-verify:CFIVerifyTests",
+    "tools/llvm-exegesis:LLVMExegesisTests",
+  ]
+
+  # Target-dependent unit tests.
+  # FIXME: This matches how they are set up in the cmake build,
+  # but if we disable an arch after building with it on, this
+  # setup leaves behind stale executables.
+  if (llvm_build_AArch64) {
+    deps += [
+      "Target/AArch64:AArch64Tests",
+      "tools/llvm-exegesis/AArch64:LLVMExegesisAArch64Tests",
+    ]
+  }
+  if (llvm_build_ARM) {
+    deps += [ "tools/llvm-exegesis/ARM:LLVMExegesisARMTests" ]
+  }
+  if (llvm_build_WebAssembly) {
+    deps += [ "Target/WebAssembly:WebAssemblyTests" ]
+  }
+  if (llvm_build_PowerPC) {
+    deps += [ "tools/llvm-exegesis/PowerPC:LLVMExegesisPowerPCTests" ]
+  }
+  if (llvm_build_X86) {
+    deps += [ "tools/llvm-exegesis/X86:LLVMExegesisX86Tests" ]
+  }
+
+  testonly = true
+}
diff --git a/utils/gn/secondary/llvm/unittests/BinaryFormat/BUILD.gn b/utils/gn/secondary/llvm/unittests/BinaryFormat/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..7b2b78e439086d08df2897b9679f84896a254562
--- /dev/null
+++ b/utils/gn/secondary/llvm/unittests/BinaryFormat/BUILD.gn
@@ -0,0 +1,15 @@
+import("//llvm/utils/unittest/unittest.gni")
+
+unittest("BinaryFormatTests") {
+  deps = [
+    "//llvm/lib/BinaryFormat",
+  ]
+  sources = [
+    "DwarfTest.cpp",
+    "MachOTest.cpp",
+    "MsgPackReaderTest.cpp",
+    "MsgPackTypesTest.cpp",
+    "MsgPackWriterTest.cpp",
+    "TestFileMagic.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/llvm/unittests/Bitcode/BUILD.gn b/utils/gn/secondary/llvm/unittests/Bitcode/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..9cb78946e829d62ef906e6f1397f5fb7ea76847b
--- /dev/null
+++ b/utils/gn/secondary/llvm/unittests/Bitcode/BUILD.gn
@@ -0,0 +1,16 @@
+import("//llvm/utils/unittest/unittest.gni")
+
+unittest("BitcodeTests") {
+  deps = [
+    "//llvm/lib/AsmParser",
+    "//llvm/lib/Bitcode/Reader",
+    "//llvm/lib/Bitcode/Writer",
+    "//llvm/lib/IR",
+    "//llvm/lib/Support",
+  ]
+  sources = [
+    "BitReaderTest.cpp",
+    "BitstreamReaderTest.cpp",
+    "BitstreamWriterTest.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/llvm/unittests/CodeGen/BUILD.gn b/utils/gn/secondary/llvm/unittests/CodeGen/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..423745ea644fc7a117c4ba571d9771a0685945e6
--- /dev/null
+++ b/utils/gn/secondary/llvm/unittests/CodeGen/BUILD.gn
@@ -0,0 +1,26 @@
+import("//llvm/utils/unittest/unittest.gni")
+
+unittest("CodeGenTests") {
+  deps = [
+    "//llvm/lib/Analysis",
+    "//llvm/lib/AsmParser",
+    "//llvm/lib/CodeGen",
+    "//llvm/lib/CodeGen/AsmPrinter",
+    "//llvm/lib/CodeGen/SelectionDAG",
+    "//llvm/lib/IR",
+    "//llvm/lib/MC",
+    "//llvm/lib/Support",
+    "//llvm/lib/Target",
+    "//llvm/lib/Target:TargetsToBuild",
+  ]
+  sources = [
+    "AArch64SelectionDAGTest.cpp",
+    "DIEHashTest.cpp",
+    "LowLevelTypeTest.cpp",
+    "MachineInstrBundleIteratorTest.cpp",
+    "MachineInstrTest.cpp",
+    "MachineOperandTest.cpp",
+    "ScalableVectorMVTsTest.cpp",
+    "TypeTraitsTest.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/llvm/unittests/CodeGen/GlobalISel/BUILD.gn b/utils/gn/secondary/llvm/unittests/CodeGen/GlobalISel/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..3afd244d055fab9e04334e63f66528b4d4085b49
--- /dev/null
+++ b/utils/gn/secondary/llvm/unittests/CodeGen/GlobalISel/BUILD.gn
@@ -0,0 +1,23 @@
+import("//llvm/utils/unittest/unittest.gni")
+
+unittest("GlobalISelTests") {
+  deps = [
+    "//llvm/lib/CodeGen",
+    "//llvm/lib/CodeGen/GlobalISel",
+    "//llvm/lib/CodeGen/MIRParser",
+    "//llvm/lib/IR",
+    "//llvm/lib/MC",
+    "//llvm/lib/Support",
+    "//llvm/lib/Target",
+    "//llvm/lib/Target:TargetsToBuild",
+  ]
+  sources = [
+    "CSETest.cpp",
+    "GISelMITest.cpp",
+    "LegalizerHelperTest.cpp",
+    "LegalizerInfoTest.cpp",
+    "MachineIRBuilderTest.cpp",
+    "PatternMatchTest.cpp",
+  ]
+  has_custom_main = true
+}
diff --git a/utils/gn/secondary/llvm/unittests/DebugInfo/CodeView/BUILD.gn b/utils/gn/secondary/llvm/unittests/DebugInfo/CodeView/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..913de50c7e6c4812864ebd94d378a8dc5f04d5f5
--- /dev/null
+++ b/utils/gn/secondary/llvm/unittests/DebugInfo/CodeView/BUILD.gn
@@ -0,0 +1,13 @@
+import("//llvm/utils/unittest/unittest.gni")
+
+unittest("DebugInfoCodeViewTests") {
+  deps = [
+    "//llvm/lib/DebugInfo/CodeView",
+    "//llvm/lib/Testing/Support",
+  ]
+  sources = [
+    "RandomAccessVisitorTest.cpp",
+    "TypeHashingTest.cpp",
+    "TypeIndexDiscoveryTest.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/llvm/unittests/DebugInfo/DWARF/BUILD.gn b/utils/gn/secondary/llvm/unittests/DebugInfo/DWARF/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..1c22a4ba1dd64d86c0119f9bb63dd98efea9b2ea
--- /dev/null
+++ b/utils/gn/secondary/llvm/unittests/DebugInfo/DWARF/BUILD.gn
@@ -0,0 +1,21 @@
+import("//llvm/utils/unittest/unittest.gni")
+
+unittest("DebugInfoDWARFTests") {
+  deps = [
+    "//llvm/lib/CodeGen/AsmPrinter",
+    "//llvm/lib/DebugInfo/DWARF",
+    "//llvm/lib/MC",
+    "//llvm/lib/Object",
+    "//llvm/lib/ObjectYAML",
+    "//llvm/lib/Support",
+    "//llvm/lib/Target:TargetsToBuild",
+    "//llvm/lib/Testing/Support",
+  ]
+  sources = [
+    "DWARFDebugInfoTest.cpp",
+    "DWARFDebugLineTest.cpp",
+    "DWARFFormValueTest.cpp",
+    "DwarfGenerator.cpp",
+    "DwarfUtils.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/llvm/unittests/DebugInfo/MSF/BUILD.gn b/utils/gn/secondary/llvm/unittests/DebugInfo/MSF/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..6c9bc06ab4cff021b2cd41abb57005f7aa9c5021
--- /dev/null
+++ b/utils/gn/secondary/llvm/unittests/DebugInfo/MSF/BUILD.gn
@@ -0,0 +1,13 @@
+import("//llvm/utils/unittest/unittest.gni")
+
+unittest("DebugInfoMSFTests") {
+  deps = [
+    "//llvm/lib/DebugInfo/MSF",
+    "//llvm/lib/Testing/Support",
+  ]
+  sources = [
+    "MSFBuilderTest.cpp",
+    "MSFCommonTest.cpp",
+    "MappedBlockStreamTest.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/llvm/unittests/DebugInfo/PDB/BUILD.gn b/utils/gn/secondary/llvm/unittests/DebugInfo/PDB/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..cfb92c6c57a9e1a55ad1812170a0165e645fcf5f
--- /dev/null
+++ b/utils/gn/secondary/llvm/unittests/DebugInfo/PDB/BUILD.gn
@@ -0,0 +1,32 @@
+import("//llvm/utils/unittest/unittest.gni")
+
+unittest("DebugInfoPDBTests") {
+  deps = [
+    "//llvm/lib/DebugInfo/CodeView",
+    "//llvm/lib/DebugInfo/MSF",
+    "//llvm/lib/DebugInfo/PDB",
+    "//llvm/lib/Testing/Support",
+  ]
+  sources = [
+    "HashTableTest.cpp",
+    "NativeSymbolReuseTest.cpp",
+    "PDBApiTest.cpp",
+    "StringTableBuilderTest.cpp",
+  ]
+
+  # DebugInfoPDBTests uses llvm::getInputFileDirectory(), which expects
+  # a file called llvm.srcdir.txt next to the test executable that contains
+  # the path of the source directory (which contains this file).
+  # lit doesn't change the cwd while running googletests, so the cwd isn't
+  # well-defined. This means this has to be an absolute path.
+  # FIXME: This doesn't work with swarming. This should really be a data
+  # dependency, and the cwd while tests requiring input files run should
+  # be required to be some fixed directory.
+  # FIXME: Also, the GN way is to write this file at build time. But since
+  # there's only one use of this, and since this is a pattern that hopefully
+  # will disappear again, and since it doesn't have any measurable performance
+  # hit, write the file at GN time.
+  # Note: This line here implicitly depends on unittest() setting output_dir to
+  # target_out_dir.
+  write_file("$target_out_dir/llvm.srcdir.txt", rebase_path("."))
+}
diff --git a/utils/gn/secondary/llvm/unittests/Demangle/BUILD.gn b/utils/gn/secondary/llvm/unittests/Demangle/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..7981d94067e524572fd7c1337d309d84e300a896
--- /dev/null
+++ b/utils/gn/secondary/llvm/unittests/Demangle/BUILD.gn
@@ -0,0 +1,12 @@
+import("//llvm/utils/unittest/unittest.gni")
+
+unittest("DemangleTests") {
+  deps = [
+    "//llvm/lib/Demangle",
+  ]
+  sources = [
+    "DemangleTest.cpp",
+    "ItaniumDemangleTest.cpp",
+    "PartialDemangleTest.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/llvm/unittests/ExecutionEngine/BUILD.gn b/utils/gn/secondary/llvm/unittests/ExecutionEngine/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..1076964e2324935771cacd622f1372cd0e855cbf
--- /dev/null
+++ b/utils/gn/secondary/llvm/unittests/ExecutionEngine/BUILD.gn
@@ -0,0 +1,16 @@
+import("//llvm/utils/unittest/unittest.gni")
+
+unittest("ExecutionEngineTests") {
+  deps = [
+    "//llvm/lib/ExecutionEngine",
+    "//llvm/lib/ExecutionEngine/Interpreter",
+    "//llvm/lib/ExecutionEngine/Orc",
+    "//llvm/lib/ExecutionEngine/RuntimeDyld",
+    "//llvm/lib/IR",
+    "//llvm/lib/MC",
+    "//llvm/lib/Support",
+  ]
+  sources = [
+    "ExecutionEngineTest.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/llvm/unittests/ExecutionEngine/MCJIT/BUILD.gn b/utils/gn/secondary/llvm/unittests/ExecutionEngine/MCJIT/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..eed7d734eb798ae5573625e38b104fdfe5006962
--- /dev/null
+++ b/utils/gn/secondary/llvm/unittests/ExecutionEngine/MCJIT/BUILD.gn
@@ -0,0 +1,29 @@
+import("//llvm/utils/unittest/unittest.gni")
+
+unittest("MCJITTests") {
+  deps = [
+    "//llvm/lib/Analysis",
+    "//llvm/lib/ExecutionEngine",
+    "//llvm/lib/ExecutionEngine/MCJIT",
+    "//llvm/lib/ExecutionEngine/RuntimeDyld",
+    "//llvm/lib/IR",
+    "//llvm/lib/MC",
+    "//llvm/lib/Support",
+    "//llvm/lib/Target",
+    "//llvm/lib/Target:NativeTarget",
+    "//llvm/lib/Transforms/IPO",
+    "//llvm/lib/Transforms/InstCombine",
+    "//llvm/lib/Transforms/Scalar",
+  ]
+  sources = [
+    "MCJITCAPITest.cpp",
+    "MCJITMemoryManagerTest.cpp",
+    "MCJITMultipleModuleTest.cpp",
+    "MCJITObjectCacheTest.cpp",
+    "MCJITTest.cpp",
+  ]
+
+  if (host_os == "win") {
+    sources += [ "MCJITTests.def" ]
+  }
+}
diff --git a/utils/gn/secondary/llvm/unittests/ExecutionEngine/Orc/BUILD.gn b/utils/gn/secondary/llvm/unittests/ExecutionEngine/Orc/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..81b66a5dc37e5685b7c55616ef546d6a5fbb6552
--- /dev/null
+++ b/utils/gn/secondary/llvm/unittests/ExecutionEngine/Orc/BUILD.gn
@@ -0,0 +1,33 @@
+import("//llvm/utils/unittest/unittest.gni")
+
+unittest("OrcJITTests") {
+  deps = [
+    "//llvm/lib/ExecutionEngine",
+    "//llvm/lib/ExecutionEngine/Orc",
+    "//llvm/lib/ExecutionEngine/RuntimeDyld",
+    "//llvm/lib/IR",
+    "//llvm/lib/Object",
+    "//llvm/lib/Support",
+    "//llvm/lib/Target:NativeTarget",
+  ]
+  sources = [
+    "CoreAPIsTest.cpp",
+    "GlobalMappingLayerTest.cpp",
+    "IndirectionUtilsTest.cpp",
+    "JITTargetMachineBuilderTest.cpp",
+    "LazyCallThroughAndReexportsTest.cpp",
+    "LazyEmittingLayerTest.cpp",
+    "LegacyAPIInteropTest.cpp",
+    "LegacyCompileOnDemandLayerTest.cpp",
+    "LegacyRTDyldObjectLinkingLayerTest.cpp",
+    "ObjectTransformLayerTest.cpp",
+    "OrcCAPITest.cpp",
+    "OrcTestCommon.cpp",
+    "QueueChannel.cpp",
+    "RPCUtilsTest.cpp",
+    "RTDyldObjectLinkingLayerTest.cpp",
+    "RemoteObjectLayerTest.cpp",
+    "SymbolStringPoolTest.cpp",
+    "ThreadSafeModuleTest.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/llvm/unittests/FuzzMutate/BUILD.gn b/utils/gn/secondary/llvm/unittests/FuzzMutate/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..2967ddcc44c10c56245048803c4e4fa30631d2ff
--- /dev/null
+++ b/utils/gn/secondary/llvm/unittests/FuzzMutate/BUILD.gn
@@ -0,0 +1,16 @@
+import("//llvm/utils/unittest/unittest.gni")
+
+unittest("FuzzMutateTests") {
+  deps = [
+    "//llvm/lib/AsmParser",
+    "//llvm/lib/FuzzMutate",
+    "//llvm/lib/IR",
+    "//llvm/lib/Support",
+  ]
+  sources = [
+    "OperationsTest.cpp",
+    "RandomIRBuilderTest.cpp",
+    "ReservoirSamplerTest.cpp",
+    "StrategiesTest.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/llvm/unittests/IR/BUILD.gn b/utils/gn/secondary/llvm/unittests/IR/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..647c6a5ba8e789889dce54173396918ce27e354a
--- /dev/null
+++ b/utils/gn/secondary/llvm/unittests/IR/BUILD.gn
@@ -0,0 +1,44 @@
+import("//llvm/utils/unittest/unittest.gni")
+
+unittest("IRTests") {
+  deps = [
+    "//llvm/lib/Analysis",
+    "//llvm/lib/AsmParser",
+    "//llvm/lib/IR",
+    "//llvm/lib/Passes",
+    "//llvm/lib/Support",
+    "//llvm/lib/Testing/Support",
+  ]
+  sources = [
+    "AsmWriterTest.cpp",
+    "AttributesTest.cpp",
+    "BasicBlockTest.cpp",
+    "CFGBuilder.cpp",
+    "ConstantRangeTest.cpp",
+    "ConstantsTest.cpp",
+    "DebugInfoTest.cpp",
+    "DebugTypeODRUniquingTest.cpp",
+    "DominatorTreeBatchUpdatesTest.cpp",
+    "DominatorTreeTest.cpp",
+    "FunctionTest.cpp",
+    "IRBuilderTest.cpp",
+    "InstructionsTest.cpp",
+    "IntrinsicsTest.cpp",
+    "LegacyPassManagerTest.cpp",
+    "MDBuilderTest.cpp",
+    "ManglerTest.cpp",
+    "MetadataTest.cpp",
+    "ModuleTest.cpp",
+    "PassBuilderCallbacksTest.cpp",
+    "PassManagerTest.cpp",
+    "PatternMatch.cpp",
+    "TypesTest.cpp",
+    "UseTest.cpp",
+    "UserTest.cpp",
+    "ValueHandleTest.cpp",
+    "ValueMapTest.cpp",
+    "ValueTest.cpp",
+    "VerifierTest.cpp",
+    "WaymarkTest.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/llvm/unittests/LineEditor/BUILD.gn b/utils/gn/secondary/llvm/unittests/LineEditor/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..0916c9108847c3f50e97f53d17ceb01aa6116c47
--- /dev/null
+++ b/utils/gn/secondary/llvm/unittests/LineEditor/BUILD.gn
@@ -0,0 +1,11 @@
+import("//llvm/utils/unittest/unittest.gni")
+
+unittest("LineEditorTests") {
+  deps = [
+    "//llvm/lib/LineEditor",
+    "//llvm/lib/Support",
+  ]
+  sources = [
+    "LineEditor.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/llvm/unittests/Linker/BUILD.gn b/utils/gn/secondary/llvm/unittests/Linker/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..a245ffd20b46f39414c74f9164af047fe6adcd09
--- /dev/null
+++ b/utils/gn/secondary/llvm/unittests/Linker/BUILD.gn
@@ -0,0 +1,12 @@
+import("//llvm/utils/unittest/unittest.gni")
+
+unittest("LinkerTests") {
+  deps = [
+    "//llvm/lib/AsmParser",
+    "//llvm/lib/IR",
+    "//llvm/lib/Linker",
+  ]
+  sources = [
+    "LinkModulesTest.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/llvm/unittests/MC/BUILD.gn b/utils/gn/secondary/llvm/unittests/MC/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..d42db2cf36a4be04c3286ccff65ee75385c9b8c3
--- /dev/null
+++ b/utils/gn/secondary/llvm/unittests/MC/BUILD.gn
@@ -0,0 +1,16 @@
+import("//llvm/utils/unittest/unittest.gni")
+
+unittest("MCTests") {
+  deps = [
+    "//llvm/lib/MC",
+    "//llvm/lib/MC/MCDisassembler",
+    "//llvm/lib/Support",
+    "//llvm/lib/Target:TargetsToBuild",
+  ]
+  sources = [
+    "Disassembler.cpp",
+    "DwarfLineTables.cpp",
+    "StringTableBuilderTest.cpp",
+    "TargetRegistry.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/llvm/unittests/MI/BUILD.gn b/utils/gn/secondary/llvm/unittests/MI/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..4ef5ebf1e01bc1829ff64a401a4f30743c81a52d
--- /dev/null
+++ b/utils/gn/secondary/llvm/unittests/MI/BUILD.gn
@@ -0,0 +1,17 @@
+import("//llvm/utils/unittest/unittest.gni")
+
+unittest("MITests") {
+  deps = [
+    "//llvm/lib/CodeGen",
+    "//llvm/lib/CodeGen/MIRParser",
+    "//llvm/lib/IR",
+    "//llvm/lib/MC",
+    "//llvm/lib/Support",
+    "//llvm/lib/Target",
+    "//llvm/lib/Target:TargetsToBuild",
+  ]
+  sources = [
+    "LiveIntervalTest.cpp",
+  ]
+  has_custom_main = true
+}
diff --git a/utils/gn/secondary/llvm/unittests/Object/BUILD.gn b/utils/gn/secondary/llvm/unittests/Object/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..74f35256dc6732eb89dd5df415c2f4c34f071aa1
--- /dev/null
+++ b/utils/gn/secondary/llvm/unittests/Object/BUILD.gn
@@ -0,0 +1,11 @@
+import("//llvm/utils/unittest/unittest.gni")
+
+unittest("ObjectTests") {
+  deps = [
+    "//llvm/lib/Object",
+  ]
+  sources = [
+    "SymbolSizeTest.cpp",
+    "SymbolicFileTest.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/llvm/unittests/ObjectYAML/BUILD.gn b/utils/gn/secondary/llvm/unittests/ObjectYAML/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..8d8575d7d3f017ce719d09ce394f4478e4065325
--- /dev/null
+++ b/utils/gn/secondary/llvm/unittests/ObjectYAML/BUILD.gn
@@ -0,0 +1,10 @@
+import("//llvm/utils/unittest/unittest.gni")
+
+unittest("ObjectYAMLTests") {
+  deps = [
+    "//llvm/lib/ObjectYAML",
+  ]
+  sources = [
+    "YAMLTest.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/llvm/unittests/OptRemarks/BUILD.gn b/utils/gn/secondary/llvm/unittests/OptRemarks/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..785e78b0e77b5e7eb0b8f6ab00a2377da55fc458
--- /dev/null
+++ b/utils/gn/secondary/llvm/unittests/OptRemarks/BUILD.gn
@@ -0,0 +1,11 @@
+import("//llvm/utils/unittest/unittest.gni")
+
+unittest("OptRemarksTests") {
+  deps = [
+    "//llvm/lib/OptRemarks",
+    "//llvm/lib/Support",
+  ]
+  sources = [
+    "OptRemarksParsingTest.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/llvm/unittests/Option/BUILD.gn b/utils/gn/secondary/llvm/unittests/Option/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..02a67af2a2f1ec097a900d33b37eec962badb3aa
--- /dev/null
+++ b/utils/gn/secondary/llvm/unittests/Option/BUILD.gn
@@ -0,0 +1,18 @@
+import("//llvm/utils/TableGen/tablegen.gni")
+import("//llvm/utils/unittest/unittest.gni")
+
+tablegen("Opts") {
+  visibility = [ ":OptionTests" ]
+  args = [ "-gen-opt-parser-defs" ]
+}
+
+unittest("OptionTests") {
+  deps = [
+    ":Opts",
+    "//llvm/lib/Option",
+    "//llvm/lib/Support",
+  ]
+  sources = [
+    "OptionParsingTest.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/llvm/unittests/Passes/BUILD.gn b/utils/gn/secondary/llvm/unittests/Passes/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..4ad9317a21e23284398ff6afbf617aba81929d8e
--- /dev/null
+++ b/utils/gn/secondary/llvm/unittests/Passes/BUILD.gn
@@ -0,0 +1,53 @@
+import("//llvm/utils/unittest/unittest.gni")
+
+# Keyed off LLVM_ENABLE_PLUGINS in the CMake build, which is usually false
+# on Windows and true elsewhere.
+if (host_os != "win") {
+  loadable_module("TestPlugin") {
+    # Put plugin next to the unit test executable.
+    output_dir = target_out_dir
+
+    sources = [
+      "TestPlugin.cpp",
+    ]
+
+    deps = [
+      # TestPlugin doesn't want to link in any LLVM code, it just needs its
+      # headers.
+      "//llvm/include/llvm/IR:public_tablegen",
+    ]
+
+    if (host_os != "mac" && host_os != "win") {
+      # The GN build currently doesn't globally pass -fPIC, but that's
+      # needed for building .so files on ELF.  Just pass it manually
+      # for loadable_modules for now.
+      cflags = [ "-fPIC" ]
+    }
+  }
+}
+
+unittest("PluginsTests") {
+  deps = [
+    "//llvm/include/llvm/Config:config",
+    "//llvm/lib/IR",
+    "//llvm/lib/Passes",
+    "//llvm/lib/Support",
+    "//llvm/lib/Testing/Support",
+  ]
+  sources = [
+    "PluginsTest.cpp",
+  ]
+
+  # If plugins are disabled, this test will disable itself at runtime.
+  # Otherwise, reconfiguring with plugins disabled will leave behind a stale
+  # executable.
+  if (host_os != "win") {
+    deps += [ ":TestPlugin" ]
+    defines = [ "LLVM_ENABLE_PLUGINS" ]
+  }
+
+  if (host_os != "mac" && host_os != "win") {
+    # Corresponds to export_executable_symbols() in cmake.
+    ldflags = [ "-rdynamic" ]
+  }
+}
diff --git a/utils/gn/secondary/llvm/unittests/ProfileData/BUILD.gn b/utils/gn/secondary/llvm/unittests/ProfileData/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..e933b510003cd730f9191747475a9fa332be58d4
--- /dev/null
+++ b/utils/gn/secondary/llvm/unittests/ProfileData/BUILD.gn
@@ -0,0 +1,15 @@
+import("//llvm/utils/unittest/unittest.gni")
+
+unittest("ProfileDataTests") {
+  deps = [
+    "//llvm/lib/IR",
+    "//llvm/lib/ProfileData",
+    "//llvm/lib/ProfileData/Coverage",
+    "//llvm/lib/Testing/Support",
+  ]
+  sources = [
+    "CoverageMappingTest.cpp",
+    "InstrProfTest.cpp",
+    "SampleProfTest.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/llvm/unittests/Support/BUILD.gn b/utils/gn/secondary/llvm/unittests/Support/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..f8d8a028504d002e46b19e1e43bc65609293bfd1
--- /dev/null
+++ b/utils/gn/secondary/llvm/unittests/Support/BUILD.gn
@@ -0,0 +1,83 @@
+import("//llvm/utils/unittest/unittest.gni")
+
+unittest("SupportTests") {
+  deps = [
+    "//llvm/lib/Support",
+    "//llvm/lib/Testing/Support",
+  ]
+  sources = [
+    "ARMAttributeParser.cpp",
+    "AlignOfTest.cpp",
+    "AllocatorTest.cpp",
+    "ArrayRecyclerTest.cpp",
+    "BinaryStreamTest.cpp",
+    "BlockFrequencyTest.cpp",
+    "BranchProbabilityTest.cpp",
+    "CachePruningTest.cpp",
+    "Casting.cpp",
+    "CheckedArithmeticTest.cpp",
+    "Chrono.cpp",
+    "CommandLineTest.cpp",
+    "CompressionTest.cpp",
+    "ConvertUTFTest.cpp",
+    "CrashRecoveryTest.cpp",
+    "DJBTest.cpp",
+    "DataExtractorTest.cpp",
+    "DebugCounterTest.cpp",
+    "DebugTest.cpp",
+    "EndianStreamTest.cpp",
+    "EndianTest.cpp",
+    "ErrnoTest.cpp",
+    "ErrorOrTest.cpp",
+    "ErrorTest.cpp",
+    "FileOutputBufferTest.cpp",
+    "FormatVariadicTest.cpp",
+    "GlobPatternTest.cpp",
+    "Host.cpp",
+    "ItaniumManglingCanonicalizerTest.cpp",
+    "JSONTest.cpp",
+    "LEB128Test.cpp",
+    "LineIteratorTest.cpp",
+    "LockFileManagerTest.cpp",
+    "MD5Test.cpp",
+    "ManagedStatic.cpp",
+    "MathExtrasTest.cpp",
+    "MemoryBufferTest.cpp",
+    "MemoryTest.cpp",
+    "NativeFormatTests.cpp",
+    "ParallelTest.cpp",
+    "Path.cpp",
+    "ProcessTest.cpp",
+    "ProgramTest.cpp",
+    "RegexTest.cpp",
+    "ReplaceFileTest.cpp",
+    "ReverseIterationTest.cpp",
+    "ScaledNumberTest.cpp",
+    "SourceMgrTest.cpp",
+    "SpecialCaseListTest.cpp",
+    "StringPool.cpp",
+    "SwapByteOrderTest.cpp",
+    "SymbolRemappingReaderTest.cpp",
+    "TarWriterTest.cpp",
+    "TargetParserTest.cpp",
+    "TaskQueueTest.cpp",
+    "ThreadLocalTest.cpp",
+    "ThreadPool.cpp",
+    "Threading.cpp",
+    "TimerTest.cpp",
+    "TrailingObjectsTest.cpp",
+    "TrigramIndexTest.cpp",
+    "TypeNameTest.cpp",
+    "TypeTraitsTest.cpp",
+    "UnicodeTest.cpp",
+    "VersionTupleTest.cpp",
+    "VirtualFileSystemTest.cpp",
+    "YAMLIOTest.cpp",
+    "YAMLParserTest.cpp",
+    "formatted_raw_ostream_test.cpp",
+    "raw_ostream_test.cpp",
+    "raw_pwrite_stream_test.cpp",
+    "raw_sha1_ostream_test.cpp",
+    "xxhashTest.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/llvm/unittests/Support/DynamicLibrary/BUILD.gn b/utils/gn/secondary/llvm/unittests/Support/DynamicLibrary/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..aac3da706d1d2f74c31bd34158702227eb57f572
--- /dev/null
+++ b/utils/gn/secondary/llvm/unittests/Support/DynamicLibrary/BUILD.gn
@@ -0,0 +1,49 @@
+import("//llvm/utils/unittest/unittest.gni")
+
+# FIXME: If we add -Wl,-z,nodelete to the global ldflags, we need to remove
+# it again for these tests (cf CMake).
+
+template("dynlib_add_module") {
+  not_needed(invoker, "*")
+
+  loadable_module(target_name) {
+    # Put plugin next to the unit test executable.
+    # This assumes that unittest() puts tests in target_out_dir.
+    output_dir = target_out_dir
+
+    sources = [
+      "PipSqueak.cpp",
+    ]
+
+    if (host_os != "mac" && host_os != "win") {
+      # The GN build currently doesn't globally pass -fPIC, but that's
+      # needed for building .so files on ELF.  Just pass it manually
+      # for loadable_modules for now.
+      cflags = [ "-fPIC" ]
+    }
+  }
+}
+
+dynlib_add_module("PipSqueak") {
+}
+
+dynlib_add_module("SecondLib") {
+}
+
+unittest("DynamicLibraryTests") {
+  deps = [
+    ":PipSqueak",
+    ":SecondLib",
+    "//llvm/include/llvm/Config:config",
+    "//llvm/lib/Support",
+  ]
+  sources = [
+    "DynamicLibraryTest.cpp",
+    "ExportedFuncs.cpp",
+  ]
+
+  if (host_os != "mac" && host_os != "win") {
+    # Corresponds to export_executable_symbols() in cmake.
+    ldflags = [ "-rdynamic" ]
+  }
+}
diff --git a/utils/gn/secondary/llvm/unittests/Target/AArch64/BUILD.gn b/utils/gn/secondary/llvm/unittests/Target/AArch64/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..9a3313d27bbb970387a85597a28674cfbb7a7bf2
--- /dev/null
+++ b/utils/gn/secondary/llvm/unittests/Target/AArch64/BUILD.gn
@@ -0,0 +1,20 @@
+import("//llvm/utils/unittest/unittest.gni")
+
+unittest("AArch64Tests") {
+  deps = [
+    "//llvm/lib/CodeGen",
+    "//llvm/lib/CodeGen/GlobalISel",
+    "//llvm/lib/CodeGen/MIRParser",
+    "//llvm/lib/CodeGen/SelectionDAG",
+    "//llvm/lib/Support",
+    "//llvm/lib/Target",
+    "//llvm/lib/Target/AArch64:LLVMAArch64CodeGen",
+    "//llvm/lib/Target/AArch64/MCTargetDesc",
+    "//llvm/lib/Target/AArch64/TargetInfo",
+    "//llvm/lib/Target/AArch64/Utils",
+  ]
+  include_dirs = [ "//llvm/lib/Target/AArch64" ]
+  sources = [
+    "InstSizes.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/llvm/unittests/Target/WebAssembly/BUILD.gn b/utils/gn/secondary/llvm/unittests/Target/WebAssembly/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..e02149f738d6bd903fc8aabd9f4ed004b8f27da9
--- /dev/null
+++ b/utils/gn/secondary/llvm/unittests/Target/WebAssembly/BUILD.gn
@@ -0,0 +1,18 @@
+import("//llvm/utils/unittest/unittest.gni")
+
+unittest("WebAssemblyTests") {
+  deps = [
+    "//llvm/lib/CodeGen",
+    "//llvm/lib/CodeGen/MIRParser",
+    "//llvm/lib/IR",
+    "//llvm/lib/MC",
+    "//llvm/lib/Support",
+    "//llvm/lib/Target/WebAssembly:LLVMWebAssemblyCodeGen",
+    "//llvm/lib/Target/WebAssembly/MCTargetDesc",
+    "//llvm/lib/Target/WebAssembly/TargetInfo",
+  ]
+  include_dirs = [ "//llvm/lib/Target/WebAssembly" ]
+  sources = [
+    "WebAssemblyExceptionInfoTest.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/llvm/unittests/TextAPI/BUILD.gn b/utils/gn/secondary/llvm/unittests/TextAPI/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..0bfa109ba75cfe443ce64dcb56e4e25358e27a19
--- /dev/null
+++ b/utils/gn/secondary/llvm/unittests/TextAPI/BUILD.gn
@@ -0,0 +1,11 @@
+import("//llvm/utils/unittest/unittest.gni")
+
+unittest("TextAPITests") {
+  deps = [
+    "//llvm/lib/Testing/Support",
+    "//llvm/lib/TextAPI",
+  ]
+  sources = [
+    "ELFYAMLTest.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/llvm/unittests/Transforms/IPO/BUILD.gn b/utils/gn/secondary/llvm/unittests/Transforms/IPO/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..1b0a86d482c90358afc480c9d92f3809fca06d01
--- /dev/null
+++ b/utils/gn/secondary/llvm/unittests/Transforms/IPO/BUILD.gn
@@ -0,0 +1,13 @@
+import("//llvm/utils/unittest/unittest.gni")
+
+unittest("IPOTests") {
+  deps = [
+    "//llvm/lib/IR",
+    "//llvm/lib/Support",
+    "//llvm/lib/Transforms/IPO",
+  ]
+  sources = [
+    "LowerTypeTests.cpp",
+    "WholeProgramDevirt.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/llvm/unittests/Transforms/Scalar/BUILD.gn b/utils/gn/secondary/llvm/unittests/Transforms/Scalar/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..eee435fc2783c6243e19bd6f0deaaf07fa6262b8
--- /dev/null
+++ b/utils/gn/secondary/llvm/unittests/Transforms/Scalar/BUILD.gn
@@ -0,0 +1,15 @@
+import("//llvm/utils/unittest/unittest.gni")
+
+unittest("ScalarTests") {
+  deps = [
+    "//llvm/lib/Analysis",
+    "//llvm/lib/AsmParser",
+    "//llvm/lib/IR",
+    "//llvm/lib/Support",
+    "//llvm/lib/Transforms/Scalar",
+    "//llvm/lib/Transforms/Utils",
+  ]
+  sources = [
+    "LoopPassManagerTest.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/llvm/unittests/Transforms/Utils/BUILD.gn b/utils/gn/secondary/llvm/unittests/Transforms/Utils/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..e4585e246c28e88a0c2352f3cec0663600bddfed
--- /dev/null
+++ b/utils/gn/secondary/llvm/unittests/Transforms/Utils/BUILD.gn
@@ -0,0 +1,23 @@
+import("//llvm/utils/unittest/unittest.gni")
+
+unittest("UtilsTests") {
+  deps = [
+    "//llvm/lib/Analysis",
+    "//llvm/lib/AsmParser",
+    "//llvm/lib/IR",
+    "//llvm/lib/Support",
+    "//llvm/lib/Transforms/Utils",
+  ]
+  sources = [
+    "ASanStackFrameLayoutTest.cpp",
+    "BasicBlockUtilsTest.cpp",
+    "CloningTest.cpp",
+    "CodeExtractorTest.cpp",
+    "FunctionComparatorTest.cpp",
+    "IntegerDivisionTest.cpp",
+    "LocalTest.cpp",
+    "SSAUpdaterBulkTest.cpp",
+    "UnrollLoopTest.cpp",
+    "ValueMapperTest.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/llvm/unittests/Transforms/Vectorize/BUILD.gn b/utils/gn/secondary/llvm/unittests/Transforms/Vectorize/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..33e1efdbb584caa25b8fbf85cec5d28f93a10aba
--- /dev/null
+++ b/utils/gn/secondary/llvm/unittests/Transforms/Vectorize/BUILD.gn
@@ -0,0 +1,18 @@
+import("//llvm/utils/unittest/unittest.gni")
+
+unittest("VectorizeTests") {
+  deps = [
+    "//llvm/lib/Analysis",
+    "//llvm/lib/AsmParser",
+    "//llvm/lib/IR",
+    "//llvm/lib/Transforms/Vectorize",
+  ]
+  sources = [
+    "VPlanDominatorTreeTest.cpp",
+    "VPlanHCFGTest.cpp",
+    "VPlanLoopInfoTest.cpp",
+    "VPlanPredicatorTest.cpp",
+    "VPlanSlpTest.cpp",
+    "VPlanTest.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/llvm/unittests/XRay/BUILD.gn b/utils/gn/secondary/llvm/unittests/XRay/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..003bfc08239a80afd0de46ae4b3ad47e48459cd8
--- /dev/null
+++ b/utils/gn/secondary/llvm/unittests/XRay/BUILD.gn
@@ -0,0 +1,19 @@
+import("//llvm/utils/unittest/unittest.gni")
+
+unittest("XRayTests") {
+  deps = [
+    "//llvm/lib/Support",
+    "//llvm/lib/Testing/Support",
+    "//llvm/lib/XRay",
+  ]
+  sources = [
+    "FDRBlockIndexerTest.cpp",
+    "FDRBlockVerifierTest.cpp",
+    "FDRProducerConsumerTest.cpp",
+    "FDRRecordPrinterTest.cpp",
+    "FDRRecordsTest.cpp",
+    "FDRTraceWriterTest.cpp",
+    "GraphTest.cpp",
+    "ProfileTest.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/llvm/unittests/tools/llvm-cfi-verify/BUILD.gn b/utils/gn/secondary/llvm/unittests/tools/llvm-cfi-verify/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..e9731264e6eeb02c805bb91e75e19becd22fd0e6
--- /dev/null
+++ b/utils/gn/secondary/llvm/unittests/tools/llvm-cfi-verify/BUILD.gn
@@ -0,0 +1,22 @@
+import("//llvm/utils/unittest/unittest.gni")
+
+unittest("CFIVerifyTests") {
+  deps = [
+    "//llvm/lib/DebugInfo/Symbolize",
+    "//llvm/lib/MC",
+    "//llvm/lib/MC/MCParser",
+    "//llvm/lib/Object",
+    "//llvm/lib/Support",
+    "//llvm/lib/Target:AllTargetsAsmParsers",
+    "//llvm/lib/Target:AllTargetsAsmPrinters",
+    "//llvm/lib/Target:AllTargetsDescs",
+    "//llvm/lib/Target:AllTargetsDisassemblers",
+    "//llvm/lib/Target:AllTargetsInfos",
+    "//llvm/tools/llvm-cfi-verify/lib",
+  ]
+  sources = [
+    "FileAnalysis.cpp",
+    "GraphBuilder.cpp",
+  ]
+  has_custom_main = true
+}
diff --git a/utils/gn/secondary/llvm/unittests/tools/llvm-exegesis/AArch64/BUILD.gn b/utils/gn/secondary/llvm/unittests/tools/llvm-exegesis/AArch64/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..e72301b6e475f2056e2ffe958d8295bad727b615
--- /dev/null
+++ b/utils/gn/secondary/llvm/unittests/tools/llvm-exegesis/AArch64/BUILD.gn
@@ -0,0 +1,25 @@
+import("//llvm/utils/unittest/unittest.gni")
+
+unittest("LLVMExegesisAArch64Tests") {
+  deps = [
+    "//llvm/lib/DebugInfo/Symbolize",
+    "//llvm/lib/MC",
+    "//llvm/lib/MC/MCParser",
+    "//llvm/lib/Object",
+    "//llvm/lib/Support",
+    "//llvm/lib/Target/AArch64",
+
+    # Exegesis reaches inside the Target/AArch64 tablegen internals and must
+    # depend on this Target/AArch64-internal build target.
+    "//llvm/lib/Target/AArch64/MCTargetDesc",
+    "//llvm/tools/llvm-exegesis/lib",
+    "//llvm/tools/llvm-exegesis/lib/AArch64",
+  ]
+  include_dirs = [
+    "//llvm/lib/Target/AArch64",
+    "//llvm/tools/llvm-exegesis/lib",
+  ]
+  sources = [
+    "TargetTest.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/llvm/unittests/tools/llvm-exegesis/ARM/BUILD.gn b/utils/gn/secondary/llvm/unittests/tools/llvm-exegesis/ARM/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..6c4f768b7308747b13bfa1f902ef81eaf22465f1
--- /dev/null
+++ b/utils/gn/secondary/llvm/unittests/tools/llvm-exegesis/ARM/BUILD.gn
@@ -0,0 +1,25 @@
+import("//llvm/utils/unittest/unittest.gni")
+
+unittest("LLVMExegesisARMTests") {
+  deps = [
+    "//llvm/lib/DebugInfo/Symbolize",
+    "//llvm/lib/MC",
+    "//llvm/lib/MC/MCParser",
+    "//llvm/lib/Object",
+    "//llvm/lib/Support",
+    "//llvm/lib/Target/ARM",
+
+    # Exegesis reaches inside the Target/ARM tablegen internals and must
+    # depend on these Target/ARM-internal build targets.
+    "//llvm/lib/Target/ARM/MCTargetDesc",
+    "//llvm/lib/Target/ARM/Utils",
+    "//llvm/tools/llvm-exegesis/lib",
+  ]
+  include_dirs = [
+    "//llvm/lib/Target/ARM",
+    "//llvm/tools/llvm-exegesis/lib",
+  ]
+  sources = [
+    "AssemblerTest.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/llvm/unittests/tools/llvm-exegesis/BUILD.gn b/utils/gn/secondary/llvm/unittests/tools/llvm-exegesis/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..9eb3c9d3037ec3d1f6e00e7ad360f93a92c04dca
--- /dev/null
+++ b/utils/gn/secondary/llvm/unittests/tools/llvm-exegesis/BUILD.gn
@@ -0,0 +1,19 @@
+import("//llvm/utils/unittest/unittest.gni")
+
+unittest("LLVMExegesisTests") {
+  deps = [
+    "//llvm/lib/DebugInfo/Symbolize",
+    "//llvm/lib/MC",
+    "//llvm/lib/MC/MCParser",
+    "//llvm/lib/Object",
+    "//llvm/lib/Support",
+    "//llvm/tools/llvm-exegesis/lib",
+  ]
+  include_dirs = [ "../../../tools/llvm-exegesis/lib" ]
+  sources = [
+    "BenchmarkRunnerTest.cpp",
+    "ClusteringTest.cpp",
+    "PerfHelperTest.cpp",
+    "RegisterValueTest.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/llvm/unittests/tools/llvm-exegesis/PowerPC/BUILD.gn b/utils/gn/secondary/llvm/unittests/tools/llvm-exegesis/PowerPC/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..bbb37bdead87e409f199a18944ed6df6ed7dd7a3
--- /dev/null
+++ b/utils/gn/secondary/llvm/unittests/tools/llvm-exegesis/PowerPC/BUILD.gn
@@ -0,0 +1,26 @@
+import("//llvm/utils/unittest/unittest.gni")
+
+unittest("LLVMExegesisPowerPCTests") {
+  deps = [
+    "//llvm/lib/DebugInfo/Symbolize",
+    "//llvm/lib/MC",
+    "//llvm/lib/MC/MCParser",
+    "//llvm/lib/Object",
+    "//llvm/lib/Support",
+    "//llvm/lib/Target/PowerPC",
+
+    # Exegesis reaches inside the Target/PowerPC tablegen internals and must
+    # depend on these Target/PowerPC-internal build targets.
+    "//llvm/lib/Target/PowerPC/MCTargetDesc",
+    "//llvm/tools/llvm-exegesis/lib",
+    "//llvm/tools/llvm-exegesis/lib/PowerPC",
+  ]
+  include_dirs = [
+    "//llvm/lib/Target/PowerPC",
+    "//llvm/tools/llvm-exegesis/lib",
+  ]
+  sources = [
+    "AnalysisTest.cpp",
+    "TargetTest.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/llvm/unittests/tools/llvm-exegesis/X86/BUILD.gn b/utils/gn/secondary/llvm/unittests/tools/llvm-exegesis/X86/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..2b59d8ba43c7877681074fdd44f2d73bd85ce890
--- /dev/null
+++ b/utils/gn/secondary/llvm/unittests/tools/llvm-exegesis/X86/BUILD.gn
@@ -0,0 +1,31 @@
+import("//llvm/utils/unittest/unittest.gni")
+
+unittest("LLVMExegesisX86Tests") {
+  deps = [
+    "//llvm/lib/DebugInfo/Symbolize",
+    "//llvm/lib/MC",
+    "//llvm/lib/MC/MCParser",
+    "//llvm/lib/Object",
+    "//llvm/lib/Support",
+    "//llvm/lib/Target/X86",
+
+    # Exegesis reaches inside the Target/X86 tablegen internals and must
+    # depend on this Target/X86-internal build target -- and so must its
+    # unittests.
+    "//llvm/lib/Target/X86/MCTargetDesc",
+    "//llvm/tools/llvm-exegesis/lib",
+    "//llvm/tools/llvm-exegesis/lib/X86",
+  ]
+  include_dirs = [
+    "//llvm/lib/Target/X86",
+    "//llvm/tools/llvm-exegesis/lib",
+  ]
+  sources = [
+    "AnalysisTest.cpp",
+    "AssemblerTest.cpp",
+    "BenchmarkResultTest.cpp",
+    "RegisterAliasingTest.cpp",
+    "SnippetGeneratorTest.cpp",
+    "TargetTest.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/llvm/utils/TableGen/tablegen.gni b/utils/gn/secondary/llvm/utils/TableGen/tablegen.gni
index 2a5520e9f1bea1c2f4ad111353b866a4363a9d97..cb588abbaa6a59aef68389ad765d578c7c57cf70 100644
--- a/utils/gn/secondary/llvm/utils/TableGen/tablegen.gni
+++ b/utils/gn/secondary/llvm/utils/TableGen/tablegen.gni
@@ -9,7 +9,7 @@
 #       Basename of the generated output file.
 #       Defaults to target name with ".inc" appended.
 #
-#   td_file (roptional)
+#   td_file (optional)
 #       The .td file to pass to llvm-tblgen.
 #       Defaults to target name with ".td" appended.
 #
@@ -25,7 +25,7 @@
 #   }
 
 template("tablegen") {
-  assert(defined(invoker.args), "args must be defined for $target_name")
+  assert(defined(invoker.args), "must set 'args' in $target_name")
 
   config_name = "${target_name}_config"
   config(config_name) {
@@ -36,8 +36,12 @@ template("tablegen") {
   action(target_name) {
     forward_variables_from(invoker, [ "visibility" ])
 
-    # FIXME: In cross builds, this should depend on the host binary.
-    tblgen_target = "//llvm/utils/TableGen:llvm-tblgen"
+    if (defined(invoker.tblgen_target)) {
+      tblgen_target = invoker.tblgen_target
+    } else {
+      tblgen_target = "//llvm/utils/TableGen:llvm-tblgen"
+    }
+    tblgen_target += "($host_toolchain)"
     tblgen_executable = get_label_info(tblgen_target, "root_out_dir") +
                         "/bin/" + get_label_info(tblgen_target, "name")
     deps = [
@@ -60,11 +64,6 @@ template("tablegen") {
     depfile = "$gen_output.d"
     td_file = rebase_path(td_file, root_build_dir)
 
-    # FIXME: The cmake build lets tablegen write to a temp file and then copies
-    # it over the final output only if it has changed, for ninja's restat
-    # optimization. Instead of doing that in cmake, llvm-tblgen should do this
-    # itself. r330742 tried this, but it caused problems. Fix those and reland,
-    # so that the gn build has the optimization too.
     args = [
              rebase_path(tblgen_executable, root_build_dir),
              "-I",
diff --git a/utils/gn/secondary/llvm/utils/llvm-lit/BUILD.gn b/utils/gn/secondary/llvm/utils/llvm-lit/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..fca5679d6af5babdf218144f0bf650304b3e7953
--- /dev/null
+++ b/utils/gn/secondary/llvm/utils/llvm-lit/BUILD.gn
@@ -0,0 +1,56 @@
+import("//clang/test/clang_lit_site_cfg_files.gni")
+import("//lld/test/lld_lit_site_cfg_files.gni")
+import("//llvm/test/llvm_lit_site_cfg_files.gni")
+import("//llvm/utils/gn/build/write_cmake_config.gni")
+
+write_cmake_config("llvm-lit") {
+  input = "llvm-lit.in"
+  output = "$root_out_dir/bin/llvm-lit"
+
+  if (host_os == "win") {
+    # llvm-lit needs suffix.py for multiprocess to find a main module.
+    output = "${output}.py"
+  }
+
+  # lit's lit/llvm/config.py shells out to llvm-config.
+  deps = [
+    "//llvm/tools/llvm-config",
+  ]
+
+  # Generate LLVM_LIT_CONFIG_MAP parameter.
+  # llvm-lit contains a mapping from each lit.cfg.py file to the corresponding
+  # generated llvm.site.cfg.py file, so llvm-lit depends on all the targets to
+  # generate the site.cfg.py file.
+  config_map = ""
+
+  deps += [
+    "//clang/test:lit_site_cfg",
+    "//clang/test:lit_unit_site_cfg",
+    "//lld/test:lit_site_cfg",
+    "//lld/test:lit_unit_site_cfg",
+    "//llvm/test:lit_site_cfg",
+    "//llvm/test:lit_unit_site_cfg",
+  ]
+
+  # Note: \n is converted into a newline by write_cmake_config.py, not by gn.
+  config_map += "map_config('" + rebase_path("//clang/test/lit.cfg.py") +
+                "', '" + rebase_path(clang_lit_site_cfg_file) + "')\n"
+  config_map += "map_config('" + rebase_path("//clang/test/Unit/lit.cfg.py") +
+                "', '" + rebase_path(clang_lit_unit_site_cfg_file) + "')\n"
+  config_map += "map_config('" + rebase_path("//lld/test/lit.cfg.py") + "', '" +
+                rebase_path(lld_lit_site_cfg_file) + "')\n"
+  config_map += "map_config('" + rebase_path("//lld/test/Unit/lit.cfg.py") +
+                "', '" + rebase_path(lld_lit_unit_site_cfg_file) + "')\n"
+  config_map += "map_config('" + rebase_path("//llvm/test/lit.cfg.py") +
+                "', '" + rebase_path(llvm_lit_site_cfg_file) + "')\n"
+  config_map += "map_config('" + rebase_path("//llvm/test/Unit/lit.cfg.py") +
+                "', '" + rebase_path(llvm_lit_unit_site_cfg_file) + "')\n"
+
+  values = [
+    "LLVM_SOURCE_DIR=" + rebase_path("//llvm"),
+    "LLVM_BINARY_DIR=" +
+        rebase_path(get_label_info("//llvm", "target_out_dir")),
+    "BUILD_MODE=.",
+    "LLVM_LIT_CONFIG_MAP=" + config_map,
+  ]
+}
diff --git a/utils/gn/secondary/llvm/utils/unittest/BUILD.gn b/utils/gn/secondary/llvm/utils/unittest/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..e4620293f2cf91c40d6aaca2d2a64fd5885008b1
--- /dev/null
+++ b/utils/gn/secondary/llvm/utils/unittest/BUILD.gn
@@ -0,0 +1,40 @@
+import("//llvm/utils/gn/build/libs/pthread/enable.gni")
+
+# Used to push the gtest include directories to things depending on :googletest.
+config("googletest_config") {
+  include_dirs = [
+    "googlemock/include",
+    "googletest/include",
+  ]
+
+  # LLVM requires C++11 but gtest doesn't correctly detect the availability
+  # of C++11 on MSVC, so we force it on.
+  defines = [
+    "GTEST_LANG_CXX11",
+    "GTEST_HAS_TR1_TUPLE=0",
+  ]
+  if (host_os == "win") {
+    defines += [ "GTEST_OS_WINDOWS" ]
+  }
+  if (!llvm_enable_threads) {
+    defines += [ "GTEST_HAS_PTHREAD=0" ]
+  }
+  defines += [ "GTEST_HAS_RTTI=0" ]
+}
+
+static_library("gtest") {
+  deps = [
+    "//llvm/lib/Support",
+  ]
+  include_dirs = [
+    "googletest",  # For including src/gtest.cc
+    "googlemock",  # For including src/gmock.cc
+  ]
+  public_configs = [ ":googletest_config" ]
+  configs -= [ "//llvm/utils/gn/build:warn_covered_switch_default" ]
+  sources = [
+    "googlemock/src/gmock-all.cc",
+    "googletest/src/gtest-all.cc",
+  ]
+  testonly = true
+}
diff --git a/utils/gn/secondary/llvm/utils/unittest/UnitTestMain/BUILD.gn b/utils/gn/secondary/llvm/utils/unittest/UnitTestMain/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..51bc7b81b93f8808aa0a3313e0c45f650bf3222a
--- /dev/null
+++ b/utils/gn/secondary/llvm/utils/unittest/UnitTestMain/BUILD.gn
@@ -0,0 +1,25 @@
+source_set("UnitTestMain") {
+  deps = [
+    "//llvm/lib/Support",
+  ]
+
+  # Make targets depending on this also depend on gtest, to get the gtest
+  # include_dir.
+  public_deps = [
+    "..:gtest",
+  ]
+  sources = [
+    "TestMain.cpp",
+  ]
+  testonly = true
+}
+
+# Nothing depends on this target, but llvm-config expects it to exist when
+# it runs with `--link-static --system-libs`, so humor it.
+static_library("gtest_main") {
+  deps = [
+    ":UnitTestMain",
+  ]
+  complete_static_lib = true
+  testonly = true
+}
diff --git a/utils/gn/secondary/llvm/utils/unittest/unittest.gni b/utils/gn/secondary/llvm/utils/unittest/unittest.gni
new file mode 100644
index 0000000000000000000000000000000000000000..f0c8309a9aff2c2d573f9a5e61ff0ed017291fb3
--- /dev/null
+++ b/utils/gn/secondary/llvm/utils/unittest/unittest.gni
@@ -0,0 +1,58 @@
+# This file defines a template for adding a unittest binary.
+#
+# It's a thin wrapper around GN's built-in executable() target type and
+# accepts the same parameters, and in addition this paramater:
+#
+#   has_custom_main (optional)
+#       [bool] If set, link against gtest instead of UnitTestMain; for tests
+#              that define their own main() function.
+#
+# Example use:
+#
+#   unittest("FormatTest") {
+#     sources = [ ... ]
+#     ...
+#   }
+
+template("unittest") {
+  executable(target_name) {
+    has_custom_main = false  # Default value.
+
+    # Foward everything (has_custom_main if set; configs, sources, deps, ...).
+    forward_variables_from(invoker, "*")
+    assert(!defined(invoker.output_dir), "cannot set unittest output_dir")
+    assert(!defined(invoker.testonly), "cannot set unittest testonly")
+
+    # Common settings for all unit tests.
+    # Unit test binaries shouldn't go right in out/gn/bin, for two reasons:
+    # 1. That's where production binaries go.
+    # 2. The CMake build doesn't put the unit tests of all projects (clang,
+    #    lld,...) in one directory, so it's not guaranteed that there won't
+    #    be name collisions between test binaries from separate projects.
+    # Each lit suite takes an foo_obj_root parameter and puts temporary files
+    # for lit tests at foo_obj_root/test and looks for unit test binaries
+    # below foo_obj_root/unittests. As long as the BUILD.gn files processing
+    # the lit.site.cfg.py.in files match the output dir here, it doesn't
+    # matter all that much where the unit test binaries go, with the weak
+    # constraints that test binaries of different projects should go in
+    # different folders, and that it's not too difficult to manually
+    # run the unit test binary if necessary. Using target_out_dir here
+    # means that //clang/unittests/Format gets its binary in
+    # out/gn/obj/clang/unittests/Format/FormatTests, which seems fine.
+    #
+    # If you change output_dir here, look through
+    # `git grep target_out_dir '*/unittests/*'` and update those too.
+    output_dir = target_out_dir
+
+    if (has_custom_main) {
+      deps += [ "//llvm/utils/unittest:gtest" ]
+    } else {
+      deps += [ "//llvm/utils/unittest/UnitTestMain" ]
+    }
+    testonly = true
+  }
+}
+
+set_defaults("unittest") {
+  configs = shared_binary_target_configs
+}
diff --git a/utils/gn/secondary/llvm/utils/yaml-bench/BUILD.gn b/utils/gn/secondary/llvm/utils/yaml-bench/BUILD.gn
new file mode 100644
index 0000000000000000000000000000000000000000..e388bf4c38ae22ee45b4a8df04ab1f4a0724257d
--- /dev/null
+++ b/utils/gn/secondary/llvm/utils/yaml-bench/BUILD.gn
@@ -0,0 +1,8 @@
+executable("yaml-bench") {
+  deps = [
+    "//llvm/lib/Support",
+  ]
+  sources = [
+    "YAMLBench.cpp",
+  ]
+}
diff --git a/utils/gn/secondary/llvm/version.gni b/utils/gn/secondary/llvm/version.gni
index b64e3f9bdec08812d2425a19ade42ddc277b6ddd..fd448d925c734a8b8d82b6b0a6cabc1d42317d11 100644
--- a/utils/gn/secondary/llvm/version.gni
+++ b/utils/gn/secondary/llvm/version.gni
@@ -1,4 +1,4 @@
-llvm_version_major = 8
+llvm_version_major = 9
 llvm_version_minor = 0
 llvm_version_patch = 0
 llvm_version = "$llvm_version_major.$llvm_version_minor.$llvm_version_patch"
diff --git a/utils/indirect_calls.py b/utils/indirect_calls.py
index b7349a6d884f87713c42738a7159d0e0d7727207..e460ff7d083620680a812120ba7c622b9bd9396f 100755
--- a/utils/indirect_calls.py
+++ b/utils/indirect_calls.py
@@ -10,6 +10,8 @@
    dump format.
 """
 
+from __future__ import print_function
+
 import os
 import sys
 import re
@@ -32,8 +34,8 @@ def look_for_indirect(file):
         result = re.search('(call|jmp).*\*', line)
         if result != None:
             # TODO: Perhaps use cxxfilt to demangle functions?
-            print function
-            print line
+            print(function)
+            print(line)
     return
 
 def main(args):
diff --git a/utils/lint/common_lint.py b/utils/lint/common_lint.py
index e982680c052ad2669365edf91373d0a69f81688d..aec907929b421b1aef11654662ea0157d21f4338 100644
--- a/utils/lint/common_lint.py
+++ b/utils/lint/common_lint.py
@@ -2,6 +2,7 @@
 #
 # Common lint functions applicable to multiple types of files.
 
+from __future__ import print_function
 import re
 
 def VerifyLineLength(filename, lines, max_length):
@@ -89,7 +90,7 @@ def RunLintOverAllFiles(linter, filenames):
   for filename in filenames:
     file = open(filename, 'r')
     if not file:
-      print 'Cound not open %s' % filename
+      print('Cound not open %s' % filename)
       continue
     lines = file.readlines()
     lint.extend(linter.RunOnFile(filename, lines))
diff --git a/utils/lint/cpp_lint.py b/utils/lint/cpp_lint.py
index 07fad5840fd64213d7288f51e9a5784e9e9438f2..2fb8cc93c1929b6c8a09d83f98a43460e30d9af3 100755
--- a/utils/lint/cpp_lint.py
+++ b/utils/lint/cpp_lint.py
@@ -6,6 +6,7 @@
 # TODO: add unittests for the verifier functions:
 # http://docs.python.org/library/unittest.html .
 
+from __future__ import print_function
 import common_lint
 import re
 import sys
@@ -86,7 +87,7 @@ class CppLint(common_lint.BaseLint):
 def CppLintMain(filenames):
   all_lint = common_lint.RunLintOverAllFiles(CppLint(), filenames)
   for lint in all_lint:
-    print '%s:%d:%s' % (lint[0], lint[1], lint[2])
+    print('%s:%d:%s' % (lint[0], lint[1], lint[2]))
   return 0
 
 
diff --git a/utils/lit/lit/TestRunner.py b/utils/lit/lit/TestRunner.py
index 4d903b4a6f44f904b9e2989fb176e998692090c7..2e885b4f8dcdd01d353375333a2835760b6f9576 100644
--- a/utils/lit/lit/TestRunner.py
+++ b/utils/lit/lit/TestRunner.py
@@ -1401,13 +1401,17 @@ class IntegratedTestKeywordParser(object):
     @staticmethod
     def _handleBooleanExpr(line_number, line, output):
         """A parser for BOOLEAN_EXPR type keywords"""
+        parts = [s.strip() for s in line.split(',') if s.strip() != '']
+        if output and output[-1][-1] == '\\':
+            output[-1] = output[-1][:-1] + parts[0]
+            del parts[0]
         if output is None:
             output = []
-        output.extend([s.strip() for s in line.split(',')])
+        output.extend(parts)
         # Evaluate each expression to verify syntax.
         # We don't want any results, just the raised ValueError.
         for s in output:
-            if s != '*':
+            if s != '*' and not s.endswith('\\'):
                 BooleanExpression.evaluate(s, [])
         return output
 
@@ -1486,6 +1490,15 @@ def parseIntegratedTestScript(test, additional_parsers=[],
         return lit.Test.Result(Test.UNRESOLVED,
                                "Test has unterminated run lines (with '\\')")
 
+    # Check boolean expressions for unterminated lines.
+    for key in keyword_parsers:
+        kp = keyword_parsers[key]
+        if kp.kind != ParserKind.BOOLEAN_EXPR:
+            continue
+        value = kp.getValue()
+        if value and value[-1][-1] == '\\':
+            raise ValueError("Test has unterminated %s lines (with '\\')" % key)
+
     # Enforce REQUIRES:
     missing_required_features = test.getMissingRequiredFeatures()
     if missing_required_features:
diff --git a/utils/lit/lit/util.py b/utils/lit/lit/util.py
index e20c4ab901532d0511a8f2badd39f613417e637a..dee70b3c542ef8bf516808c48d62c959d813447f 100644
--- a/utils/lit/lit/util.py
+++ b/utils/lit/lit/util.py
@@ -1,3 +1,5 @@
+from __future__ import print_function
+
 import errno
 import itertools
 import math
@@ -195,7 +197,7 @@ def which(command, paths=None):
 
     # Check for absolute match first.
     if os.path.isabs(command) and os.path.isfile(command):
-        return os.path.normpath(command)
+        return os.path.normcase(os.path.normpath(command))
 
     # Would be nice if Python had a lib function for this.
     if not paths:
@@ -213,7 +215,7 @@ def which(command, paths=None):
         for ext in pathext:
             p = os.path.join(path, command + ext)
             if os.path.exists(p) and not os.path.isdir(p):
-                return os.path.normpath(p)
+                return os.path.normcase(os.path.normpath(p))
 
     return None
 
diff --git a/utils/lit/setup.py b/utils/lit/setup.py
index 10de6bb4ee79b62123fa27c7de57b24963a5da2a..10b75e7fe6ee994338e632e8839ebbeedafed5e7 100644
--- a/utils/lit/setup.py
+++ b/utils/lit/setup.py
@@ -16,7 +16,7 @@ setup(
     author = lit.__author__,
     author_email = lit.__email__,
     url = 'http://llvm.org',
-    license = 'BSD',
+    license = 'Apache-2.0 with LLVM exception',
 
     description = "A Software Testing Tool",
     keywords = 'test C++ automatic discovery',
@@ -52,15 +52,15 @@ Command Guide: http://llvm.org/cmds/lit.html.
 Source
 ======
 
-The *lit* source is available as part of LLVM, in the LLVM SVN repository:
-http://llvm.org/svn/llvm-project/llvm/trunk/utils/lit.
+The *lit* source is available as part of LLVM, in the LLVM source repository:
+https://github.com/llvm/llvm-project/tree/master/llvm/utils/lit
 """,
 
     classifiers=[
         'Development Status :: 3 - Alpha',
         'Environment :: Console',
         'Intended Audience :: Developers',
-        'License :: OSI Approved :: University of Illinois/NCSA Open Source License',
+        'License :: OSI Approved :: Apache-2.0 with LLVM exception',
         'Natural Language :: English',
         'Operating System :: OS Independent',
         'Programming Language :: Python',
diff --git a/utils/lit/tests/Inputs/shtest-env/print_environment.py b/utils/lit/tests/Inputs/shtest-env/print_environment.py
index 1add4079d58fa218054d1b65681ef8d3665b1b68..ac9a80e6d5063752d67e6880a900e3463248add5 100644
--- a/utils/lit/tests/Inputs/shtest-env/print_environment.py
+++ b/utils/lit/tests/Inputs/shtest-env/print_environment.py
@@ -1,8 +1,9 @@
 #!/usr/bin/env python
 
+from __future__ import print_statement
 import os
 
 sorted_environment = sorted(os.environ.items())
 
 for name,value in sorted_environment:
-    print name,'=',value
+    print(name,'=',value)
diff --git a/utils/lit/tests/Inputs/shtest-shell/check_path.py b/utils/lit/tests/Inputs/shtest-shell/check_path.py
index c1d279700de4446f31330c7fc40d0b32543dbd0b..467505b7f293f5bb00f65831da6da9574116a851 100644
--- a/utils/lit/tests/Inputs/shtest-shell/check_path.py
+++ b/utils/lit/tests/Inputs/shtest-shell/check_path.py
@@ -1,5 +1,7 @@
 #!/usr/bin/env python
 
+from __future__ import print_function
+
 import os
 import sys
 
diff --git a/utils/lit/tests/Inputs/shtest-timeout/lit.cfg b/utils/lit/tests/Inputs/shtest-timeout/lit.cfg
index 96bf18170a8f9d74ea2e5e3581fd565c44ed98a0..6256f5a9911a0c147249376b077df8c0daf655b7 100644
--- a/utils/lit/tests/Inputs/shtest-timeout/lit.cfg
+++ b/utils/lit/tests/Inputs/shtest-timeout/lit.cfg
@@ -28,5 +28,13 @@ config.test_source_root = os.path.dirname(__file__)
 config.test_exec_root = config.test_source_root
 config.target_triple = '(unused)'
 src_root = os.path.join(config.test_source_root, '..')
-config.environment['PYTHONPATH'] = src_root
+
+pythonpath_list = [src_root]
+# Ensure the user's PYTHONPATH is included.
+if 'PYTHONPATH' in os.environ:
+    pythonpath_list.append(os.environ['PYTHONPATH'])
+if 'PYTHONPATH' in config.environment:
+    pythonpath_list.append(config.environment['PYTHONPATH'])
+config.environment['PYTHONPATH'] = os.pathsep.join(pythonpath_list)
+
 config.substitutions.append(('%{python}', '"%s"' % (sys.executable)))
diff --git a/utils/lit/tests/Inputs/testrunner-custom-parsers/test.txt b/utils/lit/tests/Inputs/testrunner-custom-parsers/test.txt
index ed118f382fee8698986746088c3d22c8d4b48cce..e28060320a2e0dbdf8200ea20b9038dc9867f6c0 100644
--- a/utils/lit/tests/Inputs/testrunner-custom-parsers/test.txt
+++ b/utils/lit/tests/Inputs/testrunner-custom-parsers/test.txt
@@ -9,5 +9,11 @@
 //
 // MY_CUSTOM: a b c
 //
+// MY_BOOL: a && (\
+// MY_BOOL: b)
+// MY_BOOL: d
+//
+// MY_BOOL_UNTERMINATED: a \
+//
 // END.
 // MY_LIST: five
diff --git a/utils/lit/tests/lit.cfg b/utils/lit/tests/lit.cfg
index 01a3431b43598388dc7c567e2aae00a6a99302b2..2af93d6314ade6dca8fc36ffb7d237cab77846c7 100644
--- a/utils/lit/tests/lit.cfg
+++ b/utils/lit/tests/lit.cfg
@@ -34,7 +34,15 @@ if llvm_src_root != None:
 else:
   lit_path = src_root
 
-config.environment['PYTHONPATH'] = lit_path # Required because some tests import the lit module
+pythonpath_list = [lit_path] # Required because some tests import the lit module
+
+# Ensure the user's PYTHONPATH is included.
+if 'PYTHONPATH' in os.environ:
+    pythonpath_list.append(os.environ['PYTHONPATH'])
+if 'PYTHONPATH' in config.environment:
+    pythonpath_list.append(config.environment['PYTHONPATH'])
+config.environment['PYTHONPATH'] = os.pathsep.join(pythonpath_list)
+
 config.substitutions.append(('%{src_root}', src_root))
 config.substitutions.append(('%{inputs}', os.path.join(
             src_root, 'tests', 'Inputs')))
diff --git a/utils/lit/tests/shtest-run-at-line.py b/utils/lit/tests/shtest-run-at-line.py
index cd0e08137ee589b841df478f335fb9e16ae39ddb..7e5d53b8e26ceea37a1c6c1e4fa135c18469cae6 100644
--- a/utils/lit/tests/shtest-run-at-line.py
+++ b/utils/lit/tests/shtest-run-at-line.py
@@ -1,7 +1,7 @@
 # Check that -vv makes the line number of the failing RUN command clear.
 # (-v is actually sufficient in the case of the internal shell.)
 #
-# RUN: not %{lit} -j 1 -vv %{inputs}/shtest-run-at-line > %t.out
+# RUN: env -u FILECHECK_OPTS not %{lit} -j 1 -vv %{inputs}/shtest-run-at-line > %t.out
 # RUN: FileCheck --input-file %t.out %s
 #
 # END.
diff --git a/utils/lit/tests/unit/TestRunner.py b/utils/lit/tests/unit/TestRunner.py
index 89209d80f5557ec267fcbbd73f45eaadcd3bda55..a9cfd869b767ddff24b81cc4d5dbbd2f3f553b7d 100644
--- a/utils/lit/tests/unit/TestRunner.py
+++ b/utils/lit/tests/unit/TestRunner.py
@@ -9,6 +9,7 @@ import os.path
 import tempfile
 
 import lit
+import lit.Test as Test
 from lit.TestRunner import ParserKind, IntegratedTestKeywordParser, \
                            parseIntegratedTestScript
 
@@ -57,9 +58,11 @@ class TestIntegratedTestKeywordParser(unittest.TestCase):
             IntegratedTestKeywordParser("MY_TAG.", ParserKind.TAG),
             IntegratedTestKeywordParser("MY_DNE_TAG.", ParserKind.TAG),
             IntegratedTestKeywordParser("MY_LIST:", ParserKind.LIST),
+            IntegratedTestKeywordParser("MY_BOOL:", ParserKind.BOOLEAN_EXPR),
             IntegratedTestKeywordParser("MY_RUN:", ParserKind.COMMAND),
             IntegratedTestKeywordParser("MY_CUSTOM:", ParserKind.CUSTOM,
-                                        custom_parse)
+                                        custom_parse),
+
         ]
 
     @staticmethod
@@ -102,6 +105,25 @@ class TestIntegratedTestKeywordParser(unittest.TestCase):
         self.assertEqual(value[0].strip(), "%dbg(MY_RUN: at line 4)  baz")
         self.assertEqual(value[1].strip(), "%dbg(MY_RUN: at line 7)  foo  bar")
 
+    def test_boolean(self):
+        parsers = self.make_parsers()
+        self.parse_test(parsers)
+        bool_parser = self.get_parser(parsers, 'MY_BOOL:')
+        value = bool_parser.getValue()
+        self.assertEqual(len(value), 2)  # there are only two run lines
+        self.assertEqual(value[0].strip(), "a && (b)")
+        self.assertEqual(value[1].strip(), "d")
+
+    def test_boolean_unterminated(self):
+        parsers = self.make_parsers() + \
+            [IntegratedTestKeywordParser("MY_BOOL_UNTERMINATED:", ParserKind.BOOLEAN_EXPR)]
+        try:
+            self.parse_test(parsers)
+            self.fail('expected exception')
+        except ValueError as e:
+            self.assertIn("Test has unterminated MY_BOOL_UNTERMINATED: lines", str(e))
+
+
     def test_custom(self):
         parsers = self.make_parsers()
         self.parse_test(parsers)
diff --git a/utils/llvm-build/llvmbuild/componentinfo.py b/utils/llvm-build/llvmbuild/componentinfo.py
index b384acd190ff53381aa98f4e1bc3965ef4df318c..0e9d08915fba9b9f137620d9d6a1eee70efe0625 100644
--- a/utils/llvm-build/llvmbuild/componentinfo.py
+++ b/utils/llvm-build/llvmbuild/componentinfo.py
@@ -2,7 +2,7 @@
 Descriptor objects for entities that are part of the LLVM project.
 """
 
-from __future__ import absolute_import
+from __future__ import absolute_import, print_function
 try:
     import configparser
 except:
@@ -461,8 +461,8 @@ def _read_components_from_parser(parser, path, subpath):
             info = type_class.parse(subpath,
                                     IniFormatParser(parser.items(section)))
         except TypeError:
-            print >>sys.stderr, "error: invalid component %r in %r: %s" % (
-                section, path, "unable to instantiate: %r" % type_name)
+            print("error: invalid component %r in %r: %s" % (
+                section, path, "unable to instantiate: %r" % type_name), file=sys.stderr)
             import traceback
             traceback.print_exc()
             raise SystemExit(1)
diff --git a/utils/llvm-build/llvmbuild/main.py b/utils/llvm-build/llvmbuild/main.py
index 4533c65063093a0a35c5b029617783cdb0d1bebb..9a9726424d4048aef3eeea5b289396ee8d45ad12 100644
--- a/utils/llvm-build/llvmbuild/main.py
+++ b/utils/llvm-build/llvmbuild/main.py
@@ -275,10 +275,9 @@ subdirectories = %s
             f.write("""\
 %s
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
@@ -545,10 +544,9 @@ subdirectories = %s
         f.write("""\
 %s
 #
-#                     The LLVM Compiler Infrastructure
-#
-# This file is distributed under the University of Illinois Open Source
-# License. See LICENSE.TXT for details.
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 #
 #===------------------------------------------------------------------------===#
 #
diff --git a/utils/llvm-compilers-check b/utils/llvm-compilers-check
index 4db8426ace88e1632a4aa4906615de04109f964b..f9bb96051cecae9111a6f69adebbe0cee9726f26 100755
--- a/utils/llvm-compilers-check
+++ b/utils/llvm-compilers-check
@@ -1,10 +1,9 @@
 #!/usr/bin/python3
 ##===- utils/llvmbuild - Build the LLVM project ----------------*-python-*-===##
 #
-#                     The LLVM Compiler Infrastructure
-#
-# This file is distributed under the University of Illinois Open Source
-# License. See LICENSE.TXT for details.
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 #
 ##===----------------------------------------------------------------------===##
 #
diff --git a/utils/llvm-gisel-cov.py b/utils/llvm-gisel-cov.py
index a74ed10f8642195e7320ce00faffe0fc7293cc2c..820fceab0d02a1c0483cb548c7bf8ded87f33c13 100644
--- a/utils/llvm-gisel-cov.py
+++ b/utils/llvm-gisel-cov.py
@@ -5,6 +5,7 @@ Summarize the information in the given coverage files.
 Emits the number of rules covered or the percentage of rules covered depending
 on whether --num-rules has been used to specify the total number of rules.
 """
+from __future__ import print_function
 
 import argparse
 import struct
@@ -59,9 +60,9 @@ def main():
   num_rules = dict(args.num_rules)
   for backend, rules_for_backend in covered_rules.items():
     if backend in num_rules:
-      print "%s: %3.2f%% of rules covered" % (backend, (float(len(rules_for_backend.keys())) / num_rules[backend]) * 100)
+      print("%s: %3.2f%% of rules covered" % (backend, float(len(rules_for_backend)) / num_rules[backend]) * 100))
     else:
-      print "%s: %d rules covered" % (backend, len(rules_for_backend.keys()))
+      print("%s: %d rules covered" % (backend, len(rules_for_backend)))
 
 if __name__ == '__main__':
   main()
diff --git a/utils/llvm-lit/llvm-lit.in b/utils/llvm-lit/llvm-lit.in
index de67b18e52c1d0d98440fabc17a9a8c9670c2fde..8700d15677d9b5001d59ca85e840089cbcea9732 100755
--- a/utils/llvm-lit/llvm-lit.in
+++ b/utils/llvm-lit/llvm-lit.in
@@ -1,4 +1,5 @@
 #!/usr/bin/env python
+# -*- coding: utf-8 -*-
 
 import os
 import sys
diff --git a/utils/llvmdo b/utils/llvmdo
index bcfc221c2b1b503af1face6804b96954853eb115..f54c752181d79e352ddc089319f14ae33e788833 100755
--- a/utils/llvmdo
+++ b/utils/llvmdo
@@ -1,10 +1,9 @@
 #!/bin/sh
 ##===- utils/llvmdo - Counts Lines Of Code -------------------*- Script -*-===##
 # 
-#                     The LLVM Compiler Infrastructure
-#
-# This file is distributed under the University of Illinois Open Source
-# License. See LICENSE.TXT for details.
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 # 
 ##===----------------------------------------------------------------------===##
 #
diff --git a/utils/llvmgrep b/utils/llvmgrep
index dc15da4961efcad5cd8c668056af99f1c7099ccd..28b2f95f979800aa6c127ded4da6bd160194b750 100755
--- a/utils/llvmgrep
+++ b/utils/llvmgrep
@@ -1,10 +1,9 @@
 #!/bin/sh
 ##===- utils/llvmgrep - Counts Lines Of Code -----------------*- Script -*-===##
 # 
-#                     The LLVM Compiler Infrastructure
-#
-# This file is distributed under the University of Illinois Open Source
-# License. See LICENSE.TXT for details.
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 # 
 ##===----------------------------------------------------------------------===##
 #
diff --git a/utils/not/not.cpp b/utils/not/not.cpp
index e35c884944ce645c9f0617de0671c5f3ceec1c3d..f00b9c8dde42b7081e2e130baaf522ceab2c9202 100644
--- a/utils/not/not.cpp
+++ b/utils/not/not.cpp
@@ -1,9 +1,8 @@
 //===- not.cpp - The 'not' testing tool -----------------------------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 // Usage:
diff --git a/utils/release/build_llvm_package.bat b/utils/release/build_llvm_package.bat
index 30767f6f3509b29a0288851aad8d5d0bfb3703d2..8038d798aa0aa890ef2a09ceb24d0a20f97e32cb 100755
--- a/utils/release/build_llvm_package.bat
+++ b/utils/release/build_llvm_package.bat
@@ -8,19 +8,26 @@ REM Usage: build_llvm_package.bat <revision>
 
 REM Prerequisites:
 REM
-REM   Visual Studio 2017, CMake, Ninja, SVN, GNUWin32,
+REM   Visual Studio 2017, CMake, Ninja, SVN, GNUWin32, SWIG, Python 3,
 REM   NSIS with the strlen_8192 patch,
 REM   Visual Studio 2017 SDK and Nuget (for the clang-format plugin),
 REM   Perl (for the OpenMP run-time).
+REM
+REM
+REM   For LLDB, SWIG version <= 3.0.8 needs to be used to work around
+REM   https://github.com/swig/swig/issues/769
 
 
 REM You need to modify the paths below:
 set vsdevcmd=C:\Program Files (x86)\Microsoft Visual Studio\2017\Professional\Common7\Tools\VsDevCmd.bat
 
+set python32_dir=C:\Users\%USERNAME%\AppData\Local\Programs\Python\Python36-32
+set python64_dir=C:\Users\%USERNAME%\AppData\Local\Programs\Python\Python36
+
 set revision=%1
 set branch=trunk
-set package_version=8.0.0-r%revision%
-set clang_format_vs_version=8.0.0.%revision%
+set package_version=9.0.0-r%revision%
+set clang_format_vs_version=9.0.0.%revision%
 set build_dir=llvm_package_%revision%
 
 echo Branch: %branch%
@@ -41,12 +48,13 @@ svn.exe export -r %revision% http://llvm.org/svn/llvm-project/clang-tools-extra/
 svn.exe export -r %revision% http://llvm.org/svn/llvm-project/lld/%branch% llvm/tools/lld || exit /b
 svn.exe export -r %revision% http://llvm.org/svn/llvm-project/compiler-rt/%branch% llvm/projects/compiler-rt || exit /b
 svn.exe export -r %revision% http://llvm.org/svn/llvm-project/openmp/%branch% llvm/projects/openmp || exit /b
+svn.exe export -r %revision% http://llvm.org/svn/llvm-project/lldb/%branch% llvm/tools/lldb || exit /b
 
 
 REM Setting CMAKE_CL_SHOWINCLUDES_PREFIX to work around PR27226.
-set cmake_flags=-DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_ASSERTIONS=ON -DLLVM_USE_CRT_RELEASE=MT -DLLVM_INSTALL_TOOLCHAIN_ONLY=ON -DCLANG_FORMAT_VS_VERSION=%clang_format_vs_version% -DPACKAGE_VERSION=%package_version% -DCMAKE_CL_SHOWINCLUDES_PREFIX="Note: including file: "
+set cmake_flags=-DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_ASSERTIONS=ON -DLLVM_INSTALL_TOOLCHAIN_ONLY=ON -DCMAKE_INSTALL_UCRT_LIBRARIES=ON -DCLANG_FORMAT_VS_VERSION=%clang_format_vs_version% -DPACKAGE_VERSION=%package_version% -DLLDB_RELOCATABLE_PYTHON=1 -DLLDB_TEST_COMPILER=%cd%\build32_stage0\bin\clang.exe -DCMAKE_CL_SHOWINCLUDES_PREFIX="Note: including file: "
 
-REM TODO: Run all tests, including lld and compiler-rt.
+REM TODO: Run the "check-all" tests.
 
 set "VSCMD_START_DIR=%CD%"
 call "%vsdevcmd%" -arch=x86
@@ -55,20 +63,24 @@ set CXX=
 mkdir build32_stage0
 cd build32_stage0
 REM Work around VS2017 bug by using MinSizeRel.
-cmake -GNinja %cmake_flags% -DCMAKE_BUILD_TYPE=MinSizeRel ..\llvm || exit /b
+cmake -GNinja %cmake_flags% -DPYTHON_HOME=%python32_dir% -DCMAKE_BUILD_TYPE=MinSizeRel ..\llvm || exit /b
 ninja all || ninja all || ninja all || exit /b
 ninja check || ninja check || ninja check || exit /b
-ninja check-clang || ninja check-clang || ninja check-clang ||  exit /b
+ninja check-clang || ninja check-clang || ninja check-clang || exit /b
+ninja check-lld || ninja check-lld || ninja check-lld || exit /b
+ninja check-sanitizer || ninja check-sanitizer || ninja check-sanitizer || exit /b
 cd..
 
 mkdir build32
 cd build32
 set CC=..\build32_stage0\bin\clang-cl
 set CXX=..\build32_stage0\bin\clang-cl
-cmake -GNinja %cmake_flags% ..\llvm || exit /b
+cmake -GNinja %cmake_flags% -DPYTHON_HOME=%python32_dir% ..\llvm || exit /b
 ninja all || ninja all || ninja all || exit /b
 ninja check || ninja check || ninja check || exit /b
-ninja check-clang || ninja check-clang || ninja check-clang ||  exit /b
+ninja check-clang || ninja check-clang || ninja check-clang || exit /b
+ninja check-lld || ninja check-lld || ninja check-lld || exit /b
+ninja check-sanitizer || ninja check-sanitizer || ninja check-sanitizer || exit /b
 ninja package || exit /b
 cd ..
 
@@ -77,7 +89,7 @@ mkdir build_vsix
 cd build_vsix
 set CC=..\build32_stage0\bin\clang-cl
 set CXX=..\build32_stage0\bin\clang-cl
-cmake -GNinja %cmake_flags% -DBUILD_CLANG_FORMAT_VS_PLUGIN=ON ..\llvm || exit /b
+cmake -GNinja %cmake_flags% -DLLVM_USE_CRT_RELEASE=MT -DBUILD_CLANG_FORMAT_VS_PLUGIN=ON -DPYTHON_HOME=%python32_dir% ..\llvm || exit /b
 ninja clang_format_vsix || exit /b
 copy ..\llvm\tools\clang\tools\clang-format-vs\ClangFormat\bin\Release\ClangFormat.vsix ClangFormat-r%revision%.vsix
 cd ..
@@ -90,19 +102,23 @@ set CXX=
 mkdir build64_stage0
 cd build64_stage0
 REM Work around VS2017 bug by using MinSizeRel.
-cmake -GNinja %cmake_flags% -DCMAKE_BUILD_TYPE=MinSizeRel ..\llvm || exit /b
+cmake -GNinja %cmake_flags% -DPYTHON_HOME=%python64_dir% -DCMAKE_BUILD_TYPE=MinSizeRel ..\llvm || exit /b
 ninja all || ninja all || ninja all || exit /b
 ninja check || ninja check || ninja check || exit /b
-ninja check-clang || ninja check-clang || ninja check-clang ||  exit /b
+ninja check-clang || ninja check-clang || ninja check-clang || exit /b
+ninja check-lld || ninja check-lld || ninja check-lld || exit /b
+ninja check-sanitizer || ninja check-sanitizer || ninja check-sanitizer || exit /b
 cd..
 
 mkdir build64
 cd build64
 set CC=..\build64_stage0\bin\clang-cl
 set CXX=..\build64_stage0\bin\clang-cl
-cmake -GNinja %cmake_flags% ..\llvm || exit /b
+cmake -GNinja %cmake_flags% -DPYTHON_HOME=%python64_dir% ..\llvm || exit /b
 ninja all || ninja all || ninja all || exit /b
 ninja check || ninja check || ninja check || exit /b
-ninja check-clang || ninja check-clang || ninja check-clang ||  exit /b
+ninja check-clang || ninja check-clang || ninja check-clang || exit /b
+ninja check-lld || ninja check-lld || ninja check-lld || exit /b
+ninja check-sanitizer || ninja check-sanitizer || ninja check-sanitizer || exit /b
 ninja package || exit /b
 cd ..
diff --git a/utils/release/export.sh b/utils/release/export.sh
index d978055f01837834fbf58c94340077b8f8688e39..9cd64a3d255fd2f197ba233e5172bee8bde13c3f 100755
--- a/utils/release/export.sh
+++ b/utils/release/export.sh
@@ -1,10 +1,9 @@
 #!/bin/sh
 #===-- tag.sh - Tag the LLVM release candidates ----------------------------===#
 #
-#                     The LLVM Compiler Infrastructure
-#
-# This file is distributed under the University of Illinois Open Source
-# License.
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 #
 #===------------------------------------------------------------------------===#
 #
diff --git a/utils/release/findRegressions-nightly.py b/utils/release/findRegressions-nightly.py
index ddf89835e2c2d9631972dbbe39aeea4e2b7eee8a..e7e13b08381241b92ed3a8226418160571a0fff1 100755
--- a/utils/release/findRegressions-nightly.py
+++ b/utils/release/findRegressions-nightly.py
@@ -1,4 +1,6 @@
 #!/usr/bin/env python
+from __future__ import print_function
+
 import re, string, sys, os, time
 
 DEBUG = 0
@@ -22,45 +24,45 @@ def parse(file):
   fname = ''
   for t in r:
     if DEBUG:
-      print t
+      print(t)
     if t[0] == 'PASS' or t[0] == 'FAIL' :
       tmp = t[2].split(testDirName)
       
       if DEBUG:
-        print tmp
+        print(tmp)
       
       if len(tmp) == 2:
         fname = tmp[1].strip('\r\n')
       else:
         fname = tmp[0].strip('\r\n')
       
-      if not test.has_key(fname) :
+      if fname not in test :
         test[fname] = {}
       
       for k in test:
         test[fname][k] = 'NA'
         test[fname][t[1]] = t[0]
         if DEBUG:
-          print test[fname][t[1]]
+          print(test[fname][t[1]])
     else :
       try:
         n = t[0].split('RESULT-')[1]
         
         if DEBUG:
-          print n;
+          print(n);
         
         if n == 'llc' or n == 'jit-comptime' or n == 'compile':
           test[fname][tp + n] = float(t[2].split(' ')[2])
           if DEBUG:
-            print test[fname][tp + n]
+            print(test[fname][tp + n])
         
         elif n.endswith('-time') :
             test[fname][exp + n] = float(t[2].strip('\r\n'))
             if DEBUG:
-              print test[fname][exp + n]
+              print(test[fname][exp + n])
         
         else :
-          print "ERROR!"
+          print("ERROR!")
           sys.exit(1)
       
       except:
@@ -73,52 +75,52 @@ def diffResults(d_old, d_new):
 
   for t in sorted(d_old.keys()) :
     if DEBUG:
-      print t
+      print(t)
         
-    if d_new.has_key(t) :
+    if t in d_new :
     
       # Check if the test passed or failed.
       for x in test:
-        if d_old[t].has_key(x):
-          if d_new[t].has_key(x):
+        if x in d_old[t]:
+          if x in d_new[t]:
             if d_old[t][x] == 'PASS':
               if d_new[t][x] != 'PASS':
-                print t + " *** REGRESSION (" + x + ")\n"
+                print(t + " *** REGRESSION (" + x + ")\n")
             else:
               if d_new[t][x] == 'PASS':
-                print t + " * NEW PASS (" + x + ")\n"
+                print(t + " * NEW PASS (" + x + ")\n")
                 
           else :
-            print t + "*** REGRESSION (" + x + ")\n"
+            print(t + "*** REGRESSION (" + x + ")\n")
         
         # For execution time, if there is no result, its a fail.
         for x in exectime:
-          if d_old[t].has_key(tp + x):
-            if not d_new[t].has_key(tp + x):
-              print t + " *** REGRESSION (" + tp + x + ")\n"
+          if tp + x in d_old[t]:
+            if tp + x not in d_new[t]:
+              print(t + " *** REGRESSION (" + tp + x + ")\n")
                 
           else :
-            if d_new[t].has_key(tp + x):
-              print t + " * NEW PASS (" + tp + x + ")\n"
+            if tp + x in d_new[t]:
+              print(t + " * NEW PASS (" + tp + x + ")\n")
 
        
         for x in comptime:
-          if d_old[t].has_key(exp + x):
-            if not d_new[t].has_key(exp + x):
-              print t + " *** REGRESSION (" + exp + x + ")\n"
+          if exp + x in d_old[t]:
+            if exp + x not in d_new[t]:
+              print(t + " *** REGRESSION (" + exp + x + ")\n")
                 
           else :
-            if d_new[t].has_key(exp + x):
-              print t + " * NEW PASS (" + exp + x + ")\n"
+            if exp + x in d_new[t]:
+              print(t + " * NEW PASS (" + exp + x + ")\n")
               
     else :
-      print t + ": Removed from test-suite.\n"
+      print(t + ": Removed from test-suite.\n")
     
 
 #Main
 if len(sys.argv) < 3 :
-    print 'Usage:', sys.argv[0], \
-          '<old log> <new log>'
+    print('Usage:', sys.argv[0], \
+          '<old log> <new log>')
     sys.exit(-1)
 
 d_old = parse(sys.argv[1])
diff --git a/utils/release/findRegressions-simple.py b/utils/release/findRegressions-simple.py
index 8d3b4cfca29b8faa1ded45b772ae11453a4e3608..7bd1523b58faa31c63fcfaaa6b1fbd1d51ae59e6 100755
--- a/utils/release/findRegressions-simple.py
+++ b/utils/release/findRegressions-simple.py
@@ -1,4 +1,6 @@
 #!/usr/bin/env python
+
+from __future__ import print_function
 import re, string, sys, os, time, math
 
 DEBUG = 0
@@ -18,20 +20,20 @@ def parse(file):
   fname = ''
   for t in r:
     if DEBUG:
-      print t
+      print(t)
 
     if t[0] == 'PASS' or t[0] == 'FAIL' :
       tmp = t[2].split('llvm-test/')
       
       if DEBUG:
-        print tmp
+        print(tmp)
 
       if len(tmp) == 2:
         fname = tmp[1].strip('\r\n')
       else:
         fname = tmp[0].strip('\r\n')
 
-      if not test.has_key(fname):
+      if fname not in test:
         test[fname] = {}
 
       test[fname][t[1] + ' state'] = t[0]
@@ -41,7 +43,7 @@ def parse(file):
         n = t[0].split('RESULT-')[1]
 
         if DEBUG:
-          print "n == ", n;
+          print("n == ", n);
         
         if n == 'compile-success':
           test[fname]['compile time'] = float(t[2].split('program')[1].strip('\r\n'))
@@ -49,7 +51,7 @@ def parse(file):
         elif n == 'exec-success':
           test[fname]['exec time'] = float(t[2].split('program')[1].strip('\r\n'))
           if DEBUG:
-            print test[fname][string.replace(n, '-success', '')]
+            print(test[fname][string.replace(n, '-success', '')])
 
         else :
           # print "ERROR!"
@@ -71,16 +73,16 @@ def diffResults(d_old, d_new):
     passes[x] = ''
 
   for t in sorted(d_old.keys()) :
-    if d_new.has_key(t):
+    if t in d_new:
 
       # Check if the test passed or failed.
       for x in ['compile state', 'compile time', 'exec state', 'exec time']:
 
-        if not d_old[t].has_key(x) and not d_new[t].has_key(x):
+        if x not in d_old[t] and x not in d_new[t]:
           continue
 
-        if d_old[t].has_key(x):
-          if d_new[t].has_key(x):
+        if x in d_old[t]:
+          if x in d_new[t]:
 
             if d_old[t][x] == 'PASS':
               if d_new[t][x] != 'PASS':
@@ -96,11 +98,11 @@ def diffResults(d_old, d_new):
           continue
 
         # For execution time, if there is no result it's a fail.
-        if not d_old[t].has_key(x) and not d_new[t].has_key(x):
+        if x not in d_old[t] and x not in d_new[t]:
           continue
-        elif not d_new[t].has_key(x):
+        elif x not in d_new[t]:
           regressions[x] += t + "\n"
-        elif not d_old[t].has_key(x):
+        elif x not in d_old[t]:
           passes[x] += t + "\n"
 
         if math.isnan(d_old[t][x]) and math.isnan(d_new[t][x]):
@@ -120,36 +122,36 @@ def diffResults(d_old, d_new):
       removed += t + "\n"
 
   if len(regressions['compile state']) != 0:
-    print 'REGRESSION: Compilation Failed'
-    print regressions['compile state']
+    print('REGRESSION: Compilation Failed')
+    print(regressions['compile state'])
 
   if len(regressions['exec state']) != 0:
-    print 'REGRESSION: Execution Failed'
-    print regressions['exec state']
+    print('REGRESSION: Execution Failed')
+    print(regressions['exec state'])
 
   if len(regressions['compile time']) != 0:
-    print 'REGRESSION: Compilation Time'
-    print regressions['compile time']
+    print('REGRESSION: Compilation Time')
+    print(regressions['compile time'])
 
   if len(regressions['exec time']) != 0:
-    print 'REGRESSION: Execution Time'
-    print regressions['exec time']
+    print('REGRESSION: Execution Time')
+    print(regressions['exec time'])
 
   if len(passes['compile state']) != 0:
-    print 'NEW PASSES: Compilation'
-    print passes['compile state']
+    print('NEW PASSES: Compilation')
+    print(passes['compile state'])
 
   if len(passes['exec state']) != 0:
-    print 'NEW PASSES: Execution'
-    print passes['exec state']
+    print('NEW PASSES: Execution')
+    print(passes['exec state'])
 
   if len(removed) != 0:
-    print 'REMOVED TESTS'
-    print removed
+    print('REMOVED TESTS')
+    print(removed)
 
 # Main
 if len(sys.argv) < 3 :
-  print 'Usage:', sys.argv[0], '<old log> <new log>'
+  print('Usage:', sys.argv[0], '<old log> <new log>')
   sys.exit(-1)
 
 d_old = parse(sys.argv[1])
diff --git a/utils/release/merge-git.sh b/utils/release/merge-git.sh
index 81d577db9d62499de19ca7124b923a792dffba05..33162f6c8131da083e2e88da8c0935365bc6ebcb 100755
--- a/utils/release/merge-git.sh
+++ b/utils/release/merge-git.sh
@@ -1,10 +1,9 @@
 #!/bin/bash
 #===-- merge-git.sh - Merge commit to the stable branch --------------------===#
 #
-#                     The LLVM Compiler Infrastructure
-#
-# This file is distributed under the University of Illinois Open Source
-# License.
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 #
 #===------------------------------------------------------------------------===#
 #
diff --git a/utils/release/merge-request.sh b/utils/release/merge-request.sh
index 333b9043af356a8faf1aa16f5ae07a1d6d4088ee..6a4ee16d788ab8bfd9017b7c32f85aa5a462d907 100755
--- a/utils/release/merge-request.sh
+++ b/utils/release/merge-request.sh
@@ -1,10 +1,9 @@
 # !/bin/bash
 #===-- merge-request.sh  ---------------------------------------------------===#
 #
-#                     The LLVM Compiler Infrastructure
-#
-# This file is distributed under the University of Illinois Open Source
-# License.
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 #
 #===------------------------------------------------------------------------===#
 #
@@ -101,6 +100,9 @@ case $stable_version in
   7.0)
     release_metabug="39106"
     ;;
+  8.0)
+    release_metabug="40331"
+    ;;
   *)
     echo "error: invalid stable version"
     exit 1
diff --git a/utils/release/merge.sh b/utils/release/merge.sh
index 698353d691581d9ea2faeb5e0f0ea87a357efe4e..ad289b6212ce701148f1cafc7a5185d828461c23 100755
--- a/utils/release/merge.sh
+++ b/utils/release/merge.sh
@@ -1,10 +1,9 @@
 #!/bin/sh
 #===-- merge.sh - Test the LLVM release candidates -------------------------===#
 #
-#                     The LLVM Compiler Infrastructure
-#
-# This file is distributed under the University of Illinois Open Source
-# License.
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 #
 #===------------------------------------------------------------------------===#
 #
diff --git a/utils/release/tag.sh b/utils/release/tag.sh
index 100831d14bd50dae56db49ae34d7c64c4d6f5c3f..01922f405ea1e9a4eb368fca3d1aed6cc5003088 100755
--- a/utils/release/tag.sh
+++ b/utils/release/tag.sh
@@ -1,10 +1,9 @@
 #!/bin/bash
 #===-- tag.sh - Tag the LLVM release candidates ----------------------------===#
 #
-#                     The LLVM Compiler Infrastructure
-#
-# This file is distributed under the University of Illinois Open Source
-# License.
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 #
 #===------------------------------------------------------------------------===#
 #
diff --git a/utils/release/test-release.sh b/utils/release/test-release.sh
index 440dee53c1b76f1a9d2758e70664ba0217b6ab0b..558e01d2eabc84ae2394efffa43d0e96968352b4 100755
--- a/utils/release/test-release.sh
+++ b/utils/release/test-release.sh
@@ -1,10 +1,9 @@
 #!/usr/bin/env bash
 #===-- test-release.sh - Test the LLVM release candidates ------------------===#
 #
-#                     The LLVM Compiler Infrastructure
-#
-# This file is distributed under the University of Illinois Open Source
-# License.
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 #
 #===------------------------------------------------------------------------===#
 #
@@ -18,6 +17,7 @@ if [ "$System" = "FreeBSD" ]; then
 else
     MAKE=make
 fi
+generator="Unix Makefiles"
 
 # Base SVN URL for the sources.
 Base_url="http://llvm.org/svn/llvm-project"
@@ -58,6 +58,7 @@ function usage() {
     echo " -test-asserts        Test with asserts on. [default: no]"
     echo " -no-compare-files    Don't test that phase 2 and 3 files are identical."
     echo " -use-gzip            Use gzip instead of xz."
+    echo " -use-ninja           Use ninja instead of make/gmake."
     echo " -configure-flags FLAGS  Extra flags to pass to the configure step."
     echo " -svn-path DIR        Use the specified DIR instead of a release."
     echo "                      For example -svn-path trunk or -svn-path branches/release_37"
@@ -112,6 +113,10 @@ while [ $# -gt 0 ]; do
                 NumJobs="$1"
             fi
             ;;
+        -use-ninja )
+            MAKE=ninja
+            generator=Ninja
+            ;;
         -build-dir | --build-dir | -builddir | --builddir )
             shift
             BuildDir="$1"
@@ -278,6 +283,8 @@ if [ "$System" != "Darwin" ]; then
   check_program_exists 'objdump'
 fi
 
+check_program_exists ${MAKE}
+
 # Make sure that the URLs are valid.
 function check_valid_urls() {
     for proj in $projects ; do
@@ -366,12 +373,12 @@ function configure_llvmCore() {
     echo "# Configuring llvm $Release-$RC $Flavor"
 
     echo "#" env CC="$c_compiler" CXX="$cxx_compiler" \
-        cmake -G "Unix Makefiles" \
+        cmake -G "$generator" \
         -DCMAKE_BUILD_TYPE=$BuildType -DLLVM_ENABLE_ASSERTIONS=$Assertions \
         $ExtraConfigureFlags $BuildDir/llvm.src \
         2>&1 | tee $LogDir/llvm.configure-Phase$Phase-$Flavor.log
     env CC="$c_compiler" CXX="$cxx_compiler" \
-        cmake -G "Unix Makefiles" \
+        cmake -G "$generator" \
         -DCMAKE_BUILD_TYPE=$BuildType -DLLVM_ENABLE_ASSERTIONS=$Assertions \
         $ExtraConfigureFlags $BuildDir/llvm.src \
         2>&1 | tee $LogDir/llvm.configure-Phase$Phase-$Flavor.log
@@ -385,16 +392,20 @@ function build_llvmCore() {
     ObjDir="$3"
     DestDir="$4"
 
+    Verbose="VERBOSE=1"
+    if [ ${MAKE} = 'ninja' ]; then
+      Verbose="-v"
+    fi
+
     cd $ObjDir
     echo "# Compiling llvm $Release-$RC $Flavor"
-    echo "# ${MAKE} -j $NumJobs VERBOSE=1"
-    ${MAKE} -j $NumJobs VERBOSE=1 \
+    echo "# ${MAKE} -j $NumJobs $Verbose"
+    ${MAKE} -j $NumJobs $Verbose \
         2>&1 | tee $LogDir/llvm.make-Phase$Phase-$Flavor.log
 
     echo "# Installing llvm $Release-$RC $Flavor"
     echo "# ${MAKE} install"
-    ${MAKE} install \
-        DESTDIR="${DestDir}" \
+    DESTDIR="${DestDir}" ${MAKE} install \
         2>&1 | tee $LogDir/llvm.install-Phase$Phase-$Flavor.log
     cd $BuildDir
 }
@@ -404,8 +415,15 @@ function test_llvmCore() {
     Flavor="$2"
     ObjDir="$3"
 
+    KeepGoing="-k"
+    if [ ${MAKE} = 'ninja' ]; then
+      # Ninja doesn't have a documented "keep-going-forever" mode, we need to
+      # set a limit on how many jobs can fail before we give up.
+      KeepGoing="-k 100"
+    fi
+
     cd $ObjDir
-    if ! ( ${MAKE} -j $NumJobs -k check-all \
+    if ! ( ${MAKE} -j $NumJobs $KeepGoing check-all \
         2>&1 | tee $LogDir/llvm.check-Phase$Phase-$Flavor.log ) ; then
       deferred_error $Phase $Flavor "check-all failed"
     fi
@@ -413,8 +431,9 @@ function test_llvmCore() {
     if [ $do_test_suite = 'yes' ]; then
       cd $TestSuiteBuildDir
       env CC="$c_compiler" CXX="$cxx_compiler" \
-          cmake $TestSuiteSrcDir -DTEST_SUITE_LIT=$Lit
-      if ! ( ${MAKE} -j $NumJobs -k check \
+          cmake $TestSuiteSrcDir -G "$generator" -DTEST_SUITE_LIT=$Lit
+
+      if ! ( ${MAKE} -j $NumJobs $KeepGoing check \
           2>&1 | tee $LogDir/llvm.check-Phase$Phase-$Flavor.log ) ; then
         deferred_error $Phase $Flavor "test suite failed"
       fi
diff --git a/utils/schedcover.py b/utils/schedcover.py
index 8c0aeeb32b53e5ac90c42a5eb87f0e36c70aca73..9532f1b554bd9726a0b37e47ad618841731652b2 100644
--- a/utils/schedcover.py
+++ b/utils/schedcover.py
@@ -39,7 +39,7 @@ def display():
     ordered_table  = sorted(table.items(), key=operator.itemgetter(0))
     ordered_models = ["itinerary", "default"]
     ordered_models.extend(sorted(models))
-    ordered_models = filter(filter_model, ordered_models)
+    ordered_models = [m for m in ordered_models if filter_model(m)]
 
     # print header
     sys.stdout.write("instruction")
diff --git a/utils/shuffle_fuzz.py b/utils/shuffle_fuzz.py
index eac34422b3fb5f67609db02ebf9cf3326c16f7d9..2d86cc0a28fa399f994e0436249b31c926d34087 100755
--- a/utils/shuffle_fuzz.py
+++ b/utils/shuffle_fuzz.py
@@ -13,6 +13,8 @@ set of transforms you want to test, and run the program. If it crashes, it found
 a bug.
 """
 
+from __future__ import print_function
+
 import argparse
 import itertools
 import random
@@ -105,22 +107,22 @@ def main():
                        else random.choice(range(shuffle_range))
                     for _ in itertools.repeat(None, width)]
                    for _ in itertools.repeat(None, args.max_shuffle_height - i)]
-                  for i in xrange(args.max_shuffle_height)]
+                  for i in range(args.max_shuffle_height)]
 
   if args.verbose:
     # Print out the shuffle sequence in a compact form.
-    print >>sys.stderr, ('Testing shuffle sequence "%s" (v%d%s):' %
-                         (args.seed, width, element_type))
+    print(('Testing shuffle sequence "%s" (v%d%s):' %
+                         (args.seed, width, element_type)), file=sys.stderr)
     for i, shuffles in enumerate(shuffle_tree):
-      print >>sys.stderr, '  tree level %d:' % (i,)
+      print('  tree level %d:' % (i,), file=sys.stderr)
       for j, s in enumerate(shuffles):
-        print >>sys.stderr, '    shuffle %d: %s' % (j, s)
-    print >>sys.stderr, ''
+        print('    shuffle %d: %s' % (j, s), file=sys.stderr)
+    print('', file=sys.stderr)
 
   # Symbolically evaluate the shuffle tree.
   inputs = [[int(j % element_modulus)
-             for j in xrange(i * width + 1, (i + 1) * width + 1)]
-            for i in xrange(args.max_shuffle_height + 1)]
+             for j in range(i * width + 1, (i + 1) * width + 1)]
+            for i in range(args.max_shuffle_height + 1)]
   results = inputs
   for shuffles in shuffle_tree:
     results = [[((results[i] if j < width else results[i + 1])[j % width]
@@ -128,15 +130,15 @@ def main():
                 for j in s]
                for i, s in enumerate(shuffles)]
   if len(results) != 1:
-    print >>sys.stderr, 'ERROR: Bad results: %s' % (results,)
+    print('ERROR: Bad results: %s' % (results,), file=sys.stderr)
     sys.exit(1)
   result = results[0]
 
   if args.verbose:
-    print >>sys.stderr, 'Which transforms:'
-    print >>sys.stderr, '  from: %s' % (inputs,)
-    print >>sys.stderr, '  into: %s' % (result,)
-    print >>sys.stderr, ''
+    print('Which transforms:', file=sys.stderr)
+    print('  from: %s' % (inputs,), file=sys.stderr)
+    print('  into: %s' % (result,), file=sys.stderr)
+    print('', file=sys.stderr)
 
   # The IR uses silly names for floating point types. We also need a same-size
   # integer type.
@@ -150,25 +152,25 @@ def main():
 
   # Now we need to generate IR for the shuffle function.
   subst = {'N': width, 'T': element_type, 'IT': integral_element_type}
-  print """
+  print("""
 define internal fastcc <%(N)d x %(T)s> @test(%(arguments)s) noinline nounwind {
 entry:""" % dict(subst,
                  arguments=', '.join(
                      ['<%(N)d x %(T)s> %%s.0.%(i)d' % dict(subst, i=i)
-                      for i in xrange(args.max_shuffle_height + 1)]))
+                      for i in range(args.max_shuffle_height + 1)])))
 
   for i, shuffles in enumerate(shuffle_tree):
    for j, s in enumerate(shuffles):
-    print """
+    print("""
   %%s.%(next_i)d.%(j)d = shufflevector <%(N)d x %(T)s> %%s.%(i)d.%(j)d, <%(N)d x %(T)s> %%s.%(i)d.%(next_j)d, <%(N)d x i32> <%(S)s>
 """.strip('\n') % dict(subst, i=i, next_i=i + 1, j=j, next_j=j + 1,
                        S=', '.join(['i32 ' + (str(si) if si != -1 else 'undef')
-                                    for si in s]))
+                                    for si in s])))
 
-  print """
+  print("""
   ret <%(N)d x %(T)s> %%s.%(i)d.0
 }
-""" % dict(subst, i=len(shuffle_tree))
+""" % dict(subst, i=len(shuffle_tree)))
 
   # Generate some string constants that we can use to report errors.
   for i, r in enumerate(result):
@@ -176,24 +178,24 @@ entry:""" % dict(subst,
       s = ('FAIL(%(seed)s): lane %(lane)d, expected %(result)d, found %%d\n\\0A' %
            {'seed': args.seed, 'lane': i, 'result': r})
       s += ''.join(['\\00' for _ in itertools.repeat(None, 128 - len(s) + 2)])
-      print """
+      print("""
 @error.%(i)d = private unnamed_addr global [128 x i8] c"%(s)s"
-""".strip() % {'i': i, 's': s}
+""".strip() % {'i': i, 's': s})
 
   # Define a wrapper function which is marked 'optnone' to prevent
   # interprocedural optimizations from deleting the test.
-  print """
+  print("""
 define internal fastcc <%(N)d x %(T)s> @test_wrapper(%(arguments)s) optnone noinline {
   %%result = call fastcc <%(N)d x %(T)s> @test(%(arguments)s)
   ret <%(N)d x %(T)s> %%result
 }
 """ % dict(subst,
            arguments=', '.join(['<%(N)d x %(T)s> %%s.%(i)d' % dict(subst, i=i)
-                                for i in xrange(args.max_shuffle_height + 1)]))
+                                for i in range(args.max_shuffle_height + 1)])))
 
   # Finally, generate a main function which will trap if any lanes are mapped
   # incorrectly (in an observable way).
-  print """
+  print("""
 define i32 @main() {
 entry:
   ; Create a scratch space to print error messages.
@@ -212,18 +214,18 @@ entry:
                  '(<%(N)d x %(IT)s> <%(input)s> to <%(N)d x %(T)s>)' %
                  dict(subst, input=', '.join(['%(IT)s %(i)d' % dict(subst, i=i)
                                               for i in input])))
-                for input in inputs]))
+                for input in inputs])))
 
   # Test that each non-undef result lane contains the expected value.
   for i, r in enumerate(result):
     if r == -1:
-      print """
+      print("""
 test.%(i)d:
   ; Skip this lane, its value is undef.
   br label %%test.%(next_i)d
-""" % dict(subst, i=i, next_i=i + 1)
+""" % dict(subst, i=i, next_i=i + 1))
     else:
-      print """
+      print("""
 test.%(i)d:
   %%v.%(i)d = extractelement <%(N)d x %(IT)s> %%v.cast, i32 %(i)d
   %%cmp.%(i)d = icmp ne %(IT)s %%v.%(i)d, %(r)d
@@ -238,9 +240,9 @@ die.%(i)d:
   call i32 @write(i32 2, i8* %%str.ptr, i32 %%length.%(i)d)
   call void @llvm.trap()
   unreachable
-""" % dict(subst, i=i, next_i=i + 1, r=r)
+""" % dict(subst, i=i, next_i=i + 1, r=r))
 
-  print """
+  print("""
 test.%d:
   ret i32 0
 }
@@ -249,7 +251,7 @@ declare i32 @strlen(i8*)
 declare i32 @write(i32, i8*, i32)
 declare i32 @sprintf(i8*, i8*, ...)
 declare void @llvm.trap() noreturn nounwind
-""" % (len(result),)
+""" % (len(result),))
 
 if __name__ == '__main__':
   main()
diff --git a/utils/shuffle_select_fuzz_tester.py b/utils/shuffle_select_fuzz_tester.py
index 88d8d75f7b9894f6669098a8adfcc0c1c775568f..6b2f94a693533db529c20c42ab71c553323b5b44 100644
--- a/utils/shuffle_select_fuzz_tester.py
+++ b/utils/shuffle_select_fuzz_tester.py
@@ -13,6 +13,7 @@ Take the output IR printed to stdout, compile it to an executable using whatever
 set of transforms you want to test, and run the program. If it crashes, it found
 a bug (an error message with the expected and actual result is printed).
 """
+from __future__ import print_function
 
 import random
 import uuid
@@ -145,7 +146,7 @@ class ShufInstr(Instruction):
 
   def calc_value(self):
     if self.value != None:
-      print 'Trying to calculate the value of a shuffle instruction twice'
+      print('Trying to calculate the value of a shuffle instruction twice')
       exit(1)
 
     result = []
@@ -179,7 +180,7 @@ class SelectInstr(Instruction):
 
   def calc_value(self):
     if self.value != None:
-      print 'Trying to calculate the value of a select instruction twice'
+      print('Trying to calculate the value of a select instruction twice')
       exit(1)
 
     result = []
@@ -343,7 +344,7 @@ def main():
                       help='Choose specific number of vector elements to be tested. (default: random)')
   args = parser.parse_args()
 
-  print '; The seed used for this test is ' + args.seed
+  print('; The seed used for this test is ' + args.seed)
 
   assert args.min_num_inputs < args.max_num_inputs , "Minimum value greater than maximum."
   assert args.type in [None, 'i8', 'i16', 'i32', 'i64', 'f32', 'f64'], "Illegal type."
@@ -362,14 +363,14 @@ def main():
 
   # print the actual test function by dumping the generated instructions.
   insts_str = ''.join([inst.dump() for inst in insts])
-  print test_template.format(ty = ty.dump(), inputs = inputs_str,
-                             instructions = insts_str, last_name = res.name)
+  print(test_template.format(ty = ty.dump(), inputs = inputs_str,
+                             instructions = insts_str, last_name = res.name))
 
   # Print the error message templates as global strings
   for i in range(len(res.value)):
     pad = ''.join(['\\00']*(31 - len(str(i)) - len(str(res.value[i]))))
-    print error_template.format(lane = str(i), exp = str(res.value[i]),
-                                padding = pad)
+    print(error_template.format(lane = str(i), exp = str(res.value[i]),
+                                padding = pad))
 
   # Prepare the runtime checks and failure handlers.
   scalar_ty = ty.get_scalar_type()
@@ -395,7 +396,7 @@ def main():
   inputs_values = [', '.join([scalar_ty.dump() + ' ' + str(i) for i in inp]) for inp in inputs_values]
   inputs = ', '.join([ty.dump() + ' <' + inp + '>' for inp in inputs_values])
 
-  print main_template.format(ty = ty.dump(), inputs = inputs, check_die = check_die)
+  print(main_template.format(ty = ty.dump(), inputs = inputs, check_die = check_die))
 
 
 if __name__ == '__main__':
diff --git a/utils/unicode-case-fold.py b/utils/unicode-case-fold.py
index 98c56839c6c8ff2db03c4e3a4f565347728ce5e0..ad8265ba7bc1b65846ce62c24048eeade70f45a6 100755
--- a/utils/unicode-case-fold.py
+++ b/utils/unicode-case-fold.py
@@ -17,9 +17,15 @@ Right now this generates a function which implements simple case folding (C+S
 entries).
 """
 
+from __future__ import print_function
+
 import sys
 import re
-import urllib2
+try:
+    from urllib.request import urlopen
+except ImportError:
+    from urllib2 import urlopen
+
 
 # This variable will body of the mappings function
 body = ""
@@ -93,7 +99,7 @@ def dump_block(b):
     body += pattern.format(last, stride(b), modulo, shift(b[0]))
 
 current_block = []
-f = urllib2.urlopen(sys.argv[1])
+f = urlopen(sys.argv[1])
 for m in mappings(f):
     if len(current_block) == 0:
         current_block.append(m)
@@ -116,22 +122,22 @@ f.close()
 
 dump_block(current_block)
 
-print '//===---------- Support/UnicodeCaseFold.cpp -------------------------------===//'
-print '//'
-print '// This file was generated by utils/unicode-case-fold.py from the Unicode'
-print '// case folding database at'
-print '//   ', sys.argv[1]
-print '//'
-print '// To regenerate this file, run:'
-print '//   utils/unicode-case-fold.py \\'
-print '//     "{}" \\'.format(sys.argv[1])
-print '//     > lib/Support/UnicodeCaseFold.cpp'
-print '//'
-print '//===----------------------------------------------------------------------===//'
-print ''
-print '#include "llvm/Support/Unicode.h"'
-print ''
-print "int llvm::sys::unicode::foldCharSimple(int C) {"
-print body
-print "  return C;"
-print "}"
+print('//===---------- Support/UnicodeCaseFold.cpp -------------------------------===//')
+print('//')
+print('// This file was generated by utils/unicode-case-fold.py from the Unicode')
+print('// case folding database at')
+print('//   ', sys.argv[1])
+print('//')
+print('// To regenerate this file, run:')
+print('//   utils/unicode-case-fold.py \\')
+print('//     "{}" \\'.format(sys.argv[1]))
+print('//     > lib/Support/UnicodeCaseFold.cpp')
+print('//')
+print('//===----------------------------------------------------------------------===//')
+print('')
+print('#include "llvm/Support/Unicode.h"')
+print('')
+print("int llvm::sys::unicode::foldCharSimple(int C) {")
+print(body)
+print("  return C;")
+print("}")
diff --git a/utils/unittest/LLVMBuild.txt b/utils/unittest/LLVMBuild.txt
index c276dd6e720ada591fbf64b4d09156895347aff3..a91a7f0c0c18b9c36e6353f3b3ca223e794237ae 100644
--- a/utils/unittest/LLVMBuild.txt
+++ b/utils/unittest/LLVMBuild.txt
@@ -1,9 +1,8 @@
 ;===- ./utils/unittest/LLVMBuild.txt ---------------------------*- Conf -*--===;
 ;
-;                     The LLVM Compiler Infrastructure
-;
-; This file is distributed under the University of Illinois Open Source
-; License. See LICENSE.TXT for details.
+; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+; See https://llvm.org/LICENSE.txt for license information.
+; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 ;
 ;===------------------------------------------------------------------------===;
 ;
diff --git a/utils/unittest/UnitTestMain/TestMain.cpp b/utils/unittest/UnitTestMain/TestMain.cpp
index 5660b44f41a30fb2393ca55537c7d8874f486cf7..1be819fab72e69b90eca6038cde24afc42a142c4 100644
--- a/utils/unittest/UnitTestMain/TestMain.cpp
+++ b/utils/unittest/UnitTestMain/TestMain.cpp
@@ -1,9 +1,8 @@
 //===--- utils/unittest/UnitTestMain/TestMain.cpp - unittest driver -------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/utils/unittest/googletest/include/gtest/internal/custom/raw-ostream.h b/utils/unittest/googletest/include/gtest/internal/custom/raw-ostream.h
index fd993db1602888cc42b98508542116f5416098a3..a9837c5037ca99fa82be8008aae16d81bdfaa928 100644
--- a/utils/unittest/googletest/include/gtest/internal/custom/raw-ostream.h
+++ b/utils/unittest/googletest/include/gtest/internal/custom/raw-ostream.h
@@ -1,9 +1,8 @@
 //===-- raw-ostream.h - Support for printing using raw_ostream --*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 // This file is not part of gtest, but extends it to support LLVM libraries.
@@ -42,8 +41,9 @@ auto printable(const T &V) -> decltype(StreamSwitch<T>::printable(V)) {
 // If raw_ostream support is enabled, we specialize for types with operator<<
 // that takes a raw_ostream.
 #if !GTEST_NO_LLVM_RAW_OSTREAM
-#include "llvm/Support/raw_ostream.h"
+#include "llvm/ADT/Optional.h"
 #include "llvm/Support/raw_os_ostream.h"
+#include "llvm/Support/raw_ostream.h"
 #include <ostream>
 namespace llvm_gtest {
 
@@ -68,6 +68,18 @@ struct StreamSwitch<T, decltype((void)(std::declval<llvm::raw_ostream &>()
                                        << ConvertibleTo<const T &>()))> {
   static const RawStreamProxy<T> printable(const T &V) { return {V}; }
 };
+
+// llvm::Optional has a template operator<<, which means it will not accept any
+// implicit conversions, so we need to special-case it here.
+template <typename T>
+struct StreamSwitch<llvm::Optional<T>,
+                    decltype((void)(std::declval<llvm::raw_ostream &>()
+                                    << std::declval<llvm::Optional<T>>()))> {
+  static const RawStreamProxy<llvm::Optional<T>>
+  printable(const llvm::Optional<T> &V) {
+    return {V};
+  }
+};
 } // namespace llvm_gtest
 #endif  // !GTEST_NO_LLVM_RAW_OSTREAM
 
diff --git a/utils/update_analyze_test_checks.py b/utils/update_analyze_test_checks.py
index b9175ae73274caade3db289f09f3e14da06475ad..4d1d912aba865565c070568c1735b6665d30a6a7 100755
--- a/utils/update_analyze_test_checks.py
+++ b/utils/update_analyze_test_checks.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2.7
+#!/usr/bin/env python
 
 """A script to generate FileCheck statements for 'opt' analysis tests.
 
@@ -29,6 +29,8 @@ The script is designed to make adding checks to a test case fast, it is *not*
 designed to be authoratitive about what constitutes a good test!
 """
 
+from __future__ import print_function
+
 import argparse
 import itertools
 import os         # Used to advertise this file's name ("autogenerated_note").
@@ -66,12 +68,12 @@ def main():
 
   opt_basename = os.path.basename(args.opt_binary)
   if (opt_basename != "opt"):
-    print >>sys.stderr, 'ERROR: Unexpected opt name: ' + opt_basename
+    print('ERROR: Unexpected opt name: ' + opt_basename, file=sys.stderr)
     sys.exit(1)
 
   for test in args.tests:
     if args.verbose:
-      print >>sys.stderr, 'Scanning for RUN lines in test file: %s' % (test,)
+      print('Scanning for RUN lines in test file: %s' % (test,), file=sys.stderr)
     with open(test) as f:
       input_lines = [l.rstrip() for l in f]
 
@@ -85,20 +87,20 @@ def main():
         run_lines.append(l)
 
     if args.verbose:
-      print >>sys.stderr, 'Found %d RUN lines:' % (len(run_lines),)
+      print('Found %d RUN lines:' % (len(run_lines),), file=sys.stderr)
       for l in run_lines:
-        print >>sys.stderr, '  RUN: ' + l
+        print('  RUN: ' + l, file=sys.stderr)
 
     prefix_list = []
     for l in run_lines:
       (tool_cmd, filecheck_cmd) = tuple([cmd.strip() for cmd in l.split('|', 1)])
 
       if not tool_cmd.startswith(opt_basename + ' '):
-        print >>sys.stderr, 'WARNING: Skipping non-%s RUN line: %s' % (opt_basename, l)
+        print('WARNING: Skipping non-%s RUN line: %s' % (opt_basename, l), file=sys.stderr)
         continue
 
       if not filecheck_cmd.startswith('FileCheck '):
-        print >>sys.stderr, 'WARNING: Skipping non-FileChecked RUN line: ' + l
+        print('WARNING: Skipping non-FileChecked RUN line: ' + l, file=sys.stderr)
         continue
 
       tool_cmd_args = tool_cmd[len(opt_basename):].strip()
@@ -119,8 +121,8 @@ def main():
         func_dict.update({prefix: dict()})
     for prefixes, opt_args in prefix_list:
       if args.verbose:
-        print >>sys.stderr, 'Extracted opt cmd: ' + opt_basename + ' ' + opt_args
-        print >>sys.stderr, 'Extracted FileCheck prefixes: ' + str(prefixes)
+        print('Extracted opt cmd: ' + opt_basename + ' ' + opt_args, file=sys.stderr)
+        print('Extracted FileCheck prefixes: ' + str(prefixes), file=sys.stderr)
 
       raw_tool_outputs = common.invoke_tool(args.opt_binary, opt_args, test)
 
@@ -134,7 +136,7 @@ def main():
     is_in_function_start = False
     prefix_set = set([prefix for prefixes, _ in prefix_list for prefix in prefixes])
     if args.verbose:
-      print >>sys.stderr, 'Rewriting FileCheck prefixes: %s' % (prefix_set,)
+      print('Rewriting FileCheck prefixes: %s' % (prefix_set,), file=sys.stderr)
     output_lines = []
     output_lines.append(autogenerated_note)
 
@@ -181,10 +183,10 @@ def main():
       is_in_function = is_in_function_start = True
 
     if args.verbose:
-      print>>sys.stderr, 'Writing %d lines to %s...' % (len(output_lines), test)
+      print('Writing %d lines to %s...' % (len(output_lines), test), file=sys.stderr)
 
     with open(test, 'wb') as f:
-      f.writelines([l + '\n' for l in output_lines])
+      f.writelines(['{}\n'.format(l).encode('utf-8') for l in output_lines])
 
 
 if __name__ == '__main__':
diff --git a/utils/update_llc_test_checks.py b/utils/update_llc_test_checks.py
index 09b49a763b6022eb4d5945b5579ce19411ceb3b0..5d2aac2509ec17184c5d7f2fed8159afb2c99b5b 100755
--- a/utils/update_llc_test_checks.py
+++ b/utils/update_llc_test_checks.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2.7
+#!/usr/bin/env python
 
 """A test case update script.
 
@@ -7,6 +7,8 @@ FileCheck patterns. It can either update all of the tests in the file or
 a single test function.
 """
 
+from __future__ import print_function
+
 import argparse
 import os         # Used to advertise this file's name ("autogenerated_note").
 import string
@@ -42,7 +44,7 @@ def main():
 
   for test in args.tests:
     if args.verbose:
-      print >>sys.stderr, 'Scanning for RUN lines in test file: %s' % (test,)
+      print('Scanning for RUN lines in test file: %s' % (test,), file=sys.stderr)
     with open(test) as f:
       input_lines = [l.rstrip() for l in f]
 
@@ -63,9 +65,9 @@ def main():
         run_lines.append(l)
 
     if args.verbose:
-      print >>sys.stderr, 'Found %d RUN lines:' % (len(run_lines),)
+      print('Found %d RUN lines:' % (len(run_lines),), file=sys.stderr)
       for l in run_lines:
-        print >>sys.stderr, '  RUN: ' + l
+        print('  RUN: ' + l, file=sys.stderr)
 
     run_list = []
     for l in run_lines:
@@ -81,11 +83,11 @@ def main():
       if len(commands) > 1:
         filecheck_cmd = commands[1]
       if not llc_cmd.startswith('llc '):
-        print >>sys.stderr, 'WARNING: Skipping non-llc RUN line: ' + l
+        print('WARNING: Skipping non-llc RUN line: ' + l, file=sys.stderr)
         continue
 
       if not filecheck_cmd.startswith('FileCheck '):
-        print >>sys.stderr, 'WARNING: Skipping non-FileChecked RUN line: ' + l
+        print('WARNING: Skipping non-FileChecked RUN line: ' + l, file=sys.stderr)
         continue
 
       llc_cmd_args = llc_cmd[len('llc'):].strip()
@@ -107,12 +109,12 @@ def main():
         func_dict.update({prefix: dict()})
     for prefixes, llc_args, triple_in_cmd in run_list:
       if args.verbose:
-        print >>sys.stderr, 'Extracted LLC cmd: llc ' + llc_args
-        print >>sys.stderr, 'Extracted FileCheck prefixes: ' + str(prefixes)
+        print('Extracted LLC cmd: llc ' + llc_args, file=sys.stderr)
+        print('Extracted FileCheck prefixes: ' + str(prefixes), file=sys.stderr)
 
       raw_tool_output = common.invoke_tool(args.llc_binary, llc_args, test)
       if not (triple_in_cmd or triple_in_ir):
-        print >>sys.stderr, "Cannot find a triple. Assume 'x86'"
+        print("Cannot find a triple. Assume 'x86'", file=sys.stderr)
 
       asm.build_function_body_dictionary_for_triple(args, raw_tool_output,
           triple_in_cmd or triple_in_ir or 'x86', prefixes, func_dict)
@@ -122,7 +124,7 @@ def main():
     func_name = None
     prefix_set = set([prefix for p in run_list for prefix in p[0]])
     if args.verbose:
-      print >>sys.stderr, 'Rewriting FileCheck prefixes: %s' % (prefix_set,)
+      print('Rewriting FileCheck prefixes: %s' % (prefix_set,), file=sys.stderr)
     output_lines = []
     output_lines.append(autogenerated_note)
 
@@ -167,10 +169,10 @@ def main():
       is_in_function = is_in_function_start = True
 
     if args.verbose:
-      print>>sys.stderr, 'Writing %d lines to %s...' % (len(output_lines), test)
+      print('Writing %d lines to %s...' % (len(output_lines), test), file=sys.stderr)
 
     with open(test, 'wb') as f:
-      f.writelines([l + '\n' for l in output_lines])
+      f.writelines(['{}\n'.format(l).encode('utf-8') for l in output_lines])
 
 
 if __name__ == '__main__':
diff --git a/utils/update_mca_test_checks.py b/utils/update_mca_test_checks.py
index 54d1cb443c359986add2246e871d6231bc849457..87ac19b39aea181f25a68977911bd6169192fcba 100755
--- a/utils/update_mca_test_checks.py
+++ b/utils/update_mca_test_checks.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2.7
+#!/usr/bin/env python
 
 """A test case update script.
 
@@ -551,7 +551,7 @@ def _write_output(test_path, input_lines, prefix_list, block_infos,  # noqa
         'Writing {} lines to {}...\n\n'.format(len(output_lines), test_path))
 
   with open(test_path, 'wb') as f:
-    f.writelines(['{}\n'.format(l).encode() for l in output_lines])
+    f.writelines(['{}\n'.format(l).encode('utf-8') for l in output_lines])
 
 def main():
   args = _parse_args()
diff --git a/utils/update_test_checks.py b/utils/update_test_checks.py
index 739fe045835a67ecdd7a34b07b2c10caa13205ad..b63a9174208d74082a425490af634ea8bbac8938 100755
--- a/utils/update_test_checks.py
+++ b/utils/update_test_checks.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2.7
+#!/usr/bin/env python
 
 """A script to generate FileCheck statements for 'opt' regression tests.
 
@@ -29,6 +29,8 @@ The script is designed to make adding checks to a test case fast, it is *not*
 designed to be authoratitive about what constitutes a good test!
 """
 
+from __future__ import print_function
+
 import argparse
 import itertools
 import os         # Used to advertise this file's name ("autogenerated_note").
@@ -66,12 +68,12 @@ def main():
 
   opt_basename = os.path.basename(args.opt_binary)
   if (opt_basename != "opt"):
-    print >>sys.stderr, 'ERROR: Unexpected opt name: ' + opt_basename
+    print('ERROR: Unexpected opt name: ' + opt_basename, file=sys.stderr)
     sys.exit(1)
 
   for test in args.tests:
     if args.verbose:
-      print >>sys.stderr, 'Scanning for RUN lines in test file: %s' % (test,)
+      print('Scanning for RUN lines in test file: %s' % (test,), file=sys.stderr)
     with open(test) as f:
       input_lines = [l.rstrip() for l in f]
 
@@ -85,20 +87,20 @@ def main():
         run_lines.append(l)
 
     if args.verbose:
-      print >>sys.stderr, 'Found %d RUN lines:' % (len(run_lines),)
+      print('Found %d RUN lines:' % (len(run_lines),), file=sys.stderr)
       for l in run_lines:
-        print >>sys.stderr, '  RUN: ' + l
+        print('  RUN: ' + l, file=sys.stderr)
 
     prefix_list = []
     for l in run_lines:
       (tool_cmd, filecheck_cmd) = tuple([cmd.strip() for cmd in l.split('|', 1)])
 
       if not tool_cmd.startswith(opt_basename + ' '):
-        print >>sys.stderr, 'WARNING: Skipping non-%s RUN line: %s' % (opt_basename, l)
+        print('WARNING: Skipping non-%s RUN line: %s' % (opt_basename, l), file=sys.stderr)
         continue
 
       if not filecheck_cmd.startswith('FileCheck '):
-        print >>sys.stderr, 'WARNING: Skipping non-FileChecked RUN line: ' + l
+        print('WARNING: Skipping non-FileChecked RUN line: ' + l, file=sys.stderr)
         continue
 
       tool_cmd_args = tool_cmd[len(opt_basename):].strip()
@@ -119,8 +121,8 @@ def main():
         func_dict.update({prefix: dict()})
     for prefixes, opt_args in prefix_list:
       if args.verbose:
-        print >>sys.stderr, 'Extracted opt cmd: ' + opt_basename + ' ' + opt_args
-        print >>sys.stderr, 'Extracted FileCheck prefixes: ' + str(prefixes)
+        print('Extracted opt cmd: ' + opt_basename + ' ' + opt_args, file=sys.stderr)
+        print('Extracted FileCheck prefixes: ' + str(prefixes), file=sys.stderr)
 
       raw_tool_output = common.invoke_tool(args.opt_binary, opt_args, test)
       common.build_function_body_dictionary(
@@ -131,7 +133,7 @@ def main():
     is_in_function_start = False
     prefix_set = set([prefix for prefixes, _ in prefix_list for prefix in prefixes])
     if args.verbose:
-      print >>sys.stderr, 'Rewriting FileCheck prefixes: %s' % (prefix_set,)
+      print('Rewriting FileCheck prefixes: %s' % (prefix_set,), file=sys.stderr)
     output_lines = []
     output_lines.append(autogenerated_note)
 
@@ -178,10 +180,10 @@ def main():
       is_in_function = is_in_function_start = True
 
     if args.verbose:
-      print>>sys.stderr, 'Writing %d lines to %s...' % (len(output_lines), test)
+      print('Writing %d lines to %s...' % (len(output_lines), test), file=sys.stderr)
 
     with open(test, 'wb') as f:
-      f.writelines([l + '\n' for l in output_lines])
+      f.writelines(['{}\n'.format(l).encode('utf-8') for l in output_lines])
 
 
 if __name__ == '__main__':
diff --git a/utils/vim/syntax/llvm.vim b/utils/vim/syntax/llvm.vim
index 9345a9c0901af9bde2d131bbd64e300cb2daf31b..56eb1b198261e248d0e926ff1c9f98f829cccd4d 100644
--- a/utils/vim/syntax/llvm.vim
+++ b/utils/vim/syntax/llvm.vim
@@ -23,7 +23,7 @@ syn match   llvmType /\<i\d\+\>/
 " The true and false tokens can be used for comparison opcodes, but it's
 " much more common for these tokens to be used for boolean constants.
 syn keyword llvmStatement add addrspacecast alloca and arcp ashr atomicrmw
-syn keyword llvmStatement bitcast br catchpad catchswitch catchret call
+syn keyword llvmStatement bitcast br catchpad catchswitch catchret call callbr
 syn keyword llvmStatement cleanuppad cleanupret cmpxchg eq exact extractelement
 syn keyword llvmStatement extractvalue fadd fast fcmp fdiv fence fmul fpext
 syn keyword llvmStatement fptosi fptoui fptrunc free frem fsub fneg getelementptr
diff --git a/utils/wciia.py b/utils/wciia.py
index eaa232f84af956123769219e86af8be0d1f4e67d..4269db2de46674535dec7386a20d5702f4a1b256 100755
--- a/utils/wciia.py
+++ b/utils/wciia.py
@@ -20,6 +20,7 @@ limitations:
 
 """
 
+from __future__ import print_function
 import os
 
 code_owners = {}
@@ -97,7 +98,7 @@ def find_owners(fpath):
 import sys
 
 if len(sys.argv) < 2:
-	print "usage " + sys.argv[0] + " file_or_folder"  
+	print("usage " + sys.argv[0] + " file_or_folder")
 	exit(-1)
 	
 # the path we are checking
@@ -105,13 +106,13 @@ path = str(sys.argv[1])
 
 # check if this is real path
 if not os.path.exists(path):
-	print "path (" + path + ") does not exist"
+	print("path (" + path + ") does not exist")
 	exit(-1)
 	
 owners_name = find_owners(path)
 
 # be grammatically correct
-print "The owner(s) of the (" + path + ") is(are) : " + str(owners_name)
+print("The owner(s) of the (" + path + ") is(are) : " + str(owners_name))
 
 exit(0)
 
@@ -119,7 +120,7 @@ exit(0)
 # not yet used 
 root = "."
 for dir,subdirList,fileList in os.walk( root , topdown=False ) :
-   print "dir :" , dir
+   print("dir :" , dir)
    for fname in fileList :
-      print "-" , fname
-   print
+      print("-" , fname)
+   print()
diff --git a/utils/yaml-bench/YAMLBench.cpp b/utils/yaml-bench/YAMLBench.cpp
index 3688db6f438b7debfe129929b824ee5efbf56c88..a9851c0f63e66a96c57efe43b585a370364bd269 100644
--- a/utils/yaml-bench/YAMLBench.cpp
+++ b/utils/yaml-bench/YAMLBench.cpp
@@ -1,9 +1,8 @@
 //===- YAMLBench - Benchmark the YAMLParser implementation ----------------===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //